Index: /main/tags/3.01/gli/.java.policy
===================================================================
--- /main/tags/3.01/gli/.java.policy	(revision 21112)
+++ /main/tags/3.01/gli/.java.policy	(revision 21112)
@@ -0,0 +1,12 @@
+
+grant {
+        permission java.io.FilePermission "<<ALL FILES>>", "read, write";
+        permission javax.sound.sampled.AudioPermission "record";
+        permission java.util.PropertyPermission "user.dir", "read";
+        permission java.util.PropertyPermission "user.home", "read";
+
+        permission java.lang.RuntimePermission "modifyThread";
+
+	permission java.util.prefs.syncInterval "write";
+	permission java.library.path "read, write";
+}
Index: /main/tags/3.01/gli/CheckJavaVersion.java
===================================================================
--- /main/tags/3.01/gli/CheckJavaVersion.java	(revision 21112)
+++ /main/tags/3.01/gli/CheckJavaVersion.java	(revision 21112)
@@ -0,0 +1,33 @@
+public class CheckJavaVersion
+{
+    static public void main(String[] args)
+    {
+	String java_version = System.getProperty("java.version");
+	System.out.println("Java version: " + java_version);
+
+	try {
+	    // Identify major version number
+	    int major_dot_position = java_version.indexOf(".");
+	    String java_version_major_string = java_version.substring(0, major_dot_position);
+	    int java_version_major = (new Integer(java_version_major_string)).intValue();
+
+	    // Identify minor version number
+	    int minor_dot_position = java_version.indexOf(".", major_dot_position + 1);
+	    String java_version_minor_string = java_version.substring(major_dot_position + 1, minor_dot_position);
+	    int java_version_minor = (new Integer(java_version_minor_string)).intValue();
+
+	    // Version of Java must be 1.4 or higher to run the GLI
+	    if (java_version_major > 1 || (java_version_major == 1 && java_version_minor >= 4)) {
+		// Valid
+		System.exit(2);
+	    }
+	    else {
+		// Invalid
+		System.exit(1);
+	    }
+	}
+	catch (Exception exception) {
+	    System.err.println("Exception: " + exception);
+	}
+    }
+}
Index: /main/tags/3.01/gli/LICENSE.txt
===================================================================
--- /main/tags/3.01/gli/LICENSE.txt	(revision 21112)
+++ /main/tags/3.01/gli/LICENSE.txt	(revision 21112)
@@ -0,0 +1,350 @@
+Copyright (c) 2003 Greenstone Digital Libraries, University of Waikato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, and distribute copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the GPL and the following conditions (where not in conflict with the GPL):
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software, as will all other notices contained in the copyrights directory.
+
+THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDER (Greenstone Digital Libraries, University of Waikato), OR THEIR STAFF BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of Greenstone Digital Libraries, University of Waikato shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization from the Greenstone Digital Libraries, University of Waikato. Distributed under GPL, as follows: 
+
+
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gli/READMEen.txt
===================================================================
--- /main/tags/3.01/gli/READMEen.txt	(revision 21112)
+++ /main/tags/3.01/gli/READMEen.txt	(revision 21112)
@@ -0,0 +1,50 @@
+The Greenstone Librarian Interface
+----------------------------------
+
+This folder contains the Greenstone Librarian Interface (GLI), a tool to
+assist you with building digital libraries using Greenstone. The GLI gives
+you access to Greenstone's functionality from an easy-to-use, 'point and
+click' interface.
+
+To use the GLI, you need a suitable version of the Java Runtime Environment
+(version 1.4.0 or newer). If you don't already have one, you can download
+one from http://java.sun.com/j2se/downloads.html
+
+Note that the GLI is run in conjunction with Greenstone, and assumes that
+it is installed in a subdirectory of your Greenstone installation. If you
+have downloaded one of the Greenstone distributions, or installed from a 
+Greenstone CD-ROM, this will be the case.
+
+
+-- Running the GLI under Windows --
+
+To run the GLI under Windows, browse to the GLI folder in your Greenstone
+installation (using Windows Explorer), and double-click on the gli.bat
+file. This file will check that Greenstone, Perl, and the Java Runtime
+Environment are installed, then start the Greenstone Librarian Interface.
+
+
+-- Running the GLI under Unix --
+
+To run the GLI under Unix, change to the GLI directory in your Greenstone
+installation, then run the gli.sh script. This script will check that
+Greenstone, Perl, and the Java Runtime Environment are installed (make sure
+they are on your search path), then start the Greenstone Librarian
+Interface.
+
+
+Please report any problems you have running or using the Librarian
+Interface to greenstone@cs.waikato.ac.nz.
+
+
+-- Compiling the GLI --
+
+If you have downloaded the Greenstone source distribution, or you have
+obtained the GLI via CVS, you will have the source code (Java) of the
+Librarian Interface. Compiling it requires a suitable version of the Java
+Software Development Kit (version 1.4.0 or newer). You can download one
+from http://java.sun.com/j2se/downloads.html
+
+To compile this source code, run the makegli.bat (Windows) or makegli.sh
+(Unix) files. Once this has been done, you can run the GLI using the
+instructions above.
Index: /main/tags/3.01/gli/READMEes.txt
===================================================================
--- /main/tags/3.01/gli/READMEes.txt	(revision 21112)
+++ /main/tags/3.01/gli/READMEes.txt	(revision 21112)
@@ -0,0 +1,54 @@
+Interfaz de la Biblioteca Digital Greenstone
+--------------------------------------------
+
+Esta carpeta contiene la Interfaz de la Biblioteca Digital Greenstone (GLI
+por sus siglas en inglés), una herramienta que le ayudará a crear y 
+construir bibliotecas digitales por medio de Greenstone. La GLI le da 
+acceso a las funciones de Greenstone desde una interfaz fácil de usar con 
+la que solo tiene que apuntar y hacer click.
+
+Para poder usar la GLI usted necesita una versión adecuada del Ambiente 
+de Ejecución Java (versión 1.4.0 o posterior). Si aún no la tiene, podrá 
+bajarla desde http://java.sun.com/j2se/downloads.html.
+
+Recuerde que la GLI se ejecuta junto con los programas Greenstone, por 
+lo que se asume que estará instalada en un subdirectorio de Greenstone. 
+Si usted ya bajó de la Web una de las versiones de Greenstone o la instaló  
+desde un CD-ROM, entonces éste será el caso.
+
+
+-- Ejecución de la GLI bajo Windows --
+
+Para correr la GLI bajo Windows explore la carpeta GLI que se encuentra 
+en su instalación Greenstone (por medio de Windows Explorer) y haga 
+doble click en el archivo gli.bat. Este archivo verificará que Greenstone, 
+Perl y el Ambiente de Ejecución Java están instalados. A continuación 
+inicie la Interfaz de la Biblioteca Digital Greenstone.
+
+
+-- Ejecución de la GLI bajo UNIX --
+
+Para correr la GLI bajo UNIX cambie al directorio GLI que se encuentra en 
+su instalación Greenstone y ejecute el guión gli.sh. Este guión 
+verificará que Greenstone, Perl y el Ambiente de Ejecución Java están 
+instalados (asegúrese de que están en su ruta de búsqueda). A 
+continuación inicie la Interfaz de la Biblioteca Digital Greenstone.
+
+
+Por favor informe sobre cualquier problema que tenga al correr o usar la 
+Interfaz de la Biblioteca Digital a la siguiente dirección:
+greenstone@cs.waikato.ac.nz.
+
+
+-- Compilación de la GLI --
+
+Si usted ha bajado la versión con el código fuente de Greenstone o ha 
+obtenido la GLI por medio de CVS, entonces tendrá el código fuente 
+(Java) de la Interfaz de la Biblioteca Digital. Para compilarlo se necesita 
+una versión adecuada del Kit de Desarrollo de Software Java (versión 
+1.4.0 o posterior). Usted puede bajarlo desde 
+http://java.sun.com/j2se/downloads.html.
+
+Para compilar este código fuente ejecute los archivos makegli.bat 
+(Windows) o makegli.sh (UNIX). Una vez que haya hecho esto usted 
+podrá correr la GLI usando las instrucciones que aparecen arriba.
Index: /main/tags/3.01/gli/READMEfr.txt
===================================================================
--- /main/tags/3.01/gli/READMEfr.txt	(revision 21112)
+++ /main/tags/3.01/gli/READMEfr.txt	(revision 21112)
@@ -0,0 +1,49 @@
+L'Interface du Bibliothécaire de Greenstone (Greenstone Librarian Interface)
+----------------------------------------------------------------------------
+
+Ce dossier contient l'Interface du Bibliothécaire de Greenstone - Greenstone 
+Librarian Interface (GLI), un outil qui vous assiste pour la création de 
+bibliothèques numérique avec Greenstone. GLI vous donne accès aux fonctions 
+de Greenstone à partir d'une interce 'pointer et cliquer' facile à utiliser.
+
+Pour utiliser GLI, vous avez besoin d'une version adéquate de Java Runtime 
+Environment (version 1.4.0 ou supérieure). Si vous n'en disposez pas, vous 
+pouvez le télécharger de http://java.sun.com/j2se/downloads.html
+
+Notez que GLI est lancé avec Greenstone et suppose qu'il est installé dans 
+un sous-répertoire de votre installation Greenstone. Si vous avez téléchargé 
+une des distributions de Greenstone ou si vous avez installé à partir d'un 
+CD-ROM, ceci sera le cas.
+
+
+-- Exécuter GLI sous Windows --
+
+Pour exécuter GLI sous Windows, parcourez votre système de fichiers jusqu'au 
+répertoire GLI (en utilisant Windows Explorer), et double-cliquez sur le 
+fichier gli.bat. Ce fichier va vérifier que Greenstone, Perl et Java Runtime 
+Environment sont installés, puis démarre Greenstone Librarian Interface.
+
+
+-- Exécuter GLI sous Unix --
+
+Pour exécuter GLI sous Unix, aller au répertoire GLI de votre installation 
+Greenstone et lancer le script gli.sh. Ce script va vérifier que Greenstone, 
+Perl et Java Runtime Environment sont installés (assurez-vous qu'ils sont 
+dans le chemin de recherche), puis démarre Greenstone Librarian Interface.
+
+
+Prière de signaler tous problèmes rencontrés lors de l'exécution ou de 
+l'utilisation de Librarian Interface à greenstone@cs.waikato.ac.nz.
+
+
+-- Compiler GLI --
+
+Si vous avez téléchargé la distribution source de Greenstone, ou si vous 
+avez obtenu Greenstone à travers CVS, alors vous avez le code source (Java) 
+de Librarian Interface. Le compiler requière une version adéquate de Java 
+Software Development Kit (version 1.4.0 ou supérieure). Vous pouvez le 
+télécharger de http://java.sun.com/j2se/downloads.html
+
+Pour compiler ce code source, exécutez makegli.bat (Windows) ou makegli.sh 
+(Unix). Une fois ceci fait, vous pouvez exécuter GLI en suivant les 
+instructions ci-dessus.
Index: /main/tags/3.01/gli/READMEru.txt-cp1251
===================================================================
--- /main/tags/3.01/gli/READMEru.txt-cp1251	(revision 21112)
+++ /main/tags/3.01/gli/READMEru.txt-cp1251	(revision 21112)
@@ -0,0 +1,51 @@
+Èíòåðôåéñ áèáëèîòåêè Greenstone
+-------------------------------
+
+Ýòà ïàïêà ñîäåðæèò èíòåðôåéñ Áèáëèîòåêè Greenstone (GLI) - èíñòðóìåíò,
+êîòîðûé ïîìîæåò Âàì â ïîñòðîåíèè öèôðîâûõ áèáëèîòåê ñ èñïîëüçîâàíèåì
+Greenstone. GLI äàžò Âàì âîçìîæíîñòü ïîëó÷èòü äîñòóï ê ôóíêöèîíàëüíûì
+âîçìîæíîñòÿì Greenstone, óäîáíîìó â ðàáîòå èíòåðôåéñó - óêàæèòå è ùåëêíèòå.
+
+×òîáû èñïîëüçîâàòü GLI, Âàì íåîáõîäèìà ïîäõîäÿùàÿ âåðñèÿ Java Runtime
+Environment (JRE) (âåðñèÿ 1.4.0 èëè áîëåå íîâàÿ). Åñëè ó Âàñ îíà
+îòñóòñòâóåò, Âû ìîæåòå çàãðóçèòü åž ñ ñàéòà
+http://java.sun.com/j2se/downloads.html
+
+Îáðàòèòå âíèìàíèå, ÷òî GLI ñâÿçàíà ñ Greenstone. Ïðåäïîëàãàåòñÿ, ÷òî îíà
+óñòàíîâëåíà â ïîääèðåêòîðèè ïðè èíñòàëëÿöèè Greenstone. Åñëè Âû çàãðóçèëè
+îäèí èç äèñòðèáóòèâîâ Greenstone èëè óñòàíîâèëè åãî ñ CD-ROM, òî ýòî áóäåò
+èìåòü ìåñòî.
+
+
+-- Çàïóñê GLI ïîä Windows --
+
+×òîáû çàïóñòèòü GLI ïîä Windows, ðàññìîòðèòå ïàïêó GLI â èíñòàëëèðîâàííîì ó
+Âàñ Greenstone, èñïîëüçóÿ Windows Explorer, è ùåëêíèòå 2 ðàçà íà ôàéë
+gli.bat. Ýòîò ôàéë ïðîâåðèò, ÷òî Greenstone, Perl è JRE óñòàíîâëåíû, ïîñëå
+÷åãî çàïóñêàåòñÿ èíòåðôåéñ Greenstone.
+
+
+-- Çàïóñê GLI ïîä Unix --
+
+×òîáû çàïóñòèòü GLI ïîä Unix, èçìåíèòå äèðåêòîðèþ GLI â âàøåì
+èíñòàëëèðîâàííîì Greenstone, çàòåì îáðàòèòåñü ê ñêðèïòó gli.sh. Ýòîò ñêðèïò
+ïðîâåðèò, ÷òî Greenstone, Perl è JRE óñòàíîâëåíû (óäîñòîâåðüòåñü, ÷òî îíè
+íàõîäÿòñÿ íà óêàçàííîì âàìè ïóòè ïîèñêà), ïîñëå ÷åãî ñòàðòóåò èíòåðôåéñ
+Greenstone.
+
+
+Ïîæàëóéñòà, ñîîáùèòå î ëþáûõ ïðîáëåìàõ, êîòîðûå ó Âàñ èìåþòñÿ ïðè çàïóñêå
+èëè èñïîëüçîâàíèè èíòåðôåéñà GS, ê greenstone@cs.waikato.ac.nz.
+
+
+-- Êîìïèëèðîâàíèå GLI --
+
+Åñëè Âû çàãðóçèëè èñòî÷íèê äèñòðèáóòèâà Greenstone èëè æå ðàñïîëàãàåòå GLI
+îò CVS, òî Âû ðàñïîëàãàåòå èñõîäíîé ïðîãðàììîé Java èíòåðôåéñà áèáëèîòåêè.
+Êîìïèëÿöèÿ åž òðåáóåò ïîäõîäÿùåé âåðñèè ïðîãðàììíîãî îáåñïå÷åíèÿ Java
+Software Development Kit (âåðñèÿ 1.4.0 èëè áîëåå íîâàÿ). Âû ìîæåòå çàãðóçèòü
+åž ñ ñàéòà http://java.sun.com/j2se/downloads.html.
+
+×òîáû ñîáðàòü ýòó èñõîäíóþ ïðîãðàììó, âîéäèòå â ôàéëû makegli.bat (Windows)
+èëè makegli.sh (Unix). Êàê òîëüêî ýòî áóäåò ñäåëàíî, ìîæåòå çàïóñòèòü GLI,
+èñïîëüçóÿ äàííûå âûøå èíñòðóêöèè.
Index: /main/tags/3.01/gli/READMEru.txt-koi8-r
===================================================================
--- /main/tags/3.01/gli/READMEru.txt-koi8-r	(revision 21112)
+++ /main/tags/3.01/gli/READMEru.txt-koi8-r	(revision 21112)
@@ -0,0 +1,51 @@
+éÎÔÅÒÆÅÊÓ ÂÉÂÌÉÏÔÅËÉ Greenstone
+-------------------------------
+
+üÔÁ ÐÁÐËÁ ÓÏÄÅÒÖÉÔ ÉÎÔÅÒÆÅÊÓ âÉÂÌÉÏÔÅËÉ Greenstone (GLI) - ÉÎÓÔÒÕÍÅÎÔ,
+ËÏÔÏÒÙÊ ÐÏÍÏÖÅÔ ÷ÁÍ × ÐÏÓÔÒÏÅÎÉÉ ÃÉÆÒÏ×ÙÈ ÂÉÂÌÉÏÔÅË Ó ÉÓÐÏÌØÚÏ×ÁÎÉÅÍ
+Greenstone. GLI ÄÁ£Ô ÷ÁÍ ×ÏÚÍÏÖÎÏÓÔØ ÐÏÌÕÞÉÔØ ÄÏÓÔÕÐ Ë ÆÕÎËÃÉÏÎÁÌØÎÙÍ
+×ÏÚÍÏÖÎÏÓÔÑÍ Greenstone, ÕÄÏÂÎÏÍÕ × ÒÁÂÏÔÅ ÉÎÔÅÒÆÅÊÓÕ - ÕËÁÖÉÔÅ É ÝÅÌËÎÉÔÅ.
+
+þÔÏÂÙ ÉÓÐÏÌØÚÏ×ÁÔØ GLI, ÷ÁÍ ÎÅÏÂÈÏÄÉÍÁ ÐÏÄÈÏÄÑÝÁÑ ×ÅÒÓÉÑ Java Runtime
+Environment (JRE) (×ÅÒÓÉÑ 1.4.0 ÉÌÉ ÂÏÌÅÅ ÎÏ×ÁÑ). åÓÌÉ Õ ÷ÁÓ ÏÎÁ
+ÏÔÓÕÔÓÔ×ÕÅÔ, ÷Ù ÍÏÖÅÔÅ ÚÁÇÒÕÚÉÔØ Å£ Ó ÓÁÊÔÁ
+http://java.sun.com/j2se/downloads.html
+
+ïÂÒÁÔÉÔÅ ×ÎÉÍÁÎÉÅ, ÞÔÏ GLI Ó×ÑÚÁÎÁ Ó Greenstone. ðÒÅÄÐÏÌÁÇÁÅÔÓÑ, ÞÔÏ ÏÎÁ
+ÕÓÔÁÎÏ×ÌÅÎÁ × ÐÏÄÄÉÒÅËÔÏÒÉÉ ÐÒÉ ÉÎÓÔÁÌÌÑÃÉÉ Greenstone. åÓÌÉ ÷Ù ÚÁÇÒÕÚÉÌÉ
+ÏÄÉÎ ÉÚ ÄÉÓÔÒÉÂÕÔÉ×Ï× Greenstone ÉÌÉ ÕÓÔÁÎÏ×ÉÌÉ ÅÇÏ Ó CD-ROM, ÔÏ ÜÔÏ ÂÕÄÅÔ
+ÉÍÅÔØ ÍÅÓÔÏ.
+
+
+-- úÁÐÕÓË GLI ÐÏÄ Windows --
+
+þÔÏÂÙ ÚÁÐÕÓÔÉÔØ GLI ÐÏÄ Windows, ÒÁÓÓÍÏÔÒÉÔÅ ÐÁÐËÕ GLI × ÉÎÓÔÁÌÌÉÒÏ×ÁÎÎÏÍ Õ
+÷ÁÓ Greenstone, ÉÓÐÏÌØÚÕÑ Windows Explorer, É ÝÅÌËÎÉÔÅ 2 ÒÁÚÁ ÎÁ ÆÁÊÌ
+gli.bat. üÔÏÔ ÆÁÊÌ ÐÒÏ×ÅÒÉÔ, ÞÔÏ Greenstone, Perl É JRE ÕÓÔÁÎÏ×ÌÅÎÙ, ÐÏÓÌÅ
+ÞÅÇÏ ÚÁÐÕÓËÁÅÔÓÑ ÉÎÔÅÒÆÅÊÓ Greenstone.
+
+
+-- úÁÐÕÓË GLI ÐÏÄ Unix --
+
+þÔÏÂÙ ÚÁÐÕÓÔÉÔØ GLI ÐÏÄ Unix, ÉÚÍÅÎÉÔÅ ÄÉÒÅËÔÏÒÉÀ GLI × ×ÁÛÅÍ
+ÉÎÓÔÁÌÌÉÒÏ×ÁÎÎÏÍ Greenstone, ÚÁÔÅÍ ÏÂÒÁÔÉÔÅÓØ Ë ÓËÒÉÐÔÕ gli.sh. üÔÏÔ ÓËÒÉÐÔ
+ÐÒÏ×ÅÒÉÔ, ÞÔÏ Greenstone, Perl É JRE ÕÓÔÁÎÏ×ÌÅÎÙ (ÕÄÏÓÔÏ×ÅÒØÔÅÓØ, ÞÔÏ ÏÎÉ
+ÎÁÈÏÄÑÔÓÑ ÎÁ ÕËÁÚÁÎÎÏÍ ×ÁÍÉ ÐÕÔÉ ÐÏÉÓËÁ), ÐÏÓÌÅ ÞÅÇÏ ÓÔÁÒÔÕÅÔ ÉÎÔÅÒÆÅÊÓ
+Greenstone.
+
+
+ðÏÖÁÌÕÊÓÔÁ, ÓÏÏÂÝÉÔÅ Ï ÌÀÂÙÈ ÐÒÏÂÌÅÍÁÈ, ËÏÔÏÒÙÅ Õ ÷ÁÓ ÉÍÅÀÔÓÑ ÐÒÉ ÚÁÐÕÓËÅ
+ÉÌÉ ÉÓÐÏÌØÚÏ×ÁÎÉÉ ÉÎÔÅÒÆÅÊÓÁ GS, Ë greenstone@cs.waikato.ac.nz.
+
+
+-- ëÏÍÐÉÌÉÒÏ×ÁÎÉÅ GLI --
+
+åÓÌÉ ÷Ù ÚÁÇÒÕÚÉÌÉ ÉÓÔÏÞÎÉË ÄÉÓÔÒÉÂÕÔÉ×Á Greenstone ÉÌÉ ÖÅ ÒÁÓÐÏÌÁÇÁÅÔÅ GLI
+ÏÔ CVS, ÔÏ ÷Ù ÒÁÓÐÏÌÁÇÁÅÔÅ ÉÓÈÏÄÎÏÊ ÐÒÏÇÒÁÍÍÏÊ Java ÉÎÔÅÒÆÅÊÓÁ ÂÉÂÌÉÏÔÅËÉ.
+ëÏÍÐÉÌÑÃÉÑ Å£ ÔÒÅÂÕÅÔ ÐÏÄÈÏÄÑÝÅÊ ×ÅÒÓÉÉ ÐÒÏÇÒÁÍÍÎÏÇÏ ÏÂÅÓÐÅÞÅÎÉÑ Java
+Software Development Kit (×ÅÒÓÉÑ 1.4.0 ÉÌÉ ÂÏÌÅÅ ÎÏ×ÁÑ). ÷Ù ÍÏÖÅÔÅ ÚÁÇÒÕÚÉÔØ
+Å£ Ó ÓÁÊÔÁ http://java.sun.com/j2se/downloads.html.
+
+þÔÏÂÙ ÓÏÂÒÁÔØ ÜÔÕ ÉÓÈÏÄÎÕÀ ÐÒÏÇÒÁÍÍÕ, ×ÏÊÄÉÔÅ × ÆÁÊÌÙ makegli.bat (Windows)
+ÉÌÉ makegli.sh (Unix). ëÁË ÔÏÌØËÏ ÜÔÏ ÂÕÄÅÔ ÓÄÅÌÁÎÏ, ÍÏÖÅÔÅ ÚÁÐÕÓÔÉÔØ GLI,
+ÉÓÐÏÌØÚÕÑ ÄÁÎÎÙÅ ×ÙÛÅ ÉÎÓÔÒÕËÃÉÉ.
Index: /main/tags/3.01/gli/classes/META-INF/MANIFEST.MF
===================================================================
--- /main/tags/3.01/gli/classes/META-INF/MANIFEST.MF	(revision 21112)
+++ /main/tags/3.01/gli/classes/META-INF/MANIFEST.MF	(revision 21112)
@@ -0,0 +1,4 @@
+Manifest-Version: 1.0
+Created-By: 1.3.0 (Sun Microsystems Inc.)
+Main-Class: org.greenstone.gatherer.Gatherer
+Classpath: .\Gatherer.jar
Index: /main/tags/3.01/gli/classes/dictionary.properties
===================================================================
--- /main/tags/3.01/gli/classes/dictionary.properties	(revision 21112)
+++ /main/tags/3.01/gli/classes/dictionary.properties	(revision 21112)
@@ -0,0 +1,1036 @@
+#*******************
+#
+#***** AboutDialog *****
+AboutDialog.Acknowledgement:Acknowledgements for proprietary programs, packages or classes...
+AboutDialog.Date:(October 2005)
+AboutDialog.Item0:'Xerces Java 1' from The Apache Software Foundation (http://www.apache.org/)
+AboutDialog.Item2:'WGet' from Free Software Foundation, Inc. (http://www.gnu.org/software/wget/wget.html)
+AboutDialog.Item3:'qflib' from Quality First Software GmbH. (http://www.qfs.de/en/qflib/index.html)
+AboutDialog.Item4:'ExtensionFileFilter.class' by Bruno Dumon and contributors (http://pollo.sourceforge.net/)
+AboutDialog.Item5:'How to disable file renaming in file chooser' - vladi21 (http://www.experts-exchange.com/)
+AboutDialog.Item6:'Autoscrolling' - Java Swing by Robert Eckstein, Marc Loy & Dave Wood (http://www.oreilly.com/catalog/jswing/chapter/dnd.beta.pdf)
+AboutDialog.Item7:'Add ghosted drag images to your JTrees' - By Andrew J. Armstrong (http://www.javaworld.com/javaworld/javatips/jw-javatip114.html)
+AboutDialog.Item8:'Stop annoying java.prefs error when you don't have superuser access' - Thanks to Walter Schatz
+AboutDialog.Java_Req:Java is a trademark of Sun Microsystems Inc. The JRE is redistributed as per the Supplemental License Terms, found at http://java.sun.com/j2se/1.4.1/j2re-1_4_1_02-license.html
+AboutDialog.Java_Req_One:This product includes code licensed from RSA Security, Inc.
+AboutDialog.Java_Req_Two:Some portions licensed from IBM are available at http\://oss.software.ibm.com/icu4j/
+AboutDialog.Thanks:Also thanks to those who posted helpful java information on the web...
+AboutDialog.Title:About...
+AboutDialog.Title_One:Greenstone Librarian Interface
+AboutDialog.Title_Two:Greenstone Digital Library, University of Waikato
+AboutDialog.Copyright:Copyright (C) 2003 New Zealand Digital Library Project
+AboutDialog.Copyright_Two:Published under the GPL. See LICENSE.txt for details.
+#***********************
+#
+#***** Audit Table *****
+AuditTable.File:Filename
+#***********************
+#
+#***** Auto Filter *****
+Autofilter.AND:AND
+Autofilter.Ascending:Ascending
+Autofilter.Case_Sensitive:Match Case
+Autofilter.Custom_Filter:Advanced
+Autofilter.Descending:Descending
+Autofilter.Filter_By_Value:Simple
+Autofilter.Name:Column Name
+Autofilter.None:None
+Autofilter.Operator:Condition
+Autofilter.OR:OR
+Autofilter.Order:Ordering
+Autofilter.Order_Tooltip:Sort the matching items in this order
+Autofilter.Remove:Remove Filter
+Autofilter.Remove_Tooltip:Remove this filter from the table column
+Autofilter.Set:Set Filter
+Autofilter.Set_Tooltip:Add this filter to the table column
+Autofilter.Table_Header_Tooltip:Filter by {0}
+Autofilter.Title:Autofilter Metadata
+Autofilter.eqeq:equal to
+Autofilter.!eq:does not equal
+Autofilter.<:is less than
+Autofilter.<eq:is less than or equal to
+Autofilter.>:is greater than
+Autofilter.>eq:is greater than or equal to
+Autofilter.^:starts with
+Autofilter.!^:does not start with
+Autofilter.$:ends with
+Autofilter.!$:does not end with
+Autofilter.?:contains
+Autofilter.!?:does not contain
+#
+#     ********************************************************
+#     *************** Collection Design Module ***************
+#     ********************************************************
+#     * The dictionary  phrases  used in the creation of the *
+#     * Colection  Design  Module. Note that we  still  have *
+#     * access  to the format  arguments (argument  32+) for *
+#     * html formatting.                                     *
+#     ********************************************************
+CDM.ArgumentConfiguration.Add:Add
+CDM.ArgumentConfiguration.Bad_Integer:<html>The argument '{0}' requires an integer value,<br>but no integer can be parsed from {1}.</html>
+CDM.ArgumentConfiguration.Custom_Header:Custom
+CDM.ArgumentConfiguration.Custom:Custom Arguments:
+CDM.ArgumentConfiguration.Error_Title:Error In Arguments
+CDM.ArgumentConfiguration.Header:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Please configure the arguments for {0}.</CENTER></FONT></STRONG></HTML>
+CDM.ArgumentConfiguration.No_Value:The argument '{0}' requires a value, but no value is specified.
+CDM.ArgumentConfiguration.Remove:Remove
+CDM.ArgumentConfiguration.Required_Argument:The argument '{0}' must be given a value.
+CDM.ArgumentConfiguration.Title:Configuring Arguments
+CDM.ClassifierManager.Add:Add Classifier
+CDM.ClassifierManager.Add_Tooltip:Use the specified classifier with this collection
+CDM.ClassifierManager.Assigned:Currently Assigned Classifiers
+CDM.ClassifierManager.Classifier:Select classifier to add:
+CDM.ClassifierManager.Classifier_Tooltip:The classifiers available for use in the collection
+CDM.ClassifierManager.Classifier_Exists:That classifier has already been assigned for this collection.
+CDM.ClassifierManager.Classifier_Str:classifier
+CDM.ClassifierManager.Classifier_XML_Parse_Failed:Unable to determine the arguments for the {0}\nclassifier. Please ensure that the classifier in question\nprovides the -xml flag by running the classinfo.pl script\nsimilar to the description of the pluginfo.pl script in\nsection 2.1 of the Greenstone Developers Guide.
+CDM.ClassifierManager.Configure:Configure Classifier
+CDM.ClassifierManager.Configure_Tooltip:Change the options of the selected classifier
+CDM.ClassifierManager.Controls:Editing Controls
+CDM.ClassifierManager.Instructions:Use this view to add, configure or remove classifiers in your collection. To add a classifier, choose it from the combobox and click 'Add Classifier'.\nTo remove a classifier, choose it from the list of assigned classifiers and click 'Remove Classifier'.\nTo configure a classifier, choose it from the list of assigned classifiers, and click 'Configure Classifier'.\n\nWhen you configure a classifier a scrollable list of its arguments will appear. Enable arguments and enter or select values as necessary.
+CDM.ClassifierManager.Remove:Remove Classifier
+CDM.ClassifierManager.Remove_Tooltip:Remove the selected classifier from the list
+CDM.ClassifierManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Classifier Selection & Configuration</CENTER></FONT></STRONG></HTML>
+CDM.ClassifierManager.Parsing.Message:Design tool is loading classifier information.
+CDM.ClassifierManager.Parsing.Title:Loading Classifiers
+CDM.FormatManager.Add:Add Format
+CDM.FormatManager.Add_Tooltip:Use the specified format command with the collection
+CDM.FormatManager.Assigned_Formats:Currently Assigned Format Commands
+CDM.FormatManager.Editor:HTML Format String
+CDM.FormatManager.Editor_Tooltip:Enter the format command here
+CDM.FormatManager.Editing_Controls:Editing Controls
+CDM.FormatManager.Enabled:Enabled
+CDM.FormatManager.Feature:Choose Feature
+CDM.FormatManager.Feature_Tooltip:The feature to format
+CDM.FormatManager.Insert:Insert
+CDM.FormatManager.Insert_Tooltip:Insert the selected variable into the format command
+CDM.FormatManager.Instructions:The web pages you see when using Greenstone are not pre-stored but are generated 'on the fly' as they are needed. Format commands are used to change the appearance of these generated pages. Some are switches that control the display of documents or parts of documents; others are more complex and require html code as an argument.\nTo add a format command, choose it from the 'feature' list. If a True/False option panel appears, select the state by clicking on the appropriate button. If a panel containing the 'Affected Feature' and 'Edit Format Strings' controls appears, you need to enter html text. You can include preset variables by selecting them from the list.\nOnce you are statisfied with your changes, press 'Add Format' to add it to the configuration file.\nRemove a format command by selecting it from the 'Assigned Format Commands' and clicking 'Remove Format'.
+CDM.FormatManager.Part:Affected Component
+CDM.FormatManager.Part_Tooltip:The aspect of the selected feature to format
+CDM.FormatManager.Remove:Remove Format
+CDM.FormatManager.Remove_Tooltip:Remove the selected format command from the list
+CDM.FormatManager.Replace:Replace Format
+CDM.FormatManager.Replace_Tooltip:Update the selected format command
+CDM.FormatManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Format Commands</CENTER></FONT></STRONG></HTML>
+CDM.FormatManager.Variable:Variables
+CDM.FormatManager.Variable_Tooltip:Variables available for use with the format command
+CDM.General.Access:This collection should be publicly accessible
+CDM.General.Browser_Title:Select An Image
+CDM.General.Collection_Extra:Collection description:
+CDM.General.Collection_Extra_Tooltip:A description of the purpose and content of the collection
+CDM.General.Collection_Name:Collection title:
+CDM.General.Collection_Name_Tooltip:Name of the collection
+CDM.General.Copy_Image:In order to use this image for your collection\nfiles must be copied into the images folder. Proceed?
+CDM.General.Email.Creator:Creator's email
+CDM.General.Email.Creator_Tooltip:E-mail address of the collection's creator
+CDM.General.Email.Maintainer:Maintainer's email
+CDM.General.Email.Maintainer_Tooltip:E-mail address of the collection's maintainer
+CDM.General.Icon_Collection:URL to 'about page' icon:
+CDM.General.Icon_Collection_Tooltip:The URL of the image to display on the collection's about page
+CDM.General.Icon_Collection_Small:URL to 'home page' icon:
+CDM.General.Icon_Collection_Small_Tooltip:The URL of the image to display on the library's home page 
+CDM.General.Image_Copy_Failed:An error has occurred which prevents the image you selected being copied.\nPlease try copying the image to the images folder of your collection manually.
+CDM.General.Image_Filter:Images (gifs, jpgs and pngs)
+CDM.General.Instructions:The design section of the Librarian Interface allows you to control many aspects of your collection's appearance. The design is split up into several sections. This section contains general options and settings. To choose a different section, click on its name in the list to the left.
+CDM.General.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>General Options</CENTER></FONT></STRONG></HTML>
+CDM.GUI.Classifiers:Browsing Classifiers
+CDM.GUI.Design_Topics:Design Sections
+CDM.GUI.Formats:Format Features
+CDM.GUI.General:General
+CDM.GUI.Indexes:Search Indexes
+CDM.GUI.MetadataSets:Metadata Sets
+CDM.GUI.Plugins:Document Plugins
+CDM.GUI.Root:CDM_ROOT
+CDM.GUI.SearchTypes:Search Types
+CDM.GUI.Subcollections:Partition Indexes
+CDM.GUI.SuperCollection:Cross-Collection Search
+CDM.GUI.Translation:Translate Text
+CDM.IndexManager.Add_Index:Add Index
+CDM.IndexManager.Add_Index_Tooltip:Build the specified index for this collection
+CDM.IndexManager.Default_Index_Indicator:[Default Index]
+CDM.IndexManager.Document:document
+CDM.IndexManager.Index_Exists:That index has already been assigned for this collection.
+CDM.IndexManager.Index_Name:Index Name:
+CDM.IndexManager.Index_Name_Tooltip:A unique name for the new index
+CDM.IndexManager.Indexes:Assigned Indexes
+CDM.IndexManager.Instructions:Here you choose what searchable indexes the collection will have.\nTo add a new index, select what material is to be indexed, choose the level of the index, and click 'Add Index'.\nTo remove an index, select it from the assigned indexes list and click 'Remove Index'.\nTo set the default index, select it from the assigned indexes list and click 'Set Default'.
+CDM.IndexManager.Level:At the level:
+CDM.IndexManager.Level_Exists:That level has already been assigned for this collection.
+CDM.IndexManager.Level_Tooltip:The document level to build the index on
+CDM.IndexManager.MGPP.Add_All_Metadata:Add All
+CDM.IndexManager.MGPP.Add_All_Metadata_Tooltip:Build a separate index for each available source, using the source as the index name
+CDM.IndexManager.MGPP.Add_Level:Add Level
+CDM.IndexManager.MGPP.Add_Level_Tooltip:Build an index on the specified level
+CDM.IndexManager.MGPP.Current_Indexes:Currently Assigned Indexes
+CDM.IndexManager.MGPP.Current_Levels:Currently Assigned Levels
+CDM.IndexManager.MGPP.Index:Index Source:
+CDM.IndexManager.MGPP.Index_Tooltip:Metadata element to build index on
+CDM.IndexManager.MGPP.Indexes:Manage Indexes
+CDM.IndexManager.MGPP.Instructions:Here you choose what searchable indexes the collection will have.\nTo add a new index, enter a unique name for the index, select material/metadata is to be indexed, and click 'Add Index'.\nIf you wish to add all of the available sources so as to have indexes built on them, then click 'Add All' (the name for each index will default to be the name of the source). Note that this is subtly different from adding the 'AllFields' source, as the ordering of the indexes is important.\nIf you wish to edit a particular index, select it from the currently assigned indexes list, edit as necessary, and then click 'Replace Index', to replace the old version.\nTo remove an index, select it from the currently assigned indexes list and click 'Remove Index'.
+CDM.IndexManager.MGPP.Level:Level Source:
+CDM.IndexManager.MGPP.Level_Name:Level Name:
+CDM.IndexManager.MGPP.Level_Name_Tooltip:A unique name for the new level index
+CDM.IndexManager.MGPP.Levels:Manage Levels
+CDM.IndexManager.MGPP.Remove_Level:Remove Level
+CDM.IndexManager.MGPP.Remove_Level_Tooltip:Remove the selected level index
+CDM.IndexManager.MGPP.Replace_Index:Replace Index
+CDM.IndexManager.MGPP.Replace_Index_Tooltip:Update the selected index
+CDM.IndexManager.MGPP.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Index And Level Selection</CENTER></FONT></STRONG></HTML>
+CDM.IndexManager.Paragraph:paragraph
+CDM.IndexManager.Remove_Index:Remove Index
+CDM.IndexManager.Remove_Index_Tooltip:Remove the selected index from the list
+CDM.IndexManager.Section:section
+CDM.IndexManager.Set_Default:Set Default Index
+CDM.IndexManager.Set_Default_Tooltip:Make the selected index the default index
+CDM.IndexManager.Source:Build index on:
+CDM.IndexManager.Source_Tooltip:The metadata elements to build the index on
+CDM.IndexManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Index Selection</CENTER></FONT></STRONG></HTML>
+CDM.LanguageManager.Add:Add Language
+CDM.LanguageManager.Add_Tooltip:Add the specified language to the list
+CDM.LanguageManager.Assigned_Languages:Assigned Languages
+CDM.LanguageManager.Default_Language:Default Language
+CDM.LanguageManager.Remove:Remove Language
+CDM.LanguageManager.Remove_Tooltip:Remove the selected language from the list
+CDM.LanguageManager.Selector:Language to add: 
+CDM.LanguageManager.Selector_Tooltip:The language to build a partition index on
+CDM.LanguageManager.Set_Default:Set Default
+CDM.LanguageManager.Set_Default_Tooltip:Make the selected language the default
+CDM.MetadataSetManager.Add:Add Metadata Set
+CDM.MetadataSetManager.Add_Tooltip:Add a new metadata set to the collection.
+CDM.MetadataSetManager.Edit:Edit Metadata Set
+CDM.MetadataSetManager.Edit_Message:Metadata sets can be edited using GEMS (Greenstone Editor for Metadata Sets).\nStart GEMS by running gems.bat (Windows) or gems.sh (Linux) from the gli folder of your Greenstone installation.
+CDM.MetadataSetManager.Edit_Message_Title:Editing Metadata Sets
+CDM.MetadataSetManager.Edit_Tooltip:Edit the selected metadata set.
+CDM.MetadataSetManager.Elements:Elements in selected Metadata Set
+CDM.MetadataSetManager.Instructions:In this design view you can add, configure and remove the Metadata Sets in your collection and what Elements they contain.
+CDM.MetadataSetManager.Remove:Remove Metadata Set
+CDM.MetadataSetManager.Remove_Tooltip:Remove the selected metadata set from the collection.
+CDM.MetadataSetManager.Sets:Available Metadata Sets
+CDM.MetadataSetManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Metadata Sets</CENTER></FONT></STRONG></HTML>
+CDM.Move.At_Bottom:Cannot move the {0} {1} lower as it is already at the bottom of the list.
+CDM.Move.At_Top:Cannot move the {0} {1} higher as it is already at the top of the list.
+CDM.Move.Cannot:Cannot move the plugin {0} any lower as RecPlug and ArcPlug must be at the bottom of the list.
+CDM.Move.Fixed:<html>RecPlug and ArcPlug are generally required to be last and so are fixed in place.<br>If you really must move them please hand edit the collect.cfg file.</html>
+CDM.Move.Move_Down:Move Down
+CDM.Move.Move_Down_Tooltip:Move the selected item down the list
+CDM.Move.Move_Up:Move Up
+CDM.Move.Move_Up_Tooltip:Move the selected item up the list
+CDM.Move.Title:Error - Cannot Move
+CDM.PlugInManager.Add:Add Plugin
+CDM.PlugInManager.Add_Tooltip:Use the specified plugin with this collection
+CDM.PlugInManager.Assigned:Currently Assigned Plugins
+CDM.PlugInManager.Configure:Configure Plugin
+CDM.PlugInManager.Configure_Tooltip:Change the options of the selected plugin
+CDM.PlugInManager.Controls:Editing Controls
+CDM.PlugInManager.Ignore:Don't Add Plugin
+CDM.PlugInManager.Ignore_Tooltip:Don't add any plugins to this collection.
+CDM.PlugInManager.Instructions:Use this view to add, configure or remove plugins from your collection. To add one choose it from the combobox and click 'Add Plugin'.\nTo configure or remove one, select it from the list of assigned plugins then:\ni)   Change its position in the plugin order by clicking on the arrow buttons. (Note: The position of RecPlug and ArcPlug are fixed).\nii)  Configure it by clicking 'Configure Plugin',\niii) Remove it by clicking 'Remove Plugin'.\n\nPlugins are configured using a pop-up design area with a scrollable list of arguments. Enable arguments and enter or select values as necessary.
+CDM.PlugInManager.PlugIn:Select plugin to add:
+CDM.PlugInManager.PlugIn_Tooltip:The plugins available for use in the collection
+CDM.PlugInManager.PlugIn_Exists:That plugin has already been assigned for this collection.
+CDM.PlugInManager.PlugIn_Str:plugin
+CDM.PlugInManager.PlugIn_XML_Parse_Failed:Unable to determine the arguments for the {0}\nplugin. Please ensure that the plugin in question provides\nthe -xml flag by running the pluginfo.pl script described\nin section 2.1 of the Greenstone Developers Guide.
+CDM.PluginManager.Plugin_Suggestion_Prompt:None of the plugins in this collection are expected to process the file "{0}". The following plugins can be used to process this file. You can add one to your collection now by selecting it and pressing the Add button. (Note that you can change the list of plugins at any time by going to "Document Plugins" on the Design Pane.)
+CDM.PlugInManager.Remove:Remove Plugin
+CDM.PlugInManager.Remove_Tooltip:Remove the selected plugin from the list
+CDM.PluginManager.SuggestedPluginListTitle:Suggested plugins for file
+CDM.PlugInManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Plugin Selection & Configuration</CENTER></FONT></STRONG></HTML>
+CDM.PlugInManager.Parsing.Message:Design tool is loading plugin information.
+CDM.PlugInManager.Parsing.Title:Loading Plugins
+CDM.SearchTypeManager.Add:Add Search Type
+CDM.SearchTypeManager.Add_Tooltip:Allow the specified type of searching with this collection
+CDM.SearchTypeManager.Assigned:Currently Assigned Search Types
+CDM.SearchTypeManager.BuildType_Selection:Indexer:
+CDM.SearchTypeManager.BuildType_Selection_Tooltip:Select the indexer to be used for this collection.
+CDM.SearchTypeManager.Enable:Enable Advanced Searches
+CDM.SearchTypeManager.Instructions:Defining the search type is an advanced feature only available when enabled (by checking the 'Enable Advanced Searches' box). Once enabled further controls for selecting and changing the order of search types become available. Select a search type from the combobox and 'Add Search Type', if enabled, to add the search type. Select an existing search type from the list, and use 'Move Up' and 'Move Down' to change the order, or 'Remove Search Type' to remove it.\n\nThe search types specify what kind of search interface should be provided: form, for fields searching, and/or plain for regular searching. The ordering is important as the first one will be used for the initial search page by default.\n\nIt is also important to note that enabling search types causes the appearance of the Indexes view to change as well. Fielded searching is a newer feature of Greenstone, and thus requires different index information. When search types is enabled all current indexes are modified to be valid indexes under the new schema, while disabling search types will cause GLI to do its best to convert new indexes back into the old style but it may not always be successful, in which case the indexes will be removed. In either case a copy of the original index command line will be available as a comment in the configuration file. 
+CDM.SearchTypeManager.Remove:Remove Search Type
+CDM.SearchTypeManager.Remove_Tooltip:Do not allow the selected type of searching with this collection
+CDM.SearchTypeManager.SearchType:Search Type
+CDM.SearchTypeManager.SearchType_Selection:Search Types:
+CDM.SearchTypeManager.SearchType_Selection_Tooltip:The search types available for use with this collection
+CDM.SearchTypeManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Search Type Selection & Ordering</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionManager.Add:Add Filter
+CDM.SubcollectionManager.Add_Tooltip:Add the specified filter to the collection
+CDM.SubcollectionManager.Assigned:Defined Subcollection Filters
+CDM.SubcollectionManager.Exclude:Exclude
+CDM.SubcollectionManager.Flags:Flags to set when matching
+CDM.SubcollectionManager.Flags_Tooltip:Regular expression flags to use when matching (eg. "i" for case-insensitive matching)
+CDM.SubcollectionManager.Include:Include
+CDM.SubcollectionManager.Inclusive:What do we do with files that match?
+CDM.SubcollectionManager.Instructions:Use this panel to refine index creation.\ni)  Define subcollection filters, which screen the collection documents based on the specified metadata values. Once defined, you can then generate partitions on any previously specified index, based on one or more of these filters.\nii) Generate index partitions based on language. This filters the collection documents so that only those in the chosen language are included.\nNote that the total number of partitions generated is a combination of all indexes, subcollection filters and languages chosen. Two indexes with two subcollection filters in two languages would yield eight index partitions.
+CDM.SubcollectionManager.Language_Controls:Assign Languages
+CDM.SubcollectionManager.Match:Regular expression to match with: 
+CDM.SubcollectionManager.Match_Tooltip:A regular expression defining the documents in the subcollection
+CDM.SubcollectionManager.Name:Subcollection filter name:
+CDM.SubcollectionManager.Name_Tooltip:The name of the new subcollection filter
+CDM.SubcollectionManager.Partitions_Disabled:This facility is currently disabled because you are operating in {0-Current Mode} mode. It can only be accessed in {1-Systems} and {2-Expert} modes. Use File->Preferences to alter the mode of operation.
+CDM.SubcollectionManager.Remove:Remove Filter
+CDM.SubcollectionManager.Remove_Tooltip:Remove the selected filter from the list
+CDM.SubcollectionManager.Replace:Replace Filter
+CDM.SubcollectionManager.Replace_Tooltip:Update the details of the selected filter
+CDM.SubcollectionManager.Source:Document attribute to match against:
+CDM.SubcollectionManager.Source_Tooltip:The document attribute to filter on
+CDM.SubcollectionManager.Subcollection_Controls:Define Filters
+CDM.SubcollectionManager.Subindex_Controls:Assign Partitions
+CDM.SubcollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Partition Indexes</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionIndexManager.Add_Subindex:Add Partition
+CDM.SubcollectionIndexManager.Add_Subindex_Tooltip:Add the specified partition to this collection
+CDM.SubcollectionIndexManager.Default_Partition_Indicator:[Default Partition]
+CDM.SubcollectionIndexManager.PartitionName:Partition Name:
+CDM.SubcollectionIndexManager.PartitionName_Tooltip:The name of the new partition
+CDM.SubcollectionIndexManager.Remove_Subindex:Remove Partition
+CDM.SubcollectionIndexManager.Remove_Subindex_Tooltip:Remove the selected partition from this collection
+CDM.SubcollectionIndexManager.Replace_Subindex:Replace Partition
+CDM.SubcollectionIndexManager.Replace_Subindex_Tooltip:Update the selected partition
+CDM.SubcollectionIndexManager.Set_Default_Subindex:Set Default
+CDM.SubcollectionIndexManager.Set_Default_Subindex_Tooltip:Make the selected partition the default
+CDM.SubcollectionIndexManager.Source:Build partition on:
+CDM.SubcollectionIndexManager.Source_Tooltip:The subcollection filters to build the partition on
+CDM.SubcollectionIndexManager.Subindexes:Assigned Subcollection Partitions
+CDM.SuperCollectionManager.Instructions:Cross-collection searching is where a single search is performed over several collections, as if all the collections were one. Specify the collections to include in a search by clicking on the appropriate collection's name in the list below. The current collection will automatically be included. \nWARNING: If the individual collections do not have the same indexes (including subcollection partitions and language partitions) as each other, cross-collection searching will not work properly. The user will only be able to search using indexes common to all collections.
+CDM.SuperCollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Cross-Collection Search Configuration</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Add:Add Translation
+CDM.TranslationManager.Add_Tooltip:Add the specified translation to the collection
+CDM.TranslationManager.Affected_Features:Features
+CDM.TranslationManager.Assigned_Fragments:Assigned Translations
+CDM.TranslationManager.Default_Text:Initial Text Fragment
+CDM.TranslationManager.Fragment_Column:Text Fragment
+CDM.TranslationManager.Instructions:Use this view to review and assign translations of text fragments in your collection. First, select an entry from the list of features (e.g. collection level metadata and classifier names). Next, select and edit a text fragment, or select a language from the combobox in order to add a new fragment. Type the text into the white text area, referring to the initial text fragment as necessary. Any changes will appear immediately in the collection.
+CDM.TranslationManager.Language:Language of translation:
+CDM.TranslationManager.Language_Column:Language
+CDM.TranslationManager.Language_Tooltip:The language being translated into
+CDM.TranslationManager.Remove:Remove Translation
+CDM.TranslationManager.Remove_Tooltip:Remove the selected translation from the collection
+CDM.TranslationManager.Replace:Replace Translation
+CDM.TranslationManager.Replace_Tooltip:Update the selected translation
+CDM.TranslationManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Text Translation</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Translation:Translated Text
+CDM.TranslationManager.Translation_Tooltip:Enter the translation of the string here
+#*******************
+#
+#***** Collection *****
+Collection.Collection:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Collection</FONT></STRONG></HTML>
+Collection.Delete_Tooltip:Delete selected files and folders
+Collection.Filter_Tooltip:Restrict the files shown in the file tree
+Collection.New_Folder_Tooltip:Create a new folder
+Collection.No_Collection:No Collection
+Collection.Stop:Stop
+Collection.Stop_Tooltip:Stop copying files
+Collection.Workspace:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Workspace</FONT></STRONG></HTML>
+#**********************
+#
+#***** CollectionBuilt *****
+CollectionBuilt.Message:The collection has been built and is ready for previewing.
+CollectionBuilt.Title:Collection Creation Result
+#***************************
+#
+#***** CollectionManager *****
+CollectionManager.Build_Cancelled:Building the collection has been cancelled.
+CollectionManager.Cannot_Create_Collection:An error has occurred and the collection could not be created.
+CollectionManager.Cannot_Create_Collection_With_Reason:The collection could not be created because:\n{0-Error message}
+CollectionManager.Cannot_Delete_Index:This collection cannot be rebuilt because the old index files cannot be deleted.\nPlease make sure that Greenstone or some other program is not using these files,\n and try again.
+CollectionManager.Cannot_Delete_Index_Log:Old index files unable to be deleted so importing stopped.
+CollectionManager.Cannot_Open:The collection at:\n{0-Collection file path}\ncannot be opened.
+CollectionManager.Cannot_Open_With_Reason:The collection at:\n{0-Collection file path}\ncannot be opened because:\n{1-Error message}
+CollectionManager.Creating_New:Creating a new collection.
+CollectionManager.Log_Created:The log folder created.
+CollectionManager.Loading_Collection:Loading collection
+CollectionManager.Loading_Collection_Please_Wait:Loading collection, please wait...
+CollectionManager.Loading_Successful:Loaded collection {0}.
+CollectionManager.Missing_Config:This collection does not have a valid collect.cfg file 
+CollectionManager.Not_Col_File:'{0}' is not a Librarian Interface Collection File (.col)
+CollectionManager.No_Config_File:The collect.cfg file has not been created by mkcol.pl.
+CollectionManager.Preview_Ready_Failed:An error has occurred which will prevent the collection being previewed.
+CollectionManager.Preview_Ready_Title:Collection Preview State.
+CollectionManager.Session_Ready:Collection {0} ready for use.
+CollectionManager.Build_Not_Moved:The building directory could not be moved.
+CollectionManager.Index_Not_Deleted:The index directory could not be deleted.
+CollectionManager.Install_Exception:Exception detected during collection install:\n{0}\nMost likely caused by Windows or Local Library holding locks on files.\nPlease make sure that none of the collection source files\nare opened in external applications, then rebuild.
+#*****************************
+#
+#***** CollectionPopupMenu *****
+CollectionPopupMenu.Delete:Delete
+CollectionPopupMenu.New_Dummy_Doc:New dummy document
+CollectionPopupMenu.New_Folder:New folder
+#*******************************
+#
+#***** CreatePane *****
+CreatePane.Build:Build Options
+CreatePane.Build_Tooltip:Options for the indexing and classification stage of collection formation
+CreatePane.Build_Collection:Build Collection
+CreatePane.Build_Collection_Tooltip:Start the collection formation process
+CreatePane.Build_Progress:Build Progress
+CreatePane.Cancel_Build:Cancel Build
+CreatePane.Cancel_Build_Tooltip:Stop the collection formation process
+CreatePane.Full_Build:Complete Rebuild
+CreatePane.Full_Build_Tooltip:Completely rebuild the collection from scratch
+CreatePane.Import:Import Options
+CreatePane.Import_Tooltip:Options for the file conversion and metadata assignment stage of collection formation
+CreatePane.Import_Progress:Import Progress
+CreatePane.Incremental_Build:Incremental Rebuild
+CreatePane.Incremental_Build_Tooltip:Faster, as only the parts of the collection building process that are necessary are performed
+CreatePane.Log:Message Log
+CreatePane.Log_Tooltip:View output from previous collection formation attempts
+CreatePane.Mode_All:all
+CreatePane.Options:Option Groups
+CreatePane.Options_Title:Collection Import & Build Options
+CreatePane.Preview_Collection:Preview Collection
+CreatePane.Preview_Collection_Tooltip:Preview the built collection
+#**********************
+#
+#***** Dates *****
+Dates.Mon:Monday
+Dates.Tue:Tuesday
+Dates.Wed:Wednesday
+Dates.Thu:Thursday
+Dates.Fri:Friday
+Dates.Sat:Saturday
+Dates.Sun:Sunday
+Dates.Jan:January
+Dates.Feb:February
+Dates.Mar:March
+Dates.Apr:April
+Dates.May:May
+Dates.Jun:June
+Dates.Jul:July
+Dates.Aug:August
+Dates.Sep:September
+Dates.Oct:October
+Dates.Nov:November
+Dates.Dec:December
+#**********************
+#
+#***** Delete Collection Prompt *****
+DeleteCollectionPrompt.Collection_Details:Selected collection's details
+DeleteCollectionPrompt.Collection_List:Available collections
+DeleteCollectionPrompt.Confirm_Delete:Please tick the box to confirm collection deletion.
+DeleteCollectionPrompt.Delete:Delete
+DeleteCollectionPrompt.Delete_Tooltip:Delete the selected collection
+DeleteCollectionPrompt.Details:Creator - {0}\nMaintainer - {1}\nDescription\n{2}
+DeleteCollectionPrompt.Failed_Delete:Collection could not be completely deleted.
+DeleteCollectionPrompt.Failed_Title:Deletion Failed!
+DeleteCollectionPrompt.No_Collection:No collection selected.
+DeleteCollectionPrompt.Successful_Delete:Collection has been deleted.
+DeleteCollectionPrompt.Successful_Title:Deletion Complete!
+DeleteCollectionPrompt.Title:Collection Deletion
+#**********************
+#
+#***** DirectoryLevelMetadata *****
+DirectoryLevelMetadata.Message:You are about to assign 'folder level' metadata to the selected folder (or folders). Such metadata will be automatically inherited by all files and folders in this folder. Furthermore this metadata cannot be removed from specific children, only replaced. To continue with this action click 'OK'.
+DirectoryLevelMetadata.Title:About to add folder level metadata
+#********************
+#
+#***** EnrichPane *****
+EnrichPane.AutoMessage:This element, {0}, belongs to the Greenstone metadata set. This set represents metadata that can usually be extracted automatically. To check what {0} metadata can be extracted create the collection using the 'Create' tab.
+EnrichPane.ExistingValues:Existing values for {0}
+EnrichPane.Expand:...
+EnrichPane.Expand_Tooltip:Edit the value in a larger text box
+EnrichPane.InheritedMetadataSelected:This piece of metadata is inherited from a folder above it, and cannot be edited. You can visit the actual folder containing the metadata by clicking on the folder icon at the start of the row.
+EnrichPane.No_File:No File Selected
+EnrichPane.No_Metadata:No Metadata Available
+EnrichPane.No_Metadata_Element:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>No Metadata Element Selected</FONT></STRONG></HTML>
+EnrichPane.Value_Field_Tooltip:Value to assign to the selected metadata element
+EnrichPane.Value_Field_Tooltip_Uneditable:Value assigned to the selected metadata element
+#***********************
+#
+#****** ExplodeMetadataPrompt *******
+ExplodeMetadataPrompt.Explode:Explode
+ExplodeMetadataPrompt.Explode_Tooltip:Explode the metadata file 
+ExplodeMetadataPrompt.Instructions:Exploding this metadata file will produce individual records with editable metadata. This is an irreversible process and the original metadata file will be removed from the collection.
+ExplodeMetadataPrompt.Failed_Title:Exploding Failed.
+ExplodeMetadataPrompt.Failed_Explode:The database {0} could not be exploded.
+ExplodeMetadataPrompt.NotExplodable:This file is not explodable.
+ExplodeMetadataPrompt.Successful_Explode:Database {0} successfully exploded.
+ExplodeMetadataPrompt.Successful_Title:Exploding Complete.
+ExplodeMetadataPrompt.Title:Explode Metadata Database
+#***********************
+#
+#****** WriteCDImagePrompt *******
+WriteCDImagePrompt.CD_Name:CD/DVD name
+WriteCDImagePrompt.Export:Write CD/DVD image
+WriteCDImagePrompt.Export_Tooltip:Write the selected collections as a Windows CD/DVD Image
+WriteCDImagePrompt.Failed_Export:The collections ({0}) could not be exported.
+WriteCDImagePrompt.Failed_Title:Export Failed
+WriteCDImagePrompt.Instructions:Export one or more collections to a self-installing Windows CD/DVD (although this works on any platform, the CD/DVD that is created only runs under Windows).
+WriteCDImagePrompt.Progress_Label:Copying Files. This could take some time...
+WriteCDImagePrompt.Size_Label:Estimated size: 
+WriteCDImagePrompt.Successful_Export:The collections ({0}) have been exported.
+WriteCDImagePrompt.Successful_Title:Export Complete
+WriteCDImagePrompt.Title:Export Collection to CD/DVD
+#************************
+#
+#****** ExportAsPrompt *******
+ExportAsPrompt.Export_Name:Export directory name
+ExportAsPrompt.Export:Export collection(s)
+ExportAsPrompt.Export_Tooltip:Export the selected collections in the chosen file format into the designated directory
+ExportAsPrompt.Failed_ExportOne:The collection, {0}, could not be exported.
+ExportAsPrompt.Failed_ExportMany:The collections -- {0} -- could not be exported.
+ExportAsPrompt.Failed_Details:See {1} for details
+ExportAsPrompt.Failed_Title:Export Failed
+ExportAsPrompt.Instructions:Export one or more collections in the chosen format to into the stipulated directory
+ExportAsPrompt.Progress_Label:Exporting Files. This could take some time...
+ExportAsPrompt.Successful_ExportOne:The collection, {0}, has been exported.
+ExportAsPrompt.Successful_ExportMany:The collections -- {0} -- have been exported.
+ExportAsPrompt.Successful_Details:Files exported to: {1}
+ExportAsPrompt.Successful_Title:Export Complete
+ExportAsPrompt.Title:Export Collections
+ExportAsPrompt.SaveAs:Export to
+ExportAsPrompt.SaveAs_Tooltip:Select the format you want to export to
+#************************
+#
+#*****ExternalCollectionPrompt********
+ExternalCollectionPrompt.NewMDS:Create a new, empty metadata set.
+ExternalCollectionPrompt.ExistingMDS:Choose one or more existing metadata sets
+ExternalCollectionPrompt.Title:Load external collection
+ExternalCollectionPrompt.Instructions_1:You are about to load a collection that has not been created by the GLI. You will be asked to select one or more metadata sets to use for this collection. Existing metadata will be imported into the selected set(s). Original metadata files will be saved in a folder called import.bak.
+ExternalCollectionPrompt.Instructions_2:You are about to load a collection that has not been created by the GLI. You need to add a metadata set to this collection. You can create a new empty metadata set to which all the metadata elements found in the collection will be automatically added. Alternatively, you can select existing metadata sets, and elements from the collection will be merged with or added to the elements already in those sets. Original metadata files will be saved in a folder called import.bak.
+#****************************************
+#
+#***** Inherited Metadata *****
+ExtractedMetadata.Message:The selected file or folder contains extracted metadata, but it is currently hidden. To view extracted metadata go to the "[F]ile" menu and select "[P]references". In the "General" properties sheet select "View Extracted Metadata", then click "[O]k".
+ExtractedMetadata.Title:This File/Folder Has Hidden Extracted Metadata
+#
+#***** FileActions *****
+FileActions.Calculating_Size:Calculating total size of selected files
+FileActions.Copying:Copying {0}
+FileActions.Deleting:Deleting {0}
+FileActions.Directories_Selected:{0} folders selected
+FileActions.Directory_Exists:<html>The folder<br>{0}<br>already exists within the collection. Files will be copied into it.</html>
+FileActions.Directory_Selected:1 folder selected
+FileActions.File_Already_Exists_No_Create:The file '{0}' already exists. 
+FileActions.File_Already_Exists_Message:{0}\nA file with the name shown above already exists at the\ndestination, and despite your previous choice the Librarian Interface\nis unable to overwrite it. Please check file permissions.
+FileActions.File_Already_Exists_Title:Error - File Already Exists.
+FileActions.File_And_Directories_Selected:1 file and {0} folders selected
+FileActions.File_And_Directory_Selected:1 file and 1 folder selected
+FileActions.File_Create_Error:The file {0} could not be created.
+FileActions.Folder_Create_Error:The folder {0} could not be created.
+FileActions.File_Exists:<html>{0} already exists in the destination folder. Overwrite it?</html>
+FileActions.File_Not_Deleted_Message:{0}\nThe file named above can not be deleted. \nPlease check that you don't have the file open\nin an external program, then try again. 
+FileActions.File_Not_Deleted_Title:Error - File Not Deleted.
+FileActions.File_Not_Found_Message:{0}\nThe file named above can not be found. Once file view\nhas refreshed, please check the file still exists.
+FileActions.File_Not_Found_Title:Error - File Not Found.
+FileActions.File_Selected:1 file selected
+FileActions.Files_And_Directory_Selected:{0} files and 1 folder selected
+FileActions.Files_Selected:{0} files selected
+FileActions.Folder_Already_Exists:The folder name '{0}' is already in use. Cannot create folder.
+FileActions.Increase_Depth:Increase Depth by 1
+FileActions.Insufficient_Space_Message:The file action has failed because of insufficient disk\nspace. For the file to copy successfully you require a\nfurther: {0}
+FileActions.Insufficient_Space_Title:Error - Insufficient Space
+FileActions.Moving:Moving {0}
+FileActions.No_Activity:No action requested
+FileActions.Possible_Cyclic_Path:{1} is {0} directories deep and may be a recursive path? Should file actioning continue?
+FileActions.Read_Only:Files from the workspace are read only and cannot be deleted.
+FileActions.Selected:{0} files and {1} folders selected
+FileActions.Unknown_File_Error_Message:Although no error message was reported during the file\naction, a final check has shown that the destination\ncopy is not the same as the source file. The reason\nfor this is unknown. Please check your media and update\nyour installed JVM.
+FileActions.Unknown_File_Error_Title:Error - Unknown File Error
+FileActions.Write_Not_Permitted_Message:The Librarian Interface does not have permission to write to\n{0}\nPlease check file permissions.
+FileActions.Write_Not_Permitted_Title:Error - Incorrect File Permissions
+FileActions.Yes_To_All:Yes to All
+#************************
+#
+#******* File Associations dialog **********
+FileAssociationDialog.Add:Add
+FileAssociationDialog.Add_Tooltip:Use this application to view files of this type
+FileAssociationDialog.Batch_File:Batch files
+FileAssociationDialog.Browse:Browse
+FileAssociationDialog.Browse_Tooltip:Choose the application to use from your filespace
+FileAssociationDialog.Browse_Tooltip_Mac:Browsing for your application is currently disabled for MacOS
+FileAssociationDialog.Browse_Title:Choose Application
+FileAssociationDialog.Close:Close
+FileAssociationDialog.Close_Tooltip:Close this dialog and return to the main window
+FileAssociationDialog.Command:Launch command:
+FileAssociationDialog.Command_File:Command files
+FileAssociationDialog.Command_Tooltip:Enter the command to launch the application to use
+FileAssociationDialog.Details:New File Association Details
+FileAssociationDialog.Executable_File:Executable files
+FileAssociationDialog.Existing_Associations:Current File Associations
+FileAssociationDialog.Extension:For files ending:
+FileAssociationDialog.Extension_Tooltip:The file extension of the files to view
+FileAssociationDialog.Instructions:Use this dialog to set what external programs are used to view a certain file-type, and what command is sent to launch this program.\nTo add a new file-type, type its extension in 'For files ending'. To add or modify a launch command, type the new command in 'Launch command' or click 'Browse' to search the file system for an appropriate program. Use the special string '%1' to indicate where the name of the file to be opened should be inserted, e.g. "C:\\Windows\\System\\MSPaint.exe %1". '%1' will be appended to the end of a command if not otherwise specified.\nMacOS users should always try the mac specific command "open [-a <application_name>] %1" first.
+FileAssociationDialog.Table.Command:Command
+FileAssociationDialog.Table.Extension:Extension
+FileAssociationDialog.Title:Edit File Associations
+FileAssociationDialog.Remove:Remove
+FileAssociationDialog.Remove_Tooltip:Remove the current selected association from the existing associations
+FileAssociationDialog.Replace:Replace
+FileAssociationDialog.Replace_Tooltip:Replace the currently selected association with the new association details
+#*******************
+#
+#******Filter***************
+Filter.0:HTM & HTML
+Filter.1:XML
+Filter.2:Text Files
+Filter.3:Images
+Filter.All_Files:All Files
+Filter.Filter_Tree:Show Files
+Filter.Invalid_Pattern:<HTML><FONT FACE={FONT} COLOR=Black>The pattern you have entered for filtering files is invalid.<BR>Please try again using regular expression syntax and with<BR>* for the wildcard character.</FONT></HTML>
+#***********************
+#
+#***** GAuthenticator *****
+GAuthenticator.Password:Password:
+GAuthenticator.Password_Tooltip:Enter your password here
+GAuthenticator.Title:Password Required.
+GAuthenticator.Username:User Name:
+GAuthenticator.Username_Tooltip:Enter your username here
+#**************************
+#
+#***** General Messages *****
+# 0 - A String representing the name and method of the class in question.
+General.Apply:Apply
+General.Apply_Tooltip:Apply the current settings but do not exit the dialog
+General.Browse:Browse...
+General.Cancel:Cancel
+General.Cancel_Tooltip:Cancel the changes (will not undo any that have been applied already)
+General.Close:Close
+General.Close_Tooltip:Close this dialog box
+General.Edit:Edit Value
+General.Error:Error
+General.No:No
+General.NotSunJava:Java vendor: {0}\nNote that the Librarian Interface has been developed and tested with Java from Sun Microsystems. 
+General.OK:OK
+General.OK_Tooltip:Accept the current settings and exit the dialog
+General.Open:Open
+General.Outstanding_Processes:<html>The Librarian Interface cannot close completely until all programs<br>started within The Librarian Interface have also exited.</html>
+General.Outstanding_Processes_Title:Waiting To Exit
+General.Pure_Cancel_Tooltip:Abort this action (no changes will be made)
+General.Review_Output:Please review output below:
+General.Usage:Usage: {0} <params>\nAll parameters are optional but the Librarian Interface may fail to work if -gsdl and -library aren't set.\n -gsdl <path>  : Force path to gsdl\n -library <path> : Path to cgi-bin\n -mozilla <path> : Enabled mozilla\n -mirror : Enable web-mirroring controls\n -laf [java|windows|motif|mac] : Look and feel\n -debug : Enable debug messages\n -no_load : Don't load previously open collection\n -load <path> : Load the specified collection
+General.View:View Value
+General.Warning:Warning
+General.Yes:Yes
+#****************************
+#
+#***** GShell *****
+GShell.BadArgument:Argument {0} is invalid.
+GShell.BadArgumentValue:Argument {0} has an invalid value.
+GShell.BadPluginOptions:The bad argument occurred in plugin {0}.
+GShell.BadClassifierOptions:The bad argument occurred in classifier {0}.
+GShell.Build.Auxilary:Creating auxiliary files and tidying up...
+GShell.Build.BuildBegun1:************** Build  Started **************
+GShell.Build.BuildCancelled:**************   Cancelled    **************
+GShell.Build.BuildComplete1:************** Build Finished **************
+GShell.Build.CompressText:Compressing text...
+GShell.Build.Index:Creating an index based on {0-index source and level}...
+GShell.Build.InfoDatabase:Creating information database...
+GShell.Build.Phind:Generating a Phind classifier. This may take some time...
+GShell.Command:Command
+GShell.Failure:Command failed.
+GShell.Import.FileNotProcessed:The file {0-target file path} was recognised but could not be processed by any plugin.
+GShell.Import.FileNotRecognised:The file {0-target file path} was not recognised by any plugin.
+GShell.Import.FileProcessing:The file {0-target file path} is being processed by {1-plugin name}.
+GShell.Import.FileProcessingError:The file {0-target file path} encountered an error during processing.
+GShell.Import.ImportBegun1:************** Import  Started **************
+GShell.Import.ImportComplete1:************** Import Finished **************
+GShell.Import.ImportComplete2:{0-number documents considered} documents were considered for processing:
+GShell.Import.ImportComplete2.1:1 document was considered for processing:
+GShell.Import.ImportComplete.Processed:{0-number processed} documents were processed and included in the collection. 
+GShell.Import.ImportComplete.Processed.1:1 document was processed and included in the collection. 
+GShell.Import.ImportComplete.Blocked:{0-number blocked} were prevented from being added by plugins (such as images in web pages).
+GShell.Import.ImportComplete.Blocked.1:1 was prevented from being added by plugins (such as images in web pages).
+GShell.Import.ImportComplete.Ignored:{0-number ignored} were unrecognised.
+GShell.Import.ImportComplete.Ignored.1:1 was unrecognised.
+GShell.Import.ImportComplete.Failed:{0-number failed} were rejected.
+GShell.Import.ImportComplete.Failed.1:1 was rejected.
+GShell.Import.ImportComplete3:\n
+GShell.Parsing_Metadata_Complete:Archived metadata extraction complete.
+GShell.Parsing_Metadata_Start:Extracting new metadata from archive files.
+GShell.Success:Command complete.
+#*********************
+#
+#***** GUI *****
+# Currently Enabled and Disabled display the same text but in different
+# colours. If necessary this can be changed.
+GUI.Create:Create
+GUI.Create_Tooltip:Build your collection using Greenstone
+GUI.Design:Design
+GUI.Design_Tooltip:Design the functionality and appearance of your collection
+GUI.Download:Download
+GUI.Download_Tooltip:Download resources from the internet for your collection
+GUI.Enrich:Enrich
+GUI.Enrich_Tooltip:Assign metadata to the files in your collection
+GUI.Gather:Gather
+GUI.Gather_Tooltip:Select the files to include in your collection
+#***************
+#
+#***** HELP *****
+Help.Contents:Contents
+Help.Title:Librarian Interface Help Pages
+#
+#***** LockFileDialog *****
+LockFileDialog.Cancel_Tooltip:Do not steal control of the lock file (collection will not open)
+LockFileDialog.Date:Date
+LockFileDialog.Error:error
+LockFileDialog.Lockfile_Message_One:The presence of a lock file suggests this collection is already open.
+LockFileDialog.Lockfile_Message_Two:Do you wish to 'steal' control of this lock file?
+LockFileDialog.Machine:Where
+LockFileDialog.Name:Name
+LockFileDialog.OK_Tooltip:Steal control of the lock file
+LockFileDialog.Title:Lock File Detected
+LockFileDialog.User:Who
+#********************
+#
+#*****MappingPrompt **********
+MappingPrompt.File:Folder
+MappingPrompt.Map:Create Shortcut
+MappingPrompt.Name:Name
+MappingPrompt.Title:Set Folder Shortcut
+MappingPrompt.Unmap:Remove Shortcut
+#*******************
+#
+#***** Menu Options *****
+Menu.Collapse:Close Folder
+Menu.Edit:Edit
+Menu.Edit_Copy:Copy (ctrl-c)
+Menu.Edit_Cut:Cut (ctrl-x)
+Menu.Edit_Paste:Paste (ctrl-v)
+Menu.Expand:Open Folder
+Menu.Explode_Metadata_Database:Explode metadata database
+Menu.File:File
+Menu.File_Associations:File Associations...
+Menu.File_CDimage:Write CD/DVD image...
+Menu.File_Close:Close
+Menu.File_Delete:Delete...
+Menu.File_Exit:Exit
+Menu.File_ExportAs:Export...
+Menu.File_New:New...
+Menu.File_Open:Open...
+Menu.File_Options:Preferences...
+Menu.File_Save:Save
+Menu.Help:Help
+Menu.Help_About:About...
+Menu.Metadata_View:Assigned Metadata for the {0}
+Menu.Open_Externally:Open in external program
+#********************
+#
+#***** MetaAudit *****
+MetaAudit.Close:Close
+MetaAudit.Close_Tooltip:Close this dialog box
+MetaAudit.Title:All Metadata
+#*********************
+#
+#***** Metadata *****
+Metadata.Element:Element
+Metadata.Value:Value
+MetadataSet.Files:Metadata Set Files
+#*********************************************
+#
+#******** MetadataImportMappingPrompt *********
+MIMP.Add:Add
+MIMP.Add_Tooltip:Add the metadata element to the selected set
+MIMP.Ignore:Ignore
+MIMP.Ignore_Tooltip:Ignore this metadata element
+MIMP.Instructions:The metadata element {0} cannot be automatically imported into the collection. Select a metadata set and either:\n(1) press 'Add' to add the metadata to that set (only if there is not already {0} metadata in the set), or\n(2) choose an element and press 'Merge' to import {0} metadata to this element, or\n(3) ignore this metadata element.
+MIMP.Merge:Merge
+MIMP.Merge_Tooltip:Map the metadata element to the specified element in the selected set
+MIMP.Source_Element:Source metadata element:
+MIMP.Target_Element:Target metadata element:
+MIMP.Target_Set:Target metadata set:
+MIMP.Title:Merging Action Required.
+#*****************************
+#
+#***** Mirroring *****
+Mirroring.ClearCache:Clear Cache
+Mirroring.ClearCache_Tooltip:Delete all downloaded files
+Mirroring.Download:Download
+Mirroring.Download_Tooltip:Start a new download job
+Mirroring.Download_Controls:Download Settings
+Mirroring.Download_Depth:Download Depth:
+Mirroring.Download_Depth.Zero:Zero
+Mirroring.Download_Depth.Unlimited:Unlimited
+Mirroring.Download_Depth_Tooltip:How many hyperlinks deep to go when downloading
+Mirroring.Download_Embedded:Download text only (no images, stylesheets etc)
+Mirroring.DownloadJob.Close:Close
+Mirroring.DownloadJob.Close_Tooltip:Remove this job from the list, stopping it if it is currently running
+Mirroring.DownloadJob.Downloading:Downloading {0}
+Mirroring.DownloadJob.Download_Complete:Download complete
+Mirroring.DownloadJob.Download_Stopped:Download halted
+Mirroring.DownloadJob.Download_Progress:Downloading in progress
+Mirroring.DownloadJob.Log:View Log
+Mirroring.DownloadJob.Log_Tooltip:View details about this download
+Mirroring.DownloadJob.Pause:Pause
+Mirroring.DownloadJob.Pause_Tooltip:Pause this download
+Mirroring.DownloadJob.Resume:Resume
+Mirroring.DownloadJob.Resume_Tooltip:Resume this download
+Mirroring.DownloadJob.Status:Downloaded {0} of {1} files ({2} warnings, {3} errors)
+Mirroring.DownloadJob.Waiting:Waiting to start
+Mirroring.DownloadJob.Waiting_User:Waiting for user action
+Mirroring.Higher_Directories:Only mirror files below this URL
+Mirroring.Invalid_URL:The URL entered is not valid. Please correct.
+Mirroring.Invalid_URL_Title:Invalid URL
+Mirroring.Preferences:Preferences...
+Mirroring.Preferences_Tooltip:Enable proxy and/or edit proxy settings
+Mirroring.Same_Host:Only mirror files within the same site
+Mirroring.Source_URL:Source URL:
+Mirroring.Source_URL_Tooltip:URL to download
+#*********************
+#
+#***** Missing EXEC *****
+MissingEXEC.Message:Warning! You will not be able to view your new collections within the Librarian Interface unless you specify the path to the Greenstone Library Webserver. You can enter this now in the text field below, or later by going to Preferences and choosing the Connection tab. Remember that a valid url must start with the protocol "http://", and will probably end with the executable name ("gsdl" for Greenstone local library server, "library" for other web servers).
+MissingEXEC.Title:Missing Greenstone Library Address
+#***** Missing EXEC_GS3 *****
+MissingEXEC_GS3.Message:Warning! You will not be able to view your new collections within the Librarian Interface unless you specify the path to the Greenstone context in the locally running Tomcat Webserver. You can enter this now in the text field below, or later by going to Preferences and choosing the Connection tab. Remember that a valid url must start with the protocol "http://". The default address is http://localhost:8080/greenstone3.
+MissingEXEC_GS3.Title:Missing Greenstone 3 Library Address
+#***** Missing GLIServer *****
+MissingGLIServer.Message:Warning! You will not be able to work with collections within the Librarian Interface unless you specify the address of the remote Greenstone "gliserver" script. You can enter this now in the text field below, or later by going to Preferences and choosing the Connection tab. Remember that a valid URL must start with the protocol "http://" and end with "gliserver".
+MissingGLIServer.Title:Missing GLIServer Address
+#***** Missing GSDL *****
+MissingGSDL.Message:Warning! You will not be able to access all of the Librarian Interface's features without a local copy of Greenstone Digital Library also installed. If you believe you are receiving this message in error please check your installation, and the paths specified in Librarian Interface batch or script file.
+MissingGSDL.Title:Missing Greenstone Path
+#***** Missing PERL *****
+MissingPERL.Message:Warning! You will not be able to create new collections, nor build existing ones, without PERL installed, and the Librarian Interface was unable to find PERL on your system. If you believe you are receiving this message in error please check your installation, and the paths specified in Librarian Interface batch or script file.
+MissingPERL.Title:Missing PERL Path
+#******* MissingWGET ********
+MissingWGET.Message:Warning! Web mirroring is disabled because the Librarian Interface failed to locate an appropriate version of WGet. If you believe you are receiving this message in error please check your installation and the paths specified in the Librarian Interface batch or script file.
+MissingWGET.Title:Missing WGet
+#******* MissingImageMagick ********
+MissingImageMagick.Message:Warning! The Librarian Interface failed to locate an appropriate version of ImageMagick, and consequently ImagePlug will not work. Please install ImageMagick if you wish to build image collections.
+MissingImageMagick.Title:Missing ImageMagick
+#*****************************
+#
+#***** New Session *****
+NewCollectionPrompt.Base_Collection:Base this collection on:
+NewCollectionPrompt.Base_Collection_Tooltip:Select an existing collection to base the new collection on
+NewCollectionPrompt.Collection_Description:Description of content:
+#NewCollectionPrompt.Collection_Email:Author's Email:
+NewCollectionPrompt.Collection_Name:Collection folder:
+#NewCollectionPrompt.Collection_Name_Tooltip:Short collection name, used internally (must be 8 characters or less)
+NewCollectionPrompt.Error:Error in new collection.
+NewCollectionPrompt.Instructions:To create a new collection fill out the fields below.
+NewCollectionPrompt.Metadata_Elements:Elements within selected set:
+NewCollectionPrompt.Metadata_Instructions1:Check the boxes beside the metadata sets to include in your collection.
+NewCollectionPrompt.Metadata_Instructions2:(Note: You can add new ones later).
+NewCollectionPrompt.Metadata_Title_Existing:Select metadata sets.
+#NewCollectionPrompt.Name_Error:You have either failed to enter a name for the collection, or the name you have choosen is already in use. Please correct.
+NewCollectionPrompt.NewCollection:-- New Collection --
+NewCollectionPrompt.Select_MDS:Available metadata sets:
+NewCollectionPrompt.Title:Create a new Collection.
+NewCollectionPrompt.Title_Clash:The title you have chosen for your collection is already in use. Continue?
+NewCollectionPrompt.Title_Error:The title field must be filled out. Please correct.
+#***** New Folder OrFilePrompt *****
+NewFolderOrFilePrompt.Default_File_Name:dummy
+NewFolderOrFilePrompt.Default_Folder_Name:new folder
+NewFolderOrFilePrompt.Destination_Name:Destination Folder
+NewFolderOrFilePrompt.File_Name:File Name
+NewFolderOrFilePrompt.File_Name_Tooltip:Enter a name for the new file
+NewFolderOrFilePrompt.Folder_Name:Folder Name
+NewFolderOrFilePrompt.Folder_Name_Tooltip:Enter a name for the new folder
+NewFolderOrFilePrompt.Title_Folder:Create a new folder
+NewFolderOrFilePrompt.Title_File:Create a new dummy document
+#***********************
+#
+NoMetadataSetsSelected.Message:If you create the collection now it will have no initial metadata sets. You will not be able to import nor assign any metadata until a set is imported or added to your collection. Do you wish to continue?
+NoMetadataSetsSelected.Title:No Metadata Sets Selected
+#**********************
+#
+#******* OldWGET *******
+NoPluginExpectedToProcessFile.Message:None of Greenstone's plugins are expected to process the file "{0}". Check that the file has the correct extension. If it is correct then you may have to use UnknownPlug to process this file.
+NoPluginExpectedToProcessFile.Title:No plugin expected to process file
+#**********************
+#
+#******* OldWGET *******
+OldWGET.Message:Warning! The version of WGet you have installed has a known bug when mirroring files which contain encoded characters such as space and tilde in their URL. While this shouldn't be a problem, we recommend installing WGet 1.9 or higher.
+OldWGET.Title:Old WGet
+#***********************
+#
+#***** OpenCollectionDialog *****
+OpenCollectionDialog.Available_Collections:Available Collections
+OpenCollectionDialog.Browse:Browse
+OpenCollectionDialog.Browse_Tooltip:Choose a collection from your filespace
+OpenCollectionDialog.Collection:Greenstone Collection (".col")
+OpenCollectionDialog.Description:Collection Description
+OpenCollectionDialog.Locked_Collection:Greenstone collection already in use
+OpenCollectionDialog.No_Description:Please select a greenstone collection (".col" file) to see description.
+OpenCollectionDialog.Normal_Collection:Greenstone collection
+OpenCollectionDialog.Open:Open
+OpenCollectionDialog.Open_Tooltip:Load the selected collection into the Librarian Interface
+OpenCollectionDialog.Title:Open Greenstone Collection
+#**********************
+#
+#***** Options Pane inside CreatePane *****
+OptionsPane.Cancelled: - cancelled
+OptionsPane.LogHistory:Log history
+OptionsPane.Successful: - successful
+OptionsPane.Unknown: - unknown
+OptionsPane.Unsuccessful: - unsuccessful
+#*******************
+#
+#***** Preferences *****
+Preferences:Preferences
+Preferences.Connection:Connection
+Preferences.Connection_Tooltip:Set the URL to the greenstone library and configure a proxy connection if appropriate
+Preferences.Connection.GLIServer_URL:Greenstone gliserver URL:
+Preferences.Connection.GLIServer_URL_Tooltip:The URL of the Greenstone "gliserver" CGI script
+Preferences.Connection.Library_Path:Library Path:
+Preferences.Connection.Library_Path_GS3:Greenstone Web Path:
+Preferences.Connection.Library_Path_Tooltip:The URL of the Greenstone web-server
+Preferences.Connection.Library_Path_Tooltip_GS3:The URL of the Greenstone web-server
+Preferences.Connection.ProgramCommand:Preview Command:
+Preferences.Connection.ProgramCommand_Tooltip:The launch command for the program used to view the collection preview. Remember to use %1 as the placeholder for the collection address, and to use speechmarks where necessary.
+Preferences.Connection.Proxy_Host:Proxy Host:
+Preferences.Connection.Proxy_Host_Tooltip:The address of your proxy server
+Preferences.Connection.Proxy_Port:Proxy Port:
+Preferences.Connection.Proxy_Port_Tooltip:The port number of your proxy server
+Preferences.Connection.Servlet:Servlet:
+Preferences.Connection.Servlet_Tooltip:The servlet to use to view your site
+Preferences.Connection.Servlet_Tooltip2:There is no servlet specified for this site, you can still build collections, but you wont be able to view them
+Preferences.Connection.Site:Site:
+Preferences.Connection.Site_Tooltip:The site you want to work in
+Preferences.Connection.Use_Proxy:Use proxy connection?
+Preferences.General:General
+Preferences.General_Tooltip:Set some general preferences
+Preferences.General.Email:User's Email:
+Preferences.General.Email_Tooltip:The email address automatically used for any collection created with the GLI
+Preferences.General.Font:Font:
+Preferences.General.Font_Tooltip:The font used for the GLI interface.
+Preferences.General.Interface_Language:Interface Language:
+Preferences.General.Interface_Language_Tooltip:Select the language you wish the interface and its controls to be displayed in. 
+Preferences.General.Restart_Required:GLI should now be restarted in order to complete updating its interface.
+Preferences.General.Show_File_Size:Show file sizes
+Preferences.General.Show_File_Size_Tooltip:Show file sizes next to entries in the Workspace and Collection file trees.
+Preferences.General.View_Extracted_Metadata:View Extracted Metadata
+Preferences.General.View_Extracted_Metadata_Tooltip:Display metadata extracted by the greenstone plugins in the metadata list in the Enrich view.
+Preferences.Mode:Mode
+Preferences.Mode_Tooltip:Click on an entry to select a suitable level of operation
+Preferences.Mode.Assistant:Library Assistant
+Preferences.Mode.Assistant_Description:Use this setting to gain access to the basic features of the Greenstone Librarian Interface: adding documents and metadata to collections, creating new collections whose structure mirrors existing ones, and building collections.
+Preferences.Mode.Librarian:Librarian
+Preferences.Mode.Librarian_Description:This setting gives normal use of the Greenstone Librarian Interface: creating and designing new collections, adding documents and metadata to collections, designing new collections, and building collections.
+Preferences.Mode.Systems:Library Systems Specialist
+Preferences.Mode.Systems_Description: This setting enables full use of the Greenstone Librarian Interface. To take advantage of all the features available, you will need to be able to formulate regular expressions.
+Preferences.Mode.Expert:Expert
+Preferences.Mode.Expert_Description:This setting enables expert use of the Greenstone Librarian Interface. It is recommended for users who are experienced with Greenstone, and are able to perform troubleshooting tasks that involve interpreting debugging output from PERL scripts. All features of the Greenstone Librarian Interface are enabled.
+Preferences.Warnings:Warnings
+Preferences.Warnings_Tooltip:Enable or disable various warning dialogs
+Preferences.Workflow:Workflow
+Preferences.Workflow_Tooltip:Design the interface workflow: set which views should be enabled
+Preferences.Workflow.Create:Create - Build your collection using Greenstone 
+Preferences.Workflow.Design:Design - Change collection appearence
+Preferences.Workflow.Download:Download - Download web-based resources
+Preferences.Workflow.Enrich:Enrich - Annotate with metadata
+Preferences.Workflow.Gather:Gather - Select files for collection
+Preferences.Workflow.Predefined.Label:Predefined Workflows
+Preferences.Workflow.Title:Check box to enable view
+#**********************
+#
+#**** PreviewCommandDialog *******
+PreviewCommandDialog.Instructions:Enter the command for opening the program to view collection preview. Remember to use %1 as the placeholder for the collection address, and to use speechmarks where necessary.
+PreviewCommandDialog.Title:Preview Command
+#**********************
+#
+#***** Remote Greenstone Server *****
+RemoteGreenstoneServer.Authentication_Message:Please enter your Greenstone username and password:
+RemoteGreenstoneServer.Error:An error has occurred on the remote Greenstone server while performing this operation:\n{0-Error message}
+RemoteGreenstoneServer.Error_Title:Remote Greenstone Server Error
+RemoteGreenstoneServer.Progress:Remote Greenstone Server Progress: 
+RemoteGreenstoneServer.Steal_Lock_Message:This collection is currently locked by user '{0-User name}'.\nDo you want to steal control of this lock file (not recommended, as work may be lost)?
+#**********************
+#
+#***** Save Collection Box *****
+SaveCollectionBox.File_Name:File Name:
+SaveCollectionBox.Files_Of_Type:Files of Type:
+SaveCollectionBox.Look_In:Look In:
+#**********************
+#
+#***** Save Progress Box *******
+SaveProgressDialog.Title:Saving Collection
+#*******************************
+#
+Server.QuitManual:The Librarian Interface was unable to close the local\nlibrary server automatically. Please exit the server\nmanually by clicking on the cross icon in its top\nright-hand corner, then click OK on this dialog.
+Server.QuitTimeOut:The greenstone local library has not responded to the\nautomatic close command for sixty seconds. Do you\nwish to wait for another minute?
+#
+#***** Sources *****
+Source.General:General
+#*******************
+#
+#***** TitleClash *****
+TitleClashes.Message:The title you have chosen for your collection is already in use. Continue?
+TitleClashes.Title:Entered Title Clashes
+#**********************
+#
+#***** Trees *****
+Tree.DownloadedFiles:Downloaded Files
+Tree.Files:files
+Tree.Home:Home Folder ({0})
+Tree.Root:Local Filespace
+Tree.World:Documents in Greenstone Collections
+#***********************
+#
+#***** Warning Dialog *****
+WarningDialog.Dont_Show_Again:Do not show this warning again
+WarningDialog.Dont_Show_Again_Message:Do not show this message again
+WarningDialog.Invalid_Value:The value you have entered in the value field\nis not valid for the specified property.
+WarningDialog.Value:Enter value:
+#
+#****** WGet *****
+WGet.Prompt:WGet Mirroring Client
+#
+#****** Workflows *****
+Workflow.AllPanels:All panels
+Workflow.LocalSourceCollectionBuilding:Local source collection building
+Workflow.DocumentEnriching:Document enriching
+
+GEMS.Already_Loaded_Set_Error: The metadata set you selected is currently open.
+GEMS.Element_Already_Exists:An element with this name already exists.
+GEMS.Set_Already_Exists:A set with this namespace already exists.
+GEMS.Cannot_Delete_Core_Sets_Error:You cannot remove a core GLI metadata set.
+
+GEMS.Preferences.Selected_Languages_Tooltip: Selected languages to display:
+
+GEMS.Add_Attribute:Add attribute
+GEMS.Add_Attribute_Tooltip:Add a new attribute to the selected item
+GEMS.Add_Element:Add element
+GEMS.Add_Element_Tooltip:Add a new element to the selected set
+GEMS.Add_Subelement:Add subelement
+GEMS.Add_Subelement_Tooltip:Add a new subelement to the selected element
+GEMS.Add_Set:Add Set
+GEMS.Attribute:Attribute
+GEMS.Attribute_Already_Exists:This attribute already exists.
+GEMS.Attribute_Name_Tooltip:The name of the new attribute
+GEMS.Attribute_Language_Tooltip:The language of the new attribute
+GEMS.Attribute_Value_Tooltip:The value of the new attribute
+GEMS.Attributes:Attributes
+GEMS.Cannot_Undo:(this action cannot be undone)
+GEMS.Cannot_Add_Elements_To_Greenstone_MDS:You cannot manually add elements to the Greenstone Metadata. If\nnecessary create a new set, then add your new element to it instead.
+GEMS.Confirm_Exit_Save: Would you like to save before exiting?
+GEMS.Confirm_Exit_Save_Title: Save Before Exit?
+GEMS.Confirm_Removal:Are you sure you wish to remove this {0}?
+GEMS.Confirm_Removal_Title:Confirm Removal
+GEMS.Edit_Attribute:Edit Attribute
+GEMS.Edit_Value:Edit Value
+GEMS.Edit_Value_Tooltip:Change the selected value
+GEMS.Element:Element
+GEMS.Element_Name_Tooltip:The name of the new element
+GEMS.Element_Details:Metadata Element Details
+GEMS.Language:Language
+GEMS.Language_Code:Language
+GEMS.Name:Name
+GEMS.Namespace:Namespace
+GEMS.New_Name:New Name
+GEMS.No_Description:No description found.
+GEMS.No_Name:No name found.
+GEMS.Open_Set:Open Set
+GEMS.Remove_Attribute:Remove attribute
+GEMS.Remove_Attribute_Tooltip:Remove the selected attribute
+GEMS.Remove_Element:Remove element
+GEMS.Remove_Element_Tooltip:Remove the selected element from the set
+GEMS.Remove_Set:Remove set
+GEMS.Remove_Set_Tooltip:Remove the selected set
+GEMS.Rename_Element:Rename element
+GEMS.Rename_Element_Tooltip:The element's new name
+GEMS.Rename_Name_Tooltip:The metadata set's new name
+GEMS.Rename_Namespace_Tooltip:The metadata set's unique namespace
+GEMS.Rename_Set:Rename set
+GEMS.Rename_Set_Tooltip:Rename the metadata set
+GEMS.Set:Set
+GEMS.Set_Details:Metadata Set Details
+GEMS.Set_Name_Tooltip:The name of the new metadata set
+GEMS.Set_Namespace_Tooltip:A unique namespace for the new metadata set
+GEMS.Subelement_Name_Tooltip:The name of the new subelement
+GEMS.Subelement_Already_Exists:A subelement with this name already exists.
+GEMS.Target:Element
+GEMS.Title:Greenstone Editor for Metadata Sets
+GEMS.Value:Value
+GEMS.Values:Values
+GEMS.inheritMetadataSet:Inherit from Metadataset:
+GEMS.Add_Set.No_Inherit:Do not inherit
Index: /main/tags/3.01/gli/classes/dictionary_es.properties
===================================================================
--- /main/tags/3.01/gli/classes/dictionary_es.properties	(revision 21112)
+++ /main/tags/3.01/gli/classes/dictionary_es.properties	(revision 21112)
@@ -0,0 +1,1016 @@
+#*******************
+#
+#***** AboutDialog *****
+AboutDialog.Acknowledgement:Reconocimientos de marca de paquetes, clases y programas propietarios...
+AboutDialog.Date:(Octubre 2005)
+AboutDialog.Item0:Xerces Java 1' de The Apache Software Foundation (http://www.apache.org/)
+AboutDialog.Item2:WGet' de Free Software Foundation, Inc. (http://www.gnu.org/software/wget/wget.html)
+AboutDialog.Item3:qflib' de Quality First Software GmbH. (http://www.qfs.de/en/qflib/index.html)
+AboutDialog.Item4:ExtensionFileFilter.class', por Bruno Dumon y asociados (http://pollo.sourceforge.net/)
+AboutDialog.Item5:How to disable file renaming in file chooser' - vladi21 (http://www.experts-exchange.com/)
+AboutDialog.Item6:Autoscrolling' - Java Swing, por Robert Eckstein, Marc Loy & Dave Wood (http://www.oreilly.com/catalog/jswing/chapter/dnd.beta.pdf)
+AboutDialog.Item7:Add ghosted drag images to your JTrees', por Andrew J. Armstrong (http://www.javaworld.com/javaworld/javatips/jw-javatip114.html)
+AboutDialog.Item8:Stop annoying java.prefs error when you don't have superuser access', cortesÃ­a de Walter Schatz
+AboutDialog.Java_Req:Java es una marca registrada de Sun Microsystems Inc. El Ambiente de EjecuciÃ³n Java (JRE por sus siglas en inglÃ©s) se distribuye de acuerdo a los tÃ©rminos de la licencia suplementaria que se encuentra en http://java.sun.com/j2se/1.4.1/j2re-1_4_1_02-license.html
+AboutDialog.Java_Req_One:Este producto incluye un cÃ³digo con licencia de uso de RSA Security, Inc.
+AboutDialog.Java_Req_Two:Algunas partes con licencia de IBM estÃ¡n disponibles en http\://oss.software.ibm.com/icu4j/
+AboutDialog.Thanks:TambiÃ©n queremos agradecer a todos aquellos que nos proporcionaron su valiosa informaciÃ³n sobre Java a travÃ©s de la Web...
+AboutDialog.Title:Acerca de...
+AboutDialog.Title_One:Interfaz de la Biblioteca Digital Greenstone
+AboutDialog.Title_Two:Biblioteca Digital Greenstone, Universidad de Waikato
+AboutDialog.Copyright:Copyright (C) 2003 New Zealand Digital Library Project
+AboutDialog.Copyright_Two:Publicado conforme a los tÃ©rminos y condiciones de la Licencia PÃºblica General. Para mayor informaciÃ³n vea LICENSE.txt
+#***********************
+#
+#***** Audit Table *****
+AuditTable.File:Nombre del archivo
+#***********************
+#
+#***** Auto Filter *****
+Autofilter.AND:Y
+Autofilter.Ascending:Ascendente
+Autofilter.Case_Sensitive:Concordancia de caso
+Autofilter.Custom_Filter:Avanzada
+Autofilter.Descending:Descendente
+Autofilter.Filter_By_Value:Sencilla
+Autofilter.Name:TÃ­tulo de la columna
+Autofilter.None:Ninguno
+Autofilter.Operator:CondiciÃ³n
+Autofilter.OR:O
+Autofilter.Order:Ordenar
+Autofilter.Order_Tooltip:Clasifica los elementos en este orden
+Autofilter.Remove:Eliminar filtro
+Autofilter.Remove_Tooltip:Quita este filtro de la columna
+Autofilter.Set:Filtrar
+Autofilter.Set_Tooltip:AÃ±ade este filtro a la columna
+Autofilter.Table_Header_Tooltip:Filtrar por {0}
+Autofilter.Title:Filtrar metadatos
+Autofilter.eqeq:igual a
+Autofilter.!eq:does not equal
+Autofilter.<:is less than
+Autofilter.<eq:is less than or equal to
+Autofilter.>:is greater than
+Autofilter.>eq:is greater than or equal to
+Autofilter.^:starts with
+Autofilter.!^:does not start with
+Autofilter.$:ends with
+Autofilter.!$:does not end with
+Autofilter.?:contains
+Autofilter.!?:does not contain
+#
+#     ********************************************************
+#     *************** Collection Design Module ***************
+#     ********************************************************
+#     * The dictionary  phrases  used in the creation of the *
+#     * Colection  Design  Module. Note that we  still  have *
+#     * access  to the format  arguments (argument  32+) for *
+#     * html formatting.                                     *
+#     ********************************************************
+CDM.ArgumentConfiguration.Add:AÃ±adir
+CDM.ArgumentConfiguration.Bad_Integer:<html>El argumento '{0}' requiere un nÃºmero entero,<br>pero no se puede obtener ningÃºn entero de {1}.</html>
+CDM.ArgumentConfiguration.Custom_Header:A la medida
+CDM.ArgumentConfiguration.Custom:Argumentos personalizados:
+CDM.ArgumentConfiguration.Error_Title:Error en los argumentos
+CDM.ArgumentConfiguration.Header:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Por favor, configure los argumentos para {0}.</CENTER></FONT></STRONG></HTML>
+CDM.ArgumentConfiguration.No_Value:El argumento â{0}â requiere un valor, pero no se ha especificado ninguno.
+CDM.ArgumentConfiguration.Remove:Eliminar
+CDM.ArgumentConfiguration.Required_Argument:El argumento '{0}' deberÃ¡ ser un valor dado.
+CDM.ArgumentConfiguration.Title:Configurando argumentos
+CDM.ClassifierManager.Add:AÃ±adir clasificador
+CDM.ClassifierManager.Add_Tooltip:Utilice el clasificador especificado con esta colecciÃ³n
+CDM.ClassifierManager.Assigned:Clasificadores asignados actualmente
+CDM.ClassifierManager.Classifier:Seleccione el clasificador que se va a aÃ±adir:
+CDM.ClassifierManager.Classifier_Tooltip:Son los clasificadores disponibles para esta colecciÃ³n
+CDM.ClassifierManager.Classifier_Exists:Ese clasificador ya se le ha asignado a esta colecciÃ³n.
+CDM.ClassifierManager.Classifier_Str:clasificador
+CDM.ClassifierManager.Classifier_XML_Parse_Failed:No se pudieron determinar los argumentos del clasificador {0}\n. Por favor, asegÃºrese que el clasificador\n contiene el indicador -xml ejecutando el guiÃ³n classinfo.pl\n, el cual es similar a la descripciÃ³n del guiÃ³n pluginfo.pl que se encuentra en\n la secciÃ³n 2.1 de la GuÃ­a del Programador de Greenstone.
+CDM.ClassifierManager.Configure:Configurar clasificador
+CDM.ClassifierManager.Configure_Tooltip:Cambia las opciones del clasificador seleccionado
+CDM.ClassifierManager.Controls:Controles de ediciÃ³n
+CDM.ClassifierManager.Instructions:Utilice esta pantalla para aÃ±adir, configurar o eliminar clasificadores de su colecciÃ³n. Para aÃ±adir un clasificador, selecciÃ³nelo del cuadro de utilerÃ­as y haga click en 'AÃ±adir Clasificador'.\n Para eliminar un clasificador, selecciÃ³nelo de la lista de clasificadores asignados y haga click en 'Eliminar Clasificador'.\n Para configurar un clasificador, selecciÃ³nelo de la lista de clasificadores asignados y haga click en 'Configurar Clasificador'.\n\n Cuando se configure un clasificador aparecerÃ¡ una lista de sus argumentos. Active los argumentos e ingrese o seleccione los valores segÃºn sea necesario.
+CDM.ClassifierManager.Remove:Eliminar clasificador
+CDM.ClassifierManager.Remove_Tooltip:Elimina el clasificador seleccionado de la lista
+CDM.ClassifierManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SelecciÃ³n y configuraciÃ³n de clasificadores</CENTER></FONT></STRONG></HTML>
+CDM.ClassifierManager.Parsing.Message:La herramienta de diseÃ±o estÃ¡ cargando la informaciÃ³n sobre el clasificador
+CDM.ClassifierManager.Parsing.Title:Cargando clasificadores
+CDM.FormatManager.Add:AÃ±adir formato
+CDM.FormatManager.Add_Tooltip:Utilice el comando de formato especificado junto con la colecciÃ³n
+CDM.FormatManager.Assigned_Formats:Comandos de formato asignados actualmente.
+CDM.FormatManager.Editor:Cadena de formato HTML
+CDM.FormatManager.Editor_Tooltip:Ingrese aquÃ­ el comando de formato
+CDM.FormatManager.Editing_Controls:Controles de ediciÃ³n
+CDM.FormatManager.Enabled:Activo
+CDM.FormatManager.Feature:Elegir funciÃ³n
+CDM.FormatManager.Feature_Tooltip:Es la funciÃ³n a la que se darÃ¡ formato
+CDM.FormatManager.Insert:Insertar
+CDM.FormatManager.Insert_Tooltip:Inserta la variable seleccionada en el comando de formato
+CDM.FormatManager.Instructions:Las pÃ¡ginas Web que usted ve cuando utiliza Greenstone no han sido guardadas previamente sino que se generan âal vueloâ conforme se van necesitando. Los comandos de formato se utilizan para cambiar la apariencia de las pÃ¡ginas generadas. Algunos son modificadores que controlan la presentaciÃ³n en pantalla de los documentos o partes de ellos, otros son mÃ¡s complejos y requieren un cÃ³digo html como argumento.\n Para aÃ±adir un comando de formato, selecciÃ³nelo de la lista de âfuncionesâ. Si aparece un cuadro de opciones Falso/Verdadero, elija el estado haciendo click en el botÃ³n correspondiente. Si aparece un cuadro con los controles âcomponente afectadoâ y âeditar cadenas de formatoâ, entonces necesita ingresar el texto html. Usted puede incluir variables predeterminadas seleccionÃ¡ndolas de la lista.\n Una vez que estÃ© satisfecho con los cambios, oprima el botÃ³n âAÃ±adir Formatoâ para agregarlo al archivo de la configuraciÃ³n.\n Elimine un comando de formato seleccionÃ¡ndolo de la lista âComandos de formato asignadosâ y haciendo click en âEliminar Formato'.
+CDM.FormatManager.Part:Componente afectado
+CDM.FormatManager.Part_Tooltip:Es el aspecto del componente afectado al que se va a dar formato
+CDM.FormatManager.Remove:Eliminar formato
+CDM.FormatManager.Remove_Tooltip:Elimina el comando de formato seleccionado de la lista
+CDM.FormatManager.Replace:Reemplazar formato
+CDM.FormatManager.Replace_Tooltip:Actualiza el comando de formato seleccionado
+CDM.FormatManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Comandos de formato</CENTER></FONT></STRONG></HTML>
+CDM.FormatManager.Variable:Variables
+CDM.FormatManager.Variable_Tooltip:Son las variables disponibles para el comando de formato
+CDM.General.Access:La colecciÃ³n serÃ¡ de acceso pÃºblico
+CDM.General.Beta:La colecciÃ³n sigue en construcciÃ³n
+CDM.General.Browser_Title:Seleccione una imagen
+CDM.General.Collection_Extra:DescripciÃ³n de la colecciÃ³n:
+CDM.General.Collection_Extra_Tooltip:Es una descripciÃ³n del propÃ³sito y los contenidos de la colecciÃ³n
+CDM.General.Collection_Name:TÃ­tulo de la colecciÃ³n:
+CDM.General.Collection_Name_Tooltip:Es el nombre de la colecciÃ³n
+CDM.General.Copy_Image:Para usar esta imagen para su colecciÃ³n, deberÃ¡n copiarse \n archivos a su carpeta de imÃ¡genes. Proceder?
+CDM.General.Email.Creator:E-mail del creador de la colecciÃ³n
+CDM.General.Email.Creator_Tooltip:es la direcciÃ³n de e-mail del creador de la colecciÃ³n
+CDM.General.Email.Maintainer:E-mail del administrador de la colecciÃ³n
+CDM.General.Email.Maintainer_Tooltip:Es la direcciÃ³n de e-mail del administrador de la colecciÃ³n
+CDM.General.Icon_Collection:URL del Ã­cono de la pÃ¡gina 'acerca de':
+CDM.General.Icon_Collection_Tooltip:Es el URL de la imagen que aparecerÃ¡ en la pÃ¡gina "Acerca de" de la biblioteca digital
+CDM.General.Icon_Collection_Small:URL del Ã­cono de la 'pÃ¡gina principal':
+CDM.General.Icon_Collection_Small_Tooltip:Es el URL de la imagen que aparecerÃ¡ en la pÃ¡gina principal de la biblioteca digital
+CDM.General.Image_Copy_Failed:Ha ocurrido un error que evita que la imagen que usted seleccionÃ³ se copie.\n Por favor trate de copiarla manualmente en la carpeta de imÃ¡genes de su colecciÃ³n.
+CDM.General.Image_Filter:ImÃ¡genes (gifs, jpgs y pngs)
+CDM.General.Instructions:La secciÃ³n de diseÃ±o de la Interfaz de la Biblioteca Digital le permite controlar varios aspectos relativos a la apariencia de su colecciÃ³n. El diseÃ±o se divide en varias secciones. Esta secciÃ³n contiene las opciones y ajustes generales. Para ver una secciÃ³n diferente, haga click en el nombre de la secciÃ³n en la lista que se encuentra a la izquierda.
+CDM.General.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Opciones generales</CENTER></FONT></STRONG></HTML>
+CDM.GUI.Classifiers:Clasificadores de exploraciÃ³n
+CDM.GUI.Design_Topics:Secciones de diseÃ±o
+CDM.GUI.Formats:Formatos
+CDM.GUI.General:General
+CDM.GUI.Indexes:Ãndices de bÃºsqueda
+CDM.GUI.MetadataSets:Conjuntos de metadatos
+CDM.GUI.Plugins:Plugins de documento
+CDM.GUI.Root:CDM_ROOT
+CDM.GUI.SearchTypes:Tipos de bÃºsqueda
+CDM.GUI.Subcollections:Particionar Ã­ndices
+CDM.GUI.SuperCollection:BÃºsqueda cruzada a travÃ©s de varias colecciones
+CDM.GUI.Translation:Traducir texto
+CDM.IndexManager.Add_Index:AÃ±adir Ã­ndice
+CDM.IndexManager.Add_Index_Tooltip:Crea el Ã­ndice especificado para esta colecciÃ³n
+CDM.IndexManager.Default_Index_Indicator:[Ãndice por omisiÃ³n]
+CDM.IndexManager.Document:documento
+CDM.IndexManager.Index_Exists:Ese Ã­ndice ya se le ha asignado a esta colecciÃ³n.
+CDM.IndexManager.Index_Name:Nombre del Ã­ndice
+CDM.IndexManager.Index_Name_Tooltip:Es un nombre exclusivo para el nuevo Ã­ndice
+CDM.IndexManager.Indexes:Ãndices asignados
+CDM.IndexManager.Instructions:AquÃ­ usted puede seleccionar los Ã­ndices de bÃºsqueda que la colecciÃ³n tendrÃ¡.\n Para aÃ±adir un nuevo Ã­ndice, seleccione el material que se va a indexar, elija el nivel del Ã­ndice y haga click en âAÃ±adir Ã­ndiceâ.\n Para eliminar un Ã­ndice, selecciÃ³nelo de la lista de Ã­ndices asignados y haga click en âEliminar Ã­ndiceâ.\n Para establecer un Ã­ndice por omisiÃ³n, selecciÃ³nelo de la lista y haga click en âPredeterminarâ'.
+CDM.IndexManager.Level:Nivel:
+CDM.IndexManager.Level_Exists:Ese nivel ya se le ha asignado a esta colecciÃ³n.
+CDM.IndexManager.Level_Tooltip:Es el nivel de documento sobre el que se crearÃ¡ el Ã­ndice
+CDM.IndexManager.MGPP.Add_All_Metadata:AÃ±adir todas
+CDM.IndexManager.MGPP.Add_All_Metadata_Tooltip:Construya un Ã­ndice separado para cada fuente disponible usando la fuente como nombre del Ã­ndice.
+CDM.IndexManager.MGPP.Add_Level:AÃ±adir nivel
+CDM.IndexManager.MGPP.Add_Level_Tooltip:Crea un Ã­ndice en el nivel especificado
+CDM.IndexManager.MGPP.Current_Indexes:Ãndices asignados
+CDM.IndexManager.MGPP.Current_Levels:Niveles asignados
+CDM.IndexManager.MGPP.Index:Fuente del Ã­ndice:
+CDM.IndexManager.MGPP.Index_Tooltip:Es elemento de metadato sobre el que crearÃ¡ el Ã­ndice
+CDM.IndexManager.MGPP.Indexes:Administrar Ã­ndices
+CDM.IndexManager.MGPP.Instructions:AquÃ­ usted elige los Ã­ndices de bÃºsqueda que la colecciÃ³n tendrÃ¡.\n Para aÃ±adir un nuevo Ã­ndice ingrese un nombre exclusivo para ese Ã­ndice, seleccione el material/metadato que se va a indexar y haga click en âAÃ±adir Ã­ndiceâ\n Si desea aÃ±adir todas las fuentes disponibles con el fin de construir Ã­ndices en ellas, entonces haga click en âAÃ±adir todosâ (el nombre para cada Ã­ndice serÃ¡ por omisiÃ³n el nombre de la fuente). Recuerde que esto es ligeramente diferente a aÃ±adir la fuente âAllFieldsâ, ya que el orden de los Ã­ndices es importante.\n Si desea editar un Ã­ndice en particular, selecciÃ³nelo de la lista de Ã­ndices asignados actualmente, edÃ­telo segÃºn sea necesario y luego haga click en âReemplazar Ã­ndiceâ para poder sustituir la versiÃ³n anterior.\n Para eliminar un Ã­ndice selecciÃ³nelo de la lista de Ã­ndices asignados actualmente y haga click en âEliminar Ã­ndiceâ.
+CDM.IndexManager.MGPP.Level:Nivel de la fuente:
+CDM.IndexManager.MGPP.Level_Name:Nombre del nivel:
+CDM.IndexManager.MGPP.Level_Name_Tooltip:Es un nombre Ãºnico para el Ã­ndice del nuevo nivel
+CDM.IndexManager.MGPP.Levels:Administrar niveles
+CDM.IndexManager.MGPP.Remove_Level:Eliminar nivel
+CDM.IndexManager.MGPP.Remove_Level_Tooltip:Elimina el Ã­ndice del nivel seleccionado
+CDM.IndexManager.MGPP.Replace_Index:Reemplazar Ã­ndice
+CDM.IndexManager.MGPP.Replace_Index_Tooltip:Actualiza el Ã­ndice seleccionado
+CDM.IndexManager.MGPP.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SelecciÃ³n de Ã­ndice y nivel</CENTER></FONT></STRONG></HTML>
+CDM.IndexManager.Paragraph:pÃ¡rrafo
+CDM.IndexManager.Remove_Index:Eliminar Ã­ndice
+CDM.IndexManager.Remove_Index_Tooltip:Elimina el Ã­ndice seleccionado de la lista
+CDM.IndexManager.Section:secciÃ³n
+CDM.IndexManager.Set_Default:Establecer un Ã­ndice por omisiÃ³n
+CDM.IndexManager.Set_Default_Tooltip:Hace que el Ã­ndice seleccionado sea el Ã­ndice por omisiÃ³n
+CDM.IndexManager.Source:Crear el Ã­ndice con base en:
+CDM.IndexManager.Source_Tooltip:Son los elementos de metadato sobre los que se crearÃ¡ el Ã­ndice
+CDM.IndexManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SelecciÃ³n del Ã­ndice</CENTER></FONT></STRONG></HTML>
+CDM.LanguageManager.Add:AÃ±adir idioma
+CDM.LanguageManager.Add_Tooltip:AÃ±ade el idioma especificado en la lista
+CDM.LanguageManager.Assigned_Languages:Idiomas asignados
+CDM.LanguageManager.Default_Language:Idioma predeterminado
+CDM.LanguageManager.Remove:Eliminar idioma
+CDM.LanguageManager.Remove_Tooltip:Elimina el idioma seleccionado de la lista
+CDM.LanguageManager.Selector:Idioma que se va a aÃ±adir
+CDM.LanguageManager.Selector_Tooltip:Es el idioma sobre el que se va a crear una particiÃ³n
+CDM.LanguageManager.Set_Default:Predeterminar
+CDM.LanguageManager.Set_Default_Tooltip:Hace que el idioma seleccionado sea el idioma por omisiÃ³n
+CDM.MetadataSetManager.Add:AÃ±adir un conjunto de metadatos
+CDM.MetadataSetManager.Add_Tooltip:AÃ±adir un nuevo conjunto de metadatos a la colecciÃ³n.
+CDM.MetadataSetManager.Edit:Editar conjunto de metadatos
+CDM.MetadataSetManager.Edit_Message:Los conjuntos de metadatos pueden editarse usando GEMS (Editor Greenstone para Conjuntos de Metadatos)./n Inicie GEMS ejecutando gems.bat (en Windows) o gems.sh (en Linux), que se encuentran en el directorio gli de su instalaciÃ³n de Greenstone.
+CDM.MetadataSetManager.Edit_Message_Title:Editando conjuntos de metadatos
+CDM.MetadataSetManager.Edit_Tooltip:Editar el conjunto de metadatos seleccionado.
+CDM.MetadataSetManager.Elements:Elementos del conjunto de metadatos seleccionado
+CDM.MetadataSetManager.Instructions:Por medio de la pantalla de diseÃ±o usted puede aÃ±adir, configurar y eliminar los conjuntos de metadatos de su colecciÃ³n, asÃ­ como los elementos que contienen.
+CDM.MetadataSetManager.Remove:Eliminar un conjunto de metadatos
+CDM.MetadataSetManager.Remove_Tooltip:Borrar el conjunto de metadatos seleccionado de la colecciÃ³n.
+CDM.MetadataSetManager.Sets:Conjuntos de metadatos disponibles
+CDM.MetadataSetManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Conjuntos de metadatos</CENTER></FONT></STRONG></HTML>
+CDM.Move.At_Bottom:{0} {1} no se puede mover mÃ¡s abajo puesto que ya se encuentra al final de la lista.
+CDM.Move.At_Top:{0} {1} no se puede mover mÃ¡s arriba puesto que ya se encuentra al principio de la lista.
+CDM.Move.Cannot:El plugin {0} no se puede mover mÃ¡s abajo ya que RecPlug y ArcPlug deben estar al final de la lista.
+CDM.Move.Fixed:<html>Por regla general RecPlug y ArcPlug siempre deben estar al final.<br>Si por alguna razÃ³n tiene que moverlos de lugar, por favor edite manualmente el archivo collect.cfg.</html>
+CDM.Move.Move_Down:Mover abajo
+CDM.Move.Move_Down_Tooltip:Mueve el elemento seleccionado hacia abajo en la lista
+CDM.Move.Move_Up:Mover arriba
+CDM.Move.Move_Up_Tooltip:Mueve el elemento seleccionado hacia arriba en la lista
+CDM.Move.Title:Error - no se puede mover
+CDM.PlugInManager.Add:AÃ±adir plugin
+CDM.PlugInManager.Add_Tooltip:Utilice el plugin especificado con esta colecciÃ³n
+CDM.PlugInManager.Assigned:Plugins asignados actualmente
+CDM.PlugInManager.Configure:Configurar un plugin
+CDM.PlugInManager.Configure_Tooltip:Cambia las opciones del plugin seleccionado
+CDM.PlugInManager.Controls:Controles de ediciÃ³n
+CDM.PlugInManager.Ignore:No Agregar Plugin
+CDM.PlugInManager.Ignore_Tooltip:No agregar ningÃºn Plugin a esta colecciÃ³n.
+CDM.PlugInManager.Instructions:Utilice esta pantalla para aÃ±adir, configurar o eliminar plugins de su colecciÃ³n. Para aÃ±adir uno selecciÃ³nelo del cuadro de utilerÃ­as y haga click en âAÃ±adir Pluginâ.\n Para configurar o eliminar uno, elÃ­jalo de la lista de plugins asignados y a continuaciÃ³n:\n i) Cambie su posiciÃ³n en el orden haciendo click en las flechas. (Recuerde: la posiciÃ³n de RecPlug y ArcPlug es fija.)\n ii) ConfigÃºrelo haciendo click en âConfigurar Pluginâ,\n o iii) ElimÃ­nelo haciendo click en âEliminar Pluginâ.\n\n Los plugins se configuran por medio de un Ã¡rea de diseÃ±o que aparece cuando se selecciona esta opciÃ³n y que contiene una lista de argumentos. Active los argumentos e ingrese o seleccione los valores segÃºn sea necesario.
+CDM.PlugInManager.PlugIn:Seleccione el plugin que se va a aÃ±adir
+CDM.PlugInManager.PlugIn_Tooltip:Son los plugins disponibles para esta colecciÃ³n
+CDM.PlugInManager.PlugIn_Exists:Ese plugin ya se le ha asignado a esta colecciÃ³n.
+CDM.PlugInManager.PlugIn_Str:plugin
+CDM.PlugInManager.PlugIn_XML_Parse_Failed:No se pudieron determinar los argumentos del plugin {0}\n. Por favor, asegÃºrese que el plugin contiene\n el indicador -xml ejecutando el guiÃ³n pluginfo.pl descrito\n en la secciÃ³n 2.1 de la GuÃ­a del Programador de Greenstone.
+CDM.PluginManager.Plugin_Suggestion_Prompt:Ninguno de los plugins de esta colecciÃ³n puede procesar el archivo "{0}". Para procesar este archivo pueden utilizarse los siguientes plugins. En este momento usted puede agregar uno a su colecciÃ³n, seleccionÃ¡ndolo y oprimiendo el botÃ³n Agregar. (Tenga en cuenta que en cualquier momento puede cambiar la lista de plugins dirigiÃ©ndose a "Document Plugins" en el Panel de DiseÃ±o.)
+CDM.PlugInManager.Remove:Eliminar plugin
+CDM.PlugInManager.Remove_Tooltip:Elimina el plugin seleccionado de la lista
+CDM.PluginManager.SuggestedPluginListTitle:Plugins sugeridos para el archivo
+CDM.PlugInManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SelecciÃ³n y configuraciÃ³n de plugins</CENTER></FONT></STRONG></HTML>
+CDM.PlugInManager.Parsing.Message:La herramienta de diseÃ±o estÃ¡ cargando la informaciÃ³n del plugin
+CDM.PlugInManager.Parsing.Title:Cargando Plugins
+CDM.SearchTypeManager.Add:AÃ±adir tipo de bÃºsqueda
+CDM.SearchTypeManager.Add_Tooltip:Permite el tipo especificado de bÃºsqueda en esta colecciÃ³n
+CDM.SearchTypeManager.Assigned:Tipos de bÃºsqueda asignados actualmente
+# -- Missing translation: CDM.SearchTypeManager.BuildType_Selection
+# -- Missing translation: CDM.SearchTypeManager.BuildType_Selection_Tooltip
+CDM.SearchTypeManager.Enable:Permitir bÃºsquedas avanzadas
+CDM.SearchTypeManager.Instructions:Definir el tipo de bÃºsqueda es una funciÃ³n avanzada que sÃ³lo estÃ¡ disponible cuando se marca el recuadro âPermitir bÃºsquedas avanzadasâ. Una vez que se activa, otros controles para seleccionar y cambiar el orden de los tipos de bÃºsqueda tambiÃ©n estÃ¡n disponibles. Seleccione un tipo de bÃºsqueda del cuadro de utilerÃ­as y haga click en âAÃ±adir tipo de bÃºsquedaâ para agregarlo. Seleccione de la lista un tipo de bÃºsqueda existente y use los botones âMover arribaâ y âMover abajoâ para cambiar su orden, o el botÃ³n âEliminar tipo de bÃºsquedaâ para quitarlo de la lista.\n\n Los tipos de bÃºsqueda especifican el tipo de bÃºsqueda con el que contarÃ¡n las interfaces: formas, bÃºsqueda por campos y/o bÃºsqueda de sÃ³lo texto. El orden es importante, ya que el primero se usarÃ¡ por omisiÃ³n para la pÃ¡gina de bÃºsqueda inicial.\n\n Asimismo es importante recordar que al activar los diferentes tipos de bÃºsqueda, la apariencia de la pantalla de Ãndices tambiÃ©n cambia. La bÃºsqueda por campos es una funciÃ³n reciente de Greenstone y por lo tanto requiere de informaciÃ³n diferente. Cuando se active la funciÃ³n de tipos de bÃºsqueda todos los Ã­ndices se modificarÃ¡n para que sean vÃ¡lidos de acuerdo al nuevo esquema, mientras que al desactivar los tipos de bÃºsqueda la Interfaz de la Biblioteca Digital harÃ¡ lo posible por convertir los nuevos Ã­ndices de regreso al estilo anterior, aunque puede ser que no siempre tenga Ã©xito, en cuyo caso los Ã­ndices se eliminarÃ¡n. De cualquier manera, una copia de la lÃ­nea de comando del Ã­ndice original estarÃ¡ disponible como comentario en el archivo de configuraciÃ³n.
+CDM.SearchTypeManager.Remove:Eliminar tipo de bÃºsqueda
+CDM.SearchTypeManager.Remove_Tooltip:No permite el tipo seleccionado de bÃºsqueda para esta colecciÃ³n
+CDM.SearchTypeManager.SearchType:Tipo de bÃºsqueda
+CDM.SearchTypeManager.SearchType_Selection:Tipos de bÃºsqueda:
+CDM.SearchTypeManager.SearchType_Selection_Tooltip:Son los tipos de bÃºsqueda disponibles para esta colecciÃ³n
+CDM.SearchTypeManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SelecciÃ³n y orden de los tipos de bÃºsqueda</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionManager.Add:AÃ±adir filtro
+CDM.SubcollectionManager.Add_Tooltip:AÃ±ade el filtro especificado a la colecciÃ³n
+CDM.SubcollectionManager.Assigned:Filtros definidos para la subcolecciÃ³n
+CDM.SubcollectionManager.Exclude:Excluir
+CDM.SubcollectionManager.Flags:Indicadores que se usarÃ¡n al momento de contrastar
+CDM.SubcollectionManager.Flags_Tooltip:Indicadores de expresiones regulares que se utilizarÃ¡n al momento de cotejar (p. ej. âiâ para comparaciones que no toman en cuenta el caso)
+CDM.SubcollectionManager.Include:Incluir
+CDM.SubcollectionManager.Inclusive:Â¿QuÃ© hacemos con los archivos que concuerdan?
+CDM.SubcollectionManager.Instructions:Utilice esta pantalla para refinar la creaciÃ³n de Ã­ndices.\n i) Defina los filtros de la subcolecciÃ³n que se emplearÃ¡n para depurar la selecciÃ³n de documentos de la colecciÃ³n con base en los valores de los metadatos especificados. Una vez definidos usted puede generar particiones sobre cualquier Ã­ndice especificado previamente a partir de uno o mÃ¡s de dichos filtros.\n ii) Genere particiones de Ã­ndices con base en el idioma. Esto filtra los documentos de la colecciÃ³n de tal manera que Ãºnicamente se incluyen aquellos documentos en el idioma seleccionado.\n Recuerde que el nÃºmero total de particiones generadas es una combinaciÃ³n de todos los Ã­ndices, filtros de subcolecciÃ³n e idiomas seleccionados. De esta manera, dos Ã­ndices con dos filtros de subcolecciÃ³n en dos idiomas producirÃ¡n ocho particiones.
+CDM.SubcollectionManager.Language_Controls:Asignar idiomas
+CDM.SubcollectionManager.Match:ExpresiÃ³n regular con la que se contrastarÃ¡:
+CDM.SubcollectionManager.Match_Tooltip:Es una expresiÃ³n regular que define los documentos de la subcolecciÃ³n
+CDM.SubcollectionManager.Name:Nombre del filtro de la subcolecciÃ³n:
+CDM.SubcollectionManager.Name_Tooltip:Es el nombre del nuevo filtro de la subcolecciÃ³n
+CDM.SubcollectionManager.Partitions_Disabled:Este programa no estÃ¡ activo en este momento ya que usted estÃ¡ operando en el modo {0-Modo Actual}. Ãnicamente se puede tener acceso a Ã©l en los modos {1-Sistemas} y {2-Avanzado}. Utilice la opciÃ³n de Preferencias que se encuentra en el menÃº de Archivo para cambiar el modo de operaciÃ³n.
+CDM.SubcollectionManager.Remove:Eliminar filtro
+CDM.SubcollectionManager.Remove_Tooltip:Quita el filtro seleccionado de la lista
+CDM.SubcollectionManager.Replace:Reemplazar filtro
+CDM.SubcollectionManager.Replace_Tooltip:Actualiza los detalles del filtro seleccionado
+CDM.SubcollectionManager.Source:Atributo del documento con el cual se va a contrastar:
+CDM.SubcollectionManager.Source_Tooltip:Es el atributo con base en el cual se filtrarÃ¡n los documentos
+CDM.SubcollectionManager.Subcollection_Controls:Definir filtros
+CDM.SubcollectionManager.Subindex_Controls:Asignar particiones
+CDM.SubcollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Particionando Ã­ndices</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionIndexManager.Add_Subindex:AÃ±adir particiÃ³n
+CDM.SubcollectionIndexManager.Add_Subindex_Tooltip:AÃ±ade la particiÃ³n especificada a esta colecciÃ³n
+CDM.SubcollectionIndexManager.Default_Partition_Indicator:[ParticiÃ³n por Defecto]
+CDM.SubcollectionIndexManager.PartitionName:Nombre de la particiÃ³n:
+CDM.SubcollectionIndexManager.PartitionName_Tooltip:Es el nombre de la nueva particiÃ³n
+CDM.SubcollectionIndexManager.Remove_Subindex:Eliminar particiÃ³n
+CDM.SubcollectionIndexManager.Remove_Subindex_Tooltip:Elimina la particiÃ³n seleccionada de esta colecciÃ³n
+CDM.SubcollectionIndexManager.Replace_Subindex:Remplazar ParticiÃ³n
+CDM.SubcollectionIndexManager.Replace_Subindex_Tooltip:Actualizar la particiÃ³n seleccionada
+CDM.SubcollectionIndexManager.Set_Default_Subindex:Predeterminar
+CDM.SubcollectionIndexManager.Set_Default_Subindex_Tooltip:Hace que la particiÃ³n seleccionada sea la particiÃ³n por omisiÃ³n
+CDM.SubcollectionIndexManager.Source:Construir particiÃ³n en:
+CDM.SubcollectionIndexManager.Source_Tooltip:Los filtros de la subcolecciÃ³n para construir la particiÃ³n en
+CDM.SubcollectionIndexManager.Subindexes:Particiones asignadas a la subcolecciÃ³n
+CDM.SuperCollectionManager.Instructions:BÃºsqueda cruzada es cuando una bÃºsqueda se realiza a travÃ©s de varias colecciones distintas, como si las colecciones fueran una sola. Especifique las colecciones que se incluirÃ¡n en la bÃºsqueda haciendo click en el nombre de la colecciÃ³n de entre la lista que aparece abajo. La colecciÃ³n actual se incluirÃ¡ automÃ¡ticamente. \n CUIDADO: Si las colecciones no tienen los mismos Ã­ndices (incluyendo particiones en subcolecciones e idiomas) la bÃºsqueda cruzada no funcionarÃ¡ correctamente. El usuario Ãºnicamente serÃ¡ capaz de buscar usando Ã­ndices comunes a todas las colecciones.
+CDM.SuperCollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>ConfiguraciÃ³n de la bÃºsqueda cruzada a travÃ©s de varias colecciones</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Add:AÃ±adir traducciÃ³n
+CDM.TranslationManager.Add_Tooltip:AÃ±ade la traducciÃ³n especificada a la colecciÃ³n
+CDM.TranslationManager.Affected_Features:Funciones
+CDM.TranslationManager.Assigned_Fragments:Traducciones asignadas
+CDM.TranslationManager.Default_Text:Fragmento de texto inicial
+CDM.TranslationManager.Fragment_Column:Fragmento de texto
+CDM.TranslationManager.Instructions:Utilice esta pantalla para revisar y asignar traducciones de fragmentos de texto a su colecciÃ³n. Primero, seleccione una entrada de la lista de funciones (p. ej. nombres de clasificadores y metadatos a nivel de colecciÃ³n). A continuaciÃ³n, seleccione y edite un fragmento de texto, o seleccione un idioma del cuadro de utilerÃ­as a fin de aÃ±adir un nuevo fragmento. Escriba el texto en el Ã¡rea en blanco, haciendo referencia al fragmento de texto inicial segÃºn sea necesario. Cualquier cambio que se haga aparecerÃ¡ inmediatamente en la colecciÃ³n.
+CDM.TranslationManager.Language:Idioma de la traducciÃ³n
+CDM.TranslationManager.Language_Column:Idioma
+CDM.TranslationManager.Language_Tooltip:Es el idioma al que se traducirÃ¡
+CDM.TranslationManager.Remove:Eliminar traducciÃ³n
+CDM.TranslationManager.Remove_Tooltip:Quita la traducciÃ³n seleccionada de la colecciÃ³n
+CDM.TranslationManager.Replace:Reemplazar traducciÃ³n
+CDM.TranslationManager.Replace_Tooltip:Actualiza la traducciÃ³n seleccionada
+CDM.TranslationManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>TraducciÃ³n de textos</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Translation:Texto traducido
+CDM.TranslationManager.Translation_Tooltip:Escriba aquÃ­ la traducciÃ³n de la cadena de texto
+#*******************
+#
+#***** Collection *****
+Collection.Collection:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>ColecciÃ³n</FONT></STRONG></HTML>
+Collection.Delete_Tooltip:Elimina los archivos y carpetas seleccionados
+Collection.Filter_Tooltip:Limita el nÃºmero de archivos mostrados en este Ã¡rbol de archivos
+Collection.New_Folder_Tooltip:Crea una nueva carpeta
+Collection.No_Collection:No hay colecciÃ³n
+Collection.Stop:Detener
+Collection.Stop_Tooltip:Detiene la copia de los archivos
+Collection.Workspace:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Ãrea de trabajo</FONT></STRONG></HTML>
+#**********************
+#
+#***** CollectionBuilt *****
+CollectionBuilt.Message:Se ha construido la colecciÃ³n y estÃ¡ lista para verse.
+CollectionBuilt.Title:Resultado del proceso de creaciÃ³n de la colecciÃ³n
+#***************************
+#
+#***** CollectionManager *****
+CollectionManager.Build_Cancelled:Se ha cancelado la construcciÃ³n de la colecciÃ³n.
+CollectionManager.Cannot_Create_Collection:Ha ocurrido un error y no se ha podido crear la colecciÃ³n
+CollectionManager.Cannot_Delete_Index:Esta colecciÃ³n no se puede reconstruir debido a que los archivos de Ã­ndices anteriores no se pueden suprimir.\n Por favor asegÃºrese que Greenstone o algÃºn otro programa no estÃ¡ usando estos archivos\n e intÃ©ntelo de nuevo.
+CollectionManager.Cannot_Delete_Index_Log:Los archivos de Ã­ndices anteriores no se pudieron suprimir, por lo cual la importaciÃ³n se detuvo.
+CollectionManager.Cannot_Open:La colecciÃ³n en:\n{0-ruta del archivo de la colecciÃ³n}\n no se pudo abrir.
+CollectionManager.Cannot_Open_With_Reason:La colecciÃ³n en:\n{0-ruta de acceso al archivo de la colecciÃ³n}\n no pudo abrirse debido a:\n{1-mensaje de error}
+CollectionManager.Creating_New:Creando una nueva colecciÃ³n.
+CollectionManager.Log_Created:Se ha creado la carpeta de registro.
+CollectionManager.Loading_Collection:Cargando colecciÃ³n
+CollectionManager.Loading_Collection_Please_Wait:Cargando colecciÃ³n, espere por favorâŠ
+CollectionManager.Loading_Successful:ColecciÃ³n cargada {0}.
+CollectionManager.Missing_Config:Esta colecciÃ³n no cuenta con un archivo collect.cfg vÃ¡lido
+CollectionManager.Not_Col_File:{0}' no es un archivo de colecciÃ³n de la Interfaz de la Biblioteca Digital (.col)
+CollectionManager.Preview_Ready_Failed:Ha ocurrido un error que evitarÃ¡ que la colecciÃ³n se pueda ver de forma preliminar.
+CollectionManager.Preview_Ready_Title:Vista preliminar de la colecciÃ³n.
+CollectionManager.Session_Ready:La colecciÃ³n {0} estÃ¡ lista para usarse.
+#*****************************
+#
+#***** CollectionPopupMenu *****
+CollectionPopupMenu.Delete:Suprimir
+CollectionPopupMenu.New_Dummy_Doc:Nuevo documento postizo
+CollectionPopupMenu.New_Folder:Nueva carpeta
+#*******************************
+#
+#***** CreatePane *****
+CreatePane.Build:Opciones de construcciÃ³n de la colecciÃ³n
+CreatePane.Build_Tooltip:Opciones para las fases de indexaciÃ³n y clasificaciÃ³n en la formaciÃ³n de la colecciÃ³n
+CreatePane.Build_Collection:Crear colecciÃ³n
+CreatePane.Build_Collection_Tooltip:Inicia el proceso de formaciÃ³n de la colecciÃ³n
+CreatePane.Build_Progress:Grado de avance en el proceso de construcciÃ³n
+CreatePane.Cancel_Build:Cancelar la construcciÃ³n
+CreatePane.Cancel_Build_Tooltip:Detiene el proceso de formaciÃ³n de la colecciÃ³n
+CreatePane.Import:Opciones de importaciÃ³n de la colecciÃ³n
+CreatePane.Import_Tooltip:Opciones para las fases de conversiÃ³n de archivos y asignaciÃ³n de metadatos en la formaciÃ³n de la colecciÃ³n
+CreatePane.Import_Progress:Grado de avance del proceso de importaciÃ³n
+CreatePane.Log:Registro de mensajes
+CreatePane.Log_Tooltip:Muestra los resultados de intentos anteriores de construcciÃ³n de la colecciÃ³n
+CreatePane.Mode_All:todos
+CreatePane.Options:Grupos opcionales
+CreatePane.Options_Title:Opciones de importaciÃ³n y construcciÃ³n de la colecciÃ³n
+CreatePane.Preview_Collection:Vista previa de la colecciÃ³n
+CreatePane.Preview_Collection_Tooltip:Vista previa de la colecciÃ³n que se ha construido
+#**********************
+#
+#***** Dates *****
+Dates.Mon:Lunes
+Dates.Tue:Martes
+Dates.Wed:MiÃ©rcoles
+Dates.Thu:Jueves
+Dates.Fri:Viernes
+Dates.Sat:SÃ¡bado
+Dates.Sun:Domingo
+Dates.Jan:Enero
+Dates.Feb:Febrero
+Dates.Mar:Marzo
+Dates.Apr:Abril
+Dates.May:Mayo
+Dates.Jun:Junio
+Dates.Jul:Julio
+Dates.Aug:Agosto
+Dates.Sep:Septiembre
+Dates.Oct:Octubre
+Dates.Nov:Noviembre
+Dates.Dec:Diciembre
+#**********************
+#
+#***** Delete Collection Prompt *****
+DeleteCollectionPrompt.Collection_Details:Detalles de la colecciÃ³n seleccionada
+DeleteCollectionPrompt.Collection_List:Colecciones disponibles
+DeleteCollectionPrompt.Confirm_Delete:Por favor marque el recuadro para confirmar que desea eliminar la colecciÃ³n.
+DeleteCollectionPrompt.Delete:Suprimir
+DeleteCollectionPrompt.Delete_Tooltip:Elimina la colecciÃ³n seleccionada
+DeleteCollectionPrompt.Details:Creador - {0}\nAdministrador - {1}\nDescripciÃ³n\n{2}
+DeleteCollectionPrompt.Failed_Delete:La colecciÃ³n no se pudo eliminar por completo.
+DeleteCollectionPrompt.Failed_Title:Â¡No se puede eliminar!
+DeleteCollectionPrompt.No_Collection:No se ha seleccionado ninguna colecciÃ³n.
+DeleteCollectionPrompt.Successful_Delete:La colecciÃ³n ha sido eliminada.
+DeleteCollectionPrompt.Successful_Title:Â¡Se ha eliminado por completo!
+DeleteCollectionPrompt.Title:EliminaciÃ³n de la colecciÃ³n
+#**********************
+#
+#***** DirectoryLevelMetadata *****
+DirectoryLevelMetadata.Message:EstÃ¡ a punto de asignar metadatos âa nivel de carpetaâ a la carpeta (o carpetas) seleccionadas. Estos metadatos serÃ¡n heredados automÃ¡ticamente por todos sus archivos y subcarpetas, y  no podrÃ¡n eliminarse de ninguno de ellos de forma individual, sino sÃ³lo reemplazarse. Para continuar haga click en âAceptarâ.
+DirectoryLevelMetadata.Title:A punto de aÃ±adir metadatos a nivel de carpeta
+#********************
+#
+#***** EnrichPane *****
+EnrichPane.Accumulate:Agregar
+EnrichPane.Accumulate_Tooltip:Agrega este valor al elemento de metadato seleccionado
+EnrichPane.AutoMessage:Este elemento, {0}, pertenece al conjunto de metadatos de Greenstone, el cual representa metadatos que normalmente pueden extraerse automÃ¡ticamente. Para saber quÃ© metadatos {0} pueden extraerse, construya una colecciÃ³n por medio de la pantalla âCrearâ.
+# -- Missing translation: EnrichPane.ExistingValues
+EnrichPane.Expand:...
+EnrichPane.Expand_Tooltip:Edita el valor en un cuadro de texto mÃ¡s grande
+# -- Missing translation: EnrichPane.InheritedMetadataSelected
+EnrichPane.No_File:No se ha seleccionado ningÃºn archivo
+EnrichPane.No_Metadata:No hay metadatos disponibles
+EnrichPane.No_Metadata_Element:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>No se ha seleccionado ningÃºn elemento de metadato</FONT></STRONG></HTML>
+EnrichPane.Overwrite:Reemplazar
+EnrichPane.Overwrite_Tooltip:Asigna este valor al elemento de metadato seleccionado, reemplazando el valor actual
+EnrichPane.Remove:Eliminar
+EnrichPane.Remove_Tooltip:Elimina este valor del elemento de metadato seleccionado
+EnrichPane.Value:Valor
+EnrichPane.Value_Field_Tooltip:Es el valor que se asignarÃ¡ al elemento de metadato seleccionado.
+EnrichPane.Value_Field_Tooltip_Uneditable:Valor asignado al meta-dato seleccionado
+#***********************
+#
+#****** ExplodeMetadataPrompt *******
+ExplodeMetadataPrompt.Explode:Explotar
+ExplodeMetadataPrompt.Explode_Tooltip:Explotar el archivo de metadatos
+ExplodeMetadataPrompt.Instructions:La explosiÃ³n de este archivo de metadatos producirÃ¡ registros individuales con metadatos editables. Este proceso es irreversible y el archivo original de metadatos serÃ¡ eliminado de la colecciÃ³n.
+ExplodeMetadataPrompt.Failed_Title:FallÃ³ la ExplosiÃ³n.
+ExplodeMetadataPrompt.Failed_Explode:No se pudo explotar la base de datos {0}
+ExplodeMetadataPrompt.NotExplodable:Este archivo no es explotable.
+ExplodeMetadataPrompt.Successful_Explode:La base de datos {0} ha sido explotada con Ã©xito
+ExplodeMetadataPrompt.Successful_Title:Se completÃ³ la ExplosiÃ³n.
+ExplodeMetadataPrompt.Title:Explotar la Base de Datos de Metadatos
+#***********************
+#
+#****** WriteCDImagePrompt *******
+WriteCDImagePrompt.CD_Name:Nombre del CD-ROM
+WriteCDImagePrompt.Export:Exportar
+WriteCDImagePrompt.Export_Tooltip:Exportar las colecciones seleccionadas al CD-ROM para Windows
+WriteCDImagePrompt.Failed_Export:Las colecciones  ({0}) no se pudieron exportar al CD-ROM
+WriteCDImagePrompt.Failed_Title:No se pudo exportar
+WriteCDImagePrompt.Instructions:Exportar una o mÃ¡s colecciones a un CD-ROM para Windows que se podrÃ¡ instalar automÃ¡ticamente (aunque esto funciona en cualquier plataforma, el CD-ROM que se crea Ãºnicamente correrÃ¡ bajo Windows).
+WriteCDImagePrompt.Progress_Label:Copiando archivos. Esto podrÃ¡ tomar algÃºn tiempo...
+WriteCDImagePrompt.Successful_Export:Se han exportado las colecciones ({0}).
+WriteCDImagePrompt.Successful_Title:ExportaciÃ³n completa
+WriteCDImagePrompt.Title:Exportar la colecciÃ³n a un CD-ROM
+#************************
+#
+#****** ExportAsPrompt *******
+ExportAsPrompt.Export_Name:Exportar el nombre del directorio
+ExportAsPrompt.Export:Exportar la(s) colecciÃ³n(ciones)
+ExportAsPrompt.Export_Tooltip:Exportar las colecciones seleccionadas en el formato de archivo escogido al directorio designado
+ExportAsPrompt.Failed_ExportOne:La colecciÃ³n, {0}, no pudo exportarse.
+ExportAsPrompt.Failed_ExportMany:Las colecciones -- {0} -- no pudieron exportarse.
+ExportAsPrompt.Failed_Details:Ver detalles en {1}
+ExportAsPrompt.Failed_Title:FallÃ³ la ExportaciÃ³n
+ExportAsPrompt.Instructions:Exportar una o mÃ¡s colecciones en el formato escogido al directorio estipulado
+ExportAsPrompt.Progress_Label:Exportando Archivos. Esto puede tomar algÃºn tiempoâŠ
+ExportAsPrompt.Successful_ExportOne:La colecciÃ³n -- {0} -- ha sido exportada
+ExportAsPrompt.Successful_ExportMany:Las colecciones -- {0} -- han sido exportadas
+ExportAsPrompt.Successful_Details:Archivos exportados a: {1}
+ExportAsPrompt.Successful_Title:ExportaciÃ³n Completada
+ExportAsPrompt.Title:Exportar Colecciones
+ExportAsPrompt.SaveAs:Exportar a
+ExportAsPrompt.SaveAs_Tooltip:Seleccione el formato al cual desea exportar
+#************************
+#
+#*****ExternalCollectionPrompt********
+ExternalCollectionPrompt.NewMDS:Crear un nuevo conjunto vacÃ­o de metadatos
+ExternalCollectionPrompt.ExistingMDS:Elija uno o mÃ¡s de los conjuntos de metadatos
+ExternalCollectionPrompt.Title:Cargar una colecciÃ³n externa
+ExternalCollectionPrompt.Instructions_1:EstÃ¡ a punto de cargar una colecciÃ³n que no ha sido creada por medio de GLI. Se le pedirÃ¡ que seleccione uno o mÃ¡s conjuntos de metadatos para esta colecciÃ³n. Los metadatos existentes se importarÃ¡n al conjunto o los conjuntos seleccionados. Los archivos de metadatos originales se guardarÃ¡n en una carpeta llamada import.bak.
+ExternalCollectionPrompt.Instructions_2:EstÃ¡ a punto de cargar una colecciÃ³n que no ha sido creada por medio de GLI. Necesita aÃ±adir un conjunto de metadatos a esta colecciÃ³n. Usted puede crear un conjunto vacÃ­o de metadatos en el que todos los elementos encontrados en la colecciÃ³n se incluirÃ¡n automÃ¡ticamente. Otra opciÃ³n es seleccionar conjuntos existentes de metadatos y los elementos de la colecciÃ³n se combinarÃ¡n o aÃ±adirÃ¡n a los elementos que ya se encuentran en dichos conjuntos. Los archivos de metadatos originales se guardarÃ¡n en una carpeta llamada import.bak.
+#****************************************
+#
+#***** Inherited Metadata *****
+ExtractedMetadata.Message:El archivo o carpeta seleccionada contiene metadatos extraÃ­dos, los cuales estÃ¡n ocultos. Para verlos vaya al menÃº "[A]rchivo" y seleccione la opciÃ³n "[P]referencias". En la hoja de propiedades "Generales" seleccione "Ver metadatos extraÃ­dos" y haga click en "[A]ceptar".
+ExtractedMetadata.Title:Este archivo/carpeta tiene metadatos extraÃ­dos ocultos
+#
+#***** FileActions *****
+FileActions.Calculating_Size:Calculando el tamaÃ±o total de los archivos seleccionados
+FileActions.Copying:Copiando {0}
+FileActions.Deleting:Eliminando {0}
+FileActions.Directories_Selected:{0} carpetas seleccionadas
+FileActions.Directory_Exists:<html>La carpeta<br>{0}<br>ya existe dentro de la colecciÃ³n. Los archivos se copiarÃ¡n en ella.</html>
+FileActions.Directory_Selected:1 carpeta seleccionada
+FileActions.File_Already_Exists_No_Create:El archivo '{0}' ya existe.
+FileActions.File_Already_Exists_Message:{0}\nYa existe un archivo con este nombre en la\n carpeta de destino y a pesar de que usted ha decidido reemplazarlo la Interfaz de la Biblioteca Digital\n no puede sobreescribirlo. Por favor verifique los permisos de acceso al archivo.
+FileActions.File_Already_Exists_Title:Error - El archivo ya existe.
+FileActions.File_And_Directories_Selected:1 archivo y {0} carpetas seleccionadas
+FileActions.File_And_Directory_Selected:1 archivo y 1 carpeta seleccionadas
+FileActions.File_Create_Error:El archivo {0} no pudo ser creado.
+FileActions.Folder_Create_Error:La carpeta {0} no pudo ser creada.
+FileActions.File_Exists:<html>{0} ya existe en la carpeta de destino. Â¿Desea reemplazarlo?</html>
+FileActions.File_Not_Found_Message:{0}\nNo se encontrÃ³ el archivo mencionado. Una vez que la vista de los archivos\n se haya refrescado, por favor verifique si el archivo aÃºn existe.
+FileActions.File_Not_Found_Title:Error - No se encontrÃ³ el archivo.
+FileActions.File_Selected:1 archivo seleccionado
+FileActions.Files_And_Directory_Selected:{0} archivos y 1 carpeta seleccionados
+FileActions.Files_Selected:{0} archivos seleccionados
+FileActions.Folder_Already_Exists:El nombre de la carpeta '{0}' ya se estÃ¡ usando. No se puede crear la carpeta.
+FileActions.Increase_Depth:Aumentar la profundidad 1 nivel
+FileActions.Insufficient_Space_Message:La acciÃ³n no se pudo realizar debido a que no hay suficiente espacio en el disco.\n Para que el archivo se pueda copiar se requiere de uno\n adicional: {0}
+FileActions.Insufficient_Space_Title:Error - Espacio insuficiente
+FileActions.Moving:Moviendo {0}
+FileActions.No_Activity:No se requiere ninguna acciÃ³n
+FileActions.Possible_Cyclic_Path:{1} tiene una profundidad de {0} directorios y puede ser una ruta recursiva. Â¿Desea continuar?
+FileActions.Read_Only:Los archivos del Ã¡rea de trabajo son de sÃ³lo lectura y no se pueden eliminar.
+FileActions.Selected:{0} archivos y {1} carpetas seleccionadas
+FileActions.Unknown_File_Error_Message:Aunque no se registrÃ³ ningÃºn mensaje de error durante la acciÃ³n\n, la revisiÃ³n final ha mostrado que la copia\n no es igual al archivo original. La razÃ³n\n de esto es desconocida. Por favor revise la media y actualice\n su JVM instalado.
+FileActions.Unknown_File_Error_Title:Error - Error de archivo desconocido
+FileActions.Write_Not_Permitted_Message:La Interfaz de la Biblioteca Digital no tiene permiso para escribir en\n{0}\n Por favor revise los permisos de acceso al archivo.
+FileActions.Write_Not_Permitted_Title:Error - Permisos de acceso al archivo incorrectos
+FileActions.Yes_To_All:Si a todo
+#************************
+#
+#******* File Associations dialog **********
+FileAssociationDialog.Add:AÃ±adir
+FileAssociationDialog.Add_Tooltip:Utilice este programa para ver archivos de este tipo
+FileAssociationDialog.Batch_File:Archivos de procesamiento por lotes
+FileAssociationDialog.Browse:Explorar
+FileAssociationDialog.Browse_Tooltip:Escoge el programa que se usarÃ¡ desde el espacio de archivos de su computadora
+FileAssociationDialog.Browse_Tooltip_Mac:La bÃºsqueda de su aplicaciÃ³n no estÃ¡ activa en este momento para MacOS
+FileAssociationDialog.Browse_Title:Elegir aplicaciÃ³n
+FileAssociationDialog.Close:Cerrar
+FileAssociationDialog.Close_Tooltip:Cerrar este cuadro de diÃ¡logo y regresar a la ventana principal
+FileAssociationDialog.Command:Comando de inicio:
+FileAssociationDialog.Command_File:Archivo de comandos
+FileAssociationDialog.Command_Tooltip:Ingrese el comando para iniciar el programa que se va a usar
+FileAssociationDialog.Details:Detalles de la nueva asociaciÃ³n
+FileAssociationDialog.Executable_File:Archivos ejecutables
+FileAssociationDialog.Existing_Associations:Asociaciones actuales
+FileAssociationDialog.Extension:Para archivos que terminan en:
+FileAssociationDialog.Extension_Tooltip:La extensiÃ³n de los archivos que se van a ver
+FileAssociationDialog.Instructions:Utilice este cuadro de diÃ¡logo para establecer los programas externos que se usarÃ¡n para ver un cierto tipo de archivo y el comando que se enviarÃ¡ para iniciar este programa.\n Para aÃ±adir un nuevo tipo de archivo escriba su extensiÃ³n en âPara archivos que terminan enâ. Para aÃ±adir o modificar un comando de inicio escriba el nuevo comando en âComando de inicioâ o haga click en âExplorarâ para buscar el sistema de archivos de un programa apropiado. Use la cadena de caracteres especiales â%1â para indicar el lugar en el que el nombre del archivo que se abrirÃ¡ deberÃ¡ insertarse, p. ej. "C:\\Windows\\System\\MSPaint.exe %1". '%1' se anexarÃ¡ al final de un comando si no se especifica de otra manera.\n Los usuarios de Mac OS siempre deberÃ¡n intentar primero el comando especÃ­fico para mac "open [-a <application_name>] %1".
+FileAssociationDialog.Table.Command:Comando
+FileAssociationDialog.Table.Extension:ExtensiÃ³n
+FileAssociationDialog.Title:Editar asociaciones de archivos
+FileAssociationDialog.Remove:Eliminar
+FileAssociationDialog.Remove_Tooltip:Elimine la asociaciÃ³n seleccionada del conjunto de asociaciones existentes
+FileAssociationDialog.Replace:Reemplazar
+FileAssociationDialog.Replace_Tooltip:Reemplace la asociaciÃ³n seleccionada con los detalles de la nueva asociaciÃ³n
+#*******************
+#
+#******Filter***************
+Filter.0:HTM y HTML
+Filter.1:XML
+Filter.2:Archivos de texto
+Filter.3:ImÃ¡genes
+Filter.All_Files:Todos los archivos
+Filter.Filter_Tree:Mostrar archivos
+Filter.Invalid_Pattern:<HTML><FONT FACE={FONT} COLOR=Black>El patrÃ³n que usted ha ingresado para filtrar archivos no es vÃ¡lido.<BR> Por favor intÃ©ntelo de nuevo usando la sintaxis de expresiones regulares con<BR>* como comodÃ­n.</FONT></HTML>
+#***********************
+#
+#***** GAuthenticator *****
+GAuthenticator.Password:ContraseÃ±a:
+GAuthenticator.Password_Tooltip:Escriba aquÃ­ su contraseÃ±a
+GAuthenticator.Title:Requiere contraseÃ±a.
+GAuthenticator.Username:Nombre de usuario:
+GAuthenticator.Username_Tooltip:Escriba aquÃ­ su nombre de usuario
+#**************************
+#
+#***** General Messages *****
+# 0 - A String representing the name and method of the class in question.
+General.Apply:Aplicar
+General.Apply_Tooltip:Aplica los ajustes, pero sin salir del cuadro de diÃ¡logo
+General.Browse:Explorar
+General.Cancel:Cancelar
+General.Cancel_Tooltip:Cancela los cambios (no desharÃ¡ ninguno de los cambios que ya se hayan aplicado)
+General.Close:Cerrar
+General.Close_Tooltip:Cierra este cuadro de diÃ¡logo
+General.Edit:Editar valor
+General.Error:Error
+General.No:No
+General.NotSunJava:Proveedor de Java: {0}\nTenga en cuenta que la Interfaz de Bibliotecario ha sido desarrollada y probada con Java de Sun Microsystems
+General.OK:Aceptar
+General.OK_Tooltip:Acepta los cambios actuales y sale del cuadro de diÃ¡logo
+General.Open:Abrir
+General.Outstanding_Processes:<html>La Interfaz de la Biblioteca Digital no se podrÃ¡ cerrar completamente hasta que se haya salido de todos los programas<br>.</html>
+General.Outstanding_Processes_Title:Esperando para salir
+General.Pure_Cancel_Tooltip:Aborta esta acciÃ³n (no se harÃ¡ ningÃºn cambio)
+General.Review_Output:Por favor revise los resultados que estÃ¡n a continuaciÃ³n:
+General.Usage:Uso: {0} <params>\nTodos los parÃ¡metros son opcionales, pero es probable que la Interfaz de la Biblioteca Digital no funcione si -gsdl y -library no se han establecido.\n -gsdl <path>  : Forzar la ruta hacia gsdl\n -library <path> : Ruta hacia cgi-bin\n -mozilla <path> : Habilitada mozilla\n -mirror : Activar los controles para el espejeo de archivos de la web\n -laf [java|windows|motif|mac] : Imagen y apariencia\n -debug : Permite depurar mensajes\n -no_load : No carga una colecciÃ³n abierta previamente\n -load <path> : Carga la colecciÃ³n especificada
+General.View:Ver Valor
+General.Warning:Advertencia
+General.Yes:Si
+#****************************
+#
+#***** GShell *****
+GShell.Build.Auxilary:Creando los archivos auxiliares y poniÃ©ndolos en orden...
+GShell.Build.BuildBegun1:************ Iniciando el proceso de construcciÃ³n de la colecciÃ³n ************
+GShell.Build.BuildCancelled:************** Cancelado **************
+GShell.Build.BuildComplete1:*********** El proceso de construcciÃ³n de la colecciÃ³n ha terminado ***********
+GShell.Build.CompressText:Comprimiendo el texto...
+GShell.Build.Index:Creando un Ã­ndice basado en {0-fuente y nivel del Ã­ndice}...
+GShell.Build.InfoDatabase:Creando una base de datos de informaciÃ³n...
+GShell.Build.Phind:Generando un clasificador Phind...
+GShell.Command:Comando
+GShell.Failure:El comando no se pudo ejecutar.
+GShell.Import.FileNotProcessed:El archivo {0-target file path} fue reconocido pero no pudo ser procesado por ningÃºn plugin.
+GShell.Import.FileNotRecognised:El archivo que se encuentra en {0-ruta del archivo objetivo} no se puede aÃ±adir debido a que su tipo no es reconocible por ninguno de los plugins de Greenstone.
+GShell.Import.FileProcessing:El archivo {0-target file path} estÃ¡ siendo procesado por {1-plugin name}.
+GShell.Import.FileProcessingError:El archivo {0-target file path} generÃ³ un error al ser procesado.
+GShell.Import.ImportBegun1:************ Iniciando el proceso de importaciÃ³n de la colecciÃ³n ************
+GShell.Import.ImportComplete1:*********** El proceso de importaciÃ³n de la colecciÃ³n ha terminado ***********
+GShell.Import.ImportComplete2:{0-nÃºmero de documentos considerados} de los documentos fueron considerados a efecto de ser procesados:
+GShell.Import.ImportComplete2.1:1 documento fue considerado a efecto de ser procesado:
+GShell.Import.ImportComplete.Processed:{0-nÃºmero procesado} se procesaron e incluidos en la colecciÃ³n.
+GShell.Import.ImportComplete.Processed.1:1 fue procesado e incluido en la colecciÃ³n.
+GShell.Import.ImportComplete.Blocked:{0-nÃºmero bloqueado} eran ser aÃ±adidos por los plugins (como por ejemplo las imÃ¡genes de las pÃ¡ginas web).
+GShell.Import.ImportComplete.Blocked.1:1 era ser aÃ±adidos por los plugins (como por ejemplo las imÃ¡genes de las pÃ¡ginas web).
+GShell.Import.ImportComplete.Ignored:{0-nÃºmero ignorado} no eran ser reconocidos por los plugins y son disponibles en la colecciÃ³n.
+GShell.Import.ImportComplete.Ignored.1:1 no era ser reconocidos por los plugins y es disponibles en la colecciÃ³n.
+GShell.Import.ImportComplete.Failed:{0-number failed} rechazados.
+GShell.Import.ImportComplete.Failed.1:1 fue rechazado.
+GShell.Import.ImportComplete3:\n
+GShell.Parsing_Metadata_Complete:ExtracciÃ³n completa de metadatos archivados.
+GShell.Parsing_Metadata_Start:Extrayendo nuevos metadatos de los archivos.
+GShell.Success:Comando completo.
+#*********************
+#
+#***** GUI *****
+# Currently Enabled and Disabled display the same text but in different
+# colours. If necessary this can be changed.
+GUI.Create:Crear
+GUI.Create_Tooltip:Le ayuda a crear su colecciÃ³n por medio del conjunto de programas Greenstone
+GUI.Design:DiseÃ±o
+GUI.Design_Tooltip:DiseÃ±a la funcionalidad y apariencia de su colecciÃ³n
+GUI.Download:Bajar
+GUI.Download_Tooltip:Baja recursos de Internet para su colecciÃ³n
+GUI.Enrich:Enriquecer
+GUI.Enrich_Tooltip:Asigna metadatos a los archivos de su colecciÃ³n
+GUI.Gather:Reunir
+GUI.Gather_Tooltip:Selecciona los archivos que se incluirÃ¡n en su colecciÃ³n
+#***************
+#
+#***** HELP *****
+Help.Contents:Contenido
+Help.Title:PÃ¡ginas de ayuda de la Interfaz de la Biblioteca Digital
+#
+#***** LockFileDialog *****
+LockFileDialog.Cancel_Tooltip:No tome el control del archivo de bloqueo (o la colecciÃ³n no se abrirÃ¡)
+LockFileDialog.Date:Fecha
+LockFileDialog.Error:error
+LockFileDialog.Lockfile_Message_One:La presencia de un archivo de bloqueo sugiere que esta colecciÃ³n estÃ¡ siendo usada por otra persona.
+LockFileDialog.Lockfile_Message_Two:Â¿Desea tomar el control de este archivo de bloqueo?
+LockFileDialog.Machine:Donde
+LockFileDialog.Name:Nombre
+LockFileDialog.OK_Tooltip:Toma el control del archivo de bloqueo
+LockFileDialog.Title:Se detectÃ³ un archivo de bloqueo
+LockFileDialog.User:Quien
+#********************
+#
+#*****MappingPrompt **********
+MappingPrompt.File:Carpeta
+MappingPrompt.Map:Crear un acceso directo
+MappingPrompt.Name:Nombre
+MappingPrompt.Title:Establecer un acceso directo a la carpeta
+MappingPrompt.Unmap:Eliminar acceso directo
+#*******************
+#
+#***** Menu Options *****
+Menu.Collapse:Cerrar carpeta
+Menu.Edit:Editar
+Menu.Edit_Copy:Copiar (ctrl-c)
+Menu.Edit_Cut:Cortar (ctrl-x)
+Menu.Edit_Paste:Pegar (ctrl-v)
+Menu.Expand:Abrir carpeta
+Menu.Explode_Metadata_Database:Explotar la base de datos de metadatos
+Menu.File:Archivo
+Menu.File_Associations:AsociaciÃ³n de archivos...
+Menu.File_CDimage:Exportar a un CD-ROM/DVD...
+Menu.File_Close:Cerrar
+Menu.File_Delete:Suprimir...
+Menu.File_Exit:Salir
+Menu.File_ExportAs:Exportar...
+Menu.File_New:Nueva...
+Menu.File_Open:Abrir...
+Menu.File_Options:Preferencias...
+Menu.File_Save:Guardar
+Menu.Help:Ayuda
+Menu.Help_About:Acerca de...
+Menu.Metadata_View:Metadatos asignados para {0}
+Menu.Open_Externally:Abrir en un programa externo
+#********************
+#
+#***** MetaAudit *****
+MetaAudit.Close:Cerrar
+MetaAudit.Close_Tooltip:Cierra este cuadro de diÃ¡logo
+MetaAudit.Title:Todos los metadatos
+#*********************
+#
+#***** Metadata *****
+Metadata.Element:Elemento
+Metadata.Value:Valor
+MetadataSet.Files:Archivos de conjunto de metadatos
+#*********************************************
+#
+#******** MetadataImportMappingPrompt *********
+MIMP.Add::AÃ±adir
+MIMP.Add_Tooltip:AÃ±ade el elemento de metadato al conjunto seleccionado
+MIMP.Ignore:Ignorar
+MIMP.Ignore_Tooltip:Ignora este elemento de metadato
+MIMP.Instructions:El elemento de metadato {0} no se puede importar automÃ¡ticamente a la colecciÃ³n. Seleccione un conjunto de metadatos y:\n(1) haga click en âAÃ±adirâ para agregar el metadato a ese conjunto (sÃ³lo si el metadato {0} no se encuentra ya en el conjunto), o\n(2) elija un elemento y haga click en âCombinarâ para importar el metadato {0} a este elemento, o\n(3) ignore este elemento de metadato.
+MIMP.Merge:Combinar
+MIMP.Merge_Tooltip:Mapea el elemento de metadato hacia el elemento especificado en el conjunto seleccionado
+MIMP.Source_Element:Elemento de metadatos origen:
+MIMP.Target_Element:Elemento de metadatos destino:
+MIMP.Target_Set:Conjunto de metadatos destino:
+MIMP.Title:Es necesario combinar los elementos.
+#*****************************
+#
+#***** Mirroring *****
+Mirroring.ClearCache:Borrar CachÃ©
+Mirroring.ClearCache_Tooltip:Borrar todos los archivos descargados
+Mirroring.Download:Bajar
+Mirroring.Download_Tooltip:Inicia una nueva descarga
+Mirroring.Download_Controls:Controles para bajar archivos de Internet
+Mirroring.Download_Depth:Profundidad de la descarga:
+Mirroring.Download_Depth.Zero:Cero
+Mirroring.Download_Depth.Unlimited:Ilimitado
+Mirroring.Download_Depth_Tooltip:CuÃ¡ntos hipervÃ­nculos se bajarÃ¡n al momento de descargar los archivos de Internet
+Mirroring.Download_Embedded:Descargar sÃ³lo texto (omitir imÃ¡genes, hojas de estilo, etc.)
+Mirroring.DownloadJob.Close:Cerrar
+Mirroring.DownloadJob.Close_Tooltip:Quitar este job de la lista, deteniÃ©ndolo si todavÃ­a estÃ¡ en ejecuciÃ³n
+Mirroring.DownloadJob.Downloading:Bajando {0}
+Mirroring.DownloadJob.Download_Complete:Descarga completa
+Mirroring.DownloadJob.Download_Stopped:Descarga detenida
+Mirroring.DownloadJob.Download_Progress:Descarga en ejecuciÃ³n
+Mirroring.DownloadJob.Log:Ver BitÃ¡cora
+Mirroring.DownloadJob.Log_Tooltip:Ver detalles acerca de esta descarga
+Mirroring.DownloadJob.Pause:Pausa
+Mirroring.DownloadJob.Pause_Tooltip:Pausa
+Mirroring.DownloadJob.Resume:Reiniciar
+Mirroring.DownloadJob.Resume_Tooltip:Reiniciar esta descarga
+Mirroring.DownloadJob.Status:{0} bajado de {1} archivos ({2} mensajes de advertencia, {3} errores)
+Mirroring.DownloadJob.Waiting:Esperando para iniciar
+Mirroring.DownloadJob.Waiting_User:Esperando instrucciones
+Mirroring.Higher_Directories:Â¿Desea bajar archivos de carpetas que estÃ©n a un nivel mÃ¡s arriba del url dado?
+Mirroring.Invalid_URL:El URL que se ingresÃ³ no es vÃ¡lido. Por favor corrÃ­jalo.
+Mirroring.Invalid_URL_Title:URL invÃ¡lida
+Mirroring.Preferences:Preferencias...
+Mirroring.Preferences_Tooltip:Habilitar proxy y/o editar la configuraciÃ³n del proxy
+Mirroring.Same_Host:SÃ³lo archivos espejo dentro del mismo sitio
+Mirroring.Source_URL:URL fuente:
+Mirroring.Source_URL_Tooltip:Es el URL que se va a bajar
+#*********************
+#
+#***** Missing EXEC *****
+MissingEXEC.Message:Â¡Cuidado! Usted no podrÃ¡ ver sus nuevas colecciÃ³n dentro de la Interfaz de la Biblioteca Digital a menos que especifique la ruta hacia el servidor web de la Biblioteca Greenstone local. Usted puede ingresarla en este momento en el campo de texto que se encuentra abajo o mÃ¡s adelante por medio del cuadro de Preferencias eligiendo la etiqueta de ConexiÃ³n. Recuerde que para que el url sea vÃ¡lido debe empezar con el protocolo "http://" y terminar con el nombre del ejecutable ("gsdl" para el servidor de la biblioteca local Greenstone, "library" para otros servidores web).
+MissingEXEC.Title:Falta la direcciÃ³n de la Biblioteca Greenstone
+#***** Missing EXEC_GS3 *****
+MissingEXEC_GS3.Message:Â¡AtenciÃ³n! No podrÃ¡ ver las nuevas colecciones con la Interfaz de Bibliotecario a menos que especifique la ruta al entorno Greenstone en el servidor local Tomcat. Puede introducirla ahora en el campo de texto situado debajo, o mÃ¡s tarde en Preferencias, pestaÃ±a ConexiÃ³n. Recuerde que un URL vÃ¡lido debe comenzar con el protocolo "http://" La direcciÃ³n predefinida es http://localhost:8080/gsdl3
+MissingEXEC_GS3.Title:No se encuentra la direcciÃ³n de la Biblioteca Greenstone
+#***** Missing GSDL *****
+MissingGSDL.Message:Â¡Cuidado! Usted no podrÃ¡ tener acceso a todas las funciones de la Interfaz de la Biblioteca Digital a menos que una copia de la Biblioteca Digital Greenstone tambiÃ©n estÃ© instalada. Si usted cree que estÃ¡ recibiendo este mensaje por error, por favor revise su instalaciÃ³n y las rutas especificadas en el archivo de guiones y procesamiento por lotes de la Interfaz de la Biblioteca Digital.
+MissingGSDL.Title:Falta la ruta de conexiÃ³n con Greenstone
+#***** Missing PERL *****
+MissingPERL.Message:Â¡Cuidado! Usted no podrÃ¡ crear nuevas colecciones ni construir las que ya existen a menos que tenga el lenguaje de programaciÃ³n PERL instalado y la Interfaz de la Biblioteca Digital no pudo encontrarlo en su sistema. Si usted cree que estÃ¡ recibiendo este mensaje por error, por favor revise su instalaciÃ³n y las rutas especificadas en el archivo de guiones y procesamiento por lotes de la Interfaz de la Biblioteca Digital.
+MissingPERL.Title:Falta la ruta PERL
+#******* MissingWGET ********
+MissingWGET.Message:AtenciÃ³n! Se ha deshabilitado la capacidad de poner la Web en espejo porque el interfaz del 'Librarian' no pudo localizar una versiÃ³n apropiada de Wget. Si usted cree que estÃ¡ recibiendo este mensaje debido a un error, por favor revise su instalaciÃ³n y las rutas especificadas en el archivo batch o script del 'Librarian'
+MissingWGET.Title:Falta Wget
+#******* MissingImageMagick ********
+MissingImageMagick.Message:AtenciÃ³n! La interfaz del bibliotecario fallÃ³ al ubicar un versiÃ³n apropiada de imageMagick y por lo tanto la imagePlug no funcionarÃ¡
+MissingImageMagick.Title:Falta ImageMagick
+#*****************************
+#
+#***** New Session *****
+NewCollectionPrompt.Base_Collection:Construir esta colecciÃ³n con base en:
+NewCollectionPrompt.Base_Collection_Tooltip:Seleccione una colecciÃ³n existente para basar en ella la nueva colecciÃ³n
+NewCollectionPrompt.Collection_Description:DescripciÃ³n de contenidos:
+#NewCollectionPrompt.Collection_Email:Author's Email:
+NewCollectionPrompt.Collection_Name:Carpeta de la colecciÃ³n:
+#NewCollectionPrompt.Collection_Name_Tooltip:Short collection name, used internally (must be 8 characters or less)
+NewCollectionPrompt.Description_Error:El campo de descripciÃ³n debe llenarse. Por favor corrija.
+NewCollectionPrompt.Error:Error en la nueva colecciÃ³n
+NewCollectionPrompt.Instructions:Para crear una nueva colecciÃ³n llene los campos que estÃ¡n a continuaciÃ³n.
+NewCollectionPrompt.Metadata_Elements:Elementos dentro del conjunto seleccionado:
+NewCollectionPrompt.Metadata_Instructions1:Marque los conjuntos de metadatos que se incluirÃ¡n en su colecciÃ³n.
+NewCollectionPrompt.Metadata_Instructions2:(Recuerde: usted puede aÃ±adir otros mÃ¡s adelante).
+NewCollectionPrompt.Metadata_Title_Existing:Seleccionar conjuntos de metadatos
+#NewCollectionPrompt.Name_Error:You have either failed to enter a name for the collection, or the name you have choosen is already in use. Please correct.
+NewCollectionPrompt.NewCollection:-- Nueva colecciÃ³n --
+NewCollectionPrompt.Select_MDS:Conjuntos de metadatos disponibles:
+NewCollectionPrompt.Title:Crear una nueva colecciÃ³n.
+NewCollectionPrompt.Title_Clash:El tÃ­tulo que usted ha elegido para su colecciÃ³n ya se estÃ¡ usando. Â¿Desea continuar?
+NewCollectionPrompt.Title_Error:El tÃ­tulo debe llenarse. Por favor corrija.
+#***** New Folder OrFilePrompt *****
+NewFolderOrFilePrompt.Default_File_Name:postizo
+NewFolderOrFilePrompt.Default_Folder_Name:nueva carpeta
+NewFolderOrFilePrompt.Destination_Name:Carpeta Destino
+NewFolderOrFilePrompt.File_Name:Nombre del Archivo
+NewFolderOrFilePrompt.File_Name_Tooltip:Dar un nombre para el nuevo archivo
+NewFolderOrFilePrompt.Folder_Name:Nombre de la Carpeta
+NewFolderOrFilePrompt.Folder_Name_Tooltip:Dar un nombre para la nueva carpeta
+NewFolderOrFilePrompt.Title_Folder:Crear una nueva carpeta
+NewFolderOrFilePrompt.Title_File:Crear un nuevo documento postizo
+#***********************
+#
+NoMetadataSetsSelected.Message:Si usted crea la colecciÃ³n en este momento no tendrÃ¡ un conjunto inicial de metadatos y no podrÃ¡ importar o asignar ningÃºn metadato hasta que un conjunto se importe o aÃ±ada a su colecciÃ³n. Â¿Desea continuar?
+NoMetadataSetsSelected.Title:No se ha seleccionado ningÃºn conjunto de metadatos
+#**********************
+#
+#******* OldWGET *******
+NoPluginExpectedToProcessFile.Message:No se encuentra ningÃºn plugin de Greenstone que pueda procesar el archivo "{0}". Compruebe que el archivo tiene la extensiÃ³n correcta. Si la extensiÃ³n es correcta entonces tendrÃ¡ que usar el plugin Unkowm para procesar este archivo.
+NoPluginExpectedToProcessFile.Title:No se encuentra un plugin que pueda procesar el archivo
+#**********************
+#
+#******* OldWGET *******
+OldWGET.Message:AtenciÃ³n! La versiÃ³n de Wget que ha instalado tiene un defecto conocido cuando se manejan archivos espejo que contienen caracteres codificados tales como espacio y tilde en su URL. Aunque este no debe ser un problema, recomendamos instalar Wget 1.9 o posterior
+OldWGET.Title:Wget viejo
+#***********************
+#
+#***** OpenCollectionDialog *****
+OpenCollectionDialog.Available_Collections:Colecciones disponibles
+OpenCollectionDialog.Browse:Explorar
+OpenCollectionDialog.Browse_Tooltip:Elija una colecciÃ³n del espacio de archivos de su computadora
+OpenCollectionDialog.Collection:ColecciÃ³n Greenstone (".col")
+OpenCollectionDialog.Description:DescripciÃ³n de la colecciÃ³n
+OpenCollectionDialog.Locked_Collection:La colecciÃ³n Greenstone ya se estÃ¡ usando
+OpenCollectionDialog.No_Description:Por favor seleccione una colecciÃ³n Greenstone (archivo ".col") para ver su descripciÃ³n.
+OpenCollectionDialog.Normal_Collection:ColecciÃ³n Greenstone
+OpenCollectionDialog.Open:Abrir
+OpenCollectionDialog.Open_Tooltip:Carga la colecciÃ³n seleccionada en la Interfaz de la Biblioteca Digital
+OpenCollectionDialog.Title:Abrir la colecciÃ³n Greenstone
+#**********************
+#
+#***** Options Pane inside CreatePane *****
+OptionsPane.Cancelled:- cancelada
+OptionsPane.LogHistory:Historial de registro
+OptionsPane.Successful:- con Ã©xito
+OptionsPane.Unknown:- desconocido
+OptionsPane.Unsuccessful:- sin Ã©xito
+#*******************
+#
+#***** Preferences *****
+Preferences:Preferencias
+Preferences.Connection:ConexiÃ³n
+Preferences.Connection_Tooltip:Escriba el URL de la Biblioteca Greenstone y configure una conexiÃ³n proxy segÃºn sea necesario
+Preferences.Connection.Library_Path:Ruta de la biblioteca
+Preferences.Connection.Library_Path_GS3:Ruta Web de Greenstone:
+Preferences.Connection.Library_Path_Tooltip:Es el URL del servidor Web Greenstone
+Preferences.Connection.Library_Path_Tooltip_GS3:URL del servidor web de Greenstone
+Preferences.Connection.ProgramCommand:Comando de vista previa:
+Preferences.Connection.ProgramCommand_Tooltip:Es el comando de inicio del programa usado para tener una vista previa de la colecciÃ³n. Recuerde usar %1 para indicar el lugar donde deberÃ¡ insertarse la direcciÃ³n de la colecciÃ³n y usar las marcas de habla donde sea necesario.
+Preferences.Connection.Proxy_Host:Servidor proxy
+Preferences.Connection.Proxy_Host_Tooltip:Es la direcciÃ³n del servidor proxy
+Preferences.Connection.Proxy_Port:Puerto para el servidor proxy
+Preferences.Connection.Proxy_Port_Tooltip:Es el nÃºmero de puerto del servidor proxy
+Preferences.Connection.Servlet:Servlet:
+Preferences.Connection.Servlet_Tooltip:Servlet a utilizar para ver su sitio
+Preferences.Connection.Servlet_Tooltip2:No se ha especificado 'servlet' para este sitio, es posible construir colecciones, pero no podrÃ¡ verlas
+Preferences.Connection.Site:Sitio:
+Preferences.Connection.Site_Tooltip:El sitio en el cual quiere trabajar
+Preferences.Connection.Use_Proxy:Â¿Desea usar la conexiÃ³n del servidor proxy?
+Preferences.General:General
+Preferences.General_Tooltip:Determine algunas preferencias generales
+Preferences.General.Email:E-mail del usuario:
+Preferences.General.Email_Tooltip:La direcciÃ³n de correo electrÃ³nico usada automÃ¡ticamente para cualquier colecciÃ³n creada por medio de GLI.
+Preferences.General.Font:Fuente:
+Preferences.General.Font_Tooltip:La fuente utilizada para el interfaz GLI.
+Preferences.General.Interface_Language:Idioma de la interfaz:
+Preferences.General.Interface_Language_Tooltip:Seleccione el idioma en el que usted desea que aparezcan los controles y la interfaz.
+Preferences.General.Restart_Required:Se debe reiniciar GLI para completar la actualizaciÃ³n de su interfaz.
+Preferences.General.Show_File_Size:Mostrar el tamaÃ±o de los archivos
+Preferences.General.Show_File_Size_Tooltip:Muestra el tamaÃ±o de los archivos junto a cada una de las entradas de las estructuras de archivos en el Ã¡rea de trabajo y en la colecciÃ³n.
+Preferences.General.View_Extracted_Metadata:Ver metadatos extraÃ­dos
+Preferences.General.View_Extracted_Metadata_Tooltip:Muestra los metadatos extraÃ­dos por los plugins de Greenstone en la lista de metadatos de la pantalla Enriquecer
+Preferences.Mode:Modo
+Preferences.Mode_Tooltip:Haga click en una entrada para seleccionar el nivel de operaciÃ³n
+Preferences.Mode.Assistant:Asistente de biblioteca
+Preferences.Mode.Assistant_Description:Utilice esta opciÃ³n para tener acceso a las funciones bÃ¡sicas de la Interfaz de la Biblioteca Digital Greenstone: aÃ±adir documentos y metadatos a las colecciones, crear nuevas colecciones cuya estructura refleje la de colecciones que ya existen y construir colecciones.
+Preferences.Mode.Librarian:Bibliotecario
+Preferences.Mode.Librarian_Description:Este control le da un uso normal de la Interfaz de la Biblioteca Digital Greenstone: crear y diseÃ±ar nuevas colecciones, aÃ±adir documentos y metadatos a las colecciones, diseÃ±ar nuevas colecciones y construir colecciones.
+Preferences.Mode.Systems:Especialista en Sistemas de Bibliotecas
+Preferences.Mode.Systems_Description:Esta opciÃ³n permite el uso completo de la Interfaz de la Biblioteca Digital Greenstone. Para aprovechar todas las funciones disponibles usted deberÃ¡ ser capaz de formular expresiones regulares.
+Preferences.Mode.Expert:Avanzado
+Preferences.Mode.Expert_Description:Este control permite el uso avanzado de la Interfaz de la Biblioteca Digital Greenstone. Se recomienda para usuarios que tienen experiencia con Greenstone y son capaces de resolver problemas que impliquen interpretar la depuraciÃ³n de los resultados de los guiones PERL: Todas las funciones de la Biblioteca Digital estÃ¡n activas.
+Preferences.Warnings:Mensajes de advertencia
+Preferences.Warnings_Tooltip:Activa o desactiva los distintos mensajes de advertencia
+Preferences.Workflow:Flujo de trabajo
+Preferences.Workflow_Tooltip:DiseÃ±e el flujo de trabajo de la interfaz: determine las pantallas que estarÃ¡n activas
+Preferences.Workflow.Create:Crear - Construye la colecciÃ³n por medio del conjunto de programas Greenstone
+Preferences.Workflow.Design:DiseÃ±o - Cambia la apariencia de la colecciÃ³n
+Preferences.Workflow.Download:Bajar - Baja recursos de la Web
+Preferences.Workflow.Enrich:Enriquecer - Enriquece los documentos con metadatos
+Preferences.Workflow.Gather:Reunir - Selecciona los archivos para formar la colecciÃ³n
+Preferences.Workflow.Predefined.Label:Flujos de trabajo predeterminados
+Preferences.Workflow.Title:Marque el cuadro para activar la pantalla
+#**********************
+#
+#**** PreviewCommandDialog *******
+PreviewCommandDialog.Instructions:Ingrese el comando para abrir el programa a fin de tener una vista previa de la colecciÃ³n. Recuerde usar %1 para indicar el lugar donde deberÃ¡ insertarse la direcciÃ³n de la colecciÃ³n y usar las marcas de habla donde sea necesario.
+PreviewCommandDialog.Title:Comando de vista previa
+#**********************
+#
+#***** Save Collection Box *****
+SaveCollectionBox.File_Name:Nombre del archivo:
+SaveCollectionBox.Files_Of_Type:Archivos de tipo:
+SaveCollectionBox.Look_In:Visita rÃ¡pida:
+#**********************
+#
+#***** Save Progress Box *******
+SaveProgressDialog.Title:Guardando la colecciÃ³n
+#*******************************
+#
+Server.QuitManual:La Interfaz de la Biblioteca Digital no pudo cerrar el servidor local\n de forma automÃ¡tica. Por favor salga manualmente del servidor\n haciendo click en el icono en forma de tache o cruz que se encuentra en el Ã¡ngulo superior\n derecho de la pantalla y luego haga click en Aceptar.
+Server.QuitTimeOut:La biblioteca local Greenstone no ha respondido durante los Ãºltimos sesenta segundos a la\n instrucciÃ³n de cerrar. Â¿Desea\n esperar otro minuto?
+#
+#***** Sources *****
+Source.General:General
+#*******************
+#
+#***** TitleClash *****
+TitleClashes.Message:El tÃ­tulo que usted ha elegido para su colecciÃ³n ya se estÃ¡ usando. Â¿Desea continuar?
+TitleClashes.Title:Este tÃ­tulo ya existe
+#**********************
+#
+#***** Trees *****
+Tree.DownloadedFiles:Archivos Descargados
+Tree.Files:archivos
+Tree.Home:Carpeta principal ({0})
+Tree.Root:Espacio de archivos local
+Tree.World:Documentos en las Colecciones Greenstone
+#***********************
+#
+#***** Warning Dialog *****
+WarningDialog.Dont_Show_Again:No mostrar este aviso otra vez.
+WarningDialog.Dont_Show_Again_Message:No mostrar este mensaje otra vez
+WarningDialog.Invalid_Value:El valor que usted ha ingresado en el campo de valores\n no es vÃ¡lido para esta propiedad.
+WarningDialog.Value:Ingrese el valor:
+#
+#****** WGet *****
+WGet.Prompt:Cliente de Espejeo Wget
+#
+#****** Workflows *****
+Workflow.AllPanels:Todos los paneles
+Workflow.LocalSourceCollectionBuilding:Construyendo la colecciÃ³n de fuente local
+Workflow.DocumentEnriching:Enriqueciendo el documento
+
+GEMS.Already_Loaded_Set_Error:El conjunto de metadatos seleccionado ya estÃ¡ abierto.
+GEMS.Element_Already_Exists:Ya existe un elemento con ese nombre
+GEMS.Set_Already_Exists:Un conjunto con este namespace ya existe
+GEMS.Cannot_Delete_Core_Sets_Error:No puede borrar un conjunto de metadatos fundamental de GLI.
+
+GEMS.Preferences.Selected_Languages_Tooltip:Seleccionar los lenguajes a mostrar:
+
+GEMS.Add_Attribute:AÃ±adir atributo
+GEMS.Add_Attribute_Tooltip:Le aÃ±ade un nuevo atributo al elemento seleccionado
+GEMS.Add_Element:AÃ±adir elemento
+GEMS.Add_Element_Tooltip:Le aÃ±ade un nuevo elemento al conjunto seleccionado
+GEMS.Add_Subelement:Agregar subelemento
+GEMS.Add_Subelement_Tooltip:Agregar un  nuevo subelemento al elemento seleccionado
+GEMS.Add_Set:AÃ±adir conjunto
+GEMS.Attribute:Atributo
+GEMS.Attribute_Already_Exists:Este atributo ya existe.
+GEMS.Attribute_Name_Tooltip:Es el nombre del nuevo atributo
+GEMS.Attribute_Language_Tooltip:Es el idioma del nuevo atributo
+GEMS.Attribute_Value_Tooltip:Es el valor del nuevo atributo
+GEMS.Attributes:Atributos
+GEMS.Cannot_Undo:(esta acciÃ³n no se puede deshacer)
+GEMS.Cannot_Add_Elements_To_Greenstone_MDS:No se pueden aÃ±adir manualmente elementos a los Metadatos Greenstone. En caso\n necesario, construya un nuevo conjunto y aÃ±Ã¡dale el nuevo elemento.
+GEMS.Confirm_Exit_Save:Â¿Deasea salvar antes de salir?
+GEMS.Confirm_Exit_Save_Title:Â¿Salvar Antes de Salir?
+GEMS.Confirm_Removal:Â¿EstÃ¡ seguro de que desea eliminar este {0}?
+GEMS.Confirm_Removal_Title:Â¿EstÃ¡ seguro de querer eliminarlo?
+GEMS.Edit_Attribute:Editar atributo
+GEMS.Edit_Value:Editar valor
+GEMS.Edit_Value_Tooltip:Cambia el valor seleccionado
+GEMS.Element:Elemento
+GEMS.Element_Name_Tooltip:Es el nombre del nuevo elemento
+GEMS.Element_Details:Detalles del elemento de metadato
+GEMS.Language:Idioma
+GEMS.Language_Code:Idioma
+GEMS.Name:Nombre
+GEMS.Namespace:Indicador del conjunto
+GEMS.New_Name:Nuevo Nombre
+GEMS.No_Description:No se encontrÃ³ la descripciÃ³n.
+GEMS.No_Name:No se encontrÃ³ el nombre.
+GEMS.Open_Set:Conjunto Abierto
+GEMS.Remove_Attribute:Remover atributo
+GEMS.Remove_Attribute_Tooltip:Elimina el atributo seleccionado
+GEMS.Remove_Element:Remover elemento
+GEMS.Remove_Element_Tooltip:Elimina el elemento seleccionado del conjunto
+GEMS.Remove_Set:Remover conjunto
+GEMS.Remove_Set_Tooltip:Elimina el conjunto seleccionado
+GEMS.Rename_Element:Renombrar elemento
+GEMS.Rename_Element_Tooltip:Nuevo nombre del elemento
+GEMS.Rename_Name_Tooltip:Nuevo nombre del conjunto de metadatos
+GEMS.Rename_Namespace_Tooltip:Espacio de nombres Ãºnico del conjunto de metadatos
+GEMS.Rename_Set:Renombrar conjunto
+GEMS.Rename_Set_Tooltip:Renombrar el conjunto de metadatos
+GEMS.Set:Conjunto
+GEMS.Set_Details:Detalles del conjunto de metadatos
+GEMS.Set_Name_Tooltip:Es el nombre del nuevo conjunto de metadatos
+GEMS.Set_Namespace_Tooltip:Es un indicador Ãºnico para el nuevo conjunto de metadatos
+GEMS.Subelement_Name_Tooltip:Nombre del nuevo subelemento
+GEMS.Subelement_Already_Exists:Ya existe un subelemento con este nombre.
+GEMS.Target:Elemento
+GEMS.Title:Editar conjuntos de metadatos
+GEMS.Value:Valor
+GEMS.Values:Valores
+GEMS.inheritMetadataSet:Heredar del conjunto de metadatos:
+GEMS.Add_Set.No_Inherit:No heredar
Index: /main/tags/3.01/gli/classes/dictionary_lv.properties
===================================================================
--- /main/tags/3.01/gli/classes/dictionary_lv.properties	(revision 21112)
+++ /main/tags/3.01/gli/classes/dictionary_lv.properties	(revision 21112)
@@ -0,0 +1,1022 @@
+#*******************
+#
+#***** AboutDialog *****
+AboutDialog.Acknowledgement:PateicÄ«bas par programmÄm, paketÄm vai klasÄm...
+AboutDialog.Date:(Oktobris 2005)
+AboutDialog.Item0:Xerces Java 1' no The Apache Software Foundation (http://www.apache.org/)
+AboutDialog.Item2:WGet' no Free Software Foundation, Inc. (http://www.gnu.org/software/wget/wget.html)
+AboutDialog.Item3:qflib' no Quality First Software GmbH. (http://www.qfs.de/en/qflib/index.html)
+AboutDialog.Item4:ExtensionFileFilter.class'  Bruno Dumon un lÄ«dzstrÄdnieki (http://pollo.sourceforge.net/)
+AboutDialog.Item5:KÄ aizliegt datnes pÄrdÄvÄÅ¡anu datnes izvÄlnÄ' - vladi21 (http://www.experts-exchange.com/)
+AboutDialog.Item6:Autoscrolling' - Java Swing pateicoties Robert Eckstein, Marc Loy & Dave Wood (http://www.oreilly.com/catalog/jswing/chapter/dnd.beta.pdf)
+AboutDialog.Item7:Add ghosted drag images to your JTrees' - pateicoties Andrew J. Armstrong (http://www.javaworld.com/javaworld/javatips/jw-javatip114.html)
+AboutDialog.Item8:NovÄrsta nepatÄ«kama java.prefs kÄŒÅ«da, ja jums nav administrÄtora pieejas' - pateicoties Walter Schatz
+AboutDialog.Java_Req:Java ir Sun Microsystems Inc. firmas zÄ«me. JRE tiek distribÅ«tÄta pie Papildus Licences NosacÄ«jumiem (Supplemental License Terms), kura atrodama uz http://java.sun.com/j2se/1.4.1/j2re-1_4_1_02-license.html
+AboutDialog.Java_Req_One:Å is produkts iekÄŒauj kodus licencÄtas RSA Security, Inc.
+AboutDialog.Java_Req_Two:DaÅŸas programmatÅ«ras daÄŒas ir IBM licencÄtas un pieejamas uz http\://oss.software.ibm.com/icu4j/
+AboutDialog.Thanks:ArÄ« pateicamies tiem, kuri publicÄja noderÄ«gu informÄciju par java uz web...
+AboutDialog.Title:Par...
+AboutDialog.Title_One:Greenstone BibliotekÄra Interfeiss (GBI)
+AboutDialog.Title_Two:Greenstone DigitÄlÄ BibliotÄka, Waikato UniversitÄte
+AboutDialog.Copyright:Copyright (C) 2003 New Zealand Digital Library Project
+AboutDialog.Copyright_Two:PublicÄts zem GPL. VairÄk skatÄ«t LICENSE.txt.
+#***********************
+#
+#***** Audit Table *****
+AuditTable.File:Datne
+#***********************
+#
+#***** Auto Filter *****
+Autofilter.AND:UN
+Autofilter.Ascending:PieaugoÅ¡s
+Autofilter.Case_Sensitive:SakritÄ«bas GadÄ«jums
+Autofilter.Custom_Filter:Uzlabots
+Autofilter.Descending:DilstoÅ¡s
+Autofilter.Filter_By_Value:VienkÄrÅ¡s
+Autofilter.Name:Kolonnas VÄrds
+Autofilter.None:NekÄds
+Autofilter.Operator:NosacÄ«jums
+Autofilter.OR:VAI
+Autofilter.Order:SakÄrtoÅ¡ana
+Autofilter.Order_Tooltip:SakÄrtot sakrÄ«toÅ¡os elementus Å¡ÄdÄ secÄ«bÄ
+Autofilter.Remove:NoÅemt Filtru
+Autofilter.Remove_Tooltip:NoÅemt Å¡o filtru no tabulas kolonnas
+Autofilter.Set:Uzdot Filtru
+Autofilter.Set_Tooltip:Pievienot Å¡o filtru tabulas kolonnai
+Autofilter.Table_Header_Tooltip:FiltrÄt pÄc {0}
+Autofilter.Title:Metadatu Autofiltrs
+Autofilter.eqeq:vienlÄ«dzÄ«gs
+Autofilter.!eq:does not equal
+Autofilter.<:is less than
+Autofilter.<eq:is less than or equal to
+Autofilter.>:is greater than
+Autofilter.>eq:is greater than or equal to
+Autofilter.^:starts with
+Autofilter.!^:does not start with
+Autofilter.$:ends with
+Autofilter.!$:does not end with
+Autofilter.?:contains
+Autofilter.!?:does not contain
+#
+#     ********************************************************
+#     *************** Collection Design Module ***************
+#     ********************************************************
+#     * The dictionary  phrases  used in the creation of the *
+#     * Colection  Design  Module. Note that we  still  have *
+#     * access  to the format  arguments (argument  32+) for *
+#     * html formatting.                                     *
+#     ********************************************************
+CDM.ArgumentConfiguration.Add:Pievienot
+CDM.ArgumentConfiguration.Bad_Integer:<html>Arguments '{0}' prasa veselu skaitli,<br>bet ne vesels skaitlis var bÅ«t analizÄts no {1}.</html>
+CDM.ArgumentConfiguration.Custom_Header:IestatÄ«Å¡ana
+CDM.ArgumentConfiguration.Custom:IestatÄ«Å¡anas Argumenti:
+CDM.ArgumentConfiguration.Error_Title:KÄŒÅ«da Argumentos
+CDM.ArgumentConfiguration.Header:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>LÅ«dzu konfigurÄt argumentus priekÅ¡ {0}.</CENTER></FONT></STRONG></HTML>
+CDM.ArgumentConfiguration.No_Value:Arguments '{0}'  prasa vÄrtÄ«bu, bet tÄ nav norÄdÄ«ta.
+CDM.ArgumentConfiguration.Remove:NoÅemt
+CDM.ArgumentConfiguration.Required_Argument:Argumentam '{0}' jÄbÅ«t norÄdÄ«ta vÄrtÄ«ba.
+CDM.ArgumentConfiguration.Title:Argumentu KonfigurÄÅ¡ana
+CDM.ClassifierManager.Add:Pievienot Klasifikatoru
+CDM.ClassifierManager.Add_Tooltip:Lietot norÄdÄ«to klasifikatoru ar Å¡o kolekciju
+CDM.ClassifierManager.Assigned:PaÅ¡laik PieÅ¡Ä·irtie Klasifikatori
+CDM.ClassifierManager.Classifier:IzvÄlieties klsifikatoru, ko pievienot:
+CDM.ClassifierManager.Classifier_Tooltip:Klasifikatori pieejami lietoÅ¡anai kolekcijÄ
+CDM.ClassifierManager.Classifier_Exists:Å is klasifikators ir jau nozÄ«mÄts Å¡ai kolekcijai.
+CDM.ClassifierManager.Classifier_Str:klasifikators
+CDM.ClassifierManager.Classifier_XML_Parse_Failed:NespÄju noteikt argumentus {0}\nklasifikatoram. LÅ«dzu pÄrliecinieties, ka klasifikators\nnodroÅ¡ina -xml karodziÅu, palaiÅŸot classinfo.pl skriptu\nlÄ«dzÄ«gi, ka skripta pluginfo.pl gadÄ«jumÄ,\nkÄ aprakstÄ«ts dokumentÄ Greenstone Developers Guide sekcijÄ 2.1.
+CDM.ClassifierManager.Configure:KonfigurÄt Klasifikatoru
+CDM.ClassifierManager.Configure_Tooltip:IzvÄlÄtÄ klasifikÄtora opciju maiÅa
+CDM.ClassifierManager.Controls:RediÄ£ÄÅ¡anas Kontrole
+CDM.ClassifierManager.Instructions:Izmantojiet Å¡o paneli lai pevienotu, noÅemtu vai konfigurÄtu klasifikatorus jÅ«su kolekcijÄ. Lai pievienotu klasifikatoru, izvÄlieties to no kombinÄtÄ saraksta\n un noklikÅ¡Ä·iniet 'Pievienot Klasifikatoru'. Lai noÅemtu klasifikatoru, izvÄlieties to no nozÄ«mÄto klasifikatoru\n saraksta un noklikÅ¡Ä·iniet 'NoÅemt Klasifikatoru'.\nLai konfigurÄtu klasifikatoru, izvÄlieties to no nozÄ«mÄto klasifikatoru saraksta, un noklikÅ¡Ä·iniet 'KonfigurÄt Klasifikatoru'.\n\nKad jÅ«s konfigurÄjiet klasifikatoru, tad parÄdÄs rullslejas argumentu saraksts. \nIzvÄlieties (atÄŒaujot) argumentus no saraksta, kuriem jÅ«s vÄlaties ievadÄ«t vai izvÄlÄties nepiecieÅ¡amÄs vÄrtÄ«bas.
+CDM.ClassifierManager.Remove:NoÅemt Klasifikatoru
+CDM.ClassifierManager.Remove_Tooltip:NoÅemt izvÄlÄto klasifikatoru no saraksta
+CDM.ClassifierManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Klasifikatora IzvÄle & KonfigurÄÅ¡ana</CENTER></FONT></STRONG></HTML>
+CDM.ClassifierManager.Parsing.Message:Dizaina rÄ«ks ielÄdÄ klasifikatoru informÄciju.
+CDM.ClassifierManager.Parsing.Title:Klasifikatoru IelÄdÄÅ¡ana
+CDM.FormatManager.Add:Pievienot FormÄtu
+CDM.FormatManager.Add_Tooltip:Izmantot specificÄto formÄta komandu ar kolekciju
+CDM.FormatManager.Assigned_Formats:PaÅ¡laik NozÄ«mÄtÄs FormÄta Komandas
+CDM.FormatManager.Editor:HTML FormÄta Rinda
+CDM.FormatManager.Editor_Tooltip:Ievadiet formÄta komandu Å¡eit
+CDM.FormatManager.Editing_Controls:RediÄ£ÄÅ¡anas Kontrole
+CDM.FormatManager.Enabled:AtÄŒauts
+CDM.FormatManager.Feature:IzvÄlÄties PazÄ«mi
+CDM.FormatManager.Feature_Tooltip:PazÄ«me kuru formatÄt
+CDM.FormatManager.Insert:Iespraust
+CDM.FormatManager.Insert_Tooltip:Iespraust izvÄlÄto mainÄ«go formÄta komandÄ
+CDM.FormatManager.Instructions:Web lappas, kuras jÅ«s redzat izmantojot Greenstone, nav iepriekÅ¡ saglabÄtas, bet tiek Ä£enerÄtas 'lidojumÄ', t.i., to pieprasÄ«Å¡anas brÄ«dÄ«. FormÄta komandas tiek izmantotas, lai izmainÄ«tu Ä£enerÄto lapu izskatu. DaÅŸi ir pÄrslÄgi (switches), kuri kontrolÄ dokumenta vai tÄ daÄŒas atrÄdÄ«Å¡anu; citi ir vairÄk komplicÄti un prasa html kodus kÄ argumentus.\nLai pievienotu formÄta komandu, izvÄlieties to no âpazÄ«mjuâ saraksta. Ja Äªsts/NeÄ«sts (True/False) opciju panelis parÄdÄs, tad izvÄlieties stÄvokli noklikÅ¡Ä·inot atbilstoÅ¡o pogu. Ja parÄdÄs panelis saturoÅ¡s kontroles âIetekmÄtÄ PazÄ«meâ un âRediÄ£Ät FormÄta Rindasâ, jums nepiecieÅ¡ams ievadÄ«t html tekstu. JÅ«s varat iekÄŒaut esoÅ¡os mainÄ«gos izvÄloties tos no saraksta.\n Kad jÅ«s esat veicis jÅ«su izmaiÅas, nospiediet âPievienot FormÄtuâ, lai to pievienotu konfigurÄcijas datnÄ (fails).\n NoÅemiet formÄta komandu, izvÄloties to no saraksta âNozÄ«mÄtÄs FormÄta Komandasâ un noklikÅ¡Ä·inot uz âNoÅemt FormÄtuâ pogas.
+CDM.FormatManager.Part:IetekmÄtÄ Komponente
+CDM.FormatManager.Part_Tooltip:IzvÄlÄtÄs pazÄ«mes aspekts, ko formatÄt
+CDM.FormatManager.Remove:NoÅemt FormÄtu
+CDM.FormatManager.Remove_Tooltip:NoÅemt izvÄlÄto formÄta komandu no saraksta
+CDM.FormatManager.Replace:NomainÄ«t FormÄtu
+CDM.FormatManager.Replace_Tooltip:AktualizÄt izvÄlÄto formÄta komandu
+CDM.FormatManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>FormÄta Komandas</CENTER></FONT></STRONG></HTML>
+CDM.FormatManager.Variable:MainÄ«gie
+CDM.FormatManager.Variable_Tooltip:MainÄ«gie pieejami izmantoÅ¡anai formÄta komandÄ
+CDM.General.Access:Å ai kolekcijai jÄbÅ«t publiski pieejamai
+CDM.General.Beta:Å Ä« kolekcija tiek vÄl konstruÄta
+CDM.General.Browser_Title:IzvÄlieties AttÄlu
+CDM.General.Collection_Extra:Kolekcijas apraksts:
+CDM.General.Collection_Extra_Tooltip:Kolekcijas satura un nodoma apraksts
+CDM.General.Collection_Name:Kolekcijas nosaukums:
+CDM.General.Collection_Name_Tooltip:Kolekcijas nosaukums
+CDM.General.Copy_Image:Lai Å¡o attÄlu (ikonu) izmantotu jÅ«su kolekcijÄ\ndatnÄm (failiem) jÄbÅ«t nokopÄtÄm images mapÄ. IzpildÄ«t?
+CDM.General.Email.Creator:Autora epasts
+CDM.General.Email.Creator_Tooltip:Kolekcijas autora e-pasta adrese
+CDM.General.Email.Maintainer:AtbalstÄ«tÄja epasts
+CDM.General.Email.Maintainer_Tooltip:Kolekcijas atbalstÄ«tÄja (apkalpotÄja) e-patsta adrese
+CDM.General.Icon_Collection:URL uz 'Par' lapas ikonu:
+CDM.General.Icon_Collection_Tooltip:URL uz attÄlu (ikonu), ko attÄlot kolekcijas 'Par' lapÄ
+CDM.General.Icon_Collection_Small:URL uz 'mÄjas' lapas ikonu:
+CDM.General.Icon_Collection_Small_Tooltip:URL uz attÄlu (ikonu), ko attÄlot bibliotÄkas mÄjas lapÄ
+CDM.General.Image_Copy_Failed:AtgadÄ«jÄs kÄŒÅ«da, kura neÄŒauj jÅ«su izvÄlÄto attÄlu(s) nokopÄt.\nLÅ«dzu mÄÄ£iniet nokopÄt attÄlu(s) uz jÅ«su kolekcijas images mapi manuÄli.
+CDM.General.Image_Filter:AttÄli (gifs, jpgs un pngs)
+CDM.General.Instructions:BibliotekÄra Interfeisa dizaina sekcija ÄŒauj jums kontrolÄt daudzus jÅ«su kolekcijas izskata aspektus. Dizains is sadalÄ«ts daudzÄs atseviÅ¡Ä·Äs sekcijÄs. Å Ä« sekcija satur vispÄrÄ«gas opcijas un iestatÄ«jumus. Lai izvÄlÄtos citu sekciju, noklikÅ¡Ä·iniet uz tÄs nosaukuma sarakstÄ pa kreisi.
+CDM.General.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>VispÄrÄ«gas Opcijas</CENTER></FONT></STRONG></HTML>
+CDM.GUI.Classifiers:PÄrlÅ«koÅ¡anas Klasifikatori
+CDM.GUI.Design_Topics:Dezaina Sekcijas
+CDM.GUI.Formats:FormatizÄÅ¡ana
+CDM.GUI.General:VispÄrÄ«gi
+CDM.GUI.Indexes:MeklÄÅ¡anas Indeksi
+CDM.GUI.MetadataSets:Metadatu Kopas
+CDM.GUI.Plugins:Dokumentu SpraudÅi
+CDM.GUI.Root:CDM_ROOT
+CDM.GUI.SearchTypes:MeklÄÅ¡anas Tipi
+CDM.GUI.Subcollections:SadalÄ«jumu Indeksi
+CDM.GUI.SuperCollection:Starp-Kolekciju MeklÄÅ¡ana
+CDM.GUI.Translation:Teksta TulkoÅ¡ana
+CDM.IndexManager.Add_Index:Pievienot Indeksu
+CDM.IndexManager.Add_Index_Tooltip:Izveidot specifisku indeksu Å¡ai kolekcijai
+CDM.IndexManager.Default_Index_Indicator:[NoklusÄjuma Indeks]
+CDM.IndexManager.Document:dokuments
+CDM.IndexManager.Index_Exists:Å is indeks esot jau bijis nozÄ«mÄts dotai kolekcijai.
+CDM.IndexManager.Index_Name:Indeksa VÄrds:
+CDM.IndexManager.Index_Name_Tooltip:UnikÄls nosaukums jaunam indeksam
+CDM.IndexManager.Indexes:NozÄ«mÄtie Indeksi
+CDM.IndexManager.Instructions:Å eit jÅ«s izvÄlÄties, kÄdi meklÄÅ¡anas indeksi bÅ«s kolekcijai.\nLai pievienotu jaunu indeksu, izvÄlieties, kÄds materiÄls tiks indeksÄts, indeksa lÄ«meni, un noklikÅ¡Ä·iniet âPievienot Indeksuâ.\nLai noÅemtu indeksu, izvÄlieties to no nozÄ«mÄto indeksu saraksta un noklikÅ¡Ä·iniet âNoÅemt Indeksuâ.\nLai uzstÄdÄ«tu indeksu pÄc noklusÄÅ¡anas, izvÄlieties to no nozÄ«mÄto indeksu saraksta un noklikÅ¡Ä·iniet âUzdot NoklusÄjuma Indeksuâ.
+CDM.IndexManager.Level:LÄ«menÄ«:
+CDM.IndexManager.Level_Exists:Å is lÄ«menis jau esot ticis nozÄ«mÄts dotai kolekcijai.
+CDM.IndexManager.Level_Tooltip:Dokumenta lÄ«menis uz kura veidot indeksu
+CDM.IndexManager.MGPP.Add_All_Metadata:Pievienot Visu
+CDM.IndexManager.MGPP.Add_All_Metadata_Tooltip:Izveidot atseviÅ¡Ä·u indeksu katram atseviÅ¡Ä·i pieejamam avotam, izmantojot avotu kÄ indeksa vÄrdu
+CDM.IndexManager.MGPP.Add_Level:Pievienot LÄ«meni
+CDM.IndexManager.MGPP.Add_Level_Tooltip:Izveidot indeksu uz norÄdÄ«to lÄ«meni
+CDM.IndexManager.MGPP.Current_Indexes:Patreiz NozÄ«mÄtie Indeksi
+CDM.IndexManager.MGPP.Current_Levels:Patreiz NozÄ«mÄtie LÄ«meÅi
+CDM.IndexManager.MGPP.Index:Indeksa Avots:
+CDM.IndexManager.MGPP.Index_Tooltip:Metadatu elements uz kura izveidot indeksu
+CDM.IndexManager.MGPP.Indexes:AdministrÄt Indeksus
+CDM.IndexManager.MGPP.Instructions:Å eit jÅ«s nosakÄt, kÄdi bÅ«s meklÄÅ¡anas indeksi kolekcijai.\nLai pievienotu jaunu indeksu, ievadiet indeksa unikÄlo vÄrdu, izvÄlieties materiÄlu/metadatus, kuri tiks indeksÄti, un noklikÅ¡Ä·iniet âPievienot Indeksuâ.\nJa jÅ«s vÄlaties pievienot indeksus visiem pieejamiem avotiem, tad noklikÅ¡Ä·iniet âPievienot Visusâ (katra indeksa vÄrds pÄc noklusÄÅ¡anas bÅ«s avota vÄrds). IevÄrojiet, ka tas atÅ¡Ä·iras no âVisiLaukiâ avota pievienoÅ¡anas, jo indeksu secÄ«ba ir svarÄ«ga.\nJa jÅ«s vÄlaties rediÄ£Ät kÄdu indeksu, izvÄlieties to no patreiz nozÄ«mÄto indeksu saraksta, rediÄ£Äjiet to, un tad noklikÅ¡Ä·iniet âNomainÄ«t Indeksuâ, lai nomainÄ«tu veco.\nLai noÅemtu indeksu, izvÄlieties to no patreiz nozÄ«mÄto indeksu saraksta un  noklikÅ¡Ä·iniet âNoÅemt Indeksuâ.
+CDM.IndexManager.MGPP.Level:LÄ«meÅa Avots:
+CDM.IndexManager.MGPP.Level_Name:LÄ«meÅa VÄrds:
+CDM.IndexManager.MGPP.Level_Name_Tooltip:UnikÄls vÄrds jauna lÄ«meÅa indeksam
+CDM.IndexManager.MGPP.Levels:AdministrÄt LÄ«meÅus
+CDM.IndexManager.MGPP.Remove_Level:NoÅemt LÄ«meni
+CDM.IndexManager.MGPP.Remove_Level_Tooltip:NoÅemt izvÄlÄtÄ lÄ«meÅa indeksu
+CDM.IndexManager.MGPP.Replace_Index:NomainÄ«t Indeksu
+CDM.IndexManager.MGPP.Replace_Index_Tooltip:AktualizÄt izvÄlÄto indeksu
+CDM.IndexManager.MGPP.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Indeksa Un LÄ«meÅa IzvÄle</CENTER></FONT></STRONG></HTML>
+CDM.IndexManager.Paragraph:paragrÄfs
+CDM.IndexManager.Remove_Index:NoÅemt Indeksu
+CDM.IndexManager.Remove_Index_Tooltip:NoÅemt izvÄlÄto indeksu no saraksta
+CDM.IndexManager.Section:sekcija
+CDM.IndexManager.Set_Default:Uzdot NoklusÄjuma Indeksu
+CDM.IndexManager.Set_Default_Tooltip:Uzdot izvÄlÄto indeksu, kÄ indeksu pÄc noklusÄjuma
+CDM.IndexManager.Source:Izveidot indeksu uz:
+CDM.IndexManager.Source_Tooltip:Metadatu elementi uz kuriem izveidot indeksus
+CDM.IndexManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Indeksu IzvÄle</CENTER></FONT></STRONG></HTML>
+CDM.LanguageManager.Add:Pievienot Valodu
+CDM.LanguageManager.Add_Tooltip:Pievienot norÄdÄ«to valodu sarakstam
+CDM.LanguageManager.Assigned_Languages:NozÄ«mÄtÄs Valodas
+CDM.LanguageManager.Default_Language:NoklusÄjuma Valoda
+CDM.LanguageManager.Remove:NoÅemt Valodu
+CDM.LanguageManager.Remove_Tooltip:NoÅemt izvÄlÄto valodu no saraksta
+CDM.LanguageManager.Selector:Valoda ko pievienot:
+CDM.LanguageManager.Selector_Tooltip:Valoda uz kuras izveidot sadalÄ«juma indeksu
+CDM.LanguageManager.Set_Default:Uzdot NoklusÄjumu
+CDM.LanguageManager.Set_Default_Tooltip:Uzdot izvÄlÄto valodu kÄ noklusÄjumu
+CDM.MetadataSetManager.Add:Pievienot Metadatu Kopu
+CDM.MetadataSetManager.Add_Tooltip:Pievienot kolekcijai jaunu metadatu kopu.
+CDM.MetadataSetManager.Edit:RediÄ£Ät Metadatu Kopu
+CDM.MetadataSetManager.Edit_Message:Metadatu kopas var bÅ«t rediÄ£Ätas, izmantojot Greenstone Metadatu Kopu Redaktoru GEMS (Greenstone Editor for Metadata Sets).\nPalaidiet GEMS ar gems.bat (Windows) vai gems.sh (Linux) no Greenstone instalÄcijas mapes ar nosaukumu gli.
+CDM.MetadataSetManager.Edit_Message_Title:Metadatu Kopu RediÄ£ÄÅ¡ana
+CDM.MetadataSetManager.Edit_Tooltip:RediÄ£Ät izvÄlÄto metadatu kopu.
+CDM.MetadataSetManager.Elements:Elementi izvÄlÄtÄ Metadatu KopÄ
+CDM.MetadataSetManager.Instructions:Å ajÄ dizaina skatÄ jÅ«s varat pievienot, konfigurÄt un dzÄst Metadatu Kopas, un definÄt kÄdus elementus tÄs satur jÅ«su kolekcijÄ.
+CDM.MetadataSetManager.Remove:DzÄst Metadatu Kopu
+CDM.MetadataSetManager.Remove_Tooltip:DzÄst izvÄlÄto metadatu kopu no kolekcijas.
+CDM.MetadataSetManager.Sets:PieejamÄs Metadatu Kopas
+CDM.MetadataSetManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Metadatu Kopas</CENTER></FONT></STRONG></HTML>
+CDM.Move.At_Bottom:Nevaru pÄrvietot {0} {1} zemÄk, jo tas jau ir paÅ¡Ä saraksta apakÅ¡Ä.
+CDM.Move.At_Top:Nevaru pÄrvietot {0} {1} augstÄk, jo tas jau ir paÅ¡Ä saraksta virsotnÄ.
+CDM.Move.Cannot:Nevaru pÄrvietot spraudni {0} zemÄk, jo RecPlug un ArcPlug jÄatrodas saraksta paÅ¡Ä apakÅ¡Ä.
+CDM.Move.Fixed:<html>RecPlug un ArcPlug parasti ir nepiecieÅ¡ams bÅ«t pÄdÄjiem un tÄdÄÄŒ ir fiksÄti savÄ vietÄ.<br>Ja jums patiesi vajag mainÄ«t viÅu pozÄ«cijas, tad lÅ«dzu rediÄ£Äjiet collect.cfg datni manuÄli.</html>
+CDM.Move.Move_Down:PÄrvietot ZemÄk
+CDM.Move.Move_Down_Tooltip:PÄrvietot izvÄlÄto elementu zemÄk sarakstÄ
+CDM.Move.Move_Up:PÄrvietot AugstÄk
+CDM.Move.Move_Up_Tooltip:PÄrvietot izvÄlÄto elementu augstÄk sarakstÄ
+CDM.Move.Title:KÄŒÅ«da - Nevaru PÄrvietot
+CDM.PlugInManager.Add:Pievienot Spraudni
+CDM.PlugInManager.Add_Tooltip:Izmantot norÄdÄ«to spraudni Å¡ajÄ kolekcijÄ
+CDM.PlugInManager.Assigned:Patreiz NozÄ«mÄtie SpraudÅi
+CDM.PlugInManager.Configure:KonfigurÄt Spraudni
+CDM.PlugInManager.Configure_Tooltip:IzmainÄ«t opcijas izvÄlÄtam spraudnim
+CDM.PlugInManager.Controls:RediÄ£ÄÅ¡anas Kontrole
+CDM.PlugInManager.Ignore:Nepievienot Spraudni
+CDM.PlugInManager.Ignore_Tooltip:Nepievienot nevienu spraudni Å¡ai kolekcijai.
+CDM.PlugInManager.Instructions:Izmantojiet Å¡o skatu, lai pievienotu, konfigurÄtu vai noÅemtu spraudÅus no jÅ«su kolekcijas. Lai pievienotu spraudni, izvÄlieties vienu no kombinÄtÄ sarasta un noklikÅ¡Ä·iniet âPievienot Spraudniâ.\nLai konfigurÄtu vai noÅemtu (dzÄstu) vienu, izvÄlieties to no pieÅ¡Ä·irto spraudÅu saraksta, un tad:\ni)  Izmainiet tÄ pozÄ«ciju spraudÅu rindÄ noklikÅ¡Ä·inot pogas ar bultÄm. (PiezÄ«me: RecPlug un ArcPlug pozÄ«cijas ir fiksÄtas).\nii)  KonfigurÄjiet to noklikÅ¡Ä·inot âKonfigurÄt Spraudniâ,\niii)  IzdzÄsiet to noklikÅ¡Ä·inot âNoÅemt Spraudniâ.\n\nSpraudÅi tiek konfigurÄti izmantojot uznirstoÅ¡u dialoga logu ar rullslejas argumentu sarakstu. AtÄŒaujiet attiecÄ«gos argumentus un ievadiet vai izvÄlieties vÄrtÄ«bas pÄc vajadzÄ«bas.
+CDM.PlugInManager.PlugIn:IzvÄlieties spraudni ko pievienot:
+CDM.PlugInManager.PlugIn_Tooltip:Pieejamie spraudÅi izmantoÅ¡anai kolekcijÄ
+CDM.PlugInManager.PlugIn_Exists:Å is spraudnis jau esot pievienots Å¡ai kolekcijai.
+CDM.PlugInManager.PlugIn_Str:spraudnis
+CDM.PlugInManager.PlugIn_XML_Parse_Failed:Nevaru noteikt argumentus {0}\nspraudnim. LÅ«dzu pÄrliecinieties, ka spraudnis nodroÅ¡ina\n -xml karodziÅu, palaiÅŸot pluginfo.pl skriptu, kÄ aprakstÄ«ts\ndokumentÄ Greenstone Developers Guide sekcijÄ 2.1.
+CDM.PluginManager.Plugin_Suggestion_Prompt:Nav sagaidÄms, ka kÄds no spraudÅiem Å¡ajÄ kolekcijÄ varÄtu apstrÄdÄt datni â{0}â.SekojoÅ¡ie spraudÅi var bÅ«t izmantoti, lai apstrÄdÄtu Å¡o datni. JÅ«s varat pievienot vienu tagad, izvÄloties to, un nospieÅŸot Pievienot pogu. (JÄpiezÄ«mÄ, ka jÅ«s varat jebkurÄ laikÄ izmainÄ«t spraudÅu sarakstu izmantojot âDokumentu SpraudÅiâ no paneÄŒa Dizains.)
+CDM.PlugInManager.Remove:NoÅemt Spraudni
+CDM.PlugInManager.Remove_Tooltip:DzÄst izvÄlÄto spraudni no saraksta
+CDM.PluginManager.SuggestedPluginListTitle:Ieteicamie spraudÅi datnei
+CDM.PlugInManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SpraudÅu IzvÄle & KonfigurÄÅ¡ana</CENTER></FONT></STRONG></HTML>
+CDM.PlugInManager.Parsing.Message:Dizaina rÄ«ks ielÄdÄ spraudÅu informÄciju.
+CDM.PlugInManager.Parsing.Title:IelÄdÄ SpraudÅus
+CDM.SearchTypeManager.Add:Pievienot MeklÄÅ¡nas Tipu
+CDM.SearchTypeManager.Add_Tooltip:AtÄŒaut noteiktu meklÄÅ¡anas tipu Å¡ajÄ kolekcijÄ
+CDM.SearchTypeManager.Assigned:Patreiz NozÄ«mÄtie MeklÄÅ¡anas Tipi
+CDM.SearchTypeManager.BuildType_Selection:IndeksÄtÄjs:  # Updated 19-Aug-2005
+CDM.SearchTypeManager.BuildType_Selection_Tooltip:IzvÄlieties indeksÄtÄju, ko lietot ar Å¡o kolekciju.  # Updated 19-Aug-2005
+CDM.SearchTypeManager.Enable:AtÄŒaut PaplaÅ¡inÄtÄs MeklÄÅ¡anas
+CDM.SearchTypeManager.Instructions:MeklÄÅ¡anas tipu definÄÅ¡ana ir paplaÅ¡inÄtÄ iespÄja un tikai pieejama, ja ir atÄŒauta (iezÄ«mÄjot âAtÄŒaut PaplaÅ¡inÄtÄs MeklÄÅ¡anasâ). Ja ir atÄŒauta, tad pÄrÄjÄs kontroles, izvÄlei un meklÄÅ¡anas tipu secÄ«bas izmaiÅai, kÄŒÅ«st pieejamas. IzvÄlieties meklÄÅ¡anas tipu no kombosaraksta un nospiediet âPievienot MeklÄÅ¡anas Tipuâ, ja poga atÄŒauta, lai pievienotu meklÄÅ¡anas tipu. IzvÄlieties eksistÄjoÅ¡o meklÄÅ¡anas tipu no saraksta, un izmantojiet âPÄrvietot AugstÄkâ un âPÄrvietot ZemÄkâ, lai izmainÄ«tu tÄ secÄ«bu sarakstÄ, vai âDzÄst MeklÄÅ¡anas Tipuâ, lai to izdzÄstu.\n\nMeklÄÅ¡anas tipi nosaka kÄda veida meklÄÅ¡anas interfeisam jÄbÅ«t nodroÅ¡inÄtam: formas, meklÄÅ¡anai pÄc lauciÅiem, un/vai vienkÄrÅ¡ai, parastai meklÄÅ¡anai. SecÄ«ba ir svarÄ«ga, jo pirmais sarakstÄ noteiks meklÄÅ¡anas interfeisu pÄc noklusÄÅ¡anas.\n\nSvarÄ«gi arÄ« ir tas, ka atÄŒaujot meklÄÅ¡anas tipus, iespaido arÄ« Indeksu skatu izmaiÅas. MeklÄÅ¡ana pÄc lauciÅiem ir Greenstone jaunÄka iespÄja, un tas prasa citÄdu indeksu informÄciju. Kad meklÄÅ¡anas tipi ir atÄŒauti, visi patreiz esoÅ¡ie indeksi tiek modificÄti uz attiecÄ«giem indeksiem pÄc jaunÄs shÄmas, un otrÄdi, aizliedzot meklÄÅ¡anas tipus, liks GBI censties konvertÄt jaunos indeksus atpakaÄŒ uz veco stilu, bet tas ne vienmÄr var bÅ«t sekmÄ«gi. Å Ädos gadÄ«jumos (nesekmÄ«gos) indeksi tiks izdzÄsti. Abos gadÄ«jumos oriÄ£inÄlÄs indeksu komandas kopija bÅ«s pieejama kÄ komentÄrs konfigurÄcijas datnÄ.
+CDM.SearchTypeManager.Remove:DzÄst MeklÄÅ¡anas Tipu
+CDM.SearchTypeManager.Remove_Tooltip:NeatÄŒaut izvÄlÄto meklÄÅ¡anas tipu ar Å¡o kolekciju
+CDM.SearchTypeManager.SearchType:MeklÄÅ¡anas Tips
+CDM.SearchTypeManager.SearchType_Selection:MeklÄÅ¡anas Tipi:
+CDM.SearchTypeManager.SearchType_Selection_Tooltip:MeklÄÅ¡anas tipi pieejami izmantoÅ¡anai ar Å¡o kolekciju
+CDM.SearchTypeManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>MeklÄÅ¡anas Tipu IzvÄle & SecÄ«ba</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionManager.Add:Pievienot Filtru
+CDM.SubcollectionManager.Add_Tooltip:Pievienot norÄdÄ«to filtru kolekcijai
+CDM.SubcollectionManager.Assigned:DefinÄtie ApkÅ¡kolekciju Filtri
+CDM.SubcollectionManager.Exclude:IzslÄgt
+CDM.SubcollectionManager.Flags:Uzdodamais karodziÅÅ¡ pie sakritÄ«bas
+CDM.SubcollectionManager.Flags_Tooltip:KarodziÅu regulÄrÄ izteiksme, kura izmantojama pie sakritÄ«bas (piem. "i" reÄ£istrnejÅ«tÄ«gai sakritÄ«bai)
+CDM.SubcollectionManager.Include:IekÄŒaut
+CDM.SubcollectionManager.Inclusive:Ko mÄs darÄm ar datnÄm, kuras sakrÄ«t?
+CDM.SubcollectionManager.Instructions:Izmantojiet Å¡o paneli smalkÄkai indeksu veidoÅ¡anai.\ni)  DefinÄjiet apakÅ¡kolekciju filtrus, kuri parÄda kolekcijas dokumentus pamatojoties uz specificÄtÄm metadatu vÄrtÄ«bÄm. Vienreiz tie definÄti, jÅ«s varat Ä£enerÄt sadalÄ«jumus uz jebkuru iepriekÅ¡ specificÄto indeksu, bÄzÄjoties uz viena vai vairÄkiem no Å¡iem filtriem.\nii)  Ä¢enerÄt indeksu sadalÄ«jumus bÄzÄjoties uz valodas. Tas izfiltrÄs kolekcijas dokumentus tÄ, ka tikai dokumenti norÄdÄ«tajÄ valodÄ tiks iekÄŒauti parÄdÄ«Å¡anai.\nJÄpiezÄ«mÄ, ka summÄrais Ä£enerÄto sadalÄ«jumu skaits ir visu indeksu, apakÅ¡kolekciju filtru un izvÄlÄto valodu kombinÄcija. Divi indeksi ar diviem apakÅ¡kolekciju filtriem divÄs valodÄs dotu iznÄkumÄ astoÅus indeksu sadalÄ«jumus.
+CDM.SubcollectionManager.Language_Controls:NozÄ«mÄt Valodas
+CDM.SubcollectionManager.Match:SakritÄ«bas regulÄrÄ izteiksme:
+CDM.SubcollectionManager.Match_Tooltip:RegulÄrÄ izteiksme, kura definÄ dokumentus apakÅ¡kolekcijÄ
+CDM.SubcollectionManager.Name:ApakÅ¡kolekcijas filtra vÄrds:
+CDM.SubcollectionManager.Name_Tooltip:JaunÄ apakÅ¡kolekcijas filtra nosaukums
+CDM.SubcollectionManager.Partitions_Disabled:Å Ä« iespÄja patreiz ir aizliegta, jo jÅ«s operÄjat {0-Current Mode} reÅŸÄ«mÄ. Tas ir tikai pieejams {1-Systems} un {2-Expert} reÅŸÄ«mos. Lietojiet Datne->IestatÄ«jumi, lai izmainÄ«tu operÄÅ¡anas reÅŸÄ«mus.
+CDM.SubcollectionManager.Remove:NoÅemt Filtru
+CDM.SubcollectionManager.Remove_Tooltip:DzÄst izvÄlÄto filtru no saraksta
+CDM.SubcollectionManager.Replace:Aizvietot Filtru
+CDM.SubcollectionManager.Replace_Tooltip:Aizvietot un aktualizÄt izvÄlÄtÄ filtra detaÄŒas
+CDM.SubcollectionManager.Source:Dokumenta atribÅ«ts pÄc kura jÄsakrÄ«t:
+CDM.SubcollectionManager.Source_Tooltip:Dokumenta atribÅ«ts pÄc kura filtrÄt
+CDM.SubcollectionManager.Subcollection_Controls:DefinÄt Filtrus
+CDM.SubcollectionManager.Subindex_Controls:NozÄ«mÄt SadalÄ«jumus
+CDM.SubcollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>SadalÄ«jumu Indeksi</CENTER></FONT></STRONG></HTML>
+CDM.SubcollectionIndexManager.Add_Subindex:Pieviento SadalÄ«jumu
+CDM.SubcollectionIndexManager.Add_Subindex_Tooltip:Pievienot norÄdÄ«to sadalÄ«jumu Å¡ai kolekcijai
+CDM.SubcollectionIndexManager.Default_Partition_Indicator:[NoklusÄjuma SadalÄ«jums]
+CDM.SubcollectionIndexManager.PartitionName:SadalÄ«juma VÄrds:
+CDM.SubcollectionIndexManager.PartitionName_Tooltip:JaunÄ sadalÄ«juma nosaukums
+CDM.SubcollectionIndexManager.Remove_Subindex:NoÅemt SadalÄ«jumu
+CDM.SubcollectionIndexManager.Remove_Subindex_Tooltip:IzdzÄst izvÄlÄto sadalÄ«jumu no Å¡Ä«s kolekcijas
+CDM.SubcollectionIndexManager.Replace_Subindex:Aizvietot SadalÄ«jumu
+CDM.SubcollectionIndexManager.Replace_Subindex_Tooltip:Azvietot izvÄlÄto sadalÄ«jumu
+CDM.SubcollectionIndexManager.Set_Default_Subindex:Uzdot NoklusÄjumu
+CDM.SubcollectionIndexManager.Set_Default_Subindex_Tooltip:Uzdot izvÄlÄto sadalÄ«jumu bÅ«t pÄc noklusÄjuma
+CDM.SubcollectionIndexManager.Source:Veidot sadalÄ«jumu uz:
+CDM.SubcollectionIndexManager.Source_Tooltip:ApakÅ¡kolekcijas filtri uz kuriem veidot sadalÄ«jumu
+CDM.SubcollectionIndexManager.Subindexes:NozÄ«mÄtie ApakÅ¡kolekcijas SadalÄ«jumi
+CDM.SuperCollectionManager.Instructions:Starp-kolekciju meklÄÅ¡ana nozÄ«mÄ vienu meklÄÅ¡anu, kura veikta pÄr daÅŸÄdÄm kolekcijÄm, it kÄ tÄs bÅ«tu kÄ viena kolekcija. NorÄdiet kolekcijas, kuras ieÄŒaut Å¡ajÄ meklÄÅ¡anÄ, noklikÅ¡Ä·inot uz attiecÄ«gÄs kolekcijas nosaukuma apakÅ¡Ä esoÅ¡ajÄ sarakstÄ. TekoÅ¡Ä kolekcija tiks iekÄŒauta meklÄÅ¡anÄ automÄtiski. \nBRÄªDINÄJUMS: Ja individuÄlÄm kolekcijÄm nav savstarpÄji vienÄdu indeksu ( tajÄ skaitÄ, apakÅ¡kolekciju  un valodas sadalÄ«jumu indeksu), starp-kolekciju meklÄÅ¡ana nedarbosies pareizi. LietotÄjs varÄs tikai meklÄt izmantojot kopÄjos indeksus visÄs izvÄlÄtÄs kolekcijÄs.
+CDM.SuperCollectionManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Starp-Kolekciju MeklÄÅ¡anas KofigurÄÅ¡ana</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Add:Pievienot Tulkojumu
+CDM.TranslationManager.Add_Tooltip:Pievienot specificÄto tulkojumu kolekcijai
+CDM.TranslationManager.Affected_Features:PazÄ«mes
+CDM.TranslationManager.Assigned_Fragments:NozÄ«mÄtie Tulkojumi
+CDM.TranslationManager.Default_Text:SÄkotnÄjs Teksta Fragments
+CDM.TranslationManager.Fragment_Column:Teksta Fragments
+CDM.TranslationManager.Instructions:Izmantojiet Å¡o paneli, lai pÄrskatÄ«tu un nozÄ«mÄtu teksta fragmentu tulkojumus jÅ«su kolekcijai. Vispirms, izvÄlieties ierakstu no pazÄ«mju saraksta (piem. kolekcijas lÄ«meÅa metadatus un klasifikatoru vÄrdus). TÄlÄk, izvÄlieties un rediÄ£Äjiet teksta fragmentu, vai izvÄlieties valodu no kombinÄtÄ saraksta, lai pievienotu jaunu fragmentu citÄ valodÄ. Ievadiet tekstu tukÅ¡Ä (baltÄ) teksta lauciÅÄ, pretim attiecÄ«gam sÄkotnÄja teksta fragmentam. Jebkuras izmaiÅas parÄdÄ«sies kolekcijÄ nekavÄjoties.
+CDM.TranslationManager.Language:Tulkojuma valoda:
+CDM.TranslationManager.Language_Column:Valoda
+CDM.TranslationManager.Language_Tooltip:Valoda uz kuru tiek tulkots
+CDM.TranslationManager.Remove:NoÅemt Tulkojumu
+CDM.TranslationManager.Remove_Tooltip:IzdzÄst izvÄlÄto tulkojumu no kolekcijas
+CDM.TranslationManager.Replace:Aizvietot Tulkojumu
+CDM.TranslationManager.Replace_Tooltip:AktualizÄt (aizvietot) izvÄlÄto tulkojumu
+CDM.TranslationManager.Title:<HTML><STRONG><FONT FACE={FONT} COLOR=Black SIZE=5><CENTER>Teksta Tulkojums</CENTER></FONT></STRONG></HTML>
+CDM.TranslationManager.Translation:Iztulkotais Teksts
+CDM.TranslationManager.Translation_Tooltip:Iztulkota teksta rindu ievadiet Å¡eit
+#*******************
+#
+#***** Collection *****
+Collection.Collection:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Kolekcija</FONT></STRONG></HTML>
+Collection.Delete_Tooltip:IzdzÄst izvÄlÄtÄs datnes un mapes
+Collection.Filter_Tooltip:IerobeÅŸot datÅu atrÄdÄ«Å¡anu datÅu kokÄ
+Collection.New_Folder_Tooltip:Izveidot jaunu mapi
+Collection.No_Collection:Nav Kolekcijas
+Collection.Stop:Stop
+Collection.Stop_Tooltip:PÄrtraukt datÅu kopÄÅ¡anu
+Collection.Workspace:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Darbatelpa</FONT></STRONG></HTML>
+#**********************
+#
+#***** CollectionBuilt *****
+CollectionBuilt.Message:Kolekcija esot izveidota un ir gatava priekÅ¡skatÄ«jumam.
+CollectionBuilt.Title:Kolekcijas IzveidoÅ¡anas RezultÄts
+#***************************
+#
+#***** CollectionManager *****
+CollectionManager.Build_Cancelled:Kolekcijas veidoÅ¡ana tikusi anulÄta.
+CollectionManager.Cannot_Create_Collection:AtgadÄ«jusies kÄŒÅ«da un kolekcija nevar bÅ«t izveidota.
+CollectionManager.Cannot_Create_Collection_With_Reason:Kolekcija nevar tikt radÄ«ta dÄÄŒ:\n{0-Error message}  # Updated 19-Aug-2005
+CollectionManager.Cannot_Delete_Index:Kolekcija nevar tikt veidota no jauna, jo nevaru izdzÄst vecÄs indeksa datnes.\n LÅ«dzu pÄrbaudiet, ka Greenstone vai cita programma neizmanto Å¡Ä«s datnes,\n un mÄÄ£iniet vÄlreiz.
+CollectionManager.Cannot_Delete_Index_Log:VecÄs indeksa datnes nevar tikt izdzÄstas, tÄpÄc importÄÅ¡ana pÄrtraukta.
+CollectionManager.Cannot_Open:Kolekcija:\n{0-Collection file path}\nnevar bÅ«t atvÄrta.
+CollectionManager.Cannot_Open_With_Reason:Kolekcija:\n{0-Collection file path}\nnevar bÅ«t atvÄrta jo:\n{1-Error message}
+CollectionManager.Creating_New:Jaunas kolekcijas radÄ«Å¡ana.
+CollectionManager.Log_Created:Log mape izveidota.
+CollectionManager.Loading_Collection:Kolekcijas ielÄdÄÅ¡ana
+CollectionManager.Loading_Collection_Please_Wait:Kolekcijas ielÄdÄÅ¡ana, lÅ«dzu uzgaidiet...
+CollectionManager.Loading_Successful:IelÄdÄtÄ kolekcija {0}.
+CollectionManager.Missing_Config:Å ai kolekcijai nav pareiza collect.cfg datne
+CollectionManager.Not_Col_File:{0}' neesot BibliotekÄra Interfeisa Kolekcijas Datne (.col)
+CollectionManager.No_Config_File:Datne collect.cfg netika radÄ«ta ar mkcol.pl.  # Updated 19-Aug-2005
+CollectionManager.Preview_Ready_Failed:AtgadÄ«jÄs kÄŒÅ«da, kura neÄŒauj, lai kolekcija tiktu priekÅ¡skatÄ«ta.
+CollectionManager.Preview_Ready_Title:Kolekcijas PriekÅ¡skatÄ«juma StÄvoklis.
+CollectionManager.Session_Ready:Kolekcija {0} gatava izmantoÅ¡anai.
+#*****************************
+#
+#***** CollectionPopupMenu *****
+CollectionPopupMenu.Delete:IzdzÄst
+CollectionPopupMenu.New_Dummy_Doc:Jauns fiktÄ«vs dokuments
+CollectionPopupMenu.New_Folder:Jauna mape
+#*******************************
+#
+#***** CreatePane *****
+CreatePane.Build:VeidoÅ¡anas Opcijas
+CreatePane.Build_Tooltip:Opcijas kolekcijas formÄÅ¡anas indeksÄcijas un klasifikÄcijas stadijÄ
+CreatePane.Build_Collection:Veidot Kolekciju
+CreatePane.Build_Collection_Tooltip:SÄkt kolekcijas formÄÅ¡anas procesu
+CreatePane.Build_Progress:VeidoÅ¡anas Progress
+CreatePane.Cancel_Build:Atcelt Veidot
+CreatePane.Cancel_Build_Tooltip:PÄrtraukt kolekcijas formÄÅ¡anas procesu
+CreatePane.Full_Build:PilnÄ«ga PÄrveidoÅ¡ana  # Updated 19-Aug-2005
+CreatePane.Full_Build_Tooltip:PilnÄ«gi pÄrveidot kolekciju no jauna  # Updated 19-Aug-2005
+CreatePane.Import:Importa Opcijas
+CreatePane.Import_Tooltip:Opcijas kolekcijas formÄÅ¡anas datÅu convertÄcijas un metadatu pieÅ¡Ä·irÅ¡anas stadijÄ
+CreatePane.Import_Progress:Importa Progress
+CreatePane.Incremental_Build:InkrementÄlÄ VeidoÅ¡ana  # Updated 19-Aug-2005
+CreatePane.Incremental_Build_Tooltip:ÄtrÄk, par cik, tikai daÄŒa no visa nepiecieÅ¡amÄ kolekcijas veidoÅ¡anas procesa tiek veikta.  # Updated 19-Aug-2005
+CreatePane.Log:PaziÅojumu Log
+CreatePane.Log_Tooltip:SkatÄ«t izvadu no iepriekÅ¡Äjiem kolekcijas formÄÅ¡anas mÄÄ£inÄjumiem
+CreatePane.Mode_All:viss
+CreatePane.Options:Opciju Grupas
+CreatePane.Options_Title:Kolekcijas Importa & VeidoÅ¡anas Opcijas
+CreatePane.Preview_Collection:PriekÅ¡skatÄ«t Kolekciju
+CreatePane.Preview_Collection_Tooltip:PriekÅ¡skatÄ«t izveidotu kolekciju
+#**********************
+#
+#***** Dates *****
+Dates.Mon:Pirmdiena
+Dates.Tue:Otrdiena
+Dates.Wed:TreÅ¡diena
+Dates.Thu:Ceturtdiena
+Dates.Fri:Piektdiena
+Dates.Sat:Sestdiena
+Dates.Sun:SvÄtdiena
+Dates.Jan:JanvÄris
+Dates.Feb:FerbruÄris
+Dates.Mar:Marts
+Dates.Apr:AprÄ«lis
+Dates.May:Maijs
+Dates.Jun:JÅ«nijs
+Dates.Jul:JÅ«lijs
+Dates.Aug:Augusts
+Dates.Sep:Septembris
+Dates.Oct:Oktobris
+Dates.Nov:Novembris
+Dates.Dec:Decembris
+#**********************
+#
+#***** Delete Collection Prompt *****
+DeleteCollectionPrompt.Collection_Details:IzvÄlÄtÄs kolekcijas detaÄŒas
+DeleteCollectionPrompt.Collection_List:Pieejamas kolekcijas
+DeleteCollectionPrompt.Confirm_Delete:LÅ«dzu atzÄ«mÄjiet rÅ«tiÅu, lai apstiprinÄtu kolekcijas izdzÄÅ¡anu.
+DeleteCollectionPrompt.Delete:IzdzÄst
+DeleteCollectionPrompt.Delete_Tooltip:IzdzÄst izvÄlÄto kolekciju
+DeleteCollectionPrompt.Details:VeidotÄjs - {0}\nAtbalstÄ«tÄjs - {1}\nApraksts\n{2}
+DeleteCollectionPrompt.Failed_Delete:Kolekcija nevar bÅ«t pilnÄ«gi izdzÄsta.
+DeleteCollectionPrompt.Failed_Title:IzdzÄÅ¡ana NeizdevÄs!
+DeleteCollectionPrompt.No_Collection:Nav izvÄlÄta kolekcija.
+DeleteCollectionPrompt.Successful_Delete:Kolekcija tikusi izdzÄsta.
+DeleteCollectionPrompt.Successful_Title:IzdzÄÅ¡ana Pabeigta!
+DeleteCollectionPrompt.Title:Kolekcijas IzdzÄÅ¡ana
+#**********************
+#
+#***** DirectoryLevelMetadata *****
+DirectoryLevelMetadata.Message:JÅ«s taisÄties pieÅ¡Ä·irt 'mapes lÄ«meÅa' metadatus izvÄlÄtai mapei (vai mapÄm). Å os metadatus automÄtiski pÄrmantos visas datnes un mapes Å¡ajÄ mapÄ. TurklÄt Å¡ie metadata nevar bÅ«t vienkÄrÅ¡i noÅemti, bet tikai aizvietoti. Lai turpinÄtu Å¡o darbÄ«bu nospiediet 'OK'.
+DirectoryLevelMetadata.Title:Par mapes lÄ«meÅa metadatu pievienoÅ¡anu
+#********************
+#
+#***** EnrichPane *****
+EnrichPane.Accumulate:Pievienot
+EnrichPane.Accumulate_Tooltip:Pievienot Å¡o vÄrtÄ«bu izvÄlÄtam metadatu elementam
+EnrichPane.AutoMessage:Å is elements, {0}, pieder Greenstone metadatu kopai. Å Ä« kopa reprezentÄ metadatus, kuri parasti var bÅ«t izvilkti automÄtiski. Lai pÄrbaudÄ«tu, kÄdi  {0} metadati var tikt izvilkti, radiet kolekciju izmantojot 'RadÄ«t' tab.
+EnrichPane.ExistingValues:EsoÅ¡Äs vÄrtÄ«bas priekÅ¡ {0}  # Updated 19-Aug-2005
+EnrichPane.Expand:...
+EnrichPane.Expand_Tooltip:RediÄ£Ät saturu lielÄkÄ teksta lodziÅÄ
+EnrichPane.InheritedMetadataSelected:Å ie metadati ir pÄrmantoti no mapes virs tÄs, un nevar bÅ«t rediÄ£Äti.
+EnrichPane.No_File:Nav IzvÄlÄtas Datnes
+EnrichPane.No_Metadata:Nav Pieejami Metadati
+EnrichPane.No_Metadata_Element:<HTML><STRONG><FONT FACE={FONT} COLOR=Black>Nav Metadatu Elements IzvÄlÄts</FONT></STRONG></HTML>
+EnrichPane.Overwrite:Aizvietot
+EnrichPane.Overwrite_Tooltip:PieÅ¡Ä·irt Å¡o vÄrtÄ«bu izvÄlÄtam metadatu elementam, aizvietojot esoÅ¡o vÄrtÄ«bu
+EnrichPane.Remove:NoÅemt
+EnrichPane.Remove_Tooltip:IzdzÄst Å¡o vÄrtÄ«bu izvÄlÄtam metadatu elementam
+EnrichPane.Value:VÄrtÄ«ba
+EnrichPane.Value_Field_Tooltip:VÄrtÄ«ba, ko pieÅ¡Ä·irt izvÄlÄtam metadatu elementam
+EnrichPane.Value_Field_Tooltip_Uneditable:VÄrtÄ«ba, kura pieÅ¡Ä·irta izvÄlÄtam metadatu elementam
+#***********************
+#
+#****** ExplodeMetadataPrompt *******
+ExplodeMetadataPrompt.Explode:SadalÄ«t
+ExplodeMetadataPrompt.Explode_Tooltip:SadalÄ«t metadatu datni
+ExplodeMetadataPrompt.Instructions:Sadalot Å¡o metadatu datni tiks producÄti individuÄli ieraksti ar rediÄ£Äjamiem metadatiem. Å is ir neatgrieÅŸams process un kolekcijas originÄlÄ metadatu datne tiks izdzÄsta.
+ExplodeMetadataPrompt.Failed_Title:SadalÄ«Å¡ana NeizdevÄs.
+ExplodeMetadataPrompt.Failed_Explode:DatubÄze {0} nevar bÅ«t sadalÄ«ta.
+ExplodeMetadataPrompt.NotExplodable:DotÄ datne nav sadalÄma.
+ExplodeMetadataPrompt.Successful_Explode:DatubÄze {0} sekmÄ«gi sadalÄ«ta.
+ExplodeMetadataPrompt.Successful_Title:SadalÄ«Å¡ana Pabeigta.
+ExplodeMetadataPrompt.Title:SadalÄ«t Metadatu DatubÄzi
+#***********************
+#
+#****** WriteCDImagePrompt *******
+WriteCDImagePrompt.CD_Name:CD/DVD vÄrds
+WriteCDImagePrompt.Export:RakstÄ«t CD/DVD attÄlu
+WriteCDImagePrompt.Export_Tooltip:RakstÄ«t izvÄlÄtÄs kolekcijas, kÄ Windows CD/DVD AttÄlus
+WriteCDImagePrompt.Failed_Export:Kolekcijas ({0}) nevarÄja bÅ«t eksportÄtas.
+WriteCDImagePrompt.Failed_Title:Eksports NeizdevÄs
+WriteCDImagePrompt.Instructions:Vienas vai vairÄku kolekciju eksports uz paÅ¡instalÄjoÅ¡os Windows CD/DVD (kaut arÄ« tas darbojas uz jebkuras platformas, CD/DVD, kas tiek radÄ«ts, palaiÅŸas tikai zem Windows).
+WriteCDImagePrompt.Progress_Label:DatÅu kopÄÅ¡ana. Tas var aizÅemt zinÄmu laiku...
+WriteCDImagePrompt.Successful_Export:Kolekcijas ({0}) esot tikuÅ¡as eksportÄtas.
+WriteCDImagePrompt.Successful_Title:Eksports Pabeigts
+WriteCDImagePrompt.Title:Kolekcijas Eksports uz CD/DVD
+#************************
+#
+#****** ExportAsPrompt *******
+ExportAsPrompt.Export_Name:Eksporta mapes nosaukums
+ExportAsPrompt.Export:EksportÄt kolekciju(as)
+ExportAsPrompt.Export_Tooltip:EksportÄt izvÄlÄtÄs kolekcijas uz noteiktu mapi un izvÄlÄtÄ formÄtÄ
+ExportAsPrompt.Failed_ExportOne:Kolekcija, {0}, nevar bÅ«t eksportÄta.
+ExportAsPrompt.Failed_ExportMany:Kolekcijas -- {0} -- nevar bÅ«t eksportÄtas.
+ExportAsPrompt.Failed_Details:SkatÄ«t {1} sÄ«kÄk
+ExportAsPrompt.Failed_Title:Eksports NeizdevÄs
+ExportAsPrompt.Instructions:EksportÄt vienu vai vairÄkas kolekcijas izvÄlÄtÄ formÄtÄ uz noteiktu mapi
+ExportAsPrompt.Progress_Label:Datnes tiek eksportÄtas. Tas var aizÅemt zinÄmu laiku...
+ExportAsPrompt.Successful_ExportOne:Kolekcija, {0}, esot tikusi eksportÄta.
+ExportAsPrompt.Successful_ExportMany:Kolekcijas -- {0} -- esot tikuÅ¡as eksportÄtas.
+ExportAsPrompt.Successful_Details:Datnes eksportÄtas uz: {1}
+ExportAsPrompt.Successful_Title:Eksports Pabeigts
+ExportAsPrompt.Title:EksportÄt Kolekcijas
+ExportAsPrompt.SaveAs:EksportÄt uz
+ExportAsPrompt.SaveAs_Tooltip:IzvÄlieties formÄtu uz kuru jÅ«s vÄlaties eksportÄt
+#************************
+#
+#*****ExternalCollectionPrompt********
+ExternalCollectionPrompt.NewMDS:RadÄ«t jaunu, tukÅ¡u metadatu kopu.
+ExternalCollectionPrompt.ExistingMDS:IzvÄlieties vienu vai vairÄkas metadatu kopas
+ExternalCollectionPrompt.Title:IelÄdÄt Äreju kolekciju
+ExternalCollectionPrompt.Instructions_1:JÅ«s gribat ielÄdÄt kolekciju, kura netika izveidota ar GBI. JÅ«s tiksiet aicinÄti izvÄlÄties vienu vai vairÄkas metadatu kopas, kuras izmantot ar doto kolekciju. EksistÄjoÅ¡ie metadati tiks importÄti izvÄlÄtajÄ(s) metadatau kopÄ(s). OriginÄlÄs metadatu datnes tiks saglabÄtas mapÄ ar nosaukumu import.bak.
+ExternalCollectionPrompt.Instructions_2:JÅ«s gribat ielÄdÄt kolekciju, kura netika izveidota ar GBI. Jums nepiecieÅ¡ams pievienot metadatu kopu Å¡ai kolekcijai. JÅ«s varat radÄ«t jaunu tukÅ¡u metadatu kopu, kurai visi atrastie metadatu elementi kolekcijÄ tiks automÄtiski pievienoti. AlternatÄ«vi, jÅ«s varat izvÄlÄties eksistÄjoÅ¡as metadatu kopas, un elementi no kolekcijas tiks apvienoti vai pievienoti jau esoÅ¡iem elementiem Å¡ajÄs kopÄs. OriÄ£inÄlÄs metadatu datnes tiks saglabÄtas mapÄ ar nosaukumu import.bak.
+#****************************************
+#
+#***** Inherited Metadata *****
+ExtractedMetadata.Message:IzvÄletÄ datne vai mape satur izvilktus metadatus, bet patreiz tie ir neredzami. Lai redzÄtu izvilktus metadatus, dodieties uz "[D]atne" izvÄlni un izvÄlieties "[I]estatÄ«jumi". "VispÄrÄ«gi" rekvizÄ«tu lapÄ izvÄlieties "SkatÄ«t Izvilktus Metadatus", tad nospieÅŸiet "[O]k".
+ExtractedMetadata.Title:Å Ä« Datne/Mape satur Neredzamus Izvilktus Metadatus
+#
+#***** FileActions *****
+FileActions.Calculating_Size:Tiek kalkulÄts izvÄlÄto datÅu kopÄjais izmÄrs
+FileActions.Copying:KopÄju {0}
+FileActions.Deleting:IzdzÄÅ¡u {0}
+FileActions.Directories_Selected:{0} mapes izvÄlÄtas
+FileActions.Directory_Exists:<html>Mape<br>{0}<br>jau eksistÄ kolekcijÄ. Datnes tiks kopÄtas tajÄ.</html>
+FileActions.Directory_Selected:1 mape izvÄlÄta
+FileActions.File_Already_Exists_No_Create:Datne '{0}' jau eksistÄ.
+FileActions.File_Already_Exists_Message:{0}\nDatne ar nosaukumu, kas parÄdÄ«ta augstÄk jau eksistÄ\nun par spÄ«ti jÅ«su iepriekÅ¡Äjai izvÄlei BibliotekÄra Interfeiss\nnevar to pÄrrakstÄ«t. LÅ«dzu pÄrbaudiet vai datne ir rakstÄ«t atÄŒauta.
+FileActions.File_Already_Exists_Title:KÄŒÅ«da - Datne Jau EksistÄ.
+FileActions.File_And_Directories_Selected:1 datne un {0} mapes izvÄlÄtas
+FileActions.File_And_Directory_Selected:1 datne un 1 mape izvÄlÄta
+FileActions.File_Create_Error:Datne {0} nevar tikt radÄ«ta.
+FileActions.Folder_Create_Error:Mape {0} nevar tikt radÄ«ta.
+FileActions.File_Exists:<html>{0} jau eksistÄ saÅÄmÄjmapÄ. PÄrrakstÄ« to?</html>
+FileActions.File_Not_Found_Message:{0}\nAugstÄk minÄtÄ datne nevar tikt atrasta. LÄ«dz ar datÅu skata\natjaunoÅ¡anu, lÅ«dzu pÄrbaudiet, ka datne vÄl eksistÄ.
+FileActions.File_Not_Found_Title:KÄŒÅ«da - Datne nav atrasta.
+FileActions.File_Selected:1 datne izvÄlÄta
+FileActions.Files_And_Directory_Selected:{0} datnes un 1 mape izvÄlÄta
+FileActions.Files_Selected:{0} datnes izvÄlÄtas
+FileActions.Folder_Already_Exists:Mape ar nosaukumu '{0}' tiek jau lietota. Nevaru radÄ«t mapi.
+FileActions.Increase_Depth:PalielinÄt DziÄŒumu par 1
+FileActions.Insufficient_Space_Message:DarbÄ«ba ar datni neizdevÄs, jo nepietiek vietas\nuz diska. Lai sekmÄ«gi nokopÄtu datni jums turmÄk\nnepiecieÅ¡ams: {0}
+FileActions.Insufficient_Space_Title:KÄŒÅ«da - Nepietiek Vietas
+FileActions.Moving:PÄrvietoju {0}
+FileActions.No_Activity:Nav pieprasÄ«tas darbÄ«bas
+FileActions.Possible_Cyclic_Path:{1} ir {0} mapes dziÄŒumÄ un var bÅ«t rekursÄ«vs ceÄŒÅ¡? Vai datÅu darbÄ«bu turpinÄt?
+FileActions.Read_Only:Datnes no darbatelpas ir tikai lasÄmas un navar bÅ«t izdzÄstas.
+FileActions.Selected:{0} datnes un {1} mapes izvÄlÄtas
+FileActions.Unknown_File_Error_Message:Lai gan nebija kÄŒÅ«da reportÄta datnes darbÄ«bas\nlaikÄ, gala pÄrbaude parÄda, ka\nkopijas datne nav tÄ pati kÄ avota datne. TÄ cÄlonis\nnav zinÄms. LÅ«dzu pÄrbaudiet jÅ«su disku un atjaunojiet\njÅ«su instalÄto JVM.
+FileActions.Unknown_File_Error_Title:KÄŒÅ«da - NezinÄma Datnes KÄŒÅ«da
+FileActions.Write_Not_Permitted_Message:BibliotekÄra Interfeisam nav atÄŒaujas rakstÄ«t\n{0}\nLÅ«dzu pÄrbaudiet datnes atÄŒaujas atribÅ«tus.
+FileActions.Write_Not_Permitted_Title:KÄŒÅ«da - Nepareizas Datnes AtÄŒaujas
+FileActions.Yes_To_All:JÄ uz Visiem
+#************************
+#
+#******* File Associations dialog **********
+FileAssociationDialog.Add:Pievienot
+FileAssociationDialog.Add_Tooltip:Izmantojiet Å¡o lietojumprogrammu, lai skatÄ«tu Å¡Ä tipa datni
+FileAssociationDialog.Batch_File:PakeÅ¡datnes
+FileAssociationDialog.Browse:PÄrlÅ«kot
+FileAssociationDialog.Browse_Tooltip:IzvÄlieties lietojumprogrammu, ko izmantot no jÅ«su datÅutelpas
+FileAssociationDialog.Browse_Tooltip_Mac:JÅ«su lietojumprogrammas pÄrlÅ«koÅ¡ana patreiz ir aizliegta priekÅ¡ MacOS
+FileAssociationDialog.Browse_Title:IzvÄlieties Lietojumprogrammu
+FileAssociationDialog.Close:AizvÄrt
+FileAssociationDialog.Close_Tooltip:Aizveriet Å¡o dialogu un atgriezieties uz galveno logu
+FileAssociationDialog.Command:PalaiÅ¡anas komanda:
+FileAssociationDialog.Command_File:Komanddatnes
+FileAssociationDialog.Command_Tooltip:Ievadiet komandu ar kuru palaist lietojumprogrammu
+FileAssociationDialog.Details:JaunÄs Datnes AsociÄcijas DetaÄŒas
+FileAssociationDialog.Executable_File:IzpildÄmÄs datnes
+FileAssociationDialog.Existing_Associations:AktuÄlÄs Datnes AsociÄcijas
+FileAssociationDialog.Extension:DatnÄm kas beidzas ar:
+FileAssociationDialog.Extension_Tooltip:SkatÄmo datÅu datnes paplaÅ¡inÄjums
+FileAssociationDialog.Instructions:Izmantojiet Å¡o dialogu, lai uzstÄdÄ«tu, kÄdas ÄrÄjÄs programmas tiks lietotas skatot noteiktu datnes-tipu, un kÄda komanda tiks nosÅ«tÄ«ta, lai  palaistu Å¡o programmu.\nLai pievienotu jaunu datnes-tipu, ievadiet tÄs paplaÅ¡inÄjumu lauciÅÄ 'DatnÄm kas beidzas ar'. Lai pievienotu vai modificÄtu palaiÅ¡anas komandu, ievadiet jaunu komandu lodziÅÄ 'PalaiÅ¡anas Komanda' vai noklikÅ¡Ä·iniet 'PÄrlÅ«kot', ja vÄlaties meklÄt datÅu sistÄmÄ attiecÄ«go programmu. Izmantojiet formÄtu '%1', lai indicÄtu vietu, kur vajadzÄtu tikt iespraustam datnes nosaukumam atveramai datnei, piemÄram, "C:\\Windows\\System\\MSPaint.exe %1". '%1' tiks pievienots komandas galÄ, ja netiks specificÄts citÄdi.\nMacOS lietotÄjiem vienmÄr vispirms jÄmÄÄ£ina specifisko mac komandu "open [-a <application_name>] %1".
+FileAssociationDialog.Table.Command:Komanda
+FileAssociationDialog.Table.Extension:PaplaÅ¡inÄjums
+FileAssociationDialog.Title:RediÄ£Ät Datnes AsociÄcijas
+FileAssociationDialog.Remove:NoÅemt
+FileAssociationDialog.Remove_Tooltip:IzdzÄst patreiz izvÄlÄto asociÄciju no eksistÄjoÅ¡o asociÄciju saraksta
+FileAssociationDialog.Replace:Aizvietot
+FileAssociationDialog.Replace_Tooltip:Aizvietot patreiz izvÄlÄto asociÄciju ar jaunÄm asociÄcijas detaÄŒÄm
+#*******************
+#
+#******Filter***************
+Filter.0:HTM & HTML
+Filter.1:XML
+Filter.2:Teksta Datnes
+Filter.3:AttÄli
+Filter.All_Files:Visas Datnes
+Filter.Filter_Tree:ParÄdÄ«t Datnes
+Filter.Invalid_Pattern:<HTML><FONT FACE={FONT} COLOR=Black>Å ablonu, ko jÅ«s esat ievadÄ«juÅ¡i datÅu filtrÄcijai, ir nederÄ«gs.<BR>LÅ«dzu mÄÄ£iniet vÄlreiz, izmantojot regulÄras izteiksmes sintaksi un ar<BR>* norÄdot aizstÄjÄjzÄ«mi.</FONT></HTML>
+#***********************
+#
+#***** GAuthenticator *****
+GAuthenticator.Password:Parole:
+GAuthenticator.Password_Tooltip:Ievadiet Å¡eit jÅ«su paroli
+GAuthenticator.Title:Parole ir NepiecieÅ¡ama.
+GAuthenticator.Username:LietotÄja VÄrds:
+GAuthenticator.Username_Tooltip:Ievadiet Å¡eit jÅ«su lietotÄjvÄrdu
+#**************************
+#
+#***** General Messages *****
+# 0 - A String representing the name and method of the class in question.
+General.Apply:Lietot
+General.Apply_Tooltip:Pielietot patreizÄjos uzstatÄ«jumus, bet neiziet no dialoga
+General.Browse:PÄrlÅ«kot...
+General.Cancel:Atcelt
+General.Cancel_Tooltip:Atcelt izmaiÅas (neatsauks jebkuru jau pielietotu izmaiÅu)
+General.Close:AizvÄrt
+General.Close_Tooltip:AizvÄrt Å¡o dialoglodziÅu
+General.Edit:RediÄ£Ät VÄrtÄ«bu
+General.Error:KÄŒÅ«da
+General.No:NÄ
+General.NotSunJava:Java vendor: {0}\nIevÄrojiet, ka BibliotekÄra Interfeiss ir ticis attÄ«stÄ«ts un testÄts ar Java no Sun Microsystems.
+General.OK:OK
+General.OK_Tooltip:AkceptÄt patreizÄjos iestatÄ«jumus un iziet no dialoga
+General.Open:AtvÄrt
+General.Outstanding_Processes:<html>BibliotekÄra Interfeiss nevar pilnÄ«gi beigt darbu, kamÄr visas programmas<br>startÄtas ar BibliotekÄra Interfeisu arÄ« nav pabeigtas.</html>
+General.Outstanding_Processes_Title:GaidoÅ¡s lai Izietu
+General.Pure_Cancel_Tooltip:PÄrtraukt Å¡o darbÄ«bu (izmaiÅas netiks veiktas)
+General.Review_Output:LÅ«dzu pÄrlÅ«kojiet izvadu zemÄk:
+General.Usage:LietoÅ¡ana: {0} <params>\nVisi parametri ir neobligÄti, bet BibliotekÄra Interfeiss var nojukt darbÄ, ja -gsdl un -library nav uzstÄdÄ«ti.\n -gsdl <path> : NorÄda ceÄŒu uz gsdl\n - library <path> : CeÄŒu uz cgi-bin\n -mozilla <path> : NorÄda ceÄŒu uz mozilla\n -mirror : AtÄŒauj web-mirroring kontroles\n -laf [java|windows|motif|mac] : Look and feel\n debug : AtÄŒauj kÄŒÅ«du paziÅojumus\n -no_load : NeielÄdÄt iepriekÅ¡ atvÄrtu kolekciju\n -load <path> : IelÄdÄt norÄdÄ«to kolekciju
+General.View:JaunÄ VÄrtÄ«ba
+General.Warning:BrÄ«dinÄjums
+General.Yes:JÄ
+#****************************
+#
+#***** GShell *****
+GShell.Build.Auxilary:PalÄ«gdatÅu radÄ«Å¡ana un apstrÄde...
+GShell.Build.BuildBegun1:************** VeidoÅ¡ana  UzsÄkta **************
+GShell.Build.BuildCancelled:**************   AnulÄts    **************
+GShell.Build.BuildComplete1:************** VeidoÅ¡ana Pabeigta **************
+GShell.Build.CompressText:Teksta SaspieÅ¡ana...
+GShell.Build.Index:Indeksa radÄ«Å¡ana bÄzÄjoties uz {0-index source and level}...
+GShell.Build.InfoDatabase:InformÄcijas datu bÄzes radÄ«Å¡ana...
+GShell.Build.Phind:Klasifikatora Phind Ä£enerÄÅ¡ana. Tas var aizÅemt zinÄmu laiku...
+GShell.Command:Komanda
+GShell.Failure:Komanda izpildÄ«jÄs nesekmÄ«gi.
+GShell.Import.FileNotProcessed:Datne {0-target file path} bija atpazÄ«ta, bet nevarÄja tikt apstrÄdÄta ar kÄdu no spraudÅiem (plugin).
+GShell.Import.FileNotRecognised:Datne {0-target file path} nebija atpazÄ«ta ne ar vienu spraudni.
+GShell.Import.FileProcessing:Datne {0-target file path} tiek apstrÄdÄta ar {1-plugin name}.
+GShell.Import.FileProcessingError:Datnes {0-target file path} apstrÄdes process sadÅ«rÄs ar kÄŒÅ«du.
+GShell.Import.ImportBegun1:************** Imports  SÄcies **************
+GShell.Import.ImportComplete1:************** Imports Pabeigts **************
+GShell.Import.ImportComplete2:{0-number documents considered} dokumenti bija izskatÄ«ti apstrÄdei:
+GShell.Import.ImportComplete2.1:1 dokuments bija izskatÄ«ts apstrÄdei:
+GShell.Import.ImportComplete.Processed:{0-number processed} dokumenti bija apstrÄdÄti un iekÄŒauti kolekcijÄ.
+GShell.Import.ImportComplete.Processed.1:1 dokuments bija apstrÄdÄts un iekÄŒauts kolekcijÄ.
+GShell.Import.ImportComplete.Blocked:{0-number blocked} bija atturÄti no pievienoÅ¡anas ar spraudÅiem (tÄdi kÄ web lappuÅ¡u attÄli).
+GShell.Import.ImportComplete.Blocked.1:1 bija atturÄts no pievienoÅ¡anas ar spraudÅiem (tÄdi kÄ web lappuÅ¡u attÄli).
+GShell.Import.ImportComplete.Ignored:{0-number ignored} bija neatpazÄ«ti.
+GShell.Import.ImportComplete.Ignored.1:1 bija neatpazÄ«ts.
+GShell.Import.ImportComplete.Failed:{0-number failed} bija noraidÄ«ti.
+GShell.Import.ImportComplete.Failed.1:1 bija noraidÄ«ts.
+GShell.Import.ImportComplete3:\n
+GShell.Parsing_Metadata_Complete:ArhivÄtu metadatu izvilkÅ¡ana pabeigta.
+GShell.Parsing_Metadata_Start:Jaunu metadatu izvilkÅ¡ana no arhÄ«vu datnÄm.
+GShell.Success:Komanda pabeigta.
+#*********************
+#
+#***** GUI *****
+# Currently Enabled and Disabled display the same text but in different
+# colours. If necessary this can be changed.
+GUI.Create:RadÄ«t
+GUI.Create_Tooltip:Veidot jÅ«su kolekciju izmantojot Greenstone
+GUI.Design:Dizains
+GUI.Design_Tooltip:JÅ«su kolekcijas funkcionalitÄtes un izskata dizains
+GUI.Download:LejupielÄde
+GUI.Download_Tooltip:LejupielÄdÄt resursus no Interneta jÅ«su kolekcijai
+GUI.Enrich:PagÄtinÄt
+GUI.Enrich_Tooltip:NozÄ«mÄt metadatus datnÄm jÅ«su kolekcijÄ
+GUI.Gather:UzkrÄt
+GUI.Gather_Tooltip:IzvÄlÄties datnes, kuras iekÄŒaut jÅ«su kolekcijÄ
+#***************
+#
+#***** HELP *****
+Help.Contents:Saturs
+Help.Title:BibliotekÄra Interfeisa PalÄ«ga Lappuses
+#
+#***** LockFileDialog *****
+LockFileDialog.Cancel_Tooltip:NepÄrtvert slÄdzejdatnes kontroli (kolekcija netiks atvÄrta)
+LockFileDialog.Date:DatÄjums
+LockFileDialog.Error:error
+LockFileDialog.Lockfile_Message_One:SlÄdzÄjdatnes (lock file) klÄtbÅ«tne norÄda uz to, ka kolekcija ir jau atvÄrta.
+LockFileDialog.Lockfile_Message_Two:Vai jÅ«s gribat 'pÄrtvert' slÄdzÄjdatnes kontroli?
+LockFileDialog.Machine:Kur
+LockFileDialog.Name:Nosaukums
+LockFileDialog.OK_Tooltip:PÄrtvert slÄdzÄjdatnes kontroli
+LockFileDialog.Title:SlÄdzÄjdatne AtklÄta
+LockFileDialog.User:KurÅ¡
+#********************
+#
+#*****MappingPrompt **********
+MappingPrompt.File:Mape
+MappingPrompt.Map:RadÄ«t ÄªsinÄjumikonu
+MappingPrompt.Name:Nosaukums
+MappingPrompt.Title:Uzdot Mapes ÄªsinÄjumikonu
+MappingPrompt.Unmap:NoÅemt ÄªsinÄjumikonu
+#*******************
+#
+#***** Menu Options *****
+Menu.Collapse:AizvÄrt Mapi
+Menu.Edit:RediÄ£Ät
+Menu.Edit_Copy:KopÄt (ctrl-c)
+Menu.Edit_Cut:Izgriezt (ctrl-x)
+Menu.Edit_Paste:IelÄ«mÄt (ctrl-v)
+Menu.Expand:AtvÄrt Mapi
+Menu.Explode_Metadata_Database:SadalÄ«t metadatu datu bÄzi
+Menu.File:Datne
+Menu.File_Associations:Datnes AsociÄcijas...
+Menu.File_CDimage:IerakstÄ«t CD/DVD attÄlu...
+Menu.File_Close:AizvÄrt
+Menu.File_Delete:IzdzÄst...
+Menu.File_Exit:Iziet
+Menu.File_ExportAs:EksportÄt...
+Menu.File_New:Jauns...
+Menu.File_Open:AtvÄrt...
+Menu.File_Options:IestatÄ«jumi...
+Menu.File_Save:SaglabÄt
+Menu.Help:PalÄ«gs
+Menu.Help_About:Par...
+Menu.Metadata_View:NozÄ«mÄtie metadati {0}
+Menu.Open_Externally:AtvÄrt ar ÄrÄjo programmu
+#********************
+#
+#***** MetaAudit *****
+MetaAudit.Close:AizvÄrt
+MetaAudit.Close_Tooltip:AizvÄrt Å¡o dialoglodziÅu
+MetaAudit.Title:Visi Metadati
+#*********************
+#
+#***** Metadata *****
+Metadata.Element:Elements
+Metadata.Value:VÄrtÄ«ba
+MetadataSet.Files:Metadatu Kopas Datnes
+#*********************************************
+#
+#******** MetadataImportMappingPrompt *********
+MIMP.Add:Pievienot
+MIMP.Add_Tooltip:Pievienot metadatu elementu atlasÄ«tai kopai
+MIMP.Ignore:IgnorÄt
+MIMP.Ignore_Tooltip:IgnorÄt Å¡o metadatu elementu
+MIMP.Instructions:Metadatu elements {0} nevar bÅ«t automÄtiski importÄts kolekcijÄ. IzvÄlieties metadatu kopu un sekojoÅ¡i:\n(1) nospiediet  'Pievienot', lai pievienotu metadatus Å¡ai kopai (pie nosacÄ«juma, ja kopÄ jau nÄesot metadatu {0}), vai\n(2) izvÄlieties elementu un nospiediet  'IekÄŒaut', lai importÄtu {0} metadatus Å¡ajÄ elementÄ, vai\n(3) ignorÄjiet Å¡o metadatu elementu.
+MIMP.Merge:IekÄŒaut
+MIMP.Merge_Tooltip:IekÄŒaut metadatu elelementu specificÄtÄ elementÄ izvÄlÄtÄ kopÄ
+MIMP.Source_Element:Avota metadatu elements:
+MIMP.Target_Element:MÄrÄ·a metadatu elements:
+MIMP.Target_Set:MÄrÄ·a metadatu kopa:
+MIMP.Title:NepieceÅ¡ama IekÄŒauÅ¡anas DarbÄ«ba.
+#*****************************
+#
+#***** Mirroring *****
+Mirroring.ClearCache:IztÄ«rÄ«t KeÅ¡u
+Mirroring.ClearCache_Tooltip:IzdzÄst visas lejupielÄdÄtas datnes
+Mirroring.Download:LejupielÄde
+Mirroring.Download_Tooltip:SÄkt jaunu lejupielÄdes uzdevumu
+Mirroring.Download_Controls:LejupielÄdes UzstatÄ«jumi
+Mirroring.Download_Depth:LejupielÄdes DziÄŒums:
+Mirroring.Download_Depth.Zero:Nulle
+Mirroring.Download_Depth.Unlimited:NeierobeÅŸots
+Mirroring.Download_Depth_Tooltip:Cik daudz hipersaites dziÄŒumÄ iet lejupielÄdÄjot
+Mirroring.Download_Embedded:LejupielÄdÄt tikai tekstu (ne attÄlus, ne stila lappuses utt)
+Mirroring.DownloadJob.Close:AizvÄrt
+Mirroring.DownloadJob.Close_Tooltip:NoÅemt Å¡o uzdevumu no saraksta, apstÄdinot to, ja tas patreiz izpildÄs
+Mirroring.DownloadJob.Downloading:{0} lejupielÄde
+Mirroring.DownloadJob.Download_Complete:LejupielÄde pabeigta
+Mirroring.DownloadJob.Download_Stopped:LejupielÄde apstÄdinÄta
+Mirroring.DownloadJob.Download_Progress:LejupielÄde ir progresÄ
+Mirroring.DownloadJob.Log:SkatÄ«t Log
+Mirroring.DownloadJob.Log_Tooltip:SkatÄ«t detalizÄtÄk par Å¡o lejupielÄdi
+Mirroring.DownloadJob.Pause:Pauze
+Mirroring.DownloadJob.Pause_Tooltip:PauzÄt Å¡o lejupielÄdi
+Mirroring.DownloadJob.Resume:AtsÄkt
+Mirroring.DownloadJob.Resume_Tooltip:AtsÄkt Å¡o lejupielÄdi
+Mirroring.DownloadJob.Status:LejupielÄdÄti {0} no {1} datnÄm ({2} brÄ«dinÄjumi, {3} kÄŒÅ«das)
+Mirroring.DownloadJob.Waiting:Gaida lai sÄktu
+Mirroring.DownloadJob.Waiting_User:Gaida uz lietotÄja darbÄ«bu
+Mirroring.Higher_Directories:Tikai atspoguÄŒot datnes zem Å¡Ä URL
+Mirroring.Invalid_URL:IevadÄ«tais URL nav pareizs. LÅ«dzu izlabojiet.
+Mirroring.Invalid_URL_Title:Nepareizs URL
+Mirroring.Preferences:IestatÄ«jumi...
+Mirroring.Preferences_Tooltip:AtÄŒaut starpnieku (proxy) un/vai rediÄ£Ät starpnieka iestatÄ«jumus
+Mirroring.Same_Host:Tikai atspoguÄŒot datnes Å¡ajÄ paÅ¡Ä vietnÄ
+Mirroring.Source_URL:Avota URL:
+Mirroring.Source_URL_Tooltip:LejupielÄdes URL
+#*********************
+#
+#***** Missing EXEC *****
+MissingEXEC.Message:BrÄ«dinÄjums! JÅ«s nevarÄsiet redzÄt jÅ«su jauno kolekciju no BibliotekÄra Interfeisa, ja jÅ«s nenorÄdÄ«siet ceÄŒu uz lokÄli darbojoÅ¡os Greenstone BibliotekÄra Webserveri. JÅ«s to varat tagad ievadÄ«t teksta lauciÅÄ zemÄk, vai vÄlÄk aizejot uz IestatÄ«jumi un izvÄloties Savienojums tabulatoru. Atcerieties, ka pareizam url jÄsÄkas ar protokolu  "http://", un jÄbeidzas ar izpildÄmÄs programmas nosaukumu ("gsdl" Greenstone lokÄlas bibliotÄkas servera gadÄ«jumÄ, "library" visu pÄrejo web serveru gadÄ«jumos (piem., Apache)).
+MissingEXEC.Title:IztrÅ«kst Greenstone BibliotÄkas Adrese
+#***** Missing EXEC_GS3 *****
+MissingEXEC_GS3.Message:BrÄ«dinÄjums! JÅ«s nevarÄsiet redzÄt jÅ«su jauno kolekciju no BibliotekÄra Interfeisa, ja jÅ«s nenorÄdÄ«siet ceÄŒu uz Greenstone kontekstu lokÄli darbojoÅ¡os Tomcat WebserverÄ«. JÅ«s to varat tagad ievadÄ«t teksta lauciÅÄ zemÄk, vai vÄlÄk aizejot uz IestatÄ«jumi un izvÄloties Savienojums tabulatoru. Atcerieties, ka pareizam url jÄsÄkas ar protokolu  http://. Adrese pÄc noklusÄjuma ir http://localhost:8080/gsdl3.
+MissingEXEC_GS3.Title:IztrÅ«kst Greenstone BibliotÄkas Adrese
+#***** Missing GSDL *****
+MissingGSDL.Message:BrÄ«dinÄjums! JÅ«s nevarÄsiet izmantot visas BibliotekÄra Interfeisa iespÄjas bez instalÄtÄs Greenstone DigitÄlÄs BibliotÄkas lokÄlÄs kopijas. LÅ«dzu, pÄrbaudiet jÅ«su instalÄciju, un ceÄŒus BibliotekÄra Interfeisa pakeÅ¡datnÄ vai skripta datnÄ.
+MissingGSDL.Title:IztrÅ«kst Greenstone CeÄŒÅ¡
+#***** Missing PERL *****
+MissingPERL.Message:BrÄ«dinÄjums! JÅ«s nevarÄsiet radÄ«t jaunas kolekcijas, kÄ arÄ« veidot esoÅ¡as, bez uzinstalÄtÄ PERL. BibliotekÄra Interfeiss nevarÄja atrast PERL jÅ«su sistÄmÄ. LÅ«dzu, pÄrbaudiet jÅ«su sistÄmas instalÄciju un ceÄŒus, kuri specificÄti BibliotekÄra Interfeisa pakeÅ¡datnÄ vai skripta datnÄ.
+MissingPERL.Title:IztrÅ«kst PERL CeÄŒÅ¡
+#******* MissingWGET ********
+MissingWGET.Message:BrÄ«dinÄjums! Web atspoguÄŒoÅ¡ana ir liegta, jo BibliotekÄra Interfeiss nesekmÄ«gi mÄÄ£inÄja atrast atbilstoÅ¡as versijas Wget. LÅ«dzu, pÄrbaudiet jÅ«su sistÄmas instalÄciju un ceÄŒus, kuri specificÄti BibliotekÄra Interfeisa pakeÅ¡datnÄ vai skripta datnÄ.
+MissingWGET.Title:IztrÅ«kst WGet
+#******* MissingImageMagick ********
+MissingImageMagick.Message:BrÄ«dinÄjums! BibliotekÄra Interfeiss nevar atrast attiecÄ«go ImageMagick versiju, un tÄpÄc ImagePlug nedarbosies. LÅ«dzu, uzinstalÄjiet ImageMagick, ja jÅ«s vÄlÄties veidot attÄlu kolekcijas.
+MissingImageMagick.Title:IztrÅ«kst ImageMagick  # Updated 19-Aug-2005
+#*****************************
+#
+#***** New Session *****
+NewCollectionPrompt.Base_Collection:BÄzÄt Å¡o kolekciju uz:
+NewCollectionPrompt.Base_Collection_Tooltip:IzvÄlieties esoÅ¡o kolekciju uz kuras bÄzÄt jauno kolekciju
+NewCollectionPrompt.Collection_Description:Satura apraksts:
+#NewCollectionPrompt.Collection_Email:Author's Email:
+NewCollectionPrompt.Collection_Name:Kolekcijas mape:
+#NewCollectionPrompt.Collection_Name_Tooltip:Short collection name, used internally (must be 8 characters or less)
+NewCollectionPrompt.Description_Error:Apraksta lauciÅam jÄbÅ«t aizpildÄ«tam. LÅ«dzu aizpildiet to.
+NewCollectionPrompt.Error:KÄŒÅ«da jaunajÄ kolekcijÄ.
+NewCollectionPrompt.Instructions:Lai radÄ«tu jaunu kolekciju, aizpildiet lauciÅus zemÄk.
+NewCollectionPrompt.Metadata_Elements:Elementi izvÄlÄtajÄ kopÄ:
+NewCollectionPrompt.Metadata_Instructions1:IzvÄlieties rÅ«tiÅas blakus metadatu kopÄm, kuras iekÄŒaut jÅ«su kolekcijÄ.
+NewCollectionPrompt.Metadata_Instructions2:(PiezÄ«me: JÅ«s varat pievienot jaunas kopas vÄlÄk).
+NewCollectionPrompt.Metadata_Title_Existing:IzvÄlieties metadatu kopas.
+#NewCollectionPrompt.Name_Error:You have either failed to enter a name for the collection, or the name you have choosen is already in use. Please correct.
+NewCollectionPrompt.NewCollection:-- Jauna Kolekcija --
+NewCollectionPrompt.Select_MDS:Pieejamas metadatu kopas:
+NewCollectionPrompt.Title:RadÄ«t jaunu Kolekciju.
+NewCollectionPrompt.Title_Clash:Nosaukums, kuru jÅ«s izvÄlÄjaties jÅ«su kolekcijai, jau tiek izmantots. TurpinÄt?
+NewCollectionPrompt.Title_Error:Nosaukuma lauciÅam jÄbÅ«t aizpildÄ«tam. LÅ«dzu izlabojiet.
+#***** New Folder OrFilePrompt *****
+NewFolderOrFilePrompt.Default_File_Name:fiktÄ«vs
+NewFolderOrFilePrompt.Default_Folder_Name:jauna mape
+NewFolderOrFilePrompt.Destination_Name:SaÅÄmÄjMape
+NewFolderOrFilePrompt.File_Name:Datnes VÄrds
+NewFolderOrFilePrompt.File_Name_Tooltip:Ievadiet nosaukumu jaunai datnei
+NewFolderOrFilePrompt.Folder_Name:Mapes VÄrds
+NewFolderOrFilePrompt.Folder_Name_Tooltip:Ievadiet nosaukumu jaunai mapei
+NewFolderOrFilePrompt.Title_Folder:RadÄ«t jaunu mapi
+NewFolderOrFilePrompt.Title_File:RadÄ«t jaunu fiktÄ«vu dokumentu
+#***********************
+#
+NoMetadataSetsSelected.Message:Ja jÅ«s radÄ«siet tagad kolekciju, tad tai nebÅ«s sÄkotnÄjas metadatu kopas. JÅ«s nevarÄsiet importÄt vai pieÅ¡Ä·irt jebkÄdus metadatus, lÄ«dz nebÅ«s importÄta vai pievienota kopa jÅ«su kolekcijai. Vai jÅ«s vÄlaties turpinÄt?
+NoMetadataSetsSelected.Title:Nav Metadatu Kopa izvÄlÄta
+#**********************
+#
+#******* OldWGET *******
+NoPluginExpectedToProcessFile.Message:IzskatÄs, ka neviens no Greenstone spraudÅiem varÄtu apstrÄdÄt datni "{0}". PÄrbaudiet, ka datnei ir pareizs paplaÅ¡inÄjums. Ja tas ir pareizs, tad jums vajadzÄtu izmantot UnknownPlug spraudni, lai apstrÄdÄtu Å¡o datni.
+NoPluginExpectedToProcessFile.Title:Nav paredzamÄ spraudÅa datnes apstrÄdei
+#**********************
+#
+#******* OldWGET *******
+OldWGET.Message:BrÄ«dinÄjums! JÅ«su instalÄtai Wget versijai ir zinÄma kÄŒÅ«da, kad tiek atspoguÄŒotas datnes, kurÄs ir URL, kas satur kodÄtas zÄ«mes, tÄdas kÄ, atstarpi  un tildi. MÄs rekomendÄjam instalÄt Wget 1.9 un augstÄk.
+OldWGET.Title:Vecs Wget
+#***********************
+#
+#***** OpenCollectionDialog *****
+OpenCollectionDialog.Available_Collections:Pieejamas Kolekcijas
+OpenCollectionDialog.Browse:PÄrlÅ«kot
+OpenCollectionDialog.Browse_Tooltip:IzvÄlieties kolekciju no jÅ«su datÅu telpas
+OpenCollectionDialog.Collection:Greenstone Kolekcija (".col")
+OpenCollectionDialog.Description:Kolekcijas Apraksts
+OpenCollectionDialog.Locked_Collection:Greenstone kolekcija jau tiek lietota
+OpenCollectionDialog.No_Description:LÅ«dzu izvÄlieties greenstone kolekciju (".col" datne), lai redzÄtu aprakstu.
+OpenCollectionDialog.Normal_Collection:Greenstone kolekcija
+OpenCollectionDialog.Open:AtvÄrt
+OpenCollectionDialog.Open_Tooltip:IelÄdÄt izvÄlÄto kolekciju BibliotekÄra InterfeisÄ
+OpenCollectionDialog.Title:AtvÄrt Greenstone Kolekciju
+#**********************
+#
+#***** Options Pane inside CreatePane *****
+OptionsPane.Cancelled: - anulÄts
+OptionsPane.LogHistory:Log vÄsture
+OptionsPane.Successful: - sekmÄ«gi
+OptionsPane.Unknown: - nezinÄms
+OptionsPane.Unsuccessful: - nesekmÄ«gi
+#*******************
+#
+#***** Preferences *****
+Preferences:IestatÄ«jumi
+Preferences.Connection:Savienojums
+Preferences.Connection_Tooltip:UzstÄdiet URL uz greenstone bibliotÄku un nokonfigurÄjiet starpniekservera savienojumu, ja vajadzÄ«gs
+Preferences.Connection.Library_Path:BibliotÄkas CeÄŒÅ¡:
+Preferences.Connection.Library_Path_GS3:Greenstone Web CeÄŒÅ¡:
+Preferences.Connection.Library_Path_Tooltip:Greenstone web-servera URL
+Preferences.Connection.Library_Path_Tooltip_GS3:Greenstone web-servera URL
+Preferences.Connection.ProgramCommand:PriekÅ¡skatÄ«juma Komanda:
+Preferences.Connection.ProgramCommand_Tooltip:Programmas palaiÅ¡anas komanda, lai skatÄ«tu kolekcijas priekÅ¡skatÄ«jumu. Atcerieties, ka jÄizmanto %1, kÄ kolekcijas adreses vietturi, un dubultpÄdiÅas, kur nepiecieÅ¡ams.
+Preferences.Connection.Proxy_Host:Starpniekserveris:
+Preferences.Connection.Proxy_Host_Tooltip:JÅ«su starpniekservera adrese
+Preferences.Connection.Proxy_Port:Starpniekservera pieslÄgvieta:
+Preferences.Connection.Proxy_Port_Tooltip:JÅ«su starpniekservera pieslÄgvietas numurs
+Preferences.Connection.Servlet:ServersÄ«klietotne:
+Preferences.Connection.Servlet_Tooltip:ServersÄ«klietotne izmantojama, lai skatÄ«tu jÅ«su vietni
+Preferences.Connection.Servlet_Tooltip2:ServersÄ«klietotne nav specificÄta Å¡ai vietnei. JÅ«s joprojÄm varat veidot kolekcijas, bet jÅ«s nevarÄsiet tÄs skatÄ«t.
+Preferences.Connection.Site:Vietne:
+Preferences.Connection.Site_Tooltip:Vietne kurÄ jÅ«s gribat strÄdÄt
+Preferences.Connection.Use_Proxy:Izmantot starpniekservera savienojumu?
+Preferences.General:VispÄrÄ«gs
+Preferences.General_Tooltip:Uzdot daÅŸus vispÄrigus iestatÄ«jumus
+Preferences.General.Email:LietotÄja Epasts:
+Preferences.General.Email_Tooltip:Epasta adrese automÄtiski izmantojama jebkurai kolekcijai radÄ«tai ar GBI
+Preferences.General.Font:Fonts:
+Preferences.General.Font_Tooltip:Fonts izmantojams GBI interfeisÄ.
+Preferences.General.Interface_Language:Interfeisa Valoda:
+Preferences.General.Interface_Language_Tooltip:IzvÄlieties valodu, kurÄ interfeiss un tÄ kontroles tiktu parÄdÄ«ti.
+Preferences.General.Restart_Required:GBI (Greenstone BibliotekÄra Interfeisam) tagad jÄrestartÄ, lai pabeigtu savu interfeisa izmaiÅu aktualizÄciju.
+Preferences.General.Show_File_Size:RÄdÄ«t datÅu izmÄrus
+Preferences.General.Show_File_Size_Tooltip:RÄdÄ«t datÅu izmÄrus pie katras ieejas DarbatelpÄ un Kolekcijas datÅu kokÄ.
+Preferences.General.View_Extracted_Metadata:SkatÄ«t Izvilktus Metadatus
+Preferences.General.View_Extracted_Metadata_Tooltip:ParÄdÄ«t greenstone spraudÅu izvilktos metadatus metadatu sarakstÄ BagÄtinÄÅ¡anas panelÄ«.
+Preferences.Mode:ReÅŸÄ«ms
+Preferences.Mode_Tooltip:NokliÅ¡Ä·iniet uz ieeju, lai izvÄletos piemÄrotu operÄÅ¡anas lÄ«meni
+Preferences.Mode.Assistant:BibliotekÄra Asistents
+Preferences.Mode.Assistant_Description:Izmantojiet Å¡o iestatÄ«jumu, lai iegÅ«tu piekÄŒuvi Greenstone BibliotekÄra Interfeisa pamata iespÄjÄm: pievienot dokumentus un metadatus kolekcijÄm, jauno kolekciju radÄ«Å¡ana, kuru struktÅ«ra atspoguÄŒo jau esoÅ¡as kolekcijas, un kolekciju veidoÅ¡ana.
+Preferences.Mode.Librarian:BibliotekÄrs
+Preferences.Mode.Librarian_Description:Å is iestatÄ«jums ÄŒauj normÄli izmantot Greenstone BibliotekÄra Interfeisu: radÄ«t un noformÄt jaunas kolekcijas, pievienot dokumentus un metadatus kolekcijÄm, jaunu kolekciju noformÄÅ¡anu (dizainu), un kolekciju veidoÅ¡anu (bÅ«vÄÅ¡anu).
+Preferences.Mode.Systems:BibliotÄku SistÄmu SpeciÄlists
+Preferences.Mode.Systems_Description: Å is iestatÄ«jums atÄŒauj pilnu Greenstone BibliotekÄra Interfeisa izmantoÅ¡anu. Lai izmantotu visu iespÄju priekÅ¡rocÄ«bas, jums bÅ«s nepiecieÅ¡ams mÄcÄt formulÄt regulÄrÄs izteiksmes.
+Preferences.Mode.Expert:Eksperts
+Preferences.Mode.Expert_Description:Å is iestatÄ«jums ÄŒauj ekspertam izmantot Greenstone BibliotekÄra Interfeisu. Tas tiek rekomendÄts lietotÄjiem, kuri ir pieredzÄjuÅ¡i darbam ar Greenstone, un ir spÄjÄ«gi veikt kÄŒÅ«du laboÅ¡anas uzdevumus, kas prasa interpretÄt atkÄŒÅ«doÅ¡anas izvadi no PERL skriptiem. Visas Greenstone BibliotekÄra Interfeisa iespÄjas ir atÄŒautas.
+Preferences.Warnings:BrÄ«dinÄjumi
+Preferences.Warnings_Tooltip:AtÄŒaut vai aizliegt daÅŸÄdus brÄ«dinÄjumu dialogus
+Preferences.Workflow:DarbplÅ«sma
+Preferences.Workflow_Tooltip:IestÄdÄ«t interfeisa darbplÅ«smu: noteikt, kurus skatus (tabulÄtorus) atÄŒaut
+Preferences.Workflow.Create:RadÄ«t - Veidot jÅ«su kolekciju izmantojot Greenstone
+Preferences.Workflow.Design:Dezains - IzmainÄ«t kolekcijas Ärieni
+Preferences.Workflow.Download:LejupielÄde - LejupielÄdÄt web-bÄzÄtos resursus
+Preferences.Workflow.Enrich:BagÄtinÄt - AprakstÄ«t ar metadatiem
+Preferences.Workflow.Gather:UzkrÄt - AtlasÄ«t datnes kolekcijai
+Preferences.Workflow.Predefined.Label:IepriekÅ¡noteiktas DarbplÅ«smas
+Preferences.Workflow.Title:IzvÄlieties rÅ«tiÅu, lai atÄŒautu skatu
+#**********************
+#
+#**** PreviewCommandDialog *******
+PreviewCommandDialog.Instructions:Ievadiet komandu, programmas palaiÅ¡anai, lai skatÄ«tu kolekcijas priekÅ¡skatÄ«jumu. Atcerieties izmantot %1, kÄ kolekcijas adreses vietturi, un dubultpÄdiÅas kur nepiecieÅ¡ams.
+PreviewCommandDialog.Title:PriekÅ¡skatÄ«juma Komanda
+#**********************
+#
+#***** Save Collection Box *****
+SaveCollectionBox.File_Name:Datnes VÄrds:
+SaveCollectionBox.Files_Of_Type:DatÅu Tips:
+SaveCollectionBox.Look_In:IelÅ«koties:
+#**********************
+#
+#***** Save Progress Box *******
+SaveProgressDialog.Title:Kolekcijas SaglabÄÅ¡ana
+#*******************************
+#
+Server.QuitManual:BibliotekÄra Interfeiss nevarÄja aizvÄrt lokÄlo\n bibliotÄkas serveri automÄtiski. LÅ«dzu izejiet no servera\nmanuÄli, noklikÅ¡Ä·inot, uz tÄ augÅ¡ÄjÄ stÅ«rÄ« labÄs\npuses krusta ikonas.
+Server.QuitTimeOut:Greenstone lokÄlÄ bibliotÄka nav atbildÄjusi uz\nautomÄtisko aizvÄrt komandu 60 sekundes. Vai jÅ«s\nvÄlaties gaidÄ«t vÄl minÅ«ti?
+#
+#***** Sources *****
+Source.General:VispÄrÄ«gs
+#*******************
+#
+#***** TitleClash *****
+TitleClashes.Message:Nosaukums, kuru jÅ«s esat izvÄlÄjuÅ¡ies jÅ«su kolekcijai, tiek jau lietots. TurpinÄt?
+TitleClashes.Title:IevadÄ«tÄ Nosaukuma nesaskaÅas
+#**********************
+#
+#***** Trees *****
+Tree.DownloadedFiles:LejupielÄdÄtÄs Datnes
+Tree.Files:datnes
+Tree.Home:SÄkummape ({0})
+Tree.Root:LokÄlÄ DatÅutelpa
+Tree.World:Dokumenti Greenstone KolekcijÄs
+#***********************
+#
+#***** Warning Dialog *****
+WarningDialog.Dont_Show_Again:NerÄdÄ«t Å¡o brÄ«dinÄjumu vÄlreiz
+WarningDialog.Dont_Show_Again_Message:NerÄdÄ«t Å¡o paziÅojumu vÄlreiz
+WarningDialog.Invalid_Value:VÄrtÄ«ba, kuru jÅ«s esat ievadÄ«juÅ¡i vÄrtÄ«bas lauciÅÄ,\nnav pareiza specificÄtam rekvizÄ«tam.
+WarningDialog.Value:Ievadiet vÄrtÄ«bu:
+#
+#****** WGet *****
+WGet.Prompt:Wget AtspoguÄŒoÅ¡anas Klients
+#
+#****** Workflows *****
+Workflow.AllPanels:Visi paneÄŒi
+Workflow.LocalSourceCollectionBuilding:LokÄla avota kolekcijas veidoÅ¡ana
+Workflow.DocumentEnriching:Dokumenta bagÄtinÄÅ¡ana
+
+GEMS.Already_Loaded_Set_Error: JÅ«su izvÄletÄ metadatu kopa patreiz ir atvÄrta.
+GEMS.Element_Already_Exists:Elements ar Å¡o vÄrdu jau eksistÄ.
+GEMS.Set_Already_Exists:Kopa ar doto vÄrdutelpu jau eksistÄ.
+GEMS.Cannot_Delete_Core_Sets_Error:JÅ«s nevarat izdzÄst GBI kodola metadatu kopu.
+
+GEMS.Preferences.Selected_Languages_Tooltip: IzvÄlieties parÄdÄ«Å¡anas valodu:
+
+GEMS.Add_Attribute:Pievienot atribÅ«tu
+GEMS.Add_Attribute_Tooltip:Pievienot jaunu atribÅ«tu izvÄlÄtam vienumam
+GEMS.Add_Element:Pievienot elementu
+GEMS.Add_Element_Tooltip:Pievienot jaunu elementu izvÄlÄtai kopai
+GEMS.Add_Subelement:Pieviento apakÅ¡elementu
+GEMS.Add_Subelement_Tooltip:Pievienot jaunu apakÅ¡elementu izvÄlÄtam elementam
+GEMS.Add_Set:Pieviento Kopu
+GEMS.Attribute:AtribÅ«ts
+GEMS.Attribute_Already_Exists:Å is atribÅ«ts jau eksistÄ.
+GEMS.Attribute_Name_Tooltip:JaunÄ atribÅ«ta nosaukums
+GEMS.Attribute_Language_Tooltip:JaunÄ atribÅ«ta valoda
+GEMS.Attribute_Value_Tooltip:JaunÄ atribÅ«ta vÄrtÄ«ba
+GEMS.Attributes:AtribÅ«ti
+GEMS.Cannot_Undo:(Å¡Ä« darbÄ«ba nevar tikt atsaukta)
+GEMS.Cannot_Add_Elements_To_Greenstone_MDS:JÅ«s nevarat manuÄli pievienot elementus Greenstone Metadatiem. Ja\nnepiecieÅ¡ams radÄ«t jaunu kopu, tad ta vietÄ pievienojiet jÅ«su jauno elementu tam.
+GEMS.Confirm_Exit_Save: Vai jÅ«s nevÄlaties saglabÄt pirms izieÅ¡anas?
+GEMS.Confirm_Exit_Save_Title: SaglabÄt Pirms IzieÅ¡anas?
+GEMS.Confirm_Removal:Vai jÅ«s esat pÄrliecinÄti, ka gribat izdzÄst Å¡o {0}?
+GEMS.Confirm_Removal_Title:ApstiprinÄt IzdzÄÅ¡anu
+GEMS.Edit_Attribute:RediÄ£Ät AttribÅ«tu
+GEMS.Edit_Value:RediÄ£Ät VÄrtÄ«bu
+GEMS.Edit_Value_Tooltip:IzmainÄ«t izvÄlÄto vÄrtÄ«bu
+GEMS.Element:Elements
+GEMS.Element_Name_Tooltip:JaunÄ elementa nosaukums
+GEMS.Element_Details:Metadatu Elementa DetaÄŒas
+GEMS.Language:Valoda
+GEMS.Language_Code:Valoda
+GEMS.Name:Nosaukums
+GEMS.Namespace:VÄrdutelpa
+GEMS.New_Name:Jauns Nosaukums
+GEMS.No_Description:Apraksts nav atrasts.
+GEMS.No_Name:Nosaukums nav atrasts.
+GEMS.Open_Set:AtvÄrt Kopu
+GEMS.Remove_Attribute:NoÅemt attribÅ«tu
+GEMS.Remove_Attribute_Tooltip:IzdzÄst izvÄlÄto atribÅ«tu
+GEMS.Remove_Element:NoÅemt elementu
+GEMS.Remove_Element_Tooltip:IzdzÄst izvÄlÄto elementu no kopas
+GEMS.Remove_Set:NoÅemt kopu
+GEMS.Remove_Set_Tooltip:IzdzÄst izvÄlÄto kopu
+GEMS.Rename_Element:PÄrdÄvÄt elementu
+GEMS.Rename_Element_Tooltip:Elementa jauns nosaukums
+GEMS.Rename_Name_Tooltip:Metadatu kopas jaunais nosaukums
+GEMS.Rename_Namespace_Tooltip:Metadatu kopas unikÄlÄ vÄrdutelpa
+GEMS.Rename_Set:PÄrdÄvÄt kopu
+GEMS.Rename_Set_Tooltip:PÄrdÄvÄt metadatu kopu
+GEMS.Set:Kopa
+GEMS.Set_Details:Metadatu Kopas DetaÄŒas
+GEMS.Set_Name_Tooltip:JaunÄs metadatu kopas vÄrds (nosaukums)
+GEMS.Set_Namespace_Tooltip:UnikÄlÄ vÄrdutelpa jaunai metadatu kopai
+GEMS.Subelement_Name_Tooltip:JaunÄ apakÅ¡elementa nosaukums
+GEMS.Subelement_Already_Exists:ApakÅ¡elements ar Å¡o nosaukumu jau eksistÄ.
+GEMS.Target:Elements
+GEMS.Title:Greenstone Metadatu Kopu Redaktors
+GEMS.Value:VÄrtÄ«ba
+GEMS.Values:VÄrtÄ«bas
+GEMS.inheritMetadataSet:PÄrmantot no Metadatu kopas:
+GEMS.Add_Set.No_Inherit:NepÄrmantot
Index: /main/tags/3.01/gli/classes/feedback.properties
===================================================================
--- /main/tags/3.01/gli/classes/feedback.properties	(revision 21112)
+++ /main/tags/3.01/gli/classes/feedback.properties	(revision 21112)
@@ -0,0 +1,264 @@
+TextComponent=Text Component
+true=True
+false=False
+Button=Button
+icon=icon
+NoImage=No Image
+ProgressBar=Progress Bar
+Slider=Slider
+Spinner=Spinner
+Label=Label
+ComboBox=Combo Box
+Selected=Selected
+NotSelected=Not Selected
+ToggleButton=Toggle Button
+MenuItem=Menu Item
+MenuBar=Menu Bar
+Window=Window
+Panel=Panel
+Scroller=Scroller
+OptionPane=Option Pane
+ViewPort=View Port
+Box=Box
+Splitters=Splitters
+List=List
+ColorChooser=Color Chooser
+PreviewPanel=Preview Panel
+ChooserPanels=Chooser Panels
+DesktopPane=Desktop Pane
+black=BLACK
+BLACK=BLACK
+blue=BLUE
+BLUE=BLUE
+cyan=CYAN
+CYAN=cyan
+gray=GRAY
+GRAY=GRAY
+green=GREEN
+GREEN=GREEN
+DARK_GRAY=DARK_GRAY
+darkGray=DARK_GRAY
+LIGHT_GRAY=LIGHT_GRAY
+lightGray=LIGHT_GRAY
+magenta=MAGENTA
+MAGENTA=MAGENTA
+orange=ORANGE
+ORANGE=ORANGE
+pink=PINK
+PINK=PINK
+red=RED
+RED=RED
+white=WHITE
+WHITE=WHITE
+YELLOW=YELLOW
+Table=Table
+row=Row
+ScrollerBar=Scroller Bar
+Tree=Tree
+Container=Container
+RootPane=Root Pane
+LayeredPane=Layered Pane
+InternalFrameTitlePane=Internal Frame Title Pane
+TableHeader=Table Header
+ToolBar=Tool Bar
+Conformation=Conformation
+ConformationText=Thank you for using this Send Feedback feature.\n\nDo you want to preview the report details you are about to send to us?\n
+Preview=Preview
+ConformationPreview=View information you about to send.
+Send=Send
+ConformationSend=Send all information inputted.
+Don'tSend=Don't Send
+NotSend=Not Send
+ConformationNotSend=Do not send all information inputted.
+content=content
+command=command
+feedback=feedback
+SendFeedback=Send Feedback
+SendFeedbackButton=Open Send feedback form
+SendFeedbackForm=Send Feedback Form
+FeedbackOpeningText=Feedback\n\nSending this report indicate that you have found the applications you were doing have some problems.You do not have to fill out the form -- any information will help.\n\nDescriptions:
+Whatkindofproblemisit=What kind of problem is it
+ContentError=Content Error
+ImageNotShow=Image Not Show
+StrangeBehaviour=Strange Behaviour
+SomethingUnexpectedHappen=Something Unexpected Happen
+HardToUse=Hard To Use
+Other=Other
+Howbadistheproblem=How bad is the problem
+Critical=Critical
+Serious=Serious
+Medium=Medium
+Minor=Minor
+Trivial=Trivial
+Doyouwanttosendthescreenshot=Do you want to send the screen shot
+Yes=Yes
+No=No
+FeedbackSendButton=Send all information inputted
+FeedbackNotSendButton=Do not send all information inputted
+FeedbackHelpText=How to report feedback\n\nYou do not have to fill in any of the form -- any information will help.\n\nClicking on thumbnail will allow you to draw scribble lines marking the problem and type more comments about it.\nIf you finish scribbling then you can click Finish Scribble which will take you back to Feedback form, or Send button which will send all the information you gave so far.\n\nYou can type your comment inside the comments text area.\n\nIf you go to any opened window and select screen shot then it will add the screen shot's thumbnail to problem details area.\n\nDetails... button will allow you to preview the xml files that are going to be send.\nSend button will send all the information away.
+FeedbackAboutText=About\n\nMany open source projects are under-utilised because normal users find the software they produce hard to use. The Send feedback button is an attempt by members of the Greenstone project to involve the users of Greenstone in making the software easier to use.\n\nEvery time you click a button in GLI, some information is collected about what you were doing in GLI at the time. This information can hopefully help the people working on the Greenstone project to figure out what caused the problems you had, and how to fix it.
+FeedbackPrivacyText=Privacy\n\nYour privacy is important to us. We assure you that we will take no information that can lead to your personal identification. Any information you provide will be used strictly for the research and for determining how to make Greenstone more usable, no other uses of the data will be allowed.\n\nNo information about anything other than GLI and the technology you use to view it (for example your screen size) will be taken.\n\nThe information we you provide will have a timestamp on it, and may be correlated with your activity within Greenstone around the time you made your complaint.
+Form=Form
+About=About
+Privacy=Privacy
+screen=screen
+ErrorLineAnd=ErrorLineAnd
+ErrorLineFor=ErrorLineFor
+Don'twanttosendscreenshot=Don't want to send the screenshot
+SendFeedback=Send Feedback
+CurrentlyOpenWindow=CurrentlyOpenWindow
+WindowActivated=Window Activated
+WindowOpened=Window Opened
+WindowClosed=Window Closed
+WindowDeactivated=Window Deactivated
+ChangeState=Change State
+Node=Node
+ValueChanged=ValueChanged
+Change=Change
+Inserted=Inserted
+Removed=Removed
+TreeStructuredChanged=Tree Structure Changed
+ItemStateChanged=Item State Changed
+ListChanged=List Changed
+ListContentsAdded=List Contents Added
+ListContentRemoved=List Content Removed
+ListValueChanged=List Value Changed
+MenuCanceled=Menu Canceled
+MenuDeselected=Menu Deselected
+MenuSelected=Menu Selected
+ColumnIndex=Column Index
+AddedToTable=Added to table
+ColumnIndexFrom=From column index
+MoveToColumnIndex=move to column index
+InTheTable=in the table
+RemovedFromTheTable= removed from the table
+ViewReport=View Report
+ViewFeedbackForm=View Feedback Form
+ViewHistoryState=View History State
+ViewCommandState=View Commands
+Close=Close
+ClosePreview=Close the preview window
+HISTORY=HISTORY
+BUGS=BUGS
+IDCode=IDCode
+LastViewedWindow=LastViewedWindow
+Optionals=Optionals
+Details=Details
+Type=Type
+Urgency=Urgency
+User=User
+Name=Name
+HomeDirectory=HomeDirectory
+WorkingDirectory=WorkingDirectory
+OpenFormTime=OpenFormTime
+SendFormTime=SendFormTime
+OperatingSystem=OperatingSystem
+JavaInformation=JavaInformation
+DefaultLocale=DefaultLocale
+Browser=Greenstone
+LocalHostName=LocalHostName
+LocalHostAddress=LocalHostAddress
+ScreenResolution=ScreenResolution
+ScreenShot=ScreenShot
+ErrorLineAndScreenShot=ErrorLineAndScreenShot
+ErrorLineForScreenShot=ErrorLineForScreenShot
+COMPONENT=COMPONENT
+Type=Type
+Title=Title
+Content=Content
+Visible=Visible
+Image=Image
+Date=Date
+Command=Command
+COMPONENTS=COMPONENTS
+LOG=LOG
+View=View
+Showwhereistheproblem=Show where is the problem
+ScreenShotLabel=Use the mouse to indicate where the problem occured. Click send once you finished.
+Rectangle=Rectangle
+RectangleToolTip=Tool to draw rectangle
+Arrow=Arrow
+ArrowToolTip=Tool to draw arrow line
+SendTheImage=Sending the image
+SMTPaddress=What is your SMTP address
+Emailaddress=What is your e-mail address
+SendAttachment=Hereby attached 2 jar files.Please download them at the same place and parser.jar is an executable jar file.\nPlease keep referencing to this ID Code\nID Code:
+Email=Email
+SMTP=SMTP
+HISTORYCOMMANDS=COMMANDS
+TotalMemory=TotalMemory
+MaxMemory=MaxMemory
+FreeMemory=FreeMemory
+ViewFileTitle=Viewing the XML file
+ViewFeedbackFile=View FeedbackInterface form file
+ViewFeedbackCommandFile=View feedback command's history file
+ViewFeedbackContentFile=View feedback's history file
+ViewContentFile=View History file
+ViewCommandFile=View Command's history file
+ParsingXMLText=Select the checkbox of the file you want to see
+View=View
+ViewToolTip=View the files selected from the check box
+CloseToolTip=Close this application
+ViewCode=Please enter the id code as written in your email
+COMMANDS=COMMANDS
+Pleasewait=Please wait this may take a while
+Java=Java
+InternalError=Sorry. There was an internal error while sending your message!Please make sure you connected to the internet.
+SendFailed=Send Failed.Please make sure you connected to the internet.Do you want to try again?
+Error=Error
+NotFoundProvider=Provider not Found!\n\nPlease enter your SMTP adress again
+NotFoundProvider2=Please enter the email address for that SMTP address.
+Emaillater=We will try to send this feedback documents next time you send another feedback document.
+ScreenShotDescription=ScreenShotDescription
+ProblemDetails=Problem Details :
+Size=Size
+Height=Height
+Width=Width
+Icon=Icon
+Viewfeedback=View Feedback
+Viewcontent=View Content
+Viewcommand=View Command
+Separator=Separator
+NoComponent=No Component (it's a null object)
+TitlePane=Metal Title Pane (Platform dependent)
+ToolTipText=ToolTipText
+Feedback=Feedback
+LookHistory=View History
+LookHistoryButton=View History of all the commands you've done
+RecreateEvent=Recreate the Event
+RecreateOpeningText=You can add your comments in the text area.Click the Screen Shot button to take the screenshot of this window.Click Send button once you finished recreating the event.
+SendRecreate=Send Recreate
+SaveRecreate=Save Recreate
+RecreateSaveButton=Save all the comments entered.
+Comment=Comment
+RecreateOption=Can you recreate the event that you want to report?
+RecreateTitle=Recreate Event
+Save=Save
+RecordReady=If you are ready to recreate the event click start.
+Start=Start
+StartRecreate=Start Recreate Event
+StartRecText=Start recording all the action you did.
+ViewFeedbackHistoryState=View Feedback History
+ViewFeedbackCommandState=View Feedback Command
+DLGmessage=We start recording every action you do now.
+Ok=Ok
+SnapShot=Screen Shot
+SnapShotText=SnapShotText
+SnapShotButton=Take screen shot of the window you are in now.
+ViewMovie=View Picture
+ViewMovieText=View all the sequence of action
+FramesPerSecond=Frames Per Second
+GenerateMovie=Frames sequences
+image=Image
+notfound=Not Found
+COMMENTS=COMMENTS
+Viewcomment=View comment
+ViewCommentFile=View Comment File
+SnapShotAppend=Screen Shot taken
+Start=Start
+Finish=Finish
+Sequences=Sequences
+Sequence=Sequence
+ImageDescription=ImageDescription
+
+
Index: /main/tags/3.01/gli/classes/xml/CollectionConfig.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/CollectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/CollectionConfig.xml	(revision 21112)
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<CollectionConfig>
+</CollectionConfig>
Index: /main/tags/3.01/gli/classes/xml/collect_cfg.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/collect_cfg.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/collect_cfg.xml	(revision 21112)
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE Configuration [
+  <!ELEMENT Configuration (Gatherer)>
+  <!ELEMENT Gatherer      (Workflow)>
+  <!ELEMENT Workflow      (Browse, Mirror, Import, Gather, Enrich, Design, Export, Preview)>
+  <!ELEMENT Browse        (#PCDATA)>
+  <!ELEMENT Mirror        (#PCDATA)>  
+  <!ELEMENT Gather        (#PCDATA)>
+  <!ELEMENT Enrich        (#PCDATA)>
+  <!ELEMENT Design        (#PCDATA)>
+  <!ELEMENT Export        (#PCDATA)>
+  <!ELEMENT Preview       (#PCDATA)>
+]>
+
+<Configuration>
+  <Gatherer>
+    <Workflow>
+	   <Browse  enabled="true"/>
+		<Mirror  enabled="true"/>
+		<Import  enabled="true"/>
+		<Gather  enabled="true"/>
+		<Enrich  enabled="true"/>
+		<Design  enabled="true"/>
+		<Export  enabled="true"/>
+		<Preview enabled="true"/>
+	 </Workflow>
+  </Gatherer>
+</Configuration>
+
+
Index: /main/tags/3.01/gli/classes/xml/config.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/config.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/config.xml	(revision 21112)
@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE Configuration [
+  <!ELEMENT Configuration  (GathererConfig, Other, DirectoryMappings, Associations)>
+  <!ELEMENT GathererConfig (Argument*)>
+  <!ELEMENT Argument       (#PCDATA)>
+  <!ATTLIST Argument
+            name           CDATA #REQUIRED>
+  <!ELEMENT Other          (Info*)>
+  <!ELEMENT Info           (Name, Desc, Arguments)>
+  <!ELEMENT Arguments      (Option*)>
+  <!ELEMENT Option         (Name, Desc, Type, Required, Default?, List?)>
+  <!ELEMENT Type           (#PCDATA)>
+  <!ELEMENT Required       (#PCDATA)>
+  <!ELEMENT Default        (#PCDATA)>
+  <!ELEMENT List           (Value*)>
+  <!ELEMENT Value          (Name, Desc?)>
+  <!ELEMENT Name           (#PCDATA)>
+  <!ELEMENT Desc           (#PCDATA)>
+  <!ELEMENT DirectoryMappings (Mapping*)>
+  <!ELEMENT Mapping           (#PCDATA)>
+  <!ATTLIST Mapping 
+            name              CDATA #REQUIRED
+	    file              CDATA #REQUIRED>
+  <!ELEMENT Associations (Entry*)>
+  <!ELEMENT Entry (#PCDATA)>
+  <!ATTLIST Entry
+            extension CDATA #REQUIRED>
+]>
+
+<Configuration version="2.62">
+  <GathererConfig>
+    <!-- ***** General Options: apply globally to the GLI execution. ***** -->
+    <!-- The location and size of the main Gatherer window. -->
+    <Argument name="general.bounds">0, 0, 800, 600</Argument>
+    <!-- Are we expected to produce a debug file. Can be overidden at start up. default = [false] -->
+    <Argument name="general.debug">false</Argument>
+    <Argument name="general.email"/>
+    <!-- The standard font used in the Gatherer. [Verdana, BOLD, 12] -->
+    <Argument name="general.font">Verdana, BOLD, 12</Argument>
+    <Argument name="general.library_url"/>
+    <Argument name="general.locale">en_NZ</Argument>
+    <!-- How many folders deep should we recurse before suspecting a cyclic path? [40] -->
+    <Argument name="general.max_folder_depth">40</Argument>
+    <!-- The mode controls what level a detail is presented to the user - ranging from a basic, no regex, no perl experience up to complete details for all processes. The values, ranging from lowest to highest detail, are: 1 = Assistant, 2 = Librarian, 3 = Systems, 4 = Expert [default 2]-->
+    <Argument name="general.mode">2</Argument>
+    <Argument name="general.open_collection"></Argument>
+    <!-- The application launching command used when previewing the collection. [default -blank-] -->
+    <Argument name="general.preview_program"></Argument>
+    <Argument name="general.proxy_host"></Argument>
+    <Argument name="general.proxy_port"></Argument>
+    <Argument name="general.show_file_size">false</Argument>
+    <!-- Do we expect to connect to the internet via a proxy server. [false] -->
+    <Argument name="general.use_proxy">false</Argument>
+    <Argument name="general.view_extracted_metadata">true</Argument>
+
+    <!-- Colouring -->
+    <Argument name="coloring.button_background">218, 237, 252</Argument>
+    <Argument name="coloring.button_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.button_selected_background">128, 180, 216</Argument>
+    <Argument name="coloring.button_selected_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.collection_heading_background">176, 208, 176</Argument>  
+    <Argument name="coloring.collection_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_selection_background">128, 180, 216</Argument>
+    <Argument name="coloring.collection_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_tree_background">224, 240, 224</Argument>
+    <Argument name="coloring.collection_tree_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_noneditable_background">192, 192, 192</Argument>
+    <Argument name="coloring.collection_noneditable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.disabled">206, 206, 206</Argument>
+    <Argument name="coloring.editable_background">255, 255, 255</Argument>
+    <Argument name="coloring.editable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.error_background">255, 0, 0</Argument>
+    <Argument name="coloring.error_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.scrollbar_background">192, 192, 192</Argument>
+    <Argument name="coloring.scrollbar_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.workspace_heading_background">128, 180, 216</Argument>
+    <Argument name="coloring.workspace_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_selection_background">176, 208, 176</Argument>
+    <Argument name="coloring.workspace_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_tree_background">218, 237, 252</Argument>
+    <Argument name="coloring.workspace_tree_foreground">0, 0, 0</Argument>
+
+    <!-- Mirroring -->
+    <!-- Mirroring has many particular options. These show the default settings. -->
+    <Argument name="mirroring.depth">0</Argument>
+    <Argument name="mirroring.no_parents">true</Argument>
+    <Argument name="mirroring.other_hosts">false</Argument>
+    <Argument name="mirroring.overwrite">false</Argument>
+    <Argument name="mirroring.page_requisites">true</Argument>
+    <Argument name="mirroring.quiet">true</Argument>
+    <Argument name="mirroring.remove_failed">false</Argument>
+
+    <!-- Warnings -->
+    <!-- Warnings can be disabled by the user. -->
+    <Argument name="warning.CollectionBuilt">true</Argument>
+    <Argument name="warning.DirectoryLevelMetadata">true</Argument>
+    <Argument name="warning.ExtractedMetadata">true</Argument>
+    <Argument name="warning.MissingEXEC">true</Argument>
+    <Argument name="warning.MissingEXEC_GS3">true</Argument>
+    <Argument name="warning.MissingGSDL">true</Argument>
+    <Argument name="warning.MissingImageMagick">true</Argument>
+    <Argument name="warning.MissingPERL">true</Argument>
+    <Argument name="warning.MissingWGET">true</Argument>
+    <Argument name="warning.NoMetadataSetsSelected">true</Argument>
+    <Argument name="warning.NoPluginExpectedToProcessFile">true</Argument>
+    <Argument name="warning.OldWGET">true</Argument>
+    <Argument name="warning.TitleClashes">true</Argument>
+
+    <!-- Workflow -->
+    <!-- Disabling a view here will cause it to be hidden from the Gatherer display. -->
+    <Argument name="workflow.download">true</Argument>
+    <Argument name="workflow.gather">true</Argument>
+    <Argument name="workflow.enrich">true</Argument>
+    <Argument name="workflow.design">true</Argument>
+    <Argument name="workflow.create">true</Argument>
+
+    <Argument name="GEMS.Preferences.Selected_Languages">en</Argument>
+  </GathererConfig>
+
+  <Other/>
+  <DirectoryMappings/>
+  <Associations>
+    <Entry extension="doc"/>
+    <Entry extension="gif"/>
+    <Entry extension="htm"/>
+    <Entry extension="html"/>
+    <Entry extension="jpg"/>
+    <Entry extension="jpeg"/>
+    <Entry extension="pdf"/>
+    <Entry extension="png"/>
+    <Entry extension="ps"/>
+    <Entry extension="rtf"/>
+    <Entry extension="txt"/>
+    <Entry extension="text"/>
+  </Associations>
+</Configuration>
Index: /main/tags/3.01/gli/classes/xml/configRemote.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/configRemote.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/configRemote.xml	(revision 21112)
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE Configuration [
+  <!ELEMENT Configuration  (GathererConfig, Other, DirectoryMappings, Associations)>
+  <!ELEMENT GathererConfig (Argument*)>
+  <!ELEMENT Argument       (#PCDATA)>
+  <!ATTLIST Argument
+            name           CDATA #REQUIRED>
+  <!ELEMENT Other          (Info*)>
+  <!ELEMENT Info           (Name, Desc, Arguments)>
+  <!ELEMENT Arguments      (Option*)>
+  <!ELEMENT Option         (Name, Desc, Type, Required, Default?, List?)>
+  <!ELEMENT Type           (#PCDATA)>
+  <!ELEMENT Required       (#PCDATA)>
+  <!ELEMENT Default        (#PCDATA)>
+  <!ELEMENT List           (Value*)>
+  <!ELEMENT Value          (Name, Desc?)>
+  <!ELEMENT Name           (#PCDATA)>
+  <!ELEMENT Desc           (#PCDATA)>
+  <!ELEMENT DirectoryMappings (Mapping*)>
+  <!ELEMENT Mapping           (#PCDATA)>
+  <!ATTLIST Mapping 
+            name              CDATA #REQUIRED
+	    file              CDATA #REQUIRED>
+  <!ELEMENT Associations (Entry*)>
+  <!ELEMENT Entry (#PCDATA)>
+  <!ATTLIST Entry
+            extension CDATA #REQUIRED>
+]>
+
+<Configuration version="2.62">
+  <GathererConfig>
+    <!-- ***** General Options: apply globally to the GLI execution. ***** -->
+    <!-- The location and size of the main Gatherer window. -->
+    <Argument name="general.bounds">0, 0, 800, 600</Argument>
+    <!-- Are we expected to produce a debug file. Can be overidden at start up. default = [false] -->
+    <Argument name="general.debug">false</Argument>
+    <!-- The standard font used in the Gatherer. [Verdana, BOLD, 12] -->
+    <Argument name="general.font">Verdana, BOLD, 12</Argument>
+    <Argument name="general.gliserver_url"/>
+    <Argument name="general.library_url"/>
+    <Argument name="general.locale">en_NZ</Argument>
+    <!-- How many folders deep should we recurse before suspecting a cyclic path? [40] -->
+    <Argument name="general.max_folder_depth">40</Argument>
+    <!-- The mode controls what level a detail is presented to the user - ranging from a basic, no regex, no perl experience up to complete details for all processes. The values, ranging from lowest to highest detail, are: 1 = Assistant, 2 = Librarian, 3 = Systems, 4 = Expert [default 2]-->
+    <Argument name="general.mode">2</Argument>
+    <Argument name="general.open_collection"></Argument>
+    <!-- The application launching command used when previewing the collection. [default -blank-] -->
+    <Argument name="general.preview_program"></Argument>
+    <Argument name="general.proxy_host"></Argument>
+    <Argument name="general.proxy_port"></Argument>
+    <Argument name="general.show_file_size">false</Argument>
+    <!-- Do we expect to connect to the internet via a proxy server. [false] -->
+    <Argument name="general.use_proxy">false</Argument>
+    <Argument name="general.view_extracted_metadata">true</Argument>
+
+    <!-- Colouring -->
+    <Argument name="coloring.button_background">218, 237, 252</Argument>
+    <Argument name="coloring.button_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.button_selected_background">128, 180, 216</Argument>
+    <Argument name="coloring.button_selected_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.collection_heading_background">176, 208, 176</Argument>  
+    <Argument name="coloring.collection_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_selection_background">128, 180, 216</Argument>
+    <Argument name="coloring.collection_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_tree_background">224, 240, 224</Argument>
+    <Argument name="coloring.collection_tree_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_noneditable_background">192, 192, 192</Argument>
+    <Argument name="coloring.collection_noneditable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.disabled">206, 206, 206</Argument>
+    <Argument name="coloring.editable_background">255, 255, 255</Argument>
+    <Argument name="coloring.editable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.error_background">255, 0, 0</Argument>
+    <Argument name="coloring.error_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.scrollbar_background">192, 192, 192</Argument>
+    <Argument name="coloring.scrollbar_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.workspace_heading_background">128, 180, 216</Argument>
+    <Argument name="coloring.workspace_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_selection_background">176, 208, 176</Argument>
+    <Argument name="coloring.workspace_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_tree_background">218, 237, 252</Argument>
+    <Argument name="coloring.workspace_tree_foreground">0, 0, 0</Argument>
+
+    <!-- Mirroring -->
+    <!-- Mirroring has many particular options. These show the default settings. -->
+    <Argument name="mirroring.depth">0</Argument>
+    <Argument name="mirroring.no_parents">true</Argument>
+    <Argument name="mirroring.other_hosts">false</Argument>
+    <Argument name="mirroring.overwrite">false</Argument>
+    <Argument name="mirroring.page_requisites">true</Argument>
+    <Argument name="mirroring.quiet">true</Argument>
+    <Argument name="mirroring.remove_failed">false</Argument>
+
+    <!-- Warnings -->
+    <!-- Warnings can be disabled by the user. -->
+    <Argument name="warning.CollectionBuilt">true</Argument>
+    <Argument name="warning.DirectoryLevelMetadata">true</Argument>
+    <Argument name="warning.ExtractedMetadata">true</Argument>
+    <Argument name="warning.InheritedMetadata">true</Argument>
+    <Argument name="warning.MissingEXEC">true</Argument>
+    <Argument name="warning.MissingGSDL">true</Argument>
+    <Argument name="warning.MissingGLIServer">true</Argument>
+    <Argument name="warning.MissingImageMagick">true</Argument>
+    <Argument name="warning.MissingPERL">true</Argument>
+    <Argument name="warning.MissingWGET">true</Argument>
+    <Argument name="warning.NoMetadataSetsSelected">true</Argument>
+    <Argument name="warning.NoPluginExpectedToProcessFile">true</Argument>
+    <Argument name="warning.OldWGET">true</Argument>
+    <Argument name="warning.TitleClashes">true</Argument>
+
+    <!-- Workflow -->
+    <!-- Disabling a view here will cause it to be hidden from the Gatherer display. -->
+    <Argument name="workflow.download">false</Argument>
+    <Argument name="workflow.gather">true</Argument>
+    <Argument name="workflow.enrich">true</Argument>
+    <Argument name="workflow.design">true</Argument>
+    <Argument name="workflow.create">true</Argument>
+  </GathererConfig>
+
+  <Other/>
+  <DirectoryMappings/>
+  <Associations>
+    <Entry extension="doc"/>
+    <Entry extension="gif"/>
+    <Entry extension="htm"/>
+    <Entry extension="html"/>
+    <Entry extension="jpg"/>
+    <Entry extension="jpeg"/>
+    <Entry extension="pdf"/>
+    <Entry extension="png"/>
+    <Entry extension="ps"/>
+    <Entry extension="rtf"/>
+    <Entry extension="txt"/>
+    <Entry extension="text"/>
+  </Associations>
+</Configuration>
Index: /main/tags/3.01/gli/classes/xml/gli.lck
===================================================================
--- /main/tags/3.01/gli/classes/xml/gli.lck	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/gli.lck	(revision 21112)
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="ASCII" ?>
+
+<LockFile>
+  <User></User>
+  <Machine></Machine>
+  <Date></Date>
+</LockFile>
Index: /main/tags/3.01/gli/classes/xml/languages.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/languages.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/languages.xml	(revision 21112)
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE Languages [
+   <!ELEMENT Languages     (Language*)>
+   <!ELEMENT Language      (#PCDATA)>
+   <!-- name: The name of the language -->
+   <!-- code: The two letter language code -->
+   <!-- gs:   Does greenstone include support for this language? -->
+   <!-- gli:  Does gli have a dictionary for this language? -->
+   <!-- mds:  Does any metadata set include support for this language? -->
+   <!ATTLIST Language
+             name          CDATA #REQUIRED
+	     code          CDATA #REQUIRED
+	     gs            CDATA "false"
+	     gli           CDATA "false"
+	     mds           CDATA "false">
+]>
+
+<Languages>
+   <Language name="ABKHAZIAN"           code="AB">IBERO-CAUCASIAN</Language>
+   <Language name="AFAN(OROMO)"         code="OM">HAMITIC</Language>
+   <Language name="AFAR"                code="AA">HAMITIC</Language>
+   <Language name="AFRIKAANS"           code="AF">GERMANIC</Language>
+   <Language name="ALBANIAN"            code="SQ">INDO-EUROPEAN (OTHER)</Language>
+   <Language name="AMHARIC"             code="AM">SEMITIC</Language>
+   <Language name="ARABIC"              code="AR" gs="true" mds="true">SEMITIC</Language>
+   <Language name="ARMENIAN"            code="HY" gs="true">INDO-EUROPEAN (OTHER)</Language>
+   <Language name="ASSAMESE"            code="AS">INDIAN</Language>
+   <Language name="AYMARA"              code="AY">AMERINDIAN</Language>
+   <Language name="AZERBAIJANI"         code="AZ">TURKIC/ALTAIC</Language>
+   <Language name="BASHKIR"             code="BA">TURKIC/ALTAIC</Language>
+   <Language name="BASQUE"              code="EU">BASQUE</Language>
+   <Language name="BENGALI(BANGLA)"     code="BN">INDIAN</Language>
+   <Language name="BHUTANI"             code="DZ">ASIAN</Language>
+   <Language name="BIHARI"              code="BH">INDIAN</Language>
+   <Language name="BISLAMA"             code="BI">[not given]</Language>
+   <Language name="BRETON"              code="BR">CELTIC</Language>
+   <Language name="BULGARIAN"           code="BG">SLAVIC</Language>
+   <Language name="BURMESE"             code="MY">ASIAN</Language>
+   <Language name="BYELORUSSIAN"        code="BE">SLAVIC</Language>
+   <Language name="CAMBODIAN"           code="KM">ASIAN</Language>
+   <Language name="CATALAN"             code="CA">ROMANCE</Language>
+   <Language name="CHINESE-SIMPLIFIED"  code="CN" gs="true" mds="true">ASIAN</Language>
+   <Language name="CHINESE-TRADITIONAL" code="ZH" gs="true">ASIAN</Language>
+   <Language name="CORSICAN"            code="CO">ROMANCE</Language>
+   <Language name="CROATIAN"            code="HR" gs="true">SLAVIC</Language>
+   <Language name="CZECH"               code="CS" gs="true" mds="true">SLAVIC</Language>
+   <Language name="DANISH"              code="DA">GERMANIC</Language>
+   <Language name="DUTCH"               code="NL" gs="true" mds="true">GERMANIC</Language>
+   <Language name="ENGLISH"             code="EN" gs="true" gli="true" mds="true">GERMANIC</Language>
+   <Language name="ESPERANTO"           code="EO">INTERNATIONAL AUX.</Language>
+   <Language name="ESTONIAN"            code="ET">FINNO-UGRIC</Language>
+   <Language name="FAROESE"             code="FO">GERMANIC</Language>
+   <Language name="FIJI"                code="FJ">OCEANIC/INDONESIAN</Language>
+   <Language name="FINNISH"             code="FI" gs="true" mds="true">FINNO-UGRIC</Language>
+   <Language name="FRENCH"              code="FR" gs="true" gli="true" mds="true">ROMANCE</Language>
+   <Language name="FRISIAN"             code="FY">GERMANIC</Language>
+   <Language name="GALICIAN"            code="GL" gs="true">ROMANCE</Language>
+   <Language name="GEORGIAN"            code="KA">IBERO-CAUCASIAN</Language>
+   <Language name="GERMAN"              code="DE" gs="true">GERMANIC</Language>
+   <Language name="GREEK"               code="EL" gs="true">LATIN/GREEK</Language>
+   <Language name="GREENLANDIC"         code="KL">ESKIMO</Language>
+   <Language name="GUARANI"             code="GN">AMERINDIAN</Language>
+   <Language name="GUJARATI"            code="GU">INDIAN</Language>
+   <Language name="HAUSA"               code="HA">NEGRO-AFRICAN</Language>
+   <Language name="HEBREW"              code="IW" gs="true">SEMITIC [*Changed back from HE in 2003 by us. Not valid iso639, HE]</Language>
+   <Language name="HINDI"               code="HI" gs="true">INDIAN</Language>
+   <Language name="HUNGARIAN"           code="HU">FINNO-UGRIC</Language>
+   <Language name="ICELANDIC"           code="IS">GERMANIC</Language>
+   <Language name="INDONESIAN"          code="ID" gs="true">OCEANIC/INDONESIAN [*Changed 1989 from original ISO 639:1988, IN]</Language>
+   <Language name="INTERLINGUA"         code="IA" mds="true">INTERNATIONAL AUX.</Language>
+   <Language name="INTERLINGUE"         code="IE">INTERNATIONAL AUX.</Language>
+   <Language name="INUKTITUT"           code="IU">[        ]</Language>
+   <Language name="INUPIAK"             code="IK">ESKIMO</Language>
+   <Language name="IRISH"               code="GA">CELTIC</Language>
+   <Language name="ITALIAN"             code="IT" gs="true">ROMANCE</Language>
+   <Language name="JAPANESE"            code="JA" gs="true">ASIAN</Language>
+   <Language name="JAVANESE"            code="JV">OCEANIC/INDONESIAN</Language>
+   <Language name="KANNADA"             code="KN" gs="true">DRAVIDIAN</Language>
+   <Language name="KASHMIRI"            code="KS">INDIAN</Language>
+   <Language name="KAZAKH"              code="KZ" gs="true">TURKIC/ALTAIC [*Changed 2003 by us. Not valid iso639, KK]</Language>
+   <Language name="KINYARWANDA"         code="RW">NEGRO-AFRICAN</Language>
+   <Language name="KIRGHIZ"             code="KY">TURKIC/ALTAIC</Language>
+   <Language name="KOREAN"              code="KO">ASIAN</Language>
+   <Language name="KURDISH"             code="KU">IRANIAN</Language>
+   <Language name="KURUNDI"             code="RN">NEGRO-AFRICAN</Language>
+   <Language name="LAOTHIAN"            code="LO">ASIAN</Language>
+   <Language name="LATIN"               code="LA">LATIN/GREEK</Language>
+   <Language name="LATVIAN (LETTISH)"   code="LV" gli="true">BALTIC</Language>
+   <Language name="LINGALA"             code="LN">NEGRO-AFRICAN</Language>
+   <Language name="LITHUANIAN"          code="LT">BALTIC</Language>
+   <Language name="MACEDONIAN"          code="MK">SLAVIC</Language>
+   <Language name="MALAGASY"            code="MG">OCEANIC/INDONESIAN</Language>
+   <Language name="MALAY"               code="MS">OCEANIC/INDONESIAN</Language>
+   <Language name="MALAYALAM"           code="ML">DRAVIDIAN</Language>
+   <Language name="MALTESE"             code="MT">SEMITIC</Language>
+   <Language name="MAORI"               code="MI" gs="true" mds="true">OCEANIC/INDONESIAN</Language>
+   <Language name="MARATHI"             code="MR" mds="true">INDIAN</Language>
+   <Language name="MOLDAVIAN"           code="MO">ROMANCE</Language>
+   <Language name="MONGOLIAN"           code="MN">[not given]</Language>
+   <Language name="NAURU"               code="NA">[not given]</Language>
+   <Language name="NEPALI"              code="NE">INDIAN</Language>
+   <Language name="NORWEGIAN"           code="NO">GERMANIC</Language>
+   <Language name="OCCITAN"             code="OC">ROMANCE</Language>
+   <Language name="ORIYA"               code="OR">INDIAN</Language>
+   <Language name="PASHTO(PUSHTO)"      code="PS">IRANIAN</Language>
+   <Language name="PERSIAN(FARSI)"      code="FA">IRANIAN</Language>
+   <Language name="POLISH"              code="PL">SLAVIC</Language>
+   <Language name="PORTUGUESE"          code="PT" gs="true">ROMANCE</Language>
+   <Language name="PUNJABI"             code="PA">INDIAN</Language>
+   <Language name="QUECHUA"             code="QU">AMERINDIAN</Language>
+   <Language name="RHAETO-ROMANCE"      code="RM">ROMANCE</Language>
+   <Language name="ROMANIAN"            code="RO">ROMANCE</Language>
+   <Language name="RUSSIAN"             code="RU" gs="true" gli="true" mds="true">SLAVIC</Language>
+   <Language name="SAMOAN"              code="SM">OCEANIC/INDONESIAN</Language>
+   <Language name="SANGHO"              code="SG">NEGRO-AFRICAN</Language>
+   <Language name="SANSKRIT"            code="SA">INDIAN</Language>
+   <Language name="SCOTS(GAELIC)"       code="GD">CELTIC</Language>
+   <Language name="SERBIAN"             code="SR">SLAVIC</Language>
+   <Language name="SERBO-CROATIAN"      code="SH">SLAVIC</Language>
+   <Language name="SESOTHO"             code="ST">NEGRO-AFRICAN</Language>
+   <Language name="SETSWANA"            code="TN">NEGRO-AFRICAN</Language>
+   <Language name="SHONA"               code="SN">NEGRO-AFRICAN</Language>
+   <Language name="SINDHI"              code="SD">INDIAN</Language>
+   <Language name="SINGHALESE"          code="SI">INDIAN</Language>
+   <Language name="SISWATI"             code="SS">NEGRO-AFRICAN</Language>
+   <Language name="SLOVAK"              code="SK">SLAVIC</Language>
+   <Language name="SLOVENIAN"           code="SL">SLAVIC</Language>
+   <Language name="SOMALI"              code="SO">HAMITIC</Language>
+   <Language name="SPANISH"             code="ES" gs="true" gli="true" mds="true">ROMANCE</Language>
+   <Language name="SUNDANESE"           code="SU">OCEANIC/INDONESIAN</Language>
+   <Language name="SWAHILI"             code="SW">NEGRO-AFRICAN</Language>
+   <Language name="SWEDISH"             code="SV">GERMANIC</Language>
+   <Language name="TAGALOG"             code="TL">OCEANIC/INDONESIAN</Language>
+   <Language name="TAJIK"               code="TG">IRANIAN</Language>
+   <Language name="TAMIL"               code="TA">DRAVIDIAN</Language>
+   <Language name="TATAR"               code="TT">TURKIC/ALTAIC</Language>
+   <Language name="TELUGU"              code="TE">DRAVIDIAN</Language>
+   <Language name="THAI"                code="TH" gs="true">ASIAN</Language>
+   <Language name="TIBETAN"             code="BO">ASIAN</Language>
+   <Language name="TIGRINYA"            code="TI">SEMITIC</Language>
+   <Language name="TONGA"               code="TO">OCEANIC/INDONESIAN</Language>
+   <Language name="TSONGA"              code="TS">NEGRO-AFRICAN</Language>
+   <Language name="TURKISH"             code="TR" gs="true">TURKIC/ALTAIC</Language>
+   <Language name="TURKMEN"             code="TK">TURKIC/ALTAIC</Language>
+   <Language name="TWI"                 code="TW">NEGRO-AFRICAN</Language>
+   <Language name="UIGUR"               code="UG">[       ]</Language>
+   <Language name="UKRAINIAN"           code="UK" gs="true">SLAVIC</Language>
+   <Language name="URDU"                code="UR">INDIAN</Language>
+   <Language name="UZBEK"               code="UZ">TURKIC/ALTAIC</Language>
+   <Language name="VIETNAMESE"          code="VI">ASIAN</Language>
+   <Language name="VOLAPUK"             code="VO">INTERNATIONAL AUX.</Language>
+   <Language name="WELSH"               code="CY">CELTIC</Language>
+   <Language name="WOLOF"               code="WO">NEGRO-AFRICAN</Language>
+   <Language name="XHOSA"               code="XH">NEGRO-AFRICAN</Language>
+   <Language name="YIDDISH"             code="YI">GERMANIC [*Changed 1989 from original ISO 639:1988, JI]</Language>
+   <Language name="YORUBA"              code="YO">NEGRO-AFRICAN</Language>
+   <Language name="ZHUANG"              code="ZA">[       ]</Language>
+   <Language name="ZULU"                code="ZU">NEGRO-AFRICAN</Language>
+</Languages>
Index: /main/tags/3.01/gli/classes/xml/metadata.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/metadata.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/metadata.xml	(revision 21112)
@@ -0,0 +1,3 @@
+<?xml version="1.0"  encoding="UTF-8" standalone="no"?>
+<!DOCTYPE DirectoryMetadata SYSTEM "http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd">
+<DirectoryMetadata/>
Index: /main/tags/3.01/gli/classes/xml/profile.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/profile.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/profile.xml	(revision 21112)
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE Profiles [
+<!ELEMENT Profiles (Action*)>
+<!ELEMENT Action (#PCDATA)>
+<!ATTLIST Action
+                source     CDATA #REQUIRED
+                target     CDATA "">
+]>
+<Profiles>
+</Profiles>
Index: /main/tags/3.01/gli/classes/xml/template.col
===================================================================
--- /main/tags/3.01/gli/classes/xml/template.col	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/template.col	(revision 21112)
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE GathererCollection [
+  <!ELEMENT GathererCollection (Argument*, GathererConfig, BuildConfig)>
+  <!ELEMENT GathererConfig    (Arguments*)>
+  <!ELEMENT BuildConfig       (Build, Import)>
+  <!ELEMENT Build             (Arguments*)>
+  <!ELEMENT Import            (Arguments*)>
+  <!ELEMENT Argument          (#PCDATA)>
+  <!ATTLIST Argument
+            name              CDATA #REQUIRED
+				enabled           CDATA "true">
+]>
+
+<GathererCollection>
+  <Argument name="built">false</Argument>
+  <Argument name="imported">false</Argument>
+  <GathererConfig/>
+  <BuildConfig>
+    <Build/>
+	 <Import/>
+  </BuildConfig>
+</GathererCollection>
+
+
+
Index: /main/tags/3.01/gli/classes/xml/template.mds
===================================================================
--- /main/tags/3.01/gli/classes/xml/template.mds	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/template.mds	(revision 21112)
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param description A description of this metadata set.                           -->
+  <!-- @param family      The namespace of this metadata set.                           -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param name        The name of this metadata set.                                -->
+  <!-- @param namespace   The abbreviated namespace of this metadata set.               -->
+  <!ELEMENT MetadataSet (Element+)>
+  <!ATTLIST MetadataSet
+				contact     CDATA #REQUIRED
+				creator     CDATA #REQUIRED
+				description CDATA #REQUIRED
+            family      CDATA #REQUIRED
+				lastchanged CDATA #REQUIRED
+            name        CDATA #REQUIRED
+				namespace   CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!ELEMENT Element (Attribute*,OptionList?,AssignedValues?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+				occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+
+  <!-- Each element also contains a tree of values that have been assigned to it. This may include a multilayer nested subject hierachy. -->
+  <!ELEMENT AssignedValues (Subject+)>
+
+  <!-- A value. -->
+  <!ELEMENT Subject (Value|Alias?|Subject*)>
+  <!ELEMENT Value (#PCDATA)>
+  <!ATTLIST Value
+            occurances CDATA "1">
+  <!ELEMENT Alias (#PCDATA)>
+]>
+
+<MetadataSet>
+</MetadataSet>
Index: /main/tags/3.01/gli/classes/xml/workflows.xml
===================================================================
--- /main/tags/3.01/gli/classes/xml/workflows.xml	(revision 21112)
+++ /main/tags/3.01/gli/classes/xml/workflows.xml	(revision 21112)
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE PredefinedWorkflows [
+  <!ELEMENT PredefinedWorkflows (Workflow*)>
+  <!ELEMENT Workflow            (#PCDATA)>
+  <!ATTLIST Workflow
+	download			  CDATA "true"
+	gather				  CDATA "true"
+	enrich				  CDATA "true"
+	design				  CDATA "true"
+	create				  CDATA "true"
+	export				  CDATA "true">
+]>
+
+<PredefinedWorkflows>
+  <Workflow>Workflow.AllPanels</Workflow>
+  <Workflow download="false">Workflow.LocalSourceCollectionBuilding</Workflow>
+  <Workflow design="false" create="false">Workflow.DocumentEnriching</Workflow>
+</PredefinedWorkflows>
Index: /main/tags/3.01/gli/clean.bat
===================================================================
--- /main/tags/3.01/gli/clean.bat	(revision 21112)
+++ /main/tags/3.01/gli/clean.bat	(revision 21112)
@@ -0,0 +1,47 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Clean up the Greenstone Librarian Interface directory --------
+
+:: This script must be run from within the directory in which it lives
+if exist clean.bat goto cleanGLI
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+
+:cleanGLI
+:: ---- Remove class files ----
+echo.
+if "%GLILANG%" == "en" echo Removing the Greenstone Librarian Interface class files...
+if "%GLILANG%" == "es" echo Eliminando los archivos de clase de la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Suppression des fichiers de classe de Greenstone Librarian Interface
+if "%GLILANG%" == "ru" echo € «¥­š¥ ä ©«®¢ ª« áá  ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone
+
+if not exist classes\org\greenstone\gatherer goto done
+
+cd classes\org\greenstone\gatherer
+if exist "*.class" del "*.class"
+if exist "cdm\*.class" del "cdm\*.class"
+if exist "collection\*.class" del "collection\*.class"
+if exist "feedback\*.class" del "feedback\*.class"
+if exist "file\*.class" del "file\*.class"
+if exist "gems\*.class" del "gems\*.class"
+if exist "gui\*.class" del "gui\*.class"
+if exist "metadata\*.class" del "metadata\*.class"
+if exist "shell\*.class" del "shell\*.class"
+if exist "util\*.class" del "util\*.class"
+cd ..\..\..\..
+
+:done
+if "%GLILANG%" == "en" echo Done!
+if "%GLILANG%" == "es" echo ­Hecho!
+if "%GLILANG%" == "fr" echo Termin!
+if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+
+:exit
+echo.
Index: /main/tags/3.01/gli/clean.sh
===================================================================
--- /main/tags/3.01/gli/clean.sh	(revision 21112)
+++ /main/tags/3.01/gli/clean.sh	(revision 21112)
@@ -0,0 +1,31 @@
+#!/bin/sh
+glilang=en
+
+##  -------- Clean up the Greenstone Librarian Interface directory --------
+
+## ---- Remove class files ----
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Eliminando los archivos de clase de la Interfaz de la Biblioteca "
+    echo "Digital Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Suppression des fichiers de classe de Greenstone Librarian Interface"
+elif [ "$glilang" == "ru" ]; then
+    echo "õÄÁÌÅÎÉÅ ÆÁÊÌÏ× ËÌÁÓÓÁ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone"
+else
+    echo "Removing the Greenstone Librarian Interface class files..."
+fi
+
+rm -rf GLI.jar
+rm -rf classes/org/greenstone/gatherer/*
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
+echo
Index: /main/tags/3.01/gli/client-gli.bat
===================================================================
--- /main/tags/3.01/gli/client-gli.bat	(revision 21112)
+++ /main/tags/3.01/gli/client-gli.bat	(revision 21112)
@@ -0,0 +1,167 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Run the Greenstone Librarian Interface --------
+
+:: This script must be run from within the directory in which it lives
+if exist client-gli.bat goto start
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+:start
+if "%OS%" == "Windows_NT" goto findGSDL
+    :: Invoke a new command processor to ensure there's enough environment space
+    if "%1" == "Second" goto findGSDL
+        command /E:2048 /C %0 Second %1 %2 %3 %4 %5 %6 %7 %8 %9
+        goto done
+
+:findGSDL
+echo.
+if "%GLILANG%" == "en" echo Greenstone Librarian Interface (GLI)
+if "%GLILANG%" == "en" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "en" echo GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+if "%GLILANG%" == "en" echo This is free software, and you are welcome to redistribute it
+
+if "%GLILANG%" == "es" echo Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "es" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÖA.
+if "%GLILANG%" == "es" echo Para mayor informaci¢n vea los trminos de la licencia en LICENSE.txt
+if "%GLILANG%" == "es" echo Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita
+
+if "%GLILANG%" == "fr" echo Interface du Bibliothcaire Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "fr" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "fr" echo GLI est fourni sans AUCUNE GARANTIE; pour des dtails, voir LICENSE.txt
+if "%GLILANG%" == "fr" echo Ceci est un logiciel libre, et vous tes invit 
+ le redistribuer
+
+if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "ru" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "ru" echo  ­¥ € ¥â   ; €¥â «š á¬. ¢ â¥ªáâ¥ LICENSE.TXT
+if "%GLILANG%" == "ru" echo â® - á¢®¡®€­® à á¯à®áâà ­ï¥¬®¥ ¯à®£à ¬¬­®¥ ®¡¥á¯¥ç¥­š¥ š ë ¬®Š¥â¥ à á¯à®áâà ­ïâì ¥£®
+
+
+:findJava
+:: ---- Check Java exists ----
+set JAVAPATH=
+
+:: Some users may set the above line manually
+if not "%JAVAPATH%" == "" goto testJava
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if Java is on the search path
+    echo %PATH%| winutil\which.exe java.exe | winutil\setvar.exe JAVAPATH > setjava.bat
+    call setjava.bat
+    del setjava.bat
+    if not "%JAVAPATH%" == "" goto testJava
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+    type nul > jre.reg
+    regedit /E jre.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Runtime Environment"
+    type jre.reg > jre.txt
+    del jre.reg
+
+    winutil\findjava.exe jdk.txt jre.txt | winutil\setvar.exe JAVAPATH > setjava.bat
+    del jdk.txt
+    del jre.txt
+    call setjava.bat
+    del setjava.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JAVAPATH%" == "" goto noJava
+
+    set JAVAPATH=%JAVAPATH%\bin
+    goto testJava
+
+:javahome
+    set JAVAPATH=%JAVA_HOME%\bin
+
+:testJava
+:: Check that a Java executable has been found
+if "%GLILANG%" == "en" echo Checking Java: %JAVAPATH%
+if "%GLILANG%" == "es" echo Revisando Java: %JAVAPATH%
+if "%GLILANG%" == "fr" echo Vrification de Java: %JAVAPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Java: %JAVAPATH%
+if exist "%JAVAPATH%\java.exe" goto checkGLI
+
+:noJava
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of Java. You must install a
+    if "%GLILANG%" == "en" echo Java Runtime Environment (version 1.4 or greater) before running the
+    if "%GLILANG%" == "en" echo Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de Java. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Ambiente de Ejecuci¢n Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de correr la Interfaz de la Biblioteca Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version adquate de Java n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Java Runtime Environment (version 1.4 ou suprieur)
+    if "%GLILANG%" == "fr" echo avant de dmarrer Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš Java.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ãáâ ­®¢šâì Java Runtime Environment (¢¥àášî 1.4 š«š ¢ëè¥) ¯¥à¥€ ¢¢®€®¬
+    if "%GLILANG%" == "ru" echo ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+:checkGLI
+:: ---- Check that the GLI has been compiled ----
+if exist "classes/org/greenstone/gatherer/Gatherer.class" goto runGLI
+if exist "GLI.jar" goto runGLI
+    echo.
+    if "%GLILANG%" == "en" echo You need to compile the Greenstone Librarian Interface (using makegli.bat)
+    if "%GLILANG%" == "en" echo before running this script.
+
+    if "%GLILANG%" == "es" echo Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone
+    if "%GLILANG%" == "es" echo (por medio de makegli.bat) antes de ejecutar este gui¢n.
+
+    if "%GLILANG%" == "fr" echo Vous devez compiler le Greenstone Interface (en utilisant makegil.bat)
+    if "%GLILANG%" == "fr" echo avant d'excuter ce script.
+
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ª®¬¯š«šà®¢ âì ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (šá¯®«ì§ãï makegli.bat)
+    if "%GLILANG%" == "ru" echo ¯¥à¥€ ¢¢®€®¬ íâ®£® áªàš¯â 
+    goto exit
+
+
+:runGLI
+:: ---- Finally, run the GLI ----
+echo.
+
+
+if "%GLILANG%" == "en" echo Running the Greenstone Librarian Interface...
+if "%GLILANG%" == "es" echo Ejecutando la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Excution de Greenstone Librarian Interface
+if "%GLILANG%" == "ru" echo ¥ªãéš© ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone...
+
+:: -Xms32M          To set minimum memory
+:: -Xmx32M          To set maximum memory
+:: -verbose:gc      To set garbage collection messages
+:: -Xincgc          For incremental garbage collection
+:: -Xprof           Function call profiling
+:: -Xloggc:<file>   Write garbage collection log
+
+"%JAVAPATH%\java" -cp classes/;GLI.jar;lib/apache.jar;lib/jp.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar org.greenstone.gatherer.GathererProg -use_remote_greenstone %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+if "%GLILANG%" == "en" echo Done!
+if "%GLILANG%" == "es" echo ­Hecho!
+if "%GLILANG%" == "fr" echo Termin!
+if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+goto done
+
+:exit
+echo.
+pause
+
+:done
+:: ---- Clean up ----
+set JAVAPATH=
Index: /main/tags/3.01/gli/client-gli.sh
===================================================================
--- /main/tags/3.01/gli/client-gli.sh	(revision 21112)
+++ /main/tags/3.01/gli/client-gli.sh	(revision 21112)
@@ -0,0 +1,174 @@
+#!/bin/sh
+glilang=en
+
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÍA."
+    echo "Para mayor información vea los términos de la licencia en LICENSE.txt"
+    echo "Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita"
+elif [ "$glilang" == "fr" ]; then
+    echo "Interface du Bibliothécaire Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI est fourni sans AUCUNE GARANTIE; pour des détails, voir LICENSE.txt"
+    echo "Ceci est un logiciel libre, et vous êtes invité à le redistribuer"
+elif [ "$glilang" == "ru" ]; then
+    echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "âéç ÎÅ ÄÁÅÔ áâóïìàôîï îéëáëéè çáòáîôéê; ÄÅÔÁÌÉ ÓÍ. × ÔÅËÓÔÅ LICENSE.TXT"
+    echo "üÔÏ - Ó×ÏÂÏÄÎÏ ÒÁÓÐÒÏÓÔÒÁÎÑÅÍÏÅ ÐÒÏÇÒÁÍÍÎÏÅ ÏÂÅÓÐÅÞÅÎÉÅ É ÷Ù ÍÏÖÅÔÅ ÒÁÓÐÒÏÓÔÒÁÎÑÔØ ÅÇÏ"
+else
+    echo "Greenstone Librarian Interface (GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt"
+    echo "This is free software, and you are welcome to redistribute it"
+fi
+echo
+
+##  -------- Run the Greenstone Librarian Interface --------
+
+# This script must be run from within the directory in which it lives
+thisdir=`pwd`
+if [ ! -f "${thisdir}/client-gli.sh" ]; then
+    if [ "$glilang" == "es" ]; then
+        echo "Este guión deberá ejecutarse desde el directorio en el que reside."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Ce script doit être exécuté à partir du répertoire dans lequel il se trouve."
+    elif [ "$glilang" == "ru" ]; then
+	echo "üÔÏÔ ÓËÒÉÐÔ ÄÏÌÖÅÎ ÂÙÔØ ×ÚÑÔ ÉÚ ÄÉÒÅËÔÏÒÉÉ, × ËÏÔÏÒÏÊ ÏÎ ÒÁÓÐÏÌÏÖÅÎ"
+    else
+	echo "This script must be run from the directory in which it resides."
+    fi
+    exit 1
+fi
+
+
+## ---- Check Java exists ----
+javapath=
+
+# Some users may set the above line manually
+if [ "$javapath" == "" ]; then
+
+    # If it is set, use the JAVA_HOME environment variable
+    if [ "$JAVA_HOME" != "" ]; then
+	javapath="$JAVA_HOME/bin/java"
+
+    # Check if Java is on the search path
+    else
+	javapath=`which java 2> /dev/null`
+    fi
+fi
+
+# Check that a Java executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Java: $javapath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Java: $javapath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Java: $javapath"
+else
+    echo "Checking Java: $javapath"
+fi
+if [ ! -x "$javapath" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo localizar una versión apropiada de Java. Usted deberá "
+	echo "instalar un Ambiente de Ejecución Java (versión 1.4 o superior) "
+	echo "antes de correr la Interfaz de la Biblioteca Digital Greenstone."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Une version adéquate de Java n'a pas pu être localisée."
+	echo "Vous devez installer un Java Runtime Environment (version 1.4 ou"
+	echo "supérieur) avant de démarrer Greenstone Librarian Interface."
+    elif [ "$glilang" == "ru" ]; then
+	echo "îÅ ÕÄÁÌÏÓØ ÏÐÒÅÄÅÌÉÔØ ÍÅÓÔÏÎÁÈÏÖÄÅÎÉÅ ÓÏÏÔ×ÅÔÓÔ×ÕÀÝÅÊ ×ÅÒÓÉÉ Java."
+	echo "÷Ù ÄÏÌÖÎÙ ÕÓÔÁÎÏ×ÉÔØ Java Runtime Environment (×ÅÒÓÉÀ 1.4 ÉÌÉ ×ÙÛÅ)"
+	echo "ÐÅÒÅÄ ××ÏÄÏÍ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone."
+    else
+	echo "Failed to locate an appropriate version of Java. You must install a"
+	echo "Java Runtime Environment (version 1.4 or greater) before running the"
+	echo "Greenstone Librarian Interface."
+    fi
+    exit 1
+fi
+
+
+## -- Check the version of Java is new enough (1.4.0 or higher) to run the GLI --
+`$javapath -classpath . CheckJavaVersion > /dev/null`
+if [ $? -ne 2 ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "La versión del Ambiente de Ejecución Java (JRE por sus siglas en "
+	echo "inglés) que usted tiene instalada es demasiado vieja para ejecutar "
+	echo "la Interfaz de la Biblioteca Digital Greenstone. Por favor instale "
+	echo "una nueva versión del Ambiente de Ejecución Java (versión 1.4 o "
+	echo "posterior) y ejecute nuevamente este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "La version de Java Runtime Environment que vous avez installée est"
+	echo "trop vielle pour faire fonctionner Greenstone Librarian Interface."
+	echo "Veuillez installer une nouvelle version du JRE (version 1.4 ou plus"
+	echo "récente) et redémarrez le script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷ÅÒÓÉÑ Java Runtime Environment, ËÏÔÏÒÕÀ ÷Ù ÕÓÔÁÎÏ×ÉÌÉ, ÏÞÅÎØ ÓÔÁÒÁ,"
+	echo "ÞÔÏÂÙ ÕÐÒÁ×ÌÑÔØ ÂÉÂÌÉÏÔÅÞÎÙÍ ÉÎÔÅÒÆÅÊÓÏÍ Greenstone. ðÏÖÁÌÕÊÓÔÁ, "
+	echo "ÕÓÔÁÎÏ×ÉÔÅ ÎÏ×ÕÀ ×ÅÒÓÉÀ JRE (×ÅÒÓÉÀ 1.4 ÉÌÉ ÂÏÌÅÅ ÎÏ×ÕÀ) É"
+	echo "ÐÅÒÅÕÓÔÁÎÏ×ÉÔÅ ÜÔÏÔ ÓËÒÉÐÔ"
+    else
+	echo "The version of the Java Runtime Environment you have installed is too"
+	echo "old to run the Greenstone Librarian Interface. Please install a new"
+	echo "version of the JRE (version 1.4 or newer) and rerun this script."
+    fi
+    exit 1
+fi
+
+## ---- Check that the GLI has been compiled ----
+if [ ! -f "classes/org/greenstone/gatherer/GathererProg.class" ] && [ ! -f "GLI.jar" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone"
+	echo "(por medio de makegli.sh) antes de ejecutar este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Vous devez compiler le Greenstone Interface (en utilisant makegli.sh)"
+	echo "avant d'exécuter ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷Ù ÄÏÌÖÎÙ ËÏÍÐÉÌÉÒÏ×ÁÔØ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone"
+	echo "(ÉÓÐÏÌØÚÕÑ makegli.sh) ÐÅÒÅÄ ××ÏÄÏÍ ÜÔÏÇÏ ÓËÒÉÐÔÁ"
+    else
+	echo "You need to compile the Greenstone Librarian Interface (using makegli.sh)"
+	echo "before running this script."
+    fi
+    exit 1
+fi
+
+
+## ---- Finally, run the GLI ----
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Ejecutando la Interfaz de la Biblioteca Digital Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Exécution de Greenstone Librarian Interface"
+elif [ "$glilang" == "ru" ]; then
+    echo "ôÅËÕÝÉÊ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone..."
+else
+    echo "Running the Greenstone Librarian Interface..."
+fi
+
+# Other arguments you can provide to GLI to work around memory limitations, or debug
+# -Xms<number>M    To set minimum memory (by default 32MB)
+# -Xmx<number>M    To set maximum memory (by default the nearest 2^n to the total remaining physical memory)
+# -verbose:gc      To set garbage collection messages
+# -Xincgc          For incremental garbage collection (significantly slows performance)
+# -Xprof           Function call profiling
+# -Xloggc:<file>   Write garbage collection log
+
+$javapath -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar org.greenstone.gatherer.GathererProg -use_remote_greenstone $*
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
Index: /main/tags/3.01/gli/copyright/apache_license.txt
===================================================================
--- /main/tags/3.01/gli/copyright/apache_license.txt	(revision 21112)
+++ /main/tags/3.01/gli/copyright/apache_license.txt	(revision 21112)
@@ -0,0 +1,64 @@
+Pollo uses the following components from the Apache projects:
+
+ - Xerces: used to parse XML files. Pollo also depends heavely
+   on Xerces' DOM implementation (including the rarely found support
+   for mutation events).
+
+ - Commons Digester: used to parse the xml-based configuration file
+   of Pollo.
+
+ - log4j
+
+For more information about this projects, visit the following sites:
+http://xml.apache.org
+http://jakarta.apache.org
+
+The following license applies to all this software (substitute
+the word Xerces for all other mentioned projects):
+
+The Apache Software License, Version 1.1
+
+
+Copyright (c) 1999 The Apache Software Foundation.  All rights 
+reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer. 
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in
+   the documentation and/or other materials provided with the
+   distribution.
+
+3. The end-user documentation included with the redistribution,
+   if any, must include the following acknowledgment:  
+      "This product includes software developed by the
+       Apache Software Foundation (http://www.apache.org/)."
+   Alternately, this acknowledgment may appear in the software itself,
+   if and wherever such third-party acknowledgments normally appear.
+
+4. The names "Xerces" and "Apache Software Foundation" must
+   not be used to endorse or promote products derived from this
+   software without prior written permission. For written 
+   permission, please contact apache@apache.org.
+
+5. Products derived from this software may not be called "Apache",
+   nor may "Apache" appear in their name, without prior written
+   permission of the Apache Software Foundation.
+
+THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
Index: /main/tags/3.01/gli/copyright/mpl_1.1.txt
===================================================================
--- /main/tags/3.01/gli/copyright/mpl_1.1.txt	(revision 21112)
+++ /main/tags/3.01/gli/copyright/mpl_1.1.txt	(revision 21112)
@@ -0,0 +1,171 @@
+MOZILLA PUBLIC LICENSE
+Version 1.1
+**********************
+
+1. Definitions.
+
+      1.0.1. "Commercial Use" means distribution or otherwise making the Covered Code available to a third party.
+
+      1.1. ''Contributor'' means each entity that creates or contributes to the creation of Modifications.
+
+      1.2. ''Contributor Version'' means the combination of the Original Code, prior Modifications used by a Contributor, and the Modifications made by that particular Contributor.
+
+      1.3. ''Covered Code'' means the Original Code or Modifications or the combination of the Original Code and Modifications, in each case including portions thereof.
+
+      1.4. ''Electronic Distribution Mechanism'' means a mechanism generally accepted in the software development community for the electronic transfer of data.
+
+      1.5. ''Executable'' means Covered Code in any form other than Source Code.
+
+      1.6. ''Initial Developer'' means the individual or entity identified as the Initial Developer in the Source Code notice required by Exhibit A.
+
+      1.7. ''Larger Work'' means a work which combines Covered Code or portions thereof with code not governed by the terms of this License.
+
+      1.8. ''License'' means this document.
+
+      1.8.1. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
+
+      1.9. ''Modifications'' means any addition to or deletion from the substance or structure of either the Original Code or any previous Modifications. When Covered Code is released as a series of files, a Modification is:
+            A. Any addition to or deletion from the contents of a file containing Original Code or previous Modifications.
+
+            B. Any new file that contains any part of the Original Code or previous Modifications.
+             
+      1.10. ''Original Code'' means Source Code of computer software code which is described in the Source Code notice required by Exhibit A as Original Code, and which, at the time of its release under this License is not already Covered Code governed by this License.
+
+      1.10.1. "Patent Claims" means any patent claim(s), now owned or hereafter acquired, including without limitation,  method, process, and apparatus claims, in any patent Licensable by grantor.
+
+      1.11. ''Source Code'' means the preferred form of the Covered Code for making modifications to it, including all modules it contains, plus any associated interface definition files, scripts used to control compilation and installation of an Executable, or source code differential comparisons against either the Original Code or another well known, available Covered Code of the Contributor's choice. The Source Code can be in a compressed or archival form, provided the appropriate decompression or de-archiving software is widely available for no charge.
+
+      1.12. "You'' (or "Your")  means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License or a future version of this License issued under Section 6.1. For legal entities, "You'' includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control'' means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity.
+
+2. Source Code License.
+
+      2.1. The Initial Developer Grant.
+      The Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license, subject to third party intellectual property claims:
+            (a)  under intellectual property rights (other than patent or trademark) Licensable by Initial Developer to use, reproduce, modify, display, perform, sublicense and distribute the Original Code (or portions thereof) with or without Modifications, and/or as part of a Larger Work; and
+
+            (b) under Patents Claims infringed by the making, using or selling of Original Code, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Code (or portions thereof).
+                          (c) the licenses granted in this Section 2.1(a) and (b) are effective on the date Initial Developer first distributes Original Code under the terms of this License.
+
+            (d) Notwithstanding Section 2.1(b) above, no patent license is granted: 1) for code that You delete from the Original Code; 2) separate from the Original Code;  or 3) for infringements caused by: i) the modification of the Original Code or ii) the combination of the Original Code with other software or devices.
+             
+      2.2. Contributor Grant.
+      Subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license
+             
+            (a)  under intellectual property rights (other than patent or trademark) Licensable by Contributor, to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof) either on an unmodified basis, with other Modifications, as Covered Code and/or as part of a Larger Work; and
+
+            (b) under Patent Claims infringed by the making, using, or selling of  Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: 1) Modifications made by that Contributor (or portions thereof); and 2) the combination of  Modifications made by that Contributor with its Contributor Version (or portions of such combination).
+
+            (c) the licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first makes Commercial Use of the Covered Code.
+
+            (d)    Notwithstanding Section 2.2(b) above, no patent license is granted: 1) for any code that Contributor has deleted from the Contributor Version; 2)  separate from the Contributor Version;  3)  for infringements caused by: i) third party modifications of Contributor Version or ii)  the combination of Modifications made by that Contributor with other software  (except as part of the Contributor Version) or other devices; or 4) under Patent Claims infringed by Covered Code in the absence of Modifications made by that Contributor.
+
+
+3. Distribution Obligations.
+
+      3.1. Application of License.
+      The Modifications which You create or to which You contribute are governed by the terms of this License, including without limitation Section 2.2. The Source Code version of Covered Code may be distributed only under the terms of this License or a future version of this License released under Section 6.1, and You must include a copy of this License with every copy of the Source Code You distribute. You may not offer or impose any terms on any Source Code version that alters or restricts the applicable version of this License or the recipients' rights hereunder. However, You may include an additional document offering the additional rights described in Section 3.5.
+
+      3.2. Availability of Source Code.
+      Any Modification which You create or to which You contribute must be made available in Source Code form under the terms of this License either on the same media as an Executable version or via an accepted Electronic Distribution Mechanism to anyone to whom you made an Executable version available; and if made available via Electronic Distribution Mechanism, must remain available for at least twelve (12) months after the date it initially became available, or at least six (6) months after a subsequent version of that particular Modification has been made available to such recipients. You are responsible for ensuring that the Source Code version remains available even if the Electronic Distribution Mechanism is maintained by a third party.
+
+      3.3. Description of Modifications.
+      You must cause all Covered Code to which You contribute to contain a file documenting the changes You made to create that Covered Code and the date of any change. You must include a prominent statement that the Modification is derived, directly or indirectly, from Original Code provided by the Initial Developer and including the name of the Initial Developer in (a) the Source Code, and (b) in any notice in an Executable version or related documentation in which You describe the origin or ownership of the Covered Code.
+
+      3.4. Intellectual Property Matters
+            (a) Third Party Claims.
+            If Contributor has knowledge that a license under a third party's intellectual property rights is required to exercise the rights granted by such Contributor under Sections 2.1 or 2.2, Contributor must include a text file with the Source Code distribution titled "LEGAL'' which describes the claim and the party making the claim in sufficient detail that a recipient will know whom to contact. If Contributor obtains such knowledge after the Modification is made available as described in Section 3.2, Contributor shall promptly modify the LEGAL file in all copies Contributor makes available thereafter and shall take other steps (such as notifying appropriate mailing lists or newsgroups) reasonably calculated to inform those who received the Covered Code that new knowledge has been obtained.
+
+            (b) Contributor APIs.
+            If Contributor's Modifications include an application programming interface and Contributor has knowledge of patent licenses which are reasonably necessary to implement that API, Contributor must also include this information in the LEGAL file.
+             
+                (c)    Representations.
+            Contributor represents that, except as disclosed pursuant to Section 3.4(a) above, Contributor believes that Contributor's Modifications are Contributor's original creation(s) and/or Contributor has sufficient rights to grant the rights conveyed by this License.
+
+
+      3.5. Required Notices.
+      You must duplicate the notice in Exhibit A in each file of the Source Code.  If it is not possible to put such notice in a particular Source Code file due to its structure, then You must include such notice in a location (such as a relevant directory) where a user would be likely to look for such a notice.  If You created one or more Modification(s) You may add your name as a Contributor to the notice described in Exhibit A.  You must also duplicate this License in any documentation for the Source Code where You describe recipients' rights or ownership rights relating to Covered Code.  You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Code. However, You may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear than any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer.
+
+      3.6. Distribution of Executable Versions.
+      You may distribute Covered Code in Executable form only if the requirements of Section 3.1-3.5 have been met for that Covered Code, and if You include a notice stating that the Source Code version of the Covered Code is available under the terms of this License, including a description of how and where You have fulfilled the obligations of Section 3.2. The notice must be conspicuously included in any notice in an Executable version, related documentation or collateral in which You describe recipients' rights relating to the Covered Code. You may distribute the Executable version of Covered Code or ownership rights under a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable version does not attempt to limit or alter the recipient's rights in the Source Code version from the rights set forth in this License. If You distribute the Executable version under a different license You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or any Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer.
+
+      3.7. Larger Works.
+      You may create a Larger Work by combining Covered Code with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Code.
+
+4. Inability to Comply Due to Statute or Regulation.
+
+      If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Code due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be included in the LEGAL file described in Section 3.4 and must be included with all distributions of the Source Code. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it.
+
+5. Application of this License.
+
+      This License applies to code to which the Initial Developer has attached the notice in Exhibit A and to related Covered Code.
+
+6. Versions of the License.
+
+      6.1. New Versions.
+      Netscape Communications Corporation (''Netscape'') may publish revised and/or new versions of the License from time to time. Each version will be given a distinguishing version number.
+
+      6.2. Effect of New Versions.
+      Once Covered Code has been published under a particular version of the License, You may always continue to use it under the terms of that version. You may also choose to use such Covered Code under the terms of any subsequent version of the License published by Netscape. No one other than Netscape has the right to modify the terms applicable to Covered Code created under this License.
+
+      6.3. Derivative Works.
+      If You create or use a modified version of this License (which you may only do in order to apply it to code which is not already Covered Code governed by this License), You must (a) rename Your license so that the phrases ''Mozilla'', ''MOZILLAPL'', ''MOZPL'', ''Netscape'', "MPL", ''NPL'' or any confusingly similar phrase do not appear in your license (except to note that your license differs from this License) and (b) otherwise make it clear that Your version of the license contains terms which differ from the Mozilla Public License and Netscape Public License. (Filling in the name of the Initial Developer, Original Code or Contributor in the notice described in Exhibit A shall not of themselves be deemed to be modifications of this License.)
+
+7. DISCLAIMER OF WARRANTY.
+
+      COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS'' BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+8. TERMINATION.
+
+      8.1.  This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. All sublicenses to the Covered Code which are properly granted shall survive any termination of this License. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive.
+
+      8.2.  If You initiate litigation by asserting a patent infringement claim (excluding declatory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You file such action is referred to as "Participant")  alleging that:
+
+      (a)  such Participant's Contributor Version directly or indirectly infringes any patent, then any and all rights granted by such Participant to You under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively, unless if within 60 days after receipt of notice You either: (i)  agree in writing to pay Participant a mutually agreeable reasonable royalty for Your past and future use of Modifications made by such Participant, or (ii) withdraw Your litigation claim with respect to the Contributor Version against such Participant.  If within 60 days of notice, a reasonable royalty and payment arrangement are not mutually agreed upon in writing by the parties or the litigation claim is not withdrawn, the rights granted by Participant to You under Sections 2.1 and/or 2.2 automatically terminate at the expiration of the 60 day notice period specified above.
+
+      (b)  any software, hardware, or device, other than such Participant's Contributor Version, directly or indirectly infringes any patent, then any rights granted to You by such Participant under Sections 2.1(b) and 2.2(b) are revoked effective as of the date You first made, used, sold, distributed, or had made, Modifications made by that Participant.
+
+      8.3.  If You assert a patent infringement claim against Participant alleging that such Participant's Contributor Version directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license.
+
+      8.4.  In the event of termination under Sections 8.1 or 8.2 above,  all end user license agreements (excluding distributors and resellers) which have been validly granted by You or any distributor hereunder prior to termination shall survive termination.
+
+9. LIMITATION OF LIABILITY.
+
+      UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
+
+10. U.S. GOVERNMENT END USERS.
+
+      The Covered Code is a ''commercial item,'' as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of ''commercial computer software'' and ''commercial computer software documentation,'' as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Code with only those rights set forth herein.
+
+11. MISCELLANEOUS.
+
+      This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by California law provisions (except to the extent applicable law, if any, provides otherwise), excluding its conflict-of-law provisions. With respect to disputes in which at least one party is a citizen of, or an entity chartered or registered to do business in the United States of America, any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California, with venue lying in Santa Clara County, California, with the losing party responsible for costs, including without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License.
+
+12. RESPONSIBILITY FOR CLAIMS.
+
+      As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
+
+13. MULTIPLE-LICENSED CODE.
+
+      Initial Developer may designate portions of the Covered Code as ?Multiple-Licensed?.  ?Multiple-Licensed? means that the Initial Developer permits you to utilize portions of the Covered Code under Your choice of the MPL or the alternative licenses, if any, specified by the Initial Developer in the file described in Exhibit A.
+
+
+EXHIBIT A -Mozilla Public License.
+
+      ``The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+      http://www.mozilla.org/MPL/
+
+      Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF
+      ANY KIND, either express or implied. See the License for the specific language governing rights and
+      limitations under the License.
+
+      The Original Code is ______________________________________.
+
+      The Initial Developer of the Original Code is ________________________. Portions created by
+       ______________________ are Copyright (C) ______ _______________________. All Rights
+      Reserved.
+
+      Contributor(s): ______________________________________.
+
+      Alternatively, the contents of this file may be used under the terms of the _____ license (the  ?[___] License?), in which case the provisions of [______] License are applicable  instead of those above.  If you wish to allow use of your version of this file only under the terms of the [____] License and not to allow others to use your version of this file under the MPL, indicate your decision by deleting  the provisions above and replace  them with the notice and other provisions required by the [___] License.  If you do not delete the provisions above, a recipient may use your version of this file under either the MPL or the [___] License."
+
+      [NOTE: The text of this Exhibit A may differ slightly from the text of the notices in the Source Code files of the Original Code. You should use the text of this Exhibit A rather than the text found in the Original Code Source Code for Your Modifications.] 
Index: /main/tags/3.01/gli/copyright/qfs_license.txt
===================================================================
--- /main/tags/3.01/gli/copyright/qfs_license.txt	(revision 21112)
+++ /main/tags/3.01/gli/copyright/qfs_license.txt	(revision 21112)
@@ -0,0 +1,11 @@
+Copyright (C) 2003 Quality First Software GmbH
+
+qflib - License
+
+qflib is distributed for free under the terms of the Mozilla Public License Version 1.1, MPL for short. It comes without warranty of any kind. The license grants you free use of qflib, access to the source and the right to modify and redistribute it.
+
+The only restrictions are that you cannot remove Quality First Software GmbH's copyright and that modifications of qflib are automatically covered by the MPL as well. Code that just uses the library through its published interface (referred to as "greater work" in the license) is free to use any license whatever.
+
+As long as these conditions are met there is nothing to inhibit commercial use or commercial distribution of qflib, e.g. together with other software on a CD-rom.
+
+We decided in favor of the MPL because it appears to be a good compromise between protecting openness and free use on the one hand and enabling interoperation with third party software under any license on the other. The text of the license is short and precise compared to others and is comprehensible for the layman. 
Index: /main/tags/3.01/gli/copyright/wget_license.txt
===================================================================
--- /main/tags/3.01/gli/copyright/wget_license.txt	(revision 21112)
+++ /main/tags/3.01/gli/copyright/wget_license.txt	(revision 21112)
@@ -0,0 +1,100 @@
+GNU GENERAL PUBLIC LICENSE
+
+Version 2, June 1991
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+675 Mass Ave, Cambridge, MA 02139, USA
+
+Everyone is permitted to copy and distribute verbatim copies
+of this license document, but changing it is not allowed.
+
+Preamble
+
+The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
+
+We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations.
+
+Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and modification follow.
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+   1. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does.
+   2. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee.
+   3. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions:
+         1. You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change.
+         2. You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License.
+         3. If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) 
+      These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License.
+   4. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following:
+         1. Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
+         2. Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
+         3. Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) 
+      The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code.
+   5. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance.
+   6. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it.
+   7. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License.
+   8. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License.
+   9. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License.
+  10. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation.
+  11. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally.
+
+      NO WARRANTY
+  12. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+  13. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 
+
+END OF TERMS AND CONDITIONS
+How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
+
+one line to give the program's name and an idea of what it does.
+Copyright (C) 19yy  name of author
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this when it starts in an interactive mode:
+
+Gnomovision version 69, Copyright (C) 19yy name of author
+Gnomovision comes with ABSOLUTELY NO WARRANTY; for details
+type `show w'.  This is free software, and you are welcome
+to redistribute it under certain conditions; type `show c' 
+for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names:
+
+Yoyodyne, Inc., hereby disclaims all copyright
+interest in the program `Gnomovision'
+(which makes passes at compilers) written 
+by James Hacker.
+
+signature of Ty Coon, 1 April 1989
+Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. 
Index: /main/tags/3.01/gli/document.bat
===================================================================
--- /main/tags/3.01/gli/document.bat	(revision 21112)
+++ /main/tags/3.01/gli/document.bat	(revision 21112)
@@ -0,0 +1,103 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Generate documentation for the Greenstone Librarian Interface --------
+
+:: This script must be run from within the directory in which it lives
+if exist document.bat goto findJavadoc
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+
+:findJavadoc
+:: ---- Check Javadoc exists ----
+set JAVADOCPATH=
+
+:: Some users may set the above line manually
+if not "%JAVADOCPATH%" == "" goto testJavadoc
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if Javadoc is on the search path
+    echo %PATH%| winutil\which javadoc.exe | winutil\setvar JAVADOCPATH > setjavadoc.bat
+    call setjavadoc.bat
+    del setjavadoc.bat
+    if not "%JAVADOCPATH%" == "" goto testJavadoc
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+
+    winutil\findjava jdk.txt | winutil\setvar JAVADOCPATH > setjavadoc.bat
+    del jdk.txt
+    call setjavadoc.bat
+    del setjavadoc.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JAVADOCPATH%" == "" goto noJavadoc
+
+    set JAVADOCPATH=%JAVADOCPATH%\bin
+    goto testJavadoc
+
+:javahome
+    set JAVADOCPATH=%JAVA_HOME%\bin
+
+:testJavadoc
+:: Check that a Javadoc executable has been found
+if "%GLILANG%" == "en" echo Checking Javadoc: %JAVADOCPATH%
+if "%GLILANG%" == "es" echo Revisando Javadoc: %JAVADOCPATH%
+if "%GLILANG%" == "fr" echo Vrification de Javadoc: %JAVADOCPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Javadoc: %JAVADOCPATH%
+if exist "%JAVADOCPATH%\javadoc.exe" goto docGLI
+
+:noJavadoc
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of Javadoc. You must install a
+    if "%GLILANG%" == "en" echo Java Development Kit (version 1.4 or greater) before generating the
+    if "%GLILANG%" == "en" echo documentation for the Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de Javadoc. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Kit de Desarrollo de Software Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de generar la documentaci¢n para la Interfaz de la Biblioteca
+    if "%GLILANG%" == "es" echo Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version approprie de Javadoc n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Kit de Dveloppement Java (version 1.4 ou suprieure) avant
+    if "%GLILANG%" == "fr" echo de produire la documentation de Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš Javadoc.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë š­áâ ««šà®¢ âì Java Development Kit (¢¥àášï 1.4 š«š ¢ëè¥)
+    if "%GLILANG%" == "ru" echo ¯à¥Š€¥, ç¥¬ £¥­¥àšà®¢ âì €®ªã¬¥­â æšî €«ï ¡š¡«š®â¥ç­®£®
+    if "%GLILANG%" == "ru" echo š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+:docGLI
+:: ---- Document the GLI ----
+echo.
+if "%GLILANG%" == "en" echo Generating documentation for the Greenstone Librarian Interface...
+if "%GLILANG%" == "es" echo Generando documentaci¢n para la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Production de documentation pour la Greenstone Librarian Interface
+if "%GLILANG%" == "ru" echo ¥­¥àšà®¢ ­š¥ €®ªã¬¥­â æšš €«ï ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone
+
+"%JAVADOCPATH%\javadoc.exe" -classpath classes/ -sourcepath src/ -source 1.4 -author -breakiterator -d docs/ -group "Greenstone Librarian Interface" org.greenstone.gatherer:org.greenstone.gatherer.cdm:org.greenstone.gatherer.collection:org.greenstone.gatherer.feedback:org.greenstone.gatherer.file:org.greenstone.gatherer.gems:org.greenstone.gatherer.gui:org.greenstone.gatherer.gui.metaaudit:org.greenstone.gatherer.gui.tree:org.greenstone.gatherer.metadata:org.greenstone.gatherer.shell:org.greenstone.gatherer.util -private -splitindex -version   org.greenstone.gatherer org.greenstone.gatherer.cdm org.greenstone.gatherer.collection org.greenstone.gatherer.feedback org.greenstone.gatherer.file org.greenstone.gatherer.gems org.greenstone.gatherer.gui org.greenstone.gatherer.gui.metaaudit org.greenstone.gatherer.gui.tree org.greenstone.gatherer.metadata org.greenstone.gatherer.shell org.greenstone.gatherer.util
+
+if "%GLILANG%" == "en" echo Done!
+if "%GLILANG%" == "es" echo ­Hecho!
+if "%GLILANG%" == "fr" echo Termin!
+if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+
+:exit
+echo.
+pause
+
+:: ---- Clean up ----
+set JAVADOCPATH=
Index: /main/tags/3.01/gli/document.sh
===================================================================
--- /main/tags/3.01/gli/document.sh	(revision 21112)
+++ /main/tags/3.01/gli/document.sh	(revision 21112)
@@ -0,0 +1,82 @@
+#!/bin/sh
+glilang=en
+
+
+##  -------- Generate documentation for the Greenstone Librarian Interface --------
+
+## ---- Check Javadoc exists ----
+javadocpath=
+
+# Some users may set the above line manually
+if [ "$javadocpath" == "" ]; then
+
+    # If it is set, use the JAVA_HOME environment variable
+    if [ "$JAVA_HOME" != "" ]; then
+	javadocpath="$JAVA_HOME/bin/javadoc"
+
+    # Check if Javadoc is on the search path
+    else
+	javadocpath=`which javadoc 2> /dev/null`
+    fi
+fi
+
+# Check that a Javadoc executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Javadoc: $javadocpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Javadoc: $javadocpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Javadoc: $javadocpath"
+else
+    echo "Checking Javadoc: $javadocpath"
+fi
+if [ ! -x "$javadocpath" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo localizar una versión apropiada de Javadoc. Usted deberá "
+        echo "instalar un Kit de Desarrollo de Software Java (versión 1.4 o "
+        echo "superior) antes de generar la documentación para la Interfaz de la "
+        echo "Biblioteca Digital Greenstone."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Une version appropriée de Javadoc n'a pas pu être localisée."
+	echo "Vous devez installer un Kit de Développement Java (version 1.4 ou"
+	echo "supérieure) avant de produire la documentation de Greenstone"
+	echo "Librarian Interface."
+    elif [ "$glilang" == "ru" ]; then
+	echo "îÅ ÕÄÁÌÏÓØ ÏÐÒÅÄÅÌÉÔØ ÍÅÓÔÏÎÁÈÏÖÄÅÎÉÅ ÓÏÏÔ×ÅÔÓÔ×ÕÀÝÅÊ ×ÅÒÓÉÉ Javadoc."
+	echo "÷Ù ÄÏÌÖÎÙ ÉÎÓÔÁÌÌÉÒÏ×ÁÔØ Java Development Kit (×ÅÒÓÉÑ 1.4 ÉÌÉ ×ÙÛÅ) "
+	echo "ÐÒÅÖÄÅ, ÞÅÍ ÇÅÎÅÒÉÒÏ×ÁÔØ ÄÏËÕÍÅÎÔÁÃÉÀ ÄÌÑ ÂÉÂÌÉÏÔÅÞÎÏÇÏ "
+	echo "ÉÎÔÅÒÆÅÊÓÁ Greenstone."
+    else
+	echo "Failed to locate an appropriate version of Javadoc. You must install a"
+	echo "Java Development Kit (version 1.4 or greater) before generating the"
+	echo "documentation for the Greenstone Librarian Interface."
+    fi
+    exit 1
+fi
+
+
+## ---- Document the GLI ----
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Generando documentación para la Interfaz de la Biblioteca Digital"
+    echo "Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Production de documentation pour la Greenstone Librarian Interface"
+elif [ "$glilang" == "ru" ]; then
+    echo "çÅÎÅÒÉÒÏ×ÁÎÉÅ ÄÏËÕÍÅÎÔÁÃÉÉ ÄÌÑ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone"
+else
+    echo "Generating documentation for the Greenstone Librarian Interface..."
+fi
+
+javadoc -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar -sourcepath src/ -source 1.4 -author -breakiterator -d docs/ -group "Greenstone Librarian Interface" org.greenstone.gatherer:org.greenstone.gatherer.cdm:org.greenstone.gatherer.collection:org.greenstone.gatherer.feedback:org.greenstone.gatherer.file:org.greenstone.gatherer.gems:org.greenstone.gatherer.gui:org.greenstone.gatherer.gui.metaaudit:org.greenstone.gatherer.gui.tree:org.greenstone.gatherer.metadata:org.greenstone.gatherer.shell:org.greenstone.gatherer.util -private -splitindex -version   org.greenstone.gatherer org.greenstone.gatherer.cdm org.greenstone.gatherer.collection org.greenstone.gatherer.feedback org.greenstone.gatherer.file org.greenstone.gatherer.gems org.greenstone.gatherer.gui org.greenstone.gatherer.gui.metaaudit org.greenstone.gatherer.gui.tree org.greenstone.gatherer.metadata org.greenstone.gatherer.shell org.greenstone.gatherer.util
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
Index: /main/tags/3.01/gli/gli.bat
===================================================================
--- /main/tags/3.01/gli/gli.bat	(revision 21112)
+++ /main/tags/3.01/gli/gli.bat	(revision 21112)
@@ -0,0 +1,293 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Run the Greenstone Librarian Interface --------
+
+:: This script must be run from within the directory in which it lives
+if exist gli.bat goto start
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+:start
+if "%OS%" == "Windows_NT" goto findGSDL
+    :: Invoke a new command processor to ensure there's enough environment space
+    if "%1" == "Second" goto findGSDL
+        command /E:2048 /C %0 Second %1 %2 %3 %4 %5 %6 %7 %8 %9
+        goto done
+
+:findGSDL
+echo.
+if "%GLILANG%" == "en" echo Greenstone Librarian Interface (GLI)
+if "%GLILANG%" == "en" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "en" echo GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+if "%GLILANG%" == "en" echo This is free software, and you are welcome to redistribute it
+
+if "%GLILANG%" == "es" echo Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "es" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÖA.
+if "%GLILANG%" == "es" echo Para mayor informaci¢n vea los trminos de la licencia en LICENSE.txt
+if "%GLILANG%" == "es" echo Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita
+
+if "%GLILANG%" == "fr" echo Interface du Bibliothcaire Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "fr" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "fr" echo GLI est fourni sans AUCUNE GARANTIE; pour des dtails, voir LICENSE.txt
+if "%GLILANG%" == "fr" echo Ceci est un logiciel libre, et vous tes invit 
+ le redistribuer
+
+if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "ru" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "ru" echo  ­¥ € ¥â   ; €¥â «š á¬. ¢ â¥ªáâ¥ LICENSE.TXT
+if "%GLILANG%" == "ru" echo â® - á¢®¡®€­® à á¯à®áâà ­ï¥¬®¥ ¯à®£à ¬¬­®¥ ®¡¥á¯¥ç¥­š¥ š ë ¬®Š¥â¥ à á¯à®áâà ­ïâì ¥£®
+
+echo.
+::  ---- Determine GSDLHOME ----
+set GSDLPATH=
+
+:: Some users may set the above line manually
+if not "%GSDLPATH%" == "" goto testGSDL
+
+    :: The default location for the GLI is a subdirectory of Greenstone
+    set GSDLPATH=..
+
+    :: If it is set, use the GSDLHOME environment variable 
+    if "%GSDLHOME%" == "" goto testGSDL
+	set GSDLPATH=%GSDLHOME%
+
+:testGSDL
+:: Check that the Greenstone installation looks OK
+if "%GLILANG%" == "en" echo Checking GSDL: %GSDLPATH%
+if "%GLILANG%" == "es" echo Revisando GSDL: %GSDLPATH%
+if "%GLILANG%" == "fr" echo Vrification de GSDL: %GSDLPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  GSDL: %GSDLPATH%
+if exist "%GSDLPATH%\setup.bat" goto prepGSDL
+    echo.
+    if "%GLILANG%" == "en" echo The Greenstone installation could not be found, or is incomplete.
+    if "%GLILANG%" == "en" echo Try reinstalling Greenstone then running this script again.
+
+    if "%GLILANG%" == "es" echo No se pudo encontrar la instalaci¢n de Greenstone o est  incompleta.
+    if "%GLILANG%" == "es" echo Trate de reinstalar Greenstone y a continuaci¢n ejecute nuevamente este gui¢n.
+
+    if "%GLILANG%" == "fr" echo L'installation de Greenstone est introuvable ou incomplte. Essayez
+    if "%GLILANG%" == "fr" echo de rinstaller Greenstone et excutez ce script 
+ nouveau.
+
+    if "%GLILANG%" == "ru" echo ­áâ ««ïæšï Greenstone ­¥ ¡ë«  ­ ©€¥­  š«š ®­  ­¥¯®«­ . ®¯à®¡ã©â¥ ¯®¢â®à­®
+    if "%GLILANG%" == "ru" echo ãáâ ­®¢šâì Greenstone,   § â¥¬ ¢¢¥áâš íâ®â áªàš¯â á­®¢ .
+    goto exit
+
+:prepGSDL
+:: Setup Greenstone, unless it has already been done
+if not "%GSDLHOME%" == "" goto doneGSDL
+    call "%GSDLPATH%\setup.bat" SetEnv > nul
+
+:doneGSDL
+:: GSDLPATH is no longer needed, since GSDLHOME should now be set
+set GSDLPATH=
+
+
+:findPerl
+::  ---- Check Perl exists ----
+set PERLPATH=
+
+:: Some users may set the above line manually
+if not "%PERLPATH%" == "" goto testPerl
+
+    :: Check if Perl is on the search path
+    echo %PATH%| winutil\which.exe perl.exe | winutil\setvar.exe PERLPATH > setperl.bat
+    call setperl.bat
+    del setperl.bat
+    if not "%PERLPATH%" == "" goto testPerl
+
+    :: If not, try GSDLHOME\bin\windows\perl\bin
+    if exist "%GSDLHOME%\bin\windows\perl\bin\perl.exe" goto gsdlPerl
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > perl.reg
+    regedit /E perl.reg "HKEY_LOCAL_MACHINE\SOFTWARE\Perl"
+    type perl.reg > perl.txt
+    del perl.reg
+
+    winutil\findperl.exe perl.txt | winutil\setvar.exe PERLPATH > setperl.bat
+    del perl.txt
+    call setperl.bat
+    del setperl.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%PERLPATH%" == "" goto noPerl
+
+    goto testPerl
+
+:gsdlPerl
+    set PERLPATH=%GSDLHOME%\bin\windows\perl\bin
+
+:testPerl
+:: Check that a Perl executable has been found
+if "%GLILANG%" == "en" echo Checking Perl: %PERLPATH%
+if "%GLILANG%" == "es" echo Revisando Perl: %PERLPATH%
+if "%GLILANG%" == "fr" echo Vrification de Perl: %PERLPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Perl: %PERLPATH%
+if exist "%PERLPATH%\perl.exe" goto findJava
+
+:noPerl
+    echo.
+    if "%GLILANG%" == "en" echo The Greenstone Librarian Interface requires perl in order to operate,
+    if "%GLILANG%" == "en" echo but perl could not be detected on your system. Please ensure that perl
+    if "%GLILANG%" == "en" echo is installed and is on your search path, then rerun this script.
+
+    if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone requiere Perl para poder
+    if "%GLILANG%" == "es" echo operar, pero ste no aparece en su sistema. Por favor aseg£rese de
+    if "%GLILANG%" == "es" echo que Perl est  instalado y se encuentra en su ruta de b£squeda.
+    if "%GLILANG%" == "es" echo A continuaci¢n ejecute nuevamente este gui¢n.
+
+    if "%GLILANG%" == "fr" echo Greenstone Librarian Interface ncessite perl pour son fonctionnement,
+    if "%GLILANG%" == "fr" echo mais perl n'a pas pu tre dtect dans votre systme. Veuillez vous 
+    if "%GLILANG%" == "fr" echo assurer que perl est install et est spcifi dans votre chemin de 
+    if "%GLILANG%" == "fr" echo recherche, puis redmarrez ce script.
+
+    if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone âà¥¡ã¥â Perl, çâ®¡ë š¬¥âì ¢®§¬®Š­®áâì
+    if "%GLILANG%" == "ru" echo à ¡®â âì, ­® Perl ­¥ ¡ë« ¢ ¢ è¥© ášáâ¥¬¥. ®Š «ã©áâ , ¯®€â¢¥à€šâ¥,
+    if "%GLILANG%" == "ru" echo çâ® Perl ãáâ ­®¢«¥­ š ­ å®€šâáï ­  ¢ è¥¬ ¯ãâš ¯®šáª , § â¥¬
+    if "%GLILANG%" == "ru" echo ¯®¢â®à­®¢¢¥€šâ¥ íâ®â áªàš¯â.
+    goto exit
+
+
+:findJava
+:: ---- Check Java exists ----
+set JAVAPATH=
+
+:: Some users may set the above line manually
+if not "%JAVAPATH%" == "" goto testJava
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if Java is on the search path
+    echo %PATH%| winutil\which.exe java.exe | winutil\setvar.exe JAVAPATH > setjava.bat
+    call setjava.bat
+    del setjava.bat
+    if not "%JAVAPATH%" == "" goto testJava
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+    type nul > jre.reg
+    regedit /E jre.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Runtime Environment"
+    type jre.reg > jre.txt
+    del jre.reg
+
+    winutil\findjava.exe jdk.txt jre.txt | winutil\setvar.exe JAVAPATH > setjava.bat
+    del jdk.txt
+    del jre.txt
+    call setjava.bat
+    del setjava.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JAVAPATH%" == "" goto noJava
+
+    set JAVAPATH=%JAVAPATH%\bin
+    goto testJava
+
+:javahome
+    set JAVAPATH=%JAVA_HOME%\bin
+
+:testJava
+:: Check that a Java executable has been found
+if "%GLILANG%" == "en" echo Checking Java: %JAVAPATH%
+if "%GLILANG%" == "es" echo Revisando Java: %JAVAPATH%
+if "%GLILANG%" == "fr" echo Vrification de Java: %JAVAPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Java: %JAVAPATH%
+if exist "%JAVAPATH%\java.exe" goto checkGLI
+
+:noJava
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of Java. You must install a
+    if "%GLILANG%" == "en" echo Java Runtime Environment (version 1.4 or greater) before running the
+    if "%GLILANG%" == "en" echo Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de Java. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Ambiente de Ejecuci¢n Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de correr la Interfaz de la Biblioteca Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version adquate de Java n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Java Runtime Environment (version 1.4 ou suprieur)
+    if "%GLILANG%" == "fr" echo avant de dmarrer Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš Java.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ãáâ ­®¢šâì Java Runtime Environment (¢¥àášî 1.4 š«š ¢ëè¥) ¯¥à¥€ ¢¢®€®¬
+    if "%GLILANG%" == "ru" echo ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+:checkGLI
+:: ---- Check that the GLI has been compiled ----
+if exist "classes/org/greenstone/gatherer/Gatherer.class" goto runGLI
+if exist "GLI.jar" goto runGLI
+    echo.
+    if "%GLILANG%" == "en" echo You need to compile the Greenstone Librarian Interface (using makegli.bat)
+    if "%GLILANG%" == "en" echo before running this script.
+
+    if "%GLILANG%" == "es" echo Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone
+    if "%GLILANG%" == "es" echo (por medio de makegli.bat) antes de ejecutar este gui¢n.
+
+    if "%GLILANG%" == "fr" echo Vous devez compiler le Greenstone Interface (en utilisant makegli.bat)
+    if "%GLILANG%" == "fr" echo avant d'excuter ce script.
+
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ª®¬¯š«šà®¢ âì ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (šá¯®«ì§ãï makegli.bat)
+    if "%GLILANG%" == "ru" echo ¯¥à¥€ ¢¢®€®¬ íâ®£® áªàš¯â 
+    goto exit
+
+
+:runGLI
+:: ---- Finally, run the GLI ----
+echo.
+
+:: ---- Set wget path manually if don't want to use the version (1.9) 
+:: that comes with Greenstone -------
+set WGETPATH=
+
+if "%GLILANG%" == "en" echo Running the Greenstone Librarian Interface...
+if "%GLILANG%" == "es" echo Ejecutando la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Excution de Greenstone Librarian Interface
+if "%GLILANG%" == "ru" echo ¥ªãéš© ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone...
+
+:: -Xms32M          To set minimum memory
+:: -Xmx32M          To set maximum memory
+:: -verbose:gc      To set garbage collection messages
+:: -Xincgc          For incremental garbage collection
+:: -Xprof           Function call profiling
+:: -Xloggc:<file>   Write garbage collection log
+
+if exist "%GSDLHOME%\server.exe" goto localLib 
+    "%JAVAPATH%\java" -cp classes/;GLI.jar;lib/apache.jar;lib/jp.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar org.greenstone.gatherer.GathererProg -gsdl %GSDLHOME% -perl %PERLPATH% -wget %WGETPATH% %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+    if "%GLILANG%" == "en" echo Done!
+    if "%GLILANG%" == "es" echo ­Hecho!
+    if "%GLILANG%" == "fr" echo Termin!
+    if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+    goto done
+
+:localLib
+    "%JAVAPATH%\java" -cp classes/;GLI.jar;lib/apache.jar;lib/jp.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar org.greenstone.gatherer.GathererProg -gsdl %GSDLHOME% -perl %PERLPATH% -local_library %GSDLHOME%\server.exe -wget %WGETPATH% %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+    if "%GLILANG%" == "en" echo Done!
+    if "%GLILANG%" == "es" echo ­Hecho!
+    if "%GLILANG%" == "fr" echo Termin!
+    if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+    goto done
+
+:exit
+echo.
+pause
+
+:done
+:: ---- Clean up ----
+set PERLPATH=
+set JAVAPATH=
+set WGETPATH=
Index: /main/tags/3.01/gli/gli.sh
===================================================================
--- /main/tags/3.01/gli/gli.sh	(revision 21112)
+++ /main/tags/3.01/gli/gli.sh	(revision 21112)
@@ -0,0 +1,293 @@
+#!/bin/sh
+glilang=en
+
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÍA."
+    echo "Para mayor información vea los términos de la licencia en LICENSE.txt"
+    echo "Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita"
+elif [ "$glilang" == "fr" ]; then
+    echo "Interface du Bibliothécaire Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI est fourni sans AUCUNE GARANTIE; pour des détails, voir LICENSE.txt"
+    echo "Ceci est un logiciel libre, et vous êtes invité à le redistribuer"
+elif [ "$glilang" == "ru" ]; then
+    echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "âéç ÎÅ ÄÁÅÔ áâóïìàôîï îéëáëéè çáòáîôéê; ÄÅÔÁÌÉ ÓÍ. × ÔÅËÓÔÅ LICENSE.TXT"
+    echo "üÔÏ - Ó×ÏÂÏÄÎÏ ÒÁÓÐÒÏÓÔÒÁÎÑÅÍÏÅ ÐÒÏÇÒÁÍÍÎÏÅ ÏÂÅÓÐÅÞÅÎÉÅ É ÷Ù ÍÏÖÅÔÅ ÒÁÓÐÒÏÓÔÒÁÎÑÔØ ÅÇÏ"
+else
+    echo "Greenstone Librarian Interface (GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt"
+    echo "This is free software, and you are welcome to redistribute it"
+fi
+echo
+
+##  -------- Run the Greenstone Librarian Interface --------
+
+# This script must be run from within the directory in which it lives
+thisdir=`pwd`
+if [ ! -f "${thisdir}/gli.sh" ]; then
+    if [ "$glilang" == "es" ]; then
+        echo "Este guión deberá ejecutarse desde el directorio en el que reside."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Ce script doit être exécuté à partir du répertoire dans lequel il se trouve."
+    elif [ "$glilang" == "ru" ]; then
+	echo "üÔÏÔ ÓËÒÉÐÔ ÄÏÌÖÅÎ ÂÙÔØ ×ÚÑÔ ÉÚ ÄÉÒÅËÔÏÒÉÉ, × ËÏÔÏÒÏÊ ÏÎ ÒÁÓÐÏÌÏÖÅÎ"
+    else
+	echo "This script must be run from the directory in which it resides."
+    fi
+    exit 1
+fi
+
+
+##  ---- Determine GSDLHOME ----
+gsdlpath=
+
+# Some users may set the above line manually
+if [ "$gsdlpath" == "" ]; then
+    # Check the environment variable first
+    if [ "$GSDLHOME" != "" ]; then
+	gsdlpath=$GSDLHOME
+
+    # If it is not set, assume that the GLI is installed as a subdirectory of Greenstone
+    else
+	pushd .. > /dev/null
+	gsdlpath=`pwd`
+	popd > /dev/null
+    fi
+fi
+
+# Check that the Greenstone installation looks OK
+if [ "$glilang" == "es" ]; then
+    echo "Revisando GSDL: $gsdlpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de GSDL: $gsdlpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ GSDL: $gsdlpath"
+else
+    echo "Checking GSDL: $gsdlpath"
+fi
+if [ ! -f "${gsdlpath}/setup.bash" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo encontrar la instalación de Greenstone o está incompleta."
+        echo "Trate de reinstalar Greenstone y a continuación ejecute nuevamente"
+	echo "este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "L'installation de Greenstone est introuvable ou incomplète."
+	echo "Essayez de réinstaller Greenstone et exécutez ce script à nouveau."
+    elif [ "$glilang" == "ru" ]; then
+	echo "éÎÓÔÁÌÌÑÃÉÑ Greenstone ÎÅ ÂÙÌÁ ÎÁÊÄÅÎÁ ÉÌÉ ÏÎÁ ÎÅÐÏÌÎÁ."
+	echo "ðÏÐÒÏÂÕÊÔÅ ÐÏ×ÔÏÒÎÏ ÕÓÔÁÎÏ×ÉÔØ Greenstone, Á ÚÁÔÅÍ ××ÅÓÔÉ ÜÔÏÔ ÓËÒÉÐÔ ÓÎÏ×Á."
+    else
+	echo "The Greenstone installation could not be found, or is incomplete."
+	echo "Try reinstalling Greenstone then running this script again."
+    fi
+    exit 1
+fi
+
+# Setup Greenstone, unless it has already been done
+if [ "$GSDLHOME" == "" ]; then
+    pushd "$gsdlpath" > /dev/null
+    source setup.bash
+    popd > /dev/null
+fi
+
+
+##  ---- Check Perl exists ----
+perlpath=
+
+# Some users may set the above line manually
+if [ "$perlpath" == "" ]; then
+    # Check if Perl is on the search path
+    perlpath=`which perl 2> /dev/null`
+fi
+
+# Check that a Perl executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Perl: $perlpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Perl: $perlpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Perl: $perlpath"
+else
+    echo "Checking Perl: $perlpath"
+fi
+if [ ! -x "$perlpath" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "La Interfaz de la Biblioteca Digital Greenstone requiere Perl para "
+	echo "poder operar, pero éste no aparece en su sistema. Por favor asegúrese "
+	echo "de que Perl está instalado y se encuentra en su ruta de búsqueda. A "
+	echo "continuación ejecute nuevamente este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Greenstone Librarian Interface nécessite perl pour son fonctionnement,"
+	echo "mais perl n'a pas pu être détecté dans votre système. Veuillez vous "
+	echo "assurer que perl est installé et est spécifié dans votre chemin de "
+	echo "recherche, puis redémarrez ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone ÔÒÅÂÕÅÔ Perl, ÞÔÏÂÙ ÉÍÅÔØ ×ÏÚÍÏÖÎÏÓÔØ"
+	echo "ÒÁÂÏÔÁÔØ, ÎÏ Perl ÎÅ ÂÙÌ × ×ÁÛÅÊ ÓÉÓÔÅÍÅ. ðÏÖÁÌÕÊÓÔÁ, ÐÏÄÔ×ÅÒÄÉÔÅ, ÞÔÏ "
+	echo "Perl ÕÓÔÁÎÏ×ÌÅÎ É ÎÁÈÏÄÉÔÓÑ ÎÁ ×ÁÛÅÍ ÐÕÔÉ ÐÏÉÓËÁ, ÚÁÔÅÍ ÐÏ×ÔÏÒÎÏ××ÅÄÉÔÅ"
+	echo "ÜÔÏÔ ÓËÒÉÐÔ."
+    else
+	echo "The Greenstone Librarian Interface requires perl in order to operate,"
+	echo "but perl could not be detected on your system. Please ensure that perl"
+	echo "is installed and is on your search path, then rerun this script."
+    fi
+    exit 1
+fi
+
+
+## ---- Check Java exists ----
+javapath=
+
+# Some users may set the above line manually
+if [ "$javapath" == "" ]; then
+
+    # If it is set, use the JAVA_HOME environment variable
+    if [ "$JAVA_HOME" != "" ]; then
+	javapath="$JAVA_HOME/bin/java"
+
+    # Check if Java is on the search path
+    else
+	javapath=`which java 2> /dev/null`
+    fi
+fi
+
+# Check that a Java executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Java: $javapath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Java: $javapath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Java: $javapath"
+else
+    echo "Checking Java: $javapath"
+fi
+if [ ! -x "$javapath" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo localizar una versión apropiada de Java. Usted deberá "
+	echo "instalar un Ambiente de Ejecución Java (versión 1.4 o superior) "
+	echo "antes de correr la Interfaz de la Biblioteca Digital Greenstone."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Une version adéquate de Java n'a pas pu être localisée."
+	echo "Vous devez installer un Java Runtime Environment (version 1.4 ou"
+	echo "supérieur) avant de démarrer Greenstone Librarian Interface."
+    elif [ "$glilang" == "ru" ]; then
+	echo "îÅ ÕÄÁÌÏÓØ ÏÐÒÅÄÅÌÉÔØ ÍÅÓÔÏÎÁÈÏÖÄÅÎÉÅ ÓÏÏÔ×ÅÔÓÔ×ÕÀÝÅÊ ×ÅÒÓÉÉ Java."
+	echo "÷Ù ÄÏÌÖÎÙ ÕÓÔÁÎÏ×ÉÔØ Java Runtime Environment (×ÅÒÓÉÀ 1.4 ÉÌÉ ×ÙÛÅ)"
+	echo "ÐÅÒÅÄ ××ÏÄÏÍ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone."
+    else
+	echo "Failed to locate an appropriate version of Java. You must install a"
+	echo "Java Runtime Environment (version 1.4 or greater) before running the"
+	echo "Greenstone Librarian Interface."
+    fi
+    exit 1
+fi
+
+
+## -- Check the version of Java is new enough (1.4.0 or higher) to run the GLI --
+`$javapath -classpath . CheckJavaVersion > /dev/null`
+if [ $? -ne 2 ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "La versión del Ambiente de Ejecución Java (JRE por sus siglas en "
+	echo "inglés) que usted tiene instalada es demasiado vieja para ejecutar "
+	echo "la Interfaz de la Biblioteca Digital Greenstone. Por favor instale "
+	echo "una nueva versión del Ambiente de Ejecución Java (versión 1.4 o "
+	echo "posterior) y ejecute nuevamente este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "La version de Java Runtime Environment que vous avez installée est"
+	echo "trop vielle pour faire fonctionner Greenstone Librarian Interface."
+	echo "Veuillez installer une nouvelle version du JRE (version 1.4 ou plus"
+	echo "récente) et redémarrez le script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷ÅÒÓÉÑ Java Runtime Environment, ËÏÔÏÒÕÀ ÷Ù ÕÓÔÁÎÏ×ÉÌÉ, ÏÞÅÎØ ÓÔÁÒÁ,"
+	echo "ÞÔÏÂÙ ÕÐÒÁ×ÌÑÔØ ÂÉÂÌÉÏÔÅÞÎÙÍ ÉÎÔÅÒÆÅÊÓÏÍ Greenstone. ðÏÖÁÌÕÊÓÔÁ, "
+	echo "ÕÓÔÁÎÏ×ÉÔÅ ÎÏ×ÕÀ ×ÅÒÓÉÀ JRE (×ÅÒÓÉÀ 1.4 ÉÌÉ ÂÏÌÅÅ ÎÏ×ÕÀ) É"
+	echo "ÐÅÒÅÕÓÔÁÎÏ×ÉÔÅ ÜÔÏÔ ÓËÒÉÐÔ"
+    else
+	echo "The version of the Java Runtime Environment you have installed is too"
+	echo "old to run the Greenstone Librarian Interface. Please install a new"
+	echo "version of the JRE (version 1.4 or newer) and rerun this script."
+    fi
+    exit 1
+fi
+
+## ---- Set wgetpath manually if you don't want to use the default Greenstone version -----
+wgetpath=
+
+## ---- Check that the GLI has been compiled ----
+if [ ! -f "classes/org/greenstone/gatherer/GathererProg.class" ] && [ ! -f "GLI.jar" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone"
+	echo "(por medio de makegli.sh) antes de ejecutar este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Vous devez compiler le Greenstone Interface (en utilisant makegli.sh)"
+	echo "avant d'exécuter ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷Ù ÄÏÌÖÎÙ ËÏÍÐÉÌÉÒÏ×ÁÔØ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone"
+	echo "(ÉÓÐÏÌØÚÕÑ makegli.sh) ÐÅÒÅÄ ××ÏÄÏÍ ÜÔÏÇÏ ÓËÒÉÐÔÁ"
+    else
+	echo "You need to compile the Greenstone Librarian Interface (using makegli.sh)"
+	echo "before running this script."
+    fi
+    exit 1
+fi
+
+if [ "$1" == "-test" ]; then
+    if [ "$glilang" == "es" ]; then
+	echo "Clase de prueba: $2"
+	echo "Argumentos $*"
+    elif [ "$glilang" == "fr" ]; then
+	echo "Classe de test: $2"
+	echo "Arguments $*"
+    elif [ "$glilang" == "ru" ]; then
+	echo "éÓÐÙÔÁÔÅÌØÎÙÊ ËÌÁÓÓ: $2"
+	echo "ðÁÒÁÍÅÔÒÙ $*"
+    else
+	echo "Testing class: $2"
+	echo "Arguments $*"
+    fi
+    $javapath -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar $2 $3
+    exit 0
+fi
+
+## ---- Finally, run the GLI ----
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Ejecutando la Interfaz de la Biblioteca Digital Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Exécution de Greenstone Librarian Interface"
+elif [ "$glilang" == "ru" ]; then
+    echo "ôÅËÕÝÉÊ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone..."
+else
+    echo "Running the Greenstone Librarian Interface..."
+fi
+
+# Other arguments you can provide to GLI to work around memory limitations, or debug
+# -Xms<number>M    To set minimum memory (by default 32MB)
+# -Xmx<number>M    To set maximum memory (by default the nearest 2^n to the total remaining physical memory)
+# -verbose:gc      To set garbage collection messages
+# -Xincgc          For incremental garbage collection (significantly slows performance)
+# -Xprof           Function call profiling
+# -Xloggc:<file>   Write garbage collection log
+
+$javapath -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar org.greenstone.gatherer.GathererProg -gsdl $GSDLHOME -wget $wgetpath $*
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
Index: /main/tags/3.01/gli/gli4gs3.bat
===================================================================
--- /main/tags/3.01/gli/gli4gs3.bat	(revision 21112)
+++ /main/tags/3.01/gli/gli4gs3.bat	(revision 21112)
@@ -0,0 +1,333 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Run the Greenstone Librarian Interface --------
+
+:: This script must be run from within the directory in which it lives
+if exist gli4gs3.bat goto start
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+:start
+if "%OS%" == "Windows_NT" goto findGSDL3
+    :: Invoke a new command processor to ensure there's enough environment space
+    if "%1" == "Second" goto findGSDL3
+        command /E:2048 /C %0 Second %1 %2 %3 %4 %5 %6 %7 %8 %9
+        goto done
+
+:findGSDL3
+echo.
+if "%GLILANG%" == "en" echo Greenstone Librarian Interface (GLI)
+if "%GLILANG%" == "en" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "en" echo GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+if "%GLILANG%" == "en" echo This is free software, and you are welcome to redistribute it
+
+if "%GLILANG%" == "es" echo Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "es" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÖA.
+if "%GLILANG%" == "es" echo Para mayor informaci¢n vea los trminos de la licencia en LICENSE.txt
+if "%GLILANG%" == "es" echo Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita
+
+if "%GLILANG%" == "fr" echo Interface du Bibliothcaire Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "fr" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "fr" echo GLI est fourni sans AUCUNE GARANTIE; pour des dtails, voir LICENSE.txt
+if "%GLILANG%" == "fr" echo Ceci est un logiciel libre, et vous tes invit 
+ le redistribuer
+
+if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "ru" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "ru" echo  ­¥ € ¥â   ; €¥â «š á¬. ¢ â¥ªáâ¥ LICENSE.TXT
+if "%GLILANG%" == "ru" echo â® - á¢®¡®€­® à á¯à®áâà ­ï¥¬®¥ ¯à®£à ¬¬­®¥ ®¡¥á¯¥ç¥­š¥ š ë ¬®Š¥â¥ à á¯à®áâà ­ïâì ¥£®
+
+echo.
+::  ---- Determine GSDL3HOME ----
+set GSDL3PATH=%GSDL3PATH%
+
+:: Some users may set the above line manually, or it may be set as an argument
+if not "%GSDL3PATH%" == "" goto testGSDL3
+
+    :: The default location for the GLI is a subdirectory of Greenstone 3
+    set GSDL3PATH=..
+
+    :: If it is set, use the GSDL3HOME environment variable
+    if "%GSDL3SRCHOME%" == "" goto testGSDL3
+	set GSDL3PATH=%GSDL3SRCHOME%
+
+:testGSDL3
+:: Check that the Greenstone 3 installation looks OK
+if "%GLILANG%" == "en" echo Checking GSDL3: %GSDL3PATH%
+if "%GLILANG%" == "es" echo Revisando GSDL3: %GSDL3PATH%
+if "%GLILANG%" == "fr" echo Vrification de GSDL3: %GSDL3PATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  GSDL3: %GSDL3PATH%
+if exist "%GSDL3PATH%\gs3-setup.bat" goto prepGSDL3
+    echo.
+    if "%GLILANG%" == "en" echo The Greenstone 3 installation could not be found, or is incomplete.
+    if "%GLILANG%" == "en" echo Try reinstalling Greenstone 3 then running this script again.
+
+    if "%GLILANG%" == "es" echo No se pudo encontrar la instalaci¢n de Greenstone 3 o est  incompleta.
+    if "%GLILANG%" == "es" echo Trate de reinstalar Greenstone 3 y a continuaci¢n ejecute nuevamente este gui¢n.
+
+    if "%GLILANG%" == "fr" echo L'installation de Greenstone 3 est introuvable ou incomplte. Essayez
+    if "%GLILANG%" == "fr" echo de rinstaller Greenstone 3 et excutez ce script 
+ nouveau.
+
+    if "%GLILANG%" == "ru" echo ­áâ ««ïæšï Greenstone 3 ­¥ ¡ë«  ­ ©€¥­  š«š ®­  ­¥¯®«­ . ®¯à®¡ã©â¥ ¯®¢â®à­®
+    if "%GLILANG%" == "ru" echo ãáâ ­®¢šâì Greenstone, 3   § â¥¬ ¢¢¥áâš íâ®â áªàš¯â á­®¢ .
+    goto exit
+
+:prepGSDL3
+:: Setup Greenstone 3, unless it has already been done
+if not "%GSDL3SRCHOME%" == "" goto doneGSDL3
+    cd | winutil\setvar.exe GLIDIR > setgli.bat
+    call setgli.bat
+    del setgli.bat
+    cd "%GSDL3PATH%"
+    call gs3-setup.bat SetEnv > nul
+    cd %GLIDIR%
+
+:doneGSDL3
+:: GSDL3PATH is no longer needed, since GSDL3HOME should now be set
+set GSDL3PATH=
+
+
+::  ---- Determine GSDLHOME ----
+set GSDLPATH=%GSDLPATH%
+
+:: Some users may set the above line manually, or it may be set in the environment
+if not "%GSDLPATH%" == "" goto testGSDL
+
+    :: The default location for Greenstone is a gs2build subdirectory of Greenstone 3
+    set GSDLPATH=%GSDL3SRCHOME%\gs2build
+
+    :: If it is set, use the GSDLHOME environment variable
+    if "%GSDLHOME%" == "" goto testGSDL
+	set GSDLPATH=%GSDLHOME%
+
+:testGSDL
+:: Check that the Greenstone installation looks OK
+if "%GLILANG%" == "en" echo Checking GSDL: %GSDLPATH%
+if "%GLILANG%" == "es" echo Revisando GSDL: %GSDLPATH%
+if "%GLILANG%" == "fr" echo Vrification de GSDL: %GSDLPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  GSDL: %GSDLPATH%
+if exist "%GSDLPATH%\setup.bat" goto prepGSDL
+    echo.
+    if "%GLILANG%" == "en" echo The Greenstone installation could not be found, or is incomplete.
+    if "%GLILANG%" == "en" echo Try reinstalling Greenstone then running this script again.
+
+    if "%GLILANG%" == "es" echo No se pudo encontrar la instalaci¢n de Greenstone o est  incompleta.
+    if "%GLILANG%" == "es" echo Trate de reinstalar Greenstone y a continuaci¢n ejecute nuevamente este gui¢n.
+
+    if "%GLILANG%" == "fr" echo L'installation de Greenstone est introuvable ou incomplte. Essayez
+    if "%GLILANG%" == "fr" echo de rinstaller Greenstone et excutez ce script 
+ nouveau.
+
+    if "%GLILANG%" == "ru" echo ­áâ ««ïæšï Greenstone ­¥ ¡ë«  ­ ©€¥­  š«š ®­  ­¥¯®«­ . ®¯à®¡ã©â¥ ¯®¢â®à­®
+    if "%GLILANG%" == "ru" echo ãáâ ­®¢šâì Greenstone,   § â¥¬ ¢¢¥áâš íâ®â áªàš¯â á­®¢ .
+    goto exit
+
+:prepGSDL
+:: Setup Greenstone, unless it has already been done
+if not "%GSDLHOME%" == "" goto doneGSDL
+    call "%GSDLPATH%\setup.bat" SetEnv > nul
+
+:doneGSDL
+:: GSDLPATH is no longer needed, since GSDLHOME should now be set
+set GSDLPATH=
+
+
+:findPerl
+::  ---- Check Perl exists ----
+set PERLPATH=
+
+:: Some users may set the above line manually
+if not "%PERLPATH%" == "" goto testPerl
+
+    :: Check if Perl is on the search path
+    echo %PATH%| winutil\which.exe perl.exe | winutil\setvar.exe PERLPATH > setperl.bat
+    call setperl.bat
+    del setperl.bat
+    if not "%PERLPATH%" == "" goto testPerl
+
+    :: If not, try GSDLHOME\bin\windows\perl\bin
+    if exist "%GSDLHOME%\bin\windows\perl\bin\perl.exe" goto gsdlPerl
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > perl.reg
+    regedit /E perl.reg "HKEY_LOCAL_MACHINE\SOFTWARE\Perl"
+    type perl.reg > perl.txt
+    del perl.reg
+
+    winutil\findperl.exe perl.txt | winutil\setvar.exe PERLPATH > setperl.bat
+    del perl.txt
+    call setperl.bat
+    del setperl.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%PERLPATH%" == "" goto noPerl
+
+    goto testPerl
+
+:gsdlPerl
+    set PERLPATH=%GSDLHOME%\bin\windows\perl\bin
+
+:testPerl
+:: Check that a Perl executable has been found
+if "%GLILANG%" == "en" echo Checking Perl: %PERLPATH%
+if "%GLILANG%" == "es" echo Revisando Perl: %PERLPATH%
+if "%GLILANG%" == "fr" echo Vrification de Perl: %PERLPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Perl: %PERLPATH%
+if exist "%PERLPATH%\perl.exe" goto findJava
+
+:noPerl
+    echo.
+    if "%GLILANG%" == "en" echo The Greenstone Librarian Interface requires perl in order to operate,
+    if "%GLILANG%" == "en" echo but perl could not be detected on your system. Please ensure that perl
+    if "%GLILANG%" == "en" echo is installed and is on your search path, then rerun this script.
+
+    if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone requiere Perl para poder
+    if "%GLILANG%" == "es" echo operar, pero ste no aparece en su sistema. Por favor aseg£rese de
+    if "%GLILANG%" == "es" echo que Perl est  instalado y se encuentra en su ruta de b£squeda.
+    if "%GLILANG%" == "es" echo A continuaci¢n ejecute nuevamente este gui¢n.
+
+    if "%GLILANG%" == "fr" echo Greenstone Librarian Interface ncessite perl pour son fonctionnement,
+    if "%GLILANG%" == "fr" echo mais perl n'a pas pu tre dtect dans votre systme. Veuillez vous 
+    if "%GLILANG%" == "fr" echo assurer que perl est install et est spcifi dans votre chemin de 
+    if "%GLILANG%" == "fr" echo recherche, puis redmarrez ce script.
+
+    if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone âà¥¡ã¥â Perl, çâ®¡ë š¬¥âì ¢®§¬®Š­®áâì
+    if "%GLILANG%" == "ru" echo à ¡®â âì, ­® Perl ­¥ ¡ë« ¢ ¢ è¥© ášáâ¥¬¥. ®Š «ã©áâ , ¯®€â¢¥à€šâ¥,
+    if "%GLILANG%" == "ru" echo çâ® Perl ãáâ ­®¢«¥­ š ­ å®€šâáï ­  ¢ è¥¬ ¯ãâš ¯®šáª , § â¥¬
+    if "%GLILANG%" == "ru" echo ¯®¢â®à­®¢¢¥€šâ¥ íâ®â áªàš¯â.
+    goto exit
+
+
+:findJava
+:: ---- Check Java exists ----
+set JAVAPATH=
+
+:: Some users may set the above line manually
+if not "%JAVAPATH%" == "" goto testJava
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if Java is on the search path
+    echo %PATH%| winutil\which.exe java.exe | winutil\setvar.exe JAVAPATH > setjava.bat
+    call setjava.bat
+    del setjava.bat
+    if not "%JAVAPATH%" == "" goto testJava
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+    type nul > jre.reg
+    regedit /E jre.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Runtime Environment"
+    type jre.reg > jre.txt
+    del jre.reg
+
+    winutil\findjava.exe jdk.txt jre.txt | winutil\setvar.exe JAVAPATH > setjava.bat
+    del jdk.txt
+    del jre.txt
+    call setjava.bat
+    del setjava.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JAVAPATH%" == "" goto noJava
+
+    set JAVAPATH=%JAVAPATH%\bin
+    goto testJava
+
+:javahome
+    set JAVAPATH=%JAVA_HOME%\bin
+
+:testJava
+:: Check that a Java executable has been found
+if "%GLILANG%" == "en" echo Checking Java: %JAVAPATH%
+if "%GLILANG%" == "es" echo Revisando Java: %JAVAPATH%
+if "%GLILANG%" == "fr" echo Vrification de Java: %JAVAPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Java: %JAVAPATH%
+if exist "%JAVAPATH%\java.exe" goto checkGLI
+
+:noJava
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of Java. You must install a
+    if "%GLILANG%" == "en" echo Java Runtime Environment (version 1.4 or greater) before running the
+    if "%GLILANG%" == "en" echo Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de Java. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Ambiente de Ejecuci¢n Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de correr la Interfaz de la Biblioteca Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version adquate de Java n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Java Runtime Environment (version 1.4 ou suprieur)
+    if "%GLILANG%" == "fr" echo avant de dmarrer Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš Java.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ãáâ ­®¢šâì Java Runtime Environment (¢¥àášî 1.4 š«š ¢ëè¥) ¯¥à¥€ ¢¢®€®¬
+    if "%GLILANG%" == "ru" echo ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+:checkGLI
+:: ---- Check that the GLI has been compiled ----
+if exist "classes/org/greenstone/gatherer/GathererProg.class" goto runGLI
+if exist "GLI.jar" goto runGLI
+    echo.
+    if "%GLILANG%" == "en" echo You need to compile the Greenstone Librarian Interface (using makegli.bat)
+    if "%GLILANG%" == "en" echo before running this script.
+
+    if "%GLILANG%" == "es" echo Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone
+    if "%GLILANG%" == "es" echo (por medio de makegli.bat) antes de ejecutar este gui¢n.
+
+    if "%GLILANG%" == "fr" echo Vous devez compiler le Greenstone Interface (en utilisant makegil.bat)
+    if "%GLILANG%" == "fr" echo avant d'excuter ce script.
+
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë ª®¬¯š«šà®¢ âì ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (šá¯®«ì§ãï makegli.bat)
+    if "%GLILANG%" == "ru" echo ¯¥à¥€ ¢¢®€®¬ íâ®£® áªàš¯â 
+    goto exit
+
+
+:runGLI
+:: ---- Finally, run the GLI ----
+echo.
+
+:: ---- Set wget path manually if don't want to use the version (1.9) 
+:: that comes with Greenstone -------
+set WGETPATH=
+
+if "%GLILANG%" == "en" echo Running the Greenstone Librarian Interface...
+if "%GLILANG%" == "es" echo Ejecutando la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Excution de Greenstone Librarian Interface
+if "%GLILANG%" == "ru" echo ¥ªãéš© ¡š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone...
+
+:: -Xms32M          To set minimum memory
+:: -Xmx32M          To set maximum memory
+:: -verbose:gc      To set garbage collection messages
+:: -Xincgc          For incremental garbage collection
+:: -Xprof           Function call profiling
+:: -Xloggc:<file>   Write garbage collection log
+
+"%JAVAPATH%\java" -cp classes/;GLI.jar;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar org.greenstone.gatherer.GathererProg -gsdl %GSDLHOME% -gsdl3 %GSDL3HOME% -gsdl3src %GSDL3SRCHOME% -perl %PERLPATH% -wget %WGETPATH% %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+if "%GLILANG%" == "en" echo Done!
+if "%GLILANG%" == "es" echo ­Hecho!
+if "%GLILANG%" == "fr" echo Termin!
+if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+goto done
+
+:exit
+echo.
+pause
+
+:done
+:: ---- Clean up ----
+set PERLPATH=
+set JAVAPATH=
+set WGETPATH=
Index: /main/tags/3.01/gli/gli4gs3.sh
===================================================================
--- /main/tags/3.01/gli/gli4gs3.sh	(revision 21112)
+++ /main/tags/3.01/gli/gli4gs3.sh	(revision 21112)
@@ -0,0 +1,347 @@
+#!/bin/sh
+glilang=en
+
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÍA."
+    echo "Para mayor información vea los términos de la licencia en LICENSE.txt"
+    echo "Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita"
+elif [ "$glilang" == "fr" ]; then
+    echo "Interface du Bibliothécaire Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI est fourni sans AUCUNE GARANTIE; pour des détails, voir LICENSE.txt"
+    echo "Ceci est un logiciel libre, et vous êtes invité à le redistribuer"
+elif [ "$glilang" == "ru" ]; then
+    echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "âéç ÎÅ ÄÁÅÔ áâóïìàôîï îéëáëéè çáòáîôéê; ÄÅÔÁÌÉ ÓÍ. × ÔÅËÓÔÅ LICENSE.TXT"
+    echo "üÔÏ - Ó×ÏÂÏÄÎÏ ÒÁÓÐÒÏÓÔÒÁÎÑÅÍÏÅ ÐÒÏÇÒÁÍÍÎÏÅ ÏÂÅÓÐÅÞÅÎÉÅ É ÷Ù ÍÏÖÅÔÅ ÒÁÓÐÒÏÓÔÒÁÎÑÔØ ÅÇÏ"
+else
+    echo "Greenstone Librarian Interface (GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt"
+    echo "This is free software, and you are welcome to redistribute it"
+fi
+echo
+
+##  -------- Run the Greenstone Librarian Interface --------
+
+# This script must be run from within the directory in which it lives
+thisdir=`pwd`
+if [ ! -f "${thisdir}/gli4gs3.sh" ]; then
+    if [ "$glilang" == "es" ]; then
+        echo "Este guión deberá ejecutarse desde el directorio en el que reside."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Ce script doit être exécuté à partir du répertoire dans lequel il se trouve."
+    elif [ "$glilang" == "ru" ]; then
+	echo "üÔÏÔ ÓËÒÉÐÔ ÄÏÌÖÅÎ ÂÙÔØ ×ÚÑÔ ÉÚ ÄÉÒÅËÔÏÒÉÉ, × ËÏÔÏÒÏÊ ÏÎ ÒÁÓÐÏÌÏÖÅÎ"
+    else
+	echo "This script must be run from the directory in which it resides."
+    fi
+    exit 1
+fi
+
+
+##  ---- Determine GSDL3HOME ----
+## may be already set or manually entered here
+gsdl3path=$gsdl3path
+
+# Some users may set the above line manually
+if [ "$gsdl3path" == "" ]; then
+    # Check the environment variable first
+    if [ "$GSDL3SRCHOME" != "" ]; then
+	gsdl3path=$GSDL3SRCHOME
+
+    # If it is not set, assume that the GLI is installed as a subdirectory of Greenstone 3
+    else
+	pushd .. > /dev/null
+	gsdl3path=`pwd`
+	popd > /dev/null
+    fi
+fi
+
+# Check that the Greenstone 3 installation looks OK
+if [ "$glilang" == "es" ]; then
+    echo "Revisando GSDL3: $gsdl3path"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de GSDL3: $gsdl3path"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ GSDL3: $gsdl3path"
+else
+    echo "Checking GSDL3: $gsdl3path"
+fi
+if [ ! -f "${gsdl3path}/gs3-setup.sh" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo encontrar la instalación de Greenstone 3 o está incompleta."
+        echo "Trate de reinstalar Greenstone 3 y a continuación ejecute nuevamente"
+	echo "este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "L'installation de Greenstone 3 est introuvable ou incomplète."
+	echo "Essayez de réinstaller Greenstone 3 et exécutez ce script à nouveau."
+    elif [ "$glilang" == "ru" ]; then
+	echo "éÎÓÔÁÌÌÑÃÉÑ Greenstone 3 ÎÅ ÂÙÌÁ ÎÁÊÄÅÎÁ ÉÌÉ ÏÎÁ ÎÅÐÏÌÎÁ."
+	echo "ðÏÐÒÏÂÕÊÔÅ ÐÏ×ÔÏÒÎÏ ÕÓÔÁÎÏ×ÉÔØ Greenstone 3, Á ÚÁÔÅÍ ××ÅÓÔÉ ÜÔÏÔ ÓËÒÉÐÔ ÓÎÏ×Á."
+    else
+	echo "The Greenstone 3 installation could not be found, or is incomplete."
+	echo "Try reinstalling Greenstone 3 then running this script again."
+    fi
+    exit 1
+fi
+
+# Setup Greenstone 3, unless it has already been done
+if [ "$GSDL3HOME" == "" ] || [ "$GSDL3SRCHOME" == "" ]; then
+    pushd $gsdl3path > /dev/null
+    source gs3-setup.sh
+    popd > /dev/null
+fi
+
+
+##  ---- Determine GSDLHOME ----
+## may be already set, or manually entered here.
+gsdlpath=$gsdlpath
+
+# Some users may set the above line manually
+if [ "$gsdlpath" == "" ]; then
+    # Check the environment variable first
+    if [ "$GSDLHOME" != "" ]; then
+	gsdlpath=$GSDLHOME
+
+    # If it is not set, assume that the gs2build subdirectory of Greenstone 3 exists
+    else
+	gsdlpath=$GSDL3SRCHOME/gs2build
+    fi
+fi
+
+# Check that the Greenstone installation looks OK
+if [ "$glilang" == "es" ]; then
+    echo "Revisando GSDL: $gsdlpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de GSDL: $gsdlpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ GSDL: $gsdlpath"
+else
+    echo "Checking GSDL: $gsdlpath"
+fi
+if [ ! -f "${gsdlpath}/setup.bash" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo encontrar la instalación de Greenstone o está incompleta."
+        echo "Trate de reinstalar Greenstone y a continuación ejecute nuevamente"
+	echo "este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "L'installation de Greenstone est introuvable ou incomplète."
+	echo "Essayez de réinstaller Greenstone et exécutez ce script à nouveau."
+    elif [ "$glilang" == "ru" ]; then
+	echo "éÎÓÔÁÌÌÑÃÉÑ Greenstone ÎÅ ÂÙÌÁ ÎÁÊÄÅÎÁ ÉÌÉ ÏÎÁ ÎÅÐÏÌÎÁ."
+	echo "ðÏÐÒÏÂÕÊÔÅ ÐÏ×ÔÏÒÎÏ ÕÓÔÁÎÏ×ÉÔØ Greenstone, Á ÚÁÔÅÍ ××ÅÓÔÉ ÜÔÏÔ ÓËÒÉÐÔ ÓÎÏ×Á."
+    else
+	echo "The Greenstone installation could not be found, or is incomplete."
+	echo "Try reinstalling Greenstone then running this script again."
+    fi
+    exit 1
+fi
+
+# Setup Greenstone, unless it has already been done
+if [ "$GSDLHOME" == "" ]; then
+    pushd $gsdlpath > /dev/null
+    source setup.bash
+    popd > /dev/null
+fi
+
+
+##  ---- Check Perl exists ----
+perlpath=
+
+# Some users may set the above line manually
+if [ "$perlpath" == "" ]; then
+    # Check if Perl is on the search path
+    perlpath=`which perl 2> /dev/null`
+fi
+
+# Check that a Perl executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Perl: $perlpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Perl: $perlpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Perl: $perlpath"
+else
+    echo "Checking Perl: $perlpath"
+fi
+if [ ! -x "$perlpath" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "La Interfaz de la Biblioteca Digital Greenstone requiere Perl para "
+	echo "poder operar, pero éste no aparece en su sistema. Por favor asegúrese "
+	echo "de que Perl está instalado y se encuentra en su ruta de búsqueda. A "
+	echo "continuación ejecute nuevamente este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Greenstone Librarian Interface nécessite perl pour son fonctionnement,"
+	echo "mais perl n'a pas pu être détecté dans votre système. Veuillez vous "
+	echo "assurer que perl est installé et est spécifié dans votre chemin de "
+	echo "recherche, puis redémarrez ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone ÔÒÅÂÕÅÔ Perl, ÞÔÏÂÙ ÉÍÅÔØ ×ÏÚÍÏÖÎÏÓÔØ"
+	echo "ÒÁÂÏÔÁÔØ, ÎÏ Perl ÎÅ ÂÙÌ × ×ÁÛÅÊ ÓÉÓÔÅÍÅ. ðÏÖÁÌÕÊÓÔÁ, ÐÏÄÔ×ÅÒÄÉÔÅ, ÞÔÏ "
+	echo "Perl ÕÓÔÁÎÏ×ÌÅÎ É ÎÁÈÏÄÉÔÓÑ ÎÁ ×ÁÛÅÍ ÐÕÔÉ ÐÏÉÓËÁ, ÚÁÔÅÍ ÐÏ×ÔÏÒÎÏ××ÅÄÉÔÅ"
+	echo "ÜÔÏÔ ÓËÒÉÐÔ."
+    else
+	echo "The Greenstone Librarian Interface requires perl in order to operate,"
+	echo "but perl could not be detected on your system. Please ensure that perl"
+	echo "is installed and is on your search path, then rerun this script."
+    fi
+    exit 1
+fi
+
+
+## ---- Check Java exists ----
+javapath=
+
+# Some users may set the above line manually
+if [ "$javapath" == "" ]; then
+
+    # If it is set, use the JAVA_HOME environment variable
+    if [ "$JAVA_HOME" != "" ]; then
+	javapath="$JAVA_HOME/bin/java"
+
+    # Check if Java is on the search path
+    else
+	javapath=`which java 2> /dev/null`
+    fi
+fi
+
+# Check that a Java executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Java: $javapath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Java: $javapath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Java: $javapath"
+else
+    echo "Checking Java: $javapath"
+fi
+if [ ! -x "$javapath" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo localizar una versión apropiada de Java. Usted deberá "
+	echo "instalar un Ambiente de Ejecución Java (versión 1.4 o superior) "
+	echo "antes de correr la Interfaz de la Biblioteca Digital Greenstone."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Une version adéquate de Java n'a pas pu être localisée."
+	echo "Vous devez installer un Java Runtime Environment (version 1.4 ou"
+	echo "supérieur) avant de démarrer Greenstone Librarian Interface."
+    elif [ "$glilang" == "ru" ]; then
+	echo "îÅ ÕÄÁÌÏÓØ ÏÐÒÅÄÅÌÉÔØ ÍÅÓÔÏÎÁÈÏÖÄÅÎÉÅ ÓÏÏÔ×ÅÔÓÔ×ÕÀÝÅÊ ×ÅÒÓÉÉ Java."
+	echo "÷Ù ÄÏÌÖÎÙ ÕÓÔÁÎÏ×ÉÔØ Java Runtime Environment (×ÅÒÓÉÀ 1.4 ÉÌÉ ×ÙÛÅ)"
+	echo "ÐÅÒÅÄ ××ÏÄÏÍ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone."
+    else
+	echo "Failed to locate an appropriate version of Java. You must install a"
+	echo "Java Runtime Environment (version 1.4 or greater) before running the"
+	echo "Greenstone Librarian Interface."
+    fi
+    exit 1
+fi
+
+
+## -- Check the version of Java is new enough (1.4.0 or higher) to run the GLI --
+`$javapath -classpath . CheckJavaVersion > /dev/null`
+if [ $? -ne 2 ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "La versión del Ambiente de Ejecución Java (JRE por sus siglas en "
+	echo "inglés) que usted tiene instalada es demasiado vieja para ejecutar "
+	echo "la Interfaz de la Biblioteca Digital Greenstone. Por favor instale "
+	echo "una nueva versión del Ambiente de Ejecución Java (versión 1.4 o "
+	echo "posterior) y ejecute nuevamente este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "La version de Java Runtime Environment que vous avez installée est"
+	echo "trop vielle pour faire fonctionner Greenstone Librarian Interface."
+	echo "Veuillez installer une nouvelle version du JRE (version 1.4 ou plus"
+	echo "récente) et redémarrez le script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷ÅÒÓÉÑ Java Runtime Environment, ËÏÔÏÒÕÀ ÷Ù ÕÓÔÁÎÏ×ÉÌÉ, ÏÞÅÎØ ÓÔÁÒÁ,"
+	echo "ÞÔÏÂÙ ÕÐÒÁ×ÌÑÔØ ÂÉÂÌÉÏÔÅÞÎÙÍ ÉÎÔÅÒÆÅÊÓÏÍ Greenstone. ðÏÖÁÌÕÊÓÔÁ, "
+	echo "ÕÓÔÁÎÏ×ÉÔÅ ÎÏ×ÕÀ ×ÅÒÓÉÀ JRE (×ÅÒÓÉÀ 1.4 ÉÌÉ ÂÏÌÅÅ ÎÏ×ÕÀ) É"
+	echo "ÐÅÒÅÕÓÔÁÎÏ×ÉÔÅ ÜÔÏÔ ÓËÒÉÐÔ"
+    else
+	echo "The version of the Java Runtime Environment you have installed is too"
+	echo "old to run the Greenstone Librarian Interface. Please install a new"
+	echo "version of the JRE (version 1.4 or newer) and rerun this script."
+    fi
+    exit 1
+fi
+
+## ---- Set wgetpath manually if you don't want to use the default Greenstone version -----
+wgetpath=
+
+## ---- Check that the GLI has been compiled ----
+if [ ! -f "classes/org/greenstone/gatherer/GathererProg.class" ] && [ ! -f "GLI.jar" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone"
+	echo "(por medio de makegli.sh) antes de ejecutar este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Vous devez compiler le Greenstone Interface (en utilisant makegli.sh)"
+	echo "avant d'exécuter ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷Ù ÄÏÌÖÎÙ ËÏÍÐÉÌÉÒÏ×ÁÔØ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone"
+	echo "(ÉÓÐÏÌØÚÕÑ makegli.sh) ÐÅÒÅÄ ××ÏÄÏÍ ÜÔÏÇÏ ÓËÒÉÐÔÁ"
+    else
+	echo "You need to compile the Greenstone Librarian Interface (using makegli.sh)"
+	echo "before running this script."
+    fi
+    exit 1
+fi
+
+if [ "$1" == "-test" ]; then
+    if [ "$glilang" == "es" ]; then
+	echo "Clase de prueba: $2"
+	echo "Argumentos $*"
+    elif [ "$glilang" == "fr" ]; then
+	echo "Classe de test: $2"
+	echo "Arguments $*"
+    elif [ "$glilang" == "ru" ]; then
+	echo "éÓÐÙÔÁÔÅÌØÎÙÊ ËÌÁÓÓ: $2"
+	echo "ðÁÒÁÍÅÔÒÙ $*"
+    else
+	echo "Testing class: $2"
+	echo "Arguments $*"
+    fi
+    $javapath -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar $2 $3
+    exit 0
+fi
+
+## ---- Finally, run the GLI ----
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Ejecutando la Interfaz de la Biblioteca Digital Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Exécution de Greenstone Librarian Interface"
+elif [ "$glilang" == "ru" ]; then
+    echo "ôÅËÕÝÉÊ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone..."
+else
+    echo "Running the Greenstone Librarian Interface..."
+fi
+
+# Other arguments you can provide to GLI to work around memory limitations, or debug
+# -Xms<number>M    To set minimum memory (by default 32MB)
+# -Xmx<number>M    To set maximum memory (by default the nearest 2^n to the total remaining physical memory)
+# -verbose:gc      To set garbage collection messages
+# -Xincgc          For incremental garbage collection (significantly slows performance)
+# -Xprof           Function call profiling
+# -Xloggc:<file>   Write garbage collection log
+
+$javapath -classpath classes/:GLI.jar:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar org.greenstone.gatherer.GathererProg -gsdl $GSDLHOME -gsdl3 $GSDL3HOME -gsdl3src $GSDL3SRCHOME -wget $wgetpath $*
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
Index: /main/tags/3.01/gli/help/en/addingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/en/addingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/addingfiles.htm	(revision 21112)
@@ -0,0 +1,89 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingfiles"><font size="5" face="Verdana"><strong>4.3: Adding Files</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section shows how to get files into your collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Dragging a file
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Multiple files
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Existing metadata
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Files can be copied into the collection by dragging and dropping.  The mouse
+pointer becomes a ghost of the selected item (or, if more than one is selected,
+the number of them).  Drop the selection into the Collection Tree to copy the
+files there (if the source was the Workspace Tree) or move them around within
+the collection (if the source was the Collection Tree).
+<p></p>
+When copying multiple files, they are all placed in the target folder at the
+same level, irrespective of the folder structure they occupied originally.
+When you copy a second file with the same name into the same folder, you are
+asked whether to overwrite the first one.  Respond "No" and the file will not be
+copied, but the others will be.  To cancel all remaining copy actions, click the
+"stop" button.
+<p></p>
+Only the "highest" items in a selection are moved.  A folder is higher than its
+children.  You cannot select files within a folder and also the folder itself.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+When you add a file, the Librarian Interface searches through the source
+folders for auxiliary files containing metadata previously assigned to the
+added file and, if it finds one, begins to import this metadata.  As the
+operation proceeds, you may be prompted (perhaps several times) for extra
+information to match the imported metadata to the metadata sets in your
+collection.  This process involves many different prompts, described in the <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importing Previously Assigned Metadata</a> section.  For a more detailed
+explanation of associating metadata with files read Chapter 2 of the Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/addingpreviouslydefinedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/addingpreviouslydefinedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/addingpreviouslydefinedmetadata.htm	(revision 21112)
@@ -0,0 +1,60 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingpreviouslydefinedmetadata"><font size="5" face="Verdana"><strong>5.4: Adding Previously Defined Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to add metadata that uses values already present in the value tree.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding to files
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding to folders or multiple files
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To add metadata that has an existing value, first select the file, then select
+the required value from the value tree, expanding hierarchy folders as
+necessary.  The value of the selected entry automatically appears in the Value
+text field (alternatively, use the value tree's auto-select and auto-complete
+features).  Click "Append" to add the metadata to the selected file.
+<p></p>
+The process of adding metadata with already-existing values to folders
+or multiple files is just the same.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/appendingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/appendingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/appendingmetadata.htm	(revision 21112)
@@ -0,0 +1,86 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="appendingmetadata"><font size="5" face="Verdana"><strong>5.3: Appending New Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to add metadata and values to files, and how to add new values for metadata elements.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding to files
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding to folders or multiple files
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Value hierarchies and editing values
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+We now add a metadata item -- both element and value -- to a file.  First select
+the file from the Collection file tree on the left.  The action causes any
+metadata previously assigned to this file to appear in the table at the right.
+<p></p>
+Next select the metadata element you want to add by clicking its row in the
+table.
+<p></p>
+Type the value into the value field.  Do not use the character "|", as it is
+used for constructing hierarchies.  When finished, click "Append" to add the new
+value as metadata for the chosen file.  The value immediately appears in the
+Metadata table.
+<p></p>
+You can also add metadata to a folder, or to several multiply selected files at
+once.  It is added to all files within the folder or selection, and to child
+folders.  Keep in mind that if you assign metadata to a folder, any new files in
+it automatically inherit the folder's values.
+<p></p>
+If you choose metadata that occurs in some of the selected files and click
+"Append", it is added to the other files in the selection too.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+You can add structure to metadata values by using paths as described in <a href="theenrichview.htm#theenrichview">The Enrich View</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/assignlanguages.htm
===================================================================
--- /main/tags/3.01/gli/help/en/assignlanguages.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/assignlanguages.htm	(revision 21112)
@@ -0,0 +1,82 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignlanguages"><font size="5" face="Verdana"><strong>6.6.3: Assign Languages</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+This section details how to restrict search indexes to particular languages.
+You do this by generating a partition using the "Assign Languages" tab of the
+"Partition Indexes" view.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Language controls
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Add language
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Remove language
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Set default language
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Clear default language
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To add a new language to partition by, use the "Assign Languages" tab to build
+an index for it.  Select the desired language from the "Language to add"
+pull-down list and click "Add Language".
+<p></p>
+To remove a language, select it from the "Language Selection" list and click
+"Remove Language".
+<p></p>
+To set the default language, select it from the list and click "Set Default".
+<p></p>
+To clear the default language, click "Clear Default".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/en/assignpartitions.htm
===================================================================
--- /main/tags/3.01/gli/help/en/assignpartitions.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/assignpartitions.htm	(revision 21112)
@@ -0,0 +1,74 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignpartitions"><font size="5" face="Verdana"><strong>6.6.2: Assign Partitions</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+This section explains how to assign a previous defined partition filter.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Add a partition
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Remove a partition
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Set default partition
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Clear default partition
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Having defined a subcollection filter, use the "Assign Partitions" tab to build
+indexes for it (or for a group of filters).  Select the desired filter (or
+filters) from the "Defined Subcollection Filters" list, enter a name for your partition in the "Partition Name" field,  and click "Add Partition".
+<p></p>
+To remove a partition, select it from the list and click "Remove Partition".
+<p></p>
+To make a partition the default one, select it from the list and click "Set
+Default".
+<p></p>
+To clear the default partition, click "Clear Default".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/en/buildsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/en/buildsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/buildsettings.htm	(revision 21112)
@@ -0,0 +1,66 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="buildsettings"><font size="5" face="Verdana"><strong>7.2: Import and Build Settings</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to access the various import and build settings.
+For more information of importing and building read Chapter 1 of the Greenstone
+Developer's Guide -- Understanding the collection-building process.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            General settings
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Import settings
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Build settings
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Controlling the various settings is done in a similar way to the
+"Configuring Arguments" window described in the <a href="plugins.htm#plugins">Document Plugins</a> section.  Some
+fields require numeric arguments, and you can either type these in or use the
+up and down arrows to increase or decrease the current value (in some cases,
+the interface restricts the range you can enter).  Others are enabled by
+clicking a checkbox (click again to disable).
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/classifiers.htm
===================================================================
--- /main/tags/3.01/gli/help/en/classifiers.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/classifiers.htm	(revision 21112)
@@ -0,0 +1,91 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="classifiers"><font size="5" face="Verdana"><strong>6.8: Classifiers</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to assign "classifiers", which are used for browsing,
+to the collection.  Under "Design Sections", click "Browsing Classifiers".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Classifier selection and configuration
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add a classifier
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove a classifier
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configure a classifier
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Alter classifier ordering
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To add a classifier, select it using the "Select classifier to add" pull-down list near the
+bottom and then click "Add Specified Classifier".  A window appears entitled
+"Configuring Arguments"; instructions for this dialog are just the same as for
+plugins (see <a href="plugins.htm#plugins">Document Plugins</a>).  Once you have configured the new
+classifier, it is added to the end of the "Currently Assigned Classifiers"
+list.
+<p></p>
+To remove a classifier, select it from the list and click "Remove Selected
+Classifier".
+<p></p>
+To change the arguments a classifier, select it from the list and click
+"Configure Selected Classifier" (or double-click on the classifier in the
+list).
+<p></p>
+The ordering of classifiers in the collection's navigation bar is reflected in
+their order here.  To change it, select the classifier you want to move and
+click "Move Up" or "Move Down".
+<p></p>
+For further information on classifiers read Chapter 2, Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/collectingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/en/collectingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/collectingfiles.htm	(revision 21112)
@@ -0,0 +1,24 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="collectingfiles"><font size="5" face="Verdana"><strong>4: Collecting Files for Your Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+Once you have a new collection you need to get some files into it.  These may
+come from your ordinary file space, or from other Greenstone collections.  Some
+may already have attached metadata.  This section describes how to import files.
+    
+    
+
+
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/creatingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/creatingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/creatingacollection.htm	(revision 21112)
@@ -0,0 +1,112 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingacollection"><font size="5" face="Verdana"><strong>2.1: Creating a New Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to create a new collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The new collection prompt
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The collection title
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Default metadata sets
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Collection description
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Creating the new collection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Cancelling the new collection
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Quick keys
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To create a new collection, open the "File" menu and choose "New".  Several
+fields need to be filled out -- but you can change their values later if you
+need to, in the design view.
+<p></p>
+"Collection title" is the text displayed at the top of your collection's home page.  It can
+be any length.
+<p></p>
+"Description of content"
+should describe, in as much detail as possible, what the collection is about.
+Use the [Enter] key to break it into paragraphs.
+<p></p>
+Finally you must specify whether the new collection will have the same
+appearance and metadata sets as an existing collection, or whether to start a
+default "New Collection".
+<p></p>
+Click "OK" to create the collection.  If you chose "New Collection" you are
+prompted for the metadata sets to use in it.  You can choose more than one, and
+you can add others later.
+<p></p>
+Clicking "Cancel" returns you to the main screen immediately.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+Buttons, like menus, have one character underlined.  To "click" the button,
+press [ALT] and the underlined character at the same time.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/creatingfolders.htm
===================================================================
--- /main/tags/3.01/gli/help/en/creatingfolders.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/creatingfolders.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingfolders"><font size="5" face="Verdana"><strong>4.2: Creating Folders</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section shows how to create new folders.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The new folder button
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The right-button menu
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Use folders in the Collection file tree to group files together and make them
+easier to find.  Folders can be placed inside folders.  There is virtually no
+limit to how many folders you can have or how deeply they can be nested.
+<p></p>
+To create a new folder, optionally select an existing folder in the Collection
+Tree and click the New Folder button.  The new folder appears within the
+selected one, or at the top level if none is selected.  You are prompted for the
+folder's name (default "New Folder").
+<p></p>
+Folders can also be created by right-clicking over a folder, choosing "New
+Folder" and proceeding as above.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/definefilters.htm
===================================================================
--- /main/tags/3.01/gli/help/en/definefilters.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/definefilters.htm	(revision 21112)
@@ -0,0 +1,75 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="definefilters"><font size="5" face="Verdana"><strong>6.6.1: Define Filters</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+The section explains how to define a partition filter.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Add a filter
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Remove a filter
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Update a filter
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Filters allows you to group together into a subcollection all documents in an
+index for which a metadata value matches a given pattern.
+<p></p>
+To create a filter, click the "Define Filters" tab and enter a name for the
+new filter into the "Name the subcollection filter" field.  Next choose a
+document attribute to match against, either a metadata element or the name of
+the file in question.  Enter a regular expression to use during the matching.
+You can toggle between "Including" documents that match the filter, or
+"Excluding" them.  Finally, you can specify any of the standard PERL regular
+expression flags to use when matching (e.g.  "i" for case-insensitive matching).
+Finally, click "Add Filter" to add the filter to the "Defined Subcollection Filters"
+list.
+<p></p>
+To remove a filter, select it from the list and click "Remove Filter".
+<p></p>
+To alter a filter, select it from the list, change any of the values that
+appear in the editing controls and click "Replace Filter" to commit the changes.
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/en/deletingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/en/deletingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/deletingcollections.htm	(revision 21112)
@@ -0,0 +1,47 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="deletingcollections"><font size="5" face="Verdana"><strong>2.4: Deleting Collections</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to permanently delete collections from your Greenstone installation.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            How to delete collections
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To permanently delete collections from your Greenstone installation, choose "Delete..." from the "File" menu. A list of your Greenstone collections appears. Select one to see its description, then tick the box at the bottom of the dialog and click "Delete" to delete the collection. This action is irreversible, so check carefully that you no longer need the collection before proceeding!
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/designingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/designingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/designingacollection.htm	(revision 21112)
@@ -0,0 +1,30 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="designingacollection"><font size="5" face="Verdana"><strong>6: Designing Your Collection's Appearance</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+Once your files are marked up with metadata, you next decide how it should
+appear to users as a Greenstone collection.  What kind of information is
+searchable?  What ways are provided to browse through the documents?  What
+languages are supported?  Where do the buttons appear on the page?  These
+things can be customized; this section describes how to do it.
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/downloadingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/en/downloadingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/downloadingfiles.htm	(revision 21112)
@@ -0,0 +1,16 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="downloadingfiles"><font size="5" face="Verdana"><strong>3: Downloading Files From the Internet</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+The "Download" view helps you download resources from the internet. This section explains the Librarian Interface's mirroring process.
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/enrichingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/enrichingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/enrichingacollection.htm	(revision 21112)
@@ -0,0 +1,27 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="enrichingacollection"><font size="5" face="Verdana"><strong>5: Enriching the Collection with Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+Having gathered several files into the collection, now enrich them with
+additional information called "metadata".  This section explains how metadata is
+created, edited, assigned and retrieved, and how to use external metadata
+sources (also see Chapter 2 of the Greenstone Developer's Guide -- Getting the most
+out of your documents).
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/exportingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/en/exportingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/exportingcollections.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="exportingcollections"><font size="5" face="Verdana"><strong>8.3: Exporting Collections to CD-ROM</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      
+This section describes how to export collections to a self-installing 
+Greenstone CD-ROM. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            How to export
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Greenstone can export one or more collections to a self-installing 
+CD-ROM for Windows. To do so, Greenstone's "Export to CD-ROM" package 
+must be installed. This is not included by default, so you may need to 
+modify your installation to include it.
+<p></p>
+To export a collection, open the "File" menu and choose "Export to 
+CD-ROM". A list of Greenstone collections appears; click on any one to 
+see its description. Tick the check boxes of the collections to export. 
+You can enter the CD-ROM's name in the box: this is what will appear in 
+the Start menu when the CD-ROM has been installed. Then click "Export". 
+The process involves copying many files and may take a few minutes.
+<p></p>
+Upon completion, Greenstone will show the name of a folder containing 
+the exported collections. Use a CD writer to copy its contents to a 
+blank CD-ROM.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/fileassociations.htm
===================================================================
--- /main/tags/3.01/gli/help/en/fileassociations.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/fileassociations.htm	(revision 21112)
@@ -0,0 +1,79 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="fileassociations"><font size="5" face="Verdana"><strong>8.2: File Associations</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+The Librarian Interface uses particular application programs to open particular
+file types.  This section explains how to assign and edit these file
+associations.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add association
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Edit association
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove association
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To alter file associations open the "File" menu and click "File
+Associations...".
+<p></p>
+To add an association, select the target file extension from the pull-down
+list, or type in a new extension (do not include the ".").  Next either type
+command that launches the desired application in the appropriate field, or
+choose the application from the "Browse" dialog.  "%1" can be used in the launch
+command to insert the name of the file being opened.  Once these are filled out,
+"Add Association" is enabled and can be clicked to add the association.
+<p></p>
+To edit an association, select an existing file extension.  Any existing
+associated command is shown in the launch command field.  Edit it, and then
+click "Replace Association".
+<p></p>
+To remove an association, select an existing file extension and click "Remove Association".
+(The file extension remains in the "For Files Ending" pull-down list.)
+<p></p>
+File associations are stored in the Librarian Interface's main folder, in a file
+called "associations.xml".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/filteringthetree.htm
===================================================================
--- /main/tags/3.01/gli/help/en/filteringthetree.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/filteringthetree.htm	(revision 21112)
@@ -0,0 +1,68 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="filteringthetree"><font size="5" face="Verdana"><strong>4.5: Filtering the Tree</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+"Filtering" the collection tree allows you to narrow down the search for particular files.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The filter control
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Custom filtering
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The "Show Files" pull-down menu underneath each tree shows a list of predefined
+filters, such as "Images".  Choosing this temporarily hides all other files in
+the tree.  To restore the tree, change the filter back to "All Files".  These
+operations do not alter the collection, nor do they affect the folders in the
+tree.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+You can specify a custom filter by typing in a pattern to match files against.
+Use standard file system abbreviations such as "*.*" or "*.doc" ("*" matches
+any characters).
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/formatstatements.htm
===================================================================
--- /main/tags/3.01/gli/help/en/formatstatements.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/formatstatements.htm	(revision 21112)
@@ -0,0 +1,111 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="formatstatements"><font size="5" face="Verdana"><strong>6.9: Format Features</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Format commands control the structure and appearance of the collection.  They
+affect such things as where buttons appear when a document is shown, and what
+links are displayed by the DateList classifier.  Format commands are not easy
+to develop, and you should read Chapter 2 of the Greenstone Developer's Guide.
+This section discusses the format settings, and how the Librarian Interface
+gives access to them.  Under "Design Sections", click "Format Features".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Formatting
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add format command
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove format command
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Update format command
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Extended formatting options
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+You can apply a format command to anything in the "Choose Feature" pull-down
+list, which includes each classifier and a predefined list of features.  When
+you select a feature, there are two types of control.  Some features are simply
+enabled or disabled, and this is controlled by a checkbox.  Others require a
+format string to be specified.  For these there is a pull-down list ("Affected 
+Component") for selecting which part of the feature the string applies to 
+(if necessary), a text area ("HTML Format String") for entering the string, 
+and a selection of predefined "Variables".  To insert a variable into the 
+current position in the format string, select it from the pull-down list and 
+click "Insert".
+<p></p>
+You can specify a default format for a particular component by selecting the
+blank feature.  This format is then applied to all applicable features unless
+otherwise specified.
+<p></p>
+To add a new format command, fill out the information as explained above and
+click "Add Format".  The new format command appears in the list of "Currently
+Assigned Format Commands".  Only one format command can be assigned to each
+feature/component combination.
+<p></p>
+To remove a format command, select it from the list and click "Remove Format".
+<p></p>
+To change a format command, select it from the list, modify the settings, and
+click "Replace Format".
+<p></p>
+For more information about variables and the feature components, read Chapter 2
+of the Greenstone Developer's Guide.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+If the "Allow Extended Options" checkbox is ticked, some advanced formatting options are enabled. The list of features that can be formatted is changed slightly, and more variables are available to be used in the format command, providing greater control over the page layout.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/generalsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/en/generalsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/generalsettings.htm	(revision 21112)
@@ -0,0 +1,59 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="generalsettings"><font size="5" face="Verdana"><strong>6.2: General Settings</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to review and alter the general settings associated
+with your collection.  First, under "Design Sections", click "General".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            General settings
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Here the values provided during collection creation can be modified.
+<p></p>
+At the top of the page is an instruction box, which appears for each of the
+different sections.  It contains a brief list of instructions to remind you
+what functionality is available.
+<p></p>
+First are the contact emails of the collection's creator and maintainer.  
+The following field allows you to change the collection title. The folder that the collection is stored in is shown next, but this cannot be edited.
+The next one specifies (in the form of a URL) the
+icon to show at the top left of the collection's "About" page, and the next is
+the icon used in the Greenstone library page to link to the collection. Then, a checkbox controls whether the collection should be publicly accessible.
+Finally comes the "Collection Description" text area as described in <a href="creatingacollection.htm#creatingacollection">Creating A New Collection</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/help.xml
===================================================================
--- /main/tags/3.01/gli/help/en/help.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/en/help.xml	(revision 21112)
@@ -0,0 +1,1457 @@
+<Document>
+  <Section name="introduction">
+    <Title>Introduction</Title>
+The Greenstone Librarian Interface is a tool for collecting and marking up documents, then building digital library collections.  It provides access to the Greenstone Digital Library Software's functionality from a graphical point and click interface.
+    <Section name="ofmiceandmenus">
+      <Title>Of Mice and Menus</Title>
+This section provides basic information about interacting with the Librarian Interface.
+If you are familiar with programs such as Internet Explorer or Microsoft
+Office and are comfortable with mouse clicks and menus, skip to the <Reference target="howtoavoidthisdocument">next section</Reference>.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Basic concepts   
+            <ContentsItem>
+              Mouse actions
+	    </ContentsItem>
+            <ContentsItem>
+              Keyboard
+	    </ContentsItem>
+          </ContentsItem>
+          <ContentsItem>
+	    Exiting the program
+          </ContentsItem>
+	</ContentsGroup>
+	<ContentsGroup linkto="advanced">
+          <ContentsItem>
+	    Quick keys
+          </ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The Librarian Interface follows Microsoft Windows conventions and draws upon
+ordinary knowledge of Windows.
+<Break/>
+Any part of the screen that you interact with, such as a button or text field,
+is called a "control".  At any given time one control, called the "focus", is
+highlighted and responds to the keyboard.  Several controls allow you to select
+parts that are highlighted in dark blue.  Some controls are greyed out to
+indicate that they are disabled.
+<Break/>
+You can move and left- or right-click the mouse in the usual way.  Many
+components also allow you to "drag" them, by clicking and holding the left
+mouse button, move them with the mouse, and "drop" them elsewhere by releasing
+the button.  Potential drop targets alter their appearance when a component
+hovers over them.
+<Break/>
+You can use the keyboard to type into text fields.  Keyboard alternatives are
+available for many controls, indicated by a key name in square brackets -- for
+example, [Tab] alters the focus.  The plus sign shows if other keys must be
+pressed at the same time.
+<Break/>
+Exit the Librarian Interface program by choosing "Exit" from the "File"
+menu.  Your collection will be saved first.
+       <Anchor name="advanced">Advanced Instructions:</Anchor>
+To access a menu, hold down [ALT] and press the corresponding letter
+(underlined).  For example, for the "File" menu press [ALT] + [F].  To choose an
+item, press the corresponding key.  For example, while in the File menu
+press [S] to "Save" a collection.
+    </Section>
+    <Section name="howtoavoidthisdocument">
+      <Title>How to Avoid Reading This Document</Title>
+Don't read this help text all the way through!  Just read enough to learn
+how to get help when you need it.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Context sensitive help
+          </ContentsItem>
+          <ContentsItem>
+            Tool tips
+          </ContentsItem>
+          <ContentsItem>
+            Other documents
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The "Help" menu item marks what may be the most appropriate help item with a
+little book icon.
+<Break/>
+For many controls, if you station the mouse over them a
+"tool tip" appears that says what they do.
+<Break/>
+Before using the Librarian Interface, first read the Greenstone
+documentation.
+    </Section>
+  </Section>
+
+  <Section name="startingoff">
+    <Title>Starting Off</Title>
+    This section covers how to create, load, save and delete collections.
+    <Section name="creatingacollection">
+      <Title>Creating a New Collection</Title>
+<ObviousSentence>This section describes how to create a new collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The new collection prompt
+          </ContentsItem>
+          <ContentsItem>
+            The collection title
+          </ContentsItem>
+          <ContentsItem>
+            Default metadata sets
+          </ContentsItem>
+          <ContentsItem>
+            Collection description
+          </ContentsItem>
+          <ContentsItem>
+            Creating the new collection
+          </ContentsItem>
+          <ContentsItem>
+            Cancelling the new collection
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Quick keys
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To create a new collection, open the "File" menu and choose "New".  Several
+fields need to be filled out -- but you can change their values later if you
+need to, in the design view.
+<Break/>
+"Collection title" is the text displayed at the top of your collection's home page.  It can
+be any length.
+<Break/>
+"Description of content"
+should describe, in as much detail as possible, what the collection is about.
+Use the [Enter] key to break it into paragraphs.
+<Break/>
+Finally you must specify whether the new collection will have the same
+appearance and metadata sets as an existing collection, or whether to start a
+default "New Collection".
+<Break/>
+Click "OK" to create the collection.  If you chose "New Collection" you are
+prompted for the metadata sets to use in it.  You can choose more than one, and
+you can add others later.
+<Break/>
+Clicking "Cancel" returns you to the main screen immediately.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+Buttons, like menus, have one character underlined.  To "click" the button,
+press [ALT] and the underlined character at the same time.
+    </Section>
+    <Section name="savingacollection">
+      <Title>Saving the Collection</Title>
+<ObviousSentence>This section describes how to save a collection, and the Save Collection Prompt.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            How to save
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Where the files are saved
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Save your work regularly by opening the "File" menu and choosing "Save".
+Saving a collection is not the same as making it ready for use in Greenstone
+(see <Reference target="producingthecollection">Producing Your Collection</Reference>).
+<Break/>
+The Librarian Interface protects your work by saving it whenever you exit the
+program or load another collection.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+Saved collections are written to a file named for the collection and with file
+extension ".col", located in a folder of the same name within your Greenstone
+installation's "collect" folder.
+    </Section>
+    <Section name="openingacollection">
+      <Title>Opening an Existing Collection</Title>
+<ObviousSentence>This section tells you how to open existing collections using the Open
+Collection prompt.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            How to open a collection
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Locked collections
+          </ContentsItem>
+	  <ContentsItem>
+	    Legacy collections
+	  </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To open an existing collection, choose "Open" from the "File" menu to get the
+Open Collection prompt.  A list of your Greenstone collections appears.
+Select one to see its description, and click "Open" to load it.  If you seek a
+collection that resides outside Greenstone's "collect" folder, click "Browse"
+for a file system browsing dialog.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+In case more than one Greenstone Librarian Interface program is running
+concurrently, the relevant directories are "locked" to prevent interference.
+On opening a collection, a small temporary lock file is created in its
+folder.  Before opening a collection, the Librarian Interface checks to ensure
+that no lock file already exists.  You can tell whether a collection is locked
+by the colour of its icon: green for a normal collection, red for a locked
+one.  However, when the Librarian Interface is exited prematurely the lock file
+is sometimes left in place.  When you open such a collection, the Librarian asks
+if you want to "steal" control of it.  Never steal a collection that someone
+else is currently working on.
+<Break/>
+When you open a collection that the Greenstone Librarian Interface did 
+not create, you will be asked to select a metadata set (or sets).  If 
+none are selected, any existing metadata will be ignored. Otherwise, 
+metadata will be imported just as it is when you drag in files with 
+existing metadata. The process is described in the <Reference target="importingpreviouslyassignedmetadata">Importing Previously Assigned Metadata</Reference> section. 
+    </Section>
+    <Section name="deletingcollections">
+      <Title>Deleting Collections</Title>
+<ObviousSentence>This section explains how to permanently delete collections from your Greenstone installation.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            How to delete collections
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To permanently delete collections from your Greenstone installation, choose "Delete..." from the "File" menu. A list of your Greenstone collections appears. Select one to see its description, then tick the box at the bottom of the dialog and click "Delete" to delete the collection. This action is irreversible, so check carefully that you no longer need the collection before proceeding!
+    </Section>
+  </Section>
+
+ <Section name="downloadingfiles">
+    <Title>Downloading Files From the Internet</Title>
+The "Download" view helps you download resources from the internet. This section explains the Librarian Interface's mirroring process.
+    <Section name="themirrorview">
+      <Title>The Download view</Title>
+This section describes how to configure a download task and control the downloading process.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The workspace tree
+          </ContentsItem>
+          <ContentsItem>
+            Download configuration
+          </ContentsItem>
+          <ContentsItem>
+            The download list
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Enabling web proxies
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Access the "Download" view by clicking its tab. The top half of the screen shows the downloading controls. The bottom half is initially empty, but will show a list of pending and completed downloading jobs. 
+<Break/>
+Files are downloaded into a folder in the workspace called "Downloaded Files" (only present when mirroring is enabled), and can be used in all collections built with the Librarian Interface.  Files in this area are named by their full web URL. A new folder is created for each host, followed by others for each part of the path. This ensures that each file is distinct.
+<Break/>
+Use the first of the download configuration controls, "Source URL", to enter the URL of a target resource. Use the "Download Depth" control to limit how many  hyperlinks to follow when downloading: Set this to 0 to download a single web page; set it to 1 to download a page and all the pages it points to. The depth limit is ignored when downloading media other than html pages. Next, there are several checkbox controls which can be set to turn on the specified feature for a specific download. Once the configuration is set up, click "Download" to start the new download job. There are two other button controls: "Preferences", which links to the connection section of the Preferences where proxy settings can be edited; and "Clear Cache", which deletes all previously downloaded files.
+<Break/>
+The download list has an entry for each web page download.  Each entry has a text region that gives details of the task along with a progress bar showing current activity. Three buttons appear to the left of each entry. "Pause" is used for pausing a currently downloading task. "View Log" opens a window showing the download log file. "Close" terminates the download and removes the task from the list.  
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+The <Reference target="preferences">Preferences section</Reference> describes how to establish an Internet connection via a proxy.  If authentication is needed, the proxy server prompts for identification and password.  The Librarian Interface does not store passwords between sessions.
+    </Section>
+  </Section>
+
+  <Section name="collectingfiles">
+    <Title>Collecting Files for Your Collection</Title>
+Once you have a new collection you need to get some files into it.  These may
+come from your ordinary file space, or from other Greenstone collections.  Some
+may already have attached metadata.  This section describes how to import files.
+    <Section name="thegatherview">
+      <Title>The Gather View</Title>
+This section introduces the Gather area that you use to select what files
+to include in the collection you are building.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            How to view the gather screen
+          </ContentsItem>
+          <ContentsItem>
+            The file trees
+          </ContentsItem>
+          <ContentsItem>
+            The status area
+          </ContentsItem>
+          <ContentsItem>
+            Control buttons
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Selection methods
+          </ContentsItem>
+          <ContentsItem>
+            Special folder mapping
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The Librarian Interface starts with the Gather view.  To return to this view
+later, click the "Gather" tab directly below the menu bar.
+<Break/>
+The two large areas titled "Workspace" and "Collection" are used to move files
+into your collection.  They contain "file trees", graphical structures that
+represent files and folders.
+<Break/>
+Select an item in the tree by clicking it.  (There are other ways; see below.)
+Double-click a folder, or single-click the switch symbol beside it, to expand (or collapse)
+its contents.  Double-click a file to open it using its associated application
+program (see <Reference target="fileassociations">File Associations</Reference>).
+<Break/>
+The Workspace file tree shows the sources of data available to the Librarian Interface -- the local file system (including disk and CD-ROM drives), the contents of existing Greenstone collections, and the cache of downloaded files if Web mirroring is enabled. You can copy and view these files but you cannot move, delete, or edit them, with the exception of the downloaded files, which can be deleted.  Navigate this space to find the files you want to include in the collection.
+<Break/>
+The Collection file tree represents the contents of the collection so
+far.  Initially, it is empty.
+<Break/>
+You can resize the spaces by mousing over the grey bar that separates the trees
+(the shape of the pointer changes) and dragging.
+<Break/>
+At the bottom of the window is a status area that shows the progress of actions involving files (copying, moving and deleting). These can take some time to complete. The "Stop" button stops any action that is currently in progress. 
+<Break/>
+Two large buttons occupy the lower right corner of the screen. "New Folder", with a picture
+of a folder, creates new folders (see <Reference target="creatingfolders">Creating folders</Reference>). 
+"Delete", with a garbage can, removes files. Clicking the Delete button will remove any selected files from the Collection file tree. Alternatively, files can be deleted by dragging them onto the Delete button. 
+       <Anchor name="advanced">Advanced Instructions:</Anchor>
+To select several sequential items, select the first and then hold down [Shift]
+and click on the last -- the selection will encompass all intervening
+items.  Select non-sequential files by holding down [Ctrl] while clicking.  Use
+these two methods together to select groups of non-adjacent items.
+<Break/>
+Certain folders -- such as the one containing your own web pages -- sometimes
+have special significance.  The Librarian Interface can map such folders to 
+the first level of the file tree.  To do this, right-click the desired
+folder.  Select "Create Shortcut", and enter a name for the folder.  To remove an item,
+right-click the mapped folder and select "Remove Shortcut".
+    </Section>
+    <Section name="creatingfolders">
+      <Title>Creating Folders</Title>
+<ObviousSentence>This section shows how to create new folders.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The new folder button
+          </ContentsItem>
+          <ContentsItem>
+            The right-button menu
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Use folders in the Collection file tree to group files together and make them
+easier to find.  Folders can be placed inside folders.  There is virtually no
+limit to how many folders you can have or how deeply they can be nested.
+<Break/>
+To create a new folder, optionally select an existing folder in the Collection
+Tree and click the New Folder button.  The new folder appears within the
+selected one, or at the top level if none is selected.  You are prompted for the
+folder's name (default "New Folder").
+<Break/>
+Folders can also be created by right-clicking over a folder, choosing "New
+Folder" and proceeding as above.
+    </Section>
+
+
+    <Section name="addingfiles">
+      <Title>Adding Files</Title>
+<ObviousSentence>This section shows how to get files into your collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Dragging a file
+          </ContentsItem>
+          <ContentsItem>
+            Multiple files
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Existing metadata
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Files can be copied into the collection by dragging and dropping.  The mouse
+pointer becomes a ghost of the selected item (or, if more than one is selected,
+the number of them).  Drop the selection into the Collection Tree to copy the
+files there (if the source was the Workspace Tree) or move them around within
+the collection (if the source was the Collection Tree).
+<Break/>
+When copying multiple files, they are all placed in the target folder at the
+same level, irrespective of the folder structure they occupied originally.
+When you copy a second file with the same name into the same folder, you are
+asked whether to overwrite the first one.  Respond "No" and the file will not be
+copied, but the others will be.  To cancel all remaining copy actions, click the
+"stop" button.
+<Break/>
+Only the "highest" items in a selection are moved.  A folder is higher than its
+children.  You cannot select files within a folder and also the folder itself.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+When you add a file, the Librarian Interface searches through the source
+folders for auxiliary files containing metadata previously assigned to the
+added file and, if it finds one, begins to import this metadata.  As the
+operation proceeds, you may be prompted (perhaps several times) for extra
+information to match the imported metadata to the metadata sets in your
+collection.  This process involves many different prompts, described in the <Reference target="importingpreviouslyassignedmetadata">Importing Previously Assigned Metadata</Reference> section.  For a more detailed
+explanation of associating metadata with files read Chapter 2 of the Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </Section>
+    <Section name="removingfiles">
+      <Title>Removing Files</Title>
+<ObviousSentence>This section describes how to remove files and folders from your collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The delete button
+          </ContentsItem>
+          <ContentsItem>
+            The delete key
+          </ContentsItem>
+          <ContentsItem>
+            Delete by drag and drop
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+There are several methods for removing files and folders.  You must first
+indicate what items to remove by selecting one or more files and folders as
+described in <Reference target="thegatherview">The Gather View</Reference>.
+<Break/>
+Once files have been selected, click the "delete" button to remove them, or
+press the [Delete] key on your keyboard, or drag them from the collection to
+the delete button and drop them there.
+    </Section>
+    <Section name="filteringthetree">
+      <Title>Filtering the Tree</Title>
+"Filtering" the collection tree allows you to narrow down the search for particular files.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The filter control
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Custom filtering
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The "Show Files" pull-down menu underneath each tree shows a list of predefined
+filters, such as "Images".  Choosing this temporarily hides all other files in
+the tree.  To restore the tree, change the filter back to "All Files".  These
+operations do not alter the collection, nor do they affect the folders in the
+tree.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+You can specify a custom filter by typing in a pattern to match files against.
+Use standard file system abbreviations such as "*.*" or "*.doc" ("*" matches
+any characters).
+    </Section>
+  </Section>
+
+  <Section name="enrichingacollection">
+    <Title>Enriching the Collection with Metadata</Title>
+Having gathered several files into the collection, now enrich them with
+additional information called "metadata".  This section explains how metadata is
+created, edited, assigned and retrieved, and how to use external metadata
+sources (also see Chapter 2 of the Greenstone Developer's Guide -- Getting the most
+out of your documents).
+    <Section name="theenrichview">
+      <Title>The Enrich View</Title>
+<ObviousSentence>This section describes how to input and edit metadata using the Enrich view.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The collection tree
+          </ContentsItem>
+          <ContentsItem>
+            The metadata editing controls
+          </ContentsItem>
+          <ContentsItem>
+            The value tree
+          </ContentsItem>
+          <ContentsItem>
+            The metadata table
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Use the Enrich view to assign metadata to the documents in the collection.
+Metadata is data about data -- typically title, author, creation date, and so
+on.  Each metadata item has two parts: "element" tells what kind of item it is
+(such as author), and "value" gives the value of that metadata element (such as
+the author's name).
+<Break/>
+On the left of the "Enrich" view is the Collection Tree. To the right is the Metadata Table, which shows metadata for any selected files or folders in the Collection Tree.  Columns are named in
+grey at the top, and can be resized by dragging the separating line. Clicking any row transfers its details to the metadata editing controls below. If several files are selected, black text indicates that the value is common to all of the
+selected files, while grey text indicates that it is not.  Black values may be
+updated or removed, while grey ones can be removed from those that have it, or
+appended to the others.
+<Break/>
+A folder icon may appear beside some metadata entries.  This indicates that the
+values are inherited from a parent (or ancestor) folder.  Inherited metadata
+cannot be edited or removed, only appended to or overwritten.  Click on the
+folder icon to go immediately to the folder where the metadata is assigned.
+<Break/>
+The metadata editing controls at the lower right appear only when a file is selected from the tree and a row is selected from the table. Use them to update, append,
+and remove the metadata value.  The value field is for entering or editing the
+metadata value.  Beside it is a button labelled "..." which, when clicked,
+opens a larger editing box.  In the buttons below, "Append" assigns the value as
+new metadata and adds it to any existing values for the selected element,
+"Replace" overwrites the selected existing value with the new one, and "Remove" clears
+the selected value.  Underneath, labelled "All Previous Values", is the "Value
+Tree".
+<Break/>
+The Value Tree expands and collapses.  Usually it is a list that shows all
+values entered previously for the selected element.  Clicking an entry
+automatically places it into the value field.  Conversely, typing in the text
+field selects the Value Tree entry that starts with the characters you have
+typed.  Pressing [Tab] auto-completes the typing with the selected value.
+<Break/>
+Metadata values can be organised into a hierarchy. This is shown in the Value Tree using folders for internal levels. Hierarchical values can be entered using the character "|" to separate the levels.  For
+example, "Cards|Red|Diamonds|Seven" might be used in a hierarchy that represents a pack of playing cards. This enables values to be grouped together. Groups can also be assigned as metadata to files.
+<Break/>
+Greenstone extracts metadata automatically from documents into a metadata set
+whose elements are prefixed by "ex.".  This has no value tree and cannot be
+edited, so the edit controls are hidden if such an entry is selected.  The
+"..." button still serves to expand the value, but the text cannot be edited.
+    </Section>
+    <Section name="selectingmetadatasets">
+      <Title>Selecting Metadata Sets</Title>
+<ObviousSentence>This section explains how to specify metadata elements and how to add metadata sets to your collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            What is a metadata set?
+          </ContentsItem>
+          <ContentsItem>
+            Controlling the metadata sets used
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Sets of predefined metadata elements are known as "metadata sets".  An example
+is the Dublin Core metadata set.  When you add a metadata set to your
+collection, its elements become available for selection.  You can have more than
+one set; to prevent name clashes a short identifier that identifies the
+metadata set is pre-pended to the element name.  For instance the Dublin Core
+element Creator becomes "dc.Creator".  Metadata sets are stored in the
+Librarian Interface's metadata folder and have the suffix ".mds".
+<Break/>
+To control the metadata sets used in a collection, use the "Metadata Sets" entry on the Design view.
+    </Section>
+    <Section name="appendingmetadata">
+      <Title>Appending New Metadata</Title>
+<ObviousSentence>This section explains how to add metadata and values to files, and how to add new values for metadata elements.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Adding to files
+          </ContentsItem>
+          <ContentsItem>
+            Adding to folders or multiple files
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Value hierarchies and editing values
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+We now add a metadata item -- both element and value -- to a file.  First select
+the file from the Collection file tree on the left.  The action causes any
+metadata previously assigned to this file to appear in the table at the right.
+<Break/>
+Next select the metadata element you want to add by clicking its row in the
+table.
+<Break/>
+Type the value into the value field.  Do not use the character "|", as it is
+used for constructing hierarchies.  When finished, click "Append" to add the new
+value as metadata for the chosen file.  The value immediately appears in the
+Metadata table.
+<Break/>
+You can also add metadata to a folder, or to several multiply selected files at
+once.  It is added to all files within the folder or selection, and to child
+folders.  Keep in mind that if you assign metadata to a folder, any new files in
+it automatically inherit the folder's values.
+<Break/>
+If you choose metadata that occurs in some of the selected files and click
+"Append", it is added to the other files in the selection too.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+You can add structure to metadata values by using paths as described in <Reference target="theenrichview">The Enrich View</Reference>.
+    </Section>
+    <Section name="addingpreviouslydefinedmetadata">
+      <Title>Adding Previously Defined Metadata</Title>
+<ObviousSentence>This section explains how to add metadata that uses values already present in the value tree.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Adding to files
+          </ContentsItem>
+          <ContentsItem>
+            Adding to folders or multiple files
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To add metadata that has an existing value, first select the file, then select
+the required value from the value tree, expanding hierarchy folders as
+necessary.  The value of the selected entry automatically appears in the Value
+text field (alternatively, use the value tree's auto-select and auto-complete
+features).  Click "Append" to add the metadata to the selected file.
+<Break/>
+The process of adding metadata with already-existing values to folders
+or multiple files is just the same.
+    </Section>
+    <Section name="updatingmetadata">
+      <Title>Updating Metadata</Title>
+<ObviousSentence>This section explains how to update the metadata assigned to a file.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Updating files
+          </ContentsItem>
+          <ContentsItem>
+            Updating folders or multiple files
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To update the value of a piece of metadata, first choose the file to which that
+value applies, and then the metadata element whose value you want to change.
+Your selection appears in the metadata edit controls.  Edit the value field and
+click "Replace" to alter the metadata.
+<Break/>
+The process is the same when updating a folder with child folders or multiple files, but you can only update metadata that is common to all files/folders selected.
+<Break/>
+The value tree shows all previous values, not just those currently assigned.
+Thus the value you have replaced will remain in the value tree.
+    </Section>
+    <Section name="removingmetadata">
+      <Title>Removing Metadata</Title>
+<ObviousSentence>This section explains how to remove metadata from a file.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Removing from files
+          </ContentsItem>
+          <ContentsItem>
+            Removing from folders or multiple files
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+You remove metadata the same way as you update it.  First select a file from the
+file tree, then use the metadata table to select the metadata. If the metadata has a value assigned, the "Remove" button becomes active.
+Click it to remove the metadata from the specified file.  Other files
+remain unchanged, and the value remains in the Value Tree.
+<Break/>
+Removing metadata from folders or multiple files works the same way.
+    </Section>
+    <Section name="reviewingmetadata">
+      <Title>Reviewing Assigned Metadata</Title>
+<ObviousSentence>This section describes how to view all metadata assigned to an entire collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            The audit table
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Right button menu
+          </ContentsItem>
+          <ContentsItem>
+            Autofilter
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Sometimes you need to see the metadata assigned to many or all files at once -- for instance,
+to determine how many files are left to work on, or to get some idea of the
+spread of dates.
+<Break/>
+Select the files you wish to examine, then right-click and choose "Assigned Metadata...". A window called
+"All Metadata", dominated by a large table with many columns, appears.  The
+first column shows file names; the rows show all metadata values assigned to
+those files.
+<Break/>
+Drawing the table can take some time if many files are selected.  You can continue to use the Librarian
+Interface while the "All Metadata" window is open.
+<Break/>
+Click "Close" to hide the window.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+You can also view the "All Metadata" table by selecting the files you wish to
+examine, right-clicking, and choosing "Assigned Metadata...".  If a folder has
+been selected, all its child files are included in the table.
+<Break/>
+When it gets too large, you can filter the "All Metadata" table by applying
+filters to the columns.  As new filters are added, only those rows that match
+them remain visible.  To set, modify or clear a filter, click on the "funnel"
+icon at the top of a column.  You are prompted for information about the filter.
+Once a filter is set, the column header changes colour.
+<Break/>
+The prompt has a "Simple" and an "Advanced" tab.  The Simple version filters
+columns so that they only show rows that contain a certain metadata value ("*"
+matches all values).  You can select metadata values from the pull-down list.
+The Advanced version allows different matching operations: must start with,
+does not contain, alphabetically less than and is equal to.  The value to be
+matched can be edited to be any string (including "*"), and you can choose
+whether the matching should be case insensitive.  Finally, you can specify a
+second matching condition that you can use to specify a range of values (by
+selecting AND) or alternative values (by selecting OR).  Below this area is a
+box that allows you to change the sort order (ascending or descending).  Once
+you have finished, click "Set Filter" to apply the new filter to the column.
+Click "Clear Filter" to remove a current filter.  Note that the filter details
+are retained even when the filter is cleared.
+<Break/>
+For example, to sort the "All Metadata" table, choose a column, select the
+default filter setting (a Simple filter on "*"), and choose ascending or
+descending ordering.
+    </Section>
+    <Section name="importingpreviouslyassignedmetadata">
+      <Title>Importing Previously Assigned Metadata</Title>
+This section describes how to import previously assigned metadata: metadata assigned to documents before they were added to the collection.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            While adding files
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+If metadata in a form recognized by the Librarian Interface has been previously
+assigned to a file -- for example, when you choose documents from an existing
+Greenstone collection -- it is imported automatically when you add the file.
+To do this, the metadata must be mapped to the metadata sets available in the
+collection.
+<Break/>
+The Librarian Interface prompts for the necessary information.  The prompt
+gives brief instructions and then shows the name of the metadata element that
+is being imported, just as it appears in the source file.  This field cannot be
+edited or changed.  Next you choose what metadata set the new element should map
+to, and then the appropriate metadata element in that set.  The system
+automatically selects the closest match, in terms of set and element, for the
+new metadata.
+<Break/>
+Having checked the mapping, you can choose "Add" to add the new metadata
+element to the chosen metadata set.  (This is only enabled if there is no
+element of the same name within the chosen set.) "Replace" maps the new element
+to the one chosen by the user.  Finally, "Ignore" does not import any metadata
+with this element name.  Once you have specified how to import a certain piece of metadata, the mapping
+information is retained for the collection's lifetime.
+<Break/>
+For details on the metadata.xml files which Greenstone uses to store the metadata, see Chapter 2 of the Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </Section>
+  </Section>
+
+  <Section name="designingacollection">
+    <Title>Designing Your Collection's Appearance</Title>
+Once your files are marked up with metadata, you next decide how it should
+appear to users as a Greenstone collection.  What kind of information is
+searchable?  What ways are provided to browse through the documents?  What
+languages are supported?  Where do the buttons appear on the page?  These
+things can be customized; this section describes how to do it.
+    <Section name="thedesignview">
+      <Title>The Design View</Title>
+This section introduces you to the design view and explains how to navigate
+between the various views within this pane.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Choosing a design section
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+With the Librarian Interface, you can configure how the collection appears to
+the user.  The configuration options are divided into different sections, each
+associated with a particular stage of navigating or presenting information.
+<Break/>
+On the left is a list of different views, and on the right are the controls
+associated with the current one.  To change to a different view, click its name
+in the list.
+<Break/>
+To understand the stages and terms involved in designing a collection, first
+read Chapters 1 and 2 of the Greenstone Developer's Guide.
+    </Section>
+    <Section name="generalsettings">
+      <Title>General Settings</Title>
+This section explains how to review and alter the general settings associated
+with your collection.  First, under "Design Sections", click "General".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            General settings
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Here the values provided during collection creation can be modified.
+<Break/>
+At the top of the page is an instruction box, which appears for each of the
+different sections.  It contains a brief list of instructions to remind you
+what functionality is available.
+<Break/>
+First are the contact emails of the collection's creator and maintainer.  
+The following field allows you to change the collection title. The folder that the collection is stored in is shown next, but this cannot be edited.
+The next one specifies (in the form of a URL) the
+icon to show at the top left of the collection's "About" page, and the next is
+the icon used in the Greenstone library page to link to the collection. Then, a checkbox controls whether the collection should be publicly accessible.
+Finally comes the "Collection Description" text area as described in <Reference target="creatingacollection">Creating A New Collection</Reference>.
+    </Section>
+    <Section name="plugins">
+      <Title>Document Plugins</Title>
+This section describes how to configure the document plugins the
+collection uses.  It explains how you specify what
+plugins to use, what parameters to pass to them, and in what order
+they occur.  Under "Design Sections", click "Document Plugins".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Add a plugin
+          </ContentsItem>
+          <ContentsItem>
+            Remove a plugin
+          </ContentsItem>
+          <ContentsItem>
+            Configure a plugin
+          </ContentsItem>
+          <ContentsItem>
+            Change plugin order
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Custom configuration
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To add a plugin, select it using the "Select plugin to add" pull-down list near the
+bottom and then click "Add Plugin".  A window appears entitled
+"Configuring Arguments"; it is described later.  Once you have configured the
+new plugin, it is added to the end of the "Currently Assigned Plugins" list.
+Note that, except for UnknownPlug, each plugin may only occur once in the list.
+<Break/>
+To remove a plugin, select it in the list and click "Remove Plugin".
+<Break/>
+Plugins are configured by providing arguments.  To alter them, select the
+plugin from the list and click "Configure Plugin" (or double-click
+the plugin). A "Configuring Arguments" dialog appears with various controls for specifying arguments.
+<Break/>
+There are different kinds of controls.  Some are checkboxes, and clicking one
+adds the appropriate option to the plugin.  Others are text strings, with a
+checkbox and a text field.  Click the box to enable the argument, then type
+appropriate text (regular expression, file path etc) in the box.  Others are
+pull-down menus from which you can select from a given set of values.  To learn what an
+argument does, let the mouse hover over its name for a moment and a description
+will appear.
+<Break/>
+When you have changed the configuration, click "OK" to commit the changes and
+close the dialog, or "Cancel" to close the dialog without changing any plugin
+arguments.
+<Break/>
+The plugins in the list are executed in order, and the ordering is sometimes
+important.  Two plugins, ArcPlug and RecPlug, are vital to the collection
+building process, and are fixed in place at the end of the list (with a
+separator line).  To change the ordering of the other ones, select the plugin you want to move
+and click "Move Up" or "Move Down".
+       <Anchor name="advanced">Advanced Instructions:</Anchor>
+The Librarian Interface does its best to determine what arguments a plugin
+supports. However, there may be cases where the user wants to specify special arguments, and for this a text field called "Custom Arguments" (at the bottom) is provided. Any text in it is appended verbatim to the end of the plugin
+command.
+    </Section>
+    <Section name="searchtypes">
+      <Title>Search Types</Title>
+This section explains how to modify a new design feature in Greenstone, Search Types, which allow fielded searching.  Under "Design Sections", click "Search Types".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            What enabling advanced searches means
+          </ContentsItem>
+          <ContentsItem>
+            Adding a new search type
+          </ContentsItem>
+          <ContentsItem>
+            Removing a search type
+          </ContentsItem>
+          <ContentsItem>
+            Changing the order of search types
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+When you enter the Search Types view, first check "Enable Advanced
+Searches", which activates the other controls.  This migrates the collection to
+Greenstone 2.4 format, which supports fielded searching, and means that (a)
+the index design is different (explained in the <Reference target="searchindexes">Search Indexes</Reference> section),
+(b) there are more text fragments to translate (see <Reference target="translatetext">Translation</Reference>), and (c) the collection will not be usable under older Greenstone
+installations.  If you later uncheck this field, most of your collection will
+be migrated back to Greenstone 2.39.  However the Librarian Interface cannot
+convert the new index specifications into older ones, so you will have to
+re-enter them manually.
+<Break/>
+To add a search type, select it from the "Search Types" list and click "Add Search Type".  Each type can only appear in the list once.
+<Break/>
+To remove a search type, select it from the "Currently Assigned Search Types"
+list and click "Remove Search Type".  The list must contain at least
+one search type.
+<Break/>
+To change to order of a search type, select it from the list and click "Move
+Up" or "Move Down". The first one will be the default.
+    </Section>
+    <Section name="searchindexes">
+      <Title>Search Indexes</Title>
+Indexes specify what parts of the collection are searchable.  This section explains how to add and remove indexes, and set a default index.  Under "Design Sections", click "Search Indexes".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Add an index
+          </ContentsItem>
+          <ContentsItem>
+            Edit an index
+          </ContentsItem>
+          <ContentsItem>
+            Remove an index
+          </ContentsItem>
+          <ContentsItem>
+            Set default index
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To add an index, type a name for it into the "Index Name" field.  Select which
+of the possible information sources to index by clicking the checkboxes beside
+them.  The list shows all the assigned metadata elements, as well the full
+text.  Having selected the data sources, choose the granularity of the index,
+using the "At the level" menu.  Once these details are complete, "Add Index"
+becomes active (unless there is an existing index with the same settings).
+Click it to add the new index.
+<Break/>
+To edit an index, select it and change the index details, then click "Replace Index".
+<Break/>
+To remove an index, select it from the list of assigned indexes and click
+"Remove Index".
+<Break/>
+The default index, the one used on the collection's search page, is tagged with
+"[Default Index]" in the "Assigned Indexes" list.  To set it, select an index
+from the list and click "Set Default".
+<Break/>
+If advanced searching is enabled (via the Search Types view), the index controls are different.  Each index
+is based on just one data source.  There is a new pseudo-data source "allfields" which provides searching across all specified indexes at once.  Levels are not
+assigned to a specific index, but apply across all indexes: thus indexes and
+levels are added separately.  Indexes are removed in the same way as above, but
+the default index can no longer be set -- it is simply the
+first index assigned.
+<Break/>
+To create indexes on all sources, click the "Add All" button. The name of each index will default to the source name. To change the name, select an index, change its details, and click "Replace Index".
+    </Section>
+    <Section name="partitionindexes">
+      <Title>Partition Indexes</Title>
+Indexes are built on particular text or metadata sources.  The search space can
+be further controlled by partitioning the index, either by language or by a
+predetermined filter.  This section describes how to do this.  Under "Design
+Sections", click "Partition Indexes".
+<Break/>
+The "Partition Indexes" view has three tabs; "Define Filters", "Assign
+Partitions" and "Assign Languages".  To learn more about partitions read about
+subcollections and subindexes in Chapter 2 of the Greenstone Developer's Guide.
+
+      <Section name="definefilters">
+        <Title>Define Filters</Title>
+<ObviousSentence>The section explains how to define a partition filter.</ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+              Add a filter
+            </ContentsItem>
+            <ContentsItem>
+              Remove a filter
+            </ContentsItem>
+            <ContentsItem>
+              Update a filter
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Simple Instructions:</Anchor>
+Filters allows you to group together into a subcollection all documents in an
+index for which a metadata value matches a given pattern.
+<Break/>
+To create a filter, click the "Define Filters" tab and enter a name for the
+new filter into the "Name the subcollection filter" field.  Next choose a
+document attribute to match against, either a metadata element or the name of
+the file in question.  Enter a regular expression to use during the matching.
+You can toggle between "Including" documents that match the filter, or
+"Excluding" them.  Finally, you can specify any of the standard PERL regular
+expression flags to use when matching (e.g.  "i" for case-insensitive matching).
+Finally, click "Add Filter" to add the filter to the "Defined Subcollection Filters"
+list.
+<Break/>
+To remove a filter, select it from the list and click "Remove Filter".
+<Break/>
+To alter a filter, select it from the list, change any of the values that
+appear in the editing controls and click "Replace Filter" to commit the changes.
+      </Section>
+      <Section name="assignpartitions">
+        <Title>Assign Partitions</Title>
+<ObviousSentence>This section explains how to assign a previous defined partition filter.</ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+              Add a partition
+            </ContentsItem>
+            <ContentsItem>
+              Remove a partition
+            </ContentsItem>
+            <ContentsItem>
+              Set default partition
+            </ContentsItem>
+            <ContentsItem>
+              Clear default partition
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Simple Instructions:</Anchor>
+Having defined a subcollection filter, use the "Assign Partitions" tab to build
+indexes for it (or for a group of filters).  Select the desired filter (or
+filters) from the "Defined Subcollection Filters" list, enter a name for your partition in the "Partition Name" field,  and click "Add Partition".
+<Break/>
+To remove a partition, select it from the list and click "Remove Partition".
+<Break/>
+To make a partition the default one, select it from the list and click "Set
+Default".
+<Break/>
+To clear the default partition, click "Clear Default".
+      </Section>
+      <Section name="assignlanguages">
+        <Title>Assign Languages</Title>
+This section details how to restrict search indexes to particular languages.
+You do this by generating a partition using the "Assign Languages" tab of the
+"Partition Indexes" view.
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+              Language controls
+            </ContentsItem>
+            <ContentsItem>
+              Add language
+            </ContentsItem>
+            <ContentsItem>
+              Remove language
+            </ContentsItem>
+            <ContentsItem>
+              Set default language
+            </ContentsItem>
+            <ContentsItem>
+              Clear default language
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Simple Instructions:</Anchor>
+To add a new language to partition by, use the "Assign Languages" tab to build
+an index for it.  Select the desired language from the "Language to add"
+pull-down list and click "Add Language".
+<Break/>
+To remove a language, select it from the "Language Selection" list and click
+"Remove Language".
+<Break/>
+To set the default language, select it from the list and click "Set Default".
+<Break/>
+To clear the default language, click "Clear Default".
+      </Section>
+    </Section>
+    <Section name="xcollectionsearching">
+      <Title>Cross-Collection Searching</Title>
+Greenstone can search across several different collections as though they were
+one.  This is done by creating a "super-collection" that comprises the
+individual collections.  Under "Design Sections", click "Cross-Collection Search".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Selecting a collection
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The Cross-Collection Search view shows a checklist of available collections.  The
+current collection is ticked and cannot be deselected.  To add another
+collection to be searched in parallel, click it in the list (click again to
+remove it).  If only one collection is selected, there is no cross-collection
+searching.
+<Break/>
+For further details, see Chapter 1 of the Greenstone Developer's Guide.
+    </Section>
+    <Section name="classifiers">
+      <Title>Classifiers</Title>
+This section explains how to assign "classifiers", which are used for browsing,
+to the collection.  Under "Design Sections", click "Browsing Classifiers".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Classifier selection and configuration
+          </ContentsItem>
+          <ContentsItem>
+            Add a classifier
+          </ContentsItem>
+          <ContentsItem>
+            Remove a classifier
+          </ContentsItem>
+          <ContentsItem>
+            Configure a classifier
+          </ContentsItem>
+          <ContentsItem>
+            Alter classifier ordering
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To add a classifier, select it using the "Select classifier to add" pull-down list near the
+bottom and then click "Add Specified Classifier".  A window appears entitled
+"Configuring Arguments"; instructions for this dialog are just the same as for
+plugins (see <Reference target="plugins">Document Plugins</Reference>).  Once you have configured the new
+classifier, it is added to the end of the "Currently Assigned Classifiers"
+list.
+<Break/>
+To remove a classifier, select it from the list and click "Remove Selected
+Classifier".
+<Break/>
+To change the arguments a classifier, select it from the list and click
+"Configure Selected Classifier" (or double-click on the classifier in the
+list).
+<Break/>
+The ordering of classifiers in the collection's navigation bar is reflected in
+their order here.  To change it, select the classifier you want to move and
+click "Move Up" or "Move Down".
+<Break/>
+For further information on classifiers read Chapter 2, Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </Section>
+    <Section name="formatstatements">
+      <Title>Format Features</Title>
+Format commands control the structure and appearance of the collection.  They
+affect such things as where buttons appear when a document is shown, and what
+links are displayed by the DateList classifier.  Format commands are not easy
+to develop, and you should read Chapter 2 of the Greenstone Developer's Guide.
+This section discusses the format settings, and how the Librarian Interface
+gives access to them.  Under "Design Sections", click "Format Features".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Formatting
+          </ContentsItem>
+          <ContentsItem>
+            Add format command
+          </ContentsItem>
+          <ContentsItem>
+            Remove format command
+          </ContentsItem>
+          <ContentsItem>
+            Update format command
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Extended formatting options
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+You can apply a format command to anything in the "Choose Feature" pull-down
+list, which includes each classifier and a predefined list of features.  When
+you select a feature, there are two types of control.  Some features are simply
+enabled or disabled, and this is controlled by a checkbox.  Others require a
+format string to be specified.  For these there is a pull-down list ("Affected 
+Component") for selecting which part of the feature the string applies to 
+(if necessary), a text area ("HTML Format String") for entering the string, 
+and a selection of predefined "Variables".  To insert a variable into the 
+current position in the format string, select it from the pull-down list and 
+click "Insert".
+<Break/>
+You can specify a default format for a particular component by selecting the
+blank feature.  This format is then applied to all applicable features unless
+otherwise specified.
+<Break/>
+To add a new format command, fill out the information as explained above and
+click "Add Format".  The new format command appears in the list of "Currently
+Assigned Format Commands".  Only one format command can be assigned to each
+feature/component combination.
+<Break/>
+To remove a format command, select it from the list and click "Remove Format".
+<Break/>
+To change a format command, select it from the list, modify the settings, and
+click "Replace Format".
+<Break/>
+For more information about variables and the feature components, read Chapter 2
+of the Greenstone Developer's Guide.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+If the "Allow Extended Options" checkbox is ticked, some advanced formatting options are enabled. The list of features that can be formatted is changed slightly, and more variables are available to be used in the format command, providing greater control over the page layout.
+    </Section>
+    <Section name="translatetext">
+      <Title>Translate Text</Title>
+This section describes the translation view, where you can define
+language-specific text fragments for parts of the collection's interface.
+Under "Design Sections", click "Translate Text".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Add translation
+          </ContentsItem>
+          <ContentsItem>
+            Remove translation
+          </ContentsItem>
+          <ContentsItem>
+            Update translation
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+First choose the an entry from the "Features" list.  The language-specific
+strings associated with this feature appear below.  Use the "Language of
+translation" pull-down list to select the target language, and type the
+translated text into the text area, referring to the "Initial Text Fragment" if
+necessary.  Click "Add Translation" when finished.
+<Break/>
+To remove an existing translation, select it in the "Assigned Translations"
+table and click "Remove Translation".
+<Break/>
+To edit a translation, select it, edit it in the "Translated Text" text
+area, and click "Replace Translation".
+    </Section>
+    <Section name="metadatasets">
+      <Title>Metadata Sets</Title>
+This section explains the metadata set review panel.
+Under "Design Sections", click "Metadata Sets".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Available metadata sets
+          </ContentsItem>
+          <ContentsItem>
+            Adding metadata sets
+          </ContentsItem>
+          <ContentsItem>
+            Editing metadata sets
+          </ContentsItem>
+          <ContentsItem>
+            Removing metadata sets
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+This view is used to review the metadata sets that the collection uses, and the
+elements that are available within each set.  Choose from the list of "Available
+Metadata Sets" in order to see details of their elements.
+<Break/>
+To use another metadata set with the loaded collection, click "Add Metadata Set" and select the metadata set file (.mds) for the new metadata set.
+<Break/>
+Editing metadata sets is done with the Greenstone Editor for Metadata Sets (GEMS). Clicking the "Edit Metadata Set" button provides information on how to run the GEMS.
+<Break/>
+If you no longer need a metadata set, select it and press "Remove Metadata Set" to remove it. If you have assigned any metadata to elements in the removed set you will be asked how to deal with this metadata when you next open the collection.
+    </Section>
+  </Section>
+
+  <Section name="producingthecollection">
+    <Title>Producing Your Collection</Title>
+Having collected the documents for the collection, annotated them with
+metadata, and designed how the collection will appear, you can now produce the
+collection using Greenstone.  This section explains how.
+    <Section name="thecreateview">
+      <Title>The Create View</Title>
+<ObviousSentence>This section explains the Create view used to produce a collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Building the collection
+	  </ContentsItem>
+          <ContentsItem>
+	    Previewing the collection
+	  </ContentsItem>
+	</ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Message log
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The Create view is used to create the collection by running Greenstone
+collection-building scripts on the information you have provided.
+Clicking "Build Collection" initiates the collection building process.  The time
+this takes depends on the size of the collection and the number of indexes
+being created (for huge collections it can be hours). A progress bar indicates how much of the process has been completed. To cancel the process at
+any time, click "Cancel Build".
+<Break/>
+<!-- added -->
+Once the collection has successfully built, clicking "Preview Collection" will launch a web browser showing the home page of the collection.
+      <Anchor name="advanced">Advanced Instructions:</Anchor>
+In Expert mode, you can use the "Message Log" entry at the left to review previous attempts to build the collection, whether successful or not. Select the log you want by clicking on the desired date in the "Log History" list.
+    </Section>
+    <Section name="buildsettings">
+      <Title>Import and Build Settings</Title>
+This section explains how to access the various import and build settings.
+For more information of importing and building read Chapter 1 of the Greenstone
+Developer's Guide -- Understanding the collection-building process.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            General settings
+          </ContentsItem>
+          <ContentsItem>
+            Import settings
+          </ContentsItem>
+          <ContentsItem>
+            Build settings
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Controlling the various settings is done in a similar way to the
+"Configuring Arguments" window described in the <Reference target="plugins">Document Plugins</Reference> section.  Some
+fields require numeric arguments, and you can either type these in or use the
+up and down arrows to increase or decrease the current value (in some cases,
+the interface restricts the range you can enter).  Others are enabled by
+clicking a checkbox (click again to disable).
+    </Section>
+  </Section>
+
+  <Section name="miscellaneous">
+    <Title>Miscellaneous</Title>
+This section describes features of the Librarian Interface that are not associated
+with any particular view.
+    <Section name="preferences">
+      <Title>Preferences</Title>
+This section explains the preferences dialog, accessed by opening "File" -> "Preferences".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>
+	    General
+	  </ContentsItem>
+	  <ContentsItem>
+	    Mode
+	  </ContentsItem>
+          <ContentsItem>
+            Workflow
+          </ContentsItem>
+          <ContentsItem>
+            Connection
+          </ContentsItem>
+          <ContentsItem>
+            Warnings
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+The first "General" option is a text field for entering your e-mail address. This will be used for the "creator" and "maintainer" collection metadata items. The next option is a pull-down list of the languages in which the Librarian Interface can be presented. If you change the dictionary by choosing one from the list, you must
+restart the Librarian Interface in order to load the new language strings from
+the dictionary.
+<Break/>
+If "View Extracted Metadata" is checked, the
+various controls dealing with metadata always show all metadata that has been
+extracted automatically from documents.  Deselecting it hides this metadata
+(although it is still available during collection design, and within the
+final Greenstone collection). If "Show file sizes" is checked, the file size is shown next to each file in the Workspace and Collection file trees in the Gather and Enrich views.
+<Break/>
+The "Mode" panel is used to control the level of detail within the interface. At its lowest setting, "Library Assistant", the design view is disabled, arguments requiring regular expressions are hidden and the collection building produces a minimal log of events. In contrast the highest setting, "Expert", provides access to all of the features of design, including plugin positioning and regular expression arguments, and also allows the full output from the collection building to be recorded in the logs. To change or review modes, click the radio button next to the mode you are interested in. You can quickly review what mode you are in by looking at the Librarian Interface's title bar.
+<Break/>
+The Librarian Interface can support different workflows by determining which of
+the various view tabs are visible.  Use the "Workflow" tab to customise what
+views are available by checking the boxes next to the views that you want to be
+available.  Alternatively, use the pull-down list at the bottom to select
+predetermined configurations.  Closing the preferences dialog establishes these
+workflow settings.  These settings are stored with the collection, not in the
+Librarian Interface configuration file.
+<Break/>
+The "Connection" tab lets you alter the path to the locally-running Greenstone library server, which is used when Previewing collections. It also lets you set proxy information for connecting to the Internet (e.g. when downloading files; see the <Reference target="downloadingfiles">Downloading Files From the Internet</Reference> section for details). Check the box to enable proxy connection and supply details of the proxy host address and port number. The proxy connection is established when you close the Preferences dialog.
+<Break/>
+During the course of a session the Librarian Interface may give warning
+messages which inform you of possibly unforeseen consequences of an action.  You
+can disable the messages by checking the "Do not show this warning again" box.
+You can re-enable warning messages using the "Warnings" tab.  Check the box
+next to warning messages you want to see again.
+    </Section>
+    <Section name="fileassociations">
+      <Title>File Associations</Title>
+The Librarian Interface uses particular application programs to open particular
+file types.  This section explains how to assign and edit these file
+associations.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Add association
+          </ContentsItem>
+          <ContentsItem>
+            Edit association
+          </ContentsItem>
+          <ContentsItem>
+            Remove association
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+To alter file associations open the "File" menu and click "File
+Associations...".
+<Break/>
+To add an association, select the target file extension from the pull-down
+list, or type in a new extension (do not include the ".").  Next either type
+command that launches the desired application in the appropriate field, or
+choose the application from the "Browse" dialog.  "%1" can be used in the launch
+command to insert the name of the file being opened.  Once these are filled out,
+"Add Association" is enabled and can be clicked to add the association.
+<Break/>
+To edit an association, select an existing file extension.  Any existing
+associated command is shown in the launch command field.  Edit it, and then
+click "Replace Association".
+<Break/>
+To remove an association, select an existing file extension and click "Remove Association".
+(The file extension remains in the "For Files Ending" pull-down list.)
+<Break/>
+File associations are stored in the Librarian Interface's main folder, in a file
+called "associations.xml".
+    </Section>
+    <Section name="exportingcollections">
+      <Title>Exporting Collections to CD-ROM</Title>
+      
+This section describes how to export collections to a self-installing 
+Greenstone CD-ROM. 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            How to export
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Simple Instructions:</Anchor>
+Greenstone can export one or more collections to a self-installing 
+CD-ROM for Windows. To do so, Greenstone's "Export to CD-ROM" package 
+must be installed. This is not included by default, so you may need to 
+modify your installation to include it.
+<Break/>
+To export a collection, open the "File" menu and choose "Export to 
+CD-ROM". A list of Greenstone collections appears; click on any one to 
+see its description. Tick the check boxes of the collections to export. 
+You can enter the CD-ROM's name in the box: this is what will appear in 
+the Start menu when the CD-ROM has been installed. Then click "Export". 
+The process involves copying many files and may take a few minutes.
+<Break/>
+Upon completion, Greenstone will show the name of a folder containing 
+the exported collections. Use a CD writer to copy its contents to a 
+blank CD-ROM.
+    </Section>
+  </Section>
+</Document>
Index: /main/tags/3.01/gli/help/en/help_index.xml
===================================================================
--- /main/tags/3.01/gli/help/en/help_index.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/en/help_index.xml	(revision 21112)
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Document><Section name="introduction"><Title>Introduction</Title><Section name="ofmiceandmenus"><Title>Of Mice and Menus</Title></Section><Section name="howtoavoidthisdocument"><Title>How to Avoid Reading This Document</Title></Section></Section><Section name="startingoff"><Title>Starting Off</Title><Section name="creatingacollection"><Title>Creating a New Collection</Title></Section><Section name="savingacollection"><Title>Saving the Collection</Title></Section><Section name="openingacollection"><Title>Opening an Existing Collection</Title></Section><Section name="deletingcollections"><Title>Deleting Collections</Title></Section></Section><Section name="downloadingfiles"><Title>Downloading Files From the Internet</Title><Section name="themirrorview"><Title>The Download view</Title></Section></Section><Section name="collectingfiles"><Title>Collecting Files for Your Collection</Title><Section name="thegatherview"><Title>The Gather View</Title></Section><Section name="creatingfolders"><Title>Creating Folders</Title></Section><Section name="addingfiles"><Title>Adding Files</Title></Section><Section name="removingfiles"><Title>Removing Files</Title></Section><Section name="filteringthetree"><Title>Filtering the Tree</Title></Section></Section><Section name="enrichingacollection"><Title>Enriching the Collection with Metadata</Title><Section name="theenrichview"><Title>The Enrich View</Title></Section><Section name="selectingmetadatasets"><Title>Selecting Metadata Sets</Title></Section><Section name="appendingmetadata"><Title>Appending New Metadata</Title></Section><Section name="addingpreviouslydefinedmetadata"><Title>Adding Previously Defined Metadata</Title></Section><Section name="updatingmetadata"><Title>Updating Metadata</Title></Section><Section name="removingmetadata"><Title>Removing Metadata</Title></Section><Section name="reviewingmetadata"><Title>Reviewing Assigned Metadata</Title></Section><Section name="importingpreviouslyassignedmetadata"><Title>Importing Previously Assigned Metadata</Title></Section></Section><Section name="designingacollection"><Title>Designing Your Collection's Appearance</Title><Section name="thedesignview"><Title>The Design View</Title></Section><Section name="generalsettings"><Title>General Settings</Title></Section><Section name="plugins"><Title>Document Plugins</Title></Section><Section name="searchtypes"><Title>Search Types</Title></Section><Section name="searchindexes"><Title>Search Indexes</Title></Section><Section name="partitionindexes"><Title>Partition Indexes</Title><Section name="definefilters"><Title>Define Filters</Title></Section><Section name="assignpartitions"><Title>Assign Partitions</Title></Section><Section name="assignlanguages"><Title>Assign Languages</Title></Section></Section><Section name="xcollectionsearching"><Title>Cross-Collection Searching</Title></Section><Section name="classifiers"><Title>Classifiers</Title></Section><Section name="formatstatements"><Title>Format Features</Title></Section><Section name="translatetext"><Title>Translate Text</Title></Section><Section name="metadatasets"><Title>Metadata Sets</Title></Section></Section><Section name="producingthecollection"><Title>Producing Your Collection</Title><Section name="thecreateview"><Title>The Create View</Title></Section><Section name="buildsettings"><Title>Import and Build Settings</Title></Section></Section><Section name="miscellaneous"><Title>Miscellaneous</Title><Section name="preferences"><Title>Preferences</Title></Section><Section name="fileassociations"><Title>File Associations</Title></Section><Section name="exportingcollections"><Title>Exporting Collections to CD-ROM</Title></Section></Section></Document>
Index: /main/tags/3.01/gli/help/en/howtoavoidthisdocument.htm
===================================================================
--- /main/tags/3.01/gli/help/en/howtoavoidthisdocument.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/howtoavoidthisdocument.htm	(revision 21112)
@@ -0,0 +1,67 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="howtoavoidthisdocument"><font size="5" face="Verdana"><strong>1.2: How to Avoid Reading This Document</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Don't read this help text all the way through!  Just read enough to learn
+how to get help when you need it.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Context sensitive help
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Tool tips
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Other documents
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The "Help" menu item marks what may be the most appropriate help item with a
+little book icon.
+<p></p>
+For many controls, if you station the mouse over them a
+"tool tip" appears that says what they do.
+<p></p>
+Before using the Librarian Interface, first read the Greenstone
+documentation.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/importingpreviouslyassignedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/importingpreviouslyassignedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/importingpreviouslyassignedmetadata.htm	(revision 21112)
@@ -0,0 +1,69 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="importingpreviouslyassignedmetadata"><font size="5" face="Verdana"><strong>5.8: Importing Previously Assigned Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to import previously assigned metadata: metadata assigned to documents before they were added to the collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            While adding files
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+If metadata in a form recognized by the Librarian Interface has been previously
+assigned to a file -- for example, when you choose documents from an existing
+Greenstone collection -- it is imported automatically when you add the file.
+To do this, the metadata must be mapped to the metadata sets available in the
+collection.
+<p></p>
+The Librarian Interface prompts for the necessary information.  The prompt
+gives brief instructions and then shows the name of the metadata element that
+is being imported, just as it appears in the source file.  This field cannot be
+edited or changed.  Next you choose what metadata set the new element should map
+to, and then the appropriate metadata element in that set.  The system
+automatically selects the closest match, in terms of set and element, for the
+new metadata.
+<p></p>
+Having checked the mapping, you can choose "Add" to add the new metadata
+element to the chosen metadata set.  (This is only enabled if there is no
+element of the same name within the chosen set.) "Replace" maps the new element
+to the one chosen by the user.  Finally, "Ignore" does not import any metadata
+with this element name.  Once you have specified how to import a certain piece of metadata, the mapping
+information is retained for the collection's lifetime.
+<p></p>
+For details on the metadata.xml files which Greenstone uses to store the metadata, see Chapter 2 of the Greenstone
+Developer's Guide -- Getting the most out of your documents.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/introduction.htm
===================================================================
--- /main/tags/3.01/gli/help/en/introduction.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/introduction.htm	(revision 21112)
@@ -0,0 +1,17 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="introduction"><font size="5" face="Verdana"><strong>1: Introduction</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+The Greenstone Librarian Interface is a tool for collecting and marking up documents, then building digital library collections.  It provides access to the Greenstone Digital Library Software's functionality from a graphical point and click interface.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/metadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/en/metadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/metadatasets.htm	(revision 21112)
@@ -0,0 +1,74 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="metadatasets"><font size="5" face="Verdana"><strong>6.11: Metadata Sets</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains the metadata set review panel.
+Under "Design Sections", click "Metadata Sets".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Available metadata sets
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding metadata sets
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Editing metadata sets
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Removing metadata sets
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+This view is used to review the metadata sets that the collection uses, and the
+elements that are available within each set.  Choose from the list of "Available
+Metadata Sets" in order to see details of their elements.
+<p></p>
+To use another metadata set with the loaded collection, click "Add Metadata Set" and select the metadata set file (.mds) for the new metadata set.
+<p></p>
+Editing metadata sets is done with the Greenstone Editor for Metadata Sets (GEMS). Clicking the "Edit Metadata Set" button provides information on how to run the GEMS.
+<p></p>
+If you no longer need a metadata set, select it and press "Remove Metadata Set" to remove it. If you have assigned any metadata to elements in the removed set you will be asked how to deal with this metadata when you next open the collection.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/miscellaneous.htm
===================================================================
--- /main/tags/3.01/gli/help/en/miscellaneous.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/miscellaneous.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="miscellaneous"><font size="5" face="Verdana"><strong>8: Miscellaneous</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+This section describes features of the Librarian Interface that are not associated
+with any particular view.
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/ofmiceandmenus.htm
===================================================================
--- /main/tags/3.01/gli/help/en/ofmiceandmenus.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/ofmiceandmenus.htm	(revision 21112)
@@ -0,0 +1,106 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="ofmiceandmenus"><font size="5" face="Verdana"><strong>1.1: Of Mice and Menus</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section provides basic information about interacting with the Librarian Interface.
+If you are familiar with programs such as Internet Explorer or Microsoft
+Office and are comfortable with mouse clicks and menus, skip to the <a href="howtoavoidthisdocument.htm#howtoavoidthisdocument">next section</a>.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Basic concepts   
+            <tr>
+<td colspan="2"> </td><td colspan="1">
+              Mouse actions
+	    </td>
+</tr>
+            
+<tr>
+<td colspan="2"> </td><td colspan="1">
+              Keyboard
+	    </td>
+</tr>
+          
+</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Exiting the program
+          </td>
+</tr>
+	
+	
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Quick keys
+          </td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The Librarian Interface follows Microsoft Windows conventions and draws upon
+ordinary knowledge of Windows.
+<p></p>
+Any part of the screen that you interact with, such as a button or text field,
+is called a "control".  At any given time one control, called the "focus", is
+highlighted and responds to the keyboard.  Several controls allow you to select
+parts that are highlighted in dark blue.  Some controls are greyed out to
+indicate that they are disabled.
+<p></p>
+You can move and left- or right-click the mouse in the usual way.  Many
+components also allow you to "drag" them, by clicking and holding the left
+mouse button, move them with the mouse, and "drop" them elsewhere by releasing
+the button.  Potential drop targets alter their appearance when a component
+hovers over them.
+<p></p>
+You can use the keyboard to type into text fields.  Keyboard alternatives are
+available for many controls, indicated by a key name in square brackets -- for
+example, [Tab] alters the focus.  The plus sign shows if other keys must be
+pressed at the same time.
+<p></p>
+Exit the Librarian Interface program by choosing "Exit" from the "File"
+menu.  Your collection will be saved first.
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+To access a menu, hold down [ALT] and press the corresponding letter
+(underlined).  For example, for the "File" menu press [ALT] + [F].  To choose an
+item, press the corresponding key.  For example, while in the File menu
+press [S] to "Save" a collection.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/openingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/openingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/openingacollection.htm	(revision 21112)
@@ -0,0 +1,88 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="openingacollection"><font size="5" face="Verdana"><strong>2.3: Opening an Existing Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section tells you how to open existing collections using the Open
+Collection prompt.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            How to open a collection
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Locked collections
+          </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Legacy collections
+	  </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To open an existing collection, choose "Open" from the "File" menu to get the
+Open Collection prompt.  A list of your Greenstone collections appears.
+Select one to see its description, and click "Open" to load it.  If you seek a
+collection that resides outside Greenstone's "collect" folder, click "Browse"
+for a file system browsing dialog.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+In case more than one Greenstone Librarian Interface program is running
+concurrently, the relevant directories are "locked" to prevent interference.
+On opening a collection, a small temporary lock file is created in its
+folder.  Before opening a collection, the Librarian Interface checks to ensure
+that no lock file already exists.  You can tell whether a collection is locked
+by the colour of its icon: green for a normal collection, red for a locked
+one.  However, when the Librarian Interface is exited prematurely the lock file
+is sometimes left in place.  When you open such a collection, the Librarian asks
+if you want to "steal" control of it.  Never steal a collection that someone
+else is currently working on.
+<p></p>
+When you open a collection that the Greenstone Librarian Interface did 
+not create, you will be asked to select a metadata set (or sets).  If 
+none are selected, any existing metadata will be ignored. Otherwise, 
+metadata will be imported just as it is when you drag in files with 
+existing metadata. The process is described in the <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importing Previously Assigned Metadata</a> section. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/partitionindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/en/partitionindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/partitionindexes.htm	(revision 21112)
@@ -0,0 +1,26 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="partitionindexes"><font size="5" face="Verdana"><strong>6.6: Partition Indexes</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Indexes are built on particular text or metadata sources.  The search space can
+be further controlled by partitioning the index, either by language or by a
+predetermined filter.  This section describes how to do this.  Under "Design
+Sections", click "Partition Indexes".
+<p></p>
+The "Partition Indexes" view has three tabs; "Define Filters", "Assign
+Partitions" and "Assign Languages".  To learn more about partitions read about
+subcollections and subindexes in Chapter 2 of the Greenstone Developer's Guide.
+
+      
+      
+      
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/plugins.htm
===================================================================
--- /main/tags/3.01/gli/help/en/plugins.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/plugins.htm	(revision 21112)
@@ -0,0 +1,113 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="plugins"><font size="5" face="Verdana"><strong>6.3: Document Plugins</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to configure the document plugins the
+collection uses.  It explains how you specify what
+plugins to use, what parameters to pass to them, and in what order
+they occur.  Under "Design Sections", click "Document Plugins".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add a plugin
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove a plugin
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configure a plugin
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Change plugin order
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Custom configuration
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To add a plugin, select it using the "Select plugin to add" pull-down list near the
+bottom and then click "Add Plugin".  A window appears entitled
+"Configuring Arguments"; it is described later.  Once you have configured the
+new plugin, it is added to the end of the "Currently Assigned Plugins" list.
+Note that, except for UnknownPlug, each plugin may only occur once in the list.
+<p></p>
+To remove a plugin, select it in the list and click "Remove Plugin".
+<p></p>
+Plugins are configured by providing arguments.  To alter them, select the
+plugin from the list and click "Configure Plugin" (or double-click
+the plugin). A "Configuring Arguments" dialog appears with various controls for specifying arguments.
+<p></p>
+There are different kinds of controls.  Some are checkboxes, and clicking one
+adds the appropriate option to the plugin.  Others are text strings, with a
+checkbox and a text field.  Click the box to enable the argument, then type
+appropriate text (regular expression, file path etc) in the box.  Others are
+pull-down menus from which you can select from a given set of values.  To learn what an
+argument does, let the mouse hover over its name for a moment and a description
+will appear.
+<p></p>
+When you have changed the configuration, click "OK" to commit the changes and
+close the dialog, or "Cancel" to close the dialog without changing any plugin
+arguments.
+<p></p>
+The plugins in the list are executed in order, and the ordering is sometimes
+important.  Two plugins, ArcPlug and RecPlug, are vital to the collection
+building process, and are fixed in place at the end of the list (with a
+separator line).  To change the ordering of the other ones, select the plugin you want to move
+and click "Move Up" or "Move Down".
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+The Librarian Interface does its best to determine what arguments a plugin
+supports. However, there may be cases where the user wants to specify special arguments, and for this a text field called "Custom Arguments" (at the bottom) is provided. Any text in it is appended verbatim to the end of the plugin
+command.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/preferences.htm
===================================================================
--- /main/tags/3.01/gli/help/en/preferences.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/preferences.htm	(revision 21112)
@@ -0,0 +1,97 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="preferences"><font size="5" face="Verdana"><strong>8.1: Preferences</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains the preferences dialog, accessed by opening "File" -&gt; "Preferences".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    General
+	  </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Mode
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Workflow
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Connection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Warnings
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The first "General" option is a text field for entering your e-mail address. This will be used for the "creator" and "maintainer" collection metadata items. The next option is a pull-down list of the languages in which the Librarian Interface can be presented. If you change the dictionary by choosing one from the list, you must
+restart the Librarian Interface in order to load the new language strings from
+the dictionary.
+<p></p>
+If "View Extracted Metadata" is checked, the
+various controls dealing with metadata always show all metadata that has been
+extracted automatically from documents.  Deselecting it hides this metadata
+(although it is still available during collection design, and within the
+final Greenstone collection). If "Show file sizes" is checked, the file size is shown next to each file in the Workspace and Collection file trees in the Gather and Enrich views.
+<p></p>
+The "Mode" panel is used to control the level of detail within the interface. At its lowest setting, "Library Assistant", the design view is disabled, arguments requiring regular expressions are hidden and the collection building produces a minimal log of events. In contrast the highest setting, "Expert", provides access to all of the features of design, including plugin positioning and regular expression arguments, and also allows the full output from the collection building to be recorded in the logs. To change or review modes, click the radio button next to the mode you are interested in. You can quickly review what mode you are in by looking at the Librarian Interface's title bar.
+<p></p>
+The Librarian Interface can support different workflows by determining which of
+the various view tabs are visible.  Use the "Workflow" tab to customise what
+views are available by checking the boxes next to the views that you want to be
+available.  Alternatively, use the pull-down list at the bottom to select
+predetermined configurations.  Closing the preferences dialog establishes these
+workflow settings.  These settings are stored with the collection, not in the
+Librarian Interface configuration file.
+<p></p>
+The "Connection" tab lets you alter the path to the locally-running Greenstone library server, which is used when Previewing collections. It also lets you set proxy information for connecting to the Internet (e.g. when downloading files; see the <a href="downloadingfiles.htm#downloadingfiles">Downloading Files From the Internet</a> section for details). Check the box to enable proxy connection and supply details of the proxy host address and port number. The proxy connection is established when you close the Preferences dialog.
+<p></p>
+During the course of a session the Librarian Interface may give warning
+messages which inform you of possibly unforeseen consequences of an action.  You
+can disable the messages by checking the "Do not show this warning again" box.
+You can re-enable warning messages using the "Warnings" tab.  Check the box
+next to warning messages you want to see again.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/producingthecollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/producingthecollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/producingthecollection.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="producingthecollection"><font size="5" face="Verdana"><strong>7: Producing Your Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+Having collected the documents for the collection, annotated them with
+metadata, and designed how the collection will appear, you can now produce the
+collection using Greenstone.  This section explains how.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/removingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/en/removingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/removingfiles.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingfiles"><font size="5" face="Verdana"><strong>4.4: Removing Files</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to remove files and folders from your collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The delete button
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The delete key
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Delete by drag and drop
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+There are several methods for removing files and folders.  You must first
+indicate what items to remove by selecting one or more files and folders as
+described in <a href="thegatherview.htm#thegatherview">The Gather View</a>.
+<p></p>
+Once files have been selected, click the "delete" button to remove them, or
+press the [Delete] key on your keyboard, or drag them from the collection to
+the delete button and drop them there.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/removingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/removingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/removingmetadata.htm	(revision 21112)
@@ -0,0 +1,58 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingmetadata"><font size="5" face="Verdana"><strong>5.6: Removing Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to remove metadata from a file.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Removing from files
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Removing from folders or multiple files
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+You remove metadata the same way as you update it.  First select a file from the
+file tree, then use the metadata table to select the metadata. If the metadata has a value assigned, the "Remove" button becomes active.
+Click it to remove the metadata from the specified file.  Other files
+remain unchanged, and the value remains in the Value Tree.
+<p></p>
+Removing metadata from folders or multiple files works the same way.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/reviewingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/reviewingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/reviewingmetadata.htm	(revision 21112)
@@ -0,0 +1,106 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="reviewingmetadata"><font size="5" face="Verdana"><strong>5.7: Reviewing Assigned Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to view all metadata assigned to an entire collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The audit table
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Right button menu
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Autofilter
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Sometimes you need to see the metadata assigned to many or all files at once -- for instance,
+to determine how many files are left to work on, or to get some idea of the
+spread of dates.
+<p></p>
+Select the files you wish to examine, then right-click and choose "Assigned Metadata...". A window called
+"All Metadata", dominated by a large table with many columns, appears.  The
+first column shows file names; the rows show all metadata values assigned to
+those files.
+<p></p>
+Drawing the table can take some time if many files are selected.  You can continue to use the Librarian
+Interface while the "All Metadata" window is open.
+<p></p>
+Click "Close" to hide the window.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+You can also view the "All Metadata" table by selecting the files you wish to
+examine, right-clicking, and choosing "Assigned Metadata...".  If a folder has
+been selected, all its child files are included in the table.
+<p></p>
+When it gets too large, you can filter the "All Metadata" table by applying
+filters to the columns.  As new filters are added, only those rows that match
+them remain visible.  To set, modify or clear a filter, click on the "funnel"
+icon at the top of a column.  You are prompted for information about the filter.
+Once a filter is set, the column header changes colour.
+<p></p>
+The prompt has a "Simple" and an "Advanced" tab.  The Simple version filters
+columns so that they only show rows that contain a certain metadata value ("*"
+matches all values).  You can select metadata values from the pull-down list.
+The Advanced version allows different matching operations: must start with,
+does not contain, alphabetically less than and is equal to.  The value to be
+matched can be edited to be any string (including "*"), and you can choose
+whether the matching should be case insensitive.  Finally, you can specify a
+second matching condition that you can use to specify a range of values (by
+selecting AND) or alternative values (by selecting OR).  Below this area is a
+box that allows you to change the sort order (ascending or descending).  Once
+you have finished, click "Set Filter" to apply the new filter to the column.
+Click "Clear Filter" to remove a current filter.  Note that the filter details
+are retained even when the filter is cleared.
+<p></p>
+For example, to sort the "All Metadata" table, choose a column, select the
+default filter setting (a Simple filter on "*"), and choose ascending or
+descending ordering.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/savingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/en/savingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/savingacollection.htm	(revision 21112)
@@ -0,0 +1,69 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="savingacollection"><font size="5" face="Verdana"><strong>2.2: Saving the Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to save a collection, and the Save Collection Prompt.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            How to save
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Where the files are saved
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Save your work regularly by opening the "File" menu and choosing "Save".
+Saving a collection is not the same as making it ready for use in Greenstone
+(see <a href="producingthecollection.htm#producingthecollection">Producing Your Collection</a>).
+<p></p>
+The Librarian Interface protects your work by saving it whenever you exit the
+program or load another collection.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+Saved collections are written to a file named for the collection and with file
+extension ".col", located in a folder of the same name within your Greenstone
+installation's "collect" folder.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/searchindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/en/searchindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/searchindexes.htm	(revision 21112)
@@ -0,0 +1,89 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchindexes"><font size="5" face="Verdana"><strong>6.5: Search Indexes</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Indexes specify what parts of the collection are searchable.  This section explains how to add and remove indexes, and set a default index.  Under "Design Sections", click "Search Indexes".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add an index
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Edit an index
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove an index
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Set default index
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To add an index, type a name for it into the "Index Name" field.  Select which
+of the possible information sources to index by clicking the checkboxes beside
+them.  The list shows all the assigned metadata elements, as well the full
+text.  Having selected the data sources, choose the granularity of the index,
+using the "At the level" menu.  Once these details are complete, "Add Index"
+becomes active (unless there is an existing index with the same settings).
+Click it to add the new index.
+<p></p>
+To edit an index, select it and change the index details, then click "Replace Index".
+<p></p>
+To remove an index, select it from the list of assigned indexes and click
+"Remove Index".
+<p></p>
+The default index, the one used on the collection's search page, is tagged with
+"[Default Index]" in the "Assigned Indexes" list.  To set it, select an index
+from the list and click "Set Default".
+<p></p>
+If advanced searching is enabled (via the Search Types view), the index controls are different.  Each index
+is based on just one data source.  There is a new pseudo-data source "allfields" which provides searching across all specified indexes at once.  Levels are not
+assigned to a specific index, but apply across all indexes: thus indexes and
+levels are added separately.  Indexes are removed in the same way as above, but
+the default index can no longer be set -- it is simply the
+first index assigned.
+<p></p>
+To create indexes on all sources, click the "Add All" button. The name of each index will default to the source name. To change the name, select an index, change its details, and click "Replace Index".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/searchtypes.htm
===================================================================
--- /main/tags/3.01/gli/help/en/searchtypes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/searchtypes.htm	(revision 21112)
@@ -0,0 +1,82 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchtypes"><font size="5" face="Verdana"><strong>6.4: Search Types</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to modify a new design feature in Greenstone, Search Types, which allow fielded searching.  Under "Design Sections", click "Search Types".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            What enabling advanced searches means
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Adding a new search type
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Removing a search type
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Changing the order of search types
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+When you enter the Search Types view, first check "Enable Advanced
+Searches", which activates the other controls.  This migrates the collection to
+Greenstone 2.4 format, which supports fielded searching, and means that (a)
+the index design is different (explained in the <a href="searchindexes.htm#searchindexes">Search Indexes</a> section),
+(b) there are more text fragments to translate (see <a href="translatetext.htm#translatetext">Translation</a>), and (c) the collection will not be usable under older Greenstone
+installations.  If you later uncheck this field, most of your collection will
+be migrated back to Greenstone 2.39.  However the Librarian Interface cannot
+convert the new index specifications into older ones, so you will have to
+re-enter them manually.
+<p></p>
+To add a search type, select it from the "Search Types" list and click "Add Search Type".  Each type can only appear in the list once.
+<p></p>
+To remove a search type, select it from the "Currently Assigned Search Types"
+list and click "Remove Search Type".  The list must contain at least
+one search type.
+<p></p>
+To change to order of a search type, select it from the list and click "Move
+Up" or "Move Down". The first one will be the default.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/selectingmetadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/en/selectingmetadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/selectingmetadatasets.htm	(revision 21112)
@@ -0,0 +1,61 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="selectingmetadatasets"><font size="5" face="Verdana"><strong>5.2: Selecting Metadata Sets</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to specify metadata elements and how to add metadata sets to your collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            What is a metadata set?
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Controlling the metadata sets used
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Sets of predefined metadata elements are known as "metadata sets".  An example
+is the Dublin Core metadata set.  When you add a metadata set to your
+collection, its elements become available for selection.  You can have more than
+one set; to prevent name clashes a short identifier that identifies the
+metadata set is pre-pended to the element name.  For instance the Dublin Core
+element Creator becomes "dc.Creator".  Metadata sets are stored in the
+Librarian Interface's metadata folder and have the suffix ".mds".
+<p></p>
+To control the metadata sets used in a collection, use the "Metadata Sets" entry on the Design view.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/startingoff.htm
===================================================================
--- /main/tags/3.01/gli/help/en/startingoff.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/startingoff.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="startingoff"><font size="5" face="Verdana"><strong>2: Starting Off</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    This section covers how to create, load, save and delete collections.
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/en/thecreateview.htm
===================================================================
--- /main/tags/3.01/gli/help/en/thecreateview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/thecreateview.htm	(revision 21112)
@@ -0,0 +1,76 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thecreateview"><font size="5" face="Verdana"><strong>7.1: The Create View</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains the Create view used to produce a collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Building the collection
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Previewing the collection
+	  </td>
+</tr>
+	
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Message log
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The Create view is used to create the collection by running Greenstone
+collection-building scripts on the information you have provided.
+Clicking "Build Collection" initiates the collection building process.  The time
+this takes depends on the size of the collection and the number of indexes
+being created (for huge collections it can be hours). A progress bar indicates how much of the process has been completed. To cancel the process at
+any time, click "Cancel Build".
+<p></p>
+
+Once the collection has successfully built, clicking "Preview Collection" will launch a web browser showing the home page of the collection.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+In Expert mode, you can use the "Message Log" entry at the left to review previous attempts to build the collection, whether successful or not. Select the log you want by clicking on the desired date in the "Log History" list.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/thedesignview.htm
===================================================================
--- /main/tags/3.01/gli/help/en/thedesignview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/thedesignview.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thedesignview"><font size="5" face="Verdana"><strong>6.1: The Design View</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section introduces you to the design view and explains how to navigate
+between the various views within this pane.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Choosing a design section
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+With the Librarian Interface, you can configure how the collection appears to
+the user.  The configuration options are divided into different sections, each
+associated with a particular stage of navigating or presenting information.
+<p></p>
+On the left is a list of different views, and on the right are the controls
+associated with the current one.  To change to a different view, click its name
+in the list.
+<p></p>
+To understand the stages and terms involved in designing a collection, first
+read Chapters 1 and 2 of the Greenstone Developer's Guide.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/theenrichview.htm
===================================================================
--- /main/tags/3.01/gli/help/en/theenrichview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/theenrichview.htm	(revision 21112)
@@ -0,0 +1,103 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="theenrichview"><font size="5" face="Verdana"><strong>5.1: The Enrich View</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to input and edit metadata using the Enrich view.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The collection tree
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The metadata editing controls
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The value tree
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The metadata table
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Use the Enrich view to assign metadata to the documents in the collection.
+Metadata is data about data -- typically title, author, creation date, and so
+on.  Each metadata item has two parts: "element" tells what kind of item it is
+(such as author), and "value" gives the value of that metadata element (such as
+the author's name).
+<p></p>
+On the left of the "Enrich" view is the Collection Tree. To the right is the Metadata Table, which shows metadata for any selected files or folders in the Collection Tree.  Columns are named in
+grey at the top, and can be resized by dragging the separating line. Clicking any row transfers its details to the metadata editing controls below. If several files are selected, black text indicates that the value is common to all of the
+selected files, while grey text indicates that it is not.  Black values may be
+updated or removed, while grey ones can be removed from those that have it, or
+appended to the others.
+<p></p>
+A folder icon may appear beside some metadata entries.  This indicates that the
+values are inherited from a parent (or ancestor) folder.  Inherited metadata
+cannot be edited or removed, only appended to or overwritten.  Click on the
+folder icon to go immediately to the folder where the metadata is assigned.
+<p></p>
+The metadata editing controls at the lower right appear only when a file is selected from the tree and a row is selected from the table. Use them to update, append,
+and remove the metadata value.  The value field is for entering or editing the
+metadata value.  Beside it is a button labelled "..." which, when clicked,
+opens a larger editing box.  In the buttons below, "Append" assigns the value as
+new metadata and adds it to any existing values for the selected element,
+"Replace" overwrites the selected existing value with the new one, and "Remove" clears
+the selected value.  Underneath, labelled "All Previous Values", is the "Value
+Tree".
+<p></p>
+The Value Tree expands and collapses.  Usually it is a list that shows all
+values entered previously for the selected element.  Clicking an entry
+automatically places it into the value field.  Conversely, typing in the text
+field selects the Value Tree entry that starts with the characters you have
+typed.  Pressing [Tab] auto-completes the typing with the selected value.
+<p></p>
+Metadata values can be organised into a hierarchy. This is shown in the Value Tree using folders for internal levels. Hierarchical values can be entered using the character "|" to separate the levels.  For
+example, "Cards|Red|Diamonds|Seven" might be used in a hierarchy that represents a pack of playing cards. This enables values to be grouped together. Groups can also be assigned as metadata to files.
+<p></p>
+Greenstone extracts metadata automatically from documents into a metadata set
+whose elements are prefixed by "ex.".  This has no value tree and cannot be
+edited, so the edit controls are hidden if such an entry is selected.  The
+"..." button still serves to expand the value, but the text cannot be edited.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/thegatherview.htm
===================================================================
--- /main/tags/3.01/gli/help/en/thegatherview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/thegatherview.htm	(revision 21112)
@@ -0,0 +1,120 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thegatherview"><font size="5" face="Verdana"><strong>4.1: The Gather View</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section introduces the Gather area that you use to select what files
+to include in the collection you are building.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            How to view the gather screen
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The file trees
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The status area
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Control buttons
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Selection methods
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Special folder mapping
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The Librarian Interface starts with the Gather view.  To return to this view
+later, click the "Gather" tab directly below the menu bar.
+<p></p>
+The two large areas titled "Workspace" and "Collection" are used to move files
+into your collection.  They contain "file trees", graphical structures that
+represent files and folders.
+<p></p>
+Select an item in the tree by clicking it.  (There are other ways; see below.)
+Double-click a folder, or single-click the switch symbol beside it, to expand (or collapse)
+its contents.  Double-click a file to open it using its associated application
+program (see <a href="fileassociations.htm#fileassociations">File Associations</a>).
+<p></p>
+The Workspace file tree shows the sources of data available to the Librarian Interface -- the local file system (including disk and CD-ROM drives), the contents of existing Greenstone collections, and the cache of downloaded files if Web mirroring is enabled. You can copy and view these files but you cannot move, delete, or edit them, with the exception of the downloaded files, which can be deleted.  Navigate this space to find the files you want to include in the collection.
+<p></p>
+The Collection file tree represents the contents of the collection so
+far.  Initially, it is empty.
+<p></p>
+You can resize the spaces by mousing over the grey bar that separates the trees
+(the shape of the pointer changes) and dragging.
+<p></p>
+At the bottom of the window is a status area that shows the progress of actions involving files (copying, moving and deleting). These can take some time to complete. The "Stop" button stops any action that is currently in progress. 
+<p></p>
+Two large buttons occupy the lower right corner of the screen. "New Folder", with a picture
+of a folder, creates new folders (see <a href="creatingfolders.htm#creatingfolders">Creating folders</a>). 
+"Delete", with a garbage can, removes files. Clicking the Delete button will remove any selected files from the Collection file tree. Alternatively, files can be deleted by dragging them onto the Delete button. 
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+To select several sequential items, select the first and then hold down [Shift]
+and click on the last -- the selection will encompass all intervening
+items.  Select non-sequential files by holding down [Ctrl] while clicking.  Use
+these two methods together to select groups of non-adjacent items.
+<p></p>
+Certain folders -- such as the one containing your own web pages -- sometimes
+have special significance.  The Librarian Interface can map such folders to 
+the first level of the file tree.  To do this, right-click the desired
+folder.  Select "Create Shortcut", and enter a name for the folder.  To remove an item,
+right-click the mapped folder and select "Remove Shortcut".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/themirrorview.htm
===================================================================
--- /main/tags/3.01/gli/help/en/themirrorview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/themirrorview.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="themirrorview"><font size="5" face="Verdana"><strong>3.1: The Download view</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes how to configure a download task and control the downloading process.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The workspace tree
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Download configuration
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            The download list
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Advanced Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enabling web proxies
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+Access the "Download" view by clicking its tab. The top half of the screen shows the downloading controls. The bottom half is initially empty, but will show a list of pending and completed downloading jobs. 
+<p></p>
+Files are downloaded into a folder in the workspace called "Downloaded Files" (only present when mirroring is enabled), and can be used in all collections built with the Librarian Interface.  Files in this area are named by their full web URL. A new folder is created for each host, followed by others for each part of the path. This ensures that each file is distinct.
+<p></p>
+Use the first of the download configuration controls, "Source URL", to enter the URL of a target resource. Use the "Download Depth" control to limit how many  hyperlinks to follow when downloading: Set this to 0 to download a single web page; set it to 1 to download a page and all the pages it points to. The depth limit is ignored when downloading media other than html pages. Next, there are several checkbox controls which can be set to turn on the specified feature for a specific download. Once the configuration is set up, click "Download" to start the new download job. There are two other button controls: "Preferences", which links to the connection section of the Preferences where proxy settings can be edited; and "Clear Cache", which deletes all previously downloaded files.
+<p></p>
+The download list has an entry for each web page download.  Each entry has a text region that gives details of the task along with a progress bar showing current activity. Three buttons appear to the left of each entry. "Pause" is used for pausing a currently downloading task. "View Log" opens a window showing the download log file. "Close" terminates the download and removes the task from the list.  
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Advanced Instructions:</a>
+<p style="text-align:justify"></p>
+The <a href="preferences.htm#preferences">Preferences section</a> describes how to establish an Internet connection via a proxy.  If authentication is needed, the proxy server prompts for identification and password.  The Librarian Interface does not store passwords between sessions.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/translatetext.htm
===================================================================
--- /main/tags/3.01/gli/help/en/translatetext.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/translatetext.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="translatetext"><font size="5" face="Verdana"><strong>6.10: Translate Text</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section describes the translation view, where you can define
+language-specific text fragments for parts of the collection's interface.
+Under "Design Sections", click "Translate Text".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Add translation
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Remove translation
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Update translation
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+First choose the an entry from the "Features" list.  The language-specific
+strings associated with this feature appear below.  Use the "Language of
+translation" pull-down list to select the target language, and type the
+translated text into the text area, referring to the "Initial Text Fragment" if
+necessary.  Click "Add Translation" when finished.
+<p></p>
+To remove an existing translation, select it in the "Assigned Translations"
+table and click "Remove Translation".
+<p></p>
+To edit a translation, select it, edit it in the "Translated Text" text
+area, and click "Replace Translation".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/updatingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/en/updatingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/updatingmetadata.htm	(revision 21112)
@@ -0,0 +1,61 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="updatingmetadata"><font size="5" face="Verdana"><strong>5.5: Updating Metadata</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+This section explains how to update the metadata assigned to a file.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Updating files
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Updating folders or multiple files
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+To update the value of a piece of metadata, first choose the file to which that
+value applies, and then the metadata element whose value you want to change.
+Your selection appears in the metadata edit controls.  Edit the value field and
+click "Replace" to alter the metadata.
+<p></p>
+The process is the same when updating a folder with child folders or multiple files, but you can only update metadata that is common to all files/folders selected.
+<p></p>
+The value tree shows all previous values, not just those currently assigned.
+Thus the value you have replaced will remain in the value tree.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/en/xcollectionsearching.htm
===================================================================
--- /main/tags/3.01/gli/help/en/xcollectionsearching.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/en/xcollectionsearching.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="xcollectionsearching"><font size="5" face="Verdana"><strong>6.7: Cross-Collection Searching</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Greenstone can search across several different collections as though they were
+one.  This is done by creating a "super-collection" that comprises the
+individual collections.  Under "Design Sections", click "Cross-Collection Search".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Simple Instructions:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Selecting a collection
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Simple Instructions:</a>
+<p style="text-align:justify"></p>
+The Cross-Collection Search view shows a checklist of available collections.  The
+current collection is ticked and cannot be deselected.  To add another
+collection to be searched in parallel, click it in the list (click again to
+remove it).  If only one collection is selected, there is no cross-collection
+searching.
+<p></p>
+For further details, see Chapter 1 of the Greenstone Developer's Guide.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/addingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/es/addingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/addingfiles.htm	(revision 21112)
@@ -0,0 +1,66 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingfiles"><font size="5" face="Verdana"><strong>4.3: A&ntilde;adiendo archivos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n muestra la forma de a&ntilde;adir archivos a su colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Arrastrando un archivo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Varios archivos</td>
+</tr>
+        
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Metadatos existentes</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Los archivos se pueden copiar arrastr&aacute;ndolos y coloc&aacute;ndolos en una colecci&oacute;n. El cursor se convierte en un fantasma del elemento seleccionado (o, si se ha seleccionado m&aacute;s de uno, del n&uacute;mero de ellos). Coloque la selecci&oacute;n en la estructura de directorios y subdirectorios de la colecci&oacute;n para copiar ah&iacute; los archivos (si la fuente son los archivos del &aacute;rea de trabajo) o mu&eacute;valos de lugar dentro de la misma colecci&oacute;n (si la fuente son los archivos de la colecci&oacute;n).
+      <p></p>
+      Cuando se copien varios archivos en una carpeta todos estar&aacute;n al mismo nivel, sin importar la estructura de las carpetas en las que se encontraban originalmente. Cuando se copie un segundo archivo con el mismo nombre en la misma carpeta se le preguntar&aacute; si desea reemplazar el archivo existente. Si la respuesta es "No" entonces el archivo no se copiar&aacute;, pero los dem&aacute;s s&iacute;. Para cancelar las acciones restantes de copiado haga click en el bot&oacute;n "Detener".
+      <p></p>
+      &Uacute;nicamente se mover&aacute;n los elementos "de m&aacute;s alto nivel" en una selecci&oacute;n. Una carpeta est&aacute; a un nivel m&aacute;s alto que los elementos que se encuentran en su interior, por lo que no se podr&aacute;n seleccionar los archivos de una carpeta y la carpeta al mismo tiempo.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Cuando se a&ntilde;ada un archivo, la Interfaz de Bibliotecario buscar&aacute; a trav&eacute;s de las carpetas fuente los archivos auxiliares que contienen los metadatos asignados previamente a dicho archivo y, si encuentra alguno, empezar&aacute; a importar estos metadatos. Conforme la operaci&oacute;n siga adelante se le pedir&aacute; (quiz&aacute; varias veces) que proporcione informaci&oacute;n adicional para conciliar los metadatos importados con los conjuntos de metadatos de su colecci&oacute;n. Este proceso incluye muchos avisos diferentes, los cuales se describen en la secci&oacute;n <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importando metadatos previamente asignados</a>. Para obtener una explicaci&oacute;n m&aacute;s detallada sobre la forma de asociar metadatos a los archivos lea el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone: Obteniendo el m&aacute;ximo beneficio de sus documentos.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/addingpreviouslydefinedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/addingpreviouslydefinedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/addingpreviouslydefinedmetadata.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingpreviouslydefinedmetadata"><font size="5" face="Verdana"><strong>5.4: A&ntilde;adiendo Metadatos Previamente Definidos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de a&ntilde;adir metadatos con valores que ya est&eacute;n presentes en la estructura de valores.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adiendo metadatos a archivos</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2"> A&ntilde;adiendo metadatos a carpetas o a varios archivos juntos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para a&ntilde;adir metadatos con un valor existente, seleccione primero el archivo y a continuaci&oacute;n el valor requerido de la estructura de valores, expandiendo las carpetas tanto como sea necesario. El valor de la entrada seleccionada aparecer&aacute; autom&aacute;ticamente en el campo de texto (otra alternativa es utilizar las funciones de selecci&oacute;n autom&aacute;tica y completamiento autom&aacute;tico de la estructura de valores). Haga click en "Agregar" para a&ntilde;adir el metadato al archivo seleccionado.
+      <p></p>
+      El proceso de a&ntilde;adir metadatos con valores existentes a carpetas o a varios archivos juntos es exactamente igual.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/appendingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/appendingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/appendingmetadata.htm	(revision 21112)
@@ -0,0 +1,70 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="appendingmetadata"><font size="5" face="Verdana"><strong>5.3: Agregando nuevos metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de a&ntilde;adir metadatos y valores a los archivos y c&oacute;mo darle nuevos valores a los elementos de los metadatos.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adiendo metadatos a los archivos</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2"> A&ntilde;adiendo metadatos a carpetas o a varios archivos juntos</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Jerarqu&iacute;as y edici&oacute;n de valores</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Ahora a&ntilde;adiremos un &iacute;tem de metadato -- tanto un elemento como un valor -- a un archivo. Primero, seleccione el archivo en la estructura de directorios y subdirectorios de la colecci&oacute;n que se encuentra a la izquierda. Esto hace que cualquier metadato asignado previamente a este archivo aparezca en la tabla de la derecha.
+      <p></p>
+      A continuaci&oacute;n, seleccione el elemento de metadato que desea a&ntilde;adir haciendo click en la fila correspondiente de la tabla.
+      <p></p>
+      Escriba el valor en el campo de valores. No use el car&aacute;cter "|", ya que &eacute;ste se utiliza para construir jerarqu&iacute;as. Una vez que haya terminado haga click en "Agregar" para a&ntilde;adir el nuevo valor como metadato del archivo seleccionado. El valor aparecer&aacute; inmediatamente en la tabla de metadatos.
+      <p></p>
+      Tambi&eacute;n se pueden a&ntilde;adir metadatos a una carpeta o a varios archivos seleccionados al mismo tiempo. Estos metadatos se a&ntilde;adir&aacute; a todos los archivos que forman parte de la carpeta o de la selecci&oacute;n, as&iacute; como a las subcarpetas. Recuerde que si usted asigna metadatos a una carpeta, todos los archivos nuevos que se encuentren en ella heredar&aacute;n autom&aacute;ticamente los valores de la carpeta.
+      <p></p>
+      Si usted eligi&oacute; un metadato que ya aparece en algunos de los archivos seleccionados y hace click en "Agregar", el metadato se a&ntilde;adir&aacute; a los dem&aacute;s archivos de la selecci&oacute;n.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Usted tambi&eacute;n puede darle estructura a los valores de los metadatos por medio de rutas, tal como se describe en la secci&oacute;n de <a href="theenrichview.htm#theenrichview">Asignaci&oacute;n de metadatos</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/assignlanguages.htm
===================================================================
--- /main/tags/3.01/gli/help/es/assignlanguages.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/assignlanguages.htm	(revision 21112)
@@ -0,0 +1,67 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignlanguages"><font size="5" face="Verdana"><strong>6.6.3: Asignar idiomas</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+	Esta secci&oacute;n describe la forma de limitar los &iacute;ndices de b&uacute;squeda a ciertos idiomas. Esto se hace generando una partici&oacute;n por medio de la pesta&ntilde;a "Asignar idiomas" que se encuentra en la pantalla "Particionar &iacute;ndices".
+	<p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+	  
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Controles de idioma</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir idioma</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar idioma</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Predeterminar un idioma por omisi&oacute;n</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Borrar el idioma predeterminado</td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+	
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+	Para a&ntilde;adir un nuevo idioma use la pesta&ntilde;a "Asignar idiomas" a fin de construirle un &iacute;ndice. Seleccione el idioma en la lista descendente "Idioma que se va a a&ntilde;adir" y haga click en "A&ntilde;adir idioma".
+	<p></p>
+	Para quitar un idioma, esc&oacute;jalo de la lista "Selecci&oacute;n de idiomas" y haga click en "Eliminar idioma".
+	<p></p>
+	Para establecer un idioma por omisi&oacute;n, selecci&oacute;nelo de la lista y haga click en "Predeterminar".
+	<p></p>
+	Para borrar el idioma predeterminado haga click en "Borrar el valor por omisi&oacute;n".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/es/assignpartitions.htm
===================================================================
--- /main/tags/3.01/gli/help/es/assignpartitions.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/assignpartitions.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignpartitions"><font size="5" face="Verdana"><strong>6.6.2: Asignar particiones</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+	Esta secci&oacute;n explica la forma de asignar un filtro de partici&oacute;n definido anteriormente.
+	<p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+	  
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir una partici&oacute;n</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar una partici&oacute;n</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Predeterminar una partici&oacute;n por omisi&oacute;n</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Borrar la partici&oacute;n predeterminada</td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+	
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+	Una vez que haya definido el filtro de una subcolecci&oacute;n, utilice la pesta&ntilde;a "Asignar particiones" para construir los &iacute;ndices del filtro (o de un grupo de filtros). Seleccione el filtro o los filtros que desee de la lista "Filtros definidos para la subcolecci&oacute;n", ingrese un nombre para la partici&oacute;n en el campo "Nombre de la partici&oacute;n" y haga click en "A&ntilde;adir partici&oacute;n".
+	<p></p>
+	Para quitar una partici&oacute;n de la lista, selecci&oacute;nela y haga click en "Eliminar partici&oacute;n".
+	<p></p>
+	Para hacer que una partici&oacute;n determinada sea el valor por omisi&oacute;n, selecci&oacute;nela de la lista y haga click en "Predeterminar".
+	<p></p>
+	Para borrar la partici&oacute;n predeterminada haga click en "Borrar el valor por omisi&oacute;n".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/es/buildsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/es/buildsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/buildsettings.htm	(revision 21112)
@@ -0,0 +1,53 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="buildsettings"><font size="5" face="Verdana"><strong>7.2: Controles de importaci&oacute;n y construcci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de tener acceso a los diferentes controles de importaci&oacute;n y construcci&oacute;n. Para mayor informaci&oacute;n lea el Cap&iacute;tulo 1 de la Gu&iacute;a del Programador de Greenstone: Comprendiendo el proceso de creaci&oacute;n de una colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Controles generales</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Controles de importaci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Controles de construcci&oacute;n</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La forma de controlar los diferentes ajustes es similar a la de la ventana "Argumentos de configuraci&oacute;n" que se describe en la secci&oacute;n de <a href="plugins.htm#plugins">Plugins</a>. Algunos campos requerir&aacute;n argumentos num&eacute;ricos y usted puede escribirlos directamente en el campo o usar las flechas que se encuentran a un lado para aumentar o disminuir el n&uacute;mero (en algunos casos la interfaz limita el rango). Otros se activan (y desactivan) haciendo click en los recuadros correspondientes.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/classifiers.htm
===================================================================
--- /main/tags/3.01/gli/help/es/classifiers.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/classifiers.htm	(revision 21112)
@@ -0,0 +1,69 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="classifiers"><font size="5" face="Verdana"><strong>6.8: Clasificadores</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de asignar "clasificadores" a una colecci&oacute;n, los cuales se usan para navegar y buscar en ella. En la parte de "Secciones de dise&ntilde;o" haga click en "Clasificadores de exploraci&oacute;n".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Selecci&oacute;n y configuraci&oacute;n de un clasificador</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir un clasificador</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar un clasificador</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Configurar un clasificador</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Cambiar el orden de los clasificadores</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para a&ntilde;adir un clasificador, esc&oacute;jalo por medio de la lista descendente "Seleccione el clasificador que se va a a&ntilde;adir" en la parte inferior de la pantalla y haga click en "A&ntilde;adir el clasificador especificado". Con esto aparecer&aacute; una ventana titulada "Argumentos de configuraci&oacute;n". Las instrucciones para este cuadro de di&aacute;logo son exactamente las mismas que para los plugins (vea la secci&oacute;n <a href="plugins.htm#plugins">Plugins</a>). Una vez que haya configurado el nuevo clasificador, &eacute;ste se a&ntilde;adir&aacute; al final de la lista de "Clasificadores asignados actualmente".
+      <p></p>
+      Para quitar un clasificador de la lista, selecci&oacute;nelo y haga click en "Eliminar el clasificador seleccionado".
+      <p></p>
+      Para cambiar los argumentos de un clasificador, selecci&oacute;nelo de la lista y haga click en "Configurar el clasificador seleccionado" o doble click en el nombre del clasificador.
+      <p></p>
+      El orden de los clasificadores en la barra de navegaci&oacute;n de la colecci&oacute;n se refleja aqu&iacute;. Para cambiarlo, seleccione el clasificador que desea mover y haga click en "Mover arriba" o "Mover abajo".
+      <p></p>
+      Para mayor informaci&oacute;n sobre los clasificadores, lea el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone: Obteniendo el m&aacute;ximo beneficio de sus documentos.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/collectingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/es/collectingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/collectingfiles.htm	(revision 21112)
@@ -0,0 +1,20 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="collectingfiles"><font size="5" face="Verdana"><strong>4: Reuniendo archivos para su colecci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Una vez que usted tiene una nueva colecci&oacute;n necesita meter algunos archivos en ella. Estos pueden venir desde la estructura de archivos de su computadora o de otras colecciones Greenstone. Algunos incluso ya pueden tener metadatos. Esta secci&oacute;n describe la forma de importar archivos.
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/creatingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/creatingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/creatingacollection.htm	(revision 21112)
@@ -0,0 +1,88 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingacollection"><font size="5" face="Verdana"><strong>2.1: Creando una nueva colecci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+       Esta secci&oacute;n describe la forma de crear una nueva colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Aviso de la nueva colecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">T&iacute;tulo de la colecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Conjuntos de metadatos por omisi&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Descripci&oacute;n de la colecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Crear una nueva colecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Cancelar la nueva colecci&oacute;n</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Teclas de m&eacute;todo abreviado</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para crear una nueva colecci&oacute;n abra el men&uacute; "Archivo" y elija la opci&oacute;n "Nueva". Deber&aacute;n llenarse varios campos, aunque en caso de que lo requiera usted podr&aacute; cambiar despu&eacute;s sus valores en la pantalla de dise&ntilde;o.
+      <p></p>
+      El "T&iacute;tulo de la colecci&oacute;n" es el texto que aparece en la parte superior de la p&aacute;gina principal de su colecci&oacute;n. Puede tener la longitud  que usted quiera.
+      <p></p>
+      La "Descripci&oacute;n del contenido" deber&aacute; describir tan detalladamente como sea posible de lo que se trata la colecci&oacute;n. Utilice la tecla [Enter] o [Intro] para dividirla en p&aacute;rrafos.
+      <p></p>
+      Finalmente deber&aacute; especificar si desea que la nueva colecci&oacute;n tenga la misma apariencia y conjunto de metadatos que una colecci&oacute;n existente o si desea iniciar una "Nueva Colecci&oacute;n" por omisi&oacute;n.
+      <p></p>
+      Haga click en "Aceptar" para crear la colecci&oacute;n. Si usted escogi&oacute; "Nueva Colecci&oacute;n" se le pedir&aacute; que elija el conjunto de metadatos que usar&aacute; en ella.  Puede elegir m&aacute;s de uno y tambi&eacute;n puede a&ntilde;adir otros m&aacute;s adelante.
+      <p></p>
+      Si hace click en "Cancelar" regresar&aacute; inmediatamente a la pantalla principal.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Los botones, al igual que los men&uacute;s, tienen una letra subrayada. Para seleccionar un bot&oacute;n oprima la tecla [ALT] y la letra que est&aacute; subrayada al mismo tiempo.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/creatingfolders.htm
===================================================================
--- /main/tags/3.01/gli/help/es/creatingfolders.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/creatingfolders.htm	(revision 21112)
@@ -0,0 +1,53 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingfolders"><font size="5" face="Verdana"><strong>4.2: Creando carpetas</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n muestra la forma de crear nuevas carpetas.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">El bot&oacute;n de nueva carpeta</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">El men&uacute; del bot&oacute;n derecho del mouse</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Utilice las carpetas que se encuentran en la estructura de directorios y subdirectorios de la colecci&oacute;n para agrupar archivos y hacer que sean m&aacute;s f&aacute;ciles de encontrar. Las carpetas pueden contener a su vez otras subcarpetas. No hay pr&aacute;cticamente ning&uacute;n l&iacute;mite en cuanto al n&uacute;mero de carpetas que se pueden tener o qu&eacute; tan profundamente anidadas puedan estar.
+      <p></p>
+      Para crear una nueva carpeta, una opci&oacute;n es seleccionar una carpeta existente en la estructura de directorios y subdirectorios de la colecci&oacute;n y hacer click en el bot&oacute;n "Nueva carpeta". La nueva carpeta aparecer&aacute; dentro de la que se seleccion&oacute; primero o en el nivel superior si no se seleccion&oacute; ninguna. Se le pedir&aacute; que le d&eacute; un nombre a la carpeta (el valor por omisi&oacute;n es "Nueva carpeta").
+      <p></p>
+      Tambi&eacute;n se pueden crear nuevas carpetas haciendo click sobre una carpeta con el bot&oacute;n derecho del mouse,  eligiendo la opci&oacute;n "Nueva carpeta" y continuando tal como se describe arriba.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/definefilters.htm
===================================================================
--- /main/tags/3.01/gli/help/es/definefilters.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/definefilters.htm	(revision 21112)
@@ -0,0 +1,59 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="definefilters"><font size="5" face="Verdana"><strong>6.6.1: Definir filtros</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+        Esta secci&oacute;n explica la forma de definir un filtro de partici&oacute;n.
+	<p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+	  
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir un filtro</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar un filtro</td>
+</tr>
+	    
+<tr>
+<td colspan="1"> </td><td colspan="2">Actualizar un filtro</td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+	
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+	Los filtros le permiten agrupar en una subcolecci&oacute;n todos los documentos de un &iacute;ndice para los cuales un valor de metadato sigue un patr&oacute;n definido.
+	<p></p>
+	Para crear un filtro haga click en la pesta&ntilde;a "Definir filtros" y escriba el nombre del nuevo filtro en el campo titulado "Nombre del filtro de la subcolecci&oacute;n". A continuaci&oacute;n elija un atributo del documento con el cual se va a contrastar, ya sea un elemento de metadato o el nombre del archivo en cuesti&oacute;n. Ingrese una expresi&oacute;n regular que se utilizar&aacute; durante la contrastaci&oacute;n. Usted puede variar libremente entre "Incluir" documentos que concuerden con el filtro o "Excluirlos". Por &uacute;ltimo, usted tambi&eacute;n puede especificar cualquiera de los indicadores est&aacute;ndar de expresiones regulares PERL para usarlos al momento de contrastar (p. ej. "i" para concordancias que no tomen en cuenta el caso). Haga click en "A&ntilde;adir filtro" para a&ntilde;adir el filtro a la lista de "Filtros definidos para la subcolecci&oacute;n".
+	<p></p>
+	Para quitar un filtro de la lista, selecci&oacute;nelo y haga click en "Eliminar filtro".
+	<p></p>
+	Para modificar un filtro, selecci&oacute;nelo de la lista, cambie cualquiera de los valores que aparecen en los controles de edici&oacute;n y haga click en "Reemplazar filtro" para guardar los cambios.
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/es/deletingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/es/deletingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/deletingcollections.htm	(revision 21112)
@@ -0,0 +1,47 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="deletingcollections"><font size="5" face="Verdana"><strong>2.4: Borrando colecciones</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica como borrar definitivamente colecciones de su instalaci&oacute;n de Greenstone.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            C&oacute;mo borrar colecciones
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para borrar definitivamente colecciones de su instalaci&oacute;n de Greenstone, elija "Borrar&hellip;" del men&uacute; "Archivo". Aparece una lista de sus colecciones. Seleccione una para ver su descripci&oacute;n, marque la casilla situada en la parte inferior del di&aacute;logo, y pulse "Borrar" para borrar la colecci&oacute;n. Esta acci&oacute;n es irreversible, por lo que debe comprobar cuidadosamente que no necesita la colecci&oacute;n antes de proceder.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/designingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/designingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/designingacollection.htm	(revision 21112)
@@ -0,0 +1,26 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="designingacollection"><font size="5" face="Verdana"><strong>6: Dise&ntilde;ando la apariencia de su colecci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Una vez que los archivos est&aacute;n marcados con metadatos, a continuaci&oacute;n se debe decidir la forma en que aparecer&aacute;n ante los usuarios como parte de una colecci&oacute;n Greenstone. &iquest;Qu&eacute; tipo de informaci&oacute;n se podr&aacute; buscar? &iquest;De qu&eacute; medios se dispone para navegar por los documentos? &iquest;En qu&eacute; idiomas? &iquest;D&oacute;nde aparecer&aacute;n los botones en la p&aacute;gina? Todas estas cosas se pueden personalizar. Esta secci&oacute;n describe la forma de hacerlo.
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/downloadingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/es/downloadingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/downloadingfiles.htm	(revision 21112)
@@ -0,0 +1,16 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="downloadingfiles"><font size="5" face="Verdana"><strong>3: Descargando archivos de Internet</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    La vista "Bajar" le ayuda a descargar recursos de internet. Esta secci&oacute;n explica el interfaz del mecanismo de puesta en espejo del Bibliotecario.
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/enrichingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/enrichingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/enrichingacollection.htm	(revision 21112)
@@ -0,0 +1,23 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="enrichingacollection"><font size="5" face="Verdana"><strong>5: Enriqueciendo la colecci&oacute;n con metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Despu&eacute;s de haber juntado varios archivos en una colecci&oacute;n, enriqu&eacute;zcalos con informaci&oacute;n adicional. Esta secci&oacute;n explica la forma en que se crean, editan, asignan y recuperan los metadatos, y la forma de usar fuentes externas de metadatos (v&eacute;ase tambi&eacute;n el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone: Obteniendo el m&aacute;ximo beneficio de sus documentos).
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/exportingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/es/exportingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/exportingcollections.htm	(revision 21112)
@@ -0,0 +1,49 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="exportingcollections"><font size="5" face="Verdana"><strong>8.3: Exportando colecciones a un CD-ROM</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la forma de exportar colecciones a un CD-ROM Greenstone que se podr&aacute; instalar autom&aacute;ticamente.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">C&oacute;mo exportar</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Greenstone puede exportar una o m&aacute;s colecciones a un CD-ROM para Windows que se podr&aacute; instalar autom&aacute;ticamente. Para ello es indispensable que el paquete "Exportar a un CD-ROM" est&eacute; instalado. Esto no se incluye por omisi&oacute;n, por lo que ser&aacute; necesario modificar la instalaci&oacute;n para incluirlo.
+      <p></p>
+      Para exportar una colecci&oacute;n abra el men&uacute; "Archivo" y seleccione la opci&oacute;n "Exportar a un CD-ROM". Con esto aparecer&aacute; una lista de colecciones Greenstone. Haga click en una para ver su descripci&oacute;n. Marque los recuadros de las colecciones que desea exportar. Usted tambi&eacute;n puede ingresar el nombre de CD-ROM en el cuadro correspondiente, el cual aparecer&aacute; en el men&uacute; de Inicio una vez que el CD-ROM se haya instalado. A continuaci&oacute;n haga click en "Exportar". El proceso incluye la copia de varios archivos y requerir&aacute; de unos cuantos minutos.
+      <p></p>
+      Una vez que haya terminado Greenstone mostrar&aacute; el nombre de una carpeta en la que se encontrar&aacute;n las colecciones que se han exportado. Utilice una quemadora de CDs para copiar sus contenidos en un CD-ROM en blanco.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/fileassociations.htm
===================================================================
--- /main/tags/3.01/gli/help/es/fileassociations.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/fileassociations.htm	(revision 21112)
@@ -0,0 +1,61 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="fileassociations"><font size="5" face="Verdana"><strong>8.2: Asociaci&oacute;n de archivos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      La Interfaz de Bibliotecario utiliza diferentes programas de aplicaci&oacute;n para abrir tipos particulares de archivos. Esta secci&oacute;n explica la forma de asignar y editar estas asociaciones.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Establecer una asociaci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Editar una asociaci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar una asociaci&oacute;n</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para modificar la asociaci&oacute;n de archivos abra el men&uacute; "Archivo" y haga click en "Asociaci&oacute;n de archivos".
+      <p></p>
+      Para establecer una asociaci&oacute;n seleccione la extensi&oacute;n del archivo de la lista descendente o escriba una nueva extensi&oacute;n (no incluya "."). A continuaci&oacute;n escriba en el campo correspondiente el comando que da inicio a la aplicaci&oacute;n deseada o elija la aplicaci&oacute;n en el cuadro "Explorar". "%1" se puede utilizar en el comando de inicio para insertar el nombre del archivo que se est&aacute; abriendo. Una vez que se han llenado estos campos, el bot&oacute;n "Establecer asociaci&oacute;n" se activar&aacute;, con lo que ya se puede hacer click en &eacute;l para establecer la asociaci&oacute;n.
+      <p></p>
+      Para editar una asociaci&oacute;n, seleccione una extensi&oacute;n de archivo existente. El comando asociado aparecer&aacute; en el campo correspondiente a comando de inicio. Ed&iacute;telo y a continuaci&oacute;n haga click en "Reemplazar asociaci&oacute;n".
+      <p></p>
+      Para eliminar una asociaci&oacute;n, seleccione la extensi&oacute;n del archivo y haga click en "Eliminar asociaci&oacute;n". La extensi&oacute;n del archivo permanecer&aacute; en una lista llamada "Para archivos que terminan en...".
+      <p></p>
+      Las asociaciones de archivos se guardan en la carpeta principal de la Interfaz de Bibliotecario en un archivo llamado "associations.xml".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/filteringthetree.htm
===================================================================
--- /main/tags/3.01/gli/help/es/filteringthetree.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/filteringthetree.htm	(revision 21112)
@@ -0,0 +1,58 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="filteringthetree"><font size="5" face="Verdana"><strong>4.5: Filtrando la estructura de directorios y subdirectorios</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      "Filtrar" la estructura de directorios y subdirectorios de la colecci&oacute;n limita la b&uacute;squeda a determinados tipos de archivos.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">El control de filtros</td>
+</tr>
+	
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Filtrado especial</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      El men&uacute; "Mostrar archivos" debajo de cada estructura de directorios y subdirectorios muestra una lista de filtros definidos previamente, como por ejemplo el de "Im&aacute;genes". Al elegir uno se ocultan temporalmente todos los dem&aacute;s archivos. Para restaurarlos, cambie el filtro a "Todos los archivos". Estas operaciones no alteran la colecci&oacute;n ni afectan las carpetas en la estructura de archivos.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Se puede especificar un filtro en particular escribiendo un patr&oacute;n que corresponda con los tipos de archivos que se desea filtrar. Utilice las abreviaturas est&aacute;ndar del sistema como "*.*" o "*.doc" ("*" equivale a cualquier caracter).
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/formatstatements.htm
===================================================================
--- /main/tags/3.01/gli/help/es/formatstatements.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/formatstatements.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="formatstatements"><font size="5" face="Verdana"><strong>6.9: Formatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Los comandos de formato controlan la estructura y apariencia de la colecci&oacute;n. Afectan aspectos como el lugar donde aparecer&aacute;n los botones cuando se muestra un documento y qu&eacute; v&iacute;nculos despliega el clasificador DateList en pantalla. Los comandos de formato no son f&aacute;ciles de desarrollar, por lo que le recomendamos que lea el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone. La presente secci&oacute;n describe los controles de los distintos formatos y la forma en que la Interfaz de Bibliotecario le da acceso a ellos. En la parte de "Secciones de dise&ntilde;o" haga click en "Formatos".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Formateo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir un comando de formato</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar un comando de formato</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Actualizar un comando de formato</td>
+</tr>
+        
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Opciones avanzadas</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Se puede aplicar un comando de formato a cualquier cosa que se encuentre en la lista llamada "Elegir funci&oacute;n", la cual incluye cada clasificador y una lista de funciones determinadas previamente. Cuando se selecciona una funci&oacute;n existen dos tipos de controles. Algunas funciones simplemente se activan o se desactivan por medio de un recuadro. Otras requieren que se especifique una cadena de formato, para las cuales hay una lista descendente (llamada "Componente afectado") que permite seleccionar la parte de la funci&oacute;n a la que se aplica la cadena (en caso necesario), un &aacute;rea de texto (llamada "cadena de formato HTML") para ingresar la cadena y una selecci&oacute;n de "variables" definidas previamente. Para insertar una variable en su posici&oacute;n actual dentro de la cadena de formato, selecci&oacute;nela de la lista descendente y haga click en "Insertar".
+      <p></p>
+      Tambi&eacute;n se puede especificar un formato predeterminado para un componente en particular seleccionando la funci&oacute;n que est&aacute; en blanco. Este formato se aplicar&aacute; a todas las funciones aplicables a menos que se especifique lo contrario.
+      <p></p>
+      Para a&ntilde;adir un nuevo formato de comando llene la informaci&oacute;n tal como se describe arriba y haga click en "A&ntilde;adir formato". El nuevo comando de formato aparecer&aacute; en la lista de "Comandos de formato asignados actualmente". Solamente se podr&aacute; asignar un comando de formato a cada combinaci&oacute;n de funci&oacute;n/componente.
+      <p></p>
+      Para quitar un comando de formato de la lista, selecci&oacute;nelo y haga click en "Eliminar formato".
+      <p></p>
+      Para cambiar un comando de formato, selecci&oacute;nelo de la lista, modifique sus par&aacute;metros y haga click en "Reemplazar formato".
+      <p></p>
+      Para mayor informaci&oacute;n acerca de las variables y los componentes de las funciones lea el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Si se marca el recuadro "Permitir opciones avanzadas" se activar&aacute;n algunas opciones de formateo avanzado. La lista de funciones que se pueden formatear cambia ligeramente y m&aacute;s variables est&aacute;n disponibles para usarlas en los comandos de formato, proporcionando as&iacute; un mayor control sobre el dise&ntilde;o de la p&aacute;gina.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/generalsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/es/generalsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/generalsettings.htm	(revision 21112)
@@ -0,0 +1,49 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="generalsettings"><font size="5" face="Verdana"><strong>6.2: Ajustes generales</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de revisar y modificar los ajustes generales asociados a una colecci&oacute;n. Primero que nada, en la parte de "Secciones de dise&ntilde;o" haga click en "General".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Ajustes generales</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Aqu&iacute; pueden modificarse los valores proporcionados durante la fase de creaci&oacute;n de la colecci&oacute;n.
+      <p></p>
+      En la parte superior de la p&aacute;gina hay un cuadro de instrucciones que aparece con cada una de las secciones, el cual contiene una breve lista de indicaciones que le recuerdan cu&aacute;les son las funciones disponibles.
+      <p></p>
+      Primero se encuentran las direcciones de correo electr&oacute;nico del creador y del administrador de la colecci&oacute;n. El siguiente campo le permite cambiar el t&iacute;tulo de la colecci&oacute;n. El directorio en el que se almacena la colecci&oacute;n se muestra al lado, pero no puede editarse. El que sigue especifica (en forma de URL) el icono que aparecer&aacute; en el &aacute;ngulo superior izquierdo de la p&aacute;gina "Acerca de" y el siguiente es el icono usado en la p&aacute;gina de la biblioteca digital Greenstone para conectarse a la colecci&oacute;n. Entonces, una casilla controla que la colecci&oacute;n sea o no accesible p&uacute;blicamente. Finalmente viene el &aacute;rea de texto que permite dar una "Descripci&oacute;n de la Colecci&oacute;n", tal como se explica en la secci&oacute;n <a href="creatingacollection.htm#creatingacollection">Creando una nueva colecci&oacute;n</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/help.xml
===================================================================
--- /main/tags/3.01/gli/help/es/help.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/es/help.xml	(revision 21112)
@@ -0,0 +1,820 @@
+<Document>
+  <Section name="introduction">
+    <Title>IntroducciÃ³n</Title>
+    La Interfaz de Bibliotecario de Greenstone es una herramienta que permite reunir y marcar documentos para luego crear colecciones digitales con ellos, dÃ¡ndole fÃ¡cil acceso a todas las funciones de los programas de software de la Biblioteca Digital Greenstone desde una interfaz grÃ¡fica de usuario con la que sÃ³lo tiene que apuntar y hacer click.
+    <Section name="ofmiceandmenus">
+      <Title>De menÃºs y ratones</Title>
+      Esta secciÃ³n proporciona informaciÃ³n bÃ¡sica para poder trabajar con la Interfaz de Bibliotecario. Si estÃ¡ familiarizado con programas como Internet Explorer o Microsoft Office y estÃ¡ acostumbrado a usar el mouse, entonces pase a la <Reference target="howtoavoidthisdocument">siguiente secciÃ³n</Reference>.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Conceptos bÃ¡sicos
+            <ContentsItem>Movimientos del mouse</ContentsItem>
+            <ContentsItem>Teclado</ContentsItem>
+          </ContentsItem>
+          <ContentsItem>Salir del programa</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>Teclas de mÃ©todo abreviado</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La Interfaz de Bibliotecario sigue las convenciones de Microsoft y se apoya en el conocimiento que comÃºnmente se tiene de Windows.
+      <Break/>
+      Cualquier parte de la pantalla con la que usted interactÃºe, como un botÃ³n o un campo de texto, se llama "control". En todo momento un botÃ³n o campo de texto llamado "control activo" aparecerÃ¡ resaltado y responderÃ¡ a las instrucciones del teclado. Asimismo, varios controles le permiten seleccionar las partes que estÃ©n resaltadas en azul marino. Algunos controles tambiÃ©n pueden aparecer sombreados para indicar que no estÃ¡n disponibles.
+      <Break/>
+      Usted puede mover y hacer click con el botÃ³n izquierdo o derecho del mouse de la forma acostumbrada. Muchos componentes tambiÃ©n se pueden "arrastrar" haciendo click en ellos y sosteniendo el botÃ³n izquierdo del mouse, para luego moverlos y "colocarlos" en algÃºn otro lugar soltando el botÃ³n. Normalmente el lugar de destino cambia de apariencia cuando un componente pasa sobre Ã©l.
+      <Break/>
+      Usted puede usar el teclado para escribir en los campos de texto. Muchos controles tambiÃ©n se pueden ejecutar por medio del teclado, lo que se indica por medio de corchetes -- por ejemplo la tecla de tabulaciÃ³n [Tab] cambia el control activo. El signo de mÃ¡s (+) indica que otras teclas deben oprimirse al mismo tiempo.
+      <Break/>
+      Salga de la Interfaz de Bibliotecario eligiendo "Salir" en el menÃº "Archivo". Su colecciÃ³n se guardarÃ¡ primero.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Para tener acceso a un menÃº, oprima el botÃ³n [ALT] y presione la letra correspondiente (la cual aparece subrayada). Por ejemplo, para desplegar el menÃº "Archivo" oprima [ALT] + [A]. Para elegir un comando presione la tecla correspondiente. Por ejemplo, en el menÃº "Archivo" presione [G] para "Guardar" la colecciÃ³n.
+    </Section>
+    <Section name="howtoavoidthisdocument">
+      <Title>Temas de ayuda</Title>
+      Â¡Usted no necesita leer todo el texto de ayuda! SÃ³lo lea lo suficiente para saber cÃ³mo obtener ayuda cuando la necesite.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Ayuda sensible al contexto</ContentsItem>
+	  <ContentsItem>Cuadros de sugerencias</ContentsItem>
+	  <ContentsItem>Otros documentos</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      El comando del menÃº "Ayuda" indica los que serÃ­an los temas de ayuda mÃ¡s apropiados por medio de un pequeÃ±o icono en forma de libro.
+      <Break/>
+      Para muchos controles si se coloca el mouse encima de ellos aparecerÃ¡ un "cuadro de sugerencias" en el cual se explica su funciÃ³n.
+      <Break/>
+      Antes de usar la Interfaz de Bibliotecario lea primero la documentaciÃ³n de Greenstone.
+    </Section>
+  </Section>
+
+  <Section name="startingoff">
+    <Title>Para empezar</Title>
+    Esta secciÃ³n explica cÃ³mo crear, cargar, guardar y borrar colecciones.
+    <Section name="creatingacollection">
+      <Title>Creando una nueva colecciÃ³n</Title>
+      <ObviousSentence> Esta secciÃ³n describe la forma de crear una nueva colecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Aviso de la nueva colecciÃ³n</ContentsItem>
+	  <ContentsItem>TÃ­tulo de la colecciÃ³n</ContentsItem>
+	  <ContentsItem>Conjuntos de metadatos por omisiÃ³n</ContentsItem>
+	  <ContentsItem>DescripciÃ³n de la colecciÃ³n</ContentsItem>
+	  <ContentsItem>Crear una nueva colecciÃ³n</ContentsItem>
+	  <ContentsItem>Cancelar la nueva colecciÃ³n</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+	  <ContentsItem>Teclas de mÃ©todo abreviado</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para crear una nueva colecciÃ³n abra el menÃº "Archivo" y elija la opciÃ³n "Nueva". DeberÃ¡n llenarse varios campos, aunque en caso de que lo requiera usted podrÃ¡ cambiar despuÃ©s sus valores en la pantalla de diseÃ±o.
+      <Break/>
+      El "TÃ­tulo de la colecciÃ³n" es el texto que aparece en la parte superior de la pÃ¡gina principal de su colecciÃ³n. Puede tener la longitud  que usted quiera.
+      <Break/>
+      La "DescripciÃ³n del contenido" deberÃ¡ describir tan detalladamente como sea posible de lo que se trata la colecciÃ³n. Utilice la tecla [Enter] o [Intro] para dividirla en pÃ¡rrafos.
+      <Break/>
+      Finalmente deberÃ¡ especificar si desea que la nueva colecciÃ³n tenga la misma apariencia y conjunto de metadatos que una colecciÃ³n existente o si desea iniciar una "Nueva ColecciÃ³n" por omisiÃ³n.
+      <Break/>
+      Haga click en "Aceptar" para crear la colecciÃ³n. Si usted escogiÃ³ "Nueva ColecciÃ³n" se le pedirÃ¡ que elija el conjunto de metadatos que usarÃ¡ en ella.  Puede elegir mÃ¡s de uno y tambiÃ©n puede aÃ±adir otros mÃ¡s adelante.
+      <Break/>
+      Si hace click en "Cancelar" regresarÃ¡ inmediatamente a la pantalla principal.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Los botones, al igual que los menÃºs, tienen una letra subrayada. Para seleccionar un botÃ³n oprima la tecla [ALT] y la letra que estÃ¡ subrayada al mismo tiempo.
+    </Section>
+    <Section name="savingacollection">
+      <Title>Guardar la colecciÃ³n</Title>
+      <ObviousSentence> Esta secciÃ³n describe la forma de guardar una colecciÃ³n y el mensaje Guardar ColecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>CÃ³mo guardar</ContentsItem>
+	</ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>DÃ³nde se guardan los archivos</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Guarde su trabajo regularmente abriendo el menÃº "Archivo" y seleccionando "Guardar". Tome en cuenta que guardar una colecciÃ³n no es lo mismo que prepararla para que se pueda usar en Greenstone (vea la secciÃ³n <Reference target="producingthecollection">Produciendo una colecciÃ³n</Reference>).
+      <Break/>
+      La Interfaz de Bibliotecario protege su trabajo guardÃ¡ndolo cada vez que salga del programa o cargue otra colecciÃ³n.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Las colecciones se guardan en un archivo que lleva el nombre de la colecciÃ³n y la extensiÃ³n ".col", ubicado en una carpeta del mismo nombre dentro de la carpeta "colecciones" del programa Greenstone.
+    </Section>
+    <Section name="openingacollection">
+      <Title>Abriendo una colecciÃ³n existente</Title>
+      <ObviousSentence>Esta secciÃ³n le dice la forma de abrir una colecciÃ³n existente por medio del mensaje Abrir ColecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>CÃ³mo abrir una colecciÃ³n</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+	  <ContentsItem>Colecciones protegidas</ContentsItem>
+	  <ContentsItem>Colecciones heredadas</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para abrir una colecciÃ³n existente elija la opciÃ³n "Abrir" en el menÃº "Archivo". Con esto aparecerÃ¡ una lista de sus colecciones Greenstone. Seleccione una para ver su descripciÃ³n y haga click en "Abrir" para cargarla. Si busca una colecciÃ³n que resida fuera de la carpeta de "colecciones" de Greenstone, entonces haga click en "Explorar" para que aparezca un cuadro de diÃ¡logo que le permita buscar en el sistema de archivos.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      En caso de que mÃ¡s de una Interfaz de Bibliotecario de Greenstone estÃ© corriendo al mismo tiempo, los directorios pertinentes se "bloquean" para evitar interferencias. Al abrir una colecciÃ³n se crea un pequeÃ±o archivo de bloqueo temporal en la carpeta. Antes de abrir una colecciÃ³n la Interfaz de Bibliotecario revisa que no haya un archivo de bloqueo en ella. Se puede saber si la colecciÃ³n estÃ¡ bloqueada por el color de su icono: verde para una colecciÃ³n normal, rojo para una colecciÃ³n bloqueada. Sin embargo, cuando se sale prematuramente de la Interfaz de Bibliotecario el archivo de bloqueo a veces permanece en la carpeta. Cuando posteriormente se abre esta colecciÃ³n, la Biblioteca Digital le pregunta si quiere tomar el control. Por favor, nunca le quite el control de una colecciÃ³n a alguien que este trabajando en ese momento con ella.
+      <Break/>
+      Cuando usted abra una colecciÃ³n que no haya sido creada por medio de la Interfaz de Bibliotecario de Greenstone se le pedirÃ¡ que seleccione uno o mÃ¡s conjuntos de metadatos. Si no selecciona ninguno, entonces los metadatos existentes se ignorarÃ¡n. De otra manera los metadatos se importarÃ¡n tal como son cuando usted arrastre los archivos. El proceso se describe en el documento llamado <Reference target="importingpreviouslyassignedmetadata">Importando Metadatos Previamente Asignados</Reference>.
+    </Section>
+    <Section name="deletingcollections">
+      <Title>Borrando colecciones</Title>
+      <ObviousSentence>Esta secciÃ³n explica como borrar definitivamente colecciones de su instalaciÃ³n de Greenstone.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            CÃ³mo borrar colecciones
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para borrar definitivamente colecciones de su instalaciÃ³n de Greenstone, elija "BorrarâŠ" del menÃº "Archivo". Aparece una lista de sus colecciones. Seleccione una para ver su descripciÃ³n, marque la casilla situada en la parte inferior del diÃ¡logo, y pulse "Borrar" para borrar la colecciÃ³n. Esta acciÃ³n es irreversible, por lo que debe comprobar cuidadosamente que no necesita la colecciÃ³n antes de proceder.
+    </Section>
+  </Section>
+
+  <Section name="downloadingfiles">
+    <Title>Descargando archivos de Internet</Title>
+    La vista "Bajar" le ayuda a descargar recursos de internet. Esta secciÃ³n explica el interfaz del mecanismo de puesta en espejo del Bibliotecario.
+    <Section name="themirrorview">
+      <Title>La vista en Bajar</Title>
+      Esta secciÃ³n describe como configurar una tarea de descarga y como controlar el proceso de descargue.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>El Ã¡rbol del Ã¡rea de trabajo</ContentsItem>
+	  <ContentsItem>ConfiguraciÃ³n de descarga</ContentsItem>
+	  <ContentsItem>Lista de descarga</ContentsItem>
+        </ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>ActivaciÃ³n de "web proxies"</ContentsItem>
+	  <ContentsItem>Modo avanzado</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Active la vista "Bajar" haciendo clic en su pestaÃ±a. La mitad superior de la pantalla contiene los controles para realizar descargas. La mitad inferior estÃ¡ vacÃ­a inicialmente, pero despuÃ©s va a mostrar una lista de tareas pendientes y terminadas.
+      <Break/>
+      Los archivos se descargan en una carpeta del espacio de trabajo denominada "Archivos Descargados" (que sÃ³lo existe cuando estÃ¡ habilitada la opciÃ³n de trabajar en espejo), y pueden utilizarse en todas las colecciones construidas con el interfaz del Bibliotecario. En esta Ã¡rea los archivos se designan con su URL completo. Para cada "host" se crea una nueva carpeta, seguida por otras para cada parte del "path". De esta manera se asegura que todos los archivos son distintos.
+      <Break/>
+      Utilice el primero de los controles de configuraciÃ³n de la descarga, "URL OrÃ­gen", para capturar el URL de un recurso deseado. Utilice el control "Profundidad de la Descarga" para limitar el nÃºmero de hiper-vÃ­nculos que se deben seguir al descargar: PÃ³ngalo en 0 para descargar una sola pÃ¡gina; pÃ³ngalo en 1 para descargar una pÃ¡gina y todas las pÃ¡ginas a las que apunta. El lÃ­mite de profundidad se ignora al descargar medios diferentes de pÃ¡ginas html. AdemÃ¡s, hay varios controles de chequeo que se pueden prender para activar determinadas capacidades durante una descarga especÃ­fica. Una vez se ha establecido la configuraciÃ³n, haga clic en "Descargar" para iniciar un nuevo proceso de descargue. Hay otros dos controles en forma de botones: "Preferencias", que enlaza con la secciÃ³n de conexiÃ³n de las Preferencias, en la cual se pueden editar las caracterÃ­sticas del "proxy"; y "Borrar CachÃ©", que borra todos los archivos descargados previamente.
+      <Break/>
+      La lista de descargas tiene una entrada por cada pÃ¡gina web descargada. Cada entrada tiene un Ã¡rea de texto que da detalles acerca de la tarea, junto con una barra que indica el avance de la actividad en ejecuciÃ³n. A la izquierda de cada entrada aparecen tres botones. "Pausa" se utiliza para detener temporalmente la descarga que se estÃ¡ ejecutando. "Ver BitÃ¡cora" abre una ventana que muestra el archivo de bitÃ¡cora de las descargas. "Cerrar" termina la descarga y quita de la lista la tarea correspondiente.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      La <Reference target="preferences">secciÃ³n Preferencias</Reference> describe como establecer una conexiÃ³n Internet por medio de un "proxy".  Si se necesita autenticaciÃ³n, el servidor "proxy" pide la identificaciÃ³n y la palabra clave. El interfaz del Bibliotecario no almacena palabras claves entre sesiones.
+    </Section>
+  </Section>
+
+  <Section name="collectingfiles">
+    <Title>Reuniendo archivos para su colecciÃ³n</Title>
+    Una vez que usted tiene una nueva colecciÃ³n necesita meter algunos archivos en ella. Estos pueden venir desde la estructura de archivos de su computadora o de otras colecciones Greenstone. Algunos incluso ya pueden tener metadatos. Esta secciÃ³n describe la forma de importar archivos.
+    <Section name="thegatherview">
+      <Title>La pantalla de selecciÃ³n de documentos</Title>
+      Esta secciÃ³n describe el funcionamiento de la pantalla "Reunir" que se utiliza para seleccionar los archivos que se van a incluir en la colecciÃ³n que se estÃ¡ construyendo.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>CÃ³mo ver la pantalla Reunir</ContentsItem>
+	  <ContentsItem>La estructura de directorios y subdirectorios</ContentsItem>
+	  <ContentsItem>El Ã¡rea de estado</ContentsItem>
+	  <ContentsItem>Botones de control</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+	  <ContentsItem>MÃ©todos de selecciÃ³n</ContentsItem>
+	  <ContentsItem>Mapeado de carpetas especiales</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La Interfaz de Bibliotecario inicia con la pantalla "Reunir". Si despuÃ©s usted desea regresar a esta pantalla haga click en la pestaÃ±a del mismo nombre que se encuentra directamente debajo de la barra de menÃºs.
+      <Break/>
+      Dos grandes Ã¡reas tituladas "Area de trabajo" y "ColecciÃ³n" se utilizan para transferir los archivos a su colecciÃ³n. Cada una contiene "Ã¡rboles de archivos", los cuales son representaciones grÃ¡ficas de la estructura de directorios y subdirectorios de sus archivos y carpetas.
+      <Break/>
+      Seleccione un elemento del Ã¡rbol haciendo click en Ã©l. (TambiÃ©n existen otras formas; vÃ©ase mÃ¡s adelante.) Haga click en el signo que se encuentra junto a una carpeta o doble click en ella para expandir (o contraer) sus contenidos. Haga doble click en un archivo para abrirlo usando un programa de aplicaciÃ³n asociado (vea la secciÃ³n <Reference target="fileassociations">AsociaciÃ³n de archivos</Reference>).
+      <Break/>
+      El Ã¡rbol de archivos del Ã¡rea de trabajo muestra los orÃ­genes de la informaciÃ³n disponible al interfaz del Bibliotecario -- el sistema local de archivos (incluyendo unidades de disco y CD-ROM), los contenidos de las colecciones existentes de Greenstone, y el cachÃ© de archivos descargados si esta activada la opciÃ³n de poner en espejo la Web. Usted puede copiar y consultar estos archivos pero no puede moverlos, eliminarlos o editarlos, con la excepciÃ³n de los archivos descargados, los cuales pueden ser eliminados. Navegue por este espacio para buscar los archivos que desee incluir en la colecciÃ³n.
+      <Break/>
+      La estructura de directorios y subdirectorios de la colecciÃ³n muestra sus contenidos. Inicialmente estÃ¡ vacÃ­a.
+      <Break/>
+      Se puede cambiar el tamaÃ±o de las Ã¡reas oprimiendo el botÃ³n del mouse sobre la barra gris que separa las estructuras de directorios y subdirectorios (la forma del cursor cambia) y arrastrÃ¡ndola.
+      <Break/>
+      En la parte inferior de la ventana se muestra un Ã¡rea de estado que informa del progreso de las acciones sobre archivos (copiando, moviendo y borrando). Puede tardar un tiempo en terminar. El botÃ³n "Detener" interrumpe cualquier acciÃ³n que se estÃ© ejecutando en ese momento.
+      <Break/>
+      Dos grandes botones ocupan la esquina inferior derecha de la pantalla: "Nueva carpeta", con la imagen de una carpeta, crea nuevas carpetas (vea la secciÃ³n <Reference target="creatingfolders">Creando carpetas</Reference>), en tanto que "Eliminar", con la imagen de un bote de basura, descarta los archivos. Al hacer click en el botÃ³n "Eliminar" se suprimirÃ¡n los archivos seleccionados de la estructura de directorios y subdirectorios de la colecciÃ³n. Asimismo, los archivos pueden eliminarse arrastrÃ¡ndolos hacia este botÃ³n.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Para seleccionar varios elementos consecutivos, seleccione el primero y a continuaciÃ³n manteniendo oprimida la tecla de mayÃºsculas o [Shift] haga click en el Ãºltimo: la selecciÃ³n incluirÃ¡ todos los elementos que se encuentran en medio. Para seleccionar varios archivos que no sean consecutivos mantenga oprimida la tecla de control o [Ctrl] mientras hace click en ellos. Utilice alternativamente estos dos mÃ©todos para seleccionar grupos de elementos que no estÃ©n juntos.
+      <Break/>
+      En ocasiones algunas carpetas, como la que contiene sus propias pÃ¡ginas Web, tienen un significado especial. La Interfaz de Bibliotecario puede asignar dichas carpetas al primer nivel de la estructura de directorios y subdirectorios. Para hacerlo haga click con el botÃ³n derecho del mouse sobre la carpeta deseada. Seleccione "Crear un acceso directo" y dÃ©le un nombre a la carpeta. Para quitar un elemento haga click con el botÃ³n derecho del mouse sobre la carpeta mapeada y seleccione "Eliminar acceso directo".
+    </Section>
+    <Section name="creatingfolders">
+      <Title>Creando carpetas</Title>
+      <ObviousSentence>Esta secciÃ³n muestra la forma de crear nuevas carpetas.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>El botÃ³n de nueva carpeta</ContentsItem>
+	  <ContentsItem>El menÃº del botÃ³n derecho del mouse</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Utilice las carpetas que se encuentran en la estructura de directorios y subdirectorios de la colecciÃ³n para agrupar archivos y hacer que sean mÃ¡s fÃ¡ciles de encontrar. Las carpetas pueden contener a su vez otras subcarpetas. No hay prÃ¡cticamente ningÃºn lÃ­mite en cuanto al nÃºmero de carpetas que se pueden tener o quÃ© tan profundamente anidadas puedan estar.
+      <Break/>
+      Para crear una nueva carpeta, una opciÃ³n es seleccionar una carpeta existente en la estructura de directorios y subdirectorios de la colecciÃ³n y hacer click en el botÃ³n "Nueva carpeta". La nueva carpeta aparecerÃ¡ dentro de la que se seleccionÃ³ primero o en el nivel superior si no se seleccionÃ³ ninguna. Se le pedirÃ¡ que le dÃ© un nombre a la carpeta (el valor por omisiÃ³n es "Nueva carpeta").
+      <Break/>
+      TambiÃ©n se pueden crear nuevas carpetas haciendo click sobre una carpeta con el botÃ³n derecho del mouse,  eligiendo la opciÃ³n "Nueva carpeta" y continuando tal como se describe arriba.
+    </Section>
+    <Section name="addingfiles">
+      <Title>AÃ±adiendo archivos</Title>
+      <ObviousSentence>Esta secciÃ³n muestra la forma de aÃ±adir archivos a su colecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Arrastrando un archivo</ContentsItem>
+	  <ContentsItem>Varios archivos</ContentsItem>
+        </ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>Metadatos existentes</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Los archivos se pueden copiar arrastrÃ¡ndolos y colocÃ¡ndolos en una colecciÃ³n. El cursor se convierte en un fantasma del elemento seleccionado (o, si se ha seleccionado mÃ¡s de uno, del nÃºmero de ellos). Coloque la selecciÃ³n en la estructura de directorios y subdirectorios de la colecciÃ³n para copiar ahÃ­ los archivos (si la fuente son los archivos del Ã¡rea de trabajo) o muÃ©valos de lugar dentro de la misma colecciÃ³n (si la fuente son los archivos de la colecciÃ³n).
+      <Break/>
+      Cuando se copien varios archivos en una carpeta todos estarÃ¡n al mismo nivel, sin importar la estructura de las carpetas en las que se encontraban originalmente. Cuando se copie un segundo archivo con el mismo nombre en la misma carpeta se le preguntarÃ¡ si desea reemplazar el archivo existente. Si la respuesta es "No" entonces el archivo no se copiarÃ¡, pero los demÃ¡s sÃ­. Para cancelar las acciones restantes de copiado haga click en el botÃ³n "Detener".
+      <Break/>
+      Ãnicamente se moverÃ¡n los elementos "de mÃ¡s alto nivel" en una selecciÃ³n. Una carpeta estÃ¡ a un nivel mÃ¡s alto que los elementos que se encuentran en su interior, por lo que no se podrÃ¡n seleccionar los archivos de una carpeta y la carpeta al mismo tiempo.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Cuando se aÃ±ada un archivo, la Interfaz de Bibliotecario buscarÃ¡ a travÃ©s de las carpetas fuente los archivos auxiliares que contienen los metadatos asignados previamente a dicho archivo y, si encuentra alguno, empezarÃ¡ a importar estos metadatos. Conforme la operaciÃ³n siga adelante se le pedirÃ¡ (quizÃ¡ varias veces) que proporcione informaciÃ³n adicional para conciliar los metadatos importados con los conjuntos de metadatos de su colecciÃ³n. Este proceso incluye muchos avisos diferentes, los cuales se describen en la secciÃ³n <Reference target="importingpreviouslyassignedmetadata">Importando metadatos previamente asignados</Reference>. Para obtener una explicaciÃ³n mÃ¡s detallada sobre la forma de asociar metadatos a los archivos lea el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone: Obteniendo el mÃ¡ximo beneficio de sus documentos.
+    </Section>
+    <Section name="removingfiles">
+      <Title>Eliminando archivos</Title>
+      <ObviousSentence>Esta secciÃ³n describe la forma de eliminar archivos y carpetas de su colecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>El botÃ³n eliminar</ContentsItem>
+	  <ContentsItem>La tecla suprimir</ContentsItem>
+	  <ContentsItem>Arrastrarlos y colocarlos en el botÃ³n "Eliminar"</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Existen varios mÃ©todos para eliminar archivos y carpetas. Primero se deben indicar los elementos que se desea eliminar seleccionando los archivos y carpetas tal como se describe en la secciÃ³n de <Reference target="thegatherview">selecciÃ³n de documentos para la colecciÃ³n</Reference>.
+      <Break/>
+      Una vez que se han seleccionado los archivos, haga click en el botÃ³n "Eliminar" o presione la tecla [Del], [Supr] o [Suprimir], dependiendo del teclado de su computadora, o arrÃ¡strelos desde la colecciÃ³n hasta el botÃ³n "Eliminar" y suÃ©ltelos ahÃ­.
+    </Section>
+    <Section name="filteringthetree">
+      <Title>Filtrando la estructura de directorios y subdirectorios</Title>
+      "Filtrar" la estructura de directorios y subdirectorios de la colecciÃ³n limita la bÃºsqueda a determinados tipos de archivos.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>El control de filtros</ContentsItem>
+	</ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>Filtrado especial</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      El menÃº "Mostrar archivos" debajo de cada estructura de directorios y subdirectorios muestra una lista de filtros definidos previamente, como por ejemplo el de "ImÃ¡genes". Al elegir uno se ocultan temporalmente todos los demÃ¡s archivos. Para restaurarlos, cambie el filtro a "Todos los archivos". Estas operaciones no alteran la colecciÃ³n ni afectan las carpetas en la estructura de archivos.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Se puede especificar un filtro en particular escribiendo un patrÃ³n que corresponda con los tipos de archivos que se desea filtrar. Utilice las abreviaturas estÃ¡ndar del sistema como "*.*" o "*.doc" ("*" equivale a cualquier caracter).
+    </Section>
+  </Section>
+
+  <Section name="enrichingacollection">
+    <Title>Enriqueciendo la colecciÃ³n con metadatos</Title>
+    DespuÃ©s de haber juntado varios archivos en una colecciÃ³n, enriquÃ©zcalos con informaciÃ³n adicional. Esta secciÃ³n explica la forma en que se crean, editan, asignan y recuperan los metadatos, y la forma de usar fuentes externas de metadatos (vÃ©ase tambiÃ©n el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone: Obteniendo el mÃ¡ximo beneficio de sus documentos).
+    <Section name="theenrichview">
+      <Title>La pantalla de asignaciÃ³n de metadatos</Title>
+      <ObviousSentence>Esta secciÃ³n describe la forma de aÃ±adir y editar metadatos por medio de la pantalla "Enriquecer".</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>La estructura de directorios y subdirectorios de la colecciÃ³n</ContentsItem>
+	  <ContentsItem>Controles de ediciÃ³n de metadatos</ContentsItem>
+	  <ContentsItem>La estructura de valores</ContentsItem>
+	  <ContentsItem>La tabla de metadatos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Utilice la pantalla "Enriquecer" para asignar metadatos a los documentos de la colecciÃ³n. Los metadatos son datos acerca de los datos: tÃ­tulo, autor, fecha de creaciÃ³n, etc. Cada metadato tiene dos partes: el "elemento" dice el tipo de metadato que es (como por ejemplo el autor) y el "valor" da la informaciÃ³n de ese elemento (como por ejemplo el nombre del autor).
+      <Break/>
+      En la parte izquierda de la pantalla "Enriquecer" se encuentra la estructura de directorios y subdirectorios de la colecciÃ³n. A la derecha estÃ¡ la Tabla de Metadatos, la cual muestra los metadatos de los archivos y carpetas seleccionados. Los nombres de las columnas estÃ¡n en gris en la parte superior y se pueden cambiar de tamaÃ±o arrastrando la lÃ­nea que las separa. Pulsar sobre cualquier lÃ­nea transfiere sus detalles a los controles inferiores de ediciÃ³n de metadatos. Si se seleccionan varios archivos, el texto en negro indica que el valor es comÃºn a todos ellos, mientras que el texto sombreado indica que no. Los valores en negro se pueden actualizar o eliminar, mientras que los que estÃ¡n sombreados se pueden eliminar de los que los tienen o agregarse a los demÃ¡s.
+      <Break/>
+      Un icono en forma de carpeta puede aparecer junto a algunas entradas de metadatos. Esto indica que los valores se heredan de la carpeta en la que se encuentra el archivo. Los metadatos heredados no se pueden editar o eliminar, sino Ãºnicamente agregarse o sobreescribirse. Haga click en el icono de la carpeta para ir inmediatamente a la carpeta en la que se asignaron los metadatos.
+      <Break/>
+      Los controles de ediciÃ³n de metadatos de la parte inferior derecha aparecen sÃ³lo cuando se selecciona un archivo de la estructura y una lÃ­nea de la tabla. UtilÃ­celos para actualizar, agregar o eliminar valores de los metadatos. El campo de valores es para ingresar o editar el valor del metadato.  Junto hay un botÃ³n con la etiqueta "...", el cual abre un cuadro de ediciÃ³n mÃ¡s grande. De los botones que se encuentran debajo, "Agregar" asigna el valor en forma de nuevo metadato y lo aÃ±ade a los valores existentes para el elemento seleccionado, "Reemplazar" sobreescribe el valor seleccionado sustituyÃ©ndolo por uno nuevo y "Eliminar" borra el valor seleccionado. MÃ¡s abajo, con la etiqueta "Todos los valores anteriores", se encuentra la estructura de valores.
+      <Break/>
+      La estructura de valores se puede expandir y contraer. Normalmente es una lista que muestra todos los valores ingresados previamente para un elemento seleccionado. Al hacer click en una entrada Ã©sta aparece automÃ¡ticamente en el campo de valores. Y a la inversa, al escribir en el campo se selecciona la entrada de la estructura de valores que empieza con los caracteres ingresados. Al oprimir [Tab] se completa automÃ¡ticamente el valor seleccionado.
+      <Break/>
+      Los valores de los metadatos se pueden organizar jerÃ¡rquicamente. Esto se muestra en la estructura de valores por medio de carpetas para los niveles internos. Los valores jerÃ¡rquicos pueden ingresarse utilizando el carÃ¡cter "|" para separar los niveles. Por ejemplo, se puede usar "Cartas|Rojo|Diamantes|Siete" en una jerarquÃ­a que representa un mazo de cartas. Esto permite agrupar los valores, en tanto que los grupos tambiÃ©n pueden asignarse en forma de metadatos a los archivos.
+      <Break/>
+      Greenstone extrae automÃ¡ticamente los metadatos de los documentos para formar un conjunto de metadatos cuyos elementos llevan el prefijo "ex.". Ãste no tiene estructura de valores y no puede editarse, por lo que los controles de ediciÃ³n estÃ¡n ocultos si se selecciona dicha entrada. El botÃ³n "..." todavÃ­a sirve para expandir el valor, pero el texto no se puede editar.
+    </Section>
+    <Section name="selectingmetadatasets">
+      <Title>Seleccionando conjuntos de metadatos</Title>
+      <ObviousSentence>Esta secciÃ³n explica la forma de especificar los elementos de los metadatos y cÃ³mo aÃ±adir conjuntos de metadatos a su colecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Â¿QuÃ© es un conjunto de metadatos?</ContentsItem>
+	  <ContentsItem>Controlando los conjuntos de metadatos usados</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Los grupos de elementos de metadatos definidos previamente son conocidos como "conjuntos de metadatos". Un ejemplo es el conjunto de metadatos de Dublin Core. Cuando se aÃ±ade un conjunto de metadatos a una colecciÃ³n, sus elementos se vuelven elegibles para que el usuario los pueda seleccionar. TambiÃ©n se puede tener mÃ¡s de un conjunto. Para evitar conflictos de nombre, un identificador que describe al conjunto de metadatos se agrega como prefijo al nombre del elemento. Por ejemplo, el elemento Creator de Dublin Core se convierte en "dc.Creator". Los conjuntos de metadatos se guardan en la carpeta de metadatos de la Interfaz de Bibliotecario y llevan el sufijo ".mds".
+      <Break/>
+      Para controlar los conjuntos de metadatos usados en una colecciÃ³n, utilice la opciÃ³n "Conjuntos de metadatos" en la vista DiseÃ±o.
+    </Section>
+    <Section name="appendingmetadata">
+      <Title>Agregando nuevos metadatos</Title>
+      <ObviousSentence>Esta secciÃ³n explica la forma de aÃ±adir metadatos y valores a los archivos y cÃ³mo darle nuevos valores a los elementos de los metadatos.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>AÃ±adiendo metadatos a los archivos</ContentsItem>
+	  <ContentsItem> AÃ±adiendo metadatos a carpetas o a varios archivos juntos</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>JerarquÃ­as y ediciÃ³n de valores</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Ahora aÃ±adiremos un Ã­tem de metadato -- tanto un elemento como un valor -- a un archivo. Primero, seleccione el archivo en la estructura de directorios y subdirectorios de la colecciÃ³n que se encuentra a la izquierda. Esto hace que cualquier metadato asignado previamente a este archivo aparezca en la tabla de la derecha.
+      <Break/>
+      A continuaciÃ³n, seleccione el elemento de metadato que desea aÃ±adir haciendo click en la fila correspondiente de la tabla.
+      <Break/>
+      Escriba el valor en el campo de valores. No use el carÃ¡cter "|", ya que Ã©ste se utiliza para construir jerarquÃ­as. Una vez que haya terminado haga click en "Agregar" para aÃ±adir el nuevo valor como metadato del archivo seleccionado. El valor aparecerÃ¡ inmediatamente en la tabla de metadatos.
+      <Break/>
+      TambiÃ©n se pueden aÃ±adir metadatos a una carpeta o a varios archivos seleccionados al mismo tiempo. Estos metadatos se aÃ±adirÃ¡ a todos los archivos que forman parte de la carpeta o de la selecciÃ³n, asÃ­ como a las subcarpetas. Recuerde que si usted asigna metadatos a una carpeta, todos los archivos nuevos que se encuentren en ella heredarÃ¡n automÃ¡ticamente los valores de la carpeta.
+      <Break/>
+      Si usted eligiÃ³ un metadato que ya aparece en algunos de los archivos seleccionados y hace click en "Agregar", el metadato se aÃ±adirÃ¡ a los demÃ¡s archivos de la selecciÃ³n.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Usted tambiÃ©n puede darle estructura a los valores de los metadatos por medio de rutas, tal como se describe en la secciÃ³n de <Reference target="theenrichview">AsignaciÃ³n de metadatos</Reference>.
+    </Section>
+    <Section name="addingpreviouslydefinedmetadata">
+      <Title>AÃ±adiendo Metadatos Previamente Definidos</Title>
+      <ObviousSentence>Esta secciÃ³n explica la forma de aÃ±adir metadatos con valores que ya estÃ©n presentes en la estructura de valores.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>AÃ±adiendo metadatos a archivos</ContentsItem>
+	  <ContentsItem> AÃ±adiendo metadatos a carpetas o a varios archivos juntos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para aÃ±adir metadatos con un valor existente, seleccione primero el archivo y a continuaciÃ³n el valor requerido de la estructura de valores, expandiendo las carpetas tanto como sea necesario. El valor de la entrada seleccionada aparecerÃ¡ automÃ¡ticamente en el campo de texto (otra alternativa es utilizar las funciones de selecciÃ³n automÃ¡tica y completamiento automÃ¡tico de la estructura de valores). Haga click en "Agregar" para aÃ±adir el metadato al archivo seleccionado.
+      <Break/>
+      El proceso de aÃ±adir metadatos con valores existentes a carpetas o a varios archivos juntos es exactamente igual.
+    </Section>
+    <Section name="updatingmetadata">
+      <Title>Actualizando metadatos</Title>
+      <ObviousSentence>Esta secciÃ³n explica la forma de actualizar los metadatos asignados a un archivo.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Actualizando los metadatos de archivos</ContentsItem>
+	  <ContentsItem> Actualizando los metadatos de carpetas o de varios archivos juntos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para actualizar el valor de una parte del metadato, elija primero el archivo al que se aplicarÃ¡ dicho valor y luego el elemento de metadato cuyo valor desea cambiar. Su selecciÃ³n aparecerÃ¡ en los controles de ediciÃ³n de metadatos. Edite el valor y haga click en "Reemplazar" para modificarlo.
+      <Break/>
+      El proceso es el mismo cuando se actualiza un directorio que contiene subdirectorios o mÃºltiples archivos, pero sÃ³lo se pueden actualizar metadatos que sean comunes a todos los archivos y carpetas seleccionados.
+      <Break/>
+      La estructura de valores muestra todos los valores anteriores y no sÃ³lo aquellos que se asignaron en este momento. AsÃ­ el valor que usted reemplazÃ³ permanecerÃ¡ en la estructura de valores.
+    </Section>
+    <Section name="removingmetadata">
+      <Title>Eliminando metadatos</Title>
+      <ObviousSentence>Esta secciÃ³n explica la forma de eliminar los metadatos de un archivo.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Eliminando metadatos de un archivo</ContentsItem>
+	  <ContentsItem>Eliminando metadatos de carpetas o de varios archivos juntos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Los metadatos se eliminan de la misma manera que se actualizan. Primero se selecciona un archivo de la estructura de directorios y subdirectorios, y se utiliza la tabla de metadatos para elegir el metadato que se desea eliminar. Si un metadato tiene asignado un valor, se activa el botÃ³n "Borrar". Haga click en Ã©l para quitar el metadato del archivo seleccionado. Los demÃ¡s archivos permanecerÃ¡n sin cambio y el valor se conservarÃ¡ en la estructura de valores.
+      <Break/>
+      Eliminar metadatos de directorios o de mÃºltiples archivos se hace de la misma forma.
+    </Section>
+    <Section name="reviewingmetadata">
+      <Title>Revisando los metadatos asignados</Title>
+      <ObviousSentence>Esta secciÃ³n describe la forma de ver todos los metadatos asignados a una colecciÃ³n completa.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>La tabla de auditorÃ­a</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+	  <ContentsItem> El menÃº del botÃ³n derecho del mouse</ContentsItem>
+	  <ContentsItem>Filtros</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      En ocasiones es necesario ver los metadatos asignados a varios o a todos los archivos al mismo tiempo, por ejemplo, para determinar la forma en que quedan muchos archivos para trabajar en ellos o para tener una idea de la diferencia de tiempo entre las fechas.
+      <Break/>
+      Seleccione los archivos que desea examinar, pulse el botÃ³n derecho del ratÃ³n y elija "Metadatos AsignadosâŠ". Con lo que aparecerÃ¡ una ventana llamada "Todos los metadatos", ocupada en su mayor parte por una tabla de varias columnas. La primera columna muestra los nombres de los archivos, mientras que las filas muestran todos los valores asignados a dichos archivos.
+      <Break/>
+      Si se seleccionaron muchos archivos es posible que transcurra algÃºn tiempo antes de que se abra la ventana. Mientras tanto usted puede seguir usando la Interfaz de Bibliotecario.
+      <Break/>
+      Haga click en "Cerrar" para ocultar la ventana.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      TambiÃ©n se puede ver la tabla de "Todos los metadatos" seleccionando los archivos que desea examinar, haciendo click con el botÃ³n derecho del mouse y eligiendo la opciÃ³n "Metadatos asignados...". Si se seleccionÃ³ una carpeta, todos sus archivos estarÃ¡n incluidos en la tabla.
+      <Break/>
+      Cuando la tabla sea demasiado grande se puede refinar aplicando los filtros a las columnas. Conforme se aÃ±adan nuevos filtros, Ãºnicamente aquellas filas que concuerden con ellos seguirÃ¡n siendo visibles. Para establecer, modificar o borrar un filtro haga click en el icono con forma de embudo que se encuentra en la parte superior de la columna. Se le indicarÃ¡ que proporcione informaciÃ³n sobre el filtro. Una vez que un filtro se ha aplicado, el encabezado de la columna cambia de color.
+      <Break/>
+      El aviso tiene una pestaÃ±a "Sencilla" y una "Avanzada". La versiÃ³n sencilla filtra las columnas de tal manera que sÃ³lo muestren las filas que contienen un determinado valor ("*" corresponde a todos los valores). Los valores de los metadatos se pueden seleccionar de la lista. La versiÃ³n avanzada permite diferentes operaciones de concordancia: "debe empezar con", "no contiene", "alfabÃ©ticamente menos que" y "es igual a". El valor se puede editar con cualquier cadena de caracteres (incluyendo "*") y se puede elegir que la concordancia no tome en cuenta el caso. Finalmente, se puede especificar una segunda condiciÃ³n de concordancia que se puede usar para especificar un rango de valores (seleccionando "Y") o valores alternativos (seleccionando "O"). Debajo de esta Ã¡rea hay un cuadro que le permite cambiar el orden de clasificaciÃ³n (ascendente o descendente). Una vez que haya terminado, haga click en "Filtrar" para aplicar el nuevo filtro a la columna. Para quitar el filtro haga click en "Borrar filtro". Recuerde que los detalles del filtro se conservan incluso cuando el filtro se borra.
+      <Break/>
+      Por ejemplo, para ordenar la tabla elija una columna, seleccione el filtro por omisiÃ³n (un filtro Sencillo en "*") y escoja el orden ascendente o descendente.
+    </Section>
+    <Section name="importingpreviouslyassignedmetadata">
+      <Title>Importando metadatos previamente asignados</Title>
+      Esta secciÃ³n describe cÃ³mo importar metadtos previamente asignados, es decir, metadatos asignados a los documentos antes de que fuesen aÃ±adidos a la colecciÃ³n.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Mientras se aÃ±aden archivos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Si se han asignado previamente metadatos en una forma reconocible por la Interfaz de Bibliotecario, por ejemplo, cuando se seleccionan documentos de una colecciÃ³n Greenstone existente, entonces Ã©stos se importarÃ¡n automÃ¡ticamente cuando usted aÃ±ada el archivo. Para poder hacer esto los metadatos deben mapearse hacia el conjunto de metadatos disponible en la colecciÃ³n.
+      <Break/>
+      La Interfaz de Bibliotecario enviarÃ¡ un mensaje solicitando la informaciÃ³n necesaria. El mensaje incluirÃ¡ breves instrucciones y a continuaciÃ³n mostrarÃ¡ el nombre del elemento de metadato que se estÃ¡ importando, tal como aparece en el archivo fuente. Este campo no se podrÃ¡ editar o cambiar. A continuaciÃ³n elija el conjunto de metadatos hacia el que el nuevo elemento deberÃ¡ mapearse y el correspondiente elemento de metadato en ese conjunto. El sistema selecciona automÃ¡ticamente la correspondencia mÃ¡s cercana en tÃ©rminos de conjunto y elemento para el nuevo metadato.
+      <Break/>
+      DespuÃ©s de revisar el mapeo, usted puede elegir "AÃ±adir" para aÃ±adir el nuevo elemento de metadato al conjunto de metadatos que se haya seleccionado. (Esto sÃ³lo es posible si no hay un elemento del mismo nombre dentro del conjunto seleccionado.) "Reemplazar" mapea el nuevo elemento hacia el que el usuario haya elegido. Finalmente, "Ignorar" no importa ningÃºn metadato con este nombre de elemento. Una vez que usted haya especificado la forma de importar una parte del metadato, la informaciÃ³n del proceso se conserva durante todo el tiempo de vida de la colecciÃ³n.
+      <Break/>
+      Para mayor informaciÃ³n sobre los archivos metadata.xml que Greenstone utiliza para guardar metadatos, vea el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone: Obteniendo el mÃ¡ximo beneficio de sus documentos.
+    </Section>
+  </Section>
+
+  <Section name="designingacollection">
+    <Title>DiseÃ±ando la apariencia de su colecciÃ³n</Title>
+    Una vez que los archivos estÃ¡n marcados con metadatos, a continuaciÃ³n se debe decidir la forma en que aparecerÃ¡n ante los usuarios como parte de una colecciÃ³n Greenstone. Â¿QuÃ© tipo de informaciÃ³n se podrÃ¡ buscar? Â¿De quÃ© medios se dispone para navegar por los documentos? Â¿En quÃ© idiomas? Â¿DÃ³nde aparecerÃ¡n los botones en la pÃ¡gina? Todas estas cosas se pueden personalizar. Esta secciÃ³n describe la forma de hacerlo.
+    <Section name="thedesignview">
+      <Title>La pantalla de diseÃ±o</Title>
+      Esta secciÃ³n le presenta la pantalla de DiseÃ±o y le explica la forma de navegar entre las diferentes opciones de este conjunto.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Eligiendo la secciÃ³n de diseÃ±o</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Con la Interfaz de Bibliotecario usted puede configurar la forma en que la colecciÃ³n aparecerÃ¡ frente al usuario. Las opciones se dividen en diferentes secciones, cada una relacionada con una etapa particular de navegaciÃ³n o presentaciÃ³n.
+      <Break/>
+      A la izquierda se encuentra una lista de las diferentes pantallas, en tanto que a la derecha estÃ¡n los controles asociados a la pantalla que estÃ¡ activa en ese momento. Para cambiar a una pantalla diferente haga click en la lista.
+      <Break/>
+      Para entender las etapas y los tÃ©rminos usados en el diseÃ±o de una colecciÃ³n, lea primero los CapÃ­tulos 1 y 2 de la GuÃ­a del Programador de Greenstone.
+    </Section>
+    <Section name="generalsettings">
+      <Title>Ajustes generales</Title>
+      Esta secciÃ³n explica la forma de revisar y modificar los ajustes generales asociados a una colecciÃ³n. Primero que nada, en la parte de "Secciones de diseÃ±o" haga click en "General".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Ajustes generales</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      AquÃ­ pueden modificarse los valores proporcionados durante la fase de creaciÃ³n de la colecciÃ³n.
+      <Break/>
+      En la parte superior de la pÃ¡gina hay un cuadro de instrucciones que aparece con cada una de las secciones, el cual contiene una breve lista de indicaciones que le recuerdan cuÃ¡les son las funciones disponibles.
+      <Break/>
+      Primero se encuentran las direcciones de correo electrÃ³nico del creador y del administrador de la colecciÃ³n. El siguiente campo le permite cambiar el tÃ­tulo de la colecciÃ³n. El directorio en el que se almacena la colecciÃ³n se muestra al lado, pero no puede editarse. El que sigue especifica (en forma de URL) el icono que aparecerÃ¡ en el Ã¡ngulo superior izquierdo de la pÃ¡gina "Acerca de" y el siguiente es el icono usado en la pÃ¡gina de la biblioteca digital Greenstone para conectarse a la colecciÃ³n. Entonces, una casilla controla que la colecciÃ³n sea o no accesible pÃºblicamente. Finalmente viene el Ã¡rea de texto que permite dar una "DescripciÃ³n de la ColecciÃ³n", tal como se explica en la secciÃ³n <Reference target="creatingacollection">Creando una nueva colecciÃ³n</Reference>.
+    </Section>
+    <Section name="plugins">
+      <Title>Plugins</Title>
+      Esta secciÃ³n describe la forma de configurar los plugins que la colecciÃ³n utiliza. Explica el modo de especificar los plugins que se utilizarÃ¡n, los parÃ¡metros que emplearÃ¡n y el orden en el que se aplicarÃ¡n. En la parte de "Secciones de diseÃ±o" haga click en "Plugins de documento".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>AÃ±adir un plugin</ContentsItem>
+	  <ContentsItem>Eliminar un plugin</ContentsItem>
+	  <ContentsItem>Configurar un plugin</ContentsItem>
+	  <ContentsItem>Cambiar el orden de los plugins</ContentsItem>
+        </ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>ConfiguraciÃ³n personalizada</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para aÃ±adir un plugin, escÃ³jalo de la lista "Seleccione el plugin que se va a aÃ±adir" que se encuentra en la parte inferior de la pantalla y haga click en "AÃ±adir plugin". AparecerÃ¡ una ventana titulada "Argumentos de configuraciÃ³n", la cual se explica mÃ¡s adelante. Una vez que usted haya configurado el nuevo plugin, este se aÃ±adirÃ¡ al final de la lista de "Plugins asignados actualmente". Note que, excepto para UnknownPlug, cada plugin sÃ³lo puede aparecer una vez en la lista.
+      <Break/>
+      Para quitar un plugin de la lista, selecciÃ³nelo y haga click en "Eliminar plugin".
+      <Break/>
+      Los plugins se configuran por medio de argumentos. Para modificarlos, seleccione un plugin de la lista y haga click en "Configurar plugin" (o doble click en el plugin). Aparece un diÃ¡logo "Configurando Argumentos" con varios controles para especificar argumentos.
+      <Break/>
+      Existen diferentes tipos de controles. Algunos son recuadros y al hacer click en ellos se aÃ±ade la opciÃ³n correspondiente al plugin. Otros son cadenas de texto con un recuadro y un campo de texto. Haga click en el cuadro para habilitarlo y escriba el argumento (expresiÃ³n regular, ruta de archivo, etc.). Otros son menÃºs descendentes desde los que se puede seleccionar un conjunto dado de valores. Para saber lo que hace un argumento, coloque el cursor del mouse encima de su nombre y la descripciÃ³n aparecerÃ¡.
+      <Break/>
+      Una vez que usted haya cambiado la configuraciÃ³n, haga click en "Aceptar" para guardar los cambios y cerrar el cuadro de diÃ¡logo o en "Cancelar" para cerrar el cuadro de diÃ¡logo sin cambiar ningÃºn argumento.
+      <Break/>
+      Los plugins de la lista se ejecutan en orden y en ocasiones Ã©ste es importante. Dos plugins, ArcPlug y RecPlug, son esenciales para el proceso de construcciÃ³n de la colecciÃ³n y se encuentran al final de la lista (separados del resto por una lÃ­nea). Para cambiar el orden de los demÃ¡s, seleccione el plugin que desea mover y haga click en "Mover arriba" o "Mover abajo".
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      La Interfaz de Bibliotecario hace lo posible por determinar los argumentos que un plugin soporta. Sin embargo, puede haber casos en los cuales el usuario quiera especificar argumentos especiales, y para esto se ofrece un campo de texto llamado "Argumentos personalizados" en la parte inferior. Cualquier texto que se escriba en Ã©l se agregarÃ¡ palabra por palabra al final del comando del plugin.
+    </Section>
+    <Section name="searchtypes">
+      <Title>Tipos de bÃºsqueda</Title>
+      Esta secciÃ³n explica la forma de modificar una nueva funciÃ³n de diseÃ±o de Greenstone llamada "Tipos de bÃºsqueda", que permite la bÃºsqueda por campos. En la parte de "Secciones de diseÃ±o" haga click en "Tipos de diseÃ±o".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Lo que significa activar las opciones de bÃºsqueda avanzada</ContentsItem>
+	  <ContentsItem>AÃ±adiendo un nuevo tipo de bÃºsqueda</ContentsItem>
+	  <ContentsItem>Eliminando un tipo de bÃºsqueda</ContentsItem>
+	  <ContentsItem>Cambiando el orden de los tipos de bÃºsqueda</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Cuando ingrese a la pantalla de tipos de bÃºsqueda marque primero el recuadro "Permitir bÃºsquedas avanzadas", lo cual activa todos los demÃ¡s controles. Esto transfiere la colecciÃ³n al formato Greenstone 2.4, el cual soporta la bÃºsqueda por campos y significa que (a) el diseÃ±o de Ã­ndices es diferente (lo cual se explica en la secciÃ³n <Reference target="searchindexes">Ãndices de bÃºsqueda</Reference>), (b) hay mÃ¡s fragmentos de texto para traducir (vea la secciÃ³n relativa a <Reference target="translatetext">TraducciÃ³n</Reference>) y (c) la colecciÃ³n no podrÃ¡ usarse con versiones anteriores de Greenstone. Si posteriormente usted quita la marca de este campo, la mayor parte de su colecciÃ³n regresarÃ¡ a Greenstone 2.39. No obstante, la Interfaz de Bibliotecario no podrÃ¡ convertir las especificaciones del nuevo Ã­ndice en las de las versiones anteriores, por lo que usted tendrÃ¡ que ingresarlas otra vez de forma manual.
+      <Break/>
+      Para aÃ±adir un tipo de bÃºsqueda, selecciÃ³nelo de la lista "Tipos de bÃºsqueda" y haga click en "AÃ±adir tipo de bÃºsqueda". Cada tipo puede aparecer en la lista solamente una vez.
+      <Break/>
+      Para quitar un tipo de bÃºsqueda, selecciÃ³nelo de la lista "Tipos de bÃºsqueda asignados actualmente" y haga click en "Eliminar tipo de bÃºsqueda". La lista deberÃ¡ contener al menos uno.
+      <Break/>
+      Para cambiar el orden de un tipo de bÃºsqueda, selecciÃ³nelo y haga click en "Mover arriba" o "Mover abajo". El primer tipo serÃ¡ el ajuste por omisiÃ³n.
+    </Section>
+    <Section name="searchindexes">
+      <Title>Ãndices de bÃºsqueda</Title>
+      Los Ã­ndices especifican las partes de la colecciÃ³n que se pueden buscar. Esta secciÃ³n explica la forma de aÃ±adir y eliminar Ã­ndices, asÃ­ como predeterminar un Ã­ndice por omisiÃ³n. En la parte correspondiente a "Secciones de diseÃ±o" haga click en "Ãndices de bÃºsqueda".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>AÃ±adir un Ã­ndice</ContentsItem>
+	  <ContentsItem>Editar un Ã­ndice</ContentsItem>
+	  <ContentsItem>Eliminar un Ã­ndice</ContentsItem>
+	  <ContentsItem>Predeterminar un Ã­ndice por omisiÃ³n</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para aÃ±adir un Ã­ndice escriba el nombre que le quiera dar en el campo "Nombre del Ã­ndice". Seleccione las fuentes de informaciÃ³n que se van a indexar haciendo click en los recuadros que se encuentran a un lado. La lista mostrarÃ¡ todos los elementos de metadatos asignados, asÃ­ como el texto completo. DespuÃ©s de seleccionar las fuentes de datos, elija la granularidad del Ã­ndice usando el menÃº de "Nivel". Una vez que se han completado estos detalles, la opciÃ³n "AÃ±adir Ã­ndice" se activa (a menos que haya un Ã­ndice con los mismos parÃ¡metros). Haga click en ella para aÃ±adir el nuevo Ã­ndice.
+      <Break/>
+      Para editar un Ã­ndice, selecciÃ³nelo, cambie los detalles del Ã­ndice, y pulse "Reemplazar Ãndice".
+      <Break/>
+      Para quitar un Ã­ndice de la lista de Ã­ndices asignados, selecciÃ³nelo y haga click en "Eliminar Ã­ndice".
+      <Break/>
+      El Ã­ndice predeterminado es el que se usa en la pÃ¡gina de bÃºsqueda de la colecciÃ³n y lleva la etiqueta "[Indice por omisiÃ³n]" en la lista de "Ãndices asignados". Para establecer un Ã­ndice por omisiÃ³n, selecciÃ³nelo de la lista y haga click en "Predeterminar".
+      <Break/>
+      Si la opciÃ³n de bÃºsqueda avanzada estÃ¡ activa (a travÃ©s de la pantalla de tipos de bÃºsqueda) los controles de indexaciÃ³n serÃ¡n diferentes. Cada Ã­ndice se basa en una sola fuente de datos. Hay una nueva fuente de pseudodatos llamada "allfields" (todos los campos) que permite buscar a travÃ©s de todos los Ã­ndices especificados al mismo tiempo. Los niveles no se asignan a un Ã­ndice en particular, sino que se aplican a todos los Ã­ndices. De esta manera los Ã­ndices y niveles se aÃ±aden por separado. Los Ã­ndices se eliminan de la misma manera que arriba, pero el Ã­ndice por omisiÃ³n ya no se puede predeterminar: pasa a ser simplemente el primer Ã­ndice asignado.
+      <Break/>
+      Para crear Ã­ndices en todas las fuentes, haga click en el botÃ³n "AÃ±adir todas". El nombre de cada Ã­ndice serÃ¡ el de la fuente por omisiÃ³n. Para cambiar el nombre, seleccione un Ã­ndice, cambie sus detalles y haga click en "Reemplazar Ã­ndice".
+    </Section>
+    <Section name="partitionindexes">
+      <Title>Particiones</Title>
+      Los Ã­ndices se construyen sobre fuentes de metadatos o textos especÃ­ficos, por lo que el espacio de bÃºsqueda se puede controlar aÃºn mÃ¡s particionando el Ã­ndice, ya sea por idioma o por medio de un filtro determinado previamente. Esta secciÃ³n describe la forma de hacerlo. En la parte de "Secciones de diseÃ±o" haga click en "Particionar Ã­ndices".
+      <Break/>
+      La pantalla "Particionar Ã­ndices" tiene tres pestaÃ±as: "Definir filtros", "Asignar particiones" y "Asignar idiomas". Para saber mÃ¡s acerca de las particiones lea la secciÃ³n correspondiente a subcolecciones y subÃ­ndices en el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone.
+      <Section name="definefilters">
+        <Title>Definir filtros</Title>
+        <ObviousSentence>Esta secciÃ³n explica la forma de definir un filtro de particiÃ³n.</ObviousSentence>
+	<Contents>
+	  <ContentsGroup linkto="simple">
+	    <ContentsItem>AÃ±adir un filtro</ContentsItem>
+	    <ContentsItem>Eliminar un filtro</ContentsItem>
+	    <ContentsItem>Actualizar un filtro</ContentsItem>
+          </ContentsGroup>
+        </Contents>
+	<Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+	Los filtros le permiten agrupar en una subcolecciÃ³n todos los documentos de un Ã­ndice para los cuales un valor de metadato sigue un patrÃ³n definido.
+	<Break/>
+	Para crear un filtro haga click en la pestaÃ±a "Definir filtros" y escriba el nombre del nuevo filtro en el campo titulado "Nombre del filtro de la subcolecciÃ³n". A continuaciÃ³n elija un atributo del documento con el cual se va a contrastar, ya sea un elemento de metadato o el nombre del archivo en cuestiÃ³n. Ingrese una expresiÃ³n regular que se utilizarÃ¡ durante la contrastaciÃ³n. Usted puede variar libremente entre "Incluir" documentos que concuerden con el filtro o "Excluirlos". Por Ãºltimo, usted tambiÃ©n puede especificar cualquiera de los indicadores estÃ¡ndar de expresiones regulares PERL para usarlos al momento de contrastar (p. ej. "i" para concordancias que no tomen en cuenta el caso). Haga click en "AÃ±adir filtro" para aÃ±adir el filtro a la lista de "Filtros definidos para la subcolecciÃ³n".
+	<Break/>
+	Para quitar un filtro de la lista, selecciÃ³nelo y haga click en "Eliminar filtro".
+	<Break/>
+	Para modificar un filtro, selecciÃ³nelo de la lista, cambie cualquiera de los valores que aparecen en los controles de ediciÃ³n y haga click en "Reemplazar filtro" para guardar los cambios.
+      </Section>
+      <Section name="assignpartitions">
+        <Title>Asignar particiones</Title>
+	<ObviousSentence>Esta secciÃ³n explica la forma de asignar un filtro de particiÃ³n definido anteriormente.</ObviousSentence>
+	<Contents>
+	  <ContentsGroup linkto="simple">
+	    <ContentsItem>AÃ±adir una particiÃ³n</ContentsItem>
+	    <ContentsItem>Eliminar una particiÃ³n</ContentsItem>
+	    <ContentsItem>Predeterminar una particiÃ³n por omisiÃ³n</ContentsItem>
+	    <ContentsItem>Borrar la particiÃ³n predeterminada</ContentsItem>
+          </ContentsGroup>
+        </Contents>
+	<Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+	Una vez que haya definido el filtro de una subcolecciÃ³n, utilice la pestaÃ±a "Asignar particiones" para construir los Ã­ndices del filtro (o de un grupo de filtros). Seleccione el filtro o los filtros que desee de la lista "Filtros definidos para la subcolecciÃ³n", ingrese un nombre para la particiÃ³n en el campo "Nombre de la particiÃ³n" y haga click en "AÃ±adir particiÃ³n".
+	<Break/>
+	Para quitar una particiÃ³n de la lista, selecciÃ³nela y haga click en "Eliminar particiÃ³n".
+	<Break/>
+	Para hacer que una particiÃ³n determinada sea el valor por omisiÃ³n, selecciÃ³nela de la lista y haga click en "Predeterminar".
+	<Break/>
+	Para borrar la particiÃ³n predeterminada haga click en "Borrar el valor por omisiÃ³n".
+      </Section>
+      <Section name="assignlanguages">
+        <Title>Asignar idiomas</Title>
+	Esta secciÃ³n describe la forma de limitar los Ã­ndices de bÃºsqueda a ciertos idiomas. Esto se hace generando una particiÃ³n por medio de la pestaÃ±a "Asignar idiomas" que se encuentra en la pantalla "Particionar Ã­ndices".
+	<Contents>
+	  <ContentsGroup linkto="simple">
+	    <ContentsItem>Controles de idioma</ContentsItem>
+	    <ContentsItem>AÃ±adir idioma</ContentsItem>
+	    <ContentsItem>Eliminar idioma</ContentsItem>
+	    <ContentsItem>Predeterminar un idioma por omisiÃ³n</ContentsItem>
+	    <ContentsItem>Borrar el idioma predeterminado</ContentsItem>
+          </ContentsGroup>
+        </Contents>
+	<Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+	Para aÃ±adir un nuevo idioma use la pestaÃ±a "Asignar idiomas" a fin de construirle un Ã­ndice. Seleccione el idioma en la lista descendente "Idioma que se va a aÃ±adir" y haga click en "AÃ±adir idioma".
+	<Break/>
+	Para quitar un idioma, escÃ³jalo de la lista "SelecciÃ³n de idiomas" y haga click en "Eliminar idioma".
+	<Break/>
+	Para establecer un idioma por omisiÃ³n, selecciÃ³nelo de la lista y haga click en "Predeterminar".
+	<Break/>
+	Para borrar el idioma predeterminado haga click en "Borrar el valor por omisiÃ³n".
+      </Section>
+    </Section>
+    <Section name="xcollectionsearching">
+      <Title>BÃºsqueda cruzada a travÃ©s de varias colecciones</Title>
+      Greenstone puede buscar a travÃ©s de varias colecciones distintas como si fueran una sola. Esto se hace creando una "supercolecciÃ³n" que abarca a las colecciones individuales. En la parte de "Secciones de diseÃ±o" haga click en "BÃºsqueda cruzada".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Seleccionando una colecciÃ³n</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La pantalla de bÃºsqueda cruzada muestra una lista de las colecciones disponibles. La colecciÃ³n actual estÃ¡ marcada y no se puede quitar de la selecciÃ³n. Para aÃ±adir otra colecciÃ³n en la cual buscar de forma paralela, haga click en ella (o haga click en ella nuevamente para quitarla de la selecciÃ³n). Si sÃ³lo se seleccionÃ³ una colecciÃ³n, entonces no habrÃ¡ bÃºsqueda cruzada.
+      <Break/>
+      Para mayor informaciÃ³n vea el CapÃ­tulo 1 de la GuÃ­a del Programador de Greenstone.
+    </Section>
+    <Section name="classifiers">
+      <Title>Clasificadores</Title>
+      Esta secciÃ³n explica la forma de asignar "clasificadores" a una colecciÃ³n, los cuales se usan para navegar y buscar en ella. En la parte de "Secciones de diseÃ±o" haga click en "Clasificadores de exploraciÃ³n".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>SelecciÃ³n y configuraciÃ³n de un clasificador</ContentsItem>
+	  <ContentsItem>AÃ±adir un clasificador</ContentsItem>
+	  <ContentsItem>Eliminar un clasificador</ContentsItem>
+	  <ContentsItem>Configurar un clasificador</ContentsItem>
+	  <ContentsItem>Cambiar el orden de los clasificadores</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para aÃ±adir un clasificador, escÃ³jalo por medio de la lista descendente "Seleccione el clasificador que se va a aÃ±adir" en la parte inferior de la pantalla y haga click en "AÃ±adir el clasificador especificado". Con esto aparecerÃ¡ una ventana titulada "Argumentos de configuraciÃ³n". Las instrucciones para este cuadro de diÃ¡logo son exactamente las mismas que para los plugins (vea la secciÃ³n <Reference target="plugins">Plugins</Reference>). Una vez que haya configurado el nuevo clasificador, Ã©ste se aÃ±adirÃ¡ al final de la lista de "Clasificadores asignados actualmente".
+      <Break/>
+      Para quitar un clasificador de la lista, selecciÃ³nelo y haga click en "Eliminar el clasificador seleccionado".
+      <Break/>
+      Para cambiar los argumentos de un clasificador, selecciÃ³nelo de la lista y haga click en "Configurar el clasificador seleccionado" o doble click en el nombre del clasificador.
+      <Break/>
+      El orden de los clasificadores en la barra de navegaciÃ³n de la colecciÃ³n se refleja aquÃ­. Para cambiarlo, seleccione el clasificador que desea mover y haga click en "Mover arriba" o "Mover abajo".
+      <Break/>
+      Para mayor informaciÃ³n sobre los clasificadores, lea el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone: Obteniendo el mÃ¡ximo beneficio de sus documentos.
+    </Section>
+    <Section name="formatstatements">
+      <Title>Formatos</Title>
+      Los comandos de formato controlan la estructura y apariencia de la colecciÃ³n. Afectan aspectos como el lugar donde aparecerÃ¡n los botones cuando se muestra un documento y quÃ© vÃ­nculos despliega el clasificador DateList en pantalla. Los comandos de formato no son fÃ¡ciles de desarrollar, por lo que le recomendamos que lea el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone. La presente secciÃ³n describe los controles de los distintos formatos y la forma en que la Interfaz de Bibliotecario le da acceso a ellos. En la parte de "Secciones de diseÃ±o" haga click en "Formatos".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Formateo</ContentsItem>
+	  <ContentsItem>AÃ±adir un comando de formato</ContentsItem>
+	  <ContentsItem>Eliminar un comando de formato</ContentsItem>
+	  <ContentsItem>Actualizar un comando de formato</ContentsItem>
+        </ContentsGroup>
+	<ContentsGroup linkto="advanced">
+	  <ContentsItem>Opciones avanzadas</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Se puede aplicar un comando de formato a cualquier cosa que se encuentre en la lista llamada "Elegir funciÃ³n", la cual incluye cada clasificador y una lista de funciones determinadas previamente. Cuando se selecciona una funciÃ³n existen dos tipos de controles. Algunas funciones simplemente se activan o se desactivan por medio de un recuadro. Otras requieren que se especifique una cadena de formato, para las cuales hay una lista descendente (llamada "Componente afectado") que permite seleccionar la parte de la funciÃ³n a la que se aplica la cadena (en caso necesario), un Ã¡rea de texto (llamada "cadena de formato HTML") para ingresar la cadena y una selecciÃ³n de "variables" definidas previamente. Para insertar una variable en su posiciÃ³n actual dentro de la cadena de formato, selecciÃ³nela de la lista descendente y haga click en "Insertar".
+      <Break/>
+      TambiÃ©n se puede especificar un formato predeterminado para un componente en particular seleccionando la funciÃ³n que estÃ¡ en blanco. Este formato se aplicarÃ¡ a todas las funciones aplicables a menos que se especifique lo contrario.
+      <Break/>
+      Para aÃ±adir un nuevo formato de comando llene la informaciÃ³n tal como se describe arriba y haga click en "AÃ±adir formato". El nuevo comando de formato aparecerÃ¡ en la lista de "Comandos de formato asignados actualmente". Solamente se podrÃ¡ asignar un comando de formato a cada combinaciÃ³n de funciÃ³n/componente.
+      <Break/>
+      Para quitar un comando de formato de la lista, selecciÃ³nelo y haga click en "Eliminar formato".
+      <Break/>
+      Para cambiar un comando de formato, selecciÃ³nelo de la lista, modifique sus parÃ¡metros y haga click en "Reemplazar formato".
+      <Break/>
+      Para mayor informaciÃ³n acerca de las variables y los componentes de las funciones lea el CapÃ­tulo 2 de la GuÃ­a del Programador de Greenstone.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      Si se marca el recuadro "Permitir opciones avanzadas" se activarÃ¡n algunas opciones de formateo avanzado. La lista de funciones que se pueden formatear cambia ligeramente y mÃ¡s variables estÃ¡n disponibles para usarlas en los comandos de formato, proporcionando asÃ­ un mayor control sobre el diseÃ±o de la pÃ¡gina.
+    </Section>
+    <Section name="translatetext">
+      <Title>TraducciÃ³n de textos</Title>
+      Esta secciÃ³n describe la pantalla de traducciÃ³n, donde se pueden definir fragmentos de texto en un idioma especÃ­fico para partes de la interfaz de la colecciÃ³n. En la parte de "Secciones de diseÃ±o" haga click en "Traducir texto".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>AÃ±adir traducciÃ³n</ContentsItem>
+	  <ContentsItem>Eliminar traducciÃ³n</ContentsItem>
+	  <ContentsItem>Actualizar traducciÃ³n</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Primero elija una entrada de la lista de "funciones". Las cadenas de texto asociadas a esta funciÃ³n aparecerÃ¡n abajo. Utilice la lista "Idioma de la traducciÃ³n" para seleccionar el idioma al que se va a traducir y escriba el texto traducido en el Ã¡rea de texto, haciendo referencia al "fragmento de texto inicial" en caso necesario. Una vez que haya terminado haga click en "AÃ±adir traducciÃ³n".
+      <Break/>
+      Para quitar una traducciÃ³n existente, selecciÃ³nela de la tabla "Traducciones asignadas" y haga click en "Eliminar traducciÃ³n".
+      <Break/>
+      Para actualizar una traducciÃ³n, selecciÃ³nela, edÃ­tela en el Ã¡rea de texto titulada "Texto traducido" y haga click en "Reemplazar traducciÃ³n".
+    </Section>
+    <Section name="metadatasets">
+      <Title>Conjuntos de metadatos</Title>
+      Esta secciÃ³n describe la pantalla de revisiÃ³n del conjunto de metadatos. En la parte de "Secciones de diseÃ±o" haga click en "Conjuntos de metadatos".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Conjuntos de metadatos disponibles</ContentsItem>
+          <ContentsItem>AÃ±adiendo conjuntos de metadatos</ContentsItem>
+          <ContentsItem>Editando conjuntos de metadatos</ContentsItem>
+          <ContentsItem>Borrando conjuntos de metadatos</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Esta pantalla se usa para revisar los conjuntos de metadatos que utiliza la colecciÃ³n, asÃ­ como los elementos que estÃ¡n disponibles dentro de cada conjunto. Elija uno de la lista de "Conjuntos de Metadatos Disponibles" para ver los detalles de sus elementos.
+      <Break/>
+      Para usar otro conjunto de metadatos con la colecciÃ³n cargada, pulse "AÃ±adir Conjunto de Metadatos", y seleccione el archivo de conjunto de metadatos (.mds) para el nuevo conjunto de metadatos.
+      <Break/>
+      La edicion de conjuntos de metadatos se hace con el Editor Greenstone para Conjuntos de Metadatos (GEMS). Pulsando el botÃ³n "Editar Conjunto de metadatos" obtendrÃ¡ informaciÃ³n sobre cÃ³mo utilizar GEMS.
+      <Break/>
+      Si ya no necesita mÃ¡s un conjunto de metadatos, selecciÃ³nelo y pulse "Borrar Conjunto de Metadatos" para eliminarlo. Si habÃ­a asignado metadatos a cualquier elemento del conjunto borrado, se le preguntarÃ¡ cÃ³mo actuar con estos metadatos cuando vuelva a abrir la colecciÃ³n.
+    </Section>
+  </Section>
+
+  <Section name="producingthecollection">
+    <Title> Produciendo una colecciÃ³n </Title>
+    Una vez que se han reunido los documentos para la colecciÃ³n, que se han enriquecido con metadatos y que se ha diseÃ±ado la forma en que la colecciÃ³n aparecerÃ¡, usted ya puede empezar a producir la colecciÃ³n. Esta secciÃ³n le explica cÃ³mo.
+    <Section name="thecreateview">
+      <Title>La pantalla de creaciÃ³n de la colecciÃ³n</Title>
+      <ObviousSentence>Esta secciÃ³n describe las caracterÃ­sticas de la pantalla "Crear" que se utiliza para producir una colecciÃ³n.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    ConstrucciÃ³n de la colecciÃ³n
+	  </ContentsItem>
+	  <ContentsItem>
+	    PresentaciÃ³n preliminar de la colecciÃ³n
+	  </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+	  <ContentsItem>Registro de mensajes</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La pantalla "Crear" se utiliza para producir la colecciÃ³n ejecutando los guiones de los programas Greenstone con base en la informaciÃ³n que usted ha proporcionado. Al hacer click en el botÃ³n "Crear colecciÃ³n" se inicia el proceso de construcciÃ³n de la colecciÃ³n. El tiempo que se lleve dependerÃ¡ del tamaÃ±o de la colecciÃ³n y del nÃºmero de Ã­ndices que se estÃ©n creando (para colecciones de gran tamaÃ±o pueden ser horas). Una barra de progreso indica cuÃ¡nto se ha completado del proceso. Para cancelar el proceso en cualquier momento haga click en el botÃ³n "Cancelar la construcciÃ³n".
+      <Break/>
+      Una vez que la colecciÃ³n se ha construido exitosamente, al seleccionar "Vista previa de la colecciÃ³n" se iniciarÃ¡ un navegador de web que mostrarÃ¡ la pÃ¡gina inicial de la colecciÃ³n.
+      <Anchor name="advanced">Instrucciones avanzadas:</Anchor>
+      En modo Experto, puede usar las entradas del "Registro de mensajes" a la izquierda, para revisar intentos previos de construcciÃ³n de la colecciÃ³n, hayan tenido Ã©xito o no. Seleccione el registro que desee haciendo click en la fecha correspondiente en la lista "Historial de registro".
+    </Section>
+    <Section name="buildsettings">
+      <Title>Controles de importaciÃ³n y construcciÃ³n</Title>
+      Esta secciÃ³n explica la forma de tener acceso a los diferentes controles de importaciÃ³n y construcciÃ³n. Para mayor informaciÃ³n lea el CapÃ­tulo 1 de la GuÃ­a del Programador de Greenstone: Comprendiendo el proceso de creaciÃ³n de una colecciÃ³n.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Controles generales</ContentsItem>
+	  <ContentsItem>Controles de importaciÃ³n</ContentsItem>
+	  <ContentsItem>Controles de construcciÃ³n</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La forma de controlar los diferentes ajustes es similar a la de la ventana "Argumentos de configuraciÃ³n" que se describe en la secciÃ³n de <Reference target="plugins">Plugins</Reference>. Algunos campos requerirÃ¡n argumentos numÃ©ricos y usted puede escribirlos directamente en el campo o usar las flechas que se encuentran a un lado para aumentar o disminuir el nÃºmero (en algunos casos la interfaz limita el rango). Otros se activan (y desactivan) haciendo click en los recuadros correspondientes.
+    </Section>
+  </Section>
+
+  <Section name="miscellaneous">
+    <Title>MiscelÃ¡nea</Title>
+    Esta secciÃ³n describe funciones y caracterÃ­sticas de la Interfaz de Bibliotecario que no estÃ¡n asociadas a ninguna pantalla en particular.
+    <Section name="preferences">
+      <Title>Preferencias</Title>
+      Esta secciÃ³n explica el cuadro de preferencias, al cual se tiene acceso por medio de la opciÃ³n "Preferencias" en el menÃº "Archivo".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>General</ContentsItem>
+	  <ContentsItem>Modo</ContentsItem>
+	  <ContentsItem>Flujo de trabajo</ContentsItem>
+	  <ContentsItem>ConexiÃ³n</ContentsItem>
+	  <ContentsItem>Mensajes de advertencia</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      La primera opciÃ³n "General" es un campo de texto para introducir su direcciÃ³n de correo electrÃ³nico. SerÃ¡ usada para los elementos de metadatos "creador" y "administrador" de la colecciÃ³n. La siguiente opciÃ³n es un menÃº desplegable de los lenguajes en los cuales se puede presentar la Interfaz de Bibliotecario. Si usted cambia el diccionario eligiendo un idioma de la lista, entonces deberÃ¡ reinicar la Interfaz de Bibliotecario para poder cargar las cadenas de texto en el nuevo idioma.
+      <Break/>
+Si se marcÃ³ el recuadro "Ver metadatos extraÃ­dos", los diferentes controles que trabajan con metadatos siempre mostrarÃ¡n todos los metadatos que se han extraÃ­do automÃ¡ticamente de los documentos. Al quitar la marca del recuadro se ocultarÃ¡n estos metadatos (aunque la opciÃ³n seguirÃ¡ disponible durante el diseÃ±o de la colecciÃ³n y en la colecciÃ³n final). Si se marcÃ³ el recuadro "Mostrar el tamaÃ±o de los archivos" el tamaÃ±o del archivo aparecerÃ¡ junto a cada uno en las estructuras de archivos del Ã¡rea de trabajo y de la colecciÃ³n que se encuentran en las pantallas "Reunir" y "Enriquecer".
+      <Break/>
+      La pantalla de "Modo" se utiliza para controlar el nivel de detalle dentro de la interfaz. En su nivel mÃ¡s bÃ¡sico, "Asistente de Biblioteca", la pantalla de diseÃ±o estÃ¡ inhabilitada, los argumentos que requieren expresiones regulares estÃ¡n ocultos y el proceso de construcciÃ³n de la colecciÃ³n registra un mÃ­nimo de eventos. En comparaciÃ³n, el nivel mÃ¡s alto, "Avanzado", le da acceso a todas las funciones de diseÃ±o, incluyendo la posiciÃ³n de los plugins y argumentos con expresiones regulares, ademÃ¡s de permitir un registro completo del proceso de construcciÃ³n de la colecciÃ³n. Para cambiar o revisar los modos, haga click en el botÃ³n que se encuentra junto al modo en el que usted estÃ© interesado. De esta manera usted puede revisar rÃ¡pidamente el modo en el que estÃ¡ viendo la barra de tÃ­tulo de la Interfaz de Bibliotecario.
+      <Break/>
+      La Interfaz de Bibliotecario puede soportar diferentes flujos de trabajo determinando cuÃ¡les de las pestaÃ±as de las diferentes pantallas estarÃ¡n visibles. Utilice la pestaÃ±a "Flujo de trabajo" para determinar quÃ© pantallas estarÃ¡n disponibles marcando los recuadros que estÃ¡n junto al nombre de cada una de ellas. De forma alternativa utilice la lista descendente que se encuentra en la parte inferior para seleccionar las configuraciones predeterminadas. Al cerrar el cuadro de preferencias se aplicarÃ¡n los ajustes. Estos se guardan junto con la colecciÃ³n, no en el archivo de configuraciÃ³n de la Interfaz de Bibliotecario.
+      <Break/>
+      La pestaÃ±a de "ConexiÃ³n" le permite cambiar la ruta que lo conecta al servidor local de la biblioteca digital Greenstone, el cual se utiliza durante la presentaciÃ³n preliminar de la colecciÃ³n. TambiÃ©n le permite establecer la informaciÃ³n de proxy para conectarse a Internet (ejemplo: al descargar archivos; ver la secciÃ³n <Reference target="downloadingfiles">Descargando Archivos del Internet</Reference> para mÃ¡s detalles). Marque la casilla para permitir la conexiÃ³n por proxy y suministre los detalles de la direcciÃ³n y el nÃºmero de puerto del host que sirve de proxy. La conexiÃ³n con el proxy se establece cuando cierre el diÃ¡logo de Preferencias.
+      <Break/>
+      En el curso de una sesiÃ³n la Interfaz de Bibliotecario podrÃ¡ enviarle mensajes de advertencia que le avisarÃ¡n sobre las posibles consecuencias no previstas de una acciÃ³n. Usted puede desactivar los mensajes marcando el recuadro "No mostrar este mensaje otra vez". Para reactivarlos utilice la pestaÃ±a "Mensajes de advertencia" y marque los recuadros de los mensajes que desea volver a ver.
+    </Section>
+    <Section name="fileassociations">
+      <Title>AsociaciÃ³n de archivos</Title>
+      La Interfaz de Bibliotecario utiliza diferentes programas de aplicaciÃ³n para abrir tipos particulares de archivos. Esta secciÃ³n explica la forma de asignar y editar estas asociaciones.
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>Establecer una asociaciÃ³n</ContentsItem>
+	  <ContentsItem>Editar una asociaciÃ³n</ContentsItem>
+	  <ContentsItem>Eliminar una asociaciÃ³n</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Para modificar la asociaciÃ³n de archivos abra el menÃº "Archivo" y haga click en "AsociaciÃ³n de archivos".
+      <Break/>
+      Para establecer una asociaciÃ³n seleccione la extensiÃ³n del archivo de la lista descendente o escriba una nueva extensiÃ³n (no incluya "."). A continuaciÃ³n escriba en el campo correspondiente el comando que da inicio a la aplicaciÃ³n deseada o elija la aplicaciÃ³n en el cuadro "Explorar". "%1" se puede utilizar en el comando de inicio para insertar el nombre del archivo que se estÃ¡ abriendo. Una vez que se han llenado estos campos, el botÃ³n "Establecer asociaciÃ³n" se activarÃ¡, con lo que ya se puede hacer click en Ã©l para establecer la asociaciÃ³n.
+      <Break/>
+      Para editar una asociaciÃ³n, seleccione una extensiÃ³n de archivo existente. El comando asociado aparecerÃ¡ en el campo correspondiente a comando de inicio. EdÃ­telo y a continuaciÃ³n haga click en "Reemplazar asociaciÃ³n".
+      <Break/>
+      Para eliminar una asociaciÃ³n, seleccione la extensiÃ³n del archivo y haga click en "Eliminar asociaciÃ³n". La extensiÃ³n del archivo permanecerÃ¡ en una lista llamada "Para archivos que terminan en...".
+      <Break/>
+      Las asociaciones de archivos se guardan en la carpeta principal de la Interfaz de Bibliotecario en un archivo llamado "associations.xml".
+    </Section>
+    <Section name="exportingcollections">
+      <Title>Exportando colecciones a un CD-ROM</Title>
+      Esta secciÃ³n describe la forma de exportar colecciones a un CD-ROM Greenstone que se podrÃ¡ instalar automÃ¡ticamente.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>CÃ³mo exportar</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instrucciones bÃ¡sicas:</Anchor>
+      Greenstone puede exportar una o mÃ¡s colecciones a un CD-ROM para Windows que se podrÃ¡ instalar automÃ¡ticamente. Para ello es indispensable que el paquete "Exportar a un CD-ROM" estÃ© instalado. Esto no se incluye por omisiÃ³n, por lo que serÃ¡ necesario modificar la instalaciÃ³n para incluirlo.
+      <Break/>
+      Para exportar una colecciÃ³n abra el menÃº "Archivo" y seleccione la opciÃ³n "Exportar a un CD-ROM". Con esto aparecerÃ¡ una lista de colecciones Greenstone. Haga click en una para ver su descripciÃ³n. Marque los recuadros de las colecciones que desea exportar. Usted tambiÃ©n puede ingresar el nombre de CD-ROM en el cuadro correspondiente, el cual aparecerÃ¡ en el menÃº de Inicio una vez que el CD-ROM se haya instalado. A continuaciÃ³n haga click en "Exportar". El proceso incluye la copia de varios archivos y requerirÃ¡ de unos cuantos minutos.
+      <Break/>
+      Una vez que haya terminado Greenstone mostrarÃ¡ el nombre de una carpeta en la que se encontrarÃ¡n las colecciones que se han exportado. Utilice una quemadora de CDs para copiar sus contenidos en un CD-ROM en blanco.
+    </Section>
+  </Section>
+</Document>
Index: /main/tags/3.01/gli/help/es/help_index.xml
===================================================================
--- /main/tags/3.01/gli/help/es/help_index.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/es/help_index.xml	(revision 21112)
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Document><Section name="introduction"><Title>IntroducciÃ³n</Title><Section name="ofmiceandmenus"><Title>De menÃºs y ratones</Title></Section><Section name="howtoavoidthisdocument"><Title>Temas de ayuda</Title></Section></Section><Section name="startingoff"><Title>Para empezar</Title><Section name="creatingacollection"><Title>Creando una nueva colecciÃ³n</Title></Section><Section name="savingacollection"><Title>Guardar la colecciÃ³n</Title></Section><Section name="openingacollection"><Title>Abriendo una colecciÃ³n existente</Title></Section><Section name="deletingcollections"><Title>Borrando colecciones</Title></Section></Section><Section name="downloadingfiles"><Title>Descargando archivos de Internet</Title><Section name="themirrorview"><Title>La vista en Bajar</Title></Section></Section><Section name="collectingfiles"><Title>Reuniendo archivos para su colecciÃ³n</Title><Section name="thegatherview"><Title>La pantalla de selecciÃ³n de documentos</Title></Section><Section name="creatingfolders"><Title>Creando carpetas</Title></Section><Section name="addingfiles"><Title>AÃ±adiendo archivos</Title></Section><Section name="removingfiles"><Title>Eliminando archivos</Title></Section><Section name="filteringthetree"><Title>Filtrando la estructura de directorios y subdirectorios</Title></Section></Section><Section name="enrichingacollection"><Title>Enriqueciendo la colecciÃ³n con metadatos</Title><Section name="theenrichview"><Title>La pantalla de asignaciÃ³n de metadatos</Title></Section><Section name="selectingmetadatasets"><Title>Seleccionando conjuntos de metadatos</Title></Section><Section name="appendingmetadata"><Title>Agregando nuevos metadatos</Title></Section><Section name="addingpreviouslydefinedmetadata"><Title>AÃ±adiendo Metadatos Previamente Definidos</Title></Section><Section name="updatingmetadata"><Title>Actualizando metadatos</Title></Section><Section name="removingmetadata"><Title>Eliminando metadatos</Title></Section><Section name="reviewingmetadata"><Title>Revisando los metadatos asignados</Title></Section><Section name="importingpreviouslyassignedmetadata"><Title>Importando metadatos previamente asignados</Title></Section></Section><Section name="designingacollection"><Title>DiseÃ±ando la apariencia de su colecciÃ³n</Title><Section name="thedesignview"><Title>La pantalla de diseÃ±o</Title></Section><Section name="generalsettings"><Title>Ajustes generales</Title></Section><Section name="plugins"><Title>Plugins</Title></Section><Section name="searchtypes"><Title>Tipos de bÃºsqueda</Title></Section><Section name="searchindexes"><Title>Ãndices de bÃºsqueda</Title></Section><Section name="partitionindexes"><Title>Particiones</Title><Section name="definefilters"><Title>Definir filtros</Title></Section><Section name="assignpartitions"><Title>Asignar particiones</Title></Section><Section name="assignlanguages"><Title>Asignar idiomas</Title></Section></Section><Section name="xcollectionsearching"><Title>BÃºsqueda cruzada a travÃ©s de varias colecciones</Title></Section><Section name="classifiers"><Title>Clasificadores</Title></Section><Section name="formatstatements"><Title>Formatos</Title></Section><Section name="translatetext"><Title>TraducciÃ³n de textos</Title></Section><Section name="metadatasets"><Title>Conjuntos de metadatos</Title></Section></Section><Section name="producingthecollection"><Title> Produciendo una colecciÃ³n </Title><Section name="thecreateview"><Title>La pantalla de creaciÃ³n de la colecciÃ³n</Title></Section><Section name="buildsettings"><Title>Controles de importaciÃ³n y construcciÃ³n</Title></Section></Section><Section name="miscellaneous"><Title>MiscelÃ¡nea</Title><Section name="preferences"><Title>Preferencias</Title></Section><Section name="fileassociations"><Title>AsociaciÃ³n de archivos</Title></Section><Section name="exportingcollections"><Title>Exportando colecciones a un CD-ROM</Title></Section></Section></Document>
Index: /main/tags/3.01/gli/help/es/howtoavoidthisdocument.htm
===================================================================
--- /main/tags/3.01/gli/help/es/howtoavoidthisdocument.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/howtoavoidthisdocument.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="howtoavoidthisdocument"><font size="5" face="Verdana"><strong>1.2: Temas de ayuda</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      &iexcl;Usted no necesita leer todo el texto de ayuda! S&oacute;lo lea lo suficiente para saber c&oacute;mo obtener ayuda cuando la necesite.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Ayuda sensible al contexto</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Cuadros de sugerencias</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Otros documentos</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      El comando del men&uacute; "Ayuda" indica los que ser&iacute;an los temas de ayuda m&aacute;s apropiados por medio de un peque&ntilde;o icono en forma de libro.
+      <p></p>
+      Para muchos controles si se coloca el mouse encima de ellos aparecer&aacute; un "cuadro de sugerencias" en el cual se explica su funci&oacute;n.
+      <p></p>
+      Antes de usar la Interfaz de Bibliotecario lea primero la documentaci&oacute;n de Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/importingpreviouslyassignedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/importingpreviouslyassignedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/importingpreviouslyassignedmetadata.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="importingpreviouslyassignedmetadata"><font size="5" face="Verdana"><strong>5.8: Importando metadatos previamente asignados</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe c&oacute;mo importar metadtos previamente asignados, es decir, metadatos asignados a los documentos antes de que fuesen a&ntilde;adidos a la colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Mientras se a&ntilde;aden archivos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Si se han asignado previamente metadatos en una forma reconocible por la Interfaz de Bibliotecario, por ejemplo, cuando se seleccionan documentos de una colecci&oacute;n Greenstone existente, entonces &eacute;stos se importar&aacute;n autom&aacute;ticamente cuando usted a&ntilde;ada el archivo. Para poder hacer esto los metadatos deben mapearse hacia el conjunto de metadatos disponible en la colecci&oacute;n.
+      <p></p>
+      La Interfaz de Bibliotecario enviar&aacute; un mensaje solicitando la informaci&oacute;n necesaria. El mensaje incluir&aacute; breves instrucciones y a continuaci&oacute;n mostrar&aacute; el nombre del elemento de metadato que se est&aacute; importando, tal como aparece en el archivo fuente. Este campo no se podr&aacute; editar o cambiar. A continuaci&oacute;n elija el conjunto de metadatos hacia el que el nuevo elemento deber&aacute; mapearse y el correspondiente elemento de metadato en ese conjunto. El sistema selecciona autom&aacute;ticamente la correspondencia m&aacute;s cercana en t&eacute;rminos de conjunto y elemento para el nuevo metadato.
+      <p></p>
+      Despu&eacute;s de revisar el mapeo, usted puede elegir "A&ntilde;adir" para a&ntilde;adir el nuevo elemento de metadato al conjunto de metadatos que se haya seleccionado. (Esto s&oacute;lo es posible si no hay un elemento del mismo nombre dentro del conjunto seleccionado.) "Reemplazar" mapea el nuevo elemento hacia el que el usuario haya elegido. Finalmente, "Ignorar" no importa ning&uacute;n metadato con este nombre de elemento. Una vez que usted haya especificado la forma de importar una parte del metadato, la informaci&oacute;n del proceso se conserva durante todo el tiempo de vida de la colecci&oacute;n.
+      <p></p>
+      Para mayor informaci&oacute;n sobre los archivos metadata.xml que Greenstone utiliza para guardar metadatos, vea el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone: Obteniendo el m&aacute;ximo beneficio de sus documentos.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/introduction.htm
===================================================================
--- /main/tags/3.01/gli/help/es/introduction.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/introduction.htm	(revision 21112)
@@ -0,0 +1,17 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="introduction"><font size="5" face="Verdana"><strong>1: Introducci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    La Interfaz de Bibliotecario de Greenstone es una herramienta que permite reunir y marcar documentos para luego crear colecciones digitales con ellos, d&aacute;ndole f&aacute;cil acceso a todas las funciones de los programas de software de la Biblioteca Digital Greenstone desde una interfaz gr&aacute;fica de usuario con la que s&oacute;lo tiene que apuntar y hacer click.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/metadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/es/metadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/metadatasets.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="metadatasets"><font size="5" face="Verdana"><strong>6.11: Conjuntos de metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la pantalla de revisi&oacute;n del conjunto de metadatos. En la parte de "Secciones de dise&ntilde;o" haga click en "Conjuntos de metadatos".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Conjuntos de metadatos disponibles</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adiendo conjuntos de metadatos</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Editando conjuntos de metadatos</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Borrando conjuntos de metadatos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Esta pantalla se usa para revisar los conjuntos de metadatos que utiliza la colecci&oacute;n, as&iacute; como los elementos que est&aacute;n disponibles dentro de cada conjunto. Elija uno de la lista de "Conjuntos de Metadatos Disponibles" para ver los detalles de sus elementos.
+      <p></p>
+      Para usar otro conjunto de metadatos con la colecci&oacute;n cargada, pulse "A&ntilde;adir Conjunto de Metadatos", y seleccione el archivo de conjunto de metadatos (.mds) para el nuevo conjunto de metadatos.
+      <p></p>
+      La edicion de conjuntos de metadatos se hace con el Editor Greenstone para Conjuntos de Metadatos (GEMS). Pulsando el bot&oacute;n "Editar Conjunto de metadatos" obtendr&aacute; informaci&oacute;n sobre c&oacute;mo utilizar GEMS.
+      <p></p>
+      Si ya no necesita m&aacute;s un conjunto de metadatos, selecci&oacute;nelo y pulse "Borrar Conjunto de Metadatos" para eliminarlo. Si hab&iacute;a asignado metadatos a cualquier elemento del conjunto borrado, se le preguntar&aacute; c&oacute;mo actuar con estos metadatos cuando vuelva a abrir la colecci&oacute;n.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/miscellaneous.htm
===================================================================
--- /main/tags/3.01/gli/help/es/miscellaneous.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/miscellaneous.htm	(revision 21112)
@@ -0,0 +1,18 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="miscellaneous"><font size="5" face="Verdana"><strong>8: Miscel&aacute;nea</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Esta secci&oacute;n describe funciones y caracter&iacute;sticas de la Interfaz de Bibliotecario que no est&aacute;n asociadas a ninguna pantalla en particular.
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/ofmiceandmenus.htm
===================================================================
--- /main/tags/3.01/gli/help/es/ofmiceandmenus.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/ofmiceandmenus.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="ofmiceandmenus"><font size="5" face="Verdana"><strong>1.1: De men&uacute;s y ratones</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n proporciona informaci&oacute;n b&aacute;sica para poder trabajar con la Interfaz de Bibliotecario. Si est&aacute; familiarizado con programas como Internet Explorer o Microsoft Office y est&aacute; acostumbrado a usar el mouse, entonces pase a la <a href="howtoavoidthisdocument.htm#howtoavoidthisdocument">siguiente secci&oacute;n</a>.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Conceptos b&aacute;sicos
+            <tr>
+<td colspan="2"> </td><td colspan="1">Movimientos del mouse</td>
+</tr>
+            
+<tr>
+<td colspan="2"> </td><td colspan="1">Teclado</td>
+</tr>
+          
+</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Salir del programa</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Teclas de m&eacute;todo abreviado</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La Interfaz de Bibliotecario sigue las convenciones de Microsoft y se apoya en el conocimiento que com&uacute;nmente se tiene de Windows.
+      <p></p>
+      Cualquier parte de la pantalla con la que usted interact&uacute;e, como un bot&oacute;n o un campo de texto, se llama "control". En todo momento un bot&oacute;n o campo de texto llamado "control activo" aparecer&aacute; resaltado y responder&aacute; a las instrucciones del teclado. Asimismo, varios controles le permiten seleccionar las partes que est&eacute;n resaltadas en azul marino. Algunos controles tambi&eacute;n pueden aparecer sombreados para indicar que no est&aacute;n disponibles.
+      <p></p>
+      Usted puede mover y hacer click con el bot&oacute;n izquierdo o derecho del mouse de la forma acostumbrada. Muchos componentes tambi&eacute;n se pueden "arrastrar" haciendo click en ellos y sosteniendo el bot&oacute;n izquierdo del mouse, para luego moverlos y "colocarlos" en alg&uacute;n otro lugar soltando el bot&oacute;n. Normalmente el lugar de destino cambia de apariencia cuando un componente pasa sobre &eacute;l.
+      <p></p>
+      Usted puede usar el teclado para escribir en los campos de texto. Muchos controles tambi&eacute;n se pueden ejecutar por medio del teclado, lo que se indica por medio de corchetes -- por ejemplo la tecla de tabulaci&oacute;n [Tab] cambia el control activo. El signo de m&aacute;s (+) indica que otras teclas deben oprimirse al mismo tiempo.
+      <p></p>
+      Salga de la Interfaz de Bibliotecario eligiendo "Salir" en el men&uacute; "Archivo". Su colecci&oacute;n se guardar&aacute; primero.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Para tener acceso a un men&uacute;, oprima el bot&oacute;n [ALT] y presione la letra correspondiente (la cual aparece subrayada). Por ejemplo, para desplegar el men&uacute; "Archivo" oprima [ALT] + [A]. Para elegir un comando presione la tecla correspondiente. Por ejemplo, en el men&uacute; "Archivo" presione [G] para "Guardar" la colecci&oacute;n.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/openingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/openingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/openingacollection.htm	(revision 21112)
@@ -0,0 +1,64 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="openingacollection"><font size="5" face="Verdana"><strong>2.3: Abriendo una colecci&oacute;n existente</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n le dice la forma de abrir una colecci&oacute;n existente por medio del mensaje Abrir Colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">C&oacute;mo abrir una colecci&oacute;n</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Colecciones protegidas</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Colecciones heredadas</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para abrir una colecci&oacute;n existente elija la opci&oacute;n "Abrir" en el men&uacute; "Archivo". Con esto aparecer&aacute; una lista de sus colecciones Greenstone. Seleccione una para ver su descripci&oacute;n y haga click en "Abrir" para cargarla. Si busca una colecci&oacute;n que resida fuera de la carpeta de "colecciones" de Greenstone, entonces haga click en "Explorar" para que aparezca un cuadro de di&aacute;logo que le permita buscar en el sistema de archivos.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      En caso de que m&aacute;s de una Interfaz de Bibliotecario de Greenstone est&eacute; corriendo al mismo tiempo, los directorios pertinentes se "bloquean" para evitar interferencias. Al abrir una colecci&oacute;n se crea un peque&ntilde;o archivo de bloqueo temporal en la carpeta. Antes de abrir una colecci&oacute;n la Interfaz de Bibliotecario revisa que no haya un archivo de bloqueo en ella. Se puede saber si la colecci&oacute;n est&aacute; bloqueada por el color de su icono: verde para una colecci&oacute;n normal, rojo para una colecci&oacute;n bloqueada. Sin embargo, cuando se sale prematuramente de la Interfaz de Bibliotecario el archivo de bloqueo a veces permanece en la carpeta. Cuando posteriormente se abre esta colecci&oacute;n, la Biblioteca Digital le pregunta si quiere tomar el control. Por favor, nunca le quite el control de una colecci&oacute;n a alguien que este trabajando en ese momento con ella.
+      <p></p>
+      Cuando usted abra una colecci&oacute;n que no haya sido creada por medio de la Interfaz de Bibliotecario de Greenstone se le pedir&aacute; que seleccione uno o m&aacute;s conjuntos de metadatos. Si no selecciona ninguno, entonces los metadatos existentes se ignorar&aacute;n. De otra manera los metadatos se importar&aacute;n tal como son cuando usted arrastre los archivos. El proceso se describe en el documento llamado <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importando Metadatos Previamente Asignados</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/partitionindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/es/partitionindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/partitionindexes.htm	(revision 21112)
@@ -0,0 +1,20 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="partitionindexes"><font size="5" face="Verdana"><strong>6.6: Particiones</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Los &iacute;ndices se construyen sobre fuentes de metadatos o textos espec&iacute;ficos, por lo que el espacio de b&uacute;squeda se puede controlar a&uacute;n m&aacute;s particionando el &iacute;ndice, ya sea por idioma o por medio de un filtro determinado previamente. Esta secci&oacute;n describe la forma de hacerlo. En la parte de "Secciones de dise&ntilde;o" haga click en "Particionar &iacute;ndices".
+      <p></p>
+      La pantalla "Particionar &iacute;ndices" tiene tres pesta&ntilde;as: "Definir filtros", "Asignar particiones" y "Asignar idiomas". Para saber m&aacute;s acerca de las particiones lea la secci&oacute;n correspondiente a subcolecciones y sub&iacute;ndices en el Cap&iacute;tulo 2 de la Gu&iacute;a del Programador de Greenstone.
+      
+      
+      
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/plugins.htm
===================================================================
--- /main/tags/3.01/gli/help/es/plugins.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/plugins.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="plugins"><font size="5" face="Verdana"><strong>6.3: Plugins</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la forma de configurar los plugins que la colecci&oacute;n utiliza. Explica el modo de especificar los plugins que se utilizar&aacute;n, los par&aacute;metros que emplear&aacute;n y el orden en el que se aplicar&aacute;n. En la parte de "Secciones de dise&ntilde;o" haga click en "Plugins de documento".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir un plugin</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar un plugin</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Configurar un plugin</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Cambiar el orden de los plugins</td>
+</tr>
+        
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Configuraci&oacute;n personalizada</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para a&ntilde;adir un plugin, esc&oacute;jalo de la lista "Seleccione el plugin que se va a a&ntilde;adir" que se encuentra en la parte inferior de la pantalla y haga click en "A&ntilde;adir plugin". Aparecer&aacute; una ventana titulada "Argumentos de configuraci&oacute;n", la cual se explica m&aacute;s adelante. Una vez que usted haya configurado el nuevo plugin, este se a&ntilde;adir&aacute; al final de la lista de "Plugins asignados actualmente". Note que, excepto para UnknownPlug, cada plugin s&oacute;lo puede aparecer una vez en la lista.
+      <p></p>
+      Para quitar un plugin de la lista, selecci&oacute;nelo y haga click en "Eliminar plugin".
+      <p></p>
+      Los plugins se configuran por medio de argumentos. Para modificarlos, seleccione un plugin de la lista y haga click en "Configurar plugin" (o doble click en el plugin). Aparece un di&aacute;logo "Configurando Argumentos" con varios controles para especificar argumentos.
+      <p></p>
+      Existen diferentes tipos de controles. Algunos son recuadros y al hacer click en ellos se a&ntilde;ade la opci&oacute;n correspondiente al plugin. Otros son cadenas de texto con un recuadro y un campo de texto. Haga click en el cuadro para habilitarlo y escriba el argumento (expresi&oacute;n regular, ruta de archivo, etc.). Otros son men&uacute;s descendentes desde los que se puede seleccionar un conjunto dado de valores. Para saber lo que hace un argumento, coloque el cursor del mouse encima de su nombre y la descripci&oacute;n aparecer&aacute;.
+      <p></p>
+      Una vez que usted haya cambiado la configuraci&oacute;n, haga click en "Aceptar" para guardar los cambios y cerrar el cuadro de di&aacute;logo o en "Cancelar" para cerrar el cuadro de di&aacute;logo sin cambiar ning&uacute;n argumento.
+      <p></p>
+      Los plugins de la lista se ejecutan en orden y en ocasiones &eacute;ste es importante. Dos plugins, ArcPlug y RecPlug, son esenciales para el proceso de construcci&oacute;n de la colecci&oacute;n y se encuentran al final de la lista (separados del resto por una l&iacute;nea). Para cambiar el orden de los dem&aacute;s, seleccione el plugin que desea mover y haga click en "Mover arriba" o "Mover abajo".
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      La Interfaz de Bibliotecario hace lo posible por determinar los argumentos que un plugin soporta. Sin embargo, puede haber casos en los cuales el usuario quiera especificar argumentos especiales, y para esto se ofrece un campo de texto llamado "Argumentos personalizados" en la parte inferior. Cualquier texto que se escriba en &eacute;l se agregar&aacute; palabra por palabra al final del comando del plugin.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/preferences.htm
===================================================================
--- /main/tags/3.01/gli/help/es/preferences.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/preferences.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="preferences"><font size="5" face="Verdana"><strong>8.1: Preferencias</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica el cuadro de preferencias, al cual se tiene acceso por medio de la opci&oacute;n "Preferencias" en el men&uacute; "Archivo".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">General</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Modo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Flujo de trabajo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Conexi&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Mensajes de advertencia</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La primera opci&oacute;n "General" es un campo de texto para introducir su direcci&oacute;n de correo electr&oacute;nico. Ser&aacute; usada para los elementos de metadatos "creador" y "administrador" de la colecci&oacute;n. La siguiente opci&oacute;n es un men&uacute; desplegable de los lenguajes en los cuales se puede presentar la Interfaz de Bibliotecario. Si usted cambia el diccionario eligiendo un idioma de la lista, entonces deber&aacute; reinicar la Interfaz de Bibliotecario para poder cargar las cadenas de texto en el nuevo idioma.
+      <p></p>
+Si se marc&oacute; el recuadro "Ver metadatos extra&iacute;dos", los diferentes controles que trabajan con metadatos siempre mostrar&aacute;n todos los metadatos que se han extra&iacute;do autom&aacute;ticamente de los documentos. Al quitar la marca del recuadro se ocultar&aacute;n estos metadatos (aunque la opci&oacute;n seguir&aacute; disponible durante el dise&ntilde;o de la colecci&oacute;n y en la colecci&oacute;n final). Si se marc&oacute; el recuadro "Mostrar el tama&ntilde;o de los archivos" el tama&ntilde;o del archivo aparecer&aacute; junto a cada uno en las estructuras de archivos del &aacute;rea de trabajo y de la colecci&oacute;n que se encuentran en las pantallas "Reunir" y "Enriquecer".
+      <p></p>
+      La pantalla de "Modo" se utiliza para controlar el nivel de detalle dentro de la interfaz. En su nivel m&aacute;s b&aacute;sico, "Asistente de Biblioteca", la pantalla de dise&ntilde;o est&aacute; inhabilitada, los argumentos que requieren expresiones regulares est&aacute;n ocultos y el proceso de construcci&oacute;n de la colecci&oacute;n registra un m&iacute;nimo de eventos. En comparaci&oacute;n, el nivel m&aacute;s alto, "Avanzado", le da acceso a todas las funciones de dise&ntilde;o, incluyendo la posici&oacute;n de los plugins y argumentos con expresiones regulares, adem&aacute;s de permitir un registro completo del proceso de construcci&oacute;n de la colecci&oacute;n. Para cambiar o revisar los modos, haga click en el bot&oacute;n que se encuentra junto al modo en el que usted est&eacute; interesado. De esta manera usted puede revisar r&aacute;pidamente el modo en el que est&aacute; viendo la barra de t&iacute;tulo de la Interfaz de Bibliotecario.
+      <p></p>
+      La Interfaz de Bibliotecario puede soportar diferentes flujos de trabajo determinando cu&aacute;les de las pesta&ntilde;as de las diferentes pantallas estar&aacute;n visibles. Utilice la pesta&ntilde;a "Flujo de trabajo" para determinar qu&eacute; pantallas estar&aacute;n disponibles marcando los recuadros que est&aacute;n junto al nombre de cada una de ellas. De forma alternativa utilice la lista descendente que se encuentra en la parte inferior para seleccionar las configuraciones predeterminadas. Al cerrar el cuadro de preferencias se aplicar&aacute;n los ajustes. Estos se guardan junto con la colecci&oacute;n, no en el archivo de configuraci&oacute;n de la Interfaz de Bibliotecario.
+      <p></p>
+      La pesta&ntilde;a de "Conexi&oacute;n" le permite cambiar la ruta que lo conecta al servidor local de la biblioteca digital Greenstone, el cual se utiliza durante la presentaci&oacute;n preliminar de la colecci&oacute;n. Tambi&eacute;n le permite establecer la informaci&oacute;n de proxy para conectarse a Internet (ejemplo: al descargar archivos; ver la secci&oacute;n <a href="downloadingfiles.htm#downloadingfiles">Descargando Archivos del Internet</a> para m&aacute;s detalles). Marque la casilla para permitir la conexi&oacute;n por proxy y suministre los detalles de la direcci&oacute;n y el n&uacute;mero de puerto del host que sirve de proxy. La conexi&oacute;n con el proxy se establece cuando cierre el di&aacute;logo de Preferencias.
+      <p></p>
+      En el curso de una sesi&oacute;n la Interfaz de Bibliotecario podr&aacute; enviarle mensajes de advertencia que le avisar&aacute;n sobre las posibles consecuencias no previstas de una acci&oacute;n. Usted puede desactivar los mensajes marcando el recuadro "No mostrar este mensaje otra vez". Para reactivarlos utilice la pesta&ntilde;a "Mensajes de advertencia" y marque los recuadros de los mensajes que desea volver a ver.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/producingthecollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/producingthecollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/producingthecollection.htm	(revision 21112)
@@ -0,0 +1,17 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="producingthecollection"><font size="5" face="Verdana"><strong>7:  Produciendo una colecci&oacute;n </strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Una vez que se han reunido los documentos para la colecci&oacute;n, que se han enriquecido con metadatos y que se ha dise&ntilde;ado la forma en que la colecci&oacute;n aparecer&aacute;, usted ya puede empezar a producir la colecci&oacute;n. Esta secci&oacute;n le explica c&oacute;mo.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/removingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/es/removingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/removingfiles.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingfiles"><font size="5" face="Verdana"><strong>4.4: Eliminando archivos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la forma de eliminar archivos y carpetas de su colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">El bot&oacute;n eliminar</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La tecla suprimir</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Arrastrarlos y colocarlos en el bot&oacute;n "Eliminar"</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Existen varios m&eacute;todos para eliminar archivos y carpetas. Primero se deben indicar los elementos que se desea eliminar seleccionando los archivos y carpetas tal como se describe en la secci&oacute;n de <a href="thegatherview.htm#thegatherview">selecci&oacute;n de documentos para la colecci&oacute;n</a>.
+      <p></p>
+      Una vez que se han seleccionado los archivos, haga click en el bot&oacute;n "Eliminar" o presione la tecla [Del], [Supr] o [Suprimir], dependiendo del teclado de su computadora, o arr&aacute;strelos desde la colecci&oacute;n hasta el bot&oacute;n "Eliminar" y su&eacute;ltelos ah&iacute;.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/removingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/removingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/removingmetadata.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingmetadata"><font size="5" face="Verdana"><strong>5.6: Eliminando metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de eliminar los metadatos de un archivo.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminando metadatos de un archivo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminando metadatos de carpetas o de varios archivos juntos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Los metadatos se eliminan de la misma manera que se actualizan. Primero se selecciona un archivo de la estructura de directorios y subdirectorios, y se utiliza la tabla de metadatos para elegir el metadato que se desea eliminar. Si un metadato tiene asignado un valor, se activa el bot&oacute;n "Borrar". Haga click en &eacute;l para quitar el metadato del archivo seleccionado. Los dem&aacute;s archivos permanecer&aacute;n sin cambio y el valor se conservar&aacute; en la estructura de valores.
+      <p></p>
+      Eliminar metadatos de directorios o de m&uacute;ltiples archivos se hace de la misma forma.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/reviewingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/reviewingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/reviewingmetadata.htm	(revision 21112)
@@ -0,0 +1,74 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="reviewingmetadata"><font size="5" face="Verdana"><strong>5.7: Revisando los metadatos asignados</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la forma de ver todos los metadatos asignados a una colecci&oacute;n completa.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La tabla de auditor&iacute;a</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2"> El men&uacute; del bot&oacute;n derecho del mouse</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Filtros</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      En ocasiones es necesario ver los metadatos asignados a varios o a todos los archivos al mismo tiempo, por ejemplo, para determinar la forma en que quedan muchos archivos para trabajar en ellos o para tener una idea de la diferencia de tiempo entre las fechas.
+      <p></p>
+      Seleccione los archivos que desea examinar, pulse el bot&oacute;n derecho del rat&oacute;n y elija "Metadatos Asignados&hellip;". Con lo que aparecer&aacute; una ventana llamada "Todos los metadatos", ocupada en su mayor parte por una tabla de varias columnas. La primera columna muestra los nombres de los archivos, mientras que las filas muestran todos los valores asignados a dichos archivos.
+      <p></p>
+      Si se seleccionaron muchos archivos es posible que transcurra alg&uacute;n tiempo antes de que se abra la ventana. Mientras tanto usted puede seguir usando la Interfaz de Bibliotecario.
+      <p></p>
+      Haga click en "Cerrar" para ocultar la ventana.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Tambi&eacute;n se puede ver la tabla de "Todos los metadatos" seleccionando los archivos que desea examinar, haciendo click con el bot&oacute;n derecho del mouse y eligiendo la opci&oacute;n "Metadatos asignados...". Si se seleccion&oacute; una carpeta, todos sus archivos estar&aacute;n incluidos en la tabla.
+      <p></p>
+      Cuando la tabla sea demasiado grande se puede refinar aplicando los filtros a las columnas. Conforme se a&ntilde;adan nuevos filtros, &uacute;nicamente aquellas filas que concuerden con ellos seguir&aacute;n siendo visibles. Para establecer, modificar o borrar un filtro haga click en el icono con forma de embudo que se encuentra en la parte superior de la columna. Se le indicar&aacute; que proporcione informaci&oacute;n sobre el filtro. Una vez que un filtro se ha aplicado, el encabezado de la columna cambia de color.
+      <p></p>
+      El aviso tiene una pesta&ntilde;a "Sencilla" y una "Avanzada". La versi&oacute;n sencilla filtra las columnas de tal manera que s&oacute;lo muestren las filas que contienen un determinado valor ("*" corresponde a todos los valores). Los valores de los metadatos se pueden seleccionar de la lista. La versi&oacute;n avanzada permite diferentes operaciones de concordancia: "debe empezar con", "no contiene", "alfab&eacute;ticamente menos que" y "es igual a". El valor se puede editar con cualquier cadena de caracteres (incluyendo "*") y se puede elegir que la concordancia no tome en cuenta el caso. Finalmente, se puede especificar una segunda condici&oacute;n de concordancia que se puede usar para especificar un rango de valores (seleccionando "Y") o valores alternativos (seleccionando "O"). Debajo de esta &aacute;rea hay un cuadro que le permite cambiar el orden de clasificaci&oacute;n (ascendente o descendente). Una vez que haya terminado, haga click en "Filtrar" para aplicar el nuevo filtro a la columna. Para quitar el filtro haga click en "Borrar filtro". Recuerde que los detalles del filtro se conservan incluso cuando el filtro se borra.
+      <p></p>
+      Por ejemplo, para ordenar la tabla elija una columna, seleccione el filtro por omisi&oacute;n (un filtro Sencillo en "*") y escoja el orden ascendente o descendente.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/savingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/es/savingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/savingacollection.htm	(revision 21112)
@@ -0,0 +1,60 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="savingacollection"><font size="5" face="Verdana"><strong>2.2: Guardar la colecci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+       Esta secci&oacute;n describe la forma de guardar una colecci&oacute;n y el mensaje Guardar Colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">C&oacute;mo guardar</td>
+</tr>
+	
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">D&oacute;nde se guardan los archivos</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Guarde su trabajo regularmente abriendo el men&uacute; "Archivo" y seleccionando "Guardar". Tome en cuenta que guardar una colecci&oacute;n no es lo mismo que prepararla para que se pueda usar en Greenstone (vea la secci&oacute;n <a href="producingthecollection.htm#producingthecollection">Produciendo una colecci&oacute;n</a>).
+      <p></p>
+      La Interfaz de Bibliotecario protege su trabajo guard&aacute;ndolo cada vez que salga del programa o cargue otra colecci&oacute;n.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Las colecciones se guardan en un archivo que lleva el nombre de la colecci&oacute;n y la extensi&oacute;n ".col", ubicado en una carpeta del mismo nombre dentro de la carpeta "colecciones" del programa Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/searchindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/es/searchindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/searchindexes.htm	(revision 21112)
@@ -0,0 +1,67 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchindexes"><font size="5" face="Verdana"><strong>6.5: &Iacute;ndices de b&uacute;squeda</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Los &iacute;ndices especifican las partes de la colecci&oacute;n que se pueden buscar. Esta secci&oacute;n explica la forma de a&ntilde;adir y eliminar &iacute;ndices, as&iacute; como predeterminar un &iacute;ndice por omisi&oacute;n. En la parte correspondiente a "Secciones de dise&ntilde;o" haga click en "&Iacute;ndices de b&uacute;squeda".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir un &iacute;ndice</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Editar un &iacute;ndice</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar un &iacute;ndice</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Predeterminar un &iacute;ndice por omisi&oacute;n</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para a&ntilde;adir un &iacute;ndice escriba el nombre que le quiera dar en el campo "Nombre del &iacute;ndice". Seleccione las fuentes de informaci&oacute;n que se van a indexar haciendo click en los recuadros que se encuentran a un lado. La lista mostrar&aacute; todos los elementos de metadatos asignados, as&iacute; como el texto completo. Despu&eacute;s de seleccionar las fuentes de datos, elija la granularidad del &iacute;ndice usando el men&uacute; de "Nivel". Una vez que se han completado estos detalles, la opci&oacute;n "A&ntilde;adir &iacute;ndice" se activa (a menos que haya un &iacute;ndice con los mismos par&aacute;metros). Haga click en ella para a&ntilde;adir el nuevo &iacute;ndice.
+      <p></p>
+      Para editar un &iacute;ndice, selecci&oacute;nelo, cambie los detalles del &iacute;ndice, y pulse "Reemplazar &Iacute;ndice".
+      <p></p>
+      Para quitar un &iacute;ndice de la lista de &iacute;ndices asignados, selecci&oacute;nelo y haga click en "Eliminar &iacute;ndice".
+      <p></p>
+      El &iacute;ndice predeterminado es el que se usa en la p&aacute;gina de b&uacute;squeda de la colecci&oacute;n y lleva la etiqueta "[Indice por omisi&oacute;n]" en la lista de "&Iacute;ndices asignados". Para establecer un &iacute;ndice por omisi&oacute;n, selecci&oacute;nelo de la lista y haga click en "Predeterminar".
+      <p></p>
+      Si la opci&oacute;n de b&uacute;squeda avanzada est&aacute; activa (a trav&eacute;s de la pantalla de tipos de b&uacute;squeda) los controles de indexaci&oacute;n ser&aacute;n diferentes. Cada &iacute;ndice se basa en una sola fuente de datos. Hay una nueva fuente de pseudodatos llamada "allfields" (todos los campos) que permite buscar a trav&eacute;s de todos los &iacute;ndices especificados al mismo tiempo. Los niveles no se asignan a un &iacute;ndice en particular, sino que se aplican a todos los &iacute;ndices. De esta manera los &iacute;ndices y niveles se a&ntilde;aden por separado. Los &iacute;ndices se eliminan de la misma manera que arriba, pero el &iacute;ndice por omisi&oacute;n ya no se puede predeterminar: pasa a ser simplemente el primer &iacute;ndice asignado.
+      <p></p>
+      Para crear &iacute;ndices en todas las fuentes, haga click en el bot&oacute;n "A&ntilde;adir todas". El nombre de cada &iacute;ndice ser&aacute; el de la fuente por omisi&oacute;n. Para cambiar el nombre, seleccione un &iacute;ndice, cambie sus detalles y haga click en "Reemplazar &iacute;ndice".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/searchtypes.htm
===================================================================
--- /main/tags/3.01/gli/help/es/searchtypes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/searchtypes.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchtypes"><font size="5" face="Verdana"><strong>6.4: Tipos de b&uacute;squeda</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de modificar una nueva funci&oacute;n de dise&ntilde;o de Greenstone llamada "Tipos de b&uacute;squeda", que permite la b&uacute;squeda por campos. En la parte de "Secciones de dise&ntilde;o" haga click en "Tipos de dise&ntilde;o".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Lo que significa activar las opciones de b&uacute;squeda avanzada</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adiendo un nuevo tipo de b&uacute;squeda</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminando un tipo de b&uacute;squeda</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Cambiando el orden de los tipos de b&uacute;squeda</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Cuando ingrese a la pantalla de tipos de b&uacute;squeda marque primero el recuadro "Permitir b&uacute;squedas avanzadas", lo cual activa todos los dem&aacute;s controles. Esto transfiere la colecci&oacute;n al formato Greenstone 2.4, el cual soporta la b&uacute;squeda por campos y significa que (a) el dise&ntilde;o de &iacute;ndices es diferente (lo cual se explica en la secci&oacute;n <a href="searchindexes.htm#searchindexes">&Iacute;ndices de b&uacute;squeda</a>), (b) hay m&aacute;s fragmentos de texto para traducir (vea la secci&oacute;n relativa a <a href="translatetext.htm#translatetext">Traducci&oacute;n</a>) y (c) la colecci&oacute;n no podr&aacute; usarse con versiones anteriores de Greenstone. Si posteriormente usted quita la marca de este campo, la mayor parte de su colecci&oacute;n regresar&aacute; a Greenstone 2.39. No obstante, la Interfaz de Bibliotecario no podr&aacute; convertir las especificaciones del nuevo &iacute;ndice en las de las versiones anteriores, por lo que usted tendr&aacute; que ingresarlas otra vez de forma manual.
+      <p></p>
+      Para a&ntilde;adir un tipo de b&uacute;squeda, selecci&oacute;nelo de la lista "Tipos de b&uacute;squeda" y haga click en "A&ntilde;adir tipo de b&uacute;squeda". Cada tipo puede aparecer en la lista solamente una vez.
+      <p></p>
+      Para quitar un tipo de b&uacute;squeda, selecci&oacute;nelo de la lista "Tipos de b&uacute;squeda asignados actualmente" y haga click en "Eliminar tipo de b&uacute;squeda". La lista deber&aacute; contener al menos uno.
+      <p></p>
+      Para cambiar el orden de un tipo de b&uacute;squeda, selecci&oacute;nelo y haga click en "Mover arriba" o "Mover abajo". El primer tipo ser&aacute; el ajuste por omisi&oacute;n.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/selectingmetadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/es/selectingmetadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/selectingmetadatasets.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="selectingmetadatasets"><font size="5" face="Verdana"><strong>5.2: Seleccionando conjuntos de metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de especificar los elementos de los metadatos y c&oacute;mo a&ntilde;adir conjuntos de metadatos a su colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">&iquest;Qu&eacute; es un conjunto de metadatos?</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Controlando los conjuntos de metadatos usados</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Los grupos de elementos de metadatos definidos previamente son conocidos como "conjuntos de metadatos". Un ejemplo es el conjunto de metadatos de Dublin Core. Cuando se a&ntilde;ade un conjunto de metadatos a una colecci&oacute;n, sus elementos se vuelven elegibles para que el usuario los pueda seleccionar. Tambi&eacute;n se puede tener m&aacute;s de un conjunto. Para evitar conflictos de nombre, un identificador que describe al conjunto de metadatos se agrega como prefijo al nombre del elemento. Por ejemplo, el elemento Creator de Dublin Core se convierte en "dc.Creator". Los conjuntos de metadatos se guardan en la carpeta de metadatos de la Interfaz de Bibliotecario y llevan el sufijo ".mds".
+      <p></p>
+      Para controlar los conjuntos de metadatos usados en una colecci&oacute;n, utilice la opci&oacute;n "Conjuntos de metadatos" en la vista Dise&ntilde;o.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/startingoff.htm
===================================================================
--- /main/tags/3.01/gli/help/es/startingoff.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/startingoff.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="startingoff"><font size="5" face="Verdana"><strong>2: Para empezar</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Esta secci&oacute;n explica c&oacute;mo crear, cargar, guardar y borrar colecciones.
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/es/thecreateview.htm
===================================================================
--- /main/tags/3.01/gli/help/es/thecreateview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/thecreateview.htm	(revision 21112)
@@ -0,0 +1,68 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thecreateview"><font size="5" face="Verdana"><strong>7.1: La pantalla de creaci&oacute;n de la colecci&oacute;n</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe las caracter&iacute;sticas de la pantalla "Crear" que se utiliza para producir una colecci&oacute;n.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Construcci&oacute;n de la colecci&oacute;n
+	  </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Presentaci&oacute;n preliminar de la colecci&oacute;n
+	  </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Registro de mensajes</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La pantalla "Crear" se utiliza para producir la colecci&oacute;n ejecutando los guiones de los programas Greenstone con base en la informaci&oacute;n que usted ha proporcionado. Al hacer click en el bot&oacute;n "Crear colecci&oacute;n" se inicia el proceso de construcci&oacute;n de la colecci&oacute;n. El tiempo que se lleve depender&aacute; del tama&ntilde;o de la colecci&oacute;n y del n&uacute;mero de &iacute;ndices que se est&eacute;n creando (para colecciones de gran tama&ntilde;o pueden ser horas). Una barra de progreso indica cu&aacute;nto se ha completado del proceso. Para cancelar el proceso en cualquier momento haga click en el bot&oacute;n "Cancelar la construcci&oacute;n".
+      <p></p>
+      Una vez que la colecci&oacute;n se ha construido exitosamente, al seleccionar "Vista previa de la colecci&oacute;n" se iniciar&aacute; un navegador de web que mostrar&aacute; la p&aacute;gina inicial de la colecci&oacute;n.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      En modo Experto, puede usar las entradas del "Registro de mensajes" a la izquierda, para revisar intentos previos de construcci&oacute;n de la colecci&oacute;n, hayan tenido &eacute;xito o no. Seleccione el registro que desee haciendo click en la fecha correspondiente en la lista "Historial de registro".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/thedesignview.htm
===================================================================
--- /main/tags/3.01/gli/help/es/thedesignview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/thedesignview.htm	(revision 21112)
@@ -0,0 +1,49 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thedesignview"><font size="5" face="Verdana"><strong>6.1: La pantalla de dise&ntilde;o</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n le presenta la pantalla de Dise&ntilde;o y le explica la forma de navegar entre las diferentes opciones de este conjunto.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eligiendo la secci&oacute;n de dise&ntilde;o</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Con la Interfaz de Bibliotecario usted puede configurar la forma en que la colecci&oacute;n aparecer&aacute; frente al usuario. Las opciones se dividen en diferentes secciones, cada una relacionada con una etapa particular de navegaci&oacute;n o presentaci&oacute;n.
+      <p></p>
+      A la izquierda se encuentra una lista de las diferentes pantallas, en tanto que a la derecha est&aacute;n los controles asociados a la pantalla que est&aacute; activa en ese momento. Para cambiar a una pantalla diferente haga click en la lista.
+      <p></p>
+      Para entender las etapas y los t&eacute;rminos usados en el dise&ntilde;o de una colecci&oacute;n, lea primero los Cap&iacute;tulos 1 y 2 de la Gu&iacute;a del Programador de Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/theenrichview.htm
===================================================================
--- /main/tags/3.01/gli/help/es/theenrichview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/theenrichview.htm	(revision 21112)
@@ -0,0 +1,69 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="theenrichview"><font size="5" face="Verdana"><strong>5.1: La pantalla de asignaci&oacute;n de metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la forma de a&ntilde;adir y editar metadatos por medio de la pantalla "Enriquecer".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La estructura de directorios y subdirectorios de la colecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Controles de edici&oacute;n de metadatos</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La estructura de valores</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La tabla de metadatos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Utilice la pantalla "Enriquecer" para asignar metadatos a los documentos de la colecci&oacute;n. Los metadatos son datos acerca de los datos: t&iacute;tulo, autor, fecha de creaci&oacute;n, etc. Cada metadato tiene dos partes: el "elemento" dice el tipo de metadato que es (como por ejemplo el autor) y el "valor" da la informaci&oacute;n de ese elemento (como por ejemplo el nombre del autor).
+      <p></p>
+      En la parte izquierda de la pantalla "Enriquecer" se encuentra la estructura de directorios y subdirectorios de la colecci&oacute;n. A la derecha est&aacute; la Tabla de Metadatos, la cual muestra los metadatos de los archivos y carpetas seleccionados. Los nombres de las columnas est&aacute;n en gris en la parte superior y se pueden cambiar de tama&ntilde;o arrastrando la l&iacute;nea que las separa. Pulsar sobre cualquier l&iacute;nea transfiere sus detalles a los controles inferiores de edici&oacute;n de metadatos. Si se seleccionan varios archivos, el texto en negro indica que el valor es com&uacute;n a todos ellos, mientras que el texto sombreado indica que no. Los valores en negro se pueden actualizar o eliminar, mientras que los que est&aacute;n sombreados se pueden eliminar de los que los tienen o agregarse a los dem&aacute;s.
+      <p></p>
+      Un icono en forma de carpeta puede aparecer junto a algunas entradas de metadatos. Esto indica que los valores se heredan de la carpeta en la que se encuentra el archivo. Los metadatos heredados no se pueden editar o eliminar, sino &uacute;nicamente agregarse o sobreescribirse. Haga click en el icono de la carpeta para ir inmediatamente a la carpeta en la que se asignaron los metadatos.
+      <p></p>
+      Los controles de edici&oacute;n de metadatos de la parte inferior derecha aparecen s&oacute;lo cuando se selecciona un archivo de la estructura y una l&iacute;nea de la tabla. Util&iacute;celos para actualizar, agregar o eliminar valores de los metadatos. El campo de valores es para ingresar o editar el valor del metadato.  Junto hay un bot&oacute;n con la etiqueta "...", el cual abre un cuadro de edici&oacute;n m&aacute;s grande. De los botones que se encuentran debajo, "Agregar" asigna el valor en forma de nuevo metadato y lo a&ntilde;ade a los valores existentes para el elemento seleccionado, "Reemplazar" sobreescribe el valor seleccionado sustituy&eacute;ndolo por uno nuevo y "Eliminar" borra el valor seleccionado. M&aacute;s abajo, con la etiqueta "Todos los valores anteriores", se encuentra la estructura de valores.
+      <p></p>
+      La estructura de valores se puede expandir y contraer. Normalmente es una lista que muestra todos los valores ingresados previamente para un elemento seleccionado. Al hacer click en una entrada &eacute;sta aparece autom&aacute;ticamente en el campo de valores. Y a la inversa, al escribir en el campo se selecciona la entrada de la estructura de valores que empieza con los caracteres ingresados. Al oprimir [Tab] se completa autom&aacute;ticamente el valor seleccionado.
+      <p></p>
+      Los valores de los metadatos se pueden organizar jer&aacute;rquicamente. Esto se muestra en la estructura de valores por medio de carpetas para los niveles internos. Los valores jer&aacute;rquicos pueden ingresarse utilizando el car&aacute;cter "|" para separar los niveles. Por ejemplo, se puede usar "Cartas|Rojo|Diamantes|Siete" en una jerarqu&iacute;a que representa un mazo de cartas. Esto permite agrupar los valores, en tanto que los grupos tambi&eacute;n pueden asignarse en forma de metadatos a los archivos.
+      <p></p>
+      Greenstone extrae autom&aacute;ticamente los metadatos de los documentos para formar un conjunto de metadatos cuyos elementos llevan el prefijo "ex.". &Eacute;ste no tiene estructura de valores y no puede editarse, por lo que los controles de edici&oacute;n est&aacute;n ocultos si se selecciona dicha entrada. El bot&oacute;n "..." todav&iacute;a sirve para expandir el valor, pero el texto no se puede editar.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/thegatherview.htm
===================================================================
--- /main/tags/3.01/gli/help/es/thegatherview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/thegatherview.htm	(revision 21112)
@@ -0,0 +1,90 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thegatherview"><font size="5" face="Verdana"><strong>4.1: La pantalla de selecci&oacute;n de documentos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe el funcionamiento de la pantalla "Reunir" que se utiliza para seleccionar los archivos que se van a incluir en la colecci&oacute;n que se est&aacute; construyendo.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">C&oacute;mo ver la pantalla Reunir</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">La estructura de directorios y subdirectorios</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">El &aacute;rea de estado</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Botones de control</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">M&eacute;todos de selecci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Mapeado de carpetas especiales</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La Interfaz de Bibliotecario inicia con la pantalla "Reunir". Si despu&eacute;s usted desea regresar a esta pantalla haga click en la pesta&ntilde;a del mismo nombre que se encuentra directamente debajo de la barra de men&uacute;s.
+      <p></p>
+      Dos grandes &aacute;reas tituladas "Area de trabajo" y "Colecci&oacute;n" se utilizan para transferir los archivos a su colecci&oacute;n. Cada una contiene "&aacute;rboles de archivos", los cuales son representaciones gr&aacute;ficas de la estructura de directorios y subdirectorios de sus archivos y carpetas.
+      <p></p>
+      Seleccione un elemento del &aacute;rbol haciendo click en &eacute;l. (Tambi&eacute;n existen otras formas; v&eacute;ase m&aacute;s adelante.) Haga click en el signo que se encuentra junto a una carpeta o doble click en ella para expandir (o contraer) sus contenidos. Haga doble click en un archivo para abrirlo usando un programa de aplicaci&oacute;n asociado (vea la secci&oacute;n <a href="fileassociations.htm#fileassociations">Asociaci&oacute;n de archivos</a>).
+      <p></p>
+      El &aacute;rbol de archivos del &aacute;rea de trabajo muestra los or&iacute;genes de la informaci&oacute;n disponible al interfaz del Bibliotecario -- el sistema local de archivos (incluyendo unidades de disco y CD-ROM), los contenidos de las colecciones existentes de Greenstone, y el cach&eacute; de archivos descargados si esta activada la opci&oacute;n de poner en espejo la Web. Usted puede copiar y consultar estos archivos pero no puede moverlos, eliminarlos o editarlos, con la excepci&oacute;n de los archivos descargados, los cuales pueden ser eliminados. Navegue por este espacio para buscar los archivos que desee incluir en la colecci&oacute;n.
+      <p></p>
+      La estructura de directorios y subdirectorios de la colecci&oacute;n muestra sus contenidos. Inicialmente est&aacute; vac&iacute;a.
+      <p></p>
+      Se puede cambiar el tama&ntilde;o de las &aacute;reas oprimiendo el bot&oacute;n del mouse sobre la barra gris que separa las estructuras de directorios y subdirectorios (la forma del cursor cambia) y arrastr&aacute;ndola.
+      <p></p>
+      En la parte inferior de la ventana se muestra un &aacute;rea de estado que informa del progreso de las acciones sobre archivos (copiando, moviendo y borrando). Puede tardar un tiempo en terminar. El bot&oacute;n "Detener" interrumpe cualquier acci&oacute;n que se est&eacute; ejecutando en ese momento.
+      <p></p>
+      Dos grandes botones ocupan la esquina inferior derecha de la pantalla: "Nueva carpeta", con la imagen de una carpeta, crea nuevas carpetas (vea la secci&oacute;n <a href="creatingfolders.htm#creatingfolders">Creando carpetas</a>), en tanto que "Eliminar", con la imagen de un bote de basura, descarta los archivos. Al hacer click en el bot&oacute;n "Eliminar" se suprimir&aacute;n los archivos seleccionados de la estructura de directorios y subdirectorios de la colecci&oacute;n. Asimismo, los archivos pueden eliminarse arrastr&aacute;ndolos hacia este bot&oacute;n.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      Para seleccionar varios elementos consecutivos, seleccione el primero y a continuaci&oacute;n manteniendo oprimida la tecla de may&uacute;sculas o [Shift] haga click en el &uacute;ltimo: la selecci&oacute;n incluir&aacute; todos los elementos que se encuentran en medio. Para seleccionar varios archivos que no sean consecutivos mantenga oprimida la tecla de control o [Ctrl] mientras hace click en ellos. Utilice alternativamente estos dos m&eacute;todos para seleccionar grupos de elementos que no est&eacute;n juntos.
+      <p></p>
+      En ocasiones algunas carpetas, como la que contiene sus propias p&aacute;ginas Web, tienen un significado especial. La Interfaz de Bibliotecario puede asignar dichas carpetas al primer nivel de la estructura de directorios y subdirectorios. Para hacerlo haga click con el bot&oacute;n derecho del mouse sobre la carpeta deseada. Seleccione "Crear un acceso directo" y d&eacute;le un nombre a la carpeta. Para quitar un elemento haga click con el bot&oacute;n derecho del mouse sobre la carpeta mapeada y seleccione "Eliminar acceso directo".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/themirrorview.htm
===================================================================
--- /main/tags/3.01/gli/help/es/themirrorview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/themirrorview.htm	(revision 21112)
@@ -0,0 +1,76 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="themirrorview"><font size="5" face="Verdana"><strong>3.1: La vista en Bajar</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe como configurar una tarea de descarga y como controlar el proceso de descargue.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">El &aacute;rbol del &aacute;rea de trabajo</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Configuraci&oacute;n de descarga</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Lista de descarga</td>
+</tr>
+        
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instrucciones avanzadas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Activaci&oacute;n de "web proxies"</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Modo avanzado</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Active la vista "Bajar" haciendo clic en su pesta&ntilde;a. La mitad superior de la pantalla contiene los controles para realizar descargas. La mitad inferior est&aacute; vac&iacute;a inicialmente, pero despu&eacute;s va a mostrar una lista de tareas pendientes y terminadas.
+      <p></p>
+      Los archivos se descargan en una carpeta del espacio de trabajo denominada "Archivos Descargados" (que s&oacute;lo existe cuando est&aacute; habilitada la opci&oacute;n de trabajar en espejo), y pueden utilizarse en todas las colecciones construidas con el interfaz del Bibliotecario. En esta &aacute;rea los archivos se designan con su URL completo. Para cada "host" se crea una nueva carpeta, seguida por otras para cada parte del "path". De esta manera se asegura que todos los archivos son distintos.
+      <p></p>
+      Utilice el primero de los controles de configuraci&oacute;n de la descarga, "URL Or&iacute;gen", para capturar el URL de un recurso deseado. Utilice el control "Profundidad de la Descarga" para limitar el n&uacute;mero de hiper-v&iacute;nculos que se deben seguir al descargar: P&oacute;ngalo en 0 para descargar una sola p&aacute;gina; p&oacute;ngalo en 1 para descargar una p&aacute;gina y todas las p&aacute;ginas a las que apunta. El l&iacute;mite de profundidad se ignora al descargar medios diferentes de p&aacute;ginas html. Adem&aacute;s, hay varios controles de chequeo que se pueden prender para activar determinadas capacidades durante una descarga espec&iacute;fica. Una vez se ha establecido la configuraci&oacute;n, haga clic en "Descargar" para iniciar un nuevo proceso de descargue. Hay otros dos controles en forma de botones: "Preferencias", que enlaza con la secci&oacute;n de conexi&oacute;n de las Preferencias, en la cual se pueden editar las caracter&iacute;sticas del "proxy"; y "Borrar Cach&eacute;", que borra todos los archivos descargados previamente.
+      <p></p>
+      La lista de descargas tiene una entrada por cada p&aacute;gina web descargada. Cada entrada tiene un &aacute;rea de texto que da detalles acerca de la tarea, junto con una barra que indica el avance de la actividad en ejecuci&oacute;n. A la izquierda de cada entrada aparecen tres botones. "Pausa" se utiliza para detener temporalmente la descarga que se est&aacute; ejecutando. "Ver Bit&aacute;cora" abre una ventana que muestra el archivo de bit&aacute;cora de las descargas. "Cerrar" termina la descarga y quita de la lista la tarea correspondiente.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instrucciones avanzadas:</a>
+<p style="text-align:justify"></p>
+      La <a href="preferences.htm#preferences">secci&oacute;n Preferencias</a> describe como establecer una conexi&oacute;n Internet por medio de un "proxy".  Si se necesita autenticaci&oacute;n, el servidor "proxy" pide la identificaci&oacute;n y la palabra clave. El interfaz del Bibliotecario no almacena palabras claves entre sesiones.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/translatetext.htm
===================================================================
--- /main/tags/3.01/gli/help/es/translatetext.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/translatetext.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="translatetext"><font size="5" face="Verdana"><strong>6.10: Traducci&oacute;n de textos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n describe la pantalla de traducci&oacute;n, donde se pueden definir fragmentos de texto en un idioma espec&iacute;fico para partes de la interfaz de la colecci&oacute;n. En la parte de "Secciones de dise&ntilde;o" haga click en "Traducir texto".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">A&ntilde;adir traducci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Eliminar traducci&oacute;n</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Actualizar traducci&oacute;n</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Primero elija una entrada de la lista de "funciones". Las cadenas de texto asociadas a esta funci&oacute;n aparecer&aacute;n abajo. Utilice la lista "Idioma de la traducci&oacute;n" para seleccionar el idioma al que se va a traducir y escriba el texto traducido en el &aacute;rea de texto, haciendo referencia al "fragmento de texto inicial" en caso necesario. Una vez que haya terminado haga click en "A&ntilde;adir traducci&oacute;n".
+      <p></p>
+      Para quitar una traducci&oacute;n existente, selecci&oacute;nela de la tabla "Traducciones asignadas" y haga click en "Eliminar traducci&oacute;n".
+      <p></p>
+      Para actualizar una traducci&oacute;n, selecci&oacute;nela, ed&iacute;tela en el &aacute;rea de texto titulada "Texto traducido" y haga click en "Reemplazar traducci&oacute;n".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/updatingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/es/updatingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/updatingmetadata.htm	(revision 21112)
@@ -0,0 +1,53 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="updatingmetadata"><font size="5" face="Verdana"><strong>5.5: Actualizando metadatos</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Esta secci&oacute;n explica la forma de actualizar los metadatos asignados a un archivo.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Actualizando los metadatos de archivos</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2"> Actualizando los metadatos de carpetas o de varios archivos juntos</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      Para actualizar el valor de una parte del metadato, elija primero el archivo al que se aplicar&aacute; dicho valor y luego el elemento de metadato cuyo valor desea cambiar. Su selecci&oacute;n aparecer&aacute; en los controles de edici&oacute;n de metadatos. Edite el valor y haga click en "Reemplazar" para modificarlo.
+      <p></p>
+      El proceso es el mismo cuando se actualiza un directorio que contiene subdirectorios o m&uacute;ltiples archivos, pero s&oacute;lo se pueden actualizar metadatos que sean comunes a todos los archivos y carpetas seleccionados.
+      <p></p>
+      La estructura de valores muestra todos los valores anteriores y no s&oacute;lo aquellos que se asignaron en este momento. As&iacute; el valor que usted reemplaz&oacute; permanecer&aacute; en la estructura de valores.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/es/xcollectionsearching.htm
===================================================================
--- /main/tags/3.01/gli/help/es/xcollectionsearching.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/es/xcollectionsearching.htm	(revision 21112)
@@ -0,0 +1,47 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="xcollectionsearching"><font size="5" face="Verdana"><strong>6.7: B&uacute;squeda cruzada a trav&eacute;s de varias colecciones</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Greenstone puede buscar a trav&eacute;s de varias colecciones distintas como si fueran una sola. Esto se hace creando una "supercolecci&oacute;n" que abarca a las colecciones individuales. En la parte de "Secciones de dise&ntilde;o" haga click en "B&uacute;squeda cruzada".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instrucciones b&aacute;sicas:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Seleccionando una colecci&oacute;n</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instrucciones b&aacute;sicas:</a>
+<p style="text-align:justify"></p>
+      La pantalla de b&uacute;squeda cruzada muestra una lista de las colecciones disponibles. La colecci&oacute;n actual est&aacute; marcada y no se puede quitar de la selecci&oacute;n. Para a&ntilde;adir otra colecci&oacute;n en la cual buscar de forma paralela, haga click en ella (o haga click en ella nuevamente para quitarla de la selecci&oacute;n). Si s&oacute;lo se seleccion&oacute; una colecci&oacute;n, entonces no habr&aacute; b&uacute;squeda cruzada.
+      <p></p>
+      Para mayor informaci&oacute;n vea el Cap&iacute;tulo 1 de la Gu&iacute;a del Programador de Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/addingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/addingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/addingfiles.htm	(revision 21112)
@@ -0,0 +1,72 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingfiles"><font size="5" face="Verdana"><strong>4.3: Ajouter des fichiers</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section indique la mani&egrave;re d'inclure des fichiers dans la collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            glisser-d&eacute;placer
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            fichiers multiples
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions avanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            m&eacute;ta-donn&eacute;es existantes
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Les fichiers peuvent &ecirc;tre copi&eacute;s dans la collection par glisser-d&eacute;placer. Le pointeur &eacute;pouse la forme de l'&eacute;l&eacute;ment s&eacute;lectionn&eacute; (ou, si la s&eacute;lection comprend plus d'un, leur nombre). D&eacute;poser la s&eacute;lection dans l'arborescence de la collection pour les y copier (si la source est l'arborescence de l'espace de travail) ou les d&eacute;placer &agrave; l'int&eacute;rieur de la collection (si la source est l'arborescence de la collection).
+      <p></p>
+      Lorsque des fichiers multiples sont copi&eacute;s, ils sont tous plac&eacute; dans le dossier cible au m&ecirc;me niveau, quelle que soit la structure du dossier qui les contenait avant. Si vous copier un second fichier avec le m&ecirc;me nom dans le m&ecirc;me dossier, la question de savoir si le premier fichier doit &ecirc;tre &eacute;cras&eacute; est pos&eacute;e. Si la r&eacute;ponse est non, le fichier ne sera pas copi&eacute;, mais les autres le seront. Pour annuler toutes les autre t&acirc;ches de copie, cliquer sur le bouton "Stop".
+      <p></p>
+      Seuls les &eacute;l&eacute;ments situ&eacute;s au-dessus dans la s&eacute;lection sont d&eacute;plac&eacute;s. Un dossier est plus &eacute;lev&eacute; que son contenu. On ne peut s&eacute;lectionner en m&ecirc;me temps l'int&eacute;rieur d'un dossier et le dossier lui-m&ecirc;me.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions avanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Lorsqu'un fichier est ajout&eacute;, l'Interface Biblioth&eacute;caire cherche les dossiers sources de fichiers auxiliaires contenant une m&eacute;ta-donn&eacute;e qui lui originellement assign&eacute;es et, s'il en trouve, commence &agrave; les importer. Au fur et &agrave; mesure que l'op&eacute;ration se poursuit, une confirmation est faite (peut-&ecirc;tre plusieurs fois) de fournir des informations compl&eacute;mentaires pour ajuster la m&eacute;ta-donn&eacute;e aux jeux de m&eacute;ta-donn&eacute;es de la collection. Ces confirmations sont d&eacute;crites dans la section <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importer des m&eacute;ta-donn&eacute;es originellement assign&eacute;es </a>. Pour une explication plus d&eacute;taill&eacute;e de l'association de m&eacute;ta-donn&eacute;es avec les fichiers, lire le chapitre 2 du Guide de d&eacute;veloppement de Greenstone -- "Exploitez vos documents au maximum".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/addingpreviouslydefinedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/addingpreviouslydefinedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/addingpreviouslydefinedmetadata.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingpreviouslydefinedmetadata"><font size="5" face="Verdana"><strong>5.4: Ajouter des M&eacute;ta-donn&eacute;es pr&eacute;d&eacute;finies</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique la mani&egrave;re d'ajouter une m&eacute;ta-donn&eacute;e qui utilise des valeurs d&eacute;j&agrave; pr&eacute;sentes dans l'arborescence des valeurs.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter aux fichiers
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter aux fichiers et aux fichiers multiples
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour ajouter une m&eacute;ta-donn&eacute;e ayant une valeur existante, s&eacute;lectionner d'abord le fichier, et ensuite la valeur requise &agrave; partir de l'arborescence des valeurs, en d&eacute;veloppant si n&eacute;cessaire les dossiers de hi&eacute;rarchie. La valeur de l'entr&eacute;e s&eacute;lectionn&eacute;e appara&icirc;t automatiquement dans le champ texte Valeur. (Utiliser alternativement la valeur auto-select de l'arborescence et les caract&eacute;ristiques auto-compl&egrave;tes. Cliquer sur "Append" pour ajouter la metadata au fichier s&eacute;lectionn&eacute;. 
+      <p></p>
+      La proc&eacute;dure d'ajouter m&eacute;ta-donn&eacute;es avec des valeurs d&eacute;j&agrave; existantes aux dossiers ou aux fichiers multiples est tout &agrave; fait identique.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/appendingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/appendingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/appendingmetadata.htm	(revision 21112)
@@ -0,0 +1,76 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="appendingmetadata"><font size="5" face="Verdana"><strong>5.3: Ajout de Nouvelles M&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment ajouter des m&eacute;ta-donn&eacute;es et des valeurs &agrave; des fichiers, et comment ajouter de nouvelles valeurs pour des &eacute;l&eacute;ments de m&eacute;ta-donn&eacute;es.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter aux fichiers
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter aux r&eacute;pertoires et aux fichiers multiples
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Hi&eacute;rarchies de valeur et &eacute;dition des valeurs
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      On ajoute un &eacute;l&eacute;ment m&eacute;ta-donn&eacute;e -- &eacute;l&eacute;ment et valeur &agrave; la fois -- &agrave; un fichier. S&eacute;lectionner d'abord le fichier &agrave; partir de l'arborescence de fichier de la collection situ&eacute;e &agrave; gauche. Cette action entra&icirc;ne l'apparition de toute m&eacute;ta-donn&eacute;e pr&eacute;-assign&eacute;e  dans le tableau de droite. 
+      <p></p>
+      S&eacute;lectionner ensuite l'&eacute;l&eacute;ment m&eacute;ta-donn&eacute;e &agrave; ajouter en cliquant sur sa rang&eacute;e dans le tableau.
+      <p></p>
+      Taper la valeur dans le champ valeur. Ne pas utiliser pas le caract&egrave;re "|", d&egrave;s lors qu'il est utilis&eacute; pour construire des hi&eacute;rarchies. Cliquer enfin sur "Append" pour ajouter la nouvelle valeur comme m&eacute;ta-donn&eacute;e pour le fichier choisi. La valeur appara&icirc;t imm&eacute;diatement dans le tableau de M&eacute;ta-donn&eacute;es.
+      <p></p>
+      Vous pouvez aussi ajouter des m&eacute;ta-donn&eacute;es &agrave; un dossier, ou &agrave; plusieurs fichiers s&eacute;lectionn&eacute;s &agrave; la fois. Elle est ajout&eacute;e &agrave; tous les fichiers ou &agrave; l'int&eacute;rieur du dossier ou de la s&eacute;l&eacute;ction, et aux sous-dossiers. Garder &agrave; l'esprit  que si des m&eacute;ta-donn&eacute;es sont ajout&eacute;es &agrave; un dossier tous les nouveaux fichiers en son sein h&eacute;rite automatiquement des valeurs du dossier.
+      <p></p>
+      Si on choisit une m&eacute;ta-donn&eacute;e qui intervient dans quelques uns des fichiers s&eacute;lectionn&eacute;s et que l'on clique sur "Coller", elle est aussi ajout&eacute;e aux autres fichiers de la s&eacute;lection.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      On peut ajouter une structure aux valeurs de m&eacute;ta-donn&eacute;es en utilisant les chemins comme d&eacute;crit dans <a href="theenrichview.htm#theenrichview">L'affichage Enrichir</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/assignlanguages.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/assignlanguages.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/assignlanguages.htm	(revision 21112)
@@ -0,0 +1,77 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignlanguages"><font size="5" face="Verdana"><strong>6.6.3: Assigner Langages</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+	Cette section donne les d&eacute;tails de la mani&egrave;re de restreindre les index &agrave; des langages particuliers. On le fait en g&eacute;n&eacute;rant une partition par le biais de l'onglet "Assigner Langages" de l'Affichage "Index de Partition".
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Contr&ocirc;les de Langage 
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Ajouter langage
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Enlever langage
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Langage par d&eacute;faut
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Enlever Langage par d&eacute;faut
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+	Pour ajouter un nouveau langage, dont la partition par le biais de l'onglet "Assigner langues" conf&egrave;re un index, S&eacute;lectionner le langage souhait&eacute; &agrave; partir de la liste "Langage &agrave; ajouter" et cliquer sur "Ajouter langue".
+	<p></p>
+	Pour supprimer une langue, s&eacute;lectionner-la &agrave; partir de la liste "S&eacute;lection de Langage" et cliquer sur "Enlever Langage". 
+	<p></p>
+	Pour installer un language par d&eacute;faut, s&eacute;lectionner-la dans la liste et cliquez sur "Par d&eacute;faut".
+	<p></p>
+	Pour enlever le langage par d&eacute;faut, cliquer sur "Enlever par D&eacute;faut".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/assignpartitions.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/assignpartitions.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/assignpartitions.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignpartitions"><font size="5" face="Verdana"><strong>6.6.2: Assigner des Partitions</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+	Cette section explique l'assignation d'un filtre de partition pr&eacute;d&eacute;fini.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Ajouter une partition
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Enlever une partition
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Partition par d&eacute;faut
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Enlever partition par d&eacute;faut
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+	Apr&egrave;s avoir d&eacute;fini un filtre de sous-collection, utiliser l'onglet "Assigner des Partitions" pour construire ses indexes  (ou ceux pour un jeu de filtres.) S&eacute;lectionner le filtre (ou les filtres) de votre choix &agrave; partir de la liste "D&eacute;finir des Filtres de Sous-collection", entrer un nom pour la partition dans le champ "Nom de Partition Name", et cliquer sur "Ajouter Partition". 
+	<p></p>
+	Pour supprimer une partition, s&eacute;lectionner-la &agrave; partir de la liste et cliquer sur "Enlever Partition".
+	<p></p>
+	Pour faire d'une partition celle par d&eacute;faut, s&eacute;lectionner-la &agrave; partir de la liste et cliquer sur "Par D&eacute;faut".
+	<p></p>
+	Pour effacer la partition par d&eacute;faut, cliquez sur "Effacer d&eacute;faut".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/buildsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/buildsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/buildsettings.htm	(revision 21112)
@@ -0,0 +1,59 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="buildsettings"><font size="5" face="Verdana"><strong>7.2: Param&egrave;tres d'Importation et de Construction</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment acc&eacute;der aux diff&eacute;rents param&egrave;tres de construction et d'importation. Pour plus d'informations sur l'importation et la construction lire le chapitre 1 du Guide du D&eacute;veloppeur de Greenstone -- Comprendre le processus de construction de la collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Param&egrave;tres G&eacute;n&eacute;raux
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Param&egrave;tres d'importation
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Param&egrave;tres de construction
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Le contr&ocirc;le des diff&eacute;rents param&egrave;tres se fait de fa&ccedil;on identique &agrave; la fen&ecirc;tre "Configurer Arguments" d&eacute;crit dans la section <a href="plugins.htm#plugins">Plugiciels de documents</a>. Certains champs requi&egrave;rent des arguments num&eacute;riques, et on peut: soit les incorporer en les saisissant, soit utiliser les fl&egrave;ches Haut et Bas pour augmenter ou diminuer la valeur pr&eacute;sente (dans certains cas, l'interface limite la classe qu'on peut entrer.) D'autres sont activ&eacute;s en cliquant sur une case de v&eacute;rification (re-cliquer pour d&eacute;sactiver.)
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/classifiers.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/classifiers.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/classifiers.htm	(revision 21112)
@@ -0,0 +1,79 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="classifiers"><font size="5" face="Verdana"><strong>6.8: Classificateurs</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment assigner &agrave; une collection des "classificateurs" qui sont utilis&eacute;s pour naviguer. Sous la section "Sections de conception", cliquer sur "Classificateurs de navigation".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configuration et s&eacute;lection d'un classificateur
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter un classificateur
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever un classificateur
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configurer un classificateur
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Changer l'ordre d'un classificateur
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour ajouter un classificateur, s&eacute;lectionner-le en utilisant la liste d&eacute;roulante "Choisir classificateur &agrave; ajouter" et cliquer sur "Ajouter Classificateur Sp&eacute;cifique". une fen&ecirc;tre qui s'intitule "Configurer Arguments" appara&icirc;t ; les instructions concernant ce dialogue sont les m&ecirc;mes que pour les plugiciels (voir <a href="plugins.htm#plugins">plugiciels de Documents</a>). Une fois le nouveau classificateur configur&eacute;, il s'ajoute &agrave; la fin de la liste des "Classificateurs couramment Assign&eacute;s". 
+      <p></p>
+      Pour supprimer un classificateur, s&eacute;lectionner-le dans la liste et cliquez sur "Enlever Classificateur Choisi".
+      <p></p>
+      Pour changer les arguments d'un classificateur, s&eacute;lectionner-le sur la liste et cliquer sur "Configurer Classificateur Choisi" (ou double-cliquer sur le classificateur sur la liste.) 
+      <p></p>
+      Le tri des classificateurs dans la barre de navigation de la collection est fourni ici dans l'ordre des classificateurs. Pour le changer, s&eacute;lectionner le classificateur &agrave; d&eacute;placer et cliquer sur "D&eacute;placer vers le haut" ou "D&eacute;placer vers le bas".
+      <p></p>
+      Pour plus d'informations sur les classificateurs lisez le chapitre 2 du Guide du D&eacute;veloppeur Greenstone -- Exploitez vos documents au maximum.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/collectingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/collectingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/collectingfiles.htm	(revision 21112)
@@ -0,0 +1,20 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="collectingfiles"><font size="5" face="Verdana"><strong>4: Collecter les fichiers pour la Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    D&egrave;s lors que l'on dispose d'une nouvelle collection, la n&eacute;cessit&eacute; se fait sentir de la doter de fichiers. Ils peuvent provenir de votre espace fichier ordinaire ou d'autres collections Greenstone. Quelques uns peuvent avoir des m&eacute;ta-donn&eacute;es attach&eacute;es.Cette section d&eacute;crit comment importer des fichiers.
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/creatingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/creatingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/creatingacollection.htm	(revision 21112)
@@ -0,0 +1,102 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingacollection"><font size="5" face="Verdana"><strong>2.1: Cr&eacute;er une nouvelle collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit la mani&egrave;re de cr&eacute;er une nouvelle collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            La nouvelle collection confirm&eacute;e
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le titre de la collection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            jeux de m&eacute;ta-donn&eacute;es par d&eacute;faut
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Description de la collection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Cr&eacute;er la nouvelle collection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Annuler la nouvelle collection
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Touches de raccourci
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour cr&eacute;er la nouvelle collection, ouvrir le menu "Fichier" et choisir "Nouveau". Plusieurs champs doivent &ecirc;tre remplis -- mais leurs valeurs sont susceptibles d'&ecirc;tre modifi&eacute;es &agrave; souhait dans l'affichage de la conception.
+      <p></p>
+      "Titre de la collection" est le texte affich&eacute; en haut de la page d'accueil de votre collection. Sa longueur peut &ecirc;tre quelconque.
+      <p></p>
+      "Description du contenu" doit d&eacute;crire avec le plus de d&eacute;tails possibles ce que la collection est. Utiliser la touche [Entr&eacute;e] pour la diviser en paragraphes.
+      <p></p>
+      On doit enfin sp&eacute;cifier soit que la nouvelle collection aura la m&ecirc;me forme et les m&ecirc;mes jeux de m&eacute;ta-donn&eacute;es que la collection existante, soit qu'elle est initi&eacute;e  "Nouvelle collection" par d&eacute;faut.
+      <p></p>
+      Cliquer sur "OK" pour cr&eacute;er la collection. Si on choisit "Nouvelle collection", les jeux de m&eacute;ta-donn&eacute;es disponibles sont affich&eacute;es. On peut en choisir plus d'une et en ajouter d'autres ult&eacute;rieurement.
+      <p></p>
+      Le clic sur "Annuler" ram&egrave;ne imm&eacute;diatement l'&eacute;cran principal.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Les boutons, tels les menus, ont un caract&egrave;re soulign&eacute;. Pour cliquer sur le bouton, appuyer en m&ecirc;me temps sur [Alt] et sur le caract&egrave;re soulign&eacute;.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/creatingfolders.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/creatingfolders.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/creatingfolders.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingfolders"><font size="5" face="Verdana"><strong>4.2: Cr&eacute;er des dossiers</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section indique la mani&egrave;re de cr&eacute;er de nouveaux dossiers
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le bouton Nouveau Dossier
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le menu bouton-droit
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Utiliser les dossiers de l'arborescence de fichiers de la collection pour rejeur les fichiers et les rendre ais&eacute;ment accessibles. Des dossiers peuvent &ecirc;tre plac&eacute;s &agrave; l'int&eacute;rieur d'autres dossiers. Aucune limite, virtuellement parlant, ni au nombre de dossiers dont on peut disposer ni &agrave; la complexit&eacute; de leur agencement.
+      <p></p>
+      Pour cr&eacute;er un nouveau dossier, on peut opter pour un dossier existant et cliquer sur le bouton Nouveau Dossier. Le nouveau dossier appara&icirc;t &agrave; l'int&eacute;rieur de celui choisi, ou au niveau sup&eacute;rieur si aucun n'est choisi. Le nom de ce dossier est r&eacute;clam&eacute; ( &agrave; d&eacute;faut, il sera nomm&eacute; "Nouveau Dossier").
+      <p></p>
+      Les dossiers peuvent aussi &ecirc;tre cr&eacute;&eacute;s en cliquant sur le dossier avec le bouton droit, et en choisissant "Nouveau Dossier" et proc&eacute;der comme ci-dessus.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/definefilters.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/definefilters.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/definefilters.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="definefilters"><font size="5" face="Verdana"><strong>6.6.1: D&eacute;finir les Filtres</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+	Cette section explique comment d&eacute;finir un filtre de partition.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ajouter un filtre
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Enlever un filtre
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Mettre &agrave; jour un filtre
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+	Les filtres permettent de jeur en sous-collection tous les documents dans un index pour laquelle une valeur m&eacute;ta-donn&eacute;e correspond &agrave; un type donn&eacute;.
+	<p></p>
+	Pour cr&eacute;er un filtre, cliquer sur le bouton "D&eacute;finir les Filtres" et entrer un nom pour le nouveau filtre dans le champ "Nom du filtre de la sous-collection". Ensuite choisir un attribut de document &agrave; faire correspondre  soit avec l'&eacute;l&eacute;ment de m&eacute;ta-donn&eacute;e soit avec le nom du fichier en question. Saisir une expression courante qu'il faut utiliser durant la correspondance. Vous pouvez choisir entre "Inclure" les documents qui correspondent au fichier ou les "Exclure". Enfin, on peut sp&eacute;cifier l'un quelconque des drapeaux standard PERL d'expression courante qu'il faut utiliser pour la correspondance (par exemple "i" pour les correspondances ne tenant pas compte des caract&egrave;res.) Pour finir, cliquez sur "Ajouter un Filtre" pour ajouter le filtre &agrave; la liste "Filtres de sous collection d&eacute;finis". 
+	<p></p>
+	Pour enlever un filtre, s&eacute;lectionner-le sur la liste et cliquer sur "Enlever Filtre".
+	<p></p>
+	Pour changer un filtre, s&eacute;lectionner-le sur la liste, changer l'une quelconque des valeurs apparaissant dans les contr&ocirc;les d'&eacute;dition et cliquer sur "Remplacer Filtre" pour valider les modifications.
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/deletingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/deletingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/deletingcollections.htm	(revision 21112)
@@ -0,0 +1,47 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="deletingcollections"><font size="5" face="Verdana"><strong>2.4: Suppression des collections</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment supprimer d&eacute;finitivement des collections de votre installation Greenstone.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Comment supprimer des collections
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour supprimer d&eacute;finitivement des collections de votre installation Greenstone, choisir "Supprimer..." dans le menu "Fichier". Une liste de vos collections Greenstone appara&icirc;t. Selectionner une collection pour vior sa description, puis cocher la case en bas du dialogue et clicquez "Supprimer" pour supprimer la collection. Cette operation est irr&eacute;versible, v&eacute;rifiez bien que vous n'avez plus besoin de cette collection avant de supprimer!
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/designingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/designingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/designingacollection.htm	(revision 21112)
@@ -0,0 +1,26 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="designingacollection"><font size="5" face="Verdana"><strong>6: D&eacute;coration de l'apparence de votre collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+Une fois les fichiers marqu&eacute;s avec les m&eacute;ta-donn&eacute;es, on peut d&eacute;cider comment ils devraient appara&icirc;tre aux utilisateurs comme une collection Greenstone. Quel type d'information peut &ecirc;tre recherch&eacute;? Quels sont les moyens qui sont fournis pour naviguer &agrave; travers les documents? Quels sont les langages support&eacute;s? o&ugrave; apparaissent les boutons sur la page? ces choses peuvent &ecirc;tre adapt&eacute;es; cette section d&eacute;crit comment proc&eacute;der.
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/downloadingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/downloadingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/downloadingfiles.htm	(revision 21112)
@@ -0,0 +1,16 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="downloadingfiles"><font size="5" face="Verdana"><strong>3: T&eacute;l&eacute;chargement de fichiers de lIinternet</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    La vue "T&eacute;l&eacute;chargement" vous permet de t&eacute;l&eacute;charger des resources de l'internet. Cette section explique le processus de miroir de l'interface biblioth&eacute;caire.
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/enrichingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/enrichingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/enrichingacollection.htm	(revision 21112)
@@ -0,0 +1,23 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="enrichingacollection"><font size="5" face="Verdana"><strong>5: Enrichir la Collection avec des m&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Apr&egrave;s avoir rassembl&eacute; plusieurs fichiers dans la collection, on les enrichit d'une information compl&eacute;mentaire appel&eacute;e "m&eacute;ta-donn&eacute;e". Cette section explique comment une m&eacute;tadonn&eacute;e est cr&eacute;&eacute;e, &eacute;dit&eacute;e, assign&eacute;e ou retir&eacute;e, et comment utiliser les sources de m&eacute;ta-donn&eacute;es externes (voir aussi le chapitre 2 du Guide de d&eacute;veloppement de Greenstone -- "Exploitez vos documents au maximum").
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/exportingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/exportingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/exportingcollections.htm	(revision 21112)
@@ -0,0 +1,49 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="exportingcollections"><font size="5" face="Verdana"><strong>8.3: Exportation de collections vers CD-ROM</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit comment exporter des collections vers un CD-ROM d'auto-installation Greenstone.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Comment exporter</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Greenstone peut exporter une ou plusieurs collections vers un CD-ROM d'auto-installation pour Windows. Pour ce faire, le paquetage Greenstone "Export to CD-ROM" doit &ecirc;tre install&eacute;. Il n'est pas inclut par d&eacute;faut, donc vous aurez peut &ecirc;tre &agrave; modifier votre installation pour l'inclure.
+      <p></p>
+      Pour exporter une collection, ouvrir le menu "Fichier" et choisir "Exporter vers CD-ROM". Une liste de collections Greenstone appara&icirc;t; cliquez sur n'importe laquelle pour obtenir une description. Cochez la case des collections &agrave; exporter. Vous pouvez entrer le nom du CD-ROM dans la case: c'est ce qui va appara&icirc;tre dans le menu de d&eacute;marrage lorsque le CD-ROM aura &eacute;t&eacute; install&eacute;. Puis cliquez sur "Exporter". Ce processus implique la copie de plusieurs fichiers et peut durer quelques minutes.
+      <p></p>
+      A la fin de l'op&eacute;ration, Greenstone va afficher le nom d'un dossier contenant les collections export&eacute;es. Utilisez un graveur de CD pour copier le contenu dans un CD vierge.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/fileassociations.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/fileassociations.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/fileassociations.htm	(revision 21112)
@@ -0,0 +1,67 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="fileassociations"><font size="5" face="Verdana"><strong>8.2: Associations de fichiers</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      L'Interface Biblioth&eacute;caire utilise des programmes d'application particuli&egrave;res pour ouvrir des types de fichiers particuliers. Cette section explique comment assigner et &eacute;diter ces associations de fichiers. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+           Ajouter association
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Editer association
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever association
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour modifier des associations ouvrez le menu "Fichier" et cliquer sur "Association de fichiers". 
+      <p></p>
+      Pour ajouter une association, s&eacute;lectionner l'extension du fichier cible dans la liste d&eacute;roulant, ou taper dans une nouvelle extension (n'y incluer pas le ".") Ensuite soit taper la commande qui lance l'application voulue dans la champ appropri&eacute;, soit choisir l'application dans le dialogue "Navigation". "%1" Peut &ecirc;tre utilis&eacute; dans la commande de lancement pour ins&eacute;rer le nom du fichier qu'on veut ouvrir. Une fois cela, "Ajouter Association" est activ&eacute;. Cliquer dessus pour ajouter l'association.
+      <p></p>
+      Pour &eacute;diter une association, s&eacute;lectionner une extension de fichier existante. Toute commande existante associ&eacute;e est affich&eacute;e dans le champ de la commande de lancement. Corriger-la, et puis cliquer sur "Remplacer Association".
+      <p></p>
+      Pour enlever une association, s&eacute;lectionner une extension de fichier existante et cliquer sur "Enlever Association". (L'extension du fichier reste dans la liste d&eacute;roulante "For Files Ending").
+      <p></p>
+      Les associations de fichiers sont stock&eacute;es dans le dossier principal du r&eacute;pertoire principal de l'Interface Biblioth&eacute;caire, dans un fichier appel&eacute; "associations.xml".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/filteringthetree.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/filteringthetree.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/filteringthetree.htm	(revision 21112)
@@ -0,0 +1,62 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="filteringthetree"><font size="5" face="Verdana"><strong>4.5: Filtrer l'arborescence</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Le "Filtrage" de l'arborescence de la collection permet de r&eacute;duire la recherche de fichiers particuliers.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le contr&ocirc;le du filtre
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Filtrage usuel
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Le menu d&eacute;roulant des "Montrer Fichiers" ("Show Files") au bas de chaque arborescence montre une liste de filtres pr&eacute;d&eacute;finis, tels "Images". Son choix cache temporairement tous les autres fichiers de l'arborescence. Pour restorer celle-ci, changer le filtre en "Tous Fichiers". Ces op&eacute;rations n'alt&egrave;rent ni la collection, ni n'affectent les dossiers dans l'arborescence.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      On peut sp&eacute;cifier un filtre usuel en saisissant un mod&egrave;le assorti aux fichiers vis&eacute;s. Utiliser un syst&egrave;me standard d'abr&eacute;viations de fichiers telles que "*.*" ou "*.doc" (*. concerne tous les caract&egrave;res).
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/formatstatements.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/formatstatements.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/formatstatements.htm	(revision 21112)
@@ -0,0 +1,90 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="formatstatements"><font size="5" face="Verdana"><strong>6.9: Caract&eacute;ristiques de Format</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Les Format Features et les commandes de format contr&ocirc;lent la structure et l'apparence de la collection. Ils influencent le lieu d'apparition des boutons lorsqu'un document est affich&eacute;, et d&eacute;terminent les liens qui sont affich&eacute;s par le classificateur DateList. Les commandes de format ne sont pas faciles &agrave; concevoir, et la lecture du chapitre 2 de Greenstone Developer's Guide est recommand&eacute;e. Cette section concerne les cadres de format, et  indique comment l'Interface Biblioth&eacute;caire en fournit l'acc&egrave;s. Sous "Sections de conception", cliquer sur "Caract&eacute;ristiques de Format".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Formatage
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter commande de format
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Enlever commande de format
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Mettre &agrave; jour commande de format
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Options de formatage &eacute;tendues
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      On peut appliquer une commande de format &agrave; n'importe quoi dans la liste d&eacute;roulante "Choisir Caract&eacute;ristique" qui comprend tous les classificateurs de m&ecirc;me qu'une liste pr&eacute;d&eacute;finie de caract&eacute;ristiques. Deux types de contr&ocirc;les existent lorsqu'on s&eacute;lectionne une caract&eacute;ristique: Certaines caract&eacute;ristiques sont simplement activ&eacute;es ou d&eacute;sactiv&eacute;es, et le contr&ocirc;le se fait gr&acirc;ce &agrave; une liste de v&eacute;rification. D'autres requi&egrave;rent qu'on sp&eacute;cifie une cha&icirc;ne de format. Pour ces caract&eacute;ristiques, il existe une liste de retrait ("Composante Affect&eacute;e") pour la s&eacute;lection des parties de la caraxt&eacute;ristique o&ugrave; s'applique la cha&icirc;ne (si n&eacute;cessaire), une zone de texte ("Cha&icirc;ne de Format HTML") o&ugrave; on saisit la cha&icirc;ne, et une s&eacute;lection de "Variables" pr&eacute;d&eacute;finies. Pour ins&eacute;rer une variable dans la position courante dans la cha&icirc;ne de format, s&eacute;lectionner la &agrave; partir de la liste de retrait et cliquer sur "Ins&eacute;rer". 
+      <p></p>
+      Vous pouvez sp&eacute;cifier un format par d&eacute;faut pour une composante particuli&egrave;re en s&eacute;lectionnant la caract&eacute;ristique vierge.Ce format est alors appliqu&eacute; &agrave; toutes les caract&eacute;ristiques applicables sauf instruction contraire. 
+      <p></p>
+      Pour ajouter une nouvelle commande de format, remplir l'information comme indiqu&eacute;e plus haut et cliquer sur "Ajouter Format". La nouvelle commande de Format appara&icirc;t dans la liste des  "Commandes de Format fr&eacute;quemment Assign&eacute;es". Seule une commande de format peut &ecirc;tre assign&eacute;e pour chaque combinaison caract&eacute;ristique/composante.
+      <p></p>
+      Pour enlever une commande de format, s&eacute;lectionner-la sur la liste et cliquer "Enlever Format".
+      <p></p>
+      Pour modifier une commande de format, s&eacute;lectionner-la  sur la liste et cliquer sur "Remplacer Format". 
+      <p></p>
+      Pour de plus amples informations sur les variables et les composantes de caract&eacute;ristiques, lire le chapitre 2 du Guide du D&eacute;veloppeur de Greenstone. 
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Si la bo&icirc;te de v&eacute;rification "Permettre options &eacute;tendues" est coch&eacute;e, certaines options avanc&eacute;es de formatage sont activ&eacute;es. La liste des caract&eacute;ristiques pouvant &ecirc;tre format&eacute;es est l&eacute;g&egrave;rement chang&eacute;e, et plusieurs variables sont disponibles dans la commande de format, offrant un contr&ocirc;le plus grand sur la mise en forme. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/generalsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/generalsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/generalsettings.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="generalsettings"><font size="5" face="Verdana"><strong>6.2: R&eacute;glages g&eacute;n&eacute;raux</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment passer en revue et changer les r&eacute;glages g&eacute;neraux associ&eacute;s &agrave; la collection. D'abord, sous "Design section" cliquer "General".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            R&eacute;glages g&eacute;n&eacute;raux
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Les valeurs fournies ici peuvent &ecirc;tre modifi&eacute;es au cours de la cr&eacute;ation de la collection.
+      <p></p>
+      En haut de la page, se trouve une bo&icirc;te &agrave; instruction, qui appara&icirc;t pour chacune des diff&eacute;rentes sections. Elle contient une br&egrave;ve liste d'instructions pour vous rappeler quelle fonctionalit&eacute; est disponible:
+      <p></p>
+      D'abord, les contacts E-mail du cr&eacute;ateur et mainteneur de la collection. Le champ suivant permet le changement de titre de la collection. Le dossier dans lequel est enregistr&eacute; la collection est affich&eacute; ensuite, mais ne peut &ecirc;tre &eacute;dit&eacute;. Celui qui suit sp&eacute;cifie (sous forme d'URL) l'&icirc;cone &agrave; montrer &agrave; l'extr&eacute;mit&eacute; sup&eacute;rieure gauche de la page "About" de la collection et la suivante est l'&icirc;cone utilis&eacute;e dans la page de la biblioth&eacute;que Greenstone &agrave; lier &agrave; la collection. Puis, une case permet de contr&ocirc;ler si la collection est accessible au public. Enfin, la zone texte de "la description de la Collection" comme d&eacute;crit <a href="creatingacollection.htm#creatingacollection">Cr&eacute;er une nouvelle collection</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/help.xml
===================================================================
--- /main/tags/3.01/gli/help/fr/help.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/help.xml	(revision 21112)
@@ -0,0 +1,1018 @@
+<Document>
+  <Section name="introduction">
+    <Title>Introduction</Title>
+    L'interface bibliothÃ©caire de Greenstone est un outil pour collecter et enrichir des documents et ensuite mettre sur pied des collections de bibliothÃšques numÃ©riques. Il fournit un accÃšs Ã  la fonctionnalitÃ© du logiciel de la BibliothÃšque numÃ©rique de Greenstone Ã  partir d'un point de vue graphique et d'une interface "clic".
+    <Section name="ofmiceandmenus">
+      <Title>Dessouris et Menus</Title>
+      La section "Dessouris et Menus" fournit une information de base sur la maniÃšre de dialoguer avec l'interface bibliothÃ©caire. Si on est familier avec des programmes comme Internet Explorer ou Microsoft Office et aussi Ã  l'aise avec les menus et les clics de la souris, passer Ã  <Reference target="howtoavoidthisdocument">section suivante</Reference>.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Concepts de base
+            <ContentsItem>TÃ¢ches de la souris</ContentsItem>
+            <ContentsItem>Clavier</ContentsItem>
+          </ContentsItem>
+          <ContentsItem>Quitter le programme</ContentsItem>
+	</ContentsGroup>
+	<ContentsGroup linkto="advanced">
+          <ContentsItem>Touches de raccourcis</ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      L'interface bibliothÃ©caire suit les conventions de Microsoft Windows et requiert une connaissance de base de Windows.
+      <Break/>
+      Une position quelconque d'interaction sur l'Ã©cran, tel qu'un bouton ou un champ de texte, est appelÃ©e "contrÃŽle". A tout moment, un contrÃŽle appelÃ© "Foyer" est mis en relief et rÃ©pond au clavier. Plusieurs contrÃŽles vous permettent de sÃ©lectionner les parties qui sont mises en relief en bleu foncÃ©. Certains contrÃŽles virent au gris pour indiquer qu'ils sont Ã©teints.
+      <Break/>
+      On peut dÃ©placer la souris, cliquer sur son bouton gauche (cliquer-gauche) ou sur son bouton droit (cliquer-droite) de faÃ§on tout Ã  fait usuelle. Plusieurs Ã©lÃ©ments sont susceptibles d'Ãªtre entrainÃ©s par glissement de la souris son bouton gauche Ã©tant maintenu appuyÃ©, et d'Ãªtre ainsi dÃ©posÃ©s quelque part en relÃ¢chant le bouton. Des cibles de dÃ©pÃŽt potentielles changent d'apparence quand un composant les atteint.
+      <Break/>
+      On peut utiliser le clavier pour la saisie Ã  l'intÃ©rieur des champs de texte. Des alternatives au clavier sont disponibles pour plusieurs contrÃŽles indiquÃ©s par un nom clÃ© entre crochets --- par exemple [Tab] change le foyer. Le signe plus indique les autres touches susceptibles d'Ãªtre appuyÃ©es en mÃªme temps.
+      <Break/>
+      Pour quitter l'Interface BibliothÃ©caire, on utilise "Sortir" du menu "Fichier". Votre collection sera d'abord sauvegardÃ©e.
+      <Anchor name="advanced">Instructions avancÃ©es:</Anchor>
+      Pour accÃ©der au menu, appuyer sur le bouton [Alt] et sur la lettre correspondante (soulignÃ©e). Par exemple, pour le menu "Fichier" appuyer sur [Alt] + [F]. Pour choisir un Ã©lÃ©ment, appuyer sur la touche correspondante. Par exemple, lorsque le menu "Fichier" est dÃ©roulÃ©, appuyer sur "S" pour sauvegarder la collection.
+    </Section>
+    <Section name="howtoavoidthisdocument">
+      <Title>Comment Ã©viter de lire ce document</Title>
+      Ne pas lire l'Aide en entier! Lire juste ce qui est nÃ©cessaire pour obtenir l'aide dont on a besoin.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>Aide contextuelle intuitive</ContentsItem>
+          <ContentsItem>Outils (Tool tips)</ContentsItem>
+          <ContentsItem>Autres documents</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      L'Ã©lÃ©ment de menu Aide indique l'Ã©lÃ©ment d'aide le plus appropriÃ© par une icÃŽne sous forme de petit livre. 
+      <Break/>
+      Le positionnement de la souris sur plusieurs contrÃŽles entraÃ®ne l'apparition d'un "Tool tip" qui dit ce qu'il faut faire.
+      <Break/>
+      Avant d'utiliser l'Interface BibliothÃ©caire, il faut lire la documentation Greenstone. La section "CrÃ©er une Nouvelle Collection" dÃ©crit comment crÃ©er une nouvelle collection. Avant d'utiliser l'Interface BiblithÃ©caire, lire d'abord la documentation de Greenstone.
+    </Section>
+  </Section>
+
+  <Section name="startingoff">
+    <Title>DÃ©marrage</Title>
+    Cette section explique comment crÃ©er, charger, enregistrer et supprimer des collections.
+    <Section name="creatingacollection">
+      <Title>CrÃ©er une nouvelle collection</Title>
+      <ObviousSentence>Cette section dÃ©crit la maniÃšre de crÃ©er une nouvelle collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            La nouvelle collection confirmÃ©e
+          </ContentsItem>
+          <ContentsItem>
+            Le titre de la collection
+          </ContentsItem>
+          <ContentsItem>
+            jeux de mÃ©ta-donnÃ©es par dÃ©faut
+          </ContentsItem>
+          <ContentsItem>
+            Description de la collection
+          </ContentsItem>
+          <ContentsItem>
+            CrÃ©er la nouvelle collection
+          </ContentsItem>
+          <ContentsItem>
+            Annuler la nouvelle collection
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Touches de raccourci
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour crÃ©er la nouvelle collection, ouvrir le menu "Fichier" et choisir "Nouveau". Plusieurs champs doivent Ãªtre remplis -- mais leurs valeurs sont susceptibles d'Ãªtre modifiÃ©es Ã  souhait dans l'affichage de la conception.
+      <Break/>
+      "Titre de la collection" est le texte affichÃ© en haut de la page d'accueil de votre collection. Sa longueur peut Ãªtre quelconque.
+      <Break/>
+      "Description du contenu" doit dÃ©crire avec le plus de dÃ©tails possibles ce que la collection est. Utiliser la touche [EntrÃ©e] pour la diviser en paragraphes.
+      <Break/>
+      On doit enfin spÃ©cifier soit que la nouvelle collection aura la mÃªme forme et les mÃªmes jeux de mÃ©ta-donnÃ©es que la collection existante, soit qu'elle est initiÃ©e  "Nouvelle collection" par dÃ©faut.
+      <Break/>
+      Cliquer sur "OK" pour crÃ©er la collection. Si on choisit "Nouvelle collection", les jeux de mÃ©ta-donnÃ©es disponibles sont affichÃ©es. On peut en choisir plus d'une et en ajouter d'autres ultÃ©rieurement.
+      <Break/>
+      Le clic sur "Annuler" ramÃšne immÃ©diatement l'Ã©cran principal.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      Les boutons, tels les menus, ont un caractÃšre soulignÃ©. Pour cliquer sur le bouton, appuyer en mÃªme temps sur [Alt] et sur le caractÃšre soulignÃ©.
+    </Section>
+    <Section name="savingacollection">
+      <Title>Enregistrer la Collection</Title>
+      <ObviousSentence>Cette section dÃ©crit la maniÃšre d'enregistrer une collection, et la confirmation d'enregistrement de la collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Comment enregistrer
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            OÃ¹ sont sauvegardÃ©s les fichiers
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Enregistrer rÃ©guliÃšrement votre travail en choisissant "Enregistrer" dans le fichier "Menu". Enregistrer une collection ne signifie pas la rendre prÃªte Ã  l'utilisation dans Greenstone (see <Reference target="producingthecollection">Confectionner votre Collection</Reference>).
+      <Break/>
+      L'Interface BiblithÃ©caire protÃšge votre travail en l'enregistrant Ã  la fermeture du programme ou Ã  l'ouverture d'une autre collection.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      Les collections sont enregistrÃ©es en un fichier choisi pour la collection avec une extension ".col", localisÃ© dans un rÃ©pertoire de mÃªme nom Ã  l'intÃ©rieur du dossier "collect" de l'installation de Greenstone.
+    </Section>
+    <Section name="openingacollection">
+      <Title>Ouvrir une collection existante</Title>
+      <ObviousSentence>Cette section indique la maniÃšre d'ouvrir des collections existantes en utilisant "PrÃªt pour Ouvrir Collection".</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>Comment ouvrir la Collection</ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>Collections verrouillÃ©es</ContentsItem>
+	  <ContentsItem>Collections de versions antÃ©rieures</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour ouvrir une collection existante, choisir "ouvrir" Ã  partir du menu "Fichier" pour obtenir le "PrÃªt Ã  l'ouverture de la collection". Une liste de vos collections Greenstone apparaÃ®t. En sÃ©lectionner une pour voir sa description, et cliquer sur "Ouvrir" pour la charger. Si l'on cherche une collection qui est hors du dossier "Collect" de Greenstone, cliquer sur "Parcourir" pour un dialogue avec le systÃšme de fichiers concernÃ©.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      Dans le cas oÃ¹ plusieurs programmes d'Interface BibliothÃ©caire de Greenstone sont exÃ©cutÃ©s simultanÃ©ment, les rÃ©pertoires concernÃ©s sont vÃ©rouillÃ©s pour prÃ©venir une interfÃ©rence. A l'ouverture d'une collection, un petit fichier de verrouillage temporaire est crÃ©Ã© dans son dossier. Avant d'ouvrir une collection, l'interface BibliothÃ©caire vÃ©rifie qu'aucun fichier de verrouillage n'existe dÃ©jÃ . Le verrouillage d'une collection est indiquÃ© par la couleur de son icÃŽne: verte pour une collection normale, rouge quand elle est vÃ©rouillÃ©e. Cependant, quand l'on quitte l'Interface BibliothÃ©caire prÃ©maturÃ©ment, le fichier de verrouillage est parfois laissÃ© en place. Quand une telle collection est ouverte, l'Interface BibliothÃ©caire demande si l'on veut se dÃ©faire de son contrÃŽle. Ne jamais sosutraire une collection en cours d'utilisation par un autre.
+      <Break/>
+      Lorsque vous ouvrez une collection qui n'a pas Ã©tÃ© crÃ©Ã©e par Greenstone Librarian Interface, on vous demande de sÃ©lectionner un ou plusieurs jeux de mÃ©ta-donnÃ©es. Si vous n'en sÃ©lectionnez aucun, les mÃ©ta-donnÃ©es existantes seront ignorÃ©es. Dans le cas contraire, les mÃ©ta-donnÃ©es seront importÃ©es comme lorsque vous cliquez et glisser dans des fichiers contenant des mÃ©ta-donnÃ©es. Ce processus est dÃ©crit dans la section <Reference target="importingpreviouslyassignedmetadata">Importation de MÃ©ta-donnÃ©es PrÃ©cÃ©demment AssignÃ©es</Reference>.
+    </Section>
+    <Section name="deletingcollections">
+      <Title>Suppression des collections</Title>
+      <ObviousSentence>Cette section explique comment supprimer dÃ©finitivement des collections de votre installation Greenstone.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Comment supprimer des collections
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour supprimer dÃ©finitivement des collections de votre installation Greenstone, choisir "Supprimer..." dans le menu "Fichier". Une liste de vos collections Greenstone apparaÃ®t. Selectionner une collection pour vior sa description, puis cocher la case en bas du dialogue et clicquez "Supprimer" pour supprimer la collection. Cette operation est irrÃ©versible, vÃ©rifiez bien que vous n'avez plus besoin de cette collection avant de supprimer!
+    </Section>
+  </Section>
+
+  <Section name="downloadingfiles">
+    <Title>TÃ©lÃ©chargement de fichiers de lIinternet</Title>
+    La vue "TÃ©lÃ©chargement" vous permet de tÃ©lÃ©charger des resources de l'internet. Cette section explique le processus de miroir de l'interface bibliothÃ©caire.
+    <Section name="themirrorview">
+      <Title>La vue TÃ©lÃ©chargement</Title>
+      Cette section dÃ©crit comment on configure une tache de tÃ©lÃ©chargement et comment on contrÃŽle le tÃ©lÃ©chargement.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            L'arbre espace de travail
+          </ContentsItem>
+          <ContentsItem>
+            Configuration de tÃ©lÃ©chargement
+          </ContentsItem>
+          <ContentsItem>
+            La liste des tÃ©lÃ©chargements
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Activation des proxies web
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      AccÃ©dez Ã  la vue "TÃ©lÃ©chargement" en cliquant son index. La moitiÃ© supÃ©rieure de l'Ã©cran montre le contrÃŽle des tÃ©lÃ©chargements. La moitiÃ© infÃ©rieure est initialement vide, mais montrera la liste des tÃ©lÃ©chargements en attente et rÃ©alisÃ©s.
+      <Break/>
+      Les fichiers sont tÃ©lÃ©chargÃ©s dans un dossier de l'espace de travail appelÃ© Â«Â Fichiers TÃ©lÃ©chargÃ©sÂ Â» (Uniquement prÃ©sent lorsque l'opÃ©ration Â«Â miroirÂ Â» est activÃ©e), et peut Ãªtre utilisÃ© pour toutes les collections construites avec l'interface bibliothÃ©caire. Les fichiers dans cette zone portent comme nom leur URL en entier. Un nouveau dossier est crÃ©Ã© pour chaque site hÃŽte, suivi par d'autres dossiers pour chaque Ã©lÃ©ment du chemin. Cela permet de s'assurer que chaque fichier a un nom distinct.
+      <Break/>
+      Utilisez le premier des contrÃŽles de la configuration de tÃ©lÃ©chargement, Â« Source URL Â», pour entrer l'URL de la ressource cible. Utilisez le contrÃŽle Â« Profondeur de TÃ©lÃ©chargement Â»  pour limiter le nombre de liens hypertextes Ã  suivre pour le tÃ©lÃ©chargement : utiliser 0 pour tÃ©lÃ©charger une seule page ; utiliser 1 pour tÃ©lÃ©charger la page et toutes les pages rÃ©fÃ©rencÃ©es par des liens hypertextes. La profondeur limite est ignorÃ©e lors du tÃ©lÃ©chargement de mÃ©dia autres que des pages html. Ensuite il y a plusieurs cases Ã  cocher qui peuvent Ãªtre cochÃ©es pour activer l'option spÃ©cifiÃ©e. Une fois la configuration prÃªte, cliquez Â« TÃ©lÃ©charger Â» pour dÃ©marrer l'opÃ©ration de tÃ©lÃ©chargement. Il y a deux autres boutons de contrÃŽle : Â« PrÃ©fÃ©rences Â», qui renvoie Ã  la section connexion de PrÃ©fÃ©rences ou les paramÃštre du proxy peuvent Ãªtre Ã©ditÃ©s ; et Â« Effacer la cache Â» qui supprimes tous les fichiers tÃ©lÃ©chargÃ©s prÃ©cÃ©demment.
+      <Break/>
+      La liste de tÃ©lÃ©chargement a une entrÃ©e pour chaque tÃ©lÃ©chargement de page Web.  Chaque entrÃ©e a une rÃ©gion de textes qui donne les dÃ©tails de la tÃ¢che avec une barre de progrÃšs montrant l'activitÃ© courante.  Trois boutons apparaissent Ã  la gauche de chaque entrÃ©e.  "Pause" est employÃ© pour pauser une tÃ¢che en cours de tÃ©lÃ©chargement.  "Voir le rÃ©sumÃ©" ouvre une fenÃªtre montrant l'historique du tÃ©lÃ©chargement.  "Fermer" termine le tÃ©lÃ©chargement et enlÃšve la tÃ¢che de la liste.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      La <Reference target="preferences">section PrÃ©fÃ©rences</Reference> explique comment Ã©tablir une connection Internet Ã  travers un proxy.  Si l'authentification est nÃ©cessaire, le proxy server demande l'identification et le mot de passe.  L'interface bibliothÃ©caire n'enregistre pas les mots de passe entre les sessions.
+    </Section>
+  </Section>
+
+  <Section name="collectingfiles">
+    <Title>Collecter les fichiers pour la Collection</Title>
+    DÃšs lors que l'on dispose d'une nouvelle collection, la nÃ©cessitÃ© se fait sentir de la doter de fichiers. Ils peuvent provenir de votre espace fichier ordinaire ou d'autres collections Greenstone. Quelques uns peuvent avoir des mÃ©ta-donnÃ©es attachÃ©es.Cette section dÃ©crit comment importer des fichiers.
+    <Section name="thegatherview">
+      <Title>L'affichage Rassembler</Title>
+cette section introduit la zone Rassembler utilisÃ©e pour selectionner les fichiers Ã  inclure dans la collection Ã  crÃ©er. 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+           comment afficher l'Ã©cran "gather"
+          </ContentsItem>
+          <ContentsItem>
+            Les arborescences de fichiers
+          </ContentsItem>
+          <ContentsItem>
+            La zone d'Ã©tat
+          </ContentsItem>
+          <ContentsItem>
+            Boutons de contrÃŽle
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            mÃ©thodes de sÃ©lection
+          </ContentsItem>
+          <ContentsItem>
+            Application de dossiers spÃ©ciaux
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      L'Interface BibliothÃ©caire dÃ©marre avec l'affichage "Rassembler". Pour revenir ultÃ©rieurement Ã  celui-ci, cliquer directement sur l'onglet "Rassembler" au bas de la barre des menus.
+      <Break/>
+      Les deux vastes zones intitulÃ©es "Espace de travail" et "Collection" servent au transfert des fichiers vers la collection. Elles contiennent des arborescences de fichiers, des structures graphiques qui reprÃ©sentent des fichiers et des dossiers.
+      <Break/>
+      Choisir un Ã©lÃ©ment dans l'arborescence en cliquant dessus. (d'autres maniÃšres de le faire existent; voir ci-dessous). Double-cliquer sur un dossier, ou cliquer simplement sur le symbole de commutateur placÃ© Ã  cÃŽtÃ© de lui, pour dÃ©velopper (ou rÃ©duire) son contenu. Double-cliquer sur un fichier pour l'ouvrir en utilisant un programme d'application associÃ© (voir <Reference target="fileassociations">Association de fichiers</Reference>).
+      <Break/>
+      L'arbre des fichiers de l'espace de travail montre les sources de donnÃ©es disponibles pour l'interface bibliothÃ©caire -- le systÃšme de fichiers local (ainsi que les unitÃ©s de disques et Cd-rom), le contenu des collection Greenstone, et la cache des fichiers tÃ©lÃ©chargÃ©s si l'option Web Â«Â miroirÂ Â» est active. Vous pouvez copier et afficher ces fichiers mais vous ne pouvez pas les dÃ©placer, supprimer ou Ã©diter, Ã  l'exception des fichiers tÃ©lÃ©chargÃ©s qui peuvent Ãªtre supprimÃ©. Vous pouvez naviguer Ã  l'intÃ©rieur de cet espace pour trouver les documents que vous souhaitez ajouter Ã  la collection.
+      <Break/>
+      L'arborescence des fichiers de la Collection ne reprÃ©sente jusque-lÃ  que le contenu de la collection. Elle est initialement vide.
+      <Break/>
+      On peut redimensionner les espaces en positionnant la souris sur la barre grise qui sÃ©pare les arborescences (le pointeur change de forme) et en la faisant glisser.
+      <Break/>
+      En bas de la fenÃªtre, il y a une barre d'Ã©tat qui montre l'Ã©volution des opÃ©rations sur les fichiers (copie, dÃ©placement et suppression). Ces opÃ©rations peuvent Ãªtre longues. Le bouton "Stop" arrÃªte toute tÃ¢che en cours.
+      <Break/>
+      Deux grands boutons occupent le coin droit infÃ©rieur de l'Ã©cran. "Nouveau dossier", avec une image d'un dossier, crÃ©e de nouveaux dossiers (voir <Reference target="creatingfolders">CrÃ©ation de dossiers</Reference>). "Supprimer", avec une corbeille, supprime les fichiers. Cliquer sur le bouton "Supprimer" dÃ©truit tous les fichiers sÃ©lectionnÃ©s de l'arborescence de fichiers de la collection. On peut de mÃªme rÃ©aliser cette suppression de fichiers en les faisant glisser jusqu'au bouton "Supprimer". 
+      <Anchor name="advanced">Instructions avancÃ©es:</Anchor>
+      Pour sÃ©lectionner plusieurs sÃ©quences d'Ã©lÃ©ments, sÃ©lectionner le premier, maintenir appuyÃ©e la touche [Maj] et cliquer sur le dernier Ã©lÃ©ment de la sÃ©quence -- la sÃ©lection comprendra tous les Ã©lÃ©ments compris entre les deux. Pour la sÃ©lection de fichiers qui ne se suivent pas, maintenir la touche [Ctrl] appuyÃ©e en cliquant sur chacun d'eux. Ces deux mÃ©thodes peuvent Ãªtre utilisÃ©es ensemble pour la sÃ©lection de jeux d'Ã©lÃ©ments non adjacents.
+      <Break/>
+      Certains dossiers -- tel que celui contenant vos propres pages Web -- ont parfois une signification spÃ©ciale. L'Interface BibliothÃ©caire peut appliquer de tels dossiers au premier niveau de l'arborescence de fichiers. Pour ce faire double-cliquer droite sur le dossier visÃ©. Choisir "CrÃ©er un Raccourci", saisir le nom du dossier. Pour enlever un Ã©lÃ©ment, cliquer-droite sur le fichier concernÃ© et sÃ©lectionner "Supprimer Raccourci".
+    </Section>
+    <Section name="creatingfolders">
+      <Title>CrÃ©er des dossiers</Title>
+      <ObviousSentence>Cette section indique la maniÃšre de crÃ©er de nouveaux dossiers</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Le bouton Nouveau Dossier
+          </ContentsItem>
+          <ContentsItem>
+            Le menu bouton-droit
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Utiliser les dossiers de l'arborescence de fichiers de la collection pour rejeur les fichiers et les rendre aisÃ©ment accessibles. Des dossiers peuvent Ãªtre placÃ©s Ã  l'intÃ©rieur d'autres dossiers. Aucune limite, virtuellement parlant, ni au nombre de dossiers dont on peut disposer ni Ã  la complexitÃ© de leur agencement.
+      <Break/>
+      Pour crÃ©er un nouveau dossier, on peut opter pour un dossier existant et cliquer sur le bouton Nouveau Dossier. Le nouveau dossier apparaÃ®t Ã  l'intÃ©rieur de celui choisi, ou au niveau supÃ©rieur si aucun n'est choisi. Le nom de ce dossier est rÃ©clamÃ© ( Ã  dÃ©faut, il sera nommÃ© "Nouveau Dossier").
+      <Break/>
+      Les dossiers peuvent aussi Ãªtre crÃ©Ã©s en cliquant sur le dossier avec le bouton droit, et en choisissant "Nouveau Dossier" et procÃ©der comme ci-dessus.
+    </Section>
+    <Section name="addingfiles">
+      <Title>Ajouter des fichiers</Title>
+      <ObviousSentence>Cette section indique la maniÃšre d'inclure des fichiers dans la collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            glisser-dÃ©placer
+          </ContentsItem>
+          <ContentsItem>
+            fichiers multiples
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            mÃ©ta-donnÃ©es existantes
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Les fichiers peuvent Ãªtre copiÃ©s dans la collection par glisser-dÃ©placer. Le pointeur Ã©pouse la forme de l'Ã©lÃ©ment sÃ©lectionnÃ© (ou, si la sÃ©lection comprend plus d'un, leur nombre). DÃ©poser la sÃ©lection dans l'arborescence de la collection pour les y copier (si la source est l'arborescence de l'espace de travail) ou les dÃ©placer Ã  l'intÃ©rieur de la collection (si la source est l'arborescence de la collection).
+      <Break/>
+      Lorsque des fichiers multiples sont copiÃ©s, ils sont tous placÃ© dans le dossier cible au mÃªme niveau, quelle que soit la structure du dossier qui les contenait avant. Si vous copier un second fichier avec le mÃªme nom dans le mÃªme dossier, la question de savoir si le premier fichier doit Ãªtre Ã©crasÃ© est posÃ©e. Si la rÃ©ponse est non, le fichier ne sera pas copiÃ©, mais les autres le seront. Pour annuler toutes les autre tÃ¢ches de copie, cliquer sur le bouton "Stop".
+      <Break/>
+      Seuls les Ã©lÃ©ments situÃ©s au-dessus dans la sÃ©lection sont dÃ©placÃ©s. Un dossier est plus Ã©levÃ© que son contenu. On ne peut sÃ©lectionner en mÃªme temps l'intÃ©rieur d'un dossier et le dossier lui-mÃªme.
+      <Anchor name="advanced">Instructions avancÃ©es:</Anchor>
+      Lorsqu'un fichier est ajoutÃ©, l'Interface BibliothÃ©caire cherche les dossiers sources de fichiers auxiliaires contenant une mÃ©ta-donnÃ©e qui lui originellement assignÃ©es et, s'il en trouve, commence Ã  les importer. Au fur et Ã  mesure que l'opÃ©ration se poursuit, une confirmation est faite (peut-Ãªtre plusieurs fois) de fournir des informations complÃ©mentaires pour ajuster la mÃ©ta-donnÃ©e aux jeux de mÃ©ta-donnÃ©es de la collection. Ces confirmations sont dÃ©crites dans la section <Reference target="importingpreviouslyassignedmetadata">Importer des mÃ©ta-donnÃ©es originellement assignÃ©es </Reference>. Pour une explication plus dÃ©taillÃ©e de l'association de mÃ©ta-donnÃ©es avec les fichiers, lire le chapitre 2 du Guide de dÃ©veloppement de Greenstone -- "Exploitez vos documents au maximum".
+    </Section>
+    <Section name="removingfiles">
+      <Title>Supprimer des Fichiers</Title>
+      <ObviousSentence>Cette section dÃ©crit la maniÃšre de supprimer des fichiers et des dossiers de la collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Le bouton Supprimer
+          </ContentsItem>
+          <ContentsItem>
+            La touche Supprimer
+          </ContentsItem>
+          <ContentsItem>
+            Supprimer par glisser-dÃ©placer
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Il existe plusieurs mÃ©thodes pour supprimer des fichiers et des dossiers. On doit d'abord indiquer les Ã©lÃ©ments Ã  supprimer en sÃ©lectionnant un ou plusieurs fichiers et dossiers comme dÃ©crit dans <Reference target="thegatherview">L'Affichage Rassembler (The Gather view)</Reference>.
+      <Break/>
+      DÃšs lors que les fichiers sont sÃ©lectionnÃ©s, cliquer sur le bouton "Supprimer" pour les dÃ©truire, ou appuyer sur la touche [Delete] de votre clavier, ou les glisser-dÃ©placer de la collection vers le bouton "Supprimer" et les y dÃ©poser.
+    </Section>
+    <Section name="filteringthetree">
+      <Title>Filtrer l'arborescence</Title>
+      Le "Filtrage" de l'arborescence de la collection permet de rÃ©duire la recherche de fichiers particuliers.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Le contrÃŽle du filtre
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Filtrage usuel
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Le menu dÃ©roulant des "Montrer Fichiers" ("Show Files") au bas de chaque arborescence montre une liste de filtres prÃ©dÃ©finis, tels "Images". Son choix cache temporairement tous les autres fichiers de l'arborescence. Pour restorer celle-ci, changer le filtre en "Tous Fichiers". Ces opÃ©rations n'altÃšrent ni la collection, ni n'affectent les dossiers dans l'arborescence.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      On peut spÃ©cifier un filtre usuel en saisissant un modÃšle assorti aux fichiers visÃ©s. Utiliser un systÃšme standard d'abrÃ©viations de fichiers telles que "*.*" ou "*.doc" (*. concerne tous les caractÃšres).
+    </Section>
+  </Section>
+
+  <Section name="enrichingacollection">
+    <Title>Enrichir la Collection avec des mÃ©ta-donnÃ©es</Title>
+    AprÃšs avoir rassemblÃ© plusieurs fichiers dans la collection, on les enrichit d'une information complÃ©mentaire appelÃ©e "mÃ©ta-donnÃ©e". Cette section explique comment une mÃ©tadonnÃ©e est crÃ©Ã©e, Ã©ditÃ©e, assignÃ©e ou retirÃ©e, et comment utiliser les sources de mÃ©ta-donnÃ©es externes (voir aussi le chapitre 2 du Guide de dÃ©veloppement de Greenstone -- "Exploitez vos documents au maximum").
+    <Section name="theenrichview">
+      <Title>L'affichage Enrichir</Title>
+      <ObviousSentence>Cette section dÃ©crit comment insÃ©rer et Ã©diter une mÃ©ta-donnÃ©e en utilisant l'affichage Enrichir.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            L'arborescence de la collection
+          </ContentsItem>
+          <ContentsItem>
+            Les contrÃŽles d'Ã©dition de mÃ©ta-donnÃ©es
+          </ContentsItem>
+          <ContentsItem>
+            L'arborescence valeur
+          </ContentsItem>
+          <ContentsItem>
+           La mÃ©ta-donnÃ©e Table
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Utiliser l'affichage Enrichir pour assigner une mÃ©ta-donnÃ©e aux documents de la collection. Une mÃ©ta-donnÃ©e est une donnÃ©e sur une donnÃ©e -- de faÃ§on typique titre, auteur, date de crÃ©ation, et ainsi de suite. Chaque Ã©lÃ©ment de mÃ©ta-donnÃ©e a deux parties: "elÃ©ment" indique le type d'Ã©lÃ©ment (par exemple auteur), "valeur" qui donne la valeur de cette mÃ©ta-donnÃ©e Ã©lÃ©ment (par exemple le nom de l'auteur).
+      <Break/>
+      Sur la gauche de l'affichage "Enrichir" est l'arborescence de la collection. A sa droite, la Table de MÃ©ta-donnÃ©es, qui montre les mÃ©ta-donnÃ©es pour tous les fichiers et dossiers sÃ©lectionnÃ©es de l'arborescence de la collection. Les colonnes sont indiquÃ©es en gris en haut, et peuvent Ãªtre redimensionnÃ©es en faisant glisser la ligne de sÃ©paration. Clicquez sur n'importe quelle ligne pour transfÃ©rer ses informations aux contÃŽles de mÃ©ta-donnÃ©es ci-dessous. Si plusieurs fichiers sont sÃ©lectionnÃ©s, un texte en noir indique que la valeur est commune Ã  tous les fichiers sÃ©lectionnÃ©s, tandis qu'un texte en gris en indique le contraire. les valeurs en noir peuvent Ãªtre mises Ã  jour ou supprimÃ©es, tandis que celles en gris peuvent Ãªtre supprimÃ©es de ceux qui l'ont, ou ajoutÃ©es aux autres.
+      <Break/>
+      Une icÃŽne de dossier peut apparaÃ®tre Ã  cÃŽtÃ© de certaines entrÃ©es de mÃ©ta-donnÃ©es. Ceci indique que les valeurs ont Ã©tÃ© hÃ©ritÃ©es d'un dossier parent. Les mÃ©ta-donnÃ©es hÃ©ritÃ©es ne peuvent Ãªtre Ã©ditÃ©es ou supprimÃ©es, mais seulement ajoutÃ©es ou Ã©crasÃ©es. Cliquer sur l'icÃŽne du dossier pour aller immÃ©diatement au dossier oÃ¹ la mÃ©ta-donnÃ©e est assignÃ©e.
+      <Break/>
+      Les contrÃŽles d'Ã©dition de mÃ©ta-donnÃ©es en bas Ã  droite apparaissent seulement quand un fichier dans l'arbre et une ligne dans la table sont sÃ©lectionnÃ©s. Utiliser les pour mettre Ã  jour, ajouter et supprimer la valeur de la mÃ©ta-donnÃ©e. Le champ valeur sert Ã  la saisie et Ã  l'Ã©dition de la valeur de la mÃ©ta-donnÃ©e. A cÃŽtÃ©, le bouton de label "..." permet d'ouvrir une grande boÃ®te d'Ã©dition. Les autres boutons situÃ©s en-dessous sont: "Coller ou ajouter" assigne la valeur comme nouvelle mÃ©ta-donnÃ©e et l'ajoute aux valeurs existantes de l'Ã©lÃ©ment sÃ©lectionnÃ©; "Remplacer" Ã©crase la valeur existante sÃ©lectionnÃ©e avec la nouvelle; "Supprimer" efface la valeur sÃ©lectionnÃ©e. Au bas, le label "Toutes les valeurs prÃ©cÃ©dentes" est la "Value Tree" ("Arborescence Valeur").
+      <Break/>
+      Celle-ci, dÃ©veloppe et comprime. C'est une liste qui, de faÃ§on usuelle, montre toutes les valeurs prÃ©cÃ©demment saisies pour un Ã©lÃ©ment sÃ©lectionnÃ©. Le clic sur une entrÃ©e la place automatiquement dans le champ valeur. Inversement, la saisie dans le champ texte sÃ©lectionne l'entrÃ©e de l'Arborescence Valeur qui dÃ©bute avec les caractÃšres saisis. Appuyer sur [Tab] complÃšte automatiquement la saisie de la valeur sÃ©lectionnÃ©e.
+      <Break/>
+      Les valeurs de mÃ©ta-donnÃ©e peuvent Ãªtre organisÃ©es en une hiÃ©rarchie.Ceci est montrÃ© dans l'Arborescence Valeur utilisant des dossiers de niveau interne.Les valeurs hiÃ©rarchiques peuvent Ãªtre saisies en utilisant le caractÃšre "|" pour sÃ©parer les niveaux. Par exemple, "Cards|Red|Diamonds|Seven" peut Ãªtre utilisÃ© dans une hiÃ©rarchie qui reprÃ©sente un jeu de cartes. Cela permet le rejeument des valeurs. Les jeux peuvent aussi Ãªtrre assignÃ©s comme mÃ©ta-donnÃ©e.
+      <Break/>
+      Greenstone extrait automatiquement des mÃ©ta-donnÃ©es Ã  partir des documents en un jeu de mÃ©ta-donnÃ©es dont les Ã©lÃ©ments sont prÃ©fixÃ©s "ex.". Ceci n'a aucune valeur et ne peut Ãªtre Ã©ditÃ©, et dÃšs lors les contrÃŽles MetaEdit sont cachÃ©s si une telle entrÃ©e est sÃ©lectionnÃ©e. Le bouton "..." sert toujours au dÃ©veloppement de la valeur, mais le texte ne peut Ãªtre Ã©ditÃ©.
+    </Section>
+    <Section name="selectingmetadatasets">
+      <Title>SÃ©lectionner les jeux de mÃ©ta-donnÃ©es</Title>
+      <ObviousSentence>Cette section explique comment spÃ©cifier les Ã©lÃ©ments mÃ©ta-donnÃ©e et comment ajouter les jeux de mÃ©ta-donnÃ©es Ã  une collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Qu'est-ce qu'un jeu de mÃ©ta-donnÃ©es?
+          </ContentsItem>
+          <ContentsItem>
+	    ContrÃŽle des jeux de mÃ©ta-donnÃ©es utilisÃ©s
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Les sÃ©ries d'Ã©lÃ©ments de mÃ©ta-donnÃ©es prÃ©dÃ©finis sont connus comme "jeux de MÃ©ta-donnÃ©es". "Dublin Core" en est un exemple. Quand on ajoute un Groupe de MÃ©ta-donnÃ©es Ã  la collection, ses Ã©lÃ©ments peuvent Ãªtre sÃ©lectionnÃ©s. On peut avoir plus d'un jeu de mÃ©ta-donnÃ©es ; pour Ã©viter des tÃ©lescopages de nom, un identificateur abrÃ©gÃ© qui identifie le jeu de mÃ©ta-donnÃ©es est suspendu au nom de l'Ã©lÃ©ment. Par exemple l'Ã©lement crÃ©ateur de Dublin Core devient "dc.Creator". Les jeux de mÃ©ta-donnÃ©es sont sauvÃ©gardÃ©s dans le dossier de mÃ©ta-donnÃ©es de l'Interface BibliothÃ©caire et ont le sufixe ".mds".
+      <Break/>
+      Pour contrÃŽler les jeux de mÃ©ta-donnÃ©es utilisÃ©s dans une collection, utilisez l'entrÃ©e "Jeux MÃ©ta-DonnÃ©es" sur la vue "Conception".
+    </Section>
+    <Section name="appendingmetadata">
+      <Title>Ajout de Nouvelles MÃ©ta-donnÃ©es</Title>
+      <ObviousSentence>Cette section explique comment ajouter des mÃ©ta-donnÃ©es et des valeurs Ã  des fichiers, et comment ajouter de nouvelles valeurs pour des Ã©lÃ©ments de mÃ©ta-donnÃ©es.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Ajouter aux fichiers
+          </ContentsItem>
+          <ContentsItem>
+            Ajouter aux rÃ©pertoires et aux fichiers multiples
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            HiÃ©rarchies de valeur et Ã©dition des valeurs
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      On ajoute un Ã©lÃ©ment mÃ©ta-donnÃ©e -- Ã©lÃ©ment et valeur Ã  la fois -- Ã  un fichier. SÃ©lectionner d'abord le fichier Ã  partir de l'arborescence de fichier de la collection situÃ©e Ã  gauche. Cette action entraÃ®ne l'apparition de toute mÃ©ta-donnÃ©e prÃ©-assignÃ©e  dans le tableau de droite. 
+      <Break/>
+      SÃ©lectionner ensuite l'Ã©lÃ©ment mÃ©ta-donnÃ©e Ã  ajouter en cliquant sur sa rangÃ©e dans le tableau.
+      <Break/>
+      Taper la valeur dans le champ valeur. Ne pas utiliser pas le caractÃšre "|", dÃšs lors qu'il est utilisÃ© pour construire des hiÃ©rarchies. Cliquer enfin sur "Append" pour ajouter la nouvelle valeur comme mÃ©ta-donnÃ©e pour le fichier choisi. La valeur apparaÃ®t immÃ©diatement dans le tableau de MÃ©ta-donnÃ©es.
+      <Break/>
+      Vous pouvez aussi ajouter des mÃ©ta-donnÃ©es Ã  un dossier, ou Ã  plusieurs fichiers sÃ©lectionnÃ©s Ã  la fois. Elle est ajoutÃ©e Ã  tous les fichiers ou Ã  l'intÃ©rieur du dossier ou de la sÃ©lÃ©ction, et aux sous-dossiers. Garder Ã  l'esprit  que si des mÃ©ta-donnÃ©es sont ajoutÃ©es Ã  un dossier tous les nouveaux fichiers en son sein hÃ©rite automatiquement des valeurs du dossier.
+      <Break/>
+      Si on choisit une mÃ©ta-donnÃ©e qui intervient dans quelques uns des fichiers sÃ©lectionnÃ©s et que l'on clique sur "Coller", elle est aussi ajoutÃ©e aux autres fichiers de la sÃ©lection.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      On peut ajouter une structure aux valeurs de mÃ©ta-donnÃ©es en utilisant les chemins comme dÃ©crit dans <Reference target="theenrichview">L'affichage Enrichir</Reference>.
+    </Section>
+    <Section name="addingpreviouslydefinedmetadata">
+      <Title>Ajouter des MÃ©ta-donnÃ©es prÃ©dÃ©finies</Title>
+      <ObviousSentence>Cette section explique la maniÃšre d'ajouter une mÃ©ta-donnÃ©e qui utilise des valeurs dÃ©jÃ  prÃ©sentes dans l'arborescence des valeurs.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Ajouter aux fichiers
+          </ContentsItem>
+          <ContentsItem>
+            Ajouter aux fichiers et aux fichiers multiples
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour ajouter une mÃ©ta-donnÃ©e ayant une valeur existante, sÃ©lectionner d'abord le fichier, et ensuite la valeur requise Ã  partir de l'arborescence des valeurs, en dÃ©veloppant si nÃ©cessaire les dossiers de hiÃ©rarchie. La valeur de l'entrÃ©e sÃ©lectionnÃ©e apparaÃ®t automatiquement dans le champ texte Valeur. (Utiliser alternativement la valeur auto-select de l'arborescence et les caractÃ©ristiques auto-complÃštes. Cliquer sur "Append" pour ajouter la metadata au fichier sÃ©lectionnÃ©. 
+      <Break/>
+      La procÃ©dure d'ajouter mÃ©ta-donnÃ©es avec des valeurs dÃ©jÃ  existantes aux dossiers ou aux fichiers multiples est tout Ã  fait identique.
+    </Section>
+    <Section name="updatingmetadata">
+      <Title>Mettre Ã  jour des MÃ©ta-donnÃ©es</Title>
+      <ObviousSentence>Cette section explique la maniÃšre de mettre Ã  jour la mÃ©ta-donnÃ©e assignÃ©e Ã  un fichier.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Mettre Ã  jour des fichiers
+          </ContentsItem>
+          <ContentsItem>
+            Mettre Ã  jour des dossiers et des fichiers multiples
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour mettre Ã  jour la valeur d'un Ã©lÃ©ment mÃ©ta-donnÃ©e, choisir d'abord le fichier auquel la valeur s'applique, et ensuite l'Ã©lÃ©ment mÃ©ta-donnÃ©e dont on veut changer la valeur. Votre sÃ©lection apparaÃ®t dans les contrÃŽles d'Ã©dition des mÃ©ta-donnÃ©es. Editer le champ valeur et cliquer sur "Remplacer" pour Ã©craser la mÃ©ta-donnÃ©e.
+      <Break/>
+      Le processus est identique pour la mise Ã  jour d'un dossier avec des sous-dossiers ou plusieurs fichiers, mais vous pouvez seulement mettre Ã  jour les mÃ©ta-donnÃ©es qui sont communes Ã  tous les fichiers/dossiers sÃ©lectionnÃ©s.
+      <Break/>
+      L'arborescence des valeurs montre toutes les valeurs prÃ©cÃ©dentes, mais pas seulement celles dÃ©signÃ©es actuellement. Ainsi la valeur que vous avez remplacÃ©e restera dans l'arbre de valeur.
+    </Section>
+    <Section name="removingmetadata">
+      <Title>Enlever les mÃ©ta-donnÃ©es</Title>
+      <ObviousSentence>Cette section explique comment retirer des mÃ©ta-donnÃ©es d'un fichier.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Enlever Ã  partir de fichiers
+          </ContentsItem>
+          <ContentsItem>
+            Enlever Ã  partir de dossiers ou de fichiers multiples
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      On enlever la mÃ©ta-donnÃ©e de la mÃªme faÃ§on que sa mise Ã  jour. SÃ©lectionner d'abord un fichier Ã  partir de l'arborescence des fichiers, utiliser ensuite le tableau de mÃ©ta-donnÃ©e pour sÃ©lectionner la mÃ©ta-donnÃ©e. Si la mÃ©ta-donnÃ©e a une valeur, le bouton "Enlever" devient actif. Cliquer-le pour enlever la mÃ©ta-donnÃ©e du fichier spÃ©cifiÃ©. D'autres fichiers demeurent inchangÃ©s et la valeur reste dans l'arborescence des valeurs. 
+      <Break/>
+      La suppression de mÃ©ta-donnÃ©es de rÃ©pertoires ou de plusieurs fichiers fonctione de la mÃªme faÃ§on.
+    </Section>
+    <Section name="reviewingmetadata">
+      <Title>Passer en revue les MÃ©ta-donnÃ©es assignÃ©es</Title>
+      <ObviousSentence>Cette section dÃ©crit la maniÃšre de visionner toutes les mÃ©ta-donnÃ©es assignÃ©es Ã  la colletion entiÃšre.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Le tableau d'audit
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Le menu bouton droit
+          </ContentsItem>
+          <ContentsItem>
+            Autofiltrage
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      On Ã©prouve parfois le besoin de voir, Ã  la fois, les mÃ©ta-donnÃ©es affectÃ©es Ã  plusieurs ou Ã  tous les fichiers -- par exemple, pour dÃ©terminer combien de fichiers sont en cours d'utilisation ou pour avoir une idÃ©e de la chronologie. 
+      <Break/>
+      SÃ©lectionnez les fichiers que vous souhaitez examiner, puis clicquez sur le bouton droit et choisissez "MÃ©ta-donnÃ©es AssociÃ©es...". Une fenÃªtre appelÃ©e "Toutes les MÃ©ta-donnÃ©es", sous forme d'un grand tableau avec plusieurs colonnes, apparaÃ®t. La premiÃšre colonne montre les noms de fichiers; les lignes montrent toutes les valeurs de mÃ©ta-donnÃ©es assignÃ©es Ã  ces fichiers.
+      <Break/>
+      Dessiner le tableau peut prendre quelque temps si plusieurs fichiers sont sÃ©lectionnÃ©s. On peut continuer Ã  utiliser l'Interface BibliothÃ©caire pendant que la fenÃªtre "Toutes MÃ©ta-donnÃ©es" est ouverte.
+      <Break/>
+      Cliquer "Fermer" pour cacher la fenÃªtre.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      On peut aussi visualiser le tableau "Toutes MÃ©tadonnÃ©es" en sÃ©lectionnant les fichiers Ã  examiner en cliquant sur le bouton droit et en choisissant "MÃ©ta-donnÃ©es AssignÃ©es...". Si un dossier a Ã©tÃ© sÃ©lectionnÃ©, tous ses fichiers sont inclus dans le tableau.
+      <Break/>
+      Quand il devient trop large, on peut filtrer le tableau "Toutes MÃ©ta-donnÃ©es" en appliquant des filtres aux colonnes. Alors que de nouveaux filtres sont ajoutÃ©s, seules les lignes qui concernent restent visibles. Pour installer, modifier ou effacer un filtre, cliquez sur l'Ã®cone "Funnel" situÃ© en haut de la colonne. Une information sur le filtre est rÃ©clamÃ©e. Une fois le filtre installÃ©, l'en-tÃªte de la colonne change de couleur.
+      <Break/>
+      La confirmation possÃšde un onglet "simple" et " advancÃ©". La version simple filtre  les colonnes afin qu'elles ne montrent que des lignes qui contiennent une certaine valeur de mÃ©ta-donnÃ©e ("*" concerne toutes les valeurs). On peut sÃ©lectionner des valeurs de mÃ©ta-donnÃ©es Ã  partir d'une liste dÃ©roulante. La version advancÃ©e permet diffÃ©rentes opÃ©rations d'harmonisation: doit commencer avec, ne contient pas, alphabÃ©tiquement et est Ã©gal Ã . La valeur Ã  harmoniser peut Ãªtre Ã©ditÃ©e et prise sous forme d'une chaÃ®ne quelconque (y compris "*"), et on peut opter pour une harmonisation insensible aux caractÃšres. Enfin, on peut spÃ©cifier une deuxiÃšme condition d'harmonisation prÃ©cisant une gamme de valeurs (en sÃ©lectionnant AND) ou des valeurs alternatives (en selectionnant OR). En dessous de cette zone, une boÃ®te qui permet de changer l'ordre de tri (ascendant ou descendant) est placÃ©e. Enfin, cliquer sur "Installer Filtre" pour appliquer le nouveau filtre Ã  la colonne. Cliquer sur "Effacer Filtre" pour enlever le filtre courant. Notez que les dÃ©tails du filtre sont retenus mÃªme si le filtre est effacÃ©.
+      <Break/>
+      Par exemple, pour trier le tableau "Toutes MÃ©ta-donnÃ©es" , choisir une colonne, sÃ©lectionner le cadre du filtre par dÃ©faut (Un filtre simple sur "*") et choisir un ordre ascendant ou descendant.
+    </Section>
+    <Section name="importingpreviouslyassignedmetadata">
+      <Title>Importer des MÃ©ta-donnÃ©es PrÃ©-assignÃ©es</Title>
+      Cette section explique comment importer des mÃ©ta-donnÃ©es affectÃ©es prÃ©cÃ©demment; mÃ©ta-donnÃ©es affectÃ©es avant d'Ãªtre ajoutÃ©es Ã  la collection.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    En ajoutant des fichiers
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Si la mÃ©ta-donnÃ©e dans une forme reconnue par l'Interface BibliothÃ©caire a Ã©tÃ© prÃ©alablement assignÃ©e Ã  un fichier, -- par exemple, quand on choisit des documents dans une collection Greenstone existante -- elle est importÃ©e automatiquement quand on ajoute le fichier. Pour ce faire, la mÃ©ta-donnÃ©e doit Ãªtre ajustÃ©e au jeu de mÃ©ta-donnÃ©es disponibles dans la collection.
+      <Break/>
+      L'Interface BibliothÃ©caire confirme l'information nÃ©cessaire. confirmation sous forme d'instructions brÃšves et qui montre ensuite le nom de l'Ã©lÃ©ment mÃ©ta-donnÃ©e qui est importÃ©e, comme il apparaÃ®t dans le fichier source. Ce champ ne peut Ãªtre ni Ã©ditÃ© ni changÃ©. Ensuite on choisit sur quel jeu de mÃ©ta-donnÃ©e le nouveau Ã©lÃ©ment devrait Ãªtre calquÃ© et enfin l'Ã©lÃ©ment mÃ©ta-donnÃ©e appropriÃ©. Le sytÃšme selectionne automatiquement le meilleur ajustement, en terme de jeu et d'Ã©lÃ©ment pour la nouvelle mÃ©ta-donnÃ©e.
+      <Break/>
+      AprÃšs cela,on peut choisir "Ajouter" pour ajouter le nouvel Ã©lÃ©ment mÃ©ta-donnÃ©e au jeu de mÃ©ta-donnÃ©es choisi. (Ceci n'est possible que si aucun Ã©lement de mÃªme nom n'est prÃ©sent Ã  l'interieur du jeu choisi.) "Remplacer" applique le nouvel Ã©lement Ã  celui choisi par l'utilisateur. Enfin, "Ignorer" n'importe aucune mÃ©ta-donnÃ©e avec ce nom d'Ã©lement. Une fois prÃ©cisÃ©e la maniÃšre d'importer un certain Ã©lÃ©ment de mÃ©ta-donnÃ©e, l'information d'application est retenue pour la durÃ©e de vie de la collection.
+      <Break/>
+      Pour des dÃ©tails concenant les fichiers mÃ©ta-donnÃ©e.xml que Greenstone utilise pour sauvegarder les mÃ©ta-donnÃ©es, il faut vous refÃ©rer au chapitre 2 du guide du DÃ©veloppeur de Greenstone -- "Exploitez vos documents au maximum".
+    </Section>
+  </Section>
+
+  <Section name="designingacollection">
+    <Title>DÃ©coration de l'apparence de votre collection</Title>
+Une fois les fichiers marquÃ©s avec les mÃ©ta-donnÃ©es, on peut dÃ©cider comment ils devraient apparaÃ®tre aux utilisateurs comme une collection Greenstone. Quel type d'information peut Ãªtre recherchÃ©? Quels sont les moyens qui sont fournis pour naviguer Ã  travers les documents? Quels sont les langages supportÃ©s? oÃ¹ apparaissent les boutons sur la page? ces choses peuvent Ãªtre adaptÃ©es; cette section dÃ©crit comment procÃ©der.
+    <Section name="thedesignview">
+      <Title>L'Affichage conception</Title>
+      Cette section prÃ©sente l'affichage de conception et explique comment on navigue entre les diffÃ©rents affichages Ã  l'intÃ©rieur de cette vitrine. 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Choisir une section de conception
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Avec l'Interface BibliothÃ©caire, on peut configurer la faÃ§on dont la collection va apparaÃ®tre Ã  l'utilisateur. Les options de configuration sont divisÃ©es en diffÃ©rentes sections, chacune associÃ©e Ã  une Ã©tape particuliÃšre de navigation ou de prÃ©sentation d'information.
+      <Break/>
+      Sur la gauche est placÃ©e une liste de diffÃ©rents affichages et Ã  droite les commandes associÃ©es Ã  celles en cours. Pour passer Ã  un  affichage diffÃ©rent, cliquer son nom dans la liste.
+      <Break/>
+      Pour comprendre les diffÃ©rentes Ã©tapes et les termes implmiquÃ©s dans la conception d'une collection, lire les chapitres 1 et 2 de Greenstone's Developer's Guide. 
+    </Section>
+    <Section name="generalsettings">
+      <Title>RÃ©glages gÃ©nÃ©raux</Title>
+      Cette section explique comment passer en revue et changer les rÃ©glages gÃ©neraux associÃ©s Ã  la collection. D'abord, sous "Design section" cliquer "General".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            RÃ©glages gÃ©nÃ©raux
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Les valeurs fournies ici peuvent Ãªtre modifiÃ©es au cours de la crÃ©ation de la collection.
+      <Break/>
+      En haut de la page, se trouve une boÃ®te Ã  instruction, qui apparaÃ®t pour chacune des diffÃ©rentes sections. Elle contient une brÃšve liste d'instructions pour vous rappeler quelle fonctionalitÃ© est disponible:
+      <Break/>
+      D'abord, les contacts E-mail du crÃ©ateur et mainteneur de la collection. Le champ suivant permet le changement de titre de la collection. Le dossier dans lequel est enregistrÃ© la collection est affichÃ© ensuite, mais ne peut Ãªtre Ã©ditÃ©. Celui qui suit spÃ©cifie (sous forme d'URL) l'Ã®cone Ã  montrer Ã  l'extrÃ©mitÃ© supÃ©rieure gauche de la page "About" de la collection et la suivante est l'Ã®cone utilisÃ©e dans la page de la bibliothÃ©que Greenstone Ã  lier Ã  la collection. Puis, une case permet de contrÃŽler si la collection est accessible au public. Enfin, la zone texte de "la description de la Collection" comme dÃ©crit <Reference target="creatingacollection">CrÃ©er une nouvelle collection</Reference>.
+    </Section>
+    <Section name="plugins">
+      <Title>Plugiciels de Document</Title>
+      Cette section dÃ©crit comment faire la configuration des plugiciels des documents que la collection utilise. Elle explique comment spÃ©cifier les plugiciels utilisÃ©s, quels paramÃštres leur passer et dans quel ordre ils se dÃ©roulent. Sous "Sections de conception", cliquer "Plugiciels de document".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Ajouter un plugiciel
+          </ContentsItem>
+          <ContentsItem>
+            Enlever un plugiciel
+          </ContentsItem>
+          <ContentsItem>
+            Configurer un plugiciel
+          </ContentsItem>
+          <ContentsItem>
+            Changer l'ordre des plugiciels
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Configuration d'usage
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour ajouter un plugiciel, sÃ©lectionner-le sur la liste dÃ©roulante de "Choisir le plugiciel Ã  ajouter" situÃ©e Ã  cÃŽtÃ© du bouton et ensuite cliquer "Ajouter plugiciel". Une fenÃªtre dÃ©nommÃ©e "Configurer les arguments" apparaÃ®t; elle est dÃ©crite plus loin. Une fois la configuration du nouveau plugiciel achevÃ©e, il est ajoutÃ© Ã  la fin de la liste "Plugiciels couramment assignÃ©s". Veuillez noter que, Ã  l'exception de UnknownPlug, chaque plugin ne peut apparaÃ®tre qu'une seule fois dans la liste.
+      <Break/>
+      Pour enlever un plugiciel, sÃ©lectionner-le sur la liste et cliquer sur "Enlever plugiciel".
+      <Break/>
+      Les plugiciels sont configurÃ©s en fournissant des arguments. Pour les changer, il faut sÃ©lectionner le plugiciel Ã  partir de la liste et cliquer "Configurer plugiciel" (Ou double-cliquer sur le plugiciel). Un dialogue "Configuration des arguments" apparaÃ®t avec diffÃ©rent contÃŽles pour spÃ©cifier les arguments.
+      <Break/>
+      Il existe plusieurs types de contrÃŽles. Certains sont des boÃ®tes de dialogue et cliquer sur l'une ajoute l'option appropriÃ©e au plugiciel. Les autres sont des chaÃ®nes de textes avec une boÃ®te de dialogue et un champ texte. Cliquer sur la boÃ®te pour activer l'argument, ensuite saisir le texte appropriÃ© dans la boÃ®te (expression normale, chemin de fichier etc). Les autres sont des menus dÃ©roulants Ã  partir desquels on peut sÃ©lectionner Ã  partir d'une sÃ©rie de valeurs dÃ©terminÃ©es. Pour apprendre ce qu'un argument peut faire, promener la souris planer sur son nom pendant un instant une description apparaÃ®tra.
+      <Break/>
+      Pour changer la configuration, cliquer sur "OK" pour valider et fermer la boÃ®te de dialogue, ou "Annuler" pour fermer la boÃ®te de dialogue sans changer les arguments du plugiciel.
+      <Break/>
+      Les plugiciels dans la liste sont executÃ©s en ordre et l'ordre est parfois important. Les deux plugiciels Qrcplug et Recplug sont vitaux au processus de construction de la collection, et sont placÃ©s Ã  la fin de la liste (avec une ligne de sÃ©paration). Pour changer l'ordre des autres, sÃ©lectionner le plugiciel Ã  dÃ©placer et cliquer sur "DÃ©placer vers le haut" ou "DÃ©placer vers le bas".
+       <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+       L'Interface BibliothÃ©caire s'Ã©vertue Ã  dÃ©terminer les arguments supportÃ©s par le plugiciel. Cependant, l'utilisateur peut avoir Ã  indiquer d'autres arguments spÃ©ciaux; pour cela un champ texte appelÃ© "Arguments SpÃ©cifiques" (en bas) est disponible. Tout texte qu'il contient est annexÃ© Ã  la fin de la commande Plugiciel.
+    </Section>
+    <Section name="searchtypes">
+      <Title>Types de recherche</Title>
+      cette section explique comment modifier une nouvelle caractÃ©ristique de conception dans Greenstone, Types de recherche, qui permet une recherche de champ. Sous "Sections de conception", cliquer sur "Types de recherche".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Que signifie activer des recheches avancÃ©es?
+          </ContentsItem>
+          <ContentsItem>
+            Ajouter un nouveau type de recherche
+          </ContentsItem>
+          <ContentsItem>
+	    Enlever un type de recherche
+          </ContentsItem>
+          <ContentsItem>
+            Changer l'ordre des types de recherche
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Lorsqu'on pÃ©nÃštre dans l'affichage Types de recherche, vÃ©rifier d'abord "Activer recherches avancÃ©es", qui active les autres contrÃŽles. la collection prend le format 2.4 de Greenstone, qui supporte la recherche basÃ©e sur les champs, et signifie que: 1. l'index de conception est diffÃ©rent (expliquÃ© dans <Reference target="searchindexes">Recherche d'index</Reference> section), 2. plusieurs fragments de textes sont Ã  traduire (voir <Reference target="translatetext">Translation</Reference>), 3. la collection est inutilisable sous des versions plus vieilles de Greenstone. Si vous dÃ©-sÃ©lectionner ce champ, la plupart de la collection retourne Ã  la version 2.39 de Greenstone 2.39. Cependant, l'Interface BibliothÃ©caire ne peut pas convertir les nouvelles spÃ©cifications d'index en celles plus anciennes; elles doivent ainsi Ãªtre saisies de nouveau manuellement. 
+      <Break/>
+      pour ajouter un type de recherche, sÃ©lectionner-le Ã  partir de la liste "Types de recherche" et cliquer sur "Ajouter Type de recherche".  Chaque type n'apparaÃ®t qu'une fois sur la liste.
+      <Break/>
+      Pour enlever un type de recherche, il faut le selectionner Ã  partir de la liste "Currently Assigned Search type" et cliquer sur "Enlever type de recheche". La liste doit contenir au moins un type de recherche.
+      <Break/>
+      pour changer l'ordre d'un type de recherche, sÃ©lectionner-le Ã  partir de la liste et cliquer sur "DÃ©placer vers le haut" ou "DÃ©placer vers le bas". Le premier sera le dÃ©faut.
+    </Section>
+    <Section name="searchindexes">
+      <Title>Index de recherche</Title>
+      Les index prÃ©cisent les parties de la collection qui doivent Ãªtre recherchÃ©es. Cette section explique comment ajouter et enlever des index et fixer un index par dÃ©faut. Sous "Sections de conception", cliquer sur "Rechercher Index".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Ajouter un index
+          </ContentsItem>
+	  <ContentsItem>
+	    Ãditer un index
+	  </ContentsItem>
+          <ContentsItem>
+            Enlever un index
+          </ContentsItem>
+          <ContentsItem>
+            Index par dÃ©faut
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour ajouter un index, taper un nom choisi dans le champ "Nom de l'Index". Selectionner les sources d'information Ã  indexer en cliquant les boÃ®tes de dialogue Ã  cÃŽtÃ© d'eux. La liste montre aussi bien tous les Ã©lÃ©ments de mÃ©ta-donnÃ©es assignÃ©s, que le texte entier. AprÃšs avoir sÃ©lectionnÃ© les sources de donnÃ©es, choisir la granularitÃ© de l'index en utilisant le menu "Au niveau". Une fois ces dÃ©tails completÃ©s, "Ajouter Index" devient actif  (Ã  moins qu'il n'ait un index existant avec les mÃªmes paramÃštres). Cliquez pour ajouter un nouvel index.
+      <Break/>
+      Pour Ã©diter un index, sÃ©lectionnez-le et changez les dÃ©tails de l'index, puis clicquez sur "Remplacer l'index".
+      <Break/>
+      Pour enlever un index, il faut le sÃ©lectionner sur la liste d'index assignÃ©s et cliquer "Enlever index". 
+      <Break/>
+      L'index par dÃ©faut, celui qui est utilisÃ© dans la page de recherche de la collection est labellÃ© parc "[Index par dÃ©faut]" dans la liste "Index assignÃ©s".
+      <Break/>
+      Si la recherche avancÃ©e est activÃ©e (voir Affichage des types de recherche), les contrÃŽles d'index sont diffÃ©rents.Chaque index est seulement basÃ© sur une source de donnÃ©es. Il existe une nouvelle pseudo-source de donnÃ©es "Tous champs" qui fournit une recherche Ã  travers tous les index spÃ©cifiÃ©s Ã  la fois. Des niveaux ne sont pas assignÃ©s Ã  un index spÃ©cifique, mais s'applique Ã  tous les index : ainsi index et niveaux sont ajoutÃ©s sÃ©parement. Les index sont enlevÃ©s de la mÃªme faÃ§on que prÃ©cÃ©demment, mais l'index par dÃ©faut ne peut plus Ãªtre mis activÃ©.. il est simplement le premier index assignÃ©. 
+      <Break/>
+      Pour crÃ©er des index sur toutes les sources cliquez le bouton "Ajouter Tous". Le nom de chaque index sera en dÃ©faut pour le nom de source. Pour changer le nom, SÃ©lectionner un index, changer ses dÃ©tails et cliquez "Remplacer Index". 
+    </Section>
+    <Section name="partitionindexes">
+      <Title>Index de Partition</Title>
+      Les index sont construits sur un texte particulier ou des sources MÃ©ta-donnÃ©es. L'espace de recherche peut Ãªtre plus tard contrÃŽlÃ© en morcelant l'index soit par langage ou par filtre choisi Ã  l'avance. Cette section en dÃ©crit la procÃ©dure. Sous "Sections de conception", cliquer "Index de Partition". 
+      <Break/>
+      L'affichage "Index de Partition" possÃšde trois onglets: "DÃ©finir les Filtres", "Assigner une partition" et "Assigner des langages". Pour en savoir davantage sur les partitions, lire sur les sous collections et les sous index au chapitre 2 de Greenstone developer's guide.
+      <Section name="definefilters">
+        <Title>DÃ©finir les Filtres</Title>
+	<ObviousSentence>Cette section explique comment dÃ©finir un filtre de partition.</ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+             Ajouter un filtre
+            </ContentsItem>
+            <ContentsItem>
+              Enlever un filtre
+            </ContentsItem>
+            <ContentsItem>
+              Mettre Ã  jour un filtre
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Instructions simples:</Anchor>
+	Les filtres permettent de jeur en sous-collection tous les documents dans un index pour laquelle une valeur mÃ©ta-donnÃ©e correspond Ã  un type donnÃ©.
+	<Break/>
+	Pour crÃ©er un filtre, cliquer sur le bouton "DÃ©finir les Filtres" et entrer un nom pour le nouveau filtre dans le champ "Nom du filtre de la sous-collection". Ensuite choisir un attribut de document Ã  faire correspondre  soit avec l'Ã©lÃ©ment de mÃ©ta-donnÃ©e soit avec le nom du fichier en question. Saisir une expression courante qu'il faut utiliser durant la correspondance. Vous pouvez choisir entre "Inclure" les documents qui correspondent au fichier ou les "Exclure". Enfin, on peut spÃ©cifier l'un quelconque des drapeaux standard PERL d'expression courante qu'il faut utiliser pour la correspondance (par exemple "i" pour les correspondances ne tenant pas compte des caractÃšres.) Pour finir, cliquez sur "Ajouter un Filtre" pour ajouter le filtre Ã  la liste "Filtres de sous collection dÃ©finis". 
+	<Break/>
+	Pour enlever un filtre, sÃ©lectionner-le sur la liste et cliquer sur "Enlever Filtre".
+	<Break/>
+	Pour changer un filtre, sÃ©lectionner-le sur la liste, changer l'une quelconque des valeurs apparaissant dans les contrÃŽles d'Ã©dition et cliquer sur "Remplacer Filtre" pour valider les modifications.
+      </Section>
+      <Section name="assignpartitions">
+        <Title>Assigner des Partitions</Title>
+	<ObviousSentence>Cette section explique l'assignation d'un filtre de partition prÃ©dÃ©fini.</ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+              Ajouter une partition
+            </ContentsItem>
+            <ContentsItem>
+              Enlever une partition
+            </ContentsItem>
+            <ContentsItem>
+              Partition par dÃ©faut
+            </ContentsItem>
+            <ContentsItem>
+             Enlever partition par dÃ©faut
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Instructions simples:</Anchor>
+	AprÃšs avoir dÃ©fini un filtre de sous-collection, utiliser l'onglet "Assigner des Partitions" pour construire ses indexes  (ou ceux pour un jeu de filtres.) SÃ©lectionner le filtre (ou les filtres) de votre choix Ã  partir de la liste "DÃ©finir des Filtres de Sous-collection", entrer un nom pour la partition dans le champ "Nom de Partition Name", et cliquer sur "Ajouter Partition". 
+	<Break/>
+	Pour supprimer une partition, sÃ©lectionner-la Ã  partir de la liste et cliquer sur "Enlever Partition".
+	<Break/>
+	Pour faire d'une partition celle par dÃ©faut, sÃ©lectionner-la Ã  partir de la liste et cliquer sur "Par DÃ©faut".
+	<Break/>
+	Pour effacer la partition par dÃ©faut, cliquez sur "Effacer dÃ©faut".
+      </Section>
+      <Section name="assignlanguages">
+        <Title>Assigner Langages</Title>
+	Cette section donne les dÃ©tails de la maniÃšre de restreindre les index Ã  des langages particuliers. On le fait en gÃ©nÃ©rant une partition par le biais de l'onglet "Assigner Langages" de l'Affichage "Index de Partition".
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+              ContrÃŽles de Langage 
+            </ContentsItem>
+            <ContentsItem>
+              Ajouter langage
+            </ContentsItem>
+            <ContentsItem>
+              Enlever langage
+            </ContentsItem>
+            <ContentsItem>
+              Langage par dÃ©faut
+            </ContentsItem>
+            <ContentsItem>
+              Enlever Langage par dÃ©faut
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">Instructions simples:</Anchor>
+	Pour ajouter un nouveau langage, dont la partition par le biais de l'onglet "Assigner langues" confÃšre un index, SÃ©lectionner le langage souhaitÃ© Ã  partir de la liste "Langage Ã  ajouter" et cliquer sur "Ajouter langue".
+	<Break/>
+	Pour supprimer une langue, sÃ©lectionner-la Ã  partir de la liste "SÃ©lection de Langage" et cliquer sur "Enlever Langage". 
+	<Break/>
+	Pour installer un language par dÃ©faut, sÃ©lectionner-la dans la liste et cliquez sur "Par dÃ©faut".
+	<Break/>
+	Pour enlever le langage par dÃ©faut, cliquer sur "Enlever par DÃ©faut".
+      </Section>
+    </Section>
+    <Section name="xcollectionsearching">
+      <Title>Recherche unique Ã  travers plusieurs collections</Title>
+      Greenstone peut rechercher dans plusieurs collections comme si elles n'en constituaient qu'une. Cela se fait par la crÃ©ation d'une "super-collection" qui contient les collections individuelles. Sous "Sections de conception", cliquez sur "Recherche unique Ã  travers plusieurs collections".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            SÃ©lectionner une collection
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      L'Affichage de Recherche unique Ã  travers des collections montre une liste de collections disponibles. La collection courante est cochÃ©e et ne peut Ãªtre dÃ©-sÃ©lectionnÃ©e. Pour ajouter une autre collection qu'on peut rechercher parallÃšlement, cliquer sur une collection dans la liste (cliquer une deuxiÃšme fois pour l'enlever). Lorsqu'une seule collection est sÃ©lectionnÃ©e, aucune recherche Ã  travers collections n'est possible.
+      <Break/>
+      Pour plus de dÃ©tails, voir Chapitre 1 du Guide du DÃ©veloppeur Greenstone.
+    </Section>
+    <Section name="classifiers">
+      <Title>Classificateurs</Title>
+      Cette section explique comment assigner Ã  une collection des "classificateurs" qui sont utilisÃ©s pour naviguer. Sous la section "Sections de conception", cliquer sur "Classificateurs de navigation".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Configuration et sÃ©lection d'un classificateur
+          </ContentsItem>
+          <ContentsItem>
+            Ajouter un classificateur
+          </ContentsItem>
+          <ContentsItem>
+            Enlever un classificateur
+          </ContentsItem>
+          <ContentsItem>
+            Configurer un classificateur
+          </ContentsItem>
+          <ContentsItem>
+            Changer l'ordre d'un classificateur
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour ajouter un classificateur, sÃ©lectionner-le en utilisant la liste dÃ©roulante "Choisir classificateur Ã  ajouter" et cliquer sur "Ajouter Classificateur SpÃ©cifique". une fenÃªtre qui s'intitule "Configurer Arguments" apparaÃ®t ; les instructions concernant ce dialogue sont les mÃªmes que pour les plugiciels (voir <Reference target="plugins">plugiciels de Documents</Reference>). Une fois le nouveau classificateur configurÃ©, il s'ajoute Ã  la fin de la liste des "Classificateurs couramment AssignÃ©s". 
+      <Break/>
+      Pour supprimer un classificateur, sÃ©lectionner-le dans la liste et cliquez sur "Enlever Classificateur Choisi".
+      <Break/>
+      Pour changer les arguments d'un classificateur, sÃ©lectionner-le sur la liste et cliquer sur "Configurer Classificateur Choisi" (ou double-cliquer sur le classificateur sur la liste.) 
+      <Break/>
+      Le tri des classificateurs dans la barre de navigation de la collection est fourni ici dans l'ordre des classificateurs. Pour le changer, sÃ©lectionner le classificateur Ã  dÃ©placer et cliquer sur "DÃ©placer vers le haut" ou "DÃ©placer vers le bas".
+      <Break/>
+      Pour plus d'informations sur les classificateurs lisez le chapitre 2 du Guide du DÃ©veloppeur Greenstone -- Exploitez vos documents au maximum.
+    </Section>
+    <Section name="formatstatements">
+      <Title>CaractÃ©ristiques de Format</Title>
+Les Format Features et les commandes de format contrÃŽlent la structure et l'apparence de la collection. Ils influencent le lieu d'apparition des boutons lorsqu'un document est affichÃ©, et dÃ©terminent les liens qui sont affichÃ©s par le classificateur DateList. Les commandes de format ne sont pas faciles Ã  concevoir, et la lecture du chapitre 2 de Greenstone Developer's Guide est recommandÃ©e. Cette section concerne les cadres de format, et  indique comment l'Interface BibliothÃ©caire en fournit l'accÃšs. Sous "Sections de conception", cliquer sur "CaractÃ©ristiques de Format".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Formatage
+          </ContentsItem>
+          <ContentsItem>
+            Ajouter commande de format
+          </ContentsItem>
+          <ContentsItem>
+             Enlever commande de format
+          </ContentsItem>
+          <ContentsItem>
+            Mettre Ã  jour commande de format
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            Options de formatage Ã©tendues
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      On peut appliquer une commande de format Ã  n'importe quoi dans la liste dÃ©roulante "Choisir CaractÃ©ristique" qui comprend tous les classificateurs de mÃªme qu'une liste prÃ©dÃ©finie de caractÃ©ristiques. Deux types de contrÃŽles existent lorsqu'on sÃ©lectionne une caractÃ©ristique: Certaines caractÃ©ristiques sont simplement activÃ©es ou dÃ©sactivÃ©es, et le contrÃŽle se fait grÃ¢ce Ã  une liste de vÃ©rification. D'autres requiÃšrent qu'on spÃ©cifie une chaÃ®ne de format. Pour ces caractÃ©ristiques, il existe une liste de retrait ("Composante AffectÃ©e") pour la sÃ©lection des parties de la caraxtÃ©ristique oÃ¹ s'applique la chaÃ®ne (si nÃ©cessaire), une zone de texte ("ChaÃ®ne de Format HTML") oÃ¹ on saisit la chaÃ®ne, et une sÃ©lection de "Variables" prÃ©dÃ©finies. Pour insÃ©rer une variable dans la position courante dans la chaÃ®ne de format, sÃ©lectionner la Ã  partir de la liste de retrait et cliquer sur "InsÃ©rer". 
+      <Break/>
+      Vous pouvez spÃ©cifier un format par dÃ©faut pour une composante particuliÃšre en sÃ©lectionnant la caractÃ©ristique vierge.Ce format est alors appliquÃ© Ã  toutes les caractÃ©ristiques applicables sauf instruction contraire. 
+      <Break/>
+      Pour ajouter une nouvelle commande de format, remplir l'information comme indiquÃ©e plus haut et cliquer sur "Ajouter Format". La nouvelle commande de Format apparaÃ®t dans la liste des  "Commandes de Format frÃ©quemment AssignÃ©es". Seule une commande de format peut Ãªtre assignÃ©e pour chaque combinaison caractÃ©ristique/composante.
+      <Break/>
+      Pour enlever une commande de format, sÃ©lectionner-la sur la liste et cliquer "Enlever Format".
+      <Break/>
+      Pour modifier une commande de format, sÃ©lectionner-la  sur la liste et cliquer sur "Remplacer Format". 
+      <Break/>
+      Pour de plus amples informations sur les variables et les composantes de caractÃ©ristiques, lire le chapitre 2 du Guide du DÃ©veloppeur de Greenstone. 
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      Si la boÃ®te de vÃ©rification "Permettre options Ã©tendues" est cochÃ©e, certaines options avancÃ©es de formatage sont activÃ©es. La liste des caractÃ©ristiques pouvant Ãªtre formatÃ©es est lÃ©gÃšrement changÃ©e, et plusieurs variables sont disponibles dans la commande de format, offrant un contrÃŽle plus grand sur la mise en forme. 
+    </Section>
+    <Section name="translatetext">
+      <Title>Traduire Texte</Title>
+      Cette section dÃ©crit l'Affichage de traduction, oÃ¹ on peut dÃ©finir des fragments de texte spÃ©cifiques Ã  un langage pour des parties de l'interface de la collection. Sous "Sections de conception", cliquer sur "Traduire Texte".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Ajouter traduction
+          </ContentsItem>
+          <ContentsItem>
+            Enlever traduction
+          </ContentsItem>
+          <ContentsItem>
+            Mettre Ã  jour traduction
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Choisir d'abord une entrÃ©e dans la liste "CaractÃ©ristiques". Les chaÃ®nes spÃ©cifiques Ã  un langage associÃ©es Ã  la caractÃ©ristiques apparaissent en bas. Utiliser la liste dÃ©roulante de "Traduction de Langage" pour sÃ©lectionner le langage cible, et saisir le texte traduit dans la zone de texte, en se refÃ©rant Ã  "Fragment de Texte Initial" si nÃ©cessaire. Cliquez sur "Ajouter Traduction" Ã  la fin de l'opÃ©ration. 
+      <Break/>
+      Pour supprimer une traduction existante, sÃ©lectionner-la Ã  partir du tableau "Traduction AssignÃ©e" et cliquer sur "Enlever Traduction". 
+      <Break/>
+      Pour Ã©diter une traduction, sÃ©lectionner-la et Ã©diter-la dans la zone de texte "Texte Traduit", et cliquer sur "Remplacer Traduction". 
+    </Section>
+    <Section name="metadatasets">
+      <Title>jeux de MÃ©ta-donnÃ©es</Title>
+      Cette section explique le panel de visualisation du jeu de mÃ©ta-donnÃ©es. Sous "Sections de conception", cliquer sur "jeux de MÃ©ta-donnÃ©es". 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            jeux de mÃ©ta-donnÃ©es disponibles
+          </ContentsItem>
+          <ContentsItem>
+            Ajout de jeux mÃ©ta-donnÃ©es
+          </ContentsItem>
+          <ContentsItem>
+            Ãdition de jeux mÃ©ta-donnÃ©es
+          </ContentsItem>
+          <ContentsItem>
+            Suppression de jeux mÃ©ta-donnÃ©es
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Cette fenÃªtre sert Ã  la revue des jeux de mÃ©ta-donnÃ©es utilisÃ©s dans la collection, et les Ã©lÃ©ments qui sont disponibles dans chaque jeu. Choisir dans la liste de "jeux de mÃ©ta-donnÃ©es disponibles" pour voir des informations dÃ©taillÃ©es Ã  ces Ã©lÃ©ments.
+      <Break/>
+      Pour utiliser un autre jeu de mÃ©ta-donnÃ©es avec la collection chargÃ©e, clicquez sur "Ajouter un jeu de mÃ©ta-donnÃ©es" et sÃ©lectionnez le fichier mÃ©ta-donnÃ©es (.mds) pour le nouveau jeu mÃ©ta-donnÃ©es.
+      <Break/>
+      L'Ã©dition de jeux mÃ©ta-donnÃ©es se fait avec l'Ã©diteur Greenstone de mÃ©ta-donnÃ©es (GEMS). En clicquant sur le bouton "Ãditer un jeux mÃ©ta-donnÃ©es" vous obtiendrez des informations sur l'utilisation de GEMS.
+      <Break/>
+      Si vous n'avez plus besoin d'un jeu mÃ©ta-donnÃ©es, sÃ©lectionnez le et clickez sur "Suppression du jeu mÃ©ta-donnÃ©es" pour l'enlever. Si vous avez affecttÃ© des mÃ©ta-donnÃ©es aux Ã©lÃ©ments du jeu supprimÃ©, on vous demandera ce que vous voulez en faire lorsque vous allez ouvrir Ã  nouveau cette collection.
+    </Section>
+  </Section>
+  <Section name="producingthecollection">
+    <Title>Pour produire votre collection</Title>
+    AprÃšs avoir rassemblÃ© les documents de la collection, les avoir annotÃ©s avec les mÃ©ta-donnÃ©es, et dÃ©fini comment la collection va apparaÃ®tre, la production de la collection en utilisant Greenstone est alors possible. Cette section vous en explique la maniÃšre.
+    <Section name="thecreateview">
+      <Title>L'Affichage CrÃ©er</Title>
+      <ObviousSentence>Cette section explique l'utilisation de l'affichage crÃ©er pour produire une collection.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Construction de la collection
+	  </ContentsItem>
+	  <ContentsItem>
+	    AperÃ§u de la collection
+	  </ContentsItem>
+	</ContentsGroup>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Journal des messages
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      L'Affichage CrÃ©er est utilisÃ© pour crÃ©er la collection en mettant en Åuvre les scripts Greenstone relatifs Ã  cette opÃ©ration. En cliquant sur "Construire Collection", on initie le processus de construction de la collection. Sa durÃ©e dÃ©pend de la taille de la collection  et du nombre d'index Ã  crÃ©er (pour les grandes collections cela peut prendre des heures). Une barre d'Ã©tat indique ou on en est dans le processus. Pour annuler le processus Ã  tout moment, cliquer sur "Annuler Construction".
+      <Break/>
+      Quand la collection est construite, appuyant "Preview Collection" initiera la page web hebergeant la collection.
+      <Anchor name="advanced">Instructions advancÃ©es:</Anchor>
+      En mode expert, vous pouvez utiliser l'entrÃ©e "Message Log" Ã  gauche pour revoir les tentatives prÃ©cÃ©dentes de construction de la collection avec ou san succÃ©s. SÃ©lectionnez le journal souhaitÃ© en cliquant sur la date correspondante dans la liste "Historique du Journal".
+    </Section>
+    <Section name="buildsettings">
+      <Title>ParamÃštres d'Importation et de Construction</Title>
+      Cette section explique comment accÃ©der aux diffÃ©rents paramÃštres de construction et d'importation. Pour plus d'informations sur l'importation et la construction lire le chapitre 1 du Guide du DÃ©veloppeur de Greenstone -- Comprendre le processus de construction de la collection.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ParamÃštres GÃ©nÃ©raux
+          </ContentsItem>
+          <ContentsItem>
+            ParamÃštres d'importation
+          </ContentsItem>
+          <ContentsItem>
+            ParamÃštres de construction
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Le contrÃŽle des diffÃ©rents paramÃštres se fait de faÃ§on identique Ã  la fenÃªtre "Configurer Arguments" dÃ©crit dans la section <Reference target="plugins">Plugiciels de documents</Reference>. Certains champs requiÃšrent des arguments numÃ©riques, et on peut: soit les incorporer en les saisissant, soit utiliser les flÃšches Haut et Bas pour augmenter ou diminuer la valeur prÃ©sente (dans certains cas, l'interface limite la classe qu'on peut entrer.) D'autres sont activÃ©s en cliquant sur une case de vÃ©rification (re-cliquer pour dÃ©sactiver.)
+    </Section>
+  </Section>
+
+  <Section name="miscellaneous">
+    <Title>Miscellaneous</Title>
+    Cette section dÃ©crit les caractÃ©ristiques de l'Interface BibliothÃ©caire qui ne sont pas associÃ©es Ã  une fenÃªtre d'affichage particuliÃšre.
+    <Section name="preferences">
+      <Title>PrÃ©fÃ©rences</Title>
+      Cette section explique le dialogue des prÃ©fÃ©rences, auquel on accÃšde en ouvrant "Fichier" -> "PrÃ©fÃ©rences". 
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>GÃ©nÃ©ral</ContentsItem>
+	  <ContentsItem>Mode</ContentsItem>
+          <ContentsItem>Flux de travail</ContentsItem>
+          <ContentsItem>Connexion</ContentsItem>
+          <ContentsItem>Avertissement</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      La premiÃšre option "General" est un champ texte pour saisir votre adresse courriel. Ceci sera utilisÃ© pour les Ã©lÃ©ments mÃ©ta-donnÃ©es "crÃ©ateur" et "Mainteneur" de la collection. L'option suivante est un liste dÃ©roulante des langues dans lesquelles l'interface bibiothÃšquaire peut Ãªtre affichÃ©e. Si on change le dictionnaire par le choix d'un autre sur la liste, on doit redÃ©marrer l'Interface BibliothÃ©caire pour charger les chaÃ®nes du nouveau langage Ã  partir du dictionnaire.
+      <Break/>
+Si on coche "Affichage MÃ©ta-donnÃ©es Ã©tendues", les diffÃ©rents contrÃŽles relatifs aux mÃ©ta-donnÃ©es affichent toujours toutes les mÃ©ta-donnÃ©es qui ont Ã©tÃ© extraites automatiquement des documents. Lorsqu'on le dÃ©sÃ©lectionne, ces mÃ©ta-donnÃ©es sont cachÃ©es (bien qu'elles restent disponibles pendant la conception de la collection, et dans la collection Greenstone finale.) Si on coche "Montrer les tailles des fichiers", la taille du fichier apparaÃ®t Ã  cÃŽtÃ© de chaque fichier dans l'espace de travail et dans les arborescences de fichiers de la collection dans les Affichages Rassembler et Enrichir.
+      <Break/>
+      Le panneau "Mode" est utilisÃ© pour contrÃŽler le niveau du dÃ©tail dans l'interface. Dans sa configuration la plus basse, "Librarian Assistant", la vue de dÃ©coration, est dÃ©sactivÃ©e, les arguments qui requiÃšrent des expressions ordinaires sont cachÃ©s, et la construction de la collection produit un enregistrement minimal d'Ã©vÃšnements. Par contre, la configuration la plus Ã©levÃ©e, "Expert", donne l'accÃšs Ã  toutes les fonctionnalitÃ©s de dÃ©coration, y compris le positionnement de plugin, et les arguments d'expressions ordinaires; elle permet aussi l'enregistrement de l'intÃ©gralitÃ© des messages lors de la construction de la collection. Pour changer ou revoir les modes, cliquez sur le bouton radio situÃ© Ã  cÃŽtÃ© du mode qui vous intÃ©resse. Vous pouvez contrÃŽler rapidement le mode dans lequel vous vous trouvez en regardant la barre de titre de Librarian Interface.
+      <Break/>
+      L'Interface BibliothÃ©caire est capable de supporter diffÃ©rents flux de travail en dÃ©terminant lesquels des diffÃ©rents onglets d'affichage sont visibles. Utiliser l'onglet "Workflow" pour adapter les fenÃªtres d'affichage disponibles en cochant les boÃ®tes qui sont Ã  cÃŽtÃ© des affichages souhaitÃ©s. On peut aussi utiliser la liste dÃ©roulante pour sÃ©lectionner des configurations prÃ©dÃ©terminÃ©es. La fermeture du dialogue des prÃ©fÃ©rences Ã©tablit ces paramÃštres de workflow (flux de travail). Ces paramÃštres sont emmagasinnÃ©s dans la collection, et non dans le fichier de configuration de l'Interface BibliothÃ©caire.
+      <Break/>
+      L'onglet "Connexion" permet changer le chemin vers le serveur de bibliothÃšque local basÃ© sur Greenstone et utilisÃ© dans l'aperÃ§u des collections. Vous pouvez aussi prÃ©ciser le proxy de connection internet (c.a.d. pour lÃ©lÃ©charger les fichiers; voir la <Reference target="downloadingfiles">TÃ©lÃ©charger les fichiers de l'Internet</Reference> section pour les details). Cochez la case pour une connection proxy et donnez l'addresse du proxy et son numÃ©ro de port. La connection proxy sera Ã©tablie aprÃšs fermeture du dialogue.
+      <Break/>
+      En cours de session, l'Interface BibliothÃ©ccaire peut dÃ©livrer des messages d'avertissement qui informent sur d'Ã©ventuelles consÃ©quences d'une action. On peut dÃ©sactiver le message  en cochant la case "Ne pas afficher cet avertissement de nouveau" ("Do not show this warning again"). On peut rÃ©activer les messages d'avertissement en utilisant l'onglet "Avertissement" ("Warning"). Cocher la boÃ®te situÃ©e Ã  cÃŽtÃ© des messages d'avertissement que l'on souhaite reconsulter. 
+    </Section>
+    <Section name="fileassociations">
+      <Title>Associations de fichiers</Title>
+      L'Interface BibliothÃ©caire utilise des programmes d'application particuliÃšres pour ouvrir des types de fichiers particuliers. Cette section explique comment assigner et Ã©diter ces associations de fichiers. 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+           Ajouter association
+          </ContentsItem>
+          <ContentsItem>
+            Editer association
+          </ContentsItem>
+          <ContentsItem>
+            Enlever association
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Pour modifier des associations ouvrez le menu "Fichier" et cliquer sur "Association de fichiers". 
+      <Break/>
+      Pour ajouter une association, sÃ©lectionner l'extension du fichier cible dans la liste dÃ©roulant, ou taper dans une nouvelle extension (n'y incluer pas le ".") Ensuite soit taper la commande qui lance l'application voulue dans la champ appropriÃ©, soit choisir l'application dans le dialogue "Navigation". "%1" Peut Ãªtre utilisÃ© dans la commande de lancement pour insÃ©rer le nom du fichier qu'on veut ouvrir. Une fois cela, "Ajouter Association" est activÃ©. Cliquer dessus pour ajouter l'association.
+      <Break/>
+      Pour Ã©diter une association, sÃ©lectionner une extension de fichier existante. Toute commande existante associÃ©e est affichÃ©e dans le champ de la commande de lancement. Corriger-la, et puis cliquer sur "Remplacer Association".
+      <Break/>
+      Pour enlever une association, sÃ©lectionner une extension de fichier existante et cliquer sur "Enlever Association". (L'extension du fichier reste dans la liste dÃ©roulante "For Files Ending").
+      <Break/>
+      Les associations de fichiers sont stockÃ©es dans le dossier principal du rÃ©pertoire principal de l'Interface BibliothÃ©caire, dans un fichier appelÃ© "associations.xml".
+    </Section>
+    <Section name="exportingcollections">
+      <Title>Exportation de collections vers CD-ROM</Title>
+      Cette section dÃ©crit comment exporter des collections vers un CD-ROM d'auto-installation Greenstone.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>Comment exporter</ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">Instructions simples:</Anchor>
+      Greenstone peut exporter une ou plusieurs collections vers un CD-ROM d'auto-installation pour Windows. Pour ce faire, le paquetage Greenstone "Export to CD-ROM" doit Ãªtre installÃ©. Il n'est pas inclut par dÃ©faut, donc vous aurez peut Ãªtre Ã  modifier votre installation pour l'inclure.
+      <Break/>
+      Pour exporter une collection, ouvrir le menu "Fichier" et choisir "Exporter vers CD-ROM". Une liste de collections Greenstone apparaÃ®t; cliquez sur n'importe laquelle pour obtenir une description. Cochez la case des collections Ã  exporter. Vous pouvez entrer le nom du CD-ROM dans la case: c'est ce qui va apparaÃ®tre dans le menu de dÃ©marrage lorsque le CD-ROM aura Ã©tÃ© installÃ©. Puis cliquez sur "Exporter". Ce processus implique la copie de plusieurs fichiers et peut durer quelques minutes.
+      <Break/>
+      A la fin de l'opÃ©ration, Greenstone va afficher le nom d'un dossier contenant les collections exportÃ©es. Utilisez un graveur de CD pour copier le contenu dans un CD vierge.
+    </Section>
+  </Section>
+</Document>
Index: /main/tags/3.01/gli/help/fr/help_index.xml
===================================================================
--- /main/tags/3.01/gli/help/fr/help_index.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/help_index.xml	(revision 21112)
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Document><Section name="introduction"><Title>Introduction</Title><Section name="ofmiceandmenus"><Title>Dessouris et Menus</Title></Section><Section name="howtoavoidthisdocument"><Title>Comment Ã©viter de lire ce document</Title></Section></Section><Section name="startingoff"><Title>DÃ©marrage</Title><Section name="creatingacollection"><Title>CrÃ©er une nouvelle collection</Title></Section><Section name="savingacollection"><Title>Enregistrer la Collection</Title></Section><Section name="openingacollection"><Title>Ouvrir une collection existante</Title></Section><Section name="deletingcollections"><Title>Suppression des collections</Title></Section></Section><Section name="downloadingfiles"><Title>TÃ©lÃ©chargement de fichiers de lIinternet</Title><Section name="themirrorview"><Title>La vue TÃ©lÃ©chargement</Title></Section></Section><Section name="collectingfiles"><Title>Collecter les fichiers pour la Collection</Title><Section name="thegatherview"><Title>L'affichage Rassembler</Title></Section><Section name="creatingfolders"><Title>CrÃ©er des dossiers</Title></Section><Section name="addingfiles"><Title>Ajouter des fichiers</Title></Section><Section name="removingfiles"><Title>Supprimer des Fichiers</Title></Section><Section name="filteringthetree"><Title>Filtrer l'arborescence</Title></Section></Section><Section name="enrichingacollection"><Title>Enrichir la Collection avec des mÃ©ta-donnÃ©es</Title><Section name="theenrichview"><Title>L'affichage Enrichir</Title></Section><Section name="selectingmetadatasets"><Title>SÃ©lectionner les jeux de mÃ©ta-donnÃ©es</Title></Section><Section name="appendingmetadata"><Title>Ajout de Nouvelles MÃ©ta-donnÃ©es</Title></Section><Section name="addingpreviouslydefinedmetadata"><Title>Ajouter des MÃ©ta-donnÃ©es prÃ©dÃ©finies</Title></Section><Section name="updatingmetadata"><Title>Mettre Ã  jour des MÃ©ta-donnÃ©es</Title></Section><Section name="removingmetadata"><Title>Enlever les mÃ©ta-donnÃ©es</Title></Section><Section name="reviewingmetadata"><Title>Passer en revue les MÃ©ta-donnÃ©es assignÃ©es</Title></Section><Section name="importingpreviouslyassignedmetadata"><Title>Importer des MÃ©ta-donnÃ©es PrÃ©-assignÃ©es</Title></Section></Section><Section name="designingacollection"><Title>DÃ©coration de l'apparence de votre collection</Title><Section name="thedesignview"><Title>L'Affichage conception</Title></Section><Section name="generalsettings"><Title>RÃ©glages gÃ©nÃ©raux</Title></Section><Section name="plugins"><Title>Plugiciels de Document</Title></Section><Section name="searchtypes"><Title>Types de recherche</Title></Section><Section name="searchindexes"><Title>Index de recherche</Title></Section><Section name="partitionindexes"><Title>Index de Partition</Title><Section name="definefilters"><Title>DÃ©finir les Filtres</Title></Section><Section name="assignpartitions"><Title>Assigner des Partitions</Title></Section><Section name="assignlanguages"><Title>Assigner Langages</Title></Section></Section><Section name="xcollectionsearching"><Title>Recherche unique Ã  travers plusieurs collections</Title></Section><Section name="classifiers"><Title>Classificateurs</Title></Section><Section name="formatstatements"><Title>CaractÃ©ristiques de Format</Title></Section><Section name="translatetext"><Title>Traduire Texte</Title></Section><Section name="metadatasets"><Title>jeux de MÃ©ta-donnÃ©es</Title></Section></Section><Section name="producingthecollection"><Title>Pour produire votre collection</Title><Section name="thecreateview"><Title>L'Affichage CrÃ©er</Title></Section><Section name="buildsettings"><Title>ParamÃštres d'Importation et de Construction</Title></Section></Section><Section name="miscellaneous"><Title>Miscellaneous</Title><Section name="preferences"><Title>PrÃ©fÃ©rences</Title></Section><Section name="fileassociations"><Title>Associations de fichiers</Title></Section><Section name="exportingcollections"><Title>Exportation de collections vers CD-ROM</Title></Section></Section></Document>
Index: /main/tags/3.01/gli/help/fr/howtoavoidthisdocument.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/howtoavoidthisdocument.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/howtoavoidthisdocument.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="howtoavoidthisdocument"><font size="5" face="Verdana"><strong>1.2: Comment &eacute;viter de lire ce document</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Ne pas lire l'Aide en entier! Lire juste ce qui est n&eacute;cessaire pour obtenir l'aide dont on a besoin.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Aide contextuelle intuitive</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Outils (Tool tips)</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Autres documents</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      L'&eacute;l&eacute;ment de menu Aide indique l'&eacute;l&eacute;ment d'aide le plus appropri&eacute; par une ic&ocirc;ne sous forme de petit livre. 
+      <p></p>
+      Le positionnement de la souris sur plusieurs contr&ocirc;les entra&icirc;ne l'apparition d'un "Tool tip" qui dit ce qu'il faut faire.
+      <p></p>
+      Avant d'utiliser l'Interface Biblioth&eacute;caire, il faut lire la documentation Greenstone. La section "Cr&eacute;er une Nouvelle Collection" d&eacute;crit comment cr&eacute;er une nouvelle collection. Avant d'utiliser l'Interface Biblith&eacute;caire, lire d'abord la documentation de Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/importingpreviouslyassignedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/importingpreviouslyassignedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/importingpreviouslyassignedmetadata.htm	(revision 21112)
@@ -0,0 +1,53 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="importingpreviouslyassignedmetadata"><font size="5" face="Verdana"><strong>5.8: Importer des M&eacute;ta-donn&eacute;es Pr&eacute;-assign&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment importer des m&eacute;ta-donn&eacute;es affect&eacute;es pr&eacute;c&eacute;demment; m&eacute;ta-donn&eacute;es affect&eacute;es avant d'&ecirc;tre ajout&eacute;es &agrave; la collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    En ajoutant des fichiers
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Si la m&eacute;ta-donn&eacute;e dans une forme reconnue par l'Interface Biblioth&eacute;caire a &eacute;t&eacute; pr&eacute;alablement assign&eacute;e &agrave; un fichier, -- par exemple, quand on choisit des documents dans une collection Greenstone existante -- elle est import&eacute;e automatiquement quand on ajoute le fichier. Pour ce faire, la m&eacute;ta-donn&eacute;e doit &ecirc;tre ajust&eacute;e au jeu de m&eacute;ta-donn&eacute;es disponibles dans la collection.
+      <p></p>
+      L'Interface Biblioth&eacute;caire confirme l'information n&eacute;cessaire. confirmation sous forme d'instructions br&egrave;ves et qui montre ensuite le nom de l'&eacute;l&eacute;ment m&eacute;ta-donn&eacute;e qui est import&eacute;e, comme il appara&icirc;t dans le fichier source. Ce champ ne peut &ecirc;tre ni &eacute;dit&eacute; ni chang&eacute;. Ensuite on choisit sur quel jeu de m&eacute;ta-donn&eacute;e le nouveau &eacute;l&eacute;ment devrait &ecirc;tre calqu&eacute; et enfin l'&eacute;l&eacute;ment m&eacute;ta-donn&eacute;e appropri&eacute;. Le syt&egrave;me selectionne automatiquement le meilleur ajustement, en terme de jeu et d'&eacute;l&eacute;ment pour la nouvelle m&eacute;ta-donn&eacute;e.
+      <p></p>
+      Apr&egrave;s cela,on peut choisir "Ajouter" pour ajouter le nouvel &eacute;l&eacute;ment m&eacute;ta-donn&eacute;e au jeu de m&eacute;ta-donn&eacute;es choisi. (Ceci n'est possible que si aucun &eacute;lement de m&ecirc;me nom n'est pr&eacute;sent &agrave; l'interieur du jeu choisi.) "Remplacer" applique le nouvel &eacute;lement &agrave; celui choisi par l'utilisateur. Enfin, "Ignorer" n'importe aucune m&eacute;ta-donn&eacute;e avec ce nom d'&eacute;lement. Une fois pr&eacute;cis&eacute;e la mani&egrave;re d'importer un certain &eacute;l&eacute;ment de m&eacute;ta-donn&eacute;e, l'information d'application est retenue pour la dur&eacute;e de vie de la collection.
+      <p></p>
+      Pour des d&eacute;tails concenant les fichiers m&eacute;ta-donn&eacute;e.xml que Greenstone utilise pour sauvegarder les m&eacute;ta-donn&eacute;es, il faut vous ref&eacute;rer au chapitre 2 du guide du D&eacute;veloppeur de Greenstone -- "Exploitez vos documents au maximum".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/introduction.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/introduction.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/introduction.htm	(revision 21112)
@@ -0,0 +1,17 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="introduction"><font size="5" face="Verdana"><strong>1: Introduction</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    L'interface biblioth&eacute;caire de Greenstone est un outil pour collecter et enrichir des documents et ensuite mettre sur pied des collections de biblioth&egrave;ques num&eacute;riques. Il fournit un acc&egrave;s &agrave; la fonctionnalit&eacute; du logiciel de la Biblioth&egrave;que num&eacute;rique de Greenstone &agrave; partir d'un point de vue graphique et d'une interface "clic".
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/metadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/metadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/metadatasets.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="metadatasets"><font size="5" face="Verdana"><strong>6.11: jeux de M&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique le panel de visualisation du jeu de m&eacute;ta-donn&eacute;es. Sous "Sections de conception", cliquer sur "jeux de M&eacute;ta-donn&eacute;es". 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            jeux de m&eacute;ta-donn&eacute;es disponibles
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajout de jeux m&eacute;ta-donn&eacute;es
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            &Eacute;dition de jeux m&eacute;ta-donn&eacute;es
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Suppression de jeux m&eacute;ta-donn&eacute;es
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Cette fen&ecirc;tre sert &agrave; la revue des jeux de m&eacute;ta-donn&eacute;es utilis&eacute;s dans la collection, et les &eacute;l&eacute;ments qui sont disponibles dans chaque jeu. Choisir dans la liste de "jeux de m&eacute;ta-donn&eacute;es disponibles" pour voir des informations d&eacute;taill&eacute;es &agrave; ces &eacute;l&eacute;ments.
+      <p></p>
+      Pour utiliser un autre jeu de m&eacute;ta-donn&eacute;es avec la collection charg&eacute;e, clicquez sur "Ajouter un jeu de m&eacute;ta-donn&eacute;es" et s&eacute;lectionnez le fichier m&eacute;ta-donn&eacute;es (.mds) pour le nouveau jeu m&eacute;ta-donn&eacute;es.
+      <p></p>
+      L'&eacute;dition de jeux m&eacute;ta-donn&eacute;es se fait avec l'&eacute;diteur Greenstone de m&eacute;ta-donn&eacute;es (GEMS). En clicquant sur le bouton "&Eacute;diter un jeux m&eacute;ta-donn&eacute;es" vous obtiendrez des informations sur l'utilisation de GEMS.
+      <p></p>
+      Si vous n'avez plus besoin d'un jeu m&eacute;ta-donn&eacute;es, s&eacute;lectionnez le et clickez sur "Suppression du jeu m&eacute;ta-donn&eacute;es" pour l'enlever. Si vous avez affectt&eacute; des m&eacute;ta-donn&eacute;es aux &eacute;l&eacute;ments du jeu supprim&eacute;, on vous demandera ce que vous voulez en faire lorsque vous allez ouvrir &agrave; nouveau cette collection.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/miscellaneous.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/miscellaneous.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/miscellaneous.htm	(revision 21112)
@@ -0,0 +1,18 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="miscellaneous"><font size="5" face="Verdana"><strong>8: Miscellaneous</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Cette section d&eacute;crit les caract&eacute;ristiques de l'Interface Biblioth&eacute;caire qui ne sont pas associ&eacute;es &agrave; une fen&ecirc;tre d'affichage particuli&egrave;re.
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/ofmiceandmenus.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/ofmiceandmenus.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/ofmiceandmenus.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="ofmiceandmenus"><font size="5" face="Verdana"><strong>1.1: Dessouris et Menus</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      La section "Dessouris et Menus" fournit une information de base sur la mani&egrave;re de dialoguer avec l'interface biblioth&eacute;caire. Si on est familier avec des programmes comme Internet Explorer ou Microsoft Office et aussi &agrave; l'aise avec les menus et les clics de la souris, passer &agrave; <a href="howtoavoidthisdocument.htm#howtoavoidthisdocument">section suivante</a>.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Concepts de base
+            <tr>
+<td colspan="2"> </td><td colspan="1">T&acirc;ches de la souris</td>
+</tr>
+            
+<tr>
+<td colspan="2"> </td><td colspan="1">Clavier</td>
+</tr>
+          
+</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Quitter le programme</td>
+</tr>
+	
+	
+<tr>
+<td colspan="3"><a href="#advanced">Instructions avanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Touches de raccourcis</td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      L'interface biblioth&eacute;caire suit les conventions de Microsoft Windows et requiert une connaissance de base de Windows.
+      <p></p>
+      Une position quelconque d'interaction sur l'&eacute;cran, tel qu'un bouton ou un champ de texte, est appel&eacute;e "contr&ocirc;le". A tout moment, un contr&ocirc;le appel&eacute; "Foyer" est mis en relief et r&eacute;pond au clavier. Plusieurs contr&ocirc;les vous permettent de s&eacute;lectionner les parties qui sont mises en relief en bleu fonc&eacute;. Certains contr&ocirc;les virent au gris pour indiquer qu'ils sont &eacute;teints.
+      <p></p>
+      On peut d&eacute;placer la souris, cliquer sur son bouton gauche (cliquer-gauche) ou sur son bouton droit (cliquer-droite) de fa&ccedil;on tout &agrave; fait usuelle. Plusieurs &eacute;l&eacute;ments sont susceptibles d'&ecirc;tre entrain&eacute;s par glissement de la souris son bouton gauche &eacute;tant maintenu appuy&eacute;, et d'&ecirc;tre ainsi d&eacute;pos&eacute;s quelque part en rel&acirc;chant le bouton. Des cibles de d&eacute;p&ocirc;t potentielles changent d'apparence quand un composant les atteint.
+      <p></p>
+      On peut utiliser le clavier pour la saisie &agrave; l'int&eacute;rieur des champs de texte. Des alternatives au clavier sont disponibles pour plusieurs contr&ocirc;les indiqu&eacute;s par un nom cl&eacute; entre crochets --- par exemple [Tab] change le foyer. Le signe plus indique les autres touches susceptibles d'&ecirc;tre appuy&eacute;es en m&ecirc;me temps.
+      <p></p>
+      Pour quitter l'Interface Biblioth&eacute;caire, on utilise "Sortir" du menu "Fichier". Votre collection sera d'abord sauvegard&eacute;e.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions avanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Pour acc&eacute;der au menu, appuyer sur le bouton [Alt] et sur la lettre correspondante (soulign&eacute;e). Par exemple, pour le menu "Fichier" appuyer sur [Alt] + [F]. Pour choisir un &eacute;l&eacute;ment, appuyer sur la touche correspondante. Par exemple, lorsque le menu "Fichier" est d&eacute;roul&eacute;, appuyer sur "S" pour sauvegarder la collection.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/openingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/openingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/openingacollection.htm	(revision 21112)
@@ -0,0 +1,64 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="openingacollection"><font size="5" face="Verdana"><strong>2.3: Ouvrir une collection existante</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section indique la mani&egrave;re d'ouvrir des collections existantes en utilisant "Pr&ecirc;t pour Ouvrir Collection".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Comment ouvrir la Collection</td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Collections verrouill&eacute;es</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Collections de versions ant&eacute;rieures</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour ouvrir une collection existante, choisir "ouvrir" &agrave; partir du menu "Fichier" pour obtenir le "Pr&ecirc;t &agrave; l'ouverture de la collection". Une liste de vos collections Greenstone appara&icirc;t. En s&eacute;lectionner une pour voir sa description, et cliquer sur "Ouvrir" pour la charger. Si l'on cherche une collection qui est hors du dossier "Collect" de Greenstone, cliquer sur "Parcourir" pour un dialogue avec le syst&egrave;me de fichiers concern&eacute;.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Dans le cas o&ugrave; plusieurs programmes d'Interface Biblioth&eacute;caire de Greenstone sont ex&eacute;cut&eacute;s simultan&eacute;ment, les r&eacute;pertoires concern&eacute;s sont v&eacute;rouill&eacute;s pour pr&eacute;venir une interf&eacute;rence. A l'ouverture d'une collection, un petit fichier de verrouillage temporaire est cr&eacute;&eacute; dans son dossier. Avant d'ouvrir une collection, l'interface Biblioth&eacute;caire v&eacute;rifie qu'aucun fichier de verrouillage n'existe d&eacute;j&agrave;. Le verrouillage d'une collection est indiqu&eacute; par la couleur de son ic&ocirc;ne: verte pour une collection normale, rouge quand elle est v&eacute;rouill&eacute;e. Cependant, quand l'on quitte l'Interface Biblioth&eacute;caire pr&eacute;matur&eacute;ment, le fichier de verrouillage est parfois laiss&eacute; en place. Quand une telle collection est ouverte, l'Interface Biblioth&eacute;caire demande si l'on veut se d&eacute;faire de son contr&ocirc;le. Ne jamais sosutraire une collection en cours d'utilisation par un autre.
+      <p></p>
+      Lorsque vous ouvrez une collection qui n'a pas &eacute;t&eacute; cr&eacute;&eacute;e par Greenstone Librarian Interface, on vous demande de s&eacute;lectionner un ou plusieurs jeux de m&eacute;ta-donn&eacute;es. Si vous n'en s&eacute;lectionnez aucun, les m&eacute;ta-donn&eacute;es existantes seront ignor&eacute;es. Dans le cas contraire, les m&eacute;ta-donn&eacute;es seront import&eacute;es comme lorsque vous cliquez et glisser dans des fichiers contenant des m&eacute;ta-donn&eacute;es. Ce processus est d&eacute;crit dans la section <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">Importation de M&eacute;ta-donn&eacute;es Pr&eacute;c&eacute;demment Assign&eacute;es</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/partitionindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/partitionindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/partitionindexes.htm	(revision 21112)
@@ -0,0 +1,20 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="partitionindexes"><font size="5" face="Verdana"><strong>6.6: Index de Partition</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Les index sont construits sur un texte particulier ou des sources M&eacute;ta-donn&eacute;es. L'espace de recherche peut &ecirc;tre plus tard contr&ocirc;l&eacute; en morcelant l'index soit par langage ou par filtre choisi &agrave; l'avance. Cette section en d&eacute;crit la proc&eacute;dure. Sous "Sections de conception", cliquer "Index de Partition". 
+      <p></p>
+      L'affichage "Index de Partition" poss&egrave;de trois onglets: "D&eacute;finir les Filtres", "Assigner une partition" et "Assigner des langages". Pour en savoir davantage sur les partitions, lire sur les sous collections et les sous index au chapitre 2 de Greenstone developer's guide.
+      
+      
+      
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/plugins.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/plugins.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/plugins.htm	(revision 21112)
@@ -0,0 +1,90 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="plugins"><font size="5" face="Verdana"><strong>6.3: Plugiciels de Document</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit comment faire la configuration des plugiciels des documents que la collection utilise. Elle explique comment sp&eacute;cifier les plugiciels utilis&eacute;s, quels param&egrave;tres leur passer et dans quel ordre ils se d&eacute;roulent. Sous "Sections de conception", cliquer "Plugiciels de document".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter un plugiciel
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever un plugiciel
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configurer un plugiciel
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Changer l'ordre des plugiciels
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configuration d'usage
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour ajouter un plugiciel, s&eacute;lectionner-le sur la liste d&eacute;roulante de "Choisir le plugiciel &agrave; ajouter" situ&eacute;e &agrave; c&ocirc;t&eacute; du bouton et ensuite cliquer "Ajouter plugiciel". Une fen&ecirc;tre d&eacute;nomm&eacute;e "Configurer les arguments" appara&icirc;t; elle est d&eacute;crite plus loin. Une fois la configuration du nouveau plugiciel achev&eacute;e, il est ajout&eacute; &agrave; la fin de la liste "Plugiciels couramment assign&eacute;s". Veuillez noter que, &agrave; l'exception de UnknownPlug, chaque plugin ne peut appara&icirc;tre qu'une seule fois dans la liste.
+      <p></p>
+      Pour enlever un plugiciel, s&eacute;lectionner-le sur la liste et cliquer sur "Enlever plugiciel".
+      <p></p>
+      Les plugiciels sont configur&eacute;s en fournissant des arguments. Pour les changer, il faut s&eacute;lectionner le plugiciel &agrave; partir de la liste et cliquer "Configurer plugiciel" (Ou double-cliquer sur le plugiciel). Un dialogue "Configuration des arguments" appara&icirc;t avec diff&eacute;rent cont&ocirc;les pour sp&eacute;cifier les arguments.
+      <p></p>
+      Il existe plusieurs types de contr&ocirc;les. Certains sont des bo&icirc;tes de dialogue et cliquer sur l'une ajoute l'option appropri&eacute;e au plugiciel. Les autres sont des cha&icirc;nes de textes avec une bo&icirc;te de dialogue et un champ texte. Cliquer sur la bo&icirc;te pour activer l'argument, ensuite saisir le texte appropri&eacute; dans la bo&icirc;te (expression normale, chemin de fichier etc). Les autres sont des menus d&eacute;roulants &agrave; partir desquels on peut s&eacute;lectionner &agrave; partir d'une s&eacute;rie de valeurs d&eacute;termin&eacute;es. Pour apprendre ce qu'un argument peut faire, promener la souris planer sur son nom pendant un instant une description appara&icirc;tra.
+      <p></p>
+      Pour changer la configuration, cliquer sur "OK" pour valider et fermer la bo&icirc;te de dialogue, ou "Annuler" pour fermer la bo&icirc;te de dialogue sans changer les arguments du plugiciel.
+      <p></p>
+      Les plugiciels dans la liste sont execut&eacute;s en ordre et l'ordre est parfois important. Les deux plugiciels Qrcplug et Recplug sont vitaux au processus de construction de la collection, et sont plac&eacute;s &agrave; la fin de la liste (avec une ligne de s&eacute;paration). Pour changer l'ordre des autres, s&eacute;lectionner le plugiciel &agrave; d&eacute;placer et cliquer sur "D&eacute;placer vers le haut" ou "D&eacute;placer vers le bas".
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+       L'Interface Biblioth&eacute;caire s'&eacute;vertue &agrave; d&eacute;terminer les arguments support&eacute;s par le plugiciel. Cependant, l'utilisateur peut avoir &agrave; indiquer d'autres arguments sp&eacute;ciaux; pour cela un champ texte appel&eacute; "Arguments Sp&eacute;cifiques" (en bas) est disponible. Tout texte qu'il contient est annex&eacute; &agrave; la fin de la commande Plugiciel.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/preferences.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/preferences.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/preferences.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="preferences"><font size="5" face="Verdana"><strong>8.1: Pr&eacute;f&eacute;rences</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique le dialogue des pr&eacute;f&eacute;rences, auquel on acc&egrave;de en ouvrant "Fichier" -&gt; "Pr&eacute;f&eacute;rences". 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">G&eacute;n&eacute;ral</td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">Mode</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Flux de travail</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Connexion</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">Avertissement</td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      La premi&egrave;re option "General" est un champ texte pour saisir votre adresse courriel. Ceci sera utilis&eacute; pour les &eacute;l&eacute;ments m&eacute;ta-donn&eacute;es "cr&eacute;ateur" et "Mainteneur" de la collection. L'option suivante est un liste d&eacute;roulante des langues dans lesquelles l'interface bibioth&egrave;quaire peut &ecirc;tre affich&eacute;e. Si on change le dictionnaire par le choix d'un autre sur la liste, on doit red&eacute;marrer l'Interface Biblioth&eacute;caire pour charger les cha&icirc;nes du nouveau langage &agrave; partir du dictionnaire.
+      <p></p>
+Si on coche "Affichage M&eacute;ta-donn&eacute;es &eacute;tendues", les diff&eacute;rents contr&ocirc;les relatifs aux m&eacute;ta-donn&eacute;es affichent toujours toutes les m&eacute;ta-donn&eacute;es qui ont &eacute;t&eacute; extraites automatiquement des documents. Lorsqu'on le d&eacute;s&eacute;lectionne, ces m&eacute;ta-donn&eacute;es sont cach&eacute;es (bien qu'elles restent disponibles pendant la conception de la collection, et dans la collection Greenstone finale.) Si on coche "Montrer les tailles des fichiers", la taille du fichier appara&icirc;t &agrave; c&ocirc;t&eacute; de chaque fichier dans l'espace de travail et dans les arborescences de fichiers de la collection dans les Affichages Rassembler et Enrichir.
+      <p></p>
+      Le panneau "Mode" est utilis&eacute; pour contr&ocirc;ler le niveau du d&eacute;tail dans l'interface. Dans sa configuration la plus basse, "Librarian Assistant", la vue de d&eacute;coration, est d&eacute;sactiv&eacute;e, les arguments qui requi&egrave;rent des expressions ordinaires sont cach&eacute;s, et la construction de la collection produit un enregistrement minimal d'&eacute;v&egrave;nements. Par contre, la configuration la plus &eacute;lev&eacute;e, "Expert", donne l'acc&egrave;s &agrave; toutes les fonctionnalit&eacute;s de d&eacute;coration, y compris le positionnement de plugin, et les arguments d'expressions ordinaires; elle permet aussi l'enregistrement de l'int&eacute;gralit&eacute; des messages lors de la construction de la collection. Pour changer ou revoir les modes, cliquez sur le bouton radio situ&eacute; &agrave; c&ocirc;t&eacute; du mode qui vous int&eacute;resse. Vous pouvez contr&ocirc;ler rapidement le mode dans lequel vous vous trouvez en regardant la barre de titre de Librarian Interface.
+      <p></p>
+      L'Interface Biblioth&eacute;caire est capable de supporter diff&eacute;rents flux de travail en d&eacute;terminant lesquels des diff&eacute;rents onglets d'affichage sont visibles. Utiliser l'onglet "Workflow" pour adapter les fen&ecirc;tres d'affichage disponibles en cochant les bo&icirc;tes qui sont &agrave; c&ocirc;t&eacute; des affichages souhait&eacute;s. On peut aussi utiliser la liste d&eacute;roulante pour s&eacute;lectionner des configurations pr&eacute;d&eacute;termin&eacute;es. La fermeture du dialogue des pr&eacute;f&eacute;rences &eacute;tablit ces param&egrave;tres de workflow (flux de travail). Ces param&egrave;tres sont emmagasinn&eacute;s dans la collection, et non dans le fichier de configuration de l'Interface Biblioth&eacute;caire.
+      <p></p>
+      L'onglet "Connexion" permet changer le chemin vers le serveur de biblioth&egrave;que local bas&eacute; sur Greenstone et utilis&eacute; dans l'aper&ccedil;u des collections. Vous pouvez aussi pr&eacute;ciser le proxy de connection internet (c.a.d. pour l&eacute;l&eacute;charger les fichiers; voir la <a href="downloadingfiles.htm#downloadingfiles">T&eacute;l&eacute;charger les fichiers de l'Internet</a> section pour les details). Cochez la case pour une connection proxy et donnez l'addresse du proxy et son num&eacute;ro de port. La connection proxy sera &eacute;tablie apr&egrave;s fermeture du dialogue.
+      <p></p>
+      En cours de session, l'Interface Biblioth&eacute;ccaire peut d&eacute;livrer des messages d'avertissement qui informent sur d'&eacute;ventuelles cons&eacute;quences d'une action. On peut d&eacute;sactiver le message  en cochant la case "Ne pas afficher cet avertissement de nouveau" ("Do not show this warning again"). On peut r&eacute;activer les messages d'avertissement en utilisant l'onglet "Avertissement" ("Warning"). Cocher la bo&icirc;te situ&eacute;e &agrave; c&ocirc;t&eacute; des messages d'avertissement que l'on souhaite reconsulter. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/producingthecollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/producingthecollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/producingthecollection.htm	(revision 21112)
@@ -0,0 +1,17 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="producingthecollection"><font size="5" face="Verdana"><strong>7: Pour produire votre collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Apr&egrave;s avoir rassembl&eacute; les documents de la collection, les avoir annot&eacute;s avec les m&eacute;ta-donn&eacute;es, et d&eacute;fini comment la collection va appara&icirc;tre, la production de la collection en utilisant Greenstone est alors possible. Cette section vous en explique la mani&egrave;re.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/removingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/removingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/removingfiles.htm	(revision 21112)
@@ -0,0 +1,61 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingfiles"><font size="5" face="Verdana"><strong>4.4: Supprimer des Fichiers</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit la mani&egrave;re de supprimer des fichiers et des dossiers de la collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le bouton Supprimer
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            La touche Supprimer
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Supprimer par glisser-d&eacute;placer
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Il existe plusieurs m&eacute;thodes pour supprimer des fichiers et des dossiers. On doit d'abord indiquer les &eacute;l&eacute;ments &agrave; supprimer en s&eacute;lectionnant un ou plusieurs fichiers et dossiers comme d&eacute;crit dans <a href="thegatherview.htm#thegatherview">L'Affichage Rassembler (The Gather view)</a>.
+      <p></p>
+      D&egrave;s lors que les fichiers sont s&eacute;lectionn&eacute;s, cliquer sur le bouton "Supprimer" pour les d&eacute;truire, ou appuyer sur la touche [Delete] de votre clavier, ou les glisser-d&eacute;placer de la collection vers le bouton "Supprimer" et les y d&eacute;poser.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/removingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/removingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/removingmetadata.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingmetadata"><font size="5" face="Verdana"><strong>5.6: Enlever les m&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment retirer des m&eacute;ta-donn&eacute;es d'un fichier.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever &agrave; partir de fichiers
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever &agrave; partir de dossiers ou de fichiers multiples
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      On enlever la m&eacute;ta-donn&eacute;e de la m&ecirc;me fa&ccedil;on que sa mise &agrave; jour. S&eacute;lectionner d'abord un fichier &agrave; partir de l'arborescence des fichiers, utiliser ensuite le tableau de m&eacute;ta-donn&eacute;e pour s&eacute;lectionner la m&eacute;ta-donn&eacute;e. Si la m&eacute;ta-donn&eacute;e a une valeur, le bouton "Enlever" devient actif. Cliquer-le pour enlever la m&eacute;ta-donn&eacute;e du fichier sp&eacute;cifi&eacute;. D'autres fichiers demeurent inchang&eacute;s et la valeur reste dans l'arborescence des valeurs. 
+      <p></p>
+      La suppression de m&eacute;ta-donn&eacute;es de r&eacute;pertoires ou de plusieurs fichiers fonctione de la m&ecirc;me fa&ccedil;on.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/reviewingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/reviewingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/reviewingmetadata.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="reviewingmetadata"><font size="5" face="Verdana"><strong>5.7: Passer en revue les M&eacute;ta-donn&eacute;es assign&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit la mani&egrave;re de visionner toutes les m&eacute;ta-donn&eacute;es assign&eacute;es &agrave; la colletion enti&egrave;re.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le tableau d'audit
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Le menu bouton droit
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Autofiltrage
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      On &eacute;prouve parfois le besoin de voir, &agrave; la fois, les m&eacute;ta-donn&eacute;es affect&eacute;es &agrave; plusieurs ou &agrave; tous les fichiers -- par exemple, pour d&eacute;terminer combien de fichiers sont en cours d'utilisation ou pour avoir une id&eacute;e de la chronologie. 
+      <p></p>
+      S&eacute;lectionnez les fichiers que vous souhaitez examiner, puis clicquez sur le bouton droit et choisissez "M&eacute;ta-donn&eacute;es Associ&eacute;es...". Une fen&ecirc;tre appel&eacute;e "Toutes les M&eacute;ta-donn&eacute;es", sous forme d'un grand tableau avec plusieurs colonnes, appara&icirc;t. La premi&egrave;re colonne montre les noms de fichiers; les lignes montrent toutes les valeurs de m&eacute;ta-donn&eacute;es assign&eacute;es &agrave; ces fichiers.
+      <p></p>
+      Dessiner le tableau peut prendre quelque temps si plusieurs fichiers sont s&eacute;lectionn&eacute;s. On peut continuer &agrave; utiliser l'Interface Biblioth&eacute;caire pendant que la fen&ecirc;tre "Toutes M&eacute;ta-donn&eacute;es" est ouverte.
+      <p></p>
+      Cliquer "Fermer" pour cacher la fen&ecirc;tre.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      On peut aussi visualiser le tableau "Toutes M&eacute;tadonn&eacute;es" en s&eacute;lectionnant les fichiers &agrave; examiner en cliquant sur le bouton droit et en choisissant "M&eacute;ta-donn&eacute;es Assign&eacute;es...". Si un dossier a &eacute;t&eacute; s&eacute;lectionn&eacute;, tous ses fichiers sont inclus dans le tableau.
+      <p></p>
+      Quand il devient trop large, on peut filtrer le tableau "Toutes M&eacute;ta-donn&eacute;es" en appliquant des filtres aux colonnes. Alors que de nouveaux filtres sont ajout&eacute;s, seules les lignes qui concernent restent visibles. Pour installer, modifier ou effacer un filtre, cliquez sur l'&icirc;cone "Funnel" situ&eacute; en haut de la colonne. Une information sur le filtre est r&eacute;clam&eacute;e. Une fois le filtre install&eacute;, l'en-t&ecirc;te de la colonne change de couleur.
+      <p></p>
+      La confirmation poss&egrave;de un onglet "simple" et " advanc&eacute;". La version simple filtre  les colonnes afin qu'elles ne montrent que des lignes qui contiennent une certaine valeur de m&eacute;ta-donn&eacute;e ("*" concerne toutes les valeurs). On peut s&eacute;lectionner des valeurs de m&eacute;ta-donn&eacute;es &agrave; partir d'une liste d&eacute;roulante. La version advanc&eacute;e permet diff&eacute;rentes op&eacute;rations d'harmonisation: doit commencer avec, ne contient pas, alphab&eacute;tiquement et est &eacute;gal &agrave;. La valeur &agrave; harmoniser peut &ecirc;tre &eacute;dit&eacute;e et prise sous forme d'une cha&icirc;ne quelconque (y compris "*"), et on peut opter pour une harmonisation insensible aux caract&egrave;res. Enfin, on peut sp&eacute;cifier une deuxi&egrave;me condition d'harmonisation pr&eacute;cisant une gamme de valeurs (en s&eacute;lectionnant AND) ou des valeurs alternatives (en selectionnant OR). En dessous de cette zone, une bo&icirc;te qui permet de changer l'ordre de tri (ascendant ou descendant) est plac&eacute;e. Enfin, cliquer sur "Installer Filtre" pour appliquer le nouveau filtre &agrave; la colonne. Cliquer sur "Effacer Filtre" pour enlever le filtre courant. Notez que les d&eacute;tails du filtre sont retenus m&ecirc;me si le filtre est effac&eacute;.
+      <p></p>
+      Par exemple, pour trier le tableau "Toutes M&eacute;ta-donn&eacute;es" , choisir une colonne, s&eacute;lectionner le cadre du filtre par d&eacute;faut (Un filtre simple sur "*") et choisir un ordre ascendant ou descendant.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/savingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/savingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/savingacollection.htm	(revision 21112)
@@ -0,0 +1,64 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="savingacollection"><font size="5" face="Verdana"><strong>2.2: Enregistrer la Collection</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit la mani&egrave;re d'enregistrer une collection, et la confirmation d'enregistrement de la collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Comment enregistrer
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            O&ugrave; sont sauvegard&eacute;s les fichiers
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Enregistrer r&eacute;guli&egrave;rement votre travail en choisissant "Enregistrer" dans le fichier "Menu". Enregistrer une collection ne signifie pas la rendre pr&ecirc;te &agrave; l'utilisation dans Greenstone (see <a href="producingthecollection.htm#producingthecollection">Confectionner votre Collection</a>).
+      <p></p>
+      L'Interface Biblith&eacute;caire prot&egrave;ge votre travail en l'enregistrant &agrave; la fermeture du programme ou &agrave; l'ouverture d'une autre collection.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Les collections sont enregistr&eacute;es en un fichier choisi pour la collection avec une extension ".col", localis&eacute; dans un r&eacute;pertoire de m&ecirc;me nom &agrave; l'int&eacute;rieur du dossier "collect" de l'installation de Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/searchindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/searchindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/searchindexes.htm	(revision 21112)
@@ -0,0 +1,75 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchindexes"><font size="5" face="Verdana"><strong>6.5: Index de recherche</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Les index pr&eacute;cisent les parties de la collection qui doivent &ecirc;tre recherch&eacute;es. Cette section explique comment ajouter et enlever des index et fixer un index par d&eacute;faut. Sous "Sections de conception", cliquer sur "Rechercher Index".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Ajouter un index
+          </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    &Eacute;diter un index
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever un index
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Index par d&eacute;faut
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour ajouter un index, taper un nom choisi dans le champ "Nom de l'Index". Selectionner les sources d'information &agrave; indexer en cliquant les bo&icirc;tes de dialogue &agrave; c&ocirc;t&eacute; d'eux. La liste montre aussi bien tous les &eacute;l&eacute;ments de m&eacute;ta-donn&eacute;es assign&eacute;s, que le texte entier. Apr&egrave;s avoir s&eacute;lectionn&eacute; les sources de donn&eacute;es, choisir la granularit&eacute; de l'index en utilisant le menu "Au niveau". Une fois ces d&eacute;tails complet&eacute;s, "Ajouter Index" devient actif  (&agrave; moins qu'il n'ait un index existant avec les m&ecirc;mes param&egrave;tres). Cliquez pour ajouter un nouvel index.
+      <p></p>
+      Pour &eacute;diter un index, s&eacute;lectionnez-le et changez les d&eacute;tails de l'index, puis clicquez sur "Remplacer l'index".
+      <p></p>
+      Pour enlever un index, il faut le s&eacute;lectionner sur la liste d'index assign&eacute;s et cliquer "Enlever index". 
+      <p></p>
+      L'index par d&eacute;faut, celui qui est utilis&eacute; dans la page de recherche de la collection est labell&eacute; parc "[Index par d&eacute;faut]" dans la liste "Index assign&eacute;s".
+      <p></p>
+      Si la recherche avanc&eacute;e est activ&eacute;e (voir Affichage des types de recherche), les contr&ocirc;les d'index sont diff&eacute;rents.Chaque index est seulement bas&eacute; sur une source de donn&eacute;es. Il existe une nouvelle pseudo-source de donn&eacute;es "Tous champs" qui fournit une recherche &agrave; travers tous les index sp&eacute;cifi&eacute;s &agrave; la fois. Des niveaux ne sont pas assign&eacute;s &agrave; un index sp&eacute;cifique, mais s'applique &agrave; tous les index : ainsi index et niveaux sont ajout&eacute;s s&eacute;parement. Les index sont enlev&eacute;s de la m&ecirc;me fa&ccedil;on que pr&eacute;c&eacute;demment, mais l'index par d&eacute;faut ne peut plus &ecirc;tre mis activ&eacute;.. il est simplement le premier index assign&eacute;. 
+      <p></p>
+      Pour cr&eacute;er des index sur toutes les sources cliquez le bouton "Ajouter Tous". Le nom de chaque index sera en d&eacute;faut pour le nom de source. Pour changer le nom, S&eacute;lectionner un index, changer ses d&eacute;tails et cliquez "Remplacer Index". 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/searchtypes.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/searchtypes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/searchtypes.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchtypes"><font size="5" face="Verdana"><strong>6.4: Types de recherche</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      cette section explique comment modifier une nouvelle caract&eacute;ristique de conception dans Greenstone, Types de recherche, qui permet une recherche de champ. Sous "Sections de conception", cliquer sur "Types de recherche".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Que signifie activer des recheches avanc&eacute;es?
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter un nouveau type de recherche
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Enlever un type de recherche
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Changer l'ordre des types de recherche
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Lorsqu'on p&eacute;n&egrave;tre dans l'affichage Types de recherche, v&eacute;rifier d'abord "Activer recherches avanc&eacute;es", qui active les autres contr&ocirc;les. la collection prend le format 2.4 de Greenstone, qui supporte la recherche bas&eacute;e sur les champs, et signifie que: 1. l'index de conception est diff&eacute;rent (expliqu&eacute; dans <a href="searchindexes.htm#searchindexes">Recherche d'index</a> section), 2. plusieurs fragments de textes sont &agrave; traduire (voir <a href="translatetext.htm#translatetext">Translation</a>), 3. la collection est inutilisable sous des versions plus vieilles de Greenstone. Si vous d&eacute;-s&eacute;lectionner ce champ, la plupart de la collection retourne &agrave; la version 2.39 de Greenstone 2.39. Cependant, l'Interface Biblioth&eacute;caire ne peut pas convertir les nouvelles sp&eacute;cifications d'index en celles plus anciennes; elles doivent ainsi &ecirc;tre saisies de nouveau manuellement. 
+      <p></p>
+      pour ajouter un type de recherche, s&eacute;lectionner-le &agrave; partir de la liste "Types de recherche" et cliquer sur "Ajouter Type de recherche".  Chaque type n'appara&icirc;t qu'une fois sur la liste.
+      <p></p>
+      Pour enlever un type de recherche, il faut le selectionner &agrave; partir de la liste "Currently Assigned Search type" et cliquer sur "Enlever type de recheche". La liste doit contenir au moins un type de recherche.
+      <p></p>
+      pour changer l'ordre d'un type de recherche, s&eacute;lectionner-le &agrave; partir de la liste et cliquer sur "D&eacute;placer vers le haut" ou "D&eacute;placer vers le bas". Le premier sera le d&eacute;faut.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/selectingmetadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/selectingmetadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/selectingmetadatasets.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="selectingmetadatasets"><font size="5" face="Verdana"><strong>5.2: S&eacute;lectionner les jeux de m&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique comment sp&eacute;cifier les &eacute;l&eacute;ments m&eacute;ta-donn&eacute;e et comment ajouter les jeux de m&eacute;ta-donn&eacute;es &agrave; une collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Qu'est-ce qu'un jeu de m&eacute;ta-donn&eacute;es?
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Contr&ocirc;le des jeux de m&eacute;ta-donn&eacute;es utilis&eacute;s
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Les s&eacute;ries d'&eacute;l&eacute;ments de m&eacute;ta-donn&eacute;es pr&eacute;d&eacute;finis sont connus comme "jeux de M&eacute;ta-donn&eacute;es". "Dublin Core" en est un exemple. Quand on ajoute un Groupe de M&eacute;ta-donn&eacute;es &agrave; la collection, ses &eacute;l&eacute;ments peuvent &ecirc;tre s&eacute;lectionn&eacute;s. On peut avoir plus d'un jeu de m&eacute;ta-donn&eacute;es ; pour &eacute;viter des t&eacute;lescopages de nom, un identificateur abr&eacute;g&eacute; qui identifie le jeu de m&eacute;ta-donn&eacute;es est suspendu au nom de l'&eacute;l&eacute;ment. Par exemple l'&eacute;lement cr&eacute;ateur de Dublin Core devient "dc.Creator". Les jeux de m&eacute;ta-donn&eacute;es sont sauv&eacute;gard&eacute;s dans le dossier de m&eacute;ta-donn&eacute;es de l'Interface Biblioth&eacute;caire et ont le sufixe ".mds".
+      <p></p>
+      Pour contr&ocirc;ler les jeux de m&eacute;ta-donn&eacute;es utilis&eacute;s dans une collection, utilisez l'entr&eacute;e "Jeux M&eacute;ta-Donn&eacute;es" sur la vue "Conception".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/startingoff.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/startingoff.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/startingoff.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="startingoff"><font size="5" face="Verdana"><strong>2: D&eacute;marrage</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Cette section explique comment cr&eacute;er, charger, enregistrer et supprimer des collections.
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/thecreateview.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/thecreateview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/thecreateview.htm	(revision 21112)
@@ -0,0 +1,70 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thecreateview"><font size="5" face="Verdana"><strong>7.1: L'Affichage Cr&eacute;er</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique l'utilisation de l'affichage cr&eacute;er pour produire une collection.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Construction de la collection
+	  </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Aper&ccedil;u de la collection
+	  </td>
+</tr>
+	
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Journal des messages
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      L'Affichage Cr&eacute;er est utilis&eacute; pour cr&eacute;er la collection en mettant en Åuvre les scripts Greenstone relatifs &agrave; cette op&eacute;ration. En cliquant sur "Construire Collection", on initie le processus de construction de la collection. Sa dur&eacute;e d&eacute;pend de la taille de la collection  et du nombre d'index &agrave; cr&eacute;er (pour les grandes collections cela peut prendre des heures). Une barre d'&eacute;tat indique ou on en est dans le processus. Pour annuler le processus &agrave; tout moment, cliquer sur "Annuler Construction".
+      <p></p>
+      Quand la collection est construite, appuyant "Preview Collection" initiera la page web hebergeant la collection.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      En mode expert, vous pouvez utiliser l'entr&eacute;e "Message Log" &agrave; gauche pour revoir les tentatives pr&eacute;c&eacute;dentes de construction de la collection avec ou san succ&eacute;s. S&eacute;lectionnez le journal souhait&eacute; en cliquant sur la date correspondante dans la liste "Historique du Journal".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/thedesignview.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/thedesignview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/thedesignview.htm	(revision 21112)
@@ -0,0 +1,51 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thedesignview"><font size="5" face="Verdana"><strong>6.1: L'Affichage conception</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section pr&eacute;sente l'affichage de conception et explique comment on navigue entre les diff&eacute;rents affichages &agrave; l'int&eacute;rieur de cette vitrine. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Choisir une section de conception
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Avec l'Interface Biblioth&eacute;caire, on peut configurer la fa&ccedil;on dont la collection va appara&icirc;tre &agrave; l'utilisateur. Les options de configuration sont divis&eacute;es en diff&eacute;rentes sections, chacune associ&eacute;e &agrave; une &eacute;tape particuli&egrave;re de navigation ou de pr&eacute;sentation d'information.
+      <p></p>
+      Sur la gauche est plac&eacute;e une liste de diff&eacute;rents affichages et &agrave; droite les commandes associ&eacute;es &agrave; celles en cours. Pour passer &agrave; un  affichage diff&eacute;rent, cliquer son nom dans la liste.
+      <p></p>
+      Pour comprendre les diff&eacute;rentes &eacute;tapes et les termes implmiqu&eacute;s dans la conception d'une collection, lire les chapitres 1 et 2 de Greenstone's Developer's Guide. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/theenrichview.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/theenrichview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/theenrichview.htm	(revision 21112)
@@ -0,0 +1,77 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="theenrichview"><font size="5" face="Verdana"><strong>5.1: L'affichage Enrichir</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit comment ins&eacute;rer et &eacute;diter une m&eacute;ta-donn&eacute;e en utilisant l'affichage Enrichir.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            L'arborescence de la collection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Les contr&ocirc;les d'&eacute;dition de m&eacute;ta-donn&eacute;es
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            L'arborescence valeur
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+           La m&eacute;ta-donn&eacute;e Table
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Utiliser l'affichage Enrichir pour assigner une m&eacute;ta-donn&eacute;e aux documents de la collection. Une m&eacute;ta-donn&eacute;e est une donn&eacute;e sur une donn&eacute;e -- de fa&ccedil;on typique titre, auteur, date de cr&eacute;ation, et ainsi de suite. Chaque &eacute;l&eacute;ment de m&eacute;ta-donn&eacute;e a deux parties: "el&eacute;ment" indique le type d'&eacute;l&eacute;ment (par exemple auteur), "valeur" qui donne la valeur de cette m&eacute;ta-donn&eacute;e &eacute;l&eacute;ment (par exemple le nom de l'auteur).
+      <p></p>
+      Sur la gauche de l'affichage "Enrichir" est l'arborescence de la collection. A sa droite, la Table de M&eacute;ta-donn&eacute;es, qui montre les m&eacute;ta-donn&eacute;es pour tous les fichiers et dossiers s&eacute;lectionn&eacute;es de l'arborescence de la collection. Les colonnes sont indiqu&eacute;es en gris en haut, et peuvent &ecirc;tre redimensionn&eacute;es en faisant glisser la ligne de s&eacute;paration. Clicquez sur n'importe quelle ligne pour transf&eacute;rer ses informations aux cont&ocirc;les de m&eacute;ta-donn&eacute;es ci-dessous. Si plusieurs fichiers sont s&eacute;lectionn&eacute;s, un texte en noir indique que la valeur est commune &agrave; tous les fichiers s&eacute;lectionn&eacute;s, tandis qu'un texte en gris en indique le contraire. les valeurs en noir peuvent &ecirc;tre mises &agrave; jour ou supprim&eacute;es, tandis que celles en gris peuvent &ecirc;tre supprim&eacute;es de ceux qui l'ont, ou ajout&eacute;es aux autres.
+      <p></p>
+      Une ic&ocirc;ne de dossier peut appara&icirc;tre &agrave; c&ocirc;t&eacute; de certaines entr&eacute;es de m&eacute;ta-donn&eacute;es. Ceci indique que les valeurs ont &eacute;t&eacute; h&eacute;rit&eacute;es d'un dossier parent. Les m&eacute;ta-donn&eacute;es h&eacute;rit&eacute;es ne peuvent &ecirc;tre &eacute;dit&eacute;es ou supprim&eacute;es, mais seulement ajout&eacute;es ou &eacute;cras&eacute;es. Cliquer sur l'ic&ocirc;ne du dossier pour aller imm&eacute;diatement au dossier o&ugrave; la m&eacute;ta-donn&eacute;e est assign&eacute;e.
+      <p></p>
+      Les contr&ocirc;les d'&eacute;dition de m&eacute;ta-donn&eacute;es en bas &agrave; droite apparaissent seulement quand un fichier dans l'arbre et une ligne dans la table sont s&eacute;lectionn&eacute;s. Utiliser les pour mettre &agrave; jour, ajouter et supprimer la valeur de la m&eacute;ta-donn&eacute;e. Le champ valeur sert &agrave; la saisie et &agrave; l'&eacute;dition de la valeur de la m&eacute;ta-donn&eacute;e. A c&ocirc;t&eacute;, le bouton de label "..." permet d'ouvrir une grande bo&icirc;te d'&eacute;dition. Les autres boutons situ&eacute;s en-dessous sont: "Coller ou ajouter" assigne la valeur comme nouvelle m&eacute;ta-donn&eacute;e et l'ajoute aux valeurs existantes de l'&eacute;l&eacute;ment s&eacute;lectionn&eacute;; "Remplacer" &eacute;crase la valeur existante s&eacute;lectionn&eacute;e avec la nouvelle; "Supprimer" efface la valeur s&eacute;lectionn&eacute;e. Au bas, le label "Toutes les valeurs pr&eacute;c&eacute;dentes" est la "Value Tree" ("Arborescence Valeur").
+      <p></p>
+      Celle-ci, d&eacute;veloppe et comprime. C'est une liste qui, de fa&ccedil;on usuelle, montre toutes les valeurs pr&eacute;c&eacute;demment saisies pour un &eacute;l&eacute;ment s&eacute;lectionn&eacute;. Le clic sur une entr&eacute;e la place automatiquement dans le champ valeur. Inversement, la saisie dans le champ texte s&eacute;lectionne l'entr&eacute;e de l'Arborescence Valeur qui d&eacute;bute avec les caract&egrave;res saisis. Appuyer sur [Tab] compl&egrave;te automatiquement la saisie de la valeur s&eacute;lectionn&eacute;e.
+      <p></p>
+      Les valeurs de m&eacute;ta-donn&eacute;e peuvent &ecirc;tre organis&eacute;es en une hi&eacute;rarchie.Ceci est montr&eacute; dans l'Arborescence Valeur utilisant des dossiers de niveau interne.Les valeurs hi&eacute;rarchiques peuvent &ecirc;tre saisies en utilisant le caract&egrave;re "|" pour s&eacute;parer les niveaux. Par exemple, "Cards|Red|Diamonds|Seven" peut &ecirc;tre utilis&eacute; dans une hi&eacute;rarchie qui repr&eacute;sente un jeu de cartes. Cela permet le rejeument des valeurs. Les jeux peuvent aussi &ecirc;trre assign&eacute;s comme m&eacute;ta-donn&eacute;e.
+      <p></p>
+      Greenstone extrait automatiquement des m&eacute;ta-donn&eacute;es &agrave; partir des documents en un jeu de m&eacute;ta-donn&eacute;es dont les &eacute;l&eacute;ments sont pr&eacute;fix&eacute;s "ex.". Ceci n'a aucune valeur et ne peut &ecirc;tre &eacute;dit&eacute;, et d&egrave;s lors les contr&ocirc;les MetaEdit sont cach&eacute;s si une telle entr&eacute;e est s&eacute;lectionn&eacute;e. Le bouton "..." sert toujours au d&eacute;veloppement de la valeur, mais le texte ne peut &ecirc;tre &eacute;dit&eacute;.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/thegatherview.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/thegatherview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/thegatherview.htm	(revision 21112)
@@ -0,0 +1,102 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thegatherview"><font size="5" face="Verdana"><strong>4.1: L'affichage Rassembler</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+cette section introduit la zone Rassembler utilis&eacute;e pour selectionner les fichiers &agrave; inclure dans la collection &agrave; cr&eacute;er. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+           comment afficher l'&eacute;cran "gather"
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Les arborescences de fichiers
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            La zone d'&eacute;tat
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Boutons de contr&ocirc;le
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions avanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            m&eacute;thodes de s&eacute;lection
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Application de dossiers sp&eacute;ciaux
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      L'Interface Biblioth&eacute;caire d&eacute;marre avec l'affichage "Rassembler". Pour revenir ult&eacute;rieurement &agrave; celui-ci, cliquer directement sur l'onglet "Rassembler" au bas de la barre des menus.
+      <p></p>
+      Les deux vastes zones intitul&eacute;es "Espace de travail" et "Collection" servent au transfert des fichiers vers la collection. Elles contiennent des arborescences de fichiers, des structures graphiques qui repr&eacute;sentent des fichiers et des dossiers.
+      <p></p>
+      Choisir un &eacute;l&eacute;ment dans l'arborescence en cliquant dessus. (d'autres mani&egrave;res de le faire existent; voir ci-dessous). Double-cliquer sur un dossier, ou cliquer simplement sur le symbole de commutateur plac&eacute; &agrave; c&ocirc;t&eacute; de lui, pour d&eacute;velopper (ou r&eacute;duire) son contenu. Double-cliquer sur un fichier pour l'ouvrir en utilisant un programme d'application associ&eacute; (voir <a href="fileassociations.htm#fileassociations">Association de fichiers</a>).
+      <p></p>
+      L'arbre des fichiers de l'espace de travail montre les sources de donn&eacute;es disponibles pour l'interface biblioth&eacute;caire -- le syst&egrave;me de fichiers local (ainsi que les unit&eacute;s de disques et Cd-rom), le contenu des collection Greenstone, et la cache des fichiers t&eacute;l&eacute;charg&eacute;s si l'option Web &laquo;&nbsp;miroir&nbsp;&raquo; est active. Vous pouvez copier et afficher ces fichiers mais vous ne pouvez pas les d&eacute;placer, supprimer ou &eacute;diter, &agrave; l'exception des fichiers t&eacute;l&eacute;charg&eacute;s qui peuvent &ecirc;tre supprim&eacute;. Vous pouvez naviguer &agrave; l'int&eacute;rieur de cet espace pour trouver les documents que vous souhaitez ajouter &agrave; la collection.
+      <p></p>
+      L'arborescence des fichiers de la Collection ne repr&eacute;sente jusque-l&agrave; que le contenu de la collection. Elle est initialement vide.
+      <p></p>
+      On peut redimensionner les espaces en positionnant la souris sur la barre grise qui s&eacute;pare les arborescences (le pointeur change de forme) et en la faisant glisser.
+      <p></p>
+      En bas de la fen&ecirc;tre, il y a une barre d'&eacute;tat qui montre l'&eacute;volution des op&eacute;rations sur les fichiers (copie, d&eacute;placement et suppression). Ces op&eacute;rations peuvent &ecirc;tre longues. Le bouton "Stop" arr&ecirc;te toute t&acirc;che en cours.
+      <p></p>
+      Deux grands boutons occupent le coin droit inf&eacute;rieur de l'&eacute;cran. "Nouveau dossier", avec une image d'un dossier, cr&eacute;e de nouveaux dossiers (voir <a href="creatingfolders.htm#creatingfolders">Cr&eacute;ation de dossiers</a>). "Supprimer", avec une corbeille, supprime les fichiers. Cliquer sur le bouton "Supprimer" d&eacute;truit tous les fichiers s&eacute;lectionn&eacute;s de l'arborescence de fichiers de la collection. On peut de m&ecirc;me r&eacute;aliser cette suppression de fichiers en les faisant glisser jusqu'au bouton "Supprimer". 
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions avanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      Pour s&eacute;lectionner plusieurs s&eacute;quences d'&eacute;l&eacute;ments, s&eacute;lectionner le premier, maintenir appuy&eacute;e la touche [Maj] et cliquer sur le dernier &eacute;l&eacute;ment de la s&eacute;quence -- la s&eacute;lection comprendra tous les &eacute;l&eacute;ments compris entre les deux. Pour la s&eacute;lection de fichiers qui ne se suivent pas, maintenir la touche [Ctrl] appuy&eacute;e en cliquant sur chacun d'eux. Ces deux m&eacute;thodes peuvent &ecirc;tre utilis&eacute;es ensemble pour la s&eacute;lection de jeux d'&eacute;l&eacute;ments non adjacents.
+      <p></p>
+      Certains dossiers -- tel que celui contenant vos propres pages Web -- ont parfois une signification sp&eacute;ciale. L'Interface Biblioth&eacute;caire peut appliquer de tels dossiers au premier niveau de l'arborescence de fichiers. Pour ce faire double-cliquer droite sur le dossier vis&eacute;. Choisir "Cr&eacute;er un Raccourci", saisir le nom du dossier. Pour enlever un &eacute;l&eacute;ment, cliquer-droite sur le fichier concern&eacute; et s&eacute;lectionner "Supprimer Raccourci".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/themirrorview.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/themirrorview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/themirrorview.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="themirrorview"><font size="5" face="Verdana"><strong>3.1: La vue T&eacute;l&eacute;chargement</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit comment on configure une tache de t&eacute;l&eacute;chargement et comment on contr&ocirc;le le t&eacute;l&eacute;chargement.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            L'arbre espace de travail
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Configuration de t&eacute;l&eacute;chargement
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            La liste des t&eacute;l&eacute;chargements
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Instructions advanc&eacute;es:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Activation des proxies web
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Acc&eacute;dez &agrave; la vue "T&eacute;l&eacute;chargement" en cliquant son index. La moiti&eacute; sup&eacute;rieure de l'&eacute;cran montre le contr&ocirc;le des t&eacute;l&eacute;chargements. La moiti&eacute; inf&eacute;rieure est initialement vide, mais montrera la liste des t&eacute;l&eacute;chargements en attente et r&eacute;alis&eacute;s.
+      <p></p>
+      Les fichiers sont t&eacute;l&eacute;charg&eacute;s dans un dossier de l'espace de travail appel&eacute; &laquo;&nbsp;Fichiers T&eacute;l&eacute;charg&eacute;s&nbsp;&raquo; (Uniquement pr&eacute;sent lorsque l'op&eacute;ration &laquo;&nbsp;miroir&nbsp;&raquo; est activ&eacute;e), et peut &ecirc;tre utilis&eacute; pour toutes les collections construites avec l'interface biblioth&eacute;caire. Les fichiers dans cette zone portent comme nom leur URL en entier. Un nouveau dossier est cr&eacute;&eacute; pour chaque site h&ocirc;te, suivi par d'autres dossiers pour chaque &eacute;l&eacute;ment du chemin. Cela permet de s'assurer que chaque fichier a un nom distinct.
+      <p></p>
+      Utilisez le premier des contr&ocirc;les de la configuration de t&eacute;l&eacute;chargement, &laquo; Source URL &raquo;, pour entrer l'URL de la ressource cible. Utilisez le contr&ocirc;le &laquo; Profondeur de T&eacute;l&eacute;chargement &raquo;  pour limiter le nombre de liens hypertextes &agrave; suivre pour le t&eacute;l&eacute;chargement : utiliser 0 pour t&eacute;l&eacute;charger une seule page ; utiliser 1 pour t&eacute;l&eacute;charger la page et toutes les pages r&eacute;f&eacute;renc&eacute;es par des liens hypertextes. La profondeur limite est ignor&eacute;e lors du t&eacute;l&eacute;chargement de m&eacute;dia autres que des pages html. Ensuite il y a plusieurs cases &agrave; cocher qui peuvent &ecirc;tre coch&eacute;es pour activer l'option sp&eacute;cifi&eacute;e. Une fois la configuration pr&ecirc;te, cliquez &laquo; T&eacute;l&eacute;charger &raquo; pour d&eacute;marrer l'op&eacute;ration de t&eacute;l&eacute;chargement. Il y a deux autres boutons de contr&ocirc;le : &laquo; Pr&eacute;f&eacute;rences &raquo;, qui renvoie &agrave; la section connexion de Pr&eacute;f&eacute;rences ou les param&egrave;tre du proxy peuvent &ecirc;tre &eacute;dit&eacute;s ; et &laquo; Effacer la cache &raquo; qui supprimes tous les fichiers t&eacute;l&eacute;charg&eacute;s pr&eacute;c&eacute;demment.
+      <p></p>
+      La liste de t&eacute;l&eacute;chargement a une entr&eacute;e pour chaque t&eacute;l&eacute;chargement de page Web.  Chaque entr&eacute;e a une r&eacute;gion de textes qui donne les d&eacute;tails de la t&acirc;che avec une barre de progr&egrave;s montrant l'activit&eacute; courante.  Trois boutons apparaissent &agrave; la gauche de chaque entr&eacute;e.  "Pause" est employ&eacute; pour pauser une t&acirc;che en cours de t&eacute;l&eacute;chargement.  "Voir le r&eacute;sum&eacute;" ouvre une fen&ecirc;tre montrant l'historique du t&eacute;l&eacute;chargement.  "Fermer" termine le t&eacute;l&eacute;chargement et enl&egrave;ve la t&acirc;che de la liste.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Instructions advanc&eacute;es:</a>
+<p style="text-align:justify"></p>
+      La <a href="preferences.htm#preferences">section Pr&eacute;f&eacute;rences</a> explique comment &eacute;tablir une connection Internet &agrave; travers un proxy.  Si l'authentification est n&eacute;cessaire, le proxy server demande l'identification et le mot de passe.  L'interface biblioth&eacute;caire n'enregistre pas les mots de passe entre les sessions.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/translatetext.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/translatetext.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/translatetext.htm	(revision 21112)
@@ -0,0 +1,63 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="translatetext"><font size="5" face="Verdana"><strong>6.10: Traduire Texte</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section d&eacute;crit l'Affichage de traduction, o&ugrave; on peut d&eacute;finir des fragments de texte sp&eacute;cifiques &agrave; un langage pour des parties de l'interface de la collection. Sous "Sections de conception", cliquer sur "Traduire Texte".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ajouter traduction
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Enlever traduction
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Mettre &agrave; jour traduction
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Choisir d'abord une entr&eacute;e dans la liste "Caract&eacute;ristiques". Les cha&icirc;nes sp&eacute;cifiques &agrave; un langage associ&eacute;es &agrave; la caract&eacute;ristiques apparaissent en bas. Utiliser la liste d&eacute;roulante de "Traduction de Langage" pour s&eacute;lectionner le langage cible, et saisir le texte traduit dans la zone de texte, en se ref&eacute;rant &agrave; "Fragment de Texte Initial" si n&eacute;cessaire. Cliquez sur "Ajouter Traduction" &agrave; la fin de l'op&eacute;ration. 
+      <p></p>
+      Pour supprimer une traduction existante, s&eacute;lectionner-la &agrave; partir du tableau "Traduction Assign&eacute;e" et cliquer sur "Enlever Traduction". 
+      <p></p>
+      Pour &eacute;diter une traduction, s&eacute;lectionner-la et &eacute;diter-la dans la zone de texte "Texte Traduit", et cliquer sur "Remplacer Traduction". 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/updatingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/updatingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/updatingmetadata.htm	(revision 21112)
@@ -0,0 +1,57 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="updatingmetadata"><font size="5" face="Verdana"><strong>5.5: Mettre &agrave; jour des M&eacute;ta-donn&eacute;es</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Cette section explique la mani&egrave;re de mettre &agrave; jour la m&eacute;ta-donn&eacute;e assign&eacute;e &agrave; un fichier.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Mettre &agrave; jour des fichiers
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Mettre &agrave; jour des dossiers et des fichiers multiples
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      Pour mettre &agrave; jour la valeur d'un &eacute;l&eacute;ment m&eacute;ta-donn&eacute;e, choisir d'abord le fichier auquel la valeur s'applique, et ensuite l'&eacute;l&eacute;ment m&eacute;ta-donn&eacute;e dont on veut changer la valeur. Votre s&eacute;lection appara&icirc;t dans les contr&ocirc;les d'&eacute;dition des m&eacute;ta-donn&eacute;es. Editer le champ valeur et cliquer sur "Remplacer" pour &eacute;craser la m&eacute;ta-donn&eacute;e.
+      <p></p>
+      Le processus est identique pour la mise &agrave; jour d'un dossier avec des sous-dossiers ou plusieurs fichiers, mais vous pouvez seulement mettre &agrave; jour les m&eacute;ta-donn&eacute;es qui sont communes &agrave; tous les fichiers/dossiers s&eacute;lectionn&eacute;s.
+      <p></p>
+      L'arborescence des valeurs montre toutes les valeurs pr&eacute;c&eacute;dentes, mais pas seulement celles d&eacute;sign&eacute;es actuellement. Ainsi la valeur que vous avez remplac&eacute;e restera dans l'arbre de valeur.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/fr/xcollectionsearching.htm
===================================================================
--- /main/tags/3.01/gli/help/fr/xcollectionsearching.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/fr/xcollectionsearching.htm	(revision 21112)
@@ -0,0 +1,49 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="xcollectionsearching"><font size="5" face="Verdana"><strong>6.7: Recherche unique &agrave; travers plusieurs collections</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Greenstone peut rechercher dans plusieurs collections comme si elles n'en constituaient qu'une. Cela se fait par la cr&eacute;ation d'une "super-collection" qui contient les collections individuelles. Sous "Sections de conception", cliquez sur "Recherche unique &agrave; travers plusieurs collections".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">Instructions simples:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            S&eacute;lectionner une collection
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> Instructions simples:</a>
+<p style="text-align:justify"></p>
+      L'Affichage de Recherche unique &agrave; travers des collections montre une liste de collections disponibles. La collection courante est coch&eacute;e et ne peut &ecirc;tre d&eacute;-s&eacute;lectionn&eacute;e. Pour ajouter une autre collection qu'on peut rechercher parall&egrave;lement, cliquer sur une collection dans la liste (cliquer une deuxi&egrave;me fois pour l'enlever). Lorsqu'une seule collection est s&eacute;lectionn&eacute;e, aucune recherche &agrave; travers collections n'est possible.
+      <p></p>
+      Pour plus de d&eacute;tails, voir Chapitre 1 du Guide du D&eacute;veloppeur Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/gen-index-xml.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-index-xml.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-index-xml.xsl	(revision 21112)
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="xml" encoding="UTF-8"/>
+
+  <xsl:template match="Document">
+    <Document>
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection"/>
+    </xsl:for-each>
+    </Document>
+  </xsl:template>
+
+  <xsl:template name="processSection">
+    <Section name="{@name}">
+      <xsl:for-each select="Title">
+        <Title>
+          <xsl:apply-templates/>
+        </Title>
+      </xsl:for-each>
+
+      <xsl:for-each select="Section">
+        <xsl:call-template name="processSection"/>
+      </xsl:for-each>
+    </Section>
+  </xsl:template>
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/gen-many-html.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-many-html.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-many-html.xsl	(revision 21112)
@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="html" encoding="UTF-8"/>
+
+
+  <xsl:template match="Anchor">
+    <p/>
+    <img height="20" src="../gatherer_small.gif" width="20"/>
+    <a name="{@name}"><xsl:text> </xsl:text><xsl:apply-templates/></a>
+    <p style="text-align:justify"/>
+  </xsl:template>
+
+
+  <xsl:template match="Break">
+    <p/>
+  </xsl:template>
+
+
+  <xsl:template match="Contents">
+    <p/>
+    <table align="center" border="2" cellpadding="2" width="480" bgcolor="#B0D0B0">
+      <tr>
+        <td align="center">
+          <font face="Verdana" size="4">
+            <strong>
+              Contents
+            </strong>
+          </font>
+        </td>
+      </tr>
+
+      <tr>
+        <td>
+          <table bgcolor="#FFFFFF" border="0" cellpadding="2" width="100%">
+            <xsl:apply-templates/>
+          </table>
+        </td>
+      </tr>
+    </table>
+    <p/>
+  </xsl:template>
+
+
+  <xsl:template match="ContentsGroup">
+    <tr>
+      <td colspan="3">
+        <a href="#{@linkto}">
+	  <xsl:variable name="linkTo" select="@linkto"/>
+	  <xsl:value-of select="parent::Contents/parent::Section/Anchor[@name=$linkTo]"/>
+	</a>
+      </td>
+    </tr>
+
+    <xsl:apply-templates/>
+  </xsl:template>
+
+
+  <xsl:template match="ContentsItem">
+    <xsl:variable name="depth" select="count(ancestor::ContentsItem) + 1"/>
+    <tr>
+      <td colspan="{$depth}">
+        <xsl:text> </xsl:text>
+      </td>
+      <td colspan="{3 - $depth}">
+        <xsl:apply-templates/>
+      </td>
+    </tr>
+  </xsl:template>
+
+
+  <xsl:template match="Document">
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection">
+	<xsl:with-param name="sectionHead" select="position()"/>
+      </xsl:call-template>
+    </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template match="Reference">
+    <a href="{@target}.htm#{@target}"><xsl:apply-templates/></a>
+  </xsl:template>
+
+
+  <xsl:template match="Section"/>
+
+
+  <xsl:template match="Title"/>
+
+
+  <xsl:template name="processSection">
+    <xsl:param name="sectionHead"/>
+
+    <html>
+      <head>
+        <title>The Greenstone Librarian Interface - Help Pages</title>
+      </head>
+      <body bgcolor="#E0F0E0">
+        <table border="2" bgcolor="#B0D0B0" cellpadding="5" cellspacing="0" width="100%">
+          <tr>
+            <td align="center" width="15%">
+              <img height="45" src="../gatherer_medium.gif" width="45"/>
+            </td>
+            <td align="center" width="*">
+              <a name="{@name}">
+                <xsl:call-template name="processTitle">
+                  <xsl:with-param name="sectionNumber" select="$sectionHead"/>
+	          <xsl:with-param name="sectionTitle" select="Title"/>
+                </xsl:call-template>
+              </a>	    
+            </td>
+            <td align="center" width="15%">
+              <img height="45" src="../gatherer_medium.gif" width="45"/>
+            </td>
+          </tr>
+        </table>
+
+        <xsl:apply-templates/>
+      </body>
+    </html>
+
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection">
+        <xsl:with-param name="sectionHead" select="concat($sectionHead, '.', position())"/>
+      </xsl:call-template>
+    </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template name="processTitle">
+    <xsl:param name="sectionNumber"/>
+    <xsl:param name="sectionTitle"/>
+
+    <font face="Verdana" size="5">
+      <strong>
+        <xsl:value-of select="concat($sectionNumber, ': ', $sectionTitle)"/>
+      </strong>
+    </font>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/gen-one-html-for-doc.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-one-html-for-doc.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-one-html-for-doc.xsl	(revision 21112)
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="html" encoding="UTF-8"/>
+
+  <xsl:template match="Anchor"/>
+
+  <xsl:template match="Break">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Contents">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Document">
+    <html>
+      <head>
+        <title>The Greenstone Librarian Interface - Help Pages</title>
+      </head>
+      <body bgcolor="#E0F0E0">
+	<xsl:for-each select="Section">
+	  <!-- Skip the first section -->
+	  <xsl:if test="position() > 1">
+	    <xsl:call-template name="processSection">
+	      <xsl:with-param name="sectionHead" select="position()"/>
+	    </xsl:call-template>
+          </xsl:if>
+	</xsl:for-each>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template match="ObviousSentence"/>
+
+  <xsl:template match="Reference">
+    <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="Section"/>
+
+  <xsl:template match="Title"/>
+
+
+  <xsl:template name="processSection">
+    <xsl:param name="sectionHead"/>
+
+    <xsl:if test="Title">
+        <xsl:call-template name="processTitle">
+          <xsl:with-param name="sectionNumber" select="$sectionHead"/>
+	  <xsl:with-param name="sectionTitle" select="Title"/>
+        </xsl:call-template>
+    </xsl:if>
+
+    <xsl:apply-templates/>
+
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection">
+        <xsl:with-param name="sectionHead" select="concat($sectionHead, '.', position())"/>
+      </xsl:call-template>
+    </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template name="processTitle">
+    <xsl:param name="sectionNumber"/>
+    <xsl:param name="sectionTitle"/>
+
+    <xsl:choose>
+      <xsl:when test="contains($sectionNumber, '.')">
+        <h4><xsl:value-of select="$sectionTitle"/></h4>
+      </xsl:when>
+      <xsl:otherwise>
+        <h3><xsl:value-of select="$sectionTitle"/></h3>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/gen-one-html.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-one-html.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-one-html.xsl	(revision 21112)
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="html" encoding="UTF-8"/>
+
+  <xsl:template match="Anchor"/>
+
+  <xsl:template match="Break">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Contents">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Document">
+    <html>
+      <head>
+        <title>The Greenstone Librarian Interface - Help Pages</title>
+      </head>
+      <body bgcolor="#E0F0E0">
+	<xsl:for-each select="Section">
+	  <xsl:call-template name="processSection">
+	    <xsl:with-param name="sectionHead" select="position()"/>
+	  </xsl:call-template>
+	</xsl:for-each>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template match="ObviousSentence"/>
+
+  <xsl:template match="Reference">
+    <a href="#{@target}"><xsl:apply-templates/></a>
+  </xsl:template>
+
+  <xsl:template match="Section"/>
+
+  <xsl:template match="Title"/>
+
+
+  <xsl:template name="processSection">
+    <xsl:param name="sectionHead"/>
+
+    <xsl:if test="Title">
+      <a name="{@name}">
+        <xsl:call-template name="processTitle">
+          <xsl:with-param name="sectionNumber" select="$sectionHead"/>
+	  <xsl:with-param name="sectionTitle" select="Title"/>
+        </xsl:call-template>
+      </a>
+    </xsl:if>
+
+    <xsl:apply-templates/>
+
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection">
+        <xsl:with-param name="sectionHead" select="concat($sectionHead, '.', position())"/>
+      </xsl:call-template>
+    </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template name="processTitle">
+    <xsl:param name="sectionNumber"/>
+    <xsl:param name="sectionTitle"/>
+
+    <xsl:choose>
+      <xsl:when test="contains($sectionNumber, '.')">
+        <h3>Section <xsl:value-of select="concat($sectionNumber, ': ', $sectionTitle)"/></h3>
+      </xsl:when>
+      <xsl:otherwise>
+        <h1>Section <xsl:value-of select="concat($sectionNumber, ': ', $sectionTitle)"/></h1>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/gen-one-tex.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-one-tex.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-one-tex.xsl	(revision 21112)
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="text" encoding="iso-8859-1"/>
+
+  <xsl:template match="Anchor"/>
+
+  <xsl:template match="Break">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Contents">
+    <p/>
+  </xsl:template>
+
+  <xsl:template match="Document-old">
+    <html>
+      <head>
+        <title>The Greenstone Librarian Interface - Help Pages</title>
+      </head>
+      <body bgcolor="#E0F0E0">
+	<xsl:for-each select="Section">
+	  <!-- Skip the first section -->
+	  <xsl:if test="position() > 1">
+	    <xsl:call-template name="processSection">
+	      <xsl:with-param name="sectionHead" select="position()"/>
+	    </xsl:call-template>
+          </xsl:if>
+	</xsl:for-each>
+      </body>
+    </html>
+  </xsl:template>
+  <xsl:template match="Document">
+    % START OF AUTO-GENERATED GLI USER GUIDE
+    <xsl:for-each select="Section">
+      <!-- Skip the first section -->
+      <xsl:if test="position() > 1">
+	<xsl:call-template name="processSection">
+	  <xsl:with-param name="sectionHead" select="position()"/>
+	</xsl:call-template>
+      </xsl:if>
+    </xsl:for-each>
+  </xsl:template>
+  <xsl:template match="ObviousSentence"/>
+
+  <xsl:template match="Reference">
+    <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="Section"/>
+
+  <xsl:template match="Title"/>
+
+
+  <xsl:template name="processSection">
+    <xsl:param name="sectionHead"/>
+
+    <xsl:if test="Title">
+        <xsl:call-template name="processTitle">
+          <xsl:with-param name="sectionNumber" select="$sectionHead"/>
+	  <xsl:with-param name="sectionTitle" select="Title"/>
+        </xsl:call-template>
+    </xsl:if>
+
+    <xsl:apply-templates/>
+
+    <xsl:for-each select="Section">
+      <xsl:call-template name="processSection">
+        <xsl:with-param name="sectionHead" select="concat($sectionHead, '.', position())"/>
+      </xsl:call-template>
+    </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template name="processTitle">
+    <xsl:param name="sectionNumber"/>
+    <xsl:param name="sectionTitle"/>
+
+    <xsl:choose>
+      <xsl:when test="contains($sectionNumber, '.')">
+        <!--<h4><xsl:value-of select="$sectionTitle"/></h4>-->
+\paragraph{\emph{<xsl:value-of select="$sectionTitle"/>}}<xsl:text>
+	</xsl:text>
+      </xsl:when>
+      <xsl:otherwise>
+        <!--<h3><xsl:value-of select="$sectionTitle"/></h3>-->
+\subsubsection{<xsl:value-of select="$sectionTitle"/>}<xsl:text>
+	</xsl:text>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/gen-text.xsl
===================================================================
--- /main/tags/3.01/gli/help/gen-text.xsl	(revision 21112)
+++ /main/tags/3.01/gli/help/gen-text.xsl	(revision 21112)
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:output method="text" encoding="UTF-8"/>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gli/help/ru/addingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/addingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/addingfiles.htm	(revision 21112)
@@ -0,0 +1,90 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingfiles"><font size="5" face="Verdana"><strong>4.3: ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÐŸ,ÐºÐ°Ðº Ð²Ð²ÐµÑÑÐž ÑÐ°Ð¹Ð»Ñ Ð² ÐÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐµÑÐµÐŒÐµÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÐµ(ÑÐŸÑÑÐ°Ð²ÐœÑÐµ) ÑÐ°Ð¹Ð»Ñ
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð¡ÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð€Ð°Ð¹Ð»Ñ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŒÑÑÑÑ Ð¿ÑÑÐµÐŒ ÐžÑ
+ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ Ðž ÐŸÑÐ¿ÑÑÐºÐ°ÐœÐžÑ. Ð£ÐºÐ°Ð·Ð°ÑÐµÐ»Ñ ÐŒÑÑÐž ÐŸÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑÑÑ ÐœÐ°ÐŽ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐŒ ÐžÐ»Ðž, ÐµÑÐ»Ðž Ð²ÑÐ±ÐžÑÐ°ÐµÑÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐžÐœ,ÐœÐ°ÐŽ ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÐŒÐž. ÐÑÐŸÐ¿ÑÑÑÐžÑÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» Ð² ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð·ÐŽÐµÑÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ (ÐµÑÐ»Ðž ÐžÑÑÐŸÑÐœÐžÐºÐŸÐŒ Ð±ÑÐ»ÐŸ ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž).
+<p></p>
+  ÐÑÐž ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÑ
+(ÑÐŸÑÑÐ°Ð²ÐœÑÑ
+) ÑÐ°Ð¹Ð»ÐŸÐ²,Ð²ÑÐµ ÐŸÐœÐž Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑÑÑ Ð² ÑÐµÐ»ÐµÐ²ÑÑ Ð¿Ð°Ð¿ÐºÑ ÐœÐ° ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ ÑÑÐŸÐ²ÐµÐœÑ,ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÐŸ ÐŸÑ ÑÑÑÑÐºÑÑÑÑ Ð¿Ð°Ð¿ÐºÐž,ÐºÐŸÑÐŸÑÑÑ ÐŸÐœÐž Ð·Ð°ÐœÐžÐŒÐ°Ð»Ðž Ð¿ÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐºÐŸÐ¿ÐžÑÑÐµÑÐµ Ð²ÑÐŸÑÐŸÐ¹ ÑÐ°Ð¹Ð» Ñ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² ÑÑ Ð¶Ðµ ÑÐ°ÐŒÑÑ Ð¿Ð°Ð¿ÐºÑ, ÐÐ°Ñ Ð·Ð°Ð¿ÑÐŸÑÑÑ, Ð·Ð°ÑÐµÑÐµÑÑ Ð»Ðž Ð¿ÐµÑÐ²ÑÐ¹. ÐÑÐ²ÐµÑÑÑÐµ "ÐÐµÑ" Ðž ÑÑÐŸÑ ÑÐ°Ð¹Ð» ÐœÐµ Ð±ÑÐŽÐµÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑÑÑ,ÐœÐŸ ÐŽÑÑÐ³ÐžÐµ Ð±ÑÐŽÑÑ. Ð§ÑÐŸÐ±Ñ ÐŸÑÐŒÐµÐœÐžÑÑ Ð²ÑÐµ ÐŸÑÑÐ°ÑÑÐžÐµÑÑ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ Ð¿ÐŸ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "Ð¡ÑÐŸÐ¿".   <p></p>
+ Ð¢ÐŸÐ»ÑÐºÐŸ "ÑÐ°ÐŒÑÐµ Ð²ÑÑÐŸÐºÐžÐµ"ÐžÐ· ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² Ð±ÑÐŽÑÑ Ð¿ÐµÑÐµÐŒÐµÑÐµÐœÑ. ÐÐ°Ð¿ÐºÐ° Ð²ÑÑÐµ, ÑÐµÐŒ ÐµÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐœÑÐµ. ÐÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ ÑÐ°Ð¹Ð»Ñ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð¿Ð°Ð¿ÐºÐž,Ð° ÑÐ°ÐºÐ¶Ðµ ÑÐ°ÐŒÑ Ð¿Ð°Ð¿ÐºÑ ÐœÐµÐ¿ÐŸÑÑÐµÐŽÑÑÐ²ÐµÐœÐœÐŸ. 
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÑÐ°Ð¹Ð», ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑ Ð¿ÐŸÐžÑÐº Ð² ÐžÑÑ
+ÐŸÐŽÐœÑÑ
+ Ð¿Ð°Ð¿ÐºÐ°Ñ
+ Ð²ÑÐ¿ÐŸÐŒÐŸÐ³Ð°ÑÐµÐ»ÑÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÐŒÐŸÐŒÑ ÑÐ°Ð¹Ð»Ñ,Ðž ÐµÑÐ»Ðž ÐŸÐœ ÐœÐ°Ñ
+ÐŸÐŽÐžÑ ÐµÐ³ÐŸ,ÑÐŸ ÐœÐ°ÑÐžÐœÐ°ÐµÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÐŽÐ°ÐœÐœÑÐµ. ÐÑÐ»Ðž Ð¿ÑÐŸÑÐµÑÑ Ð¿ÑÐŸÐŽÐŸÐ»Ð¶Ð°ÐµÑÑÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð±ÑÑÑ Ð·Ð°Ð¿ÑÐŸÑÐµÐœÑ (Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ°Ð·) ÐŸ ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž,ÑÑÐŸÐ±Ñ ÐŽÐŸÑÑÐžÑÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÐµÐŒÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž,ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐŒÐž Ð² ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð­ÑÐŸÑ Ð¿ÑÐŸÑÐµÑÑ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐŒÐœÐŸÐ³ÐžÐµ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ Ð·Ð°Ð¿ÑÐŸÑÑ, ÐŸÐ¿ÐžÑÐ°ÐœÐœÑÐµ Ð² ÑÐ°Ð·ÐŽÐµÐ»Ðµ<a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</a>.ÐÐ»Ñ Ð±ÐŸÐ»ÐµÐµ ÐŽÐµÑÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÐŸÐ±ÑÑÑÐœÐµÐœÐžÑ Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÑ
+ Ñ ÑÐ°Ð¹Ð»Ð°ÐŒÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐžÑÐ°Ð¹ÑÐµ ÐÐ»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone - ÐÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð±ÐŸÐ»ÑÑÐžÐœÑÑÐ²Ð° Ð²Ð°ÑÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ².  
+     </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/addingpreviouslydefinedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/addingpreviouslydefinedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/addingpreviouslydefinedmetadata.htm	(revision 21112)
@@ -0,0 +1,60 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="addingpreviouslydefinedmetadata"><font size="5" face="Verdana"><strong>5.4: ÐÐŸÐ±Ð°Ð²Ð»ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,ÑÐ¶Ðµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÑÐµ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ Ðº Ð¿Ð°Ð¿ÐºÐ°ÐŒ ÐžÐ»Ðž Ðº ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð», Ð·Ð°ÑÐµÐŒ ÑÑÐµÐ±ÑÐµÐŒÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐŽÐµÑÐµÐ²Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÑÐ°ÑÑÐžÑÑÑ Ð¿Ð°Ð¿ÐºÐž ÐžÐµÑÐ°ÑÑ
+ÐžÐž Ð¿ÐŸ ÐŒÐµÑÐµ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž. ÐÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² Ð¿ÐŸÐ»Ðµ ÑÐµÐºÑÑÐ° (Ð°Ð»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÐŸ ÐŒÐŸÐ¶ÐœÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð°Ð²ÑÐŸÐ²ÑÐ±ÐŸÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ Ðž ÑÑÐµÐŽÑÑÐ² Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐŸÐ³ÐŸ Ð·Ð°Ð²ÐµÑÑÐµÐœÐžÑ). Ð©ÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐŒÑ ÑÐ°Ð¹Ð»Ñ.
+<p></p>
+ÐÑÐŸÑÐµÑÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÑÐ¶Ðµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒÐž Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒÐž Ðº Ð¿Ð°Ð¿ÐºÐ°ÐŒ ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐ°ÐºÐŸÐ¹ Ð¶Ðµ.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/appendingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/appendingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/appendingmetadata.htm	(revision 21112)
@@ -0,0 +1,94 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="appendingmetadata"><font size="5" face="Verdana"><strong>5.3: ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðž ÐžÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ðº ÑÐ°Ð¹Ð»Ð°ÐŒ Ðž ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐ»Ñ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŸÐ¿ÐŸÐ»ÐœÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ¿ÐŸÐ»ÐœÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÑÐŸÑÑÐ°Ð²ÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐœÐ°ÑÐµÐœÐžÐµ ÐžÐµÑÐ°ÑÑ
+ÐžÐ¹ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð¢ÐµÐ¿ÐµÑÑ ÐŒÑ ÐŽÐŸÐ±Ð°Ð²ÐžÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ - Ðž ÑÐ°ÐŒ ÑÐ»ÐµÐŒÐµÐœÑ,Ðž ÐµÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ - Ðº ÑÐ°Ð¹Ð»Ñ. Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐµÐŒ ÑÐ°Ð¹Ð» ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð¢Ð°ÐºÐŸÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ Ð²ÑÐ·ÑÐ²Ð°ÐµÑ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÐµ Ð»ÑÐ±ÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÑÐ²ÑÐ·Ð°ÐœÐœÑÑ
+ Ñ ÑÑÐžÐŒ ÑÐ°Ð¹Ð»ÐŸÐŒ,Ð² ÑÐ°Ð±Ð»ÐžÑÐµ ÑÐ¿ÑÐ°Ð²Ð°. 
+<p></p>
+ÐÐ°ÑÐµÐŒ Ð²ÑÐ±ÐµÑÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐ¹ Ñ
+ÐŸÑÐžÐŒ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ,Ð¿ÑÑÐµÐŒ ÑÐµÐ»ÑÐºÐ° ÑÑÐŽÐŸÐŒ Ñ ÐœÐžÐŒ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ. 
+ <p></p>
+ ÐÐ¿ÐžÑÐžÑÐµ ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð² Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. ÐÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð·ÐœÐ°Ðº "|",Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ ÐŸÐœ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐµÑÐ°ÑÑ
+ÐžÐ¹. ÐÐŸ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÐž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.Ð­ÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.  
+<p></p>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐ°ÐºÐ¶Ðµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº Ð¿Ð°Ð¿ÐºÐµ ÐžÐ»Ðž ÑÑÐ°Ð·Ñ Ðº ÐœÐµÐºÐŸÑÐŸÑÐŸÐŒÑ ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ². Ð­ÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ ÑÐ°Ð¹Ð»Ð°ÐŒ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð¿Ð°Ð¿ÐºÐž ÐžÐ»Ðž Ðº ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ Ðž Ðº Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐœÑÐŒ Ð¿Ð°Ð¿ÐºÐ°ÐŒ. ÐÐŒÐµÐ¹ÑÐµ Ð² Ð²ÐžÐŽÑ,ÑÑÐŸ ÐµÑÐ»Ðž ÐÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÐ»Ðž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº Ð¿Ð°Ð¿ÐºÐµ, ÑÐŸ Ð»ÑÐ±ÑÐµ ÐœÐŸÐ²ÑÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÐœÐµÐ¹ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÑÐœÐ°ÑÐ»ÐµÐŽÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐžÐ· ÑÑÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž.  
+ <p></p>
+ ÐÑÐ»Ðž ÐÑ Ð²ÑÐ±ÐµÑÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐµÑÑÑÑ Ð² ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÐžÐ· ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÐŸ ÐŸÐœÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑÑÑÑ ÑÐ°ÐºÐ¶Ðµ Ðº ÐŽÑÑÐ³ÐžÐŒ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ ÑÐ°Ð¹Ð»Ð°ÐŒ.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ Ðº Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð¿ÑÑÑ, ÐŸÐ¿ÐžÑÐ°ÐœÐœÑÐ¹ Ð² <a href="theenrichview.htm#theenrichview">Ð Ð°ÑÑÐžÑÐµÐœÐœÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/assignlanguages.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/assignlanguages.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/assignlanguages.htm	(revision 21112)
@@ -0,0 +1,77 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignlanguages"><font size="5" face="Verdana"><strong>6.6.3: ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŽÐµÑÐ°Ð»ÐžÐ·ÐžÑÐŸÐ²Ð°ÐœÐŸ,ÐºÐ°Ðº ÐŸÐ³ÑÐ°ÐœÐžÑÐžÑÑ ÐžÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ° ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐŒÐž ÑÐ·ÑÐºÐ°ÐŒÐž. ÐÑ ÑÐŸÐ²ÐµÑÑÐ°ÐµÑÐµ ÑÑÐŸ Ð¿ÑÐž ÑÐµÐ³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž" Ð² Ð¿ÐµÑÐµÑÐœÐµ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ".
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐ·ÑÐºÐ°ÐŒÐž
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ·ÑÐº
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ·ÑÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÑÐžÑÑÐºÐ° ÑÐ·ÑÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐ·ÑÐº ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž" ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐ° ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ. ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÑÐ¹ ÑÐ·ÑÐº ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ° "Ð¯Ð·ÑÐºÐž ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ·ÑÐº"
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÑÐ±ÐŸÑÑ ÑÐ·ÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÑÐ·ÑÐº Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÐžÑÑÐžÑÑ ÑÐ·ÑÐº Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/assignpartitions.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/assignpartitions.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/assignpartitions.htm	(revision 21112)
@@ -0,0 +1,71 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="assignpartitions"><font size="5" face="Verdana"><strong>6.6.2: ÐÐ°Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐœÐ°Ð·ÐœÐ°ÑÐžÑÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐ¹ ÑÐžÐ»ÑÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ.   
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÑÐžÑÑÐºÐ° ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐŒÐµÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÐ¹ ÑÐžÐ»ÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°Ð·ÐœÐ°ÑÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ" ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐŸÐ² ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐžÐ»Ðž ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿Ñ ÑÐžÐ»ÑÑÑÐŸÐ². ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÑÐ¹ ÑÐžÐ»ÑÑÑ (ÐžÐ»Ðž ÑÐžÐ»ÑÑÑÑ) ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐžÐ»ÑÑÑÑ ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž", Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ".
+<p></p>
+  Ð§ÑÐŸÐ±Ñ Ð¿ÑÐŸÐžÐ·Ð²ÐµÑÑÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐŸÑÐžÑÑÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/buildsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/buildsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/buildsettings.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="buildsettings"><font size="5" face="Verdana"><strong>7.2: ÐÐŒÐ¿ÐŸÑÑ Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑÐŒ ÐžÐŒÐ¿ÐŸÑÑÐ° Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ. ÐÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹ Ð¿ÐŸ ÐžÐŒÐ¿ÐŸÑÑÑ Ðž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÑÐžÑÐ°Ð¹ÑÐµ Ð³Ð»Ð°Ð²Ñ 1 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.  
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐžÐŒÐ¿ÐŸÑÑÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒÐž ÑÐŽÐµÐ»Ð°ÐœÐŸ Ð¿ÐŸÐŽÐŸÐ±ÐœÐŸ ÑÐŸÐŒÑ ÑÐ¿ÐŸÑÐŸÐ±Ñ, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ¿ÐžÑÐ°Ðœ Ð¿ÑÐž "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²" Ð² <a href="plugins.htm#plugins">ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</a>. ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ»Ñ ÑÑÐµÐ±ÑÑÑ ÑÐžÑÐ»ÐµÐœÐœÑÑ
+ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ². ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑ
+ ÐžÐ»Ðž ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°ÑÑ ÐžÐ»Ðž ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÑÑÑÐµÐ»ÐºÐž Ð²Ð²ÐµÑÑ
+ Ðž Ð²ÐœÐžÐ·, ÑÑÐŸÐ±Ñ ÑÐ²ÐµÐ»ÐžÑÐžÑÑ ÐžÐ»Ðž ÑÐŒÐµÐœÑÑÐžÑÑ ÑÐµÐºÑÑÐµÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ (Ð² ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÑÐ»ÑÑÐ°ÑÑ
+ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŸÐ³ÑÐ°ÐœÐžÑÐžÐ²Ð°ÐµÑ ÐŽÐžÐ°Ð¿Ð°Ð·ÐŸÐœ, Ð² ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐŸÐ¹ÑÐž). ÐÑÑÐ³ÐžÐµ Ð¿ÐŸÐ»Ñ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž Ð¿ÑÐž ÑÐµÐ»ÑÐºÐµ Ð¿ÐŸ ÐºÐœÐŸÐ¿ÐºÐµ (ÐÐŸÐ²ÑÐŸÑÐœÑÐ¹ ÑÐµÐ»ÑÐŸÐº ÐŽÐµÐ»Ð°ÐµÑ ÐžÑ
+ ÐœÐµÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž). 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/classifiers.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/classifiers.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/classifiers.htm	(revision 21112)
@@ -0,0 +1,82 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="classifiers"><font size="5" face="Verdana"><strong>6.8: ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ°. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐŸÑÐŒÐŸÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ²". 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐ±ÐŸÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Ðž ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÐ»ÐµÐŽÐŸÐ²Ð°ÑÐµÐ»ÑÐœÐŸÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ²
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "ÐÑÐ±ÐŸÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ð²ÐœÐžÐ·Ñ ÐŸÐºÐœÐ°,Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐºÐ°Ð·Ð°ÐœÐœÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐµ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²".ÐÐœÑÑÑÑÐºÑÐžÐž ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° ÑÐµ Ð¶Ðµ ÑÐ°ÐŒÑÐµ,ÑÑÐŸ ÐºÐ°ÑÐ°ÑÑÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² (<a href="plugins.htm#plugins">ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</a>).ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÑÑÐŸÑÐŒÐžÑÐŸÐ²Ð°Ð»Ðž ÐœÐŸÐ²ÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ, ÐŸÐœ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° "Ð¢ÐµÐºÑÑÐžÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ".  
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°" (ÐžÐ»Ðž ÑÐµÐ»ÐºÐœÑÑÑ ÐŽÐ²Ð° ÑÐ°Ð·Ð° ÐœÐ° ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐµ Ð² ÑÐ¿ÐžÑÐºÐµ).
+<p></p>
+ ÐÐŸÑÑÐŽÐŸÐº ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² Ð² ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐŸÐœÐœÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐž ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ ÑÐŸÐŒÑ,ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐºÐ°Ð·Ð°Ðœ Ð·ÐŽÐµÑÑ.Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ, ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ, Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·".
+<p></p>
+ ÐÐ»Ñ ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐµÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² ÑÐžÑÐ°Ð¹ÑÐµ Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/collectingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/collectingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/collectingfiles.htm	(revision 21112)
@@ -0,0 +1,23 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="collectingfiles"><font size="5" face="Verdana"><strong>4: Ð¡Ð±ÐŸÑ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŽÐ»Ñ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ ÐÑÐ»Ðž ÐÑ ÑÐŸÐ·ÐŽÐ°ÐµÑÐµ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ,ÑÐŸ Ð²ÐŸÐ·ÐœÐžÐºÐ°ÐµÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÑ Ð²Ð²ÐŸÐŽÐ° Ð² ÐœÐµÐµ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÐœÐž ÐŒÐŸÐ³ÑÑ Ð¿ÐŸÑÑÑÐ¿ÐžÑÑ ÐžÐ· Ð²Ð°ÑÐµÐ³ÐŸ ÐŸÐ±ÑÑÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° ÐžÐ· ÐŽÑÑÐ³ÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Greenstone. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ ÑÐ¶Ðµ ÐžÐŒÐµÑÑ Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐµÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ.Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ.
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/creatingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/creatingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/creatingacollection.htm	(revision 21112)
@@ -0,0 +1,111 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingacollection"><font size="5" face="Verdana"><strong>2.1: Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+       
+       
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÑÑÑÑÐŸÐµ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐ°Ð³Ð»Ð°Ð²ÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐŒÐµÐœÐ° ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÑÑÑÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐŒÐµÐœÑ "File" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ Ð² ÐœÑÐŒ Ð¿ÑÐœÐºÑ "New". ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ»Ñ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÑÐ·Ð°ÑÐµÐ»ÑÐœÐŸ Ð·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÑ, ÐœÐŸ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°ÐŒÐµÐœÐžÑÑ ÐžÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ð¿ÐŸÐ·Ð¶Ðµ, ÐµÑÐ»Ðž ÐÑ Ð² ÑÑÐŸÐŒ ÐœÑÐ¶ÐŽÐ°ÐµÑÐµÑÑ Ñ ÑÐŸÑÐºÐž Ð·ÑÐµÐœÐžÑ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°.
+ <p></p>
+ÐÐ°Ð³Ð»Ð°Ð²ÐžÐµ "ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ/Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž" - ÑÑÐŸ ÑÐµÐºÑÑ, ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÑÐ¹ ÐœÐ°Ð²ÐµÑÑ
+Ñ ÐŽÐŸÐŒÐ°ÑÐœÐµÐ¹ ÑÑÑÐ°ÐœÐžÑÑ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐœÐŸ ÐŒÐŸÐ¶ÐµÑ ÐžÐŒÐµÑÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐ»ÑÐœÑÑ ÐŽÐ»ÐžÐœÑ.
+<p></p>
+"ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ" ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²ÐžÑÑ ÐºÐ°Ðº ÐŒÐŸÐ¶ÐœÐŸ ÐŽÐµÑÐ°Ð»ÑÐœÐµÐµ ÑÐŸ,ÐŸ ÑÑÐŒ ÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ [Enter] ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ,ÑÑÐŸÐ±Ñ ÑÐ°Ð·Ð±ÐžÑÑ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ ÐœÐ° Ð¿Ð°ÑÐ°Ð³ÑÐ°ÑÑ.
+<p></p>
+ ÐÐ°ÐºÐŸÐœÐµÑ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ, ÐŽÐŸÐ»Ð¶ÐœÐ° Ð»Ðž ÐœÐŸÐ²Ð°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐŒÐµÑÑ ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ Ð²ÐžÐŽ Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸ Ðž ÑÑÑÐµÑÑÐ²ÑÑÑÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐžÐ»Ðž Ð¶Ðµ ÐœÐ°ÑÐ°ÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ "ÐÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ".
+<p></p>
+Ð©ÐµÐ»ÐºÐœÐžÑÐµ "OK", ÑÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. ÐÑÐ»Ðž ÐÑ Ð²ÑÐ±ÑÐ°Ð»Ðž "New Collection", ÑÐŸ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð²Ð²ÐµÑÑÐž ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐžÑ
+ Ð² ÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐœÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, Ð° ÐŒÐŸÐ¶ÐµÑÐµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŽÑÑÐ³ÐžÐµ Ð¿ÐŸÐ·Ð¶Ðµ. 
+<p></p>
+ÐÐ°Ð¶Ð°ÑÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž "Cancel" ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð²ÐŸÐ·Ð²ÑÐ°ÑÐ°ÐµÑ ÐÐ°Ñ Ðº Ð³Ð»Ð°Ð²ÐœÐŸÐŒÑ ÑÐºÑÐ°ÐœÑ. 
+    <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐœÐŸÐ¿ÐºÐž, ÑÐžÐ¿Ð° ÐŒÐµÐœÑ, ÐžÐŒÐµÑÑ ÐŸÐŽÐžÐœ Ð¿ÐŸÐŽÑÑÑÐºÐœÑÑÑÐ¹ ÑÐžÐŒÐ²ÐŸÐ». Ð§ÑÐŸÐ±Ñ Ð²ÑÐ·Ð²Ð°ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐœÐ°Ð¶ÐŒÐžÑÐµ [ALT] Ðž ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ Ð¿ÐŸÐŽÑÐµÑÐºÐœÑÑÑÐ¹ ÑÐžÐŒÐ²ÐŸÐ». 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/creatingfolders.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/creatingfolders.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/creatingfolders.htm	(revision 21112)
@@ -0,0 +1,61 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="creatingfolders"><font size="5" face="Verdana"><strong>4.2: Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÑÑ ÐœÐŸÐ²ÑÐµ Ð¿Ð°Ð¿ÐºÐž
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐœÐŸÐ¿ÐºÐ° ÐœÐŸÐ²ÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐµÐœÑ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐž ÐŒÑÑÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ "ÐÐ°Ð¿ÐºÐž" Ð² ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐŒ ÐŽÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð²ÐŒÐµÑÑÐµ Ðž Ð±ÐŸÐ»ÐµÐµ Ð»ÐµÐ³ÐºÐŸ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑ ÐžÑ
+. ÐÐ°Ð¿ÐºÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð¿ÐŸÐŒÐµÑÐµÐœÑ Ð²ÐœÑÑÑÐž Ð¿Ð°Ð¿ÐŸÐº. Ð€Ð°ÐºÑÐžÑÐµÑÐºÐž ÐœÐµÑ ÐŸÐ³ÑÐ°ÐœÐžÑÐµÐœÐžÑ ÐœÐ° ÑÐŸ,ÑÐºÐŸÐ»ÑÐºÐŸ Ð¿Ð°Ð¿ÐŸÐº ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐŒÐµÑÑ ÐžÐ»Ðž ÐºÐ°Ðº Ð³Ð»ÑÐ±ÐŸÐºÐŸ ÐŸÐœÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð²Ð»ÐŸÐ¶ÐµÐœÑ.
+<p></p>
+  Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ Ð¿Ð°Ð¿ÐºÑ, Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐ»ÑÐœÐŸ Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÑÑ Ð¿Ð°Ð¿ÐºÑ Ð² ÐÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°". ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÐžÐ»Ðž ÐœÐ° Ð²ÑÑÑÐµÐŒ ÑÑÐŸÐ²ÐœÐµ,ÐµÑÐ»Ðž ÐœÐž ÐŸÐŽÐœÐ° ÐœÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐ°.ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÑÐŸÑÐ²ÐŸÐžÑÑ Ð¿Ð°Ð¿ÐºÐµ ÐžÐŒÑ (Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°").
+<p></p>
+ ÐÐ°Ð¿ÐºÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐ°ÐºÐ¶Ðµ ÑÐŸÐ·ÐŽÐ°ÐœÑ ÑÐµÐ»ÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐž ÐŒÑÑÐž Ð¿ÐŸ Ð¿Ð°Ð¿ÐºÐµ,Ð²ÑÐ±ÐžÑÐ°Ñ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°" Ðž ÐŽÐµÐ¹ÑÑÐ²ÑÑ,ÐºÐ°Ðº ÑÐºÐ°Ð·Ð·Ð°ÐœÐŸ Ð²ÑÑÐµ.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/definefilters.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/definefilters.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/definefilters.htm	(revision 21112)
@@ -0,0 +1,69 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="definefilters"><font size="5" face="Verdana"><strong>6.6.1: ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+        
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ.
+        <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+          
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </td>
+</tr>
+            
+<tr>
+<td colspan="1"> </td><td colspan="2">
+               ÐÐ·ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </td>
+</tr>
+          
+        
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+        
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð€ÐžÐ»ÑÑÑÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð² Ð¿ÐŸÐŽÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð²ÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ð¿ÐŸ  ÐžÐœÐŽÐµÐºÑÑ,ÐŽÐ»Ñ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐ¹ ÐŒÐŸÐŽÐµÐ»Ðž.
+ <p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÑÐžÐ»ÑÑÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ" Ðž Ð²Ð²ÐµÐŽÐžÑÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŽÐ»Ñ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐžÐ»ÑÑÑÐ° Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÑÐžÐ»ÑÑÑÐ° ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž".ÐÐ°ÑÐµÐŒ Ð²ÑÐ±ÐµÑÐžÑÐµ Ñ
+Ð°ÑÐ°ÐºÑÐµÑÐœÑÐµ Ð¿ÑÐžÐ·ÐœÐ°ÐºÐž ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐµ ÐžÐ»Ðž ÑÐ»ÐµÐŒÐµÐœÑÐ°ÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐžÐ»Ðž ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°ÐµÐŒÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.ÐÐ²ÐµÐŽÐžÑÐµ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ,ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð² ÑÐµÑÐµÐœÐžÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÑÑÐŸÐ³ÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ.ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐºÐ»ÑÑÐ°ÑÑ "ÐÐºÐ»ÑÑÐµÐœÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²", ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑ ÑÐžÐ»ÑÑÑÑ, ÐžÐ»Ðž "ÐÑÐºÐ»ÑÑÐµÐœÐžÐµ" ÐžÑ
+.ÐÐ°ÐºÐŸÐœÐµÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð»ÑÐ±ÐŸÐ¹ ÑÑÐ°ÐœÐŽÐ°ÑÑ PERL, Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÑÐµ ÑÐ»Ð°Ð³Ðž Ð²ÑÑÐ°Ð¶ÐµÐœÐžÑ,ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð¿ÑÐž ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ("i" ÐŽÐ»Ñ ÑÐ»ÑÑÐ°Ñ ÐœÐµÑÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐŸÐ³ÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ),ÐÐ°ÐºÐŸÐœÐµÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ Ðº ÑÐ¿ÐžÑÐºÑ "Ð€ÐžÐ»ÑÑÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹".
+ <p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ". 
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ°,Ð·Ð°ÐŒÐµÐœÐžÑÐµ Ð»ÑÐ±ÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ Ð·Ð°ÐºÑÐµÐ¿ÐžÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ.
+      </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/deletingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/deletingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/deletingcollections.htm	(revision 21112)
@@ -0,0 +1,48 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="deletingcollections"><font size="5" face="Verdana"><strong>2.4: Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÐœÐµÐžÐ·ÐŒÐµÐœÐœÐŸ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐ· ÐžÐœÑÑÐ°Ð»ÑÑÐžÐŸÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž Greenstone.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐ°Ðº ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+      ÐÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ ÐœÐµÐžÐ·ÐŒÐµÐœÐœÐŸ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐ· ÐžÐœÑÑÐ°Ð»ÑÑÐžÐŸÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž Greenstone, Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ..." ÐžÐ· ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»". ÐÑÐºÑÐŸÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº Ð²ÑÐµÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Greenstone. ÐÑÐ±ÐµÑÐžÑÐµ ÐœÑÐ¶ÐœÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÐ° ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ, Ð¿ÐŸÑÐ»Ðµ ÑÑÐŸÐ³ÐŸ Ð¿ÐŸÑÑÐ°Ð²ÑÐµ Ð³Ð°Ð»ÐŸÑÐºÑ Ð² Ð¿ÑÐœÐºÑÐµ Ð²ÐœÐžÐ·Ñ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ", ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. Ð­ÑÐ° ÐŸÐ¿ÐµÑÐ°ÑÐžÑ Ð±ÐµÐ·Ð²ÐŸÐ·Ð²ÑÐ°ÑÐœÐ°, Ð¿ÐŸÑÑÐŸÐŒÑ Ð¿ÑÐŸÐ²ÐµÑÑÑÐµ Ð²ÐœÐžÐŒÐ°ÑÐµÐ»ÑÐœÐŸ, ÐŽÐµÐ¹ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐŸ Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµÐœÑÐ¶ÐœÐ°!
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/designingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/designingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/designingacollection.htm	(revision 21112)
@@ -0,0 +1,26 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="designingacollection"><font size="5" face="Verdana"><strong>6: ÐÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ ÐÐŸÐ³ÐŽÐ° ÐÐ°ÑÐž ÑÐ°Ð¹Ð»Ñ ÑÐœÐ°Ð±Ð¶ÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž,ÐŽÐ°Ð»ÐµÐµ ÐÑ ÑÐµÑÐ°ÐµÑÐµ,Ð² ÐºÐ°ÐºÐŸÐŒ Ð²ÐžÐŽÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Greenstone ÐŽÐŸÐ»Ð¶ÐœÐ° Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐ° Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»ÑÐŒ.ÐÐ°ÐºÐŸÐ³ÐŸ Ð²ÐžÐŽÐ° ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŽÐŸÑÑÑÐ¿ÐœÐ° ÐŽÐ»Ñ Ð¿ÐŸÐžÑÐºÐ°? ÐÐ°ÐºÐžÐµ Ð¿ÑÑÐž Ð¿ÑÐµÐŽÑÑÐŒÐŸÑÑÐµÐœÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°? ÐÐ°ÐºÐžÐµ ÑÐ·ÑÐºÐž Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑÑÑ? ÐÐŽÐµ ÑÐ°Ð·ÐŒÐµÑÑÐžÑÑ ÐºÐœÐŸÐ¿ÐºÐž ÐœÐ° ÑÑÑÐ°ÐœÐžÑÐµ? Ð­ÑÐž Ð²ÐµÑÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐœÐ°ÑÑÑÐŸÐµÐœÑ;Ð² ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ.
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/downloadingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/downloadingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/downloadingfiles.htm	(revision 21112)
@@ -0,0 +1,16 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="downloadingfiles"><font size="5" face="Verdana"><strong>3: ÐÐ°Ð³ÑÑÐ·ÐžÑÑ ÑÐ°Ð¹Ð»Ñ ÐžÐ· ÐžÐœÑÐµÑÐœÐµÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    ÐÐžÐŽ "ÐÐµÑÐºÐ°Ð»ÐŸ" Ð¿ÐŸÐŒÐŸÐ¶ÐµÑ Ð²Ð°ÐŒ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐžÐ· ÐÐœÑÐµÑÐœÐµÑÐ°. ÐÐ°ÐœÐœÐ°Ñ ÑÐµÐºÑÐžÑ ÐŸÐ±ÑÑÑÐœÑÐµÑ Ð¿ÑÐŸÑÐµÑÑ Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð² ÐÐœÑÐµÑÑÐµÐ¹ÑÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ.
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/enrichingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/enrichingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/enrichingacollection.htm	(revision 21112)
@@ -0,0 +1,28 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="enrichingacollection"><font size="5" face="Verdana"><strong>5: ÐÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ ÐÐŒÐµÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐœÐ°Ð¿ÐŸÐ»ÐœÐžÑÐµ ÐžÑ
+ ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹,ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐ¹ "ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ".Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÑÑÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,ÐºÐ°Ðº ÐŸÐœÐž ÑÐµÐŽÐ°ÐºÑÐžÑÑÑÑÑÑ, ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑÑÑ Ðž Ð²ÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÑÑ,Ð° ÑÐ°ÐºÐ¶Ðµ ÐºÐ°Ðº ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ Ð²ÐœÐµÑÐœÐžÐµ ÐžÑÑÐŸÑÐœÐžÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+    
+ (ÑÐ°ÐºÐ¶Ðµ ÑÐŒ.ÐÐ»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone - ÐÐŸÐ»ÑÑÐµÐœÐžÐµ Ð±ÐŸÐ»ÑÑÐµÐ¹ ÑÐ°ÑÑÐž ÐžÐ· Ð²Ð°ÑÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²). 
+    
+    
+    
+    
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/exportingcollections.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/exportingcollections.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/exportingcollections.htm	(revision 21112)
@@ -0,0 +1,53 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="exportingcollections"><font size="5" face="Verdana"><strong>8.3: Ð­ÐºÑÐ¿ÐŸÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐœÐ° CD-ROM</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      
+Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ, ÐºÐ°Ðº ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð² ÑÐ°ÐŒÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÐžÐ¹ÑÑ CD-ROM Greenstone. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐ°Ðº ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+Greenstone ÐŒÐŸÐ¶ÐµÑ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŸÐŽÐœÑ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ð² ÑÐ°ÐŒÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÐžÐ¹ÑÑ CD-ROM Ð¿ÐŸÐŽ Windows. Ð§ÑÐŸÐ±Ñ ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ, ÐŽÐŸÐ»Ð¶ÐµÐœ  Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ Ð¿Ð°ÐºÐµÑ "Ð­ÐºÑÐ¿ÐŸÑÑ Ð² CD-ROM". ÐÐœ ÐœÐµ Ð²ÐºÐ»ÑÑÐµÐœ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ, Ð¿ÐŸÑÑÐŸÐŒÑ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐÐ°ÑÑ ÐžÐœÑÑÐ°Ð»Ð»ÑÑÐžÑ, ÑÑÐŸÐ±Ñ ÐµÐ³ÐŸ Ð²ÐºÐ»ÑÑÐžÑÑ.
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð­ÐºÑÐ¿ÐŸÑÑ Ð² CD-ROM". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÐŸÐ±ÑÐ°ÐœÐžÐ¹ Greenstone; ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð»ÑÐ±ÐŸÐ¹, ÑÑÐŸÐ±Ñ ÑÐ²ÐžÐŽÐµÑÑ ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ. ÐÐŸÐŒÐµÑÑÑÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐµ Ð¿ÐŸÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐŽÐ»Ñ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²Ð²ÐµÑÑÐž ÐžÐŒÑ CD-ROM Ð² Ð¿ÐŸÐ»Ðµ: ÑÑÐŸ ÑÐŸ, ÑÑÐŸ Ð¿ÐŸÑÐ²ÐžÑÑÑ Ð² ÑÑÐ°ÑÑÐŸÐ²ÐŸÐŒ ÐŒÐµÐœÑ, ÐºÐŸÐ³ÐŽÐ° CD-ROM Ð±ÑÐŽÐµÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ. ÐÐ°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð­ÐºÑÐ¿ÐŸÑÑ". ÐÑÐŸÑÐµÑÑ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž ÐŒÐŸÐ¶ÐµÑ Ð·Ð°ÐœÑÑÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐŒÐžÐœÑÑ.
+<p></p>
+ ÐÐŸÑÐ»Ðµ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÑ Greenstone Ð¿ÐŸÐºÐ°Ð¶ÐµÑ ÐžÐŒÑ Ð¿Ð°Ð¿ÐºÐž, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ¹ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð¿ÐžÑÑÑÐžÐ¹ ÐºÐŸÐŒÐ¿Ð°ÐºÑ-ÐŽÐžÑÐº ÐŽÐ»Ñ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐµÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÐœÐ° ÑÐžÑÑÑÐ¹ CD-ROM.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/fileassociations.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/fileassociations.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/fileassociations.htm	(revision 21112)
@@ -0,0 +1,68 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="fileassociations"><font size="5" face="Verdana"><strong>8.2: ÐÑÑÐŸÑÐžÐ°ÑÐžÐž ÑÐ°Ð¹Ð»Ð°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐžÐºÐ»Ð°ÐŽÐœÑÐµ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ, ÑÑÐŸÐ±Ñ ÐŸÑÐºÑÑÐ²Ð°ÑÑ ÑÐ¿ÐµÑÐžÑÐžÑÐœÑÐµ ÑÐžÐ¿Ñ ÑÐ°Ð¹Ð»ÐŸÐ². Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ±ÑÑÑÐœÑÐµÑ, ÐºÐ°Ðº ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+  Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐŸÑÐºÑÑÐ²Ð°ÑÑ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž ÑÐµÐ»ÐºÐ°ÑÑ "ÐÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ".
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐµÐ»ÐµÐ²ÐŸÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ° ÐžÐ»Ðž ÐœÐ°Ð¿ÐžÑÐžÑÐµ ÐœÐŸÐ²ÐŸÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ (ÐœÐµ Ð²ÐºÐ»ÑÑÐ°Ñ "."). ÐÐ°ÑÐµÐŒ ÐžÐ»Ðž ÐœÐ°Ð¿ÐžÑÐžÑÐµ Ð² ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐŒ Ð¿ÐŸÐ»Ðµ ÐºÐŸÐŒÐ°ÐœÐŽÑ,Ð·Ð°Ð¿ÑÑÐºÐ°ÑÑÑÑ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÐŸÐµ Ð¿ÑÐžÐ»ÐŸÐ¶ÐµÐœÐžÐµ,ÐžÐ»Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿ÑÐžÐ»ÐŸÐ¶ÐµÐœÐžÐµ ÐžÐ· ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ³ÐŸ ÐŸÐºÐœÐ° "ÐÑÐŸÑÐŒÐŸÑÑ". ".%1" ÐŒÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð² ÐºÐŸÐŒÐ°ÐœÐŽÐµ Ð·Ð°Ð¿ÑÑÐºÐ°, ÑÑÐŸÐ±Ñ Ð²ÑÑÐ°Ð²ÐžÑÑ ÐžÐŒÑ ÑÐ°Ð¹Ð»Ð°, ÐºÐŸÑÐŸÑÑÐ¹ ÐŽÐŸÐ»Ð¶ÐµÐœ ÐŸÑÐºÑÑÑÑÑÑ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐŸÐœÐž Ð·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÑ,Ð¿ÑÐœÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž" ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒ Ðž ÐŒÐŸÐ¶ÐœÐŸ ÑÐµÐ»ÑÐºÐŸÐŒ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ.  
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°. ÐÑÐ±Ð°Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐ°Ñ ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÑÐ²ÑÐ·Ðž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ Ð² Ð¿ÐŸÐ»Ðµ Ð·Ð°Ð¿ÑÑÐºÐ° ÐºÐŸÐŒÐ°ÐœÐŽ. ÐÑÑÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ ÐµÐµ Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ".
+ <p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ".(Ð Ð°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐŸÑÑÐ°ÐµÑÑÑ Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ "ÐÐ»Ñ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ²").
+ <p></p>
+ ÐÑÐŸÑÐžÐ°ÑÐžÐ²ÐœÑÐµ ÑÐ°Ð¹Ð»Ñ Ñ
+ÑÐ°ÐœÑÑÑÑ Ð² Ð³Ð»Ð°Ð²ÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°, Ð² ÑÐ°Ð¹Ð»Ðµ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐŒ "associations.xml".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/filteringthetree.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/filteringthetree.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/filteringthetree.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="filteringthetree"><font size="5" face="Verdana"><strong>4.5: Ð€ÐžÐ»ÑÑÑÐŸÐ²Ð°ÐœÐžÐµ ÐŽÐµÑÐµÐ²Ð°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+"Ð€ÐžÐ»ÑÑÑÐŸÐ²Ð°ÐœÐžÐµ" ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÑÑÐ¶Ð°ÑÑ Ð¿ÐŸÐžÑÐº ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐžÐ»ÑÑÑÐŸÐŒ
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+              ÐÐ°ÑÑÑÐŸÐ¹ÐºÐ° ÑÐžÐ»ÑÑÑÐ°
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÑÐœÐºÑ "ÐÐŸÐºÐ°Ð·Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ" Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÐŒÐµÐœÑ Ð¿ÐŸÐŽ ÐºÐ°Ð¶ÐŽÑÐŒ ÐŽÐµÑÐµÐ²ÐŸÐŒ Ð²ÑÐ²ÐŸÐŽÐžÑ ÑÐ¿ÐžÑÐŸÐº Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÑ
+ ÑÐžÐ»ÑÑÑÐŸÐ² ÑÐžÐ¿Ð° "Images".ÐÑÐ±ÐµÑÐžÑÐµ Ðž Ð²ÑÐµ Ð²ÑÐµÐŒÐµÐœÐœÐŸ ÑÐºÑÑÑÑÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÐŽÐµÑÐµÐ²Ðµ.Ð§ÑÐŸÐ±Ñ Ð²ÐŸÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐŽÐµÑÐµÐ²ÐŸ,Ð·Ð°ÐŒÐµÐœÐžÑÐµ ÑÐžÐ»ÑÑÑ ÑÐœÐŸÐ²Ð° ÐœÐ° "ÐÑÐµ ÑÐ°Ð¹Ð»Ñ".Ð­ÑÐž ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÐœÐµ ÐžÐ·ÐŒÐµÐœÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð¿ÑÐž ÑÑÐŸÐŒ ÐŸÐœÐž ÐœÐµ Ð·Ð°ÑÑÐ°Ð³ÐžÐ²Ð°ÑÑ Ð¿Ð°Ð¿ÐºÐž Ð² ÐŽÐµÑÐµÐ²Ðµ. 
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°ÐŽÐ°ÑÑ ÐœÐ°ÑÑÑÐŸÐ¹ÐºÑ ÑÐžÐ»ÑÑÑÐ°,ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°Ð² Ð² ÑÑ
+ÐµÐŒÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐ°ÐœÐŽÐ°ÑÑÐœÑÐµ Ð°Ð±Ð±ÑÐµÐ²ÐžÐ°ÑÑÑÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ¹ ÑÐžÑÑÐµÐŒÑ ÑÐžÐ¿Ð° "*.*" ÐžÐ»Ðž "*.doc" ("*" ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð»ÑÐ±ÑÐŒ ÑÐžÐŒÐ²ÐŸÐ»Ð°ÐŒ). 
+     </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/formatstatements.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/formatstatements.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/formatstatements.htm	(revision 21112)
@@ -0,0 +1,98 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="formatstatements"><font size="5" face="Verdana"><strong>6.9: ÐÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÑÐŸÑÐŒÐ°ÑÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ ÐÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑ ÑÑÑÑÐºÑÑÑÐŸÐ¹ Ðž Ð²ÐžÐŽÐŸÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐœÐž Ð·Ð°ÑÑÐ°Ð³ÐžÐ²Ð°ÑÑ ÑÐ°ÐºÐžÐµ Ð²ÐµÑÐž,ÐºÐ°Ðº ÐŒÐµÑÑÐŸ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÑ ÐºÐœÐŸÐ¿ÐºÐž,ÐºÐŸÐ³ÐŽÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ,Ðž ÐºÐ°ÐºÐžÐµ ÑÐ²ÑÐ·Ðž ÐŸÑÐŸÐ±ÑÐ°Ð¶Ð°ÑÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐŒ DateList. ÐÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° ÐœÐµ Ð»ÐµÐ³ÐºÐŸ ÑÐ°Ð·ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÑÑ,Ðž ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÑÐŸÑÐžÑÐ°ÑÑ Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÐ¶ÐŽÐ°ÑÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÑÐŸÑÐŒÐ°ÑÐ° Ðž ÑÐŸ, ÐºÐ°Ðº ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÐµÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÐœÐžÐŒ. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÑÐŸÑÐŒÐ°ÑÐ°".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð€ÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐžÐŒÐµÐœÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° Ðº ÑÐµÐŒÑ-ÐœÐžÐ±ÑÐŽÑ Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ "ÐÑÐ±ÐŸÑ ÑÑÐµÐŽÑÑÐ²",ÐºÐŸÑÐŸÑÑÐ¹ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐºÐ°Ð¶ÐŽÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ Ðž Ð²ÑÑÑÐŸÐµÐœÐœÑÐ¹ ÑÐ¿ÐžÑÐŸÐº ÑÑÐµÐŽÑÑÐ². ÐÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ ÑÑÐµÐŽÑÑÐ²Ð°, ÐµÑÑÑ ÐŽÐ²Ð° ÑÐžÐ¿Ð° ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ²Ð»ÑÑÑÑÑ Ð¿ÑÐŸÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐžÐ»Ðž ÐœÐµÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž,ÐŸÐœÐž ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑÑÑ ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž. ÐÑÑÐ³ÐžÐµ ÑÑÐµÐ±ÑÑÑ, ÑÑÐŸÐ±Ñ Ð±ÑÐ»Ð° ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐ° ÑÐŸÑÐŒÐ°ÑÐœÐ°Ñ ÑÑÑÐŸÐºÐ°. ÐÐ»Ñ ÐœÐžÑ
+ ÐžÐŒÐµÐµÑÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "ÐÐŸÐ·ÐŽÐµÐ¹ÑÑÐ²ÑÑÑÐžÐ¹ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ", ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ, Ðº ÐºÐ°ÐºÐŸÐ¹ ÑÐ°ÑÑÐž ÑÑÐµÐŽÑÑÐ² ÑÑÑÐŸÐºÐ° ÐŸÐ±ÑÐ°ÑÐ°ÐµÑÑÑ,ÐµÑÐ»Ðž ÑÑÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ, ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ°   ("Ð¡ÑÑÐŸÐºÐ° ÑÐŸÑÐŒÐ°ÑÐ° HTML") ÐŽÐ»Ñ Ð²Ð²ÐŸÐŽÐ° ÑÑÑÐŸÐºÐž, Ðž Ð²ÑÐ±ÐŸÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ "ÐÐµÑÐµÐŒÐµÐœÐœÑÑ
+". Ð§ÑÐŸÐ±Ñ Ð²ÑÑÐ°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ Ð² ÑÐµÐºÑÑÑÑ Ð¿ÐŸÐ·ÐžÑÐžÑ Ð² ÑÐŸÑÐŒÐ°ÑÐœÑÑ ÑÑÑÐŸÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ°  Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÑÐ°Ð²ÐžÑÑ". 
+<p></p>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐŽÐ»Ñ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÐ°, Ð²ÑÐ±ÐžÑÐ°Ñ Ð±Ð»Ð°ÐœÐº ÑÐ²ÐŸÐ¹ÑÑÐ². Ð­ÑÐŸÑ ÑÐŸÑÐŒÐ°Ñ Ð·Ð°ÑÐµÐŒ Ð¿ÑÐžÐŒÐµÐœÑÐµÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ Ð¿ÑÐžÐŒÐµÐœÐžÐŒÑÐŒ ÑÐ²ÐŸÐ¹ÑÑÐ²Ð°ÐŒ, ÐµÑÐ»Ðž ÐŽÑÑÐ³ÐŸÐµ ÐœÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐŸ.
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°,Ð²ÐœÐµÑÐžÑÐµ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ,ÐºÐ°Ðº ÐŸÐ±ÑÑÑÐœÐµÐœÐŸ Ð²ÑÑÐµ, Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".ÐÐŸÐ²Ð°Ñ ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÑÐŸÑÐŒÐ°ÑÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÐžÑÐºÐµ "Ð¢ÐµÐºÑÑÐžÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°".Ð¢ÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐœÐ° ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÑÐ²ÑÐ·Ð°ÐœÐ° Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÐºÐŸÐŒÐ±ÐžÐœÐ°ÑÐžÐµÐ¹ ÑÐ²ÐŸÐ¹ÑÑÐ²ÐŸ/ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ.
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".
+<p></p>
+  Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐŒÐ· ÑÐ¿ÐžÑÐºÐ°, ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".
+<p></p>
+ ÐÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹ ÐŸ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ
+ Ðž ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÐ°Ñ
+ ÑÑÐµÐŽÑÑÐ² ÐŸÐ±ÑÐ°ÑÐžÑÐµÑÑ Ðº Ð³Ð»Ð°Ð²Ðµ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑÐ»Ðž ÐºÐœÐŸÐ¿ÐºÐ° "ÐÐŸÐ·Ð²ÐŸÐ»ÐµÐœÐœÑÐµ ÑÐ°ÑÑÐžÑÐµÐœÐœÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ" Ð¿ÐŸÐŒÐµÑÐµÐœÐ°,ÑÐŸ ÑÐ°Ð·ÑÐµÑÐµÐœÑ ÐœÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÑÐŸÐŽÐ²ÐžÐœÑÑÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÑ . Ð¡Ð¿ÐžÑÐŸÐº ÑÑÐµÐŽÑÑÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÑ, ÐœÐµÐŒÐœÐŸÐ³ÐŸ ÐžÐ·ÐŒÐµÐœÐµÐœ Ðž Ð±ÐŸÐ»ÑÑÐµÐµ ÑÐžÑÐ»ÐŸ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ
+ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐŒÐ°ÐœÐŽÐŸÐ¹ ÑÐŸÑÐŒÐ°ÑÐ°, ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°Ñ Ð±ÐŸÐ»ÑÑÐµÐµ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÑÑÐ°ÐœÐžÑÐµÐ¹ ÐŸÑÐžÐ³ÐžÐœÐ°Ð»-ÐŒÐ°ÐºÐµÑÐ°.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/generalsettings.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/generalsettings.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/generalsettings.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="generalsettings"><font size="5" face="Verdana"><strong>6.2: ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ðž ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŸÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž, ÐŸÑÐœÐŸÑÑÑÐžÐµÑÑ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð² "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ±ÑÐžÐµ"
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐŽÐµÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,Ð¿ÑÐµÐŽÑÑÐŒÐŸÑÑÐµÐœÐœÑÐµ Ð¿ÑÐž ÑÐŸÐ·ÐŽÐ°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÑ.
+<p></p>
+ ÐÐ°Ð²ÐµÑÑ
+Ñ ÑÑÑÐ°ÐœÐžÑÑ ÐžÐŒÐµÐµÑÑÑ Ð¿ÐŸÐ»Ðµ ÐžÐœÑÑÑÑÐºÑÐžÐž,ÐºÐŸÑÐŸÑÐ°Ñ ÐžÐŒÐµÐµÑÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð·ÐŽÐµÐ»Ð°.ÐÐœÐŸ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐºÑÐ°ÑÐºÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº ÑÐºÐ°Ð·Ð°ÐœÐžÐ¹,ÑÑÐŸÐ±Ñ ÐœÐ°Ð¿ÐŸÐŒÐœÐžÑÑ ÐÐ°ÐŒ,ÐºÐ°ÐºÐžÐµ ÑÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐµ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÐž ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž.
+<p></p>
+ Ð¡ÐœÐ°ÑÐ°Ð»Ð° ÐžÐŽÑÑ ÐºÐŸÐœÑÐ°ÐºÑÐœÑÐµ Ð°ÐŽÑÐµÑÐ° ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ ÑÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑÐµÐ³ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð»ÐžÑÐ°. Ð¡Ð»ÐµÐŽÑÑÑÐµÐµ Ð¿ÐŸÐ»Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð¡Ð»ÐµÐŽÑÑÐµÐµ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð¿Ð°Ð¿ÐºÑ, Ð² ÐºÐŸÑÐŸÑÐŸÐ¹ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐœÐŸ ÐœÐµ Ð¿ÐŸÐŽÐ»ÐµÐ¶ÐžÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑÐŒ. Ð¡Ð»ÐµÐŽÑÑÑÐµÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ (Ð² ÑÐŸÑÐŒÐµ URL) ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ,ÑÑÐŸÐ±Ñ Ð¿ÐŸÐºÐ°Ð·Ð°ÑÑ ÐœÐ°Ð²ÐµÑÑ
+Ñ ÑÐ»ÐµÐ²Ð° ÐŸÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÑÑÐ°ÐœÐžÑÑ ÐŸ ÐœÐµÐ¹, Ðž ÑÐ»ÐµÐŽÑÑÑÐµÐµ - ÐžÐºÐŸÐœÐºÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐ°Ñ Ð² Greenstone ÑÑÑÐ°ÐœÐžÑÐ° ÐŽÐ»Ñ ÑÐ²ÑÐ·Ðž Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹. ÐÐŸÑÐ»Ðµ ÑÑÐŸÐ³ÐŸ, Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐŽÐµÐ»ÐžÑÑ Ð¿ÑÐœÐºÑ Ð³Ð°Ð»ÐŸÑÐºÐŸÐ¹, ÑÐºÐ°Ð·Ð°Ð², ÐŽÐŸÑÑÑÐ¿ÐœÐ° Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŽÑÑÐ³ÐžÐŒ. ÐÐ°ÐºÐŸÐœÐµÑ, ÑÐ»ÐµÐŽÑÐµÑ ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ° "ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž",Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÐ°Ñ ÑÐ°ÐœÐµÐµ Ð² <a href="creatingacollection.htm#creatingacollection">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</a>.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/help.xml
===================================================================
--- /main/tags/3.01/gli/help/ru/help.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/help.xml	(revision 21112)
@@ -0,0 +1,1427 @@
+<Document>
+  <Section name="introduction">
+    <Title>ÐÐ²ÐµÐŽÐµÐœÐžÐµ</Title>
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Greenstone - ÑÑÐŸ ÐžÐœÑÑÑÑÐŒÐµÐœÑ ÐŽÐ»Ñ ÑÐ±ÐŸÑÐ° Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž
+ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ñ Ð¿ÐŸÑÐ»ÐµÐŽÑÑÑÐžÐŒ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÐžÑÑÐŸÐ²ÑÑ
+ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐº.ÐÐœ
+ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÐµÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÑÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐŒ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑÐŒ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐœÐŸÐ³ÐŸ ÐŸÐ±ÐµÑÐ¿ÐµÑÐµÐœÐžÑ
+Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž Greenstone Ð³ÑÐ°ÑÐžÑÐµÑÐºÐžÐŒ Ð¿ÑÑÐµÐŒ Ðž ÑÐ°Ð±ÐŸÑÐŸÐ¹ ÐŒÑÑÑÑ.
+    <Section name="ofmiceandmenus">
+      <Title>ÐÑÑÑ Ðž ÐÐµÐœÑ</Title>
+Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŽÐ°ÐµÑ Ð±Ð°Ð·ÐŸÐ²ÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð²Ð·Ð°ÐžÐŒÐŸÐŽÐµÐ¹ÑÑÐ²ÐžÐž Ñ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ  Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž. 
+ÐÑÐ»Ðž ÐÑ Ð·ÐœÐ°ÐºÐŸÐŒÑ Ñ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐ°ÐŒÐž ÑÐžÐ¿Ð° Internet Explorer ÐžÐ»Ðž Microsoft office, Ð»ÐµÐ³ÐºÐŸ ÐŸÐ±ÑÐ°ÑÐ°ÐµÑÐµÑÑ Ñ ÐŒÑÑÑÑ Ðž ÐŒÐµÐœÑ, Ð¿ÑÐŸÐ¿ÑÑÑÐžÑÐµ <Reference target="howtoavoidthisdocument">Ðž Ð¿ÐµÑÐµÐ¹ÐŽÐžÑÐµ Ðº ÑÐ»ÐµÐŽÑÑÑÐµÐŒÑ ÑÐ°Ð·ÐŽÐµÐ»Ñ </Reference>.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    ÐÑÐœÐŸÐ²ÐœÑÐµ Ð¿ÐŸÐœÑÑÐžÑ ÐºÐŸÐœÑÐµÐ¿ÑÐžÐž
+            <ContentsItem>
+               ÐÐµÐ¹ÑÑÐ²ÐžÑ ÐŒÑÑÐž
+	    </ContentsItem>
+            <ContentsItem>
+               ÐÐ»Ð°Ð²ÐžÐ°ÑÑÑÐ°
+	    </ContentsItem>
+          </ContentsItem>
+          <ContentsItem>
+	    ÐÐµÑÐµÑ
+ÐŸÐŽ Ðº Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐµ
+          </ContentsItem>
+	</ContentsGroup>
+	<ContentsGroup linkto="advanced">
+          <ContentsItem>
+	    ÐÑÑÑÑÑÐµ ÐºÐ»ÑÑÐž
+          </ContentsItem>
+	</ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ ÑÐŸÐ³Ð»Ð°ÑÐµÐœÐžÑÐŒ Windows Microsoft Ðž ÑÑÐµÐ±ÑÐµÑ ÐŸÐ±ÑÑÐœÐŸÐ³ÐŸ Ð·ÐœÐ°ÐœÐžÑ Windows.   
+ <Break/>
+ ÐÑÐ±Ð°Ñ ÑÐ°ÑÑÑ ÑÐºÑÐ°ÐœÐ°, Ñ ÐºÐŸÑÐŸÑÐŸÐ¹ ÐÑ Ð²Ð·Ð°ÐžÐŒÐŸÐŽÐµÐ¹ÑÑÐ²ÑÐµÑÐµ, ÑÐžÐ¿Ð° ÐºÐœÐŸÐ¿ÐºÐž ÐžÐ»Ðž ÐŸÐ±Ð»Ð°ÑÑÐž ÑÐµÐºÑÑÐ°,ÐœÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµÐŒ. Ð Ð»ÑÐ±ÐŸÐµ Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐµ Ð²ÑÐµÐŒÑ ÐŸÐŽÐžÐœ ÑÐ»ÐµÐŒÐµÐœÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÑÐ¹ "ÑÐŸÐºÑÑ",Ð²ÑÐŽÐµÐ»ÑÐµÑÑÑ Ð¿ÐŸÐŽÑÐ²ÐµÑÐºÐŸÐ¹ Ðž ÐŸÑÐºÐ»ÐžÐºÐ°ÐµÑÑÑ ÐœÐ° ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÑ. ÐÐµÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ Ð²ÑÐ±ÐžÑÐ°ÑÑ ÑÑÐ°ÑÑÐºÐž, ÐºÐŸÑÐŸÑÑÐµ Ð²ÑÐŽÐµÐ»ÑÑÑÑÑ ÑÐµÐŒÐœÐŸÑÐžÐœÐžÐŒ ÑÐ²ÐµÑÐŸÐŒ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐŸÑÐŸÐ±ÑÐ°Ð¶Ð°ÑÑÑÑ Ñ Ð¿ÐŸÐœÐžÐ¶ÐµÐœÐœÐŸÐ¹ ÑÑÐºÐŸÑÑÑÑ, Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°Ñ ÑÐµÐŒ ÑÐ°ÐŒÑÐŒ, ÑÑÐŸ ÐŸÐœÐž Ð·Ð°Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÑ.
+<Break/>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑÑÑ Ðž ÐŽÐµÐ¹ÑÑÐ²ÐŸÐ²Ð°ÑÑ  Ð»ÐµÐ²ÐŸÐ¹ ÐžÐ»Ðž Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž ÐŒÑÑÐž ÐŸÐ±ÑÑÐœÑÐŒ ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ. ÐÐœÐŸÐ³ÐžÐµ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÑ ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ "ÑÑÐœÑÑÑ" ÐžÑ
+, ÐœÐ°Ð¶ÐžÐŒÐ°Ñ Ðž ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ñ Ð»ÐµÐ²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐŒÑÑÐž, Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑ ÐžÑ
+ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐŒÑÑÐž Ðž ÐŸÑÑÐ°Ð²ÐžÑÑ ÐžÑ
+ Ð² ÐŽÑÑÐ³ÐŸÐŒ ÐŒÐµÑÑÐµ, ÐŸÑÐ¿ÑÑÐºÐ°Ñ ÐºÐœÐŸÐ¿ÐºÑ. ÐÐŸÑÐµÐœÑÐžÐ°Ð»ÑÐœÑÐµ ÐŸÐ±ÑÐµÐºÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐžÐ·ÐŒÐµÐœÑÑÑ ÑÐ²ÐŸÐ¹ Ð²ÐžÐŽ, ÐºÐŸÐ³ÐŽÐ° ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ ÑÐºÐŸÐ»ÑÐ·ÐžÑ Ð¿ÐŸ ÐœÐžÐŒ.  
+<Break/>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÑ ÐŽÐ»Ñ Ð·Ð°Ð¿ÐžÑÐž Ð² ÐŸÐ±Ð»Ð°ÑÑÐž ÑÐµÐºÑÑÐ°. ÐÐ»ÑÑÐµÑÐœÐ°ÑÐžÐ²Ñ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÐµ ÐŽÐŸÑÑÑÐ¿ÐœÑ ÐŽÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÑÑ
+ ÐºÐ»ÑÑÐµÐ²ÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² ÐºÐ²Ð°ÐŽÑÐ°ÑÐœÑÑ
+ ÑÐºÐŸÐ±ÐºÐ°Ñ
+. Ð Ð¿ÑÐžÐŒÐµÑÑ, [Tab] ÐžÐ·ÐŒÐµÐœÑÐµÑ ÑÐŸÐºÑÑ. ÐÐœÐ°Ðº Ð¿Ð»ÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÑÑÐŸ ÐŽÑÑÐ³ÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐœÐ°Ð¶Ð°ÑÑ ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ. 
+<Break/>
+ÐÑÐ¹ÐŽÐžÑÐµ ÐžÐ· Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°, Ð²ÑÐ±ÐžÑÐ°Ñ Ð¿ÑÐœÐºÑ "ÐÑÑ
+ÐŸÐŽ" Ð² ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»Ð°". ÐÐ°ÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐŽÐµÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ  ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐ°.
+<Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÐŒÐµÐœÑ, ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ð¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ [ALT] Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ Ð±ÑÐºÐ²Ñ (Ð¿ÐŸÐŽÑÐµÑÐºÐœÑÑÑÑ).ÐÐ°Ð¿ÑÐžÐŒÐµÑ,ÐŽÐ»Ñ Ð²ÑÐ±ÐŸÑÐ° ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" ÐœÐ°Ð¶ÐŒÐžÑÐµ [ALT] + [F]. ÐÐ»Ñ Ð²ÑÐ±ÐŸÑÐ° ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÑÐœÐºÑÐ°, ÐœÐ°Ð¶ÐŒÐžÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, ÐµÑÐ»Ðž ÐÑ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÐµÑÑ Ð² ÐŒÐµÐœÑ Ð€Ð°Ð¹Ð» ÐœÐ°Ð¶ÐŒÐžÑÐµ [S], ÑÑÐŸÐ±Ñ "Ð¡ÐŸÑ
+ÑÐ°ÐœÐžÑÑ"ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+</Section>
+<Section name="howtoavoidthisdocument">
+<Title>ÐÐ°Ðº ÐžÐ·Ð±ÐµÐ¶Ð°ÑÑ ÑÑÐµÐœÐžÑ ÑÑÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Title>
+ ÐÐµ ÑÐžÑÐ°Ð¹ÑÐµ ÑÑÐŸÑ ÑÐµÐºÑÑ Ð¿ÐŸÐŒÐŸÑÐž Ð¿ÐŸÐ»ÐœÐŸÑÑÑÑ! ÐÐŸÑÑÐ°ÑÐŸÑÐœÐŸ Ð¿ÑÐŸÑÐµÑÑÑ, ÐºÐ°Ðº ÑÑÐŒÐµÑÑ Ð¿ÐŸÐ»ÑÑÐžÑÑ Ð¿ÐŸÐŒÐŸÑÑ, ÐºÐŸÐ³ÐŽÐ° ÐÑ ÐœÑÐ¶ÐŽÐ°ÐµÑÐµÑÑ Ð² ÑÑÐŸÐŒ.  
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+           Ð§ÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐ°Ñ ÐºÐŸÐœÑÐµÐºÑÑÐœÐ°Ñ Ð¿ÐŸÐŒÐŸÑÑ
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐœÑÑÑÑÐŒÐµÐœÑ Ð¿ÐŸÐŽÑÐºÐ°Ð·ÐŸÐº
+          </ContentsItem>
+          <ContentsItem>
+             ÐÑÑÐ³ÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÑÐœÐºÑÑ ÐŒÐµÐœÑ "ÐÐŸÐŒÐŸÑÑ" ÑÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÐµ ÑÐ°Ð·ÐŽÐµÐ»Ñ,ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐœÐ°ÐžÐ±ÐŸÐ»ÐµÐµ Ð°ÐŽÐµÐºÐ²Ð°ÑÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÐŸÐŒÐŸÑÐž, Ðž ÑÐŸÐ¿ÑÐŸÐ²ÐŸÐ¶ÐŽÐ°ÑÑÑÑ ÐœÐµÐ±ÐŸÐ»ÑÑÐžÐŒ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµÐŒ ÐºÐœÐžÐ³Ðž  
+<Break/>
+ÐÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐµÑÐ»Ðž ÐÑ ÑÐ°Ð·ÐŒÐµÑÐ°ÐµÑÐµ ÐŒÑÑÑ ÐœÐ° ÐœÐžÑ
+,Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ 
+"Ð¿ÐŸÐŽÑÐºÐ°Ð·ÐºÐ°",ÐºÐŸÑÐŸÑÐ°Ñ ÐŸÐ±ÑÑÑÐœÑÐµÑ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ,ÐºÐŸÑÐŸÑÑÐµ ÐŸÐœÐž Ð²ÑÐ¿ÐŸÐ»ÐœÑÑÑ  
+<Break/>
+ ÐÐµÑÐµÐŽ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ, ÑÐœÐ°ÑÐ°Ð»Ð° Ð¿ÑÐŸÑÑÐžÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÑÐžÑ Greenstone/
+    </Section>
+  </Section>
+  <Section name="startingoff">
+    <Title>ÐÐŸ ÑÑÐ°ÑÑÐ°</Title>
+    Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÑÑ, Ð·Ð°Ð³ÑÑÐ¶Ð°ÑÑ Ðž ÑÐŽÐ°Ð»ÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+    <Section name="creatingacollection">
+ <Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>      
+       
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÑÑÑÑÐŸÐµ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐ°Ð³Ð»Ð°Ð²ÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             ÐÑÐŒÐµÐœÐ° ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÐŸÑÑÑÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐŒÐµÐœÑ "File" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ Ð² ÐœÑÐŒ Ð¿ÑÐœÐºÑ "New". ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ»Ñ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÑÐ·Ð°ÑÐµÐ»ÑÐœÐŸ Ð·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÑ, ÐœÐŸ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°ÐŒÐµÐœÐžÑÑ ÐžÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ð¿ÐŸÐ·Ð¶Ðµ, ÐµÑÐ»Ðž ÐÑ Ð² ÑÑÐŸÐŒ ÐœÑÐ¶ÐŽÐ°ÐµÑÐµÑÑ Ñ ÑÐŸÑÐºÐž Ð·ÑÐµÐœÐžÑ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°.
+ <Break/>
+ÐÐ°Ð³Ð»Ð°Ð²ÐžÐµ "ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ/Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž" - ÑÑÐŸ ÑÐµÐºÑÑ, ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÑÐ¹ ÐœÐ°Ð²ÐµÑÑ
+Ñ ÐŽÐŸÐŒÐ°ÑÐœÐµÐ¹ ÑÑÑÐ°ÐœÐžÑÑ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐœÐŸ ÐŒÐŸÐ¶ÐµÑ ÐžÐŒÐµÑÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐ»ÑÐœÑÑ ÐŽÐ»ÐžÐœÑ.
+<Break/>
+"ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ" ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²ÐžÑÑ ÐºÐ°Ðº ÐŒÐŸÐ¶ÐœÐŸ ÐŽÐµÑÐ°Ð»ÑÐœÐµÐµ ÑÐŸ,ÐŸ ÑÑÐŒ ÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ [Enter] ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ,ÑÑÐŸÐ±Ñ ÑÐ°Ð·Ð±ÐžÑÑ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ ÐœÐ° Ð¿Ð°ÑÐ°Ð³ÑÐ°ÑÑ.
+<Break/>
+ ÐÐ°ÐºÐŸÐœÐµÑ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ, ÐŽÐŸÐ»Ð¶ÐœÐ° Ð»Ðž ÐœÐŸÐ²Ð°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐŒÐµÑÑ ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ Ð²ÐžÐŽ Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸ Ðž ÑÑÑÐµÑÑÐ²ÑÑÑÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐžÐ»Ðž Ð¶Ðµ ÐœÐ°ÑÐ°ÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ "ÐÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ".
+<Break/>
+Ð©ÐµÐ»ÐºÐœÐžÑÐµ "OK", ÑÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. ÐÑÐ»Ðž ÐÑ Ð²ÑÐ±ÑÐ°Ð»Ðž "New Collection", ÑÐŸ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð²Ð²ÐµÑÑÐž ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐžÑ
+ Ð² ÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐœÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, Ð° ÐŒÐŸÐ¶ÐµÑÐµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŽÑÑÐ³ÐžÐµ Ð¿ÐŸÐ·Ð¶Ðµ. 
+<Break/>
+ÐÐ°Ð¶Ð°ÑÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž "Cancel" ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð²ÐŸÐ·Ð²ÑÐ°ÑÐ°ÐµÑ ÐÐ°Ñ Ðº Ð³Ð»Ð°Ð²ÐœÐŸÐŒÑ ÑÐºÑÐ°ÐœÑ. 
+    <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐœÐŸÐ¿ÐºÐž, ÑÐžÐ¿Ð° ÐŒÐµÐœÑ, ÐžÐŒÐµÑÑ ÐŸÐŽÐžÐœ Ð¿ÐŸÐŽÑÑÑÐºÐœÑÑÑÐ¹ ÑÐžÐŒÐ²ÐŸÐ». Ð§ÑÐŸÐ±Ñ Ð²ÑÐ·Ð²Ð°ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐœÐ°Ð¶ÐŒÐžÑÐµ [ALT] Ðž ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ Ð¿ÐŸÐŽÑÐµÑÐºÐœÑÑÑÐ¹ ÑÐžÐŒÐ²ÐŸÐ». 
+    </Section>
+    <Section name="savingacollection">
+      <Title>Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ðž ÐºÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ ÐµÑ ÐœÐµÐ·Ð°ÐŒÐµÐŽÐ»ÐžÑÐµÐ»ÑÐœÐŸ. 
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+            ÐÐŽÐµ ÑÐ°Ð¹Ð»Ñ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð¡ÐŸÑ
+ÑÐ°ÐœÑÐ¹ÑÐµ ÐÐ°ÑÑ ÑÐ°Ð±ÐŸÑÑ ÑÐµÐ³ÑÐ»ÑÑÐœÐŸ, ÐŸÑÐºÑÑÐ²Ð°Ñ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž Ð²ÑÐ±ÐžÑÐ°Ñ Ð² ÐœÑÐŒ Ð¿ÑÐœÐºÑ "Ð¡ÐŸÑ
+ÑÐ°ÐœÐžÑÑ". Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž - ÑÑÐŸ ÐœÐµ ÑÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ, ÐºÐ°Ðº  ÑÐŽÐµÐ»Ð°ÑÑ ÐµÑ Ð³ÐŸÑÐŸÐ²ÐŸÐ¹ Ðº ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ð² Greenstone.
+ (see <Reference target="producingthecollection">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Reference>).
+<Break/>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð·Ð°ÑÐžÑÐ°ÐµÑ ÐÐ°ÑÑ ÑÐ°Ð±ÐŸÑÑ Ð¿ÑÑÑÐŒ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÑ ÐµÑ Ð²ÑÑÐºÐžÐ¹ ÑÐ°Ð·, ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÑ
+ÐŸÐŽÐžÑÐµ ÐžÐ· Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ ÐžÐ»Ðž Ð·Ð°Ð³ÑÑÐ¶Ð°ÐµÑÐµ ÐŽÑÑÐ³ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð·Ð°Ð¿ÐžÑÑÐ²Ð°ÑÑÑÑ  Ð² ÑÐ°Ð¹Ð», ÐžÐŒÐµÑÑÐžÐ¹ ÑÐŸ Ð¶Ðµ ÐžÐŒÑ, ÑÑÐŸ Ðž ÐžÐŒÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ ".col", Ðž ÑÐ°ÑÐ¿ÐŸÐ»Ð°Ð³Ð°ÐµÑÑÑ Ð² Ð¿Ð°Ð¿ÐºÐµ ÑÐŸÐ³ÐŸ Ð¶Ðµ ÐœÐ°ÐžÐŒÐµÐœÐŸÐ²Ð°ÐœÐžÑ, ÑÑÐŸ Ðž Ð¿Ð°Ð¿ÐºÐ° "collect" Ð¿ÑÐž ÐžÐœÑÑÐ°Ð»Ð»ÑÑÐžÐž Greenstone.
+    </Section>
+    <Section name="openingacollection">
+      <Title>ÐÑÐºÑÑÑÐžÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐŸÐŸÐ±ÑÐ°ÐµÑÑÑ,ÐºÐ°Ðº ÐŸÑÐºÑÑÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Open Collection prompt.
+     </ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐ°Ðº ÐŸÑÐºÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÐ°ÐºÑÑÑÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+	  <ContentsItem>
+	     ÐÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐ°ÑÐ»ÐµÐŽÑÑÐ²Ð°
+	  </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ ÐŸÑÐºÑÑÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÑÐºÑÑÑÑ" Ð² ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»",ÑÑÐŸÐ±Ñ Ð±ÑÑÑÑÐŸ Ð¿ÐµÑÐµÐ¹ÑÐž Ðº ÐÑÐºÑÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÐÐ°ÑÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ð² Greenstone. ÐÑÐ±ÐµÑÐžÑÐµ ÐŸÐŽÐœÑ,ÑÑÐŸÐ±Ñ ÑÐ²ÐžÐŽÐµÑÑ ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ   "OÑÐºÑÑÑÑ", ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐµÐµ. ÐÑÐ»Ðž ÐÑ ÐžÑÐžÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ Ð²ÐœÐµ Ð¿Ð°Ð¿ÐºÐž "Ð¡ollect Greenstone",ÑÐŸ ÑÐµÐ»ÐºÐœÐžÑÐµ "Browse" ÐŽÐ»Ñ Ð²ÑÐ·ÐŸÐ²Ð° ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ¹ ÑÐžÑÑÐµÐŒÑ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÑÐ°Ð¹Ð»ÐŸÐ².
+ <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð ÑÐ»ÑÑÐ°Ðµ, ÐµÑÐ»Ðž Ð±ÐŸÐ»ÑÑÐµ ÑÐµÐŒ ÐŸÐŽÐœÐ° Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐ° ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Greenstone Ð²Ð²ÐŸÐŽÐžÑÑÑ ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ, ÑÐµÐ»ÐµÐ²Ð°ÐœÑÐœÑÐµ ÐŽÐžÑÐµÐºÑÑÐžÐž "Ð·Ð°ÐºÑÑÑÑ",ÑÑÐŸÐ±Ñ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐžÑÑ ÐœÐ°Ð»ÐŸÐ¶ÐµÐœÐžÐµ.ÐÑÐž ÐŸÑÐºÑÑÑÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐµÐ±ÐŸÐ»ÑÑÐŸÐ¹ Ð²ÑÐµÐŒÐµÐœÐœÑÐ¹ Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÐ¹ ÑÐ°Ð¹Ð» ÑÐŸÐ·ÐŽÐ°ÐµÑÑÑ Ð² ÐµÑ Ð¿Ð°Ð¿ÐºÐµ.ÐÐµÑÐµÐŽ ÐŸÑÐºÑÑÑÐžÐµÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑ Ð¿ÑÐŸÐ²ÐµÑÐºÑ,ÑÑÐŸÐ±Ñ ÑÐ±ÐµÐŽÐžÑÑÑÑ,ÑÑÐŸ ÐœÐžÐºÐ°ÐºÐžÑ
+ Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐœÐµ ÑÑÑÐµÑÑÐ²ÑÐµÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐºÐ°Ð·Ð°ÑÑ,Ð·Ð°ÐºÑÑÑÐ° Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, Ð¿ÐŸ ÑÐ²ÐµÑÑ ÐµÑ ÐžÐºÐŸÐœÐºÐž:Ð·ÐµÐ»ÑÐœÑÐ¹ - ÐŽÐ»Ñ ÐœÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐºÑÐ°ÑÐœÑÐ¹ - ÐŽÐ»Ñ Ð·Ð°ÐºÑÑÑÐŸÐ¹. ÐÐŽÐœÐ°ÐºÐŸ,ÐºÐŸÐ³ÐŽÐ° ÐžÐ· ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Ð²ÑÑ
+ÐŸÐŽÑÑ Ð¿ÑÐµÐ¶ÐŽÐµÐ²ÑÐµÐŒÐµÐœÐœÐŸ, Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÐ¹ ÑÐ°Ð¹Ð» ÐžÐœÐŸÐ³ÐŽÐ° ÐŸÑÑÐ°ÐµÑÑÑ ÐœÐ° ÐŒÐµÑÑÐµ. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŸÑÐºÑÑÐ²Ð°ÐµÑÐµ ÑÐ°ÐºÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ° Ð·Ð°Ð¿ÑÐ°ÑÐžÐ²Ð°ÐµÑ,Ñ
+ÐŸÑÐžÑÐµ Ð»Ðž ÐÑ "ÑÐºÑÐ°ÑÑÑ" ÐµÐ³ÐŸ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ. ÐÐžÐºÐŸÐ³ÐŽÐ° ÐœÐµ ÐŽÐµÐ»Ð°Ð¹ÑÐµ ÑÑÐŸÐ³ÐŸ,ÐµÑÐ»Ðž ÐºÑÐŸ-ÑÐŸ ÐµÑÐµ Ð² ÑÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ Ð²ÑÐµÐŒÑ ÑÐ°Ð±ÐŸÑÐ°ÐµÑ.   
+<Break/>
+ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŸÑÐºÑÑÐ²Ð°ÐµÑÐµ ÐºÐŸÐ»ÐµÐºÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐ° ÐœÐµ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°  Greenstone, ÐÐ°Ñ Ð¿ÐŸÐ¿ÑÐŸÑÑÑ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐ°Ð±ÐŸÑ (ÐžÐ»Ðž ÐœÐ°Ð±ÐŸÑÑ) ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÑÐ»Ðž ÐœÐž ÐŸÐŽÐžÐœ ÐœÐµ ÐŸÑÐŸÐ±ÑÐ°Ðœ, Ð»ÑÐ±ÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð±ÑÐŽÑÑ ÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÑÑÑÑ. ÐÐœÐ°ÑÐµ, ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð±ÑÐŽÑÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÑ ÑÐ°Ðº Ð¶Ðµ, ÐºÐ°Ðº Ðž ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐµÑÐµ Ð² ÑÐ°Ð¹Ð»Ñ Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž. ÐÑÐŸÑÐµÑÑ ÐŸÐ¿ÐžÑÐ°Ðœ Ð² <Reference target="importingpreviouslyassignedmetadata">ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð²ÑÐ±ÑÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Reference>. 
+    </Section>
+    <Section name="deletingcollections">
+      <Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹</Title>
+      <ObviousSentence>Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÐœÐµÐžÐ·ÐŒÐµÐœÐœÐŸ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐ· ÐžÐœÑÑÐ°Ð»ÑÑÐžÐŸÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž Greenstone.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐ°Ðº ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+      ÐÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ ÐœÐµÐžÐ·ÐŒÐµÐœÐœÐŸ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐ· ÐžÐœÑÑÐ°Ð»ÑÑÐžÐŸÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž Greenstone, Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ..." ÐžÐ· ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»". ÐÑÐºÑÐŸÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº Ð²ÑÐµÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Greenstone. ÐÑÐ±ÐµÑÐžÑÐµ ÐœÑÐ¶ÐœÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÐ° ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ, Ð¿ÐŸÑÐ»Ðµ ÑÑÐŸÐ³ÐŸ Ð¿ÐŸÑÑÐ°Ð²ÑÐµ Ð³Ð°Ð»ÐŸÑÐºÑ Ð² Ð¿ÑÐœÐºÑÐµ Ð²ÐœÐžÐ·Ñ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ", ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. Ð­ÑÐ° ÐŸÐ¿ÐµÑÐ°ÑÐžÑ Ð±ÐµÐ·Ð²ÐŸÐ·Ð²ÑÐ°ÑÐœÐ°, Ð¿ÐŸÑÑÐŸÐŒÑ Ð¿ÑÐŸÐ²ÐµÑÑÑÐµ Ð²ÐœÐžÐŒÐ°ÑÐµÐ»ÑÐœÐŸ, ÐŽÐµÐ¹ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐŸ Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµÐœÑÐ¶ÐœÐ°!
+    </Section>
+  </Section>
+
+  <Section name="downloadingfiles">
+    <Title>ÐÐ°Ð³ÑÑÐ·ÐžÑÑ ÑÐ°Ð¹Ð»Ñ ÐžÐ· ÐžÐœÑÐµÑÐœÐµÑ</Title>
+    ÐÐžÐŽ "ÐÐµÑÐºÐ°Ð»ÐŸ" Ð¿ÐŸÐŒÐŸÐ¶ÐµÑ Ð²Ð°ÐŒ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐžÐ· ÐÐœÑÐµÑÐœÐµÑÐ°. ÐÐ°ÐœÐœÐ°Ñ ÑÐµÐºÑÐžÑ ÐŸÐ±ÑÑÑÐœÑÐµÑ Ð¿ÑÐŸÑÐµÑÑ Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð² ÐÐœÑÐµÑÑÐµÐ¹ÑÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ.
+    <Section name="themirrorview">
+      <Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐÐ°Ð³ÑÑÐ·ÐžÑÑ</Title>
+      Ð­ÑÐ° ÑÐµÐºÑÐžÑ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐºÐ°Ðº ÐºÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÑÑ Ð·Ð°ÐŽÐ°ÑÑ Ðž ÐºÐŸÐœÑÑÐŸÐ»Ñ Ð¿ÑÐŸÑÐµÑÑÐ° Ð·Ð°Ð³ÑÑÐ·ÐºÐž.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+          ÐÐµÑÐµÐ²ÐŸ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð°
+	  </ContentsItem>
+          <ContentsItem>
+          ÐÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ Ð·Ð°Ð³ÑÑÐ·ÐºÐž
+	  </ContentsItem>
+          <ContentsItem>
+          Ð¡Ð¿ÐžÑÐŸÐº Ð·Ð°Ð³ÑÑÐ·ÐºÐž
+	  </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+          Ð Ð°Ð·ÑÐµÑÐžÑÑ Ð¿ÑÐŸÐºÑÐž
+	  </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+      ÐÐŸÑÑÑÐ¿ Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÑ "ÐÐ°Ð³ÑÑÐ·ÐžÑÑ" ÐœÐ°Ð¶Ð°ÑÐµÐŒ ÑÑÐŸÐ¹ Ð·Ð°ÐºÐ»Ð°ÐŽÐºÐž. ÐÐµÑÑ
+ÐœÑÑ Ð¿ÐŸÐ»ÐŸÐ²ÐžÐœÐ° ÑÐºÑÐ°ÐœÐ° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐºÐŸÐœÑÑÐŸÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž. ÐÐžÐ¶ÐœÑÑ Ð¿ÐŸÐ»ÐŸÐ²ÐžÐœÐ° Ð² ÐœÐ°ÑÐ°Ð»Ðµ Ð¿ÑÑÑÐ°, ÐŸÐŽÐœÐ°ÐºÐŸ Ð¿ÐŸÐºÐ°Ð¶ÐµÑ Ð¿ÐµÑÐµÑÐµÐœÑ ÐŸÐ¶ÐžÐŽÐ°ÐµÐŒÑÑ
+ Ðž Ð·Ð°ÐºÐŸÐœÑÐµÐœÐœÑÑ
+ Ð·Ð°Ð³ÑÑÐ·ÐŸÐº.
+      <Break/>
+      Ð€Ð°Ð¹Ð»Ñ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÐµ Ð² Ð¿Ð°Ð¿ÐºÑ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° ÐœÐ°Ð·ÑÐ²Ð°ÑÑÑÑ "ÐÐ°Ð³ÑÑÐ¶Ð°ÐµÐŒÑÐµ ÑÐ°Ð¹Ð»Ñ" (ÑÐŸÐ»ÑÐºÐŸ ÑÐµÐ¹ÑÐ°Ñ ÐºÐŸÐ³ÐŽÐ° Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð²ÐºÐ»ÑÑÐµÐœÐŸ), Ðž ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐ° Ð²ÑÐµÐŒÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐœÑÐŒÐž Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ. Ð€Ð°Ð¹Ð»Ñ Ð² ÑÑÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐžÐžÐŒÐµÐœÑÑÑÑÑ Ð¿ÐŸ ÐžÑ
+ Ð¿ÐŸÐ»ÐœÐŸÐŒÑ URL Ð°ÐŽÑÐµÑÑ.  Ð¡ÐŸÐ·ÐŽÐ°ÐµÑÑÑ ÐœÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ° ÐœÐ° ÐºÐ°Ð¶ÐŽÐŸÐŒ Ñ
+ÐŸÑÑÐµ, ÐœÐ°ÑÐ»ÐµÐŽÑÐµÐŒÐ°Ñ ÐŽÑÑÐ³ÐžÐŒÐž ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÑÐ°Ð³ÐŒÐµÐœÑÐ° Ð¿ÑÑÐž.  Ð­ÑÐŸ Ð³Ð°ÑÐ°ÐœÑÐžÑÑÐµÑ ÐŸÑÐ»ÐžÑÐžÐµ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.
+      <Break/>
+      ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð¿ÐµÑÐ²ÑÐ¹ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÑÑ
+ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐŸÐº, "Source URL",  ÑÑÐŸÐ±Ñ Ð²Ð²ÐµÑÑÐž URL Ð°ÐŽÑÐµÑ Ð² ÑÐµÐ»ÐµÐ²ÐŸÐ¹ ÑÐµÑÑÑÑ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐŸÐœÑÑÐŸÐ»Ñ "ÐÐ»ÑÐ±ÐžÐœÑ ÐÐ°Ð³ÑÑÐ·ÐºÐž" ÐŽÐ»Ñ ÐŸÐ³ÑÐ°ÐœÐžÑÐµÐœÐžÑ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²Ð° Ð³ÐžÐ¿ÐµÑÑÑÑÐ»ÐŸÐº ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž: Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ 0 ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÐŸÐŽÐœÐŸÐ¹ Ð²ÐµÐ± ÑÑÑÐ°ÐœÐžÑÑ; Ð·ÐœÐ°ÑÐµÐœÐžÐµ 1 ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐŸÐŽÐœÑ ÑÑÑÐ°ÐœÐžÑÑ Ðž Ð²ÑÐµ ÑÑÑÐ°ÐœÐžÑÑ ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ ÐœÐµÐ¹ . Ð¿ÑÐµÐŽÐµÐ» Ð³Ð»ÑÐ±ÐžÐœÑ ÑÐ°ÑÐ¿ÑÐŸÑÑÑÐ°ÐœÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐœÐ° html ÑÑÑÐ°ÐœÐžÑÑ. Ð¡Ð»ÐµÐŽÑÐ¹ÑÐµÐµ, Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐ° ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ "Ð³Ð°Ð»ÐŸÑÐµÐº", ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ, ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ ÑÑÐŸÐ±Ñ Ð²ÐºÐ»ÑÑÐžÑÑ ÑÐºÐ°Ð·Ð°ÐœÐœÑÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÑ ÐºÐŸÐœÐºÑÐµÑÐœÐŸÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐºÐž. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐ°, ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÐ°Ð³ÑÑÐ·ÐºÐ°", ÑÑÐŸÐ±Ñ ÐœÐ°ÑÐ°ÑÑ ÐœÐŸÐ²ÑÑ Ð·Ð°Ð³ÑÑÐ·ÐºÑ. ÐÑÑÑ ÐŽÐ²Ðµ ÐŽÑÑÐ³ÐžÑ
+ ÐºÐœÐŸÐ¿ÐºÐž: Ð² ÑÐ°Ð·ÐŽÐµÐ»Ðµ "ÐÐ°ÑÑÑÐŸÐ¹ÐºÐž", ÐºÐŸÑÐŸÑÑÐµ ÑÐ²ÑÐ·Ð°ÐœÑ Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐŸÐŒ ÐœÐ°ÑÑÑÐŸÐ¹ÐºÐž ÑÐ²ÑÐ·Ðž, Ð³ÐŽÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑÐ° ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÑ; Ðž "ÐÑÐžÑÑÐºÐ° ÐÑÑ", ÐºÐŸÑÐŸÑÐ°Ñ ÑÐŽÐ°Ð»ÑÐµÑ Ð²ÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ.
+      <Break/>
+      Ð¡Ð¿ÐžÑÐŸÐº Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÐžÐŒÐµÐµÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ Ð·Ð°Ð³ÑÑÐ¶Ð°ÐµÐŒÐŸÐ¹ Ð²ÐµÐ± ÑÑÑÐ°ÐœÐžÑÑ. ÐÐ°Ð¶ÐŽÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐŒÐµÐµÑ ÑÐµÐºÑÑ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÐžÐ¹ ÐŽÐµÑÐ°Ð»Ðž Ð·Ð°ÐŽÐ°ÑÐž Ð²ÐŒÐµÑÑÐµ Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐŸÑÐŸÐŒ Ð¿ÑÐŸÐ³ÑÐµÑÑÐ° ÐŽÐµÐŒÐŸÐœÑÑÑÐžÑÑÑÑÐžÐŒ ÑÐµÐºÑÑÑÑ ÐŽÐµÑÑÐµÐ»ÑÐœÐŸÑÑÑ. Ð¢ÑÐž ÐºÐœÐŸÐ¿ÐºÐž Ð¿ÐŸÑÐ²Ð»ÑÑÑÑÑ ÑÐ»ÐµÐ²Ð° ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ. "ÐÐ°ÑÐ·Ð°" ÐŽÐ»Ñ Ð¿ÑÐžÐŸÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÑ ÑÐµÐºÑÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÐµÑÑÐ°. "Ð¡ÐŒÐŸÑÑÐµÑÑ ÐÐŸÐ³Ðž" ÐŸÑÐºÑÑÐ²Ð°ÐµÑ ÐŸÐºÐœÐŸ ÑÐ°Ð¹Ð»Ð° Ð·Ð°Ð³ÑÑÐ·ÐºÐž. "ÐÐ°ÐºÑÑÑÑ" Ð·Ð°Ð²ÐµÑÑÐ°ÐµÑ Ð·Ð°Ð³ÑÑÐ·ÐºÑ Ð·Ð°ÐŽÐ°ÑÐž ÐžÐ· ÑÐ¿ÐžÑÐºÐ°.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+      <Reference target="preferences">Ð¡ÐµÐºÑÐžÑ ÐÐ°ÑÑÑÐŸÐµÐº</Reference> ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐºÐ°Ðº ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐžÐœÑÐµÑÐœÐµÑ ÑÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ ÑÐµÑÐµÐ· Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑ.  ÐÑÐ»Ðž ÐœÑÐ¶ÐœÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐ°Ñ Ð°ÑÑÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÑ, Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑ Ð·Ð°Ð¿ÑÐŸÑÐžÑ ÐžÐŒÑ Ðž Ð¿Ð°ÑÐŸÐ»Ñ.  ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ ÐœÐµ ÑÐŸÐ·ÑÐ°ÐœÑÐµÑ Ð¿Ð°ÑÐŸÐ»Ðž ÐŒÐµÐ¶ÐŽÑ ÑÐµÑÑÐžÑÐŒÐž.
+    </Section>
+  </Section>
+
+  <Section name="collectingfiles">
+    <Title>Ð¡Ð±ÐŸÑ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŽÐ»Ñ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>
+ ÐÑÐ»Ðž ÐÑ ÑÐŸÐ·ÐŽÐ°ÐµÑÐµ ÐœÐŸÐ²ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ,ÑÐŸ Ð²ÐŸÐ·ÐœÐžÐºÐ°ÐµÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÑ Ð²Ð²ÐŸÐŽÐ° Ð² ÐœÐµÐµ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÐœÐž ÐŒÐŸÐ³ÑÑ Ð¿ÐŸÑÑÑÐ¿ÐžÑÑ ÐžÐ· Ð²Ð°ÑÐµÐ³ÐŸ ÐŸÐ±ÑÑÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° ÐžÐ· ÐŽÑÑÐ³ÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Greenstone. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ ÑÐ¶Ðµ ÐžÐŒÐµÑÑ Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐµÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ.Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ.
+    <Section name="thegatherview">
+      <Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°ÐºÐŸÐ¿ÐžÑÐµÐ»Ñ</Title>
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» Ð²Ð²ÐŸÐŽÐžÑ Ð² ÐŸÐ±Ð»Ð°ÑÑÑ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ,ÐºÐŸÑÐŸÑÑÑ ÐÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÐµ,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ,ÐºÐ°ÐºÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð² ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÐµÐŒÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐ°Ðº Ð¿ÑÐŸÑÐŒÐ°ÑÑÐžÐ²Ð°ÑÑ ÑÐºÑÐ°Ðœ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ 
+            </ContentsItem>
+          <ContentsItem>
+             Ð€Ð°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ±Ð»Ð°ÑÑÑ ÑÐŸÑÑÐŸÑÐœÐžÑ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐœÐŸÐ¿ÐºÐž ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÐµÑÐŸÐŽÑ ÐŸÑÐ±ÐŸÑÐ°
+          </ContentsItem>
+          <ContentsItem>
+             Ð¡Ð¿ÐµÑÐžÐ°Ð»ÑÐœÐ°Ñ Ð¿Ð°Ð¿ÐºÐ° ÐŸÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐœÐ°ÑÐžÐœÐ°ÐµÑÑÑ Ñ ÐŸÐ±Ð·ÐŸÑÐ° ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ. Ð§ÑÐŸÐ±Ñ Ð²ÐµÑÐœÑÑÑÑÑ Ðº ÑÑÐŸÐŒÑ ÐŸÐ±Ð·ÐŸÑÑ Ð¿ÐŸÐ·Ð¶Ðµ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÑÐžÐŒÐ²ÐŸÐ»Ðµ "Gather", ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÑÐŒ ÐœÐµÐ¿ÐŸÑÑÐµÐŽÑÑÐ²ÐµÐœÐœÐŸ ÐœÐžÐ¶Ðµ Ð»ÐžÐœÐµÐ¹ÐºÐž ÐŒÐµÐœÑ.  
+<Break/>
+ÐÐ²Ðµ Ð±ÐŸÐ»ÑÑÐžÐµ ÐŸÐ±Ð»Ð°ÑÑÐž,ÐœÐ°Ð·Ð²Ð°ÐœÐœÑÐµ "Ð Ð°Ð±ÐŸÑÐµÐµ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²ÐŸ" Ðž "ÐoÐ»Ð»ÐµÐºÑÐžÑ" ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ,ÑÑÐŸÐ±Ñ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ ÑÐ°Ð¹Ð»Ñ Ð² ÐÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. ÐÐœÐž Ð²ÐºÐ»ÑÑÐ°ÑÑ "ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ",Ð³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÑÑÑÑÐºÑÑÑÑ,ÐºÐŸÑÐŸÑÑÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ Ðž Ð¿Ð°Ð¿ÐºÐž. 
+<Break/>
+ÐÑÐ±ÐµÑÐžÑÐµ ÑÐ»ÐµÐŒÐµÐœÑ ÐŽÐµÑÐµÐ²Ð°, ÐœÐ°Ð¶Ð°Ð² ÐœÐ° ÐœÐµÐ³ÐŸ Ð»ÐµÐ²ÐŸÐ¹ ÐºÐ»Ð°Ð²ÐžÑÐµÐ¹ ÐŒÑÑÐž. (Ð¢ÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ ÐŒÐŸÐ¶ÐœÐŸ ÑÐŽÐµÐ»Ð°ÑÑ ÐžÐœÐ°ÑÐµ, ÑÐŒ. ÐœÐžÐ¶Ðµ). ÐÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð°Ð¿ÐºÑ, ÐžÐ»Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° ÑÐžÐŒÐ²ÐŸÐ» ÑÑÐŽÐŸÐŒ Ñ ÐœÐµÐ¹, ÑÑÐŸÐ±Ñ ÑÐ°Ð·Ð²ÐµÑÐœÑÑÑ (ÐžÐ»Ðž ÑÐ²ÐµÑÐœÑÑÑ) ÐµÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÐŒÐŸÐµ. ÐÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ ÐœÐ° ÐœÑÐ¶ÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐµÐ³ÐŸ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÐŸÐ¹ ÐŽÐ°ÐœÐœÐŸÐŒÑ ÑÐžÐ¿Ñ ÑÐ°Ð¹Ð»ÐŸÐ² Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ (ÑÐŒ.<Reference target="fileassociations">ÐÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐµ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ</Reference>).
+<Break/>
+ÐÐµÑÐµÐ²ÐŸ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÑÑÐŸÑÐœÐžÐºÐž ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ Ð² ÐÐœÑÐµÑÑÐµÐ¹ÑÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ -- ÑÐµÐºÑÑÑÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÑÑ ÑÐžÑÑÐµÐŒÑ (Ð²ÐºÐ»ÑÑÐ°Ñ ÐŽÐžÑÐºÐŸÐ²ÐŸÐŽ Ðž CD-ROM), ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐÑÐžÐœÑÑÐŸÑÐœ, Ðž ÐºÑÑ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², Ð¿ÑÐž ÑÑÐ»ÐŸÐ²ÐžÐž Ð²ÐºÐ»ÑÑÐµÐœÐžÑ Ð²ÐµÐ± Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐžÐŽÐµÑÑ Ðž ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÑÐ°Ð¹Ð»Ñ, ÐŸÐŽÐœÐ°ÐºÐŸ ÐœÐµ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐœÐŸÑÐžÑÑ, ÑÐŽÐ°Ð»ÑÑÑ, ÐžÐ»Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÑ
+, Ð·Ð° ÐžÑÐºÐ»ÑÑÐµÐœÐžÐµÐŒ ÑÐ¶Ðµ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ.  ÐÑÑÑÐµÑÑÐ²Ð»ÑÑÑ ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÑ Ð² ÑÑÐŸÐŒ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ðµ ÑÑÐŸÐ±Ñ ÐœÐ°Ð¹ÑÐž ÑÐ°Ð¹Ð»Ñ ÐºÐŸÑÐŸÑÑÐµ Ð²Ñ Ñ
+ÐŸÑÐžÑÐµ Ð²ÐºÐ»ÑÑÐžÑÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+<Break/>
+Ð€Ð°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÑÐµÑ Ð¿ÐŸÐ»ÐœÐŸÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ ÐŸÐœÐŸ Ð¿ÑÑÑÐŸ.
+<Break/>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÑÐ°Ð·ÐŒÐµÑÑ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð°, Ð¿ÐµÑÐµÐŽÐ²ÐžÐ³Ð°Ñ ÐŒÑÑÑ Ð¿ÐŸ ÑÐµÑÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ, ÐºÐŸÑÐŸÑÐ°Ñ ÐŸÑÐŽÐµÐ»ÑÐµÑ ÐŽÐµÑÐµÐ²ÑÑ (ÑÐŸÑÐŒÐ° ÑÐºÐ°Ð·Ð°ÑÐµÐ»Ñ ÐžÐ·ÐŒÐµÐœÑÐµÑÑÑ),Ðž Ð±ÑÐºÑÐžÑÑÑ ÐžÑ
+.
+<Break/>
+ÐÐœÐžÐ·Ñ ÐŸÐºÐœÐ° ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐ° ÑÑÐ°ÑÑÑÐœÐ°Ñ ÐŸÐ±Ð»Ð°ÑÑÑ, ÐºÐŸÑÐŸÑÐ°Ñ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð¿ÑÐŸÐ³ÑÐµÑÑ ÐŸÐ¿ÐµÑÐ°ÑÐžÐ¹ ÐœÐ°ÐŽ ÑÐ°Ð¹Ð»Ð°ÐŒÐž (ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐµ, Ð¿ÐµÑÐµÐŒÐµÑÐµÐœÐžÐµ, ÑÐŽÐ°Ð»ÐµÐœÐžÐµ). ÐÐœÐž ÐŒÐŸÐ³ÑÑ Ð·Ð°ÐœÑÑÑ ÐºÐ°ÐºÐŸÐµ-ÑÐŸ Ð²ÑÐµÐŒÑ. ÐÐœÐŸÐ¿ÐºÐ° "Ð¡ÑÐŸÐ¿" ÐŸÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑ Ð»ÑÐ±ÐŸÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ Ð¿ÑÐŸÐžÑÑ
+ÐŸÐŽÐžÑ. 
+<Break/>
+ ÐÐ²Ðµ Ð±ÐŸÐ»ÑÑÐžÐµ ÐºÐœÐŸÐ¿ÐºÐž Ð·Ð°ÐœÐžÐŒÐ°ÑÑ ÐœÐžÐ¶ÐœÐžÐ¹ Ð¿ÑÐ°Ð²ÑÐ¹ ÑÐ³ÐŸÐ» ÑÐºÑÐ°ÐœÐ°."ÐÐŸÐ²Ð°Ñ ÐÐ°Ð¿ÐºÐ°",Ñ ÐºÐ°ÑÑÐžÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž, ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÐµ Ð¿Ð°Ð¿ÐºÐž (ÑÐŒ. <Reference target="creatingfolders">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº</Reference>). ÐÐœÐŸÐ¿ÐºÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ"ÑÐŽÐ°Ð»ÑÐµÑ ÑÐ°Ð¹Ð»Ñ Ñ ÐœÐµÐœÑÐ¶ÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹.Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ" ÑÐŽÐ°Ð»ÐžÑ Ð»ÑÐ±ÑÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐ»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÐŸ, ÑÐ°Ð¹Ð»Ñ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ Ð¿ÑÑÐµÐŒ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ ÐžÑ
+ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÑ "Ð£ÐŽÐ°Ð»ÐžÑÑ". 
+       <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÐŸÐ±ÑÐ°ÑÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ Ð¿ÐŸÑÐ»ÐµÐŽÐŸÐ²Ð°ÑÐµÐ»ÑÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ²,Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿ÐµÑÐ²ÑÐ¹ Ðž Ð·Ð°ÑÐµÐŒ ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ñ [Shift],ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð¿ÐŸÑÐ»ÐµÐŽÐœÐµÐŒ - Ð¿ÑÐž ÑÑÐŸÐŒ Ð±ÑÐŽÑÑ ÐŸÑÐŸÐ±ÑÐ°ÐœÑ Ð²ÑÐµ ÐŸÑ
+Ð²Ð°ÑÐµÐœÐœÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ.ÐÑÐ±ÐŸÑ ÐœÐµ ÑÐ»ÐµÐŽÑÑÑÐžÑ
+ ÐŸÐŽÐžÐœ Ð·Ð° ÐŽÑÑÐ³ÐžÐŒ ÑÐ°Ð¹Ð»ÐŸÐ² Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ Ð¿ÑÐž ÐœÐ°Ð¶Ð°ÑÐŸÐ¹ ÐºÐ»Ð°Ð²ÐžÑÐµ [Ctrl] Ðž ÑÐµÐ»ÑÐºÐµ ÐœÐ° Ð²ÑÐ±ÐžÑÐ°ÐµÐŒÑÑ
+ ÑÐ°Ð¹Ð»Ð°Ñ
+. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐž ÐŽÐ²Ð° ÐŒÐµÑÐŸÐŽÐ° Ð²ÐŒÐµÑÑÐµ ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿Ñ ÐœÐµÑÐŒÐµÐ¶ÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ².  
+<Break/>
+ ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿Ð°Ð¿ÐºÐž - ÑÐžÐ¿Ð° ÑÐµÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÐÐ°ÑÐž ÑÐŸÐ±ÑÑÐ²ÐµÐœÐœÑÐµ Ð²ÐµÐ±-ÑÑÑÐ°ÐœÐžÑÑ - ÐžÐœÐŸÐ³ÐŽÐ° ÐžÐŒÐµÑÑ ÐŸÑÐŸÐ±ÑÑ Ð²Ð°Ð¶ÐœÐŸÑÑÑ.ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÐµÑÐµÐœÐµÑÑÐž ÑÐ°ÐºÐžÐµ Ð¿Ð°Ð¿ÐºÐž ÐœÐ° Ð¿ÐµÑÐ²ÑÐ¹ ÑÑÐŸÐ²ÐµÐœÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð°. Ð§ÑÐŸÐ±Ñ ÑÐŽÐµÐ»Ð°ÑÑ ÑÑÐŸ, ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ ÐŒÑÑÐž ÐœÐ° Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ. ÐÑÐ±ÐµÑÐžÑÐµ "Ð¡ÐŸÐ·ÐŽÐ°ÑÑ ÑÐŸÐºÑÐ°ÑÐµÐœÐžÐµ" Ðž Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ Ð¿Ð°Ð¿ÐºÐž. ÐÐ»Ñ ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ ÐŒÑÑÐž ÐœÐ° Ð¿ÐŸÐŒÐµÑÐµÐœÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐŸÐºÑÐ°ÑÐµÐœÐžÑ".
+    </Section>
+    <Section name="creatingfolders">
+      <Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÑÑ ÐœÐŸÐ²ÑÐµ Ð¿Ð°Ð¿ÐºÐž
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐœÐŸÐ¿ÐºÐ° ÐœÐŸÐ²ÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐµÐœÑ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐž ÐŒÑÑÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ "ÐÐ°Ð¿ÐºÐž" Ð² ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐŒ ÐŽÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð²ÐŒÐµÑÑÐµ Ðž Ð±ÐŸÐ»ÐµÐµ Ð»ÐµÐ³ÐºÐŸ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑ ÐžÑ
+. ÐÐ°Ð¿ÐºÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð¿ÐŸÐŒÐµÑÐµÐœÑ Ð²ÐœÑÑÑÐž Ð¿Ð°Ð¿ÐŸÐº. Ð€Ð°ÐºÑÐžÑÐµÑÐºÐž ÐœÐµÑ ÐŸÐ³ÑÐ°ÐœÐžÑÐµÐœÐžÑ ÐœÐ° ÑÐŸ,ÑÐºÐŸÐ»ÑÐºÐŸ Ð¿Ð°Ð¿ÐŸÐº ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐŒÐµÑÑ ÐžÐ»Ðž ÐºÐ°Ðº Ð³Ð»ÑÐ±ÐŸÐºÐŸ ÐŸÐœÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð²Ð»ÐŸÐ¶ÐµÐœÑ.
+<Break/>
+  Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÑ Ð¿Ð°Ð¿ÐºÑ, Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐ»ÑÐœÐŸ Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÑÑ Ð¿Ð°Ð¿ÐºÑ Ð² ÐÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°". ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÐžÐ»Ðž ÐœÐ° Ð²ÑÑÑÐµÐŒ ÑÑÐŸÐ²ÐœÐµ,ÐµÑÐ»Ðž ÐœÐž ÐŸÐŽÐœÐ° ÐœÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐ°.ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÑÐŸÑÐ²ÐŸÐžÑÑ Ð¿Ð°Ð¿ÐºÐµ ÐžÐŒÑ (Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°").
+<Break/>
+ ÐÐ°Ð¿ÐºÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐ°ÐºÐ¶Ðµ ÑÐŸÐ·ÐŽÐ°ÐœÑ ÑÐµÐ»ÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐž ÐŒÑÑÐž Ð¿ÐŸ Ð¿Ð°Ð¿ÐºÐµ,Ð²ÑÐ±ÐžÑÐ°Ñ "ÐÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ°" Ðž ÐŽÐµÐ¹ÑÑÐ²ÑÑ,ÐºÐ°Ðº ÑÐºÐ°Ð·Ð·Ð°ÐœÐŸ Ð²ÑÑÐµ.
+    </Section>
+    <Section name="addingfiles">
+      <Title>ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÐŸ,ÐºÐ°Ðº Ð²Ð²ÐµÑÑÐž ÑÐ°Ð¹Ð»Ñ Ð² ÐÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ </ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐµÑÐµÐŒÐµÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÐµ(ÑÐŸÑÑÐ°Ð²ÐœÑÐµ) ÑÐ°Ð¹Ð»Ñ
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             Ð¡ÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð€Ð°Ð¹Ð»Ñ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŒÑÑÑÑ Ð¿ÑÑÐµÐŒ ÐžÑ
+ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ Ðž ÐŸÑÐ¿ÑÑÐºÐ°ÐœÐžÑ. Ð£ÐºÐ°Ð·Ð°ÑÐµÐ»Ñ ÐŒÑÑÐž ÐŸÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑÑÑ ÐœÐ°ÐŽ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐŒ ÐžÐ»Ðž, ÐµÑÐ»Ðž Ð²ÑÐ±ÐžÑÐ°ÐµÑÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐžÐœ,ÐœÐ°ÐŽ ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÐŒÐž. ÐÑÐŸÐ¿ÑÑÑÐžÑÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» Ð² ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð·ÐŽÐµÑÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ (ÐµÑÐ»Ðž ÐžÑÑÐŸÑÐœÐžÐºÐŸÐŒ Ð±ÑÐ»ÐŸ ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž).
+<Break/>
+  ÐÑÐž ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÑ
+(ÑÐŸÑÑÐ°Ð²ÐœÑÑ
+) ÑÐ°Ð¹Ð»ÐŸÐ²,Ð²ÑÐµ ÐŸÐœÐž Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑÑÑ Ð² ÑÐµÐ»ÐµÐ²ÑÑ Ð¿Ð°Ð¿ÐºÑ ÐœÐ° ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ ÑÑÐŸÐ²ÐµÐœÑ,ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÐŸ ÐŸÑ ÑÑÑÑÐºÑÑÑÑ Ð¿Ð°Ð¿ÐºÐž,ÐºÐŸÑÐŸÑÑÑ ÐŸÐœÐž Ð·Ð°ÐœÐžÐŒÐ°Ð»Ðž Ð¿ÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐºÐŸÐ¿ÐžÑÑÐµÑÐµ Ð²ÑÐŸÑÐŸÐ¹ ÑÐ°Ð¹Ð» Ñ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² ÑÑ Ð¶Ðµ ÑÐ°ÐŒÑÑ Ð¿Ð°Ð¿ÐºÑ, ÐÐ°Ñ Ð·Ð°Ð¿ÑÐŸÑÑÑ, Ð·Ð°ÑÐµÑÐµÑÑ Ð»Ðž Ð¿ÐµÑÐ²ÑÐ¹. ÐÑÐ²ÐµÑÑÑÐµ "ÐÐµÑ" Ðž ÑÑÐŸÑ ÑÐ°Ð¹Ð» ÐœÐµ Ð±ÑÐŽÐµÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑÑÑ,ÐœÐŸ ÐŽÑÑÐ³ÐžÐµ Ð±ÑÐŽÑÑ. Ð§ÑÐŸÐ±Ñ ÐŸÑÐŒÐµÐœÐžÑÑ Ð²ÑÐµ ÐŸÑÑÐ°ÑÑÐžÐµÑÑ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ Ð¿ÐŸ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "Ð¡ÑÐŸÐ¿".   <Break/>
+ Ð¢ÐŸÐ»ÑÐºÐŸ "ÑÐ°ÐŒÑÐµ Ð²ÑÑÐŸÐºÐžÐµ"ÐžÐ· ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² Ð±ÑÐŽÑÑ Ð¿ÐµÑÐµÐŒÐµÑÐµÐœÑ. ÐÐ°Ð¿ÐºÐ° Ð²ÑÑÐµ, ÑÐµÐŒ ÐµÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐœÑÐµ. ÐÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ ÑÐ°Ð¹Ð»Ñ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð¿Ð°Ð¿ÐºÐž,Ð° ÑÐ°ÐºÐ¶Ðµ ÑÐ°ÐŒÑ Ð¿Ð°Ð¿ÐºÑ ÐœÐµÐ¿ÐŸÑÑÐµÐŽÑÑÐ²ÐµÐœÐœÐŸ. 
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÑÐ°Ð¹Ð», ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑ Ð¿ÐŸÐžÑÐº Ð² ÐžÑÑ
+ÐŸÐŽÐœÑÑ
+ Ð¿Ð°Ð¿ÐºÐ°Ñ
+ Ð²ÑÐ¿ÐŸÐŒÐŸÐ³Ð°ÑÐµÐ»ÑÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÐŒÐŸÐŒÑ ÑÐ°Ð¹Ð»Ñ,Ðž ÐµÑÐ»Ðž ÐŸÐœ ÐœÐ°Ñ
+ÐŸÐŽÐžÑ ÐµÐ³ÐŸ,ÑÐŸ ÐœÐ°ÑÐžÐœÐ°ÐµÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÐŽÐ°ÐœÐœÑÐµ. ÐÑÐ»Ðž Ð¿ÑÐŸÑÐµÑÑ Ð¿ÑÐŸÐŽÐŸÐ»Ð¶Ð°ÐµÑÑÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð±ÑÑÑ Ð·Ð°Ð¿ÑÐŸÑÐµÐœÑ (Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ°Ð·) ÐŸ ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž,ÑÑÐŸÐ±Ñ ÐŽÐŸÑÑÐžÑÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÐµÐŒÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž,ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐŒÐž Ð² ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð­ÑÐŸÑ Ð¿ÑÐŸÑÐµÑÑ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐŒÐœÐŸÐ³ÐžÐµ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ Ð·Ð°Ð¿ÑÐŸÑÑ, ÐŸÐ¿ÐžÑÐ°ÐœÐœÑÐµ Ð² ÑÐ°Ð·ÐŽÐµÐ»Ðµ<Reference target="importingpreviouslyassignedmetadata">ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Reference>.ÐÐ»Ñ Ð±ÐŸÐ»ÐµÐµ ÐŽÐµÑÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÐŸÐ±ÑÑÑÐœÐµÐœÐžÑ Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÑ
+ Ñ ÑÐ°Ð¹Ð»Ð°ÐŒÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐžÑÐ°Ð¹ÑÐµ ÐÐ»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone - ÐÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð±ÐŸÐ»ÑÑÐžÐœÑÑÐ²Ð° Ð²Ð°ÑÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ².  
+     </Section>
+    <Section name="removingfiles">
+      <Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÐŽÐ°Ð»ÑÑÑ ÑÐ°Ð¹Ð»Ñ Ðž Ð¿Ð°Ð¿ÐºÐž ÐžÐ· ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.  
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐœÐŸÐ¿ÐºÐ° ÑÐŽÐ°Ð»ÐµÐœÐžÑ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ»Ð°Ð²ÐžÑÐ° ÑÐŽÐ°Ð»ÐµÐœÐžÑ
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð¿ÑÑÐµÐŒ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ Ðž ÐŸÑÐ¿ÑÑÐºÐ°ÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð¡ÑÑÐµÑÑÐ²ÑÐµÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐŒÐµÑÐŸÐŽÐŸÐ² ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž Ð¿Ð°Ð¿ÐŸÐº. ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÑÐœÐ°ÑÐ°Ð»Ð° ÑÐºÐ°Ð·Ð°ÑÑ,ÐºÐ°ÐºÐžÐµ ÐžÐ· ÐœÐžÑ
+ ÑÐŽÐ°Ð»ÑÑÑ, Ð²ÑÐ±ÐžÑÐ°Ñ ÐŸÐŽÐžÐœ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž Ð¿Ð°Ð¿ÐŸÐº, ÐºÐ°Ðº ÐŸÐ¿ÐžÑÐ°ÐœÐŸ Ð² <Reference target="thegatherview">ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÐ¹</Reference>.
+<Break/>
+ ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÐ°Ð¹Ð»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÑ, ÐŽÐ»Ñ ÐžÑ
+ ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐµÐ»ÐºÐ°ÑÑ ÐºÐœÐŸÐ¿ÐºÑ "Ð£ÐŽÐ°Ð»ÐžÑÑ"ÐžÐ»Ðž ÐœÐ°Ð¶ÐžÐŒÐ°ÑÑ ÐºÐºÐ»Ð°Ð²ÐžÑÑ [Ð£ÐŽÐ°Ð»ÐžÑÑ] ÐœÐ° ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÐµ, ÐžÐ»Ðž Ð¶Ðµ ÑÑÐœÑÑ ÐžÑ
+ ÐžÐ· ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŽÐŸ ÐºÐœÐŸÐ¿ÐºÐž "Ð£ÐŽÐ°Ð»ÐžÑÑ" Ðž ÑÐ°ÐŒ ÐŸÑÐ¿ÑÑÐºÐ°ÑÑ.  
+    </Section>
+    <Section name ="filteringthetree">
+      <Title>Ð€ÐžÐ»ÑÑÑÐŸÐ²Ð°ÐœÐžÐµ ÐŽÐµÑÐµÐ²Ð°</Title>
+"Ð€ÐžÐ»ÑÑÑÐŸÐ²Ð°ÐœÐžÐµ" ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÑÑÐ¶Ð°ÑÑ Ð¿ÐŸÐžÑÐº ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐžÐ»ÑÑÑÐŸÐŒ
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+              ÐÐ°ÑÑÑÐŸÐ¹ÐºÐ° ÑÐžÐ»ÑÑÑÐ°
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents >
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÑÐœÐºÑ "ÐÐŸÐºÐ°Ð·Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ" Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÐŒÐµÐœÑ Ð¿ÐŸÐŽ ÐºÐ°Ð¶ÐŽÑÐŒ ÐŽÐµÑÐµÐ²ÐŸÐŒ Ð²ÑÐ²ÐŸÐŽÐžÑ ÑÐ¿ÐžÑÐŸÐº Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÑ
+ ÑÐžÐ»ÑÑÑÐŸÐ² ÑÐžÐ¿Ð° "Images".ÐÑÐ±ÐµÑÐžÑÐµ Ðž Ð²ÑÐµ Ð²ÑÐµÐŒÐµÐœÐœÐŸ ÑÐºÑÑÑÑÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÐŽÐµÑÐµÐ²Ðµ.Ð§ÑÐŸÐ±Ñ Ð²ÐŸÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐŽÐµÑÐµÐ²ÐŸ,Ð·Ð°ÐŒÐµÐœÐžÑÐµ ÑÐžÐ»ÑÑÑ ÑÐœÐŸÐ²Ð° ÐœÐ° "ÐÑÐµ ÑÐ°Ð¹Ð»Ñ".Ð­ÑÐž ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÐœÐµ ÐžÐ·ÐŒÐµÐœÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð¿ÑÐž ÑÑÐŸÐŒ ÐŸÐœÐž ÐœÐµ Ð·Ð°ÑÑÐ°Ð³ÐžÐ²Ð°ÑÑ Ð¿Ð°Ð¿ÐºÐž Ð² ÐŽÐµÑÐµÐ²Ðµ. 
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°ÐŽÐ°ÑÑ ÐœÐ°ÑÑÑÐŸÐ¹ÐºÑ ÑÐžÐ»ÑÑÑÐ°,ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°Ð² Ð² ÑÑ
+ÐµÐŒÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐ°ÐœÐŽÐ°ÑÑÐœÑÐµ Ð°Ð±Ð±ÑÐµÐ²ÐžÐ°ÑÑÑÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ¹ ÑÐžÑÑÐµÐŒÑ ÑÐžÐ¿Ð° "*.*" ÐžÐ»Ðž "*.doc" ("*" ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð»ÑÐ±ÑÐŒ ÑÐžÐŒÐ²ÐŸÐ»Ð°ÐŒ). 
+     </Section>
+  </Section>
+
+  <Section name="enrichingacollection">
+    <Title>ÐÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž</Title>
+ ÐÐŒÐµÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐœÐ°Ð¿ÐŸÐ»ÐœÐžÑÐµ ÐžÑ
+ ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹,ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐ¹ "ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ".Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÑÑÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,ÐºÐ°Ðº ÐŸÐœÐž ÑÐµÐŽÐ°ÐºÑÐžÑÑÑÑÑÑ, ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑÑÑ Ðž Ð²ÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÑÑ,Ð° ÑÐ°ÐºÐ¶Ðµ ÐºÐ°Ðº ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ Ð²ÐœÐµÑÐœÐžÐµ ÐžÑÑÐŸÑÐœÐžÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+    
+ (ÑÐ°ÐºÐ¶Ðµ ÑÐŒ.ÐÐ»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone - ÐÐŸÐ»ÑÑÐµÐœÐžÐµ Ð±ÐŸÐ»ÑÑÐµÐ¹ ÑÐ°ÑÑÐž ÐžÐ· Ð²Ð°ÑÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²). 
+    <Section name="theenrichview">
+      <Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ</Title>
+<ObviousSentence>T Ð ÑÑÐŸÐž ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº Ð²Ð²ÐŸÐŽÐžÑÑ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ.
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐŸÐœÑÑÐŸÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐœÐžÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹
+          </ContentsItem>
+          <ContentsItem>
+             Ð¢Ð°Ð±Ð»ÐžÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ ÐŽÐ»Ñ Ð·Ð°ÐŽÐ°ÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. 
+MeÑÐ°ÐŽÐ°ÐœÐœÑÐµ - ÑÑÐŸ ÐŽÐ°ÐœÐœÑÐµ ÐŸ ÐŽÐ°ÐœÐœÑÑ
+ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ,ÐÐ²ÑÐŸÑ,ÐÐ°ÑÐ° ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ Ðž Ñ.ÐŽ. ÐÐ°Ð¶ÐŽÑÐ¹ Ð¿ÑÐœÐºÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐŸÑÑÐŸÐžÑ ÐžÐ· ÐŽÐ²ÑÑ
+ ÑÐ°ÑÑÐµÐ¹: "ÑÐ»ÐµÐŒÐµÐœÑ"- Ðº ÐºÐ°ÐºÐŸÐŒÑ ÑÐžÐ¿Ñ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŸÐœ ÐŸÑÐœÐŸÑÐžÑÑÑ (Ðº Ð¿ÑÐžÐŒÐµÑÑ,ÐÐ²ÑÐŸÑ),Ðž "Ð·ÐœÐ°ÑÐµÐœÐžÐµ"- ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÑÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ Ð»ÑÐ±ÑÑ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐžÐ»Ðž Ð¿Ð°Ð¿ÐŸÐº Ð² ÐÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð¡ÑÐŸÐ»Ð±ÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ ÑÐµÑÑÐŒ ÑÐ²ÐµÑÐŸÐŒ Ð²Ð²ÐµÑÑ
+Ñ Ðž ÐžÑ
+ ÑÐ°Ð·ÐŒÐµÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÑÐœ Ð¿ÑÑÑÐŒ Ð¿ÐµÑÐµÐŽÐ²ÐžÐ¶ÐµÐœÐžÑ Ð»ÐžÐœÐžÐž ÑÐ°Ð·ÐŽÐµÐ»Ð°. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° Ð»ÑÐ±ÐŸÐ¹ ÑÑÑÐŸÐºÐµ Ð¿ÐµÑÐµÐœÐŸÑÐžÑ ÐµÑ ÐŽÐµÑÐ°Ð»Ðž Ð²ÐœÐžÐ· ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐ°Ð¶Ð°Ð² ÐœÐ° Ð»ÑÐ±ÐŸÐ¹ ÑÑÐŽ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ, Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ðž ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐµÑÐ°Ð»Ðž ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð² ÐœÐžÐ¶ÐœÐµÐ¹ ÑÐµÐºÑÐžÐž ÐŸÐºÐœÐ°. Ð§ÐµÑÐœÑÐ¹ ÑÐµÐºÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐ±ÑÐžÐŒ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²,ÑÐŸÐ³ÐŽÐ° ÐºÐ°Ðº ÑÐµÑÑÐ¹ ÑÐµÐºÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ,ÑÑÐŸ ÑÑÐŸ - ÐœÐµ ÑÐ°Ðº.Ð§ÐµÑÐœÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐµÐœÑ, Ð² ÑÐŸ Ð²ÑÐµÐŒÑ ÐºÐ°Ðº ÑÐµÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ ÐžÐ· ÑÐµÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐµÑÑ ÐžÑ
+,ÐžÐ»Ðž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ðº ÐŽÑÑÐ³ÐžÐŒ.  
+ <Break/>
+ ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐºÐž ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÑÐ²Ð»ÑÑÑÑÑ ÐŸÐºÐŸÐ»ÐŸ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ Ð²Ñ
+ÐŸÐŽÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.Ð­ÑÐŸ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐœÐ° ÑÐŸ,ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÑÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÑ ÐŸÑ Ð¿Ð°Ð¿ÐŸÐº ÑÐŸÐŽÐžÑÐµÐ»ÐµÐ¹ (ÐžÐ»Ðž Ð¿ÑÐµÐŽÐºÐŸÐ²). Ð£ÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐœÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐµÐœÑ, Ð° ÑÐŸÐ»ÑÐºÐŸ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° ÐžÐ»Ðž Ð¿ÐµÑÐµÐ¿ÐžÑÐ°ÐœÑ.Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐž Ð¿Ð°Ð¿ÐºÐž,ÑÑÐŸÐ±Ñ ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð¿ÐµÑÐµÐ¹ÑÐž Ð² ÑÑ Ð¿Ð°Ð¿ÐºÑ, Ð³ÐŽÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð·Ð°ÐŽÐ°ÐœÑ.  
+<Break/>
+ÐÐŸÐœÑÑÐŸÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐœÐžÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÐœÐžÐ·Ñ ÑÐ¿ÑÐ°Ð²Ð° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐµÑÐ»Ðž ÑÐ°Ð¹Ð» Ð²ÑÐ±ÑÐ°Ðœ ÐžÐ· ÐŽÐµÑÐµÐ²Ð°, Ð° ÑÑÐŽ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐžÑ
+, ÑÑÐŸÐ±Ñ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ, ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐœÐ°ÑÐµÐœÐžÐµ Ð¿ÐŸÐ»Ñ Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐŸ ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ Ð²Ð²ÐµÑÑÐž ÐžÐ»Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐºÐŸÐ»ÐŸ ÐœÐµÐ³ÐŸ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐ°Ñ ÐºÐœÐŸÐ¿ÐºÐ°, ÐºÐŸÑÐŸÑÐ°Ñ Ð¿ÑÐž ÑÐµÐ»ÑÐºÐµ ÐŸÑÐºÑÑÐ²Ð°ÐµÑ Ð±ÐŸÐ»ÑÑÐµÐµ Ð¿ÐŸÐ»Ðµ ÐŽÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÑ. Ð Ð°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÐ°Ñ ÐœÐžÐ¶Ðµ ÐºÐœÐŸÐ¿ÐºÐ° "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ" Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðž ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÑ
+ Ðº ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ ÐŽÐ»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ°. ÐÐœÐŸÐ¿ÐºÐ° "ÐÐ°ÐŒÐµÐœÐžÑÑ" Ð¿ÑÐžÐŒÐµÐœÑÐµÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ðº ÐœÐŸÐ²ÑÐŒ, Ð° "Ð£ÐŽÐ°Ð»ÐžÑÑ" ÑÑÐžÑÐ°ÐµÑ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ.ÐÐœÐžÐ·Ñ Ð¿ÑÐœÐºÑÐ°,ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐŸÐ³ÐŸ ÐºÐ°Ðº "ÐÑÐµ Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ", ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ "ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹".  
+<Break/> 
+ ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐŒÐŸÐ¶ÐµÑ ÑÐ°ÑÑÐžÑÑÑÑÑÑ Ðž ÑÐ°Ð·ÑÑÑÐ°ÑÑÑÑ. ÐÐ±ÑÑÐœÐŸ ÑÑÐŸ - ÑÐ¿ÐžÑÐŸÐº, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, Ð²Ð²ÐµÐŽÐµÐœÐœÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÐŽÐ»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ°. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° Ð²Ñ
+ÐŸÐŽÐµ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÑÐ°Ð·ÐŒÐµÑÐ°ÐµÑ ÐµÐ³ÐŸ Ð² Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. Ð¡ ÐŽÑÑÐ³ÐŸÐ¹ ÑÑÐŸÑÐŸÐœÑ,Ð¿ÑÐž ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°ÐœÐœÐŸÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÐž Ð² ÑÐµÐºÑÑÐŸÐ²ÐŸÐŒ Ð¿ÐŸÐ»Ðµ Ð²ÑÐ±ÐŸÑÐºÐž ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐÐµÑÐµÐ²Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÑÐžÑÑÐµÐŒÐ° ÐœÐ°ÑÐžÐœÐ°ÐµÑ ÑÐ°Ð±ÐŸÑÑ Ñ ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐÑ Ð²Ð²ÐµÐ»Ðž. ÐÐ°Ð¶Ð°ÑÐžÐµ [Tab] Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð·Ð°Ð²ÐµÑÑÐ°ÐµÑ Ð¿ÐµÑÐ°ÑÐ°ÐœÐžÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ.
+ <Break/>
+ÐÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÑÐ³Ð°ÐœÐžÐ·ÐŸÐ²Ð°ÐœÑ Ð² ÐžÐµÑÐ°ÑÑ
+ÐžÑ. Ð­ÑÐŸ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÐŸ Ð² ÐÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð¿Ð°Ð¿ÐºÐž ÐŽÐ»Ñ Ð²ÐœÑÑÑÐµÐœÐœÐžÑ
+ ÑÑÐŸÐ²ÐœÐµÐ¹. ÐÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð²Ð²ÐµÐŽÐµÐœÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð·ÐœÐ°Ðº "|" ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ ÑÑÐŸÐ²ÐœÐµÐ¹. ÐÐ°Ð¿ÑÐžÐŒÐµÑ,"Cards|Red|Diamonds|Seven" ÐŒÐŸÐ³ Ð±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð² ÐžÐµÑÐ°ÑÑ
+ÐžÐž, Ð¿ÑÐžÐŒÐµÑÐŸÐŒ ÐºÐŸÑÐŸÑÐŸÐ¹ ÑÐ²Ð»ÑÐµÑÑÑ Ð¿Ð°ÐºÐµÑ ÐžÐ³ÑÑ Ð² ÐºÐ°ÑÑÑ.Ð­ÑÐŸ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ð²ÐŒÐµÑÑÐµ. ÐÑÑÐ¿Ð¿Ñ ÐŒÐŸÐ³ÑÑ ÑÐ°ÐºÐ¶Ðµ ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑÑÑ ÐºÐ°Ðº ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº ÑÐ°Ð¹Ð»Ð°ÐŒ.  
+<Break/> 
+ Greenstone Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ð² ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÐ»ÐµÐŒÐµÐœÑÑ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐŒÐµÑÑ Ð¿ÑÐµÑÐžÐºÑ "ex.". Ð­ÑÐŸ ÐœÐµ ÐžÐŒÐµÐµÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐµÑÐµÐ²Ð° Ðž ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ,Ð¿ÐŸÑÐŸÐŒÑ ÑÑÐŸ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÑÐºÑÑÑÐŸ,ÐµÑÐ»Ðž ÑÐ°ÐºÐŸÐ¹ ÑÐžÐ¿ Ð²Ð²ÐŸÐŽÐ° Ð²ÑÐ±ÑÐ°Ðœ.ÐÐœÐŸÐ¿ÐºÐ° "..." Ð²ÑÑ ÐµÑÑ ÑÐ»ÑÐ¶ÐžÑ ÐŽÐ»Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,ÐœÐŸ ÑÐµÐºÑÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°Ðœ. 
+    </Section>
+    <Section name="selectingmetadatasets">
+      <Title>ÐÑÐ±ÐŸÑ ÐœÐ°Ð±ÐŸÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence>Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ»ÐµÐŒÐµÐœÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðž ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Ð§ÑÐŸ ÑÐ°ÐºÐŸÐµ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+?
+          </ContentsItem>
+          <ContentsItem>
+	    ÐÑÐ±ÑÐ°Ðœ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ·Ð²ÐµÑÑÐœÐ° ÐºÐ°Ðº "ÐœÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+". ÐÑÐžÐŒÐµÑÐŸÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÐºÐŸÐŒÐ¿Ð»ÐµÐºÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÐŸ ÐµÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÑ ÑÑÐ°ÐœÐŸÐ²ÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐŽÐ»Ñ Ð²ÑÐ±ÐŸÑÐ°. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐŒÐµÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐžÐœ ÐœÐ°Ð±ÐŸÑ. ÐÐ»Ñ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐµÐœÐžÑ ÑÑÐŸÐ»ÐºÐœÐŸÐ²ÐµÐœÐžÑ ÐŸÐŽÐžÐœÐ°ÐºÐŸÐ²ÑÑ
+ ÐžÐŒÑÐœ Ðº ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÑÐµÑÑÑ ÐºÐŸÑÐŸÑÐºÐžÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ,ÐºÐŸÑÐŸÑÑÐ¹ Ð²ÑÐŽÐµÐ»ÑÐµÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ»ÐµÐŒÐµÐœÑ Dublin Core "Creator" ÑÑÐ°ÐœÐµÑ "dc.Creator".ÐÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ
+ÑÐ°ÐœÑÑÑÑ Ð² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Ðž ÐžÐŒÐµÑÑ ÑÑÑÑÐžÐºÑ ".mds".
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÐŸÐ²Ð°ÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÑ
+ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+" Ð² Ð¿ÑÐŸÑÐŒÐŸÑÑÐµ "ÐÐžÐ·Ð°Ð¹Ðœ".
+    </Section>
+    <Section name="appendingmetadata">
+      <Title>ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence>Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðž ÐžÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ðº ÑÐ°Ð¹Ð»Ð°ÐŒ Ðž ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐ»Ñ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐŸÐ¿ÐŸÐ»ÐœÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐŸÐ¿ÐŸÐ»ÐœÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÑÐŸÑÑÐ°Ð²ÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÐœÐ°ÑÐµÐœÐžÐµ ÐžÐµÑÐ°ÑÑ
+ÐžÐ¹ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð¢ÐµÐ¿ÐµÑÑ ÐŒÑ ÐŽÐŸÐ±Ð°Ð²ÐžÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ - Ðž ÑÐ°ÐŒ ÑÐ»ÐµÐŒÐµÐœÑ,Ðž ÐµÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ - Ðº ÑÐ°Ð¹Ð»Ñ. Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐµÐŒ ÑÐ°Ð¹Ð» ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð¢Ð°ÐºÐŸÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ Ð²ÑÐ·ÑÐ²Ð°ÐµÑ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÐµ Ð»ÑÐ±ÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÑÐ²ÑÐ·Ð°ÐœÐœÑÑ
+ Ñ ÑÑÐžÐŒ ÑÐ°Ð¹Ð»ÐŸÐŒ,Ð² ÑÐ°Ð±Ð»ÐžÑÐµ ÑÐ¿ÑÐ°Ð²Ð°. 
+<Break/>
+ÐÐ°ÑÐµÐŒ Ð²ÑÐ±ÐµÑÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐ¹ Ñ
+ÐŸÑÐžÐŒ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ,Ð¿ÑÑÐµÐŒ ÑÐµÐ»ÑÐºÐ° ÑÑÐŽÐŸÐŒ Ñ ÐœÐžÐŒ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ. 
+ <Break/>
+ ÐÐ¿ÐžÑÐžÑÐµ ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð² Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. ÐÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð·ÐœÐ°Ðº "|",Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ ÐŸÐœ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐµÑÐ°ÑÑ
+ÐžÐ¹. ÐÐŸ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÐž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.Ð­ÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.  
+<Break/>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐ°ÐºÐ¶Ðµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº Ð¿Ð°Ð¿ÐºÐµ ÐžÐ»Ðž ÑÑÐ°Ð·Ñ Ðº ÐœÐµÐºÐŸÑÐŸÑÐŸÐŒÑ ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ². Ð­ÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ ÑÐ°Ð¹Ð»Ð°ÐŒ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð¿Ð°Ð¿ÐºÐž ÐžÐ»Ðž Ðº ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ Ðž Ðº Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐœÑÐŒ Ð¿Ð°Ð¿ÐºÐ°ÐŒ. ÐÐŒÐµÐ¹ÑÐµ Ð² Ð²ÐžÐŽÑ,ÑÑÐŸ ÐµÑÐ»Ðž ÐÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÐ»Ðž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº Ð¿Ð°Ð¿ÐºÐµ, ÑÐŸ Ð»ÑÐ±ÑÐµ ÐœÐŸÐ²ÑÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÐœÐµÐ¹ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÑÐœÐ°ÑÐ»ÐµÐŽÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐžÐ· ÑÑÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž.  
+ <Break/>
+ ÐÑÐ»Ðž ÐÑ Ð²ÑÐ±ÐµÑÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐµÑÑÑÑ Ð² ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÐžÐ· ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÐŸ ÐŸÐœÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑÑÑÑ ÑÐ°ÐºÐ¶Ðµ Ðº ÐŽÑÑÐ³ÐžÐŒ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐŒ ÑÐ°Ð¹Ð»Ð°ÐŒ.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ Ðº Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð¿ÑÑÑ, ÐŸÐ¿ÐžÑÐ°ÐœÐœÑÐ¹ Ð² <Reference target="theenrichview">Ð Ð°ÑÑÐžÑÐµÐœÐœÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ</Reference>.
+    </Section>
+    <Section name="addingpreviouslydefinedmetadata">
+      <Title>ÐÐŸÐ±Ð°Ð²Ð»ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,ÑÐ¶Ðµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÑÐµ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ Ðº Ð¿Ð°Ð¿ÐºÐ°ÐŒ ÐžÐ»Ðž Ðº ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð», Ð·Ð°ÑÐµÐŒ ÑÑÐµÐ±ÑÐµÐŒÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐŽÐµÑÐµÐ²Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÑÐ°ÑÑÐžÑÑÑ Ð¿Ð°Ð¿ÐºÐž ÐžÐµÑÐ°ÑÑ
+ÐžÐž Ð¿ÐŸ ÐŒÐµÑÐµ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž. ÐÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² Ð¿ÐŸÐ»Ðµ ÑÐµÐºÑÑÐ° (Ð°Ð»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÐŸ ÐŒÐŸÐ¶ÐœÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð°Ð²ÑÐŸÐ²ÑÐ±ÐŸÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ Ðž ÑÑÐµÐŽÑÑÐ² Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐŸÐ³ÐŸ Ð·Ð°Ð²ÐµÑÑÐµÐœÐžÑ). Ð©ÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐŒÑ ÑÐ°Ð¹Ð»Ñ.
+<Break/>
+ÐÑÐŸÑÐµÑÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÑÐ¶Ðµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒÐž Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒÐž Ðº Ð¿Ð°Ð¿ÐºÐ°ÐŒ ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ñ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐ°ÐºÐŸÐ¹ Ð¶Ðµ.
+    </Section>
+    <Section name="updatingmetadata">
+      <Title>ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÐŒÐŸÐŽÐµÑÐœÐžÐ·ÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐµ Ðº ÑÐ°Ð¹Ð»Ñ.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÑÐ°ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð»,Ðº ÐºÐŸÑÐŸÑÐŸÐŒÑ ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð¿ÑÐžÐŒÐµÐœÐµÐœÐŸ,Ð° Ð·Ð°ÑÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐÑ Ñ
+ÐŸÑÐžÑÐµ ÐžÐ·ÐŒÐµÐœÐžÑÑ.Ð ÐµÐ·ÑÐ»ÑÑÐ°Ñ Ð²ÑÐ±ÐŸÑÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. Ð ÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ", ÑÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ.
+<Break/>
+ÐÑÐŸÑÐµÑÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÑ ÐŽÐ°ÐœÐœÑÑ
+ Ð¿Ð°Ð¿ÐºÐž Ñ Ð¿ÐŸÐŽÐ¿Ð°Ð¿ÐºÐ°ÐŒÐž ÐžÐ»Ðž ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹, ÐœÐŸ Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ, ÐŸÐ±ÑÐžÐµ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²/Ð¿Ð°Ð¿ÐŸÐº.
+<Break/>
+ ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, Ð° ÐœÐµ ÑÐŸÐ»ÑÐºÐŸ ÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÑ Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ. Ð¢Ð°ÐºÐžÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ, Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ ÐÑ Ð·Ð°ÐŒÐµÐœÐžÐ»Ðž, ÐŸÑÑÐ°ÑÑÑÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.
+    </Section>
+    <Section name="removingmetadata">
+      <Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÐŽÐ°Ð»ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ².</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ² 
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐžÐ· Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ð° ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÑÐ¿ÐŸÐ»ÐœÑÐµÑÑÑ ÑÐ°ÐºÐžÐŒ Ð¶Ðµ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ,ÐºÐ°Ðº Ðž ÐžÑ
+ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ. Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð» ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð°,Ð·Ð°ÑÐµÐŒ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÐ°Ð±Ð»ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÐµ. ÐÑÐ»Ðž Ðº ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÐŸÐŒÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐœÐŸÐ¿ÐºÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ" Ð°ÐºÑÐžÐ²ÐžÑÑÐµÑÑÑ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐœÐµÐ¹,ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÑÐºÐ°Ð·Ð°ÐœÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°. ÐÑÑÐ³ÐžÐµ ÑÐ°Ð¹Ð»Ñ ÐŸÑÑÐ°ÑÑÑÑ ÐœÐµÐžÐ·ÐŒÐµÐœÐœÑÐŒÐž, Ðž ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŸÑÑÐ°ÑÑÑÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.
+<Break/>
+ÐÑÐŸÑÐµÑÑ ÑÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ñ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹.
+    </Section>
+    <Section name="reviewingmetadata">
+      <Title>ÐÑÐŸÑÐŒÐŸÑÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+<ObviousSentence>Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ð²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ Ð¿ÐŸÐ»ÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.</ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐœÑÑÐŸÐ»ÑÐœÐ°Ñ ÑÐ°Ð±Ð»ÐžÑÐ°
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÑÐ°Ð²Ð°Ñ ÐºÐœÐŸÐ¿ÐºÐ° ÐŒÐµÐœÑ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ²ÑÐŸÑÐžÐ»ÑÑÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐœÐŸÐ³ÐŽÐ° ÐÐ°ÐŒ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÐžÐ»Ðž Ð²ÑÐµÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÑÐ°Ð·Ñ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ,ÑÑÐŸÐ±Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ,ÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŸÑÑÐ°ÑÑÑÑ ÐŽÐ»Ñ Ð²ÐŸÐ·ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÐžÐ»Ðž Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐœÐµÐºÐŸÑÐŸÑÑÐµ ÐžÐŽÐµÐž ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÑÐ°ÑÐ¿ÑÐŸÑÑÑÐ°ÐœÐµÐœÐžÑ ÐŽÐ°ÐœÐœÑÑ
+.
+<Break/>
+ÐÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð»Ñ, ÐºÐŸÑÐŸÑÑÐµ Ð²Ñ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, Ð¿ÐŸÑÐŸÐŒ ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° Ð¿ÑÐ°Ð²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐŒÑÑÐž Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐ°Ð·ÐœÐ°ÑÐµÐœÑÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ...". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ,ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐµ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ", Ð·Ð°ÐœÑÑÐŸÐµ Ð±ÐŸÐ»ÑÑÐŸÐ¹ ÑÐ°Ð±Ð»ÐžÑÐµÐ¹,ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ¹ ÐŒÐœÐŸÐ³ÐŸ ÐºÐŸÐ»ÐŸÐœÐŸÐº.ÐÐµÑÐ²Ð°Ñ ÐºÐŸÐ»ÐŸÐœÐºÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÐŒÐµÐœÐ° ÑÐ°Ð¹Ð»ÐŸÐ²;ÑÑÑÐŸÐºÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ Ð²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÑÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².  
+ <Break/>
+ Ð ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐ°Ð±Ð»ÐžÑÑ ÐŒÐŸÐ¶ÐµÑ Ð·Ð°ÐœÑÑÑ ÐœÐµÐºÐŸÑÐŸÑÐŸÐµ Ð²ÑÐµÐŒÑ,ÐµÑÐ»Ðž Ð²ÑÐ¿Ð°Ð»ÐŸ ÐŒÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐŸÐŽÐŸÐ»Ð¶Ð°ÑÑ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ Ð¿ÑÐž ÐŸÑÐºÑÑÑÐŸÐŒ ÐŸÐºÐœÐµ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ". 
+<Break/>
+Ð©ÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐºÑÑÑÑ", ÑÑÐŸÐ±Ñ ÑÐºÑÑÑÑ ÐŸÐºÐœÐŸ.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐ°ÐºÐ¶Ðµ ÑÐ°ÑÑÐŒÐŸÑÑÐµÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð²ÑÐ±ÐžÑÐ°Ñ ÑÐµ ÑÐ°Ð¹Ð»Ñ, ÐºÐŸÑÐŸÑÑÐµ ÐÑ Ð¶ÐµÐ»Ð°ÐµÑÐµ ÐžÐ·ÑÑÐžÑÑ, ÑÐµÐ»ÐºÐœÑÐ² Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ Ðž Ð²ÑÐ±ÐžÑÐ°Ñ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ".ÐÑÐ»Ðž Ð±ÑÐ»Ð° Ð²ÑÐ±ÑÐ°ÐœÐ° Ð¿Ð°Ð¿ÐºÐ°,ÑÐŸ Ð²ÑÐµ ÐµÑ ÐŽÐŸÑÐµÑÐœÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² ÑÐ°Ð±Ð»ÐžÑÑ.  
+<Break/>
+ ÐÐŸÐ³ÐŽÐ° ÐŸÐœÐ° ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ ÑÐ»ÐžÑÐºÐŸÐŒ Ð±ÐŸÐ»ÑÑÐŸÐ¹,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÑÑÐžÐ»ÑÑÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð¿ÑÐžÐŒÐµÐœÑÑ ÑÐžÐ»ÑÑÑÑ Ðº ÐºÐŸÐ»ÐŸÐœÐºÐ°ÐŒ.ÐÐŸÑÐºÐŸÐ»ÑÐºÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ ÐœÐŸÐ²ÑÐµ ÑÐžÐ»ÑÑÑÑ, ÑÐŸÐ»ÑÐºÐŸ ÑÐµ ÑÑÑÐŸÐºÐž,ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑ ÐžÐŒ,ÐŸÑÑÐ°ÑÑÑÑ Ð²ÐžÐŽÐžÐŒÑÐŒÐž.Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ,ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐ»Ðž ÐŸÑÐžÑÑÐžÑÑ ÑÐžÐ»ÑÑÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÐŸ ÐžÐºÐŸÐœÐºÐµ "ÐÐŸÑÐŸÐœÐºÐ°" ÐœÐ°Ð²ÐµÑÑ
+Ñ ÐºÐŸÐ»ÐŸÐœÐºÐž. Ð£ Ð²Ð°Ñ Ð±ÑÐŽÐµÑ Ð·Ð°Ð¿ÑÐŸÑÐµÐœÐ° ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ ÑÐžÐ»ÑÑÑÐµ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÐžÐ»ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ,ÐžÐ·ÐŒÐµÐœÑÐµÑÑÑ ÑÐ²ÐµÑ ÐºÐŸÐ»ÐŸÐœÐºÐž.
+<Break/>
+ ÐÐ°Ð¿ÑÐŸÑ ÐžÐŒÐµÐµÑ ÑÐžÐŒÐ²ÐŸÐ»Ñ "ÐÑÐŸÑÑÐŸÐ¹" Ðž "ÐÑÐŸÐŽÐ²ÐžÐœÑÑÑÐ¹".ÐÑÐŸÑÑÐ°Ñ Ð²ÐµÑÑÐžÑ ÑÐžÐ»ÑÑÑÑÐµÑ ÐºÐŸÐ»ÐŸÐœÐºÐž ÑÐ°Ðº,ÑÑÐŸ ÐŸÐœÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÐŸÐ»ÑÐºÐŸ ÑÑÑÐŸÐºÐž, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ("*" ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð²ÑÐµÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ).ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ°.ÐÑÐŸÐŽÐ²ÐžÐœÑÑÐ°Ñ Ð²ÐµÑÑÐžÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð²ÑÐ¿ÐŸÐ»ÐœÑÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ:ÐÐŸÐ»Ð¶ÐµÐœ ÐœÐ°ÑÐ°ÑÑÑÑ Ñ,ÐÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ,Ð Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ,ÐÐµÐœÑÑÐµ ÑÐµÐŒ, Ð­ÐºÐ²ÐžÐ²Ð°Ð»ÐµÐœÑÐµÐœ. ÐÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð±ÑÐŽÐµÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐŸÐ²Ð°ÑÑ, ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ ÑÐ°Ðº,ÑÑÐŸÐ±Ñ Ð±ÑÑÑ Ð»ÑÐ±ÐŸÐ¹ ÑÑÑÐŸÐºÐŸÐ¹,Ð²ÐºÐ»ÑÑÐ°Ñ "*". ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ,ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð»Ðž ÑÑÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ Ð±ÑÑÑ ÐœÐµ ÑÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÑÐŒ ÑÐ»ÑÑÐ°ÐµÐŒ. ÐÐ°ÐºÐŸÐœÐµÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð²ÑÐŸÑÐŸÐµ ÑÑÐ»ÐŸÐ²ÐžÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ,ÐºÐŸÑÐŸÑÐŸÐµ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŽÐ»Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÐ°ÐœÐ³Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ (Ð²ÑÐ±ÐžÑÐ°Ñ Ð Ð±ÐŸÐ»ÑÑÐŸÐµ) ÐžÐ»Ðž Ð°Ð»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÑÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ (Ð²ÑÐ±ÐžÑÐ°Ñ ÐÐÐ).ÐÐžÐ¶Ðµ ÑÑÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐž ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ Ð¿ÐŸÐ»Ðµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž (Ð²ÐŸÐ·ÑÐ°ÑÑÐ°ÑÑÐžÐ¹ ÐžÐ»Ðž Ð¿ÐŸÐœÐžÐ¶Ð°ÑÐžÐ¹ÑÑ).ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ Ð·Ð°ÐºÐŸÐœÑÐžÑÐµ,ÐœÐ°Ð¶ÐŒÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐžÐ»ÑÑÑÐ°",ÑÑÐŸÐ±Ñ Ð¿ÑÐžÐŒÐµÐœÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐžÐ»ÑÑÑ Ðº ÐºÐŸÐ»ÐŸÐœÐºÐµ.ÐÐ°Ð¶ÐŒÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐµÐºÑÑÐžÐ¹ ÑÐžÐ»ÑÑÑ.ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ ÐœÐ° ÑÐŸ,ÑÑÐŸ ÐŽÐµÑÐ°Ð»Ðž ÑÐžÐ»ÑÑÑÐ° ÑÐŸÑ
+ÑÐ°ÐœÑÑÑÑÑ, ÐŽÐ°Ð¶Ðµ ÐºÐŸÐ³ÐŽÐ° ÑÐžÐ»ÑÑÑ ÐŸÑÐžÑÐµÐœ.
+ <Break/>
+ÐÐ°Ð¿ÑÐžÐŒÐµÑ,ÑÑÐŸÐ±Ñ ÐŸÑÑÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð²ÑÐ±ÐµÑÐžÑÐµ ÐºÐŸÐ»ÐŸÐœÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÑ ÑÐžÐ»ÑÑÑÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ (Ð¿ÑÐŸÑÑÐŸÐ¹ ÑÐžÐ»ÑÑÑ ÐœÐ° "*") Ðž Ð²ÐŸÐ·ÑÐ°ÑÑÐ°ÑÑÐžÐ¹ ÐžÐ»Ðž Ð¿ÐŸÐœÐžÐ¶Ð°ÑÑÐžÐ¹ÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº.   
+    </Section>
+    <Section name="importingpreviouslyassignedmetadata">
+      <Title>ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+      Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐµ ÑÐ°ÐœÑÑÐµ. Ð ÐµÑÑ ÐžÐŽÐµÑ ÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐ»Ðž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÐ°ÐœÑÑÐµ.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÑÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐž ÑÐ°Ð¹Ð»ÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑÐ»Ðž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð² ÑÐŸÑÐŒÐµ,ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°Ð²Ð°ÐµÐŒÐŸÐ¹ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ,Ð±ÑÐ»Ðž Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÑ ÑÐ°Ð¹Ð»Ñ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ,ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÐžÐ· ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone - ÐŸÐœÐž ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÑÑÑÑ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž,ÐºÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÑÐ°Ð¹Ð».Ð§ÑÐŸÐ±Ñ ÑÐŽÐµÐ»Ð°ÑÑ ÑÑÐŸ, ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ Ð² ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°Ñ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ Ð² ÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+<Break/>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð·Ð°Ð¿ÑÐ°ÑÐžÐ²Ð°ÐµÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ.ÐÐ°Ð¿ÑÐŸÑ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐºÑÐ°ÑÐºÐžÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž Ðž Ð·Ð°ÑÐµÐŒ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÐŒÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÑÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐŸÐœ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÐžÑÑ
+ÐŸÐŽÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ.Ð­ÑÐŸ Ð¿ÐŸÐ»Ðµ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ ÐžÐ»Ðž ÐžÐ·ÐŒÐµÐœÐµÐœÐŸ.ÐÐ°ÑÐµÐŒ ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ,ÐºÐ°ÐºÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ,Ð° Ð·Ð°ÑÐµÐŒ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐ¹ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² ÑÑÐžÑ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°Ñ
+.Ð¡ÐžÑÑÐµÐŒÐ° Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð²ÑÐ±ÐžÑÐ°ÐµÑ ÑÐ°ÐŒÐŸÐµ Ð±Ð»ÐžÐ·ÐºÐŸÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ Ð² ÑÐµÑÐŒÐžÐœÐ°Ñ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ðž ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŽÐ»Ñ ÐœÐŸÐ²ÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. 
+<Break/>
+ ÐÑÐŸÐ²ÐµÑÐžÐ² ÑÑÐž ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐžÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº Ð²ÑÐ±ÑÐ°ÐœÐœÑÐŒ ÐœÐ°Ð±ÐŸÑÐ°ÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.Ð­ÑÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐŸÐ»ÑÐºÐŸ ÑÐŸÐ³ÐŽÐ°,ÐºÐŸÐ³ÐŽÐ° ÐŸÑÑÑÑÑÑÐ²ÑÐµÑ ÑÐ»ÐµÐŒÐµÐœÑ Ñ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð²ÑÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐŸÐº; "ÐÐ°ÐŒÐµÐœÐžÑÑ" ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐ°ÐµÑ Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ðº ÑÐ¶Ðµ ÑÐ°ÐœÐµÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐŒÑ  Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»ÐµÐŒ;ÐœÐ°ÐºÐŸÐœÐµÑ, "ÐÐµ ÑÑÐžÑÑÐ²Ð°ÑÑ", ÑÑÐŸÐ±Ñ ÐœÐµ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐœÐžÐºÐ°ÐºÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÑÑÐžÐŒ ÐžÐŒÐµÐœÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑÐ°. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÐ»Ðž,ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐœÐµÐºÐŸÑÐŸÑÑÑ ÑÐ°ÑÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð·Ð°Ð¿ÐžÑÐ°ÐœÐœÐ°Ñ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÑÐŸÑ
+ÑÐ°ÐœÑÐµÑÑÑ ÐœÐ° Ð²ÑÐµÐŒÑ ÑÑÑÐµÑÑÐ²ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+<Break/>
+ ÐÐ»Ñ ÐŽÐµÑÐ°Ð»ÐµÐ¹ ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² metadata.xml, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ Ð² Greenstone ÐŽÐ»Ñ Ñ
+ÑÐ°ÐœÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÐŒ.Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+ </Section>
+  </Section>
+
+  <Section name="designingacollection">
+    <Title>ÐÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>
+ ÐÐŸÐ³ÐŽÐ° ÐÐ°ÑÐž ÑÐ°Ð¹Ð»Ñ ÑÐœÐ°Ð±Ð¶ÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž,ÐŽÐ°Ð»ÐµÐµ ÐÑ ÑÐµÑÐ°ÐµÑÐµ,Ð² ÐºÐ°ÐºÐŸÐŒ Ð²ÐžÐŽÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Greenstone ÐŽÐŸÐ»Ð¶ÐœÐ° Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐ° Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»ÑÐŒ.ÐÐ°ÐºÐŸÐ³ÐŸ Ð²ÐžÐŽÐ° ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŽÐŸÑÑÑÐ¿ÐœÐ° ÐŽÐ»Ñ Ð¿ÐŸÐžÑÐºÐ°? ÐÐ°ÐºÐžÐµ Ð¿ÑÑÐž Ð¿ÑÐµÐŽÑÑÐŒÐŸÑÑÐµÐœÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°? ÐÐ°ÐºÐžÐµ ÑÐ·ÑÐºÐž Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑÑÑ? ÐÐŽÐµ ÑÐ°Ð·ÐŒÐµÑÑÐžÑÑ ÐºÐœÐŸÐ¿ÐºÐž ÐœÐ° ÑÑÑÐ°ÐœÐžÑÐµ? Ð­ÑÐž Ð²ÐµÑÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐœÐ°ÑÑÑÐŸÐµÐœÑ;Ð² ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ.
+    <Section name="thedesignview">
+      <Title>ÐÐžÐŽ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°</Title>
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» Ð²Ð²ÐŸÐŽÐžÑ ÐÐ°Ñ Ð² ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°ÐµÐŒÑÐ¹ Ð¿ÑÐŸÐµÐºÑ Ðž ÐŸÐ±ÑÑÑÐœÑÐµÑ,ÐºÐ°Ðº Ð²ÑÐ±ÑÐ°ÑÑ ÐŒÐµÐ¶ÐŽÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÐŒÐž Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð°ÐºÑÐžÐ²ÐœÐŸÐ¹ ÑÐ°ÑÑÐž ÐŸÐºÐœÐ°.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÑÐ±ÐŸÑ ÑÐ°Ð·ÐŽÐµÐ»Ð° Ð¿ÑÐŸÐµÐºÑÐ°
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑÐž Ð¿ÐŸÐŒÐŸÑÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐŸÑÐŒÐžÑÐŸÐ²Ð°ÑÑ Ð²ÐžÐŽÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÑÐµÐŒÐŸÐ¹ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»Ñ.ÐÐ°ÑÐžÐ°ÐœÑÑ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÑ ÐœÐ° ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÑÐ°Ð·ÐŽÐµÐ»Ñ, ÐºÐ°Ð¶ÐŽÑÐ¹ ÐžÐ· ÐºÐŸÑÐŸÑÑÑ
+ ÑÐ²ÑÐ·Ð°Ðœ ÑÐŸ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐ¹ ÑÑÐ°ÐŽÐžÐµÐ¹ ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐž ÐžÐ»Ðž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž.
+<Break/>
+ Ð¡Ð»ÐµÐ²Ð° ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐ¹,Ð° ÑÐ¿ÑÐ°Ð²Ð° - ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÐºÑÑÐžÐŒ ÑÐ¿ÐžÑÐºÐŸÐŒ.Ð§ÑÐŸÐ±Ñ Ð¿ÐµÑÐµÐ¹ÑÐž ÐœÐ° Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐµÐ³ÐŸ ÐžÐŒÑ Ð² ÑÐ¿ÐžÑÐºÐµ. 
+<Break/>
+ Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐœÐžÐŒÐ°ÑÑ ÑÑÐ°ÐŽÐžÐž Ðž ÑÐµÑÐŒÐžÐœÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐµ Ð¿ÑÐž Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÐœÐ°ÑÐ°Ð»Ð° Ð¿ÑÐŸÑÑÐžÑÐµ Ð³Ð»Ð°Ð²Ñ 1 Ðž 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone. 
+    </Section>
+    <Section name="generalsettings">
+      <Title>ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ðž ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŸÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž, ÐŸÑÐœÐŸÑÑÑÐžÐµÑÑ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð² "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ±ÑÐžÐµ"
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐŽÐµÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,Ð¿ÑÐµÐŽÑÑÐŒÐŸÑÑÐµÐœÐœÑÐµ Ð¿ÑÐž ÑÐŸÐ·ÐŽÐ°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÑ.
+<Break/>
+ ÐÐ°Ð²ÐµÑÑ
+Ñ ÑÑÑÐ°ÐœÐžÑÑ ÐžÐŒÐµÐµÑÑÑ Ð¿ÐŸÐ»Ðµ ÐžÐœÑÑÑÑÐºÑÐžÐž,ÐºÐŸÑÐŸÑÐ°Ñ ÐžÐŒÐµÐµÑÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð·ÐŽÐµÐ»Ð°.ÐÐœÐŸ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐºÑÐ°ÑÐºÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº ÑÐºÐ°Ð·Ð°ÐœÐžÐ¹,ÑÑÐŸÐ±Ñ ÐœÐ°Ð¿ÐŸÐŒÐœÐžÑÑ ÐÐ°ÐŒ,ÐºÐ°ÐºÐžÐµ ÑÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐµ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÐž ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž.
+<Break/>
+ Ð¡ÐœÐ°ÑÐ°Ð»Ð° ÐžÐŽÑÑ ÐºÐŸÐœÑÐ°ÐºÑÐœÑÐµ Ð°ÐŽÑÐµÑÐ° ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ ÑÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑÐµÐ³ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð»ÐžÑÐ°. Ð¡Ð»ÐµÐŽÑÑÑÐµÐµ Ð¿ÐŸÐ»Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð¡Ð»ÐµÐŽÑÑÐµÐµ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð¿Ð°Ð¿ÐºÑ, Ð² ÐºÐŸÑÐŸÑÐŸÐ¹ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐœÐŸ ÐœÐµ Ð¿ÐŸÐŽÐ»ÐµÐ¶ÐžÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑÐŒ. Ð¡Ð»ÐµÐŽÑÑÑÐµÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ (Ð² ÑÐŸÑÐŒÐµ URL) ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ,ÑÑÐŸÐ±Ñ Ð¿ÐŸÐºÐ°Ð·Ð°ÑÑ ÐœÐ°Ð²ÐµÑÑ
+Ñ ÑÐ»ÐµÐ²Ð° ÐŸÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÑÑÐ°ÐœÐžÑÑ ÐŸ ÐœÐµÐ¹, Ðž ÑÐ»ÐµÐŽÑÑÑÐµÐµ - ÐžÐºÐŸÐœÐºÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐ°Ñ Ð² Greenstone ÑÑÑÐ°ÐœÐžÑÐ° ÐŽÐ»Ñ ÑÐ²ÑÐ·Ðž Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹. ÐÐŸÑÐ»Ðµ ÑÑÐŸÐ³ÐŸ, Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐŽÐµÐ»ÐžÑÑ Ð¿ÑÐœÐºÑ Ð³Ð°Ð»ÐŸÑÐºÐŸÐ¹, ÑÐºÐ°Ð·Ð°Ð², ÐŽÐŸÑÑÑÐ¿ÐœÐ° Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐŽÑÑÐ³ÐžÐŒ. ÐÐ°ÐºÐŸÐœÐµÑ, ÑÐ»ÐµÐŽÑÐµÑ ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ° "ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž",Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÐ°Ñ ÑÐ°ÐœÐµÐµ Ð² <Reference target="creatingacollection">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Reference>.
+    </Section>
+    <Section name="plugins">
+      <Title>ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÐ°ÐœÐŸ,ÐºÐ°Ðº ÑÐŸÑÐŒÐžÑÐŸÐ²Ð°ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐŽÐ»Ñ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹.Ð ÐœÑÐŒ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ,ÐºÐ°ÐºÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ, ÐºÐ°ÐºÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿ÐµÑÐµÐŽÐ°Ð²Ð°ÑÑ Ðº ÐœÐžÐŒ Ðž Ð² ÐºÐ°ÐºÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ ÐŸÐœÐž Ð²Ñ
+ÐŸÐŽÑÑ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ
+ "ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²" Ð² Ð¿ÑÐœÐºÑÐµ "Ð Ð°Ð·ÐŽÐµÐ»Ñ Ð¿ÑÐŸÐµÐºÑÐ°".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </ContentsItem>
+          <ContentsItem>
+            Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </ContentsItem>
+          <ContentsItem>
+             Ð€ÐŸÑÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ·ÐŒÐµÐœÐµÐœÐžÐµ Ð¿ÐŸÑÑÐŽÐºÐ° Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             ÐÐ°ÑÑÑÐŸÐ¹ÐºÐ° ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž</Anchor>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ "ÐÑÐ±ÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ ÐŸÐºÐŸÐ»ÐŸ ÐŸÑÐœÐŸÐ²Ð°ÐœÐžÑ,Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ Ñ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµÐŒ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²";ÐŸÐœÐŸ ÐŸÐ¿ÐžÑÐ°ÐœÐŸ ÐœÐžÐ¶Ðµ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÑÑÐŸÑÐŒÐžÑÐŸÐ²Ð°Ð»Ðž ÐœÐŸÐ²ÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ, ÐŸÐœ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° "Ð£ÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐµ ÑÐµÐºÑÑÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ". ÐÐ°ÐŒÐµÑÑÑÐµ, ÑÑÐŸ, ÐºÑÐŸÐŒÐµ UnknownPlug, ÐºÐ°Ð¶ÐŽÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð²ÑÐ²ÐµÐŽÐµÐœ ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐžÐœ ÑÐ°Ð· Ð² ÑÐ¿ÐžÑÐºÐµ.
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ð² ÑÐ¿ÐžÑÐºÐµ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ".
+<Break/>
+ ÐÐ»Ð°Ð³ÐžÐœÑ ÑÐŸÑÐŒÐžÑÑÑÑÑÑ Ð¿ÑÑÐµÐŒ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ². Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÑ
+, Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð»Ð°Ð³ÐžÐœ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð€ÐŸÑÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ²"(ÐžÐ»Ðž ÐŽÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ ÐœÐ° Ð¿Ð»Ð°Ð³ÐžÐœÐµ). ÐÐžÐ°Ð»ÐŸÐ³ "ÐÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐµ ÐÑÐ³ÑÐŒÐµÐœÑÑ" Ð²ÐŸÐ·ÐœÐžÐºÐ°ÐµÑ Ñ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž ÐŸÐ¿ÑÐžÑÐŒÐž ÐŽÐ»Ñ ÑÐºÐ°Ð·Ð°ÐœÐžÑ Ð°ÑÐ³ÑÐŒÐµÐœÑÐŸÐ².
+<Break/>
+ Ð¡ÑÑÐµÑÑÐ²ÑÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ Ð²ÐžÐŽÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÐ²Ð»ÑÑÑÑÑ ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÑÐŒÐž ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž Ðž ÑÐµÐ»ÑÐŸÐº ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ ÐŸÐ¿ÑÐžÑ Ðº Ð¿Ð»Ð°Ð³ÐžÐœÑ. ÐÑÑÐ³ÐžÐµ Ð²ÑÑÑÑÐ¿Ð°ÑÑ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÑÑÑÐŸÐºÐž ÑÐµÐºÑÑÐ° Ñ ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ Ðž ÐŸÐ±Ð»Ð°ÑÑÑÑ ÑÐµÐºÑÑÐ°. Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð¿ÐŸÐ»Ðµ,ÑÑÐŸÐ±Ñ Ð²ÑÐ·Ð²Ð°ÑÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑ,Ð·Ð°ÑÐµÐŒ ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°Ð¹ÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐ¹ ÑÐµÐºÑÑ (Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ,Ð¿ÑÑÑ ÑÐ°Ð¹Ð»Ð° Ðž Ñ.ÐŽ.) Ð² Ð¿ÐŸÐ»Ðµ. ÐÑÑÐ³ÐžÐµ - ÑÑÐŸ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐµ ÐŒÐµÐœÑ,Ð² ÐºÐŸÑÐŸÑÑÑ
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÐžÑÐ°ÑÑ ÐœÑÐ¶ÐœÐŸÐµ ÐžÐ· Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐœÐ°Ð±ÐŸÑÐ° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐœÑÑÑ,ÑÑÐŸ ÐŽÐµÐ»Ð°ÐµÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑ, ÐŽÐ°Ð¹ÑÐµ ÐŒÑÑÐž ÐœÐ° ÐŒÐ³ÐœÐŸÐ²ÐµÐœÐžÐµ Ð¿ÑÐŸÐ¹ÑÐž Ð¿ÐŸ ÐµÐ³ÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ Ðž ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ Ð¿ÐŸÑÐ²ÐžÑÑÑ. 
+<Break/>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐžÐ·ÐŒÐµÐœÐžÐ»Ðž ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ,ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÐ", ÑÑÐŸÐ±Ñ Ð¿ÑÐžÐœÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ Ðž Ð·Ð°ÐºÑÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³,ÐžÐ»Ðž "ÐÑÐŒÐµÐœÐžÑÑ",ÑÑÐŸÐ±Ñ Ð·Ð°ÐºÑÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³ Ð±ÐµÐ· ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ Ð»ÑÐ±ÑÑ
+ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ² Ð¿Ð»Ð°Ð³ÐžÐœÐ°. 
+ <Break/>
+ ÐÐ»Ð°Ð³ÐžÐœÑ Ð² ÑÐ¿ÐžÑÐºÐµ ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÑ Ð² ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ,ÑÐ°Ðº ÐºÐ°Ðº ÑÐ¿ÐŸÑÑÐŽÐŸÑÐµÐœÐžÐµ ÐžÐœÐŸÐ³ÐŽÐ° Ð²Ð°Ð¶ÐœÐŸ. ÐÐ²Ð° Ð¿Ð»Ð°Ð³ÐžÐœÐ° - ArcPlug Ðž RecPlug - ÑÐ²Ð»ÑÑÑÑÑ Ð¶ÐžÐ·ÐœÐµÐœÐœÐŸ Ð²Ð°Ð¶ÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÑÐŸÑÐµÑÑÐ° ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ Ð² ÐºÐŸÐœÑÐµ ÑÐ¿ÐžÑÐ°ÐºÐ° (Ñ ÐŸÑÐŽÐµÐ»ÑÑÑÐµÐ¹ Ð»ÐžÐœÐžÐµÐ¹).Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÐŽÑÑÐ³ÐžÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ²,Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð»Ð°Ð³ÐžÐœ,ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·".  
+       <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŸÑÐµÐœÑ Ñ
+ÐŸÑÐŸÑÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ,ÐºÐ°ÐºÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑ. ÐÐŽÐœÐ°ÐºÐŸ, ÐµÑÑÑ ÑÐ»ÑÑÐ°Ðž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»Ñ Ñ
+ÐŸÑÐµÑ ÑÐºÐ°Ð·Ð°ÑÑ ÑÐ¿ÐµÑÐžÐ°Ð»ÑÐœÑÐµ Ð°ÑÐ³ÑÐŒÐµÐœÑÑ. ÐÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐµÑÑÑ Ð²ÐœÐžÐ·Ñ ÑÐµÐºÑÑÐŸÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ "ÐÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÐµ ÐÑÐ³ÑÐŒÐµÐœÑÑ". ÐÑÐ±ÐŸÐ¹ ÑÐµÐºÑÑ Ð² ÑÑÐŸÐŒ Ð¿ÐŸÐ»Ðµ ÐŽÐŸÑÐ»ÐŸÐ²ÐœÐŸ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ². 
+    </Section>
+    <Section name="searchtypes">
+      <Title>ÐÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÐœÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐžÐ·Ð°Ð¹ÐœÐ° Ð² Greenstone; Ð²ÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°,ÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð¿ÐŸÐ»ÑÐŒ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ Ð²ÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ° Ð² "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐ°Ðº Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÑÑ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ°ÑÑÐžÑÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÐŸÐžÑÐºÐ°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ³ÐŸ Ð²ÐžÐŽÐ° Ð¿ÐŸÐžÑÐºÐ°
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð²ÐžÐŽÐ° Ð¿ÐŸÐžÑÐºÐ°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ·ÐŒÐµÐœÐµÐœÐžÐµ Ð¿ÐŸÑÑÐŽÐºÐ° Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐœÐ°ÑÐžÐœÐ°ÐµÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°, ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑ ÑÐ°ÑÑÐžÑÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÐŸÐžÑÐºÐ°",ÐºÐŸÑÐŸÑÑÐ¹ Ð°ÐºÑÐžÐ²ÐžÐ·ÐžÑÑÐµÑ ÐŽÑÑÐ³ÐžÐµ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ.Ð­ÑÐŸ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð² ÑÐŸÑÐŒÐ°Ñ Greenstone 2.4, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð¿ÐŸÐ»ÑÐŒ Ðž ÐŸÐ·ÐœÐ°ÑÐ°ÐµÑ, ÑÑÐŸ (Ð°) - ÐžÐœÐŽÐµÐºÑ ÐŽÐžÐ·Ð°Ð¹ÐœÐ° ÑÐ²Ð»ÑÐµÑÑÑ ÐŸÑÐ»ÐžÑÐ°ÑÑÐžÐŒÑÑ,ÐŸÐ±ÑÑÑÐœÐµÐœ Ð²
+ (<Reference target="searchindexes">ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°</Reference> section);
+(b) - ÐžÐŒÐµÐµÑÑÑ Ð±ÐŸÐ»ÑÑÐµ ÑÑÐ°Ð³ÐŒÐµÐœÑÐŸÐ² ÑÐµÐºÑÑÐ° ÐŽÐ»Ñ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ° (<Reference target="translatetext">ÐÐµÑÐµÐ²ÐŸÐŽ</Reference>);(c) - ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµ Ð±ÑÐŽÐµÑ Ð³ÐŸÐŽÐœÐ° Ðº ÑÐ¿ÐŸÑÑÐµÐ±Ð»ÐµÐœÐžÑ Ð¿ÑÐž Ð±ÐŸÐ»ÐµÐµ ÑÑÐ°ÑÑÑ
+ Ð²ÐµÑÑÐžÑÑ
+ Greenstone. ÐÑÐ»Ðž ÐÑ Ð¿ÐŸÐ·Ð¶Ðµ ÐœÐµ Ð¿ÑÐŸÐ²ÐµÑÐžÑÐµ ÑÑÐŸ Ð¿ÐŸÐ»Ðµ,ÑÐŸ Ð±ÐŸÐ»ÑÑÐ°Ñ ÑÐ°ÑÑÑ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð²ÐµÑÐœÑÑÑÑ Ðº Greenstone 2.39.ÐÐŽÐœÐ°ÐºÐŸ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð¿ÑÐµÐŸÐ±ÑÐ°Ð·ÐŸÐ²Ð°ÑÑ ÐœÐŸÐ²ÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐ° Ð² Ð±ÐŸÐ»ÐµÐµ ÑÑÐ°ÑÑÐµ,ÑÐ°Ðº ÑÑÐŸ ÐÑ Ð±ÑÐŽÐµÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð²ÐµÑÐœÑÑÑ ÐžÑ
+ Ð²ÑÑÑÐœÑÑ.
+ <Break/>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°".ÐÐ°Ð¶ÐŽÑÐ¹ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ° ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÑÐ²ÐžÑÑÑÑ Ð² ÑÐ¿ÐžÑÐºÐµ ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐžÐœ ÑÐ°Ð·.
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "Ð¢ÐµÐºÑÑÐžÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°".Ð¡Ð¿ÐžÑÐŸÐº ÐŽÐŸÐ»Ð¶ÐµÐœ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ Ð¿ÐŸ ÐºÑÐ°Ð¹ÐœÐµÐ¹ ÐŒÐµÑÐµ ÐŸÐŽÐžÐœ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°. 
+ <Break/>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·". ÐÐµÑÐ²ÑÐ¹ ÐžÐ· ÐœÐŒÑ
+ Ð±ÑÐŽÐµÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ.
+    </Section>
+    <Section name="searchindexes">
+      <Title>ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°</Title>
+ ÐÐœÐŽÐµÐºÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑ,ÐºÐ°ÐºÐžÐµ ÑÐ°ÑÑÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÐŸÐžÑÐºÐ°. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ðž ÑÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑÑ,Ð° ÑÐ°ÐºÐ¶Ðµ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ.ÐÐŸÐŽ "Ð Ð°Ð·ÐŽÐµÐ»Ð°ÐŒÐž ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ ÐžÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ
+          </ContentsItem>
+	  <ContentsItem>
+	    ÐÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÐœÐŽÐµÐºÑ
+	  </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐžÐœÐŽÐµÐºÑÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ,Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ ÐœÐµÐ³ÐŸ Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÐžÐœÐŽÐµÐºÑÐ°".ÐÑÐ±ÐµÑÐžÑÐµ, ÐºÐ°ÐºÐžÐµ ÐžÐ· Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÑ
+ ÐžÑÑÐŸÑÐœÐžÐºÐŸÐ² ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐžÑÑ Ðº ÐžÐœÐŽÐµÐºÑÑ, ÑÐµÐ»ÐºÐ°Ñ ÐœÐ° Ð¿ÐŸÐ»Ðµ ÐŸÐºÐŸÐ»ÐŸ ÐœÐžÑ
+. Ð¡Ð¿ÐžÑÐŸÐº Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð° ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ»ÐœÑÐ¹ ÑÐµÐºÑÑ. Ð Ð°ÑÐ¿ÐŸÐ»Ð°Ð³Ð°Ñ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐŒÐž ÐžÑÑÐŸÑÐœÐžÐºÐ°ÐŒÐž ÐŽÐ°ÐœÐœÑÑ
+, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÐµÐ¿ÐµÐœÑ ÐŽÐµÑÐ°Ð»ÐžÐ·Ð°ÑÐžÐž ÐžÐœÐŽÐµÐºÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐŒÐµÐœÑ "ÐÐ° ÑÑÐŸÐ²ÐœÐµ". ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÑÐž ÐŽÐµÑÐ°Ð»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ,Ð¿ÑÐœÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ" ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ Ð°ÐºÑÐžÐ²ÐœÑÐŒ (ÐµÑÐ»Ðž ÐœÐµÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÐžÐœÐŽÐµÐºÑÐ° Ñ ÑÐµÐŒÐž Ð¶Ðµ ÑÐ°ÐŒÑÐŒÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒÐž).Ð©ÐµÐºÐœÐžÑÐµ ÐœÐ° ÐœÑÐŒ, ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÐžÐœÐŽÐµÐºÑ.
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÐœÐŽÐµÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ðž ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÐŽÐµÑÐ°Ð»Ðž. ÐÐŸÐŽÑÐ²ÐµÑÐŽÐžÑÐµ, ÐœÐ°Ð¶Ð°Ð² ÐœÐ° "ÐÐ°ÐŒÐµÐœÐžÑÑ ÐžÐœÐŽÐµÐºÑ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ² Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ".
+<Break/>
+ ÐÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ - ÑÐŸÑ, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐœÐ° Ð¿ÐŸÐžÑÐºÐŸÐ²ÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,Ð¿ÐŸÐŒÐµÑÐµÐœ ÑÑÐ³Ð°ÐŒÐž "[ÐÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ]" Ð² ÑÐ¿ÐžÑÐºÐµ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐœÐŽÐµÐºÑÑ".Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐµÐ³ÐŸ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐžÐœÐŽÐµÐºÑ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<Break/>
+ ÐÑÐ»Ðž Ð¿ÐŸÐ·Ð²ÐŸÐ»ÐµÐœ ÑÐ°ÑÑÐžÑÐµÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº (ÑÐµÑÐµÐ· Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°), ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÐœÐŽÐµÐºÑÐŸÐŒ ÐŸÑÐ»ÐžÑÐ°ÐµÑÑÑ.ÐÐ°Ð¶ÐŽÑÐ¹ ÐžÐœÐŽÐµÐºÑ Ð±Ð°Ð·ÐžÑÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐœÐ° ÐŸÐŽÐœÐŸÐŒ ÐžÑÑÐŸÑÐœÐžÐºÐµ ÐŽÐ°ÐœÐœÑÑ
+.ÐÐŒÐµÐµÑÑÑ ÐœÐŸÐ²ÑÐ¹ ÐžÑÑÐŸÑÐœÐžÐº Ð¿ÑÐµÐ²ÐŽÐŸÐŽÐ°ÐœÐœÑÑ
+ "Ð²ÑÐµ Ð¿ÐŸÐ»Ñ",ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÑÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð²ÑÐµÐŒ ÑÐºÐ°Ð·Ð°ÐœÐœÑÐŒ ÐžÐœÐŽÐµÐºÑÐ°ÐŒ ÑÑÐ°Ð·Ñ.Ð£ÑÐŸÐ²ÐœÐž ÐœÐµ Ð¿ÑÐžÐ²ÑÐ·Ð°ÐœÑ Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐŸÐŒÑ ÐžÐœÐŽÐµÐºÑÑ,Ð° Ð¿ÑÐžÐŒÐµÐœÑÑÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ ÐžÐœÐŽÐµÐºÑÐ°ÐŒ.Ð¢Ð°ÐºÐžÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ,ÐžÐœÐŽÐµÐºÑÑ Ðž ÑÑÐŸÐ²ÐœÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑÑÑÑ ÐŸÑÐŽÐµÐ»ÑÐœÐŸ.ÐÐœÐŽÐµÐºÑÑ ÑÐŽÐ°Ð»ÑÑÑÑÑ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ,ÐºÐ°Ðº Ð±ÑÐ»ÐŸ ÑÐºÐ°Ð·Ð°ÐœÐŸ Ð²ÑÑÐµ,ÐœÐŸ ÐžÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ Ð±ÐŸÐ»ÑÑÐµ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ. Ð­ÑÐŸ Ð¿ÑÐŸÑÑÐŸ Ð¿ÐµÑÐ²ÑÐ¹ ÐžÐ· ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ².   
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐžÐœÐŽÐµÐºÑÑ ÐœÐ° Ð²ÑÐµÑ
+ ÐžÑÑÐŸÑÐœÐžÐºÐ°Ñ
+,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐºÐœÐŸÐ¿ÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÑÐµ". ÐÐŒÑ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐžÐœÐŽÐµÐºÑÐ° Ð±ÑÐŽÐµÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÑÐ²ÑÐ·Ð°ÐœÐŸ Ñ ÐžÐŒÐµÐœÐµÐŒ ÐžÑÑÐŸÑÐœÐžÐºÐ°.Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐŒÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐžÐœÐŽÐµÐºÑ,ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÐµÐ³ÐŸ ÐŽÐµÑÐ°Ð»Ðž Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÐžÐœÐŽÐµÐºÑ".   
+    </Section>
+    <Section name="partitionindexes">
+      <Title>ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ</Title>
+ ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÑÑÑÐŸÐµÐœÑ ÐœÐ° ÐŽÐµÑÐ°Ð»ÑÐœÐŸÐŒ ÑÐµÐºÑÑÐµ ÐžÐ»Ðž ÐžÑÑÐŸÑÐœÐžÐºÐ°Ñ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐµÑÑÐŸ Ð¿ÐŸÐžÑÐºÐ° ÐŒÐŸÐ¶ÐµÑ Ð² ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐµÐŒ ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑÑÑ Ð¿ÑÑÑÐŒ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐŸÐ² Ð»ÐžÐ±ÐŸ Ð¿ÐŸ ÑÐ·ÑÐºÑ ÐžÐ»Ðž Ð¿ÐŸ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐŒ ÑÐžÐ»ÑÑÑÐ°ÐŒ.Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ. ÐÐœÑÑÑÐž "Ð Ð°Ð·ÐŽÐµÐ»ÐŸÐ² ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ".
+<Break/>
+ÐÐžÐŽ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ" ÐžÐŒÐµÐµÑ ÑÑÐž Ð·ÐœÐ°ÑÐµÐœÐžÑ:"ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ","ÐÐ°Ð·ÐœÐ°ÑÐžÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÐµ" Ðž "ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ·ÑÐºÐž". ÐÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ Ð±ÐŸÐ»ÑÑÐµ ÑÐ·ÐœÐ°ÑÑ ÐŸ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐž, ÑÐžÑÐ°Ð¹ÑÐµ  ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+ Ðž Ð¿ÐŸÐŽÐžÐœÐŽÐµÐºÑÐ°Ñ
+ Ð² Ð³Ð»Ð°Ð²Ðµ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+
+      <Section name="definefilters">
+        <Title>ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ.</ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </ContentsItem>
+            <ContentsItem>
+               Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </ContentsItem>
+            <ContentsItem>
+               ÐÐ·ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð€ÐžÐ»ÑÑÑÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð² Ð¿ÐŸÐŽÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð²ÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ð¿ÐŸ  ÐžÐœÐŽÐµÐºÑÑ,ÐŽÐ»Ñ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐ¹ ÐŒÐŸÐŽÐµÐ»Ðž.
+ <Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÑÐžÐ»ÑÑÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ" Ðž Ð²Ð²ÐµÐŽÐžÑÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŽÐ»Ñ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐžÐ»ÑÑÑÐ° Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÑÐžÐ»ÑÑÑÐ° ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž".ÐÐ°ÑÐµÐŒ Ð²ÑÐ±ÐµÑÐžÑÐµ Ñ
+Ð°ÑÐ°ÐºÑÐµÑÐœÑÐµ Ð¿ÑÐžÐ·ÐœÐ°ÐºÐž ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐµ ÐžÐ»Ðž ÑÐ»ÐµÐŒÐµÐœÑÐ°ÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐžÐ»Ðž ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°ÐµÐŒÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.ÐÐ²ÐµÐŽÐžÑÐµ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ,ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð² ÑÐµÑÐµÐœÐžÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÑÑÐŸÐ³ÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ.ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐºÐ»ÑÑÐ°ÑÑ "ÐÐºÐ»ÑÑÐµÐœÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²", ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑ ÑÐžÐ»ÑÑÑÑ, ÐžÐ»Ðž "ÐÑÐºÐ»ÑÑÐµÐœÐžÐµ" ÐžÑ
+.ÐÐ°ÐºÐŸÐœÐµÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð»ÑÐ±ÐŸÐ¹ ÑÑÐ°ÐœÐŽÐ°ÑÑ PERL, Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÑÐµ ÑÐ»Ð°Ð³Ðž Ð²ÑÑÐ°Ð¶ÐµÐœÐžÑ,ÑÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð¿ÑÐž ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ("i" ÐŽÐ»Ñ ÑÐ»ÑÑÐ°Ñ ÐœÐµÑÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐŸÐ³ÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ),ÐÐ°ÐºÐŸÐœÐµÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐžÐ»ÑÑÑ Ðº ÑÐ¿ÐžÑÐºÑ "Ð€ÐžÐ»ÑÑÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹".
+ <Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐžÐ»ÑÑÑ". 
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ°,Ð·Ð°ÐŒÐµÐœÐžÑÐµ Ð»ÑÐ±ÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ Ð·Ð°ÐºÑÐµÐ¿ÐžÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ.
+      </Section>
+      <Section name="assignpartitions">
+        <Title>ÐÐ°Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐœÐ°Ð·ÐœÐ°ÑÐžÑÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐ¹ ÑÐžÐ»ÑÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ.   </ObviousSentence>
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ
+            </ContentsItem>
+            <ContentsItem>
+              Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ
+            </ContentsItem>
+            <ContentsItem>
+              Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </ContentsItem>
+            <ContentsItem>
+               ÐÑÐžÑÑÐºÐ° ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐŒÐµÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÐ¹ ÑÐžÐ»ÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°Ð·ÐœÐ°ÑÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ" ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐŸÐ² ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐžÐ»Ðž ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿Ñ ÑÐžÐ»ÑÑÑÐŸÐ². ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÑÐ¹ ÑÐžÐ»ÑÑÑ (ÐžÐ»Ðž ÑÐžÐ»ÑÑÑÑ) ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐžÐ»ÑÑÑÑ ÑÑÐ±ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž", Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ".
+<Break/>
+  Ð§ÑÐŸÐ±Ñ Ð¿ÑÐŸÐžÐ·Ð²ÐµÑÑÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐŸÑÐžÑÑÐžÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+      </Section>
+      <Section name="assignlanguages">
+        <Title>ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŽÐµÑÐ°Ð»ÐžÐ·ÐžÑÐŸÐ²Ð°ÐœÐŸ,ÐºÐ°Ðº ÐŸÐ³ÑÐ°ÐœÐžÑÐžÑÑ ÐžÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ° ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐŒÐž ÑÐ·ÑÐºÐ°ÐŒÐž. ÐÑ ÑÐŸÐ²ÐµÑÑÐ°ÐµÑÐµ ÑÑÐŸ Ð¿ÑÐž ÑÐµÐ³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž" Ð² Ð¿ÐµÑÐµÑÐœÐµ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ".
+        <Contents>
+          <ContentsGroup linkto="simple">
+            <ContentsItem>
+               Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐ·ÑÐºÐ°ÐŒÐž
+            </ContentsItem>
+            <ContentsItem>
+               ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ·ÑÐº
+            </ContentsItem>
+            <ContentsItem>
+               Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº
+            </ContentsItem>
+            <ContentsItem>
+               Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ·ÑÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </ContentsItem>
+            <ContentsItem>
+               ÐÑÐžÑÑÐºÐ° ÑÐ·ÑÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+            </ContentsItem>
+          </ContentsGroup>
+        </Contents>
+        <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐ·ÑÐº ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž" ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐ° ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ. ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÑÐ¹ ÑÐ·ÑÐº ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ° "Ð¯Ð·ÑÐºÐž ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐ·ÑÐº"
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÑÐ±ÐŸÑÑ ÑÐ·ÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐ·ÑÐº".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÑÐ·ÑÐº Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÐžÑÑÐžÑÑ ÑÐ·ÑÐº Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+      </Section>
+    </Section>
+    <Section name="xcollectionsearching">
+      <Title>Ð¡ÐŸÐ²ÐŒÐµÑÑÐœÑÐ¹ Ð¿ÐŸÐžÑÐº Ð² ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+</Title>
+ Greenstone Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑ Ð¿ÐŸÐžÑÐº Ð² ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+, ÐºÐ°Ðº ÐµÑÐ»Ðž Ð±Ñ ÐŸÐœÐž Ð±ÑÐ»Ðž ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÑ.Ð­ÑÐŸ ÑÐŽÐµÐ»Ð°ÐœÐŸ ÐŽÐ»Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ "Ð¡ÑÐ¿ÐµÑÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž", ÐºÐŸÑÐŸÑÐ°Ñ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐžÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐžÑÐº Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÑÐ±ÐŸÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÐµÐ¶ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐŸÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐºÐŸÐœÑÑÐŸÐ»ÑÐœÑÐ¹ ÑÐ¿ÐžÑÐŸÐº ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹.Ð¢ÐµÐºÑÑÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð¿ÐŸÐŒÐµÑÐµÐœÑ Ðž ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÑÐœÐ°. Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŽÑÑÐ³ÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐžÑÐº Ð±ÑÐŽÐµÑ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑÑÑÑ Ð¿Ð°ÑÐ°Ð»Ð»ÐµÐ»ÑÐœÐŸ, ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐœÐžÑ
+ Ð² ÑÐ¿ÐžÑÐºÐµ (ÑÐµÐ»ÐºÐœÐžÑÐµ ÑÐœÐŸÐ²Ð°, ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐžÑ
+). ÐÑÐ»Ðž ÐŸÑÐŸÐ±ÑÐ°ÐœÐ° ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐœÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÑÐŸ ÐŒÐµÐ¶ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐŸÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº ÐœÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐµÐœ.
+<Break/>
+ ÐÐ»Ñ ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐžÑ
+ ÐŽÐµÑÐ°Ð»ÐµÐ¹ ÑÐŒ.Ð³Ð»Ð°Ð²Ñ 1 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+    </Section>
+    <Section name="classifiers">
+      <Title>ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ°. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐŸÑÐŒÐŸÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ²". 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÑÐ±ÐŸÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Ðž ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÐ»ÐµÐŽÐŸÐ²Ð°ÑÐµÐ»ÑÐœÐŸÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ²
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "ÐÑÐ±ÐŸÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ð²ÐœÐžÐ·Ñ ÐŸÐºÐœÐ°,Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐºÐ°Ð·Ð°ÐœÐœÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐµ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²".ÐÐœÑÑÑÑÐºÑÐžÐž ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° ÑÐµ Ð¶Ðµ ÑÐ°ÐŒÑÐµ,ÑÑÐŸ ÐºÐ°ÑÐ°ÑÑÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² (<Reference target="plugins">ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Reference>).ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÑÑÐŸÑÐŒÐžÑÐŸÐ²Ð°Ð»Ðž ÐœÐŸÐ²ÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ, ÐŸÐœ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° "Ð¢ÐµÐºÑÑÐžÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ".  
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°" (ÐžÐ»Ðž ÑÐµÐ»ÐºÐœÑÑÑ ÐŽÐ²Ð° ÑÐ°Ð·Ð° ÐœÐ° ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐµ Ð² ÑÐ¿ÐžÑÐºÐµ).
+<Break/>
+ ÐÐŸÑÑÐŽÐŸÐº ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² Ð² ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐŸÐœÐœÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐž ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ ÑÐŸÐŒÑ,ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐºÐ°Ð·Ð°Ðœ Ð·ÐŽÐµÑÑ.Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ, ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ, Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·".
+<Break/>
+ ÐÐ»Ñ ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐµÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² ÑÐžÑÐ°Ð¹ÑÐµ Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+    </Section>
+    <Section name="formatstatements">
+      <Title>ÐÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÑÐŸÑÐŒÐ°ÑÐ°</Title>
+ ÐÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑ ÑÑÑÑÐºÑÑÑÐŸÐ¹ Ðž Ð²ÐžÐŽÐŸÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐœÐž Ð·Ð°ÑÑÐ°Ð³ÐžÐ²Ð°ÑÑ ÑÐ°ÐºÐžÐµ Ð²ÐµÑÐž,ÐºÐ°Ðº ÐŒÐµÑÑÐŸ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÑ ÐºÐœÐŸÐ¿ÐºÐž,ÐºÐŸÐ³ÐŽÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ,Ðž ÐºÐ°ÐºÐžÐµ ÑÐ²ÑÐ·Ðž ÐŸÑÐŸÐ±ÑÐ°Ð¶Ð°ÑÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐŒ DateList. ÐÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° ÐœÐµ Ð»ÐµÐ³ÐºÐŸ ÑÐ°Ð·ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÑÑ,Ðž ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÑÐŸÑÐžÑÐ°ÑÑ Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÐ¶ÐŽÐ°ÑÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÑÐŸÑÐŒÐ°ÑÐ° Ðž ÑÐŸ, ÐºÐ°Ðº ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÐµÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÐœÐžÐŒ. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÑÐŸÑÐŒÐ°ÑÐ°".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             Ð€ÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐºÐŸÐŒÐ°ÐœÐŽ ÑÐŸÑÐŒÐ°ÑÐ°
+          </ContentsItem>
+        </ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐžÐŒÐµÐœÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ° Ðº ÑÐµÐŒÑ-ÐœÐžÐ±ÑÐŽÑ Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ "ÐÑÐ±ÐŸÑ ÑÑÐµÐŽÑÑÐ²",ÐºÐŸÑÐŸÑÑÐ¹ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐºÐ°Ð¶ÐŽÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ Ðž Ð²ÑÑÑÐŸÐµÐœÐœÑÐ¹ ÑÐ¿ÐžÑÐŸÐº ÑÑÐµÐŽÑÑÐ². ÐÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ ÑÑÐµÐŽÑÑÐ²Ð°, ÐµÑÑÑ ÐŽÐ²Ð° ÑÐžÐ¿Ð° ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ²Ð»ÑÑÑÑÑ Ð¿ÑÐŸÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐžÐ»Ðž ÐœÐµÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž,ÐŸÐœÐž ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑÑÑ ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž. ÐÑÑÐ³ÐžÐµ ÑÑÐµÐ±ÑÑÑ, ÑÑÐŸÐ±Ñ Ð±ÑÐ»Ð° ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐ° ÑÐŸÑÐŒÐ°ÑÐœÐ°Ñ ÑÑÑÐŸÐºÐ°. ÐÐ»Ñ ÐœÐžÑ
+ ÐžÐŒÐµÐµÑÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "ÐÐŸÐ·ÐŽÐµÐ¹ÑÑÐ²ÑÑÑÐžÐ¹ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ", ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ, Ðº ÐºÐ°ÐºÐŸÐ¹ ÑÐ°ÑÑÐž ÑÑÐµÐŽÑÑÐ² ÑÑÑÐŸÐºÐ° ÐŸÐ±ÑÐ°ÑÐ°ÐµÑÑÑ,ÐµÑÐ»Ðž ÑÑÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ, ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ°   ("Ð¡ÑÑÐŸÐºÐ° ÑÐŸÑÐŒÐ°ÑÐ° HTML") ÐŽÐ»Ñ Ð²Ð²ÐŸÐŽÐ° ÑÑÑÐŸÐºÐž, Ðž Ð²ÑÐ±ÐŸÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ "ÐÐµÑÐµÐŒÐµÐœÐœÑÑ
+". Ð§ÑÐŸÐ±Ñ Ð²ÑÑÐ°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ Ð² ÑÐµÐºÑÑÑÑ Ð¿ÐŸÐ·ÐžÑÐžÑ Ð² ÑÐŸÑÐŒÐ°ÑÐœÑÑ ÑÑÑÐŸÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ°  Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÑÑÐ°Ð²ÐžÑÑ". 
+<Break/>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐŽÐ»Ñ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÐ°, Ð²ÑÐ±ÐžÑÐ°Ñ Ð±Ð»Ð°ÐœÐº ÑÐ²ÐŸÐ¹ÑÑÐ². Ð­ÑÐŸÑ ÑÐŸÑÐŒÐ°Ñ Ð·Ð°ÑÐµÐŒ Ð¿ÑÐžÐŒÐµÐœÑÐµÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ Ð¿ÑÐžÐŒÐµÐœÐžÐŒÑÐŒ ÑÐ²ÐŸÐ¹ÑÑÐ²Ð°ÐŒ, ÐµÑÐ»Ðž ÐŽÑÑÐ³ÐŸÐµ ÐœÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐŸ.
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°,Ð²ÐœÐµÑÐžÑÐµ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ,ÐºÐ°Ðº ÐŸÐ±ÑÑÑÐœÐµÐœÐŸ Ð²ÑÑÐµ, Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".ÐÐŸÐ²Ð°Ñ ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÑÐŸÑÐŒÐ°ÑÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÐžÑÐºÐµ "Ð¢ÐµÐºÑÑÐžÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°".Ð¢ÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐœÐ° ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÑÐ²ÑÐ·Ð°ÐœÐ° Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÐºÐŸÐŒÐ±ÐžÐœÐ°ÑÐžÐµÐ¹ ÑÐ²ÐŸÐ¹ÑÑÐ²ÐŸ/ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ.
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".
+<Break/>
+  Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ ÑÐŸÑÐŒÐ°ÑÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÑ ÐŒÐ· ÑÐ¿ÐžÑÐºÐ°, ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÑÐŸÑÐŒÐ°Ñ".
+<Break/>
+ ÐÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹ ÐŸ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ
+ Ðž ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÐ°Ñ
+ ÑÑÐµÐŽÑÑÐ² ÐŸÐ±ÑÐ°ÑÐžÑÐµÑÑ Ðº Ð³Ð»Ð°Ð²Ðµ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ ÐÑÐ»Ðž ÐºÐœÐŸÐ¿ÐºÐ° "ÐÐŸÐ·Ð²ÐŸÐ»ÐµÐœÐœÑÐµ ÑÐ°ÑÑÐžÑÐµÐœÐœÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ" Ð¿ÐŸÐŒÐµÑÐµÐœÐ°,ÑÐŸ ÑÐ°Ð·ÑÐµÑÐµÐœÑ ÐœÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÑÐŸÐŽÐ²ÐžÐœÑÑÑÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÐžÑ . Ð¡Ð¿ÐžÑÐŸÐº ÑÑÐµÐŽÑÑÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°ÐœÑ, ÐœÐµÐŒÐœÐŸÐ³ÐŸ ÐžÐ·ÐŒÐµÐœÐµÐœ Ðž Ð±ÐŸÐ»ÑÑÐµÐµ ÑÐžÑÐ»ÐŸ Ð¿ÐµÑÐµÐŒÐµÐœÐœÑÑ
+ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐŒÐ°ÐœÐŽÐŸÐ¹ ÑÐŸÑÐŒÐ°ÑÐ°, ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°Ñ Ð±ÐŸÐ»ÑÑÐµÐµ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÑÑÐ°ÐœÐžÑÐµÐ¹ ÐŸÑÐžÐ³ÐžÐœÐ°Ð»-ÐŒÐ°ÐºÐµÑÐ°.
+    </Section>
+    <Section name="translatetext">
+      <Title>ÐÐµÑÐµÐ²ÐŸÐŽ ÑÐµÐºÑÑÐ°</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°, Ð³ÐŽÐµ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ·ÑÐº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ ÑÑÐ°Ð³ÐŒÐµÐœÑÐŸÐ² ÑÐµÐºÑÑÐ° ÐŽÐ»Ñ ÑÐ°ÑÑÐµÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ðµ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐµÑÐµÐ²ÐµÑÑÐž ÑÐµÐºÑÑ".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÐ±ÐœÐŸÐ²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ Ð²Ñ
+ÐŸÐŽ Ð² ÑÐ¿ÐžÑÐŸÐº "Ð¡Ð²ÐŸÐ¹ÑÑÐ²Ð°". ÐÐžÐ¶Ðµ Ð¿ÐŸÑÐ²ÑÑÑÑ ÑÑÑÐŸÐºÐž ÑÐ¿ÐµÑÐžÑÐžÑÐœÐŸÐ³ÐŸ ÑÐ·ÑÐºÐ°, ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ ÑÑÐžÐŒ ÑÐ²ÐŸÐ¹ÑÑÐ²ÐŸÐŒ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "Ð¯Ð·ÑÐº Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°",ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐ¹ ÑÐ·ÑÐº Ðž Ð²Ð²ÐµÑÑÐž Ð¿ÐµÑÐµÐ²ÐµÐŽÐµÐœÐœÑÐ¹ ÑÐµÐºÑÑ Ð² ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ°, ÐŸÐ±ÑÐ°ÑÐ°ÑÑÑ Ðº "ÐÐ°ÑÐ°Ð»ÑÐœÐŸÐŒÑ ÑÑÐ°Ð³ÐŒÐµÐœÑÑ ÑÐµÐºÑÑÐ°",ÐµÑÐ»Ðž ÑÑÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ. ÐÐŸ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÐž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐ¹ Ð¿ÐµÑÐµÐ²ÐŸÐŽ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ Ð¿ÐµÑÐµÐ²ÐŸÐŽÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ, ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ Ð² ÑÐµÐºÑÑÐŸÐ²ÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐž "ÐÐµÑÐµÐ²ÐµÐŽÐµÐœÐœÑÐ¹ ÑÐµÐºÑÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+    </Section>
+    <Section name="metadatasets">
+      <Title>Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title>
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ±ÑÑÑÐœÑÐµÑ ÐŸÐ±Ð·ÐŸÑÐœÑÑ Ð¿Ð°ÐœÐµÐ»Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ðµ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+".
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐŸÑÑÑÐ¿ÐœÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+          <ContentsItem>
+            ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+          <ContentsItem>
+            Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+          <ContentsItem>
+            Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð­ÑÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ,ÑÑÐŸÐ±Ñ ÑÐ°ÑÑÐŒÐŸÑÑÐµÑÑ ÐœÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹,Ðž ÑÐ»ÐµÐŒÐµÐœÑÑ,ÐºÐŸÑÐŸÑÑÐµ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐœÐ°Ð±ÐŸÑÐ°. ÐÑÐ±ÐµÑÐžÑÐµ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐŸÑÑÑÐ¿ÐœÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+",ÑÑÐŸÐ±Ñ Ð²ÐžÐŽÐµÑÑ ÐŽÐµÑÐ°Ð»Ðž ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ².
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŽÑÑÐ³ÐŸÐ¹ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð» Ñ ÐœÐ°Ð±ÐŸÑÐŸÐŒ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ (.mds).
+<Break/>
+Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐŸ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Greenstone Editor for Metadata Sets (GEMS). ÐÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÑ "Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+", ÑÑÐŸÐ±Ñ Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ GEMS.
+<Break/>
+ÐÑÐ»Ðž ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²Ð°ÐŒ Ð±ÐŸÐ»ÑÑÐµ ÐœÐµ ÐœÑÐ¶ÐµÐœ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+". ÐÑÐ»Ðž Ð²Ñ ÐœÐ°Ð·ÐœÐ°ÑÐžÐ»Ðž ÐºÐ°ÐºÐžÐµ-Ð»ÐžÐ±ÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ Ðº ÑÐ»ÐµÐŒÐµÐœÑÐ°ÐŒ Ð² ÑÐŽÐ°Ð»ÐµÐœÐœÐŸÐŒ ÐœÐ°Ð±ÐŸÑÐµ, Ñ Ð²Ð°Ñ ÑÐ¿ÑÐŸÑÑÑ ÐºÐ°Ðº Ñ ÐœÐžÐŒÐž Ð¿ÐŸÑÑÑÐ¿ÐžÑÑ Ð¿ÑÐž ÑÐ»ÐµÐŽÑÑÐµÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐºÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+    </Section>
+  </Section>
+
+  <Section name="producingthecollection">
+    <Title>ÐÐŸÑÑÑÐŸÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title>
+ ÐÐŒÐµÑ ÑÐŸÐ±ÑÐ°ÐœÐœÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÐŽÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð°ÐœÐœÐŸÑÐžÑÐŸÐ²Ð°Ð² ÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž Ðž ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÐ°Ð² ÐŽÐžÐ·Ð°Ð¹Ðœ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐµÐ¿ÐµÑÑ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Greenstone. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ.
+    <Section name="thecreateview">
+      <Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ</Title>
+<ObviousSentence> Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. </ObviousSentence>
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+	    Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+	  </ContentsItem>
+	  <ContentsItem>
+	    ÐÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÐ¹ Ð¿ÑÐŸÑÐŒÐŸÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+	  </ContentsItem>
+	</ContentsGroup>
+        <ContentsGroup linkto="advanced">
+          <ContentsItem>
+             Ð ÐµÐ³ÐžÑÑÑÐ°ÑÐžÑ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ,ÑÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ,Ð²ÑÐ¿ÐŸÐ»ÐœÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐµ ÑÑÐµÐœÐ°ÑÐžÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone ÐžÐ· ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž, ÐºÐŸÑÐŸÑÑÑ ÐÑ Ð¿ÐŸÐŽÐ³ÐŸÑÐŸÐ²ÐžÐ»Ðž. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° "ÐÐŸÑÑÑÐŸÐµÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž" ÐœÐ°ÑÐžÐœÐ°ÐµÑ Ð¿ÑÐŸÑÐµÑÑ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐµÐŒÑ, ÐºÐŸÑÐŸÑÐŸÐµ ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ Ð¿ÐŸÑÑÐµÐ±ÑÐµÑÑÑ, Ð·Ð°Ð²ÐžÑÐžÑ ÐŸÑ ÑÐ°Ð·ÐŒÐµÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž ÑÐžÑÐ»Ð° ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÐµÐŒÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ² (ÐŽÐ»Ñ ÐŸÐ³ÑÐŸÐŒÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÑÐŸ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐ°ÑÑ). ÐÐœÐŽÐžÐºÐ°ÑÐŸÑ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÑ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÐœÐ°ÑÐºÐŸÐ»ÑÐºÐŸ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœ Ð¿ÑÐŸÑÐµÑÑ. Ð§ÑÐŸÐ±Ñ ÐŸÑÐŒÐµÐœÐžÑÑ Ð¿ÑÐŸÑÐµÑÑ Ð² Ð»ÑÐ±ÐŸÐµ Ð²ÑÐµÐŒÑ, ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÑÐŒÐµÐœÐ° Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ".
+      <Break/>
+      ÐÐ°Ð¶Ð°Ð² ÐœÐ° "ÐÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÐ¹ Ð¿ÑÐŸÑÐŒÐŸÑÑ" Ð¿ÐŸÑÐ»Ðµ ÑÐŸÐ³ÐŸ, ÐºÐ°Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐ»Ð° ÑÑÐ¿ÐµÑÐœÐŸ ÑÐŸÐ±ÑÐ°ÐœÐ°, ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ Ð²ÐµÐ±-Ð±ÑÐ°ÑÐ·ÐµÑ Ñ Ð³Ð»Ð°Ð²ÐœÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+      <Anchor name="advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+      Ð ÐŒÐŸÐŽÑÑÐµ Ð­ÐºÑÐ¿ÐµÑÑ Ð²Ñ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð·Ð°Ð¿ÐžÑÑ "ÐÐŸÐ³ Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐ¹" ÑÐ»ÐµÐ²Ð°, ÑÑÐŸÐ±Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, Ð±ÑÐ»Ðž Ð»Ðž Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð¿ÐŸÐ¿ÑÑÐºÐž ÑÐºÐŸÐœÑÑÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÑÑÐ¿ÐµÑÐœÑ ÐžÐ»Ðž ÐœÐµÑ. ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÐµÐŒÑÑ Ð·Ð°Ð¿ÐžÑÑ,ÑÐµÐ»ÐºÐ°Ñ Ð¿ÐŸ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐ¹ ÐŽÐ°ÑÐµ Ð² ÑÐ¿ÐžÑÐºÐµ "ÐÑÑÐŸÑÐžÑ Ð·Ð°Ð¿ÐžÑÐž".
+    </Section>
+    <Section name="buildsettings">
+      <Title>ÐÐŒÐ¿ÐŸÑÑ Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑÐŒ ÐžÐŒÐ¿ÐŸÑÑÐ° Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ. ÐÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹ Ð¿ÐŸ ÐžÐŒÐ¿ÐŸÑÑÑ Ðž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÑÐžÑÐ°Ð¹ÑÐµ Ð³Ð»Ð°Ð²Ñ 1 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.  
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐžÐŒÐ¿ÐŸÑÑÐ°
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒÐž ÑÐŽÐµÐ»Ð°ÐœÐŸ Ð¿ÐŸÐŽÐŸÐ±ÐœÐŸ ÑÐŸÐŒÑ ÑÐ¿ÐŸÑÐŸÐ±Ñ, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ¿ÐžÑÐ°Ðœ Ð¿ÑÐž "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²" Ð² <Reference target="plugins">ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Reference>. ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ»Ñ ÑÑÐµÐ±ÑÑÑ ÑÐžÑÐ»ÐµÐœÐœÑÑ
+ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ². ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑ
+ ÐžÐ»Ðž ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°ÑÑ ÐžÐ»Ðž ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÑÑÑÐµÐ»ÐºÐž Ð²Ð²ÐµÑÑ
+ Ðž Ð²ÐœÐžÐ·, ÑÑÐŸÐ±Ñ ÑÐ²ÐµÐ»ÐžÑÐžÑÑ ÐžÐ»Ðž ÑÐŒÐµÐœÑÑÐžÑÑ ÑÐµÐºÑÑÐµÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ (Ð² ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÑÐ»ÑÑÐ°ÑÑ
+ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŸÐ³ÑÐ°ÐœÐžÑÐžÐ²Ð°ÐµÑ ÐŽÐžÐ°Ð¿Ð°Ð·ÐŸÐœ, Ð² ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐŸÐ¹ÑÐž). ÐÑÑÐ³ÐžÐµ Ð¿ÐŸÐ»Ñ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž Ð¿ÑÐž ÑÐµÐ»ÑÐºÐµ Ð¿ÐŸ ÐºÐœÐŸÐ¿ÐºÐµ (ÐÐŸÐ²ÑÐŸÑÐœÑÐ¹ ÑÐµÐ»ÑÐŸÐº ÐŽÐµÐ»Ð°ÐµÑ ÐžÑ
+ ÐœÐµÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž). 
+    </Section>
+  </Section>
+
+  <Section name="miscellaneous">
+    <Title>Ð Ð°Ð·ÐœÐŸÐµ</Title>
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°,ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÑÐ²ÑÐ·Ð°ÐœÑ ÐœÐž Ñ ÐºÐ°ÐºÐžÐŒ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐŒ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµÐŒ.
+    <Section name="preferences">
+      <Title>ÐÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÑ</Title>
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³ Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹,ÐŽÐŸÑÑÑÐ¿ÐœÑÐ¹ Ð¿ÑÐž ÐŸÑÐºÑÑÑÐžÐž "Ð€Ð°Ð¹Ð»Ð°", Ð° Ð·Ð°ÑÐµÐŒ "ÐÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹".
+      <Contents>
+        <ContentsGroup linkto="simple">
+	  <ContentsItem>
+	    ÐÐ±ÑÐµÐµ
+	  </ContentsItem>
+	  <ContentsItem>
+	    Ð¡Ð¿ÐŸÑÐŸÐ±
+	  </ContentsItem>
+          <ContentsItem>
+            Ð¢ÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐ¹ Ð¿ÑÐŸÑÐµÑÑ
+          </ContentsItem>
+          <ContentsItem>
+             Ð¡Ð²ÑÐ·Ñ
+          </ContentsItem>
+          <ContentsItem>
+             ÐÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+ÐÐµÑÐ²Ð°Ñ ÐŸÐ¿ÑÐžÑ "ÐÐ±ÑÐµÐµ" - ÑÑÐŸ ÑÐµÐºÑÑÐŸÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŽÐ»Ñ Ð²Ð²ÐµÐŽÐµÐœÐžÑ Ð°ÐŽÑÐµÑÐ° Ð²Ð°ÑÐµÐ¹ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ. ÐÐœ Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°Ðœ Ð² Ð·ÐœÐ°ÑÐµÐœÐžÐž ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ "Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ" Ðž "ÐÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÑÐ¹". Ð¡Ð»ÐµÐŽÑÑÐ°Ñ ÐŸÐ¿ÑÐžÑ - ÑÑÐŸ Ð²ÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐµ ÐŒÐµÐœÑ Ñ ÑÐ·ÑÐºÐ°ÐŒÐž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœ. ÐÑÐ»Ðž ÐÑ ÐžÐ·ÐŒÐµÐœÑÐµÑÐµ ÑÐ»ÐŸÐ²Ð°ÑÑ, Ð²ÑÐ±ÐžÑÐ°Ñ ÐºÐ°ÐºÐŸÐ¹-Ð»ÐžÐ±ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ°, ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÐµÑÐµÐ·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ, ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐœÐŸÐ²ÑÐµ ÑÑÑÐŸÐºÐž ÑÐ·ÑÐºÐ° ÐžÐ· ÑÑÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ.
+<Break/>
+ÐÑÐ»Ðž Ð²ÑÐ±ÑÐ°ÐœÐ° "ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÐŒÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+", ÑÐ°Ð·Ð»ÐžÑÐœÐŸÐµ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ, ÑÐ²ÑÐ·Ð°ÐœÐœÐŸÐµ Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž, Ð²ÑÐµÐ³ÐŽÐ° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐ»Ðž Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÐžÐ·Ð²Ð»ÐµÑÐµÐœÑ ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ². ÐÑÐŒÐµÐœÐ° ÑÑÐŸÐ³ÐŸ Ð²ÑÐ±ÐŸÑÐ° ÑÐºÑÑÐ²Ð°ÐµÑ ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, Ñ
+ÐŸÑÑ ÐŸÐœÐž Ð²ÑÑ ÐµÑÑ  ÐŽÐŸÑÑÑÐ¿ÐœÑ Ð² ÑÐµÑÐµÐœÐžÐµ ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÐºÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð² ÑÐ°ÐŒÐºÐ°Ñ
+ ÐŸÐºÐŸÐœÑÐ°ÑÐµÐ»ÑÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone. ÐÑÐ»Ðž Ð²ÑÐ±ÑÐ°ÐœÐŸ "ÐÐŸÐºÐ°Ð·Ð°ÑÑ ÑÐ°Ð·ÐŒÐµÑÑ ÑÐ°Ð¹Ð»Ð°", ÑÐŸ ÑÐ°Ð·ÐŒÐµÑ ÑÐ°Ð¹Ð»Ð° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ ÑÑÐŽÐŸÐŒ Ñ ÐºÐ°Ð¶ÐŽÑÐŒ ÑÐ°Ð¹Ð»ÐŸÐŒ Ð² ÑÐ°Ð±ÐŸÑÐµÐŒ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ðµ, Ð° ÑÐ°ÐºÐ¶Ðµ Ð² ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐŒ ÐŽÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð² Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÑ
+ Gather Ðž Enrich.
+<Break/>
+ÐÐ°ÐœÐµÐ»Ñ "Ð¡Ð¿ÐŸÑÐŸÐ±" ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÑÐŸÐ²ÐœÐµÐŒ Ð¿ÐŸÐŽÑÐŸÐ±ÐœÐŸÑÑÐµÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°. ÐÐ° ÑÐ°ÐŒÐŸÐŒ ÐœÐžÐ¶ÐœÐµÐŒ ÑÑÐŸÐ²ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐŸÐº ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ "ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ Ð¿ÐŸÐŒÐŸÑÐœÐžÐº", Ð¿ÑÐž ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐž ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÐŸ, Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ, ÑÑÐµÐ±ÑÑÑÐžÐµ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐ³ÐŸ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÑ, ÑÐºÑÑÑÑ Ðž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ ÐŒÐžÐœÐžÐŒÐ°Ð»ÑÐœÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÐŒ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ Ð·Ð°Ð¿ÐžÑÐµÐ¹. ÐÐ°ÐŸÐ±ÐŸÑÐŸÑ, ÐœÐ°ÐžÐ²ÑÑÑÐ°Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° "Ð­ÐºÑÐ¿ÐµÑÑ" ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÐµÑ ÐŽÐŸÑÑÑÐ¿ ÐºÐŸ Ð²ÑÐµÐŒ ÑÑÐµÐŽÑÑÐ²Ð°ÐŒ Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÑ, Ð²ÐºÐ»ÑÑÐ°Ñ Ð¿ÐŸÐ·ÐžÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² Ðž Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ², Ð° ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÐŸÐ»ÐœÑÐ¹ Ð²ÑÑ
+ÐŸÐŽ ÐžÐ· Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð·Ð°ÑÐžÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð² Ð¶ÑÑÐœÐ°Ð»Ðµ Ð·Ð°Ð¿ÐžÑÐµÐ¹. Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐ»Ðž Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ ÑÐ¿ÐŸÑÐŸÐ±Ñ, ÐºÐ»ÐžÐºÐœÐžÑÐµ ÑÐ°ÐŽÐžÐŸ-ÐºÐœÐŸÐ¿ÐºÑ, ÑÐ»ÐµÐŽÑÑÑÑÑ Ð·Ð° ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÐœÑÐµÑÐµÑÑÐµÑ ÐÐ°Ñ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð±ÑÑÑÑÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, ÐºÐ°ÐºÐŸÐ¹ ÑÐ¿ÐŸÑÐŸÐ± ÐÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÐµ, ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°Ñ Ð»ÐžÐœÐµÐ¹ÐºÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ Ð² ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐµ. 
+<Break/>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÑÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐŸÑÐµÑÑÑ Ð¿ÑÐž ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÐž, ÐºÐ°ÐºÐžÐµ ÐžÐ· ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐ¹ ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐ² ÑÐ²Ð»ÑÑÑÑÑ Ð²ÐžÐŽÐžÐŒÑÐŒÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "Ð¢ÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐŸÑÐµÑÑÑ", ÑÑÐŸÐ±Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ, ÐºÐ°ÐºÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž, Ð¿ÑÐŸÐ²ÐµÑÑÑ Ð¿ÐŸÐ»Ñ ÑÑÐŽÐŸÐŒ Ñ ÑÐµÐŒÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÐŒÐž, ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž. ÐÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ ÐŒÐŸÐ¶ÐœÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð²ÐœÐžÐ·Ñ ÑÐºÑÐ°ÐœÐ°,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž. ÐÐ°ÐºÑÑÑÐžÐµ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹ Ð·Ð°ÐºÑÐµÐ¿Ð»ÑÐµÑ ÑÑÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÑÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐŸÐ³ÐŸ Ð¿ÑÐŸÑÐµÑÑÐ°. Ð­ÑÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ñ
+ÑÐ°ÐœÑÑÑÑ Ð²ÐŒÐµÑÑÐµ Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹,Ð° ÐœÐµ Ð² ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°.
+<Break/>
+ÐÐ»Ð°Ð²ÐžÑÐ° "Ð¡ÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ" Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ Ð¿ÑÑÑ Ðº Ð»ÐŸÐºÐ°Ð»ÑÐœÐŸÐŒÑ ÑÐµÑÐ²ÐµÑÑ Greenstone, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸÐ³ÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐŸÐŒÐµ ÑÐŸÐ³ÐŸ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐŒÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð° Ðº ÐÐœÑÐµÑÐœÐµÑÑ Ð¿ÑÐŸÐºÑÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÑÐ°Ð¹Ð»ÐŸÐ²; ÑÐŒÐŸÑÑÐž ÑÐ°Ð·ÐŽÐµÐ» <Reference target="downloadingfiles">ÐÐ°Ð³ÑÑÐ¶Ð°ÑÑ Ð€Ð°Ð¹Ð»Ñ ÐžÐ· ÐÐœÑÐµÑÐœÐµÑÐ°</Reference> ÐŽÐ»Ñ Ð¿ÐŸÐŽÑÐŸÐ±ÐœÐŸÑÑÐµÐ¹). ÐÐŸÐŒÐµÑÑÑÐµ ÑÐ»Ð°Ð³ÐŸÐ²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ, ÑÑÐŸÐ±Ñ Ð²ÐºÐ»ÑÑÐžÑÑ ÐŒÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð°, Ðž ÑÐºÐ°Ð¶ÐžÑÐµ ÐŽÐµÑÐ°Ð»Ðž Ð°ÐŽÑÐµÑÐ° ÑÐµÑÐ²ÐµÑÐ° Ð¿ÑÐŸÐºÑÐž Ðž ÐœÐŸÐŒÐµÑ Ð¿ÐŸÑÑÐ°. ÐÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð° Ð±ÑÐŽÐµÑ Ð°ÐºÑÐžÐ²ÐžÑÐŸÐ²Ð°Ðœ Ð¿ÑÐž Ð·Ð°ÐºÑÑÑÐžÐž ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ³ÐŸ ÐŸÐºÐœÐ° ÐÐ¿ÑÐžÐ¹.
+<Break/>
+Ð ÑÐµÑÐµÐœÐžÐµ ÑÐµÑÑÐžÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ ÐŽÐ°Ð²Ð°ÑÑ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐŽÐžÑÐµÐ»ÑÐœÑÐµ ÑÐŸÐŸÐ±ÑÐµÐœÐžÑ, ÐºÐŸÑÐŸÑÑÐµ ÐžÐœÑÐŸÑÐŒÐžÑÑÑÑ ÐÐ°Ñ ÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÑ
+ ÐœÐµÐ¿ÑÐµÐŽÐ²ÐžÐŽÐµÐœÐœÑÑ
+ Ð¿ÐŸÑÐ»ÐµÐŽÑÑÐ²ÐžÑÑ
+ ÐÐ°ÑÐžÑ
+ ÐŽÐµÐ¹ÑÑÐ²ÐžÐ¹. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÑÑ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐµ, Ð²ÑÐ±ÐžÑÐ°Ñ Ð¿ÐŸÐ»Ðµ "ÐÐµ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÑÐŸ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ ÑÐœÐŸÐ²Ð°". ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐŸÐ·ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÐµ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐ°ÑÑÐžÑ
+ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ". ÐÑÐ±ÐµÑÐžÑÐµ Ð¿ÐŸÐ»Ðµ ÑÑÐŽÐŸÐŒ Ñ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐ°ÑÑÐžÐŒÐž ÑÐŸÐŸÐ±ÑÐµÐœÐžÑÐŒÐž, ÐµÑÐ»Ðž ÐÑ Ñ
+ÐŸÑÐžÑÐµ ÑÐœÐŸÐ²Ð° Ð¿ÐŸÐ»ÑÑÐ°ÑÑ ÐžÑ
+.
+     </Section>
+    <Section name="fileassociations">
+      <Title>ÐÑÑÐŸÑÐžÐ°ÑÐžÐž ÑÐ°Ð¹Ð»Ð°</Title>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐžÐºÐ»Ð°ÐŽÐœÑÐµ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ, ÑÑÐŸÐ±Ñ ÐŸÑÐºÑÑÐ²Ð°ÑÑ ÑÐ¿ÐµÑÐžÑÐžÑÐœÑÐµ ÑÐžÐ¿Ñ ÑÐ°Ð¹Ð»ÐŸÐ². Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ±ÑÑÑÐœÑÐµÑ, ÐºÐ°Ðº ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ.
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </ContentsItem>
+          <ContentsItem>
+             Ð£ÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+  Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐŸÑÐºÑÑÐ²Ð°ÑÑ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž ÑÐµÐ»ÐºÐ°ÑÑ "ÐÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ".
+<Break/>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐµÐ»ÐµÐ²ÐŸÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ° ÐžÐ»Ðž ÐœÐ°Ð¿ÐžÑÐžÑÐµ ÐœÐŸÐ²ÐŸÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ (ÐœÐµ Ð²ÐºÐ»ÑÑÐ°Ñ "."). ÐÐ°ÑÐµÐŒ ÐžÐ»Ðž ÐœÐ°Ð¿ÐžÑÐžÑÐµ Ð² ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐŒ Ð¿ÐŸÐ»Ðµ ÐºÐŸÐŒÐ°ÐœÐŽÑ,Ð·Ð°Ð¿ÑÑÐºÐ°ÑÑÑÑ Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÐŸÐµ Ð¿ÑÐžÐ»ÐŸÐ¶ÐµÐœÐžÐµ,ÐžÐ»Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿ÑÐžÐ»ÐŸÐ¶ÐµÐœÐžÐµ ÐžÐ· ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ³ÐŸ ÐŸÐºÐœÐ° "ÐÑÐŸÑÐŒÐŸÑÑ". ".%1" ÐŒÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð² ÐºÐŸÐŒÐ°ÐœÐŽÐµ Ð·Ð°Ð¿ÑÑÐºÐ°, ÑÑÐŸÐ±Ñ Ð²ÑÑÐ°Ð²ÐžÑÑ ÐžÐŒÑ ÑÐ°Ð¹Ð»Ð°, ÐºÐŸÑÐŸÑÑÐ¹ ÐŽÐŸÐ»Ð¶ÐµÐœ ÐŸÑÐºÑÑÑÑÑÑ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐŸÐœÐž Ð·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÑ,Ð¿ÑÐœÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÐž" ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒ Ðž ÐŒÐŸÐ¶ÐœÐŸ ÑÐµÐ»ÑÐºÐŸÐŒ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ.  
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°. ÐÑÐ±Ð°Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐ°Ñ ÐºÐŸÐŒÐ°ÐœÐŽÐ° ÑÐ²ÑÐ·Ðž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ Ð² Ð¿ÐŸÐ»Ðµ Ð·Ð°Ð¿ÑÑÐºÐ° ÐºÐŸÐŒÐ°ÐœÐŽ. ÐÑÑÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ ÐµÐµ Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ".
+ <Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð°ÑÑÐŸÑÐžÐ°ÑÐžÑ".(Ð Ð°ÑÑÐžÑÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐŸÑÑÐ°ÐµÑÑÑ Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ "ÐÐ»Ñ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ²").
+ <Break/>
+ ÐÑÐŸÑÐžÐ°ÑÐžÐ²ÐœÑÐµ ÑÐ°Ð¹Ð»Ñ Ñ
+ÑÐ°ÐœÑÑÑÑ Ð² Ð³Ð»Ð°Ð²ÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°, Ð² ÑÐ°Ð¹Ð»Ðµ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐŒ "associations.xml".
+    </Section>
+    <Section name="exportingcollections">
+      <Title>Ð­ÐºÑÐ¿ÐŸÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐœÐ° CD-ROM</Title>
+      
+Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ, ÐºÐ°Ðº ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð² ÑÐ°ÐŒÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÐžÐ¹ÑÑ CD-ROM Greenstone. 
+      <Contents>
+        <ContentsGroup linkto="simple">
+          <ContentsItem>
+            ÐÐ°Ðº ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ
+          </ContentsItem>
+        </ContentsGroup>
+      </Contents>
+      <Anchor name="simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</Anchor>
+Greenstone ÐŒÐŸÐ¶ÐµÑ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŸÐŽÐœÑ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ð² ÑÐ°ÐŒÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÐžÐ¹ÑÑ CD-ROM Ð¿ÐŸÐŽ Windows. Ð§ÑÐŸÐ±Ñ ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ, ÐŽÐŸÐ»Ð¶ÐµÐœ  Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ Ð¿Ð°ÐºÐµÑ "Ð­ÐºÑÐ¿ÐŸÑÑ Ð² CD-ROM". ÐÐœ ÐœÐµ Ð²ÐºÐ»ÑÑÐµÐœ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ, Ð¿ÐŸÑÑÐŸÐŒÑ ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐÐ°ÑÑ ÐžÐœÑÑÐ°Ð»Ð»ÑÑÐžÑ, ÑÑÐŸÐ±Ñ ÐµÐ³ÐŸ Ð²ÐºÐ»ÑÑÐžÑÑ.
+<Break/>
+ Ð§ÑÐŸÐ±Ñ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð­ÐºÑÐ¿ÐŸÑÑ Ð² CD-ROM". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÐŸÐ±ÑÐ°ÐœÐžÐ¹ Greenstone; ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð»ÑÐ±ÐŸÐ¹, ÑÑÐŸÐ±Ñ ÑÐ²ÐžÐŽÐµÑÑ ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ. ÐÐŸÐŒÐµÑÑÑÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐµ Ð¿ÐŸÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐŽÐ»Ñ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²Ð²ÐµÑÑÐž ÐžÐŒÑ CD-ROM Ð² Ð¿ÐŸÐ»Ðµ: ÑÑÐŸ ÑÐŸ, ÑÑÐŸ Ð¿ÐŸÑÐ²ÐžÑÑÑ Ð² ÑÑÐ°ÑÑÐŸÐ²ÐŸÐŒ ÐŒÐµÐœÑ, ÐºÐŸÐ³ÐŽÐ° CD-ROM Ð±ÑÐŽÐµÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ. ÐÐ°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð­ÐºÑÐ¿ÐŸÑÑ". ÐÑÐŸÑÐµÑÑ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž ÐŒÐŸÐ¶ÐµÑ Ð·Ð°ÐœÑÑÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐŒÐžÐœÑÑ.
+<Break/>
+ ÐÐŸÑÐ»Ðµ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÑ Greenstone Ð¿ÐŸÐºÐ°Ð¶ÐµÑ ÐžÐŒÑ Ð¿Ð°Ð¿ÐºÐž, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ¹ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð¿ÐžÑÑÑÐžÐ¹ ÐºÐŸÐŒÐ¿Ð°ÐºÑ-ÐŽÐžÑÐº ÐŽÐ»Ñ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐµÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÐœÐ° ÑÐžÑÑÑÐ¹ CD-ROM.
+    </Section>
+  </Section>
+</Document>
Index: /main/tags/3.01/gli/help/ru/help_index.xml
===================================================================
--- /main/tags/3.01/gli/help/ru/help_index.xml	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/help_index.xml	(revision 21112)
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Document><Section name="introduction"><Title>ÐÐ²ÐµÐŽÐµÐœÐžÐµ</Title><Section name="ofmiceandmenus"><Title>ÐÑÑÑ Ðž ÐÐµÐœÑ</Title></Section><Section name="howtoavoidthisdocument"><Title>ÐÐ°Ðº ÐžÐ·Ð±ÐµÐ¶Ð°ÑÑ ÑÑÐµÐœÐžÑ ÑÑÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Title></Section></Section><Section name="startingoff"><Title>ÐÐŸ ÑÑÐ°ÑÑÐ°</Title><Section name="creatingacollection"><Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title></Section><Section name="savingacollection"><Title>Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title></Section><Section name="openingacollection"><Title>ÐÑÐºÑÑÑÐžÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title></Section><Section name="deletingcollections"><Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹</Title></Section></Section><Section name="downloadingfiles"><Title>ÐÐ°Ð³ÑÑÐ·ÐžÑÑ ÑÐ°Ð¹Ð»Ñ ÐžÐ· ÐžÐœÑÐµÑÐœÐµÑ</Title><Section name="themirrorview"><Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐÐ°Ð³ÑÑÐ·ÐžÑÑ</Title></Section></Section><Section name="collectingfiles"><Title>Ð¡Ð±ÐŸÑ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŽÐ»Ñ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title><Section name="thegatherview"><Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°ÐºÐŸÐ¿ÐžÑÐµÐ»Ñ</Title></Section><Section name="creatingfolders"><Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº</Title></Section><Section name="addingfiles"><Title>ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</Title></Section><Section name="removingfiles"><Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</Title></Section><Section name="filteringthetree"><Title>Ð€ÐžÐ»ÑÑÑÐŸÐ²Ð°ÐœÐžÐµ ÐŽÐµÑÐµÐ²Ð°</Title></Section></Section><Section name="enrichingacollection"><Title>ÐÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž</Title><Section name="theenrichview"><Title>ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ</Title></Section><Section name="selectingmetadatasets"><Title>ÐÑÐ±ÐŸÑ ÐœÐ°Ð±ÐŸÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="appendingmetadata"><Title>ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="addingpreviouslydefinedmetadata"><Title>ÐÐŸÐ±Ð°Ð²Ð»ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="updatingmetadata"><Title>ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="removingmetadata"><Title>Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="reviewingmetadata"><Title>ÐÑÐŸÑÐŒÐŸÑÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section><Section name="importingpreviouslyassignedmetadata"><Title>ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section></Section><Section name="designingacollection"><Title>ÐÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title><Section name="thedesignview"><Title>ÐÐžÐŽ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°</Title></Section><Section name="generalsettings"><Title>ÐÐ±ÑÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž</Title></Section><Section name="plugins"><Title>ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</Title></Section><Section name="searchtypes"><Title>ÐÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°</Title></Section><Section name="searchindexes"><Title>ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°</Title></Section><Section name="partitionindexes"><Title>ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ</Title><Section name="definefilters"><Title>ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ</Title></Section><Section name="assignpartitions"><Title>ÐÐ°Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ</Title></Section><Section name="assignlanguages"><Title>ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÐ·ÑÐºÐž</Title></Section></Section><Section name="xcollectionsearching"><Title>Ð¡ÐŸÐ²ÐŒÐµÑÑÐœÑÐ¹ Ð¿ÐŸÐžÑÐº Ð² ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+</Title></Section><Section name="classifiers"><Title>ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ</Title></Section><Section name="formatstatements"><Title>ÐÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÑÐŸÑÐŒÐ°ÑÐ°</Title></Section><Section name="translatetext"><Title>ÐÐµÑÐµÐ²ÐŸÐŽ ÑÐµÐºÑÑÐ°</Title></Section><Section name="metadatasets"><Title>Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</Title></Section></Section><Section name="producingthecollection"><Title>ÐÐŸÑÑÑÐŸÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</Title><Section name="thecreateview"><Title>Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ</Title></Section><Section name="buildsettings"><Title>ÐÐŒÐ¿ÐŸÑÑ Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ</Title></Section></Section><Section name="miscellaneous"><Title>Ð Ð°Ð·ÐœÐŸÐµ</Title><Section name="preferences"><Title>ÐÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÑ</Title></Section><Section name="fileassociations"><Title>ÐÑÑÐŸÑÐžÐ°ÑÐžÐž ÑÐ°Ð¹Ð»Ð°</Title></Section><Section name="exportingcollections"><Title>Ð­ÐºÑÐ¿ÐŸÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐœÐ° CD-ROM</Title></Section></Section></Document>
Index: /main/tags/3.01/gli/help/ru/howtoavoidthisdocument.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/howtoavoidthisdocument.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/howtoavoidthisdocument.htm	(revision 21112)
@@ -0,0 +1,66 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="howtoavoidthisdocument"><font size="5" face="Verdana"><strong>1.2: ÐÐ°Ðº ÐžÐ·Ð±ÐµÐ¶Ð°ÑÑ ÑÑÐµÐœÐžÑ ÑÑÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+
+ ÐÐµ ÑÐžÑÐ°Ð¹ÑÐµ ÑÑÐŸÑ ÑÐµÐºÑÑ Ð¿ÐŸÐŒÐŸÑÐž Ð¿ÐŸÐ»ÐœÐŸÑÑÑÑ! ÐÐŸÑÑÐ°ÑÐŸÑÐœÐŸ Ð¿ÑÐŸÑÐµÑÑÑ, ÐºÐ°Ðº ÑÑÐŒÐµÑÑ Ð¿ÐŸÐ»ÑÑÐžÑÑ Ð¿ÐŸÐŒÐŸÑÑ, ÐºÐŸÐ³ÐŽÐ° ÐÑ ÐœÑÐ¶ÐŽÐ°ÐµÑÐµÑÑ Ð² ÑÑÐŸÐŒ.  
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+           Ð§ÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÐ°Ñ ÐºÐŸÐœÑÐµÐºÑÑÐœÐ°Ñ Ð¿ÐŸÐŒÐŸÑÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐœÑÑÑÑÐŒÐµÐœÑ Ð¿ÐŸÐŽÑÐºÐ°Ð·ÐŸÐº
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÑÐ³ÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÑÐœÐºÑÑ ÐŒÐµÐœÑ "ÐÐŸÐŒÐŸÑÑ" ÑÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÐµ ÑÐ°Ð·ÐŽÐµÐ»Ñ,ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐœÐ°ÐžÐ±ÐŸÐ»ÐµÐµ Ð°ÐŽÐµÐºÐ²Ð°ÑÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÐŸÐŒÐŸÑÐž, Ðž ÑÐŸÐ¿ÑÐŸÐ²ÐŸÐ¶ÐŽÐ°ÑÑÑÑ ÐœÐµÐ±ÐŸÐ»ÑÑÐžÐŒ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµÐŒ ÐºÐœÐžÐ³Ðž  
+<p></p>
+ÐÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐµÑÐ»Ðž ÐÑ ÑÐ°Ð·ÐŒÐµÑÐ°ÐµÑÐµ ÐŒÑÑÑ ÐœÐ° ÐœÐžÑ
+,Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ 
+"Ð¿ÐŸÐŽÑÐºÐ°Ð·ÐºÐ°",ÐºÐŸÑÐŸÑÐ°Ñ ÐŸÐ±ÑÑÑÐœÑÐµÑ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ,ÐºÐŸÑÐŸÑÑÐµ ÐŸÐœÐž Ð²ÑÐ¿ÐŸÐ»ÐœÑÑÑ  
+<p></p>
+ ÐÐµÑÐµÐŽ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ, ÑÐœÐ°ÑÐ°Ð»Ð° Ð¿ÑÐŸÑÑÐžÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÑÐžÑ Greenstone/
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/importingpreviouslyassignedmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/importingpreviouslyassignedmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/importingpreviouslyassignedmetadata.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="importingpreviouslyassignedmetadata"><font size="5" face="Verdana"><strong>5.8: ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐµ ÑÐ°ÐœÑÑÐµ. Ð ÐµÑÑ ÐžÐŽÐµÑ ÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐ»Ðž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÐ°ÐœÑÑÐµ.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐž ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑÐ»Ðž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð² ÑÐŸÑÐŒÐµ,ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°Ð²Ð°ÐµÐŒÐŸÐ¹ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ,Ð±ÑÐ»Ðž Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÑ ÑÐ°Ð¹Ð»Ñ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ,ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÐžÐ· ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone - ÐŸÐœÐž ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÑÑÑÑ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž,ÐºÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÑÐ°Ð¹Ð».Ð§ÑÐŸÐ±Ñ ÑÐŽÐµÐ»Ð°ÑÑ ÑÑÐŸ, ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ Ð² ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°Ñ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ Ð² ÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+<p></p>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð·Ð°Ð¿ÑÐ°ÑÐžÐ²Ð°ÐµÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ.ÐÐ°Ð¿ÑÐŸÑ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐºÑÐ°ÑÐºÐžÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž Ðž Ð·Ð°ÑÐµÐŒ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÐŒÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÑÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐŸÐœ Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÐžÑÑ
+ÐŸÐŽÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ.Ð­ÑÐŸ Ð¿ÐŸÐ»Ðµ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ ÐžÐ»Ðž ÐžÐ·ÐŒÐµÐœÐµÐœÐŸ.ÐÐ°ÑÐµÐŒ ÐÑ Ð²ÑÐ±ÐžÑÐ°ÐµÑÐµ,ÐºÐ°ÐºÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ,Ð° Ð·Ð°ÑÐµÐŒ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐ¹ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² ÑÑÐžÑ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°Ñ
+.Ð¡ÐžÑÑÐµÐŒÐ° Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð²ÑÐ±ÐžÑÐ°ÐµÑ ÑÐ°ÐŒÐŸÐµ Ð±Ð»ÐžÐ·ÐºÐŸÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ Ð² ÑÐµÑÐŒÐžÐœÐ°Ñ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ðž ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŽÐ»Ñ ÐœÐŸÐ²ÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. 
+<p></p>
+ ÐÑÐŸÐ²ÐµÑÐžÐ² ÑÑÐž ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐžÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ",ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº Ð²ÑÐ±ÑÐ°ÐœÐœÑÐŒ ÐœÐ°Ð±ÐŸÑÐ°ÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.Ð­ÑÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐŸÐ»ÑÐºÐŸ ÑÐŸÐ³ÐŽÐ°,ÐºÐŸÐ³ÐŽÐ° ÐŸÑÑÑÑÑÑÐ²ÑÐµÑ ÑÐ»ÐµÐŒÐµÐœÑ Ñ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð²ÑÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÑÑÐ°ÐœÐŸÐ²ÐŸÐº; "ÐÐ°ÐŒÐµÐœÐžÑÑ" ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐ°ÐµÑ Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ðº ÑÐ¶Ðµ ÑÐ°ÐœÐµÐµ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐŒÑ  Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»ÐµÐŒ;ÐœÐ°ÐºÐŸÐœÐµÑ, "ÐÐµ ÑÑÐžÑÑÐ²Ð°ÑÑ", ÑÑÐŸÐ±Ñ ÐœÐµ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐœÐžÐºÐ°ÐºÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ ÑÑÐžÐŒ ÐžÐŒÐµÐœÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑÐ°. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÐ»Ðž,ÐºÐ°Ðº ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐœÐµÐºÐŸÑÐŸÑÑÑ ÑÐ°ÑÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð·Ð°Ð¿ÐžÑÐ°ÐœÐœÐ°Ñ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÑÐŸÑ
+ÑÐ°ÐœÑÐµÑÑÑ ÐœÐ° Ð²ÑÐµÐŒÑ ÑÑÑÐµÑÑÐ²ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+<p></p>
+ ÐÐ»Ñ ÐŽÐµÑÐ°Ð»ÐµÐ¹ ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² metadata.xml, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ Ð² Greenstone ÐŽÐ»Ñ Ñ
+ÑÐ°ÐœÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÐŒ.Ð³Ð»Ð°Ð²Ñ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+ </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/introduction.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/introduction.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/introduction.htm	(revision 21112)
@@ -0,0 +1,21 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="introduction"><font size="5" face="Verdana"><strong>1: ÐÐ²ÐµÐŽÐµÐœÐžÐµ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Greenstone - ÑÑÐŸ ÐžÐœÑÑÑÑÐŒÐµÐœÑ ÐŽÐ»Ñ ÑÐ±ÐŸÑÐ° Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž
+ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ñ Ð¿ÐŸÑÐ»ÐµÐŽÑÑÑÐžÐŒ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÐžÑÑÐŸÐ²ÑÑ
+ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐº.ÐÐœ
+ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÐµÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÑÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐŒ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑÐŒ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐœÐŸÐ³ÐŸ ÐŸÐ±ÐµÑÐ¿ÐµÑÐµÐœÐžÑ
+Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž Greenstone Ð³ÑÐ°ÑÐžÑÐµÑÐºÐžÐŒ Ð¿ÑÑÐµÐŒ Ðž ÑÐ°Ð±ÐŸÑÐŸÐ¹ ÐŒÑÑÑÑ.
+    
+
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/metadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/metadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/metadatasets.htm	(revision 21112)
@@ -0,0 +1,89 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="metadatasets"><font size="5" face="Verdana"><strong>6.11: Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ±ÑÑÑÐœÑÐµÑ ÐŸÐ±Ð·ÐŸÑÐœÑÑ Ð¿Ð°ÐœÐµÐ»Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ðµ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŸÑÑÑÐ¿ÐœÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð­ÑÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ,ÑÑÐŸÐ±Ñ ÑÐ°ÑÑÐŒÐŸÑÑÐµÑÑ ÐœÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹,Ðž ÑÐ»ÐµÐŒÐµÐœÑÑ,ÐºÐŸÑÐŸÑÑÐµ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐœÐ°Ð±ÐŸÑÐ°. ÐÑÐ±ÐµÑÐžÑÐµ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐŸÑÑÑÐ¿ÐœÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+",ÑÑÐŸÐ±Ñ Ð²ÐžÐŽÐµÑÑ ÐŽÐµÑÐ°Ð»Ðž ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ².
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŽÑÑÐ³ÐŸÐ¹ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+" Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð» Ñ ÐœÐ°Ð±ÐŸÑÐŸÐŒ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ (.mds).
+<p></p>
+Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐŸ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Greenstone Editor for Metadata Sets (GEMS). ÐÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÑ "Ð ÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+", ÑÑÐŸÐ±Ñ Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ GEMS.
+<p></p>
+ÐÑÐ»Ðž ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²Ð°ÐŒ Ð±ÐŸÐ»ÑÑÐµ ÐœÐµ ÐœÑÐ¶ÐµÐœ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+". ÐÑÐ»Ðž Ð²Ñ ÐœÐ°Ð·ÐœÐ°ÑÐžÐ»Ðž ÐºÐ°ÐºÐžÐµ-Ð»ÐžÐ±ÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ Ðº ÑÐ»ÐµÐŒÐµÐœÑÐ°ÐŒ Ð² ÑÐŽÐ°Ð»ÐµÐœÐœÐŸÐŒ ÐœÐ°Ð±ÐŸÑÐµ, Ñ Ð²Ð°Ñ ÑÐ¿ÑÐŸÑÑÑ ÐºÐ°Ðº Ñ ÐœÐžÐŒÐž Ð¿ÐŸÑÑÑÐ¿ÐžÑÑ Ð¿ÑÐž ÑÐ»ÐµÐŽÑÑÐµÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐºÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/miscellaneous.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/miscellaneous.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/miscellaneous.htm	(revision 21112)
@@ -0,0 +1,18 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="miscellaneous"><font size="5" face="Verdana"><strong>8: Ð Ð°Ð·ÐœÐŸÐµ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°,ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÑÐ²ÑÐ·Ð°ÐœÑ ÐœÐž Ñ ÐºÐ°ÐºÐžÐŒ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐŒ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµÐŒ.
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/ofmiceandmenus.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/ofmiceandmenus.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/ofmiceandmenus.htm	(revision 21112)
@@ -0,0 +1,101 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="ofmiceandmenus"><font size="5" face="Verdana"><strong>1.1: ÐÑÑÑ Ðž ÐÐµÐœÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» ÐŽÐ°ÐµÑ Ð±Ð°Ð·ÐŸÐ²ÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð²Ð·Ð°ÐžÐŒÐŸÐŽÐµÐ¹ÑÑÐ²ÐžÐž Ñ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ  Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž. 
+ÐÑÐ»Ðž ÐÑ Ð·ÐœÐ°ÐºÐŸÐŒÑ Ñ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐ°ÐŒÐž ÑÐžÐ¿Ð° Internet Explorer ÐžÐ»Ðž Microsoft office, Ð»ÐµÐ³ÐºÐŸ ÐŸÐ±ÑÐ°ÑÐ°ÐµÑÐµÑÑ Ñ ÐŒÑÑÑÑ Ðž ÐŒÐµÐœÑ, Ð¿ÑÐŸÐ¿ÑÑÑÐžÑÐµ <a href="howtoavoidthisdocument.htm#howtoavoidthisdocument">Ðž Ð¿ÐµÑÐµÐ¹ÐŽÐžÑÐµ Ðº ÑÐ»ÐµÐŽÑÑÑÐµÐŒÑ ÑÐ°Ð·ÐŽÐµÐ»Ñ </a>.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÑÐœÐŸÐ²ÐœÑÐµ Ð¿ÐŸÐœÑÑÐžÑ ÐºÐŸÐœÑÐµÐ¿ÑÐžÐž
+            <tr>
+<td colspan="2"> </td><td colspan="1">
+               ÐÐµÐ¹ÑÑÐ²ÐžÑ ÐŒÑÑÐž
+	    </td>
+</tr>
+            
+<tr>
+<td colspan="2"> </td><td colspan="1">
+               ÐÐ»Ð°Ð²ÐžÐ°ÑÑÑÐ°
+	    </td>
+</tr>
+          
+</td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÐµÑÐµÑ
+ÐŸÐŽ Ðº Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐµ
+          </td>
+</tr>
+	
+	
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÑÑÑÑÑÐµ ÐºÐ»ÑÑÐž
+          </td>
+</tr>
+	
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ ÑÐŸÐ³Ð»Ð°ÑÐµÐœÐžÑÐŒ Windows Microsoft Ðž ÑÑÐµÐ±ÑÐµÑ ÐŸÐ±ÑÑÐœÐŸÐ³ÐŸ Ð·ÐœÐ°ÐœÐžÑ Windows.   
+ <p></p>
+ ÐÑÐ±Ð°Ñ ÑÐ°ÑÑÑ ÑÐºÑÐ°ÐœÐ°, Ñ ÐºÐŸÑÐŸÑÐŸÐ¹ ÐÑ Ð²Ð·Ð°ÐžÐŒÐŸÐŽÐµÐ¹ÑÑÐ²ÑÐµÑÐµ, ÑÐžÐ¿Ð° ÐºÐœÐŸÐ¿ÐºÐž ÐžÐ»Ðž ÐŸÐ±Ð»Ð°ÑÑÐž ÑÐµÐºÑÑÐ°,ÐœÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµÐŒ. Ð Ð»ÑÐ±ÐŸÐµ Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐµ Ð²ÑÐµÐŒÑ ÐŸÐŽÐžÐœ ÑÐ»ÐµÐŒÐµÐœÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÑÐ¹ "ÑÐŸÐºÑÑ",Ð²ÑÐŽÐµÐ»ÑÐµÑÑÑ Ð¿ÐŸÐŽÑÐ²ÐµÑÐºÐŸÐ¹ Ðž ÐŸÑÐºÐ»ÐžÐºÐ°ÐµÑÑÑ ÐœÐ° ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÑ. ÐÐµÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ Ð²ÑÐ±ÐžÑÐ°ÑÑ ÑÑÐ°ÑÑÐºÐž, ÐºÐŸÑÐŸÑÑÐµ Ð²ÑÐŽÐµÐ»ÑÑÑÑÑ ÑÐµÐŒÐœÐŸÑÐžÐœÐžÐŒ ÑÐ²ÐµÑÐŸÐŒ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐŸÑÐŸÐ±ÑÐ°Ð¶Ð°ÑÑÑÑ Ñ Ð¿ÐŸÐœÐžÐ¶ÐµÐœÐœÐŸÐ¹ ÑÑÐºÐŸÑÑÑÑ, Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°Ñ ÑÐµÐŒ ÑÐ°ÐŒÑÐŒ, ÑÑÐŸ ÐŸÐœÐž Ð·Ð°Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÑ.
+<p></p>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑÑÑ Ðž ÐŽÐµÐ¹ÑÑÐ²ÐŸÐ²Ð°ÑÑ  Ð»ÐµÐ²ÐŸÐ¹ ÐžÐ»Ðž Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž ÐŒÑÑÐž ÐŸÐ±ÑÑÐœÑÐŒ ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ. ÐÐœÐŸÐ³ÐžÐµ ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑÑ ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ ÐÐ°ÐŒ "ÑÑÐœÑÑÑ" ÐžÑ
+, ÐœÐ°Ð¶ÐžÐŒÐ°Ñ Ðž ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ñ Ð»ÐµÐ²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐŒÑÑÐž, Ð¿ÐµÑÐµÐŒÐµÑÐ°ÑÑ ÐžÑ
+ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐŒÑÑÐž Ðž ÐŸÑÑÐ°Ð²ÐžÑÑ ÐžÑ
+ Ð² ÐŽÑÑÐ³ÐŸÐŒ ÐŒÐµÑÑÐµ, ÐŸÑÐ¿ÑÑÐºÐ°Ñ ÐºÐœÐŸÐ¿ÐºÑ. ÐÐŸÑÐµÐœÑÐžÐ°Ð»ÑÐœÑÐµ ÐŸÐ±ÑÐµÐºÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐžÐ·ÐŒÐµÐœÑÑÑ ÑÐ²ÐŸÐ¹ Ð²ÐžÐŽ, ÐºÐŸÐ³ÐŽÐ° ÐºÐŸÐŒÐ¿ÐŸÐœÐµÐœÑ ÑÐºÐŸÐ»ÑÐ·ÐžÑ Ð¿ÐŸ ÐœÐžÐŒ.  
+<p></p>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÑ ÐŽÐ»Ñ Ð·Ð°Ð¿ÐžÑÐž Ð² ÐŸÐ±Ð»Ð°ÑÑÐž ÑÐµÐºÑÑÐ°. ÐÐ»ÑÑÐµÑÐœÐ°ÑÐžÐ²Ñ ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÐµ ÐŽÐŸÑÑÑÐ¿ÐœÑ ÐŽÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ, ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÑÑ
+ ÐºÐ»ÑÑÐµÐ²ÑÐŒ ÐžÐŒÐµÐœÐµÐŒ Ð² ÐºÐ²Ð°ÐŽÑÐ°ÑÐœÑÑ
+ ÑÐºÐŸÐ±ÐºÐ°Ñ
+. Ð Ð¿ÑÐžÐŒÐµÑÑ, [Tab] ÐžÐ·ÐŒÐµÐœÑÐµÑ ÑÐŸÐºÑÑ. ÐÐœÐ°Ðº Ð¿Ð»ÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÑÑÐŸ ÐŽÑÑÐ³ÐžÐµ ÐºÐ»Ð°Ð²ÐžÑÐž ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐœÐ°Ð¶Ð°ÑÑ ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ. 
+<p></p>
+ÐÑÐ¹ÐŽÐžÑÐµ ÐžÐ· Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°, Ð²ÑÐ±ÐžÑÐ°Ñ Ð¿ÑÐœÐºÑ "ÐÑÑ
+ÐŸÐŽ" Ð² ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»Ð°". ÐÐ°ÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐŽÐµÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ  ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐ°.
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐŽÐŸÑÑÑÐ¿ Ðº ÐŒÐµÐœÑ, ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ð¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ [ALT] Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ Ð±ÑÐºÐ²Ñ (Ð¿ÐŸÐŽÑÐµÑÐºÐœÑÑÑÑ).ÐÐ°Ð¿ÑÐžÐŒÐµÑ,ÐŽÐ»Ñ Ð²ÑÐ±ÐŸÑÐ° ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" ÐœÐ°Ð¶ÐŒÐžÑÐµ [ALT] + [F]. ÐÐ»Ñ Ð²ÑÐ±ÐŸÑÐ° ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÑÐœÐºÑÐ°, ÐœÐ°Ð¶ÐŒÐžÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, ÐµÑÐ»Ðž ÐÑ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÐµÑÑ Ð² ÐŒÐµÐœÑ Ð€Ð°Ð¹Ð» ÐœÐ°Ð¶ÐŒÐžÑÐµ [S], ÑÑÐŸÐ±Ñ "Ð¡ÐŸÑ
+ÑÐ°ÐœÐžÑÑ"ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+</body>
+</html>
Index: /main/tags/3.01/gli/help/ru/openingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/openingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/openingacollection.htm	(revision 21112)
@@ -0,0 +1,80 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="openingacollection"><font size="5" face="Verdana"><strong>2.3: ÐÑÐºÑÑÑÐžÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐŸÐŸÐ±ÑÐ°ÐµÑÑÑ,ÐºÐ°Ðº ÐŸÑÐºÑÑÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Open Collection prompt.
+     
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ°Ðº ÐŸÑÐºÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ°ÐºÑÑÑÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	     ÐÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐ°ÑÐ»ÐµÐŽÑÑÐ²Ð°
+	  </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ ÐŸÑÐºÑÑÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÑÐºÑÑÑÑ" Ð² ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»",ÑÑÐŸÐ±Ñ Ð±ÑÑÑÑÐŸ Ð¿ÐµÑÐµÐ¹ÑÐž Ðº ÐÑÐºÑÑÑÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÐÐ°ÑÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ð² Greenstone. ÐÑÐ±ÐµÑÐžÑÐµ ÐŸÐŽÐœÑ,ÑÑÐŸÐ±Ñ ÑÐ²ÐžÐŽÐµÑÑ ÐµÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ   "OÑÐºÑÑÑÑ", ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐµÐµ. ÐÑÐ»Ðž ÐÑ ÐžÑÐžÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ Ð²ÐœÐµ Ð¿Ð°Ð¿ÐºÐž "Ð¡ollect Greenstone",ÑÐŸ ÑÐµÐ»ÐºÐœÐžÑÐµ "Browse" ÐŽÐ»Ñ Ð²ÑÐ·ÐŸÐ²Ð° ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ¹ ÑÐžÑÑÐµÐŒÑ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÑÐ°Ð¹Ð»ÐŸÐ².
+ <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð ÑÐ»ÑÑÐ°Ðµ, ÐµÑÐ»Ðž Ð±ÐŸÐ»ÑÑÐµ ÑÐµÐŒ ÐŸÐŽÐœÐ° Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐ° ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Greenstone Ð²Ð²ÐŸÐŽÐžÑÑÑ ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ, ÑÐµÐ»ÐµÐ²Ð°ÐœÑÐœÑÐµ ÐŽÐžÑÐµÐºÑÑÐžÐž "Ð·Ð°ÐºÑÑÑÑ",ÑÑÐŸÐ±Ñ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐžÑÑ ÐœÐ°Ð»ÐŸÐ¶ÐµÐœÐžÐµ.ÐÑÐž ÐŸÑÐºÑÑÑÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐµÐ±ÐŸÐ»ÑÑÐŸÐ¹ Ð²ÑÐµÐŒÐµÐœÐœÑÐ¹ Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÐ¹ ÑÐ°Ð¹Ð» ÑÐŸÐ·ÐŽÐ°ÐµÑÑÑ Ð² ÐµÑ Ð¿Ð°Ð¿ÐºÐµ.ÐÐµÑÐµÐŽ ÐŸÑÐºÑÑÑÐžÐµÐŒ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑ Ð¿ÑÐŸÐ²ÐµÑÐºÑ,ÑÑÐŸÐ±Ñ ÑÐ±ÐµÐŽÐžÑÑÑÑ,ÑÑÐŸ ÐœÐžÐºÐ°ÐºÐžÑ
+ Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐœÐµ ÑÑÑÐµÑÑÐ²ÑÐµÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐºÐ°Ð·Ð°ÑÑ,Ð·Ð°ÐºÑÑÑÐ° Ð»Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, Ð¿ÐŸ ÑÐ²ÐµÑÑ ÐµÑ ÐžÐºÐŸÐœÐºÐž:Ð·ÐµÐ»ÑÐœÑÐ¹ - ÐŽÐ»Ñ ÐœÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐºÑÐ°ÑÐœÑÐ¹ - ÐŽÐ»Ñ Ð·Ð°ÐºÑÑÑÐŸÐ¹. ÐÐŽÐœÐ°ÐºÐŸ,ÐºÐŸÐ³ÐŽÐ° ÐžÐ· ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Ð²ÑÑ
+ÐŸÐŽÑÑ Ð¿ÑÐµÐ¶ÐŽÐµÐ²ÑÐµÐŒÐµÐœÐœÐŸ, Ð±Ð»ÐŸÐºÐžÑÑÑÑÐžÐ¹ ÑÐ°Ð¹Ð» ÐžÐœÐŸÐ³ÐŽÐ° ÐŸÑÑÐ°ÐµÑÑÑ ÐœÐ° ÐŒÐµÑÑÐµ. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŸÑÐºÑÑÐ²Ð°ÐµÑÐµ ÑÐ°ÐºÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ° Ð·Ð°Ð¿ÑÐ°ÑÐžÐ²Ð°ÐµÑ,Ñ
+ÐŸÑÐžÑÐµ Ð»Ðž ÐÑ "ÑÐºÑÐ°ÑÑÑ" ÐµÐ³ÐŸ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ. ÐÐžÐºÐŸÐ³ÐŽÐ° ÐœÐµ ÐŽÐµÐ»Ð°Ð¹ÑÐµ ÑÑÐŸÐ³ÐŸ,ÐµÑÐ»Ðž ÐºÑÐŸ-ÑÐŸ ÐµÑÐµ Ð² ÑÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ Ð²ÑÐµÐŒÑ ÑÐ°Ð±ÐŸÑÐ°ÐµÑ.   
+<p></p>
+ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŸÑÐºÑÑÐ²Ð°ÐµÑÐµ ÐºÐŸÐ»ÐµÐºÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐ° ÐœÐµ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°  Greenstone, ÐÐ°Ñ Ð¿ÐŸÐ¿ÑÐŸÑÑÑ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐ°Ð±ÐŸÑ (ÐžÐ»Ðž ÐœÐ°Ð±ÐŸÑÑ) ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÑÐ»Ðž ÐœÐž ÐŸÐŽÐžÐœ ÐœÐµ ÐŸÑÐŸÐ±ÑÐ°Ðœ, Ð»ÑÐ±ÑÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð±ÑÐŽÑÑ ÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÑÑÑÑ. ÐÐœÐ°ÑÐµ, ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð±ÑÐŽÑÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÑ ÑÐ°Ðº Ð¶Ðµ, ÐºÐ°Ðº Ðž ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐµÑÐµ Ð² ÑÐ°Ð¹Ð»Ñ Ñ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž. ÐÑÐŸÑÐµÑÑ ÐŸÐ¿ÐžÑÐ°Ðœ Ð² <a href="importingpreviouslyassignedmetadata.htm#importingpreviouslyassignedmetadata">ÐÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð²ÑÐ±ÑÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</a>. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/partitionindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/partitionindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/partitionindexes.htm	(revision 21112)
@@ -0,0 +1,25 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="partitionindexes"><font size="5" face="Verdana"><strong>6.6: ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÑÑÑÐŸÐµÐœÑ ÐœÐ° ÐŽÐµÑÐ°Ð»ÑÐœÐŸÐŒ ÑÐµÐºÑÑÐµ ÐžÐ»Ðž ÐžÑÑÐŸÑÐœÐžÐºÐ°Ñ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐµÑÑÐŸ Ð¿ÐŸÐžÑÐºÐ° ÐŒÐŸÐ¶ÐµÑ Ð² ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐµÐŒ ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑÑÑ Ð¿ÑÑÑÐŒ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐŸÐ² Ð»ÐžÐ±ÐŸ Ð¿ÐŸ ÑÐ·ÑÐºÑ ÐžÐ»Ðž Ð¿ÐŸ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐŒ ÑÐžÐ»ÑÑÑÐ°ÐŒ.Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ. ÐÐœÑÑÑÐž "Ð Ð°Ð·ÐŽÐµÐ»ÐŸÐ² ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ".
+<p></p>
+ÐÐžÐŽ "ÐÐœÐŽÐµÐºÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ" ÐžÐŒÐµÐµÑ ÑÑÐž Ð·ÐœÐ°ÑÐµÐœÐžÑ:"ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐžÐ»ÑÑÑÑ","ÐÐ°Ð·ÐœÐ°ÑÐžÑÑ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÐµ" Ðž "ÐÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ·ÑÐºÐž". ÐÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ Ð±ÐŸÐ»ÑÑÐµ ÑÐ·ÐœÐ°ÑÑ ÐŸ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÐž, ÑÐžÑÐ°Ð¹ÑÐµ  ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+ Ðž Ð¿ÐŸÐŽÐžÐœÐŽÐµÐºÑÐ°Ñ
+ Ð² Ð³Ð»Ð°Ð²Ðµ 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+
+      
+      
+      
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/plugins.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/plugins.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/plugins.htm	(revision 21112)
@@ -0,0 +1,101 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="plugins"><font size="5" face="Verdana"><strong>6.3: ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÐ°ÐœÐŸ,ÐºÐ°Ðº ÑÐŸÑÐŒÐžÑÐŸÐ²Ð°ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐŽÐ»Ñ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹.Ð ÐœÑÐŒ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ,ÐºÐ°ÐºÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ, ÐºÐ°ÐºÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿ÐµÑÐµÐŽÐ°Ð²Ð°ÑÑ Ðº ÐœÐžÐŒ Ðž Ð² ÐºÐ°ÐºÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ ÐŸÐœÐž Ð²Ñ
+ÐŸÐŽÑÑ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ
+ "ÐÐ»Ð°Ð³ÐžÐœÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²" Ð² Ð¿ÑÐœÐºÑÐµ "Ð Ð°Ð·ÐŽÐµÐ»Ñ Ð¿ÑÐŸÐµÐºÑÐ°".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð€ÐŸÑÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ·ÐŒÐµÐœÐµÐœÐžÐµ Ð¿ÐŸÑÑÐŽÐºÐ° Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ°ÑÑÑÐŸÐ¹ÐºÐ° ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž</a>
+<p style="text-align:justify"></p>
+Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ "ÐÑÐ±ÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐŽÐ»Ñ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ" Ð² ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐŒ ÑÐ¿ÐžÑÐºÐµ ÐŸÐºÐŸÐ»ÐŸ ÐŸÑÐœÐŸÐ²Ð°ÐœÐžÑ,Ðž Ð·Ð°ÑÐµÐŒ ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ Ñ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµÐŒ "ÐÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ²";ÐŸÐœÐŸ ÐŸÐ¿ÐžÑÐ°ÐœÐŸ ÐœÐžÐ¶Ðµ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ ÑÑÐŸÑÐŒÐžÑÐŸÐ²Ð°Ð»Ðž ÐœÐŸÐ²ÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ, ÐŸÐœ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° "Ð£ÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐµ ÑÐµÐºÑÑÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ". ÐÐ°ÐŒÐµÑÑÑÐµ, ÑÑÐŸ, ÐºÑÐŸÐŒÐµ UnknownPlug, ÐºÐ°Ð¶ÐŽÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð²ÑÐ²ÐµÐŽÐµÐœ ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐžÐœ ÑÐ°Ð· Ð² ÑÐ¿ÐžÑÐºÐµ.
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ð² ÑÐ¿ÐžÑÐºÐµ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ".
+<p></p>
+ ÐÐ»Ð°Ð³ÐžÐœÑ ÑÐŸÑÐŒÐžÑÑÑÑÑÑ Ð¿ÑÑÐµÐŒ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ². Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÑ
+, Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð»Ð°Ð³ÐžÐœ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð€ÐŸÑÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ²"(ÐžÐ»Ðž ÐŽÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ ÐœÐ° Ð¿Ð»Ð°Ð³ÐžÐœÐµ). ÐÐžÐ°Ð»ÐŸÐ³ "ÐÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐµ ÐÑÐ³ÑÐŒÐµÐœÑÑ" Ð²ÐŸÐ·ÐœÐžÐºÐ°ÐµÑ Ñ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž ÐŸÐ¿ÑÐžÑÐŒÐž ÐŽÐ»Ñ ÑÐºÐ°Ð·Ð°ÐœÐžÑ Ð°ÑÐ³ÑÐŒÐµÐœÑÐŸÐ².
+<p></p>
+ Ð¡ÑÑÐµÑÑÐ²ÑÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ Ð²ÐžÐŽÑ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ. ÐÐµÐºÐŸÑÐŸÑÑÐµ ÑÐ²Ð»ÑÑÑÑÑ ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÑÐŒÐž ÐºÐœÐŸÐ¿ÐºÐ°ÐŒÐž Ðž ÑÐµÐ»ÑÐŸÐº ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÑÑ ÐŸÐ¿ÑÐžÑ Ðº Ð¿Ð»Ð°Ð³ÐžÐœÑ. ÐÑÑÐ³ÐžÐµ Ð²ÑÑÑÑÐ¿Ð°ÑÑ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÑÑÑÐŸÐºÐž ÑÐµÐºÑÑÐ° Ñ ÐœÐµÐ·Ð°Ð²ÐžÑÐžÐŒÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ Ðž ÐŸÐ±Ð»Ð°ÑÑÑÑ ÑÐµÐºÑÑÐ°. Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð¿ÐŸÐ»Ðµ,ÑÑÐŸÐ±Ñ Ð²ÑÐ·Ð²Ð°ÑÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑ,Ð·Ð°ÑÐµÐŒ ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°Ð¹ÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐ¹ ÑÐµÐºÑÑ (Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ,Ð¿ÑÑÑ ÑÐ°Ð¹Ð»Ð° Ðž Ñ.ÐŽ.) Ð² Ð¿ÐŸÐ»Ðµ. ÐÑÑÐ³ÐžÐµ - ÑÑÐŸ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐµ ÐŒÐµÐœÑ,Ð² ÐºÐŸÑÐŸÑÑÑ
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÐžÑÐ°ÑÑ ÐœÑÐ¶ÐœÐŸÐµ ÐžÐ· Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐœÐ°Ð±ÐŸÑÐ° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐœÑÑÑ,ÑÑÐŸ ÐŽÐµÐ»Ð°ÐµÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑ, ÐŽÐ°Ð¹ÑÐµ ÐŒÑÑÐž ÐœÐ° ÐŒÐ³ÐœÐŸÐ²ÐµÐœÐžÐµ Ð¿ÑÐŸÐ¹ÑÐž Ð¿ÐŸ ÐµÐ³ÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ Ðž ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ Ð¿ÐŸÑÐ²ÐžÑÑÑ. 
+<p></p>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐžÐ·ÐŒÐµÐœÐžÐ»Ðž ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ,ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÐ", ÑÑÐŸÐ±Ñ Ð¿ÑÐžÐœÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ Ðž Ð·Ð°ÐºÑÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³,ÐžÐ»Ðž "ÐÑÐŒÐµÐœÐžÑÑ",ÑÑÐŸÐ±Ñ Ð·Ð°ÐºÑÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³ Ð±ÐµÐ· ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ Ð»ÑÐ±ÑÑ
+ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ² Ð¿Ð»Ð°Ð³ÐžÐœÐ°. 
+ <p></p>
+ ÐÐ»Ð°Ð³ÐžÐœÑ Ð² ÑÐ¿ÐžÑÐºÐµ ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÑ Ð² ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ,ÑÐ°Ðº ÐºÐ°Ðº ÑÐ¿ÐŸÑÑÐŽÐŸÑÐµÐœÐžÐµ ÐžÐœÐŸÐ³ÐŽÐ° Ð²Ð°Ð¶ÐœÐŸ. ÐÐ²Ð° Ð¿Ð»Ð°Ð³ÐžÐœÐ° - ArcPlug Ðž RecPlug - ÑÐ²Ð»ÑÑÑÑÑ Ð¶ÐžÐ·ÐœÐµÐœÐœÐŸ Ð²Ð°Ð¶ÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÑÐŸÑÐµÑÑÐ° ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ Ð² ÐºÐŸÐœÑÐµ ÑÐ¿ÐžÑÐ°ÐºÐ° (Ñ ÐŸÑÐŽÐµÐ»ÑÑÑÐµÐ¹ Ð»ÐžÐœÐžÐµÐ¹).Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÐŽÑÑÐ³ÐžÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ²,Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð»Ð°Ð³ÐžÐœ,ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ,Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·".  
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŸÑÐµÐœÑ Ñ
+ÐŸÑÐŸÑÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ,ÐºÐ°ÐºÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑ. ÐÐŽÐœÐ°ÐºÐŸ, ÐµÑÑÑ ÑÐ»ÑÑÐ°Ðž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»Ñ Ñ
+ÐŸÑÐµÑ ÑÐºÐ°Ð·Ð°ÑÑ ÑÐ¿ÐµÑÐžÐ°Ð»ÑÐœÑÐµ Ð°ÑÐ³ÑÐŒÐµÐœÑÑ. ÐÐ»Ñ ÑÑÐŸÐ³ÐŸ ÐµÑÑÑ Ð²ÐœÐžÐ·Ñ ÑÐµÐºÑÑÐŸÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ "ÐÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÐµ ÐÑÐ³ÑÐŒÐµÐœÑÑ". ÐÑÐ±ÐŸÐ¹ ÑÐµÐºÑÑ Ð² ÑÑÐŸÐŒ Ð¿ÐŸÐ»Ðµ ÐŽÐŸÑÐ»ÐŸÐ²ÐœÐŸ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÑÑ Ð² ÐºÐŸÐœÐµÑ ÐºÐŸÐŒÐ°ÐœÐŽÑ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ². 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/preferences.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/preferences.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/preferences.htm	(revision 21112)
@@ -0,0 +1,103 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="preferences"><font size="5" face="Verdana"><strong>8.1: ÐÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ ÐŽÐžÐ°Ð»ÐŸÐ³ Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹,ÐŽÐŸÑÑÑÐ¿ÐœÑÐ¹ Ð¿ÑÐž ÐŸÑÐºÑÑÑÐžÐž "Ð€Ð°Ð¹Ð»Ð°", Ð° Ð·Ð°ÑÐµÐŒ "ÐÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÐ±ÑÐµÐµ
+	  </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Ð¡Ð¿ÐŸÑÐŸÐ±
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            Ð¢ÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐ¹ Ð¿ÑÐŸÑÐµÑÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð¡Ð²ÑÐ·Ñ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÐµÑÐ²Ð°Ñ ÐŸÐ¿ÑÐžÑ "ÐÐ±ÑÐµÐµ" - ÑÑÐŸ ÑÐµÐºÑÑÐŸÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŽÐ»Ñ Ð²Ð²ÐµÐŽÐµÐœÐžÑ Ð°ÐŽÑÐµÑÐ° Ð²Ð°ÑÐµÐ¹ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ. ÐÐœ Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°Ðœ Ð² Ð·ÐœÐ°ÑÐµÐœÐžÐž ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ "Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ" Ðž "ÐÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÑÐ¹". Ð¡Ð»ÐµÐŽÑÑÐ°Ñ ÐŸÐ¿ÑÐžÑ - ÑÑÐŸ Ð²ÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐµ ÐŒÐµÐœÑ Ñ ÑÐ·ÑÐºÐ°ÐŒÐž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœ. ÐÑÐ»Ðž ÐÑ ÐžÐ·ÐŒÐµÐœÑÐµÑÐµ ÑÐ»ÐŸÐ²Ð°ÑÑ, Ð²ÑÐ±ÐžÑÐ°Ñ ÐºÐ°ÐºÐŸÐ¹-Ð»ÐžÐ±ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ°, ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÐµÑÐµÐ·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ, ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐœÐŸÐ²ÑÐµ ÑÑÑÐŸÐºÐž ÑÐ·ÑÐºÐ° ÐžÐ· ÑÑÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ.
+<p></p>
+ÐÑÐ»Ðž Ð²ÑÐ±ÑÐ°ÐœÐ° "ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÐŒÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+", ÑÐ°Ð·Ð»ÐžÑÐœÐŸÐµ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ, ÑÐ²ÑÐ·Ð°ÐœÐœÐŸÐµ Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž, Ð²ÑÐµÐ³ÐŽÐ° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐ»Ðž Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÐžÐ·Ð²Ð»ÐµÑÐµÐœÑ ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ². ÐÑÐŒÐµÐœÐ° ÑÑÐŸÐ³ÐŸ Ð²ÑÐ±ÐŸÑÐ° ÑÐºÑÑÐ²Ð°ÐµÑ ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, Ñ
+ÐŸÑÑ ÐŸÐœÐž Ð²ÑÑ ÐµÑÑ  ÐŽÐŸÑÑÑÐ¿ÐœÑ Ð² ÑÐµÑÐµÐœÐžÐµ ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÐºÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð² ÑÐ°ÐŒÐºÐ°Ñ
+ ÐŸÐºÐŸÐœÑÐ°ÑÐµÐ»ÑÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone. ÐÑÐ»Ðž Ð²ÑÐ±ÑÐ°ÐœÐŸ "ÐÐŸÐºÐ°Ð·Ð°ÑÑ ÑÐ°Ð·ÐŒÐµÑÑ ÑÐ°Ð¹Ð»Ð°", ÑÐŸ ÑÐ°Ð·ÐŒÐµÑ ÑÐ°Ð¹Ð»Ð° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑÑÑ ÑÑÐŽÐŸÐŒ Ñ ÐºÐ°Ð¶ÐŽÑÐŒ ÑÐ°Ð¹Ð»ÐŸÐŒ Ð² ÑÐ°Ð±ÐŸÑÐµÐŒ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ðµ, Ð° ÑÐ°ÐºÐ¶Ðµ Ð² ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐŒ ÐŽÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð² Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÑ
+ Gather Ðž Enrich.
+<p></p>
+ÐÐ°ÐœÐµÐ»Ñ "Ð¡Ð¿ÐŸÑÐŸÐ±" ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÑÐŸÐ²ÐœÐµÐŒ Ð¿ÐŸÐŽÑÐŸÐ±ÐœÐŸÑÑÐµÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°. ÐÐ° ÑÐ°ÐŒÐŸÐŒ ÐœÐžÐ¶ÐœÐµÐŒ ÑÑÐŸÐ²ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐŸÐº ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ "ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ Ð¿ÐŸÐŒÐŸÑÐœÐžÐº", Ð¿ÑÐž ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐž ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÐŸ, Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ, ÑÑÐµÐ±ÑÑÑÐžÐµ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐ³ÐŸ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÑ, ÑÐºÑÑÑÑ Ðž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ ÐŒÐžÐœÐžÐŒÐ°Ð»ÑÐœÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÐŒ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ Ð·Ð°Ð¿ÐžÑÐµÐ¹. ÐÐ°ÐŸÐ±ÐŸÑÐŸÑ, ÐœÐ°ÐžÐ²ÑÑÑÐ°Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° "Ð­ÐºÑÐ¿ÐµÑÑ" ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÐµÑ ÐŽÐŸÑÑÑÐ¿ ÐºÐŸ Ð²ÑÐµÐŒ ÑÑÐµÐŽÑÑÐ²Ð°ÐŒ Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÑ, Ð²ÐºÐ»ÑÑÐ°Ñ Ð¿ÐŸÐ·ÐžÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² Ðž Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÐŸÐ², Ð° ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÐŸÐ»ÐœÑÐ¹ Ð²ÑÑ
+ÐŸÐŽ ÐžÐ· Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÑÐŸÐ±Ñ Ð·Ð°ÑÐžÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð² Ð¶ÑÑÐœÐ°Ð»Ðµ Ð·Ð°Ð¿ÐžÑÐµÐ¹. Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐ»Ðž Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ ÑÐ¿ÐŸÑÐŸÐ±Ñ, ÐºÐ»ÐžÐºÐœÐžÑÐµ ÑÐ°ÐŽÐžÐŸ-ÐºÐœÐŸÐ¿ÐºÑ, ÑÐ»ÐµÐŽÑÑÑÑÑ Ð·Ð° ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÐœÑÐµÑÐµÑÑÐµÑ ÐÐ°Ñ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð±ÑÑÑÑÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, ÐºÐ°ÐºÐŸÐ¹ ÑÐ¿ÐŸÑÐŸÐ± ÐÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÐµ, ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°Ñ Ð»ÐžÐœÐµÐ¹ÐºÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ Ð² ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐµ. 
+<p></p>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÑÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐŸÑÐµÑÑÑ Ð¿ÑÐž ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÐž, ÐºÐ°ÐºÐžÐµ ÐžÐ· ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐ¹ ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐ² ÑÐ²Ð»ÑÑÑÑÑ Ð²ÐžÐŽÐžÐŒÑÐŒÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐ»Ð°Ð²ÐžÑÑ "Ð¢ÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐžÐµ Ð¿ÑÐŸÑÐµÑÑÑ", ÑÑÐŸÐ±Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ, ÐºÐ°ÐºÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž, Ð¿ÑÐŸÐ²ÐµÑÑÑ Ð¿ÐŸÐ»Ñ ÑÑÐŽÐŸÐŒ Ñ ÑÐµÐŒÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÐŒÐž, ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž. ÐÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ ÐŒÐŸÐ¶ÐœÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð²ÐœÐžÐ·Ñ ÑÐºÑÐ°ÐœÐ°,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž. ÐÐ°ÐºÑÑÑÐžÐµ ÐŽÐžÐ°Ð»ÐŸÐ³Ð° Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐ¹ Ð·Ð°ÐºÑÐµÐ¿Ð»ÑÐµÑ ÑÑÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž ÑÐµÑ
+ÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐŸÐ³ÐŸ Ð¿ÑÐŸÑÐµÑÑÐ°. Ð­ÑÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐž Ñ
+ÑÐ°ÐœÑÑÑÑ Ð²ÐŒÐµÑÑÐµ Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐµÐ¹,Ð° ÐœÐµ Ð² ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ°.
+<p></p>
+ÐÐ»Ð°Ð²ÐžÑÐ° "Ð¡ÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ" Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ Ð¿ÑÑÑ Ðº Ð»ÐŸÐºÐ°Ð»ÑÐœÐŸÐŒÑ ÑÐµÑÐ²ÐµÑÑ Greenstone, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸÐ³ÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐŸÐŒÐµ ÑÐŸÐ³ÐŸ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐŒÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð° Ðº ÐÐœÑÐµÑÐœÐµÑÑ Ð¿ÑÐŸÐºÑÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÑÐ°Ð¹Ð»ÐŸÐ²; ÑÐŒÐŸÑÑÐž ÑÐ°Ð·ÐŽÐµÐ» <a href="downloadingfiles.htm#downloadingfiles">ÐÐ°Ð³ÑÑÐ¶Ð°ÑÑ Ð€Ð°Ð¹Ð»Ñ ÐžÐ· ÐÐœÑÐµÑÐœÐµÑÐ°</a> ÐŽÐ»Ñ Ð¿ÐŸÐŽÑÐŸÐ±ÐœÐŸÑÑÐµÐ¹). ÐÐŸÐŒÐµÑÑÑÐµ ÑÐ»Ð°Ð³ÐŸÐ²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ, ÑÑÐŸÐ±Ñ Ð²ÐºÐ»ÑÑÐžÑÑ ÐŒÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð°, Ðž ÑÐºÐ°Ð¶ÐžÑÐµ ÐŽÐµÑÐ°Ð»Ðž Ð°ÐŽÑÐµÑÐ° ÑÐµÑÐ²ÐµÑÐ° Ð¿ÑÐŸÐºÑÐž Ðž ÐœÐŸÐŒÐµÑ Ð¿ÐŸÑÑÐ°. ÐÐŸÐŽÑÐ»Ñ ÐŽÐŸÑÑÑÐ¿Ð° Ð±ÑÐŽÐµÑ Ð°ÐºÑÐžÐ²ÐžÑÐŸÐ²Ð°Ðœ Ð¿ÑÐž Ð·Ð°ÐºÑÑÑÐžÐž ÐŽÐžÐ°Ð»ÐŸÐ³ÐŸÐ²ÐŸÐ³ÐŸ ÐŸÐºÐœÐ° ÐÐ¿ÑÐžÐ¹.
+<p></p>
+Ð ÑÐµÑÐµÐœÐžÐµ ÑÐµÑÑÐžÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐŒÐŸÐ¶ÐµÑ ÐŽÐ°Ð²Ð°ÑÑ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐŽÐžÑÐµÐ»ÑÐœÑÐµ ÑÐŸÐŸÐ±ÑÐµÐœÐžÑ, ÐºÐŸÑÐŸÑÑÐµ ÐžÐœÑÐŸÑÐŒÐžÑÑÑÑ ÐÐ°Ñ ÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÑ
+ ÐœÐµÐ¿ÑÐµÐŽÐ²ÐžÐŽÐµÐœÐœÑÑ
+ Ð¿ÐŸÑÐ»ÐµÐŽÑÑÐ²ÐžÑÑ
+ ÐÐ°ÑÐžÑ
+ ÐŽÐµÐ¹ÑÑÐ²ÐžÐ¹. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð·Ð°Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÑÑ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐµ, Ð²ÑÐ±ÐžÑÐ°Ñ Ð¿ÐŸÐ»Ðµ "ÐÐµ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÑÐŸ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ ÑÐœÐŸÐ²Ð°". ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐŸÐ·ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸÑÐ²Ð»ÐµÐœÐžÐµ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐ°ÑÑÐžÑ
+ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐºÐ»Ð°Ð²ÐžÑÑ "ÐÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐµÐœÐžÐµ". ÐÑÐ±ÐµÑÐžÑÐµ Ð¿ÐŸÐ»Ðµ ÑÑÐŽÐŸÐŒ Ñ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐ°ÑÑÐžÐŒÐž ÑÐŸÐŸÐ±ÑÐµÐœÐžÑÐŒÐž, ÐµÑÐ»Ðž ÐÑ Ñ
+ÐŸÑÐžÑÐµ ÑÐœÐŸÐ²Ð° Ð¿ÐŸÐ»ÑÑÐ°ÑÑ ÐžÑ
+.
+     </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/producingthecollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/producingthecollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/producingthecollection.htm	(revision 21112)
@@ -0,0 +1,18 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="producingthecollection"><font size="5" face="Verdana"><strong>7: ÐÐŸÑÑÑÐŸÐµÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+ ÐÐŒÐµÑ ÑÐŸÐ±ÑÐ°ÐœÐœÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÐŽÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð°ÐœÐœÐŸÑÐžÑÐŸÐ²Ð°Ð² ÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž Ðž ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÐ°Ð² ÐŽÐžÐ·Ð°Ð¹Ðœ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐµÐ¿ÐµÑÑ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Greenstone. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÑÐŸ ÑÐŽÐµÐ»Ð°ÑÑ.
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/removingfiles.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/removingfiles.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/removingfiles.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingfiles"><font size="5" face="Verdana"><strong>4.4: Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ,ÐºÐ°Ðº ÑÐŽÐ°Ð»ÑÑÑ ÑÐ°Ð¹Ð»Ñ Ðž Ð¿Ð°Ð¿ÐºÐž ÐžÐ· ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.  
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐœÐŸÐ¿ÐºÐ° ÑÐŽÐ°Ð»ÐµÐœÐžÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ»Ð°Ð²ÐžÑÐ° ÑÐŽÐ°Ð»ÐµÐœÐžÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð¿ÑÑÐµÐŒ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ Ðž ÐŸÑÐ¿ÑÑÐºÐ°ÐœÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð¡ÑÑÐµÑÑÐ²ÑÐµÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐŒÐµÑÐŸÐŽÐŸÐ² ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž Ð¿Ð°Ð¿ÐŸÐº. ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÑÐœÐ°ÑÐ°Ð»Ð° ÑÐºÐ°Ð·Ð°ÑÑ,ÐºÐ°ÐºÐžÐµ ÐžÐ· ÐœÐžÑ
+ ÑÐŽÐ°Ð»ÑÑÑ, Ð²ÑÐ±ÐžÑÐ°Ñ ÐŸÐŽÐžÐœ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž Ð¿Ð°Ð¿ÐŸÐº, ÐºÐ°Ðº ÐŸÐ¿ÐžÑÐ°ÐœÐŸ Ð² <a href="thegatherview.htm#thegatherview">ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÐ¹</a>.
+<p></p>
+ ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÐ°Ð¹Ð»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÑ, ÐŽÐ»Ñ ÐžÑ
+ ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐµÐ»ÐºÐ°ÑÑ ÐºÐœÐŸÐ¿ÐºÑ "Ð£ÐŽÐ°Ð»ÐžÑÑ"ÐžÐ»Ðž ÐœÐ°Ð¶ÐžÐŒÐ°ÑÑ ÐºÐºÐ»Ð°Ð²ÐžÑÑ [Ð£ÐŽÐ°Ð»ÐžÑÑ] ÐœÐ° ÐºÐ»Ð°Ð²ÐžÐ°ÑÑÑÐµ, ÐžÐ»Ðž Ð¶Ðµ ÑÑÐœÑÑ ÐžÑ
+ ÐžÐ· ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŽÐŸ ÐºÐœÐŸÐ¿ÐºÐž "Ð£ÐŽÐ°Ð»ÐžÑÑ" Ðž ÑÐ°ÐŒ ÐŸÑÐ¿ÑÑÐºÐ°ÑÑ.  
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/removingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/removingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/removingmetadata.htm	(revision 21112)
@@ -0,0 +1,62 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="removingmetadata"><font size="5" face="Verdana"><strong>5.6: Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÐŽÐ°Ð»ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ².
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ² 
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐžÐ· Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²Ð° ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÑÐ¿ÐŸÐ»ÐœÑÐµÑÑÑ ÑÐ°ÐºÐžÐŒ Ð¶Ðµ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ,ÐºÐ°Ðº Ðž ÐžÑ
+ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ. Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð» ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð°,Ð·Ð°ÑÐµÐŒ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÐ°Ð±Ð»ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÐµ. ÐÑÐ»Ðž Ðº ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÐŸÐŒÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐœÐŸÐ¿ÐºÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ" Ð°ÐºÑÐžÐ²ÐžÑÑÐµÑÑÑ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐœÐµÐ¹,ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÑÐºÐ°Ð·Ð°ÐœÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°. ÐÑÑÐ³ÐžÐµ ÑÐ°Ð¹Ð»Ñ ÐŸÑÑÐ°ÑÑÑÑ ÐœÐµÐžÐ·ÐŒÐµÐœÐœÑÐŒÐž, Ðž ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŸÑÑÐ°ÑÑÑÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.
+<p></p>
+ÐÑÐŸÑÐµÑÑ ÑÐŽÐ°Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ñ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/reviewingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/reviewingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/reviewingmetadata.htm	(revision 21112)
@@ -0,0 +1,97 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="reviewingmetadata"><font size="5" face="Verdana"><strong>5.7: ÐÑÐŸÑÐŒÐŸÑÑ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ð²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ Ð¿ÐŸÐ»ÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐœÑÑÐŸÐ»ÑÐœÐ°Ñ ÑÐ°Ð±Ð»ÐžÑÐ°
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐ°Ð²Ð°Ñ ÐºÐœÐŸÐ¿ÐºÐ° ÐŒÐµÐœÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ²ÑÐŸÑÐžÐ»ÑÑÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐœÐŸÐ³ÐŽÐ° ÐÐ°ÐŒ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ,Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ ÐŒÐœÐŸÐ³ÐžÑ
+ ÐžÐ»Ðž Ð²ÑÐµÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÑÐ°Ð·Ñ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ,ÑÑÐŸÐ±Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ,ÑÐºÐŸÐ»ÑÐºÐŸ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŸÑÑÐ°ÑÑÑÑ ÐŽÐ»Ñ Ð²ÐŸÐ·ÐŽÐµÐ¹ÑÑÐ²ÐžÑ ÐžÐ»Ðž Ð¿ÐŸÐ»ÑÑÐžÑÑ ÐœÐµÐºÐŸÑÐŸÑÑÐµ ÐžÐŽÐµÐž ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÐŸ ÑÐ°ÑÐ¿ÑÐŸÑÑÑÐ°ÐœÐµÐœÐžÑ ÐŽÐ°ÐœÐœÑÑ
+.
+<p></p>
+ÐÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð»Ñ, ÐºÐŸÑÐŸÑÑÐµ Ð²Ñ Ñ
+ÐŸÑÐžÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, Ð¿ÐŸÑÐŸÐŒ ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° Ð¿ÑÐ°Ð²ÑÑ ÐºÐœÐŸÐ¿ÐºÑ ÐŒÑÑÐž Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐ°Ð·ÐœÐ°ÑÐµÐœÑÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ...". ÐÐŸÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐºÐœÐŸ,ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÐŸÐµ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ", Ð·Ð°ÐœÑÑÐŸÐµ Ð±ÐŸÐ»ÑÑÐŸÐ¹ ÑÐ°Ð±Ð»ÐžÑÐµÐ¹,ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ¹ ÐŒÐœÐŸÐ³ÐŸ ÐºÐŸÐ»ÐŸÐœÐŸÐº.ÐÐµÑÐ²Ð°Ñ ÐºÐŸÐ»ÐŸÐœÐºÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÐŒÐµÐœÐ° ÑÐ°Ð¹Ð»ÐŸÐ²;ÑÑÑÐŸÐºÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ Ð²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÑÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².  
+ <p></p>
+ Ð ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐ°Ð±Ð»ÐžÑÑ ÐŒÐŸÐ¶ÐµÑ Ð·Ð°ÐœÑÑÑ ÐœÐµÐºÐŸÑÐŸÑÐŸÐµ Ð²ÑÐµÐŒÑ,ÐµÑÐ»Ðž Ð²ÑÐ¿Ð°Ð»ÐŸ ÐŒÐœÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ².ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐŸÐŽÐŸÐ»Ð¶Ð°ÑÑ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐŒ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐŸÐŒ Ð¿ÑÐž ÐŸÑÐºÑÑÑÐŸÐŒ ÐŸÐºÐœÐµ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ". 
+<p></p>
+Ð©ÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐºÑÑÑÑ", ÑÑÐŸÐ±Ñ ÑÐºÑÑÑÑ ÐŸÐºÐœÐŸ.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐ°ÐºÐ¶Ðµ ÑÐ°ÑÑÐŒÐŸÑÑÐµÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð²ÑÐ±ÐžÑÐ°Ñ ÑÐµ ÑÐ°Ð¹Ð»Ñ, ÐºÐŸÑÐŸÑÑÐµ ÐÑ Ð¶ÐµÐ»Ð°ÐµÑÐµ ÐžÐ·ÑÑÐžÑÑ, ÑÐµÐ»ÐºÐœÑÐ² Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ Ðž Ð²ÑÐ±ÐžÑÐ°Ñ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ".ÐÑÐ»Ðž Ð±ÑÐ»Ð° Ð²ÑÐ±ÑÐ°ÐœÐ° Ð¿Ð°Ð¿ÐºÐ°,ÑÐŸ Ð²ÑÐµ ÐµÑ ÐŽÐŸÑÐµÑÐœÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² ÑÐ°Ð±Ð»ÐžÑÑ.  
+<p></p>
+ ÐÐŸÐ³ÐŽÐ° ÐŸÐœÐ° ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ ÑÐ»ÐžÑÐºÐŸÐŒ Ð±ÐŸÐ»ÑÑÐŸÐ¹,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÑÑÐžÐ»ÑÑÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð¿ÑÐžÐŒÐµÐœÑÑ ÑÐžÐ»ÑÑÑÑ Ðº ÐºÐŸÐ»ÐŸÐœÐºÐ°ÐŒ.ÐÐŸÑÐºÐŸÐ»ÑÐºÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ ÐœÐŸÐ²ÑÐµ ÑÐžÐ»ÑÑÑÑ, ÑÐŸÐ»ÑÐºÐŸ ÑÐµ ÑÑÑÐŸÐºÐž,ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑ ÐžÐŒ,ÐŸÑÑÐ°ÑÑÑÑ Ð²ÐžÐŽÐžÐŒÑÐŒÐž.Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ,ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐ»Ðž ÐŸÑÐžÑÑÐžÑÑ ÑÐžÐ»ÑÑÑ,ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÐŸ ÐžÐºÐŸÐœÐºÐµ "ÐÐŸÑÐŸÐœÐºÐ°" ÐœÐ°Ð²ÐµÑÑ
+Ñ ÐºÐŸÐ»ÐŸÐœÐºÐž. Ð£ Ð²Ð°Ñ Ð±ÑÐŽÐµÑ Ð·Ð°Ð¿ÑÐŸÑÐµÐœÐ° ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ ÑÐžÐ»ÑÑÑÐµ. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÐžÐ»ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ,ÐžÐ·ÐŒÐµÐœÑÐµÑÑÑ ÑÐ²ÐµÑ ÐºÐŸÐ»ÐŸÐœÐºÐž.
+<p></p>
+ ÐÐ°Ð¿ÑÐŸÑ ÐžÐŒÐµÐµÑ ÑÐžÐŒÐ²ÐŸÐ»Ñ "ÐÑÐŸÑÑÐŸÐ¹" Ðž "ÐÑÐŸÐŽÐ²ÐžÐœÑÑÑÐ¹".ÐÑÐŸÑÑÐ°Ñ Ð²ÐµÑÑÐžÑ ÑÐžÐ»ÑÑÑÑÐµÑ ÐºÐŸÐ»ÐŸÐœÐºÐž ÑÐ°Ðº,ÑÑÐŸ ÐŸÐœÐž Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÑÐŸÐ»ÑÐºÐŸ ÑÑÑÐŸÐºÐž, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ("*" ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð²ÑÐµÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ).ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ· ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐµÐ³ÐŸ ÑÐ¿ÐžÑÐºÐ°.ÐÑÐŸÐŽÐ²ÐžÐœÑÑÐ°Ñ Ð²ÐµÑÑÐžÑ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð²ÑÐ¿ÐŸÐ»ÐœÑÑÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ:ÐÐŸÐ»Ð¶ÐµÐœ ÐœÐ°ÑÐ°ÑÑÑÑ Ñ,ÐÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ,Ð Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ,ÐÐµÐœÑÑÐµ ÑÐµÐŒ, Ð­ÐºÐ²ÐžÐ²Ð°Ð»ÐµÐœÑÐµÐœ. ÐÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð±ÑÐŽÐµÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐŸÐ²Ð°ÑÑ, ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ ÑÐ°Ðº,ÑÑÐŸÐ±Ñ Ð±ÑÑÑ Ð»ÑÐ±ÐŸÐ¹ ÑÑÑÐŸÐºÐŸÐ¹,Ð²ÐºÐ»ÑÑÐ°Ñ "*". ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÑÐ±ÑÐ°ÑÑ,ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð»Ðž ÑÑÐŸ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ Ð±ÑÑÑ ÐœÐµ ÑÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÑÐŒ ÑÐ»ÑÑÐ°ÐµÐŒ. ÐÐ°ÐºÐŸÐœÐµÑ,ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð²ÑÐŸÑÐŸÐµ ÑÑÐ»ÐŸÐ²ÐžÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ,ÐºÐŸÑÐŸÑÐŸÐµ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŽÐ»Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÐ°ÐœÐ³Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ (Ð²ÑÐ±ÐžÑÐ°Ñ Ð Ð±ÐŸÐ»ÑÑÐŸÐµ) ÐžÐ»Ðž Ð°Ð»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÑÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ (Ð²ÑÐ±ÐžÑÐ°Ñ ÐÐÐ).ÐÐžÐ¶Ðµ ÑÑÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐž ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ Ð¿ÐŸÐ»Ðµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐÐ°ÐŒ ÐžÐ·ÐŒÐµÐœÑÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž (Ð²ÐŸÐ·ÑÐ°ÑÑÐ°ÑÑÐžÐ¹ ÐžÐ»Ðž Ð¿ÐŸÐœÐžÐ¶Ð°ÑÐžÐ¹ÑÑ).ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐÑ Ð·Ð°ÐºÐŸÐœÑÐžÑÐµ,ÐœÐ°Ð¶ÐŒÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐžÐ»ÑÑÑÐ°",ÑÑÐŸÐ±Ñ Ð¿ÑÐžÐŒÐµÐœÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÑÐžÐ»ÑÑÑ Ðº ÐºÐŸÐ»ÐŸÐœÐºÐµ.ÐÐ°Ð¶ÐŒÐžÑÐµ "ÐÑÐžÑÑÐžÑÑ ÑÐžÐ»ÑÑÑ",ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÐµÐºÑÑÐžÐ¹ ÑÐžÐ»ÑÑÑ.ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ ÐœÐ° ÑÐŸ,ÑÑÐŸ ÐŽÐµÑÐ°Ð»Ðž ÑÐžÐ»ÑÑÑÐ° ÑÐŸÑ
+ÑÐ°ÐœÑÑÑÑÑ, ÐŽÐ°Ð¶Ðµ ÐºÐŸÐ³ÐŽÐ° ÑÐžÐ»ÑÑÑ ÐŸÑÐžÑÐµÐœ.
+ <p></p>
+ÐÐ°Ð¿ÑÐžÐŒÐµÑ,ÑÑÐŸÐ±Ñ ÐŸÑÑÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐ°Ð±Ð»ÐžÑÑ "ÐÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ",Ð²ÑÐ±ÐµÑÐžÑÐµ ÐºÐŸÐ»ÐŸÐœÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÑÐ°ÐœÐŸÐ²ÐºÑ ÑÐžÐ»ÑÑÑÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ (Ð¿ÑÐŸÑÑÐŸÐ¹ ÑÐžÐ»ÑÑÑ ÐœÐ° "*") Ðž Ð²ÐŸÐ·ÑÐ°ÑÑÐ°ÑÑÐžÐ¹ ÐžÐ»Ðž Ð¿ÐŸÐœÐžÐ¶Ð°ÑÑÐžÐ¹ÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº.   
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/savingacollection.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/savingacollection.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/savingacollection.htm	(revision 21112)
@@ -0,0 +1,77 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="savingacollection"><font size="5" face="Verdana"><strong>2.2: Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ðž ÐºÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ ÐµÑ ÐœÐµÐ·Ð°ÐŒÐµÐŽÐ»ÐžÑÐµÐ»ÑÐœÐŸ. 
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ°Ðº ÑÐŸÑ
+ÑÐ°ÐœÐžÑÑ
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŽÐµ ÑÐ°Ð¹Ð»Ñ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð¡ÐŸÑ
+ÑÐ°ÐœÑÐ¹ÑÐµ ÐÐ°ÑÑ ÑÐ°Ð±ÐŸÑÑ ÑÐµÐ³ÑÐ»ÑÑÐœÐŸ, ÐŸÑÐºÑÑÐ²Ð°Ñ ÐŒÐµÐœÑ "Ð€Ð°Ð¹Ð»" Ðž Ð²ÑÐ±ÐžÑÐ°Ñ Ð² ÐœÑÐŒ Ð¿ÑÐœÐºÑ "Ð¡ÐŸÑ
+ÑÐ°ÐœÐžÑÑ". Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž - ÑÑÐŸ ÐœÐµ ÑÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ, ÐºÐ°Ðº  ÑÐŽÐµÐ»Ð°ÑÑ ÐµÑ Ð³ÐŸÑÐŸÐ²ÐŸÐ¹ Ðº ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ð² Greenstone.
+ (see <a href="producingthecollection.htm#producingthecollection">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž</a>).
+<p></p>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð·Ð°ÑÐžÑÐ°ÐµÑ ÐÐ°ÑÑ ÑÐ°Ð±ÐŸÑÑ Ð¿ÑÑÑÐŒ ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐžÑ ÐµÑ Ð²ÑÑÐºÐžÐ¹ ÑÐ°Ð·, ÐºÐŸÐ³ÐŽÐ° ÐÑ Ð²ÑÑ
+ÐŸÐŽÐžÑÐµ ÐžÐ· Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ ÐžÐ»Ðž Ð·Ð°Ð³ÑÑÐ¶Ð°ÐµÑÐµ ÐŽÑÑÐ³ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð¡ÐŸÑ
+ÑÐ°ÐœÐµÐœÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð·Ð°Ð¿ÐžÑÑÐ²Ð°ÑÑÑÑ  Ð² ÑÐ°Ð¹Ð», ÐžÐŒÐµÑÑÐžÐ¹ ÑÐŸ Ð¶Ðµ ÐžÐŒÑ, ÑÑÐŸ Ðž ÐžÐŒÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ ".col", Ðž ÑÐ°ÑÐ¿ÐŸÐ»Ð°Ð³Ð°ÐµÑÑÑ Ð² Ð¿Ð°Ð¿ÐºÐµ ÑÐŸÐ³ÐŸ Ð¶Ðµ ÐœÐ°ÐžÐŒÐµÐœÐŸÐ²Ð°ÐœÐžÑ, ÑÑÐŸ Ðž Ð¿Ð°Ð¿ÐºÐ° "collect" Ð¿ÑÐž ÐžÐœÑÑÐ°Ð»Ð»ÑÑÐžÐž Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/searchindexes.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/searchindexes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/searchindexes.htm	(revision 21112)
@@ -0,0 +1,85 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchindexes"><font size="5" face="Verdana"><strong>6.5: ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ ÐÐœÐŽÐµÐºÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑ,ÐºÐ°ÐºÐžÐµ ÑÐ°ÑÑÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÑÐ²Ð»ÑÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐŽÐ»Ñ Ð¿ÐŸÐžÑÐºÐ°. Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ðž ÑÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑÑ,Ð° ÑÐ°ÐºÐ¶Ðµ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ.ÐÐŸÐŽ "Ð Ð°Ð·ÐŽÐµÐ»Ð°ÐŒÐž ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ ÐžÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ
+          </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÐœÐŽÐµÐºÑ
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐžÐœÐŽÐµÐºÑÐ° Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ,Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ ÐœÐµÐ³ÐŸ Ð² Ð¿ÐŸÐ»Ðµ "ÐÐŒÑ ÐžÐœÐŽÐµÐºÑÐ°".ÐÑÐ±ÐµÑÐžÑÐµ, ÐºÐ°ÐºÐžÐµ ÐžÐ· Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÑÑ
+ ÐžÑÑÐŸÑÐœÐžÐºÐŸÐ² ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÐžÑÑ Ðº ÐžÐœÐŽÐµÐºÑÑ, ÑÐµÐ»ÐºÐ°Ñ ÐœÐ° Ð¿ÐŸÐ»Ðµ ÐŸÐºÐŸÐ»ÐŸ ÐœÐžÑ
+. Ð¡Ð¿ÐžÑÐŸÐº Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð·Ð°ÐŽÐ°ÐœÐœÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð° ÑÐ°ÐºÐ¶Ðµ Ð¿ÐŸÐ»ÐœÑÐ¹ ÑÐµÐºÑÑ. Ð Ð°ÑÐ¿ÐŸÐ»Ð°Ð³Ð°Ñ Ð²ÑÐ±ÑÐ°ÐœÐœÑÐŒÐž ÐžÑÑÐŸÑÐœÐžÐºÐ°ÐŒÐž ÐŽÐ°ÐœÐœÑÑ
+, Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÑÐµÐ¿ÐµÐœÑ ÐŽÐµÑÐ°Ð»ÐžÐ·Ð°ÑÐžÐž ÐžÐœÐŽÐµÐºÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÐŒÐµÐœÑ "ÐÐ° ÑÑÐŸÐ²ÐœÐµ". ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÑÑÐž ÐŽÐµÑÐ°Ð»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ,Ð¿ÑÐœÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÐœÐŽÐµÐºÑ" ÑÑÐ°ÐœÐŸÐ²ÐžÑÑÑ Ð°ÐºÑÐžÐ²ÐœÑÐŒ (ÐµÑÐ»Ðž ÐœÐµÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÐžÐœÐŽÐµÐºÑÐ° Ñ ÑÐµÐŒÐž Ð¶Ðµ ÑÐ°ÐŒÑÐŒÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ°ÐŒÐž).Ð©ÐµÐºÐœÐžÑÐµ ÐœÐ° ÐœÑÐŒ, ÑÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐŸÐ²ÑÐ¹ ÐžÐœÐŽÐµÐºÑ.
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÐœÐŽÐµÐºÑ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ðž ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÐŽÐµÑÐ°Ð»Ðž. ÐÐŸÐŽÑÐ²ÐµÑÐŽÐžÑÐµ, ÐœÐ°Ð¶Ð°Ð² ÐœÐ° "ÐÐ°ÐŒÐµÐœÐžÑÑ ÐžÐœÐŽÐµÐºÑ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ² Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÐžÐœÐŽÐµÐºÑ".
+<p></p>
+ ÐÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ - ÑÐŸÑ, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐœÐ° Ð¿ÐŸÐžÑÐºÐŸÐ²ÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž,Ð¿ÐŸÐŒÐµÑÐµÐœ ÑÑÐ³Ð°ÐŒÐž "[ÐÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ]" Ð² ÑÐ¿ÐžÑÐºÐµ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐœÐŽÐµÐºÑÑ".Ð§ÑÐŸÐ±Ñ ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐµÐ³ÐŸ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐžÐœÐŽÐµÐºÑ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ".
+<p></p>
+ ÐÑÐ»Ðž Ð¿ÐŸÐ·Ð²ÐŸÐ»ÐµÐœ ÑÐ°ÑÑÐžÑÐµÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº (ÑÐµÑÐµÐ· Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°), ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÐœÐŽÐµÐºÑÐŸÐŒ ÐŸÑÐ»ÐžÑÐ°ÐµÑÑÑ.ÐÐ°Ð¶ÐŽÑÐ¹ ÐžÐœÐŽÐµÐºÑ Ð±Ð°Ð·ÐžÑÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐœÐ° ÐŸÐŽÐœÐŸÐŒ ÐžÑÑÐŸÑÐœÐžÐºÐµ ÐŽÐ°ÐœÐœÑÑ
+.ÐÐŒÐµÐµÑÑÑ ÐœÐŸÐ²ÑÐ¹ ÐžÑÑÐŸÑÐœÐžÐº Ð¿ÑÐµÐ²ÐŽÐŸÐŽÐ°ÐœÐœÑÑ
+ "Ð²ÑÐµ Ð¿ÐŸÐ»Ñ",ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÑÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð²ÑÐµÐŒ ÑÐºÐ°Ð·Ð°ÐœÐœÑÐŒ ÐžÐœÐŽÐµÐºÑÐ°ÐŒ ÑÑÐ°Ð·Ñ.Ð£ÑÐŸÐ²ÐœÐž ÐœÐµ Ð¿ÑÐžÐ²ÑÐ·Ð°ÐœÑ Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐŸÐŒÑ ÐžÐœÐŽÐµÐºÑÑ,Ð° Ð¿ÑÐžÐŒÐµÐœÑÑÑÑÑ ÐºÐŸ Ð²ÑÐµÐŒ ÐžÐœÐŽÐµÐºÑÐ°ÐŒ.Ð¢Ð°ÐºÐžÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ,ÐžÐœÐŽÐµÐºÑÑ Ðž ÑÑÐŸÐ²ÐœÐž ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑÑÑÑ ÐŸÑÐŽÐµÐ»ÑÐœÐŸ.ÐÐœÐŽÐµÐºÑÑ ÑÐŽÐ°Ð»ÑÑÑÑÑ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ,ÐºÐ°Ðº Ð±ÑÐ»ÐŸ ÑÐºÐ°Ð·Ð°ÐœÐŸ Ð²ÑÑÐµ,ÐœÐŸ ÐžÐœÐŽÐµÐºÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ Ð±ÐŸÐ»ÑÑÐµ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ. Ð­ÑÐŸ Ð¿ÑÐŸÑÑÐŸ Ð¿ÐµÑÐ²ÑÐ¹ ÐžÐ· ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ².   
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐžÐœÐŽÐµÐºÑÑ ÐœÐ° Ð²ÑÐµÑ
+ ÐžÑÑÐŸÑÐœÐžÐºÐ°Ñ
+,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐºÐœÐŸÐ¿ÐºÑ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÑÐµ". ÐÐŒÑ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐžÐœÐŽÐµÐºÑÐ° Ð±ÑÐŽÐµÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÑÐ²ÑÐ·Ð°ÐœÐŸ Ñ ÐžÐŒÐµÐœÐµÐŒ ÐžÑÑÐŸÑÐœÐžÐºÐ°.Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐžÐŒÑ,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐžÐœÐŽÐµÐºÑ,ÐžÐ·ÐŒÐµÐœÐžÑÐµ ÐµÐ³ÐŸ ÐŽÐµÑÐ°Ð»Ðž Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ ÐžÐœÐŽÐµÐºÑ".   
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/searchtypes.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/searchtypes.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/searchtypes.htm	(revision 21112)
@@ -0,0 +1,79 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="searchtypes"><font size="5" face="Verdana"><strong>6.4: ÐÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÐž ÐœÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐžÐ·Ð°Ð¹ÐœÐ° Ð² Greenstone; Ð²ÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°,ÐºÐŸÑÐŸÑÑÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÑÑ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð¿ÐŸÐ»ÑÐŒ. Ð©ÐµÐ»ÐºÐœÐžÑÐµ Ð²ÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ° Ð² "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐ°Ðº Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÑÑ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ°ÑÑÐžÑÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÐŸÐžÑÐºÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ ÐœÐŸÐ²ÐŸÐ³ÐŸ Ð²ÐžÐŽÐ° Ð¿ÐŸÐžÑÐºÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ Ð²ÐžÐŽÐ° Ð¿ÐŸÐžÑÐºÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ·ÐŒÐµÐœÐµÐœÐžÐµ Ð¿ÐŸÑÑÐŽÐºÐ° Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐŸÐ³ÐŽÐ° ÐÑ ÐœÐ°ÑÐžÐœÐ°ÐµÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°, ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑ ÑÐ°ÑÑÐžÑÐµÐœÐœÐŸÐ³ÐŸ Ð¿ÐŸÐžÑÐºÐ°",ÐºÐŸÑÐŸÑÑÐ¹ Ð°ÐºÑÐžÐ²ÐžÐ·ÐžÑÑÐµÑ ÐŽÑÑÐ³ÐžÐµ ÑÑÐµÐŽÑÑÐ²Ð° ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ.Ð­ÑÐŸ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð² ÑÐŸÑÐŒÐ°Ñ Greenstone 2.4, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ Ð¿ÐŸÐžÑÐº Ð¿ÐŸ Ð¿ÐŸÐ»ÑÐŒ Ðž ÐŸÐ·ÐœÐ°ÑÐ°ÐµÑ, ÑÑÐŸ (Ð°) - ÐžÐœÐŽÐµÐºÑ ÐŽÐžÐ·Ð°Ð¹ÐœÐ° ÑÐ²Ð»ÑÐµÑÑÑ ÐŸÑÐ»ÐžÑÐ°ÑÑÐžÐŒÑÑ,ÐŸÐ±ÑÑÑÐœÐµÐœ Ð²
+ (<a href="searchindexes.htm#searchindexes">ÐÐœÐŽÐµÐºÑÑ Ð¿ÐŸÐžÑÐºÐ°</a> section);
+(b) - ÐžÐŒÐµÐµÑÑÑ Ð±ÐŸÐ»ÑÑÐµ ÑÑÐ°Ð³ÐŒÐµÐœÑÐŸÐ² ÑÐµÐºÑÑÐ° ÐŽÐ»Ñ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ° (<a href="translatetext.htm#translatetext">ÐÐµÑÐµÐ²ÐŸÐŽ</a>);(c) - ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµ Ð±ÑÐŽÐµÑ Ð³ÐŸÐŽÐœÐ° Ðº ÑÐ¿ÐŸÑÑÐµÐ±Ð»ÐµÐœÐžÑ Ð¿ÑÐž Ð±ÐŸÐ»ÐµÐµ ÑÑÐ°ÑÑÑ
+ Ð²ÐµÑÑÐžÑÑ
+ Greenstone. ÐÑÐ»Ðž ÐÑ Ð¿ÐŸÐ·Ð¶Ðµ ÐœÐµ Ð¿ÑÐŸÐ²ÐµÑÐžÑÐµ ÑÑÐŸ Ð¿ÐŸÐ»Ðµ,ÑÐŸ Ð±ÐŸÐ»ÑÑÐ°Ñ ÑÐ°ÑÑÑ ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð²ÐµÑÐœÑÑÑÑ Ðº Greenstone 2.39.ÐÐŽÐœÐ°ÐºÐŸ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð¿ÑÐµÐŸÐ±ÑÐ°Ð·ÐŸÐ²Ð°ÑÑ ÐœÐŸÐ²ÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÐžÐœÐŽÐµÐºÑÐ° Ð² Ð±ÐŸÐ»ÐµÐµ ÑÑÐ°ÑÑÐµ,ÑÐ°Ðº ÑÑÐŸ ÐÑ Ð±ÑÐŽÐµÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð²ÐµÑÐœÑÑÑ ÐžÑ
+ Ð²ÑÑÑÐœÑÑ.
+ <p></p>
+ Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°,Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "ÐÐžÐŽÑ Ð¿ÐŸÐžÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°".ÐÐ°Ð¶ÐŽÑÐ¹ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ° ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÑÐ²ÐžÑÑÑÑ Ð² ÑÐ¿ÐžÑÐºÐµ ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐžÐœ ÑÐ°Ð·.
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° "Ð¢ÐµÐºÑÑÐžÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°".Ð¡Ð¿ÐžÑÐŸÐº ÐŽÐŸÐ»Ð¶ÐµÐœ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ Ð¿ÐŸ ÐºÑÐ°Ð¹ÐœÐµÐ¹ ÐŒÐµÑÐµ ÐŸÐŽÐžÐœ Ð²ÐžÐŽ Ð¿ÐŸÐžÑÐºÐ°. 
+ <p></p>
+ Ð§ÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ Ð¿ÐŸÑÑÐŽÐŸÐº ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÐžÑ Ð²ÐžÐŽÐŸÐ² Ð¿ÐŸÐžÑÐºÐ°, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ ÐžÐ· ÑÐ¿ÐžÑÐºÐ° Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²Ð²ÐµÑÑ
+" ÐžÐ»Ðž "ÐÐ²ÐžÐ³Ð°ÑÑÑÑ Ð²ÐœÐžÐ·". ÐÐµÑÐ²ÑÐ¹ ÐžÐ· ÐœÐŒÑ
+ Ð±ÑÐŽÐµÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/selectingmetadatasets.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/selectingmetadatasets.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/selectingmetadatasets.htm	(revision 21112)
@@ -0,0 +1,74 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="selectingmetadatasets"><font size="5" face="Verdana"><strong>5.2: ÐÑÐ±ÐŸÑ ÐœÐ°Ð±ÐŸÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ»ÐµÐŒÐµÐœÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðž ÐºÐ°Ðº ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Ð§ÑÐŸ ÑÐ°ÐºÐŸÐµ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+?
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÑÐ±ÑÐ°Ðœ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð£ÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ·Ð²ÐµÑÑÐœÐ° ÐºÐ°Ðº "ÐœÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+". ÐÑÐžÐŒÐµÑÐŸÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core. ÐÐŸÐ³ÐŽÐ° ÐÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÐµÑÐµ ÐºÐŸÐŒÐ¿Ð»ÐµÐºÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðº ÐÐ°ÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÐŸ ÐµÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÑ ÑÑÐ°ÐœÐŸÐ²ÑÑÑÑ ÐŽÐŸÑÑÑÐ¿ÐœÑÐŒÐž ÐŽÐ»Ñ Ð²ÑÐ±ÐŸÑÐ°. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐŒÐµÑÑ Ð±ÐŸÐ»ÑÑÐµ,ÑÐµÐŒ ÐŸÐŽÐžÐœ ÐœÐ°Ð±ÐŸÑ. ÐÐ»Ñ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐµÐœÐžÑ ÑÑÐŸÐ»ÐºÐœÐŸÐ²ÐµÐœÐžÑ ÐŸÐŽÐžÐœÐ°ÐºÐŸÐ²ÑÑ
+ ÐžÐŒÑÐœ Ðº ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ð¿ÑÐžÑÐŸÐµÐŽÐžÐœÑÐµÑÑÑ ÐºÐŸÑÐŸÑÐºÐžÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ,ÐºÐŸÑÐŸÑÑÐ¹ Ð²ÑÐŽÐµÐ»ÑÐµÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ»ÐµÐŒÐµÐœÑ Dublin Core "Creator" ÑÑÐ°ÐœÐµÑ "dc.Creator".ÐÐ°Ð±ÐŸÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ñ
+ÑÐ°ÐœÑÑÑÑ Ð² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° Ðž ÐžÐŒÐµÑÑ ÑÑÑÑÐžÐºÑ ".mds".
+<p></p>
+Ð§ÑÐŸÐ±Ñ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÐŸÐ²Ð°ÑÑ ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÑ
+ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð²ÑÐ±ÐµÑÐžÑÐµ "ÐÐ°Ð±ÐŸÑ ÐÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+" Ð² Ð¿ÑÐŸÑÐŒÐŸÑÑÐµ "ÐÐžÐ·Ð°Ð¹Ðœ".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/startingoff.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/startingoff.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/startingoff.htm	(revision 21112)
@@ -0,0 +1,19 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="startingoff"><font size="5" face="Verdana"><strong>2: ÐÐŸ ÑÑÐ°ÑÑÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+    
+    Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÑÐ°ÑÑÐºÐ°Ð·Ð°ÐœÐŸ, ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÑÑ, Ð·Ð°Ð³ÑÑÐ¶Ð°ÑÑ Ðž ÑÐŽÐ°Ð»ÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+    
+    
+    
+    
+  </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/thecreateview.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/thecreateview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/thecreateview.htm	(revision 21112)
@@ -0,0 +1,72 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thecreateview"><font size="5" face="Verdana"><strong>7.1: Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ,ÐºÐ°Ðº ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. 
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+	  </td>
+</tr>
+	  
+<tr>
+<td colspan="1"> </td><td colspan="2">
+	    ÐÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÐ¹ Ð¿ÑÐŸÑÐŒÐŸÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+	  </td>
+</tr>
+	
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð ÐµÐ³ÐžÑÑÑÐ°ÑÐžÑ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ,ÑÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ,Ð²ÑÐ¿ÐŸÐ»ÐœÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐµ ÑÑÐµÐœÐ°ÑÐžÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone ÐžÐ· ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž, ÐºÐŸÑÐŸÑÑÑ ÐÑ Ð¿ÐŸÐŽÐ³ÐŸÑÐŸÐ²ÐžÐ»Ðž. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° "ÐÐŸÑÑÑÐŸÐµÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž" ÐœÐ°ÑÐžÐœÐ°ÐµÑ Ð¿ÑÐŸÑÐµÑÑ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÑÐµÐŒÑ, ÐºÐŸÑÐŸÑÐŸÐµ ÐŽÐ»Ñ ÑÑÐŸÐ³ÐŸ Ð¿ÐŸÑÑÐµÐ±ÑÐµÑÑÑ, Ð·Ð°Ð²ÐžÑÐžÑ ÐŸÑ ÑÐ°Ð·ÐŒÐµÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž ÑÐžÑÐ»Ð° ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÐµÐŒÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ² (ÐŽÐ»Ñ ÐŸÐ³ÑÐŸÐŒÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÑÑÐŸ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐ°ÑÑ). ÐÐœÐŽÐžÐºÐ°ÑÐŸÑ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÑ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÐœÐ°ÑÐºÐŸÐ»ÑÐºÐŸ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœ Ð¿ÑÐŸÑÐµÑÑ. Ð§ÑÐŸÐ±Ñ ÐŸÑÐŒÐµÐœÐžÑÑ Ð¿ÑÐŸÑÐµÑÑ Ð² Ð»ÑÐ±ÐŸÐµ Ð²ÑÐµÐŒÑ, ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÑÐŒÐµÐœÐ° Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ".
+      <p></p>
+      ÐÐ°Ð¶Ð°Ð² ÐœÐ° "ÐÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÑÐ¹ Ð¿ÑÐŸÑÐŒÐŸÑÑ" Ð¿ÐŸÑÐ»Ðµ ÑÐŸÐ³ÐŸ, ÐºÐ°Ðº ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐ»Ð° ÑÑÐ¿ÐµÑÐœÐŸ ÑÐŸÐ±ÑÐ°ÐœÐ°, ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ Ð²ÐµÐ±-Ð±ÑÐ°ÑÐ·ÐµÑ Ñ Ð³Ð»Ð°Ð²ÐœÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÐµÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+      Ð ÐŒÐŸÐŽÑÑÐµ Ð­ÐºÑÐ¿ÐµÑÑ Ð²Ñ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð·Ð°Ð¿ÐžÑÑ "ÐÐŸÐ³ Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐ¹" ÑÐ»ÐµÐ²Ð°, ÑÑÐŸÐ±Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ, Ð±ÑÐ»Ðž Ð»Ðž Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð¿ÐŸÐ¿ÑÑÐºÐž ÑÐºÐŸÐœÑÑÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÑÑÐ¿ÐµÑÐœÑ ÐžÐ»Ðž ÐœÐµÑ. ÐÑÐ±ÐµÑÐžÑÐµ Ð¶ÐµÐ»Ð°ÐµÐŒÑÑ Ð·Ð°Ð¿ÐžÑÑ,ÑÐµÐ»ÐºÐ°Ñ Ð¿ÐŸ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐ¹ ÐŽÐ°ÑÐµ Ð² ÑÐ¿ÐžÑÐºÐµ "ÐÑÑÐŸÑÐžÑ Ð·Ð°Ð¿ÐžÑÐž".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/thedesignview.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/thedesignview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/thedesignview.htm	(revision 21112)
@@ -0,0 +1,55 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thedesignview"><font size="5" face="Verdana"><strong>6.1: ÐÐžÐŽ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» Ð²Ð²ÐŸÐŽÐžÑ ÐÐ°Ñ Ð² ÑÐ°ÑÑÐŒÐ°ÑÑÐžÐ²Ð°ÐµÐŒÑÐ¹ Ð¿ÑÐŸÐµÐºÑ Ðž ÐŸÐ±ÑÑÑÐœÑÐµÑ,ÐºÐ°Ðº Ð²ÑÐ±ÑÐ°ÑÑ ÐŒÐµÐ¶ÐŽÑ ÑÐ°Ð·Ð»ÐžÑÐœÑÐŒÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑÐŒÐž Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ Ð°ÐºÑÐžÐ²ÐœÐŸÐ¹ ÑÐ°ÑÑÐž ÐŸÐºÐœÐ°.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÑÐ±ÐŸÑ ÑÐ°Ð·ÐŽÐµÐ»Ð° Ð¿ÑÐŸÐµÐºÑÐ°
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÑÐž Ð¿ÐŸÐŒÐŸÑÐž ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÐŸÐ³ÐŸ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÑÐŸÑÐŒÐžÑÐŸÐ²Ð°ÑÑ Ð²ÐžÐŽÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÑÐµÐŒÐŸÐ¹ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»Ñ.ÐÐ°ÑÐžÐ°ÐœÑÑ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÑ ÐœÐ° ÑÐ°Ð·Ð»ÐžÑÐœÑÐµ ÑÐ°Ð·ÐŽÐµÐ»Ñ, ÐºÐ°Ð¶ÐŽÑÐ¹ ÐžÐ· ÐºÐŸÑÐŸÑÑÑ
+ ÑÐ²ÑÐ·Ð°Ðœ ÑÐŸ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐ¹ ÑÑÐ°ÐŽÐžÐµÐ¹ ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐž ÐžÐ»Ðž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž.
+<p></p>
+ Ð¡Ð»ÐµÐ²Ð° ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÑÐ¿ÐžÑÐŸÐº ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐ¹,Ð° ÑÐ¿ÑÐ°Ð²Ð° - ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÐºÑÑÐžÐŒ ÑÐ¿ÐžÑÐºÐŸÐŒ.Ð§ÑÐŸÐ±Ñ Ð¿ÐµÑÐµÐ¹ÑÐž ÐœÐ° Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐµÐ³ÐŸ ÐžÐŒÑ Ð² ÑÐ¿ÐžÑÐºÐµ. 
+<p></p>
+ Ð§ÑÐŸÐ±Ñ Ð¿ÐŸÐœÐžÐŒÐ°ÑÑ ÑÑÐ°ÐŽÐžÐž Ðž ÑÐµÑÐŒÐžÐœÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐµ Ð¿ÑÐž Ð¿ÑÐŸÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÑÐœÐ°ÑÐ°Ð»Ð° Ð¿ÑÐŸÑÑÐžÑÐµ Ð³Ð»Ð°Ð²Ñ 1 Ðž 2 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/theenrichview.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/theenrichview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/theenrichview.htm	(revision 21112)
@@ -0,0 +1,113 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="theenrichview"><font size="5" face="Verdana"><strong>5.1: ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+T Ð ÑÑÐŸÐž ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ, ÐºÐ°Ðº Ð²Ð²ÐŸÐŽÐžÑÑ Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ.
+
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐŸÐœÑÑÐŸÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐœÐžÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð¢Ð°Ð±Ð»ÐžÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐŸÐ±ÐŸÐ³Ð°ÑÐµÐœÐžÑ ÐŽÐ»Ñ Ð·Ð°ÐŽÐ°ÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. 
+MeÑÐ°ÐŽÐ°ÐœÐœÑÐµ - ÑÑÐŸ ÐŽÐ°ÐœÐœÑÐµ ÐŸ ÐŽÐ°ÐœÐœÑÑ
+ - ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ,ÐÐ²ÑÐŸÑ,ÐÐ°ÑÐ° ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ Ðž Ñ.ÐŽ. ÐÐ°Ð¶ÐŽÑÐ¹ Ð¿ÑÐœÐºÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐŸÑÑÐŸÐžÑ ÐžÐ· ÐŽÐ²ÑÑ
+ ÑÐ°ÑÑÐµÐ¹: "ÑÐ»ÐµÐŒÐµÐœÑ"- Ðº ÐºÐ°ÐºÐŸÐŒÑ ÑÐžÐ¿Ñ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŸÐœ ÐŸÑÐœÐŸÑÐžÑÑÑ (Ðº Ð¿ÑÐžÐŒÐµÑÑ,ÐÐ²ÑÐŸÑ),Ðž "Ð·ÐœÐ°ÑÐµÐœÐžÐµ"- ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÑÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ Ð»ÑÐ±ÑÑ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐžÐ»Ðž Ð¿Ð°Ð¿ÐŸÐº Ð² ÐÐµÑÐµÐ²Ðµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð¡ÑÐŸÐ»Ð±ÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÑ ÑÐµÑÑÐŒ ÑÐ²ÐµÑÐŸÐŒ Ð²Ð²ÐµÑÑ
+Ñ Ðž ÐžÑ
+ ÑÐ°Ð·ÐŒÐµÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÑÐœ Ð¿ÑÑÑÐŒ Ð¿ÐµÑÐµÐŽÐ²ÐžÐ¶ÐµÐœÐžÑ Ð»ÐžÐœÐžÐž ÑÐ°Ð·ÐŽÐµÐ»Ð°. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° Ð»ÑÐ±ÐŸÐ¹ ÑÑÑÐŸÐºÐµ Ð¿ÐµÑÐµÐœÐŸÑÐžÑ ÐµÑ ÐŽÐµÑÐ°Ð»Ðž Ð²ÐœÐžÐ· ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐ°Ð¶Ð°Ð² ÐœÐ° Ð»ÑÐ±ÐŸÐ¹ ÑÑÐŽ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ, Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ðž ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐµÑÐ°Ð»Ðž ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð² ÐœÐžÐ¶ÐœÐµÐ¹ ÑÐµÐºÑÐžÐž ÐŸÐºÐœÐ°. Ð§ÐµÑÐœÑÐ¹ ÑÐµÐºÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ, ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ²Ð»ÑÐµÑÑÑ ÐŸÐ±ÑÐžÐŒ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²,ÑÐŸÐ³ÐŽÐ° ÐºÐ°Ðº ÑÐµÑÑÐ¹ ÑÐµÐºÑÑ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ,ÑÑÐŸ ÑÑÐŸ - ÐœÐµ ÑÐ°Ðº.Ð§ÐµÑÐœÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐµÐœÑ, Ð² ÑÐŸ Ð²ÑÐµÐŒÑ ÐºÐ°Ðº ÑÐµÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ ÐžÐ· ÑÐµÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÐžÐŒÐµÑÑ ÐžÑ
+,ÐžÐ»Ðž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ðº ÐŽÑÑÐ³ÐžÐŒ.  
+ <p></p>
+ ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐºÐž ÐŒÐŸÐ¶ÐµÑ Ð¿ÐŸÑÐ²Ð»ÑÑÑÑÑ ÐŸÐºÐŸÐ»ÐŸ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ Ð²Ñ
+ÐŸÐŽÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.Ð­ÑÐŸ ÑÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐœÐ° ÑÐŸ,ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÑÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÑ ÐŸÑ Ð¿Ð°Ð¿ÐŸÐº ÑÐŸÐŽÐžÑÐµÐ»ÐµÐ¹ (ÐžÐ»Ðž Ð¿ÑÐµÐŽÐºÐŸÐ²). Ð£ÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐœÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐµÐœÑ, Ð° ÑÐŸÐ»ÑÐºÐŸ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÑ Ð² ÐºÐŸÐœÐµÑ ÑÐ¿ÐžÑÐºÐ° ÐžÐ»Ðž Ð¿ÐµÑÐµÐ¿ÐžÑÐ°ÐœÑ.Ð©ÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐž Ð¿Ð°Ð¿ÐºÐž,ÑÑÐŸÐ±Ñ ÐœÐµÐŒÐµÐŽÐ»ÐµÐœÐœÐŸ Ð¿ÐµÑÐµÐ¹ÑÐž Ð² ÑÑ Ð¿Ð°Ð¿ÐºÑ, Ð³ÐŽÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð·Ð°ÐŽÐ°ÐœÑ.  
+<p></p>
+ÐÐŸÐœÑÑÐŸÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐœÐžÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ Ð²ÐœÐžÐ·Ñ ÑÐ¿ÑÐ°Ð²Ð° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐµÑÐ»Ðž ÑÐ°Ð¹Ð» Ð²ÑÐ±ÑÐ°Ðœ ÐžÐ· ÐŽÐµÑÐµÐ²Ð°, Ð° ÑÑÐŽ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐžÑ
+, ÑÑÐŸÐ±Ñ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ, ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ ÐžÐ»Ðž ÑÐŽÐ°Ð»ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐœÐ°ÑÐµÐœÐžÐµ Ð¿ÐŸÐ»Ñ Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐŸ ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ Ð²Ð²ÐµÑÑÐž ÐžÐ»Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐºÐŸÐ»ÐŸ ÐœÐµÐ³ÐŸ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐ°Ñ ÐºÐœÐŸÐ¿ÐºÐ°, ÐºÐŸÑÐŸÑÐ°Ñ Ð¿ÑÐž ÑÐµÐ»ÑÐºÐµ ÐŸÑÐºÑÑÐ²Ð°ÐµÑ Ð±ÐŸÐ»ÑÑÐµÐµ Ð¿ÐŸÐ»Ðµ ÐŽÐ»Ñ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÑ. Ð Ð°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÐ°Ñ ÐœÐžÐ¶Ðµ ÐºÐœÐŸÐ¿ÐºÐ° "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð² ÐºÐŸÐœÐµÑ" Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ðž ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐžÑ
+ Ðº ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒ ÐŽÐ»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ°. ÐÐœÐŸÐ¿ÐºÐ° "ÐÐ°ÐŒÐµÐœÐžÑÑ" Ð¿ÑÐžÐŒÐµÐœÑÐµÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ðº ÐœÐŸÐ²ÑÐŒ, Ð° "Ð£ÐŽÐ°Ð»ÐžÑÑ" ÑÑÐžÑÐ°ÐµÑ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ.ÐÐœÐžÐ·Ñ Ð¿ÑÐœÐºÑÐ°,ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐŸÐ³ÐŸ ÐºÐ°Ðº "ÐÑÐµ Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ", ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ "ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹".  
+<p></p> 
+ ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐŒÐŸÐ¶ÐµÑ ÑÐ°ÑÑÐžÑÑÑÑÑÑ Ðž ÑÐ°Ð·ÑÑÑÐ°ÑÑÑÑ. ÐÐ±ÑÑÐœÐŸ ÑÑÐŸ - ÑÐ¿ÐžÑÐŸÐº, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, Ð²Ð²ÐµÐŽÐµÐœÐœÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ ÐŽÐ»Ñ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ ÑÐ»ÐµÐŒÐµÐœÑÐ°. Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° Ð²Ñ
+ÐŸÐŽÐµ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÑÐ°Ð·ÐŒÐµÑÐ°ÐµÑ ÐµÐ³ÐŸ Ð² Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹. Ð¡ ÐŽÑÑÐ³ÐŸÐ¹ ÑÑÐŸÑÐŸÐœÑ,Ð¿ÑÐž ÐœÐ°Ð¿ÐµÑÐ°ÑÐ°ÐœÐœÐŸÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÐž Ð² ÑÐµÐºÑÑÐŸÐ²ÐŸÐŒ Ð¿ÐŸÐ»Ðµ Ð²ÑÐ±ÐŸÑÐºÐž ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐÐµÑÐµÐ²Ð° Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÑÐžÑÑÐµÐŒÐ° ÐœÐ°ÑÐžÐœÐ°ÐµÑ ÑÐ°Ð±ÐŸÑÑ Ñ ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐÑ Ð²Ð²ÐµÐ»Ðž. ÐÐ°Ð¶Ð°ÑÐžÐµ [Tab] Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž Ð·Ð°Ð²ÐµÑÑÐ°ÐµÑ Ð¿ÐµÑÐ°ÑÐ°ÐœÐžÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ.
+ <p></p>
+ÐÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÑÐ³Ð°ÐœÐžÐ·ÐŸÐ²Ð°ÐœÑ Ð² ÐžÐµÑÐ°ÑÑ
+ÐžÑ. Ð­ÑÐŸ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÐŸ Ð² ÐÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð¿Ð°Ð¿ÐºÐž ÐŽÐ»Ñ Ð²ÐœÑÑÑÐµÐœÐœÐžÑ
+ ÑÑÐŸÐ²ÐœÐµÐ¹. ÐÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð²Ð²ÐµÐŽÐµÐœÑ,ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð·ÐœÐ°Ðº "|" ÐŽÐ»Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐžÑ ÑÑÐŸÐ²ÐœÐµÐ¹. ÐÐ°Ð¿ÑÐžÐŒÐµÑ,"Cards|Red|Diamonds|Seven" ÐŒÐŸÐ³ Ð±Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð² ÐžÐµÑÐ°ÑÑ
+ÐžÐž, Ð¿ÑÐžÐŒÐµÑÐŸÐŒ ÐºÐŸÑÐŸÑÐŸÐ¹ ÑÐ²Ð»ÑÐµÑÑÑ Ð¿Ð°ÐºÐµÑ ÐžÐ³ÑÑ Ð² ÐºÐ°ÑÑÑ.Ð­ÑÐŸ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²Ð°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ð²ÐŒÐµÑÑÐµ. ÐÑÑÐ¿Ð¿Ñ ÐŒÐŸÐ³ÑÑ ÑÐ°ÐºÐ¶Ðµ ÐœÐ°Ð·ÐœÐ°ÑÐ°ÑÑÑÑ ÐºÐ°Ðº ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ðº ÑÐ°Ð¹Ð»Ð°ÐŒ.  
+<p></p> 
+ Greenstone Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ð² ÐœÐ°Ð±ÐŸÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÐ»ÐµÐŒÐµÐœÑÑ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐŒÐµÑÑ Ð¿ÑÐµÑÐžÐºÑ "ex.". Ð­ÑÐŸ ÐœÐµ ÐžÐŒÐµÐµÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐµÑÐµÐ²Ð° Ðž ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐŸ,Ð¿ÐŸÑÐŸÐŒÑ ÑÑÐŸ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÑÐºÑÑÑÐŸ,ÐµÑÐ»Ðž ÑÐ°ÐºÐŸÐ¹ ÑÐžÐ¿ Ð²Ð²ÐŸÐŽÐ° Ð²ÑÐ±ÑÐ°Ðœ.ÐÐœÐŸÐ¿ÐºÐ° "..." Ð²ÑÑ ÐµÑÑ ÑÐ»ÑÐ¶ÐžÑ ÐŽÐ»Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ,ÐœÐŸ ÑÐµÐºÑÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°Ðœ. 
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/thegatherview.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/thegatherview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/thegatherview.htm	(revision 21112)
@@ -0,0 +1,119 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="thegatherview"><font size="5" face="Verdana"><strong>4.1: ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐœÐ°ÐºÐŸÐ¿ÐžÑÐµÐ»Ñ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð­ÑÐŸÑ ÑÐ°Ð·ÐŽÐµÐ» Ð²Ð²ÐŸÐŽÐžÑ Ð² ÐŸÐ±Ð»Ð°ÑÑÑ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ,ÐºÐŸÑÐŸÑÑÑ ÐÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÐµ,ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ,ÐºÐ°ÐºÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð² ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÐµÐŒÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÐ°Ðº Ð¿ÑÐŸÑÐŒÐ°ÑÑÐžÐ²Ð°ÑÑ ÑÐºÑÐ°Ðœ ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ 
+            </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð€Ð°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±Ð»Ð°ÑÑÑ ÑÐŸÑÑÐŸÑÐœÐžÑ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐœÐŸÐ¿ÐºÐž ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ
+          </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐµÑÐŸÐŽÑ ÐŸÑÐ±ÐŸÑÐ°
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð¡Ð¿ÐµÑÐžÐ°Ð»ÑÐœÐ°Ñ Ð¿Ð°Ð¿ÐºÐ° ÐŸÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ ÐœÐ°ÑÐžÐœÐ°ÐµÑÑÑ Ñ ÐŸÐ±Ð·ÐŸÑÐ° ÐœÐ°ÐºÐŸÐ¿Ð»ÐµÐœÐžÑ. Ð§ÑÐŸÐ±Ñ Ð²ÐµÑÐœÑÑÑÑÑ Ðº ÑÑÐŸÐŒÑ ÐŸÐ±Ð·ÐŸÑÑ Ð¿ÐŸÐ·Ð¶Ðµ,ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÑÐžÐŒÐ²ÐŸÐ»Ðµ "Gather", ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐœÑÐŒ ÐœÐµÐ¿ÐŸÑÑÐµÐŽÑÑÐ²ÐµÐœÐœÐŸ ÐœÐžÐ¶Ðµ Ð»ÐžÐœÐµÐ¹ÐºÐž ÐŒÐµÐœÑ.  
+<p></p>
+ÐÐ²Ðµ Ð±ÐŸÐ»ÑÑÐžÐµ ÐŸÐ±Ð»Ð°ÑÑÐž,ÐœÐ°Ð·Ð²Ð°ÐœÐœÑÐµ "Ð Ð°Ð±ÐŸÑÐµÐµ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²ÐŸ" Ðž "ÐoÐ»Ð»ÐµÐºÑÐžÑ" ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ,ÑÑÐŸÐ±Ñ Ð¿ÐµÑÐµÐŒÐµÑÑÐžÑÑ ÑÐ°Ð¹Ð»Ñ Ð² ÐÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ. ÐÐœÐž Ð²ÐºÐ»ÑÑÐ°ÑÑ "ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ",Ð³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÑÑÑÑÐºÑÑÑÑ,ÐºÐŸÑÐŸÑÑÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ Ðž Ð¿Ð°Ð¿ÐºÐž. 
+<p></p>
+ÐÑÐ±ÐµÑÐžÑÐµ ÑÐ»ÐµÐŒÐµÐœÑ ÐŽÐµÑÐµÐ²Ð°, ÐœÐ°Ð¶Ð°Ð² ÐœÐ° ÐœÐµÐ³ÐŸ Ð»ÐµÐ²ÐŸÐ¹ ÐºÐ»Ð°Ð²ÐžÑÐµÐ¹ ÐŒÑÑÐž. (Ð¢ÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ ÐŒÐŸÐ¶ÐœÐŸ ÑÐŽÐµÐ»Ð°ÑÑ ÐžÐœÐ°ÑÐµ, ÑÐŒ. ÐœÐžÐ¶Ðµ). ÐÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿Ð°Ð¿ÐºÑ, ÐžÐ»Ðž ÐœÐ°Ð¶ÐŒÐžÑÐµ ÐœÐ° ÑÐžÐŒÐ²ÐŸÐ» ÑÑÐŽÐŸÐŒ Ñ ÐœÐµÐ¹, ÑÑÐŸÐ±Ñ ÑÐ°Ð·Ð²ÐµÑÐœÑÑÑ (ÐžÐ»Ðž ÑÐ²ÐµÑÐœÑÑÑ) ÐµÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÐŒÐŸÐµ. ÐÐ²ÐŸÐ¹ÐœÑÐŒ ÑÐµÐ»ÑÐºÐŸÐŒ ÐœÐ° ÐœÑÐ¶ÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÐŸÑÐºÑÐŸÐ¹ÑÐµ ÐµÐ³ÐŸ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÐŸÐ¹ ÐŽÐ°ÐœÐœÐŸÐŒÑ ÑÐžÐ¿Ñ ÑÐ°Ð¹Ð»ÐŸÐ² Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ (ÑÐŒ.<a href="fileassociations.htm#fileassociations">ÐÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐµ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ</a>).
+<p></p>
+ÐÐµÑÐµÐ²ÐŸ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐžÑÑÐŸÑÐœÐžÐºÐž ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ Ð² ÐÐœÑÐµÑÑÐµÐ¹ÑÐµ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ -- ÑÐµÐºÑÑÑÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÑÑ ÑÐžÑÑÐµÐŒÑ (Ð²ÐºÐ»ÑÑÐ°Ñ ÐŽÐžÑÐºÐŸÐ²ÐŸÐŽ Ðž CD-ROM), ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ ÐÑÐžÐœÑÑÐŸÑÐœ, Ðž ÐºÑÑ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², Ð¿ÑÐž ÑÑÐ»ÐŸÐ²ÐžÐž Ð²ÐºÐ»ÑÑÐµÐœÐžÑ Ð²ÐµÐ± Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ. ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐžÐŽÐµÑÑ Ðž ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐž ÑÐ°Ð¹Ð»Ñ, ÐŸÐŽÐœÐ°ÐºÐŸ ÐœÐµ ÐŒÐŸÐ¶ÐµÑÐµ Ð¿ÐµÑÐµÐœÐŸÑÐžÑÑ, ÑÐŽÐ°Ð»ÑÑÑ, ÐžÐ»Ðž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÐžÑ
+, Ð·Ð° ÐžÑÐºÐ»ÑÑÐµÐœÐžÐµÐŒ ÑÐ¶Ðµ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ.  ÐÑÑÑÐµÑÑÐ²Ð»ÑÑÑ ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÑ Ð² ÑÑÐŸÐŒ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ðµ ÑÑÐŸÐ±Ñ ÐœÐ°Ð¹ÑÐž ÑÐ°Ð¹Ð»Ñ ÐºÐŸÑÐŸÑÑÐµ Ð²Ñ Ñ
+ÐŸÑÐžÑÐµ Ð²ÐºÐ»ÑÑÐžÑÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+<p></p>
+Ð€Ð°Ð¹Ð»ÐŸÐ²ÐŸÐµ ÐŽÐµÑÐµÐ²ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÑÐµÑ Ð¿ÐŸÐ»ÐœÐŸÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ ÐŸÐœÐŸ Ð¿ÑÑÑÐŸ.
+<p></p>
+ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÑÐ°Ð·ÐŒÐµÑÑ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð°, Ð¿ÐµÑÐµÐŽÐ²ÐžÐ³Ð°Ñ ÐŒÑÑÑ Ð¿ÐŸ ÑÐµÑÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ, ÐºÐŸÑÐŸÑÐ°Ñ ÐŸÑÐŽÐµÐ»ÑÐµÑ ÐŽÐµÑÐµÐ²ÑÑ (ÑÐŸÑÐŒÐ° ÑÐºÐ°Ð·Ð°ÑÐµÐ»Ñ ÐžÐ·ÐŒÐµÐœÑÐµÑÑÑ),Ðž Ð±ÑÐºÑÐžÑÑÑ ÐžÑ
+.
+<p></p>
+ÐÐœÐžÐ·Ñ ÐŸÐºÐœÐ° ÑÐ°ÑÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐ° ÑÑÐ°ÑÑÑÐœÐ°Ñ ÐŸÐ±Ð»Ð°ÑÑÑ, ÐºÐŸÑÐŸÑÐ°Ñ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð¿ÑÐŸÐ³ÑÐµÑÑ ÐŸÐ¿ÐµÑÐ°ÑÐžÐ¹ ÐœÐ°ÐŽ ÑÐ°Ð¹Ð»Ð°ÐŒÐž (ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐµ, Ð¿ÐµÑÐµÐŒÐµÑÐµÐœÐžÐµ, ÑÐŽÐ°Ð»ÐµÐœÐžÐµ). ÐÐœÐž ÐŒÐŸÐ³ÑÑ Ð·Ð°ÐœÑÑÑ ÐºÐ°ÐºÐŸÐµ-ÑÐŸ Ð²ÑÐµÐŒÑ. ÐÐœÐŸÐ¿ÐºÐ° "Ð¡ÑÐŸÐ¿" ÐŸÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑ Ð»ÑÐ±ÐŸÐµ ÐŽÐµÐ¹ÑÑÐ²ÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ Ð¿ÑÐŸÐžÑÑ
+ÐŸÐŽÐžÑ. 
+<p></p>
+ ÐÐ²Ðµ Ð±ÐŸÐ»ÑÑÐžÐµ ÐºÐœÐŸÐ¿ÐºÐž Ð·Ð°ÐœÐžÐŒÐ°ÑÑ ÐœÐžÐ¶ÐœÐžÐ¹ Ð¿ÑÐ°Ð²ÑÐ¹ ÑÐ³ÐŸÐ» ÑÐºÑÐ°ÐœÐ°."ÐÐŸÐ²Ð°Ñ ÐÐ°Ð¿ÐºÐ°",Ñ ÐºÐ°ÑÑÐžÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž, ÑÐŸÐ·ÐŽÐ°ÑÑ ÐœÐŸÐ²ÑÐµ Ð¿Ð°Ð¿ÐºÐž (ÑÐŒ. <a href="creatingfolders.htm#creatingfolders">Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº</a>). ÐÐœÐŸÐ¿ÐºÐ° "Ð£ÐŽÐ°Ð»ÐžÑÑ"ÑÐŽÐ°Ð»ÑÐµÑ ÑÐ°Ð¹Ð»Ñ Ñ ÐœÐµÐœÑÐ¶ÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐµÐ¹.Ð©ÐµÐ»ÑÐŸÐº ÐœÐ° ÐºÐœÐŸÐ¿ÐºÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ" ÑÐŽÐ°Ð»ÐžÑ Ð»ÑÐ±ÑÐµ ÐŸÑÐŸÐ±ÑÐ°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐ»ÑÑÐµÑÐœÐ°ÑÐžÐ²ÐœÐŸ, ÑÐ°Ð¹Ð»Ñ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐŽÐ°Ð»ÐµÐœÑ Ð¿ÑÑÐµÐŒ Ð¿ÐµÑÐµÑÑÐ³ÐžÐ²Ð°ÐœÐžÑ ÐžÑ
+ ÐœÐ° ÐºÐœÐŸÐ¿ÐºÑ "Ð£ÐŽÐ°Ð»ÐžÑÑ". 
+       <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÐŸÐ±ÑÐ°ÑÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ Ð¿ÐŸÑÐ»ÐµÐŽÐŸÐ²Ð°ÑÐµÐ»ÑÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ²,Ð²ÑÐ±ÐµÑÐžÑÐµ Ð¿ÐµÑÐ²ÑÐ¹ Ðž Ð·Ð°ÑÐµÐŒ ÑÐŽÐµÑÐ¶ÐžÐ²Ð°Ñ [Shift],ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° Ð¿ÐŸÑÐ»ÐµÐŽÐœÐµÐŒ - Ð¿ÑÐž ÑÑÐŸÐŒ Ð±ÑÐŽÑÑ ÐŸÑÐŸÐ±ÑÐ°ÐœÑ Ð²ÑÐµ ÐŸÑ
+Ð²Ð°ÑÐµÐœÐœÑÐµ ÑÐ»ÐµÐŒÐµÐœÑÑ.ÐÑÐ±ÐŸÑ ÐœÐµ ÑÐ»ÐµÐŽÑÑÑÐžÑ
+ ÐŸÐŽÐžÐœ Ð·Ð° ÐŽÑÑÐ³ÐžÐŒ ÑÐ°Ð¹Ð»ÐŸÐ² Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ Ð¿ÑÐž ÐœÐ°Ð¶Ð°ÑÐŸÐ¹ ÐºÐ»Ð°Ð²ÐžÑÐµ [Ctrl] Ðž ÑÐµÐ»ÑÐºÐµ ÐœÐ° Ð²ÑÐ±ÐžÑÐ°ÐµÐŒÑÑ
+ ÑÐ°Ð¹Ð»Ð°Ñ
+. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐž ÐŽÐ²Ð° ÐŒÐµÑÐŸÐŽÐ° Ð²ÐŒÐµÑÑÐµ ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿Ñ ÐœÐµÑÐŒÐµÐ¶ÐœÑÑ
+ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ².  
+<p></p>
+ ÐÐµÐºÐŸÑÐŸÑÑÐµ Ð¿Ð°Ð¿ÐºÐž - ÑÐžÐ¿Ð° ÑÐµÑ
+,ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÐÐ°ÑÐž ÑÐŸÐ±ÑÑÐ²ÐµÐœÐœÑÐµ Ð²ÐµÐ±-ÑÑÑÐ°ÐœÐžÑÑ - ÐžÐœÐŸÐ³ÐŽÐ° ÐžÐŒÐµÑÑ ÐŸÑÐŸÐ±ÑÑ Ð²Ð°Ð¶ÐœÐŸÑÑÑ.ÐÐžÐ±Ð»ÐžÐŸÑÐµÑÐœÑÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÐµÑÐµÐœÐµÑÑÐž ÑÐ°ÐºÐžÐµ Ð¿Ð°Ð¿ÐºÐž ÐœÐ° Ð¿ÐµÑÐ²ÑÐ¹ ÑÑÐŸÐ²ÐµÐœÑ ÑÐ°Ð¹Ð»ÐŸÐ²ÐŸÐ³ÐŸ ÐŽÐµÑÐµÐ²Ð°. Ð§ÑÐŸÐ±Ñ ÑÐŽÐµÐ»Ð°ÑÑ ÑÑÐŸ, ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ ÐŒÑÑÐž ÐœÐ° Ð¶ÐµÐ»Ð°ÑÐµÐ»ÑÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ. ÐÑÐ±ÐµÑÐžÑÐµ "Ð¡ÐŸÐ·ÐŽÐ°ÑÑ ÑÐŸÐºÑÐ°ÑÐµÐœÐžÐµ" Ðž Ð²Ð²ÐµÐŽÐžÑÐµ ÐžÐŒÑ ÐŽÐ»Ñ Ð¿Ð°Ð¿ÐºÐž. ÐÐ»Ñ ÑÐŽÐ°Ð»ÐµÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÑÐµÐ»ÐºÐœÐžÑÐµ Ð¿ÑÐ°Ð²ÐŸÐ¹ ÐºÐœÐŸÐ¿ÐºÐŸÐ¹ ÐŒÑÑÐž ÐœÐ° Ð¿ÐŸÐŒÐµÑÐµÐœÐœÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ Ðž Ð²ÑÐ±ÐµÑÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÐŸÐºÑÐ°ÑÐµÐœÐžÑ".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/themirrorview.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/themirrorview.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/themirrorview.htm	(revision 21112)
@@ -0,0 +1,89 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="themirrorview"><font size="5" face="Verdana"><strong>3.1: ÐÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐÐ°Ð³ÑÑÐ·ÐžÑÑ</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+      Ð­ÑÐ° ÑÐµÐºÑÐžÑ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐºÐ°Ðº ÐºÐŸÐœÑÐžÐ³ÑÑÐžÑÐŸÐ²Ð°ÑÑ Ð·Ð°ÐŽÐ°ÑÑ Ðž ÐºÐŸÐœÑÑÐŸÐ»Ñ Ð¿ÑÐŸÑÐµÑÑÐ° Ð·Ð°Ð³ÑÑÐ·ÐºÐž.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+          ÐÐµÑÐµÐ²ÐŸ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð°
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+          ÐÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ Ð·Ð°Ð³ÑÑÐ·ÐºÐž
+	  </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+          Ð¡Ð¿ÐžÑÐŸÐº Ð·Ð°Ð³ÑÑÐ·ÐºÐž
+	  </td>
+</tr>
+        
+        
+<tr>
+<td colspan="3"><a href="#advanced">Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+          Ð Ð°Ð·ÑÐµÑÐžÑÑ Ð¿ÑÐŸÐºÑÐž
+	  </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+      ÐÐŸÑÑÑÐ¿ Ðº Ð¿ÑÐŸÑÐŒÐŸÑÑÑ "ÐÐ°Ð³ÑÑÐ·ÐžÑÑ" ÐœÐ°Ð¶Ð°ÑÐµÐŒ ÑÑÐŸÐ¹ Ð·Ð°ÐºÐ»Ð°ÐŽÐºÐž. ÐÐµÑÑ
+ÐœÑÑ Ð¿ÐŸÐ»ÐŸÐ²ÐžÐœÐ° ÑÐºÑÐ°ÐœÐ° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐºÐŸÐœÑÑÐŸÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž. ÐÐžÐ¶ÐœÑÑ Ð¿ÐŸÐ»ÐŸÐ²ÐžÐœÐ° Ð² ÐœÐ°ÑÐ°Ð»Ðµ Ð¿ÑÑÑÐ°, ÐŸÐŽÐœÐ°ÐºÐŸ Ð¿ÐŸÐºÐ°Ð¶ÐµÑ Ð¿ÐµÑÐµÑÐµÐœÑ ÐŸÐ¶ÐžÐŽÐ°ÐµÐŒÑÑ
+ Ðž Ð·Ð°ÐºÐŸÐœÑÐµÐœÐœÑÑ
+ Ð·Ð°Ð³ÑÑÐ·ÐŸÐº.
+      <p></p>
+      Ð€Ð°Ð¹Ð»Ñ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÐµ Ð² Ð¿Ð°Ð¿ÐºÑ ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²Ð° ÐœÐ°Ð·ÑÐ²Ð°ÑÑÑÑ "ÐÐ°Ð³ÑÑÐ¶Ð°ÐµÐŒÑÐµ ÑÐ°Ð¹Ð»Ñ" (ÑÐŸÐ»ÑÐºÐŸ ÑÐµÐ¹ÑÐ°Ñ ÐºÐŸÐ³ÐŽÐ° Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð²ÐºÐ»ÑÑÐµÐœÐŸ), Ðž ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐ° Ð²ÑÐµÐŒÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒÐž Ð¿ÐŸÑÑÑÐŸÐµÐœÐœÑÐŒÐž Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ. Ð€Ð°Ð¹Ð»Ñ Ð² ÑÑÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐžÐžÐŒÐµÐœÑÑÑÑÑ Ð¿ÐŸ ÐžÑ
+ Ð¿ÐŸÐ»ÐœÐŸÐŒÑ URL Ð°ÐŽÑÐµÑÑ.  Ð¡ÐŸÐ·ÐŽÐ°ÐµÑÑÑ ÐœÐŸÐ²Ð°Ñ Ð¿Ð°Ð¿ÐºÐ° ÐœÐ° ÐºÐ°Ð¶ÐŽÐŸÐŒ Ñ
+ÐŸÑÑÐµ, ÐœÐ°ÑÐ»ÐµÐŽÑÐµÐŒÐ°Ñ ÐŽÑÑÐ³ÐžÐŒÐž ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÑÐ°Ð³ÐŒÐµÐœÑÐ° Ð¿ÑÑÐž.  Ð­ÑÐŸ Ð³Ð°ÑÐ°ÐœÑÐžÑÑÐµÑ ÐŸÑÐ»ÐžÑÐžÐµ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°.
+      <p></p>
+      ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð¿ÐµÑÐ²ÑÐ¹ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÑÑ
+ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐŸÐº, "Source URL",  ÑÑÐŸÐ±Ñ Ð²Ð²ÐµÑÑÐž URL Ð°ÐŽÑÐµÑ Ð² ÑÐµÐ»ÐµÐ²ÐŸÐ¹ ÑÐµÑÑÑÑ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐºÐŸÐœÑÑÐŸÐ»Ñ "ÐÐ»ÑÐ±ÐžÐœÑ ÐÐ°Ð³ÑÑÐ·ÐºÐž" ÐŽÐ»Ñ ÐŸÐ³ÑÐ°ÐœÐžÑÐµÐœÐžÑ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²Ð° Ð³ÐžÐ¿ÐµÑÑÑÑÐ»ÐŸÐº ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž: Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ 0 ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÐŸÐŽÐœÐŸÐ¹ Ð²ÐµÐ± ÑÑÑÐ°ÐœÐžÑÑ; Ð·ÐœÐ°ÑÐµÐœÐžÐµ 1 ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ·ÐžÑÑ ÐŸÐŽÐœÑ ÑÑÑÐ°ÐœÐžÑÑ Ðž Ð²ÑÐµ ÑÑÑÐ°ÐœÐžÑÑ ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ ÐœÐµÐ¹ . Ð¿ÑÐµÐŽÐµÐ» Ð³Ð»ÑÐ±ÐžÐœÑ ÑÐ°ÑÐ¿ÑÐŸÑÑÑÐ°ÐœÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐœÐ° html ÑÑÑÐ°ÐœÐžÑÑ. Ð¡Ð»ÐµÐŽÑÐ¹ÑÐµÐµ, Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐ° ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ "Ð³Ð°Ð»ÐŸÑÐµÐº", ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ, ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ ÑÑÐŸÐ±Ñ ÑÑÐŸÐ±Ñ Ð²ÐºÐ»ÑÑÐžÑÑ ÑÐºÐ°Ð·Ð°ÐœÐœÑÑ ÐŸÑÐŸÐ±ÐµÐœÐœÐŸÑÑÑ ÐºÐŸÐœÐºÑÐµÑÐœÐŸÐ¹ Ð·Ð°Ð³ÑÑÐ·ÐºÐž. ÐÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐ°, ÐœÐ°Ð¶ÐŒÐžÑÐµ "ÐÐ°Ð³ÑÑÐ·ÐºÐ°", ÑÑÐŸÐ±Ñ ÐœÐ°ÑÐ°ÑÑ ÐœÐŸÐ²ÑÑ Ð·Ð°Ð³ÑÑÐ·ÐºÑ. ÐÑÑÑ ÐŽÐ²Ðµ ÐŽÑÑÐ³ÐžÑ
+ ÐºÐœÐŸÐ¿ÐºÐž: Ð² ÑÐ°Ð·ÐŽÐµÐ»Ðµ "ÐÐ°ÑÑÑÐŸÐ¹ÐºÐž", ÐºÐŸÑÐŸÑÑÐµ ÑÐ²ÑÐ·Ð°ÐœÑ Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐŸÐŒ ÐœÐ°ÑÑÑÐŸÐ¹ÐºÐž ÑÐ²ÑÐ·Ðž, Ð³ÐŽÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑÐ° ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÑ; Ðž "ÐÑÐžÑÑÐºÐ° ÐÑÑ", ÐºÐŸÑÐŸÑÐ°Ñ ÑÐŽÐ°Ð»ÑÐµÑ Ð²ÑÐµ Ð¿ÑÐµÐŽÐ²Ð°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ.
+      <p></p>
+      Ð¡Ð¿ÐžÑÐŸÐº Ð·Ð°Ð³ÑÑÐ·ÐºÐž ÐžÐŒÐµÐµÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ Ð·Ð°Ð³ÑÑÐ¶Ð°ÐµÐŒÐŸÐ¹ Ð²ÐµÐ± ÑÑÑÐ°ÐœÐžÑÑ. ÐÐ°Ð¶ÐŽÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐŒÐµÐµÑ ÑÐµÐºÑÑ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÐžÐ¹ ÐŽÐµÑÐ°Ð»Ðž Ð·Ð°ÐŽÐ°ÑÐž Ð²ÐŒÐµÑÑÐµ Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐŸÑÐŸÐŒ Ð¿ÑÐŸÐ³ÑÐµÑÑÐ° ÐŽÐµÐŒÐŸÐœÑÑÑÐžÑÑÑÑÐžÐŒ ÑÐµÐºÑÑÑÑ ÐŽÐµÑÑÐµÐ»ÑÐœÐŸÑÑÑ. Ð¢ÑÐž ÐºÐœÐŸÐ¿ÐºÐž Ð¿ÐŸÑÐ²Ð»ÑÑÑÑÑ ÑÐ»ÐµÐ²Ð° ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ. "ÐÐ°ÑÐ·Ð°" ÐŽÐ»Ñ Ð¿ÑÐžÐŸÑÑÐ°ÐœÐŸÐ²ÐºÐž Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÑ ÑÐµÐºÑÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÐµÑÑÐ°. "Ð¡ÐŒÐŸÑÑÐµÑÑ ÐÐŸÐ³Ðž" ÐŸÑÐºÑÑÐ²Ð°ÐµÑ ÐŸÐºÐœÐŸ ÑÐ°Ð¹Ð»Ð° Ð·Ð°Ð³ÑÑÐ·ÐºÐž. "ÐÐ°ÐºÑÑÑÑ" Ð·Ð°Ð²ÐµÑÑÐ°ÐµÑ Ð·Ð°Ð³ÑÑÐ·ÐºÑ Ð·Ð°ÐŽÐ°ÑÐž ÐžÐ· ÑÐ¿ÐžÑÐºÐ°.
+      <p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="advanced"> Ð Ð°ÑÑÐžÑÐµÐœÐœÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+      
+<a href="preferences.htm#preferences">Ð¡ÐµÐºÑÐžÑ ÐÐ°ÑÑÑÐŸÐµÐº</a> ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑ ÐºÐ°Ðº ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÐžÐœÑÐµÑÐœÐµÑ ÑÐŸÐµÐŽÐžÐœÐµÐœÐžÐµ ÑÐµÑÐµÐ· Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑ.  ÐÑÐ»Ðž ÐœÑÐ¶ÐœÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐ°Ñ Ð°ÑÑÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÑ, Ð¿ÑÐŸÐºÑÐž ÑÐµÑÐ²ÐµÑ Ð·Ð°Ð¿ÑÐŸÑÐžÑ ÐžÐŒÑ Ðž Ð¿Ð°ÑÐŸÐ»Ñ.  ÐÐœÑÐµÑÑÐµÐ¹Ñ ÐÐžÐ±Ð»ÐžÐŸÑÐµÐºÐ°ÑÑ ÐœÐµ ÑÐŸÐ·ÑÐ°ÐœÑÐµÑ Ð¿Ð°ÑÐŸÐ»Ðž ÐŒÐµÐ¶ÐŽÑ ÑÐµÑÑÐžÑÐŒÐž.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/translatetext.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/translatetext.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/translatetext.htm	(revision 21112)
@@ -0,0 +1,66 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="translatetext"><font size="5" face="Verdana"><strong>6.10: ÐÐµÑÐµÐ²ÐŸÐŽ ÑÐµÐºÑÑÐ°</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÐµÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°, Ð³ÐŽÐµ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÑÐ·ÑÐº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ ÑÑÐ°Ð³ÐŒÐµÐœÑÐŸÐ² ÑÐµÐºÑÑÐ° ÐŽÐ»Ñ ÑÐ°ÑÑÐµÐ¹ ÐžÐœÑÐµÑÑÐµÐ¹ÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð "Ð Ð°Ð·ÐŽÐµÐ»Ðµ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐµÑÐµÐ²ÐµÑÑÐž ÑÐµÐºÑÑ".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÐœÐŸÐ²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð¡ÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ Ð²Ñ
+ÐŸÐŽ Ð² ÑÐ¿ÐžÑÐŸÐº "Ð¡Ð²ÐŸÐ¹ÑÑÐ²Ð°". ÐÐžÐ¶Ðµ Ð¿ÐŸÑÐ²ÑÑÑÑ ÑÑÑÐŸÐºÐž ÑÐ¿ÐµÑÐžÑÐžÑÐœÐŸÐ³ÐŸ ÑÐ·ÑÐºÐ°, ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ ÑÑÐžÐŒ ÑÐ²ÐŸÐ¹ÑÑÐ²ÐŸÐŒ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÐœÐžÑÐ¿Ð°ÐŽÐ°ÑÑÐžÐ¹ ÑÐ¿ÐžÑÐŸÐº "Ð¯Ð·ÑÐº Ð¿ÐµÑÐµÐ²ÐŸÐŽÐ°",ÑÑÐŸÐ±Ñ Ð²ÑÐ±ÑÐ°ÑÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐ¹ ÑÐ·ÑÐº Ðž Ð²Ð²ÐµÑÑÐž Ð¿ÐµÑÐµÐ²ÐµÐŽÐµÐœÐœÑÐ¹ ÑÐµÐºÑÑ Ð² ÐŸÐ±Ð»Ð°ÑÑÑ ÑÐµÐºÑÑÐ°, ÐŸÐ±ÑÐ°ÑÐ°ÑÑÑ Ðº "ÐÐ°ÑÐ°Ð»ÑÐœÐŸÐŒÑ ÑÑÐ°Ð³ÐŒÐµÐœÑÑ ÑÐµÐºÑÑÐ°",ÐµÑÐ»Ðž ÑÑÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ. ÐÐŸ ÐŸÐºÐŸÐœÑÐ°ÐœÐžÐž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐ±Ð°Ð²ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐ¹ Ð¿ÐµÑÐµÐ²ÐŸÐŽ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ Ð² ÑÐ°Ð±Ð»ÐžÑÐµ "ÐÐ°ÐŽÐ°ÐœÐœÑÐµ Ð¿ÐµÑÐµÐ²ÐŸÐŽÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "Ð£ÐŽÐ°Ð»ÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+<p></p>
+ Ð§ÑÐŸÐ±Ñ ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ, Ð²ÑÐ±ÐµÑÐžÑÐµ ÐµÐ³ÐŸ, ÐŸÑÑÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ Ð² ÑÐµÐºÑÑÐŸÐ²ÐŸÐ¹ ÐŸÐ±Ð»Ð°ÑÑÐž "ÐÐµÑÐµÐ²ÐµÐŽÐµÐœÐœÑÐ¹ ÑÐµÐºÑÑ" Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽ".
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/updatingmetadata.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/updatingmetadata.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/updatingmetadata.htm	(revision 21112)
@@ -0,0 +1,65 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="updatingmetadata"><font size="5" face="Verdana"><strong>5.5: ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Ð ÑÑÐŸÐŒ ÑÐ°Ð·ÐŽÐµÐ»Ðµ ÐŸÐ±ÑÑÑÐœÑÐµÑÑÑ, ÐºÐ°Ðº ÐŒÐŸÐŽÐµÑÐœÐžÐ·ÐžÑÐŸÐ²Ð°ÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐµ Ðº ÑÐ°Ð¹Ð»Ñ.
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+             ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ Ð¿Ð°Ð¿ÐŸÐº ÐžÐ»Ðž ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐŸ ÑÐ°Ð¹Ð»ÐŸÐ²
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ Ð§ÑÐŸÐ±Ñ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÑÐ°ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,ÑÐœÐ°ÑÐ°Ð»Ð° Ð²ÑÐ±ÐµÑÐžÑÐµ ÑÐ°Ð¹Ð»,Ðº ÐºÐŸÑÐŸÑÐŸÐŒÑ ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð¿ÑÐžÐŒÐµÐœÐµÐœÐŸ,Ð° Ð·Ð°ÑÐµÐŒ ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+,Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐÑ Ñ
+ÐŸÑÐžÑÐµ ÐžÐ·ÐŒÐµÐœÐžÑÑ.Ð ÐµÐ·ÑÐ»ÑÑÐ°Ñ Ð²ÑÐ±ÐŸÑÐ° Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ Ð² ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÐž ÑÐµÐŽÐ°ÐºÑÐžÑÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. Ð ÐµÐŽÐ°ÐºÑÐžÑÑÐ¹ÑÐµ Ð¿ÐŸÐ»Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ðž ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐ°ÐŒÐµÐœÐžÑÑ", ÑÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ.
+<p></p>
+ÐÑÐŸÑÐµÑÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÑ ÐŽÐ°ÐœÐœÑÑ
+ Ð¿Ð°Ð¿ÐºÐž Ñ Ð¿ÐŸÐŽÐ¿Ð°Ð¿ÐºÐ°ÐŒÐž ÐžÐ»Ðž ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹, ÐœÐŸ Ð²Ñ ÑÐŒÐŸÐ¶ÐµÑÐµ ÐŸÐ±ÐœÐŸÐ²ÐžÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ, ÐŸÐ±ÑÐžÐµ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ²/Ð¿Ð°Ð¿ÐŸÐº.
+<p></p>
+ ÐÐµÑÐµÐ²ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ Ð²ÑÐµ Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, Ð° ÐœÐµ ÑÐŸÐ»ÑÐºÐŸ ÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÑ Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ. Ð¢Ð°ÐºÐžÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ, Ð·ÐœÐ°ÑÐµÐœÐžÐµ, ÐºÐŸÑÐŸÑÐŸÐµ ÐÑ Ð·Ð°ÐŒÐµÐœÐžÐ»Ðž, ÐŸÑÑÐ°ÑÑÑÑ Ð² ÐŽÐµÑÐµÐ²Ðµ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/ru/xcollectionsearching.htm
===================================================================
--- /main/tags/3.01/gli/help/ru/xcollectionsearching.htm	(revision 21112)
+++ /main/tags/3.01/gli/help/ru/xcollectionsearching.htm	(revision 21112)
@@ -0,0 +1,62 @@
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The Greenstone Librarian Interface - Help Pages</title>
+</head>
+<body bgcolor="#E0F0E0">
+<table width="100%" cellspacing="0" cellpadding="5" bgcolor="#B0D0B0" border="2">
+<tr>
+<td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td><td width="*" align="center"><a name="xcollectionsearching"><font size="5" face="Verdana"><strong>6.7: Ð¡ÐŸÐ²ÐŒÐµÑÑÐœÑÐ¹ Ð¿ÐŸÐžÑÐº Ð² ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+</strong></font></a></td><td width="15%" align="center"><img width="45" src="../gatherer_medium.gif" height="45"></td>
+</tr>
+</table>
+      
+ Greenstone Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑ Ð¿ÐŸÐžÑÐº Ð² ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÑ
+ ÑÐ°Ð·Ð»ÐžÑÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+, ÐºÐ°Ðº ÐµÑÐ»Ðž Ð±Ñ ÐŸÐœÐž Ð±ÑÐ»Ðž ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÑ.Ð­ÑÐŸ ÑÐŽÐµÐ»Ð°ÐœÐŸ ÐŽÐ»Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ "Ð¡ÑÐ¿ÐµÑÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž", ÐºÐŸÑÐŸÑÐ°Ñ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐžÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.Ð "Ð Ð°Ð·ÐŽÐµÐ»Ð°Ñ
+ ÐŽÐžÐ·Ð°Ð¹ÐœÐ°" ÑÐµÐ»ÐºÐœÐžÑÐµ "ÐÐŸÐžÑÐº Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+".
+      <p></p>
+<table bgcolor="#B0D0B0" width="480" cellpadding="2" border="2" align="center">
+<tr>
+<td align="center"><font size="4" face="Verdana"><strong>
+              Contents
+            </strong></font></td>
+</tr>
+<tr>
+<td>
+<table width="100%" cellpadding="2" border="0" bgcolor="#FFFFFF">
+        
+<tr>
+<td colspan="3"><a href="#simple">ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a></td>
+</tr>
+          
+<tr>
+<td colspan="1"> </td><td colspan="2">
+            ÐÑÐ±ÐŸÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+          </td>
+</tr>
+        
+      
+</table>
+</td>
+</tr>
+</table>
+<p></p>
+      
+<p></p>
+<img width="20" src="../gatherer_small.gif" height="20"><a name="simple"> ÐÑÐŸÑÑÑÐµ ÐžÐœÑÑÑÑÐºÑÐžÐž:</a>
+<p style="text-align:justify"></p>
+ ÐÐµÐ¶ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐŸÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÑ
+ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÑ ÐºÐŸÐœÑÑÐŸÐ»ÑÐœÑÐ¹ ÑÐ¿ÐžÑÐŸÐº ÐŽÐŸÑÑÑÐ¿ÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹.Ð¢ÐµÐºÑÑÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð¿ÐŸÐŒÐµÑÐµÐœÑ Ðž ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐŸÑÑÐµÑÐœÐ°. Ð§ÑÐŸÐ±Ñ ÐŽÐŸÐ±Ð°Ð²ÐžÑÑ ÐŽÑÑÐ³ÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð¿ÐŸÐžÑÐº Ð±ÑÐŽÐµÑ Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑÑÑÑ Ð¿Ð°ÑÐ°Ð»Ð»ÐµÐ»ÑÐœÐŸ, ÑÐµÐ»ÐºÐœÐžÑÐµ ÐœÐ° ÐœÐžÑ
+ Ð² ÑÐ¿ÐžÑÐºÐµ (ÑÐµÐ»ÐºÐœÐžÑÐµ ÑÐœÐŸÐ²Ð°, ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ ÐžÑ
+). ÐÑÐ»Ðž ÐŸÑÐŸÐ±ÑÐ°ÐœÐ° ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐœÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÑÐŸ ÐŒÐµÐ¶ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐŸÐœÐœÑÐ¹ Ð¿ÐŸÐžÑÐº ÐœÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐµÐœ.
+<p></p>
+ ÐÐ»Ñ ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐžÑ
+ ÐŽÐµÑÐ°Ð»ÐµÐ¹ ÑÐŒ.Ð³Ð»Ð°Ð²Ñ 1 Ð ÑÐºÐŸÐ²ÐŸÐŽÑÑÐ²Ð° ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÑÐžÐºÐ° Greenstone.
+    </body>
+</html>
Index: /main/tags/3.01/gli/help/splithelpdocument.pl
===================================================================
--- /main/tags/3.01/gli/help/splithelpdocument.pl	(revision 21112)
+++ /main/tags/3.01/gli/help/splithelpdocument.pl	(revision 21112)
@@ -0,0 +1,66 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# splithelpdocument.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+sub main
+{
+    local @lines = <STDIN>;
+
+    local $filestart = 0;
+    local $filename = "";
+    for ($i = 0; $i < scalar(@lines); $i++) {
+	local $line = $lines[$i];
+
+	if ($line =~ /\<a name=\"([^\"]+)\"\>/ && $filename eq "") {
+	    $filename = $1;
+	}
+
+	if ($line =~ /^\<html\>/ && $i > $filestart) {
+	    open(FILE_OUT, ">$filename.htm") or die "Error: Could not write $filename.htm.\n";
+	    for ($j = $filestart; $j < $i; $j++) {
+		print FILE_OUT $lines[$j];
+	    }
+	    close(FILE_OUT);
+
+	    $filestart = $i;
+	    $filename = "";
+	}
+    }
+
+    # Deal with the last file
+    if ($filename ne "") {
+	open(FILE_OUT, ">$filename.htm") or die "Error: Could not write $filename.htm.\n";
+	for ($j = $filestart; $j < $i; $j++) {
+	    print FILE_OUT $lines[$j];
+	}
+	close(FILE_OUT);
+    }
+}
+
+
+&main(@ARGV);
Index: /main/tags/3.01/gli/makegli.bat
===================================================================
--- /main/tags/3.01/gli/makegli.bat	(revision 21112)
+++ /main/tags/3.01/gli/makegli.bat	(revision 21112)
@@ -0,0 +1,160 @@
+@echo off
+set GLILANG=en
+
+
+::  -------- Compile the Greenstone Librarian Interface --------
+
+echo.
+if "%GLILANG%" == "en" echo Greenstone Librarian Interface (GLI)
+if "%GLILANG%" == "en" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "en" echo GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+if "%GLILANG%" == "en" echo This is free software, and you are welcome to redistribute it
+
+if "%GLILANG%" == "es" echo Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "es" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "es" echo La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÖA.
+if "%GLILANG%" == "es" echo Para mayor informaci¢n vea los trminos de la licencia en LICENSE.txt
+if "%GLILANG%" == "es" echo Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita
+
+if "%GLILANG%" == "fr" echo Interface du Bibliothcaire Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "fr" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "fr" echo GLI est fourni sans AUCUNE GARANTIE; pour des dtails, voir LICENSE.txt
+if "%GLILANG%" == "fr" echo Ceci est un logiciel libre, et vous tes invit 
+ le redistribuer
+
+if "%GLILANG%" == "ru" echo š¡«š®â¥ç­ë© š­â¥àä¥©á Greenstone (Greenstone Librarian Interface - GLI)
+if "%GLILANG%" == "ru" echo Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato
+if "%GLILANG%" == "ru" echo  ­¥ € ¥â   ; €¥â «š á¬. ¢ â¥ªáâ¥ LICENSE.TXT
+if "%GLILANG%" == "ru" echo â® - á¢®¡®€­® à á¯à®áâà ­ï¥¬®¥ ¯à®£à ¬¬­®¥ ®¡¥á¯¥ç¥­š¥ š ë ¬®Š¥â¥ à á¯à®áâà ­ïâì ¥£®
+
+echo.
+
+:: This script must be run from within the directory in which it lives
+if exist makegli.bat goto findJavac
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+
+:findJavac
+:: ---- Check Javac exists ----
+set JAVACPATH=
+
+:: Some users may set the above line manually
+if not "%JAVACPATH%" == "" goto testJavac
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if Javac is on the search path
+    echo %PATH%| winutil\which.exe javac.exe | winutil\setvar.exe JAVACPATH > setjavac.bat
+    call setjavac.bat
+    del setjavac.bat
+    if not "%JAVACPATH%" == "" goto testJavac
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+
+    winutil\findjava.exe jdk.txt | winutil\setvar.exe JAVACPATH > setjavac.bat
+    del jdk.txt
+    call setjavac.bat
+    del setjavac.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JAVACPATH%" == "" goto noJavac
+
+    set JAVACPATH=%JAVACPATH%\bin
+    goto testJavac
+
+:javahome
+    set JAVACPATH=%JAVA_HOME%\bin
+
+:testJavac
+:: Check that a Javac executable has been found
+if "%GLILANG%" == "en" echo Checking Javac: %JAVACPATH%
+if "%GLILANG%" == "es" echo Revisando Javac: %JAVACPATH%
+if "%GLILANG%" == "fr" echo Vrification de Javac: %JAVACPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  Javac: %JAVACPATH%
+if exist "%JAVACPATH%\javac.exe" goto makeGLI
+
+:noJavac
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of Javac. You must install a
+    if "%GLILANG%" == "en" echo Java Development Kit (version 1.4 or greater) before compiling the
+    if "%GLILANG%" == "en" echo Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de Javac. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Kit de Desarrollo de Software Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de generar la documentaci¢n para la Interfaz de la Biblioteca
+    if "%GLILANG%" == "es" echo Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version approprie de Javac n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Kit de Dveloppement Java (version 1.4 ou suprieure) 
+    if "%GLILANG%" == "fr" echo avant de produire la documentation de Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš Javac.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë š­áâ ««šà®¢ âì Java Development Kit (¢¥àášï 1.4 š«š ¢ëè¥)
+    if "%GLILANG%" == "ru" echo ¯à¥Š€¥, ç¥¬ £¥­¥àšà®¢ âì €®ªã¬¥­â æšî €«ï ¡š¡«š®â¥ç­®£®
+    if "%GLILANG%" == "ru" echo š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+:makeGLI
+:: ---- Compile the GLI ----
+
+if "%1" == "" goto makeAll
+    :: If a file has been specified as a command-line argument, just compile that file
+    echo.
+    if "%GLILANG%" == "en" echo Compiling %1 and dependent classes...
+    if "%GLILANG%" == "es" echo Compilando %1 y clases dependientes...
+    if "%GLILANG%" == "fr" echo Compilation de %1 et des classes dpendantes,,,
+    if "%GLILANG%" == "ru" echo ®¬¯š«šà®¢ ­š¥ %1 š § ¢šáš¬ë¥ ª« ááë...
+
+    "%JAVACPATH%\javac.exe" -d classes/ -sourcepath src/ -classpath classes/;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar %1
+
+    if "%GLILANG%" == "en" echo Done!
+    if "%GLILANG%" == "es" echo ­Hecho!
+    if "%GLILANG%" == "fr" echo Termin!
+    if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+    goto done
+
+:makeAll
+:: Otherwise compile the lot...
+
+:: Remove any existing class files first
+call clean.bat
+
+if "%GLILANG%" == "en" echo Compiling the Greenstone Librarian Interface...
+if "%GLILANG%" == "es" echo Compilando la Interfaz de la Biblioteca Digital Greenstone...
+if "%GLILANG%" == "fr" echo Compilation de Greenstone Librarian Interface,,,
+if "%GLILANG%" == "ru" echo ®¬¯š«šà®¢ ­š¥ ¡š¡«š®â¥ç­®£® š­â¥àä¥©á  Greenstone...
+
+:: Compile the GLI
+"%JAVACPATH%\javac.exe" -d classes/ -sourcepath src/ -classpath classes/;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar src/org/greenstone/gatherer/GathererProg.java src/org/greenstone/gatherer/util/DragTreeSelectionModel.java 
+"%JAVACPATH%\javac.exe" -d classes/ -sourcepath src/ -classpath classes/;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar src/org/greenstone/gatherer/GathererApplet.java
+
+:: Compile the GEMS
+"%JAVACPATH%\javac.exe" -d classes/ -sourcepath src/ -classpath classes/;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar src/org/greenstone/gatherer/gems/GEMS.java
+
+:: Compile the stand-alone programs used by the GLI applet
+"%JAVACPATH%\javac.exe" -d classes/ -sourcepath src/ -classpath classes/;lib/apache.jar;lib/qfslib.jar;lib/mail.jar;lib/activation.jar src/org/greenstone/gatherer/util/Unzip.java
+
+if "%GLILANG%" == "en" echo Done!
+if "%GLILANG%" == "es" echo ­Hecho!
+if "%GLILANG%" == "fr" echo Termin!
+if "%GLILANG%" == "ru" echo ë¯®«­¥­®!
+goto done
+
+:exit
+echo.
+pause
+
+:done
+:: ---- Clean up ----
+set JAVACPATH=
Index: /main/tags/3.01/gli/makegli.sh
===================================================================
--- /main/tags/3.01/gli/makegli.sh	(revision 21112)
+++ /main/tags/3.01/gli/makegli.sh	(revision 21112)
@@ -0,0 +1,142 @@
+#!/bin/sh
+glilang=en
+
+echo
+if [ "$glilang" == "es" ]; then
+    echo "Interfaz de la Biblioteca Digital Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "La Interfaz de la Biblioteca Digital Greenstone NO INCLUYE ABSOLUTAMENTE NINGUNA GARANTÍA."
+    echo "Para mayor información vea los términos de la licencia en LICENSE.txt"
+    echo "Este es un software abierto, por lo que lo invitamos a que lo distribuya de forma gratuita"
+elif [ "$glilang" == "fr" ]; then
+    echo "Interface du Bibliothécaire Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI est fourni sans AUCUNE GARANTIE; pour des détails, voir LICENSE.txt"
+    echo "Ceci est un logiciel libre, et vous êtes invité à le redistribuer"
+elif [ "$glilang" == "ru" ]; then
+    echo "âÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone (Greenstone Librarian Interface - GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "âéç ÎÅ ÄÁÅÔ áâóïìàôîï îéëáëéè çáòáîôéê; ÄÅÔÁÌÉ ÓÍ. × ÔÅËÓÔÅ LICENSE.TXT"
+    echo "üÔÏ - Ó×ÏÂÏÄÎÏ ÒÁÓÐÒÏÓÔÒÁÎÑÅÍÏÅ ÐÒÏÇÒÁÍÍÎÏÅ ÏÂÅÓÐÅÞÅÎÉÅ É ÷Ù ÍÏÖÅÔÅ ÒÁÓÐÒÏÓÔÒÁÎÑÔØ ÅÇÏ"
+else
+    echo "Greenstone Librarian Interface (GLI)"
+    echo "Copyright (C) 2005, New Zealand Digital Library Project, University Of Waikato"
+    echo "GLI comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt"
+    echo "This is free software, and you are welcome to redistribute it"
+fi
+echo
+
+##  -------- Compile the Greenstone Librarian Interface --------
+
+## ---- Check Javac exists ----
+javacpath=
+
+# Some users may set the above line manually
+if [ "$javacpath" == "" ]; then
+
+    # If it is set, use the JAVA_HOME environment variable
+    if [ "$JAVA_HOME" != "" ]; then
+	javacpath="$JAVA_HOME/bin/javac"
+
+    # Check if Javac is on the search path
+    else
+	javacpath=`which javac 2> /dev/null`
+    fi
+fi
+
+# Check that a Javac executable has been found
+if [ "$glilang" == "es" ]; then
+    echo "Revisando Javac: $javacpath"
+elif [ "$glilang" == "fr" ]; then
+    echo "Vérification de Javac: $javacpath"
+elif [ "$glilang" == "ru" ]; then
+    echo "ðÒÏ×ÅÒËÁ Javac: $javacpath"
+else
+    echo "Checking Javac: $javacpath"
+fi
+if [ ! -x "$javacpath" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "No se pudo localizar una versión apropiada de Javac."
+	echo "Por favor instale una nueva versión del Kit de Desarrollo de"
+	echo "Software Java (versión 1.4 o posterior) y ejecute nuevamente"
+	echo "este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Une version appropriée de Javac n'a pas pu être localisée."
+	echo "Veuillez installer une nouvelle version de Java SDK (version 1.4 ou"
+	echo "plus récente) et redémarrez ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "îÅ ÕÄÁÌÏÓØ ÏÐÒÅÄÅÌÉÔØ ÍÅÓÔÏÎÁÈÏÖÄÅÎÉÅ ÓÏÏÔ×ÅÔÓÔ×ÕÀÝÅÊ ×ÅÒÓÉÉ Javac."
+	echo "ðÏÖÁÌÕÊÓÔÁ, ÕÓÔÁÎÏ×ÉÔÅ ÎÏ×ÕÀ ×ÅÒÓÉÀ Java SDK (×ÅÒÓÉÀ 1.4 ÉÌÉ ÂÏÌÅÅ"
+	echo "ÎÏ×ÕÀ) É ÐÅÒÅÕÓÔÁÎÏ×ÉÔÅ ÜÔÏÔ ÓËÒÉÐÔ."
+    else
+	echo "Failed to locate an appropriate version of Javac. You must install a"
+	echo "Java Development Kit (version 1.4 or greater) before compiling the"
+	echo "Greenstone Librarian Interface."
+    fi
+    exit 1
+fi
+
+## ---- Compile the GLI ----
+
+# If a file has been specified at the command-line, just compile that file
+if [ ! "x$*" = "x" ] ; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "Compilando $* y clases dependientes..."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Compilation de $* et des classes dépendantes,,,"
+    elif [ "$glilang" == "ru" ]; then
+	echo "ëÏÍÐÉÌÉÒÏ×ÁÎÉÅ $* É ÚÁ×ÉÓÉÍÙÅ ËÌÁÓÓÙ..."
+    else
+	echo "Compiling $* and dependent classes..."
+    fi
+
+    $javacpath -deprecation -d classes/ -sourcepath src/ -classpath classes/:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar $*
+
+    if [ "$glilang" == "es" ]; then
+	echo "¡Hecho!"
+    elif [ "$glilang" == "fr" ]; then
+	echo "Terminé!"
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷ÙÐÏÌÎÅÎÏ!"
+    else
+	echo "Done!"
+    fi
+    exit 0
+fi
+
+# Otherwise compile the lot...
+
+# Remove any existing class files first
+./clean.sh
+
+if [ "$glilang" == "es" ]; then
+    echo "Compilando la Interfaz de la Biblioteca Digital Greenstone..."
+elif [ "$glilang" == "fr" ]; then
+    echo "Compilation de Greenstone Librarian Interface,,,"
+elif [ "$glilang" == "ru" ]; then
+    echo "ëÏÍÐÉÌÉÒÏ×ÁÎÉÅ ÂÉÂÌÉÏÔÅÞÎÏÇÏ ÉÎÔÅÒÆÅÊÓÁ Greenstone..."
+else
+    echo "Compiling the Greenstone Librarian Interface..."
+fi
+
+# Compile the GLI
+$javacpath -deprecation -d classes/ -sourcepath src/ -classpath classes/:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar src/org/greenstone/gatherer/GathererProg.java
+$javacpath -deprecation -d classes/ -sourcepath src/ -classpath classes/:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar src/org/greenstone/gatherer/GathererApplet.java
+
+# Compile the GEMS
+$javacpath -deprecation -d classes/ -sourcepath src/ -classpath classes/:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar src/org/greenstone/gatherer/gems/GEMS.java
+
+# Compile the stand-alone programs used by the GLI applet
+$javacpath -deprecation -d classes/ -sourcepath src/ -classpath classes/:lib/apache.jar:lib/qfslib.jar:lib/mail.jar:lib/activation.jar src/org/greenstone/gatherer/util/Unzip.java
+
+if [ "$glilang" == "es" ]; then
+    echo "¡Hecho!"
+elif [ "$glilang" == "fr" ]; then
+    echo "Terminé!"
+elif [ "$glilang" == "ru" ]; then
+    echo "÷ÙÐÏÌÎÅÎÏ!"
+else
+    echo "Done!"
+fi
Index: /main/tags/3.01/gli/makejar.bat
===================================================================
--- /main/tags/3.01/gli/makejar.bat	(revision 21112)
+++ /main/tags/3.01/gli/makejar.bat	(revision 21112)
@@ -0,0 +1,175 @@
+echo off
+set GLILANG=en
+
+:: This script must be run from within the directory in which it lives
+if exist makejar.bat goto findJar
+    if "%GLILANG%" == "en" echo This script must be run from the directory in which it resides.
+    if "%GLILANG%" == "es" echo Este gui¢n deber  ejecutarse desde el directorio en el que reside.
+    if "%GLILANG%" == "fr" echo Ce script doit tre excut 
+ partir du rpertoire dans lequel il se trouve.
+    if "%GLILANG%" == "ru" echo â®â áªàš¯â €®«Š¥­ ¡ëâì ¢§ïâ š§ €šà¥ªâ®àšš, ¢ ª®â®à®© ®­ à á¯®«®Š¥­
+    goto exit
+
+
+
+
+:findJar
+:: ---- Check jar exists ----
+set JARPATH=
+
+:: Some users may set the above line manually
+if not "%JARPATH%" == "" goto testJar
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto javahome
+
+    :: Check if jar is on the search path
+    echo %PATH%| winutil\which.exe jar.exe | winutil\setvar.exe JARPATH > setjar.bat
+    call setjar.bat
+    del setjar.bat
+    if not "%JARPATH%" == "" goto testJar
+
+    :: Still haven't found anything, so try looking in the registry (gulp!)
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+
+    winutil\findjava.exe jdk.txt | winutil\setvar.exe JARPATH > setjar.bat
+    del jdk.txt
+    call setjar.bat
+    del setjar.bat
+
+    :: If nothing was found in the registry, we're stuck
+    if "%JARPATH%" == "" goto noJar
+
+    set JARPATH=%JARPATH%\bin
+    goto testJar
+
+:javahome
+    set JARPATH=%JAVA_HOME%\bin
+
+:testJar
+:: Check that a jar executable has been found
+if "%GLILANG%" == "en" echo Checking jar: %JARPATH%
+if "%GLILANG%" == "es" echo Revisando jar: %JARPATH%
+if "%GLILANG%" == "fr" echo Vrification de jar: %JARPATH%
+if "%GLILANG%" == "ru" echo à®¢¥àª  jar: %JARPATH%
+if exist "%JARPATH%\jar.exe" goto checkCompile
+
+:noJar
+    echo.
+    if "%GLILANG%" == "en" echo Failed to locate an appropriate version of jar. You must install a
+    if "%GLILANG%" == "en" echo Java Development Kit (version 1.4 or greater) before compiling the
+    if "%GLILANG%" == "en" echo Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "es" echo No se pudo localizar una versi¢n apropiada de jar. Usted deber 
+    if "%GLILANG%" == "es" echo instalar un Kit de Desarrollo de Software Java (versi¢n 1.4 o superior)
+    if "%GLILANG%" == "es" echo antes de generar la documentaci¢n para la Interfaz de la Biblioteca
+    if "%GLILANG%" == "es" echo Digital Greenstone.
+
+    if "%GLILANG%" == "fr" echo Une version approprie de jar n'a pas pu tre localise. Vous devez
+    if "%GLILANG%" == "fr" echo installer un Kit de Dveloppement Java (version 1.4 ou suprieure) 
+    if "%GLILANG%" == "fr" echo avant de produire la documentation de Greenstone Librarian Interface.
+
+    if "%GLILANG%" == "ru" echo ¥ ã€ «®áì ®¯à¥€¥«šâì ¬¥áâ®­ å®Š€¥­š¥ á®®â¢¥âáâ¢ãîé¥© ¢¥àášš jar.
+    if "%GLILANG%" == "ru" echo ë €®«Š­ë š­áâ ««šà®¢ âì Java Development Kit (¢¥àášï 1.4 š«š ¢ëè¥)
+    if "%GLILANG%" == "ru" echo ¯à¥Š€¥, ç¥¬ £¥­¥àšà®¢ âì €®ªã¬¥­â æšî €«ï ¡š¡«š®â¥ç­®£®
+    if "%GLILANG%" == "ru" echo š­â¥àä¥©á  Greenstone.
+    goto exit
+
+
+
+:checkCompile
+:: Check that the GLI has been compiled 
+if exist classes\org\greenstone\gatherer\GathererProg.class goto makeJar
+    if "%GLILANG%" == "es"  (
+	echo Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone
+	echo (por medio de makegli.sh) antes de ejecutar este guión.
+    )
+    if "%GLILANG%" == "fr" (
+	echo Vous devez compiler le Greenstone Interface (en utilisant makegli.sh)
+	echo avant d'exécuter ce script.
+    )
+    if "%GLILANG%" == "ru" (
+	echo ÷Ù ÄÏÌÖÎÙ ËÏÍÐÉÌÉÒÏ×ÁÔØ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone
+	echo (ÉÓÐÏÌØÚÕÑ makegli.sh) ÐÅÒÅÄ ××ÏÄÏÍ ÜÔÏÇÏ ÓËÒÉÐÔÁ
+    )
+    if "%GLILANG%" == "en" (
+	echo You need to compile the Greenstone Librarian Interface (using makegli.sh)
+	echo before running this script.
+    )
+    goto exit
+fi
+
+
+:makeJar
+:: All the GLI class files and supporting libraries are put into the "jar" directory
+
+echo Assuming that Java code is freshly compiled...
+
+if NOT exist jar (
+  mkdir jar
+
+  cd lib
+  %JARPATH%\jar xf activation.jar com javax
+  %JARPATH%\jar xf apache.jar com org
+  %JARPATH%\jar xf mail.jar com javax
+  %JARPATH%\jar xf qfslib.jar de
+
+  move com ..\jar\com     >NUL
+  move de ..\jar\de       >NUL
+  move javax ..\jar\javax >NUL
+  move org ..\jar\org     >NUL
+
+  cd ..
+)
+
+:: Copy the latest version of the GLI classes into the jar directory
+if exist jar\org\greenstone (
+  rmdir /q /s jar\org\greenstone
+)
+xcopy /i /e /q classes\org\greenstone jar\org\greenstone
+
+:: Some of the things to go into the JAR file are optional, and may not exist
+set optional=
+if exist collect.zip (
+    set optional=%optional% collect.zip
+)
+if exist classifiers.dat (
+    set optional=%optional% classifiers.dat
+)
+if exist plugins.dat (
+    set optional=%optional% plugins.dat
+)
+
+:: Recreate the metadata.zip file (contains the GLI metadata directory)
+if exist jar\org\greenstone (
+  del /f metadata.zip
+)
+winutil\zip.exe -r metadata.zip metadata >NUL
+
+:: Jar everything up
+%JARPATH%\jar cf GLI.jar .java.policy metadata.zip %optional% help -C classes dictionary.properties -C classes dictionary_es.properties -C classes dictionary_fr.properties -C classes dictionary_ru.properties -C classes feedback.properties -C classes images -C classes xml -C jar com -C jar de -C jar javax -C jar org
+
+
+
+:: ---- Make signed JAR file for the applet, if desired ----
+if  (%1) == (-sign) (
+
+    if not exist appletstore (
+      %JARPATH%\keytool -genkey -alias privateKey -keystore appletstore -storepass greenstone
+    )
+
+    if exist SignedGatherer.jar del /f SignedGatherer.jar
+    if exist appletpasswd (
+      echo Using jarsigner to make signed jar file ...
+      %JARPATH%\jarsigner -keystore appletstore -signedjar SignedGatherer.jar GLI.jar privateKey < appletpasswd >NUL 2>NUL
+      echo ... done.
+    ) ELSE (
+      %JARPATH%\jarsigner -keystore appletstore -signedjar SignedGatherer.jar GLI.jar privateKey
+    )
+    echo Installing SignedGatherer in ..\bin\java
+    move SignedGatherer.jar ..\bin\java\SignedGatherer.jar
+)
+
Index: /main/tags/3.01/gli/makejar.sh
===================================================================
--- /main/tags/3.01/gli/makejar.sh	(revision 21112)
+++ /main/tags/3.01/gli/makejar.sh	(revision 21112)
@@ -0,0 +1,90 @@
+#!/bin/sh
+glilang=en
+
+
+# This script must be run from within the directory in which it lives
+thisdir=`pwd`
+if [ ! -f "${thisdir}/makejar.sh" ]; then
+    if [ "$glilang" == "es" ]; then
+        echo "Este guión deberá ejecutarse desde el directorio en el que reside."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Ce script doit être exécuté à partir du répertoire dans lequel il se trouve."
+    elif [ "$glilang" == "ru" ]; then
+	echo "üÔÏÔ ÓËÒÉÐÔ ÄÏÌÖÅÎ ÂÙÔØ ×ÚÑÔ ÉÚ ÄÉÒÅËÔÏÒÉÉ, × ËÏÔÏÒÏÊ ÏÎ ÒÁÓÐÏÌÏÖÅÎ"
+    else
+	echo "This script must be run from the directory in which it resides."
+    fi
+    exit 1
+fi
+
+## ---- Check that the GLI has been compiled ----
+if [ ! -f "classes/org/greenstone/gatherer/GathererProg.class" ]; then
+    echo
+    if [ "$glilang" == "es" ]; then
+	echo "Usted necesita compilar la Interfaz de la Biblioteca Digital Greenstone"
+	echo "(por medio de makegli.sh) antes de ejecutar este guión."
+    elif [ "$glilang" == "fr" ]; then
+	echo "Vous devez compiler le Greenstone Interface (en utilisant makegli.sh)"
+	echo "avant d'exécuter ce script."
+    elif [ "$glilang" == "ru" ]; then
+	echo "÷Ù ÄÏÌÖÎÙ ËÏÍÐÉÌÉÒÏ×ÁÔØ ÂÉÂÌÉÏÔÅÞÎÙÊ ÉÎÔÅÒÆÅÊÓ Greenstone"
+	echo "(ÉÓÐÏÌØÚÕÑ makegli.sh) ÐÅÒÅÄ ××ÏÄÏÍ ÜÔÏÇÏ ÓËÒÉÐÔÁ"
+    else
+	echo "You need to compile the Greenstone Librarian Interface (using makegli.sh)"
+	echo "before running this script."
+    fi
+    exit 1
+fi
+
+echo "Assuming that Java code is freshly compiled..."
+
+# All the GLI class files and supporting libraries are put into the "jar" directory
+if [ ! -d jar ]; then
+  mkdir jar
+
+  cd lib
+  jar xf activation.jar com javax
+  jar xf apache.jar com org
+  jar xf mail.jar com javax
+  jar xf qfslib.jar de
+
+  mv com de javax org ../jar/.
+  cd ..
+fi
+
+# Copy the latest version of the GLI classes into the jar directory
+rm -rf jar/org/greenstone
+cp -r classes/org/greenstone jar/org
+
+# Some of the things to go into the JAR file are optional, and may not exist
+optional=
+if [ -f collect.zip ]; then
+    optional="$optional collect.zip"
+fi
+if [ -f classifiers.dat ]; then
+    optional="$optional classifiers.dat"
+fi
+if [ -f plugins.dat ]; then
+    optional="$optional plugins.dat"
+fi
+
+# Recreate the metadata.zip file (contains the GLI metadata directory)
+rm -f metadata.zip
+zip -r metadata.zip metadata >/dev/null
+
+# Jar everything up
+jar cf GLI.jar .java.policy metadata.zip $optional help \
+    -C classes dictionary.properties -C classes dictionary_es.properties \
+    -C classes dictionary_fr.properties -C classes dictionary_ru.properties \
+    -C classes feedback.properties -C classes images -C classes xml \
+    -C jar com -C jar de -C jar javax -C jar org
+
+
+# ---- Make signed JAR file for the applet, if desired ----
+if [ "$1" == "-sign" ]; then
+    rm -f SignedGatherer.jar
+    echo "greenstone" | jarsigner -keystore .greenstonestore -signedjar SignedGatherer.jar GLI.jar privateKey 2> /dev/null
+    echo
+    echo "Installing SignedGatherer in ../bin/java"
+    mv SignedGatherer.jar ../bin/java/.
+fi
Index: /main/tags/3.01/gli/metadata/adding_translations.txt
===================================================================
--- /main/tags/3.01/gli/metadata/adding_translations.txt	(revision 21112)
+++ /main/tags/3.01/gli/metadata/adding_translations.txt	(revision 21112)
@@ -0,0 +1,88 @@
+How to add a translation for a metadata set.
+
+Note: When using the word element, we are speaking of a document object model element. Whereas when we say Element we are talking about a particular document object model element used to describe an item from the current set.
+
+1. Prepare your source text in a UTF8 encoded document, or use a fully Unicode compliant text editor. Open the metadata set file that you want to edit.
+
+Example: To add a new language to the dublin.mds, open the file in a UTF8 editor.
+
+2. In MetadataSet DOM element edit, or add if necessary, an argument named "language_dependent". As its value add a list of comma separated attribute names. These are the names of the attributes which are taken to be language dependent (see 4). 
+
+Example: While editing the dublin.mds scroll down to the start of the MetadataSet xml block, which looks like:
+
+<MetadataSet 
+  contact     = "dcmi-feedback@dublincore.org"
+  creator     = "Dublin Core Metadata Initiative"
+  family      = "Dublin Core"
+  lastchanged = "Tue Jun 04 10:35:30 2002 - jmt12 on blackbird"
+  namespace   = "dc" 
+>
+
+and add the mentioned line so it reads something like:
+
+<MetadataSet 
+  contact     = "dcmi-feedback@dublincore.org"
+  creator     = "Dublin Core Metadata Initiative"
+  family      = "Dublin Core"
+  lastchanged = "Tue Jun 04 10:35:30 2002 - jmt12 on blackbird"
+  namespace   = "dc" 
+  language_dependent = "identifier, definition, comment"
+>
+
+where identifier, definition and command are the attributes that we wish to be language specific.
+
+3. Near the top of the document add a language specific set description block of the form
+
+  <SetLanguage code="**">
+    <Name>**</Name>
+    <Description>**</Description>
+  </SetLanguage>
+
+  Replace the ** with the relevant detail. Code is the two letter language code according to ISO 3166.
+
+Example: To add the French translation to the dublin.mds scroll down to just below the MetadataSet xml block, and add the SetLanguage xml block:
+
+  <SetLanguage code="fr">
+    <Name language="fr">El\303\251ments de m\303\251tadonn\303\251es du Dublin Core, Version 1.1: Description de R\303\251ference</Name>
+    <Description language="fr">Ce document r\303\251sume...</Description>
+  </SetLanguage>
+
+4. For each Element xml block defined in the metadata set, add a new Language xml block and within it add the language specfic Attributes. The block is of the form:
+
+    <Language code="**">
+       <Attribute name="**">**</Attribute>
+       ...
+    </Language>
+
+  Replace the ** with the relevant detail. Once again code is the two letter language code according to ISO 3166. Note that the names specified in the name argument of an Attribute must also be listed in the language_dependent argument of the MetadataSet (see 1)
+
+Example: Thus to add a French translation for the Element "Title", scroll down to the Element xml block defining Title. This should start:
+
+  <Element name="Title" remember_values="true">
+  ...
+  ...
+  ...
+  </Element>
+
+Now add in a new Language block so it looks something like this:
+
+  <Element name="Title" remember_values="true">
+  ...
+     <!-- French -->
+     <Language code="fr">
+        <Attribute name="identifier">Titre</Attribute>
+        <Attribute name="definition">Le nom donn\303\251 \303\240  la ressource.</Attribute>
+        <Attribute name="comment">Typiquement, un titre sera le nom par lequel la ressource est officiellement connue.</Attribute>
+     </Language>
+  ...
+  </Element>    
+
+5. Finally edit the file <gli_install>/classes/xml/languages.xml. Locate the appropriate Language and add the argument mds="true" to enable this language. This controls what languages appear on the preferences page within the GLI.
+
+Example: Finally to enable the French translation, find the entry in the languages.xml file that reads:
+
+  <Language name="FRENCH"              code="FR" ... >ROMANCE</Language>
+
+And change it so it now says:
+
+  <Language name="FRENCH"              code="FR" ... mds="true">ROMANCE</Language> 
Index: /main/tags/3.01/gli/metadata/agls.mds
===================================================================
--- /main/tags/3.01/gli/metadata/agls.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/agls.mds	(revision 21112)
@@ -0,0 +1,334 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param description A description of this metadata set.                           -->
+  <!-- @param family      The namespace of this metadata set.                           -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param name        The name of this metadata set.                                -->
+  <!-- @param namespace   The abbreviated namespace of this metadata set.               -->
+  <!ELEMENT MetadataSet (Element+)>
+  <!ATTLIST MetadataSet
+				contact     CDATA #REQUIRED
+				creator     CDATA #REQUIRED
+				description CDATA #REQUIRED
+            family      CDATA #REQUIRED
+				lastchanged CDATA #REQUIRED
+            name        CDATA #REQUIRED
+				namespace   CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!ELEMENT Element (Attribute*,OptionList?,AssignedValues?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+				occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+
+  <!-- Each element also contains a tree of values that have been assigned to it. This may include a multilayer nested subject hierachy. -->
+  <!ELEMENT AssignedValues (Subject+)>
+
+  <!-- A value. -->
+  <!ELEMENT Subject (Value|Alias?|Subject*)>
+  <!ELEMENT Value (#PCDATA)>
+  <!ATTLIST Value
+            occurances CDATA "1">
+  <!ELEMENT Alias (#PCDATA)>
+]>
+<MetadataSet contact="agls@naa.gov.au" creator="The National Archives of Australia" lastchanged="Friday December 3 2004 03:07:42 - mgw5 on narya.cs.waikato.ac.nz" namespace="agls">
+    <Name language="en">Australian Government Locator Service Metadata Element Set, Version 1.3</Name>
+    <Element name="Creator">
+        <Attribute language="en" name="identifier">Creator</Attribute>
+        <Attribute language="en" name="obligation">Mandatory if known</Attribute>
+        <Attribute language="en" name="definition">An entity primarily responsible for making the content of the resource.</Attribute>
+        <Attribute language="en" name="comment">Examples of a Creator include a person, or an organisation.</Attribute>
+    </Element>
+    <Element name="Date">
+        <Attribute language="en" name="identifier">Date</Attribute>
+        <Attribute language="en" name="definition">A date of an event in the lifecycle of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory</Attribute>
+        <Attribute language="en" name="comment">Typically, Date will be associated with the creator or availability of the resource. Recommended best practice for encoding the date value is defined in a profile of ISO 8601 [W3CDTF] and follows the YYYY-MM-DD format.</Attribute>
+        <Element name="created">
+            <Attribute language="en" name="identifier">Created</Attribute>
+            <Attribute language="en" name="definition">Creation date of the resource</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+        </Element>
+        <Element name="modified">
+            <Attribute language="en" name="identifier">Modified</Attribute>
+            <Attribute language="en" name="definition">Creation date of the resource</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+        </Element>
+        <Element name="valid">
+            <Attribute language="en" name="identifier">Valid</Attribute>
+            <Attribute language="en" name="comment">Typically, a date the resource becomes valid or ceases to be valid, or the date range for which the resource is valid.</Attribute>
+            <Attribute language="en" name="definition">A date (often a range) of validity of a resource.</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+        </Element>
+        <Element name="issued">
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">A date on which the rescource was made formally available in its current form.</Attribute>
+            <Attribute language="en" name="identifier">Issued</Attribute>
+        </Element>
+    </Element>
+    <Element name="Title">
+        <Attribute language="en" name="identifier">Title</Attribute>
+        <Attribute language="en" name="definition">A name given to the resource.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory</Attribute>
+        <Attribute language="en" name="comment">This qualifier could include abbreviations and acronyms by which a resource may be known.</Attribute>
+        <Element name="alternative">
+            <Attribute language="en" name="identifier">Alternative</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">Any form of the title used as a substitute or alternative to the formal title or the resource</Attribute>
+            <Attribute language="en" name="comment">This qualifier could include abbreviations and acronyms by which a resource may be known.</Attribute>
+        </Element>
+    </Element>
+    <Element name="Availability">
+        <Attribute language="en" name="definition">How the resource can be obtained or contact information for obtaining the resource.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory for offline resources.</Attribute>
+        <Attribute language="en" name="comment">The Availability element is primarily used for non-electronic resources to provide information on how to obtain physical access to the resource.</Attribute>
+        <Attribute language="en" name="identifier">Availability</Attribute>
+    </Element>
+    <Element name="Function">
+        <Attribute language="en" name="identifier">Function</Attribute>
+        <Attribute language="en" name="definition">The business function of the organisation to which the resource relates.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory if no Subject element specified.</Attribute>
+        <Attribute language="en" name="comment">Used to indicate the business role of the resource in terms of business functions and activities. Functions are the major units of activity which organisations pursue in order to meet the mission and goals of the organisation. They are defined in the Australian Records Management Standard (AS 4390) as the largest unit of business activity in an organisation. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+    </Element>
+    <Element name="Identifier">
+        <Attribute language="en" name="identifier">Resource Identifier</Attribute>
+        <Attribute language="en" name="definition">An unambiguous reference to the resource within a given context.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory for online resources.</Attribute>
+        <Attribute language="en" name="comment">Recommended best practice is to identify the resource by means of a string or number conforming to a formal identification system. Example formal identification systems include the Uniform Resource Identifier (URI) (including the Uniform Resource Locator (URL)), the Digital Object Identifier (DOI) and the International Standard Book Number (ISBN).</Attribute>
+    </Element>
+    <Element name="Publisher">
+        <Attribute language="en" name="identifier">Publisher</Attribute>
+        <Attribute language="en" name="definition">An entity responsible for making the resource available.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory for information resources.</Attribute>
+        <Attribute language="en" name="comment">This field is often the name of the organisation that owns or controls or publishes the resource. It is not recommended that this element be used for the name of the entity which merely acts as the host for a website.</Attribute>
+    </Element>
+    <Element name="Subject">
+        <Attribute language="en" name="identifier">Subject and Keywords</Attribute>
+        <Attribute language="en" name="definition">A subject and/or topic of the content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Mandatory if no Function element specified.</Attribute>
+        <Attribute language="en" name="comment">Typically, a Subject will be expressed as keywords, key phrases or classification codes that describe a topic of the resource content. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+    </Element>
+    <Element name="Audience">
+        <Attribute language="en" name="identifier">Audience</Attribute>
+        <Attribute language="en" name="definition">A target audience of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Types of audiences commonly used in this element include particular industry sectors, education levels, skill levels, occupations, and EEO categories. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+    </Element>
+    <Element name="Contributor">
+        <Attribute language="en" name="identifier">Contributor</Attribute>
+        <Attribute language="en" name="definition">An entity responsible for making a contribution to the content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Typically, a contributor will be an entity that has played an important but secondary role in creating the content of the resource and is not specified in the creator element.</Attribute>
+    </Element>
+    <Element name="Coverage">
+        <Attribute language="en" name="identifier">Coverage</Attribute>
+        <Attribute language="en" name="definition">The extent or scope of the content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Coverage will typically include spatial location (a place name or geographic coordinates), temporal period (a period label, date, or date range) or jurisdiction (such as a named administrative entity). Recommended best practice is to select a value from a controlled vocabulary (for example, the Thesaurus of Geographic Names [TGN]) and that, where appropriate, named places or time periods be used in preference to numeric identifiers such as sets of coordinates or date ranges. </Attribute>
+        <Element name="jurisdiction">
+            <Attribute language="en" name="identifier">Jurisdiction</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The name of the political/administrative entity covered by the content of the resource.</Attribute>
+            <Attribute language="en" name="comment">Jurisdiction is a description of the territory over which a particular government exercises its authority or a particualar business transacts its operations, to which the resource content is applicable.</Attribute>
+        </Element>
+        <Element name="spatial">
+            <Attribute language="en" name="identifier">Spatial</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">Spatial (geographic) characteristics of the intellectual content of the resource.</Attribute>
+        </Element>
+        <Element name="temporal">
+            <Attribute language="en" name="identifier">Temporal</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">Temporal characteristics of the intellectual content of the resource.</Attribute>
+        </Element>
+        <Element name="postcode">
+            <Attribute language="en" name="identifier">Postcode</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">Australian postcode(s) applicable to the spatial coverage of the resource content.</Attribute>
+            <Attribute language="en" name="comment">Postcode refers to the actual Australian postcode(s) which is relevant to the spatial coverage of the resource content. This qualifier will be of particular use in describing services.</Attribute>
+        </Element>
+    </Element>
+    <Element name="Description">
+        <Attribute language="en" name="identifier">Description</Attribute>
+        <Attribute language="en" name="definition">An account of the content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Description may include but is not limited to: an abstract, table of contents, reference to a graphical representation of content (eg a thumbnail of an image), or a free-text account of the content.</Attribute>
+    </Element>
+    <Element name="Format">
+        <Attribute language="en" name="identifier">Format</Attribute>
+        <Attribute language="en" name="definition">The physical or digital manifestation of the resource. </Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Typically, Format may include the media-type or dimensions of the resource. Format may be used to determine the software, hardware or other equipment needed to display or operate the resource. Examples of dimensions include size and duration. Recommended best practice is to select a value from a controlled vocabulary (for example, the list of Internet Media Types [MIME] defining computer media formats).</Attribute>
+        <Element name="extent">
+            <Attribute language="en" name="identifier">Extent</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">The size or duration of the resource.</Attribute>
+            <Attribute language="en" name="comment">The extent qualifier allows the description of the physical dimensions, file size or duration of the resource.</Attribute>
+        </Element>
+        <Element name="medium">
+            <Attribute language="en" name="identifier">Medium</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">The material or physical carrier of the resource.</Attribute>
+        </Element>
+    </Element>
+    <Element name="Language">
+        <Attribute language="en" name="identifier">Language</Attribute>
+        <Attribute language="en" name="definition">A language of the intellectual content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Recommended best practice for the values of the Language element is defined by RFC 3066 [RFC 3066] which includes a two-letter Language Code (taken from the ISO 639 standard [ISO 639]), followed optionally, by a two-letter Country Code (taken from the ISO 3166 standard [ISO 3166]). For example, &apos;en&apos; for English, &apos;fr&apos; for French, or &apos;en-uk&apos; for English used in the United Kingdom.</Attribute>
+    </Element>
+    <Element name="Mandate">
+        <Attribute language="en" name="identifier">Mandate</Attribute>
+        <Attribute language="en" name="definition">A specific warrant which requires the resource to be created or provided.</Attribute>
+        <Attribute language="en" name="obligation">Optional </Attribute>
+        <Attribute language="en" name="comment">The element is useful to indicate the specific legal mandate which requires the resource being described to be created or provided to the public. The content of this element will usually be a reference to a specific Act, Regulation or Case, but may be a URI pointing to the legal instrument in question. </Attribute>
+        <Element name="act">
+            <Attribute language="en" name="identifier">Act</Attribute>
+            <Attribute language="en" name="definition">A reference to a specific State or Federal Act which requires the creation or provision of the resource</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+        </Element>
+        <Element name="regulation">
+            <Attribute language="en" name="identifier">Regulation</Attribute>
+            <Attribute language="en" name="definition">A reference to a specific regulation which requires the creation or provision of the resource.</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+        </Element>
+        <Element name="case">
+            <Attribute language="en" name="identifier">Case Law</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">A reference to a specific case which requires the creation or provision of the resource</Attribute>
+        </Element>
+    </Element>
+    <Element name="Relation">
+        <Attribute language="en" name="identifier">Relation</Attribute>
+        <Attribute language="en" name="definition">A reference to a related resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+        <Element name="isVersionOf">
+            <Attribute language="en" name="identifier">Is Version Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is a version, edition, or adaption of the referenced resource. Changes in version imply substative changes in content rather than differences in format.</Attribute>
+        </Element>
+        <Element name="hasVersion">
+            <Attribute language="en" name="identifier">Has Version</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource has a version, edition, or adaptation, namely, the referenced resource.</Attribute>
+        </Element>
+        <Element name="isReplacedBy">
+            <Attribute language="en" name="identifier">Is Replaced By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is supplanted, displaced, or superseded by the referenced resource.</Attribute>
+        </Element>
+        <Element name="replaces">
+            <Attribute language="en" name="identifier">Replaces</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resources supplants, displaces, or supersedes the referenced reource.</Attribute>
+        </Element>
+        <Element name="isRequiredBy">
+            <Attribute language="en" name="identifier">Is Required By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is required by the referenced resource, either physically or logically. </Attribute>
+        </Element>
+        <Element name="requires">
+            <Attribute language="en" name="identifier">Requires</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource requires the referenced resource to support its function, delivery, or coherence of content. </Attribute>
+        </Element>
+        <Element name="isPartOf">
+            <Attribute language="en" name="identifier">Is Part Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is a physical or logical part of the referenced resource. </Attribute>
+        </Element>
+        <Element name="hasPart">
+            <Attribute language="en" name="identifier">Has Part</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource includes the referenced resource either physically or logically. </Attribute>
+        </Element>
+        <Element name="isReferencedBy">
+            <Attribute language="en" name="identifier">Is Referenced By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is referenced, cited, or otherwise pointed to by the referenced resource. </Attribute>
+        </Element>
+        <Element name="references">
+            <Attribute language="en" name="identifier">References</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource references, cites, or otherwise points to the referenced resource. </Attribute>
+        </Element>
+        <Element name="isFormatOf">
+            <Attribute language="en" name="identifier">Is Format Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource is the same intellectual content of the referenced resource, but presented in another format. </Attribute>
+        </Element>
+        <Element name="hasFormat">
+            <Attribute language="en" name="identifier">Has Format</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource pre-existed the referenced resource, which is essentially the same intellectual content presented in another format. </Attribute>
+        </Element>
+        <Element name="isBasisFor">
+            <Attribute language="en" name="identifier">Is Basis For</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Attribute language="en" name="definition">The described resource pre-existed the referenced resource, which is a performance, production, derivation, translation, or interpretation of the described resource. </Attribute>
+        </Element>
+        <Element name="isBasedOn">
+            <Attribute language="en" name="identifier">Is Based On</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement </Attribute>
+            <Attribute language="en" name="definition">The described resource is a performance, production, derivation, translation, or interpretation of the referenced resource. </Attribute>
+        </Element>
+    </Element>
+    <Element name="Rights">
+        <Attribute language="en" name="identifier">Rights Management</Attribute>
+        <Attribute language="en" name="definition">Information about rights held in and over the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Typically, the Rights element will contain a rights management statement for the resource, or refer to a service providing such information. Rights information often encompasses Intellectual Property Rights (IPR), Copyright, and various Property Rights. If the Rights element is absent, no assumptions can be made about the status of these and other rights with respect to the resource.</Attribute>
+    </Element>
+    <Element name="Source">
+        <Attribute language="en" name="identifier">Source</Attribute>
+        <Attribute language="en" name="definition">A reference to a resource from which the present resource is derived.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">The present resource may be derived from the Source resource in whole or in part. Recommended best practice is to reference the source resource by means of a string or number conforming to a formal identification system.</Attribute>
+    </Element>
+    <Element name="Type">
+        <Attribute language="en" name="identifier">Type</Attribute>
+        <Attribute language="en" name="defintion">The nature or genre of the content of the resource.</Attribute>
+        <Attribute language="en" name="obligation">Optional</Attribute>
+        <Attribute language="en" name="comment">Type includes terms describing general categories, genres, or aggregation levels for content. Recommended best practice is to select a value from a controlled vocabulary (for example, the working draft list of Dublin Core Types [DCT1]). To describe the physical or digital manifestation of the resource, use the FORMAT element.</Attribute>
+        <Element name="category">
+            <Attribute language="en" name="identifier">Category</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement </Attribute>
+            <Attribute language="en" name="definition">The generic type of the resource being described. </Attribute>
+            <Attribute language="en" name="comment">The value for this qualifier must be one of either service, document, or agency. </Attribute>
+        </Element>
+        <Element name="aggregtionLevel">
+            <Attribute language="en" name="identifier">Aggregtion Level</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">The level of aggregation of the resource being described.</Attribute>
+            <Attribute language="en" name="comment">There are only two values possible for this qualifier, either item or collection.</Attribute>
+        </Element>
+        <Element name="documentType">
+            <Attribute language="en" name="identifier">Document Type</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">The form of the resource where category = document.</Attribute>
+            <Attribute language="en" name="comment">Document is used in its widest sense and includes such things as software sound files and images. </Attribute>
+        </Element>
+        <Element name="serviceType">
+            <Attribute language="en" name="identifier">Service Type</Attribute>
+            <Attribute language="en" name="qualifier type">Element refinement</Attribute>
+            <Attribute language="en" name="definition">The type of service being offered where category = service. </Attribute>
+        </Element>
+    </Element>
+</MetadataSet>
Index: /main/tags/3.01/gli/metadata/dls.mds
===================================================================
--- /main/tags/3.01/gli/metadata/dls.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/dls.mds	(revision 21112)
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The namespace of this metadata set.                           -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The abbreviated namespace of this metadata set.               -->
+  <!ELEMENT MetadataSet (Element+)>
+  <!ATTLIST MetadataSet
+				contact     CDATA #REQUIRED
+				creator     CDATA #REQUIRED
+            family      CDATA #REQUIRED
+				lastchanged CDATA #REQUIRED
+				namespace   CDATA #REQUIRED>
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- An element in the metadata set. -->
+  <!ELEMENT Element (Attribute*,OptionList?)>
+  <!ATTLIST Element
+				hierarchy  CDATA "false"
+            name       CDATA #REQUIRED
+				occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+
+<MetadataSet contact="nzdl.org" creator="New Zealand Digital Libraries Project: Greenstone" family="Development Library Subset" lastchanged="Tue Jun 04 10:35:30 2002 - jmt12 on blackbird" namespace="dls">
+
+  <Name language="en">Development Library Subset Example Metadata</Name>
+
+  <Description language="en">A metadata set file for use in The Gatherer, created from original metadata found within the Development Library Subset.</Description>
+
+  <Element name="Title" remember_values="true">
+    <Attribute name="comment" language="en"></Attribute>
+    <Attribute name="definition" language="en">The title of this resource.</Attribute>
+    <Attribute name="identifier" language="en">Title</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+  <Element name="Organization" hierarchy="true" remember_values="true">
+    <Attribute name="comment" language="en"></Attribute>
+    <Attribute name="definition" language="en">The organization responsible for producing this resource.</Attribute>
+    <Attribute name="identifier" language="en">Organization</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+  <Element name="Subject" hierarchy="true" remember_values="true">
+    <Attribute name="comment" language="en">Currently uses a hierarchy.</Attribute>
+    <Attribute name="definition" language="en">The subject of this resource.</Attribute>
+    <Attribute name="identifier" language="en">Subject And Keywords</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+  <Element name="Keyword" hierarchy="true" remember_values="true">
+    <Attribute name="comment" language="en">Informal.</Attribute>
+    <Attribute name="definition" language="en">A more specific indication of what the resource can be used for.</Attribute>
+    <Attribute name="identifier" language="en">Keyword</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+  <Element name="Language" remember_values="true">
+    <Attribute name="comment" language="en"></Attribute>
+    <Attribute name="definition" language="en">The language of this resource.</Attribute>
+    <Attribute name="identifier" language="en">Language</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+  <Element name="AZList" hierarchy="true" remember_values="true">
+    <Attribute name="comment" language="en">Not the same as an AZList classifier.</Attribute>
+    <Attribute name="definition" language="en">A customizable AZList based on document titles.</Attribute>
+    <Attribute name="identifier" language="en">AZList</Attribute>
+    <Attribute name="version">1.0</Attribute>
+  </Element>
+
+</MetadataSet>
+
Index: /main/tags/3.01/gli/metadata/dublin.mds
===================================================================
--- /main/tags/3.01/gli/metadata/dublin.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/dublin.mds	(revision 21112)
@@ -0,0 +1,1834 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The name of this sets family.                                 -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The namespace of this metadata set.                           -->
+  <!ELEMENT MetadataSet ((Name|Description|SetLanguage|Element)+)>
+  <!ATTLIST MetadataSet
+	    contact            CDATA #REQUIRED
+	    creator            CDATA #REQUIRED
+	    description        CDATA #REQUIRED
+            family             CDATA #REQUIRED
+	    lastchanged        CDATA #REQUIRED
+            name               CDATA #REQUIRED
+	    namespace          CDATA #REQUIRED
+	    language_dependent CDATA "">
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT SetLanguage (Name, Description?)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!-- Note that the name attribute isn't really Dublin Core compliant - it was just the first thing I came up with. -->
+  <!ELEMENT Element (Attribute*,Language*,OptionList?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+	    occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->	
+  <!-- Given that name was already used for an Element, I decided to use identifier for the language specific 'name', which I've been told isn't really compliant with Dublin Core. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA "">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT Language (Attribute*)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+
+<MetadataSet contact="dcmi-feedback@dublincore.org" creator="Dublin Core Metadata Initiative" family="Dublin Core" lastchanged="Tue Jun 04 10:35:30 2002 - jmt12 on blackbird" namespace="dc" language_dependent="identifier, definition, comment">
+
+  <!-- English Version sourced from http://www.dublincore.org/documents/dces/ -->
+  <SetLanguage code="en">
+    <Name language="en">Dublin Core Metadata Element Set, Version 1.1: Reference Description</Name>
+    <Description language="en">This document is the reference description, version 1.1 of the Dublin Core Metadata Element Set. This document supersedes the Dublin Core Metadata Element Set, version 1.0. See the Dublin Core Home Page (http://dublincore.org) for further information about the workshops, reports, working group papers, projects, and new developments concerning the Dublin Core Metadata Element set</Description>
+  </SetLanguage>
+
+  <!-- Maori Version sourced from http://www.tki.org.nz/r/maori/pedagogy/dc_elements_m.php -->
+  <SetLanguage code="mi">
+    <Name>Dublin Core Huanga - Maori (He Tauira)</Name>
+  </SetLanguage>
+
+    <!-- French Version sourced from http://www-rocq.inria.fr/%7Evercoust/METADATA/DC-fr.1.1.html -->
+    <!--  by Anne-Marie Vercoustre, Inria -->
+  <SetLanguage code="fr">
+    <Name language="fr">ElÃ©ments de mÃ©tadonnÃ©es du Dublin Core, Version 1.1: Description de RÃ©ference</Name>
+    <Description language="fr">Ce document rÃ©sume les derniÃšres dÃ©finitions des Ã©lÃ©ments demÃ©tadonnÃ©es du Dublin Core originellement dÃ©finis dans le document[RFC2413]. Ces nouvelles dÃ©finitions seront officiellement connues comme la Version 1.1. Elles utilisent un standard dÃ©fini formellement pour la description d'Ã©lÃ©ments de mÃ©tadonnÃ©es. Cette formalisation contribue Ã  amÃ©liorer la cohÃ©rence avec d'autres communautÃ©s dÃ©crivant des mÃ©tadonnÃ©es et Ã  augmenter la prÃ©cision, la portÃ©e et la cohÃ©rence interne de la dÃ©finition des Ã©lÃ©ments du Dublin Core.</Description>
+  </SetLanguage>
+
+  <!-- Russian Version sourced from -->
+  <!-- by Olga Barysheva, National Library of Russia -->
+  <SetLanguage code="ru">
+    <Name language="ru">ÐÐ°Ð±ÐŸÑ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core (ÐÑÐ±Ð»ÐžÐœÑÐºÐŸÐ³ÐŸ ÑÐŽÑÐ°) ÐÐµÑÑÐžÑ 1.1: Ð¡Ð¿ÑÐ°Ð²ÐŸÑÐœÐŸÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ</Name>
+    <Description language="ru">ÐÐ°ÑÑÐŸÑÑÐžÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÐŸÐ±ÐŸÐ±ÑÐ°ÐµÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core (ÐÑÐ±Ð»ÐžÐœÑÐºÐŸÐ³ÐŸ ÑÐŽÑÐ°), Ð¿ÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ Ð² RFC2413 (ÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Dublin Core ÐŽÐ»Ñ ÐœÐ°Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÑ ÑÐµÑÑÑÑÐŸÐ²). Ð­ÑÐž ÐœÐŸÐ²ÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÐŸÑÐžÑÐžÐ°Ð»ÑÐœÐŸ ÐžÐ·Ð²ÐµÑÑÐœÑ ÐºÐ°Ðº ÐÐµÑÑÐžÑ 1.1.</Description>
+  </SetLanguage>
+
+  <!-- Spanish Version sourced from http://www.rediris.es/metadata/. Please note that this is a translation for Dublin Core v1.0 -->
+  <!-- by Javi Masa, RedIRIS -->
+  <SetLanguage code="es">
+    <Name language="es">Elementos del conjunto de metadatas de Dublin Core: DescripciÃ³n de Referencia</Name>
+    <Description language="es">Este documento corresponde a una traducciÃ³n del original en lengua inglesa, con fecha 02/11/1997 que se encuentra en http://purl.org/metadata/dublin_core_elements. Este documento es una descripciÃ³n de referencia de los elementos del conjunto de "Dublin Core". Para mÃ¡s informaciÃ³n sobre reuniones de trabajo, documentos de trabajo, proyectos y nuevos desarrollos relacionados con el conjunto de elementos que aquÃ­ se describen consultar la "Home Page" de "Dublin Core" (http://purl.org/metadata/dublin_core). Los elementos poseen nombres descriptivos que pretenden transmitir un significado semÃ¡ntico a los mismos. Para promover una interoperabilidad global, una descripciÃ³n del valor de algunos elementos prodrÃ¡ ser asociada a vocabularios controlados. Se asume que otros vocabularios controlados serÃ¡n desarrollados para asegurar esta interoperabilidad en dominios especÃ­ficos. Cada elemento es opcional y puede repetirse. AdemÃ¡s, los elementos pueden aparecer en cualquier orden. Aunque algunos entornos, como HTML, no diferencian entre mayÃºsculas y minÃºsculas, es recomendable escribir correctamente cada metadata segÃºn su definiciÃ³n para evitar conflictos con otros entornos, como XML (Extensible Markup Language) http://www.w3.org/TR/PR-xml</Description>
+  </SetLanguage>
+
+  <!-- Arabic Version sourced from http://haddouti.de/DC_arabic.html -->
+  <!-- maintained by Hachim Haddouti. -->
+  <SetLanguage code="ar">
+  <Name>Ø¯ØšÙÙÙ ÙÙØ±</Name>
+  <Description>ÙØ±Ø­Øš ØšÙÙ
+ ÙÙ ØµÙØ­Ø©  Ø¯ ØšÙÙÙ ÙÙ  Ø± ØšØ§ÙØ¹Ø±ØšÙ    ÙÙ  ÙØ°Ù Ø§ÙØµÙØ­Ø©  Ù ØšØ§ÙØªØ¹Ø§ÙÙ Ù
+Ø¹ Ø§ÙØ§Ø®ÙØ§Ù Ø§ÙØ¹Ø±Øš Ù
+Ù Ø§ÙÙ
+ÙØªØšØ§ØªØ Ø¯Ø§Ø± Ø§ÙÙØŽØ±Ø Ø§ÙØšØ­ÙØ« Ø§ÙØ¹ÙÙ
+ÙØ© Ù    ÙØ±ØŽØ© Ø¹Ù
+ÙWorking Group On Multilingual Dublin Core (DC-International) Ø</Description>
+  </SetLanguage>
+
+  <!-- Chinese - Simplified Version from http://dc.library.sh.cn/DC/1-1.htm -->
+  <!-- by Liu Wei, Shanghai Library -->
+  <SetLanguage code="cn">
+    <Name>éœæææ žå¿å
+æ°æ®å
+çŽ é1.1çïŒåèæè¿°</Name>
+    <Description>æ¬ææ¡£å
+³äºéœæææ žå¿å
+æ°æ®å
+çŽ é1.1çæ¬çåèæè¿°ãæ­€æè¿°äžå·²ç»æäº€ç»ISOçšäºåœé
+æç¥šçNISOå®æ¹æ åä¿æäžèŽã    éœæææ žå¿å
+æ°æ®éæ¯äžç§è·šé¢åçä¿¡æ¯èµæºæè¿°æ åãè¿éçä¿¡æ¯èµæºè¢«å®ä¹äžºâä»»äœå
+·ææ è¯çäžè¥¿âãæ­€å®ä¹æ¥èªäºTim Berners-Leeç­äººæèçInternet RFC2396 "Uniform Resource Identifiers (URI): Generic Syntax"æä»¶ãéœæææ žå¿å
+æ°æ®çåºçšé¢åæ²¡ææ ¹æ¬çéå¶ã</Description>
+  </SetLanguage>
+
+  <!-- Czech Version from http://www.ics.muni.cz/dublin_core/DC-czech-1.1.html -->
+  <!-- Ustav vypocetni techniky Masarykovy univerzity v Brne -->
+  <SetLanguage code="cs">
+    <Name>Soubor metadatovÃœch prvkÅ¯ Dublin Core, verze 1.1: referenÄnÃ­ popis</Name>
+    <Description>Tento dokument je ÄeskÃœm pÅekladem anglickÃ© verze referenÄnÃ­ho popisu souboru metadatovÃœch prvkÅ¯  Dublin Core, verze 1.1 [DCEN1.1]. Tento dokument nahrazuje ÄeskÃœ pÅeklad souboru metadatovÃœch prvkÅ¯ Dublin Core, verze 1.0 [DCCS1.0]. DalÅ¡Ã­ informace o pracovnÃ­ch seminÃ¡ÅÃ­ch (workshopech), zprÃ¡vÃ¡ch, studiÃ­ch pracovnÃ­ch skupin, projektech a dalÅ¡Ã­m vÃœvoji souboru metadatovÃœch prvkÅ¯ Dublin Core najdete na domovskÃ© strÃ¡nce Dublin Core (http://purl.org/dc)</Description>
+  </SetLanguage>
+
+  <!-- Dutch Version from http://www.kb.nl/coop/donor/project-nl-index.html?/coop/donor/rapporten/DCsimpleformat.html -->
+  <!-- by ... -->
+  <SetLanguage code="nl">
+    <Name>Rapporten Dublin Core Simple Format</Name>
+    <Description>Dit document beschrijft welke Dublin Core metadata elementen er zijn en geeft een korte definitie van elk element. Voor uitgebreide uitleg en gebruik van de Dublin Core elementen wordt u verwezen naar de gebruikershandleiding. Deze vertaling geeft ook de keuzes aan voor standaarden (taalcodes, datum codes) die wij in DONOR verband gaan hanteren. In zoverre kan deze vertaling afwijken van het oorspronkelijke document "Description of Dublin Core Elements" http://purl.org/metadata/dublin_core_elements"</Description>
+  </SetLanguage>
+
+  <!-- Finnish Version from http://www.lib.helsinki.fi/dublin_core/dcref-fin.html -->
+  <!-- by Jani Stenvall and Juha Hakala, Helsinki University Library -->
+  <SetLanguage code="fi">
+    <Name></Name>
+    <Description></Description>
+  </SetLanguage>
+
+  <!-- Interlingua Version sourced from http://www.nautilus.com.br/%7Eensjo/ia/purl.org/dc/documents/rec-dces-19990702.htm -->
+  <!-- by Emerson Jose` Silveira da Costa -->
+  <SetLanguage code="ia">
+    <Name>Collection de elementos de metadatos de Dublin Core, version 1.1: Description de referentia</Name>
+    <Description>Isto es un traduction de un Recommendation del Dublin Core Metadata Initiative. Le publication como un recommendation significa que le specificationes es stabile e su adoption es supportate per le communitate Dublin Core (http://purl.org/dc/).</Description>
+  </SetLanguage>
+
+  <!-- Marathi Version sourced from http://bioinfo.ernet.in/library/dc/ele.txt -->
+  <!-- by by Shubhada Nagarkar, Bioinformatics Centre, University of Pune -->
+  <SetLanguage code="mr">
+    <Name>à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿ à€žà¥à€ à€à€µà¥à€€à¥à€€à¥ à¥§.à¥§ à€žà€à€Šà€°à¥à€­ à€žà¥à€ªà€·à¥à€à¥à€à€°à€£ à¥§.à¥§ à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿà€à€µà¥à€€à¥à€€à¥ à¥§.à¥§.</Name>
+    <Description>à¥§.à¥§ à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿ à€¹à¥ à€µà¥à€à€µà¥à€à€³à¥à€¯à€Ÿ à€µà€¿à€·à€¯à€Ÿà€€à¥à€² à€®à€Ÿà€¹à¥à€€à¥ à€žà€Ÿà€ à¥à€¯à€Ÿà€à€µà€Šà¥à€Šà€²à€à¥ à€®à€Ÿà€¹à¥à€€à¥ à€Šà¥à€£à¥à€¯à€Ÿà€à¥ à€®à€Ÿà€šà€ à€à€¹à¥.  à€¯à€Ÿà€€ à€®à€Ÿà€¹à¥à€€à¥à€žà€Ÿà€ à€Ÿ à€®à¥à€¹à€£à€à¥ à€à¥à€¯à€Ÿà€²à€Ÿ à€à€³à€ à€à€¹à¥ à€
+à€žà€Ÿ à€à€žà€€à¥.  à€
+à€¶à€Ÿ à€ªà¥à€°à€à€Ÿà€°à€à¥ à€µà¥à€¯à€Ÿà€à¥à€¯à€Ÿ à€®à€Ÿà€¹à¥à€€à¥ à€à€Ÿà€³à¥à€¯à€µà€°à¥à€² à€. à€à€«.à€žà¥. à¥šà¥©à¥¯à¥¬URI: Generic Syntax -- Tim Berners Lee et al. à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à€Ÿà€šà€ à€à¥à€£à€€à¥à€¯à€Ÿà€¹à¥ à€ªà¥à€°à€à€Ÿà€°à€à¥à€¯à€Ÿ à€®à€Ÿà€¹à¥à€€à¥ à€žà€Ÿà€ à¥à€¯à€Ÿà€à€žà€Ÿà€ à¥ à€µà€Ÿà€ªà€°à€€à€Ÿ à€¯à¥à€€à¥.  à€€à¥à€¯à€Ÿà€žà€Ÿà€ à¥ à€à¥à€£à€€à€Ÿà€¹à¥ à€®à€à¥à€à€Ÿà€µ à€šà€Ÿà€¹à¥.</Description>
+  </SetLanguage>
+
+  <Element name="Title" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+      <Attribute name="identifier">Title</Attribute>
+      <Attribute name="definition">A name given to the resource.</Attribute>
+      <Attribute name="comment">Typically, a Title will be a name by which the resource is formally known.</Attribute>
+    </Language>
+
+    <!-- Maori -->
+    <Language code="mi">
+      <Attribute name="identifier">Taitara</Attribute>
+      <Attribute name="definition">Te ingoa ka tapaina tetahi rauemi.</Attribute>
+      <Attribute name="comment">Ko te tikanga, ko te Taitara te ingoa ka mohio whanuitia te ingoa.</Attribute>
+    </Language>
+
+    <!-- French -->
+    <Language code="fr">
+      <Attribute name="identifier">Titre</Attribute>
+      <Attribute name="definition">Le nom donnÃ© Ã  la ressource.</Attribute>
+      <Attribute name="comment">Typiquement, un titre sera le nom par lequel la ressource est officiellement connue.</Attribute>
+    </Language>
+
+    <!-- Russian -->
+    <Language code="ru">
+      <Attribute name="identifier">ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ</Attribute>
+      <Attribute name="definition">ÐÐŒÑ, ÐŽÐ°ÐœÐœÐŸÐµ ÑÐµÑÑÑÑÑ.</Attribute>
+      <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÐžÐŒÑ, Ð¿ÐŸÐŽ ÐºÐŸÑÐŸÑÑÐŒ ÑÐµÑÑÑÑ ÐŸÑÐžÑÐžÐ°Ð»ÑÐœÐŸ ÐžÐ·Ð²ÐµÑÑÐµÐœ.</Attribute>
+    </Language>
+
+    <!-- Spanish -->
+    <Language code="es">
+      <Attribute name="identifier">TÃ­tulo</Attribute>
+      <Attribute name="definition">El nombre dado a un recurso, usualmente por el autor.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙØ¹ÙÙØ§Ù</Attribute>
+      <Attribute name="definition">ÙØ°Ø§ Ø§ÙØ§Ø³Ù
+ ÙØ¹Ø·Ù ÙÙÙ
+ØµØ¯Ø±, Ù
+Ù ÙØšÙ Ø§ÙÙ
+ØšØ¯Ø¹ Ø§Ù Ø§ÙÙØ§ØŽØ±</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">èµæºå</Attribute>
+      <Attribute name="definition">èµäºèµæºçåç§°ã</Attribute>
+      <Attribute name="comment">èµæºåäžè¬æèµæºå¯¹è±¡æ­£åŒå
+¬åŒçåç§°ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">NÃ¡zev</Attribute>
+      <Attribute name="definition">JmÃ©no danÃ© zdroji.</Attribute>
+      <Attribute name="comment">NÃ¡zev bude typicky jmÃ©no, pod nÃ­mÅŸ je zdroj oficiÃ¡lnÄ znÃ¡m.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Titel</Attribute>
+      <Attribute name="definition">De naam van de Internetbron. Meestal gegeven door de auteur, maker of uitgever</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language> 
+
+    <!-- Interlingua -->
+    <Language code="ia">
+      <Attribute name="identifier">Titulo</Attribute>
+      <Attribute name="definition">Un nomine date al ressource.</Attribute>
+      <Attribute name="comment">Typicamente se utiliza un nomine per le qual le ressource es formalmente cognite.</Attribute>
+    </Language>  
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€¶à¥à€°à¥à€·à€</Attribute>
+      <Attribute name="definition">à€ªà€°à¥à€¯à€Ÿà€¯à¥ à€¶à¥à€°à¥à€·à€</Attribute>
+      <Attribute name="comment">à€žà€à€à¥à€·à€¿à€ªà¥à€€ ,  à€­à€Ÿà€·à€Ÿà€à€€à€°à¥à€€  à€¶à¥à€°à¥à€·à€</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->	   
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Character String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Creator" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+      <Attribute name="identifier">Creator</Attribute>
+      <Attribute name="definition">An entity primarily responsible for making the content of the resource.</Attribute>
+      <Attribute name="comment">Examples of a Creator include a person, an organisation, or a service. Typically, the name of a Creator should be used to indicate the entity.</Attribute>
+    </Language>
+
+    <!-- Maori -->
+    <Language code="mi">
+      <Attribute name="identifier">Kaihanga</Attribute>
+      <Attribute name="definition">Te putahi, nana, na ratou ranei i matua oti ai te hanganga o te rauemi.</Attribute>
+      <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaihanga. Ko te tikanga, ko te Kaihanga te ingoa ka whakamahia hei tohu i te putahi na ratou te hanganga.</Attribute>
+    </Language>
+
+    <!-- French -->
+    <Language code="fr">
+      <Attribute name="identifier">CrÃ©ateur</Attribute>
+      <Attribute name="definition">L'entitÃ© principalement responsable de la crÃ©ation du contenu de la ressource.</Attribute>
+      <Attribute name="comment">Exemples de CrÃ©ateur incluent une personne, une organisation, ou un service. Typiquement, un nom du CrÃ©ateur devrait Ãªtre utilisÃ© pour dÃ©signer cette entitÃ©.</Attribute>
+    </Language>
+
+    <!-- Russian -->
+    <Language code="ru">
+      <Attribute name="identifier">Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+      <Attribute name="definition">ÐÐžÑÐŸ, ÐœÐµÑÑÑÐµÐµ Ð¿ÐµÑÐ²ÐžÑÐœÑÑ ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÑÑÑ Ð·Ð° ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+      <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+    </Language>
+
+    <!-- Spanish -->
+    <Language code="es">
+      <Attribute name="identifier">Autor o Creador</Attribute>
+      <Attribute name="definition">La persona u organizaciÃ³n responsable de la creaciÃ³n del contenido intelectual del recurso. Por ejemplo, los autores en el caso de documentos escritos, artistas, fotografos e ilustradores en el caso de recursos visuales.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙØ§ØªØš Ø§Ù Ø§ÙÙ
+ØšØ¯Ø¹</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø£ÙÙØ§  Ø¹Ù ÙØŽØŠ Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ± Ù ÙÙÙ
+ØµØ¯Ø± , Ø¹ÙÙ Ø³ØšÙÙ Ø§ÙÙ
+Ø«Ø§Ù , Ø§ÙÙ
+Ø€ÙÙÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙØ«Ø§ØŠÙ Ø§ÙÙ
+ÙØªÙØšØ© , Ù Ø§ÙÙÙØ§ÙÙÙ , Ø§ÙÙ
+ØµÙØ±ÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">åå»ºè
+</Attribute>
+      <Attribute name="definition">åå»ºèµæºå
+å®¹çäž»èŠèŽ£ä»»è
+ã</Attribute>
+      <Attribute name="comment">åå»ºè
+çå®äŸå
+æ¬äžªäººïŒç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšåå»ºè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">TvÅ¯rce</Attribute>
+      <Attribute name="definition">Entita primÃ¡rnÄ odpovÄdnÃ¡ za vytvoÅenÃ­ obsahu zdroje.</Attribute>
+      <Attribute name="comment">PÅÃ­klady TvÅ¯rce zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no tvÅ¯rce.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Auteur of maker</Attribute>
+      <Attribute name="definition">De auteur, maker of organisatie die primair verantwoordelijk is voor de intellectuele inhoud van het werk. Bijvoorbeeld auteurs in geval van geschreven documenten, artiesten, fotografen of illustrators in geval van visuele bronnen.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+      <Attribute name="identifier">Creator</Attribute>
+      <Attribute name="definition">Le entitate primarimente responsabile pro producer le contento del ressource.</Attribute>
+      <Attribute name="comment">Exemplos de creator include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de creator pro indicar iste entitate.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€šà€¿à€°à¥à€®à€Ÿà€€à€Ÿ, à€à€šà€</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€šà€¿à€°à¥à€®à€Ÿà€£  à€à€°à€£à€Ÿà€°à€Ÿ</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ.  à€²à¥à€à€, à€²à¥à€à€¿à€à€Ÿ,  à€µà¥à€¯à€à¥à€€à¥, à€žà€à€žà¥à€¥à€Ÿ,  à€žà€à€ªà€Ÿà€Šà€, à€žà€à€ªà€Ÿà€Šà€¿à€à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Subject" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Subject and Keywords</Attribute>
+    <Attribute name="definition">The topic of the content of the resource.</Attribute>
+    <Attribute name="comment">Typically, a Subject will be expressed as keywords, key phrases or classification codes that describe a topic of the resource. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Kaupapa</Attribute>
+    <Attribute name="definition">Te take/ nga take o te kaupapa, o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tikanga, ka whakaaturia te kaupapa o te rauemi e nga kupu matua, nga kimahi matua, nga tohu wehewehenga whakaahua ranei.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Sujet</Attribute>
+    <Attribute name="definition">Le sujet du contenu de la ressource.</Attribute>
+    <Attribute name="comment">Typiquement, le sujet sera dÃ©crit par un ensemble de mots-clefs ou de phrases ou un code de classification qui prÃ©cisent le sujet de la ressource. L'utilisation de vocabulaires contrÃŽlÃ©s et de schÃ©mas formels de classification est encouragÃ©e.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÑÐµÐŽÐŒÐµÑ Ðž ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð°</Attribute>
+    <Attribute name="definition">Ð¢ÐµÐŒÐ° ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÑÐµÐŽÐŒÐµÑ Ð²ÑÑÐ°Ð¶Ð°ÐµÑÑÑ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÐ»ÐŸÐ², ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÑÐ°Ð· ÐžÐ»Ðž  ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÐºÐŸÐŽÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑ ÑÐµÐŒÑ ÑÐµÑÑÑÑÐ°. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ ÐžÐ»Ðž ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÑ
+ÐµÐŒÑ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Claves</Attribute>
+    <Attribute name="definition">Los tÃ³picos del recurso. TÃ­picamente, Subject expresarÃ¡ las claves o frases que describen el tÃ­tulo o el contenido del recurso. Se fomentarÃ¡ el uso de vocabularios controlados y de sistemas de clasificaciÃ³n formales. Se pueden ver ejemplos de vocabularios controlados en: Controlled vocabularies, thesauri and classification systems available in the WWW. DC Subject</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙ
+ÙØ¶ÙØ¹ Ù Ø§ÙÙÙÙ
+Ø§Øª Ø§ÙØ±ØŠÙØ³ÙØ©</Attribute>
+      <Attribute name="definition">ÙÙ Ù
+ÙØ¶ÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±. ÙÙ
+ÙØ°Ø¬ÙØ§  ÙØ¹ØšØ± Ø¹Ù Ø§ÙÙ
+ÙØ¶ÙØ¹ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+Ù
+ÙØªØ§Ø­  Ø§ÙÙÙÙ
+Ø§Øª  Ø§Ù Ø¬Ù
+Ù Ø§ÙØªÙ  ØªØµÙ Ù
+ÙØ¶ÙØ¹ Ø§Ù Ù
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø± , Ù ÙØ³ØªØ­Ø³Ù Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ù
+ÙØ±Ø¯Ø§Øª Ù
+Ø®Ø·Ø·Ø©Ù Ù
+Ø­ÙÙ
+Ø©  ÙØ§Ø¯ÙØ© ØªØµÙÙÙ ØŽÙÙÙØ©</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">äž»é¢åå
+³é®è¯</Attribute>
+      <Attribute name="definition">èµæºå
+å®¹çäž»é¢æè¿°ã</Attribute>
+      <Attribute name="comment">åŠæèŠæè¿°ç¹å®èµæºçæäžäž»é¢ïŒäžè¬éçšå
+³é®è¯ãå
+³é®å­ç­è¯­æåç±»å·ïŒæå¥œäž»é¢åå
+³é®è¯ä»åæ§    è¯è¡šæè§èçåç±»äœç³»äž­ååŒã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">PÅedmÄt a klÃ­ÄovÃ¡ slova</Attribute>
+      <Attribute name="definition">TÃ©ma obsahu zdroje.</Attribute>
+      <Attribute name="comment">PÅedmÄt bude typicky vyjÃ¡dÅen pomocÃ­ klÃ­ÄovÃœch slov, klÃ­ÄovÃœch frÃ¡zÃ­ nebo klasifikaÄnÃ­ch znakÅ¯ popisujÃ­cÃ­ch tÃ©ma zdroje. DoporuÄuje se vybÃ­rat hodnotu prvku z ÅÃ­zenÃ©ho slovnÃ­ku nebo formÃ¡lnÃ­ho klasifikaÄnÃ­ho schÃ©matu.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Onderwerp en trefwoorden</Attribute>
+      <Attribute name="definition">Het onderwerp van de Internetbron. Het onderwerp wordt beschreven in korte zinnen of door het gebruik van trefwoorden. Het gebruik van gecontroleerde trefwoorden of classificatie schema's wordt aanbevolen.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Subjecto e parolas-clave</Attribute>
+    <Attribute name="definition">Le topico del contento del ressource.</Attribute>
+    <Attribute name="comment">Typicamente le subjecto es expresse per medio de parolas-clave, phrases-clave, o codices de classification que describe le topico del ressource. Le melior practica recommendate es selectionar un valor a partir de un vocabulario controlate o un schema formal de classification.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€µà€¿à€·à€¯</Attribute>
+      <Attribute name="definition">à€µà€¿à€·à€¯</Attribute>
+      <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà¥ à€¯à€Ÿà€€à¥à€² à€à€¶à€¯ à€žà¥à€ªà€·à¥à€ à€à€°à€£à€Ÿà€°à¥ à€¶à€¬à¥à€Š, à€µà€¿à€·à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£ à€à€°à€€à€Ÿà€šà€Ÿ à€¶à€¬à¥à€Šà€žà€à€à¥à€°à€¹ à€
+à€¥à€µà€Ÿ à€¶à€¬à¥à€Š à€à¥à€·à€Ÿà€à€Ÿ à€µà€Ÿà€ªà€° à€à€°à€Ÿà€µà€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Description" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Description</Attribute>
+    <Attribute name="definition">An account of the content of the resource.</Attribute>
+    <Attribute name="comment">Description may include but is not limited to: an abstract, table of contents, reference to a graphical representation of content or a free-text account of the content.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">	
+    <Attribute name="identifier">Whakaaturanga</Attribute>
+    <Attribute name="definition">He whakaaturanga o nga kaupapa o te rauemi.</Attribute>
+    <Attribute name="comment">Ka uru atu, engari ehara noa ko: te whakarapopototanga korero, te ihirangi, te tohutanga o te kaupapa ma te whakaahua, te whakaaturanga kore kupu ranei, o te kauapapa.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Description</Attribute>
+    <Attribute name="definition">Une description du contenu de la ressource.</Attribute>
+    <Attribute name="comment">Une Description peut contenir, mais ce n'est pas limitatif: un rÃ©sumÃ©, une table des matiÃšres, une rÃ©fÃ©rence Ã  une reprÃ©sentation graphique du contenu, ou un texte libre sur le contenu.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ</Attribute>
+    <Attribute name="definition">Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐµ ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐž ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐŸ (Ð¿ÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) Ð² Ð²ÐžÐŽÐµ: ÑÐµÑÐµÑÐ°ÑÐ°, ÐŸÐ³Ð»Ð°Ð²Ð»ÐµÐœÐžÑ, ÑÑÑÐ»ÐºÐž ÐœÐ° Ð³ÑÐ°ÑÐžÑÐµÑÐºÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÐžÐ»Ðž Ð¿ÑÐŸÑÑÐŸÐ³ÐŸ ÑÐµÐºÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐžÐ·Ð»ÐŸÐ¶ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">DescripciÃ³n</Attribute>
+    <Attribute name="definition">Una descripciÃ³n textual del recurso, tal como un resumen en el caso de un documento o una descripciÃ³n del contenido en el caso de un documento visual.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙØµÙ</Attribute>
+      <Attribute name="definition">ÙÙ ÙØµÙ ÙØµÙ  ÙÙ
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø±, ÙÙØ¶Ù
+ Ø§ÙÙ
+ÙØ®ØµØ§Øª ÙÙ Ø­Ø§ÙØ© Ø§ÙÙØ«Ø§ØŠÙ Ù
+Ø«Ù Ø§ÙØ§ØŽÙØ§Ø¡ Ø§ÙÙ
+ÙØªÙØšØ© Ø§Ù ØªÙØµÙÙ Ø§ÙÙ
+Ø­ØªÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">è¯Žæ</Attribute>
+      <Attribute name="definition">èµæºå
+å®¹çè§£é</Attribute>
+      <Attribute name="comment">è¯Žæå¯ä»¥å
+æ¬äœäžéäºä»¥äžå
+å®¹ïŒææãç®åœãå¯¹ä»¥åŸåœ¢æ¥æ­ç€ºå
+å®¹çèµæºèèšçæå­è¯Žæãæè
+äž    äžªæå
+³èµæºå
+å®¹çèªç±ææ¬æè¿°ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Popis</Attribute>
+      <Attribute name="definition">VysvÄtlenÃ­ obsahu zdroje.</Attribute>
+      <Attribute name="comment">Popis mÅ¯ÅŸe obsahovat (mimo jinÃ©): abstrakt, obsah, odkaz na grafickÃ© nebo volnÃ© textovÃ© vyjÃ¡dÅenÃ­ obsahu.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Omschrijving</Attribute>
+      <Attribute name="definition">Een omschrijving van de inhoud van de Internetbron. Abstracts of inhoudsomschrijvingen.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Description</Attribute>
+    <Attribute name="definition">Un description del contento del ressource.</Attribute>
+    <Attribute name="comment">Le description pote includer, ma non se limita a: summario, indice, referentia a un representation graphic del contento o un texto libere a proposito del contento.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€µà€°à¥à€£à€š</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà€Ÿà€à¥à€µà€°à¥à€£à€š</Attribute>
+      <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€šà€¿à€µà¥à€Šà€š, à€à€²à¥à€, à€à€¿à€€à¥à€°à€°à¥à€ª</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Publisher" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Publisher</Attribute>
+    <Attribute name="definition">An entity responsible for making the resource available.</Attribute>
+    <Attribute name="comment">Examples of a Publisher include a person, an organisation, or a service. Typically, the name of a Publisher should be used to indicate the entity.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Kaiwhakaputa</Attribute>
+    <Attribute name="definition">Te putahi nana i whakaputa te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaputa. Ko te tikanga, ka whakamahia te ingoa o te Kaiwhakaputa hei tohu i te putahi.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Editeur</Attribute>
+    <Attribute name="definition">L'entitÃ© responsable de la diffusion de la ressource, dans sa forme actuelle, tels, un dÃ©partement universitaire, une entreprise.</Attribute>
+    <Attribute name="comment">Exemples d'Editeurs incluent une personne, une organisation, ou un service. Typiquement, le nom d'une maison d'Ã©dition devrait Ãªtre utilisÃ© ici.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+    <Attribute name="definition">ÐÐžÑÐŸ, ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÐµ Ð·Ð° Ð²Ð²ÐŸÐŽ ÑÐµÑÑÑÑÐ° Ð² ÐŸÐ±ÑÐ°ÑÐµÐœÐžÐµ.</Attribute>
+    <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Editor</Attribute>
+    <Attribute name="definition">>La entidad responsable de hacer que el recurso se encuentre disponible en la red en su formato actual, por ejemplo la empresa editora, un departamento universitario u otro tipo de organizaciÃ³n.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙØ§ØŽØ±</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙÙÙØŠØ© Ø§ÙÙ
+Ø³Ø€ÙÙØ© Ø¹Ù ØµÙØ§Ø¹Ø© Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙÙ
+ØªÙÙØ± ÙÙ Ø­Ø§ÙØªÙ Ø§ÙØ­Ø§Ø¶Ø±Ø©, Ù
+Ø«Ù Ø¯Ø§Ø± Ø§ÙÙØŽØ±, ÙØ³Ù
+ Ø¬Ø§Ù
+Ø¹Ù Ø§Ù Ù
+Ø€Ø³Ø³Ø© Ø§Ø¹Ù
+Ø§Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">åºçè
+</Attribute>
+      <Attribute name="definition">äœ¿èµæºæäžºå¯ä»¥è·åŸå¹¶å¯çšçèŽ£ä»»è
+ã</Attribute>
+      <Attribute name="comment">åºçè
+çå®äŸå
+æ¬äžªäœïŒç»ç»ïŒææå¡ãäžè¬èèšïŒåºè¯¥çšåºçè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Vydavatel</Attribute>
+      <Attribute name="definition">Entita odpovÄdnÃ¡ za zpÅÃ­stupnÄnÃ­ zdroje.</Attribute>
+      <Attribute name="comment">PÅÃ­klady Vydavatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no vydavatele.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Uitgever</Attribute>
+      <Attribute name="definition">De entiteit die verantwoordelijk is voor het beschikbaarstellen van de Internetbron in de huidig vorm. Bijvoorbeeld een uitgeversbedrijf, een universiteitsafdeling.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Editor</Attribute>
+    <Attribute name="definition">Un entitate responsabile pro disponibilizar le ressource.</Attribute>
+    <Attribute name="comment">Exemplos de editor include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de editor pro indicar le entitate.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€¶à€</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ  à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ.à€ à€ªà¥à€°à€à€Ÿà€¶à€ , à€ªà¥à€°à€à€Ÿà€¶à€š  à€žà€à€žà¥à€¥à€Ÿ, à€žà¥à€µà€Ÿ    à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€ªà¥à€² à€°à¥à€ªà€Ÿà€€  à€
+à€¥à€µà€Ÿ à€à€à€à€°à€šà¥à€ à€µà€° à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Contributor" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Contributor</Attribute>
+    <Attribute name="definition">An entity responsible for making contributions to the content of the resource.</Attribute>
+    <Attribute name="comment">Examples of a Contributor include a person, an organisation, or a service. Typically, the name of a Contributor should be used to indicate the entity.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Kaiwhakaaro</Attribute>
+    <Attribute name="definition">Te putahi nana, na ratou ranei nga whakakaupapatanga whakaaro o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaaro. Ko te tikanga ka whakamahia te ingoa o te Kaiwhakaaro hei tohu i te putahi.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Contributeur</Attribute>
+    <Attribute name="definition">Une entitÃ© qui a contribuÃ© Ã  la crÃ©ation du contenu de la ressource.</Attribute>
+    <Attribute name="comment">Exemples de Contributeur incluent une personne, une organisation, ou un service. Typiquement, le nom d'un contributeur devrait Ãªtre utilisÃ© ici pour dÃ©signer l'entitÃ©.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ</Attribute>
+    <Attribute name="definition">ÐÐžÑÐŸ, Ð²ÐœÐµÑÑÐµÐµ Ð²ÐºÐ»Ð°ÐŽ Ð² ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Otros Colaboradores</Attribute>
+    <Attribute name="definition">Una persona u organizacion que haya tenido una contribuciÃ³n intelectual significativa en la creaciÃ³n del recurso pero cuyas contribuciones son secundarias en comparaciÃ³n a las de las personas u organizaciones especificadas en el elementeo Creator (por ejemplo, editor, ilustrador y traductor).</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙ
+Ø³Ø§ÙÙ
+ Ø§ÙØ§Ø®Ø±</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙØºÙØ± Ù
+Ø°ÙÙØ± ØµØ±Ø§Ø­Ø© ÙÙ Ø¹Ø§Ù
+Ù " Ø§ÙÙ
+ØšØ¯Ø¹ " Ù Ø§ÙØ°Ù ÙØ¯Ù
+ Ù
+Ø³Ø§ÙÙ
+Ø§Øª ÙÙØ±ÙØ© ÙØ§Ù
+Ø© ÙÙÙ
+ØµØ¯Ø±, ÙÙÙ Ù
+Ø³Ø§ÙÙ
+ØªÙØªØšÙÙ    Ø«Ø§ÙÙÙØ© Ù
+ÙØ§Ø±ÙØ© Ù
+Ø¹ Ø§ÙØŽØ®Øµ Ø£Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙÙ
+Ø°ÙÙØ±Ø© ÙØµØ§ ÙÙ Ø¹Ø§Ù
+Ù " Ø§ÙÙ
+ÙØŽØŠ " Ù
+Ø«ÙØ§Ù Ù
+ØµØ­Ø­, Ø§ÙÙ
+ØªØ±Ø¬Ù
+, Ø§Ù Ø§ÙÙ
+ÙØ³Ø±</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">å
+¶ä»èŽ£ä»»è
+</Attribute>
+      <Attribute name="definition">å¯¹èµæºçå
+å®¹äœåºèŽ¡ç®çå
+¶ä»å®äœã</Attribute>
+      <Attribute name="comment">å
+¶ä»èŽ£ä»»è
+çå®äŸå¯å
+æ¬äžªäººãç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšå
+¶ä»èŽ£ä»»è
+çåå­æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">PÅispÄvatel</Attribute>
+      <Attribute name="definition">Entita, kterÃ¡ pÅispÄla k vytvoÅenÃ­ obsahu zdroje.</Attribute>
+      <Attribute name="comment">PÅÃ­klady PÅispÄvatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no pÅispÄvatele.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Andere medewerkers</Attribute>
+      <Attribute name="definition">Een persoon of organisatie die een belangrijke bijdrage heeft geleverd, maar die secondair is aan de persoon of organisatie die bij "Creator" genoemd wordt. Gedacht kan worden aan editors, vertalers en illustrators.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Contribuente</Attribute>
+    <Attribute name="definition">Un entitate responsabile pro facer contributiones al contento del ressource.</Attribute>
+    <Attribute name="comment">Exemplos de contribuente include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de contribuente pro indicar le entitate.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier"> à€žà€¹à€¯à¥à€à€, à€žà€¹à€¯à¥à€à¥, à€žà€¹à€Ÿà€¯à¥à€¯à€</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€žà€à€à€²à¥à€€ à€à€°à€£à¥à¥à€Ÿà€ž  à€®à€Šà€€ à€à€°à€£à€Ÿà€°à¥ à€µà¥à€¯à€à¥à€€à¥</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ.à€ à€µà¥à€¯à€à¥à€€à¥, à€žà¥à€µà€Ÿ, à€žà€à€žà¥à€¥à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Date" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Date</Attribute>
+    <Attribute name="definition">A date associated with an event in the life cycle of the resource.</Attribute>
+    <Attribute name="comment">Typically, Date will be associated with the creation or availability of the resource.  Recommended best practice for encoding the date value is defined in a profile of ISO 8601 and follows the YYYY-MM-DD format.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Ra</Attribute>
+    <Attribute name="definition">He ra e whai panga ana ki te huringa ora o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tikanga, ka whai panga te ra ki te hanganga, te wateatanga ranei, o te rauemi. Ko te huarahi tino pai hei tohu i te uara o te ra (date value) kua whakaritea i roto i te whakaritenga IS0 8601 [W3CDTF], a, e whai atu ana ki te whakatakotoranga YYYY-MM-DD.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Date</Attribute>
+    <Attribute name="definition">Une date associÃ©e avec un Ã©vÃ©nement dans le cycle de vie de la ressource.</Attribute>
+    <Attribute name="comment">Typiquement, une date sera associÃ©e Ã  la crÃ©ation ou Ã  la publication d'une ressource. Il est fortement recommandÃ© d'encoder la valeur de la date en utilisant le format dÃ©fini par l'ISO 8601 [W3CDTF] sous la forme AAAA-MM-JJ.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÐ°ÑÐ°</Attribute>
+    <Attribute name="definition">ÐÐ°ÑÐ°, ÑÐ²ÑÐ·Ð°ÐœÐœÐ°Ñ Ñ ÑÐŸÐ±ÑÑÐžÐµÐŒ Ð² Ð¶ÐžÐ·ÐœÐµÐœÐœÐŸÐŒ ÑÐžÐºÐ»Ðµ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÐ°ÑÐ° Ð°ÑÑÐŸÑÐžÐžÑÑÐµÑÑÑ Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµÐŒ ÐžÐ»Ðž ÐŽÐŸÑÑÑÐ¿ÐœÐŸÑÑÑÑ ÑÐµÑÑÑÑÐ°. ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐŽÐ»Ñ ÐºÐŸÐŽÐžÑÐŸÐ²ÐºÐž Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐ°ÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÑÐŸÑÐžÐ»Ñ ISO 8601 (Ð€ÐŸÑÐŒÐ°ÑÑ ÐŽÐ°ÑÑ Ðž Ð²ÑÐµÐŒÐµÐœÐž W3C), ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ ÑÐŸÑÐŒÐ°Ñ ÐÐÐÐ-ÐÐ-ÐÐ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Fecha</Attribute>
+    <Attribute name="definition">Una fecha en la que el recurso se puso a disposiciÃ³n del usuario en su forma actual. Esta fecha no ha de confundirse con la que pertenece al elemento Coverage, que serÃ­a asociada con el recurso sÃ³lo en la medida en que el contenido intelectual estÃ¡ de algÃºn modo relacionado con esa fecha. Recomendamos la utiliazaciÃ³n de uno de los formatos definidos en el documento "Date and Time Formats", http://www.w3.org/TR/NOTE-datetime basado en la norma ISO 8601 que incluye, entre otras, fechas en el formato AAAA y AAAA-MM-DD. De esta forma la fecha 1994-11-05 corresponderÃ­a al 5 de Noviembre de 1994.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙØªØ§Ø±Ø®</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙÙ
+ØªØ¹ÙÙ ØšÙÙØª ÙØŽØŠ Ø£Ù ØªÙÙØ± Ø§ÙÙ
+ØµØ¯Ø± . Ù ÙØ¬Øš Ø£Ù ÙØ§ ÙØ®ÙØ· ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙØ¹Ø§ØŠØ¯ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØºØ·ÙØ© , Ù Ø§ÙØ° Ù ÙØ¬Øš Ø£Ù    ÙÙØ­Ù ØšØ§ÙÙ
+ØµØ¯Ø± ÙÙØ· Ù
+Ù ÙØ§Ø­ÙØ© Ø§ÙÙ
+Ø­ØªÙÙØ§ÙÙÙØ±Ù ÙÙÙÙÙ ÙÙØ¹Ø§ Ù
+Ø§ ÙÙ ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ®. Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø³Ø«Ø­Ø³ÙØ© ÙÙØ§Ø­Ø³Ù Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ    Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙ ÙÙ Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø¹ØªÙ
+Ø¯Ø© ÙÙ ØªØŽÙÙÙ (ISO 8601) ØµÙØºØ© ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601) Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© http://www.w3.org/TR/NOTE-datetime, W3C. ÙÙÙ    ÙØ°Ø§ Ø§ÙÙØ³Ù , Ù
+Ø«ÙØ§, ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ 1994-11-04 ÙÙØ§ØžØ± ÙÙÙ
+ 5 ÙÙÙÙ
+ØšØ± 1994</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">æ¥æ</Attribute>
+      <Attribute name="definition">äžèµæºçåœåšæäž­çäžäžªäºä»¶çžå
+³çæ¶éŽã</Attribute>
+      <Attribute name="comment">äžè¬èèšïŒæ¥æåºäžèµæºçåå»ºæåºçæ¥æçžå
+³ãå»ºè®®éçšçæ¥ææ ŒåŒåºç¬ŠåISO 8601 [W3CDTF]è§    èïŒå¹¶äœ¿çšYYYY-MM-DDçæ ŒåŒã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Datum</Attribute>
+      <Attribute name="definition">Datum spojenÃ© s urÄitou udÃ¡lostÃ­ bÄhem existence zdroje.</Attribute>
+      <Attribute name="comment">Datum bude typicky spojeno s vytvoÅenÃ­m nebo zpÅÃ­stupnÄnÃ­m zdroje. DoporuÄenÃœ postup pro kÃ³dovÃ¡nÃ­ hodnoty data je definovÃ¡n v ISO 8601 [W3CDTF]: formÃ¡t RRRR-MM-DD.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Datum</Attribute>
+      <Attribute name="definition">De datum van de totstandkoming of beschikbaarstelling van een Internetbron. Deze datum moet niet verward worden met de datum in "Coverage". De datum in "Coverage" geeft de tijdsperiode van de inhoud van de Internetbron aan. Voorgeschreven wordt de ISO 8601 standaard voor datum, zoals beschreven in W3C Technical Note http://www.w3.org/TR/NOTE-datetime. Hierin wordt het formaat YYYY-MM-DD beschreven. Het formaat YYYY-MM-DD wordt de voorgeschreven standaard. 3 September 1998 wordt in dit geval 1998-09-03.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Data</Attribute>
+    <Attribute name="definition">Un data associate a un evento in le cyclo vital del ressource.</Attribute>
+    <Attribute name="comment">Typicamente le data es associate al creation o disponibilization del ressource. Le practica recommendate pro codificar le valor del datas es definite in un profilo de ISO 8601 [W3CDTF] e seque le formato AAAA-MM-DD.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€€à€Ÿà€°à¥à€,  à€Šà€¿à€šà€Ÿà€à€</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€€à€¯à€Ÿà€° à€à¥à€²à¥à€¯à€Ÿà€à¥ à€€à€Ÿà€°à¥à€</Attribute>
+      <Attribute name="comment">à€€à€Ÿà€°à¥à€  à€²à€¿à€¹à¥à€£à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª à€ à€µà€°à¥à€·. à€®à€¹à€¿à€šà€Ÿ. à€Šà€¿à€µà€ž
+à¥§à¥¯à¥«à¥Š. à¥Šà¥ª. à¥Šà¥ª</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Type" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Resource Type</Attribute>
+    <Attribute name="definition">The nature or genre of the content of the resource.</Attribute>
+    <Attribute name="comment">Type includes terms describing general categories, functions, genres, or aggregation levels for content. Recommended best practice is to select a value from a controlled vocabulary (for example, the working draft list of Dublin Core Types). To describe the physical or digital manifestation of the resource, use the FORMAT element.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Tumomo</Attribute>
+    <Attribute name="definition">Te ahua, te tu-ahuatanga ranei o nga kaupapa o te rauemi.</Attribute>
+    <Attribute name="comment">Kei roto i te Tumomo nga kupu whakaatu i nga wehewehenga whanui, nga aheinga, nga tu-ahuatanga, te whakaritenga ranei o nga taumata o te/ nga kaupapa. Ko te huarahi tino pai kua tohua, ko te tipako i tetahi uara mai i nga kupu kua whakaritea (hei tauira, ko te tauira rarangi e whakamahia ana o nga Dublin Core Types [DCT1]. Hei whakaatu i te whakatinanatanga o te rauemi, whakamahia ko te huanga WHAKATAKOTORANGA (FORMAT).</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Type</Attribute>
+    <Attribute name="definition">La nature ou le genre du contenu de la ressource.</Attribute>
+    <Attribute name="comment">Type inclut des termes dÃ©crivant des catÃ©gories, fonctions ou genres gÃ©nÃ©raux pour le contenu, ou des niveaux d'agrÃ©gation. Il est recommandÃ© de choisir la valeur du type dans une liste de vocabulaire contrÃŽlÃ© (par exemple, la liste provisoire de Types du Dublin Core[DCT1]). Pour dÃ©crire la matÃ©rialisation physique ou digitale de la ressource, il faut utiliser l'Ã©lÃ©ment Format.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">Ð¢ÐžÐ¿ ÑÐµÑÑÑÑÐ°</Attribute>
+    <Attribute name="definition">ÐÑÐžÑÐŸÐŽÐ° ÐžÐ»Ðž Ð¶Ð°ÐœÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">Ð¢ÐžÐ¿ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐµÑÐŒÐžÐœÑ, ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑÐžÐµ ÐŸÐ±ÑÐžÐµ ÐºÐ°ÑÐµÐ³ÐŸÑÐžÐž, ÑÑÐœÐºÑÐžÐž, Ð¶Ð°ÐœÑÑ ÐžÐ»Ðž ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÐœÑÐµ ÑÑÐŸÐ²ÐœÐž ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÐµÐºÑÐ° ÑÐ¿ÐžÑÐºÐ° Ð¢ÐžÐ¿ÐŸÐ² Dublin Core DCT1). ÐÐ»Ñ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ ÑÐžÐ·ÐžÑÐµÑ-ÐºÐŸÐ³ÐŸ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐµÑÑÑÑÐ° ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÑÐ»ÐµÐŒÐµÐœÑ Ð€ÐŸÑÐŒÐ°Ñ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Tipo del Recurso</Attribute>
+    <Attribute name="definition">La categorÃ­a del recurso, por ejemplo pÃ¡gina personal, romance, poema, minuta, diccionario. Para asegurar la interoperabilidad, Type deberÃ­a ser seleccionado de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo. En http://sunsite.berkeley.edu/Metadata/types.html se puede consultar el estado actual de la discusiÃ³n en torno a este tema.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">ÙÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+      <Attribute name="definition">ÙÙ ÙØŠØ© Ø§ÙÙ
+ØµØ¯Ø± , Ù
+Ø«ÙØ§ Ø§ÙØµÙØ­Ø© Ø§ÙØ§Ù
+, Ø±ÙØ§ÙØ© , ÙØ·Ø¹Ø© ØŽØ¹Ø±ÙØ© , ÙØ±ÙØ© Ø¹Ù
+Ù , ØªÙØ±ÙØ± ÙÙÙ , Ù
+ÙØ§Ù , ÙØ§Ù
+ÙØ³ . Ù Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù    ÙØ¬Øš Ø£Ù ÙØ®ØªØ§Ø± Ø§ÙÙÙØ¹ Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© ÙØ§ÙØªÙ ÙÙÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">èµæºç±»å</Attribute>
+      <Attribute name="definition">èµæºå
+å®¹çç¹åŸæç±»åã</Attribute>
+      <Attribute name="comment">èµæºç±»åå
+æ¬æè¿°èµæºå
+å®¹çäžè¬èçŽïŒåèœïŒç§å±ïŒæèç±»å±æ¬¡çæ¯è¯­ãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒäŸåŠDCMIç±»åè¯æ±è¡š[DCMITYPE]ïŒãèŠæè¿°èµæºçç©çææ°å­åè¡šç°åœ¢åŒïŒè¯·äœ¿çšâæ ŒåŒïŒFORMATïŒâå
+çŽ ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Typ zdroje</Attribute>
+      <Attribute name="definition">Povaha nebo druh obsahu zdroje.</Attribute>
+      <Attribute name="comment">Typ obsahuje termÃ­ny popisujÃ­cÃ­ obecnÃ© kategorie, funkce, druhy nebo agregaÄnÃ­ ÃºrovnÄ obsahu. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. pracovnÃ­ nÃ¡vrh seznamu TypÅ¯ Dublin Core [DCT1]). K popisu fyzickÃ© nebo digitÃ¡lnÃ­ reprezentace zdroje uÅŸijte prvek FormÃ¡t.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Bestands type</Attribute>
+      <Attribute name="definition">Het soort Internetbron, bijvoorbeeld home page, gedicht, working paper, technical report, essay, woordenboek etc. De DC working groups zijn momenteel bezig met twee voorstellen; een minimalist draft: http://sunsite.berkeley.edu/Metadata/minimalist.html en een structuralist draft: http://sunsite.berkely.edu/Metadata/structuralist.html. Het wordt sterk aanbevolen om "Type" van deze lijsten te selecteren.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Typo del ressource</Attribute>
+    <Attribute name="definition">Le natura o genere del contento del ressource.</Attribute>
+    <Attribute name="comment">Le typo del ressource include terminos que describe categorias general, functiones, generes, o nivellos de aggregation pro le contento. Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le lista de schizzos de travalio de typos de Dublin Core [DCT1]). Pro describer le manifestation physic o digital del ressource, utiliza le elemento Formato.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€°</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿà€ªà¥à€°à€à€Ÿà€°</Attribute>
+      <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£, à€µà€°à¥à€à€µà€Ÿà€°à¥à€®à€§à¥à€¯à¥ à€à€à€Ÿà€Šà¥à€¯à€Ÿ à€šà¥à€à€Šà¥à€à€Ÿ à€®à¥à€à¥à€¯ à€à€à€ªà¥à€°à€à€Ÿà€°</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Format" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Format</Attribute>
+    <Attribute name="definition">The physical or digital manifestation of the resource.</Attribute>
+    <Attribute name="comment">Typically, Format may include the media-type or dimensions of the resource. Format may be used to determine the software, hardware or other equipment needed to display or operate the resource. Examples of dimensions include size and duration. Recommended best practice is to select a value from a controlled vocabulary (for example, the list of Internet Media Types [MIME] defining computer media formats).</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Whakatakotoranga</Attribute>
+    <Attribute name="definition">Te whakatinanatanga, te whakaputanga a-mati o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tikanga, ka taea te Whakatakotoranga te whakamahi hei whakarite i te rawa-ngohe (software), te rawa-maro (hardware), ma etahi taputapu ranei me matua whiwhi, hei whakahaere i te rauemi. Ko te rahi me te roanga etahi tauira hanga. Ko te tino huarahi kua tohua ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, te rarangi o nga Tumomo Papaho Ipurangi (Internet Media Types [MIME]) e whakaatu ana i nga whakatakotoranga papaho rorohiko.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Format</Attribute>
+    <Attribute name="definition">La matÃ©rialisation physique ou digitale de la ressource.</Attribute>
+    <Attribute name="comment">Typiquement, Format peut inclure le media ou les dimensions de la ressource. Format peut Ãªtre utilisÃ© pour prÃ©ciser le logiciel, le matÃ©riel ou autre Ã©quipement nÃ©cessaire pour afficher ou faire fonctionner la ressource. Exemples de dimensions incluent la taille et la durÃ©e. Il est recommandÃ© de choisir la valeur du format dans une liste de vocabulaire contrÃŽlÃ©(par exemple, la liste des types de media dÃ©finis sur Internet [MIME]).</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">Ð€ÐŸÑÐŒÐ°Ñ</Attribute>
+    <Attribute name="definition">Ð€ÐžÐ·ÐžÑÐµÑÐºÐŸÐµ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐžÐ¿ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž (ÐŒÐµÐŽÐžÐ°-ÑÐžÐ¿) ÐžÐ»Ðž Ð²ÐµÐ»ÐžÑÐžÐœÑ ÑÐµÑÑÑÑÐ°. Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐŽÐ»Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð°Ð¿Ð¿Ð°ÑÐ°ÑÐœÐŸÐ³ÐŸ, Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐœÐŸÐ³ÐŸ ÐŸÐ±ÐµÑÐ¿ÐµÑÐµÐœÐžÑ ÐžÐ»Ðž ÐŽÑÑÐ³ÐŸÐ³ÐŸ ÐŸÐ±ÐŸÑÑÐŽÐŸÐ²Ð°ÐœÐžÑ, ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÐ³ÐŸ ÐŽÐ»Ñ Ð²ÐŸÑÐ¿ÑÐŸÐžÐ·Ð²ÐµÐŽÐµÐœÐžÑ ÐžÐ»Ðž ÑÐ°Ð±ÐŸÑÑ Ñ ÑÐµÑÑÑÑÐŸÐŒ. ÐÑÐžÐŒÐµÑÑ Ð²ÐµÐ»ÐžÑÐžÐœÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐ°Ð·ÐŒÐµÑ Ðž Ð¿ÑÐŸÐŽÐŸÐ»Ð¶ÐžÑÐµÐ»ÑÐœÐŸÑÑÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ¿ÐžÑÐºÐ° ÐÐµÐŽÐžÐ°-ÑÐžÐ¿ÐŸÐ² ÐÐœÑÐµÑÐœÐµÑ MIME, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑÐµÐ³ÐŸ ÐºÐŸÐŒÐ¿ÑÑÑÐµÑÐœÑÐµ ÑÐŸÑÐŒÐ°ÑÑ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž).</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Formato</Attribute>
+    <Attribute name="definition">El formato de datos de un recurso, usado para identificar el software y posiblemente, el hardware que se necesitarÃ­a para mostrar el recurso. Para asegurar la interoperabilidad, los valores de Format deberÃ­an ser seleccionados de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙØµÙØºØ©</Attribute>
+      <Attribute name="definition">ÙØ°Ù ØµÙØºØ© Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø± , Ù ØªØ³ØªØ®Ø¯Ù
+ ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙØšØ±Ù
+Ø¬ÙØ§Øª ÙØ±ØšÙ
+Ø§ Ø§ÙØµÙØš ÙÙØ­Ø§Ø³ÙØš Ù Ø§ÙØªÙ ÙÙ
+ÙÙ Ø£Ù ÙØ­ØªØ§Ø¬    ÙÙØ§ ÙØ¹Ø±Ø¶ Ø£Ù ØªØŽØºÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù, ÙØ¬Øš Ø£Ù ØªØ®ØªØ§Ø± Ø§ÙØµÙØºØ© Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§    Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">æ ŒåŒ</Attribute>
+      <Attribute name="definition">èµæºçç©çææ°å­è¡šç°åœ¢åŒã</Attribute>
+      <Attribute name="comment">äžè¬èèšïŒæ ŒåŒå¯èœå
+æ¬èµæºçåªäœç±»åæèµæºçå€§å°ïŒæ ŒåŒå
+çŽ å¯ä»¥çšæ¥å³å®å±ç€ºææäœèµæºæé    çèœ¯ç¡¬ä»¶æå
+¶ä»çžåºè®Ÿå€ïŒäŸåŠå€§å°å
+æ¬èµæºæå çå­åšç©ºéŽåæç»­æ¶éŽãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒïŒäŸåŠInternetåªäœç±»å[MIME]å®ä¹è®¡ç®æºåªäœæ ŒåŒïŒã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">FormÃ¡t</Attribute>
+      <Attribute name="definition">FyzickÃ¡ nebo digitÃ¡lnÃ­ reprezentace zdroje.</Attribute>
+      <Attribute name="comment">Typicky mÅ¯ÅŸe FormÃ¡t obsahovat typ mÃ©dia nebo rozmÄry zdroje. FormÃ¡t lze pouÅŸÃ­vat k urÄenÃ­ softwaru, hardwaru nebo dalÅ¡Ã­ho zaÅÃ­zenÃ­ potÅebnÃ©ho k zobrazenÃ­ nebo k manipulaci se zdrojem. PÅÃ­klady rozmÄrÅ¯ zahrnujÃ­ velikost a dobu trvÃ¡nÃ­. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. seznam typÅ¯ internetovÃœch mÃ©diÃ­ [MIME] definujÃ­cÃ­ formÃ¡ty poÄÃ­taÄovÃœch mÃ©diÃ­).</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Format</Attribute>
+      <Attribute name="definition">Het data formaat van het Internetbestand. Dit gegeven kan gebruikt worden om systeemvereisten (hard- en software) te identificeren die nodig zijn voor raadpleging of bediening van de Internetbron. "Format" is nog in ontwikkeling bij de DC workshop. Uit compatibiliteitsoverwegingen wordt aanbevolen formaatgegevens te selecteren uit een gecontroleerde lijst. Formaten kunnen van de Internet Media Types (MIME types) lijst gekozen worden: http://www.isi.edu/in-notes/iana/assignments/media-types/media-types</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Formato</Attribute>
+    <Attribute name="definition">Le manifestation physic o digital del ressource.</Attribute>
+    <Attribute name="comment">Typicamente le formato pote includer le typo de medio o dimensiones del ressource. Le formato pote esser utilizate pro determinar le software o le equipamento necessari pro exhibir o operar le ressource. Exemplos de dimensiones include grandessa e duration. Le practica recommendate es selectionar un valor de un vocabulario controlate (per exemplo, le lista de typos de medios de Internet [MIME] que defini formatos de medios).</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€°à€à€šà€Ÿ</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€°à€à€šà€Ÿ, à€šà€®à¥à€šà€Ÿà€¬à¥à€Š à€°à€à€šà€Ÿ</Attribute>
+      <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª, à€°à€à€šà€Ÿà€à€Šà€Ÿ. à€€à¥à€¯à€Ÿà€à¥ à€°à¥à€ª à€à€žà¥ à€ªà¥à€žà¥à€€à€à€°à¥à€ª, à€žà€à€à€£à€à¥à€¯ à€°à¥à€ª, à€µà€¿à€Šà¥à€¯à¥à€€ à€ªà€°à€®à€Ÿà€£à¥à€à€à¥à€¯à€Ÿ à€žà€¹à€Ÿà€¯à¥à€¯à€Ÿà€šà¥ à€žà€à€à€£à€à¥à€à¥à€€ à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Identifier" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Resource Identifier</Attribute>
+    <Attribute name="definition">An unambiguous reference to the resource within a given context.</Attribute>
+    <Attribute name="comment">Recommended best practice is to identify the resource by means of a string or number conforming to a formal identification system. Example formal identification systems include the Uniform Resource Identifier (URI) (including the Uniform Resource Locator (URL)), the Digital Object Identifier (DOI) and the International Standard Book Number (ISBN).</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Taututanga</Attribute>
+    <Attribute name="definition">He tohutoro e hangai ana ki te rauemi i roto i tetahi horopaki.</Attribute>
+    <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tautu i te rauemi ma tetahi aho (string), tau ranei, e hangai ana ki tetahi punaha taututanga okawa. Ko etahi punaha taututanga okawa ko te Taututanga Rauemi Orite (Uniform Resource Identifier - URI), te Taututanga Rauemi Tohu-wahi (Uniform Resource Locator - URL), Te Mea A-mati (Digital Object Identifier - DOI) me te Tau Pukapuka Aro Whanui o te Ao (International Standard Book Number - ISBN).</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Identifiant</Attribute>
+    <Attribute name="definition">Une rÃ©fÃ©rence non ambiguÃ« Ã  la ressource dans un contexte donnÃ©</Attribute>
+    <Attribute name="comment">Il est recommandÃ© d'identifier la ressource par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un sytÃšme formel d'identification. Exemples de systemes formels d'identification incluent le "Uniform Resource Identifier" (URI) (qui inclut le "Uniform Resource Locator" (URL)), le "Digital Object Identifier"(DOI) et le "International Standard Book Number"(ISBN).</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÑÐµÑÑÑÑÐ°</Attribute>
+    <Attribute name="definition">ÐÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÐ°Ñ ÑÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐºÐŸÐœÑÐµÐºÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ. ÐÑÐžÐŒÐµÑÑ ÑÐŸÑÐŒÐ°Ð»ÑÐœÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÑÐžÑÑÐµÐŒ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° (URI), (Ð²ÐºÐ»ÑÑÐ°Ñ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÑÐ¹ ÐÐŸÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° URL), ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐŠÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐÐ±ÑÐµÐºÑÐ° (DOI) Ðž ÐÐµÐ¶ÐŽÑÐœÐ°ÑÐŸÐŽÐœÑÐ¹ Ð¡ÑÐ°ÐœÐŽÐ°ÑÑÐœÑÐ¹ ÐÐœÐžÐ¶ÐœÑÐ¹ ÐÐŸÐŒÐµÑ (ISBN).</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Identificador del Recurso</Attribute>
+    <Attribute name="definition">Secuencia de caracteres usados para identificar unÃ­vocamente un recurso. Ejemplos para recursos en lÃ­nea pueden ser URLs y URNs (cuando estÃ©n implementados). Para otros recursos pueden ser usados otros formatos de identificadores, como por ejemplo ISBN ("International Standard Book Number" - NÃºmero Internacional Normalizado para Libros)</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ù
+Ø¹Ø±Ù Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+      <Attribute name="definition">ÙÙ Ù
+Ø¬Ù
+ÙØ¹Ø© Ø­Ø±ÙÙØ© Ø£Ù Ø±ÙÙ
+ ÙØ³ØªØ®Ø¯Ù
+ ÙØªØ¹Ø±ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§Ø­Ø¯ÙØ§ : Ø§ÙØ§Ù
+Ø«ÙØ© ÙØ°Ø§ÙÙ ÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙØŽØšÙÙØ©ØªØ¶Ù
+ ) URNs, URLs Ø¹ÙØ¯ ØšÙØ§ØŠÙØ§ (    ÙØ°ÙÙ ÙÙ
+ÙÙ Ø§Ø¹ØªØšØ§Ø± Ø§ÙÙ
+Ø¹Ø±ÙØ§Øª Ø§ÙØ¹Ø§ÙÙ
+ÙØ© Ø§ÙØ§Ø­Ø§Ø¯ÙØ© Ù
+Ø«Ù Ø§ÙØ±ÙÙ
+ Ø§ÙØ¯ÙÙÙ ÙØªØµÙÙÙ Ø§ÙÙØ§ØªØš (ISBN) Ù
+Ø±ØŽØ­Ø© ÙÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">èµæºæ è¯ç¬Š</Attribute>
+      <Attribute name="definition">åšç¹å®çèåŽå
+ç»äºèµæºçäžäžªæç¡®çæ è¯ã</Attribute>
+      <Attribute name="comment">å»ºè®®å¯¹èµæºçæ è¯éçšç¬Šåæäžæ­£åŒæ è¯äœç³»çå­ç¬Šäž²åæ°å­ç»åãäŸåŠæ­£åŒçæ è¯äœç³»å
+æ¬ç»äžèµ    æºæ è¯ç¬Š(URI) (å
+æ¬åäžèµæºå®äœç¬ŠURL)ãæ°å­å¯¹è±¡æ è¯ç¬Š(DOI)ååœé
+æ åä¹Šå·(ISBN)ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">IdentifikÃ¡tor zdroje</Attribute>
+      <Attribute name="definition">JednoznaÄnÃœ odkaz na zdroj v rÃ¡mci danÃ©ho kontextu.</Attribute>
+      <Attribute name="comment">DoporuÄuje se
+identifikovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯. PÅÃ­klady formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯ zahrnujÃ­ Uniform Resource Identifier (URI) (vÄetnÄ Uniform Resource Locator (URL)), Digital Object Identifier (DOI) a MezinÃ¡rodnÃ­ standardnÃ­ ÄÃ­slo knihy (ISBN).</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Bestandsidentificatie</Attribute>
+      <Attribute name="definition">Een unieke string of nummer ter identificatie van de Internetbron. Voorbeelden zijn URL's of URN's Andere voorbeelden zijn International Standard Book Number (ISBN) of International Standard Serial Number (ISSN)</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Identificator del ressource</Attribute>
+    <Attribute name="definition">Un referentia inambigue al ressource intra un date contexto.</Attribute>
+    <Attribute name="comment">Se recommenda identificar le ressource per medio de un catena de characteres o numero conforme a un systema formal de identification. Exemplos de systemas formal de identification include le Uniform Resource Identifier (URI) (includente le Uniform Resource Locator (URL)), le Digital Object Identifier (DOI) e le International Standard Book Number (ISBN).</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€à€³à€</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿ à€à€ à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ. à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ, à€µà€¿à€¶à€¿à€·à¥à€  à€žà¥à€¥à€³,  à€ªà¥à€žà¥à€€à€à€Ÿà€à€Ÿ à€à€à€€à€°à€°à€Ÿà€·à¥à€à¥à€°à¥à€¯  à€®à€Ÿà€šà€ à€à¥à€°à€®à€Ÿà€à€,  à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€à€³à€ à€ªà€à€µà€¿à€£à€Ÿà€°à€Ÿ à€à€à€à€°à€šà¥à€ à€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Source" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Source</Attribute>
+    <Attribute name="definition">A Reference to a resource from which the present resource is derived.</Attribute>
+    <Attribute name="comment">The present resource may be derived from the Source resource in whole or in part.  Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Matapuna</Attribute>
+    <Attribute name="definition">He tohutoro ki tetahi rauemi koira te matapuna o te rauemi e tirohia ana.</Attribute>
+    <Attribute name="comment">Tera pea i puta te katoa, tetahi wahanga ranei o te rauemi, e tirohia ana, mai i te Matapuna rauemi. Ko te tino huarahi kua tohua ko te tohutoro ki te rauemi ma te aho, ma te tau ranei e hangai ana ki tetahi punaha taututanga okawa.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Source</Attribute>
+    <Attribute name="definition">Une rÃ©fÃ©rence Ã  une ressource Ã  partir de laquelle la ressource actuelle a Ã©tÃ© dÃ©rivÃ©e.</Attribute>
+    <Attribute name="comment">La ressource actuelle peut avoir Ã©tÃ© dÃ©rivÃ©e d'une autre ressource source, en totalitÃ© ou en partie. Il est recommandÃ© de refÃ©rencer cette source par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un systÃšme formel d'identification.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÑÑÐŸÑÐœÐžÐº</Attribute>
+    <Attribute name="definition">Ð¡ÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ, ÐžÐ· ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐœÐ°ÑÑÐŸÑÑÐžÐ¹.</Attribute>
+    <Attribute name="comment">ÐÐ°ÑÑÐŸÑÑÐžÐ¹ ÑÐµÑÑÑÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐžÐ· ÐÑÑÐŸÑÐœÐžÐºÐ° ÑÐµÐ»ÐžÐºÐŸÐŒ ÐžÐ»Ðž ÑÐ°ÑÑÐžÑÐœÐŸ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Fuente</Attribute>
+    <Attribute name="definition">Secuencia de caracteres utilizado para identificar unÃ­vocamente un trabajo a partir del cual proviene el recurso actual. Por ejemplo, es posible usar Source con la fecha de 1603 como descripciÃ³n de una pelÃ­cula basada en una obra de Shakespeare, pero es preferible, en ese caso, usar Relation "IsBasedOn" con una referencia a un recurso distinto cuya descripciÃ³n contenga el elemento Date con valor 1603.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+      <Attribute name="definition">Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø§Øª Ø§ÙØ®Ø§ØµØ© ØšÙ
+ØµØ¯Ø± Ø«Ø§ÙÙ Ù Ø§ÙØ°Ù Ø§Ø³ØªÙ
+Ø¯ Ù
+ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ. Ù
+Ø¹ Ø£ÙÙ Ù
+Ù Ø§ÙÙ
+ÙØµÙ ØšÙ Ø£Ù ØªÙØ¯Ù
+ Ø§ÙØ¹ÙØ§Ù
+Ù    Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ ÙÙØ· , ÙÙÙ
+ÙÙ Ø£Ù ÙØ­ÙÙ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ®, Ø§ÙÙ
+ÙØŽØŠ , Ø§ÙØµÙØºØ© , Ø§ÙÙ
+Ø¹Ø±Ù , Ø§Ù Ø§ÙØ© Ù
+ÙØ© - Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø§Ø®Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ø¹ÙØ¯Ù
+Ø§ ØªØ¹ØªØšØ± Ù
+ÙÙ
+Ø© ÙØ§ÙØªØŽØ§Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ , Ù ÙØ°Ø§ÙÙ ÙÙØµÙ Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙÙÙ Ø§Ø­Ø³Ù     Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+ Ø¹Ø§Ù
+Ù Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">æ¥æº</Attribute>
+      <Attribute name="definition">ç°æèµæºæ¥æºçåç
+§ã</Attribute>
+      <Attribute name="comment">åœåèµæºå¯èœéšåæå
+šéšæºèªè¯¥å
+çŽ ææ è¯çèµæºïŒå»ºè®®å¯¹è¿äžèµæºçæ è¯éçšäžäžªç¬Šåæ­£åŒæ è¯ç³»    ç»çå­äž²åæ°å­ç»å ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Zdroj</Attribute>
+      <Attribute name="definition">Odkaz na zdroj, z nÄhoÅŸ je popisovanÃœ zdroj odvozen.</Attribute>
+      <Attribute name="comment">PopisovanÃœ zdroj mÅ¯ÅŸe bÃœt zcela nebo zÄÃ¡sti odvozen ze zdroje uvedenÃ©ho v prvku Zdroj. DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Bron</Attribute>
+      <Attribute name="definition">Informatie over een tweede bron waar de huidige Internetbron is van afgeleid. Bij "Source" mag een datum, auteur of maker, formaat, identificatie of andere metadata voor de tweede bron worden ingevuld. Voorbeeld: het is mogelijk om een "Source" datum 1603 in een omschrijving van een 1996 film bewerking van een Shakespeare stuk te gebruiken. "Source" is niet bruikbaar wanneer de huidige bron de orginele is.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Fonte</Attribute>
+    <Attribute name="definition">Un referentia a un ressource del qual se deriva le ressource presente.</Attribute>
+    <Attribute name="comment">Le ressource presente pote derivar se del ressource-fonte in toto o in parte. Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o un numero conforme a un systema formal de identification.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€à€à€®</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿ à€à€à€® à€žà€Ÿà€à€à€£à€Ÿà€°à€Ÿ à€à€à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ. à€ªà¥à€žà¥à€€à€, à€®à€Ÿà€žà€¿à€,  à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿà€¯à€Ÿà€à€à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Language" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Language</Attribute>
+    <Attribute name="definition">A language of the intellectual content of the resource.</Attribute>
+    <Attribute name="comment" language="en">Recommended best practice for the values of the Language element is defined by RFC 1766 which includes a two-letter Language Code (taken from the ISO 639 standard, followed optionally, by a two-letter Country Code (taken from the ISO 3166 standard). For example, 'en' for English, 'fr' for French, or 'en-uk' for English used in the United Kingdom.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Reo</Attribute>
+    <Attribute name="definition">Te reo o te whakakaupapatanga o nga matauranga o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tino huarahi kua tohua mo nga uara o te huanga Reo kua whakaritea ki te RFC 1766 [RFC1766] kei roto te Tohu Reo kupu-rua (two-letter Language Code) (no te aro whanui ISO 639 [ISO639], ka taea e tetahi tohu kupu-rua te whai atu (no te aro whanui ISO 3166) [ISO366]). Hei tauira, te 'en' mo te English, te 'fr' mo te French, 'te en-uk' mo te English e whakamahia ana i Ingarangi (UK).</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Langue</Attribute>
+    <Attribute name="definition">La langue du contenu intellectuel de la ressource.</Attribute>
+    <Attribute name="comment">Il est recommandÃ© d'utiliser comme valeur de l'Ã©lÃ©ment Langue celles definies par la RFC 1766 [RFC1766] qui comprend un code de langage Ã  deux caractÃšres(venant du standard ISO 639 [ISO639]), Ã©ventuellement suivi d'un code Ã  deux lettres pour le pays (venant du standard ISO 3166 [ISO3166] ou en franÃ§ais [ISO3166]). Par exemple, 'en' pour l'anglais, 'fr' pour le franÃ§ais, ou 'en-uk' pour l'anglais utilisÃ© au Royaume-Uni.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">Ð¯Ð·ÑÐº</Attribute>
+    <Attribute name="definition">Ð¯Ð·ÑÐº ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ð¯Ð·ÑÐº, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÐŒÐŸÐµ RFC 1766 (Ð¢ÑÐ³Ðž ÐŽÐ»Ñ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐ·ÑÐºÐ°), Ð²ÐºÐ»ÑÑÐ°ÑÑÐžÐŒ ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÐ·ÑÐºÐ° (Ð²Ð·ÑÑÑÐ¹ ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 639 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÐ·ÑÐºÐŸÐ²)), Ð·Ð° ÐºÐŸÑÐŸÑÑÐŒ ÑÐ»ÐµÐŽÑÐµÑ (Ð¿ÑÐž ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÑÑÐ°ÐœÑ (Ð²Ð·ÑÑÑÐŒÐž ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 3166 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÑÑÐ°Ðœ)). ÐÐ°Ð¿ÑÐžÐŒÐµÑ, "en" - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, "fr" - ÐŽÐ»Ñ ÑÑÐ°ÐœÑÑÐ·ÑÐºÐŸÐ³ÐŸ, "en-uk" - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ Ð² ÐÐµÐ»ÐžÐºÐŸÐ±ÑÐžÑÐ°ÐœÐžÐž.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Lengua</Attribute>
+    <Attribute name="definition">Lengua/s del contenido intelectual del recurso. Practicamente el contenido de este campo deberÃ­a coincidir con los de la RFC 1766 (Tags para la identificaciÃ³n de lenguas, http://ds.internic.net/rfc/rfc1766.txt); por ejemplo: en, es, de, fi, ja y zh.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙÙØºØ©</Attribute>
+      <Attribute name="definition">ÙØ°Ù ÙÙ ÙØºØ© Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . ÙÙÙ
+Ø§ ÙØ§Ù  Ø¹Ù
+ÙÙØ§ , Ù
+Ø­ØªÙÙ ÙØžØ§ Ø§ÙØ­ÙÙ ÙØ¬Øš Ø£Ù ÙØªØ·Ø§ØšÙ Ù
+Ø¹ RFC 1766    ] Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ th, ja, fr, es, fi, de, en Ù zh ÙØ±Ù
+ÙØ² ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙÙØºØ© [</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">è¯­ç§</Attribute>
+      <Attribute name="definition">æè¿°èµæºç¥è¯å
+å®¹çè¯­ç§ã</Attribute>
+      <Attribute name="comment">å»ºè®®æ¬å
+çŽ çåŒéçšRFC 3066[RFC3066]ïŒè¯¥æ åäžISO 639 [ISO639]äžèµ·å®ä¹äºç±äž€äžªåäžäžªè±    æå­æ¯ç»æçäž»è¯­èšæ ç­Ÿåå­æ ç­ŸäŸåŠå
+æ¬âenâæâengâæ¥è¡šç€ºEnglish, "akk" æ¥è¡šç€º    "en-GB"è¡šç€ºè±åœè±è¯­ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Jazyk</Attribute>
+      <Attribute name="definition">Jazyk intelektuÃ¡lnÃ­ho obsahu zdroje.</Attribute>
+      <Attribute name="comment">DoporuÄenÃœ postup pro hodnoty prvku Jazyk je definovÃ¡n v RFC 1766 [RFC1766]: dvouznakovÃœ kÃ³d jazyka (pÅevzatÃœ z normy ISO 639 [ISO639]) volitelnÄ nÃ¡sledovanÃœ dvouznakovÃœm kÃ³dem zemÄ (pÅevzatÃœm z normy ISO 3166 [ISO3166]). NapÅÃ­klad 'en' pro angliÄtinu, 'fr' pro francouzÅ¡tinu, 'cs' pro ÄeÅ¡tinu, nebo 'en-uk' pro angliÄtinu uÅŸÃ­vanou ve SpojenÃ©m krÃ¡lovstvÃ­.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Taal</Attribute>
+      <Attribute name="definition">De taal waarin het document is beschreven. Voorgeschreven wordt de ISO639-1 norm, een twee karakter taalcode van de ISO 639(639-1 en 639-2) norm. Taalcodes kunnen van een lijst worden gekozen: http://www.ics.uci.edu/pub/ietf/http/related/iso639.txt, Eventueel kan ISO3166 (landencodes) gebruikt worden om aan te geven in welk gebied de taal wordt gesproken. Deze kunnen ook van een lijst worden gekozen:  http://www.ics.uci.edu/pub/ietf/http/related/iso3166.txt.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Lingua</Attribute>
+    <Attribute name="definition">Le lingua del contento intellectual del ressource.</Attribute>
+    <Attribute name="comment">Le practica recommendate pro le valores del elemento lingua es definite per RFC 1766 [RFC1766], que include un codice de lingua de duo litteras (prendite del standard ISO 639 [ISO639]), sequite optionalmente per un codice de pais de duo litteras (prendite del standard ISO 3166 [ISO3166]). Per exemplo, 'ia' pro interlingua, 'fr' pro francese, o 'en-uk' pro le anglese del Regno Unite.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€­à€Ÿà€·à€Ÿ</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à€Ÿà€·à€Ÿ</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ. à€®à€°à€Ÿà€ à¥ - à€­à€Ÿà€°à€€à¥à€¯ à€­à€Ÿà€·à€Ÿ</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Relation" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Relation</Attribute>
+    <Attribute name="definition">A reference to a related resource.</Attribute>
+    <Attribute name="comment">Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Whai Panga</Attribute>
+    <Attribute name="definition">He tohutoro ki tetahi rauemi whai panga.</Attribute>
+    <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tohutoro ki te rauemi ma te aho, te tau ranei, e hangai ana ki te punaha taututanga okawa.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Relation</Attribute>
+    <Attribute name="definition">Une rÃ©fÃ©rence Ã  une autre ressource qui a un rapport avec cette ressource.</Attribute>
+    <Attribute name="comment">Il est recommandÃ© de rÃ©fÃ©rencer cette ressource par une chaÃ®ne de caractÃšres ou un numÃ©ro conforme Ã  un systÃšme formel d'identification.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÑÐœÐŸÑÐµÐœÐžÐµ</Attribute>
+    <Attribute name="definition">²ÐµÐœÐœÑÐ¹ ÑÐµÑÑÑÑ.</Attribute>
+    <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">RelaciÃ³n</Attribute>
+    <Attribute name="definition">Un identificador de un segundo recurso y su relaciÃ³n con el recurso actual. Este elemento permite enlazar los recursos relacionados y las descripciones de los recursos. Por ejemplo: IsVersionOf - Incluye la ediciÃ³n de un trabajo, IsBasedOn - La traducciÃ³n de un trabajo, IsPartOf - Un capÃ­tulo de un libro, IsFormatOf - Un mecanismo de transformaciÃ³n de un conjunto de datos en una imagen. Para asegurar la interoperabilidad, las relaciones deberÃ­an ser seleccionadas de una lista de elementos que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+      <Attribute name="definition">ÙØ°Ø§ Ù
+Ø¹Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ù Ø¹ÙØ§ÙØªÙ ØšØ§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ . ÙØ³Ù
+Ø­ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù ØšØªÙØ¶ÙØ­ Ø§ÙØ±ÙØ§ØšØ· ØšÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+ØªØ¹ÙÙØ©     ØšØšØ¹Ø¶ÙØ§ Ù ØªÙØµÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ Ø·ØšØ¹Ø© Ø§ÙØ¹Ù
+Ù ) ÙØ³Ø®Ø© Ù
+Ù ( ØªØ±Ø¬Ù
+Ø© Ø¹Ù
+Ù ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ ( ÙØµÙ Ù
+Ù ÙØªØ§Øš ) Ø¬Ø²Ø¡ Ù
+Ù (    Ù ØªØ­ÙÙÙÙ Ù
+ÙÙØ§ÙÙÙÙØ© ÙÙ
+Ø¬Ù
+ÙØ¹Ø© Ù
+Ø¹ÙÙÙ
+Ø§ØªÙØ© Ø§ÙÙ ØµÙØ±Ø© ) ØµÙØºØ© Ù
+Ù ( Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù , ÙØ¬Øš Ø§Ø®ØªÙØ§Ø± Ø§ÙØ¹ÙØ§ÙØ§Øª Ù
+Ù    Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">å
+³è</Attribute>
+      <Attribute name="definition">æ¶åççžå
+³èµæºã</Attribute>
+      <Attribute name="comment">èåœçžå
+³èµæºæ¶ïŒå»ºè®®äœ¿çšç¬Šåæ­£åŒæ è¯ç³»ç»çå­ç¬Šäž²æè
+æ°å­ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">Vztah</Attribute>
+      <Attribute name="definition">Odkaz na pÅÃ­buznÃœ zdroj.</Attribute>
+      <Attribute name="comment">DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Relatie</Attribute>
+      <Attribute name="definition">Een identificatie van een tweede bron en de relatie met de huide Internetbron. Voorbeelden zijn een editie van een werk (IsVersionOf), een vertaling van een werk (IsBasedOn), een hoofdstuk van een boek (IsPartOf) en een mechanische transformatie van een dataset in een plaatje (IsFormatOf). De DC Working Groups zijn bezig met een lijst met de meest verwachte relaties. Aanbevolen wordt om de relaties van deze lijst te gebruiken: http://purl.oclc.org/metadata/dublin_core/wrelationdraft.html.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="definition">Relation</Attribute>
+    <Attribute name="definition">Un referentia a un ressource correlate.</Attribute>
+    <Attribute name="comment">Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o numero conforme a un systema de identification formal.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€šà€Ÿà€€à¥ , à€žà€à€¬à€à€§</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€¶à¥ à€žà€à€¬à€à€§ à€Šà€Ÿà€à€µà€£à€Ÿà€°à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ.  à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ(URL)</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Coverage" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Coverage</Attribute>
+    <Attribute name="definition">The extent or scope of the content of the resource.</Attribute>
+<Attribute name="comment">Coverage will typically include spatial location (a place name or geographic coordinates), temporal period (a period label, date, or date range) or jurisdiction (such as a named administrative entity). Recommended best practice is to select a value from a controlled vocabulary (for example, the Thesaurus of Geographic Names [TGN]) and that, where appropriate, named places or time periods be used in preference to numeric identifiers such as sets of coordinates or date ranges.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Kapitanga</Attribute>
+    <Attribute name="definition">Te roanga, te whanuitanga ranei o te whakakaupapatanga o te rauemi.</Attribute>
+    <Attribute name="comment">Ko te tikanga, ka uru atu te wahi mokowa (te ingoa o tetahi wahi, whakapiritanga matawhenua ranei), whakaritenga wa (he tapatanga wa (period label, ra, whakatakotoranga whanui o nga ra), te whakaritenga ture (penei i te putahi whakahaere). Ko te tino huarahi kua tohua, ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, i tetahi Puna Kupu Ingoa Matawhenua (Geographic Names [TGN] i nga wahi tika, ko nga wahi me nga whakaritenga wa ka whakamahia i mua i te whakamahitanga o nga taututanga tau penei i nga huinga whakapiritanga me nga whakatakotoranga whanui o nga ra.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Couverture</Attribute>
+    <Attribute name="definition">La portÃ©e ou la couverture spatio-temporelle de la ressource.</Attribute>
+    <Attribute name="comment">La couverture typiquement inclut une position gÃ©ographique (le nom d'un lieu ou ses coordonnÃ©es), une pÃ©riode de temps (un nom de pÃ©riode, une date, ou un intervalle de temps) ou une juridiction (telle que le nom d'une entitÃ© administrative). Il est recommandÃ© de choisir la  valeur de Couverture dans un vocabulaire contrÃŽlÃ© (par exemple, un thÃ©saurus de noms gÃ©ographiques, comme[TGN]) et, quand cela est appropriÃ©, des noms de lieux ou de pÃ©riodes plutÃŽt que des identifiants numÃ©riques tels que des coordonnÃ©es ou des intervalles de dates. </Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÑ
+Ð²Ð°Ñ</Attribute>
+    <Attribute name="definition">ÐÑÐŸÑÑÐ¶ÐµÐœÐœÐŸÑÑÑ Ðž Ð³ÑÐ°ÐœÐžÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÑ
+Ð²Ð°Ñ ÐŸÐ±ÑÐºÐœÐŸÐ²ÐµÐœÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²ÐµÐœÐœÐŸÐµ ÐŒÐµÑÑÐŸÐœÐ°Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÐµ (ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŒÐµÑÑÐœÐŸÑÑÐž ÐžÐ»Ðž Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÐºÐŸÐŸÑÐŽÐžÐœÐ°ÑÑ), Ð²ÑÐµÐŒÐµÐœÐœÐŸÐ¹ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐŸÐº (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ Ð¿ÐµÑÐžÐŸÐŽÐ°, ÐŽÐ°ÑÐ° ÐžÐ»Ðž ÑÑÐŽ ÐŽÐ°Ñ) ÐžÐ»Ðž ÑÑÐžÑÐŽÐžÐºÑÐžÑ (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ ÐµÐŽÐžÐœÐžÑÑ Ð°ÐŽÐŒÐžÐœÐžÑÑÑÐ°ÑÐžÐ²ÐœÐŸÐ³ÐŸ ÐŽÐµÐ»ÐµÐœÐžÑ). ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð¢ÐµÐ·Ð°ÑÑÑÑÐ° Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ TGN), Ðž, ÐµÑÐ»Ðž ÑÑÐŸ ÑÐŒÐµÑÑÐœÐŸ, Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐµ ÐŸÑÐŽÐ°ÐµÑÑÑ ÑÐºÐ°Ð·Ð°ÐœÐžÑÐŒ ÐŒÐµÑÑÐœÐŸÑÑÐµÐ¹ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐºÐŸÐ² Ð²ÐŒÐµÑÑÐŸ ÑÐžÑÑÐŸÐ²ÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ², ÐºÐ°Ðº, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐžÑÑÐµÐŒ ÐºÐŸÐŸÑÐŽÐžÐœÐ°Ñ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ ÑÑÐŽÐŸÐ².</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Cobertura</Attribute>
+    <Attribute name="definition">La caracterÃ­stica de cobertura espacial y/o temporal del contenido intelectual del recurso. La cobertura espacial se refiere a una regiÃ³n fÃ­sica (por ejemplo, sector celestial); uso de coordenadas (por ejemplo, longitud y latitud) o nombres de lugares extraidos de una lista controlada. La cobertura temporal se refiere al contenido del recurso en vez de a cuando fue creado o puesto accesible ya que este Ãºltimo pertenece al elemento Date. Se usa el mismo formato basado en http://www.w3.org/TR/NOTE-datetime.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø§ÙØªØºØ·ÙØ©</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙØ®ØµØ§ØŠØµ Ø§ÙÙ
+ÙØ§ÙÙØ© Ø§Ù Ø§ÙØ²Ù
+ÙÙØ© ÙÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . Ø§ÙØªØºØ·ÙØ© Ø§ÙÙ
+ÙØ§ÙÙØ© ØªØŽÙØ± Ø§ÙÙ Ø¬ÙØ© Ø·ØšÙØ¹ÙØ© )Ù
+Ø«Ù , ÙØ·Ø§Ø¹ Ø³Ù
+Ø§ÙÙ (    Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ø§ÙØ§Ø­Ø¯Ø§Ø«ÙØ§Øª ) Ù
+Ø«Ù  Ø®Ø· Ø§ÙØ·ÙÙ Ù Ø®Ø· Ø§ÙØ¹Ø±Ø¶ ( ÙÙ Ø£Ø³Ù
+Ø§Ø¡ Ø§Ù
+ÙÙØ© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù . Ø§ÙØªØºØ·ÙØ© Ø§ÙØ²Ù
+ÙÙØ©    Ø«Ø¹ÙØ¯ Ø§ÙÙ Ù
+Ø§ ÙØ±Ø¬Ø¹ Ø§ÙÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ù ÙÙØ³ Ø§Ù Ù
+ØªÙ ØªÙ
+ ØªÙÙÙØ±Ù ) Ø§ÙØ§Ø®ÙØ± ÙØªØ¹ÙÙ ØšØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® ( Ø§Ø³ØªØ®Ø¯Ø§Ù
+ ÙÙØ³ ØµÙØºØ© Ø§ÙØªØ§Ø±ÙØ®  Ø§Ù Ø§ÙÙÙØª    ) Ø¹Ø§Ø¯Ø© Ù
+Ø¯Ù ] ( ØµÙØ§ØºØ§Øª Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601 Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© [http://www.w3.org/TR/NOTE-datetime, W3C    ÙÙ
+Ø§ ÙÙ Ù
+ÙØµÙ ØšÙ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® Ø§Ù ÙØªØ±Ø§Øª Ø²Ù
+ÙÙØ© Ù
+Ù ÙØ§ØŠÙ
+Ø© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">èŠçèåŽ</Attribute>
+      <Attribute name="definition">èµæºå
+å®¹ææ¶åçå€å»¶äžèŠçèåŽ</Attribute>
+      <Attribute name="comment">èŠçèåŽäžè¬å
+æ¬ç©ºéŽäœçœ®ïŒäžäžªå°åæå°çåæ ïŒãæ¶éŽåºéŽïŒäžäžªæ¶éŽæ è¯ïŒæ¥ææäžäžªæ¥æè    åŽïŒãæè
+æéïŒæ¯åŠåœåçææå®äœïŒæšèèŠçèåŽæå¥œæ¯åèªäºäžäžªåæ§è¯è¡šïŒäŸåŠå°åè¯åº    [TGN]ïŒïŒå¹¶åºå°œå¯èœå°äœ¿çšç±æ°å­è¡šç€ºçåæ ææ¥æåºéŽæ¥æè¿°å°åäžæ¶éŽæ®µã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">PokrytÃ­</Attribute>
+      <Attribute name="definition">Rozsah nebo zÃ¡bÄr obsahu zdroje.</Attribute>
+      <Attribute name="comment">PokrytÃ­ bude typicky obsahovat prostorovÃ© umÃ­stÄnÃ­ (jmÃ©no mÃ­sta nebo zemÄpisnÃ© souÅadnice), ÄasovÃ© obdobÃ­ (oznaÄenÃ­ obdobÃ­, datum nebo ÄasovÃœ interval) nebo jurisdikci (napÅ. jmÃ©no administrativnÃ­ jednotky). DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. Thesaurus of Geographic Names [TGN]) a tam, kde je to vhodnÃ©, uvÃ¡dÄt mÃ­sta nebo ÄasovÃ© Ãºseky spÃ­Å¡e jmÃ©nem neÅŸ ÄÃ­selnÃœmi identifikÃ¡tory jako jsou soubory souÅadnic nebo ÄasovÃ© intervaly.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Dekking</Attribute>
+      <Attribute name="definition">De ruimtelijke of tijdelijke karakteristieken van de intellectuele inhoud van de Internetbron. Ruimtelijke dekking verwijst naar een geografisch gebied. Gebruik coordinaten (longitude (lengtegraad) en latitude (breedtegraad)) of plaatsnamen die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Dekking in de tijd geeft de tijdsperiode aan waarop de inhoud van de Internetbron betrekking heeft en niet wanneer de Internetbron is gemaakt of beschikbaar is gesteld (dit laatste hoort ingevult te worden bij "Date"). Gebruik dezelfde datum/tijd formaat als bij "Date"; W3C Technical Note http://www.w3.org/TR/NOTE-datetime of tijdsperioden die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Coverage is in ontwikkeling bij de Dublin Core Working group http://www.alexandria.ucsb.edu/docs/metadata/dc_coverage.html.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Copertura</Attribute>
+    <Attribute name="definition">Le extension physic o abstracte del contento del ressource.</Attribute>
+    <Attribute name="comment">Le copertura typicamente include un location spatial (un nomine de localitate o coordinatas geographic), un periodo de tempore (un etiquetta de periodo, data, o gamma de datas), o un jurisdiction (tal como un entitate administrative nominate). Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le Thesaurus of Geographical Names [TGN]) e que, quando appropriate, localitates o periodos de tempore nominate sia utilizate in preferentia a identificatores numeric tales como collectiones de coordinatas o gammas de datas.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+      <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à¥à€à¥à€²à€¿à€  à€®à€Ÿà€¹à€¿à€€à¥</Attribute>
+      <Attribute name="comment">à€à€Šà€Ÿ. à€­à¥à€à¥à€²à€¿à€ à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+  <Element name="Rights" remember_values="true">
+    <!-- English -->
+    <Language code="en">
+    <Attribute name="identifier">Rights Management</Attribute>
+    <Attribute name="definition">Information about rights held in and over the resource.</Attribute>
+    <Attribute name="comment">Typically, a Rights element will contain a rights management statement for the resource, or reference a service providing such information. Rights information often encompasses Intellectual Property Rights (IPR), Copyright, and various Property Rights. If the Rights element is absent, no assumptions can be made about the status of these and other rights with respect to the resource.</Attribute>
+    </Language>
+    <!-- Maori -->
+    <Language code="mi">
+    <Attribute name="identifier">Motika</Attribute>
+    <Attribute name="definition">Ko nga mohiohio ki nga motika o tetahi rauemi.</Attribute>
+    <Attribute name="comment">Ko te tikanga, kei te huanga Motika e pupuri ana he korero mo te whakaritenga motika mo te rauemi, he ratonga tohutoro e whakarato ana ranei i enei tumomo mohiohio. He nui nga wa, puta ai i roto i te mohiohio Motika te Mana Whakairo Hinegaro (Intellectual Property Rights - (IPR)), te Manatarua me nga Tuahua Mana Taonga (Property Rights). Ki te kore te huanga Motika, kaore e taea te pohehe noa ki te mana o enei, nga motika ranei e pa ana ki te rauemi.</Attribute>
+    </Language>
+    <!-- French -->
+    <Language code="fr">
+    <Attribute name="identifier">Droits</Attribute>
+    <Attribute name="definition">Information sur les droits sur et au sujet de la ressource.</Attribute>
+    <Attribute name="comment">Typiquement, un Ã©lÃ©ment Droits contiendra un Ã©tat du droit Ã  gÃ©rer une ressource, ou la refÃ©rence Ã  un service fournissant cette information. Ces droits souvent couvrent les droits de propriÃ©tÃ© intellectuelle (IPR), Copyright, et divers droits de propriÃ©tÃ©. Si l'Ã©lÃ©ment Droits est absent, aucune hypothÃšse ne peut Ãªtre faite sur l'Ã©tat de ces droits, ou de tout autre, par rapport Ã  la ressource.</Attribute>
+    </Language>
+    <!-- Russian -->
+    <Language code="ru">
+    <Attribute name="identifier">ÐÑÐ°Ð²ÐŸÐ²ÐŸÐµ ÑÐµÐ³ÑÐ»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ</Attribute>
+    <Attribute name="definition">ÐÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð¿ÑÐ°Ð²Ð°Ñ
+ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐ°ÑÑÑ Ð²ÐœÑÑÑÐž Ðž Ð²ÐœÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+    <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÑÐ»ÐµÐŒÐµÐœÑ ÐÑÐ°Ð²Ð° ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐµ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÑÑ
+ ÐœÐŸÑÐŒÐ°Ñ
+, ÑÐµÐ³ÑÐ»ÐžÑÑÑÑÐžÑ
+ ÑÑÐœÐºÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐµÑÑÑÑÐ° ÐžÐ»Ðž ÑÑÑÐ»ÐºÑ ÐœÐ° ÑÐ»ÑÐ¶Ð±Ñ, Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÑÑ ÑÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ. ÐÑÐ°Ð²ÐŸÐ²Ð°Ñ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸÐ±ÑÑÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐ²ÐµÐŽÐµÐœÐžÑ ÐŸ ÐÑÐ°Ð²Ð°Ñ
+ ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ¹ ÑÐŸÐ±ÑÑÐ²ÐµÐœÐœÐŸÑÑÐž, ÐÐ²ÑÐŸÑÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²Ðµ Ðž ÐŽÑÑÐ³ÐžÑ
+ ÐÐŒÑÑÐµÑÑÐ²ÐµÐœÐœÑÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+. ÐÑÑÑÑÑÑÐ²ÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐÑÐ°Ð²Ð° ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÑÐ²Ð»ÑÑÑÑÑ ÐŸÑÐœÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŽÐ»Ñ ÐºÐ°ÐºÐžÑ
+-Ð»ÐžÐ±ÐŸ Ð¿ÑÐµÐŽÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐ¹ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÐŸÐŒ ÑÑÐ°ÑÑÑÐµ Ðž ÐŽÑÑÐ³ÐžÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+, ÐºÐ°ÑÐ°ÑÑÐžÑ
+ÑÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+    </Language>
+    <!-- Spanish -->
+    <Language code="es">
+    <Attribute name="identifier">Derechos</Attribute>
+    <Attribute name="definition">Una referencia (URL, por ejemplo) para una nota sobre derechos de autor, para un servicio de gestiÃ³n de derechos o para un servicio que darÃ¡ informaciÃ³n sobre tÃ©rminos y condiciones de acceso a un recurso. Una especificaciÃ³n formal del elemento Rights se encuentra catualmente en discusiÃ³n y por lo tanto su uso se considera experimental.</Attribute>
+    </Language>
+
+    <!-- Arabic -->
+    <Language code="ar">
+      <Attribute name="identifier">Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø©</Attribute>
+      <Attribute name="definition">ÙÙ Ø§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ®Ø¯Ù
+Ø© ØªÙØ¯Ù
+ Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø±</Attribute>
+    </Language>
+
+    <!-- Chinese - Simplified -->
+    <Language code="cn">
+      <Attribute name="identifier">æé</Attribute>
+      <Attribute name="definition">æå
+³èµæºæ¬èº«ææçæè¢«èµäºçæéä¿¡æ¯</Attribute>
+      <Attribute name="comment">äžè¬èèšïŒæéå
+çŽ åºå
+æ¬äžäžªå¯¹èµæºçæéå£°æïŒæè
+æ¯å¯¹æäŸè¿äžä¿¡æ¯çæå¡çåç
+§ãæéäžè¬    å
+æ¬ç¥è¯äº§æ(IPR)ïŒçææå
+¶ä»åç§åæ ·çäº§æãåŠææ²¡ææéå
+çŽ çæ æ³šïŒäžå¯ä»¥å¯¹äžèµæºçžå
+³    çäžè¿°æå
+¶ä»æå©çæ
+åµäœåºä»»äœåå®ã</Attribute>
+    </Language>
+
+    <!-- Czech -->
+    <Language code="cs">
+      <Attribute name="identifier">PrÃ¡va</Attribute>
+      <Attribute name="definition">Informace o prÃ¡vech vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+      <Attribute name="comment">Prvek PrÃ¡va bude typicky obsahovat ustanovenÃ­ o sprÃ¡vÄ autorskÃœch/vlastnickÃœch prÃ¡v ke zdroji nebo odkaz na sluÅŸbu poskytujÃ­cÃ­ takovÃ©to informace. Informace o prÃ¡vech Äasto zahrnujÃ­ prÃ¡va duÅ¡evnÃ­ho vlastnictvÃ­ (IPR), autorskÃ© prÃ¡vo a rÅ¯znÃ¡ vlastnickÃ¡ prÃ¡va. JestliÅŸe prvek PrÃ¡va chybÃ­, nelze odvozovat ÅŸÃ¡dnÃ© pÅedpoklady o stavu tÄchto a jinÃœch prÃ¡v vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+    </Language>
+
+    <!-- Dutch -->
+    <Language code="nl">
+      <Attribute name="identifier">Copyright</Attribute>
+      <Attribute name="definition">Een copyright verklaring of een link naar een copyright verklaring of een link naar een service die informatie geeft over de copyright van de Internetbron.</Attribute>
+    </Language>
+
+    <!-- Finnish -->
+    <Language code="fi">
+      <Attribute name="identifier"></Attribute>
+      <Attribute name="definition"></Attribute>
+      <Attribute name="comment"></Attribute>
+    </Language>
+
+    <!-- Interlingua -->
+    <Language code="ia">
+    <Attribute name="identifier">Gestion de derectos</Attribute>
+    <Attribute name="definition">Information super derectos detenite super e a proposito del ressource.</Attribute>
+    <Attribute name="comment">Typicamente le elemento derectos continera un declaration de gestion de derectos pro le ressource, o un referentia a un servicio que provide tal information. Information de derectos frequentemente involve derectos de proprietate intellectual (IPR), copyright e varie derectos de proprietate. Si le elemento Derectos es absente, necun hypothese pote esser facite quanto al stato de iste e altere derectos concernente ressource.</Attribute>
+    </Language>
+
+    <!-- Marathi -->
+    <Language code="mr">
+      <Attribute name="identifier">à€¹à€à¥à€</Attribute>
+      <Attribute name="definition">à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+      <Attribute name="comment">à€žà¥à€µà€€à€à€¹à€Ÿà€à¥ à€²à¥à€à€š à€à€Ÿà€ªà¥à€š à€
+à€¥à€µà€Ÿ à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€° à€ªà¥à€°à€à€Ÿà€¶à€¿à€€ à€à€°à€£à¥à€¯à€Ÿà€à€Ÿ à€²à¥à€à€ à€
+à€¥à€µà€Ÿ à€²à¥à€à€¿à€à¥à€à€Ÿ à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+    </Language>
+
+    <!-- Non-language specific -->
+    <Attribute name="version">1.1</Attribute>
+    <Attribute name="registration_authority" language="en">Dublin Core Metadata Initiative</Attribute>
+    <Attribute name="obligation">Optional</Attribute>
+    <Attribute name="datatype">Charater String</Attribute>
+    <Attribute name="maximum_occurance">Unlimited</Attribute>
+  </Element>
+
+</MetadataSet>
Index: /main/tags/3.01/gli/metadata/ex.mds
===================================================================
--- /main/tags/3.01/gli/metadata/ex.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/ex.mds	(revision 21112)
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The name of this sets family.                                 -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The namespace of this metadata set.                           -->
+  <!ELEMENT MetadataSet ((Name|Description|Element)+)>
+  <!ATTLIST MetadataSet
+				contact     CDATA #REQUIRED
+				creator     CDATA #REQUIRED
+				description CDATA #REQUIRED
+            family      CDATA #REQUIRED
+				lastchanged CDATA #REQUIRED
+            name        CDATA #REQUIRED
+				namespace   CDATA #REQUIRED>
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- An element in the metadata set. -->
+  <!ELEMENT Element (Attribute*,OptionList?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+				occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+
+<MetadataSet contact="nzdl.org" creator="New Zealand Digital Libraries Project: Greenstone" family="Extracted Greenstone" lastchanged="Tue May 20 10:30:32 2003 - jmt12 on narya" namespace="ex" >
+
+  <Name language="en">Extracted Greenstone Metadata 1.0</Name>
+
+  <Description language="en">This document is the reference description, version 1.0 of the automatically extracted Greenstone Metadata Element Set.</Description>
+
+  <Element name="Encoding">
+    <Attribute name="comment" language="en">Not always available.</Attribute>
+    <Attribute name="definition" language="en">The encoding of the imported document as extracted by Greenstone.</Attribute>
+    <Attribute name="identifier" language="en">Encoding</Attribute>
+  </Element>
+
+  <Element name="Language">
+    <Attribute name="comment" language="en">Not always available.</Attribute>
+    <Attribute name="definition" language="en">The language of the imported document as extracted by Greenstone.</Attribute>
+    <Attribute name="identifier" language="en">Language</Attribute>
+  </Element>
+
+  <Element name="Source">
+    <Attribute name="comment" language="en">Hidden in the Librarian Interface as it already shown by the various file trees.</Attribute>
+    <Attribute name="definition" language="en">The filename of the document imported by Greenstone.</Attribute>
+    <Attribute name="identifier" language="en">Source</Attribute>
+  </Element>
+
+  <Element name="Title">
+    <Attribute name="comment" language="en">Not always available.</Attribute>
+    <Attribute name="definition" language="en">The title of the imported document as extracted by Greenstone.</Attribute>
+    <Attribute name="identifier" language="en">Title</Attribute>
+  </Element>
+
+</MetadataSet>
Index: /main/tags/3.01/gli/metadata/nzgls.mds
===================================================================
--- /main/tags/3.01/gli/metadata/nzgls.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/nzgls.mds	(revision 21112)
@@ -0,0 +1,1830 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The name of this sets family.                                 -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The namespace of this metadata set.                           -->
+  <!ELEMENT MetadataSet ((Name|Description|SetLanguage|Element)+)>
+  <!ATTLIST MetadataSet
+	    contact            CDATA #REQUIRED
+	    creator            CDATA #REQUIRED
+	    description        CDATA #REQUIRED
+            family             CDATA #REQUIRED
+	    lastchanged        CDATA #REQUIRED
+            name               CDATA #REQUIRED
+	    namespace          CDATA #REQUIRED
+	    language_dependent CDATA "">
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT SetLanguage (Name, Description?)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!-- Note that the name attribute isn't really Dublin Core compliant - it was just the first thing I came up with. -->
+  <!ELEMENT Element (Attribute*,Language*,OptionList?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+	    occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->	
+  <!-- Given that name was already used for an Element, I decided to use identifier for the language specific 'name', which I've been told isn't really compliant with Dublin Core. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA "">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT Language (Attribute*)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+<MetadataSet contact="nzgls@ssc.govt.nz" creator="Archives New Zealand" language_dependent="identifier, definition, comment" lastchanged="Fri Dec 03 14:35 2004 - mgw5 on narya" namespace="nzgls">
+    <!-- English Version sourced from http://www.e-government.govt.nz/nzgls/standard/ -->
+    <SetLanguage code="en">
+        <Name language="en">New Zealand Government Locator Service Metadata Standard version 2.1</Name>
+        <Description language="en">The NZGLS metadata standard is the official New Zealand Government standard for creating discovery-level metadata (see Cabinet Circular CO (02) 3). The standard is based closely on two well established standards: the Dublin Core Metadata Element Set and the Australian Government Locator Service.</Description>
+    </SetLanguage>
+    <Element name="Title" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Title</Attribute>
+            <Attribute name="definition">A name given to the resource.</Attribute>
+            <Attribute name="comment">Typically, a Title will be a name by which the resource is formally known.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Taitara</Attribute>
+            <Attribute name="definition">Te ingoa ka tapaina tetahi rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ko te Taitara te ingoa ka mohio whanuitia te ingoa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Titre</Attribute>
+            <Attribute name="definition">Le nom donnÃ© Ã  la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, un titre sera le nom par lequel la ressource est officiellement connue.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ÐÐŒÑ, ÐŽÐ°ÐœÐœÐŸÐµ ÑÐµÑÑÑÑÑ.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÐžÐŒÑ, Ð¿ÐŸÐŽ ÐºÐŸÑÐŸÑÑÐŒ ÑÐµÑÑÑÑ ÐŸÑÐžÑÐžÐ°Ð»ÑÐœÐŸ ÐžÐ·Ð²ÐµÑÑÐµÐœ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">TÃ­tulo</Attribute>
+            <Attribute name="definition">El nombre dado a un recurso, usualmente por el autor.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØ¹ÙÙØ§Ù</Attribute>
+            <Attribute name="definition">ÙØ°Ø§ Ø§ÙØ§Ø³Ù
+ ÙØ¹Ø·Ù ÙÙÙ
+ØµØ¯Ø±, Ù
+Ù ÙØšÙ Ø§ÙÙ
+ØšØ¯Ø¹ Ø§Ù Ø§ÙÙØ§ØŽØ±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºå</Attribute>
+            <Attribute name="definition">èµäºèµæºçåç§°ã</Attribute>
+            <Attribute name="comment">èµæºåäžè¬æèµæºå¯¹è±¡æ­£åŒå
+¬åŒçåç§°ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">NÃ¡zev</Attribute>
+            <Attribute name="definition">JmÃ©no danÃ© zdroji.</Attribute>
+            <Attribute name="comment">NÃ¡zev bude typicky jmÃ©no, pod nÃ­mÅŸ je zdroj oficiÃ¡lnÄ znÃ¡m.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Titel</Attribute>
+            <Attribute name="definition">De naam van de Internetbron. Meestal gegeven door de auteur, maker of uitgever</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Titulo</Attribute>
+            <Attribute name="definition">Un nomine date al ressource.</Attribute>
+            <Attribute name="comment">Typicamente se utiliza un nomine per le qual le ressource es formalmente cognite.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€¶à¥à€°à¥à€·à€</Attribute>
+            <Attribute name="definition">à€ªà€°à¥à€¯à€Ÿà€¯à¥ à€¶à¥à€°à¥à€·à€</Attribute>
+            <Attribute name="comment">à€žà€à€à¥à€·à€¿à€ªà¥à€€ ,  à€­à€Ÿà€·à€Ÿà€à€€à€°à¥à€€  à€¶à¥à€°à¥à€·à€</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Creator" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Creator</Attribute>
+            <Attribute name="definition">An entity primarily responsible for making the content of the resource.</Attribute>
+            <Attribute name="comment">Examples of a Creator include a person, an organisation, or a service. Typically, the name of a Creator should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaihanga</Attribute>
+            <Attribute name="definition">Te putahi, nana, na ratou ranei i matua oti ai te hanganga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaihanga. Ko te tikanga, ko te Kaihanga te ingoa ka whakamahia hei tohu i te putahi na ratou te hanganga.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">CrÃ©ateur</Attribute>
+            <Attribute name="definition">L&apos;entitÃ© principalement responsable de la crÃ©ation du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Exemples de CrÃ©ateur incluent une personne, une organisation, ou un service. Typiquement, un nom du CrÃ©ateur devrait Ãªtre utilisÃ© pour dÃ©signer cette entitÃ©.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, ÐœÐµÑÑÑÐµÐµ Ð¿ÐµÑÐ²ÐžÑÐœÑÑ ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÑÑÑ Ð·Ð° ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Autor o Creador</Attribute>
+            <Attribute name="definition">La persona u organizaciÃ³n responsable de la creaciÃ³n del contenido intelectual del recurso. Por ejemplo, los autores en el caso de documentos escritos, artistas, fotografos e ilustradores en el caso de recursos visuales.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØ§ØªØš Ø§Ù Ø§ÙÙ
+ØšØ¯Ø¹</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø£ÙÙØ§  Ø¹Ù ÙØŽØŠ Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ± Ù ÙÙÙ
+ØµØ¯Ø± , Ø¹ÙÙ Ø³ØšÙÙ Ø§ÙÙ
+Ø«Ø§Ù , Ø§ÙÙ
+Ø€ÙÙÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙØ«Ø§ØŠÙ Ø§ÙÙ
+ÙØªÙØšØ© , Ù Ø§ÙÙÙØ§ÙÙÙ , Ø§ÙÙ
+ØµÙØ±ÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">åå»ºè
+</Attribute>
+            <Attribute name="definition">åå»ºèµæºå
+å®¹çäž»èŠèŽ£ä»»è
+ã</Attribute>
+            <Attribute name="comment">åå»ºè
+çå®äŸå
+æ¬äžªäººïŒç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšåå»ºè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">TvÅ¯rce</Attribute>
+            <Attribute name="definition">Entita primÃ¡rnÄ odpovÄdnÃ¡ za vytvoÅenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady TvÅ¯rce zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no tvÅ¯rce.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Auteur of maker</Attribute>
+            <Attribute name="definition">De auteur, maker of organisatie die primair verantwoordelijk is voor de intellectuele inhoud van het werk. Bijvoorbeeld auteurs in geval van geschreven documenten, artiesten, fotografen of illustrators in geval van visuele bronnen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Creator</Attribute>
+            <Attribute name="definition">Le entitate primarimente responsabile pro producer le contento del ressource.</Attribute>
+            <Attribute name="comment">Exemplos de creator include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de creator pro indicar iste entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€šà€¿à€°à¥à€®à€Ÿà€€à€Ÿ, à€à€šà€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€šà€¿à€°à¥à€®à€Ÿà€£  à€à€°à€£à€Ÿà€°à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.  à€²à¥à€à€, à€²à¥à€à€¿à€à€Ÿ,  à€µà¥à€¯à€à¥à€€à¥, à€žà€à€žà¥à€¥à€Ÿ,  à€žà€à€ªà€Ÿà€Šà€, à€žà€à€ªà€Ÿà€Šà€¿à€à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Function">
+        <Attribute language="en" name="identifier">Function</Attribute>
+        <Attribute language="en" name="obligation">Mandatory</Attribute>
+        <Attribute language="en" name="definition">The business function of the organisation to which the resource relates</Attribute>
+        <Attribute language="en" name="comment">Used to indicate the business role of the resource in terms of business functions and activities. Functions are the major units of activity which organisations pursue in order to meet the mission and goals of the organisation. They are defined in the Australian Records Management Standard [AS4390] as the largest unit of business activity in an organisation. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme. Resources being described within the NZ Government should use the Functions of New Zealand (FONZ) thesaurus. Use of this encoding scheme is mandatory for metadata prepared for use by the NZ Government Portal.</Attribute>
+    </Element>
+    <Element name="Subject" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Subject and Keywords</Attribute>
+            <Attribute name="definition">The topic of the content of the resource.</Attribute>
+            <Attribute name="comment">Typically, a Subject will be expressed as keywords, key phrases or classification codes that describe a topic of the resource. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaupapa</Attribute>
+            <Attribute name="definition">Te take/ nga take o te kaupapa, o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka whakaaturia te kaupapa o te rauemi e nga kupu matua, nga kimahi matua, nga tohu wehewehenga whakaahua ranei.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Sujet</Attribute>
+            <Attribute name="definition">Le sujet du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, le sujet sera dÃ©crit par un ensemble de mots-clefs ou de phrases ou un code de classification qui prÃ©cisent le sujet de la ressource. L&apos;utilisation de vocabulaires contrÃŽlÃ©s et de schÃ©mas formels de classification est encouragÃ©e.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐµÐŽÐŒÐµÑ Ðž ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð°</Attribute>
+            <Attribute name="definition">Ð¢ÐµÐŒÐ° ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÑÐµÐŽÐŒÐµÑ Ð²ÑÑÐ°Ð¶Ð°ÐµÑÑÑ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÐ»ÐŸÐ², ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÑÐ°Ð· ÐžÐ»Ðž  ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÐºÐŸÐŽÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑ ÑÐµÐŒÑ ÑÐµÑÑÑÑÐ°. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ ÐžÐ»Ðž ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÑ
+ÐµÐŒÑ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Claves</Attribute>
+            <Attribute name="definition">Los tÃ³picos del recurso. TÃ­picamente, Subject expresarÃ¡ las claves o frases que describen el tÃ­tulo o el contenido del recurso. Se fomentarÃ¡ el uso de vocabularios controlados y de sistemas de clasificaciÃ³n formales. Se pueden ver ejemplos de vocabularios controlados en: Controlled vocabularies, thesauri and classification systems available in the WWW. DC Subject</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+ÙØ¶ÙØ¹ Ù Ø§ÙÙÙÙ
+Ø§Øª Ø§ÙØ±ØŠÙØ³ÙØ©</Attribute>
+            <Attribute name="definition">ÙÙ Ù
+ÙØ¶ÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±. ÙÙ
+ÙØ°Ø¬ÙØ§  ÙØ¹ØšØ± Ø¹Ù Ø§ÙÙ
+ÙØ¶ÙØ¹ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+Ù
+ÙØªØ§Ø­  Ø§ÙÙÙÙ
+Ø§Øª  Ø§Ù Ø¬Ù
+Ù Ø§ÙØªÙ  ØªØµÙ Ù
+ÙØ¶ÙØ¹ Ø§Ù Ù
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø± , Ù ÙØ³ØªØ­Ø³Ù Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ù
+ÙØ±Ø¯Ø§Øª Ù
+Ø®Ø·Ø·Ø©Ù Ù
+Ø­ÙÙ
+Ø©  ÙØ§Ø¯ÙØ© ØªØµÙÙÙ ØŽÙÙÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">äž»é¢åå
+³é®è¯</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çäž»é¢æè¿°ã</Attribute>
+            <Attribute name="comment">åŠæèŠæè¿°ç¹å®èµæºçæäžäž»é¢ïŒäžè¬éçšå
+³é®è¯ãå
+³é®å­ç­è¯­æåç±»å·ïŒæå¥œäž»é¢åå
+³é®è¯ä»åæ§    è¯è¡šæè§èçåç±»äœç³»äž­ååŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PÅedmÄt a klÃ­ÄovÃ¡ slova</Attribute>
+            <Attribute name="definition">TÃ©ma obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅedmÄt bude typicky vyjÃ¡dÅen pomocÃ­ klÃ­ÄovÃœch slov, klÃ­ÄovÃœch frÃ¡zÃ­ nebo klasifikaÄnÃ­ch znakÅ¯ popisujÃ­cÃ­ch tÃ©ma zdroje. DoporuÄuje se vybÃ­rat hodnotu prvku z ÅÃ­zenÃ©ho slovnÃ­ku nebo formÃ¡lnÃ­ho klasifikaÄnÃ­ho schÃ©matu.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Onderwerp en trefwoorden</Attribute>
+            <Attribute name="definition">Het onderwerp van de Internetbron. Het onderwerp wordt beschreven in korte zinnen of door het gebruik van trefwoorden. Het gebruik van gecontroleerde trefwoorden of classificatie schema&apos;s wordt aanbevolen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Subjecto e parolas-clave</Attribute>
+            <Attribute name="definition">Le topico del contento del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le subjecto es expresse per medio de parolas-clave, phrases-clave, o codices de classification que describe le topico del ressource. Le melior practica recommendate es selectionar un valor a partir de un vocabulario controlate o un schema formal de classification.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà€¿à€·à€¯</Attribute>
+            <Attribute name="definition">à€µà€¿à€·à€¯</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà¥ à€¯à€Ÿà€€à¥à€² à€à€¶à€¯ à€žà¥à€ªà€·à¥à€ à€à€°à€£à€Ÿà€°à¥ à€¶à€¬à¥à€Š, à€µà€¿à€·à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£ à€à€°à€€à€Ÿà€šà€Ÿ à€¶à€¬à¥à€Šà€žà€à€à¥à€°à€¹ à€
+à€¥à€µà€Ÿ à€¶à€¬à¥à€Š à€à¥à€·à€Ÿà€à€Ÿ à€µà€Ÿà€ªà€° à€à€°à€Ÿà€µà€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Description" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">An account of the content of the resource.</Attribute>
+            <Attribute name="comment">Description may include but is not limited to: an abstract, table of contents, reference to a graphical representation of content or a free-text account of the content.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whakaaturanga</Attribute>
+            <Attribute name="definition">He whakaaturanga o nga kaupapa o te rauemi.</Attribute>
+            <Attribute name="comment">Ka uru atu, engari ehara noa ko: te whakarapopototanga korero, te ihirangi, te tohutanga o te kaupapa ma te whakaahua, te whakaaturanga kore kupu ranei, o te kauapapa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">Une description du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Une Description peut contenir, mais ce n&apos;est pas limitatif: un rÃ©sumÃ©, une table des matiÃšres, une rÃ©fÃ©rence Ã  une reprÃ©sentation graphique du contenu, ou un texte libre sur le contenu.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐµ ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐž ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐŸ (Ð¿ÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) Ð² Ð²ÐžÐŽÐµ: ÑÐµÑÐµÑÐ°ÑÐ°, ÐŸÐ³Ð»Ð°Ð²Ð»ÐµÐœÐžÑ, ÑÑÑÐ»ÐºÐž ÐœÐ° Ð³ÑÐ°ÑÐžÑÐµÑÐºÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÐžÐ»Ðž Ð¿ÑÐŸÑÑÐŸÐ³ÐŸ ÑÐµÐºÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐžÐ·Ð»ÐŸÐ¶ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">DescripciÃ³n</Attribute>
+            <Attribute name="definition">Una descripciÃ³n textual del recurso, tal como un resumen en el caso de un documento o una descripciÃ³n del contenido en el caso de un documento visual.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØµÙ</Attribute>
+            <Attribute name="definition">ÙÙ ÙØµÙ ÙØµÙ  ÙÙ
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø±, ÙÙØ¶Ù
+ Ø§ÙÙ
+ÙØ®ØµØ§Øª ÙÙ Ø­Ø§ÙØ© Ø§ÙÙØ«Ø§ØŠÙ Ù
+Ø«Ù Ø§ÙØ§ØŽÙØ§Ø¡ Ø§ÙÙ
+ÙØªÙØšØ© Ø§Ù ØªÙØµÙÙ Ø§ÙÙ
+Ø­ØªÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">è¯Žæ</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çè§£é</Attribute>
+            <Attribute name="comment">è¯Žæå¯ä»¥å
+æ¬äœäžéäºä»¥äžå
+å®¹ïŒææãç®åœãå¯¹ä»¥åŸåœ¢æ¥æ­ç€ºå
+å®¹çèµæºèèšçæå­è¯Žæãæè
+äž    äžªæå
+³èµæºå
+å®¹çèªç±ææ¬æè¿°ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Popis</Attribute>
+            <Attribute name="definition">VysvÄtlenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">Popis mÅ¯ÅŸe obsahovat (mimo jinÃ©): abstrakt, obsah, odkaz na grafickÃ© nebo volnÃ© textovÃ© vyjÃ¡dÅenÃ­ obsahu.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Omschrijving</Attribute>
+            <Attribute name="definition">Een omschrijving van de inhoud van de Internetbron. Abstracts of inhoudsomschrijvingen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">Un description del contento del ressource.</Attribute>
+            <Attribute name="comment">Le description pote includer, ma non se limita a: summario, indice, referentia a un representation graphic del contento o un texto libere a proposito del contento.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà€°à¥à€£à€š</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà€Ÿà€à¥à€µà€°à¥à€£à€š</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€šà€¿à€µà¥à€Šà€š, à€à€²à¥à€, à€à€¿à€€à¥à€°à€°à¥à€ª</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Publisher" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Publisher</Attribute>
+            <Attribute name="definition">An entity responsible for making the resource available.</Attribute>
+            <Attribute name="comment">Examples of a Publisher include a person, an organisation, or a service. Typically, the name of a Publisher should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaiwhakaputa</Attribute>
+            <Attribute name="definition">Te putahi nana i whakaputa te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaputa. Ko te tikanga, ka whakamahia te ingoa o te Kaiwhakaputa hei tohu i te putahi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Editeur</Attribute>
+            <Attribute name="definition">L&apos;entitÃ© responsable de la diffusion de la ressource, dans sa forme actuelle, tels, un dÃ©partement universitaire, une entreprise.</Attribute>
+            <Attribute name="comment">Exemples d&apos;Editeurs incluent une personne, une organisation, ou un service. Typiquement, le nom d&apos;une maison d&apos;Ã©dition devrait Ãªtre utilisÃ© ici.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÐµ Ð·Ð° Ð²Ð²ÐŸÐŽ ÑÐµÑÑÑÑÐ° Ð² ÐŸÐ±ÑÐ°ÑÐµÐœÐžÐµ.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Editor</Attribute>
+            <Attribute name="definition">&gt;La entidad responsable de hacer que el recurso se encuentre disponible en la red en su formato actual, por ejemplo la empresa editora, un departamento universitario u otro tipo de organizaciÃ³n.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØ§ØŽØ±</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙÙÙØŠØ© Ø§ÙÙ
+Ø³Ø€ÙÙØ© Ø¹Ù ØµÙØ§Ø¹Ø© Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙÙ
+ØªÙÙØ± ÙÙ Ø­Ø§ÙØªÙ Ø§ÙØ­Ø§Ø¶Ø±Ø©, Ù
+Ø«Ù Ø¯Ø§Ø± Ø§ÙÙØŽØ±, ÙØ³Ù
+ Ø¬Ø§Ù
+Ø¹Ù Ø§Ù Ù
+Ø€Ø³Ø³Ø© Ø§Ø¹Ù
+Ø§Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">åºçè
+</Attribute>
+            <Attribute name="definition">äœ¿èµæºæäžºå¯ä»¥è·åŸå¹¶å¯çšçèŽ£ä»»è
+ã</Attribute>
+            <Attribute name="comment">åºçè
+çå®äŸå
+æ¬äžªäœïŒç»ç»ïŒææå¡ãäžè¬èèšïŒåºè¯¥çšåºçè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Vydavatel</Attribute>
+            <Attribute name="definition">Entita odpovÄdnÃ¡ za zpÅÃ­stupnÄnÃ­ zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady Vydavatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no vydavatele.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Uitgever</Attribute>
+            <Attribute name="definition">De entiteit die verantwoordelijk is voor het beschikbaarstellen van de Internetbron in de huidig vorm. Bijvoorbeeld een uitgeversbedrijf, een universiteitsafdeling.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Editor</Attribute>
+            <Attribute name="definition">Un entitate responsabile pro disponibilizar le ressource.</Attribute>
+            <Attribute name="comment">Exemplos de editor include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de editor pro indicar le entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€¶à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ  à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.à€ à€ªà¥à€°à€à€Ÿà€¶à€ , à€ªà¥à€°à€à€Ÿà€¶à€š  à€žà€à€žà¥à€¥à€Ÿ, à€žà¥à€µà€Ÿ    à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€ªà¥à€² à€°à¥à€ªà€Ÿà€€  à€
+à€¥à€µà€Ÿ à€à€à€à€°à€šà¥à€ à€µà€° à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Mandate">
+        <Attribute language="en" name="identifier">Mandate</Attribute>
+        <Attribute language="en" name="obligation">Recommended</Attribute>
+        <Attribute language="en" name="definition">A specific warrant which requires the resource to be created or provided.</Attribute>
+        <Attribute language="en" name="comment">The element is useful to indicate the specific legal mandate that requires the resource being described to be created or provided to the public. The content of this element will usually be a reference to a specific Act, Regulation or Case, but may be a URI pointing to the legal instrument in question.</Attribute>
+        <Element name="act">
+            <Attribute language="en" name="identifier">Act</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">A reference to a specific Act of Parliament which requires the creation or provision of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="regulation">
+            <Attribute language="en" name="identifier">Regulation</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">A reference to a specific regulation which requires the creation or provision of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="rules">
+            <Attribute language="en" name="identifier">Rules</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The specific rule or bylaw which requires the creation or provision of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="case">
+            <Attribute language="en" name="identifier">Case Law</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">A reference to a specific case which requires the creation or provision of the resource.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Contributor" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Contributor</Attribute>
+            <Attribute name="definition">An entity responsible for making contributions to the content of the resource.</Attribute>
+            <Attribute name="comment">Examples of a Contributor include a person, an organisation, or a service. Typically, the name of a Contributor should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaiwhakaaro</Attribute>
+            <Attribute name="definition">Te putahi nana, na ratou ranei nga whakakaupapatanga whakaaro o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaaro. Ko te tikanga ka whakamahia te ingoa o te Kaiwhakaaro hei tohu i te putahi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Contributeur</Attribute>
+            <Attribute name="definition">Une entitÃ© qui a contribuÃ© Ã  la crÃ©ation du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Exemples de Contributeur incluent une personne, une organisation, ou un service. Typiquement, le nom d&apos;un contributeur devrait Ãªtre utilisÃ© ici pour dÃ©signer l&apos;entitÃ©.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, Ð²ÐœÐµÑÑÐµÐµ Ð²ÐºÐ»Ð°ÐŽ Ð² ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Otros Colaboradores</Attribute>
+            <Attribute name="definition">Una persona u organizacion que haya tenido una contribuciÃ³n intelectual significativa en la creaciÃ³n del recurso pero cuyas contribuciones son secundarias en comparaciÃ³n a las de las personas u organizaciones especificadas en el elementeo Creator (por ejemplo, editor, ilustrador y traductor).</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+Ø³Ø§ÙÙ
+ Ø§ÙØ§Ø®Ø±</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙØºÙØ± Ù
+Ø°ÙÙØ± ØµØ±Ø§Ø­Ø© ÙÙ Ø¹Ø§Ù
+Ù &quot; Ø§ÙÙ
+ØšØ¯Ø¹ &quot; Ù Ø§ÙØ°Ù ÙØ¯Ù
+ Ù
+Ø³Ø§ÙÙ
+Ø§Øª ÙÙØ±ÙØ© ÙØ§Ù
+Ø© ÙÙÙ
+ØµØ¯Ø±, ÙÙÙ Ù
+Ø³Ø§ÙÙ
+ØªÙØªØšÙÙ    Ø«Ø§ÙÙÙØ© Ù
+ÙØ§Ø±ÙØ© Ù
+Ø¹ Ø§ÙØŽØ®Øµ Ø£Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙÙ
+Ø°ÙÙØ±Ø© ÙØµØ§ ÙÙ Ø¹Ø§Ù
+Ù &quot; Ø§ÙÙ
+ÙØŽØŠ &quot; Ù
+Ø«ÙØ§Ù Ù
+ØµØ­Ø­, Ø§ÙÙ
+ØªØ±Ø¬Ù
+, Ø§Ù Ø§ÙÙ
+ÙØ³Ø±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">å
+¶ä»èŽ£ä»»è
+</Attribute>
+            <Attribute name="definition">å¯¹èµæºçå
+å®¹äœåºèŽ¡ç®çå
+¶ä»å®äœã</Attribute>
+            <Attribute name="comment">å
+¶ä»èŽ£ä»»è
+çå®äŸå¯å
+æ¬äžªäººãç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšå
+¶ä»èŽ£ä»»è
+çåå­æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PÅispÄvatel</Attribute>
+            <Attribute name="definition">Entita, kterÃ¡ pÅispÄla k vytvoÅenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady PÅispÄvatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no pÅispÄvatele.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Andere medewerkers</Attribute>
+            <Attribute name="definition">Een persoon of organisatie die een belangrijke bijdrage heeft geleverd, maar die secondair is aan de persoon of organisatie die bij &quot;Creator&quot; genoemd wordt. Gedacht kan worden aan editors, vertalers en illustrators.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Contribuente</Attribute>
+            <Attribute name="definition">Un entitate responsabile pro facer contributiones al contento del ressource.</Attribute>
+            <Attribute name="comment">Exemplos de contribuente include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de contribuente pro indicar le entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier"> à€žà€¹à€¯à¥à€à€, à€žà€¹à€¯à¥à€à¥, à€žà€¹à€Ÿà€¯à¥à€¯à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€žà€à€à€²à¥à€€ à€à€°à€£à¥à¥à€Ÿà€ž  à€®à€Šà€€ à€à€°à€£à€Ÿà€°à¥ à€µà¥à€¯à€à¥à€€à¥</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.à€ à€µà¥à€¯à€à¥à€€à¥, à€žà¥à€µà€Ÿ, à€žà€à€žà¥à€¥à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Date" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Date</Attribute>
+            <Attribute name="definition">A date associated with an event in the life cycle of the resource.</Attribute>
+            <Attribute name="comment">Typically, Date will be associated with the creation or availability of the resource.  Recommended best practice for encoding the date value is defined in a profile of ISO 8601 and follows the YYYY-MM-DD format.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Ra</Attribute>
+            <Attribute name="definition">He ra e whai panga ana ki te huringa ora o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka whai panga te ra ki te hanganga, te wateatanga ranei, o te rauemi. Ko te huarahi tino pai hei tohu i te uara o te ra (date value) kua whakaritea i roto i te whakaritenga IS0 8601 [W3CDTF], a, e whai atu ana ki te whakatakotoranga YYYY-MM-DD.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Date</Attribute>
+            <Attribute name="definition">Une date associÃ©e avec un Ã©vÃ©nement dans le cycle de vie de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, une date sera associÃ©e Ã  la crÃ©ation ou Ã  la publication d&apos;une ressource. Il est fortement recommandÃ© d&apos;encoder la valeur de la date en utilisant le format dÃ©fini par l&apos;ISO 8601 [W3CDTF] sous la forme AAAA-MM-JJ.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ°ÑÐ°</Attribute>
+            <Attribute name="definition">ÐÐ°ÑÐ°, ÑÐ²ÑÐ·Ð°ÐœÐœÐ°Ñ Ñ ÑÐŸÐ±ÑÑÐžÐµÐŒ Ð² Ð¶ÐžÐ·ÐœÐµÐœÐœÐŸÐŒ ÑÐžÐºÐ»Ðµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÐ°ÑÐ° Ð°ÑÑÐŸÑÐžÐžÑÑÐµÑÑÑ Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµÐŒ ÐžÐ»Ðž ÐŽÐŸÑÑÑÐ¿ÐœÐŸÑÑÑÑ ÑÐµÑÑÑÑÐ°. ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐŽÐ»Ñ ÐºÐŸÐŽÐžÑÐŸÐ²ÐºÐž Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐ°ÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÑÐŸÑÐžÐ»Ñ ISO 8601 (Ð€ÐŸÑÐŒÐ°ÑÑ ÐŽÐ°ÑÑ Ðž Ð²ÑÐµÐŒÐµÐœÐž W3C), ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ ÑÐŸÑÐŒÐ°Ñ ÐÐÐÐ-ÐÐ-ÐÐ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Fecha</Attribute>
+            <Attribute name="definition">Una fecha en la que el recurso se puso a disposiciÃ³n del usuario en su forma actual. Esta fecha no ha de confundirse con la que pertenece al elemento Coverage, que serÃ­a asociada con el recurso sÃ³lo en la medida en que el contenido intelectual estÃ¡ de algÃºn modo relacionado con esa fecha. Recomendamos la utiliazaciÃ³n de uno de los formatos definidos en el documento &quot;Date and Time Formats&quot;, http://www.w3.org/TR/NOTE-datetime basado en la norma ISO 8601 que incluye, entre otras, fechas en el formato AAAA y AAAA-MM-DD. De esta forma la fecha 1994-11-05 corresponderÃ­a al 5 de Noviembre de 1994.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØªØ§Ø±Ø®</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙÙ
+ØªØ¹ÙÙ ØšÙÙØª ÙØŽØŠ Ø£Ù ØªÙÙØ± Ø§ÙÙ
+ØµØ¯Ø± . Ù ÙØ¬Øš Ø£Ù ÙØ§ ÙØ®ÙØ· ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙØ¹Ø§ØŠØ¯ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØºØ·ÙØ© , Ù Ø§ÙØ° Ù ÙØ¬Øš Ø£Ù    ÙÙØ­Ù ØšØ§ÙÙ
+ØµØ¯Ø± ÙÙØ· Ù
+Ù ÙØ§Ø­ÙØ© Ø§ÙÙ
+Ø­ØªÙÙØ§ÙÙÙØ±Ù ÙÙÙÙÙ ÙÙØ¹Ø§ Ù
+Ø§ ÙÙ ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ®. Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø³Ø«Ø­Ø³ÙØ© ÙÙØ§Ø­Ø³Ù Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ    Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙ ÙÙ Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø¹ØªÙ
+Ø¯Ø© ÙÙ ØªØŽÙÙÙ (ISO 8601) ØµÙØºØ© ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601) Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© http://www.w3.org/TR/NOTE-datetime, W3C. ÙÙÙ    ÙØ°Ø§ Ø§ÙÙØ³Ù , Ù
+Ø«ÙØ§, ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ 1994-11-04 ÙÙØ§ØžØ± ÙÙÙ
+ 5 ÙÙÙÙ
+ØšØ± 1994</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ¥æ</Attribute>
+            <Attribute name="definition">äžèµæºçåœåšæäž­çäžäžªäºä»¶çžå
+³çæ¶éŽã</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæ¥æåºäžèµæºçåå»ºæåºçæ¥æçžå
+³ãå»ºè®®éçšçæ¥ææ ŒåŒåºç¬ŠåISO 8601 [W3CDTF]è§    èïŒå¹¶äœ¿çšYYYY-MM-DDçæ ŒåŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Datum</Attribute>
+            <Attribute name="definition">Datum spojenÃ© s urÄitou udÃ¡lostÃ­ bÄhem existence zdroje.</Attribute>
+            <Attribute name="comment">Datum bude typicky spojeno s vytvoÅenÃ­m nebo zpÅÃ­stupnÄnÃ­m zdroje. DoporuÄenÃœ postup pro kÃ³dovÃ¡nÃ­ hodnoty data je definovÃ¡n v ISO 8601 [W3CDTF]: formÃ¡t RRRR-MM-DD.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Datum</Attribute>
+            <Attribute name="definition">De datum van de totstandkoming of beschikbaarstelling van een Internetbron. Deze datum moet niet verward worden met de datum in &quot;Coverage&quot;. De datum in &quot;Coverage&quot; geeft de tijdsperiode van de inhoud van de Internetbron aan. Voorgeschreven wordt de ISO 8601 standaard voor datum, zoals beschreven in W3C Technical Note http://www.w3.org/TR/NOTE-datetime. Hierin wordt het formaat YYYY-MM-DD beschreven. Het formaat YYYY-MM-DD wordt de voorgeschreven standaard. 3 September 1998 wordt in dit geval 1998-09-03.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Data</Attribute>
+            <Attribute name="definition">Un data associate a un evento in le cyclo vital del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le data es associate al creation o disponibilization del ressource. Le practica recommendate pro codificar le valor del datas es definite in un profilo de ISO 8601 [W3CDTF] e seque le formato AAAA-MM-DD.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€€à€Ÿà€°à¥à€,  à€Šà€¿à€šà€Ÿà€à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€€à€¯à€Ÿà€° à€à¥à€²à¥à€¯à€Ÿà€à¥ à€€à€Ÿà€°à¥à€</Attribute>
+            <Attribute name="comment">à€€à€Ÿà€°à¥à€  à€²à€¿à€¹à¥à€£à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª à€ à€µà€°à¥à€·. à€®à€¹à€¿à€šà€Ÿ. à€Šà€¿à€µà€ž à¥§à¥¯à¥«à¥Š. à¥Šà¥ª. à¥Šà¥ª</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="created">
+            <Attribute language="en" name="identifier">Created</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Date of creation of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="modified">
+            <Attribute language="en" name="identifier">Modified</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Date on which the resource was changed.</Attribute>
+            </Language>
+        </Element>
+        <Element name="valid">
+            <Attribute language="en" name="identifier">Valid</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Date (often a range) of validity of a resource.</Attribute>
+                <Attribute name="comment">Typically, a date the resource becomes valid or ceases to be valid, or the date range for which the resource is valid.</Attribute>
+            </Language>
+        </Element>
+        <Element name="issued">
+            <Attribute language="en" name="identifier">Issued</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Date of formal issuance (e.g. publication) of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="available">
+            <Attribute language="en" name="identifier">Available</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Date (often a range) that the resource will become or did become available.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Type" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Resource Type</Attribute>
+            <Attribute name="definition">The nature or genre of the content of the resource.</Attribute>
+            <Attribute name="comment">Type includes terms describing general categories, functions, genres, or aggregation levels for content. Recommended best practice is to select a value from a controlled vocabulary (for example, the working draft list of Dublin Core Types). To describe the physical or digital manifestation of the resource, use the FORMAT element.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Tumomo</Attribute>
+            <Attribute name="definition">Te ahua, te tu-ahuatanga ranei o nga kaupapa o te rauemi.</Attribute>
+            <Attribute name="comment">Kei roto i te Tumomo nga kupu whakaatu i nga wehewehenga whanui, nga aheinga, nga tu-ahuatanga, te whakaritenga ranei o nga taumata o te/ nga kaupapa. Ko te huarahi tino pai kua tohua, ko te tipako i tetahi uara mai i nga kupu kua whakaritea (hei tauira, ko te tauira rarangi e whakamahia ana o nga Dublin Core Types [DCT1]. Hei whakaatu i te whakatinanatanga o te rauemi, whakamahia ko te huanga WHAKATAKOTORANGA (FORMAT).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Type</Attribute>
+            <Attribute name="definition">La nature ou le genre du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Type inclut des termes dÃ©crivant des catÃ©gories, fonctions ou genres gÃ©nÃ©raux pour le contenu, ou des niveaux d&apos;agrÃ©gation. Il est recommandÃ© de choisir la valeur du type dans une liste de vocabulaire contrÃŽlÃ© (par exemple, la liste provisoire de Types du Dublin Core[DCT1]). Pour dÃ©crire la matÃ©rialisation physique ou digitale de la ressource, il faut utiliser l&apos;Ã©lÃ©ment Format.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¢ÐžÐ¿ ÑÐµÑÑÑÑÐ°</Attribute>
+            <Attribute name="definition">ÐÑÐžÑÐŸÐŽÐ° ÐžÐ»Ðž Ð¶Ð°ÐœÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">Ð¢ÐžÐ¿ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐµÑÐŒÐžÐœÑ, ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑÐžÐµ ÐŸÐ±ÑÐžÐµ ÐºÐ°ÑÐµÐ³ÐŸÑÐžÐž, ÑÑÐœÐºÑÐžÐž, Ð¶Ð°ÐœÑÑ ÐžÐ»Ðž ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÐœÑÐµ ÑÑÐŸÐ²ÐœÐž ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÐµÐºÑÐ° ÑÐ¿ÐžÑÐºÐ° Ð¢ÐžÐ¿ÐŸÐ² Dublin Core DCT1). ÐÐ»Ñ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ ÑÐžÐ·ÐžÑÐµÑ-ÐºÐŸÐ³ÐŸ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐµÑÑÑÑÐ° ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÑÐ»ÐµÐŒÐµÐœÑ Ð€ÐŸÑÐŒÐ°Ñ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Tipo del Recurso</Attribute>
+            <Attribute name="definition">La categorÃ­a del recurso, por ejemplo pÃ¡gina personal, romance, poema, minuta, diccionario. Para asegurar la interoperabilidad, Type deberÃ­a ser seleccionado de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo. En http://sunsite.berkeley.edu/Metadata/types.html se puede consultar el estado actual de la discusiÃ³n en torno a este tema.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">ÙÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">ÙÙ ÙØŠØ© Ø§ÙÙ
+ØµØ¯Ø± , Ù
+Ø«ÙØ§ Ø§ÙØµÙØ­Ø© Ø§ÙØ§Ù
+, Ø±ÙØ§ÙØ© , ÙØ·Ø¹Ø© ØŽØ¹Ø±ÙØ© , ÙØ±ÙØ© Ø¹Ù
+Ù , ØªÙØ±ÙØ± ÙÙÙ , Ù
+ÙØ§Ù , ÙØ§Ù
+ÙØ³ . Ù Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù    ÙØ¬Øš Ø£Ù ÙØ®ØªØ§Ø± Ø§ÙÙÙØ¹ Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© ÙØ§ÙØªÙ ÙÙÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºç±»å</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çç¹åŸæç±»åã</Attribute>
+            <Attribute name="comment">èµæºç±»åå
+æ¬æè¿°èµæºå
+å®¹çäžè¬èçŽïŒåèœïŒç§å±ïŒæèç±»å±æ¬¡çæ¯è¯­ãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒäŸåŠDCMIç±»åè¯æ±è¡š[DCMITYPE]ïŒãèŠæè¿°èµæºçç©çææ°å­åè¡šç°åœ¢åŒïŒè¯·äœ¿çšâæ ŒåŒïŒFORMATïŒâå
+çŽ ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Typ zdroje</Attribute>
+            <Attribute name="definition">Povaha nebo druh obsahu zdroje.</Attribute>
+            <Attribute name="comment">Typ obsahuje termÃ­ny popisujÃ­cÃ­ obecnÃ© kategorie, funkce, druhy nebo agregaÄnÃ­ ÃºrovnÄ obsahu. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. pracovnÃ­ nÃ¡vrh seznamu TypÅ¯ Dublin Core [DCT1]). K popisu fyzickÃ© nebo digitÃ¡lnÃ­ reprezentace zdroje uÅŸijte prvek FormÃ¡t.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bestands type</Attribute>
+            <Attribute name="definition">Het soort Internetbron, bijvoorbeeld home page, gedicht, working paper, technical report, essay, woordenboek etc. De DC working groups zijn momenteel bezig met twee voorstellen; een minimalist draft: http://sunsite.berkeley.edu/Metadata/minimalist.html en een structuralist draft: http://sunsite.berkely.edu/Metadata/structuralist.html. Het wordt sterk aanbevolen om &quot;Type&quot; van deze lijsten te selecteren.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Typo del ressource</Attribute>
+            <Attribute name="definition">Le natura o genere del contento del ressource.</Attribute>
+            <Attribute name="comment">Le typo del ressource include terminos que describe categorias general, functiones, generes, o nivellos de aggregation pro le contento. Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le lista de schizzos de travalio de typos de Dublin Core [DCT1]). Pro describer le manifestation physic o digital del ressource, utiliza le elemento Formato.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€°</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿà€ªà¥à€°à€à€Ÿà€°</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£, à€µà€°à¥à€à€µà€Ÿà€°à¥à€®à€§à¥à€¯à¥ à€à€à€Ÿà€Šà¥à€¯à€Ÿ à€šà¥à€à€Šà¥à€à€Ÿ à€®à¥à€à¥à€¯ à€à€à€ªà¥à€°à€à€Ÿà€°</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="category">
+            <Attribute language="en" name="identifier">Category</Attribute>
+            <Attribute language="en" name="type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The generic type of the resource being described.</Attribute>
+                <Attribute name="comment">The value for this qualifier must be one of service, document, or agency.</Attribute>
+            </Language>
+            <Attribute language="en" name="source">AGLS</Attribute>
+            <Attribute language="en" name="service">used when describing a service directly, not a document about a service.</Attribute>
+            <Attribute language="en" name="document">used for all resources which are not services or agencies, even if they are no traditional &amp;quot;documents&amp;quot; - for example, a sculpture;</Attribute>
+            <Attribute language="en" name="agency">used to describe an organisation</Attribute>
+        </Element>
+        <Element name="aggregationLevel">
+            <Attribute language="en" name="identifier">Aggregation Level</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The level of aggregation of the resource being described.</Attribute>
+                <Attribute name="comment">There are only two values possible for this qualifier, either item or collection.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Format" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">The physical or digital manifestation of the resource.</Attribute>
+            <Attribute name="comment">Typically, Format may include the media-type or dimensions of the resource. Format may be used to determine the software, hardware or other equipment needed to display or operate the resource. Examples of dimensions include size and duration. Recommended best practice is to select a value from a controlled vocabulary (for example, the list of Internet Media Types [MIME] defining computer media formats).</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whakatakotoranga</Attribute>
+            <Attribute name="definition">Te whakatinanatanga, te whakaputanga a-mati o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka taea te Whakatakotoranga te whakamahi hei whakarite i te rawa-ngohe (software), te rawa-maro (hardware), ma etahi taputapu ranei me matua whiwhi, hei whakahaere i te rauemi. Ko te rahi me te roanga etahi tauira hanga. Ko te tino huarahi kua tohua ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, te rarangi o nga Tumomo Papaho Ipurangi (Internet Media Types [MIME]) e whakaatu ana i nga whakatakotoranga papaho rorohiko.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">La matÃ©rialisation physique ou digitale de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, Format peut inclure le media ou les dimensions de la ressource. Format peut Ãªtre utilisÃ© pour prÃ©ciser le logiciel, le matÃ©riel ou autre Ã©quipement nÃ©cessaire pour afficher ou faire fonctionner la ressource. Exemples de dimensions incluent la taille et la durÃ©e. Il est recommandÃ© de choisir la valeur du format dans une liste de vocabulaire contrÃŽlÃ©(par exemple, la liste des types de media dÃ©finis sur Internet [MIME]).</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð€ÐŸÑÐŒÐ°Ñ</Attribute>
+            <Attribute name="definition">Ð€ÐžÐ·ÐžÑÐµÑÐºÐŸÐµ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐžÐ¿ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž (ÐŒÐµÐŽÐžÐ°-ÑÐžÐ¿) ÐžÐ»Ðž Ð²ÐµÐ»ÐžÑÐžÐœÑ ÑÐµÑÑÑÑÐ°. Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐŽÐ»Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð°Ð¿Ð¿Ð°ÑÐ°ÑÐœÐŸÐ³ÐŸ, Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐœÐŸÐ³ÐŸ ÐŸÐ±ÐµÑÐ¿ÐµÑÐµÐœÐžÑ ÐžÐ»Ðž ÐŽÑÑÐ³ÐŸÐ³ÐŸ ÐŸÐ±ÐŸÑÑÐŽÐŸÐ²Ð°ÐœÐžÑ, ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÐ³ÐŸ ÐŽÐ»Ñ Ð²ÐŸÑÐ¿ÑÐŸÐžÐ·Ð²ÐµÐŽÐµÐœÐžÑ ÐžÐ»Ðž ÑÐ°Ð±ÐŸÑÑ Ñ ÑÐµÑÑÑÑÐŸÐŒ. ÐÑÐžÐŒÐµÑÑ Ð²ÐµÐ»ÐžÑÐžÐœÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐ°Ð·ÐŒÐµÑ Ðž Ð¿ÑÐŸÐŽÐŸÐ»Ð¶ÐžÑÐµÐ»ÑÐœÐŸÑÑÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ¿ÐžÑÐºÐ° ÐÐµÐŽÐžÐ°-ÑÐžÐ¿ÐŸÐ² ÐÐœÑÐµÑÐœÐµÑ MIME, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑÐµÐ³ÐŸ ÐºÐŸÐŒÐ¿ÑÑÑÐµÑÐœÑÐµ ÑÐŸÑÐŒÐ°ÑÑ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž).</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Formato</Attribute>
+            <Attribute name="definition">El formato de datos de un recurso, usado para identificar el software y posiblemente, el hardware que se necesitarÃ­a para mostrar el recurso. Para asegurar la interoperabilidad, los valores de Format deberÃ­an ser seleccionados de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØµÙØºØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ù ØµÙØºØ© Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø± , Ù ØªØ³ØªØ®Ø¯Ù
+ ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙØšØ±Ù
+Ø¬ÙØ§Øª ÙØ±ØšÙ
+Ø§ Ø§ÙØµÙØš ÙÙØ­Ø§Ø³ÙØš Ù Ø§ÙØªÙ ÙÙ
+ÙÙ Ø£Ù ÙØ­ØªØ§Ø¬    ÙÙØ§ ÙØ¹Ø±Ø¶ Ø£Ù ØªØŽØºÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù, ÙØ¬Øš Ø£Ù ØªØ®ØªØ§Ø± Ø§ÙØµÙØºØ© Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§    Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ ŒåŒ</Attribute>
+            <Attribute name="definition">èµæºçç©çææ°å­è¡šç°åœ¢åŒã</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæ ŒåŒå¯èœå
+æ¬èµæºçåªäœç±»åæèµæºçå€§å°ïŒæ ŒåŒå
+çŽ å¯ä»¥çšæ¥å³å®å±ç€ºææäœèµæºæé    çèœ¯ç¡¬ä»¶æå
+¶ä»çžåºè®Ÿå€ïŒäŸåŠå€§å°å
+æ¬èµæºæå çå­åšç©ºéŽåæç»­æ¶éŽãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒïŒäŸåŠInternetåªäœç±»å[MIME]å®ä¹è®¡ç®æºåªäœæ ŒåŒïŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">FormÃ¡t</Attribute>
+            <Attribute name="definition">FyzickÃ¡ nebo digitÃ¡lnÃ­ reprezentace zdroje.</Attribute>
+            <Attribute name="comment">Typicky mÅ¯ÅŸe FormÃ¡t obsahovat typ mÃ©dia nebo rozmÄry zdroje. FormÃ¡t lze pouÅŸÃ­vat k urÄenÃ­ softwaru, hardwaru nebo dalÅ¡Ã­ho zaÅÃ­zenÃ­ potÅebnÃ©ho k zobrazenÃ­ nebo k manipulaci se zdrojem. PÅÃ­klady rozmÄrÅ¯ zahrnujÃ­ velikost a dobu trvÃ¡nÃ­. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. seznam typÅ¯ internetovÃœch mÃ©diÃ­ [MIME] definujÃ­cÃ­ formÃ¡ty poÄÃ­taÄovÃœch mÃ©diÃ­).</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">Het data formaat van het Internetbestand. Dit gegeven kan gebruikt worden om systeemvereisten (hard- en software) te identificeren die nodig zijn voor raadpleging of bediening van de Internetbron. &quot;Format&quot; is nog in ontwikkeling bij de DC workshop. Uit compatibiliteitsoverwegingen wordt aanbevolen formaatgegevens te selecteren uit een gecontroleerde lijst. Formaten kunnen van de Internet Media Types (MIME types) lijst gekozen worden: http://www.isi.edu/in-notes/iana/assignments/media-types/media-types</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Formato</Attribute>
+            <Attribute name="definition">Le manifestation physic o digital del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le formato pote includer le typo de medio o dimensiones del ressource. Le formato pote esser utilizate pro determinar le software o le equipamento necessari pro exhibir o operar le ressource. Exemplos de dimensiones include grandessa e duration. Le practica recommendate es selectionar un valor de un vocabulario controlate (per exemplo, le lista de typos de medios de Internet [MIME] que defini formatos de medios).</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€°à€à€šà€Ÿ</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€°à€à€šà€Ÿ, à€šà€®à¥à€šà€Ÿà€¬à¥à€Š à€°à€à€šà€Ÿ</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª, à€°à€à€šà€Ÿà€à€Šà€Ÿ. à€€à¥à€¯à€Ÿà€à¥ à€°à¥à€ª à€à€žà¥ à€ªà¥à€žà¥à€€à€à€°à¥à€ª, à€žà€à€à€£à€à¥à€¯ à€°à¥à€ª, à€µà€¿à€Šà¥à€¯à¥à€€ à€ªà€°à€®à€Ÿà€£à¥à€à€à¥à€¯à€Ÿ à€žà€¹à€Ÿà€¯à¥à€¯à€Ÿà€šà¥ à€žà€à€à€£à€à¥à€à¥à€€ à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="extent">
+            <Attribute language="en" name="identifier">Extent</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The size or duration of the resource.</Attribute>
+                <Attribute name="comment">The extent qualifier allows the description of the physical dimensions, file size or duration of the resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="medium">
+            <Attribute language="en" name="identifier">Medium</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The material or physical carrier of the resource.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Availability">
+        <Attribute language="en" name="identifier">Availability</Attribute>
+        <Attribute language="en" name="obligation">Conditional - mandatory when adding an agency, service, or off-line document; optional when adding an on-line document.</Attribute>
+        <Attribute language="en" name="definition">How the resource can be obtained or contact information for obtaining the resource.</Attribute>
+        <Attribute language="en" name="comment">The Availability element is primarily used for non-electronic resources to provide information on how to obtain physical access to the resource. The Availability element should be repeated to show multiple access points - for example, a service where someone could pay a bill at two separate organisations. Generally, full details of how any service can be obtained, or contact information, should be provided. If a structured value is to be given (including for example name and address information) it is recommended that the NZGLS Agent Encoding scheme be used. Alternatively, a link to a directory, authority file or similar set of identity information may be provided.</Attribute>
+    </Element>
+    <Element name="Identifier" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Resource Identifier</Attribute>
+            <Attribute name="definition">An unambiguous reference to the resource within a given context.</Attribute>
+            <Attribute name="comment">Recommended best practice is to identify the resource by means of a string or number conforming to a formal identification system. Example formal identification systems include the Uniform Resource Identifier (URI) (including the Uniform Resource Locator (URL)), the Digital Object Identifier (DOI) and the International Standard Book Number (ISBN).</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Taututanga</Attribute>
+            <Attribute name="definition">He tohutoro e hangai ana ki te rauemi i roto i tetahi horopaki.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tautu i te rauemi ma tetahi aho (string), tau ranei, e hangai ana ki tetahi punaha taututanga okawa. Ko etahi punaha taututanga okawa ko te Taututanga Rauemi Orite (Uniform Resource Identifier - URI), te Taututanga Rauemi Tohu-wahi (Uniform Resource Locator - URL), Te Mea A-mati (Digital Object Identifier - DOI) me te Tau Pukapuka Aro Whanui o te Ao (International Standard Book Number - ISBN).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Identifiant</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence non ambiguÃ« Ã  la ressource dans un contexte donnÃ©</Attribute>
+            <Attribute name="comment">Il est recommandÃ© d&apos;identifier la ressource par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un sytÃšme formel d&apos;identification. Exemples de systemes formels d&apos;identification incluent le &quot;Uniform Resource Identifier&quot; (URI) (qui inclut le &quot;Uniform Resource Locator&quot; (URL)), le &quot;Digital Object Identifier&quot;(DOI) et le &quot;International Standard Book Number&quot;(ISBN).</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÑÐµÑÑÑÑÐ°</Attribute>
+            <Attribute name="definition">ÐÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÐ°Ñ ÑÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐºÐŸÐœÑÐµÐºÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ. ÐÑÐžÐŒÐµÑÑ ÑÐŸÑÐŒÐ°Ð»ÑÐœÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÑÐžÑÑÐµÐŒ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° (URI), (Ð²ÐºÐ»ÑÑÐ°Ñ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÑÐ¹ ÐÐŸÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° URL), ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐŠÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐÐ±ÑÐµÐºÑÐ° (DOI) Ðž ÐÐµÐ¶ÐŽÑÐœÐ°ÑÐŸÐŽÐœÑÐ¹ Ð¡ÑÐ°ÐœÐŽÐ°ÑÑÐœÑÐ¹ ÐÐœÐžÐ¶ÐœÑÐ¹ ÐÐŸÐŒÐµÑ (ISBN).</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Identificador del Recurso</Attribute>
+            <Attribute name="definition">Secuencia de caracteres usados para identificar unÃ­vocamente un recurso. Ejemplos para recursos en lÃ­nea pueden ser URLs y URNs (cuando estÃ©n implementados). Para otros recursos pueden ser usados otros formatos de identificadores, como por ejemplo ISBN (&quot;International Standard Book Number&quot; - NÃºmero Internacional Normalizado para Libros)</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ù
+Ø¹Ø±Ù Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">ÙÙ Ù
+Ø¬Ù
+ÙØ¹Ø© Ø­Ø±ÙÙØ© Ø£Ù Ø±ÙÙ
+ ÙØ³ØªØ®Ø¯Ù
+ ÙØªØ¹Ø±ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§Ø­Ø¯ÙØ§ : Ø§ÙØ§Ù
+Ø«ÙØ© ÙØ°Ø§ÙÙ ÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙØŽØšÙÙØ©ØªØ¶Ù
+ ) URNs, URLs Ø¹ÙØ¯ ØšÙØ§ØŠÙØ§ (    ÙØ°ÙÙ ÙÙ
+ÙÙ Ø§Ø¹ØªØšØ§Ø± Ø§ÙÙ
+Ø¹Ø±ÙØ§Øª Ø§ÙØ¹Ø§ÙÙ
+ÙØ© Ø§ÙØ§Ø­Ø§Ø¯ÙØ© Ù
+Ø«Ù Ø§ÙØ±ÙÙ
+ Ø§ÙØ¯ÙÙÙ ÙØªØµÙÙÙ Ø§ÙÙØ§ØªØš (ISBN) Ù
+Ø±ØŽØ­Ø© ÙÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºæ è¯ç¬Š</Attribute>
+            <Attribute name="definition">åšç¹å®çèåŽå
+ç»äºèµæºçäžäžªæç¡®çæ è¯ã</Attribute>
+            <Attribute name="comment">å»ºè®®å¯¹èµæºçæ è¯éçšç¬Šåæäžæ­£åŒæ è¯äœç³»çå­ç¬Šäž²åæ°å­ç»åãäŸåŠæ­£åŒçæ è¯äœç³»å
+æ¬ç»äžèµ    æºæ è¯ç¬Š(URI) (å
+æ¬åäžèµæºå®äœç¬ŠURL)ãæ°å­å¯¹è±¡æ è¯ç¬Š(DOI)ååœé
+æ åä¹Šå·(ISBN)ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">IdentifikÃ¡tor zdroje</Attribute>
+            <Attribute name="definition">JednoznaÄnÃœ odkaz na zdroj v rÃ¡mci danÃ©ho kontextu.</Attribute>
+            <Attribute name="comment">DoporuÄuje se identifikovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯. PÅÃ­klady formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯ zahrnujÃ­ Uniform Resource Identifier (URI) (vÄetnÄ Uniform Resource Locator (URL)), Digital Object Identifier (DOI) a MezinÃ¡rodnÃ­ standardnÃ­ ÄÃ­slo knihy (ISBN).</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bestandsidentificatie</Attribute>
+            <Attribute name="definition">Een unieke string of nummer ter identificatie van de Internetbron. Voorbeelden zijn URL&apos;s of URN&apos;s Andere voorbeelden zijn International Standard Book Number (ISBN) of International Standard Serial Number (ISSN)</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Identificator del ressource</Attribute>
+            <Attribute name="definition">Un referentia inambigue al ressource intra un date contexto.</Attribute>
+            <Attribute name="comment">Se recommenda identificar le ressource per medio de un catena de characteres o numero conforme a un systema formal de identification. Exemplos de systemas formal de identification include le Uniform Resource Identifier (URI) (includente le Uniform Resource Locator (URL)), le Digital Object Identifier (DOI) e le International Standard Book Number (ISBN).</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€à€³à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿ à€à€ à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ, à€µà€¿à€¶à€¿à€·à¥à€  à€žà¥à€¥à€³,  à€ªà¥à€žà¥à€€à€à€Ÿà€à€Ÿ à€à€à€€à€°à€°à€Ÿà€·à¥à€à¥à€°à¥à€¯  à€®à€Ÿà€šà€ à€à¥à€°à€®à€Ÿà€à€,  à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€à€³à€ à€ªà€à€µà€¿à€£à€Ÿà€°à€Ÿ à€à€à€à€°à€šà¥à€ à€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Source" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Source</Attribute>
+            <Attribute name="definition">A Reference to a resource from which the present resource is derived.</Attribute>
+            <Attribute name="comment">The present resource may be derived from the Source resource in whole or in part.  Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Matapuna</Attribute>
+            <Attribute name="definition">He tohutoro ki tetahi rauemi koira te matapuna o te rauemi e tirohia ana.</Attribute>
+            <Attribute name="comment">Tera pea i puta te katoa, tetahi wahanga ranei o te rauemi, e tirohia ana, mai i te Matapuna rauemi. Ko te tino huarahi kua tohua ko te tohutoro ki te rauemi ma te aho, ma te tau ranei e hangai ana ki tetahi punaha taututanga okawa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Source</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence Ã  une ressource Ã  partir de laquelle la ressource actuelle a Ã©tÃ© dÃ©rivÃ©e.</Attribute>
+            <Attribute name="comment">La ressource actuelle peut avoir Ã©tÃ© dÃ©rivÃ©e d&apos;une autre ressource source, en totalitÃ© ou en partie. Il est recommandÃ© de refÃ©rencer cette source par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un systÃšme formel d&apos;identification.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÑÐŸÑÐœÐžÐº</Attribute>
+            <Attribute name="definition">Ð¡ÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ, ÐžÐ· ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐœÐ°ÑÑÐŸÑÑÐžÐ¹.</Attribute>
+            <Attribute name="comment">ÐÐ°ÑÑÐŸÑÑÐžÐ¹ ÑÐµÑÑÑÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐžÐ· ÐÑÑÐŸÑÐœÐžÐºÐ° ÑÐµÐ»ÐžÐºÐŸÐŒ ÐžÐ»Ðž ÑÐ°ÑÑÐžÑÐœÐŸ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Fuente</Attribute>
+            <Attribute name="definition">Secuencia de caracteres utilizado para identificar unÃ­vocamente un trabajo a partir del cual proviene el recurso actual. Por ejemplo, es posible usar Source con la fecha de 1603 como descripciÃ³n de una pelÃ­cula basada en una obra de Shakespeare, pero es preferible, en ese caso, usar Relation &quot;IsBasedOn&quot; con una referencia a un recurso distinto cuya descripciÃ³n contenga el elemento Date con valor 1603.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø§Øª Ø§ÙØ®Ø§ØµØ© ØšÙ
+ØµØ¯Ø± Ø«Ø§ÙÙ Ù Ø§ÙØ°Ù Ø§Ø³ØªÙ
+Ø¯ Ù
+ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ. Ù
+Ø¹ Ø£ÙÙ Ù
+Ù Ø§ÙÙ
+ÙØµÙ ØšÙ Ø£Ù ØªÙØ¯Ù
+ Ø§ÙØ¹ÙØ§Ù
+Ù    Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ ÙÙØ· , ÙÙÙ
+ÙÙ Ø£Ù ÙØ­ÙÙ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ®, Ø§ÙÙ
+ÙØŽØŠ , Ø§ÙØµÙØºØ© , Ø§ÙÙ
+Ø¹Ø±Ù , Ø§Ù Ø§ÙØ© Ù
+ÙØ© - Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø§Ø®Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ø¹ÙØ¯Ù
+Ø§ ØªØ¹ØªØšØ± Ù
+ÙÙ
+Ø© ÙØ§ÙØªØŽØ§Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ , Ù ÙØ°Ø§ÙÙ ÙÙØµÙ Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙÙÙ Ø§Ø­Ø³Ù     Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+ Ø¹Ø§Ù
+Ù Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ¥æº</Attribute>
+            <Attribute name="definition">ç°æèµæºæ¥æºçåç
+§ã</Attribute>
+            <Attribute name="comment">åœåèµæºå¯èœéšåæå
+šéšæºèªè¯¥å
+çŽ ææ è¯çèµæºïŒå»ºè®®å¯¹è¿äžèµæºçæ è¯éçšäžäžªç¬Šåæ­£åŒæ è¯ç³»    ç»çå­äž²åæ°å­ç»å ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Zdroj</Attribute>
+            <Attribute name="definition">Odkaz na zdroj, z nÄhoÅŸ je popisovanÃœ zdroj odvozen.</Attribute>
+            <Attribute name="comment">PopisovanÃœ zdroj mÅ¯ÅŸe bÃœt zcela nebo zÄÃ¡sti odvozen ze zdroje uvedenÃ©ho v prvku Zdroj. DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bron</Attribute>
+            <Attribute name="definition">Informatie over een tweede bron waar de huidige Internetbron is van afgeleid. Bij &quot;Source&quot; mag een datum, auteur of maker, formaat, identificatie of andere metadata voor de tweede bron worden ingevuld. Voorbeeld: het is mogelijk om een &quot;Source&quot; datum 1603 in een omschrijving van een 1996 film bewerking van een Shakespeare stuk te gebruiken. &quot;Source&quot; is niet bruikbaar wanneer de huidige bron de orginele is.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Fonte</Attribute>
+            <Attribute name="definition">Un referentia a un ressource del qual se deriva le ressource presente.</Attribute>
+            <Attribute name="comment">Le ressource presente pote derivar se del ressource-fonte in toto o in parte. Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o un numero conforme a un systema formal de identification.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€à€à€®</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿ à€à€à€® à€žà€Ÿà€à€à€£à€Ÿà€°à€Ÿ à€à€à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€ªà¥à€žà¥à€€à€, à€®à€Ÿà€žà€¿à€,  à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿà€¯à€Ÿà€à€à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Language" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Language</Attribute>
+            <Attribute name="definition">A language of the intellectual content of the resource.</Attribute>
+            <Attribute language="en" name="comment">Recommended best practice for the values of the Language element is defined by RFC 1766 which includes a two-letter Language Code (taken from the ISO 639 standard, followed optionally, by a two-letter Country Code (taken from the ISO 3166 standard). For example, &apos;en&apos; for English, &apos;fr&apos; for French, or &apos;en-uk&apos; for English used in the United Kingdom.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Reo</Attribute>
+            <Attribute name="definition">Te reo o te whakakaupapatanga o nga matauranga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua mo nga uara o te huanga Reo kua whakaritea ki te RFC 1766 [RFC1766] kei roto te Tohu Reo kupu-rua (two-letter Language Code) (no te aro whanui ISO 639 [ISO639], ka taea e tetahi tohu kupu-rua te whai atu (no te aro whanui ISO 3166) [ISO366]). Hei tauira, te &apos;en&apos; mo te English, te &apos;fr&apos; mo te French, &apos;te en-uk&apos; mo te English e whakamahia ana i Ingarangi (UK).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Langue</Attribute>
+            <Attribute name="definition">La langue du contenu intellectuel de la ressource.</Attribute>
+            <Attribute name="comment">Il est recommandÃ© d&apos;utiliser comme valeur de l&apos;Ã©lÃ©ment Langue celles definies par la RFC 1766 [RFC1766] qui comprend un code de langage Ã  deux caractÃšres(venant du standard ISO 639 [ISO639]), Ã©ventuellement suivi d&apos;un code Ã  deux lettres pour le pays (venant du standard ISO 3166 [ISO3166] ou en franÃ§ais [ISO3166]). Par exemple, &apos;en&apos; pour l&apos;anglais, &apos;fr&apos; pour le franÃ§ais, ou &apos;en-uk&apos; pour l&apos;anglais utilisÃ© au Royaume-Uni.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¯Ð·ÑÐº</Attribute>
+            <Attribute name="definition">Ð¯Ð·ÑÐº ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ð¯Ð·ÑÐº, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÐŒÐŸÐµ RFC 1766 (Ð¢ÑÐ³Ðž ÐŽÐ»Ñ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐ·ÑÐºÐ°), Ð²ÐºÐ»ÑÑÐ°ÑÑÐžÐŒ ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÐ·ÑÐºÐ° (Ð²Ð·ÑÑÑÐ¹ ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 639 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÐ·ÑÐºÐŸÐ²)), Ð·Ð° ÐºÐŸÑÐŸÑÑÐŒ ÑÐ»ÐµÐŽÑÐµÑ (Ð¿ÑÐž ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÑÑÐ°ÐœÑ (Ð²Ð·ÑÑÑÐŒÐž ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 3166 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÑÑÐ°Ðœ)). ÐÐ°Ð¿ÑÐžÐŒÐµÑ, &quot;en&quot; - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, &quot;fr&quot; - ÐŽÐ»Ñ ÑÑÐ°ÐœÑÑÐ·ÑÐºÐŸÐ³ÐŸ, &quot;en-uk&quot; - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ Ð² ÐÐµÐ»ÐžÐºÐŸÐ±ÑÐžÑÐ°ÐœÐžÐž.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Lengua</Attribute>
+            <Attribute name="definition">Lengua/s del contenido intelectual del recurso. Practicamente el contenido de este campo deberÃ­a coincidir con los de la RFC 1766 (Tags para la identificaciÃ³n de lenguas, http://ds.internic.net/rfc/rfc1766.txt); por ejemplo: en, es, de, fi, ja y zh.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØºØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ù ÙÙ ÙØºØ© Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . ÙÙÙ
+Ø§ ÙØ§Ù  Ø¹Ù
+ÙÙØ§ , Ù
+Ø­ØªÙÙ ÙØžØ§ Ø§ÙØ­ÙÙ ÙØ¬Øš Ø£Ù ÙØªØ·Ø§ØšÙ Ù
+Ø¹ RFC 1766    ] Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ th, ja, fr, es, fi, de, en Ù zh ÙØ±Ù
+ÙØ² ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙÙØºØ© [</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">è¯­ç§</Attribute>
+            <Attribute name="definition">æè¿°èµæºç¥è¯å
+å®¹çè¯­ç§ã</Attribute>
+            <Attribute name="comment">å»ºè®®æ¬å
+çŽ çåŒéçšRFC 3066[RFC3066]ïŒè¯¥æ åäžISO 639 [ISO639]äžèµ·å®ä¹äºç±äž€äžªåäžäžªè±    æå­æ¯ç»æçäž»è¯­èšæ ç­Ÿåå­æ ç­ŸäŸåŠå
+æ¬âenâæâengâæ¥è¡šç€ºEnglish, &quot;akk&quot; æ¥è¡šç€º    &quot;en-GB&quot;è¡šç€ºè±åœè±è¯­ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Jazyk</Attribute>
+            <Attribute name="definition">Jazyk intelektuÃ¡lnÃ­ho obsahu zdroje.</Attribute>
+            <Attribute name="comment">DoporuÄenÃœ postup pro hodnoty prvku Jazyk je definovÃ¡n v RFC 1766 [RFC1766]: dvouznakovÃœ kÃ³d jazyka (pÅevzatÃœ z normy ISO 639 [ISO639]) volitelnÄ nÃ¡sledovanÃœ dvouznakovÃœm kÃ³dem zemÄ (pÅevzatÃœm z normy ISO 3166 [ISO3166]). NapÅÃ­klad &apos;en&apos; pro angliÄtinu, &apos;fr&apos; pro francouzÅ¡tinu, &apos;cs&apos; pro ÄeÅ¡tinu, nebo &apos;en-uk&apos; pro angliÄtinu uÅŸÃ­vanou ve SpojenÃ©m krÃ¡lovstvÃ­.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Taal</Attribute>
+            <Attribute name="definition">De taal waarin het document is beschreven. Voorgeschreven wordt de ISO639-1 norm, een twee karakter taalcode van de ISO 639(639-1 en 639-2) norm. Taalcodes kunnen van een lijst worden gekozen: http://www.ics.uci.edu/pub/ietf/http/related/iso639.txt, Eventueel kan ISO3166 (landencodes) gebruikt worden om aan te geven in welk gebied de taal wordt gesproken. Deze kunnen ook van een lijst worden gekozen:  http://www.ics.uci.edu/pub/ietf/http/related/iso3166.txt.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Lingua</Attribute>
+            <Attribute name="definition">Le lingua del contento intellectual del ressource.</Attribute>
+            <Attribute name="comment">Le practica recommendate pro le valores del elemento lingua es definite per RFC 1766 [RFC1766], que include un codice de lingua de duo litteras (prendite del standard ISO 639 [ISO639]), sequite optionalmente per un codice de pais de duo litteras (prendite del standard ISO 3166 [ISO3166]). Per exemplo, &apos;ia&apos; pro interlingua, &apos;fr&apos; pro francese, o &apos;en-uk&apos; pro le anglese del Regno Unite.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€­à€Ÿà€·à€Ÿ</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à€Ÿà€·à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€®à€°à€Ÿà€ à¥ - à€­à€Ÿà€°à€€à¥à€¯ à€­à€Ÿà€·à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Relation" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Relation</Attribute>
+            <Attribute name="definition">A reference to a related resource.</Attribute>
+            <Attribute name="comment">Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whai Panga</Attribute>
+            <Attribute name="definition">He tohutoro ki tetahi rauemi whai panga.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tohutoro ki te rauemi ma te aho, te tau ranei, e hangai ana ki te punaha taututanga okawa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Relation</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence Ã  une autre ressource qui a un rapport avec cette ressource.</Attribute>
+            <Attribute name="comment">Il est recommandÃ© de rÃ©fÃ©rencer cette ressource par une chaÃ®ne de caractÃšres ou un numÃ©ro conforme Ã  un systÃšme formel d&apos;identification.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐœÐŸÑÐµÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ï¿œÐµÐœÐœÑÐ¹ ÑÐµÑÑÑÑ.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">RelaciÃ³n</Attribute>
+            <Attribute name="definition">Un identificador de un segundo recurso y su relaciÃ³n con el recurso actual. Este elemento permite enlazar los recursos relacionados y las descripciones de los recursos. Por ejemplo: IsVersionOf - Incluye la ediciÃ³n de un trabajo, IsBasedOn - La traducciÃ³n de un trabajo, IsPartOf - Un capÃ­tulo de un libro, IsFormatOf - Un mecanismo de transformaciÃ³n de un conjunto de datos en una imagen. Para asegurar la interoperabilidad, las relaciones deberÃ­an ser seleccionadas de una lista de elementos que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ø§ Ù
+Ø¹Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ù Ø¹ÙØ§ÙØªÙ ØšØ§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ . ÙØ³Ù
+Ø­ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù ØšØªÙØ¶ÙØ­ Ø§ÙØ±ÙØ§ØšØ· ØšÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+ØªØ¹ÙÙØ©     ØšØšØ¹Ø¶ÙØ§ Ù ØªÙØµÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ Ø·ØšØ¹Ø© Ø§ÙØ¹Ù
+Ù ) ÙØ³Ø®Ø© Ù
+Ù ( ØªØ±Ø¬Ù
+Ø© Ø¹Ù
+Ù ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ ( ÙØµÙ Ù
+Ù ÙØªØ§Øš ) Ø¬Ø²Ø¡ Ù
+Ù (    Ù ØªØ­ÙÙÙÙ Ù
+ÙÙØ§ÙÙÙÙØ© ÙÙ
+Ø¬Ù
+ÙØ¹Ø© Ù
+Ø¹ÙÙÙ
+Ø§ØªÙØ© Ø§ÙÙ ØµÙØ±Ø© ) ØµÙØºØ© Ù
+Ù ( Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù , ÙØ¬Øš Ø§Ø®ØªÙØ§Ø± Ø§ÙØ¹ÙØ§ÙØ§Øª Ù
+Ù    Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">å
+³è</Attribute>
+            <Attribute name="definition">æ¶åççžå
+³èµæºã</Attribute>
+            <Attribute name="comment">èåœçžå
+³èµæºæ¶ïŒå»ºè®®äœ¿çšç¬Šåæ­£åŒæ è¯ç³»ç»çå­ç¬Šäž²æè
+æ°å­ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Vztah</Attribute>
+            <Attribute name="definition">Odkaz na pÅÃ­buznÃœ zdroj.</Attribute>
+            <Attribute name="comment">DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Relatie</Attribute>
+            <Attribute name="definition">Een identificatie van een tweede bron en de relatie met de huide Internetbron. Voorbeelden zijn een editie van een werk (IsVersionOf), een vertaling van een werk (IsBasedOn), een hoofdstuk van een boek (IsPartOf) en een mechanische transformatie van een dataset in een plaatje (IsFormatOf). De DC Working Groups zijn bezig met een lijst met de meest verwachte relaties. Aanbevolen wordt om de relaties van deze lijst te gebruiken: http://purl.oclc.org/metadata/dublin_core/wrelationdraft.html.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="definition">Relation</Attribute>
+            <Attribute name="definition">Un referentia a un ressource correlate.</Attribute>
+            <Attribute name="comment">Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o numero conforme a un systema de identification formal.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€šà€Ÿà€€à¥ , à€žà€à€¬à€à€§</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€¶à¥ à€žà€à€¬à€à€§ à€Šà€Ÿà€à€µà€£à€Ÿà€°à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.  à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ(URL)</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="isVersionOf">
+            <Attribute language="en" name="identifier">Is Version Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is a version, edition, or adaptation of the referenced resource. Changes in version imply substantive changes in content rather than differences in format.</Attribute>
+            </Language>
+        </Element>
+        <Element name="hasVersion">
+            <Attribute language="en" name="identifier">Has Version</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource has a version, edition, or adaptation, namely, the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isReplacedBy">
+            <Attribute language="en" name="identifier">Is Replaced By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is supplanted, displaced, or superseded by the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="replaces">
+            <Attribute language="en" name="identifier">Replaces</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource supplants, displaces, or supersedes the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isRequiredBy">
+            <Attribute language="en" name="identifier">Is Required By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is required by the referenced resource, either physically or logically.</Attribute>
+            </Language>
+        </Element>
+        <Element name="requires">
+            <Attribute language="en" name="identifier">Requires</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource requires the referenced resource to support its function, delivery, or coherence of content.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isPartOf">
+            <Attribute language="en" name="identifier">Is Part Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is a physical or logical part of the referenced resource.</Attribute>
+                <Attribute name="comment">Used to describe the relationship between a service and relevant documents (e.g. web pages and forms)</Attribute>
+            </Language>
+        </Element>
+        <Element name="hasPart">
+            <Attribute language="en" name="identifier">Has Part</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource includes the referenced resource either physically or logically.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isReferencedBy">
+            <Attribute language="en" name="identifier">Is Referenced By</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is referenced, cited, or otherwise pointed to by the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="references">
+            <Attribute language="en" name="identifier">References</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource references, cites, or otherwise points to the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isFormatOf">
+            <Attribute language="en" name="identifier">Is Format Of</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is the same intellectual content of the referenced resource, but presented in another format.</Attribute>
+            </Language>
+        </Element>
+        <Element name="hasFormat">
+            <Attribute language="en" name="identifier">Has Format</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource pre-existed the referenced resource, which is essentially the same intellectual content presented in another format.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isBasisFor">
+            <Attribute language="en" name="identifier">Is Basis For</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource pre-existed the referenced resource, which is a performance, production, derivation, translation, or interpretation of the described resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isBasedOn">
+            <Attribute language="en" name="identifier">Is Based On</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The described resource is a performance, production, derivation, translation, or interpretation of the referenced resource.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Coverage" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Coverage</Attribute>
+            <Attribute name="definition">The extent or scope of the content of the resource.</Attribute>
+            <Attribute name="comment">Coverage will typically include spatial location (a place name or geographic coordinates), temporal period (a period label, date, or date range) or jurisdiction (such as a named administrative entity). Recommended best practice is to select a value from a controlled vocabulary (for example, the Thesaurus of Geographic Names [TGN]) and that, where appropriate, named places or time periods be used in preference to numeric identifiers such as sets of coordinates or date ranges.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kapitanga</Attribute>
+            <Attribute name="definition">Te roanga, te whanuitanga ranei o te whakakaupapatanga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka uru atu te wahi mokowa (te ingoa o tetahi wahi, whakapiritanga matawhenua ranei), whakaritenga wa (he tapatanga wa (period label, ra, whakatakotoranga whanui o nga ra), te whakaritenga ture (penei i te putahi whakahaere). Ko te tino huarahi kua tohua, ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, i tetahi Puna Kupu Ingoa Matawhenua (Geographic Names [TGN] i nga wahi tika, ko nga wahi me nga whakaritenga wa ka whakamahia i mua i te whakamahitanga o nga taututanga tau penei i nga huinga whakapiritanga me nga whakatakotoranga whanui o nga ra.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Couverture</Attribute>
+            <Attribute name="definition">La portÃ©e ou la couverture spatio-temporelle de la ressource.</Attribute>
+            <Attribute name="comment">La couverture typiquement inclut une position gÃ©ographique (le nom d&apos;un lieu ou ses coordonnÃ©es), une pÃ©riode de temps (un nom de pÃ©riode, une date, ou un intervalle de temps) ou une juridiction (telle que le nom d&apos;une entitÃ© administrative). Il est recommandÃ© de choisir la  valeur de Couverture dans un vocabulaire contrÃŽlÃ© (par exemple, un thÃ©saurus de noms gÃ©ographiques, comme[TGN]) et, quand cela est appropriÃ©, des noms de lieux ou de pÃ©riodes plutÃŽt que des identifiants numÃ©riques tels que des coordonnÃ©es ou des intervalles de dates. </Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑ
+Ð²Ð°Ñ</Attribute>
+            <Attribute name="definition">ÐÑÐŸÑÑÐ¶ÐµÐœÐœÐŸÑÑÑ Ðž Ð³ÑÐ°ÐœÐžÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑ
+Ð²Ð°Ñ ÐŸÐ±ÑÐºÐœÐŸÐ²ÐµÐœÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²ÐµÐœÐœÐŸÐµ ÐŒÐµÑÑÐŸÐœÐ°Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÐµ (ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŒÐµÑÑÐœÐŸÑÑÐž ÐžÐ»Ðž Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÐºÐŸÐŸÑÐŽÐžÐœÐ°ÑÑ), Ð²ÑÐµÐŒÐµÐœÐœÐŸÐ¹ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐŸÐº (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ Ð¿ÐµÑÐžÐŸÐŽÐ°, ÐŽÐ°ÑÐ° ÐžÐ»Ðž ÑÑÐŽ ÐŽÐ°Ñ) ÐžÐ»Ðž ÑÑÐžÑÐŽÐžÐºÑÐžÑ (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ ÐµÐŽÐžÐœÐžÑÑ Ð°ÐŽÐŒÐžÐœÐžÑÑÑÐ°ÑÐžÐ²ÐœÐŸÐ³ÐŸ ÐŽÐµÐ»ÐµÐœÐžÑ). ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð¢ÐµÐ·Ð°ÑÑÑÑÐ° Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ TGN), Ðž, ÐµÑÐ»Ðž ÑÑÐŸ ÑÐŒÐµÑÑÐœÐŸ, Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐµ ÐŸÑÐŽÐ°ÐµÑÑÑ ÑÐºÐ°Ð·Ð°ÐœÐžÑÐŒ ÐŒÐµÑÑÐœÐŸÑÑÐµÐ¹ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐºÐŸÐ² Ð²ÐŒÐµÑÑÐŸ ÑÐžÑÑÐŸÐ²ÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ², ÐºÐ°Ðº, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐžÑÑÐµÐŒ ÐºÐŸÐŸÑÐŽÐžÐœÐ°Ñ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ ÑÑÐŽÐŸÐ².</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Cobertura</Attribute>
+            <Attribute name="definition">La caracterÃ­stica de cobertura espacial y/o temporal del contenido intelectual del recurso. La cobertura espacial se refiere a una regiÃ³n fÃ­sica (por ejemplo, sector celestial); uso de coordenadas (por ejemplo, longitud y latitud) o nombres de lugares extraidos de una lista controlada. La cobertura temporal se refiere al contenido del recurso en vez de a cuando fue creado o puesto accesible ya que este Ãºltimo pertenece al elemento Date. Se usa el mismo formato basado en http://www.w3.org/TR/NOTE-datetime.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØªØºØ·ÙØ©</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØ®ØµØ§ØŠØµ Ø§ÙÙ
+ÙØ§ÙÙØ© Ø§Ù Ø§ÙØ²Ù
+ÙÙØ© ÙÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . Ø§ÙØªØºØ·ÙØ© Ø§ÙÙ
+ÙØ§ÙÙØ© ØªØŽÙØ± Ø§ÙÙ Ø¬ÙØ© Ø·ØšÙØ¹ÙØ© )Ù
+Ø«Ù , ÙØ·Ø§Ø¹ Ø³Ù
+Ø§ÙÙ (    Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ø§ÙØ§Ø­Ø¯Ø§Ø«ÙØ§Øª ) Ù
+Ø«Ù  Ø®Ø· Ø§ÙØ·ÙÙ Ù Ø®Ø· Ø§ÙØ¹Ø±Ø¶ ( ÙÙ Ø£Ø³Ù
+Ø§Ø¡ Ø§Ù
+ÙÙØ© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù . Ø§ÙØªØºØ·ÙØ© Ø§ÙØ²Ù
+ÙÙØ©    Ø«Ø¹ÙØ¯ Ø§ÙÙ Ù
+Ø§ ÙØ±Ø¬Ø¹ Ø§ÙÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ù ÙÙØ³ Ø§Ù Ù
+ØªÙ ØªÙ
+ ØªÙÙÙØ±Ù ) Ø§ÙØ§Ø®ÙØ± ÙØªØ¹ÙÙ ØšØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® ( Ø§Ø³ØªØ®Ø¯Ø§Ù
+ ÙÙØ³ ØµÙØºØ© Ø§ÙØªØ§Ø±ÙØ®  Ø§Ù Ø§ÙÙÙØª    ) Ø¹Ø§Ø¯Ø© Ù
+Ø¯Ù ] ( ØµÙØ§ØºØ§Øª Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601 Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© [http://www.w3.org/TR/NOTE-datetime, W3C    ÙÙ
+Ø§ ÙÙ Ù
+ÙØµÙ ØšÙ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® Ø§Ù ÙØªØ±Ø§Øª Ø²Ù
+ÙÙØ© Ù
+Ù ÙØ§ØŠÙ
+Ø© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èŠçèåŽ</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹ææ¶åçå€å»¶äžèŠçèåŽ</Attribute>
+            <Attribute name="comment">èŠçèåŽäžè¬å
+æ¬ç©ºéŽäœçœ®ïŒäžäžªå°åæå°çåæ ïŒãæ¶éŽåºéŽïŒäžäžªæ¶éŽæ è¯ïŒæ¥ææäžäžªæ¥æè    åŽïŒãæè
+æéïŒæ¯åŠåœåçææå®äœïŒæšèèŠçèåŽæå¥œæ¯åèªäºäžäžªåæ§è¯è¡šïŒäŸåŠå°åè¯åº    [TGN]ïŒïŒå¹¶åºå°œå¯èœå°äœ¿çšç±æ°å­è¡šç€ºçåæ ææ¥æåºéŽæ¥æè¿°å°åäžæ¶éŽæ®µã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PokrytÃ­</Attribute>
+            <Attribute name="definition">Rozsah nebo zÃ¡bÄr obsahu zdroje.</Attribute>
+            <Attribute name="comment">PokrytÃ­ bude typicky obsahovat prostorovÃ© umÃ­stÄnÃ­ (jmÃ©no mÃ­sta nebo zemÄpisnÃ© souÅadnice), ÄasovÃ© obdobÃ­ (oznaÄenÃ­ obdobÃ­, datum nebo ÄasovÃœ interval) nebo jurisdikci (napÅ. jmÃ©no administrativnÃ­ jednotky). DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. Thesaurus of Geographic Names [TGN]) a tam, kde je to vhodnÃ©, uvÃ¡dÄt mÃ­sta nebo ÄasovÃ© Ãºseky spÃ­Å¡e jmÃ©nem neÅŸ ÄÃ­selnÃœmi identifikÃ¡tory jako jsou soubory souÅadnic nebo ÄasovÃ© intervaly.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Dekking</Attribute>
+            <Attribute name="definition">De ruimtelijke of tijdelijke karakteristieken van de intellectuele inhoud van de Internetbron. Ruimtelijke dekking verwijst naar een geografisch gebied. Gebruik coordinaten (longitude (lengtegraad) en latitude (breedtegraad)) of plaatsnamen die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Dekking in de tijd geeft de tijdsperiode aan waarop de inhoud van de Internetbron betrekking heeft en niet wanneer de Internetbron is gemaakt of beschikbaar is gesteld (dit laatste hoort ingevult te worden bij &quot;Date&quot;). Gebruik dezelfde datum/tijd formaat als bij &quot;Date&quot;; W3C Technical Note http://www.w3.org/TR/NOTE-datetime of tijdsperioden die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Coverage is in ontwikkeling bij de Dublin Core Working group http://www.alexandria.ucsb.edu/docs/metadata/dc_coverage.html.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Copertura</Attribute>
+            <Attribute name="definition">Le extension physic o abstracte del contento del ressource.</Attribute>
+            <Attribute name="comment">Le copertura typicamente include un location spatial (un nomine de localitate o coordinatas geographic), un periodo de tempore (un etiquetta de periodo, data, o gamma de datas), o un jurisdiction (tal como un entitate administrative nominate). Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le Thesaurus of Geographical Names [TGN]) e que, quando appropriate, localitates o periodos de tempore nominate sia utilizate in preferentia a identificatores numeric tales como collectiones de coordinatas o gammas de datas.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à¥à€à¥à€²à€¿à€  à€®à€Ÿà€¹à€¿à€€à¥</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€­à¥à€à¥à€²à€¿à€ à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="jurisdiction">
+            <Attribute language="en" name="identifier">Jurisdiction</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">The name of the political/administrative entity covered by the content of the resource.</Attribute>
+                <Attribute name="comment">Jurisdiction is a description of the territory over which a particular government exercises its authority or a particular business transacts its operations, to which the resource content is applicable.</Attribute>
+            </Language>
+        </Element>
+        <Element name="spatial">
+            <Attribute language="en" name="identifier">Spatial</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Spatial characteristics of the intellectual content of the resource.</Attribute>
+                <Attribute name="comment">Spatial (geographic) coverage or locations or areas covered in the content of a resource. Use of the New Zealand Geographic Place Names Database (LINZ) may be appropriate. In some cases the DCMI Box Encoding Scheme may be suitable. Refer: http//dublincore.org.documents/2000/07/28/dcmi-box/</Attribute>
+            </Language>
+        </Element>
+        <Element name="temporal">
+            <Attribute language="en" name="identifier">Temporal</Attribute>
+            <Attribute language="en" name="qualifier type">element refinement</Attribute>
+            <Language code="en">
+                <Attribute name="definition">Temporal characteristics of the intellectual content of the resource.</Attribute>
+                <Attribute name="comment">The refinement &amp;quot;temporal&amp;quot; refers to time periods that are covered by or discussed in the content of resource. Time periods will generally be best expressed as date ranges, although named periods may be appropriate. The recommended encoding schemes for dates and date ranges are noted in the section on Date.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Rights" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Rights Management</Attribute>
+            <Attribute name="definition">Information about rights held in and over the resource.</Attribute>
+            <Attribute name="comment">Typically, a Rights element will contain a rights management statement for the resource, or reference a service providing such information. Rights information often encompasses Intellectual Property Rights (IPR), Copyright, and various Property Rights. If the Rights element is absent, no assumptions can be made about the status of these and other rights with respect to the resource.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Motika</Attribute>
+            <Attribute name="definition">Ko nga mohiohio ki nga motika o tetahi rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, kei te huanga Motika e pupuri ana he korero mo te whakaritenga motika mo te rauemi, he ratonga tohutoro e whakarato ana ranei i enei tumomo mohiohio. He nui nga wa, puta ai i roto i te mohiohio Motika te Mana Whakairo Hinegaro (Intellectual Property Rights - (IPR)), te Manatarua me nga Tuahua Mana Taonga (Property Rights). Ki te kore te huanga Motika, kaore e taea te pohehe noa ki te mana o enei, nga motika ranei e pa ana ki te rauemi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Droits</Attribute>
+            <Attribute name="definition">Information sur les droits sur et au sujet de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, un Ã©lÃ©ment Droits contiendra un Ã©tat du droit Ã  gÃ©rer une ressource, ou la refÃ©rence Ã  un service fournissant cette information. Ces droits souvent couvrent les droits de propriÃ©tÃ© intellectuelle (IPR), Copyright, et divers droits de propriÃ©tÃ©. Si l&apos;Ã©lÃ©ment Droits est absent, aucune hypothÃšse ne peut Ãªtre faite sur l&apos;Ã©tat de ces droits, ou de tout autre, par rapport Ã  la ressource.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐ°Ð²ÐŸÐ²ÐŸÐµ ÑÐµÐ³ÑÐ»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ÐÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð¿ÑÐ°Ð²Ð°Ñ
+ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐ°ÑÑÑ Ð²ÐœÑÑÑÐž Ðž Ð²ÐœÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÑÐ»ÐµÐŒÐµÐœÑ ÐÑÐ°Ð²Ð° ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐµ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÑÑ
+ ÐœÐŸÑÐŒÐ°Ñ
+, ÑÐµÐ³ÑÐ»ÐžÑÑÑÑÐžÑ
+ ÑÑÐœÐºÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐµÑÑÑÑÐ° ÐžÐ»Ðž ÑÑÑÐ»ÐºÑ ÐœÐ° ÑÐ»ÑÐ¶Ð±Ñ, Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÑÑ ÑÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ. ÐÑÐ°Ð²ÐŸÐ²Ð°Ñ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸÐ±ÑÑÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐ²ÐµÐŽÐµÐœÐžÑ ÐŸ ÐÑÐ°Ð²Ð°Ñ
+ ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ¹ ÑÐŸÐ±ÑÑÐ²ÐµÐœÐœÐŸÑÑÐž, ÐÐ²ÑÐŸÑÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²Ðµ Ðž ÐŽÑÑÐ³ÐžÑ
+ ÐÐŒÑÑÐµÑÑÐ²ÐµÐœÐœÑÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+. ÐÑÑÑÑÑÑÐ²ÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐÑÐ°Ð²Ð° ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÑÐ²Ð»ÑÑÑÑÑ ÐŸÑÐœÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŽÐ»Ñ ÐºÐ°ÐºÐžÑ
+-Ð»ÐžÐ±ÐŸ Ð¿ÑÐµÐŽÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐ¹ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÐŸÐŒ ÑÑÐ°ÑÑÑÐµ Ðž ÐŽÑÑÐ³ÐžÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+, ÐºÐ°ÑÐ°ÑÑÐžÑ
+ÑÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Derechos</Attribute>
+            <Attribute name="definition">Una referencia (URL, por ejemplo) para una nota sobre derechos de autor, para un servicio de gestiÃ³n de derechos o para un servicio que darÃ¡ informaciÃ³n sobre tÃ©rminos y condiciones de acceso a un recurso. Una especificaciÃ³n formal del elemento Rights se encuentra catualmente en discusiÃ³n y por lo tanto su uso se considera experimental.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø©</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ®Ø¯Ù
+Ø© ØªÙØ¯Ù
+ Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æé</Attribute>
+            <Attribute name="definition">æå
+³èµæºæ¬èº«ææçæè¢«èµäºçæéä¿¡æ¯</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæéå
+çŽ åºå
+æ¬äžäžªå¯¹èµæºçæéå£°æïŒæè
+æ¯å¯¹æäŸè¿äžä¿¡æ¯çæå¡çåç
+§ãæéäžè¬    å
+æ¬ç¥è¯äº§æ(IPR)ïŒçææå
+¶ä»åç§åæ ·çäº§æãåŠææ²¡ææéå
+çŽ çæ æ³šïŒäžå¯ä»¥å¯¹äžèµæºçžå
+³    çäžè¿°æå
+¶ä»æå©çæ
+åµäœåºä»»äœåå®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PrÃ¡va</Attribute>
+            <Attribute name="definition">Informace o prÃ¡vech vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+            <Attribute name="comment">Prvek PrÃ¡va bude typicky obsahovat ustanovenÃ­ o sprÃ¡vÄ autorskÃœch/vlastnickÃœch prÃ¡v ke zdroji nebo odkaz na sluÅŸbu poskytujÃ­cÃ­ takovÃ©to informace. Informace o prÃ¡vech Äasto zahrnujÃ­ prÃ¡va duÅ¡evnÃ­ho vlastnictvÃ­ (IPR), autorskÃ© prÃ¡vo a rÅ¯znÃ¡ vlastnickÃ¡ prÃ¡va. JestliÅŸe prvek PrÃ¡va chybÃ­, nelze odvozovat ÅŸÃ¡dnÃ© pÅedpoklady o stavu tÄchto a jinÃœch prÃ¡v vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Copyright</Attribute>
+            <Attribute name="definition">Een copyright verklaring of een link naar een copyright verklaring of een link naar een service die informatie geeft over de copyright van de Internetbron.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Gestion de derectos</Attribute>
+            <Attribute name="definition">Information super derectos detenite super e a proposito del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le elemento derectos continera un declaration de gestion de derectos pro le ressource, o un referentia a un servicio que provide tal information. Information de derectos frequentemente involve derectos de proprietate intellectual (IPR), copyright e varie derectos de proprietate. Si le elemento Derectos es absente, necun hypothese pote esser facite quanto al stato de iste e altere derectos concernente ressource.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€¹à€à¥à€</Attribute>
+            <Attribute name="definition">à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+            <Attribute name="comment">à€žà¥à€µà€€à€à€¹à€Ÿà€à¥ à€²à¥à€à€š à€à€Ÿà€ªà¥à€š à€
+à€¥à€µà€Ÿ à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€° à€ªà¥à€°à€à€Ÿà€¶à€¿à€€ à€à€°à€£à¥à€¯à€Ÿà€à€Ÿ à€²à¥à€à€ à€
+à€¥à€µà€Ÿ à€²à¥à€à€¿à€à¥à€à€Ÿ à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+</MetadataSet>
Index: /main/tags/3.01/gli/metadata/qdublin.mds
===================================================================
--- /main/tags/3.01/gli/metadata/qdublin.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/qdublin.mds	(revision 21112)
@@ -0,0 +1,1966 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The name of this sets family.                                 -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The namespace of this metadata set.                           -->
+  <!ELEMENT MetadataSet ((Name|Description|SetLanguage|Element)+)>
+  <!ATTLIST MetadataSet
+	    contact            CDATA #REQUIRED
+	    creator            CDATA #REQUIRED
+	    description        CDATA #REQUIRED
+            family             CDATA #REQUIRED
+	    lastchanged        CDATA #REQUIRED
+            name               CDATA #REQUIRED
+	    namespace          CDATA #REQUIRED
+	    language_dependent CDATA "">
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT SetLanguage (Name, Description?)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!-- Note that the name attribute isn't really Dublin Core compliant - it was just the first thing I came up with. -->
+  <!ELEMENT Element (Attribute*,Language*,OptionList?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+	    occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->	
+  <!-- Given that name was already used for an Element, I decided to use identifier for the language specific 'name', which I've been told isn't really compliant with Dublin Core. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA "">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT Language (Attribute*)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+<MetadataSet contact="dcmi-feedback@dublincore.org" creator="Dublin Core Metadata Initiative" family="Dublin Core" language_dependent="identifier, definition, comment" lastchanged="Tue Jun 04 10:35:30 2002 - jmt12 on blackbird" namespace="dc">
+    <!-- English Version sourced from http://www.dublincore.org/documents/dces/ -->
+    <SetLanguage code="en">
+        <Name language="en">Qualified Dublin Core Metadata Element Set, Version 1.1: Reference Description</Name>
+        <Description language="en">This document is the reference description, version 1.1 of the Dublin Core Metadata Element Set. This document supersedes the Dublin Core Metadata Element Set, version 1.0. See the Dublin Core Home Page (http://dublincore.org) for further information about the workshops, reports, working group papers, projects, and new developments concerning the Dublin Core Metadata Element set</Description>
+    </SetLanguage>
+    <!-- Maori Version sourced from http://www.tki.org.nz/r/maori/pedagogy/dc_elements_m.php -->
+    <SetLanguage code="mi">
+        <Name>Dublin Core Huanga - Maori (He Tauira)</Name>
+    </SetLanguage>
+    <!-- French Version sourced from http://www-rocq.inria.fr/%7Evercoust/METADATA/DC-fr.1.1.html -->
+    <!--  by Anne-Marie Vercoustre, Inria -->
+    <SetLanguage code="fr">
+        <Name language="fr">ElÃ©ments de mÃ©tadonnÃ©es du Dublin Core, Version 1.1: Description de RÃ©ference</Name>
+        <Description language="fr">Ce document rÃ©sume les derniÃšres dÃ©finitions des Ã©lÃ©ments demÃ©tadonnÃ©es du Dublin Core originellement dÃ©finis dans le document[RFC2413]. Ces nouvelles dÃ©finitions seront officiellement connues comme la Version 1.1. Elles utilisent un standard dÃ©fini formellement pour la description d&apos;Ã©lÃ©ments de mÃ©tadonnÃ©es. Cette formalisation contribue Ã  amÃ©liorer la cohÃ©rence avec d&apos;autres communautÃ©s dÃ©crivant des mÃ©tadonnÃ©es et Ã  augmenter la prÃ©cision, la portÃ©e et la cohÃ©rence interne de la dÃ©finition des Ã©lÃ©ments du Dublin Core.</Description>
+    </SetLanguage>
+    <!-- Russian Version sourced from -->
+    <!-- by Olga Barysheva, National Library of Russia -->
+    <SetLanguage code="ru">
+        <Name language="ru">ÐÐ°Ð±ÐŸÑ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core (ÐÑÐ±Ð»ÐžÐœÑÐºÐŸÐ³ÐŸ ÑÐŽÑÐ°) ÐÐµÑÑÐžÑ 1.1: Ð¡Ð¿ÑÐ°Ð²ÐŸÑÐœÐŸÐµ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐµ</Name>
+        <Description language="ru">ÐÐ°ÑÑÐŸÑÑÐžÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÐŸÐ±ÐŸÐ±ÑÐ°ÐµÑ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐ² ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Dublin Core (ÐÑÐ±Ð»ÐžÐœÑÐºÐŸÐ³ÐŸ ÑÐŽÑÐ°), Ð¿ÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÐŸ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ Ð² RFC2413 (ÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Dublin Core ÐŽÐ»Ñ ÐœÐ°Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÑ ÑÐµÑÑÑÑÐŸÐ²). Ð­ÑÐž ÐœÐŸÐ²ÑÐµ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ ÐŸÑÐžÑÐžÐ°Ð»ÑÐœÐŸ ÐžÐ·Ð²ÐµÑÑÐœÑ ÐºÐ°Ðº ÐÐµÑÑÐžÑ 1.1.</Description>
+    </SetLanguage>
+    <!-- Spanish Version sourced from http://www.rediris.es/metadata/. Please note that this is a translation for Dublin Core v1.0 -->
+    <!-- by Javi Masa, RedIRIS -->
+    <SetLanguage code="es">
+        <Name language="es">Elementos del conjunto de metadatas de Dublin Core: DescripciÃ³n de Referencia</Name>
+        <Description language="es">Este documento corresponde a una traducciÃ³n del original en lengua inglesa, con fecha 02/11/1997 que se encuentra en http://purl.org/metadata/dublin_core_elements. Este documento es una descripciÃ³n de referencia de los elementos del conjunto de &quot;Dublin Core&quot;. Para mÃ¡s informaciÃ³n sobre reuniones de trabajo, documentos de trabajo, proyectos y nuevos desarrollos relacionados con el conjunto de elementos que aquÃ­ se describen consultar la &quot;Home Page&quot; de &quot;Dublin Core&quot; (http://purl.org/metadata/dublin_core). Los elementos poseen nombres descriptivos que pretenden transmitir un significado semÃ¡ntico a los mismos. Para promover una interoperabilidad global, una descripciÃ³n del valor de algunos elementos prodrÃ¡ ser asociada a vocabularios controlados. Se asume que otros vocabularios controlados serÃ¡n desarrollados para asegurar esta interoperabilidad en dominios especÃ­ficos. Cada elemento es opcional y puede repetirse. AdemÃ¡s, los elementos pueden aparecer en cualquier orden. Aunque algunos entornos, como HTML, no diferencian entre mayÃºsculas y minÃºsculas, es recomendable escribir correctamente cada metadata segÃºn su definiciÃ³n para evitar conflictos con otros entornos, como XML (Extensible Markup Language) http://www.w3.org/TR/PR-xml</Description>
+    </SetLanguage>
+    <!-- Arabic Version sourced from http://haddouti.de/DC_arabic.html -->
+    <!-- maintained by Hachim Haddouti. -->
+    <SetLanguage code="ar">
+        <Name>Ø¯ØšÙÙÙ ÙÙØ±</Name>
+        <Description>ÙØ±Ø­Øš ØšÙÙ
+ ÙÙ ØµÙØ­Ø©  Ø¯ ØšÙÙÙ ÙÙ  Ø± ØšØ§ÙØ¹Ø±ØšÙ    ÙÙ  ÙØ°Ù Ø§ÙØµÙØ­Ø©  Ù ØšØ§ÙØªØ¹Ø§ÙÙ Ù
+Ø¹ Ø§ÙØ§Ø®ÙØ§Ù Ø§ÙØ¹Ø±Øš Ù
+Ù Ø§ÙÙ
+ÙØªØšØ§ØªØ Ø¯Ø§Ø± Ø§ÙÙØŽØ±Ø Ø§ÙØšØ­ÙØ« Ø§ÙØ¹ÙÙ
+ÙØ© Ù    ÙØ±ØŽØ© Ø¹Ù
+ÙWorking Group On Multilingual Dublin Core (DC-International) Ø</Description>
+    </SetLanguage>
+    <!-- Chinese - Simplified Version from http://dc.library.sh.cn/DC/1-1.htm -->
+    <!-- by Liu Wei, Shanghai Library -->
+    <SetLanguage code="cn">
+        <Name>éœæææ žå¿å
+æ°æ®å
+çŽ é1.1çïŒåèæè¿°</Name>
+        <Description>æ¬ææ¡£å
+³äºéœæææ žå¿å
+æ°æ®å
+çŽ é1.1çæ¬çåèæè¿°ãæ­€æè¿°äžå·²ç»æäº€ç»ISOçšäºåœé
+æç¥šçNISOå®æ¹æ åä¿æäžèŽã    éœæææ žå¿å
+æ°æ®éæ¯äžç§è·šé¢åçä¿¡æ¯èµæºæè¿°æ åãè¿éçä¿¡æ¯èµæºè¢«å®ä¹äžºâä»»äœå
+·ææ è¯çäžè¥¿âãæ­€å®ä¹æ¥èªäºTim Berners-Leeç­äººæèçInternet RFC2396 &quot;Uniform Resource Identifiers (URI): Generic Syntax&quot;æä»¶ãéœæææ žå¿å
+æ°æ®çåºçšé¢åæ²¡ææ ¹æ¬çéå¶ã</Description>
+    </SetLanguage>
+    <!-- Czech Version from http://www.ics.muni.cz/dublin_core/DC-czech-1.1.html -->
+    <!-- Ustav vypocetni techniky Masarykovy univerzity v Brne -->
+    <SetLanguage code="cs">
+        <Name>Soubor metadatovÃœch prvkÅ¯ Dublin Core, verze 1.1: referenÄnÃ­ popis</Name>
+        <Description>Tento dokument je ÄeskÃœm pÅekladem anglickÃ© verze referenÄnÃ­ho popisu souboru metadatovÃœch prvkÅ¯  Dublin Core, verze 1.1 [DCEN1.1]. Tento dokument nahrazuje ÄeskÃœ pÅeklad souboru metadatovÃœch prvkÅ¯ Dublin Core, verze 1.0 [DCCS1.0]. DalÅ¡Ã­ informace o pracovnÃ­ch seminÃ¡ÅÃ­ch (workshopech), zprÃ¡vÃ¡ch, studiÃ­ch pracovnÃ­ch skupin, projektech a dalÅ¡Ã­m vÃœvoji souboru metadatovÃœch prvkÅ¯ Dublin Core najdete na domovskÃ© strÃ¡nce Dublin Core (http://purl.org/dc)</Description>
+    </SetLanguage>
+    <!-- Dutch Version from http://www.kb.nl/coop/donor/project-nl-index.html?/coop/donor/rapporten/DCsimpleformat.html -->
+    <!-- by ... -->
+    <SetLanguage code="nl">
+        <Name>Rapporten Dublin Core Simple Format</Name>
+        <Description>Dit document beschrijft welke Dublin Core metadata elementen er zijn en geeft een korte definitie van elk element. Voor uitgebreide uitleg en gebruik van de Dublin Core elementen wordt u verwezen naar de gebruikershandleiding. Deze vertaling geeft ook de keuzes aan voor standaarden (taalcodes, datum codes) die wij in DONOR verband gaan hanteren. In zoverre kan deze vertaling afwijken van het oorspronkelijke document &quot;Description of Dublin Core Elements&quot; http://purl.org/metadata/dublin_core_elements&quot;</Description>
+    </SetLanguage>
+    <!-- Finnish Version from http://www.lib.helsinki.fi/dublin_core/dcref-fin.html -->
+    <!-- by Jani Stenvall and Juha Hakala, Helsinki University Library -->
+    <SetLanguage code="fi">
+        <Name/>
+        <Description/>
+    </SetLanguage>
+    <!-- Interlingua Version sourced from http://www.nautilus.com.br/%7Eensjo/ia/purl.org/dc/documents/rec-dces-19990702.htm -->
+    <!-- by Emerson Jose` Silveira da Costa -->
+    <SetLanguage code="ia">
+        <Name>Collection de elementos de metadatos de Dublin Core, version 1.1: Description de referentia</Name>
+        <Description>Isto es un traduction de un Recommendation del Dublin Core Metadata Initiative. Le publication como un recommendation significa que le specificationes es stabile e su adoption es supportate per le communitate Dublin Core (http://purl.org/dc/).</Description>
+    </SetLanguage>
+    <!-- Marathi Version sourced from http://bioinfo.ernet.in/library/dc/ele.txt -->
+    <!-- by by Shubhada Nagarkar, Bioinformatics Centre, University of Pune -->
+    <SetLanguage code="mr">
+        <Name>à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿ à€žà¥à€ à€à€µà¥à€€à¥à€€à¥ à¥§.à¥§ à€žà€à€Šà€°à¥à€­ à€žà¥à€ªà€·à¥à€à¥à€à€°à€£ à¥§.à¥§ à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿà€à€µà¥à€€à¥à€€à¥ à¥§.à¥§.</Name>
+        <Description>à¥§.à¥§ à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à¥à€à€Ÿà€¡à¥à€à€Ÿ à€¹à¥ à€µà¥à€à€µà¥à€à€³à¥à€¯à€Ÿ à€µà€¿à€·à€¯à€Ÿà€€à¥à€² à€®à€Ÿà€¹à¥à€€à¥ à€žà€Ÿà€ à¥à€¯à€Ÿà€à€µà€Šà¥à€Šà€²à€à¥ à€®à€Ÿà€¹à¥à€€à¥ à€Šà¥à€£à¥à€¯à€Ÿà€à¥ à€®à€Ÿà€šà€ à€à€¹à¥.  à€¯à€Ÿà€€ à€®à€Ÿà€¹à¥à€€à¥à€žà€Ÿà€ à€Ÿ à€®à¥à€¹à€£à€à¥ à€à¥à€¯à€Ÿà€²à€Ÿ à€à€³à€ à€à€¹à¥ à€
+à€žà€Ÿ à€à€žà€€à¥.  à€
+à€¶à€Ÿ à€ªà¥à€°à€à€Ÿà€°à€à¥ à€µà¥à€¯à€Ÿà€à¥à€¯à€Ÿ à€®à€Ÿà€¹à¥à€€à¥ à€à€Ÿà€³à¥à€¯à€µà€°à¥à€² à€. à€à€«.à€žà¥. à¥šà¥©à¥¯à¥¬URI: Generic Syntax -- Tim Berners Lee et al. à€¡à€¬à¥à€²à€¿à€š à€à¥à€° à€®à€Ÿà€šà€ à€à¥à€£à€€à¥à€¯à€Ÿà€¹à¥ à€ªà¥à€°à€à€Ÿà€°à€à¥à€¯à€Ÿ à€®à€Ÿà€¹à¥à€€à¥ à€žà€Ÿà€ à¥à€¯à€Ÿà€à€žà€Ÿà€ à¥ à€µà€Ÿà€ªà€°à€€à€Ÿ à€¯à¥à€€à¥.  à€€à¥à€¯à€Ÿà€žà€Ÿà€ à¥ à€à¥à€£à€€à€Ÿà€¹à¥ à€®à€à¥à€à€Ÿà€µ à€šà€Ÿà€¹à¥.</Description>
+    </SetLanguage>
+    <Element name="Title" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Title</Attribute>
+            <Attribute name="definition">A name given to the resource.</Attribute>
+            <Attribute name="comment">Typically, a Title will be a name by which the resource is formally known.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Taitara</Attribute>
+            <Attribute name="definition">Te ingoa ka tapaina tetahi rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ko te Taitara te ingoa ka mohio whanuitia te ingoa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Titre</Attribute>
+            <Attribute name="definition">Le nom donnÃ© Ã  la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, un titre sera le nom par lequel la ressource est officiellement connue.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ÐÐŒÑ, ÐŽÐ°ÐœÐœÐŸÐµ ÑÐµÑÑÑÑÑ.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÐžÐŒÑ, Ð¿ÐŸÐŽ ÐºÐŸÑÐŸÑÑÐŒ ÑÐµÑÑÑÑ ÐŸÑÐžÑÐžÐ°Ð»ÑÐœÐŸ ÐžÐ·Ð²ÐµÑÑÐµÐœ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">TÃ­tulo</Attribute>
+            <Attribute name="definition">El nombre dado a un recurso, usualmente por el autor.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØ¹ÙÙØ§Ù</Attribute>
+            <Attribute name="definition">ÙØ°Ø§ Ø§ÙØ§Ø³Ù
+ ÙØ¹Ø·Ù ÙÙÙ
+ØµØ¯Ø±, Ù
+Ù ÙØšÙ Ø§ÙÙ
+ØšØ¯Ø¹ Ø§Ù Ø§ÙÙØ§ØŽØ±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºå</Attribute>
+            <Attribute name="definition">èµäºèµæºçåç§°ã</Attribute>
+            <Attribute name="comment">èµæºåäžè¬æèµæºå¯¹è±¡æ­£åŒå
+¬åŒçåç§°ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">NÃ¡zev</Attribute>
+            <Attribute name="definition">JmÃ©no danÃ© zdroji.</Attribute>
+            <Attribute name="comment">NÃ¡zev bude typicky jmÃ©no, pod nÃ­mÅŸ je zdroj oficiÃ¡lnÄ znÃ¡m.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Titel</Attribute>
+            <Attribute name="definition">De naam van de Internetbron. Meestal gegeven door de auteur, maker of uitgever</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Titulo</Attribute>
+            <Attribute name="definition">Un nomine date al ressource.</Attribute>
+            <Attribute name="comment">Typicamente se utiliza un nomine per le qual le ressource es formalmente cognite.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€¶à¥à€°à¥à€·à€</Attribute>
+            <Attribute name="definition">à€ªà€°à¥à€¯à€Ÿà€¯à¥ à€¶à¥à€°à¥à€·à€</Attribute>
+            <Attribute name="comment">à€žà€à€à¥à€·à€¿à€ªà¥à€€ ,  à€­à€Ÿà€·à€Ÿà€à€€à€°à¥à€€  à€¶à¥à€°à¥à€·à€</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="alternative">
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Alternative</Attribute>
+                <Attribute name="definition">Any form of the title used as a substitute or alternative to the formal title of the resource.</Attribute>
+                <Attribute name="comment">This qualifier can include Title abbreviations as well as translations.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+    </Element>
+    <Element name="Creator" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Creator</Attribute>
+            <Attribute name="definition">An entity primarily responsible for making the content of the resource.</Attribute>
+            <Attribute name="comment">Examples of a Creator include a person, an organisation, or a service. Typically, the name of a Creator should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaihanga</Attribute>
+            <Attribute name="definition">Te putahi, nana, na ratou ranei i matua oti ai te hanganga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaihanga. Ko te tikanga, ko te Kaihanga te ingoa ka whakamahia hei tohu i te putahi na ratou te hanganga.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">CrÃ©ateur</Attribute>
+            <Attribute name="definition">L&apos;entitÃ© principalement responsable de la crÃ©ation du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Exemples de CrÃ©ateur incluent une personne, une organisation, ou un service. Typiquement, un nom du CrÃ©ateur devrait Ãªtre utilisÃ© pour dÃ©signer cette entitÃ©.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, ÐœÐµÑÑÑÐµÐµ Ð¿ÐµÑÐ²ÐžÑÐœÑÑ ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÑÑÑ Ð·Ð° ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Autor o Creador</Attribute>
+            <Attribute name="definition">La persona u organizaciÃ³n responsable de la creaciÃ³n del contenido intelectual del recurso. Por ejemplo, los autores en el caso de documentos escritos, artistas, fotografos e ilustradores en el caso de recursos visuales.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØ§ØªØš Ø§Ù Ø§ÙÙ
+ØšØ¯Ø¹</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø£ÙÙØ§  Ø¹Ù ÙØŽØŠ Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ± Ù ÙÙÙ
+ØµØ¯Ø± , Ø¹ÙÙ Ø³ØšÙÙ Ø§ÙÙ
+Ø«Ø§Ù , Ø§ÙÙ
+Ø€ÙÙÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙØ«Ø§ØŠÙ Ø§ÙÙ
+ÙØªÙØšØ© , Ù Ø§ÙÙÙØ§ÙÙÙ , Ø§ÙÙ
+ØµÙØ±ÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">åå»ºè
+</Attribute>
+            <Attribute name="definition">åå»ºèµæºå
+å®¹çäž»èŠèŽ£ä»»è
+ã</Attribute>
+            <Attribute name="comment">åå»ºè
+çå®äŸå
+æ¬äžªäººïŒç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšåå»ºè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">TvÅ¯rce</Attribute>
+            <Attribute name="definition">Entita primÃ¡rnÄ odpovÄdnÃ¡ za vytvoÅenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady TvÅ¯rce zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no tvÅ¯rce.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Auteur of maker</Attribute>
+            <Attribute name="definition">De auteur, maker of organisatie die primair verantwoordelijk is voor de intellectuele inhoud van het werk. Bijvoorbeeld auteurs in geval van geschreven documenten, artiesten, fotografen of illustrators in geval van visuele bronnen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Creator</Attribute>
+            <Attribute name="definition">Le entitate primarimente responsabile pro producer le contento del ressource.</Attribute>
+            <Attribute name="comment">Exemplos de creator include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de creator pro indicar iste entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€šà€¿à€°à¥à€®à€Ÿà€€à€Ÿ, à€à€šà€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€šà€¿à€°à¥à€®à€Ÿà€£  à€à€°à€£à€Ÿà€°à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.  à€²à¥à€à€, à€²à¥à€à€¿à€à€Ÿ,  à€µà¥à€¯à€à¥à€€à¥, à€žà€à€žà¥à€¥à€Ÿ,  à€žà€à€ªà€Ÿà€Šà€, à€žà€à€ªà€Ÿà€Šà€¿à€à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Subject" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Subject and Keywords</Attribute>
+            <Attribute name="definition">The topic of the content of the resource.</Attribute>
+            <Attribute name="comment">Typically, a Subject will be expressed as keywords, key phrases or classification codes that describe a topic of the resource. Recommended best practice is to select a value from a controlled vocabulary or formal classification scheme.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaupapa</Attribute>
+            <Attribute name="definition">Te take/ nga take o te kaupapa, o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka whakaaturia te kaupapa o te rauemi e nga kupu matua, nga kimahi matua, nga tohu wehewehenga whakaahua ranei.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Sujet</Attribute>
+            <Attribute name="definition">Le sujet du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, le sujet sera dÃ©crit par un ensemble de mots-clefs ou de phrases ou un code de classification qui prÃ©cisent le sujet de la ressource. L&apos;utilisation de vocabulaires contrÃŽlÃ©s et de schÃ©mas formels de classification est encouragÃ©e.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐµÐŽÐŒÐµÑ Ðž ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð°</Attribute>
+            <Attribute name="definition">Ð¢ÐµÐŒÐ° ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÑÐµÐŽÐŒÐµÑ Ð²ÑÑÐ°Ð¶Ð°ÐµÑÑÑ Ñ Ð¿ÐŸÐŒÐŸÑÑÑ ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÐ»ÐŸÐ², ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÑÐ°Ð· ÐžÐ»Ðž  ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÐºÐŸÐŽÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑ ÑÐµÐŒÑ ÑÐµÑÑÑÑÐ°. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ ÐžÐ»Ðž ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÑ
+ÐµÐŒÑ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Claves</Attribute>
+            <Attribute name="definition">Los tÃ³picos del recurso. TÃ­picamente, Subject expresarÃ¡ las claves o frases que describen el tÃ­tulo o el contenido del recurso. Se fomentarÃ¡ el uso de vocabularios controlados y de sistemas de clasificaciÃ³n formales. Se pueden ver ejemplos de vocabularios controlados en: Controlled vocabularies, thesauri and classification systems available in the WWW. DC Subject</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+ÙØ¶ÙØ¹ Ù Ø§ÙÙÙÙ
+Ø§Øª Ø§ÙØ±ØŠÙØ³ÙØ©</Attribute>
+            <Attribute name="definition">ÙÙ Ù
+ÙØ¶ÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±. ÙÙ
+ÙØ°Ø¬ÙØ§  ÙØ¹ØšØ± Ø¹Ù Ø§ÙÙ
+ÙØ¶ÙØ¹ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+Ù
+ÙØªØ§Ø­  Ø§ÙÙÙÙ
+Ø§Øª  Ø§Ù Ø¬Ù
+Ù Ø§ÙØªÙ  ØªØµÙ Ù
+ÙØ¶ÙØ¹ Ø§Ù Ù
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø± , Ù ÙØ³ØªØ­Ø³Ù Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ù
+ÙØ±Ø¯Ø§Øª Ù
+Ø®Ø·Ø·Ø©Ù Ù
+Ø­ÙÙ
+Ø©  ÙØ§Ø¯ÙØ© ØªØµÙÙÙ ØŽÙÙÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">äž»é¢åå
+³é®è¯</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çäž»é¢æè¿°ã</Attribute>
+            <Attribute name="comment">åŠæèŠæè¿°ç¹å®èµæºçæäžäž»é¢ïŒäžè¬éçšå
+³é®è¯ãå
+³é®å­ç­è¯­æåç±»å·ïŒæå¥œäž»é¢åå
+³é®è¯ä»åæ§    è¯è¡šæè§èçåç±»äœç³»äž­ååŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PÅedmÄt a klÃ­ÄovÃ¡ slova</Attribute>
+            <Attribute name="definition">TÃ©ma obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅedmÄt bude typicky vyjÃ¡dÅen pomocÃ­ klÃ­ÄovÃœch slov, klÃ­ÄovÃœch frÃ¡zÃ­ nebo klasifikaÄnÃ­ch znakÅ¯ popisujÃ­cÃ­ch tÃ©ma zdroje. DoporuÄuje se vybÃ­rat hodnotu prvku z ÅÃ­zenÃ©ho slovnÃ­ku nebo formÃ¡lnÃ­ho klasifikaÄnÃ­ho schÃ©matu.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Onderwerp en trefwoorden</Attribute>
+            <Attribute name="definition">Het onderwerp van de Internetbron. Het onderwerp wordt beschreven in korte zinnen of door het gebruik van trefwoorden. Het gebruik van gecontroleerde trefwoorden of classificatie schema&apos;s wordt aanbevolen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Subjecto e parolas-clave</Attribute>
+            <Attribute name="definition">Le topico del contento del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le subjecto es expresse per medio de parolas-clave, phrases-clave, o codices de classification que describe le topico del ressource. Le melior practica recommendate es selectionar un valor a partir de un vocabulario controlate o un schema formal de classification.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà€¿à€·à€¯</Attribute>
+            <Attribute name="definition">à€µà€¿à€·à€¯</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà¥ à€¯à€Ÿà€€à¥à€² à€à€¶à€¯ à€žà¥à€ªà€·à¥à€ à€à€°à€£à€Ÿà€°à¥ à€¶à€¬à¥à€Š, à€µà€¿à€·à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£ à€à€°à€€à€Ÿà€šà€Ÿ à€¶à€¬à¥à€Šà€žà€à€à¥à€°à€¹ à€
+à€¥à€µà€Ÿ à€¶à€¬à¥à€Š à€à¥à€·à€Ÿà€à€Ÿ à€µà€Ÿà€ªà€° à€à€°à€Ÿà€µà€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Description" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">An account of the content of the resource.</Attribute>
+            <Attribute name="comment">Description may include but is not limited to: an abstract, table of contents, reference to a graphical representation of content or a free-text account of the content.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whakaaturanga</Attribute>
+            <Attribute name="definition">He whakaaturanga o nga kaupapa o te rauemi.</Attribute>
+            <Attribute name="comment">Ka uru atu, engari ehara noa ko: te whakarapopototanga korero, te ihirangi, te tohutanga o te kaupapa ma te whakaahua, te whakaaturanga kore kupu ranei, o te kauapapa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">Une description du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Une Description peut contenir, mais ce n&apos;est pas limitatif: un rÃ©sumÃ©, une table des matiÃšres, une rÃ©fÃ©rence Ã  une reprÃ©sentation graphique du contenu, ou un texte libre sur le contenu.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐµ ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐž ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ¿ÐžÑÐ°ÐœÐžÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐŸ (Ð¿ÐŸ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) Ð² Ð²ÐžÐŽÐµ: ÑÐµÑÐµÑÐ°ÑÐ°, ÐŸÐ³Ð»Ð°Ð²Ð»ÐµÐœÐžÑ, ÑÑÑÐ»ÐºÐž ÐœÐ° Ð³ÑÐ°ÑÐžÑÐµÑÐºÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÐžÐ»Ðž Ð¿ÑÐŸÑÑÐŸÐ³ÐŸ ÑÐµÐºÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐžÐ·Ð»ÐŸÐ¶ÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">DescripciÃ³n</Attribute>
+            <Attribute name="definition">Una descripciÃ³n textual del recurso, tal como un resumen en el caso de un documento o una descripciÃ³n del contenido en el caso de un documento visual.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØµÙ</Attribute>
+            <Attribute name="definition">ÙÙ ÙØµÙ ÙØµÙ  ÙÙ
+Ø­ØªÙÙ Ø§ÙÙ
+ØµØ¯Ø±, ÙÙØ¶Ù
+ Ø§ÙÙ
+ÙØ®ØµØ§Øª ÙÙ Ø­Ø§ÙØ© Ø§ÙÙØ«Ø§ØŠÙ Ù
+Ø«Ù Ø§ÙØ§ØŽÙØ§Ø¡ Ø§ÙÙ
+ÙØªÙØšØ© Ø§Ù ØªÙØµÙÙ Ø§ÙÙ
+Ø­ØªÙÙ ÙÙ Ø­Ø§ÙØ© Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+Ø±ØŠÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">è¯Žæ</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çè§£é</Attribute>
+            <Attribute name="comment">è¯Žæå¯ä»¥å
+æ¬äœäžéäºä»¥äžå
+å®¹ïŒææãç®åœãå¯¹ä»¥åŸåœ¢æ¥æ­ç€ºå
+å®¹çèµæºèèšçæå­è¯Žæãæè
+äž    äžªæå
+³èµæºå
+å®¹çèªç±ææ¬æè¿°ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Popis</Attribute>
+            <Attribute name="definition">VysvÄtlenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">Popis mÅ¯ÅŸe obsahovat (mimo jinÃ©): abstrakt, obsah, odkaz na grafickÃ© nebo volnÃ© textovÃ© vyjÃ¡dÅenÃ­ obsahu.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Omschrijving</Attribute>
+            <Attribute name="definition">Een omschrijving van de inhoud van de Internetbron. Abstracts of inhoudsomschrijvingen.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Description</Attribute>
+            <Attribute name="definition">Un description del contento del ressource.</Attribute>
+            <Attribute name="comment">Le description pote includer, ma non se limita a: summario, indice, referentia a un representation graphic del contento o un texto libere a proposito del contento.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà€°à¥à€£à€š</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥, à€à¥à€à€Ÿà€šà€žà€Ÿà€§à€šà€Ÿà€à¥à€µà€°à¥à€£à€š</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€šà€¿à€µà¥à€Šà€š, à€à€²à¥à€, à€à€¿à€€à¥à€°à€°à¥à€ª</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="abstract">
+            <Language code="en">
+                <Attribute name="identifier">Abstract</Attribute>
+                <Attribute name="definition">A summary of the content of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="tableOfContents">
+            <Language code="en">
+                <Attribute name="identifier">Table Of Contents</Attribute>
+                <Attribute name="definition">A list of subunits of the content of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+    </Element>
+    <Element name="Publisher" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Publisher</Attribute>
+            <Attribute name="definition">An entity responsible for making the resource available.</Attribute>
+            <Attribute name="comment">Examples of a Publisher include a person, an organisation, or a service. Typically, the name of a Publisher should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaiwhakaputa</Attribute>
+            <Attribute name="definition">Te putahi nana i whakaputa te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaputa. Ko te tikanga, ka whakamahia te ingoa o te Kaiwhakaputa hei tohu i te putahi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Editeur</Attribute>
+            <Attribute name="definition">L&apos;entitÃ© responsable de la diffusion de la ressource, dans sa forme actuelle, tels, un dÃ©partement universitaire, une entreprise.</Attribute>
+            <Attribute name="comment">Exemples d&apos;Editeurs incluent une personne, une organisation, ou un service. Typiquement, le nom d&apos;une maison d&apos;Ã©dition devrait Ãªtre utilisÃ© ici.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, ÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸÐµ Ð·Ð° Ð²Ð²ÐŸÐŽ ÑÐµÑÑÑÑÐ° Ð² ÐŸÐ±ÑÐ°ÑÐµÐœÐžÐµ.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Editor</Attribute>
+            <Attribute name="definition">&gt;La entidad responsable de hacer que el recurso se encuentre disponible en la red en su formato actual, por ejemplo la empresa editora, un departamento universitario u otro tipo de organizaciÃ³n.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØ§ØŽØ±</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙÙÙØŠØ© Ø§ÙÙ
+Ø³Ø€ÙÙØ© Ø¹Ù ØµÙØ§Ø¹Ø© Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙÙ
+ØªÙÙØ± ÙÙ Ø­Ø§ÙØªÙ Ø§ÙØ­Ø§Ø¶Ø±Ø©, Ù
+Ø«Ù Ø¯Ø§Ø± Ø§ÙÙØŽØ±, ÙØ³Ù
+ Ø¬Ø§Ù
+Ø¹Ù Ø§Ù Ù
+Ø€Ø³Ø³Ø© Ø§Ø¹Ù
+Ø§Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">åºçè
+</Attribute>
+            <Attribute name="definition">äœ¿èµæºæäžºå¯ä»¥è·åŸå¹¶å¯çšçèŽ£ä»»è
+ã</Attribute>
+            <Attribute name="comment">åºçè
+çå®äŸå
+æ¬äžªäœïŒç»ç»ïŒææå¡ãäžè¬èèšïŒåºè¯¥çšåºçè
+çåç§°æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Vydavatel</Attribute>
+            <Attribute name="definition">Entita odpovÄdnÃ¡ za zpÅÃ­stupnÄnÃ­ zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady Vydavatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no vydavatele.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Uitgever</Attribute>
+            <Attribute name="definition">De entiteit die verantwoordelijk is voor het beschikbaarstellen van de Internetbron in de huidig vorm. Bijvoorbeeld een uitgeversbedrijf, een universiteitsafdeling.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Editor</Attribute>
+            <Attribute name="definition">Un entitate responsabile pro disponibilizar le ressource.</Attribute>
+            <Attribute name="comment">Exemplos de editor include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de editor pro indicar le entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€¶à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ  à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.à€ à€ªà¥à€°à€à€Ÿà€¶à€ , à€ªà¥à€°à€à€Ÿà€¶à€š  à€žà€à€žà¥à€¥à€Ÿ, à€žà¥à€µà€Ÿ    à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€ªà¥à€² à€°à¥à€ªà€Ÿà€€  à€
+à€¥à€µà€Ÿ à€à€à€à€°à€šà¥à€ à€µà€° à€à€ªà€²à€¬à¥à€§ à€à€°à¥à€š à€Šà¥à€£à€Ÿà€°à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Contributor" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Contributor</Attribute>
+            <Attribute name="definition">An entity responsible for making contributions to the content of the resource.</Attribute>
+            <Attribute name="comment">Examples of a Contributor include a person, an organisation, or a service. Typically, the name of a Contributor should be used to indicate the entity.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kaiwhakaaro</Attribute>
+            <Attribute name="definition">Te putahi nana, na ratou ranei nga whakakaupapatanga whakaaro o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tangata, he putahitanga, he ratonga ranei etahi tauira Kaiwhakaaro. Ko te tikanga ka whakamahia te ingoa o te Kaiwhakaaro hei tohu i te putahi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Contributeur</Attribute>
+            <Attribute name="definition">Une entitÃ© qui a contribuÃ© Ã  la crÃ©ation du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Exemples de Contributeur incluent une personne, une organisation, ou un service. Typiquement, le nom d&apos;un contributeur devrait Ãªtre utilisÃ© ici pour dÃ©signer l&apos;entitÃ©.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ</Attribute>
+            <Attribute name="definition">ÐÐžÑÐŸ, Ð²ÐœÐµÑÑÐµÐµ Ð²ÐºÐ»Ð°ÐŽ Ð² ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑÐžÐŒÐµÑÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð¿ÐµÑÑÐŸÐœÑ, ÐŸÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ ÐžÐ»Ðž ÑÐ»ÑÐ¶Ð±Ñ. ÐÐ±ÑÑÐœÐŸ ÐžÐŒÑ Ð¡ÐŸÐžÑÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐžÐœÐŽÐžÐºÐ°ÑÐžÐž ÐŸÐ±ÑÐµÐºÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Otros Colaboradores</Attribute>
+            <Attribute name="definition">Una persona u organizacion que haya tenido una contribuciÃ³n intelectual significativa en la creaciÃ³n del recurso pero cuyas contribuciones son secundarias en comparaciÃ³n a las de las personas u organizaciones especificadas en el elementeo Creator (por ejemplo, editor, ilustrador y traductor).</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+Ø³Ø§ÙÙ
+ Ø§ÙØ§Ø®Ø±</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØŽØ®Øµ Ø§Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙØºÙØ± Ù
+Ø°ÙÙØ± ØµØ±Ø§Ø­Ø© ÙÙ Ø¹Ø§Ù
+Ù &quot; Ø§ÙÙ
+ØšØ¯Ø¹ &quot; Ù Ø§ÙØ°Ù ÙØ¯Ù
+ Ù
+Ø³Ø§ÙÙ
+Ø§Øª ÙÙØ±ÙØ© ÙØ§Ù
+Ø© ÙÙÙ
+ØµØ¯Ø±, ÙÙÙ Ù
+Ø³Ø§ÙÙ
+ØªÙØªØšÙÙ    Ø«Ø§ÙÙÙØ© Ù
+ÙØ§Ø±ÙØ© Ù
+Ø¹ Ø§ÙØŽØ®Øµ Ø£Ù Ø§ÙÙ
+Ø€Ø³Ø³Ø© Ø§ÙÙ
+Ø°ÙÙØ±Ø© ÙØµØ§ ÙÙ Ø¹Ø§Ù
+Ù &quot; Ø§ÙÙ
+ÙØŽØŠ &quot; Ù
+Ø«ÙØ§Ù Ù
+ØµØ­Ø­, Ø§ÙÙ
+ØªØ±Ø¬Ù
+, Ø§Ù Ø§ÙÙ
+ÙØ³Ø±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">å
+¶ä»èŽ£ä»»è
+</Attribute>
+            <Attribute name="definition">å¯¹èµæºçå
+å®¹äœåºèŽ¡ç®çå
+¶ä»å®äœã</Attribute>
+            <Attribute name="comment">å
+¶ä»èŽ£ä»»è
+çå®äŸå¯å
+æ¬äžªäººãç»ç»ææé¡¹æå¡ãäžè¬èèšïŒçšå
+¶ä»èŽ£ä»»è
+çåå­æ¥æ è¯è¿äžæ¡ç®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PÅispÄvatel</Attribute>
+            <Attribute name="definition">Entita, kterÃ¡ pÅispÄla k vytvoÅenÃ­ obsahu zdroje.</Attribute>
+            <Attribute name="comment">PÅÃ­klady PÅispÄvatele zahrnujÃ­ osobu, organizaci nebo sluÅŸbu. K oznaÄenÃ­ entity se typicky pouÅŸÃ­vÃ¡ jmÃ©no pÅispÄvatele.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Andere medewerkers</Attribute>
+            <Attribute name="definition">Een persoon of organisatie die een belangrijke bijdrage heeft geleverd, maar die secondair is aan de persoon of organisatie die bij &quot;Creator&quot; genoemd wordt. Gedacht kan worden aan editors, vertalers en illustrators.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Contribuente</Attribute>
+            <Attribute name="definition">Un entitate responsabile pro facer contributiones al contento del ressource.</Attribute>
+            <Attribute name="comment">Exemplos de contribuente include un persona, un organization, o un servicio. Typicamente se utiliza un nomine de contribuente pro indicar le entitate.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier"> à€žà€¹à€¯à¥à€à€, à€žà€¹à€¯à¥à€à¥, à€žà€¹à€Ÿà€¯à¥à€¯à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€žà€à€à€²à¥à€€ à€à€°à€£à¥à¥à€Ÿà€ž  à€®à€Šà€€ à€à€°à€£à€Ÿà€°à¥ à€µà¥à€¯à€à¥à€€à¥</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.à€ à€µà¥à€¯à€à¥à€€à¥, à€žà¥à€µà€Ÿ, à€žà€à€žà¥à€¥à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Date" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Date</Attribute>
+            <Attribute name="definition">A date associated with an event in the life cycle of the resource.</Attribute>
+            <Attribute name="comment">Typically, Date will be associated with the creation or availability of the resource.  Recommended best practice for encoding the date value is defined in a profile of ISO 8601 and follows the YYYY-MM-DD format.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Ra</Attribute>
+            <Attribute name="definition">He ra e whai panga ana ki te huringa ora o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka whai panga te ra ki te hanganga, te wateatanga ranei, o te rauemi. Ko te huarahi tino pai hei tohu i te uara o te ra (date value) kua whakaritea i roto i te whakaritenga IS0 8601 [W3CDTF], a, e whai atu ana ki te whakatakotoranga YYYY-MM-DD.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Date</Attribute>
+            <Attribute name="definition">Une date associÃ©e avec un Ã©vÃ©nement dans le cycle de vie de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, une date sera associÃ©e Ã  la crÃ©ation ou Ã  la publication d&apos;une ressource. Il est fortement recommandÃ© d&apos;encoder la valeur de la date en utilisant le format dÃ©fini par l&apos;ISO 8601 [W3CDTF] sous la forme AAAA-MM-JJ.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐ°ÑÐ°</Attribute>
+            <Attribute name="definition">ÐÐ°ÑÐ°, ÑÐ²ÑÐ·Ð°ÐœÐœÐ°Ñ Ñ ÑÐŸÐ±ÑÑÐžÐµÐŒ Ð² Ð¶ÐžÐ·ÐœÐµÐœÐœÐŸÐŒ ÑÐžÐºÐ»Ðµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÐÐ°ÑÐ° Ð°ÑÑÐŸÑÐžÐžÑÑÐµÑÑÑ Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÐµÐŒ ÐžÐ»Ðž ÐŽÐŸÑÑÑÐ¿ÐœÐŸÑÑÑÑ ÑÐµÑÑÑÑÐ°. ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐŽÐ»Ñ ÐºÐŸÐŽÐžÑÐŸÐ²ÐºÐž Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŽÐ°ÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÐµ Ð¿ÑÐŸÑÐžÐ»Ñ ISO 8601 (Ð€ÐŸÑÐŒÐ°ÑÑ ÐŽÐ°ÑÑ Ðž Ð²ÑÐµÐŒÐµÐœÐž W3C), ÐºÐŸÑÐŸÑÐŸÐµ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ ÑÐŸÑÐŒÐ°Ñ ÐÐÐÐ-ÐÐ-ÐÐ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Fecha</Attribute>
+            <Attribute name="definition">Una fecha en la que el recurso se puso a disposiciÃ³n del usuario en su forma actual. Esta fecha no ha de confundirse con la que pertenece al elemento Coverage, que serÃ­a asociada con el recurso sÃ³lo en la medida en que el contenido intelectual estÃ¡ de algÃºn modo relacionado con esa fecha. Recomendamos la utiliazaciÃ³n de uno de los formatos definidos en el documento &quot;Date and Time Formats&quot;, http://www.w3.org/TR/NOTE-datetime basado en la norma ISO 8601 que incluye, entre otras, fechas en el formato AAAA y AAAA-MM-DD. De esta forma la fecha 1994-11-05 corresponderÃ­a al 5 de Noviembre de 1994.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØªØ§Ø±Ø®</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙÙ
+ØªØ¹ÙÙ ØšÙÙØª ÙØŽØŠ Ø£Ù ØªÙÙØ± Ø§ÙÙ
+ØµØ¯Ø± . Ù ÙØ¬Øš Ø£Ù ÙØ§ ÙØ®ÙØ· ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ® Ø§ÙØ¹Ø§ØŠØ¯ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØºØ·ÙØ© , Ù Ø§ÙØ° Ù ÙØ¬Øš Ø£Ù    ÙÙØ­Ù ØšØ§ÙÙ
+ØµØ¯Ø± ÙÙØ· Ù
+Ù ÙØ§Ø­ÙØ© Ø§ÙÙ
+Ø­ØªÙÙØ§ÙÙÙØ±Ù ÙÙÙÙÙ ÙÙØ¹Ø§ Ù
+Ø§ ÙÙ ÙØ°Ø§ Ø§ÙØªØ§Ø±ÙØ®. Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø³Ø«Ø­Ø³ÙØ© ÙÙØ§Ø­Ø³Ù Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ    Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙ ÙÙ Ø§ÙØµÙØºØ© Ø§ÙÙ
+Ø¹ØªÙ
+Ø¯Ø© ÙÙ ØªØŽÙÙÙ (ISO 8601) ØµÙØºØ© ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601) Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© http://www.w3.org/TR/NOTE-datetime, W3C. ÙÙÙ    ÙØ°Ø§ Ø§ÙÙØ³Ù , Ù
+Ø«ÙØ§, ØªØ§Ø±ÙØ® Ø§ÙÙÙÙ
+ 1994-11-04 ÙÙØ§ØžØ± ÙÙÙ
+ 5 ÙÙÙÙ
+ØšØ± 1994</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ¥æ</Attribute>
+            <Attribute name="definition">äžèµæºçåœåšæäž­çäžäžªäºä»¶çžå
+³çæ¶éŽã</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæ¥æåºäžèµæºçåå»ºæåºçæ¥æçžå
+³ãå»ºè®®éçšçæ¥ææ ŒåŒåºç¬ŠåISO 8601 [W3CDTF]è§    èïŒå¹¶äœ¿çšYYYY-MM-DDçæ ŒåŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Datum</Attribute>
+            <Attribute name="definition">Datum spojenÃ© s urÄitou udÃ¡lostÃ­ bÄhem existence zdroje.</Attribute>
+            <Attribute name="comment">Datum bude typicky spojeno s vytvoÅenÃ­m nebo zpÅÃ­stupnÄnÃ­m zdroje. DoporuÄenÃœ postup pro kÃ³dovÃ¡nÃ­ hodnoty data je definovÃ¡n v ISO 8601 [W3CDTF]: formÃ¡t RRRR-MM-DD.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Datum</Attribute>
+            <Attribute name="definition">De datum van de totstandkoming of beschikbaarstelling van een Internetbron. Deze datum moet niet verward worden met de datum in &quot;Coverage&quot;. De datum in &quot;Coverage&quot; geeft de tijdsperiode van de inhoud van de Internetbron aan. Voorgeschreven wordt de ISO 8601 standaard voor datum, zoals beschreven in W3C Technical Note http://www.w3.org/TR/NOTE-datetime. Hierin wordt het formaat YYYY-MM-DD beschreven. Het formaat YYYY-MM-DD wordt de voorgeschreven standaard. 3 September 1998 wordt in dit geval 1998-09-03.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Data</Attribute>
+            <Attribute name="definition">Un data associate a un evento in le cyclo vital del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le data es associate al creation o disponibilization del ressource. Le practica recommendate pro codificar le valor del datas es definite in un profilo de ISO 8601 [W3CDTF] e seque le formato AAAA-MM-DD.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€€à€Ÿà€°à¥à€,  à€Šà€¿à€šà€Ÿà€à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ à€€à€¯à€Ÿà€° à€à¥à€²à¥à€¯à€Ÿà€à¥ à€€à€Ÿà€°à¥à€</Attribute>
+            <Attribute name="comment">à€€à€Ÿà€°à¥à€  à€²à€¿à€¹à¥à€£à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª à€ à€µà€°à¥à€·. à€®à€¹à€¿à€šà€Ÿ. à€Šà€¿à€µà€ž à¥§à¥¯à¥«à¥Š. à¥Šà¥ª. à¥Šà¥ª</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="available">
+            <Language code="en">
+                <Attribute name="identifier">Available</Attribute>
+                <Attribute name="definition">Date (often a range) that the resource will become or did become available.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="created">
+            <Language code="en">
+                <Attribute name="identifier">Created</Attribute>
+                <Attribute name="definition">Date of creation of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="dateAccepted">
+            <Language code="en">
+                <Attribute name="identifier">Date Accepted</Attribute>
+                <Attribute name="definition">Date of acceptance of the resource (e.g. of thesis by university department, of article by journal, etc.).</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+        <Element name="dateCopyrighted">
+            <Language code="en">
+                <Attribute name="identifier">Date Copyrighted</Attribute>
+                <Attribute name="definition">Date of a statement of copyright.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+        <Element name="dateSubmitted">
+            <Language code="en">
+                <Attribute name="identifier">Date Submitted</Attribute>
+                <Attribute name="definition">Date of submission of the resource (e.g. thesis, articles, etc.).</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+        <Element name="issued">
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Issued</Attribute>
+                <Attribute name="definition">Date of formal issuance (e.g., publication) of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="modified">
+            <Language code="en">
+                <Attribute name="identifier">Modified</Attribute>
+                <Attribute name="definition">Date on which the resource was changed.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="valid">
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Valid</Attribute>
+                <Attribute name="definition">Date (often a range) of validity of a resource.</Attribute>
+            </Language>
+        </Element>
+    </Element>
+    <Element name="Type" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Resource Type</Attribute>
+            <Attribute name="definition">The nature or genre of the content of the resource.</Attribute>
+            <Attribute name="comment">Type includes terms describing general categories, functions, genres, or aggregation levels for content. Recommended best practice is to select a value from a controlled vocabulary (for example, the working draft list of Dublin Core Types). To describe the physical or digital manifestation of the resource, use the FORMAT element.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Tumomo</Attribute>
+            <Attribute name="definition">Te ahua, te tu-ahuatanga ranei o nga kaupapa o te rauemi.</Attribute>
+            <Attribute name="comment">Kei roto i te Tumomo nga kupu whakaatu i nga wehewehenga whanui, nga aheinga, nga tu-ahuatanga, te whakaritenga ranei o nga taumata o te/ nga kaupapa. Ko te huarahi tino pai kua tohua, ko te tipako i tetahi uara mai i nga kupu kua whakaritea (hei tauira, ko te tauira rarangi e whakamahia ana o nga Dublin Core Types [DCT1]. Hei whakaatu i te whakatinanatanga o te rauemi, whakamahia ko te huanga WHAKATAKOTORANGA (FORMAT).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Type</Attribute>
+            <Attribute name="definition">La nature ou le genre du contenu de la ressource.</Attribute>
+            <Attribute name="comment">Type inclut des termes dÃ©crivant des catÃ©gories, fonctions ou genres gÃ©nÃ©raux pour le contenu, ou des niveaux d&apos;agrÃ©gation. Il est recommandÃ© de choisir la valeur du type dans une liste de vocabulaire contrÃŽlÃ© (par exemple, la liste provisoire de Types du Dublin Core[DCT1]). Pour dÃ©crire la matÃ©rialisation physique ou digitale de la ressource, il faut utiliser l&apos;Ã©lÃ©ment Format.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¢ÐžÐ¿ ÑÐµÑÑÑÑÐ°</Attribute>
+            <Attribute name="definition">ÐÑÐžÑÐŸÐŽÐ° ÐžÐ»Ðž Ð¶Ð°ÐœÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">Ð¢ÐžÐ¿ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐµÑÐŒÐžÐœÑ, ÐŸÐ¿ÐžÑÑÐ²Ð°ÑÑÐžÐµ ÐŸÐ±ÑÐžÐµ ÐºÐ°ÑÐµÐ³ÐŸÑÐžÐž, ÑÑÐœÐºÑÐžÐž, Ð¶Ð°ÐœÑÑ ÐžÐ»Ðž ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÐœÑÐµ ÑÑÐŸÐ²ÐœÐž ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ°Ð±ÐŸÑÐµÐ³ÐŸ Ð¿ÑÐŸÐµÐºÑÐ° ÑÐ¿ÐžÑÐºÐ° Ð¢ÐžÐ¿ÐŸÐ² Dublin Core DCT1). ÐÐ»Ñ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ ÑÐžÐ·ÐžÑÐµÑ-ÐºÐŸÐ³ÐŸ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÑÐµÑÑÑÑÐ° ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÑÐ»ÐµÐŒÐµÐœÑ Ð€ÐŸÑÐŒÐ°Ñ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Tipo del Recurso</Attribute>
+            <Attribute name="definition">La categorÃ­a del recurso, por ejemplo pÃ¡gina personal, romance, poema, minuta, diccionario. Para asegurar la interoperabilidad, Type deberÃ­a ser seleccionado de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo. En http://sunsite.berkeley.edu/Metadata/types.html se puede consultar el estado actual de la discusiÃ³n en torno a este tema.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">ÙÙØ¹ Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">ÙÙ ÙØŠØ© Ø§ÙÙ
+ØµØ¯Ø± , Ù
+Ø«ÙØ§ Ø§ÙØµÙØ­Ø© Ø§ÙØ§Ù
+, Ø±ÙØ§ÙØ© , ÙØ·Ø¹Ø© ØŽØ¹Ø±ÙØ© , ÙØ±ÙØ© Ø¹Ù
+Ù , ØªÙØ±ÙØ± ÙÙÙ , Ù
+ÙØ§Ù , ÙØ§Ù
+ÙØ³ . Ù Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù    ÙØ¬Øš Ø£Ù ÙØ®ØªØ§Ø± Ø§ÙÙÙØ¹ Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© ÙØ§ÙØªÙ ÙÙÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºç±»å</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹çç¹åŸæç±»åã</Attribute>
+            <Attribute name="comment">èµæºç±»åå
+æ¬æè¿°èµæºå
+å®¹çäžè¬èçŽïŒåèœïŒç§å±ïŒæèç±»å±æ¬¡çæ¯è¯­ãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒäŸåŠDCMIç±»åè¯æ±è¡š[DCMITYPE]ïŒãèŠæè¿°èµæºçç©çææ°å­åè¡šç°åœ¢åŒïŒè¯·äœ¿çšâæ ŒåŒïŒFORMATïŒâå
+çŽ ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Typ zdroje</Attribute>
+            <Attribute name="definition">Povaha nebo druh obsahu zdroje.</Attribute>
+            <Attribute name="comment">Typ obsahuje termÃ­ny popisujÃ­cÃ­ obecnÃ© kategorie, funkce, druhy nebo agregaÄnÃ­ ÃºrovnÄ obsahu. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. pracovnÃ­ nÃ¡vrh seznamu TypÅ¯ Dublin Core [DCT1]). K popisu fyzickÃ© nebo digitÃ¡lnÃ­ reprezentace zdroje uÅŸijte prvek FormÃ¡t.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bestands type</Attribute>
+            <Attribute name="definition">Het soort Internetbron, bijvoorbeeld home page, gedicht, working paper, technical report, essay, woordenboek etc. De DC working groups zijn momenteel bezig met twee voorstellen; een minimalist draft: http://sunsite.berkeley.edu/Metadata/minimalist.html en een structuralist draft: http://sunsite.berkely.edu/Metadata/structuralist.html. Het wordt sterk aanbevolen om &quot;Type&quot; van deze lijsten te selecteren.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Typo del ressource</Attribute>
+            <Attribute name="definition">Le natura o genere del contento del ressource.</Attribute>
+            <Attribute name="comment">Le typo del ressource include terminos que describe categorias general, functiones, generes, o nivellos de aggregation pro le contento. Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le lista de schizzos de travalio de typos de Dublin Core [DCT1]). Pro describer le manifestation physic o digital del ressource, utiliza le elemento Formato.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€ªà¥à€°à€à€Ÿà€°</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿà€ªà¥à€°à€à€Ÿà€°</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€µà€°à¥à€à¥à€à€°à€£, à€µà€°à¥à€à€µà€Ÿà€°à¥à€®à€§à¥à€¯à¥ à€à€à€Ÿà€Šà¥à€¯à€Ÿ à€šà¥à€à€Šà¥à€à€Ÿ à€®à¥à€à¥à€¯ à€à€à€ªà¥à€°à€à€Ÿà€°</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Format" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">The physical or digital manifestation of the resource.</Attribute>
+            <Attribute name="comment">Typically, Format may include the media-type or dimensions of the resource. Format may be used to determine the software, hardware or other equipment needed to display or operate the resource. Examples of dimensions include size and duration. Recommended best practice is to select a value from a controlled vocabulary (for example, the list of Internet Media Types [MIME] defining computer media formats).</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whakatakotoranga</Attribute>
+            <Attribute name="definition">Te whakatinanatanga, te whakaputanga a-mati o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka taea te Whakatakotoranga te whakamahi hei whakarite i te rawa-ngohe (software), te rawa-maro (hardware), ma etahi taputapu ranei me matua whiwhi, hei whakahaere i te rauemi. Ko te rahi me te roanga etahi tauira hanga. Ko te tino huarahi kua tohua ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, te rarangi o nga Tumomo Papaho Ipurangi (Internet Media Types [MIME]) e whakaatu ana i nga whakatakotoranga papaho rorohiko.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">La matÃ©rialisation physique ou digitale de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, Format peut inclure le media ou les dimensions de la ressource. Format peut Ãªtre utilisÃ© pour prÃ©ciser le logiciel, le matÃ©riel ou autre Ã©quipement nÃ©cessaire pour afficher ou faire fonctionner la ressource. Exemples de dimensions incluent la taille et la durÃ©e. Il est recommandÃ© de choisir la valeur du format dans une liste de vocabulaire contrÃŽlÃ©(par exemple, la liste des types de media dÃ©finis sur Internet [MIME]).</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð€ÐŸÑÐŒÐ°Ñ</Attribute>
+            <Attribute name="definition">Ð€ÐžÐ·ÐžÑÐµÑÐºÐŸÐµ ÐžÐ»Ðž ÑÐžÑÑÐŸÐ²ÐŸÐµ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐžÐ¿ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž (ÐŒÐµÐŽÐžÐ°-ÑÐžÐ¿) ÐžÐ»Ðž Ð²ÐµÐ»ÐžÑÐžÐœÑ ÑÐµÑÑÑÑÐ°. Ð€ÐŸÑÐŒÐ°Ñ ÐŒÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐŽÐ»Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð°Ð¿Ð¿Ð°ÑÐ°ÑÐœÐŸÐ³ÐŸ, Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐœÐŸÐ³ÐŸ ÐŸÐ±ÐµÑÐ¿ÐµÑÐµÐœÐžÑ ÐžÐ»Ðž ÐŽÑÑÐ³ÐŸÐ³ÐŸ ÐŸÐ±ÐŸÑÑÐŽÐŸÐ²Ð°ÐœÐžÑ, ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÐ³ÐŸ ÐŽÐ»Ñ Ð²ÐŸÑÐ¿ÑÐŸÐžÐ·Ð²ÐµÐŽÐµÐœÐžÑ ÐžÐ»Ðž ÑÐ°Ð±ÐŸÑÑ Ñ ÑÐµÑÑÑÑÐŸÐŒ. ÐÑÐžÐŒÐµÑÑ Ð²ÐµÐ»ÐžÑÐžÐœÑ Ð²ÐºÐ»ÑÑÐ°ÑÑ ÑÐ°Ð·ÐŒÐµÑ Ðž Ð¿ÑÐŸÐŽÐŸÐ»Ð¶ÐžÑÐµÐ»ÑÐœÐŸÑÑÑ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐ¿ÐžÑÐºÐ° ÐÐµÐŽÐžÐ°-ÑÐžÐ¿ÐŸÐ² ÐÐœÑÐµÑÐœÐµÑ MIME, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑÐµÐ³ÐŸ ÐºÐŸÐŒÐ¿ÑÑÑÐµÑÐœÑÐµ ÑÐŸÑÐŒÐ°ÑÑ Ð°ÑÐŽÐžÐŸÐ²ÐžÐ·ÑÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐž).</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Formato</Attribute>
+            <Attribute name="definition">El formato de datos de un recurso, usado para identificar el software y posiblemente, el hardware que se necesitarÃ­a para mostrar el recurso. Para asegurar la interoperabilidad, los valores de Format deberÃ­an ser seleccionados de entre una lista de valores que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØµÙØºØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ù ØµÙØºØ© Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø± , Ù ØªØ³ØªØ®Ø¯Ù
+ ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙØšØ±Ù
+Ø¬ÙØ§Øª ÙØ±ØšÙ
+Ø§ Ø§ÙØµÙØš ÙÙØ­Ø§Ø³ÙØš Ù Ø§ÙØªÙ ÙÙ
+ÙÙ Ø£Ù ÙØ­ØªØ§Ø¬    ÙÙØ§ ÙØ¹Ø±Ø¶ Ø£Ù ØªØŽØºÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù, ÙØ¬Øš Ø£Ù ØªØ®ØªØ§Ø± Ø§ÙØµÙØºØ© Ù
+Ù Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§    Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ ŒåŒ</Attribute>
+            <Attribute name="definition">èµæºçç©çææ°å­è¡šç°åœ¢åŒã</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæ ŒåŒå¯èœå
+æ¬èµæºçåªäœç±»åæèµæºçå€§å°ïŒæ ŒåŒå
+çŽ å¯ä»¥çšæ¥å³å®å±ç€ºææäœèµæºæé    çèœ¯ç¡¬ä»¶æå
+¶ä»çžåºè®Ÿå€ïŒäŸåŠå€§å°å
+æ¬èµæºæå çå­åšç©ºéŽåæç»­æ¶éŽãå»ºè®®éçšæ¥èªäºåæ§è¯è¡š    äž­çåŒïŒïŒäŸåŠInternetåªäœç±»å[MIME]å®ä¹è®¡ç®æºåªäœæ ŒåŒïŒã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">FormÃ¡t</Attribute>
+            <Attribute name="definition">FyzickÃ¡ nebo digitÃ¡lnÃ­ reprezentace zdroje.</Attribute>
+            <Attribute name="comment">Typicky mÅ¯ÅŸe FormÃ¡t obsahovat typ mÃ©dia nebo rozmÄry zdroje. FormÃ¡t lze pouÅŸÃ­vat k urÄenÃ­ softwaru, hardwaru nebo dalÅ¡Ã­ho zaÅÃ­zenÃ­ potÅebnÃ©ho k zobrazenÃ­ nebo k manipulaci se zdrojem. PÅÃ­klady rozmÄrÅ¯ zahrnujÃ­ velikost a dobu trvÃ¡nÃ­. DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. seznam typÅ¯ internetovÃœch mÃ©diÃ­ [MIME] definujÃ­cÃ­ formÃ¡ty poÄÃ­taÄovÃœch mÃ©diÃ­).</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Format</Attribute>
+            <Attribute name="definition">Het data formaat van het Internetbestand. Dit gegeven kan gebruikt worden om systeemvereisten (hard- en software) te identificeren die nodig zijn voor raadpleging of bediening van de Internetbron. &quot;Format&quot; is nog in ontwikkeling bij de DC workshop. Uit compatibiliteitsoverwegingen wordt aanbevolen formaatgegevens te selecteren uit een gecontroleerde lijst. Formaten kunnen van de Internet Media Types (MIME types) lijst gekozen worden: http://www.isi.edu/in-notes/iana/assignments/media-types/media-types</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Formato</Attribute>
+            <Attribute name="definition">Le manifestation physic o digital del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le formato pote includer le typo de medio o dimensiones del ressource. Le formato pote esser utilizate pro determinar le software o le equipamento necessari pro exhibir o operar le ressource. Exemplos de dimensiones include grandessa e duration. Le practica recommendate es selectionar un valor de un vocabulario controlate (per exemplo, le lista de typos de medios de Internet [MIME] que defini formatos de medios).</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€°à€à€šà€Ÿ</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€°à€à€šà€Ÿ, à€šà€®à¥à€šà€Ÿà€¬à¥à€Š à€°à€à€šà€Ÿ</Attribute>
+            <Attribute name="comment">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€žà¥à€µà€°à¥à€ª, à€°à€à€šà€Ÿà€à€Šà€Ÿ. à€€à¥à€¯à€Ÿà€à¥ à€°à¥à€ª à€à€žà¥ à€ªà¥à€žà¥à€€à€à€°à¥à€ª, à€žà€à€à€£à€à¥à€¯ à€°à¥à€ª, à€µà€¿à€Šà¥à€¯à¥à€€ à€ªà€°à€®à€Ÿà€£à¥à€à€à¥à€¯à€Ÿ à€žà€¹à€Ÿà€¯à¥à€¯à€Ÿà€šà¥ à€žà€à€à€£à€à¥à€à¥à€€ à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="extent">
+            <Language code="en">
+                <Attribute name="identifier">Extent</Attribute>
+                <Attribute name="definition">The size or duration of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="medium">
+            <Language code="en">
+                <Attribute name="identifier">Medium</Attribute>
+                <Attribute name="definition">The material or physical carrier of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+    </Element>
+    <Element name="Identifier" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Resource Identifier</Attribute>
+            <Attribute name="definition">An unambiguous reference to the resource within a given context.</Attribute>
+            <Attribute name="comment">Recommended best practice is to identify the resource by means of a string or number conforming to a formal identification system. Example formal identification systems include the Uniform Resource Identifier (URI) (including the Uniform Resource Locator (URL)), the Digital Object Identifier (DOI) and the International Standard Book Number (ISBN).</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Taututanga</Attribute>
+            <Attribute name="definition">He tohutoro e hangai ana ki te rauemi i roto i tetahi horopaki.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tautu i te rauemi ma tetahi aho (string), tau ranei, e hangai ana ki tetahi punaha taututanga okawa. Ko etahi punaha taututanga okawa ko te Taututanga Rauemi Orite (Uniform Resource Identifier - URI), te Taututanga Rauemi Tohu-wahi (Uniform Resource Locator - URL), Te Mea A-mati (Digital Object Identifier - DOI) me te Tau Pukapuka Aro Whanui o te Ao (International Standard Book Number - ISBN).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Identifiant</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence non ambiguÃ« Ã  la ressource dans un contexte donnÃ©</Attribute>
+            <Attribute name="comment">Il est recommandÃ© d&apos;identifier la ressource par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un sytÃšme formel d&apos;identification. Exemples de systemes formels d&apos;identification incluent le &quot;Uniform Resource Identifier&quot; (URI) (qui inclut le &quot;Uniform Resource Locator&quot; (URL)), le &quot;Digital Object Identifier&quot;(DOI) et le &quot;International Standard Book Number&quot;(ISBN).</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÑÐµÑÑÑÑÐ°</Attribute>
+            <Attribute name="definition">ÐÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÐ°Ñ ÑÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐºÐŸÐœÑÐµÐºÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ. ÐÑÐžÐŒÐµÑÑ ÑÐŸÑÐŒÐ°Ð»ÑÐœÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÑ
+ ÑÐžÑÑÐµÐŒ Ð²ÐºÐ»ÑÑÐ°ÑÑ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° (URI), (Ð²ÐºÐ»ÑÑÐ°Ñ Ð£ÐœÐžÑÐžÑÐžÑÐŸÐ²Ð°ÐœÐœÑÑÐ¹ ÐÐŸÐºÐ°ÑÐŸÑ Ð ÐµÑÑÑÑÐ° URL), ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐŠÐžÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐÐ±ÑÐµÐºÑÐ° (DOI) Ðž ÐÐµÐ¶ÐŽÑÐœÐ°ÑÐŸÐŽÐœÑÐ¹ Ð¡ÑÐ°ÐœÐŽÐ°ÑÑÐœÑÐ¹ ÐÐœÐžÐ¶ÐœÑÐ¹ ÐÐŸÐŒÐµÑ (ISBN).</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Identificador del Recurso</Attribute>
+            <Attribute name="definition">Secuencia de caracteres usados para identificar unÃ­vocamente un recurso. Ejemplos para recursos en lÃ­nea pueden ser URLs y URNs (cuando estÃ©n implementados). Para otros recursos pueden ser usados otros formatos de identificadores, como por ejemplo ISBN (&quot;International Standard Book Number&quot; - NÃºmero Internacional Normalizado para Libros)</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ù
+Ø¹Ø±Ù Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">ÙÙ Ù
+Ø¬Ù
+ÙØ¹Ø© Ø­Ø±ÙÙØ© Ø£Ù Ø±ÙÙ
+ ÙØ³ØªØ®Ø¯Ù
+ ÙØªØ¹Ø±ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§Ø­Ø¯ÙØ§ : Ø§ÙØ§Ù
+Ø«ÙØ© ÙØ°Ø§ÙÙ ÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙØŽØšÙÙØ©ØªØ¶Ù
+ ) URNs, URLs Ø¹ÙØ¯ ØšÙØ§ØŠÙØ§ (    ÙØ°ÙÙ ÙÙ
+ÙÙ Ø§Ø¹ØªØšØ§Ø± Ø§ÙÙ
+Ø¹Ø±ÙØ§Øª Ø§ÙØ¹Ø§ÙÙ
+ÙØ© Ø§ÙØ§Ø­Ø§Ø¯ÙØ© Ù
+Ø«Ù Ø§ÙØ±ÙÙ
+ Ø§ÙØ¯ÙÙÙ ÙØªØµÙÙÙ Ø§ÙÙØ§ØªØš (ISBN) Ù
+Ø±ØŽØ­Ø© ÙÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èµæºæ è¯ç¬Š</Attribute>
+            <Attribute name="definition">åšç¹å®çèåŽå
+ç»äºèµæºçäžäžªæç¡®çæ è¯ã</Attribute>
+            <Attribute name="comment">å»ºè®®å¯¹èµæºçæ è¯éçšç¬Šåæäžæ­£åŒæ è¯äœç³»çå­ç¬Šäž²åæ°å­ç»åãäŸåŠæ­£åŒçæ è¯äœç³»å
+æ¬ç»äžèµ    æºæ è¯ç¬Š(URI) (å
+æ¬åäžèµæºå®äœç¬ŠURL)ãæ°å­å¯¹è±¡æ è¯ç¬Š(DOI)ååœé
+æ åä¹Šå·(ISBN)ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">IdentifikÃ¡tor zdroje</Attribute>
+            <Attribute name="definition">JednoznaÄnÃœ odkaz na zdroj v rÃ¡mci danÃ©ho kontextu.</Attribute>
+            <Attribute name="comment">DoporuÄuje se identifikovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯. PÅÃ­klady formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯ zahrnujÃ­ Uniform Resource Identifier (URI) (vÄetnÄ Uniform Resource Locator (URL)), Digital Object Identifier (DOI) a MezinÃ¡rodnÃ­ standardnÃ­ ÄÃ­slo knihy (ISBN).</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bestandsidentificatie</Attribute>
+            <Attribute name="definition">Een unieke string of nummer ter identificatie van de Internetbron. Voorbeelden zijn URL&apos;s of URN&apos;s Andere voorbeelden zijn International Standard Book Number (ISBN) of International Standard Serial Number (ISSN)</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Identificator del ressource</Attribute>
+            <Attribute name="definition">Un referentia inambigue al ressource intra un date contexto.</Attribute>
+            <Attribute name="comment">Se recommenda identificar le ressource per medio de un catena de characteres o numero conforme a un systema formal de identification. Exemplos de systemas formal de identification include le Uniform Resource Identifier (URI) (includente le Uniform Resource Locator (URL)), le Digital Object Identifier (DOI) e le International Standard Book Number (ISBN).</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€à€³à€</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à€Ÿ à€à€ à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ, à€µà€¿à€¶à€¿à€·à¥à€  à€žà¥à€¥à€³,  à€ªà¥à€žà¥à€€à€à€Ÿà€à€Ÿ à€à€à€€à€°à€°à€Ÿà€·à¥à€à¥à€°à¥à€¯  à€®à€Ÿà€šà€ à€à¥à€°à€®à€Ÿà€à€,  à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€à€³à€ à€ªà€à€µà€¿à€£à€Ÿà€°à€Ÿ à€à€à€à€°à€šà¥à€ à€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="bibliographicCitation">
+            <Attribute language="en" name="status">conforming</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Bibliographic Citation</Attribute>
+                <Attribute name="definition">A bibliographic reference for the resource. </Attribute>
+                <Attribute name="comment">Recommended practice is to include sufficient bibliographic detail to identify the resource as unambiguously as possible, whether or not the citation is in a standard form.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+    </Element>
+    <Element name="Source" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Source</Attribute>
+            <Attribute name="definition">A Reference to a resource from which the present resource is derived.</Attribute>
+            <Attribute name="comment">The present resource may be derived from the Source resource in whole or in part.  Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Matapuna</Attribute>
+            <Attribute name="definition">He tohutoro ki tetahi rauemi koira te matapuna o te rauemi e tirohia ana.</Attribute>
+            <Attribute name="comment">Tera pea i puta te katoa, tetahi wahanga ranei o te rauemi, e tirohia ana, mai i te Matapuna rauemi. Ko te tino huarahi kua tohua ko te tohutoro ki te rauemi ma te aho, ma te tau ranei e hangai ana ki tetahi punaha taututanga okawa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Source</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence Ã  une ressource Ã  partir de laquelle la ressource actuelle a Ã©tÃ© dÃ©rivÃ©e.</Attribute>
+            <Attribute name="comment">La ressource actuelle peut avoir Ã©tÃ© dÃ©rivÃ©e d&apos;une autre ressource source, en totalitÃ© ou en partie. Il est recommandÃ© de refÃ©rencer cette source par une chaÃ®ne de caractÃšre ou un nombre conforme Ã  un systÃšme formel d&apos;identification.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÑÐŸÑÐœÐžÐº</Attribute>
+            <Attribute name="definition">Ð¡ÑÑÐ»ÐºÐ° ÐœÐ° ÑÐµÑÑÑÑ, ÐžÐ· ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐœÐ°ÑÑÐŸÑÑÐžÐ¹.</Attribute>
+            <Attribute name="comment">ÐÐ°ÑÑÐŸÑÑÐžÐ¹ ÑÐµÑÑÑÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐžÐ· ÐÑÑÐŸÑÐœÐžÐºÐ° ÑÐµÐ»ÐžÐºÐŸÐŒ ÐžÐ»Ðž ÑÐ°ÑÑÐžÑÐœÐŸ. ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Fuente</Attribute>
+            <Attribute name="definition">Secuencia de caracteres utilizado para identificar unÃ­vocamente un trabajo a partir del cual proviene el recurso actual. Por ejemplo, es posible usar Source con la fecha de 1603 como descripciÃ³n de una pelÃ­cula basada en una obra de Shakespeare, pero es preferible, en ese caso, usar Relation &quot;IsBasedOn&quot; con una referencia a un recurso distinto cuya descripciÃ³n contenga el elemento Date con valor 1603.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙ
+ØµØ¯Ø±</Attribute>
+            <Attribute name="definition">Ø§ÙÙ
+Ø¹ÙÙÙ
+Ø§Øª Ø§ÙØ®Ø§ØµØ© ØšÙ
+ØµØ¯Ø± Ø«Ø§ÙÙ Ù Ø§ÙØ°Ù Ø§Ø³ØªÙ
+Ø¯ Ù
+ÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ. Ù
+Ø¹ Ø£ÙÙ Ù
+Ù Ø§ÙÙ
+ÙØµÙ ØšÙ Ø£Ù ØªÙØ¯Ù
+ Ø§ÙØ¹ÙØ§Ù
+Ù    Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ ÙÙØ· , ÙÙÙ
+ÙÙ Ø£Ù ÙØ­ÙÙ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ®, Ø§ÙÙ
+ÙØŽØŠ , Ø§ÙØµÙØºØ© , Ø§ÙÙ
+Ø¹Ø±Ù , Ø§Ù Ø§ÙØ© Ù
+ÙØ© - Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø§Ø®Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ø¹ÙØ¯Ù
+Ø§ ØªØ¹ØªØšØ± Ù
+ÙÙ
+Ø© ÙØ§ÙØªØŽØ§Ù Ø§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ , Ù ÙØ°Ø§ÙÙ ÙÙØµÙ Ø¹ÙØ¯ Ø§ÙØªØ·ØšÙÙÙÙ Ø§Ø­Ø³Ù     Ø§ÙÙ
+ØªØ¹Ø§Ø±Ù Ø¹ÙÙÙ ØšØ§Ø³ØªØ®Ø¯Ø§Ù
+ Ø¹Ø§Ù
+Ù Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æ¥æº</Attribute>
+            <Attribute name="definition">ç°æèµæºæ¥æºçåç
+§ã</Attribute>
+            <Attribute name="comment">åœåèµæºå¯èœéšåæå
+šéšæºèªè¯¥å
+çŽ ææ è¯çèµæºïŒå»ºè®®å¯¹è¿äžèµæºçæ è¯éçšäžäžªç¬Šåæ­£åŒæ è¯ç³»    ç»çå­äž²åæ°å­ç»å ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Zdroj</Attribute>
+            <Attribute name="definition">Odkaz na zdroj, z nÄhoÅŸ je popisovanÃœ zdroj odvozen.</Attribute>
+            <Attribute name="comment">PopisovanÃœ zdroj mÅ¯ÅŸe bÃœt zcela nebo zÄÃ¡sti odvozen ze zdroje uvedenÃ©ho v prvku Zdroj. DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Bron</Attribute>
+            <Attribute name="definition">Informatie over een tweede bron waar de huidige Internetbron is van afgeleid. Bij &quot;Source&quot; mag een datum, auteur of maker, formaat, identificatie of andere metadata voor de tweede bron worden ingevuld. Voorbeeld: het is mogelijk om een &quot;Source&quot; datum 1603 in een omschrijving van een 1996 film bewerking van een Shakespeare stuk te gebruiken. &quot;Source&quot; is niet bruikbaar wanneer de huidige bron de orginele is.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Fonte</Attribute>
+            <Attribute name="definition">Un referentia a un ressource del qual se deriva le ressource presente.</Attribute>
+            <Attribute name="comment">Le ressource presente pote derivar se del ressource-fonte in toto o in parte. Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o un numero conforme a un systema formal de identification.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€à€à€®</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿ à€à€à€® à€žà€Ÿà€à€à€£à€Ÿà€°à€Ÿ à€à€à€
+à€°à¥à€¥à¥ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€ªà¥à€žà¥à€€à€, à€®à€Ÿà€žà€¿à€,  à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿà€¯à€Ÿà€à€à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Language" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Language</Attribute>
+            <Attribute name="definition">A language of the intellectual content of the resource.</Attribute>
+            <Attribute language="en" name="comment">Recommended best practice for the values of the Language element is defined by RFC 1766 which includes a two-letter Language Code (taken from the ISO 639 standard, followed optionally, by a two-letter Country Code (taken from the ISO 3166 standard). For example, &apos;en&apos; for English, &apos;fr&apos; for French, or &apos;en-uk&apos; for English used in the United Kingdom.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Reo</Attribute>
+            <Attribute name="definition">Te reo o te whakakaupapatanga o nga matauranga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua mo nga uara o te huanga Reo kua whakaritea ki te RFC 1766 [RFC1766] kei roto te Tohu Reo kupu-rua (two-letter Language Code) (no te aro whanui ISO 639 [ISO639], ka taea e tetahi tohu kupu-rua te whai atu (no te aro whanui ISO 3166) [ISO366]). Hei tauira, te &apos;en&apos; mo te English, te &apos;fr&apos; mo te French, &apos;te en-uk&apos; mo te English e whakamahia ana i Ingarangi (UK).</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Langue</Attribute>
+            <Attribute name="definition">La langue du contenu intellectuel de la ressource.</Attribute>
+            <Attribute name="comment">Il est recommandÃ© d&apos;utiliser comme valeur de l&apos;Ã©lÃ©ment Langue celles definies par la RFC 1766 [RFC1766] qui comprend un code de langage Ã  deux caractÃšres(venant du standard ISO 639 [ISO639]), Ã©ventuellement suivi d&apos;un code Ã  deux lettres pour le pays (venant du standard ISO 3166 [ISO3166] ou en franÃ§ais [ISO3166]). Par exemple, &apos;en&apos; pour l&apos;anglais, &apos;fr&apos; pour le franÃ§ais, ou &apos;en-uk&apos; pour l&apos;anglais utilisÃ© au Royaume-Uni.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">Ð¯Ð·ÑÐº</Attribute>
+            <Attribute name="definition">Ð¯Ð·ÑÐº ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° Ð¯Ð·ÑÐº, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÐŒÐŸÐµ RFC 1766 (Ð¢ÑÐ³Ðž ÐŽÐ»Ñ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐ·ÑÐºÐ°), Ð²ÐºÐ»ÑÑÐ°ÑÑÐžÐŒ ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÐ·ÑÐºÐ° (Ð²Ð·ÑÑÑÐ¹ ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 639 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÐ·ÑÐºÐŸÐ²)), Ð·Ð° ÐºÐŸÑÐŸÑÑÐŒ ÑÐ»ÐµÐŽÑÐµÑ (Ð¿ÑÐž ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸÑÑÐž) ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐ¹ ÐºÐŸÐŽ ÑÑÑÐ°ÐœÑ (Ð²Ð·ÑÑÑÐŒÐž ÐžÐ· ÑÑÐ°ÐœÐŽÐ°ÑÑÐ° ISO 3166 (ÐÐŸÐŽÑ ÐŽÐ»Ñ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐžÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÑÑÐ°Ðœ)). ÐÐ°Ð¿ÑÐžÐŒÐµÑ, &quot;en&quot; - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, &quot;fr&quot; - ÐŽÐ»Ñ ÑÑÐ°ÐœÑÑÐ·ÑÐºÐŸÐ³ÐŸ, &quot;en-uk&quot; - ÐŽÐ»Ñ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐŸÐ³ÐŸ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ Ð² ÐÐµÐ»ÐžÐºÐŸÐ±ÑÐžÑÐ°ÐœÐžÐž.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Lengua</Attribute>
+            <Attribute name="definition">Lengua/s del contenido intelectual del recurso. Practicamente el contenido de este campo deberÃ­a coincidir con los de la RFC 1766 (Tags para la identificaciÃ³n de lenguas, http://ds.internic.net/rfc/rfc1766.txt); por ejemplo: en, es, de, fi, ja y zh.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙÙØºØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ù ÙÙ ÙØºØ© Ø§ÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . ÙÙÙ
+Ø§ ÙØ§Ù  Ø¹Ù
+ÙÙØ§ , Ù
+Ø­ØªÙÙ ÙØžØ§ Ø§ÙØ­ÙÙ ÙØ¬Øš Ø£Ù ÙØªØ·Ø§ØšÙ Ù
+Ø¹ RFC 1766    ] Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ th, ja, fr, es, fi, de, en Ù zh ÙØ±Ù
+ÙØ² ÙÙØªØ¹Ø±ÙÙ ØšØ§ÙÙØºØ© [</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">è¯­ç§</Attribute>
+            <Attribute name="definition">æè¿°èµæºç¥è¯å
+å®¹çè¯­ç§ã</Attribute>
+            <Attribute name="comment">å»ºè®®æ¬å
+çŽ çåŒéçšRFC 3066[RFC3066]ïŒè¯¥æ åäžISO 639 [ISO639]äžèµ·å®ä¹äºç±äž€äžªåäžäžªè±    æå­æ¯ç»æçäž»è¯­èšæ ç­Ÿåå­æ ç­ŸäŸåŠå
+æ¬âenâæâengâæ¥è¡šç€ºEnglish, &quot;akk&quot; æ¥è¡šç€º    &quot;en-GB&quot;è¡šç€ºè±åœè±è¯­ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Jazyk</Attribute>
+            <Attribute name="definition">Jazyk intelektuÃ¡lnÃ­ho obsahu zdroje.</Attribute>
+            <Attribute name="comment">DoporuÄenÃœ postup pro hodnoty prvku Jazyk je definovÃ¡n v RFC 1766 [RFC1766]: dvouznakovÃœ kÃ³d jazyka (pÅevzatÃœ z normy ISO 639 [ISO639]) volitelnÄ nÃ¡sledovanÃœ dvouznakovÃœm kÃ³dem zemÄ (pÅevzatÃœm z normy ISO 3166 [ISO3166]). NapÅÃ­klad &apos;en&apos; pro angliÄtinu, &apos;fr&apos; pro francouzÅ¡tinu, &apos;cs&apos; pro ÄeÅ¡tinu, nebo &apos;en-uk&apos; pro angliÄtinu uÅŸÃ­vanou ve SpojenÃ©m krÃ¡lovstvÃ­.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Taal</Attribute>
+            <Attribute name="definition">De taal waarin het document is beschreven. Voorgeschreven wordt de ISO639-1 norm, een twee karakter taalcode van de ISO 639(639-1 en 639-2) norm. Taalcodes kunnen van een lijst worden gekozen: http://www.ics.uci.edu/pub/ietf/http/related/iso639.txt, Eventueel kan ISO3166 (landencodes) gebruikt worden om aan te geven in welk gebied de taal wordt gesproken. Deze kunnen ook van een lijst worden gekozen:  http://www.ics.uci.edu/pub/ietf/http/related/iso3166.txt.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Lingua</Attribute>
+            <Attribute name="definition">Le lingua del contento intellectual del ressource.</Attribute>
+            <Attribute name="comment">Le practica recommendate pro le valores del elemento lingua es definite per RFC 1766 [RFC1766], que include un codice de lingua de duo litteras (prendite del standard ISO 639 [ISO639]), sequite optionalmente per un codice de pais de duo litteras (prendite del standard ISO 3166 [ISO3166]). Per exemplo, &apos;ia&apos; pro interlingua, &apos;fr&apos; pro francese, o &apos;en-uk&apos; pro le anglese del Regno Unite.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€­à€Ÿà€·à€Ÿ</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à€Ÿà€·à€Ÿ</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€®à€°à€Ÿà€ à¥ - à€­à€Ÿà€°à€€à¥à€¯ à€­à€Ÿà€·à€Ÿ</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="Relation" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Relation</Attribute>
+            <Attribute name="definition">A reference to a related resource.</Attribute>
+            <Attribute name="comment">Recommended best practice is to reference the resource by means of a string or number conforming to a formal identification system.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Whai Panga</Attribute>
+            <Attribute name="definition">He tohutoro ki tetahi rauemi whai panga.</Attribute>
+            <Attribute name="comment">Ko te tino huarahi kua tohua, ko te tohutoro ki te rauemi ma te aho, te tau ranei, e hangai ana ki te punaha taututanga okawa.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Relation</Attribute>
+            <Attribute name="definition">Une rÃ©fÃ©rence Ã  une autre ressource qui a un rapport avec cette ressource.</Attribute>
+            <Attribute name="comment">Il est recommandÃ© de rÃ©fÃ©rencer cette ressource par une chaÃ®ne de caractÃšres ou un numÃ©ro conforme Ã  un systÃšme formel d&apos;identification.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐœÐŸÑÐµÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ï¿œÐµÐœÐœÑÐ¹ ÑÐµÑÑÑÑ.</Attribute>
+            <Attribute name="comment">ÐÐ»Ñ Ð¿ÑÐ°ÐºÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ ÐžÐŽÐµÐœÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÑÑÑÑ Ð¿ÐŸÑÑÐµÐŽÑÑÐ²ÐŸÐŒ ÑÑÑÐŸÐºÐž ÐžÐ»Ðž ÑÐžÑÐ»Ð°, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐ³ÐŸ ÑÐŸÑÐŒÐ°Ð»ÑÐœÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ ÑÐžÑÑÐµÐŒÐµ.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">RelaciÃ³n</Attribute>
+            <Attribute name="definition">Un identificador de un segundo recurso y su relaciÃ³n con el recurso actual. Este elemento permite enlazar los recursos relacionados y las descripciones de los recursos. Por ejemplo: IsVersionOf - Incluye la ediciÃ³n de un trabajo, IsBasedOn - La traducciÃ³n de un trabajo, IsPartOf - Un capÃ­tulo de un libro, IsFormatOf - Un mecanismo de transformaciÃ³n de un conjunto de datos en una imagen. Para asegurar la interoperabilidad, las relaciones deberÃ­an ser seleccionadas de una lista de elementos que actualmente se encuentra bajo desarrollo en un grupo de trabajo.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØ¹ÙØ§ÙØ©</Attribute>
+            <Attribute name="definition">ÙØ°Ø§ Ù
+Ø¹Ø±Ù ÙÙÙ
+ØµØ¯Ø± Ø§ÙØ«Ø§ÙÙ Ù Ø¹ÙØ§ÙØªÙ ØšØ§ÙÙ
+ØµØ¯Ø± Ø§ÙØ­Ø§ÙÙ . ÙØ³Ù
+Ø­ ÙØ°Ø§ Ø§ÙØ¹Ø§Ù
+Ù ØšØªÙØ¶ÙØ­ Ø§ÙØ±ÙØ§ØšØ· ØšÙÙ Ø§ÙÙ
+ØµØ§Ø¯Ø± Ø§ÙÙ
+ØªØ¹ÙÙØ©     ØšØšØ¹Ø¶ÙØ§ Ù ØªÙØµÙÙ Ø§ÙÙ
+ØµØ¯Ø± . Ø§ÙØ§Ù
+Ø«ÙØ© ØªØ¶Ù
+ Ø·ØšØ¹Ø© Ø§ÙØ¹Ù
+Ù ) ÙØ³Ø®Ø© Ù
+Ù ( ØªØ±Ø¬Ù
+Ø© Ø¹Ù
+Ù ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ ( ÙØµÙ Ù
+Ù ÙØªØ§Øš ) Ø¬Ø²Ø¡ Ù
+Ù (    Ù ØªØ­ÙÙÙÙ Ù
+ÙÙØ§ÙÙÙÙØ© ÙÙ
+Ø¬Ù
+ÙØ¹Ø© Ù
+Ø¹ÙÙÙ
+Ø§ØªÙØ© Ø§ÙÙ ØµÙØ±Ø© ) ØµÙØºØ© Ù
+Ù ( Ù
+Ù Ø£Ø¬Ù Ø§ÙØªØ¹Ø§Ù
+Ù Ø§ÙÙ
+ØŽØªØ±Ù , ÙØ¬Øš Ø§Ø®ØªÙØ§Ø± Ø§ÙØ¹ÙØ§ÙØ§Øª Ù
+Ù    Ø§ÙÙØ§ØŠÙ
+Ø© Ø§ÙÙ
+Ø±ÙÙ
+Ø© Ù Ø§ÙØªÙ ÙØªÙ
+ Ø§Ø¹Ø¯Ø§Ø¯ÙØ§ Ø­Ø§ÙÙØ§ Ø¶Ù
+Ù Ø­ÙÙØ§Øª ÙØ±ØŽØ© Ø¹Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">å
+³è</Attribute>
+            <Attribute name="definition">æ¶åççžå
+³èµæºã</Attribute>
+            <Attribute name="comment">èåœçžå
+³èµæºæ¶ïŒå»ºè®®äœ¿çšç¬Šåæ­£åŒæ è¯ç³»ç»çå­ç¬Šäž²æè
+æ°å­ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">Vztah</Attribute>
+            <Attribute name="definition">Odkaz na pÅÃ­buznÃœ zdroj.</Attribute>
+            <Attribute name="comment">DoporuÄuje se odkazovat zdroj znakovÃœm ÅetÄzcem nebo ÄÃ­slem odpovÃ­dajÃ­cÃ­m nÄkterÃ©mu z formÃ¡lnÃ­ch identifikaÄnÃ­ch systÃ©mÅ¯.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Relatie</Attribute>
+            <Attribute name="definition">Een identificatie van een tweede bron en de relatie met de huide Internetbron. Voorbeelden zijn een editie van een werk (IsVersionOf), een vertaling van een werk (IsBasedOn), een hoofdstuk van een boek (IsPartOf) en een mechanische transformatie van een dataset in een plaatje (IsFormatOf). De DC Working Groups zijn bezig met een lijst met de meest verwachte relaties. Aanbevolen wordt om de relaties van deze lijst te gebruiken: http://purl.oclc.org/metadata/dublin_core/wrelationdraft.html.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="definition">Relation</Attribute>
+            <Attribute name="definition">Un referentia a un ressource correlate.</Attribute>
+            <Attribute name="comment">Le practica recommendate es facer referentia al ressource per medio de un catena de characteres o numero conforme a un systema de identification formal.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€šà€Ÿà€€à¥ , à€žà€à€¬à€à€§</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€¶à¥ à€žà€à€¬à€à€§ à€Šà€Ÿà€à€µà€£à€Ÿà€°à€Ÿ à€žà€à€Šà€°à¥à€­</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ.  à€®à€Ÿà€¹à€¿à€€à¥ à€à€Ÿà€³à¥à€¯à€Ÿà€µà€°à¥à€² à€ªà€€à¥à€€à€Ÿ(URL)</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="conformsTo">
+            <Language code="en">
+                <Attribute name="identifier">Conforms To</Attribute>
+                <Attribute name="definition">A reference to an established standard to which the resource conforms.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="hasFormat">
+            <Language code="en">
+                <Attribute name="identifier">Has Format</Attribute>
+                <Attribute name="definition">The described resource pre-existed the referenced resource, which is essentially the same intellectual content presented in another format.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="hasPart">
+            <Language code="en">
+                <Attribute name="identifier">Has Part</Attribute>
+                <Attribute name="definition">The described resource includes the referenced resource either physically or logically.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="hasVersion">
+            <Language code="en">
+                <Attribute name="identifier">Has Version</Attribute>
+                <Attribute name="definition">The described resource has a version, edition, or adaptation, namely, the referenced resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="isFormatOf">
+            <Language code="en">
+                <Attribute name="identifier">Is Format Of</Attribute>
+                <Attribute name="definition">The described resource is the same intellectual content of the referenced resource, but presented in another format.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="isPartOf">
+            <Language code="en">
+                <Attribute name="identifier">Is Part Of</Attribute>
+                <Attribute name="definition">The described resource is a physical or logical part of the referenced resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="isReferencedBy">
+            <Language code="en">
+                <Attribute name="identifier">Is Referenced By</Attribute>
+                <Attribute name="definition">The described resource is referenced, cited, or otherwise pointed to by the referenced resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="isReplacedBy">
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Is Replaced By</Attribute>
+                <Attribute name="definition">The described resource is supplanted, displaced, or superseded by the referenced resource.</Attribute>
+            </Language>
+        </Element>
+        <Element name="isRequiredBy">
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Is Required By</Attribute>
+                <Attribute name="definition">The described resource is required by the referenced resource, either physically or logically.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+        <Element name="isVersionOf">
+            <Language code="en">
+                <Attribute name="identifier">Is Version Of</Attribute>
+                <Attribute name="definition">The described resource is a version, edition, or adaptation of the referenced resource. Changes in version imply substantive changes in content rather than differences in format.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="references">
+            <Language code="en">
+                <Attribute name="identifier">References</Attribute>
+                <Attribute name="definition">The described resource references, cites, or otherwise points to the referenced resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+        <Element name="replaces">
+            <Language code="en">
+                <Attribute name="status">recommended</Attribute>
+                <Attribute name="identifier">Replaces</Attribute>
+                <Attribute name="definition">The described resource supplants, displaces, or supersedes the referenced resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+        <Element name="requires">
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Requires</Attribute>
+                <Attribute name="definition">The described resource requires the referenced resource to support its function, delivery, or coherence of content.</Attribute>
+            </Language>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+    </Element>
+    <Element name="Coverage" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Coverage</Attribute>
+            <Attribute name="definition">The extent or scope of the content of the resource.</Attribute>
+            <Attribute name="comment">Coverage will typically include spatial location (a place name or geographic coordinates), temporal period (a period label, date, or date range) or jurisdiction (such as a named administrative entity). Recommended best practice is to select a value from a controlled vocabulary (for example, the Thesaurus of Geographic Names [TGN]) and that, where appropriate, named places or time periods be used in preference to numeric identifiers such as sets of coordinates or date ranges.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Kapitanga</Attribute>
+            <Attribute name="definition">Te roanga, te whanuitanga ranei o te whakakaupapatanga o te rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, ka uru atu te wahi mokowa (te ingoa o tetahi wahi, whakapiritanga matawhenua ranei), whakaritenga wa (he tapatanga wa (period label, ra, whakatakotoranga whanui o nga ra), te whakaritenga ture (penei i te putahi whakahaere). Ko te tino huarahi kua tohua, ko te tipako uara mai i tetahi whakaritenga kupu (hei tauira, i tetahi Puna Kupu Ingoa Matawhenua (Geographic Names [TGN] i nga wahi tika, ko nga wahi me nga whakaritenga wa ka whakamahia i mua i te whakamahitanga o nga taututanga tau penei i nga huinga whakapiritanga me nga whakatakotoranga whanui o nga ra.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Couverture</Attribute>
+            <Attribute name="definition">La portÃ©e ou la couverture spatio-temporelle de la ressource.</Attribute>
+            <Attribute name="comment">La couverture typiquement inclut une position gÃ©ographique (le nom d&apos;un lieu ou ses coordonnÃ©es), une pÃ©riode de temps (un nom de pÃ©riode, une date, ou un intervalle de temps) ou une juridiction (telle que le nom d&apos;une entitÃ© administrative). Il est recommandÃ© de choisir la  valeur de Couverture dans un vocabulaire contrÃŽlÃ© (par exemple, un thÃ©saurus de noms gÃ©ographiques, comme[TGN]) et, quand cela est appropriÃ©, des noms de lieux ou de pÃ©riodes plutÃŽt que des identifiants numÃ©riques tels que des coordonnÃ©es ou des intervalles de dates. </Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑ
+Ð²Ð°Ñ</Attribute>
+            <Attribute name="definition">ÐÑÐŸÑÑÐ¶ÐµÐœÐœÐŸÑÑÑ Ðž Ð³ÑÐ°ÐœÐžÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÑ
+Ð²Ð°Ñ ÐŸÐ±ÑÐºÐœÐŸÐ²ÐµÐœÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ Ð¿ÑÐŸÑÑÑÐ°ÐœÑÑÐ²ÐµÐœÐœÐŸÐµ ÐŒÐµÑÑÐŸÐœÐ°Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÐµ (ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŒÐµÑÑÐœÐŸÑÑÐž ÐžÐ»Ðž Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÐºÐŸÐŸÑÐŽÐžÐœÐ°ÑÑ), Ð²ÑÐµÐŒÐµÐœÐœÐŸÐ¹ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐŸÐº (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ Ð¿ÐµÑÐžÐŸÐŽÐ°, ÐŽÐ°ÑÐ° ÐžÐ»Ðž ÑÑÐŽ ÐŽÐ°Ñ) ÐžÐ»Ðž ÑÑÐžÑÐŽÐžÐºÑÐžÑ (ÑÐºÐ°Ð·Ð°ÐœÐžÐµ ÐµÐŽÐžÐœÐžÑÑ Ð°ÐŽÐŒÐžÐœÐžÑÑÑÐ°ÑÐžÐ²ÐœÐŸÐ³ÐŸ ÐŽÐµÐ»ÐµÐœÐžÑ). ÐÐ° Ð¿ÑÐ°ÐºÑÐžÐºÐµ ÑÐµÐºÐŸÐŒÐµÐœÐŽÑÐµÑÑÑ Ð²ÑÐ±ÐžÑÐ°ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐžÐ· ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÐŒÐŸÐ³ÐŸ ÑÐ»ÐŸÐ²Ð°ÑÑ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð¢ÐµÐ·Ð°ÑÑÑÑÐ° Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ TGN), Ðž, ÐµÑÐ»Ðž ÑÑÐŸ ÑÐŒÐµÑÑÐœÐŸ, Ð¿ÑÐµÐŽÐ¿ÐŸÑÑÐµÐœÐžÐµ ÐŸÑÐŽÐ°ÐµÑÑÑ ÑÐºÐ°Ð·Ð°ÐœÐžÑÐŒ ÐŒÐµÑÑÐœÐŸÑÑÐµÐ¹ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ Ð¿ÑÐŸÐŒÐµÐ¶ÑÑÐºÐŸÐ² Ð²ÐŒÐµÑÑÐŸ ÑÐžÑÑÐŸÐ²ÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ², ÐºÐ°Ðº, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐžÑÑÐµÐŒ ÐºÐŸÐŸÑÐŽÐžÐœÐ°Ñ ÐžÐ»Ðž Ð²ÑÐµÐŒÐµÐœÐœÑÑ
+ ÑÑÐŽÐŸÐ².</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Cobertura</Attribute>
+            <Attribute name="definition">La caracterÃ­stica de cobertura espacial y/o temporal del contenido intelectual del recurso. La cobertura espacial se refiere a una regiÃ³n fÃ­sica (por ejemplo, sector celestial); uso de coordenadas (por ejemplo, longitud y latitud) o nombres de lugares extraidos de una lista controlada. La cobertura temporal se refiere al contenido del recurso en vez de a cuando fue creado o puesto accesible ya que este Ãºltimo pertenece al elemento Date. Se usa el mismo formato basado en http://www.w3.org/TR/NOTE-datetime.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø§ÙØªØºØ·ÙØ©</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØ®ØµØ§ØŠØµ Ø§ÙÙ
+ÙØ§ÙÙØ© Ø§Ù Ø§ÙØ²Ù
+ÙÙØ© ÙÙÙ
+Ø­ØªÙÙ Ø§ÙÙÙØ±Ù ÙÙÙ
+ØµØ¯Ø± . Ø§ÙØªØºØ·ÙØ© Ø§ÙÙ
+ÙØ§ÙÙØ© ØªØŽÙØ± Ø§ÙÙ Ø¬ÙØ© Ø·ØšÙØ¹ÙØ© )Ù
+Ø«Ù , ÙØ·Ø§Ø¹ Ø³Ù
+Ø§ÙÙ (    Ø§Ø³ØªØ®Ø¯Ø§Ù
+ Ø§ÙØ§Ø­Ø¯Ø§Ø«ÙØ§Øª ) Ù
+Ø«Ù  Ø®Ø· Ø§ÙØ·ÙÙ Ù Ø®Ø· Ø§ÙØ¹Ø±Ø¶ ( ÙÙ Ø£Ø³Ù
+Ø§Ø¡ Ø§Ù
+ÙÙØ© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù . Ø§ÙØªØºØ·ÙØ© Ø§ÙØ²Ù
+ÙÙØ©    Ø«Ø¹ÙØ¯ Ø§ÙÙ Ù
+Ø§ ÙØ±Ø¬Ø¹ Ø§ÙÙÙ Ø§ÙÙ
+ØµØ¯Ø± Ù ÙÙØ³ Ø§Ù Ù
+ØªÙ ØªÙ
+ ØªÙÙÙØ±Ù ) Ø§ÙØ§Ø®ÙØ± ÙØªØ¹ÙÙ ØšØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® ( Ø§Ø³ØªØ®Ø¯Ø§Ù
+ ÙÙØ³ ØµÙØºØ© Ø§ÙØªØ§Ø±ÙØ®  Ø§Ù Ø§ÙÙÙØª    ) Ø¹Ø§Ø¯Ø© Ù
+Ø¯Ù ] ( ØµÙØ§ØºØ§Øª Ø§ÙÙÙÙ
+ Ù Ø§ÙÙÙØª ) Ø§Ø¹ØªÙ
+Ø§Ø¯Ø§ Ø¹ÙÙ (ISO 8601 Ø§ÙÙ
+ÙØ§Ø­ØžØ© Ø§ÙÙÙÙØ© [http://www.w3.org/TR/NOTE-datetime, W3C    ÙÙ
+Ø§ ÙÙ Ù
+ÙØµÙ ØšÙ ÙØ¹Ø§Ù
+Ù Ø§ÙØªØ§Ø±ÙØ® Ø§Ù ÙØªØ±Ø§Øª Ø²Ù
+ÙÙØ© Ù
+Ù ÙØ§ØŠÙ
+Ø© Ù
+Ø³ØªØ­ÙÙ
+Ø© Ø§Ù Ù
+ØªØ¬ÙØ§Øª ØšØ§ÙÙØ§Ù
+Ù</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">èŠçèåŽ</Attribute>
+            <Attribute name="definition">èµæºå
+å®¹ææ¶åçå€å»¶äžèŠçèåŽ</Attribute>
+            <Attribute name="comment">èŠçèåŽäžè¬å
+æ¬ç©ºéŽäœçœ®ïŒäžäžªå°åæå°çåæ ïŒãæ¶éŽåºéŽïŒäžäžªæ¶éŽæ è¯ïŒæ¥ææäžäžªæ¥æè    åŽïŒãæè
+æéïŒæ¯åŠåœåçææå®äœïŒæšèèŠçèåŽæå¥œæ¯åèªäºäžäžªåæ§è¯è¡šïŒäŸåŠå°åè¯åº    [TGN]ïŒïŒå¹¶åºå°œå¯èœå°äœ¿çšç±æ°å­è¡šç€ºçåæ ææ¥æåºéŽæ¥æè¿°å°åäžæ¶éŽæ®µã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PokrytÃ­</Attribute>
+            <Attribute name="definition">Rozsah nebo zÃ¡bÄr obsahu zdroje.</Attribute>
+            <Attribute name="comment">PokrytÃ­ bude typicky obsahovat prostorovÃ© umÃ­stÄnÃ­ (jmÃ©no mÃ­sta nebo zemÄpisnÃ© souÅadnice), ÄasovÃ© obdobÃ­ (oznaÄenÃ­ obdobÃ­, datum nebo ÄasovÃœ interval) nebo jurisdikci (napÅ. jmÃ©no administrativnÃ­ jednotky). DoporuÄuje se vybÃ­rat hodnotu z ÅÃ­zenÃ©ho slovnÃ­ku (napÅ. Thesaurus of Geographic Names [TGN]) a tam, kde je to vhodnÃ©, uvÃ¡dÄt mÃ­sta nebo ÄasovÃ© Ãºseky spÃ­Å¡e jmÃ©nem neÅŸ ÄÃ­selnÃœmi identifikÃ¡tory jako jsou soubory souÅadnic nebo ÄasovÃ© intervaly.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Dekking</Attribute>
+            <Attribute name="definition">De ruimtelijke of tijdelijke karakteristieken van de intellectuele inhoud van de Internetbron. Ruimtelijke dekking verwijst naar een geografisch gebied. Gebruik coordinaten (longitude (lengtegraad) en latitude (breedtegraad)) of plaatsnamen die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Dekking in de tijd geeft de tijdsperiode aan waarop de inhoud van de Internetbron betrekking heeft en niet wanneer de Internetbron is gemaakt of beschikbaar is gesteld (dit laatste hoort ingevult te worden bij &quot;Date&quot;). Gebruik dezelfde datum/tijd formaat als bij &quot;Date&quot;; W3C Technical Note http://www.w3.org/TR/NOTE-datetime of tijdsperioden die van een gecontroleerde lijst komen of volledig uitgeschreven zijn. Coverage is in ontwikkeling bij de Dublin Core Working group http://www.alexandria.ucsb.edu/docs/metadata/dc_coverage.html.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Copertura</Attribute>
+            <Attribute name="definition">Le extension physic o abstracte del contento del ressource.</Attribute>
+            <Attribute name="comment">Le copertura typicamente include un location spatial (un nomine de localitate o coordinatas geographic), un periodo de tempore (un etiquetta de periodo, data, o gamma de datas), o un jurisdiction (tal como un entitate administrative nominate). Le practica recommendate es selectionar un valor a partir de un vocabulario controlate (per exemplo, le Thesaurus of Geographical Names [TGN]) e que, quando appropriate, localitates o periodos de tempore nominate sia utilizate in preferentia a identificatores numeric tales como collectiones de coordinatas o gammas de datas.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+            <Attribute name="definition">à€®à€Ÿà€¹à€¿à€€à¥à€žà€Ÿà€ à¥à€¯à€Ÿà€à¥ à€­à¥à€à¥à€²à€¿à€  à€®à€Ÿà€¹à€¿à€€à¥</Attribute>
+            <Attribute name="comment">à€à€Šà€Ÿ. à€­à¥à€à¥à€²à€¿à€ à€µà¥à€¯à€Ÿà€ªà¥à€€à¥</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="spatial">
+            <Attribute language="en" name="status">recommended</Attribute>
+            <Language code="en">
+                <Attribute name="identifier">Spatial</Attribute>
+                <Attribute name="definition">Spatial characteristics of the intellectual content of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+        </Element>
+        <Element name="temporal">
+            <Language code="en">
+                <Attribute name="identifier">Temporal</Attribute>
+                <Attribute name="definition">Temporal characteristics of the intellectual content of the resource.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+    </Element>
+    <Element name="Rights" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Rights Management</Attribute>
+            <Attribute name="definition">Information about rights held in and over the resource.</Attribute>
+            <Attribute name="comment">Typically, a Rights element will contain a rights management statement for the resource, or reference a service providing such information. Rights information often encompasses Intellectual Property Rights (IPR), Copyright, and various Property Rights. If the Rights element is absent, no assumptions can be made about the status of these and other rights with respect to the resource.</Attribute>
+        </Language>
+        <!-- Maori -->
+        <Language code="mi">
+            <Attribute name="identifier">Motika</Attribute>
+            <Attribute name="definition">Ko nga mohiohio ki nga motika o tetahi rauemi.</Attribute>
+            <Attribute name="comment">Ko te tikanga, kei te huanga Motika e pupuri ana he korero mo te whakaritenga motika mo te rauemi, he ratonga tohutoro e whakarato ana ranei i enei tumomo mohiohio. He nui nga wa, puta ai i roto i te mohiohio Motika te Mana Whakairo Hinegaro (Intellectual Property Rights - (IPR)), te Manatarua me nga Tuahua Mana Taonga (Property Rights). Ki te kore te huanga Motika, kaore e taea te pohehe noa ki te mana o enei, nga motika ranei e pa ana ki te rauemi.</Attribute>
+        </Language>
+        <!-- French -->
+        <Language code="fr">
+            <Attribute name="identifier">Droits</Attribute>
+            <Attribute name="definition">Information sur les droits sur et au sujet de la ressource.</Attribute>
+            <Attribute name="comment">Typiquement, un Ã©lÃ©ment Droits contiendra un Ã©tat du droit Ã  gÃ©rer une ressource, ou la refÃ©rence Ã  un service fournissant cette information. Ces droits souvent couvrent les droits de propriÃ©tÃ© intellectuelle (IPR), Copyright, et divers droits de propriÃ©tÃ©. Si l&apos;Ã©lÃ©ment Droits est absent, aucune hypothÃšse ne peut Ãªtre faite sur l&apos;Ã©tat de ces droits, ou de tout autre, par rapport Ã  la ressource.</Attribute>
+        </Language>
+        <!-- Russian -->
+        <Language code="ru">
+            <Attribute name="identifier">ÐÑÐ°Ð²ÐŸÐ²ÐŸÐµ ÑÐµÐ³ÑÐ»ÐžÑÐŸÐ²Ð°ÐœÐžÐµ</Attribute>
+            <Attribute name="definition">ÐÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð¿ÑÐ°Ð²Ð°Ñ
+ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐ°ÑÑÑ Ð²ÐœÑÑÑÐž Ðž Ð²ÐœÐµ ÑÐµÑÑÑÑÐ°.</Attribute>
+            <Attribute name="comment">ÐÐ±ÑÑÐœÐŸ ÑÐ»ÐµÐŒÐµÐœÑ ÐÑÐ°Ð²Ð° ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐµ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÑÑ
+ ÐœÐŸÑÐŒÐ°Ñ
+, ÑÐµÐ³ÑÐ»ÐžÑÑÑÑÐžÑ
+ ÑÑÐœÐºÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐµÑÑÑÑÐ° ÐžÐ»Ðž ÑÑÑÐ»ÐºÑ ÐœÐ° ÑÐ»ÑÐ¶Ð±Ñ, Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÑÑÑÑÑ ÑÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ. ÐÑÐ°Ð²ÐŸÐ²Ð°Ñ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸÐ±ÑÑÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÑÐ²ÐµÐŽÐµÐœÐžÑ ÐŸ ÐÑÐ°Ð²Ð°Ñ
+ ÐžÐœÑÐµÐ»Ð»ÐµÐºÑÑÐ°Ð»ÑÐœÐŸÐ¹ ÑÐŸÐ±ÑÑÐ²ÐµÐœÐœÐŸÑÑÐž, ÐÐ²ÑÐŸÑÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²Ðµ Ðž ÐŽÑÑÐ³ÐžÑ
+ ÐÐŒÑÑÐµÑÑÐ²ÐµÐœÐœÑÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+. ÐÑÑÑÑÑÑÐ²ÐžÐµ ÑÐ»ÐµÐŒÐµÐœÑÐ° ÐÑÐ°Ð²Ð° ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÑÐ²Ð»ÑÑÑÑÑ ÐŸÑÐœÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŽÐ»Ñ ÐºÐ°ÐºÐžÑ
+-Ð»ÐžÐ±ÐŸ Ð¿ÑÐµÐŽÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐ¹ ÐŸ Ð¿ÑÐ°Ð²ÐŸÐ²ÐŸÐŒ ÑÑÐ°ÑÑÑÐµ Ðž ÐŽÑÑÐ³ÐžÑ
+ Ð¿ÑÐ°Ð²Ð°Ñ
+, ÐºÐ°ÑÐ°ÑÑÐžÑ
+ÑÑ ÑÐµÑÑÑÑÐ°.</Attribute>
+        </Language>
+        <!-- Spanish -->
+        <Language code="es">
+            <Attribute name="identifier">Derechos</Attribute>
+            <Attribute name="definition">Una referencia (URL, por ejemplo) para una nota sobre derechos de autor, para un servicio de gestiÃ³n de derechos o para un servicio que darÃ¡ informaciÃ³n sobre tÃ©rminos y condiciones de acceso a un recurso. Una especificaciÃ³n formal del elemento Rights se encuentra catualmente en discusiÃ³n y por lo tanto su uso se considera experimental.</Attribute>
+        </Language>
+        <!-- Arabic -->
+        <Language code="ar">
+            <Attribute name="identifier">Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø©</Attribute>
+            <Attribute name="definition">ÙÙ Ø§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ§ÙØ§Ø¯Ø© Ø­ÙÙÙ Ø§ ÙØ§Ø¯Ø§Ø±Ø© , Ù
+Ø¹Ø±Ù Ù
+Ø±ØªØšØ· ØšØ®Ø¯Ù
+Ø© ØªÙØ¯Ù
+ Ù
+Ø¹ÙÙÙ
+Ø§Øª Ø¹Ù Ø­ÙÙÙ Ø§ÙØ§Ø¯Ø§Ø±Ø© Ø§ÙØ®Ø§ØµØ© ØšØ§ÙÙ
+ØµØ¯Ø±</Attribute>
+        </Language>
+        <!-- Chinese - Simplified -->
+        <Language code="cn">
+            <Attribute name="identifier">æé</Attribute>
+            <Attribute name="definition">æå
+³èµæºæ¬èº«ææçæè¢«èµäºçæéä¿¡æ¯</Attribute>
+            <Attribute name="comment">äžè¬èèšïŒæéå
+çŽ åºå
+æ¬äžäžªå¯¹èµæºçæéå£°æïŒæè
+æ¯å¯¹æäŸè¿äžä¿¡æ¯çæå¡çåç
+§ãæéäžè¬    å
+æ¬ç¥è¯äº§æ(IPR)ïŒçææå
+¶ä»åç§åæ ·çäº§æãåŠææ²¡ææéå
+çŽ çæ æ³šïŒäžå¯ä»¥å¯¹äžèµæºçžå
+³    çäžè¿°æå
+¶ä»æå©çæ
+åµäœåºä»»äœåå®ã</Attribute>
+        </Language>
+        <!-- Czech -->
+        <Language code="cs">
+            <Attribute name="identifier">PrÃ¡va</Attribute>
+            <Attribute name="definition">Informace o prÃ¡vech vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+            <Attribute name="comment">Prvek PrÃ¡va bude typicky obsahovat ustanovenÃ­ o sprÃ¡vÄ autorskÃœch/vlastnickÃœch prÃ¡v ke zdroji nebo odkaz na sluÅŸbu poskytujÃ­cÃ­ takovÃ©to informace. Informace o prÃ¡vech Äasto zahrnujÃ­ prÃ¡va duÅ¡evnÃ­ho vlastnictvÃ­ (IPR), autorskÃ© prÃ¡vo a rÅ¯znÃ¡ vlastnickÃ¡ prÃ¡va. JestliÅŸe prvek PrÃ¡va chybÃ­, nelze odvozovat ÅŸÃ¡dnÃ© pÅedpoklady o stavu tÄchto a jinÃœch prÃ¡v vztahujÃ­cÃ­ch se k popisovanÃ©mu zdroji.</Attribute>
+        </Language>
+        <!-- Dutch -->
+        <Language code="nl">
+            <Attribute name="identifier">Copyright</Attribute>
+            <Attribute name="definition">Een copyright verklaring of een link naar een copyright verklaring of een link naar een service die informatie geeft over de copyright van de Internetbron.</Attribute>
+        </Language>
+        <!-- Finnish -->
+        <Language code="fi">
+            <Attribute name="identifier"/>
+            <Attribute name="definition"/>
+            <Attribute name="comment"/>
+        </Language>
+        <!-- Interlingua -->
+        <Language code="ia">
+            <Attribute name="identifier">Gestion de derectos</Attribute>
+            <Attribute name="definition">Information super derectos detenite super e a proposito del ressource.</Attribute>
+            <Attribute name="comment">Typicamente le elemento derectos continera un declaration de gestion de derectos pro le ressource, o un referentia a un servicio que provide tal information. Information de derectos frequentemente involve derectos de proprietate intellectual (IPR), copyright e varie derectos de proprietate. Si le elemento Derectos es absente, necun hypothese pote esser facite quanto al stato de iste e altere derectos concernente ressource.</Attribute>
+        </Language>
+        <!-- Marathi -->
+        <Language code="mr">
+            <Attribute name="identifier">à€¹à€à¥à€</Attribute>
+            <Attribute name="definition">à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+            <Attribute name="comment">à€žà¥à€µà€€à€à€¹à€Ÿà€à¥ à€²à¥à€à€š à€à€Ÿà€ªà¥à€š à€
+à€¥à€µà€Ÿ à€®à€Ÿà€¹à€¿à€€à¥à€à€Ÿà€³à¥à€¯à€Ÿà€µà€° à€ªà¥à€°à€à€Ÿà€¶à€¿à€€ à€à€°à€£à¥à€¯à€Ÿà€à€Ÿ à€²à¥à€à€ à€
+à€¥à€µà€Ÿ à€²à¥à€à€¿à€à¥à€à€Ÿ à€¬à¥à€§à¥à€Šà€¿à€ à€žà€à€ªà€Šà¥à€à€Ÿà€¹à€à¥à€</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.1</Attribute>
+        <Attribute language="en" name="registration_authority">Dublin Core Metadata Initiative</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Charater String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+        <Element name="accessRights">
+            <Language code="en">
+                <Attribute name="identifier">Access Rights</Attribute>
+                <Attribute name="definition">Information about who can access the resource or an indication of its security status.</Attribute>
+                <Attribute name="comment">Access Rights may include information regarding access or restrictions based on privacy, security or other regulations.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+        <Element name="license">
+            <Language code="en">
+                <Attribute name="identifier">License</Attribute>
+                <Attribute name="definition">A legal document giving official permission to do something with the resource.</Attribute>
+                <Attribute name="comment">Recommended best practice is to identify the license using a URI. Examples of such licenses can be found at http://creativecommons.org/licenses/.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+    </Element>
+    <Element name="Audience">
+        <Language code="en">
+            <Attribute name="identifier">Audience</Attribute>
+            <Attribute name="definition">A class of entity for whom the resource is intended or useful.</Attribute>
+            <Attribute name="comment">A class of entity may be determined by the creator or the publisher or by a third party.</Attribute>
+        </Language>
+        <Element name="educationLevel">
+            <Language code="en">
+                <Attribute name="identifier">Audience Education Level</Attribute>
+                <Attribute name="definition">A general statement describing the education or training context. Alternatively, a more specific statement of the location of the audience in terms of its progression through an education or training context.</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">conforming</Attribute>
+        </Element>
+        <Element name="mediator">
+            <Language code="en">
+                <Attribute name="definition">A class of entity that mediates access to the resource and for whom the resource is intended or useful.</Attribute>
+                <Attribute name="comment">The audiences for a resource are of two basic classes: (1) an ultimate beneficiary of the resource, and (2) frequently, an entity that mediates access to the resource. The mediator element refinement represents the second of these two classes.</Attribute>
+                <Attribute name="identifier">Mediator</Attribute>
+            </Language>
+            <Attribute language="en" name="type">element-refinement</Attribute>
+            <Attribute language="en" name="status">recommended</Attribute>
+        </Element>
+    </Element>
+    <Element name="Provenance">
+        <Language code="en">
+            <Attribute name="identifier">Provenance</Attribute>
+            <Attribute name="definition">A statement of any changes in ownership and custody of the resource since its creation that are significant for its authenticity, integrity and interpretation.</Attribute>
+            <Attribute name="comment">The statement may include a description of any changes successive custodians made to the resource.</Attribute>
+        </Language>
+    </Element>
+    <Element name="RightsHolder">
+        <Language code="en">
+            <Attribute name="definition">A person or organization owning or managing rights over the resource.</Attribute>
+            <Attribute name="comment">Recommended best practice is to use the URI or name of the Rights Holder to indicate the entity.</Attribute>
+            <Attribute name="identifier">Rights Holder</Attribute>
+        </Language>
+        <Attribute language="en" name="status">conforming</Attribute>
+    </Element>
+</MetadataSet>
Index: /main/tags/3.01/gli/metadata/rfc1807.mds
===================================================================
--- /main/tags/3.01/gli/metadata/rfc1807.mds	(revision 21112)
+++ /main/tags/3.01/gli/metadata/rfc1807.mds	(revision 21112)
@@ -0,0 +1,462 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE MetadataSet [
+
+  <!-- The outer tag of any metadata set. Contains some number of elements.             -->
+  <!-- @param contact     A contact address for the creator.                            -->
+  <!-- @param creator     The name of the entity resposibly for this set.               -->
+  <!-- @param family      The name of this sets family.                                 -->
+  <!-- @param lastchanged The full date, user name and machine name of the last editor. --> 
+  <!--   uses the form:                                                                 -->
+  <!--   Day Mon(th) dd hh:mm:ss yyyy '-' <username> on <machine>                       -->
+  <!-- @param namespace   The namespace of this metadata set.                           -->
+  <!ELEMENT MetadataSet ((Name|Description|SetLanguage|Element)+)>
+  <!ATTLIST MetadataSet
+	    contact            CDATA #REQUIRED
+	    creator            CDATA #REQUIRED
+	    description        CDATA #REQUIRED
+            family             CDATA #REQUIRED
+	    lastchanged        CDATA #REQUIRED
+            name               CDATA #REQUIRED
+	    namespace          CDATA #REQUIRED
+	    language_dependent CDATA "">
+
+  <!-- The name of a metadata set. Is language specific. -->
+  <!ELEMENT Name (#PCDATA)>
+  <!ATTLIST Name
+            language CDATA "en">
+
+  <!-- The description of a metadata set. Is language specific. -->
+  <!ELEMENT Description (#PCDATA)>
+  <!ATTLIST Description
+            language CDATA "en">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT SetLanguage (Name, Description?)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- An element in the metadata set. -->
+  <!-- Note that the name attribute isn't really Dublin Core compliant - it was just the first thing I came up with. -->
+  <!ELEMENT Element (Attribute*,Language*,OptionList?)>
+  <!ATTLIST Element
+            name       CDATA #REQUIRED
+	    occurances CDATA "0">
+
+  <!-- Each element has several attribute children. -->	
+  <!-- Given that name was already used for an Element, I decided to use identifier for the language specific 'name', which I've been told isn't really compliant with Dublin Core. -->
+  <!ELEMENT Attribute (#PCDATA)>
+  <!ATTLIST Attribute
+            name CDATA "">
+
+  <!-- When used in conjuction with defer-node-expansion, allows for better memory use when dealing with several or more languages. -->
+  <!ELEMENT Language (Attribute*)>
+  <!ATTLIST Language
+	    code CDATA #REQUIRED>
+
+  <!-- For listing predefined values for this element. -->
+  <!ELEMENT OptionList (Value+)>
+  <!ATTLIST OptionList
+            restricted (true|false) "false">
+]>
+<MetadataSet contact="rlasher@forsythe.stanford.edu" creator="The Internet Engineering Task Force" family="RFC 1807" language_dependent="identifier, definition, comment" lastchanged="Fri Dec 3 14:10 NZST 2004 - mgw5 on narya" namespace="rfc1807">
+    <SetLanguage code="en">
+        <Name language="en">RFC 1807 Metadata Element Set, Version TR-v2.1</Name>
+        <Description language="en">    This document is the reference description for Internet Engineering Task Force (IETF) Request for Comments (RFC) 1807 metadata standard for bibliographic records, expressed as a metadata set for Greenstone.  </Description>
+    </SetLanguage>
+    <Element name="bib-version" occurances="1" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Bib-version</Attribute>
+            <Attribute name="definition">The format version of RFC-1807 record.</Attribute>
+            <Attribute name="comment">This is the first field of any record. It is a mandatory field. It identifies the version of the format used to create this bibliographic record. This RFC defines BIB-Version TR-v2.1 BIB-VERSIONs that start with the letter X (case independent) are considered experimental. Bib-records sent with such a BIB-VERSION should NOT be incorporated in the permanent database of the recipient. Using this version of this format, this field is always: Format: BIB-VERSION:: CS-TR-v2.1  </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Compulsory</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="minimum_occurance">1</Attribute>
+        <Attribute name="maximum_occurance">1</Attribute>
+    </Element>
+    <Element name="id" occurances="1" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Id</Attribute>
+            <Attribute name="definition">The identifier for the RFC-1807 record.</Attribute>
+            <Attribute name="comment">This is the second field of any record. It is also a mandatory field. The ID field identifies the bibliographic record and is used in management of these records. Its format is &quot;ID:: XXX//YYY&quot;, where XXX is the publisher-ID (the controlled symbol of the publisher) and YYY is the ID (e.g., report number) of the publication as assigned by the publisher. This ID is typically printed on the cover, and may contain slashes. The organization symbols &quot;DUMMY&quot; and &quot;TEST&quot; (case independent) are reserved for test records that should NOT be incorporated in the permanent database of the recipients. Format: ID:: &lt;publisher-ID&gt;//&lt;free-text&gt; Example: ID:: OUKS//CS-TR-91-123 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Compulsory</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="minimum_occurance">1</Attribute>
+        <Attribute name="maximum_occurance">1</Attribute>
+    </Element>
+    <Element name="entry" occurances="1" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Entry</Attribute>
+            <Attribute name="definition">The date RFC-1807 record was created.</Attribute>
+            <Attribute name="comment">This is a mandatory field. It is the date of creating this bibliographic record. The format for ENTRY date is &quot;Month Day, Year&quot;. The month must be alphabetic (spelled out). The &quot;Day&quot; is a 1- or 2-digit number. The &quot;Year&quot; is a 4-digit number. Format: ENTRY:: &lt;date&gt; Example: ENTRY:: January 15, 1992 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Compulsory</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="minimum_occurance">1</Attribute>
+        <Attribute name="maximum_occurance">1</Attribute>
+    </Element>
+    <Element name="organization" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Organization</Attribute>
+            <Attribute name="definition">The full name of the publishing organization</Attribute>
+            <Attribute name="comment">It is the full name spelled out (no acronyms, please) of the publishing organization. The use of this name is controlled together with the controlled symbol of the publisher (as discussed above for the ID field). Avoid acronyms because there are many common acronyms, such as ISI and USC. Please provide it in ascending order, such as &quot;X University, Y Department&quot; (not &quot;Y Department, X University&quot;) Format:  ORGANIZATION:: &lt;free-text&gt; Example:  ORGANIZATION:: Stanford University, Department of Computer Science  </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="title" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Title</Attribute>
+            <Attribute name="definition">The title of the work</Attribute>
+            <Attribute name="comment">This is the title of the work as assigned by the author. This field should include the complete title with all the subtitles, if any. Format:  TITLE:: &lt;free-text&gt;  Example:  TITLE:: The Computerization of Oceanview with High Speed Fiber Optics Communication </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="type" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Type</Attribute>
+            <Attribute name="definition">The type of publication</Attribute>
+            <Attribute name="comment">Indicates the type of publication (summary, final project report, etc.) as assigned by the issuing organization. Format:  TYPE:: &lt;free-text&gt; Example:  TYPE:: Technical Report  </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="revision" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Revision</Attribute>
+            <Attribute name="definition">If the record is a revision of a previously issued record</Attribute>
+            <Attribute name="comment">Indicates that the current bibliographic record is a revision of a previously issued record and is intended to replace it. Revision information consists of a date and/or followed by a semicolon and by text in an open ended format. The revised bibliographic record should contain a complete record for the publication, not just a list of changes to the old record. If revision is omitted, the record is assumed to be a new record and not a revision.  If the revision date is specified as 0, this is assumed to be January 1, 1900 (the previous RFC, used revision data of 0, 1, 2, 3, etc. this specification is for programs that might process records from RFC1357). The text before the semicolon in this field is a date of the form month day, year. Any record with a more recent revision date replaces completely any record with an earlier revision date (supplied either explicitly or by default). Use the text to describe the revision. Reasons to send out a revised record include an error in the original, or change in the access information. Format:  REVISION:: January 1, 1995; &lt;free-text&gt; Example: REVISION:: January 1, 1995; FTP information added </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="withdraw" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Withdraw</Attribute>
+            <Attribute name="definition">Withdraw means the document is no longer available.</Attribute>
+            <Attribute name="comment">Withdraw means the document is no longer available. Some Institutions choose to delete the record others remove some of the fields. It is up to each institution to decide how to process withdraw records. A withdraw record has all of the mandatory fields plus the withdraw field and a mandatory revision field. The Withdraw field should indicate the reason for the withdraw in free text. Example for withdrawing a bibliographic record:: BIB-VERSION::  CS-TR-v2.1 ID:: OUKS//CS-TR-91-123 ENTRY:: January 21, 1995 ORGANIZATION:: Oceanview University, Kansas, Computer Science TITLE:: The Computerization of Oceanview with High Speed Fiber Optics Communication REVISION:: January 21, 1995 WITHDRAW:: Withdrawn, found to be irrelevant END::  OUKS//CS-TR-91-123  </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="author" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Author</Attribute>
+            <Attribute name="definition">The author</Attribute>
+            <Attribute name="comment">Personal names only. Normal last name first inversion. Editors should be listed here as well, identified with the usual &quot;(ed.)&quot; as shown below in the last example. If the report was not authored by a person (e.g., it was authored by a committee or a panel) use CORP-AUTHOR (see below) instead of AUTHOR. Multiple authors are entered by using multiple lines, each in the form of &quot;AUTHOR:: &lt;free-text&gt;&quot;. The system preserves the order of the authors. Format: AUTHOR:: &lt;free-text&gt; Example:  AUTHOR:: Finnegan, James A. AUTHOR:: Pooh, Winnie The AUTHOR:: Lastname, Firstname (ed.) </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="corp-author" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Corp-author</Attribute>
+            <Attribute name="definition">The corporate author</Attribute>
+            <Attribute name="comment">The corporate author (e.g., a committee or a panel) that authored the report, which may be different from the ORGANIZATION issuing the report. In entering the corporate name please omit initial &quot;the&quot; or &quot;a&quot;.  If it is really part of the name, please invert it. Format: CORP-AUTHOR:: &lt;free-text&gt; Example: CORP-AUTHOR:: Committee on long-range computing </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="contact" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Contact</Attribute>
+            <Attribute name="definition">The contact for the author(s)</Attribute>
+            <Attribute name="comment">The contact for the author(s). Open-ended, most likely E-mail and postal addresses. A CONTACT field for each author should be provided, separately, or for all the AUTHOR fields. E-mail addresses should always be in &quot;pointy brackets&quot; (as in the example below). Format: CONTACT:: &lt;free-text&gt; Example: CONTACT:: Prof. J. A. Finnegan, CS Dept, Oceanview Univ., Oceanview, Kansas, 54321 Tel: 913-456-7890 &lt;Finnegan@cs.ouks.edu&gt; </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="date" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Date</Attribute>
+            <Attribute name="definition">The publication date</Attribute>
+            <Attribute name="comment">The publication date. The formats are &quot;Month Year&quot; and &quot;Month Day, Year&quot;. The month must be alphabetic (spelled out). The &quot;Day&quot; is a 1- or 2-digit number. The &quot;Year&quot; is a 4- digit number. Format:  DATE:: &lt;date&gt; Example: DATE:: January 1992 Example:  DATE:: January 15, 1992  </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="pages" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Pages</Attribute>
+            <Attribute name="definition">Total number of pages</Attribute>
+            <Attribute name="comment">Total number of pages, without being too picky about it. Final numbered page is actually preferred, if it is a reasonable approximation to the total number of pages. Format: PAGES:: &lt;number&gt; Example:  PAGES:: 48 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="copyright" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Copyright</Attribute>
+            <Attribute name="definition">Copyright information</Attribute>
+            <Attribute name="comment">Copyright information. Open ended format. The COPYRIGHT field applies to the cited report, rather than to the current bibliographic record. Format:  COPYRIGHT:: &lt;free-text&gt; Example: COPYRIGHT:: Copyright for the report (c) 1991, by J. A. Finnegan. All rights reserved. Permission is granted for any academic use of the report. </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="handle" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Handle</Attribute>
+            <Attribute name="definition">Handles are unique identifiers that are used to rerieve location data.</Attribute>
+            <Attribute name="comment">Handles are unique permanent identifiers that are  used in the Handle Management System to retrieve location data. A handle is a printable string which when given to a handle server returns the location of the data. Handles are used to identify digital objects stored within a digital library. If the technical report is available in electronic form, the Handle MUST be supplied in the bibliographic record. Format is &quot;HANDLE:: hdl:&lt;naming authority&gt;/string  of characters&quot;. The string of characters can be the report number of the technical report as assigned by the publisher. For more information on handles and handle servers see the CNRI WEB page at http://www.cnri.reston.va.us. NOTE: White space in HANDLE due to line wrap is ignored. Format: HANDLE:: hdl:&lt;naming authority&gt;/string of characters Example: HANDLE:: hdl:oceanview.electr/CS-TR-91-123</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="other_access" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Other_access</Attribute>
+            <Attribute name="definition">Other access information like URLs and URNs</Attribute>
+            <Attribute name="comment">For URLs, URNs, and other yet to be invented  formatted retrieval systems. Only one URL or URN per occurrence of the field. URL and URN information is available in the internet drafts from the IETF (Internet Engineering Task Force). The most recent drafts can be found on the CNRI WEB page at http://www.cnri.reston.va.us. NOTE: White space in a URL or URN due to line wrap is ignored. Format: OTHER_ACCESS:: URL:&lt;URL&gt; OTHER_ACCESS:: URN:&lt;URN&gt; Example: OTHER_ACCESS:: URL:http://elib.stanford.edu/Document/STANFORD.CS:CS-TN-94-1 Example: OTHER_ACCESS:: URL:ftp://JUPITER.CS.OUKS.EDU/PUBS/computerization.txt. When the URN standard is finalized naming authorities will be registered and URNs will be viable unique identifiers. Until then this is a place holder. For the latest URN drafts see CNRI WEB page at http://www.cnri.reston.va.us </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="retrieval" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Retrieval</Attribute>
+            <Attribute name="definition">Describes how to get a copy of the full text</Attribute>
+            <Attribute name="comment">Open-ended format describing how to get a copy of the full text. This is an optional, repeatable field. No limitations are placed on the dissemination of the bibliographic records. If there are limitations on the dissemination of the publication, it should be protected by some means such as passwords. This format does not address this protection. Format:  RETRIEVAL:: &lt;free-text&gt; RETRIEVAL:: for full text with color pictures send a self-addressed stamped envelope to Prof. J.A. Finnegan, CS Dept, Oceanview University, Oceanview, KS 54321 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="keyword" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Keyword</Attribute>
+            <Attribute name="definition">Keywords</Attribute>
+            <Attribute name="comment">Specify any keywords, controlled or uncontrolled. This is an optional, repeatable field. Multiple keywords are entered using multiple lines in the form of  &quot;KEYWORD::  &lt;free-text&gt;.  Format: KEYWORD:: &lt;free-text&gt; Example: KEYWORD:: Scientific Communication KEYWORD:: Communication Theory </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="cr-category" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Cr-category</Attribute>
+            <Attribute name="definition">The Computer Reviews Category index</Attribute>
+            <Attribute name="comment">Specify the CR-category.  The CR-category (the Computer Reviews Category) index (e.g., &quot;B.3&quot;) should always be included, optionally followed by the name of that category. If the name is specified it should be fully specified with parent levels as needed to clarify it, as in the second example below. Use multiple lines for multiple categories. Every year, the January issue of CR has the full list of these categories, with a detailed discussion of the CR Classification System, and a full index. Typically the full index appears in every January issue, and the top two levels in every issue. Format: CR-CATEGORY:: &lt;free-text&gt; Example: CR-CATEGORY:: D.1 Example: CR-CATEGORY:: B.3 Hardware, Memory Structures </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="period" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Period</Attribute>
+            <Attribute name="definition">Time period covered</Attribute>
+            <Attribute name="comment">Time period covered (date range). Applicable primarily to progress reports, etc. Any format is acceptable, as long as the two dates are separated with &quot; to &quot; (the word &quot;to&quot; surrounded by spaces) and each date is in the format allowed for dates, as described above for the date field. Format: PERIOD:: &lt;date&gt; to &lt;date&gt;  Example:  PERIOD:: January 1990 to March 1990</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="series" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Series</Attribute>
+            <Attribute name="definition">Series title</Attribute>
+            <Attribute name="comment">Series title, including volume number within series. Open-ended format, with producing institution strongly  encouraged to be internally consistent. Format: SERIES:: &lt;free-text&gt;  Example: SERIES:: Communication </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="monitoring" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Monitoring</Attribute>
+            <Attribute name="definition">The monitoring organization(s)</Attribute>
+            <Attribute name="comment">The name(s) of the monitoring organization(s). Format: MONITORING:: &lt;free-text&gt; Example: MONITORING:: ONR </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="funding" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Funding</Attribute>
+            <Attribute name="definition">The funding organization(s)</Attribute>
+            <Attribute name="comment">The name(s) of the funding organization(s). Format: FUNDING:: &lt;free-text&gt; Example: FUNDING:: ARPA </Attribute>
+            <Attribute name="comment">The name(s) of the funding organization(s). Format: FUNDING:: &lt;free-text&gt; Example: FUNDING:: ARPA </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="contract" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Contract</Attribute>
+            <Attribute name="definition">The contract number(s)</Attribute>
+            <Attribute name="comment">The contract number(s). Format: CONTRACT:: &lt; free-text&gt;  Example: CONTRACT:: MMA-90-23-456 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="grant" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Grant</Attribute>
+            <Attribute name="definition">The grant number(s)</Attribute>
+            <Attribute name="comment">The grant number(s). Format: GRANT:: &lt;free-text&gt; Example:  GRANT:: NASA-91-2345</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="language" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Language</Attribute>
+            <Attribute name="definition">The language in which the report is written</Attribute>
+            <Attribute name="comment">The language in which the report is written. Please use the full English name of that language. Please include the Abstract in English, if possible. If the language is not specified, English is assumed. Format: LANGUAGE:: &lt;free-text&gt; Example: LANGUAGE:: English  Example: LANGUAGE:: French</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="notes" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Notes</Attribute>
+            <Attribute name="definition">Miscellaneous free text</Attribute>
+            <Attribute name="comment">Miscellaneous free text. Format: NOTES:: &lt;free-text&gt; Example: NOTES:: This report is the full version of the paper with the same title in IEEE Trans ASSP Dec 1976 </Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+    <Element name="abstract" occurances="58" remember_values="true">
+        <!-- English -->
+        <Language code="en">
+            <Attribute name="identifier">Abstract</Attribute>
+            <Attribute name="definition">The abstract of the report</Attribute>
+            <Attribute name="comment">Highly recommended, but not mandatory. Even though no limit is defined for its length, it is suggested not to expect applications to be able to handle more than 10,000 characters. The ABSTRACT is expected to be used for subject searching since titles are not enough. Even if the report is not in English, an English ABSTRACT is preferable. If no formal abstract appears on document, the producers of the bibliographic records are encouraged to use pieces of the introduction, first paragraph, etc. Format:  ABSTRACT:: xxxx .............. xxxxxxxx</Attribute>
+        </Language>
+        <!-- Non-language specific -->
+        <Attribute name="version">1.0</Attribute>
+        <Attribute language="en" name="registration_authority">The Internet Engineering Task Force</Attribute>
+        <Attribute name="obligation">Optional</Attribute>
+        <Attribute name="datatype">Character String</Attribute>
+        <Attribute name="maximum_occurance">Unlimited</Attribute>
+    </Element>
+</MetadataSet>
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/Configuration.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/Configuration.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/Configuration.java	(revision 21112)
@@ -0,0 +1,1003 @@
+/**
+ *#########################################################################
+ * Configuration
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.io.*;
+import java.lang.ref.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.*;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class stores the various configurable settings inside the Gatherer, both during a session, and between sessions in the form of XML. However not all data members are retained during xml serialization. To further improve efficiency, the property-name -> DOM Element pairs are stored in a SoftReferenced Hashtable.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Configuration
+    extends Hashtable {
+
+    static final public boolean COLLECTION_SPECIFIC = true;
+    static final public boolean GENERAL_SETTING = true;
+
+    static final public int ASSISTANT_MODE = 1;
+    static final public int LIBRARIAN_MODE = 2;
+    static final public int SYSTEMS_MODE   = 3;
+    static final public int EXPERT_MODE    = 4; 
+
+    /** The string identifying an argument's name attribute. */
+    static final private String ARGUMENT_NAME = "name";
+    /** The name of the general Gatherer configuration file. */
+    static public String CONFIG_XML = "config.xml";
+    /** The name of the general Gatherer configuration file for running with GS3. */
+    static public String GS3_CONFIG_XML = "config3.xml";
+
+    /** The name of the root element of the subtree containing gatherer configuration options. This is required as the document itself may contain several other subtrees of settings (such as in the case of a '.col' file). */
+    static final private String GATHERER_CONFIG = "GathererConfig";
+    /** The string identifying an argument element. */
+    static final private String GATHERER_CONFIG_ARGUMENT = "Argument";
+
+    static final private String GENERAL_EMAIL_SETTING = "general.email";
+    /** the lang attribute of the other element */
+    static final private String LANGUAGE = "lang";
+    /** The name of a Name Element. */
+    static final private String NAME = "Name";
+    /** The name of the other arguments element. */
+    static final private String OTHER = "Other";
+    /** The name of an information Element within the Other subtree. */
+    static final private String OTHER_INFO = "Info";
+    /** The name of the general Gatherer configuration template. */
+    static public String TEMPLATE_CONFIG_XML = "xml/config.xml";
+    /** The first of three patterns used during tokenization, this pattern handles a comma separated list. */
+    static final private String TOKENIZER_PATTERN1 = " ,\n\t";
+    /** The second of three patterns used during tokenization, this pattern handles an underscore separated list. */
+    static final private String TOKENIZER_PATTERN2 = "_\n\t";
+    /** The last of three patterns used during tokenization, this pattern handles an comma separated list containing spaces. */
+    static final private String TOKENIZER_PATTERN3 = ",\n\t";
+
+    static public Configuration self = null;
+    static private String gli_user_directory_path = null;
+
+    static public boolean just_updated_config_xml_file = false;
+    /** The path to the Greenstone Suite installation directory. */
+    static public String gsdl_path = "";
+    /** If we are using GLI in Greenstone 3, the path to gsdl3 home (== web dir) */
+    static public String gsdl3_path = "";
+    /** If we are using GLI in Greenstone 3, the path to gsdl3 src directory (== gsdl3 dir)*/
+    static public String gsdl3_src_path = "";
+    /** The path to the PERL executable, up to and including Perl.exe. */
+    static public String perl_path = "";
+    /** The password for the proxy server indicated above. */
+    static public String proxy_pass = null;
+    /** The username for the proxy server indicated above. */
+    static public String proxy_user = null;
+    /** The screen size of the desktop the Gatherer will be displayed on. */
+    static public Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+    /** The site name if we are using GS3 */
+    static public String site_name = "";
+    /** The servlet path if we are using GS3 */
+    static public String servlet_path = "";
+    /** Collection level configuration (which in some cases overrides general configuration. */
+    static private Document collection_config;
+    /** The general configuration settings. */
+    static private Document general_config;
+
+    /** The element, from glis config file, that contains the various directory mappings. */
+    static private Element directory_mappings_element;
+
+    static private int cache_hit = 0;
+    static private int cache_miss = 0;
+
+    static private String wget_path = null;
+    static private String wget_version_str = StaticStrings.NO_WGET_STR;
+
+    static public URL gliserver_url = null;
+    static public URL library_url = null;
+
+
+    /** Constructor.
+     * @param gsdl_path The path to the Greenstone directory as a <strong>String</strong>.
+     * @param gsdl3_path The path to the Greenstone 3 home (web) directory as a <strong>String</strong>.
+     * @param gsdl3_src_path The path to the Greenstone 3 src directory as a <strong>String</strong>.
+     * @param site_name The name of the Greenstone 3 site currently in use.
+     */
+    public Configuration(String gli_user_directory_path, String gsdl_path, String gsdl3_path, String gsdl3_src_path, String site_name)
+    {
+	super();
+	self = this;
+
+	this.gli_user_directory_path = gli_user_directory_path;
+	this.gsdl_path = gsdl_path;
+	this.gsdl3_path = gsdl3_path;
+	this.gsdl3_src_path = gsdl3_src_path;
+	this.site_name = site_name;
+
+	// Try to load the configuration file from the user specific location
+	String config_xml_file_name = CONFIG_XML;
+	if (gsdl3_path != null) {
+	    config_xml_file_name = GS3_CONFIG_XML;
+	}
+
+	// If the existing user config.xml file isn't recent enough, backup the old version and update it
+	File config_xml_file = new File(gli_user_directory_path + config_xml_file_name);
+	if (config_xml_file != null && config_xml_file.exists()) {
+	    just_updated_config_xml_file = updateUserConfigXMLFileIfNecessary(config_xml_file);
+	    general_config = XMLTools.parseXMLFile(config_xml_file);
+	}
+
+	// Create a new config from the default configuration file from our xml library
+	if (general_config == null) {
+	    general_config = XMLTools.parseXMLFile(TEMPLATE_CONFIG_XML, true);
+	    just_updated_config_xml_file = true;
+	}
+
+	// Determine the Directory Mappings element
+	directory_mappings_element = (Element) XMLTools.getNodeFromNamed(general_config.getDocumentElement(), StaticStrings.DIRECTORY_MAPPINGS_ELEMENT);
+
+	// Re-establish the color settings.
+	updateUI();
+
+	// Read the Greenstone library URL from the config file
+	String library_url_string = getString("general.library_url", true);
+	if (!library_url_string.equals("")) {
+	    try {
+		library_url = new URL(library_url_string);
+	    }
+	    catch (MalformedURLException exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+
+	// Read the Greenstone gliserver URL from the config file, if using a remote Greenstone
+	if (Gatherer.isGsdlRemote) {
+	    String gliserver_url_string = getString("general.gliserver_url", true);
+	    if (!gliserver_url_string.equals("")) {
+		try {
+		    gliserver_url = new URL(gliserver_url_string);
+		}
+		catch (MalformedURLException exception) {
+		    DebugStream.printStackTrace(exception);
+		}
+	    }
+	}
+
+	if (gsdl3_path != null) {
+	    if (this.site_name == null || this.site_name.equals("")) {
+		this.site_name = getString("general.site_name", true);
+		this.servlet_path = getString("general.servlet_path", true);
+		if (this.site_name.equals("")) {
+		    this.site_name = "localsite"; // can we assume these??
+		    this.servlet_path = "/library";
+		    setString("general.site_name", true, this.site_name);
+		    setString("general.servlet_path", true, this.servlet_path);
+		}
+		
+	    } else {
+		// we set the current one in the config
+		setString("general.site_name", true, this.site_name);
+		if (this.servlet_path != null && !this.servlet_path.equals("")) {
+		    setString("general.servlet_path", true, this.servlet_path);
+		}
+	    }
+	}
+    }
+
+
+    static private boolean updateUserConfigXMLFileIfNecessary(File config_xml_file)
+    {
+	// Check if the configuration file actually needs updating by looking at the version numbers
+	Document new_document = XMLTools.parseXMLFile(TEMPLATE_CONFIG_XML, true);
+	Document old_document = XMLTools.parseXMLFile(config_xml_file);
+
+	String new_version = new_document.getDocumentElement().getAttribute(StaticStrings.VERSION_ATTRIBUTE);
+	String old_version = old_document.getDocumentElement().getAttribute(StaticStrings.VERSION_ATTRIBUTE);
+	if (new_version.compareTo(old_version) <= 0) {
+	    // Don't need to update file
+	    return false;
+	}
+
+	System.err.println("Updating user config.xml from version " + old_version + " to version " + new_version + "...");
+
+	// Build up the new user config.xml file from the template config.xml with the user's preferences added
+	// Make a backup of the old config.xml file first
+	config_xml_file.renameTo(new File(config_xml_file.getAbsolutePath() + ".old"));
+
+	// Check all of the Argument elements in the new config.xml file
+	NodeList new_argument_elements_nodelist = new_document.getElementsByTagName("Argument");
+	NodeList old_argument_elements_nodelist = old_document.getElementsByTagName("Argument");
+	for (int i = 0; i < new_argument_elements_nodelist.getLength(); i++) {
+	    Element new_argument_element = (Element) new_argument_elements_nodelist.item(i);
+	    String new_argument_element_name = new_argument_element.getAttribute("name");
+	    String new_argument_element_value = XMLTools.getElementTextValue(new_argument_element);
+
+	    // Did this Argument have a non-default value set?
+	    for (int j = old_argument_elements_nodelist.getLength() - 1; j >= 0; j--) {
+		Element old_argument_element = (Element) old_argument_elements_nodelist.item(j);
+		String old_argument_element_name = old_argument_element.getAttribute("name");
+
+		// Argument found
+		if (old_argument_element_name.equals(new_argument_element_name)) {
+		    String old_argument_element_value = XMLTools.getElementTextValue(old_argument_element);
+		    if (!old_argument_element_value.equals(new_argument_element_value)) {
+			XMLTools.setElementTextValue(new_argument_element, old_argument_element_value);
+		    }
+
+		    // We don't care about this option any more
+		    old_argument_element.getParentNode().removeChild(old_argument_element);
+		    break;
+		}
+	    }
+	}
+
+	// Are there any old Argument elements left over? Keep these around for old time's sake
+	old_argument_elements_nodelist = old_document.getElementsByTagName("Argument");
+	if (old_argument_elements_nodelist.getLength() > 0) {
+	    NodeList new_gathererconfig_elements_nodelist = new_document.getElementsByTagName("GathererConfig");
+	    if (new_gathererconfig_elements_nodelist.getLength() > 0) {
+		Element new_gathererconfig_element = (Element) new_gathererconfig_elements_nodelist.item(0);
+		new_gathererconfig_element.appendChild(new_document.createComment(" Legacy options "));
+
+		// Add the legacy Arguments to the GathererConfig element
+		for (int i = 0; i < old_argument_elements_nodelist.getLength(); i++) {
+		    Element old_argument_element = (Element) old_argument_elements_nodelist.item(i);
+		    new_gathererconfig_element.appendChild(new_document.importNode(old_argument_element, true));
+		}
+	    }
+	}
+
+	// Keep all the Mapping elements
+	NodeList new_directorymappings_elements_nodelist = new_document.getElementsByTagName("DirectoryMappings");
+	if (new_directorymappings_elements_nodelist.getLength() > 0) {
+	    Element new_directorymappings_element = (Element) new_directorymappings_elements_nodelist.item(0);
+
+	    NodeList old_mapping_elements_nodelist = old_document.getElementsByTagName("Mapping");
+	    for (int i = 0; i < old_mapping_elements_nodelist.getLength(); i++) {
+		Element old_mapping_element = (Element) old_mapping_elements_nodelist.item(i);
+		new_directorymappings_element.appendChild(new_document.importNode(old_mapping_element, true));
+	    }
+	}
+
+	// Check all of the Associations elements in the new config.xml file
+	NodeList new_entry_elements_nodelist = new_document.getElementsByTagName("Entry");
+	NodeList old_entry_elements_nodelist = old_document.getElementsByTagName("Entry");
+	for (int i = 0; i < new_entry_elements_nodelist.getLength(); i++) {
+	    Element new_entry_element = (Element) new_entry_elements_nodelist.item(i);
+	    String new_entry_element_name = new_entry_element.getAttribute("extension");
+	    String new_entry_element_value = XMLTools.getElementTextValue(new_entry_element);
+
+	    // Did this Entry have a non-default value set?
+	    for (int j = old_entry_elements_nodelist.getLength() - 1; j >= 0; j--) {
+		Element old_entry_element = (Element) old_entry_elements_nodelist.item(j);
+		String old_entry_element_name = old_entry_element.getAttribute("extension");
+
+		// Entry found
+		if (old_entry_element_name.equals(new_entry_element_name)) {
+		    String old_entry_element_value = XMLTools.getElementTextValue(old_entry_element);
+		    if (!old_entry_element_value.equals(new_entry_element_value)) {
+			XMLTools.setElementTextValue(new_entry_element, old_entry_element_value);
+		    }
+
+		    // We don't care about this option any more
+		    old_entry_element.getParentNode().removeChild(old_entry_element);
+		    break;
+		}
+	    }
+	}
+
+	// Are there any old Entry elements left over? We want to keep these
+	old_entry_elements_nodelist = old_document.getElementsByTagName("Entry");
+	if (old_entry_elements_nodelist.getLength() > 0) {
+	    NodeList new_associations_elements_nodelist = new_document.getElementsByTagName("Associations");
+	    if (new_associations_elements_nodelist.getLength() > 0) {
+		Element new_associations_element = (Element) new_associations_elements_nodelist.item(0);
+
+		// Add the legacy Entries to the Associations element
+		for (int i = 0; i < old_entry_elements_nodelist.getLength(); i++) {
+		    Element old_entry_element = (Element) old_entry_elements_nodelist.item(i);
+		    new_associations_element.appendChild(new_document.importNode(old_entry_element, true));
+		}
+	    }
+	}
+
+	// Write out the updated config.xml file
+	XMLTools.writeXMLFile(config_xml_file, new_document);
+	return true;
+    }
+
+
+    /** Add a special directory mapping. */
+    static public boolean addDirectoryMapping(String name, File file) {
+	boolean result = false;
+	try {
+	    // Ensure the name isn't already in use.
+	    boolean found = false;
+	    NodeList mappings = directory_mappings_element.getElementsByTagName(StaticStrings.MAPPING_ELEMENT);
+	    for(int i = 0; !found && i < mappings.getLength(); i++) {
+		Element mapping_element = (Element) mappings.item(i);
+		if(mapping_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equalsIgnoreCase(name)) {
+		    found = true;
+		}
+		mapping_element = null;
+	    }
+	    // Otherwise add the mapping.
+	    if(!found) {
+		Element mapping_element = general_config.createElement(StaticStrings.MAPPING_ELEMENT);
+		mapping_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
+		mapping_element.setAttribute(StaticStrings.FILE_ATTRIBUTE, file.toString());
+		directory_mappings_element.appendChild(mapping_element);
+		result = true;
+		mapping_element = null;
+	    }
+	    mappings = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    } /** addDirectoryMapping(String name, String file) **/
+
+    /** The default get action retrieves the named property from the desired configuration, and returns a true or false. */
+    static public boolean get(String property, boolean general) {
+	String raw = getString(property, general);
+	return (raw != null && raw.equalsIgnoreCase("true"));
+    }
+
+    /** Retrieve all of the configuration preferences which match a certain string. They are returned as a hash map of property names to String objects. */
+    static public HashMap getAll(String property_pattern, boolean general) {
+	HashMap properties = new HashMap();
+	try {
+	    // Locate the appropriate element
+	    Element document_element = null;
+	    if(general) {
+		document_element = general_config.getDocumentElement();
+	    }
+	    else if(collection_config != null) {
+		document_element = collection_config.getDocumentElement();
+	    }
+	    if(document_element != null) {
+		// Retrieve the Gatherer element
+		Element gatherer_element = (Element) XMLTools.getNodeFromNamed(document_element, GATHERER_CONFIG);
+		NodeList arguments = gatherer_element.getElementsByTagName(GATHERER_CONFIG_ARGUMENT);
+		for(int i = 0; i < arguments.getLength(); i++) {
+		    Element argument_element = (Element) arguments.item(i);
+		    if(argument_element.getAttribute(ARGUMENT_NAME).matches(property_pattern)) {
+			String result = XMLTools.getValue(argument_element);
+			// Store a mapping in the cache. Sometimes we will overwrite an existing value (say for collection and general level workflow options) but the processing overhead of detecting these clashes far exceeds any savings.
+			self.put(argument_element.getAttribute(ARGUMENT_NAME) + general, new SoftReference(argument_element));
+			// Add mapping to the properties we're going to return
+			properties.put(argument_element.getAttribute(ARGUMENT_NAME), result);
+		    }
+		}
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return properties;
+    }
+
+    /** Retrieve the information subtree containing the arguments for the desired external program. If the program has marked superclasses append their arguments as well. */
+    static public Element getArguments(String filename) {
+	Element argument_element = null;
+	try {
+	    // Retrieve the other information subtree.
+	    Element document_element = general_config.getDocumentElement();
+	    Element other_element = (Element) XMLTools.getNodeFromNamed(document_element, OTHER);
+	    NodeList argument_elements = other_element.getElementsByTagName(OTHER_INFO);
+	    for(int i = 0; argument_element == null && i < argument_elements.getLength(); i++) {
+		Element possible_element = (Element) argument_elements.item(i);
+		Element possible_name_element = (Element) XMLTools.getNodeFromNamed(possible_element, NAME);
+		String possible_name = XMLTools.getValue(possible_name_element);
+		///ystem.err.println("Does " + possible_name + " equal " + filename);
+		if(possible_name.equalsIgnoreCase(filename)) {
+		    argument_element = possible_element;
+		}
+		possible_name = null;
+		possible_name_element = null;
+		possible_element = null;
+	    }
+	    argument_elements = null;
+	    other_element = null;
+	    document_element = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return argument_element;
+    }
+
+    /** Gets the language for the other arguments subtree 
+     @return empty string if no language specified */
+    static public String getArgumentsLanguage() {
+	Element document_element = general_config.getDocumentElement();
+	Element other_element = (Element) XMLTools.getNodeFromNamed(document_element, OTHER);
+	String lang = other_element.getAttribute(LANGUAGE);
+	if (lang == null) {
+	    lang = "en";
+	}
+	return lang;
+    }
+
+    /** Retrieve the value of the named property as a Rectangle. */
+    static public Rectangle getBounds(String property, boolean general) {
+	Rectangle result = null;
+	try {
+	    String raw = getString(property, general);
+	    // Rectangle is (x, y, width, height)
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN1);
+	    int x = Integer.parseInt(tokenizer.nextToken());
+	    int y = Integer.parseInt(tokenizer.nextToken());
+	    int width = Integer.parseInt(tokenizer.nextToken());
+	    int height = Integer.parseInt(tokenizer.nextToken());
+	    result = new Rectangle(x, y, width, height);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    }
+
+    /** Retrieve the value of the named property as a Color. */
+    static public Color getColor(String property, boolean general) {
+	Color result = Color.white; // Default
+	try {
+	    String raw = getString(property, general);
+	    // Color is a RGB triplet list, comma separated (also remove whitespace)
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN1);
+	    int red = Integer.parseInt(tokenizer.nextToken());
+	    int green = Integer.parseInt(tokenizer.nextToken());
+	    int blue = Integer.parseInt(tokenizer.nextToken());
+	    result = new Color(red, green, blue);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    }
+
+
+    /** Retrieve the special directory mappings associated with this collection.
+     * @return A <strong>HashMap</strong> containing mappings from names to directories.
+     */
+    static public HashMap getDirectoryMappings() {
+	HashMap special_directories = new HashMap();
+	try {
+	    // Ensure the name isn't already in use.
+	    boolean found = false;
+	    NodeList mappings = directory_mappings_element.getElementsByTagName(StaticStrings.MAPPING_ELEMENT);
+	    for(int i = 0; !found && i < mappings.getLength(); i++) {
+		Element mapping_element = (Element) mappings.item(i);
+		String name = mapping_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+		File file = new File(mapping_element.getAttribute(StaticStrings.FILE_ATTRIBUTE));
+		special_directories.put(name, file);
+		file = null;
+		name = null;
+		mapping_element = null;
+	    }
+	    mappings = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return special_directories;
+    } /** getDirectoryMappings() */
+
+    /** Retrieve the current users email. These are always stored in the general settings.
+     * @return the email address, if it is set, as a String
+     */
+    static public String getEmail() {
+	String email = getString(GENERAL_EMAIL_SETTING, true);
+	return (email.length() > 0 ? email : null);
+    }
+
+    static public Element getFileAssociations() {
+	NodeList file_association_elements = general_config.getDocumentElement().getElementsByTagName(StaticStrings.ASSOCIATIONS_ELEMENT);
+	return (Element) file_association_elements.item(0);
+    }
+
+    /** Retrieve the value of the named property as a FontUIResource. */
+    static public FontUIResource getFont(String property, boolean general) {
+	FontUIResource result = new FontUIResource("Times New Roman", Font.PLAIN, 10);
+	try {
+	    String raw = getString(property, general);
+	    // Font is a face, style, size triplet.
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN3);
+	    String face = tokenizer.nextToken().trim();
+	    ///ystem.err.println("Face:  " + face);
+	    int style = Font.PLAIN;
+	    String temp = tokenizer.nextToken().toUpperCase().trim();
+	    ///ystem.err.println("Style: " + temp);
+	    if(temp.equals("BOLD")) {
+		style = Font.BOLD;
+	    }
+	    else if(temp.equals("ITALIC")) {
+		style = Font.ITALIC;
+	    }
+	    int size = Integer.parseInt(tokenizer.nextToken().trim());
+	    ///ystem.err.println("Size:  " + size);
+	    result = new FontUIResource(face, style, size);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    }
+
+    /** Retrieve the value of the named property as an integer. */
+    static public int getInt(String property, boolean general) {
+	int result = 0;
+	try {
+	    String raw = getString(property, general);
+	    result = Integer.parseInt(raw);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    }
+
+    /** Retrieves the current interface language two letter code. */
+    static public String getLanguage() {
+	Locale locale = getLocale("general.locale", GENERAL_SETTING);
+	String code = "en"; // Default
+	if(locale != null) {
+	    code = locale.getLanguage();
+	}
+	return code;
+    }
+
+    /** Retrieve the value of the named property as a Locale. */
+    static public Locale getLocale(String property, boolean general) {
+	Locale result = Locale.getDefault();
+	try {
+	    String raw = getString(property, general);
+	    // Locale is a underscore separated code.
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN2);
+	    String language = tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		String country = tokenizer.nextToken();
+		result = new Locale(language, country);
+	    }
+	    else {
+		result = new Locale(language);
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return result;
+    }
+
+    /** Because modes will soon be an integral part of GLI, they get their own easy to remember methods such as this one to get the mode.
+     * @return an int representing the mode
+     */
+    static public int getMode() {
+	return getInt("general.mode", GENERAL_SETTING);
+    }
+
+    /** Return the current mode as a string for use in title bar etc
+     * @return the mode as a String
+     */
+    static public String getModeAsString() {
+	String result;
+	switch(getInt("general.mode", GENERAL_SETTING)) {
+	case ASSISTANT_MODE:
+	    result = Dictionary.get("Preferences.Mode.Assistant");
+	    break;
+	case SYSTEMS_MODE:
+	    result = Dictionary.get("Preferences.Mode.Systems");
+	    break;
+	case EXPERT_MODE:
+	    result = Dictionary.get("Preferences.Mode.Expert");
+	    break;
+	default:
+	    result = Dictionary.get("Preferences.Mode.Librarian");
+	}
+	return result;
+    }
+
+    static public String getPreviewCommand() {
+	return getString("general.preview_program", GENERAL_SETTING);
+    }
+
+    static public String getServletPath() {
+	return servlet_path;
+    }
+
+    /** Retrieve the value of the named property, and noting whether we consult the general or collection specific configuration. */
+    static public String getString(String property, boolean general) {
+	// Its up to this method to find the appropriate node and retrieve the data itself.
+	String result = "";
+	try {
+	    // First of all we look in the cache to see if we have a match.
+	    SoftReference reference = (SoftReference) self.get(property + general);
+	    if(reference != null) {
+		Element argument_element = (Element) reference.get();
+		if(argument_element != null) {
+		    cache_hit++;
+		    result = XMLTools.getValue(argument_element);
+		}
+	    }
+	    // We may have missed in the cache, or the reference may have been consumed.
+	    if(result.length() == 0) {
+		cache_miss++;
+		// Locate the appropriate element
+		Element document_element = null;
+		if(general) {
+		    document_element = general_config.getDocumentElement();
+		}
+		else if(collection_config != null) {
+		    document_element = collection_config.getDocumentElement();
+		}
+		if(document_element != null) {
+		    // Retrieve the Gatherer element
+		    Element gatherer_element = (Element) XMLTools.getNodeFromNamed(document_element, GATHERER_CONFIG);
+		    NodeList arguments = gatherer_element.getElementsByTagName(GATHERER_CONFIG_ARGUMENT);
+		    for(int i = 0; result.length() == 0 && i < arguments.getLength(); i++) {
+			Element argument_element = (Element) arguments.item(i);
+			if(argument_element.getAttribute(ARGUMENT_NAME).equalsIgnoreCase(property)) {
+			    result = XMLTools.getValue(argument_element);
+			    // Store a mapping in the cache. Sometimes we will overwrite an existing value (say for collection and general level workflow options) but the processing overhead of detecting these clashes far exceeds any savings.
+			    self.put(property + general, new SoftReference(argument_element));
+			}
+		    }
+		}
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	// If we still have no result, and the search was made in the collection configuration, retrieve the general one instead.
+	if(result.length() == 0 && !general) {
+	    result = getString(property, true);
+	}
+	return result;
+    }
+
+    static public String getGS3ScriptPath() {
+	return gsdl3_src_path + "bin" + File.separator + "script" + File.separator;
+    }
+
+    static public String getWGetPath() {
+	return (wget_path != null ? wget_path : "");
+    }
+
+    static public String getWGetVersion() {
+	return (wget_version_str != null ? wget_version_str : StaticStrings.NO_WGET_STR);
+    }
+
+    /** Remove a previously defined special directory mapping.
+     * @param name The name of the mapping to remove as a <strong>String</strong>.
+     * @return The <strong>File</strong> of the mapping removed.
+     */
+    static public File removeDirectoryMapping(String name) {
+	File file = null;
+	try {
+	    // Ensure the name isn't already in use.
+	    boolean found = false;
+	    NodeList mappings = directory_mappings_element.getElementsByTagName(StaticStrings.MAPPING_ELEMENT);
+	    for(int i = 0; !found && i < mappings.getLength(); i++) {
+		Element mapping_element = (Element) mappings.item(i);
+		if(mapping_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equalsIgnoreCase(name)) {
+		    file = new File(XMLTools.getValue(mapping_element));
+		    directory_mappings_element.removeChild(mapping_element);
+		    found = true;
+		}
+		mapping_element = null;
+	    }
+	    mappings = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return file;
+    } /** removeDirectoryMapping(String name) */
+
+    /** Export the general configuration to file. */
+    static public void save() {
+	///ystem.err.println("Hits " + cache_hit + " vs Misses " + cache_miss);
+	// We first try exporting to a user specific place
+	File user_config_xml = null;
+	String config_xml_name = CONFIG_XML;
+	if (gsdl3_path!=null) {
+	    config_xml_name = GS3_CONFIG_XML;
+	}
+	try {
+	    user_config_xml = new File(gli_user_directory_path + config_xml_name);
+	    ///ystem.err.println("Trying to save to: " + user_config_xml.getAbsolutePath());
+	    ///ystem.err.println("Writing.");
+	    user_config_xml.getParentFile().mkdirs();
+	    XMLTools.writeXMLFile(new File(user_config_xml.getAbsolutePath()), general_config);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	    user_config_xml = null;
+	}
+    }
+
+    /** Set the named property, from the specified configuration, using the given boolean value. */
+    static public void set(String property, boolean general, boolean value) {
+	if(property.startsWith("workflow")) {
+	    DebugStream.println("Set property: " + property + ", general=" + general + ", value=" + value);
+	}
+	setString(property, general, (value ? "true" : "false"));
+    }
+
+    /** Add a subtree of argument information to the other arguments part of the general configuration. This overwrites any such existing subtree. */
+    static public void setArguments(Element arguments_element) {
+	try {
+	    Element document_element = general_config.getDocumentElement();
+	    Element other_element = (Element) XMLTools.getNodeFromNamed(document_element, OTHER);
+	    // Retrieve the name of the information
+	    Element arguments_name_element = (Element)XMLTools.getNodeFromNamed(arguments_element, NAME);
+	    String filename = XMLTools.getValue(arguments_element);
+	    // Find any argument information subtree starting with the same name
+	    Element obsolete_arguments_element = getArguments(filename);
+	    // Create a copy of the arguments_element within our tree (import).
+	    Element our_arguments_element = (Element) general_config.importNode(arguments_element, true);
+	    // Now we insert this new node into the tree. If a previous node exists we replace it instead.
+	    if(obsolete_arguments_element == null) {
+		other_element.appendChild(our_arguments_element);
+	    }
+	    else {
+		other_element.replaceChild(our_arguments_element, obsolete_arguments_element);
+	    }
+	    our_arguments_element = null;
+	    obsolete_arguments_element = null;
+	    filename = null;
+	    arguments_name_element = null;
+	    other_element = null;
+	    document_element = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.println("Error in Configuration.setArguments(): " + exception);
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+    /** Sets the language for the other arguments subtree */
+    static public void setArgumentsLanguage(String lang) {
+	Element document_element = general_config.getDocumentElement();
+	Element other_element = (Element) XMLTools.getNodeFromNamed(document_element, OTHER);
+	other_element.setAttribute(LANGUAGE, lang);
+    }
+
+	
+    /** Set the collection configuration. */
+    static public void setCollectionConfiguration(Document collection_config) {
+	// clear the cached values
+	self.clear();
+	collection_config = collection_config;
+	updateUI();
+	///atherer.println("Collection configuration set.");
+    }
+
+    /** Set the named property, from the specified configuration, using the given Rectangle value. */
+    static public void setBounds(String property, boolean general, Rectangle value) {
+	StringBuffer text = new StringBuffer("");
+	text.append(value.x);
+	text.append(", ");
+	text.append(value.y);
+	text.append(", ");
+	text.append(value.width);
+	text.append(", ");
+	text.append(value.height);
+	setString(property, general, text.toString());
+    }
+
+    /** Set the named property, from the specified configuration, using the given Color value. */
+    static public void setColor(String property, boolean general, Color value) {
+	StringBuffer text = new StringBuffer("");
+	text.append(value.getRed());
+	text.append(", ");
+	text.append(value.getGreen());
+	text.append(", ");
+	text.append(value.getBlue());
+	setString(property, general, text.toString());
+    }
+
+    /** Establish the current users email.
+     * @param email the email as a String
+     */
+    static public void setEmail(String email) {
+	setString(GENERAL_EMAIL_SETTING, true, email);
+    }
+
+    /** Set the named property, from the specified configuration, using the given Font value. */
+    static public void setFont(String property, boolean general, Font value) {
+	StringBuffer text = new StringBuffer("");
+	text.append(value.getName());
+	text.append(", ");
+	switch(value.getStyle()) {
+	case Font.BOLD:
+	    text.append("BOLD");
+	    break;
+	case Font.ITALIC:
+	    text.append("ITALIC");
+	    break;
+	default:
+	    text.append("PLAIN");
+	}
+	text.append(", ");
+	text.append(value.getSize());
+	setString(property, general, text.toString());
+    }
+
+    /** Set the named property, from the specified configuration, using the given integer value. */
+    static public void setInt(String property, boolean general, int value) {
+        setString(property, general, String.valueOf(value));
+    } 
+
+    /** Set the named property, from the specified configuration, using the given Locale value. */
+    static public void setLocale(String property, boolean general, Locale value) {
+	StringBuffer text = new StringBuffer("");
+	text.append(value.getLanguage());
+	String country = value.getCountry();
+	if(country != null && country.length() > 0) {
+	    text.append("_");
+	    text.append(country);
+	}
+	country = null;
+	setString(property, general, text.toString());
+    }
+
+    /** Because modes will soon be an integral part of GLI, they get their own easy to remember methods such as this one to set the mode.
+     * @param value the new value for mode
+     */
+    static public void setMode(int value) {
+	setInt("general.mode", GENERAL_SETTING, value);
+    }
+
+    static public void setPreviewCommand(String value) {
+	setString("general.preview_program", GENERAL_SETTING, value);
+    }
+
+    static public void setSiteAndServlet(String site, String servlet) {
+	site_name = site;
+	servlet_path = servlet;
+	setString("general.site_name", GENERAL_SETTING, site);
+	setString("general.servlet_path", GENERAL_SETTING, servlet);
+	
+    }
+    /** Sets the value of the named property argument using the given string. */
+    static public void setString(String property, boolean general, String value) {
+	DebugStream.println("Set configuration property: " + property + " = " + value + (general ? "" : " [Collection]"));
+	try {
+	    Document document = general_config;
+	    if(!general && collection_config != null) {
+		document = collection_config;
+	    }
+	    if(document != null) {
+		Element argument_element = null;
+		// Try to retrieve from cache
+		SoftReference reference = (SoftReference) self.get(property + general);
+		if(reference != null) {
+		    argument_element = (Element) reference.get();
+		}
+		if(argument_element == null) {
+		    Element document_element = document.getDocumentElement();
+		    Element gatherer_element = (Element) XMLTools.getNodeFromNamed(document_element, GATHERER_CONFIG);
+		    NodeList arguments = document_element.getElementsByTagName(GATHERER_CONFIG_ARGUMENT);
+		    boolean found = false;
+		    for(int i = 0; argument_element == null && i < arguments.getLength(); i++) {
+			Element possible_element = (Element) arguments.item(i);
+			if(possible_element.getAttribute(ARGUMENT_NAME).equalsIgnoreCase(property)) {
+			    argument_element = possible_element;
+			}
+		    }
+		    // If argument element is still null, create it in the target document.
+		    if(argument_element == null) {
+			argument_element = document.createElement(GATHERER_CONFIG_ARGUMENT);
+			argument_element.setAttribute(ARGUMENT_NAME, property);
+			gatherer_element.appendChild(argument_element);
+		    }
+		    // Update cache
+		    self.put(property + general, new SoftReference(argument_element));
+
+		}
+		if(value == null) {
+		    value = "";
+		}
+		// Now remove any current text node children.
+		NodeList children = argument_element.getChildNodes();
+		for(int i = 0; i < children.getLength(); i++) {
+		    argument_element.removeChild(children.item(i));
+		}
+		// Add a new text node child with the new value
+		argument_element.appendChild(document.createTextNode(value));
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+    static public void setWGetPath(String path) {
+	wget_path  = path;
+    }
+
+    static public void setWGetVersion(String version) {
+	wget_version_str = version;
+    }
+
+    static private void updateUI() {
+	// Buttons
+	UIManager.put("Button.select", new ColorUIResource(getColor("coloring.button_selected_background", false)));
+	UIManager.put("Button.background", new ColorUIResource(getColor("coloring.button_background", false)));
+	UIManager.put("Button.foreground", new ColorUIResource(getColor("coloring.button_foreground", false)));
+
+	UIManager.put("ToggleButton.background", new ColorUIResource(getColor("coloring.button_background", false)));
+	UIManager.put("ToggleButton.foreground", new ColorUIResource(getColor("coloring.button_foreground", false)));
+	UIManager.put("ToggleButton.select", new ColorUIResource(getColor("coloring.button_selected_background", false)));
+
+	// All the things with a lovely Collection green background
+	UIManager.put("OptionPane.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+	UIManager.put("Panel.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+	UIManager.put("Label.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+	UIManager.put("TabbedPane.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+	UIManager.put("SplitPane.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+	UIManager.put("CheckBox.background", new ColorUIResource(getColor("coloring.collection_heading_background", false)));
+
+
+	// Editable coloring
+	UIManager.put("ComboBox.background", new ColorUIResource(getColor("coloring.collection_tree_background", false))); // Indicate clickable
+	UIManager.put("Tree.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("Tree.textBackground", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("ProgressBar.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("TextArea.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("TextField.background", new ColorUIResource(getColor("coloring.editable_background", false)));
+	UIManager.put("Table.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("List.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+	UIManager.put("RadioButton.background", new ColorUIResource(getColor("coloring.collection_tree_background", false)));
+
+	// Selection color
+	UIManager.put("TabbedPane.selected", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("Tree.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("ComboBox.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("ProgressBar.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("TextArea.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("TextField.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+	UIManager.put("List.selectionBackground", new ColorUIResource(getColor("coloring.collection_selection_background", false)));
+
+	// Scroll bar stuff
+	UIManager.put("ScrollBar.background", new ColorUIResource(getColor("coloring.scrollbar_background", false)));
+	UIManager.put("ScrollBar.thumb", new ColorUIResource(getColor("coloring.scrollbar_foreground", false)));
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/DebugStream.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/DebugStream.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/DebugStream.java	(revision 21112)
@@ -0,0 +1,120 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer;
+
+
+import java.io.*;
+import java.util.*;
+
+
+public class DebugStream
+{
+    static private boolean debugging_enabled = false;
+    static private PrintStream debug_stream = null;
+
+
+    static public void closeDebugStream()
+    {
+	if (debug_stream != null) {
+	    try {
+		debug_stream.flush();
+		debug_stream.close();
+	    }
+	    catch (Exception exception) {
+		exception.printStackTrace();
+	    }
+	}
+    }
+
+
+    static public void enableDebugging()
+    {
+	debugging_enabled = true;
+    }
+
+
+    static public boolean isDebuggingEnabled()
+    {
+	return debugging_enabled;
+    }
+
+
+    static synchronized public void print(Properties properties)
+    {
+	if (debugging_enabled) {
+	    if (debug_stream != null) {
+		properties.list(debug_stream);
+	    }
+	    properties.list(System.err);
+	}
+    }
+
+
+    /** Print a message to the debug stream. */
+    static synchronized public void print(String message)
+    {
+	if (debugging_enabled) {
+	    if (debug_stream != null) {
+		debug_stream.print(message);
+	    }
+	    System.err.print(message);
+	}
+    }
+
+
+    /** Print a message to the debug stream. */
+    static synchronized public void println(String message)
+    {
+	if (debugging_enabled) {
+	    if (debug_stream != null) {
+		debug_stream.println(message);
+	    }
+	    System.err.println(message);
+	}
+    }
+
+
+    /** Print a stack trace to the debug stream. */
+    static synchronized public void printStackTrace(Exception exception)
+    {
+	if (debugging_enabled && debug_stream != null) {
+	    exception.printStackTrace(debug_stream);
+	}
+	exception.printStackTrace();
+    }
+
+
+    static public void setDebugFile(String debug_file_path)
+    {
+	try {
+	    debug_stream = new PrintStream(new FileOutputStream(debug_file_path));
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/Dictionary.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/Dictionary.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/Dictionary.java	(revision 21112)
@@ -0,0 +1,364 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.*;
+import javax.swing.text.*;
+import javax.swing.tree.*;
+
+/** Extends the ResourceBundle class to allow for the automatic insertion of arguments.<BR>
+ * <BR>
+ * Property files usable by this class have the Backus-Naur form: <BR>
+ * <BR>
+ * FileLine ::= Comment | Mapping <BR>
+ * Comment  ::= '#' SString <BR>
+ * Mapping  ::= NZString ':' SString ( Argument SString )* <BR>
+ * NZString ::= ( Char | Int ) SString <BR>
+ * Argument ::= '{' Int '}' <BR>
+ * SString  ::= String . ['"','#',...] -> ['\"','\#',...] <BR>
+ * <BR>
+ * In order to add a new dictionary Locale, simply copy the existing dictionary.properties files, replace the values (Strings after the ':') with the new language specific ones being careful to maintain formatting and Gatherer placed arguments, then save the new dictionary as: <br>
+ * <BR>
+ * dictionary_<I>locale</I>.properties<BR>
+ * <BR>
+ * where locale is made of two two-letter codes seperated by an underscore. The first code is in lower-case and defines the language. The second is in upper-case and defines the country. For example the default dictionary could also correctly be called:
+ * <BR>
+ * dictionary_en_NZ.properties<BR>
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Dictionary
+    extends HashMap {
+
+    /** The font used when displaying various html text. */
+    static private FontUIResource font = null;
+    /** A reference to remind us of the current locale. */
+    static private Locale locale = null;
+    /** The ResourceBundle which contains the raw key-value mappings.
+	Loaded from a file named "dictionary<I>locale</I>.properties. */
+    static private ResourceBundle dictionary = null;
+
+
+    public Dictionary(Locale locale, FontUIResource font)
+    {
+	super();
+
+	// Initialize
+	this.font = font;
+	this.locale = ((locale == null) ? Locale.getDefault() : locale);
+	this.dictionary = ResourceBundle.getBundle("dictionary", this.locale);
+    }
+
+
+    /** Retrieve the two letter code of the current language we are using, according to the stored locale.
+     * @return A <strong>String</strong> containing the two letter ISO639 language code.
+     */
+    static public String getLanguage()
+    {
+	return locale.getLanguage();
+    }
+
+
+    static public String get(String key)
+    {
+	return get(key, (String[]) null);
+    }
+
+
+    static public String get(String key, String arg)
+    {
+	String[] args = new String[1];
+	args[0] = arg;
+	return get(key, args);
+    }
+
+
+    /** Used to retrieve a property value from the Locale specific ResourceBundle, based upon the key and arguments supplied. If the key cannot be found or if some other part of the call fails a default (English) error message is returned. <BR>
+     * Here the get recieves a second argument which is an array of Strings used to populate argument fields, denoted {<I>n</I>}, within the value String returned.
+     * @param key A <strong>String</strong> which is mapped to a initial String within the ResourceBundle.
+     * @param args A <strong>String[]</strong> used to populate argument fields within the complete String.
+     * @return A <strong>String</strong> which has been referenced by the key String and that either contains no argument fields, or has had the argument fields automatiically populated with formatting Strings of with argument String provided in the get call.
+     */
+    static public String get(String key, String[] args)
+    {
+	try {
+	    String initial_raw = dictionary.getString(key);
+
+	    // Convert into Unicode
+	    String initial = "";
+	    try {
+		// This "ISO-8859-1" looks out of place, but it is very important.
+		// It is essential to call getBytes with an 8-bit encoding, otherwise
+		// Java kindly deems some characters "undisplayable", and replaces
+		// them with question marks. This is NOT good.
+		initial = new String(initial_raw.getBytes("ISO-8859-1"), "UTF-8");
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+		return initial_raw;
+	    }
+
+	    // Remove any comments from the string
+	    if (initial.indexOf("#") != -1) {
+		initial = initial.substring(0, initial.indexOf("#"));
+	    }
+
+	    // If the string contains arguments we have to insert them.
+	    String complete = "";
+
+	    // While we still have initial string left.
+	    while (initial.length() > 0 && initial.indexOf('{') != -1 && initial.indexOf('}') != -1) {
+		// Remove preamble
+		int opening = initial.indexOf('{');
+		int closing = initial.indexOf('}');
+		int comment_mark = initial.indexOf('-', opening); // May not exist
+		if (comment_mark > closing) { // May also be detecting a later comment
+		    comment_mark = -1;
+		}
+		complete = complete + initial.substring(0, opening);
+
+		// Parse arg_num
+		String arg_str = null;
+		if (comment_mark != -1) {
+		    arg_str = initial.substring(opening + 1, comment_mark);
+		}
+		else {
+		    arg_str = initial.substring(opening + 1, closing);
+		}
+
+		if (closing + 1 < initial.length()) {
+		    initial = initial.substring(closing + 1);
+		}
+		else {
+		    initial = "";
+		}
+
+		// Insert argument
+		if (arg_str.equals("FONT")) {
+		    complete = complete + (font != null ? font.getFontName() : "Arial");
+		}
+		else {
+		    int arg_num = Integer.parseInt(arg_str);
+		    if (args != null && 0 <= arg_num && arg_num < args.length) {
+			complete = complete + args[arg_num];
+		    }
+		}
+	    }
+	    complete = complete + initial;
+
+	    return complete;
+	}
+	catch (Exception exception) {
+	    System.err.println("Missing value for key: " + key);
+	    // DebugStream.printStackTrace(e);
+	    return key;
+	}
+    }
+
+
+    static public void registerBoth(Component component, String text_key, String tooltip_key)
+    {
+	setText(component, text_key);
+	setTooltip(component, tooltip_key);
+    }
+
+
+    static public void registerText(Component component, String text_key)
+    {
+	setText(component, text_key);
+    }
+
+
+    static public void registerText(Component component, String text_key, String[] args)
+    {
+	setText(component, text_key, args);
+    }
+
+
+    static public void registerTooltip(Component component, String tooltip_key)
+    {
+	setTooltip(component, tooltip_key);
+    }
+
+
+    static public void registerTooltipText(Component component, String tooltip)
+    {
+	setTooltipText(component, tooltip);
+    }
+
+
+    static public void setBoth(Component component, String text_key, String tooltip_key)
+    {
+	setText(component, text_key);
+	setTooltip(component, tooltip_key);
+    }
+
+
+    static public void setText(Component component, String text_key)
+    {
+	setText(component, text_key, null);
+    }
+
+
+    static public void setText(Component component, String text_key, String[] args)
+    {
+	if (component != null) {
+	    // Update the component using the AWTEvent queue
+	    SwingUtilities.invokeLater(new ComponentUpdateTask(component, get(text_key, args), null));
+	}
+    }
+
+
+    static public void setTooltip(Component component, String tooltip_key)
+    {
+	if (component != null) {
+	    // Update the component using the AWTEvent queue
+	    SwingUtilities.invokeLater(new ComponentUpdateTask(component, null, get(tooltip_key)));
+	}
+    }
+
+
+    static public void setTooltipText(Component component, String tooltip)
+    {
+	if (component != null) {
+	    // Update the component using the AWTEvent queue
+	    SwingUtilities.invokeLater(new ComponentUpdateTask(component, null, tooltip));
+	}
+    }
+
+
+    static private class ComponentUpdateTask
+	implements Runnable {
+
+	private Component component = null;
+	private String text = null;
+	private String tooltip = null;
+
+
+	public ComponentUpdateTask(Component component, String text, String tooltip)
+	{
+	    this.component = component;
+	    this.text = text;
+	    this.tooltip = tooltip;
+	}
+
+
+	public void run()
+	{
+	    // If the component has text
+	    if (text != null) {
+		if (component instanceof AbstractButton) {
+		    ((AbstractButton) component).setText(text);
+		}
+		else if (component instanceof Frame) {
+		    ((Frame) component).setTitle(text);
+		}
+		else if (component instanceof JDialog) {
+		    ((JDialog) component).setTitle(text);
+		}
+		else if (component instanceof JLabel) {
+		    ((JLabel) component).setText(text);
+		}
+		else if (component instanceof JProgressBar) {
+		    ((JProgressBar) component).setString(text);
+		}
+		else if (component instanceof JTextComponent) {
+		    ((JTextComponent) component).setText(text);
+		    ((JTextComponent) component).setCaretPosition(0);
+		}
+		else {
+		    System.err.println("Unhandled component: " + component.getClass());
+		}
+	    }
+
+	    // If the component has a tooltip
+	    if (tooltip != null) {
+		if (component instanceof JComponent) {
+		    ((JComponent) component).setToolTipText(tooltip);
+		}
+	    }
+	}
+    }
+
+
+    /**
+     * Register a tab pane component. This will be deprecated eventually. */
+    static public void register(JTabbedPane component)
+    {
+	if (component != null) {
+  	    String[] args = new String[component.getTabCount()];
+
+  	    // Iterate through the tabbed panes tabs, updating values and recording the original key of each item in args.
+	    for (int i = 0; i < args.length; i++) {
+  		if (args[i] == null) {
+  		    args[i] = component.getTitleAt(i);
+  		}
+  		String value = get(args[i], (String[]) null);
+  		String tooltip = get(args[i] + "_Tooltip", (String[]) null);
+  		JTabbedPaneChangeTask task = new JTabbedPaneChangeTask(component, i, value, tooltip);
+  		SwingUtilities.invokeLater(task);
+  	    }
+  	}
+    }
+
+
+    /** Updates a tabbed panes tab title and tooltip. */
+    static private class JTabbedPaneChangeTask
+    	implements Runnable {
+
+    	private JTabbedPane component;
+    	private int index;
+    	private String value;
+    	private String tooltip;
+
+    	public JTabbedPaneChangeTask(JTabbedPane component, int index, String value, String tooltip) {
+  	    this.component = component;
+    	    this.index = index;
+  	    this.value = value;
+   	    this.tooltip = tooltip;
+    	}
+
+   	public void run() {
+    	    component.setTitleAt(index, value);
+  	    component.setToolTipTextAt(index, tooltip);
+    	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/GAuthenticator.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/GAuthenticator.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/GAuthenticator.java	(revision 21112)
@@ -0,0 +1,188 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */ 
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.gui.GLIButton;
+
+/** Provides a graphic authenticator for network password requests.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class GAuthenticator
+    extends Authenticator
+{
+    static public Hashtable authentications = new Hashtable();
+
+    /** Indicates if this authentication prompt been cancelled, and if so rolls-back authentication. */
+    private boolean authentication_cancelled = false;
+    /** The button used to cancel a prompt. */
+    private JButton cancel_button = null;
+    /** The button used to submit the login/password. */
+    private JButton ok_button = null;
+    /** A reference to the dialog prompt created so inner classes can dispose of it. */
+    private JDialog dialog = null;
+    /** The password is a special starred out password field. */
+    private JPasswordField password = null;
+    /** The default size of this dialog. */
+    static final private Dimension SIZE = new Dimension(410,140);
+
+    /** Constructor. */
+    public GAuthenticator() {
+    }
+
+    /** Prompt the user for authentication using a pretty dialog box.
+     * @return A <strong>PasswordAuthentication</strong> object containing the login and password valuees the user has submitted.
+     * @see org.greenstone.gatherer.GAuthenticator.AuthenticationActionListener
+     * @see org.greenstone.gatherer.GAuthenticator.RequestFocusListener
+     */
+    protected PasswordAuthentication getPasswordAuthentication() {
+	// Component definition.
+	dialog = new JDialog (Gatherer.g_man, Dictionary.get("GAuthenticator.Title"), true);
+	dialog.setModal(true);
+	dialog.setSize(SIZE);
+	JPanel content_pane = (JPanel) dialog.getContentPane();
+	JLabel title_label = new JLabel(getMessageString());
+	JPanel user_panel = new JPanel();
+	JLabel username_label = new JLabel();
+	Dictionary.setText(username_label, "GAuthenticator.Username");
+	JTextField username = new JTextField();
+	Dictionary.setTooltip(username, "GAuthenticator.Username_Tooltip");
+	JPanel password_panel = new JPanel();
+	JLabel password_label = new JLabel();
+	Dictionary.setText(password_label, "GAuthenticator.Password");
+	password = new JPasswordField();
+	password.setEchoChar('*');
+	Dictionary.setTooltip(password, "GAuthenticator.Password_Tooltip");
+	JPanel button_panel = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+
+	// Connection
+	cancel_button.addActionListener(new AuthenticationActionListener(true));
+	ok_button.addActionListener(new AuthenticationActionListener(false));
+	password.addActionListener(new AuthenticationActionListener(false));
+	username.addActionListener(new RequestFocusListener(password));
+
+	// Layout
+	user_panel.setLayout(new GridLayout(1,2));
+	user_panel.add(username_label);
+	user_panel.add(username);
+
+	password_panel.setLayout(new GridLayout(1,2));
+	password_panel.add(password_label);
+	password_panel.add(password);
+
+	button_panel.setLayout(new GridLayout(1,2));
+	button_panel.add(ok_button);
+	button_panel.add(cancel_button);
+
+	content_pane.setLayout(new GridLayout(4,1,0,2));
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.add(title_label);
+	content_pane.add(user_panel);
+	content_pane.add(password_panel);
+	content_pane.add(button_panel);
+
+	// Position the window
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	dialog.setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	dialog.setVisible(true);
+	if(!authentication_cancelled) {
+	    // Store the authentication
+	    authentications.put(getRequestingHost() + ":" + getRequestingPort(), username.getText() + "@" + new String(password.getPassword()));
+	    return new PasswordAuthentication(username.getText(), password.getPassword());
+	} else {
+	    return null;
+	}
+    }
+
+
+    /** This is defined so it can be overridden by subclasses (getRequestingPrompt is final). */
+    protected String getMessageString()
+    {
+	return getRequestingPrompt();
+    }
+
+
+    /** Detects actions upon any control that attempt to submit the current details for authentication. */
+    private class AuthenticationActionListener
+	implements ActionListener {
+	/** <i>true</i> if this authentication action cancels the authentication, <i>false</i> otherwise. */
+	private boolean cancel_action = false;
+	/** Constructor.
+	 * @param cancel_action <i>true</i> if this authentication action cancels the authentication, <i>false</i> otherwise.
+	 */
+	public AuthenticationActionListener(boolean cancel_action) {
+	    this.cancel_action = cancel_action;
+	}
+	/** Any implementation of an ActionListener must include this method so that we can be informed when an action has been performed on our registered controls, allowing us to dispose of the authentication dialog after determining if this is a submit action or a cancel one.
+	 * @param event An <strong>ActionEvent</strong> with information about the event that fired this method.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    authentication_cancelled = cancel_action;
+	    dialog.dispose();
+	}
+    }
+
+    /** This listener detects actions on registered controls, and when they occur ensures the focus is moved to some targetted component. */
+    private class RequestFocusListener
+	implements ActionListener {
+	/*The <strong>Component</strong> you wish to gain focus when an action is performed on a registered control. */
+	private Component target = null;
+	/** Constructor.
+	 * @param target The <strong>Component</strong> you wish to gain focus when an action is performed on a registered control.
+	 */
+	public RequestFocusListener(Component target) {
+	    this.target = target;
+	}
+	/** Any implementation of an ActionListener must include this method so that we can be informed when an action has been performed on our registered controls, allowing us to request focus in the target control.
+	 * @param event An <strong>ActionEvent</strong> with information about the event that fired this method.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    target.requestFocus();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/Gatherer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/Gatherer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/Gatherer.java	(revision 21112)
@@ -0,0 +1,1005 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.lang.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.GAuthenticator;
+import org.greenstone.gatherer.cdm.ClassifierManager;
+import org.greenstone.gatherer.cdm.PluginManager;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.feedback.ActionRecorderDialog;
+import org.greenstone.gatherer.file.FileManager;
+import org.greenstone.gatherer.file.FileAssociationManager;
+import org.greenstone.gatherer.gui.GUIManager;
+import org.greenstone.gatherer.gui.URLField;
+import org.greenstone.gatherer.gui.WarningDialog;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/** Containing the top-level "core" for the Gatherer, this class is the
+ * common core for the GLI application and applet. It first parses the
+ * command line arguments, preparing to update the configuration as
+ * required. Next it loads several important support classes such as the
+ * Configuration and Dictionary. Finally it creates the other important
+ * managers and sends them on their way.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Gatherer
+{
+    static private Dimension size = new Dimension(800, 540);
+
+    /** Has the exit flag been set? */
+    static public boolean exit = false;
+
+    static private String gli_directory_path = null;
+    static private String gli_user_directory_path = null;
+
+    /** All of the external applications that must exit before we close the Gatherer. */
+    static private Vector apps = new Vector();
+    static private String non_standard_collect_directory_path = null;
+    static public String open_collection_file_path = null;
+    /** A public reference to the FileAssociationManager. */
+    static public FileAssociationManager assoc_man;
+    /** A public reference to the CollectionManager. */
+    static public CollectionManager c_man;
+    /** a reference to the Servlet Configuration is GS3 */
+    static public ServletConfiguration servlet_config;
+    /** A public reference to the FileManager. */
+    static public FileManager f_man;
+    /** A public reference to the GUIManager. */
+    static public GUIManager g_man = null;
+    static private boolean g_man_built = false;
+
+    /** We are using the GLI for GS3 */
+    static public boolean GS3 = false;
+
+    static public boolean isApplet = false;
+    static public boolean isGsdlRemote = false;
+
+    // feedback stuff
+    /** is the feedback feature enabled? */
+    static public boolean feedback_enabled = true;
+    /** the action recorder dialog */
+    static public ActionRecorderDialog feedback_dialog = null;
+
+    // Refresh reasons
+    static public final int COLLECTION_OPENED   = 0;
+    static public final int COLLECTION_CLOSED   = 1;
+    static public final int COLLECTION_REBUILT  = 2;
+    static public final int PREFERENCES_CHANGED = 3;
+
+
+    /** Magic to allow Enter to fire the default button. */
+    static {
+      KeyStroke enter = KeyStroke.getKeyStroke(KeyEvent.VK_ENTER, 0);
+      Keymap map = JTextComponent.getKeymap(JTextComponent.DEFAULT_KEYMAP);
+      map.removeKeyStrokeBinding(enter);
+    }
+
+
+    public Gatherer(String[] args) 
+    {
+	JarTools.initialise(this); 
+
+	GetOpt go = new GetOpt(args);
+
+	// If feedback is enabled, set up the recorder dialog
+	if (go.feedback_enabled) {
+	    // Use the default locale for now - this will be changed by the Gatherer run method
+	    feedback_enabled = true;
+	    feedback_dialog = new ActionRecorderDialog(Locale.getDefault());
+	}
+
+	// Are we using a remote Greenstone?
+	if (go.use_remote_greenstone) {
+	    isGsdlRemote = true;
+
+	    // We don't have a local Greenstone!
+	    go.gsdl_path = null;
+
+	    // We have to use our own collect directory since we can't use the Greenstone one
+	    setCollectDirectoryPath(getGLIUserDirectoryPath() + "collect" + File.separator);
+	}
+	// No, we have a local Greenstone
+	else {
+	    LocalGreenstone.setDirectoryPath(go.gsdl_path);
+	}
+
+	// Users may specify a non-standard collect directory (eg. when running one GLI in a network environment)
+	if (go.collect_directory_path != null) {
+	    setCollectDirectoryPath(go.collect_directory_path);
+	}
+
+	// More special code for running with a remote Greenstone
+	if (isGsdlRemote) {
+	    Configuration.TEMPLATE_CONFIG_XML = "xml/configRemote.xml";
+	    Configuration.CONFIG_XML = "configRemote.xml";
+	    Configuration.GS3_CONFIG_XML = "config3Remote.xml";
+
+	    File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	    if (!collect_directory.exists() && !collect_directory.mkdir()) {
+		System.err.println("Warning: Unable to make directory: " + collect_directory);
+	    }
+
+	    File plugins_dat_file = new File(getGLIUserDirectoryPath() + "plugins.dat");
+	    if (!plugins_dat_file.exists()) {
+		JarTools.extractFromJar("plugins.dat", getGLIUserDirectoryPath(), false);
+	    }
+
+	    File classifiers_dat_file = new File(getGLIUserDirectoryPath() + "classifiers.dat");
+	    if (!classifiers_dat_file.exists()) {
+		JarTools.extractFromJar("classifiers.dat", getGLIUserDirectoryPath(), false);
+	    }
+	}
+
+	init(go.gsdl_path, go.gsdl3_path, go.gsdl3_src_path, go.local_library_path, go.library_url_string, 
+	     go.gliserver_url_string, go.debug, go.perl_path, go.no_load, go.filename, go.site_name,
+	     go.servlet_path, go.wget_path);
+    }
+
+
+    public void init(String gsdl_path, String gsdl3_path, String gsdl3_src_path, String local_library_path,
+		     String library_url_string, String gliserver_url_string, boolean debug_enabled,
+		     String perl_path, boolean no_load, String open_collection, 
+		     String site_name, String servlet_path, String wget_path) 
+    {
+	if (gsdl3_path != null && !gsdl3_path.equals("")) {
+	    this.GS3 = true;
+	} else {
+	    gsdl3_path = null;
+	    gsdl3_src_path = null;
+	}
+
+	// Create the debug stream if required
+	if (debug_enabled) {
+	    DebugStream.enableDebugging();
+
+	    Calendar now = Calendar.getInstance();
+	    String debug_file_path = "debug" + now.get(Calendar.DATE) + "-" + now.get(Calendar.MONTH) + "-" + now.get(Calendar.YEAR) + ".txt";
+
+	    // Debug file is created in the user's GLI directory
+	    debug_file_path = getGLIUserDirectoryPath() + debug_file_path;
+	    DebugStream.println("Debug file path: " + debug_file_path);
+	    DebugStream.setDebugFile(debug_file_path);
+	    DebugStream.print(System.getProperties());
+	}
+
+	try {
+	    // Load GLI config file
+	    new Configuration(getGLIUserDirectoryPath(), gsdl_path, gsdl3_path, gsdl3_src_path, site_name);
+	    if (Configuration.just_updated_config_xml_file) {
+		// Delete the plugins.dat and classifiers.dat files
+		PluginManager.clearPluginCache();
+		ClassifierManager.clearClassifierCache();
+	    }
+
+	    if (GS3) {
+		// Load Greenstone 3 servelt configuration
+		servlet_config = new ServletConfiguration(gsdl3_path);
+	    }
+
+	    // Check we know where Perl is
+	    Configuration.perl_path = perl_path;
+	    if (isGsdlRemote && Utility.isWindows() && Configuration.perl_path != null) {
+		if (Configuration.perl_path.toLowerCase().endsWith("perl.exe")) {
+		    Configuration.perl_path = Configuration.perl_path.substring(0, Configuration.perl_path.length() - "perl.exe".length());
+		}
+		if (Configuration.perl_path.endsWith(File.separator)) {
+		    Configuration.perl_path = Configuration.perl_path.substring(0, Configuration.perl_path.length() - File.separator.length());
+		}
+	    }
+
+	    if (GS3 && Configuration.servlet_path == null) {
+		Configuration.servlet_path = servlet_config.getServletPath(Configuration.site_name);
+	    }
+
+	    // the feedback dialog has been loaded with a default locale, 
+	    // now set the user specified one
+	    if (feedback_enabled && feedback_dialog != null) {
+		feedback_dialog.setLocale(Configuration.getLocale("general.locale", true));
+	    }
+
+	    // Read Dictionary
+	    new Dictionary(Configuration.getLocale("general.locale", true), Configuration.getFont("general.font", true));
+
+	    // check that we are using Sun Java
+	    String java_vendor = System.getProperty("java.vendor");
+	    if (!java_vendor.equals("Sun Microsystems Inc.")) {
+		System.err.println(Dictionary.get("General.NotSunJava", java_vendor));
+	    }
+
+	    // Unless we're using remote building, we need to know where the local Greenstone is
+	    if (!isGsdlRemote && gsdl_path == null) {
+		missingGSDL();
+	    }
+
+	    // Start up the local library server, if that's what we want
+	    if (Utility.isWindows() && local_library_path != null && !isGsdlRemote && !GS3) {
+		LocalLibraryServer.start(gsdl_path, local_library_path);
+	    }
+
+	    // The "-library_url" option overwrites anything in the config files
+	    if (library_url_string != null && library_url_string.length() > 0) {
+		try {
+		    System.err.println("Setting library_url to " + library_url_string + "...");
+		    Configuration.library_url = new URL(library_url_string);
+		}
+		catch (MalformedURLException error) {
+		    DebugStream.printStackTrace(error);
+		}
+	    }
+
+	    // Check that we now know the Greenstone library URL, since we need this for previewing collections
+	    DebugStream.println("Configuration.library_url = " + Configuration.library_url);
+	    if (Configuration.library_url == null) {
+		missingEXEC();
+	    }
+
+	    // The "-gliserver_url" option overwrites anything in the config files
+	    if (gliserver_url_string != null && gliserver_url_string.length() > 0) {
+		try {
+		    System.err.println("Setting gliserver_url to " + gliserver_url_string + "...");
+		    Configuration.gliserver_url = new URL(gliserver_url_string);
+		}
+		catch (MalformedURLException error) {
+		    DebugStream.printStackTrace(error);
+		}
+	    }
+
+	    // If we're using a remote Greenstone we need to know where the gliserver script is
+	    DebugStream.println("Configuration.gliserver_url = " + Configuration.gliserver_url);
+	    if (isGsdlRemote) {
+		if (Configuration.gliserver_url == null) {
+		    missingGLIServer();
+		}
+		if (Configuration.gliserver_url != null) {
+		    gliserver_url_string = Configuration.gliserver_url.toString();
+		}
+	    }
+
+	    // Check for ImageMagick
+	    if (Gatherer.isGsdlRemote) {
+		DebugStream.println("Not checking for ImageMagick.");
+	    }
+	    else if (!(new ImageMagickTest()).found()) {
+		// Time for a warning message
+		missingImageMagick();
+	    }
+
+	    if (Gatherer.isGsdlRemote) {
+		DebugStream.println("Not checking for perl path/exe");
+	    }
+	    else {
+		// Perl path is a little different as it is perfectly ok to
+		// start the GLI without providing a perl path
+		boolean found_perl = false;
+		if (Configuration.perl_path != null) {
+		    // See if the file pointed to actually exists
+		    File perl_file = new File(Configuration.perl_path);
+		    found_perl = perl_file.exists();
+		    perl_file = null;
+		}
+		if (Configuration.perl_path == null || !found_perl) {
+		    // Run test to see if we can run perl as is.
+		    PerlTest perl_test = new PerlTest();
+		    if (perl_test.found()) {
+			// If so replace the perl path with the system
+			// default (or null for unix).
+			Configuration.perl_path = perl_test.toString();
+			found_perl = true;
+		    }
+		}
+		if (!found_perl) {
+		    // Time for an error message.
+		    missingPERL();
+		}
+	    }
+
+	    boolean download_workflow = Configuration.get("workflow.download", false);
+	    if (download_workflow) {
+		// If the WGet path hasn't been specified by the user, try the Greenstone one
+		if (wget_path == null) {
+		    String wget_file_path = gsdl_path + "bin" + File.separator;
+		    if (Utility.isWindows()) {
+			wget_file_path += "windows" + File.separator + "wget.exe";
+		    }
+		    else if (Utility.isMac()) {
+			wget_file_path += "darwin" + File.separator + "wget";
+		    }
+		    else {
+			wget_file_path += "linux" + File.separator + "wget";
+		    }
+		    if (new File(wget_file_path).exists()) {
+			wget_path = wget_file_path;
+		    }
+		}
+
+		// If we still don't have a WGet path, give up (there isn't anything else we can do)
+		String wget_version_str = null;
+		if (wget_path == null) {
+		    wget_version_str = StaticStrings.NO_WGET_STR;
+		    missingWGET();
+		}
+		// Otherwise check the version of WGet
+		else {
+		    wget_version_str = testWGetVersion(wget_path);
+		    if (wget_version_str.equals(StaticStrings.WGET_OLD_STR)) {
+			oldWGET();
+		    }
+		}
+
+		// Store the new values in the configuration
+		DebugStream.println("WGet path: " + wget_path);
+		Configuration.setWGetPath(wget_path);
+		Configuration.setWGetVersion(wget_version_str);
+	    }
+
+	    // Set the default font for all Swing components.
+	    FontUIResource default_font = Configuration.getFont("general.font", true);
+	    Enumeration keys = UIManager.getDefaults().keys();
+	    while (keys.hasMoreElements()) {
+		Object key = keys.nextElement();
+		Object value = UIManager.get(key);
+		if (value instanceof FontUIResource) {
+		    UIManager.put(key, default_font);
+		}
+	    }
+
+	    // Set up proxy
+	    setProxy();
+	    // Now we set up an Authenticator
+	    Authenticator.setDefault(new GAuthenticator());
+
+	    assoc_man = new FileAssociationManager();
+	    // Create File Manager
+	    f_man = new FileManager();
+	    // Create Collection Manager
+	    c_man = new CollectionManager();
+
+	    if (GS3) {
+		if (site_name==null) {
+		    site_name = Configuration.site_name;
+		    servlet_path = null; // need to reset this
+		}
+		if (servlet_path == null) {
+		    servlet_path = Configuration.getServletPath();
+		}
+	    }
+
+	    open_collection_file_path = open_collection;
+	    if (open_collection_file_path == null) {
+		open_collection_file_path = Configuration.getString("general.open_collection", true);
+	    }
+	    if (no_load || open_collection_file_path.equals("")) {
+		open_collection_file_path = null;
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+
+	// Create GUI Manager (last) or else suffer the death of a thousand NPE's
+	g_man = new GUIManager(size);
+    }
+
+
+    public void openGUI()
+    {
+	// Size and place the frame on the screen
+	Rectangle bounds = Configuration.getBounds("general.bounds", true);
+	if (bounds == null) {
+	    // Choose a sensible default value
+	    bounds = new Rectangle(0, 0, 640, 480);
+	}
+	
+	// Ensure width and height are reasonable
+	size = bounds.getSize();
+	if (size.width < 640) {
+	    size.width = 640;
+	}
+	else if (size.width > Configuration.screen_size.width) {
+	    size.width = Configuration.screen_size.width;
+	}
+	if (size.height < 480) {
+	    size.height = 480;
+	}
+	else if (size.height > Configuration.screen_size.height) {
+	    size.height = Configuration.screen_size.height;
+	}
+
+	if (!g_man_built) {
+
+	    g_man.display();
+
+	    // Place the window in the desired location on the screen, if this is do-able (not under most linux window managers apparently. In fact you're lucky if they listen to any of your screen size requests).
+	    g_man.setLocation(bounds.x, bounds.y);
+	    g_man.setVisible(true);
+
+	    // After the window has been made visible, check that it is in the correct place
+	    // sometimes java places a window not in the correct place, 
+	    // but with an offset. If so, we work out what the offset is
+	    // and change the desired location to take that into account 
+	    Point location = g_man.getLocation();
+	    int x_offset = bounds.x - location.x;
+	    int y_offset = bounds.y - location.y;
+	    // If not, offset the window to move it into the correct location
+	    if (x_offset > 0 || y_offset > 0) {
+		///ystem.err.println("changing the location to "+(bounds.x + x_offset)+" "+ (bounds.y + y_offset));
+		g_man.setLocation(bounds.x + x_offset, bounds.y + y_offset);
+	    }
+
+	    // The 'after-display' triggers several events which don't occur until after the visual components are actually available on screen. Examples of these would be the various html renderings, as they can't happen offscreen.
+	    g_man.afterDisplay();
+	    g_man_built = true;
+	}
+	else {
+	    g_man.setVisible(true);
+	}
+    }
+
+
+    /** Exits the Gatherer after ensuring that things needing saving are saved.
+     * @see java.io.FileOutputStream
+     * @see java.io.PrintStream
+     * @see java.lang.Exception
+     * @see javax.swing.JOptionPane
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.gui.GUIManager
+     */
+    static public void exit()
+    {
+	System.err.println("In Gatherer.exit()...");
+	exit = true;
+
+	// Save the file associations
+	if (assoc_man != null) {
+	    assoc_man.save();
+	    assoc_man = null;
+	}
+
+	// Get the gui to deallocate
+	g_man.destroy();
+	g_man_built = false;
+
+	// Flush debug
+	DebugStream.closeDebugStream();
+
+	// If we started a server, we should try to stop it.
+	if (LocalLibraryServer.isRunning() == true) {
+	    LocalLibraryServer.stop();
+	}
+
+	// If we're using a remote Greenstone server we need to make sure that all jobs have completed first
+	if (isGsdlRemote) {
+	    RemoteGreenstoneServer.exit();
+	}
+
+	// Make sure we haven't started up any processes that are still running
+	if (apps.size() == 0) {
+	    // If we're running as an applet we don't actually quit (just hide the main GLI window)
+	    if (!Gatherer.isApplet) {
+		// This is the end...
+		System.exit(0);
+	    }
+	}
+	else {
+	    JOptionPane.showMessageDialog(g_man, Dictionary.get("General.Outstanding_Processes"), Dictionary.get("General.Outstanding_Processes_Title"), JOptionPane.ERROR_MESSAGE);
+	    g_man.setVisible(false);
+	}
+    }
+
+
+    /** Returns the path of the Greenstone "collect" directory. */
+    static public String getCollectDirectoryPath()
+    {
+	if (non_standard_collect_directory_path != null) {
+	    return non_standard_collect_directory_path;
+	}
+
+	if (!GS3) {
+	    return Configuration.gsdl_path + "collect" + File.separator;
+	}
+	else {
+	    return getSitesDirectoryPath() + Configuration.site_name + File.separator + "collect" + File.separator;
+	}
+    }
+
+
+    /** Returns the path of the GLI directory. */
+    static public String getGLIDirectoryPath()
+    {
+	return gli_directory_path;
+    }
+
+
+    /** Returns the path of the GLI "metadata" directory. */
+    static public String getGLIMetadataDirectoryPath()
+    {
+	return getGLIDirectoryPath() + "metadata" + File.separator;
+    }
+
+
+    /** Returns the path of the GLI user directory. */
+    static public String getGLIUserDirectoryPath()
+    {
+	return gli_user_directory_path;
+    }
+
+
+    /** Returns the path of the GLI user "cache" directory. */
+    static public String getGLIUserCacheDirectoryPath()
+    {
+	return getGLIUserDirectoryPath() + "cache" + File.separator;
+    }
+
+
+    /** Returns the path of the GLI user "log" directory. */
+    static public String getGLIUserLogDirectoryPath()
+    {
+	return getGLIUserDirectoryPath() + "log" + File.separator;
+    }
+
+
+    static public String getSitesDirectoryPath()
+    {
+	return Configuration.gsdl3_path + "sites" + File.separator;
+    }
+
+
+    static public void setCollectDirectoryPath(String collect_directory_path)
+    {
+	non_standard_collect_directory_path = collect_directory_path;
+	if (!non_standard_collect_directory_path.endsWith(File.separator)) {
+	    non_standard_collect_directory_path = non_standard_collect_directory_path + File.separator;
+	}
+    }
+
+
+    static public void setGLIDirectoryPath(String gli_directory_path_arg)
+    {
+	gli_directory_path = gli_directory_path_arg;
+    }
+
+
+    static public void setGLIUserDirectoryPath(String gli_user_directory_path_arg)
+    {
+	gli_user_directory_path = gli_user_directory_path_arg;
+
+	// Ensure the GLI user directory exists
+	File gli_user_directory = new File(gli_user_directory_path);
+	if (!gli_user_directory.exists() && !gli_user_directory.mkdirs()) {
+	    System.err.println("Error: Unable to make directory: " + gli_user_directory);
+	}
+    }
+
+
+    static public void refresh(int refresh_reason)
+    {
+	if (g_man != null) {
+	    g_man.refresh(refresh_reason, c_man.ready());
+	}
+
+	// Now is a good time to force a garbage collect
+	System.gc();
+    }
+
+
+    // used to send reload coll messages to the tomcat server
+    static public void configGS3Server(String site, String command) {
+
+	try {
+	    // need to add the servlet name to the exec address
+	    String raw_url = Configuration.library_url.toString() + Configuration.getServletPath() + command;
+	    URL url = new URL(raw_url);
+	    DebugStream.println("Action: " + raw_url);
+	    HttpURLConnection library_connection = (HttpURLConnection) url.openConnection();
+	    int response_code = library_connection.getResponseCode();
+	    if(HttpURLConnection.HTTP_OK <= response_code && response_code < HttpURLConnection.HTTP_MULT_CHOICE) {
+		DebugStream.println("200 - Complete.");
+	    }
+	    else {
+		DebugStream.println("404 - Failed.");
+	    }
+	    url = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+
+    /** Used to 'spawn' a new child application when a file is double clicked.
+     * @param file The file to open 
+     * @see org.greenstone.gatherer.Gatherer.ExternalApplication
+     */
+    static public void spawnApplication(File file) {
+	String [] commands = assoc_man.getCommand(file);
+	if(commands != null) {
+	    ExternalApplication app = new ExternalApplication(commands);
+	    apps.add(app);
+	    app.start();
+	}
+	else {
+	    ///ystem.err.println("No open command available.");
+	}
+    }
+
+
+    static public void spawnApplication(String command)
+    {
+	ExternalApplication app = new ExternalApplication(command);
+	apps.add(app);
+	app.start();
+    }
+
+
+    /** Used to 'spawn' a new  browser application or reset an existing one when the preview button is clicked 
+     * @param url The url to open the browser at
+     * @see org.greenstone.gatherer.Gatherer.BrowserApplication
+     */    
+    static public void spawnBrowser(String url) {
+	String command = assoc_man.getBrowserCommand(url);
+	if (command != null) {
+	    BrowserApplication app = new BrowserApplication(command, url);
+	    apps.add(app);
+	    app.start();
+	}
+	else {
+	    ///ystem.err.println("No browser command available."); 
+	}
+    }
+
+
+    /** Prints a warning message about a missing library path, which means the final collection cannot be previewed in the Gatherer.
+     */
+    static private void missingEXEC() {
+	WarningDialog dialog;
+	if (GS3) {
+	    dialog = new WarningDialog("warning.MissingEXEC_GS3", "MissingEXEC_GS3.Title", Dictionary.get("MissingEXEC_GS3.Message"), "general.library_url", false);
+	}
+	else {
+	    dialog = new WarningDialog("warning.MissingEXEC", "MissingEXEC.Title", Dictionary.get("MissingEXEC.Message"), "general.library_url", false);
+	}
+	dialog.setValueField(new URLField(Configuration.getColor("coloring.editable_foreground", false), Configuration.getColor("coloring.editable_background", false)));
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+
+	String library_url_string = Configuration.getString("general.library_url", true);
+	if (!library_url_string.equals("")) {
+	    try {
+		Configuration.library_url = new URL(library_url_string);
+	    }
+	    catch (MalformedURLException exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+    }
+
+
+    static private void missingGLIServer()
+    {
+	WarningDialog dialog = new WarningDialog("warning.MissingGLIServer", "MissingGLIServer.Title", Dictionary.get("MissingGLIServer.Message"), "general.gliserver_url", false);
+	dialog.setValueField(new URLField(Configuration.getColor("coloring.editable_foreground", false), Configuration.getColor("coloring.editable_background", false)));
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+
+	String gliserver_url_string = Configuration.getString("general.gliserver_url", true);
+	if (!gliserver_url_string.equals("")) {
+	    try {
+		Configuration.gliserver_url = new URL(gliserver_url_string);
+	    }
+	    catch (MalformedURLException exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+    }
+
+
+    /** Prints a warning message about a missing GSDL path, which although not fatal pretty much ensures nothing will work properly in the GLI.
+     */
+    static private void missingGSDL() {
+	WarningDialog dialog = new WarningDialog("warning.MissingGSDL", "MissingGSDL.Title", Dictionary.get("MissingGSDL.Message"), null, false);
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+    }
+
+    /** Prints a warning message about missing a valid ImageMagick path, which although not fatal means building image collections won't work */
+    static private void missingImageMagick() {
+	WarningDialog dialog = new WarningDialog("warning.MissingImageMagick", "MissingImageMagick.Title", Dictionary.get("MissingImageMagick.Message"), null, false);
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+    }
+
+    /** Prints a warning message about missing a valid PERL path, which although not fatal pretty much ensures no collection creation/building will work properly in the GLI. */
+    static private void missingPERL() {
+	WarningDialog dialog = new WarningDialog("warning.MissingPERL", "MissingPERL.Title", Dictionary.get("MissingPERL.Message"), null, false);
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+    }
+
+    /** Prints a warning message about missing a valid WGet path, which although not fatal means mirroring won't work */
+    static public void missingWGET() {
+	WarningDialog dialog = new WarningDialog("warning.MissingWGET", "MissingWGET.Title", Dictionary.get("MissingWGET.Message"), null, false);
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+    }
+
+    /** Prints a warning message about having an old version of WGet. not fatal, but mirroring may not work properly */
+    static public void oldWGET() {
+	WarningDialog dialog = new WarningDialog("warning.OldWGET", "OldWGET.Title", Dictionary.get("OldWGET.Message"), null, false);
+	dialog.display();
+	dialog.dispose();
+	dialog = null;
+    }
+
+
+    /** Sets up the proxy connection by setting JVM Environment flags and creating a new Authenticator.
+     * @see java.lang.Exception
+     * @see java.lang.System
+     * @see java.net.Authenticator
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.GAuthenticator
+     */
+    static public void setProxy() {
+	try {// Can throw several exceptions
+	    if(Configuration.get("general.use_proxy", true)) {
+		System.setProperty("http.proxyType", "4");
+		System.setProperty("http.proxyHost", Configuration.getString("general.proxy_host", true));
+		System.setProperty("http.proxyPort", Configuration.getString("general.proxy_port", true));
+		System.setProperty("http.proxySet", "true");
+	    } else {
+		System.setProperty("http.proxySet", "false");
+	    }
+	} catch (Exception error) {
+	    DebugStream.println("Error in Gatherer.initProxy(): " + error);
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+
+    // TODO fill this in
+    private String testWGetVersion(String wget_path) {
+	return StaticStrings.WGET_STR;
+    }
+    
+    /** This private class contains an instance of an external application running within a JVM shell. It is important that this process sits in its own thread, but its more important that when we exit the Gatherer we don't actually System.exit(0) the Gatherer object until the user has volunteerily ended all of these child processes. Otherwise when we quit the Gatherer any changes the users may have made in external programs will be lost and the child processes are automatically deallocated. */
+    static private class ExternalApplication
+	extends Thread {
+	private Process process = null;
+	/** The initial command string given to this sub-process. */
+	private String command = null;
+	private String[] commands = null;
+	/** Constructor.
+	 * @param command The initial command <strong>String</strong>.
+	 */
+	public ExternalApplication(String command) {
+	    this.command = command;
+	}
+
+	public ExternalApplication(String[] commands) {
+	    this.commands = commands;
+	}
+	/** We start the child process inside a new thread so it doesn't block the rest of Gatherer.
+	 * @see java.lang.Exception
+	 * @see java.lang.Process
+	 * @see java.lang.Runtime
+	 * @see java.lang.System
+	 * @see java.util.Vector
+	 */
+	public void run() {
+	    // Call an external process using the args.
+	    try {
+		if(commands != null) {
+		    StringBuffer whole_command = new StringBuffer();
+		    for(int i = 0; i < commands.length; i++) {
+			whole_command.append(commands[i]);
+			whole_command.append(" ");
+		    }
+		    DebugStream.println("Running " + whole_command.toString());
+		    Runtime rt = Runtime.getRuntime();
+		    process = rt.exec(commands);
+		    process.waitFor();
+		}
+		else {
+		    DebugStream.println("Running " + command);
+		    Runtime rt = Runtime.getRuntime();
+		    process = rt.exec(command);
+		    process.waitFor();
+		}
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	    // Remove ourself from Gatherer list of threads.
+	    apps.remove(this);
+	    // Call exit if we were the last outstanding child process thread.
+	    if (apps.size() == 0 && exit == true) {
+		System.exit(0);
+	    }
+	}
+	public void stopExternalApplication() {
+	    if(process != null) {
+		process.destroy();
+	    }
+	}
+    }
+    /** This private class contains an instance of an external application running within a JVM shell. It is important that this process sits in its own thread, but its more important that when we exit the Gatherer we don't actually System.exit(0) the Gatherer object until the user has volunteerily ended all of these child processes. Otherwise when we quit the Gatherer any changes the users may have made in external programs will be lost and the child processes are automatically deallocated. */
+    static private class BrowserApplication
+	extends Thread {
+	private Process process = null;
+	/** The initial command string given to this sub-process. */
+	private String command = null;
+	private String url = null;
+	private String[] commands = null;
+
+	public BrowserApplication(String command, String url) {
+	    StringTokenizer st = new StringTokenizer(command);
+	    int num_tokens = st.countTokens();
+	    this.commands = new String [num_tokens];
+	    int i=0;
+	    while (st.hasMoreTokens()) {
+		commands[i] = st.nextToken();
+		i++;
+	    }
+	    //this.commands = commands;
+	    this.url = url;
+	}
+	/** We start the child process inside a new thread so it doesn't block the rest of Gatherer.
+	 * @see java.lang.Exception
+	 * @see java.lang.Process
+	 * @see java.lang.Runtime
+	 * @see java.lang.System
+	 * @see java.util.Vector
+	 */
+	public void run() {
+	    // Call an external process using the args.
+	    if(commands == null) {
+		apps.remove(this);
+		return;
+	    }
+	    try {
+		String prog_name = commands[0];
+		String lower_name = prog_name.toLowerCase();
+		if (lower_name.indexOf("mozilla") != -1 || lower_name.indexOf("netscape") != -1) { 
+		    DebugStream.println("found mozilla or netscape, trying remote it");
+		    // mozilla and netscape, try using a remote command to get things in the same window
+		    String [] new_commands = new String[] {prog_name, "-raise", "-remote", "openURL("+url+",new-tab)"};
+		    printArray(new_commands);
+
+		    Runtime rt = Runtime.getRuntime();
+		    process = rt.exec(new_commands);
+		    int exitCode = process.waitFor();
+		    if (exitCode != 0) { // if Netscape or mozilla was not open
+			DebugStream.println("couldn't do remote, trying original command");
+			printArray(commands);
+			process = rt.exec(commands); // try the original command
+		    }
+		} else {
+		    // just run what we have been given
+		    StringBuffer whole_command = new StringBuffer();
+		    for(int i = 0; i < commands.length; i++) {
+			whole_command.append(commands[i]);
+			whole_command.append(" ");
+		    }
+		    DebugStream.println("Running " + whole_command.toString());
+		    Runtime rt = Runtime.getRuntime();
+		    process = rt.exec(commands);
+		    process.waitFor();
+		}
+	    }
+	    
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	    // Remove ourself from Gatherer list of threads.
+	    apps.remove(this);
+	    // Call exit if we were the last outstanding child process thread.
+	    if (apps.size() == 0 && exit == true) {
+		System.exit(0);
+	    }
+	}
+	public void printArray(String [] array) {
+	    for(int i = 0; i < array.length; i++) {
+		DebugStream.print(array[i]+" ");
+		System.err.println(array[i]+" ");
+	    }
+	}
+	public void stopBrowserApplication() {
+	    if(process != null) {
+		process.destroy();
+	    }
+	}
+    }
+
+
+    private class ImageMagickTest
+    {
+	public boolean found()
+	{
+	    try {
+		String[] command = new String[2];
+		command[0] = (Utility.isWindows() ? "identify.exe" : "identify");
+		command[1] = "-version";
+		Process image_magick_process = Runtime.getRuntime().exec(command);
+		image_magick_process.waitFor();
+		return (image_magick_process.exitValue() == 0);
+	    }
+	    catch (Exception exception) {
+		return false;
+	    }
+	}
+    }
+
+
+    private class PerlTest
+    {
+	private String[] command = new String[2];
+
+	public PerlTest()
+	{
+	    command[0] = (Utility.isWindows() ? Utility.PERL_EXECUTABLE_WINDOWS : Utility.PERL_EXECUTABLE_UNIX);
+	    command[1] = "-version";
+	}
+
+	public boolean found()
+	{
+	    try {
+		Process perl_process = Runtime.getRuntime().exec(command);
+		perl_process.waitFor();
+		return (perl_process.exitValue() == 0);
+	    }
+	    catch (Exception exception) {
+		return false;
+	    }
+	}
+
+	public String toString() {
+	    return command[0];
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererApplet.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererApplet.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererApplet.java	(revision 21112)
@@ -0,0 +1,219 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer;
+
+import java.applet.*;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.net.*;
+import javax.swing.JApplet;
+
+import java.io.*;
+import javax.swing.*;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+public class GathererApplet extends JApplet implements ActionListener
+{
+    private Gatherer gatherer = null;
+
+
+    protected String fullLibraryURL(String address) 
+    {
+	String full_address = "";
+
+        // make sure the URL has protocol, host, and file
+        if (address.startsWith("http:")) {
+            // the address has all the necessary components
+	    full_address = address;
+        } 
+	else if (address.startsWith("/")) {
+            // there is not protocol and host
+            URL document = getDocumentBase();
+	    int port_no = document.getPort();
+
+            String port = (port_no>0) ? ":" + port_no : "";
+            full_address = "http://" + document.getHost() + port  + address;
+	}
+
+	return full_address;
+    }
+
+
+    /* This is the entry point for the GLI applet */
+    public void init()
+    {
+	// Check if the user has agreed to the requested security settings
+	try {
+	    // Work-around to reduce number of 
+	    // 'Could not lock user prefs' error messages.
+	    // Thanks to Walter Schatz from the java forums.
+	    System.setProperty("java.util.prefs.syncInterval","2000000");
+	}
+	catch (Exception exception) {
+	    getContentPane().add(new JLabel("Greenstone Librarian Interface Applet deactivated", JLabel.CENTER));
+	    return;
+	}
+
+	// Ensure platform specific LAF
+	try {
+	    UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+
+	// Determine the GLI user directory path
+	String gli_user_directory_path = System.getProperty("user.home") + File.separator;
+        if (Utility.isWindows()) {
+	    String windows_home_parameter = getParameter("windowsHome");
+	    if (windows_home_parameter != null && !windows_home_parameter.equals("")) {
+		gli_user_directory_path = windows_home_parameter + File.separator + "GLI" + File.separator;
+	    }
+	    else {
+		gli_user_directory_path += "Application Data" + File.separator + "Greenstone" + File.separator + "GLI" + File.separator;
+	    }
+	}
+        else {
+            gli_user_directory_path += ".gli" + File.separator;
+        }
+	Gatherer.setGLIUserDirectoryPath(gli_user_directory_path);
+
+	// If we're running as an applet we don't have a local GLI, so use the user directory path as the GLI path
+	Gatherer.setGLIDirectoryPath(gli_user_directory_path);
+
+	// Set some global variables so we know how we're running
+	Gatherer.isApplet = true;
+	String library_url_string = fullLibraryURL(getParameter("gwcgi"));
+	String gliserver_url_string = library_url_string.substring(0, library_url_string.lastIndexOf('/') + 1) + "gliserver";
+
+        // String debug_param = getParameter("debug");
+        // if ((debug_param != null) && (debug_param.matches("true"))) {
+        //     go.debug = true;
+        // }
+
+	String[] args = { "-use_remote_greenstone",
+			  "-gliserver_url", gliserver_url_string,
+			  "-library_url", library_url_string,
+			  "-debug",
+	                };
+
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	if (!collect_directory.exists()) {
+	    if (JarTools.isInJar("collect.zip")) {
+		// Dig out collect.zip from JAR file and unzip it (thereby
+		// creating collect directory and descendants.
+		//   This is done to seed gsdl home folder with config files
+		//   for documented example collections so "base this on
+		//   existing collection" works
+		unzipFromJar("collect.zip", Gatherer.getGLIUserDirectoryPath());
+	    }
+	    else {
+		// Prepare an empty collect dir for use
+		if (!collect_directory.mkdir()) {
+		    System.err.println("Warning: Unable to make directory: " + collect_directory);
+		}
+	    }
+	}
+
+	File metadata_directory = new File(Gatherer.getGLIMetadataDirectoryPath());
+	if (!metadata_directory.exists()) {
+	    // dig out metadata.zip from JAR file and unzip it
+	    unzipFromJar("metadata.zip", Gatherer.getGLIDirectoryPath());
+	}
+
+	// Create an instance of the Gatherer class, which will parse the args and prepare the rest of the GLI
+	gatherer = new Gatherer(args);
+
+	// Create a button for the user to press to open the GLI GUI
+	JButton launch_GLI_button = new JButton("Launch Greenstone Librarian Interface ...");
+	launch_GLI_button.addActionListener(this);
+	getContentPane().add(launch_GLI_button);
+    }
+
+
+    public void start() 
+    {
+	System.err.println("Start called");
+    }
+
+
+    public void stop()
+    {
+	System.err.println("Stop called");
+
+    }
+
+
+    public void destroy()
+    {
+	System.err.println("Destroy called");
+	gatherer.exit();
+	System.err.println("Done gatherer exit.");
+    }
+
+
+    public void actionPerformed(ActionEvent e)
+    {
+	gatherer.openGUI();
+
+	// If there was an open collection last session, reopen it
+	new OpenCollectionFromLastTimeTask().start();
+    }
+
+
+    /** This is in it's own thread because this action cannot happen on the GUI thread. */
+    private class OpenCollectionFromLastTimeTask
+	extends Thread
+    {
+	public void run()
+	{
+	    Gatherer.c_man.openCollectionFromLastTime();
+	}
+    }
+
+
+    /** 
+     * Method which unzips a given metadata resoure.
+     * @param jar_zip_fname The name of the jar file as a <strong>String</strong>.
+     * @param dst_dir The destination directory for unzipping, also as a <strong>String</strong>.
+     * @see JarTools.extractFromJar(String, String, boolean)
+     */
+    static public void unzipFromJar(String jar_zip_fname, String dst_dir)
+    {
+	if (!dst_dir.endsWith(File.separator)) {
+	    dst_dir += File.separator;
+	}
+
+	// !! TO DO: Test this !!
+	JarTools.extractFromJar(jar_zip_fname, dst_dir, true);
+	ZipTools.unzipFile(dst_dir + jar_zip_fname, dst_dir);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererProg.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererProg.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/GathererProg.java	(revision 21112)
@@ -0,0 +1,85 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, NZDL Project, University of Waikato
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+
+import java.io.*;
+import javax.swing.*;
+import org.greenstone.gatherer.util.Utility;
+
+
+/**
+ * Containing the main() method, this class is the entry point for the
+ * stand-alone (non-Applet) GLI. The main point of this class is to set
+ * some stand-alone specific properties and then create an instance of the
+ * Gatherer class.
+ * @author John Thompson, NZDL Project, University of Waikato
+ */
+public class GathererProg
+{
+    /** The entry point into the stand-alone GLI.
+     * @param args A collection of arguments that may include: initial screen size, dictionary, path to the GSDL etc.
+     */
+    static public void main(String[] args)
+    {
+	// A serious hack, but its good enough to stop crappy 'Could not
+	// lock user prefs' error messages.  Thanks to Walter Schatz from
+	// the java forums.
+	System.setProperty("java.util.prefs.syncInterval","2000000"); 
+	// => results in one message every 600 hours!
+
+	// Ensure platform specific LAF
+	try {
+	    UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+
+	// Determine the GLI user directory path
+	String gli_user_directory_path = System.getProperty("user.home") + File.separator;
+        if (Utility.isWindows()) {
+	    gli_user_directory_path += "Application Data" + File.separator + "Greenstone" + File.separator + "GLI" + File.separator;
+        }
+        else {
+            gli_user_directory_path += ".gli" + File.separator;
+        }
+	Gatherer.setGLIUserDirectoryPath(gli_user_directory_path);
+
+	// We have a local GLI
+	Gatherer.setGLIDirectoryPath(System.getProperty("user.dir") + File.separator);
+
+	// Create an instance of the Gatherer class, which will parse the args and prepare the rest of the GLI
+	Gatherer gatherer = new Gatherer(args);
+
+	// Display the GUI immediately
+	gatherer.openGUI();
+
+	// If there was an open collection last session, reopen it
+	Gatherer.c_man.openCollectionFromLastTime();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/GetOpt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/GetOpt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/GetOpt.java	(revision 21112)
@@ -0,0 +1,240 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson and David Bainbridge, 
+ *         Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.io.*;
+import org.greenstone.gatherer.util.StaticStrings; 
+import org.greenstone.gatherer.util.Utility;
+
+public class GetOpt
+{
+    public boolean debug = false; 
+    public boolean feedback_enabled = false;
+    public boolean no_load = false;
+    public boolean use_remote_greenstone = false;
+
+    public String collect_directory_path = null;
+    public String filename = null;
+    public String gliserver_url_string = null;
+    public String gsdl_path = null;
+    public String gsdl3_path = null;
+    public String gsdl3_src_path = null;
+    public String library_url_string = null;
+    public String local_library_path = null;
+    public String perl_path = null;
+    public String site_name = null; // for GS3
+    public String servlet_path = null;
+    public String wget_path = null;
+
+
+    public GetOpt(String[] args)
+    {
+	// Default dictionary. Only used for starting error messages.
+	Dictionary dictionary = new Dictionary(null, null); 
+
+	// Parse arguments
+	int argument_index = 0;
+	String next_token = null;
+
+	while(argument_index < args.length || next_token != null) {
+	    // 1. We start by attempting to parse an argument name. An argument
+	    //    must start with a '-', and should not contain spaces. If
+	    //    anything else is encountered it is ignored.
+
+	    String argument_name = null;
+	    if(next_token == null) {
+		next_token = args[argument_index];
+		argument_index++;
+	    }
+
+	    if(next_token.startsWith(StaticStrings.MINUS_CHARACTER)) {
+		// Trim second '-' just to be kind to Unixy-type people
+		if(next_token.startsWith(StaticStrings.MINUS_CHARACTER + StaticStrings.MINUS_CHARACTER)) {
+		    argument_name = next_token.substring(1);
+		}
+		else {
+		    argument_name = next_token;
+		}
+	    }
+	    next_token = null;
+	    // 2. If we now have an argument name we continue by attempting
+	    //    to parse a value. A value is taken to be the sequence of
+	    //    space seperated Strings between the last argument name
+	    //    and up to but not including the next argument name. Of
+	    //    course an argument needn't have any value (ie -debug,
+	    //    -help), in which case value will be null.
+	    if(argument_name != null) {
+		String argument_value = null;
+		StringBuffer argument_value_buffer = new StringBuffer("");
+		while(argument_index < args.length && next_token == null) {
+		    next_token = args[argument_index];
+		    argument_index++;
+		    // If we just parsed an arbitary String then append it to value, followed by a single space
+		    if(!next_token.startsWith(StaticStrings.MINUS_CHARACTER)) {
+			argument_value_buffer.append(next_token);
+			argument_value_buffer.append(StaticStrings.SPACE_CHARACTER);
+			next_token = null;
+		    }
+		    // If the argument token retrieved is an argument name,
+		    // then leave it in next_token, which will cause the
+		    // argument parsing process to move onto the next step.
+		}
+		// If a value now exists in argument buffer, retrieve
+		// it. Remove the last character as it will be an erroneous
+		// space.
+		if(argument_value_buffer.length() > 0) {
+		    argument_value = argument_value_buffer.substring(0, argument_value_buffer.length() - 1);
+		}
+		
+		// 3. We now have the argument name, and any associated
+		//    value. We are ready to store the data in the
+		//    appropriate variables.
+		DebugStream.println("Parsed Argument: name=" + argument_name + (argument_value != null ? (", value=" + argument_value) : ", no value"));
+		// 3a. First those arguments that have no associated value
+		if(argument_value == null) {
+		    if(argument_name.equals(StaticStrings.HELP_ARGUMENT)) {
+			System.out.println(Dictionary.get("General.Usage"));
+			System.exit(0);
+		    }
+		    // Run GLI in debug mode. Produces debug log plus extra
+		    // messages.
+		    else if(argument_name.equals(StaticStrings.DEBUG_ARGUMENT)) {
+			debug = true;
+		    }
+		    // Run GLI with feedback enabled. 
+		    else if(argument_name.equals(StaticStrings.FEEDBACK_ARGUMENT)) {
+			feedback_enabled = true;
+		    }
+		    // Forces no loading on previous collection.
+		    else if(argument_name.equals(StaticStrings.NO_LOAD_ARGUMENT)) {
+			no_load = true;
+			filename = null;
+		    }
+		    // Use a remote Greenstone rather than a local one
+		    else if (argument_name.equals(StaticStrings.USE_REMOTE_GREENSTONE_ARGUMENT)) {
+			use_remote_greenstone = true;
+		    }
+		}
+		// 3b. Now for those that do
+		else {
+		    // Parse the path to the GSDL. Required argument.
+		    if(argument_name.equals(StaticStrings.GSDL_ARGUMENT)) {
+			if(argument_value.endsWith(File.separator)) {
+			    gsdl_path = argument_value;
+			}
+			else {
+			    gsdl_path = argument_value + File.separator;
+			}
+		    }
+		    // GSDL3 path
+		    if(argument_name.equals(StaticStrings.GSDL3_ARGUMENT)) {
+			if(argument_value.endsWith(File.separator)) {
+			    gsdl3_path = argument_value;
+			}
+			else {
+			    gsdl3_path = argument_value + File.separator;
+			}
+		    }
+		    // GSDL3 src path
+		    if(argument_name.equals(StaticStrings.GSDL3_SRC_ARGUMENT)) {
+			if(argument_value.endsWith(File.separator)) {
+			    gsdl3_src_path = argument_value;
+			}
+			else {
+			    gsdl3_src_path = argument_value + File.separator;
+			}
+		    }
+		    
+		    else if (argument_name.equals(StaticStrings.SITE_ARGUMENT)) {
+			site_name = argument_value;
+		    }
+		    else if (argument_name.equals(StaticStrings.SERVLET_ARGUMENT)) {
+			servlet_path = argument_value;
+		    }
+		    // Specify a non-standard collect directory to use (for running one GLI in a network environment)
+		    else if (argument_name.equals(StaticStrings.COLLECTDIR_ARGUMENT)) {
+			collect_directory_path = argument_value;
+			System.err.println("Non standard collect directory specified: " + collect_directory_path);
+		    }
+		    // Specify a collection to load initially. Could be used for file associations.
+		    else if(argument_name.equals(StaticStrings.LOAD_ARGUMENT)) {
+			filename = argument_value;
+			no_load = false;
+		    }
+		    // Parse the file path of the local library server
+		    else if (argument_name.equals(StaticStrings.LOCAL_LIBRARY_ARGUMENT)) {
+			local_library_path = argument_value;
+		    }
+		    // Manually specify the Greenstone library URL
+		    else if (argument_name.equals(StaticStrings.LIBRARY_URL_ARGUMENT)) {
+			library_url_string = argument_value;
+		    }
+		    // Specify the URL to the gliserver CGI script for remote collection building
+		    else if (argument_name.equals(StaticStrings.GLISERVER_URL_ARGUMENT)) {
+			gliserver_url_string = argument_value;
+		    }
+		    // Parse the path to PERL. If not provided it assumes
+		    // perl should be availble on the PATH.
+		    else if(argument_name.equals(StaticStrings.PERL_ARGUMENT)) {
+			perl_path = argument_value;
+			// Test whether this points to the Perl bin
+			// directory or the Perl executable itself.
+			File perl_file = new File(perl_path);
+			if(perl_file.isDirectory()) {
+			    // If this is windows we create a child file
+			    // perl.exe, otherwise we create perl
+			    if(Utility.isWindows()) {
+				perl_file = new File(perl_file, Utility.PERL_EXECUTABLE_WINDOWS);
+			    }
+			    else {
+				perl_file = new File(perl_file, Utility.PERL_EXECUTABLE_UNIX);
+			    }
+			    // And store this new path.
+			    perl_path = perl_file.getAbsolutePath();
+			    perl_file = null;
+			}
+			// Otherwise it is fine as it is
+		    }
+		    // Parse the path to WGet
+		    else if (argument_name.equals(StaticStrings.WGET_ARGUMENT)) {
+			// Check the specified WGet file exists
+			if (argument_value != null && !argument_value.equals("")) {
+			    File wget_file = new File(argument_value);
+			    if (wget_file.exists()) {
+				wget_path = argument_value;
+			    } 
+			}
+		    }
+		}
+	    }
+	    // Argument name was null, nothing to be done.
+	}
+	next_token = null;
+	// Arguments all parsed.
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalGreenstone.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalGreenstone.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalGreenstone.java	(revision 21112)
@@ -0,0 +1,64 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer;
+
+
+import java.io.*;
+
+
+/**
+ * This class contains static functions that only make sense when running with a local Greenstone.
+ * None of these functions should be called if the GLI is using a remote Greenstone.
+ */
+public class LocalGreenstone
+{
+    static private String gsdl_directory_path = null;
+
+
+    static public String getBinScriptDirectoryPath()
+    {
+	return gsdl_directory_path + "bin" + File.separator + "script" + File.separator;
+    }
+
+
+    static public String getDirectoryPath()
+    {
+	return gsdl_directory_path;
+    }
+
+
+    static public String getTmpDirectoryPath()
+    {
+	return gsdl_directory_path + "tmp" + File.separator;
+    }
+
+
+    static public void setDirectoryPath(String gsdl_directory_path_arg)
+    {
+	gsdl_directory_path = gsdl_directory_path_arg;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalLibraryServer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalLibraryServer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/LocalLibraryServer.java	(revision 21112)
@@ -0,0 +1,255 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer;
+
+
+import java.io.*;
+import java.lang.*;
+import java.net.*;
+import javax.swing.*;
+import org.greenstone.gatherer.util.GSDLSiteConfig;
+
+
+public class LocalLibraryServer
+{
+    static final private String ADD_COMMAND = "?a=config&cmd=add-collection&c=";
+    static final private String RELEASE_COMMAND = "?a=config&cmd=release-collection&c=";
+    static final private String QUIT_COMMAND = "?a=config&cmd=kill";
+
+    static private GSDLSiteConfig gsdlsite_cfg_file = null;
+    static private File local_library_server_file = null;
+
+    static private boolean running = false;
+
+    static public void addCollection(String collection_name)
+    {
+	config(ADD_COMMAND + collection_name);
+	// This is very important -- it ensures that the above command has finished
+	config("");
+    }
+
+
+    // Used to send messages to the local library
+    // Warning: this has a lot of potential for nasty race conditions
+    // The response code is returned immediately -- but this does not mean the local
+    //   library action has finished!
+    static private void config(String command)
+    {
+	if (Configuration.library_url == null) {
+	    System.err.println("Error: Trying to configure local library with null Configuration.library_url!");
+	    return;
+	}
+
+	try {
+	    URL url = new URL(Configuration.library_url.toString() + command);
+	    HttpURLConnection library_connection = (HttpURLConnection) url.openConnection();
+	    int response_code = library_connection.getResponseCode();
+	    if (response_code >= HttpURLConnection.HTTP_OK && response_code < HttpURLConnection.HTTP_MULT_CHOICE) {
+		DebugStream.println("200 - Complete.");
+	    }
+	    else {
+		DebugStream.println("404 - Failed.");
+	    }
+	}
+	catch (Exception ex) {
+	    DebugStream.printStackTrace(ex);
+	}
+    }
+
+
+    static public boolean isRunning()
+    {
+	if (!running) return false;
+	gsdlsite_cfg_file.load();
+	if (gsdlsite_cfg_file.getURL() == null)  return false;
+	return true;
+    }
+
+    static public void releaseCollection(String collection_name)
+    {
+	config(RELEASE_COMMAND + collection_name);
+	// This is very important -- it ensures that the above command has finished
+	config("");
+    }
+
+
+    static public void start(String gsdl_path, String local_library_server_file_path)
+    {
+	// Check the local library server.exe file exists
+	local_library_server_file = new File(local_library_server_file_path);
+	if (!local_library_server_file.exists()) {
+	    DebugStream.println("No local library at given file path.");
+
+	    local_library_server_file = new File(gsdl_path + "server.exe");
+	    if (!local_library_server_file.exists()) {
+		DebugStream.println("No local library at all.");
+		return;
+	    }
+	}
+
+	// Check if the server is already running
+	gsdlsite_cfg_file = new GSDLSiteConfig(local_library_server_file);
+	String url = gsdlsite_cfg_file.getURL();
+	if (url != null) {
+	    // If it is already running then set the Greenstone web server address and we're done
+	    try {
+		Configuration.library_url = new URL(url);
+		running = true;
+		return;
+	    }
+	    catch (MalformedURLException exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+
+	// Configure the server for immediate entry
+	gsdlsite_cfg_file.set();
+
+	// Spawn local library server process
+	String local_library_server_command = local_library_server_file.getAbsolutePath() + " " + gsdlsite_cfg_file.getSiteConfigFilename();
+	Gatherer.spawnApplication(local_library_server_command);
+
+	// Wait until program has started, by reloading and checking the URL field
+	gsdlsite_cfg_file.load();
+	int attempt_count = 0;
+	while (gsdlsite_cfg_file.getURL() == null) {
+	    new OneSecondWait();  // Wait one second (give or take)
+	    gsdlsite_cfg_file.load();
+	    attempt_count++;
+
+	    // After waiting a minute ask the user whether they want to wait another minute
+	    if (attempt_count == 60) {
+		int try_again = JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("Server.QuitTimeOut"), Dictionary.get("General.Warning"), JOptionPane.YES_NO_OPTION);
+		if (try_again == JOptionPane.NO_OPTION) {
+		    return;
+		}
+		attempt_count = 0;
+	    }
+	}
+
+	// Ta-da. Now the url should be available
+	try {
+	    Configuration.library_url = new URL(gsdlsite_cfg_file.getURL());
+	}
+	catch (MalformedURLException exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+
+	// A quick test involves opening a connection to get the home page for this collection
+	try {
+	    DebugStream.println("Try connecting to server on config url: '" + Configuration.library_url+ "'");
+	    URLConnection connection = Configuration.library_url.openConnection();
+	    connection.getContent();
+	}
+	catch (IOException bad_url_connection) {
+	    try {
+		// If this fails then we try changing the url to be localhost
+		Configuration.library_url = new URL(gsdlsite_cfg_file.getLocalHostURL());
+		DebugStream.println("Try connecting to server on local host: '" + Configuration.library_url + "'");
+		URLConnection connection = Configuration.library_url.openConnection();
+		connection.getContent();
+	    }
+	    catch (IOException worse_url_connection) {
+		DebugStream.println("Can't connect to server on either address.");
+		Configuration.library_url = null;
+		return;
+	    }
+	}
+
+	running = true;
+    }
+
+
+    static public void stop()
+    {
+	if (running == false) {
+	    return;
+	}
+
+	// Send the command for it to exit.
+	config(QUIT_COMMAND);
+
+	// Wait until program has stopped, by reloading and checking the URL field
+	gsdlsite_cfg_file.load();
+	int attempt_count = 0;
+	while (gsdlsite_cfg_file.getURL() != null) {
+	    new OneSecondWait();  // Wait one second (give or take)
+	    gsdlsite_cfg_file.load();
+	    attempt_count++;
+
+	    // After waiting a minute ask the user whether they want to wait another minute
+	    if (attempt_count == 60) {
+		int try_again = JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("Server.QuitTimeOut"), Dictionary.get("General.Warning"), JOptionPane.YES_NO_OPTION);
+		if (try_again == JOptionPane.NO_OPTION) {
+		    return;
+		}
+		attempt_count = 0;
+	    }
+	}
+
+	//if(gsdlsite_cfg_file.getURL() != null) {
+	//JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Server.QuitManual"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	//}
+
+	// Restore the gsdlsite_cfg.
+	gsdlsite_cfg_file.restore();
+
+	// If the local server is still running then our changed values will get overwritten.
+	if (gsdlsite_cfg_file.getURL() != null) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Server.QuitFailed"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	}
+
+	running = false;
+    }
+    
+    static public void checkServerRunning() {
+	if (!running) return; // don't worry about it if its not supposed to be running 
+	gsdlsite_cfg_file.load();
+	if (gsdlsite_cfg_file.getURL() == null) {
+	    // need to restart the server again
+	    gsdlsite_cfg_file.set();
+
+	    // Spawn local library server process
+	    String local_library_server_command = local_library_server_file.getAbsolutePath() + " " + gsdlsite_cfg_file.getSiteConfigFilename();
+	    Gatherer.spawnApplication(local_library_server_command);
+
+	}
+    }
+    static private class OneSecondWait
+    {
+	public OneSecondWait()
+	{
+	    synchronized(this) {
+		try {
+		    wait(1000);
+		}
+		catch (InterruptedException exception) {
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/ServletConfiguration.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/ServletConfiguration.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/ServletConfiguration.java	(revision 21112)
@@ -0,0 +1,157 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.io.*;
+import java.lang.ref.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.*;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class stores the mapping between sites and servlets available for them. The info is read in from the web.xml file.
+ * @author Katherine Don, New Zealand Digital Library, University of Waikato
+ * @version 
+ */
+public class ServletConfiguration {
+    
+    static public String ADD_COMMAND = "?a=s&sa=a&st=collection&sn=";
+    static public String DEACTIVATE_COMMAND = "?a=s&sa=d&st=collection&sn=";
+    private String gsdl3_path = "";
+    private ArrayList sites = null;
+    private HashMap mappings = null;
+
+    public ServletConfiguration(String gsdl3_path) {
+	
+	this.gsdl3_path = gsdl3_path;
+	
+	//String web_xml_path = gsdl3_path + File.separator + "web" + File.separator + "WEB-INF"+ File.separator + "web.xml";
+	File web_xml = new File(gsdl3_path + "WEB-INF"+ File.separator + "web.xml");
+
+	if (!web_xml.exists()) {
+	    DebugStream.println("Error: no web.xml found at "+web_xml.toString());
+	    return;
+	}
+
+	this.sites = new ArrayList();
+	// find the sites
+	File start = new File(Utility.getSitesDir(gsdl3_path));
+	File possible_sites[] = start.listFiles();
+	ArrayTools.sort(possible_sites);
+	for (int i=0; possible_sites != null && i < possible_sites.length; i++) {
+	    File site_config = new File(possible_sites[i], "siteConfig.xml");
+	    if (site_config.exists()) {
+		this.sites.add(possible_sites[i].getName());
+	    }
+	}
+	
+	this.mappings = new HashMap();
+	Document web_config = XMLTools.parseXMLFile(web_xml.getAbsolutePath(), false);
+       
+	NodeList servlet_mappings = web_config.getElementsByTagName("servlet-mapping");
+	// make a little map class
+	HashMap url_mappings = new HashMap();
+	for (int i=0; i<servlet_mappings.getLength(); i++) {
+	    Element map = (Element)servlet_mappings.item(i);
+	    String name = XMLTools.getValue(XMLTools.getNodeFromNamed(map, "servlet-name"));
+	    String pattern = XMLTools.getValue(XMLTools.getNodeFromNamed(map, "url-pattern"));
+	    url_mappings.put(name, pattern);
+
+	}
+
+	NodeList servlets = web_config.getElementsByTagName("servlet");
+	for (int i=0; i<servlets.getLength(); i++) {
+	    Element servlet = (Element)servlets.item(i);
+	    String name = XMLTools.getValue(XMLTools.getNodeFromNamed(servlet, "servlet-name"));
+	    String description = XMLTools.getValue(XMLTools.getNodeFromNamed(servlet, "description"));
+	    String site = getServletSite(servlet);
+	       
+	    if (site != null) {
+		ArrayList this_site = (ArrayList)mappings.get(site);
+		if (this_site == null) {
+		    this_site = new ArrayList();
+		}
+		String url = (String)url_mappings.get(name);
+		this_site.add(url);
+		mappings.put(site, this_site);
+	    }
+	    
+	}
+
+	web_config = null;
+	url_mappings = null;
+    }
+    
+    public ArrayList getSites() {
+	return this.sites;
+    }
+
+    public ArrayList getServletsForSite(String site) {
+	return (ArrayList)this.mappings.get(site);
+    }
+    public String getServletPath(String site) {
+	
+	// for now, get the first one
+	ArrayList servlets = (ArrayList)mappings.get(site);
+	if(servlets == null) {
+	    return null;
+	}
+	return (String)servlets.get(0);
+    }
+
+    private String getServletSite(Element servlet) {
+	
+	NodeList params = servlet.getElementsByTagName("init-param");
+	if ( params == null || params.getLength() == 0) {
+	    return null;
+	}
+	String site = null;
+	for (int i=0; i<params.getLength()&& site == null; i++) {
+	    String p_name = XMLTools.getValue(XMLTools.getNodeFromNamed(params.item(i), "param-name"));
+	    if (p_name.equals("site_name")) {
+		site = XMLTools.getValue(XMLTools.getNodeFromNamed(params.item(i), "param-value"));
+	    }
+	}
+	return site;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/WGet.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/WGet.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/WGet.java	(revision 21112)
@@ -0,0 +1,325 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer;
+
+import java.awt.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.collection.DownloadJob;
+
+/** This class provides access to the functionality of the WGet program, either by calling it via a shell script or by the JNI. It maintains a queue of pending jobs, and the component for showing these tasks to the user.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class WGet
+    extends Thread {
+
+    /** <i>true</i> if there is a task currently being carried out, <i>false</i> otherwise. */
+    private boolean busy = false;
+    /** <i>true</i> if verbose debug messages should be displayed, <i>false</i> otherwise. */
+    private boolean debug = false;
+    /** <i>true</i> if successfully completed tasks should be automatically removed from the job queue. */
+    private boolean remove_complete_jobs = true;
+
+    private JPanel filler_pane = null;
+    /** The panel that the task list will be shown in. */
+    private JPanel list_pane;
+    /** The job currently underway. */
+    private DownloadJob job;
+    /** A scroll pane which will be used to display the list of pending tasks. */
+    private JScrollPane list_scroll;
+    /** A queue of download tasks. */
+    private Vector job_queue;
+    /** A static flag used to switch between simple and advanced modes. If <i>true</i> the Process object is used to externally call the Wget program. If <i>false</i> the native WGet libraries are statically loaded and the JNI used to download directly. */
+    static final private boolean simple = true;
+    /** Load the WGet native library. */
+    static {
+	if(!simple) {
+	    System.load(System.getProperty("user.dir") + File.separator + "libgatherer.so");
+	}
+    }
+
+    /** Constructor. Nothing special. */
+    public WGet() {
+	job = null;
+	job_queue = new Vector();
+	filler_pane = new JPanel();
+	list_pane = new JPanel();
+	list_pane.setLayout(new BoxLayout(list_pane, BoxLayout.Y_AXIS));
+	//list_pane.setLayout(new GridLayout(height_count,1));
+	list_scroll = new JScrollPane(list_pane);
+	//list_scroll.setHorizontalScrollBarPolicy(JScrollPane.HORIZONTAL_SCROLLBAR_NEVER);
+    }
+
+    /** Called by the WGet native code to inform us of a new download starting within the given job.
+     * @param url The url that is being downloaded, as a <strong>String</strong>.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized void addDownload(String url) {
+	job.addDownload(url);
+    }
+
+    /* Used to advise the DownloadJob of a newly parsed link. Its up to DownloadJob to decide if it already knows about this url, and if not to update its progress bar.
+     * @param url The url in question as a <strong>String</strong>.
+     * @param type <i>true</i> if this is an internal link, <i>false</i> for and external one.
+     * @return A <i>boolean</i> with a value of <i>true</i> indicating if the url was added, <i>false</i> otherwise.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized boolean addLink(String url, int type) {
+	return job.addLink(url, type);
+    }
+
+
+    /** This method is called to delete a certain job from the queue.
+     * This job may be pending, complete or even in progress. However
+     * if it is currently downloading then the delete method must
+     * wait until the native job has cleanly exited before removing
+     * the job.
+     * @param delete_me The <strong>DownloadJob</strong> that is to be deleted.
+     */
+    public void deleteDownloadJob(DownloadJob delete_me) {
+	if (delete_me == job) {
+	    // While this seems wasteful its only for the briefest moment.
+	    while(busy) {
+	    }
+	    job = null;
+	}
+	if (delete_me.hasSignalledStop()) {
+	    list_pane.remove(delete_me.getProgressBar());
+	    job_queue.remove(delete_me);
+	    // Unfortunately removing a task gets a bit more complicated as we have to resize the filler
+	    list_pane.remove(filler_pane);
+	    if(job_queue.size() > 0) {
+		Dimension progress_bar_size = delete_me.getProgressBar().getPreferredSize();
+		Dimension list_pane_size = list_pane.getSize();
+		int height = list_pane_size.height - (job_queue.size() * progress_bar_size.height);
+		progress_bar_size = null;
+		if(height > 0) {
+		    filler_pane.setPreferredSize(new Dimension(list_pane_size.width, height));
+		    list_pane.add(filler_pane);
+		}
+		list_pane_size = null;
+	    }
+	    list_pane.updateUI();
+	}
+	else {
+	    DebugStream.println("Somehow we're trying to delete a job that is still running.");
+	}
+    }
+
+    /** Called by the WGet native code when the current download,
+     * for the indicated job, is completed. In turn all download
+     * listeners are informed.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized void downloadComplete() {
+	job.downloadComplete();
+    }
+
+    /** Called by the WGet native code when the requested download returns
+     * a status code other than 200-399 for the specified download job.
+     * for.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized void downloadFailed() {
+	// Add the failed download as a new job if the user so requests.
+	job.downloadFailed();
+    }
+
+    /** Called by the WGet native code when some non-fatal error has caused
+     * a download to fail. An example of a warning would be if a file can't
+     * be downloaded as doing so would clobber an existing file and the -nc
+     * flag is set.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized void downloadWarning() {
+	job.downloadWarning();
+    }
+
+    /** Used by other graphic functions to get a reference to the
+     * scroll pane containing the current list of jobs.
+     */
+    public JScrollPane getDownloadJobList() {
+	return list_scroll;
+    }
+
+    /** Returns the current state of the stop flag for the job indicated.
+     * @return A boolean representing whether the user has requested to
+     * stop.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized boolean hasSignalledStop() {
+	return job.hasSignalledStop();
+    }
+
+    /** Creates a new mirroring job on the queue given the target url and the destination (private, public). All other details are harvested from the config file, but these two must be  captured from the GUI's current state.
+     * @param no_parents
+     * @param other_hosts
+     * @param page_requisites
+     * @param url a URL which points to the root url for the mirroring
+     * @param depth
+     * @param destination the destination file as a String
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     * @see org.greenstone.gatherer.gui.DownloadProgressBar
+     * @see org.greenstone.gatherer.util.GURL
+     */
+    public void newDownloadJob(boolean no_parents, boolean other_hosts, boolean page_requisites, URL url, int depth, String destination) {
+	// Create the job and fill in the details from gatherer.config.
+	DebugStream.println("About to create a new job");
+
+	DownloadJob new_job = new DownloadJob(Configuration.get("mirroring.debug", false), no_parents, other_hosts, page_requisites, Configuration.get("mirroring.quiet", false), url, depth, destination, Configuration.proxy_pass, Configuration.proxy_user, this, simple);
+
+	// Tell it to run as soon as possible
+	new_job.setState(DownloadJob.RUNNING);
+
+	// Add to job_queue job list.
+	job_queue.add(new_job);
+	// Now add it to the visual component, job list.
+	list_pane.remove(filler_pane);
+	Dimension progress_bar_size = new_job.getProgressBar().getPreferredSize();
+	Dimension list_pane_size = list_pane.getSize();
+	int height = list_pane_size.height - (job_queue.size() * progress_bar_size.height);
+	progress_bar_size = null;
+	list_pane.add(new_job.getProgressBar());
+	if(height > 0) {
+	    filler_pane.setPreferredSize(new Dimension(list_pane_size.width, height));
+	    list_pane.add(filler_pane);
+	}
+	list_pane_size = null;
+	//list_pane.setAlignmentX(Component.LEFT_ALIGNMENT);
+	list_pane.updateUI();
+	new_job = null;
+	synchronized(this) {
+	    notify(); // Just incase its sleeping.
+	}
+    }
+
+    /** Called by the WGet native code to signal the current progress of
+     * downloading for the specified job.
+     * @param current A long representing the number of bytes that have
+     * been downloaded since the last update.
+     * @param expected A long representing the total number of bytes
+     * expected for this download.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public synchronized void updateProgress(long current, long expected) {
+	job.updateProgress(current, expected);
+    }
+
+    /* There may be times when the download thread is sleeping, but the
+     * user has indicated that a previously paused job should now begin
+     * again. The flag within the job will change, so we tell the thread
+     * to start again.
+     */
+    public void resumeThread() {
+	synchronized(this) {
+	    notify(); // Just incase its sleeping.
+	}
+    }
+
+    /* This begins the WGet thread, which simply iterates through the waiting
+     * jobs attempting each one. Successful downloads are removed from the
+     * waiting list.
+     * @see org.greenstone.gatherer.collection.DownloadJob
+     */
+    public void run() {
+	while(true) {
+	    // If there are jobs job_queue and we have more room.
+	    if(job_queue.size() > 0) {
+		int index = 0;
+		while(job_queue.size() > 0 && index < job_queue.size()) {
+		    // Get the first job that isn't stopped.
+		    job = (DownloadJob) job_queue.get(index);
+		    if(job.getState() == DownloadJob.RUNNING) {
+			DebugStream.println("DownloadJob " + job.toString() + " Begun.");
+			// A lock to prevent us deleting this job while its being
+			// run, unless you want things to go really wrong.
+			busy = true;
+			if(simple) {
+			    job.callWGet();
+			}
+			else {
+			    job.callWGetNative();
+			}
+			busy = false;
+			DebugStream.println("DownloadJob " + job.toString() + " complete.");
+			// And if the user has requested that complete jobs
+		       	// be removed, then remove it from the list.
+			//deleteDownloadJob(job);
+			job = null;
+		    }
+		    index++;
+		}
+	    }
+	    // In order to save processor time, I'll suspend the thread
+	    // if theres no advantage to it running. Actions such as
+	    // new or complete jobs will resume the thread.
+	    else {
+		try {
+		    synchronized(this) {
+			DebugStream.println("WGet thread is waiting for DownloadJobs.");
+			wait();
+		    }
+		} catch (InterruptedException e) {
+		    // Time to get going again.
+		}
+	    }
+	} // End While.
+    }
+    
+    /* Link to the call to the WGet Native method for downloading.
+     * @param argc An int representing the number of elements in argv.
+     * @param argv An array of objects passed as arguments to wget.
+     * @param debug Whether the native code should show java-only debug
+     * messages.
+     */
+    public native int wget(int argc, Object argv[], boolean debug);
+
+    /* Link to the call to the WGet Native method for converting url links.
+     * @param argc An int representing the number of object elements in argv.
+     * @param argv An array of objects passed as arguments to wget.
+     * @param urlc An int representing the number of url string elements in urlv.
+     * @param urlv An array of strings representing urls.
+     * @param filec An int representing the number of file string elements in filev.
+     * @param filev An array of strings representing files.
+     */
+    public native void wren(int argc, Object argv[], int urlc, Object urlv[], int filec, Object filev[]);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Argument.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Argument.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Argument.java	(revision 21112)
@@ -0,0 +1,621 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Greenstone Librarian Interface (GLI) application, 
+ * part of the Greenstone digital library software suite from the New 
+ * Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
+ *
+ * Author: John Thompson 
+ *         Greenstone Project, New Zealand Digital Library
+ *         University of Waikato
+ *         http://www.nzdl.org
+ *
+ * Copyright (C) 2004 New Zealand Digital Library, University of Waikato
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.cdm;
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.Collection;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class contains all the details about a single argument that can be passed to this plugin, including option lists if the parameters are restricted.
+ * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+ * @version 2.41 final
+ */
+public class Argument
+    implements Comparable, Serializable {
+    /** An element of the argument type enumeration specifying a combobox control. */
+    static final public byte ENUM = 0;
+    /** An element of the argument type enumeration specifying a checkbox control. */
+    static final public byte FLAG = 1;
+    /** An element of the argument type enumeration specifying a tree control. */
+    static final public byte HIERARCHY = 2;
+    /** An element of the argument type enumeration specifying a spinner control. */
+    static final public byte INTEGER = 3;
+    /** An element of the argument type enumeration specifying a language combobox control. */
+    static final public byte LANGUAGE = 4;
+    /** An element of the argument type enumeration specifying a list control. */
+    static final public byte METADATA = 5;
+    /** An element of the argument type enumeration specifying a metadata combobox control. */
+    static final public byte METADATUM = 6;
+    /** An element of the argument type enumeration specifying a text field. */
+    static final public byte STRING = 7;
+    /** An element of the argument type enumeration specifying a regular expression text field. */
+    static final public byte REGEXP = 8;
+    /** true if this argument should actually be hidden within the GLI. This is iportant for arguments such as import dir or other location critical arguments. */
+    private boolean hidden_gli = false;
+    /** <i>true</i> if this argument is required for the applicable script to work properly, <i>false</i> otherwise. */
+    private boolean required = false;
+    /** The type of this argument. Used to be an int, but bytes are cheaper. */
+    private byte type = STRING;
+    /** The maximum value an integer based control can have. */
+    private int maximum = Integer.MAX_VALUE;
+    /** The minimum value an integer based control can have. */
+    private int minimum = Integer.MIN_VALUE;
+    /** Every argument has a detail mode level at which it becomes available to the user to edit. 
+     * @see org.greenstone.gatherer.Configuration
+     */
+    private int mode_level = Configuration.LIBRARIAN_MODE;
+    /** The DOM element this argument is built around, if any. */
+    private Element element;
+    /** If the argument is of type ENUM then this map holds all the various options. Each entry is an &lt;option value&gt; -&gt; &lt;description&gt; mapping. */
+    private HashMap list = null;
+    /** A default value for parameter-type arguments. May be a Perl pattern. */
+    private String default_value = null;
+    /** The text description of this argument parsed from the pluginfo output. */
+    private String description = null;
+    /** The argument flag as it appears in the command. Also used as the unique identifier of an argument. */
+    private String name = null;
+    /** The plugin that owns this argument, for the purposes of visualising inheritance. */
+    private String owner = null;
+
+    /** Default Constructor. */
+    public Argument() {
+    }
+
+    /** Another constructor but this one is a little more interesting as it takes a DOM element.
+     * @param element the Element this argument is based around
+     */
+    public Argument(Element element) {
+	this.element = element;
+    }
+
+    /** Method to add an element to the option list.
+     * @param name the name value of the option as a String
+     * @param desc the description of this options as a String
+     */
+    public void addOption(String name, String desc) {
+	if(type == ENUM && name != null) {
+	    if(desc == null) {
+		desc = "";
+	    }
+	    if(list == null) {
+		list = new HashMap();
+	    }
+	    list.put(name, desc);
+	}
+    }
+
+    /** Method to compare two arguments for ordering.
+     * @param object the argument we are comparing to, as an Object
+     * @return an int specifying the argument order, using values as set out in String
+     * @see org.greenstone.gatherer.cdm.Argument
+     */
+    public int compareTo(Object object) {
+	if(object instanceof Argument) {
+	    return getName().compareTo(((Argument)object).getName());
+	}
+	else {
+	    return toString().compareTo(object.toString());
+	}
+    }
+
+    /** Create a copy of this argument.
+     * @return a newly created Argument with the same details as this one
+     */
+    public Argument copy() {
+	Argument copy = new Argument();
+	copy.setDefaultValue(default_value);
+	copy.setDescription(description);
+	copy.setOptions(list);
+	copy.setOwner(owner);
+	copy.setName(name);
+	copy.setRequired(required);
+	copy.setType(type);
+	copy.setMinimum(minimum);
+	copy.setMaximum(maximum);
+	copy.setModeLevel(mode_level);
+	return copy;
+    }
+
+    /** Method to determine if two arguments are equal.
+     * @param object the argument to test against, as an Object
+     * @return true if the arguments names match, false otherwise
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Method to retrieve the value of default_value.
+     * @return a String containing the default value
+     */
+    public String getDefaultValue() {
+	return default_value;
+    }
+
+    /** Method to retrieve this arguments description.
+     * @return a String containing the description
+     */
+    public String getDescription() {
+	return description;
+    }
+
+    /** Method to retrieve the description of a certain list option value.
+     * @param key the String whose description we are searching for
+     * @return the description of the desired key as a String which may be empty if no such key exists
+     */
+    public String getDescription(String key) {
+	if(list.containsKey(key)) {
+	    return (String)list.get(key);
+	}
+	return "";
+    }
+
+    /** Retrieve the upper bound of a range based argument.
+     * @return the maximum as an int
+     */
+    public int getMaximum() {
+	return maximum;
+    }
+
+    /** Retrieve the lower bound of a range based argument.
+     * @return the minimum as an int
+     */
+    public int getMinimum() {
+	return minimum;
+    }
+
+    /** Retrieves the mode level at which this argument should become available. Any higher levels should also see this argument.
+     * @return the mode level as an int
+     */
+    public int getModeLevel() {
+	return mode_level;
+    }
+
+    /** Method to retrieve the value of name.
+     * @return a String containing the argument name
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */
+    public String getName() {
+	if(name == null && element != null) {
+	    name = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+	}
+	return name;
+    }
+
+    /** Method to retrieve the option list for this argument.
+     * @return a HashMap containing &lt;option value&gt; -&gt; &lt;description&gt; entries
+     */
+    public HashMap getOptions() {
+	return list;
+    }
+
+    /** Retrieve the name of the owner of this argument.
+     * @return the owners name as a String
+     */
+    public String getOwner() {
+	return owner;
+    }
+    
+    /** Method to determine the type of this argument.
+     * @return a byte specifying the type
+     */
+    public byte getType() {
+	return type;
+    }
+
+    /** Method to retrieve the value of value.
+     * @return the value of value as a String
+     * @see org.greenstone.gatherer.Gatherer#c_man
+     * @see org.greenstone.gatherer.collection.CollectionManager#getCollection
+     */
+    public String getValue()
+    {
+	// Only assigned arguments have values.
+	if (element == null) {
+	    return null;
+	}
+
+	return XMLTools.getValue(element);
+    }
+
+
+    /** Method to determine if this argument has been assigned.
+     * @return true if it has, false otherwise
+     * @see org.greenstone.gatherer.util.StaticStrings#ASSIGNED_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#TRUE_STR
+     */
+    public boolean isAssigned() {
+	return (element != null && element.getAttribute(StaticStrings.ASSIGNED_ATTRIBUTE).equals(StaticStrings.TRUE_STR));
+    }
+
+    /** Determine if this is a custom argument ie one that has been parsed from the config file but doesn't have a matching entry in the argument library.
+     * @return true if this argument is a custom, false otherwise
+     * @see org.greenstone.gatherer.util.StaticStrings#CUSTOM_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#TRUE_STR
+     */
+    public boolean isCustomArgument() {
+	return (element != null && element.getAttribute(StaticStrings.CUSTOM_ATTRIBUTE).equals(StaticStrings.TRUE_STR));
+    }
+
+    /** Determine if this argument is hidden in GLI
+     * @return true if the argument is hidden, false otherwise
+     */
+    public boolean isHiddenGLI() {
+	return hidden_gli;
+    }
+
+    /** Method to determine of this argument is required for the associated script to work.
+     * @return true if this argument is required, false otherwise
+     */
+    public boolean isRequired() {
+	return required;
+    }
+
+    /** Method to allow for the activation of arguments that might never have their setValue() method called.
+     * @param assigned the desired state as a boolean
+     * @see org.greenstone.gatherer.util.StaticStrings#ASSIGNED_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#FALSE_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#TRUE_STR
+     */
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, (assigned ? StaticStrings.TRUE_STR : StaticStrings.FALSE_STR));
+	}
+    }
+
+    /** Set the custom argument flag.
+     * @param custom true to make this argument custom, false otherwise
+     * @see org.greenstone.gatherer.util.StaticStrings#CUSTOM_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#FALSE_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#TRUE_STR
+     */
+    public void setCustomArgument(boolean custom) {
+	if(element != null) {
+	    element.setAttribute(StaticStrings.CUSTOM_ATTRIBUTE, (custom ? StaticStrings.TRUE_STR : StaticStrings.FALSE_STR));
+	}
+    }
+
+    /** Sets the value of default_value.
+     * @param default_value The new value for default_value as a <strong>String</strong>.
+     */
+    public void setDefaultValue(String default_value) {
+	this.default_value = default_value;
+    }
+
+    /** Set the value of desc.
+     * @param description the new value of desc as a String
+     */
+    public void setDescription(String description) {
+	this.description = description;
+    }
+
+    /** Set the element this argument should be based upon.
+     * @param element the Element
+     */
+    public void setElement(Element element) {
+	this.element = element;
+    }
+
+    /** Mark this argument as being hidden in GLI. */
+    public void setHiddenGLI() {
+	hidden_gli = true;
+    }
+
+    /** Set the upper bound for a range type argument.
+     * @param maximum the maximum as an int
+     */
+    public void setMaximum(int maximum) {
+	this.maximum = maximum;
+    }
+
+    /** Set the lower bound for a range type argument.
+     * @param minimum the minimum as an int
+     */
+    public void setMinimum(int minimum) {
+	this.minimum = minimum;
+    }
+
+    /** Set the detail mode level where this argument will become available.
+     * @param mode_level the mode level as an int
+     */
+    public void setModeLevel(int mode_level) {
+	this.mode_level = mode_level;
+    }
+
+    /** Set the value of name.
+     * @param name the new value of name as a String
+     */
+    public void setName(String name) {
+	this.name = name;
+    }
+
+    /** Sets the value of the options list.
+     * @param list the new options list as a HashMap
+     */
+    public void setOptions(HashMap list) {
+	this.list = list;
+    }
+
+    /** Set the owner of this argument.
+     * @param owner the name of the owner of this argument as a String
+     */
+    public void setOwner(String owner) {
+	this.owner = owner;
+    }
+
+    /** Set the value of required.
+     * @param required the new value of required as a boolean
+     */
+    public void setRequired(boolean required) {
+	this.required = required;
+    }
+
+    /** Set the value of type.
+     * @param type the new value of type as an byte
+     */
+    public void setType(byte type) {
+	this.type = type;
+    }
+
+    /** Set the value of type, by matching a type to the given string.
+     * @param new_type a String which contains the name of a certain argument type
+     * @see org.greenstone.gatherer.util.StaticStrings#ENUM_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#FLAG_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#HIERARCHY_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#INT_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#METADATA_TYPE_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#METADATUM_TYPE_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#REGEXP_STR
+     */
+    public void setType(String new_type) {
+	if(new_type.equalsIgnoreCase(StaticStrings.ENUM_STR)) {
+	    this.type = ENUM;
+	    list = new HashMap();
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.FLAG_STR)) {
+	    this.type = FLAG;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.HIERARCHY_STR)) {
+	    this.type = HIERARCHY;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.INT_STR)) {
+	    this.type = INTEGER;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.LANGUAGE_STR)) {
+	    this.type = LANGUAGE;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.METADATA_TYPE_STR)) {
+	    this.type = METADATA;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.METADATUM_TYPE_STR)) {
+	    this.type = METADATUM;
+	}
+	else if(new_type.equalsIgnoreCase(StaticStrings.REGEXP_STR)) {
+	    this.type = REGEXP;
+	}
+	else {
+	    this.type = STRING;
+	}
+    }
+
+    /** Method to set the value of this argument.
+     * @param value the new value for the argument
+     * @see org.greenstone.gatherer.Gatherer#println
+     */
+    public void setValue(String value) {
+	if(element != null) {
+	    XMLTools.setValue(element, value);
+	}
+	else {
+	    DebugStream.println("Argument.setValue(" + value + ") called on a base Argument.");
+	}
+    }
+
+    /** Set the values vector to the given values. Currently I just assign the new values, whereas I may later want to implement a deep clone.
+     * @param values an ArrayList of values
+     * @see org.greenstone.gatherer.Gatherer#println
+     */
+    public void setValues(ArrayList values) {
+	if(element != null) {
+	    StringBuffer value = new StringBuffer();
+	    int value_length = values.size();
+	    for(int i = 0; i < value_length; i++) {
+		value.append(values.get(i));
+		value.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	    value.deleteCharAt(value.length() - 1); // Remove last ','
+	    XMLTools.setValue(element, value.toString());
+	}
+	else {
+	    DebugStream.println("Argument.setValues([" + values.size() + " items]) called on a base Argument.");
+	}
+    }
+
+    /** Method for translating the data of this class into a string.
+     * @return a String containing a fragment of the total arguments string
+     * @see org.greenstone.gatherer.Gatherer#c_man
+     * @see org.greenstone.gatherer.collection.CollectionManager#getCollection
+     * @see org.greenstone.gatherer.util.StaticStrings#COMMA_CHARACTER
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#SPACE_CHARACTER
+     * @see org.greenstone.gatherer.util.StaticStrings#SPEECH_CHARACTER
+     */
+    public String toString()
+    {
+	StringBuffer text = new StringBuffer("-");
+
+	if (element == null) {
+	    return text.toString();
+	}
+
+	if (name == null) {
+	    name = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+	}
+	text.append(name);
+
+	String value = XMLTools.getValue(element);
+	if (value.length() == 0) {
+	    return text.toString();
+	}
+
+	text.append(StaticStrings.SPACE_CHARACTER);
+
+// 	// Handle metadata elements specially
+// 	if (type == METADATA || type == METADATUM) {
+// 	    // Tokenize the string
+// 	    StringTokenizer tokenizer = new StringTokenizer(value, ",");
+// 	    while (tokenizer.hasMoreTokens()) {
+// 		String token = tokenizer.nextToken();
+
+// 		MetadataElement metadata_element = MetadataTools.getMetadataElementWithDisplayName(token);
+// 		if (metadata_element != null) {
+// 		    text.append(metadata_element.getFullName());
+// 		}
+// 		else {
+// 		    text.append(token);
+// 		}
+
+// 		if (tokenizer.hasMoreTokens()) {
+// 		    text.append(StaticStrings.COMMA_CHARACTER);
+// 		}
+// 	    }
+// 	    return text.toString();
+// 	}
+
+	// If the value contains a space, add speech marks
+	//   (Except for metadata elements, which won't have spaces when written out to collect.cfg)
+	if (value.indexOf(StaticStrings.SPACE_CHARACTER) != -1 && !(type == METADATUM || type == METADATA)) {
+	    value = StaticStrings.SPEECH_CHARACTER + value + StaticStrings.SPEECH_CHARACTER;
+	}
+
+	text.append(value);
+	return text.toString();
+    } 
+
+    /** parse the <Option> XML from eg import.pl -xml or pluginfo.pl -xml */
+    public void parseXML(Element option) {
+
+	for(Node node = option.getFirstChild(); node != null; node = node.getNextSibling()) {
+	    String node_name = node.getNodeName();
+	    if(node_name.equals("Name")) {
+		setName(XMLTools.getValue(node));
+	    }
+	    else if(node_name.equals("Desc")) {
+		setDescription(XMLTools.getValue(node));
+	    }
+	    else if(node_name.equals("Type")) {
+		setType(XMLTools.getValue(node));
+	    }
+	    else if(node_name.equals("Default")) {
+		setDefaultValue(XMLTools.getValue(node));
+	    }
+	    else if(node_name.equals("Required")) {
+		String v = XMLTools.getValue(node);
+		if(v != null && v.equals("yes")) {
+		    setRequired(true);
+		}
+	    }
+	    else if(node_name.equals("List")) {
+		// Two final loops are required to parse lists.
+		for(Node value = node.getFirstChild(); value != null; value = value.getNextSibling()) {
+		    if(value.getNodeName().equals("Value")) {
+			String key = null;
+			String desc = "";
+			for(Node subvalue = value.getFirstChild(); subvalue != null; subvalue = subvalue.getNextSibling()) {
+			    node_name = subvalue.getNodeName();
+			    if(node_name.equals("Name")) {
+				key = XMLTools.getValue(subvalue);
+			    }
+			    else if(node_name.equals("Desc")) {
+				desc = XMLTools.getValue(subvalue);
+			    }
+			}
+			if(key != null) {
+			    addOption(key, desc);
+			}
+		    }
+		}
+	    }
+	    else if(node_name.equals("Range")) {
+		String range_raw = XMLTools.getValue(node);
+		int index = -1;
+		if((index = range_raw.indexOf(StaticStrings.COMMA_CHARACTER)) != -1) {
+		    if(index > 0) {
+			try {
+			    String first_number = range_raw.substring(0, index);
+			    setMinimum(Integer.parseInt(first_number));
+			    first_number = null;
+			}
+			catch(Exception exception) {
+			}
+		    }
+		    
+		    if(index + 1 < range_raw.length()) {
+			try {
+			    String second_number = range_raw.substring(index + 1);
+			    setMaximum(Integer.parseInt(second_number));
+			    second_number = null;    
+			}
+			catch(Exception exception) {
+			}
+		    }
+		}
+		// Else it wasn't a valid range anyway, so ignore it
+	    }
+	    else if(node_name.equals("HiddenGLI")) {
+		setHiddenGLI();
+	    }
+	    else if(node_name.equals("ModeGLI")) {
+		String mode_level_str = XMLTools.getValue(node);
+		try {
+		    int mode_level = Integer.parseInt(mode_level_str);
+		    setModeLevel(mode_level);
+		}
+		catch(Exception exception) {
+		    DebugStream.println("Exception in Argument.parseXML() - Unexpected but non-fatal");
+		    DebugStream.printStackTrace(exception);
+		}
+	    }
+	    
+	} // for each option
+	
+    } // parseXML
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentConfiguration.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentConfiguration.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentConfiguration.java	(revision 21112)
@@ -0,0 +1,1033 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.ModalDialog;
+import org.greenstone.gatherer.gui.SimpleMenuBar;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/** This class provides us with a dialog box which allows us to edit the arguments of either a PlugIn or a Classifier.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ * @see org.greenstone.gatherer.cdm.Classifier
+ * @see org.greenstone.gatherer.cdm.PlugIn
+ */
+public class ArgumentConfiguration
+    extends ModalDialog
+    implements ActionListener {
+    /** The data whose arguments we are editing. */
+    private ArgumentContainer data = null;
+    /** Argument these argument controls coloured or uncoloured (alternates to indicate inheritance). */
+    private boolean coloured = false;
+    /** Whether we have successfully edited the arguments associated with the ArgumentContainer or if we have failed to enter required arguments and have instead cancelled (which would cause argument additions to roll back). */
+    private boolean success = false;
+    /** A button to cancel this dialog. */
+    private JButton cancel = null;
+    /** A button to accept the changes and close the dialog. */
+    private JButton ok = null;
+    /** A reference to the ourselves so our inner classes can dispose us like a dialog. */
+    private ArgumentConfiguration self = null;
+    /** The central pane where a list of known arguments is displayed. */
+    private JPanel central_pane = null;
+    /** The field for entering custom arguments. */
+    private JTextField custom = null;
+    /** The panel for the custom arguments */
+    private JPanel custom_pane = null;
+    /** The name of the owner of the last argument control. */
+    private String previous_owner = null;
+    /** The size used for an argument label. */
+    static final private Dimension LABEL_SIZE = new Dimension(225, 25);
+    /** Size of a list. */
+    static final private Dimension LIST_SIZE = new Dimension(380, 50);
+    /** The size used for the dialog. */
+    static final private Dimension SIZE = new Dimension(800, 425);
+
+    /** Constructor.
+     * @param data The plugin or classifier whose arguments we are configuring, in the form of its supported <strong>ArgumentContainer</strong> interface.
+     * @see org.greenstone.gatherer.Configuration
+     */
+    public ArgumentConfiguration(ArgumentContainer data) {
+	super(Gatherer.g_man);
+	this.data = data;
+	this.self = this;
+
+	// Create
+	setModal(true);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("designingacollection")); // can we tell whether we are doing a classifier or plugin, to make the help more specific??
+	Dictionary.setText(this, "CDM.ArgumentConfiguration.Title");
+
+	central_pane = new JPanel();
+	JPanel content_pane = (JPanel) getContentPane();
+
+	custom_pane = new JPanel();
+	String custom_str = data.getCustom();
+	if (custom_str != null) {
+	    custom = new JTextField(custom_str);
+	}
+	else {
+	    custom = new JTextField();
+	}
+	JLabel custom_label = new JLabel();
+	custom_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.setText(custom_label, "CDM.ArgumentConfiguration.Custom");
+
+	JPanel header_pane = new JPanel();
+	JLabel header = new JLabel();
+	header.setHorizontalAlignment(JLabel.CENTER);
+	header.setOpaque(true);
+	String args[] = new String[1];
+	args[0] = data.getName();
+	Dictionary.setText(header, "CDM.ArgumentConfiguration.Header", args);
+
+	JPanel button_pane = new JPanel();
+	cancel = new GLIButton();
+	cancel.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	ok = new GLIButton();
+	ok.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok, "General.OK", "General.OK_Tooltip");
+
+	// Listeners
+	cancel.addActionListener(this);
+	ok.addActionListener(this);
+	ok.addActionListener(CollectionDesignManager.all_change_listener);
+
+	// Layout
+	custom_pane.setLayout(new BorderLayout());
+	custom_pane.add(custom_label, BorderLayout.WEST);
+	custom_pane.add(custom, BorderLayout.CENTER);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2));
+	button_pane.add(ok);
+	button_pane.add(cancel);
+
+	central_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	central_pane.setLayout(new BoxLayout(central_pane, BoxLayout.Y_AXIS));
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(header, BorderLayout.NORTH);
+	content_pane.add(new JScrollPane(central_pane), BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Now generate a set of controls for each of the arguments.
+	generateControls();
+
+	// Display on screen.
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	screen_size = null;
+    }
+
+    /** Any implementation of ActionListener must include this method so that we can be informed when an action has occured on one of the controls we are listening to.
+     * @param event An <strong>ActionEvent</strong> containing pertinant information about the event that fired this call.
+     * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl
+     * @see org.greenstone.gatherer.cdm.ArgumentContainer
+     */
+    public void actionPerformed(ActionEvent event) {
+	boolean cont = true;
+	if (event.getSource() == ok) {
+	    // Clear the current focus to ensure components such as combobox have correctly updated
+	    custom.requestFocus();
+	    // Update the details stored in the data objects arguments.
+	    data.setCustom(custom.getText());
+	    // Loop through each of the controls in the central pane, updating the matching argument as necessary.
+	    for(int i = 0; i < central_pane.getComponentCount(); i++) {
+		Component component = central_pane.getComponent(i);
+		if(component instanceof ArgumentControl) {
+		   cont = cont && ((ArgumentControl)component).updateArgument();
+		}
+	    }
+	    if(cont) {
+		success = true;
+	    }
+	}
+	if(cont) {
+	    dispose();
+	}
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	cancel = null;
+	central_pane = null;
+	custom_pane = null;
+	custom = null;
+	data = null;
+	ok = null;
+	self = null;
+    }
+
+    /** Method which actually forces the dialog to be shown on screen.
+     * @return <i>true</i> if the user completed configuration and pressed ok, <i>false</i> otherwise.
+     */
+    public boolean display() {
+	setVisible(true);
+	return success;
+    }
+
+    private void addHeader(String name, Color color) {
+	JPanel header = new JPanel();
+	header.setBackground(color);
+	JPanel inner_pane = new JPanel();
+	inner_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,5,5,5), BorderFactory.createRaisedBevelBorder()));
+	inner_pane.setBackground(color);
+	JLabel header_label = new JLabel("<html><strong>" + name + "</strong></html>");
+	header_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	header_label.setHorizontalAlignment(JLabel.CENTER);
+	header_label.setOpaque(true);
+
+	// Layout
+	inner_pane.setLayout(new BorderLayout());
+	inner_pane.add(header_label, BorderLayout.CENTER);
+
+	header.setLayout(new BorderLayout());
+	header.add(inner_pane, BorderLayout.CENTER);
+	central_pane.add(header);
+    }
+
+    /** Method to iterate through the arguments associated with whatever argument container we are building an argument control view for, creating the appropriate controls for each.
+     * @see org.greenstone.gatherer.cdm.Argument
+     * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl
+     */
+    private void generateControls() {
+	ArrayList arguments = data.getArguments(true, false);
+	int total_height = 250;
+	int mode = Configuration.getMode();
+	for(int i = 0; i < arguments.size(); i++) {
+	    Argument argument = (Argument) arguments.get(i);
+	    if(mode > Configuration.LIBRARIAN_MODE || !(argument.getType() == Argument.REGEXP)) {
+		ArgumentControl argument_control = new ArgumentControl(argument);
+		total_height = total_height - argument_control.getPreferredSize().height;
+		central_pane.add(argument_control);
+	    }
+	}
+	// now add in the custom args bit
+	coloured = !coloured;
+	Color color = (coloured ? Configuration.getColor("coloring.collection_heading_background", false) : Configuration.getColor("coloring.collection_tree_background", false));
+	addHeader(Dictionary.get("CDM.ArgumentConfiguration.Custom_Header"), color);
+	custom_pane.setBackground(color);
+	central_pane.add(custom_pane);
+	if(total_height > 0) {
+	    JPanel filler = new JPanel();
+	    filler.setPreferredSize(new Dimension(100, total_height));
+	    filler.setSize(new Dimension(100, total_height));
+	    central_pane.add(filler);
+	}
+    }
+
+    /** This class encapsulates all the technical difficulty of creating a specific control based on an Argument. */
+    private class ArgumentControl
+	extends JPanel {
+	/** The Argument this control will be based on. */
+	private Argument argument = null;
+
+	private Color colour_one = Configuration.getColor("coloring.collection_heading_background", false);
+	private Color colour_two = Configuration.getColor("coloring.collection_tree_background", false);
+	/** One of a possible two buttons available for adding to this control. */
+	private JButton one = null;
+	/** The second of two buttons available for adding to this control. */
+	private JButton two = null;
+	/** A checkbox to allow enabling or diabling of this Argument. */
+	private JCheckBox enabled = null;
+	/** Some form of editor component, such as a JComboBox or JTextField, used to set parameters to an Argument. */
+	private JComponent value = null;
+	/** Can be used in place of the other editor components if a list is required. */
+	private JList list = null;
+	/** Constructor.
+	 * @param argument The <strong>Argument</strong> this control will be built around.
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.AddListener
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.EnabledListener
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.HierarchyListener
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.ListOption
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.RemoveListener
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.ToolTipUpdater
+	 */
+	public ArgumentControl(Argument argument) {
+	    this.argument = argument;
+	    ///ystem.err.println("generating controls for arg "+argument.getName());
+	    String tip = "<html>" + argument.getDescription() + "</html>";
+	    tip = Utility.formatHTMLWidth(tip, 60);
+
+	    // If this is the first control, there is no history.
+	    if(previous_owner == null) {
+		///ystem.err.println("previous owner is null");
+		previous_owner = argument.getOwner();
+		addHeader(previous_owner, colour_two);
+	    }
+	    // Otherwise if the owner of the control has changed since the last argument, toggle the colouring of the control.
+	    else if(previous_owner != argument.getOwner()) {
+		///ystem.err.println("previous owner is different from current owner");
+		coloured = !coloured;
+		previous_owner = argument.getOwner();
+		addHeader(previous_owner, (coloured ? colour_one : colour_two));
+	    }
+	    // Create
+	    if(coloured) {
+		setBackground(colour_one);
+	    }
+	    else {
+		setBackground(colour_two);
+	    }
+	    JLabel owner_label = new JLabel(argument.getOwner());
+	    owner_label.setOpaque(false);
+	    JLabel label = new JLabel(argument.getName());
+	    label.setOpaque(false);
+	    label.setPreferredSize(LABEL_SIZE);
+	    label.setToolTipText(tip);
+	    enabled = new JCheckBox(argument.getName());
+	    enabled.setOpaque(false);
+	    enabled.setPreferredSize(LABEL_SIZE);
+	    enabled.setToolTipText(tip);
+	    JPanel inner_pane = new JPanel();
+	    inner_pane.setOpaque(false);
+	    String existing_value = argument.getValue();
+	    String default_value = argument.getDefaultValue();
+
+	    switch(argument.getType()) {
+	    case Argument.ENUM:
+		// Build an option model, wrapping each entry of the list table.
+		HashMap arg_list = argument.getOptions();
+		ArrayList options_model = new ArrayList();
+		Iterator it = arg_list.keySet().iterator();
+		while(it.hasNext()) {
+		    String key = (String) it.next();
+		    options_model.add(new ListOption(key, (String)arg_list.get(key)));
+		}
+		Collections.sort(options_model);
+		value = new GComboBox(options_model.toArray(), false);
+		((JComboBox)value).addActionListener(new ToolTipUpdater());
+		if(existing_value != null && existing_value.length() > 0) {
+		    // Select the correct value. Since they're all text strings we better iterate to be safe.
+		    selectValue((JComboBox)value, existing_value);
+		}
+		else if(default_value != null) {
+		    ///ystem.err.println("Default for argument: " + argument.getName());
+		    // Same as above except for default value.
+		    selectValue((JComboBox)value, default_value);
+		}
+		break;
+
+	    case Argument.FLAG:
+		// Only need the check box.
+		break;
+
+	    case Argument.HIERARCHY:
+		// I don't think these are used any more...
+		break;
+
+	    case Argument.INTEGER:
+		// Build a spinner
+		int initial_value=0;
+		// If there was an original value, set it.
+		if(existing_value != null && !existing_value.equals("")) {
+		    try {
+			initial_value = Integer.parseInt(existing_value);
+			//spinner.setValue(new Integer(existing_value));
+		    }
+		    catch (Exception error) {
+			DebugStream.println("ArgumentConfiguration Error: "+error);
+		    }
+		} else if (default_value != null && !default_value.equals("")) {
+		    try {
+			initial_value = Integer.parseInt(default_value);
+			//spinner.setValue(new Integer(default_value));
+		    }
+		    catch (Exception error) {
+			DebugStream.println("ArgumentConfiguration Error: "+error);
+		    }
+		}
+		if (initial_value < argument.getMinimum()) {
+		    initial_value = argument.getMinimum();
+		} else if (initial_value > argument.getMaximum()) {
+		    initial_value = argument.getMaximum();
+		}
+
+		JSpinner spinner = new JSpinner(new SpinnerNumberModel(initial_value, argument.getMinimum(), argument.getMaximum(), 1));
+
+		// And remember it
+		value = spinner;
+		break;
+
+	    case Argument.REGEXP:
+	    case Argument.STRING:
+		// If there is already a value set for this argument, use it
+		if (existing_value != null && !existing_value.equals("")) {
+		    value = new JTextField(existing_value);
+		    break;
+		}
+
+		// Use the default value, if there is one
+		if (default_value != null && !default_value.equals("")) {
+		    value = new JTextField(default_value);
+		    break;
+		}
+
+// 		// Special test just for the hfile field.
+// 		if (argument.getName().equals("hfile")) {
+// 		    // Work through previous controls looking for the metadata one.
+// 		    for (int i = 0; i < central_pane.getComponentCount(); i++) {
+// 			Object object = central_pane.getComponent(i);
+// 			if (object instanceof ArgumentControl) {
+// 			    ArgumentControl control = (ArgumentControl) object;
+// 			    if (control.toString().equals("metadata")) {
+// 				Object temp = control.getValue();
+// 				if (temp != null) {
+// 				    value = new JTextField(temp.toString() + ".txt");
+//                                  break;
+// 				}
+// 			    }
+// 			}
+// 		    }
+// 		}
+
+		// Blank field
+		value = new JTextField();
+		break;
+
+	    case Argument.LANGUAGE:
+		value = new GComboBox(CollectionDesignManager.language_manager.getLanguageCodes().toArray(), false);
+		// we want to display the language name not the code
+		((JComboBox)value).setRenderer(new LanguageListCellRenderer());
+		// Now ensure we have the existing value or default value selected if either exist in our known languages
+		String lang_name = null;
+		String selected_code = existing_value;
+		if(existing_value != null && !existing_value.equals("")) {
+		    lang_name = CollectionDesignManager.language_manager.getLanguageName(existing_value);
+		}
+		if(lang_name == null && default_value != null) {
+		    lang_name = CollectionDesignManager.language_manager.getLanguageName(default_value);
+		    selected_code = default_value;
+		}
+		if (lang_name != null) {
+		    ((JComboBox)value).setSelectedItem(selected_code);
+		}
+		break;
+
+	    case Argument.METADATUM:
+	    case Argument.METADATA:
+		value = new GComboBox(MetadataSetManager.getEveryMetadataSetElement(), false);
+
+		// Editable for advanced modes (allows things like dc.Title,ex.Title)
+		if (Configuration.getMode() > Configuration.ASSISTANT_MODE) {
+		    ((JComboBox) value).setEditable(true);
+		}
+
+		// Now ensure we have the existing value or default value selected if either exist.
+		if (existing_value != null && existing_value.length() > 0) {
+		    boolean found = selectValue((JComboBox) value, existing_value);
+		    // It's possible that this is a custom value and so doesn't exist in the combobox
+		    if (!found) {
+			// If so, add it then select it
+			((JComboBox) value).addItem(existing_value);
+			((JComboBox) value).setSelectedItem(existing_value);
+		    }
+		}
+		else if (default_value != null) {
+		    selectValue((JComboBox) value, default_value);
+		}
+		break;
+
+// ---- Special interface for adding and ordering multiple metadata items ----
+// Turned off at Ian's request!
+//  	    case Argument.METADATA:
+//  		// Comma separated metadata values.
+//  		ArrayList values = argument.getValues();
+//  		value = new GComboBox(MetadataSetManager.getEveryMetadataSetElement(), false);
+//  		//((JComboBox)value).setEditable(false);
+//  		DefaultListModel model = new DefaultListModel();
+//  		list = new JList(model);
+//  		list.setVisibleRowCount(3);
+//  		for(int i = 0; i < values.size(); i++) {
+//  		    model.addElement(values.get(i));
+//  		}
+
+//  		one = new GLIButton();
+//  		one.addActionListener(new AddListener((JComboBox)value, list));
+//  		one.setMnemonic(KeyEvent.VK_A);
+//  		Dictionary.setBoth(one, "CDM.ArgumentConfiguration.Add", "CDM.ArgumentConfiguration.Add_Tooltip");
+//  		two = new GLIButton();
+//  		two.addActionListener(new RemoveListener(list));
+//  		two.setMnemonic(KeyEvent.VK_R);
+//  		Dictionary.setBoth(two, "CDM.ArgumentConfiguration.Remove", "CDM.ArgumentConfiguration.Remove_Tooltip");
+
+//  		if(argument.getValues().size() > 0 || argument.isRequired()) {
+//  		    enabled.setSelected(true);
+//  		    list.setBackground(Color.white);
+//  		    list.setEnabled(true);
+//  		    one.setEnabled(true);
+//  		    two.setEnabled(true);
+//  		    value.setEnabled(true);
+//  		}
+//  		else {
+//  		    enabled.setSelected(false);
+//  		    list.setBackground(Color.lightGray);
+//  		    list.setEnabled(false);
+//  		    one.setEnabled(false);
+//  		    two.setEnabled(false);
+//  		    value.setEnabled(false);
+//  		}
+//  		break;
+	    } // end of switch
+
+	    // Enable or disable as necessary.
+	    if(argument.isRequired() || argument.isAssigned()) {
+		enabled.setSelected(true);
+		if(value != null) {
+		    value.setOpaque(true);
+		    value.setBackground(Color.white);
+		    value.setEnabled(true);
+		    if(value instanceof JSpinner) {
+			// Set enabled
+			JComponent c = ((JSpinner)value).getEditor();
+			if ( c instanceof JSpinner.DefaultEditor ) {
+			    JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+			    JFormattedTextField field = editor.getTextField();
+			    field.setEditable(true);
+			    field.setBackground(Color.white);
+			}
+		    }
+		}
+	    }
+	    else {
+		enabled.setSelected(false);
+		if(value != null) {
+		    value.setOpaque(true);
+		    value.setBackground(Color.lightGray);
+		    value.setEnabled(false);
+		    if(value instanceof JSpinner) {
+			// Set enabled
+			JComponent c = ((JSpinner)value).getEditor();
+			if ( c instanceof JSpinner.DefaultEditor ) {
+			    JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+			    JFormattedTextField field = editor.getTextField();
+			    field.setEditable(false);
+			    field.setBackground(Color.lightGray);
+			}
+		    }
+		}
+	    }
+	    // Listener
+	    if(value != null && !argument.isRequired()) {
+		enabled.addActionListener(new EnabledListener(one, two, list, value));
+	    }
+
+	    // Layout
+	    inner_pane.setLayout(new BorderLayout());
+	    if (argument.isRequired()) {
+		inner_pane.add(label, BorderLayout.WEST);
+	    }
+	    else {
+		inner_pane.add(enabled, BorderLayout.WEST);
+	    }
+
+	    if (list == null) {
+		enabled.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+		if (value != null) {
+		    inner_pane.add(value, BorderLayout.CENTER);
+		}
+	    }
+	    else {
+		JPanel left_pane = new JPanel(new BorderLayout());
+		left_pane.add(new JLabel(""), BorderLayout.NORTH);
+		left_pane.add(value, BorderLayout.CENTER);
+		left_pane.add(one, BorderLayout.SOUTH);
+		left_pane.setOpaque(false);
+
+		JPanel right_pane = new JPanel(new BorderLayout());
+		right_pane.add(new JScrollPane(list), BorderLayout.CENTER);
+		right_pane.add(two, BorderLayout.SOUTH);
+		right_pane.setOpaque(false);
+
+		JPanel control_pane = new JPanel(new GridLayout(1, 2));
+		control_pane.add(left_pane);
+		control_pane.add(right_pane);
+
+		inner_pane.add(control_pane, BorderLayout.CENTER);
+	    }
+
+	    setLayout(new BorderLayout());
+	    add(inner_pane, BorderLayout.CENTER);
+	}
+
+       public Argument getArgument() {
+	  return argument;
+       }
+
+	public Object getValue() {
+	    if(value instanceof JComboBox) {
+		return ((JComboBox)value).getSelectedItem();
+	    }
+	    else if(value instanceof JTextField) {
+		return ((JTextField)value).getText();
+	    }
+	    return null;
+	}
+	/** Identifies this control by returning the name of the Argument it is based on.
+	 * @return The name of the Argument as a <strong>String</strong>.
+	 * @see org.greenstone.gatherer.cdm.Argument
+	 */
+	public String toString() {
+	    return argument.getName();
+	}
+	/** Updates the enwrapped Argument using the values provided by the controls.
+	 * @return <i>true</i> if the update was successful, <i>false</i> otherwise.
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.ListOption
+	 * @see org.greenstone.gatherer.cdm.Language
+	 */
+	public boolean updateArgument() {
+	    if(enabled.isSelected() || argument.isRequired()) {
+		argument.setAssigned(false);
+		String result = null;
+		switch(argument.getType()) {
+		case Argument.ENUM:
+		    ListOption option = (ListOption)((JComboBox)value).getSelectedItem();
+		    if(option != null && option.getValue().length() > 0) {
+			argument.setValue(option.getValue());
+		    }
+		    else {
+			String args[] = new String[1];
+			args[0] = argument.getName();
+			if(argument.isRequired()) {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.Required_Argument", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			}
+			// They've left the field blank
+			else {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.No_Value", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			    argument.setValue(null);
+			}
+			args = null;
+			return false;
+		    }
+		    argument.setAssigned(true);
+		    return true;
+		case Argument.FLAG:
+		    // Should have already been handled above.
+		    argument.setAssigned(true);
+		    return true;
+		case Argument.INTEGER:
+		    result = ((JSpinner)value).getValue().toString();
+		    if(result.length() > 0) {
+			// Test if the value entered is a valid int.
+			try {
+			    int x = Integer.parseInt(result);
+			}
+			catch(NumberFormatException nfe) {
+			    String args[] = new String[2];
+			    args[0] = argument.getName();
+			    args[1] = result;
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.Bad_Integer", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			    args = null;
+			    return false;
+			}
+			argument.setValue(result);
+		    }
+		    else {
+			String args[] = new String[1];
+			args[0] = argument.getName();
+			if(argument.isRequired()) {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.Required_Argument", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			}
+			// They've left the field blank
+			else {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.No_Value", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			    argument.setValue(null);
+			}
+			args = null;
+			return false;
+		    }
+		    argument.setAssigned(true);
+		    return true;
+		case Argument.LANGUAGE:
+		    String language = (((JComboBox)value).getSelectedItem()).toString();
+		    argument.setValue(language);
+		    // Kinda lucked out here. Its impossible not to choose an entry from these comboboxes as they are restricted.
+		    argument.setAssigned(true);
+		    return true;
+		case Argument.METADATUM:
+		case Argument.METADATA:
+		    Object new_value_raw = ((JComboBox) value).getSelectedItem();
+		    if (new_value_raw instanceof MetadataElement) {
+			argument.setValue(((MetadataElement) new_value_raw).getFullName());
+		    }
+		    else {
+			// But we have to be careful as an arbitary string object could be zero length
+			String new_value = new_value_raw.toString();
+			///ystem.err.println("The current value is: " + new_value);
+			if(new_value.length() > 0) {
+			    argument.setValue(new_value);
+			}
+			else {
+			    String args[] = new String[1];
+			    args[0] = argument.getName();
+			    if(argument.isRequired()) {
+				JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.Required_Argument", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			    }
+			    // They've left the field blank
+			    else {
+				JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.No_Value", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+				argument.setValue(null);
+			    }
+			    args = null;
+			    return false;
+			}
+		    }
+		    argument.setAssigned(true);
+		    return true;
+//  		case Argument.METADATA:
+//  		    DefaultListModel model = (DefaultListModel)list.getModel();
+//  		    ArrayList values = new ArrayList();
+//  		    for(int i = 0; i < model.size(); i++) {
+//  			values.add(model.get(i));
+//  		    }
+//  		    argument.setValues(values);
+//  		    argument.setAssigned(true);
+//  		    return true;
+		case Argument.HIERARCHY:
+// 		    argument.setValue(((JComboBox)value).getSelectedItem().toString());
+// 		    // Kinda lucked out here. Its impossible not to choose an entry from these comboboxes as they are restricted.
+// 		    argument.setAssigned(true);
+		    return true;
+		case Argument.REGEXP:
+		case Argument.STRING:
+		    result = ((JTextField)value).getText();
+		    if(result.length() > 0) {
+			argument.setValue(result);
+		    }
+		    else {
+			String args[] = new String[1];
+			args[0] = argument.getName();
+			if(argument.isRequired()) {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.Required_Argument", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			}
+			// They've left the field blank
+			else {
+			    JOptionPane.showMessageDialog(self, Dictionary.get("CDM.ArgumentConfiguration.No_Value", args), Dictionary.get("CDM.ArgumentConfiguration.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			    argument.setValue(null);
+			}
+			args = null;
+			return false;
+		    }
+		    argument.setAssigned(true);
+		    return true;
+		}
+		return false;
+	    }
+	    else {
+		argument.setAssigned(false);
+		return true;
+	    }
+	}
+	/** Method to ensure that a certain value is selected, if it exists within that combobox to begin with.
+	 * @param combobox The <strong>JComboBox</strong> whose selection we are trying to preset.
+	 * @param target The desired value of the selection as a <strong>String</strong>.
+	 * @return true if the item was found and selected, false otherwise
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl.ListOption
+	 */
+	public boolean selectValue(JComboBox combobox, String target) {
+	    ///ystem.err.println("Searching for the target string: " + target);
+	    for(int i = 0; i < combobox.getItemCount(); i++) {
+		Object object = combobox.getItemAt(i);
+		if(object instanceof ListOption) {
+		    ListOption lo = (ListOption) object;
+		    ///ystem.err.print("/tChecking: " + lo.getValue() + "... ");
+		    if(lo.getValue().startsWith(target)) {
+			///ystem.err.println("Match!");
+			combobox.setSelectedIndex(i);
+			return true;
+		    }
+		    /*
+		    else {
+			System.err.println("No Match.");
+		    }
+		    */
+		}
+		else if (object instanceof MetadataElement) {
+		    if(object.toString().equals(target)) {
+			combobox.setSelectedIndex(i);
+			return true;
+		    }
+		}
+	    }
+	    return false;
+	}
+	/** Forces the control into an 'enabled' mode. */
+	public void setEnabled() {
+	    enabled.setSelected(true);
+	}
+	/** Explicitly sets the value of a JTextField type control to the given String.
+	 * @param value_str The new value of the control as a <strong>String</strong>.
+	 */
+	public void setValue(String value_str) {
+	    ((JTextField)value).setText(value_str);
+	}
+// 	/** Listener which adds entries to a list from a combobox when fired. */
+// 	private class AddListener
+// 	    implements ActionListener {
+// 	    /** The model behind the target list. */
+// 	    private DefaultListModel model = null;
+//       	    /** The source for data to be added to the list. */
+// 	    private JComboBox source = null;
+// 	    /** The list to add data to. */
+// 	    private JList target = null;
+// 	    /** Constructor.
+// 	     * @param source A <strong>JComboBox</strong> which serves as the source for data.
+// 	     * @param target A <strong>JList</strong> which serves as the target for data.
+// 	     */
+// 	    public AddListener(JComboBox source, JList target) {
+// 		this.model = (DefaultListModel) target.getModel();
+// 		this.source = source;
+// 		this.target = target;
+// 	    }
+// 	    /** When the add button is clicked, we attempt to add the selected metadata from the source into the target.
+// 	     * @param event An <strong>ActionEvent</strong> containing information about the event.
+// 	     */
+// 	    public void actionPerformed(ActionEvent event) {
+// 		ElementWrapper element = (ElementWrapper) source.getSelectedItem();
+// 		String name = element.toString();
+// 		if (!model.contains(name)) {
+// 		    model.addElement(name);
+// 		}
+// 	    }
+// 	}
+	/** Listens for actions apon the enable checkbox, and if detected enables or diables control appropriately. */
+	private class EnabledListener
+	    implements ActionListener {
+	    /** One of two possible buttons that might have their enabled state changed by this listener. */
+	    private JButton one = null;
+	    /** One of two possible buttons that might have their enabled state changed by this listener. */
+	    private JButton two = null;
+	    /** An editor component, such as a JComboBox or JTextField, that might have its enabled state changed by this listener. */
+	    private JComponent target = null;
+	    /** A list which might have its enabled state changed by this listener. */
+	    private JList list = null;
+	    /** Constructor.
+	     * @param one A <strong>JButton</strong> whose enabled state is determined by the listener, or <i>null</i> if no button.
+	     * @param two A <strong>JButton</strong> whose enabled state is determined by the listener, or <i>null</i> if no button.
+	     * @param list A <strong>JList</strong> whose enabled state is determined by the listener, or <i>null</i> if no list.
+	     * @param target A <strong>JComponent</strong> whose enabled state is determined by the listener, or <i>null</i> if no component.
+	     */
+	    public EnabledListener(JButton one, JButton two, JList list, JComponent target) {
+		this.list = list;
+		this.one = one;
+		this.target = target;
+		this.two = two;
+	    }
+	    /** Any implementation of ActionListener must include this method so that we can be informed when an action has been performed on or registered check box, prompting us to change the state of the other controls as per the users request.
+	     * @param event An <strong>ActionEvent</strong> containing information about the click.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		JCheckBox source = (JCheckBox)event.getSource();
+		if(source.isSelected()) {
+		    target.setBackground(Color.white);
+		    target.setEnabled(true);
+		    if(target instanceof JSpinner) {
+			// Set enabled
+			JComponent c = ((JSpinner)target).getEditor();
+			if ( c instanceof JSpinner.DefaultEditor ) {
+			    JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+			    JFormattedTextField field = editor.getTextField();
+			    field.setEditable(true);
+			    field.setBackground(Color.white);
+			}
+		    }
+		    if(one != null && two != null && list != null) {
+			one.setEnabled(true);
+			two.setEnabled(true);
+			list.setBackground(Color.white);
+			list.setEnabled(true);
+		    }
+		}
+		else {
+		    target.setBackground(Color.lightGray);
+		    target.setEnabled(false);
+		    if(target instanceof JSpinner) {
+			// Set enabled
+			JComponent c = ((JSpinner)target).getEditor();
+			if ( c instanceof JSpinner.DefaultEditor ) {
+			    JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+			    JFormattedTextField field = editor.getTextField();
+			    field.setEditable(false);
+			    field.setBackground(Color.lightGray);
+			}
+		    }
+		    if(one != null && two != null && list != null) {
+			one.setEnabled(false);
+			two.setEnabled(false);
+			list.setBackground(Color.lightGray);
+			list.setEnabled(false);
+		    }
+		}
+	    }
+	}
+	/** If a metadata element is selected that requires an hfile, then this listener defaults that hfile. */
+// 	private class HierarchyListener
+// 	    implements ItemListener {
+// 	    /** Any implementation of ItemListener must include this method so that we can be informed when an item from the list is selected, and generate a predetermined hfile for that selection.
+// 	     * @param event An <strong>ItemEvent</strong> containing information about the selection.
+// 	     * @see org.greenstone.gatherer.cdm.ArgumentConfiguration.ArgumentControl
+// 	     */
+// 	    public void itemStateChanged(ItemEvent event) {
+// 		// Determine if the selected element represents a hierarchy.
+// 		Object temp = ((JComboBox)value).getSelectedItem();
+// 		String filename = temp.toString();
+// 		// Search for a argument control called hfile and enable and set value.
+// 		for(int i = 0; i < central_pane.getComponentCount(); i++) {
+// 		    Object object = central_pane.getComponent(i);
+// 		    if(object instanceof ArgumentControl) {
+// 			ArgumentControl control = (ArgumentControl) object;
+// 			if(control.toString().equals("hfile")) {
+// 			    control.setValue(filename + ".txt");
+// 			    control.setEnabled(true);
+// 			}
+// 		    }
+// 		}
+// 	    }
+// 	}
+	/** A ListOption is a compound item which is constructed from several Strings. That magic part is that the length of screen real-estate used by the text version of this item is limited. */
+	private class ListOption
+	    implements Comparable {
+	    /** The maximum length of this String version of this item. */
+	    private int MAX_DESC = 65;
+	    /** The description of the value for this item. */
+	    private String description = null;
+	    /** A cached value for the text value of this option, as it never changes after the first call to toString(). */
+	    private String text = null;
+	    /** The value for this item. */
+	    private String value = null;
+	    /** Constructor.
+	     * @param value The value for this item as a <strong>String</strong>.
+	     * @param description The description of the value as a <strong>String</strong>.
+	     */
+	    public ListOption(String value, String description) {
+		this.description = description;
+		this.value = value;
+	    }
+	    /** Compare two possible ListOption objects for ordering.
+	     * @param object The <strong>Object</strong> to compare to.
+	     * @return An <i>int</i> indicating order as explained in String.
+	     * @see java.lang.String#compareTo
+	     */
+	    public int compareTo(Object object) {
+		return toString().compareTo(object.toString());
+	    }
+	    /** Tests two possible ListOption objects for equality. Uses the result from compareTo().
+	     * @param object The <strong>Object</strong> which may be equal.
+	     * @return <i>true</i> if the objects are equal, <i>false</i> otherwise.
+	     */
+	    public boolean equals(Object object) {
+		return (compareTo(object) == 0);
+	    }
+	    /** Retrieve the description of this list item.
+	     * @return The description as a <strong>String</strong>.
+	     */
+	    public String getDesc() {
+		return description;
+	    }
+	    /** Retrieve the value of this list item.
+	     * @return The value as a <strong>String</strong>.
+	     */
+	    public String getValue() {
+		return value;
+	    }
+       	    /** Convert this object into a nice readable String.
+	     * @return A <strong>String</strong> representing this object.
+	     */
+	    public String toString() {
+		if(text == null) {
+		    if(description.length() >= MAX_DESC) {
+		    	text = value + StaticStrings.MINUS_CHARACTER + description.substring(0, MAX_DESC) + StaticStrings.TRUNCATED_STRING;
+		    }
+		    else {
+		       text = value + StaticStrings.MINUS_CHARACTER + description;
+		    }
+		}
+		return text;
+	    }
+	}
+	/** Listener which removes entries from a list from a combobox when fired. */
+	private class RemoveListener
+	    implements ActionListener {
+	    /** The model behind the target list. */
+	    private DefaultListModel model = null;
+	    /** The list to remove data from. */
+	    private JList target = null;
+	    /** Constructor.
+	     * @param target A <strong>JList</strong>.
+	     */
+	    public RemoveListener(JList target) {
+		this.model = (DefaultListModel) target.getModel();
+		this.target = target;
+	    }
+	    /** When the remove button is clicked, we attempt to remove the selected metadata from the target.
+	     * @param event An <strong>ActionEvent</strong> containing information about the event.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!target.isSelectionEmpty()) {
+		    int index = target.getSelectedIndex();
+		    model.remove(index);
+		}
+	    }
+	}
+	/** Listener that sets the tooltip associated to a combobox to the tooltip relevant to the selected item. */
+	private class ToolTipUpdater
+	    implements ActionListener {
+	    /** Any implementation of an ActionListener must include this method so that we can be informed when the selection in a combobox has changed and update the tooltip accordingly.
+	     * @param event An <strong>ActionEvent</strong> containing information about the action that fired this call.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		JComboBox source = (JComboBox)event.getSource();
+		Object object = source.getSelectedItem();
+		if(object instanceof ListOption) {
+		    ListOption lo = (ListOption)object;
+		    if(lo != null) {
+			String description = Utility.formatHTMLWidth(lo.getDesc(), 60);
+			source.setToolTipText(description);
+		    }
+		    else {
+			source.setToolTipText(StaticStrings.EMPTY_STR);
+		    }
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentContainer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentContainer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ArgumentContainer.java	(revision 21112)
@@ -0,0 +1,58 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      20/05/02
+ * Revised:      03/07/03
+ **************************************************************************************/
+import java.util.ArrayList;
+
+/** This class provides an interface to any class that contains a list of Arguments.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public interface ArgumentContainer {
+    /** Method to retrieve the list of arguments from this container. Note that this method returns both the containers arguments plus its 'supers' arguments if any, and alphabetically orders them.
+     * @param include_normal true to filter so normal arguments are returned
+     * @param include_custom true to filter so custom arguments are returned
+     * @return the arguments within a ArrayList
+     */
+    public ArrayList getArguments(boolean include_normal, boolean include_custom);
+    /** Method to retrieve this containers custom argument string.
+     * @return the custom arguments as a String
+     */
+    public String getCustom();
+    /** Method to retrieve the name associated with this argument container.
+     * @return the name as a String
+     */
+    public String getName();
+    /** Method to set the custom arguments string.
+     * @param custom the new custom argument String
+     */
+    public void setCustom(String custom);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Classifier.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Classifier.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Classifier.java	(revision 21112)
@@ -0,0 +1,412 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      01/05/02
+ * Revised:      16/08/02 Optimized and Commented.
+ *               11/07/03 DOM support
+ **************************************************************************************/
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.w3c.dom.*;
+
+/** This class is responsible for storing information from a parsed classinfo.pl call in such a way that it allows easy access to parsed details for the purposes of user design and specification of classifiers.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Classifier
+    extends ArrayList
+    implements ArgumentContainer, Comparable, DOMProxyListEntry, Serializable {
+
+    static final public String CLASSIFIER_PREFIX = "CL";
+
+    private boolean is_abstract = false;
+
+    /** A reference to the classifier that this one inherits from. */
+    private Classifier super_classifier = null;
+    /** The element this classifier is based upon. */
+    private Element element;
+    /** A description of this classifier. */
+    private String description = null;
+    /** The name of the classifier as it would appear in the collect.cfg file. */
+    private String name = null;
+    /** This string is filled out the first time this classifier is created, and remains unchanged there-after. It is used to match up with Format commands that may not yet have been instantiated (and thus only have offline references along the lines of 'CL1' to figure out what Classifier they want.) */
+    private String old_position_string = null;
+
+    /** Constructor used only in DOMProxyListModel initializations.
+     */
+    public Classifier() {
+    }
+
+    public Classifier(Element element, Classifier base_classifier) {
+	super();
+	this.element = element;
+	this.name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	///atherer.println("Establishing Classifier: " + name);
+	// Parse in any argument options for this classifier, keeping a list of the ones found
+	HashMap known_arguments = new HashMap();
+	NodeList option_elements = element.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
+	int option_elements_length = option_elements.getLength();
+	for(int i = 0; i < option_elements_length; i++) {
+	    Element option_element = (Element) option_elements.item(i);
+	    Argument argument = new Argument(option_element);
+	    ///atherer.println("Rebuilding existing argument: " + argument.getName());
+	    argument.setOwner(name);
+	    add(argument);
+	    known_arguments.put(argument.getName(), argument);
+	}
+	// If a base classifier was given
+	if(base_classifier != null) {
+	    // Copy the details, and add a reference to whatever base_classifiers super classifier is.
+	    description = base_classifier.getDescription();
+	    // Now search through the 'dummy' arguments belonging to the base classifier. For each found, if it is already assigned, fill out further details such as type. If any are found that are not already assigned for this classifier, copy them and add them, but without a value.
+	    ArrayList all_arguments = base_classifier.getArguments(true, true);
+	    int argument_count = all_arguments.size();
+	    for(int j = 0; j < argument_count; j++) {
+		Argument base_argument = (Argument) all_arguments.get(j);
+		String base_argument_name = base_argument.getName();
+		///atherer.println("Library indicates this classifier should have an argument: " + base_argument_name);
+		Argument existing_argument = (Argument) known_arguments.get(base_argument_name);
+		// Found an existing argument. Complete its details
+		if(existing_argument != null) {
+		    ///atherer.println("Found existing argument. Filling out details.");
+		    existing_argument.setCustomArgument(false);
+		    existing_argument.setDefaultValue(base_argument.getDefaultValue());
+		    existing_argument.setDescription(base_argument.getDescription());
+		    existing_argument.setOptions(base_argument.getOptions());
+		    existing_argument.setRequired(base_argument.isRequired());
+		    existing_argument.setType(base_argument.getType());
+		}
+		// No existing argument. Copy base_argument and add it, but do not set its assigned flag. That should be set the first time its changed by the user.
+		else {
+		    ///atherer.println("No such argument. Adding new, unassigned, argument.");
+		    // The trick thing is that we have to create a new element in the DOM as well.
+		    Argument new_argument = base_argument.copy();
+		    new_argument.setOwner(name);
+		    Element argument_element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.OPTION_ELEMENT);
+		    argument_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, base_argument_name);
+		    argument_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
+		    argument_element.setAttribute(StaticStrings.CUSTOM_ATTRIBUTE, StaticStrings.FALSE_STR);
+		    new_argument.setElement(argument_element);
+		    // All done. Add it.
+		    element.appendChild(argument_element);
+		    add(new_argument);
+		}
+	    }
+	}
+	old_position_string = getPositionString();
+    }
+
+    /** Constructor.
+     * @param name The name of this classifier as a <strong>String</strong>.
+     * @param description A description of this classifier as a <strong>String</strong>.
+     * @param super_classifier The super class of this classifier, as a <strong>Classifier</strong>.
+     */
+    public Classifier(String name, String description, Classifier super_classifier) {
+	super();
+	this.description = description;
+	this.name = name;
+	this.super_classifier = super_classifier;
+    }
+
+    /** Method to add an argument to this classifier. Only adds the argument if it isn't already present.
+     * @param argument The <strong>Argument</strong> to add.
+     */
+    public void addArgument(Argument argument) {
+	if(element == null && !contains(argument)) {
+	    add(argument);
+	    argument.setOwner(name);
+	}
+    }
+
+    /** Method to compare two classifiers for ordering.
+     * @param object The classifier we are comparing to, as an <strong>Object</strong>.
+     * @return An <i>int</i> specifying the classifier order, using values as set out in String.
+     * @see java.lang.String#compareTo
+     */
+    public int compareTo(Object object) {
+	if(object == null) {
+	    return -1;
+	}
+	return toString().compareTo(object.toString());
+    }
+
+    /** The assigned classifier constructor.
+     * @param element the DOM Element this classifier is based upon
+     */
+    public DOMProxyListEntry create(Element element) {
+	String classifier_name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	// Determine the base classifier from the classifier name
+	Classifier base_classifier = CollectionDesignManager.classifier_manager.getBaseClassifier(classifier_name);
+	Classifier classifier = new Classifier(element, base_classifier);
+	base_classifier = null;
+	classifier_name = null;
+	return classifier;
+    }
+
+    /** Method to determine if two classifiers are equal.
+     * @param object The classifier to test against, as an <strong>Object</strong>.
+     * @return <i>true</i> if the classifier names match, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Method to retrieve an argument by its name.
+     * @param name The name of the argument as a <strong>String</strong>.
+     * @return The <strong>Argument</strong> requested, or <i>null</i> if no such argument.
+     */
+    public Argument getArgument(String name) {
+	// The name given may still include the '-'
+	if(name.startsWith("-")) {
+	    name = name.substring(1);
+	}
+	ArrayList arguments = getArguments(true, true);
+	for(int i = 0; i < arguments.size(); i++) {
+	    Argument argument = (Argument)arguments.get(i);
+	    if(argument.getName().equals(name)) {
+		return argument;
+	    }
+	}
+	return null;
+    }
+
+     /** Retrieve all of the arguments available to this base classifier, including its super classifiers arguments. Some complexity is added by allowing the caller to choose whether they want normal arguments, custom arguments, or both.
+      * @return an ArrayList of all of the arguments, starting with those for this classifier and ending with the arguments for basplug or similiar root classifier
+      */
+    public ArrayList getArguments(boolean include_normal, boolean include_custom) {
+	ArrayList arguments = new ArrayList();
+	if(include_normal && include_custom) {
+	    arguments.addAll(this);
+	}
+	else {
+	    int size = size();
+	    for(int i = 0; i < size; i++) {
+		Argument argument = (Argument) get(i);
+		if(argument.isCustomArgument()) {
+		    if(include_custom && !arguments.contains(argument)) {
+			arguments.add(argument);
+		    }
+		}
+		else {
+		    if(include_normal && !arguments.contains(argument)) {
+			arguments.add(argument);
+		    }
+		}
+		argument = null;
+	    }
+	}
+	if(super_classifier != null) {
+	    ArrayList remainder = super_classifier.getArguments(include_normal, include_custom);
+	    remainder.removeAll(arguments);
+	    arguments.addAll(remainder);
+	}
+	return arguments;
+    }
+
+    /** Method to retrieve a classifiers custom argument information. Custom arguments are defined to be those that have not got matching arguments in the base reference classifier from the library. Of course if there is no base classifier then all arguments are considered to be custom.
+     * @return the custom arguments as a String
+     */
+    public String getCustom() {
+	StringBuffer custom_text = new StringBuffer();
+	// Retrieve all of the arguments, and append any that are custom into one long string
+	ArrayList arguments = getArguments(false, true);
+	int arguments_size = arguments.size();
+	boolean first = true;
+	for(int i = 0; i < arguments_size; i++) {
+	    Argument argument = (Argument) arguments.get(i);
+	    if(argument.isAssigned()) {
+		if(!first) {
+		    custom_text.append(" ");
+		}
+		custom_text.append(argument.toString());
+		first = false;
+	    }
+	}
+	return custom_text.toString();
+    }
+
+    public String getDescription() {
+	return description;
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    /** Method to retrieve a classifiers name.
+     * @return A <strong>String</strong> containing the classifiers name.
+     */
+    public String getName() {
+	if(name == null && element != null) {
+	    name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	}
+	return name;
+    }
+
+    /* private String getOldPositionString() {
+	return old_position_string;
+	} */
+
+    /** Generate the string showing this classifiers position. */
+    public String getPositionString() {
+	String position_string = CLASSIFIER_PREFIX;
+	if(element != null) {
+	    // Determine our place in the collect.cfg file
+	    int position_int = CollectionDesignManager.classifier_manager.indexOf(this) + 1;
+	    if(position_int != -1) {
+		position_string = position_string + position_int;
+	    }
+	}
+	return position_string;
+    }
+
+    public boolean isAbstract() {
+	return is_abstract;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    /** Set the custom arguments. This turns out to be quite tricky. We must parse in the string, searching for arguments (for that we use a handy method in CollectionConfiguration). Next, for each argument, we check if we already know about it. If so we update its value, otherwise we create a new argument and assign it (must assign!).
+     * @param custom_str the custom arguments all splodged together in one String
+     */
+    public void setCustom(String custom_str) {
+	HashMap raw_arguments = CollectionConfiguration.parseArguments(new CommandTokenizer(custom_str));
+	ArrayList custom_arguments = getArguments(false, true);
+	int size = custom_arguments.size();
+	for(int i = 0; i < size; i++) {
+	    Argument argument = (Argument) custom_arguments.get(i);
+	    String original_argument_name = StaticStrings.MINUS_CHARACTER + argument.getName();
+	    if(raw_arguments.containsKey(original_argument_name)) {
+		// Set as assigned
+		argument.setAssigned(true);
+		String argument_value = (String)raw_arguments.remove(original_argument_name);
+		if(argument_value != null) {
+		    argument.setValue(argument_value);
+		    argument_value = null;
+		}
+	    }
+	    // We've removed it from our custom statement, so unassign
+	    else {
+		argument.setAssigned(false);
+	    }
+	    argument = null;
+	}
+	// Any left over, add to the classifier
+	Iterator argument_names = raw_arguments.keySet().iterator();
+	while(argument_names.hasNext()) {
+	    String argument_name = (String) argument_names.next();
+	    String argument_value = (String) raw_arguments.get(argument_name);
+	    // The tricky thing is that we have to create a new element in the DOM as well.
+	    Element argument_element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.OPTION_ELEMENT);
+	    argument_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, argument_name.substring(1));
+	    argument_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
+	    argument_element.setAttribute(StaticStrings.CUSTOM_ATTRIBUTE, StaticStrings.TRUE_STR);
+	    Argument argument = new Argument(argument_element);
+	    argument_name = null;
+	    if(argument_value != null) {
+		argument.setValue(argument_value);
+		argument_value = null;
+	    }
+	    // All done. Add it.
+	    element.appendChild(argument_element);
+	    add(argument);
+	    argument_element = null;
+	}
+	raw_arguments = null;
+    }
+
+    /** Method to set the value of desc.
+     * @param description The new value of desc as a <strong>String</strong>.
+     */
+    public void setDescription(String description) {
+	this.description = description;
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+    }
+
+    public void setIsAbstract(boolean is_abstract) {
+	this.is_abstract = is_abstract;
+    }
+
+    /** Method to set the value of name.
+     * @param name The new value of name as a <strong>String</strong>.
+     */
+    public void setName(String name) {
+	this.name = name;
+    }
+
+    /** Method to set the value of the super_classifier.
+     * @param super_classifier The new value of super_classifier as a <strong>Classifier</strong>, or <i>null</i> if this class has no inheritance.
+     */
+    public void setSuper(Classifier super_classifier) {
+	this.super_classifier = super_classifier;
+    }
+
+
+    /** Method to print out this classifier as it would appear to the user in the interface
+     * @return A <strong>String</strong> containing a single classifier command.
+     */
+    public String toString()
+    {
+	if (element == null) {
+	    return name;
+	}
+
+	if (name == null) {
+	    name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	}
+
+	StringBuffer text = new StringBuffer(StaticStrings.CLASSIFY_STR);
+	text.append(" ");
+	text.append(name);
+
+	ArrayList arguments = getArguments(true, true);
+	for (int i = 0; i < arguments.size(); i++) {
+	    Argument argument = (Argument) arguments.get(i);
+	    if (argument.isAssigned()) {
+		text.append(" ");
+		text.append(argument.toString());
+	    }
+	}
+
+	return text.toString();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ClassifierManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ClassifierManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ClassifierManager.java	(revision 21112)
@@ -0,0 +1,904 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.jar.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.apache.xerces.parsers.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+import org.xml.sax.*;
+
+/** This class is responsible for keeping track of all the classifiers assigned to this collection, and providing methods for adding and removing them.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class ClassifierManager
+    extends DOMProxyListModel {
+
+    /** A list of known, but currently unassigned, classifiers. */
+    private ArrayList library = null;
+    /** The controls for editing the contents of this manager. */
+    private Control controls = null;
+
+    private DOMProxyListModel model;
+
+    /** Constructor.
+     * @see org.greenstone.gatherer.cdm.DynamicListModel
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    public ClassifierManager() {
+	super(CollectionDesignManager.collect_config.getDocumentElement(), CollectionConfiguration.CLASSIFY_ELEMENT, new Classifier());
+	this.model = this;
+	DebugStream.println("ClassifierManager: " + getSize() + " classifiers parsed.");
+	// Reload/Create the library
+	loadClassifiers();
+	saveClassifiers();
+    }
+
+    /** Method to add a new classifier to library.
+     * @param classifier The new <strong>Classifier</strong>.
+     * @see org.greenstone.gatherer.cdm.DynamicListModel
+     */
+    private void addClassifier(Classifier classifier) {
+	if(!library.contains(classifier)) {
+	    library.add(classifier);
+	}
+    }
+
+    /** Method to assign a classifier.
+     * @param classifier The base <strong>Classifier</strong> to assign.
+     * @see org.greenstone.gatherer.cdm.DynamicListModel
+     */
+    private void assignClassifier(Classifier classifier) {
+	if(!contains(classifier)) {
+	    Element element = classifier.getElement();
+	    // Locate where we should insert this new classifier.
+	    Node target_node = CollectionConfiguration.findInsertionPoint(element);
+	    add(root, classifier, target_node);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    public static boolean clearClassifierCache() {
+
+	DebugStream.println("deleting classifiers.dat");
+	File class_file = new File(Gatherer.getGLIUserDirectoryPath() + "classifiers.dat");
+	if (class_file.exists()) {
+	    return Utility.delete(class_file);
+	}
+	return true;
+    }
+    /** Destructor.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
+     * @see org.greenstone.gatherer.cdm.DynamicListModel
+     */
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+	library.clear();
+	library = null;
+    }
+
+    public Classifier getBaseClassifier(String name) {
+	int library_size = library.size();
+	for(int i = 0; i < library_size; i++) {
+	    Classifier classifier = (Classifier) library.get(i);
+	    if(classifier.getName().equals(name)) {
+		return classifier;
+	    }
+	}
+	// No success.
+	return null;
+    }
+
+    /** Method to retrieve the classifier with the given index.
+     * @param index The index of the desired classifier as an <i>int</i>.
+     * @return The requested Classifier or <i>null</i> if no such classifier exists.
+     */
+    public Classifier getClassifier(int index) {
+	if(0 <= index && index < getSize()) {
+	    return (Classifier) getElementAt(index);
+	}
+	return null;
+    }
+
+    /** Method to retrieve the control for this manager.
+     * @return the Control for editing classifiers
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    // Build controls
+	    this.controls = new ClassifierControl();
+	}
+	return controls;
+    }
+
+    public ArrayList getHierarchyClassifiers() {
+	ArrayList result = new ArrayList();
+	for(int i = 0; i < getSize(); i++) {
+	    Classifier classifier = (Classifier) getElementAt(i);
+	    if(classifier.getName().equalsIgnoreCase(StaticStrings.HIERARCHY_CLASSIFIER)) {
+		result.add(classifier);
+	    }
+	    classifier = null;
+	}
+	return result;
+
+    }
+
+    /** Determine if the Phind classifier has been assigned.
+     * @return true if it has, false otherwise
+     */
+    public boolean isPhindClassifierAssigned() {
+	for(int i = 0; i < getSize(); i++) {
+	    Classifier classifier = (Classifier) getElementAt(i);
+	    if(classifier.getName().equalsIgnoreCase(StaticStrings.PHIND_CLASSIFIER)) {
+		return true;
+	    }
+	    classifier = null;
+	}
+	return false;
+    }
+
+    /** Method to move a classifier in the list order.
+     * @param classifier the Classifier you want to move.
+     * @param direction true to move the classifier up, false to move it down.
+     * @param all true to move to move all the way, false for a single step.
+     */
+    private void moveClassifier(Classifier classifier, boolean direction, boolean all) {
+	if(getSize() < 2) {
+	    DebugStream.println("Not enough classifiers to allow moving.");
+	    return;
+	}
+	if(all) {
+	    // Move to top
+	    if(direction) {
+		// Remove the moving classifier
+		remove(classifier);
+		// Retrieve the first classifier
+		Classifier first_classifier = (Classifier) getElementAt(0);
+		// Add the moving classifier before the first classifier
+		addBefore(classifier, first_classifier);
+		first_classifier = null;
+	    }
+	    else {
+		// Remove the moving classifier
+		remove(classifier);
+		// And add after last classifier
+		add(getSize(), classifier);
+	    }
+	}
+	else {
+	    // Try to move the classifier one step in the desired direction.
+	    int index = indexOf(classifier);
+	    ///ystem.err.println("Index of " + classifier + " = " + index);
+	    if(direction) {
+		index--;
+		if(index < 0) {
+		    String args[] = new String[2];
+		    args[0] = Dictionary.get("CDM.ClassifierManager.Classifier");
+		    args[1] = classifier.getName();
+		    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.Move.At_Top", args), Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+		    return;
+		}
+		remove(classifier);
+		add(index, classifier);
+	    }
+	    else {
+		index++;
+		if(index >= getSize()) {
+		    String args[] = new String[2];
+		    args[0] = Dictionary.get("CDM.ClassifierManager.Classifier_Str");
+		    args[1] = classifier.getName();
+		    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.Move.At_Bottom", args), Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+		    return;
+		}
+		remove(classifier);
+		add(index, classifier);
+	    }
+	}
+	
+	Gatherer.c_man.configurationChanged();
+	// tell the format manager to update the names of its format statements
+	Gatherer.c_man.getCollection().cdm.format_manager.refresh();
+    }
+    
+    /** This method removes an assigned classifier. I was tempted to call it unassign, but remove is more consistant. Note that there is no way to remove a classifier from the library.
+     * @param classifier The Classifier to remove
+     * @see org.greenstone.gatherer.cdm.DynamicListModel
+     */
+    private void removeClassifier(Classifier classifier) {
+	remove(classifier);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    /** Method to cache the current contents of library (known classifiers) to file.
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void saveClassifiers() {
+	try {
+	    File classifiers_dat_file = new File(Gatherer.getGLIUserDirectoryPath() + "classifiers.dat");
+	    FileOutputStream file = new FileOutputStream(classifiers_dat_file);
+	    ObjectOutputStream out = new ObjectOutputStream(file);
+	    out.writeObject(library);
+	    out.close();
+	}
+	catch (Exception error) {
+	}
+    }
+
+    private Object[] getAvailable() {
+	ArrayList available = new ArrayList();
+	int library_size = library.size();
+	for(int i = 0; i < library_size; i++) {
+	    Classifier classifier = (Classifier) library.get(i);
+	    if(!classifier.isAbstract()) {
+		available.add(classifier);
+	    }
+	    classifier = null;
+	}
+	return available.toArray();
+    }
+
+    /** Method to extract just the classifiers name from a file object.
+     * @param classifier The <strong>File</strong> which references a certain classifier.
+     * @return A <strong>String</strong> containing just the classifiers name, without extension.
+     */
+    private String getClassifierName(String filename) {
+	String name = filename;
+	if(name.indexOf(".") != -1) {
+	    name = name.substring(0, name.indexOf("."));
+	}
+	return name;
+    }
+
+    /** Method to load the details of a single plug-in.
+     * @param classifier The classifier <strong>File</strong> you wish to load.
+     */
+    private void loadClassifier(String classifier, String lang) {
+	///ystem.err.println("Attempting to parse " + classifier);
+	Document document = null;
+	InputStream input_stream = null;
+
+	long start;
+	long end;
+	// Run classinfo on this classifier, and then send the results for parsing.
+	try {
+	    StringBuffer xml = null;
+	    if (Gatherer.isGsdlRemote) {
+		String classinfo_output = RemoteGreenstoneServer.getScriptOptions("classinfo.pl", "&classifier=" + classifier);
+		xml = new StringBuffer(classinfo_output);
+	    }
+	    else {
+		String args[] = null;
+		if(Utility.isWindows()) {
+		    args = new String[6];
+		    if(Configuration.perl_path != null) {
+			args[0] = Configuration.perl_path;
+		    }
+		    else {
+			args[0] = "Perl.exe";
+		    }
+		    args[1] = LocalGreenstone.getBinScriptDirectoryPath() + "classinfo.pl";
+		    args[2] = "-xml";
+		    args[3] = "-language";
+		    args[4] = lang;
+		    args[5] = getClassifierName(classifier);
+		}
+		else {
+		    args = new String[5];
+		    args[0] = "classinfo.pl";
+		    args[1] = "-xml";
+		    args[2] = "-language";
+		    args[3] = lang;
+		    args[4] = getClassifierName(classifier);
+		}
+
+		// Create the process.
+		Runtime runtime = Runtime.getRuntime();
+		Process process = runtime.exec(args);
+
+		input_stream = process.getErrorStream();
+		xml = Utility.readXMLStream(input_stream);
+	    }
+
+	    document = CollectionDesignManager.XMLStringToDOM(xml,classifier);
+	}
+	catch (Exception error) {
+	    error.printStackTrace();
+	    ///ystem.err.println("Error: Cannot parse " + getClassifierName(classifier));
+	}
+	if(document != null) {
+	    parseXML(document.getDocumentElement());
+	}
+    }
+
+    /** Method to initially load information from the standard classifiers within the gsdl Perl library.
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void loadClassifiers() {
+	// Attempt to restore the cached file.
+	File classifiers_dat_file = new File(Gatherer.getGLIUserDirectoryPath() + "classifiers.dat");
+	try {
+	    FileInputStream file = new FileInputStream(classifiers_dat_file);
+	    ObjectInputStream input = new ObjectInputStream(file);
+	    library = (ArrayList) input.readObject();
+	}
+	catch (Exception error) {
+	    DebugStream.println("Unable to open " + classifiers_dat_file);
+	}
+
+	if(library == null) {
+	    library = new ArrayList();
+
+	    if (Gatherer.isGsdlRemote) {
+		String classinfo_output = RemoteGreenstoneServer.getScriptOptions("classinfo.pl", "&listall=");
+		loadClassifiers(new StringBuffer(classinfo_output));
+	    }
+
+	    else {
+		// Retrieve the gsdl home directory...
+		String directory = LocalGreenstone.getDirectoryPath();
+		directory = directory + "perllib" + File.separator + "classify" + File.separator;
+
+		String current_lang = Configuration.getLanguage();
+		File files[] = (new File(directory)).listFiles();
+		if(files != null) {
+		    // Create a progress indicator.
+		    ParsingProgress progress = new ParsingProgress(Dictionary.get("CDM.ClassifierManager.Parsing.Title"), Dictionary.get("CDM.ClassifierManager.Parsing.Message"), files.length);
+		    for(int i = 0; i < files.length; i++) {
+			// We only want to check Perl Modules.
+			if(files[i].getName().endsWith(".pm")) {
+			    loadClassifier(files[i].getName(), current_lang);
+			}
+			progress.inc();
+		    }
+		    progress.dispose();
+		    progress.destroy();
+		    progress = null;
+		}
+	    }
+	}
+    }
+
+
+
+    /** Method to load classifier information from a specified input stream (could be local or through URL). Of course no classifiers may be found at this location.
+     * @param input_stream An <strong>InputStream</strong> indicating the where list of classifiers -- encoded in XML -- can be read from
+     */
+    private void loadClassifiers(StringBuffer xml)
+    {
+	Document document = CollectionDesignManager.XMLStringToDOM(xml, "-listall");
+
+	// Parse XML to build up list of classifier names
+	Node root = document.getDocumentElement();
+
+	NamedNodeMap attributes = root.getAttributes();
+	Node length_node = attributes.getNamedItem("length");
+	String num_classifiers_str = length_node.getNodeValue();
+	int num_classifiers = Integer.parseInt(num_classifiers_str);
+	String class_list[] = new String[num_classifiers];
+
+	Node node = root.getFirstChild();
+	int i = 0;
+	while (node != null) {
+	    String node_name = node.getNodeName();
+	    if (node_name.equalsIgnoreCase("ClassifyName")) {
+		String name = XMLTools.getValue(node);
+		class_list[i] = name;
+		i++;		
+	    }
+
+	    node = node.getNextSibling();
+	}
+
+	String current_lang = Configuration.getLanguage();
+	if (num_classifiers>0) {
+	    // Create a progress indicator.
+	    ParsingProgress progress = new ParsingProgress(Dictionary.get("CDM.ClassifierManager.Parsing.Title"), Dictionary.get("CDM.ClassifierManager.Parsing.Message"), num_classifiers);
+
+	    for (i=0; i<num_classifiers; i++) {
+		String classifier = class_list[i];
+		loadClassifier(classifier, current_lang);
+		progress.inc();
+	    }
+	    progress.dispose();
+	    progress.destroy();
+	    progress = null;
+	}
+    }
+
+
+    /** Parses a DOM tree model turning it into a Classifier and its associated arguments.
+     * @param root The <strong>Node</strong> at the root of the DOM model.
+     * @return A newly created <strong>Classifier</strong> based on the information parsed from the DOM model.
+     * @see org.greenstone.gatherer.cdm.Argument
+     */
+    private Classifier parseXML(Node root) {
+	Classifier classifier = new Classifier();
+	String node_name = null;
+	for(Node node = root.getFirstChild(); node != null;
+	    node = node.getNextSibling()) {
+	    node_name = node.getNodeName();
+	    if(node_name.equals("Name")) {
+		String name = XMLTools.getValue(node);
+		// We can save ourselves some processing time if a classifier with this name already exists in our manager. If so retrieve it and return it.
+		Classifier existing = getBaseClassifier(name);
+		if(existing != null) {
+		    return existing;
+		}
+		classifier.setName(name);
+	    }
+	    else if(node_name.equals("Desc")) {
+		classifier.setDescription(XMLTools.getValue(node));
+	    }
+	    else if(node_name.equals("Abstract")) {
+		classifier.setIsAbstract(XMLTools.getValue(node).equalsIgnoreCase(CollectionConfiguration.YES_STR));
+	    }
+	    // Parse the multitude of arguments.
+	    else if(node_name.equals("Arguments")) {
+		for(Node arg = node.getFirstChild(); arg != null; arg = arg.getNextSibling()) {
+		    node_name = arg.getNodeName();
+		    // An option.
+		    if(node_name.equals("Option")) {
+			Argument argument = new Argument();
+			argument.parseXML((Element)arg);
+			classifier.addArgument(argument);
+		    }
+		    // A super classifier class.
+		    else if(node_name.equals("ClassInfo")) {
+			Classifier super_classifier = parseXML(arg);
+			classifier.setSuper(super_classifier);
+		    }
+		}
+	    }
+	}
+	if(classifier.getName() != null) {
+	    addClassifier(classifier);
+	    return classifier;
+	}
+	return null;
+    }
+
+    /** A class which provides controls for assigned and editing classifiers. */
+    private class ClassifierControl
+	extends JPanel
+	implements Control {
+	/** A combobox containing all of the known classifiers, including those that may have already been assigned. */
+	private GComboBox classifier = null;
+	/** Button for adding classifiers. */
+	private JButton add = null;
+	/** Button for configuring the selected classifier. */
+	private JButton configure = null;
+
+	//private JButton move_bottom_button;
+
+	private JButton move_down_button;
+
+	//private JButton move_top_button;
+
+	private JButton move_up_button;
+
+	/** Button to remove the selected classifier. */
+	private JButton remove = null;
+
+	/** A list of assigned classifiers. */
+	private JList classifier_list = null;
+	/** The text area containing instructions on the use of this control. */
+	private JTextArea instructions = null;
+
+	/** Constructor.
+	 * @see org.greenstone.gatherer.cdm.ClassifierManager.ClassifierControl.AddListener
+	 * @see org.greenstone.gatherer.cdm.ClassifierManager.ClassifierControl.ConfigureListener
+	 * @see org.greenstone.gatherer.cdm.ClassifierManager.ClassifierControl.RemoveListener
+	 */
+	public ClassifierControl() {
+	    Collections.sort(library);
+	    // Create
+	    add = new GLIButton();
+	    add.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add, "CDM.ClassifierManager.Add", "CDM.ClassifierManager.Add_Tooltip");
+	    JPanel button_pane = new JPanel();
+	    JPanel central_pane = new JPanel();
+	    configure = new GLIButton();
+	    configure.setEnabled(false);
+	    configure.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerBoth(configure, "CDM.ClassifierManager.Configure", "CDM.ClassifierManager.Configure_Tooltip");
+	    JPanel header_pane = new JPanel();
+	    instructions = new JTextArea();
+	    instructions.setEditable(false);
+	    instructions.setLineWrap(true);
+	    instructions.setRows(6);
+	    instructions.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions, "CDM.ClassifierManager.Instructions");
+
+	    ClassifierComboboxListener ccl = new ClassifierComboboxListener();
+	    classifier = new GComboBox(getAvailable());
+	    classifier.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    classifier.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    classifier.setEditable(true);
+	    classifier.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    classifier.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    if(classifier.getItemCount() > 0) {
+		classifier.setSelectedIndex(0);
+		ccl.itemStateChanged(new ItemEvent(classifier, 0, null, ItemEvent.SELECTED));
+	    }
+
+	    JLabel classifier_label = new JLabel();
+	    Dictionary.registerText(classifier_label, "CDM.ClassifierManager.Classifier");
+	    classifier_list = new JList(model);
+	    classifier_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    JLabel classifier_list_label = new JLabel();
+	    classifier_list_label.setHorizontalAlignment(JLabel.CENTER);
+	    classifier_list_label.setOpaque(true);
+	    Dictionary.registerText(classifier_list_label, "CDM.ClassifierManager.Assigned");
+	    JPanel classifier_list_pane = new JPanel();
+	    JPanel classifier_pane = new JPanel();
+	    remove = new GLIButton();
+	    remove.setEnabled(false);
+	    remove.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove, "CDM.ClassifierManager.Remove", "CDM.ClassifierManager.Remove_Tooltip");
+
+	    JLabel title = new JLabel();
+	    title.setHorizontalAlignment(JLabel.CENTER);
+	    title.setOpaque(true);
+	    Dictionary.registerText(title, "CDM.ClassifierManager.Title");
+
+	    JPanel temp = new JPanel(new BorderLayout());
+
+	    JPanel move_button_pane = new JPanel();
+
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    // Listeners
+	    add.addActionListener(new AddListener());
+	    add.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    classifier.addItemListener(ccl);
+	    configure.addActionListener(new ConfigureListener());
+	    configure.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove.addActionListener(new RemoveListener());
+	    remove.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    classifier_list.addMouseListener(new ClickListener());
+	    classifier_list.addListSelectionListener(new ListListener());
+	    ccl = null;
+
+	    MoveListener ml = new MoveListener();
+	    //move_bottom_button.addActionListener(ml);
+	    move_down_button.addActionListener(ml);
+	    move_down_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    //move_top_button.addActionListener(ml);
+	    move_up_button.addActionListener(ml);
+	    move_up_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+
+	    // Layout
+	    title.setBorder(BorderFactory.createEmptyBorder(0,0,2,0));
+
+	    instructions.setBorder(BorderFactory.createEmptyBorder(2,5,2,5));
+
+	    header_pane.setLayout(new BorderLayout());
+	    header_pane.add(title, BorderLayout.NORTH);
+	    header_pane.add(new JScrollPane(instructions), BorderLayout.CENTER);
+
+	    move_button_pane.setLayout(new GridLayout(4,1));
+	    move_button_pane.add(move_up_button);
+	    move_button_pane.add(new JPanel());
+	    move_button_pane.add(new JPanel());
+	    move_button_pane.add(move_down_button);
+
+	    classifier_list_label.setBorder(BorderFactory.createEmptyBorder(0,2,0,2));
+
+	    classifier_list_pane.setLayout(new BorderLayout());
+	    classifier_list_pane.add(classifier_list_label, BorderLayout.NORTH);
+	    classifier_list_pane.add(new JScrollPane(classifier_list), BorderLayout.CENTER);
+	    classifier_list_pane.add(move_button_pane, BorderLayout.EAST);
+
+	    classifier_label.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+
+	    classifier_pane.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	    classifier_pane.setLayout(new BorderLayout(5,0));
+	    classifier_pane.add(classifier_label, BorderLayout.WEST);
+	    classifier_pane.add(classifier, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1, 3));
+	    button_pane.add(add);
+	    button_pane.add(configure);
+	    button_pane.add(remove);
+
+	    // Scope these mad bordering skillz.
+	    temp.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,0,5,0), BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("CDM.ClassifierManager.Controls")), BorderFactory.createEmptyBorder(2,2,2,2))));
+	    temp.add(classifier_pane, BorderLayout.NORTH);
+	    temp.add(button_pane, BorderLayout.SOUTH);
+
+	    central_pane.setLayout(new BorderLayout());
+	    central_pane.add(classifier_list_pane, BorderLayout.CENTER);
+	    central_pane.add(temp, BorderLayout.SOUTH);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_pane, BorderLayout.NORTH);
+	    add(central_pane, BorderLayout.CENTER);
+	}
+
+	/** Method which acts like a destructor, tidying up references to persistant objects.
+	 */
+	public void destroy() {
+	    add = null;
+	    classifier = null;
+	    classifier_list = null;
+	    configure = null;
+	    instructions = null;
+	    remove = null;
+	}
+
+	/** This method is overridden to ensure the instructions are scrolled to top, before the super classes updateUI() is called.
+	 */
+	public void gainFocus() {
+	    if(instructions != null) {
+		instructions.setCaretPosition(0);
+	    }
+	}
+
+	public void loseFocus() {
+	}
+
+	/** This class listens for actions upon the add button in the controls, and if detected calls the assignClassifier() method.
+	 */
+	private class AddListener
+	    implements ActionListener {
+	    /** Any implementation of ActionListener must include this method so that we can be informed when an action has occured on one of our target controls, so that we can add the selected Classifier.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     * @see org.greenstone.gatherer.Gatherer
+	     * @see org.greenstone.gatherer.cdm.ArgumentConfiguration
+	     * @see org.greenstone.gatherer.cdm.Classifier
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Object selected_object = classifier.getSelectedItem();
+		// If there is something in the combobox, but we haven't registered a selection, then add the object and select it!
+		if(selected_object != null) {
+		    // Retrieve the base classifier
+		    Classifier base_classifier = getBaseClassifier(selected_object.toString());
+
+		    // Create a new element in the DOM
+		    Element element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.CLASSIFY_ELEMENT);
+		    Classifier new_classifier = null;
+		    if(base_classifier != null) {
+			DebugStream.println("Creating Classifier based on existing Classifer.");
+			element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, base_classifier.getName());
+			new_classifier = new Classifier(element, base_classifier);
+		    }
+		    else {
+			DebugStream.println("Creating new custom Classifier.");
+			element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, selected_object.toString());
+			new_classifier = new Classifier(element, null);
+			// Also we add the custom classifier name to the combobox for convienence.
+			classifier.addItem(selected_object);
+		    }
+		    element = null;
+		    // Automatically chain to configuration. This ensures required arguments are filled out.
+		    ArgumentConfiguration ac = new ArgumentConfiguration(new_classifier);
+		    if(ac.display()) {
+			if(!model.contains(new_classifier)) {
+			    assignClassifier(new_classifier);
+			    classifier_list.setSelectedValue(new_classifier, true);
+			}
+			else {
+			    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.ClassifierManager.Classifier_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+			}
+		    }
+		    ac = null;
+		    new_classifier = null;
+		}
+	    }
+	}
+
+    /** This listener reacts to changes in the current selection of the classifier combobox. */
+    private class ClassifierComboboxListener
+        implements ItemListener {
+        /** When a user selects a certain plugin, update the tooltip to show the plugin description. */
+        public void itemStateChanged(ItemEvent event) {
+            if(event.getStateChange() == ItemEvent.SELECTED) {
+                // Retrieve the selected plugin
+                Object current_selection = classifier.getSelectedItem();
+                // And reset the tooltip. If the plugin is null or is a string, then go back to the default message
+                if(current_selection == null || current_selection instanceof String) {
+                    Dictionary.registerTooltip(classifier, "CDM.ClassifierManager.Classifier_Tooltip");
+                }
+                else {
+                    Classifier current_classifier = (Classifier) current_selection;
+                    Dictionary.registerTooltipText(classifier, Utility.formatHTMLWidth(current_classifier.getDescription(), 40));
+                    current_classifier = null;
+                }
+                current_selection = null;
+            }
+        }
+    }
+
+	/** Listens for double clicks apon the list and react as if the configure button was pushed. */
+	private class ClickListener
+	    extends MouseAdapter {
+	    /** Called whenever the mouse is clicked over a registered component, we use this to chain through to the configure prompt.
+	     * @param event A <strong>MouseEvent</strong> containing information about the mouse click.
+	     */
+	    public void mouseClicked(MouseEvent event) {
+		if(event.getClickCount() == 2 ) {
+		    if(!classifier_list.isSelectionEmpty()) {
+			Classifier classifier = (Classifier) classifier_list.getSelectedValue();
+			ArgumentConfiguration ac = new ArgumentConfiguration(classifier);
+			if(ac.display()) {
+			    refresh(classifier);
+			}
+			ac.destroy();
+			ac = null;
+			// cos I can't be bothered checking every argument to see if it has changed or not, we'll asasume that the configuration has changed if someone has clicked configure
+			Gatherer.c_man.configurationChanged();
+		    }
+		}
+	    }
+	}
+
+	/** This class listens for actions upon the configure button in the controls, and if detected creates a new ArgumentConfiguration dialog box to allow for configuration.
+	 */
+	private class ConfigureListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     * @see org.greenstone.gatherer.cdm.ArgumentConfiguration
+	     * @see org.greenstone.gatherer.cdm.Classifier
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!classifier_list.isSelectionEmpty()) {
+		   Classifier classifier = (Classifier) classifier_list.getSelectedValue();
+		   ArgumentConfiguration ac = new ArgumentConfiguration(classifier);
+		   if(ac.display()) {
+		       refresh(classifier);
+		   }
+		   ac.destroy();
+		   ac = null;
+		   // cos I can't be bothered checking every argument to see if it has changed or not, we'll asasume that the configuration has changed if someone has clicked configure
+		   Gatherer.c_man.configurationChanged();
+		}
+	    }
+	}
+
+	/** listens for changes in the list selection and enables the configure and remove buttons if there is a selection, disables them if there is no selection */
+	private class ListListener
+	    implements ListSelectionListener {
+
+	    public void valueChanged(ListSelectionEvent e) {
+		if (!e.getValueIsAdjusting()) { // we get two events for one change in list selection - use the false one ( the second one)
+		    if (classifier_list.isSelectionEmpty()) {
+			move_up_button.setEnabled(false);
+			move_down_button.setEnabled(false);
+			configure.setEnabled(false);
+			remove.setEnabled(false);
+		    }
+		    else {
+			configure.setEnabled(true);
+			remove.setEnabled(true);
+			int selected_index = classifier_list.getSelectedIndex();
+			move_up_button.setEnabled(selected_index !=0);
+			move_down_button.setEnabled(selected_index != model.getSize()-1);
+		    }
+		}
+	    }
+	}
+
+	/** Listens for actions apon the move buttons in the manager controls, and if detected calls the <i>moveClassifier()</i> method of the manager with the appropriate details. */
+	private class MoveListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!classifier_list.isSelectionEmpty()) {
+		    Object object = classifier_list.getSelectedValue();
+		    if(object instanceof Classifier) {
+			Classifier classifier = (Classifier) object;
+			//if(event.getSource() == move_top_button) {
+			//    moveClassifier(classifier, true, true);
+			//}
+			//else 
+			if(event.getSource() == move_up_button) {
+			    moveClassifier(classifier, true, false);
+			}
+			else if(event.getSource() == move_down_button) {
+			    moveClassifier(classifier, false, false);
+			}
+			//else {
+			//    moveClassifier(classifier, false, true);
+			//}
+			classifier_list.setSelectedValue(classifier, true);
+		    }
+		}
+	    }
+	}
+
+	/** This class listens for actions upon the remove button in the controls, and if detected calls the <i>removeClassifier()</i> method.
+	 */
+	private class RemoveListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(classifier_list.isSelectionEmpty()) {
+		    remove.setEnabled(false);
+		    return;
+		}
+		int selected_index = classifier_list.getSelectedIndex();
+		Object selected_classifier = classifier_list.getSelectedValue();
+		if (!(selected_classifier instanceof Classifier)) {
+		    return; // what else could we have here???
+		}
+		removeClassifier((Classifier)selected_classifier);
+
+		if (selected_index >= classifier_list.getModel().getSize()) {
+		    selected_index--;
+		}
+		if (selected_index >=0) {
+		    classifier_list.setSelectedIndex(selected_index);
+		} else {
+		    // no more classifiers in the list
+		    remove.setEnabled(false);
+		}
+		
+	    }
+	}
+    }
+    
+
+    
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionConfiguration.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionConfiguration.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionConfiguration.java	(revision 21112)
@@ -0,0 +1,1691 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.DOMTree;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.greenstone.gatherer.util.ZipTools;
+import org.w3c.dom.*;
+
+/** This class provides either access to a pseudo-G3 document, or parses a collect.cfg file in such a way as to provide an xml-type view of its content. This later version is useful as it allows the manipulation and free form editing of a legacy collect.cfg file while still allowing the various CDM data managers to base themselves directly on this model (whereas they used to be independant ListModels which clobbered the ordering of unparsed commands).
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class CollectionConfiguration
+    extends StaticStrings {
+    
+    static final public String ENCODING = "UTF-8";
+
+    static public Document document;
+
+    static public void main(String[] args) {
+	if(args.length >= 1) {
+	    File file = new File(args[0]);
+	    CollectionConfiguration collect_cfg = new CollectionConfiguration(file);
+	    collect_cfg.save(true);
+	    collect_cfg.save(false);
+	    collect_cfg = null;
+	}
+	else {
+	    System.out.println("Usage: CollectionConfiguration <filename>");
+	}
+    }
+
+    /** Find the best insertion position for the given DOM Element. This should try to match command tag, and if found should then try to group by name or type (eg CollectionMeta), or append to end is no such grouping exists (eg PlugIns). Failing a command match it will check against the command order for the best insertion location.
+     * @param target_element the command Element to be inserted
+     * @return the Element which the given command should be inserted before, or null to append to end of list
+     */
+    static public Node findInsertionPoint(Element target_element) {
+	///ystem.err.println("Find insertion point: " + target_element.getNodeName());
+	String target_element_name = target_element.getNodeName();
+	Element document_element = document.getDocumentElement();
+	// Try to find commands with the same tag.
+	NodeList matching_elements = document_element.getElementsByTagName(target_element_name);
+	// If we found matching elements, then we have our most likely insertion location, so check within for groupings
+	if(matching_elements.getLength() != 0) {
+	    ///ystem.err.println("Found matching elements.");
+	    // Only CollectionMeta are grouped.
+	    if(target_element_name.equals(COLLECTIONMETADATA_ELEMENT)) {
+		///ystem.err.println("Dealing with collection metadata");
+		// Special case: CollectionMeta can be added at either the start or end of a collection configuration file. However the start position is reserved for special metadata, so if no non-special metadata can be found we must append to the end.
+		// So if the command to be added is special add it immediately after any other special command
+		if(target_element.getAttribute(SPECIAL_ATTRIBUTE).equals(TRUE_STR)) {
+		    int index = 0;
+		    Element matched_element = (Element) matching_elements.item(index);
+		    Element sibling_element = (Element) matched_element.getNextSibling();
+		    while(sibling_element.getAttribute(SPECIAL_ATTRIBUTE).equals(TRUE_STR)) {
+			index++;
+			matched_element = (Element) matching_elements.item(index);
+			sibling_element = (Element) matched_element.getNextSibling();
+		    }
+		    if(sibling_element.getNodeName().equals(NEWLINE_ELEMENT)) {
+			Element newline_element = document.createElement(NEWLINE_ELEMENT);
+			document_element.insertBefore(newline_element, sibling_element);
+		    }
+		    return sibling_element;
+		}
+		// Otherwise try to find a matching 'name' and add after the last one in that group.
+		else {
+		    int index = 0;
+		    target_element_name = target_element.getAttribute(NAME_ATTRIBUTE);
+		    boolean found = false;
+		    // Skip all of the special metadata
+		    Element matched_element = (Element) matching_elements.item(index);
+		    while(matched_element.getAttribute(SPECIAL_ATTRIBUTE).equals(TRUE_STR)) {
+			index++;
+			matched_element = (Element) matching_elements.item(index);
+		    }
+		    // Begin search
+		    while(!found && matched_element != null) {
+			if(matched_element.getAttribute(NAME_ATTRIBUTE).equals(target_element_name)) {
+			    found = true;
+			}
+			else {
+			    index++;
+			    matched_element = (Element) matching_elements.item(index);
+			}
+		    }
+		    // If we found a match, we need to continue checking until we find the last name match.
+		    if(found) {
+			index++;
+			Element previous_sibling = matched_element;
+			Element sibling_element = (Element) matching_elements.item(index);
+			while(sibling_element != null && sibling_element.getAttribute(NAME_ATTRIBUTE).equals(target_element_name)) {
+			    previous_sibling = sibling_element;
+			    index++;
+			    sibling_element = (Element) matching_elements.item(index);
+			}
+			// Previous sibling now holds the command immediately before where we want to add, so find its next sibling and add to that. In this one case we can ignore new lines!
+			return previous_sibling.getNextSibling();
+		    }
+		    // If not found we just add after last metadata element
+		    else {
+			Element last_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
+			return last_element.getNextSibling();
+		    }
+		}
+
+	    }
+	    else {
+		///ystem.err.println("Not dealing with collection meta.");
+		Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
+		// One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
+		Node sibling_element = matched_element.getNextSibling();
+		if(sibling_element != null && sibling_element.getNodeName().equals(NEWLINE_ELEMENT)) {
+		    Element newline_element = document.createElement(NEWLINE_ELEMENT);
+		    document_element.insertBefore(newline_element, sibling_element);
+		}
+		return sibling_element; // Note that this may be null
+	    }
+	}
+	///ystem.err.println("No matching elements found.");
+	// Locate where this command is in the ordering
+	int command_index = -1;
+	for(int i = 0; command_index == -1 && i < COMMAND_ORDER.length; i++) {
+	    if(COMMAND_ORDER[i].equals(target_element_name)) {
+		command_index = i;
+	    }
+	}
+	///ystem.err.println("Command index is: " + command_index);
+	// Now move forward, checking for existing elements in each of the preceeding command orders.
+	int preceeding_index = command_index - 1;
+	///ystem.err.println("Searching before the target command.");
+	while(preceeding_index >= 0) {
+	    matching_elements = document_element.getElementsByTagName(COMMAND_ORDER[preceeding_index]);
+	    // If we've found a match
+	    if(matching_elements.getLength() > 0) {
+		// We add after the last element
+		Element matched_element = (Element) matching_elements.item(matching_elements.getLength() - 1);
+		// One final quick test. If the matched element is immediately followed by a NewLine command, then we insert another NewLine after the matched command, then return the NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
+		Node sibling_element = matched_element.getNextSibling();
+		if(sibling_element != null && sibling_element.getNodeName().equals(NEWLINE_ELEMENT)) {
+		    Element newline_element = document.createElement(NEWLINE_ELEMENT);
+		    document_element.insertBefore(newline_element, sibling_element);
+		}
+		return sibling_element; // Note that this may be null
+	    }
+	    preceeding_index--;
+	}
+	// If all that fails, we now move backwards through the commands
+	int susceeding_index = command_index + 1;
+	///ystem.err.println("Searching after the target command.");
+	while(susceeding_index < COMMAND_ORDER.length) {
+	    matching_elements = document_element.getElementsByTagName(COMMAND_ORDER[susceeding_index]);
+	    // If we've found a match
+	    if(matching_elements.getLength() > 0) {
+		// We add before the first element
+		Element matched_element = (Element) matching_elements.item(0);
+		// One final quick test. If the matched element is immediately preceeded by a NewLine command, then we insert another NewLine before the matched command, then return this new NewLine instead (thus the about to be inserted command will be placed between the two NewLines)
+		Node sibling_element = matched_element.getPreviousSibling();
+		if(sibling_element != null && sibling_element.getNodeName().equals(NEWLINE_ELEMENT)) {
+		    Element newline_element = document.createElement(NEWLINE_ELEMENT);
+		    document_element.insertBefore(newline_element, sibling_element);
+		}
+		return sibling_element; // Note that this may be null
+	    }
+	    susceeding_index++;
+	}
+	// Well. Apparently there are no other commands in this collection configuration. So append away...
+	return null;
+    }
+
+    static public String toString(Element command_element, boolean show_extracted_namespace) {
+	String command_element_name = command_element.getNodeName();
+	if(command_element_name.equals(CLASSIFY_ELEMENT)) {
+	    return classifyToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(FORMAT_ELEMENT)) {
+	    return formatToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(INDEXES_ELEMENT)) {
+	    return indexesToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(INDEX_DEFAULT_ELEMENT)) {
+	    return indexDefaultToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(LANGUAGES_ELEMENT)) {
+	    return languagesToString(command_element);
+	}
+	else if(command_element_name.equals(LANGUAGE_DEFAULT_ELEMENT)) {
+	    return languageDefaultToString(command_element);
+	}
+	else if(command_element_name.equals(LEVELS_ELEMENT)) {
+	    return levelsToString(command_element);
+	}
+	else if(command_element_name.equals(COLLECTIONMETADATA_ELEMENT)) {
+	    return metadataToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(COLLECTIONMETADATA_CREATOR_ELEMENT)) {
+	    return metadataToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(COLLECTIONMETADATA_MAINTAINER_ELEMENT)) {
+	    return metadataToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(COLLECTIONMETADATA_PUBLIC_ELEMENT)) {
+	    return metadataToString(command_element, show_extracted_namespace);
+	}
+	else if (command_element_name.equals(BUILDTYPE_ELEMENT)) {
+	    return metadataToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(PLUGIN_ELEMENT)) {
+	    return pluginToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(SEARCHTYPE_ELEMENT)) {
+	    return searchtypeToString(command_element);
+	}
+	else if(command_element_name.equals(SUBCOLLECTION_ELEMENT)) {
+	    return subcollectionToString(command_element, show_extracted_namespace);
+	}
+	else if(command_element_name.equals(SUBCOLLECTION_DEFAULT_INDEX_ELEMENT)) {
+	    return subcollectionDefaultIndexToString(command_element);
+	}
+	else if(command_element_name.equals(SUBCOLLECTION_INDEXES_ELEMENT)) {
+	    return subcollectionIndexesToString(command_element);
+	}
+	else if(command_element_name.equals(SUPERCOLLECTION_ELEMENT)) {
+	    return supercollectionToString(command_element);
+	}
+	else if(command_element_name.equals(UNKNOWN_ELEMENT)) {
+	    return unknownToString(command_element);
+	}
+	return "";
+    }
+
+    /** Parses arguments from a tokenizer and returns a HashMap of mappings. The tricky bit here is that not all entries in the HashMap are name->value pairs, as some arguments are boolean and are turned on by their presence. Arguments are denoted by a '-' prefix.
+     * @param tokenizer a CommandTokenizer based on the unconsumed portion of a command string
+     * @return a HashMap containing the arguments parsed
+     */
+    static public HashMap parseArguments(CommandTokenizer tokenizer) {
+	HashMap arguments = new HashMap();
+	String name = null;
+	String value = null;
+	while(tokenizer.hasMoreTokens() || name != null) {
+	    // First we retrieve a name if we need one.
+	    if(name == null) {
+		name = tokenizer.nextToken();
+	    }
+	    // Now we attempt to retrieve a value
+	    if(tokenizer.hasMoreTokens()) {
+		value = tokenizer.nextToken();
+		// Test if the value is actually a name, and if so add the name by itself, then put value into name so that it is parsed correctly during the next loop.
+		if(value.startsWith(StaticStrings.MINUS_CHARACTER)) {
+		    arguments.put(name, null);
+		    name = value;
+		}
+		// Otherwise we have a typical name->value pair ready to go
+		else {
+		    arguments.put(name, value);
+		    name = null;
+		}
+	    }
+	    // Otherwise its a binary flag
+	    else {
+		arguments.put(name, null);
+		name = null;
+	    }
+	}
+	return arguments;
+    }
+
+    static private ArrayList known_metadata;
+
+    /** Gives the preferred ordering of commands */
+    static final private String[] COMMAND_ORDER = {StaticStrings.COLLECTIONMETADATA_CREATOR_ELEMENT, StaticStrings.COLLECTIONMETADATA_MAINTAINER_ELEMENT, StaticStrings.COLLECTIONMETADATA_PUBLIC_ELEMENT,  StaticStrings.SEARCHTYPE_ELEMENT, StaticStrings.BUILDTYPE_ELEMENT, StaticStrings.PLUGIN_ELEMENT, StaticStrings.INDEXES_ELEMENT, StaticStrings.LEVELS_ELEMENT, StaticStrings.INDEX_DEFAULT_ELEMENT, StaticStrings.LANGUAGES_ELEMENT, StaticStrings.LANGUAGE_DEFAULT_ELEMENT, StaticStrings.SUBCOLLECTION_ELEMENT, StaticStrings.SUBCOLLECTION_INDEXES_ELEMENT, StaticStrings.SUBCOLLECTION_DEFAULT_INDEX_ELEMENT, StaticStrings.SUPERCOLLECTION_ELEMENT, StaticStrings.CLASSIFY_ELEMENT, StaticStrings.FORMAT_ELEMENT, StaticStrings.COLLECTIONMETADATA_ELEMENT};
+
+    /** ************************** Public Data Members ***************************/
+
+    /** ************************** Private Data Members ***************************/
+    
+    private File collect_cfg_file;
+
+    /** ************************** Public Methods ***************************/
+
+    public CollectionConfiguration(File collect_cfg_file)
+    {
+	this.collect_cfg_file = collect_cfg_file;
+	// If collect_cfg is xml we can load it straight away
+	String collect_config_name = collect_cfg_file.getName();
+	if(collect_config_name.equals(COLLECTCONFIGURATION_XML)) {
+	    document = XMLTools.parseXMLFile(collect_cfg_file);
+	}
+	// Otherwise if this is a legacy collect.cfg file then read in the template and send to magic parser
+	else if(collect_config_name.equals(COLLECT_CFG)) {
+	    document = XMLTools.parseXMLFile("xml/CollectionConfig.xml", true);
+	    parse(collect_cfg_file);
+	}
+    }
+
+    /** This debug facility shows the currently loaded collect.cfg or CollectConfig.xml file as a DOM tree. */
+    public void display() {
+	JDialog dialog = new JDialog(Gatherer.g_man, "Collection Configuration", false);
+	dialog.setSize(400,400);
+	JPanel content_pane = (JPanel) dialog.getContentPane();
+	final DOMTree tree = new DOMTree(document);
+	JButton refresh_button = new GLIButton("Refresh Tree");
+	refresh_button.setMnemonic(KeyEvent.VK_R);
+	refresh_button.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent event) {
+		    tree.setDocument(document);
+		}
+	    });
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JScrollPane(tree), BorderLayout.CENTER);
+	content_pane.add(refresh_button, BorderLayout.SOUTH);
+	dialog.setVisible(true);
+    }
+
+
+    public Element getCreator() {
+	Element element = getOrCreateElementByTagName(COLLECTIONMETADATA_CREATOR_ELEMENT, null, null);
+	element.setAttribute(NAME_ATTRIBUTE, COLLECTIONMETADATA_CREATOR_STR);
+	element.setAttribute(SPECIAL_ATTRIBUTE, TRUE_STR);
+	return element;
+    }
+
+    public Element getDocumentElement() {
+	return document.getDocumentElement();
+    }
+
+    public File getFile() {
+	return collect_cfg_file;
+    }
+
+    /** Retrieve or create the languages Element. */
+    public Element getLanguages() {
+	return getOrCreateElementByTagName(LANGUAGES_ELEMENT, null, null);
+    }
+
+    public Element getLevels() {
+	return getOrCreateElementByTagName(LEVELS_ELEMENT, null, null);
+    }
+
+    public Element getMaintainer() {
+	Element element = getOrCreateElementByTagName(COLLECTIONMETADATA_MAINTAINER_ELEMENT, null, null);
+	element.setAttribute(NAME_ATTRIBUTE, COLLECTIONMETADATA_MAINTAINER_STR);
+	element.setAttribute(SPECIAL_ATTRIBUTE, TRUE_STR);
+	return element;
+    }
+
+    /** Retrieve or create the indexes Element. Note that this method behaves differently from the other getBlah methods, in that it also has to keep in mind that indexes come in two flavours, MG and MGPP. */
+    public Element getMGIndexes() {
+	return getOrCreateElementByTagName(INDEXES_ELEMENT, MGPP_ATTRIBUTE, FALSE_STR);
+    }
+
+    public Element getMGPPIndexes() {
+	return getOrCreateElementByTagName(INDEXES_ELEMENT, MGPP_ATTRIBUTE, TRUE_STR);
+    }
+
+    public Element getPublic() {
+	Element element = getOrCreateElementByTagName(COLLECTIONMETADATA_PUBLIC_ELEMENT, null, null);
+	element.setAttribute(NAME_ATTRIBUTE, COLLECTIONMETADATA_PUBLIC_STR);
+	element.setAttribute(SPECIAL_ATTRIBUTE, TRUE_STR);
+	return element;
+    }
+
+    public Element getBuildType() {
+	Element element = getOrCreateElementByTagName(BUILDTYPE_ELEMENT, null, null);
+	element.setAttribute(NAME_ATTRIBUTE, BUILDTYPE_STR);
+	element.setAttribute(SPECIAL_ATTRIBUTE, TRUE_STR);
+	return element;
+
+    }
+    /** Retrieve or create the searchtype element. */
+    public Element getSearchType() {
+	///ystem.err.println("Get or create element by tag name: " + name);
+	Element document_element = document.getDocumentElement();
+	NodeList elements = document_element.getElementsByTagName(SEARCHTYPE_ELEMENT);
+	int elements_length = elements.getLength();
+	if(elements_length > 0) {
+	    document_element = null;
+	    return (Element) elements.item(0);
+	}
+	// Create the element
+	Element element = document.createElement(SEARCHTYPE_ELEMENT);
+	Node target_node = findInsertionPoint(element);
+	if(target_node != null) {
+	    document_element.insertBefore(element, target_node);
+	}
+	else {
+	    document_element.appendChild(element);
+	}
+	document_element = null;
+	// Append a default search type node - form
+	Element a_searchtype_element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.CONTENT_ELEMENT);
+	a_searchtype_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, SearchTypeManager.SEARCH_TYPES[0]);
+	element.appendChild(a_searchtype_element);
+	return element;
+    }
+
+    /** Retrieve or create the subindexes Element. */
+    public Element getSubIndexes() {
+	return getOrCreateElementByTagName(SUBCOLLECTION_INDEXES_ELEMENT, null, null);
+    }
+
+    /** Retrieve or create the supercollections Element. */
+    public Element getSuperCollection() {
+	return getOrCreateElementByTagName(SUPERCOLLECTION_ELEMENT, null, null);
+    }
+
+    public boolean ready() {
+	return document != null;
+    }
+
+    public void save() {
+	save(false);
+    }
+
+    public void save(boolean force_xml) {
+	if(collect_cfg_file.exists()) {
+	    File original_file = new File(collect_cfg_file.getParentFile(), COLLECT_CFG);
+	    File backup_file = new File(collect_cfg_file.getParentFile(), "collect.bak");
+	    if(backup_file.exists()) {
+		backup_file.delete();
+	    }
+	    if(!original_file.renameTo(backup_file)) {
+		DebugStream.println("Can't rename collect.cfg");
+	    }
+	}
+	if(force_xml || collect_cfg_file.getName().equals(COLLECTCONFIGURATION_XML)) {
+	    ///ystem.err.println("Writing XML");
+	    XMLTools.writeXMLFile(new File(collect_cfg_file.getParentFile(), COLLECTCONFIGURATION_XML), document);
+	}
+	else {
+	    ///ystem.err.println("Writing text");
+	    try {		
+		OutputStream ostream = new FileOutputStream(collect_cfg_file);
+		Writer file_writer = new OutputStreamWriter(ostream, ENCODING);
+		//FileWriter file_writer = new FileWriter(collect_cfg_file, false);
+		BufferedWriter buffered_writer = new BufferedWriter(file_writer);
+		Element collect_config_element = document.getDocumentElement();
+		NodeList command_elements = collect_config_element.getChildNodes();
+		boolean just_wrote_newline = false; // Prevent two or more newlines in a row
+		for(int i = 0; i < command_elements.getLength(); i++) {
+		    Node command_node = command_elements.item(i);
+		    if(command_node instanceof Element) {
+			Element command_element = (Element) command_node;
+			// The only thing left are NewLine elements
+			if(command_element.getNodeName().equals(NEWLINE_ELEMENT) && !just_wrote_newline) {
+			    buffered_writer.newLine();
+			    just_wrote_newline = true;
+			}
+			// Anything else we write to file, but only if it has been assigned, the exception being the Indexes element which just get commented if unassigned (a side effect of MG && MGPP compatibility)
+			else if(!command_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(FALSE_STR) || command_element.getNodeName().equals(INDEXES_ELEMENT) || command_element.getNodeName().equals(INDEX_DEFAULT_ELEMENT) || command_element.getNodeName().equals(LEVELS_ELEMENT)){
+			    String command;
+			    // format statements we write out with ex. still present
+			    if (command_element.getNodeName().equals(FORMAT_ELEMENT)) {
+				command = toString(command_element, true);
+			    } else {
+				command = toString(command_element, false);
+			    }
+			    if(command != null && command.length()> 0 ) {
+				write(buffered_writer, command);
+				buffered_writer.newLine();
+				just_wrote_newline = false;
+			    }
+			}
+		    }
+		}
+		buffered_writer.close();
+
+		// If we're using a remote Greenstone server, upload the new collect.cfg file
+		if (Gatherer.isGsdlRemote) {
+		    String collection_name = Gatherer.c_man.getCollection().getName();
+		    RemoteGreenstoneServer.uploadCollectionFile(collection_name, collect_cfg_file);
+		}
+	    }
+	    catch (Exception exception) {
+		DebugStream.println("Error in CollectionConfiguration.save(boolean): " + exception);
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+    }
+
+    /** ************************** Private Methods ***************************/
+
+    static private String classifyToString(Element command_element, boolean show_extracted_namespace)
+    {
+	StringBuffer text = new StringBuffer(CLASSIFY_STR);
+	text.append(TAB_CHARACTER);
+	text.append(command_element.getAttribute(TYPE_ATTRIBUTE));
+	text.append(SPACE_CHARACTER);
+	NodeList option_elements = command_element.getElementsByTagName(OPTION_ELEMENT);
+	int option_elements_length = option_elements.getLength();
+	for(int j = 0; j < option_elements_length; j++) {
+	    Element option_element = (Element) option_elements.item(j);
+	    if(option_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(TRUE_STR)) {
+		text.append(StaticStrings.MINUS_CHARACTER);
+		text.append(option_element.getAttribute(NAME_ATTRIBUTE));
+		String value_str = XMLTools.getValue(option_element);
+
+		// Remove the extracted metadata namespaces if required
+		if (value_str.length() > 0) {
+		    StringTokenizer string_tokenizer = new StringTokenizer(value_str, ",");
+		    value_str = "";
+		    while (string_tokenizer.hasMoreElements()) {
+			String raw_token = (String) string_tokenizer.nextElement();
+			String token = raw_token.trim();
+			MetadataElement metadata_element = MetadataTools.getMetadataElementWithDisplayName(token);
+			if (metadata_element != null) {
+			    token = metadata_element.getFullName();
+			}
+
+			if (token.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+			    token = token.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+			}
+
+			value_str = value_str + token;
+			if (string_tokenizer.hasMoreElements()) {
+			    value_str = value_str + ",";
+			}
+		    }
+		}
+
+		text.append(SPACE_CHARACTER);
+		if (value_str.indexOf(SPACE_CHARACTER) == -1) {
+		    text.append(value_str);
+		}
+		else {
+		    text.append(SPEECH_CHARACTER);
+		    text.append(value_str);
+		    text.append(SPEECH_CHARACTER);
+		}
+		value_str = null;
+		if(j < option_elements_length - 1) {
+		    text.append(SPACE_CHARACTER);
+		}
+	    }
+	    option_element = null;
+	}
+	option_elements = null;
+	return text.toString();
+    }
+
+    static private String formatToString(Element command_element, boolean show_extracted_namespace) {
+	StringBuffer text = new StringBuffer(FORMAT_STR);
+	text.append(SPACE_CHARACTER);
+	text.append(command_element.getAttribute(NAME_ATTRIBUTE));
+	text.append(SPACE_CHARACTER);
+	String value_str = command_element.getAttribute(VALUE_ATTRIBUTE);
+	if(value_str.length() != 0) {
+	    text.append(value_str);
+	}
+	else {
+	    // Remember to encode format string to Greenstone specification
+	    value_str = Codec.transform(XMLTools.getValue(command_element), Codec.DOM_TO_GREENSTONE);
+	    // Remove any references to a namespace for extracted metadata
+	    if (!show_extracted_namespace) {
+		String match_string = "\\[" + MetadataSetManager.EXTRACTED_METADATA_NAMESPACE + "\\.";
+		value_str = value_str.replaceAll(match_string, "[");
+	    }
+
+	    text.append(SPEECH_CHARACTER);
+	    text.append(value_str);
+	    text.append(SPEECH_CHARACTER);
+	}
+	value_str = null;
+	return text.toString();
+    }
+
+    /** Retrieve or create the indexes Element. */
+    static private Element getOrCreateElementByTagName(String name, String conditional_attribute, String required_value) {
+	Element document_element = document.getDocumentElement();
+	NodeList elements = document_element.getElementsByTagName(name);
+	int elements_length = elements.getLength();
+	if(elements_length > 0) {
+	    if(conditional_attribute == null) {
+		document_element = null;
+		return (Element) elements.item(0);
+	    }
+	    else {
+		for(int i = 0; i < elements_length; i++) {
+		    Element element = (Element) elements.item(i);
+		    if(element.getAttribute(conditional_attribute).equals(required_value)) {
+			document_element = null;
+			return element;
+		    }
+		    element = null;
+		}
+	    }
+	}
+	// Create the element
+	Element element = document.createElement(name);
+	// If there was a property set it
+	if(conditional_attribute != null) {
+	    element.setAttribute(conditional_attribute, required_value);
+	}
+	Node target_node = findInsertionPoint(element);
+	if(target_node != null) {
+	    document_element.insertBefore(element, target_node);
+	}
+	else {
+	    document_element.appendChild(element);
+	}
+	document_element = null;
+	return element;
+    }
+
+    static private String indexesToString(Element command_element, boolean show_extracted_namespace) {
+	boolean comment_only = false;
+	StringBuffer text = new StringBuffer("");
+	if(command_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(FALSE_STR)) {
+	    text.append("#");
+	    comment_only = true;
+	}
+	text.append(INDEX_STR);
+	text.append(TAB_CHARACTER);
+	if(!comment_only) {
+	    text.append(TAB_CHARACTER);
+	}
+	NodeList index_elements = command_element.getElementsByTagName(INDEX_ELEMENT);
+	if (index_elements.getLength() == 0) { // no indexes
+	    return "";
+	}
+	// For each index, write its level, a colon, then concatenate its child content elements into a single comma separated list
+	int index_elements_length = index_elements.getLength();
+	for(int j = 0; j < index_elements_length; j++) {
+	    Element index_element = (Element) index_elements.item(j);
+	    String level_str = index_element.getAttribute(LEVEL_ATTRIBUTE);
+	    if(level_str.length() > 0) {
+		text.append(level_str);
+		text.append(StaticStrings.COLON_CHARACTER);
+	    }
+	    NodeList content_elements = index_element.getElementsByTagName(CONTENT_ELEMENT);
+	    int content_elements_length = content_elements.getLength();
+	    // Don't output anything if no indexes are set
+	    if(content_elements_length == 0) {
+		return null;
+	    }
+	    for(int k = 0; k < content_elements_length; k++) {
+		Element content_element = (Element) content_elements.item(k);
+		String name_str = content_element.getAttribute(NAME_ATTRIBUTE);
+		if(!show_extracted_namespace && name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+		    name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+		}
+		text.append(name_str);
+		name_str = null;
+		if(k < content_elements_length - 1) {
+		    text.append(StaticStrings.COMMA_CHARACTER);
+		}
+		content_element = null;
+	    }
+	    if(j < index_elements_length - 1) {
+		text.append(SPACE_CHARACTER);
+	    }
+	    content_elements = null;
+	    index_element = null;
+	}
+	index_elements = null;
+	return text.toString();
+    }
+
+    static private String indexDefaultToString(Element command_element, boolean show_extracted_namespace) {
+	StringBuffer text = new StringBuffer("");
+	if(command_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(FALSE_STR)) {
+	    text.append("#");
+	}
+	text.append(INDEX_DEFAULT_STR);
+	text.append(TAB_CHARACTER);
+	text.append(command_element.getAttribute(LEVEL_ATTRIBUTE));
+	text.append(StaticStrings.COLON_CHARACTER);
+	NodeList content_elements = command_element.getElementsByTagName(CONTENT_ELEMENT);
+	int content_elements_length = content_elements.getLength();
+	for(int j = 0; j < content_elements_length; j++) {
+	    Element content_element = (Element) content_elements.item(j);
+	    String name_str = content_element.getAttribute(NAME_ATTRIBUTE);
+	    if(!show_extracted_namespace && name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+		name_str = name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+	    }
+	    text.append(name_str);
+	    name_str = null;
+	    if(j < content_elements_length - 1) {
+		text.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	    content_element = null;
+	}
+	content_elements = null;
+	return text.toString();
+    }
+
+    static private String languagesToString(Element command_element) {
+	StringBuffer text = new StringBuffer(LANGUAGES_STR);
+	text.append(TAB_CHARACTER);
+	// Retrieve all the languages and write them out in a space separated list
+	NodeList language_elements = command_element.getElementsByTagName(LANGUAGE_ELEMENT);
+	int language_elements_length = language_elements.getLength();
+	if(language_elements_length == 0) {
+	    return null;
+	}
+	for(int j = 0; j < language_elements_length; j++) {
+	    Element language_element = (Element) language_elements.item(j);
+	    text.append(language_element.getAttribute(NAME_ATTRIBUTE));
+	    if(j < language_elements_length - 1) {
+		text.append(SPACE_CHARACTER);
+	    }
+	}
+	return text.toString();
+    }
+
+    static private String languageDefaultToString(Element command_element) {
+	StringBuffer text = new StringBuffer(LANGUAGE_DEFAULT_STR);
+	text.append(TAB_CHARACTER);
+	text.append(command_element.getAttribute(NAME_ATTRIBUTE));
+	return text.toString();
+    }
+
+    static private String levelsToString(Element command_element) {
+	StringBuffer text = new StringBuffer("");
+	if(!command_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(TRUE_STR)) {
+	    text.append("#");
+	}
+	text.append(LEVELS_STR);
+	text.append(TAB_CHARACTER);
+	text.append(TAB_CHARACTER);
+	NodeList content_elements = command_element.getElementsByTagName(CONTENT_ELEMENT);
+	int content_elements_length = content_elements.getLength();
+	// Don't output anything if no levels are set.
+	if(content_elements_length == 0) {
+	    return null;
+	}
+	for(int i = 0; i < content_elements_length; i++) {
+	    Element content_element = (Element) content_elements.item(i);
+	    text.append(content_element.getAttribute(NAME_ATTRIBUTE));
+	    text.append(SPACE_CHARACTER);
+	}
+	return text.substring(0, text.length() - 1);
+    }
+
+    static private String metadataToString(Element command_element, boolean text_value) {
+	// lets first check the value - if its empty, don't bother sticking it in the config file
+	String value_str = XMLTools.getValue(command_element);
+	if (value_str.equals("")) {
+	    return "";
+	}
+	boolean special = false;
+	
+	StringBuffer text = new StringBuffer("");
+	String name_str = command_element.getAttribute(NAME_ATTRIBUTE);
+	// If the name is one of the special four, we don't write the collectionmeta first. Note maintainer and buildtype are singled out for 'prittying' reasons.
+	if(name_str.equals(COLLECTIONMETADATA_MAINTAINER_STR)|| name_str.equals(BUILDTYPE_STR) ) {
+	    text.append(name_str);
+	    text.append(TAB_CHARACTER);
+	    special = true;
+	}
+	else if (name_str.equals(COLLECTIONMETADATA_CREATOR_STR) || name_str.equals(COLLECTIONMETADATA_PUBLIC_STR) ) {
+	    text.append(name_str);
+	    text.append(TAB_CHARACTER);
+	    text.append(TAB_CHARACTER);
+	    special = true;
+	}
+	else {
+	    text.append(COLLECTIONMETADATA_STR);
+	    text.append(TAB_CHARACTER);
+	    text.append(name_str);
+	    text.append(SPACE_CHARACTER);
+	    String language_str = command_element.getAttribute(LANGUAGE_ATTRIBUTE);
+	    text.append(LBRACKET_CHARACTER);
+	    text.append(LANGUAGE_ARGUMENT);
+	    text.append(language_str);
+	    text.append(RBRACKET_CHARACTER);
+	    text.append(SPACE_CHARACTER);
+
+	    if(known_metadata != null) {
+		known_metadata.add(name_str);
+	    }
+	    language_str = null;
+	}
+	name_str = null;
+
+	// The value string we retrieved will be encoded for xml, so we now decode it - to text if text_value set. This parameter was originally show_extracted_namespace, but sincethis is only true for 'toString()' commands from within the CDM, its good enough to determine if this toString() will be used to display on screen, or write to collect.cfg
+	if(text_value == CollectionMeta.TEXT) {
+	    value_str = Codec.transform(value_str, Codec.DOM_TO_TEXT);
+	}
+	else {
+	    value_str = Codec.transform(value_str, Codec.DOM_TO_GREENSTONE);
+	}
+
+	// We don't wrap the email addresses in quotes, nor the other special metadata
+	if(special) {
+	    text.append(value_str);
+	}
+	else {
+	    text.append(SPEECH_CHARACTER);
+	    text.append(value_str);
+	    text.append(SPEECH_CHARACTER);
+	}
+	value_str = null;
+	return text.toString();
+    }
+
+    /** Parse a collect.cfg into a DOM model representation. 
+     * note we are ignoring 2.39 compatibility now. */
+    private void parse(File collect_cfg_file) {
+	try {
+
+	    Element collect_cfg_element = document.getDocumentElement();
+	    // Read in the file one command at a time.
+	    InputStream istream = new FileInputStream(collect_cfg_file);
+	    Reader in_reader = new InputStreamReader(istream, ENCODING);
+	    BufferedReader in = new BufferedReader(in_reader);
+	    String command_str = null;
+	    while((command_str = in.readLine()) != null) {
+		boolean append_element = true;
+		Element command_element = null;
+		// A command may be broken over several lines.
+		command_str = command_str.trim();
+		boolean eof = false;
+		while(!eof && command_str.endsWith(NEWLINE_CHARACTER)) {
+		    String next_line = in.readLine();
+		    if(next_line != null) {
+			next_line = next_line.trim();
+			if(next_line.length() > 0) {
+			    // Remove the new line character
+			    command_str = command_str.substring(0, command_str.lastIndexOf(NEWLINE_CHARACTER));
+			    // And append the next line, which due to the test above must be non-zero length
+			    command_str = command_str + next_line;
+			}
+			next_line = null;
+		    }
+		    // If we've reached the end of the file theres nothing more we can do
+		    else {
+			eof = true;
+		    }
+		}
+		// If there is still a new line character, then we remove it and hope for the best
+		if(command_str.endsWith(NEWLINE_CHARACTER)) {
+		    command_str = command_str.substring(0, command_str.lastIndexOf(NEWLINE_CHARACTER));
+		}
+		// Now we've either got a command to parse...
+		if(command_str.length() != 0) {
+		    // Start trying to figure out what it is
+		    //StringTokenizer tokenizer = new StringTokenizer(command_str);
+		    // Instead of a standard string tokenizer I'm going to use the new version of CommandTokenizer, which is not only smart enough to correctly notice speech marks and correctly parse them out, but now also takes the input stream so it can rebuild tokens that stretch over several lines.
+		    CommandTokenizer tokenizer = new CommandTokenizer(command_str, in);
+		    String command_type = tokenizer.nextToken().toLowerCase();
+		    // Why can't you switch on strings eh? We pass it to the various subparsers who each have a bash at parsing the command. If none can parse the command, an unknown element is created
+		    if(command_element == null && command_type.equals(CLASSIFY_STR)) {
+			command_element = parseClassify(command_str);
+		    }
+		    if(command_element == null && command_type.equals(FORMAT_STR)) {
+			command_element = parseFormat(tokenizer); // Revised to handle multiple lines
+		    }
+		    if(command_element == null && (command_type.equals(INDEX_STR)  || command_type.equals(COMMENTED_INDEXES_STR))) {
+			command_element = parseIndex(command_str);
+		    }
+		    if(command_element == null && (command_type.equals(INDEX_DEFAULT_STR) || command_type.equals(COMMENTED_INDEX_DEFAULT_STR))) {
+			
+			command_element = parseIndexDefault(command_str);
+		    }
+		    if(command_element == null && command_type.equals(LANGUAGES_STR)) {
+			command_element = parseLanguage(command_str);
+		    }
+		    if(command_element == null && command_type.equals(LANGUAGE_DEFAULT_STR)) {
+			command_element = parseLanguageDefault(command_str);
+		    }
+		    if(command_element == null && (command_type.equals(LEVELS_STR) || command_type.equals(COMMENTED_LEVELS_STR))) {
+			command_element = parseLevels(command_str);
+		    }
+		    if(command_element == null && command_type.equals(COLLECTIONMETADATA_STR)) {
+			command_element = parseMetadata(tokenizer); // Revised to handle multiple lines
+		    }
+		    if(command_element == null && (command_type.equals(COLLECTIONMETADATA_PUBLIC_STR) || command_type.equals(COLLECTIONMETADATA_CREATOR_STR) || command_type.equals(COLLECTIONMETADATA_MAINTAINER_STR) || command_type.equals(BUILDTYPE_STR))) {
+			command_element = parseMetadataSpecial(command_str);
+		    }
+		    if(command_element == null && command_type.equals(PLUGIN_STR)) {
+			command_element = parsePlugIn(command_str);
+		    }
+		    if(command_element == null && command_type.equals(SEARCHTYPE_STR)) {
+			command_element = parseSearchType(command_str);
+		    }
+		    if(command_element == null && command_type.equals(SUBCOLLECTION_STR)) {
+			command_element = parseSubCollection(command_str);
+		    }
+		    if(command_element == null && command_type.equals(SUBCOLLECTION_DEFAULT_INDEX_STR)) {
+			command_element = parseSubCollectionDefaultIndex(command_str);
+		    }
+		    if(command_element == null && command_type.equals(SUBCOLLECTION_INDEX_STR)) {
+			command_element = parseSubCollectionIndex(command_str);
+		    }
+		    if(command_element == null && (command_type.equals(SUPERCOLLECTION_STR) || command_type.equals(CCS_STR))) {
+			command_element = parseSuperCollection(command_str);
+		    }
+		    // Doesn't match any known type
+		    command_type = null;
+		    if(command_element == null) {
+			// No-one knows what to do with this command, so we create an Unknown command element
+			command_element = document.createElement(UNKNOWN_ELEMENT);
+			XMLTools.setValue(command_element, command_str);
+		    }
+		}
+		// Or an empty line to remember for later
+		else {
+		    command_element = document.createElement(NEWLINE_ELEMENT);
+		}
+		// Now command element shouldn't be null so we append it to the collection config DOM, but only if we haven't been told not to add it
+		//if(append_element) {
+		    collect_cfg_element.appendChild(command_element);
+		    //}
+	    }
+
+	}
+	catch(Exception exception) {
+	    DebugStream.println("Error in CollectionConfiguration.parse(java.io.File): " + exception);
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+    
+    private Element parseClassify(String command_str) {
+	Element command_element = null;
+	try {
+	    CommandTokenizer tokenizer = new CommandTokenizer(command_str);
+	    // Check the token count. The token count from a command tokenizer isn't guarenteed to be correct, but it does give the maximum number of available tokens according to the underlying StringTokenizer (some of which may actually be append together by the CommandTokenizer as being a single argument).
+	    if(tokenizer.countTokens() >= 2) {  // Must support "classify Phind" (no args)
+		command_element = document.createElement(CLASSIFY_ELEMENT);
+		// First token is classify
+		tokenizer.nextToken();
+		// The next token is the classifier type
+		command_element.setAttribute(TYPE_ATTRIBUTE, tokenizer.nextToken());
+		// Now we parse out the remaining arguments into a hashmapping from name to value
+		HashMap arguments = parseArguments(tokenizer);
+		// Assign the arguments as Option elements, but watch out for the metadata argument as we treat that differently
+		Iterator names = arguments.keySet().iterator();
+		while(names.hasNext()) {
+		    String name = (String) names.next();
+		    String value = (String) arguments.get(name); // Can be null
+		    // The metadata argument gets added as the content attribute
+		    if (name.equals(METADATA_ARGUMENT) && value != null) {
+			// Add the extracted namespace onto un-namespaced metadata names
+			StringTokenizer string_tokenizer = new StringTokenizer(value, ",");
+			value = "";
+			while (string_tokenizer.hasMoreElements()) {
+			    String token = (String) string_tokenizer.nextElement();
+
+			    if (token.indexOf(StaticStrings.NS_SEP) == -1) {
+				token = StaticStrings.EXTRACTED_NAMESPACE + token;
+			    }
+			    else {
+				MetadataElement metadata_element = MetadataTools.getMetadataElementWithName(token);
+				if (metadata_element != null) {
+				    token = metadata_element.getDisplayName();
+				}
+			    }
+
+			    if (!value.equals("")) {
+				value = value + ",";
+			    }
+			    value = value + token;
+			}
+		    }
+		    // Everything else is an Option Element
+		    Element option_element = document.createElement(OPTION_ELEMENT);
+		    option_element.setAttribute(NAME_ATTRIBUTE, name.substring(1));
+		    if(value != null) {
+			// Remove any speech marks appended in strings containing whitespace
+			if(value.startsWith(SPEECH_CHARACTER) && value.endsWith(SPEECH_CHARACTER)) {
+			    value = value.substring(1, value.length() - 1);
+			}
+			XMLTools.setValue(option_element, value);
+		    }
+		    option_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+		    option_element.setAttribute(CUSTOM_ATTRIBUTE, TRUE_STR);
+		    command_element.appendChild(option_element);
+		    option_element = null;
+		    name = null;
+		    value = null;
+		}
+		names = null;
+		arguments = null;
+	    }
+	    tokenizer = null;
+	}
+	catch(Exception error) {
+	}
+	return command_element;
+    }
+
+    private Element parseFormat(CommandTokenizer tokenizer) {
+	Element command_element = null;
+	try {
+	    command_element = document.createElement(FORMAT_ELEMENT);
+	    String name_str = tokenizer.nextToken();
+	    String value_str = tokenizer.nextToken();
+	    if(name_str != null && value_str != null) {
+		command_element.setAttribute(NAME_ATTRIBUTE, name_str);
+		// If the value is true or false we add it as an attribute
+		if(value_str.equalsIgnoreCase(TRUE_STR) || value_str.equalsIgnoreCase(FALSE_STR)) {
+		    command_element.setAttribute(VALUE_ATTRIBUTE, value_str.toLowerCase());
+		}
+		// Otherwise it gets added as a text node
+		else {
+		    // Ready the value str (which can contain all sorts of funky characters) for writing as a DOM value
+		    value_str = Codec.transform(value_str, Codec.GREENSTONE_TO_DOM);
+		    XMLTools.setValue(command_element, value_str);
+		}
+	    }
+	    else {
+		command_element = null;
+	    }
+	    name_str = null;
+	    value_str = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	    command_element = null;
+	}
+	return command_element;
+    }
+
+    private Element parseIndex(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    String command = tokenizer.nextToken();
+	    command_element = document.createElement(INDEXES_ELEMENT);
+	    command_element.setAttribute(ASSIGNED_ATTRIBUTE, (command.equals(INDEX_STR) ? TRUE_STR : FALSE_STR));
+	    command = null;
+	    if(!tokenizer.hasMoreTokens()) {
+
+		// there are no indexes
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, FALSE_STR);
+		command_element.setAttribute(MGPP_ATTRIBUTE, FALSE_STR); // for now
+		tokenizer = null;
+		return command_element;
+	    }
+
+	    while(tokenizer.hasMoreTokens()) {
+		Element index_element = document.createElement(INDEX_ELEMENT);
+		String index_str = tokenizer.nextToken();
+		// There are two types of index we have to consider. Old G2.38 and earlier use level:source tuplets while G2.39+ have just a single, non-comma separated list where order is important.
+		boolean old_index;
+		if(index_str.indexOf(COLON_CHARACTER) != -1) {
+		    old_index = true;
+		    index_element.setAttribute(LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));
+		    index_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
+		    command_element.setAttribute(MGPP_ATTRIBUTE, FALSE_STR);
+		}
+		else {
+		    command_element.setAttribute(MGPP_ATTRIBUTE, TRUE_STR);
+		    old_index = false;
+		}
+		StringTokenizer content_tokenizer = new StringTokenizer(index_str, StaticStrings.COMMA_CHARACTER);
+		while(content_tokenizer.hasMoreTokens()) {
+		    Element content_element = document.createElement(CONTENT_ELEMENT);
+		    String content_str = content_tokenizer.nextToken();
+		    // Since the contents of indexes have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
+		    if(content_str.indexOf(StaticStrings.NS_SEP) == -1) {
+			if(content_str.equals(StaticStrings.TEXT_STR) || (!old_index && content_str.equals(StaticStrings.ALLFIELDS_STR))) {
+				// Our special strings are OK.
+			}
+			else {
+			    content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
+			}
+		    }
+		    content_element.setAttribute(NAME_ATTRIBUTE, content_str);
+		    index_element.appendChild(content_element);
+		    content_element = null;
+		}
+		content_tokenizer = null;
+		index_str = null;
+		command_element.appendChild(index_element);
+		index_element = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+	return command_element;
+    }
+
+    private Element parseIndexDefault(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 2) {
+		command_element = document.createElement(INDEX_DEFAULT_ELEMENT);
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, ((tokenizer.nextToken()).equals(INDEX_DEFAULT_STR) ? TRUE_STR : FALSE_STR));
+		String index_str = tokenizer.nextToken();
+		command_element.setAttribute(LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));
+		String content_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
+		StringTokenizer content_tokenizer = new StringTokenizer(content_str, StaticStrings.COMMA_CHARACTER);
+		while(content_tokenizer.hasMoreTokens()) {
+		    Element content_element = document.createElement(CONTENT_ELEMENT);
+		    content_element.setAttribute(NAME_ATTRIBUTE, content_tokenizer.nextToken());
+		    command_element.appendChild(content_element);
+		    content_element = null;
+		}
+		content_tokenizer = null;
+		content_str = null;
+		content_str = null;
+		index_str = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+    private Element parseBuildType(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 2) {
+		command_element = document.createElement(INDEX_DEFAULT_ELEMENT);
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, ((tokenizer.nextToken()).equals(INDEX_DEFAULT_STR) ? TRUE_STR : FALSE_STR));
+		String index_str = tokenizer.nextToken();
+		command_element.setAttribute(LEVEL_ATTRIBUTE, index_str.substring(0, index_str.indexOf(StaticStrings.COLON_CHARACTER)));
+		String content_str = index_str.substring(index_str.indexOf(StaticStrings.COLON_CHARACTER) + 1);
+		StringTokenizer content_tokenizer = new StringTokenizer(content_str, StaticStrings.COMMA_CHARACTER);
+		while(content_tokenizer.hasMoreTokens()) {
+		    Element content_element = document.createElement(CONTENT_ELEMENT);
+		    content_element.setAttribute(NAME_ATTRIBUTE, content_tokenizer.nextToken());
+		    command_element.appendChild(content_element);
+		    content_element = null;
+		}
+		content_tokenizer = null;
+		content_str = null;
+		content_str = null;
+		index_str = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseLanguage(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		command_element = document.createElement(LANGUAGES_ELEMENT);
+		while(tokenizer.hasMoreTokens()) {
+		    Element language_element = document.createElement(LANGUAGE_ELEMENT);
+		    language_element.setAttribute(NAME_ATTRIBUTE, tokenizer.nextToken());
+		    command_element.appendChild(language_element);
+		    language_element = null;
+		}
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseLanguageDefault(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 2) {
+		command_element = document.createElement(LANGUAGE_DEFAULT_ELEMENT);
+		tokenizer.nextToken();
+		String default_language_str = tokenizer.nextToken();
+		command_element.setAttribute(NAME_ATTRIBUTE, default_language_str);
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
+		default_language_str = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseLevels(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    // First token is command type (levels)
+	    String command = tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		command_element = document.createElement(LEVELS_ELEMENT);
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, (command.equals(LEVELS_STR) ? TRUE_STR : FALSE_STR));
+		while(tokenizer.hasMoreTokens()) {
+		    Element level_element = document.createElement(CONTENT_ELEMENT);
+		    level_element.setAttribute(NAME_ATTRIBUTE, tokenizer.nextToken());
+		    command_element.appendChild(level_element);
+		    level_element = null;
+		}
+	    }
+	    command = null;
+	}
+	catch(Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseMetadata(CommandTokenizer tokenizer) {
+	Element command_element = null;
+	try {
+	    command_element = document.createElement(COLLECTIONMETADATA_ELEMENT);
+	    String name_str = tokenizer.nextToken();
+	    String value_str = tokenizer.nextToken();
+	    if(name_str != null && value_str != null) {
+		String language_str = Configuration.getLanguage();
+		// Check if the value string is actually a language string
+		if(value_str.startsWith(LBRACKET_CHARACTER) && value_str.endsWith(RBRACKET_CHARACTER)) {
+		    language_str = value_str.substring(value_str.indexOf(LANGUAGE_ARGUMENT) + 2, value_str.length() - 1);
+		    value_str = tokenizer.nextToken();
+		}
+		if(value_str != null) {
+		    // Ready the value str (which can contain all sorts of funky characters) for writing as a DOM value
+		    value_str = Codec.transform(value_str, Codec.GREENSTONE_TO_DOM);
+		    command_element.setAttribute(NAME_ATTRIBUTE, name_str);
+		    command_element.setAttribute(LANGUAGE_ATTRIBUTE, language_str);
+		    command_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+		    XMLTools.setValue(command_element, value_str);
+		}
+		else {
+		    command_element = null;
+		}
+		language_str = null;
+	    }
+	    else {
+		command_element = null;
+	    }
+	    name_str = null;
+	    value_str = null;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	    command_element = null;
+	}
+	return command_element;
+    }
+
+    private Element parseMetadataSpecial(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 2) {
+		String name_str = tokenizer.nextToken();
+		String value_str = tokenizer.nextToken();
+		if (name_str.equals(COLLECTIONMETADATA_CREATOR_STR)) {
+		    command_element = document.createElement(COLLECTIONMETADATA_CREATOR_ELEMENT);
+		}
+		else if(name_str.equals(COLLECTIONMETADATA_MAINTAINER_STR)) {
+		    command_element = document.createElement(COLLECTIONMETADATA_MAINTAINER_ELEMENT);
+		}
+		else if(name_str.equals(COLLECTIONMETADATA_PUBLIC_STR)) {
+		    command_element = document.createElement(COLLECTIONMETADATA_PUBLIC_ELEMENT);
+		}
+		else if (name_str.equals(BUILDTYPE_STR)) {
+		    command_element = document.createElement(BUILDTYPE_ELEMENT);
+		}
+		if(command_element != null) {
+		    command_element.setAttribute(NAME_ATTRIBUTE, name_str);
+		    command_element.setAttribute(LANGUAGE_ATTRIBUTE, ENGLISH_LANGUAGE_STR);
+		    command_element.setAttribute(SPECIAL_ATTRIBUTE, TRUE_STR);
+		    command_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+		    if(value_str.startsWith(SPEECH_CHARACTER) && value_str.endsWith(SPEECH_CHARACTER)) {
+			value_str = value_str.substring(1, value_str.length() - 1);
+		    }
+		    XMLTools.setValue(command_element, value_str);
+		}
+		value_str = null;
+		name_str = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parsePlugIn(String command_str) {
+	Element command_element = null;
+	boolean use_metadata_files = false;
+	try {
+	    CommandTokenizer tokenizer = new CommandTokenizer(command_str);
+	    // Check the token count. The token count from a command tokenizer isn't guarenteed to be correct, but it does give the maximum number of available tokens according to the underlying StringTokenizer (some of which may actually be append together by the CommandTokenizer as being a single argument).
+	    if(tokenizer.countTokens() >= 2) {
+		command_element = document.createElement(PLUGIN_ELEMENT);
+		// First token is plugin
+		tokenizer.nextToken();
+		// The next token is the type
+		String type = tokenizer.nextToken();
+		command_element.setAttribute(TYPE_ATTRIBUTE, type);
+		// Now we parse out the remaining arguments into a hashmapping from name to value
+		HashMap arguments = parseArguments(tokenizer);
+		// Assign the arguments as Option elements, but watch out for the metadata argument as we treat that differently
+		Iterator names = arguments.keySet().iterator();
+		while(names.hasNext()) {
+		    String name = (String) names.next();
+		    String value = (String) arguments.get(name); // Can be null
+		    Element option_element = document.createElement(OPTION_ELEMENT);
+		    if(name.substring(1).equals(USE_METADATA_FILES_ARGUMENT)) {
+			use_metadata_files = true;
+		    }
+		    option_element.setAttribute(NAME_ATTRIBUTE, name.substring(1));
+		    option_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+		    option_element.setAttribute(CUSTOM_ATTRIBUTE, TRUE_STR); // All arguments are considered to be custom until matched against base plugins arguments
+		    if(value != null) {
+			// Remove any speech marks appended in strings containing whitespace
+			if(value.startsWith(SPEECH_CHARACTER) && value.endsWith(SPEECH_CHARACTER)) {
+			    value = value.substring(1, value.length() - 1);
+			}
+			if(name.equals(METADATA_ARGUMENT)) {
+			    // The metadata argument must be the fully qualified name of a metadata element, so if it doesn't yet have a namespace, append the extracted metadata namespace.
+			    if(value.indexOf(StaticStrings.NS_SEP) == -1) {
+				value = StaticStrings.EXTRACTED_NAMESPACE + value;
+			    }
+			}
+			XMLTools.setValue(option_element, value);
+		    }
+		    command_element.appendChild(option_element);
+		    option_element = null;
+		    name = null;
+		    value = null;
+		}
+
+		// We must have some RecPlug options: use_metadata_files
+		if (type.equals(RECPLUG_STR)) {
+		    if (!use_metadata_files) {
+			Element option_element = document.createElement(OPTION_ELEMENT);
+			option_element.setAttribute(NAME_ATTRIBUTE, USE_METADATA_FILES_ARGUMENT);
+			option_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+			option_element.setAttribute(CUSTOM_ATTRIBUTE, TRUE_STR);
+			command_element.appendChild(option_element);
+			option_element = null;
+		    }
+		}
+		type = null;
+		names = null;
+		arguments = null;
+	    }
+	    tokenizer = null;
+	}
+	catch(Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseSearchType(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    // First token is command type (levels)
+	    tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		command_element = document.createElement(SEARCHTYPE_ELEMENT);
+		command_element.setAttribute(ASSIGNED_ATTRIBUTE, TRUE_STR);
+		while(tokenizer.hasMoreTokens()) {
+		    Element search_element = document.createElement(CONTENT_ELEMENT);
+		    search_element.setAttribute(NAME_ATTRIBUTE, tokenizer.nextToken());
+		    command_element.appendChild(search_element);
+		    search_element = null;
+		}
+	    }
+	}
+	catch(Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseSubCollection(String command_str) {
+	Element command_element = null;
+	try {
+	    CommandTokenizer tokenizer = new CommandTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 3) {
+		command_element = document.createElement(SUBCOLLECTION_ELEMENT);
+		// First token is command type
+		tokenizer.nextToken();
+		// Then subcollection identifier
+		command_element.setAttribute(NAME_ATTRIBUTE, tokenizer.nextToken());
+		// Then finally the pattern used to build the subcollection partition
+		String full_pattern_str = tokenizer.nextToken();
+		// Set inclusion/exclusion flag and remove any exclamation mark
+		boolean exclusion = full_pattern_str.startsWith(EXCLAMATION_CHARACTER);
+		if (exclusion) {
+		    full_pattern_str = full_pattern_str.substring(1, full_pattern_str.length());
+		    command_element.setAttribute(TYPE_ATTRIBUTE, EXCLUDE_STR);
+		}
+		else {
+		    command_element.setAttribute(TYPE_ATTRIBUTE, INCLUDE_STR);
+		}
+		StringTokenizer pattern_tokenizer = new StringTokenizer(full_pattern_str, SEPARATOR_CHARACTER);
+		if(pattern_tokenizer.countTokens() >= 2) {
+		    String content_str = pattern_tokenizer.nextToken();
+		    // Since the contents of indexes have to be certain keywords, or metadata elements, if the content isn't a keyword and doesn't yet have a namespace, append the extracted metadata namespace.
+		    if(!content_str.equals(StaticStrings.FILENAME_STR) && content_str.indexOf(StaticStrings.NS_SEP) == -1) {
+			content_str = StaticStrings.EXTRACTED_NAMESPACE + content_str;
+		    }
+		    command_element.setAttribute(CONTENT_ATTRIBUTE, content_str);
+		    XMLTools.setValue(command_element, pattern_tokenizer.nextToken());
+		    if(pattern_tokenizer.hasMoreTokens()) {
+			command_element.setAttribute(OPTIONS_ATTRIBUTE, pattern_tokenizer.nextToken());
+		    }
+		}
+		pattern_tokenizer = null;
+	    }
+	}
+	catch(Exception exception) {
+	    exception.printStackTrace();
+	}
+	return command_element;
+    }
+
+    private Element parseSubCollectionDefaultIndex(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() == 2) {
+		command_element = document.createElement(SUBCOLLECTION_DEFAULT_INDEX_ELEMENT);
+		tokenizer.nextToken();
+		//command_element.setAttribute(CONTENT_ATTRIBUTE, tokenizer.nextToken());
+		String content_str = tokenizer.nextToken();
+		StringTokenizer content_tokenizer = new StringTokenizer(content_str, StaticStrings.COMMA_CHARACTER);
+		while(content_tokenizer.hasMoreTokens()) {
+		    Element content_element = document.createElement(CONTENT_ELEMENT);
+		    content_element.setAttribute(NAME_ATTRIBUTE, content_tokenizer.nextToken());
+		    command_element.appendChild(content_element);
+		    content_element = null;
+		}
+		content_tokenizer = null;
+		content_str = null;
+	    }
+	    tokenizer = null;
+	}
+	catch(Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseSubCollectionIndex(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		command_element = document.createElement(SUBCOLLECTION_INDEXES_ELEMENT);
+	    }
+	    while(tokenizer.hasMoreTokens()) {
+		Element subcollectionindex_element = document.createElement(INDEX_ELEMENT);
+		//command_element.setAttribute(CONTENT_ATTRIBUTE, tokenizer.nextToken());
+		String content_str = tokenizer.nextToken();
+		StringTokenizer content_tokenizer = new StringTokenizer(content_str, StaticStrings.COMMA_CHARACTER);
+		while(content_tokenizer.hasMoreTokens()) {
+		    Element content_element = document.createElement(CONTENT_ELEMENT);
+		    content_element.setAttribute(NAME_ATTRIBUTE, content_tokenizer.nextToken());
+		    subcollectionindex_element.appendChild(content_element);
+		    content_element = null;
+		}
+		content_tokenizer = null;
+		content_str = null;
+		command_element.appendChild(subcollectionindex_element);
+		subcollectionindex_element = null;
+	    }
+	    tokenizer = null;
+	}
+	catch (Exception exception) {
+	}
+	return command_element;
+    }
+
+    private Element parseSuperCollection(String command_str) {
+	Element command_element = null;
+	try {
+	    StringTokenizer tokenizer = new StringTokenizer(command_str);
+	    if(tokenizer.countTokens() >= 3) {
+		command_element = document.createElement(SUPERCOLLECTION_ELEMENT);
+		tokenizer.nextToken();
+		while(tokenizer.hasMoreTokens()) {
+		    Element collection_element = document.createElement(COLLECTION_ELEMENT);
+		    collection_element.setAttribute(NAME_ATTRIBUTE, tokenizer.nextToken());
+		    command_element.appendChild(collection_element);
+		    collection_element = null;
+		}
+	    }
+	    tokenizer = null;
+	}
+	catch(Exception exception) {
+	}
+	return command_element;
+    }
+
+    static private String pluginToString(Element command_element, boolean show_extracted_namespace) {
+	StringBuffer text = new StringBuffer();
+	if(!command_element.getAttribute(SEPARATOR_ATTRIBUTE).equals(TRUE_STR)) {
+	    text.append(PLUGIN_STR);
+	    text.append(TAB_CHARACTER);
+	    text.append(TAB_CHARACTER);
+	    text.append(command_element.getAttribute(TYPE_ATTRIBUTE));
+	    // Retrieve, and output, the arguments
+	    NodeList option_elements = command_element.getElementsByTagName(OPTION_ELEMENT);
+	    int option_elements_length = option_elements.getLength();
+	    if(option_elements_length > 0) {
+		text.append(SPACE_CHARACTER);
+		for(int j = 0; j < option_elements_length; j++) {
+		    Element option_element = (Element) option_elements.item(j);
+		    if(option_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(TRUE_STR)) {
+			text.append(StaticStrings.MINUS_CHARACTER);
+			text.append(option_element.getAttribute(NAME_ATTRIBUTE));
+			String value_str = XMLTools.getValue(option_element);
+			if(!show_extracted_namespace && value_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+			    value_str = value_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+			}
+			if(value_str.length() > 0) {
+			    text.append(SPACE_CHARACTER);
+			    if(value_str.indexOf(SPACE_CHARACTER) == -1) {
+				text.append(value_str);
+			    }
+			    else {
+				text.append(SPEECH_CHARACTER);
+				text.append(value_str);
+				text.append(SPEECH_CHARACTER);
+			    }
+			}
+			value_str = null;
+			if(j < option_elements_length - 1) {
+			    text.append(SPACE_CHARACTER);
+			}
+		    }
+		    option_element = null;
+		}
+	    }
+	    option_elements = null;
+	}
+	return text.toString();
+    }
+
+    static private String searchtypeToString(Element command_element) {
+	if(command_element.getAttribute(ASSIGNED_ATTRIBUTE).equals(TRUE_STR)) {
+	    StringBuffer text = new StringBuffer(SEARCHTYPE_STR);
+	    text.append(TAB_CHARACTER);
+	    NodeList search_elements = command_element.getElementsByTagName(CONTENT_ELEMENT);
+	    int search_elements_length = search_elements.getLength();
+	    for(int i = 0; i < search_elements_length; i++) {
+		Element search_element = (Element) search_elements.item(i);
+		text.append(search_element.getAttribute(NAME_ATTRIBUTE));
+		text.append(SPACE_CHARACTER);
+	    }
+	    return text.substring(0, text.length() - 1);
+	}
+	else {
+	    return null;
+	}
+    }
+
+    static private String subcollectionToString(Element command_element, boolean show_extracted_namespace) {
+	StringBuffer text = new StringBuffer(SUBCOLLECTION_STR);
+	text.append(SPACE_CHARACTER);
+	text.append(command_element.getAttribute(NAME_ATTRIBUTE));
+	text.append(SPACE_CHARACTER);
+	text.append(TAB_CHARACTER);
+	text.append(SPEECH_CHARACTER);
+	if(command_element.getAttribute(TYPE_ATTRIBUTE).equals(EXCLUDE_STR)) {
+	    text.append(EXCLAMATION_CHARACTER);
+	}
+	String content_str = command_element.getAttribute(CONTENT_ATTRIBUTE);
+	if(!show_extracted_namespace && content_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+	    content_str = content_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+	}
+	text.append(content_str);
+	content_str = null;
+	text.append(SEPARATOR_CHARACTER);
+	text.append(XMLTools.getValue(command_element));
+	text.append(SEPARATOR_CHARACTER);
+	String options_str = command_element.getAttribute(OPTIONS_ATTRIBUTE);
+	if(options_str.length() > 0) {
+	    text.append(options_str);
+	}
+	options_str = null;
+	text.append(SPEECH_CHARACTER);
+	return text.toString();
+    }
+
+    static private String subcollectionDefaultIndexToString(Element command_element) {
+	StringBuffer text = new StringBuffer(SUBCOLLECTION_DEFAULT_INDEX_STR);
+	text.append(TAB_CHARACTER);
+	NodeList content_elements = command_element.getElementsByTagName(CONTENT_ELEMENT);
+	int content_elements_length = content_elements.getLength();
+	for(int j = 0; j < content_elements_length; j++) {
+	    Element content_element = (Element) content_elements.item(j);
+	    text.append(content_element.getAttribute(NAME_ATTRIBUTE));
+	    if(j < content_elements_length - 1) {
+		text.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	}
+	return text.toString();
+    }
+
+    static private String subcollectionIndexesToString(Element command_element) {
+	StringBuffer text = new StringBuffer(SUBCOLLECTION_INDEX_STR);
+	text.append(TAB_CHARACTER);
+	// Retrieve all of the subcollection index partitions
+	NodeList subcollectionindex_elements = command_element.getElementsByTagName(INDEX_ELEMENT);
+	int subcollectionindex_elements_length = subcollectionindex_elements.getLength();
+	if(subcollectionindex_elements_length == 0) {
+	    return null;
+	}
+	for(int j = 0; j < subcollectionindex_elements_length; j++) {
+	    Element subcollectionindex_element = (Element) subcollectionindex_elements.item(j);
+	    NodeList content_elements = subcollectionindex_element.getElementsByTagName(CONTENT_ELEMENT);
+	    int content_elements_length = content_elements.getLength();
+	    for(int k = 0; k < content_elements_length; k++) {
+		Element content_element = (Element) content_elements.item(k);
+		text.append(content_element.getAttribute(NAME_ATTRIBUTE));
+		if(k < content_elements_length - 1) {
+		    text.append(StaticStrings.COMMA_CHARACTER);
+		}
+	    }
+	    if(j < subcollectionindex_elements_length - 1) {
+		text.append(SPACE_CHARACTER);
+	    }
+	}
+	return text.toString();
+    }
+
+    static private String supercollectionToString(Element command_element) {
+	NodeList content_elements = command_element.getElementsByTagName(COLLECTION_ELEMENT);
+	int content_elements_length = content_elements.getLength();
+	if(content_elements_length > 1) {
+	    StringBuffer text = new StringBuffer(SUPERCOLLECTION_STR);
+	    text.append(TAB_CHARACTER);
+	    for(int j = 0; j < content_elements_length; j++) {
+		Element content_element = (Element) content_elements.item(j);
+		text.append(content_element.getAttribute(NAME_ATTRIBUTE));
+		if(j < content_elements_length - 1) {
+		    text.append(SPACE_CHARACTER);
+		}
+	    }
+	    return text.toString();
+	}
+	return null;
+    }
+
+    static private String unknownToString(Element command_element) {
+	return XMLTools.getValue(command_element);
+    }
+
+    /** Write the text to the buffer. This is used so we don't have to worry about storing intermediate String values just so we can calaulate length and offset.
+     * @param writer the BufferedWriter to which the str will be written
+     * @param str the String to be written
+     */
+    private void write(BufferedWriter writer, String str)
+	throws IOException {
+	writer.write(str, 0, str.length());
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionDesignManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionDesignManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionDesignManager.java	(revision 21112)
@@ -0,0 +1,321 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.apache.xerces.parsers.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+import org.w3c.dom.*;
+import org.xml.sax.*;
+
+/** This manager provides access to submanagers, which in turn provide tools for the designing of Greenstone collections via the information stored in etc/collect.cfg. This class acts as a hub for the managers that handle specific parts of the configuration such as classifiers, format strings and language settings.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class CollectionDesignManager {
+    /** This listener listens for any event on any of the components in any of the sub-views, and marks the collection as needing saving if any change occurs. */
+    static public CDMChangeListener change_listener;
+    /** These listeners listen to changes in the Design mode so as to allow incremental building */
+    static public DesignChangeListener all_change_listener;
+    static public DesignChangeListener buildcol_change_listener;
+    static public DesignChangeListener collect_cfg_change_listener;
+    /** A list of classifiers to use at build time. */
+    static public ClassifierManager classifier_manager;
+    /** The CollectionConfiguration object on which this CDM will be based. */
+    static public CollectionConfiguration collect_config;
+    /** A manager of collection level metadata. */
+    static public CollectionMetaManager collectionmeta_manager;
+    /** A list of formating strings to use at build time. */
+    static public FormatManager format_manager;
+    /** The manager in charge of displaying this manager and the controls for other managers. */
+    static public GeneralManager general_manager;
+    /** List of indexes to be built, and the default index. */
+    static public IndexManager index_manager;
+    /** Contains instructions dealing with the collection language. */
+    static public LanguageManager language_manager;
+    /** A simple manager for the visual review of metadata sets. */
+    static public MetadataSetView metadataset_view;
+    /** A list of plugins to use at build time. */
+    static public PluginManager plugin_manager;
+    /** The manager in charge of all aspects of searchtypes. We also ask this manager whether we are MG or MGPP enabled. */
+    static public SearchTypeManager searchtype_manager;
+    /** Contains: A list of subcollections, (defined on metadatadata), a list of which subcollection indexes to build and the default subcollection index. */
+    static public SubcollectionManager subcollection_manager;
+
+    static public SubcollectionIndexManager subcollectionindex_manager;
+    /** A supercollection command allows a single search to be conducted across several collections. It is a very basic command and so avoids all the crazy model stuff that exists in most of the design managers. */
+    static public SuperCollectionManager supercollection_manager; // Just cause I could ;p
+    /** The text translation manager. */
+    static public TranslationView translation_view;
+    /** These mark what needs to happen when building a collection where ONLY design options have been changed.
+        The build requirements of the higher numbers must include doing everything from the lower numbers. */
+    static final public int ALL = 3;
+    static final public int BUILDCOL = 2;
+    static final public int UPDATE_COLLECT_CFG = 1;
+    static final public int NOTHING = 0;
+    static private int rebuildTypeRequired = NOTHING; //Rebuild type required if only design options have changed
+    static public boolean update_collect_cfg_required = false;
+
+    /** Constructor. Loads a certain collection configuration file, which is parsed into a DOM. This model is then registered with the command information managers, each of whom knows how to, and provides controls to, alter certain commands.
+     * @param collect_config_file the File representing a collection configuration file either in its text (G2) or xml (G3) form
+     */
+    public CollectionDesignManager(File collect_config_file) {
+	DebugStream.println("Initializaing CollectionDesignModule.");
+	change_listener = new CDMChangeListener();
+	all_change_listener = new DesignChangeListener(ALL);
+	buildcol_change_listener = new DesignChangeListener(BUILDCOL);
+	collect_cfg_change_listener = new DesignChangeListener(UPDATE_COLLECT_CFG);
+	// Parse the collection configuration
+	collect_config = new CollectionConfiguration(collect_config_file);
+	if (DebugStream.isDebuggingEnabled()) {
+	    collect_config.display();
+	}
+	loadDesignDetails();
+	DebugStream.println("CollectionDesignModule loaded.");
+    }
+
+    /** Reloads the various managers to ensure they have built themselves from the latest details available in the collection configuration class
+     * @see org.greenstone.gatherer.cdm.ClassifierManager
+     * @see org.greenstone.gatherer.cdm.CollectionMetaManager
+     * @see org.greenstone.gatherer.cdm.FormatManager
+     * @see org.greenstone.gatherer.cdm.GeneralManager
+     * @see org.greenstone.gatherer.cdm.IndexManager
+     * @see org.greenstone.gatherer.cdm.LanguageManager
+     * @see org.greenstone.gatherer.cdm.MetadataSetView
+     * @see org.greenstone.gatherer.cdm.PluginManager
+     * @see org.greenstone.gatherer.cdm.SearchTypeManager
+     * @see org.greenstone.gatherer.cdm.SubcollectionIndexManager
+     * @see org.greenstone.gatherer.cdm.SubcollectionManager
+     * @see org.greenstone.gatherer.cdm.SuperCollectionManager
+     * @see org.greenstone.gatherer.cdm.TranslationView
+     */
+    private void loadDesignDetails() {
+	// Create the command information managers, registering the config file with each as necessary
+	language_manager = new LanguageManager(collect_config.getLanguages());
+	collectionmeta_manager = new CollectionMetaManager();
+	classifier_manager = new ClassifierManager();
+	general_manager = new GeneralManager();
+
+	searchtype_manager = new SearchTypeManager(collect_config.getSearchType());
+	if(searchtype_manager.isSearchTypeEnabled()) {
+	    // note this this could be lucene now, the format is the same as for mgpp
+	    index_manager = new IndexManager(collect_config.getMGPPIndexes());
+	}
+	else {
+	    index_manager = new IndexManager(collect_config.getMGIndexes());
+	}
+
+	metadataset_view = new MetadataSetView();
+	plugin_manager = new PluginManager();
+	plugin_manager.placeSeparator();
+	subcollection_manager = new SubcollectionManager();
+	subcollectionindex_manager = new SubcollectionIndexManager(collect_config.getSubIndexes());
+	supercollection_manager = new SuperCollectionManager(collect_config.getSuperCollection());
+	translation_view = new TranslationView();
+	format_manager = new FormatManager(); // Parse formats at the very end, given that they depend upon several other managers to appear properly.
+    }
+
+    /** This method deconstructs each of the managers, causing them to dispose of their controls.
+     */
+    public void destroy() {
+	// Remove visual the component from its parent.
+	if(general_manager.getParent() != null) {
+	    general_manager.getParent().remove(general_manager);
+	}
+	// Remove references from persistant listeners.
+	classifier_manager.destroy();
+	classifier_manager = null;
+	format_manager.destroy();
+	format_manager = null;
+	general_manager.destroy();
+	general_manager = null;
+	index_manager.destroy();
+	index_manager = null;
+	language_manager.destroy();
+	language_manager = null;
+	metadataset_view.destroy();
+	metadataset_view = null;
+	plugin_manager.destroy();
+	plugin_manager = null;
+	subcollection_manager.destroy();
+	subcollection_manager = null;
+	supercollection_manager.destroy();
+	supercollection_manager = null;
+	translation_view.destroy();
+	translation_view = null;
+    }
+
+    /** Display the GUI interface for the CollectionDesignManager in the centre of the indicated panel.
+     * @param target the JPanel you wish to display the gui on
+     */
+    public void display(JPanel target) {
+	target.add(general_manager, BorderLayout.CENTER);
+    }
+    /** When the tab on the JTabbedPane that contains the GUI is selected, this method is called to ensure that the controls are all up to date, in terms of references to metadata etc.
+     */
+    public void gainFocus() {
+	general_manager.gainFocus();
+    }
+
+    /** Called when the detail mode has changed which in turn may cause several design elements to be available/hidden
+     * @param mode the new mode as an int
+     */
+    public void modeChanged(int mode) {
+	plugin_manager.modeChanged(mode);
+	subcollection_manager.modeChanged(mode);
+    }
+
+    /** The cdm is considered to be ready if the collect.cfg file was found and parsed and the collection title is not error.
+     * @return true if the collection is ready, false otherwise
+     */
+    public boolean ready() {
+	return collect_config.ready();
+    }
+
+    /** Ensure that saving will actually work. This can return false if there is a collection filename clash for instance. */
+    public boolean canSave() {
+	return (general_manager == null || general_manager.canSave());
+    }
+
+
+    public void save() {
+	save(false);
+    }
+    /** Cause the current collection configuration to be written out to disk.
+     */
+    public void save(boolean force_save)
+    {
+	
+	general_manager.loseFocus();
+	if (!update_collect_cfg_required && !force_save) {
+	    DebugStream.println("Asked to save the config file, but don't need to");
+	    return;
+	}
+
+	DebugStream.println("Saving the collection config file");
+	
+	// Release collection as necessary
+	String collection_name = Gatherer.c_man.getCollection().getName();
+	boolean collection_released = false;
+
+	if (Gatherer.c_man.built() && LocalLibraryServer.isRunning() == true) {
+	    // Release the collection
+	    LocalLibraryServer.releaseCollection(collection_name);
+	    collection_released = true;
+	}
+
+	collect_config.save();
+
+	// Read collection
+	if (collection_released) {
+	    // Now re-add collection to server to force format commands to be processed
+	    LocalLibraryServer.addCollection(collection_name);
+	}
+
+	// Unset formats changed
+	update_collect_cfg_required = false;
+	
+    }
+
+
+    static public Document XMLStringToDOM(StringBuffer xml, String form) 
+    {
+	Document document = null;
+
+	// If something has gone horribly wrong then xml will be empty.
+	if(xml.length() > 0) {
+	    try {
+		// Then read the xml from the piped input stream.
+		StringReader xml_sr = new StringReader(xml.toString());
+		InputSource source = new InputSource(xml_sr);
+		DOMParser parser = new DOMParser();
+		parser.parse(source);
+		document = parser.getDocument();
+	    }
+	    catch (Exception error) {
+		System.err.println("Failed when trying to parse XML stream ");
+		error.printStackTrace();
+	    }
+	}
+	else {
+	    //DebugStream.println("Zero length argument xml detected for: " + form);
+	    String[] margs = new String[1];
+	    margs[0] = form;
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.PlugInManager.PlugIn_XML_Parse_Failed", margs), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);  // !! TO DO: This is bad -- this function is used for more than just plugins
+	}
+
+	return document;
+    }
+
+    public static int getRebuildTypeRequired() {
+	return rebuildTypeRequired;
+    }
+    public static void resetRebuildTypeRequired() {
+	setRebuildTypeRequired(NOTHING);
+    }
+    public static void setRebuildTypeRequired(int number) {
+	rebuildTypeRequired = number;
+    }
+
+    /**
+     * What exactly does this do?
+     */
+    private class CDMChangeListener
+	implements ActionListener, DocumentListener {
+
+	/** Gives notification that an event has happened */
+	public void actionPerformed(ActionEvent event) {
+	    Gatherer.c_man.getCollection().setSaved(false);
+	}
+
+	/** Gives notification that an attribute or set of attributes changed. */
+	public void changedUpdate(DocumentEvent e) {
+	    Gatherer.c_man.getCollection().setSaved(false);
+	}
+
+	/** Gives notification that there was an insert into the document. */
+	public void insertUpdate(DocumentEvent e) {
+	    Gatherer.c_man.getCollection().setSaved(false);
+	}
+
+	/** Gives notification that a portion of the document has been removed. */
+	public void removeUpdate(DocumentEvent e) {
+	    Gatherer.c_man.getCollection().setSaved(false);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMeta.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMeta.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMeta.java	(revision 21112)
@@ -0,0 +1,186 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class encapsulates a single collection level metadata assignment, which constitutes a name, language and value.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class CollectionMeta
+    implements DOMProxyListEntry {
+
+    static final public boolean TEXT = true;
+    static final public boolean GREENSTONE = false;
+
+    private boolean dummy = false;
+    private Element element = null;
+    private String text = null;
+
+    /** Constructor.
+     * @param element the Element from which we will determine metadata details
+     */
+    public CollectionMeta(Element element) {
+	this.element = element;
+    }
+
+    /** Constructor to create a new piece of metadata given its name. */
+    public CollectionMeta(String name) {
+	element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
+	element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
+	element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, Configuration.getLanguage());
+	element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
+    }
+
+    /** Constructor to create a new piece of metadata given its name. */
+    public CollectionMeta(String name, String language) {
+	element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.COLLECTIONMETADATA_ELEMENT);
+	element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
+	element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, language);
+	element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
+    }
+
+    /** Constructor to create a new piece of metadata given its name. */
+    public CollectionMeta(String name, String language, boolean dummy) {
+	this(name, language);
+	this.dummy = dummy;
+    }
+
+    /** Method to compare two collection metadata objects to calculate their respective ordering.
+     * @param object the other metadata to compare to, as an Object
+     * @return an int which is less than 0 if this object proceeds the given object, 0 if they are equal and greater than 0 otherwise.
+     * @see org.greenstone.gatherer.cdm.Language
+     */
+    public int compareTo(Object object) {
+	return toString().compareTo(object.toString());
+    }
+
+    /** Factory constructor. */
+    public DOMProxyListEntry create(Element element) {
+	return new CollectionMeta(element);
+    }
+
+    /** Method to compare two collection metadata objects for equality.
+     * @param object The other metadata to compare to, as an <strong>Object</strong>.
+     * @return A <i>boolean</i> value of <i>true</i> if the object are equal, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    /** Method to retrieve the value of language.
+     * @return the value of language as a <strong>String</strong>.
+     */
+    public String getLanguage() {
+	// Retrieve the language string
+	return element.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE);
+    }
+
+    /** Method to retrieve the value of name.
+     * @return the name attribute of the collection meta as a String 
+     */
+    public String getName() {
+	return element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+    }
+    /** Method to retrieve the value of value (well great choice of name there).
+     * @return The value of value as a <strong>String</strong>.
+     */
+    public String getValue(boolean text_value) {
+	String raw_value = XMLTools.getValue(element);
+	// Decode the raw value depending on whether the user asked for the TEXT or GREENSTONE version
+	if(text_value == TEXT) {
+	    return Codec.transform(raw_value, Codec.DOM_TO_TEXT);
+	}
+	else {
+	    return Codec.transform(raw_value, Codec.DOM_TO_GREENSTONE);
+	}
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public boolean isDummy() {
+	return dummy;
+    }
+
+    /** Determine if this metadata is one of the four special pieces of metadata.
+     * @return true if this metadata is special, false otherwise.
+     */
+    public boolean isSpecial() {
+	return (element != null && element.getAttribute(StaticStrings.SPECIAL_ATTRIBUTE).equals(StaticStrings.TRUE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	text = null;
+    }
+
+    /** Change the value of value. 
+     * @param raw_value the new value as a String.
+     */
+    public void setValue(String raw_value) {
+	// we need to check if the value has changed
+	String current_value = XMLTools.getValue(element);
+	String new_value = Codec.transform(raw_value, Codec.TEXT_TO_DOM);
+	if (!current_value.equals(new_value)) {
+	// Only raw html text can be given to setValue so we need to encode it
+	    XMLTools.setValue(element, new_value);
+	    text = null; // Reset text
+	    // And determine if this makes the metadata assigned
+	    setAssigned(raw_value != null && raw_value.length() > 0);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    /** Method to print out this class as it would appear within the collection configuration file.
+     * @return A <strong>String</strong> containing the text value of this class.
+     */
+    public String toString() {
+	if(text == null) {
+	    text = CollectionConfiguration.toString(element, true);
+	}
+	return text;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMetaManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMetaManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CollectionMetaManager.java	(revision 21112)
@@ -0,0 +1,211 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+/** This class is responsible for maintaining a list of assigned collection level metadata, and for allows manipulations on the aforementioned data.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class CollectionMetaManager 
+    extends DOMProxyListModel {
+    
+    /** Constructor. */
+    public CollectionMetaManager() {
+	super(CollectionDesignManager.collect_config.getDocumentElement(), StaticStrings.COLLECTIONMETADATA_ELEMENT, new CollectionMeta(""));
+	DebugStream.println("CollectionMetaManager: " + getSize() + " metadata parsed.");
+    }
+    /** Method to add a new piece of metadata.
+     * @param metadata the new CollectionMeta
+     */
+    public void addMetadatum(CollectionMeta metadata) {
+	if(!contains(metadata)) {
+	    Element element = metadata.getElement();
+	    // Locate where we should insert this new metadata. 
+	    Node target_node = CollectionConfiguration.findInsertionPoint(element);
+	    add(root, metadata, target_node);
+	    if (!metadata.getValue(CollectionMeta.TEXT).equals("")) {
+		// if just adding an empty metadata, the configuration hasn't changed.
+		Gatherer.c_man.configurationChanged();
+	    }
+	}
+	//Gatherer.c_man.configurationChanged();
+    }
+
+    public CollectionMeta get(int i) {
+	return (CollectionMeta) getElementAt(i);
+    }
+
+    /** Retrieve the languages in use for the metadata assigned to this collection
+     * @return an TreeSet containing the languages used
+     */
+    public TreeSet getLanguages() {
+	TreeSet result = new TreeSet();
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    CollectionMeta metadata = (CollectionMeta) getElementAt(i);
+	    String language = metadata.getLanguage();
+	    result.add(language);
+	}
+	return result;
+    }
+
+    /** Retrieve all of the general metadata. */
+    public ArrayList getMetadata() {
+	ArrayList result = new ArrayList();
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    CollectionMeta metadata = (CollectionMeta) getElementAt(i);
+	    if(!metadata.getName().startsWith(StaticStrings.STOP_CHARACTER)) {
+		result.add(metadata);
+	    }
+	}
+	return result;
+    }
+	 
+    /** Retrieve all of the metadata for the given feature, regardless of language. */
+    public ArrayList getMetadata(String name) {
+	ArrayList result = new ArrayList();
+	int size = getSize(); // Refresh DOM Model
+	for(int i = 0; i < size; i++) {
+	    CollectionMeta metadata = (CollectionMeta) getElementAt(i);
+	    if(metadata.getName().equals(name)) {
+		result.add(metadata);
+	    }
+	}
+	return result;
+    }
+
+    /** Retrieve the named piece of metadata, in the default language, if available. If no such metadata is available then it is created.
+     * @param name the name of the metadatum to retrieve as a String
+     * @return the dom Element containing the specified metadatum
+     */
+    public CollectionMeta getMetadatum(String name) {
+	return getMetadatum(name, true);
+    }
+
+    public CollectionMeta getMetadatum(String name, boolean add_if_not_found) {
+	//DebugStream.println("Get the metadata for " + name + " in the default language.");
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    CollectionMeta metadatum = (CollectionMeta) getElementAt(i);
+	    if(metadatum.getName().equals(name) && metadatum.getLanguage().equals(Configuration.getLanguage())) {
+		DebugStream.println("Found '" + metadatum + "'");
+		return metadatum;
+	    }
+	    else {
+		//DebugStream.println("No match with: " + metadatum.getName() + " [l=" + metadatum.getLanguage() + "] \"" + metadatum.getValue() + "\"");
+	    }
+	    metadatum = null;
+	}
+	if(add_if_not_found) {
+	    CollectionMeta result = new CollectionMeta(name);
+	    addMetadatum(result);
+	    DebugStream.println("Added new metadata: " + name);
+	    return result;
+	}
+	else {
+	    return null;
+	}
+    }
+
+    /** Method to retrieve a certain piece of metadata based on its name and language.
+     * @param name the name of the metadata as an Object (as it may actually be a refernce to an Index or SubIndex)
+     * @param language the language of the metadata.
+     * @param partial <i>true</i> to return the first partial match (ie matches name but not language).
+     * @return The <strong>CollectionMeta</strong> requested, or <i>null</i> if no such metadata.
+     */
+    public CollectionMeta getMetadata(String name, String language, boolean partial) { 
+	CollectionMeta partial_match = null;
+	for(int i = 0; i < getSize(); i++) {
+	    CollectionMeta metadata = (CollectionMeta) getElementAt(i);
+	    Object metadata_name = metadata.getName();
+	    // We test the case of an object match (ie Index to Index)...
+	    if(metadata_name.equals(name)) {
+		if (metadata.getLanguage().equals(language)) {
+		    return metadata;
+		}
+		partial_match = metadata;
+	    }
+	}
+	if(partial) {
+	    return partial_match;
+	}
+	return null;
+    }
+
+    /** Method to remove a piece of metadata.
+     * @param metadata metadata
+     */
+    public void removeMetadata(CollectionMeta metadata) {
+	if(metadata != null) {
+	    String name = metadata.getName();
+	    String language = metadata.getLanguage();
+	    for(int i = 0; i < getSize(); i++) {
+		CollectionMeta other = (CollectionMeta) getElementAt(i);
+		if(name.equals(other.getName()) && language.equals(other.getLanguage())) {
+		    remove(i);
+		    Gatherer.c_man.configurationChanged();
+		    return;
+		}
+		other = null;
+	    }	
+	    language = null;
+	    name = null;
+	}
+    }
+
+    /** Removes all of the metadata with a certain name, regardless of language or value. */
+    public void removeMetadata(String name) {
+	boolean removed_entry = false;
+	for(int i = getSize(); i != 0; i--) {
+	    CollectionMeta other = (CollectionMeta) getElementAt(i - 1);
+	    if(name.equals(other.getName())) {
+		remove(i - 1);
+		removed_entry = true;
+	    }
+	    other = null;
+	}	
+	if(removed_entry) {
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    public int size() {
+	return getSize();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CommandTokenizer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CommandTokenizer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/CommandTokenizer.java	(revision 21112)
@@ -0,0 +1,225 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Title:		Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Company:	  The University of Waikato
+ * Written:	  07/05/02
+ * Revised:	  03/10/02 - Commented
+ **************************************************************************************/
+import java.io.BufferedReader;
+import java.util.StringTokenizer;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.StaticStrings;
+
+/** This class provides an extension to the standard StringTokenizer in that it recognizes quotes (or some form of bracketting) enclose a single token so in something like: <br>format Search '&lt;table&gt;&lt;img src=... &lt;/table&gt;'<br> the formatting string is parsed as a single token. Unfortunately this makes countTokens() unreliable for exact measurement of tokens remaining, and only useful for determining if there are tokens left to be processed (includes any that have already been read into command buffer).
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class CommandTokenizer {
+    
+    static final public int BRACKET_ENCLOSED		= 0;
+    static final public int DOUBLE_QUOTE_ENCLOSED	= 1;
+    static final public int NORMAL			= 2;
+    static final public int QUOTE_ENCLOSED		= 3;
+    
+    private BufferedReader in_stream;
+    private int count = -1;
+    private StringTokenizer internal_tokenizer;
+    private boolean strip_quotes = true;
+    /** Basic Constructor. Used to parse tokens from a string keeping tokens surrounded by speechmarks or square brackets intact. Thus something like:<br>
+     * collectionmeta collectionextra [l = en] "Hello World"<br>
+     * is tokenized thus<br>
+     * {'collectionmeta', 'collectionextra', 'l = en', 'Hello World'}
+     * @param command the command String you wish to tokenize
+     */
+    public CommandTokenizer(String command) {
+	this.internal_tokenizer = new StringTokenizer(command);
+	this.in_stream = null;
+    }
+    
+    /** Advanced Constructor. As above but with one major difference. Since it is provided an input stream (presumably where the command string originated from), it is able to parse a quote enclosed command token that stretches over several lines. Each newline is preserved in the resulting token. There is an extra bitchslap here as comething like a collection extra might have html code in them that contain escaped speechmarks, so extra care must be taken not to break at them. Thus something like:<br>
+     * collectionmeta collectionextra [l = en] "<br>
+     *     an example of the crazy as description we sometimes get which includes of all things something like <a href=\"this.html\"<br>
+     *     >this</a> which you could easily see might be a problem if I parse this niavely."<br>
+     * is tokenized thus<br>
+     * {'collectionmeta', 'collectionextra', 'l = en', '\nan example of the crazy as description we sometimes get which includes of all things something like <a href=\"this.html\"\n>this</a> which you could easily see might be a problem if I parse this niavely.'}
+     * @param command the command String you wish to tokenize
+     * @param in_stream a BufferedReader from which the tokenizer can draw further lines as necessary
+     */
+    public CommandTokenizer(String command, BufferedReader in_stream) {
+	///atherer.println("***** CommandTokenizer *****\nparse:\t" + command + "\n****************************");
+	this.internal_tokenizer = new StringTokenizer(command);
+	this.in_stream = in_stream;
+    }
+    
+    public CommandTokenizer(String command, BufferedReader in_stream, boolean strip_quotes) {
+	this.internal_tokenizer = new StringTokenizer(command);
+	this.in_stream = in_stream;
+	this.strip_quotes = strip_quotes;
+    }
+	
+    /** Returns the minumum number of remaining tokens before the tokenizer runs out of string. There may be more tokens than this count, but never less. The discrepancy is due to internal functionality and the fact we can't read ahead in the string or associated stream without risking the need for unpredictable push-back
+     * @return the minumum number of tokens available as an int
+     */
+    public int countTokens() {
+	if(count == 0 && internal_tokenizer.countTokens() > 1) {
+	    return 1;
+	}
+	if(count == -1) {
+	    count = internal_tokenizer.countTokens();
+	}
+	return count;
+    }
+    
+    /** Determine if there are still tokens available.
+     * @return true if there are more tokens, false otherwise
+     */
+    public boolean hasMoreTokens() {
+	return internal_tokenizer.hasMoreTokens();
+    }
+    
+    /** Method to retrieve the next token from the command, taking care to group tokens enclosed in speech marks.
+     * @return a String containing the next token from the command
+     */
+    public String nextToken() {
+	String result = null;
+	if(internal_tokenizer.hasMoreTokens()) {
+	    StringBuffer buffer = new StringBuffer(internal_tokenizer.nextToken());
+	    switch(buffer.charAt(0)) {
+	    case StaticStrings.DOUBLEQUOTE_CHAR:
+		///ystem.err.println("Building token wrapped by double quotes.");
+		result = buildToken(buffer, StaticStrings.DOUBLEQUOTE_CHAR, this.strip_quotes);
+		break;
+	    case StaticStrings.SINGLEQUOTE_CHAR:
+		///ystem.err.println("Building token wrapped by single quotes.");
+		result = buildToken(buffer, StaticStrings.SINGLEQUOTE_CHAR, this.strip_quotes);
+		break;
+	    case StaticStrings.OPENBRACKET_CHAR:
+		///ystem.err.println("Building token wrapped by brackets.");
+		result = buildToken(buffer, StaticStrings.CLOSEBRACKET_CHAR, false);
+		break;
+	    default:
+		///ystem.err.println("Returning plain string.");
+		result = buffer.toString();
+	    }
+	    buffer = null;
+	}
+	// Because of our tricky counting system we never want to have negative tokens remaining. In fact, unless the internal string buffer is empty, we will return a count of 1 anyway
+	if(count > 0) {
+	    count = count - 1;
+	}
+	///ystem.err.println("----- CommandTokenizer -----\ntoken:\t" + result + "\n----------------------------");
+	return result;
+    }
+    
+    /** Parse in the next token. paying heed to enclosing characters demands, escaped characters, newlines and empty buffers and consequential unexpected end of tokens
+     * @param buffer the StringBuffer in which the partial token is stored (at the first bit that caused this method to be called)
+     * @param end_char the sentinel char we are watching for as it encloses a token
+     * @param strip_characters a boolean denoting whether the enclosing characters should be stripped off
+     * @return the token, either in its entirity less the enclosing characters if required or, if an unexpected end occured, whatever we parsed without its starting enclosing character, again only if required. In fact if we weren't asked to strip characters then we add the enclosing character back in
+     */
+    private String buildToken(StringBuffer buffer, char end_char, boolean strip_characters) {
+	while(buffer.length() == 1 || buffer.charAt(buffer.length() - 1) != end_char || (buffer.length() > 3 && buffer.charAt(buffer.length() - 2) == StaticStrings.BACKSLASH_CHAR)) {
+	    try {
+		// The first version is for the basic tokenizer which has no idea of an input stream, so runs out tokens at the same time as the internal tokenizer does
+		if(internal_tokenizer.hasMoreTokens()) {
+		    buffer.append(StaticStrings.SPACE_CHAR);
+		    buffer.append(internal_tokenizer.nextToken());
+		}
+		// While the second version can draw more lines from the stream until eof occurs
+		else if(in_stream != null) {
+		    String line_str = null;
+		    while(!internal_tokenizer.hasMoreTokens() && (line_str = in_stream.readLine()) != null) {
+			///atherer.println("+++++ CommandTokenizer +++++\nappend:\t" + line_str + "\n+++++++++++++++++++++++++++++");
+			// Its at this stage the our token count becomes completely putu
+			internal_tokenizer = new StringTokenizer(line_str);
+			buffer.append(StaticStrings.NEW_LINE_CHAR); // A new line in the final token
+		    }
+		    line_str = null;
+		    if(internal_tokenizer.hasMoreTokens()) {
+			// Don't add a space if we just added a newline
+			if(buffer.charAt(buffer.length() - 1) != StaticStrings.NEW_LINE_CHAR) {
+			    buffer.append(StaticStrings.SPACE_CHAR);
+			}
+			buffer.append(internal_tokenizer.nextToken());
+		    }
+		    // We've prematurely run out of content, so throw the dummy, or at least return whatever we managed to parse sans its opening character
+		    else {
+			if(strip_characters) {
+			    return buffer.substring(1);
+			}
+			else {
+			    buffer.append(end_char);
+			    return buffer.toString();
+			}
+		    }
+		}
+		// We've prematurely run out of content, so throw the dummy, or at least return whatever we managed to parse sans its opening character
+		else {
+		    if(strip_characters) {
+			return buffer.substring(1);
+		    }
+		    else {
+			buffer.append(end_char);
+			return buffer.toString();
+		    }
+		}
+	    }
+	    // Exception thrown when we attempted reading from the input stream, so throw the dummy, or at least return whatever we managed to parse sans its opening character
+	    catch(Exception exception) {
+		DebugStream.printStackTrace(exception);
+		if(strip_characters) {
+		    return buffer.substring(1);
+		}
+		else {
+		    buffer.append(end_char);
+		    return buffer.toString();
+		}
+	    }
+	}
+	// Return the string sans enclosing characters
+	if(buffer.length() >= 2 && strip_characters) {
+	    return buffer.substring(1, buffer.length() - 1);
+	}
+	else {
+	    return buffer.toString();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Control.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Control.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Control.java	(revision 21112)
@@ -0,0 +1,36 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public interface Control {
+    public void destroy();
+    public void gainFocus();
+    public void loseFocus();
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListEntry.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListEntry.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListEntry.java	(revision 21112)
@@ -0,0 +1,38 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import org.w3c.dom.*;
+
+public interface DOMProxyListEntry 
+    extends Comparable {
+    public DOMProxyListEntry create(Element element);
+    public Element getElement();
+    public boolean isAssigned();
+    public void setAssigned(boolean assigned);
+    public void setElement(Element element);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DOMProxyListModel.java	(revision 21112)
@@ -0,0 +1,265 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.DebugStream;
+import org.w3c.dom.*;
+
+/** This class provides ListModel like access to a list of nodes within a DOM model.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class DOMProxyListModel
+    extends AbstractListModel {
+    protected Element root;
+    private DOMProxyListEntry class_type;
+    private HashMap cache = new HashMap();
+    private NodeList children = null;
+    private String tag_name;
+
+    /** Constructor.
+     * @param root the Element at the root of the subtree to be searched for appropriate child elements
+     * @param tag_name the name of appropriate elements as a String
+     * @param class_type the type of object to wrap the elements returned in, as a DOMProxyListEntry
+     */
+    public DOMProxyListModel(Element root, String tag_name, DOMProxyListEntry class_type) {
+	this.class_type = class_type;
+	this.root = root;
+	this.tag_name = tag_name;
+	this.children = this.root.getElementsByTagName(this.tag_name);
+    }
+
+    /** Used to add an element into the underlying dom, and fire the appropriate repaint events. This version always adds the new element at the very head of the DOM. */
+    public synchronized void add(DOMProxyListEntry entry) {
+	Element element = entry.getElement();
+	if(root.hasChildNodes()) {
+	    Node sibling = root.getFirstChild();
+	    root.insertBefore(element, sibling);
+	    sibling = null;
+	}
+	else {
+	    root.appendChild(element);
+	}
+	element = null;
+	// Regardless fire update event
+	cache.clear();
+	fireIntervalAdded(this, 0, 0);
+    }
+
+    /** Used to add an element into the underlying dom, and fire the appropriate repaint events.
+     * @param index the index where the element should be inserted (relative of the other elements in this proxy list)
+     * @param entry the <strong>DOMProxyListEntry</strong> to be inserted
+     */
+    public synchronized void add(int index, DOMProxyListEntry entry) {
+	///atherer.println("Add entry at " + index + " where size = " + getSize());
+	Element element = entry.getElement();
+	// retrieve the node where we want to insert
+	if(index < children.getLength()) {
+	    Node sibling = children.item(index);
+	    // Find the parent node
+	    Node parent_node = sibling.getParentNode();
+	    parent_node.insertBefore(element, sibling);
+	    sibling = null;
+	}
+	// If the index is too large, we are adding to the end of our list of entries. However you have to remember that this list is only a viewport on the entire DOM so there might be entries following this group that we actually want to insert before (not append at the very end!)
+	else {
+	    // Retrieve the currently last entry
+	    index = children.getLength() - 1;
+	    Node sibling = null;
+	    Node parent_node = null;
+	    if(index >= 0) {
+		sibling = children.item(index);
+		parent_node = sibling.getParentNode();
+		sibling = sibling.getNextSibling();
+	    }
+	    if(sibling != null && parent_node != null) {
+		parent_node.insertBefore(element, sibling);
+	    }
+	    // Add to the root node
+	    else {
+		index = 0;
+		root.appendChild(element);
+	    }
+	}
+	// Regardless fire update event
+	cache.clear();
+	fireIntervalAdded(this, index, index);
+    }
+
+    /** Used to add an element into the underlying dom, and fire the appropriate repaint events. This version inserts the new entry immediately -after- the given entry in the DOM.
+     * @param entry the DOMProxyListEntry to be inserted
+     * @param preceeding_entry the DOMProxyListEntry immediately before where we want the new entry
+     */
+    public synchronized void addAfter(DOMProxyListEntry entry, DOMProxyListEntry preceeding_entry) {
+	Element element = entry.getElement();
+	Element preceeding_sibling = preceeding_entry.getElement();
+	Node parent_node = preceeding_sibling.getParentNode();
+	Node following_sibling = preceeding_sibling.getNextSibling();
+	if(following_sibling != null) {
+	    parent_node.insertBefore(element, following_sibling);
+	}
+	else {
+	    parent_node.appendChild(element);
+	}
+	// Regardless fire update event
+	cache.clear();
+	int index = indexOf(entry);
+	fireIntervalAdded(this, index, index);
+    }
+
+    /** Used to add an element into the underlying dom, and fire the appropriate repaint events. This version inserts the new entry immediately -before- the given entry in the DOM.
+     * @param entry the DOMProxyListEntry to be inserted
+     * @param following_entry the DOMProxyListEntry immediately after where we want the new entry
+     */
+    public synchronized void addBefore(DOMProxyListEntry entry, DOMProxyListEntry following_entry) {
+	Element element = entry.getElement();
+	Element following_sibling = following_entry.getElement();
+	Node parent_node = following_sibling.getParentNode();
+	parent_node.insertBefore(element, following_sibling);
+	// Regardless fire update event
+	cache.clear();
+	int index = indexOf(entry);
+	fireIntervalAdded(this, index, index);
+    }
+
+    public synchronized void add(Node parent, DOMProxyListEntry entry, Node sibling) {
+	Element child = entry.getElement();
+	if(sibling != null) {
+	    parent.insertBefore(child, sibling);
+	}
+	else {
+	    parent.appendChild(child);
+	}
+	cache.clear();
+	int index = indexOf(entry);
+	fireIntervalAdded(this, index, index);
+    }
+
+    /** Used to add an element into the underlying dom, and fire the appropriate repaint events. This version always adds the new element at the end of the children. */
+    public synchronized void append(DOMProxyListEntry entry) {
+	Element element = entry.getElement();
+	root.appendChild(element);
+	element = null;
+	// Regardless fire update event
+	cache.clear();
+	fireIntervalAdded(this, 0, 0);
+    }
+   
+    public synchronized ArrayList children() {
+	ArrayList child_list = new ArrayList();
+	int child_count = children.getLength();
+	for(int i = 0; i < child_count; i++) {
+	    child_list.add(getElementAt(i));
+	}
+	return child_list;
+    }
+
+    public synchronized boolean contains(Object entry) {
+	boolean found = false;
+	int size = getSize();
+	for(int i = 0; !found && i < size; i++) {
+	    DOMProxyListEntry sibling = (DOMProxyListEntry) getElementAt(i);
+	    if(sibling.equals(entry)) {
+		found = true;
+	    }
+	}
+	return found;
+    }
+
+    public synchronized Object getElementAt(int index)
+    {
+	Object object = cache.get(new Integer(index));
+	if (object != null) {
+	    return object;
+	}
+
+	// Retrieve the required element
+	Element element = (Element) children.item(index);
+	DebugStream.println("Element at index " + index + " not in cache: " + element);
+
+	// Now wrap it in the object of the users choice
+	object = class_type.create(element);
+	cache.put(new Integer(index), object);
+	return object;
+    }
+
+    public synchronized int indexOf(DOMProxyListEntry entry) {
+	Element element = entry.getElement();
+	int children_length = children.getLength();
+	for(int i = 0; i < children_length; i++) {
+	    Node node = children.item(i);
+	    if(element == node) {
+		return i;
+	    }
+	}
+	return -1;
+    }
+
+    public synchronized int getSize() {
+	if(children == null) {
+	    children = root.getElementsByTagName(tag_name);
+	}
+	return children.getLength();
+    }
+
+    public synchronized void refresh() {
+	fireContentsChanged(this, 0, getSize());
+    }
+
+    public synchronized void refresh(DOMProxyListEntry entry) {
+	int index = indexOf(entry);
+	fireContentsChanged(this, index, index);
+    }
+
+    public synchronized void remove(DOMProxyListEntry entry) {
+	remove(indexOf(entry));
+    } 
+
+    public synchronized void remove(int index) {
+	// Retrieve the required element
+	Node node = children.item(index);
+	// Find its parent
+	Node parent_node = node.getParentNode();
+	// Now remove it
+	parent_node.removeChild(node);
+	// Refresh model
+	cache.clear();
+	fireIntervalRemoved(this, index, index);
+    }
+
+    /** Changes the 'root' element that this list sources its information from. 
+     * @param  root the new root Element
+     */
+    public synchronized void setRoot(Element root) {
+	this.children = null;
+	cache.clear();
+	this.root = root;
+	this.children = this.root.getElementsByTagName(this.tag_name);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DesignChangeListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DesignChangeListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DesignChangeListener.java	(revision 21112)
@@ -0,0 +1,85 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Matthew Whyte, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.cdm;
+
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.DebugStream;
+import javax.swing.event.*;
+import java.awt.event.*;
+
+/**
+ * This handles events generated by changing design options.
+ * The rebuildTypeRequired is updated if the rebuild type associated with this
+ * listener requires more 'work' than what is currently set in the {@link CollectionDesignManager}.
+ * @author Matthew Whyte
+ */
+public class DesignChangeListener implements ActionListener, DocumentListener {
+    
+    /** The rebuild type required associated with this listener */
+    private final int rebuildType;
+    
+    /** Constructor. Sets the build type required. */
+    public DesignChangeListener(int type) {
+	rebuildType = type;
+    }
+
+    public void maybeSetRebuildRequired() {
+	if(rebuildType > CollectionDesignManager.getRebuildTypeRequired()) {
+	    CollectionDesignManager.setRebuildTypeRequired(rebuildType);
+	}
+	//We need to keep track of this seperately, as collect.cfg is uploaded if needed, when the collect.cfg is saved.
+	if(rebuildType == CollectionDesignManager.UPDATE_COLLECT_CFG) {
+	    CollectionDesignManager.update_collect_cfg_required = true;
+	}
+    }
+    
+    /** Gives notification that an event has happened */
+    public void actionPerformed(ActionEvent event) {
+	//DebugStream.println("ActionEvent in Design from: " + event.getSource());
+	maybeSetRebuildRequired();
+    }
+
+    /** Gives notification that an attribute or set of attributes changed. */
+    public void changedUpdate(DocumentEvent e) {
+	maybeSetRebuildRequired();
+	//DebugStream.println("changedUpdate in Design.");
+    }
+    
+    /** Gives notification that there was an insert into the document. */
+    public void insertUpdate(DocumentEvent e) {
+	maybeSetRebuildRequired();
+	//DebugStream.println("insertUpdate in Design."); //debug
+    }
+    
+    /** Gives notification that a portion of the document has been removed. */
+    public void removeUpdate(DocumentEvent e) {
+	maybeSetRebuildRequired();
+	//DebugStream.println("removeUpdate in Design."); //debug
+    }
+    
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DynamicListModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DynamicListModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/DynamicListModel.java	(revision 21112)
@@ -0,0 +1,119 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Title:        Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Company:      The University of Waikato
+ * Written:      20/05/02
+ * Revised:      03/10/02 - Commented
+ **************************************************************************************/
+import javax.swing.ComboBoxModel;
+import javax.swing.DefaultListModel;
+
+/** This class extends the functionality of the DefaultListModel to include simple methods for refreshing the contents of this model if one of its elements changes.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class DynamicListModel
+    extends DefaultListModel
+    implements ComboBoxModel {
+
+    private boolean auto_order = false;
+
+    private Object object = null;
+
+    public void addElement(Object element) {
+	if(auto_order) {
+	    // Insert the object in its alphabetically correct place.
+	    int position = 0;
+	    boolean found = false;
+	    while(!found && position < size()) {
+		///ystem.err.print("Compare " + element.toString() + " to " + get(position).toString() + ": ");
+		int order = element.toString().compareTo(get(position).toString());
+		if(order < 0) {
+		    add(position, element);
+		    found = true;
+		    ///ystem.err.println("Greater than. Insert");
+		}
+		else if(order == 0) {
+		    found = true;
+		    ///ystem.err.println("Equal. End.");
+		}
+		else {
+		    position++;
+		    ///ystem.err.println("Less than. Carry on.");
+		}
+	    }
+	    if(!found) {
+		super.addElement(element);
+		///ystem.err.println("Out of elements. Insert");
+	    }
+	}
+	else {
+	    super.addElement(element);
+	}
+    }
+
+    /* private DynamicListModel shallowCopy() {
+	DynamicListModel copy = new DynamicListModel();
+	copy.setAutoOrder(auto_order);
+	for(int i = 0; i < size(); i++) {
+	    copy.addElement(get(i));
+	}
+	return copy;
+	} */
+
+    public Object getSelectedItem() {
+	///ystem.err.println("Get item: " + object);
+	return object;
+    }
+    /** Notify all controls that are based on this list model that its contents have changed, and they should repaint themselves. */
+    public void refresh() {
+	fireContentsChanged(this, 0, size());
+    }
+
+    public void setAutoOrder(boolean auto_order) {
+	this.auto_order = auto_order;
+    }
+
+    public void setSelectedItem(Object object) {
+	///ystem.err.println("Set item: " + object);
+	this.object = object;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Format.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Format.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Format.java	(revision 21112)
@@ -0,0 +1,349 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class encapsulates all the information about a format command from the collection configuration. This includes both the 'general' formatting commands, and the commands targetted at specific classifiers or classification component types. This class is, unfortunately, the most complex in terms of DOM model support, since we have to maintain a live reference to any classifier this format applies to. We temporarily use the feature name as the format name attribute but since the order of classifiers can change, this of course gets horribly out of date. Before the CollectionDesignManager saves the CollectionConfiguration it calls a method in the FormatManager to force all of the Formats to update their name attributes.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Format 
+    implements DOMProxyListEntry {
+
+    /** The default features (not all of these are in the Greenstone Developer's Guide). */
+    static final public String DEFAULT_FEATURES[] = { "", "AllowExtendedOptions", "Document", "DocumentArrowsBottom", "DocumentArrowsTop", "DocumentButtons", "DocumentContents", "DocumentHeading", "DocumentImages", "DocumentText", "DocumentTitles", "DocumentUseHTML", "RelatedDocuments", "Search" };
+    /** The list of known feature parts. */
+    static final public String DEFAULT_PARTS[] = { "", "DateList", "HList", "VList" };
+
+    /** Only "" and "Search" can have parts */
+    static public boolean canHavePart(String name) {
+	return (name.equalsIgnoreCase("") || name.equalsIgnoreCase("Search") || name.equalsIgnoreCase("Document"));
+    }
+	
+    static public String generateName(Object feature, String part) {
+	if(feature instanceof Classifier) {
+	    return ((Classifier)feature).getPositionString() + part;
+	}
+	else {
+	    return feature.toString() + part;
+	}
+    }
+
+    /** Returns true if the parameter is a boolean type, false otherwise. */
+    static public boolean isParamType(String name)
+    {
+	if (name.equalsIgnoreCase("AllowExtendedOptions") || name.equalsIgnoreCase("DocumentArrowsBottom") || name.equalsIgnoreCase("DocumentArrowsTop") || name.equalsIgnoreCase("DocumentContents") || name.equalsIgnoreCase("DocumentImages") || name.equalsIgnoreCase("DocumentTitles") || name.equalsIgnoreCase("DocumentUseHTML")) {
+	    return true;
+	}
+	else {
+	    return false;
+	}
+    }
+
+    /** If this format is altering a Classifier then this is the classifier in question. */
+    private Classifier classifier = null;
+    /** The element this format is based on. */
+    private Element element = null;
+    /** We keep a copy of the part because its slightly more computationally tricky to calculate. */
+    private String part = null;
+    /** Cached result of toString. */
+    private String text = null;
+
+    /** Constructor only to be used during DOMProxyListModel initialization. */
+    public Format() {
+    }
+
+    public Format(Element element) {
+	this.element = element;
+	// Immediately check if we are dealing with a classifier, by retrieving the feature name. If it is a classifier, then restore the live reference immediately before anything changes it.
+	Object object = getFeature();
+
+	// There may be format statements for no-longer-existing classifiers
+	if (object == null) {
+	    DebugStream.println("No matching classifier for format statement!");
+	    return;
+	}
+
+	if(object instanceof Classifier) {
+	    classifier = (Classifier) object;
+	}
+	else {
+	    String feature = (String) object;
+	    if(feature.toUpperCase().startsWith(Classifier.CLASSIFIER_PREFIX)) {
+		// Extract the integer index
+		int index = Integer.parseInt(feature.substring(Classifier.CLASSIFIER_PREFIX.length()));
+		// Subtract one (as java is 0-2 where G2 is 1-3)
+		index = index - 1;
+		// Retrieve the appropriate classifier
+		classifier = CollectionDesignManager.classifier_manager.getClassifier(index);
+	    }
+	}
+    }
+
+    /** Constructor for a flag format command.
+     * @param feature The <Strong>Object</strong> this format affects.
+     * @param part The specific part of the control to format as a <strong>String</strong>.
+     * @param state A <i>boolean</i> indicating this formats state.
+     */
+    public Format(Object feature, String part, boolean state) {
+	element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.FORMAT_ELEMENT);
+	setName(feature, part);
+	setState(state);
+    }
+
+    /** Constructor for a format-string format command.
+     * @param feature The <Strong>Object</strong> this format affects.
+     * @param part The specific part of the control to format as a <strong>String</strong>.
+     * @param value The format <strong>String</strong> which may be a label name, or a piece of HTML formatting.
+     */
+    public Format(Object feature, String part, String value) {
+	element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.FORMAT_ELEMENT);
+	setName(feature, part);
+	setValue(value);
+    }
+
+    public boolean canHavePart() {
+	return canHavePart(getName());
+    }
+
+    public int compareTo(Object object) {
+	if(object == null) { // It seems that occasionally compareTo is called and somehow null ends up in comparison.
+	    return -1;
+	}
+	if(object instanceof Format && element != null) {
+	    return element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE).compareTo(((Format)object).getName());
+	}
+	return toString().compareTo(object.toString());
+    }
+
+    public DOMProxyListEntry create(Element element) {
+	return new Format(element);
+    }
+
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    /** Method to retrieve the value of feature, which may either be a String or a Classifier.
+     * @return The value of feature as an <strong>Object</strong>.
+     */
+    public Object getFeature() {
+	if(classifier != null) {
+	    return classifier;
+	}
+	else if(element != null) {
+	    String name = element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	    // Remove part
+	    String part = getPart();
+	    String feature;
+	    if(part != null) {
+		feature = name.substring(0, name.length() - part.length());
+	    }
+	    else {
+		feature = name;
+	    }
+	    part = null;
+	    name = null;
+	    // If the feature now refers to a classifier, retrieve it.
+	    if(feature.toUpperCase().startsWith(Classifier.CLASSIFIER_PREFIX)) {
+		// Extract the integer index
+		int index = Integer.parseInt(feature.substring(Classifier.CLASSIFIER_PREFIX.length()));
+		// Subtract one (as java is 0-2 where G2 is 1-3)
+		index = index - 1;
+		// Retrieve the appropriate classifier
+		classifier = CollectionDesignManager.classifier_manager.getClassifier(index);
+		return classifier;
+	    }
+	    else {
+		return feature;
+	    }
+	}
+	else {
+	    return "Error";
+	}
+    }
+
+    public String getName() {
+	if(element != null) {
+	    return element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	}
+	return "Error";
+    }
+
+    /** Method to retrieve the value of part. Of course there may not be one, in which case return ""
+     * @return The value of part as a <Strong>String</strong>.
+     */
+    public String getPart() {
+	if(part == null && element != null) {
+	    // To determine a part, we retrieve the Format name, then look for one of our recognized parts
+	    String name = element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	    // DEFAULT_PARTS[0] is an empty string to correctly enable comboboxes.
+	    name = name.toLowerCase();
+	    for(int i = 1; part == null && i < DEFAULT_PARTS.length; i++) {
+		if(name.endsWith(DEFAULT_PARTS[i].toLowerCase())) {
+		    part = DEFAULT_PARTS[i];
+		}
+	    }
+	    if(part == null) {
+		part = DEFAULT_PARTS[0];
+	    }
+	}
+	return part;
+    }
+
+    /** Method to retrieve the value of state.
+     * @return value of state as a <i>boolean</i>.
+     */
+    public boolean getState() {
+	return (element != null && element.getAttribute(CollectionConfiguration.VALUE_ATTRIBUTE).equals(CollectionConfiguration.TRUE_STR));
+    }
+
+    /** Retrieve the value of value.
+     * @return A <strong>String</strong> which is the value of value.
+     */
+    public String getValue() {
+	String value = "Error";
+	if(element != null) {
+	   value = XMLTools.getValue(element);
+	   // Decode into text
+	   value = Codec.transform(value, Codec.DOM_TO_TEXT);
+	}
+	return value;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public boolean isParamType() {
+	return (element != null && element.getAttribute(CollectionConfiguration.VALUE_ATTRIBUTE).length() > 0);
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	part = null;
+	text = null;
+	// Establish the classifier reference, if necessary
+	getFeature();
+    }
+
+    public void setName(Object feature, String part) {
+	// Can't do anything unless an element exists.
+	if(element != null) {
+	    this.part = part; // Easier for later on.
+	    if(feature instanceof Classifier) {
+		classifier = (Classifier) feature;
+		element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, classifier.getPositionString() + part);
+	    }
+	    else {
+		element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, feature.toString() + part);
+	    }
+	    text = null;
+	}
+    }
+
+    /** Set the value of state.
+     * @param state The new value for state as a <i>boolean</i>.
+     */
+    public void setState(boolean state) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.VALUE_ATTRIBUTE, (state ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	    text = null;
+	}
+    }
+
+    /** Set the value of value. Hmmm.
+     * @param value The new value from value as a <strong>String</strong>.
+     */
+    public void setValue(String value) {
+	if(element != null) {
+	    // Strip off any enclosing speech marks
+	    if(value.startsWith(CollectionConfiguration.SPEECH_CHARACTER) && value.startsWith(CollectionConfiguration.SPEECH_CHARACTER)) {
+		value = value.substring(1, value.length() - 1);
+	    }
+	    // Encode
+	    value = Codec.transform(value, Codec.TEXT_TO_DOM);
+	    // Store in DOM
+	    XMLTools.setValue(element, value);
+	    text = null;
+	}
+    }
+
+    /** Method to translate this classes information into a line of text as you would expect in the collection configuration file.
+     * @return A <strong>String</strong> containing the format command.
+     */
+    public String toString() {
+	if(text == null && element != null) {
+	    StringBuffer temp = new StringBuffer(CollectionConfiguration.FORMAT_STR);
+	    temp.append(" ");
+	    temp.append(element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE));
+	    temp.append(" ");
+	    String value = element.getAttribute(CollectionConfiguration.VALUE_ATTRIBUTE);
+	    if(value.length() > 0) {
+		temp.append(value);
+	    }
+	    else {
+		value = XMLTools.getValue(element);
+		if(value.equalsIgnoreCase(CollectionConfiguration.TRUE_STR) || value.equalsIgnoreCase(CollectionConfiguration.FALSE_STR)) {
+		    temp.append(value);
+		}
+		else {
+		    temp.append("\"");
+		    // Retrieve the DOM encoded value, decode and then append
+		    value = Codec.transform(value, Codec.DOM_TO_TEXT);
+		    temp.append(value);
+		    temp.append("\"");
+		}
+	    }
+	    text = temp.toString();
+	    temp = null;
+	}
+	return text;
+    }
+
+    public void update() {
+	if(classifier != null) {
+	    element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, classifier.getPositionString() + getPart());
+	    text = null;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/FormatManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/FormatManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/FormatManager.java	(revision 21112)
@@ -0,0 +1,1011 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+/** This class maintains a list of format statements, and allows the addition and removal of these statements.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class FormatManager
+    extends DOMProxyListModel {
+
+    static final private String BLANK = "blank";
+    static final private String FLAG  = "flag";
+    static final private String VALUE = "value";
+
+    static final private String DATELIST = "DateList";
+    static final private String DATELIST_DEFAULT_FORMAT = "<td>[link][icon][/link]</td>\n<td>[highlight]{Or}{[dls.Title],[dc.Title],[ex.Title],Untitled}[/highlight]</td>\n<td>[ex.Date]</td>";
+    static final private String HLIST = "HList";
+    static final private String HLIST_DEFAULT_FORMAT = "[link][highlight][ex.Title][/highlight][/link]";
+    static final private String VLIST = "VList";
+    static final private String VLIST_DEFAULT_FORMAT = "<td valign=top>[link][icon][/link]</td>\n<td valign=top>[ex.srclink]{Or}{[ex.thumbicon],[ex.srcicon]}[ex./srclink]</td>\n<td valign=top>[highlight]\n{Or}{[dls.Title],[dc.Title],[ex.Title],Untitled}\n[/highlight]{If}{[ex.Source],<br><i>([ex.Source])</i>}</td>";
+    //    static final private String INVISIBLE = "Invisible";
+    //static final private String INVISIBLE_DEFAULT_FORMAT = "";
+
+    static final private String DOCUMENTHEADING = "DocumentHeading";
+    static final private String DOCUMENTHEADING_DEFAULT_FORMAT = "{Or}{[parent(Top):Title],[Title],untitled}<br>";
+
+    static final private String DOCUMENTTEXT = "DocumentText";
+    static final private String DOCUMENTTEXT_DEFAULT_FORMAT = "<center><table width=_pagewidth_><tr><td>[Text]</td></tr></table></center>";
+
+    static final private String DOCUMENTBUTTONS = "DocumentButtons";
+    static final private String DOCUMENTBUTTONS_DEFAULT_FORMAT = "Detach|Highlight";
+    
+    /** The controls used to edit the format commands. */
+    private Control controls = null;
+    /** A reference to ourselves so inner classes can get at the model. */
+    private DOMProxyListModel model = null;
+
+    /** Constructor. */
+    public FormatManager() {
+	super(CollectionDesignManager.collect_config.getDocumentElement(), CollectionConfiguration.FORMAT_ELEMENT, new Format());
+	this.model = this;
+	DebugStream.println("FormatManager: parsed " + getSize() + " format statements.");
+	// Establish all of the format objects, so that classifier indexes are initially correct (subsequent refreshes of the model will be sufficient to keep these up to date, as long as we start with a live reference to a classifier.
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    getElementAt(i);
+	}
+
+	// Ensure the default formats for DateList, HList and VList are assigned
+	if (getFormat(DATELIST) == null) {
+	    addFormat(new Format("",DATELIST,DATELIST_DEFAULT_FORMAT));
+	}
+
+	if (getFormat(HLIST) == null) {
+	    addFormat(new Format("",HLIST,HLIST_DEFAULT_FORMAT));
+	}
+
+	if (getFormat(VLIST) == null) {
+	    addFormat(new Format("",VLIST,VLIST_DEFAULT_FORMAT));
+	}
+
+	if (getFormat(DOCUMENTHEADING) == null) {
+	    addFormat(new Format(DOCUMENTHEADING, "", DOCUMENTHEADING_DEFAULT_FORMAT));
+	}
+	if (getFormat(DOCUMENTTEXT) == null) {
+	    addFormat(new Format(DOCUMENTTEXT,"",DOCUMENTTEXT_DEFAULT_FORMAT));
+	}
+	if (getFormat(DOCUMENTBUTTONS) == null) {
+	    addFormat(new Format(DOCUMENTBUTTONS,"",DOCUMENTBUTTONS_DEFAULT_FORMAT));
+	}
+	
+    }
+
+    /** Method to add a new format to this manager.
+     * @param format The <strong>Format</strong> to add.
+     */
+    private void addFormat(Format format) {
+	if(!contains(format)) {
+	    Element element = format.getElement();
+	    // Locate where we should insert this new classifier. 
+	    Node target_node = CollectionConfiguration.findInsertionPoint(element);
+	    add(root, format, target_node);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+    }
+
+    /** Gets the format indicated by the index.
+     * @param index The location of the desired format, as an <i>int</i>.
+     */
+    private Format getFormat(int index) {
+	Format result = null;
+	if(0 < index && index < getSize()) {
+	    result = (Format) getElementAt(index);
+	}
+	return result;
+    }
+
+    private Format getFormat(String name) {
+	int model_size = getSize();
+	for(int index = 0; index < model_size; index++) {
+	    Format format = (Format) getElementAt(index);
+	    if(format.getName().equals(name)) {
+		return format;
+	    }
+	}
+	return null;
+    }
+    
+    /** Method to retrieve this managers controls.
+     * @return the Control for this collection.
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    controls = new FormatControl();
+	}
+	return controls;
+    }
+
+    /** updates the format model */
+    public synchronized void refresh() {
+	for(int i = 0; i < getSize(); i++) {
+	    Format format = (Format) getElementAt(i);
+	    format.update();
+	    format = null;
+	}
+	super.refresh();
+    }
+    /** Method to remove a format.
+     * @param format The <strong>Format</strong> to remove.
+     */
+    private void removeFormat(Format format) {
+	remove(format);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    private HashMap buildDefaultMappings(ArrayList features_model, ArrayList parts_model) {
+	DebugStream.println("buildDefaultMappings(): replace me with something that reads in a data xml file.");
+	return new HashMap();
+    }
+
+    private ArrayList buildFeatureModel() {
+	// Rebuild feature model.
+	ArrayList feature_model = new ArrayList();
+	// Add the set options
+	for(int i = 0; i < Format.DEFAULT_FEATURES.length; i++) {
+	    feature_model.add(new Entry(Format.DEFAULT_FEATURES[i]));
+	}
+	// Now the classifiers.
+	for(int j = 0; j < CollectionDesignManager.classifier_manager.getSize(); j++) {
+	    feature_model.add(new Entry(CollectionDesignManager.classifier_manager.getClassifier(j)));
+	}
+	Collections.sort(feature_model);
+	return feature_model;
+    }
+
+    private ArrayList buildPartModel() {
+	DebugStream.println("buildPartModel(): replace me with something that reads in a data xml file.");
+	ArrayList part_model = new ArrayList();
+	part_model.add(new Part("", ""));
+	part_model.add(new Part(DATELIST, DATELIST_DEFAULT_FORMAT));
+	part_model.add(new Part(HLIST, HLIST_DEFAULT_FORMAT));
+	//part_model.add(new Part(INVISIBLE, INVISIBLE_DEFAULT_FORMAT));
+	part_model.add(new Part(VLIST, VLIST_DEFAULT_FORMAT));
+	return part_model;
+    }
+
+    private ArrayList buildVariableModel() {
+	DebugStream.println("buildVariableModel(): replace me with something that reads in a data xml file.");
+	ArrayList variable_model = new ArrayList();
+	variable_model.add("[Text]");
+	ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+	for (int i = 0; i < every_metadata_set_element.size(); i++) {
+	    variable_model.add("[" + ((MetadataElement) every_metadata_set_element.get(i)).getFullName() + "]");
+	}
+	variable_model.add("[link]");
+	variable_model.add("[/link]");
+	variable_model.add("[icon]");
+	variable_model.add("[numleafdocs]");
+	variable_model.add("[num]");
+	variable_model.add("[parent():_]");
+	variable_model.add("[parent(Top):_]");
+	variable_model.add("[parent(All'_'):_]");
+	return variable_model;
+    }
+
+    private class FormatControl
+	extends JPanel 
+	implements Control {
+	
+	private ArrayList feature_model;
+	private ArrayList part_model;
+	private ArrayList variable_model;
+	private boolean ignore_event = false;
+	private boolean ready = false; // Are these controls available to be refreshed
+	private CardLayout card_layout;
+	private HashMap default_mappings;
+	private JButton add_button;
+	private JButton insert_button;
+	private JButton remove_button;
+	private JButton replace_button;
+	private JCheckBox enabled_checkbox;
+	private JComboBox feature_combobox;
+	private JComboBox part_combobox;
+	private JComboBox variable_combobox;
+	private JList format_list;
+	private JTextArea instructions_textarea;
+	private JTextArea editor_textarea;
+	private JPanel blank_pane;
+	private JPanel control_pane;
+	private JPanel part_pane;
+	private JPanel selection_pane;
+	private String view_type;
+
+	private final Dimension FIELD_SIZE = new Dimension(200, 30);
+	public FormatControl() {
+	    feature_model = buildFeatureModel();
+	    part_model = buildPartModel();
+	    variable_model = buildVariableModel();
+	    default_mappings = buildDefaultMappings(feature_model, part_model);
+
+	    // Create
+	    JPanel instructions_pane = new JPanel();
+	    JLabel title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    title_label.setOpaque(true);
+	    Dictionary.registerText(title_label, "CDM.FormatManager.Title");
+
+	    instructions_textarea = new JTextArea();
+	    instructions_textarea.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    instructions_textarea.setEditable(false);
+	    instructions_textarea.setLineWrap(true);
+	    instructions_textarea.setRows(6);
+	    instructions_textarea.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions_textarea, "CDM.FormatManager.Instructions");
+
+	    JLabel format_label = new JLabel();
+	    Dictionary.registerText(format_label, "CDM.FormatManager.Assigned_Formats");
+	    format_list = new JList(model);
+
+	    selection_pane = new JPanel();
+	    JPanel feature_pane = new JPanel();
+	    JLabel feature_label = new JLabel();
+	    Dictionary.registerText(feature_label, "CDM.FormatManager.Feature");
+	    feature_combobox = new JComboBox(feature_model.toArray());
+	    feature_combobox.setPreferredSize(FIELD_SIZE);
+	    feature_combobox.setEditable(false);
+	    Dictionary.registerTooltip(feature_combobox, "CDM.FormatManager.Feature_Tooltip");
+
+	    part_pane = new JPanel();
+	    JLabel part_label = new JLabel();
+	    Dictionary.registerText(part_label, "CDM.FormatManager.Part");
+
+	    part_combobox = new JComboBox(part_model.toArray());
+	    part_combobox.setPreferredSize(FIELD_SIZE);
+	    part_combobox.setEditable(false);
+	    Dictionary.registerTooltip(part_combobox, "CDM.FormatManager.Part_Tooltip");
+
+	    blank_pane = new JPanel();
+	    
+	    JPanel center_pane = new JPanel();
+
+	    card_layout = new CardLayout();
+	    control_pane = new JPanel();
+
+	    JPanel blank_pane = new JPanel();
+
+	    JPanel editor_pane = new JPanel();
+	    JPanel editor_header_pane = new JPanel();
+	    JLabel editor_label = new JLabel();
+	    Dictionary.registerText(editor_label, "CDM.FormatManager.Editor");
+
+	    editor_textarea = new JTextArea();
+	    editor_textarea.setBackground(Configuration.getColor("coloring.editable_background", false));
+	    editor_textarea.setCaretPosition(0);
+	    editor_textarea.setLineWrap(true);
+	    editor_textarea.setRows(6);
+	    editor_textarea.setWrapStyleWord(false);
+	    Dictionary.registerTooltip(editor_textarea, "CDM.FormatManager.Editor_Tooltip");
+
+	    JPanel variable_pane = new JPanel();
+	    JLabel variable_label = new JLabel();
+	    Dictionary.registerText(variable_label, "CDM.FormatManager.Variable");
+	    variable_combobox = new JComboBox(variable_model.toArray());
+	    Dictionary.registerTooltip(variable_combobox, "CDM.FormatManager.Variable_Tooltip");
+
+	    insert_button = new GLIButton();
+	    insert_button.setMnemonic(KeyEvent.VK_I);
+	    Dictionary.registerBoth(insert_button, "CDM.FormatManager.Insert", "CDM.FormatManager.Insert_Tooltip");
+
+	    JPanel flag_pane = new JPanel();
+	    enabled_checkbox = new JCheckBox();
+	    Dictionary.registerText(enabled_checkbox, "CDM.FormatManager.Enabled");
+
+	    JPanel button_pane = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(false);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.FormatManager.Add", "CDM.FormatManager.Add_Tooltip");
+	    replace_button = new GLIButton();
+	    replace_button.setEnabled(false);
+	    replace_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerBoth(replace_button, "CDM.FormatManager.Replace", "CDM.FormatManager.Replace_Tooltip");
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.FormatManager.Remove", "CDM.FormatManager.Remove_Tooltip");
+
+	    // Connect
+	    add_button.addActionListener(new AddListener());
+	    add_button.addActionListener(CollectionDesignManager.collect_cfg_change_listener);
+	    insert_button.addActionListener(new InsertListener());
+	    insert_button.addActionListener(CollectionDesignManager.collect_cfg_change_listener);
+	    remove_button.addActionListener(new RemoveListener());
+	    remove_button.addActionListener(CollectionDesignManager.collect_cfg_change_listener);
+	    replace_button.addActionListener(new ReplaceListener());
+	    replace_button.addActionListener(CollectionDesignManager.collect_cfg_change_listener);
+	    enabled_checkbox.addActionListener(new EnabledListener());
+	    enabled_checkbox.addActionListener(CollectionDesignManager.collect_cfg_change_listener);
+	    feature_combobox.addActionListener(new FeatureListener());
+	    part_combobox.addActionListener(new PartListener());
+	    editor_textarea.getDocument().addDocumentListener(new EditorListener());
+	    format_list.addListSelectionListener(new FormatListListener());
+
+	    // Layout
+	    instructions_pane.setLayout(new BorderLayout());
+	    instructions_pane.add(title_label, BorderLayout.NORTH);
+	    instructions_pane.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+	    instructions_pane.add(format_label, BorderLayout.SOUTH);
+
+	    feature_pane.setLayout(new BorderLayout(5,0));
+	    feature_pane.add(feature_label, BorderLayout.WEST);
+	    feature_pane.add(feature_combobox, BorderLayout.CENTER);
+
+	    part_pane.setLayout(new BorderLayout(5, 0));
+	    part_pane.add(part_label, BorderLayout.WEST);
+	    part_pane.add(part_combobox, BorderLayout.CENTER);
+
+	    //selection_pane.setLayout(new GridLayout(2,1,0,2));
+	    //selection_pane.add(feature_pane);
+	    //selection_pane.add(part_pane);
+
+	    flag_pane.add(enabled_checkbox);
+
+	    editor_header_pane.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	    editor_header_pane.setLayout(new GridLayout(1,3));
+	    editor_header_pane.add(editor_label);
+
+	    variable_pane.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	    variable_pane.setLayout(new GridLayout(1,3));
+	    variable_pane.add(variable_label);
+	    variable_pane.add(variable_combobox);
+	    variable_pane.add(insert_button);
+
+	    editor_pane.setLayout(new BorderLayout());
+	    editor_pane.add(editor_header_pane, BorderLayout.NORTH);
+	    editor_pane.add(new JScrollPane(editor_textarea), BorderLayout.CENTER);
+	    //editor_pane.add(variable_pane, BorderLayout.SOUTH);
+
+	    selection_pane.setLayout(new BorderLayout());
+	    selection_pane.add(part_pane, BorderLayout.NORTH);
+	    selection_pane.add(editor_pane, BorderLayout.CENTER);
+	    selection_pane.add(variable_pane, BorderLayout.SOUTH);
+
+	    control_pane.setLayout(card_layout);
+	    control_pane.add(flag_pane, FLAG);
+	    control_pane.add(selection_pane, VALUE);
+	    control_pane.add(blank_pane, BLANK);
+
+	    button_pane.setLayout(new GridLayout(1,3));
+	    button_pane.add(add_button);
+	    button_pane.add(replace_button);
+	    button_pane.add(remove_button);
+
+	    center_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("CDM.FormatManager.Editing_Controls")), BorderFactory.createEmptyBorder(2,2,2,2)));
+	    center_pane.setLayout(new BorderLayout());
+	    center_pane.add(feature_pane, BorderLayout.NORTH);
+	    center_pane.add(control_pane, BorderLayout.CENTER);
+	    center_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(instructions_pane, BorderLayout.NORTH);
+	    add(new JScrollPane(format_list), BorderLayout.CENTER);
+	    add(center_pane, BorderLayout.SOUTH);
+	    ready = true;
+	}
+
+	public void destroy() {
+	}
+       
+	/** Overriden to ensure that the instructions pane is scrolled to the top.
+	 */
+	public void gainFocus() {
+	    // This is only necessary if the components have been realized
+	    if(ready) {
+		model.refresh();
+		feature_model = buildFeatureModel();
+		// Remember the current selection
+		Object selected_object = feature_combobox.getSelectedItem();
+		feature_combobox.setModel(new DefaultComboBoxModel(feature_model.toArray()));
+		// Now restore the selected object as best as possible
+		feature_combobox.setSelectedItem(selected_object);
+		selected_object = null;
+		if(instructions_textarea != null) {
+		    instructions_textarea.setCaretPosition(0);
+		}
+	    }   
+	}
+
+	public void loseFocus() {
+	    // Force all of the Formats to update their names with the correct values.
+	    // do we need to do this on loseFocus???
+	    model.refresh();
+	}
+
+	/** Listens for clicks on the add button, and if the relevant details are provided adds a new format. Note that formats are responsible for codecing the values into something that can be a) stored in a DOM and b) written to file */
+	private class AddListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event)
+	    {
+		ignore_event = true; // Prevent format_list excetera propagating events
+
+		Entry entry = (Entry) feature_combobox.getSelectedItem();
+		Object f = entry.getFeature();
+		String p = "";
+		if (entry.canHavePart()) {
+		    p = part_combobox.getSelectedItem().toString();
+		}
+
+		// Add a new format string of the appropriate type
+		Format format = null;
+		if (view_type.equals(FLAG)) {
+		    format = new Format(f, p, enabled_checkbox.isSelected());
+		}
+		else {
+		    format = new Format(f, p, editor_textarea.getText());
+		}
+		addFormat(format);
+
+		// Save the collection configuration file immediately
+		Gatherer.g_man.design_pane.saveConfiguration();
+
+		add_button.setEnabled(false);
+		replace_button.setEnabled(false);
+		remove_button.setEnabled(true);
+
+		// Update list selection
+		format_list.setSelectedValue(format, true);
+		format = null;
+		p = null;
+		f = null;
+		entry = null;
+		ignore_event = false;
+	    }
+	}
+    
+	private class EditorListener
+	    implements DocumentListener {
+ 
+	    public void changedUpdate(DocumentEvent e) {
+		update();
+	    }
+          
+	    public void insertUpdate(DocumentEvent e) {
+		update();
+	    }
+          
+	    public void removeUpdate(DocumentEvent e) {
+		update();
+	    }
+
+	    public void update() {
+		// Determine if replace should be enabled
+		if(!format_list.isSelectionEmpty()) {
+		    Format format = (Format)format_list.getSelectedValue();
+		    replace_button.setEnabled(!format.isParamType() && editor_textarea.getText() != format.getValue());
+		}
+		else {
+		    replace_button.setEnabled(false);
+		}
+	    }
+	}
+
+	private class EnabledListener
+	    implements ActionListener {
+	    
+	    public void actionPerformed(ActionEvent event) {
+		// If there is a current format selected, and the value of enable_checkbox is now different than to value in it, then enable to replace button.
+		if(!format_list.isSelectionEmpty()) {
+		    Format format = (Format)format_list.getSelectedValue();
+		    replace_button.setEnabled(format.isParamType() && enabled_checkbox.isSelected() != format.getState());
+		}
+		// Thats it. Add would have been enabled upon feature/part selection depending if no existing format, um, existed.
+	    }
+	}
+
+	private class FeatureListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		if(!ignore_event) {
+		    ignore_event = true;
+		    Entry entry = (Entry) feature_combobox.getSelectedItem();
+		    // Step one: reset part
+		    if (entry.canHavePart()) {
+			part_combobox.setEnabled(true);
+			part_combobox.setSelectedIndex(part_combobox.getModel().getSize()-1);
+
+		    } else {
+			part_combobox.setEnabled(false);
+			part_combobox.setSelectedIndex(0);
+		    }
+		    // Step two: the rest
+		    
+		    String name = entry.toString();
+		    // Add is only enabled if there isn't already a format for the choosen feature and part. Create a dummy format and test if itsa already in the model
+		    Object f = entry.getFeature();
+		    Part part = (Part)part_combobox.getSelectedItem();
+		    String pname = part.getName();
+		    // You can never add anything to blank-blank
+		    if(f.toString().length() == 0 && pname.length() == 0) {
+			add_button.setEnabled(false);
+			replace_button.setEnabled(false);
+			remove_button.setEnabled(false);
+		    }
+		    else {
+		       Format format = getFormat(Format.generateName(f, pname));
+		       // If there is an existing feature, select it, and use it to determine what controls are visible
+		       if(format != null) {
+			  ///ystem.err.println("There is an existing format!");
+			    format_list.setSelectedValue(format, true);
+			    // Now use type to determine what controls are visible, and what have initial values.
+			    if(format.isParamType()) {
+			       ///ystem.err.println("Flag type");
+				// Flags first.
+				///election_pane.remove(part_pane);
+				card_layout.show(control_pane, FLAG);
+				view_type = FLAG;
+				// Initial value
+				enabled_checkbox.setSelected(format.getState());
+			    }
+			    else {
+				///ystem.err.println("Value type");
+				///election_pane.add(part_pane);
+				card_layout.show(control_pane, VALUE);
+				view_type = VALUE;
+				// Initial value
+				editor_textarea.setText(format.getValue());
+			    }
+			    control_pane.updateUI();
+			    remove_button.setEnabled(true);
+		       }
+		       // Otherwise there is no existing format, so we proceed by checking against the feature name
+			else {
+			   ///ystem.err.println("No existing format");
+			    format_list.clearSelection();
+			    if(Format.isParamType(name)) {
+			       ///ystem.err.println("Flag type");
+				// Flags first.
+				///election_pane.remove(part_pane);
+				card_layout.show(control_pane, FLAG);
+				view_type = FLAG;
+				// Initial value
+				enabled_checkbox.setSelected(false);
+			    }
+			    else {
+			       ///ystem.err.println("Value type");
+				///election_pane.add(part_pane);
+				card_layout.show(control_pane, VALUE);
+				view_type = VALUE;
+				// Initial value
+				editor_textarea.setText(part.getDefaultFormat());
+			    }
+			    add_button.setEnabled(true);
+			}
+			format = null;
+			name = null;
+		    }
+		    part = null;
+		    pname = null;
+		    f = null;
+		    replace_button.setEnabled(false);
+		    name = null;
+		    entry = null;
+		    ignore_event = false;
+		}
+	    }
+	}
+
+	private class FormatListListener
+	    implements ListSelectionListener {
+	    public void valueChanged(ListSelectionEvent event) {
+		if(!ignore_event && !event.getValueIsAdjusting()) {
+		    if(!format_list.isSelectionEmpty()) {
+			ignore_event = true;
+			Format format = (Format)format_list.getSelectedValue();
+			// Try to match the target, remembering the entries within are Entry's
+			Entry an_entry = new Entry(format.getFeature());
+			feature_combobox.setSelectedItem(an_entry);
+			// If we didn't match anything, add it and select it again
+			Entry result_entry = (Entry) feature_combobox.getSelectedItem();
+			if(!an_entry.equals(result_entry)) {
+			    feature_combobox.insertItemAt(an_entry, feature_combobox.getItemCount());
+			    feature_combobox.setSelectedItem(an_entry);
+			}
+
+			// Now use type to determine what controls are visible, and what have initial values.
+			///ystem.err.println("Determine the correct type.");
+			if(format.isParamType()) {
+			    ///ystem.err.println("Flag type - remove part");
+			    // Flags first.
+			    ///SwingUtilities.invokeLater(new GUIChangeTask(/election_pane, part_pane, blank_pane, false));
+			   card_layout.show(control_pane, FLAG);
+			   view_type = FLAG;
+			   // Initial value
+			   enabled_checkbox.setSelected(format.getState());
+			}
+			else {
+			   ///ystem.err.println("Value type - show part");
+			    ///SwingUtilities.invokeLater(new GUIChangeTask(/election_pane, part_pane, blank_pane, true));
+			   card_layout.show(control_pane, VALUE);
+			   view_type = VALUE;
+			   // Initial values
+			   
+			   // Try to match the part.
+			   String part_entry = format.getPart();
+			   // Set Selected Item doesn't work so I'll do this manually
+			   boolean found = false;
+			   for(int i=0; i < part_combobox.getItemCount(); i++) {
+			      Part a_part = (Part) part_combobox.getItemAt(i);
+			      if(a_part.equals(part_entry)) {
+				 part_combobox.setSelectedItem(a_part);
+				 found = true;
+			      }
+			      a_part = null;
+			   }
+			   // If we didn't match anything, add it and select it again
+			   if(!found) {
+			      Part a_part = new Part(part_entry, "");
+			      part_combobox.insertItemAt(a_part, part_combobox.getItemCount());
+			      part_combobox.setSelectedItem(a_part);
+			   }
+			   
+			   editor_textarea.setText(format.getValue());
+			}
+			//control_pane.updateUI();
+			remove_button.setEnabled(true);
+			ignore_event = false;
+		    }
+		    else {
+			remove_button.setEnabled(false);
+		    }
+		    add_button.setEnabled(false);
+		    replace_button.setEnabled(false);
+		}
+	    }
+	}
+
+	private class InsertListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		editor_textarea.insert((String)variable_combobox.getSelectedItem(), editor_textarea.getCaretPosition());
+	    }
+	}
+       
+	private class PartListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		if(!ignore_event) {
+		    // Add is only enabled if there isn't already a format for the choosen feature and part. Create a dummy format and test if its already in the model
+		    Entry entry = (Entry) feature_combobox.getSelectedItem();
+		    Object f = entry.getFeature();
+		    Part part = (Part) part_combobox.getSelectedItem();
+		    String pname = part.getName();
+		    // You can never add anything to blank-blank
+		    if(f.toString().length() == 0 && pname.length() == 0) {
+			add_button.setEnabled(false);
+			replace_button.setEnabled(false);
+			remove_button.setEnabled(false);
+		    }
+		    else {
+			String name = Format.generateName(f, pname);
+			Format format = getFormat(name);
+			// If there is an existing feature, select it.
+			if(format != null) {
+			    format_list.setSelectedValue(format, true);
+			    // Now use type to determine what controls are visible, and what have initial values.
+			    if(format.isParamType()) {
+				// Flags first.
+				///election_pane.remove(part_pane);
+				card_layout.show(control_pane, FLAG);
+				view_type = FLAG;
+				// Initial value
+				enabled_checkbox.setSelected(format.getState());
+			    }
+			    else {
+				///election_pane.add(part_pane);
+				card_layout.show(control_pane, VALUE);
+				view_type = VALUE;
+				// Initial value
+				editor_textarea.setText(format.getValue());
+			    }
+			    control_pane.updateUI();
+			    remove_button.setEnabled(true);
+			}
+			else {
+			    format_list.clearSelection();
+			    if(Format.isParamType(name)) {
+				// Flags first.
+				///election_pane.remove(part_pane);
+				card_layout.show(control_pane, FLAG);
+				view_type = FLAG;
+				// Initial value
+				enabled_checkbox.setSelected(false);
+			    }
+			    else {
+				///election_pane.add(part_pane);
+				card_layout.show(control_pane, VALUE);
+				view_type = VALUE;
+				// Initial value
+				editor_textarea.setText(part.getDefaultFormat());
+			    }
+			    add_button.setEnabled(true);
+			}
+			format = null;
+			name = null;
+		    }
+		    
+		    pname = null;
+		    part = null;
+		    f = null;
+		    entry = null;
+		    replace_button.setEnabled(false);
+		}
+	    }
+	}
+
+	private class RemoveListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event)
+	    {
+		if (!format_list.isSelectionEmpty()) {
+		    // Remove the current format
+		    removeFormat((Format)format_list.getSelectedValue());
+
+		    // Save the collection configuration file immediately
+		    Gatherer.g_man.design_pane.saveConfiguration();
+
+		    // Change buttons
+		    add_button.setEnabled(true);
+		    replace_button.setEnabled(false);
+		}
+
+		remove_button.setEnabled(false);
+	    }
+	}
+
+	private class ReplaceListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event)
+	    {
+		ignore_event = true; // Prevent format_list excetera propagating events
+
+		if (!format_list.isSelectionEmpty()) {
+		    // Remove the current format
+		    removeFormat((Format) format_list.getSelectedValue());
+
+		    Entry entry = (Entry) feature_combobox.getSelectedItem();
+		    Object f = entry.getFeature();
+		    String p = "";
+		    if (entry.canHavePart()) {
+			p = part_combobox.getSelectedItem().toString();
+		    }
+
+		    // Add a new format string of the appropriate type
+		    Format format = null;
+		    if (view_type.equals(FLAG)) {
+			format = new Format(f, p, enabled_checkbox.isSelected());
+		    }
+		    else {
+			format = new Format(f, p, editor_textarea.getText());
+		    }
+		    addFormat(format);
+
+		    // Save the collection configuration file immediately
+		    Gatherer.g_man.design_pane.saveConfiguration();
+
+		    add_button.setEnabled(false);
+		    remove_button.setEnabled(true);
+		    // Update list selection
+		    format_list.setSelectedValue(format, true);
+		    format = null;
+		    p = null;
+		    f = null;
+		    entry = null;
+		}
+
+		replace_button.setEnabled(false);
+		ignore_event = false;
+	    }
+	}
+    }
+
+    /** This object provides a wrapping around an entry in the format target control, which is tranparent as to whether it is backed by a String or a Classifier. */
+    private class Entry
+	implements Comparable {
+	private Classifier classifier = null;
+	private String text = null;
+	
+	public Entry(Object object) {
+	    if(object instanceof Classifier) {
+		classifier = (Classifier)object;
+	    }
+	    else if(object instanceof String) {
+		text = (String)object;
+	    }
+	    else {
+		text = "";
+	    }
+	}
+	
+	public Entry(String text) {
+	    this.text = text;
+	}
+	
+	public boolean canHavePart() {
+	    if (classifier !=null) return true;
+	    return Format.canHavePart(text);
+	}
+
+	public int compareTo(Object object) {
+	    if(object == null) {
+		return 1;
+	    }
+	    if(toString() == null) {
+		return -1;
+	    }
+	    else {
+		String object_str = object.toString();
+		if(object_str == null) {
+		    return 1;
+		}
+		return toString().compareTo(object_str);
+	    }
+	}
+	
+	public boolean equals(Object object) {
+	    if(compareTo(object) == 0) {
+		return true;
+	    }
+	    return false;
+	}
+	
+	public Classifier getClassifier() {
+	    return classifier;
+	}
+	
+	public Object getFeature() {
+	    if(classifier != null) {
+		return classifier;
+	    }
+	    return text;
+	}
+	
+	public String toString() {
+	    if(classifier != null) {
+		// Return the classifier - less the 'classify ' prefix and with its CL index shown
+		return classifier.getPositionString() + StaticStrings.COLON_CHARACTER + StaticStrings.SPACE_CHARACTER + classifier.toString().substring(9);
+	    }
+	    return text;
+	}
+    }
+
+    /*
+      private class GUIChangeTask
+      implements Runnable {
+      private boolean to_add;
+      private JPanel child;
+      private JPanel parent;
+      private JPanel replacement;
+      
+      public GUIChangeTask(JPanel parent, JPanel child, JPanel replacement, boolean to_add) {
+      this.child = child;
+      this.parent = parent;
+      this.replacement = replacement;
+      this.to_add = to_add;
+      }
+      
+      public void run() {
+      if(to_add) {
+      parent.remove(replacement);
+      parent.add(child);
+      parent.updateUI();
+      }
+      else {
+      parent.remove(child);
+      parent.add(replacement);
+      parent.updateUI();
+      }
+      }
+      }
+    */
+
+   /** This class encapsulates all of the information associated with a certain component part of a feature within a html page returned from the receptioninst. */
+   private class Part
+      implements Comparable {
+      /** The default format string for this part */
+      private String default_format = null;
+      /** The name of this part */
+      private String name = null;
+      /** Constructor - must be provided with all of the details of a part as there are no other setter methods.
+       * @param name the name of this part
+       * @param default_format the default format string for this part
+       */
+      public Part(String name, String default_format) {
+	 this.default_format = default_format;
+	 this.name = name;
+      }
+      /** Compare this part to another object in terms of ordering
+       * @param obj the other Object
+       * @return <0 if the object is before, 0 if equal to, and >0 if the object is after this part
+       */
+      public int compareTo(Object obj) {
+	 return name.compareTo(obj.toString());
+      }
+      
+      /** Determine if the part is equivelent to some other object
+       * @param obj the other Object
+       * @return true is the two objects are equal
+       */
+      public boolean equals(Object obj) {
+	 return name.equals(obj.toString());
+      }
+      
+      /** Retrieve the default format string for this part
+       * @return the default format String
+       */
+      public String getDefaultFormat() {
+	 // Retrieve the format for the super format - either VList or HList
+	 Format default_format_object = getFormat(name);
+	 if(default_format_object != null) {
+	    return default_format_object.getValue();
+	 }
+	 else {
+	    return this.default_format;
+	 }
+      }
+      /** Retrieve the name of this part
+       * @return the name as a String
+       */
+      public String getName() {
+	 return name;
+      }
+      /** Produce a string representation of this part, which in this case is simply the name again
+       * @return the name as a String
+       */
+      public String toString() {
+	 return name;
+      }
+   }  
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/GeneralManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/GeneralManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/GeneralManager.java	(revision 21112)
@@ -0,0 +1,696 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.regex.*;
+import javax.swing.*;
+import javax.swing.filechooser.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.EmailField;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.NewCollectionDetailsPrompt;
+import org.greenstone.gatherer.gui.OpenCollectionDialog;
+import org.greenstone.gatherer.gui.WarningDialog;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.StaticStrings;
+
+/** This class is responsible for generating the necessary GUI components. It does this by calling the getEditControls() method of the appropriate class (i.e. IndexManager for index related edits). It also is in charge of correctly adding (and removing) listeners, using phrases from the <strong>Dictionary</strong> rather than the text keys inserted in the component classes, and several other aspects of the design page, including the config file section tree.
+* @author John Thompson, Greenstone Digital Library, University of Waikato
+* @version 2.3d
+*/
+public class GeneralManager
+    extends JPanel {
+    /** The available subscreens. */
+    static final private String CONTENTS[] = { "CDM.GUI.General", "CDM.GUI.Plugins", "CDM.GUI.SearchTypes", "CDM.GUI.Indexes", "CDM.GUI.Subcollections", "CDM.GUI.SuperCollection", "CDM.GUI.Classifiers", "CDM.GUI.Formats", "CDM.GUI.Translation",  "CDM.GUI.MetadataSets" };
+    /** The preferred size of the contents tree. */
+    static final private Dimension TREE_SIZE = new Dimension(200, 500);
+    /** The controls used to modify the general options. */
+    private Control controls;
+    /** The panel apon which is rendered the currently selected section screen. */
+    private Control view = null;
+    /** A tree to serve as a 'table of contents' for this design tool. We decided on a tree rather than a list, as it allows us to break sections into subsections if they become to complicated. */
+    private DesignTree tree;
+    /** Constructor. */
+    public GeneralManager() {
+	super();
+	DebugStream.println("GeneralManager: Main GUI components created.");
+	// Assignments
+	this.controls = new GeneralControl();
+
+	// Creation
+	JPanel tree_pane = new JPanel();
+	JLabel title = new JLabel();
+	Dictionary.registerText(title, "CDM.GUI.Design_Topics");
+	tree = new DesignTree();
+	view = controls;
+
+	// Connect
+	tree.addTreeSelectionListener(new TreeListener());
+
+	// Layout
+	tree_pane.setLayout(new BorderLayout());
+	tree_pane.setPreferredSize(TREE_SIZE);
+	tree_pane.add(title, BorderLayout.NORTH);
+	tree_pane.add(new JScrollPane(tree), BorderLayout.CENTER);
+
+	setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+	setLayout(new BorderLayout());
+	add(tree_pane, BorderLayout.WEST);
+	add((JPanel)view, BorderLayout.CENTER);
+    }
+
+    public boolean canSave() {
+	return (controls == null || ((GeneralControl)controls).canSave());
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	controls.destroy();
+	controls = null;
+	tree = null;
+	view = null;
+    }
+
+    /** Called whenever the detail mode changes to ensure the contents tree is at an appropriate level (ie with Partitions disabled in lower levels)
+     * Written 1543-07-01-2004, and made obsolete by a new list of requirements 1611-07-01-2004. *sigh* 
+     * @param mode the mode level as an int
+     */
+    /* public void modeChanged(int mode) {
+	tree.resetModel(mode);
+    } */
+
+    /** Force the display to show a certain pane of controls.
+     * @param type a String giving the name of the information manager or view we wish to display
+     */
+    public void setSelectedView(String type) {
+	tree.setSelectedView(type);
+    }
+
+    /** Refresh the values on our controls that could be stale due to changes in other components. */
+    public void gainFocus() {
+	if (view != null) {
+	    view.gainFocus();
+	}
+    }
+
+    /** Saves the state of the controls in the current view. */
+   public void loseFocus() {
+      if (view != null) {
+	 view.loseFocus();
+      }
+    }
+
+    /** This class is resposible for generating the controls for the editing of general options.
+     * @return the Control for editing the general options
+     */
+    private Control getControls() {
+	return controls;
+    }
+
+    /** This class represents the visual component of the general options stored in the CollectionDesignManager. */
+    private class GeneralControl
+	extends JPanel
+	implements Control {
+
+	private boolean has_been_warned = false;
+	private boolean ready = false;
+	private CollectionMeta collection_extra_collectionmeta;
+	private CollectionMeta collection_name_collectionmeta;
+	private CollectionMeta creator_collectionmeta;
+	private CollectionMeta icon_collection_collectionmeta;
+	private CollectionMeta icon_collection_small_collectionmeta;
+	private CollectionMeta maintainer_collectionmeta;
+	private CollectionMeta public_collectionmeta;
+	/** The creators email. */
+	private EmailField creator_emailfield;
+	/** The maintainers email. */
+	private EmailField maintainer_emailfield;
+	/** Button to browse for the collection about page icon. */
+	private JButton browse_about_icon_button;
+	/** Button to browse for the collection home page icon. */
+	private JButton browse_home_icon_button;
+	/** The checkbox controlling public access to the collection. */
+	private JCheckBox public_checkbox;
+	private JLabel creator_label;
+	private JLabel description_label;
+	private JLabel icon_label;
+	private JLabel maintainer_label;
+	private JLabel name_label;
+	private JLabel small_icon_label;
+	private JLabel title_label;
+	/** The text field used to edit the file name of the collections icon. */
+	private JTextField icon_textfield;
+	/** The text field used to edit the collections title. */
+	private JTextField name_textfield;
+	/** The text field used to edit the file name of the collections small icon. */
+	private JTextField small_icon_textfield;
+	/** A text area used to modify the collection description. */
+	private JTextArea description_textarea;
+	private JTextArea instructions_textarea;
+	/** Constructor. */
+	public GeneralControl() {
+	    super();
+	    // Retrieve some of the model elements, those we know aren't language dependant
+	    public_collectionmeta = new CollectionMeta(CollectionDesignManager.collect_config.getPublic());
+
+	    // Creation
+	    JPanel instruction_panel = new JPanel();
+	    title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title_label, "CDM.General.Title");
+
+	    instructions_textarea = new JTextArea();
+	    instructions_textarea.setCaretPosition(0);
+	    instructions_textarea.setEditable(false);
+	    instructions_textarea.setLineWrap(true);
+	    instructions_textarea.setRows(6);
+	    instructions_textarea.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions_textarea, "CDM.General.Instructions");
+
+	    JPanel all_details_panel = new JPanel();
+	    JPanel details_panel = new JPanel();
+	    JPanel fields_panel = new JPanel();
+
+	    creator_label = new JLabel();
+	    Dictionary.registerText(creator_label, "CDM.General.Email.Creator");
+
+	    creator_emailfield = new EmailField(Configuration.getColor("coloring.error_background", false));
+	    Dictionary.registerTooltip(creator_emailfield, "CDM.General.Email.Creator_Tooltip");
+
+	    maintainer_label = new JLabel();
+	    Dictionary.registerText(maintainer_label, "CDM.General.Email.Maintainer");
+
+	    maintainer_emailfield = new EmailField(Configuration.getColor("coloring.error_background", false));
+	    Dictionary.registerTooltip(maintainer_emailfield, "CDM.General.Email.Maintainer_Tooltip");
+
+	    name_label = new JLabel();
+	    Dictionary.registerText(name_label, "CDM.General.Collection_Name");
+	    name_textfield = new JTextField("CDM.General.Collection_Name");
+	    Dictionary.registerTooltip(name_textfield, "CDM.General.Collection_Name_Tooltip");
+	    JLabel short_name_label = new JLabel();
+	    Dictionary.registerText(short_name_label, "NewCollectionPrompt.Collection_Name");
+	    JTextField short_name_textfield = new JTextField(Gatherer.c_man.getCollection().getName());
+	    short_name_textfield.setEditable(false);
+	    short_name_textfield.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    JPanel icon_panel = new JPanel();
+	    icon_label = new JLabel();
+	    Dictionary.registerText(icon_label, "CDM.General.Icon_Collection");
+	    icon_textfield = new JTextField("CDM.General.Icon_Collection");
+	    Dictionary.registerTooltip(icon_textfield, "CDM.General.Icon_Collection_Tooltip");
+	    browse_about_icon_button = new GLIButton();
+	    browse_about_icon_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerText(browse_about_icon_button, "General.Browse");
+	    JPanel small_icon_panel = new JPanel();
+	    small_icon_label = new JLabel("CDM.General.Icon_Collection_Small");
+	    Dictionary.registerText(small_icon_label, "CDM.General.Icon_Collection_Small");
+	    small_icon_textfield = new JTextField("CDM.General.Icon_Collection_Small");
+	    Dictionary.registerTooltip(small_icon_textfield, "CDM.General.Icon_Collection_Small_Tooltip");
+	    browse_home_icon_button = new GLIButton();
+	    browse_home_icon_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerText(browse_home_icon_button, "General.Browse");
+
+	    // public
+	    JPanel box_panel = new JPanel();
+	    public_checkbox = new JCheckBox("", public_collectionmeta.getValue(CollectionMeta.TEXT).equals(CollectionConfiguration.TRUE_STR));
+	    Dictionary.registerText(public_checkbox, "CDM.General.Access");
+
+	    JPanel description_panel = new JPanel();
+	    description_label = new JLabel();
+	    Dictionary.registerText(description_label, "CDM.General.Collection_Extra");
+	    description_textarea = new JTextArea();
+	    description_textarea.setBackground(Configuration.getColor("coloring.editable_background", false));
+	    Dictionary.registerTooltip(description_textarea, "CDM.General.Collection_Extra_Tooltip");
+	    // Connection
+	    BrowseListener browse_listener = new BrowseListener(StaticStrings.IMAGES_PATH_RELATIVE_TO_GSDL_PREFIX);
+	    browse_about_icon_button.addActionListener(browse_listener);
+	    browse_home_icon_button.addActionListener(browse_listener);
+	    browse_listener = null;
+	    public_checkbox.addActionListener(CollectionDesignManager.change_listener);
+	    public_checkbox.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    creator_emailfield.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    creator_emailfield.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    description_textarea.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    description_textarea.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    icon_textfield.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    icon_textfield.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    maintainer_emailfield.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    maintainer_emailfield.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    name_textfield.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    name_textfield.getDocument().addDocumentListener(new CollectionTitleUpdater());
+	    name_textfield.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    small_icon_textfield.getDocument().addDocumentListener(CollectionDesignManager.change_listener);
+	    small_icon_textfield.getDocument().addDocumentListener(CollectionDesignManager.buildcol_change_listener);
+	    //Note: unfortunately just loading the General Design panel fires the buildcol_change_listener (even if nothing is changed).
+
+	    // Layout
+	    instruction_panel.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    instruction_panel.setLayout(new BorderLayout());
+	    instruction_panel.add(title_label, BorderLayout.NORTH);
+	    instruction_panel.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+
+	    fields_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    fields_panel.setLayout(new BorderLayout(5,2));
+	    
+	    JPanel fields_label_panel = new JPanel();
+	    fields_label_panel.setLayout(new GridLayout(6,1));
+
+	    JPanel fields_box_panel = new JPanel();
+	    fields_box_panel.setLayout(new GridLayout(6,1));
+    
+	    // creator
+	    fields_label_panel.add(creator_label);
+	    fields_box_panel.add(creator_emailfield);
+
+	    // maintainer
+	    fields_label_panel.add(maintainer_label);
+	    fields_box_panel.add(maintainer_emailfield);
+
+	    // title
+	    fields_label_panel.add(name_label);
+	    fields_box_panel.add(name_textfield);
+
+	    // collection short name
+	    fields_label_panel.add(short_name_label);
+	    fields_box_panel.add(short_name_textfield);
+
+	    // icon
+	    fields_label_panel.add(icon_label);
+	    fields_box_panel.add(icon_panel);
+
+	    // small icon
+	    fields_label_panel.add(small_icon_label);
+	    fields_box_panel.add(small_icon_panel);
+
+	    fields_panel.add(fields_label_panel, BorderLayout.WEST);
+	    fields_panel.add(fields_box_panel, BorderLayout.CENTER);
+
+	    icon_panel.setLayout(new BorderLayout());
+	    icon_panel.add(icon_textfield, BorderLayout.CENTER);
+	    icon_panel.add(browse_about_icon_button, BorderLayout.EAST);
+
+	    small_icon_panel.setLayout(new BorderLayout());
+	    small_icon_panel.add(small_icon_textfield, BorderLayout.CENTER);
+	    small_icon_panel.add(browse_home_icon_button, BorderLayout.EAST);
+
+	    box_panel.setLayout(new GridLayout(1,1,5,2));
+	    box_panel.add(public_checkbox);
+
+	    description_panel.setLayout(new BorderLayout());
+	    description_panel.add(description_label, BorderLayout.NORTH);
+	    description_panel.add(new JScrollPane(description_textarea), BorderLayout.CENTER);
+	    
+	    details_panel.setLayout(new BorderLayout());
+	    details_panel.add(fields_panel, BorderLayout.NORTH);
+	    details_panel.add(box_panel, BorderLayout.CENTER);
+
+	    all_details_panel.setLayout(new BorderLayout());
+	    all_details_panel.add(details_panel, BorderLayout.NORTH);
+	    all_details_panel.add(description_panel, BorderLayout.CENTER);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(instruction_panel, BorderLayout.NORTH);
+	    add(all_details_panel, BorderLayout.CENTER);
+	}
+
+	/** Destructor. */
+	public void destroy() {
+	}
+
+	/** Called to refresh the components. */
+	public void gainFocus() {
+	    // Retrieve all of the elements that are dependant on default language.
+	    collection_extra_collectionmeta = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);
+	    collection_name_collectionmeta = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_COLLECTIONNAME_STR);
+	    
+	    creator_collectionmeta = new CollectionMeta(CollectionDesignManager.collect_config.getCreator());
+	    icon_collection_collectionmeta = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_ICONCOLLECTION_STR);
+	    icon_collection_small_collectionmeta = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR);
+	    maintainer_collectionmeta = new CollectionMeta(CollectionDesignManager.collect_config.getMaintainer());
+	    // Make sure the components are up to date
+	    creator_emailfield.setText(creator_collectionmeta.getValue(CollectionMeta.TEXT));
+	    creator_emailfield.setCaretPosition(0);
+	    description_textarea.setText(collection_extra_collectionmeta.getValue(CollectionMeta.TEXT));
+	    description_textarea.setCaretPosition(0);
+	    icon_textfield.setText(icon_collection_collectionmeta.getValue(CollectionMeta.TEXT));
+	    icon_textfield.setCaretPosition(0);
+	    maintainer_emailfield.setText(maintainer_collectionmeta.getValue(CollectionMeta.TEXT));
+	    maintainer_emailfield.setCaretPosition(0);
+	    name_textfield.setText(collection_name_collectionmeta.getValue(CollectionMeta.TEXT));
+	    name_textfield.setCaretPosition(0);
+	    small_icon_textfield.setText(icon_collection_small_collectionmeta.getValue(CollectionMeta.TEXT));
+	    small_icon_textfield.setCaretPosition(0);
+	    public_checkbox.setSelected(public_collectionmeta.getValue(CollectionMeta.TEXT).equals(CollectionConfiguration.TRUE_STR));
+	    ready = true;
+	}
+
+	public boolean canSave() {
+	    boolean title_clash_warning = true;
+	    // Oh, and if the collection title is already in use warn them about it
+	    String title = name_textfield.getText();
+	    // I just happen to have a handy method in the new details prompt to detect this very thing
+	    if(NewCollectionDetailsPrompt.titleClashes(title, CollectionDesignManager.collect_config.getFile())) {
+		// Determine if the user wants to be warned about this
+		WarningDialog dialog = new WarningDialog("warning.TitleClashes", "TitleClashes.Title", Dictionary.get("TitleClashes.Message"), null, true);
+		if(dialog.display() == JOptionPane.OK_OPTION) {
+		    // If they have said yes, then carry on with the assignement
+		    collection_name_collectionmeta.setValue(title);
+		    has_been_warned = true;
+		}
+		// Otherwise we don't assign anything. In fact we have to restore the frame title back to its original value
+		else {
+		    title_clash_warning = false;
+		    String collection_title = collection_name_collectionmeta.getValue(CollectionMeta.TEXT);
+		    String collection_name = Gatherer.c_man.getCollection().getName();
+		    Gatherer.g_man.setTitle(collection_title, collection_name);
+		    collection_name = null;
+		    collection_title = null;
+		}
+		dialog.dispose();
+		dialog = null;
+	    }
+	    // No clash, no worries.
+	    else if(collection_name_collectionmeta != null) {
+		collection_name_collectionmeta.setValue(title);
+	    }
+	    return title_clash_warning;
+	}
+
+	/** Called to store the current value of the components. */
+	public void loseFocus() {
+	    // String values. Have to test if this component has actually ever recieved focus anyway.
+	    if(ready) {
+		// Boolean values
+		public_collectionmeta.setValue((public_checkbox.isSelected() ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+		String creator_email_str = creator_emailfield.getText();
+		creator_collectionmeta.setValue(creator_email_str);
+		// If the email is currently empty, store this email
+		if(Configuration.getEmail() == null) {
+		    Configuration.setEmail(creator_email_str); // Store the email address in the configuration
+		}
+		collection_extra_collectionmeta.setValue(description_textarea.getText());
+		icon_collection_collectionmeta.setValue(icon_textfield.getText());
+		maintainer_collectionmeta.setValue(maintainer_emailfield.getText());
+		icon_collection_small_collectionmeta.setValue(small_icon_textfield.getText());
+
+		// Oh, and if the collection title is already in use warn them about it
+		String title = name_textfield.getText();
+		// I just happen to have a handy method in the new details prompt to detect this very thing
+		if(!has_been_warned && NewCollectionDetailsPrompt.titleClashes(title, CollectionDesignManager.collect_config.getFile())) {
+		    // Determine if the user wants to be warned about this
+		    WarningDialog dialog = new WarningDialog("warning.TitleClashes", "TitleClashes.Title", Dictionary.get("TitleClashes.Message"), null, true);
+		    if(dialog.display() == JOptionPane.OK_OPTION) {
+			// If they have said yes, then carry on with the assignement
+			collection_name_collectionmeta.setValue(title);
+		    }
+		    // Otherwise we don't assign anything. In fact we have to restore the frame title back to its original value
+		    else {
+			String collection_title = collection_name_collectionmeta.getValue(CollectionMeta.TEXT);
+			String collection_name = Gatherer.c_man.getCollection().getName();
+			Gatherer.g_man.setTitle(collection_title, collection_name);
+			collection_name = null;
+			collection_title = null;
+		    }
+		    dialog.dispose();
+		    dialog = null;
+		}
+		// No clash, no worries.
+		else {
+		    collection_name_collectionmeta.setValue(title);
+		}
+		title = null;
+
+		ready = false;
+		has_been_warned = false;
+	    }  
+	}
+
+	/** Listens for a click on either browse button, and when detected opens a file browser window initially pointed at the images folder of the collection. Once a user has selected a path, does it's best to construct the best address to the resource, such as _httpcollection_/images/about.gif */
+	private class BrowseListener
+	    implements ActionListener {
+
+	    private String prefix;
+
+	    public BrowseListener(String prefix_raw) {
+		this.prefix = prefix_raw.replaceAll(StaticStrings.COLNAME_PATTERN, Gatherer.c_man.getCollection().getName());
+	    }
+
+	    public void actionPerformed(ActionEvent event) {
+		// Open an almost standard file browser to the images folder of the current collection
+		File images_folder = new File(Gatherer.c_man.getCollectionImagesDirectoryPath());
+		// If images isn't already there, create it
+		if(!images_folder.exists()) {
+		    images_folder.mkdirs();
+		}
+		JFileChooser file_chooser = new JFileChooser(images_folder);
+		file_chooser.setAcceptAllFileFilterUsed(false);
+		file_chooser.setDialogTitle(Dictionary.get("CDM.General.Browser_Title"));
+		file_chooser.setFileFilter(new ImageFilter());
+		file_chooser.setSize(400,300);
+		OpenCollectionDialog.disableRename(file_chooser);
+		int value = file_chooser.showOpenDialog(Gatherer.g_man);
+		// If the user hasn't cancelled, retrieve the file path selected
+		if(value == JFileChooser.APPROVE_OPTION) {
+		    // If the file isn't in the images folder, then ask the user if they want to copy it there
+		    File file = file_chooser.getSelectedFile();
+		    if(!file.getParentFile().equals(images_folder)) {
+			if(true) { //JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("CDM.General.Copy_Image"), Dictionary.get("General.Warning"), JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) {
+			    // Copy the file
+			    try {
+				File collection_image_file = new File(images_folder, file.getName());
+				Gatherer.f_man.getQueue().copyFile(file, collection_image_file, null);
+
+				// If we're using a remote Greenstone server, upload the image
+				if (Gatherer.isGsdlRemote) {
+				    RemoteGreenstoneServer.uploadCollectionFile(Gatherer.c_man.getCollection().getName(), collection_image_file);
+				}
+			    }
+			    catch(Exception exception) {
+				DebugStream.printStackTrace(exception);
+				// Show warning
+				JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.General.Image_Copy_Failed"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+				// Nothing else we can do.
+				return;
+			    }
+			}
+			else {
+			    // Nothing we can safely do
+			    return;
+			}
+		    }
+		    // Create the path starting _httpcollection_/images/<filename>
+		    String path = prefix + file.getName();
+		    if(event.getSource() == browse_about_icon_button) {
+			icon_textfield.setText(path);
+		    }
+		    else {
+			small_icon_textfield.setText(path);
+		    }
+		    path = null;
+		}
+	    }
+
+	    /** ImageFilter.java is a 1.4 example used by FileChooserDemo2.java. */
+	    private class ImageFilter
+		extends javax.swing.filechooser.FileFilter {
+
+		private Pattern pattern = null;
+
+		public ImageFilter() {
+		    pattern = Pattern.compile(".*\\.(gif|png|jpe?g)");
+		}
+
+		// Accept all directories and all .col files
+		public boolean accept(File f) {
+		    String filename = f.getName().toLowerCase();
+		    Matcher matcher = pattern.matcher(filename);
+		    return f.isDirectory() || matcher.matches();
+		}
+
+		// The description of this filter
+		public String getDescription() {
+		    return Dictionary.get("CDM.General.Image_Filter");
+		}
+	    }
+	}
+
+	private class CollectionTitleUpdater
+	    implements DocumentListener {
+	    /** Gives notification that an attribute or set of attributes changed. */
+	    public void changedUpdate(DocumentEvent e) {
+		setTitle();
+	    }
+	    /** Gives notification that there was an insert into the document. */
+	    public void insertUpdate(DocumentEvent e) {
+		setTitle();
+	    }
+	    /** Gives notification that a portion of the document has been removed. */
+	    public void removeUpdate(DocumentEvent e) {
+		setTitle();
+	    }
+	    
+	    private void setTitle() {
+		// Set the title
+		String collection_title = name_textfield.getText();
+		String collection_name = Gatherer.c_man.getCollection().getName();
+		Gatherer.g_man.setTitle(collection_title, collection_name);
+		collection_title = null;
+		collection_name = null;
+	    }
+	}
+    }
+
+    /** This tree provides a 'table of contents' for the various components of the design process (collection configuration in more technical terms). */
+    private class DesignTree
+	extends JTree {
+	private DesignNode root = null;
+	/** Constructor. Automatically generates all of the nodes, in the order of CONTENTS. */
+	public DesignTree() {
+	    super();
+	    resetModel(Configuration.getMode());
+	    expandRow(0);
+	    setRootVisible(false);
+	    setSelectionRow(0);
+	}
+
+	/** Reset the model used by the design page contents tree. This is necessary to hide the partitions entry when in lower detail modes
+	 * @param mode the current detail mode as an int
+	 */
+	public void resetModel(int mode) {
+	    root = new DesignNode("CDM.GUI.Root");
+	    // Now add the design categories.
+	    for(int i = 0; i < CONTENTS.length; i++) {
+		root.add(new DesignNode(CONTENTS[i]));
+	    }
+	    this.setModel(new DefaultTreeModel(root));
+	    updateUI();
+	}
+	/** Set the current view to the one specified.
+	 * @param type the name of the desired view as a String
+	 */
+	public void setSelectedView(String type) {
+	    type = Dictionary.get(type);
+	    for(int i = 0; i < root.getChildCount(); i++) {
+		DesignNode child = (DesignNode) root.getChildAt(i);
+		if(child.toString().equals(type)) {
+		    TreePath path = new TreePath(child.getPath());
+		    setSelectionPath(path);
+		}
+	    }
+	}
+    }
+    /** A tree node that retains a reference to one of the possible design sub-views relating to the different sub-managers. */
+    private class DesignNode
+	extends DefaultMutableTreeNode {
+	/** Constructor.
+	 * @param object The <strong>Object</strong> assigned to this node.
+	 */
+	public DesignNode(String object) {
+	    super(object);
+	}
+	/** Retrieve a textual representation of the object.
+	 * @return a String
+	 */
+	public String toString() {
+	    // return Dictionary.get("CDM.GUI." + (String)getUserObject());
+	    return Dictionary.get((String) getUserObject());
+	}
+    }
+    /** Listens for selection changes in the 'contents' tree, and switches to the appropriate view. */
+    private class TreeListener
+	implements TreeSelectionListener {
+	/** Called whenever the selection changes, we must update the view so it matches the node selected.
+	 * @param event A <strong>TreeSelectionEvent</strong> containing more information about the tree selection.
+	 * @see org.greenstone.gatherer.cdm.ClassifierManager
+	 * @see org.greenstone.gatherer.cdm.CollectionDesignManager
+	 * @see org.greenstone.gatherer.cdm.CollectionMetaManager
+	 * @see org.greenstone.gatherer.cdm.FormatManager
+	 * @see org.greenstone.gatherer.cdm.LanguageManager
+	 * @see org.greenstone.gatherer.cdm.MetadataSetView
+	 * @see org.greenstone.gatherer.cdm.SubcollectionManager
+	 * @see org.greenstone.gatherer.cdm.TranslationView
+	 * @see org.greenstone.gatherer.cdm.PlugInManager
+	 */
+	public void valueChanged(TreeSelectionEvent event) {
+	    if(!tree.isSelectionEmpty()) {
+		TreePath path = tree.getSelectionPath();
+		DesignNode node = (DesignNode)path.getLastPathComponent();
+		String type = (String)node.getUserObject();
+		// Wait begins
+		Gatherer.g_man.wait(true);
+		// Save information in current view
+		view.loseFocus();
+		remove((JPanel)view);
+		// Change panes.
+		if(type == CONTENTS[0]) {
+		    view = getControls();
+		}
+		else if(type == CONTENTS[1]) {
+		    view = CollectionDesignManager.plugin_manager.getControls();
+		}
+		else if(type == CONTENTS[2]) {
+		    view = CollectionDesignManager.searchtype_manager.getControls();
+		}
+		else if(type == CONTENTS[3]) {
+		    view = CollectionDesignManager.index_manager.getControls();
+		}
+		else if(type == CONTENTS[4]) {
+		    view = CollectionDesignManager.subcollection_manager.getControls();
+		}
+		else if(type == CONTENTS[5]) {
+		    view = CollectionDesignManager.supercollection_manager.getControls();
+		}
+		else if(type == CONTENTS[6]) {
+		    view = CollectionDesignManager.classifier_manager.getControls();
+		}
+		else if(type == CONTENTS[7]) {
+		    view = CollectionDesignManager.format_manager.getControls();
+		}
+		else if(type == CONTENTS[8]) {
+		    view = CollectionDesignManager.translation_view.getControls();
+		}
+		else if(type == CONTENTS[9]) {
+		    view = CollectionDesignManager.metadataset_view.getControls();
+		}
+		add((JPanel)view, BorderLayout.CENTER);
+		// Update information on visible pane
+		view.gainFocus();
+		repaint();
+		// Wait ends
+		Gatherer.g_man.wait(false);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Index.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Index.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Index.java	(revision 21112)
@@ -0,0 +1,319 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class encapsulates a single indexing pair.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class Index
+    implements Comparable, DOMProxyListEntry {
+    /** An values of items in the index level enumeration. */
+    static public final String LEVEL[] = {CollectionConfiguration.DOCUMENT_STR, CollectionConfiguration.PARAGRAPH_STR, CollectionConfiguration.SECTION_STR};
+
+    private ArrayList sources = null;
+    /** The level of this index (if old sckool MG). */
+    private int level = -1;
+    /** The element this index is based upon. */
+    private Element element = null;
+    /** The unique, if cryptic, identifier of an index. */
+    private String id = null;
+
+    /** Default constructor, which should only be used during DOMProxyListModel creation. */
+    public Index() {
+    }
+
+    /** Constructor. */
+    public Index(Element element) {
+	this.element = element;
+    }
+
+    /** Constructor for a newly assigned index. */
+    public Index(ArrayList sources) {
+	this.sources = sources;
+	// Create a new element
+	Document document = CollectionDesignManager.collect_config.document;
+	this.element = document.createElement(CollectionConfiguration.INDEX_ELEMENT);
+	// For each source add a content element
+	int size = sources.size();
+	for(int i = 0; i < size; i++) {
+	    Element content_element = document.createElement(CollectionConfiguration.CONTENT_ELEMENT);
+	    Object source_object = sources.get(i);
+	    if (source_object instanceof MetadataElement) {
+		// System.err.println("Constructing new Index with MetadataElement source...");
+		content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, ((MetadataElement) source_object).getFullName());
+	    }
+	    else {
+		// System.err.println("Constructing new Index with String source...");
+		content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, source_object.toString());
+	    }
+	    element.appendChild(content_element);
+	    content_element = null;
+	}
+	document = null;
+    }
+
+    /** Constructor for a newly assigned index with specified level (old skool). */
+    public Index(int level, ArrayList sources) {
+	this(sources);
+	this.level = level;
+	element.setAttribute(CollectionConfiguration.LEVEL_ATTRIBUTE, LEVEL[level]);
+    }
+
+    public Index(String level, ArrayList sources) {
+	this(sources);
+	for(int i = 0; i < LEVEL.length; i++) {
+	    if(LEVEL[i].equalsIgnoreCase(level)) {
+		this.level = i;
+	    }
+	}
+	element.setAttribute(CollectionConfiguration.LEVEL_ATTRIBUTE, LEVEL[this.level]);
+    }
+
+    /** Method to compare two indexes.
+     * @param object The other index as an <strong>Object</strong>.
+     * @return An <i>int</i> which indicates how the indexes compare.
+     * @see java.lang.String
+     */
+    public int compareTo(Object object) {
+	if(object == null) {
+	    return -1;
+	}
+	String id = getID();
+	return id.compareTo(((Index)object).getID());
+    }
+    
+    public DOMProxyListEntry create(Element element) {
+	return new Index(element);
+    }
+
+    /** Method to test for the equality of two indexes.
+     * @param object The other index as an <strong>Object</strong>.
+     * @return A <i>boolean</i> which is <i>true</i> if the two indexes are equal, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+    
+    public Element getElement() {
+	return element;
+    }
+    
+    /** Method to get the value of level.
+     * @return the level as a int
+     */
+    public int getLevel() {
+	if(level == -1 && element != null) {
+	    String level_str = element.getAttribute(CollectionConfiguration.LEVEL_ATTRIBUTE);
+	    for(int i = 0; level == -1 && i < LEVEL.length; i++) {
+		if(level_str.equals(LEVEL[i])) {
+		    level = i;
+		}
+	    }
+	    level_str = null;
+	}
+	return level;
+    }
+
+    public String getID() {
+	if(element == null) {
+	    id="";
+	}
+	else if(id == null) {
+	    StringBuffer id_buffer = new StringBuffer();
+	    // Write level information, if any.
+	    int level = getLevel();
+	    if(0 <= level && level < 3) {
+		id_buffer.append(LEVEL[level]);
+		id_buffer.append(StaticStrings.COLON_CHARACTER);
+	    }
+	    // Write data information. Retrieve each of the content sources and add them in a comma separated list.
+	    ArrayList sources = getSources();
+	    int sources_size = sources.size();
+	    for(int i = 0; i < sources_size; i++) {
+		Object source_object = sources.get(i);
+		if (source_object instanceof MetadataElement) {
+		    String full_element_name = ((MetadataElement)source_object).getFullName();
+		    if(full_element_name.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+		       id_buffer.append(full_element_name.substring(StaticStrings.EXTRACTED_NAMESPACE.length()));
+		    }
+		    else {
+			id_buffer.append(full_element_name);
+		    }
+		}
+		else {
+		    id_buffer.append(source_object.toString());
+		}
+		id_buffer.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	    sources = null;
+	    id = id_buffer.substring(0, id_buffer.length() - 1);
+	}
+	return id;
+    }
+
+    /** Tries to retrieve this indexes name according to the CollectionMetaManager. */
+    public String getName() {
+	CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(StaticStrings.STOP_CHARACTER + getID(), false);
+	if(metadatum != null) {
+	    return metadatum.getValue(CollectionMeta.TEXT);
+	}
+	return "";
+    }
+
+    /** Retrieve the sources of this index.
+     * @return the sources as an ArrayList
+     */
+    public ArrayList getSources() {
+	if(sources == null) {
+	    sources = new ArrayList();
+	    NodeList content_elements = element.getElementsByTagName(CollectionConfiguration.CONTENT_ELEMENT);
+	    int content_elements_length = content_elements.getLength();
+	    for(int i = 0; i < content_elements_length; i++) {
+		Element content_element = (Element) content_elements.item(i);
+		String metadata_element_name_full = (String) content_element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+		MetadataElement metadata_element = MetadataTools.getMetadataElementWithDisplayName(metadata_element_name_full);
+		if (metadata_element != null) {
+		    sources.add(metadata_element);
+		}
+		else {
+		    sources.add(metadata_element_name_full);
+		}
+	    }
+	    content_elements = null;
+	}
+	return sources;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	this.level = -1;
+	this.id = null;
+	this.sources = null;
+    }
+
+    /** Method to set the level of this index which can only be used for the default index. 
+     * @param new_level the new level as an int
+     */
+    public void setLevel(int new_level) {
+	// System.err.println("SetLevel(" + new_level + ")");
+	if(element != null && element.getNodeName().equals(CollectionConfiguration.INDEX_DEFAULT_ELEMENT)) {
+	    element.setAttribute(CollectionConfiguration.LEVEL_ATTRIBUTE, LEVEL[new_level]);
+	    this.id = null; // Regenerate ID.
+	    this.level = new_level;
+	}
+	else {
+	    DebugStream.println("Error! Called setLevel() of index other than the default.");
+	}
+    }
+
+    /** Method to set the sources for this index which can only be used for the default index.
+     * @param sources an ArrayList of source names
+     */
+    public void setSources(ArrayList sources) {
+	if(element != null && element.getNodeName().equals(CollectionConfiguration.INDEX_DEFAULT_ELEMENT)) {
+	    // Erase old sources
+	    XMLTools.clear(element);
+	    // For each entry in the sources array add a new content element.
+	    int size = sources.size();
+	    for(int i = 0; i < size; i++) {
+		Element content_element = element.getOwnerDocument().createElement(CollectionConfiguration.CONTENT_ELEMENT);
+		Object source_object = sources.get(i);
+		if (source_object instanceof MetadataElement) {
+		    String name = ((MetadataElement) source_object).getFullName();
+		    content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, name);
+		    name = null;
+		}
+		else {
+		    //DebugStream.println("Found String as source: " + source_object.toString());
+		    content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, source_object.toString());
+		}
+		source_object = null;
+		element.appendChild(content_element);
+		content_element = null;
+	    }
+	    this.id = null; // Regenerate ID.
+	    this.sources = sources;
+	}
+	else {
+	    DebugStream.println("Error! Called setSource() of index other than the default.");
+	}
+    }
+
+    /** Method to turn this object into a string representation ready to be placed in the collection configuration file.
+     * @return A <strong>String</strong> containing the information of this class.
+     */
+    public String toString() {
+	StringBuffer text_buffer = new StringBuffer("");
+	// Generate language dependant id (include extracted metadata namespace)
+	// Write level information, if any.
+	int level = getLevel();
+	if(0 <= level && level < 3) {
+	    text_buffer.append(LEVEL[level]);
+	    text_buffer.append(StaticStrings.COLON_CHARACTER);
+	}
+	// Write data information. Retrieve each of the content sources and add them in a comma separated list.
+	ArrayList sources = getSources();
+	int sources_size = sources.size();
+	for(int i = 0; i < sources_size; i++) {
+	    String source_name = (sources.get(i)).toString();
+	    text_buffer.append(source_name);
+	    if(i < sources_size - 1) {
+		text_buffer.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	}
+	sources = null;
+	CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(StaticStrings.STOP_CHARACTER + getID(), false);
+	if(metadatum != null) {
+	    text_buffer.append(" \"");
+	    text_buffer.append(metadatum.getValue(CollectionMeta.TEXT));
+	    text_buffer.append("\"");
+	}
+	return text_buffer.toString();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/IndexManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/IndexManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/IndexManager.java	(revision 21112)
@@ -0,0 +1,1834 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+/** This class is resposible for storing the indexes which have been assigned to this collection and the default index, and providing methods for interacting with both these data pools. It also knows how to turn itself into a String as it would be displayed in the collection configuration file.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class IndexManager
+    extends DOMProxyListModel {
+
+    static final private Dimension FIELD_SIZE = new Dimension(200,30);
+    static final private String MGINDEXES = "mg indexes";
+    static final private String MGPPINDEXES = "mgpp indexes";
+
+    /** The controls for editing the indexes. */
+    private Control controls = null;
+    /** A model of the levels, also based on the DOM. */
+    private DOMProxyListModel levels_model = null;
+    /** A reference to ourselves so our inner methods have access. */
+    private DOMProxyListModel model = null;
+    /** The default index. */
+    private Index default_index = null;
+
+    /** Constructor. */
+    public IndexManager(Element indexes) {
+	super(indexes, CollectionConfiguration.INDEX_ELEMENT, new Index());
+	DebugStream.println("IndexManager: " + getSize() + " indexes parsed.");
+	model = this;
+	// Parse and retrieve the default index
+	NodeList default_index_elements = CollectionDesignManager.collect_config.getDocumentElement().getElementsByTagName(CollectionConfiguration.INDEX_DEFAULT_ELEMENT);
+	if(default_index_elements.getLength() > 0) {
+	    default_index = new Index((Element)default_index_elements.item(0));
+	}
+	// Parse and retrieve the levels element
+	Element levels_element = CollectionDesignManager.collect_config.getLevels();
+	levels_model = new DOMProxyListModel(levels_element, CollectionConfiguration.CONTENT_ELEMENT, new Level());
+	DebugStream.println(" + " + levels_model.getSize() + " levels parsed.");
+    }
+
+    /** Method to add a new index.
+     * @param index The <strong>Index</strong> to add.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void addIndex(Index index, CollectionMeta metadatum) {
+	///ystem.err.println("Adding an index: " + index.toString());
+	if(!contains(index)) {
+	    CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+	    // Retrieve the currently last index
+	    if(getSize() > 0) {
+		Index last_index = (Index)getElementAt(getSize() - 1);
+		addAfter(index, last_index);
+		
+	    }
+	    else {
+		add(index);
+		// Also set this index as the default one, but only if there are levels available (ie mg only)
+		if(index.getLevel() != -1) {
+		    setDefault(index);
+		}
+	    }
+	    Gatherer.c_man.configurationChanged();
+	}
+	else {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.IndexManager.Index_Exists"), Dictionary.get("General.Warning"), JOptionPane.WARNING_MESSAGE);
+	}
+    }
+
+    private void addLevel(Level level, CollectionMeta metadatum) {
+	if(!levels_model.contains(level)) {
+	    CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+	    // Retrieve the currently last level
+	    if(levels_model.getSize() > 0) {
+		Level last_level = (Level)levels_model.getElementAt(levels_model.getSize() - 1);
+		levels_model.addAfter(level, last_level);
+	    }
+	    else {
+		levels_model.add(level);
+	    }
+	    Gatherer.c_man.configurationChanged();
+	}
+	else {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.IndexManager.Level_Exists"), Dictionary.get("General.Warning"), JOptionPane.WARNING_MESSAGE);
+	}
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+	default_index = null;
+	model = null;
+    }
+
+    /** Method to acquire the controls for editing the indexes.
+     * @return the Control
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    // Build controls
+	    controls = new IndexControl();
+	}
+	return controls;
+    }
+
+    /** Method to get the default index.
+     * @return The default <strong>Index</strong>.
+     */
+    public Index getDefault() {
+	if(default_index != null && default_index.isAssigned()) {
+	    return default_index;
+	}
+	else {
+	    return null;
+	}
+    }
+
+    /** Method to retrieve a certain index, as referenced by an index number.
+     * @param index An <i>int</i> which indicates the position of the desired index.
+     * @return The <strong>Index</strong> at the given index, or <i>null</i> if no such index exists.
+     */
+    public Index getIndex(int index) {
+	if(0 <= index && index < getSize()) {
+	    return (Index)getElementAt(index);
+	}
+	return null;
+    }
+
+    /** Method to retrieve a certain index, given its id.
+     * @param id the id of the index as a String
+     * @return the Index that matches id, or null if no such index exists
+     */
+    public Index getIndex(String id) {
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    Index index = (Index) getElementAt(i);
+	    if(index.getID().equals(id)) {
+		return index;
+	    }
+	}
+	return null;
+    }
+
+    public ArrayList getIndexes() {
+	return children();
+    }
+
+    public Level getLevel(String name) {
+	int levels_model_size = levels_model.getSize();
+	for(int i = 0; i < levels_model_size; i++) {
+	    Level level = (Level) levels_model.getElementAt(i);
+	    if(level.getLevel().equals(name)) {
+		return level;
+	    }
+	}
+	return null;
+    }
+
+    public int getNumLevels() {
+	return levels_model.getSize();
+    }
+    private void moveIndex(Index index, boolean move_up)
+    {
+	// Determine the current position of the index
+	int position = indexOf(index);
+	// Determine if it can be moved, ie if its not already at the top trying to move up, or at the bottom trying to move down.
+	if(position == -1) {
+	    return;
+	}
+	if(position == 0 && move_up) {
+	    return;
+	}
+	if(position == (getSize()) - 1 && !move_up) {
+	    return;
+	}
+	
+	// Ok, move up
+	if (move_up) {
+	    position--;
+	    remove(index);
+	    add(position, index);
+	}
+
+	// Or move down
+	else {
+	    position++;
+	    remove(index);
+	    add(position, index);
+	}
+
+	// Schedule the collection for saving
+	Gatherer.c_man.configurationChanged();
+    }
+
+
+    private void moveLevel(Level level, boolean move_up) {
+	// Determine the leveles current position
+	int position = levels_model.indexOf(level);
+	// Determine if it can be moved, ie if its not already at the top trying to move up, or at the bottom trying to move down.
+	if(position == -1) {
+	    return;
+	}
+	if(position == 0 && move_up) {
+	    return;
+	}
+	if(position == (levels_model.getSize()) - 1 && !move_up) {
+	    return;
+	}
+	// Ok, move up
+	if(move_up) {
+	    position--;
+	    levels_model.remove(level);
+	    levels_model.add(position, level);
+	}
+	// Or move down
+	else {
+	    position++;
+	    levels_model.remove(level);
+	    levels_model.add(position, level);
+	}
+	// Schedule the collection for saving
+	Gatherer.c_man.configurationChanged();
+    }
+    
+    /** Method to remove a certain index.
+     * @param index the Index to remove.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
+     * @see org.greenstone.gatherer.cdm.CollectionMetaManager
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void removeIndex(Index index) {
+	if(index != null) {
+	    // Remove any current metadata from this index
+	    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + index.getID());
+	    // Check if the index removed happens to be the default index
+	    if(default_index != null && default_index.equals(index)) {
+		// If so our first solution is to set the first index to be default
+		if(getSize() > 0) {
+		    Index another_index = (Index) getElementAt(0);
+		    setDefault(another_index);
+		    another_index = null;
+		}
+		else {
+		    default_index.setAssigned(false);
+		}
+	    }
+	    // Remove the index
+	    remove(index);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    private void removeLevel(Level level) {
+	if(level != null) {
+	    // Remove any current metadata from this level
+	    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + level.getLevel());
+	    // Remove the level
+	    levels_model.remove(level);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    /** Method to set the default index.
+     * @param index the new default Index
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    public void setDefault(Index index) {
+	if(index != null) {
+	    if(default_index == null) {
+		// Create the default index element, and place immediately after indexes element.
+		Element default_index_element = root.getOwnerDocument().createElement(CollectionConfiguration.INDEX_DEFAULT_ELEMENT);
+		default_index = new Index(default_index_element);
+		Node target_node = CollectionConfiguration.findInsertionPoint(default_index_element);
+		if(target_node != null) {
+		    root.getOwnerDocument().getDocumentElement().insertBefore(default_index_element, target_node);
+		}
+		else {
+		    root.getOwnerDocument().getDocumentElement().appendChild(default_index_element);
+		}
+	    }
+	    default_index.setAssigned(true);
+	    default_index.setLevel(index.getLevel());
+	    default_index.setSources(index.getSources());
+	}
+	else {
+	    if(default_index != null) {
+		default_index.setAssigned(false);
+	    }
+	}
+	Gatherer.c_man.configurationChanged();
+    }
+
+    /** This method is reponsible for changing the underlying Index commands from MG to MGPP and back again. This turns out to be easyish for MG->MGPP and very hard for the reverse. For the former we remove the level fragment and make sure the same levels are set, then we produce a list of the sources involved, breaking down comma seperated lists and making sure each item it unique. Changing back the other way turns out to be impossible, so we don't (beyond generating document:text, section:text and paragraph:text if text is an index and the respective levels are present). In either case we start by creating a comment containing the old index information.
+     * @param  state true to enable MGPP indexes, false to use standard MG style ones
+     */
+    public void setMGPPEnabled(boolean state) {
+	if(state == root.getAttribute(CollectionConfiguration.MGPP_ATTRIBUTE).equals(CollectionConfiguration.TRUE_STR)) {
+	    // we are not changing state
+	    return;
+	    
+	}
+	// change MG -> MGPP
+	if(state) {
+	    Element mg_element = root;
+	    // Retrieve and assign the MGPP indexes element.
+	    Element mgpp_element = CollectionDesignManager.collect_config.getMGPPIndexes();
+	    mgpp_element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.TRUE_STR);
+	    levels_model.root.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.TRUE_STR);
+	    // If the MGPP indexes element is empty (ie was created by CollectionConfiguration), generate new MGPP index from the existing index
+	    NodeList indexes = mgpp_element.getElementsByTagName(CollectionConfiguration.INDEX_ELEMENT);
+	    if(indexes.getLength() == 0) {
+		ArrayList levels_list = new ArrayList();
+		ArrayList sources_list = new ArrayList();
+		// We first use details from the default index if any
+		if(default_index != null) {
+		    int level_int = default_index.getLevel();
+		    if(0 <= level_int && level_int < 3) {
+			String level = Index.LEVEL[level_int];
+			if(!levels_list.contains(level)) {
+			    levels_list.add(level);
+			}
+			level = null;
+		    }
+		    ArrayList sources = default_index.getSources();
+		    sources.removeAll(sources_list);
+		    sources_list.addAll(sources);
+		}
+		int size = getSize();
+		for(int i = 0; i < size; i++) {
+		    Index index = (Index) getElementAt(i);
+		    int level_int = index.getLevel();
+		    if(0 <= level_int && level_int < 3) {
+			String level = Index.LEVEL[level_int];
+			if(!levels_list.contains(level)) {
+			    levels_list.add(level);
+			}
+			level = null;
+		    }
+		    ArrayList sources = index.getSources();
+		    sources.removeAll(sources_list);
+		    sources_list.addAll(sources);
+		    index = null;
+		}
+		// Replace mg element with mgpp element
+		setRoot(mgpp_element);
+		
+		// We now have a list of sources and a list of levels, so create new indexes and levels based on these
+		int sources_list_size = sources_list.size();
+		for(int j = 0; j < sources_list_size; j++) {
+		    Object source_object = sources_list.get(j);
+		    String source_str = null;
+		    if(source_object instanceof MetadataElement) {
+			source_str = ((MetadataElement) source_object).getFullName();
+		    }
+		    else {
+			source_str = source_object.toString();
+		    }
+		    ArrayList new_sources = new ArrayList();
+		    new_sources.add(source_object);
+		    source_object = null;
+		    Index new_index = new Index(new_sources);
+		    // Try to retrieve existing metadatum
+		    source_str = new_index.getID();
+		    CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.STOP_CHARACTER + source_str, false);
+		    // If no metadata was found, add new pseudo metadata using the id
+		    if(metadatum == null) {
+			metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + source_str);
+			metadatum.setAssigned(true);
+			metadatum.setValue(source_str);
+		    }
+		    // If it was found, ensure it is assigned
+		    else {
+			metadatum.setAssigned(true);
+		    }
+		    source_str = null;
+		    addIndex(new_index, metadatum);
+		    metadatum = null;
+		    new_index = null;
+		    new_sources = null;
+		    source_str = null;
+		}
+		int levels_list_size = levels_list.size();
+		for(int k = 0; k < levels_list_size; k++) {
+		    Level new_level = new Level((String)levels_list.get(k));
+		    if(!levels_model.contains(new_level)) {
+			levels_model.add(levels_model.getSize(), new_level);
+		    }
+		    new_level = null;
+		}
+	    }
+	    else {
+		// Replace mg element with mgpp element
+		setRoot(mgpp_element);
+	    }
+	    // Unassign MG element and default index
+	    mg_element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.FALSE_STR);
+	    mg_element = null;
+	    if(default_index != null) {
+		default_index.setAssigned(false);
+	    }
+	}
+	// change MGPP -> MG
+	else {
+	    Element mgpp_element = root;
+	    // Retrieve and assign MG element and default index element
+	    Element mg_element = CollectionDesignManager.collect_config.getMGIndexes();
+	    mg_element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.TRUE_STR);
+	    if(default_index != null) {
+		default_index.setAssigned(true);
+	    }
+	    // If mg element has no indexes, and the current mgpp index include a text one, then generate text indexes for each of the registered levels.
+	    NodeList indexes = mgpp_element.getElementsByTagName(CollectionConfiguration.INDEX_ELEMENT);
+	    if(indexes.getLength() == 0) {
+		Index index = getIndex(CollectionConfiguration.TEXT_STR);
+		if(index != null) {
+		    // Replace mgpp element with mg element
+		    setRoot(mg_element);
+		    int level_size = levels_model.getSize();
+		    for(int i = 0; i < level_size; i++) {
+			Level level = (Level) levels_model.getElementAt(i);
+			Index new_index = new Index(level.getLevel(), index.getSources());
+			// Try to retrieve existing metadatum
+			String source_str = new_index.getID();
+			CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.STOP_CHARACTER + source_str, false);
+			// If no metadata was found, add new pseudo metadata using the id
+			if(metadatum == null) {
+			    metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + source_str);
+			    metadatum.setAssigned(true);
+			    metadatum.setValue(source_str);
+			}
+			// If it was found, ensure it is assigned
+			else {
+			    metadatum.setAssigned(true);
+			}
+			source_str = null;
+			addIndex(new_index, metadatum);
+			new_index = null;
+			level = null;
+		    }
+		}
+	    }
+	    else {
+		// Replace mgpp element with mg element
+		setRoot(mg_element);
+	    }
+	    // Unassign mgpp element and levels
+	    mgpp_element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.FALSE_STR);
+	    levels_model.root.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.FALSE_STR);
+	}
+	
+    }
+    
+
+    /** This class creates a set of controls for editing the indexes. */
+    private class IndexControl
+	extends JPanel
+	implements Control {
+
+	private CardLayout card_layout;
+	private CheckList source_list;
+	private JButton add_button;
+	private JButton move_down_button;
+	private JButton move_up_button;
+	private JButton remove_button;
+	private JButton replace_button;
+	private JButton set_default_button;
+	private JComboBox level_combobox;
+	private JList index_list;
+	private JTextArea instruction_textarea;
+	private JTextField name_textfield ;
+	// when we are in mgpp mode, we want to remove the mg listener 
+	// the underlying model is the same between the two JLists, and
+	// both listeners get notified when the model changes, but the mg
+	// listener doesn't care about mgpp events
+	private ListSelectionListener index_list_listener;
+ 	
+	private MGPPControl mgppindexes_control;
+
+	/** Constructor.
+	 * @see org.greenstone.gatherer.Configuration
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.cdm.IndexManager.IndexControl.AddListener
+	 * @see org.greenstone.gatherer.cdm.IndexManager.IndexControl.NameListener
+	 * @see org.greenstone.gatherer.cdm.IndexManager.IndexControl.RemoveListener
+	 * @see org.greenstone.gatherer.cdm.IndexManager.IndexControl.SetDefaultListener
+	 * @see org.greenstone.gatherer.collection.CollectionManager
+	 */
+	public IndexControl() {
+	    super();
+
+	    ArrayList new_data = new ArrayList();
+	    new_data.add(CollectionConfiguration.TEXT_STR);
+	    new_data.addAll(MetadataSetManager.getEveryMetadataSetElement());
+	    // Creation
+	    JPanel mgindexes_panel = new JPanel();
+	    card_layout = new CardLayout();
+
+	    // MG Index Controls
+
+	    JPanel mg_indexes_pane = new JPanel();
+
+	    JPanel instruction_pane = new JPanel();
+	    JLabel title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title_label, "CDM.IndexManager.Title");
+
+	    instruction_textarea = new JTextArea();
+	    instruction_textarea.setEditable(false);
+	    instruction_textarea.setLineWrap(true);
+	    instruction_textarea.setRows(6);
+	    instruction_textarea.setWrapStyleWord(true);
+	    Dictionary.registerText(instruction_textarea, "CDM.IndexManager.Instructions");
+
+	    JPanel assigned_indexes_pane = new JPanel();
+	    JLabel index_label = new JLabel();
+	    Dictionary.registerText(index_label, "CDM.IndexManager.Indexes");
+	    index_list = new JList(model);
+	    index_list.setCellRenderer(new IndexListRenderer());
+	    index_list.setVisibleRowCount(2);
+	    JPanel movement_pane = new JPanel();
+
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    set_default_button = new GLIButton();
+	    set_default_button.setEnabled(false);
+	    set_default_button.setMnemonic(KeyEvent.VK_S);
+	    Dictionary.registerBoth(set_default_button, "CDM.IndexManager.Set_Default", "CDM.IndexManager.Set_Default_Tooltip");
+
+	    JPanel index_pane = new JPanel();
+	    JPanel details_pane = new JPanel();
+	    JPanel labels_pane = new JPanel();
+	    JPanel boxes_pane = new JPanel();
+	    JPanel content_pane = new JPanel();
+
+	    JLabel name_label = new JLabel();
+	    Dictionary.registerText(name_label, "CDM.IndexManager.Index_Name");
+	    name_textfield = new JTextField();
+	    name_textfield.setPreferredSize(FIELD_SIZE);
+	    Dictionary.registerTooltip(name_textfield, "CDM.IndexManager.Index_Name_Tooltip");
+
+	    JLabel source_label = new JLabel();
+	    Dictionary.registerText(source_label, "CDM.IndexManager.Source");
+	    source_list = new CheckList(false);
+	    source_list.setListData(new_data);
+	    Dictionary.registerTooltip(source_list, "CDM.IndexManager.Source_Tooltip");
+
+	    JLabel level_label = new JLabel();
+	    Dictionary.registerText(level_label, "CDM.IndexManager.Level");
+	    level_combobox = new JComboBox();
+	    level_combobox.setPreferredSize(FIELD_SIZE);
+	    level_combobox.addItem(CollectionConfiguration.DOCUMENT_STR);//Dictionary.get("CDM.IndexManager.Document"));
+	    level_combobox.addItem(CollectionConfiguration.PARAGRAPH_STR);//Dictionary.get("CDM.IndexManager.Paragraph"));
+	    level_combobox.addItem(CollectionConfiguration.SECTION_STR);//Dictionary.get("CDM.IndexManager.Section"));
+	    level_combobox.setEditable(false);
+	    Dictionary.registerTooltip(level_combobox, "CDM.IndexManager.Level_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(false);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.IndexManager.Add_Index", "CDM.IndexManager.Add_Index_Tooltip");
+
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.IndexManager.Remove_Index", "CDM.IndexManager.Remove_Index_Tooltip");
+	    replace_button = new GLIButton();
+	    replace_button.setEnabled(false);
+	    replace_button.setMnemonic(KeyEvent.VK_P);
+	    Dictionary.registerBoth(replace_button, "CDM.IndexManager.MGPP.Replace_Index", "CDM.IndexManager.MGPP.Replace_Index_Tooltip");
+
+	    // This class is getting a little crowded, so I'll generate the many mgpp controls in a inner class.
+	    mgppindexes_control = new MGPPControl();
+
+	    // Listeners
+	    add_button.addActionListener(new AddListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_down_button.addActionListener(new MoveListener(false));
+	    move_down_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_up_button.addActionListener(new MoveListener(true));
+	    move_up_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    replace_button.addActionListener(new ReplaceListener());
+	    replace_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    set_default_button.addActionListener(new SetDefaultListener());
+	    set_default_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    name_textfield.getDocument().addDocumentListener(new NameListener());
+	    level_combobox.addItemListener(new LevelListener());
+	    // Create the index_list_listener but don't add in to the list - this gets done in gainFocus, we don't want it twice.
+	    index_list_listener = new IndexListListener();
+	    source_list.addListSelectionListener(new SourceListListener());
+
+	    // Layout
+	    instruction_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    instruction_pane.setLayout(new BorderLayout());
+	    instruction_pane.add(title_label, BorderLayout.NORTH);
+	    instruction_pane.add(new JScrollPane(instruction_textarea), BorderLayout.CENTER);
+
+	    movement_pane.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+	    movement_pane.setLayout(new GridLayout(3,1));
+	    movement_pane.add(move_up_button);
+	    movement_pane.add(move_down_button);
+	    movement_pane.add(set_default_button);
+
+	    assigned_indexes_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    assigned_indexes_pane.setLayout(new BorderLayout());
+	    assigned_indexes_pane.add(index_label, BorderLayout.NORTH);
+	    assigned_indexes_pane.add(new JScrollPane(index_list), BorderLayout.CENTER);
+	    assigned_indexes_pane.add(movement_pane, BorderLayout.EAST);
+
+	    labels_pane.setLayout(new BorderLayout());
+	    labels_pane.setBorder(BorderFactory.createEmptyBorder(5, 5, 10, 5));
+	    labels_pane.add(name_label, BorderLayout.NORTH);
+	    labels_pane.add(source_label, BorderLayout.CENTER);
+	    labels_pane.add(level_label, BorderLayout.SOUTH);
+	    
+	    boxes_pane.setLayout(new BorderLayout());
+	    boxes_pane.add(name_textfield, BorderLayout.NORTH);
+	    boxes_pane.add(new JScrollPane(source_list), BorderLayout.CENTER);
+	    boxes_pane.add(level_combobox, BorderLayout.SOUTH);
+	    
+	    details_pane.setLayout(new BorderLayout());
+	    details_pane.add(labels_pane, BorderLayout.WEST);
+	    details_pane.add(boxes_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,3));
+	    button_pane.add(add_button);
+	    button_pane.add(replace_button);
+	    button_pane.add(remove_button);
+
+	    index_pane.setLayout(new BorderLayout());
+	    index_pane.add(details_pane, BorderLayout.CENTER);
+	    index_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(assigned_indexes_pane, BorderLayout.NORTH);
+	    content_pane.add(index_pane, BorderLayout.CENTER);
+
+	    mgindexes_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    mgindexes_panel.setLayout(new BorderLayout());
+	    mgindexes_panel.add(instruction_pane, BorderLayout.NORTH);
+	    mgindexes_panel.add(content_pane, BorderLayout.CENTER);
+
+	    setLayout(card_layout);
+	    add(mgindexes_panel, MGINDEXES);
+	    add(mgppindexes_control, MGPPINDEXES);
+	}
+
+	
+	/* Destructor, removes persistant listeners from the Dictionary.
+	 */
+	public void destroy() {
+	    mgppindexes_control.destroy();
+	    mgppindexes_control = null;
+	}
+
+	public void gainFocus() {
+	    boolean mgpp_enabled = CollectionDesignManager.searchtype_manager.isSearchTypeEnabled();
+	    if(instruction_textarea != null) {
+		instruction_textarea.setCaretPosition(0);
+	    }
+	    if(mgpp_enabled) {
+		// remove the mg index list listener first
+		index_list.removeListSelectionListener(index_list_listener);
+		mgppindexes_control.gainFocus();
+	    }
+	    else {
+		// remove the mgpp listener first
+		mgppindexes_control.removeIndexListListener();
+		// add the mg listener back in
+		index_list.addListSelectionListener(index_list_listener);
+		// Reload the assigned indexes list.
+		ArrayList new_data = new ArrayList();
+		new_data.add(CollectionConfiguration.TEXT_STR);
+		new_data.addAll(MetadataSetManager.getEveryMetadataSetElement());
+		// reset the model in the list and combobox
+		source_list.setListData(new_data);
+		new_data = null;
+		// refresh the indexList
+		index_list.updateUI();
+		// if there is one selected, fill in the controls
+		updateControlsWithSelectedIndex();
+	    }
+	    // Bring the appropriate card to the front
+	    if(mgpp_enabled) {
+		card_layout.show(this, MGPPINDEXES);
+	    }
+	    else {
+		card_layout.show(this, MGINDEXES);
+	    }
+	}
+
+	public void loseFocus() {
+	}
+
+	private void updateControlsWithSelectedIndex() {
+	    
+	    Index selected_index = (Index)index_list.getSelectedValue();
+	    if (selected_index == null) {
+		return;
+	    }
+	    String id = selected_index.getID();
+	    if (id == null || id.equals("")) {
+		return;
+	    }
+	    String name = CollectionDesignManager.collectionmeta_manager.getMetadatum("." + id).getValue(true);
+	    name_textfield.setText(name);
+	    level_combobox.setSelectedIndex(selected_index.getLevel());
+	    source_list.clearTicked();
+	    ArrayList sources = selected_index.getSources();
+	    source_list.setTickedObjects(sources.toArray());
+	   
+	}
+	
+	private void validateAddButton()
+	{
+	    boolean add_enabled = false;
+	    boolean replace_enabled = false;
+
+	    // Indexes must have a name
+	    if (name_textfield.getText().length() == 0) {
+		add_enabled = false;
+	    }
+	    // Can't add a new index if no sources are selected
+	    else if (source_list.isNothingTicked()) {
+		add_enabled = false;
+	    }
+	    // If we get this far, create a dummy index and see if it's already assigned in the collection
+	    else {
+		Object object[] = source_list.getTicked().toArray();
+		ArrayList sources = new ArrayList();
+		for(int i = 0; i < object.length; i++) {
+		    sources.add(object[i]);
+		}
+		object = null;
+		Index index = new Index(level_combobox.getSelectedIndex(), sources);
+		sources = null;
+		if (model.contains(index)) {
+		    add_enabled = false;
+
+		    // Here we need to check if we have changed the name - if so, we can enable the replace button
+		    if (index_list.getSelectedIndex() != -1) {
+			Index selected_index = (Index)index_list.getSelectedValue();
+			String name = name_textfield.getText();
+			String index_name = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.STOP_CHARACTER+selected_index.getID()).getValue(true);
+			if (!index_name.equals(name)){
+			    replace_enabled = true;
+			}
+		    }
+		}
+		else {
+		    add_enabled = true;
+
+		    // We have a new index, do we have something selected in the index list? if so, enable the replace button
+		    if (index_list.getSelectedIndex() != -1) {
+			replace_enabled = true;
+		    }
+		}
+	    }
+
+	    // We should now know the add_button state
+	    add_button.setEnabled(add_enabled);
+	    replace_button.setEnabled(replace_enabled);
+	}
+
+
+	private class AddListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		String name = name_textfield.getText();
+		if (!source_list.isNothingTicked() && name.length() != 0) {
+		    ArrayList sources = source_list.getTicked();
+		    Index index = new Index(level_combobox.getSelectedIndex(), sources);
+
+		    // Before we add the index to the model, we have to add the collection metadata for this
+		    CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    metadatum.setValue(name);
+
+		    // Finally, add the index
+		    addIndex(index, metadatum);
+		    index_list.setSelectedValue(index, true);
+		}
+	    }
+	}
+
+
+	/** Listens for selections within the list on the IndexManager controls, and if a change is detected enables, or disables, controls appropriately. */
+	private class IndexListListener
+	    implements ListSelectionListener
+	{
+	    /** This method is called whenever the source list selection changes. When it does we need to fill in the various parts of the list description panel
+	     * @param event A <strong>ListSelectionEvent</strong> containing further information about the list selection.
+	     */
+	    public void valueChanged(ListSelectionEvent event)
+	    {
+		if (event.getValueIsAdjusting()) {
+		    return;
+		}
+
+		Object value = index_list.getSelectedValue();
+		if (value == null) {
+		    move_down_button.setEnabled(false);
+		    move_up_button.setEnabled(false);
+		    remove_button.setEnabled(false);
+		    replace_button.setEnabled(false);
+		    set_default_button.setEnabled(false);
+		    return;
+		}
+
+		// Enable the buttons appropriately
+		remove_button.setEnabled(true);
+		set_default_button.setEnabled(default_index == null || !default_index.equals(value));
+		int i = index_list.getSelectedIndex();
+		int size = index_list.getModel().getSize();
+		if (i > 0) {
+		    move_up_button.setEnabled(true);
+		}
+		else {
+		    move_up_button.setEnabled(false);
+		}
+		if (i < size-1){
+		    move_down_button.setEnabled(true);
+		}
+		else {
+		    move_down_button.setEnabled(false);
+		}
+
+		// Need to fill in the rest of the bits
+		updateControlsWithSelectedIndex();		
+	    }
+	}
+
+	private class IndexListRenderer
+	    extends DefaultListCellRenderer {
+
+	    /** Return a component that has been configured to display the specified value. */
+	    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+		JLabel component = (JLabel) super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+		if(default_index != null && default_index.equals(value)) {
+		    component.setText(component.getText() + " " + Dictionary.get("CDM.IndexManager.Default_Index_Indicator"));
+		}
+		return component;
+	    }
+
+	}
+
+	private class LevelListener
+	    implements ItemListener {
+	    public void itemStateChanged(ItemEvent event) {
+		if(event.getStateChange() == ItemEvent.SELECTED) {
+		    // just want to validate the add button
+		    validateAddButton();
+
+		}
+	    }
+	}
+
+	private class MoveListener
+	    implements ActionListener {
+
+	    private boolean move_up;
+
+	    public MoveListener(boolean move_up) {
+		this.move_up = move_up;
+	    }
+
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the selected index
+		Index index = (Index) index_list.getSelectedValue();
+		if(index != null) {
+		    moveIndex(index, move_up);
+		    // Ensure the index that moved is still selected
+		    index_list.setSelectedValue(index, true);
+		    index = null;
+		}
+	    }
+	}
+
+
+	/** Listens for key presses within the name field, and enabled or disables controls as appropriate. */
+	private class NameListener
+	    implements DocumentListener
+	{
+	    public void changedUpdate(DocumentEvent e)
+	    {
+		validateAddButton();
+	    }
+
+	    public void insertUpdate(DocumentEvent e)
+	    {
+		validateAddButton();
+	    }
+
+	    public void removeUpdate(DocumentEvent e)
+	    {
+		validateAddButton();
+	    }
+	}
+
+
+	private class RemoveListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		int i = index_list.getSelectedIndex();
+		if (i != -1) {
+		    removeIndex((Index) index_list.getSelectedValue());
+		}
+		int size = index_list.getModel().getSize();
+		if (i == size) {
+		    i--;
+		}
+		index_list.setSelectedIndex(i);
+
+		// This will produce an event on the list, updating the other buttons
+		if (size == 0) {
+		    // We have removed the last index, should be able to add what's filled in currently, if valid
+		    validateAddButton();
+		}
+	    }
+	}
+
+
+	private class ReplaceListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		if (index_list.isSelectionEmpty()) {
+		    // This should never happen, but just in case...
+		    replace_button.setEnabled(false);
+		    return;
+		}
+
+		// We'll just remove the old one and add the new one
+		removeIndex((Index) index_list.getSelectedValue());
+		replace_button.setEnabled(false);
+		add_button.setEnabled(true);
+		add_button.doClick();
+		add_button.setEnabled(false);
+	    }
+	}
+
+
+	private class SetDefaultListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event) {
+		Index index = (Index) index_list.getSelectedValue();
+		if(index != null) {
+		    setDefault(index);
+		    // This should cause a repaint of just the desired row
+		    index_list.setSelectedValue(index, true);
+		}
+		set_default_button.setEnabled(false);
+	    }
+	}
+
+
+	private class SourceListListener
+	    implements ListSelectionListener
+	{
+	    public void valueChanged(ListSelectionEvent event)
+	    {
+		validateAddButton();
+	    }
+	}
+    }
+
+    private class MGPPControl
+	extends JPanel {
+
+	private GComboBox index_combobox;
+	private GComboBox level_combobox;
+	private JButton add_index_button;
+	private JButton add_all_button;
+	private JButton add_level_button;
+	private JButton move_index_down_button;
+	private JButton move_level_down_button;
+	private JButton move_index_up_button;
+	private JButton move_level_up_button;
+	private JButton replace_index_button;
+	private JButton remove_index_button;
+	private JButton remove_level_button;
+	private JLabel current_levels_label;
+	private JLabel current_indexes_label;
+	private JLabel index_label;
+	private JLabel index_name_label;
+	private JLabel level_label;
+	private JLabel level_name_label;
+	private JLabel move_index_down_label;
+	private JLabel move_level_down_label;
+	private JLabel move_index_up_label;
+	private JLabel move_level_up_label;
+	private JLabel title_label;
+	private JList current_levels_list;
+	private JList current_indexes_list;
+	private JTabbedPane tabbed_pane;
+	private JTextArea instructions_textarea;
+	private JTextField index_name_field;
+	private JTextField level_name_field;
+	
+	// when we are in mg mode, we want to remove the mgpp listener 
+	// the underlying model is the same between the two JLists, and
+	// both listeners get notified when the model changes, but the mgpp
+	// listener doesn't care about mg events
+	private ListSelectionListener mgpp_index_list_listener;
+	
+	public MGPPControl() {
+	    // Create Indexes
+	    JPanel indexes_panel = new JPanel();
+
+	    JPanel current_indexes_panel = new JPanel();
+
+	    current_indexes_label = new JLabel();
+	    Dictionary.registerText(current_indexes_label, "CDM.IndexManager.MGPP.Current_Indexes");
+	    current_indexes_list = new JList(model);
+	    current_indexes_list.setVisibleRowCount(5);
+	    current_indexes_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	    JPanel index_movement_panel = new JPanel();
+
+	    move_index_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_index_up_button.setEnabled(false);
+	    move_index_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_index_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_index_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_index_down_button.setEnabled(false);
+	    move_index_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_index_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    JPanel index_spacer_panel = new JPanel();
+	    JPanel index_body_panel = new JPanel();
+	    JPanel index_details_panel = new JPanel();
+	    JPanel index_labels_panel = new JPanel();
+
+	    index_name_label = new JLabel();
+	    Dictionary.registerText(index_name_label, "CDM.IndexManager.Index_Name");
+
+	    index_name_field = new JTextField();
+	    index_name_field.setPreferredSize(FIELD_SIZE);
+	    Dictionary.registerTooltip(index_name_field, "CDM.IndexManager.Index_Name_Tooltip");
+
+	    JPanel index_boxes_panel = new JPanel();
+
+	    index_label = new JLabel("CDM.IndexManager.MGPP.Index");
+	    Dictionary.registerText(index_label, "CDM.IndexManager.MGPP.Index");
+
+	    index_combobox = new GComboBox();
+	    index_combobox.setPreferredSize(FIELD_SIZE);
+	    index_combobox.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    index_combobox.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    index_combobox.setEditable(false);  // Warning: setting this to true opens a big can of worms
+	    index_combobox.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    index_combobox.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.registerTooltip(index_combobox, "CDM.IndexManager.MGPP.Index_Tooltip");
+
+	    JPanel index_button_panel = new JPanel();
+
+	    add_index_button = new GLIButton();
+	    add_index_button.setEnabled(false);
+	    add_index_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_index_button, "CDM.IndexManager.Add_Index", "CDM.IndexManager.Add_Index_Tooltip");
+
+	    add_all_button = new GLIButton();
+	    add_all_button.setEnabled(true);
+	    add_all_button.setMnemonic(KeyEvent.VK_L);
+	    Dictionary.registerBoth(add_all_button, "CDM.IndexManager.MGPP.Add_All_Metadata", "CDM.IndexManager.MGPP.Add_All_Metadata_Tooltip");
+
+	    replace_index_button = new GLIButton();
+	    replace_index_button.setEnabled(false);
+	    replace_index_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerBoth(replace_index_button, "CDM.IndexManager.MGPP.Replace_Index", "CDM.IndexManager.MGPP.Replace_Index_Tooltip");
+
+	    remove_index_button = new GLIButton();
+	    remove_index_button.setEnabled(false);
+	    remove_index_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(remove_index_button, "CDM.IndexManager.Remove_Index", "CDM.IndexManager.Remove_Index_Tooltip");
+
+	    JPanel index_empty_panel = new JPanel();
+
+	    // Connect Indexes
+	    EnableAddIndexListener index_eal = new EnableAddIndexListener();
+	    add_index_button.addActionListener(new AddIndexActionListener());
+	    add_all_button.addActionListener(new AddAllActionListener());
+	    
+	    // Create the index_list_listener but don't add in to the list - this gets done in gainFocus, we don't want it twice.
+	    mgpp_index_list_listener = new CurrentIndexesListSelectionListener();
+	    index_combobox.addActionListener(index_eal);
+	    ((JTextField)index_combobox.getEditor().getEditorComponent()).getDocument().addDocumentListener(index_eal);
+	    index_name_field.getDocument().addDocumentListener(index_eal);
+	    move_index_down_button.addActionListener(new MoveIndexDownListener());
+	    move_index_up_button.addActionListener(new MoveIndexUpListener());
+	    replace_index_button.addActionListener(new ReplaceIndexActionListener());
+	    remove_index_button.addActionListener(new RemoveIndexActionListener(index_eal));
+
+	    // Layout Indexes
+	    index_movement_panel.setLayout(new GridLayout(2,1));
+	    index_movement_panel.add(move_index_up_button);
+	    index_movement_panel.add(move_index_down_button);
+
+	    current_indexes_panel.setLayout(new BorderLayout());
+	    current_indexes_panel.add(current_indexes_label, BorderLayout.NORTH);
+	    current_indexes_panel.add(new JScrollPane(current_indexes_list), BorderLayout.CENTER);
+	    current_indexes_panel.add(index_movement_panel, BorderLayout.EAST);
+
+
+	    index_labels_panel.setLayout(new GridLayout(2,1));
+	    index_labels_panel.add(index_name_label);
+	    index_labels_panel.add(index_label);
+
+	    index_boxes_panel.setLayout(new GridLayout(2,1));
+	    index_boxes_panel.add(index_name_field);
+	    index_boxes_panel.add(index_combobox);
+
+	    index_details_panel.setLayout(new BorderLayout(5,0));
+	    index_details_panel.add(index_labels_panel, BorderLayout.WEST);
+	    index_details_panel.add(index_boxes_panel, BorderLayout.CENTER);
+
+	    index_button_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    index_button_panel.setLayout(new GridLayout(2,2,5,0));
+	    index_button_panel.add(add_index_button);
+	    index_button_panel.add(add_all_button);
+	    index_button_panel.add(replace_index_button);
+	    index_button_panel.add(remove_index_button);
+
+	    index_body_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    index_body_panel.setLayout(new BorderLayout());
+	    index_body_panel.add(index_details_panel, BorderLayout.CENTER);
+	    index_body_panel.add(index_button_panel, BorderLayout.SOUTH);
+
+	    index_spacer_panel.setLayout(new BorderLayout());
+	    index_spacer_panel.add(index_body_panel, BorderLayout.NORTH);
+	    index_spacer_panel.add(index_empty_panel, BorderLayout.CENTER);
+
+	    indexes_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    indexes_panel.setLayout(new BorderLayout());
+	    indexes_panel.add(current_indexes_panel, BorderLayout.NORTH);
+	    indexes_panel.add(index_spacer_panel, BorderLayout.CENTER);
+
+	    // Create Levels
+	    JPanel levels_panel = new JPanel();
+
+	    JPanel current_levels_panel = new JPanel();
+
+	    current_levels_label = new JLabel();
+	    Dictionary.registerText(current_levels_label, "CDM.IndexManager.MGPP.Current_Levels");
+
+	    current_levels_list = new JList(levels_model);
+	    current_levels_list.setVisibleRowCount(5);
+	    current_levels_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    JPanel level_movement_panel = new JPanel();
+
+	    move_level_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_level_up_button.setEnabled(false);
+	    move_level_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_level_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_level_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_level_down_button.setEnabled(false);
+	    move_level_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_level_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    JPanel level_spacer_panel = new JPanel();
+	    JPanel level_body_panel = new JPanel();
+	    JPanel level_details_panel = new JPanel();
+	    JPanel level_labels_panel = new JPanel();
+	    JPanel level_boxes_panel = new JPanel();
+
+	    level_name_label = new JLabel();
+	    Dictionary.registerText(level_name_label, "CDM.IndexManager.MGPP.Level_Name");
+
+	    level_name_field = new JTextField();
+	    level_name_field.setPreferredSize(FIELD_SIZE);
+	    Dictionary.registerTooltip(level_name_field, "CDM.IndexManager.MGPP.Level_Name_Tooltip");
+
+
+	    level_label = new JLabel();
+	    Dictionary.registerText(level_label, "CDM.IndexManager.MGPP.Level");
+
+	    level_combobox = new GComboBox(Index.LEVEL);
+	    level_combobox.setPreferredSize(FIELD_SIZE);
+	    level_combobox.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    level_combobox.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    level_combobox.setEditable(false);
+	    level_combobox.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    level_combobox.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.registerTooltip(level_combobox, "CDM.IndexManager.Level_Tooltip");
+
+	    JPanel level_button_panel = new JPanel();
+
+	    add_level_button = new GLIButton("CDM.IndexManager.MGPP.Add_Level");
+	    add_level_button.setEnabled(false);
+	    add_level_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_level_button, "CDM.IndexManager.MGPP.Add_Level", "CDM.IndexManager.MGPP.Add_Level_Tooltip");
+
+	    remove_level_button = new GLIButton("CDM.IndexManager.MGPP.Remove_Level");
+	    remove_level_button.setEnabled(false);
+	    remove_level_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(remove_level_button, "CDM.IndexManager.MGPP.Remove_Level", "CDM.IndexManager.MGPP.Remove_Level_Tooltip");
+
+	    JPanel level_empty_panel = new JPanel();
+
+	    // Connect Levels
+	    EnableAddLevelListener level_eal = new EnableAddLevelListener();
+	    add_level_button.addActionListener(new AddLevelActionListener());
+	    current_levels_list.addListSelectionListener(new CurrentLevelsListSelectionListener());
+	    level_combobox.addActionListener(level_eal);
+	    ((JTextField)level_combobox.getEditor().getEditorComponent()).getDocument().addDocumentListener(level_eal);
+	    level_name_field.getDocument().addDocumentListener(level_eal);
+	    move_level_down_button.addActionListener(new MoveLevelDownListener());
+	    move_level_up_button.addActionListener(new MoveLevelUpListener());
+	    remove_level_button.addActionListener(new RemoveLevelActionListener(level_eal));
+	    // Layout Levels
+
+	    level_movement_panel.setLayout(new GridLayout(2,1));
+	    level_movement_panel.add(move_level_up_button);
+	    level_movement_panel.add(move_level_down_button);
+
+	    current_levels_panel.setLayout(new BorderLayout());
+	    current_levels_panel.add(current_levels_label, BorderLayout.NORTH);
+	    current_levels_panel.add(new JScrollPane(current_levels_list), BorderLayout.CENTER);
+	    current_levels_panel.add(level_movement_panel, BorderLayout.EAST);
+
+	    level_labels_panel.setLayout(new GridLayout(2,1));
+	    level_labels_panel.add(level_name_label);
+	    level_labels_panel.add(level_label);
+
+	    level_boxes_panel.setLayout(new GridLayout(2,1));
+	    level_boxes_panel.add(level_name_field);
+	    level_boxes_panel.add(level_combobox);
+
+	    level_details_panel.setLayout(new BorderLayout(5,0));
+	    level_details_panel.add(level_labels_panel, BorderLayout.WEST);
+	    level_details_panel.add(level_boxes_panel, BorderLayout.CENTER);
+
+	    level_button_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    level_button_panel.setLayout(new GridLayout(1,2,5,0));
+	    level_button_panel.add(add_level_button);
+	    level_button_panel.add(remove_level_button);
+
+	    level_body_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    level_body_panel.setLayout(new BorderLayout());
+	    level_body_panel.add(level_details_panel, BorderLayout.CENTER);
+	    level_body_panel.add(level_button_panel, BorderLayout.SOUTH);
+
+	    level_spacer_panel.setLayout(new BorderLayout());
+	    level_spacer_panel.add(level_body_panel, BorderLayout.NORTH);
+	    level_spacer_panel.add(level_empty_panel, BorderLayout.CENTER);
+
+	    levels_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    levels_panel.setLayout(new BorderLayout());
+	    levels_panel.add(current_levels_panel, BorderLayout.NORTH);
+	    levels_panel.add(level_spacer_panel, BorderLayout.CENTER);
+
+	    // Create General
+	    JPanel header_panel = new JPanel();
+
+	    title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title_label, "CDM.IndexManager.MGPP.Title");
+
+	    instructions_textarea = new JTextArea();
+	    instructions_textarea.setEditable(false);
+	    instructions_textarea.setLineWrap(true);
+	    instructions_textarea.setRows(6);
+	    instructions_textarea.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions_textarea, "CDM.IndexManager.MGPP.Instructions");
+
+	    tabbed_pane = new JTabbedPane();
+
+	    // Layout General
+	    header_panel.setBorder(BorderFactory.createEmptyBorder(0,0,2,0));
+	    header_panel.setLayout(new BorderLayout());
+	    header_panel.add(title_label, BorderLayout.NORTH);
+	    header_panel.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+
+	    tabbed_pane.add(Dictionary.get("CDM.IndexManager.MGPP.Indexes"), indexes_panel);
+	    tabbed_pane.add(Dictionary.get("CDM.IndexManager.MGPP.Levels"), levels_panel);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_panel, BorderLayout.NORTH);
+	    add(tabbed_pane, BorderLayout.CENTER);
+	}
+
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	    // add in the mgpp listener
+	    current_indexes_list.addListSelectionListener(mgpp_index_list_listener);
+	    // Reload the assigned indexes list.
+	    index_combobox.removeAllItems();
+	    index_combobox.addItem(CollectionConfiguration.ALLFIELDS_STR);
+	    index_combobox.addItem(CollectionConfiguration.METADATA_STR);
+	    index_combobox.addItem(CollectionConfiguration.TEXT_STR);
+	    ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+	    for(int i = 0; i < every_metadata_set_element.size(); i++) {
+		index_combobox.addItem(every_metadata_set_element.get(i));
+	    }
+	    // Ensure the level manager has at least documents assigned
+	    if(levels_model.getSize() == 0) {
+		Level level = new Level(CollectionConfiguration.DOCUMENT_STR);
+		// Create new metadatum
+		CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + CollectionConfiguration.DOCUMENT_STR);
+		metadatum.setValue(CollectionConfiguration.DOCUMENT_STR);
+		// Assign new level
+		addLevel(level, metadatum);	    
+		level = null;
+	    }
+
+	    // refresh
+	    current_indexes_list.updateUI();
+	}
+	
+	public void removeIndexListListener() {
+
+	    current_indexes_list.removeListSelectionListener(mgpp_index_list_listener);
+	}
+	
+	private class AddIndexActionListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the name
+		String name = index_name_field.getText();
+		// Retrieve the source
+		Object source = index_combobox.getSelectedItem();
+
+		// If this object isn't yet in the combobox add it.
+		if(index_combobox.getSelectedIndex() == -1) {
+		    index_combobox.insertItemAt(source, index_combobox.getItemCount());
+		}
+		// Create new index
+		ArrayList sources = new ArrayList();
+		sources.add(source);
+		Index index = new Index(sources);
+		if(!model.contains(index)) {
+		    // Create new metadatum
+		    CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    metadatum.setValue(name);
+		    // Assign new index
+		    addIndex(index, metadatum);
+		    current_indexes_list.setSelectedValue(index, true);
+		}
+		// Done. Disable add
+		add_index_button.setEnabled(false);
+	    }
+	}
+
+	private class AddAllActionListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event) {
+		for(int i = 0; i < index_combobox.getItemCount(); i++) {
+		    Object source = index_combobox.getItemAt(i);
+		    // Create new index
+		    ArrayList sources = new ArrayList();
+		    sources.add(source);
+		    Index index = new Index(sources);
+		    sources = null;
+		    if(!model.contains(index)) {
+			// Determine the metadatum value
+			String name = source.toString();
+			if(name.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+			    name = name.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+			}
+			// Create new metadatum
+			CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + index.getID());
+			metadatum.setValue(name);
+			name = null;
+			// Assign new index
+			addIndex(index, metadatum);
+		    }
+		    source = null;
+		    index = null;
+		}
+		// Done. Disable add
+		add_index_button.setEnabled(false);
+	    }
+	}
+
+	private class AddLevelActionListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the name
+		String name = level_name_field.getText();
+		// Retrieve the source
+		String source = (String)level_combobox.getSelectedItem();
+		Level level = new Level(source);
+		// Create new metadatum
+		CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + source);
+		metadatum.setValue(name);
+		// Assign new level
+		int size = levels_model.getSize();
+		addLevel(level, metadatum);
+		current_levels_list.setSelectedValue(level, true);
+		add_level_button.setEnabled(false);
+	    }
+	}
+
+	private class CurrentIndexesListSelectionListener
+	    implements ListSelectionListener {
+	    public void valueChanged(ListSelectionEvent event) {
+		if(event.getValueIsAdjusting()) {
+		    return;
+		}
+		
+		Index index = (Index)current_indexes_list.getSelectedValue();
+		if(index != null) {
+		    String full_text = index.toString();
+		    if(full_text.indexOf("\"") != -1) {
+			index_name_field.setText(index.getName());
+		    }
+		    ArrayList sources = index.getSources();
+		    index_combobox.setSelectedItem(sources.get(0));
+		    
+		    
+		    move_index_down_button.setEnabled((model.indexOf(index) < model.getSize() - 1));
+		    move_index_up_button.setEnabled((model.indexOf(index) > 0));
+		    remove_index_button.setEnabled(true);
+		}
+		else {
+		    move_index_down_button.setEnabled(false);
+		    move_index_up_button.setEnabled(false);
+		    remove_index_button.setEnabled(false);
+		}
+	    }	    
+	}
+
+	private class CurrentLevelsListSelectionListener
+	    implements ListSelectionListener {
+	    public void valueChanged(ListSelectionEvent event) {
+		
+		if(event.getValueIsAdjusting()) {
+		    return;
+		}
+		
+		Level level = (Level)current_levels_list.getSelectedValue();
+		if(level != null) {
+		    String full_text = level.toString();
+		    if(full_text.indexOf("\"") != -1) {
+			level_name_field.setText(level.getName());
+		    }
+		    String level_id = level.getLevel();
+		    level_combobox.setSelectedItem(level_id);
+		    move_level_down_button.setEnabled((levels_model.indexOf(level) < levels_model.getSize() - 1));
+		    move_level_up_button.setEnabled((levels_model.indexOf(level) > 0));
+		    remove_level_button.setEnabled(true);
+		}
+		else {
+		    move_level_down_button.setEnabled(false);
+		    move_level_up_button.setEnabled(false);
+		    remove_level_button.setEnabled(false);
+		}
+	    }
+	}
+
+	private class EnableAddIndexListener
+	    implements ActionListener, DocumentListener {
+	    /** Called whenever a selection action occurs on the combobox.
+	     * @param  event an ActionEvent containing information about the selection event
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		validateAddButtonIndex();
+	    }
+
+	    /** Gives notification that an attribute or set of attributes changed.
+	     * @param  event a DocumentEvent containing information about the text changed
+	     */
+	    public void changedUpdate(DocumentEvent event) {
+		validateAddButtonIndex();
+	    }
+
+	    /** Gives notification that there was an insert into the document.
+	     * @param  event a DocumentEvent containing information about the text added
+	     */
+	    public void insertUpdate(DocumentEvent event) {
+		validateAddButtonIndex();
+	    }
+
+	    /** Gives notification that a portion of the document has been removed.
+	     * @param  event a DocumentEvent containing information about the text removed
+	     */
+	    public void removeUpdate(DocumentEvent event) {
+		validateAddButtonIndex();
+	    }
+
+	    /** Change the enable state of the add button depending on the current value in the search type combobox. */
+	    public void validateAddButtonIndex() {
+		String name = index_name_field.getText();
+		Object selected_object = index_combobox.getSelectedItem();
+		boolean add_enabled = false;
+		boolean replace_enabled = false;
+
+		if(name.length() > 0 && selected_object != null) {
+		    // Unfortunately we have to generate a valid id
+		    String id = null;
+		    // Special cases
+		    if (selected_object instanceof String && (selected_object.equals(CollectionConfiguration.ALLFIELDS_STR) || selected_object.equals(CollectionConfiguration.METADATA_STR) || selected_object.equals(CollectionConfiguration.TEXT_STR))) {
+			id = (String) selected_object;
+		    }
+		    else if (selected_object instanceof MetadataElement) {
+			id = ((MetadataElement) selected_object).getFullName();
+		    }
+		    else {
+			System.err.println("Internal Error: Unexpected selected_object in IndexManager.");
+			return;
+		    }
+		    if(id.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+			id = id.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+		    }
+		    
+		    Index index = getIndex(id);
+		    if(index == null) {
+			add_enabled = true;
+			// we have a new index. is something selected? if so, we can replace
+			if (current_indexes_list.getSelectedIndex() != -1) {
+			    replace_enabled = true;
+			}
+		    }
+		    else {
+			// the index is already there
+			add_enabled = false;
+			// but if the name is changed, we can do a replace
+			if(!name.equals(index.getName())) {
+			    replace_enabled = true;
+			}
+			
+		    }
+		}
+		else {
+		    add_enabled = false;
+		    replace_enabled = false;
+		    
+		}
+
+		// We should now know the button state
+		add_index_button.setEnabled(add_enabled);
+		replace_index_button.setEnabled(replace_enabled);
+		
+	    }
+	}
+
+	private class EnableAddLevelListener
+	    implements ActionListener, DocumentListener {
+	    /** Called whenever a selection action occurs on the combobox.
+	     * @param  event an ActionEvent containing information about the selection event
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		validateAddButtonLevel();
+	    }
+
+	    /** Gives notification that an attribute or set of attributes changed.
+	     * @param  event a DocumentEvent containing information about the text changed
+	     */
+	    public void changedUpdate(DocumentEvent event) {
+		validateAddButtonLevel();
+	    }
+
+	    /** Gives notification that there was an insert into the document.
+	     * @param  event a DocumentEvent containing information about the text added
+	     */
+	    public void insertUpdate(DocumentEvent event) {
+		validateAddButtonLevel();
+	    }
+
+	    /** Gives notification that a portion of the document has been removed.
+	     * @param  event a DocumentEvent containing information about the text removed
+	     */
+	    public void removeUpdate(DocumentEvent event) {
+		validateAddButtonLevel();
+	    }
+    
+	    /** Change the enable state of the add button depending on the current value in the search type combobox. */
+	    public void validateAddButtonLevel() {
+		String name = level_name_field.getText();
+		Object selected_object = level_combobox.getSelectedItem();
+		if(name.length() > 0 && selected_object != null) {
+		    add_level_button.setEnabled(getLevel((String)selected_object) == null);
+		}
+		else {
+		    add_level_button.setEnabled(false);
+		}
+	    }
+	}
+
+	private class MoveIndexDownListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the first selected item
+		Index index = (Index) current_indexes_list.getSelectedValue();
+		moveIndex(index, false);
+		current_indexes_list.setSelectedValue(index, true);
+	    }
+	}
+
+	private class MoveLevelDownListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the first selected item
+		Level level = (Level) current_levels_list.getSelectedValue();
+		moveLevel(level, false);
+		current_levels_list.setSelectedValue(level, true);
+	    }
+	}
+
+	private class MoveIndexUpListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the first selected item
+		Index index = (Index) current_indexes_list.getSelectedValue();
+		moveIndex(index, true);
+		current_indexes_list.setSelectedValue(index, true);
+	    }
+	}
+
+	private class MoveLevelUpListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the first selected item
+		Level level = (Level) current_levels_list.getSelectedValue();
+		moveLevel(level, true);
+		current_levels_list.setSelectedValue(level, true);
+	    }
+	}
+
+	/** Replace really only replaces the string. */
+	private class ReplaceIndexActionListener
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event) {
+		// We'll just remove the old one and add the new one
+		removeIndex((Index) current_indexes_list.getSelectedValue());
+		replace_index_button.setEnabled(false);
+		add_index_button.setEnabled(true);
+		add_index_button.doClick(); // this adds the new one
+		add_index_button.setEnabled(false);
+		
+		//Object[] selected_objects = current_indexes_list.getSelectedValues();
+		//if(selected_objects.length == 1) {
+		Index index = (Index)current_indexes_list.getSelectedValue();
+		if (index != null) {
+		//Index index = (Index) selected_objects[0];
+		    // Remove old name
+		    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    // Enter new name
+		    String name = index_name_field.getText();
+		    // Create new metadatum
+		    CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    //index = null;
+		    metadatum.setValue(name);
+		    name = null;
+		    // Assign new index
+		    CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+		    metadatum = null;
+		}
+		//current_indexes_list.setSelectedValue(selected_objects[0], true);
+		current_indexes_list.setSelectedValue(index, true);
+		// Done. Disable add
+		add_index_button.setEnabled(false);
+		replace_index_button.setEnabled(false);
+	    }
+	}
+
+	private class RemoveIndexActionListener
+	    implements ActionListener {
+
+	    private EnableAddIndexListener eal = null;
+
+	    public RemoveIndexActionListener(EnableAddIndexListener eal) {
+		this.eal = eal;
+	    }
+
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the selected items
+		//Object[] selected_objects = current_indexes_list.getSelectedValues();
+		// Clear selection
+		//current_indexes_list.clearSelection();
+		//for(int i = 0; i < selected_objects.length; i++) {
+		//    Index index = (Index) selected_objects[i];
+		    // Remove any related metadata
+		//    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    // Remove the index
+		//    removeIndex(index);
+		//}
+		// Disable remove button
+		//remove_index_button.setEnabled(false);
+		// Check if add should be reenabled
+		//eal.validateAddButtonIndex();
+		int i = current_indexes_list.getSelectedIndex();
+		if (i != -1) {
+		    Index index = (Index) current_indexes_list.getSelectedValue();
+		    // Remove any related metadata
+		    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + index.getID());
+		    removeIndex(index);
+		    
+		//if (i != -1) {
+		//removeIndex((Index) index_list.getSelectedValue());
+		    //}
+		    int size = current_indexes_list.getModel().getSize();
+		    if (i == size) {
+			i--;
+		    }
+		    current_indexes_list.setSelectedIndex(i);
+		}
+		if (i==-1) {
+		    // empty list, disable remove button
+		    remove_index_button.setEnabled(false);
+		}
+		// Check if add should be reenabled
+		eal.validateAddButtonIndex();
+
+		// This will produce an event on the list, updating the other buttons
+		//if (size == 0) {
+		    // We have removed the last index, should be able to add what's filled in currently, if valid
+		// validateAddButton();
+		//	}
+	    }
+	}
+
+	private class RemoveLevelActionListener
+	    implements ActionListener {
+
+	    private EnableAddLevelListener eal = null;
+
+	    public RemoveLevelActionListener(EnableAddLevelListener eal) {
+		this.eal = eal;
+	    }
+
+	    public void actionPerformed(ActionEvent event) {
+		
+		int i = current_levels_list.getSelectedIndex();
+		if (i != -1) {
+		    Level level = (Level) current_levels_list.getSelectedValue();
+		    CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + level.getLevel());
+		    removeLevel(level);
+		    if (i == current_levels_list.getModel().getSize()) {
+			i--;
+		    }
+		    current_levels_list.setSelectedIndex(i);
+		}
+		// Retrieve the selected items
+		//Object[] selected_objects = current_levels_list.getSelectedValues();
+		// Clear selection
+		//current_levels_list.clearSelection();
+		//for(int i = 0; i < selected_objects.length; i++) {
+		
+		    //Level level = (Level) selected_objects[i];
+		    // Remove any related metadata
+		    //CollectionDesignManager.collectionmeta_manager.removeMetadata(CollectionConfiguration.STOP_CHARACTER + level.getLevel());
+		    // Remove the index
+		    //removeLevel(level);
+		//}
+		
+		if (i==-1) {
+		    // Disable remove button
+		    remove_level_button.setEnabled(false);
+		}
+		// If there are no levels left, put document back in
+// 		if(levels_model.getSize() == 0) {
+// 			Level level = new Level(CollectionConfiguration.DOCUMENT_STR);
+// 			// Create new metadatum
+// 			CollectionMeta metadatum = new CollectionMeta(CollectionConfiguration.STOP_CHARACTER + CollectionConfiguration.DOCUMENT_STR);
+// 			metadatum.setValue(CollectionConfiguration.DOCUMENT_STR);
+// 			// Assign new level
+// 			addLevel(level, metadatum);	    
+// 			level = null;
+// 		}
+		// Check if add should be reenabled
+		eal.validateAddButtonLevel();
+	    }
+	}
+    } // MGPPControls
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Language.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Language.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Language.java	(revision 21112)
@@ -0,0 +1,140 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+
+import org.w3c.dom.*;
+
+/** A pretty unexciting extension of a two character string, in that it has a field which details if its the default language.
+* @author John Thompson, Greenstone Digital Library, University of Waikato
+* @version 2.1
+*/
+public class Language
+    implements Comparable, DOMProxyListEntry {
+    /** The Element this language entry is based upon. */
+    private Element element = null;
+    /** The two character code for this language. */
+    private String code = null;
+    /** The name of this language. */
+    private String name = null;
+ 
+    public Language() {
+    }
+
+    public Language(Element element) {
+	this.element = element;
+    }
+
+    /** Constructor for a brand new language. */
+    public Language(String code) {
+	this.code = code;
+	// Create the new element
+	element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.LANGUAGE_ELEMENT);
+	element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, code);
+	element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, CollectionConfiguration.TRUE_STR);
+    }
+
+    /** Method to compare two languages for ordering purposes.
+     * @param object the other language as an Object
+     * @return an int which indicates the order between this language and the given one: < 0, 0 or > 0 for before, equal to or after respectively
+     */
+    public int compareTo(Object object) {
+	return toString().compareTo(object.toString());
+    }
+
+    public DOMProxyListEntry create(Element element) {
+	return new Language(element);
+    }
+
+    /** Method to test for the equality of two languages.
+     * @param object The other language as an <strong>Object</strong>.
+     * @return <i>true</i> if the languages are equal, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	//return (compareTo(object) == 0);
+	// two langs are equal if their codes are equal
+	if (object instanceof Language) {
+	    return getCode().equals(((Language)object).getCode());
+	}
+	return false;
+    }
+
+    /** Method to retrieve the code of this language.
+     * @return A <strong>String</strong> representing the two letter code.
+     */
+    public String getCode() {
+	if(code == null && element != null) {
+	    code = element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	}
+	return code;
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    public String getName() {
+	if(name == null) {
+	    String code = getCode();
+	    name = CollectionDesignManager.language_manager.getLanguageName(code);
+	}
+	return name;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.TRUE_STR));
+    }
+
+    public void setAssigned(boolean value) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (value ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setCode(String new_code) {
+	code = new_code;
+	// Set element
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, new_code);
+	}
+	// Reset name
+	name = null;
+    }
+
+    public void setElement(Element new_element) {
+	element = new_element;
+	code = null;
+	name = null;
+    }
+
+    /** Method to display the language code.
+     * @return A <strong>String</strong> representing the language code.
+     */
+    public String toString() {
+	return getName();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageListCellRenderer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageListCellRenderer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageListCellRenderer.java	(revision 21112)
@@ -0,0 +1,43 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      11/07/03
+ * Revised:      
+ **************************************************************************************/
+import java.awt.*;
+import javax.swing.*;
+
+/** Provides a renderer which transforms a language code (such as en or fr) into its appropriate name (English and French respectively) before generating the rubberstamping component. */
+public class LanguageListCellRenderer
+    extends DefaultListCellRenderer {
+    
+    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+	return super.getListCellRendererComponent(list, CollectionDesignManager.language_manager.getLanguageName((String)value), index, isSelected, cellHasFocus);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/LanguageManager.java	(revision 21112)
@@ -0,0 +1,570 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class manages the language commands, remembering both a list of languages to build indexes in, plus the default language.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class LanguageManager
+    extends DOMProxyListModel {
+
+    static public Document LANGUAGES_DOCUMENT = XMLTools.parseXMLFile("xml/languages.xml", true);
+
+    static final private Dimension COMPONENT_SIZE = new Dimension(125,25);
+
+    /** The visual controls for this manager. */
+    private Control controls = null;
+    /** A reference to this class as a model, for the inner controls class. */
+    private DOMProxyListModel model = null;
+    /** A hashtable of code->name mappings of known languages. */
+    private LinkedHashMap known_languages = null;
+    /** The default language object. */
+    private Language default_language = null;
+ 
+   /** Constructor. */
+    public LanguageManager(Element languages_element) {
+	super(languages_element, CollectionConfiguration.LANGUAGE_ELEMENT, new Language());
+	
+	DebugStream.println("LanguageManager: " + getSize() + " languages parsed.");
+
+	this.model = this;
+	// Retrieve the default language
+	NodeList default_language_elements = CollectionDesignManager.collect_config.getDocumentElement().getElementsByTagName(CollectionConfiguration.LANGUAGE_DEFAULT_ELEMENT);
+	if(default_language_elements.getLength() > 0) {
+	    default_language = new Language((Element)default_language_elements.item(0));
+	}
+	// Load a series of code->language mappings into known_languages, by reading from the 'languages.xml' file, which is essentially a subset of the ISO 639 Standard.
+	known_languages = new LinkedHashMap();
+	/*
+	  try {
+	    File in_file = new File("languages.dat");
+	    FileReader in_reader = new FileReader(in_file);
+	    BufferedReader in = new BufferedReader(in_reader);
+	    String entry = null;
+	    while((entry = in.readLine()) != null) {
+		if(!entry.startsWith("#")) {
+		    StringTokenizer tokenizer = new StringTokenizer(entry);
+		    if(tokenizer.countTokens() >= 2) {
+			String name = tokenizer.nextToken();
+			String code = tokenizer.nextToken().toLowerCase();
+			known_languages.put(code, name);
+		    }
+		}
+	    }
+	    in.close();
+	    in_reader.close();
+	    in = null;
+	    in_reader = null;
+	    in_file = null;
+	}
+	catch (Exception error) {
+	    error.printStackTrace();
+	}
+	*/
+	NodeList language_elements = LANGUAGES_DOCUMENT.getDocumentElement().getElementsByTagName(CollectionConfiguration.LANGUAGE_ELEMENT);
+	for(int i = 0; i < language_elements.getLength(); i++) {
+	    Element language_element = (Element) language_elements.item(i);
+	    String code = language_element.getAttribute(CollectionConfiguration.CODE_ATTRIBUTE);
+	    String name = language_element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	    known_languages.put(code.toLowerCase(), name);
+	    name = null;
+	    code = null;
+	    language_element = null;
+	}
+    }
+
+    /** Method to add a new language.
+     * @param language The <strong>Language</strong> to add.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void addLanguage(Language language) {
+	if(!contains(language)) {
+	    Element element = language.getElement();
+	    append(language);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+	known_languages.clear();
+	known_languages = null;
+	default_language = null;
+    }
+
+    /** Method to retrieve the control for this manager.
+     * @return the Control for editing the language partitions
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    // Build controls
+	    controls = new LanguageControl();
+	}
+	return controls;
+    }
+
+    /** Method to retrieve the default language code.
+     * @return A <strong>Language</strong> containing a two letter code.
+     */
+    /* private Language getDefaultLanguage() {
+	// If no default is set...
+	if(default_language != null && default_language.isAssigned()) {
+	    return default_language;
+	}
+	return null;
+	} */
+
+    /** Method to retrieve a certain language object by its code.
+     * @param code The two letter code of a language, as a <strong>String</strong>.
+     * @return The <strong>Language</strong> that matches the given code, or <i>null</i> if no such language exists.
+     */
+    public Language getLanguage(String code) {
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    Language language = (Language) getElementAt(i);
+	    if(language.getCode().equals(code)) {
+		return language;
+	    }
+	}
+	return null;
+    }
+
+    public ArrayList getLanguages() {
+	return children();
+    }
+
+    /** Method to return a list of the known language codes.
+     * @return an ArrayList containing the series of known language codes as per the languages.dat file
+     */
+    public ArrayList getLanguageCodes() {
+	return new ArrayList(known_languages.keySet());
+    }
+
+    public String getLanguageName(String code) {
+	return (String) known_languages.get(code);
+    }
+
+   private int moveLanguage(Language lang, boolean move_up)
+    {
+	// Determine the current position of the language
+	int position = indexOf(lang);
+	int new_position;
+
+	// Attempt to move the language up
+	if (move_up) {
+	    // Check it's not already at the top
+	    if (position == 0) {
+		return position;
+	    }
+
+	    // This automatically removes the language first, as an Element can only exist once in a particular document
+	    new_position = position - 1;
+	    addBefore(lang, (Language) getElementAt(new_position));
+	}
+
+	// Attempt to move the language down
+	else {
+	    // Check it's not already at the bottom
+	    if (position == (getSize()) - 1) {
+		return position;
+	    }
+
+	    // This automatically removes the language first, as an Element can only exist once in a particular document
+	    new_position = position + 1;
+	    addAfter(lang, (Language) getElementAt(new_position));
+	}
+
+	// Schedule the collection for saving
+	Gatherer.c_man.configurationChanged();
+	return new_position;
+    }
+
+    /** Method to remove a certain language.
+     * @param language The <strong>Language</strong> to remove.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void removeLanguage(Language language) {
+	remove(language);
+	if(default_language != null && default_language.equals(language)) {
+	    default_language = null;
+	}
+	Gatherer.c_man.configurationChanged();
+    }
+
+    /** Method to set the default language.
+     * @param language The <strong>Language</strong> to use as a default, or <i>null</i> for no default.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    public void setDefault(Language language) {
+	if(language != null) {
+	    if(default_language == null) {
+		// Create the default index element, and place immediately after indexes element.
+		Element default_language_element = root.getOwnerDocument().createElement(CollectionConfiguration.LANGUAGE_DEFAULT_ELEMENT);
+		default_language = new Language(default_language_element);
+		Node target_node = CollectionConfiguration.findInsertionPoint(default_language_element);
+		if(target_node != null) {
+		    root.getOwnerDocument().getDocumentElement().insertBefore(default_language_element, target_node);
+		}
+		else {
+		    root.getOwnerDocument().getDocumentElement().appendChild(default_language_element);
+		}
+	    }
+	    default_language.setAssigned(true);
+	    default_language.setCode(language.getCode());
+	}
+	else {
+	    if(default_language != null) {
+		default_language.setAssigned(false);
+	    }
+	}
+	Gatherer.c_man.configurationChanged();
+    }
+
+
+    /** This class represents the visual component of the Language Manager. */
+    private class LanguageControl
+	extends JPanel 
+	implements Control {
+	/** The button to add a new language support. */
+	private JButton add_button = null;
+	/** The button to remove a supported language. */
+	private JButton remove_button = null;
+	/** button to move a language up in the list */
+	private JButton move_down_button;
+	/** button to move a language down in the list */
+	private JButton move_up_button;
+	/** The button to set the current language as the default one. */
+	private JButton set_default_button = null;
+	/** A combobox listing the available supported languages. */
+	private JComboBox language_combobox = null;
+	/** A list of currently supported languages. */
+	private JList selected_languages_list = null;
+	/** Constructor.
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.AddListener
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.ClearDefaultListener
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.ListListener
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.RemoveListener
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.SelectorListener
+	 * @see org.greenstone.gatherer.cdm.LanguageManager.LanguageControl.SetDefaultListener
+	 */
+	public LanguageControl() {
+	    super();
+	    // Creation.
+	    JPanel center_panel = new JPanel();
+
+	    JLabel selected_languages_list_label = new JLabel();
+	    Dictionary.registerText(selected_languages_list_label, "CDM.LanguageManager.Assigned_Languages");
+	    selected_languages_list = new JList(model);
+	    selected_languages_list.setCellRenderer(new MyLanguageListCellRenderer());
+	    selected_languages_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    selected_languages_list.setVisibleRowCount(5);
+	    JPanel details_panel = new JPanel();
+
+	    JPanel control_panel = new JPanel();
+	    JLabel selector_label = new JLabel();
+	    Dictionary.registerText(selector_label, "CDM.LanguageManager.Selector");
+
+	    language_combobox = new JComboBox(getLanguageCodes().toArray());
+	    language_combobox.setPreferredSize(COMPONENT_SIZE);
+	    language_combobox.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    language_combobox.setRenderer(new LanguageListCellRenderer());
+	    Dictionary.registerTooltip(language_combobox, "CDM.LanguageManager.Selector_Tooltip");
+
+	    JPanel movement_pane = new JPanel();
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    set_default_button = new GLIButton();
+	    set_default_button.setEnabled(false);
+	    set_default_button.setMnemonic(KeyEvent.VK_S);
+	    Dictionary.registerBoth(set_default_button, "CDM.LanguageManager.Set_Default", "CDM.LanguageManager.Set_Default_Tooltip");
+
+	    JPanel button_panel = new JPanel();
+
+	    add_button = new GLIButton();
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.LanguageManager.Add", "CDM.LanguageManager.Add_Tooltip");
+	    if (getLanguage((String)language_combobox.getItemAt(0))==null) {
+		add_button.setEnabled(true);
+	    } else {
+		add_button.setEnabled(false);
+	    }
+	    remove_button = new GLIButton();
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    remove_button.setEnabled(false);
+	    Dictionary.registerBoth(remove_button, "CDM.LanguageManager.Remove", "CDM.LanguageManager.Remove_Tooltip");
+
+	    // Set up and connect listeners.
+	    add_button.addActionListener(new AddListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_down_button.addActionListener(new MoveListener(false));
+	    move_down_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_up_button.addActionListener(new MoveListener(true));
+	    move_up_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    language_combobox.addActionListener(new SelectorListener());
+	    set_default_button.addActionListener(new SetDefaultListener());
+	    set_default_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    selected_languages_list.addListSelectionListener(new ListListener());
+
+	    // Layout components
+	    movement_pane.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+	    movement_pane.setLayout(new GridLayout(3,1));
+	    movement_pane.add(move_up_button);
+	    movement_pane.add(move_down_button);
+	    movement_pane.add(set_default_button);
+
+	    control_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    control_panel.setLayout(new BorderLayout());
+	    control_panel.add(selector_label, BorderLayout.WEST);
+	    control_panel.add(language_combobox, BorderLayout.CENTER);
+
+	    center_panel.setLayout(new BorderLayout());
+	    center_panel.add(selected_languages_list_label, BorderLayout.NORTH);
+	    center_panel.add(new JScrollPane(selected_languages_list), BorderLayout.CENTER);
+	    center_panel.add(movement_pane, BorderLayout.EAST);
+	    center_panel.add(control_panel, BorderLayout.SOUTH);
+
+	    button_panel.setLayout(new GridLayout(1,2));
+	    button_panel.add(add_button);
+	    button_panel.add(remove_button);
+// 	    button_panel.add(set_button);
+// 	    button_panel.add(clear_button);
+
+	    //when we go to a multi list, remove inner_panel and empty_panel
+	    JPanel empty_panel = new JPanel();
+	    JPanel inner_panel = new JPanel();
+
+	    inner_panel.setLayout(new BorderLayout());
+	    inner_panel.add(center_panel, BorderLayout.CENTER);
+	    inner_panel.add(button_panel, BorderLayout.SOUTH);
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    //add(center_panel, BorderLayout.CENTER);
+	    //add(button_panel, BorderLayout.SOUTH);
+	    add(inner_panel, BorderLayout.NORTH);
+	    add(empty_panel, BorderLayout.CENTER);
+	}
+	
+	/** Destructor.	*/
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	}
+
+	public void loseFocus() {
+	}
+
+	/** Listens for actions apon the 'add' button in the LanguageManager controls, and if detected calls the add method of the manager with a newly created language. */
+	private class AddListener
+	    implements ActionListener {
+	    /** Add a new language support.
+	     * @param event an ActionEvent
+	     * @see org.greenstone.gatherer.cdm.Language
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		String language_code = (String) language_combobox.getSelectedItem();
+		if(language_code != null) {
+		    addLanguage(new Language(language_code));
+		}
+		// add some collectionmeta
+		String lang_name = getLanguageName(language_code);
+		CollectionMeta metadatum = new CollectionMeta("." + language_code);
+		metadatum.setValue(lang_name);
+		CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+
+		add_button.setEnabled(false);
+	    }
+	}
+
+	/** Listens for actions apon the 'remove' button in the LanguageManager controls, and if detected calls the remove method of the manager with the language selected for removal. */
+	private class RemoveListener
+	    implements ActionListener {
+	    /** Remove the currently selected language, if any.
+	     * @param event An <strong>ActionEvent</strong>.
+	     * @see org.greenstone.gatherer.cdm.Language
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Language delete_me = (Language)selected_languages_list.getSelectedValue();
+		if(delete_me != null) {
+		    // Remove any current metadata for this language
+		    CollectionDesignManager.collectionmeta_manager.removeMetadata("." + delete_me.getCode());
+		    removeLanguage(delete_me);
+		    if(default_language != null && default_language.equals(delete_me)) {
+			setDefault(null);
+			//clear_button.setEnabled(false);
+			//default_language_field.setText("");
+		    }
+		}
+		remove_button.setEnabled(false);
+	    }
+	}
+
+	/** Listens for selections within the combobox on the LanguageManager controls, and if a change is detected enables, or disables, controls appropriately. */
+	private class SelectorListener
+	    implements ActionListener {
+	    /** Enable or disable controls depeding on selection.
+	     * @param event An <strong>ActionEvent</strong>.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		String language_code = (String) language_combobox.getSelectedItem();
+		if(language_code != null && getLanguage(language_code) == null) {
+		    add_button.setEnabled(true);
+		}
+		else {
+		    add_button.setEnabled(false);
+		}
+	    }
+	}
+
+	/** Listens for actions apon the 'set default' button in the LanguageManager controls, and if detected calls the <i>setDefault()</i> method of the manager with the language selected for default. */
+	private class SetDefaultListener
+	    implements ActionListener {
+	    /** Set the default index to the one currently selected, if any.
+	     * @param event An <strong>ActionEvent</strong>.
+	     * @see org.greenstone.gatherer.cdm.Language
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Language selected_language = (Language) selected_languages_list.getSelectedValue();
+		if(selected_language != null) {
+		    setDefault(selected_language);
+		    //clear_button.setEnabled(true);
+		    // This should cause a repaint of just the desired row
+		    selected_languages_list.setSelectedValue(selected_language, true);
+		}
+		set_default_button.setEnabled(false);
+	    }
+	}
+
+	private class MoveListener
+	    implements ActionListener
+	{
+	    private boolean move_up;
+
+	    public MoveListener(boolean move_up)
+	    {
+		this.move_up = move_up;
+	    }
+
+	    public void actionPerformed(ActionEvent event)
+	    {
+		// Retrieve the selected language
+		Language language = (Language) selected_languages_list.getSelectedValue();
+		if (language != null) {
+		    int new_position = moveLanguage(language, move_up);
+		    // Ensure the language that moved is still selected
+		    selected_languages_list.setSelectedIndex(new_position);
+		}
+	    }
+	}
+
+
+	/** Listens for selections within the list on the LanguageManager controls, and if a change is detected enables, or disables, controls appropriately. */
+	private class ListListener
+	    implements ListSelectionListener {
+	    /** Enable or disable controls depending on the current list selection.
+	     * @param event A <strong>ListSelectionEvent</strong>.
+	     */
+	    public void valueChanged(ListSelectionEvent event) {
+		if (event.getValueIsAdjusting()) {
+		    return;
+		}
+		if(selected_languages_list.isSelectionEmpty()) {
+		    remove_button.setEnabled(false);
+		    set_default_button.setEnabled(false);
+		    move_down_button.setEnabled(false);
+ 		    move_up_button.setEnabled(false);
+		    return;
+		}
+		
+		int i = selected_languages_list.getSelectedIndex();
+ 		int size = selected_languages_list.getModel().getSize();
+		Language selected_lang = (Language)selected_languages_list.getSelectedValue();
+		remove_button.setEnabled(true);
+		set_default_button.setEnabled(default_language == null || !default_language.equals(selected_lang));
+		
+		if (i > 0) {
+ 		    move_up_button.setEnabled(true);
+ 		}
+ 		else {
+ 		    move_up_button.setEnabled(false);
+ 		}
+ 		if (i < size-1){
+ 		    move_down_button.setEnabled(true);
+ 		}
+ 		else {
+ 		    move_down_button.setEnabled(false);
+ 		}
+	    }
+	}
+
+	private class MyLanguageListCellRenderer
+	    extends DefaultListCellRenderer
+	{
+	    /** Return a component that has been configured to display the specified value. */
+	    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+		JLabel component = (JLabel) super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+		if (default_language != null && default_language.equals(value)) {
+		    component.setText(component.getText() + " [" + Dictionary.get("CDM.LanguageManager.Default_Language")+"]");
+		}
+		return component;
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Level.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Level.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Level.java	(revision 21112)
@@ -0,0 +1,174 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      16/07/03
+ * Revised:      
+ **************************************************************************************/
+import org.w3c.dom.*;
+
+/** This class represents a single search type setting which means that, at the moment, it is either based on an Content element containing the string 'plain' or one on the string 'form'. The ordering of these elements within the Level element is important.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class Level
+    implements DOMProxyListEntry {
+
+    /** The Element this level object will source its information from. */
+    private Element element;
+
+    /** Constructor used only during DOMProxyListModel initialization. */
+    public Level() {
+    }
+
+    /** Normal constructor.
+     * @param  element the Element this object will find its data from
+     */
+    public Level(Element element) {
+	this.element = element;
+    }
+
+    /** Creation of a brand new Level 
+     * @param  name the name of this type as a String
+     */
+    public Level(String level) {
+	this.element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.CONTENT_ELEMENT);
+	this.element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, level);
+    }
+
+    /** Compare two objects for ordering. 
+     * @param  object the other Object to compare to
+     * @return <0 if this level should be before the given object, 0 if they are equal, and >0 if it should be after
+     * since we can only have one instance of a particular level, we don't need to look at the name when comparing.
+     */
+    public int compareTo(Object object) {
+	if(object == null) {
+	    return -1;
+	}
+	// hope we never compare a Level with something that is not a Level!
+	return getLevel().compareTo(((Level)object).getLevel());
+    }
+
+    /** Factory-like method to allow DOMProxyListModel to generate new entries.
+     * @param  element the Element the new level should be based on
+     * @return a newly created DOMProxyListEntry for the given element
+     */
+    public DOMProxyListEntry create(Element element) {
+	return new Level(element);
+    }
+
+    /** Determine if this level is equavilent to another object.
+     * @param  object the other Other to match against
+     * @return true if the two are equal, false otherwise
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Retrieve the element this DOMProxyListEntry is based upon. Specified by the interface.
+     * @return the Element in question
+     */
+    public Element getElement() {
+	return element;
+    }
+
+    /** Retrieve the level.
+     * @return the level as a String
+     */
+    public String getLevel() {
+	return element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+    }
+
+    /** Retrieve the display name
+     */
+    public String getName() {
+	CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(CollectionConfiguration.STOP_CHARACTER + getLevel(), false);
+	if(metadatum != null) {
+	    return metadatum.getValue(CollectionMeta.TEXT);
+	}
+	return "";
+    }
+    
+   /** Determine is this command has been assigned, either because it already existed in the collection configuration, or because it has been explicitly set by the user. Non-assigned entries imply they have been added by the GLI to ensure consistancy (and avoid NPE's!)
+     * @return true if this command has been assigned, false otherwise
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     */
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+    
+    /** Set the assigned state.
+     * @param  assigned the desired state of assigned as a boolean
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     */
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    /** Set the element that this DOMProxyListEntry is base on.
+     * @param  element the new Element that this entry should source informatin from
+     */
+    public void setElement(Element element) {
+	this.element = element;
+    }
+
+    /** Set the name of this level.
+     * @param  name the new name for this level, as a String
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     */
+    public void setLevel(String level) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, level);
+	}
+    }
+
+    /** Produce a text representation of this level.
+     * @return a String showing this level
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     */
+    public String toString() {
+	String text ="";
+	if(element == null) {
+	    text = "#Error";
+	}
+	else {
+	    String level = getLevel();
+	    StringBuffer text_buffer = new StringBuffer(level);
+	    text_buffer.append(" \"");
+	    text_buffer.append(getName());
+	    text_buffer.append("\"");
+	
+	    text = text_buffer.toString();
+	    text_buffer = null;
+	}
+	
+	return text;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/MetadataSetView.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/MetadataSetView.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/MetadataSetView.java	(revision 21112)
@@ -0,0 +1,391 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.filechooser.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.MetadataElementListCellRenderer;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+
+
+/** This class only knows how to produce a simple visual representation of the currently imported metadata sets. It is also read-only, so should be fairly straight forward.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class MetadataSetView
+    extends DynamicListModel {
+
+    /** The visual contols used to review the metadata sets. */
+    private Control controls = null;
+    /** A reference to ourselves so our inner classes can refer to us. */
+    private DynamicListModel model = null;
+
+
+    public MetadataSetView()
+    {
+	model = this;
+
+	// Initialise the model
+	refreshModel();
+
+	// Build the controls
+	controls = new MetadataSetViewControls();
+    }
+
+
+    public void destroy()
+    {
+	controls.destroy();
+	controls = null;
+	model = null;
+    }
+
+
+    /** A method for retrieve the controls for this manager.
+     */
+    public Control getControls() {
+	return controls;
+    }
+
+
+    public void refreshModel()
+    {
+	// Remove any metadata sets from the model that are no longer loaded
+	ArrayList loaded_metadata_sets = MetadataSetManager.getMetadataSets();
+	for (int i = size() - 1; i >= 0; i--) {
+	    MetadataSet metadata_set = (MetadataSet) get(i);
+	    if (loaded_metadata_sets.contains(metadata_set) == false) {
+		// System.err.println("Metadata set in list no longer loaded...removing.");
+		remove(i);
+	    }
+	}
+
+	// Add any metadata sets that are loaded but not in the model
+	for (int i = 0; i < loaded_metadata_sets.size(); i++) {
+	    MetadataSet loaded_metadata_set = (MetadataSet) loaded_metadata_sets.get(i);
+	    boolean in_model = false;
+	    for (int j = 0; j < size(); j++) {
+		MetadataSet metadata_set = (MetadataSet) get(j);
+		if (metadata_set.equals(loaded_metadata_set)) {
+		    in_model = true;
+		    break;
+		}
+	    }
+
+	    if (in_model == false) {
+		// System.err.println("Metadata set loaded but not in list...adding.");
+		addElement(loaded_metadata_set);
+	    }
+	}
+    }
+
+
+    /** This class creates and lays-out the various controls for reviewing the metadata sets, and their commands as they would appear in the collection configuration file. */
+    private class MetadataSetViewControls
+	extends JPanel
+	implements Control {
+	/** Adds a new set to the collection*/
+	private JButton add_button;
+	/** Will eventually launch GEMS for editing the selected set, for now just gives a message */
+	private JButton edit_button;
+	/** REmoves a set from the collection */
+	private JButton remove_button;
+	/** The label denoting the element list. */
+	private JLabel element_label = null;
+	/** The label denoting the set list. */
+	private JLabel set_label = null;
+	/** The title of these controls. */
+	private JLabel title = null;
+	/** The list of elements for the choosen set. */
+	private JList element_list = null;
+	/** The list of sets in this collection. */
+	private JList set_list = null;
+	/** The panel onto which all other panels will be placed. */
+	private JPanel central_pane = null;
+	/** The panel onto which the element list will be placed. */
+	private JPanel element_pane = null;
+	/** The panel containing the title and instructions. */
+	private JPanel header_pane = null;
+	/** The panel containing the set list. */
+	private JPanel set_pane = null;
+
+	private JScrollPane element_list_scroll_pane;
+	/** The text area of inline instructions. */
+	private JTextArea instructions = null;
+
+	private MetadataElementListCellRenderer element_list_cell_renderer;
+	private MetadataSetListSelectionListener list_listener;
+
+	/* Constructor.
+	 * @see org.greenstone.gatherer.Dictionary
+	 * @see org.greenstone.gatherer.cdm.MetadataSetView.ListListener
+	 */
+	public MetadataSetViewControls()
+	{
+	    // Create visual components
+	    central_pane = new JPanel();
+	    element_label = new JLabel();
+	    element_label.setHorizontalAlignment(JLabel.CENTER);
+	    element_label.setOpaque(true);
+	    Dictionary.registerText(element_label, "CDM.MetadataSetManager.Elements");
+	    element_list = new JList();
+	    element_list_scroll_pane = new JScrollPane();
+	    element_list_scroll_pane.setViewportView(element_list);
+	    element_list_cell_renderer = new MetadataElementListCellRenderer();
+	    element_list.setCellRenderer(element_list_cell_renderer);
+	    element_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    element_pane = new JPanel();
+	    header_pane = new JPanel();
+	    instructions = new JTextArea();
+	    instructions.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    instructions.setEditable(false);
+	    instructions.setLineWrap(true);
+	    instructions.setRows(6);
+	    instructions.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions, "CDM.MetadataSetManager.Instructions");
+	    set_label = new JLabel();
+	    set_label.setHorizontalAlignment(JLabel.CENTER);
+	    set_label.setOpaque(true);
+	    Dictionary.registerText(set_label, "CDM.MetadataSetManager.Sets");
+	    set_list = new JList(model);
+	    set_list.setCellRenderer(new MetadataSetListCellRenderer());
+	    set_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	    set_pane = new JPanel();
+	    title = new JLabel();
+	    title.setHorizontalAlignment(JLabel.CENTER);
+	    title.setOpaque(true);
+	    Dictionary.registerText(title, "CDM.MetadataSetManager.Title");
+
+	    JPanel button_pane = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(true);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.MetadataSetManager.Add", "CDM.MetadataSetManager.Add_Tooltip");
+	    edit_button = new GLIButton();
+	    edit_button.setEnabled(false);
+	    edit_button.setMnemonic(KeyEvent.VK_E);
+	    Dictionary.registerBoth(edit_button, "CDM.MetadataSetManager.Edit", "CDM.MetadataSetManager.Edit_Tooltip");
+	    
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.MetadataSetManager.Remove", "CDM.MetadataSetManager.Remove_Tooltip");
+	    list_listener = new MetadataSetListSelectionListener();
+
+	    // Add listeners
+	    add_button.addActionListener(new AddButtonListener());
+	    edit_button.addActionListener(new EditButtonListener());
+	    remove_button.addActionListener(new RemoveButtonListener());
+	    set_list.addListSelectionListener(list_listener);
+	    // Layout
+	    instructions.setBorder(BorderFactory.createEmptyBorder(2,5,2,5));
+
+	    header_pane.setLayout(new BorderLayout());
+	    header_pane.add(title, BorderLayout.NORTH);
+	    header_pane.add(new JScrollPane(instructions), BorderLayout.CENTER);
+
+	    set_pane.setLayout(new BorderLayout());
+	    set_pane.add(set_label, BorderLayout.NORTH);
+	    set_pane.add(new JScrollPane(set_list), BorderLayout.CENTER);
+
+	    element_pane.setLayout(new BorderLayout());
+	    element_pane.add(element_label, BorderLayout.NORTH);
+	    element_pane.add(element_list_scroll_pane, BorderLayout.CENTER);
+
+	    central_pane.setLayout(new GridLayout(2,1));
+	    central_pane.add(set_pane);
+	    central_pane.add(element_pane);
+	    
+	    button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    button_pane.setLayout(new GridLayout(1,3,0,0));
+	    button_pane.add(add_button);
+	    button_pane.add(edit_button);
+	    button_pane.add(remove_button);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_pane, BorderLayout.NORTH);
+	    add(central_pane, BorderLayout.CENTER);
+	    add(button_pane, BorderLayout.SOUTH);
+	}
+
+
+	public void destroy() { }
+
+
+	public void gainFocus()
+	{
+	    // Ensure the instructions area is scrolled to the top
+	    if (instructions != null) {
+		instructions.setCaretPosition(0);
+	    }
+
+	    // If no current selection, select first available set
+	    if (set_list.isSelectionEmpty() && set_list.getModel().getSize() > 0) {
+		set_list.setSelectedIndex(0);
+		list_listener.valueChanged(new ListSelectionEvent(set_list, 0, 0, true));
+	    }
+	    if (!set_list.isSelectionEmpty()) {
+		edit_button.setEnabled(true);
+		remove_button.setEnabled(true);
+	    } else {
+		edit_button.setEnabled(false);
+		remove_button.setEnabled(false);
+	    }
+		
+	}
+
+
+	public void loseFocus() { }
+
+
+	/** Listens for clicks on the add button. */
+	private class AddButtonListener
+	    implements ActionListener {
+
+	    /** Called when the add button is clicked.
+	     * @param event an ActionEvent containing information about the mouse click
+	     */
+	    public void actionPerformed(ActionEvent event)
+	    {
+		JFileChooser chooser = new JFileChooser(new File(Gatherer.getGLIMetadataDirectoryPath()));
+		chooser.setFileFilter(new MetadataSet.MetadataSetFileFilter());
+		int return_val = chooser.showDialog(Gatherer.g_man, Dictionary.get("CDM.MetadataSetManager.Add"));
+		if (return_val == JFileChooser.APPROVE_OPTION) {
+		    Gatherer.c_man.importMetadataSet(new MetadataSet(chooser.getSelectedFile()));
+		    refreshModel();
+
+		    // The metadata.xml files may possibly contain metadata for this newly added metadata set
+		    // Re-read all the metadata.xml files to account for this case
+		    MetadataXMLFileManager.clearMetadataXMLFiles();
+		    MetadataXMLFileManager.loadMetadataXMLFiles(new File(Gatherer.c_man.getCollectionImportDirectoryPath()));
+		}
+	    }
+	}
+
+
+	/** Listens for clicks on the edit button. */
+	private class EditButtonListener
+	    implements ActionListener {
+
+	    /** Called when the edit button is clicked.
+	     * @param event an ActionEvent containing information about the mouse click
+	     */
+	    public void actionPerformed(ActionEvent event)
+	    {
+		// do a pop up message
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.MetadataSetManager.Edit_Message"), Dictionary.get("CDM.MetadataSetManager.Edit_Message_Title"), JOptionPane.INFORMATION_MESSAGE);
+
+	    }
+	}
+
+
+	private class MetadataSetListCellRenderer
+	    implements ListCellRenderer
+	{
+	    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus)
+	    {
+		DefaultListCellRenderer default_list_cell_renderer = new DefaultListCellRenderer();
+		JLabel list_cell_label = (JLabel) default_list_cell_renderer.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+
+		if (value instanceof MetadataSet) {
+		    MetadataSet metadata_set = (MetadataSet) value;
+		    String metadata_set_name = MetadataTools.getMetadataSetAttribute(metadata_set, "Name", Configuration.getLanguage(), "en");
+		    list_cell_label.setText("metadataset " + metadata_set.getNamespace() + " \"" + metadata_set_name + "\"");
+		}
+
+		return list_cell_label;
+	    }
+	}
+
+
+	private class MetadataSetListSelectionListener
+	    implements ListSelectionListener {
+
+	    public void valueChanged(ListSelectionEvent event)
+	    {
+		// Wait until we get a stable event
+		if (event.getValueIsAdjusting()) {
+		    return;
+		}
+
+		// Now we can process it
+		if (!set_list.isSelectionEmpty()) {
+		    MetadataSet metadata_set = (MetadataSet) set_list.getSelectedValue();
+		    element_list.setListData(new Vector(metadata_set.getMetadataSetElements()));
+  		    remove_button.setEnabled(true);
+		    edit_button.setEnabled(true);
+
+  		    // Finally check the directionality and scroll as necessary
+  		    JScrollBar scroll_bar = element_list_scroll_pane.getHorizontalScrollBar();
+  		    if (element_list_cell_renderer.getDirectionality() == Character.DIRECTIONALITY_RIGHT_TO_LEFT) {
+			scroll_bar.setValue(scroll_bar.getMaximum());
+  		    }
+  		    else {
+			scroll_bar.setValue(scroll_bar.getMinimum());
+		    }
+		}
+		else {
+		    remove_button.setEnabled(false);
+		    edit_button.setEnabled(false);
+		}
+	    }
+	}
+
+
+	/** Listens for clicks on the remove button. */
+	private class RemoveButtonListener
+	    implements ActionListener {
+
+	    /** Called when the remove button is clicked.
+	     * @param event an ActionEvent containing information about the mouse click
+	     */
+	    public void actionPerformed(ActionEvent event)
+	    {
+		MetadataSet metadata_set = (MetadataSet) set_list.getSelectedValue();
+		Gatherer.c_man.removeMetadataSet(metadata_set);
+		refreshModel();
+		element_list.setListData(new Vector());
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ParsingProgress.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ParsingProgress.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/ParsingProgress.java	(revision 21112)
@@ -0,0 +1,101 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Title:        Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Copyright:    Copyright (c) 2001
+ * Company:      The University of Waikato
+ * Written:      03/05/02
+ * Revised:      20/05/02
+ *               17/11/02 - Commented
+ **************************************************************************************/
+import java.awt.*;
+import javax.swing.*;
+
+/** This class provides a progress bar to be displayed whenever the module must reload the plugin and classifier information, either automatically or if the user indicates a reload is needed.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class ParsingProgress
+    extends JDialog {
+    /** The content pane within the dialog box. */
+    private JPanel content_pane = null;
+    /** The progress bar itself. */
+    private JProgressBar progress = null;
+    /** The default size of the progress dialog. */
+    static final Dimension SIZE = new Dimension(600,75);
+
+    /** Constructor.
+     * @param title The title to show on the dialog, as a <strong>String</strong>.
+     * @param message The message to show on the dialog, as a <strong>String</strong>.
+     * @param max The total number of plugins/classifiers that have to be parsed before the progress bar can be disposed of, as an <i>int</i>.
+     */
+    public ParsingProgress(String title, String message, int max) {
+	super();
+	this.setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+	this.setSize(SIZE);
+	this.setTitle(title);
+
+	// Creation
+	this.content_pane = (JPanel) getContentPane();
+	this.progress = new JProgressBar();
+	this.progress.setMaximum(max);
+
+	// Layout
+	this.content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	this.content_pane.setLayout(new BorderLayout());
+	this.content_pane.add(new JLabel(message), BorderLayout.NORTH);
+	this.content_pane.add(progress, BorderLayout.CENTER);
+
+	// Center and display
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - SIZE.width) / 2,
+		    (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+
+    public void destroy() {
+    }
+
+    /** Method which increments the progress count by one, which should be called after every successful parsing of a classifier or plugin.
+     */
+    public void inc() {
+	progress.setValue(progress.getValue() + 1);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Plugin.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Plugin.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Plugin.java	(revision 21112)
@@ -0,0 +1,440 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+/** This class is responsible for storing information from a parsed pluginfo call in such a way that it allows easy access to parsed details for the purposes of user design and specification of plugins. */
+public class Plugin
+    extends ArrayList
+    implements ArgumentContainer, Comparable, DOMProxyListEntry, Serializable {
+    /** The DOM Element this assigned Plugin is modelled on. */
+    private Element element;
+    private boolean does_explode_metadata_databases = false; // Only for Base
+    private boolean is_abstract = false; // Only for Base
+    /** The parent Plugin this one inherits from, if any. */
+    private Plugin super_plugin; // Only for Base
+    private String description; // Only for Base
+    private String name; // Only for Base
+
+    /** Constructor used in DOMProxyListModel initializations, and Library Level. Used for Base plugins (those in the list of available plugins, not ones that are in the DOMProxyList)
+     */
+    public Plugin() {
+    }
+    
+    /** Constructor used for the plugins that are in the DOMProxyList */
+    // Every time the list of plugins in the assigned plugins box changes, (eg plugin added or removed, not when plugin configured), the plugins seem to be regenerated, using the element from the old plugin. All known args get added to the element the first time this happens - we need to add them to the arguments list in the order they are found in the base plugins though, not this order.
+    public Plugin(Element element, Plugin base_plugin) {
+	super();
+	this.element = element;
+	this.name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	//DebugStream.println("Establishing Plugin: " + name);
+	// Parse in any argument options for this plugin, keeping a list of the ones found
+	HashMap known_arguments = new HashMap();
+	NodeList option_elements = element.getElementsByTagName(StaticStrings.OPTION_ELEMENT);
+	int option_elements_length = option_elements.getLength();
+	for(int i = 0; i < option_elements_length; i++) {
+	    Element option_element = (Element) option_elements.item(i);
+	    Argument argument = new Argument(option_element);
+	    //DebugStream.println("Rebuilding existing argument: " + argument.getName());
+	    known_arguments.put(argument.getName(), argument);
+	    if (base_plugin == null) {
+		// otherwise they will all be added in the next step
+		argument.setOwner(name);
+		add(argument);
+	    } 
+	}
+	// If a base plugin was given
+	if(base_plugin != null) {
+	    //DebugStream.println("Based on previous plugin.");
+	    // Copy the details, and add a reference to whatever base_plugins super plugin is. ?? 
+	    description = base_plugin.getDescription();
+	    // Now search through the 'dummy' arguments belonging to the base plugin. For each found, if it is already assigned, fill out further details such as type. If any are found that are not already assigned for this plugin, copy them and add them, but without a value.
+	    ArrayList all_arguments = base_plugin.getArguments(true, true);
+	    int argument_count = all_arguments.size();
+	    for(int j = 0; j < argument_count; j++) {
+		Argument base_argument = (Argument) all_arguments.get(j);
+		String base_argument_name = base_argument.getName();
+		//DebugStream.println("Library indicates this plugin should have an argument: " + base_argument_name);
+		Argument existing_argument = (Argument) known_arguments.get(base_argument_name);
+		// Found an existing argument. Complete its details
+		if(existing_argument != null) {
+		    //DebugStream.println("Found existing argument. Filling out details.");
+		    existing_argument.setCustomArgument(false);
+		    existing_argument.setDefaultValue(base_argument.getDefaultValue());
+		    existing_argument.setDescription(base_argument.getDescription());
+		    existing_argument.setOptions(base_argument.getOptions());
+		    existing_argument.setRequired(base_argument.isRequired());
+		    existing_argument.setType(base_argument.getType());
+		    existing_argument.setMinimum(base_argument.getMinimum());
+		    existing_argument.setMaximum(base_argument.getMaximum());
+		    existing_argument.setOwner(base_argument.getOwner());
+		    add(existing_argument); 
+		    
+		}
+		// No existing argument. Copy base_argument and add it, but do not set its assigned flag. That should be set the first time its changed by the user.
+		else {
+		    //DebugStream.println("No such argument. Adding new, unassigned, argument.");
+		    // The trick thing is that we have to create a new element in the DOM as well.
+		    Argument new_argument = base_argument.copy();
+		    Element argument_element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.OPTION_ELEMENT);
+		    argument_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, base_argument_name);
+		    argument_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
+		    argument_element.setAttribute(StaticStrings.CUSTOM_ATTRIBUTE, StaticStrings.FALSE_STR);
+		    // All done. Add it.
+		    // if we are not in the first collection opened, then this 
+		    /// will belong to a different document, so need to import 
+		    // it.
+		    Node new_element = element.getOwnerDocument().importNode(argument_element, true);
+		    new_argument.setElement((Element) new_element);
+		    element.appendChild(new_element);
+		    add(new_argument);
+		}
+	    }
+	}
+    }
+
+    /** Method to add an argument to this base plugin. Only adds the argument if it isn't already present, and only if this is a base plugin (ie not based on DOM).
+     * @param argument the Argument to add
+     */
+    public void addArgument(Argument argument) {
+	if(element == null && !contains(argument)) {
+	    add(argument);
+	    argument.setOwner(name);
+	}
+    }
+
+    /** Method to compare two plugins for ordering.
+     * @param object The plugin we are comparing to, as an <strong>Object</strong>.
+     * @return An <i>int</i> specifying the plugin order, using values as set out in <strong>String</strong>.
+     * @see java.lang.String#compareTo
+     */
+    public int compareTo(Object object) {
+	if(object instanceof Plugin) {
+	    return name.compareTo(((Plugin)object).getName());
+	}
+	return -1;
+    }
+
+    /** The assigned plugin constructor.
+     * @param element the DOM Element this plugin is based upon
+     */
+    public DOMProxyListEntry create(Element element) {
+	String plugin_name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	// Determine the base plugin from the plugin name
+	Plugin base_plugin = CollectionDesignManager.plugin_manager.getBasePlugin(plugin_name);
+	Plugin plugin = new Plugin(element, base_plugin);
+	base_plugin = null;
+	plugin_name = null;
+	return plugin;
+    }
+
+
+    /** Checks whether the plugin this instance is based on processes metadata databases that can be exploded. */
+    public boolean doesExplodeMetadataDatabases()
+    {
+	Plugin base_plugin = CollectionDesignManager.plugin_manager.getBasePlugin(getName());
+	if (base_plugin == null) {
+	    return false;
+	}
+	return base_plugin.does_explode_metadata_databases;
+    }
+
+
+    /** Checks whether this plugin instance will process the specified file (given its process_exp). */
+    public boolean doesProcessFile(File file)
+    {
+	// Check the filename against the plugin's process_exp value
+	ArrayList arguments = getArguments(true, true);
+	for (int i = 0; i < arguments.size(); i++) {
+	    Argument argument = (Argument) arguments.get(i);
+	    if (argument.getName().equals("process_exp")) {
+		// Try the assigned value first, for when the user has manually set the value
+		String regular_expression = argument.getValue();
+		if (regular_expression == null || regular_expression.equals("")) {
+		    // Not set, so use the default value
+		    regular_expression = argument.getDefaultValue();
+		    if (regular_expression.equals("")) {
+			continue;
+		    }
+		}
+
+		// The $ at the end doesn't seem to work in Java, so need to add ".*" at the start
+		if (regular_expression.startsWith("(?i)")) {
+		    // Don't mess up case-insensitive matching though
+		    regular_expression = "(?i)" + ".*" + regular_expression.substring("(?i)".length());
+		}
+		else {
+		    regular_expression = ".*" + regular_expression;
+		}
+
+		// If the filename matches the regular expression, this plugin will deal with the file in some way
+		if (file.getName().matches(regular_expression)) {
+		    return true;
+		}
+	    }
+	}
+
+	// This plugin will (probably) not deal with the specified file
+	return false;
+    }
+
+
+    /** Method to determine if two plugins are equal.
+     * @param object The plugin to test against, as an <strong>Object</strong>.
+     * @return <i>true</i> if the plugin names match, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Method to retrieve an argument by its name.
+     * @param name The name of the argument as a <strong>String</strong>.
+     * @return The <strong>Argument</strong> requested, or <i>null</i> if no such argument.
+     */
+    public Argument getArgument(String name) {
+	// The name given may still include the '-'
+	if(name.startsWith("-")) {
+	    name = name.substring(1);
+	}
+	ArrayList arguments = getArguments(true, true);
+	for(int i = 0; i < arguments.size(); i++) {
+	    Argument argument = (Argument)arguments.get(i);
+	    if(argument.getName().equals(name)) {
+		return argument;
+	    }
+	}
+	return null;
+    }
+
+    /** Retrieve all of the arguments available to this base plugin, including its super plugins arguments. Some complexity is added by allowing the caller to choose whether they want normal arguments, custom arguments, or both.
+     * @return an ArrayList of all of the arguments, starting with those for this plugin and ending with the arguments for basplug or similiar root plugin
+     */
+    public ArrayList getArguments(boolean include_normal, boolean include_custom) {
+	ArrayList arguments = new ArrayList();
+	if(include_normal && include_custom) {
+	    arguments.addAll(this);
+	}
+	else {
+	    int size = size();
+	    for(int i = 0; i < size; i++) {
+		Argument argument = (Argument) get(i);
+		if(argument.isCustomArgument()) {
+		    if(include_custom && !arguments.contains(argument)) {
+			arguments.add(argument);
+		    }
+		}
+		else {
+		    if(include_normal && !arguments.contains(argument)) {
+			arguments.add(argument);
+		    }
+		}
+		argument = null;
+	    }
+	}
+	if(super_plugin != null) {
+	    ArrayList remainder = super_plugin.getArguments(include_normal, include_custom);
+	    remainder.removeAll(arguments);
+	    arguments.addAll(remainder);
+	}
+	return arguments;
+    }
+
+    /** Method to retrieve a plugins custom argument information. Custom arguments are defined to be those that have not got matching arguments in the base reference plugin from the library. Of course if there is no base plugin then all arguments are considered to be custom.
+     * @return the custom arguments as a String
+     */
+    public String getCustom() {
+	StringBuffer custom_text = new StringBuffer();
+	// Retrieve all of the arguments, and append any that are custom into one long string
+	ArrayList arguments = getArguments(false, true);
+	int arguments_size = arguments.size();
+	boolean first = true;
+	for(int i = 0; i < arguments_size; i++) {
+	    Argument argument = (Argument) arguments.get(i);
+	    if(argument.isAssigned()) {
+		if(!first) {
+		    custom_text.append(" ");
+		}
+		custom_text.append(argument.toString());
+		first = false;
+	    }
+	}
+	return custom_text.toString();
+    }
+
+    public String getDescription() {
+	return description;
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    /** Method to retrieve a plugins name.
+     * @return A <strong>String</strong> containing the plugins name.
+     */
+    public String getName() {
+	if(name == null && element != null) {
+	    name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	}
+	return name;
+    }
+
+    public boolean isAbstract() {
+	return is_abstract;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public boolean isSeparator() {
+	return (element != null && element.getAttribute(StaticStrings.SEPARATOR_ATTRIBUTE).equals(StaticStrings.TRUE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    /** Set the custom arguments. This turns out to be quite tricky. We must parse in the string, searching for arguments (for that we use a handy method in CollectionConfiguration). Next, for each argument, we check if we already know about it. If so we update its value, otherwise we create a new argument and assign it (must assign!).
+     * @param custom_str the custom arguments all splodged together in one String
+     */
+    public void setCustom(String custom_str) {
+	HashMap raw_arguments = CollectionConfiguration.parseArguments(new CommandTokenizer(custom_str));
+	ArrayList custom_arguments = getArguments(false, true);
+	int size = custom_arguments.size();
+	for(int i = 0; i < size; i++) {
+	    Argument argument = (Argument) custom_arguments.get(i);
+	    String original_argument_name = StaticStrings.MINUS_CHARACTER + argument.getName();
+	    if(raw_arguments.containsKey(original_argument_name)) {
+		// Set as assigned
+		argument.setAssigned(true);
+		String argument_value = (String)raw_arguments.remove(original_argument_name);
+		if(argument_value != null) {
+		    argument.setValue(argument_value);
+		    argument_value = null;
+		}
+	    }
+	    // We've removed it from our custom statement, so unassign
+	    else {
+		argument.setAssigned(false);
+	    }
+	    argument = null;
+	}
+	// Any left over, add to the plugin
+	Iterator argument_names = raw_arguments.keySet().iterator();
+	while(argument_names.hasNext()) {
+	    String argument_name = (String) argument_names.next();
+	    String argument_value = (String) raw_arguments.get(argument_name);
+	    // The tricky thing is that we have to create a new element in the DOM as well.
+	    Element argument_element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.OPTION_ELEMENT);
+	    argument_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, argument_name.substring(1));
+	    argument_element.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
+	    argument_element.setAttribute(StaticStrings.CUSTOM_ATTRIBUTE, StaticStrings.TRUE_STR);
+	    Argument argument = new Argument(argument_element);
+	    argument_name = null;
+	    if(argument_value != null) {
+		argument.setValue(argument_value);
+		argument_value = null;
+	    }
+	    // All done. Add it.
+	    element.appendChild(argument_element);
+	    add(argument);
+	    argument_element = null;
+	}
+	raw_arguments = null;
+    }
+
+    /** Method to set the value of desc.
+     * @param description The new value of desc as a <strong>String</strong>.
+     */
+    public void setDescription(String description) {
+	this.description = description;
+    }
+
+    public void setDoesExplodeMetadataDatabases(boolean does_explode_metadata_databases) {
+	this.does_explode_metadata_databases = does_explode_metadata_databases;
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+    }
+
+    public void setIsAbstract(boolean is_abstract) {
+	this.is_abstract = is_abstract;
+    }
+
+    /** Method to set the value of name.
+     * @param name The new value of name as a <strong>String</strong>.
+     */
+    public void setName(String name) {
+	this.name = name;
+    }
+
+    /** Method to set the value of the super_plugin.
+     * @param super_plugin The new value of super_plugin as a <strong>Plugin</strong>, or <i>null</i> if this class has no inheritance.
+     */
+    public void setSuper(Plugin super_plugin) {
+	this.super_plugin = super_plugin;
+    }
+
+    /** Method to print out this plugin as it would appear as a command within the collection configuration file.
+     * @return A <strong>String</strong> containing a single plugin command.
+     */
+    public String toString() {
+	if(element != null) {
+	    if(name == null) {
+		name = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE);
+	    }
+	    StringBuffer text = new StringBuffer(StaticStrings.PLUGIN_STR);
+	    text.append(" ");
+	    text.append(name);
+	    text.append(" ");
+	    ArrayList arguments = getArguments(true, true);
+	    int arguments_size = arguments.size();
+	    for(int i = 0; i < arguments_size; i++) {
+		Argument argument = (Argument)arguments.get(i);
+		if(argument.isAssigned()) {
+		    text.append(argument.toString());
+		    text.append(" ");
+		}
+		argument = null;
+	    }
+	    return text.substring(0, text.length() - 1);
+	}
+	// Basic Plugin
+	else {
+	    return name;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/PluginManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/PluginManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/PluginManager.java	(revision 21112)
@@ -0,0 +1,1333 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.plaf.*;
+import javax.swing.plaf.basic.*;
+import org.apache.xerces.parsers.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.collection.CollectionContentsChangedListener;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.ModalDialog;
+import org.greenstone.gatherer.gui.WarningDialog;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+import org.xml.sax.*;
+
+/** This class is for maintaining a list of known plug-ins, and importing new plugins using the parser. */
+public class PluginManager
+    extends DOMProxyListModel
+    implements CollectionContentsChangedListener
+{
+    /** The library 'reserve' of base plugins. */
+    private ArrayList library = null;
+    /** When asking how many rows are in the model, and if this variables value is true, then this modifier alters the number returned. This funtionality is used to hide the last three rows of the list in low detail modes. */
+    private boolean modify_row_count = false;
+    /** The controls for editing the contents of this manager. */
+    private Control controls = null;
+    private DOMProxyListModel model;
+    private JPanel separator;
+    private Plugin separator_plugin;
+    /** Constructor.
+     */
+    public PluginManager() {
+	super(CollectionDesignManager.collect_config.getDocumentElement(), CollectionConfiguration.PLUGIN_ELEMENT, new Plugin());
+	DebugStream.println("PluginManager: " + super.getSize() + " plugins parsed.");
+	model = this;
+	// Reload/Create the library
+	loadPlugins(); // adds all the plugins to the library
+	savePlugins();
+	// Create the separator, cause we can reuse it.
+	separator = getSeparator();
+
+	// Listen for CollectionContentsChanged events, so we can give plugin hints when new files are added
+	Gatherer.c_man.addCollectionContentsChangedListener(this);
+    }
+
+    /** Method to add a new plugin to the library
+     * @param plugin the new base Plugin
+     */
+    private void addPlugin(Plugin plugin) {
+  	if(!library.contains(plugin)) {
+  	    library.add(plugin);
+  	}
+    }
+
+    /** Method to assign a plugin
+     * @param plugin the Plugin to assign
+     */
+    private void assignPlugin(Plugin plugin) {
+	if(plugin.getName().equals(StaticStrings.RECPLUG_STR) || plugin.getName().equals(StaticStrings.ARCPLUG_STR)) {
+	    addAfter(plugin, separator_plugin); // Adds after separator
+	} else {
+	    addBefore(plugin, separator_plugin);
+	}
+	Gatherer.c_man.configurationChanged();
+    }
+
+    public static boolean clearPluginCache() {
+
+	DebugStream.println("deleting plugins.dat");
+	File plugin_file = new File(Gatherer.getGLIUserDirectoryPath() + "plugins.dat");
+	if (plugin_file.exists()) {
+	    return Utility.delete(plugin_file);
+	}
+	return true;
+    }
+
+
+    /** Destructor. */
+    public void destroy()
+    {
+	Gatherer.c_man.removeCollectionContentsChangedListener(this);
+
+	if (controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+
+	library.clear();
+	library = null;
+    }
+
+
+    /** This function listens for new files being added to the collection and hints about suitable plugins. */
+    public void fileAddedToCollection(File file)
+    {
+	// First check the plugins already assigned in the collection
+	for (int i = 0; i < super.getSize(); i++) {
+	    Plugin assigned_plugin = (Plugin) getElementAt(i);
+	    if (assigned_plugin.isSeparator() == false && assigned_plugin.doesProcessFile(file) == true) {
+		// This file will be processed by an assigned plugin, so no suggestion is necessary
+		DebugStream.println("Processed by assigned plugin: " + assigned_plugin);
+		return;
+	    }
+	}
+
+	// Next try the plugins NOT already assigned in the collection
+	ArrayList suitable_plugins = new ArrayList();
+	Object[] unassigned_plugins = getAvailable();
+	for (int i = 0; i < unassigned_plugins.length; i++) {
+	    Plugin unassigned_plugin = (Plugin) unassigned_plugins[i];
+	    if (unassigned_plugin.doesProcessFile(file) == true) {
+		DebugStream.println("Processed by unassigned plugin: " + unassigned_plugin);
+		suitable_plugins.add(unassigned_plugin);
+	    }
+	}
+
+	// If there appear to be no suitable plugins, warn the user about this and be done
+	if (suitable_plugins.size() == 0) {
+	    String[] args = new String[1];
+	    args[0] = file.getName();
+	    WarningDialog warning_dialog = new WarningDialog("warning.NoPluginExpectedToProcessFile", "NoPluginExpectedToProcessFile.Title", Dictionary.get("NoPluginExpectedToProcessFile.Message", args), null, false);
+	    warning_dialog.display();
+	    warning_dialog.dispose();
+	    return;
+	}
+
+	// Generate a dialog 
+	new PluginSuggestionPrompt(file.getName(), suitable_plugins);
+    }
+
+
+    public boolean isFileExplodable(File file)
+    {
+
+	for (int i = 0; i < library.size(); i++) {
+	    Plugin plugin = (Plugin) library.get(i);
+	    if (plugin.doesExplodeMetadataDatabases() == true && plugin.doesProcessFile(file) == true) { 
+		return true;
+	    }
+	}
+
+	return false;
+    }
+
+    public Plugin getExploderPlugin(File file)
+    {
+	for (int i = 0; i < library.size(); i++) {
+	    Plugin plugin = (Plugin) library.get(i);
+	    if (plugin.doesProcessFile(file) == true && plugin.doesExplodeMetadataDatabases() == true) {
+		return plugin;
+	    }
+	}
+	return null;
+	
+    }
+
+    /** Method to retrieve the control for this manager.
+     * @return the Control
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    // Build controls
+	    controls = new PluginControl();
+	}
+	return controls;
+    }
+
+    /** Retrieve the base pluging of the given name, or null if no such plugin.
+     * @param name the name of the base plugin to retrieve as a String
+     * @return the Plugin requested or null if no such plugin
+     */
+    public Plugin getBasePlugin(String name) {
+	int library_size = library.size();
+	for(int i = 0; i < library_size; i++) {
+	    Plugin plugin = (Plugin) library.get(i);
+	    if(plugin.getName().equals(name)) {
+		return plugin;
+	    }
+	}
+	// No success.
+	return null;
+    }
+
+    /** Overrides getSize in DOMProxyListModel to take into account the row count modifier used to hide the last three rows in lower detail modes
+     * @return an int indicating the number of rows in the model, or more correctly the desired number of rows to display
+     */
+    public int getSize() {
+	int result = super.getSize();
+	if(modify_row_count) {
+	    result = result-3;
+	}
+	return result;
+    }
+
+    /** Called when the detail mode has changed which in turn may cause several design elements to be available/hidden
+     * @param mode the new mode as an int
+     */
+    public void modeChanged(int mode) {
+	if(controls != null) {
+	    ((PluginControl)controls).modeChanged(mode);
+	}
+    }    
+
+    /** Method to move a plugin in the list order.
+     * @param plugin the Plugin you want to move.
+     * @param direction true to move the plugin up, false to move it down.
+     * @param all true to move to move all the way, false for a single step.
+     */
+    // why are all the error notices there when the buttons are disabled is you cant move???
+    private void movePlugin(Plugin plugin, boolean direction, boolean all) {
+	// Can't ever move RecPlug or ArcPlug
+	if(super.getSize() < 4) {
+	    //DebugStream.println("Not enough plugins to allow moving.");
+	    return;
+	}
+	if(plugin.getName().equals(StaticStrings.ARCPLUG_STR) || plugin.getName().equals(StaticStrings.RECPLUG_STR)) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.Move.Fixed"), Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+	    return;
+	}
+	if(all) {
+	    // Move to top
+	    if(direction) {
+		// Remove the moving plugin
+		remove(plugin);
+		// Retrieve the first plugin
+		Plugin first_plugin = (Plugin) getElementAt(0);
+		// Add the moving plugin before the first plugin
+		addBefore(plugin, first_plugin);
+		first_plugin = null;
+		Gatherer.c_man.configurationChanged();
+	    }
+	    else {
+		// Remove the moving plugin
+		remove(plugin);
+		// Add the moving plugin before the separator
+		addBefore(plugin, separator_plugin);
+		Gatherer.c_man.configurationChanged();
+	    }
+	}
+	else {
+	    // Try to move the plugin one step in the desired direction.
+	    int index = indexOf(plugin);
+	    ///ystem.err.println("Index of " + plugin + " = " + index);
+	    if(direction) {
+		index--;
+		if(index < 0) {
+		    String args[] = new String[2];
+		    args[0] = Dictionary.get("CDM.PlugInManager.PlugIn_Str");
+		    args[1] = plugin.getName();
+		    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.Move.At_Top", args), Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+		    return;
+		}
+		remove(plugin);
+		add(index, plugin);
+		Gatherer.c_man.configurationChanged();
+	    }
+	    else {
+		index++;
+		Plugin next_plugin = (Plugin) getElementAt(index);
+		if(next_plugin.isSeparator()) {
+		    String args[] = new String[1];
+		    args[0] = plugin.getName();
+		    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.Move.Cannot", args), Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+		    // Still not going to move RecPlug or ArcPlug.
+		    return;
+		}
+		remove(plugin);
+		add(index, plugin);
+		Gatherer.c_man.configurationChanged();
+	    }
+	}
+    }
+
+    /** We attempt to place the separator between the unfixed and the fixed plugins. Since we only know of two fixed plugins, we search for either of them, and place the separator before them.
+     */
+    public void placeSeparator() {
+	///ystem.err.println("Placing separator.");
+	int separator_index = super.getSize();
+	if(separator_index > 0) { 
+	    boolean found_fixed = false;
+	    int index = separator_index - 1;
+	    while(index > 0) {
+		Plugin plugin = (Plugin) getElementAt(index);
+		String name = plugin.getName();
+		if(name.equals(StaticStrings.RECPLUG_STR) || name.equals(StaticStrings.ARCPLUG_STR)) {
+		    found_fixed = true;
+		    index--;
+		}
+		else {
+		    if(found_fixed) {
+			separator_index = index + 1;
+			index = -1;
+		    }
+		    else {
+			index--;
+		    }
+		}
+		name = null;
+		plugin = null;
+	    }
+	}
+	Element element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.PLUGIN_ELEMENT);
+	element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, CollectionConfiguration.SEPARATOR_ATTRIBUTE);
+	element.setAttribute(CollectionConfiguration.SEPARATOR_ATTRIBUTE, CollectionConfiguration.TRUE_STR);
+	separator_plugin = new Plugin(element, null);
+	///atherer.println("Adding plugin separator at: " + separator_index);
+	add(separator_index, separator_plugin);
+    }
+
+    /** This method removes an assigned plugin. I was tempted to call it unassign, but remove is more consistant. Note that there is no way to remove a plugin from the library.
+     * @param plugin The <strong>Plugin</strong> to remove.
+     */
+    private void removePlugin(Plugin plugin) {
+	remove(plugin);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    /** Method to cache the current contents of library (known plugins) to file.
+     */
+    private void savePlugins() {
+	try {
+	    File plugins_dat_file = new File(Gatherer.getGLIUserDirectoryPath() + "plugins.dat");
+	    FileOutputStream file = new FileOutputStream(plugins_dat_file);
+	    ObjectOutputStream out = new ObjectOutputStream(file);
+	    out.writeObject(library);
+	    out.close();
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+    /** Inform the model to hide/show the last three lines on the list.
+     * @param modify_row_count true to hide the last three lines, false otherwise
+     */
+    private void setHideLines(boolean modify_row_count) {
+	this.modify_row_count = modify_row_count;
+	int original_size = super.getSize();
+	if(modify_row_count) {
+	    fireIntervalRemoved(this, original_size - 4, original_size - 1);
+	}
+	else {
+	    fireIntervalAdded(this, original_size - 4, original_size - 1);
+	}
+    }
+
+    /** Determine the current separator index. */
+    private int findSeparatorIndex() {
+	int separator_index = super.getSize() - 1;
+	while(separator_index >= 0) {
+	    Plugin search = (Plugin) getElementAt(separator_index);
+	    if(search.isSeparator()) {
+		return separator_index;
+	    }
+	    separator_index--;
+	}
+	return separator_index;
+    }
+
+
+    /** Retrieve a list of those plugins that are in library but not in the assigned plugins. */
+    private Object[] getAvailable() {
+	ArrayList available = new ArrayList();
+	int library_size = library.size();
+	for(int i = 0; i < library_size; i++) {
+	    Plugin plugin = (Plugin) library.get(i);
+	    if(!plugin.isAbstract()) {
+		available.add(plugin);
+	    }
+	    plugin = null;
+	}
+	// Now go through the assigned plugins, and remove any that match.
+	available.removeAll(children());
+	//DebugStream.println("There are a total of " + library.size() + " plugins in the library.");
+	//DebugStream.println("However " + children().size() + " are in use,");
+	//DebugStream.println("So only " + available.size() + " remain.");
+	Collections.sort(available);
+	return available.toArray();
+    }
+
+    /** Method to extract just the plugins name from a file object.
+     * @param plugin The <strong>File</strong> which references a certain plugin.
+     * @return A <strong>String</strong> containing just the plugins name, without extension.
+     */
+    private String getPluginName(String filename) {
+	String name = filename;
+	if(name.indexOf(".") != -1) {
+	    name = name.substring(0, name.indexOf("."));
+	}
+	return name;
+    }
+
+
+    /** Method to load the details of a single plug-in.
+     * @param plugin The plugin <strong>File</strong> you wish to load.
+     */
+    private void loadPlugin(String plugin, String lang) {
+	Document document = null;
+	InputStream input_stream = null;
+	
+	// Run pluginfo on this plugin, and then send the results for parsing.
+	try {
+	    StringBuffer xml = null;
+	    if (Gatherer.isGsdlRemote) {
+		String pluginfo_output = RemoteGreenstoneServer.getScriptOptions("pluginfo.pl", "&plugin=" + plugin);
+		xml = new StringBuffer(pluginfo_output);
+	    }
+	    else {
+		String args[] = null;
+		if(Utility.isWindows()) {
+		    args = new String[6];
+		    if(Configuration.perl_path != null) {
+			args[0] = Configuration.perl_path;
+		    }
+		    else {
+			args[0] = "Perl.exe";
+		    }
+		    args[1] = LocalGreenstone.getBinScriptDirectoryPath() + "pluginfo.pl";
+		    args[2] = "-xml";
+		    args[3] = "-language";
+		    args[4] = lang;
+		    args[5] = getPluginName(plugin);
+		}
+		else {
+		    args = new String[5];
+		    args[0] = "pluginfo.pl";
+		    args[1] = "-xml";
+		    args[2] = "-language";
+		    args[3] = lang;
+		    args[4] = getPluginName(plugin);
+		}
+
+		// Create the process.
+		Runtime runtime = Runtime.getRuntime();
+		Process process = runtime.exec(args);
+		input_stream = process.getErrorStream();
+		xml = Utility.readXMLStream(input_stream);
+	    }
+
+	    document = CollectionDesignManager.XMLStringToDOM(xml,plugin);
+	}
+	catch (Exception error) {
+	    System.err.println("Failed when trying to parse: " + plugin);
+	    error.printStackTrace();
+	}
+	if(document != null) {
+	    parseXML(document.getDocumentElement());
+	}
+    }
+
+    /** Method to initially load information from the standard plug-ins within the gsdl Perl library.
+     */
+    private void loadPlugins() {
+	// Attempt to restore the cached file.
+	File plugins_dat_file = new File(Gatherer.getGLIUserDirectoryPath() + "plugins.dat");
+	try {
+	    FileInputStream file = new FileInputStream(plugins_dat_file);
+	    ObjectInputStream input = new ObjectInputStream(file);
+	    library = (ArrayList) input.readObject();
+	}
+	catch (Exception error) {
+	    DebugStream.println("Unable to open " + plugins_dat_file);
+	}
+
+	if(library == null) {
+	    library = new ArrayList();
+
+	    if (Gatherer.isGsdlRemote) {
+		String pluginfo_output = RemoteGreenstoneServer.getScriptOptions("pluginfo.pl", "&listall=");
+		loadPlugins(new StringBuffer(pluginfo_output));
+	    }
+
+	    else {
+		// Retrieve the gsdl home directory...
+		String directory = LocalGreenstone.getDirectoryPath();
+		directory = directory + "perllib" + File.separator + "plugins" + File.separator;
+		loadPlugins(new File(directory));
+	    }
+	}
+    }
+
+
+
+
+    /** Method to load plug-in information from a specified input stream (could be local or through URL). Of course no plug-ins may be found at this location.
+     * @param input_stream An <strong>InputStream</strong> indicating the where list of plugins -- encoded in XML -- can be read from
+     */
+    private void loadPlugins(StringBuffer xml)
+    {
+	Document document = CollectionDesignManager.XMLStringToDOM(xml, "-listall");
+
+	// Parse XML to build up list of plugin names
+	Node root = document.getDocumentElement();
+
+	NamedNodeMap attributes = root.getAttributes();
+	Node length_node = attributes.getNamedItem("length");
+	String num_plugins_str = length_node.getNodeValue();
+	int num_plugins = Integer.parseInt(num_plugins_str);
+	String plugin_list[] = new String[num_plugins];
+
+	Node node = root.getFirstChild();
+	int i = 0;
+	while (node != null) {
+	    String node_name = node.getNodeName();
+	    if (node_name.equalsIgnoreCase("PluginName")) {
+		String name = XMLTools.getValue(node);
+		plugin_list[i] = name;
+		i++;		
+	    }
+
+	    node = node.getNextSibling();
+	}
+
+
+	boolean is_windows = Utility.isWindows();
+	boolean is_mac = Utility.isMac();
+
+	String current_lang = Configuration.getLanguage();
+	if (num_plugins>0) {
+	    // Create a progress indicator.
+	    ParsingProgress progress = new ParsingProgress(Dictionary.get("CDM.PlugInManager.Parsing.Title"), Dictionary.get("CDM.PlugInManager.Parsing.Message"), num_plugins);
+
+	    for (i=0; i<num_plugins; i++) {
+		String plugin = plugin_list[i];
+		if (plugin.equals("GMLPlug.pm") || ((is_windows || is_mac) && plugin.equals("DBPlug.pm"))) {
+		    // don't load GMLPlug or DBPlug for windows
+		} else {
+		    loadPlugin(plugin, current_lang);
+		}
+		
+		progress.inc();
+	    }
+	    progress.dispose();
+	    progress.destroy();
+	    progress = null;
+	}
+    }
+
+
+    /** Method to load plug-in information from a specified directory. Of course no plug-ins may be found at this location.
+     * @param directory A <strong>File</strong> indicating the directory to be scanned for plug-ins.
+     */
+    private void loadPlugins(File directory) {
+	File files[] = directory.listFiles();
+	boolean is_windows = Utility.isWindows();
+	boolean is_mac = Utility.isMac();
+	String current_lang = Configuration.getLanguage();
+	if(files != null) {
+	    // Create a progress indicator.
+	    ParsingProgress progress = new ParsingProgress(Dictionary.get("CDM.PlugInManager.Parsing.Title"), Dictionary.get("CDM.PlugInManager.Parsing.Message"), files.length);
+	    for(int i = 0; i < files.length; i++) {
+		// We only want to check Perl Modules.
+		if(files[i].getName().endsWith(".pm")) {
+		    if (files[i].getName().equals("GMLPlug.pm") || ((is_windows || is_mac) && files[i].getName().equals("DBPlug.pm"))) {
+			// don't load GMLPlug or DBPlug for windows
+		    } else {
+			loadPlugin(files[i].getName(), current_lang);
+		    }
+		}
+		progress.inc();
+	    }
+	    progress.dispose();
+	    progress.destroy();
+	    progress = null;
+	}
+    }
+
+    private Plugin parseXML(Node root) {
+	Plugin plugin = new Plugin();
+	String node_name = null;
+	for (Node node = root.getFirstChild(); node != null; node = node.getNextSibling()) {
+	    node_name = node.getNodeName();
+	    if(node_name.equalsIgnoreCase("Name")) {
+		String name = XMLTools.getValue(node);
+		// We can save ourselves some processing time if a plugin with this name already exists in our manager. If so retrieve it and return it.
+		Plugin existing = getBasePlugin(name);
+		if(existing != null) {
+		    return existing;
+		}
+		plugin.setName(name);
+	    }
+	    else if (node_name.equalsIgnoreCase("Desc")) {
+		plugin.setDescription(XMLTools.getValue(node));
+	    }
+	    else if (node_name.equalsIgnoreCase("Abstract")) {
+		plugin.setIsAbstract(XMLTools.getValue(node).equalsIgnoreCase(StaticStrings.YES_STR));
+	    }
+	    else if (node_name.equalsIgnoreCase("Explodes")) {
+		plugin.setDoesExplodeMetadataDatabases(XMLTools.getValue(node).equalsIgnoreCase(StaticStrings.YES_STR));
+		//System.err.println("Plugin " + plugin.getName() + " explodes metadata databases: " + plugin.doesExplodeMetadataDatabases());
+	    }
+	    // Parse the multitude of arguments
+	    else if(node_name.equalsIgnoreCase("Arguments")) {
+		for(Node arg = node.getFirstChild(); arg != null; arg = arg.getNextSibling()) {
+		    node_name = arg.getNodeName();
+		    // An option.
+		    if(node_name.equalsIgnoreCase("Option")) {
+			Argument argument = new Argument();
+			argument.parseXML((Element)arg);
+			plugin.addArgument(argument);
+		    }
+		    // A super plugin class.
+		    else if(node_name.equalsIgnoreCase("PlugInfo")) {
+			Plugin super_plugin = parseXML(arg);
+			plugin.setSuper(super_plugin);
+		    }
+		}
+	    }
+	}
+	if(plugin.getName() != null) {
+	    addPlugin(plugin);
+	    return plugin;
+	}
+	return null;
+    }
+
+    /** A class which provodes controls for assigned and editing plugins. */
+    private class PluginControl
+	extends JPanel
+	implements Control {
+	/** Button for adding plugins. */
+	private JButton add = null;
+	/** Button for configuring the selected plugin. */
+	private JButton configure = null;
+	/** Buttom to move an assinged plugin as low in the order as possible. */
+	//private JButton move_bottom_button = null;
+	/** Button to move an assigned plugin one position lower in the order. */
+	private JButton move_down_button = null;
+	/** Button to move an assigned plugin as high in the order as possible. */
+	//private JButton move_top_button = null;
+	/** Button to move an assigned plugin one position higher in the order. */
+	private JButton move_up_button = null;
+	/** Button to remove the selected plugin. */
+	private JButton remove = null;
+	/** A combobox containing all of the known plugins, including those that may have already been assigned. */
+	private GComboBox plugin = null;
+	/** The label next to the plugin combobox. */
+	private JLabel plugin_label = null;
+	/** The label above the assigned plugin list. */
+	private JLabel plugin_list_label = null;
+	/** The title of this view. */
+	private JLabel title = null;
+	/** A list of assigned plugins. */
+	private JList plugin_list = null;
+	/** The area where the add, configure and remove buttons are placed. */
+	private JPanel button_pane = null;
+	/** The region which divides the central portion of the view into list and controls */
+	private JPanel central_pane = null;
+	/** The area where title label and instructions sit. */
+	private JPanel header_pane = null;
+	/** The area where movement buttons are placed. */
+	private JPanel movement_pane = null;
+	/** The small region containing the plugin combobox and its label. */
+	private JPanel plugin_pane = null;
+	/** The pane containing the assigned plugin list and its label. */
+	private JPanel plugin_list_pane = null;
+	/** The text area containing instructions on the use of this control. */
+	private JTextArea instructions = null;
+
+	/** Constructor.
+	 */
+	public PluginControl() {
+	    // Create
+	    add = new GLIButton();
+	    add.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add, "CDM.PlugInManager.Add", "CDM.PlugInManager.Add_Tooltip");
+
+	    button_pane = new JPanel();
+	    central_pane = new JPanel();
+
+	    configure = new GLIButton();
+	    configure.setEnabled(false);
+	    configure.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerBoth(configure, "CDM.PlugInManager.Configure", "CDM.PlugInManager.Configure_Tooltip");
+
+	    header_pane = new JPanel();
+
+	    instructions = new JTextArea();
+	    instructions.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    instructions.setEditable(false);
+	    instructions.setLineWrap(true);
+	    instructions.setRows(6);
+	    instructions.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions, "CDM.PlugInManager.Instructions");
+
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    movement_pane = new JPanel();
+
+	    PluginComboboxListener picl = new PluginComboboxListener();
+	    plugin = new GComboBox(getAvailable());
+	    plugin.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    plugin.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    plugin.setEditable(true);
+	    plugin.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    plugin.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    picl.itemStateChanged(new ItemEvent(plugin, 0, null, ItemEvent.SELECTED));
+
+	    plugin_label = new JLabel();
+	    Dictionary.registerText(plugin_label, "CDM.PlugInManager.PlugIn");
+
+	    plugin_list = new JList(model);
+	    plugin_list.setCellRenderer(new ListRenderer());
+	    plugin_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    plugin_list_label = new JLabel();
+	    plugin_list_label.setHorizontalAlignment(JLabel.CENTER);
+	    plugin_list_label.setOpaque(true);
+	    Dictionary.registerText(plugin_list_label, "CDM.PlugInManager.Assigned");
+
+	    plugin_list_pane = new JPanel();
+	    plugin_pane = new JPanel();
+
+	    remove = new GLIButton();
+	    remove.setEnabled(false);
+	    remove.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove, "CDM.PlugInManager.Remove", "CDM.PlugInManager.Remove_Tooltip");
+
+	    title = new JLabel();
+	    title.setHorizontalAlignment(JLabel.CENTER);
+	    title.setOpaque(true);
+	    Dictionary.registerText(title, "CDM.PlugInManager.Title");
+
+	    // Listeners
+	    add.addActionListener(new AddListener()); //all_change_listener is listening to the ArgumentConfiguration
+	    configure.addActionListener(new ConfigureListener());
+	    MoveListener ml = new MoveListener();
+	    //move_bottom_button.addActionListener(ml);
+	    move_down_button.addActionListener(ml);
+	    move_down_button.addActionListener(CollectionDesignManager.all_change_listener);
+	    //move_top_button.addActionListener(ml);
+	    move_up_button.addActionListener(ml);
+	    move_up_button.addActionListener(CollectionDesignManager.all_change_listener);
+	    plugin.addItemListener(picl);
+	    remove.addActionListener(new RemoveListener());
+	    remove.addActionListener(CollectionDesignManager.all_change_listener);
+	    plugin_list.addMouseListener(new ClickListener());
+	    plugin_list.addListSelectionListener(new ListListener());
+	    picl = null;
+
+	    // Layout
+	    title.setBorder(BorderFactory.createEmptyBorder(0,0,2,0));
+
+	    instructions.setBorder(BorderFactory.createEmptyBorder(2,5,2,5));
+
+	    header_pane.setLayout(new BorderLayout());
+	    header_pane.add(title, BorderLayout.NORTH);
+	    header_pane.add(new JScrollPane(instructions), BorderLayout.CENTER);
+
+	    plugin_list_label.setBorder(BorderFactory.createEmptyBorder(0,2,0,2));
+
+	    movement_pane.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+	    movement_pane.setLayout(new GridLayout(4,1));
+	    movement_pane.add(move_up_button);
+	    movement_pane.add(new JPanel());
+	    movement_pane.add(new JPanel());
+	    movement_pane.add(move_down_button);
+
+	    plugin_list_pane.setLayout(new BorderLayout());
+	    plugin_list_pane.add(plugin_list_label, BorderLayout.NORTH);
+	    plugin_list_pane.add(new JScrollPane(plugin_list), BorderLayout.CENTER);
+	    modeChanged(Configuration.getMode()); // Whether the movement buttons are visible is mode dependant
+
+	    plugin_label.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+
+	    plugin_pane.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	    plugin_pane.setLayout(new BorderLayout(5,0));
+	    plugin_pane.add(plugin_label, BorderLayout.WEST);
+	    plugin_pane.add(plugin, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,3));
+	    button_pane.add(add);
+	    button_pane.add(configure);
+	    button_pane.add(remove);
+
+	    // Scope these mad bordering skillz.
+	    JPanel temp = new JPanel(new BorderLayout());
+	    temp.setBorder
+		(BorderFactory.createCompoundBorder
+		 (BorderFactory.createEmptyBorder(5,0,5,0),
+		  BorderFactory.createCompoundBorder
+		  (BorderFactory.createTitledBorder(Dictionary.get("CDM.PlugInManager.Controls")),
+		   BorderFactory.createEmptyBorder(2,2,2,2))));
+
+	    temp.add(plugin_pane, BorderLayout.NORTH);
+	    temp.add(button_pane, BorderLayout.SOUTH);
+
+	    central_pane.setLayout(new BorderLayout());
+	    central_pane.add(plugin_list_pane, BorderLayout.CENTER);
+	    central_pane.add(temp, BorderLayout.SOUTH);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_pane, BorderLayout.NORTH);
+	    add(central_pane, BorderLayout.CENTER);
+	}
+
+	/** Method which acts like a destructor, tidying up references to persistant objects.
+	 */
+	public void destroy() {
+	}
+
+	/** This method is overridden to ensure the instructions are scrolled to top, before the super classes updateUI() is called.
+	 */
+	public void gainFocus() {
+	    if(instructions != null) {
+		instructions.setCaretPosition(0);
+	    }
+	    super.updateUI();
+	}
+
+	public void loseFocus() {
+	}
+
+	/** The current detail mode controls two aspects of plugin manager: whether the movement buttons are visible and whether the fixed position plugins Arc and RecPlug are in the list
+	 * @param mode the current mode as an int, which can be matched against static ints in the Configuration class
+	 */
+	public void modeChanged(int mode) {
+	    // First of all we clear the current selection, as there can be some serious problems if the user selects the plugins we're hiding, or had the last plugin selected before we unhid the last three
+	    plugin_list.clearSelection();
+	    // The first change is dependant on whether the user is systems mode or higher
+	    if(mode >= Configuration.SYSTEMS_MODE) {
+		// Show movement buttons
+		plugin_list_pane.add(movement_pane, BorderLayout.EAST);
+		// Do we show Arc and RecPlugs or hide them and the separator line
+		setHideLines(!(mode >= Configuration.EXPERT_MODE));
+	    }
+	    // Otherwise hide the movement buttons and fixed plugins
+	    else {
+		plugin_list_pane.remove(movement_pane);
+		setHideLines(true);
+	    }
+	    plugin_list_pane.updateUI();
+	}
+
+	/** This class listens for actions upon the add button in the controls, and if detected calls the <i>assignPlugin()</i> method. */
+	private class AddListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Object selected_object = plugin.getSelectedItem();
+		if(selected_object != null) {
+		    // Retrieve the base plugin if any
+		    Plugin base_plugin = getBasePlugin(selected_object.toString());
+
+		    // Create a new element in the DOM
+		    Element element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.PLUGIN_ELEMENT);
+		    // Remember that the plugin supplied might be a custom string rather than a base plugin
+		    Plugin new_plugin = null;
+		    if(base_plugin != null) {
+			//DebugStream.println("New Plugin based on existing Plugin");
+			element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, base_plugin.getName());
+			new_plugin = new Plugin(element, base_plugin);
+		    }
+		    else {
+			//DebugStream.println("New Custom Plugin");
+			element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, selected_object.toString());
+			new_plugin = new Plugin(element, null);
+		    }
+		    if(!model.contains(new_plugin) || new_plugin.getName().equals(StaticStrings.UNKNOWNPLUG_STR)) {
+			// Automatically chain to configuration. This ensures required arguments are filled out.
+			ArgumentConfiguration ac = new ArgumentConfiguration(new_plugin);
+			if(ac.display()) {
+			    assignPlugin(new_plugin);
+			    plugin_list.setSelectedValue(new_plugin, true);
+			    // Since we weren't cancelled, and if there was a base plugin, ensure it no longer is shown as available, unless it is the UnknownPlugin which can be added several times
+			    if(base_plugin != null && !base_plugin.getName().equals(StaticStrings.UNKNOWNPLUG_STR)) {
+				plugin.removeItem(base_plugin);
+			    }
+			}
+			ac = null;
+			new_plugin = null;
+			plugin.setSelectedIndex(0);
+		    }
+		    else {
+			JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CDM.PlugInManager.PlugIn_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		    }
+		    base_plugin = null;
+		}
+	    }
+	}
+
+	/** Listens for double clicks apon the list and react as if the configure button was pushed. */
+	private class ClickListener
+	    extends MouseAdapter {
+	    /** Called whenever the mouse is clicked over a registered component, we use this to chain through to the configure prompt.
+	     * @param event A <strong>MouseEvent</strong> containing information about the mouse click.
+	    */
+	    public void mouseClicked(MouseEvent event) {
+		if(event.getClickCount() == 2 ) {
+		    if(!plugin_list.isSelectionEmpty()) {
+			Plugin plugin = (Plugin) plugin_list.getSelectedValue();
+			if(!plugin.isSeparator()) {
+			    ArgumentConfiguration ac = new ArgumentConfiguration(plugin);
+			    if(ac.display()) {
+				refresh(plugin);
+			    }
+			    ac.destroy();
+			    ac = null;
+			    // cos I can't be bothered checking every argument to see if it has changed or not, we'll asasume that the configuration has changed if someone has clicked configure
+			    Gatherer.c_man.configurationChanged();
+			}
+		    }
+		}
+	    }
+	}
+
+	/** This class listens for actions upon the configure button in the controls, and if detected creates a new <strong>ArgumentConfiguration</strong> dialog box to allow for configuration.
+	 * @see org.greenstone.gatherer.cdm.ArgumentConfiguration
+	 */
+	private class ConfigureListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!plugin_list.isSelectionEmpty()) {
+		    Plugin plugin = (Plugin) plugin_list.getSelectedValue();
+		    if(!plugin.isSeparator()) {
+			ArgumentConfiguration ac = new ArgumentConfiguration(plugin);
+			if(ac.display()) {
+			    refresh(plugin);
+			}
+			ac.destroy();
+			ac = null;
+			// cos I can't be bothered checking every argument to see if it has changed or not, we'll asasume that the configuration has changed if someone has clicked configure
+			Gatherer.c_man.configurationChanged();
+		    }
+		}
+	    }
+	}
+
+	/** listens for changes in the list selection and enables the configure and remove buttons if there is a selection, disables them if there is no selection */
+	private class ListListener
+	    implements ListSelectionListener {
+
+	    public void valueChanged(ListSelectionEvent e) {
+		if (!e.getValueIsAdjusting()) { // we get two events for one change in list selection - use the false one (the second one)
+		    if (plugin_list.isSelectionEmpty()) {
+			//move_top_button.setEnabled(false);
+			move_up_button.setEnabled(false);
+			move_down_button.setEnabled(false);
+			//move_bottom_button.setEnabled(false);
+			configure.setEnabled(false);
+			remove.setEnabled(false);
+		    }
+		    else {
+			Plugin selected_plugin = (Plugin) plugin_list.getSelectedValue();
+			if(selected_plugin.isSeparator()) {
+			    //move_top_button.setEnabled(false);
+			    move_up_button.setEnabled(false);
+			    move_down_button.setEnabled(false);
+			    //move_bottom_button.setEnabled(false);
+			    configure.setEnabled(false);
+			    remove.setEnabled(false);
+			}
+			else {
+			    configure.setEnabled(true);
+			    String plugin_name = selected_plugin.getName();
+			    // Some buttons are only available for plugins other than ArcPlug and RecPlug 
+			    if(plugin_name.equals(StaticStrings.ARCPLUG_STR) || plugin_name.equals(StaticStrings.RECPLUG_STR) ) {
+				//move_top_button.setEnabled(false);
+				move_up_button.setEnabled(false);
+				move_down_button.setEnabled(false);
+				//move_bottom_button.setEnabled(false);
+				remove.setEnabled(false);
+			    }
+			    else {
+				// don't let people remove special plugins such GAPlug an METSPlug,
+				// unless they are in systems mode or above
+				int mode = Configuration.getMode();
+				for (int i=0; i<StaticStrings.KEEP_PLUG.length; i++) {
+				    if ((plugin_name.equals(StaticStrings.KEEP_PLUG[i])) &&
+					(mode < Configuration.SYSTEMS_MODE)) {
+					remove.setEnabled(false); 
+					break;
+				    } else {
+					remove.setEnabled(true);
+				    }
+				}
+				    
+				// Move ups are only enabled if the selected plugin isn't already at the top 
+				Plugin first_plugin = (Plugin) getElementAt(0);
+				if(!first_plugin.equals(selected_plugin)) {
+				    //move_top_button.setEnabled(true);
+				    move_up_button.setEnabled(true);
+				}
+				else {
+				    //move_top_button.setEnabled(false);
+				    move_up_button.setEnabled(false);
+				}
+				// And move downs are only allowed when the selected plugin isn't at an index one less than the separator line.
+				int separator_index = findSeparatorIndex();
+				int selected_index = plugin_list.getSelectedIndex();
+				if(selected_index < separator_index - 1) {
+				    move_down_button.setEnabled(true);
+				    //move_bottom_button.setEnabled(true);
+				}
+				else {
+				    move_down_button.setEnabled(false);
+				    //move_bottom_button.setEnabled(false);
+				}
+			    }
+			    selected_plugin = null;
+			    plugin_name = null;
+			}
+		    }
+		}
+	    }
+	}
+
+	/** A special list renderer which is able to render separating lines as well. */
+	private class ListRenderer
+	    extends DefaultListCellRenderer {
+	    /** Return a component that has been configured to display the specified value. That component's paint method is then called to "render" the cell. If it is necessary to compute the dimensions of a list because the list cells do not have a fixed size, this method is called to generate a component on which getPreferredSize  can be invoked.
+	     * @param list - The <strong>JList</strong> we're painting.
+	     * @param value - The value returned by list.getModel().getElementAt(index) as an <strong>Object</strong>.
+	     * @param index - The cells index as an <i>int</i>.
+	     * @param isSelected - <i>true</i> if the specified cell was selected.
+	     * @param cellHasFocus - <i>true</i> if the specified cell has the focus.
+	     * @return A <strong>Component</strong> whose paint() method will render the specified value.
+	     * @see javax.swing.JList
+	     * @see javax.swing.JSeparator
+	     * @see javax.swing.ListModel
+	     * @see javax.swing.ListSelectionModel
+	     */
+	    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+		Plugin plugin = (Plugin) value;
+		if(plugin.isSeparator()) {
+		    return separator;
+		}
+		else {
+		    return super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+		}
+	    }
+	}
+
+
+	/** Listens for actions apon the move buttons in the manager controls, and if detected calls the <i>movePlugin()</i> method of the manager with the appropriate details. */
+	private class MoveListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if (!plugin_list.isSelectionEmpty()) {
+		    Object object = plugin_list.getSelectedValue();
+		    if (object instanceof Plugin) {
+			Plugin plugin = (Plugin) object;
+			//if (event.getSource() == move_top_button) {
+			//    movePlugin(plugin, true, true);
+			//}
+			//else 
+			if (event.getSource() == move_up_button) {
+			    movePlugin(plugin, true, false);
+			}
+			else if (event.getSource() == move_down_button) {
+			    movePlugin(plugin, false, false);
+			}
+			//else if (event.getSource() == move_bottom_button) {
+			//    movePlugin(plugin, false, true);
+			//}
+			plugin_list.setSelectedValue(plugin, true);
+		    }
+		}
+	    }
+	}
+
+	/** This listener reacts to changes in the current selection of the plugin combobox. */
+	private class PluginComboboxListener
+        implements ItemListener {
+        /** When a user selects a certain plugin, update the tooltip to show the plugin description. */
+        public void itemStateChanged(ItemEvent event) {
+            if(event.getStateChange() == ItemEvent.SELECTED) {
+                // Retrieve the selected plugin
+                Object current_selection = plugin.getSelectedItem();
+                // And reset the tooltip. If the plugin is null or is a string, then go back to the default message
+                if(current_selection == null || current_selection instanceof String) {
+                    Dictionary.registerTooltip(plugin, "CDM.PlugInManager.PlugIn_Tooltip");
+                }
+                else {
+                    Plugin current_plugin = (Plugin) current_selection;
+                    Dictionary.registerTooltipText(plugin, Utility.formatHTMLWidth(current_plugin.getDescription(), 40));
+                    current_plugin = null;
+                }
+                current_selection = null;
+            }
+        }
+    }
+	
+	/** This class listens for actions upon the remove button in the controls, and if detected calls the <i>removePlugin()</i> method.
+	 */
+	private class RemoveListener
+	    implements ActionListener {
+	    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured on one of our target controls.
+	     * @param event An <strong>ActionEvent</strong> containing information garnered from the control action.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		int selected_index = plugin_list.getSelectedIndex();
+		if(selected_index != -1) {
+		    Plugin selected_plugin = (Plugin) plugin_list.getSelectedValue();
+		    removePlugin(selected_plugin);
+		    selected_plugin = null;
+		    // Select the next plugin if available
+		    if(selected_index < plugin_list.getModel().getSize()) {
+			// If the new selection is above the separator we can remove it
+			if(selected_index < findSeparatorIndex()) {
+			    plugin_list.setSelectedIndex(selected_index);
+			    
+			    // don't let people remove special plugins such GAPlug an METSPlug,
+			    // unless they are in systems mode or above
+			    int mode = Configuration.getMode();
+			    for (int i=0; i<StaticStrings.KEEP_PLUG.length; i++) {
+				String selected_plugin_name
+				    = ((Plugin)plugin_list.getSelectedValue()).getName();
+				if ((selected_plugin_name.equals(StaticStrings.KEEP_PLUG[i])) &&
+				    (mode < Configuration.SYSTEMS_MODE)) {
+				    remove.setEnabled(false); 
+				    break;
+				} else {
+				    remove.setEnabled(true);
+				}
+			    }					
+			}
+			// Otherwise select the first non-removable plugin
+			else {
+			    plugin_list.setSelectedIndex(selected_index + 1);
+			    remove.setEnabled(false);
+			}
+		    }
+		    else {
+			remove.setEnabled(false);
+		    }
+		    // Refresh the available plugins
+		    plugin.setModel(new DefaultComboBoxModel(getAvailable()));
+		}
+		else {
+		    remove.setEnabled(false);
+		}
+	    }
+	}
+    }
+    
+
+    private class PluginSuggestionPrompt
+	extends ModalDialog
+	implements ActionListener
+    {
+	private Dimension size = new Dimension(480, 240);
+	private GComboBox suitable_plugins_combobox = null;
+	private GLIButton add_button = null;
+	private GLIButton ignore_button = null;
+
+	public PluginSuggestionPrompt(String filename, ArrayList suitable_plugins)
+	{
+	    super(Gatherer.g_man, true);
+	    setModal(true);
+	    setSize(size);
+	    Dictionary.setText(this, "CDM.PluginManager.SuggestedPluginListTitle");
+
+	    String[] args = new String[1];
+	    args[0] = filename;
+
+	    JTextArea instructions_textarea = new JTextArea();
+	    instructions_textarea.setCaretPosition(0);
+	    instructions_textarea.setEditable(false);
+	    instructions_textarea.setLineWrap(true);
+	    instructions_textarea.setRows(5);
+	    instructions_textarea.setWrapStyleWord(true);
+	    Dictionary.setText(instructions_textarea, "CDM.PluginManager.Plugin_Suggestion_Prompt", args);
+
+	    JLabel suitable_plugins_label = new JLabel();
+	    Dictionary.registerText(suitable_plugins_label, "CDM.PlugInManager.PlugIn");
+	    suitable_plugins_label.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+
+	    suitable_plugins_combobox = new GComboBox(suitable_plugins);
+	    suitable_plugins_combobox.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    suitable_plugins_combobox.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    suitable_plugins_combobox.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    suitable_plugins_combobox.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+
+	    JPanel suitable_plugins_pane = new JPanel();
+	    //suitable_plugins_pane.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	    suitable_plugins_pane.setLayout(new BorderLayout(5,0));
+	    suitable_plugins_pane.add(suitable_plugins_label, BorderLayout.WEST);
+	    suitable_plugins_pane.add(suitable_plugins_combobox, BorderLayout.CENTER);
+
+	    add_button = new GLIButton();
+	    Dictionary.setBoth(add_button, "CDM.PlugInManager.Add", "CDM.PlugInManager.Add_Tooltip");
+	    ignore_button = new GLIButton();
+	    Dictionary.setBoth(ignore_button, "CDM.PlugInManager.Ignore","CDM.PlugInManager.Ignore_Tooltip" );
+
+	    add_button.addActionListener(this);
+	    ignore_button.addActionListener(this);
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setLayout(new GridLayout(1,2,5,0));
+	    button_pane.add(add_button);
+	    button_pane.add(ignore_button);
+
+	    JPanel controls_pane = new JPanel();
+	    controls_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    controls_pane.setLayout(new GridLayout(2,1,0,5));
+	    controls_pane.add(suitable_plugins_pane);
+	    controls_pane.add(button_pane);
+
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(instructions_textarea, BorderLayout.CENTER);
+	    content_pane.add(controls_pane, BorderLayout.SOUTH);
+
+	    // Show
+	    Dimension screen_size = Configuration.screen_size;
+	    setLocation((screen_size.width - size.width) / 2, (screen_size.height - size.height) / 2);
+	    setVisible(true);
+	}
+
+	public void actionPerformed(ActionEvent event) {
+	    
+	    if(event.getSource() == add_button) {
+		// add the selected plugin to the list
+		Object selected_object = suitable_plugins_combobox.getSelectedItem();
+		Plugin base_plugin = getBasePlugin(selected_object.toString());
+
+		// Create a new element in the DOM
+		Element element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.PLUGIN_ELEMENT);
+		// Remember that the plugin supplied might be a custom string rather than a base plugin
+		Plugin new_plugin = null;
+		if(base_plugin != null) {
+		    //DebugStream.println("New Plugin based on existing Plugin");
+		    element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, base_plugin.getName());
+		    new_plugin = new Plugin(element, base_plugin);
+		}
+		else {
+			//DebugStream.println("New Custom Plugin");
+		    element.setAttribute(CollectionConfiguration.TYPE_ATTRIBUTE, selected_object.toString());
+		    new_plugin = new Plugin(element, null);
+		}
+		assignPlugin(new_plugin);
+	    } // else do nothing
+	    
+	    // close the dialog
+	    setVisible(false);
+	}
+
+    }
+
+
+    /** Creates a list separator.
+     * Found on Google Groups. Code courtesy of Paul Farwell.
+     */
+    private JPanel getSeparator() {
+	// We put the separator inside a panel to control its appearance
+	JPanel _sepPanel = new JPanel();
+	_sepPanel.setOpaque(false);
+	_sepPanel.setBorder(BorderFactory.createEmptyBorder(1, 3, 1, 3));
+	_sepPanel.setLayout(new BoxLayout(_sepPanel, BoxLayout.Y_AXIS));
+	_sepPanel.add(Box.createRigidArea(new Dimension(0, 4)));
+	// We have to be a little careful here, as the default UI for separators under MacOS is a blank box. Instead we force a BasicUI look
+	_sepPanel.add(new BasicSeparator());
+	_sepPanel.add(Box.createRigidArea(new Dimension(0, 4)));
+	return _sepPanel;
+    }
+
+    /** This class behaves just like a normal JSeparator except that, no matter what the current settings in the UIManager are, it always paints itself with BasicSeparatorUI. */
+    private class BasicSeparator
+	extends JSeparator {
+
+	private ComponentUI basic_ui;
+
+	public BasicSeparator() {
+	    super();
+	    basic_ui = new BasicSeparatorUI();
+	}
+
+	public void paintComponent(Graphics g) {
+	    if (basic_ui != null) {
+		basic_ui.update(g, this);
+	    }   
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchType.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchType.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchType.java	(revision 21112)
@@ -0,0 +1,159 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      16/07/03
+ * Revised:      
+ **************************************************************************************/
+import org.w3c.dom.*;
+
+/** This class represents a single search type setting which means that, at the moment, it is either based on an Content element containing the string 'plain' or one on the string 'form'. The ordering of these elements within the SearchType element is important.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class SearchType
+    implements DOMProxyListEntry {
+
+    /** The Element this searchtype object will source its information from. */
+    private Element element;
+    /** A cached version of the string representation, for speed of painting (given toString() might be called several times during a list repaint). */
+    private String text;
+
+    /** Constructor used only during DOMProxyListModel initialization. */
+    public SearchType() {
+    }
+
+    /** Normal constructor.
+     * @param  element the Element this object will find its data from
+     */
+    public SearchType(Element element) {
+	this.element = element;
+    }
+
+    /** Creation of a brand new SearchType 
+     * @param  name the name of this type as a String
+     */
+    public SearchType(String name) {
+	element = CollectionDesignManager.collect_config.document.createElement(CollectionConfiguration.CONTENT_ELEMENT);
+	element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, name);
+	text = null;
+    }
+
+    /** Compare two objects for ordering. 
+     * @param  object the other Object to compare to
+     * @return <0 if this searchtype should be before the given object, 0 if they are equal, and >0 if it should be after
+     */
+    public int compareTo(Object object) {
+	if(object == null) {
+	    return -1;
+	}
+	return toString().compareTo(object.toString());
+    }
+
+    /** Factory-like method to allow DOMProxyListModel to generate new entries.
+     * @param  element the Element the new searchtype should be based on
+     * @return a newly created DOMProxyListEntry for the given element
+     */
+    public DOMProxyListEntry create(Element element) {
+	return new SearchType(element);
+    }
+
+    /** Determine if this searchtype is equavilent to another object.
+     * @param  object the other Other to match against
+     * @return true if the two are equal, false otherwise
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Retrieve the element this DOMProxyListEntry is based upon. Specified by the interface.
+     * @return the Element in question
+     */
+    public Element getElement() {
+	return element;
+    }
+
+    /** Retrieve the name of this searchtype.
+     * @return the name as a String
+     */
+    public String getName() {
+	return toString();
+    }
+
+    /** Determine is this command has been assigned, either because it already existed in the collection configuration, or because it has been explicitly set by the user. Non-assigned entries imply they have been added by the GLI to ensure consistancy (and avoid NPE's!)
+     * @return true if this command has been assigned, false otherwise
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.util.StaticStrings
+     */
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+    
+    /** Set the assigned state.
+     * @param  assigned the desired state of assigned as a boolean
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.util.StaticStrings
+     */
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    /** Set the element that this DOMProxyListEntry is base on.
+     * @param  element the new Element that this entry should source informatin from
+     */
+    public void setElement(Element element) {
+	this.element = element;
+	this.text = null;
+    }
+
+    /** Set the name of this searchtype.
+     * @param  name the new name for this searchtype, as a String
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.util.StaticStrings
+     */
+    public void setName(String name) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, name);
+	    this.text = null;
+	}
+    }
+
+    /** Produce a text representation of this searchtype.
+     * @return a String showing this searchtype
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.util.StaticStrings
+     */
+    public String toString() {
+	if(text == null && element != null) {
+	    text = element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	}
+	return text;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchTypeManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchTypeManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SearchTypeManager.java	(revision 21112)
@@ -0,0 +1,545 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:      16/07/03
+ * Revised:
+ **************************************************************************************/
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.util.JarTools;
+import org.w3c.dom.*;
+
+/** This class maintains an ordered list of the search types available in the collection (MGPP/lucene command available in G2.39 or later). Currently only 'form' and 'plain' are valid.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class SearchTypeManager
+    extends DOMProxyListModel
+{
+    static final private Dimension LABEL_SIZE = new Dimension(150, 25);
+
+    static final public String BUILD_TYPE_LUCENE = "lucene";
+    static final public String BUILD_TYPE_MGPP = "mgpp";
+    
+    static final public String[] SEARCH_TYPES = {  "plain" , "form"};
+    static final public String[] BUILD_TYPES = {  BUILD_TYPE_LUCENE, BUILD_TYPE_MGPP };
+    
+    /** the buildtype element in the config file  - searchtypemanager looks 
+     * after this now */
+    public CollectionMeta build_type = null;
+    /** The controls used to edit the search types. */
+    private Control controls = null;
+    /** A reference to ourselves so our inner classes have access. */
+    private DOMProxyListModel model;
+
+    public SearchTypeManager(Element searchtypes_element) {
+	super(searchtypes_element, CollectionConfiguration.CONTENT_ELEMENT, new SearchType());
+	this.model = this;
+	DebugStream.println("SearchTypeManager: parsed " + getSize() + " search types.");
+	build_type = new CollectionMeta(CollectionDesignManager.collect_config.getBuildType());
+
+    }
+
+    private void addSearchType(SearchType searchtype) {
+	if(!contains(searchtype)) {
+	    add(getSize(), searchtype);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+    
+    public Control getControls() {
+	if(controls == null) {
+	    controls = new SearchTypeControl();
+	}
+	return controls;
+    }
+
+    /** Return a list of the currently assigned search types as a comma separated string.
+     * @return a String
+     */
+    public String getSearchTypes() {
+	StringBuffer search_types = new StringBuffer();
+	ArrayList types = children();
+	for (int i = 0; i < types.size(); i++) {
+	    if (i>0) {
+		search_types.append(",");
+	    }
+	    search_types.append(((SearchType)types.get(i)).getName());
+	}
+	return search_types.toString();
+    }
+    
+    /** By examining the SearchType 'root' we were created with, determine if mgpp is enabled.
+     * @return true if MGPP is enabled, false otherwise
+     */
+    public boolean isSearchTypeEnabled() {
+	return root.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.TRUE_STR);
+    }
+    
+    public boolean isMGPP() {
+	return isSearchTypeEnabled() && build_type.getValue(CollectionMeta.TEXT).equals(BUILD_TYPE_MGPP);
+    }
+
+    public boolean isLucene() {
+	return isSearchTypeEnabled() && build_type.getValue(CollectionMeta.TEXT).equals(BUILD_TYPE_LUCENE);
+    }
+    
+    private void moveSearchType(SearchType search_type, boolean direction) {
+	// Try to move the classifier one step in the desired direction.
+	int index = indexOf(search_type);
+	if(direction) {
+	    index--;
+	}
+	else {
+	    index++;
+	}
+	// Check we aren't at an edge
+	if((direction && index < 0) || (!direction && index >= getSize())) {
+	    String args[] = new String[2];
+	    args[0] = Dictionary.get("CDM.SearchTypeManager.SearchType");
+	    args[1] = search_type.toString();
+	    String message = null;
+	    if (direction) {
+		message = Dictionary.get("CDM.Move.At_Top", args);
+	    }
+	    else {
+		message = Dictionary.get("CDM.Move.At_Bottom", args);
+	    }
+	    JOptionPane.showMessageDialog(Gatherer.g_man, message, Dictionary.get("CDM.Move.Title"), JOptionPane.ERROR_MESSAGE);
+	    return;
+	}
+	remove(search_type);
+	add(index, search_type);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    private void removeSearchType(SearchType searchtype) {
+	if(contains(searchtype)) {
+	    remove(searchtype);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    private class SearchTypeControl
+	extends JPanel
+	implements Control {
+
+	private GComboBox search_type_combobox;
+	private GComboBox build_type_combobox;
+
+	private JButton add_button;
+	private JButton move_down_button;
+	private JButton move_up_button;
+	private JButton remove_button;
+
+	private JCheckBox enable_advanced_searches_checkbox;
+
+	private JLabel current_search_types_label;
+	private JLabel search_type_label;
+	private JLabel title_label;
+
+	private JList current_search_types_list;
+
+	private JTextArea instructions_textarea;
+
+	public SearchTypeControl() {
+	
+	    // Creation
+	    title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title_label, "CDM.SearchTypeManager.Title");
+
+	    JPanel instructions_panel = new JPanel();
+	    instructions_textarea = new JTextArea();
+	    instructions_textarea.setCaretPosition(0);
+	    instructions_textarea.setEditable(false);
+	    instructions_textarea.setLineWrap(true);
+	    instructions_textarea.setRows(6);
+	    instructions_textarea.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions_textarea, "CDM.SearchTypeManager.Instructions");
+
+	    JPanel spacer_panel = new JPanel();
+
+	    JPanel empty_panel = new JPanel();
+
+	    JPanel inner_panel = new JPanel();
+
+	    JPanel indexer_type_panel = new JPanel();
+	    JPanel build_type_panel = new JPanel();
+
+	    enable_advanced_searches_checkbox = new JCheckBox();
+	    Dictionary.registerText(enable_advanced_searches_checkbox, "CDM.SearchTypeManager.Enable");
+
+	    build_type_combobox = new GComboBox(BUILD_TYPES);
+	    build_type_combobox.setPreferredSize(LABEL_SIZE);
+	    build_type_combobox.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    build_type_combobox.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    build_type_combobox.setEditable(true);
+	    build_type_combobox.setSelectedIndex(1);// mgpp is at 1
+	    String selected_value = build_type.getValue(CollectionMeta.TEXT);
+	    if (!selected_value.equals("")) {
+		for (int i=0; i<build_type_combobox.getItemCount();i++) {
+		    String item = (String)build_type_combobox.getItemAt(i);
+		    if (item.equals(selected_value)) {
+			build_type_combobox.setSelectedIndex(i);
+			break;
+		    }
+		}
+	    } else {
+		//build_type.setValue((String)build_type_combobox.getSelectedItem());
+		build_type.setAssigned(isSearchTypeEnabled());
+
+	    }
+	    build_type_combobox.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    build_type_combobox.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.registerTooltip(build_type_combobox, "CDM.SearchTypeManager.BuildType_Selection_Tooltip");
+
+	    JLabel build_type_label = new JLabel();
+	    Dictionary.registerText(build_type_label, "CDM.SearchTypeManager.BuildType_Selection");
+	    JPanel current_search_types_panel = new JPanel();
+	    current_search_types_label = new JLabel();
+	    Dictionary.registerText(current_search_types_label, "CDM.SearchTypeManager.Assigned");
+	    current_search_types_list = new JList(model);
+	    current_search_types_list.setVisibleRowCount(3);
+
+	    JPanel movement_panel = new JPanel();
+
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    JPanel search_type_panel = new JPanel();
+	    search_type_label = new JLabel();
+	    Dictionary.registerText(search_type_label, "CDM.SearchTypeManager.SearchType_Selection");
+	    search_type_combobox = new GComboBox(SEARCH_TYPES);
+	    search_type_combobox.setPreferredSize(LABEL_SIZE);
+	    search_type_combobox.setBackgroundNonSelectionColor(Configuration.getColor("coloring.editable_background", false));
+	    search_type_combobox.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	    search_type_combobox.setEditable(true);
+	    search_type_combobox.setSelectedIndex(0); 
+	    search_type_combobox.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	    search_type_combobox.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.registerTooltip(search_type_combobox, "CDM.SearchTypeManager.SearchType_Selection_Tooltip");
+
+	    JPanel button_panel = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(false);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.SearchTypeManager.Add", "CDM.SearchTypeManager.Add_Tooltip");
+
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.SearchTypeManager.Remove", "CDM.SearchTypeManager.Remove_Tooltip");
+	    
+	  
+	    // Connection
+	    add_button.addActionListener(new AddActionListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    build_type_combobox.addActionListener(new BuildTypeActionListener());
+	    build_type_combobox.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    current_search_types_list.addListSelectionListener(new CurrentSearchTypesListSelectionListener());
+	    enable_advanced_searches_checkbox.addActionListener(new EnableAdvancedSearchesActionListener());
+	    enable_advanced_searches_checkbox.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_up_button.addActionListener(new MoveListener(true));
+	    move_up_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_down_button.addActionListener(new MoveListener(false));
+	    move_down_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveActionListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    SearchTypesActionDocumentListener stadl = new SearchTypesActionDocumentListener();
+	    search_type_combobox.addActionListener(stadl);
+	    search_type_combobox.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    ((JTextField)search_type_combobox.getEditor().getEditorComponent()).getDocument().addDocumentListener(stadl);
+
+	    // Layout
+	    instructions_panel.setBorder(BorderFactory.createEmptyBorder(0,0,2,0));
+	    instructions_panel.setLayout(new BorderLayout());
+	    instructions_panel.add(title_label, BorderLayout.NORTH);
+	    instructions_panel.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+
+	    movement_panel.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+	    movement_panel.setLayout(new GridLayout(2,1,5,0));
+	    movement_panel.add(move_up_button);
+	    movement_panel.add(move_down_button);
+
+	    current_search_types_panel.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	    current_search_types_panel.setLayout(new BorderLayout());
+	    current_search_types_panel.add(current_search_types_label, BorderLayout.NORTH);
+	    current_search_types_panel.add(new JScrollPane(current_search_types_list), BorderLayout.CENTER);
+	    current_search_types_panel.add(movement_panel, BorderLayout.EAST);
+
+	    build_type_panel.setLayout(new BorderLayout());
+	    build_type_panel.add(build_type_label, BorderLayout.WEST);
+	    build_type_panel.add(build_type_combobox, BorderLayout.CENTER);
+
+	    indexer_type_panel.setLayout(new GridLayout(1,2,0,5));
+	    indexer_type_panel.add(enable_advanced_searches_checkbox);
+	    indexer_type_panel.add(build_type_panel);
+	    
+	    button_panel.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	    button_panel.setLayout(new GridLayout(1,2,0,5));
+	    button_panel.add(add_button);
+	    button_panel.add(remove_button);
+
+	    search_type_panel.setLayout(new BorderLayout(5,3));
+	    search_type_panel.add(search_type_label, BorderLayout.WEST);
+	    search_type_panel.add(search_type_combobox, BorderLayout.CENTER);
+	    search_type_panel.add(button_panel, BorderLayout.SOUTH);
+
+	    inner_panel.setLayout(new BorderLayout());
+	    inner_panel.add(indexer_type_panel, BorderLayout.NORTH);
+	    inner_panel.add(current_search_types_panel, BorderLayout.CENTER);
+	    inner_panel.add(search_type_panel, BorderLayout.SOUTH);
+
+	    spacer_panel.setLayout(new BorderLayout());
+	    spacer_panel.add(inner_panel, BorderLayout.NORTH);
+	    spacer_panel.add(empty_panel, BorderLayout.CENTER);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(instructions_panel, BorderLayout.NORTH);
+	    add(spacer_panel, BorderLayout.CENTER);
+	}
+
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	    instructions_textarea.setCaretPosition(0);
+	    validateControls(isSearchTypeEnabled());
+	}
+
+	public void loseFocus() {
+
+	}
+
+	private void validateControls(boolean advanced_search_enabled) {
+	    // Enable or disable controls based on whether MGPP is enabled
+	    // validate add button, which depends on the current combobox selection and the contents of the assigned search types list
+	    Object selected_item = search_type_combobox.getSelectedItem();
+	    add_button.setEnabled(advanced_search_enabled && selected_item != null && !model.contains(selected_item));
+	    // validate other controls.
+	    current_search_types_list.setEnabled(advanced_search_enabled);
+	    build_type_combobox.setEnabled(advanced_search_enabled);
+	    enable_advanced_searches_checkbox.setSelected(advanced_search_enabled);
+	    search_type_combobox.setEnabled(advanced_search_enabled);
+	    remove_button.setEnabled(current_search_types_list.getModel().getSize() > 1 && !current_search_types_list.isSelectionEmpty() && advanced_search_enabled);
+	}
+
+	/** Listenes for actions on the Add button, and if detected adds a new search type. */
+	private class AddActionListener
+	    implements ActionListener {
+	    /** Called when someone actions the Add button.
+	     * @param  event an ActionEvent containing information about the add button click
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Object selected_item = search_type_combobox.getSelectedItem();
+		if(selected_item != null) {
+		    if(search_type_combobox.getSelectedIndex() == -1) {
+			search_type_combobox.insertItemAt(selected_item, search_type_combobox.getItemCount());
+		    }
+		    // Add the search type
+		    SearchType new_searchtype = new SearchType((String)selected_item);
+		    addSearchType(new_searchtype);
+		}
+		add_button.setEnabled(false);
+	    }
+	}
+
+	/** Listens for selections within the search types list and updates the remove button appropriately. */
+	private class CurrentSearchTypesListSelectionListener
+	    implements ListSelectionListener {
+	    /** Called when the selection in the list changes.
+	     * @param  event a ListSelectionEvent containing information about the selection change
+	     */
+	    public void valueChanged(ListSelectionEvent event) {
+		if(!event.getValueIsAdjusting()) {
+		    SearchType search_type = null;
+		    if ((search_type = (SearchType) current_search_types_list.getSelectedValue()) != null) {
+			int index = model.indexOf(search_type);
+			// Move up is only enabled if the current selection isn't at index 0
+			move_up_button.setEnabled(index != 0);
+			// Move down is only enabled if the current selection isn't at index getSize() - 1
+			move_down_button.setEnabled(index != model.getSize() - 1);
+			// Remove is only enabled if this isn't the last search type
+			remove_button.setEnabled(current_search_types_list.getModel().getSize() > 1);
+		    }
+		    else {
+			move_up_button.setEnabled(false);
+			move_down_button.setEnabled(false);
+			remove_button.setEnabled(false);
+		    }
+		}
+	    }
+	}
+
+	/** The most complex listener in this class, this listens for changes to the enable advanced searches checkbox, and when they occur it not only updates the controls on this page, but asks the IndexManager to action the appropriate changes to the underlying DOM so as to support either MG or MGPP styles of indexes. */
+	private class EnableAdvancedSearchesActionListener
+	    implements ActionListener {
+	    /** Called whenever the checkbox is checked or unchecked.
+	     * @param  event an ActionEvent containing information about the checking action
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		Gatherer.g_man.wait(true);
+		boolean advanced_search_enabled = enable_advanced_searches_checkbox.isSelected();
+		
+		model.root.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (advanced_search_enabled ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+		CollectionDesignManager.index_manager.setMGPPEnabled(advanced_search_enabled);
+		validateControls(advanced_search_enabled);
+		if ((!build_type.isAssigned() && advanced_search_enabled) || 
+		    (build_type.isAssigned() && !advanced_search_enabled)) {
+		    
+		    build_type.setAssigned(advanced_search_enabled);
+		    Gatherer.c_man.configurationChanged();
+		}
+		if (build_type.isAssigned() && build_type.getValue(true).equals("")) {
+		    build_type.setValue((String)build_type_combobox.getSelectedItem());
+		}
+		Gatherer.g_man.wait(false);
+	    }
+	}
+
+	private class BuildTypeActionListener
+	    implements	ActionListener {
+
+	    public void actionPerformed(ActionEvent event) {
+		if (!build_type.getValue(CollectionMeta.TEXT).equals((String)build_type_combobox.getSelectedItem())) {
+		    build_type.setValue((String)build_type_combobox.getSelectedItem());	
+		}
+	    }
+	    
+	}
+	/** Listenes for actions on the Remove button, and if detected removes the currently selected search types. */
+	private class RemoveActionListener
+	    implements ActionListener {
+	    /** Called when someone actions the Remove button.
+	     * @param  event an ActionEvent containing information about the remove button click
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!current_search_types_list.isSelectionEmpty()) {
+		    Object[] selected_items = current_search_types_list.getSelectedValues();
+		    for(int i = 0; model.getSize() > 1 && i < selected_items.length; i++) {
+			removeSearchType((SearchType)selected_items[i]);
+		    }
+		}
+		Object selected_object = search_type_combobox.getSelectedItem();
+		if(selected_object != null) {
+		    add_button.setEnabled(!model.contains(selected_object));
+		}
+		else {
+		    add_button.setEnabled(false);
+		}
+		remove_button.setEnabled(false);
+	    }
+	}
+
+	private class MoveListener
+	    implements ActionListener {
+	    private boolean move_up;
+	    public MoveListener(boolean move_up) {
+		this.move_up = move_up;
+	    }
+	    public void actionPerformed(ActionEvent event) {
+		// Retrieve the first selected search type (if any)
+		SearchType search_type = null;
+		if((search_type = (SearchType) current_search_types_list.getSelectedValue()) != null) {
+		    // Move search type
+		    moveSearchType(search_type, move_up);
+		    // Reselect the moved search type
+		    current_search_types_list.setSelectedValue(search_type, true);
+		}
+		// No selection - no movement
+		else {
+		    move_up_button.setEnabled(false);
+		    move_down_button.setEnabled(false);
+		}
+	    }
+	}
+
+	/** Listens for changes in the search types combobox, and enabled add button appropriately. */
+	private class SearchTypesActionDocumentListener
+	    implements ActionListener, DocumentListener {
+	    /** Called whenever a selection action occurs on the combobox.
+	     * @param  event an ActionEvent containing information about the selection event
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		validateAddButton();
+	    }
+
+	    /** Gives notification that an attribute or set of attributes changed.
+	     * @param  event a DocumentEvent containing information about the text changed
+	     */
+	    public void changedUpdate(DocumentEvent event) {
+		validateAddButton();
+	    }
+	    /** Gives notification that there was an insert into the document.
+	     * @param  event a DocumentEvent containing information about the text added
+	     */
+	    public void insertUpdate(DocumentEvent event) {
+		validateAddButton();
+	    }
+
+	    /** Gives notification that a portion of the document has been removed.
+	     * @param  event a DocumentEvent containing information about the text removed
+	     */
+	    public void removeUpdate(DocumentEvent event) {
+		validateAddButton();
+	    }
+
+	    /** Change the enable state of the add button depending on the current value in the search type combobox. */
+	    private void validateAddButton() {
+		Object selected_object = search_type_combobox.getSelectedItem();
+		if(selected_object != null) {
+		    add_button.setEnabled(!model.contains(selected_object));
+		}
+		else {
+		    add_button.setEnabled(false);
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Subcollection.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Subcollection.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/Subcollection.java	(revision 21112)
@@ -0,0 +1,256 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class encapsulates one subcollection entry in the collection configuration file.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class Subcollection
+    implements Comparable, DOMProxyListEntry {
+    /** A <i>boolean</i> which is <i>true</i> if the condition is an include one, <i>false</i> otherwise. */
+    private boolean include = true;
+    /** The DOM Element this Subcollection is based upon. */
+    private Element element = null;
+    /** A String containing a Perl expression which is used as the pattern by which to filter this subcollection. */
+    private String pattern = null;
+    /** A series of flags to be used when matching the expression. */
+    private String flags = null;
+    /** A String which is a unique identifier of a subcollection. */
+    private String name = null;
+    /** Either the fully qualified name of the metadata whose value should be matched against the given expression, or <i>null</i> if you wish to match against the file name. */
+    private String source = null;
+    private String text = null;
+
+    /** Default constructor which should only be used during DOMProxyListModel initialization. */
+    public Subcollection() {
+    }
+
+    /** Constructor for representing an existing assigned Subcollection.
+     * @param element the Element this subcollection is based upon
+     */
+    public Subcollection(Element element) {
+	this.element = element;
+    }
+
+    /** Constructor for assigning a brand new Subcollection.
+     * @param name a unique identifier for this collection as a String
+     * @param include true if this the pattern should be an inclusion filter, false for exclusion
+     * @param source either the fully qualified name of an element as a String, or null to filter filenames
+     * @param pattern the matching pattern as a String
+     * @param flags any flags to use while matching, as a String
+     */
+    public Subcollection(String name, boolean include, String source, String pattern, String flags) {
+	// Cache the details
+	this.flags = flags;
+	this.include = include;
+	this.name = name;
+	this.pattern = pattern;
+	if(source != null) {
+	    this.source = source;
+	}
+	else {
+	    this.source = StaticStrings.FILENAME_STR;
+	}
+	// Create a new DOM Element with the appropriate attributes and text value
+	element = CollectionDesignManager.collect_config.document.createElement(StaticStrings.SUBCOLLECTION_ELEMENT);
+	element.setAttribute(StaticStrings.CONTENT_ATTRIBUTE, source);
+	element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
+	element.setAttribute(StaticStrings.OPTIONS_ATTRIBUTE, flags);
+	element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, (include ? StaticStrings.INCLUDE_STR : StaticStrings.EXCLUDE_STR));
+	XMLTools.setValue(element, pattern);
+    }
+
+    /** Method to compare two subcollections.
+     * @param object he other subcollection to compare to, as an Object
+     * @return an int which is &gt;0, 0, or &lt;0 if this subcollection is before, equal to, or after the target object respectively.
+     */
+    public int compareTo(Object object)
+    {
+	if (object == null) {
+	    return -1;
+	}
+
+	// Object may be a String
+	if (object instanceof String) {
+	    return getName().compareTo((String) object);
+	}
+
+	// Otherwise assume it is a Subcollection object
+	return getName().compareTo(((Subcollection) object).getName());
+    }
+
+
+    public DOMProxyListEntry create(Element element) {
+	return new Subcollection(element);
+    }
+
+    /** Method to check two subcollections for equality.
+     * @param object the other subcollection to compare to, as an Object
+     * @return true if the subcollections are equal, false otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+
+    /** Retrieve the DOM Element this Subcollection is based upon.
+     * @return an Element
+     */
+    public Element getElement() {
+	return element;
+    }
+
+    /** Method to get the value of flags.
+     * @return the value of flags as a String
+     */
+    public String getFlags() {
+	if(flags == null && element != null) {
+	    flags = element.getAttribute(StaticStrings.OPTIONS_ATTRIBUTE);
+	}
+	return flags;
+    }
+
+    /** Method to get the value of name.
+     * @return String The value of name as a <strong>String</string>.
+     */
+    public String getName() {
+	if(name == null && element != null) {
+	    name = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+	}
+	return name;
+    }
+
+    public String getPattern() {
+	if(pattern == null && element != null) {
+	    pattern = XMLTools.getValue(element);
+	}
+	return pattern;
+    }
+
+    /** Method to get the name of the source of the strings used in pattern matching.
+     * @return a String which is either the fully qualified name of a metadata element, or filename
+     */
+    public String getSource()
+    {
+	if (source == null && element != null) {
+	    source = element.getAttribute(StaticStrings.CONTENT_ATTRIBUTE);
+	}
+
+	return source;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    /** Method to get the value of include.
+     * @return boolean true if this is an inclusion filter, false otherwise
+     */
+    public boolean isInclusive() {
+	if (element != null) {
+	    include = element.getAttribute(StaticStrings.TYPE_ATTRIBUTE).equals(StaticStrings.INCLUDE_STR);
+	}
+
+	return include;
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	include = true;
+	flags = null;
+	name = null;
+	pattern = null;
+	source = null;
+	text = null;
+    }
+
+    public void setFlags(String new_flags) {
+	if(element != null) {
+	    // Change element
+	    element.setAttribute(StaticStrings.OPTIONS_ATTRIBUTE, new_flags);
+	    flags = new_flags;
+	    text = null;
+	}
+    }
+
+    public void setInclusive(boolean new_include) {
+	if(element != null) {
+	    // Change element
+	    include = new_include;
+	    element.setAttribute(StaticStrings.TYPE_ATTRIBUTE, (include ? StaticStrings.INCLUDE_STR : StaticStrings.EXCLUDE_STR));
+	    text = null;
+	}
+    }
+
+    public void setName(String new_name) {
+	if(element != null) {
+	    // Change element
+	    element.setAttribute(StaticStrings.NAME_ATTRIBUTE, new_name);
+	    name = new_name;
+	    text = null;
+	}
+    }
+
+    public void setPattern(String new_pattern) {
+	if(element != null) {
+	    // Change element
+	    XMLTools.setValue(element, new_pattern);
+	    pattern = new_pattern;
+	    text = null;
+	}
+    }
+
+    public void setSource(String new_source) {
+	if(element != null) {
+	    // Change element
+	    element.setAttribute(StaticStrings.CONTENT_ATTRIBUTE, new_source);
+	    source = new_source;
+	    text = null;
+	}
+    }
+
+    /** Method to display the contents of this class as it would appear in the collection configuration file.
+     * @return a String representing this subcollection
+     */
+    public String toString() {
+	if(text == null && element != null) {
+	    text = CollectionConfiguration.toString(element, true);
+	}
+	return text;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndex.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndex.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndex.java	(revision 21112)
@@ -0,0 +1,185 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class encapsulates a single indexing pair.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class SubcollectionIndex
+    implements Comparable, DOMProxyListEntry {
+
+    private ArrayList sources = null;
+    /** The element this index is based upon. */
+    private Element element = null;
+    /** The unique, if cryptic, identifier of an index. */
+    private String id = null;
+
+    /** Default constructor, which should only be used during DOMProxyListModel creation. */
+    public SubcollectionIndex() {
+    }
+
+    /** Constructor. */
+    public SubcollectionIndex(Element element) {
+	this.element = element;
+    }
+
+    public SubcollectionIndex(Object[] raw_sources) {
+	this.sources = new ArrayList();
+	// Create a new element
+	Document document = CollectionDesignManager.collect_config.document;
+	element = document.createElement(CollectionConfiguration.INDEX_ELEMENT);
+	// For each source add a content element
+	int size = raw_sources.length;
+	for(int i = 0; i < size; i++) {
+	    Subcollection subcollection = (Subcollection) raw_sources[i];
+	    String subcollection_name = subcollection.getName();
+	    sources.add(subcollection_name);
+	    Element content_element = document.createElement(CollectionConfiguration.CONTENT_ELEMENT);
+	    content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, subcollection_name);
+	    element.appendChild(content_element);
+	    content_element = null;
+	}
+	document = null;
+    }
+
+    /** Method to compare two indexes.
+     * @param object The other index as an <strong>Object</strong>.
+     * @return An <i>int</i> which indicates how the indexes compare.
+     * @see java.lang.String
+     */
+    public int compareTo(Object object) {
+	return getID().compareTo(((SubcollectionIndex)object).getID());
+    }
+    
+    public DOMProxyListEntry create(Element element) {
+	return new SubcollectionIndex(element);
+    }
+
+    /** Method to test for the equality of two indexes.
+     * @param object The other index as an <strong>Object</strong>.
+     * @return A <i>boolean</i> which is <i>true</i> if the two indexes are equal, <i>false</i> otherwise.
+     */
+    public boolean equals(Object object) {
+	return (compareTo(object) == 0);
+    }
+    
+    public Element getElement() {
+	return element;
+    }
+
+    public String getID() {
+	if(id == null) {
+	    StringBuffer id_buffer = new StringBuffer();
+	    // Write data information. Retrieve each of the content sources and add them in a comma separated list.
+	    ArrayList sources = getSources();
+	    int sources_size = sources.size();
+	    for(int i = 0; i < sources_size; i++) {
+		id_buffer.append((String)sources.get(i));
+		id_buffer.append(StaticStrings.COMMA_CHARACTER);
+	    }
+	    sources = null;
+	    id = id_buffer.substring(0, id_buffer.length() - 1);
+	}
+	return id;
+    }
+
+    /** Retrieve the sources of this index.
+     * @return the sources as an ArrayList
+     */
+    public ArrayList getSources() {
+	if(sources == null) {
+	    sources = new ArrayList();
+	    NodeList content_elements = element.getElementsByTagName(CollectionConfiguration.CONTENT_ELEMENT);
+	    int content_elements_length = content_elements.getLength();
+	    for(int i = 0; i < content_elements_length; i++) {
+		Element content_element = (Element) content_elements.item(i);
+		sources.add(content_element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE));
+	    }
+	    content_elements = null;
+	    Collections.sort(sources);
+	}
+	return sources;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && !element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.FALSE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	this.id = null;
+	this.sources = null;
+    }
+
+    /** Method to set the sources for this index which can only be used for the default index.
+     * @param sources an ArrayList of source names
+     */
+    public void setSources(ArrayList sources) {
+	if(element != null) {
+	    // Erase old sources
+	    XMLTools.clear(element);
+	    // For each entry in the sources array add a new content element.
+	    int size = sources.size();
+	    for(int i = 0; i < size; i++) {
+		Element content_element = element.getOwnerDocument().createElement(CollectionConfiguration.CONTENT_ELEMENT);
+		content_element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, (String) sources.get(i));
+		element.appendChild(content_element);
+		content_element = null;
+	    }
+	    this.id = null; // Regenerate ID.
+	    this.sources = sources;
+	}
+    }
+
+    /** Method to turn this object into a string representation ready to be placed in the collection configuration file.
+     * @return A <strong>String</strong> containing the information of this class.
+     */
+    public String toString() {
+	String id = getID();
+	StringBuffer text = new StringBuffer(id);
+	CollectionMeta metadatum = CollectionDesignManager.collectionmeta_manager.getMetadatum(StaticStrings.STOP_CHARACTER + id, false);
+	if(metadatum != null) {
+	    text.append(" \"");
+	    text.append(metadatum.getValue(CollectionMeta.TEXT));
+	    text.append("\"");
+	}
+	return text.toString();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndexManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndexManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionIndexManager.java	(revision 21112)
@@ -0,0 +1,664 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.JarTools;
+import org.w3c.dom.*;
+
+
+/** This class maintains a list of indexes partitions for the purpose of defining subcollections.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class SubcollectionIndexManager
+    extends DOMProxyListModel
+{
+    static final private Dimension FIELD_SIZE = new Dimension(200, 30);
+
+    private Control controls;
+    private DOMProxyListModel model;
+    private SubcollectionIndex default_index;
+
+
+    /** Constructor. */
+    public SubcollectionIndexManager(Element subindexes)
+    {
+	super(subindexes, CollectionConfiguration.INDEX_ELEMENT, new SubcollectionIndex());
+	DebugStream.println("SubcollectionIndexManager: " + getSize() + " subcollection indexes parsed.");
+	model = this;
+
+	// Parse and retrieve the default index
+	NodeList default_index_elements = CollectionDesignManager.collect_config.getDocumentElement().getElementsByTagName(CollectionConfiguration.SUBCOLLECTION_DEFAULT_INDEX_ELEMENT);
+	if (default_index_elements.getLength() > 0) {
+	    default_index = new SubcollectionIndex((Element) default_index_elements.item(0));
+	}
+    }
+
+
+    /** Method to add a subindex.
+     * @param subindex a SubcollectionIndex
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void addSubcollectionIndex(SubcollectionIndex subcollection_index)
+    {
+	if (!contains(subcollection_index)) {
+	    add(getSize(), subcollection_index);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+
+    public void destroy()
+    {
+	if (controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+	default_index = null;
+	model = null;
+    }
+
+
+    public Control getControls()
+    {
+	if (controls == null) {
+	    controls = new SubcollectionIndexControls();
+	}
+	return controls;
+    }
+
+
+    /** Method to get all of the subindexes set.
+     * @return an ArrayList containing all the defined indexes
+     */
+    public ArrayList getSubcollectionIndexes()
+    {
+	return children();
+    }
+
+
+    private int moveSubcollectionIndex(SubcollectionIndex subcollection_index, boolean move_up)
+    {
+	// Determine the current position of the subcollection index
+	int position = indexOf(subcollection_index);
+	int new_position;
+
+	// Attempt to move the subcollection index up
+	if (move_up) {
+	    // Check it's not already at the top
+	    if (position == 0) {
+		return position;
+	    }
+
+	    // This automatically removes the index first, as an Element can only exist once in a particular document
+	    new_position = position - 1;
+	    addBefore(subcollection_index, (SubcollectionIndex) getElementAt(new_position));
+	}
+
+	// Attempt to move the subcollection index down
+	else {
+	    // Check it's not already at the bottom
+	    if (position == (getSize()) - 1) {
+		return position;
+	    }
+
+	    // This automatically removes the index first, as an Element can only exist once in a particular document
+	    new_position = position + 1;
+	    addAfter(subcollection_index, (SubcollectionIndex) getElementAt(new_position));
+	}
+
+	// Schedule the collection for saving
+	Gatherer.c_man.configurationChanged();
+	return new_position;
+    }
+
+
+    /** Method to remove a certain subcollection index. */
+    private void removeSubcollectionIndex(SubcollectionIndex subcollection_index)
+    {
+	if (subcollection_index != null) {
+	    // Remove any current metadata from this index
+	    CollectionDesignManager.collectionmeta_manager.removeMetadata("." + subcollection_index.getID());
+
+	    // Check if the index removed happens to be the default index
+	    if (default_index != null && default_index.equals(subcollection_index)) {
+		// If so our first solution is to set the first subcollection index to be the default
+		if (getSize() > 0) {
+		    setDefaultSubcollectionIndex((SubcollectionIndex) getElementAt(0));
+		}
+		else {
+		    default_index.setAssigned(false);
+		}
+	    }
+
+	    // Remove the index
+	    remove(subcollection_index);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+
+    /** Method to remove all of the subindexes that contain a certain subcollection.
+     * @param subcollection the Subcollection that has been removed
+     * @see org.greenstone.gatherer.cdm.Subcollection
+     * @see org.greenstone.gatherer.cdm.SubcollectionIndex
+     */
+    public void removeSubcollectionIndexes(Subcollection subcollection)
+    {
+	String subcollection_name = subcollection.getName();
+	int size = getSize();
+	for(int i = size - 1; i >= 0; i--) {
+	    SubcollectionIndex subcollection_index = (SubcollectionIndex) getElementAt(i);
+	    if (subcollection_index.getSources().contains(subcollection_name)) {
+		removeSubcollectionIndex(subcollection_index);
+	    }
+	}
+    }
+
+
+    /** Method to set the default subcollection index.
+     * @param index The <strong>SubcollectionIndex</strong> to use as the default index.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.cdm.SubcollectionIndex
+     */
+    private void setDefaultSubcollectionIndex(SubcollectionIndex subcollection_index)
+    {
+ 	if (subcollection_index != null) {
+ 	    if (default_index == null) {
+		// Create the default index element, and place immediately after indexes element.
+		Element default_index_element = root.getOwnerDocument().createElement(CollectionConfiguration.SUBCOLLECTION_DEFAULT_INDEX_ELEMENT);
+		default_index = new SubcollectionIndex(default_index_element);
+		Node target_node = CollectionConfiguration.findInsertionPoint(default_index_element);
+		if (target_node != null) {
+		    root.getOwnerDocument().getDocumentElement().insertBefore(default_index_element, target_node);
+		}
+		else {
+		    root.getOwnerDocument().getDocumentElement().appendChild(default_index_element);
+		}
+ 	    }
+ 	    default_index.setAssigned(true);
+ 	    default_index.setSources(subcollection_index.getSources());
+ 	}
+ 	else {
+ 	    if (default_index != null) {
+ 		default_index.setAssigned(false);
+ 	    }
+ 	}
+ 	Gatherer.c_man.configurationChanged();
+    }
+
+
+    /** This class creates a set of controls for editing the indexes. */
+    private class SubcollectionIndexControls
+	extends JPanel
+	implements Control
+    {
+	private CheckList source_list;
+	private JButton add_button;
+	private JButton move_down_button;
+	private JButton move_up_button;
+	private JButton remove_button;
+	private JButton replace_button;
+	private JButton set_default_button;
+	private JList subcollection_index_list;
+	private JTextField subcollection_index_name_textfield;
+
+
+	/** Constructor.
+	 */
+	public SubcollectionIndexControls()
+	{
+	    super();
+
+	    ArrayList sources = new ArrayList();
+	    ListModel source_model = CollectionDesignManager.subcollection_manager;
+	    for (int i = 0; i < source_model.getSize(); i++) {
+		sources.add(source_model.getElementAt(i));
+	    }
+
+	    // Creation
+	    JPanel assigned_indexes_pane = new JPanel();
+	    JLabel index_label = new JLabel();
+	    Dictionary.registerText(index_label, "CDM.SubcollectionIndexManager.Subindexes");
+ 	    subcollection_index_list = new JList(model);
+	    subcollection_index_list.setCellRenderer(new SubcollectionIndexListCellRenderer());
+ 	    subcollection_index_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    subcollection_index_list.setVisibleRowCount(2);
+
+	    JPanel movement_pane = new JPanel();
+	    move_up_button = new JButton("", JarTools.getImage("arrow-up.gif"));
+	    move_up_button.setEnabled(false);
+	    move_up_button.setMnemonic(KeyEvent.VK_U);
+	    Dictionary.registerBoth(move_up_button, "CDM.Move.Move_Up", "CDM.Move.Move_Up_Tooltip");
+
+	    move_down_button = new JButton("", JarTools.getImage("arrow-down.gif"));
+	    move_down_button.setEnabled(false);
+	    move_down_button.setMnemonic(KeyEvent.VK_D);
+	    Dictionary.registerBoth(move_down_button, "CDM.Move.Move_Down", "CDM.Move.Move_Down_Tooltip");
+
+	    set_default_button = new GLIButton();
+	    set_default_button.setEnabled(false);
+	    set_default_button.setMnemonic(KeyEvent.VK_S);
+	    Dictionary.registerBoth(set_default_button, "CDM.SubcollectionIndexManager.Set_Default_Subindex", "CDM.SubcollectionIndexManager.Set_Default_Subindex_Tooltip");
+
+	    JPanel index_pane = new JPanel();
+	    JPanel details_pane = new JPanel();
+	    JPanel labels_pane = new JPanel();
+	    JPanel boxes_pane = new JPanel();
+	    JPanel content_pane = new JPanel();
+
+	    JLabel name_label = new JLabel();
+	    Dictionary.registerText(name_label, "CDM.SubcollectionIndexManager.PartitionName");
+	    subcollection_index_name_textfield = new JTextField();
+	    subcollection_index_name_textfield.setPreferredSize(FIELD_SIZE);
+	    Dictionary.registerTooltip(subcollection_index_name_textfield, "CDM.SubcollectionIndexManager.PartitionName_Tooltip");
+
+	    JLabel source_label = new JLabel();
+	    Dictionary.registerText(source_label, "CDM.SubcollectionIndexManager.Source");
+	    source_list = new CheckList(false);
+	    source_list.setListData(sources);
+	    Dictionary.registerTooltip(source_list, "CDM.SubcollectionIndexManager.Source_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(false);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.SubcollectionIndexManager.Add_Subindex", "CDM.SubcollectionIndexManager.Add_Subindex_Tooltip");
+
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.SubcollectionIndexManager.Remove_Subindex", "CDM.SubcollectionIndexManager.Remove_Subindex_Tooltip");
+
+	    replace_button = new GLIButton();
+	    replace_button.setEnabled(false);
+	    replace_button.setMnemonic(KeyEvent.VK_P);
+	    Dictionary.registerBoth(replace_button, "CDM.SubcollectionIndexManager.Replace_Subindex", "CDM.SubcollectionIndexManager.Replace_Subindex_Tooltip");
+
+	    // Listeners
+	    add_button.addActionListener(new AddListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_down_button.addActionListener(new MoveListener(false));
+	    move_down_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    move_up_button.addActionListener(new MoveListener(true));
+	    move_up_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    replace_button.addActionListener(new ReplaceListener());
+	    replace_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    set_default_button.addActionListener(new SetDefaultListener());
+	    set_default_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    subcollection_index_name_textfield.getDocument().addDocumentListener(new NameListener());
+	    subcollection_index_list.addListSelectionListener(new SubcollectionIndexListListener());
+	    source_list.addListSelectionListener(new SourceListListener());
+
+	    // Layout
+	    movement_pane.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+	    movement_pane.setLayout(new GridLayout(3,1));
+	    movement_pane.add(move_up_button);
+	    movement_pane.add(move_down_button);
+	    movement_pane.add(set_default_button);
+
+	    assigned_indexes_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    assigned_indexes_pane.setLayout(new BorderLayout());
+	    assigned_indexes_pane.add(index_label, BorderLayout.NORTH);
+	    assigned_indexes_pane.add(new JScrollPane(subcollection_index_list), BorderLayout.CENTER);
+	    assigned_indexes_pane.add(movement_pane, BorderLayout.EAST);
+
+	    labels_pane.setLayout(new BorderLayout());
+	    labels_pane.setBorder(BorderFactory.createEmptyBorder(5, 5, 10, 5));
+	    labels_pane.add(name_label, BorderLayout.NORTH);
+	    labels_pane.add(source_label, BorderLayout.CENTER);
+	    
+	    boxes_pane.setLayout(new BorderLayout());
+	    boxes_pane.add(subcollection_index_name_textfield, BorderLayout.NORTH);
+	    boxes_pane.add(new JScrollPane(source_list), BorderLayout.CENTER);
+	    
+	    details_pane.setLayout(new BorderLayout());
+	    details_pane.add(labels_pane, BorderLayout.WEST);
+	    details_pane.add(boxes_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,3));
+	    button_pane.add(add_button);
+	    button_pane.add(replace_button);
+	    button_pane.add(remove_button);
+
+	    index_pane.setLayout(new BorderLayout());
+	    index_pane.add(details_pane, BorderLayout.CENTER);
+	    index_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(assigned_indexes_pane, BorderLayout.NORTH);
+	    content_pane.add(index_pane, BorderLayout.CENTER);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(content_pane, BorderLayout.CENTER);
+	}
+
+
+	public void destroy()
+	{
+	}
+
+
+	public void gainFocus()
+	{
+ 	    // Reload the source list
+ 	    ArrayList sources = new ArrayList();
+ 	    ListModel source_model = CollectionDesignManager.subcollection_manager;
+ 	    for (int i = 0; i < source_model.getSize(); i++) {
+ 		sources.add(source_model.getElementAt(i));
+ 	    }
+ 	    source_list.setListData(sources);
+
+ 	    // Refresh the subcollection index list
+ 	    subcollection_index_list.updateUI();
+
+ 	    // If there is one selected, fill in the controls
+ 	    updateControlsWithSelectedIndex();
+	}
+
+
+	public void loseFocus()
+	{
+	}
+
+
+	private void updateControlsWithSelectedIndex()
+	{
+	    SubcollectionIndex selected_index = (SubcollectionIndex) subcollection_index_list.getSelectedValue();
+	    if (selected_index == null) {
+		return;
+	    }
+
+	    // Display the selected subcollection index's name
+	    String id = selected_index.getID();
+	    String name = CollectionDesignManager.collectionmeta_manager.getMetadatum("." + id).getValue(true);
+	    subcollection_index_name_textfield.setText(name);
+
+	    // Display the selected subcollection index's sources
+	    source_list.clearTicked();
+	    source_list.setTickedObjects(selected_index.getSources().toArray());
+	}
+
+
+	private void validateControls()
+	{
+	    boolean add_enabled = false;
+	    boolean replace_enabled = false;
+
+	    // Indexes must have a name
+	    if (subcollection_index_name_textfield.getText().length() == 0) {
+		add_enabled = false;
+	    }
+	    // Can't add a new index if no sources are selected
+	    else if (source_list.isNothingTicked()) {
+		add_enabled = false;
+	    }
+	    // If we get this far, create a dummy index and see if it's already assigned in the collection
+	    else {
+ 		ArrayList sources = source_list.getTicked();
+ 		SubcollectionIndex subcollection_index = new SubcollectionIndex(sources.toArray());
+
+		// Subcollection index already exists: can't add, but can replace if the name has changed
+ 		if (model.contains(subcollection_index)) {
+ 		    add_enabled = false;
+
+ 		    // Here we need to check if we have changed the name - if so, we can enable the replace button
+		    if (subcollection_index_list.getSelectedIndex() != -1) {
+			String name = subcollection_index_name_textfield.getText();
+ 			SubcollectionIndex selected_index = (SubcollectionIndex) subcollection_index_list.getSelectedValue();
+ 			String selected_name = CollectionDesignManager.collectionmeta_manager.getMetadatum("." + selected_index.getID()).getValue(true);
+ 			if (!name.equals(selected_name)) {
+ 			    replace_enabled = true;
+ 			}
+ 		    }
+		}
+
+		// Subcollection index doesn't already exist: can add, or replace if there is something selected
+		else {
+ 		    add_enabled = true;
+ 		    if (subcollection_index_list.getSelectedIndex() != -1) {
+ 			replace_enabled = true;
+ 		    }
+ 		}
+	    }
+
+	    // We should now know the add_button state
+	    add_button.setEnabled(add_enabled);
+	    replace_button.setEnabled(replace_enabled);
+	}
+
+
+	private class AddListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		String name = subcollection_index_name_textfield.getText();
+		if (!source_list.isNothingTicked() && name.length() != 0) {
+		    ArrayList sources = source_list.getTicked();
+		    SubcollectionIndex subcollection_index = new SubcollectionIndex(sources.toArray());
+
+		    // Before we add the index to the model, we have to add the collection metadata for this
+		    CollectionMeta metadatum = new CollectionMeta("." + subcollection_index.getID());
+		    metadatum.setValue(name);
+ 		    CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+
+		    // Finally, add the subcollection index
+		    addSubcollectionIndex(subcollection_index);
+		    subcollection_index_list.setSelectedIndex(model.getSize() - 1);
+		}
+	    }
+	}
+
+
+	private class MoveListener
+	    implements ActionListener
+	{
+	    private boolean move_up;
+
+	    public MoveListener(boolean move_up)
+	    {
+		this.move_up = move_up;
+	    }
+
+	    public void actionPerformed(ActionEvent event)
+	    {
+		// Retrieve the selected subcollection index
+		SubcollectionIndex subcollection_index = (SubcollectionIndex) subcollection_index_list.getSelectedValue();
+		if (subcollection_index != null) {
+		    int new_position = moveSubcollectionIndex(subcollection_index, move_up);
+		    // Ensure the subcollection index that moved is still selected
+		    subcollection_index_list.setSelectedIndex(new_position);
+		}
+	    }
+	}
+
+
+	/** Listens for key presses within the name field, and enabled or disables controls as appropriate. */
+	private class NameListener
+	    implements DocumentListener
+	{
+	    public void changedUpdate(DocumentEvent e)
+	    {
+		validateControls();
+	    }
+
+	    public void insertUpdate(DocumentEvent e)
+	    {
+		validateControls();
+	    }
+
+	    public void removeUpdate(DocumentEvent e)
+	    {
+		validateControls();
+	    }
+	}
+
+
+	private class RemoveListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		int i = subcollection_index_list.getSelectedIndex();
+		if (i != -1) {
+		    removeSubcollectionIndex((SubcollectionIndex) subcollection_index_list.getSelectedValue());
+		}
+
+		int size = subcollection_index_list.getModel().getSize();
+		if (i == size) {
+		    i--;
+		}
+		subcollection_index_list.setSelectedIndex(i);
+		validateControls();
+	    }
+	}
+
+
+	private class ReplaceListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		if (subcollection_index_list.isSelectionEmpty()) {
+		    // This should never happen, but just in case...
+		    replace_button.setEnabled(false);
+		    return;
+		}
+
+		// We'll just remove the old one and add the new one
+		removeSubcollectionIndex((SubcollectionIndex) subcollection_index_list.getSelectedValue());
+		replace_button.setEnabled(false);
+		add_button.setEnabled(true);
+		add_button.doClick();
+		add_button.setEnabled(false);
+	    }
+	}
+
+
+	private class SetDefaultListener
+	    implements ActionListener
+	{
+	    public void actionPerformed(ActionEvent event)
+	    {
+		SubcollectionIndex subcollection_index = (SubcollectionIndex) subcollection_index_list.getSelectedValue();
+		if (subcollection_index != null) {
+		    setDefaultSubcollectionIndex(subcollection_index);
+		    // This should cause a repaint of just the desired row
+		    subcollection_index_list.setSelectedValue(subcollection_index, true);
+		}
+		set_default_button.setEnabled(false);
+	    }
+	}
+
+
+	private class SourceListListener
+	    implements ListSelectionListener
+	{
+	    public void valueChanged(ListSelectionEvent event)
+	    {
+		validateControls();
+	    }
+	}
+
+
+	/** Listens for selections within the list on the SubcollectionIndexManager controls, and if a change is detected enables, or disables, controls appropriately. */
+	private class SubcollectionIndexListListener
+	    implements ListSelectionListener
+	{
+	    /** This method is called whenever the source list selection changes. When it does we need to fill in the various parts of the list description panel
+	     * @param event A <strong>ListSelectionEvent</strong> containing further information about the list selection.
+	     */
+	    public void valueChanged(ListSelectionEvent event)
+	    {
+		if (event.getValueIsAdjusting()) {
+		    return;
+		}
+
+ 		int i = subcollection_index_list.getSelectedIndex();
+ 		int size = subcollection_index_list.getModel().getSize();
+ 		if (i < 0 || i >= size) {
+ 		    move_down_button.setEnabled(false);
+ 		    move_up_button.setEnabled(false);
+ 		    remove_button.setEnabled(false);
+ 		    replace_button.setEnabled(false);
+ 		    set_default_button.setEnabled(false);
+ 		    return;
+ 		}
+
+ 		// Enable the buttons appropriately
+ 		remove_button.setEnabled(true);
+ 		SubcollectionIndex selected_index = (SubcollectionIndex) subcollection_index_list.getSelectedValue();
+ 		set_default_button.setEnabled(default_index == null || !default_index.equals(selected_index));
+ 		if (i > 0) {
+ 		    move_up_button.setEnabled(true);
+ 		}
+ 		else {
+ 		    move_up_button.setEnabled(false);
+ 		}
+ 		if (i < size-1){
+ 		    move_down_button.setEnabled(true);
+ 		}
+ 		else {
+ 		    move_down_button.setEnabled(false);
+ 		}
+
+ 		// Need to fill in the rest of the bits
+		updateControlsWithSelectedIndex();
+	    }
+	}
+
+
+	private class SubcollectionIndexListCellRenderer
+	    extends DefaultListCellRenderer
+	{
+	    /** Return a component that has been configured to display the specified value. */
+	    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+		JLabel component = (JLabel) super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+		if (default_index != null && default_index.equals(value)) {
+		    component.setText(component.getText() + " " + Dictionary.get("CDM.SubcollectionIndexManager.Default_Partition_Indicator"));
+		}
+		return component;
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SubcollectionManager.java	(revision 21112)
@@ -0,0 +1,575 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.NonWhitespaceField;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+/** This class maintains a list of subcollections within our collection.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class SubcollectionManager
+    extends DOMProxyListModel {
+
+    static final private String DISABLED_CONTROLS = "Disabled";
+    static final private String ENABLED_CONTROLS  = "Normal";
+    static final private String CLASS_DICTIONARY_NAME = "CDM.SubcollectionManager.";
+
+    /** The controls used to edit the settings of this manager. */
+    private Control controls = null;
+
+    private DOMProxyListModel model;
+
+    /** Constructor. 
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
+     * @see org.greenstone.gatherer.cdm.DOMProxyListModel
+     * @see org.greenstone.gatherer.cdm.Subcollection
+     */
+    public SubcollectionManager() {
+	super(CollectionDesignManager.collect_config.getDocumentElement(), StaticStrings.SUBCOLLECTION_ELEMENT, new Subcollection());
+	DebugStream.println("SubcollectionManager: " + getSize() + " subcollections parsed.");
+	this.model = this;
+    }
+
+    /** Method to add a new subcollection.
+     * @param subcollection the Subcollection to add
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.cdm.CollectionConfiguration
+     * @see org.greenstone.gatherer.cdm.DOMProxyListModel
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void addSubcollection(Subcollection subcollection) {
+	if(!contains(subcollection)) {
+	    Element element = subcollection.getElement();
+	    // Locate where we should insert this new subcollection. 
+	    Node target_node = CollectionConfiguration.findInsertionPoint(element);
+	    // Failing that we insert immediately after a language string
+	    add(root, subcollection, target_node);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+    }
+
+    /** Method to retrieve the controls for this manager.
+     * @return the Control used to edit the subcollection data
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    controls = new SubcollectionControl();
+	}
+	return controls;
+    }
+
+    /** Method to retrieve a certain subcollection by its name.
+     * @param name a String which is used as the key for finding the matching subcollection
+     * @return the requested Subcollection or null if no such subcollection exists.
+     */
+    private Subcollection getSubcollection(String name) {
+	Subcollection result = null;
+	int size = getSize();
+	for(int i = 0; i < size; i++) {
+	    Subcollection subcollection = (Subcollection) getElementAt(i);
+	    if(subcollection.getName().equals(name)) {
+		result = subcollection;
+	    }
+	}
+	return result;
+    }
+
+
+    /** Called when the detail mode has changed which in turn may cause several design elements to be available/hidden
+     * @param mode the new mode as an int
+     */
+    public void modeChanged(int mode) {
+	if(controls != null) {
+	    ((SubcollectionControl)controls).modeChanged(mode);
+	}
+    }    
+
+    /** Method to remove the given subcollection.
+     * @param subcollection the Subcollection you want to remove
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     */
+    private void removeSubcollection(Subcollection subcollection) {
+	remove(subcollection);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    private void updateSubcollection(Subcollection subcollection, String name, boolean include, String source, String pattern, String flags) {
+	subcollection.setFlags(flags);
+	subcollection.setInclusive(include);
+	subcollection.setName(name);
+	subcollection.setPattern(pattern);
+	subcollection.setSource(source);
+	refresh(subcollection);
+	Gatherer.c_man.configurationChanged();
+    }
+
+    /** This class creates a JPanel containing serveral more controls used for editing subcollection information. */
+    private class SubcollectionControl
+	extends JPanel 
+	implements Control, ChangeListener
+    {
+	private CardLayout card_layout;
+	private JButton add_button;
+	private JButton remove_button;
+	private JButton update_button;
+	private JComboBox source_combobox;
+	private JList subcollection_list;
+	private JPanel border_pane;
+	private JTabbedPane tabbed_pane;
+	private JTextArea instructions_area;
+	private JTextField flags_field;
+	private JTextField match_field;
+	private JTextField name_field;
+	private JRadioButton exclude_button;
+	private JRadioButton include_button;
+
+	/** Constructor */
+	public SubcollectionControl() {
+	    // Create
+	    JPanel header_pane = new JPanel();
+	    instructions_area = new JTextArea();
+	    instructions_area.setEditable(false);
+	    instructions_area.setLineWrap(true);
+	    instructions_area.setRows(6);
+	    instructions_area.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions_area, "CDM.SubcollectionManager.Instructions");
+
+	    border_pane = new JPanel();
+	    card_layout = new CardLayout();
+
+	    tabbed_pane = new JTabbedPane();
+	    tabbed_pane.addChangeListener(this);
+	    JLabel title = new JLabel();
+	    title.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title, "CDM.SubcollectionManager.Title");
+
+	    JPanel button_pane_3 = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    add_button.setEnabled(false);
+	    Dictionary.registerBoth(add_button, "CDM.SubcollectionManager.Add", "CDM.SubcollectionManager.Add_Tooltip");
+	    remove_button = new GLIButton();
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    remove_button.setEnabled(false);
+	    Dictionary.registerBoth(remove_button, "CDM.SubcollectionManager.Remove", "CDM.SubcollectionManager.Remove_Tooltip");
+	    update_button = new GLIButton();
+	    update_button.setMnemonic(KeyEvent.VK_C);
+	    update_button.setEnabled(false);
+	    Dictionary.registerBoth(update_button, "CDM.SubcollectionManager.Replace", "CDM.SubcollectionManager.Replace_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    JPanel button_pane_1 = new JPanel();
+	    include_button = new JRadioButton();
+	    include_button.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    include_button.setMnemonic(KeyEvent.VK_I);
+	    include_button.setOpaque(false);
+	    Dictionary.registerText(include_button, "CDM.SubcollectionManager.Include");
+	    exclude_button = new JRadioButton();
+	    exclude_button.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    exclude_button.setMnemonic(KeyEvent.VK_X);
+	    exclude_button.setOpaque(false);
+	    Dictionary.registerText(exclude_button, "CDM.SubcollectionManager.Exclude");
+
+	    JLabel flags_label = new JLabel();
+	    Dictionary.registerText(flags_label, "CDM.SubcollectionManager.Flags");
+	    flags_field = new NonWhitespaceField();
+	    Dictionary.registerTooltip(flags_field, "CDM.SubcollectionManager.Flags_Tooltip");
+
+	    JPanel inclusive_pane = new JPanel();
+	    JLabel inclusive_label = new JLabel();
+	    Dictionary.registerText(inclusive_label, "CDM.SubcollectionManager.Inclusive");
+
+	    JLabel match_label = new JLabel();
+	    Dictionary.registerText(match_label, "CDM.SubcollectionManager.Match");
+	    match_field = new JTextField();
+	    Dictionary.registerTooltip(match_field, "CDM.SubcollectionManager.Match_Tooltip");
+
+	    JLabel name_label = new JLabel();
+	    Dictionary.registerText(name_label, "CDM.SubcollectionManager.Name");
+	    name_field = new NonWhitespaceField();
+	    Dictionary.registerTooltip(name_field, "CDM.SubcollectionManager.Name_Tooltip");
+
+	    JLabel source_label = new JLabel();
+	    Dictionary.registerText(source_label, "CDM.SubcollectionManager.Source");
+	    ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+	    Vector source_model = new Vector(every_metadata_set_element);
+	    source_model.add(0, StaticStrings.FILENAME_STR);
+	    source_combobox = new JComboBox(source_model);
+	    Dictionary.registerTooltip(source_combobox, "CDM.SubcollectionManager.Source_Tooltip");
+
+	    subcollection_list = new JList(model);
+	    subcollection_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    JPanel subcollection_pane = new JPanel();
+	    ButtonGroup bg = new ButtonGroup();
+	    bg.add(include_button);
+	    bg.add(exclude_button);
+	    include_button.setSelected(true);
+	    JPanel subcollection_list_pane = new JPanel();
+	    JLabel subcollection_list_label = new JLabel();
+	    Dictionary.registerText(subcollection_list_label, "CDM.SubcollectionManager.Assigned");
+
+	    // Create a message pane which explains why these controls are not currently active
+	    JPanel message_pane = new JPanel();
+	    String args[] = new String[3];
+	    args[0] = Configuration.getModeAsString();
+	    args[1] = Dictionary.get("Preferences.Mode.Systems");
+	    args[2] = Dictionary.get("Preferences.Mode.Expert");
+	    JTextArea message_textarea = new JTextArea();
+	    Dictionary.registerText(message_textarea, "CDM.SubcollectionManager.Partitions_Disabled", args);
+	    message_textarea.setEditable(false);
+	    message_textarea.setHighlighter(null); // Prevent highlighting
+	    message_textarea.setLineWrap(true);
+	    message_textarea.setOpaque(false); // Make it transparent
+	    message_textarea.setWrapStyleWord(true);
+
+	    // Add listeners
+	    SubCollectionChangeListener cl = new SubCollectionChangeListener();
+	    add_button.addActionListener(new AddSubCollectionListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveSubCollectionListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    update_button.addActionListener(new UpdateSubCollectionListener());
+	    update_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    exclude_button.addActionListener(cl);
+	    include_button.addActionListener(cl);
+	    source_combobox.addActionListener(cl);
+	    flags_field.getDocument().addDocumentListener(cl);
+	    match_field.getDocument().addDocumentListener(cl);
+	    name_field.getDocument().addDocumentListener(cl);
+	    subcollection_list.addListSelectionListener(new SubCollectionListListener());
+
+	    // Layout
+	    instructions_area.setBorder(BorderFactory.createEmptyBorder(2,5,2,5));
+
+	    header_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    header_pane.setLayout(new BorderLayout());
+	    header_pane.add(title, BorderLayout.NORTH);
+	    header_pane.add(new JScrollPane(instructions_area), BorderLayout.CENTER);
+
+	    inclusive_pane.setLayout(new GridLayout());
+	    inclusive_pane.add(include_button);
+	    inclusive_pane.add(exclude_button);
+
+	    button_pane_1.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    button_pane_1.setLayout(new GridLayout(5, 2));
+	    button_pane_1.add(name_label);
+	    button_pane_1.add(name_field);
+	    button_pane_1.add(source_label);
+	    button_pane_1.add(source_combobox);
+	    button_pane_1.add(match_label);
+	    button_pane_1.add(match_field);
+	    button_pane_1.add(inclusive_label);
+	    button_pane_1.add(inclusive_pane);
+	    button_pane_1.add(flags_label);
+	    button_pane_1.add(flags_field);
+
+	    button_pane_3.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    button_pane_3.setLayout(new GridLayout(1,3));
+	    button_pane_3.add(add_button);
+	    button_pane_3.add(update_button);
+	    button_pane_3.add(remove_button);
+
+	    button_pane.setLayout(new BorderLayout());
+	    button_pane.add(button_pane_1, BorderLayout.CENTER);
+	    button_pane.add(button_pane_3, BorderLayout.SOUTH);
+
+	    subcollection_list_pane.setLayout(new BorderLayout());
+	    subcollection_list_pane.add(subcollection_list_label, BorderLayout.NORTH);
+	    subcollection_list_pane.add(new JScrollPane(subcollection_list), BorderLayout.CENTER);
+	    subcollection_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    subcollection_pane.setLayout(new BorderLayout());
+	    subcollection_pane.add(subcollection_list_pane, BorderLayout.CENTER);
+	    subcollection_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    tabbed_pane.addTab(Dictionary.get("CDM.SubcollectionManager.Subcollection_Controls"), subcollection_pane);
+	    tabbed_pane.addTab(Dictionary.get("CDM.SubcollectionManager.Subindex_Controls"), (JPanel) CollectionDesignManager.subcollectionindex_manager.getControls());
+	    tabbed_pane.addTab(Dictionary.get("CDM.SubcollectionManager.Language_Controls"), (JPanel) CollectionDesignManager.language_manager.getControls());
+
+	    message_pane.setBorder(BorderFactory.createEmptyBorder(0,5,5,5));
+	    message_pane.setLayout(new GridLayout(3,1,0,0));
+	    message_pane.add(new JPanel());
+	    message_pane.add(message_textarea);
+	    message_pane.add(new JPanel());
+
+	    border_pane.setBorder(BorderFactory.createEmptyBorder(0,5,5,5));
+	    border_pane.setLayout(card_layout);
+	    if(Configuration.getMode() > Configuration.LIBRARIAN_MODE) {
+		border_pane.add(tabbed_pane, ENABLED_CONTROLS);
+		border_pane.add(message_pane, DISABLED_CONTROLS);
+	    }
+	    else {
+		border_pane.add(message_pane, DISABLED_CONTROLS);
+		border_pane.add(tabbed_pane, ENABLED_CONTROLS);
+	    }
+
+	    setLayout(new BorderLayout());
+	    add(header_pane, BorderLayout.NORTH);
+	    add(border_pane, BorderLayout.CENTER);
+	}
+       	
+	/** Method to unregister any listeners to avoid memory leaks.
+	 */
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	    // Rebuild the sources combobox
+	    ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+	    Vector source_model = new Vector(every_metadata_set_element);
+	    source_model.add(0, "Filename"); // Add filename as a possible source.
+	    source_combobox.setModel(new DefaultComboBoxModel(source_model));
+	}
+
+	public void loseFocus() {
+	}
+	
+	/** Called when the detail mode has changed which in turn controls if any of the partition controls are visible, or if they are instead replaced with a message explaining why they are not.
+	 * @param mode the new mode as an int
+	 */
+	public void modeChanged(int mode) {
+	    if(mode > Configuration.LIBRARIAN_MODE) {
+		card_layout.show(border_pane, ENABLED_CONTROLS);
+	    }
+	    else {
+		card_layout.show(border_pane, DISABLED_CONTROLS);
+	    }
+	}
+
+	public void stateChanged(ChangeEvent event)
+	{
+	    if (tabbed_pane.getSelectedIndex() == 1) {
+		CollectionDesignManager.subcollectionindex_manager.getControls().gainFocus();
+	    }
+	    if (tabbed_pane.getSelectedIndex() == 2) {
+		CollectionDesignManager.language_manager.getControls().gainFocus();
+	    }
+	}
+
+	/** Listens for actions apon the 'add' button in the SubcollectionManager controls, and if detected calls the addSubcollection method of the manager with a newly created subcollection. */
+	private class AddSubCollectionListener
+	    implements ActionListener {
+	    /** Any implementation of ActionListener must include this method so we can be informed when an action has been performed on one of our target controls. In this case we wish to retrieve information from the various edit controls, and if we have sufficient data to build a new subcollection do so.
+	     * @param event An <strong>ActionEvent</strong> containing information about the event.
+	     * @see org.greenstone.gatherer.cdm.Subcollection
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		String name = name_field.getText();
+		String source = null;
+		Object object = source_combobox.getSelectedItem();
+		if (object instanceof MetadataElement) {
+		    MetadataElement metadata_element = (MetadataElement) object;
+		    source = metadata_element.getFullName();
+		}
+		else {
+		    source = object.toString();
+		}
+		String pattern = match_field.getText();
+		String flags = flags_field.getText();
+		if(name.length() > 0 && (source == null || source.length() > 0) && pattern.length() > 0) {
+		    Subcollection subcollection = new Subcollection(name, include_button.isSelected(), source, pattern, flags);
+		    addSubcollection(subcollection);
+		    // Change the selection to the new subcollection
+		    subcollection_list.setSelectedValue(subcollection, true);
+		}
+		add_button.setEnabled(false);
+	    }
+	}
+
+	/** This class listens for any key entry in a text field, selection change in a combobox or button click, and updates a subcollection as appropriate. Its also convenient to use this class to test if the add button should be active yet. */
+	private class SubCollectionChangeListener
+	    implements DocumentListener, ActionListener {
+	    /** Any implementation of ActionListener must include this method so we can be informed when an action has been performed on one of our target controls. In this case we want to record that somethings changed, then validate the controls.
+	     * @param event An <strong>ActionEvent</strong> containing information about the event.
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		validateAdd();
+	    }
+
+	    public void changedUpdate(DocumentEvent event) {
+		validateAdd();
+	    }
+
+	    public void insertUpdate(DocumentEvent event) {
+		validateAdd();
+
+	    }
+
+	    public void removeUpdate(DocumentEvent event) {
+		validateAdd();
+	    }
+
+	    /** Method to validate the current subcollection editor values, and enable or disable controls (add button) based on said values. */
+	    private void validateAdd() {
+		if(name_field.getText().length() > 0 && match_field.getText().length() > 0) {
+		    if (getSubcollection(name_field.getText()) == null) {
+			add_button.setEnabled(true);
+		    } else {
+			add_button.setEnabled(false);
+		    }
+		}
+		else {
+		    add_button.setEnabled(false);
+		}
+	    }
+	}
+
+	/** Listens for actions apon the 'remove' button in the SubcollectionManager controls, and if detected calls the remove method of the manager with the SubIndex selected for removal. */
+	private class RemoveSubCollectionListener
+	    implements ActionListener {
+	    /** Any implementation of ActionListener must include this method so we can be informed when an action has been performed on one of our target controls. In this case we want to check if they have a subcolleciton selected, and if so remove both it and any subindexes based on it.
+	     * @param event An <strong>ActionEvent</strong> containing information about the event.
+	     * @see org.greenstone.gatherer.cdm.Subcollection
+	     */
+	    public void actionPerformed(ActionEvent event) {
+		if(!subcollection_list.isSelectionEmpty()) {
+		    Subcollection subcollection = (Subcollection)subcollection_list.getSelectedValue();
+		    removeSubcollection(subcollection);
+		    // And remove subcollection indexes dependant on this subcollection
+		    CollectionDesignManager.subcollectionindex_manager.removeSubcollectionIndexes(subcollection);
+		}
+		remove_button.setEnabled(false);
+	    }
+	}
+
+	private class UpdateSubCollectionListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		if(!subcollection_list.isSelectionEmpty()) {
+		    Subcollection subcollection = (Subcollection)subcollection_list.getSelectedValue();
+		    String name = name_field.getText();
+		    String source = null;
+		    Object object = source_combobox.getSelectedItem();
+		    if (object instanceof MetadataElement) {
+			MetadataElement metadata_element = (MetadataElement) object;
+			source = metadata_element.getFullName();
+		    }
+		    else {
+			source = object.toString();
+		    }
+		    String pattern = match_field.getText();
+		    String flags = flags_field.getText();
+		    if(name.length() > 0 && (source == null || source.length() > 0) && pattern.length() > 0) {
+			updateSubcollection(subcollection, name, include_button.isSelected(), source, pattern, flags);
+		    }
+		}
+	    }
+	}
+
+	/** This class listens for selections in the list on the subcollections pane of the SubcollectionManager, and updates the controls as necessary to reflect selection. */
+	private class SubCollectionListListener
+	    implements ListSelectionListener {
+	    /** Any implementation of ListSelectionListener must include this method so we can be informed when the selection changes. In this case we want to execute any changes the users made to the entry, then update the controls with details of the new selection.
+	     * @param event A <strong>ListSelectionEvent</strong> containing information related to this event.
+	     * @see org.greenstone.gatherer.cdm.Subcollection
+	     */
+	    public void valueChanged(ListSelectionEvent event) {
+		// Wait until the event stabilises to avoid processing it multiple times
+		if (event.getValueIsAdjusting() == true) {
+		    return;
+		}
+		// Now the entry
+		if(!subcollection_list.isSelectionEmpty()) {
+		    Subcollection subcollection = (Subcollection) subcollection_list.getSelectedValue();
+		    flags_field.setText(subcollection.getFlags());
+		    include_button.setSelected(subcollection.isInclusive());
+		    exclude_button.setSelected(!subcollection.isInclusive());
+		    match_field.setText(subcollection.getPattern());
+		    name_field.setText(subcollection.getName());
+		    String s = subcollection.getSource();
+		    int pos = 0;
+		    Object value = source_combobox.getItemAt(pos);
+		    //ystem.err.println("Search for: " + s);
+		    while (value != null) {
+			if (value instanceof MetadataElement) {
+			    MetadataElement metadata_element = (MetadataElement) value;
+			    String metadata_element_name = metadata_element.getFullName();
+			    //ystem.err.print("Compare to: " + e_name);
+			    if (metadata_element_name.equals(s)) {
+				source_combobox.setSelectedIndex(pos);
+				value = null;
+				//ystem.err.println(" - Match");
+			    }
+			    else {
+				pos++;
+				value = source_combobox.getItemAt(pos);
+				//ystem.err.println(" - Fail");
+			    }
+			}
+			else if(value.toString().equals(s)) {
+			    source_combobox.setSelectedIndex(pos);
+			    value = null;
+			}
+			else {
+			    pos++;
+			    value = source_combobox.getItemAt(pos);
+			}
+		    }
+		    // Can't add one thats already there.
+		    add_button.setEnabled(false);
+		    // You can update or remove it though...
+		    remove_button.setEnabled(true);
+		    update_button.setEnabled(true);
+		}
+		else {
+		    flags_field.setText("");
+		    include_button.setSelected(true);
+		    match_field.setText("");
+		    name_field.setText("");
+		    source_combobox.setSelectedIndex(0);
+		    remove_button.setEnabled(false);
+		    update_button.setEnabled(false);
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollection.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollection.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollection.java	(revision 21112)
@@ -0,0 +1,99 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+* This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+/**************************************************************************************
+ * Written:     02/07/03
+ * Revised:     
+ **************************************************************************************/
+import org.w3c.dom.Element;
+
+public class SuperCollection
+    implements DOMProxyListEntry {
+
+    private Element element;
+    private String name;
+
+    public SuperCollection() {
+	element = null;
+	name = null;
+    }
+
+    public SuperCollection(Element element) {
+	this.element = element;
+	name = null;
+    }
+
+    public int compareTo(Object other) {
+	return this.getName().compareTo(other.toString());
+    }
+
+    public DOMProxyListEntry create(Element element) {
+	return new SuperCollection(element);
+    } 
+
+    public boolean equals(Object other) {
+	return (compareTo(other) == 0);
+    }
+
+    public Element getElement() {
+	return element;
+    }
+
+    public String getName() {
+	if(name == null && element != null) {
+	    name = element.getAttribute(CollectionConfiguration.NAME_ATTRIBUTE);
+	}
+	return name;
+    }
+
+    public boolean isAssigned() {
+	return (element != null && element.getAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE).equals(CollectionConfiguration.TRUE_STR));
+    }
+
+    public void setAssigned(boolean assigned) {
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.ASSIGNED_ATTRIBUTE, (assigned ? CollectionConfiguration.TRUE_STR : CollectionConfiguration.FALSE_STR));
+	}
+    }
+
+    public void setElement(Element element) {
+	this.element = element;
+	name = null;
+    }
+
+    public void setName(String name) {
+	this.name = name;
+	if(element != null) {
+	    element.setAttribute(CollectionConfiguration.NAME_ATTRIBUTE, name);
+	}
+    }
+
+    public String toString() {
+	return getName();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollectionManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollectionManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/SuperCollectionManager.java	(revision 21112)
@@ -0,0 +1,254 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+* This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.CheckListEntry;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.w3c.dom.*;
+
+/** This class contains the information about what supercollection has been specified (if any) and methods for changing this information. Note that there is a major difference between this manager and the others in that its DOM model is never used directly in any list component. It is only used to decide whether a certain entry in the actual checklist is checked. */
+public class SuperCollectionManager
+    extends DOMProxyListModel {
+
+    static final public String SUPERCOLLECTION_COMMAND = "supercollection";
+    static final public String CCS_COMMAND = "ccs";
+    private ArrayList collection_checklist_model = null; // Model used to actually populate list
+    private Control controls = null;
+    private DOMProxyListModel model = null;
+    private String current_coll_name = null;
+    private boolean superCollectionChanged = false;
+
+    public SuperCollectionManager(Element supercollections_element) {
+	super(supercollections_element, StaticStrings.COLLECTION_ELEMENT, new SuperCollection());
+	DebugStream.println("SuperCollectionManager: " + getSize() + " supercollection members parsed.");
+	this.model = this;
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+	if(collection_checklist_model != null) {
+	    collection_checklist_model.clear();
+	    collection_checklist_model = null;
+	}
+    }
+
+    private void addSuperCollection(SuperCollection supercollection) {
+	if(!contains(supercollection)) {
+	    add(getSize(), supercollection);
+	    Gatherer.c_man.configurationChanged();
+	}
+	
+    }
+
+    /** Method to retrieve the control for this manager.
+     * @return the Control for editing supercollection settings
+     */
+    public Control getControls() {
+	if(controls == null) {
+	    // Build controls
+	    this.controls = new SuperCollectionControl();
+	}
+	return controls;
+    }
+
+    public SuperCollection getSuperCollection(String collection_name) {
+	SuperCollection result = null;
+	int size = getSize();
+	for(int i = 0; result == null && i < size; i++) {
+	    SuperCollection supercollection = (SuperCollection) getElementAt(i);
+	    if(supercollection.getName().equals(collection_name)) {
+		result = supercollection;
+	    }
+	    supercollection = null;
+	}
+	return result;
+    }
+
+    private void removeSuperCollection(SuperCollection supercollection) {
+	if(contains(supercollection)) {
+	    remove(supercollection);
+	    Gatherer.c_man.configurationChanged();
+	}
+    }
+
+    /** Provides controls for altering the SuperCollection settings. */
+    private class SuperCollectionControl
+	extends JPanel
+	implements Control {
+
+	private boolean init = true;
+	private CheckList collection_checklist = null;
+
+	SuperCollectionControl() {
+	    super();
+
+	    // Creation
+	    JPanel header_panel = new JPanel();
+
+	    JLabel title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    title_label.setOpaque(true);
+	    Dictionary.registerText(title_label, "CDM.SuperCollectionManager.Title");
+
+	    JTextArea instructions = new JTextArea();
+	    instructions.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    instructions.setCaretPosition(0);
+	    instructions.setEditable(false);
+	    instructions.setLineWrap(true);
+	    instructions.setRows(6);
+	    instructions.setWrapStyleWord(true);
+	    Dictionary.registerText(instructions, "CDM.SuperCollectionManager.Instructions");
+
+	    collection_checklist = new CheckList(false);
+	    buildModel();
+	    collection_checklist.setListData(collection_checklist_model);
+
+	    // Layout
+	    header_panel.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    header_panel.setLayout(new BorderLayout());
+	    header_panel.add(title_label, BorderLayout.NORTH);
+	    header_panel.add(new JScrollPane(instructions), BorderLayout.CENTER);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_panel, BorderLayout.NORTH);
+	    add(new JScrollPane(collection_checklist), BorderLayout.CENTER);
+	}
+
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	}
+
+	public void loseFocus() {
+	    CollectionDesignManager.collect_cfg_change_listener.maybeSetRebuildRequired(); // !! TO DO: This is crap
+	    int super_collections_count = 0;
+	    // Retrieve the current supercollections
+	    ArrayList supercollections = children();
+	    // Now iterate through the checklist, and for each checked box found ensure the Supercollection exists, and ensure its assigned. Remove any supercollections altered in this way from the temporary array list
+	    // we ignore the current coll at the moment
+	    int size = collection_checklist_model.size();
+	    for(int i = 0; i < size; i++) {
+		CheckListEntry entry = (CheckListEntry) collection_checklist_model.get(i);
+		if(entry.isSelected()) {
+		    String collection_name = (String) entry.getProperty();
+		    if (!collection_name.equals(current_coll_name)) {
+			SuperCollection supercollection = getSuperCollection(collection_name);
+			// Create the supercollection element if necessary
+			if(supercollection == null) {
+			    Element supercollection_element = root.getOwnerDocument().createElement(StaticStrings.COLLECTION_ELEMENT);
+			    supercollection = new SuperCollection(supercollection_element);
+			    supercollection.setName(collection_name);
+			    addSuperCollection(supercollection);
+			}
+			else {
+			    supercollections.remove(supercollection);
+			}
+			supercollection.setAssigned(true);
+			super_collections_count++;
+		    }
+		}
+	    }
+	    if (super_collections_count > 0) {
+		// we have some super colls, add in the current collection
+		SuperCollection supercollection = getSuperCollection(current_coll_name);
+		// Create the supercollection element if necessary
+		if(supercollection == null) {
+		    Element supercollection_element = root.getOwnerDocument().createElement(StaticStrings.COLLECTION_ELEMENT);
+		    supercollection = new SuperCollection(supercollection_element);
+		    supercollection.setName(current_coll_name);
+		    addSuperCollection(supercollection);
+		}
+		else {
+		    supercollections.remove(supercollection);
+		}
+		model.root.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.TRUE_STR);
+	    } else {
+		// current collection is the only one - don't bother adding it, because assigned is false, and we don't want to update the config file for an unassigned item.
+		model.root.setAttribute(StaticStrings.ASSIGNED_ATTRIBUTE, StaticStrings.FALSE_STR);
+	    }
+	    
+	    // Any collections left in the temporary list have been unselected, so delete them
+	    for(int j = supercollections.size(); j > 0; j--) {
+		SuperCollection supercollection = (SuperCollection) supercollections.get(j - 1);
+		removeSuperCollection(supercollection);
+	    }
+	}
+
+	
+	private void buildModel()
+	{
+	    collection_checklist_model = new ArrayList();
+	    current_coll_name = Gatherer.c_man.getCollection().getName();
+
+	    File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	    File[] possible_collections = collect_directory.listFiles();
+	    for(int i = 0; possible_collections != null && i < possible_collections.length; i++) {
+		File collect_cfg_file = new File(possible_collections[i], Utility.CONFIG_FILE);
+		if (collect_cfg_file.exists()) {
+		    BasicCollectionConfiguration collect_cfg = new BasicCollectionConfiguration(collect_cfg_file);
+		    StringBuffer title_buffer = new StringBuffer(collect_cfg.getName());
+		    title_buffer.append(StaticStrings.SPACE_CHARACTER);
+		    title_buffer.append(StaticStrings.OPEN_PARENTHESIS_CHARACTER);
+		    title_buffer.append(possible_collections[i].getName());
+		    title_buffer.append(StaticStrings.CLOSE_PARENTHESIS_CHARACTER);
+		    String collection_title = title_buffer.toString();
+		    title_buffer = null;
+		    String collection_name = possible_collections[i].getName();
+
+		    // We have to block the model collection.
+		    if (collect_cfg.getName().equals("**title**")) {
+			continue;
+		    }
+
+		    // The current collection is always selected.
+		    CheckListEntry entry = new CheckListEntry(collection_title);
+		    entry.setProperty(collection_name);
+		    entry.setSelected(getSuperCollection(collection_name) != null || collection_name.equals(current_coll_name));
+		    entry.setFixed(collection_name.equals(current_coll_name));
+		    collection_checklist_model.add(entry);
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/TranslationView.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/TranslationView.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/cdm/TranslationView.java	(revision 21112)
@@ -0,0 +1,726 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.cdm;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+
+/** This class provides a graphical interface to allow a user to quickly and conviently (ie all in one place) translate the text fragments associated with general metadata and indexes into each of the assigned languages in the collection. It should provide clear controls for the editing of these text fragments, plus clear indicate what languages still need further translation, which it will do through a combination of coloring and other visual indicators.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class TranslationView {
+
+    private Control controls;
+
+    static final private Dimension COMPONENT_SIZE = new Dimension(225,25);
+    static final private int LANGUAGE_WIDTH = 75;
+    static final private String GENERAL_STR = "General:";
+    static final private String INDEX_STR = "Index:";
+    static final private String PARTITION_STR = "Partitions:";
+
+    public TranslationView() {
+	DebugStream.println("TranslationView: Initialized.");
+    }
+
+    public void destroy() {
+	if(controls != null) {
+	    controls.destroy();
+	    controls = null;
+	}
+    }
+
+    public Control getControls() {
+	if(controls == null) {
+	    controls = new TranslationControl();
+	}
+	return controls;
+    }
+
+    private Object[] getFeaturesList() {
+	ArrayList features_model = new ArrayList();
+
+	// Add the indexes
+	ArrayList indexes = CollectionDesignManager.index_manager.getIndexes();
+	int indexes_size = indexes.size();
+	for(int j = 0; j < indexes_size; j++) {
+	    Object bob = new BobTheMagicalComparableWrapper(indexes.get(j));
+	    if(!features_model.contains(bob)) {
+		features_model.add(bob);
+	    }
+	}
+
+	// We also add subindexes.
+	ArrayList subindexes = CollectionDesignManager.subcollectionindex_manager.getSubcollectionIndexes();
+	int subindexes_size = subindexes.size();
+	for(int j = 0; j < subindexes_size; j++) {
+	    Object bob = new BobTheMagicalComparableWrapper(subindexes.get(j));
+	    if(!features_model.contains(bob)) {
+		features_model.add(bob);
+	    }
+	}
+
+	// Finally we add any remaining metadata as general
+	ArrayList general_metadata = CollectionDesignManager.collectionmeta_manager.getMetadata();
+	for(int i = 0; i < general_metadata.size(); i++) {
+	    CollectionMeta metadata = (CollectionMeta) general_metadata.get(i);
+	    if(!metadata.isSpecial()) {
+		Object bob = new BobTheMagicalComparableWrapper(metadata.getName());
+		if(!features_model.contains(bob)) {
+		    features_model.add(bob);
+		}
+	    }
+	}
+
+	Collections.sort(features_model);
+
+	return features_model.toArray();
+    }
+
+    private class BobTheMagicalComparableWrapper
+	implements Comparable {
+	private Object content;
+	private String text;
+	BobTheMagicalComparableWrapper(Object content) {
+	    this.content = content;
+	}
+	public int compareTo(Object object) {
+	    if(object == null) {
+		object = "";
+	    }
+	    if(text == null) {
+		toString();
+	    }
+	    return text.compareTo(object.toString());
+	}
+
+	/** Equals is used by contains and since we want to prevent double up of metadata we compare differently than in compareTo. */
+	public boolean equals(Object object) {
+	    return (object != null && content.toString().equals(((BobTheMagicalComparableWrapper)object).getContent().toString()));
+	}
+
+	public Object getContent() {
+	    return content;
+	}
+
+	public String getName() {
+	    if(content instanceof Index) {
+		return CollectionConfiguration.STOP_CHARACTER + ((Index)content).getID();
+	    }
+	    else if (content instanceof SubcollectionIndex) {
+	    	return CollectionConfiguration.STOP_CHARACTER + ((SubcollectionIndex)content).getID();
+	    }
+	    else {
+		return content.toString();
+	    }
+	}
+
+	public String toString() {
+	    if(text == null) {
+		String temp = content.toString();
+		if(temp.indexOf(CollectionConfiguration.SPACE_CHARACTER) != -1) {
+		    temp = temp.substring(0, temp.indexOf(CollectionConfiguration.SPACE_CHARACTER));
+		}
+		if(content instanceof Index) {
+		    text = INDEX_STR + temp;
+		}
+		else if(content instanceof SubcollectionIndex) {
+		    text = PARTITION_STR + temp;
+		}
+		else {
+		    text = GENERAL_STR + temp;
+		}
+	    }
+	    return text;
+	}
+    }
+
+    private class TranslationControl
+	extends JPanel 
+	implements Control {
+
+	private boolean ignore_event = false;
+	private FragmentTableModel fragment_table_model;
+	private JButton add_button;
+	private JButton remove_button;
+	private JButton replace_button;
+	private JComboBox language_combobox;
+	private JList features_list;
+	private JTable fragment_table;
+	private JTextArea default_area;
+	private JTextArea translation_area;
+
+	TranslationControl() {
+	    fragment_table_model = new FragmentTableModel("", new TreeSet(), new ArrayList());
+	    // Creation
+	    JPanel header_panel = new JPanel();
+	    JLabel title_label = new JLabel();
+	    title_label.setHorizontalAlignment(JLabel.CENTER);
+	    Dictionary.registerText(title_label, "CDM.TranslationManager.Title");
+
+	    JTextArea instruction_area = new JTextArea();
+	    instruction_area.setEditable(false);
+	    instruction_area.setLineWrap(true);
+	    instruction_area.setRows(6);
+	    instruction_area.setWrapStyleWord(true);
+	    Dictionary.registerText(instruction_area, "CDM.TranslationManager.Instructions");
+
+	    JPanel center_panel = new JPanel();
+
+	    JPanel selection_panel = new JPanel();
+
+	    JPanel component_selection_panel = new JPanel();
+	    JLabel component_label = new JLabel();
+	    Dictionary.registerText(component_label, "CDM.TranslationManager.Affected_Features");
+	    features_list = new JList(getFeaturesList());
+	    features_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	    JPanel fragment_selection_panel = new JPanel();
+	    JLabel fragment_label = new JLabel();
+	    Dictionary.registerText(fragment_label, "CDM.TranslationManager.Assigned_Fragments");
+
+	    fragment_table = new JTable(fragment_table_model);
+	    fragment_table.setAutoResizeMode(JTable.AUTO_RESIZE_LAST_COLUMN);
+	    fragment_table.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    fragment_table.setColumnSelectionAllowed(false);
+	    fragment_table.setDefaultRenderer(Object.class, new FragmentTableRenderer());
+	    fragment_table.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	    JScrollPane table_scroll = new JScrollPane(fragment_table);
+	    table_scroll.getViewport().setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    table_scroll.setOpaque(true);
+
+	    JPanel south_panel = new JPanel();
+
+	    JPanel text_panel = new JPanel();
+
+	    JPanel language_panel = new JPanel();
+	    JLabel language_label = new JLabel();
+	    Dictionary.registerText(language_label, "CDM.TranslationManager.Language");
+	    language_combobox = new JComboBox(CollectionDesignManager.language_manager.getLanguageCodes().toArray());
+	    language_combobox.setPreferredSize(COMPONENT_SIZE);
+	    language_combobox.setRenderer(new LanguageListCellRenderer());
+	    Dictionary.registerTooltip(language_combobox, "CDM.TranslationManager.Language_Tooltip");
+
+	    JPanel default_text_panel = new JPanel();
+	    JLabel default_label = new JLabel();
+	    Dictionary.registerText(default_label, "CDM.TranslationManager.Default_Text");
+	    default_area = new JTextArea();
+	    default_area.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    default_area.setEditable(false);
+	    default_area.setLineWrap(true);
+	    default_area.setWrapStyleWord(true);
+
+	    JPanel translated_text_panel = new JPanel();
+	    JLabel translation_label = new JLabel();
+	    Dictionary.registerText(translation_label, "CDM.TranslationManager.Translation");
+	    translation_area = new JTextArea();
+	    translation_area.setBackground(Configuration.getColor("coloring.disabled", false));
+	    translation_area.setEnabled(false);
+	    translation_area.setLineWrap(true);
+	    translation_area.setWrapStyleWord(true);
+	    Dictionary.registerTooltip(translation_area, "CDM.TranslationManager.Translation_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    add_button = new GLIButton();
+	    add_button.setEnabled(false);
+	    add_button.setMnemonic(KeyEvent.VK_A);
+	    Dictionary.registerBoth(add_button, "CDM.TranslationManager.Add", "CDM.TranslationManager.Add_Tooltip");
+	    replace_button = new GLIButton();
+	    replace_button.setEnabled(false);
+	    replace_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerBoth(replace_button, "CDM.TranslationManager.Replace", "CDM.TranslationManager.Replace_Tooltip");
+	    remove_button = new GLIButton();
+	    remove_button.setEnabled(false);
+	    remove_button.setMnemonic(KeyEvent.VK_R);
+	    Dictionary.registerBoth(remove_button, "CDM.TranslationManager.Remove", "CDM.TranslationManager.Remove_Tooltip");
+
+	    // Connection
+	    add_button.addActionListener(new AddListener());
+	    add_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    remove_button.addActionListener(new RemoveListener());
+	    remove_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    replace_button.addActionListener(new ReplaceListener());
+	    replace_button.addActionListener(CollectionDesignManager.buildcol_change_listener);
+	    language_combobox.addActionListener(new LanguageActionListener());
+	    translation_area.getDocument().addDocumentListener(new TranslationDocumentListener());
+	    features_list.addListSelectionListener(new FeaturesListSelectionListener());
+	    fragment_table.getSelectionModel().addListSelectionListener(new FragmentListSelectionListener());
+
+	    // Layout
+	    header_panel.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    header_panel.setLayout(new BorderLayout());
+	    header_panel.add(title_label, BorderLayout.NORTH);
+	    header_panel.add(new JScrollPane(instruction_area), BorderLayout.CENTER);
+
+	    component_selection_panel.setLayout(new BorderLayout());
+	    component_selection_panel.add(component_label, BorderLayout.NORTH);
+	    component_selection_panel.add(new JScrollPane(features_list), BorderLayout.CENTER);
+
+	    fragment_selection_panel.setLayout(new BorderLayout());
+	    fragment_selection_panel.add(fragment_label, BorderLayout.NORTH);
+	    fragment_selection_panel.add(table_scroll, BorderLayout.CENTER);
+
+	    selection_panel.setLayout(new GridLayout(2,1,0,5));
+	    selection_panel.add(component_selection_panel);
+	    selection_panel.add(fragment_selection_panel);
+
+	    default_text_panel.setLayout(new BorderLayout());
+	    default_text_panel.add(default_label, BorderLayout.NORTH);
+	    default_text_panel.add(new JScrollPane(default_area), BorderLayout.CENTER);
+
+	    translated_text_panel.setLayout(new BorderLayout());
+	    translated_text_panel.add(translation_label, BorderLayout.NORTH);
+	    translated_text_panel.add(new JScrollPane(translation_area), BorderLayout.CENTER);
+
+	    text_panel.setLayout(new GridLayout(1,2,5,0));
+	    text_panel.add(default_text_panel);
+	    text_panel.add(translated_text_panel);
+
+	    language_panel.setLayout(new BorderLayout(5,0));
+	    language_panel.add(language_label, BorderLayout.WEST);
+	    language_panel.add(language_combobox, BorderLayout.CENTER);
+
+	    south_panel.setLayout(new BorderLayout());
+	    south_panel.add(language_panel, BorderLayout.NORTH);
+	    south_panel.add(text_panel, BorderLayout.CENTER);
+
+	    center_panel.setLayout(new GridLayout(2,1,0,5));
+	    center_panel.add(selection_panel);
+	    center_panel.add(south_panel);
+
+	    button_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	    button_pane.setLayout(new GridLayout(1,3));
+	    button_pane.add(add_button);
+	    button_pane.add(replace_button);
+	    button_pane.add(remove_button);
+
+	    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    setLayout(new BorderLayout());
+	    add(header_panel, BorderLayout.NORTH);
+	    add(center_panel, BorderLayout.CENTER);
+	    add(button_pane, BorderLayout.SOUTH);
+	}
+
+	public void destroy() {
+	}
+
+	public void gainFocus() {
+	    // Rebuild the features model, just incase features have been added or removed.
+	    if(features_list != null) {
+		Object selected_feature = features_list.getSelectedValue();
+		features_list.setListData(getFeaturesList());
+		if(selected_feature != null) {
+		    features_list.setSelectedValue(selected_feature, true);
+		}
+	    }
+	}
+
+	public void loseFocus() {
+	}
+
+
+	private class AddListener
+	    implements ActionListener {
+	    
+	    public void actionPerformed(ActionEvent event) {
+		ignore_event = true;
+		String translation_text = translation_area.getText();
+		if(translation_text.length() > 0) {
+		    // If there is no current fragment table selection, but translation text is not an empty string, then this is a new fragment. Create new collection metadata, refresh the fragment table, then ensure that the new metadata is selected. Remember that it is the selected metadata to which changes will be applied.
+		    BobTheMagicalComparableWrapper selected_feature = (BobTheMagicalComparableWrapper)features_list.getSelectedValue();
+		    String language = (String) language_combobox.getSelectedItem();
+		    CollectionMeta metadatum = new CollectionMeta(selected_feature.getName(), language);
+		    metadatum.setValue(translation_text);
+		    CollectionDesignManager.collectionmeta_manager.addMetadatum(metadatum);
+		    fragment_table.clearSelection();
+		    ArrayList metadata = CollectionDesignManager.collectionmeta_manager.getMetadata(selected_feature.getName());
+		    fragment_table_model.setData(selected_feature.getName(), CollectionDesignManager.collectionmeta_manager.getLanguages(), metadata);
+		    int index = fragment_table_model.getMetadataIndexByLanguage(language);
+		    ///ystem.err.println("I want to select the row " + index + " out of " + metadata.size());
+		    fragment_table.setRowSelectionInterval(index, index);
+		    metadatum = null;
+		    language = null;
+		    selected_feature = null;
+		    translation_text = null;
+		    remove_button.setEnabled(true);
+		    Gatherer.c_man.configurationChanged();
+		}
+		add_button.setEnabled(false);
+		replace_button.setEnabled(false);
+		ignore_event = false;
+	    }
+	}
+
+	private class RemoveListener
+	    implements ActionListener {
+	    
+	    public void actionPerformed(ActionEvent event) {
+		ignore_event = true;
+		int index = -1;
+		if((index = fragment_table.getSelectedRow()) != -1) {
+		    CollectionMeta metadata = fragment_table_model.getMetadata(index);
+		    CollectionDesignManager.collectionmeta_manager.removeMetadata(metadata);
+		    fragment_table_model.remove(index);
+		    // If a remove occured then enable add
+		    add_button.setEnabled(true);
+		    Gatherer.c_man.configurationChanged();
+		}
+		// Either way disable replace and remove
+		replace_button.setEnabled(false);
+		remove_button.setEnabled(false);
+		ignore_event = false;
+	    }
+	}
+
+	private class ReplaceListener 
+	    implements ActionListener {
+
+	    public void actionPerformed(ActionEvent event) {
+		ignore_event = true;
+		int index = -1;
+		if((index = fragment_table.getSelectedRow()) != -1) {
+		    String translation_text = translation_area.getText();
+		    // Get the selected collection metadata
+		    CollectionMeta metadata = fragment_table_model.getMetadata(index);
+		    // Update the fragment metadata and ask the table to repaint the appropriate row
+		    metadata.setValue(translation_text);
+		    fragment_table_model.fireTableRowsUpdated(index, index);
+		    metadata = null;
+		    remove_button.setEnabled(true);
+		    Gatherer.c_man.configurationChanged();
+		}
+		// Either way disable replace
+		add_button.setEnabled(false);
+		replace_button.setEnabled(false);
+		ignore_event = false;
+	    }
+	}
+
+	private class FeaturesListSelectionListener
+	    implements ListSelectionListener {
+	    public void valueChanged(ListSelectionEvent event) {
+		if(!event.getValueIsAdjusting() && !ignore_event) {
+		    ///ystem.err.println("FeaturesListSelectionListener");
+		    if(!features_list.isSelectionEmpty()) {
+			// Determine the features name. Remember to remove our helpful source prefix (delimited by ':').
+			BobTheMagicalComparableWrapper selected_feature = (BobTheMagicalComparableWrapper)features_list.getSelectedValue();
+			// Retrieve the default language translation, or otherwise the first match, to be default text.
+			CollectionMeta default_metadata = CollectionDesignManager.collectionmeta_manager.getMetadatum(selected_feature.getName());
+			if(default_metadata != null) {
+			    default_area.setText(default_metadata.getValue(CollectionMeta.TEXT));
+			}
+			// Update the text fragment table.
+			fragment_table.clearSelection();
+			fragment_table_model.setData(selected_feature.getName(), CollectionDesignManager.collectionmeta_manager.getLanguages(), CollectionDesignManager.collectionmeta_manager.getMetadata(selected_feature.getName()));
+			selected_feature = null;
+			// Now we check whatever value is currently selected in the combobox to see if it is valid to add.
+			String language_name = (String) language_combobox.getSelectedItem();
+			int index = fragment_table_model.getMetadataIndexByLanguage(language_name);
+			if(index != -1) {
+			    CollectionMeta metadata = fragment_table_model.getMetadata(index);
+			    fragment_table.setRowSelectionInterval(index, index);
+			    if(metadata != null) {
+				translation_area.setText(metadata.getValue(CollectionMeta.TEXT));
+			    }
+			    else {
+				translation_area.setText("");
+			    }
+			    metadata = null;
+			}
+			else {
+			    translation_area.setText("");
+			}
+			// Update and enable the text area
+			translation_area.setEnabled(true);
+			translation_area.setBackground(Configuration.getColor("coloring.editable_background", false));
+		    }
+		    else {
+			default_area.setText("");
+			fragment_table.clearSelection();
+			fragment_table_model.setData("", new TreeSet(), new ArrayList());
+			translation_area.setText("");
+			// Update and disable the text area
+			translation_area.setText("");
+			translation_area.setEnabled(false);
+			translation_area.setBackground(Configuration.getColor("coloring.disabled", false));
+		    }
+		}
+	    }
+	}
+
+	private class FragmentListSelectionListener
+	    implements ListSelectionListener {
+	    public void valueChanged(ListSelectionEvent event) {
+		if(!event.getValueIsAdjusting() && !ignore_event) {
+		    ignore_event = true;
+		    ///ystem.err.println("FragmentListSelectionListener");
+		    int index = -1;
+		    if((index = fragment_table.getSelectedRow()) != -1) {
+			// A row has been selected. Retrieve the collection metadata.
+			CollectionMeta metadatum = fragment_table_model.getMetadata(index);
+			if(metadatum != null) {
+			    // Update the combobox to show the current language.
+			    String language = metadatum.getLanguage();
+			    ///ystem.err.println("Searching for the language: " + language_name);
+			    for(int i = 0; i < language_combobox.getItemCount(); i++) {
+				if(language_combobox.getItemAt(i).toString().equals(language)) {
+				    language_combobox.setSelectedIndex(i);
+				}
+			    }
+			    translation_area.setText(metadatum.getValue(CollectionMeta.TEXT));
+			    remove_button.setEnabled(!metadatum.isDummy());
+			}
+			else {
+			    remove_button.setEnabled(false);
+			}
+		    }
+		    else {
+			translation_area.setText("");
+			remove_button.setEnabled(false);
+		    }
+		    ignore_event = false;
+		}
+	    }
+	}
+
+	private class FragmentTableModel
+	    extends AbstractTableModel {
+
+	    private ArrayList metadata;
+	    private String feature_name;
+	    private TreeSet languages;
+
+	    FragmentTableModel(String feature_name, TreeSet languages, ArrayList metadata) {
+		this.feature_name = feature_name;
+		this.languages = languages;
+		this.metadata = metadata;
+	    }
+	    public int getRowCount() {
+	       // The row count is equal to the maximum number of languages currently assigned in the collection.
+		return languages.size();
+	    }
+	    public int getColumnCount() {
+		return 2;
+	    }
+
+	    public String getColumnName(int column) {
+		if (column == 0) {
+		    return Dictionary.get("CDM.TranslationManager.Language_Column");
+		}
+		if (column == 1) {
+		    return Dictionary.get("CDM.TranslationManager.Fragment_Column");
+		}
+		return "";
+	    }
+
+	    /** Attempt to retrieve the metadata associated with a certain row - however not all rows actually have metadata with them (say the French row where no metadata has been set).
+	     * @param row the row number as an int
+	     * @return the CollectionMeta requested, which may be a dummy metadata pair
+	     */
+	    public CollectionMeta getMetadata(int row) {
+		// Determine what language we are talking about
+		String language = null;
+		Iterator language_iterator = languages.iterator();
+		int current_row = 0;
+		while(current_row != row && language_iterator.hasNext()) {
+		    language_iterator.next();
+		    current_row++;
+		}
+		if(current_row == row) {
+		    language = (String) language_iterator.next();
+		    return getMetadata(language);
+		}
+		language_iterator = null;
+		return null;
+	    }
+
+	    public CollectionMeta getMetadata(String language) {
+		// Attempt to retrieve metadata with that language
+		for(int i = 0; i < metadata.size(); i++) {
+		    CollectionMeta metadatum = (CollectionMeta) metadata.get(i);
+		    if(metadatum.getLanguage().equals(language)) {
+			return metadatum;
+		    }
+		}
+		return new CollectionMeta(feature_name, language, true);
+	    }
+
+	    public int getMetadataIndexByLanguage(String language) {
+		///ystem.err.println("Find the index for the language " + language + " (out of " + languages.size() + ")");
+		// First we have to iterate to the correct place
+		Iterator language_iterator = languages.iterator();
+		int current_row = 0;
+		while(language_iterator.hasNext()) {
+		    String target = (String)language_iterator.next();
+		    if(language.equals(target)) {
+			///ystem.err.println("Matches " + target);
+			return current_row;
+		    }
+		    else {
+			///ystem.err.println("Doesn't match " + target);
+			current_row++;
+		    }
+		}
+		///ystem.err.println("No such language in model: -1");
+		return -1;
+	    }
+
+	    public Object getValueAt(int row, int column) {
+		if(0 <= row && row < languages.size()) {
+		    // First we have to iterate to the correct place
+		    Iterator language_iterator = languages.iterator();
+		    int current_row = 0;
+		    while(current_row != row && language_iterator.hasNext()) {
+			language_iterator.next();
+			current_row++;
+		    }
+		    if(current_row == row) {
+			String language = (String)language_iterator.next();
+			if(column == 0) {
+			    return CollectionDesignManager.language_manager.getLanguageName(language);
+			}
+			else {
+			    CollectionMeta metadatum = getMetadata(language);
+			    return metadatum.getValue(CollectionMeta.TEXT);
+			}
+		    }
+		    language_iterator = null;
+		}
+		return "#Error";
+	    }
+
+	    public void remove(int index) {
+		metadata.remove(index);
+		fireTableRowsDeleted(index, index);
+	    }
+
+	    public void setData(String feature_name, TreeSet languages, ArrayList metadata) {
+		this.feature_name = feature_name;
+		this.languages = languages;
+		this.metadata = metadata;
+		fireTableDataChanged();
+	    }
+
+	    public int size() {
+		return metadata.size();
+	    }
+	}
+
+	private class FragmentTableRenderer
+	    extends DefaultTableCellRenderer {
+	    /** Retrieve the component used to rubberstamp the table based on the given parameters.
+	     * @param table The <strong>JTable</strong> to rendered.
+	     * @param value The <strong>Object</strong> whose representation will be shown in the table row.
+	     * @param isSelected <i>true</i> iff the column/row is selected.
+	     * @param hasFocus <i>true</i> iff the column/row is in focus.
+	     * @param row An <i>int</i> specifying the target row.
+	     * @param column An <i>int</i> specifying the target column.
+	     * @see org.greenstone.gatherer.cdm.Language
+	     */
+	    public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) {
+		JComponent component = (JComponent) super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column);
+		if(isSelected) {
+		    component.setBackground(Configuration.getColor("coloring.workspace_heading_background", false));
+		}
+		else {
+		    component.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+		}
+		return component;
+	    }
+	}
+
+	private class LanguageActionListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		if(!ignore_event) {
+		    ignore_event = true;
+		    ///ystem.err.println("LanguageActionListener");
+		    // If the newly selected language value already exists in the fragment table, and that row isn't the one selected, then select that row the text area.
+		    String language_name = language_combobox.getSelectedItem().toString();
+		    int index = fragment_table_model.getMetadataIndexByLanguage(language_name);
+		    if(index != -1) {
+			CollectionMeta metadata = fragment_table_model.getMetadata(index);
+			fragment_table.setRowSelectionInterval(index, index);
+			translation_area.setText(metadata.getValue(CollectionMeta.TEXT));
+		    }
+		    else {
+			fragment_table.clearSelection();
+			translation_area.setText("");
+		    }
+		    // Ready the text area
+		    translation_area.setEnabled(true);
+		    translation_area.setBackground(Configuration.getColor("coloring.editable_background", false));
+		    ignore_event = false;
+		}
+	    }
+	}
+
+	private class TranslationDocumentListener
+	    implements DocumentListener {
+	    /** Gives notification that an attribute or set of attributes changed. */
+	    public void changedUpdate(DocumentEvent e) {
+		update();
+	    }
+	    /** Gives notification that there was an insert into the document. */
+	    public void insertUpdate(DocumentEvent e) {
+		update();
+	    }
+	    /** Gives notification that a portion of the document has been removed. */
+	    public void removeUpdate(DocumentEvent e) {
+		update();
+	    }
+	    /** The text area has changed in some way. Given that this can only happed when we are editing or adding a text fragment we better respond appropriately. */
+	    private void update() {
+		String translation_text = translation_area.getText();
+		// Determine if add should be enabled. You can only add if the current text fragment doesn't already exist in the fragment table for the given language
+		String language = (String) language_combobox.getSelectedItem();
+		CollectionMeta metadatum = fragment_table_model.getMetadata(language);
+		add_button.setEnabled(translation_text.length() > 0 && (metadatum.isDummy() || fragment_table_model.getMetadataIndexByLanguage(language) == -1));
+		language = null;
+		// Determine if replace should be enabled. Replace is only enabled it the text fragment is different from the one in the current fragment table selection.
+		if(fragment_table.getSelectedRowCount() > 0) {
+		    replace_button.setEnabled(!metadatum.isDummy() && !translation_text.equals(fragment_table.getValueAt(fragment_table.getSelectedRow(), 1)));
+		}
+		// Nothing selected, nothing to replace
+		else {
+		    replace_button.setEnabled(false);
+		}
+		translation_text = null;
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/BasicCollectionConfiguration.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/BasicCollectionConfiguration.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/BasicCollectionConfiguration.java	(revision 21112)
@@ -0,0 +1,213 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.collection;
+
+import java.io.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.cdm.CommandTokenizer;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+/** This class provides access to a collection configuration file. This version accepts either a valid xml document or a historical collect.cfg file. */
+public class BasicCollectionConfiguration
+    implements Comparable {
+    
+    private File file;
+    private String creator = "";
+    private String description = "";
+    private String maintainer = "";
+    private String name = "";
+    
+    private String site = null; // used for gs3 colls
+    
+    public BasicCollectionConfiguration(File file) {
+	this.file = file;
+	try {
+	    String filename = file.getName().toLowerCase();
+	    if(!file.exists()) {
+		return;
+	    }
+	    if(filename.endsWith(".cfg")) {
+		FileInputStream fis = new FileInputStream(file);
+		InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
+		BufferedReader br = new BufferedReader(isr);
+		String command = null;
+		while((command = br.readLine()) != null) {
+		    if(command.length() > 0) {
+			// We have to test the end of command for the special character '\'. If found, remove it and append the next line, then repeat.
+			while(command.trim().endsWith("\\")) {
+			    command = command.substring(0, command.lastIndexOf("\\"));
+			    String next_line = br.readLine();
+			    if (next_line != null) {
+				command = command + "\n" + next_line;
+			    }
+			    next_line = null;
+			}
+			CommandTokenizer tokenizer = new CommandTokenizer(command, br);
+			String command_type_str = tokenizer.nextToken();
+			if (command_type_str == null) {
+			    // Bad command. Do nothing
+			    continue;
+			}
+
+			command_type_str = command_type_str.toLowerCase();
+			if(command_type_str.equals(StaticStrings.COLLECTIONMETADATA_CREATOR_STR)) {
+			    creator = tokenizer.nextToken();
+			}
+			else if(command_type_str.equals(StaticStrings.COLLECTIONMETADATA_MAINTAINER_STR)) {
+			    maintainer = tokenizer.nextToken();
+			}
+			else if(command_type_str.equalsIgnoreCase(StaticStrings.COLLECTIONMETADATA_STR)) {
+			    String meta_type_str = tokenizer.nextToken();
+			    String value_str = tokenizer.nextToken();
+			    // check for language
+			    String language_str = StaticStrings.ENGLISH_LANGUAGE_STR; // assume the no-lang ones are english, but we shouldn't really do this.
+			    if(meta_type_str != null && value_str != null) {
+				meta_type_str = meta_type_str.toLowerCase();
+				if(value_str.startsWith(StaticStrings.LBRACKET_CHARACTER) && value_str.endsWith(StaticStrings.RBRACKET_CHARACTER)) {
+				    language_str = value_str.substring(value_str.indexOf(StaticStrings.EQUALS_CHARACTER) + 1, value_str.length() - 1);
+				    language_str = language_str.toLowerCase();
+				    value_str = tokenizer.nextToken();
+				}
+				// now we can work out which coll meta we are dealing with
+				if (meta_type_str.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR)) {
+				    // If this is the first collection title found, then use it, otherwise search for one that more closely matches our choosen interface language
+				    if (name == null || language_str.equals(Dictionary.getLanguage())) {
+					name = Codec.transform(value_str, Codec.GREENSTONE_TO_TEXT);
+				    }
+				}
+				else if (meta_type_str.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR)) {
+				    // Again we are either looking for the first description, then after that a language specific one
+				    if (description == null || language_str.equals(Dictionary.getLanguage())) {
+					description = Codec.transform(value_str, Codec.GREENSTONE_TO_TEXT);
+				    }
+				}
+			    }
+			    language_str = null;
+			    value_str = null;
+			    meta_type_str = null;
+			} else {
+			    // we want to process all the tokens to make sure we get rid of multi line commands before trying to find the next one
+			    while (tokenizer.hasMoreTokens()) {
+			    	tokenizer.nextToken();
+			    }
+
+			} // end of coll meta bit
+			command_type_str = null;
+			tokenizer = null;
+		    } // if command.length > 0
+		} // while
+		command = null;
+		br.close();
+		isr.close();
+		br = null;
+		isr = null;
+	    } // cfg file
+	    ///ystem.err.println("Parsed collect.cfg");
+	    ///ystem.err.println("name		= " + name);
+	    ///ystem.err.println("creator	 = " + creator);
+	    ///ystem.err.println("maintainer  = " + maintainer);
+	    ///ystem.err.println("description = " + description);
+	}
+	catch(Exception error) {
+	    DebugStream.println("Error in CollectionConfiguration.<init>(): " + error);
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+    /** Compare this configuration to another for ordering, which essentially compares two strings for ordering.
+     * @param other the other Object which is presumably another basic collection configuration
+     * @return an integer which is either <0, 0 or >0 if this configuration is naturally less than, equal to or greater than the target object
+     */
+    public int compareTo(Object other) {
+	if(other == null) {
+	    return -1;
+	}
+	return toString().compareTo(other.toString());
+    }
+
+    public boolean equals(Object other) {
+	return (compareTo(other) == 0);
+    }
+
+    /** Retrieve the creators email for this collection.
+     * @return a String
+     */
+    public String getCreator() {
+	return creator;
+    }
+
+    public String getDescription() {
+	return description;
+    }
+
+    public File getFile() {
+	return file;
+    }
+    
+    public String getMaintainer() {
+	return maintainer;
+    }
+
+    public String getName() {
+	return name;
+    }
+    
+    /** Retrieve the short name for this collection which, given this current file is in <col_name>/etc/collect.cfg, is the name of this file's parent file's parent.
+     * @return the short name of this collection as a String
+     */
+    public String getShortName() {
+	return file.getParentFile().getParentFile().getName();
+    }
+    
+    /** set the site for this coll */
+    public void setSite(String site) {
+	this.site = site;
+    }
+	
+    
+    /** Display the title for this collection. */
+    public String toString() {
+	if (this.site == null) {
+	    return getName() + StaticStrings.SPACE_CHARACTER + StaticStrings.OPEN_PARENTHESIS_CHARACTER + getShortName() + StaticStrings.CLOSE_PARENTHESIS_CHARACTER;
+	} else {
+	    return getName() + StaticStrings.SPACE_CHARACTER + StaticStrings.OPEN_PARENTHESIS_CHARACTER + getShortName()+", "+this.site + StaticStrings.CLOSE_PARENTHESIS_CHARACTER;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/Collection.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/Collection.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/Collection.java	(revision 21112)
@@ -0,0 +1,406 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.collection;
+
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.cdm.CollectionMeta;
+import org.greenstone.gatherer.cdm.CollectionMetaManager;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** Collection provides a common collection point for all the information about a certain collection build session. It keeps a record of several other managers that actually handle the content of the collection, such as metadata sets and metadata itself.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3c
+ */
+public class Collection {
+    /** A reference to the Collection Design Manager. */
+    public CollectionDesignManager cdm;
+    /** A reference to the build ScriptOptions. */
+    public ScriptOptions build_options;
+    /** A reference to the import ScriptOptions. */
+    public ScriptOptions import_options;
+    /** true if an error has occurred during construction */
+    public boolean error = false;
+    /** <i>true</i> if the currently loaded collection has been saved since the last significant change, <i>false</i> otherwise. */
+    private boolean saved = false;
+    /*<i>true</i> if the currently loaded collection has had files added since its last build */
+    private boolean filesChanged = false;
+    /*<i>true</i> if the currently loaded collection has had metadata added since its last build */
+    private boolean metadataChanged = false;
+    /** The document around which this collection class is based. */
+    private Document document;
+    /** The file the collection is in (the file may not actually exist, such in the case of a legacy collection)! */
+    private File file;
+    /** The name of the argument element. */
+    static final private String ARGUMENT = "Argument";
+    static final private String BASE_COLLECTION = "base_collection";
+    /** The name of the build element. */
+    static final private String BUILD = "Build";
+    /** The name of the build config element. */
+    static final private String BUILD_CONFIG = "BuildConfig";
+    /** The name of the collection xml template. */
+    static final private String COLLECTION_XML_TEMPLATE = "xml/template.col";
+    /** The name of the import element. */
+    static final private String IMPORT = "Import";
+    /** The name of the imported attribute. */
+    static final private String IMPORTED = "imported";
+
+    /** Constructor. */
+    public Collection(File collection_xml) {
+	this.file = collection_xml;
+	// Try to load this collections details.
+	document = XMLTools.parseXMLFile(collection_xml);
+	// If that fails load the default settings for a collection.
+	if(document == null) {
+	    document = XMLTools.parseXMLFile(COLLECTION_XML_TEMPLATE, true);
+	}
+	if (document == null) {
+	    error = true;
+	    return;
+	}
+	// Point the Configuration class at our gatherer config arguments.
+	Configuration.setCollectionConfiguration(document);
+	if (Gatherer.g_man != null) {
+	    Gatherer.g_man.updateUI();
+	}
+	// Finally create all of the child managers that are directly dependant on a collection
+	build_options = new ScriptOptions(getBuildValues(), "buildcol.pl", true);
+	import_options = new ScriptOptions(getImportValues(), "import.pl", true);
+    }
+    
+    /** Destructor.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
+     */
+    public void destroy() {
+	cdm.destroy();
+	Configuration.setCollectionConfiguration(null);
+	if (Gatherer.g_man != null) {
+	    Gatherer.g_man.updateUI();
+	}
+	cdm = null;
+	document = null;
+    }
+    
+    /** Determine the path to the base collection.
+     * @return the path as a String
+     */
+    public String getBaseCollection() {
+	return getString(BASE_COLLECTION);
+    }
+    
+    /** Determine the number of documents and folders in this collection. */
+    public int getCount() {
+	return getCount((TreeNode)Gatherer.c_man.getCollectionTreeModel().getRoot(), true, true);
+    }
+    
+    /** Calculates the number of documents in this collection. */
+    public int getDocumentCount() {
+	return getCount((TreeNode)Gatherer.c_man.getCollectionTreeModel().getRoot(), false, true);
+    }
+    
+    /** Retrieve the description of this collection.
+     * @return a String
+     */
+    public String getDescription() {
+	if(cdm == null) {
+	    return StaticStrings.EMPTY_STR;
+	}
+	CollectionMeta collection_extra_collectionmeta = cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);
+	return collection_extra_collectionmeta.getValue(CollectionMeta.TEXT);
+    }
+    
+    /** Retrieve the authors email for this collection.
+     * @return The email as a <strong>String</strong>.
+     */
+    public String getEmail() {
+	if(cdm == null) {
+	    return StaticStrings.EMPTY_STR;
+	}
+	CollectionMeta creator_collectionmeta = new CollectionMeta(CollectionDesignManager.collect_config.getCreator());
+	return creator_collectionmeta.getValue(CollectionMeta.TEXT);
+    }
+
+    /** Retrieve the short name for this collection.
+     * @return The name as a <strong>String</strong>.
+     */
+    public String getName() {
+	return file.getParentFile().getName();
+    }
+    
+    /** Determine if this collection has been saved since the last major change.
+     * @return <i>true</i> if it has been saved recently, <i>false</i> otherwise.
+     */
+    public boolean getSaved() {
+	return saved;
+    }
+
+    /** Determine if this collection has had its collection files changed since the last build.
+     * @return <i>true</i> if they have been saved, <i>false</i> otherwise.
+     */
+    public boolean getFilesChanged() {
+	return filesChanged;
+    }
+    
+    /** Set the flag that marks weather or not the files has been changed since the last build.
+	@param changed if files has been added/removed, as a <strong>boolean</strong>
+    */
+    public void setFilesChanged(boolean changed) {
+	filesChanged = changed;
+    }
+
+    /** Determine if this collection has had metadata changed since the last build.
+     * @return <i>true</i> if it has been changed, <i>false</i> otherwise.
+     */
+    public boolean getMetadataChanged() {
+	return metadataChanged;
+    }
+    
+    /** Set the flag that marks weather or not the metadata has been changed since the last build.
+	@param changed if metadata has been added/changed/removed, as a <strong>boolean</strong>
+    */
+    public void setMetadataChanged(boolean changed) {
+	metadataChanged = changed;
+    }
+
+    /** Retrieve the title of this collection.
+     * @return The title as a <strong>String</strong>.
+     */
+    public String getTitle() {
+	if(cdm == null) {
+	    return StaticStrings.EMPTY_STR;
+	}
+	CollectionMeta collection_name_collectionmeta = cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR);
+	return collection_name_collectionmeta.getValue(CollectionMeta.TEXT);
+    }
+
+    /** Save this xml document to the given file. */
+    public void save() {
+	XMLTools.writeXMLFile(file, document);
+	saved = true;
+    }
+
+    public void setBaseCollection(String base_collection) {
+	set(BASE_COLLECTION, base_collection);
+    }
+    
+    /** Set the value of imported to the given value.
+     * @param value The new value for imported, <i>true</i> if the collection has been imported successfully, <i>false</i> otherwise.
+     */
+    public void setImported(boolean value) {
+	set(IMPORTED, value);
+	saved = false;
+    }
+
+    /** Set the value of saved to the given value.
+     * @param value The new value for saved, <i>true</i> if the collection has been saved recently, <i>false</i> otherwise.
+     */
+    public void setSaved(boolean value) {
+	saved = value;
+    }
+
+    /** Set the value of title to the given value.
+     * @param title The new <strong>String</strong> title.
+     */
+    public void setTitle(String title) {
+	if(cdm != null) {
+	    CollectionMeta collection_name_collectionmeta = cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR);
+	    collection_name_collectionmeta.setValue(title);
+	}
+    }
+
+    /** Method called to return a textual representation of a class, which in this case is the collections title.
+     * @return A <strong>String</strong> containing the collections title.
+     */
+    public String toString() {
+	return getTitle();
+    }
+    
+    /** Get the value of a collection argument. */
+    private boolean get(String name) {
+	boolean result = false;
+	try {
+	    Element document_element = document.getDocumentElement();
+	    NodeList arguments = document_element.getElementsByTagName(ARGUMENT);
+	    boolean found = false;
+	    for(int i = 0; !found && i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		if(argument_element.getParentNode() == document_element) {
+		    if(argument_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equalsIgnoreCase(name)) {
+			String value = XMLTools.getValue(argument_element);
+			if(value.equalsIgnoreCase(StaticStrings.TRUE_STR)) {
+			    result = true;
+			}
+			found = true;
+			value = null;
+		    }
+		}
+		argument_element = null;
+	    }
+	    arguments = null;
+	    document_element = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return result;
+    }
+    
+    /** Get the value of a collection argument. */
+    private String getString(String name) {
+	String result = "";
+	try {
+	    Element document_element = document.getDocumentElement();
+	    NodeList arguments = document_element.getElementsByTagName(ARGUMENT);
+	    boolean found = false;
+	    for(int i = 0; !found && i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		if(argument_element.getParentNode() == document_element) {
+		    if(argument_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equalsIgnoreCase(name)) {
+			result = XMLTools.getValue(argument_element);
+			found = true;
+		    }
+		}
+		argument_element = null;
+	    }
+	    arguments = null;
+	    document_element = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return result;
+    }
+    
+    /** Method to retrieve the current build options associated with this Collection. */
+    private Element getBuildValues() {
+	Element build_values_element = null;
+	try {
+	    Element document_element = document.getDocumentElement();
+	    Element build_config_element = (Element) XMLTools.getNodeFromNamed(document_element, BUILD_CONFIG);
+	    build_values_element = (Element) XMLTools.getNodeFromNamed(build_config_element, BUILD);
+	    build_config_element = null;
+	    document_element = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return build_values_element;
+    }
+    
+    /** Count either documents or folders, depending on the state of the given boolean. */
+    private int getCount(TreeNode node, boolean count_folders, boolean count_files) {
+	int count = 0;
+	File file = ((CollectionTreeNode)node).getFile();
+	if(file.isFile() && count_files) {
+	    count++;
+	}
+	else if(file.isDirectory() && count_folders) {
+	    count++;
+	}
+	for(int i = 0; !file.getName().equals("CVS") && i < node.getChildCount(); i++) {
+	    count = count + getCount(node.getChildAt(i), count_folders, count_files);
+	}
+	return count;
+    }
+    
+    /** Method to retrieve the current import options associated with this Collection. */
+    public Element getImportValues() {
+	Element import_values_element = null;
+	try {
+	    Element document_element = document.getDocumentElement();
+	    Element build_config_element = (Element) XMLTools.getNodeFromNamed(document_element, BUILD_CONFIG);
+	    import_values_element = (Element) XMLTools.getNodeFromNamed(build_config_element, IMPORT);
+	    build_config_element = null;
+	    document_element = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return import_values_element;
+    }
+    
+    /** Set the value of a collection argument. */
+    private void set(String name, boolean value) {
+	set(name, (value ? StaticStrings.TRUE_STR : StaticStrings.FALSE_STR));
+    }
+    
+    private void set(String name, String value) {
+	try {
+	    Element document_element = document.getDocumentElement();
+	    NodeList arguments = document_element.getElementsByTagName(ARGUMENT);
+	    boolean found = false;
+	    for(int i = 0; !found && i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		if(argument_element.getParentNode() == document_element) {
+		    if(argument_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equalsIgnoreCase(name)) {
+			// Strip any current value nodes.
+			while(argument_element.hasChildNodes()) {
+			    argument_element.removeChild(argument_element.getFirstChild());
+			}
+			// Append new value
+			argument_element.appendChild(document.createTextNode(value));
+			found = true;
+		    }
+		}
+		argument_element = null;
+	    }
+	    // Append it
+	    if(!found) {
+		Element argument_element = document.createElement(ARGUMENT);
+		argument_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, name);
+		argument_element.appendChild(document.createTextNode(value));
+		document_element.appendChild(argument_element);
+		argument_element = null;
+	    }
+	    arguments = null;
+	    document_element = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionContentsChangedListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionContentsChangedListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionContentsChangedListener.java	(revision 21112)
@@ -0,0 +1,37 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.collection;
+
+
+import java.io.File;
+
+
+public interface CollectionContentsChangedListener
+{
+    /** Notification that a new file has been added to the collection. */
+    public void fileAddedToCollection(File file);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionManager.java	(revision 21112)
@@ -0,0 +1,1554 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.collection;
+
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.filechooser.FileSystemView;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.ServletConfiguration;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.cdm.CollectionMeta;
+import org.greenstone.gatherer.cdm.CollectionMetaManager;
+import org.greenstone.gatherer.cdm.CommandTokenizer;
+import org.greenstone.gatherer.gui.ExternalCollectionPrompt;
+import org.greenstone.gatherer.gui.LockFileDialog;
+import org.greenstone.gatherer.gui.ModalProgressPopup;
+import org.greenstone.gatherer.gui.NewCollectionMetadataPrompt;
+import org.greenstone.gatherer.gui.WarningDialog;
+import org.greenstone.gatherer.metadata.DocXMLFileManager;
+import org.greenstone.gatherer.metadata.MetadataChangedListener;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+import org.greenstone.gatherer.metadata.ProfileXMLFileManager;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.shell.GShellEvent;
+import org.greenstone.gatherer.shell.GShellListener;
+import org.greenstone.gatherer.shell.GShellProgressMonitor;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class manages many aspects of the collection, from its creation via scripts, data access via methods and its importing and building into the final collection. It is also resposible for firing appropriate event when significant changes have occured within the collection, and for creating a new metadata set manager as necessary.
+ * @author John Thompson
+ * @version 2.3
+ */
+public class CollectionManager
+    implements GShellListener, MetadataChangedListener
+{
+    /** Are we currently in the process of building? */
+    private boolean building = false;
+    /** Are we currently in the process of importing? */
+    private boolean importing = false;
+    /** The objects listening for CollectionContentsChanged events. */
+    private ArrayList collection_contents_changed_listeners = new ArrayList();
+    /** The collection this manager is managing! */
+    static private Collection collection = null;
+    /** The collection_model. */
+    private CollectionTreeModel collection_model = null;
+    /** An inner class listener responsible for noting tree changes and resetting saved when they occur. */
+    private FMTreeModelListener fm_tree_model_listener = null;
+    /** The monitor resposible for parsing the build process. */
+    private GShellProgressMonitor build_monitor = null;
+    /** The monitor resposible for parsing the import process. */
+    private GShellProgressMonitor import_monitor = null;
+
+    /** Whether we have downloaded the collection configurations in this session, when using a remote Greenstone. */
+    static public boolean downloaded_collection_configurations = false;
+
+    /** The name of the standard lock file. */
+    static final public String LOCK_FILE = "gli.lck";
+
+    /** Used to indicate the source of the message is the file collection methods. */
+    static final public int COLLECT   = 3;
+    /** Used to indicate the source of the message is the building methods. */
+    static final public int BUILDING  = 5;
+
+    /** Constructor. */
+    public CollectionManager() {
+	// Initialisation.
+	this.building = false;
+	this.importing = false;
+	this.collection = null;
+
+	MetadataXMLFileManager.addMetadataChangedListener(this);
+
+	// If using a remote Greenstone server, delete the local collect directory because it will be out of date
+	if (Gatherer.isGsdlRemote) {
+	    System.err.println("Deleting user's local collect directory...");
+	    Utility.delete(new File(Gatherer.getCollectDirectoryPath()));
+	    System.err.println("Done.");
+	    new File(Gatherer.getCollectDirectoryPath()).mkdirs();
+	}
+    }
+
+
+    public void addCollectionContentsChangedListener(CollectionContentsChangedListener listener)
+    {
+	collection_contents_changed_listeners.add(listener);
+    }
+
+
+    /** This method calls the builcol.pl scripts via a GShell so as to not lock up the processor.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.Collection
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.shell.GShell
+     * @see org.greenstone.gatherer.shell.GShellListener
+     * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void buildCollection(boolean incremental_build)
+    {
+	System.err.println("In CollectionManager.buildCollection(), incremental_build: " + incremental_build);
+	System.err.println("Is event dispatch thread: " + SwingUtilities.isEventDispatchThread());
+	building = true;
+
+	// Generate the buildcol.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if ((Utility.isWindows()) && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "buildcol.pl");
+	command_parts_list.add("-gli");
+	command_parts_list.add("-language");
+	command_parts_list.add(Configuration.getLanguage());
+	command_parts_list.add("-collectdir");
+	command_parts_list.add(getCollectDirectory());
+
+	// If the user hasn't manually specified "-keepold" or "-removeold" then pick one based on incremental_build
+	if (!collection.build_options.getValueEnabled("keepold") && !collection.build_options.getValueEnabled("removeold")) {
+	    command_parts_list.add(incremental_build ? "-keepold" : "-removeold");
+	}
+
+	String[] build_options = collection.build_options.getValues();
+	for (int i = 0; i < build_options.length; i++) {
+	    command_parts_list.add(build_options[i]);
+	}
+
+	command_parts_list.add(collection.getName());
+
+	// Run the buildcol.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell shell = new GShell(command_parts, GShell.BUILD, BUILDING, this, build_monitor, GShell.GSHELL_BUILD);
+	shell.addGShellListener(Gatherer.g_man.create_pane);
+	shell.start();
+	System.err.println("CollectionManager.buildCollection().return");
+    }
+
+
+    /** Used to determine whether the currently active collection has been built.
+     * @return A boolean indicating the built status of the collection.
+     */
+    public boolean built() {
+	if(collection != null) {
+	    // Determine if the collection has been built by looking for the build.cfg file
+	    File build_cfg_file = new File(getCollectionIndexDirectoryPath() + "build.cfg");
+	    return build_cfg_file.exists();
+	}
+	return false;
+    }
+
+    /** a test method to see if we can delete a directory/file - returns false is the file or any of the contents of a directory cannot be deleted */
+    public boolean canDelete(File file) {
+	if (!file.isDirectory()) {
+	    return file.canWrite();
+	}
+	File [] file_list = file.listFiles();
+	for (int i=0; i<file_list.length; i++) {
+	    if (!canDelete(file_list[i])) {
+		return false;
+	    }
+	}
+	return true;
+    }
+    /** Called to close the current collection and remove its lock file.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.Collection
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void closeCollection() {
+	DebugStream.println("Close collection: " + collection.getName());
+
+	// Remove the lock on this file, then remove the collection.
+	File lock_file = new File(getCollectionDirectoryPath() + LOCK_FILE);
+	lock_file.delete();
+	if (lock_file.exists()) {
+	    System.err.println("Warning: Lockfile was not successfully deleted.");
+	}
+
+	// Remove the lock file on the server
+	if (Gatherer.isGsdlRemote) {
+	    RemoteGreenstoneServer.deleteCollectionFile(collection.getName(), lock_file);
+	}
+
+	MetadataSetManager.clearMetadataSets();
+	MetadataXMLFileManager.clearMetadataXMLFiles();
+	DocXMLFileManager.clearDocXMLFiles();
+	ProfileXMLFileManager.clearProfileXMLFile();
+
+	collection = null;
+	collection_model = null;
+	Configuration.setCollectionConfiguration(null);
+	Gatherer.refresh(Gatherer.COLLECTION_CLOSED);
+	if (Gatherer.g_man != null) {
+	    Gatherer.g_man.updateUI();  // !!! Necessary?
+	}
+    }
+
+    /** Method that is called whenever something has changed in the configuration of this collection. */
+    public void configurationChanged() {
+	if(collection != null) {
+	    collection.setSaved(false);
+	    collection.cdm.update_collect_cfg_required = true;
+	}
+    }
+
+    public void convertToGS3Collection() {
+	// Generate the convert_coll_from_gs2.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if ((Utility.isWindows()) && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(Configuration.getGS3ScriptPath() + "convert_coll_from_gs2.pl");
+	command_parts_list.add("-collectdir");
+	command_parts_list.add(getCollectDirectory());
+    	command_parts_list.add(collection.getName());
+
+	// Run the convert_coll_from_gs2.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell process = new GShell(command_parts, GShell.CONVERT, COLLECT, this, null, GShell.GSHELL_CONVERT);
+	process.addGShellListener(this);
+	process.run(); // Don't bother threading this... yet
+       
+    }
+
+    /** When basing a new collection on an existing one, we need to copy 
+     *	over some extra directories: images and macros  
+     */
+    private boolean copyExtraBaseCollStuff(File new_coll_dir, File base_coll_dir) {
+	if (!new_coll_dir.isDirectory() || !base_coll_dir.isDirectory()) {
+	    return false;
+	}
+	DebugStream.println("Copying images and macros dirs from the base collection");
+	try {
+	    // do the images dir
+	    File base_coll_images = new File(base_coll_dir, "images");
+	    if (base_coll_images.isDirectory()) {
+		// copy all the images over
+		File new_coll_images = new File(new_coll_dir, "images");
+		new_coll_images.mkdirs();
+		
+		// copy the contents over
+		Gatherer.f_man.getQueue().copyDirectoryContents(base_coll_images, new_coll_images, null);
+	    }
+	}
+	catch (Exception e) {
+	    DebugStream.println("Couldn't copy over the images dir from the base collection: "+e.toString());
+	}
+	try {
+	    // do the macros dir
+	    File base_coll_macros = new File(base_coll_dir, Utility.MACROS_DIR);
+	    if (base_coll_macros.isDirectory()) {
+		// copy all the macros over
+		File new_coll_macros = new File(new_coll_dir, Utility.MACROS_DIR);
+		new_coll_macros.mkdirs();
+		
+		// copy the contents over
+		Gatherer.f_man.getQueue().copyDirectoryContents(base_coll_macros, new_coll_macros, null);
+	    } 
+	}
+	catch (Exception e) {
+	    DebugStream.println("Couldn't copy over the macros dir from the base collection: "+e.toString());
+	}
+	return true;
+    }
+
+    /** Used to set the current collection to the given collection. Note that this call should -always- be proceeded by a ready call, and if the collection is ready and the saved flag is unset then the user should be prompted to save. Also note that this method creates yet another GShell to run buildcol.pl.
+     * @param description a description of the collection as a String
+     * @param email the email address of the author/maintainer as a String
+     * @param name the short name of the collection, which will subsequently be used to refer to this particular collection, as a String
+     * @param title the longer title of the collection as a String
+     * @param base_collection_directory if the user has chosen to base their new collection on an existing one, this is the directory where this base collection can be found, as a File, otherwise its null
+     * @param metadata_sets if the user has decided to select several metadata sets with which to initially populate the GLI then this is an ArrayList of metadata set file names, otherwise its null
+     */
+    public void createCollection(String description, String email, String name, String title, File base_collection_directory, ArrayList metadata_sets)
+    {
+	try {
+	    // first make sure that the collect directory exists
+	    File collect_dir = new File(getCollectDirectory());
+	    if (!collect_dir.exists()) {
+		collect_dir.mkdirs();
+	    }
+
+	    // Create a progress monitor
+	    ProgressMonitor progress = new ProgressMonitor(Gatherer.g_man, Dictionary.get("CollectionManager.Creating_New"), "mkcol.pl", 0, 4);
+
+	    // Create the new collection
+	    makeCollection(name, email);
+	    progress.setProgress(1);
+
+	    // Check that the collection has been created successfully
+	    String collection_directory_path = getCollectionDirectoryPath(name);
+	    if (!new File(collection_directory_path).exists()) {
+		// If there is no collection directory then the creation was unsuccessful, or cancelled
+		System.err.println("No collection directory...");
+		progress.close();
+		return;
+	    }
+
+	    // Check for the existence of the collect.cfg file
+	    File collect_cfg_file = new File(collection_directory_path + "etc" + File.separator + "collect.cfg");
+	    if (!collect_cfg_file.exists()) {
+		System.err.println("Error: no collect.cfg file has been created!");
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Create_Collection_With_Reason", Dictionary.get("CollectionManager.No_Config_File")), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE); 
+		progress.close();
+		return;
+	    }
+ 	    
+	    // ACTIVE_DIR/log/
+	    File log_dir = new File(collection_directory_path + "log");
+	    log_dir.mkdirs();
+	    progress.setNote(Dictionary.get("CollectionManager.Log_Created"));
+
+	    // Make sure an import folder exists
+	    File import_directory = new File(collection_directory_path + "import");
+	    if (!import_directory.exists()) {
+		import_directory.mkdirs();
+	    }
+
+	    progress.setProgress(2);
+
+	    // Now create the collection object around the directory.
+	    collection = new Collection(new File(collection_directory_path, name + ".col"));
+
+	    // "-removeold" is on by default for import.pl
+	    collection.import_options.setValue("removeold", true, null);
+
+	    MetadataSetManager.clearMetadataSets();
+	    MetadataXMLFileManager.clearMetadataXMLFiles();
+	    DocXMLFileManager.clearDocXMLFiles();
+
+	    // Import default metadata sets, if any
+	    for (int i = 0; metadata_sets != null && i < metadata_sets.size(); i++) {
+		importMetadataSet((MetadataSet) metadata_sets.get(i));
+	    }
+
+	    ProfileXMLFileManager.loadProfileXMLFile(new File(collection_directory_path + "metadata"));
+
+	    // Before creating the CollectionDesignManager check if we are basing it upon some other collection
+	    if (base_collection_directory != null) {
+		DebugStream.println("Basing new collection on existing one: " + base_collection_directory);
+
+		// If we're using a remote Greenstone server, download the collection shell to get the files needed
+		if (Gatherer.isGsdlRemote) {
+		    String base_collection_name = base_collection_directory.getName();
+		    RemoteGreenstoneServer.downloadCollection(base_collection_name);
+		}
+
+		collection.setBaseCollection(base_collection_directory.getAbsolutePath());
+		// copy over other needed directories
+		copyExtraBaseCollStuff(new File(collection_directory_path), base_collection_directory);
+
+		// Try to import any existing metadata sets for this collection
+		// Look in base_collection_directory/metadata and import any metadata sets found.
+		File base_metadata_directory = new File(base_collection_directory, "metadata");
+		ArrayList base_metadata_sets = MetadataSetManager.listMetadataSets(base_metadata_directory);
+		if (base_metadata_sets != null) {
+		    for (int i = 0; i < base_metadata_sets.size(); i++) {
+			importMetadataSet((MetadataSet) base_metadata_sets.get(i));
+		    }
+		}
+		else {
+		    DebugStream.println("This base collection has no metadata directory.");
+		}
+
+		// If no sets were imported...
+		if (MetadataSetManager.getMetadataSets().size() == 0) {
+		    // Prompt the user so that they can choose at least one initial metadata set. We're sneaky here and just create a ncm_prompt
+		    DebugStream.println("This collection has no metadata sets. Present the user with the metadata set selection prompt.");
+		    NewCollectionMetadataPrompt ncm_prompt = new NewCollectionMetadataPrompt();
+		    // If cancelled then they really do mean to start a collection with no metadata sets.
+		    if (!ncm_prompt.isCancelled()) {
+			ArrayList initial_sets = ncm_prompt.getSets();
+			for (int i = 0; initial_sets != null && i < initial_sets.size(); i++) {
+			    importMetadataSet((MetadataSet) initial_sets.get(i));
+			}
+		    }
+		    ncm_prompt.dispose();
+		    ncm_prompt = null;
+		}
+
+		// Now we update our collect.cfg
+		DebugStream.println("Copy and update collect.cfg from base collection.");
+		updateCollectionCFG(new File(base_collection_directory, Utility.CONFIG_FILE), new File(collection_directory_path, Utility.CONFIG_FILE), description, email, title);
+	    }
+
+	    // Always import the extracted metadata set if we didn't already
+	    if (MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE) == null) {
+		File extracted_metadata_set_file = new File(Gatherer.getGLIMetadataDirectoryPath() + MetadataSetManager.EXTRACTED_METADATA_NAMESPACE + StaticStrings.METADATA_SET_EXTENSION);
+		importMetadataSet(new MetadataSet(extracted_metadata_set_file));
+	    }
+
+	    collection.cdm = new CollectionDesignManager(new File(getCollectionConfigFilePath()));
+
+	    // We always set title and description here rather than calling mkcol.pl with Unicode arguments
+	    CollectionMeta collection_name_collectionmeta = collection.cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR);
+	    collection_name_collectionmeta.setValue(title);
+	    CollectionMeta collection_extra_collectionmeta = collection.cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR);
+	    collection_extra_collectionmeta.setValue(description);
+
+	    // Now that we have a CDM, update several settings, such as if we created this collection by basing it on another, set it as public automatically
+	    if (base_collection_directory != null) {
+		// Update the creator and maintainer
+		CollectionMeta creator_collectionmeta = new CollectionMeta(collection.cdm.collect_config.getCreator());
+		creator_collectionmeta.setValue(email);
+		creator_collectionmeta = null;
+		CollectionMeta maintainer_collectionmeta = new CollectionMeta(collection.cdm.collect_config.getMaintainer());
+		maintainer_collectionmeta.setValue(email);
+		maintainer_collectionmeta = null;
+
+		// All collections based on others are automatically public
+		CollectionMeta public_collectionmeta = new CollectionMeta(collection.cdm.collect_config.getPublic());
+		public_collectionmeta.setValue(StaticStrings.TRUE_STR);
+		public_collectionmeta = null;
+
+		// Finally reset the icons
+		CollectionMeta icon_collection_collectionmeta = collection.cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR);
+		icon_collection_collectionmeta.setValue(StaticStrings.EMPTY_STR);
+		icon_collection_collectionmeta = null;
+		CollectionMeta icon_collection_small_collectionmeta = collection.cdm.collectionmeta_manager.getMetadatum(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR);
+		icon_collection_small_collectionmeta.setValue(StaticStrings.EMPTY_STR);
+		icon_collection_small_collectionmeta = null;
+	    }
+
+	    collection.cdm.save(true);
+	    progress.setProgress(3);
+
+	    // Create a lock file
+	    createLockFile(new File(collection_directory_path, LOCK_FILE));
+
+	    progress.setProgress(4);
+	    progress.setNote(Dictionary.get("CollectionManager.Session_Ready", name));
+	    progress.close();
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+
+    private void createLockFile(File lock_file)
+    {
+	try {
+	    Document default_lockfile = XMLTools.parseXMLFile("xml/" + LOCK_FILE, true);
+	    String user_name = System.getProperty("user.name");
+	    Element person_element = (Element) XMLTools.getNodeFromNamed(default_lockfile.getDocumentElement(), "User");
+	    person_element.appendChild(default_lockfile.createTextNode(user_name));
+	    person_element = null;
+	    user_name = null;
+	    String machine_name = Utility.getMachineName();
+	    Element machine_element = (Element) XMLTools.getNodeFromNamed(default_lockfile.getDocumentElement(), "Machine");
+	    machine_element.appendChild(default_lockfile.createTextNode(machine_name));
+	    machine_element = null;
+	    machine_name = null;
+	    String date_time = Utility.getDateString();
+	    Element date_element = (Element) XMLTools.getNodeFromNamed(default_lockfile.getDocumentElement(), "Date");
+	    date_element.appendChild(default_lockfile.createTextNode(date_time));
+	    date_element = null;
+	    date_time = null;
+	    XMLTools.writeXMLFile(lock_file, default_lockfile);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+
+    public boolean deleteCollection(String collection_name)
+    {
+	return Utility.delete(new File(getCollectionDirectoryPath(collection_name)));
+    }
+
+
+    public void fireFileAddedToCollection(File file)
+    {
+	// Send the event off to all the CollectionContentsChangedListeners
+	for (int i = 0; i < collection_contents_changed_listeners.size(); i++) {
+	    ((CollectionContentsChangedListener) collection_contents_changed_listeners.get(i)).fileAddedToCollection(file);
+	}
+    }
+
+
+    /** Retrieve the current collection.
+     * @return The <strong>Collection</strong> itself.
+     */
+    public Collection getCollection() {
+	return collection;
+    }
+
+
+    static public String getCollectionDirectoryPath()
+    {
+	return Gatherer.getCollectDirectoryPath() + collection.getName() + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;"
+     * @return A <strong>String</strong> containing the directory name.
+     */
+    static public String getCollectionDirectoryPath(String collection_name)
+    {
+	return Gatherer.getCollectDirectoryPath() + collection_name + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's .col file, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/&lt;col_name&gt;.col"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionFilePath()
+    {
+	return getCollectionDirectoryPath() + collection.getName() + ".col";
+    }
+
+
+    /** Constructs the absolute filename of the collection's archives directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/archives/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionArchivesDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "archives" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's building directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/building/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionBuildingDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "building" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's collect.cfg file, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/etc/collect.cfg"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionConfigFilePath()
+    {
+	return getCollectionEtcDirectoryPath() + "collect.cfg";
+    }
+
+
+    /** Constructs the absolute filename of the collection's etc directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/etc/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionEtcDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "etc" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's images directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/images/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionImagesDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "images" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's import directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/import/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionImportDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "import" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's index directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/index/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionIndexDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "index" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's log directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/log/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionLogDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "log" + File.separator;
+    }
+
+
+    /** Constructs the absolute filename of the collection's metadata directory, which should resemble "$GSDLHOME/collect/&lt;col_name&gt;/metadata/"
+     * @return A <strong>String</strong> containing the filename.
+     */
+    public String getCollectionMetadataDirectoryPath()
+    {
+	return getCollectionDirectoryPath() + "metadata" + File.separator;
+    }
+
+
+    /** Retrieve the tree model associated with the current collection. */
+    public CollectionTreeModel getCollectionTreeModel()
+    {
+	if (collection_model == null && collection != null) {
+	    // Use the import directory to generate a new CollectionTreeModel
+	    collection_model = new CollectionTreeModel(new CollectionTreeNode(new File(getCollectionImportDirectoryPath())));
+	    // Ensure that the manager is a change listener for the tree.
+	    if (fm_tree_model_listener == null) {
+		fm_tree_model_listener = new FMTreeModelListener();
+	    }
+	    collection_model.addTreeModelListener(fm_tree_model_listener);
+	}
+	return collection_model;
+    }
+
+
+    /** This method when called, creates a new GShell in order to run the import.pl script.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.shell.GShell
+     * @see org.greenstone.gatherer.shell.GShellListener
+     * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void importCollection() {
+	importing = true;
+
+	if (!saved()) {
+	    DebugStream.println("CollectionManager.importCollection().forcesave");
+	    import_monitor.saving();
+	    saveCollection();
+	}
+
+	System.err.println("CollectionManager.importCollection()");
+	System.err.println("Is event dispatch thread: " + SwingUtilities.isEventDispatchThread());
+	//check that we can remove the old index before starting import
+	File index_dir = new File(getCollectionIndexDirectoryPath());
+	if (index_dir.exists()) {
+	    DebugStream.println("Old Index = " + index_dir.getAbsolutePath()+", testing for deletability");
+	    if (!canDelete(index_dir)) {
+		// tell the user
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Delete_Index"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE); 
+		// tell the gui manager
+		// a message for the building log 
+		GShellEvent event = new GShellEvent(this, 0, GShell.IMPORT, Dictionary.get("CollectionManager.Cannot_Delete_Index_Log"), GShell.ERROR);
+		Gatherer.g_man.create_pane.message(event);
+		event = new GShellEvent(this, 0, GShell.IMPORT, "", GShell.ERROR);
+		Gatherer.g_man.create_pane.processComplete(event);
+		importing = false;
+		return;
+	    }
+	}
+
+	// Generate the import.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if ((Utility.isWindows()) && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "import.pl");
+	command_parts_list.add("-gli");
+	command_parts_list.add("-language");
+	command_parts_list.add(Configuration.getLanguage());
+	command_parts_list.add("-collectdir");
+	command_parts_list.add(getCollectDirectory());
+
+	String[] import_options = collection.import_options.getValues();
+	for (int i = 0; i < import_options.length; i++) {
+	    command_parts_list.add(import_options[i]);
+	}
+
+	command_parts_list.add(collection.getName());
+
+	// Run the import.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell shell = new GShell(command_parts, GShell.IMPORT, BUILDING, this, import_monitor, GShell.GSHELL_IMPORT);
+	shell.addGShellListener(Gatherer.g_man.create_pane);
+	shell.start();
+	System.err.println("CollectionManager.importCollection().return");
+
+	importing = false;
+    }
+
+
+    public void importMetadataSet(MetadataSet external_metadata_set)
+    {
+	// Copy the .mds file into the collection's "metadata" folder...
+	File external_metadata_set_file = external_metadata_set.getMetadataSetFile();
+
+	// ...but not if it is the redundant "hidden.mds" file
+	if (external_metadata_set_file.getName().equals("hidden.mds")) {
+	    return;
+	}
+
+	// ...and only if it doesn't already exist
+	File metadata_set_file = new File(getCollectionMetadataDirectoryPath(), external_metadata_set_file.getName());
+	if (!metadata_set_file.exists()) {
+	    try {
+		Gatherer.f_man.getQueue().copyFile(external_metadata_set_file, metadata_set_file, null);
+
+		// If we're using a remote Greenstone server, upload the metadata file
+		if (Gatherer.isGsdlRemote) {
+		    RemoteGreenstoneServer.uploadCollectionFile(collection.getName(), metadata_set_file);
+		}
+	    }
+	    catch (Exception ex) {
+		ex.printStackTrace();
+	    }
+
+	    // Load it into the MetadataSetManager
+	    MetadataSetManager.loadMetadataSet(metadata_set_file);
+	}
+    }
+
+
+    /** Determine if we are currently in the middle of importing (and thus, in this case, we can't allow the log writer to exit). Boy was this a mission to track down. The cascade of crap rolls out something like this: Joe Schmo clicks 'Build Collection', which calls the importCollection() method above, which in turn saves the collection with a saveTask, which fires a collectionChanged message once its finished, which drives the list of logs shown on the create pane to update, which fires a itemChanged() event to the OptionsPane who dutifully tells the current log writer thread to finish up writing (all zero lines its been asked to write) and then die. Wereapon Joe Schmo gets a pretty log to look at, but it isn't actually being written to file so the next time he tries to view it faeces hits the air motion cooling device. Joy.
+     * @return true if the gli is currently importing
+     */
+    public boolean isImporting() {
+	return importing;
+    }
+
+
+    public void loadCollection(String collection_file_path)
+    {
+	// Display a modal progress popup to indicate that the collection is being loaded
+	ModalProgressPopup load_collection_progress_popup = new ModalProgressPopup(Dictionary.get("CollectionManager.Loading_Collection"), Dictionary.get("CollectionManager.Loading_Collection_Please_Wait"));
+	load_collection_progress_popup.display();
+
+	// Load the collection on a separate thread so the progress bar updates correctly
+	(new LoadCollectionTask(collection_file_path, load_collection_progress_popup)).start();
+    }
+
+
+    private class LoadCollectionTask
+	extends Thread
+    {
+	private String collection_file_path = null;
+	private ModalProgressPopup load_collection_progress_popup = null;
+
+	public LoadCollectionTask(String collection_file_path, ModalProgressPopup load_collection_progress_popup)
+	{
+	    this.collection_file_path = collection_file_path;
+	    this.load_collection_progress_popup = load_collection_progress_popup;
+	}
+
+	public void run()
+	{
+	    loadCollectionInternal(collection_file_path);
+	    load_collection_progress_popup.close();
+	}
+    }
+
+
+    /** Attempts to load the given collection. Currently uses simple serialization of the collection class.
+     * @param location The path to the collection as a <strong>String</strong>.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.Collection
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void loadCollectionInternal(String location)
+    {
+	DebugStream.println("Loading collection " + location + "...");
+
+	if (Gatherer.isGsdlRemote) {
+	    String collection_name = location.substring(location.lastIndexOf(File.separator) + 1, location.length() - ".col".length());
+	    if (RemoteGreenstoneServer.downloadCollection(collection_name).equals("")) {
+		return;
+	    }
+	}
+
+	boolean non_gli_collection = false;
+
+	// Check we have actually been given a .col file.
+	if (!location.endsWith(".col")) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Not_Col_File", location), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    DebugStream.println("CollectionManager.loadCollection: Haven't been given a .col file.");
+	    return;
+	}
+
+	// Check that the collection configuration file is available
+	File collection_file = new File(location);
+
+	// Ensure that the directory exists
+	File collection_directory = collection_file.getParentFile();
+	if (collection_directory == null || !collection_directory.exists()) {
+	    // We can't open this
+	    System.err.println("CollectionManager.loadCollection: No collection directory.");
+	    return;
+	}
+
+	File collection_config_file = new File(collection_directory, Utility.CONFIG_FILE);
+	if (!collection_config_file.exists()) {
+	    System.err.println("CollectionManager.loadCollection: No config file.");
+	    collection_directory = null;
+	    collection_config_file = null;
+	    return;
+	}
+
+	// Ensure that an import directory exists for this collection
+	File collection_import_directory = new File(collection_directory, Utility.IMPORT_DIR);
+	if (!collection_import_directory.exists()) {
+	    collection_import_directory.mkdir();
+	}
+
+	// Special case of a user trying to open an old greenstone collection.
+	File collection_metadata_directory = new File(collection_directory, Utility.META_DIR);
+	if (!collection_metadata_directory.exists()) {
+	    DebugStream.println("Loading non-gatherer collection...");
+	    non_gli_collection = true;
+	}
+
+	// Now determine if a lock already exists on this collection.
+	String name = collection_directory.getName();
+	File lock_file = new File(collection_file.getParentFile(), LOCK_FILE);
+	if (lock_file.exists()) {
+	    LockFileDialog dialog = new LockFileDialog(Gatherer.g_man, name, lock_file);
+	    int choice = dialog.getChoice();
+	    dialog.dispose();
+	    dialog = null;
+
+	    if (choice != LockFileDialog.YES_OPTION) {
+		// user has cancelled
+		lock_file = null;
+		collection_directory = null;
+		collection_config_file = null;
+		return;
+	    }
+
+	    lock_file.delete();
+	}
+
+	try {
+	    // Create a lock file.
+	    createLockFile(lock_file);
+	    // This lock file may not have been created so check
+	    if(!lock_file.canWrite()) {
+		// The lock file cannot be written to. Most likely cause incorrect file permissions.
+		System.err.println("Cannot write lock file!");
+		String args[] = new String[2];
+		args[0] = location;
+		args[1] = Dictionary.get("FileActions.Write_Not_Permitted_Title");
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Open_With_Reason", args), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		args = null;
+		return;
+	    }
+
+	    // Open the collection file
+	    this.collection = new Collection(collection_file);
+	    if (collection.error) {
+		collection = null;
+		// Remove lock file
+		if (lock_file.exists()) {
+		    lock_file.delete();
+		}
+		throw(new Exception(Dictionary.get("CollectionManager.Missing_Config"))); // this error message does not agree with the error
+	    }
+
+	    MetadataSetManager.clearMetadataSets();
+	    MetadataSetManager.loadMetadataSets(collection_metadata_directory);
+
+	    ProfileXMLFileManager.loadProfileXMLFile(collection_metadata_directory);
+ 
+	    // If this is a non-GLI (legacy) collection, ask the user to choose some metadata sets
+	    if (non_gli_collection) {
+		if (!addSomeMetadataSets(collection_directory)) {
+		    lock_file = null;
+		    collection_directory = null;
+		    closeCollection();
+		    return;
+		}
+
+		// Recurse the import folder tree, backing up the metadata.xml files before they are edited
+		LegacyCollectionImporter.backupMetadataXMLFiles(collection_directory);
+	    }
+
+	    // Read through the metadata.xml files in the import directory, building up the metadata value trees
+	    MetadataXMLFileManager.clearMetadataXMLFiles();
+	    MetadataXMLFileManager.loadMetadataXMLFiles(collection_import_directory);
+
+	    // Read through the doc.xml files in the archives directory
+	    File collection_archives_directory = new File(getCollectionArchivesDirectoryPath());
+	    DocXMLFileManager.clearDocXMLFiles();
+	    DocXMLFileManager.loadDocXMLFiles(collection_archives_directory);
+
+	    collection.cdm = new CollectionDesignManager(collection_config_file);
+	    if (non_gli_collection) {
+		// Change the classifiers to use the namespaced element names
+		LegacyCollectionImporter.updateClassifiers(collection.cdm);
+	    }
+
+	    // We're done. Let everyone know.
+	    DebugStream.println(Dictionary.get("CollectionManager.Loading_Successful", name));
+	    Gatherer.refresh(Gatherer.COLLECTION_OPENED);
+	}
+	catch (Exception error) {
+	    // There is obviously no existing collection present.
+	    DebugStream.printStackTrace(error);
+	    if(error.getMessage() != null) {
+		String[] args = new String[2];
+		args[0] = location;
+		args[1] = error.getMessage();
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Open_With_Reason", args), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    }
+	    else {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Open", location), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    }
+	}
+
+	lock_file = null;
+	collection_directory = null;
+	collection_config_file = null;
+    }
+
+
+    private void makeCollection(String name, String email)
+    {
+	// Generate the mkcol.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if (Utility.isWindows() && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "mkcol.pl");
+
+	command_parts_list.add("-collectdir");
+	command_parts_list.add(getCollectDirectory());
+	command_parts_list.add("-win31compat");
+	command_parts_list.add((Gatherer.isGsdlRemote) ? "false" : "true");
+
+	if (email != null && !email.equals("")) {
+	    command_parts_list.add("-creator");
+	    command_parts_list.add(email);
+	}
+
+	command_parts_list.add(name);
+
+	// Run the mkcol.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell process = new GShell(command_parts, GShell.NEW, COLLECT, this, null, GShell.GSHELL_NEW);
+	process.run(); // Don't bother threading this... yet
+    }
+
+
+    /** Any implementation of GShellListener must include this method to allow the GShell to send messages to listeners. However in this case the CollectionManager is in no way interested in what the messages are, just the import events which have a specific type and are handled elsewhere. Thus we can safely ignore this event.
+     * @param event A <strong>GShellEvent</strong> which contains a the message.
+     */
+    public synchronized void message(GShellEvent event) {
+    }
+
+
+    public void metadataChanged(CollectionTreeNode[] file_nodes)
+    {
+	if (collection != null) {
+	    collection.setMetadataChanged(true);
+	}
+    }
+
+
+    public void openCollectionFromLastTime()
+    {
+	// If there was an open collection last session, reopen it
+	if (Gatherer.open_collection_file_path != null && !Gatherer.isGsdlRemote) {
+	    // If we're using a remote Greenstone server we must download the collection configurations first
+	    // if (Gatherer.isGsdlRemote && !CollectionManager.downloaded_collection_configurations) {
+	    // if (RemoteGreenstoneServer.downloadCollectionConfigurations().equals("")) {
+		    // Something went wrong downloading the collection configurations
+		    // return;
+	    // }
+
+	    // CollectionManager.downloaded_collection_configurations = true;
+	    // }
+
+	    // Load the collection now
+	    loadCollection(Gatherer.open_collection_file_path);
+	}
+    }
+
+
+    /** This call is fired whenever a process within a GShell created by this class begins.
+     * @param event A <strong>GShellEvent</strong> containing information about the GShell process.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.gui.GUIManager
+     * @see org.greenstone.gatherer.shell.GShell
+     */
+    public synchronized void processBegun(GShellEvent event) {
+	DebugStream.println("CollectionManager.processBegun(" + event.getType() + ")");
+	///ystem.err.println("ProcessBegun " + event.getType());
+	// If this is one of the types where we wish to lock user control
+	Gatherer.g_man.lockCollection((event.getType() == GShell.IMPORT), true);
+    }
+    /** This call is fired whenever a process within a GShell created by this class ends.
+     * @param event A <strong>GShellEvent</strong> containing information about the GShell process.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.gui.GUIManager
+     * @see org.greenstone.gatherer.shell.GShell
+     */
+    public synchronized void processComplete(GShellEvent event) {
+	System.err.println("CollectionManager.processComplete(" + event.getType() + ")");
+	Gatherer.g_man.lockCollection((event.getType() == GShell.IMPORT), false);
+	///ystem.err.println("Recieved process complete event - " + event);
+	// If we were running an import, now run a build.
+	if(event.getType() == GShell.IMPORT && event.getStatus() == GShell.OK) {
+	    // Finish import.
+	    collection.setImported(true);
+	    buildCollection(false);
+	}
+	// If we were running a build, now is when we move files across.
+	else if(event.getType() == GShell.BUILD && event.getStatus() == GShell.OK) {
+	    if(installCollection()) {
+		// If we have a local library running then ask it to add our newly create collection
+		if (LocalLibraryServer.isRunning() == true) {
+		    LocalLibraryServer.addCollection(collection.getName());
+		}
+		else if (Gatherer.GS3) {
+		    convertToGS3Collection();
+		    Gatherer.configGS3Server(Configuration.site_name, ServletConfiguration.ADD_COMMAND + collection.getName());
+		}  
+
+		// Fire a collection changed first to update the preview etc buttons
+		Gatherer.refresh(Gatherer.COLLECTION_REBUILT);
+
+  		// Now display a message dialog saying its all built
+  		WarningDialog collection_built_warning_dialog = new WarningDialog("warning.CollectionBuilt", "CollectionBuilt.Title", Dictionary.get("CollectionBuilt.Message"), null, false);
+  		collection_built_warning_dialog.setMessageOnly(true); // Not a warning
+  		collection_built_warning_dialog.display();
+  		collection_built_warning_dialog.dispose();
+  		collection_built_warning_dialog = null;
+
+		//Set nothing as needing rebuilding, as a build has just finished :-)
+		CollectionDesignManager.resetRebuildTypeRequired();
+	    }
+	    else {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Preview_Ready_Failed"), Dictionary.get("CollectionManager.Preview_Ready_Title"), JOptionPane.ERROR_MESSAGE);
+		Gatherer.refresh(Gatherer.COLLECTION_REBUILT);
+		DebugStream.println("Status is ok but !installCollection()");
+	    }
+	}
+	else if (event.getStatus() == GShell.CANCELLED) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Build_Cancelled"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    Gatherer.g_man.repaint();
+	}
+	else if (event.getStatus() == GShell.ERROR) {
+	    DebugStream.println("There was an error in the gshell:"+ event.getMessage());
+	    if (event.getType() == GShell.NEW) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Cannot_Create_Collection"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    } else {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Preview_Ready_Failed"), Dictionary.get("CollectionManager.Preview_Ready_Title"), JOptionPane.ERROR_MESSAGE);
+		Gatherer.refresh(Gatherer.COLLECTION_REBUILT);
+	    }
+	    
+	    Gatherer.g_man.repaint(); // It appears Java's own dialogs have the same not always painting correct area bug that I suffer from. Well I don't suffer from it personally, but my ModalDialog components do.
+	}
+    }
+
+
+    /** Determine if the manager is ready for actions apon its collection.
+     * @return A <i>boolean</i> which is <i>true</i> to indicate a collection has been loaded and thus the collection is ready for editing, <i>false</i> otherwise.
+     */
+    static public synchronized boolean ready() {
+	if(collection != null) {
+	    return true;
+	}
+	else {
+	    return false;
+	}
+    }
+
+
+    /** This method associates the collection build monitor with the build monitor created in CreatePane.
+     * @param monitor A <strong>GShellProgressMonitor</strong> which we will use as the build monitor.
+     */
+    public void registerBuildMonitor(GShellProgressMonitor monitor) {
+	build_monitor = monitor;
+    }
+    /** This method associates the collection import monitor with the import monitor created in CreatePane.
+     * @param monitor A <strong>GShellProgressMonitor</strong> which we will use as the import monitor.
+     */
+    public void registerImportMonitor(GShellProgressMonitor monitor) {
+	import_monitor = monitor;
+    }
+
+
+    public void removeCollectionContentsChangedListener(CollectionContentsChangedListener listener)
+    {
+	collection_contents_changed_listeners.remove(listener);
+    }
+
+
+    public void removeMetadataSet(MetadataSet metadata_set)
+    {
+	System.err.println("Removing metadata set...");
+
+	// Delete the .mds file from the collection's "metadata" folder...
+	File metadata_set_file = metadata_set.getMetadataSetFile();
+
+	// ...but not if it is the "ex.mds" file
+	if (metadata_set_file.getName().equals("ex.mds")) {
+	    return;
+	}
+
+	// ...and only if it exists
+	if (metadata_set_file.exists()) {
+	    metadata_set_file.delete();
+
+	    // Unload it from the MetadataSetManager
+	    MetadataSetManager.unloadMetadataSet(metadata_set);
+
+	    // If we're using a remote Greenstone server, delete the metadata file on the server
+	    if (Gatherer.isGsdlRemote) {
+		RemoteGreenstoneServer.deleteCollectionFile(collection.getName(), metadata_set_file);
+	    }
+	}
+    }
+
+
+    /** Used to check whether all open collections have a 'saved' state.
+     * @return A <i>boolean</i> which is <i>true</i> if the collection has been saved.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    public boolean saved() {
+	boolean result = true;
+	if(collection != null) {
+	    result = collection.getSaved();
+	}
+	return result;
+    }
+
+
+    /** Saves the currently loaded collection. */
+    public void saveCollection()
+    {
+	DebugStream.println("Saving collection " + collection.getName() + "...");
+
+	// Change cursor to hourglass
+	Gatherer.g_man.wait(true);
+
+	// Create a backup of the collection file, just in case anything goes wrong
+	File collection_file = new File(getCollectionFilePath());
+	if (collection_file.exists()) {
+	    File collection_file_backup = new File(collection_file.getAbsolutePath() + "~");
+	    if (!collection_file.renameTo(collection_file_backup)) {
+		DebugStream.println("Error in CollectionManager.saveCollection(): could not create backup file.");
+	    }
+	    collection_file_backup.deleteOnExit();
+	}
+
+	// Write out the collection file
+	collection.save();
+
+	// Write out the collection configuration file
+	Gatherer.g_man.design_pane.saveConfiguration();
+
+	// Change cursor back to normal
+	Gatherer.g_man.wait(false);
+    }
+
+
+    /** I started giving the user the choice of using an existing meta set or creating a new one. The second option being so that they didn't have to add/merge/ignore each element, they could all be added automatically. However, I am not sure where the merge prompt gets called from, and it is not essential, so I am leaving it for now - it should be added back in and finished. [kjdon] */
+    private boolean addSomeMetadataSets(File collection_dir) {
+
+
+	ExternalCollectionPrompt external_prompt = new ExternalCollectionPrompt();
+	int meta_choice = external_prompt.getMetadataChoice();
+	boolean cancelled = external_prompt.isCancelled();
+	if (cancelled) {
+	    return false;
+	}
+
+	// now we reuse the newcoll metadata prompt for the user to select metadata sets
+	NewCollectionMetadataPrompt ncm_prompt = new NewCollectionMetadataPrompt(true);
+	if (ncm_prompt.isCancelled()) {
+	    return false;
+	}
+	ArrayList metadata_sets = ncm_prompt.getSets();
+	// Import default metadata sets if any.
+	for(int i = 0; metadata_sets != null && i < metadata_sets.size(); i++) {
+	    importMetadataSet((MetadataSet) metadata_sets.get(i));
+	}
+
+	// Always import the extracted metadata set
+	File extracted_metadata_set_file = new File(Gatherer.getGLIMetadataDirectoryPath() + MetadataSetManager.EXTRACTED_METADATA_NAMESPACE + StaticStrings.METADATA_SET_EXTENSION);
+	importMetadataSet(new MetadataSet(extracted_metadata_set_file));
+
+	return true;
+    }
+
+
+    // used as arg in the perl scripts
+    private String getCollectDirectory() {
+	String collect_dir = Gatherer.getCollectDirectoryPath();
+
+	// Remove erroneous file windows file separator as it causes problems when running import.pl
+	if(collect_dir.length() > 2 && collect_dir.endsWith("\\")) {
+	    collect_dir = collect_dir.substring(0, collect_dir.length() - 1);
+	}
+
+	return collect_dir;
+    }
+
+
+    /** Install collection by moving its files from building to index after a successful build.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private boolean installCollection()
+    {
+	DebugStream.println("Build complete. Moving files.");
+
+	try {
+	    // Ensure that the local library has released this collection so we can delete the index directory
+	    if (LocalLibraryServer.isRunning() == true) {
+		LocalLibraryServer.releaseCollection(collection.getName());
+	    }
+	    // deactivate it in tomcat so that windows will release the index files
+	    if (Gatherer.GS3) {
+		Gatherer.configGS3Server(Configuration.site_name, ServletConfiguration.DEACTIVATE_COMMAND + collection.getName());
+	    }
+	    File index_dir = new File(getCollectionIndexDirectoryPath());
+	    DebugStream.println("Index = " + index_dir.getAbsolutePath());
+
+	    File building_dir = new File(getCollectionBuildingDirectoryPath());
+	    DebugStream.println("Building = " + building_dir.getAbsolutePath());
+
+	    // Get the build mode from the build options
+	    String build_mode = collection.build_options.getValue("mode");
+
+	    // Special case for build mode "all": replace index dir with building dir
+	    if (build_mode == null || build_mode.equals(Dictionary.get("CreatePane.Mode_All"))) {
+		// Remove the old index directory
+		if (index_dir.exists()) {
+		    Utility.delete(index_dir);
+
+		    // Wait for a couple of seconds, just for luck
+		    wait(2000);
+
+		    // Check the delete worked
+		    if (index_dir.exists()) {
+			throw new Exception(Dictionary.get("CollectionManager.Index_Not_Deleted"));
+		    }
+		}
+
+		if (Gatherer.isGsdlRemote) {
+		    RemoteGreenstoneServer.deleteCollectionFile(collection.getName(), new File(getCollectionIndexDirectoryPath()));
+		    RemoteGreenstoneServer.moveCollectionFile(collection.getName(), new File(getCollectionBuildingDirectoryPath()), new File(getCollectionIndexDirectoryPath()));
+		}
+
+		// Move the building directory to become the new index directory
+		if (building_dir.renameTo(index_dir) == false) {
+		    throw new Exception(Dictionary.get("CollectionManager.Build_Not_Moved"));
+		}
+	    }
+
+	    // Otherwise copy everything in the building dir into the index dir
+	    else {
+		moveContentsInto(building_dir, index_dir);
+	    }
+	}
+	catch (Exception exception) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("CollectionManager.Install_Exception", exception.getMessage()), "Error", JOptionPane.ERROR_MESSAGE);
+	    return false;
+	}
+	return true;
+    }
+
+
+    /** Moves all the files in one directory into another, overwriting existing files */
+    private void moveContentsInto(File source_directory, File target_directory)
+    {
+	File[] source_files = source_directory.listFiles();
+	for (int i = 0; i < source_files.length; i++) {
+	    File source_file = source_files[i];
+	    File target_file = new File(target_directory, source_file.getName());
+
+	    if (source_file.isDirectory()) {
+		moveContentsInto(source_file, target_file);
+		source_file.delete();
+	    }
+	    else {
+		if (target_file.exists()) {
+		    target_file.delete();
+		}
+
+		source_file.renameTo(target_file);
+	    }
+	}
+    }
+
+
+    private void updateCollectionCFG(File base_cfg, File new_cfg, String description, String email, String title)
+    {
+	boolean first_name = true;
+	boolean first_extra = true;
+	String collection_path = (base_cfg.getParentFile().getParentFile()).getAbsolutePath();
+
+	// Now read in base_cfg line by line, parsing important onces and/or replacing them with information pertinent to our collection. Each line is then written back out to the new collect.cfg file.
+	try {
+	    BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(base_cfg), "UTF-8"));
+	    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new_cfg), "UTF-8"));
+	    String command = null;
+	    while((command = in.readLine()) != null) {
+		if (command.length()==0) {
+		    // output a new line
+		    out.newLine();
+		    continue;
+		}
+		// We have to test the end of command for the special character '\'. If found, remove it and append the next line, then repeat.
+		while(command.trim().endsWith("\\")) {
+		    command = command.substring(0, command.lastIndexOf("\\"));
+		    String next_line = in.readLine();
+		    if(next_line != null) {
+			command = command + next_line;
+		    }
+		}
+		// commands can extend over more than one line so use the CommandTokenizer which takes care of that
+		CommandTokenizer tokenizer = new CommandTokenizer(command, in, false);
+		String command_type_str = tokenizer.nextToken().toLowerCase();
+
+		if (command_type_str.equals(StaticStrings.COLLECTIONMETADATA_STR)) {
+		    // read the whole thing in, but for collectionname, collectionextra, iconcollection, iconcollectionsmall we will ignore them
+		    StringBuffer new_command = new StringBuffer(command_type_str);
+		    String meta_name = tokenizer.nextToken();
+		    new_command.append(' ');
+		    new_command.append(meta_name);
+		    while (tokenizer.hasMoreTokens()) {
+			new_command.append(' ');
+			new_command.append(tokenizer.nextToken());
+		    }
+		    if (meta_name.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONNAME_STR) || meta_name.equals(StaticStrings.COLLECTIONMETADATA_COLLECTIONEXTRA_STR) || meta_name.equals(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTION_STR) || meta_name.equals(StaticStrings.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR)) {
+			// dont save
+		    } else {
+			write(out, new_command.toString());
+		    }
+		    new_command = null;
+		    continue;
+		} // if collectionmeta
+
+		if(command_type_str.equals(Utility.CFG_CLASSIFY)) {
+		    StringBuffer text = new StringBuffer(command_type_str);
+		    // Read in the classifier command watching for hfile, metadata and sort arguments.
+		    String buttonname = null;
+		    String hfile = null;
+		    String new_metadata = null;
+		    String old_metadata = null;
+		
+		    while(tokenizer.hasMoreTokens()) {
+			String token = tokenizer.nextToken();
+			if(token.equals(Utility.CFG_CLASSIFY_HFILE)) {
+			    if(tokenizer.hasMoreTokens()) {
+				text.append(" ");
+				text.append(token);
+				token = tokenizer.nextToken();
+				hfile = token;
+			    }
+			}
+			else if(token.equals(Utility.CFG_CLASSIFY_METADATA)) {
+			    if(tokenizer.hasMoreTokens()) {
+				text.append(" ");
+				text.append(token);
+				String temp_metadata = tokenizer.nextToken();
+				String replacement = ProfileXMLFileManager.getMetadataElementFor(temp_metadata);
+				if (replacement != null && !replacement.equals("")) {
+				    token = replacement;
+				    old_metadata = temp_metadata;
+				    new_metadata = replacement;
+				}
+				else {
+				    token = temp_metadata;
+				}
+				temp_metadata = null;
+				replacement = null;
+			    }
+			}
+			else if(token.equals(Utility.CFG_CLASSIFY_SORT)) {
+			    if(tokenizer.hasMoreTokens()) {
+				text.append(" ");
+				text.append(token);
+				String temp_metadata = tokenizer.nextToken();
+				String replacement = ProfileXMLFileManager.getMetadataElementFor(temp_metadata);
+				if (replacement != null && !replacement.equals("")) {
+				    token = replacement;
+				}
+				else {
+				    token = temp_metadata;
+				}
+				temp_metadata = null;
+				replacement = null;
+			    }
+			}
+			else if(token.equals(Utility.CFG_CLASSIFY_BUTTONNAME)) {
+			    buttonname = token;
+			}
+			text.append(' ');
+			text.append(token);
+			token = null;
+		    }
+		   
+		    // If we replaced the metadata argument and didn't encounter a buttonname, then add one now pointing back to the old metadata name in order to accomodate macro files which required such names (buttonname is metadata name by default)!
+		    if(old_metadata != null && new_metadata != null && buttonname == null) {
+			text.append(' ');
+			text.append(Utility.CFG_CLASSIFY_BUTTONNAME);
+			text.append(' ');
+			text.append(old_metadata);
+		    }
+		    command = text.toString();
+		    // Replace the hfile if we found it
+		    if(hfile != null && new_metadata != null) {
+			command = command.replaceAll(hfile, new_metadata + ".txt");
+		    }
+		
+		    buttonname = null;
+		    hfile = null;
+		    new_metadata = null;
+		    old_metadata = null;
+		    write(out, command);
+		} else {
+		    // the rest of the commands just want a string - we read in all the tokens from the tokeniser and get rid of it.
+		    StringBuffer new_command = new StringBuffer(command_type_str);
+		    while (tokenizer.hasMoreTokens()) {
+			new_command.append(' ');
+			new_command.append(tokenizer.nextToken());
+		    }
+
+		    command = new_command.toString();
+		    
+		    // There is still one special case, that of the format command. In such a command we have to search for [<target>] to ensure we don't change parts of the format which have nothing to do with the metadata elements.
+		    // we really want to build up the whole command here
+		    boolean format_command = command_type_str.equals(Utility.CFG_FORMAT);
+		    HashMap metadata_mapping = ProfileXMLFileManager.getMetadataMapping();
+		    if (metadata_mapping != null) {
+			Iterator keys = metadata_mapping.keySet().iterator();
+			while (keys.hasNext()) {
+			    String target = (String) keys.next();
+			    String replacement = (String) metadata_mapping.get(target);
+			    if (replacement != null && !replacement.equals("")) {
+				if (format_command) {
+				    target = "\\[" + target + "\\]";
+				    replacement = "{Or}{[" + replacement + "]," + target + "}";
+				}
+				command = command.replaceAll(target, replacement);
+			    }
+			}
+		    }
+
+		    write(out, command);
+		}
+		tokenizer = null;
+	    }
+	    in.close();
+	    in = null;
+	    out.flush();
+	    out.close();
+	    out = null;
+	}
+	catch(Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	// All done, I hope.
+    }
+    
+    private void write(BufferedWriter out, String message)
+	throws Exception {
+	out.write(message, 0, message.length());
+	out.newLine();
+    }
+
+
+    /** The CollectionManager class is getting too confusing by half so I'll implement this TreeModelListener in a private class to make responsibility clear. */
+    private class FMTreeModelListener
+	implements TreeModelListener {
+	/** Any action that changes one of the tree models within a collection, which are the only models we listen to, mean the collections contents have changed and so saved should be set to false.
+	 * @param event A <strong>TreeModelEvent</strong> encompassing all the information about the event which has changed the tree.
+	 */
+	public void treeNodesChanged(TreeModelEvent event) {
+	    if(collection != null) {
+		collection.setSaved(false);
+		collection.setFilesChanged(true);
+	    }
+	}
+	/** Any action that changes one of the tree models within a collection, which are the only models we listen to, mean the collections contents have changed and so saved should be set to false.
+	 * @param event A <strong>TreeModelEvent</strong> encompassing all the information about the event which has changed the tree.
+	 */
+	public void treeNodesInserted(TreeModelEvent event) {
+	    if(collection != null) {
+		collection.setSaved(false);
+		collection.setFilesChanged(true);
+	    }
+	}
+	/** Any action that changes one of the tree models within a collection, which are the only models we listen to, mean the collections contents have changed and so saved should be set to false.
+	 * @param event A <strong>TreeModelEvent</strong> encompassing all the information about the event which has changed the tree.
+	 */
+	public void treeNodesRemoved(TreeModelEvent event) {
+	    if(collection != null) {
+		collection.setSaved(false);
+		collection.setFilesChanged(true);
+
+	    }
+	}
+	/** Any action that changes one of the tree models within a collection, which are the only models we listen to, mean the collections contents have changed and so saved should be set to false.
+	 * @param event A <strong>TreeModelEvent</strong> encompassing all the information about the event which has changed the tree.
+	 */
+	public void treeStructureChanged(TreeModelEvent event) {
+	    if(collection != null) {
+		collection.setSaved(false);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTree.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTree.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTree.java	(revision 21112)
@@ -0,0 +1,60 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.collection;
+
+import java.awt.*;
+import javax.swing.*;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.gui.tree.DragTreeCellRenderer;
+
+
+public class CollectionTree
+    extends DragTree
+{
+    public CollectionTree(String name, CollectionTreeModel collection_tree_model, boolean mixed_selection)
+    {
+	super(name, collection_tree_model, mixed_selection);
+	setCellRenderer(new CollectionTreeCellRenderer());
+    }
+
+
+    private class CollectionTreeCellRenderer
+	extends DragTreeCellRenderer
+    {
+	public Component getTreeCellRendererComponent(JTree tree, Object value, boolean sel, boolean expanded, boolean leaf, int row, boolean hasFocus)
+	{
+	    JLabel tree_cell = (JLabel) super.getTreeCellRendererComponent(tree, value, sel, expanded, leaf, row, hasFocus);
+
+	    // Mark explodable files with a different icon
+	    if (value instanceof CollectionTreeNode && ((CollectionTreeNode) value).isExplodable()) {
+		tree_cell.setIcon(CollectionTreeNode.GREEN_FILE_ICON);
+	    }
+
+	    return tree_cell;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeModel.java	(revision 21112)
@@ -0,0 +1,41 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.collection;
+
+
+import org.greenstone.gatherer.file.FileSystemModel;
+
+
+/** This class represents ... */
+public class CollectionTreeModel
+    extends FileSystemModel
+{
+    public CollectionTreeModel(CollectionTreeNode root_node)
+    {
+	super(root_node);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeNode.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeNode.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/CollectionTreeNode.java	(revision 21112)
@@ -0,0 +1,68 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.collection;
+
+
+import java.io.*;
+import javax.swing.*;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.file.FileNode;
+import org.greenstone.gatherer.util.JarTools;
+
+
+/** This class represents one node in the collection tree. */
+public class CollectionTreeNode
+    extends FileNode
+{
+    static final public ImageIcon GREEN_FILE_ICON = JarTools.getImage("greenfile.gif", true);
+
+    /** Is this file a metadata database that is explodable with the explode_metadata_databases.pl script? */
+    private boolean is_explodable = false;
+
+
+    public CollectionTreeNode(File file)
+    {
+	super(file);
+
+	this.is_explodable = CollectionDesignManager.plugin_manager.isFileExplodable(file);
+    }
+
+
+    public FileNode addChildNode(File file)
+    {
+	CollectionTreeNode child_node = new CollectionTreeNode(file);
+	child_node.setModel(model);
+	child_node.setParent(this);
+	return child_node;
+    }
+
+
+    public boolean isExplodable()
+    {
+	return is_explodable;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/DownloadJob.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/DownloadJob.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/DownloadJob.java	(revision 21112)
@@ -0,0 +1,574 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.collection;
+
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.GAuthenticator;
+import org.greenstone.gatherer.WGet;
+import org.greenstone.gatherer.file.WorkspaceTree;
+import org.greenstone.gatherer.gui.DownloadProgressBar;
+import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
+import org.greenstone.gatherer.util.GURL;
+import org.greenstone.gatherer.util.SynchronizedTreeModelTools;
+import org.greenstone.gatherer.util.Utility;
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.0
+ */
+public class DownloadJob
+    implements ActionListener {
+	 
+    private boolean debug;
+    private boolean higher_directories;
+    private boolean no_parents;
+    private boolean other_hosts; 
+    private boolean page_requisites; 
+    private boolean quiet;
+
+    private AppendLineOnlyFileDocument download_log;
+
+    private DownloadProgressBar progress;
+
+    private GURL initial = null;
+    private GURL url = null;
+
+    // private TreeModel model;
+
+    private int depth;
+    private int previous_state;
+    private int state;
+
+    private String current_url;
+    private String destination;
+    private String proxy_pass;
+    private String proxy_user;
+
+    private Vector encountered_urls;
+    private Vector failed_urls;
+
+    private WGet mummy;
+
+    public static int COMPLETE = 0;
+    public static int PAUSED   = 1;
+    public static int RUNNING  = 2;
+    public static int STOPPED  = 3;
+
+    /**
+     */
+    public DownloadJob(boolean debug, boolean no_parents, boolean other_hosts, boolean page_requisites, boolean quiet, URL initial, int depth, String destination, String proxy_pass, String proxy_user, WGet mummy, boolean simple) {
+	// this.model = model;
+
+	String log_filename = Gatherer.getGLIUserLogDirectoryPath() + "wget" + initial.hashCode() + ".log";
+	File log_file = new File(log_filename);
+	if(log_file.exists()) {
+	    log_file.delete();
+	}
+	File parent_log_file = log_file.getParentFile();
+	parent_log_file.mkdirs();
+	parent_log_file = null;
+	log_file = null;
+
+	this.debug = debug;
+	this.download_log = new AppendLineOnlyFileDocument(log_filename, false);
+	this.no_parents = no_parents;
+	this.other_hosts = other_hosts;
+	this.page_requisites = page_requisites;
+	this.quiet = quiet;
+	this.initial = new GURL(initial);
+	this.depth = depth;
+	this.destination = destination;
+	this.proxy_pass = proxy_pass;
+	this.proxy_user = proxy_user;
+	this.mummy = mummy;
+
+	progress = new DownloadProgressBar(this, initial.toString(), simple);
+
+	encountered_urls = new Vector();
+	failed_urls = new Vector();
+
+	previous_state = STOPPED;
+	state = STOPPED;
+    }
+
+    /** Depending on which button on the progress bar was pushed,
+     * this method will affect the state of the DownloadJob and perhaps make
+     * calls to wget.class if necessary.
+     * @param event The ActionEvent fired from within the DownloadProgressBar
+     * which we must respond to.
+     */
+    public void actionPerformed(ActionEvent event) {
+	// The stop_start_button is used to alternately start or stop the
+	// job. If the current state of the job is paused then this
+	// restart is logically equivelent to a resume.
+	if(event.getSource() == progress.stop_start_button) {
+	    previous_state = state;
+	    if (state == RUNNING) {
+		state = STOPPED;
+	    } else {
+		//previous_state = state;
+		state = RUNNING;
+		mummy.resumeThread();
+	    }
+	}
+	else if (event.getSource() == progress.close_button) {
+	    if(state == RUNNING) {
+		previous_state = state;
+		state = STOPPED; // do we need to do anything else to stop this?
+	    }
+	    // else {
+	    mummy.deleteDownloadJob(this);
+		// }
+	}
+    }
+
+    /** Called by the WGet native code to inform us of a new download starting.
+     * @param raw_url The url that is being downloaded, as a String.
+     */
+    public void addDownload(String raw_url) {
+	if(!encountered_urls.contains(raw_url)) {
+	    encountered_urls.add(raw_url);
+	}
+	// Regardless create a new GURL
+	current_url = raw_url;
+	url = new GURL(raw_url);
+	progress.addDownload(raw_url);
+    }
+
+    /** Used to advise the DownloadJob of a newly parsed link. Its up to DownloadJob
+     * to decide if it already knows about this url, and if not to
+     * update its progress bar.
+     * @param raw_url The url in question as a String.
+     * @param type Whether the link is an internal or external link.
+     * @return A boolean indicating if the url was added.
+     */
+    public boolean addLink(String raw_url, int type) {
+	///ystem.out.println("addLink("+url+", "+type+")");
+	if(!encountered_urls.contains(raw_url)) {
+	    // Add it to the urls we've seen.
+	    encountered_urls.add(raw_url);
+	    // Add it the to links for the current GURL.
+
+	    // Add it to the progress file count.
+	    progress.increaseFileCount();
+	    return true;
+	}
+	// Regardless add it to the children links of the current GURL
+	initial.addLink(raw_url);
+
+	// We've seen it before. Don't count it again.
+	return false;
+    }
+
+    public void callWGet() {
+	// Build parameter string. Note that we never clobber, and we continue if possible
+
+	// want to always download newer files,  convert non-relative links to relative, always use directories, and only try twice to get a file before giving up
+	String command = Configuration.getWGetPath() + " -N -k -x -t 2 "; // + " -nc -c "; 
+
+	if (no_parents) {
+	    command = command + "-np ";
+	}
+	if(depth < 0) {
+	    // Infinite recursion
+	    command = command + "-r ";
+	}
+	else if (depth == 0) {
+	    // Just this page.
+	}
+	else if (depth > 0) {
+	    // Recursion to the specified depth.
+	    command = command + "-r -l" + depth + " ";
+	}
+
+	String proxy_url = "";
+	// Determine if we have to use a proxy.
+	if(Configuration.get("general.use_proxy", true)) {
+	    String proxy_host = Configuration.getString("general.proxy_host", true);
+	    String proxy_port = Configuration.getString("general.proxy_port", true);
+	    // Find out whether the user has already authenticated themselves
+	    String user_pass = null;
+	    String address = proxy_host + ":" + proxy_port;
+	    int count = 0;
+	    while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
+		Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
+		count++;
+	    }
+	    if(count >= 3) {
+		state = STOPPED;
+		return;
+	    }
+	    if(user_pass.indexOf("@") != -1) {
+		
+		// Write the use proxy command - we don't do this anymore, instead we set environment variables - hopefully these can't be spied on like the follwoing can (using ps) - actually the environment stuff didn't work for windows, so lets go back to this
+		if (Utility.isWindows()) {
+		    command = command + "-e httpproxy=" + proxy_host + ":" + proxy_port + "/ --proxy-user=" + user_pass.substring(0, user_pass.indexOf("@")) + " --proxy-passwd=" + user_pass.substring(user_pass.indexOf("@") + 1) + " -Y on ";
+		} else {
+		    String user_name = user_pass.substring(0, user_pass.indexOf("@"));
+		    String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
+		    proxy_url = user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";
+		}
+
+	    }
+	    else {
+		DebugStream.println("Unknown user/pass");
+	    }
+	}
+		
+	// The user can choose to mirror all of the page requisites...
+ 	if(page_requisites) {
+ 	    command = command + "-p ";
+ 	}
+
+	// Download files from other hosts
+	if(other_hosts) {
+	    command = command + "-H ";
+	}
+
+	// Finally tell it the site to download.
+	command = command + initial.toString();
+
+	if(previous_state == DownloadJob.COMPLETE) {
+	    progress.mirrorBegun(true, true);
+	}
+	else {
+	    progress.mirrorBegun(false, true);
+	}		  
+
+	File dest_file = new File(destination);
+	if (!dest_file.exists()) {
+	    dest_file.mkdirs();
+	}
+	// Run it
+	try {
+	    //DebugStream.println("Cmd: " + command); // don't print it out cos it may have the password in it
+	    Runtime rt = Runtime.getRuntime();
+	    String [] env = null;
+	    if (!proxy_url.equals("")) {
+		env = new String[2];
+		env[0] = "http_proxy=http://"+proxy_url;
+		env[1] = "ftp_proxy=ftp://"+proxy_url;
+	    }
+	    Process prcs = rt.exec(command, env, dest_file);
+	    InputStreamReader isr = new InputStreamReader(prcs.getErrorStream());
+	    BufferedReader br = new BufferedReader(isr);
+	    // Capture the standard error stream and seach for two particular occurances.
+	    String line;
+	    boolean ignore_for_robots = false;
+	    while ((line = br.readLine()) != null && state != STOPPED) {
+		
+		DebugStream.println(line);
+		download_log.appendLine(line);
+		// The first magic special test is to see if we've just
+		// asked for the robots.txt file. If so we ignore
+		// the next add and then the next complete/error.
+		if(line.lastIndexOf("robots.txt;") != -1) {
+		    DebugStream.println("***** Requesting robot.txt");
+		    ignore_for_robots = true;
+		}
+		// If line contains "=> `" display text as the 
+		// currently downloading url. Unique to add download.
+		else if(line.lastIndexOf("=> `") != -1) {
+		    if(!ignore_for_robots) {
+			// Add download
+			String new_url = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
+			addDownload("http:/" + new_url);
+		    }
+		}
+		// If line contains "/s) - `" set currently 
+		// downloading url to "Download Complete".
+		else if(line.lastIndexOf("/s) - `") != -1) {
+		    String current_file_downloading = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
+		    if(!ignore_for_robots) {
+			DebugStream.println("Not ignore for robots");
+			// Download complete
+			downloadComplete(current_file_downloading);
+		    }
+		    else {
+			DebugStream.println("Ignore for robots");
+			ignore_for_robots = false;
+		    }
+		}
+		// The already there line begins "File `..." However this
+		// is only true in english, so instead I looked and there
+		// are few (if any at all) other messages than those above
+		// and not overwriting messages that use " `" so we'll 
+		// look for that. Note this method is not guarenteed to be
+		// unique like the previous two.
+		else if(line.lastIndexOf(" `") != -1) {
+		    // Not Overwriting
+		    DebugStream.println("Already there.");
+		    String new_url = 
+			line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
+		    addDownload("http:/" + new_url);
+		    downloadWarning();
+		}
+		// Any other important message starts with the time in the form hh:mm:ss
+		else if(line.length() > 7) {
+		    if(line.charAt(2) == ':' && line.charAt(5) == ':') {
+			if(!ignore_for_robots) {
+			    DebugStream.println("Error.");
+			    downloadFailed();
+			}
+			else {
+			    ignore_for_robots = false;
+			}
+		    }
+		}
+	    }
+	    if(state == STOPPED) {
+		isr.close();
+		prcs.destroy(); // This doesn't always work, but it's worth a try
+	    }
+	    else {
+		// Now display final message based on exit value
+		prcs.waitFor();
+	    }
+	}
+	catch (Exception ioe) {
+	    //message(Utility.ERROR, ioe.toString());
+	    DebugStream.printStackTrace(ioe);
+	}
+	// If we've got to here and the state isn't STOPPED then the
+	// job is complete.
+	if(state == DownloadJob.RUNNING) {
+	    progress.mirrorComplete();
+	    previous_state = state;
+	    state = DownloadJob.COMPLETE;
+	    
+	}
+	// refresh the workspace tree
+	Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
+    }
+    
+	
+    /** The most important part of the DownloadJob class, this method is
+     * responsible for calling the WGet native methods used to 
+     * mirror the indicated url. By this stage all the variables
+     * necessary should be set and we need only build up the
+     * parameter string and make the call.
+     */
+    public void callWGetNative() {
+	Vector args = new Vector();
+
+	// Let the DownloadProgressBar know we're starting, just in case
+	// the user hasn't told us to. If this is the second time the
+	// urls downloaded and the first attempt was successful (ie 
+	// the previous job was complete), then we have the case where 
+	// the user is forcing us to remirror. Reset all the values etc
+	// if this is the case then reset the variables.
+	// Note that this can cause the result line to look something 
+	// like this.
+	// Downloaded 12 of 12 files (8 warnings, 0 errors).
+	// The warnings would be something like, 'File already downloaded'
+	// but the total number of files and the file successfully 
+	// downloaded will be correct.
+	if(previous_state == DownloadJob.COMPLETE) {
+	    progress.mirrorBegun(true, false);
+	}
+	else {
+	    progress.mirrorBegun(false, false);
+	}
+
+	// Parse arguments into array.
+	args.add(Configuration.getWGetPath());
+	args.add("-d");
+	args.add("-o");
+	args.add("debug.txt");
+		  
+	if(destination != null) {
+	    args.add("-P");
+	    args.add(destination);
+	}
+
+	if(depth < 0) {
+	    // Infinite recursion
+	    args.add("-r");
+	}
+	else if (depth == 0) {
+	    // Just this page.
+	}
+	else if (depth > 0) {
+	    // Recursion to the specified depth.
+	    args.add("-r");
+	    args.add("-l");
+	    args.add("" + depth + ""); // Hacky
+	}
+
+	if(previous_state == PAUSED) {
+	    args.add("-nc");
+	    args.add("-c");
+	}
+
+	if(proxy_user != null) {
+	    args.add("--proxy-user=" + proxy_user);
+	    args.add("--proxy-passwd=" + proxy_pass);
+	}
+		  
+	if(page_requisites) {
+	    args.add("-p");
+	}
+
+	if(quiet) {
+	    args.add("-q");
+	}
+
+	if(other_hosts) {
+	    args.add("-H");
+	}
+
+	args.add(initial.toString());
+
+	DebugStream.println("Calling wget ");
+	for(Enumeration e = args.elements(); e.hasMoreElements();) {
+	    DebugStream.println(e.nextElement() + " ");
+	}
+	DebugStream.println("");
+
+	// Run home to mummy.
+	int value = mummy.wget(args.size(), args.toArray(), debug);
+
+	// If we've got to here and the state isn't STOPPED then the job is complete.
+	if(state == RUNNING) {
+	    progress.mirrorComplete();
+	    previous_state = state;
+	    state = COMPLETE;
+	}
+    }
+
+    /** Called by the WGet native code when the current download is 
+     * completed. In turn all download listeners are informed.
+     */
+    public void downloadComplete() {
+	progress.downloadComplete();
+	url = null;
+	current_url = null;
+    }
+
+
+    public void downloadComplete(String current_file_downloading)
+    {
+	progress.downloadComplete();
+	DebugStream.println("Download complete: " + current_file_downloading);
+    }
+
+
+    /** Called by the WGet native code when the requested download returns
+     * a status code other than 200.
+     */
+    public void downloadFailed() {
+	failed_urls.add(current_url); // It is the current url that failed
+	progress.downloadFailed();
+	DebugStream.println("Download failed: " + current_url);
+    } 
+	 
+    /**
+     */
+    public void downloadWarning() {
+	progress.downloadWarning();
+    }
+
+
+    /** 
+     * @return A String representing the initial urls host (root node
+     * of tree that we are mirroring).
+     */
+    public String getHost() {
+	return url.getHost();
+    }
+
+    public AppendLineOnlyFileDocument getLogDocument() {
+	return download_log;
+    }
+
+    /**
+     * @return Returns the progress bar associated with this job.
+     */
+    public DownloadProgressBar getProgressBar() {
+	return progress;
+    }
+
+    /** Called to discover if the user wanted this thread to run or if
+     * it is paused.
+     * @return An int representing the current DownloadJob state.
+     */
+    public int getState() {
+	return state;
+    }
+
+    /** Returns the current state of the stop flag for this job.
+     * @return A boolean representing whether the user has requested to
+     * stop.
+     */
+    public boolean hasSignalledStop() {
+	if(state == DownloadJob.STOPPED || state == DownloadJob.PAUSED || 
+	   state == DownloadJob.COMPLETE) {
+	    return true;
+	}
+	return false;
+    }
+
+    public void setState(int state) {
+	previous_state = this.state;
+	this.state = state;
+    }
+
+    /** A convinence call.
+     * @return A String representing the url of the initial url (root node of the mirrored tree).
+     */
+    public String toString() {
+	return initial.toString();
+    }
+
+    /** Called by the WGet native code to signal the current progress of 
+     * downloading.
+     * @param current A long representing the number of bytes that have 
+     * been downloaded since the last update. 
+     * @param expected A long representing the total number of bytes
+     * expected for this download.
+     */
+    public void updateProgress(long current, long expected) {
+	progress.updateProgress(current, expected);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/LegacyCollectionImporter.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/LegacyCollectionImporter.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/LegacyCollectionImporter.java	(revision 21112)
@@ -0,0 +1,138 @@
+package org.greenstone.gatherer.collection;
+
+
+import java.io.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.Argument;
+import org.greenstone.gatherer.cdm.Classifier;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.metadata.ProfileXMLFileManager;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+
+/**
+ * Importing legacy collections involves two main steps:
+ *
+ * 1. Read the existing metadata.xml files and namespace them, usually by prompting the user
+ *    to specify the mapping between old metadata elements and new elements.
+ *
+ * 2. Fix up the classify commands in the collect.cfg file to specify the new element names.
+ *
+ * @author Michael Dewsnip
+ */
+public class LegacyCollectionImporter 
+{
+    // This copies all the existing metadata.xml files into a backup directory
+    static public void backupMetadataXMLFiles(File collection_dir)
+    {
+	File import_dir = new File(collection_dir, Utility.IMPORT_DIR);
+	File import_bak_dir = new File(collection_dir, Utility.IMPORT_BAK_DIR);
+	import_bak_dir.mkdir();
+	copyMetadataXMLFiles(import_dir, import_bak_dir);
+    }
+
+
+    static private void copyMetadataXMLFiles(File source_dir, File backup_dir)
+    {
+	if (source_dir == null || !source_dir.exists()) {
+	    return;
+	}
+
+	// Find the metadata file in this dir
+	File metadata_xml_file_file = new File(source_dir, "metadata.xml");
+	if (metadata_xml_file_file.exists()) {
+	    File new_metadata_xml_file_file = new File(backup_dir, "metadata.xml");
+	    try {
+		backup_dir.mkdirs();
+		Gatherer.f_man.getQueue().copyFile(metadata_xml_file_file, new_metadata_xml_file_file, null);
+		if (!new_metadata_xml_file_file.exists()) {
+		    throw new Exception("");
+		}
+	    }
+	    catch (Exception e) {
+		DebugStream.println("Exception: couldn't copy the file " + metadata_xml_file_file.getPath() + e.getMessage());
+	    }
+	}
+
+	// Now go through child directories
+	File [] children = source_dir.listFiles();
+	for (int i = 0; i < children.length; i++) {
+	    File child = children[i];
+	    if (child.isDirectory()) {
+		copyMetadataXMLFiles(child, new File(backup_dir, child.getName()));
+	    }
+	}
+    }
+
+
+    static public void updateClassifiers(CollectionDesignManager cdm)
+    {
+	// Update the metadata elements in each of the classifiers
+	for (int i = 0; i < cdm.classifier_manager.getSize(); i++) {
+	    Classifier classifier = cdm.classifier_manager.getClassifier(i);
+	    // System.err.println("Classifier: " + classifier);
+
+	    // Update the "-metadata" value
+	    Argument metadata_argument = classifier.getArgument("-metadata");
+	    String metadata_argument_value = metadata_argument.getValue();
+	    String metadata_element_name_full = mapLegacyMetadataElementName(metadata_argument_value);
+	    if (metadata_element_name_full != null && !metadata_element_name_full.equals("")) {
+		String metadata_element_display_name_full = MetadataTools.getDisplayNameForMetadataElementWithName(metadata_element_name_full);
+		if (metadata_element_display_name_full != null) {
+		    metadata_argument_value = metadata_element_display_name_full;
+		    metadata_argument.setValue(metadata_argument_value);
+		}
+	    }
+
+  	    // Update the "-sort" value
+	    Argument sort_argument = classifier.getArgument("-sort");
+	    if (sort_argument != null) {
+		String sort_element_name_full = mapLegacyMetadataElementName(sort_argument.getValue());
+		if (sort_element_name_full != null && !sort_element_name_full.equals("")) {
+		    String sort_element_display_name_full = MetadataTools.getDisplayNameForMetadataElementWithName(sort_element_name_full);
+		    if (sort_element_display_name_full != null) {
+			sort_argument.setValue(sort_element_display_name_full);
+		    }
+		}
+	    }
+
+	    // Set the "-buttonname" value if it isn't already set
+	    Argument buttonname_argument = classifier.getArgument("-buttonname");
+	    if (buttonname_argument != null && buttonname_argument.getValue().equals("")) {
+		int namespace_index = metadata_argument_value.indexOf(".");
+		if (namespace_index != -1) {
+		    buttonname_argument.setValue(metadata_argument_value.substring(namespace_index + 1));
+		}
+		else {
+		    buttonname_argument.setValue(metadata_argument_value);
+		}
+		buttonname_argument.setAssigned(true);
+	    }
+
+	    // With Hierarchy classifiers, update the hfile arguments
+  	    if (classifier.getName().equalsIgnoreCase(StaticStrings.HIERARCHY_CLASSIFIER)) {
+		// Update the "-hfile" value
+		Argument hfile_argument = classifier.getArgument(StaticStrings.HFILE_ARGUMENT);
+		hfile_argument.setValue(metadata_element_name_full + ".txt");
+  	    }
+
+	    // System.err.println("Classifier (after): " + classifier);
+	}
+    }
+
+
+    static private String mapLegacyMetadataElementName(String metadata_element_name)
+    {
+ 	// Remove the extracted namespace if it has been added
+ 	if (metadata_element_name.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
+ 	    metadata_element_name = metadata_element_name.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
+ 	}
+
+ 	// Update the metadata value to the new (namespaced) one
+	return ProfileXMLFileManager.getMetadataElementFor(metadata_element_name);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/ScriptOptions.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/ScriptOptions.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/collection/ScriptOptions.java	(revision 21112)
@@ -0,0 +1,333 @@
+package org.greenstone.gatherer.collection;
+
+import java.io.*;
+import java.lang.ref.*;
+import java.net.*;
+import java.util.*;
+import org.apache.xerces.parsers.DOMParser;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.cdm.Argument;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+import org.xml.sax.InputSource;
+
+/** Build options uses the argument list found in config.xml and the current settings from the loaded collection configuration to represent the options the user wants to use during import and build. If there are no arguments stored in config.xml, or if the user indicates the argument are out of date, this class tries to parse new ones. */
+public class ScriptOptions {
+    /** The root element of the argument tree. */
+    private Element arguments_element;
+    /** The root element of the values tree. */
+    private Element values_element;
+    /** A cache of previously created arguments. */
+    private Hashtable arguments_cache = new Hashtable();
+    /** whether the values should be saved or not */
+    /** The name of an argument element. */
+    static final private String ARGUMENT = "Argument";
+    /** The name of the enabled attribute. */
+    static final private String ENABLED = "enabled";
+    /** The name of the 'false' value. */
+    static final private String FALSE = "false";
+    /** The name of a name attribute. */
+    static final private String NAME = "name";
+    /** The name of an option element. */
+    static final private String OPTION = "Option";
+    /** The name of the 'true' value. */
+    static final private String TRUE = "true";
+    /** When constructing a ScriptOptions object we first try to retrieve the valid arguments from config.xml - if saved is true. If that fails we then try to parse the arguments noticed when calling script_name -xml. If that also fails, we try to load the default arguments from the xml template library. */
+    public ScriptOptions(Element values_element, String script_name, boolean saved) {
+	// first check the saved arguments language
+	String interface_lang = Configuration.getLanguage();
+	String args_lang = Configuration.getArgumentsLanguage();
+	if (saved && interface_lang.equals(args_lang)) {
+	    // Try to retrieve the arguments for import and build.
+	    arguments_element = Configuration.getArguments(script_name);
+	}
+	// If that fails try to reconstruct the arguments
+	if(arguments_element == null) {
+	    boolean used_defaults = false;
+	    arguments_element = loadArguments(script_name, interface_lang);
+	   
+	    // And if that too fails, load the default argument templates
+	    if(arguments_element == null) {
+		// replace .pl with .xml to get the template name
+		String template_name = script_name.replaceAll("\\.pl$", "\\.xml");
+		Document document = XMLTools.parseXMLFile("xml/" + template_name, true);
+		if (document != null) {
+		    arguments_element = document.getDocumentElement();
+		    document = null;
+		    used_defaults = true;
+
+		} else {
+		    return;
+		}
+		///atherer.println("Loaded default BO arguments from templates.");
+	    }
+	    else {
+		///atherer.println("Loaded BO arguments from scripts.");
+	    }
+	    // By now we should definately have the arguments. However the reason we are here is because they are not stored in the config.xml file, to make sure they are stored now.
+	    if (saved) {
+		if (used_defaults) {
+		    Configuration.setArgumentsLanguage("en");
+		} else {
+		    Configuration.setArgumentsLanguage(interface_lang);
+		}
+		Configuration.setArguments(arguments_element);
+	    }
+	}
+	else {
+	    DebugStream.println("Loaded BO arguments from config.xml");
+	}
+	// Now take a note of the values too.
+	this.values_element = values_element;
+    }
+
+    /** Retrieve the indexth argument */
+    public Argument getArgument(int index) {
+	Argument argument = null;
+	// Try to find the argument in the cache.
+	SoftReference reference = (SoftReference) arguments_cache.get(new Integer(index));
+	if(reference != null) {
+	    argument = (Argument) reference.get();
+	}
+	// Otherwise generate a new argument.
+	if(argument == null) {
+	    NodeList option_list = arguments_element.getElementsByTagName(OPTION);
+	    if(0 <= index && index < option_list.getLength()) {
+		argument = new Argument();
+		argument.parseXML((Element) option_list.item(index));
+	    }
+	}
+	return argument;
+    }
+
+
+    /** Retrieve the number of arguments.
+     */
+    public int getArgumentCount() {
+	// Determining the total count is easy.
+	NodeList argument_elements = arguments_element.getElementsByTagName(OPTION);
+	return argument_elements.getLength();
+    }
+
+    /** Retrieve all the argument names and values  */
+    public String[] getArguments() {
+	String arguments[] = null;
+	NodeList argument_list = null;
+	argument_list = values_element.getElementsByTagName(ARGUMENT);
+	for(int i = 0; i < argument_list.getLength(); i++) {
+	    String[] temp = new String[2];
+	    Element argument = (Element) argument_list.item(i);
+	    ArrayTools.add(arguments, argument.getAttribute(NAME));
+	    String value = XMLTools.getValue(argument);
+	    if(value != null && value.length() > 0) {
+		ArrayTools.add(arguments, value);
+	    }
+	}
+	return arguments;
+    }
+
+    /** Retrieve the value of a certain argument. */
+    public String getValue(String name) {
+	return getValue(name, false);
+    }
+
+    /** Determine if the named argument value is enabled or disabled. */
+    public boolean getValueEnabled(String name) {
+	boolean result = false;
+	String value = getValue(name, true);
+	if(value != null && value.length() > 0) {
+	    result = (value.equalsIgnoreCase(TRUE));
+	}
+	return result;
+    }
+
+    /** Retrieve all of the values as a String array ready to added to the script call. */
+    public String[] getValues() {
+	ArrayList values = new ArrayList();
+	try {
+	    NodeList arguments = values_element.getElementsByTagName(ARGUMENT);
+	    for(int i = 0; i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		// Determine if this argument is enabled.
+		if(argument_element.getAttribute(ENABLED).equalsIgnoreCase(TRUE)) {
+		    // First store the name of the argument prefixed with a '-'
+		    values.add("-" + argument_element.getAttribute(NAME));
+		    // Now retrieve the value.
+		    String argument_value = Codec.transform(XMLTools.getValue(argument_element), Codec.DOM_TO_TEXT);
+		    // If there is a value, tokenize it by commas only.
+		    if(argument_value != null && argument_value.length() > 0) {
+			values.add(argument_value);
+		    }
+		    argument_value = null;
+		}
+		argument_element = null;
+	    }
+	    arguments = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return ArrayTools.arrayListToStringArray(values);
+    }
+
+   /** Remove the named value from the arguments */
+    public void removeValue(String name) {
+	try {
+	    NodeList arguments = values_element.getElementsByTagName(ARGUMENT);
+	    boolean found = false;
+	    for(int i = 0; !found && i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		// Is this the argument we want.
+		if(argument_element.getAttribute(NAME).equalsIgnoreCase(name)) {
+		    values_element.removeChild(argument_element);
+		    found = true;
+		}
+		argument_element = null;
+	    }
+	    arguments = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+    /** Set the state of some argument. Note that value may be either a single String, an ArrayList of Strings or null. If enable is false then any existing argument for the named argument is disabled. */
+    public void setValue(String name, boolean enable, String value) {
+	///ystem.err.println("Set value: " + (arguments_element == build_values_element ? "Build" : "Import") + ", " + name + ", " + enable + ", " + value);
+	try {
+	    Document document = values_element.getOwnerDocument();
+	    NodeList arguments = values_element.getElementsByTagName(ARGUMENT);
+	    boolean found = false;
+	    for(int i = 0; i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		// If this the argument named.
+		if(argument_element.getAttribute(NAME).equalsIgnoreCase(name)) {
+		    found = true;
+		    // Set whether this argument is enabled
+		    argument_element.setAttribute(ENABLED, (enable ? TRUE : FALSE));
+		    // Now we set the value, depending or what it is.
+		    if(value == null) {
+			// Nothing to do.
+		    }
+		    else {
+			// Remove existing text nodes.
+			while(argument_element.hasChildNodes()) {
+			    argument_element.removeChild(argument_element.getFirstChild());
+			}
+			argument_element.appendChild(document.createTextNode((String)value));
+		    }
+		}
+		argument_element = null;
+	    }
+	    // If we haven't found an instance of this argument, but should have, then add it.
+	    if(!found && (enable || value != null)) {
+		Element argument_element = document.createElement(ARGUMENT);
+		argument_element.setAttribute(NAME, name);
+		argument_element.setAttribute(ENABLED, (enable ? TRUE : FALSE));
+		// Now we set the value, depending or what it is.
+		if(value == null) {
+		    // Nothing to do.
+		}
+		else {
+		    argument_element.appendChild(document.createTextNode((String)value));
+		}
+		values_element.appendChild(argument_element);
+	    }
+	    arguments = null;
+	    document = null;
+	    // Make sure the collection knows to save.
+	    Gatherer.c_man.getCollection().setSaved(false);
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+ 
+    private String getValue(String name, boolean is_enabled) {
+	String result = null;
+	try {
+	    NodeList arguments = values_element.getElementsByTagName(ARGUMENT);
+	    for(int i = 0; result == null && i < arguments.getLength(); i++) {
+		Element argument_element = (Element) arguments.item(i);
+		// Is this the argument we want.
+		if(argument_element.getAttribute(NAME).equalsIgnoreCase(name)) {
+		    // Are we simply determining if this argument is enabled
+		    if(is_enabled) {
+			result = argument_element.getAttribute(ENABLED);
+		    }
+		    else {
+			String argument_value = XMLTools.getValue(argument_element);
+			if(argument_value != null) {
+			    result = argument_value;
+			}
+			argument_value = null;
+		    }
+		}
+		argument_element = null;
+	    }
+	    arguments = null;
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+	return result;
+    }
+
+
+    private Element loadArguments(String filename, String lang) {
+	Element arguments_element = null;
+	InputStream input_stream = null;
+
+	// Run the required program.
+	try {
+	    Document document;
+	    if (Gatherer.isGsdlRemote) {
+		String script_output = RemoteGreenstoneServer.getScriptOptions(filename, "");
+		document = XMLTools.parseXML(new StringReader(script_output));
+	    }
+	    else {
+		String args[];
+		if(Utility.isWindows()) {
+		    args = new String[6];
+		    args[0] = Configuration.perl_path;
+		    args[1] = "-S";
+		    args[2] = LocalGreenstone.getBinScriptDirectoryPath() + filename;
+		    args[3] = "-xml";
+		    args[4] = "-language";
+		    args[5] = lang;
+		}
+		else {
+		    args = new String[4];
+		    args[0] = LocalGreenstone.getBinScriptDirectoryPath() + filename;
+		    args[1] = "-xml";
+		    args[2] = "-language";
+		    args[3] = lang;
+		}
+
+		// Create the process.
+		Runtime runtime = Runtime.getRuntime();
+		Process process = runtime.exec(args);
+
+		input_stream = process.getErrorStream();
+		document = XMLTools.parseXML(input_stream);
+	    }
+
+	    arguments_element = document.getDocumentElement();
+	}
+	catch (Exception error) {
+	    DebugStream.println("Error in ScriptOptions.loadArguments(): " + error);
+	    DebugStream.printStackTrace(error);
+	}
+	return arguments_element;
+    }
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ActionRecorderDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ActionRecorderDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ActionRecorderDialog.java	(revision 21112)
@@ -0,0 +1,2135 @@
+package org.greenstone.gatherer.feedback;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.tree.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import java.io.*;
+import java.awt.image.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.border.*;
+import java.text.*;
+
+import org.greenstone.gatherer.util.JarTools;
+
+/**
+ * This is the special class listener. 
+ * <p>This class will record all the action did to any of the
+ * component inside the window that were open by the application. 
+ * <br> This listener will start listening since the beginning of the application 
+ * until the user quits from the application. It will record :
+ * <br>1. All the information about the window and its components where the action occured.
+ * <br>2. All the information about all windows and its components that were open when the action occured.
+ * <br>Note : point 1 and 2 only start when user wants to send feedback.
+ * <br>3. The action's command the user did.
+ * <br>4. The date of when the action occured.
+ * <br><br>The history of the action will be stored in history.log file, which then will be compressed
+ * to a zip file called MyZipFile.zip.</p>
+ * <p>The listener this class implements are only the listener interface that a javax.swing.* component
+ * could have as stated in API.Some of the listener however is not implemented here as it is considered the event
+ * listened by those listener are not essential to be recorded as they not making any major changes
+ * to the application.</p> 
+ * <p>This listener will only record maximum last 60 instructions user did to the application.</p>
+ * <p>In this class also we can add some stuff globally through out all the window open by
+ * the application.
+ * <br>This is the class will add a menu called Feedback through out all the window of type JFrame or JDialog
+ * open by the application.This menu will allow user to either sending feedback or viewing the history
+ * of all the action user did so far.
+ * @author Veronica Liesaputra
+ */
+public class ActionRecorderDialog
+    implements ActionListener,TreeModelListener,ItemListener,ListDataListener,ChangeListener,
+	       ListSelectionListener,MenuListener,TableColumnModelListener,TreeSelectionListener
+{
+    /**
+     * This hold the title of the window where the action occured.
+     */
+    private String myframe;
+
+    /**
+     * This hold the title of the window where user called send feedback.
+     */
+    private String feedbackframe;
+
+    /**
+     * This hold the instance of ComponentInformation to get all the information inside a window.
+     */
+    private ComponentInformation compInfo;
+    
+    /**
+     * This will hold a record of the action that user just did at the moment.
+     */
+    private History log;
+
+    /**
+     * This will reference to the window where the action occured.
+     */
+    private Window point;
+   
+    /**
+     * This hold the vector of all the History instances of all the record of action user did
+     * so far.
+     * This vector will hold the history of all the 30 actions user just did.
+     * There are 2 situation for this vector :
+     * <br>1. Before user choose to send feedback.
+     * <br>Here, If we want to add another record to the vector when the vector size already 30 then,
+     * all the 30 records inside the vector will be saved in the temp_history.log file. The vector will
+     * be cleared and then added with the new record.
+     * <br>If the user quits the application, then whats in the vector will be saved to history.log file.
+     * <br>2. After user choose to send feedback.
+     * <br>When user choose to send feedback,then what happen is what ever inside the vector at the moment
+     * will be saved to history.log file and then vector will be cleared and it will add the new instruction.
+     * <br> If we want to add another record to the vector when the vector size already 30 then,
+     * all the 30 records inside the vector will be saved in the temp_feedbackhist.log file. The vector will
+     * be cleared and then added with the new record.
+     * <br> If the user finished with the feedback then it will cleared all the vector and load the vector
+     * inside the history.log to it and add the new record to the vector.
+     * <br> If the user quits the application, then whats in the vector will not be saved.
+     */
+    private static Vector vector;
+
+    /**
+     * This is the flag to tell whether or not the image for the window where the action occured is
+     * finished being added properly to the appropriate record.
+     * If savefinish = true then it means that the thread of adding image to the record stop, otherwise
+     * it means its not yet finished so don't stop the thread.
+     */
+    private static boolean savefinish = true;
+
+    /**
+     * This is the thread that contains all the code that the application should be when the user exiting
+     * the application.
+     */
+    private Thread writeThread;
+
+    /**
+     * This is the thread that contains all the code to add the image of the window where the action
+     * occured to the proper record.
+     * This thread will stop if savefinish = true.
+     */
+    private Thread saveThread;
+
+    /**
+     * This is the runtime of this application.
+     */
+    private Runtime run;
+
+    /**
+     * This variable will hold the resource of the words that is stored in feedback.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    /**
+     * This is an instance of ScreenShot that will allow this application to take screen shot of the whole
+     * screen or just the screen shot of the window where the actions occured.
+     */
+    private ScreenShot sh;
+
+    /**
+     * This HashMap stored the pair of model and its component.
+     * This will allow this listener to know which component the model where the actions occured belongs to.
+     */
+    private HashMap modelHash;
+
+    /**
+     * This is the flag variable to sign that the user start the Reporting Feedback sequence.
+     * If start_rec = false means that the user did not start reporting feedback sequence.
+     * Otherwise, it means the user start reporting feedback sequence, here there will be changes in the
+     * content of the vector used to save all the records.This also will change the appearance of the
+     * window that are currently open by the application. it will make toolbar value = true, rec = true and
+     * stop_rec = false.
+     */
+    private boolean start_rec = false;
+
+    /**
+     * This is the flag variable to sign that the user are now recording the Reporting Feedback sequence.
+     * If rec = true, it means its recording, otherwise its false.It also will decide
+     * the GUI appearance of the window that are currently open by the application.
+     */
+    private boolean rec = false;
+
+    /**
+     * This is the flag variable to sign that user are finished the Reporting Feedback sequence.
+     * If stop_rec = false, then it means user stop the Reporting Feedback sequence.So here, there will
+     * be changes in the vector's content. If rec = true then it will also delete the added toolbar on the currently open
+     * window by the application and its also will delete all the history and file taken when user doing the
+     * Reporting Feedback sequence.
+     */
+    private boolean stop_rec = true;
+
+    /**
+     * This is the thread do task to take history of the actions.
+     */
+    private Thread saveallThread;
+
+    /**
+     * This is the flag to say whether or not the send feedback button been pushed by user.
+     */
+    private boolean start_fd = false;
+
+    /**
+     * This is a blocker to say that user cannot do anything while we taking history.
+     */
+    private MouseListener mouse_blocker_listener = new MouseAdapter() {};
+
+    /**
+     * This is the feedback interface window.
+     */
+    private FeedbackInterface frame;
+
+    /**
+     * This is an instance of CompListener that will add listener to all the components inside it.
+     */
+    private CompListener compList;
+
+    /**
+     * This is the accelerator key (shortcuts key) for taking screenshot.
+     */
+    private KeyStroke shKey;
+
+    /**
+     * This is the accelerator key (shortcuts key) for reporting feedback.
+     */
+    private KeyStroke fdKey;
+
+    /**
+     * This is the accelerator key (shortcuts key) for viewing history.
+     */
+    private KeyStroke histKey;
+
+    /**
+     * This is the accelerator key (shortcuts key) for sending feedback straight away.
+     */
+    private KeyStroke sndfdKey;
+
+    /**
+     * This is the window that is active at the moment.
+     */
+    private Window activeWindow;
+
+    /**
+     * This is the modal window where user wants to take the screeshot or sending feedback.
+     */
+    private Window modalWindow;
+
+    /**
+     * This constructor will seeting up the data member to its appropriate value.
+     * It will get the vector that are stored in the history.log file and also it will
+     * setup the writeThread.
+     * @param currentLocale is the locale user choose all the text to be displayed as.
+     */
+    public ActionRecorderDialog (Locale currentLocale)
+    {
+	messages = ResourceBundle.getBundle("feedback",currentLocale);
+	setup_UIManager();
+
+	fdKey = KeyStroke.getKeyStroke("F2");
+	histKey = KeyStroke.getKeyStroke("F3");
+	shKey = KeyStroke.getKeyStroke("F5");
+	sndfdKey = KeyStroke.getKeyStroke("F12");
+	/*shKey = KeyStroke.getKeyStroke(KeyEvent.VK_SLASH,InputEvent.CTRL_MASK|
+	  InputEvent.SHIFT_MASK);*/
+
+	final ZipFile zip;
+	zip = new ZipFile();
+	zip.unZipFile();
+	vector = getPrev_log("history.log");
+	sh = new ScreenShot();
+	modelHash = new HashMap();
+	compInfo = new ComponentInformation(this,currentLocale,modelHash);
+        compList = new CompListener(this,modelHash);
+	EventQueue eq;
+	eq = Toolkit.getDefaultToolkit().getSystemEventQueue();
+	eq.push(new MyEventQueue());
+	run = Runtime.getRuntime();
+	Runnable writeRun;
+	writeRun = new Runnable()
+	    {
+		public void run()
+		{
+		    try
+			{
+			    if (stop_rec == true)
+				{
+				    File f;
+				    f = new File("history.log");
+				    FileOutputStream fos;
+				    fos = new FileOutputStream("history.log");
+				    ObjectOutputStream oos;
+				    oos = new ObjectOutputStream(fos);
+				    oos.writeObject(vector);
+				    vector.removeAllElements();
+				    vector = getPrev_log("temp_history.log");
+				    oos.writeObject(vector);
+				    oos.close();
+				    f = new File("temp_history.log");
+				    f.delete();
+				}
+
+			    File f2;
+			    f2 = new File("feedbackhist.log");
+			    f2.delete();
+			    f2 = new File("temp_feedbackhist.log");
+			    f2.delete();
+
+			    zip.sendZipFile();
+			    writeThread = null;
+			}
+		    catch (IOException exp) {}
+		}
+	    };
+
+	writeThread = new Thread(writeRun);
+	run.addShutdownHook(writeThread);
+    }
+
+    /** change the locale for the resource bundle*/
+    public void setLocale(Locale loc) {
+	messages = ResourceBundle.getBundle("feedback",loc);
+    }
+    /**
+     * This method will setup some of the GUI appearance that window should have when the user
+     * choose to start the Reporting Feedback sequence.
+     */
+    public void setup_UIManager ()
+    {
+	// why do you do all this???
+	//UIManager.put("TabbedPane.selected",(new Color(224,240,224)));
+	//UIManager.put("Button.select",(new Color(224,240,224)));
+	//UIManager.put("ScrollBar.background",(new Color(176,208,176)));
+	//UIManager.put("ScrollBar.highlight",(new Color(224,240,224)));
+	//UIManager.put("ScrollBar.thumb",(new Color(224,240,224)));
+	//UIManager.put("ScrollBar.track",(new Color(224,240,224)));
+	//UIManager.put("Tree.textBackground",new Color(224,240,224));
+	//UIManager.put("Label.font",new Font("Dialog",Font.PLAIN,12));
+	//UIManager.put("ComboBox.font",new Font("Dialog",Font.PLAIN,12));
+	//UIManager.put("TextArea.font",new Font("Dialog",Font.PLAIN,12));
+    }
+
+    /**
+     * This method will get the information about all the components inside the window that are
+     * open by the application when the action occured and save it to vector.
+     * @param comp its the component where the user did the action.
+     * @param command its the action's command the user did.
+     */
+    private void saving (final Component comp,final String command)
+    {
+	if ((command.equals(messages.getString("SendFeedback"))) ||
+	    ((command.equals("FastSend")) && (stop_rec == false)))
+	    {
+		start_fd = true;
+	    }
+
+	Runnable saveall;
+	saveall = new Runnable ()
+	    {
+		public void run ()
+		{
+		    try
+			{
+			    Runnable savework;
+			    savework = new Runnable ()
+				{
+				    public void run ()
+				    {
+					Date curr_date;
+					curr_date = new Date();
+					log = new History(curr_date,command);
+
+					if ((start_fd == true)&&(command.equals(messages.getString("WindowOpened"))))
+					    {
+						start_rec = true;
+						rec = true;
+						start_fd = false;
+					    }
+
+					getFrame(comp,command);
+					saveState(command);
+					saveLog();
+
+					if ((point != null) && (rec == true))
+					    {
+						if (command.equals(messages.getString("WindowClosed")))
+						    {
+							if (point instanceof JDialog)
+							    if (((JDialog)point).isModal() == true)
+								if (point == modalWindow)
+								    frame.setComment(false);
+						    }
+					    }
+				    }
+				};
+
+			    SwingUtilities.invokeAndWait(savework);
+			}
+		    catch (Exception ex) {ex.printStackTrace();}
+		}
+	    };
+
+	saveallThread = new Thread(saveall);
+	saveallThread.start();
+    }
+
+    /**
+     * This method will get the screen shot of the window where the actions occured and
+     * add the image also its dimension to the passed record.
+     * It will caused the saveThread to start so it can allows other stuff to be run concurrently
+     * and this thread will be stopped when the screen shot of the window alreadyfinished taken.
+     * @param hst is the record where the image and its dimension should be stored to.
+     */
+    public void saveImage (final History hst)
+    {
+	Runnable saveRun;
+	saveRun = new Runnable()
+	    {
+		History hist = hst;
+		public void run()
+		{
+		    String img,width,height;
+		    Rectangle rect;
+		    int w, h,dimwidth,dimheight;
+		 
+		    if (point != null)
+			{
+			    dimwidth = sh.getWidth();
+			    dimheight = sh.getHeight();
+
+			    w = point.getWidth();
+			    h = point.getHeight();
+
+			    try
+				{
+				    rect = new Rectangle(point.getLocationOnScreen(),
+							 new Dimension(w,h));
+			    
+				    if (w >= dimwidth)
+					w = dimwidth - 50;
+				    
+				    width = "" +  w;
+				    
+				    if (h >= dimheight)
+					h = dimheight - 50;
+				    
+				    height = "" + h;
+				}
+			    catch (IllegalComponentStateException exp)
+				{
+				    rect = null;
+				    width = null;
+				    height = null;
+				}
+			}
+		    else
+			{
+			    rect = null;
+			    width = null;
+			    height = null;
+			}
+		    
+		    if (rect != null)
+			{
+			    img = sh.getImage(rect);
+			    if (savefinish == true)
+				{
+				    hist.setImage(img,height,width);
+				}
+			}
+		}
+	    };
+	
+	Thread saveThread;
+	saveThread = new Thread(saveRun);
+	saveThread.start();
+    }
+
+    /**
+     * This method will setup the savefinish value.
+     * (Precondition: (save != null))
+     * @param save the value savefinish should have.
+     */
+    public static void setSavefinish (boolean save)
+    {
+	savefinish = save;
+    }
+
+
+    /**
+     * This method will make the JMenu that is should be added to all the
+     * window opened by the application.
+     * <br>If rec = false then the JMenu will give user option to start the Reporting Feedback sequence
+     * or Sending feedback or to view the history of actions they have done so far.
+     * <br>If rec = true then the JMenu will only give user option to view the history of actions
+     * they have done so far or sending feedback or taking screenshot.
+     * @return JMenu that should be added to all window open.
+     */
+    public JMenu makeMenu ()
+    {
+	JMenu menu;
+	JMenuItem item,item2,item3;
+	ImageIcon icon,icon2,icon3;
+
+	icon3 = JarTools.getImage("fastsend.gif");
+	item3 = new JMenuItem(messages.getString("SendFeedback"), 
+			      new ImageIcon(icon3.getImage().getScaledInstance
+					    (20,20,Image.SCALE_SMOOTH)));
+	item3.setActionCommand("FastSend");
+	item3.addActionListener(this);
+	item3.setMnemonic(KeyEvent.VK_N);
+	item3.setAccelerator(sndfdKey);
+	item3.setToolTipText("Send feedback now");
+
+	item = null;
+
+	if (rec == false)
+	    {
+		icon = JarTools.getImage("feedback.gif");
+		item = new JMenuItem("Report Feedback",
+				     new ImageIcon(icon.getImage().getScaledInstance
+						   (20,20,Image.SCALE_SMOOTH)));
+		item.setActionCommand(messages.getString("SendFeedback"));
+		item.addActionListener(this);
+		item.setMnemonic(KeyEvent.VK_F);
+		item.setAccelerator(fdKey);
+		item.setToolTipText("Report feedback");
+	    }
+	else
+	    {
+		icon = JarTools.getImage("camera.gif");
+		item = new JMenuItem(messages.getString("ScreenShot"),
+				     new ImageIcon(icon.getImage().getScaledInstance
+						   (20,20,Image.SCALE_SMOOTH)));
+		item.setActionCommand("ScreenShot");
+		item.setMnemonic(KeyEvent.VK_S);
+		item.addActionListener(this);
+		item.setAccelerator(shKey);
+		item.setToolTipText("Take screenshot of the whole screen.");
+	    }
+
+	icon2 = JarTools.getImage("history.gif");
+	item2 = new JMenuItem(messages.getString("LookHistory"),
+			      new ImageIcon(icon2.getImage().getScaledInstance
+					    (20,20,Image.SCALE_SMOOTH)));
+	item2.setActionCommand(messages.getString("LookHistory"));
+	item2.addActionListener(this);
+	item2.setMnemonic(KeyEvent.VK_H);
+	item2.setAccelerator(histKey);
+	item2.setToolTipText(messages.getString("LookHistoryButton"));
+
+	menu = new JMenu (messages.getString("Feedback"));
+	menu.setMnemonic(KeyEvent.VK_D);
+
+
+	if (item != null)
+	    menu.add(item);
+
+	menu.add(item3);
+
+	menu.add(item2);
+
+	if (rec == true)
+	    menu.setBackground(new Color(176,209,217));
+	else
+	    menu.setBackground(new Color(204,204,204));
+
+	return menu;
+    }
+
+    public void refreshLocationAndSize(Window window) {
+
+	window.setLocation(window.getX(),window.getY());
+	
+	Toolkit kit;
+	kit = Toolkit.getDefaultToolkit();
+	Dimension screenSize;
+	screenSize = kit.getScreenSize();
+	int screenHeight;
+	screenHeight = screenSize.height;
+	int screenWidth;
+	screenWidth = screenSize.width;
+	
+	if ((window.getSize().width > screenWidth) &&
+	    (window.getSize().height > screenHeight)) {
+	    window.setSize(window.getPreferredSize());
+	}
+    }
+
+
+    /**
+     * This method will setup the GUI that this window should have at the moment.
+     * If start_rec is true then JMenu don't have the option to start Reporting Feedback sequence in it.
+     * If stop_rec is true and rec is true then JMenu there should be an option to start Reporting Feedback sequence in it.
+     * The JMenu will only be affected window of type JDialog and JFrame only.
+     * (Precondition: (window != null))
+     * @param window its the window that we want to set the GUI
+     */
+    public void setUI (Window window)
+    {	
+	if (start_rec == true) {
+	    if (window instanceof JFrame) {
+		JFrame a;
+		a = (JFrame) window;
+		
+		JMenuBar menuBar;
+		menuBar = a.getJMenuBar();
+		
+		if (menuBar != null) {
+		    
+		    JMenu temp_menu;
+		    temp_menu = menuBar.getMenu(menuBar.getMenuCount() - 1);
+		    temp_menu.setBackground(new Color(176,209,217));
+		    temp_menu.remove(0);
+		    
+		    JMenuItem item;
+		    ImageIcon icon;
+		    icon = JarTools.getImage("camera.gif");
+		    item = new JMenuItem(messages.getString("ScreenShot"),
+					 new ImageIcon(icon.getImage().getScaledInstance
+						       (20,20,Image.SCALE_SMOOTH)));
+		    item.setActionCommand("ScreenShot");
+		    item.setMnemonic(KeyEvent.VK_S);
+		    item.addActionListener(this);
+		    item.setAccelerator(shKey);
+		    item.setToolTipText("Take screenshot of this window");
+		    
+		    temp_menu.insert(item,0);
+		}
+		
+		refreshLocationAndSize(a);
+		a.validate();
+		a.repaint();
+	    }
+	    else if (window instanceof JDialog) {
+		
+		JDialog a;
+		a = (JDialog) window;
+		JMenuBar menuBar;
+		menuBar = a.getJMenuBar();
+		
+		if (menuBar != null) {
+		    
+		    JMenu temp_menu;
+		    temp_menu = menuBar.getMenu(menuBar.getMenuCount() - 1);
+		    temp_menu.setBackground(new Color(176,209,217));
+		    temp_menu.remove(0);
+		    
+		    JMenuItem item;
+		    ImageIcon icon;
+		    icon = JarTools.getImage("camera.gif");
+		    item = new JMenuItem(messages.getString("ScreenShot"),
+					 new ImageIcon(icon.getImage().getScaledInstance
+						       (20,20,Image.SCALE_SMOOTH)));
+		    item.setActionCommand("ScreenShot");
+		    item.setMnemonic(KeyEvent.VK_S);
+		    item.addActionListener(this);
+		    item.setAccelerator(shKey);
+		    item.setToolTipText("Take screenshot of this window");
+		    
+		    temp_menu.insert(item,0);
+		}
+		refreshLocationAndSize(a);
+		a.validate();
+		a.repaint();	
+	    }
+	    else if (window instanceof JWindow) {
+		JWindow a;
+		a = (JWindow) window;
+		
+		((JComponent) a.getContentPane()).unregisterKeyboardAction(fdKey);
+		
+		((JComponent) a.getContentPane()).registerKeyboardAction(this, "ScreenShot", shKey,JComponent.WHEN_IN_FOCUSED_WINDOW);
+	    }
+	    else
+		return;
+	}
+	else {
+	    
+	    if ((stop_rec == true) && (rec == true)) {
+		
+		if (window instanceof JFrame) {
+		    JFrame a;
+		    a = (JFrame) window;
+		    
+		    JMenuBar menuBar;
+		    menuBar = a.getJMenuBar();
+		    
+		    if (menuBar != null) {
+			
+			JMenu temp_menu;
+			temp_menu = menuBar.getMenu(menuBar.getMenuCount() - 1);
+			temp_menu.setBackground(new Color(204,204,204));
+			temp_menu.remove(0);
+			
+			ImageIcon icon;
+			JMenuItem item;
+			
+			icon = JarTools.getImage("feedback.gif");
+			item = new JMenuItem("Report Feedback", new ImageIcon(icon.getImage().getScaledInstance(20,20,Image.SCALE_SMOOTH)));
+			item.setActionCommand(messages.getString("SendFeedback"));
+			item.addActionListener(this);
+			item.setMnemonic(KeyEvent.VK_F);
+			item.setAccelerator(fdKey);
+			item.setToolTipText(messages.getString("SendFeedbackButton"));
+			
+			temp_menu.insert(item,0);
+		    }
+		    a.setSize(a.getPreferredSize());
+		    a.validate();
+		    a.repaint();
+		}
+		else if (window instanceof JDialog) {
+		    JDialog a;
+		    a = (JDialog) window;
+		    
+		    JMenuBar menuBar;
+		    menuBar = a.getJMenuBar();
+		    
+		    if (menuBar != null) {
+			
+			JMenu temp_menu;
+			temp_menu = menuBar.getMenu(menuBar.getMenuCount() - 1);
+			temp_menu.setBackground(new Color(204,204,204));
+			temp_menu.remove(0);
+			
+			ImageIcon icon;
+			JMenuItem item;
+			
+			icon = JarTools.getImage("feedback.gif");
+			item = new JMenuItem("Report Feedback", new ImageIcon(icon.getImage().getScaledInstance(20,20,Image.SCALE_SMOOTH)));
+			item.setActionCommand(messages.getString("SendFeedback"));
+			item.addActionListener(this);
+			item.setMnemonic(KeyEvent.VK_F);
+			item.setAccelerator(fdKey);
+			item.setToolTipText(messages.getString("SendFeedbackButton"));
+			
+			temp_menu.insert(item,0);
+		    }
+		    a.setSize(a.getPreferredSize());
+		    a.validate();
+		    a.repaint();
+		}
+		else if (window instanceof JWindow) {
+		    JWindow a;
+		    a = (JWindow) window;
+		    
+		    ((JComponent) a.getContentPane()).registerKeyboardAction(this, messages.getString("SendFeedback"), fdKey, JComponent.WHEN_IN_FOCUSED_WINDOW);
+		    
+		    ((JComponent) a.getContentPane()).unregisterKeyboardAction(shKey);
+		    
+		}
+		else
+		    return;
+	    }
+	    else
+		return;
+	}
+    }
+    
+    /**
+     * This will make an Window[] convert to ArrayList that contain list of Window.
+     * (Precondition: (windows != null))
+     * @param windows the array that we want to convert to ArrayList
+     * @return        the array list that contains all object in array windows.
+     */
+    public ArrayList addToArrayList (Window[] windows,ArrayList arr)
+    {
+	int i;
+
+	for (i = 0 ; i < windows.length ; i++)
+	    {
+		arr.add(windows[i]);
+	    }
+	
+	return arr;
+    }
+
+    /**
+     * This method will get all the information of the window that are currently open when the action
+     * occured and these windows are not the window where the action occured.
+     * It will add these windows information to the current record and it will also setting up the 
+     * GUI for each of these windows.
+     * @param command the action's command user did.
+     */
+    public void saveState(String command)
+    {
+	Window[] frame_windows;
+	ArrayList windows;
+	Frame[] frames;
+	int i,j;
+
+	if (log == null)
+	    return;
+	
+	windows = new ArrayList();
+
+	frames = Frame.getFrames();
+	for (i = 0 ; i<frames.length;i++) 
+	    {
+		frame_windows = frames[i].getOwnedWindows();
+		windows = addToArrayList(frame_windows,windows);
+		
+		for (j = 0;j < frame_windows.length ; j++)
+		    {
+			Window window;
+			window = frame_windows[j];
+
+			if (window == point)
+			    {
+				Container cont;
+				cont = window.getParent();
+
+				if ((cont != null) && (cont instanceof Window) &&
+				    (windows.contains(cont) == false)&&(cont != point))
+				    {
+					if (cont instanceof ReportDetails.View) {}
+					else if (cont instanceof Movie) {}
+					else
+					    {
+						setUI((Window) cont);
+						
+						/*if (rec == true)
+						    {
+							log.addComponent(compInfo.getInside((Window) cont),
+									 messages.getString("CurrentlyOpenWindow"));
+						    }
+						    else*/
+						    compList.getInside((Window) cont);
+						
+						windows.add(cont);
+					    }
+				    }	
+				else
+				    {}
+			    }
+			else
+			    {
+				if (window instanceof ReportDetails.View) {}
+				else if (window instanceof Movie) {}
+				else
+				    {
+					if ((window != null) && (window != point))
+					    {
+						setUI(window);
+						
+						if (rec == true)
+						    {
+							log.addComponent(compInfo.getInside((Window) window),
+									 messages.getString("CurrentlyOpenWindow"));
+						    }
+						else
+						    compList.getInside((Window) window);
+						
+						Container cont;
+						cont = window.getParent();
+						
+						if ((cont != null) && (cont instanceof Window) &&
+						    (cont != point) && (windows.contains(cont) == false))
+						    {
+							if (cont instanceof ReportDetails.View) {}
+							else if (cont instanceof Movie) {}
+							else
+							    {
+								setUI((Window)cont);
+								
+								/*if (rec == true)
+								    {
+									log.addComponent(compInfo.getInside((Window) cont),
+											 messages.getString("CurrentlyOpenWindow"));
+								    }
+								    else*/
+								    compList.getInside((Window) cont);
+
+								windows.add(cont);
+							    }
+						    }
+						else {}
+					    }
+				    }
+			    }
+		    }
+	    }
+    }
+
+    /**
+     * This method will get all the information about the window where the actions occured and it also will setting up
+     * how hthe GUI should look like in this window and it will also take the image of this window.
+     * @param comp is the component where the actions happened.
+     * @param command is the command of user's action
+     */
+    public void getFrame (Component comp,String command)
+    {
+	boolean cls;
+	
+	cls = false;
+	point = null;
+	
+	Container frame; 
+	myframe = null;
+	
+	if (comp == null)
+	    return;
+	else if (comp instanceof Window)
+	    frame = (Container) comp;
+	else
+	    frame = SwingUtilities.getWindowAncestor(comp);
+	
+	if (frame instanceof Window)
+	    {
+		JMenu menu;
+		point = (Window) frame;
+
+		point.addMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+
+		
+		if ((frame instanceof ReportDetails.View)||
+		    (frame instanceof Movie))
+		    {
+			log = null;
+			return;
+		    }
+		
+		if (command.equals(messages.getString("WindowClosed")))
+		    {
+			cls = true;
+			if (frame instanceof FeedbackInterface)
+			    {
+				stop_rec = true;
+			    }
+			else
+			    {}
+		    }
+		
+		if  (command.equals(messages.getString("WindowOpened")))
+		    {
+			menu = makeMenu();
+		    }
+		else
+		    {
+			menu = null;
+		    }
+		
+		if (frame instanceof JFrame)
+		    {
+			JFrame a;
+			a = (JFrame) frame;
+			
+			myframe = a.getTitle();
+			
+			if (menu != null)
+			    {
+				JMenuBar menuBar;
+				menuBar = a.getJMenuBar();
+				
+				if (menuBar == null)
+				    {
+					menuBar = new JMenuBar();
+					menuBar.add(menu);
+					a.setJMenuBar(menuBar);
+				    }
+				else
+				    menuBar.add(menu);
+			    }
+			
+			if ((menu != null))
+			    {
+				refreshLocationAndSize(a);
+				a.validate();
+				a.repaint();
+			    }
+		    }
+		else if (frame instanceof JDialog)
+		    {   
+			JDialog a;
+			a = (JDialog) frame;
+			
+			myframe = a.getTitle();
+			
+			if (menu != null)
+			    {
+				JMenuBar menuBar;
+				menuBar = a.getJMenuBar();
+				
+				if (menuBar == null)
+				    {
+					menuBar = new JMenuBar();
+					menuBar.add(menu);
+					a.setJMenuBar(menuBar);
+				    }
+				else
+				    menuBar.add(menu);
+			    }
+		    
+			if ((menu != null))
+			    {
+				refreshLocationAndSize(a);
+				a.validate();
+				a.repaint();
+			    }
+		    }
+		else if (frame instanceof JWindow)
+		    {
+			JWindow a;
+			a = (JWindow) frame;
+			
+			if (menu != null)
+			    {
+				((JComponent) a.getContentPane()).registerKeyboardAction(this,
+											 messages.getString("LookHistory"),
+											 histKey,
+											 JComponent.WHEN_IN_FOCUSED_WINDOW);
+				
+				if (rec == false)
+				    {
+					((JComponent) a.getContentPane()).registerKeyboardAction(this,
+												 messages.getString("SendFeedback"),
+												 fdKey,
+												 JComponent.WHEN_IN_FOCUSED_WINDOW);
+					
+					((JComponent) a.getContentPane()).unregisterKeyboardAction(shKey);
+				    }
+				else
+				    {
+					((JComponent) a.getContentPane()).unregisterKeyboardAction(fdKey);
+					
+					((JComponent) a.getContentPane()).registerKeyboardAction(this,
+												 "ScreenShot",
+												shKey,
+												 JComponent.WHEN_IN_FOCUSED_WINDOW);
+				    }
+			    }
+
+			if (cls == true)
+			    {
+				((JComponent) a.getContentPane()).unregisterKeyboardAction(fdKey);
+				
+				((JComponent) a.getContentPane()).unregisterKeyboardAction(shKey);
+			    }
+		    }
+		
+		/*if ((command.equals(messages.getString("WindowClosed")))||
+		  (log == null))
+		  {}
+		  else
+		  saveImage(log);*/
+
+		/*if (rec == true)
+		    log.addComponent(compInfo.getInside(point),command);
+		    else*/
+		    compList.getInside(point);
+
+		if (myframe == null)
+		    myframe = " ";
+		
+		log.setTitle(myframe);
+		
+		return;
+	    }
+	else
+	    return;
+    }
+
+    /**
+     * This method will add the current record to the vector.
+     */
+    public void saveLog()
+    {
+	if (start_rec == true)
+	    {
+		save_HistLog("history.log");
+		vector.removeAllElements();
+		System.gc();
+		start_rec = false;
+		stop_rec = false;
+		rec = true;
+	    }
+	
+	if (log != null)
+	    vector.add(0,log);
+
+	if (vector.size() > 30)
+	    {
+		if (rec == false)
+		    {
+			save_tempHistLog("history.log");
+		    }
+		else
+		    save_tempHistLog("feedbackhist.log");
+		vector.removeAllElements();
+		//vector.removeElementAt(vector.size() - 1);
+		System.gc();
+	    }
+
+	if ((rec == true) && (stop_rec == true))
+	    {
+		save_HistLog("feedbackhist.log");
+		vector.removeAllElements();
+		System.gc();
+		vector = getPrev_log("history.log");
+		File f2;
+		f2 = new File("feedbackhist.log");
+		f2.delete();
+		f2 = new File("temp_feedbackhist.log");
+		f2.delete();
+		start_rec = false;
+		stop_rec = true;
+		rec = false;
+	    }
+
+	if (point != null)
+	    {
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+		point.removeMouseListener(mouse_blocker_listener);
+	    }
+    }
+
+    /**
+     * This method will save whatever in the vector at the moment to a file
+     * with the specified filename.
+     * Then it will also save the vector inside the file with the name "temp_"
+     * concatenated with the specified filename into it.
+     * (Precondition : (filename != null))
+     * @param filename its the name of the file where the vector should be saved.
+     */
+    public void save_HistLog(String filename)
+    {
+	 try 
+	     {
+		 File f;
+		 f = new File(filename);
+		 FileOutputStream fos;
+		 fos = new FileOutputStream(f);
+		 ObjectOutputStream oos;
+		 oos = new ObjectOutputStream(fos);
+		 oos.writeObject(vector);
+		 vector.removeAllElements();
+		 System.gc();
+		 vector = getPrev_log("temp_"+ filename);
+		 oos.writeObject(vector);
+		 oos.close();
+		 f = new File("temp_" + filename);
+		 f.delete();
+		 writeThread = null;
+	     }
+	 catch (IOException exp) {}
+    }
+   
+    /**
+     * This method will save whatever in the vector at the moment to a file with a 
+     * name "temp_" concatenated with the specified filename into it.
+     * (Precondition : (filename != null))
+     * @param filename its part of the name of the file where the vector should be saved.
+     */
+    public void save_tempHistLog(String filename)
+    {
+	try
+	    {
+		File f;
+		f = new File("temp_"+ filename);
+		FileOutputStream fos;
+		fos = new FileOutputStream(f);
+		ObjectOutputStream oos;
+		oos = new ObjectOutputStream(fos);
+		oos.writeObject(vector);
+		oos.close();
+	    }
+	catch (IOException exp) {}
+    }
+
+    /**
+     * This method will get vector that is stored in a file with the specified filename and
+     * it will also save the second vector in that file to a file with a name
+     * temp_history.log.
+     * If file with the specified filename does not exist then it will return a new vector.
+     * (Precondition : (filename != null))
+     * @param filename is the name of the file where we want to get the vector.
+     * @return         the first vector saved in the file with the specified filename.
+     */
+    public Vector getPrev_log(String filename)
+    {
+	Vector stack;
+	ObjectInputStream ois;
+	stack = null;
+	try
+	    {
+		File f;
+		f = new File(filename);
+		if (f.exists() == true)
+		    {
+			FileInputStream fis;
+			fis = new FileInputStream(f);
+			ois = new ObjectInputStream(fis);
+			stack = (Vector) ois.readObject();
+
+			Vector stack2;
+			stack2 = null;
+
+			try
+			    {
+				stack2 = (Vector) ois.readObject();
+			    }
+			catch (EOFException e) {stack2 = null;}
+			
+			if (stack2 != null)
+			    {
+				File f2;
+				f2 = new File("temp_history.log");
+				FileOutputStream fos;
+				fos = new FileOutputStream(f2);
+				ObjectOutputStream oos;
+				oos = new ObjectOutputStream(fos);
+				oos.writeObject(stack2);
+				oos.close();
+				stack2.removeAllElements();
+				stack2 = null;
+				System.gc();
+			    }
+
+			ois.close();
+			return stack;
+		    }
+		else
+		    stack = new Vector(); 
+	    }
+	catch (IOException exp) {exp.printStackTrace();}
+	catch (ClassNotFoundException exp2) {System.out.println("class exp");}
+	
+	return stack;
+    }
+ 
+    /**
+     * This method will make sure that there is something running at the moment.
+     */
+    private static void ensureEventThread()
+    {
+	if (SwingUtilities.isEventDispatchThread())
+	    {
+		return;
+	    }
+	throw new RuntimeException("something running");
+    }
+    
+    /**
+     * This method will called the Conformation window to show up.
+     * @param err_array this is the array containing all the information taken when user doing the Feedback form.
+     * @param imgFile this is the array conataining the String representation of the images taken when user doing the 
+     *                Feedback form.
+     */
+    /*public void getVector(String[] err_array,String[] imgFile)
+    {
+	Conformation rd;
+	rd = new Conformation(vector,err_array,imgFile,messages);
+	return;
+	}*/
+
+    /**
+     * This method will get the vector that hold the history.
+     * @return the vector that hold the history stored so far.
+     */
+    public Vector getVector()
+    {
+	return vector;
+    }
+
+    /**
+     * This class will catch the window opened and window closed event.
+     * If an application open or close a window it will be listened by this class so the information
+     * about that window and all the other open window can be took.
+     */
+    class MyEventQueue extends EventQueue
+    { 
+	/**
+	 * This method  will catch the window opened and window closed event.
+	 * If an application open or close a window it will be listened by this class so the information
+	 * about that window and all the other open window can be took.
+	 * @param evt its the event.
+	 */
+	protected void dispatchEvent(AWTEvent evt) 
+	{
+	    super.dispatchEvent(evt);
+	    int id;
+	    id = evt.getID();
+	
+	    if ((id == WindowEvent.WINDOW_OPENED)||(id == WindowEvent.WINDOW_CLOSED))
+		{   
+		    String command;
+		    command = " ";
+		    
+		    if (id == WindowEvent.WINDOW_OPENED) command = messages.getString("WindowOpened");
+		    else if (id == WindowEvent.WINDOW_CLOSED) command = messages.getString("WindowClosed");
+		   
+		    WindowEvent e;
+		    e = (WindowEvent) evt;
+		    Window comp;
+		    comp =  e.getWindow();
+		 
+		    if ((comp instanceof ReportDetails.View)||
+			(comp instanceof Movie))
+			return;
+
+		    saving(comp,command);
+		}
+	}
+    }
+
+    /**
+     * This method will take the screenshot of the whole screen.
+     * @param p the window where user select to take screenshot.
+     * @return  the screenshot image.
+     */
+    //public byte[] recreateEvent (final Window p)
+    public BufferedImage recreateEvent (final Window p)
+    {
+	setup_UIManager();
+	Rectangle rect;
+	int w, h;
+	
+	if (p != null)
+	    {
+		w = p.getWidth();
+		h = p.getHeight();
+		
+		try
+		    {
+			rect = new Rectangle(p.getLocationOnScreen(),
+				 	     new Dimension(w,h));
+		    }
+		catch (IllegalComponentStateException exp)
+		    {
+			rect = null;
+		    }
+	    }
+	else
+	    {
+		rect = null;
+	    }
+	
+	if (rect != null)
+	    {
+		//byte[] img;
+		BufferedImage img;
+		img = sh.captureScreen();
+		
+		return img;
+	    }
+	else
+	    return null;
+    }
+
+    /**
+     * This method will listen to any action event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ActionEvent.
+     */
+    public void actionPerformed(ActionEvent e)
+    {
+	String command;
+	command = e.getActionCommand();
+	Component comp;
+	comp = (Component)e.getSource();
+	
+	saving(comp,command);
+
+	if (myframe == null)
+	    myframe = " ";
+	
+	if("FastSend".equals(e.getActionCommand()))
+	    {
+		point.addMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+
+		BufferedImage img;
+		boolean modal;
+			
+		modal = false;
+
+		img = recreateEvent(point);
+		
+		if (stop_rec == true)
+		    {
+			feedbackframe = myframe;
+			
+			JDialog.setDefaultLookAndFeelDecorated(true);
+			stop_rec = false;
+			frame = null;
+			frame = new FeedbackInterface(sh,feedbackframe,messages,this);
+			if ((img != null) && (savefinish == true))
+			    {
+				int x,y,w,h,iw,ih;
+				double wrat,hrat;
+				Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+				
+				iw = (int) (dim.getWidth() - 60);
+				ih = (int) (dim.getHeight() * 0.75) - 50;
+				
+				wrat = iw/((double)dim.getWidth());
+				hrat = ih/((double)dim.getHeight());
+				
+				w = (int) (point.getWidth() * wrat);
+				h = (int) (point.getHeight() * hrat);
+				x = (int) (point.getLocationOnScreen().x * wrat);
+				y = (int) (point.getLocationOnScreen().y * hrat);
+
+				//frame.addScreenPanel(feedbackframe,sh.getBuffImage(),img);
+				frame.addScreenPanel(feedbackframe,img,x,y,w,h);
+			    }
+			frame.setVisible(false);
+		    }
+		else
+		    {
+			if ((img != null) &&(savefinish == true))
+			    {
+				int x,y,w,h,iw,ih;
+				double wrat,hrat;
+				Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+				
+				iw = (int) (dim.getWidth() - 60);
+				ih = (int) (dim.getHeight() * 0.75) - 50;
+				
+				wrat = iw/((double)dim.getWidth());
+				hrat = ih/((double)dim.getHeight());
+				
+				w = (int) (point.getWidth() * wrat);
+				h = (int) (point.getHeight() * hrat);
+				x = (int) (point.getLocationOnScreen().x * wrat);
+				y = (int) (point.getLocationOnScreen().y * hrat);
+				
+				//frame.addScreenPanel(feedbackframe,sh.getBuffImage(),img);
+				if (point instanceof JDialog)
+				    modal = ((JDialog) point).isModal();
+				
+				feedbackframe = myframe;
+				frame.addScreenPanel(feedbackframe,img,x,y,w,h);
+			    }
+		    }
+		
+		ActionEvent evt;
+		evt = new ActionEvent(frame.send_button,ActionEvent.ACTION_PERFORMED,messages.getString("Send"));
+		
+		frame.actionPerformed(evt);
+
+		point.removeMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));		 
+	    }
+
+	if(messages.getString("SendFeedback").equals(e.getActionCommand()))
+	    {
+		setup_UIManager();
+		//byte[] img;
+		BufferedImage img;
+		boolean modal;
+
+		point.addMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+
+		modal = false;
+		
+		img = recreateEvent(point);
+		feedbackframe = myframe;
+		if (point instanceof JDialog)
+		    modal = ((JDialog)point).isModal();
+			
+		JDialog.setDefaultLookAndFeelDecorated(true);
+		stop_rec = false;
+		frame = null;
+		frame = new FeedbackInterface(sh,feedbackframe,messages,this);
+		if ((img != null) && (savefinish == true))
+		    {
+			int x,y,w,h,iw,ih;
+			double wrat,hrat;
+			Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+
+			iw = (int) (dim.getWidth() - 60);
+			ih = (int) (dim.getHeight() * 0.75) - 50;
+			
+			wrat = iw/((double)dim.getWidth());
+			hrat = ih/((double)dim.getHeight());
+			
+			w = (int) (point.getWidth() * wrat);
+			h = (int) (point.getHeight() * hrat);
+			x = (int) (point.getLocationOnScreen().x * wrat);
+			y = (int) (point.getLocationOnScreen().y * hrat);
+		
+			//frame.addScreenPanel(feedbackframe,sh.getBuffImage(),img);
+			frame.addScreenPanel(feedbackframe,img,x,y,w,h);
+			frame.setComment(modal);
+			
+			if (modal = true)
+			    {
+				modalWindow = point;
+			    }
+		    }
+
+		point.removeMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));	
+	    }
+	
+	if ("ScreenShot".equals(e.getActionCommand()))
+	    {
+		point.addMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+
+		//byte[] img;
+		BufferedImage img;
+		boolean modal;
+		modal = false;
+		img = recreateEvent(point);
+		
+		if ((img != null) &&(savefinish == true))
+		    {
+			int x,y,w,h,iw,ih;
+			double wrat,hrat;
+			Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+
+			iw = (int) (dim.getWidth() - 60);
+			ih = (int) (dim.getHeight() * 0.75) - 50;
+			
+			wrat = iw/((double)dim.getWidth());
+			hrat = ih/((double)dim.getHeight());
+			
+			w = (int) (point.getWidth() * wrat);
+			h = (int) (point.getHeight() * hrat);
+			x = (int) (point.getLocationOnScreen().x * wrat);
+			y = (int) (point.getLocationOnScreen().y * hrat);
+
+			//frame.addScreenPanel(feedbackframe,sh.getBuffImage(),img);
+			if (point instanceof JDialog)
+			    modal = ((JDialog) point).isModal();
+
+			feedbackframe = myframe;
+			frame.addScreenPanel(feedbackframe,img,x,y,w,h);
+			frame.setComment(modal);
+
+			if (modal == true)
+			    modalWindow = point;
+		    }
+
+		point.removeMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));	
+	    }
+
+	if (messages.getString("LookHistory").equals(e.getActionCommand()))
+	    {
+		point.addMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+
+		setup_UIManager();
+		ReportDetails rd;
+		rd = new ReportDetails(vector,null,null,messages,rec);
+
+		point.removeMouseListener(mouse_blocker_listener);
+		point.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));	
+	    }
+    }
+   
+
+    /**
+     * This method will listen to any Change event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ChangeEvent.
+     */
+    public void stateChanged(ChangeEvent e)
+    {
+	String command;
+	command = messages.getString("ChangeState");
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any tree selection event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TreeSelectionEvent.
+     */
+    public void valueChanged(TreeSelectionEvent e)
+    {
+	MutableTreeNode node;
+        node = (MutableTreeNode)
+	    (e.getPath().getLastPathComponent());
+	
+	if (node == null) return;
+
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	String command;
+	command = messages.getString("Node") + " ";
+	if (node instanceof DefaultMutableTreeNode)
+	    command = command + ((DefaultMutableTreeNode)node).getUserObject() + " ";
+	command = command + messages.getString("ValueChanged");
+	
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any tree nodes change in the tree model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TreeModelEvent.
+     */
+    public void treeNodesChanged(TreeModelEvent e)
+    {
+	MutableTreeNode node;
+        node = (MutableTreeNode)
+                 (e.getTreePath().getLastPathComponent());
+
+	try {
+            int index;
+	    index = e.getChildIndices()[0];
+            node = (MutableTreeNode)(node.getChildAt(index));
+        } catch (NullPointerException exc) {}
+
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("Node");
+
+	if (node instanceof DefaultMutableTreeNode)
+	    command = command + " "  + ((DefaultMutableTreeNode)node).getUserObject();
+
+	command = command + " " + messages.getString("Change");
+	
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any tree nodes inserted in the tree model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TreeModelEvent.
+     */
+    public void treeNodesInserted(TreeModelEvent e)
+    {
+	MutableTreeNode node;
+        node = (MutableTreeNode)
+                 (e.getTreePath().getLastPathComponent());
+
+	try {
+            int index;
+	    index = e.getChildIndices()[0];
+            node = (MutableTreeNode)(node.getChildAt(index));
+        } catch (NullPointerException exc) {}
+	
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	String command;
+	command = messages.getString("Node");
+	if (node instanceof DefaultMutableTreeNode)
+	    command = command + " "  + ((DefaultMutableTreeNode)node).getUserObject();
+
+	command = command + " " + messages.getString("Inserted");
+	
+	saving(comp,command);
+    }
+    
+    /**
+     * This method will listen to any tree nodes removed in the tree model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TreeModelEvent.
+     */
+    public void treeNodesRemoved(TreeModelEvent e)
+    {
+	MutableTreeNode node;
+        node = (MutableTreeNode)
+                 (e.getTreePath().getLastPathComponent());
+
+	try {
+            int index;
+	    index = e.getChildIndices()[0];
+            node = (MutableTreeNode)(node.getChildAt(index));
+        } catch (NullPointerException exc) {}
+
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("Node");
+	if (node instanceof DefaultMutableTreeNode)
+	    command = command + " "  + ((DefaultMutableTreeNode)node).getUserObject();
+	command = command + " " + messages.getString("Removed");
+	
+	saving(comp,command);
+    }
+ 
+    /**
+     * This method will listen to any tree structure change in the tree model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TreeModelEvent.
+     */
+    public void treeStructureChanged(TreeModelEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	String command;
+	command = messages.getString("TreeStructuredChanged");
+	
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any item state change in the item  event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ItemEvent.
+     */
+    public void itemStateChanged(ItemEvent e)
+    {
+	Component comp;
+	comp = (Component) e.getSource();
+	String command;
+	command = ((String) e.getItem()) + " " + 
+	    messages.getString("ItemStateChanged");
+
+	saving(comp,command);
+    } 
+
+    /**
+     * This method will listen to any contents change in the list data event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ListDataEvent.
+     */
+    public void contentsChanged (ListDataEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	String command;
+	command = messages.getString("ListChanged");
+
+	saving(comp,command);
+    } 
+
+    /**
+     * This method will listen to any interval added in the list data event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ListDataEvent.
+     */
+    public void intervalAdded (ListDataEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+	
+	String command;
+	command = messages.getString("ListContentsAdded");
+
+	saving(comp,command);
+    } 
+
+    /**
+     * This method will listen to any interval removed in the list data event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ListDataEvent.
+     */
+    public void intervalRemoved (ListDataEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("ListContentRemoved");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any value changed in the list data event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the ListDataEvent.
+     */
+    public void valueChanged (ListSelectionEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("ListValueChanged");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any menu canceled in the menu event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the MenuEvent.
+     */
+    public void menuCanceled (MenuEvent e)
+    {
+	Component comp;
+	comp = (Component) e.getSource();
+	String command;
+	command = messages.getString("MenuCanceled");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any menu deselected in the menu event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the MenuEvent.
+     */
+    public void menuDeselected (MenuEvent e)
+     {
+	Component comp;
+	comp = (Component) e.getSource();
+	String command;
+	command = messages.getString("MenuDeselected");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any menu selected in the menu event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the MenuEvent.
+     */
+    public void menuSelected(MenuEvent e)
+    {
+	Component comp;
+	comp = (Component) e.getSource();
+	String command;
+	command = messages.getString("MenuSelected");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any column added in the table column model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TableColumnModelEvent.
+     */
+    public void columnAdded (TableColumnModelEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("ColumnIndex") + " "  + e.getToIndex() 
+	    + messages.getString("AddedToTable");
+
+	saving(comp,command);
+    }
+
+    public void columnMarginChanged(ChangeEvent e) {}
+
+    /**
+     * This method will listen to any column moved in the table column model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TableColumnModelEvent.
+     */
+    public void columnMoved(TableColumnModelEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("ColumnIndexFrom") + " " + 
+	    e.getFromIndex() + " " + 
+	    messages.getString("MovedToColumnIndex") + " "  + e.getToIndex() +
+	    " " + messages.getString("InTheTable");
+
+	saving(comp,command);
+    }
+
+    /**
+     * This method will listen to any column removed in the table column model event that occured.
+     * It will allow this special listener class to record all the information inside all the
+     * window open when the action occured.
+     * @param e the TableColumnModelEvent.
+     */
+    public void columnRemoved(TableColumnModelEvent e)
+    {
+	Object obj;
+	obj = e.getSource();
+	
+	Component comp;
+
+	if (obj instanceof Component)
+	    {
+		comp = (Component)e.getSource();
+	    }
+	else
+	    {
+		if (modelHash.containsKey(obj) == true)
+		    {
+			comp = (Component) modelHash.get(obj);
+		    }
+		else
+		    {
+			comp = null;
+			System.out.println("weird" + obj.toString());
+		    }
+	    }
+
+	String command;
+	command = messages.getString("ColumnIndex") + " "  + e.getFromIndex() +
+	    " " + messages.getString("RemovedFromTheTable");
+
+	saving(comp,command);
+    }
+
+    public void columnSelectionChanged(ListSelectionEvent e) {}
+}
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Base64.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Base64.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Base64.java	(revision 21112)
@@ -0,0 +1,1281 @@
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+/**
+ * Encodes and decodes to and from Base64 notation.
+ *
+ * <p>
+ * Change Log:
+ * </p>
+ * <ul>
+ *  <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the
+ *   encoded data was a single byte.</li>
+ *  <li>v2.0 - I got rid of methods that used booleans to set options. 
+ *   Now everything is more consolidated and cleaner. The code now detects
+ *   when data that's being decoded is gzip-compressed and will decompress it
+ *   automatically. Generally things are cleaner. You'll probably have to
+ *   change some method calls that you were making to support the new
+ *   options format (<tt>int</tt>s that you "OR" together).</li>
+ *  <li>v1.5.1 - Fixed bug when decompressing and decoding to a             
+ *   byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>.      
+ *   Added the ability to "suspend" encoding in the Output Stream so        
+ *   you can turn on and off the encoding if you need to embed base64       
+ *   data in an otherwise "normal" stream (like an XML file).</li>  
+ *  <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself.
+ *      This helps when using GZIP streams.
+ *      Added the ability to GZip-compress objects before encoding them.</li>
+ *  <li>v1.4 - Added helper methods to read/write files.</li>
+ *  <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li>
+ *  <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream
+ *      where last buffer being read, if not completely full, was not returned.</li>
+ *  <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li>
+ *  <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li>
+ * </ul>
+ *
+ * <p>
+ * I am placing this code in the Public Domain. Do with it as you will.
+ * This software comes with no guarantees or warranties but with
+ * plenty of well-wishing instead!
+ * Please visit <a href="http://iharder.net/xmlizable">http://iharder.net/base64</a>
+ * periodically to check for updates or to contribute improvements.
+ * </p>
+ *
+ * @author Robert Harder
+ * @author rob@iharder.net
+ * @version 2.0
+ */
+public class Base64
+{
+    
+/* ********  P U B L I C   F I E L D S  ******** */   
+    
+    
+    /** No options specified. Value is zero. */
+    public final static int NO_OPTIONS = 0;
+    
+    /** Specify encoding. */
+    public final static int ENCODE = 1;
+    
+    
+    /** Specify decoding. */
+    public final static int DECODE = 0;
+    
+    
+    /** Specify that data should be gzip-compressed. */
+    public final static int GZIP = 2;
+    
+    
+    /** Don't break lines when encoding (violates strict Base64 specification) */
+    public final static int DONT_BREAK_LINES = 8;
+    
+    
+/* ********  P R I V A T E   F I E L D S  ******** */  
+    
+    
+    /** Maximum line length (76) of Base64 output. */
+    private final static int MAX_LINE_LENGTH = 76;
+    
+    
+    /** The equals sign (=) as a byte. */
+    private final static byte EQUALS_SIGN = (byte)'=';
+    
+    
+    /** The new line character (\n) as a byte. */
+    private final static byte NEW_LINE = (byte)'\n';
+    
+    
+    /** The 64 valid Base64 values. */
+    private final static byte[] ALPHABET =
+    {
+        (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G',
+        (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N',
+        (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', 
+        (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z',
+        (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g',
+        (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n',
+        (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', 
+        (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z',
+        (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', 
+        (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/'
+    };
+    
+    /** 
+     * Translates a Base64 value to either its 6-bit reconstruction value
+     * or a negative number indicating some other meaning.
+     **/
+    private final static byte[] DECODABET =
+    {   
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,                 // Decimal  0 -  8
+        -5,-5,                                      // Whitespace: Tab and Linefeed
+        -9,-9,                                      // Decimal 11 - 12
+        -5,                                         // Whitespace: Carriage Return
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 14 - 26
+        -9,-9,-9,-9,-9,                             // Decimal 27 - 31
+        -5,                                         // Whitespace: Space
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,              // Decimal 33 - 42
+        62,                                         // Plus sign at decimal 43
+        -9,-9,-9,                                   // Decimal 44 - 46
+        63,                                         // Slash at decimal 47
+        52,53,54,55,56,57,58,59,60,61,              // Numbers zero through nine
+        -9,-9,-9,                                   // Decimal 58 - 60
+        -1,                                         // Equals sign at decimal 61
+        -9,-9,-9,                                      // Decimal 62 - 64
+        0,1,2,3,4,5,6,7,8,9,10,11,12,13,            // Letters 'A' through 'N'
+        14,15,16,17,18,19,20,21,22,23,24,25,        // Letters 'O' through 'Z'
+        -9,-9,-9,-9,-9,-9,                          // Decimal 91 - 96
+        26,27,28,29,30,31,32,33,34,35,36,37,38,     // Letters 'a' through 'm'
+        39,40,41,42,43,44,45,46,47,48,49,50,51,     // Letters 'n' through 'z'
+        -9,-9,-9,-9                                 // Decimal 123 - 126
+        /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 127 - 139
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 140 - 152
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 153 - 165
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 166 - 178
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 179 - 191
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 192 - 204
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 205 - 217
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 218 - 230
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 231 - 243
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9         // Decimal 244 - 255 */
+    };
+    
+    private final static byte BAD_ENCODING    = -9; // Indicates error in encoding
+    private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding
+    private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding
+
+    
+    /** Defeats instantiation. */
+    private Base64(){}
+    
+    
+    
+/* ********  E N C O D I N G   M E T H O D S  ******** */    
+    
+    
+    /**
+     * Encodes the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     *
+     * @param threeBytes the array to convert
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes )
+    {   
+        return encode3to4( threeBytes, 3 );
+    }   // end encodeToBytes
+    
+    
+    
+    /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     *
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes, int numSigBytes )
+    {  
+        byte[] dest = new byte[4];
+        encode3to4( threeBytes, 0, numSigBytes, dest, 0 );
+        return dest;
+    }
+    
+     /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     * Code can reuse a byte array by passing a four-byte array as <var>b4</var>.
+     *
+     * @param b4 A reusable byte array to reduce array instantiation
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.5.1
+     */
+    private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes )
+    {
+        encode3to4( threeBytes, 0, numSigBytes, b4, 0 );
+        return b4;
+    }   // end encode3to4
+
+    
+    /**
+     * Encodes up to three bytes of the array <var>source</var>
+     * and writes the resulting four Base64 bytes to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 3 for
+     * the <var>source</var> array or <var>destOffset</var> + 4 for
+     * the <var>destination</var> array.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param numSigBytes the number of significant bytes in your array
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the <var>destination</var> array
+     * @since 1.3
+     */
+    private static byte[] encode3to4( 
+     byte[] source, int srcOffset, int numSigBytes,
+     byte[] destination, int destOffset )
+    {
+        //           1         2         3  
+        // 01234567890123456789012345678901 Bit position
+        // --------000000001111111122222222 Array position from threeBytes
+        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
+        //          >>18  >>12  >> 6  >> 0  Right shift necessary
+        //                0x3f  0x3f  0x3f  Additional AND
+        
+        // Create buffer with zero-padding if there are only one or two
+        // significant bytes passed in the array.
+        // We have to shift left 24 in order to flush out the 1's that appear
+        // when Java treats a value as negative that is cast from a byte to an int.
+        int inBuff =   ( numSigBytes > 0 ? ((source[ srcOffset     ] << 24) >>>  8) : 0 )
+                     | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 )
+                     | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 );
+
+        switch( numSigBytes )
+        {
+            case 3:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = ALPHABET[ (inBuff       ) & 0x3f ];
+                return destination;
+                
+            case 2:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            case 1:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = EQUALS_SIGN;
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            default:
+                return destination;
+        }   // end switch
+    }   // end encode3to4
+    
+    
+    
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * The object is not GZip-compressed before being encoded.
+     *
+     * @param serializableObject The object to encode
+     * @return The Base64-encoded object
+     * @since 1.4
+     */
+    public static String encodeObject( java.io.Serializable serializableObject )
+    {
+        return encodeObject( serializableObject, NO_OPTIONS );
+    }   // end encodeObject
+    
+
+
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     * @param serializableObject The object to encode
+     * @param options Specified options
+     * @return The Base64-encoded object
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeObject( java.io.Serializable serializableObject, int options )
+    {
+        // Streams
+        java.io.ByteArrayOutputStream  baos  = null; 
+        java.io.OutputStream           b64os = null; 
+        java.io.ObjectOutputStream     oos   = null; 
+        java.util.zip.GZIPOutputStream gzos  = null;
+        
+        // Isolate options
+        int gzip           = (options & GZIP);
+        int dontBreakLines = (options & DONT_BREAK_LINES);
+        
+        try
+        {
+            // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream
+            baos  = new java.io.ByteArrayOutputStream();
+            b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+    
+            // GZip?
+            if( gzip == GZIP )
+            {
+                gzos = new java.util.zip.GZIPOutputStream( b64os );
+                oos  = new java.io.ObjectOutputStream( gzos );
+            }   // end if: gzip
+            else
+                oos   = new java.io.ObjectOutputStream( b64os );
+            
+            oos.writeObject( serializableObject );
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            return null;
+        }   // end catch
+        finally
+        {
+            try{ oos.close();   } catch( Exception e ){}
+            try{ gzos.close();  } catch( Exception e ){}
+            try{ b64os.close(); } catch( Exception e ){}
+            try{ baos.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return new String( baos.toByteArray() );
+    }   // end encode
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source )
+    {
+        return encodeBytes( source, 0, source.length, NO_OPTIONS );
+    }   // end encodeBytes
+    
+
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int options )
+    {   
+        return encodeBytes( source, 0, source.length, options );
+    }   // end encodeBytes
+    
+    
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source, int off, int len )
+    {
+        return encodeBytes( source, off, len, NO_OPTIONS );
+    }   // end encodeBytes
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int off, int len, int options )
+    {
+        // Isolate options
+        int dontBreakLines = ( options & DONT_BREAK_LINES );
+        int gzip           = ( options & GZIP   );
+        
+        // Compress?
+        if( gzip == GZIP )
+        {
+            java.io.ByteArrayOutputStream  baos  = null;
+            java.util.zip.GZIPOutputStream gzos  = null;
+            Base64.OutputStream            b64os = null;
+            
+    
+            try
+            {
+                // GZip -> Base64 -> ByteArray
+                baos = new java.io.ByteArrayOutputStream();
+                b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+                gzos  = new java.util.zip.GZIPOutputStream( b64os ); 
+            
+                gzos.write( source, off, len );
+                gzos.close();
+            }   // end try
+            catch( java.io.IOException e )
+            {
+                e.printStackTrace();
+                return null;
+            }   // end catch
+            finally
+            {
+                try{ gzos.close();  } catch( Exception e ){}
+                try{ b64os.close(); } catch( Exception e ){}
+                try{ baos.close();  } catch( Exception e ){}
+            }   // end finally
+
+            return new String( baos.toByteArray() );
+        }   // end if: compress
+        
+        // Else, don't compress. Better not to use streams at all then.
+        else
+        {
+            // Convert option to boolean in way that code likes it.
+            boolean breakLines = dontBreakLines == 0;
+            
+            int    len43   = len * 4 / 3;
+            byte[] outBuff = new byte[   ( len43 )                      // Main 4:3
+                                       + ( (len % 3) > 0 ? 4 : 0 )      // Account for padding
+                                       + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines      
+            int d = 0;
+            int e = 0;
+            int len2 = len - 2;
+            int lineLength = 0;
+            for( ; d < len2; d+=3, e+=4 )
+            {
+                encode3to4( source, d+off, 3, outBuff, e );
+
+                lineLength += 4;
+                if( breakLines && lineLength == MAX_LINE_LENGTH )
+                {   
+                    outBuff[e+4] = NEW_LINE;
+                    e++;
+                    lineLength = 0;
+                }   // end if: end of line
+            }   // en dfor: each piece of array
+
+            if( d < len )
+            {
+                encode3to4( source, d+off, len - d, outBuff, e );
+                e += 4;
+            }   // end if: some padding needed
+
+            return new String( outBuff, 0, e );
+        }   // end else: don't compress
+        
+    }   // end encodeBytes
+    
+
+    
+    
+    
+/* ********  D E C O D I N G   M E T H O D S  ******** */
+    
+    
+    /**
+     * Decodes the first four bytes of array <var>fourBytes</var>
+     * and returns an array up to three bytes long with the
+     * decoded values.
+     *
+     * @param fourBytes the array with Base64 content
+     * @return array with decoded values
+     * @since 1.3
+     */
+    private static byte[] decode4to3( byte[] fourBytes )
+    {
+        byte[] outBuff1 = new byte[3];
+        int    count    = decode4to3( fourBytes, 0, outBuff1, 0 );
+        byte[] outBuff2 = new byte[ count ];
+        
+        for( int i = 0; i < count; i++ )
+            outBuff2[i] = outBuff1[i];
+        
+        return outBuff2;
+    }
+    
+    
+    
+    
+    /**
+     * Decodes four bytes from array <var>source</var>
+     * and writes the resulting bytes (up to three of them)
+     * to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 4 for
+     * the <var>source</var> array or <var>destOffset</var> + 3 for
+     * the <var>destination</var> array.
+     * This method returns the actual number of bytes that 
+     * were converted from the Base64 encoding.
+     * 
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the number of decoded bytes converted
+     * @since 1.3
+     */
+    private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset )
+    {
+        // Example: Dk==
+        if( source[ srcOffset + 2] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 );
+            
+            destination[ destOffset ] = (byte)( outBuff >>> 16 );
+            return 1;
+        }
+        
+        // Example: DkL=
+        else if( source[ srcOffset + 3 ] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6 );
+            
+            destination[ destOffset     ] = (byte)( outBuff >>> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>>  8 );
+            return 2;
+        }
+        
+        // Example: DkLE
+        else
+        {
+            try{
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 )
+            //              | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6)
+                          | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF )      );
+
+            
+            destination[ destOffset     ] = (byte)( outBuff >> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>  8 );
+            destination[ destOffset + 2 ] = (byte)( outBuff       );
+
+            return 3;
+            }catch( Exception e){
+                System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset     ] ]  ) );
+                System.out.println(""+source[srcOffset+1]+  ": " + ( DECODABET[ source[ srcOffset + 1 ] ]  ) );
+                System.out.println(""+source[srcOffset+2]+  ": " + ( DECODABET[ source[ srcOffset + 2 ] ]  ) );
+                System.out.println(""+source[srcOffset+3]+  ": " + ( DECODABET[ source[ srcOffset + 3 ] ]  ) );
+                return -1;
+            }   //e nd catch
+        }
+    }   // end decodeToBytes
+    
+    
+    
+    
+    /**
+     * Very low-level access to decoding ASCII characters in
+     * the form of a byte array. Does not support automatically
+     * gunzipping or any other "fancy" features.
+     *
+     * @param source The Base64 encoded data
+     * @param off    The offset of where to begin decoding
+     * @param len    The length of characters to decode
+     * @return decoded data
+     * @since 1.3
+     */
+    public static byte[] decode( byte[] source, int off, int len )
+    {
+        int    len34   = len * 3 / 4;
+        byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output
+        int    outBuffPosn = 0;
+        
+        byte[] b4        = new byte[4];
+        int    b4Posn    = 0;
+        int    i         = 0;
+        byte   sbiCrop   = 0;
+        byte   sbiDecode = 0;
+        for( i = off; i < off+len; i++ )
+        {
+            sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits
+            sbiDecode = DECODABET[ sbiCrop ];
+            
+            if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better
+            {
+                if( sbiDecode >= EQUALS_SIGN_ENC )
+                {
+                    b4[ b4Posn++ ] = sbiCrop;
+                    if( b4Posn > 3 )
+                    {
+                        outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn );
+                        b4Posn = 0;
+                        
+                        // If that was the equals sign, break out of 'for' loop
+                        if( sbiCrop == EQUALS_SIGN )
+                            break;
+                    }   // end if: quartet built
+                    
+                }   // end if: equals sign or better
+                
+            }   // end if: white space, equals sign or better
+            else
+            {
+                System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" );
+                return null;
+            }   // end else: 
+        }   // each input character
+                                   
+        byte[] out = new byte[ outBuffPosn ];
+        System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); 
+        return out;
+    }   // end decode
+    
+    
+    
+    
+    /**
+     * Decodes data from Base64 notation, automatically
+     * detecting gzip-compressed data and decompressing it.
+     *
+     * @param s the string to decode
+     * @return the decoded data
+     * @since 1.4
+     */
+    public static byte[] decode( String s )
+    {   
+        byte[] bytes = s.getBytes();
+        bytes = decode( bytes, 0, bytes.length );
+        
+        // Check to see if it's gzip-compressed
+        // GZIP Magic Two-Byte Number: 0x8b1f (35615)
+        if( bytes.length >= 2 )
+        {
+            
+            int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);       
+            if(
+            bytes != null &&      // In case decoding returned null
+            bytes.length >= 4  && // Don't want to get ArrayIndexOutOfBounds exception
+            java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) 
+            {
+                java.io.ByteArrayInputStream  bais = null;
+                java.util.zip.GZIPInputStream gzis = null;
+                java.io.ByteArrayOutputStream baos = null;
+                byte[] buffer = new byte[2048];
+                int    length = 0;
+
+                try
+                {
+                    baos = new java.io.ByteArrayOutputStream();
+                    bais = new java.io.ByteArrayInputStream( bytes );
+                    gzis = new java.util.zip.GZIPInputStream( bais );
+
+                    while( ( length = gzis.read( buffer ) ) >= 0 )
+                    {
+                        baos.write(buffer,0,length);
+                    }   // end while: reading input
+
+                    // No error? Get new bytes.
+                    bytes = baos.toByteArray();
+
+                }   // end try
+                catch( java.io.IOException e )
+                {
+                    // Just return originally-decoded bytes
+                }   // end catch
+                finally
+                {
+                    try{ baos.close(); } catch( Exception e ){}
+                    try{ gzis.close(); } catch( Exception e ){}
+                    try{ bais.close(); } catch( Exception e ){}
+                }   // end finally
+
+            }   // end if: gzipped
+        }   // end if: bytes.length >= 2
+        
+        return bytes;
+    }   // end decode
+
+
+    
+
+    /**
+     * Attempts to decode Base64 data and deserialize a Java
+     * Object within. Returns <tt>null</tt> if there was an error.
+     *
+     * @param encodedObject The Base64 data to decode
+     * @return The decoded and deserialized object
+     * @since 1.5
+     */
+    public static Object decodeToObject( String encodedObject )
+    {
+        // Decode and gunzip if necessary
+        byte[] objBytes = decode( encodedObject );
+        
+        java.io.ByteArrayInputStream  bais = null;
+        java.io.ObjectInputStream     ois  = null;
+        Object obj = null;
+        
+        try
+        {
+            bais = new java.io.ByteArrayInputStream( objBytes );
+            ois  = new java.io.ObjectInputStream( bais );
+        
+            obj = ois.readObject();
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        catch( java.lang.ClassNotFoundException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        finally
+        {
+            try{ bais.close(); } catch( Exception e ){}
+            try{ ois.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return obj;
+    }   // end decodeObject
+    
+    
+    /* ********  I N N E R   C L A S S   I N P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * It will read data from another
+     * {@link java.io.InputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterInputStream
+     * @since 1.3
+     */
+    public static class InputStream extends java.io.FilterInputStream
+    {
+        private int     options;        // Options specified
+        private boolean encode;         // Encoding or decoding
+        private int     position;       // Current position in the buffer
+        private byte[]  buffer;         // Small buffer holding converted data
+        private int     bufferLength;   // Length of buffer (3 or 4)
+        private int     numSigBytes;    // Number of meaningful bytes in the buffer
+        private int     lineLength;
+        private boolean breakLines;     // Break lines at less than 80 characters
+        
+        
+        /**
+         * Constructs a InputStream in DECODE mode.
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @since 1.3
+         */
+        public InputStream( java.io.InputStream in )
+        {   
+            this( in, DECODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a InputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code>
+         *
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @param options Specified options
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 2.0
+         */
+        public InputStream( java.io.InputStream in, int options )
+        {   
+            super( in );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.breakLines = breakLines;
+            this.encode = encode;
+            this.bufferLength = encode ? 4 : 3;
+            this.buffer   = new byte[ bufferLength ];
+            this.position = -1;
+            this.lineLength = 0;
+        }   // end constructor
+        
+        /**
+         * Reads enough of the input stream to convert
+         * to/from Base64 and returns the next byte.
+         *
+         * @return next byte
+         * @since 1.3
+         */
+        public int read() throws java.io.IOException 
+        { 
+            // Do we need to get data?
+            if( position < 0 )
+            {
+                if( encode )
+                {
+                    byte[] b3 = new byte[3];
+                    int numBinaryBytes = 0;
+                    for( int i = 0; i < 3; i++ )
+                    {
+                        try
+                        { 
+                            int b = in.read();
+                            
+                            // If end of stream, b is -1.
+                            if( b >= 0 )
+                            {
+                                b3[i] = (byte)b;
+                                numBinaryBytes++;
+                            }   // end if: not end of stream
+                            
+                        }   // end try: read
+                        catch( java.io.IOException e )
+                        {   
+                            // Only a problem if we got no data at all.
+                            if( i == 0 )
+                                throw e;
+                            
+                        }   // end catch
+                    }   // end for: each needed input byte
+                    
+                    if( numBinaryBytes > 0 )
+                    {
+                        encode3to4( b3, 0, numBinaryBytes, buffer, 0 );
+                        position = 0;
+                        numSigBytes = 4;
+                    }   // end if: got data
+                    else
+                    {
+                        return -1;
+                    }   // end else
+                }   // end if: encoding
+                
+                // Else decoding
+                else
+                {
+                    byte[] b4 = new byte[4];
+                    int i = 0;
+                    for( i = 0; i < 4; i++ )
+                    {
+                        // Read four "meaningful" bytes:
+                        int b = 0;
+                        do{ b = in.read(); }
+                        while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC );
+                        
+                        if( b < 0 )
+                            break; // Reads a -1 if end of stream
+                        
+                        b4[i] = (byte)b;
+                    }   // end for: each needed input byte
+                    
+                    if( i == 4 )
+                    {
+                        numSigBytes = decode4to3( b4, 0, buffer, 0 );
+                        position = 0;
+                    }   // end if: got four characters
+                    else if( i == 0 ){
+                        return -1;
+                    }   // end else if: also padded correctly
+                    else
+                    {
+                        // Must have broken out from above.
+                        throw new java.io.IOException( "Improperly padded Base64 input." );
+                    }   // end 
+                    
+                }   // end else: decode
+            }   // end else: get data
+            
+            // Got data?
+            if( position >= 0 )
+            {
+                // End of relevant data?
+                if( /*!encode &&*/ position >= numSigBytes )
+                    return -1;
+                
+                if( encode && breakLines && lineLength >= MAX_LINE_LENGTH )
+                {
+                    lineLength = 0;
+                    return '\n';
+                }   // end if
+                else
+                {
+                    lineLength++;   // This isn't important when decoding
+                                    // but throwing an extra "if" seems
+                                    // just as wasteful.
+                    
+                    int b = buffer[ position++ ];
+
+                    if( position >= bufferLength )
+                        position = -1;
+
+                    return b & 0xFF; // This is how you "cast" a byte that's
+                                     // intended to be unsigned.
+                }   // end else
+            }   // end if: position >= 0
+            
+            // Else error
+            else
+            {   
+                // When JDK1.4 is more accepted, use an assertion here.
+                throw new java.io.IOException( "Error in Base64 code reading stream." );
+            }   // end else
+        }   // end read
+        
+        
+        /**
+         * Calls {@link #read} repeatedly until the end of stream
+         * is reached or <var>len</var> bytes are read.
+         * Returns number of bytes read into array or -1 if
+         * end of stream is encountered.
+         *
+         * @param dest array to hold values
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @return bytes read into array or -1 if end of stream is encountered.
+         * @since 1.3
+         */
+        public int read( byte[] dest, int off, int len ) throws java.io.IOException
+        {
+            int i;
+            int b;
+            for( i = 0; i < len; i++ )
+            {
+                b = read();
+                
+                //if( b < 0 && i == 0 )
+                //    return -1;
+                
+                if( b >= 0 )
+                    dest[off + i] = (byte)b;
+                else if( i == 0 )
+                    return -1;
+                else
+                    break; // Out of 'for' loop
+            }   // end for: each byte read
+            return i;
+        }   // end read
+        
+    }   // end inner class InputStream
+    
+    
+    
+    
+    
+    
+    /* ********  I N N E R   C L A S S   O U T P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * A OutputStream will write data to another
+     * {@link java.io.OutputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterOutputStream
+     * @since 1.3
+     */
+    public static class OutputStream extends java.io.FilterOutputStream
+    {
+        private int     options;
+        private boolean encode;
+        private int     position;
+        private byte[]  buffer;
+        private int     bufferLength;
+        private int     lineLength;
+        private boolean breakLines;
+        private byte[]  b4; // Scratch used in a few places
+        private boolean suspendEncoding;
+        
+        /**
+         * Constructs a OutputStream in ENCODE mode.
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out )
+        {   
+            this( out, ENCODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a OutputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code>
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @param options Specified options.
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out, int options )
+        {   
+            super( out );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.bufferLength = encode ? 3 : 4;
+            this.buffer       = new byte[ bufferLength ];
+            this.position     = 0;
+            this.lineLength   = 0;
+            this.suspendEncoding = false;
+            this.b4           = new byte[4];
+        }   // end constructor
+        
+        
+        /**
+         * Writes the byte to the output stream after
+         * converting to/from Base64 notation.
+         * When encoding, bytes are buffered three
+         * at a time before the output stream actually
+         * gets a write() call.
+         * When decoding, bytes are buffered four
+         * at a time.
+         *
+         * @param theByte the byte to write
+         * @since 1.3
+         */
+        public void write(int theByte) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theByte );
+                return;
+            }   // end if: supsended
+            
+            // Encode?
+            if( encode )
+            {
+                buffer[ position++ ] = (byte)theByte;
+                if( position >= bufferLength )  // Enough to encode.
+                {
+                    out.write( encode3to4( b4, buffer, bufferLength ) );
+
+                    lineLength += 4;
+                    if( breakLines && lineLength >= MAX_LINE_LENGTH )
+                    {
+                        out.write( NEW_LINE );
+                        lineLength = 0;
+                    }   // end if: end of line
+
+                    position = 0;
+                }   // end if: enough to output
+            }   // end if: encoding
+
+            // Else, Decoding
+            else
+            {
+                // Meaningful Base64 character?
+                if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC )
+                {
+                    buffer[ position++ ] = (byte)theByte;
+                    if( position >= bufferLength )  // Enough to output.
+                    {
+                        int len = Base64.decode4to3( buffer, 0, b4, 0 );
+                        out.write( b4, 0, len );
+                        //out.write( Base64.decode4to3( buffer ) );
+                        position = 0;
+                    }   // end if: enough to output
+                }   // end if: meaningful base64 character
+                else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC )
+                {
+                    throw new java.io.IOException( "Invalid character in Base64 data." );
+                }   // end else: not white space either
+            }   // end else: decoding
+        }   // end write
+        
+        
+        
+        /**
+         * Calls {@link #write} repeatedly until <var>len</var> 
+         * bytes are written.
+         *
+         * @param theBytes array from which to read bytes
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @since 1.3
+         */
+        public void write( byte[] theBytes, int off, int len ) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theBytes, off, len );
+                return;
+            }   // end if: supsended
+            
+            for( int i = 0; i < len; i++ )
+            {
+                write( theBytes[ off + i ] );
+            }   // end for: each byte written
+            
+        }   // end write
+        
+        
+        
+        /**
+         * Method added by PHIL. [Thanks, PHIL. -Rob]
+         * This pads the buffer without closing the stream.
+         */
+        public void flushBase64() throws java.io.IOException 
+        {
+            if( position > 0 )
+            {
+                if( encode )
+                {
+                    out.write( encode3to4( b4, buffer, position ) );
+                    position = 0;
+                }   // end if: encoding
+                else
+                {
+                    throw new java.io.IOException( "Base64 input not properly padded." );
+                }   // end else: decoding
+            }   // end if: buffer partially full
+
+        }   // end flush
+
+        
+        /** 
+         * Flushes and closes (I think, in the superclass) the stream. 
+         *
+         * @since 1.3
+         */
+        public void close() throws java.io.IOException
+        {
+            // 1. Ensure that pending characters are written
+            flushBase64();
+
+            // 2. Actually close the stream
+            // Base class both flushes and closes.
+            super.close();
+            
+            buffer = null;
+            out    = null;
+        }   // end close
+        
+        
+        
+        /**
+         * Suspends encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void suspendEncoding() throws java.io.IOException 
+        {
+            flushBase64();
+            this.suspendEncoding = true;
+        }   // end suspendEncoding
+        
+        
+        /**
+         * Resumes encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void resumeEncoding()
+        {
+            this.suspendEncoding = false;
+        }   // end resumeEncoding
+        
+        
+        
+    }   // end inner class OutputStream
+    
+    
+}   // end class Base64
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CombineGraphs.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CombineGraphs.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CombineGraphs.java	(revision 21112)
@@ -0,0 +1,138 @@
+/** This class is not currently used and therefore will not be compiled by makegli. */
+
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import javax.imageio.*;
+import java.awt.*;
+import java.awt.image.*;
+import com.sun.image.codec.jpeg.*;
+import javax.swing.*;
+
+/**
+ *  CombineGraphs class is a class to combine 2 images file into 1 jpeg file
+ *  called test.jpg using point by point comparison.
+ *  <p>User need to supplied the valid exist filenames of the 2 images that 
+ *  they want to combine.If in any one point in 2nd image file is not transparent
+ *  then that point will be drawn to the 1st image file.</p>  
+ *  <p>To run this class properly the format is :
+ *  <br><i>java CombineGraphs filename1 filename2</i>
+ *  <br>where filename1 is the names of the first image file.
+ *  <br>and filename2 is the names of the second image file. This file must
+ *  contains transparent.</p>
+ *  <p>For example: If user wants to combine the screenshot image file with the
+ *  image file of the scribble lines, then
+ *  <br>filename1 will be the filename of the screenshot image file, and
+ *  <br>filename2 will be the filename of the scribble lines image file.</p>
+ *
+ *  @author Veronica Liesaputra
+ */
+
+public class CombineGraphs extends JPanel
+{
+    /**
+     * This is a variable to stored the image that is stored inside f1.
+     * After the combine method called this variable will store the resulted image of
+     * combining 2 images together.
+     */
+    private BufferedImage bi1;
+
+    /**
+     * This is a variable to stored the image that is stored inside f2.
+     */
+    private BufferedImage bi2;
+
+    /**
+     * This is a variable to stored the first image file that the user supplied.
+     */
+    private File f1;
+
+    /**
+     * This is a variable to stored the second image file that the user supplied.
+     */
+    private File f2;
+    
+    /**
+     * It will read the image files user supplied and stored the image
+     * to the appropriate bufferedimage variable.
+     * (Precondition: (file1 != null) && (file2 != null))
+     * @param file1   the name of the first image file.
+     * @param file2   the name of the second image file.
+     */
+    public CombineGraphs(String file1,String file2)
+    {
+	try
+	    {
+		f1 = new File(file1);
+		f2 = new File(file2);
+		bi1 = ImageIO.read(f1);
+		bi2 = ImageIO.read(f2);	
+	    }
+	catch(IOException exp) {}
+    }   
+
+    /**
+     * This method will combine the 2 buffered images into 1 buffered image and 
+     * saved it to an output file in jpeg format.
+     * (Precondition: (filename != null))
+     * @param filename the name of the output file.
+     * (Postcondition: the combined image is stored in a jpeg file with the name
+     *                 as specified in filename.)
+     */
+    public void combine(String filename)
+    {
+
+	/*using bi1 as the base layer that we use to compare point by point with bi2. 
+	  if in one point in bi2 its not transparent then we draw that point to bi1.*/ 
+	Graphics2D G1;
+	
+	G1 = bi1.createGraphics();
+	
+	int width = bi2.getWidth();
+	int height = bi2.getHeight();
+
+	for (int y = 0 ; y < height ; y++) 
+	    {
+		for (int x = 0 ; x < width ; x++)
+		    {
+			int rgb = bi2.getRGB(x,y);
+			if (rgb != 0)
+			    {
+				G1.setColor(new Color(rgb,true));
+				G1.drawLine(x,y,x,y);
+			    }
+		    }
+	    }
+	
+	// Save the combined bufferedimage into a jpeg output file.
+	try {
+	    File file = new File(filename);
+	    FileOutputStream out = new FileOutputStream(file);
+	    
+	    JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(out);
+	    JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(bi1);
+	    param.setQuality(1.0f, false);
+	    encoder.setJPEGEncodeParam(param);
+	    encoder.encode(bi1);
+	    
+	} catch (Exception ex) {}
+    }
+
+    /**
+     * This the main method that need 2 inputs from the user for it to
+     * work properly. 
+     * The inputs are the 2 image filenames that user want to combine.
+     */
+    public static void main (String[] args)
+    {
+	CombineGraphs cb = new CombineGraphs(args[0],args[1]);
+	cb.combine("test.jpg");
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompGroup.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompGroup.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompGroup.java	(revision 21112)
@@ -0,0 +1,60 @@
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+
+/**
+ * The instance of this class will hold a UserComponent and the status of the UserComponent.
+ * For each of the CompGroup instance will contains all the information and the status of
+ * a window that is opened by the application the user used.
+ * @author Veronica Liesaputra
+ */
+public class CompGroup implements Serializable
+{
+    /**
+     *  This variable will hold the status of the window that is opened by the application.
+     */
+    private String comp_status;
+
+    /**
+     *  This variable will hold the information about the window that is opened by the
+     *  application.
+     */
+    private UserComponent comp;
+
+    public CompGroup ()
+    {
+
+    }
+    
+    /**
+     *  It will make CompGroup instance that hold the information and status of the window
+     *  thats is opened by the application.
+     *  @param curr_comp  information about the window opened by the application.
+     *  @param curr_status status of the window opened by the application.
+     */
+    public CompGroup (UserComponent curr_comp,String curr_status)
+    {
+	comp_status = curr_status;
+	comp = curr_comp;
+    }
+    
+    /**
+     * This method will return the status of the window opened by the application.
+     * @return the status of the window.
+     */
+    public String getStatus ()
+    {
+	return comp_status;
+    }
+    
+    /**
+     * This method will return the information about the window opened by the application.
+     * @return the information about the window.
+     */
+    public UserComponent getComponent()
+    {
+	return comp;
+    }
+}
+
+  
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/CompListener.java	(revision 21112)
@@ -0,0 +1,944 @@
+package org.greenstone.gatherer.feedback;
+
+import org.omg.IOP.ComponentIdHelper;
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import javax.swing.table.*;
+import javax.swing.text.*;
+import java.awt.image.*;
+import javax.imageio.ImageIO;
+import javax.swing.plaf.*;
+import javax.swing.plaf.basic.*;
+import javax.swing.colorchooser.*;
+import java.awt.event.*;
+import javax.swing.event.*;
+import java.text.MessageFormat;
+import javax.swing.plaf.metal.MetalIconFactory.*;
+import javax.swing.plaf.metal.MetalIconFactory;
+import javax.swing.plaf.basic.BasicInternalFrameTitlePane;
+import javax.swing.plaf.metal.MetalInternalFrameTitlePane;
+
+/**
+ *  In general, this class will get add special listener 
+ *  to a Container and of all the components inside it.
+ *  <p>This class will also add special listener to the Container and each of the components inside 
+ *  the Container. So, it will allows special listener  to record and listen each of the actions 
+ *  the user do to the Container or any of the components inside it. The special listener
+ *  is an instance of ActionRecorderDialog.</p>
+ *  <p>This class will only recognized all the components that is belongs to the javax.swing package
+ *  which are all the implementing classes of RootPaneContainer interface and all the direct known
+ *  subclasses of JComponent. All the other components will be stated simply as Container 
+ *  and user cannot really get much information out of it.</p>
+ *  <p>The added special listener to this component is as what its allowed in API, if the component
+ *  have some modified listener added to it then that action will not be known and recorded by
+ *  the special listener.</p>
+ *  
+ *  @author Veronica Liesaputra
+ */
+public class CompListener
+{
+    /**
+     *  This variable will hold the special listener that will listen and record any of the 
+     *  actions user do to the Container and any of the components inside it.
+     */
+    private ActionRecorderDialog m;
+   
+    /**
+     *  This variable will hold the pair of the component and the model of the component.
+     *  With this variable allows ActionRecorderDialog listener to know which component the model that fires an action
+     *  belongs to. In this HashMap the model is the key and the component where the model belongs to is
+     *  the value. 
+     */
+    private HashMap hash;
+    
+    /**
+     *  This constructor will set the special listener, the locale and the hashmap to be used in
+     *  this Container.
+     *  (Precondition: dialog != null)
+     *  @param dialog  its the special listener that will listen and record all the action
+     *                 user do to any of the component inside the Container.
+     *  @param h       its the HashMap that will hold the pair of model with the component it belongs.
+     */
+    public CompListener 
+	(ActionRecorderDialog dialog,HashMap h)
+    {
+	m = dialog;
+	hash = h;
+    }
+	
+    /**
+     *  If the JTextComponent is a JTextField then it will add the special action listener to it.
+     *  (Precondition: txt != null)
+     *  @param txt its the JTextComponent we want to get all the information from.
+     */
+    public void getTextAreaInfo(JTextComponent txt)
+    {
+	if (txt instanceof JTextField)
+	    {
+		((JTextField)txt).removeActionListener(m);
+		((JTextField)txt).addActionListener(m);
+	    }
+    }
+
+    /**
+     *  This method will add the special action listener to the JButton.
+     *  (Precondition: button != null)
+     *  @param button its the JButton we want to get the information from.
+     */
+    public void getButtonInfo (JButton button)
+    {
+	button.removeActionListener(m);
+	button.addActionListener(m);
+    }
+
+     /**
+     *  This method will get add the special change listener to the JProgressBar.
+     *  (Precondition: bar != null)
+     *  @param  bar   its the JProgressBar we want to get information from.
+     */
+    public void getProgressBarInfo (JProgressBar bar)
+    {
+	bar.removeChangeListener(m);
+	bar.addChangeListener(m);
+    }
+    
+    /**
+     *  This method will add the special change listener to the JSlider.
+     *  (Precondition: slider != null)
+     *  @param  slider its the JSlider we want to get information from.
+     */
+    public void getSliderInfo (JSlider slider)
+    {
+	slider.removeChangeListener(m);
+	slider.addChangeListener(m);
+    }
+
+    /**
+     *  This method will add the special change listener to the JSpinner.
+     *  (Precondition: spinner != null)
+     *  @param spinner its the JSpinner we want to the information from.
+     */
+    public void getSpinnerInfo (JSpinner spinner)
+    {
+	spinner.removeChangeListener(m);
+	spinner.addChangeListener(m);
+    }
+    
+    /**
+     * Dummy method to get JLabel.
+     */
+    public void getLabelInfo(JLabel lbl)
+    {}
+    
+    /**
+     *  This method will add the special action listener to the JComboBox.
+     *  (Precondition: box != null)
+     *  @param box    its the JComboBox we want to get information from.
+     */
+    public void getComboBoxInfo(JComboBox box)
+    {
+	ComboBoxModel cm;
+	cm = box.getModel();
+
+	box.removeActionListener(m);
+	box.addActionListener(m);
+    }
+
+    /**
+     *  This method will add the special action listener to the JToggleButton.
+     *  (Precondition: button != null)
+     *  @param button its a JToggleButton inside the window.
+     */
+    public void getToggleButtonInfo (JToggleButton button)
+    {
+	button.removeActionListener(m);
+	button.addActionListener(m);
+    }
+
+    /**
+     *  This method will add the special action listener to the JMenuItem.
+     *  (Precondition: menuitem != null)
+     *  @param menuitem its the JMenuItem we want to get the information from.
+     */
+    public void getMenuItemInfo (JMenuItem menuitem)
+    {
+	menuitem.removeActionListener(m);
+	menuitem.addActionListener(m);
+    }
+
+    /**
+     *  This method will get all the components inside JMenuBar.
+     *  (Precondition: menubar != null)
+     *  @param menubar its the JMenuBar we want to get the information from.
+     */
+    public void getMenuBarInfo (JMenuBar menubar)
+    {
+	int i;
+	for (i= 0 ; i < menubar.getMenuCount() ; i++)
+	    {
+		getInside(menubar.getMenu(i));
+	    }
+    }
+
+    /**
+     *  This method will add the special action and menu listener to the JMenu and
+     *  get the components inside JMenu.
+     *  (Precondition: menu != null)
+     *  @param menu   its the JMenu we want to the the information from.
+     */
+    public void getMenuInfo (JMenu menu)
+    {
+	menu.removeActionListener(m);
+	menu.removeMenuListener(m);
+	menu.addActionListener(m);
+	menu.addMenuListener(m);
+
+	int i;
+	for (i= 0 ; i < menu.getMenuComponentCount() ; i++)
+	    {
+		getInside((Container)menu.getMenuComponent(i));
+	    }
+    }
+    
+    /**
+     *  This method will get all the components inside the JApplet.
+     *  (Precondition: applet != null)
+     *  @param applet its the JApplet we want to get the information from.
+     */
+    public void getAppletInfo (JApplet applet)
+    {
+	Component[] group;
+	group = applet.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)applet.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JWindow.
+     *  (Precondition: window != null)
+     *  @param window its the JWindow we want to get the information from.
+     */
+    public void getWindowInfo (JWindow window)
+    {
+	Component[] group;
+	group = window.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)window.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JDialog.
+     *  (Precondition: dialog != null)
+     *  @param dialog its the JDialog we want to get the information from.
+     */
+    public void getDialogInfo (JDialog dialog)
+    {
+	Component[] group;
+	group = dialog.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)dialog.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JInternalFrame.
+     *  (Precondition: frame != null)
+     *  @param frame  its the JInternalFrame we want to get the information from.
+     */
+    public void getInternalFrameInfo (JInternalFrame frame)
+    {
+	Component[] group;
+	group = frame.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)frame.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JFrame.
+     *  (Precondition: frame != null)
+     *  @param frame  its the JFrame we want to get the information from.
+     */
+    public void getFrameInfo (JFrame frame)
+    {
+	Component[] group;
+	group = frame.getComponents();
+	int i;
+	
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)frame.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JPanel.
+     *  (Precondition: pane != null)
+     *  @param pane   its the JPanel we want to get the information from.
+     */
+    public void getPanelInfo (JPanel pane)
+    {
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)pane.getComponent(i));
+		    }
+	    }
+    }
+    
+    /**
+     *  This method will get all the components inside the JScrollPane.
+     *  (Precondition: scroll != null)
+     *  @param scroll its the JScrollPane we want to get the information from.
+     */
+    public void getScrollInfo (JScrollPane scroll)
+    {
+	Component[] group;
+	group = scroll.getComponents();
+	int i;
+	
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)group[i]);
+		    }
+	    }
+    }
+
+    /**
+     *  This method will add special change listener to the JTabbedPane and get all
+     *  the components inside it.
+     *  (Precondition: tab != null)
+     *  @param tab   its the JTabbedPane we want to get the information from.
+     */
+    public void getTabbedPaneInfo (JTabbedPane tab)
+    {
+	int j;
+	
+	tab.removeChangeListener(m);
+	tab.addChangeListener(m);
+
+	for (j = 0 ; j < tab.getTabCount(); j++)
+	    {	
+		
+		Component group = tab.getComponentAt(j);
+		getInside((Container)group);
+		
+	    }
+    }
+    
+    /**
+     *  This is the dummy method to ge the option pane info.
+     */
+    public void getOptionPaneInfo (JOptionPane op)
+    {}
+
+    /**
+     *  This method will get components inside the JViewPort.
+     *  (Precondition: vp != null)
+     *  @param vp   its the JViewPort we want to get the information from.
+     */
+    public void getViewPortInfo (JViewport vp)
+    {
+	Component[] group;
+	group = vp.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)group[i]);
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get components inside the Box.
+     *  (Precondition: box != null)
+     *  @param box  its the Box we want to get the information from.
+     */
+    public void getBoxInfo (Box box)
+    {
+	Component[] group;
+	group = box.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)group[i]);
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get components inside the JSplitPane.
+     *  (Precondition: split != null)
+     *  @param split  its the JSplitPane we want to get the information from.
+     */
+    public void getSplitPaneInfo (JSplitPane split)
+    {
+	Component[] group;
+	group = split.getComponents();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)group[i]);
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get add special list selection listener to the JList.
+     *  (Precondition: list != null)
+     *  @param list  its the JList we want to get the information from.
+     */
+    public void getListInfo (JList list)
+    {
+	ListModel ls;
+	ls = list.getModel();
+	
+	list.removeListSelectionListener(m);
+	list.addListSelectionListener(m);
+	
+	if (ls != null)
+	    {
+		if (hash.containsKey(ls) == false)
+		    hash.put(ls,list);
+		ls.removeListDataListener(m);
+		ls.addListDataListener(m);
+	    }
+    }
+
+    /**
+     *  This method will add special change listener to the JColorChooser.
+     *  (Precondition: color != null)
+     *  @param color  its the JColorChooser we want to get the information from.
+     */
+    public void getColorInfo (JColorChooser color)
+    {
+	ColorSelectionModel cm;
+	cm = color.getSelectionModel();
+	
+	if (cm != null)
+	    {
+		if (hash.containsKey(cm) == false)
+		    hash.put(cm,color);
+		cm.removeChangeListener(m);
+		cm.addChangeListener(m);
+	    }
+
+	getInside((Container) color.getPreviewPanel());
+    }
+
+    /**
+     *  This method will add special action listener to the JFileChooser.
+     *  (Precondition: fc != null)
+     *  @param fc  its the JFileChooser we want to get the information from.
+     */
+    public void getFileChooserInfo (JFileChooser fc)
+    {
+	fc.removeActionListener(m);
+	fc.addActionListener(m);
+    }		
+
+    /**
+     *  This method will add special list selection and column model listener 
+     *  to the JTable.
+     *  (Precondition: table != null)
+     *  @param table  its the JTable we want to get the information from.
+     */
+    public void getTableInfo (JTable table)
+    {
+	TableModel tm;
+	tm = table.getModel();
+	
+	TableColumnModel tcm;
+	tcm = table.getColumnModel();
+	ListSelectionModel lm;
+	lm = table.getSelectionModel(); 
+
+	if (lm != null)
+	    {
+		if (hash.containsKey(lm) == false)
+		    hash.put(lm,table);
+		lm.removeListSelectionListener(m);
+		lm.addListSelectionListener(m);
+	    }
+	
+	if (tcm != null)
+	    {
+		if (hash.containsKey(tcm) == false)
+		    hash.put(tcm,table);
+		tcm.removeColumnModelListener(m);
+		tcm.addColumnModelListener(m);
+	    }
+    }
+
+    /**
+     *  This is a dummy method to get the JScrollBar.
+     */
+    public void getScrollBarInfo (JScrollBar scroll)
+    {
+    }
+
+    /**
+     *  This method will add special tree selection listener to the JTree.
+     *  (Precondition: tree != null)
+     *  @param tree  its the JTree we want to get the information from.
+     */
+    public void getTreeInfo (JTree tree)
+    {
+	TreeModel tm;
+	tm = tree.getModel();
+	
+	tree.removeTreeSelectionListener(m);
+	tree.addTreeSelectionListener(m);
+	
+	if (tm != null)
+	    {
+		if (hash.containsKey(tm) == false)
+		    hash.put(tm,tree);
+		tm.removeTreeModelListener(m);
+		tm.addTreeModelListener(m);
+	    }
+    }
+       
+    /**
+     *  This method is a dummy method to get Container.
+     */
+    public void getCompInfo (Container pane)
+    {
+    }
+
+    /**
+     *  This method will get all components inside the JPopupMenu.
+     *  (Precondition: popup != null)
+     *  @param popup  its the JPopUpMenu we want to get the information from.
+     */
+    public void getPopUpMenuInfo (JPopupMenu popup)
+    {
+	Component[] group;
+	group = popup.getComponents();
+	SingleSelectionModel sm;
+	sm = popup.getSelectionModel();
+	int i;
+
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)popup.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JRootPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JRootPane we want to get the information from.
+     */
+    public void getRootPaneInfo (JRootPane pane)
+    {
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+	
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			
+			getInside((Container)pane.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JLayeredPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JLayeredPane we want to get the information from.
+     */
+    public void getLayeredPaneInfo (JLayeredPane pane)
+    {
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+	
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)pane.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     * This is a dummy method to get BasicInternalFrameTitlePane.
+     */
+    public void getInternalFrameTitleInfo (BasicInternalFrameTitlePane pane)
+    {
+    }
+
+    /**
+     *  This method will add special column model and list selection
+     *  listener to JTableHeader.
+     *  (Precondition: header != null)
+     *  @param header  its the JTableHeader we want to get the information from.
+     */
+    public void getTableHeaderInfo (JTableHeader header)
+    {
+	TableColumnModel tcm;
+	tcm = header.getColumnModel();
+	ListSelectionModel lm;
+	lm = tcm.getSelectionModel();
+
+	if (tcm != null)
+	    {
+		if (hash.containsKey(tcm) == false)
+		    hash.put(tcm,header);
+		tcm.removeColumnModelListener(m);
+		tcm.addColumnModelListener(m);
+	    }
+
+	if (lm != null)
+	    {
+		if (hash.containsKey(lm) == false)
+		    hash.put(lm,header);
+		lm.removeListSelectionListener(m);
+		lm.addListSelectionListener(m);
+	    }
+    }
+
+    /**
+     *  This method will get all components inside the JToolBar.
+     *  (Precondition: tool != null)
+     *  @param tool  its the JToolBar we want to get the information from.
+     */
+    public void getToolBarInfo (JToolBar tool)
+    {
+	Component[] group;
+	group = tool.getComponents();
+	int i;
+	
+	if (group != null)
+	    {
+		for (i = 0; i < group.length ; i++)
+		    {
+			getInside((Container)tool.getComponent(i));
+		    }
+	    }
+    }
+
+    /**
+     *  This method will get all the components inside the JToolTip.
+     *  (Precondition: tip != null)
+     *  @param tip  its the JToolTip we want to get the information from.
+     */
+    public void getToolTipInfo (JToolTip tip)
+    {
+	Component group;
+	group = tip.getComponent();
+	int i;
+	
+	if (group != null)
+	    {
+		getInside((Container) group);
+	    }
+    }
+
+    /**
+     *  This is the dummy method to get JSeparator.
+     */
+    public void getSeparatorInfo (JSeparator sept)
+    {
+    }
+
+    /**
+     *  This method will get all the components inside the JDesktopPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JDesktopPane we want to get the information from.
+     */
+    public void getDesktopPaneInfo (JDesktopPane pane)
+    {
+	JInternalFrame[]  group;
+	group = pane.getAllFrames();
+
+	if (group != null)
+	    {
+		int i;
+		
+		for ( i = 0 ; i < group.length ; i++ )
+		    {
+			getInside((Container) group[i]);
+		    }
+	    }
+    }
+
+    /**
+     *  This is a dummy method to catch null components.
+     */
+    public void getNullInfo ()
+    {
+    }
+
+    /**
+     *  This method will add a special listener to a container and components inside it.
+     *  This method is the gateway to all other methods in this class.
+     *  @param comp its the Container we want to get the information from.
+     */
+    public void getInside(Container comp)
+    {
+	if (comp == null)
+	    {
+		getNullInfo();
+		return;
+	    }
+	else if (comp instanceof JApplet)
+	    {
+		getAppletInfo ((JApplet) comp);
+		return;
+	    }
+	else if (comp instanceof JSeparator)
+	    {
+		getSeparatorInfo ((JSeparator) comp);
+		return;
+	    }
+	else if (comp instanceof JWindow)
+	    {
+		getWindowInfo((JWindow) comp);
+		return;
+	    }
+	else if (comp instanceof JDialog)
+	    {
+		getDialogInfo((JDialog) comp);
+		return;
+	    }
+	else if (comp instanceof JComboBox)
+	    {
+		getComboBoxInfo((JComboBox) comp);
+		return;
+	    }
+	else if (comp instanceof JLabel)
+	    {
+		getLabelInfo((JLabel) comp);
+		return;
+	    }
+	else if (comp instanceof JTextComponent)
+	    {
+		getTextAreaInfo((JTextComponent) comp);
+		return;
+	    }
+	else if (comp instanceof JFrame)
+	    {
+		getFrameInfo((JFrame) comp);
+		return;
+	    }
+	else if (comp instanceof JPanel)
+	    {
+		getPanelInfo((JPanel) comp);
+		return;
+	    }
+	else if (comp instanceof JScrollPane)
+	    {
+		getScrollInfo((JScrollPane) comp);
+		return;
+	    }
+	else if (comp instanceof JScrollBar)
+	    {
+		getScrollBarInfo ((JScrollBar) comp);
+		return;
+	    }
+	else if (comp instanceof JViewport)
+	    {
+		getViewPortInfo((JViewport) comp);
+		return;
+	    }
+	else if (comp instanceof JTabbedPane)
+	    {
+		getTabbedPaneInfo((JTabbedPane) comp);
+		return;
+	    }
+	else if (comp instanceof Box)
+	    {
+		getBoxInfo((Box) comp);
+		return;
+	    }
+	else if (comp instanceof JToggleButton)
+	    {
+		getToggleButtonInfo((JToggleButton) comp);
+		return;
+	    }
+	else if (comp instanceof JSplitPane)
+	    {
+		getSplitPaneInfo ((JSplitPane) comp);
+		return;
+	    }
+	else if (comp instanceof JList)
+	    {
+		getListInfo ((JList) comp);
+		return;
+	    }
+	else if (comp instanceof JTree)
+	    {
+		getTreeInfo ((JTree) comp);
+		return;
+	    }
+	else if (comp instanceof JTable)
+	    {
+		getTableInfo ((JTable) comp);
+		return;
+	    }
+	else if (comp instanceof JSlider)
+	    {
+		getSliderInfo ((JSlider) comp);
+		return;
+	    }
+	else if (comp instanceof JSpinner)
+	    {
+		getSpinnerInfo ((JSpinner) comp);
+		return;
+	    }
+	else if (comp instanceof JProgressBar)
+	    {
+		getProgressBarInfo ((JProgressBar) comp);
+		return;
+	    }
+	else if (comp instanceof JButton)
+	    {
+		getButtonInfo ((JButton) comp);
+		return;
+	    }
+	else if (comp instanceof JMenu)
+	    {
+		getMenuInfo ((JMenu) comp);
+		return;
+	    }
+	else if (comp instanceof JMenuBar)
+	    {
+		getMenuBarInfo ((JMenuBar) comp);
+		return;
+	    }
+	else if (comp instanceof JMenuItem)
+	    {
+		getMenuItemInfo ((JMenuItem) comp);
+		return;
+	    }
+	else if (comp instanceof JOptionPane)
+	    {
+		getOptionPaneInfo ((JOptionPane) comp);
+		return;
+	    }
+	else if (comp instanceof BasicInternalFrameTitlePane)
+	    {
+		getInternalFrameTitleInfo ((BasicInternalFrameTitlePane) comp);
+		return;
+	    }
+	else if (comp instanceof JColorChooser)
+	    {
+		getColorInfo ((JColorChooser) comp);
+		return;
+	    }
+	else if (comp instanceof JFileChooser)
+	    {
+		getFileChooserInfo ((JFileChooser) comp);
+		return;
+	    }
+	else if (comp instanceof JDesktopPane)
+	    {
+		getDesktopPaneInfo ((JDesktopPane) comp);
+		return;
+	    }
+	else if (comp instanceof JInternalFrame)
+	    {
+		getInternalFrameInfo ((JInternalFrame) comp);
+		return;
+	    }
+	else if (comp instanceof JLayeredPane)
+	    {
+		getLayeredPaneInfo ((JLayeredPane) comp);
+		return;
+	    }
+	else if (comp instanceof JPopupMenu)
+	    {
+		getPopUpMenuInfo ((JPopupMenu) comp);
+		return;
+	    }
+	else if (comp instanceof JRootPane)
+	    {
+		getRootPaneInfo ((JRootPane) comp);
+		return;
+	    }
+	else if (comp instanceof JTableHeader)
+	    {
+		getTableHeaderInfo ((JTableHeader) comp);
+		return;
+	    }
+	else if (comp instanceof JToolBar)
+	    {
+		getToolBarInfo ((JToolBar) comp);
+		return;
+	    }
+	else if (comp instanceof JToolTip)
+	    {
+		getToolTipInfo ((JToolTip) comp);
+		return;
+	    }
+	else
+	    {
+		getCompInfo (comp);
+		return;
+	    }
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ComponentInformation.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ComponentInformation.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ComponentInformation.java	(revision 21112)
@@ -0,0 +1,2261 @@
+package org.greenstone.gatherer.feedback;
+
+import org.omg.IOP.ComponentIdHelper;
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import javax.swing.table.*;
+import javax.swing.text.*;
+import java.awt.image.*;
+import javax.imageio.ImageIO;
+import javax.swing.plaf.*;
+import javax.swing.plaf.basic.*;
+import javax.swing.colorchooser.*;
+import java.awt.event.*;
+import javax.swing.event.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.plaf.metal.MetalIconFactory.*;
+import javax.swing.plaf.metal.MetalIconFactory;
+import javax.swing.plaf.basic.BasicInternalFrameTitlePane;
+import javax.swing.plaf.metal.MetalInternalFrameTitlePane;
+
+/**
+ *  In general, this class will get all information and add special listener 
+ *  to a Container and of all the components inside it.
+ *  <p>This class will get the information about the Container and all the components inside it.
+ *  The information we get is like whether or not the component is visible and the contents of it.
+ *  This class will also add special listener to the Container and each of the components inside 
+ *  the Container. So, it will allows special listener  to record and listen each of the actions 
+ *  the user do to the Container or any of the components inside it. The special listener
+ *  is an instance of ActionRecorderDialog.</p>
+ *  <p>This class will only recognized all the components that is belongs to the javax.swing package
+ *  which are all the implementing classes of RootPaneContainer interface and all the direct known
+ *  subclasses of JComponent. All the other components will be stated simply as Container 
+ *  and user cannot really get much information out of it.</p>
+ *  <p>The added special listener to this component is as what its allowed in API, if the component
+ *  have some modified listener added to it then that action will not be known and recorded by
+ *  the special listener.</p>
+ *  
+ *  @author Veronica Liesaputra
+ */
+public class ComponentInformation
+{
+    /**
+     *  This variable will hold the special listener that will listen and record any of the 
+     *  actions user do to the Container and any of the components inside it.
+     */
+    private ActionRecorderDialog m;
+    
+    /**
+     *  This variable will hold the resource of the words that is stored in Messages.properties file. 
+     *  The calling using messages.getString(someString) will caused someString to be translated
+     *  into some other string that is hold in that file.usually it will caused it to be translated
+     *  to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    /**
+     *  This variable will hold the pair of the component and the model of the component.
+     *  With this variable allows ActionRecorderDialog listener to know which component the model that fires an action
+     *  belongs to. In this HashMap the model is the key and the component where the model belongs to is
+     *  the value. 
+     */
+    private HashMap hash;
+    
+    /**
+     *  This constructor will set the special listener, the locale and the hashmap to be used in
+     *  this Container.
+     *  (Precondition: dialog != null)
+     *  @param dialog  its the special listener that will listen and record all the action
+     *                 user do to any of the component inside the Container.
+     *  @param currentLocale its the locale that the user choose all the information to be displayed as.
+     *  @param h       its the HashMap that will hold the pair of model with the component it belongs.
+     */
+    public ComponentInformation 
+	(ActionRecorderDialog dialog,Locale currentLocale,HashMap h)
+    {
+	m = dialog;
+	hash = h;
+	messages = ResourceBundle.getBundle("feedback", currentLocale);
+    }
+	
+    /**
+     *  This method will get all information about the JTextComponent.
+     *  If the JTextComponent is a JTextField then it will add the special action listener to it.
+     *  (Precondition: txt != null)
+     *  @param txt its the JTextComponent we want to get all the information from.
+     *  @return    a UserComponent that hold the information about this
+     *             JTextComponent.
+     */
+    public  UserComponent getTextAreaInfo(JTextComponent txt)
+    {
+	UserComponent file;
+	file = new UserComponent(); 
+	String text;
+	text = txt.getText();
+
+	file.startContent(1);
+	if (txt instanceof JTextArea)
+	    file.saveContent("JTextArea");
+	else if (txt instanceof JTextPane)
+	    file.saveContent("JTextPane");
+	else if (txt instanceof JEditorPane)
+	    file.saveContent("JEditorPane");
+	else if (txt instanceof JTextField)
+	    {
+		((JTextField)txt).removeActionListener(m);
+		((JTextField)txt).addActionListener(m);
+		
+		if (txt instanceof JFormattedTextField)
+		    {
+			file.saveContent("JFormattedTextField");
+		    }
+		else if (txt instanceof JPasswordField)
+		    {
+			file.saveContent("JPasswordField");
+		    }
+		else
+		    {
+			file.saveContent("JTextField");
+		    }
+	    }
+
+	file.startContent(2);
+	file.saveContent(messages.getString("TextComponent"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	file.startContent(4);
+	file.saveContent(txt.getSelectedText());
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + txt.isVisible()));
+
+	file.startContent(7);
+	file.saveContent(txt.getToolTipText());
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add the special action listener to the JButton.
+     *  (Precondition: button != null)
+     *  @param button its the JButton we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JButton.
+     */
+    public UserComponent getButtonInfo (JButton button)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	String text;
+	text = button.getText();
+
+	button.removeActionListener(m);
+	button.addActionListener(m);
+
+	file.startContent(1);
+	file.saveContent("JButton");
+
+	file.startContent(2);
+	file.saveContent(messages.getString("Button"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+
+	file.startContent(4);
+	file.saveContent(messages.getString("" + button.isSelected()));
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + button.isVisible()));
+	
+	file.startContent(6);
+	Icon img = button.getIcon();
+	if (img != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+
+	file.startContent(7);
+	file.saveContent(button.getToolTipText());
+
+	return file;
+    }
+
+     /**
+     *  This method will get all the information and add the special change listener to the JProgressBar.
+     *  (Precondition: bar != null)
+     *  @param  bar   its the JProgressBar we want to get information from.
+     *  @return       a UserComponent that will hold the information about this JProgressBar.
+     */
+    public UserComponent getProgressBarInfo (JProgressBar bar)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	String text;
+	text = bar.getString();
+
+	bar.removeChangeListener(m);
+	bar.addChangeListener(m);
+	
+	file.startContent(1);
+	file.saveContent("JProgressBar");
+
+	file.startContent(2);
+	file.saveContent(messages.getString("ProgressBar"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + bar.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(bar.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information and add the special change listener to the JSlider.
+     *  (Precondition: slider != null)
+     *  @param  slider its the JSlider we want to get information from.
+     *  @return        a UserComponent that will hold the information about this JSlider.
+     */
+    public UserComponent getSliderInfo (JSlider slider)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	String text;
+	text = "" + slider.getValue();
+	
+	slider.removeChangeListener(m);
+	slider.addChangeListener(m);
+
+	file.startContent(1);
+	file.saveContent("JSlider");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Slider"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + slider.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(slider.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add the special change listener to the JSpinner.
+     *  (Precondition: spinner != null)
+     *  @param spinner its the JSpinner we want to the information from.
+     *  @return        a UserComponent that will hold the information about this JSpinner.
+     */
+    public UserComponent getSpinnerInfo (JSpinner spinner)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	String text;
+	text = "" + spinner.getValue();
+	
+	spinner.removeChangeListener(m);
+	spinner.addChangeListener(m);
+
+	file.startContent(1);
+	file.saveContent("JSpinner");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Spinner"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + spinner.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(spinner.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information of  the JLabel.
+     *  (Precondition: lbl != null)
+     *  @param lbl    its the JLabel we want to get information from.
+     *  @return       a UserComponent that will hold the information about this JLabel.
+     */
+    public UserComponent getLabelInfo(JLabel lbl)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	String text;
+	text = lbl.getText();
+
+	file.startContent(1);
+	file.saveContent("JLabel");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Label"));
+	
+	file.startContent(3);
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + lbl.isVisible()));
+
+	file.startContent(6);
+	Icon img = lbl.getIcon();
+	if (img != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(lbl.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information and add the special action listener to the JComboBox.
+     *  (Precondition: box != null)
+     *  @param box    its the JComboBox we want to get information from.
+     *  @return       a UserComponent that will hold the information about this JComboBox.
+     */
+    public  UserComponent getComboBoxInfo(JComboBox box)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	ComboBoxModel cm;
+	cm = box.getModel();
+
+	box.removeActionListener(m);
+	box.addActionListener(m);
+	
+	file.startContent(1);
+	file.saveContent("JComboBox");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("ComboBox"));
+	
+	file.startContent(3);
+	int i;
+	for (i = 0; i < box.getItemCount() ; i++)
+	    {
+		UserComponent file2;
+		file2 = new UserComponent();
+		//Assuming that they don't put any picture or images
+		String text;
+		Object objmsg;
+
+		objmsg = box.getItemAt(i);
+
+		if (objmsg instanceof String)
+		    text = (String) objmsg + "\n";
+		else
+		    text = objmsg.toString() + "\n" ;
+		
+		file2.startContent(3);
+		if (text != null)
+		    file2.saveContent(text);
+		else
+		    file2.saveContent(" ");
+		
+		file2.startContent(4);
+		if (i == box.getSelectedIndex())
+		    file2.saveContent(messages.getString("Selected"));
+		else
+		    file2.saveContent(messages.getString("NotSelected"));
+		
+		file.saveContent(file2);
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + box.isVisible()));
+
+	file.startContent(7);
+	file.saveContent(box.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add the special action listener to the JToggleButton.
+     *  (Precondition: button != null)
+     *  @param button its a JToggleButton inside the window
+     *  @return       a UserComponent that will hold the information about this JToggleButton.
+     */
+    public UserComponent getToggleButtonInfo (JToggleButton button)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	button.removeActionListener(m);
+	button.addActionListener(m);
+
+	file.startContent(1);
+	if (button instanceof JCheckBox)
+	    file.saveContent("JCheckBox");
+	else
+	    file.saveContent("JRadioButton");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("ToggleButton"));
+	
+	file.startContent(3);
+	String text;
+	text = button.getText();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	file.startContent(4);
+	if (button.isSelected() == true)
+	    {
+		file.saveContent(messages.getString("Selected"));
+	    }
+	else
+	    file.saveContent(messages.getString("NotSelected"));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + button.isVisible()));
+	
+	file.startContent(6);
+	Icon img = button.getIcon();
+	if (img != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(button.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add the special action listener to the JMenuItem.
+     *  (Precondition: menuitem != null)
+     *  @param menuitem its the JMenuItem we want to get the information from.
+     *  @return         a UserComponent that will hold the information about this JMenuItem.
+     */
+    public UserComponent getMenuItemInfo (JMenuItem menuitem)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	menuitem.removeActionListener(m);
+	menuitem.addActionListener(m);
+	
+	file.startContent(1);
+	file.saveContent("JMenuItem");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("MenuItem"));
+	
+	file.startContent(3);
+	String text;
+	text = menuitem.getText();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	file.startContent(4);
+	if (menuitem.isSelected() == true)
+	    {
+		file.saveContent(messages.getString("Selected"));
+	    }
+	else
+	    file.saveContent(messages.getString("NotSelected"));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  menuitem.isVisible()));
+	
+	file.startContent(6);
+	Icon img = menuitem.getIcon();
+	if (img  != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(menuitem.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JMenuBar.
+     *  (Precondition: menubar != null)
+     *  @param menubar its the JMenuBar we want to get the information from.
+     *  @return        a UserComponent that will hold the information about this JMenuBar.
+     */
+    public  UserComponent getMenuBarInfo (JMenuBar menubar)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	file.startContent(1);
+	file.saveContent("JMenuBar");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("MenuBar"));
+	
+	file.startContent(3);
+	int i;
+	for (i= 0 ; i < menubar.getMenuCount() ; i++)
+	    {
+		file.saveContent(getInside(menubar.getMenu(i)));
+	    }
+	
+	file.startContent(4);
+	if (menubar.isSelected() == true)
+	    {
+		file.saveContent(messages.getString("Selected"));
+	    }
+	else
+	    file.saveContent(messages.getString("NotSelected"));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  menubar.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(menubar.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add the special action and menu listener to the JMenu.
+     *  (Precondition: menu != null)
+     *  @param menu   its the JMenu we want to the the information from.
+     *  @return       a UserComponent that will hold the information about this JMenu.
+     */
+    public  UserComponent getMenuInfo (JMenu menu)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	menu.removeActionListener(m);
+	menu.removeMenuListener(m);
+	menu.addActionListener(m);
+	menu.addMenuListener(m);
+
+	file.startContent(1);
+	file.saveContent("JMenu");
+	
+	file.startContent(2);
+	String text;
+	text = menu.getText();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	file.startContent(3);
+	int i;
+	for (i= 0 ; i < menu.getMenuComponentCount() ; i++)
+	    {
+		file.saveContent(getInside((Container)menu.getMenuComponent(i)));
+	    }
+	
+	file.startContent(4);
+	if (menu.isSelected() == true)
+	    {
+		file.saveContent(messages.getString("Selected"));
+	    }
+	else
+	    file.saveContent(messages.getString("NotSelected"));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  menu.isVisible()));
+	
+	file.startContent(6);
+	Icon img = menu.getIcon();
+	if (img  != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(menu.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information of the JApplet.
+     *  (Precondition: applet != null)
+     *  @param applet its the JApplet we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JApplet.
+     */
+    public  UserComponent getAppletInfo (JApplet applet)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = applet.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JApplet");
+	
+	file.startContent(2);
+	String text;
+	text = applet.getAppletInfo();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)applet.getComponent(i)));
+		    }
+	    }
+	
+	file.startContent(4);
+	file.saveContent(messages.getString("" + applet.isActive()));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + applet.isVisible()));
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JWindow.
+     *  (Precondition: window != null)
+     *  @param window its the JWindow we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JWindow.
+     */
+    public  UserComponent getWindowInfo (JWindow window)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = window.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JWindow");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Window"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)window.getComponent(i)));
+		    }
+	    }
+	
+	file.startContent(4);
+	file.saveContent(messages.getString("" + window.isActive()));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + window.isVisible()));
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JDialog.
+     *  (Precondition: dialog != null)
+     *  @param dialog its the JDialog we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JDialog.
+     */
+    public UserComponent getDialogInfo (JDialog dialog)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = dialog.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JDialog");
+	
+	file.startContent(2);
+	String text = dialog.getTitle();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)dialog.getComponent(i)));
+		    }
+	    }
+	
+	file.startContent(4);
+	file.saveContent(messages.getString("" + dialog.isActive()));
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + dialog.isVisible()));
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JInternalFrame.
+     *  (Precondition: frame != null)
+     *  @param frame  its the JInternalFrame we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JInternalFrame.
+     */
+    public UserComponent getInternalFrameInfo (JInternalFrame frame)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = frame.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JInternalFrame");
+	
+	file.startContent(2);
+	String text;
+	text = frame.getTitle();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)frame.getComponent(i)));
+		    }
+	    }
+	
+	file.startContent(4);
+	file.saveContent(messages.getString("" + frame.isSelected()));
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + frame.isVisible()));
+	
+	file.startContent(6);
+	Icon img = frame.getFrameIcon();
+	if (img != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(frame.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JFrame.
+     *  (Precondition: frame != null)
+     *  @param frame  its the JFrame we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JFrame.
+     */
+    public UserComponent getFrameInfo (JFrame frame)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = frame.getComponents();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JFrame");
+	
+	file.startContent(2);
+	String text;
+	text = frame.getTitle();
+	if (text != null)
+	    file.saveContent(text);
+	else
+	    file.saveContent(" ");
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)frame.getComponent(i)));
+		    }
+	    }
+	
+	file.startContent(4);
+	file.saveContent(messages.getString("" + frame.isActive()));
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + frame.isVisible()));
+	
+	file.startContent(6);
+	Image image;
+	image = frame.getIconImage();
+	if (image != null)
+	    {
+		ImageIcon img;
+		img = new ImageIcon(image);
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JPanel.
+     *  (Precondition: pane != null)
+     *  @param pane   its the JPanel we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JPanel.
+     */
+    public UserComponent getPanelInfo (JPanel pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JPanel");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Panel"));
+
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)pane.getComponent(i)));
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(pane.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information of the JScrollPane.
+     *  (Precondition: scroll != null)
+     *  @param scroll its the JScrollPane we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JScrollPane.
+     */
+    public UserComponent getScrollInfo (JScrollPane scroll)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = scroll.getComponents();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JScrollPane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Scroller"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)group[i]));
+		    }
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + scroll.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(scroll.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special change listener to the JTabbedPane.
+     *  (Precondition: tab != null)
+     *  @param tab   its the JTabbedPane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JTabbedPane.
+     */
+    public UserComponent getTabbedPaneInfo (JTabbedPane tab)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	int j;
+	
+	tab.removeChangeListener(m);
+	tab.addChangeListener(m);
+
+	file.startContent(1);
+	file.saveContent("JTabbedPane");
+	
+	file.startContent(3);
+	for (j = 0 ; j < tab.getTabCount(); j++)
+	    {	
+		UserComponent file2;
+		file2 = new UserComponent();
+
+		file2.startContent(2);
+		String title;
+		title = tab.getTitleAt(j);
+		if (title != null)
+		    file2.saveContent(title);
+		else
+		    file2.saveContent(" ");
+		
+		file2.startContent(6);
+		Icon img;
+		img = tab.getIconAt(j);
+		if (img != null)
+		    {
+			file2.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+		    }
+		else
+		    {
+			file2.saveContent(messages.getString("NoImage"));
+		    }
+		
+		file2.startContent(3);
+		Component group = tab.getComponentAt(j);
+		file2.saveContent(getInside((Container)group));
+		
+		file2.startContent(4);
+		if (j == tab.getSelectedIndex())
+		    file2.saveContent(messages.getString("Selected"));
+		else
+		    file2.saveContent(messages.getString("NotSelected"));
+		
+		file.saveContent(file2);
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + tab.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(tab.getToolTipText());
+
+	return file;
+    }
+    
+    /**
+     *  This method will get all the information of the JOptionPane.
+     *  (Precondition: op != null)
+     *  @param op   its the JOptionPane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JOptionPane.
+     */
+    public UserComponent getOptionPaneInfo (JOptionPane op)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	file.startContent(1);
+	file.saveContent("JOptionPane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("OptionPane"));
+	
+	file.startContent(3);
+	Object objmsg;
+	objmsg = op.getMessage();
+
+	if (objmsg != null)
+	    {
+		String text;
+		if (objmsg instanceof String)
+		    text = (String) objmsg + "\n";
+		else
+		    text = objmsg.toString() + "\n" ;
+		file.saveContent(text);
+	    }
+	else
+	    file.saveContent(" ");
+	
+	Object[] obj;
+	int i;
+	obj = op.getOptions();
+	if (obj != null)
+	    {
+		for (i = 0; i < obj.length ; i++)
+		    {
+			if (obj[i] instanceof String)
+			    file.saveContent((String) obj[i]);
+			else
+			    file.saveContent(obj[i].toString());
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + op.isVisible()));
+	
+	file.startContent(6);
+	Icon img = op.getIcon();
+	if (img != null)
+	    {
+		file.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+	    }
+	else
+	    {
+		file.saveContent(messages.getString("NoImage"));
+	    }
+	
+	file.startContent(7);
+	file.saveContent(op.getToolTipText());
+
+	return file;
+	
+    }
+
+    /**
+     *  This method will get all the information of the JViewPort.
+     *  (Precondition: vp != null)
+     *  @param vp   its the JViewPort we want to get the information from.
+     *  @return     a UserComponent that will hold the information about this JViewPort.
+     */
+    public  UserComponent getViewPortInfo (JViewport vp)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = vp.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JViewPort");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("ViewPort"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)group[i]));
+		    }
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + vp.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(vp.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the Box.
+     *  (Precondition: box != null)
+     *  @param box  its the Box we want to get the information from.
+     *  @return     a UserComponent that will hold the information about this Box.
+     */
+    public UserComponent getBoxInfo (Box box)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = box.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("Box");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Box"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)group[i]));
+		    }
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + box.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(box.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JSplitPane.
+     *  (Precondition: split != null)
+     *  @param split  its the JSplitPane we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JSplitPane.
+     */
+    public  UserComponent getSplitPaneInfo (JSplitPane split)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = split.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JSplitPane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Splitters"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)group[i]));
+		    }
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + split.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(split.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special list selection listener to the JList.
+     *  (Precondition: list != null)
+     *  @param list  its the JList we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JList.
+     */
+    public UserComponent getListInfo (JList list)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	ListModel ls;
+	ls = list.getModel();
+	int i;
+	
+	list.removeListSelectionListener(m);
+	list.addListSelectionListener(m);
+	
+	if (ls != null)
+	    {
+		if (hash.containsKey(ls) == false)
+		    hash.put(ls,list);
+		ls.removeListDataListener(m);
+		ls.addListDataListener(m);
+	    }
+
+	file.startContent(1);
+	file.saveContent("JList");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("List"));
+	
+	if (ls != null)
+	    {
+		file.startContent(3);
+		for ( i = 0 ; i < ls.getSize() ; i++)
+		    {
+			UserComponent file2;
+			file2 = new UserComponent();
+			//Assume that its all a text
+			file2.startContent(3);
+			String text;
+			Object obj;
+			obj = ls.getElementAt(i);
+			
+			if (obj instanceof String)
+			    text = (String) obj;
+			else
+			    text = obj.toString();
+			
+			if (text != null)
+			    file2.saveContent(text);
+			else
+			    file2.saveContent(" ");
+			
+			file2.startContent(4);
+			if ( i == list.getSelectedIndex())
+			    {
+				file2.saveContent(messages.getString("Selected"));
+			    }
+			else
+			    file2.saveContent(messages.getString("NotSelected"));
+			
+			file.saveContent(file2);
+		    }
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + list.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(list.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special change listener to the JColorChooser.
+     *  (Precondition: color != null)
+     *  @param color  its the JColorChooser we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JColorChooser.
+     */
+    public UserComponent getColorInfo (JColorChooser color)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	ColorSelectionModel cm;
+	cm = color.getSelectionModel();
+	int i;
+	
+	if (cm != null)
+	    {
+		if (hash.containsKey(cm) == false)
+		    hash.put(cm,color);
+		cm.removeChangeListener(m);
+		cm.addChangeListener(m);
+	    }
+	
+	file.startContent(1);
+	file.saveContent("JColorChooser");
+
+	file.startContent(2);
+	file.saveContent(messages.getString("ColorChooser"));
+
+	file.startContent(3);
+	UserComponent file2;
+	file2 = new UserComponent();
+	file2.startContent(2);
+	file2.saveContent(messages.getString("PreviewPanel"));
+
+	file2.startContent(3);
+	file2.saveContent(getInside((Container) color.getPreviewPanel()));
+	
+	file.saveContent(file2);
+
+	file2 = new UserComponent();
+	file2.startContent(2);
+	file2.saveContent(messages.getString("ChooserPanels"));
+
+	AbstractColorChooserPanel[] cp;
+	cp = color.getChooserPanels();
+	if ((cp != null) && (cp.length != 0))
+	    {
+		file2.startContent(3);
+		for (i = 0 ; i < cp.length ; i++)
+		    {
+			UserComponent file3;
+			file3 = new UserComponent();
+			file3.startContent(2);
+			file3.saveContent(cp[i].getDisplayName());
+			
+			ColorSelectionModel cpsm;
+			cpsm = cp[i].getColorSelectionModel();
+
+			if (cpsm != null)
+			    {
+				file3.startContent(3);
+				file3.saveContent("" + 
+						  cpsm.getSelectedColor());
+			    }
+
+			file3.startContent(6);
+			Icon img = cp[i].getSmallDisplayIcon();
+			if (img  != null)
+			    {
+				file3.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+			    }
+			else
+			    {
+				file3.saveContent(messages.getString("NoImage"));
+			    }
+			
+			file3.startContent(6);
+			img = cp[i].getLargeDisplayIcon();
+			if (img  != null)
+			    {
+				file3.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+			    }
+			else
+			    {
+				file3.saveContent(messages.getString("NoImage"));
+			    }
+			file2.saveContent(file3);
+		    }
+	    }
+		
+	file.saveContent(file2);    
+	
+	file.startContent(4);
+	file.saveContent("" + cm.getSelectedColor().toString());
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  color.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(color.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special action listener to the JFileChooser.
+     *  (Precondition: fc != null)
+     *  @param fc  its the JFileChooser we want to get the information from.
+     *  @return    a UserComponent that will hold the information about this JFileChooser.
+     */
+    public UserComponent getFileChooserInfo (JFileChooser fc)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	int i;
+
+	fc.removeActionListener(m);
+	fc.addActionListener(m);
+
+	file.startContent(1);
+	file.saveContent("JFileChooser");
+	
+	file.startContent(2);
+	file.saveContent(fc.getDialogTitle());
+
+	File[] f;
+	f = fc.getSelectedFiles();
+	
+	if (f != null)
+	    {
+		file.startContent(3);
+		
+		for ( i = 0 ; i < f.length ; i++)
+		    {
+			UserComponent file2;
+			file2 = new UserComponent();
+			file2.startContent(1);
+			file2.saveContent(fc.getTypeDescription(f[i]));
+			
+			file2.startContent(2);
+			file2.saveContent(fc.getName(f[i]));
+			
+			file2.startContent(6);
+			Icon img = fc.getIcon(f[i]);
+			if (img  != null)
+			    {
+				file2.saveImage(img,messages.getString("icon")+img.hashCode()+".jpg");
+			    }
+			else
+			    {
+				file2.saveContent(messages.getString("NoImage"));
+			    }
+			file.saveContent(file2);
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  fc.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(fc.getToolTipText());
+
+	return file;
+    }		
+
+    /**
+     *  This method will get all the information and add special list selection and column model listener 
+     *  to the JTable.
+     *  (Precondition: table != null)
+     *  @param table  its the JTable we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JTable.
+     */
+    public UserComponent getTableInfo (JTable table)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	TableModel tm;
+	tm = table.getModel();
+	int i,j;
+	String text;
+
+	TableColumnModel tcm;
+	tcm = table.getColumnModel();
+	ListSelectionModel lm;
+	lm = table.getSelectionModel(); 
+
+	if (lm != null)
+	    {
+		if (hash.containsKey(lm) == false)
+		    hash.put(lm,table);
+		lm.removeListSelectionListener(m);
+		lm.addListSelectionListener(m);
+	    }
+	
+	if (tcm != null)
+	    {
+		if (hash.containsKey(tcm) == false)
+		    hash.put(tcm,table);
+		tcm.removeColumnModelListener(m);
+		tcm.addColumnModelListener(m);
+	    }
+
+	file.startContent(1);
+	file.saveContent("JTable");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Table"));
+	
+	if (tm != null)
+	    {
+		file.startContent(3);
+		for ( i = 0 ; i < tm.getRowCount() ; i++)
+		    {
+			UserComponent file2;
+			file2 = new UserComponent();
+			
+			file2.startContent(2);
+			text = messages.getString("row") +" " + (i + 1);
+			file2.saveContent(text);
+			
+			file2.startContent(3);
+			for ( j = 0 ; j < tm.getColumnCount() ; j++)
+			    {
+				UserComponent file3;
+				file3 = new UserComponent();
+				
+				file3.startContent(2);
+				file3.saveContent(tm.getColumnName(j));
+				
+				file3.startContent(3);
+				
+				Object obj;
+				obj = tm.getValueAt(i,j);
+				
+				if (obj instanceof String)
+				    file3.saveContent((String)obj);
+				else
+				    file3.saveContent(obj.toString());
+				
+				file3.startContent(4);
+				if (table.isCellSelected(i,j) == true)
+				    {
+					file3.saveContent(messages.getString("Selected"));
+				    }
+				else
+				    file3.saveContent(messages.getString("NotSelected"));
+				
+				file2.saveContent(file3);
+			    }
+			
+			file.saveContent(file2);
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  table.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(table.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JScrollBar.
+     *  (Precondition: scroll != null)
+     *  @param scroll  its the JScrollBar we want to get the information from.
+     *  @return         a UserComponent that will hold the information about this JScrollBar.
+     */
+    public  UserComponent getScrollBarInfo (JScrollBar scroll)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	file.startContent(1);
+	file.saveContent("JScrollBar");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("ScrollerBar"));
+	
+	file.startContent(3);
+	file.saveContent(""+ scroll.getOrientation()+ " :"+ scroll.getValue());
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + scroll.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(scroll.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special tree selection listener to the JTree.
+     *  (Precondition: tree != null)
+     *  @param tree  its the JTree we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JTree.
+     */
+    public UserComponent getTreeInfo (JTree tree)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	TreeModel tm;
+	tm = tree.getModel();
+	int i;
+	
+	tree.removeTreeSelectionListener(m);
+	tree.addTreeSelectionListener(m);
+	
+	if (tm != null)
+	    {
+		if (hash.containsKey(tm) == false)
+		    hash.put(tm,tree);
+		tm.removeTreeModelListener(m);
+		tm.addTreeModelListener(m);
+	    }
+	
+	file.startContent(1);
+	file.saveContent("JTree");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Tree"));
+	
+	file.startContent(3);
+	Object root;
+	root = tm.getRoot();
+	file.saveContent(getChildInfo(tm,root,tree));
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" +  tree.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(tree.getToolTipText());
+
+	return file;
+    }
+       
+    /**
+     *  This method will get all the information about the parent node and all the childs of it inside the JTree.
+     *  (Precondition: (tree != null) && (tm != null) && (parent != null))
+     *  @param tm      its the tree model of the JTree.
+     *  @param parent  its the parent node that we want to get all the child information from.
+     *  @param tree    its the JTree where the parent node belongs to.
+     *  @return        a UserComponent that will hold the information about this parent node.  
+     */
+    private UserComponent getChildInfo (TreeModel tm,Object parent,JTree tree)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	if (tm.isLeaf(parent) == false)
+	    {
+		file.startContent(2);
+		String text;
+		text = parent.toString();
+		if (text != null)
+		    file.saveContent(text);
+		else
+		    file.saveContent(" ");
+		
+		int j;
+		for (j = 0 ; j < tm.getChildCount(parent) ; j++)
+		    {
+			file.startContent(3);
+			Object child;
+			child = tm.getChild(parent,j);
+			file.saveContent(getChildInfo(tm,child,tree));
+		    }
+	    }
+	else
+	    {
+		file.startContent(2);
+		String text;
+		text = parent.toString();
+		if (text != null)
+		    file.saveContent(text);
+		else
+		    file.saveContent(" ");
+	    }
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of a Container that is not belongs to any of the more specified
+     *  method stated in this class.
+     *  (Precondition: pane != null)
+     *  @param pane  its the Container component we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this Container.
+     */
+    public UserComponent getCompInfo (Container pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("Container");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Container"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		file.saveContent(messages.getString("TitlePane"));
+		file.saveContent(getInside(null));
+		/*for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)pane.getComponent(i)));
+			}*/
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JPopupMenu.
+     *  (Precondition: popup != null)
+     *  @param popup  its the JPopUpMenu we want to get the information from.
+     *  @return       a UserComponent that will hold the information about this JPopupMenu.
+     */
+    public UserComponent getPopUpMenuInfo (JPopupMenu popup)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = popup.getComponents();
+	SingleSelectionModel sm;
+	sm = popup.getSelectionModel();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("JPopUpMenu");
+
+	file.startContent(2);
+	String text;
+	text = popup.getLabel();
+	if (text != null)
+	    file.saveContent(popup.getLabel());
+	else
+	    file.saveContent(" ");
+
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)popup.getComponent(i)));
+		    }
+	    }
+
+	if (sm != null)
+	    {
+		file.startContent(4);
+		file.saveContent("" + sm.getSelectedIndex());
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + popup.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(popup.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JRootPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JRootPane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JRootPane.
+     */
+    public UserComponent getRootPaneInfo (JRootPane pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JRootPane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("RootPane"));
+
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			
+			file.saveContent(getInside((Container)pane.getComponent(i)));
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(pane.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JLayeredPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JLayeredPane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JLayeredPane.
+     */
+    public UserComponent getLayeredPaneInfo (JLayeredPane pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JLayeredPane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("LayeredPane"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)pane.getComponent(i)));
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(pane.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the BasicInternalFrameTitlePane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the BasicInternalFrameTitlePane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this BasicInternalFrameTitlePane.
+     */
+    public UserComponent getInternalFrameTitleInfo (BasicInternalFrameTitlePane pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = pane.getComponents();
+	int i;
+
+	file.startContent(1);
+	file.saveContent("BasicInternalFrameTitlePane");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("InternalFrameTitlePane"));
+	
+	/*if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			Component c;
+			c = pane.getComponent(i);
+			
+			else
+			    file.saveContent(getInside((Container)pane.getComponent(i)));
+		    }
+		    }*/
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(pane.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information and add special column model and list selection
+     *  listener to JTableHeader.
+     *  (Precondition: header != null)
+     *  @param header  its the JTableHeader we want to get the information from.
+     *  @return        a UserComponent that will hold the information about this JTableHeader.
+     */
+    public UserComponent getTableHeaderInfo (JTableHeader header)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	TableColumnModel tcm;
+	tcm = header.getColumnModel();
+	ListSelectionModel lm;
+	lm = tcm.getSelectionModel();
+
+	if (tcm != null)
+	    {
+		if (hash.containsKey(tcm) == false)
+		    hash.put(tcm,header);
+		tcm.removeColumnModelListener(m);
+		tcm.addColumnModelListener(m);
+	    }
+
+	if (lm != null)
+	    {
+		if (hash.containsKey(lm) == false)
+		    hash.put(lm,header);
+		lm.removeListSelectionListener(m);
+		lm.addListSelectionListener(m);
+	    }
+
+	file.startContent(1);
+	file.saveContent("JTableHeader");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("TableHeader"));
+
+	if (tcm != null)
+	    {
+		int[] select = tcm.getSelectedColumns();
+
+		if (select != null)
+		    {
+			file.startContent(3);
+			for (int i = 0 ; i < select.length ; i++)
+			    {
+				UserComponent file2;
+				file2 = new UserComponent();
+				file2.startContent(4);
+				file2.saveContent("" + select[i]);
+				
+				file.saveContent(file2);
+			    }
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + header.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(header.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JToolBar.
+     *  (Precondition: tool != null)
+     *  @param tool  its the JToolBar we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JToolBar.
+     */
+    public UserComponent getToolBarInfo (JToolBar tool)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component[] group;
+	group = tool.getComponents();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JToolBar");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("ToolBar"));
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		for (i = 0; i < group.length ; i++)
+		    {
+			file.saveContent(getInside((Container)tool.getComponent(i)));
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + tool.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(tool.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JToolTip.
+     *  (Precondition: tip != null)
+     *  @param tip  its the JToolTip we want to get the information from.
+     *  @return     a UserComponent that will hold the information about this JToolTip.
+     */
+    public UserComponent getToolTipInfo (JToolTip tip)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	Component group;
+	group = tip.getComponent();
+	int i;
+	
+	file.startContent(1);
+	file.saveContent("JToolTip");
+	
+	file.startContent(2);
+	String text;
+	text = tip.getTipText();
+	if (text != null)
+	    file.saveContent(tip.getTipText());
+	else
+	    file.saveContent(" ");
+	
+	if (group != null)
+	    {
+		file.startContent(3);
+		file.saveContent(getInside((Container) group));
+	    }
+	
+	file.startContent(5);
+	file.saveContent(messages.getString("" + tip.isVisible()));
+	
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JSeparator.
+     *  (Precondition: sept != null)
+     *  @param sept  its the JSeparator we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JSeparator.
+     */
+    public UserComponent getSeparatorInfo (JSeparator sept)
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	file.startContent(1);
+	file.saveContent("JSeparator");
+	
+	file.startContent(2);
+	file.saveContent(messages.getString("Separator"));
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + sept.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(sept.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will get all the information of the JDesktopPane.
+     *  (Precondition: pane != null)
+     *  @param pane  its the JDesktopPane we want to get the information from.
+     *  @return      a UserComponent that will hold the information about this JDesktopPane.
+     */
+    public UserComponent getDesktopPaneInfo (JDesktopPane pane)
+    {
+	UserComponent file;
+	file = new UserComponent();
+	JInternalFrame[]  group;
+	group = pane.getAllFrames();
+
+	file.startContent(1);
+	file.saveContent("JDesktopPane");
+
+	file.startContent(2);
+	file.saveContent(messages.getString("DesktopPane"));
+
+	if (group != null)
+	    {
+		int i;
+		file.startContent(3);
+		for ( i = 0 ; i < group.length ; i++ )
+		    {
+			file.saveContent (getInside((Container) group[i]));
+		    }
+	    }
+
+	file.startContent(5);
+	file.saveContent(messages.getString("" + pane.isVisible()));
+	
+	file.startContent(7);
+	file.saveContent(pane.getToolTipText());
+
+	return file;
+    }
+
+    /**
+     *  This method will give you a UserComponent that hold the information saying
+     *  the component is null (no component).
+     *  @return  a UserComponent that will hold the information saying its a null component.
+     */
+    public UserComponent getNullInfo ()
+    {
+	UserComponent file;
+	file = new UserComponent();
+
+	file.startContent(1);
+	file.saveContent(messages.getString("NoComponent"));
+
+	return file;
+    }
+
+    /**
+     *  This method will get all information about any kind of Container.
+     *  This method is the gateway to all other methods in this class and will give you all
+     *  information about the Container and whats inside it.If you dont know what the specific class
+     *  of your container then you go here and it will sort which appropriate method 
+     *  this container should go to in this class.
+     *  @param comp its the Container we want to get the information from.
+     *  @return     a UserComponent that hold the information about this Container.
+     */
+    public  UserComponent getInside(Container comp)
+    {
+	if (comp == null)
+	    {
+		return getNullInfo();
+	    }
+	else if (comp instanceof JApplet)
+	    {
+		return getAppletInfo ((JApplet) comp);
+	    }
+	else if (comp instanceof JSeparator)
+	    {
+		return getSeparatorInfo ((JSeparator) comp);
+	    }
+	else if (comp instanceof JWindow)
+	    {
+		return getWindowInfo((JWindow) comp);
+	    }
+	else if (comp instanceof JDialog)
+	    {
+		return getDialogInfo((JDialog) comp);
+	    }
+	else if (comp instanceof JComboBox)
+	    {
+		return getComboBoxInfo((JComboBox) comp);
+	    }
+	else if (comp instanceof JLabel)
+	    {
+		return getLabelInfo((JLabel) comp);
+	    }
+	else if (comp instanceof JTextComponent)
+	    {
+		return getTextAreaInfo((JTextComponent) comp);
+	    }
+	else if (comp instanceof JFrame)
+	    {
+		return getFrameInfo((JFrame) comp);
+	    }
+	else if (comp instanceof JPanel)
+	    {
+		return getPanelInfo((JPanel) comp);
+	    }
+	else if (comp instanceof JScrollPane)
+	    {
+		return getScrollInfo((JScrollPane) comp);
+	    }
+	else if (comp instanceof JScrollBar)
+	    {
+		return getScrollBarInfo ((JScrollBar) comp);
+	    }
+	else if (comp instanceof JViewport)
+	    {
+		return getViewPortInfo((JViewport) comp);
+	    }
+	else if (comp instanceof JTabbedPane)
+	    {
+		return getTabbedPaneInfo((JTabbedPane) comp);
+	    }
+	else if (comp instanceof Box)
+	    {
+		return getBoxInfo((Box) comp);
+	    }
+	else if (comp instanceof JToggleButton)
+	    {
+		return getToggleButtonInfo((JToggleButton) comp);
+	    }
+	else if (comp instanceof JSplitPane)
+	    {
+		return getSplitPaneInfo ((JSplitPane) comp);
+	    }
+	else if (comp instanceof JList)
+	    {
+		return getListInfo ((JList) comp);
+	    }
+	else if (comp instanceof JTree)
+	    {
+		return getTreeInfo ((JTree) comp);
+	    }
+	else if (comp instanceof JTable)
+	    {
+		return getTableInfo ((JTable) comp);
+	    }
+	else if (comp instanceof JSlider)
+	    {
+		return getSliderInfo ((JSlider) comp);
+	    }
+	else if (comp instanceof JSpinner)
+	    {
+		return getSpinnerInfo ((JSpinner) comp);
+	    }
+	else if (comp instanceof JProgressBar)
+	    {
+		return getProgressBarInfo ((JProgressBar) comp);
+	    }
+	else if (comp instanceof JButton)
+	    {
+		return getButtonInfo ((JButton) comp);
+	    }
+	else if (comp instanceof JMenu)
+	    {
+		return getMenuInfo ((JMenu) comp);
+	    }
+	else if (comp instanceof JMenuBar)
+	    {
+		return getMenuBarInfo ((JMenuBar) comp);
+	    }
+	else if (comp instanceof JMenuItem)
+	    {
+		return getMenuItemInfo ((JMenuItem) comp);
+	    }
+	else if (comp instanceof JOptionPane)
+	    {
+		return getOptionPaneInfo ((JOptionPane) comp);
+	    }
+	else if (comp instanceof BasicInternalFrameTitlePane)
+	    {
+		return getInternalFrameTitleInfo ((BasicInternalFrameTitlePane) comp);
+	    }
+	else if (comp instanceof JColorChooser)
+	    {
+		return getColorInfo ((JColorChooser) comp);
+	    }
+	else if (comp instanceof JFileChooser)
+	    {
+		return getFileChooserInfo ((JFileChooser) comp);
+	    }
+	else if (comp instanceof JDesktopPane)
+	    {
+		return getDesktopPaneInfo ((JDesktopPane) comp);
+	    }
+	else if (comp instanceof JInternalFrame)
+	    {
+		return getInternalFrameInfo ((JInternalFrame) comp);
+	    }
+	else if (comp instanceof JLayeredPane)
+	    {
+		return getLayeredPaneInfo ((JLayeredPane) comp);
+	    }
+	else if (comp instanceof JPopupMenu)
+	    {
+		return getPopUpMenuInfo ((JPopupMenu) comp);
+	    }
+	else if (comp instanceof JRootPane)
+	    {
+		return getRootPaneInfo ((JRootPane) comp);
+	    }
+	else if (comp instanceof JTableHeader)
+	    {
+		return getTableHeaderInfo ((JTableHeader) comp);
+	    }
+	else if (comp instanceof JToolBar)
+	    {
+		return getToolBarInfo ((JToolBar) comp);
+	    }
+	else if (comp instanceof JToolTip)
+	    {
+		return getToolTipInfo ((JToolTip) comp);
+	    }
+	else
+	    {
+		return getCompInfo (comp);
+	    }
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/FeedbackInterface.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/FeedbackInterface.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/FeedbackInterface.java	(revision 21112)
@@ -0,0 +1,1654 @@
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import java.awt.*;
+import java.awt.image.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import java.util.*;
+import java.net.*;
+import java.rmi.server.*;
+import javax.imageio.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import com.sun.image.codec.jpeg.*;
+import javax.swing.Timer;
+import org.greenstone.gatherer.util.Utility;
+
+/**
+ * This class will show the FeedbackInterface form.
+ * It will allow user to described about the problem better.Here we will also take some more information available
+ * in the system such as the size of the memory available and the screen size.User also could take more screenshot
+ * so they will be able to say the sequence of events that lead up to the problem.
+ * <br>If the user don't want to send this form, then all the information taken from this form will be deleted 
+ * and user will be quitted from the Reporting Feedback sequence.
+ *
+ * @author Veronica Liesaputra
+ */
+public class FeedbackInterface extends JDialog
+    implements ActionListener
+{
+    /**
+     * This is the button user need to pressed if they want to view the files.
+     * This button will be disabled when the timer is running and finish = false.
+     */
+    private JButton view_button;
+    
+    /**
+     * This button means that the user finished give the information and
+     * want to send it.
+     */
+    public JButton send_button;
+
+    /**
+     * This button means that the user don't want to send information and
+     * back to normal application.
+     * All the data taken start from user choose to start doing the Feedback sequence 
+     * up to this point will be delete.
+     */
+    private JButton notsend_button; 
+
+    /**
+     * This is the text area where user can explain more about the problem.
+     */
+    private JTextArea problem_details;
+
+    /**
+     * This is the combo box that allow user to specify the poblem type.
+     */
+    private JComboBox problem_type;
+
+    /**
+     * This is the combo box that allow user to specify the urgency of the problem.
+     */
+    private JComboBox problem_urgency;
+
+    /**
+     * This combo box will ask whether or not user want to send the screenshot.
+     */
+    private JComboBox problem_screenshot;
+
+    /**
+     * This text field allow user to type the smtp address of their email that
+     * they want this application to use to send all the data.
+     * If they don't fill this field then it will set to the same
+     * smtp adress of where the data will be send to.
+     */
+    private JTextField smtp_address;
+
+    /**
+     * This text field allow user to type the email adress that they want this
+     * application to use to send all the data.
+     * If they don't fill this field then it will set to the same email
+     * adress of where the data will be send to.
+     */
+    private JTextField email_address;
+
+    /**
+     * This is the name of the window the user lastly used.
+     * This is the name of the window where the user called to start the Feedback
+     * sequence.
+     */
+    private String frameName;
+
+    /**
+     * This is the unique code generated for each time user start doing the Feedback
+     * sequence.
+     * This code will be used as reference to which feedback that the user send.
+     */
+    private static String code;
+
+    /**
+     * This is the date when the user start open this FeedbackInterface form.
+     */
+    private Date curr_date;
+    
+    /**
+     * This is the special listener class that record all the action user did in application.
+     */
+    private static ActionRecorderDialog dialog;
+
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    /**
+     * This is the screen shot instance that will allow the application to take screen shot of the whole
+     * screen or only the window that the user currently doing their action.
+     */
+    private ScreenShot screen;
+
+    /**
+     * This is the thread that allows the encoding of the image file to String and the saving of image to
+     * a jpeg file happens concurrently.
+     */
+    private Thread save;
+
+    /**
+     * This is the hashmap that hold the screen shot and the panel where the screenshot is held.
+     */
+    private HashMap picAndPane;
+
+    /**
+     * This is the panel that hold all the panels containing the screenshots taken.
+     */
+    private JPanel picture_scroll;
+
+    /**
+     * This is how many rows of screenshots panel displayed at the moment.
+     */
+    private int row;
+
+    /**
+     * This is the array list that hold all the screenshots panel displayed.
+     */
+    private ArrayList paneArray;
+
+    /**
+     * This button will remove the last screenshot panels.
+     */
+    private JButton remove;
+
+    /**
+     * This is a reference to feedback interface displayed at the moment.
+     */
+    private FeedbackInterface frame;
+
+    /**
+     * This is the thread to send all the xml files back to the developer.
+     */
+    private Thread sendThread;
+
+    /**
+     * This is the flag to tell whether or not the sending files finished or not.
+     * It is also flag to tell whether or not the viewing files finished or not.
+     */
+    private static boolean finish;
+
+    /**
+     * This is the timer to displayed the progress bar to tell user to wait while
+     * we sending the files or while we generate the xml files to be viewed.
+     * If finish = false then wait cursor,progress bar and wait label will be displayed.
+     * If finish = true then default cursor will be displayed.
+     */
+    private Timer timer;
+
+    /**
+     * Progress bar displayed when user sending or want to view details.
+     */
+    private JProgressBar progressBar;
+
+    /**
+     * Label displayed to tell user to wait for a while.
+     */
+    private JLabel wait_lbl;
+
+    /**
+     * This is the background colour for the window.
+     */
+    private Color bckcolor;
+    
+    /**
+     * This is the panel to displayed all descriptions panel.
+     */
+    private JPanel tempPane;
+
+    /**
+     * This is the thread that will be running when user pressed the preview button.
+     * This thread will show the 6 xml files that are going to be send. It will run concurrently
+     * with the timer, if value of finish = true, this thread will be null-ed.
+     */
+    private Thread prevThread;
+    
+    /**
+     * This is the flag to say whether or not this window should be dispose when timer is stop
+     * and finish = true.
+     * If selected = true then this window should be dispose, otherwise don't dispose it.
+     */
+    private boolean selected = false;
+    
+    /**
+     * This is a blocker to say that user cannot do anything while waiting.
+     */
+    private MouseListener mouse_blocker_listener = new MouseAdapter() {};
+ 
+    /**
+     * Will make a dialog window that allows people to write more about the problem that want
+     * to send.
+     * This is the GUI for user to write on Feedback form. It is also setting the up the appropriate 
+     * data member with the value passed on to this constructor.
+     * (Precondition: (sh != null) && (dlg != null) && (msg != null))
+     * @param sh  an instance of ScreenShot that will allow application to take screenshot.
+     * @param frame_name name of the window where user called Feedback form from.
+     * @param msg holding the resource of the words that is stored in Messages.properties file.
+     * @param dlg holding the special listener.
+     */
+    public FeedbackInterface(ScreenShot sh,String frame_name,
+			     ResourceBundle msg,ActionRecorderDialog dlg)
+    {
+	super();
+
+	frame = this;
+
+	bckcolor = new Color(176,209,217);
+
+	picAndPane = new HashMap();
+
+	messages = msg;
+	frameName = frame_name;
+	dialog = dlg;
+
+	FeedbackInterface.setDefaultLookAndFeelDecorated(true);
+	setDefaultCloseOperation(WindowConstants.DO_NOTHING_ON_CLOSE);
+	setTitle(messages.getString("SendFeedbackForm"));
+	
+	screen = sh;
+	Container container;
+	container = getContentPane();
+	createUI(container);
+	curr_date = new Date();
+	
+	Toolkit kit;
+	kit = Toolkit.getDefaultToolkit();
+	Dimension screenSize;
+	screenSize = kit.getScreenSize();
+	int screenHeight;
+	screenHeight = screenSize.height;
+	int screenWidth;
+	screenWidth = screenSize.width;
+	Dimension size;
+	size = this.getSize();
+	setLocation((screenWidth - size.width - 500) , 
+		    (screenHeight - size.height - 700));
+	
+	setBackground(bckcolor);
+	pack();
+	setModal(false);
+	setVisible(true);
+    }
+
+    /**
+     * This method is setting up the content pane for the Feedback form window.
+     * (Precondition: (window != null))
+     * @param window the content pane of Feedback form window. 
+     */
+    public void createUI(Container window)
+    {
+	JPanel container;
+	container = new JPanel();
+	container.setBackground(bckcolor);
+	container.setLayout(new BoxLayout(container,BoxLayout.PAGE_AXIS));
+	
+	String opening_text;
+	opening_text = messages.getString("FeedbackOpeningText");
+	JTextArea textArea;
+	textArea = new JTextArea(opening_text);
+	textArea.setMinimumSize(new Dimension(100,100));
+	textArea.setOpaque(false);
+	textArea.setEditable(false);
+	textArea.setLineWrap(true);
+	textArea.setWrapStyleWord(true);
+	textArea.setBorder(new EmptyBorder(10,10,0,10));
+	textArea.setBackground(bckcolor);
+	container.add(textArea);
+
+	tempPane = new JPanel();
+	tempPane.setLayout(new BorderLayout());
+	tempPane.setBackground(new Color(176,208,176));
+
+	paneArray = new ArrayList();
+	picture_scroll = new JPanel();
+	row = 1;
+	picture_scroll.setLayout(new GridLayout(0,1));
+	picture_scroll.setBackground(new Color(176,208,176));
+	tempPane.add(picture_scroll,BorderLayout.PAGE_START);
+
+	JPanel btnPanel;
+	btnPanel = new JPanel();
+	btnPanel.setBackground(new Color(176,208,176));
+	remove = new JButton("Remove");
+	remove.setActionCommand("Remove");
+	remove.addActionListener(new ActionListener ()
+	    {
+		public void actionPerformed(ActionEvent e)
+		{
+		    Object obj;
+		    JPanel pane;
+
+		    if (paneArray.size() == 0)
+			System.out.println("There is no panel here!");
+		    else
+			{
+			    pane = (JPanel) paneArray.get((row - 2));
+			    obj = picAndPane.get(pane);
+			    obj = null;
+			    picAndPane.remove(pane);
+			    paneArray.remove((row - 2));
+			    
+			    if (paneArray.size() == 0)
+				remove.setVisible(false);
+
+			    picture_scroll.remove(pane);
+			
+			    row--;
+			    
+			    picture_scroll.setSize(picture_scroll.getPreferredSize());
+			    picture_scroll.revalidate();
+			    picture_scroll.repaint();
+			    tempPane.setSize(tempPane.getPreferredSize());
+			    tempPane.revalidate();
+			    tempPane.repaint();
+			}
+		}
+	    });
+	remove.setToolTipText("Remove last thumbnail");
+	remove.setBackground(new Color(176,208,176));
+	remove.setVisible(false);
+	btnPanel.add(remove);
+	tempPane.add(btnPanel,BorderLayout.PAGE_END);
+   
+	JScrollPane pict_scroll;
+	pict_scroll = new JScrollPane(tempPane);
+	pict_scroll.setBackground(bckcolor);
+	pict_scroll.setPreferredSize(new Dimension(300,300));
+	pict_scroll.setBorder(new EmptyBorder(0,10,10,10));
+	container.add(pict_scroll);
+
+	JPanel pane;
+	pane = new JPanel();
+	pane.setBackground(bckcolor);
+	pane.setLayout(new GridLayout(0,2));
+	pane.setBorder(new EmptyBorder(10,10,10,10));
+		
+	JLabel txt_type;
+	txt_type = new JLabel(messages.getString("Whatkindofproblemisit") + "?    ");
+	txt_type.setBackground(bckcolor);
+	pane.add(txt_type);
+	
+	String [] type = {" ",messages.getString("ContentError"),
+			  messages.getString("ImageNotShow"),
+			  messages.getString("StrangeBehaviour"),
+			  messages.getString("SomethingUnexpectedHappen"),
+			  messages.getString("HardToUse"),
+			  messages.getString("Other")};
+	problem_type = new JComboBox(type);
+	problem_type.addActionListener(this);
+	problem_type.setBackground(bckcolor);
+	pane.add(problem_type);
+
+	JLabel txt_urgent;
+	txt_urgent = new JLabel(messages.getString("Howbadistheproblem")+"?  ");
+	txt_urgent.setBackground(bckcolor);
+	pane.add(txt_urgent);
+
+	String[] urgent = {" ",messages.getString("Critical"),
+		  messages.getString("Serious"),
+		  messages.getString("Medium"),messages.getString("Minor"),
+		  messages.getString("Trivial")};
+	problem_urgency = new JComboBox(urgent);
+	problem_urgency.addActionListener(this);
+	problem_urgency.setBackground(bckcolor);
+	pane.add(problem_urgency);
+
+	/*JLabel txt_screenshot;
+	txt_screenshot= new JLabel(messages.getString("Doyouwanttosendthescreenshot") + "?  ");
+	txt_screenshot.setBackground(new Color(108,168,108));
+	pane.add(txt_screenshot);
+
+	String[] screenshot = {messages.getString("Yes"),
+			       messages.getString("No")};
+	problem_screenshot = new JComboBox(screenshot);
+	problem_screenshot.addActionListener(this);
+	problem_screenshot.setBackground(new Color(108,168,108));
+	pane.add(problem_screenshot);*/
+	
+	
+	/*JLabel txt_smtp;
+	txt_smtp = new JLabel(messages.getString("SMTPaddress") + "?  ");
+	txt_smtp.setBackground(new Color(176,208,176));
+	pane.add(txt_smtp);
+
+	smtp_address = new JTextField();
+	smtp_address.setBackground(new Color(224,240,224));
+	pane.add(smtp_address);*/
+	
+
+	/*JLabel txt_email;
+	txt_email = new JLabel(messages.getString("Emailaddress") + "?  ");
+	txt_email.setBackground(new Color(176,208,176));
+	pane.add(txt_email);
+
+	email_address = new JTextField();
+	email_address.setBackground(new Color(224,240,224));
+	pane.add(email_address);*/
+	
+	JLabel empty;
+	empty = new JLabel ("   ");
+	empty.setBackground(bckcolor);
+	pane.add(empty);
+	JLabel empty2;
+	empty2 = new JLabel ("   ");
+	empty2.setBackground(bckcolor);
+	pane.add(empty2);
+	container.add(pane);
+
+	JPanel panely;
+	panely = new JPanel();
+	panely.setLayout(new GridLayout(0,3));
+	panely.setBackground(bckcolor);
+	panely.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+	
+	view_button = new JButton("Details . . .");
+	view_button.setActionCommand(messages.getString("Preview"));
+	view_button.addActionListener(this);
+	view_button.setToolTipText(messages.getString("ConformationPreview"));
+	view_button.setBackground(bckcolor);
+	panely.add(view_button); 
+
+	send_button = new JButton(messages.getString("Send"));
+	send_button.setActionCommand(messages.getString("Send"));
+	send_button.addActionListener(this);
+	send_button.setDefaultCapable(true);
+	send_button.setToolTipText(messages.getString("FeedbackSendButton"));
+	send_button.setBackground(bckcolor);
+	panely.add(send_button);
+	
+	notsend_button = new JButton("Cancel");
+	notsend_button.setActionCommand(messages.getString("NotSend"));
+	notsend_button.addActionListener(this);
+	notsend_button.setToolTipText(messages.getString("FeedbackNotSendButton"));
+	notsend_button.setBackground(bckcolor);
+	panely.add(notsend_button);
+	container.add(panely);
+
+	//Align the left edges of the components.
+	textArea.setAlignmentX(Component.LEFT_ALIGNMENT);
+	pict_scroll.setAlignmentX(Component.LEFT_ALIGNMENT);
+	pane.setAlignmentX(Component.LEFT_ALIGNMENT);
+	panely.setAlignmentX(Component.LEFT_ALIGNMENT);
+
+	String about;
+	about = messages.getString("FeedbackAboutText");
+	JTextArea about_txt;
+	about_txt = new JTextArea(about);
+	about_txt.setEditable(false);
+	about_txt.setLineWrap(true);
+	about_txt.setWrapStyleWord(true);
+	about_txt.setBackground(bckcolor);
+	about_txt.setBorder(new EmptyBorder(10,10,10,10));
+
+	String privacy;
+	privacy = messages.getString("FeedbackPrivacyText");
+	JTextArea privacy_txt;
+	privacy_txt = new JTextArea(privacy);
+	privacy_txt.setEditable(false);
+	privacy_txt.setLineWrap(true);
+	privacy_txt.setWrapStyleWord(true);
+	privacy_txt.setBackground(bckcolor);
+	privacy_txt.setBorder(new EmptyBorder(10,10,10,10));
+ 
+	JTabbedPane tab;
+	tab = new JTabbedPane();
+	tab.setBackground(bckcolor);
+	tab.addTab(messages.getString("Form"),container);
+	tab.addTab(messages.getString("About"),about_txt);
+	tab.addTab(messages.getString("Privacy"),privacy_txt);
+	tab.setBorder(new EmptyBorder(5,5,5,5));
+	
+	JPanel panelx;
+	panelx = new JPanel();
+	panelx.setLayout(new BoxLayout(panelx,BoxLayout.PAGE_AXIS));
+	panelx.setBackground(bckcolor);
+	panelx.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+	
+	wait_lbl = new JLabel(messages.getString("Pleasewait"));
+	wait_lbl.setBackground(bckcolor);
+	wait_lbl.setVisible(false);
+	panelx.add(wait_lbl);
+	
+	progressBar = new JProgressBar ();
+	progressBar.setBackground(bckcolor);
+	progressBar.setVisible(false);
+	panelx.add(progressBar);
+
+	JLabel emp2;
+	emp2 = new JLabel ("   ");
+	emp2.setBackground(bckcolor);
+	panelx.add(emp2);
+
+	wait_lbl.setAlignmentX(Component.CENTER_ALIGNMENT);
+	progressBar.setAlignmentX(Component.CENTER_ALIGNMENT);
+	emp2.setAlignmentX(Component.CENTER_ALIGNMENT);
+
+	final String p1;
+	final String p2;
+	p1 = "Tab";
+	p2 = "ProgressBar";
+
+	final JPanel cards;
+	cards = new JPanel(new CardLayout());
+	cards.setBackground(bckcolor);
+	cards.add(tab,p1);
+	cards.add(panelx,p2);
+
+	window.add(cards);
+
+	
+	timer = new Timer(1000, new ActionListener()
+	    {
+		public void actionPerformed(ActionEvent evt)
+		{
+		    if (finish == true)
+			{
+			    if (sendThread != null)
+				{
+				    sendThread.interrupt();
+				    sendThread = null;
+				}
+
+			    if (prevThread != null)
+				{
+				    prevThread.interrupt();
+				    prevThread = null;
+				}
+			    
+			    Component glass_pane;
+			    glass_pane = frame.getGlassPane();
+			    
+			    glass_pane.setVisible(false);
+			    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+			    glass_pane.removeMouseListener(mouse_blocker_listener);
+			    
+			    wait_lbl.setVisible(false);
+			    progressBar.setIndeterminate(false);
+			    progressBar.setVisible(false);
+			    
+			    frame.setTitle(messages.getString("SendFeedbackForm"));
+			    
+			    CardLayout cl = (CardLayout)(cards.getLayout());
+			    cl.show(cards,p1);
+			    
+			    if (selected == true)
+				frame.dispose();
+
+			    timer.stop();
+			}
+		    else
+			{
+			    Component glass_pane;
+			    glass_pane = frame.getGlassPane();
+			    
+			    glass_pane.addMouseListener(mouse_blocker_listener);
+			    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+			    glass_pane.setVisible(true);
+
+			    frame.setTitle("Please wait...");
+
+			    CardLayout cl = (CardLayout)(cards.getLayout());
+			    cl.show(cards,p2);
+
+			    wait_lbl.setVisible(true);
+			    progressBar.setIndeterminate(true);
+			    progressBar.setVisible(true);
+			}
+		}
+	    });
+    }
+
+    /**
+     * This method will set the finish variable values.
+     * When the sending files is finished, the sending application  will set the finish value to true.
+     * @param fin whether or not the send files is finished.
+     */
+    public static void setFinish (boolean fin)
+    {
+	finish = fin;
+    }
+
+    /**
+     * This is the class to represent the whole panel that hold the screenshot and the comments
+     * about the screenshot.
+     */
+    private class ThumbnailPic implements Serializable
+    {
+	/**
+	 * This is the panel that hold a thumbnail for a screenshot and a comments text area.
+	 */
+	private JPanel picpane;
+
+	/**
+	 * This hold the BufferedImages for the whole screen version of screenshot.
+	 */
+	private BufferedImage[] screenimages;
+
+	/**
+	 * This hold the BufferedImages for the window version only.
+	 */
+	private BufferedImage[] windowimages;
+
+	/**
+	 * This is the text area where user can enter their comments.
+	 */
+	private JTextArea problem_details;
+
+	/**
+	 * This is a flag to tell us which image user wants to send.Its defaults value is false.
+	 * If iswindow = true then it means user wants to send the window version only.
+	 * If iswindow = false then it means user wants to send the whole screen version.
+	 */
+	private boolean iswindow;
+
+	/**
+	 * This is the titles for the panel.
+	 * Usually this is the title of the window where user wants to do the screenshot.
+	 * If the title is more than 40 characters then we will trim it and add ... at the back of it.
+	 */
+	private String titles;
+
+	/**
+	 * This is the width of the image user wants to send.
+	 */
+	private String iw;
+
+	/**
+	 * This is the height of the image user wants to send.
+	 */
+	private String ih;
+
+	/**
+	 * This is the x-coordinate where the window located inside the whole screen image.
+	 */
+	private int xcoord;
+
+	/**
+	 * This is the y-coordinate where the window located inside the whole screen image.
+	 */
+	private int ycoord;
+
+	/**
+	 * This is the width of the window inside the whole screen image.
+	 */
+	private int width;
+
+	/**
+	 * This is the height of the window inside the whole screen image.
+	 */
+	private int height;
+
+	/**
+	 * This is the constructor that will setup the panel that contains
+	 * a thumbnail screenshot and a comments text area.
+	 * @param image the whole screen screenshot image.
+	 * @param text  the title for this pane.
+	 * @param x     the x-coordinate of the window inside the whole screen image.
+	 * @param y     the y-coordinate of the window inside the whole screen image.
+	 * @param w     the width of the window inside the whole screen image.
+	 * @param h     the height of the window inside the whole screen  image. 
+	 */
+	//public ThumbnailPic (BufferedImage screen,byte[] image,String text)
+	public ThumbnailPic (BufferedImage image,String text,int x,int y,int w,int h)
+	{
+	    xcoord = x;
+	    ycoord = y;
+	    width = w;
+	    height = h;
+	    iswindow = false;
+	    titles = text;
+	    screenimages = new BufferedImage[3];
+	    windowimages = new BufferedImage[3];
+	    //screenimages[0] = screen;
+	    screenimages[0] = image;
+	    screenimages[1] = null;
+	    screenimages[2] = null;
+	    windowimages[0] = null;
+	    windowimages[1] = null;
+	    windowimages[2] = null;
+	    addPanel(image);
+	}
+
+	/**
+	 * This method will setup the components inside the panel properly.
+	 * @param image the whole screen screenshot image.
+	 */
+	//public void addPanel (byte[] image)
+	public void addPanel (BufferedImage image)
+	{
+	    picpane = new JPanel();
+	    picpane.setLayout(new BorderLayout());
+	    
+	    final JButton picture;
+	    picture = new JButton();
+
+	    int w;
+	    int h;
+
+	    w = (int) (150/2);
+	    h = (int) (150/2);
+	    
+	    iw = "" + image.getWidth();
+	    ih = "" + image.getHeight();
+
+	    ImageIcon thumbnail;
+	    thumbnail = new ImageIcon(image.getScaledInstance(w,h,Image.SCALE_SMOOTH));
+
+	    picture.setIcon(thumbnail);
+	    picture.setText("Draw . . .");
+	    picture.setActionCommand("Scribble");
+	    picture.addActionListener(new ActionListener()
+		{
+		    public void actionPerformed (ActionEvent e)
+		    {
+			frame.addMouseListener(mouse_blocker_listener);
+			frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+			SelectPicture select;
+			select = new SelectPicture(screenimages,windowimages,iswindow,problem_details.getText(),messages);
+			frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+			frame.removeMouseListener(mouse_blocker_listener);	
+			select.setWindowBounds(xcoord,ycoord,width,height);
+			
+			iswindow = select.getIsWindow();
+
+			screenimages[0] = select.getImage3();
+			screenimages[1] = select.getImage();
+			screenimages[2] = select.getImage2();
+
+			if (iswindow == true)
+			    {
+				windowimages[0] = select.getWindowImage3();
+				windowimages[1] = select.getWindowImage();
+				windowimages[2] = select.getWindowImage2();
+			    }
+			else
+			    {
+				windowimages[0] = null;
+				windowimages[1] = null;
+				windowimages[2] = null;
+			    }
+			
+			problem_details.setText(select.getDetails());
+
+			int w,h;
+			w = (int) (150/2);
+			h = (int) (150/2);
+			
+			ImageIcon thumbnail;
+
+			if (iswindow == false)
+			    thumbnail = new ImageIcon(screenimages[1].getScaledInstance(w,h,Image.SCALE_SMOOTH));
+			else
+			    thumbnail = new ImageIcon(windowimages[1].getScaledInstance(w,h,Image.SCALE_SMOOTH));
+
+			picture.setIcon(thumbnail);
+			
+			if (select.getSendNow() == true)
+			    {
+				ActionEvent evt;
+				evt = new ActionEvent(send_button,ActionEvent.ACTION_PERFORMED,messages.getString("Send"));
+
+				frame.actionPerformed(evt);
+			    }
+		    }
+		});
+	    picture.setToolTipText("Click if you want to draw on it");
+	    picture.setMargin(new Insets(5,5,5,5));
+	    picture.setVerticalTextPosition(AbstractButton.BOTTOM);
+	    picture.setHorizontalTextPosition(AbstractButton.CENTER);
+	    picture.setBackground(new Color(176,208,176));
+	    picpane.add(picture,BorderLayout.LINE_START);
+	    
+	    JPanel txtPane;
+	    txtPane = new JPanel(new BorderLayout());
+	    txtPane.setBackground(new Color(176,208,176));
+	    txtPane.setBorder(new EmptyBorder(5,5,5,5));
+
+	    JLabel lbldesc = new JLabel("Comments :");
+	    lbldesc.setBackground(new Color(176,208,176));
+	    txtPane.add(lbldesc,BorderLayout.PAGE_START);
+
+	    problem_details = new JTextArea();
+	    problem_details.setWrapStyleWord(true);
+	    problem_details.setEditable(true);
+	    problem_details.setBackground(new Color(224,240,224));
+	   
+	    JScrollPane scroll;
+	    scroll = new JScrollPane(problem_details);
+	    scroll.setBackground(new Color(176,208,176));
+	    scroll.setPreferredSize(new Dimension(300,150));
+	    txtPane.add(scroll,BorderLayout.LINE_START);
+	   
+	    picpane.setSize(new Dimension(150,150));
+	    picpane.add(txtPane,BorderLayout.CENTER);
+	}
+
+	/**
+	 * This method will get the width of the image user wants to send.
+	 * @return the width of the image.
+	 */
+	public String getWidth()
+	{
+	    if (iswindow == false)
+		return "" + screenimages[0].getWidth();
+	    else
+		{
+		    return "" + windowimages[0].getWidth();
+		}
+	}
+
+	/**
+	 * This method will get the height of the image user wants to send.
+	 * @return the height of the image.
+	 */
+	public String getHeight()
+	{
+	    if (iswindow == false)
+		return "" + screenimages[0].getHeight();
+	    else
+		{
+		    return "" + windowimages[0].getHeight();
+		}  
+	}
+
+	/**
+	 * This method will get the titles of this panel.
+	 * @return the panel's title.
+	 */
+	public String getTitles()
+	{
+	    return titles;
+	}
+
+	/**
+	 * This method will get user comments inside the text area.
+	 * @return the user's comments.
+	 */
+	public String getProblemDetails ()
+	{
+	    String txt;
+	    txt = problem_details.getText();
+	    
+	    if (txt == null)
+		return " ";
+	    else
+		return txt;
+	}
+
+	/**
+	 * This method will setting up the text inside the text area.
+	 * @param comments the text set to the text area.
+	 */
+	public void setComments (String comments)
+	{
+	    problem_details.setText(comments);
+	}
+
+	/**
+	 * This method will get the image that user wants to send.
+	 * @return image going to be send.
+	 */
+	public BufferedImage[] getImages ()
+	{
+	    if (iswindow == false)
+		return screenimages;
+	    else
+		return windowimages;
+	}
+
+	/**
+	 * This method will get the non-scribble lines image that user wants to send.
+	 * @return the string representation of the non-scribble lines image.
+	 */
+	public String getScreenImage ()
+	{
+	    if (iswindow == false)
+		return convert(screenimages[0]);
+	    else
+		return convert(windowimages[1]);
+	}
+
+	/**
+	 * This method will get the scribble lines image that user wants to send.
+	 * @return the string representation of the scribble lines image.
+	 */
+	public String getScreenAndLineImage ()
+	{
+	    if (iswindow == false)
+		{
+		    if (screenimages[1]!= null)
+			return convert(screenimages[1]);
+		    else
+			return convert(screenimages[0]);
+		}
+	    else
+		{
+		    if (windowimages[1]!= null)
+			return convert(windowimages[1]);
+		    else
+			return convert(windowimages[0]);
+		}
+	}
+
+	/**
+	 * This method will get only scribble lines image that user wants to send.
+	 * @return the string representation of the only scribble lines image.
+	 */
+	public String getLineForScreenImage ()
+	{
+	    if (iswindow == false)
+		{
+		    if (screenimages[2] != null)
+			return convert(screenimages[2]);
+		    else
+			return "";
+		}
+	    else
+		{
+		    if (windowimages[2] != null)
+			return convert(windowimages[2]);
+		    else
+			return "";
+		}
+	}
+
+	/**
+	 * This method will get the panel that hold a thumbnail and a text area.
+	 * @return panel that represent the thumbnail and the text area.
+	 */
+	public JPanel getPanel()
+	{
+	    return picpane;
+	}
+    }
+
+    /**
+     * This method will setting up the comments to be displayed inside the latest added panel.
+     * So if the window where user wants to take the screenshot at the moment is a modal window
+     * then it will setting the comments to ask user to close the window before they can enter
+     * their comments there.
+     * @param modal the modality of the window where user wants to take the screenshot.
+     */
+    public void setComment (boolean modal)
+    {
+	if (paneArray.size() == 0)
+	    System.out.println("There is no panel here!");
+	else
+	    {
+		JPanel pane;
+		pane = (JPanel) paneArray.get((row - 2));
+		ThumbnailPic obj;
+		obj = (ThumbnailPic) picAndPane.get(pane);
+		String text;
+		if (modal == true)
+		    text = "Please close '" + obj.getTitles() + "' window \nbefore you can enter comments here.";
+		else
+		    text = " ";
+		obj.setComments(text);
+	    }
+    }
+
+    /**
+     * This method will add a panel that contains the thumbnail button and the text area.
+     * @param title the window's title.
+     * @param image the whole screen image.
+     * @param x     the window's x-coordinate in the image.
+     * @param y     the window's y-coordinate in the image.
+     * @param width the window's width in the image.
+     * @param height the window's height in the image.
+     */
+    //public void addScreenPanel (String title,BufferedImage screen,byte[] image)
+    public void addScreenPanel (String title,BufferedImage image,int x,int y,int width,int height)
+    {
+	final JPanel pane;
+	pane = new JPanel();
+	pane.setLayout(new BorderLayout());
+	pane.setBackground(new Color(176,208,176));
+
+	Border loweredetched;
+	loweredetched = BorderFactory.createEtchedBorder(EtchedBorder.LOWERED);
+
+	String ttl;
+	if (title.length() > 41)
+	    ttl = title.substring(0,40) + "  . . . ";
+	else
+	    ttl = title;
+
+	TitledBorder border;
+	border = BorderFactory.createTitledBorder(loweredetched, title);
+	border.setTitleJustification(TitledBorder.LEFT);
+	
+	pane.setBorder(BorderFactory.createCompoundBorder(new EmptyBorder(5,5,5,5),border));
+	
+	ThumbnailPic pic;
+	//pic = new ThumbnailPic(screen,image,title);
+	pic = new ThumbnailPic(image,title,x,y,width,height);
+	picAndPane.put(pane,pic);
+	pane.add(pic.getPanel(),BorderLayout.LINE_START);
+
+	remove.setVisible(true);
+	picture_scroll.add(pane);
+	paneArray.add(pane);
+	row++;
+	picture_scroll.setSize(picture_scroll.getPreferredSize());
+	picture_scroll.revalidate();
+	picture_scroll.repaint();
+	tempPane.setSize(tempPane.getPreferredSize());
+	tempPane.revalidate();
+	tempPane.repaint();
+    }
+
+    /**
+     * This method will generate the unique code that will be used as the 
+     * reference number of the feedback that user are about to send.
+     * @return the unique code.
+     */
+    public String getPrevData ()
+    {
+	try
+	    {
+		UID id = new UID();
+		code =  "" + InetAddress.getLocalHost().hashCode() + id.hashCode();
+	    }
+	catch (UnknownHostException exp) {}
+	return code;
+    }
+
+    /**
+     * This method allows another class to get the unique code that can be used as 
+     * the reference number of the feedback that user are about to send.
+     * This method should be called only if 
+     * getPrevData() method already been called before.Otherwise it will return null. 
+     * @return the unique code
+     */
+    public static String getCode()
+    {
+	return code;
+    }
+
+    /**
+     * This method will encode BufferedImage into their String representation using
+     * Base64 encoder.
+     * (Precondition: (image != null))
+     * @param image is the image that we want to encode to String.
+     */
+    public String convert (BufferedImage image)
+    {
+	String txt;
+	txt = null;
+
+	try
+	    {
+		ByteArrayOutputStream stream = new ByteArrayOutputStream();
+		ImageIO.write(image,"jpeg",stream);
+		byte[] bytes = stream.toByteArray();
+
+		txt = Base64.encodeBytes(bytes);
+		
+	    }
+	catch(IOException ex) {}
+
+	return txt;
+    }
+
+    /**
+     * This method is to make sure that there is no thread running at the moment.
+     */
+    private void ensureEventThread()
+    {
+	if(SwingUtilities.isEventDispatchThread())
+	    {
+		return;
+	    }
+	throw new RuntimeException(messages.getString("file"));
+    }
+
+    /**
+     * This method will create all 6 xml files, jarred them into one file, deletes all xml files and
+     * sending it.
+     * The xml files for:
+     * <br> 1.The history & information of all the actions done and windows opened by the application
+     *        after user choose to start the sequence of sending feedback.
+     * <br> 2.The history of all the commands (actions) done by the user before choosing
+     *        after start the sequence of sending feedback.
+     * <br> 3.The informations we get from the feedback interface form period.
+     * <br> will be getting from vector and the temp_feedbackhist.log file.
+     * (Precondition: (sx != null) && (vector != null) && (err_array != null))
+     * @param sx the xml parser.
+     * @param vector the vector that hold the history and information.
+     * @param err_array the arrays of information we take from the feedback interface period.
+     * @param imgFile   the arrays of information we take from the panels inside feedback interface form.
+     */
+    private void sendState (SaveToXML sx,Vector vector,String[] err_array,String[] imgFile)
+    {
+	String dirname;
+	dirname = "xmlfeedback/";
+	
+	File dir = new File("xmlfeedback");
+	if (dir.isDirectory() == false)
+	    dir.mkdir();
+
+	sx.saveFeedback(err_array,imgFile);
+	
+	sx.open(dirname + "feedbackcontent" + FeedbackInterface.getCode() + ".xml",
+		"HISTORY");
+	SaveToXML sx2;
+	sx2 = new SaveToXML(messages);
+	sx2.open(dirname + "feedbackcommand" + FeedbackInterface.getCode() + ".xml",
+		 "COMMANDS");
+	
+	for (int i = 0 ; i < vector.size() ; i++)
+	    {
+		History log;
+		log = (History) vector.get(i);
+		log.sendXML(sx);
+		log.sendXMLComm(sx2);
+	    }
+	
+	Vector stack;
+	stack = getPrev_log();
+	if (stack != null)
+	    {
+		for (int i = 0 ; i < stack.size() ; i++)
+		    {
+			History log;
+			log = (History) stack.get(i);
+			log.sendXML(sx);
+			log.sendXMLComm(sx2);
+		    }
+		stack.removeAllElements();
+		System.gc();
+		stack = null;
+	    }
+	
+	sx2.close("COMMANDS");
+	sx.close("HISTORY");
+	
+	getHistPrev_log(sx,sx2);
+	
+	ZipFile zip;
+	zip = new ZipFile();
+	zip.sendZipXMLFile();
+
+	File f;
+	f = new File (dirname + "feedback" + FeedbackInterface.getCode() + ".xml");
+	f.delete();
+	f = new File (dirname + "feedbackcontent" + FeedbackInterface.getCode() + ".xml");
+	f.delete();
+	f = new File (dirname + "feedbackcommand" + FeedbackInterface.getCode() + ".xml");
+	f.delete();
+	/*f = new File (dirname + "content" + FeedbackInterface.getCode() + ".xml");
+	  f.delete();*/
+	f = new File (dirname + "command" + FeedbackInterface.getCode() + ".xml");
+	f.delete();
+	
+	String[] args = new String[5];
+	args[0] = "vl6@cs.waikato.ac.nz";
+	args[1] = dirname + "Jar" + FeedbackInterface.getCode() + "File.jar";
+	args[4] = "lib/parser.jar";
+	args[2] = err_array[19];
+	args[3] = err_array[20];
+	SendHTTP sm = new SendHTTP();
+	//SendMail sm = new SendMail();
+	sm.sendMail(args,messages,FeedbackInterface.getCode());
+    }
+	
+    /**
+     * This method will create the 2 xml files.
+     * The xml files are:
+     * <br> 1.The history & information of all the actions done and windows opened by the application
+     *         before user choose to start the sequence of sending feedback.
+     * <br> 2.The history of all the commands (actions) done by the user before choosing
+     *         to start the sequence of sending feedback.
+     * <br> The information will be getting from history.log file and the temp_history.log file.
+     * (Precondition: (sx != null) && (sx2 != null))
+     * @param sx the xml parser for the history xml file.
+     * @param sx2 the xml parser for the command xml file.
+     */
+    private void getHistPrev_log(SaveToXML sx,SaveToXML sx2)
+    {
+	Vector stack = null;
+	try
+	    {
+		File f = new File("history.log");
+		if (f.exists() == true)
+		    {
+			FileInputStream fis = new FileInputStream(f);
+			ObjectInputStream ois = new ObjectInputStream(fis);
+			
+			stack = (Vector) ois.readObject();
+			
+			String dirname;
+			dirname = "xmlfeedback/";
+
+			/*sx.open(dirname + "content" + FeedbackInterface.getCode() + ".xml",
+			  "HISTORY");*/
+			sx2.open(dirname + "command" + FeedbackInterface.getCode() + ".xml",
+				 "COMMANDS");
+			
+			if (stack != null)
+			    {
+				for (int i = 0 ; i < stack.size() ; i++)
+				    {
+					History log;
+					log = (History) stack.get(i);
+				        //log.sendXML(sx);
+					log.sendXMLComm(sx2);
+				    }
+				stack.removeAllElements();
+				System.gc();
+				stack = null;
+			    }
+			
+			stack = (Vector) ois.readObject();
+			
+			if (stack != null)
+			    {
+				for (int i = 0 ; i < stack.size() ; i++)
+				    {
+					History log;
+					log = (History) stack.get(i);
+				        //log.sendXML(sx);
+					log.sendXMLComm(sx2);
+				    }
+				stack.removeAllElements();
+				System.gc();
+				stack = null;
+			    }
+			
+			sx2.close("COMMANDS");
+			//sx.close("HISTORY");
+			
+			ois.close();
+		    }
+	    }
+	catch (IOException exp) {exp.printStackTrace();}
+	catch (ClassNotFoundException exp2)
+	    {
+		System.out.println("class exp");}
+    }
+
+    /**
+     * This method will get the Vector stored in temp_feedbackhist.log file.
+     * @return the vector stored in temp_feedbackhist.log file.
+     */
+    public Vector getPrev_log()
+    {
+	Vector stack = null;
+	try
+	    {
+		File f = new File("temp_feedbackhist.log");
+		if (f.exists() == true)
+		    {
+			FileInputStream fis = new FileInputStream(f);
+			ObjectInputStream ois = new ObjectInputStream(fis);
+			stack = (Vector) ois.readObject();
+			ois.close();
+			return stack;
+		    }
+		else
+		    stack = null;
+	    }
+	catch (IOException exp) {exp.printStackTrace();}
+	catch (ClassNotFoundException exp2)
+	    {
+		System.out.println("class exp");}
+	
+	return stack;
+    }
+
+    /**
+     * This is the thread that will start when user click send button and it will
+     * make 6 xml files,jarred them together into 1 jar file, deletes the 6 xml files and send it.
+     * @param vector the vector stored the current actions history.
+     * @param err_array the array that stored all the information took during feedback interface period.
+     * @param imgFile the array of that stored all the information from all the panels inside feedback interface form.
+     */
+    public void sendMethod (final Vector vector,final String err_array[], final String imgFile[])
+    {
+	ensureEventThread();
+	Runnable sendRun;
+	sendRun = new Runnable()
+	    {
+		public void run ()
+		{
+		    finish = false;
+		    selected = true;
+		    SaveToXML sx;
+		    sx = new SaveToXML(messages);
+		    sendState(sx,vector,err_array,imgFile);
+		    finish = true;
+		}
+	    };
+	sendThread = new Thread(sendRun);
+	sendThread.start();
+    }
+
+    /**
+     * This method will called the window that will display the preview of all
+     * the file going to be send.
+     * @param vector the vector stored the current actions history.
+     * @param err_array the array that stored all the information took during feedback interface period.
+     * @param imgFile the array of that stored all the information from all the panels inside feedback interface form.
+     */
+    public void viewMethod (final Vector vector,final String err_array[], final String imgFile[])
+    {
+	ensureEventThread();
+	Runnable prevRun;
+	prevRun = new Runnable()
+	    {
+		public void run ()
+		{
+		    finish = false;
+		    selected = false;
+		    ReportDetails rd;
+		    rd = new ReportDetails(vector,err_array,imgFile,messages,true);
+		    finish = true;
+		}
+	    };
+	
+	prevThread = new Thread(prevRun);
+	prevThread.start();
+    }
+
+    /**
+     * This method will save a BufferedImage to a jpeg file with the specified filename.
+     * This method is a thread that will caused the thread save to be started and stop
+     * if the saving process is finished.
+     * (Precondition: (I != null))
+     * @param I is the BufferedImage that we want to save.
+     * @param picScreen name of the jpeg file where we want to store the BufferedImage
+     */
+    public void saveGraph(final BufferedImage I,final String picScreen)
+    {
+	ensureEventThread();
+	Runnable saveRun;
+	saveRun = new Runnable ()
+	    {
+		public void run()
+		{
+		    try 
+			{
+			    File file;
+			    file = new File(picScreen);
+			    FileOutputStream out;
+			    out = new FileOutputStream(file);
+			    
+			    JPEGImageEncoder encoder;
+			    encoder = JPEGCodec.createJPEGEncoder(out);
+			    JPEGEncodeParam param;
+			    param = encoder.getDefaultJPEGEncodeParam(I);
+			    param.setQuality(1.0f, false);
+			    encoder.setJPEGEncodeParam(param);
+			    encoder.encode(I);
+			    out.close();
+			    save = null;
+			} 
+		    catch (Exception ex) {}
+		}
+	    };
+	
+	save = new Thread(saveRun);
+	save.start();
+    }
+
+    /**
+     * This method will get all the informations from the all the panels inside the feedback interface form.
+     * @return the arrays of information from all the panels.
+     */
+    public String[] getimgFile()
+    {
+	String[] imgFile;	
+	imgFile = null;
+
+	/*String answer;
+	  answer = (String) problem_screenshot.getSelectedItem();
+	  if(answer == messages.getString("Yes"))
+	  {*/
+	
+	if (paneArray.size() > 0)
+	    {
+		imgFile = new String[(paneArray.size() * 10)];
+		
+		int step;
+		for (step = 0 ; step < (paneArray.size()*10) ; step=step+10)
+		    {
+			JPanel pane;
+			ThumbnailPic obj;
+			pane = (JPanel) paneArray.get((step/10));
+			obj = (ThumbnailPic) picAndPane.get(pane);
+			
+			//obj.setSize();
+			
+			String title;
+			title = obj.getTitles();
+			
+			imgFile[step+0] = title;
+			imgFile[step+1] = obj.getScreenImage();
+			title = "Window" + pane.hashCode() + "File";
+			imgFile[step+2] = messages.getString("screen")+title+step+".jpg";
+			imgFile[step+3] = obj.getScreenAndLineImage();
+			imgFile[step+4] = messages.getString("ErrorLineAnd")+imgFile[step+2];
+			imgFile[step+5] = obj.getLineForScreenImage();
+			imgFile[step+6] = messages.getString("ErrorLineFor")+imgFile[step+2];
+			imgFile[step+7] = obj.getWidth();
+			imgFile[step+8] = obj.getHeight();
+			imgFile[step+9] = obj.getProblemDetails();
+		    }
+	    }
+		
+	return imgFile;
+    }
+
+    /**
+     * This method will get all the informations taken during feedback interface period, such as os version.
+     * @return then arrays of informations.
+     */
+    public String[] geterr_array()
+    {
+	String[] err_array;
+	err_array = null;
+
+	String os_name;
+	os_name = System.getProperty("os.name") + " " + System.getProperty("os.version") 
+	    + " " + System.getProperty("os.arch");
+	
+	Date problem_date;
+	String err_date,java_info,screen_res;
+	problem_date = new Date();
+	err_date = problem_date.toString();
+	java_info = System.getProperty("java.vendor")+ " " + System.getProperty("java.class.version");
+	Dimension dim;
+	dim = Toolkit.getDefaultToolkit().getScreenSize();
+	screen_res = "" + dim.width + "x" + dim.height;
+	
+	try
+	    {
+		err_array = new String[24];
+
+		err_array[0] = code;
+		err_array[1] = frameName;
+		err_array[2] = "";
+		err_array[3] = (String) problem_type.getSelectedItem();
+		err_array[4] = (String) problem_urgency.getSelectedItem();
+		err_array[5] = System.getProperty("user.name");
+		err_array[6] = System.getProperty("user.home");
+		err_array[7] = System.getProperty("user.dir");
+		err_array[8] = curr_date.toString();
+		err_array[9] = err_date;
+		err_array[10] = os_name;
+		err_array[11] = java_info;
+		err_array[12] = "" + Locale.getDefault();
+		err_array[13] = Utility.PROGRAM_NAME + " " + Utility.PROGRAM_VERSION; 
+		err_array[14] = InetAddress.getLocalHost().getHostName();
+		err_array[15] = InetAddress.getLocalHost().getHostAddress();
+		err_array[16] = screen_res;
+		err_array[17] = "";
+		
+		/*String test_txt;
+		  test_txt = smtp_address.getText();
+		  if ((test_txt == null)||(test_txt.length() < 5))*/
+		err_array[19] = "mail.waikato.ac.nz";
+		/*else
+		  err_array[19] = test_txt;
+		  
+		  test_txt = email_address.getText();
+		  if ((test_txt == null)||(test_txt.length() < 5))*/
+		err_array[20] = "vl6@cs.waikato.ac.nz";
+		//else
+		//err_array[20] = test_txt;
+		
+		Runtime run;
+		run = Runtime.getRuntime();
+		err_array[21] = formatFileSize(run.totalMemory());
+		err_array[22] = formatFileSize(run.maxMemory());
+		err_array[23] = formatFileSize(run.freeMemory());
+	    }
+	catch(UnknownHostException exp) {}
+
+	return err_array;
+    }
+
+    /**
+     * This method will format the file size given into more elegant form,e.g. 2KB
+     * @param length the file size.
+     * @return the more elegant string representations of the file size.
+     */
+    public String formatFileSize(long length) 
+    {
+	String BYTE_SUFFIX;
+	BYTE_SUFFIX = " B";
+	long GIGABYTE;
+	GIGABYTE = 1024000000l;
+	String GIGABYTE_SUFFIX;
+	GIGABYTE_SUFFIX = " GB";
+	long KILOBYTE;
+	KILOBYTE = 1024l;
+	String KILOBYTE_SUFFIX;
+	KILOBYTE_SUFFIX = " KB";
+	long MEGABYTE;
+	MEGABYTE = 1024000l;
+	String MEGABYTE_SUFFIX;
+	MEGABYTE_SUFFIX = " MB";
+
+	StringBuffer result;
+	result = new StringBuffer("");
+	float number;
+	number = 0f;
+	String suffix;
+	suffix = null;
+	
+	if(length >= GIGABYTE) 
+	    {
+		number = (float) length / (float) GIGABYTE;
+		suffix = GIGABYTE_SUFFIX;
+	    }
+	else if(length >= MEGABYTE) 
+	    {
+		number = (float) length / (float) MEGABYTE;
+		suffix = MEGABYTE_SUFFIX;
+	    }
+	else if(length >= KILOBYTE) 
+	    {
+		number = (float) length / (float) KILOBYTE;
+		suffix = KILOBYTE_SUFFIX;
+	    }
+	else 
+	    {
+		return length + BYTE_SUFFIX;
+	    }
+	
+	String number_str;
+	number_str = Float.toString(number);
+	char number_char[] = number_str.toCharArray();
+	int pos,i;
+	pos = 0;
+	
+	while(number_char != null && pos < number_char.length && number_char[pos] != '.')
+	    {
+		result.append(number_char[pos]);
+		pos++;
+	    }
+
+	if(pos < number_char.length) 
+	    {
+		result.append(number_char[pos]);
+		pos++;
+		
+		for(i = 0; i < 2 && pos < number_char.length; i++, pos++)
+		    result.append(number_char[pos]);
+	   
+		while(pos < number_char.length && number_char[pos] != 'E') 
+		    pos++;
+	   
+		while(pos < number_char.length) 
+		    {
+			result.append(number_char[pos]);
+			pos++;
+		    }
+	    }
+	
+	result.append(suffix);
+	
+	return result.toString();
+    }
+
+    /**
+     * This method will decide what action should be done if the user entered a button in
+     * the Feedback form.
+     * <br>If user click cancel button then all the information taken in this form will be deleted
+     * and user will be quitted from the Reporting Feedback sequence.
+     * <br>If user click the send button then all the information taken will be stored in array which
+     * then will be saved into xml files and sent when user confirm to send all the files.
+     * <br>If user click the details button then they can view all the xml files that are going to be send.
+     * @param e the button that user clicked
+     */
+    public void actionPerformed (ActionEvent e)
+    {
+	getPrevData();
+
+	if(messages.getString("Send").equals(e.getActionCommand()))
+	    {
+		String[] imgFile;
+		String[] err_array;
+
+		imgFile = getimgFile();
+		err_array = geterr_array();
+
+		timer.start();
+		
+		sendMethod(dialog.getVector(),err_array,imgFile);
+	    }
+
+	if(messages.getString("Preview").equals(e.getActionCommand()))
+	    {
+		String[] imgFile;
+		String[] err_array;
+
+		imgFile = getimgFile();
+		err_array = geterr_array();
+
+		timer.start();
+		
+		viewMethod(dialog.getVector(),err_array,imgFile); 
+	    }
+
+	if(messages.getString("NotSend").equals(e.getActionCommand()))
+	    {
+		dispose();
+	    }
+    }
+}
+	
+	
+
+	
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Graphs.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Graphs.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Graphs.java	(revision 21112)
@@ -0,0 +1,360 @@
+package org.greenstone.gatherer.feedback;
+
+import java.awt.image.*;
+import java.awt.geom.*;
+import java.awt.geom.Line2D.*;
+import java.awt.geom.Line2D.Double;
+import java.io.*;
+import javax.swing.*;
+import java.awt.*;
+
+/**
+ * This is the class that will allow user to draw a same line to 2 different BufferedImage
+ * at the same time.
+ * The first BufferedImage is BufferedImage with some Image as the based and then
+ * we draw a line on it, and the second BufferedImage is just a transparent Image and then we 
+ * draw a line on it. The line drawn in first BufferedImage is exactly the same with the line
+ * drawn in second BufferedImage.The Image drawn in the first bufferedimage is the
+ * screen shot image of the whole screen.
+ * @author Veronica Liesaputra
+ */
+public class Graphs
+{ 
+    /**
+     * This is the first buffered image.
+     * It will drawn with an image first to be its based, and a line  
+     * that are identical to line drawn the second BufferedImage, will be drawn on it.
+     */
+    private BufferedImage I = null;
+
+    /**
+     * This is the second buffered image.
+     * Its a transaparent buffered image and a line that are indetical to line
+     * drawn in the first BufferedImage, will be drawn on it. 
+     */
+    private BufferedImage I2;
+
+    /**
+     * This is the Graphics for the first BufferedImage.
+     * This will allow user to draw a line and an image on the first buffered image.
+     */
+    private Graphics2D G;
+    
+    /**
+     * This is the Graphics for the second BufferedImage.
+     * This will allow user to draw a line on the second buffered image.
+     */
+    private Graphics2D G2;
+
+    /**
+     * This is the arrays of the 3 BufferedImage.
+     */
+    private BufferedImage[] screen;
+
+    /**
+     * This is the graphics for screen[0] BufferedImage.
+     * This will allow user to draw something on the screen[0] image.
+     */
+    private Graphics2D G3;
+  
+    /**
+     * This is the width of the image.
+     */
+    private double imwidth;
+
+    /**
+     * This is the height of the image.
+     */
+    private double imheight;
+
+    /**
+     * This is the constructor that will initialised the BufferedImage and
+     * its Graphics.
+     * (Precondition: (sh != null))
+     * @param sh is the images.
+     */
+    public Graphs (BufferedImage[] sh)
+    {
+	screen = sh;
+
+	if (screen[0] != null)
+	    { 
+		imwidth = screen[0].getWidth();
+		imheight = screen[0].getHeight();
+
+
+		if((screen[1] == null)&& (screen[2] == null))
+		    {
+			I = new BufferedImage(screen[0].getWidth(),screen[0].getHeight(),
+					      BufferedImage.TYPE_INT_RGB);
+			I2 = new BufferedImage(I.getWidth(),I.getHeight(),
+					       BufferedImage.TYPE_INT_ARGB);
+		    }
+		else
+		    {
+			I = screen[1];
+			I2 = screen[2];
+		    }
+		start();
+	    }
+	else
+	    System.out.println("weird");
+    }
+
+    /**
+     * This method will make sure that everything is null-ed and dispose properly.
+     */
+    public void flush()
+    {
+	G.dispose();
+	G = null;
+	G3.dispose();
+	G3 = null;
+	G2.dispose();
+	G2 = null;
+
+	int i;
+	for ( i = 0 ; i < 3; i++)
+	    {
+		if (screen[i] != null)
+		    {
+			screen[i].flush();
+			screen[i] = null;
+		    }
+	    }
+
+	I.flush();
+	I = null;
+	I2.flush();
+	I2 = null;
+	System.gc();
+    }
+
+    /**
+     * This method will cleared the scribble lines in the images.
+     */
+    public void reset()
+    {
+	I.flush();
+	I = null;
+	System.gc();
+	I2.flush();
+	I2 = null;
+	G.dispose();
+	System.gc();
+	G2.dispose();
+	G = null;
+	G2 = null;
+	System.gc();
+
+	I = new BufferedImage(screen[0].getWidth(),
+			      screen[0].getHeight(),
+			      BufferedImage.TYPE_INT_RGB);
+	I2 = new BufferedImage(I.getWidth(),I.getHeight(),
+			       BufferedImage.TYPE_INT_ARGB);
+
+	screen[1] = null;
+	screen[2] = null;
+
+	start();
+    }
+    /**
+     * This method will setup the 3 Graphics for the 3 BufferedImage.
+     * Here the graphics for the bufferedimage will be created and the appropriate image
+     * will be drawn to the first bufferedimage.  
+     */
+    public void start ()
+    {
+	G3 = screen[0].createGraphics();
+	G3.drawImage((Image) screen[0],0,0,new JLabel());
+
+	G = I.createGraphics();
+	if (screen[1] == null)
+	    G.drawImage((Image) screen[0],0,0,new JLabel());
+	else
+	    G.drawImage ((Image) screen[1],0,0,new JLabel());
+	G2 = I2.createGraphics();
+    }
+ 
+    /**
+     * This method will draw a red line on both BufferedImage from the source point
+     * to the destination point.
+     * @param x1 is the x-coordinate of the source point.
+     * @param y1 is the y-coordinate of the source point.
+     * @param x2 is the x-coordinate of the destination point.
+     * @param y2 is the y-coordinate of the destination point. 
+     */
+    public void drawLines(int x1,int y1,int x2,int y2)
+    {
+	G2.setColor(Color.red);
+	G.setColor(Color.red);
+
+	G.setStroke(new BasicStroke(2.5f));
+	G2.setStroke(new BasicStroke(2.5f));
+
+	G2.drawLine(x1,y1,x2,y2);
+	G.drawLine(x1,y1,x2,y2);
+    }
+
+    
+    /**
+     * This method will draw a white line on all 3 BufferedImage from the source point
+     * to the destination point.
+     * @param x1 is the x-coordinate of the source point.
+     * @param y1 is the y-coordinate of the source point.
+     * @param x2 is the x-coordinate of the destination point.
+     * @param y2 is the y-coordinate of the destination point. 
+     */
+    public void eraseLines(int x1,int y1,int x2,int y2)
+    {
+	G3.setColor(Color.white);
+	G2.setColor(Color.white);
+	G.setColor(Color.white);
+
+	G.setStroke(new BasicStroke(25.5f));
+	G2.setStroke(new BasicStroke(25.5f));
+	G3.setStroke(new BasicStroke(25.5f));
+	
+	G3.drawLine(x1,y1,x2,y2);
+	G2.drawLine(x1,y1,x2,y2);
+	G.drawLine(x1,y1,x2,y2);
+    }
+
+    /**
+     * This method will get the first BufferedImage.
+     * This is the BufferedImage where it will already be drawn with the
+     * screen shot image and the lines.
+     * @return the first BufferedImage.
+     */
+    public BufferedImage getImage ()
+    {
+	return I;
+    }
+
+    /**
+     * This method will get the second BufferedImage.
+     * This is the BufferedImage where its base is transparent and it will
+     * already be drawn with the lines.
+     * @return the second BufferedImage.
+     */
+    public BufferedImage getImage2()
+    {
+	return I2;
+    }
+
+    /**
+     * This method will get the image without scribble lines only the white lines.
+     * @return the image without scribble lines.
+     */
+    public BufferedImage getScreenImage()
+    {
+	return screen[0];
+    }
+
+    /**
+     * This method will get the window out of the whole screen image.
+     * @param xcoord the xcoordinate of the window in the image.
+     * @param ycoord the ycoordinate of the window in the image.
+     * @param width  the width of the window in the image.
+     * @param height the height of the window in the image.
+     * @return the BufferedImage of the window only from the 3 buffered image.
+     */
+    public BufferedImage[] getWindowVersion(int xcoord,int ycoord,int width,int height)
+    {
+	BufferedImage[] window;
+	window = new BufferedImage[3];
+
+	Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+	
+	if (xcoord < 0 ) 
+	    xcoord = 0;
+	if (ycoord < 0)
+	    ycoord = 0;
+
+	if (xcoord >= imwidth)
+	    xcoord = (int)(imwidth - 1);
+	if (ycoord >= imheight)
+	    ycoord = (int)(imheight - 1);
+	if ((xcoord + width) > imwidth)
+	    width = (int)(imwidth - xcoord);
+	if ((ycoord + height) > imheight)
+	    height = (int)(imheight - ycoord);
+
+	if (width <= 0)
+	    width = 1;
+	if (height <= 0)
+	    height = 1;
+
+	window[0] = getClipVersion(screen[0],xcoord,ycoord,width,height);
+	
+	window[1] = getClipVersion(I,xcoord,ycoord,width,height);
+	
+	window[2] = getClipVersion(I2,xcoord,ycoord,width,height);
+	
+	return window;
+    }
+    
+    /**
+     * This method will return the subimage of the big buffered image.
+     * @param tag the original buffered image.
+     * @param x   the x location of subimage.
+     * @param y   the y location of subimage.
+     * @param w   the width of the subimage.
+     * @param h   the height of the subimage.
+     * @return    the subimage.
+     */
+    public BufferedImage getClipVersion (BufferedImage tag,int x, int y, int w, int h) 
+    {
+	BufferedImage sub;
+	sub = tag.getSubimage(x,y,w,h);
+	
+	return sub;
+    }
+
+    /**
+     * This method will return the screen image version of the give buffered image.
+     * @param sh the buffered image that we going to draw in the screen image version.
+     * @param xcoord   the x location of subimage.
+     * @param ycoord   the y location of subimage.
+     * @param w   the width of the subimage.
+     * @param h   the height of the subimage.
+     */
+    public void setScreenVersion(BufferedImage[] sh,int xcoord,int ycoord,int w, int h)
+    {
+	if (xcoord < 0 ) 
+	    xcoord = 0;
+	if (ycoord < 0)
+	    ycoord = 0;
+
+	if (xcoord >= imwidth)
+	    xcoord = (int) (imwidth - 1);
+	if (ycoord >= imheight)
+	    ycoord = (int) (imheight - 1);
+	if ((xcoord + w) > imwidth)
+	    w = (int) (imwidth - xcoord);
+	if ((ycoord + h) > imheight)
+	    h = (int) (imheight - ycoord);
+
+	if (w <= 0)
+	    w = 1;
+	if (h <= 0)
+	    h = 1;
+       
+	G3.drawImage(sh[0], xcoord,ycoord,new JLabel());
+
+        
+        G.drawImage(sh[1], xcoord,ycoord,new JLabel());
+	
+	
+	G2.drawImage(sh[2], xcoord,ycoord,new JLabel());
+    }
+}
+    
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/History.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/History.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/History.java	(revision 21112)
@@ -0,0 +1,284 @@
+package org.greenstone.gatherer.feedback;
+
+import java.util.*;
+import java.awt.image.*;
+import java.io.*;
+import javax.imageio.*;
+import javax.swing.*;
+import java.awt.*;
+
+/**
+ * This class is one record of the action user did to any of the component
+ * inside the window that the application open that is recorded by the 
+ * special listener class.
+ * An instance of this class will hold all the information about the action
+ * user did at that moment,all information about all the components inside the
+ * window where the action occured and all information about all the components
+ * inside all the windows that were opened when the action occured.
+ * @author Veronica Liesaputra
+ */
+public class History implements Serializable
+{
+    /**
+     * This is the date of when the action is done by the user.
+     */
+    private String date;
+    
+    /**
+     * This is the command that user did.
+     */
+    private String command;
+    
+    /**
+     * This is the title of the window where the action occured.
+     */
+    private String title;
+
+    /**
+     * This hold all information about all the components inside the
+     * window where the action occured and all information about all the components
+     * inside all the windows that were open when the action occured.
+     */
+    private ArrayList component;
+
+    /**
+     * This hold the image of the window where the action occured.
+     */
+    private String image = null;
+
+    /**
+     * This is the image file name, if then user decide to save the image to a jpeg file.
+     */
+    private String imageName = "";
+
+    /**
+     * This is the height of image of the window where the action occured. 
+     */
+    private String height;
+
+    /**
+     * This is the width of image of the window where the action occured.
+     */
+    private String width;
+
+    /**
+     * This constructor will be initializing the date,command and component value.
+     * (Precondition: (curr_date != null))
+     * @param curr_date is the date when the action occured.
+     * @param curr_command is the command the user did.
+     */
+    public History (Date curr_date,String curr_command)
+    {
+	date = curr_date.toString();
+	command = curr_command;
+	component = new ArrayList();
+    }
+
+    /**
+     * This method will be setting up the image attributes to the passed values.
+     * It will setting the image of the window where the action occured and its height 
+     * and width.It will also generate the supposed file name for this image.
+     * (Precondition: (imag != null) && (h != null) && (w != null))
+     * @param img the image of the window where the action occured.
+     * @param h   image's height.
+     * @param w   image's width.
+     */
+    public void setImage(String img,String h,String w)
+    {
+	image = img;
+	height = h;
+	width = w;
+	StringTokenizer st;
+	st = new StringTokenizer(date,": ");
+	while (st.hasMoreTokens()) 
+	    {
+		imageName += st.nextToken();
+	    }
+	imageName += ".jpg";
+    }
+
+    /**
+     * This method will add a new CompGroup instance that hold information and
+     * the status of a window that were open when the action occured.
+     * @param comp UserComponent instance that hold information about all
+     *             components inside the window.
+     * @param curr_status The current status of the window.
+     */
+    public void addComponent (UserComponent comp,String curr_status)
+    {
+	component.add(new CompGroup(comp,curr_status));
+    }
+
+    /**
+     * This method will get the image's height of the window where the action
+     * occured.
+     * @return image's height.
+     */
+    public String getHeight ()
+    {
+	return height;
+    }
+
+    /**
+     * This method will get the image's width of the window where the action
+     * occured.
+     * @return image's width.
+     */
+    public String getWidth ()
+    {
+	return width;
+    }
+
+    /**
+     * This method will set the title of the window where the action occured.
+     * @param name window's title.
+     */
+    public void setTitle (String name)
+    {
+	title = name;
+    }
+
+    /**
+     * This method will get the title of the window where the action occured.
+     * @return the window's title.
+     */
+    public String getTitle ()
+    {
+	return title;
+    }
+
+    /**
+     * This method will get the date when the action occured.
+     * @return the date when the action occured.
+     */
+    public String getDate()
+    {
+	return date;
+    }
+
+    /**
+     * This method will get the command that user did.
+     * @return the command user did.
+     */
+    public String getCommand()
+    {
+	return command;
+    }
+
+    /**
+     * This method will get the array of CompGroup instances that hold
+     * all the information and status of all the window that were open when 
+     * the action occured.
+     * @return the array list of the CompGroup instances.
+     */
+    public ArrayList getArray()
+    {
+	return component;
+    }
+
+    /**
+     * This method will get the image of the window where the action occured.
+     * @return image the window's image.
+     */
+    public String getImage()
+    {
+	return image;
+    }
+    
+    /**
+     * This method will get the supposed image's file name.
+     * @return image's file name.
+     */
+    public String getImageName()
+    {
+	return imageName;
+    }
+ 
+    /**
+     * This method will stored this instance to an xml file.
+     * It will only store the date,title,command,image and all
+     * image attributes of this instance to the xml file.
+     * @param sx is the xml file where we want to store.
+     */
+    public void sendXMLComm (SaveToXML sx)
+    {
+	sx.startContent(10);
+	
+	sx.startContent(7);
+	sx.saveContent(date);
+	sx.closeContent(7);
+	
+	sx.startContent(2);
+	sx.saveContent(title);
+	sx.closeContent(2);
+	
+	sx.startContent(8);
+	sx.saveContent(command);
+	sx.closeContent(8);
+
+	if (image != null)
+	    {
+		sx.startContent(6);
+		sx.saveImage(image,imageName,width,height);
+		sx.closeContent(6);
+	    }
+	
+	sx.closeContent(10);
+    }
+
+    /**
+     * This method will stored this instance to an xml file.
+     * It will store all the data members of this instance to
+     * the xml file.
+     * @param sx is the xml file where we want to store.
+     */
+    public void sendXML (SaveToXML sx)
+    {
+	sx.startContent(10);
+
+	sx.startContent(7);
+	sx.saveContent(date);
+	sx.closeContent(7);
+
+	sx.startContent(2);
+	sx.saveContent(title);
+	sx.closeContent(2);
+
+	sx.startContent(8);
+	sx.saveContent(command);
+	sx.closeContent(8);
+
+	if (image != null)
+	    {
+		sx.startContent(6);
+		sx.saveImage(image,imageName,width,height);
+		sx.closeContent(6);
+	    }
+	
+	int j;
+	for (j = 0 ; j < component.size() ; j++)
+	    {
+		sx.startContent(9);
+		CompGroup comp;
+		comp = (CompGroup) component.get(j);
+		
+		sx.startContent(8);
+		sx.saveContent(comp.getStatus());
+		sx.closeContent(8);
+
+		sx.startContent(0);
+		comp.getComponent().sendingXML(sx);
+		sx.closeContent(0);
+		
+		sx.closeContent(9);
+	    }
+	sx.closeContent(10);
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Movie.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Movie.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Movie.java	(revision 21112)
@@ -0,0 +1,376 @@
+package org.greenstone.gatherer.feedback;
+
+import java.awt.image.*;
+import com.sun.image.codec.jpeg.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.awt.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.text.NumberFormatter;
+import java.text.NumberFormat;
+import java.beans.*;
+import java.util.ArrayList;
+
+/**
+ * This is the class to generate a animation type from a list of ImageIcon.
+ * It will display a JDialog window and then user can specify the frames
+ * per second rate that the user want to use to view the list of the ImageIcon.
+ * @author Veronica Liesaputra
+ */
+public class Movie extends JDialog
+    implements ActionListener,ChangeListener,PropertyChangeListener
+{
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    /**
+     * This is the label that we use to display the ImageIcon.
+     */
+    private JLabel picture;
+
+    /**
+     * This is a textField where user can set how many frames per second they want to view the
+     * ImageIcon.If user enter value that is bigger than the FPS_MAX then it will beep and user
+     * will need to enter new valid value or it will stay as the previous valid value.
+     * If user enter 0 then it will stop the animation.
+     */
+    private JFormattedTextField textField;
+
+    /**
+     * This is a slider where user can set how many frames per second they want to view the
+     * ImageIcon.If user slide to 0 then animation stop.
+     */
+    private JSlider framesPerSecond;
+    
+    /**
+     * This is the minimum frame per second value.
+     */
+    private int FPS_MIN = 0;
+
+    /**
+     * This is the maximum frame per second value. The value is the twice the amount of 
+     * ImageIcon in the list.
+     */
+    private int FPS_MAX;
+
+    /**
+     * This is the initial frame per second value to view the ImageIcon.
+     */
+    private int FPS_INIT = 1;
+
+    /**
+     * This is the ImageIcon index that are currently showing in the label.
+     */
+    private int frameNumber;
+
+    /**
+     * This is the total number of ImageIcon in the list.
+     */
+    private int NUM_FRAMES;
+
+    /**
+     * This hold the list of the ImageIcon that are to be displayed.
+     */
+    private ArrayList images;
+
+    /**
+     * This is telling how long is the delay for each frame to be refreshed.
+     */
+    private int delay;
+
+    /**
+     * This is the timer that generate the animation of the list of ImageIcon.
+     * The animation will be paused twice per cycle by restarting the timer.
+     * If the the prame per second chosen by the user is 0 then the timer
+     * will be stop.
+     */
+    private Timer timer;
+
+    /**
+     * This the flag to say when the animation should stop.
+     * If frozen = true then the animation should stop otherwise the animation
+     * should keep running.
+     */
+    private boolean frozen = false;
+
+    /**
+     * This will sho a modal-dialog and set up the value for the animation to run properly 
+     * according to the list of ImageIcon passed in.
+     * @param msg  hold the resource of the words that is stored in Messages.properties file.
+     * @param img  hold the list of ImageIcon to be displayed.
+     */
+    public Movie (ResourceBundle msg,ArrayList img)
+    {
+	super();
+
+	images = img;
+	NUM_FRAMES = img.size(); 
+	FPS_MAX = img.size() * 2;
+
+	messages = msg; 
+	setTitle(messages.getString("GenerateMovie"));
+	setModal(true);
+	setBackground(new Color (176,208,176));
+    }
+
+    /**
+     * This method will seeting up the content pane of the dialog window.
+     * It also setting up the timer and the delay.
+     * @return the content pane of the window.
+     */
+    public JPanel create_UI()
+    {
+	JPanel cont;
+	cont = new JPanel();
+	cont.setOpaque(true);
+	cont.setBackground(new Color (176,208,176));
+
+	JButton cls;
+	cls = new JButton(messages.getString("Close"));
+	cls.addActionListener(new ActionListener ()
+	    {
+		public void actionPerformed (ActionEvent e)
+		{
+		    dispose();
+		    if (timer != null)
+			timer.stop();
+		}
+	    });
+	cls.setDefaultCapable(true);
+	cls.setActionCommand(messages.getString("Close"));
+	cls.setBackground(new Color(176,208,176));
+
+
+	if (images.size() <= 0)
+	    {
+		JLabel empty_lbl;
+		empty_lbl = new JLabel ("No Image to be displayed   ");
+		cont.add(empty_lbl);
+		cont.add(cls);
+		return cont;
+	    }
+
+	cont.setLayout(new BoxLayout(cont, BoxLayout.PAGE_AXIS));
+
+	delay = 1000 / FPS_INIT;
+
+	JLabel sliderLabel;
+	sliderLabel = new JLabel(messages.getString("FramesPerSecond") + ": ", JLabel.CENTER);
+	sliderLabel.setBackground(new Color(176,208,176));
+        sliderLabel.setAlignmentX(Component.CENTER_ALIGNMENT);
+
+        NumberFormat numberFormat;
+	numberFormat = NumberFormat.getIntegerInstance();
+        NumberFormatter formatter;
+	formatter = new NumberFormatter(numberFormat);
+        formatter.setMinimum(new Integer(FPS_MIN));
+        formatter.setMaximum(new Integer(FPS_MAX));
+
+        textField = new JFormattedTextField(formatter);
+	textField.setBackground(new Color(224,240,224));
+        textField.setValue(new Integer(FPS_INIT));
+        textField.setColumns(5);
+        textField.addPropertyChangeListener(this);
+
+        textField.getInputMap().put(KeyStroke.getKeyStroke(
+                                        KeyEvent.VK_ENTER, 0),
+                                        "check");
+        textField.getActionMap().put("check", new AbstractAction() 
+	    {
+		public void actionPerformed(ActionEvent e) 
+		{
+		    if (!textField.isEditValid()) 
+			{ 
+			    Toolkit.getDefaultToolkit().beep();
+			    textField.selectAll();
+			} 
+		    else
+			{ 
+			    try 
+				{                  
+				    textField.commitEdit();
+				} 
+			    catch (java.text.ParseException exc) { }
+			}
+		}
+	    });
+
+        framesPerSecond = new JSlider(JSlider.HORIZONTAL,
+                                      FPS_MIN, FPS_MAX, FPS_INIT);
+        framesPerSecond.addChangeListener(this);
+	framesPerSecond.setBackground(new Color(176,208,176));
+	framesPerSecond.setMajorTickSpacing(10);
+        framesPerSecond.setMinorTickSpacing(1);
+        framesPerSecond.setPaintTicks(true);
+        framesPerSecond.setPaintLabels(true);
+        framesPerSecond.setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
+	
+        picture = new JLabel();
+	Dimension dim;
+	int iw,ih;
+	dim = Toolkit.getDefaultToolkit().getScreenSize();
+	iw = (int) (dim.width - 30);
+	ih = (int) (dim.height * 0.75);
+	picture.setPreferredSize(new Dimension(iw,ih));
+        picture.setHorizontalAlignment(JLabel.CENTER);
+        picture.setAlignmentX(Component.CENTER_ALIGNMENT);
+        picture.setBorder(BorderFactory.createCompoundBorder(
+                BorderFactory.createLoweredBevelBorder(),
+                BorderFactory.createEmptyBorder(10,10,10,10)));
+	picture.setBackground(new Color(176,208,176));
+        updatePicture(0);
+
+	
+        JPanel labelAndTextField;
+	labelAndTextField = new JPanel();
+        labelAndTextField.add(sliderLabel);
+        labelAndTextField.add(textField);
+	labelAndTextField.add(cls);
+	labelAndTextField.setBackground(new Color(176,208,176));
+
+	cont.add(picture);
+	cont.add(framesPerSecond);
+        cont.add(labelAndTextField);
+       
+        cont.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+
+        timer = new Timer(delay, this);
+        timer.setInitialDelay(delay * 7); 
+        timer.setCoalesce(true);
+	timer.start();
+	
+	return cont;
+    }
+
+    /**
+     * This method will tell what should happen when the slider's arrow
+     * change its position.
+     * <br>If user adjusting the frame per second value using the slider
+     * then it will set the Textfield to have the value that slider point 
+     * to at the moment.If the value is 0 then it will stop the animation
+     * otherwise it will keep animate with the new frame per second value.
+     * @param e is the slider when the slider's arrow change its position.
+     */
+    public void stateChanged(ChangeEvent e) 
+    {
+        JSlider source;
+	source = (JSlider)e.getSource();
+        int fps;
+	fps = (int)source.getValue();
+        if (!source.getValueIsAdjusting()) 
+	    { 
+		textField.setValue(new Integer(fps));
+		if (fps == 0) 
+		    {
+			if (!frozen) 
+			    stopAnimation();
+		    } 
+		else 
+		    {
+			delay = 1000 / fps;
+			timer.setDelay(delay);
+			timer.setInitialDelay(delay * 10);
+			if (frozen) 
+			    startAnimation();
+		    }
+	    } 
+	else 
+	    { 
+		textField.setText(String.valueOf(fps));
+	    }
+    }
+   
+    /**
+     * This method will set the slider value when the user adjusting
+     * the frame per second value using the textfield.
+     * <br>If user adjusting the frame per second value using text field, the
+     * slider's arrow position will be changed into that value.
+     * @param e the text field
+     */
+    public void propertyChange(PropertyChangeEvent e) 
+    {
+        if ("value".equals(e.getPropertyName())) 
+	    {
+		Number value;
+		value = (Number)e.getNewValue();
+		if (framesPerSecond != null && value != null) 
+		    {
+			framesPerSecond.setValue(value.intValue());
+		    }
+	    }
+    }
+
+    /**
+     * This method will start the timer.
+     */
+    public void startAnimation() 
+    {
+        timer.start();
+        frozen = false;
+    }
+
+    /**
+     * This method will stop the timer.
+     */
+    public void stopAnimation() 
+    {
+        timer.stop();
+        frozen = true;
+    }
+
+    /**
+     * This method will tell what should happen when the timer is start.
+     * When the timer start it will increase the framenumber and will
+     * update the label with the new image icon.
+     * If the timer already display all the image icon in the list, it 
+     * will restart the timer and reanimate the image icon again.
+     * @param e fires when the timer start.
+     */
+    public void actionPerformed(ActionEvent e) 
+    {
+        if (frameNumber == (NUM_FRAMES - 1))
+            frameNumber = 0;
+        else 
+            frameNumber++;
+        
+        updatePicture(frameNumber);
+
+        if ((frameNumber == (NUM_FRAMES - 1)) ||
+	    (frameNumber==(NUM_FRAMES/2 - 1)))
+            timer.restart();
+    }
+
+    /**
+     * This will update the label with the new ImageIcon.
+     * If the ImageIcon is null then the label will display an error message
+     * instead of the picture.
+     * @param frameNum this is the index of the ImageIcon to be displayed. 
+     */
+    protected void updatePicture(int frameNum) 
+    {
+        if (images.get(frameNumber) != null) 
+	    {
+		picture.setIcon((ImageIcon) images.get(frameNumber));
+	    } 
+	else 
+	    {
+		picture.setText(messages.getString("image")+" #"
+				+ frameNumber + " " +
+				messages.getString("notfound"));
+	    }
+    }
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/MyTextArea.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/MyTextArea.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/MyTextArea.java	(revision 21112)
@@ -0,0 +1,85 @@
+/** This class is not currently used and therefore will not be compiled by makegli. */
+
+package org.greenstone.gatherer.feedback;
+
+import javax.swing.*;
+import javax.swing.text.*;
+import java.util.Date;
+import java.awt.Color;
+
+/**
+ * This class will allow us to changed what should be inputted to the 
+ * text area as the user inputted the text to the text area.
+ * @author Veronica Liesaputra
+ */
+public class MyTextArea extends JTextArea 
+{
+    /**
+     * Creating an empty JTextArea.
+     */
+    public MyTextArea ()
+    {
+	super();
+    }
+
+    /**
+     * Creating a JTextArea already appended with cols.
+     * (Precondition: (cols != null))
+     * @param cols the text to be appended.
+     */
+    public MyTextArea(String cols) 
+    {
+	super(cols);
+    }
+ 
+    /**
+     * It will give the default model this text area should use.
+     * @return the default model this text area used.
+     */
+    protected Document createDefaultModel() 
+    {
+	return new UpperCaseDocument();
+    }
+ 
+    /**
+     * This class will be used as the default model of MyTextArea.
+     * Here we can control and modified the string that is going to be inserted to 
+     * the document.
+     */
+    static class UpperCaseDocument extends PlainDocument 
+    {
+	/**
+	 * This method will caused the document to be appended with the date followed by newline
+	 * everytime user trying to insert a new line to the document.
+	 * (Precondition : (offs >= 0) && (str != null))
+	 * @param offs is the offset placted to insert the string.
+	 * @param str is the string to be inserted.
+	 * @param a is the attribute set to be used.
+	 */
+	public void insertString(int offs, String str, AttributeSet a) 
+	    throws BadLocationException 
+	{
+	    if (str == null) 
+		{
+		    return;
+		}
+	    
+	    char[] upper;
+	    upper = str.toCharArray();
+
+	    if (str.endsWith("\n") == true)
+		{
+		    String str2;
+		    str2 = str + ((new Date()).toString()) + "\n\n>";
+		    upper = str2.toCharArray();
+		}
+	   
+	    super.insertString(offs, new String(upper), a);
+	}
+    }
+}
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ReportDetails.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ReportDetails.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ReportDetails.java	(revision 21112)
@@ -0,0 +1,1304 @@
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import java.awt.image.*;
+import javax.swing.*;
+import java.awt.*;
+import java.util.*;
+import javax.swing.event.MouseInputAdapter;
+import java.awt.event.*;
+import javax.swing.border.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.tree.*;
+
+/**
+ * This class can give preview of all the history of actions user did or give preview of all the
+ * xml files that are ready to be send and it also can give preview of all the screen shot of
+ * all window taken when user did the actions.
+ * If user select Details button then they will view all xml files that are going to be send.
+ * If user select View History then they will see the history record that we took so far.
+ * @author Veronica Liesaputra
+ */
+public class ReportDetails extends WindowAdapter 
+{
+    /**
+     * This is the tree that will be used to hold the feedback form,the history stored inside the vector 
+     * at the moment, and hold the comment user did when they recreating the event.
+     * This tree's data partly will be supplied from whats inside vector. 
+     */
+    private JTree tree;
+    /**
+     * This is the root node for tree.
+     */
+    private DefaultMutableTreeNode rootNode;
+    /**
+     * This hold the parent node of the current node that is currently just being added to tree.
+     */
+    private DefaultMutableTreeNode parentNode;
+    /**
+     * This hold the current node that is currently just being added to tree.
+     */
+    private DefaultMutableTreeNode currNode;
+    /**
+     * This is the tree model for tree.
+     */
+    private DefaultTreeModel treeModel;
+
+    /**
+     * This is the tree that will be used to hold the commands (actions) done by the user inside the vector
+     * at the moment.This tree builds from partial data of tree and which data entered to this tree
+     * is signed by the value of comm_bool and comm_start. 
+     */
+    private JTree tree2;
+    /**
+     * This is the root node for tree2.
+     */
+    private DefaultMutableTreeNode rootNode2;
+    /**
+     * This is the parent node of the current node that is currently just being added to tree2.
+     */
+    private DefaultMutableTreeNode parentNode2;
+    /**
+     * This is the current node that is currently just being added to tree2.
+     */
+    private DefaultMutableTreeNode currNode2;
+    /**
+     * This is the tree model for tree2.
+     */
+    private DefaultTreeModel treeModel2;
+
+    /**
+     * This is the tree that will be used to hold the history of all the actions
+     * done by the user before they choose to do the Reporting Feedback sequence.
+     */
+    private JTree tree3;
+    /**
+     * This is the root node for tree3.
+     */
+    private DefaultMutableTreeNode rootNode3;
+    /**
+     * This is the parent node for the current node that is currently just being added to tree3.
+     */
+    private DefaultMutableTreeNode parentNode3;
+    /**
+     * This is the current node that is currently just being added to tree3.
+     */
+    private DefaultMutableTreeNode currNode3;
+    /**
+     * This is the tree model for tree3.
+     */
+    private DefaultTreeModel treeModel3;
+
+    /**
+     * This is the tree that will be used to hold the commands (actions) done by the user 
+     * before they choose to do the Reporting Feedback sequence.This tree builds from partial data of tree3 
+     * and which data entered to this tree2 is signed by the value of comm_bool and comm_start. 
+     */
+    private JTree tree4;
+    /**
+     * This is the root node for tree4.
+     */
+    private DefaultMutableTreeNode rootNode4;
+    /**
+     * This is the parent node for the current node that is currently just being added to tree4.
+     */
+    private DefaultMutableTreeNode parentNode4;
+    /**
+     * This is the current node that is currently just being added to tree4.
+     */
+    private DefaultMutableTreeNode currNode4;
+    /**
+     * This is the tree model for tree4.
+     */
+    private DefaultTreeModel treeModel4;
+
+    /**
+     * This is the panel to display tree4.
+     */
+    private JPanel pane4;
+    /**
+     * This is the scroll pane for displaying tree3, so it is scrollable.
+     */
+    private JScrollPane scroll3;
+    /**
+     * This is the scroll pane for displaying tree4, so it is scrollable.
+     */
+    private JScrollPane scroll4;
+
+    /**
+     * This is the panel to display tree2.
+     */
+    private JPanel pane2;
+    /**
+     * This is the scroll pane for displaying tree2, so it is scrollable.
+     */
+    private JScrollPane scroll2;
+    /**
+     * This is the scroll pane for displaying tree, so it is scrollable.
+     */
+    private JScrollPane scroll;
+   
+    /**
+     * This is all the information taken from the Feedback form and information that
+     * was taken from the system.
+     */
+    private String[] err_details;
+    
+    /**
+     * This is the vector that hold all information about the history of all
+     * the actions user did.
+     * If rec = true, it means it hold the history of all the actions and information
+     * of all the windows were open when user did the actions during reporting feedback
+     * sequence.
+     * If rec = false, it means it hold the history of all the actions and information
+     * of all the windows were open when user did the actions before and after
+     * reporting feedback sequence.
+     */
+    private Vector vector;
+    
+    /**
+     * This is a variable to know which tag xml data that wants to be added to the JTree. 
+     */
+    private int count;
+    
+    /**
+     * This is a flag variable to sign whether or not the next data is to be added to
+     * the command JTree. 
+     */
+    private boolean comm_bool = false;
+    /**
+     * This is a flag variable to sign whether or not the next history is to be added to
+     * the command JTree.
+     */
+    private boolean comm_start = false;
+    /**
+     * This is a flag variable to sign whether or not at the moment we building a new tree
+     * or build from the existing tree.
+     */
+    private boolean new_tree = false;
+    
+    /**
+     * This is a flag variable to sign which type of preview that user wants at the moment.
+     * If its true, then its means user will view all the 6 xml files that will be send.
+     * If its false, then its means user only wants to view the history and command user just did
+     * before and after the reporting feedback sequence.
+     */
+    private boolean rec = false;
+   
+    /**
+     * This is array list of all the image icons that are inside all the command JTree.
+     */
+    private ArrayList movie;
+
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    private String[] img;
+   
+    /**
+     * This constructor will be seeting up the data member to its appropriate value.
+     * (Precondition : (msg != null) && (bool != null))
+     * @param stack this is all vector to be supplied to tree.
+     * @param err_array holding all the information taken when user doing the FeedbackInterface form.
+     * @param imgFile holding all the information inside all the panels in the FeedbackInterface form.
+     * @param msg hold the resource of the words that is stored in Messages.properties file.
+     * @param bool hold the falg value of rec.
+     */
+    public ReportDetails(Vector stack,String[] err_array,String[] imgFile,ResourceBundle msg,boolean bool)
+    {
+	img = imgFile;
+	vector = stack;
+	err_details = err_array;
+	messages = msg;
+	rec = bool;
+	movie = new ArrayList();
+	makeNewWindow();
+    }
+
+    /**
+     * This method will make a modal-dialog window that will display all the preview.
+     */
+    public void makeNewWindow()
+    {
+	JDialog frame;
+	frame = new View (this);
+	frame.setModal(true);
+	frame.setVisible(true);
+    }
+
+    /**
+     * This class is the dialog window that will give the preview.
+     */
+    class View extends JDialog implements ActionListener
+    {
+	/**
+	 * This variable is holding the current window that display the preview.
+	 */
+	private JDialog frame;
+	/**
+	 * This variable holding the owner of this window.
+	 */
+	private ReportDetails framework = null;
+
+	/**
+	 * This constructor will make the dialog window for previewing and setting up the
+	 * content pane of this window.
+	 * @param controller is the owner of this window.
+	 */
+	public View (ReportDetails controller)
+	{
+	    super();
+	    framework = controller;
+	    frame = this;
+
+	    setTitle(messages.getString("ViewReport"));  
+	    setBackground(new Color(176,208,176));
+	    setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE);
+	    setDefaultLookAndFeelDecorated(true);
+	 
+	    JPanel cont;
+	    cont = new JPanel();
+	    cont.setLayout(new BoxLayout(cont,BoxLayout.PAGE_AXIS));
+	    cont.setBackground(new Color(176,208,176));
+	
+	    JTabbedPane tab;
+	    tab = new JTabbedPane();
+	    tab.setBackground(new Color(176,208,176));
+
+	    new_tree = false;
+
+	    if (err_details != null)
+		{
+		    tab.addTab(messages.getString("ViewFeedbackForm"), 
+			       createFeedback_UI(err_details,img));
+		}
+	    
+	    if (rec == true)
+		{
+		    tab.addTab(messages.getString("ViewFeedbackHistoryState"),
+			       createFeedbackHistory_UI(vector));
+		    tab.addTab(messages.getString("ViewFeedbackCommandState"),pane2);
+		    /*tab.addTab(messages.getString("ViewHistoryState"),
+		      createHistory_UI());*/
+		    createHistory_UI();
+		    tab.addTab(messages.getString("ViewCommandState"),pane4);
+		}
+	    else
+		{
+		    /*tab.addTab(messages.getString("ViewHistoryState"),
+		      createFeedbackHistory_UI(vector));*/
+		    createFeedbackHistory_UI(vector);
+		    tab.addTab(messages.getString("ViewCommandState"),pane2);
+		}
+
+	    cont.add(tab);
+
+	    JLabel lbl;
+	    lbl = new JLabel("   ");
+	    lbl.setBackground(new Color(176,208,176));
+
+	    cont.add(lbl);
+
+	    JPanel pane3;
+	    pane3 = new JPanel();
+	    pane3.setBackground(new Color(176,208,176));
+	    JButton button;
+	    button = new JButton(messages.getString("Close"));
+	    button.setActionCommand(messages.getString("Close"));
+	    button.addActionListener(this);
+	    button.setDefaultCapable(true);
+	    button.setToolTipText(messages.getString("ClosePreview"));
+	    button.setBackground(new Color(176,208,176));
+	   
+	    /*JButton button2;
+	    button2 = new JButton(messages.getString("ViewMovie"));
+	    button2.setActionCommand(messages.getString("ViewMovie"));
+	    button2.addActionListener(new ActionListener()
+		{
+		    public void actionPerformed (ActionEvent e)
+		    {
+			JDialog.setDefaultLookAndFeelDecorated(true);
+			JDialog frame;
+			frame = new Movie(messages,movie);
+
+			frame.setContentPane(((Movie)frame).create_UI());
+			frame.pack();
+			frame.setVisible(true);
+		    }
+		});
+	    button2.setToolTipText(messages.getString("ViewMovieText"));
+	    button2.setBackground(new Color(176,208,176));
+	    pane3.add(button2);*/
+	    pane3.add(button);
+	    pane3.setBorder(new EmptyBorder(new Insets(0,10,10,10)));
+
+	    cont.add(pane3);
+
+	    cont.setBorder(new EmptyBorder(new Insets(5,5,5,5)));
+	    getContentPane().add(cont);
+
+	    tab.setAlignmentX(Component.LEFT_ALIGNMENT);
+	    lbl.setAlignmentX(Component.LEFT_ALIGNMENT);
+	    pane3.setAlignmentX(Component.LEFT_ALIGNMENT);
+	    
+	    addWindowListener(framework);
+	    Toolkit kit;
+	    kit = Toolkit.getDefaultToolkit();
+	    Dimension screenSize;
+	    screenSize = kit.getScreenSize();
+	    int screenHeight;
+	    screenHeight = screenSize.height;
+	    int screenWidth;
+	    screenWidth = screenSize.width;
+	    Dimension size;
+	    size = this.getSize();
+	    setLocation((screenWidth -size.width)/ 4, 
+			(screenHeight - size.height)/ 4);
+
+	    pack();
+	    setResizable(false);
+	}
+	
+	/**
+	 * This method will get the vector that is stored inside temp_feedbackhist.log
+	 * if rec is true, and will get the vector that is stored inside temp_history.log
+	 * if rec is false. If file is not exist then it will return null. 
+	 * @return vector that is stored in the file.
+	 */
+	public Vector getPrev_log()
+	{
+	    Vector stack = null;
+	    try
+		{
+		    File f;
+		    if (rec == true)
+			{
+			    f = new File("temp_feedbackhist.log");
+			}
+		    else
+			{
+			    f = new File("temp_history.log");
+			}
+
+		    if (f.exists() == true)
+		    {
+			FileInputStream fis = new FileInputStream(f);
+			ObjectInputStream ois = new ObjectInputStream(fis);
+			stack = (Vector) ois.readObject();
+			ois.close();
+			return stack;
+		    }
+		    else
+			stack = null; 
+		}
+	    catch (IOException exp) {exp.printStackTrace();}
+	    catch (ClassNotFoundException exp2) 
+		{
+		    System.out.println("class exp");}
+	    
+	    return stack;
+	}
+
+	/**
+	 * This method will update the graphics of this window and it will
+	 * set variable finish in Conformation to true.
+	 * @param g is the graphics for this window.
+	 */
+	public void paint (Graphics g)
+	{
+	    super.paint(g); 
+	    FeedbackInterface.setFinish(true);
+	}
+
+	/**
+	 * This method will close the preview window if user select close button.
+	 * @param e action event user did.
+	 */
+	public void actionPerformed (ActionEvent e)
+	{   
+	    if(messages.getString("Close").equals(e.getActionCommand()))
+		{ 
+		    dispose();
+		}
+	}
+
+	/**
+	 * This method will add a record to history tree and the command tree.
+	 * @param log is the record to be added.
+	 */
+	public void getHist(History log)
+	{
+	    comm_bool = true;
+	    startContent(10);
+
+	    startContent(7);
+	    saveContent(log.getDate());
+	    closeContent(7);
+	    
+	    startContent(2);
+	    saveContent(log.getTitle());
+	    closeContent(2);
+	    
+	    startContent(8);
+	    saveContent(log.getCommand());
+	    closeContent(8);
+	  
+	    String image;
+	    image = log.getImage();
+
+	    if (image != null)
+		{
+		    byte[] img;
+		    img = Base64.decode(image);
+		    
+		    String width,height;
+		    width = log.getWidth();
+		    height = log.getHeight();
+		    
+		    int w,h;
+		    w = Integer.parseInt(width);
+		    h = Integer.parseInt(height);
+
+		    ImageIcon icon,small_icon;
+		    small_icon = new ImageIcon(img);
+		    icon = new ImageIcon
+			(small_icon.getImage().getScaledInstance(w,h,Image.SCALE_SMOOTH));
+
+		    movie.add(0,icon);
+
+		    startContent(6);
+		    saveImage(log.getImageName(),width,height);
+		    closeContent(6);
+		}
+	    comm_bool = false;
+	    
+	    ArrayList component;
+	    component = log.getArray();
+	    int j;
+	    for (j = 0 ; j < component.size() ; j++)
+		{
+		    startContent(9);
+		    CompGroup comp;
+		    comp = (CompGroup) component.get(j);
+		    
+		    startContent(8);
+		    saveContent(comp.getStatus());
+		    closeContent(8);
+		    
+		    startContent(0);
+		    UserComponent cmp;
+		    cmp = comp.getComponent();
+		    showReport(cmp);
+		    closeContent(0);
+		    
+		    closeContent(9);
+		}
+	    
+	    comm_bool = true;
+	    closeContent(10);
+	    comm_bool = false;
+	}
+    
+	/**
+	 * This method will make the tree that contains the history and the commands of all the actions and window 
+	 * open when user did the action that stored in vector.
+	 * @param vector the vector that contains the information need to be display in tree.
+	 * @return       the panel that display tree and tree2.
+	 */
+	public JPanel createFeedbackHistory_UI(Vector vector)
+	{
+	    scroll = null;
+	    int k;
+
+	    tree = null;
+	    rootNode = null;
+	    currNode = null;
+	    parentNode =  null; 
+	    treeModel = null;
+
+	    tree2 = null;
+	    rootNode2 = null;
+	    currNode2 = null;
+	    parentNode2 =  null; 
+	    treeModel2 = null;
+	    
+	    comm_bool = true;
+	    comm_start = true;
+	    new_tree = false;
+
+	    startElement(messages.getString("HISTORY"));
+ 
+	    for (k = 0 ; k < vector.size() ; k++)
+		{
+		    History log;
+		    log = (History) vector.get(k);
+		    getHist(log);
+		}
+
+	    Vector stack;
+	    stack = getPrev_log();
+	    if (stack != null)
+		{
+		    for (k = 0 ; k < stack.size() ; k++)
+			{
+			    History log;
+			    log = (History) stack.get(k);
+			    getHist(log);
+			}
+		    stack.removeAllElements();
+		    System.gc();
+		    stack = null;
+		}
+
+	    comm_bool = true;
+	    endElement(messages.getString("HISTORY"));
+	    comm_bool = false;
+	    
+	    comm_start = false;
+	
+	    pane2 = new JPanel();
+	    pane2.add(scroll2);
+	    pane2.setBackground(new Color(176,208,176));
+	    pane2.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+
+	    JPanel pane;
+	    pane = new JPanel();
+	    pane.add(scroll);
+	    pane.setBackground(new Color(176,208,176));
+	    pane.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+	    
+	    return pane;
+	}
+
+	/**
+	 * This method will make the tree that contains the history and the commands of all the actions and window 
+	 * open when user did the action before user choose the reporting feedback sequence.
+	 * @return the panel contains the preview of this history and commands before user choose to start
+	 *         the reporting feedback sequence.
+	 */
+	public JPanel createHistory_UI()
+	{
+	    scroll3 = null;
+	    int k;
+
+	    tree3 = null;
+	    rootNode3 = null;
+	    currNode3 = null;
+	    parentNode3 =  null; 
+	    treeModel3 = null;
+
+	    tree4 = null;
+	    rootNode4 = null;
+	    currNode4 = null;
+	    parentNode4 =  null; 
+	    treeModel4 = null;
+	    
+	    comm_bool = true;
+	    comm_start = true;
+	    new_tree = true;
+
+	    startElement(messages.getString("HISTORY"));
+ 
+	    try
+		{
+		    File f = new File("history.log");
+		    FileInputStream fis = new FileInputStream(f);
+		    ObjectInputStream ois = new ObjectInputStream(fis);
+		    
+		    Vector stack;
+		    stack = (Vector) ois.readObject();
+
+		    if (stack != null)
+			{
+			    for (k = 0 ; k < stack.size() ; k++)
+				{
+				    History log;
+				    log = (History) stack.get(k);
+				    getHist(log);
+				}
+			    stack.removeAllElements();
+			    System.gc();
+			    stack = null;
+			}
+		    
+		    stack = (Vector) ois.readObject();
+		    
+		    if (stack != null)
+			{
+			    for (k = 0 ; k < stack.size() ; k++)
+				{
+				    History log;
+				    log = (History) stack.get(k);
+				    getHist(log);
+				}
+			    stack.removeAllElements();
+			    System.gc();
+			    stack = null;
+			}
+		
+		    ois.close();
+		}
+	    catch (IOException exp) {exp.printStackTrace();}
+	    catch (ClassNotFoundException exp2) 
+		{
+		    System.out.println("class exp");}
+	    
+	    comm_bool = true;
+	    endElement(messages.getString("HISTORY"));
+	    comm_bool = false;
+	    
+	    comm_start = false;
+	
+	    pane4 = new JPanel();
+	    pane4.add(scroll4);
+	    pane4.setBackground(new Color(176,208,176));
+	    pane4.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+
+	    JPanel pane;
+	    pane = new JPanel();
+	    pane.add(scroll3);
+	    pane.setBackground(new Color(176,208,176));
+	    pane.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+	    
+	    return pane;
+	}
+
+	/**
+	 * This method display all the information taken when user did the Feedback form.
+	 * @param err_details is the information taken when user did the Feedback form.
+	 * @param img         is the information from all the panels inside the Feedback form.
+	 * @return            panel displaying the information.
+	 */
+	public JPanel createFeedback_UI(String[] err_details,String[] img)
+	{
+	    scroll = null;
+	 
+	    tree = null;
+	    rootNode = null;
+	    currNode = null;
+	    parentNode =  null; 
+	    treeModel = null;
+
+	    tree2 = null;
+	    rootNode2 = null;
+	    currNode2 = null;
+	    parentNode2 =  null; 
+	    treeModel2 = null;
+
+	    startElement(messages.getString("BUGS"));
+	    
+	    startElement(messages.getString("IDCode"));
+	    characters(err_details[0].toCharArray(),0,err_details[0].length());
+	    endElement(messages.getString("IDCode"));
+	
+	    startElement(messages.getString("LastViewedWindow"));
+	    characters(err_details[1].toCharArray(),0,err_details[1].length());
+	    endElement(messages.getString("LastViewedWindow"));
+	    
+	    startElement(messages.getString("Optionals"));
+	   
+	    startElement(messages.getString("Type"));
+	    characters(err_details[3].toCharArray(),0,err_details[3].length());
+	    endElement(messages.getString("Type"));
+		
+	    startElement(messages.getString("Urgency"));
+	    characters(err_details[4].toCharArray(),0,err_details[4].length());
+	    endElement(messages.getString("Urgency"));
+	    
+	    endElement(messages.getString("Optionals"));
+	
+	    startElement(messages.getString("User"));
+	    
+	    startElement(messages.getString("Name"));
+	    characters(err_details[5].toCharArray(),0,err_details[5].length());
+	    endElement(messages.getString("Name"));
+	    
+	    startElement(messages.getString("HomeDirectory"));
+	    characters(err_details[6].toCharArray(),0,err_details[6].length());
+	    endElement(messages.getString("HomeDirectory"));
+	    
+	    startElement(messages.getString("WorkingDirectory"));
+	    characters(err_details[7].toCharArray(),0,err_details[7].length());
+	    endElement(messages.getString("WorkingDirectory"));
+	    
+	    startElement(messages.getString("SMTP"));
+	    characters(err_details[19].toCharArray(),0,err_details[19].length());
+	    endElement(messages.getString("SMTP"));
+	    
+	    startElement(messages.getString("Email"));
+	    characters(err_details[20].toCharArray(),0,err_details[20].length());
+	    endElement(messages.getString("Email"));
+
+	    endElement(messages.getString("User"));
+	    
+	    startElement(messages.getString("OpenFormTime"));
+	    characters(err_details[8].toCharArray(),0,err_details[8].length());
+	    endElement(messages.getString("OpenFormTime"));
+
+	    startElement(messages.getString("SendFormTime"));
+	    characters(err_details[9].toCharArray(),0,err_details[9].length());
+	    endElement(messages.getString("SendFormTime"));
+	    
+	    startElement(messages.getString("OperatingSystem"));
+	    characters(err_details[10].toCharArray(),0,err_details[10].length());
+	    endElement(messages.getString("OperatingSystem"));
+
+	    startElement(messages.getString("Java"));
+	    
+	    startElement(messages.getString("JavaInformation"));
+	    characters(err_details[11].toCharArray(),0,err_details[11].length());
+	    endElement(messages.getString("JavaInformation"));
+	    
+	    startElement(messages.getString("TotalMemory"));
+	    characters(err_details[21].toCharArray(),0,err_details[21].length());
+	    endElement(messages.getString("TotalMemory"));
+
+	    startElement(messages.getString("MaxMemory"));
+	    characters(err_details[22].toCharArray(),0,err_details[22].length());
+	    endElement(messages.getString("MaxMemory"));
+
+	    startElement(messages.getString("FreeMemory"));
+	    characters(err_details[23].toCharArray(),0,err_details[23].length());
+	    endElement(messages.getString("FreeMemory"));
+
+	    startElement(messages.getString("DefaultLocale"));
+	    characters(err_details[12].toCharArray(),0,err_details[12].length());
+	    endElement(messages.getString("DefaultLocale"));
+	
+
+	    endElement(messages.getString("Java"));
+	    
+	    startElement(messages.getString("Browser"));
+	    characters(err_details[13].toCharArray(),0,err_details[13].length());
+	    endElement(messages.getString("Browser"));
+	
+	    startElement(messages.getString("LocalHostName"));
+	    characters(err_details[14].toCharArray(),0,err_details[14].length());
+	    endElement(messages.getString("LocalHostName"));
+	
+	    startElement(messages.getString("LocalHostAddress"));
+	    characters(err_details[15].toCharArray(),0,err_details[15].length());
+	    endElement(messages.getString("LocalHostAddress"));
+	    
+	    startElement(messages.getString("ScreenResolution"));
+	    characters(err_details[16].toCharArray(),0,err_details[16].length());
+	    endElement(messages.getString("ScreenResolution"));
+
+	    if (img!=null)
+		{
+		    startElement("Sequences");
+		    int i;
+		    for ( i = 0 ; i < img.length ; i= i + 10)
+			{
+			    startElement("Sequence");
+			    
+			    startElement("Title");
+			    characters(img[i].toCharArray(),0,img[i].length());
+			    endElement("Title");
+			    
+			    startElement("Details");
+			    characters(img[i+9].toCharArray(),0,img[i+9].length());
+			    endElement("Details");
+			    
+			    startElement("ImageDescription");
+			    
+			    startElement("ScreenShot");
+			    startElement("Image");
+			    saveImage(img[i+2],img[i+7],img[i+8]);
+			    endElement("Image");
+			    endElement("ScreenShot");
+			    
+			    startElement("ErrorLineAndScreenShot");
+			    startElement("Image");
+			    saveImage(img[i+4],img[i+7],img[i+8]);
+			    endElement("Image");
+			    endElement("ErrorLineAndScreenShot");
+			    
+			    startElement("ErrorLineForScreenShot");
+			    startElement("Image");
+			    saveImage(img[i+6],img[i+7],img[i+8]); 
+			    endElement("Image");
+			    endElement("ErrorLineForScreenShot");
+			  
+			    endElement("ImageDescription");
+			    
+			    endElement("Sequence");
+			}
+		    endElement("Sequences");
+		}
+
+	    endElement(messages.getString("BUGS"));
+	    
+	    JPanel pane;
+	    pane = new JPanel();
+	    pane.add(scroll);
+	    pane.setBackground(new Color(176,208,176));
+	    pane.setBorder(new EmptyBorder(new Insets(10,10,10,10)));
+
+	    return pane;
+	}
+	
+	/**
+	 * This will add the UserComponent to the tree.
+	 * @param sx is the UserComponent to be added. 
+	 */
+	public void showReport (UserComponent sx)
+	{
+	    String type;
+	    type = sx.getType();
+	    String title;
+	    title = sx.getTitle();
+	    ArrayList content;
+	    content = sx.getContent();
+	    String selected;
+	    selected = sx.getSelected();
+	    String visible;
+	    visible = sx.getVisible();
+	    String image;
+	    image = sx.getImage();
+	    String img;
+	    img = sx.getImageFileName();
+	    String width;
+	    width = sx.getWidth();
+	    String height;
+	    height = sx.getHeight();
+	    String tooltip;
+	    tooltip = sx.getToolTip();
+	    
+	    if (type != null)
+		{
+		    startContent(1);
+		    saveContent(type);
+		    closeContent(1);
+		}
+	    
+	    if (title != null)
+		{
+		    startContent(2);
+		    saveContent(title);
+		    closeContent(2);
+		}
+	    
+	    if (content != null)
+		{
+		    startContent(3);
+		    int j;
+		    for ( j = 0 ; j < content.size() ; j++)
+			{
+			    if (content.get(j) instanceof String)
+				saveContent((String) content.get(j));
+			    else if (content.get(j) instanceof UserComponent)
+				{
+				    startContent(0);
+				    UserComponent cmp;
+				    cmp = (UserComponent) content.get(j);
+				    showReport(cmp);
+				    closeContent(0);
+				}
+			}
+		    closeContent(3);
+		}
+	    
+	    if (selected != null)
+		{
+		    startContent(4);
+		    saveContent(selected);
+		    closeContent(4);
+		}
+	    
+	    if (visible != null)
+		{
+		    startContent(5);
+		    saveContent(visible);
+		    closeContent(5);
+		}
+	
+	    if (img != null)
+		{
+		    startContent(6);
+		    saveImage(img,width,height);
+		    closeContent(6);
+		}
+	    else
+		{
+		    if (image != null)
+			{ 
+			    startContent(11);
+			    saveContent(image);
+			    closeContent(11);
+			} 
+		}
+
+	    if (tooltip != null)
+		{
+		    startContent(12);
+		    saveContent(tooltip);
+		    closeContent(12);
+		}
+	}
+	
+	/**
+	 * This method will give what type of information this node hold according the number
+	 * @param num the number that describe what type of information it hold.
+	 */
+	public void startContent (int num)
+	{
+	    switch(num)
+		{
+		case 0: startElement(messages.getString("COMPONENT"));
+		    break;
+		case 1: startElement(messages.getString("Type"));
+		    break;
+		case 2: startElement(messages.getString("Title"));
+		    break;
+		case 3: startElement(messages.getString("Content"));
+		    break;
+		case 4: startElement(messages.getString("Selected"));
+		    break;
+		case 5: startElement(messages.getString("Visible"));  
+		    break;
+		case 6: startElement(messages.getString("Image"));
+		    break;
+		case 7: startElement(messages.getString("Date"));
+		    break;
+		case 8: startElement(messages.getString("Command"));
+		    break;
+		case 9: startElement(messages.getString("COMPONENTS"));
+		    break;
+		case 10:startElement(messages.getString("LOG"));
+		    break;
+		case 11:startElement(messages.getString("Icon"));
+		    break;
+		case 12:startElement(messages.getString("ToolTipText"));
+		    break;
+		case 13:startElement(messages.getString("Comment"));
+		    break;
+		}
+	}
+	
+	/**
+	 * This will add the text content to the current node.
+	 * @param text is the text content of the current node.
+	 */
+	public void saveContent (String text)
+	{
+	    if (text == null)
+		text = "\t";
+	    characters(text.toCharArray(),0,text.length());
+	}
+	
+	/**
+	 * This will say that what type of information that has been finished added
+	 * accroding its number.
+	 * @param num  the number that describe what type of information it hold.
+	 */
+	public void closeContent (int num)
+	{
+	    switch(num)
+		{
+		case 0: endElement(messages.getString("COMPONENT"));
+		    break;
+		case 1: endElement(messages.getString("Type"));
+		    break;
+		case 2: endElement(messages.getString("Title"));
+		    break;
+		case 3: endElement(messages.getString("Content"));
+		    break;
+		case 4: endElement(messages.getString("Selected"));
+		    break;
+		case 5: endElement(messages.getString("Visible"));
+		    break;
+		case 6: endElement(messages.getString("Image"));
+		    break;
+		case 7: endElement(messages.getString("Date"));
+		    break;
+		case 8: endElement(messages.getString("Command"));
+		    break;
+		case 9: endElement(messages.getString("COMPONENTS"));
+		    break;
+		case 10:endElement(messages.getString("LOG"));
+		    break;
+		case 11:endElement(messages.getString("Icon"));
+		    break;
+		case 12:endElement(messages.getString("ToolTipText"));
+		    break;
+		case 13:endElement(messages.getString("Comment"));
+		    break;
+		}
+	}
+	
+	/**
+	 * This method will add the this string to the appropriate place in the tree.
+	 * @param type its the type to be added to the tree.
+	 */
+	public void startElement (String type)
+	{
+	   
+	    if ((comm_bool == true)&&(comm_start == true))
+		{
+		    if (new_tree == true)
+			{
+			    if (type == messages.getString("HISTORY"))
+				{
+				    rootNode4 = new DefaultMutableTreeNode(messages.getString("HISTORYCOMMANDS"));
+				    treeModel4 = new DefaultTreeModel(rootNode4);
+				    
+				    parentNode4 = rootNode4;
+				    tree4 = new JTree(treeModel4);
+				    tree4.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+				    tree4.setBackground(new Color(224,240,224));
+				    tree4.setEditable(false);
+				    tree4.setShowsRootHandles(true);
+				}  
+			    else
+				{
+				    currNode4 = new DefaultMutableTreeNode(type + " : ");
+				    treeModel4.insertNodeInto(currNode4,parentNode4,parentNode4.getChildCount());
+				    parentNode4 = currNode4;
+				} 
+			}
+		    else
+			{
+			    if (type == messages.getString("HISTORY"))
+				{
+				    rootNode2 = new DefaultMutableTreeNode(messages.getString("HISTORYCOMMANDS"));
+				    treeModel2 = new DefaultTreeModel(rootNode2);
+				    
+				    parentNode2 = rootNode2;
+				    tree2 = new JTree(treeModel2);
+				    tree2.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+				    tree2.setBackground(new Color(224,240,224));
+				    tree2.setEditable(false);
+				    tree2.setShowsRootHandles(true);
+				}  
+			    else
+				{
+				    currNode2 = new DefaultMutableTreeNode(type + " : ");
+				    treeModel2.insertNodeInto(currNode2,parentNode2,parentNode2.getChildCount());
+				    parentNode2 = currNode2;
+				} 
+			}
+		}
+		
+	    if ((type == messages.getString("COMMENTS"))||(type == messages.getString("BUGS"))||
+		(type == messages.getString("HISTORY")))
+		{
+		    if (new_tree == true)
+			{
+			    rootNode3 = new DefaultMutableTreeNode(type);
+			    treeModel3 = new DefaultTreeModel(rootNode3);
+			    
+			    parentNode3 = rootNode3;
+			    tree3 = new JTree(treeModel3);
+			    tree3.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+			    
+			    tree3.setBackground(new Color(224,240,224));
+			    tree3.setEditable(false);
+			    tree3.setShowsRootHandles(true);
+			}
+		    else
+			{
+			    rootNode = new DefaultMutableTreeNode(type);
+			    treeModel = new DefaultTreeModel(rootNode);
+			    
+			    parentNode = rootNode;
+			    tree = new JTree(treeModel);
+			    tree.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+			    
+			    tree.setBackground(new Color(224,240,224));
+			    tree.setEditable(false);
+			    tree.setShowsRootHandles(true);
+			}
+		}
+	    else
+		 {
+		     if (new_tree == true)
+			 {
+			     currNode3 = new DefaultMutableTreeNode(type + " : ");
+			     treeModel3.insertNodeInto(currNode3,parentNode3,parentNode3.getChildCount());
+			     parentNode3 = currNode3;
+			 }
+		     else
+			 {
+			     currNode = new DefaultMutableTreeNode(type + " : ");
+			     treeModel.insertNodeInto(currNode,parentNode,parentNode.getChildCount());
+			     parentNode = currNode;
+			 }
+		 }
+	}
+	
+	/**
+	 * This method is to mark that we finished adding the type to the tree.
+	 * @param type type that finished added to the tree.
+	 */
+	public void endElement (String type)
+	{
+	    if ((type == messages.getString("HISTORY"))||(type == messages.getString("BUGS"))||
+		(type == messages.getString("COMMENTS")))
+		{
+		    if (new_tree == true)
+			{
+			    scroll3 = new JScrollPane(tree3);
+			    scroll3.setPreferredSize(new Dimension(600,600));
+			    if (comm_bool == true)
+				{
+				    scroll4 = new JScrollPane(tree4);
+				    scroll4.setPreferredSize(new Dimension(600,600));
+				}
+			}
+		    else
+			{
+			    scroll = new JScrollPane(tree);
+			    scroll.setPreferredSize(new Dimension(600,600));
+			    if (comm_bool == true)
+				{
+				    scroll2 = new JScrollPane(tree2);
+				    scroll2.setPreferredSize(new Dimension(600,600));
+				}
+			}
+		}
+	    else
+		{
+		    if (new_tree == true)
+			{
+			    parentNode3 = (DefaultMutableTreeNode) parentNode3.getParent();
+			    
+			    if ((comm_bool == true)&&(comm_start == true))
+				{
+				    if (parentNode4 == null)  return;
+				    parentNode4 = (DefaultMutableTreeNode) parentNode4.getParent();
+				}  
+			}
+		    else
+			{
+			    parentNode = (DefaultMutableTreeNode) parentNode.getParent();
+			    
+			    if ((comm_bool == true)&&(comm_start == true))
+				{
+				    if (parentNode2 == null)  return;
+				    parentNode2 = (DefaultMutableTreeNode) parentNode2.getParent();
+				}  
+			}
+		}
+	}
+	
+	/*public BufferedImage iconToImage(Icon icon, Component targetComponent) 
+	{
+	    int w = icon.getIconWidth();
+	    int h = icon.getIconHeight();
+	    GraphicsConfiguration gc = targetComponent.getGraphicsConfiguration();
+	    BufferedImage image = gc.createCompatibleImage(w, h);
+	    Graphics2D g = image.createGraphics();
+	    icon.paintIcon(targetComponent,g,0,0);
+	    g.dispose();
+	    return image;
+	}
+	
+	public BufferedImage savingImage (ImageIcon img)
+	{
+	    JFrame f = new JFrame();
+	    f.setVisible(false);
+	    BufferedImage image = iconToImage(img,f);
+	    f.dispose();
+	    return image;
+	    }*/
+	
+	/**
+	 * This will add the image filename,width and height to the tree.
+	 * @param filename image's filename
+	 * @param width    image's width
+	 * @param height   image's height
+	 */
+	public void saveImage (String filename,String width,String height)
+	{
+	    startElement(messages.getString("Title"));
+	    characters(filename.toCharArray(),0,filename.length());
+	    endElement(messages.getString("Title"));
+	
+	    startElement(messages.getString("Size"));
+	    
+	    startElement(messages.getString("Width"));
+	    characters(width.toCharArray(),0,width.length());
+	    endElement(messages.getString("Width"));
+	    
+	    startElement(messages.getString("Height"));
+	    characters(height.toCharArray(),0,height.length());
+	    endElement(messages.getString("Height"));
+
+	    endElement(messages.getString("Size"));
+
+	    /*JFrame f = new JFrame();
+	    f.setVisible(false);
+	    BufferedImage image = savingImage(img);
+	    BufferedImage bi = new BufferedImage(400, 400, BufferedImage.TYPE_INT_RGB);
+	    Graphics2D big = bi.createGraphics();
+	    big.drawImage(image.getScaledInstance(400,400,Image.SCALE_SMOOTH),0,0,f);
+	    f.dispose();
+	    
+	
+	    JLabel label = new JLabel(new ImageIcon(bi));
+	    label.setPreferredSize(new Dimension(400,400));
+	    currNode = new DefaultMutableTreeNode(label);
+	    treeModel.insertNodeInto(currNode,parentNode,parentNode.getChildCount());*/
+	}
+	
+	/**
+	 * This method will add the characters to the current node's type.
+	 * @param ch is the char[] that is need to be added to the current node's type.
+	 * @param start is where we should start taking from ch.
+	 * @param len is the length of charcacters we should take from ch
+	 */
+	public void characters (char[] ch, int start, int len)
+	{
+	    String text;
+	    text = new String(ch,start,len);
+	    String text1;
+	    text1 = text.trim();
+	    if (text1.length() > 0)
+		{
+		    if ((comm_bool == true)&&(comm_start == true))
+			{
+			    if (new_tree == true)
+				{
+				    String txt2;
+				    txt2 = (String) currNode4.getUserObject();
+				    currNode4.setUserObject(txt2 + text1);
+				}
+			    else 
+				{
+				    String txt2;
+				    txt2 = (String) currNode2.getUserObject();
+				    currNode2.setUserObject(txt2 + text1);
+				}
+			}
+
+		    if (new_tree == true)
+			{
+			    String txt;
+			    txt = (String) currNode3.getUserObject();
+			    currNode3.setUserObject(txt + text1);
+			}
+		    else
+			{
+			    String txt;
+			    txt = (String) currNode.getUserObject();
+			    currNode.setUserObject(txt + text1);
+			}
+		}
+	}
+    }
+ 
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SaveToXML.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SaveToXML.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SaveToXML.java	(revision 21112)
@@ -0,0 +1,482 @@
+package org.greenstone.gatherer.feedback;
+
+import java.awt.image.*;
+import java.io.*;// SAX classes.
+import javax.imageio.*;
+import org.xml.sax.*;
+import org.xml.sax.helpers.*;//JAXP 1.1
+import javax.xml.parsers.*;
+import javax.xml.transform.*;
+import javax.xml.transform.stream.*;
+import javax.xml.transform.sax.*; 
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import java.util.*;
+
+/**
+ * This class will make xml files from the information given.
+ * @author Veronica Liesaputra
+ */
+public class SaveToXML
+{
+    /**
+     * This is the file where the xml file will be saved.
+     */
+    private FileOutputStream fos;
+    /**
+     * This is the transformer handler for the xml file.
+     */
+    private TransformerHandler hd;
+    /**
+     * This is the attribute implementation for the xml file.
+     */
+    private AttributesImpl atts;
+     /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+
+    /**
+     * This constructor will setup what languange should be use.
+     * @param msg will hold the resource of the words that is stored in Messages.properties file.
+     */
+    public SaveToXML(ResourceBundle msg)
+    {
+	messages = msg;
+    }
+
+    /**
+     * This will save the image, image filename,width and height to the xml file.
+     * @param image    image
+     * @param filename image's filename
+     * @param width    image's width
+     * @param height   image's height
+     */
+    public void saveImage (String image,String filename,String width,String height)
+    {
+	try
+	    {
+		hd.startElement("","","Title",atts);
+		hd.characters(filename.toCharArray(),0,filename.length());
+		hd.endElement("","","Title");
+		hd.startElement("","","Size",atts);
+		hd.startElement("","","Width",atts);
+		hd.characters(width.toCharArray(),0,width.length());
+		hd.endElement("","","Width");
+		hd.startElement("","","Height",atts);
+		hd.characters(height.toCharArray(),0,height.length());
+		hd.endElement("","","Height");
+		hd.endElement("","","Size");
+		hd.startElement("","","View",atts);
+		hd.characters(image.toCharArray(),0,image.length());
+		hd.endElement("","","View");
+	    }
+	catch(SAXException sax) {}
+    }
+
+    /**
+     * This method will create and setup the xml file.
+     * @param xmlname this is the file name for the xml file.
+     * @param type    this is the first tag that open the xml file.
+     */
+    public void open (String xmlname,String type)
+    {
+	try
+	    {
+		File f;
+		f = new File(xmlname);
+		fos = new FileOutputStream(f);
+		PrintWriter out;
+		out = new PrintWriter(fos);
+		StreamResult streamResult;
+		streamResult = new StreamResult(out);
+		SAXTransformerFactory tf;
+		tf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
+		hd = tf.newTransformerHandler();
+		Transformer serializer;
+		serializer = hd.getTransformer();
+		
+		serializer.setOutputProperty(OutputKeys.ENCODING,"UTF-8");
+		//serializer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM,dtdname);
+		serializer.setOutputProperty(OutputKeys.INDENT,"yes");
+			
+		hd.setResult(streamResult);
+		atts = new AttributesImpl();
+		
+		hd.startDocument();
+		
+		hd.startElement("","",type,atts);   
+	    }
+	catch (IOException e){
+	    System.err.println ("Unable to write to file");
+	    System.exit(-1);
+	}catch (TransformerConfigurationException tce){
+	    System.err.println("Error in: TransformerConfigurationException");
+	}catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+
+    /**
+     * This method will setup to close the xml file.
+     * @param type    this is the last tag that close the xml file.
+     */
+    public void close(String type)
+    {
+	try
+	    {
+		hd.endElement("","",type);
+		hd.endDocument();
+		fos.close(); 
+	    }
+	catch (IOException e){
+	    System.err.println ("Unable to write to file");
+	    System.exit(-1);
+	}catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+
+    /**
+     * This method will add the open tag according to the number.
+     * @param num the number to say which type of tag it is.
+     */
+    public void startContent (int num)
+    {
+	try
+	    {
+		switch(num)
+		    {
+		    case 0: hd.startElement("","","COMPONENT",atts);
+			break;
+		    case 1: hd.startElement("","","Type",atts);
+			break;
+		    case 2: hd.startElement("","","Title",atts);
+			break;
+		    case 3: hd.startElement("","","Content",atts);
+			break;
+		    case 4: hd.startElement("","","Selected",atts);
+			break;
+		    case 5: hd.startElement("","","Visible",atts);  
+			break;
+		    case 6: hd.startElement("","","Image",atts);
+			break;
+		    case 7: hd.startElement("","","Date",atts);
+			break;
+		    case 8: hd.startElement("","","Command",atts);
+			break;
+		    case 9: hd.startElement("","","COMPONENTS",atts);
+			break;
+		    case 10:hd.startElement("","","LOG",atts);
+			break;
+		    case 11:hd.startElement("","","Icon",atts);
+			break;
+		    case 12:hd.startElement("","","ToolTipText",atts);
+			break;
+		    case 13:hd.startElement("","","Comment",atts);
+			break;
+		    }
+	    }
+	catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x;
+	    x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+
+    /**
+     * This method will add the text to the tag.
+     * @param text is the text to be saved in xml files.
+     */
+    public void saveContent (String text)
+    {
+	try
+	    {
+		if (text == null)
+		    text = "\t";
+		hd.characters(text.toCharArray(),0,text.length());
+	    }
+	catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x;
+	    x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+
+    /**
+     * This method will add the close tag according to the number.
+     * @param num the number to say which type of tag it is.
+     */
+    public void closeContent (int num)
+    {
+	try
+	    {
+		switch(num)
+		    {
+		    case 0: hd.endElement("","","COMPONENT");
+			break;
+		    case 1: hd.endElement("","","Type");
+			break;
+		    case 2: hd.endElement("","","Title");
+			break;
+		    case 3: hd.endElement("","","Content");
+			break;
+		    case 4: hd.endElement("","","Selected");
+			break;
+		    case 5: hd.endElement("","","Visible");
+			break;
+		    case 6: hd.endElement("","","Image");
+			break;
+		    case 7: hd.endElement("","","Date");
+			break;
+		    case 8: hd.endElement("","","Command");
+			break;
+		    case 9: hd.endElement("","","COMPONENTS");
+			break;
+		    case 10:hd.endElement("","","LOG");
+			break;
+		    case 11:hd.endElement("","","Icon");
+			break;
+		    case 12:hd.endElement("","","ToolTipText");
+			break;
+		    case 13:hd.endElement("","","Comment");
+			break;
+		    }
+	    }
+	catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x;
+	    x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+
+    /**
+     * This method save to xml all the information taken when user did the Feedback form.
+     * @param err_details is the information taken when user did the Feedback form.
+     * @param img         is the images taken when user did the Feedback form
+     */ 
+    public void saveFeedback (String[] err_details,String[] img)
+    {
+	try
+	    {
+		String dirname;
+		dirname = "xmlfeedback/";
+		FileOutputStream fos;
+		fos = new FileOutputStream(dirname + "feedback" + err_details[0] + 
+					   ".xml");
+		PrintWriter out;
+		out = new PrintWriter(fos);
+		StreamResult streamResult;
+		streamResult = new StreamResult(out);
+		SAXTransformerFactory tf;
+		tf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
+		hd = tf.newTransformerHandler();
+		Transformer serializer;
+		serializer = hd.getTransformer();
+		serializer.setOutputProperty(OutputKeys.ENCODING,"UTF-8");
+		serializer.setOutputProperty(OutputKeys.INDENT,"yes");
+		hd.setResult(streamResult);
+		
+		
+		hd.startDocument();
+		atts = new AttributesImpl();
+		hd.startElement("","","BUGS",atts);
+		
+		hd.startElement("","","IDCode",atts);
+		hd.characters(err_details[0].toCharArray(),0,err_details[0].length());
+		hd.endElement("","","IDCode");
+		
+		hd.startElement("","","LastViewedWindow",atts);
+		hd.characters(err_details[1].toCharArray(),0,err_details[1].length());
+		hd.endElement("","","LastViewedWindow");
+
+		hd.startElement("","","Optionals",atts);
+		
+		hd.startElement("","","Type",atts);
+		hd.characters(err_details[3].toCharArray(),0,err_details[3].length());
+		hd.endElement("","","Type");
+		
+		hd.startElement("","","Urgency",atts);
+		hd.characters(err_details[4].toCharArray(),0,err_details[4].length());
+		hd.endElement("","","Urgency");
+		hd.endElement("","","Optionals");
+
+		hd.startElement("","","User",atts);
+		hd.startElement("","","Name",atts);
+		hd.characters(err_details[5].toCharArray(),0,err_details[5].length());
+		hd.endElement("","","Name");
+
+		hd.startElement("","","HomeDirectory",atts);
+		hd.characters(err_details[6].toCharArray(),0,err_details[6].length());
+		hd.endElement("","","HomeDirectory");
+
+		hd.startElement("","","WorkingDirectory",atts);
+		hd.characters(err_details[7].toCharArray(),0,err_details[7].length());
+		hd.endElement("","","WorkingDirectory");
+
+		hd.startElement("","","SMTP",atts);
+		hd.characters(err_details[19].toCharArray(),0,err_details[19].length());
+		hd.endElement("","","SMTP");
+
+		hd.startElement("","","Email",atts);
+		hd.characters(err_details[20].toCharArray(),0,err_details[20].length());
+		hd.endElement("","","Email");
+
+		hd.endElement("","","User");
+
+		hd.startElement("","","OpenFormTime",atts);
+		hd.characters(err_details[8].toCharArray(),0,err_details[8].length());
+		hd.endElement("","","OpenFormTime");
+
+		hd.startElement("","","SendFormTime",atts);
+		hd.characters(err_details[9].toCharArray(),0,err_details[9].length());
+		hd.endElement("","","SendFormTime");
+
+		hd.startElement("","","OperatingSystem",atts);
+		hd.characters(err_details[10].toCharArray(),0,err_details[10].length());
+		hd.endElement("","","OperatingSystem");
+
+		hd.startElement("","","Java",atts);
+
+		hd.startElement("","","JavaInformation",atts);
+		hd.characters(err_details[11].toCharArray(),0,err_details[11].length());
+		hd.endElement("","","JavaInformation");
+
+		hd.startElement("","","TotalMemory",atts);
+		hd.characters(err_details[21].toCharArray(),0,err_details[21].length());
+		hd.endElement("","","TotalMemory");
+
+		hd.startElement("","","MaxMemory",atts);
+		hd.characters(err_details[22].toCharArray(),0,err_details[22].length());
+		hd.endElement("","","MaxMemory");
+
+		hd.startElement("","","FreeMemory",atts);
+		hd.characters(err_details[23].toCharArray(),0,err_details[23].length());
+		hd.endElement("","","FreeMemory");
+
+		hd.startElement("","","DefaultLocale",atts);
+		hd.characters(err_details[12].toCharArray(),0,err_details[12].length());
+		hd.endElement("","","DefaultLocale");
+		
+		hd.endElement("","","Java");
+
+		hd.startElement("","","Browser",atts);
+		hd.characters(err_details[13].toCharArray(),0,err_details[13].length());
+		hd.endElement("","","Browser");
+
+		hd.startElement("","","LocalHostName",atts);
+		hd.characters(err_details[14].toCharArray(),0,err_details[14].length());
+		hd.endElement("","","LocalHostName");
+
+		hd.startElement("","","LocalHostAddress",atts);
+		hd.characters(err_details[15].toCharArray(),0,err_details[15].length());
+		hd.endElement("","","LocalHostAddress");
+
+		hd.startElement("","","ScreenResolution",atts);
+		hd.characters(err_details[16].toCharArray(),0,err_details[16].length());
+		hd.endElement("","","ScreenResolution");
+
+		if (img!= null)
+		    {
+			hd.startElement("","","Sequences",atts);
+			int i;
+			for ( i = 0 ; i < img.length ; i= i + 10)
+			    {
+				hd.startElement("","","Sequence",atts);
+				
+				hd.startElement("","","Title",atts);
+				hd.characters(img[i].toCharArray(),0,img[i].length());
+				hd.endElement("","","Title");
+				
+				hd.startElement("","","Details",atts);
+				hd.characters(img[i+9].toCharArray(),0,img[i+9].length());
+				hd.endElement("","","Details");
+				
+				hd.startElement("","","ImageDescription",atts);
+				
+				hd.startElement("","","ScreenShot",atts);
+				hd.startElement("","","Image",atts);
+				saveImage(img[i+1],img[i+2],img[i+7],img[i+8]);
+				hd.endElement("","","Image");
+				hd.endElement("","","ScreenShot");
+				
+				hd.startElement("","","ErrorLineAndScreenShot",atts);
+				hd.startElement("","","Image",atts);
+				saveImage(img[i+3],img[i+4],img[i+7],img[i+8]);
+				hd.endElement("","","Image");
+				hd.endElement("","","ErrorLineAndScreenShot");
+				
+				hd.startElement("","","ErrorLineForScreenShot",atts);
+				hd.startElement("","","Image",atts);
+				saveImage(img[i+5],img[i+6],img[i+7],img[i+8]); 
+				hd.endElement("","","Image");
+				hd.endElement("","","ErrorLineForScreenShot");
+				
+				hd.endElement("","","ImageDescription");
+				
+				hd.endElement("","","Sequence");
+			    }
+			hd.endElement("","","Sequences");
+		    }
+
+		hd.endElement("","","BUGS");
+		hd.endDocument();
+		fos.close();
+	    }
+	catch (IOException e){
+	    System.err.println ("Unable to write to file");
+	    System.exit(-1);
+	}catch (TransformerConfigurationException tce){
+	    System.err.println("Error in: TransformerConfigurationException");
+	}catch (SAXException spe) {
+	    // Error generated by the parser
+	    System.err.println("Error in: (SAXException");
+	    // Use the contained exception, if any
+	    Exception  x;
+	    x = spe;
+	    if (spe.getException() != null)
+		x = spe.getException();
+	    x.printStackTrace();
+	}
+    }
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ScreenShot.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ScreenShot.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ScreenShot.java	(revision 21112)
@@ -0,0 +1,273 @@
+package org.greenstone.gatherer.feedback;
+
+import java.awt.*;
+import java.applet.Applet;
+import java.awt.image.*;
+import java.awt.event.WindowEvent;
+import java.awt.event.WindowListener;
+import java.awt.event.WindowAdapter;
+import java.awt.event.*;
+import java.awt.image.BufferedImage;
+import java.awt.image.DataBuffer;
+import java.awt.geom.GeneralPath;
+import java.io.*;
+import javax.imageio.*;
+import javax.swing.*; 
+import com.sun.image.codec.jpeg.*;
+import java.awt.geom.*;
+
+/**
+ * This class can take screen shot of the whole screen size or screen shot of a specified
+ * rectangle only.
+ * @author Veronica Liesaputra
+ */
+public class ScreenShot
+{
+    /**
+     * This is the buffered image that hold the screen shot.
+     */
+    private BufferedImage img;
+    
+    /**
+     * The dimension of the whole screen size.
+     */
+    private final static Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+    
+    /**
+     * Rectangle with a dimension of a whole screen size.
+     */
+    private Rectangle mRect = new Rectangle(dim);
+    
+    /**
+     * The desired width of the image.
+     */
+    private int iw;
+
+    /**
+     * The desired height of the image.
+     */
+    private int ih;
+
+    /**
+     * BufferedImage that hold the screen shot image with the desired height and width.
+     */
+    private BufferedImage bi;
+    
+    /**
+     * This is the robot that allows to take screen shot.
+     */
+    private Robot mRobot;
+  
+    /**
+     * This constructor will setup the robot to alow taking a screen shot.
+     */
+    public ScreenShot () 
+    {
+	try
+	    {
+		mRobot = new Robot();
+		iw = (int) (dim.getWidth() - 60);
+		ih = (int) (dim.getHeight() * 0.75) - 50;
+	    }
+	catch (AWTException exp) {}
+    }
+
+    /**
+     * This method will only take screen shot of the particular rectangle.
+     * @param rect is the specified rectangle that we want to take the screen shot of.
+     * @return     the image taken.
+     */
+    //public byte[] getImageIcon (Rectangle rect)
+    public BufferedImage getImageIcon (Rectangle rect)
+    {
+	BufferedImage image;
+	image = null;
+	
+	//try
+	//  {
+		img = mRobot.createScreenCapture(rect);
+		
+		int w,h;
+		boolean rescale;
+
+		rescale = false;
+		w = rect.width;
+		
+		if (w >= iw)
+		    {
+			w = iw - 50;
+			rescale = true;
+		    }
+		
+		h = rect.height;
+		
+		if ( h >= ih)
+		    {
+			h = ih  - 50;
+			rescale = true;
+		    }
+		
+		Graphics2D big;
+		big = null;
+		
+		if (rescale == true)
+		    {
+			image = new BufferedImage(w,h, BufferedImage.TYPE_INT_RGB);
+			big = image.createGraphics();
+			big.drawImage(img.getScaledInstance(w,h,Image.SCALE_SMOOTH),0,0,null);
+		    }
+		else
+		    {
+			image = img;
+		    }
+
+		
+		/*ByteArrayOutputStream stream;
+		stream = new ByteArrayOutputStream();
+		ImageIO.write(image,"jpeg",stream);
+		byte[] bytes;
+		bytes = stream.toByteArray();*/
+		
+		if (big != null)
+		    big.dispose();
+		//image = null;
+		//img = null;
+		ActionRecorderDialog.setSavefinish(true);
+		return image;
+		//return bytes;
+		//  }
+		//catch(IOException ex) {ex.printStackTrace();}
+	
+	//return null;
+    }
+
+    /**
+     * This method will only take screen shot of the particular rectangle and encode
+     * the image to its string representation.
+     * @param rect is the specified rectangle that we want to take the screen shot of.
+     * @return     the string representation of the image taken.
+     */
+    public String getImage (Rectangle rect)
+    {
+	String txt;
+	txt = null;
+
+	BufferedImage image;
+	image = null;
+	
+	try
+	    {
+		img = mRobot.createScreenCapture(rect);
+		
+		int w,h;
+		boolean rescale;
+
+		rescale = false;
+		w = rect.width;
+		
+		if (w >= iw)
+		    {
+			w = iw - 50;
+			rescale = true;
+		    }
+		
+		h = rect.height;
+		
+		if ( h >= ih)
+		    {
+			h = ih  - 50;
+			rescale = true;
+		    }
+	
+		Graphics2D big;
+		big = null;
+
+		if (rescale == true)
+		    {
+			image = new BufferedImage(w,h, BufferedImage.TYPE_INT_RGB);
+			big = image.createGraphics();
+			big.drawImage(img.getScaledInstance(w,h,Image.SCALE_SMOOTH),0,0,null);
+		    }
+		else
+		    {
+			image = img;
+		    }
+
+		ByteArrayOutputStream stream;
+		stream = new ByteArrayOutputStream();
+		ImageIO.write(image,"gif",stream);
+		byte[] bytes;
+		bytes = stream.toByteArray();
+		txt = Base64.encodeBytes(bytes);
+		
+		bytes = null;
+		if (big != null)
+		    big.dispose();
+		image = null;
+		img = null;
+	    }
+	catch(IOException ex) {ex.printStackTrace();}
+	
+	ActionRecorderDialog.setSavefinish(true);
+	
+	return txt;
+    }
+
+    /**
+     * This method will return the buffered image of the whole screen size screen shot
+     * with the desired width and height.
+     * @return the buffered image with the desired height and width.
+     */
+    public BufferedImage getBuffImage()
+    {
+	return bi;
+    }
+
+    /**
+     * This method will take screen shot of the whole screen and resize it to the
+     * desired width and height.
+     */
+    public BufferedImage captureScreen()
+    {
+	img = mRobot.createScreenCapture(mRect);
+	
+	Graphics2D big;
+	bi = new BufferedImage(iw,ih, BufferedImage.TYPE_INT_RGB);
+	big = bi.createGraphics();
+	big.drawImage(img.getScaledInstance(iw,ih,Image.SCALE_SMOOTH),0,0,null);
+	big.dispose();
+	big = null;
+
+	return bi;
+    }
+
+    /**
+     * This method will return the desired width.
+     * @return the desired width.
+     */
+    public int getWidth()
+    {
+	return iw;
+    }
+
+    /**
+     * This method will return the desired height.
+     * @return the desired height.
+     */
+    public int getHeight()
+    {
+	return ih;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SelectPicture.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SelectPicture.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SelectPicture.java	(revision 21112)
@@ -0,0 +1,819 @@
+package org.greenstone.gatherer.feedback;
+
+import java.awt.image.*;
+import javax.swing.BoxLayout;
+import javax.swing.ImageIcon;
+import javax.swing.JLabel;
+import javax.swing.*;
+import javax.swing.event.MouseInputAdapter;
+import java.awt.*;
+import java.awt.geom.*;
+import java.awt.geom.Line2D.*;
+import java.awt.geom.Line2D.Double;
+import java.awt.event.*;
+import java.awt.event.MouseEvent;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.border.*;
+import javax.swing.BorderFactory;
+
+import org.greenstone.gatherer.util.JarTools;
+
+/**
+ * This class allows user to make scribble lines in the image.
+ * This class will display an image to a label and if the user drag inside the label
+ * then it will draw coloured line as the user drag the mouse.
+ * Image here is the screen shot of the whole screen size.
+ * @author Veronica Liesaputra
+ */
+public class SelectPicture extends WindowAdapter
+{
+    /**
+     * This is the images that stored all the images for the whole screen size image.
+     */
+    private BufferedImage[] screen;
+  
+    /**
+     * This is the images that stored all the images for the window image.
+     */
+    private BufferedImage[] window;
+
+    /**
+     * This is the buffered image that contains the line that are drawn all
+     * over the screen image.
+     */
+    private BufferedImage bi;
+    
+    /**
+     * This is the buffered image of the line that are drawn all over the screen image.
+     */
+    private BufferedImage bi2;
+
+    /**
+     * This is the buffered image of the screen image.
+     */
+    private BufferedImage bi3;
+
+    /**
+     * This is the buffered image that contains the line that are drawn all over the window image.
+     */
+    private BufferedImage bi4;
+
+    /**
+     * This is the buffered image of the line that are drawn all over the window image.
+     */
+    private BufferedImage bi5;
+
+    /**
+     * This is the buffered image of the window image.
+     */
+    private BufferedImage bi6;
+
+    /**
+     * This is the window's x-coord in the screen image.
+     */
+    private int xcoord;
+
+    /**
+     * This is the window's y-coord in the screen image.
+     */
+    private int ycoord;
+
+    /**
+     * This is the window's width in the screen image.
+     */
+    private int width;
+
+    /**
+     * This is the window's height in the screen image.
+     */
+    private int height;
+    
+    /**
+     * This is to block all the input in this window while waiting.
+     */
+    private MouseListener mouse_blocker_listener = new MouseAdapter() {};
+    
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle messages;
+    
+    /**
+     * This variable hold the comments that user entered when they are drawing lines into the image.
+     */
+    private String details;
+    
+    /**
+     * This variable is a flag to say whether or not user wants to send the feedback straight away.
+     */
+    private boolean sendNow;
+
+    /**
+     * This variable is a flag to say whether the user wants to draw the scribble lines or
+     * to erase the image.
+     */
+    private boolean drawNow;
+
+    /**
+     * This variable is a flag to say whether or not the image user wants to send is window
+     * or the whole screen image.
+     */
+    private boolean windowNow;
+
+    /**
+     * This method will make a modal-dialog that will allow user to draw a line all over the image.
+     * @param sh the buffered images of the screen image.
+     * @param wh the buffered images of the window image.
+     * @param iswindow the flag to say whether user wants to send window or whole screen image.
+     * @param screen_details the comments that user typed for this screenshot.
+     * @param msg hold the resource of the words that is stored in Messages.properties file.
+     */
+    public SelectPicture(BufferedImage[] sh,BufferedImage[] wh,boolean iswindow,String screen_details,ResourceBundle msg)
+    {
+	windowNow = iswindow;
+	drawNow = true;
+	details = screen_details;
+	messages = msg;
+	window = wh;
+	screen = sh;
+    }
+    
+    /**
+     * This method setting up the window's bounds inside the screen image.
+     * @param x the window's x-coordinate in the screen image.
+     * @param y the window's y-coordniate in the screen image.
+     * @param w the window's width in the screen image.
+     * @param h the window's height in the screen image.
+     */
+    public void setWindowBounds (int x,int y,int w,int h)
+    {
+	xcoord = x;
+	ycoord = y;
+	width = w;
+	height = h;
+	makeNewWindow();
+    }
+
+    /**
+     * This method will get which one user wants to send the window or screen image.
+     * @return the flag whether user wants to send window or screen image. 
+     */
+    public boolean getIsWindow()
+    {
+	return windowNow;
+    }
+
+    /**
+     * This method will get the screen image of the screenshot with the line drawn all over it.
+     * @return  this is the screen image with the line drawn all over it.
+     */
+    public BufferedImage getImage()
+    {
+	return bi;
+    }
+
+    /**
+     * This method will get the screen image of the screenshot.
+     * @return  this is the screen image.
+     */
+    public BufferedImage getImage3()
+    {
+	return bi5;
+    }
+
+    /**
+     * This method will get the screen image of only the line that drawn all over the image of
+     * the screenshot.
+     * @return  this is the screen image of only the line.
+     */
+    public BufferedImage getImage2()
+    {
+	return bi2;
+    }
+
+    /**
+     * This method will get the window image of the screenshot with the line drawn all over it.
+     * @return  this is the image with the line drawn all over it.
+     */
+    public BufferedImage getWindowImage()
+    {
+	return bi3;
+    }
+
+    /**
+     * This method will get the window image of the screenshot.
+     * @return  this is the window image.
+     */
+    public BufferedImage getWindowImage3()
+    {
+	return bi6;
+    }
+
+    /**
+     * This method will get the window image of only the line that drawn all over the image of
+     * the screenshot.
+     * @return  this is the window image of only the line.
+     */
+    public BufferedImage getWindowImage2()
+    {
+	return bi4;
+    }
+
+    /**
+     * This method will give the comments user made when they are drawing line over the picture.
+     * @return this is the comment user made.
+     */
+    public String getDetails ()
+    {
+	return details;
+    }
+
+    public boolean getSendNow()
+    {
+	return sendNow;
+    }
+
+    /**
+     * This method will make a modal-dialog that will have the image of the screen shot in the label,
+     * allow user to draw a line all over the image and comments it as well. 
+     */
+    public void makeNewWindow()
+    {
+	JDialog f;
+	f = new Selecting(this);
+	f.setLocation(0,0);
+    }
+
+    /**
+     * This class will make a modal-dialog that will have the image of the screen shot in the label,
+     * allow user to draw a line all over the image and comments it as well. 
+     */
+    class Selecting extends JDialog implements ActionListener
+    {
+	/**
+	 * This is the modal-dialog.
+	 */
+	private JDialog frame;
+	
+	/**
+	 * This is the special label that allows user to draw a line all over the icon set in the label.
+	 */
+	private SelectionArea area;
+	
+	/**
+	 * This is the text area where user can enter the comments they want to add.
+	 */
+	private JTextArea problem_details;
+	
+	/**
+	 * This is the owner of this modal-dialog.
+	 */
+	private SelectPicture framework = null;
+	
+	/**
+	 * This is the graphs that will allow to draw the line all over the screen image in the label and getting
+	 * the image of it.
+	 */
+	private Graphs gp;
+
+	/**
+	 * This is the graphs that will allow to draw the line all over the window image in the label and getting
+	 * the image of it.
+	 */
+	private Graphs gp2;
+
+	/**
+	 * This constructor will make  a modal-dialog that will have the image of the screen shot in the label,
+	 * allow user to draw a line all over the image and comments it as well. 
+	 * @param controller this is the owner of the modal-dialog.
+	 */
+	public Selecting(SelectPicture controller)
+	{
+	    Container contentPane;
+   
+	    contentPane = getContentPane();
+
+	    gp = new Graphs(screen);
+	    
+	    window = gp.getWindowVersion(xcoord,ycoord,width,height);
+	  
+	    gp2 = new Graphs(window);
+
+	    buildUI(contentPane);
+	
+	    framework = controller;
+	   
+	    setTitle(messages.getString("Showwhereistheproblem"));
+	    setDefaultLookAndFeelDecorated(true);
+	    setModal(true);
+
+	    frame = this;
+	    addWindowListener(framework);
+	    setBackground(new Color(176,208,176)); 
+	    
+	    setLocation(0,0);
+	  
+	    pack();
+	    setVisible(true);
+	}
+
+	/**
+	 * This method will setup the contentpane of the modal-dialog.It will set the cursor for
+	 * the label to be from the file pen.gif and set the image icon for the label.
+	 * @param content this is the contentPane of the modal-dialog.
+	 */
+	private void buildUI(Container content) 
+	{
+	    final JPanel container;
+	    container = new JPanel();
+	    container.setLayout(new BoxLayout(container, BoxLayout.PAGE_AXIS));
+	    container.setBackground(new Color(176,208,176));
+	    container.setBorder(new EmptyBorder(5,5,5,5));
+
+	   
+	    ImageIcon image = createImageIcon(windowNow);
+	    area = new SelectionArea(image, this);
+	    area.setBackground(new Color(224,240,224));
+	    area.setBorder(BorderFactory.createCompoundBorder
+			   (BorderFactory.createLineBorder(Color.blue,4),
+			    new EmptyBorder(10,10,10,10)));
+	   
+	    ImageIcon imgicon,imgicon2;
+	    Image img,img2;
+	    imgicon = JarTools.getImage("pen.gif");
+	    img = imgicon.getImage();
+	    img = img.getScaledInstance(24,24,Image.SCALE_SMOOTH);
+	    imgicon2 = JarTools.getImage("eraserpen.gif");
+	    img2 = imgicon2.getImage();
+	    final Cursor cursor;
+	    final Cursor cursor2;
+	    cursor = Toolkit.getDefaultToolkit().createCustomCursor(img,new Point(0,0),"crayon");
+	    cursor2 = Toolkit.getDefaultToolkit().createCustomCursor(img2,new Point(0,0),"eraser");
+	    
+	    area.setCursor(cursor);
+
+	    Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+	    final JPanel iconarea = new JPanel();
+	    iconarea.setBackground(new Color(176,208,176));
+	    iconarea.add(area);
+	    container.add(iconarea);
+	    
+	    String screen_label;
+	    screen_label = " " + messages.getString("ScreenShotLabel");
+	    JLabel label;
+	    label = new JLabel(screen_label);
+	    label.setBackground(new Color(176,208,176));
+	    container.add(label);
+
+	    JPanel panes;
+	    panes = new JPanel();
+	    panes.setLayout(new GridLayout(0,2));
+	    panes.setBackground(new Color(176,208,176));
+
+	    problem_details = new JTextArea(details);
+	    problem_details.setWrapStyleWord(true);
+	    problem_details.setEditable(true);
+	    problem_details.setBackground(new Color(224,240,224));
+	    JScrollPane scroll;
+	    scroll = new JScrollPane(problem_details);
+	    scroll.setBackground(new Color(176,208,176));
+	    scroll.setPreferredSize(new Dimension(50,100));
+	    scroll.setBorder(BorderFactory.createTitledBorder(
+							      new EmptyBorder(0,0,0,0),
+							      "Screen shot " + 
+							      messages.getString("ProblemDetails")));
+
+	    panes.add(scroll);
+
+	    JPanel pane;
+	    pane = new JPanel();
+	    pane.setLayout(new GridLayout(2,3));
+	    pane.setBackground(new Color(176,208,176));
+	    pane.setBorder(new EmptyBorder(15,15,15,15));
+
+	    if ((window == null) || (screen == null)) {System.out.println("How come!");}
+	    else
+		{
+		    final JButton toggle;
+		    
+		    if (windowNow == false)
+			{
+			    toggle = new JButton("GLI window."); 
+			    toggle.setToolTipText("Showing only GLI window.");
+			}
+		    else
+			{
+			    toggle = new JButton("Whole screen.");
+			    toggle.setToolTipText("Showing whole screen.");
+			}
+		    toggle.setActionCommand("Toggle");
+		    toggle.addActionListener(new ActionListener ()
+			{
+			    public void actionPerformed (ActionEvent e)
+			    {
+				frame.addMouseListener(mouse_blocker_listener);
+				frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+				if (windowNow == true)
+				    {
+					windowNow = false;
+					toggle.setText("GLI window.");
+					toggle.setToolTipText("Showing only GLI window.");
+					window[0] = gp2.getScreenImage();
+					window[1] = gp2.getImage();
+					window[2] = gp2.getImage2();
+					gp.setScreenVersion(window,xcoord,ycoord,width,height);
+					area.setIcon(new ImageIcon(gp.getImage()));
+				    }
+				else
+				    {
+					windowNow = true;
+					toggle.setText("Whole screen.");
+					toggle.setToolTipText("Showing whole screen.");
+					
+					window[0].flush();
+					window[0] = null;
+					window[1].flush();
+					window[1] = null;
+					window[2].flush();
+					window[2] = null;
+					window = null;
+					BufferedImage tmp;
+					gp2.getImage().flush();
+					tmp = gp2.getImage();
+					tmp.flush();
+					tmp = null;
+					gp2.getImage2().flush();
+					tmp = gp2.getImage2();
+					tmp.flush();
+					tmp = null;
+					gp2.flush();
+					gp2 = null;
+					System.gc();
+
+					window = gp.getWindowVersion(xcoord,ycoord,width,height);
+					gp2 = new Graphs(window);
+					area.setIcon(new ImageIcon(gp2.getImage()));
+				    }
+
+				frame.pack();
+				frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+				frame.removeMouseListener(mouse_blocker_listener);	
+			    }
+			});
+		    toggle.setBackground(new Color(176,208,176));
+		    pane.add(toggle);
+		}
+				     
+
+	    JButton pen;
+	    pen = new JButton(JarTools.getImage("penicon.gif"));
+	    pen.setText("Pen");
+	    pen.setPreferredSize(new Dimension(30,20));
+	    pen.setActionCommand("Pen");
+	    pen.setToolTipText("Draw.");
+	    pen.addActionListener(new ActionListener ()
+		{
+		    public void actionPerformed (ActionEvent e)
+		    {
+			area.setCursor(cursor);
+			drawNow = true;
+		    }
+		});
+	    pen.setBackground(new Color(176,208,176));
+	    pane.add(pen);
+
+	    JButton eraser;
+	    eraser = new JButton(JarTools.getImage("eraser.gif"));
+	    eraser.setText("Eraser");
+	    eraser.setActionCommand("Eraser");
+	    eraser.setPreferredSize(new Dimension(30,20));
+	    eraser.setToolTipText("Erase.");
+	    eraser.addActionListener(new ActionListener ()
+		{
+		    public void actionPerformed (ActionEvent e)
+		    {
+			area.setCursor(cursor2);
+			drawNow = false;
+		    }
+		});
+	    eraser.setBackground(new Color(176,208,176));
+	    pane.add(eraser);
+
+	    if ((window == null) || (screen == null)) 
+		{
+		    JLabel empty = new JLabel(" ");
+		    empty.setBackground(new Color(176,208,176));
+		    pane.add(empty);
+		}
+
+	    JButton button;
+	    button = new JButton("Clear Scribble");
+	    button.setActionCommand("Clear");
+	    button.setToolTipText("Clear all the scribble lines.");
+	    button.addActionListener(this);
+	    button.setBackground(new Color(176,208,176));
+	    pane.add(button);
+
+	    JButton button2;
+	    button2 = new JButton("Finish Scribble");
+	    button2.setActionCommand(messages.getString("Send"));
+	    button2.setToolTipText("Back to Feedback form.");
+	    button2.addActionListener(this);
+	    button2.setBackground(new Color(176,208,176));
+	    pane.add(button2);
+
+	    JButton button3;
+	    button3 = new JButton("Send");
+	    button3.setActionCommand("SendNow");
+	    button3.setToolTipText("Finish reporting feedback and send all information now.");
+	    button3.addActionListener(this);
+	    button3.setBackground(new Color(176,208,176));
+	    pane.add(button3);
+
+	    panes.add(pane);
+
+	    container.add(panes);
+	    
+	    panes.setAlignmentX(Component.LEFT_ALIGNMENT);
+	    label.setAlignmentX(Component.LEFT_ALIGNMENT);
+	    iconarea.setAlignmentX(Component.LEFT_ALIGNMENT);
+	   
+	    container.setSize(container.getPreferredSize());
+	    content.add(container);
+	}
+
+	/**
+	 * This method will tell what the program should do when user click the buttons.
+	 * @param e this is the event when user click button.
+	 */
+	public void actionPerformed (ActionEvent e)
+	{
+	    if(messages.getString("Send").equals(e.getActionCommand()))
+		{
+		    frame.addMouseListener(mouse_blocker_listener);
+		    frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+		    sendNow = false;
+		    if (windowNow == false)
+			{
+			    window = gp.getWindowVersion(xcoord,ycoord,width,height);
+			}
+		    else
+			{
+			    window[0] = gp2.getScreenImage();
+			    window[1] = gp2.getImage();
+			    window[2] = gp2.getImage2();
+			    gp.setScreenVersion(window,xcoord,ycoord,width,height);
+			}
+ 
+		    bi5 = gp.getScreenImage();
+		    bi = gp.getImage();
+		    bi2 = gp.getImage2();
+		    bi3 = window[1];
+		    bi4 = window[2];
+		    bi6 = window[0];
+
+		    details = problem_details.getText();
+		    frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+		    frame.removeMouseListener(mouse_blocker_listener);	
+		    dispose();
+		}
+	    if ("Clear".equals(e.getActionCommand()))
+		{
+		    if (windowNow == false)
+			{
+			    gp.reset();
+			    area.setIcon(new ImageIcon(gp.getScreenImage()));
+			}
+		    else
+			{
+			    gp2.reset();
+			    area.setIcon(new ImageIcon(gp2.getScreenImage()));
+			}
+		}
+
+	    if ("SendNow".equals(e.getActionCommand()))
+		{
+		    frame.addMouseListener(mouse_blocker_listener);
+		    frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+		    sendNow = true;
+
+		    if (windowNow == false)
+			{
+			    window = gp.getWindowVersion(xcoord,ycoord,width,height);
+			}
+		    else
+			{
+			    window[0] = gp2.getScreenImage();
+			    window[1] = gp2.getImage();
+			    window[2] = gp2.getImage2();
+			    gp.setScreenVersion(window,xcoord,ycoord,width,height);
+			}
+ 
+		    bi5 = gp.getScreenImage();
+		    bi = gp.getImage();
+		    bi2 = gp.getImage2();
+		    bi3 = window[1];
+		    bi4 = window[2];
+		    bi6 = window[0];
+
+		    details = problem_details.getText();
+		    frame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+		    frame.removeMouseListener(mouse_blocker_listener);	
+		    dispose();
+		}
+	}
+
+	/**
+	 * This method will create ImageIcon of the buffered image get from the screen shot instance.
+	 * The image is screen shot whole screen size image.
+	 * @return ImageIcon of the screen shot image.
+	 */
+	protected ImageIcon createImageIcon(boolean iswindow)
+	{
+	    if (iswindow == false)
+		{
+		    if (screen[1] == null)
+			return new ImageIcon(screen[0]);
+		    else
+			return new ImageIcon(screen[1]);
+		}
+	    else
+		{
+		    if (window[1] == null)
+			return new ImageIcon(window[0]);
+		    else
+			return new ImageIcon(window[1]);
+		}
+	}
+	
+	/**
+	 * This is the special class, that will allows user to draw a line all over the label when 
+	 * user dragged the mouse inside the area.
+	 */
+      	private class SelectionArea extends JLabel 
+	{
+	    /**
+	     * This is the owner of this label.
+	     */
+	    private Selecting controller;
+	    /**
+	     * This is the x coordinate of the first point.
+	     */
+	    private int x1;
+	    /**
+	     * This is the x coordinate of the second point.
+	     */
+	    private int x2;
+	    /**
+	     * This is the y coordinate of the first point.
+	     */
+	    private int y1;
+	    /**
+	     * This is the y coordinate of the second point.
+	     */
+	    private int y2;
+	    /**
+	     * This is the flag to say that user is over the label of the screen image 
+	     * and they are doing the mouse input from there.
+	     */
+	    private boolean ismouse;
+
+	    /**
+	     * This method will create a label with the specified image 
+	     * icon and it will added with mouse listener that will
+	     * allow user to draw a line all over the image.
+	     * @param image image icon to be added to the label.
+	     * @param controller this is the owner of this label.
+	     */
+	    public SelectionArea(ImageIcon image, Selecting controller) 
+	    {
+		super(image);
+		this.controller = controller;
+		setOpaque(true);
+		
+		MyListener myListener;
+		myListener = new MyListener();
+		addMouseListener(myListener);
+		addMouseMotionListener(myListener);
+	    } 
+	    
+	    /**
+	     * This is the modified mouse listener that will allow user to draw
+	     * a line all over the label while user dragging the mouse.
+	     */
+	    private class MyListener extends MouseInputAdapter 
+	    {
+		/**
+		 * This method will make a red dot in the place where user pressed the mouse.
+		 * @param e the mouse event
+		 */
+		public void mousePressed(MouseEvent e) 
+		{
+		    int x;
+		    x = e.getX();
+		    int y;
+		    y = e.getY();
+		
+		    x2 = x;
+		    y2 = y;
+		    
+		    x1 = x2;
+		    y1 = y2;
+		}
+
+		/**
+		 * This method will make a red line from (x1,y1) to the coordinate where
+		 * the mouse is located now.
+		 * @param e the mouse event.
+		 */
+		public void mouseDragged(MouseEvent e) 
+		{
+		    ismouse = true;
+		    updateSize(e);
+		}
+
+		
+		/**
+		 * This method will make a red line from (x1,y1) to the coordinate where
+		 * the mouse is located now.
+		 * @param e the mouse event.
+		 */
+		public void updateSize(MouseEvent e) 
+		{
+		    int x;
+		    x = e.getX();
+		    int y;
+		    y = e.getY();
+		
+		    x2 = x;
+		    y2 = y;
+	
+		    repaint();    
+		}
+	    }
+	    
+	    /**
+	     * This method will repaint all the component inside the dialog-window.
+	     * It will draw a red line from (x1,y1) to (x2,y2) coordinates and
+	     * updates the value of x1,y1 and the icon for the label. 
+	     * @param g the graphics of this dialog-window.
+	     */
+	    public void paintComponent(Graphics g) 
+	    {
+		super.paintComponent(g);
+
+		if (ismouse == true)
+		    {
+			if (windowNow == false)
+			    {
+				if (drawNow == true)
+				    gp.drawLines(x1,y1,x2,y2);
+				else
+				    gp.eraseLines(x1,y1,x2,y2);
+			    }
+			else
+			    {
+				if (drawNow == true)
+				    gp2.drawLines(x1,y1,x2,y2);
+				else
+				    gp2.eraseLines(x1,y1,x2,y2);
+			    }
+			
+			x1 = x2;
+			y1 = y2;
+
+			if (windowNow == false)
+			    {
+				area.setIcon(new ImageIcon(gp.getImage()));
+			    }
+			else
+			    {
+				area.setIcon(new ImageIcon(gp2.getImage()));
+			    }
+			
+			ismouse = false;
+		    }
+	    }
+	}
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendHTTP.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendHTTP.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendHTTP.java	(revision 21112)
@@ -0,0 +1,421 @@
+package org.greenstone.gatherer.feedback;
+
+import java.util.Properties;
+import java.net.*;
+import java.io.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.*;
+import java.awt.Dimension;
+
+/**
+ * This class will send the 2 jar files to the server using HTTP Post.
+ */
+public class SendHTTP
+{
+    /**
+     * This is variable will hold the arguments provided by user in order to send 
+     */
+    private String[] argument;
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle msg;
+    /**
+     * This variable will hold the jar file unique code.
+     * For example: Jar123File.jar then file_code = 123
+     */
+    private String file_code;
+    /**
+     * This is the flag that will say whether or not the sending is successful.
+     * If failed = false means its succesful otherwise its failed.
+     */
+    private boolean failed = false;
+
+    /**
+     * Crates instance of SendHTTP
+     */
+    public SendHTTP () {}
+  
+    /**
+     * This method will send 2 files back to the server.
+     * It will also send  the code if the sending is failed then it will give user a message saying whats
+     * the possible error of it, and it will make the jar file changed into
+     * Unsend_Jar123File.jar file.
+     * If the sending is succesful then it will try to send all the other jar files
+     * that were failed to send before.And then it will deletes all the jar files
+     * that are send successfully.
+     * @param args the argument provided to send the email.
+     * @param messages hold the resource of the words that is stored in Messages.properties file.
+     * @param code the code of the email to be send.
+     */
+    public void  sendMail(String[] args,ResourceBundle messages,String code) 
+    {
+	argument = args;
+	msg = messages;
+	file_code = code;
+
+	try
+	    {
+		String BND = "---------------------------7d021a37605f0";
+		String hostURL =  "http://kanuka.cs.waikato.ac.nz:8090/glifeedback/Upload";
+
+		URL mURL = new URL(hostURL+"?name=posting");
+		URLConnection conn = mURL.openConnection();
+		
+		if (conn instanceof HttpURLConnection)
+		    ((HttpURLConnection) conn).setRequestMethod("POST");
+		 
+		conn.setDoInput(true);
+		conn.setDoOutput(true);
+		conn.setUseCaches(false);
+		conn.setDefaultUseCaches(false);
+		conn.setRequestProperty("Accept","*/*");
+		conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + BND);
+		conn.setRequestProperty("Connection","Keep-Alive");
+		conn.setRequestProperty("Cache-Control","no-cache");
+
+		DataOutputStream out;
+		out = new DataOutputStream(conn.getOutputStream());
+
+		//--- Sending subject -------
+		String txt;
+		txt = "--" + BND + "\r\n";
+		out.writeBytes(txt);
+
+		txt = "Content-Disposition: form-data; ";
+		out.writeBytes(txt);
+
+		String subject;
+		subject = "Send Feedback" + code;
+		txt = "name=\"" + "Subject" + "\"\r\n"; 
+		out.writeBytes(txt);
+		
+		txt = "\r\n";
+		out.writeBytes(txt);
+
+		txt = subject + "\r\n";
+		out.writeBytes(txt);
+		
+		out.flush();
+
+		//--- Sending text ------
+		txt = "--" + BND + "\r\n";
+		out.writeBytes(txt);
+
+		txt = "Content-Disposition: form-data; ";
+		out.writeBytes(txt);
+
+		String attmnt;
+		attmnt = "We have succesfully received your feedback report.\n" +
+		    "To check what files we have received go to this link: " + hostURL +
+		    "/?name=" + code + "\n" + 
+		    "Please keep referencing to this ID Code for this feedback report.\n" +
+		    "ID Code: " + code;
+
+		txt = "name=\"" + "Text" + "\"\r\n";
+		out.writeBytes(txt);
+		
+		txt = "\r\n";
+		out.writeBytes(txt);
+		
+		txt = attmnt + "\r\n";
+		out.writeBytes(txt);
+
+		out.flush();
+
+		//--------- Sending xml-jar file
+		FileInputStream fis;
+		File f;
+		String name,path;
+		
+		f = new File(args[1]);
+		
+		txt = "--" + BND + "\r\n";
+		out.writeBytes(txt);
+
+		txt = "Content-Disposition: form-data; ";
+		out.writeBytes(txt);
+		
+		name = f.getName();
+		txt = "name=\"" + name + "\"; ";
+		out.writeBytes(txt);
+		
+		path = f.getAbsolutePath();
+		txt = "filename=\"" + path + "\"\r\n";
+		out.writeBytes(txt);
+
+		txt = "Content-Type: aplication/octet-stream\r\n";
+		out.writeBytes(txt);
+
+		txt = "\r\n";
+		out.writeBytes(txt);
+
+		fis = new FileInputStream(f);
+		byte[] data = new byte[1024];		
+		int r = 0;
+		while((r = fis.read(data, 0, data.length)) != -1) 
+		    {			
+			out.write(data, 0, r);		
+		    }
+		fis.close();
+
+		txt = "\r\n";
+		out.writeBytes(txt);
+
+		out.flush();
+
+		//--- Sending parser.jar ----
+		txt = "--" + BND + "\r\n";
+		out.writeBytes(txt);
+		
+		txt = "Content-Disposition: form-data; ";
+		out.writeBytes(txt);
+		
+		f = new File(args[4]);
+		
+		name = f.getName();
+		txt = "name=\"" + name + "\"; ";
+		out.writeBytes(txt);
+		
+		path = f.getAbsolutePath();
+		txt = "filename=\"" + path + "\"\r\n";
+		out.writeBytes(txt);
+
+		txt = "Content-Type: application/octet-stream\r\n";
+		out.writeBytes(txt);
+
+		txt = "\r\n";
+		out.writeBytes(txt);
+
+		fis = new FileInputStream(f);
+	        data = new byte[1024];		
+		r = 0;
+		while((r = fis.read(data, 0, data.length)) != -1) 
+		    {			
+			out.write(data, 0, r);		
+		    }
+		fis.close();
+		
+		txt = "\r\n";
+		out.writeBytes(txt);
+
+		out.flush();
+
+		//----- finished sending -----
+		txt = "--" + BND + "--";
+		out.writeBytes(txt);
+
+		out.flush();
+		out.close();
+
+		//----- getting the response ------
+		BufferedReader reader;
+		reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+		String responseLine;
+		responseLine = null;
+
+		StringBuffer response;
+		response = new StringBuffer();
+		while ((responseLine = reader.readLine()) != null)
+		    response.append(responseLine+"\n");
+
+		reader.close();
+	 
+
+		//------ our application's response to server's response
+		/*if (response.equals("failed"))
+		    {
+			failed = true;
+		
+			JOptionPane.showMessageDialog(null,messages.getString("InternalError")+ "\n" + 
+						      messages.getString("Emaillater"),messages.getString("Error"),
+						      JOptionPane.ERROR_MESSAGE);
+			//Conformation.setFinish(true);
+		    }
+		else if (response.equals("success"))
+		    failed = false;
+		else
+		{*/
+			failed = false;
+			JDialog.setDefaultLookAndFeelDecorated(false);
+			JDialog dlg = new JDialog();
+			JLabel lbl = new JLabel (response.toString());
+			
+			dlg.getContentPane().add(lbl);
+			dlg.pack();
+			dlg.setVisible(true);
+			// }
+
+		//Conformation.setFinish(true);
+	    }
+	catch (MalformedURLException exp) 
+	    {
+		failed = true;
+		
+		JOptionPane.showMessageDialog(null,"malformed URL" + "\n" + 
+					      messages.getString("Emaillater"),messages.getString("Error"),
+					      JOptionPane.ERROR_MESSAGE); 
+		System.out.println("malformed URL"); 
+	    }
+	catch (IOException e) 
+	    { 
+		failed = true;
+		
+		JOptionPane.showMessageDialog(null,messages.getString("InternalError")+ "\n" + 
+					      messages.getString("Emaillater"),messages.getString("Error"),
+					      JOptionPane.ERROR_MESSAGE);e.printStackTrace();
+	    }
+
+	String dirname;
+	dirname = "xmlfeedback/";
+	
+	if (failed == false)
+	    {
+		File f;
+		f = new File(dirname + "Jar" + file_code + "File.jar");
+		f.delete();
+		
+		File file;
+		file = new File(dirname + ".");
+		File[] fileList;
+		fileList = file.listFiles();
+		
+		int y;
+		for (y = 0 ; y < fileList.length ; y++)
+		    {
+			String filename;
+			filename = isRightJarFile(fileList[y]);
+			if (filename != null)
+			    {
+				renameTo(fileList[y],new File(filename));
+				argument[1] = filename;
+				sendMail(argument,msg,getCode(filename));
+			    }
+		    }
+	    }
+	else
+	    {
+		File f2;
+		f2 = new File(dirname + "Jar" + file_code + "File.jar");
+		renameTo(f2,new File(dirname + "Unsend_Jar" + file_code + "File.jar"));
+	    }
+	
+    }
+    
+    /**
+     * This method will rename file.
+     * @param src is the name of the file to be renamed.
+     * @param dest is the desired file name.
+     */
+    private void renameTo (File src, File dest)
+    {
+	if (src.renameTo(dest) == false)
+	    {
+		try
+		    {
+			FileInputStream in;
+			in = new FileInputStream(src);
+			FileOutputStream out;
+			out = new FileOutputStream(dest);
+			int next;
+			
+			while (true)
+			    {
+				next = in.read();
+				if (next == -1)
+				    break;
+				else
+				    out.write((byte) next);
+			    }
+			
+			in.close();
+			out.close();
+		    }
+		catch (IOException exp) {exp.printStackTrace();}
+	    }
+    }
+
+    /**
+     * This method will get the extension of the file.
+     * @param f is the file that we want to get the extension.
+     * @return  the file extension
+     */
+    private String getExtension(File f)
+    {
+	String ext;
+	ext = null;
+	String s;
+	s = f.getName();
+	int i;
+	i = s.lastIndexOf('.');
+	
+	if (i > 0 &&  i < s.length() - 1)
+	    {
+		ext = s.substring(i+1).toLowerCase();
+	    }
+	return ext;
+    }
+    
+    /**
+     * This method will get all the name of the file after "_" if the file its a jar file.
+     * For example Unsend_Jar123File.jar then this method will return Jar123File, otherwise
+     * file with any other format will return null.
+     * @param f is the file that we want to get the name.
+     */
+    private String isRightJarFile(File f)
+    {
+	String ext;
+	ext = null;
+	
+	if (f.isDirectory() == true)
+	    return null;
+	
+	if (getExtension(f).compareTo("jar") == 0)
+	    {
+		String s;
+		s = f.getName();
+		int i;
+		i = s.lastIndexOf("_");
+
+		if (i > 0 &&  i < s.length() - 1)
+		    {
+			ext = s.substring(i+1,s.length());
+		    }
+		else
+		    ext = null;
+		
+		return ext;
+	    }
+	else
+	    return null;
+    }
+    
+    /**
+     * This method will getting the code that inside the Jar filename.
+     * For example Jar123File then this method will return 123. 
+     */
+    private String getCode (String s)
+	{
+	    int i;
+	    i = s.lastIndexOf("Jar");
+	    int j;
+	    j = s.indexOf("File");
+	    String code;
+	    code = s.substring((i+3),j);
+	    return code;
+	}
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendMail.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendMail.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SendMail.java	(revision 21112)
@@ -0,0 +1,388 @@
+/** This class is not currently used and therefore will not be compiled by makegli. */
+
+package org.greenstone.gatherer.feedback;
+
+import java.util.Properties;
+import javax.mail.*;
+import javax.mail.internet.*;
+import javax.activation.*;
+import java.io.*;
+import java.util.Locale;
+import java.util.ResourceBundle;
+import java.text.MessageFormat;
+import javax.swing.*;
+
+/**
+ * This is class will provides the ability to send email that contains 2 attachment file.
+ * The arguments that user need to provide are email address of the recepient,first file
+ * to be attached, SMTP address of sender email address, email adress of the sender and
+ * second file to be attached and the code. It has to be in this order.
+ * Here we will expect the 2 file to be attached were parser.jar and jar file of all the
+ * 6 xml files made in the Reporting Feedback sequence.
+ * <br>The email send will have subject = Send Feedback followed with the code.
+ * <br>The message will be : 
+ * <br>Hereby attached 2 jar files.Please download them at the same place and parser.jar is an executable jar file.
+ * <br>Please keep referencing to this ID Code.
+ * <br>ID Code: code
+ * This email will only can be send to one recepient only.
+ * @author Veronica Liesaputra
+ */
+public class SendMail 
+{
+    /**
+     * This is variable will hold the arguments provided by user in order to send 
+     * 2 attachment files through email.
+     */
+    private String[] argument;
+    /**
+     * This variable will hold the resource of the words that is stored in Messages.properties file.
+     * The calling using messages.getString(someString) will caused someString to be translated
+     * into some other string that is hold in that file.usually it will caused it to be translated
+     * to the language that the user use or choose to have as stated in Locale.
+     */
+    private static ResourceBundle msg;
+    /**
+     * This variable will hold the jar file unique code.
+     * For example: Jar123File.jar then file_code = 123
+     */
+    private String file_code;
+    /**
+     * This is the flag that will say whether or not the sending is successful.
+     * If failed = false means its succesful otherwise its failed.
+     */
+    private boolean failed = false;
+
+    /**
+     * Crates instance of SendMail
+     */
+    public SendMail () {}
+  
+    /**
+     * This method will be send an email with 2 file attachment to a recepient
+     * email address using the to email address and to smpt adress as given in
+     * the arguments. This email will also have the code as specified code. 
+     * If the sending is failed then it will give user a message saying whats
+     * the possible error of it, and it will make the jar file changed into
+     * Unsend_Jar123File.jar file.
+     * If the sending is succesful then it will try to send all the other jar files
+     * that were failed to send before.And then it will deletes all the jar files
+     * that are send successfully.
+     * @param args the argument provided to send the email.
+     * @param messages hold the resource of the words that is stored in Messages.properties file.
+     * @param code the code of the email to be send.
+     */
+    public void  sendMail(String[] args,ResourceBundle messages,String code) 
+    {
+	
+	argument = args;
+	msg = messages;
+	file_code = code;
+
+	if(args.length<5) 
+	    {
+		System.out.println("Args for example:");
+		System.out.println("String[] args = new String(5);");
+		System.out.println("args[0] = " + "recepient@maildomain.com;");
+		System.out.println("args[1] = " + "Jar123.jar;");
+		System.out.println("args[2] = " + "mail.waikato.ac.nz;");
+		System.out.println("args[3] = " + "to@cs.waikato.ac.nz;");
+		System.out.println("args[4] = " + "parser.jar;");
+		return;
+	    }
+	
+	File attachment;
+	attachment =new File(args[1]);
+	
+	if(attachment.isDirectory()) 
+	    {
+		System.out.println("Attachment should not be a directory.\n It should be a file.");
+		return;
+	    }
+	if(!attachment.exists()) 
+	    {
+		System.out.println("Attachment file not found.");
+		return;
+	    }
+	
+	Properties p;
+	p = System.getProperties();
+	
+	p.put("mail.smtp.host",args[2]);
+	
+	Session session;
+	session =Session.getInstance(p);
+	
+	Message mail;
+	mail =new MimeMessage(session);
+	
+	Transport t;
+
+	try 
+	    {
+     
+		Address [] addresses;
+		addresses = new Address[]{new InternetAddress(args[0])};
+	    
+		mail.setFrom(new InternetAddress(args[3]));
+	    
+		mail.setSubject("Send Feedback" + code);
+	    
+		MimeBodyPart part1;
+		part1 =new MimeBodyPart();
+		String attmnt;
+		attmnt = "Hereby attached 2 jar files.Please download them at the same place and parser.jar " +
+		    "is an executable jar file.\n" + "Please keep referencing to this ID Code.\n" +
+		    "ID Code:";
+		part1.setText(attmnt+" " + code);
+			
+		MimeBodyPart part2;
+		part2 =new MimeBodyPart();				
+		FileDataSource source;
+		source =new FileDataSource(args[1]);
+		part2.setDataHandler(new DataHandler(source));
+		part2.setFileName(source.getName());
+	    
+		MimeBodyPart part3;
+		part3 = new MimeBodyPart();
+		FileDataSource source2;
+		source2 = new FileDataSource(args[4]);
+		part3.setDataHandler(new DataHandler(source2));
+		part3.setFileName(source2.getName());
+
+		Multipart part;
+		part =new MimeMultipart();
+	    
+		part.addBodyPart(part1);
+		part.addBodyPart(part2);
+		part.addBodyPart(part3);
+	    
+		mail.setContent(part);	
+	    
+		t = session.getTransport(addresses[0]);
+	    
+		t.addTransportListener(new javax.mail.event.TransportAdapter()
+		    {
+			public void messageNotDelivered(javax.mail.event.TransportEvent te) 
+			{
+			    Address a[];
+			    a =te.getInvalidAddresses();
+			    failed = true;
+			    System.out.println("Message not delivered for the following addresses:");
+			    int i;
+			    for(i=0;i<a.length;i++)
+				{
+				    System.out.println(a[i]);
+				}
+			}
+		    });
+		
+		t.connect();
+	  
+	   
+		t.sendMessage(mail,addresses);
+	    
+		
+		t.close();
+		
+	    }
+	catch (NoSuchProviderException e) 
+	    {
+		failed = true;
+		System.out.println("Provider not found.");
+		String s;
+		s = (String)JOptionPane.showInputDialog(messages.getString("NotFoundProvider"));
+		
+		if ((s != null) && (s.length() > 0))
+		    {
+			argument[2] = s;
+			
+			String s2;
+			s2 = (String)JOptionPane.showInputDialog(messages.getString("NotFoundProvider2"));
+			
+			if ((s2 != null) && (s2.length() > 0))
+			    argument[3] = s2;
+
+			sendMail(argument,msg,file_code);
+			//Conformation.setFinish(true);
+			return;
+		    }
+		else
+		    JOptionPane.showMessageDialog(null,messages.getString("Emaillater"),messages.getString("Error"),
+						  JOptionPane.INFORMATION_MESSAGE);
+		
+	    }
+	catch (SendFailedException nspe) 
+	    {
+		failed = true;
+		int ans;
+		ans = JOptionPane.showConfirmDialog(null,messages.getString("SendFailed"),
+							messages.getString("Error"),JOptionPane.YES_NO_OPTION,
+							JOptionPane.INFORMATION_MESSAGE);
+		if (ans == JOptionPane.YES_OPTION)
+		    {
+			sendMail(argument,msg,file_code);
+			//Conformation.setFinish(true);
+			return;
+		    }
+		else
+		    JOptionPane.showMessageDialog(null,messages.getString("Emaillater"),messages.getString("Error"),
+						  JOptionPane.INFORMATION_MESSAGE);
+	    }
+	catch (Exception nspe) 
+	    {
+		failed = true;
+		JOptionPane.showMessageDialog(null,messages.getString("InternalError")+ "\n" + 
+					      messages.getString("Emaillater"),messages.getString("Error"),
+					      JOptionPane.ERROR_MESSAGE);
+		//Conformation.setFinish(true);
+	    }
+
+	String dirname;
+	dirname = "xmlfeedback/";
+
+	if (failed == false)
+	    {
+		File f;
+		f = new File(dirname + "Jar" + file_code + "File.jar");
+		f.delete();
+
+		File file;
+		file = new File(dirname + ".");
+		File[] fileList;
+		fileList = file.listFiles();
+		
+		int y;
+		for (y = 0 ; y < fileList.length ; y++)
+		    {
+			String filename;
+			filename = isRightJarFile(fileList[y]);
+			if (filename != null)
+			    {
+				renameTo(fileList[y],new File(filename));
+				argument[1] = filename;
+				sendMail(argument,msg,getCode(filename));
+			    }
+		    }
+	    }
+	else
+	    {
+		File f2;
+		f2 = new File(dirname + "Jar" + file_code + "File.jar");
+		renameTo(f2,new File(dirname + "Unsend_Jar" + file_code + "File.jar"));
+	    }
+
+	//Conformation.setFinish(true);
+    }
+
+    /**
+     * This method will rename file.
+     * @param src is the name of the file to be renamed.
+     * @param dest is the desired file name.
+     */
+    private void renameTo (File src, File dest)
+    {
+	if (src.renameTo(dest) == false)
+	    {
+		try
+		    {
+			FileInputStream in;
+			in = new FileInputStream(src);
+			FileOutputStream out;
+			out = new FileOutputStream(dest);
+			int next;
+			
+			while (true)
+			    {
+				next = in.read();
+				if (next == -1)
+				    break;
+				else
+				    out.write((byte) next);
+			    }
+			
+			in.close();
+			out.close();
+		    }
+		catch (IOException exp) {exp.printStackTrace();}
+	    }
+    }
+
+    /**
+     * This method will get the extension of the file.
+     * @param f is the file that we want to get the extension.
+     * @return  the file extension
+     */
+    private String getExtension(File f)
+    {
+	String ext;
+	ext = null;
+	String s;
+	s = f.getName();
+	int i;
+	i = s.lastIndexOf('.');
+	
+	if (i > 0 &&  i < s.length() - 1)
+	    {
+		ext = s.substring(i+1).toLowerCase();
+	    }
+	return ext;
+    }
+    
+    /**
+     * This method will get all the name of the file after "_" if the file its a jar file.
+     * For example Unsend_Jar123File.jar then this method will return Jar123File, otherwise
+     * file with any other format will return null.
+     * @param f is the file that we want to get the name.
+     */
+    private String isRightJarFile(File f)
+    {
+	String ext;
+	ext = null;
+	
+	if (f.isDirectory() == true)
+	    return null;
+	
+	if (getExtension(f).compareTo("jar") == 0)
+	    {
+		String s;
+		s = f.getName();
+		int i;
+		i = s.lastIndexOf("_");
+
+		if (i > 0 &&  i < s.length() - 1)
+		    {
+			ext = s.substring(i+1,s.length());
+		    }
+		else
+		    ext = null;
+		
+		return ext;
+	    }
+	else
+	    return null;
+    }
+    
+    /**
+     * This method will getting the code that inside the Jar filename.
+     * For example Jar123File then this method will return 123. 
+     */
+    private String getCode (String s)
+	{
+	    int i;
+	    i = s.lastIndexOf("Jar");
+	    int j;
+	    j = s.indexOf("File");
+	    String code;
+	    code = s.substring((i+3),j);
+	    return code;
+	}
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SeparateGraphs.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SeparateGraphs.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/SeparateGraphs.java	(revision 21112)
@@ -0,0 +1,105 @@
+/** This class is not currently used and therefore will not be compiled by makegli. */
+
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import javax.imageio.*;
+import java.awt.*;
+import java.awt.image.*;
+import com.sun.image.codec.jpeg.*;
+import javax.swing.*;
+
+/**
+ * This class will separate a image file into 2 jpeg image files by taking
+ * all the point in that image file that has the color red and save all that
+ * points to another image file, while all the other point to another image
+ * file.
+ * @author Veronica Liesaputra
+ */
+public class SeparateGraphs
+{
+    /**
+     * This is the BufferedImage that we want to separate.
+     */
+    private BufferedImage bi;
+
+    /**
+     * This constructor will save the image inside the file with the specified 
+     * filename to the BufferedImage.
+     */
+    public SeparateGraphs(String filename)
+    {
+	try
+	    {
+		File f = new File(filename);
+		bi = ImageIO.read(f);
+	    }
+	catch(IOException exp) {}
+    }
+
+    /**
+     * This method will separate bi into 2 image and save it to
+     * jpeg files with the specified filename.
+     * The 2 image separated will be, 1 image is the image of all the points in bi that
+     * have the color red and another image is the image of all other points in bi that
+     * do not have the color red.
+     * @param file1 this is the filename for the image of all other points that do not have color red.
+     * @param file2 this is the filename for the image of all points that have the color red.
+     */
+    public void separate(String file1,String file2)
+    {
+	BufferedImage bi1,bi2;
+	
+	int width = bi.getWidth();
+	int height = bi.getHeight();
+	
+	bi1 = new BufferedImage(width,height,BufferedImage.TYPE_INT_RGB);
+	bi2 = new BufferedImage(width,height,BufferedImage.TYPE_INT_ARGB);
+
+	for (int y = 0 ; y < height ; y++) 
+	    {
+		for (int x = 0 ; x < width ; x++)
+		    {
+			int rgb = bi.getRGB(x,y);
+			Color c = new Color(rgb,true);
+			if (c != Color.RED)
+			    {
+				bi1.setRGB(x,y,rgb);
+			    }
+			else
+			    {
+				bi2.setRGB(x,y,rgb);
+			    }
+		    }
+	    }
+
+	save(bi1,file1);
+	save(bi2,file2);
+    }
+
+    /**
+     * This method will save the BufferedImage into a jpeg file with the specified filenames.
+     * @param b is the BufferedImage to be saved.
+     * @param filename is the name of the jpeg files.
+     */
+    public void save(BufferedImage b,String filename)
+    {
+	try 
+	    {
+		File file = new File(filename);
+		FileOutputStream out = new FileOutputStream(file);
+	    
+		JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(out);
+		JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(b);
+		param.setQuality(1.0f, false);
+		encoder.setJPEGEncodeParam(param);
+		encoder.encode(b);
+		
+	    } catch (Exception ex) {}
+    }
+}
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Separating.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Separating.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/Separating.java	(revision 21112)
@@ -0,0 +1,108 @@
+/** This class is not currently used and therefore will not be compiled by makegli. */
+
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import javax.imageio.*;
+import java.awt.*;
+import java.awt.image.*;
+import com.sun.image.codec.jpeg.*;
+import javax.swing.*;
+
+/**
+ * This method will separate 1 image file into 2 image  with the help of
+ * another file and sace the difference into a jpeg files.
+ * For example : We have 1 image file which is a screen shot and another image file
+ *               is the screenshot with some line in it.
+ *               This class will separate the screen shot and the line into 2 separates 
+ *               images.And then it will save the line into a jpeg files.
+ * 
+ * @author Veronica Liesaputra
+ */
+public class Separating
+{
+    /**
+     * This is the BufferedImage of image inside the reference image file.
+     */
+    private BufferedImage bi;
+    /**
+     * This is the BufferedImage of image that we want to separate into
+     * 2 different image using the reference file.
+     */
+    private BufferedImage src_bi;
+
+    /**
+     * It will read whats inside the 2 given filenames and put it in the
+     * right BufferedImage.
+     * @param filename1 this is the file name of the reference image file.
+     * @param filename2 this is the file name of the file we want to separate into 2.
+     */
+    public Separating(String filename1,String filename2)
+    {
+	try
+	    {
+		File f1 = new File(filename1);
+		File f2 = new File(filename2);
+		bi = ImageIO.read(f1);
+		src_bi = ImageIO.read(f2);
+	    }
+	catch(IOException exp) {}
+    }
+
+    /**
+     * This method will get the difference that exist between bi and src_bi and
+     * will save the difference inside a jpeg file with the specified file name.
+     * @param file1 is the name of the file where we want to save the image.
+     */
+    public void separate(String file1)
+    {
+	BufferedImage bi1;
+	
+	int width = bi.getWidth();
+	int height = bi.getHeight();
+	
+	bi1 = new BufferedImage(width,height,BufferedImage.TYPE_INT_RGB);
+
+	for (int y = 0 ; y < height ; y++) 
+	    {
+		for (int x = 0 ; x < width ; x++)
+		    {
+			int rgb = bi.getRGB(x,y);
+			int src_rgb = src_bi.getRGB(x,y);
+			
+			if (src_rgb != rgb)
+			    {
+				bi1.setRGB(x,y,rgb);
+			    }
+		    }
+	    }
+
+	save(bi1,file1);
+    }
+
+    /**
+     * This method will save a BufferedImage to a jpeg file with the specified filename.
+     * @param b is the BufferedImage to be saved.
+     * @param filename is the name of the jpeg file.
+     */
+    public void save(BufferedImage b,String filename)
+    {
+	try 
+	    {
+		File file = new File(filename);
+		FileOutputStream out = new FileOutputStream(file);
+	    
+		JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(out);
+		JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(b);
+		param.setQuality(1.0f, false);
+		encoder.setJPEGEncodeParam(param);
+		encoder.encode(b);
+		
+	    } catch (Exception ex) {}
+    }
+}
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/UserComponent.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/UserComponent.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/UserComponent.java	(revision 21112)
@@ -0,0 +1,439 @@
+package org.greenstone.gatherer.feedback;
+
+import java.util.*;
+import java.awt.image.*;
+import java.io.*;
+import javax.imageio.*;
+import javax.swing.*;
+import java.awt.*;
+
+/**
+ * This class willl hold the information about a component and the contents inside the component.
+ * @author Veronica Liesaputra
+ */
+public class UserComponent implements Serializable
+{
+    /**
+     * This variable will hold what is the type of instance this component is.
+     */
+    private String type = null;
+
+    /**
+     * This variable will store the title of this component.
+     */
+    private String title = null;
+
+    /**
+     * This varaible will store the array list of the contents of this component.
+     */
+    private ArrayList content = null;
+
+    /**
+     * This variable will store the status of this component, whether or not
+     * this component is selected by the user.
+     */
+    private String selected = null;
+
+    /**
+     * This variable will store the status of this component, whether or not
+     * this component is visible.
+     */
+    private String visible = null;
+
+    /**
+     * This variable will hold the descrption text saying that there is no
+     * image in this component.
+     */
+    private String image = null;
+
+    /**
+     * This is the supposed filename of the image that is in this component should use if 
+     * we want to save it to a jpeg file.
+     */
+    private String img = null;
+
+    /**
+     * This is the string representation of the image that is in this component.
+     */
+    private String imgFile = null;
+
+    /**
+     * This is the index that will help storing the data to the correct data member.
+     */
+    private int index;
+
+    /**
+     * This is the height of the image that is in this component.
+     */
+    private String height;
+
+    /**
+     * This is the width of the image that is in this component.
+     */
+    private String width;
+
+    /**
+     * This variable will hold the tooltiptext set for this component.
+     */
+    private String tooltip;
+
+    /**
+     * This will initialized content to a new empty Arrray list.
+     */
+    public UserComponent()
+    {
+	content = new ArrayList();
+    }
+
+    /**
+     * This method will get the width of the image that is inside this component.
+     * @return image's width.
+     */
+    public String getWidth()
+    {
+	return width;
+    }
+
+    /**
+     * This method will get the height of the image that is inside this component.
+     * @return image's height.
+     */
+    public String getHeight()
+    {
+	return height;
+    }
+
+    /**
+     * This method will get the type of instance this component is.
+     * @return component's type.
+     */
+    public String getType()
+    {
+	return type;
+    }
+
+    /**
+     * This method will get the tooltiptext set for this component.
+     * @return component's tooltiptext.
+     */
+    public String getToolTip()
+    {
+	return tooltip;
+    }
+
+    /**
+     * This method will get the title for this component.
+     * @return component's title.
+     */
+    public String getTitle()
+    {
+	return title;
+    }
+
+    /**
+     * This method will get the content inside this component.
+     * @return component's content.
+     */
+    public ArrayList getContent()
+    {
+	return content;
+    }
+
+    /**
+     * This method will get whether or not this component is 
+     * selected by the user.
+     * @return component selected or not.
+     */
+    public String getSelected()
+    {
+	return selected;
+    }
+
+    /**
+     * This method will get whether or not this component is 
+     * visible.
+     * @return component visible or not.
+     */
+    public String getVisible()
+    {
+	return visible;
+    }
+
+    /**
+     * This method will get the descrption text saying that there is no
+     * image in this component.
+     * @return the description text.
+     */
+    public String getImage()
+    {
+	return image;
+    }
+
+    /**
+     * This method will get the supposed filename of the image that is in this component should use if 
+     * we want to save it to a jpeg file.
+     * @return the image file name.
+     */
+    public String getImageFileName()
+    {
+	return img;
+    }
+
+    /**
+     * This method will get the string representation of the image that is in this component.
+     * @return the image.
+     */
+    public String getImageFile()
+    {
+	return imgFile;
+    }
+
+    /**
+     * This method will set the index value.
+     * (Precondition: (num > 0))
+     * @param num the new index value.
+     */
+    public void startContent (int num)
+    {
+	index = num;
+    }
+    
+    /**
+     * This method will add the UserComponent to the content array list.
+     * @param comp the content of the component.
+     */
+    public void saveContent (UserComponent comp)
+    {
+	content.add(comp);
+    }
+
+    /**
+     * This method will set the data member value to the text passed
+     * to this method depending on the index value.
+     * @param text the value to be set to the data member.
+     */
+    public void saveContent (String text)
+    {
+	if (text == null)
+	    text = " ";
+
+	switch (index)
+	    {
+	    case 1 : type = text; break;
+	    case 2 : title = text; break;
+	    case 3 : content.add(text); break;
+	    case 4 : selected = text;break;
+            case 5 : visible = text;break;
+	    case 6 : image = text; break;
+	    case 7 : tooltip = text;break;
+            }
+    }
+
+    /**
+     * This method will encode the ImageIcon into its String representation using Based64encoder and
+     * it will also set the image's filename,height and width.
+     * @param image is the image to be converted.
+     * @param filename is the file name of the image.
+     */
+    public void saveImage (ImageIcon image,String filename) 
+    {
+	/*int w;
+	w = image.getIconWidth();
+	int h;
+	h = image.getIconHeight();
+
+	height = "" + h;
+	width = "" + w;
+	img = filename;
+	
+	if (w > 50)
+	    w = w - 50;
+	
+	if (h > 50)
+	    h = h - 50;
+	
+	ImageIcon icon;
+	icon = new ImageIcon(image.getImage().getScaledInstance(w,h,Image.SCALE_SMOOTH));
+   
+	if ( w < 0 || h < 0)
+	    {
+		System.out.println(img);
+		return;
+	    }
+
+	BufferedImage bi;
+	bi =new BufferedImage(w,h, BufferedImage.TYPE_INT_RGB);
+	Graphics2D big;
+	big = bi.createGraphics();
+	
+	big.drawImage(icon.getImage(),0,0,null);
+	
+	try	
+	    {
+		ByteArrayOutputStream stream;
+		stream = new ByteArrayOutputStream();
+		ImageIO.write(bi,"jpeg",stream);
+		byte[] bytes;
+		bytes = stream.toByteArray();
+		imgFile = Base64.encodeBytes(bytes);
+	    }
+	catch (IOException exp) {}
+	
+	big.dispose();
+	image = null;
+	bi = null;
+	icon = null;*/
+    }
+
+    /**
+     * This method will encode the Icon into its String representation using Based64encoder and
+     * it will also set the image's filename,height and width.
+     * @param image is the image to be converted.
+     * @param filename is the file name of the image
+     */
+    public void saveImage (Icon image,String filename)
+    {
+	if (image == null)
+	    System.out.println("null image??");
+	else if (image instanceof ImageIcon)
+	    saveImage((ImageIcon)image, filename);
+	else
+	    System.out.println("Its a icon");
+	    //saveIcon (image,filename);
+    }
+
+    /**
+     * This method will encode the Icon into its String representation using Based64encoder and
+     * it will also set the image's filename,height and width.
+     * @param image is the image to be converted.
+     * @param filename is the file name of the image.
+     */
+    private void saveIcon (Icon image,String filename)
+    {
+	JFrame lbl = new JFrame();
+	lbl.setVisible(false);
+
+	ImageIcon icon;
+	Image img;
+	img = iconToImage(image,lbl);
+	icon = new ImageIcon(img);
+	lbl.dispose();
+	lbl = null;
+	saveImage(icon,filename);
+    }
+
+    /**
+     * This method will save to content of this UserComponent to an xml file.
+     * The xml file should already have an opening tag and closing tag first before calling
+     * this method or it wont make a valid xml file.
+     * @param sx this is the xml file where we want to save it.
+     */
+    public void sendingXML (SaveToXML sx)
+    {
+	if (type != null)
+	    {
+		sx.startContent(1);
+		sx.saveContent(type);
+		sx.closeContent(1);
+	    }
+
+	if (title != null)
+	    {
+		sx.startContent(2);
+		sx.saveContent(title);
+		sx.closeContent(2);
+	    }
+
+	if (content != null)
+	    {
+		sx.startContent(3);
+		for (int j = 0 ; j < content.size() ; j++)
+		    {
+			if (content.get(j) instanceof String)
+			    sx.saveContent((String) content.get(j));
+			else
+			    {
+				sx.startContent(0);
+				UserComponent cmp;
+				cmp = (UserComponent) content.get(j);
+				cmp.sendingXML(sx);
+				sx.closeContent(0);
+			    }
+		    }
+		sx.closeContent(3);
+	    }
+
+	if (selected != null)
+	    {
+		sx.startContent(4);
+		sx.saveContent(selected);
+		sx.closeContent(4);
+	    }
+
+	if (visible != null)
+	    {
+		sx.startContent(5);
+		sx.saveContent(visible);
+		sx.closeContent(5);
+	    }
+
+	if (img != null)
+	    {
+		sx.startContent(6);
+		sx.saveImage(imgFile,img,width,height);
+		sx.closeContent(6);
+	    }
+	else
+	    {
+		if (image != null)
+		    {
+			sx.startContent(11);
+			sx.saveContent(image);
+			sx.closeContent(11);
+		    }
+	    }
+
+	if (tooltip != null)
+	    {
+		sx.startContent(12);
+		sx.saveContent(tooltip);
+		sx.closeContent(12);
+	    }
+    }
+
+    /**
+     * This method will convert an Icon to a BufferedImage.
+     * @param icon icon to be converted.
+     * @param targetComponent is the component where we want to draw this BufferedImage.
+     * @return is the BufferedImage of the icon.
+     */
+    public BufferedImage iconToImage(Icon icon, Component targetComponent) 
+    {
+	int w,h;
+	w = icon.getIconWidth();
+	h = icon.getIconHeight();
+	GraphicsConfiguration gc;
+	gc = targetComponent.getGraphicsConfiguration();
+	BufferedImage image;
+	image = gc.createCompatibleImage(w, h,Transparency.TRANSLUCENT);
+	
+	Graphics2D g;
+	g = image.createGraphics();
+	icon.paintIcon(targetComponent,g,0,0);
+	g.dispose();
+	return image;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ZipFile.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ZipFile.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/feedback/ZipFile.java	(revision 21112)
@@ -0,0 +1,266 @@
+package org.greenstone.gatherer.feedback;
+
+import java.io.*;
+import java.util.jar.*;
+import java.util.zip.*;
+
+/**
+ * This class will help to zip and unzip files.
+ * @author Veronica Liesaputra
+ */
+public class ZipFile 
+{
+    /**
+     * This is the specified buffer size.
+     */
+    private static final int BUFFER = 2048;
+    
+    public ZipFile ()
+    {}
+
+    /**
+     * This method will unzip the MyZipFile.zip file and delete the file.
+     */
+    public void unZipFile ()
+    {
+	try
+	    {
+		BufferedOutputStream dest;
+		dest = null;
+		FileInputStream fis;
+		fis = new FileInputStream("MyZipFile.zip");
+		ZipInputStream zis;
+		zis = new ZipInputStream(new BufferedInputStream(fis));
+		ZipEntry entry;
+		while((entry = zis.getNextEntry()) != null) 
+		    {
+			int count;
+			byte data[];
+			data = new byte[BUFFER];
+			// write the files to the disk
+			File f;
+			f = new File(entry.getName());
+			if (isRightLogExtension(f) == true)
+			    {
+				FileOutputStream fos;
+				fos = new FileOutputStream(entry.getName());
+				dest = new BufferedOutputStream(fos, BUFFER);
+				while ((count = zis.read(data, 0, BUFFER)) != -1) 
+				    {
+					dest.write(data, 0, count);
+				    }
+				dest.flush();
+				dest.close();
+			    }
+		    }
+		zis.close();
+		File f3;
+		f3 = new File("MyZipFile.zip");
+		f3.delete();
+	    } 
+	catch (FileNotFoundException fexp) {}
+	catch(Exception e) 
+	    {
+		e.printStackTrace();
+	    }
+    } 
+
+    /**
+     * This method will zip all the xml files into one Jar file.
+     */
+    public void sendZipXMLFile ()
+     {
+	try 
+	    {
+		String dirname;
+		dirname = "xmlfeedback/";
+		BufferedInputStream origin;
+		origin = null;
+		FileOutputStream dest;
+		dest = new FileOutputStream(dirname + "Jar" + 
+					    FeedbackInterface.getCode() + 
+					    "File.jar");
+		JarOutputStream out;
+		out = new JarOutputStream(new BufferedOutputStream(dest));
+		out.setMethod(ZipOutputStream.DEFLATED);
+		byte data[];
+		data = new byte[BUFFER];
+		
+		File f;
+		f = new File(dirname + ".");
+		File[] fileList;
+		fileList = f.listFiles();
+		File[] files;
+		files = new File[fileList.length + 8];
+		
+		int index,j,i;
+		index = 0;
+		for (j = 0 ; j < fileList.length ; j++)
+		    {
+			String name;
+			name = fileList[j].getName();
+			
+			if (isRightXMLExtension(fileList[j]) == true)
+			    {
+				files[index] = fileList[j];
+				index++;
+			    }
+		    }
+		
+		for ( i = 0; i < index; i++) 
+		    {
+			FileInputStream fi;
+			fi = new FileInputStream(files[i]);
+			origin = new BufferedInputStream(fi, BUFFER);
+			ZipEntry entry;
+			entry = new ZipEntry(files[i].getName());
+			out.putNextEntry(entry);
+			int count;
+			while((count = origin.read(data, 0,BUFFER)) != -1) 
+			    {
+				out.write(data, 0, count);
+			    }
+			origin.close();
+		    }	
+
+		out.close();
+	    } 
+	catch(Exception e) 
+	    {
+		e.printStackTrace();
+	    }
+    }
+
+    /**
+     * This method will zip all .log file to MyZipFile.zip file and it will
+     * delete the history.log file.
+     */
+    public void sendZipFile ()
+    {
+	try 
+	    {
+		BufferedInputStream origin;
+		origin = null;
+		FileOutputStream dest;
+		dest = new FileOutputStream("MyZipFile.zip");
+		ZipOutputStream out;
+		out = new ZipOutputStream(new BufferedOutputStream(dest));
+		out.setMethod(ZipOutputStream.DEFLATED);
+		byte data[];
+		data = new byte[BUFFER];
+		
+		File f;
+		f = new File(".");
+		File[] fileList;
+		fileList = f.listFiles();
+		File[] files;
+		files = new File[fileList.length];
+		
+		int index,i,j;
+		index = 0;
+		for ( j = 0 ; j < fileList.length ; j++)
+		    {
+			if (isRightLogExtension(fileList[j]) == true)
+			    {
+				files[index] = fileList[j];
+				index++;
+			    }
+		    }
+		
+		for (i = 0; i < index; i++) 
+		    {
+			FileInputStream fi;
+			fi = new FileInputStream(files[i]);
+			origin = new BufferedInputStream(fi, BUFFER);
+			ZipEntry entry;
+			entry = new ZipEntry(files[i].getName());
+			out.putNextEntry(entry);
+			int count;
+			while((count = origin.read(data, 0,BUFFER)) != -1) 
+			    {
+				out.write(data, 0, count);
+			    }
+			origin.close();
+		    }
+		out.close();
+		File f2;
+		f2 = new File("history.log");
+		f2.delete();
+	    } 
+	catch(Exception e) 
+	    {
+		e.printStackTrace();
+	    }
+    }
+    
+    /**
+     * This method will get the extension of the given file.
+     * @param f the file we want to know the extension.
+     * @return  the file extension.
+     */
+    public String getExtension(File f)
+    {
+        String ext;
+	ext = null;
+        String s;
+	s = f.getName();
+        int i;
+	i = s.lastIndexOf('.');
+	
+        if (i > 0 &&  i < s.length() - 1)
+	    {
+		ext = s.substring(i+1).toLowerCase();
+	    }
+        return ext;
+    }
+   
+    /**
+     * This method will check whether or not the give file has
+     * an .xml extension.
+     * @param f the file we want to check the extension.
+     * @return  whether or not the file has xml as its file extension.
+     */
+    public boolean isRightXMLExtension(File f)
+    {
+	if (f.isDirectory() == true)
+	    {
+		return false;
+	    }
+	String ext;
+	ext = getExtension(f);
+	if ((ext != null) && (ext.compareTo("xml") == 0))
+	    return true;
+	else
+	    return false;
+    }
+
+    /**
+     * This method will check whether or not the give file has
+     * an .log extension.
+     * @param f the file we want to check the extension.
+     * @return  whether or not the file has log as its file extension.
+     */
+    public boolean isRightLogExtension(File f)
+    {
+	if (f.isDirectory() == true)
+	    return false;
+	
+	String ext;
+	ext = getExtension(f);
+	
+	if ((ext != null) && (ext.compareTo("log") == 0))
+	    return true;
+	else
+	    return false;
+    }
+}
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAlreadyExistsException.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAlreadyExistsException.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAlreadyExistsException.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gatherer.file;
+
+import java.lang.Exception;
+
+public class FileAlreadyExistsException
+    extends Exception {
+
+    public FileAlreadyExistsException() {
+	super();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAssociationManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAssociationManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileAssociationManager.java	(revision 21112)
@@ -0,0 +1,333 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import java.io.*;
+import java.util.regex.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.gui.FileAssociationDialog;
+import org.greenstone.gatherer.gui.PreviewCommandDialog;
+import org.greenstone.gatherer.util.ExternalProgram;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+public class FileAssociationManager 
+    extends AbstractTableModel {
+    static final public String FILENAME_ARG = "%1";
+    static final private String ESCAPE = "\\\\"; // '\'
+    static final private String ESCAPED_ESCAPE = "\\\\\\\\"; // '\\'
+    static final private String SPACE = " ";
+    static final private String ESCAPED_SPACE = "\\\\ ";
+    private Element associations_element;
+    private File data_file;   
+
+    public FileAssociationManager() {
+	// Retrieve the associations_element from the config
+	associations_element = Configuration.getFileAssociations();
+	// Initialize the associations. This involves looking through all current associations searching for those with a command of "".
+	if(associations_element != null) {
+	    NodeList entries = associations_element.getElementsByTagName(StaticStrings.ENTRY_ELEMENT);
+	    for(int i = 0; i < entries.getLength(); i++) {
+		Element entry = (Element) entries.item(i);
+		String command = XMLTools.getValue(entry);
+		// If we encounter a command of ""...
+		if(command.length() == 0) {
+		    // if we are on windows, we default to the start command
+		    if(Utility.isWindows()) {
+			if (Utility.isWindows9x()) {
+			    XMLTools.setValue(entry, StaticStrings.WIN_9X_OPEN_COMMAND);
+			} else {
+			    XMLTools.setValue(entry, StaticStrings.WIN_OPEN_COMMAND);
+			}
+		    }
+		    // and if we are on mac, we default to the open program
+		    else if(Utility.isMac()) {
+			XMLTools.setValue(entry, StaticStrings.MAC_OPEN_COMMAND);
+		    }
+		}
+		command = null;
+		entry = null;
+	    }
+	    entries = null;
+	}
+	else {
+	    DebugStream.println("Didn't parse anything. About to crash.");
+	}
+    }
+    
+    public void edit() {
+	FileAssociationDialog dialog = new FileAssociationDialog(this);
+	dialog.display(null);
+	dialog = null;		  
+    }
+
+    public String getBrowserCommand(String url) {
+	DebugStream.println("Get browser command: " + url);
+	// First off we try to retrieve one from the configuration
+	String command = Configuration.getPreviewCommand();
+	// If that worked, substitute in the url and return
+	if(command != null && command.length() > 0) {
+	    command = command.replaceAll("%1", url);
+	    DebugStream.println("Result = " + command);
+	    return command;
+	}
+	command = null;
+	// Failing that we have a guess at a sensible default 
+	if(Utility.isWindows()) {
+	    // we use cmd and start
+	    if (Utility.isWindows9x()) {
+		command = StaticStrings.WIN_9X_OPEN_COMMAND;//"command.com /c start \""+url+"\"";
+	    } else {
+		command = StaticStrings.WIN_OPEN_COMMAND;//"cmd.exe /c start \"\" \""+url+"\"";
+	    }
+	} else if (Utility.isMac()) {
+	    command = StaticStrings.MAC_OPEN_COMMAND; // "open %1"
+	} else {
+	    // we try to look for a browser
+	    String [] browsers = new String [] {"mozilla", "netscape"};
+	    for (int i=0; i<browsers.length; i++) {
+		if (isAvailable(browsers[i])) {
+		    command = browsers[i]+ " %1";
+		    break;
+		}
+	    }
+	}
+
+	// if we still haven't found something, prompt the user
+	if (command == null) {
+	    PreviewCommandDialog dialog = new PreviewCommandDialog();
+	    command = dialog.display();
+	    dialog.dispose();
+	    dialog = null;
+	}
+
+	// Store the result if any
+	if(command != null && !command.equals("")) {
+	    Configuration.setPreviewCommand(command);
+	    command = command.replaceAll(FILENAME_ARG, url);
+	    DebugStream.println("Result = " + command);
+	    return command;
+	}
+	// if we haven't got a command by now, we'll never get one
+	DebugStream.println("Result = null");
+	return null;
+	
+    }
+
+    public int getColumnCount() {
+	return 2;
+    }
+
+    public String getColumnName(int column) {
+	String name;
+	switch(column) {
+	case 0:
+	    name = Dictionary.get("FileAssociationDialog.Table.Extension");
+	    break;
+	default:
+	    name = Dictionary.get("FileAssociationDialog.Table.Command");
+	}
+	return name;
+    }
+   
+    public String [] getCommand(File file) {
+	String command = null;
+	String [] commands = null;
+	if(file.isFile()) {
+	    // Determine extension
+	    String filename = file.getAbsolutePath();
+	    String extension = filename.substring(filename.lastIndexOf(".") + 1);
+	    // Try to retrieve a value from cache
+	    Element entry = getCommand(extension);
+	    if(entry != null) {
+		///ystem.err.println("Retrieved Value From Cache");
+		command = XMLTools.getValue(entry); 
+	    }
+	    if(command == null || command.length() == 0) {
+		///ystem.err.println("No Existing Command");
+		// If command is null, and we are on windows try searching the registry.
+		if(Utility.isWindows()) {
+		    //try the start command
+		    if (Utility.isWindows9x()) {
+			command = StaticStrings.WIN_9X_OPEN_COMMAND;
+		    } else {
+			command = StaticStrings.WIN_OPEN_COMMAND;
+		    }
+		    
+		}
+		
+		// If we are on a mac, default to using the open program
+		else if(Utility.isMac()) {
+		    command = StaticStrings.MAC_OPEN_COMMAND;
+		}
+
+		// Otherwise display the dialog and ask the user to enter launching command.
+		if(command == null || command.length() == 0) {
+		    ///ystem.err.println("Show Dialog");
+		    // Show the dialog which forces a user to select the launch command for a certain file.
+		    FileAssociationDialog dialog = new FileAssociationDialog(this);
+		    command = dialog.display(extension);
+		    dialog = null;
+		}
+
+		// Hopefully by now we have a command, or else we're never going to get one. Add the association.
+		if (command != null && !command.equals("")) {
+		    // If no previous entry existed create one.
+		    if(entry == null) {
+			entry = associations_element.getOwnerDocument().createElement(StaticStrings.ENTRY_ELEMENT);
+			entry.setAttribute(StaticStrings.EXTENSION_ATTRIBUTE, extension);
+			associations_element.appendChild(entry);
+		    }
+		    // Replace the text in this node. Remember to replace the dummy filename with %1 - I dont think the filename will ever be in the comand now
+		    //XMLTools.setValue(entry, command.replaceAll(filename, FILENAME_ARG));
+		    XMLTools.setValue(entry, command);
+		}
+	    }
+
+	    if (command != null && !command.equals("")) {
+		// Make the command into a string []		
+		commands = command.split(" ");
+
+		// Now substitute any occurrences of %1 with its filename
+		// Note this is done after the split on spaces to avoid
+		// any conflict with filenames with spaces in them.
+
+		// We have to fix filename under windows to escape the backslashes
+		filename = filename.replaceAll(ESCAPE, ESCAPED_ESCAPE);
+
+		for (int i=0; i<commands.length; i++) {
+		    // Replace %1 with the appropriate filename
+		    commands[i] = commands[i].replaceAll(FILENAME_ARG, filename);
+		}
+	    }
+	    
+	    entry = null;
+	    extension = null;
+	    filename = null;
+	}
+	return commands;
+    }
+
+    public Element getCommand(String target_extension) {
+	NodeList entries = associations_element.getElementsByTagName(StaticStrings.ENTRY_ELEMENT);
+	for(int i = 0; i < entries.getLength(); i++) { 
+	    Element entry = (Element) entries.item(i);
+	    String extension = entry.getAttribute(StaticStrings.EXTENSION_ATTRIBUTE);
+	    if(extension.equalsIgnoreCase(target_extension)) {
+		entries = null;
+		extension = null;
+		return entry;
+	    }
+	}
+	entries = null;
+	return null;
+    }
+
+    public String getCommandString(String target_extension) {
+	Element entry = getCommand(target_extension);
+	if(entry != null) {
+	    return XMLTools.getValue(entry);
+	}
+	else {
+	    return "";
+	}
+    }
+
+    public String getExtension(int index) {
+	NodeList entries = associations_element.getElementsByTagName(StaticStrings.ENTRY_ELEMENT);
+	if(0 <= index && index < entries.getLength()) {
+	    Element entry = (Element) entries.item(index);
+	    return entry.getAttribute(StaticStrings.EXTENSION_ATTRIBUTE);
+	}
+	return "";
+    }
+
+    public int getRowCount() {
+	return size();
+    }
+
+    public Object getValueAt(int row, int column) {
+	String extension = getExtension(row);
+	switch(column) {
+	case 0:
+	    return extension;
+	default:
+	    return getCommandString(extension);
+	}
+    }
+
+    public void save() {
+    }
+
+    public void setCommand(String extension, String command) {
+	DebugStream.println("Set Launch: " + extension + " with " + command);
+	// Retrieve any existing entry for this extension
+	Element entry = getCommand(extension);
+	// If no previous entry existed create one.
+	if(entry == null && command != null) {
+	    entry = associations_element.getOwnerDocument().createElement(StaticStrings.ENTRY_ELEMENT);
+	    entry.setAttribute(StaticStrings.EXTENSION_ATTRIBUTE, extension);
+	    associations_element.appendChild(entry);
+	}
+
+	if(command != null) {
+	    // Replace the text in this node. If the user has used filename instead of %1 then too bad.
+	    XMLTools.setValue(entry, command);
+	}
+	else {
+	    // Remove the entry
+	    associations_element.removeChild(entry);
+	}
+	entry = null;
+	fireTableDataChanged(); // Can't be anymore efficient as DOM does not gareuntee ordering of new child nodes is consistant
+    }
+
+    public int size() {
+	NodeList entries = associations_element.getElementsByTagName(StaticStrings.ENTRY_ELEMENT);
+	return entries.getLength();
+    }
+
+
+    protected boolean isAvailable(String program) {
+	try {
+	    ExternalProgram e = new ExternalProgram("which", program);
+	    e.exitProgram();
+	    String out = e.getLineOfProgramOutput();
+	    if (out == null) {
+		return false;
+	    }
+	    return true;
+	} catch (Exception exc) {
+	    return false;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileFilter.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileFilter.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileFilter.java	(revision 21112)
@@ -0,0 +1,10 @@
+package org.greenstone.gatherer.file;
+
+public class FileFilter {
+    public boolean exclude;
+    public String filter;
+    public FileFilter(String filter, boolean exclude) {
+	this.exclude = exclude;
+	this.filter = filter;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileJob.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileJob.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileJob.java	(revision 21112)
@@ -0,0 +1,156 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import javax.swing.tree.*;
+import org.greenstone.gatherer.util.DragComponent;
+
+/** This object encapsulates all the information about a certain file movement job. This job be to create a copy of a file from one place to another, or to simply delete a file. 
+ * @author John Thompson
+ * @version 2.3c
+ */
+public class FileJob {
+    /** true to mark that this file has already been copied, false otherwise. */
+    public boolean done = false;
+
+    public boolean folder_level = false;
+    /** The type of this movement as an byte. */
+    public byte type = 0;
+    /** The DragComponent source of this file, most likely a GTree. */
+    public DragComponent source       = null;
+    /** The DragComponent to move the file to, again most likely a GTree. */
+    public DragComponent target       = null;
+    /** The unique identifier shared by all jobs created by the same action. */
+    private long id = 0;
+    /** The path to the destination node as a string. Used because any reference based path or variables quickly becomes obsolete. */
+    private TreePath destination_path = null;
+    /** The path to the origin node as a string. Used because any reference based path or variables quickly becomes obsolete. */
+    private TreePath origin_path = null;
+    /** An element of the job type enumeration indicating a copy action. */
+    static final public byte COPY   = 1;
+    /** An element of the job type enumeration indicating a delete action. */
+    static final public byte DELETE = 2;
+    /** An element of the job type enumeration indicating a move action. */
+    static final public byte MOVE   = 3;
+    /** An element of the job type enumeration indicating a new folder action. */
+    static final public byte NEW_FOLDER = 4;
+
+    /** Constructor.
+     * @param id A unique identifier for this job (and others created with a single gesture) as a long.
+     * @param source The DragComponent source of this file, most likely a GTree.
+     * @param orig The FileNode you wish to mode.
+     * @param target The DragComponent to move the file to, again most likely a GTree.
+     * @param dest The files new FileNode parent within the target.
+     * @param type The type of this movement as an int, either COPY or DELETE.
+     */
+    public FileJob(long id, DragComponent source, FileNode orig, DragComponent target, FileNode dest, byte type) {
+	this.id = id;
+	this.source = source;
+	this.target = target;
+	this.type = type;
+	///ystem.err.println("New Job: " + type + ", " + source + ", " + target);
+	// Dont store FileNodes which can go stale. Store paths instead, which are used to locate current 'fresh' versions of nodes.
+	if(dest != null) {
+	    this.destination_path = new TreePath(dest.getPath());
+	    ///ystem.err.println("Destination Path: " + destination_path);
+	}
+	if(orig != null) {
+	    this.origin_path = new TreePath(orig.getPath());
+	    ///atherer.println("Origin Path: " + origin_path);
+	}
+    }
+
+    /** Retrieve the destination node. Watch out for stale versions by always attempting to load the node at destination_path first. */
+    public FileNode getDestination() {
+	FileNode destination = null;
+	if(destination_path != null) {
+	    if(target != null) {
+		FileSystemModel model = (FileSystemModel)target.getTreeModel();
+		destination = model.getNode(destination_path);
+	    }
+	    // If the above fails, a stale copy may be better than nothing.
+	    else {
+		destination = (FileNode) destination_path.getLastPathComponent();
+	    }
+	}
+	return destination;
+    }
+
+    /** Retrieve the origin node. Watch out for stale versions by always attempting to load the node at origin_path first. */
+    public FileNode getOrigin() {
+	FileNode origin = null;
+	if(origin_path != null) {
+	    if(source != null) {
+		FileSystemModel model = (FileSystemModel)source.getTreeModel();
+		origin = model.getNode(origin_path);
+	    }
+	    // If the above fails, a stale copy may be better than nothing.
+	    else {
+		origin = (FileNode) origin_path.getLastPathComponent();
+	    }
+	}
+	return origin;
+    }
+
+    /** Retrieve the id for this job. */
+    public long ID() {
+	return id;
+    }
+
+    public String toString() {
+	StringBuffer text = new StringBuffer("");
+	switch(type) {
+	case COPY:
+	    text.append("copy ");
+	    break;
+	case DELETE:
+	    text.append("delete ");
+	    break;
+	case MOVE:
+	    text.append("move ");
+	    break;
+	default:
+	    text.append("unknown ");
+	}
+	FileNode origin = getOrigin();
+	if(origin != null) {
+	    text.append(origin.getFile().getAbsolutePath());
+	}
+	else {
+	    text.append("ERROR!");
+	}
+	text.append(" -> ");
+	FileNode destination = getDestination();
+	if(destination != null) {
+	    text.append(destination.getFile().getAbsolutePath());
+	}  
+	else {
+	    text.append("Recycle Bin");
+	}
+	return text.toString();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileManager.java	(revision 21112)
@@ -0,0 +1,332 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, NZDL Project, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import java.io.File;
+import javax.swing.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.gui.LongProgressBar;
+import org.greenstone.gatherer.gui.NewFolderOrFilePrompt;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.DragComponent;
+import org.greenstone.gatherer.util.Utility;
+
+
+/** Manages the moving of files within a separate thread.
+ * @author John Thompson, NZDL Project, University of Waikato
+ */
+public class FileManager
+{
+    /** Not only the queue of files to be moved, but also the object that moves them. */
+    static private FileQueue file_queue = null;
+
+    public static int FILE_TYPE = 0;
+    public static int FOLDER_TYPE = 1;
+
+
+    /** Constructor.
+     * @see org.greenstone.gatherer.file.FileQueue
+     */
+    public FileManager()
+    {
+	file_queue = new FileQueue();
+	file_queue.start();
+    }
+
+
+    /** Determine what action should be carried out by the file queue, and add all of the necessary file jobs. */
+    public void action(DragComponent source, FileNode[] source_nodes, DragComponent target, FileNode target_node)
+    {
+	// Check there is something to do
+	if (source_nodes == null || source_nodes.length == 0) {
+	    return;
+	}
+
+	// We need a unique ID for each file task
+	long id = System.currentTimeMillis();
+
+	// If source and target are the same we're moving
+	if (source == target) {
+	    // Start a new move FileTask and we're done
+	    (new FileTask(id, source, source_nodes, target, target_node, FileJob.MOVE)).start();
+	    return;
+	}
+
+	// If target isn't the RecycleBin, we're copying
+	if (!(target instanceof RecycleBin)) {
+	    // Start a new copy FileTask and we're done
+	    (new FileTask(id, source, source_nodes, target, target_node, FileJob.COPY)).start();
+	    return;
+	}
+
+	// We're deleting... but first make sure source isn't read-only
+	boolean read_only_source = false;
+
+	// The workspace tree is read-only...
+	if (source.toString().equals("Workspace")) {
+	    read_only_source = true;
+
+	    // ...except for files from the "Downloaded Files" folder
+	    String downloaded_files_folder_path = Gatherer.getGLIUserCacheDirectoryPath();
+	    for (int i = 0; i < source_nodes.length; i++) {
+		// Is this the "Downloaded Files" folder?
+		if (source_nodes[i].getFile().getAbsolutePath().startsWith(downloaded_files_folder_path)) {
+		    // Can only delete from the "Downloaded Files" folder if a collection is open
+		    if (Gatherer.c_man.ready() == false) {
+			return;
+		    }
+		    read_only_source = false;
+		}
+	    }
+	}
+
+	// The source is read-only, so tell the user and abort
+	if (read_only_source) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Read_Only"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	    return;
+	}
+
+	// Start a new delete FileTask and we're done
+	(new FileTask(id, source, source_nodes, target, target_node, FileJob.DELETE)).start();
+    }
+
+
+    /** Retrieves the file queue object. */
+    public FileQueue getQueue()
+    {
+	return file_queue;
+    }
+
+
+    private class FileTask
+	extends Thread
+    {
+	private long id;
+	private DragComponent source;
+	private FileNode[] source_nodes;
+	private DragComponent target;
+	private FileNode target_node;
+	private byte type;
+
+
+	public FileTask(long id, DragComponent source, FileNode[] source_nodes, DragComponent target, FileNode target_node, byte type)
+	{
+	    this.id = id;
+	    this.source = source;
+	    this.source_nodes = source_nodes;
+	    this.target = target;
+	    this.target_node = target_node;
+	    this.type = type;
+	}
+
+
+	public void run()
+	{
+	    // Reset the progress bar and set it to indeterminate while calculating its size
+	    LongProgressBar progress_bar = file_queue.getProgressBar();
+	    progress_bar.reset();
+	    progress_bar.setIndeterminate(true);
+
+	    // Calculate the progress bar size
+	    boolean cancelled = file_queue.calculateSize(source_nodes);
+	    if (!cancelled) {
+		file_queue.addJob(id, source, source_nodes, target, target_node, type, true);
+		if (Gatherer.isGsdlRemote) {
+		    String collection_name = Gatherer.c_man.getCollection().getName();
+
+		    // Perform the appropriate action based on the job type (RemoteGreenstoneServer will queue)
+		    if (type == FileJob.COPY) {
+			// Copies: upload all the files at once in one zip file
+			File[] source_files = new File[source_nodes.length];
+			for (int i = 0; i < source_nodes.length; i++) {
+			    source_files[i] = source_nodes[i].getFile();
+			}
+			RemoteGreenstoneServer.uploadFilesIntoCollection(collection_name, source_files, target_node.getFile());
+		    }
+		    else if (type == FileJob.DELETE) {
+			// Deletes: delete each top-level file/directory one at a time
+			for (int i = 0; i < source_nodes.length; i++) {
+			    RemoteGreenstoneServer.deleteCollectionFile(collection_name, source_nodes[i].getFile());
+			}
+		    }
+		    else if (type == FileJob.MOVE) {
+			// Moves: move each top-level file/directory one at a time
+			for (int i = 0; i < source_nodes.length; i++) {
+			    RemoteGreenstoneServer.moveCollectionFile(collection_name, source_nodes[i].getFile(), target_node.getFile());
+			}
+		    }
+		}
+	    }
+
+	    progress_bar.setIndeterminate(false);
+	    progress_bar.clear();
+	}
+    }
+
+
+    public void openFileInExternalApplication(File file)
+    {
+	// This must go in a separate thread because we need the progress bar to work (remote Greenstone server)
+	new OpenFileInExternalApplicationTask(file).start();
+    }
+
+
+    private class OpenFileInExternalApplicationTask
+	extends Thread
+    {
+	private File file = null;
+
+	public OpenFileInExternalApplicationTask(File file)
+	{
+	    this.file = file;
+	}
+
+	public void run()
+	{
+	    // If we're using a remote Greenstone server, we need to download the file before viewing it...
+	    if (Gatherer.isGsdlRemote) {
+		// ... but only if it is inside the collection and we haven't already downloaded it
+		if (file.getAbsolutePath().startsWith(Gatherer.getCollectDirectoryPath()) && file.length() == 0) {
+		    if (RemoteGreenstoneServer.downloadCollectionFile(Gatherer.c_man.getCollection().getName(), file).equals("")) {
+			// Something has gone wrong downloading the file
+			return;
+		    }
+		}
+	    }
+
+	    // View the file in an external application
+	    Gatherer.spawnApplication(file);
+	}
+    }
+
+
+    public void newDummyDoc(DragTree tree, CollectionTreeNode parent_node){
+	newFolderOrDummyDoc(tree, parent_node, FILE_TYPE);
+    }
+
+
+    public void newFolder(DragTree tree, CollectionTreeNode parent_node) {
+	newFolderOrDummyDoc(tree, parent_node, FOLDER_TYPE);
+    }
+
+
+    protected void newFolderOrDummyDoc(DragTree tree, CollectionTreeNode parent_node, int type) {
+	(new NewFolderOrDummyDocumentTask(tree, parent_node, type)).start();
+    }
+
+
+    private class NewFolderOrDummyDocumentTask
+	extends Thread
+    {
+	private DragTree tree = null;
+	private CollectionTreeNode parent_node = null;
+	private int type;
+
+	public NewFolderOrDummyDocumentTask(DragTree tree, CollectionTreeNode parent_node, int type)
+	{
+	    this.tree = tree;
+	    this.parent_node = parent_node;
+	    this.type = type;
+	}
+
+	public void run()
+	{
+	    // Ask the user for the directories name.
+	    String extension = "";
+	    if (type == FILE_TYPE) {
+		extension = ".nul";
+	    }
+
+	    NewFolderOrFilePrompt new_folder_prompt = new NewFolderOrFilePrompt(parent_node, type, extension);
+	    String name = new_folder_prompt.display();
+	    new_folder_prompt.dispose();
+	    new_folder_prompt = null;
+
+	    // And if the name is non-null...
+	    if (name != null) {
+		FileSystemModel model = (FileSystemModel) tree.getModel();
+		File folder_file = new File(parent_node.getFile(), name);
+
+		//... check if it already exists.
+		if (folder_file.exists()) { 
+		    if (type == FILE_TYPE) {
+			JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Already_Exists_No_Create", name), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		    }
+		    else {
+			JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Folder_Already_Exists", name), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		    }
+		}
+		// Otherwise create it.
+		else {
+		    try {
+			if (type == FILE_TYPE) {
+			    folder_file.createNewFile();
+			}
+			else {
+			    folder_file.mkdirs();
+			    if (Gatherer.isGsdlRemote) {
+				long id = System.currentTimeMillis();
+				RemoteGreenstoneServer.newCollectionDirectory(Gatherer.c_man.getCollection().getName(), folder_file);
+			    }
+			}
+
+			// Update the parent node to show the new folder
+			parent_node.refresh();
+
+			// Refresh workspace tree (collection tree is done automatically)
+			Gatherer.g_man.refreshWorkspaceTree(DragTree.COLLECTION_CONTENTS_CHANGED);
+		    }
+		    catch (Exception exception) {
+			if (type == FILE_TYPE) {
+			    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Create_Error", name), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+			}
+			else {
+			    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Folder_Create_Error", name), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+			}
+		    }
+		}
+		
+		folder_file = null;
+		model = null;
+	    }
+	    name = null;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileNode.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileNode.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileNode.java	(revision 21112)
@@ -0,0 +1,357 @@
+package org.greenstone.gatherer.file;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import javax.swing.*;
+import javax.swing.filechooser.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.ArrayTools;
+
+
+public abstract class FileNode
+    implements MutableTreeNode
+{
+    protected boolean allows_children = true;
+    protected ArrayList child_nodes = null;
+    protected ArrayList child_nodes_unfiltered = null;
+    protected File file = null;
+    protected FileSystemModel model = null;
+    protected MutableTreeNode parent = null;
+
+
+    public FileNode(File file)
+    {
+	this.file = file;
+
+	// Files cannot have children
+	if (file != null && file.isFile()) {
+	    // Cache this result to prevent unceasing missing disk messages being thrown if the
+	    // removable media was, um, removed after directory mapped
+	    this.allows_children = false;
+	}
+    }
+
+
+    /** Returns the children of the node as an Enumeration. */
+    public Enumeration children()
+    {
+	return new FileEnumeration();
+    }
+
+
+    /** Returns true if the receiver allows children.  */
+    public boolean getAllowsChildren()
+    {
+	return allows_children;
+    }
+
+
+    /** Returns the child TreeNode at index childIndex. */
+    public TreeNode getChildAt(int index)
+    {
+	return (TreeNode) child_nodes.get(index);
+    }
+
+
+    /** Returns the number of children TreeNodes the receiver contains. */
+    public int getChildCount()
+    {
+	int size = 0;
+
+	// We don't map if this is a system root, or we risk the 50,000 Disk not found error messages of death
+	if (isFileSystemRoot()) {
+	    size = 1;  // Size is always non-zero for a system root
+	}
+	else {
+	    map();
+	}
+
+	// Use the number of (filtered) child nodes
+	if (child_nodes != null) {
+	    size = child_nodes.size();
+	}
+
+	return size;
+    }
+
+
+    /** Returns the index of node in the receivers children. */
+    public int getIndex(TreeNode node)
+    {
+	if (child_nodes != null) {
+	    return child_nodes.indexOf(node);
+	}
+
+	return -1;
+    }
+
+
+    /** Returns the parent TreeNode of the receiver. */
+    public TreeNode getParent()
+    {
+	return parent;
+    }
+
+
+    /** Adds child to the receiver at index. */
+    public void insert(MutableTreeNode child, int index)
+    {
+	DebugStream.println("Insert " + child + " in " + this + " at index " + index + " [Model: " + model + "]");
+	if (child == null) {
+	    return;
+	}
+
+	try {
+	    FileNode child_node = (FileNode) child;
+	    child_nodes.add(index, child_node);
+	    child_node.model = model;
+	    child_node.parent = this;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+
+    /** Returns true if the receiver is a leaf. */
+    public boolean isLeaf()
+    {
+	return (allows_children == false);
+    }
+
+
+    /** Removes the child at index from the receiver. */
+    public void remove(int index)
+    {
+	if (index >= 0 && index < child_nodes.size()) {
+	    child_nodes.remove(index);
+	}
+    }
+
+
+    /** Removes node from the receiver. */
+    public void remove(MutableTreeNode node)
+    {
+	remove(getIndex(node));
+    }
+
+
+    /** Removes the receiver from its parent. */
+    public void removeFromParent()
+    {
+	parent.remove(this);
+	parent = null;
+    }
+
+
+    /** Resets the user object of the receiver to object. */
+    public void setUserObject(Object object) {
+	try {
+	    file = (File) object;
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+
+    // -------------------------------------------------------------------------------
+
+
+    public void add(MutableTreeNode child)
+    {
+	insert(child, child_nodes.size());
+    }
+
+
+    protected abstract FileNode addChildNode(File file);
+
+
+    /** Compare two FileNodes for equality. */
+    public boolean equals(FileNode node)
+    {
+	if (node == null) {
+	    // Definitely not a match
+	    return false;
+	}
+
+	if (file != null) {
+	    return file.equals(node.getFile());
+	}
+	else {
+	    return toString().equals(node.toString());
+	}
+    }
+
+
+    /** Retrieve the file node at the given index, regardless of filters set. */
+    public FileNode getChildAtUnfiltered(int index)
+    {
+ 	if (index >= 0 && index < size()) {
+ 	    return (FileNode) child_nodes_unfiltered.get(index);
+ 	}
+ 	return null;
+    }
+
+
+    public File getFile() {
+	return file;
+    }
+
+
+    /** Retrieves the tree path from the root node to this node. */
+    public TreeNode[] getPath() {
+	int count = 0;
+	TreeNode current = this;
+	while(current != null) {
+	    count++;
+	    current = current.getParent();
+	}
+	TreeNode[] path = new TreeNode[count];
+	current = this;
+	while(current != null) {
+	    path[count - 1] = current;
+	    count--;
+	    current = current.getParent();
+	}
+	return path;
+    }
+
+
+    public boolean isFileSystemRoot() {
+	if (file != null) {
+	    return FileSystemView.getFileSystemView().isFileSystemRoot(file);
+	}
+	else {
+	    return false;
+	}
+    }
+
+
+    /** Overridden if necessary by subclasses. */
+    public boolean isInLoadedCollection()
+    {
+	return false;
+    }
+
+
+    /** Overridden if necessary by subclasses. */
+    public boolean isReadOnly()
+    {
+	return false;
+    }
+
+
+    public void map()
+    {
+	// If this node has already been mapped, don't bother doing it again
+	if (child_nodes != null) {
+	    return;
+	}
+	child_nodes = new ArrayList();
+
+	// General case, only map if children are allowed
+	if (file != null && getAllowsChildren()) {
+	    File[] files = file.listFiles();
+	    if (files != null && files.length > 0) {
+		// Sort the child files
+		ArrayTools.sort(files);
+
+		// Now add them to child_nodes_unfiltered
+		child_nodes_unfiltered = new ArrayList();
+		for (int i = 0; i < files.length; i++) {
+		    FileNode child_node = this.addChildNode(files[i]);
+		    child_nodes_unfiltered.add(child_node);
+		}
+
+		// Apply the filters set in the model
+		FileFilter[] filters = model.getFilters();
+		for (int i = 0; filters != null && i < filters.length; i++) {
+		    files = ArrayTools.filter(files, filters[i].filter, filters[i].exclude);
+		}
+
+		// Add the files left after filtering to child_nodes
+		for (int i = 0; i < files.length; i++) {
+		    // Use the FileNode object in child_nodes_unfiltered rather than creating another
+		    for (int j = 0; j < child_nodes_unfiltered.size(); j++) {
+			if (((FileNode) child_nodes_unfiltered.get(j)).getFile().equals(files[i])) {
+			    child_nodes.add(child_nodes_unfiltered.get(j));
+			    break;
+			}
+		    }
+		}
+	    }
+
+	    model.nodeStructureChanged(this);
+	}
+    }
+
+
+    public void refresh()
+    {
+  	unmap();
+  	map();
+    }
+
+
+    public void setModel(FileSystemModel model) {
+	this.model = model;
+    }
+
+    public void setParent(MutableTreeNode parent) {
+	this.parent = parent;
+    }
+
+
+    /** Return the total number of child files for the file this node represents, irrespective of filters set. */
+    public int size() {
+	if (child_nodes_unfiltered != null) {
+	    return child_nodes_unfiltered.size();
+	}
+	return 0;
+    }
+
+
+    public String toString()
+    {
+	if (isFileSystemRoot()) {
+	    return file.getAbsolutePath();
+	}
+	else {
+	    return file.getName();
+	}
+    }
+
+
+    /** Unmap this node's children. */
+    public void unmap()
+    {
+	DebugStream.println("Unmapping " + this + "...");
+	child_nodes_unfiltered = null;
+	child_nodes = null;
+    }
+
+
+    private class FileEnumeration
+	implements Enumeration
+    {
+	private int index = 0;
+
+	/** Tests if this enumeration contains more elements. */
+	public boolean hasMoreElements() {
+	    return (index < child_nodes.size());
+	}
+
+	/** Returns the next element of this enumeration if this enumeration object has at least one more element to provide. */
+	public Object nextElement() {
+	    Object result = null;
+	    if (index < child_nodes.size()) {
+		result = child_nodes.get(index);
+		index++;
+	    }
+	    return result;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileOpenActionListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileOpenActionListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileOpenActionListener.java	(revision 21112)
@@ -0,0 +1,101 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Gatherer;
+
+/** This class listens for mouse clicks and responds to double-clicks (spawn a new application to view the selected file) and right mouse button clicks (popup menu). 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3b
+ */
+public class FileOpenActionListener
+    extends MouseAdapter 
+    implements TreeExpansionListener {
+    /** This flag gets toggled to <i>true</i> if the listener determines that the next mouse clicked event it will recieve is actually caused by the tree expanding or collapsing. */
+    private boolean ignore = false;
+    /** The constructor. */
+    public FileOpenActionListener() {
+    }
+    /** Any subclass of MouseAdapter can override this method to respond to mouse click events. In this case we want to start an external application if someone double clicks on an appropriate file record. 
+     * @param event A <strong>MouseEvent</strong> containing further information about the mouse click performed.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.file.FileNode
+     * @see org.greenstone.gatherer.gui.tree.DragTree
+     */ 
+    public void mouseClicked(MouseEvent event) {
+	///ystem.err.println("Mouse clicked");
+	if(!ignore) {
+	    if(event.getClickCount() >= 2) {
+		// Find the file we're clicking on.
+		JTree tree = (JTree)event.getSource();
+		TreePath path = tree.getClosestPathForLocation(event.getX(), event.getY());
+		if(path != null) {
+		    FileNode record = (FileNode)path.getLastPathComponent();
+		    ///ystem.err.println("Double clicked on " + record);
+		    File file = record.getFile();
+		    if (file != null && file.isFile()) {
+			///ystem.err.println("Running " + file);
+			Gatherer.f_man.openFileInExternalApplication(file);
+		    }
+		}
+	    }
+	}
+	else {
+	    ///ystem.err.println("Caused by tree expansion / collapse. Ignoring.");
+	    ignore = false;
+	}
+    }
+    /** Any implementation of TreeExpansionListener must include this method so that we can be informed when a tree node has been collapsed, thus indicating that any mouse click events about to be recieved are most likely related to this event.
+     * @param event A <strong>TreeExpansionEvent</strong> containing information about the node collapsed.
+     */
+    public void treeCollapsed(TreeExpansionEvent event) {
+	///ystem.err.println("Tree Collapsed");
+	ignore = true;
+    }
+    /** Any implementation of TreeExpansionListener must include this method so that we can be informed when a tree node has been expanded, thus indicating that any mouse click events about to be recieved are most likely related to this event.
+     * @param event A <strong>TreeExpansionEvent</strong> containing information about the node expanded.
+     */
+    public void treeExpanded(TreeExpansionEvent event) {
+	///ystem.err.println("Tree Expanded");
+	ignore = true;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileQueue.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileQueue.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileQueue.java	(revision 21112)
@@ -0,0 +1,694 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.gui.LongProgressBar;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.metadata.MetadataValue;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+import org.greenstone.gatherer.util.DragComponent;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.SynchronizedTreeModelTools;
+import org.greenstone.gatherer.util.Utility;
+
+/** A threaded object which processes a queue of file actions such as copying and movement. It also handles updating the various trees involved so they are an accurate representation of the file system they are meant to match.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class FileQueue
+    extends Thread
+{
+    /** When someone requests the movement queue to be dumped this cancel flag is set to true. */
+    private boolean cancel_action = false;
+    /** The button which controls the stopping of the file queue. */
+    private JButton stop_button = null;
+    /** true if the user has selected yes to all from a file 'clash' dialog. */
+    private boolean yes_to_all = false;
+    /** A label explaining the current moving files status. */
+    private JLabel file_status = null;
+    /** A list containing a queue of waiting movement jobs. */
+    private ArrayList queue = null;
+    /** A progress bar which shows how many bytes, out of the total size of bytes, has been moved. */
+    private LongProgressBar progress = null;
+
+
+    /** Constructor.
+     */
+    public FileQueue() {
+	DebugStream.println("FileQueue started.");
+	this.queue = new ArrayList();
+	file_status = new JLabel();
+	Dictionary.setText(file_status, "FileActions.No_Activity");
+	progress = new LongProgressBar();
+	progress.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	progress.setForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
+	progress.setString(Dictionary.get("FileActions.No_Activity"));
+	progress.setStringPainted(true);
+    }
+
+    /** Requeue an existing job into the queue.
+     * @param job A previously created FileJob.
+     */
+    synchronized private void addJob(FileJob job, int position) {
+	job.done = true; // Ensure that the requeued job is marked as done.
+	queue.add(position, job);
+	notify();
+    }
+
+    /** Add a new job to the queue, specifiying as many arguments as is necessary to complete this type of job (ie delete needs no target information).
+     * @param id A long id unique to all jobs created by a single action.
+     * @param source The DragComponent source of this file, most likely a DragTree.
+     * @param child The FileNode you wish to mode.
+     * @param target The DragComponent to move the file to, again most likely a DragTree.
+     * @param parent The files new FileNode parent within the target.
+     * @param type The type of this movement as an int, either COPY or DELETE.
+     */
+    public void addJob(long id, DragComponent source, FileNode[] children, DragComponent target, FileNode parent, byte type, boolean folder_level)
+    {
+	// Queue the sub-job(s) (this may fail if we are asked to delete a read only file)
+	for (int i = 0; i < children.length; i++) {
+	    addJob(id, source, children[i], target, parent, type, folder_level, -1);
+	}
+    }
+    
+    synchronized private void addJob(long id, DragComponent source, FileNode child, DragComponent target, FileNode parent, byte type, boolean folder_level, int position) {
+	FileJob job = new FileJob(id, source, child, target, parent, type);
+	job.folder_level = folder_level;
+        DebugStream.println("Adding job: " + job);
+	if(position != -1 && position <= queue.size() + 1) {
+	    queue.add(position, job);
+	}
+	else {
+	    queue.add(job);
+	}
+	notify();
+    }
+
+    /** Calculates the total deep file size of the selected file nodes.
+     * @param files a FileNode[] of selected files
+     * @return true if a cancel was signalled, false otherwise
+     * @see org.greenstone.gatherer.file.FileManager.Task#run()
+     */
+    public boolean calculateSize(FileNode[] files)
+    {
+	file_status.setText(Dictionary.get("FileActions.Calculating_Size"));
+	progress.setString(Dictionary.get("FileActions.Calculating_Size"));
+
+	// Calculate the total file size of all the selected file nodes
+	Vector remaining = new Vector();
+	for (int i = 0; !cancel_action && i < files.length; i++) {
+	    remaining.add(files[i]);
+	}
+	while (!cancel_action && remaining.size() > 0) {
+	    FileNode node = (FileNode) remaining.remove(0);
+	    if (node.isLeaf()) {
+		progress.addMaximum(node.getFile().length());
+	    }
+	    else {
+		for (int i = 0; !cancel_action && i < node.getChildCount(); i++) {
+		    remaining.add(node.getChildAt(i));
+		}
+	    }
+	}
+
+	// Now we return if calculation was cancelled so that the FileManagers Task can skip the addJob phase correctly.
+	if (cancel_action) {
+	    cancel_action = false; // reset
+	    return true;
+	}
+	else {
+	    return false;
+	}
+    }
+
+    /** This method is called to cancel the job queue at the next available moment. */
+    public void cancelAction() {
+	cancel_action = true;
+    }
+
+
+    private int countFolderDepth(File file)
+    {
+	int depth = 0;
+	while (file != null) {
+	    depth++;
+	    file = file.getParentFile();
+	}
+	return depth;
+    }
+
+
+    /** Format the given filename path string so that it is no longer than the given width. If it is wider replace starting directories with ...
+     * @param key The key <strong>String</Strong> used to retrieve a phrase from the dictionary for this item.
+     * @param raw The raw filename path <strong>String</strong>.
+     * @param width The maximum width as an <i>int</i>.
+     * @return A path <strong>String</strong> no longer than width.
+     */
+    private String formatPath(String key, String raw, int width)
+    {
+	JLabel label = new JLabel(Dictionary.get(key, raw));
+	int position = -1;
+	while(label.getPreferredSize().width > width && (position = raw.indexOf(File.separator)) != -1) {
+	    raw = "..." + raw.substring(position + 1);
+	    label.setText(Dictionary.get(key, raw));
+	}
+	if(raw.indexOf(File.separator) == -1 && raw.startsWith("...")) {
+	    raw = raw.substring(3);
+	}
+	return raw;
+    }
+
+
+    /** Access to the file state label. */
+    public JLabel getFileStatus() {
+	return file_status;
+    }
+
+    /** Access to the progress bar. */
+    public LongProgressBar getProgressBar() {
+	return progress;
+    }
+
+
+    /** The run method exists in every thread, and here it is used to work its way through the queue of Jobs. If no jobs are waiting and it cans, it waits until a job arrives. If a job is present then it is either COPIED or DELETED, with the records being copied or removed as necessary, and directories being recursed through. Finally the user can press cancel to cause the loop to prematurely dump the job queue then wait.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.file.FileJob
+     * @see org.greenstone.gatherer.file.FileNode
+     * @see org.greenstone.gatherer.gui.LongProgressBar
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void run()
+    {
+	super.setName("FileQueue");
+
+	while (!Gatherer.exit) {
+	    try {
+		// Retrieve the next job
+		int position = queue.size() - 1;
+		FileJob job = null;
+		if (position >= 0) {
+		    job = (FileJob) queue.remove(position);
+		}
+
+		if (job != null) {
+		    ///ystem.err.println("Found job: " + job);
+		    // Enabled stop button
+		    stop_button.setEnabled(true);
+		    // The user can cancel this individual action at several places, so keep track if the state is 'ready' for the next step.
+		    boolean ready = true;
+		    FileNode origin_node = job.getOrigin();
+		    FileNode destination_node = job.getDestination();
+		    FileSystemModel source_model = (FileSystemModel)job.source.getTreeModel();
+		    FileSystemModel target_model = (FileSystemModel)job.target.getTreeModel();
+		    if(destination_node == null) {
+			// Retrieve the root node of the target model instead. A delete, or course, has no target file so all deleted files are added to the root of the Recycle Bin model.
+			destination_node = (FileNode) target_model.getRoot();
+		    }
+
+		    // Extract common job details.
+		    File source_file = origin_node.getFile();
+		    File target_file = null;
+		    // Determine the target file for a copy or move.
+		    if (job.type == FileJob.COPY || job.type == FileJob.MOVE) {
+			// use the name of the filenode instead of the name of the file - these should be the same except for the collection directories where we want the collection name to be used, not 'import' which is the underlying name
+			target_file = new File(destination_node.getFile(), origin_node.toString());
+		    }
+		    // To copy a file, copy it then add any metadata found at the source. If this file was already in our collection then we must ensure the lastest version of its metadata.xml has been saved to disk. To copy a directory simply create the directory at the destination, then add all of its children files as new jobs.
+		    if((job.type == FileJob.COPY || job.type == FileJob.MOVE) && !job.done) {
+			///ystem.err.println("Copy/Move: " + origin_node);
+
+			// The number one thing to check is whether we are in a cyclic loop. The easiest way is just to check how deep we are
+			int max_folder_depth = Configuration.getInt("general.max_folder_depth", Configuration.COLLECTION_SPECIFIC);
+			boolean continue_over_depth = false;
+			if (countFolderDepth(source_file) > max_folder_depth) {
+			    Object[] options = { Dictionary.get("General.Yes"), Dictionary.get("General.No"), Dictionary.get("FileActions.Increase_Depth") };
+			    String args[] = { String.valueOf(max_folder_depth), source_file.getAbsolutePath() };
+			    int result = JOptionPane.showOptionDialog(Gatherer.g_man, Utility.formatHTMLWidth(Dictionary.get("FileActions.Possible_Cyclic_Path", args), 80), Dictionary.get("General.Warning"), JOptionPane.DEFAULT_OPTION, JOptionPane.WARNING_MESSAGE, null, options, options[1]);
+			    args = null;
+			    options = null;
+			    switch(result) {
+			    case 0: // Yes
+				continue_over_depth = true;
+				break;
+			    case 2: // Continue and increase depth
+				continue_over_depth = true;
+				Configuration.setInt("general.max_folder_depth", Configuration.COLLECTION_SPECIFIC, (max_folder_depth + 1));
+				break;
+			    }
+			}
+			else {
+			    continue_over_depth = true;
+			}
+			   
+			if(continue_over_depth) {
+			    FileNode new_node = null;
+			    // Check if file exists, and action as necessary. Be aware the user can choose to cancel the action all together (where upon ready becomes false).
+			    if(target_file.exists()) {
+				// We've previously been told
+				if(yes_to_all) {
+				    // Remove the old file and tree entry.
+				    target_file.delete();
+				    ready = true;
+				}
+				else {
+				    ///atherer.println("Opps! This filename already exists. Give the user some options.");
+				    Object[] options = { Dictionary.get("General.Yes"), Dictionary.get("FileActions.Yes_To_All"), Dictionary.get("General.No"), Dictionary.get("General.Cancel") };
+				    int result = JOptionPane.showOptionDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Exists", target_file.getName()), Dictionary.get("General.Warning"), JOptionPane.DEFAULT_OPTION, JOptionPane.WARNING_MESSAGE, null, options, options[0]);
+				    switch(result) {
+				    case 1: // Yes To All
+					yes_to_all = true;
+				    case 0: // Yes
+					// Remove the old file and tree entry.
+					if(destination_node != null) {
+					    TreePath destination_path = new TreePath(destination_node.getPath());
+					    CollectionTreeNode temp_target_node = new CollectionTreeNode(target_file);  // !!! , target_model, true);
+					    TreePath target_path = destination_path.pathByAddingChild(temp_target_node);
+					    SynchronizedTreeModelTools.removeNodeFromParent(target_model, target_model.getNode(target_path));
+					    target_path = null;
+					    temp_target_node = null;
+					    destination_path = null;
+					}
+					target_file.delete();
+					ready = true;
+					break;
+				    case 3: // No To All
+					cancel_action = true;
+				    case 2: // No
+				    default:
+					ready = false;
+					// Increment progress by size of potentially copied file
+					progress.addValue(origin_node.getFile().length());
+				    }
+				}
+			    }
+			    // We proceed with the copy/move if the ready flag is still set. If it is that means there is no longer any existing file of the same name.
+			    if(ready) {
+				// update status area
+				String args[] = new String[1];
+				args[0] = "" + (queue.size() + 1) + "";
+				if(job.type == FileJob.COPY) {
+				    args[0] = formatPath("FileActions.Copying", source_file.getAbsolutePath(), file_status.getSize().width);
+				    file_status.setText(Dictionary.get("FileActions.Copying", args));
+				}
+				else {
+				    args[0] = formatPath("FileActions.Moving", source_file.getAbsolutePath(), file_status.getSize().width);
+				    file_status.setText(Dictionary.get("FileActions.Moving", args));
+				}
+				args = null;
+				
+				// If source is a file
+				if(source_file.isFile()) {
+				    // copy the file. If anything goes wrong the copy file should throw the appropriate exception. No matter what exception is thrown (bar an IOException) we display some message, perhaps take some action, then cancel the remainder of the pending file jobs. No point in being told your out of hard drive space for each one of six thousand files eh?
+				    try {
+					copyFile(source_file, target_file, progress);
+				    }
+				    // If we can't find the source file, then the most likely reason is that the file system has changed since the last time it was mapped. Warn the user that the requested file can't be found, then force a refresh of the source folder involved.
+				    catch(FileNotFoundException fnf_exception) {
+					DebugStream.printStackTrace(fnf_exception);
+					cancel_action = true;
+					// Show warning.
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Not_Found_Message", source_file.getName()), Dictionary.get("FileActions.File_Not_Found_Title"), JOptionPane.ERROR_MESSAGE);
+					// Force refresh of source folder.
+					source_model.refresh(new TreePath(((FileNode)origin_node.getParent()).getPath()));
+				    }
+				    catch(FileAlreadyExistsException fae_exception) {
+					DebugStream.printStackTrace(fae_exception);
+					cancel_action = true;
+					// Show warning.
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Already_Exists_Message", target_file.getName()), Dictionary.get("FileActions.File_Already_Exists_Title"), JOptionPane.ERROR_MESSAGE);
+					// Nothing else can be done by the Gatherer.
+				    }
+				    catch(InsufficientSpaceException is_exception) {
+					DebugStream.printStackTrace(is_exception);
+					cancel_action = true;
+					// Show warning. The message body of the expection explains how much more space is required for this file copy.
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Insufficient_Space_Message", is_exception.getMessage()), Dictionary.get("FileActions.Insufficient_Space_Title"), JOptionPane.ERROR_MESSAGE);
+					// Nothing else can be done by the Gatherer. In fact if we are really out of space I'm not even sure we can quit safely.
+				    }
+				    catch (ReadNotPermittedException rnp_exception) {
+					if (DebugStream.isDebuggingEnabled()) {
+					    DebugStream.printStackTrace(rnp_exception);
+					}
+					cancel_action = true;
+					// Show warning
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Read_Not_Permitted_Message", source_file.getAbsolutePath()), Dictionary.get("FileActions.Write_Not_Permitted_Title"), JOptionPane.ERROR_MESSAGE);
+					// Nothing else we can do.
+				    }
+				    catch(UnknownFileErrorException ufe_exception) {
+					DebugStream.printStackTrace(ufe_exception);
+					cancel_action = true;
+					// Show warning
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Unknown_File_Error_Message"), Dictionary.get("FileActions.Unknown_File_Error_Title"), JOptionPane.ERROR_MESSAGE);
+					// Nothing else we can do.
+				    }
+				    catch(WriteNotPermittedException wnp_exception) {
+					if (DebugStream.isDebuggingEnabled()) {
+					    DebugStream.printStackTrace(wnp_exception);
+					}
+					cancel_action = true;
+					// Show warning
+					JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.Write_Not_Permitted_Message", target_file.getAbsolutePath()), Dictionary.get("FileActions.Write_Not_Permitted_Title"), JOptionPane.ERROR_MESSAGE);
+					// Nothing else we can do.
+				    }
+				    catch(IOException exception) {
+					// Can't really do much about this.
+					DebugStream.printStackTrace(exception);
+				    }
+				    // If not cancelled
+				    if (!cancel_action) {
+					// Create a dummy FileNode with the correct structure (so getPath works)
+					new_node = new CollectionTreeNode(target_file);
+					SynchronizedTreeModelTools.insertNodeInto(target_model, destination_node, new_node);
+				    }
+				}
+				// Else
+				else if(source_file.isDirectory()) {
+				    // create new record
+				    CollectionTreeNode directory_record = new CollectionTreeNode(target_file);
+				    SynchronizedTreeModelTools.insertNodeInto(target_model, destination_node, directory_record);
+				    // Why is this not happening eh?
+				    directory_record.setParent(destination_node);
+				    if(!target_file.exists()) {
+					// make the directory
+					target_file.mkdirs();
+					new_node = directory_record;
+				    }
+				    // Else inform the users that a directory already exists and files will be copied into it
+				    //else {
+				    //    JOptionPane.showMessageDialog(null, Dictionary.get("FileActions.Directory_Exists", target_file.toString()), Dictionary.get("General.Warning"), JOptionPane.WARNING_MESSAGE);
+				    //}
+				    // Queue non-filtered child files for copying. If this directory already existed, the child records will have to generate the undo jobs, as we don't want to entirely delete this directory if it already existed.
+				    FileNode child_record = null;
+				    // In order to have a sane copy proceedure (rather than always copying last file first as it used to) we always add the child node at the position the parent was removed from. Consider the file job 'a' at the end of the queue which generates three new jobs 'b', 'c' and 'd'. The resulting flow should look like this.
+				    // -- Starting queue                   ...[a]
+				    // remove(position) = 'a'              ...
+				    // add(position, 'b')                  ...[b]
+				    // add(position, 'c')                  ...[c][b]
+				    // add(position, 'd')                  ...[d][c][b]
+				    // Next loop
+				    // remove(position) = 'b'              ...[d][c]
+				    //for(int i = 0; i < origin_node.getChildCount(); i++) {
+				    for (int i=origin_node.getChildCount()-1; i>=0; i--) {
+					child_record = (FileNode) origin_node.getChildAt(i);
+					addJob(job.ID(), job.source, child_record, job.target, directory_record, job.type, false, position);
+				    }
+				    child_record = null;
+				    directory_record = null;
+				}
+				// The file wasn't found!
+				else {
+				    cancel_action = true;
+				    // Show warning.
+				    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Not_Found_Message", source_file.getName()), Dictionary.get("FileActions.File_Not_Found_Title"), JOptionPane.ERROR_MESSAGE);
+				    // Force refresh of source folder.
+				    source_model.refresh(new TreePath(((FileNode)origin_node.getParent()).getPath()));
+				}
+
+				// If we haven't been cancelled and we created a new FileNode during the above phase, now is the time to deal with metadata
+				if (!cancel_action && new_node != null) {
+				    // If the file came from inside our collection...
+				    if (job.source.toString().equals("Collection")) {
+					// Get the non-folder level metadata assigned to the origin node...
+					ArrayList assigned_metadata = MetadataXMLFileManager.getMetadataAssignedDirectlyToFile(source_file);
+					// ...and remove it from the original node and assign it to the new folder
+					MetadataXMLFileManager.removeMetadata((CollectionTreeNode) origin_node, assigned_metadata);
+					MetadataXMLFileManager.addMetadata((CollectionTreeNode) new_node, assigned_metadata);
+				    }
+				    // If it came from the workspace search for metadata assigned to the file
+				    else if (job.source.toString().equals("Workspace")) {
+					ArrayList assigned_metadata = MetadataXMLFileManager.getMetadataAssignedDirectlyToExternalFile(origin_node.getFile());
+					MetadataXMLFileManager.addMetadata((CollectionTreeNode) new_node, assigned_metadata);
+				    }
+
+				    if (job.type == FileJob.COPY && new_node.getFile().isFile()) {
+					Gatherer.c_man.fireFileAddedToCollection(new_node.getFile());
+				    }
+				}
+				new_node = null;
+			    }
+			}
+		    }
+		    // If we haven't been cancelled, and we've been asked to delete a directory/file, or perhaps as part of a move, we delete the file. This involves removing any existing metadata and then copying the file to the recycled bin (for a delete only), then deleting the file. When deleting a directory record from the tree (or from the filesystem for that matter) we must ensure that all of the descendant records have already been removed. If we fail to do this the delete will fail, or you will be bombarded with hundreds of 'Parent node of null not allowed' error messages. Also be aware that if the user has cancelled just this action, because of say a name clash, then we shouldn't do any deleting of any sort dammit.
+		    if(!cancel_action && ready && (job.type == FileJob.DELETE || job.type == FileJob.MOVE)) {
+			// Update the progress bar for this job
+			if (source_file.isFile()) {
+			    progress.addValue(source_file.length());
+			}
+
+			// If the source is a file or an empty directory (but not the root node of a tree)
+			File[] child_list = source_file.listFiles();
+			if (source_file.isFile() || (child_list != null && child_list.length == 0 && origin_node.getParent() != null)) {
+			    // Update status area
+			    String args[] = new String[1];
+			    args[0] = formatPath("FileActions.Deleting", source_file.getAbsolutePath(), file_status.getSize().width);
+			    file_status.setText(Dictionary.get("FileActions.Deleting", args));
+
+			    // If it is a metadata.xml file, we must unload it
+			    if (source_file.getName().equals(StaticStrings.METADATA_XML)) {
+				MetadataXMLFileManager.unloadMetadataXMLFile(source_file);
+			    }
+
+			    // Remove the metadata assigned directly to the file
+			    ArrayList assigned_metadata = MetadataXMLFileManager.getMetadataAssignedDirectlyToFile(origin_node.getFile());
+			    MetadataXMLFileManager.removeMetadata((CollectionTreeNode) origin_node, assigned_metadata);
+
+			    // Remove from model
+			    FileNode parent_record = (FileNode) origin_node.getParent();
+			    if (parent_record != null) {
+				SynchronizedTreeModelTools.removeNodeFromParent(source_model, origin_node);
+			    }
+
+			    // Delete the source file
+			    if (!Utility.delete(source_file)) {
+				// Show message that we couldn't delete
+				JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("FileActions.File_Not_Deleted_Message", source_file.getName()), Dictionary.get("FileActions.File_Not_Deleted_Title"), JOptionPane.ERROR_MESSAGE);
+			    }
+			}
+			// Else the source is a directory and it has children remaining
+			else if(child_list != null && child_list.length > 0) {
+			    // Don't worry about all this for true file move actions.
+			    if(job.type == FileJob.DELETE) {
+				// queue all of its children, (both filtered and non-filtered), but for deleting only. Don't queue jobs for a current move event, as they would be queued as part of copying. I have no idea way, per sec, however the children within the origin node are always invalid during deletion (there are several copies of some nodes?!?). I'll check that each child is only added once.
+				origin_node.refresh();
+				for(int i = 0; i < origin_node.size(); i++) {
+				    FileNode child_record = (FileNode) origin_node.getChildAtUnfiltered(i);
+				    ///atherer.println("Queuing: " + child_record);
+				    addJob(job.ID(), job.source, child_record, job.target, destination_node, FileJob.DELETE, false, position);
+				}  
+			    }
+			    // Requeue a delete job -after- the children have been dealt with. Remember I've reversed the direction of the queue so sooner is later. Te-he. Also have to remember that we have have followed this path to get here for a move job: Copy Directory -> Queue Child Files -> Delete Directory (must occur after child files) -> Queue Directory.
+			    // One special case. Do not requeue root nodes. Don't requeue jobs marked as done.
+			    if(origin_node.getParent() != null && !job.done) {
+				///atherer.println("Requeuing: " + origin_node.getFile().getAbsolutePath());
+				job.type = FileJob.DELETE; // You only requeue jobs that are deletes, as directories must be inspected before children, but deleted after.
+				addJob(job, position);
+			    }
+			    else {
+				DebugStream.println("I've already done this job twice. I refuse to requeue it again!");
+			    }
+			}
+		    }
+		    job = null;
+		    source_file = null;
+		    target_file = null;
+		    origin_node = null;
+
+		    // We only break out of the while loop if we are out of files or the action was cancelled
+		    if (cancel_action) {
+			// Empty queue
+			clearJobs();
+			cancel_action = false;
+		    }
+		}
+		else { // job == null
+		    // Disable stop button
+		    if (stop_button != null) {
+			stop_button.setEnabled(false);
+		    }
+		    synchronized(this) {
+			// Force both workspace and collection trees to refresh
+			if (Gatherer.g_man != null) {
+			    Gatherer.g_man.refreshWorkspaceTree(DragTree.COLLECTION_CONTENTS_CHANGED);
+			    Gatherer.g_man.refreshCollectionTree(DragTree.COLLECTION_CONTENTS_CHANGED);
+			}
+
+			// Reset status area
+			file_status.setText(Dictionary.get("FileActions.No_Activity"));
+			progress.reset();
+			progress.setString(Dictionary.get("FileActions.No_Activity"));
+			yes_to_all = false;
+			try {
+			    wait();
+			}
+			catch (InterruptedException exception) {}
+		    }
+		}
+	    }
+	    catch (Exception error) {
+		DebugStream.printStackTrace(error);
+	    }
+	}
+    }
+
+
+    /** Register the button that will be responsible for stopping executing file actions. 
+     * @param stop_button a JButton
+     */
+    public void registerStopButton(JButton stop_button) {
+	this.stop_button = stop_button;
+    }
+
+
+    synchronized private void clearJobs() {
+	queue.clear();
+    }
+    
+    /** Copy the contents from the source directory to the destination 
+     * directory.
+     * @param source The source directory
+     * @param destination The destination directory
+     * @param progress A progress bar to monitor copying progress
+     * @see org.greenstone.gatherer.Gatherer
+     */
+    public void copyDirectoryContents(File source, File destination, LongProgressBar progress) 
+	throws FileAlreadyExistsException, FileNotFoundException, InsufficientSpaceException, IOException, ReadNotPermittedException, UnknownFileErrorException, WriteNotPermittedException
+    {
+	if (!source.isDirectory()) return;
+	// check that dest dirs exist
+	destination.mkdirs();
+	
+	File [] src_files = source.listFiles();
+	if (src_files.length == 0) return; // nothing to copy
+	for (int i=0; i<src_files.length; i++) {
+	    File f = src_files[i];
+	    String f_name = f.getName();
+	    File new_file = new File(destination, f_name);
+	    if (f.isDirectory()) {
+		copyDirectoryContents(f, new_file, progress);
+	    } else if (f.isFile()) {
+		copyFile(f, new_file, progress);
+	    }
+	}
+	
+    }
+    
+    /** Copy a file from the source location to the destination location.
+     * @param source The source File.
+     * @param destination The destination File.
+     * @see org.greenstone.gatherer.Gatherer
+     */
+    public void copyFile(File source, File destination, LongProgressBar progress)
+	throws FileAlreadyExistsException, FileNotFoundException, InsufficientSpaceException, IOException, ReadNotPermittedException, UnknownFileErrorException, WriteNotPermittedException {
+	if(source.isDirectory()) {
+	    destination.mkdirs();
+	}
+	else {
+	    // Check if the origin file exists.
+	    if (!source.exists()) {
+		DebugStream.println("Couldn't find the source file.");
+		throw(new FileNotFoundException());
+	    }
+
+	    // Make sure the destination file does not exist.
+	    if (destination.exists()) {
+		throw(new FileAlreadyExistsException());
+	    }
+
+	    // Open an input stream to the source file
+	    FileInputStream f_in = null;
+	    try {
+		f_in = new FileInputStream(source);
+	    }
+	    catch (FileNotFoundException exception) {
+		// A FileNotFoundException translates into a ReadNotPermittedException in this case
+		throw new ReadNotPermittedException(exception.toString());		
+	    }
+
+	    // Create an necessary directories for the target file
+	    File dirs = destination.getParentFile();
+	    dirs.mkdirs();
+
+	    // Open an output stream to the target file
+	    FileOutputStream f_out = null;
+	    try {
+		f_out = new FileOutputStream(destination);
+	    }
+	    catch (FileNotFoundException exception) {
+		// A FileNotFoundException translates into a WriteNotPermittedException in this case
+		throw new WriteNotPermittedException(exception.toString());
+	    }
+
+	    // Copy the file
+	    byte data[] = new byte[Utility.BUFFER_SIZE];
+	    int data_size = 0;
+	    while((data_size = f_in.read(data, 0, Utility.BUFFER_SIZE)) != -1 && !cancel_action) {
+		long destination_size = destination.length();
+		try {
+		    f_out.write(data, 0, data_size);
+		}
+		// If an IO exception occurs, we can do some maths to determine if the number of bytes written to the file was less than expected. If so we assume a InsufficientSpace exception. If not we just throw the exception again.
+		catch (IOException io_exception) {
+		    if(destination_size + (long) data_size > destination.length()) {
+			// Determine the difference (which I guess is in bytes).
+			long difference = (destination_size + (long) data_size) - destination.length();
+			// Transform that into a human readable string.
+			String message = Utility.formatFileLength(difference);
+			throw(new InsufficientSpaceException(message));
+		    }
+		    else {
+			throw(io_exception);
+		    }
+		}
+		if(progress != null) {
+		    progress.addValue(data_size);
+		}
+	    }
+	    // Flush and close the streams to ensure all bytes are written.
+	    f_in.close();
+	    f_out.close();
+	    // We have now, in theory, produced an exact copy of the source file. Check this by comparing sizes.
+	    if(!destination.exists() || (!cancel_action && source.length() != destination.length())) {
+		throw(new UnknownFileErrorException());
+	    }
+	    // If we were cancelled, ensure that none of the destination file exists.
+	    if(cancel_action) {
+		destination.delete();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystem.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystem.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystem.java	(revision 21112)
@@ -0,0 +1,63 @@
+package org.greenstone.gatherer.file;
+
+import java.io.File;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+
+
+public class FileSystem
+{
+    static public WorkspaceTreeNode getLocalFilespaceNode(FileSystemModel model)
+    {
+	// Get all the available roots mounted on the filesystem
+	File[] roots = File.listRoots();
+	if (roots == null) {
+	    return null;
+	}
+
+	// If there is just one root use it as the tree root (Unix)
+	if (roots.length == 1) {
+	    return new WorkspaceTreeNode(roots[0], Dictionary.get("Tree.Root"));
+	}
+
+	// Otherwise build a dummy node which has the roots as children (Windows)
+	else {
+	    return new WorkspaceTreeNode(null, Dictionary.get("Tree.Root"));
+	}
+    }
+
+
+    static public WorkspaceTreeNode getHomeFolderNode()
+    {
+	// Get the name of the OS we're running on
+	String os_name = System.getProperty("os.name");
+	if (os_name == null) {
+	    return null;
+	}
+
+	// Make sure we're running on a multiuser OS where the idea of a "user home" is valid
+	DebugStream.println("Property os.name: " + os_name);
+	String os_name_lc = os_name.toLowerCase();
+	if (os_name_lc.equals("windows 95") || os_name_lc.equals("windows 98") || os_name_lc.equals("windows me")) {
+	    return null;
+	}
+
+	// Get the user's home folder
+ 	String home_folder_str = System.getProperty("user.home");
+	if (home_folder_str == null || home_folder_str.length() == 0) {
+	    return null;
+	}
+
+	// Generate a special mapping to the user home folder
+	File home_folder = new File(home_folder_str);
+	return new WorkspaceTreeNode(home_folder, Dictionary.get("Tree.Home", home_folder.getName()));
+    }
+
+
+    static public WorkspaceTreeNode getDownloadedFilesNode()
+    {
+	// Return a mapping to the downloaded files folder
+	return new WorkspaceTreeNode(new File(Gatherer.getGLIUserCacheDirectoryPath()), Dictionary.get("Tree.DownloadedFiles"));
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystemModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystemModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/FileSystemModel.java	(revision 21112)
@@ -0,0 +1,266 @@
+package org.greenstone.gatherer.file;
+
+import java.io.File;
+import java.util.*;
+import java.util.regex.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.util.SynchronizedTreeModelTools;
+
+public class FileSystemModel
+    extends DefaultTreeModel 
+    implements TreeExpansionListener, TreeWillExpandListener
+{
+    private DragTree tree = null;
+    private FileFilter current_filter = null;
+    private FileFilter[] filters = null;
+    /** The filters in place for any file system model. */
+    private FileFilter[] default_filters = { new FileFilter("\\..*", true), new FileFilter("metadata\\.xml", true) };
+
+    public FileSystemModel(FileNode root) {
+	super(root);
+	root.setModel(this);
+	root.map();
+    }
+
+    public int getChildCount(Object parent) {
+	return ((TreeNode)parent).getChildCount();
+    }
+
+    public FileFilter[] getFilters() {
+	if(filters == null) {
+	    if(current_filter != null) {
+		filters = new FileFilter[default_filters.length + 1];
+		filters[default_filters.length] = current_filter;
+	    }
+	    else {
+		filters = new FileFilter[default_filters.length];
+	    }
+	    System.arraycopy(default_filters, 0, filters, 0, default_filters.length);
+	}
+	return filters;
+    }
+	 
+    /** Retrieve the node denoted by the given tree path. Note that this isn't equivelent to saying path.lastPathComponent, as the references within the path may be stale. */
+    public FileNode getNode(TreePath path) {
+	FileNode last = (FileNode)path.getLastPathComponent();
+	DebugStream.println("Last Path Component = " + last);
+	return last;
+	/*
+	///atherer.println("**** getNode(" + path + ") ****");
+	FileNode current = (FileNode)root;
+	// Special case for the root node. Check the first path component is the root node.
+	FileNode first_node = (FileNode)path.getPathComponent(0);
+	if(current.equals(first_node)) {
+	    DebugStream.println("First path component matches root node.");
+	    // For each path with this tree path
+	    for(int i = 1; current != null && i < path.getPathCount(); i++) {
+		// Retrieve the stale path
+		Object stale_object = path.getPathComponent(i);
+		FileNode stale_node = null;
+		if(stale_object instanceof FileNode) {
+		    stale_node = (FileNode) stale_object;
+		}
+		DebugStream.print("Searching for '" + stale_object + "': ");
+		// Locate the fresh node by searching current's children. Remember to ensure that current is mapped.
+		boolean found = false;
+
+		// First we search through the mapped children
+		for(int j = 0; !found && j < current.getChildCount(); j++) {
+		    FileNode child_node = (FileNode) current.getChildAt(j);
+		    DebugStream.print(child_node + " ");
+		    if((stale_node != null && stale_node.equals(child_node)) || stale_object.toString().equals(child_node.toString())) {
+			found = true;
+			current = child_node;
+			DebugStream.println("Found!");
+		    }
+		    child_node = null;
+		}
+		// Failing that we search through all the children, including filtered files
+		for(int j = 0; !found && j < current.size(); j++) {
+		    FileNode child_node = (FileNode) current.getChildAtUnfiltered(j);
+		    DebugStream.print(child_node + " ");
+		    if((stale_node != null && stale_node.equals(child_node)) || stale_object.toString().equals(child_node.toString())) {
+			found = true;
+			current = child_node;
+			DebugStream.println("Found!");
+		    }
+		    child_node = null;
+		}
+		// If no match is found, then set current to null and exit.
+		if(!found) {
+		    current = null;
+		    DebugStream.println("Not Found!");
+		}
+		else {
+		    DebugStream.println("Returning node: " + new TreePath(current.getPath()));
+		}
+		// Repeat as necessary
+	    }
+	}
+	return current;
+	*/
+    }
+
+    public void insertNodeInto(MutableTreeNode newChild, MutableTreeNode parent, int index) {
+	///ystem.err.println("insertNodeInto(" + newChild + ", " + parent + ", " + index + ")");
+	super.insertNodeInto(newChild, parent, index);
+    }
+
+
+    public void refresh(TreePath path)
+    {
+	// Can only refresh if the model is currently being displayed in a tree
+	if (tree == null) {
+	    return;
+	}
+
+	// If no path is set, take the path to the root node (ie. update the whole tree)
+	if (path == null) {
+	    // System.err.println("\nFileSystemModel.refresh(entire tree).");
+	    path = new TreePath(((FileNode) root).getPath());
+
+	    // Make sure the root node is expanded
+	    tree.expandPath(path);
+	}
+	// else {
+	//     System.err.println("\nFileSystemModel.refresh(" + path + ").");
+	// }
+
+	// Record all the expanded paths under this node
+	Enumeration old_expanded_paths_enumeration = tree.getExpandedDescendants(path);
+	if (old_expanded_paths_enumeration == null) {
+	    return;
+	}
+
+	// Map and unmap the node to refresh its contents
+	FileNode node = (FileNode) path.getLastPathComponent();
+	node.refresh();
+
+	// Fire the appropriate event
+	nodeStructureChanged(node);
+
+	// Sort the old expanded paths by length, smallest first
+	ArrayList old_expanded_paths_list = Collections.list(old_expanded_paths_enumeration);
+	Collections.sort(old_expanded_paths_list, new TreePathComparator());
+
+	// Restore each of the expanded paths
+	for (int i = 0; i < old_expanded_paths_list.size(); i++) {
+  	    TreePath old_expanded_path = (TreePath) old_expanded_paths_list.get(i);
+	    // System.err.println("Expanded path: " + old_expanded_path);
+
+	    // Build up the new path in the tree
+	    TreePath current_path = new TreePath(path.getPath());
+  	    FileNode current_node = node;
+
+	    // Traverse the tree to find the node to expand (or find it no longer exists)
+	    while (!current_path.toString().equals(old_expanded_path.toString())) {
+		// System.err.println("Current path: " + current_path);
+
+		FileNode old_expanded_node =
+		    (FileNode) old_expanded_path.getPathComponent(current_path.getPathCount());
+		// System.err.println("Looking for: " + old_expanded_node);
+
+		// Find the child node that matches the next element in the path
+		boolean found = false;
+		for (int j = 0; j < current_node.getChildCount(); j++) {
+		    FileNode child_node = (FileNode) current_node.getChildAt(j);
+		    // System.err.println("Child node: " + child_node);
+		    if (child_node.equals(old_expanded_node)) {
+			// System.err.println("Found!");
+			current_path = current_path.pathByAddingChild(child_node);
+			current_node = child_node;
+			found = true;
+			break;
+		    }
+		}
+
+		// The node was not found, so we cannot expand this path
+		if (!found) {
+		    // System.err.println("Not found...");
+		    break;
+		}
+	    }
+
+	    // If we have built up the correct path, expand it
+	    if (current_path.toString().equals(old_expanded_path.toString())) {
+		tree.expandPath(current_path);
+	    }
+	}
+    }
+
+
+    private class TreePathComparator
+	implements Comparator {
+
+	public int compare(Object o1, Object o2)
+	{
+	    return (((TreePath) o1).getPathCount() - ((TreePath) o2).getPathCount());
+	}
+    }
+
+
+    public void setFilter(String pattern) {
+  	if(pattern != null) {
+  	    current_filter = new FileFilter(pattern, false);
+  	}
+  	else {
+  	    current_filter = null;
+  	}
+  	filters = null;
+    }
+
+    public void setTree(DragTree tree) {
+	this.tree = tree;
+    }
+
+    public String toString() {
+	if(tree != null) {
+	    return tree.toString();
+	}
+	return "FileSystemModel";
+    }
+
+    /** Called whenever an item in the tree has been collapsed. */
+    public void treeCollapsed(TreeExpansionEvent event) {
+	// Deallocate the affected nodes children. Don't need to do this in a swing worker, as the nodes children are currently not visable.
+	TreePath path = event.getPath();
+	FileNode node = (FileNode) path.getLastPathComponent();
+	node.unmap();
+	// Fire the appropriate event.
+	nodeStructureChanged(node);
+    }
+
+    /** Called whenever an item in the tree has been expanded. */
+    public void treeExpanded(TreeExpansionEvent event) {
+    }
+
+    /** Invoked whenever a node in the tree is about to be collapsed. */
+    public void treeWillCollapse(TreeExpansionEvent event) 
+	throws ExpandVetoException {
+	// Veto the event if the user is attempting to collapse the root node (regardless of whether it is visible).
+	TreePath path = event.getPath();
+	if(path.getPathCount() == 1) {
+	    throw new ExpandVetoException(event, "Cannot collapse root node!");
+	}
+    }
+
+    /** Invoked whenever a node in the tree is about to be expanded. */
+    public void treeWillExpand(TreeExpansionEvent event) 
+	throws ExpandVetoException {
+	// Set the wait cursor.
+	Gatherer.g_man.wait(true);
+	// Allocate the children. Don't need to do this in a swing worker, as the nodes children are currently not visable.
+	TreePath path = event.getPath();
+	FileNode node = (FileNode) path.getLastPathComponent();
+	///ystem.err.println("Mapping: " + node);
+	node.map();
+	///ystem.err.println(" -> node has " + node.getChildCount() + " children");
+	nodeStructureChanged(node);
+	// Restore the cursor.
+	Gatherer.g_man.wait(false);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/InsufficientSpaceException.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/InsufficientSpaceException.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/InsufficientSpaceException.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gatherer.file;
+
+import java.lang.Exception;
+
+public class InsufficientSpaceException
+    extends Exception {
+
+    public InsufficientSpaceException(String message) {
+	super(message);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/ReadNotPermittedException.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/ReadNotPermittedException.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/ReadNotPermittedException.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gatherer.file;
+
+import java.lang.Exception;
+
+public class ReadNotPermittedException
+    extends Exception
+{
+    public ReadNotPermittedException(String message) {
+	super(message);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/RecycleBin.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/RecycleBin.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/RecycleBin.java	(revision 21112)
@@ -0,0 +1,188 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.file;
+
+import java.awt.*;
+import java.awt.datatransfer.Transferable;
+import java.awt.dnd.*;
+import java.awt.event.*;
+import java.awt.geom.AffineTransform;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.DragComponent;
+import org.greenstone.gatherer.util.DragGroup;
+import org.greenstone.gatherer.util.JarTools;
+
+
+public class RecycleBin
+    extends JButton
+    implements DragComponent, DropTargetListener {
+
+    private boolean ignore = false;
+    /** The group encompasses all of the objects you plan to drag and drop within, and ensures that only one has focus (as clearly identified by the colour of the selection field or, in this particular case, the background) and that actions only occur between components in the same group. */
+    private DragGroup group;
+    /** In order to make this button a drop target we have to create a DropTarget instance with the button as its target. */
+    private DropTarget drop_target;
+    private FileSystemModel model;
+    /** What sort of action should a drag resemble. Not really used as we override with custom drag icon. */
+    private int drag_action = DnDConstants.ACTION_MOVE;
+    /** The last point the mouse was at. Used to repaint 'spoilt' area. */
+    private Point pt_last = null;
+    /** The area covered by the drag ghost, our custom drag icon. */
+    private Rectangle ra_ghost = new Rectangle();
+
+
+    public RecycleBin()
+    {
+	super(JarTools.getImage("bin.gif"));
+	this.drop_target = new DropTarget(this, drag_action, this, true);
+
+	setBackground(Configuration.getColor("coloring.button_background", true));
+	setForeground(Configuration.getColor("coloring.button_foreground", true));
+	setOpaque(true);
+
+	this.model = new FileSystemModel(new RecycleBinNode(new File(Gatherer.getGLIUserDirectoryPath() + "recycle")));
+    }
+
+
+    /** In order for the appearance to be consistant, given we may be in the situation where the pointer has left our focus but the ghost remains, this method allows other members of the GGroup to tell this component to repair the 'spoilt' region left by its ghost. */
+    public void clearGhost(){
+    }
+
+
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the drag focus enters this component. We want to provide some sort of indication whether the current component is an acceptable drop target as well as indicating focus. */
+    public void dragEnter(DropTargetDragEvent event) {
+	group.grabFocus(this);
+	setBackground(Configuration.getColor("coloring.button_selected_background", true));
+	setForeground(Configuration.getColor("coloring.button_selected_foreground", true));
+    }
+
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the drag focus leaves this component. We need to indicate that we have lost focus. */
+    public void dragExit(DropTargetEvent event) {
+	setBackground(Configuration.getColor("coloring.button_background", true));
+	setForeground(Configuration.getColor("coloring.button_foreground", true));
+    }
+
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the drag moves in this component. This is where we repaint our ghost icon at the tip of the mouse pointer. */
+    public void dragOver(DropTargetDragEvent event) {
+	Graphics2D g2 = (Graphics2D) getGraphics();
+	Point pt = event.getLocation();
+	if(pt_last != null && pt.equals(pt_last)) {
+	    return;
+	}
+	pt_last = pt;
+	if(!DragSource.isDragImageSupported()) {
+	    // Erase the last ghost image and or cue line
+	    paintImmediately(ra_ghost.getBounds());
+	    // Remember where you are about to draw the new ghost image
+	    ra_ghost.setRect(pt.x - group.mouse_offset.x, pt.y - group.mouse_offset.y, group.image_ghost.getWidth(), group.image_ghost.getHeight());
+	    // Draw the ghost image
+	    g2.drawImage(group.image_ghost, AffineTransform.getTranslateInstance(ra_ghost.getX(), ra_ghost.getY()), null);
+	}
+    }
+
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the drag ends, ie the transferable is dropped. This in turn triggers a series of add events preceded by a pre() and followed by a post(). */
+    public void drop(DropTargetDropEvent event) {
+	ignore = true;
+	group.grabFocus(this);
+	setBackground(Configuration.getColor("coloring.button_background", true));
+	setForeground(Configuration.getColor("coloring.button_foreground", true));
+	Transferable transferable = event.getTransferable();
+	try {
+	    DragComponent source = group.getSource();
+	    TreePath[] selection = group.getSelection();
+	    FileNode[] source_nodes = new FileNode[selection.length];
+	    for(int i = 0; i < source_nodes.length; i++) {
+		source_nodes[i] = (FileNode) selection[i].getLastPathComponent();
+	    }
+	    event.acceptDrop(drag_action);
+	    // Action delete
+	    Gatherer.f_man.action(source, source_nodes, this, null);
+	    group.setSource(null);
+	    group.setSelection(null);
+	}
+	catch(Exception error) {
+	    error.printStackTrace();
+	    event.rejectDrop();
+	}
+	ignore = false;
+	// Clear up the group.image_ghost
+	paintImmediately(ra_ghost.getBounds());
+	event.getDropTargetContext().dropComplete(true);
+    }
+
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the action to be taken upon drop changes. We never change so we don't do anything here. */
+    public void dropActionChanged(DropTargetDragEvent event) {
+    }
+
+    /** Used to notify this component that it has gained focus by some method other that mouse focus. */
+    public void gainFocus() {
+    }
+
+    /** Any implementation of DragComponent must include this method so that a outsider can get at the underlying tree model behind the component. */
+    public FileSystemModel getTreeModel(){
+  	return (FileSystemModel) model;
+    }
+
+    public boolean ignore() {
+	return ignore;
+    }
+
+    /** This method is used to inform this component when it loses focus by means other than a drag mouse event, and should indicate this somehow. */
+    public void loseFocus() {
+    }
+
+    public void setGroup(DragGroup group) {
+	this.group = group;
+    }
+
+
+    public class RecycleBinNode
+	extends FileNode
+    {
+	public RecycleBinNode(File file)
+	{
+	    super(file);
+	}
+
+	public FileNode addChildNode(File file) { return null; }
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/UnknownFileErrorException.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/UnknownFileErrorException.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/UnknownFileErrorException.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gatherer.file;
+
+import java.lang.Exception;
+
+public class UnknownFileErrorException
+    extends Exception {
+
+    public UnknownFileErrorException() {
+	super();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTree.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTree.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTree.java	(revision 21112)
@@ -0,0 +1,143 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.file;
+
+import java.io.File;
+import java.util.Enumeration;
+import javax.swing.tree.TreePath;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.tree.DragTree;
+
+
+public class WorkspaceTree
+    extends DragTree
+{
+    static public final int LIBRARY_CONTENTS_CHANGED = 10;
+    static public final int DOWNLOADED_FILES_CHANGED = 11;
+    static public final int FOLDER_SHORTCUTS_CHANGED = 12;
+
+
+    public WorkspaceTree(String name)
+    {
+	super(name, WorkspaceTreeModel.getWorkspaceTreeModel(), true);
+    }
+
+
+    public void addDirectoryMapping(String name, File file)
+    {
+	// Update the information stored in the user's configuration
+	Configuration.addDirectoryMapping(name, file);
+
+	// Now update the tree
+	refresh(FOLDER_SHORTCUTS_CHANGED);
+    }
+
+
+    public void refresh(int refresh_reason)
+    {
+	DebugStream.println("WorkspaceTree::refresh()... ");
+
+	// The method for displaying the tree has changed - redraw the tree
+  	if (refresh_reason == DragTree.TREE_DISPLAY_CHANGED) {
+  	    DebugStream.println("...Reason: tree display changed.");
+  	    updateUI();
+  	}
+
+	// The loaded collection has changed - currently do nothing
+	if (refresh_reason == DragTree.LOADED_COLLECTION_CHANGED) {
+	    DebugStream.println("...Reason: loaded collection changed.");
+	}
+
+	// The collection's contents have changed - refresh collection's import folder
+	if (refresh_reason == DragTree.COLLECTION_CONTENTS_CHANGED) {
+	    DebugStream.println("...Reason: collection contents changed.");
+	    String collection_import_directory_path = Gatherer.c_man.getCollectionImportDirectoryPath();
+	    refreshEveryNodeShowingFolder(collection_import_directory_path);
+	}
+
+	// The collections in the library have changed - refresh the collect directory
+	if (refresh_reason == WorkspaceTree.LIBRARY_CONTENTS_CHANGED) {
+	    DebugStream.println("...Reason: library contents changed.");
+	    String collect_directory_path = Gatherer.getCollectDirectoryPath();
+	    refreshEveryNodeShowingFolder(collect_directory_path);
+	    WorkspaceTreeModel.refreshGreenstoneCollectionsNode();
+	}
+
+	// The downloaded files have changed - refresh that node
+	if (refresh_reason == WorkspaceTree.DOWNLOADED_FILES_CHANGED) {
+	    DebugStream.println("...Reason: downloaded files changed.");
+	    WorkspaceTreeModel.refreshDownloadedFilesNode();
+	}
+
+	// The folder shortcuts have changed - refresh only them
+	if (refresh_reason == WorkspaceTree.FOLDER_SHORTCUTS_CHANGED) {
+	    DebugStream.println("...Reason: folder shortcuts changed.");
+	    WorkspaceTreeModel.refreshFolderShortcuts();
+	}
+    }
+
+
+    private void refreshEveryNodeShowingFolder(String folder_path_str)
+    {
+	// Search through the expanded tree paths, checking if any show the given folder
+	TreePath tree_root_path = new TreePath(getModel().getRoot());
+	Enumeration expanded_tree_paths = getExpandedDescendants(tree_root_path);
+	while (expanded_tree_paths.hasMoreElements()) {
+	    TreePath expanded_tree_path = (TreePath) expanded_tree_paths.nextElement();
+	    WorkspaceTreeNode tree_node = (WorkspaceTreeNode) expanded_tree_path.getLastPathComponent();
+
+	    File tree_node_file = tree_node.getFile();
+	    if (tree_node_file == null) {
+		continue;
+	    }
+
+	    // Get the path of the open tree node
+	    String tree_node_path_str = tree_node_file.toString();
+	    if (!tree_node_path_str.endsWith(File.separator)) {
+		tree_node_path_str = tree_node_path_str + File.separator;
+	    }
+
+	    // If the specified folder is open, it must be refreshed
+	    if (tree_node_path_str.equals(folder_path_str)) {
+		System.err.println("Must refresh node " + tree_node_path_str + "!");
+		((WorkspaceTreeModel) getModel()).refresh(expanded_tree_path);
+	    }
+	}
+    }
+
+
+    public void removeDirectoryMapping(WorkspaceTreeNode target)
+    {
+	// Update the information stored in the user's configuration
+	Configuration.removeDirectoryMapping(target.toString());
+
+	// Update tree
+	refresh(FOLDER_SHORTCUTS_CHANGED);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeModel.java	(revision 21112)
@@ -0,0 +1,179 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.file;
+
+import java.io.*;
+import java.util.*;
+import javax.swing.tree.TreePath;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.SynchronizedTreeModelTools;
+
+
+public class WorkspaceTreeModel
+    extends FileSystemModel
+{
+    static private WorkspaceTreeModel workspace_tree_model = null;
+    static private WorkspaceTreeNode workspace_tree_root = null;
+
+    static private WorkspaceTreeNode greenstone_collections_node = null;
+    static private WorkspaceTreeNode local_filespace_node = null;
+    static private WorkspaceTreeNode downloaded_files_node = null;
+    static private WorkspaceTreeNode[] folder_shortcuts = null;
+
+
+    public WorkspaceTreeModel(WorkspaceTreeNode root_node)
+    {
+	super(root_node);
+    }
+
+
+    static public WorkspaceTreeNode[] getFolderShortcuts()
+    {
+	// Return any predefined special directories
+	HashMap mappings = Configuration.getDirectoryMappings();
+	WorkspaceTreeNode[] mapping_nodes = new WorkspaceTreeNode[mappings.size()];
+	Iterator mappings_iterator = mappings.keySet().iterator();
+	for (int i = 0; mappings_iterator.hasNext(); i++) {
+	    String mapping_name = (String) mappings_iterator.next();
+	    File mapping_file = (File) mappings.get(mapping_name);
+	    mapping_nodes[i] = new WorkspaceTreeNode(mapping_file, mapping_name);
+	}
+	return mapping_nodes;
+    }
+
+
+    static public WorkspaceTreeModel getWorkspaceTreeModel()
+    {
+	// Create a root node to contain the various nodes in the workspace tree
+ 	workspace_tree_root = new WorkspaceTreeNode(null, "ABS_ROOT");
+	workspace_tree_model = new WorkspaceTreeModel(workspace_tree_root);
+
+	// Add the "Documents in Greenstone Collections" node
+	greenstone_collections_node = new WorkspaceTreeNode(null, Dictionary.get("Tree.World"));
+	workspace_tree_root.add(greenstone_collections_node);
+
+	// Add the "Local Filespace" node
+	local_filespace_node = FileSystem.getLocalFilespaceNode(workspace_tree_model);
+	workspace_tree_root.add(local_filespace_node);
+
+	// Add the "Home Folder" node
+	workspace_tree_root.add(FileSystem.getHomeFolderNode());
+
+	// Add the "Downloaded Files" node, if the Download pane is enabled
+	if (Gatherer.g_man.download_pane != null) {
+	    downloaded_files_node = FileSystem.getDownloadedFilesNode();
+	    workspace_tree_root.add(downloaded_files_node);
+	}
+
+	// Add any folder shortcuts the user has created
+	refreshFolderShortcuts();
+
+ 	return workspace_tree_model;
+    }
+
+
+    static public void refreshGreenstoneCollectionsNode()
+    {
+	greenstone_collections_node.refresh();
+    }
+
+
+    static public void refreshDownloadedFilesNode()
+    {
+	downloaded_files_node.refresh();
+    }
+
+
+    static public void refreshFolderShortcuts()
+    {
+	// Check for new/deleted folder shortcuts
+	WorkspaceTreeNode[] old_folder_shortcuts = folder_shortcuts;
+	folder_shortcuts = getFolderShortcuts();
+
+	// Remove any deleted shortcuts from the tree
+	if (old_folder_shortcuts != null) {
+	    for (int i = 0; i < old_folder_shortcuts.length; i++) {
+		if (!doesArrayContain(folder_shortcuts, old_folder_shortcuts[i])) {
+		    DebugStream.println("Deleted shortcut: " + old_folder_shortcuts[i]);
+		    SynchronizedTreeModelTools.removeNodeFromParent(workspace_tree_model, old_folder_shortcuts[i]);
+		}
+	    }
+	}
+
+	// Add any new shortcuts to the tree
+	if (folder_shortcuts != null) {
+	    for (int i = 0; i < folder_shortcuts.length; i++) {
+		if (!doesArrayContain(old_folder_shortcuts, folder_shortcuts[i])) {
+		    DebugStream.println("Added shortcut: " + folder_shortcuts[i]);
+		    SynchronizedTreeModelTools.insertNodeInto(workspace_tree_model, workspace_tree_root, folder_shortcuts[i], false);
+		}
+	    }
+	}
+    }
+
+
+    // -- This code is necessary to support the workspace tree file filter --
+    public void refresh(TreePath path)
+    {
+	// If we're not refreshing the whole tree just refresh a certain path
+	if (path != null) {
+	    super.refresh(path);
+	    return;
+	}
+
+	// Refresh each of the nodes in the workspace tree
+	for (int i = 0; i < workspace_tree_root.getChildCount(); i++) {
+	    WorkspaceTreeNode child_node = (WorkspaceTreeNode) workspace_tree_root.getChildAt(i);
+	    super.refresh(new TreePath(child_node.getPath()));
+	}
+
+	// Refresh the local filespace tree specially (it is not unmapped so not refreshed)
+	for (int i = 0; i < local_filespace_node.getChildCount(); i++) {
+	    WorkspaceTreeNode child_node = (WorkspaceTreeNode) local_filespace_node.getChildAt(i);
+	    super.refresh(new TreePath(child_node.getPath()));
+	}
+    }
+
+
+    static private boolean doesArrayContain(Object[] array, Object item)
+    {
+	if (array == null) {
+	    return false;
+	}
+
+	for (int i = 0; i < array.length; i++) {
+	    if (item.toString().equals(array[i].toString())) {
+		return true;
+	    }
+	}
+
+	return false;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeNode.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeNode.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WorkspaceTreeNode.java	(revision 21112)
@@ -0,0 +1,196 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.file;
+
+
+import java.io.*;
+import java.util.*;
+import javax.swing.filechooser.FileSystemView;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;  // !!! Don't like this here
+import org.greenstone.gatherer.collection.Collection;  // !!! Don't like this here
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+
+/** This class represents one node in the workspace tree. */
+public class WorkspaceTreeNode
+    extends FileNode
+{
+    private boolean is_gs3_site_node = false;
+    private boolean is_in_loaded_collection = false;
+    private String title = null;
+
+
+    public WorkspaceTreeNode(File file)
+    {
+	super(file);
+    }
+
+
+    public WorkspaceTreeNode(File file, String title)
+    {
+	this(file);
+	this.title = title;
+    }
+
+
+    public FileNode addChildNode(File file)
+    {
+	WorkspaceTreeNode child_node = new WorkspaceTreeNode(file);
+	child_node.setModel(model);
+	child_node.setParent(this);
+	return child_node;
+    }
+
+
+    /** Is this file node within the currently loaded collection? */
+    public boolean isInLoadedCollection()
+    {
+	if (is_in_loaded_collection) {
+	    return true;
+	}
+	else {
+	    FileNode parent = (FileNode) getParent();
+	    if (parent != null) {
+		return parent.isInLoadedCollection();
+	    }
+	}
+	return false;
+    }
+
+
+    public boolean isReadOnly()
+    {
+	// The workspace tree is read only
+	return true;
+    }
+
+
+    public void map()
+    {
+	// Special Case: "Documents in Greenstone Collections" -- map the collections installed in Greenstone
+	if (file == null) { // a special mapping folder
+	    if (child_nodes != null) {
+		// don't bother mapping again if we already have children
+		return;
+	    }
+	    child_nodes = new ArrayList();
+	    if (title.equals(Dictionary.get("Tree.World")) && Gatherer.GS3) {
+		// the Greenstone collections folder for GS3 - this contains a 
+		// folder for each site
+
+		File start = new File(Utility.getSitesDir(Configuration.gsdl3_path));
+		File sites[] = start.listFiles();
+		ArrayTools.sort(sites);
+		for (int i = 0; sites != null && i < sites.length; i++) {
+		    File collect_dir = new File(sites[i], "collect");
+		    if (!collect_dir.exists()) {
+			continue;
+		    }
+
+		    WorkspaceTreeNode child = new WorkspaceTreeNode(null, sites[i].getName());
+		    child.is_gs3_site_node = true;
+		    child.unmap();
+		    child.setModel(model);
+		    child.setParent(this);
+		    child.map();
+		    child_nodes.add(child);
+		}
+		model.nodeStructureChanged(this);
+		
+	    } else if (title.equals(Dictionary.get("Tree.World")) || is_gs3_site_node) {
+		// For each of the children directories, which are collections...
+		File start;
+		if (is_gs3_site_node) {
+		    start = new File(Gatherer.getSitesDirectoryPath() + title + File.separator + "collect" + File.separator);
+		} else {
+		    start = new File(Gatherer.getCollectDirectoryPath());
+		}
+		File cols[] = start.listFiles();
+		ArrayTools.sort(cols);
+
+		// We add their import directories, except for the model collection
+		for (int i = 0; cols != null && i < cols.length; i++) {
+		    if (!cols[i].getName().equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+			File import_dir = new File(cols[i], StaticStrings.IMPORT_FOLDER);
+			if (!import_dir.exists()) {
+			    continue;
+			}
+
+			// we don't care if there is no config file
+			BasicCollectionConfiguration collect_cfg = new BasicCollectionConfiguration(new File(cols[i], Utility.CONFIG_FILE));
+			WorkspaceTreeNode collection_root = new WorkspaceTreeNode(import_dir, collect_cfg.toString());
+			collection_root.setParent(this);
+			collection_root.setModel(model);
+
+			// One last piece of magic so we can determine the current collection
+			Collection collection = Gatherer.c_man.getCollection();
+			if (collection != null) {
+			    collection_root.is_in_loaded_collection = cols[i].getName().equals(collection.getName());
+			}
+			child_nodes.add(collection_root);
+		    }
+		}
+		model.nodeStructureChanged(this);
+	    }
+	    else if (title.equals(Dictionary.get("Tree.Root"))) {
+		// Special Case: "Local Filespace" on Windows -- create a node for each filesystem root (drive letter)
+		
+		// Sort the roots into alphabetical order
+		File[] roots = File.listRoots();
+		ArrayTools.sort(roots);
+		for (int i = 0; i < roots.length; i++) {
+		    // Only add root if it isn't a floppy drive
+		    // this used to cause problems, I don't think it does now...
+		    //if (!FileSystemView.getFileSystemView().isFloppyDrive(roots[i])) {
+		    child_nodes.add(addChildNode(roots[i]));
+			// }
+		}
+	    }
+	} // if file == null
+	
+	// General case
+	else {
+	    super.map();
+	}
+    }
+
+
+    public String toString()
+    {
+	if (title != null) {
+	    return title;
+	}
+
+	return super.toString();
+    }
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WriteNotPermittedException.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WriteNotPermittedException.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/file/WriteNotPermittedException.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gatherer.file;
+
+import java.lang.Exception;
+
+public class WriteNotPermittedException
+    extends Exception {
+
+    public WriteNotPermittedException(String message) {
+	super(message);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/Attribute.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/Attribute.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/Attribute.java	(revision 21112)
@@ -0,0 +1,85 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+public class Attribute 
+    implements Comparable {
+    public String language = null;
+    public String name = null;
+    public String value = null;
+    public String text = null;
+    private boolean language_specific = false;
+    public Attribute(String name, String value) {
+	this.name = name;
+	this.value = value;
+    }
+    public Attribute(String name, String language, String value) {
+	this.language = language;
+	this.language_specific = true;
+	this.name = name;
+	this.value = value;
+    }
+    public int compareTo(Object other) {
+	return toString().compareTo(other.toString());
+    }
+    public boolean equals(Object other) {
+	return toString().equals(other.toString());
+    }
+    public String get(int column) {
+	switch(column) {
+	case 0:
+	    return name;
+	case 1:
+	    if(language_specific) {
+		return language;
+	    }
+	default:
+	    return value;
+	}
+    }
+    public String toString() {
+	if(text == null) {
+	    if(language_specific) {
+		text = name + " [" + language + "] = " + value;
+	    }
+	    else {
+		text = name + " = " + value;
+	    }
+	}
+	return text;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/AttributeTableModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/AttributeTableModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/AttributeTableModel.java	(revision 21112)
@@ -0,0 +1,244 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.util.Utility;
+
+
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class AttributeTableModel 
+    extends AbstractTableModel {
+    private Attribute last_attribute = null;
+    private int last_index = -1;
+    private int longest[] = null;
+    private int column_zero_width = 0;
+    private JScrollPane scroll_pane = null;
+    private JTable table = null;
+    private String default_value = null;
+    private String names[] = null;
+    private TreeSet attributes = null;
+    static final private int table_width = 550;
+    public AttributeTableModel(TreeSet attributes, String one, String two, String default_value) {
+	super();
+	this.attributes = attributes;
+	this.default_value = default_value;
+	this.longest = new int[2];
+	this.names = new String[2];
+	names[0] = one;
+	names[1] = two;
+    }
+
+    public AttributeTableModel(TreeSet attributes, String one, String two, String three, String default_value) {
+	super();
+	this.attributes = attributes;
+	this.default_value = default_value;
+	this.longest = new int[3];
+	this.names = new String[3];
+	names[0] = one;
+	names[1] = two;
+	names[2] = three;
+    }
+    public int add(Attribute attribute) {
+	attributes.add(attribute);
+	// Determine the row index
+	boolean found = false;
+	int index = 0;
+	Iterator iterator = attributes.iterator();
+	for(int i = 0; !found && iterator.hasNext(); i++) {
+	    if(attribute == iterator.next()) {
+		found = true;
+		index = i;
+	    }
+	}
+	// Fire event
+	fireTableRowsInserted(index, index);
+	return index;
+    }
+
+    /**
+     * Checks to see if the table contains a given value in a given column
+     *
+     * @param value the <strong>String</strong> to check for
+     * @param column the column number to check in, as an <strong>int</strong>
+     * @return boolean - true if the table contains the value.
+     */
+    public boolean contains(String value, int column) {
+	boolean result = false;
+	Iterator iterator = attributes.iterator();
+	while(iterator.hasNext() && !result) {
+	    Attribute attribute = (Attribute) iterator.next();
+	    if(value.equals(attribute.get(column))) {
+		result = true;
+	    }
+	}  
+	return result;
+    }
+
+    /**
+     * Checks to see if the table contains a given value in a given language
+     *
+     * @param value the <strong>String</strong> to check for
+     * @param language_code the language code to check for, as a <strong>String</strong>
+     * @return boolean - true if the table contains the value.
+     */
+    public boolean contains(String value, String language_code) {
+	Iterator iterator = attributes.iterator();
+	while(iterator.hasNext()) {
+	    Attribute attribute = (Attribute) iterator.next();
+	    String attributeLanguage = attribute.language;
+	    if(attributeLanguage == null) {
+		attributeLanguage = "";
+	    }	    
+	    if(language_code == null) {
+		language_code = "";
+	    }
+	  
+	    if(value.equals(attribute.name) && language_code.equalsIgnoreCase(attribute.language)) {
+		return true;
+	    }
+	}
+	return false;
+    }
+
+    public Attribute getAttribute(int row) {
+	if(row != last_index) {
+	    last_index = row;
+	    last_attribute = null;
+				// Retrieve the value from the set using its iterator
+	    Iterator iterator = attributes.iterator();
+	    for(int i = 0; i < row && iterator.hasNext(); i++) {
+		iterator.next();
+	    }
+	    if(iterator.hasNext()) {
+		last_attribute = (Attribute) iterator.next();
+	    }
+	    iterator = null;
+	}
+	return last_attribute;
+    }
+
+    public int getColumnCount() {
+	return names.length;
+    }
+
+    public String getColumnName(int col) {
+	return names[col];
+    }
+
+    public int getRowCount() {
+	int count = 0;
+	if(attributes != null) {
+	    count = attributes.size();
+	}
+	return count;
+    }
+
+    public Object getValueAt(int row, int col) {
+	String value = null;
+	Attribute attribute = getAttribute(row);
+	if(attribute != null) {
+	    value = attribute.get(col);
+	}
+	if(value == null) {
+	    value = default_value;
+	}
+	// Update table column width if necessary
+	if(value.length() > longest[col]) {
+	    Component component = table.getCellRenderer(row, col).getTableCellRendererComponent(table, value, false, false, 0, 0);
+	    int width = component.getPreferredSize().width + 10;
+	    TableColumn column = table.getColumnModel().getColumn(col);
+	    if(width > column.getPreferredWidth()) {
+		// Calculate default column width
+		int default_width = 0;
+		int total_width = scroll_pane.getSize().width;
+		if(names.length == 2) {
+		    if(col == 0) {
+			default_width = total_width / 3;
+		    }
+		    else {
+			default_width = 2 * (total_width / 3);
+		    }
+		}
+		else {
+		    switch(col) {
+		    case 0:
+			default_width = (2 * total_width) / 9;
+			break;
+		    case 1:
+			default_width = total_width / 9;
+			break;
+		    default:
+			default_width = 2 * (total_width / 3);
+			break;
+		    }
+		}
+		if(width > default_width) {
+		    column.setPreferredWidth(width);
+		}
+		else {
+		    column.setPreferredWidth(default_width);
+		}
+	    }
+	    longest[col] = value.length();
+	}
+	return Utility.stripNL(value);
+    }
+    public void removeRow(final int row) {
+	///ystem.err.println("Remove row " + row);
+	Attribute attribute = getAttribute(row);
+	if(attribute != null) {
+	    attributes.remove(attribute);
+				// Arg... this caused me a headache for hours.
+	    last_index = -1;
+	    fireTableRowsDeleted(row, row);
+	}
+    }
+    public void setScrollPane(JScrollPane scroll_pane) {
+	this.scroll_pane = scroll_pane;
+    }
+    public void setTable(JTable table) {
+	this.table = table;
+	this.longest = new int[getColumnCount()];
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ComboArea.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ComboArea.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ComboArea.java	(revision 21112)
@@ -0,0 +1,202 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.util.Utility;
+
+/** Combines the idea of a combobox with a text area so that multiline entries can be displayed properly. Basically takes a GCombobox, butchers out the JTextField editor while retaining the button and the data model, then uses a JTextArea for the editor instead. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class ComboArea
+    extends JPanel {
+    /** Should the listener ignore any change events fired (most likely as a result of the listener in the first place). */
+    private boolean ignore = false;
+    /** The combobox we are going to utilize. */
+    private GComboBox combobox = null;
+    /** The text area involved. Not quite a JTextArea, the smarter part is that is allows coloring contrary to the UIManager settings. */
+    private SmarterTextArea text_area = null;
+    /** Create a new ComboArea demarked by a particular label. The label is given here so that it fits nicely with the text area and button.
+     * @param text The label text to be shown above text area as a <strong>String</strong>.
+     * @param label_size The <strong>Dimension</strong> to use for the label.
+     * @see org.greenstone.gatherer.gui.ComboArea.ComboAreaListener
+     * @see org.greenstone.gatherer.gui.ComboArea.Renderer
+     */
+    public ComboArea(String text, Dimension label_size) {
+	// Creation
+	JPanel title_pane = new JPanel();
+	title_pane.setOpaque(false);
+	JLabel label = new JLabel(text);
+	label.setOpaque(false);
+	label.setPreferredSize(label_size);
+	JPanel text_pane = new JPanel();
+	text_area = new SmarterTextArea();
+	text_area.setRows(3);
+	JPanel button_pane = new JPanel();
+	Dimension button_size = new Dimension(label_size.height, label_size.height);
+	button_pane.setMaximumSize(button_size);
+	button_pane.setMinimumSize(button_size);
+	button_pane.setPreferredSize(button_size);
+	button_pane.setSize(button_size);
+	combobox = new GComboBox();
+	combobox.setPreferredSize(label_size);
+	combobox.setRenderer(new Renderer());
+	// Connection
+	combobox.addActionListener(new ComboAreaListener());
+	// Layout
+	title_pane.setLayout(new BorderLayout());
+	title_pane.add(label, BorderLayout.WEST);
+
+	text_pane.setLayout(new BorderLayout());
+	text_pane.add(new JScrollPane(text_area), BorderLayout.CENTER);
+
+	setLayout(new BorderLayout());
+	add(title_pane, BorderLayout.NORTH);
+	add(text_pane, BorderLayout.CENTER);
+    }
+
+    /** Add an element to our combobox model. Remember to tell the listeners to ignore this action unless you want an infinite loop.
+     * @param item The <strong>Object</strong> to add.
+     */
+    public void add(Object item) {
+	ignore = true;
+	combobox.add(item.toString());
+	ignore = false;
+    }
+
+    /** Clear the combobox model.
+     */
+    public void clear() {
+	combobox.clear();
+    }
+
+    /** Retrieve the text currently displayed in the text area.
+     * @return A <strong>String</strong> containing the required text.
+     */
+    public String getText() {
+	return text_area.getText();
+    }
+
+    /** Retrieve a reference to the text area.
+     * @return A <strong>JTextComponent</strong> reference.
+     */
+    public JTextComponent getTextComponent() {
+	return text_area;
+    }
+
+    /** Change the selected item in the combobox to the one given. If such an object doesn't exist in the model, ignore it. Notice that this time we don't ignore, as we want a selection to update the text field.
+     * @param item The <strong>Object</strong> that we want to select from the combobox.
+     */
+    public void setSelectedItem(Object item) {
+	combobox.setSelectedItem(item);
+    }
+
+    /** Set the text shown in the text area to the value provided. Note that this has no effect on the combobox, nor will this value be stored in the model.
+     * @param text The <strong>String</strong> to display.
+     */
+    public void setText(String text) {
+	text_area.setText(text);
+    }
+
+    /** This clas listens for changes in the selected item in the combobox and if applicable updates the text area to match. */
+    private class ComboAreaListener
+	implements ActionListener {
+	/** Whenever a new object is selected from the combobox list, determine if this is an appropriate time to update the text area and if so do so.
+	 * @param event An <strong>ActionEvent</strong> encompassing all the relevant data about an action having been performed.
+			*/
+	public void actionPerformed(ActionEvent event) {
+	    Object object = combobox.getSelectedItem();
+	    if(object != null && !ignore) {
+		text_area.setText(object.toString());
+	    }
+	}
+    }
+    /** This renderer is used to ensure that the labels shown in the combobox drop-down list are both correct in colouring and legible. The later is important as the entry itself may actually span several lines. */
+    private class Renderer
+	extends DefaultListCellRenderer {
+	/** Retrieve the component used to 'rubberstamp' the desired entry in the combobox.
+	 * @param list The <strong>JList</strong> this component will be renderer in.
+	 * @param value The <strong>Object</strong> whose value should be represented by the component returned.
+	 * @param index An <i>int</i> giving the objects index in the list of objects.
+	 * @param isSelected <i>true</i> if this object is currently selected, <i>false</i> otherwise.
+	 * @param cellHasFocus <i>true</i> if this object is currently in focus, <i>false</i> otherwise.
+	 * @see org.greenstone.gatherer.util.Utility
+	 */
+	public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+	    JLabel temp = null;
+	    if(value != null) {
+		// Only show a single line by removing new lines.
+		StringBuffer text = new StringBuffer(Utility.stripNL((String)value));
+		// Generate prototype label
+		temp = (JLabel) super.getListCellRendererComponent(list, text.toString(), index, isSelected, cellHasFocus);
+		// Trim down the value string so it fits nicely. Start by shortening it to the number of characters determined from the smarter text area.
+		int preferred_characters = text_area.calculateColumnCount();
+		if(preferred_characters > 3 && text.length() > preferred_characters) {
+		    text.delete(preferred_characters - 3, text.length());
+		    text.append("...");
+		}
+		temp.setText(text.toString());
+		// If we are still too long, shorten by one character at a time until it fits
+		while(temp.getPreferredSize().width > text_area.getSize().width && text.length() > 4) {
+		    // Remove the last four characters and mark as being chopped
+		    text.delete(text.length() - 4, text.length());
+		    text.append("...");
+		    // Assign new string, rinse and repeat.
+		    temp.setText(text.toString());
+		}
+	    }
+	    else {
+		temp = (JLabel) super.getListCellRendererComponent(list, "", index, isSelected, cellHasFocus);
+	    }
+				// Colouring should be the same as the dependent we are based on.
+	    if(isSelected) {
+		temp.setBackground(text_area.getSelectionColor());
+		temp.setForeground(text_area.getSelectedTextColor());
+	    }
+	    else {
+		temp.setBackground(text_area.getBackground());
+		temp.setForeground(text_area.getForeground());
+	    }
+	    return temp;
+	} 
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ElementWrapper.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ElementWrapper.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/ElementWrapper.java	(revision 21112)
@@ -0,0 +1,292 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+/**************************************************************************************
+ * Title:        Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Company:      The University of Waikato
+ * Written:        /  /01
+ * Revised:      03/10/02 - Commented
+ **************************************************************************************/
+import java.util.TreeSet;
+import java.util.ArrayList;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** This class provides a convience wrapper around a DOM model Element to allow Components such as the MetadataTable to display this information properly.
+ * @author John Thompson
+ * @version 2.3
+ */
+public class ElementWrapper
+    implements Comparable {
+    /** The DOM element this wrapper is wrapped around. */
+    private Element element = null;
+    /** A string prefix identifying the metadata set namespace. */
+    private String namespace = "";
+    private ArrayList subelements = null; //The list of subelements
+
+    /** Constructor for elements with no namespace necessary.
+     * @param element The DOM <strong>Element</strong> this is to be based on.
+     */
+    public ElementWrapper(Element element) {
+	this.element = element;
+	Element parent =  (Element)element.getParentNode();
+	if (parent != null)
+	    {
+		this.namespace = parent.getAttribute("namespace");
+	    }
+	
+	subelements = XMLTools.getChildElementsByTagName(element, "Element");
+	//Should read in each subelement's value tree?
+    }
+
+    public void addAttribute(String name, String value) {
+	MSMUtils.addElementAttribute(element, name, "en", value);
+    }
+
+    public void addAttribute(String name, String language, String value) {
+	MSMUtils.addElementAttribute(element, name, language, value);
+    }
+
+
+    /**
+       Check to see if the element contains a particular subelement
+       @return true if element already contains a subelement of name name.
+       @param name the name of the subelement to check for, as a <strong>String</strong> 
+       @author Matthew Whyte
+       Date last modified: 19/01/04
+    */
+    public boolean containsSubelement(String name)
+    {
+	//System.err.println("Want to see if subelement: " + name + " exists."); //debug
+	for(int i = 0; i < subelements.size(); i++)
+	    {
+		Element sibling = (Element)subelements.get(i);
+		String sibling_name = sibling.getAttribute("name");
+		if(sibling_name.equals(name))
+		    {
+			return true;
+		    }
+	    }
+       
+	return false;
+    }
+
+    /*
+      Check to see if the element contains any subelements
+      @return tree if element contains subelements
+      @author Matthew Whyte
+      Date last modified: 2/02/05
+     */
+    public boolean containsSubelements()
+    {
+	return(subelements.size() != 0);
+    }
+
+    /**
+       Method to add a subelement to the list of subelements.
+       @param element the subelement to add to the list, as a <strong>Element</strong>
+       @author Matthew Whyte
+       Date last modified: 27/01/05
+     */
+    public void addSubelementToList(Element element)
+    {
+	subelements.add(element);
+    }
+
+    /**
+       Method to remove a subelement from the list of subelements.
+       @param element the subelement to remove from the list, as a <strong>Element</strong>
+       @author Matthew Whyte
+       Date last modified: 27/01/05
+     */
+    public void removeSubelementFromList(Element element)
+    {
+	subelements.remove(element);
+    }
+
+    /**
+       Method to acquire a list of all the subelements in the element.
+       @return A <strong>NodeList</strong> containing all of this element's subelements.
+
+       @author: Matthew Whyte
+       @date last modified: 19/01/04
+     */
+    public ArrayList getSubelements()
+    {
+	return subelements;
+    }
+
+
+    /** Create a copy of this element wrapper.
+     * @return A new <strong>ElementWrapper</strong> based on the same element as this one.
+     */
+    public ElementWrapper copy() {
+	Element element_copy = (Element) element.cloneNode(true);
+	return new ElementWrapper(element_copy);
+    }
+    /** Compare two element wrappers for ordering.
+     * @param object An <strong>Object</strong> which is most likely another element wrapper to be compared with.
+     * @return An <i>int</i> indicating order, -1, 0, 1 if this element wrapper is less than, equal to or greater than the given object.
+     */
+    public int compareTo(Object object) {
+	return toString().compareTo(object.toString());
+    }
+    /** Decrement the number of occurances of this metadata element.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public void dec() {
+	MSMUtils.setOccurance(element, -1);
+    }
+    /** Test if two ElementWrappers are equal.
+     * @param object The <strong>Object</strong> to test against.
+     */
+   public boolean equals(Object object) {
+      if(object == null) {
+	 return false;
+      }
+      if(object instanceof ElementWrapper) {
+	 String our_full_name = MSMUtils.getFullName(element, namespace);
+	 String their_full_name = MSMUtils.getFullName(((ElementWrapper)object).getElement());
+	 return our_full_name.equals(their_full_name);
+      }
+      return toString().equals(object.toString());
+    }
+    /** Retrieve the attributes associated with the element this element wrapper is built around.
+     * @return A <strong>TreeSet</strong> of the attributes.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public TreeSet getAttributes() {
+	return MSMUtils.getAttributes(element);
+    }
+    /** Retrieve the element this is wrapped around.
+     * @return A DOM <strong>Element</strong> which represents a metadata element.
+     */
+    public Element getElement() {
+	return element;
+    }
+    /** Retrieve the identity of this element (not necessary the same as this elements name). Identity is language and locale dependent.
+     * @return The identity as a <strong>String</strong>.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public String getIdentity() {
+	return MSMUtils.getIdentifier(element);
+    }
+
+    /** Retrieve the name of this element. Name is unique.
+	@return The name (without the namespace) as a <strong>String</strong>.
+	@see org.greenstone.gatherer.msm.MSMUtils
+	@author Matthew Whyte
+	@date last modified: 21/01/04
+     */
+    public String getNameOnly() {
+	return MSMUtils.getNameOnly(element);
+    }
+
+    /** Retrieve the name of this element. Name is unique.
+     * @return The name as a <strong>String</strong>.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public String getName() {
+		return MSMUtils.getFullName(element, namespace);
+    }
+
+
+    /** Retrieve the namespace prefix for this element wrapper.
+     * @return A <strong>String</strong> containing the namespace or "" if there is no namespace for this element.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public String getNamespace() {
+	if (!namespace.equals("")) {
+	    return namespace;
+	}
+	String name = getName();
+	int pos;
+	if((pos = name.indexOf(MSMUtils.NS_SEP)) != -1) {
+	    return name.substring(0, pos);
+	}
+	else {
+	    return "";
+	}
+    }
+    /** Look for the occurances 'field' of the element and return it if found.
+     * @return An <i>int</i> which matches the number in the occurances attribute of the element, or 0 if no such attribute.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public int getOccurances() {
+	return MSMUtils.getOccurances(element);
+    }
+    /** This method is essentially the same as getDescription() in that it does indeed return this metaelements description. However this method uses the Utility function formatHTMLWidth() to ensure the String can be displayed in a tool-tip window using html markup.
+     * @return A String containing the HTML formatted versions of definition and content (comment).
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public String getToolTip() {
+	// Add HTML formatting
+	return Utility.formatHTMLWidth(MSMUtils.getDescription(element), 60);
+    }
+
+    /** Increment the number of occurances of this metadata element.
+     * @see org.greenstone.gatherer.msm.MSMUtils
+     */
+    public void inc() {
+	MSMUtils.setOccurance(element, 1);
+    }
+
+    public boolean isHierarchy() {
+	return element.getAttribute(StaticStrings.HIERARCHY_ATTRIBUTE).equalsIgnoreCase(StaticStrings.TRUE_STR);
+    }
+
+    /** Removes an Attribute node from the element. */
+    public boolean removeAttribute(String name, String language, String value) {
+	boolean sucess = MSMUtils.removeElementAttribute(element, name, language, value);
+	if(!sucess)
+	    {
+		//This should not happen.
+		System.err.println("Error removing attribute \'" + name + "\' with value \'" + value +"\'"); 
+	    }
+	return sucess;
+    }
+
+    public String toString() {
+	return getNamespace() + MSMUtils.NS_SEP + getIdentity();
+    }
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMS.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMS.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMS.java	(revision 21112)
@@ -0,0 +1,2590 @@
+ /**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at th * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################## ##########################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.io.PrintStream;
+import java.io.FileInputStream;
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.util.*;
+import javax.swing.*;
+import java.lang.String;
+import javax.swing.event.*;
+import javax.swing.filechooser.*;
+import javax.swing.text.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.GetOpt;
+import org.greenstone.gatherer.gui.GComboBox;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.HelpFrame;
+import org.greenstone.gatherer.gui.ModalDialog;
+import org.greenstone.gatherer.gui.NonWhitespaceField;
+import org.greenstone.gatherer.util.Codec;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+
+/** Provides a GUI and relevant suite of tools for the editing of the metadata set associated with this collection. Again I have tried to capture a file manager type feel, with a tree showing the various set-element relations to the left of the dialog, and the right side showing details on the current tree selection. When a set is selected these details include a list of attributes, while when an element is selected this list is joined by another showing assigned values. In order for the editor to be stable and consistant with the rest of the tool, care must be taken for fire appropriate events whenever the sets are changed. There is also the addded complexity of determining what actions have to occur in order for a users edit of an assigned value to be completed, i.e. if the user chooses to remove a value then a call must be made to record_set.root.removeMetadata() to ensure all such values are actually removed, so it is not enough just to remove the value from the value model. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3b
+ */ 
+public class GEMS
+    extends JFrame
+{
+    static final public int ADD_SET    = 0;
+    static final public int NORMAL     = 1;
+    static final public int REMOVE_SET = 2;
+
+    /** The default size of the editor dialog. */
+    static final private Dimension ADD_ELEMENT_SIZE = new Dimension(400,125);
+    static final private Dimension RENAME_ELEMENT_SIZE = new Dimension(400, 95);
+    static final private Dimension ADD_SET_SIZE = new Dimension(400,150);
+    static final private Dimension RENAME_SET_SIZE = new Dimension(400,120);
+    static final private Dimension OPEN_SETS_SIZE = new Dimension(400,135);
+    static final private Dimension ADD_OR_EDIT_ATTRIBUTE_SIZE = new Dimension(600,325);
+    static final private Dimension COMPONENT_SIZE = new Dimension(300,30);
+    static final private Dimension SIZE = new Dimension(800,480);
+    static final private String BLANK = "blank";
+    static final private String ELEMENT = "element";
+    static final private String SET = "set";
+
+    static final public String METADATA_SET_TEMPLATE = "xml" + File.separator + "template.mds";
+
+    //all of our popup menu items
+    static final public int T_COLLECTION = 0;
+    static final public int T_ELEMENT    = 1;
+    static final public int T_SET        = 4;
+
+    //static final public String MAX_LOADED_SETS = 64;
+    static public Configuration config = null;
+    private MetadataSetManager msm = new MetadataSetManager(this);
+
+    public AddElementActionListener add_element_action_listener = null;
+    public  AddSetActionListener add_set_action_listener = null;
+    /** The class used to handle add or edit attribute actions has to be globally available so that we can dispose of its dialog properly. */
+    public AddOrEditAttributeActionListener add_or_edit_attribute_action_listener = null;
+    private boolean ignore = false;
+    /** A card layout is used to switch between the two differing detail views. */ 
+    private CardLayout card_layout = null;
+
+    /** Um, the size of the screen I'd guess. */
+    private Dimension screen_size = null;
+    private ElementWrapper current_element = null;
+    /** A reference to ourselves so our inner classes can dispose of us. */
+    private GEMS self = null;
+    private int current_attribute = -1;
+    private int current_attribute_type = -1;
+
+    //private JLabel element_name = null;
+    private JLabel set_name = null;
+    private JScrollPane element_attributes_scroll = null;
+    private JScrollPane set_attributes_scroll = null;
+    private JPanel details_pane = null;
+    private GEMSModel model = null;
+    private GEMSNode current_node = null;
+    private MetadataSet current_set = null;
+    public RemoveSetActionListener remove_set_action_listener;
+    private SmarterTable element_attributes = null;
+    private SmarterTable set_attributes = null;
+    public org.w3c.dom.Element rootelement;
+   
+    //PopupListener provides right-click functionality to any component that needs it
+    //It is a centralized manager for all popups, then we use evt.Source() and determine where the
+    //event was performed
+    private PopupListener popupListener = null;
+ 
+    /** A tree that represents the current metadata sets associated with this collection. */
+    private SmarterTree mds_tree = null;
+    private String dialog_options[] = null;
+    private boolean atLeastOneSetChanged = false;
+
+    static public void main(String[] args)
+    {
+	// Parse arguments
+	GetOpt go = new GetOpt(args);
+
+	if (go.debug) {
+	    DebugStream.enableDebugging();
+
+	    Calendar now = Calendar.getInstance();
+	    String debug_file_path = "debug" + now.get(Calendar.DATE) + "-" + now.get(Calendar.MONTH) + "-" + now.get(Calendar.YEAR) + ".txt";
+
+	    // Debug file is created in the GLI directory
+	    DebugStream.println("Debug file path: " + debug_file_path);
+	    DebugStream.setDebugFile(debug_file_path);
+	}
+
+        GEMS GEMS;
+	GEMS = new GEMS(go.gsdl_path);
+    }
+
+
+    /** Constructor.
+     * @param set a MetadataSet that should be initially selected
+     * @param action a systematic action that should be performed
+     */
+    public GEMS(String gsdl_path)
+    {
+	// Determine the GLI user directory path
+	String gli_user_directory_path = System.getProperty("user.home") + File.separator;
+        if (Utility.isWindows()) {
+	    gli_user_directory_path += "Application Data" + File.separator + "Greenstone" + File.separator + "GLI" + File.separator;
+        }
+        else {
+            gli_user_directory_path += ".gli" + File.separator;
+        }
+
+	//Load configuration file
+	this.self = this;
+	JarTools.initialise(this);
+	config = new Configuration(gli_user_directory_path, gsdl_path, null, null, null);
+
+	// Read Dictionary
+	new Dictionary(Configuration.getLocale("general.locale", true), Configuration.getFont("general.font", true));
+
+	this.dialog_options = new String[2];
+	this.screen_size = config.screen_size;
+        
+	// Initialise some common images
+	popupListener = new PopupListener(this);
+
+	// Load help
+	HelpFrame help = new HelpFrame();
+
+	dialog_options[0] = Dictionary.get("General.OK");
+	dialog_options[1] = Dictionary.get("General.Cancel");
+
+	// Load all the core metadata sets (in the GLI "metadata" directory)
+	model = new GEMSModel();
+	File metadata_directory = new File(getGLIMetadataDirectoryPath());
+	if (metadata_directory.exists()) {
+	    // Load just those .mds files in this directory, and return them
+	    File[] directory_files = metadata_directory.listFiles();
+	    for (int i = 0; i < directory_files.length; i++) {
+		File child_file = directory_files[i];
+		if (!child_file.isDirectory() && child_file.getName().endsWith(".mds")) {
+		    MetadataSet metadata_set = msm.loadMetadataSet(child_file);
+		    model.add(null, metadata_set, GEMSNode.SET);
+		}
+	    }
+	}
+
+	// Creation
+	setDefaultCloseOperation(DISPOSE_ON_CLOSE);
+	setSize(SIZE);
+	setJMenuBar(new GEMSMenuBar());
+	Dictionary.setText(this, "GEMS.Title");
+
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));	  
+	JPanel upper_pane = new JPanel();
+	upper_pane.setOpaque(false);
+	JPanel set_details_pane = new JPanel();
+	set_details_pane.setOpaque(false);
+	JPanel set_name_pane = new JPanel();
+	set_name_pane.setOpaque(false);
+	JLabel set_name_label = new JLabel();
+	set_name_label.setOpaque(false);
+	JPanel element_details_pane = new JPanel();
+	element_details_pane.setOpaque(false);	  
+	JPanel element_inner_pane = new JPanel();
+	element_inner_pane.setOpaque(false);
+	JPanel blank_pane = new JPanel(); // Some blank panel
+	blank_pane.setOpaque(false);
+
+	details_pane = new JPanel();
+	details_pane.setOpaque(false);
+	card_layout = new CardLayout();
+	set_name = new JLabel();
+	
+	JPanel mds_tree_pane = new JPanel();
+	mds_tree_pane.setOpaque(false);
+	mds_tree_pane.setPreferredSize(new Dimension(300,500));
+
+	mds_tree = new SmarterTree(model);
+	mds_tree.setCellRenderer(new GEMSTreeCellRenderer());
+	mds_tree.setRootVisible(false);
+	mds_tree.setBackground(config.getColor("coloring.collection_tree_background", false));
+	mds_tree.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	mds_tree.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	mds_tree.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+        //add popupevent handler for tree items
+        mds_tree.addMouseListener(popupListener);
+	mds_tree.addKeyListener(new KeyPressedHandler());
+	mds_tree.addTreeSelectionListener(new MDSTreeSelectionListener());
+		  
+	JPanel set_attributes_pane = new JPanel();
+	set_attributes_pane.setOpaque(false);
+	set_attributes_scroll = new JScrollPane();
+	set_attributes_scroll.getViewport().setBackground(config.getColor("coloring.collection_tree_background", false));
+	set_attributes_scroll.setOpaque(true);
+	set_attributes = new SmarterTable();
+	set_attributes.setAutoResizeMode(JTable.AUTO_RESIZE_OFF);
+	set_attributes.setBackground(config.getColor("coloring.collection_tree_background", false));
+	set_attributes.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	set_attributes.setHeadingBackground(config.getColor("coloring.collection_heading_background", false));
+	set_attributes.setHeadingForeground(config.getColor("coloring.collection_heading_foreground", false));
+	set_attributes.setOpaque(false);
+	set_attributes.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	set_attributes.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	set_attributes.getSelectionModel().setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	set_attributes.addMouseListener(popupListener);	
+	set_attributes.addMouseListener(new AddOrEditAttributeActionListener());
+	set_attributes.addKeyListener(new RemoveAttributeActionListener()); //Listen for DELETE key
+	set_attributes.getSelectionModel().addListSelectionListener(new AttributesListSelectionListener(set_attributes));
+		  
+	JPanel element_attributes_pane = new JPanel();
+	element_attributes_pane.setOpaque(false);
+	element_attributes_scroll = new JScrollPane();
+	element_attributes_scroll.getViewport().setBackground(config.getColor("coloring.collection_tree_background", false));
+	element_attributes_scroll.setOpaque(true);
+	element_attributes = new SmarterTable();
+	element_attributes.setAutoResizeMode(JTable.AUTO_RESIZE_OFF);
+	element_attributes.setBackground(config.getColor("coloring.collection_tree_background", false));
+	element_attributes.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	element_attributes.setHeadingBackground(config.getColor("coloring.collection_heading_background", false));
+	element_attributes.setHeadingForeground(config.getColor("coloring.collection_heading_foreground", false));
+	element_attributes.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	element_attributes.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	element_attributes.getSelectionModel().setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+        element_attributes.addMouseListener(popupListener);
+	element_attributes.addMouseListener(new AddOrEditAttributeActionListener());
+	element_attributes.addKeyListener(new RemoveAttributeActionListener()); //For the DELETE key
+	element_attributes.getSelectionModel().addListSelectionListener(new AttributesListSelectionListener(element_attributes));
+
+	add_element_action_listener = new AddElementActionListener();
+	add_set_action_listener = new AddSetActionListener();
+	add_or_edit_attribute_action_listener = new AddOrEditAttributeActionListener();
+	remove_set_action_listener = new RemoveSetActionListener();
+
+	// Layout
+	mds_tree_pane.setLayout(new BorderLayout());
+	mds_tree_pane.add(new JScrollPane(mds_tree), BorderLayout.CENTER);
+		  
+	//Set panes
+	set_name_pane.setLayout(new BorderLayout());
+	set_name_pane.add(set_name_label, BorderLayout.WEST);
+	set_name_pane.add(set_name, BorderLayout.CENTER);
+				
+	set_attributes_scroll.setViewportView(set_attributes);
+	set_attributes_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("GEMS.Attributes")), BorderFactory.createEmptyBorder(2,2,2,2)));
+	set_attributes_pane.setLayout(new BorderLayout());
+	set_attributes_pane.add(set_attributes_scroll, BorderLayout.CENTER);
+		
+	set_details_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("GEMS.Set_Details")), BorderFactory.createEmptyBorder(2,2,2,2)));
+	set_details_pane.setLayout(new BorderLayout());
+	set_details_pane.add(set_name_pane, BorderLayout.NORTH);
+	set_details_pane.add(set_attributes_pane, BorderLayout.CENTER);
+		
+	//Element panes
+	element_attributes_scroll.setViewportView(element_attributes);
+	element_attributes_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("GEMS.Attributes")), BorderFactory.createEmptyBorder(2,2,2,2)));
+	element_attributes_pane.setLayout(new BorderLayout());
+	element_attributes_pane.add(element_attributes_scroll, BorderLayout.CENTER);
+		  
+	element_inner_pane.setLayout(new BorderLayout());
+	element_inner_pane.add(element_attributes_pane);
+		  
+	element_details_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("GEMS.Element_Details")), BorderFactory.createEmptyBorder(2,2,2,2)));
+	element_details_pane.setLayout(new BorderLayout());
+	element_details_pane.add(element_inner_pane, BorderLayout.CENTER);
+
+	//General panes
+	details_pane.setBorder(BorderFactory.createEmptyBorder(0,5,0,0));
+	details_pane.setLayout(card_layout);
+	details_pane.add(blank_pane, BLANK);
+	details_pane.add(set_details_pane, SET);
+	details_pane.add(element_details_pane, ELEMENT);
+
+	upper_pane.setLayout(new BorderLayout());
+	upper_pane.add(mds_tree_pane, BorderLayout.WEST);
+	upper_pane.add(details_pane, BorderLayout.CENTER);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(upper_pane, BorderLayout.CENTER);
+
+	// initialise the selection
+	mds_tree.setSelectionRow(0);
+	// Display
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+
+
+    private void exit()
+    {
+        if(atLeastOneSetChanged == true)
+	    {
+		int result = JOptionPane.showConfirmDialog(self, Dictionary.get("GEMS.Confirm_Exit_Save", Dictionary.get("GEMS.Element")), Dictionary.get("GEMS.Confirm_Exit_Save_Title"), JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.QUESTION_MESSAGE);
+
+		if(result == 0)
+		    {
+			DebugStream.closeDebugStream();
+			msm.save();
+			System.exit(0);
+		    }
+		else if(result == 1)
+		    {
+			DebugStream.closeDebugStream();
+			System.exit(0);
+		    }
+		else if(result == 2)
+		    {
+			//do nothing
+		    }
+	    }
+	else
+	    {
+		//Just exit
+		DebugStream.closeDebugStream();
+		System.exit(0);
+	    }
+    }
+
+
+    static public String getGLIDirectoryPath()
+    {
+	return System.getProperty("user.dir") + File.separator;
+    }
+
+
+    static public String getGLIMetadataDirectoryPath()
+    {
+	return getGLIDirectoryPath() + "metadata" + File.separator;
+    }
+
+    
+    private class GEMSMenuBar
+	extends JMenuBar
+	implements ActionListener
+    {
+	private JMenu file = null;
+	private JMenu edit = null;
+	private JMenu help = null;
+
+	public JMenuItem file_exit   = null;
+	public JMenuItem file_new    = null;
+	public JMenuItem file_open   = null;
+	public JMenuItem file_save   = null;
+	public JMenuItem edit_copy   = null;
+	public JMenuItem edit_cut    = null;
+	public JMenuItem edit_paste  = null;
+	public JMenuItem help_help   = null;
+        public JMenuItem file_delete = null;
+        public JMenuItem file_preferences = null;
+	public GEMSMenuBar()
+	{
+	    file = new JMenu();
+	    file.setMnemonic(KeyEvent.VK_F);
+	    Dictionary.registerText(file, "Menu.File");
+
+	    file_exit = new JMenuItem();
+	    file_exit.addActionListener(this);
+	    file_exit.setMnemonic(KeyEvent.VK_X);
+	    Dictionary.registerText(file_exit, "Menu.File_Exit");
+
+	    file_new = new JMenuItem();
+	    file_new.addActionListener(this);
+	    file_new.setMnemonic(KeyEvent.VK_N);
+	    Dictionary.registerText(file_new, "Menu.File_New");
+
+	    file_open = new JMenuItem();
+	    file_open.addActionListener(this);
+	    file_open.setMnemonic(KeyEvent.VK_O);
+	    Dictionary.registerText(file_open, "Menu.File_Open");
+
+	    file_save = new JMenuItem();
+	    file_save.addActionListener(this);
+	    file_save.setMnemonic(KeyEvent.VK_S);
+            Dictionary.registerText(file_save, "Menu.File_Save");
+            
+            file_delete = new JMenuItem();
+	    file_delete.addActionListener(new KeyPressedHandler()); //KeyPressedHandler handles deleting stuff.
+	    file_delete.setMnemonic(KeyEvent.VK_S);
+            Dictionary.registerText(file_delete, "Menu.File_Delete");
+            
+            file_preferences = new JMenuItem();
+	    file_preferences.addActionListener(this);
+	    file_preferences.setMnemonic(KeyEvent.VK_S);
+            Dictionary.registerText(file_preferences, "Menu.File_Options");
+
+	    // Layout (file menu)
+	    file.add(file_new);
+	    file.add(file_open);
+	    file.add(file_save);
+            file.add(file_delete);
+	    file.add(new JSeparator());
+            file.add(file_preferences);
+            file.add(new JSeparator());
+	    file.add(file_exit);
+            
+	    // Edit menu
+	    edit = new JMenu();
+	    edit.setMnemonic(KeyEvent.VK_E);
+	    Dictionary.registerText(edit, "Menu.Edit");
+
+	    edit_cut = new JMenuItem();
+	    edit_cut.addActionListener(this);
+	    edit_cut.setMnemonic(KeyEvent.VK_X);
+	    Dictionary.registerText(edit_cut, "Menu.Edit_Cut");
+
+	    edit_copy = new JMenuItem();
+	    edit_copy.addActionListener(this);
+	    edit_copy.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.registerText(edit_copy, "Menu.Edit_Copy");
+
+	    edit_paste = new JMenuItem();
+	    edit_paste.addActionListener(this);
+	    edit_paste.setMnemonic(KeyEvent.VK_V);
+	    Dictionary.registerText(edit_paste, "Menu.Edit_Paste");
+
+	    // Layout (edit menu)
+	    edit.add(edit_cut);
+	    edit.add(edit_copy);
+	    edit.add(edit_paste);
+
+	    // Help menu. Is not used! --Matthew
+	    // help = new JMenu();
+	    // help.setIcon(JarTools.getImage("help.gif"));
+	    // Dictionary.setText(help, "Menu.Help");
+
+	    //help_help = new JMenuItem();
+	    //help_help.addActionListener(this);
+	    //Dictionary.registerText(help_help, "Menu.Help");
+
+	    // Layout (help menu)
+	    //help.add(help_help);
+
+	    // Layout (menu bar)
+	    this.add(file);
+	    this.add(Box.createHorizontalStrut(15));
+	    this.add(edit);
+	    this.add(Box.createHorizontalGlue());
+	    //this.add(help);
+	}
+
+
+	public void actionPerformed(ActionEvent event)
+	{
+	    Object event_source = event.getSource();
+
+	    // File -> New
+	    if (event_source == file_new) {
+		AddSetActionListener rah = new AddSetActionListener();
+		rah.setVisible(true);
+		// rah.show();
+	 
+		return;
+	    }
+
+	    // File -> Open
+	    if (event_source == file_open) {
+                
+                //we first launch the dialog to determine where the jfilechooser should be opened
+                //ie: for collection specific sets - attila dec10-04
+                String set_directory = getGLIMetadataDirectoryPath();
+                
+                SetSelectionDialog m = new SetSelectionDialog();
+		m.setVisible(true);
+                // m.show();
+                    
+                //once we have clicked 'ok' on the jdialog above, then we should have
+                //the directory pointer in string set_directory...and pass it to the jfilechoose constru
+		return;
+	    }
+
+	    // File -> Save
+	    if (event_source == file_save) {
+		msm.save();
+                atLeastOneSetChanged = false;
+		return;
+	    }
+
+	    // File -> Exit
+	    if (event_source == file_exit) {
+                exit(); //The exit() method checks if data needs saving.
+	    }
+
+	    // File -> Preferences
+            if(event_source == file_preferences){
+              GEMSPreferences GemsPreferences = new GEMSPreferences();  
+            }
+
+	    // Edit -> Cut
+	    if (event_source == edit_cut) {
+		try {
+		    KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		    // Get the component with selected text as a JTextComponent
+		    JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();
+		    // Cut the text to the clipboard
+		    text.cut();
+		}
+		catch (ClassCastException cce) {
+		    // If the component is not a text component ignore the cut command
+		    DebugStream.println(cce.toString());
+		}
+		return;
+	    }
+
+	    // Edit -> Copy
+	    if (event_source == edit_copy) {
+		try {
+		    KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		    // Get the component with selected text as a JTextComponent
+		    JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();//getFocusOwner();
+		    text.copy();
+		}
+		catch (Exception cce) {
+		    // If the component is not a text component ignore the copy command
+		    DebugStream.println(cce.toString());
+		}
+		return;
+	    }
+
+	    // Edit -> Paste
+	    if (event_source == edit_paste) {
+		try {
+		    KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		    // Get the component with selected text as a JTextComponent
+		    JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();
+		    // Cut the text to the clipboard
+		    text.paste();
+		}
+		catch (ClassCastException cce) {
+		    // If the component is not a text component ignore the paste command
+		    DebugStream.println(cce.toString());
+		}
+		return;
+	    }
+
+	    // Help -> Help
+	    if (event_source == help_help) {
+		HelpFrame.setView("editingmetadatasets");
+		return;
+	    }
+	}
+    }
+
+
+    private class MetadataSetFileFilter
+	extends FileFilter
+    {
+	public boolean accept(File file)
+	{
+	    String file_name = file.getName().toLowerCase();
+	    return(file_name.endsWith(".mds") || file_name.indexOf(".") == -1);
+	}
+
+	public String getDescription()
+	{
+	    return Dictionary.get("MetadataSet.Files");
+	}
+    }
+
+
+    private class ActionTask
+	extends Thread {
+	private int action;
+	public ActionTask(int action) {
+	    this.action = action;
+	}
+	public void run() {
+	    boolean complete = false;
+	    while(!complete) {
+		if(self.isVisible()) {
+		    switch(action) {
+		    case ADD_SET:
+			add_set_action_listener.actionPerformed(new ActionEvent(this, 0, "Blarg!"));
+			break;
+		    case REMOVE_SET:
+			if(!mds_tree.isSelectionEmpty()) {
+			    remove_set_action_listener.actionPerformed(new ActionEvent(this, 0, "Blarg!"));
+			}
+			break;
+		    }
+		    complete = true;
+		}
+		else {
+		    try {
+			synchronized(this) {
+			    wait(100);
+			}
+		    }
+		    catch(Exception exception) {
+		    }
+		}
+	    }
+	}
+    }
+    //callback method
+     protected void processWindowEvent(WindowEvent evt){
+       
+       if(evt.getID() == WindowEvent.WINDOW_CLOSING){
+           
+          exit();
+       }
+   }
+   //called from MetadataSetManager...I don't think it should be here anymore, but for now it's ok
+ 
+   public void warn_cant_load_set(){
+            JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Already_Loaded_Set_Error"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+        }
+    public void dispose() {
+        
+        
+         //if anything has changed, warn user that they should save
+        if(atLeastOneSetChanged == true) {
+            
+            int result = JOptionPane.showOptionDialog(self, Dictionary.get("GEMS.Confirm_Exit_Save", Dictionary.get("GEMS.Element")), Dictionary.get("GEMS.Confirm_Exit_Save_Title"), JOptionPane.DEFAULT_OPTION, JOptionPane.QUESTION_MESSAGE, null, dialog_options, dialog_options[0]);
+         //   System.out.println("Exit: "+result);
+            if(result == 0) {
+                msm.save();
+            }
+        }
+        atLeastOneSetChanged = false;
+	// Destructor
+	card_layout = null;
+	screen_size = null;
+	self = null;
+	details_pane = null;
+	element_attributes = null;
+	set_attributes = null;
+	set_name = null;
+	mds_tree = null;
+        
+        popupListener = null;
+	// Dispose of inner dialogs
+	if (add_element_action_listener != null) {
+	    add_element_action_listener.dispose();
+	    add_element_action_listener = null;
+	}
+	if (add_or_edit_attribute_action_listener != null) {
+	    add_or_edit_attribute_action_listener.dispose();
+	    add_or_edit_attribute_action_listener = null;
+	}
+	remove_set_action_listener = null;
+
+        //close
+	super.dispose();
+    }
+
+
+    /**
+       Class to handle adding or modifying element attributes.
+     */
+    private class AddOrEditAttributeActionListener
+	extends ModalDialog
+	implements ActionListener, MouseListener {
+	private boolean add_type = true;
+	private ComboArea value = null;
+	private JButton cancel_button = null;
+	private JButton ok_button = null;
+	private JComboBox language_box = null;
+	private GComboBox name = null;
+	private HashMap name_to_values = null;
+	private JLabel target = null;
+	private JLabel target_label = null;
+	private Vector attributeLists = null;      
+
+	org.w3c.dom.Document lang = XMLTools.parseXMLFile(getGLIDirectoryPath() + "classes" + File.separator + "xml" + File.separator + "languages.xml", false);
+	
+	/** Constructor. */
+	public AddOrEditAttributeActionListener() {
+	    super(self);
+	    setModal(true);
+            attributeLists = new Vector();
+            attributeLists.add("identifier");
+            attributeLists.add("comment");
+            attributeLists.add("definition");
+            attributeLists.add("name");
+
+            attributeLists.add("creator");
+            attributeLists.add("date");
+            attributeLists.add("contact");
+            attributeLists.add("family");
+            
+            attributeLists.add("lastchanged");
+            attributeLists.add("language_dependant");
+            attributeLists.add("version");
+            attributeLists.add("datatype");
+            
+            attributeLists.add("obligation");
+            attributeLists.add("maximum_occurence");
+            attributeLists.add("registration_authority");
+            
+	    setSize(ADD_OR_EDIT_ATTRIBUTE_SIZE);
+	    name_to_values = new HashMap();
+
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel upper_pane = new JPanel();
+	    upper_pane.setOpaque(false);
+
+	    JPanel labels_pane = new JPanel();
+	    JPanel boxes_pane = new JPanel();
+
+	    target_label = new JLabel();
+	    target_label.setOpaque(false);
+	    target = new JLabel();
+	    target.setOpaque(false);
+
+	    JLabel name_label = new JLabel();
+	    name_label.setOpaque(false);
+	    Dictionary.setText(name_label, "GEMS.Name");
+	    name = new GComboBox();
+	    name.setEditable(true);
+	    Dictionary.setTooltip(name, "GEMS.Attribute_Name_Tooltip");
+
+	    JLabel language_label = new JLabel();
+	    language_label.setOpaque(false);
+	    Dictionary.setText(language_label, "GEMS.Language");
+	    language_box = new JComboBox();
+            
+            for(int n = 0; n < attributeLists.size(); n++){
+                name.add(attributeLists.get(n));
+            }
+ 
+	    Dictionary.setTooltip(language_box, "GEMS.Attribute_Language_Tooltip");
+
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+	    value = new ComboArea(Dictionary.get("GEMS.Values"), COMPONENT_SIZE);
+	    value.setOpaque(false);
+
+	    JTextArea v_text_area = (JTextArea) value.getTextComponent();
+	    v_text_area.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    v_text_area.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    v_text_area.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    v_text_area.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(v_text_area, "GEMS.Attribute_Value_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	    // Connection
+	    TextFieldEnabler ok_button_enabler = new TextFieldEnabler(ok_button);
+	    cancel_button.addActionListener(this);
+	    name.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    ok_button_enabler.add((JTextComponent)name.getEditor()); // Assuming the default editor is a JTextField!
+
+	    // Layout
+	    labels_pane.setLayout(new GridLayout(3,1,0,5));
+	    labels_pane.add(target_label);
+	    labels_pane.add(name_label);
+	    labels_pane.add(language_label);
+
+	    boxes_pane.setLayout(new GridLayout(3,1,0,5));
+	    boxes_pane.add(target);
+	    boxes_pane.add(name);
+	    boxes_pane.add(language_box);
+	    
+	    upper_pane.setLayout(new BorderLayout(5,0));
+	    upper_pane.add(labels_pane, BorderLayout.WEST);
+	    upper_pane.add(boxes_pane, BorderLayout.CENTER);
+
+	    value.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+
+	    button_pane.setLayout(new GridLayout(1,2,5,0));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    center_pane.setLayout(new BorderLayout());
+	    center_pane.add(value, BorderLayout.CENTER);
+	    center_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(upper_pane, BorderLayout.NORTH);
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+
+	    setLocation((config.screen_size.width - ADD_OR_EDIT_ATTRIBUTE_SIZE.width) / 2, (config.screen_size.height - ADD_OR_EDIT_ATTRIBUTE_SIZE.height) / 2);
+
+	    GEMSLanguageManager GemsLangManager = new GEMSLanguageManager(getGLIDirectoryPath() + "classes" + File.separator + "xml" + File.separator + "languages.xml");
+        
+            Vector languagecodes = GemsLangManager.getLanguageCodes();
+            
+            for(int p = 0; p < languagecodes.size(); p++){
+             
+                language_box.addItem(languagecodes.get(p).toString());
+            }
+           
+           String code = Configuration.getLanguage();
+           language_box.setSelectedItem(code);
+        }
+
+       //Handle double clicks in the table.
+       public void mouseClicked(MouseEvent e)
+	{
+	    if(e.getSource() == element_attributes || e.getSource() == set_attributes)
+		{
+		    if(e.getClickCount() == 2) //Double click
+			{
+			    //Show the Edit Attribute dialog
+			    this.menuEditValue();
+			    this.editAttribute();
+			}
+		}
+	}
+	//These methods are required by interface MouseListener
+	public void mouseEntered(MouseEvent e){}
+	public void mouseExited(MouseEvent e){}
+	public void mousePressed(MouseEvent e){}
+	public void mouseReleased(MouseEvent e){}
+
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to add a new attribute to the selected set or element.
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+	    AttributeTableModel model = null;
+           
+	    if(current_element != null)
+		{
+		    target.setText(current_element.getName());
+		    Dictionary.setText(target_label, "GEMS.Target");
+		}
+	    else if(current_set != null)
+		{
+		    Dictionary.setText(target_label, "GEMS.Set");
+		    target.setText(current_set.toString());
+		    Dictionary.setText(target_label, "GEMS.Target");
+		}
+	    
+	     if (source == popupListener.menuEditValue)
+		 {
+		     this.menuEditValue();              
+		 }
+ 
+            if(source == ok_button) {
+		boolean success = true;
+
+		// Add or edit a set/element attribute
+		switch(current_attribute_type) {
+		case GEMSNode.SET:
+		    model = (AttributeTableModel) set_attributes.getModel();
+		    success = addOrEditSetAttribute(model);
+		    break;
+		case GEMSNode.ELEMENT:
+		    model = (AttributeTableModel) element_attributes.getModel();
+		    success = addOrEditElementAttribute(model);
+		    break;
+		}
+                 
+		if (success) {
+		    // Hide dialog
+		    atLeastOneSetChanged = true;
+		    setVisible(false);
+                    return;
+		}
+		else
+		    {
+			//If no success (an attribute with that name already exists), do nothing.
+		    }
+	    }
+	    else if(source == cancel_button) {
+		// Hide dialog
+		setVisible(false);
+                return;
+	    }
+	    else if(source == name) {
+		//System.err.println("source is name");
+                Object object = name.getSelectedItem();
+		if(object != null) {
+		    java.util.List values = (java.util.List) name_to_values.get(object.toString());
+                    
+                    if(value != null && values != null){
+                        value.clear();
+                        for(int i = 0; i < values.size(); i++) {
+                            value.add(values.get(i));
+                        }
+                    }
+		}
+	    }
+	    else if (source == popupListener.menuAddAttributeAttribute || source == popupListener.menuAddAttributeSet || source == popupListener.menuAddAttributeElement){
+
+		Dictionary.setText(this, "GEMS.Add_Attribute");
+		for(int i = 0; i < attributeLists.size(); i++) {
+		    name.add(attributeLists.get(i).toString());    
+                    
+		}
+		name.setEnabled(true);
+		setVisible(true);
+            }
+           
+		else if(current_attribute != -1) {
+		    this.editAttribute();
+		}
+
+	    source = null;
+	}
+
+       private void menuEditValue()
+       {
+	   AttributeTableModel tablemodel = null;
+	  
+	   String prevLang = new String("");
+	   String prevValue = new String("");
+
+	   add_type = false;
+	   //grab the table model based on what is currently selected
+	   switch(current_attribute_type) {
+	   case GEMSNode.SET:
+	       tablemodel = (AttributeTableModel) set_attributes.getModel();
+	       //grab value in table
+	       prevLang =  tablemodel.getValueAt(current_attribute,0).toString();
+	       prevValue = tablemodel.getValueAt(current_attribute,1).toString();
+	       
+	       break;
+	   case GEMSNode.ELEMENT:
+	       tablemodel = (AttributeTableModel) element_attributes.getModel();
+	       //grab values in table( lang and value)
+	       prevLang =  tablemodel.getValueAt(current_attribute,1).toString();
+	       prevValue = tablemodel.getValueAt(current_attribute,2).toString();
+	       
+	       break;
+	   default:
+	       //by default grab element_attr's
+	       tablemodel = (AttributeTableModel) element_attributes.getModel();
+	       prevLang =  tablemodel.getValueAt(current_attribute,1).toString();
+	       prevValue = tablemodel.getValueAt(current_attribute,2).toString();
+	       
+	       break;
+	   }         
+	   language_box.setEnabled(false);
+	   
+           //if a set attr is selected, then do not do any lang operations, other than setting index to -1
+	   if(current_attribute_type != GEMSNode.SET) {
+	       
+	       //set the current language to show up in language_box(no lang if no lang specified)
+	       if(prevLang == null || prevLang.compareTo(" ")==0 || prevLang.compareTo("") == 0 || prevLang.compareTo("  ")==0) {
+		   language_box.setSelectedIndex(-1);
+		   
+	       }
+	       else {
+		   language_box.setSelectedItem(prevLang.toLowerCase());
+	       }
+	   }// end currattr != GEMSnode.set
+	   else {
+	       
+	       language_box.setSelectedIndex(-1);
+	       
+	   }// end currattr == GEMSnode.set
+	   value.setText(prevValue);
+       }
+
+       private void editAttribute()
+       {
+	   //System.err.println("just entered editAttribute"); //debug
+	   AttributeTableModel model = null;
+	   Dictionary.setText(this, "GEMS.Edit_Attribute");
+	   
+	   switch(current_attribute_type) {
+	   case GEMSNode.ELEMENT:
+	       model = (AttributeTableModel) element_attributes.getModel();
+	       break;
+	   case GEMSNode.SET:
+	       model = (AttributeTableModel) set_attributes.getModel();
+	       break;
+	   }
+	   add_type = false;
+	   String name_str = (String) model.getValueAt(current_attribute, 0);
+	   String value_str = (String) model.getValueAt(current_attribute, model.getColumnCount() - 1);
+	   model = null;
+	   // Retrieve the appropriate value model
+	   java.util.List values = (java.util.List) name_to_values.get(name_str);
+	   // Only possible for collection file selections.
+	   if(values == null) {
+	       values = msm.getElements();
+	   }
+	   name.setSelectedItem(name_str);
+	   name_str = null;
+	   for(int i = 0; i < values.size(); i++) {
+	       Object temp_value = values.get(i);
+	       if(temp_value instanceof ElementWrapper) {
+		   value.add(new NameElementWrapperEntry(temp_value));
+	       }
+	       else {
+		   value.add(temp_value);
+	       }
+	   }
+	   values = null;
+	   value.setSelectedItem(value_str);
+	   value_str = null;
+	   setVisible(true);   
+       }
+
+	private boolean addOrEditSetAttribute(AttributeTableModel model)
+	{
+	    String new_name_str = name.getSelectedItem().toString();
+	    String value_str = Codec.transform(Codec.transformUnicode(value.getText()), Codec.TEXT_TO_DOM);
+	    String language_code = (String) language_box.getSelectedItem();
+	    String old_name_str = null;
+	    boolean cont = false;
+	    boolean edit = false;
+
+	    if(!add_type && current_attribute != -1) //If edit, grab the attribute's old name
+		{
+		    old_name_str = (String) set_attributes.getValueAt(current_attribute, 0);
+		    edit = true;
+		}
+
+	    // Check that there isn't already an entry for this attribute
+	    if (!model.contains(new_name_str, language_code) || (edit && new_name_str.equals(old_name_str)))
+		{
+		    if(edit) //If edit, remove old attribute
+			{ 
+			    current_set.removeAttribute(old_name_str);
+			    model.removeRow(current_attribute);
+			}
+		    
+		    // Add the new attribute
+		    current_set.addAttribute(new_name_str, value_str);
+		    // Update the attribute table
+		    model.add(new Attribute(new_name_str, value_str));
+		    cont = true;
+		}
+	    // Otherwise show an error message and do not proceed
+	    else
+		{
+		    cont = false;
+		    JOptionPane.showMessageDialog(self,
+						  Dictionary.get("GEMS.Attribute_Already_Exists"),
+						  Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		}
+       
+	    value_str = null;
+	    new_name_str = null;
+	    return cont;
+	}
+
+
+	private boolean addOrEditElementAttribute(AttributeTableModel model)
+	{
+	    String new_name_str = name.getSelectedItem().toString();
+	    String old_name_str = null;
+	    String language_code = (String) language_box.getSelectedItem();
+	    String value_str = Codec.transform(Codec.transformUnicode(value.getText()), Codec.TEXT_TO_DOM);
+	    String old_value_str = null;
+	    String old_lang_code = null;
+	    boolean cont = false;
+	    boolean edit = false;
+
+	    if(!add_type && current_attribute != -1) //An edit
+		{
+		    Attribute old_attribute = model.getAttribute(current_attribute);
+		    old_value_str = old_attribute.value;
+		    old_lang_code = old_attribute.language;
+		    old_name_str = (String) element_attributes.getValueAt(current_attribute, 0); //get the old name.
+		    edit = true;
+		}
+
+	    //Check that there isn't already an entry for this attribute
+	    if (!model.contains(new_name_str, language_code) || (edit && new_name_str.equals(old_name_str)))
+		{
+		    if(edit) //If this is an edit, remove the old attribute
+			{
+			    current_element.removeAttribute(old_name_str, old_lang_code, old_value_str);
+			    model.removeRow(current_attribute);
+			}
+
+		    // Add the new attribute
+		    //System.err.println("current_element is: " + current_element.toString()); //debug
+		    current_element.addAttribute(new_name_str, language_code, value_str);
+		    // Update the attribute table
+		    int row = model.add(new Attribute(new_name_str, language_code, value_str));
+		    element_attributes.setRowSelectionInterval(row, row);
+		    cont = true;
+		}
+	    // Otherwise show an error message and do not proceed
+	    else
+		{
+		    cont = false;
+		    JOptionPane.showMessageDialog(self,
+						  Dictionary.get("GEMS.Attribute_Already_Exists"),
+						  Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		} 
+	
+	    value_str = null;
+	    language_code = null;
+	    new_name_str = null;
+	    return cont;
+	}
+    }
+
+   
+    private class AddSubelementActionListener extends AddElementActionListener implements ActionListener {
+
+	public AddSubelementActionListener() {
+	    Dictionary.setText(this, "GEMS.Add_Subelement");
+	    Dictionary.setTooltip(name_field, "GEMS.Subelement_Name_Tooltip");
+	}
+
+	public void actionPerformed(ActionEvent event)
+	{
+	    Object source = event.getSource();
+	    current_element = current_node.getElement(); //current_element here is used generically for a element or subelement.
+
+	    if(source == ok_button) //Add then dispose
+		{
+		    String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+		    current_set = msm.getSet(current_element.getNamespace());
+		    if(current_set != null)
+			{
+			     // If this subelement doesn't already exist.
+			    if(!current_element.containsSubelement(name_str))
+				{
+				    // Add it,
+				    String language_code = config.getLanguage();
+				    ElementWrapper subelement = current_set.addElement(name_str, language_code, current_node);
+
+				    final GEMSNode new_node = model.add(current_node, subelement, GEMSNode.ELEMENT); //Then update the tree.
+				   
+				    //SwingUtilities.invokeLater is used so that expandPath works on new parents.
+				    SwingUtilities.invokeLater(new Runnable() {
+					    public void run() {
+						mds_tree.expandPath(new TreePath(current_node.getPath())); //Expand element so can see subelement.
+						mds_tree.getSelectionModel().setSelectionPath(new TreePath(new_node)); //works, but node on tree doesn't look hilighted!
+					    }
+					});
+
+				    //Done. Finish up.
+				    subelement = null;
+				    setVisible(false);
+				    atLeastOneSetChanged = true; //mark as changed
+				}
+			    else
+				{
+				    //A subelement with the same name exists. Display an error message.
+				    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Subelement_Already_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+				}
+			}
+		    else
+			{
+			    //This should not happen.
+			    System.err.println("Error: current_set is null in AddSubelementActionListener");
+			}
+		}
+	    else if(source == cancel_button)
+		{
+		    setVisible(false);
+		}
+	    else
+		{
+		    set_field.setText(current_node.toString()); //set to the name of the subelement
+		    name_field.setText("");
+		    setVisible(true);
+		}
+	}
+
+	public void dispose()
+	{
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	    set_field = null;
+	    super.dispose();
+	}
+    }
+
+
+    private class AddElementActionListener
+	extends ModalDialog
+	implements ActionListener {
+	protected JButton cancel_button = null;
+	protected JButton ok_button = null;
+	protected JLabel set_field = null;
+	protected NonWhitespaceField name_field = null;
+	public AddElementActionListener() {
+	    super(self);
+	    setModal(true);
+	    setSize(ADD_ELEMENT_SIZE);
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+	    Dictionary.setText(this, "GEMS.Add_Element");
+
+	    JPanel labels_pane = new JPanel();
+	    labels_pane.setOpaque(false);
+	    JLabel set_label = new JLabel();
+	    set_label.setOpaque(false);
+	    Dictionary.setText(set_label, "GEMS.Set");
+	    set_field = new JLabel();
+	    set_field.setOpaque(false);
+
+	    JPanel values_pane = new JPanel();
+	    values_pane.setOpaque(false);
+	    JLabel name_label = new JLabel();
+	    name_label.setOpaque(false);
+	    Dictionary.setText(name_label, "GEMS.Name");
+	    name_field = new NonWhitespaceField();
+	    name_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    name_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    name_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    name_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(name_field, "GEMS.Element_Name_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	    TextFieldEnabler ok_button_enabler = new TextFieldEnabler(ok_button);
+
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    ok_button_enabler.add(name_field);
+
+	    // Layout
+	    labels_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    labels_pane.setLayout(new GridLayout(2,1));
+	    labels_pane.add(set_label);
+	    labels_pane.add(name_label);
+
+	    values_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    values_pane.setLayout(new GridLayout(2,1));
+	    values_pane.add(set_field);
+	    values_pane.add(name_field);
+
+	    center_pane.setLayout(new BorderLayout(5,5));
+	    center_pane.add(labels_pane, BorderLayout.WEST);
+	    center_pane.add(values_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,2,0,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    setLocation((config.screen_size.width - ADD_ELEMENT_SIZE.width) / 2, (config.screen_size.height - ADD_ELEMENT_SIZE.height) / 2);
+	}
+
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+	    if(source == ok_button) {
+		// Add then dispose
+		String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+		// If this element doesn't already exist.
+		if(!current_set.containsElement(name_str)) {
+		    // Add it,
+		    String language_code = config.getLanguage();
+		    ElementWrapper element = current_set.addElement(name_str, language_code, null);
+
+		    // Then update the tree
+		    model.add(current_node, element, GEMSNode.ELEMENT);
+		    //SwingUtilities.invokeLater is used so that expandPath works on new parents.
+		    SwingUtilities.invokeLater(new Runnable() {
+			    public void run() {
+				mds_tree.expandPath(new TreePath(current_node.getPath())); //expand set.
+			    }
+			});
+		    
+		    // Done
+		    element = null;
+		    setVisible(false);
+                    
+                    //mark as changed
+                    atLeastOneSetChanged = true;
+                    current_set.setSetChanged(true);
+                 
+		}
+		// Otherwise show an error message and do not proceed.
+		else {
+		    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Element_Already_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		}
+		name_str = null;
+	    }
+	    else if(source == cancel_button) {
+		// Dispose
+		setVisible(false);
+	    }
+	    else {
+		if(current_set != null) { //Does this ever happen? --Matthew
+		    // You can't manually add elements to the Greenstone metadata set.
+		    if(!current_set.getNamespace().equals("")) {
+			set_field.setText(current_set.toString());
+			name_field.setText("");
+			// Display
+			setVisible(true);
+		    }
+		    // Warn the user that they can't do that dave. 
+		    else {
+                        
+			JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Cannot_Add_Elements_To_Greenstone_MDS"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		    }
+		}
+	    }
+	    source = null;
+	}
+ 
+	public void dispose() {
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	    set_field = null;
+	    super.dispose();
+	}
+    }
+	 
+   
+    private class AddSetActionListener
+	extends ModalDialog
+	implements ActionListener {
+	private JButton cancel_button = null;
+	private JButton ok_button = null;
+	private JTextField name_field = null;
+	private JTextField namespace_field = null;
+        private JComboBox existingSetJComboBox = new JComboBox();
+            
+	private String DO_NOT_INHERIT = Dictionary.get("GEMS.Add_Set.No_Inherit");
+	
+	public AddSetActionListener() {
+	    super(self);
+	    setModal(true);
+	    setSize(ADD_SET_SIZE);
+	    Dictionary.setText(this, "GEMS.Add_Set");
+
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+
+	    JPanel label_pane = new JPanel();
+	    JPanel boxes_pane = new JPanel();
+
+	    JLabel namespace_label = new JLabel();
+	    namespace_label.setOpaque(false);
+	    Dictionary.setText(namespace_label, "GEMS.Namespace");
+	    namespace_field = TransformCharacterTextField.createNamespaceTextField();
+	    namespace_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    namespace_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    namespace_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    namespace_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(namespace_field, "GEMS.Set_Namespace_Tooltip");
+
+	    JLabel name_label = new JLabel();
+	    name_label.setOpaque(false);
+	    Dictionary.setText(name_label, "GEMS.Name");
+	    name_field = new JTextField();
+	    name_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    name_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    name_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    name_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(name_field, "GEMS.Set_Name_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	    TextFieldEnabler ok_button_enabler = new TextFieldEnabler(ok_button);
+
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    ok_button_enabler.add(name_field);
+	    ok_button_enabler.add(namespace_field);
+
+	    // Layout
+	    label_pane.setLayout(new GridLayout(3,1));
+	    label_pane.add(name_label);
+	    label_pane.add(namespace_label);
+            boxes_pane.setLayout(new GridLayout(3,1));
+	    boxes_pane.add(name_field);
+	    boxes_pane.add(namespace_field);
+            
+            JLabel inheritJLabel = new JLabel(); 
+            Dictionary.setText(inheritJLabel, "GEMS.inheritMetadataSet");
+	   
+            label_pane.add(inheritJLabel);
+	    
+            int getSetsLen = msm.getSets().toArray().length;
+            Vector theSets = msm.getSets();
+            
+            for(int k = 0; k < theSets.size(); k++){
+                MetadataSet ms = (MetadataSet)theSets.elementAt(k);
+                
+                existingSetJComboBox.addItem(new String(ms.getName() + "                                                                             #% " + ms.getFile().toString()));
+                
+            }
+            //add sets to existingSetJComboBox
+            
+            JLabel dummyl = new JLabel();
+            Dictionary.setText(dummyl, "GEMS.Add_Set.No_Inherit");
+            
+	    existingSetJComboBox.addItem(DO_NOT_INHERIT);
+            existingSetJComboBox.setSelectedIndex(getSetsLen);
+            boxes_pane.add(existingSetJComboBox);
+             
+	    center_pane.setLayout(new BorderLayout(5,0));
+	    center_pane.add(label_pane, BorderLayout.WEST);
+	    center_pane.add(boxes_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,2,0,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+         
+	    setLocation((config.screen_size.width - ADD_SET_SIZE.width) / 2, (config.screen_size.height - ADD_SET_SIZE.height) / 2);
+	}
+
+        private boolean a_set_exists_with_this_namespace(String namespace){
+            Vector p = msm.getSets();
+            boolean flag = false;
+            for(int k = 0; k < p.size(); k++){
+                MetadataSet s = (MetadataSet)p.get(k);
+                
+                if(s.getNamespace().compareTo(namespace) == 0){
+                    flag = true;     
+                }
+            }
+            
+            return flag;
+        }
+
+
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+            
+	    if(source == ok_button) {
+                //this code below handles whether we inherit from an existing set or not
+             
+                String namespace_str = namespace_field.getText();
+		String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+		
+		// first check whether the namespace is valid, i.e. hasn't been used before
+		if(a_set_exists_with_this_namespace(namespace_str)) {
+		    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Set_Already_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		    return; // return back to the dialog
+		}
+		
+		//not based on another set
+		if (existingSetJComboBox.getSelectedItem().toString().equals(DO_NOT_INHERIT)){			
+		    MetadataSet set = msm.addSet(namespace_str, name_str);
+		    
+		    //mark as set changed = true
+		    set.setSetChanged(true);
+		    atLeastOneSetChanged = true;
+			
+		    // Update tree.
+		    model.add(null, set, GEMSNode.SET);
+		    // Done
+		    set = null;
+		    setVisible(false);
+		    
+		}
+                else {
+		    //Should change this to use XML parsing utilites --Matthew
+                    //lets grab the metadata we want to inherit from...
+                    String anitem = existingSetJComboBox.getSelectedItem().toString();
+                   
+		    //setup to read the filename from mds file
+		    String [] items = anitem.split("#%");
+		    try {
+			FileInputStream fstream = new FileInputStream(items[1].trim());
+			BufferedReader in = new BufferedReader(new InputStreamReader(fstream, "UTF8"));
+			    
+			FileOutputStream out;
+			PrintWriter p;
+			//setup to write to filename from existing file
+			try {
+			    out = new FileOutputStream(getGLIMetadataDirectoryPath() + namespace_str + ".mds");
+			    OutputStreamWriter pout = new OutputStreamWriter(out, "UTF8");
+			    p = new PrintWriter(pout);
+			    
+			    while(in.ready()){
+				String nextline = in.readLine();
+				
+				if(nextline.matches(".+namespace=\".+\".*") == true) {
+				    nextline = nextline.replaceFirst("namespace=\".+\"","namespace=\"" + namespace_str+ "\"");                          
+				}
+				else {
+				    nextline = nextline.replaceFirst(">.+</Name>",">" + name_str + "</Name>");
+				}
+				p.print(nextline + "\n");
+			    }
+			    p.close();
+			    
+			    MetadataSet loadedset = msm.loadMetadataSet(new File(getGLIMetadataDirectoryPath() + namespace_str + ".mds"));
+			    model.add(null, loadedset, GEMSNode.SET);
+			}
+			catch(Exception e){
+			    System.out.println("Print file error");
+			}
+		    }
+		    catch(Exception e){
+			System.out.println("File Input error");
+		    }
+		    
+		}
+		
+		name_str = null;
+		namespace_str = null;
+		setVisible(false);
+	    } //if(source == ok_button)
+	    else if(source == cancel_button) {
+		setVisible(false);
+	    }
+	    else {
+		name_field.setText("");
+		setVisible(true);
+	    }
+	} //actionPerformed
+		  
+	public void dispose() {
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	    super.dispose();
+	}
+    }
+
+	////
+    private class SetSelectionDialog
+	extends ModalDialog
+	implements ActionListener {
+	private JButton cancel_button = null;
+	private JButton ok_button = null;
+        private JComboBox existingSetJComboBox = new JComboBox();
+            
+	public SetSelectionDialog() {
+	    super(self);
+	    setModal(true);
+	    setSize(OPEN_SETS_SIZE);
+	    Dictionary.setText(this, "GEMS.Open_Set");
+
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+
+	    JPanel boxes_pane = new JPanel();
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+
+	    // Layout
+            boxes_pane.setLayout(new GridLayout(4,1));
+
+            JLabel inheritJLabel = new JLabel(); 
+            Dictionary.setText(inheritJLabel, "GEMS.inheritMetadataSet");
+	   
+            File[] col_dir = new File(Configuration.gsdl_path + File.separator + "collect").listFiles();
+            
+            existingSetJComboBox.addItem("Default Metadata Sets                                                                                        #% " + getGLIMetadataDirectoryPath());
+                
+            for(int k = 0; k < col_dir.length; k++){
+                  if(col_dir[k].isDirectory() && col_dir[k].getName().compareTo("CVS") != 0) {
+                    
+                       File metadir = new File(col_dir[k].toString() + File.separator + "metadata");
+                    if(metadir.exists())   
+                        existingSetJComboBox.addItem(new String(col_dir[k].getName() + " Collection Sets                                                                                                     #% " + col_dir[k].toString() + File.separator + "metadata"));
+                 
+                  }
+                
+            }
+            //add sets to existingSetJComboBox
+            
+            existingSetJComboBox.setSelectedIndex(0);
+            JLabel dummyl = new JLabel();
+            Dictionary.setText(dummyl, "GEMS.Add_Set.No_Inherit");
+            
+            boxes_pane.add(new JLabel("Select which group of metadata sets you wish to edit:\n\n\n\n"));
+            boxes_pane.add(new JLabel("\n"));
+            boxes_pane.add(existingSetJComboBox);
+            boxes_pane.add(new JLabel("\n"));
+
+            button_pane.setLayout(new GridLayout(1,3,0,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+            
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(boxes_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+         
+	    setLocation((config.screen_size.width - ADD_SET_SIZE.width) / 2, (config.screen_size.height - ADD_SET_SIZE.height) / 2);
+	}
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+   
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+            
+	    if(source == ok_button) {
+                //this code below handles whether we inherit from an existing set or not
+             
+               // String namespace_str = namespace_field.getText();
+		//String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+                String loc = (String) existingSetJComboBox.getSelectedItem();
+                
+                String [] whereto = loc.split("#%");
+                String set_directory = whereto[1].trim();
+                
+                //load the jfilechooser:
+                 
+		JFileChooser metadata_set_file_chooser = new JFileChooser(new File(set_directory));
+		metadata_set_file_chooser.setFileFilter(new MetadataSetFileFilter());
+		if (metadata_set_file_chooser.showDialog(this, Dictionary.get("General.Open")) == JFileChooser.APPROVE_OPTION) {
+		    MetadataSet metadata_set = msm.loadMetadataSet(metadata_set_file_chooser.getSelectedFile());
+		    // Update tree
+                    //only update tree if the metadata_set return is not null(it will be null if set is already
+                    //opened-Attila - oct 19 0 4
+                    if(metadata_set != null) {
+                        model.add(null, metadata_set, GEMSNode.SET);
+                    }
+		}
+                
+         
+                setVisible(false);
+            }
+	    else if(source == cancel_button) {
+		setVisible(false);
+	    }
+	    else {
+		setVisible(true);
+	    }
+	}
+		  
+	public void dispose() {
+	    cancel_button = null;
+	    ok_button = null;
+	    super.dispose();
+	}
+    }
+
+    /*
+      Class to handle removing set or element attributes.
+    */
+    private class RemoveAttributeActionListener
+	implements ActionListener, KeyListener {
+
+	/**
+	   Any implementation of KeyListener must include these mothods.
+	   When the DELETE key is pressed, fire the actionPerformed event.
+	   @param e A <strong>KeyEvent</strong> containing information about the event.
+	*/
+	public void keyTyped(KeyEvent e){
+	    //Do nothing
+	}
+	    public void keyPressed(KeyEvent e)
+		{
+		    if(e.getKeyCode() == java.awt.event.KeyEvent.VK_DELETE)
+			{
+			    //DELETE key pressed
+			    actionPerformed((ActionEvent)null); //is there a better way?
+			}
+		}
+	    public void keyReleased(KeyEvent e) {
+		//Do nothing
+	    }
+		    
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    if(current_attribute != -1) {
+		int result = JOptionPane.showOptionDialog(self, Dictionary.get("GEMS.Confirm_Removal", Dictionary.get("GEMS.Attribute")), Dictionary.get("GEMS.Confirm_Removal_Title"), JOptionPane.DEFAULT_OPTION, JOptionPane.QUESTION_MESSAGE, null, dialog_options, dialog_options[0]);
+		// Remove this attribute
+		if(result == 0) {
+		    ignore = true;
+		    //Attributes are tricky little beasties, as they can come from several different places and depend upon what other little beasties have been selected
+		    //Has an element been selected
+		    if(current_element != null) {
+			// Remove element attribute
+			String name = (String) element_attributes.getValueAt(current_attribute, 0);
+			String language = (String) element_attributes.getValueAt(current_attribute, 1);
+			String value = (String) element_attributes.getValueAt(current_attribute, 2);
+			current_element.removeAttribute(name, language, value);
+
+			// Refresh table
+			((AttributeTableModel)element_attributes.getModel()).removeRow(current_attribute);
+		    }
+		    else if(current_set != null) {
+			String name = (String) set_attributes.getValueAt(current_attribute, 0);
+			// Remove set attribute
+			current_set.removeAttribute(name);
+			// Refresh table
+			((AttributeTableModel)set_attributes.getModel()).removeRow(current_attribute);
+		    }
+		    ignore = false;
+		    atLeastOneSetChanged= true;
+		}
+	    }
+	    else
+		{
+		    System.err.println("Tried to remove attribute, but current_attribute == -1!"); //This should not happen
+		}
+	}
+    }
+
+    /*
+      Class to handle deletion of elements (including subelements).
+
+      Last modified 2/02/05 by Matthew Whyte
+     */
+    private class RemoveElementActionListener
+	implements ActionListener {
+
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to provide an editing prompt.
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+         
+	    if(current_element != null) {
+		int result = JOptionPane.showOptionDialog(self, Dictionary.get("GEMS.Confirm_Removal", Dictionary.get("GEMS.Element")), Dictionary.get("GEMS.Confirm_Removal_Title"), JOptionPane.DEFAULT_OPTION, JOptionPane.QUESTION_MESSAGE, null, dialog_options, dialog_options[0]);
+		// Remove this attribute
+		if(result == 0) {
+		    //Get the parent element
+		    GEMSNode parent_node = (GEMSNode)current_node.getParent();
+		    ElementWrapper parent_wrapped = parent_node.getElement();
+
+		    ignore = true;
+		    msm.removeElement(current_element, parent_wrapped);
+		    // Clear selection
+		    mds_tree.clearSelection();
+		    model.remove(current_node);
+
+		    // Show a blank panel.
+		    card_layout.show(details_pane, BLANK);
+		    ignore = false;
+                    
+                     atLeastOneSetChanged = true;
+		}
+	    }
+	    else {
+		System.err.println("No element currently selected."); //This should not happen!
+	    }
+	}
+    }
+
+
+    /**
+       Class to handle renaming of a set and/or it's namespace.
+
+       @author: Matthew Whyte
+       Date last modified: 10/01/05
+    */
+    private class RenameSetActionListener extends ModalDialog implements ActionListener {
+
+	private JButton cancel_button = null;
+	private JButton ok_button = null;
+	private JTextField name_field = null;
+	private JTextField namespace_field = null;
+            
+	public RenameSetActionListener() {
+	    super(self);
+	    setModal(true);
+	    setSize(RENAME_SET_SIZE);
+	    Dictionary.setText(this, "GEMS.Rename_Set");
+
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+
+	    JPanel label_pane = new JPanel();
+	    JPanel boxes_pane = new JPanel();
+
+	    JLabel namespace_label = new JLabel();
+	    namespace_label.setOpaque(false);
+	    Dictionary.setText(namespace_label, "GEMS.Namespace");
+	    namespace_field = TransformCharacterTextField.createNamespaceTextField();
+	    namespace_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    namespace_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    namespace_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    namespace_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(namespace_field, "GEMS.Rename_Namespace_Tooltip");
+
+	    JLabel name_label = new JLabel();
+	    name_label.setOpaque(false);
+	    Dictionary.setText(name_label, "GEMS.Name");
+	    name_field = new JTextField();
+	    name_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    name_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    name_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    name_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(name_field, "GEMS.Rename_Name_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	    TextFieldEnabler ok_button_enabler = new TextFieldEnabler(ok_button);
+
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    ok_button_enabler.add(name_field);
+	    ok_button_enabler.add(namespace_field);
+
+	    // Layout
+	    label_pane.setLayout(new GridLayout(2,1));
+	    label_pane.add(name_label);
+	    label_pane.add(namespace_label);
+            boxes_pane.setLayout(new GridLayout(2,1));
+	    boxes_pane.add(name_field);
+	    boxes_pane.add(namespace_field);
+            
+	    center_pane.setLayout(new BorderLayout(5,0));
+	    center_pane.add(label_pane, BorderLayout.WEST);
+	    center_pane.add(boxes_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,2,0,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+         
+	    setLocation((config.screen_size.width - RENAME_SET_SIZE.width) / 2, (config.screen_size.height - RENAME_SET_SIZE.height) / 2);
+	}
+
+        private boolean a_set_exists_with_this_namespace(String namespace){
+            Vector p = msm.getSets();
+            boolean flag = false;
+            for(int k = 0; k < p.size(); k++){
+                MetadataSet s = (MetadataSet)p.get(k);
+                
+                if(s.getNamespace().compareTo(namespace) == 0){
+                    flag = true;     
+                }
+            }
+            return flag;
+        }
+
+
+	/** 
+	    Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component,
+	    allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+         */
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+            
+	    if(source == ok_button)
+		{
+		    String namespace_str = namespace_field.getText();
+		    String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+		    //Note: The name in name_str is language dependent. However, currently GEMS does not create new sets with different names for different languages(?).
+		    String current_set_namespace = current_set.getNamespace();
+
+		    //Ensure that the namespace is unique. 
+			if(!(a_set_exists_with_this_namespace(namespace_str)) || (namespace_str.equals(current_set_namespace)))
+			{
+			    //Hide the set that are about to rename
+			    mds_tree.clearSelection();
+			    model.remove(current_node);
+
+			    //Rename set & namespace
+			    msm.renameSet(current_set, namespace_str, name_str);
+			    //Add (renamed) set back to tree
+			    model.add(null, current_set, GEMSNode.SET);
+
+			    //Clean up
+			    card_layout.show(details_pane, BLANK); // Show a blank panel.
+			    ignore = false;
+			    atLeastOneSetChanged = true;
+			    setVisible(false);
+			}
+			    
+		    // Otherwise show an error message and do not proceed.
+		    else
+			{
+			    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Set_Already_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+			}
+		    name_str = null;
+		    
+		}
+
+	    else if(source == cancel_button)
+		{
+		    setVisible(false); //hide dialog
+		}
+	    else
+		{
+		    name_field.setText(current_set.getName());
+		    namespace_field.setText(current_set.getNamespace());
+		    setVisible(true); //Show the dialog
+		}   
+	    source = null;
+	}
+	
+	public void dispose() {
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	    super.dispose();
+	}
+    }
+
+   
+
+ /**
+       Class to handle renaming of an element.
+
+       @author: Matthew Whyte
+       Date last modified: 10/01/05
+    */
+    private class RenameElementActionListener extends ModalDialog implements ActionListener {
+
+	protected JButton cancel_button = null;
+	protected JButton ok_button = null;
+	//private JLabel label = null;
+	protected NonWhitespaceField name_field = null;
+
+	public RenameElementActionListener() {
+	    super(self);
+	    setModal(true);
+	    setSize(RENAME_ELEMENT_SIZE);
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    content_pane.setBackground(config.getColor("coloring.collection_heading_background", false));
+	    JPanel center_pane = new JPanel();
+	    center_pane.setOpaque(false);
+
+	    JPanel labels_pane = new JPanel();
+	    labels_pane.setOpaque(false);
+
+	    JPanel values_pane = new JPanel();
+	    values_pane.setOpaque(false);
+	    JLabel name_label = new JLabel();
+	    name_label.setOpaque(false);
+	    Dictionary.setText(name_label, "GEMS.New_Name");
+	    name_field = new NonWhitespaceField();
+	    name_field.setBackground(config.getColor("coloring.collection_tree_background", false));
+	    name_field.setForeground(config.getColor("coloring.collection_tree_foreground", false));
+	    name_field.setSelectionColor(config.getColor("coloring.collection_selection_background", false));
+	    name_field.setSelectedTextColor(config.getColor("coloring.collection_selection_foreground", false));
+	    Dictionary.setTooltip(name_field, "GEMS.Rename_Element_Tooltip");
+
+	    JPanel button_pane = new JPanel();
+	    button_pane.setOpaque(false);
+
+	    ok_button = new GLIButton();
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    getRootPane().setDefaultButton(ok_button);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	    TextFieldEnabler ok_button_enabler = new TextFieldEnabler(ok_button);
+
+	    Dictionary.setText(this, "GEMS.Rename_Element");
+	    
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    ok_button_enabler.add(name_field);
+
+	    // Layout
+	    labels_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    labels_pane.setLayout(new GridLayout(1,1));
+	    //labels_pane.add(set_label);
+	    labels_pane.add(name_label);
+
+	    values_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    values_pane.setLayout(new GridLayout(1,1));
+	    //values_pane.add(label);
+	    values_pane.add(name_field);
+
+	    center_pane.setLayout(new BorderLayout(5,5));
+	    center_pane.add(labels_pane, BorderLayout.WEST);
+	    center_pane.add(values_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,2,0,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	    setLocation((config.screen_size.width - ADD_ELEMENT_SIZE.width) / 2, (config.screen_size.height - ADD_ELEMENT_SIZE.height) / 2);
+	}
+
+
+	/**
+	   Any implementation of ActionListener must include this method so that we can be informed when an action has occured.
+	   @param event An <string>ActionEvent</strong> containing information about the event.
+	*/
+	public void actionPerformed(ActionEvent event)
+	{
+	    Object source = event.getSource();
+
+	    if(source == ok_button)
+		{
+		    TreeNode selPath = current_node.getParent();
+		    GEMSNode parent = (GEMSNode)selPath;
+		    MetadataSet currentSet = msm.getSet(current_element.getNamespace());
+		    String name_str = Codec.transform(Codec.transformUnicode(name_field.getText()), Codec.TEXT_TO_DOM);
+		    GEMSNode new_node = null;
+
+		    // If this element doesn't already exist.
+		    if(!currentSet.containsElement(name_str)) //Do the renaming!
+			{   
+			    model.remove(current_node); //Hide the element that are about to rename
+			    msm.renameElement(current_element, name_str); //rename the element
+			    //Add the new element
+			    new_node = model.add(parent, current_element, GEMSNode.ELEMENT);
+			    
+			    //mds_tree.setSelectionPath(new_node_path); //would like to do this but doesn't hilight tree :-(
+			    card_layout.show(details_pane, BLANK); // Show a blank panel.
+			    
+			    //finish up
+			    currentSet.setSetChanged(true);
+			    atLeastOneSetChanged = true;
+			    setVisible(false);
+			}
+			    
+		    // Otherwise show an error message and do not proceed.
+		    else
+			{
+			    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Element_Already_Exists"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+			}
+		    name_str = null;
+		    
+		}
+
+	    else if(source == cancel_button)
+		{
+		    setVisible(false); //hide dialog
+		}
+	    else
+		{
+		    name_field.setText(current_element.getNameOnly()); //show old name (without namespace)
+		    setVisible(true); //Show the dialog
+		}
+		    
+	    source = null;
+	}
+	
+	public void dispose() {
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	    super.dispose();
+	}
+    }
+
+
+    /**
+       Class to handle DELETE key presses on the tree.
+       The class then fires the appropriate event to remove that element/set.
+       Attributes in the table are handled directly by the RemoveAttributeActionListener
+
+       Author: Matthew Whyte
+       Date last modified: 25/01/05
+    */
+    private class KeyPressedHandler implements KeyListener, ActionListener {	
+
+	// All implementations of KeyListener requires these methods
+	public void keyTyped(KeyEvent e){
+	    //Do nothing
+	}
+	public void keyPressed(KeyEvent e)
+	{
+
+	    if(e.getKeyCode() == java.awt.event.KeyEvent.VK_DELETE) //DELETE key pressed
+	    {
+		this.removeItem();
+	    }
+	}
+	public void keyReleased(KeyEvent e)
+	    {
+		//Do nothing
+	    }
+
+	public void actionPerformed(ActionEvent event)
+	{
+	    this.removeItem();
+	}
+
+	public void removeItem()
+	{
+	    boolean cont = true;
+	    
+	    /**
+	       Check to see if more than one tree node/leaf is selected. If so don't continue.
+	       
+	       This could be updated to delete multiple nodes/leaves in the futere.
+	       But not me and not today!
+	    */
+	    TreePath[] multiSelectionCheck = mds_tree.getSelectionPaths();
+	    if(multiSelectionCheck.length > 1)
+		{
+		    cont = false;
+		    Toolkit.getDefaultToolkit().beep();
+		}
+	    
+	    if(cont)
+		{
+		    //Check to see if pressed on Set or Element.
+		    TreePath selPath = mds_tree.getSelectionPath();
+		    GEMSNode t = (GEMSNode)selPath.getLastPathComponent();
+		    
+		    if(t.type == T_ELEMENT) //Remove the element
+			{
+			    RemoveElementActionListener removeElement = new RemoveElementActionListener();
+			    removeElement.actionPerformed((ActionEvent)null);
+			}
+		    else if(t.type == T_SET) //Remove the whole set
+			{
+			    RemoveSetActionListener removeSet = new RemoveSetActionListener();
+			    removeSet.actionPerformed((ActionEvent)null);
+			}
+		}
+	}
+    }
+    
+
+    private class RemoveSetActionListener
+	implements ActionListener {
+
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action as occured on our registered component, allowing us to
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    if(current_set != null) {
+                
+                //If the current_set_filename is dls or dublin or ex and in the GLI metadata directory, then do
+                //not let anyone remove them --Attila
+                File current_set_filename = current_set.getFile();
+                if(current_set_filename.toString().compareTo(new String(getGLIMetadataDirectoryPath() + "dls.mds")) == 0 ||
+                    current_set_filename.toString().compareTo(new String(getGLIMetadataDirectoryPath() + "dublin.mds")) == 0 ||
+                    current_set_filename.toString().compareTo(new String(getGLIMetadataDirectoryPath() + "ex.mds")) == 0){
+                  
+                    JOptionPane.showMessageDialog(self, Dictionary.get("GEMS.Cannot_Delete_Core_Sets_Error"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+                    return;
+                }
+                
+                
+		int result = JOptionPane.showOptionDialog(self, Dictionary.get("GEMS.Confirm_Removal", Dictionary.get("GEMS.Set") + " " +  Dictionary.get("GEMS.Cannot_Undo")), Dictionary.get("GEMS.Confirm_Removal_Title"), JOptionPane.DEFAULT_OPTION, JOptionPane.QUESTION_MESSAGE, null, dialog_options, dialog_options[0]);
+		// Remove the currently selected set
+		if(result == 0) {
+                    
+		    ignore = true;
+		    msm.removeSet(current_set);
+		    // Clear selection
+		    mds_tree.clearSelection();
+		    model.remove(current_node);
+		    card_layout.show(details_pane, BLANK); //Show a blank panel.
+		    ignore = false;
+ 
+		}
+	    }
+	}
+    }
+	 
+    /** Used to enable a certain target component if and only if all the registered text fields contain non-zero length strings. */
+    private class TextFieldEnabler
+	implements DocumentListener {
+	private boolean ignore = false;
+	private Component target = null;
+	private JTextComponent[] fields = null;
+	public TextFieldEnabler(Component target) {
+	    super();
+	    this.target = target;
+	}
+	public void add(JTextComponent field) {
+	    // Record this field as being one we depend upon
+	    if(fields == null) {
+		fields = new JTextComponent[1];
+		fields[0] = field;
+	    }
+	    else {
+		JTextComponent[] temp = new JTextComponent[fields.length + 1];
+		System.arraycopy(fields, 0, temp, 0, fields.length);
+		temp[fields.length] = field;
+		fields = temp;
+		temp = null;
+	    }
+	    // Add the appropriate listener
+	    field.getDocument().addDocumentListener(this);
+	}
+	/** Gives notification that an attribute or set of attributes changed. */
+	public void changedUpdate(DocumentEvent e) {
+	    canEnable();
+	}
+	/** Gives notification that there was an insert into the document. */
+	public void insertUpdate(DocumentEvent e) {
+	    canEnable();
+	}
+        /** Gives notification that a portion of the document has been removed. */
+	public void removeUpdate(DocumentEvent e) {
+	    canEnable();
+	}
+	private void canEnable() {
+	    if(!ignore) {
+		ignore = true;
+		boolean can_enable = true;
+		for(int i = 0; can_enable && i < fields.length; i++) {
+		    can_enable = can_enable && (fields[i].getText().length() > 0);
+		}
+		target.setEnabled(can_enable);
+		ignore = false;
+	    }
+	}
+    }
+
+    private class AttributesListSelectionListener
+	implements ListSelectionListener
+    {
+	private JTable table = null;
+	public AttributesListSelectionListener(JTable table)
+	{
+	    this.table = table;
+	}
+
+	public void valueChanged(ListSelectionEvent event)
+	{
+	    if(!event.getValueIsAdjusting())
+		{
+		    current_attribute = table.getSelectedRow();
+		}
+	}
+    }
+
+    private class MDSTreeSelectionListener
+	implements TreeSelectionListener {
+	public void valueChanged(TreeSelectionEvent event) {
+	    if(!ignore) {
+		// Clear all variables based on previous tree selection
+		current_element = null;
+		current_set = null;
+		// Now process the node selected
+		TreePath path = event.getPath();
+		current_node = (GEMSNode)path.getLastPathComponent();
+		// What we show depends on the node type...
+		AttributeTableModel atm = null;
+		TreeSet attributes = null;
+		current_attribute_type = current_node.getType();
+		switch(current_attribute_type) {
+
+		case GEMSNode.ELEMENT:
+		    current_element = current_node.getElement();
+		    atm = current_node.getModel();
+		    if(atm == null) {
+			atm = new AttributeTableModel(current_element.getAttributes(), Dictionary.get("GEMS.Name"), Dictionary.get("GEMS.Language_Code"), Dictionary.get("GEMS.Value"), "");
+			//current_node.setModel(atm);
+		    }
+		    element_attributes.setModel(atm);
+		    atm.setScrollPane(element_attributes_scroll);
+		    atm.setTable(element_attributes);
+
+		    card_layout.show(details_pane, ELEMENT);
+		    break;
+		case GEMSNode.SET:
+		    current_set = current_node.getSet();
+		    atm = current_node.getModel();
+		    if(atm == null) {
+			NamedNodeMap temp = current_set.getAttributes();
+			attributes = new TreeSet();
+			for(int i = 0; i < temp.getLength(); i++) {
+			    Attr attribute = (Attr) temp.item(i);
+			    // We don't show the namespace attribute, as it is used as a unique primary key and should never be changed or removed in itself.
+			    if(!attribute.getName().equals("namespace")) {
+				attributes.add(new Attribute(attribute.getName(), attribute.getValue()));
+			    }
+			    attribute = null;
+			}
+			atm = new AttributeTableModel(attributes, Dictionary.get("GEMS.Name"), Dictionary.get("GEMS.Value"), "");
+			temp = null;
+		    }
+		    set_attributes.setModel(atm);
+		    atm.setScrollPane(set_attributes_scroll);
+		    atm.setTable(set_attributes);
+						 
+		    card_layout.show(details_pane, SET);
+		    attributes = null;
+		    break;
+
+		default:
+		    // Show a blank panel.
+		    card_layout.show(details_pane, BLANK);
+		}
+		attributes = null;
+		path = null;
+		atm = null;
+	    }
+	}
+    }
+
+    private class GEMSTreeCellRenderer
+	extends DefaultTreeCellRenderer {
+	public Component getTreeCellRendererComponent(JTree tree, Object value, boolean selected, boolean expanded, boolean leaf, int row, boolean hasFocus) {
+	    super.getTreeCellRendererComponent(tree, value, selected, expanded, leaf, row, hasFocus);
+	    setToolTipText(value.toString());
+	    return this;
+	}
+    }
+    
+
+    //PopupListener is our main manager for all right-click popups
+    class PopupListener extends MouseAdapter implements ActionListener{
+	JPopupMenu setPopup;
+	JPopupMenu elementPopup;
+	JPopupMenu attributePopup;
+
+	//all of our popup menu items EVER!
+	JMenuItem menuAddAttributeSet;
+	JMenuItem menuAddElement;
+	JMenuItem menuRenameSet;
+	JMenuItem menuRemoveSet;
+
+	JMenuItem menuAddAttributeElement;
+	JMenuItem menuAddSubelement;
+	JMenuItem menuRenameElement;
+	JMenuItem menuDeleteElement;
+
+	JMenuItem menuAddAttributeAttribute;
+	JMenuItem menuEditValue;
+	JMenuItem menuDeleteAttribute;
+	
+        GEMS GEMSRef = null;
+        PopupListener(GEMS g) {
+	    setPopup = new JPopupMenu();
+	    elementPopup = new JPopupMenu();
+	    attributePopup = new JPopupMenu();
+            
+            //reference to our parent(GEMS), because some of the actionlisteners would not
+            //work unless they were declared in GEMS, and I need a way to reference GEMS
+	    GEMSRef = g;
+	    
+	    //Create the menu items
+	    menuAddAttributeSet = new JMenuItem();
+	    Dictionary.setBoth(menuAddAttributeSet, "GEMS.Add_Attribute", "GEMS.Add_Attribute_Tooltip");
+	    menuAddElement =new JMenuItem();
+	    Dictionary.setBoth(menuAddElement,"GEMS.Add_Element","GEMS.Add_Element_Tooltip");
+	    menuRenameSet = new JMenuItem();
+	    Dictionary.setBoth(menuRenameSet,"GEMS.Rename_Set", "GEMS.Rename_Set_Tooltip");
+            menuRemoveSet = new JMenuItem();
+	    Dictionary.setBoth(menuRemoveSet, "GEMS.Remove_Set","GEMS.Remove_Set_Tooltip");
+            
+            menuAddAttributeElement=new JMenuItem();
+            Dictionary.setBoth(menuAddAttributeElement,"GEMS.Add_Attribute", "GEMS.Add_Attribute_Tooltip");
+	    menuAddSubelement=new JMenuItem();
+            Dictionary.setBoth(menuAddSubelement,"GEMS.Add_Subelement", "GEMS.Add_Subelement_Tooltip");
+            menuRenameElement=new JMenuItem();
+            Dictionary.setBoth(menuRenameElement,"GEMS.Rename_Element", "GEMS.Rename_Element_Tooltip");
+            menuDeleteElement=new JMenuItem();
+	    Dictionary.setBoth(menuDeleteElement,"GEMS.Remove_Element", "GEMS.Remove_Element_Tooltip" );
+
+	    menuAddAttributeAttribute=new JMenuItem();
+	    Dictionary.setBoth(menuAddAttributeAttribute,"GEMS.Add_Attribute", "GEMS.Add_Attribute_Tooltip");
+            menuEditValue=new JMenuItem();
+            Dictionary.setBoth(menuEditValue,"GEMS.Edit_Value", "GEMS.Edit_Value_Tooltip");
+            menuDeleteAttribute=new JMenuItem();
+	    Dictionary.setBoth(menuDeleteAttribute,"GEMS.Remove_Attribute", "GEMS.Remove_Attribute_Tooltip");
+             
+	    //Associate the menu items with the appropriate popups:
+	    setPopup.add(menuAddAttributeSet);
+	    setPopup.add(menuAddElement);
+	    setPopup.add(menuRenameSet);
+	    setPopup.add(menuRemoveSet);
+            
+	    elementPopup.add(menuAddAttributeElement);
+	    elementPopup.add(menuAddSubelement);
+	    elementPopup.add(menuRenameElement);
+	    elementPopup.add(menuDeleteElement);
+            
+	    attributePopup.add(menuAddAttributeAttribute);
+	    attributePopup.add(menuEditValue);
+	    attributePopup.add(menuDeleteAttribute);
+	    
+	    //Add appropriate action listeners to the menu items.
+	    menuAddAttributeSet.addActionListener(new AddOrEditAttributeActionListener());
+	    menuAddElement.addActionListener(new AddElementActionListener());
+	    menuRenameSet.addActionListener(new RenameSetActionListener());
+	    menuRemoveSet.addActionListener(new RemoveSetActionListener());
+	    
+	    menuAddAttributeElement.addActionListener(new AddOrEditAttributeActionListener());
+	    menuAddSubelement.addActionListener(new AddSubelementActionListener());
+	    menuRenameElement.addActionListener(new RenameElementActionListener());
+	    menuDeleteElement.addActionListener(new RemoveElementActionListener());
+	   
+	    menuEditValue.addActionListener(new AddOrEditAttributeActionListener());
+	    menuDeleteAttribute.addActionListener(new RemoveAttributeActionListener());
+	    menuAddAttributeAttribute.addActionListener(new AddOrEditAttributeActionListener());            
+        }
+
+        public void mouseClicked(MouseEvent e) {
+            maybeShowPopup(e);
+        }
+        public void mousePressed(MouseEvent e){
+        maybeShowPopup(e);
+	}
+        public void mouseReleased(MouseEvent e) {
+            maybeShowPopup(e);
+        }
+
+        private void maybeShowPopup(MouseEvent e) {
+	    TreePath selPath = mds_tree.getPathForLocation(e.getX(), e.getY());
+	    Point p = null;
+            if (e.isPopupTrigger()) {
+		
+		if(e.getSource() == mds_tree) {
+		    try {
+			GEMSNode t = (GEMSNode)selPath.getLastPathComponent();
+			
+			mds_tree.setSelectionPath(selPath); //Select node right-clicked on
+			
+			if(t.type == T_ELEMENT){
+			    elementPopup.show(e.getComponent(), e.getX(), e.getY());
+			}
+			else if(t.type == T_SET){
+			    setPopup.show(e.getComponent(), e.getX(), e.getY());
+			}
+		    }
+		    catch(Exception NullPointerException) {
+			//Right-clicked in tree area, but no item to select. Do nothing.
+		    }
+		}
+		else if(e.getSource() == set_attributes){ //When right-click on table of set attributes
+		    //Select the table row that was right-clicked on.
+		    p = e.getPoint();
+		    set_attributes.changeSelection(set_attributes.rowAtPoint(p), set_attributes.columnAtPoint(p), false, false);
+		    attributePopup.show(e.getComponent(), e.getX(), e.getY());  
+		}
+		else if(e.getSource() == element_attributes)  { //When right-click on table of element attributes
+		    //Select the table row that was right-clicked on.
+		    p = e.getPoint();
+		    element_attributes.changeSelection(element_attributes.rowAtPoint(p), element_attributes.columnAtPoint(p), false, false);
+		    attributePopup.show(e.getComponent(), e.getX(), e.getY());
+		}
+            }  
+        }
+      
+      public void actionPerformed(ActionEvent e) {
+	  //This should never happen!
+	  System.err.println("Error: actionPerformed was called in class PopupListener. This should never happen!");
+      }
+    }
+  
+    /** Wrap a ElementWrapper so we get its name when we ask for toString(), not its identifier. */
+    private class NameElementWrapperEntry 
+	implements Comparable {
+	private ElementWrapper element_wrapper = null;
+	public NameElementWrapperEntry(Object object) {
+	    this.element_wrapper = (ElementWrapper) object;
+	}
+	public int compareTo(Object object) {
+	    return element_wrapper.compareTo(object);
+	}
+	public boolean equals(Object object) {
+	    return element_wrapper.equals(object);
+	}
+	public String toString() {
+	    return element_wrapper.getName();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSLanguageManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSLanguageManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSLanguageManager.java	(revision 21112)
@@ -0,0 +1,149 @@
+ /**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at th * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *############################################## ##########################
+ */
+
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.cdm.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.w3c.dom.*;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.FactoryConfigurationError;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.w3c.dom.Document;
+import org.w3c.dom.DOMException;
+
+/** This class manages the language commands, remembering both a list of languages to build indexes in, plus the default language.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ * Created on October 27, 2004, 3:31 PM
+ * Modified 1/02/05 by Matthew Whyte
+ */
+ public class GEMSLanguageManager {
+     private Document document;
+     private DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+     private Vector languageCodes;
+     private Vector languageCodesAndNames;
+     public GEMSLanguageManager(String languagexml_location) {
+	 
+	 this.languageCodes = new Vector();
+	 this.languageCodesAndNames = new Vector();
+
+	 try {
+	     
+	     File f = new File(languagexml_location);
+	     
+	     DocumentBuilder builder = factory.newDocumentBuilder();
+	     document = builder.parse(f);
+	     org.w3c.dom.Element docelement = document.getDocumentElement();
+	     
+	     //get all language nodes
+	     NodeList mynodelist = docelement.getChildNodes();
+	     
+	     //get a single language text
+	     //System.out.println(mynodelist.item(1).getChildNodes().item(0).getNodeValue()); //debug
+	     
+	     for(int j = 1; j < mynodelist.getLength(); j++){
+		 
+		 //read text value for the element
+		 NodeList childnodesofelement = mynodelist.item(j).getChildNodes();
+		 NamedNodeMap nnm = mynodelist.item(j).getAttributes();
+		 
+		 if(childnodesofelement.getLength() == 1)
+		     {
+			 String code = nnm.item(0).getNodeValue();
+			 String name = nnm.item(4).getNodeValue();
+			 String name_value = childnodesofelement.item(0).getNodeValue();
+			 //language_box.addItem(code.toLowerCase());
+			 this.languageCodes.add(code.toLowerCase());
+			 this.languageCodesAndNames.add(name.toLowerCase() + "               \t(" + code.toLowerCase() + ")");
+		     } //end if
+	     }//end for
+	 } //end try
+	 catch(SAXException sxe) {
+	     System.out.println("sxe exception\n");
+	 }
+	 catch(ParserConfigurationException pce) {
+             System.out.println("pce exception\n");
+	 }
+	 catch(IOException ioe){
+	     System.out.println("ioe exception\n");
+	 }
+	 
+	 //end create dom object for language_box
+	 
+     }
+     public Vector getLanguageCodesAndNames() {
+	 return languageCodesAndNames;
+     }
+
+     public Vector getLanguageCodes() { 
+	 return languageCodes;
+     }
+     public Object[] getLanguageCodesToArray() { 
+	 return languageCodes.toArray();
+     }
+     public String getEnabledLanguageCodes() {
+	 String enabled_languages =  Configuration.getString("Gems.Preferences.Selected_Languages", true);
+	 return enabled_languages;   
+     }
+     public String[] getEnabledLanguageCodesToArray() {
+	 String enabled_languages =  Configuration.getString("Gems.Preferences.Selected_Languages", true); 
+	 String [] enabled_languages_split = enabled_languages.split(",");
+	 // String [] trimmed_enabled_lang_split = null;
+	 
+	 //  for(int p = 0; p < enabled_languages_split.length; p++) {
+	 //trimmed_enabled_lang_split[p] = enabled_languages_split[p].trim(); 
+	 //  System.out.println("gemslang: trimm: " + trimmed_enabled_lang_split[p] + " orig: "+ enabled_languages_split[p].trim());
+	 // }//for p < enabled lang split
+	 
+	 return enabled_languages_split;
+     }  
+ }
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSModel.java	(revision 21112)
@@ -0,0 +1,79 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import javax.swing.tree.DefaultTreeModel;
+import org.greenstone.gatherer.util.SynchronizedTreeModelTools;
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class GEMSModel 
+    extends DefaultTreeModel {
+
+    public GEMSModel() {
+	super(new GEMSNode());
+    }
+
+    public GEMSNode add(GEMSNode parent, Object object, int type) {
+	if(parent == null) {
+	    parent = (GEMSNode) root;
+	}
+
+	GEMSNode new_node = new GEMSNode(type, object, null);
+	SynchronizedTreeModelTools.insertNodeInto(this, parent, new_node);
+
+	return new GEMSNode(type, object, parent);
+    }
+
+    public GEMSNode getProfileNode() {
+	GEMSNode node = (GEMSNode) root;
+	return (GEMSNode) node.getChildAt(node.getChildCount() - 1);
+    }
+
+    /**
+       Remove a node from the tree.
+       @param node the node to remove, as a <strong>GEMSNode</strong>
+       @author Matthew Whyte
+       @date last modified: 21/01/04
+     */
+    public void remove(GEMSNode node)
+    {
+	SynchronizedTreeModelTools.removeNodeFromParent(this, node);
+	return;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSNode.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSNode.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSNode.java	(revision 21112)
@@ -0,0 +1,221 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.io.*;
+import java.util.*;
+import javax.swing.tree.*;
+import javax.swing.JPopupMenu;
+import java.awt.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.w3c.dom.*;
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ * Modified 2/02/05 by Matthew Whyte
+ */
+public class GEMSNode
+    extends DefaultMutableTreeNode{
+    private AttributeTableModel model = null;
+    public int type = 0;
+    private String text = null;
+    static final public int COLLECTION = 0;
+    static final public int ELEMENT    = 1;
+    static final public int PROFILER   = 2;
+    static final public int ROOT       = 3; 
+    static final public int SET        = 4;
+    //static final public int SUBELEMENT = 5; 
+
+    public GEMSNode() {
+	this.type = ROOT;
+    }
+
+    public GEMSNode(int type, Object object, GEMSNode parent) {
+	this.userObject = object;
+	this.parent = parent;
+	this.type = type;
+	this.text = null;
+    }
+
+    public Enumeration children() {
+	if(children == null) {
+	    mapChildren();
+	}		  
+	return children.elements();
+    }
+
+    public boolean getAllowsChildren() {
+	return true;
+    }
+
+    public TreeNode getChildAt(int index) {
+	if(children == null) {
+	    mapChildren();
+	}
+	return (TreeNode) children.get(index);
+    }
+    public int getChildCount() {
+	if(children == null) {
+	    mapChildren();
+	}
+	return children.size();
+    }
+    public ElementWrapper getElement() {
+	ElementWrapper result = null;
+	if(type == ELEMENT) {
+	    result = (ElementWrapper) userObject;
+	}
+	return result;	
+    }
+
+    public int getIndex(TreeNode node) {
+	if(children == null) {
+	    mapChildren();
+	}
+	return children.indexOf(node);
+    }
+    public AttributeTableModel getModel() {
+	return model;
+    }
+    public TreeNode getParent() {
+	return parent;
+    }
+    public MetadataSet getSet() {
+	MetadataSet result = null;
+	if(type == SET) {
+	    result = (MetadataSet) userObject;
+	}
+	return result;
+    }
+    public int getType() {
+	return type;
+    }
+    public boolean isLeaf() {
+	if(children == null) {
+	    mapChildren();
+	}
+	return children.size() == 0;
+    }
+    public void setModel(AttributeTableModel model) {
+	this.model = model;
+    }
+
+    public String toString()
+    {
+	if(text == null)
+	    {
+		if(userObject != null)
+		    {
+			if(userObject instanceof ElementWrapper)
+			    {
+				text = ((ElementWrapper)userObject).getName();
+			    }
+			else
+			    {
+				text = userObject.toString();
+			    }
+		    }
+		else
+		    {
+			text = "error";
+		    }
+	    }
+	return text;
+    }
+
+    public void mapChildren()
+    {
+	children = new Vector();
+
+	// How we build children depends on the node type
+	if (type == SET)
+	    {
+		// Add the elements as children
+		MetadataSet set = (MetadataSet) userObject;
+		ArrayList elements = set.getElements();
+		for (int i = 0; i < elements.size(); i++)
+		    {
+			children.add(new GEMSNode(ELEMENT, new ElementWrapper((Element) elements.get(i)), this));
+		    }
+	    }
+	if (type == ELEMENT) //Add subelements. As elements and subelements are identical, this process continues.
+	    {
+		ArrayList subelements = this.getElement().getSubelements();
+		for(int j = 0; j < subelements.size(); j++)
+		    {
+			children.add(new GEMSNode(ELEMENT, new ElementWrapper((Element)subelements.get(j)), this));
+		    }
+	    }
+    }
+    
+   class GemsNodeMouseAdapter
+    extends MouseAdapter {
+       
+      public void mouseClicked(MouseEvent e) {
+        JPopupMenu setPopup = new JPopupMenu("asdf");
+         if (e.isPopupTrigger()) {       
+	     setPopup.show(e.getComponent(),
+                           e.getX(), e.getY());
+	 }
+    }    
+    
+    public void mouseEntered(MouseEvent e) {
+    }    
+  
+    public void mouseExited(MouseEvent e) {
+    }
+    
+    public void mousePressed(MouseEvent e) {
+         JPopupMenu setPopup = new JPopupMenu("asdf");
+         if (e.isPopupTrigger()) {               
+	     setPopup.show(e.getComponent(),
+                           e.getX(), e.getY());
+	 }
+    }
+       
+    public void mouseReleased(MouseEvent e) {
+         JPopupMenu setPopup = new JPopupMenu("asdf");
+         if (e.isPopupTrigger()) {                  
+	     setPopup.show(e.getComponent(),
+                           e.getX(), e.getY());     
+	 }
+    }  
+   }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSPreferences.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSPreferences.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/GEMSPreferences.java	(revision 21112)
@@ -0,0 +1,406 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.border.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.cdm.LanguageManager;
+import org.greenstone.gatherer.gui.*;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.w3c.dom.*;
+
+public class GEMSPreferences extends ModalDialog {
+
+    static final public String CONNECTION_PREFS = "connection";
+    static final public String GENERAL_PREFS = "general";
+    static final private Dimension LABEL_SIZE = new Dimension(280, 25);
+    static final private Dimension SIZE = new Dimension(640, 345);
+
+    private JButton apply_button;
+    private JButton cancel_button;
+    private JButton ok_button;
+
+    private JComboBox language_combobox;
+    private JList language_limited_jlist;  
+    private JList language_code_jlist;
+    private JLabel interface_language_label;
+    private JLabel language_label;
+
+    private JTabbedPane tab_pane;
+    private GEMSPreferences self;
+
+    public GEMSPreferences() {
+	// Initialize
+	super((JFrame) null, true);
+	this.self = this;
+	setSize(SIZE);
+	Dictionary.registerText(this, "Preferences");
+	setJMenuBar(new SimpleMenuBar("preferences"));
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	tab_pane = new JTabbedPane();
+	JPanel general_preferences = createGeneralPreferences();
+	tab_pane.add("Preferences.General", general_preferences);
+        
+	Dictionary.register(tab_pane);
+
+	JPanel button_pane = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+    
+	Dictionary.registerBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	apply_button = new GLIButton();
+	apply_button.setMnemonic(KeyEvent.VK_A);
+	Dictionary.registerBoth(apply_button, "General.Apply", "General.Apply_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(new OKButtonListener(true));
+	apply_button.addActionListener(new OKButtonListener(false));
+	cancel_button.addActionListener(new CancelButtonListener());
+
+	// Layout
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,2,2,2));
+	button_pane.setLayout(new GridLayout(1,3,0,5));
+	button_pane.add(ok_button);
+	button_pane.add(apply_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(tab_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+        setLocation(400, 300);
+	tab_pane.setSelectedComponent(general_preferences);
+	
+	// Clean up
+	general_preferences = null;
+	cancel_button = null;
+	ok_button = null;
+	button_pane = null;
+	tab_pane = null;
+	content_pane = null;
+
+	setVisible(true);
+    }
+
+    private JPanel createGeneralPreferences() {
+	JPanel general_pane = new JPanel();
+
+	// Build the model of available languages
+	ArrayList dictionary_model = new ArrayList();
+
+	// The new method makes use of the successor to the languages.dat file, classes/xml/languages.xml
+	// Should update to also give the full name of the language --Matthew
+	NodeList language_elements = LanguageManager.LANGUAGES_DOCUMENT.getDocumentElement().getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
+	for(int i = 0; i < language_elements.getLength(); i++) {
+	    Element language_element = (Element) language_elements.item(i);
+	    if((language_element.getAttribute(StaticStrings.GLI_ATTRIBUTE)).equalsIgnoreCase(StaticStrings.TRUE_STR) || (language_element.getAttribute(StaticStrings.MDS_ATTRIBUTE)).equalsIgnoreCase(StaticStrings.TRUE_STR)) {
+		Locale locale = new Locale(language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE));
+		String description = language_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+		DictionaryEntry entry = new DictionaryEntry(description, locale);
+		if(!dictionary_model.contains(entry)) {
+		    dictionary_model.add(entry);
+		}
+		entry = null;
+		description = null;
+		locale = null;
+	    }
+	    language_element = null;
+	}
+	language_elements = null;
+
+	// Users email
+	JPanel lang_limited_pane = new JPanel();
+	interface_language_label = new JLabel();
+	interface_language_label.setPreferredSize(LABEL_SIZE);
+        
+	Dictionary.registerText(interface_language_label, "GEMS.Preferences.Selected_Languages_Tooltip");
+
+	// Language
+	JPanel language_pane = new JPanel();
+	language_label = new JLabel();
+	language_label.setPreferredSize(LABEL_SIZE);
+   
+	Dictionary.registerText(language_label, "Preferences.General.Interface_Language");
+	language_combobox = new JComboBox(dictionary_model.toArray());
+	Dictionary.registerTooltip(language_combobox, "Preferences.General.Interface_Language_Tooltip");
+	// Try to locate and select the current language
+	String language_code = Configuration.getLanguage();
+	for(int b = 0; b < language_combobox.getItemCount(); b++) {
+	    DictionaryEntry entry = (DictionaryEntry) language_combobox.getItemAt(b);
+	    if(language_code.equalsIgnoreCase(entry.getLocale().getLanguage())) {
+		language_combobox.setSelectedIndex(b);
+	    }
+	}
+        
+        //create language selected box, and high light the apprpropate ones
+	int [] selectedLanguages = new int[300];
+	int selectedLanguagesLength = 0;
+                 
+        GEMSLanguageManager gemsLangManager = new GEMSLanguageManager(GEMS.getGLIDirectoryPath() + "classes" + File.separator + "xml" + File.separator + "languages.xml");
+
+	//Create the visible JList, and then another one with just the language codes. 
+	language_code_jlist = new JList(gemsLangManager.getLanguageCodesToArray());  
+	language_limited_jlist = new JList(gemsLangManager.getLanguageCodesAndNames());
+
+	//Set the list up to display tabs properly
+  	TabListCellRenderer renderer = new TabListCellRenderer();
+	renderer.setTabs(new int[] {50, 200, 300});
+	language_limited_jlist.setCellRenderer(renderer);
+
+        String enabled_languages =  Configuration.getString("GEMS.Preferences.Selected_Languages", true);
+	//we have a string: enabled_languages that contains an array of strings delimited by comma's
+        String []enabled_languages_split = enabled_languages.split(",");
+        Object []language_codes_array = gemsLangManager.getLanguageCodesToArray();
+        
+        //for each enabled_languages_split, check which languages match, and store the index in selectedLanguages
+        for(int k = 0; k < enabled_languages_split.length; k++) {
+            
+            for(int p =0; p < language_codes_array.length; p++){
+                
+                //if the codes  match, then include the index p into the vector selectedLanguages
+                if (language_codes_array[p].toString().toLowerCase().trim().compareTo(enabled_languages_split[k].toLowerCase().trim()) == 0)
+		    {
+			selectedLanguages[selectedLanguagesLength] = p;
+			selectedLanguagesLength++;
+			language_limited_jlist.addSelectionInterval(p,p);
+		    }   
+            }   
+        }
+
+	// Layout
+        JScrollPane langScrollPane = new JScrollPane(language_limited_jlist);
+        langScrollPane.setPreferredSize(new Dimension(100, 100));
+	lang_limited_pane.setLayout(new GridLayout(1,3));
+	lang_limited_pane.add(interface_language_label);
+	lang_limited_pane.add(langScrollPane);
+        
+	language_pane.setLayout(new BorderLayout());
+	language_pane.add(language_label, BorderLayout.WEST);
+	language_pane.add(language_combobox, BorderLayout.CENTER);
+        
+	general_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	general_pane.setLayout(null);
+	general_pane.add(lang_limited_pane);
+	general_pane.add(language_pane);
+	//general_pane.add(view_extracted_metadata_checkbox);
+        Insets general_pane_insets = general_pane.getInsets();
+        
+        lang_limited_pane.setBounds(20+general_pane_insets.left, 20+general_pane_insets.top, 
+                           lang_limited_pane.getPreferredSize().width, lang_limited_pane.getPreferredSize().height);
+        
+        language_pane.setBounds(20+general_pane_insets.left, 130+general_pane_insets.top, 
+                            language_pane.getPreferredSize().width, language_pane.getPreferredSize().height);
+       
+	return general_pane;
+    }
+
+
+    private class OKButtonListener
+	implements ActionListener {
+	private boolean close;
+	public OKButtonListener(boolean close) {
+	    this.close = close;
+	}
+	public void actionPerformed(ActionEvent event) {
+	    // Submit the various changes
+
+            //save interface language
+	    String current_lang = Configuration.getLanguage();
+	    String new_lang = ((DictionaryEntry)language_combobox.getSelectedItem()).getLocale().getLanguage();
+	    if (!current_lang.equals(new_lang)) {
+		Configuration.setLocale("general.locale", Configuration.GENERAL_SETTING, ((DictionaryEntry)language_combobox.getSelectedItem()).getLocale());
+		JOptionPane.showMessageDialog(null, Dictionary.get("Preferences.General.Restart_Required"), Dictionary.get("General.Warning"), JOptionPane.WARNING_MESSAGE);
+	    }
+            
+	    //Set language_code_jlist to have the same selections as the language_limited_jlist.
+	    //(before, there was only one JList, and it only contained the language codes).
+	    int[] selection = language_limited_jlist.getSelectedIndices();
+	    language_code_jlist.setSelectedIndices(selection);
+	    Object[] selectedValues = language_code_jlist.getSelectedValues();
+	    String concatString = new String();
+	    
+	    for(int k = 0; k < selectedValues.length; k++) { 
+		if(selectedValues.length-k > 1) 
+		    concatString = concatString.concat(selectedValues[k].toString() +",");
+		else
+		    concatString = concatString.concat(selectedValues[k].toString());
+             }
+            
+            Configuration.setString("GEMS.Preferences.Selected_Languages",true, concatString);
+            
+	    // Always save configuration changes immediately (in case the GLI crashes)
+	    Configuration.save();
+
+	    // Hide dialog
+	    if(close) {
+		self.dispose();
+	    }
+	}
+    }
+
+    private class CancelButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    self.dispose();
+	}
+    }
+
+    private class DictionaryEntry
+	implements Comparable {
+	private Locale locale;
+	private String description;
+	public DictionaryEntry(Locale locale) {
+	    this.description = null;
+	    this.locale = locale;
+	}
+	public DictionaryEntry(String description, Locale locale) {
+	    this.description = description;
+	    this.locale = locale;
+	}
+	public int compareTo(Object object) {
+	    return toString().compareTo(object.toString());
+	}
+	public boolean equals(Object object) {
+	    return toString().equals(object.toString());
+	}
+	public Locale getLocale() {
+	    return locale;
+	}
+	public String toString() {
+	    if(description != null) {
+		return description;
+	    }
+	    else {
+		return locale.getDisplayName();
+	    }
+	}
+    }
+}
+
+
+/**
+   A custom list renderer, to display tabs in the JList properly.
+
+   @see http://manning.com/sbe/files/uts2/Chapter10html/Chapter10.htm
+   Added by Matthew Whyte
+   Date last modified: 1/02/05
+ */
+class TabListCellRenderer extends JLabel implements ListCellRenderer
+{
+    protected static Border m_noFocusBorder;
+    protected FontMetrics m_fm = null;
+    protected Insets m_insets = new Insets(0, 0, 0, 0);
+    
+    protected int m_defaultTab = 50;
+    protected int[] m_tabs = null;
+    
+    public TabListCellRenderer()
+    {
+	super();
+	m_noFocusBorder = new EmptyBorder(1, 1, 1, 1);
+	setOpaque(true);
+	setBorder(m_noFocusBorder);
+    }
+
+    public Component getListCellRendererComponent(JList list,
+						  Object value, int index, boolean isSelected, boolean cellHasFocus)     
+    {         
+	setText(value.toString());
+	
+	setBackground(isSelected ? list.getSelectionBackground() : list.getBackground());
+	setForeground(isSelected ? list.getSelectionForeground() : list.getForeground());
+	
+	setFont(list.getFont());
+	setBorder((cellHasFocus) ? UIManager.getBorder("List.focusCellHighlightBorder") : m_noFocusBorder);
+	
+	return this;
+    }
+    
+    public void setDefaultTab(int defaultTab) { m_defaultTab = defaultTab; }
+    
+    public int getDefaultTab() { return m_defaultTab; }
+
+    public void setTabs(int[] tabs) { m_tabs = tabs; }
+
+    public int[] getTabs() { return m_tabs; }
+
+    public int getTab(int index)
+    {
+	if (m_tabs == null)
+	    return m_defaultTab*index;
+		
+	int len = m_tabs.length;
+	if (index >= 0 && index < len)
+	    return m_tabs[index];
+
+	return m_tabs[len-1] + m_defaultTab*(index-len+1);
+    }
+
+
+    public void paint(Graphics g)
+    {
+	m_fm = g.getFontMetrics();
+	
+	g.setColor(getBackground());
+	g.fillRect(0, 0, getWidth(), getHeight());
+	getBorder().paintBorder(this, g, 0, 0, getWidth(), getHeight());
+
+	g.setColor(getForeground());
+	g.setFont(getFont());
+	m_insets = getInsets();
+	int x = m_insets.left;
+	int y = m_insets.top + m_fm.getAscent();
+
+	StringTokenizer	st = new StringTokenizer(getText(), "\t");
+	while (st.hasMoreTokens()) 
+	{
+	      String sNext = st.nextToken();
+	      g.drawString(sNext, x, y);
+		x += m_fm.stringWidth(sNext);
+
+		if (!st.hasMoreTokens())
+		    break;
+		int index = 0;
+		while (x >= getTab(index))
+		    index++;
+		x = getTab(index);
+	}
+    }   
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MSMUtils.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MSMUtils.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MSMUtils.java	(revision 21112)
@@ -0,0 +1,639 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.cdm.CommandTokenizer;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+/** This class contains a plethora of methods associated with handling the content of <strong>MetadataSet</strong>s and the <strong>Element</strong>s within. For example this is where you will find methods for comparing various parts of two <strong>MetadataSet</strong>s for equality. It also has methods for extracting common and useful data from <strong>Element</strong>s such as the AssignedValue nodes.
+ * @author John Thompson, Greenstone Digital Libraries
+ * @version 2.3
+ */
+public class MSMUtils {
+    static public int NONE = 0;     //An element of the enumeration of type filter.
+    static public int VALUES = 1;   //An element of the enumeration of type filter.
+    static public int ALIASES = 2;  //An element of the enumeration of type filter.
+    static public int BOTH = 3;     //An element of the enumeration of type filter.
+    static public char NS_SEP= '.'; //The character used to separate name space from metadata element.
+    static public char SF_SEP= '^'; //The character used to separate subfields from metadata element.
+
+
+    /** Method to add an attribute element to the given element. This method makes use of the language_dependent attribute of the document to not only determine if the attribute is language dependent, but also to see whether a Language element should be created if doesn't already exist.
+     * @param element_element the Element to add the attribute element to
+     * @param attribute_name_str the name of the new attribute to add as a String
+     * @param language_code_str the two letter code String of the language this attribute is to be added as
+     * @param value_str the String to be assigned as the attribute elements value
+     * @see org.greenstone.gatherer.msm.MSMUtils#isAttributeLanguageDependent
+     * @see org.greenstone.gatherer.msm.MSMUtils#setValue(Element, String)
+     * @see org.greenstone.gatherer.util.StaticStrings#ATTRIBUTE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#CODE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */ 
+    static public void addElementAttribute(Element element_element, String attribute_name_str, String language_code_str, String value_str) {
+	Document document = element_element.getOwnerDocument();
+	// Create the basic new attribute (everything except language attribute)
+	Element attribute_element = document.createElement(StaticStrings.ATTRIBUTE_ELEMENT);
+	attribute_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, attribute_name_str);
+	MSMUtils.setValue(attribute_element, value_str);
+
+	// Start off by determining if we have to add this node in the new multilingual optimized way
+	if(isAttributeLanguageDependent(document, attribute_name_str)) {
+	    boolean found = false;
+	    // Try to retrieve a language element for the given language code
+	    //NodeList language_elements = element_element.getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
+	    ArrayList language_elements = XMLTools.getChildElementsByTagName(element_element, StaticStrings.LANGUAGE_ELEMENT);
+	    for(int i = 0; i < language_elements.size(); i++) {
+		Element language_element = (Element) language_elements.get(i);
+		if(language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE).equals(language_code_str)) {
+		    found = true;
+		    // Add attribute
+		    language_element.appendChild(attribute_element);
+		}
+		language_element = null;
+	    }
+	    language_elements = null;
+	    // If it still hasn't been found, then add it
+	    if(!found) {
+		Element language_element = document.createElement(StaticStrings.LANGUAGE_ELEMENT);
+		language_element.setAttribute(StaticStrings.CODE_ATTRIBUTE, language_code_str);
+		element_element.appendChild(language_element);
+		// Add attribute
+		language_element.appendChild(attribute_element);
+		language_element = null;
+	    }
+	}
+	// Just add the attribute the old fashioned way
+	else {
+	    attribute_element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, language_code_str);
+	    element_element.appendChild(attribute_element);
+	}
+	// Clean up
+	attribute_element = null;
+	document = null;
+    }
+
+
+    /** Retrieve all of the attributes for the given element as a tree set. Note that this requires significant manipulation if the source is a multilingual optimized metadata set.
+     * @param element the Element whose attributes we wish to catalog
+     * @return a TreeSet of the attributes sorted by their natural ordering
+     * @see org.greenstone.gatherer.msm.MSMUtils#getValue(Node)
+     * @see org.greenstone.gatherer.util.StaticStrings#ATTRIBUTE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#CODE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */
+    static public TreeSet getAttributes(Element element) {
+	TreeSet attribute_tree = new TreeSet();
+
+        GEMSLanguageManager gemsLangManager = new GEMSLanguageManager(GEMS.getGLIDirectoryPath() + "classes" + File.separator + "xml" + File.separator + "languages.xml");
+        
+        String[] enabled_langs = gemsLangManager.getEnabledLanguageCodesToArray();
+                       
+	for(Node node = element.getFirstChild(); node != null; node = node.getNextSibling()) {
+	    if(node instanceof Element) {
+		Element some_element = (Element) node;
+		String some_element_name = some_element.getNodeName();
+		if(some_element_name.equals(StaticStrings.ATTRIBUTE_ELEMENT)) {
+		    attribute_tree.add(new Attribute(some_element.getAttribute(StaticStrings.NAME_ATTRIBUTE), some_element.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE), MSMUtils.getValue(some_element)));
+                    // System.out.println(some_element_name +  "<- some_element nodes' value");
+                }
+		else if(some_element_name.equals(StaticStrings.LANGUAGE_ELEMENT)) {
+		    String language_code = some_element.getAttribute(StaticStrings.CODE_ATTRIBUTE);
+		    NodeList attribute_elements = some_element.getElementsByTagName(StaticStrings.ATTRIBUTE_ELEMENT);
+                    
+		    for(int i = 0; i < attribute_elements.getLength(); i++) {
+			Element attribute_element = (Element) attribute_elements.item(i);
+                     
+                            for(int k = 0; k < enabled_langs.length; k++){
+                                
+                               if(enabled_langs[k].trim().compareTo(some_element.getAttributes().item(0).getNodeValue().toString()) == 0){
+				   attribute_tree.add(new Attribute(attribute_element.getAttribute(StaticStrings.NAME_ATTRIBUTE), language_code, MSMUtils.getValue(attribute_element)));
+				   attribute_element = null;
+				   
+				   break;
+                               }
+                            }
+			    attribute_element = null;
+		   } 
+		    attribute_elements = null;
+		    language_code = null;
+		}
+		some_element_name = null;
+		some_element = null;
+	    }
+	}
+	return attribute_tree;
+    }
+
+
+    /** Method to construct an elements description by retrieving the correct attribute.
+     * @param element the Element whose name we wish to retrieve
+     * @return a String which is the elements description, or an empty string if no description exists
+     * @see org.greenstone.gatherer.msm.MSMUtils#getElementAttribute
+     * @see org.greenstone.gatherer.util.StaticStrings#COMMENT_VALUE
+     * @see org.greenstone.gatherer.util.StaticStrings#DEFINITION_VALUE
+     * @see org.greenstone.gatherer.util.StaticStrings#EMPTY_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#SPACE_CHARACTER
+     */
+    static public String getDescription(Element element) {
+	String language_code_str = Configuration.getLanguage();
+	StringBuffer description = new StringBuffer(StaticStrings.EMPTY_STR);
+	description.append(getElementAttribute(element, StaticStrings.DEFINITION_VALUE, language_code_str));
+	if(description.length() > 0) {
+	    description.append(StaticStrings.SPACE_CHARACTER);
+	}
+	description.append(getElementAttribute(element, StaticStrings.COMMENT_VALUE, language_code_str));
+	language_code_str = null;
+	return description.toString();
+    }
+
+
+    /** Retrieve the value for the requested attribute in the required language. Once again this method must be aware of the differences between the old metadata sets and the new multilingual optimized ones.
+     * @param element_element the Element whose attributes we are searching through
+     * @param attribute_name_str the name of the desired attribute as a String
+     * @param language_code_str the two letter code String indicating the desired language
+     * @see org.greenstone.gatherer.msm.MSMUtils#getValue
+     * @see org.greenstone.gatherer.msm.MSMUtils#isAttributeLanguageDependent
+     * @see org.greenstone.gatherer.util.StaticStrings#ATTRIBUTE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#CODE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#EMPTY_STR
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */
+    static private String getElementAttribute(Element element_element, String attribute_name_str, String language_code_str) {
+	boolean found = false;
+	String result = StaticStrings.EMPTY_STR;
+	Document document = element_element.getOwnerDocument();
+	// Determine if the attribute is language specific
+	if(isAttributeLanguageDependent(document, attribute_name_str)) {
+	    NodeList language_elements = element_element.getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
+	    for(int i = 0; !found && i < language_elements.getLength(); i++) {
+		Element language_element = (Element) language_elements.item(i);
+		if(language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE).equals(language_code_str)) {
+		    NodeList attribute_elements = language_element.getElementsByTagName(StaticStrings.ATTRIBUTE_ELEMENT);
+		    for(int j = 0; !found && j < attribute_elements.getLength(); j++) {
+			Element attribute_element = (Element) attribute_elements.item(j);
+			if(attribute_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(attribute_name_str)) {
+			    found = true;
+			    result = MSMUtils.getValue(attribute_element);
+			}
+			attribute_element = null;
+		    }
+		    attribute_elements = null;
+		}
+		language_element = null;
+	    }
+	    language_elements = null;
+	}
+	else {
+	    boolean first_match = false;
+	    NodeList attribute_elements = element_element.getElementsByTagName(StaticStrings.ATTRIBUTE_ELEMENT);
+	    for(int k = 0; !found && k < attribute_elements.getLength(); k++) {
+		Element attribute_element = (Element) attribute_elements.item(k);
+		// We don't want to consider those attributes found inside language elements
+		if(attribute_element.getParentNode() == element_element) {
+		    ///ystem.err.println("First level");
+		    String target_name_str = attribute_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+		    String target_language_str = attribute_element.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE);
+		    ///ystem.err.println("Does " + target_name_str + " equal " + attribute_name_str + "?");
+		    if(attribute_name_str.equals(target_name_str)) {
+			///ystem.err.println("Does " + target_language_str + " equal " + language_code_str + "?");
+			if(language_code_str.equals(target_language_str)) {
+			    ///ystem.err.println("Perfect match!");
+			    found = true;
+			    result = MSMUtils.getValue(attribute_element);
+			}
+			else if((result == StaticStrings.EMPTY_STR || first_match) && isLegacyMDS(document)) {
+			    ///ystem.err.println("Legacy MDS");
+			    // Special case for old style documents, where the english match is good enough
+			    if(target_language_str.equals(StaticStrings.ENGLISH_LANGUAGE_STR)) {
+				///ystem.err.println("English plate.");
+				result = MSMUtils.getValue(attribute_element);
+			    }
+			    // Super special case where the first match is better than nothing
+			    else if(result == StaticStrings.EMPTY_STR && !first_match) {
+				///ystem.err.println("First match.");
+				first_match = true;
+				result = MSMUtils.getValue(attribute_element);
+			    }
+			}
+		    }
+		    target_language_str = null;
+		    target_name_str = null;
+		}
+		attribute_element = null;
+	    }
+	    attribute_elements = null;
+	}
+	document = null;
+	return result;
+    }
+
+
+
+    /** Method to construct an elements fully qualified name. Note that this is different from a nodes identifier. Think of name as a short, unique reference to a metadata element, whereas identifier can be much longer, language specific and non-unique.
+     * @param element An <strong>Element</strong> whose name we are interested in.
+     * @return A <strong>String</strong> representing this given elements fully namespace qualified name.
+     */
+    static final public String getFullName(Element element) {
+	return getFullName(element, "");
+
+    }
+
+
+    /** Method to construct an elements fully qualified name. Note that this is different from a nodes identifier. Think of name as a short, unique reference to a metadata element, whereas identifier can be much longer, language specific and non-unique.
+     * @param element An <strong>Element</strong> whose name we are interested in.
+     * @return A <strong>String</strong> representing this given elements fully namespace qualified name.
+     */
+    static final public String getFullName(Element element, String namespace) {
+	StringBuffer name_buffer = new StringBuffer();
+	if(element == null) {
+	    return "Error";
+	}
+	// First get the root node.
+	Document document = element.getOwnerDocument();
+	Element root = document.getDocumentElement();
+	document = null;
+	// Retrieve this elements name
+	name_buffer.append(element.getAttribute("name"));
+
+	// Now we check if element has a parent node, other than root. If so we begin building up the full name
+	//For some reason this doesn't work in specific cases of subelements.
+	Element parent_element = (Element) element.getParentNode();
+	while(parent_element != null && parent_element != root) { 
+	    name_buffer.insert(0, SF_SEP);
+	    name_buffer.insert(0, parent_element.getAttribute("name"));
+	    parent_element = (Element)parent_element.getParentNode();
+	}
+	parent_element = null;
+	// Finally insert the namespace and we are all done.
+	if(root != null) {
+	    namespace = root.getAttribute("namespace");
+	} 
+	root = null;
+	// If no root, or no namespace found, assume its extracted (at least then they can't edit it)
+	if(namespace == null || namespace.equals("")) {
+	    namespace = MetadataSetManager.EXTRACTED_METADATA_NAMESPACE;
+	}
+	name_buffer.insert(0, NS_SEP);
+	name_buffer.insert(0, namespace);
+	namespace = null;
+	return name_buffer.toString();
+    } // static public String getFullName(Element element)
+
+
+    /**
+       Method to get an element's name, not including its namespace.
+       @param element an <strong>Element</strong> whose name we are interested in.
+       @return A <strong>Strong</strong> representing the given elements name
+       @author Matthew Whyte
+       @date last modified: 21/01/04
+     */
+    static final public String getNameOnly(Element element)
+    {
+	StringBuffer name_buffer = new StringBuffer();
+	if(element == null) {
+	    return "Error";
+	}
+	// Retrieve this elements name
+	name_buffer.append(element.getAttribute("name"));
+
+	return name_buffer.toString();
+    }
+
+
+    /** Method to construct an elements name (sic identifier) by retrieving the correct attribute, language specific.
+     * @param element the Element whose name we wish to retrieve
+     * @return a String which is the elements identifier, or an empty string if no identifier exists
+     * @see org.greenstone.gatherer.msm.MSMUtils#getElementAttribute
+     * @see org.greenstone.gatherer.util.StaticStrings#IDENTIFIER_VALUE
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */
+    static final public String getIdentifier(Element element) {
+	String identifier = getElementAttribute(element, StaticStrings.IDENTIFIER_VALUE, Configuration.getLanguage());
+	// Failing the above we return the nodes name instead.
+	if(identifier == null || identifier.length() == 0) {
+	    identifier = element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+	}
+
+	return identifier;
+    }
+
+
+    /** Method to retrieve from the node given, a certain child node with the specified name.
+     * @param parent The <strong>Node</strong> whose children should be searched.
+     * @param name The required nodes name as a <strong>String</strong>.
+     * @return The requested <strong>Node</strong> if it is found, <i>null</i> otherwise.
+     */ 
+    static final public Node getNodeFromNamed(Node parent, String name) {
+	Node child = null;
+	for(Node i = parent.getFirstChild(); i != null && child == null; 
+	    i = i.getNextSibling()) {
+	    if(i.getNodeName().equals(name)) {
+		child = i;
+	    }
+	}
+	return child;
+    }
+
+
+    /** Look for the occurances 'field' of the element and return it if found.
+     * @return An <i>int</i> which matches the number in the occurances attribute of the element, or 0 if no such attribute.
+     */
+    static final public int getOccurances(Element element) {
+	int count = 0;
+	String number = null;
+	if((number = element.getAttribute("occurances")) != null) {
+	    try {
+		count = Integer.parseInt(number);
+	    }
+	    catch(Exception error) {
+		count = 0;
+	    }
+	}
+	return count;
+    }	 
+
+
+    /** Method to retrieve the value of a given node (not the assigned values tree!).
+     * @param element The <strong>Element</strong> whose value we wish to find.
+     * @return The value found as a <strong>String</strong>, or <i>null</i> if this element has no value.
+     */
+    static final public String getValue(Node element) {
+	// If we've been given a subject node first retrieve its value node.
+	if(element.getNodeName().equals("Subject")) {
+	    element = getNodeFromNamed(element, "Value");
+	}
+	// If we've got a value node, then reconstruct the text. Remember that DOM will split text over 256 characters into several text nodes
+	if(element != null && element.hasChildNodes()) {
+	   StringBuffer text_buffer = new StringBuffer();
+	   NodeList text_nodes = element.getChildNodes();
+	   for(int i = 0; i < text_nodes.getLength(); i++) {
+	      Node possible_text = text_nodes.item(i);
+	      if(possible_text.getNodeName().equals(StaticStrings.TEXT_NODE)) {
+		 text_buffer.append(possible_text.getNodeValue());
+	      }
+	   }
+	   return text_buffer.toString();
+	}
+	return "";
+    }
+
+
+    /** Determine if the named attribute is language specific for this collection. This information is found in a DOM attribute of the document element, as a comma separated list of attribute names.
+     * @param document the Document for which we wish to check the language requirements
+     * @param attribute_name_str the name of the attribute we a testing as a String
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGEDEPENDENT_ATTRIBUTE
+     * Modified by Matthew Whyte 4/02/05
+     */
+    static private boolean isAttributeLanguageDependent(Document document, String attribute_name_str)
+    {
+	//gives a list of comma seperated language specific attributes.
+	String language_specific_attributes = document.getDocumentElement().getAttribute(StaticStrings.LANGUAGEDEPENDENT_ATTRIBUTE).toLowerCase();
+	//System.err.println("lang_specific: " + language_specific_attributes); //debug
+
+	//Seperate out language_specific_attributes
+	String[] attributes_split = language_specific_attributes.split(",");
+
+	//Compare each to attribute_name_str
+	for(int i = 0; i < attributes_split.length; i++)
+	    {
+		if(attributes_split[i].trim().compareTo(attribute_name_str.toLowerCase()) == 0)
+		    return true;
+	    }
+	return false;
+    }
+
+
+    /** A method to remove a specific attribute element from an element. This attribute must match in name, language and in value before being removed. Note that this method supports both legacy and multilingual optimized versions of the mds.
+     * @param element_element the Element which represent the metadata element we are altering
+     * @param attribute_name_str the name of the attribute to remove as a String
+     * @param language_code_str the language code we must match as a String
+     * @param value_str the value String which also must match before we remove anything
+     * @return true if the desired attribute was successfully found and removed, false otherwise
+     * @see org.greenstone.gatherer.msm.MSMUtils#isAttributeLanguageDependent
+     * @see org.greenstone.gatherer.msm.MSMUtils#getValue(Node)
+     * @see org.greenstone.gatherer.util.StaticStrings#ATTRIBUTE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#CODE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGE_ELEMENT
+     * @see org.greenstone.gatherer.util.StaticStrings#NAME_ATTRIBUTE
+     */
+    static public boolean removeElementAttribute(Element element_element, String attribute_name_str, String language_code_str, String value_str) {
+	//Determine the if this is one of the language specific attributes.
+	//This can sometimes give a false result, if an attribute is made language specific half way through creation of a metadata set (or vice versa).
+	if(isAttributeLanguageDependent(element_element.getOwnerDocument(), attribute_name_str))
+	    {
+		/*
+		Multilingual Optimized version, eg:
+		<Language code="en">
+		    <Attribute name="identifier">second</Attribute>
+		</Language>
+		*/
+		return removeElementAttributeNew(element_element, attribute_name_str, language_code_str, value_str, false);
+	    }
+	else
+	    {
+		/*
+		The old way, eg:
+		<Attribute language="en" name="identifier">second</Attribute>
+		*/
+		return removeElementAttributeOld(element_element, attribute_name_str, language_code_str, value_str, false);
+	    }
+	
+    }
+    static final private boolean removeElementAttributeNew(Element element_element, String attribute_name_str, String language_code_str, String value_str, boolean run_once)
+    {
+	// Retrieve the language elements, and determine the correct one
+	NodeList language_elements = element_element.getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
+	for(int i = 0; i < language_elements.getLength(); i++) {
+	    Element language_element = (Element) language_elements.item(i);
+	    if(language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE).equalsIgnoreCase(language_code_str)) {
+		NodeList attribute_elements = language_element.getElementsByTagName(StaticStrings.ATTRIBUTE_ELEMENT);
+		for(int j = 0; j < attribute_elements.getLength(); j++) {
+		    Element attribute_element = (Element) attribute_elements.item(j);
+		    String target_name_str = attribute_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+		    String target_value_str = MSMUtils.getValue(attribute_element);
+		    if(attribute_name_str.equals(target_name_str) && value_str.equals(target_value_str)) {
+			language_element.removeChild(attribute_element);
+			if(attribute_elements.getLength() == 0) {
+			    element_element.removeChild(language_element);
+			}
+			target_value_str = null;
+			target_name_str = null;
+			attribute_element = null;
+			attribute_elements = null;
+			language_element = null;
+			language_elements = null;
+			return true;
+		    }
+		    target_value_str = null;
+		    target_name_str = null;
+		    attribute_element = null;			
+		}
+		attribute_elements = null;
+	    }
+	    language_element = null;
+	}
+	language_elements = null;
+	// Not found. Try the old way before failing.
+	if(!run_once)
+	    return removeElementAttributeOld(element_element, attribute_name_str, language_code_str, value_str, true);
+	return false;
+    }
+    static final private boolean removeElementAttributeOld(Element element_element, String attribute_name_str, String language_code_str, String value_str, boolean run_once)
+    {
+	// Find the attribute to remove
+	NodeList attribute_elements = element_element.getElementsByTagName(StaticStrings.ATTRIBUTE_ELEMENT);
+	for (int k = 0; k < attribute_elements.getLength(); k++) {
+	    Element attribute_element = (Element) attribute_elements.item(k);
+	    // Remember to ignore any attributes that live within nested language elements
+	    if (attribute_element.getParentNode() == element_element && attribute_element.getAttribute(StaticStrings.NAME_ATTRIBUTE).equals(attribute_name_str) && attribute_element.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE).equalsIgnoreCase(language_code_str) && MSMUtils.getValue(attribute_element).equals(value_str)) {
+		// Match found, so remove the attribute node and return
+		element_element.removeChild(attribute_element);
+		attribute_element = null;
+		attribute_elements = null;
+		return true;
+	    }
+	    attribute_element = null;
+	}
+	attribute_elements = null;
+	// No match found. Try the new way before failing.
+	if(!run_once)
+	    return removeElementAttributeNew(element_element, attribute_name_str, language_code_str, value_str, true);
+	return false;
+    }
+
+
+    /** can only be used to set the english value */
+    static final public void setIdentifier(Node element, String value) {
+	// Get the 'identifier' Element
+	for(Node node = element.getFirstChild(); node != null;
+	    node = node.getNextSibling()) {
+	    if(node.getNodeName().equals("Attribute")) {
+		Element target = (Element)node;
+		if(target.getAttribute("name").equals("identifier") && target.getAttribute("language").equals("en")) {
+		    Node text = target.getFirstChild();
+		    text.setNodeValue(value);
+		    break;
+		}
+	    }
+	}
+    }
+
+
+    /** Set the value of the element attribute occurances.
+     * @param element The <strong>Element</strong> to change.
+     * @param value The value to change by as an <i>int</i>.
+     */
+    static final public void setOccurance(Element element, int value) {
+	Integer new_value = new Integer(getOccurances(element) + value);
+	element.setAttribute("occurances", new_value.toString());
+    }
+
+
+    /** Set the #text node value of some element.
+     * @param element the Element whose value we wish to set
+     * @param value the new value for the element as a String
+     */
+    static final public void setValue(Element element, String value) {
+	// Remove any existing child node(s)
+	clear(element);
+	// Add new text node.
+	if (value != null) {
+	    element.appendChild(element.getOwnerDocument().createTextNode(value));
+	}
+    }
+
+
+    /** This method also traverses the tree, but this one is used to gather all the values and aliases at once, and to 'prune' the tree if necessary.
+     * @param current The current root <strong>Node</strong> of this AssignedValues tree subtree.
+     * @param return_filter This <i>int</i> specifies what nodes from the tree should be returned, where;<br>VALUES  = return values only<br>ALIASES = return aliases only<br>BOTH    = return both values and aliases<br>NONE    = return nothing.
+     * @param remove_leaves A leaf node is a subject that contains no child subjects. If this <i>boolean</i> is set to <i>true</i> then the leaf nodes will be removed from the tree. 
+     * @return A <strong>Node[]</strong> containing whatever values or aliases have been found during the trees traversal.
+     */
+    static final public Node[] traverseTree(Node current, int return_filter, boolean remove_leaves) {
+	Node leaves[] = null;
+	String name = current.getNodeName();
+	if(name.equals("Value") && (return_filter == VALUES || return_filter == BOTH)) {
+	    leaves = ArrayTools.add(leaves, current);
+	}
+	else if(name.equals("Alias") && (return_filter == ALIASES || return_filter == BOTH)) {
+	    leaves = ArrayTools.add(leaves, current);
+	}
+	else if(name.equals("Subject")) {
+	    boolean has_subject_child = false;
+	    Node children[] = ArrayTools.nodeListToNodeArray(current.getChildNodes());
+	    for(int i = 0; i < children.length; i++) {
+		if(children[i].getNodeName().equals("Subject")) {
+		    has_subject_child = true;
+		}
+		leaves = ArrayTools.add(leaves, traverseTree(children[i], return_filter, remove_leaves));
+	    }
+	    if(!has_subject_child && remove_leaves) {
+		Node parent = current.getParentNode();
+		parent.removeChild(current);
+	    }
+	}
+	else if(name.equals("AssignedValues")) {
+	    Node children[] = ArrayTools.nodeListToNodeArray(current.getChildNodes());
+	    for(int i = 0; i < children.length; i++) {
+		leaves = ArrayTools.add(leaves, traverseTree(children[i], return_filter, remove_leaves));
+	    }
+	}															  
+	return leaves;
+    }
+
+
+    /** Determine if the given document is a legacy MDS or a new multilingual one. The easiest way to tell is whether there is a language_dependant attribute in the document element.
+     * @param document the Document to test
+     * @return true if this is an old mds, false otherwise
+     * @see org.greenstone.gatherer.util.StaticStrings#LANGUAGEDEPENDANT_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#EMPTY_STR
+     */
+    static private boolean isLegacyMDS(Document document) {
+	///ystem.err.println("isLegacyMDS(): l_d = " + document.getDocumentElement().getAttribute(StaticStrings.LANGUAGEDEPENDANT_ATTRIBUTE));
+	return (document.getDocumentElement().getAttribute(StaticStrings.LANGUAGEDEPENDENT_ATTRIBUTE)).equals(StaticStrings.EMPTY_STR);
+    }
+
+
+    /** Remove all of the child nodes from a certain node. */
+    static final public void clear(Node parent) {
+	while(parent.hasChildNodes()) {
+	    parent.removeChild(parent.getFirstChild());
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSet.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSet.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSet.java	(revision 21112)
@@ -0,0 +1,549 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import org.apache.xerces.dom.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+/** An semi-data class to hold details about a loaded metadata set, it also provides some methods to manipulating the data within. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3b
+ */
+public class MetadataSet {
+    /** The <Strong>Document</strong> of the DOM model. */
+    private Document document = null;
+    /** The document <strong>Element</strong> of the DOM model. */
+    private Element  root     = null;
+    /** The <strong>File</strong> this metadata set was loaded from. */
+    public String old_filename = null;
+    private File     file     = null;
+    /** The list of metadata elements which are, of course, children of the root node. */
+    private ArrayList elements = null;
+    
+    // the nmae of the collection if this set is collection-specific
+    private String collection = null;
+    //set_changed is check in msm(metadatasetmanager).save() and if this is true then we save
+    //the collection, otherwise we leave it as is
+    private boolean setChanged = false;
+    private String current_language_code;
+    /** The description of this metadata set. Cached as it takes more computation time. */
+    private String description = null;
+    /** The name of this metadata set. Cached as it takes more computation time. */
+    private String name = null;
+    /** An element of the tree pruning filter enumeration, that indicates all nodes in the tree should be retained. */
+    static final int ALL_VALUES    = 1;
+    /** An element of the tree pruning filter enumeration, that indicates only metadata Subject nodes or higher should remain after pruning. */
+    static final int SUBJECTS_ONLY = 2;
+    /** An element of the tree pruning filter enumeration, that indicates no value nodes i.e. the entire AssignedValues subtree, should remain after pruning. */
+    static final int NO_VALUES     = 3;
+
+    public MetadataSet(String metadata_template) {
+	
+	this.file = new File(metadata_template);
+	this.document = XMLTools.parseXMLFile(metadata_template, true);
+
+	init(true); // use class loader
+    }
+
+    /** Constructor.
+     * @param file The file the metadata set should be loaded from. 
+     */
+    public MetadataSet(File file) {
+	this.file = file;
+	this.document = XMLTools.parseXMLFile(file);
+
+	init(false); // don't use class loader
+    }
+
+    /** Copy constructor. 
+     * @param original The original metadata set to copy from.
+     */
+    public MetadataSet(MetadataSet original) {
+	// We have to create a new document.
+	document = new DocumentImpl(original.getDocument().getDoctype());
+	root = (Element) document.importNode(original.getDocument().getDocumentElement(), true);
+	document.appendChild(root);
+	elements = XMLTools.getChildElementsByTagName(root, "Element");
+	//elements = root.getElementsByTagName("Element");
+	file = original.getFile();
+    }
+  
+    public boolean getSetChanged() {
+        
+        return setChanged;    
+    }
+    public void setSetChanged(boolean val) {
+         
+        setChanged = val;
+    }
+
+
+    public void setCollection(String coll) {
+	collection = coll;
+
+    }
+    /** Add a mds level attribute.
+     * @param name The name of the attribute to add as a <Strong>String</strong>.
+     * @param value The value as a <strong>String</strong>.
+     */
+    public void addAttribute(String name, String value) {
+	root.setAttribute(name, value);
+    }
+
+
+    /** Add a new default metadata element with the given name to this metadata set.
+	This method also handles the addtion of 'subelements'
+     * @param name The name of this element as a <strong>String</strong>.
+     * @param parent The parent element as an <strong>GEMSNode</strong>. If null, the parent is the metadataset root.
+     * @return An <strong>ElementWrapper</strong> around the newly created element or null if the element was not created.
+     */
+    public ElementWrapper addElement(String name, String language, GEMSNode parent)
+    {
+	Element parent_element = null;
+
+	//The element that we are creating
+	Element element = document.createElementNS("","Element");
+	element.setAttribute("name",name);
+
+	//Add identifier attribute. This handles if we use the new or old language identification
+	MSMUtils.addElementAttribute(element, "identifier", language, name);
+	
+	if(parent == null) //A element
+	    {
+		parent_element = root;
+		elements.add(element); //Add the new element to the list of 'em
+	    }
+	else //A subelement!
+	    {
+		//Add the new subelement to the list of 'em
+		parent.getElement().addSubelementToList(element);
+		parent_element = parent.getElement().getElement(); //get the parent as an Element
+	    }
+
+	parent_element.appendChild(element);
+
+	return new ElementWrapper(element);
+    }
+
+
+    public int compare(Element e1, Element e2) {
+	int result = 0;
+	// Check that they're not the same element.
+	if(e1 != e2) {
+	    int index_e1 = -1;
+	    int index_e2 = -1;
+	    // Locate the indexes for each element.
+	    for(int i = 0; i < elements.size(); i++) {
+		Node element = (Node)elements.get(i);
+		if(element == e1) {
+		    index_e1 = i;
+		}
+		if(element == e2) {
+		    index_e2 = i;
+		}
+	    }
+	    if(index_e1 < index_e2) {
+		result = -1;
+	    }
+	    else {
+		result = 1;
+	    }
+	}
+	return result;
+    }
+
+    /** A method to determine if this metadata set contains an element with a certain name (case sensitive).
+     * @param name A <strong>String</strong> which is the name of the element whose presence we are checking.
+     * @return A <i>boolean</i> which is <i>true</i> if the named element exists, <i>false</i> otherwise.
+     */
+    public boolean containsElement(String name) {
+	for(int i = 0; i < elements.size(); i++) {
+	    Element sibling = (Element) elements.get(i);
+	    String sibling_name = sibling.getAttribute("name");
+	    if(sibling_name.equals(name)) {
+		return true;
+	    }
+	}
+	return false;
+    }
+
+    public NamedNodeMap getAttributes() {
+	return root.getAttributes();
+    }
+
+    /** Method to retrieve the contact address of the metadata set creator.
+     * @return A <strong>String</strong> containing the address. 
+     */
+    /* private String getContact() {
+	return root.getAttribute("contact");
+	} */
+    /** Method to retrieve the name of the creator of this metadata set. 
+     * @return A <strong>String</strong> containing the name.
+     */
+    public String getCreator() {
+	return root.getAttribute("creator");
+    }
+    /** Method to retrieve the description of this metadata set. Note that this is language specific, so we determine the desired language from the Dictionary. If no such entry exists, first try returning the english version and failing that the first description found.
+     * @return The description as a <strong>String</strong>.
+     */
+    public String getDescription() {
+	if(current_language_code != null && !Configuration.getLanguage().equals(current_language_code)) {
+	    description = null;
+	}
+	if(description == null) {
+	    description = getAttribute(StaticStrings.DESCRIPTION_ELEMENT, Dictionary.get("GEMS.No_Description"));
+	}
+	return description;
+    }
+
+    /** Method to retrieve the <strong>Document</strong> associated with this metadata set.
+     * @return The <strong>Document</strong> representing this metadata set.
+     */
+    public Document getDocument() {
+	return document;
+    }
+    /** Method to retrieve the metadata element indicated by an index.
+     * @param index An <i>int</i> specifying the required element.
+     * @return The <strong>Element</strong> at the index.
+     */
+    public Element getElement(int index) {
+	return (Element)elements.get(index);
+    }
+    /** This method is used to acquire a reference to the element which matches the given metadata. Note that this is not the same as <i>metadata.getElement()</i> as the reference returned by it may now be obsolete.
+     * @param metadata A <strong>Metadata</strong> object representing an element and value assignment.
+     * @return A 'live' reference to an <strong>Element</strong> which is the same as that referenced by the given metadata, or <i>null</i> if there is no such element.
+     */
+//     public Element getElement(Metadata metadata) {
+// 	return metadata.getElement().getElement();
+//     }
+    /** This method is used to acquire a reference to the element which has the name specified. Note that this is not the same as <i>metadata.getElement()</i> as the reference returned by it may now be obsolete.
+     * @param name A <strong>String</strong> stating the desired objects name.
+     * @return A 'live' reference to an <strong>Element</strong> which is the same as that referenced by the given metadata, or <i>null</i> if there is no such element.
+     */
+    public Element getElement(String name) {
+	// Strip any namespace.
+	while(name.indexOf(".") != -1 && !name.equals(".")) {
+	    name = name.substring(name.indexOf(".") + 1);
+	}
+	///ystem.err.println("Get element named " + name);
+	for(int i = 0; i < elements.size(); i++) {
+	    Element element = (Element) elements.get(i);
+	    ///ystem.err.println("Compare to: " + element.getAttribute("name"));
+	    if(element.getAttribute("name").equals(name)) {
+		return element;
+	    }
+	}
+	return null;
+    }
+
+    public Element getElement(Element parent_element, String name) {
+	DebugStream.println("Get element named " + name + " from " + parent_element.getAttribute("name"));
+	ArrayList elements = XMLTools.getChildElementsByTagName(parent_element, "Element");
+	for(int i = 0; i < elements.size(); i++) {
+	    Element element = (Element) elements.get(i);
+	    if(element.getAttribute("name").equals(name) && element.getParentNode() == parent_element) {
+		elements = null;
+		return element;
+	    }
+	}
+	elements = null;
+	return null;
+    }
+    /** Method to acquire a list of all the elements in this metadata set. 
+     * @return A <strong>NodeList</strong> containing all of this sets elements.
+     */
+    public ArrayList getElements() {
+	return elements;
+    }
+
+    /** Method to retrieve a list of all the elements in this metadata set, sorted.
+     * @return A <strong>Vector</strong> containing all of the elements of this sets, sorted.
+     */
+//     public Vector getElementsSorted() {
+// 	Vector elements_list = new Vector();
+// 	for (int i = 0; i < elements.size(); i++) {
+// 	    elements_list.add(new ElementWrapper((Element) elements.get(i)));
+// 	}
+// 	Collections.sort(elements_list, MSMUtils.METADATA_COMPARATOR);
+// 	return elements_list;
+//     }
+
+    /** Method to retrieve the original file this metadata set was created from.
+     * @return A <strong>File</strong>.
+     */
+    public File getFile() {
+	return file;
+    }
+    /** Get the last changed attribute.
+     * @return Last changed as a <strong>String</strong>.
+     */
+    public String getLastChanged() {
+	return root.getAttribute("lastchanged");
+    }
+
+    /** Method to get this metadata sets name. Note that this is language specific, so we determine the desired language from the Dictionary.
+	If no such entry exists, first try returning the english version and failing that the first name found.
+     * @return A <strong>String</strong> which contains its name.
+     */
+    public String getName() {
+	if(current_language_code != null && !Configuration.getLanguage().equals(current_language_code)) {
+	   // name = null;
+            return null;
+	}
+	if(name == null) {
+            
+	    //name = getAttribute(StaticStrings.NAME_ELEMENT, Dictionary.get("MSM.No_Name"));
+	   return getAttribute(StaticStrings.NAME_ELEMENT, Dictionary.get("GEMS.No_Name"));
+	
+        }
+     
+	return name;
+    }
+    /** Method to retrieve this metadata sets namespace.
+     * @return The namespace as a <strong>String</strong>.
+     */
+    public String getNamespace() {
+	return root.getAttribute("namespace");
+    }
+
+    /**
+       Method to change this metadata set's namespace
+       @param: The new namespace as a <strong>String</strong>
+     */
+    public void setNamespace(String new_namespace)
+    {
+	root.setAttribute("namespace", new_namespace);
+    }
+
+    /** Method to retrieve the root element, i.e. the Document Element, of the DOM model behind this metadata set.
+     * @return An <strong>Element</strong> which is at the root of the modal.
+     */ 
+    public Element getRoot() {
+	return root;
+    }
+
+    /** Remove a mds level attribute.
+     * @param name The name of the attribute to remove.
+     */
+    public void removeAttribute(String name) {
+	root.removeAttribute(name);
+    }
+
+    /** Method to remove the given element from this metadata set.
+     * @param element The <strong>Element</strong> to be removed.
+     * @param parent The parent element (wrapped, so can access its subelement list), as an <strong>ElementWrapper</strong>. If null, the parent is root.
+     */
+    public void removeElement(Element element, ElementWrapper parent) {
+	try {
+	    ElementWrapper element_wrapped = new ElementWrapper(element);
+
+	    if(parent == null)
+		{
+		    elements.remove(element);
+		    root.removeChild(element);
+		}
+	    else
+		{
+		    parent.removeSubelementFromList(element);
+		    parent.getElement().removeChild(element);
+		}
+	    
+	} catch(Exception DOMException) { //The element does not exist!
+	    System.err.println("Exception: '"+ DOMException + "' occured. Cannot remove element because it does not exist.");
+	}
+    }
+
+ /**
+       Rename the metadata element.
+       @param oldElement An <strong>Element</strong> we wish to rename
+       @param newName A <strong>String</strong> which contains the new name of the element.
+       @return An <strong>ElementWrapper</strong> around the newly named element
+    */
+    public void renameElement(Element oldElement, String newName)
+    {
+	oldElement.setAttribute("name", newName); //Rename the element
+    }
+	
+
+    /** Set one of the mds level attributes.
+     * @param name The attribute to change.
+     * @param value its new value.
+     */
+    public void setAttribute(String name, String value) {
+	root.setAttribute(name, value);
+    }
+    /** Once the metadata set has been saved to a different location, this is used to update the file parameter.
+     * @param file The new location of this metadata set <strong>File</strong>.
+     */
+    public void setFile(File file) {
+	this.file = file;
+    }
+
+    public void setName(String name) {
+	// Retrieve the name element. We look for the first english one.
+	boolean found_it = false;
+	Element name_element = null;
+	Element metadataset_element = document.getDocumentElement();
+	NodeList name_elements = metadataset_element.getElementsByTagName(StaticStrings.NAME_ELEMENT);
+	for(int i = 0; (i < name_elements.getLength() && !found_it); i++) {
+	    Element possible_name_element = (Element) name_elements.item(i);
+	    if(possible_name_element.getAttribute(StaticStrings.LANGUAGE_ATTRIBUTE).equals("en")) {
+		// Found it.
+		found_it = true;
+		name_element = possible_name_element;
+		//System.out.println("Found english name: " + name_element); //debug
+	    }
+	}
+	// If there is none add one. Note that we can only add english metadata sets. Although others can edit them to add further names as necessary.
+	if(name_element == null) {
+	    name_element = document.createElement(StaticStrings.NAME_ELEMENT);
+	    name_element.setAttribute(StaticStrings.LANGUAGE_ATTRIBUTE, "en");
+	    metadataset_element.insertBefore(name_element, metadataset_element.getFirstChild());
+	}
+	// Replace the text node
+	while(name_element.hasChildNodes()) {
+	    name_element.removeChild(name_element.getFirstChild());
+	}
+	name_element.appendChild(document.createTextNode(name));
+    }
+
+    /** Method to determine the number of elements in this set.
+     * @return An <i>int</i> specifying the element count.
+     */
+    public int size() {
+	return elements.size();
+    }
+    /** Method to translate this class into a meaningful string, which in this case is the metadata sets name.
+     * @return The metadata sets name as a <strong>String</strong>.
+     */
+    public String toString() {
+       
+	String name = getName();
+	// If there is no given name, then use the namespace as there is garaunteed to be one of them.
+	if(name == null || name.length() == 0) {
+	    name = root.getAttribute("namespace");
+	}
+	// Append namespace
+	String namespace = root.getAttribute("namespace");
+	if(namespace == null || namespace.equals("")) {
+	    namespace = MetadataSetManager.EXTRACTED_METADATA_NAMESPACE;
+	}
+	name = name + " (" + namespace + ")";
+	if (collection == null) 
+	    return name;
+	return "["+collection+"] "+name;
+    }
+
+    /** This method retrieves the required attribute from the Metadata Set, typically it's name or it's description. Note that this method is language dependent, and moreover supports both legacy metadata sets and the new sets optimized for multiple languages.
+     * @param element_name the name of the type of element the required information is in as a String
+     * @param default_string the value to return if no such element is found also as a String
+     * @see org.greenstone.gatherer.Configuration#getLanguage()
+     * @see org.greenstone.gatherer.msm.MSMUtils#getValue(Node)
+     * @see org.greenstone.gatherer.util.StaticStrings#CODE_ATTRIBUTE
+     * @see org.greenstone.gatherer.util.StaticStrings#SETLANGUAGE_ELEMENT
+     */
+    private String getAttribute(String element_name, String default_string) {
+	String result = null;
+
+	// Determine the language code.
+	current_language_code = Configuration.getLanguage();
+
+	///ystem.err.println("Searching for the " + element_name + " in " + current_language_code);
+
+	// New Metadata Set Format makes use of deferred-node-expansion to save memory - rather than create nodes for a name and description in each language, nodes which have potentially huge strings, we instead create simplier SETLANGUAGE nodes, and then only expand the one in the desired language. Of course if a user happens to change to every available language slightly more memory will be used than in the old method. For instance consider the DLS with 25 languages, each with a name node of 50 bytes and an descriptions of 500. Thus old style > 13750 bytes while new style < 600.
+	NodeList set_language_elements = document.getElementsByTagName(StaticStrings.SETLANGUAGE_ELEMENT);
+	for(int b = 0; b < set_language_elements.getLength(); b++) {
+	    Element set_language_element = (Element) set_language_elements.item(b);
+	    String code = set_language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE).toLowerCase();
+	    if(code.equals(current_language_code)) {
+		NodeList specific_elements = set_language_element.getElementsByTagName(element_name);
+		if(specific_elements.getLength() > 0) {
+		    Element specific_element = (Element) specific_elements.item(0);
+		    result = MSMUtils.getValue(specific_element);
+		    specific_element = null;
+		}
+		specific_elements = null;
+	    }
+	    code = null;
+	    set_language_element = null;
+	}
+	set_language_elements = null;
+	// And we may be all done
+	if(result != null) {
+	    return result;
+	}
+
+	// Failing that we move on to an older style search - start by recovering all Name elements
+	NodeList possible_elements = document.getElementsByTagName(element_name);
+	// Iterate through the available names looking for the appropriate one. Also make note of the first name, then overwrite it with any english one.
+	boolean found = false;
+	for(int i = 0; !found && i < possible_elements.getLength(); i++) {
+	    Element possible_element = (Element) possible_elements.item(i);
+	    String possible_element_code = possible_element.getAttribute("language").toLowerCase();
+	    if(possible_element_code.equals(current_language_code) || name == null) {
+		result = MSMUtils.getValue(possible_element);
+		found = true;
+	    }
+	    possible_element_code = null;
+	    possible_element = null;
+	}
+	possible_elements = null;
+	// Failing all that set an error message
+	if(result == null) {
+	    result = default_string;
+	}
+	return result;
+    }
+
+    private void init(boolean use_classloader) {
+
+	if(this.document != null) {
+	    this.root = document.getDocumentElement();
+	    this.elements = XMLTools.getChildElementsByTagName(root, "Element");
+	}
+	else {
+	    DebugStream.println("Error! Missing mds file: " + file.getAbsolutePath());
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSetManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSetManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/MetadataSetManager.java	(revision 21112)
@@ -0,0 +1,435 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.io.*;
+import java.util.*;
+import java.util.StringTokenizer;
+import javax.swing.*;
+import javax.swing.filechooser.*;
+import javax.swing.tree.TreePath;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.file.*;
+import org.greenstone.gatherer.util.XMLTools;
+import org.apache.xerces.parsers.*;
+import org.apache.xml.serialize.*;
+import org.w3c.dom.*;
+import org.xml.sax.*;
+
+/** This class is responsible for managing all the metadata sets in the collection and for providing methods for manipulating the aforementioned sets contents. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3b
+ */
+public class MetadataSetManager
+{
+    static public final String EXTRACTED_METADATA_NAMESPACE = "ex";
+
+    static final public String HIDDEN = "hidden"; //The name of the hidden, or system, metadata set.
+    static final public int MAX_LOADED_SETS = 64;
+    static public CollectionManager col_man = null;
+    static private Hashtable mds_hashtable = new Hashtable(); //A mapping from metadata namespace to metadata set.
+    public GEMS gemsptr;
+    
+
+    /** Constructor. */
+    public MetadataSetManager(GEMS gemscopy) {
+        gemsptr = gemscopy;
+        col_man = new CollectionManager();
+    }
+
+
+    public MetadataSet addSet(String namespace, String name) {
+	MetadataSet mds = new MetadataSet(GEMS.METADATA_SET_TEMPLATE);
+	mds.setAttribute("creator", "The Greenstone Librarian Interface");
+	// Calculate lastchanged to right now on this machine by this user
+	String user_name = System.getProperty("user.name");
+	String machine_name = Utility.getMachineName();
+	mds.setAttribute("lastchanged", Utility.getDateString() + " - " + user_name + " on " + machine_name);
+	// And the remaining attributes.
+	mds.setAttribute("namespace", namespace);
+	// Add the name element.
+	mds.setName(name);
+	// this is the location for a global metadata set
+	File mds_file = new File(GEMS.getGLIMetadataDirectoryPath() + namespace + ".mds");
+	mds.setFile(mds_file);
+	mds_hashtable.put(namespace, mds);
+	// fireSetChanged(mds);
+	return mds;
+    }
+
+
+    /**
+       Method to rename a metadata set.
+       @param: current_set: The metadata set to rename
+       @param: new_namspace: The new namespace for the current_set
+       @param: new_name: The new (language-dependent) name for the current_set. Multiple names in different languages is not yet implemented in GEMS, but when it is, this method will need to be updated.
+
+       @pre: There is not another metadata set with namespace new_namespace. Ie namespaces are unique.
+       @post: The Name and namespace in the metadata file will have been updated. The filename will stay the same.
+
+       @author Matthew Whyte
+       Date last modified: 10/01/12
+    */
+    public void renameSet(MetadataSet current_set, String new_namespace, String new_name)
+    {
+	if(current_set != null)
+	    {
+		String old_namespace = current_set.getNamespace();
+		current_set.setNamespace(new_namespace); //Update the namespace
+		current_set.setName(new_name); //Update the name
+		mds_hashtable.remove(old_namespace); //Remove the old set from the hashtable
+		mds_hashtable.put(new_namespace, current_set); //Add the new set to the hashtable
+	    }
+	else
+	    {
+		//This should not happen.
+		System.err.println("Error: no such set \'" + current_set + "\'");
+	    }
+    }
+
+
+    /** Destructor. 
+     */
+    public void destroy() {
+	mds_hashtable.clear();
+    }
+
+
+    /** Used to get all the (non-hidden) elements in this manager.
+     * @return A Vector of ElementWrappers.
+     */
+    public Vector getElements() {
+	return getElements(false);
+    }
+
+    public Vector getElements(boolean all) {
+	return getElements(all, false);
+    }
+
+
+    /** Used to get all the elements in this manager.
+     * @param all <i>true</i> if all elements, including hidden, should be returned.
+     * @return A Vector of ElementWrappers.
+     */
+    public Vector getElements(boolean all, boolean force_extracted) {
+	Vector all_elements = new Vector();
+	for(Enumeration keys = mds_hashtable.keys(); keys.hasMoreElements(); ) {
+	    MetadataSet mds = (MetadataSet)mds_hashtable.get(keys.nextElement());
+	    if((!mds.getNamespace().equals(EXTRACTED_METADATA_NAMESPACE) && !mds.getNamespace().equals(HIDDEN)) 
+	       || (mds.getNamespace().equals(EXTRACTED_METADATA_NAMESPACE) && (Configuration.get("general.view_extracted_metadata", Configuration.COLLECTION_SPECIFIC) || force_extracted)) 
+	       || (mds.getNamespace().equals(EXTRACTED_METADATA_NAMESPACE) && mds.getNamespace().equals(HIDDEN) && all)) {
+		ArrayList set_elements = mds.getElements();
+		///ystem.err.println("The set " + mds + " has " + set_elements.getLength() + " elements.");
+		for(int i = 0; i < set_elements.size(); i++) {
+		    Element raw_element = (Element)set_elements.get(i);
+		    ElementWrapper element = new ElementWrapper(raw_element);
+		    // For now we do not add subfield elements and their parents, just the subfields.
+		    NodeList child_elements = raw_element.getElementsByTagName("Element");
+		    if(child_elements.getLength() == 0) {
+			all_elements.add(element);
+		    }
+		}
+	    }
+	}
+	Collections.sort(all_elements);
+	return all_elements;
+    }
+
+
+    /** Retrieve a metadata element by its index.
+     * @param index The specified index as an int.
+     * @return An ElementWrapper containing the specied element, or <i>null</i> is no such element exists.
+     */
+    public ElementWrapper getElement(int index) {
+	Vector elements = getElements(false);
+	ElementWrapper result = null;
+	if(0 <= index && index < elements.size()) {
+	    result = (ElementWrapper) elements.get(index);
+	}
+	return result;
+    }
+
+    /** Retrieve a metadata element by looking at the current metadata element. Note that this 'index' element may now be disconnected from the DOM model, so we have to reload the target element by the string method.
+     * @param element The possibly out-of-data MetadataElement.
+     * @return An ElementWrapper containing the specied element, or <i>null</i> is no such element exists.
+     */
+    public ElementWrapper getElement(ElementWrapper element) {
+	return getElement(element.toString());
+    }
+
+    /** Retrieve a metadata element by its fully qualified name.
+     * @param name The elements name as a String.
+     * @return An ElementWrapper containing the specied element, or <i>null</i> is no such element exists.
+     */
+
+    public ElementWrapper getElement(String name) {
+	return getElement(name, false);
+    }
+
+    public ElementWrapper getElement(String name, boolean perfect) {
+	///ystem.err.println("Retrieve element " + name);
+	if(name == null) {
+	    ///ystem.err.println("No name!");
+	    return null;
+	}
+	ElementWrapper result = null;
+ 	MetadataSet set = null;
+	String element = null;
+	// First we seperate off what set it is in, where we have '<set><namespace_separator><element>'.
+	if(name.indexOf(MSMUtils.NS_SEP) != -1) {
+	    String namespace = name.substring(0, name.indexOf(MSMUtils.NS_SEP));
+	    // Retrieve the correct set if possible.
+	    set = (MetadataSet)mds_hashtable.get(namespace);
+	    namespace = null;
+	    // Now retrieve the element name.
+	    element = name.substring(name.indexOf(MSMUtils.NS_SEP) + 1);
+	}
+	// If we are looking for a perfect match, we can assume that no namespace means extracted metadata
+	else if(!perfect) {
+	    // No namespace so assume that its extracted metadata.
+	    set = (MetadataSet)mds_hashtable.get(EXTRACTED_METADATA_NAMESPACE);
+	    element = name;
+	}
+	if(set != null) {
+	    // Now we have a set we are ready to locate the requested element. Break the remaining element name down by the subfield separator, attempting to retrieve the element indicated by each step.
+	    if(element.indexOf(MSMUtils.SF_SEP) != -1) {
+		StringTokenizer tokenizer = new StringTokenizer(element, new Character(MSMUtils.SF_SEP).toString());
+		// Has to be at least two tokens
+		if(tokenizer.countTokens() >= 2) {
+		    Element current_element = set.getElement(tokenizer.nextToken());
+		    while(current_element != null && tokenizer.hasMoreTokens()) {
+			current_element = set.getElement(current_element, tokenizer.nextToken());
+		    }
+		    if(current_element != null) {
+			result = new ElementWrapper(current_element);
+			current_element = null;
+		    }
+		}
+		tokenizer = null;
+	    }
+	    // No subelements - much easier.
+	    if(result == null) {
+		///ystem.err.print("Trying to match element " + element +"?");
+		Element temp = set.getElement(element);
+		if(temp != null) {
+		    result = new ElementWrapper(temp);
+		}
+		temp = null;
+	    }
+	    set = null;
+	}
+	element = null;
+	return result;
+    }
+
+    /** Retrieve a certain named element from a certain named set.
+     * @param set The metadata set whose element you want.
+     * @param name The name of the element.
+     * @return An ElementWrapper around the requested element, or null if no such set or element.
+     */
+    public ElementWrapper getElement(String set, String name) {
+	if(mds_hashtable.containsKey(set)) {
+	    MetadataSet temp = (MetadataSet)mds_hashtable.get(set);
+	    return new ElementWrapper(temp.getElement(name));
+	}
+	return null;
+    }
+
+
+    /** Retrieve the named metadata set.
+     * @param name The sets name as a String.
+     * @return The MetadataSet as named, or null if no such set.
+     */
+    public MetadataSet getSet(String namespace) {
+	if(mds_hashtable.containsKey(namespace)) {
+	    return (MetadataSet) mds_hashtable.get(namespace);
+	}
+	return null;
+    }
+
+    /** Method to retrieve all of the metadata sets loaded in this collection. 
+     * @return A Vector of metadata sets.
+     */
+    public Vector getSets() {
+	return getSets(true);
+    }
+
+    public Vector getSets(boolean include_greenstone_extracted) {
+	Vector result = new Vector();
+	for(Enumeration keys = mds_hashtable.keys(); keys.hasMoreElements(); ) {
+	    MetadataSet set = (MetadataSet)mds_hashtable.get(keys.nextElement());
+	    if(!set.getNamespace().equals(HIDDEN) && (include_greenstone_extracted || !set.getNamespace().equals(EXTRACTED_METADATA_NAMESPACE))) {
+		result.add(set);
+	    }
+	}
+	return result;
+    }
+  
+
+    public MetadataSet loadMetadataSet(File metadata_set_file)
+    {
+        File[] all_sets = new File[MAX_LOADED_SETS];
+        MetadataSet tempSet;
+        int i = 0;
+        System.err.println("Loading metadata set file " + metadata_set_file + "...");
+	MetadataSet metadata_set = new MetadataSet(metadata_set_file);
+        
+        //write out existing namespaces to all_sets[]...we use this to determine if a set is
+        //already loaded-Attila 
+        for(Enumeration keys = mds_hashtable.keys(); keys.hasMoreElements(); ) {
+	    //System.err.println("keys: " + keys.nextElement() + "\n"); //debug
+	    tempSet = (MetadataSet)mds_hashtable.get(keys.nextElement());        
+            all_sets[i] = tempSet.getFile();
+            i++;
+	}
+        
+        //Now check our metadata.set's namespace to see if it is already loaded in all_sets
+        for(int p = 0 ; p < i; p++) {
+	    if(metadata_set.getFile().compareTo(all_sets[p]) == 0) {
+		gemsptr.warn_cant_load_set();
+		return null; 
+	    }
+        }
+    
+        //check if this metadata set belongs to a collection
+        //if so, then change the metadata set's name in the interface to (collection) dublin core...
+	String sep = new String(metadata_set.getFile().separator);
+	StringTokenizer st = new StringTokenizer(metadata_set.getFile().toString(), sep);
+	String colName = new String("");
+	
+        while ( st.hasMoreTokens() )
+        {
+           if(st.nextToken().compareTo("collect") == 0) {
+	       colName = st.nextToken();  
+	       break;
+           }   
+        }
+
+	if (colName.equals("")) {
+	    mds_hashtable.put(metadata_set.getNamespace(), metadata_set);
+	} else {
+	    metadata_set.setCollection(colName);
+	    mds_hashtable.put(metadata_set.getNamespace(), metadata_set);
+	}
+	return metadata_set;
+    }
+
+
+    /**
+       Method to remove an element or subelement from a metadata set
+       @param element the element to remove, as an <strong>ElementWrapper</strong>
+       @param parent the parent of the element, as an <strong>ElementWrapper</strong> This is required if removing a subelement (can be null - this means the parent is the metadataset root.)
+       @return void
+       Modified by Matthew Whyte 19/01/04
+
+       This should really be totally replaced by MetadataSet.removeElement --Matthew
+     */
+    public void removeElement(ElementWrapper element, ElementWrapper parent) {
+	// Retrieve the metadata set this element belongs to.
+	String namespace = element.getNamespace();
+	//System.err.println("namespace is: " + namespace); //debug
+	MetadataSet set = (MetadataSet) mds_hashtable.get(namespace);
+	if(set != null) {
+	    //Remove the element
+	    set.removeElement(element.getElement(), parent);
+	}
+	else {
+	    //This should not happen - no such set, no such element.
+	    System.err.println("Cannot remove element \'" + element + "\' because there is no such set \'" + namespace + "\'");
+	}
+    }
+
+
+    /**
+       Rename the metadata element.
+       @param oldElement An <strong>ElementWrapper</strong> we wish to rename
+       @param newName A <strong>String</strong> which contains the new name of the element.
+       @return An <strong>ElementWrapper</strong> around the newly named element
+    */
+    public void renameElement(ElementWrapper oldElement, String newName)
+    {
+	//Retrive the metadata set this element belongs to.
+	String namespace = oldElement.getNamespace();
+	MetadataSet set = (MetadataSet)mds_hashtable.get(namespace);
+	if(set != null)
+	    {
+		//System.err.println("newName is: " + newName); //debug
+		set.renameElement(oldElement.getElement(), newName); //Rename the element
+	    }
+	else
+	    {
+		System.err.println("No such set " + namespace);
+	    }
+    }
+
+
+    public void save() {
+	File file = new File(GEMS.getGLIMetadataDirectoryPath());
+       
+	if (!file.exists()) {
+	    System.out.println("trying to save a collection and the metadata directory does not exist - creating it!");
+	    
+	    file.mkdir();
+	}
+	// And make back ups of all existing metadata set files.
+	File temp[] = file.listFiles();
+	
+	// Now save the latest versions of the metadata sets.
+	for(Enumeration keys = mds_hashtable.keys(); keys.hasMoreElements(); ) {
+	    String namespace = (String) keys.nextElement();
+	    MetadataSet set = (MetadataSet) mds_hashtable.get(namespace);
+	    try {
+		// just use the existing file path
+		File mds_file = set.getFile();
+		XMLTools.writeXMLFile(mds_file, set.getDocument());
+		set.setSetChanged(false);
+	    }
+	    catch (Exception error) {
+		error.printStackTrace();
+	    }
+	}
+    }
+    
+    public void removeSet(MetadataSet set) {
+	mds_hashtable.remove(set.getNamespace());
+        File f =  set.getFile();
+        f.delete();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTable.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTable.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTable.java	(revision 21112)
@@ -0,0 +1,76 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.*;
+import javax.swing.JTable;
+import javax.swing.table.*;
+
+/** An extension of a JTable that allows for the background, foreground, heading and selection colors to be set in single actions. */
+public class SmarterTable
+    extends JTable {
+
+    public void setBackground(Color color) {
+	super.setBackground(color);
+	TableCellRenderer renderer = getDefaultRenderer(String.class);
+	((DefaultTableCellRenderer)renderer).setBackground(color);
+    }
+
+    public void setForeground(Color color) {
+	super.setForeground(color);
+	TableCellRenderer renderer = getDefaultRenderer(String.class);
+	((DefaultTableCellRenderer)renderer).setForeground(color);
+    }
+
+    public void setHeadingBackground(Color color) {
+	JTableHeader header = getTableHeader();
+	header.setBackground(color);
+    }
+
+    public void setHeadingForeground(Color color) {
+	JTableHeader header = getTableHeader();
+	header.setForeground(color);
+    }
+
+    public void setSelectionColor(Color color) {
+	super.setSelectionBackground(color);
+    }
+	 
+    public void setSelectedTextColor(Color color) {
+	super.setSelectionForeground(color);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTextArea.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTextArea.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTextArea.java	(revision 21112)
@@ -0,0 +1,79 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import javax.swing.JTextArea;
+import javax.swing.text.*;
+
+public class SmarterTextArea
+    extends JTextArea {
+
+    private boolean init = true;
+
+    public SmarterTextArea() {
+	super();
+    }
+
+    public SmarterTextArea(String text, boolean editable) {
+	super();
+	if(!editable) {
+	    setDocument(new UnchangingDocument());
+	}
+	setText(text);
+	init = false;
+    }
+
+    public int calculateColumnCount() {
+	return (getWidth() / getColumnWidth());
+    }
+
+    private class UnchangingDocument 
+	extends PlainDocument {
+	public void insertString(int offs, String str, AttributeSet a) 
+	    throws BadLocationException {
+	    if(init) {
+		super.insertString(offs, str, a);
+	    }
+	}
+	public void remove(int offs, int len)  
+	    throws BadLocationException {
+	    if (init) {
+		super.remove(offs, len);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTree.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTree.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/SmarterTree.java	(revision 21112)
@@ -0,0 +1,85 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.awt.Color;
+import javax.swing.JTree;
+import javax.swing.tree.*;
+import java.awt.event.*;
+
+/** An extension of a JTree that allows for the background, foreground and selection colors to be set in single actions. */
+public class SmarterTree extends JTree implements ActionListener{
+
+    public SmarterTree() {
+	super();
+    }
+
+    public SmarterTree(TreeModel model) {
+	super(model);
+    }
+
+    public boolean isSelectionEmpty() {
+	return super.isSelectionEmpty();
+    }
+
+    public void setBackground(Color color) {
+	super.setBackground(color);
+	if(cellRenderer != null) {
+	    ((DefaultTreeCellRenderer)cellRenderer).setBackgroundNonSelectionColor(color);
+	}
+    }
+
+    public void setForeground(Color color) {
+	super.setForeground(color);
+	if(cellRenderer != null) {
+	    ((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(color);
+	}
+    }
+
+    public void setSelectionColor(Color color) {
+	((DefaultTreeCellRenderer)cellRenderer).setBackgroundSelectionColor(color);
+    }
+	 
+    public void setSelectedTextColor(Color color) {
+	((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(color);
+    }
+    
+    public void actionPerformed(ActionEvent e) {
+        System.out.println("Action!");
+    }
+    
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/TransformCharacterTextField.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/TransformCharacterTextField.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gems/TransformCharacterTextField.java	(revision 21112)
@@ -0,0 +1,105 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gems;
+
+import java.util.*;
+import javax.swing.*;
+import javax.swing.text.*;
+
+
+/** A JTextField that doesn't allow certain characters, either blocking or replacing them
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class TransformCharacterTextField 
+    extends JTextField {
+    
+    private boolean block_all_except_mappings = false;
+    static private HashMap mappings = new HashMap();
+
+    static public TransformCharacterTextField createNamespaceTextField() {
+	TransformCharacterTextField text = new TransformCharacterTextField();
+	text.setBlockAllExceptMappings(true);
+	int i = 'a';
+	int j = 'A';
+	for( ; i <= 'z'; i++, j++) {
+	    text.replaceCharacter((char)i, (char)i);
+	    text.replaceCharacter((char)j, (char)i);
+	}
+	return text;
+    }
+
+    public TransformCharacterTextField() {
+	super();
+    }
+
+    public TransformCharacterTextField(String text) {
+	super(text);
+    }
+
+    public void replaceCharacter(char x, char y) {
+	mappings.put(new Character(x), new Character(y));
+    }
+
+    public void setBlockAllExceptMappings(boolean block_all_except_mappings) {
+	this.block_all_except_mappings = block_all_except_mappings;
+	setText(""); // Have to reset
+    }
+
+    protected Document createDefaultModel() {
+  	return new TransformCharacterTextDocument();
+    }
+
+    private class TransformCharacterTextDocument 
+	extends PlainDocument {
+		  
+	public void insertString(int offs, String str, AttributeSet a) 
+	    throws BadLocationException {
+				
+	    if (str == null) {
+		return;
+	    }
+	    char[] raw = str.toCharArray();
+	    StringBuffer result = new StringBuffer("");
+	    for (int i = 0; i < raw.length; i++) {
+		Character key = new Character(raw[i]);
+		if(mappings.containsKey(key)) {
+		    Character replacement = (Character) mappings.get(key);
+		    if(replacement != null) {
+			result.append(replacement.charValue());
+			replacement = null;
+		    }
+		}
+		else if(!block_all_except_mappings) {
+		    result.append(raw[i]);
+		}
+		key = null;
+	    }
+	    super.insertString(offs, result.toString(), a);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/AboutDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/AboutDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/AboutDialog.java	(revision 21112)
@@ -0,0 +1,167 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Greenstone Librarian Interface (GLI) application, 
+ * part of the Greenstone digital library software suite from the New 
+ * Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
+ *
+ * Author: John Thompson
+ *         Greenstone Project, New Zealand Digital Library
+ *         University of Waikato
+ *         http://www.nzdl.org
+ *
+ * Copyright (C) 2004 New Zealand Digital Library, University of Waikato
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.Utility;
+
+/** Generates a pretty about dialog which not only thanks those for their contributions but also meets our legal requirements if we wish to ship the JVM with GLI.
+ * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+ * @version 2.41 final
+ */
+// RICOH SOURCE CODE PUBLIC LICENSE - http://www.risource.org/RPL/RPL-1.0A.shtml
+public class AboutDialog
+    extends JDialog {
+    /** The default size of the about dialog. */
+    static final private Dimension SIZE = new Dimension(600, 325);
+    /** The size of the GLI icon to display on dialog. */
+    static final private int ICON_SIZE = 65;
+    /** A reference to ourself so that our inner classes can dismiss us. */
+    private AboutDialog self;
+    /** The button used for dismissing the about dialog. */
+    private JButton close_button;
+    /** The constructor not only builds, but displays the about dialog. This method doesn't return until the dialog is dismissed.
+     * @param parent the JFrame which owns this dialog for use in centering the dialog
+     * @see org.greenstone.gatherer.Dictionary#get
+     * @see org.greenstone.gatherer.Dictionary#setBoth
+     * @see org.greenstone.gatherer.Dictionary#setText
+     * @see org.greenstone.gatherer.gui.AboutDialog.CloseButtonListener
+     * @see org.greenstone.gatherer.gui.GLIButton
+     * @see org.greenstone.gatherer.util.Utility#PROGRAM_NAME
+     * @see org.greenstone.gatherer.util.Utility#PROGRAM_VERSION
+     */
+    public AboutDialog(JFrame parent) {
+	super(parent, Dictionary.get("AboutDialog.Title"), true);
+	this.self = this;
+	setSize(SIZE);
+		  
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel upper_pane = new JPanel();
+	ImageIcon icon = JarTools.getImage("gatherer_medium.gif");
+	ImageIcon scaled_icon = new ImageIcon(icon.getImage().getScaledInstance(ICON_SIZE, ICON_SIZE, Image.SCALE_DEFAULT));
+	JLabel icon_label = new JLabel(scaled_icon);
+	JPanel title_pane = new JPanel();
+	JLabel title_one_label = new JLabel();
+	Dictionary.setText(title_one_label, "AboutDialog.Title_One");
+	JLabel title_two_label = new JLabel(Utility.PROGRAM_NAME + " " + Utility.PROGRAM_VERSION + " " + Dictionary.get("AboutDialog.Date"));
+	JLabel title_three_label = new JLabel();
+	Dictionary.setText(title_three_label, "AboutDialog.Title_Two");
+	JLabel copyright_label = new JLabel();
+	Dictionary.setText(copyright_label, "AboutDialog.Copyright");
+	JLabel gpl_label = new JLabel();
+	Dictionary.setText(gpl_label, "AboutDialog.Copyright_Two");
+
+	JTextArea text = new JTextArea();
+	text.setLineWrap(true);
+	text.setWrapStyleWord(true);
+
+	JPanel button_pane = new JPanel();
+	close_button = new GLIButton();
+	close_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(close_button, "General.Close", "General.Close_Tooltip");
+
+	// Connection
+	close_button.addActionListener(new CloseButtonListener());
+
+	// Layout
+	icon_label.setBorder(BorderFactory.createEmptyBorder(0,0,0,10));
+
+	title_pane.setLayout(new GridLayout(5,1,0,2));
+	title_pane.add(title_one_label);
+	title_pane.add(title_two_label);
+	title_pane.add(title_three_label);
+	title_pane.add(copyright_label);
+	title_pane.add(gpl_label);
+		  
+	upper_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	upper_pane.setLayout(new BorderLayout());
+	upper_pane.add(icon_label, BorderLayout.WEST);
+	upper_pane.add(title_pane, BorderLayout.CENTER);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new BorderLayout());
+	button_pane.add(close_button, BorderLayout.EAST);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(upper_pane, BorderLayout.NORTH);
+	content_pane.add(new JScrollPane(text), BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Build text content
+	text.append(Dictionary.get("AboutDialog.Java_Req"));
+	text.append("\n");
+	text.append(Dictionary.get("AboutDialog.Java_Req_One"));
+	text.append("\n");
+	text.append(Dictionary.get("AboutDialog.Java_Req_Two"));
+	text.append("\n\n");
+	text.append("*****" + Dictionary.get("AboutDialog.Acknowledgement") + "*****");
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item0"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item2"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item3"));
+	text.append("\n\n");
+	text.append("*****" + Dictionary.get("AboutDialog.Thanks") + "*****");
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item4"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item5"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item6"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item7"));
+	text.append("\n\n");
+	text.append(Dictionary.get("AboutDialog.Item8"));
+	text.setCaretPosition(0);
+
+	// Show
+	Rectangle frame_bounds = parent.getBounds();
+	setLocation(frame_bounds.x + (frame_bounds.width - SIZE.width) / 2, frame_bounds.y + (frame_bounds.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+    /** Listens for actions upon the close button, and when detected closes the dialog. */
+    private class CloseButtonListener
+	implements ActionListener {
+	/** Called whenever an action occurs on the close button, thus asking the dialog to close.
+	 * @param event an ActionEvent containing information about the button press
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    self.setVisible(false);
+	    self.dispose();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ArgumentControl.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ArgumentControl.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ArgumentControl.java	(revision 21112)
@@ -0,0 +1,322 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.cdm.Argument;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+
+public class ArgumentControl
+    extends JPanel {
+    static protected Dimension LABEL_SIZE = new Dimension(180, 25);
+    static protected Dimension ROW_SIZE = new Dimension(610, 30);
+    static protected Dimension SPINNER_SIZE = new Dimension(100, 25);
+    static protected String DESCRIPTION_SEP = " + ";
+
+    protected Argument argument;
+    protected JComponent value_control;
+    protected JCheckBox enabled;
+
+    public ArgumentControl(Argument argument, boolean enable, String value) {
+	super();
+	this.argument = argument;
+	String tooltip = Utility.formatHTMLWidth("<html>" + argument.getDescription() + "</html>", 60);
+	setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	setLayout(new BorderLayout());
+	setPreferredSize(ROW_SIZE);
+	setMaximumSize(ROW_SIZE);
+	
+	// Try to determine what the value_controls value should be.
+	if(value == null) {
+	    value = argument.getDefaultValue();
+	}
+	// Create a correct value control based on the argument provided.
+	switch(argument.getType()) {
+	case Argument.ENUM:
+	case Argument.METADATUM:
+	    JComboBox combobox = new JComboBox();
+	    combobox.setEnabled(enable);
+	    combobox.setToolTipText(tooltip);
+	    combobox.setBackground(enable ? Color.white : Color.lightGray);
+	    
+	    // Build an option model, wrapping each entry of the list table
+	    if (argument.getType() == Argument.ENUM) {
+		HashMap arg_list = argument.getOptions();
+		Iterator it = arg_list.keySet().iterator();
+		ArrayList options = new ArrayList();
+		while (it.hasNext()) {
+		    options.add((String) it.next());
+		}
+		Collections.sort(options);
+		for (int i=0; i<options.size(); i++) {
+		    combobox.addItem(options.get(i));
+		}
+		// Connect this so if a value is selected the tooltip updates accordingly
+		combobox.addActionListener(new ToolTipUpdater(arg_list));
+	    }
+	    if (argument.getType() == Argument.METADATUM) {
+		ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+		for (int i = 0; i < every_metadata_set_element.size(); i++) {
+		    combobox.addItem(every_metadata_set_element.get(i));
+		}
+	    }
+
+	    if (value != null) {
+		// Set the selected string
+		for (int i = 0; i < combobox.getItemCount(); i++) {
+		    if (combobox.getItemAt(i).toString().equals(value)) {
+			combobox.setSelectedIndex(i);
+		    }
+		}
+	    }
+	    // Layout
+	    add(combobox, BorderLayout.CENTER);
+	    // And remember
+	    value_control = combobox;
+	    break;
+	case Argument.FLAG:
+	    // Only need the check box.
+	    value_control = null;
+	    break;
+	case Argument.INTEGER:
+	    // Build a spinner
+	    int initial_value=0;
+	    if (value != null) {
+		try {
+		    initial_value = Integer.parseInt(value);
+		} catch (Exception exception) {
+		}
+	    }
+	    if (initial_value < argument.getMinimum()) {
+		initial_value = argument.getMinimum();
+	    } else if (initial_value > argument.getMaximum()) {
+		initial_value = argument.getMaximum();
+	    }
+	    JSpinner spinner = new JSpinner(new SpinnerNumberModel(initial_value, argument.getMinimum(), argument.getMaximum(), 1));		
+	    spinner.setEnabled(enable);
+	    spinner.setPreferredSize(SPINNER_SIZE);
+	    spinner.setToolTipText(tooltip);
+	    // Set enabled
+	    JComponent c = spinner.getEditor();
+	    if ( c instanceof JSpinner.DefaultEditor ) {
+		JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+		JFormattedTextField field = editor.getTextField();
+		field.setEditable(enable);
+		if(enable) {
+		    field.setBackground(Color.white);
+		}
+		else {
+		    field.setBackground(Color.lightGray);
+		}
+	    }
+	    // Layout
+	    add(new JLabel(), BorderLayout.CENTER);
+	    add(spinner, BorderLayout.EAST);
+	    // And remember it
+	    value_control = spinner;
+	    break;
+	case Argument.REGEXP:
+	case Argument.STRING:
+	    // Use a standard text field
+	    JTextField textfield = new JTextField();
+	    textfield.setEnabled(enable);
+	    textfield.setToolTipText(tooltip);
+	    // Set enabled
+	    if(enable) {
+		textfield.setBackground(Color.white);
+	    }
+	    else {
+		textfield.setBackground(Color.lightGray);
+	    }
+	    // If there was an original value, set it.
+	    if(value != null) {
+		textfield.setText(value);
+	    }
+	    // Layout
+	    add(textfield, BorderLayout.CENTER);
+	    // And remember it
+	    value_control = textfield;
+	    break;
+	}
+
+	// If the argument is required, then you don't get a choice of whether it is enabled.
+	if(argument.isRequired()) {
+	    JLabel label = new JLabel(argument.getName());
+	    label.setOpaque(false);
+	    label.setPreferredSize(LABEL_SIZE);
+	    label.setToolTipText(tooltip);
+	    add(label, BorderLayout.WEST);
+	}
+	else {
+	    enabled = new JCheckBox(argument.getName(), enable);
+	    enabled.setOpaque(false);
+	    enabled.setPreferredSize(LABEL_SIZE);
+	    enabled.setToolTipText(tooltip);
+	    // Connect
+	    enabled.addActionListener(new EnabledListener(value_control));
+	    // Layout
+	    add(enabled, BorderLayout.WEST);
+	}
+    }
+
+    public String getName() {
+	return argument.getName();
+    }
+
+    public String getValue() {
+	String value = null;
+	if(value_control == null) {
+	    // Flag value, nothing to do.
+	}
+	else if(value_control instanceof JTextField) {
+	    value = ((JTextField)value_control).getText();
+	}
+	else if(value_control instanceof JSpinner) {
+	    value = ((JSpinner)value_control).getValue().toString();
+	}
+	else if(value_control instanceof JComboBox) {
+	    Object selected_item = ((JComboBox)value_control).getSelectedItem();
+	    if (selected_item != null) {
+		value = selected_item.toString();
+	    }
+	}
+	return value;
+    }
+    /** Retrieve the control used for storing values.
+     * @return JComponent
+     */
+    public JComponent getValueControl() {
+	return value_control;
+    }
+
+    public boolean isEnabled() {
+	if (enabled == null) {
+	    return true; // always enabled
+	} 
+	return enabled.isSelected();
+    }
+
+    /** Method to ensure that a certain value is selected, if it exists within that combobox to begin with.
+     * @param combobox the JComboBox whose selection we are trying to preset
+     * @param target The desired value of the selection as a String
+     * @return true if the item was found and selected, false otherwise
+     */
+    public boolean selectValue(JComboBox combobox, String target) {
+	for(int i = 0; i < combobox.getItemCount(); i++) {
+	    if(combobox.getItemAt(i).toString().equals(target)) {
+		combobox.setSelectedIndex(i);
+		return true;
+	    }
+	}
+	return false;
+    }
+
+    /** Listens for actions apon the enable checkbox, and if detected enables or diables control appropriately. */
+    private class EnabledListener
+	implements ActionListener {
+	/** An editor component, such as a JComboBox or JTextField, that might have its enabled state changed by this listener. */
+	private JComponent target = null;
+	/** Constructor. */
+	public EnabledListener(JComponent target) {
+	    this.target = target;
+	}
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action has been performed on or registered check box, prompting us to change the state of the other controls as per the users request.
+	 * @param event An <strong>ActionEvent</strong> containing information about the click.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    JCheckBox source = (JCheckBox)event.getSource();
+	    if(target != null) {
+		if(source.isSelected()) {
+		    target.setBackground(Color.white);
+		    target.setEnabled(true);
+		}
+		else {
+		    target.setBackground(Color.lightGray);
+		    target.setEnabled(false);
+		}
+		// Special case of stupid JSpinners who don't let their backgrounds change properly.
+		if(target instanceof JSpinner) {
+		    JSpinner spinner = (JSpinner) target;
+		    JComponent c = spinner.getEditor();
+		    if ( c instanceof JSpinner.DefaultEditor ) {
+			JSpinner.DefaultEditor editor = (JSpinner.DefaultEditor) c;
+			JFormattedTextField field = editor.getTextField();
+			field.setEditable(source.isSelected());
+			if(source.isSelected()) {
+			    field.setBackground(Color.white);
+			}
+			else {
+			    field.setBackground(Color.lightGray);
+			}
+		    }
+		}
+	    }
+	}
+    }
+
+   /** Listener that sets the tooltip associated to a combobox to the tooltip relevant to the selected item. */
+    private class ToolTipUpdater
+	implements ActionListener {
+	private HashMap arg_list;
+	public ToolTipUpdater(HashMap arg_list) {
+	    this.arg_list = arg_list;
+	}
+	/** Any implementation of an ActionListener must include this method so that we can be informed when the selection in a combobox has changed and update the tooltip accordingly.
+	 * @param event An <strong>ActionEvent</strong> containing information about the action that fired this call.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    JComboBox source = (JComboBox)event.getSource();
+	    String key = (String) source.getSelectedItem();
+	    if(arg_list != null) {
+		String description = (String) arg_list.get(key);
+		if(description != null) {
+		    description = Utility.formatHTMLWidth(description, 60);
+		    source.setToolTipText(description);
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/CreatePane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/CreatePane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/CreatePane.java	(revision 21112)
@@ -0,0 +1,990 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.cdm.SearchTypeManager;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.collection.Collection;
+import org.greenstone.gatherer.shell.GBasicProgressMonitor;
+import org.greenstone.gatherer.shell.GBuildProgressMonitor;
+import org.greenstone.gatherer.shell.GImportProgressMonitor;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.shell.GShellEvent;
+import org.greenstone.gatherer.shell.GShellListener;
+import org.greenstone.gatherer.shell.GShellProgressMonitor;
+import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+
+/** This class provides a GUI view showing some statistics on your current collection, and options for building it. As the collection is built this initial view is replaced with one showing progress bars and a message log of the creation process. This log can be later accessed via the options tree located in the center of the initial pane. This class is also responsible for creating the GShellProgressMonitors that are then attatched to external shell processes, and calling the methods in the CollectionManager for actually importing and building the collection. <br><BR>
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class CreatePane
+    extends JPanel
+    implements GShellListener {
+
+    static private Dimension ARGUMENT_SIZE = new Dimension(800,90);
+    /** The threshold for when the simple view is replaced by the complex one. */
+    static private final int THRESHOLD = Configuration.EXPERT_MODE;
+    /** An identifier for the control panel within the card layout. */
+    static private String CONTROL  = "Control";
+    /** An identifier for the progress panel within the card layout. */
+    static private String PROGRESS = "Progress";
+    /** An identifier for the simple panel within the card layout. */
+    static private String SIMPLE   = "Simple";
+
+    /** Determines the current view that should be shown for this pane. */
+    public boolean processing = false;
+    /** The options pane generates all the various option sheet configuations. */
+    public OptionsPane options_pane = null;
+    private AppendLineOnlyFileDocument document;
+    /** A card layout is used to store the separate configuration and progress panes. */
+    private CardLayout card_layout = null;
+    /** Stores a reference to the current collection. */
+    private Collection previous_collection = null;
+    /** This monitor tracks the build processes progress. */
+    private GShellProgressMonitor build_monitor = null;
+    /** This monitor tracks the import processes progress. */
+    private GShellProgressMonitor import_monitor = null;
+    /** The button for begining the building processes. */
+    private JButton build_button = null;
+    /** The button for stopping the building processes. */
+    private JButton cancel_button = null;
+    /** The button for viewing the collection. */
+    private JButton preview_button = null;
+    private JButton simple_build_button;
+    private JButton simple_cancel_button;
+    private JButton simple_preview_button;
+    /** The radio buttons for chnaging the build type (incremental/full) */
+    private JRadioButton full_build_radio_button;
+    private JRadioButton incremental_build_radio_button;
+    private JRadioButton sfull_build_radio_button;
+    private JRadioButton sincremental_build_radio_button;
+    /** The label displaying the number of documents in this collection. */
+    private JLabel document_count = null;
+    /** The label alongside the build progress bar gets some special treatment so... */
+    private JLabel progress_build_label = null;
+    /** The label alongside the import progress bar gets some special treatment so... */
+    private JLabel progress_import_label = null;
+    private JPanel bar_area;
+    /** The panel on which buttons are rendered on higher detail modes. */
+    private JPanel button_pane;
+    /** The pane which contains the controls for configuration. */
+    private JPanel control_pane = null;
+    /** The pane which has the card layout as its manager. */
+    private JPanel main_pane = null;
+    /** The pane which contains the progress information. */
+    private JPanel progress_pane = null;
+    /** The pane on the right-hand side - shows the requested options view */
+    private JPanel right = null;
+    /** The simple panel on which all of the available arguments are rendered. */
+    private JPanel sargument_configuration_panel;
+    /** The panel on which buttons are rendered on lower details modes. */
+    private JPanel sbutton_panel;
+    /** A simplified version for lower detail modes containing both the control and progress panes smooshed together. */
+    private JPanel simple_panel;
+    /** The inner panel of the simple pane which is global so that the bar_area can be added and removed from it. */
+    private JPanel sinner_panel;
+    /** The outer panel of the simple pane which is global so that the arguments pane can be added and removed from it. */
+    private JPanel souter_panel;
+    /** The scrolling pane for the log. */
+    private JScrollPane log_scroll;
+    /** The scrolling pane the simple arguments are rendered within. */
+    private JScrollPane sargument_configuration_scroll;
+    /** the scrolling pane the righthand side is inside - only used for import and build options. the log and log list are not inside a scrollpane */
+    private JScrollPane scroll_pane;
+    /** The message log for the entire session. */
+    private JTextArea log_textarea;
+    /** A tree used to display the currently available option views. */
+    private OptionTree tree = null;
+    /** An array used to pass arguments with dictionary calls. */
+    private String args[] = null;
+    /** The homepage address of the current collection */
+    private String homepage = null;
+
+
+    /** The constructor creates important helper classes, and initializes all the components.
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.gui.CreatePane.BuildButtonListener
+     * @see org.greenstone.gatherer.gui.CreatePane.CancelButtonListener
+     * @see org.greenstone.gatherer.gui.CreatePane.OptionTree
+     * @see org.greenstone.gatherer.gui.OptionsPane
+     * @see org.greenstone.gatherer.shell.GBasicProgressMonitor
+     * @see org.greenstone.gatherer.shell.GBuildProgressMonitor
+     * @see org.greenstone.gatherer.shell.GImportProgressMonitor
+     * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+     */
+    public CreatePane() {
+	Collection collection = Gatherer.c_man.getCollection();
+	log_textarea = new JTextArea();
+	log_scroll = new JScrollPane(log_textarea);
+
+	// Create components
+	card_layout = new CardLayout();
+	// Control Pane
+	control_pane = new JPanel();
+	tree = new OptionTree();
+	button_pane = new JPanel();
+
+	// Progress Pane
+	progress_pane = new JPanel(); // One owner of the bar component
+	bar_area = new JPanel(); // This component will be shared about
+
+	progress_import_label = new JLabel();
+	Dictionary.registerText(progress_import_label, "CreatePane.Import_Progress");
+
+	import_monitor = new GImportProgressMonitor(); //GBasicProgressMonitor();
+	Gatherer.c_man.registerImportMonitor(import_monitor);
+
+	progress_build_label = new JLabel();
+	Dictionary.registerText(progress_build_label, "CreatePane.Build_Progress");
+
+	build_monitor = new GBuildProgressMonitor(import_monitor.getSharedProgress()); //GBasicProgressMonitor();
+	Gatherer.c_man.registerBuildMonitor(build_monitor);
+
+	// And the simple panel
+	simple_panel = new JPanel();
+	sbutton_panel = new JPanel();
+	sinner_panel = new JPanel();
+
+	//Radio buttons
+	incremental_build_radio_button = new JRadioButton();
+	Dictionary.registerBoth(incremental_build_radio_button, "CreatePane.Incremental_Build", "CreatePane.Incremental_Build_Tooltip");
+	full_build_radio_button = new JRadioButton();
+	Dictionary.registerBoth(full_build_radio_button, "CreatePane.Full_Build", "CreatePane.Full_Build_Tooltip");
+	sincremental_build_radio_button = new JRadioButton();
+	Dictionary.registerBoth(sincremental_build_radio_button, "CreatePane.Incremental_Build", "CreatePane.Incremental_Build_Tooltip");
+	sfull_build_radio_button = new JRadioButton();
+	Dictionary.registerBoth(sfull_build_radio_button, "CreatePane.Full_Build", "CreatePane.Full_Build_Tooltip");
+
+	// Buttons
+	BuildButtonListener bbl = new BuildButtonListener();
+	CancelButtonListener cbl = new CancelButtonListener();
+	PreviewButtonListener pbl = new PreviewButtonListener();
+
+	build_button = new GLIButton();
+	build_button.addActionListener(bbl);
+	build_button.setMnemonic(KeyEvent.VK_B);
+	Dictionary.registerBoth(build_button, "CreatePane.Build_Collection", "CreatePane.Build_Collection_Tooltip");
+
+	cancel_button = new GLIButton();
+	cancel_button.addActionListener(cbl);
+	cancel_button.setEnabled(false);
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(cancel_button, "CreatePane.Cancel_Build", "CreatePane.Cancel_Build_Tooltip");
+
+	preview_button = new GLIButton();
+	preview_button.addActionListener(pbl);
+	if(Gatherer.c_man != null) {
+	    preview_button.setEnabled(Gatherer.c_man.built());
+	}
+	else {
+	    preview_button.setEnabled(false);
+	}
+	preview_button.setMnemonic(KeyEvent.VK_P);
+	Dictionary.registerBoth(preview_button, "CreatePane.Preview_Collection", "CreatePane.Preview_Collection_Tooltip");
+
+	simple_build_button = new GLIButton();
+	simple_build_button.addActionListener(bbl);
+	simple_build_button.setMnemonic(KeyEvent.VK_B);
+	Dictionary.registerBoth(simple_build_button, "CreatePane.Build_Collection", "CreatePane.Build_Collection_Tooltip");
+
+	simple_cancel_button = new GLIButton();
+	simple_cancel_button.addActionListener(cbl);
+	simple_cancel_button.setEnabled(false);
+	simple_cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(simple_cancel_button, "CreatePane.Cancel_Build", "CreatePane.Cancel_Build_Tooltip");
+
+	simple_preview_button = new GLIButton();
+	simple_preview_button.addActionListener(pbl);
+	if(Gatherer.c_man != null) {
+	    simple_preview_button.setEnabled(Gatherer.c_man.built());
+	}
+	else {
+	    simple_preview_button.setEnabled(false);
+	}
+	simple_preview_button.setMnemonic(KeyEvent.VK_P);
+	Dictionary.registerBoth(simple_preview_button, "CreatePane.Preview_Collection", "CreatePane.Preview_Collection_Tooltip");
+
+	bbl = null;
+	cbl = null;
+	pbl = null;
+    }
+
+
+    public void destroy() {
+	if(document != null) {
+	    document.destroy();
+	}
+    }
+    
+    /** This method is called to actually layout the components.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     */
+    public void display() {
+
+	int current_mode = Configuration.getMode();
+
+	//Complete/incremental build options panel
+	//For some reason I need to create seperate objects for each mode. Is there a better way??
+	ButtonGroup build_type_group = new ButtonGroup();
+	build_type_group.add(incremental_build_radio_button);
+	full_build_radio_button.setSelected(true);
+	build_type_group.add(full_build_radio_button);
+	ButtonGroup sbuild_type_group = new ButtonGroup();
+	sbuild_type_group.add(sincremental_build_radio_button);
+	sfull_build_radio_button.setSelected(true);
+	sbuild_type_group.add(sfull_build_radio_button);
+	JPanel build_type_pane = new JPanel(new GridLayout(2,1));
+	build_type_pane.add(full_build_radio_button);
+	build_type_pane.add(incremental_build_radio_button);
+	JPanel sbuild_type_pane = new JPanel(new GridLayout(2,1));
+	sbuild_type_pane.add(sfull_build_radio_button);
+	sbuild_type_pane.add(sincremental_build_radio_button);
+	
+	
+
+	// Build control_pane
+	JPanel left = new JPanel();
+	left.setBorder(BorderFactory.createEmptyBorder(0,5,5,5));
+	left.setLayout(new BorderLayout());
+	left.add(tree, BorderLayout.CENTER);
+	left.add(build_type_pane, BorderLayout.SOUTH);
+	//left.add(full_build_radio_button, BorderLayout.SOUTH);
+
+	right = new JPanel();
+	right.setBorder(BorderFactory.createEmptyBorder(0,0,5,5));
+	right.setLayout(new BorderLayout());
+
+	JPanel options_area = new JPanel();
+	options_area.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,5,5,5), BorderFactory.createTitledBorder(Dictionary.get("CreatePane.Options_Title"))));
+	options_area.setLayout(new BorderLayout());
+	options_area.add(left, BorderLayout.WEST);
+	options_area.add(right, BorderLayout.CENTER);
+
+	button_pane = new JPanel();
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,10,10,10));
+	button_pane.setLayout(new GridLayout(1,4));
+	button_pane.add(build_button);
+	button_pane.add(cancel_button);
+	button_pane.add(preview_button);
+
+	control_pane.setLayout(new BorderLayout());
+	control_pane.add(options_area, BorderLayout.CENTER);
+	control_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Build progress_pane
+	JPanel labels_pane = new JPanel();
+	labels_pane.setLayout(new GridLayout(2,1,0,5));
+	labels_pane.add(progress_import_label);
+	labels_pane.add(progress_build_label);
+
+	JPanel monitors_pane = new JPanel();
+	monitors_pane.setLayout(new GridLayout(2,1,0,5));
+	monitors_pane.add(import_monitor.getProgress());
+	monitors_pane.add(build_monitor.getProgress());
+
+	bar_area.setBorder(BorderFactory.createEmptyBorder(10,10,5,10));
+	bar_area.setLayout(new BorderLayout(5,5));
+	bar_area.add(labels_pane, BorderLayout.WEST);
+	bar_area.add(monitors_pane, BorderLayout.CENTER);
+
+	progress_pane.setBorder(BorderFactory.createEmptyBorder(20,20,20,20));
+	progress_pane.setLayout(new BorderLayout());
+	progress_pane.add(bar_area, BorderLayout.NORTH);
+	if(current_mode >= THRESHOLD) {
+	    progress_pane.add(log_scroll, BorderLayout.CENTER);
+	}
+
+	// Simple panel
+	sbutton_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	sbutton_panel.setLayout(new GridLayout(1,3));
+	sbutton_panel.add(simple_build_button);
+	sbutton_panel.add(simple_cancel_button);
+	sbutton_panel.add(simple_preview_button);
+
+	JPanel simple_bar_area = new JPanel(new GridLayout(2,1));
+	simple_bar_area.setBorder(BorderFactory.createEmptyBorder(0,5,0,0));
+	simple_bar_area.add(import_monitor.getSharedProgress());
+	simple_bar_area.add(sbutton_panel);
+
+	sinner_panel.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	sinner_panel.setLayout(new BorderLayout());
+	sinner_panel.add(sbuild_type_pane, BorderLayout.WEST);
+	//sinner_panel.add(full_build_radio_button, BorderLayout.WEST);
+	sinner_panel.add(simple_bar_area, BorderLayout.CENTER);
+	
+	if(options_pane != null) {
+	    sargument_configuration_panel = options_pane.buildImport(null);
+	    sargument_configuration_panel = options_pane.buildBuild(sargument_configuration_panel);
+	}
+	else {
+	    sargument_configuration_panel = new JPanel();
+	}
+	sargument_configuration_scroll = new JScrollPane(sargument_configuration_panel);
+	sargument_configuration_scroll.setPreferredSize(ARGUMENT_SIZE);
+	souter_panel = new JPanel();
+	souter_panel.setSize(new Dimension(400,800));
+	souter_panel.setLayout(new GridLayout(2,1));
+	souter_panel.add(sargument_configuration_scroll);
+	souter_panel.add(sinner_panel);
+
+	simple_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	simple_panel.setLayout(new BorderLayout());
+	simple_panel.add(souter_panel, BorderLayout.NORTH);
+	if(current_mode < THRESHOLD) {
+	    simple_panel.add(log_scroll, BorderLayout.CENTER);
+	}
+
+	// Main pane
+	main_pane = new JPanel(card_layout);
+	if(current_mode < THRESHOLD) { // Simple mode - add first
+	    main_pane.add(simple_panel, SIMPLE);
+	}
+	main_pane.add(control_pane, CONTROL);
+	main_pane.add(progress_pane, PROGRESS);
+	if(current_mode >= THRESHOLD) { // Expert mode - add last
+	    main_pane.add(simple_panel, SIMPLE);
+	}
+
+	this.setLayout(new BorderLayout());
+	this.add(main_pane, BorderLayout.CENTER);
+    }
+
+    /** This method is called whenever the 'Create' tab is selected from the view bar. It allows this view to perform any preactions required prior to display. In this case this entails gathering up to date information about the status of the collection including number of documents etc.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.gui.CreatePane.OptionTree
+     */
+    public void gainFocus() {
+	if(Configuration.getMode() < THRESHOLD) {
+	    card_layout.show(main_pane, SIMPLE);
+	}
+	else if(!processing) {
+	    // Move the buttons to control
+	    control_pane.add(button_pane, BorderLayout.SOUTH);
+	    card_layout.show(main_pane, CONTROL);
+	}
+	else {
+	    // Move the buttons to progress
+	    progress_pane.add(button_pane, BorderLayout.SOUTH);
+	    card_layout.show(main_pane, PROGRESS);
+	}
+	// Refresh the set of controls.
+	TreePath path = tree.getSelectionPath();
+	tree.clearSelection();
+	tree.setSelectionPath(path);
+
+	//Disable the full/incremental buttons if the collection has not been built.
+	if(Gatherer.c_man.built()) {
+	    full_build_radio_button.setEnabled(true);
+	    incremental_build_radio_button.setEnabled(true);
+	    sfull_build_radio_button.setEnabled(true);
+	    sincremental_build_radio_button.setEnabled(true);
+	}
+	else {
+	    full_build_radio_button.setEnabled(false);
+	    incremental_build_radio_button.setEnabled(false);
+	    sfull_build_radio_button.setEnabled(false);
+	    sincremental_build_radio_button.setEnabled(false);
+	}
+    }
+
+    /** We are informed when this view pane loses focus so we can update build options. */
+    public void loseFocus() {
+	tree.valueChanged(null);
+    }
+
+    /**
+     * Check to see if doing an incremental build is chosen.
+     * If not, the full rebuild option must be chosen.
+     * @return boolean - true if the collection should be rebuilt incrementally.
+     */
+    public boolean isIncremental() {
+	boolean incremental = false;
+
+	//This is horrible. What if the mode is changed, and different options are chosen on different modes??
+	if(Gatherer.c_man.built()) {
+	    if(Configuration.getMode() >= THRESHOLD) {
+		incremental = incremental_build_radio_button.isSelected();
+	    }
+	    else {
+		incremental = sincremental_build_radio_button.isSelected();
+	    }
+	}
+	else {
+	    incremental = false; //The collection has not yet been built
+	}
+	return incremental;
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed of messages from the GShell.
+     * @param event A <strong>GShellEvent</strong> that contains, amoung other things, the message.
+     */
+    public synchronized void message(GShellEvent event) {
+	// Ignore the messages from RecPlug (used for progress bars)
+	if (event.getMessage().startsWith("import.pl> RecPlug - ")) {
+	    return;
+	}
+	document.appendLine(event.getMessage());
+	log_textarea.setCaretPosition(document.getLengthToNearestLine());
+    }
+
+    /** Called when the detail mode has changed which in turn may cause several import/build configuration arguments to be available/hidden
+     * @param mode the new mode as an int
+     */
+    public void modeChanged(int mode) {
+	// If we are below the complexity threshold ensure the simple controls are being shown
+	if(Configuration.getMode() < THRESHOLD) {
+	    // Update the arguments
+	    souter_panel.remove(sargument_configuration_scroll);
+	    souter_panel.remove(sinner_panel);
+	    if(options_pane != null) {
+		sargument_configuration_panel = options_pane.buildImport(null);
+		sargument_configuration_panel = options_pane.buildBuild(sargument_configuration_panel);
+	    }
+	    else {
+		sargument_configuration_panel = new JPanel();
+	    }
+	    sargument_configuration_scroll = new JScrollPane(sargument_configuration_panel);
+	    sargument_configuration_scroll.setPreferredSize(ARGUMENT_SIZE);
+	    souter_panel.add(sargument_configuration_scroll);
+	    souter_panel.add(sinner_panel);
+	    // Remove the shared components from the expert panels
+	    progress_pane.remove(log_scroll);
+	    // And add to simple one
+	    simple_panel.add(log_scroll, BorderLayout.CENTER);
+	    // And bring the card to the front
+	    card_layout.show(main_pane, SIMPLE);
+	}
+	// And if we are above the threshold change to the complex controls
+	else {
+	    // Remove the shared components from the simple panel
+	    simple_panel.remove(log_scroll);
+	    // And add then to the expert ones
+	    progress_pane.add(log_scroll, BorderLayout.CENTER);
+	    // And bring the appropriate card to the front
+	    if(!processing) {
+		control_pane.add(button_pane, BorderLayout.SOUTH);
+		card_layout.show(main_pane, CONTROL);
+	    }
+	    else {
+		progress_pane.add(button_pane, BorderLayout.SOUTH);
+		card_layout.show(main_pane, PROGRESS);
+	    }
+	}
+	tree.valueChanged(null); // Ensure tree argument panels are rebuilt
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell begins its task. Implementation side-effect, not actually used.
+     * @param event A <strong>GShellEvent</strong> that contains details of the initial state of the <strong>GShell</strong> before task comencement.
+     */
+    public synchronized void processBegun(GShellEvent event) {
+	// We don't care. We'll get a more acurate start from the progress monitors.
+    }
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell completes its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the final state of the <strong>GShell</strong> after task completion.
+     */
+    public synchronized void processComplete(GShellEvent event) {
+	System.err.println("In CreatePane::processComplete(" + event.getType() + ")...");
+	if(event.getStatus() == GShell.OK) {
+	    if(event.getType() == GShell.BUILD) {
+		processing = false;
+		build_button.setEnabled(true);
+		cancel_button.setEnabled(false);
+		//preview_button.setEnabled(true);
+		simple_build_button.setEnabled(true);
+		simple_cancel_button.setEnabled(false);
+		//simple_preview_button.setEnabled(true);
+		int status = event.getStatus();
+		document.setSpecialCharacter(OptionsPane.SUCCESSFUL);
+		options_pane.resetFileEntry();
+		build_monitor.reset();
+		import_monitor.reset();
+		if(Configuration.getMode() >= THRESHOLD) {
+		    control_pane.add(button_pane, BorderLayout.SOUTH);
+		    card_layout.show(main_pane, CONTROL);
+		}
+	    }
+	    // Otherwise its completed import but still got build to go
+	}
+	else {
+	    processing = false;
+	    cancel_button.setEnabled(false);
+	    build_button.setEnabled(true);
+	    // The build may have failed, but a previous index may still be in place
+	    //preview_button.setEnabled(Gatherer.c_man.built());
+
+	    simple_build_button.setEnabled(true);
+	    simple_cancel_button.setEnabled(false);
+	    //simple_preview_button.setEnabled(Gatherer.c_man.built());
+	    if(event.getStatus() == GShell.CANCELLED) {
+		document.setSpecialCharacter(OptionsPane.CANCELLED);
+	    }
+	    else {
+		document.setSpecialCharacter(OptionsPane.UNSUCCESSFUL);
+	    }
+	    options_pane.resetFileEntry();
+	    if(Configuration.getMode() >= THRESHOLD) {
+		control_pane.add(button_pane, BorderLayout.SOUTH);
+		card_layout.show(main_pane, CONTROL);
+	    }
+	}
+    }
+
+
+    /** This method is invoked at any time there has been a significant change in the collection, such as saving, loading or creating.
+     * @param ready A <strong>boolean</strong> indicating if a collection is currently available.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.CollectionManager
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.gui.OptionsPane
+     */
+    public void refresh(int refresh_reason, boolean ready)
+    {
+	if (Gatherer.c_man == null || !ready) {
+	    return;
+	}
+	Collection current_collection = Gatherer.c_man.getCollection();
+	if (current_collection != previous_collection && !Gatherer.c_man.isImporting()) {
+	    this.options_pane = new OptionsPane(current_collection.import_options, current_collection.build_options);
+	    if (previous_collection != null) {
+		// clear the log
+		Document log_document = log_textarea.getDocument();
+		if (log_document instanceof AppendLineOnlyFileDocument) {
+		    ((AppendLineOnlyFileDocument) log_document).destroy();
+		}
+	    }
+	    previous_collection = current_collection;
+	   
+	} 
+	// If we are in simple mode, we have to rebuild the simple arguments list
+	if(Configuration.getMode() < THRESHOLD) {
+	    souter_panel.remove(sargument_configuration_scroll);
+	    souter_panel.remove(sinner_panel);
+	    sargument_configuration_panel = options_pane.buildImport(null);
+	    sargument_configuration_panel = options_pane.buildBuild(sargument_configuration_panel);
+	    sargument_configuration_scroll = new JScrollPane(sargument_configuration_panel);
+	    sargument_configuration_scroll.setPreferredSize(ARGUMENT_SIZE);
+	    souter_panel.add(sargument_configuration_scroll);
+	    souter_panel.add(sinner_panel);
+	}
+	tree.valueChanged(null);
+
+	// Validate the preview button
+	if (Gatherer.c_man.built() && Configuration.library_url != null) {
+	    preview_button.setEnabled(true);
+	    simple_preview_button.setEnabled(true);
+	}
+	else {
+	    preview_button.setEnabled(false);
+	    simple_preview_button.setEnabled(false);
+	}
+    }
+
+
+    private void configureHomeURL() {	
+	// set up the home page for the current collection
+	Collection current_collection = Gatherer.c_man.getCollection();
+	String site = Configuration.site_name; // for GS3 colls
+	String extra_args = "";
+	SearchTypeManager st_man = current_collection.cdm.searchtype_manager;
+	if (!Gatherer.GS3 && st_man.isSearchTypeEnabled()) {
+	    if (st_man.isMGPP()) {
+		extra_args = "&ct=1";
+	    } else if (st_man.isLucene()) {
+		extra_args = "&ct=2";
+	    }
+	    // we need some more args on the url
+	    String search_types = st_man.getSearchTypes();
+	    if (search_types.equals("")) {
+		extra_args += "&qt=0&qto=3";
+	    } else if (search_types.equals("plain")) {
+		extra_args += "&qt=0&qto=1";
+	    } else if (search_types.equals("form")) {
+		extra_args += "&qt=1&qto=2";
+	    } else if (search_types.equals("plain,form")) {
+		extra_args += "&qt=0&qto=3";
+	    } else if (search_types.equals("form,plain")) {
+		extra_args += "&qt=1&qto=3";
+	    }
+	}
+	// Remember to add unique timestamp
+	if (Gatherer.GS3) {
+	    homepage = Configuration.library_url.toString() + Configuration.getServletPath()+ "?a=p&sa=about&c=" + current_collection.getName()+"&l="+Configuration.getLanguage(); //+extra_args + StaticStrings.TIMESTAMP_ARGUMENT + System.currentTimeMillis();
+	} else {
+	    homepage = Configuration.library_url.toString() + "?a=p&p=about&c=" + current_collection.getName()+"&l="+Configuration.getLanguage()+extra_args + StaticStrings.TIMESTAMP_ARGUMENT + System.currentTimeMillis();
+	}
+    }
+    
+
+    /** This class serves as the listener for actions on the build button. */
+    private class BuildButtonListener
+	implements ActionListener {
+	/**
+	 * This method checks to see what needs to be done for a build, and starts the process off.
+	 * A complete build proccess is started by {@link CollectionManager.importCollection()}, which then imports and builds the collection.
+	 * However, this doesn't always happen, as sometimes an incremental build will do :-)
+	 * @param event An <strong>ActionEvent</strong> who, thanks to the power of object oriented programming, we don't give two hoots about.
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.collection.CollectionManager
+	 * @see org.greenstone.gatherer.gui.BuildOptions
+	 * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    Collection collection = Gatherer.c_man.getCollection();
+	    String collection_name = collection.getName();
+	    if(!collection.getMetadataChanged() && !collection.getFilesChanged() && isIncremental()) {
+		//Only design options have changes, and we want to be smart in the way we handle them.
+		int rebuildTypeRequired = CollectionDesignManager.getRebuildTypeRequired();
+		if(rebuildTypeRequired == CollectionDesignManager.UPDATE_COLLECT_CFG && Gatherer.isGsdlRemote) {
+		    //Special case when the GLI is an applet: Upload the collect.cfg to server 
+		    //This is already handled by CollectionDesignManager.save()
+		    DebugStream.println("Just want to upload collect.cfg, but this has already been done");
+		}
+		else if(rebuildTypeRequired == CollectionDesignManager.BUILDCOL) {
+
+		    //Just run the buildcol command.
+		    DebugStream.println("Just want to run buildcol.pl");
+		    prepareForBuild();
+		    Gatherer.c_man.buildCollection(isIncremental());
+		}
+		else if(rebuildTypeRequired == CollectionDesignManager.ALL) {
+		    //Do a usual build.
+		    DebugStream.println("Want to do a complete build");
+		    prepareForBuild();
+		    Gatherer.c_man.importCollection(); //This starts the building process.
+		}
+		else {
+		    //Nothing at all needs doing.
+		    //This is bad HCI. Maybe should disable the build button in this situation?
+		    System.err.println("Want to build the collection but nothing needs doing.");
+		}
+	    }
+	    else {
+		//Do a complete build.
+		prepareForBuild();
+		Gatherer.c_man.importCollection(); //This starts the building process.
+	    }
+	    //Re-setting the rebuildTypeRequired is handled by CollectionManager.processComplete(GShellEvent)
+	}
+
+	/**
+	 * This does some stuff that is needed before a collection can be built.
+	 * For example, buttons are enabled/disabled, and certain flags are set.
+	 * This is called from {@link actionPerformed(ActionEvent)}
+	 */
+	private void prepareForBuild() {
+	    // First we force the build options to be updated if we haven't already.
+	    tree.valueChanged(null);
+	    
+	    // Remember that for lower thresholds the above doesn't work, so try this instead
+	    if(Configuration.getMode() < THRESHOLD) {
+		options_pane.update(sargument_configuration_panel);
+	    }
+	    
+	    // Now go about building.
+	    build_button.setEnabled(false);
+	    cancel_button.setEnabled(true);
+	    preview_button.setEnabled(false);
+	    
+	    simple_build_button.setEnabled(false);
+	    simple_cancel_button.setEnabled(true);
+	    simple_preview_button.setEnabled(false);
+	    
+	    document = options_pane.createNewLogDocument();
+	    log_textarea.setDocument(document);
+	    options_pane.log_textarea.setDocument(document);
+	    // Change the view.
+	    processing = true;
+	    if(Configuration.getMode() >= THRESHOLD) {
+		progress_pane.add(button_pane, BorderLayout.SOUTH);
+		card_layout.show(main_pane, PROGRESS);
+	    }
+	    // Reset the stop flag in all the process monitors, just incase.
+	    ((GBuildProgressMonitor)build_monitor).reset();
+	    import_monitor.setStop(false);
+	}
+    }
+    
+    
+    /** This class serves as the listener for actions on the cancel button. */
+    private class CancelButtonListener
+	implements ActionListener {
+	/** This method attempts to cancel the current GShell task. It does this by first telling CollectionManager not to carry out any further action. This it turn tells the GShell to break from the current job immediately, without waiting for the processEnded message, and then kills the thread in an attempt to stop the process. The results of such an action are debatable.
+	 * @param event An <strong>ActionEvent</strong> who, thanks to the power of object oriented programming, we don't give two hoots about.
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.collection.CollectionManager
+	 * @see org.greenstone.gatherer.gui.BuildOptions
+	 * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    build_button.setEnabled(false);
+	    cancel_button.setEnabled(false);
+	    preview_button.setEnabled(false);
+
+	    simple_build_button.setEnabled(false);
+	    simple_cancel_button.setEnabled(false);
+	    simple_preview_button.setEnabled(false);
+
+	    processing = false;
+	    document.setSpecialCharacter(OptionsPane.CANCELLED);
+	    if(Configuration.getMode() >= THRESHOLD) {
+		control_pane.add(button_pane, BorderLayout.SOUTH);
+		card_layout.show(main_pane, CONTROL);
+	    }
+	    // Set the stop flag in all the process monitor.
+	    import_monitor.setStop(true);
+	    import_monitor.reset();
+	    build_monitor.setStop(true);
+	    build_monitor.reset();
+
+	    // Remove the collection lock.
+	    //Gatherer.g_man.lockCollection(false, false);
+	}
+    }
+
+    private class PreviewButtonListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent event) {
+	    Gatherer.c_man.getCollection().cdm.save(); // save the config file just in case
+	    configureHomeURL();
+	    // check that the local library server is stil running  - doesn't do anything if its not supposed to be running
+	    LocalLibraryServer.checkServerRunning();
+	    
+	    Gatherer.spawnBrowser(homepage);
+
+	}
+    }
+    
+    /** The OptionTree is simply a tree structure that has a root node labelled "Options" and then has a child node for each available options screen. These screens are either combinations of the available import and build arguments, or a message log detailing the shell processes progress. */
+    private class OptionTree
+	extends JTree
+	implements TreeSelectionListener {
+	/** The model behind the tree. */
+	private DefaultTreeModel model = null;
+	/** The previous options view displayed, which is sometimes needed to refresh properly. */
+	private JPanel previous_pane = null;
+	/** The node corresponding to the building options view. */
+	private OptionTreeNode building  = null;
+	/** The node corresponding to the importing options view. */
+	private OptionTreeNode importing = null;
+	/** The node corresponding to the log view. */
+	private OptionTreeNode log       = null;
+	/** The root node of the options tree, which has no associated options view. */
+	private OptionTreeNode options   = null;
+	/** Constructor.
+	 * @see org.greenstone.gatherer.gui.CreatePane.OptionTreeNode
+	 */
+	public OptionTree() {
+	    super();
+
+	    ToolTipManager.sharedInstance().registerComponent(this);
+	    addTreeSelectionListener(this);
+	    getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+	    setCellRenderer(new ToolTipTreeCellRenderer());
+	    setRootVisible(false);
+	    setToggleClickCount(1);
+
+	    // Create tree.
+	    building = new OptionTreeNode(Dictionary.get("CreatePane.Build"));
+	    building.setToolTipText(Dictionary.get("CreatePane.Build_Tooltip"));
+	    importing = new OptionTreeNode(Dictionary.get("CreatePane.Import"));
+	    importing.setToolTipText(Dictionary.get("CreatePane.Import_Tooltip"));
+	    log = new OptionTreeNode(Dictionary.get("CreatePane.Log"));
+	    log.setToolTipText(Dictionary.get("CreatePane.Log_Tooltip"));
+	    options = new OptionTreeNode(Dictionary.get("CreatePane.Options"));
+
+	    model = new DefaultTreeModel(options);
+	    setModel(model);
+	    model.insertNodeInto(importing, options, 0);
+	    model.insertNodeInto(building, options, 1);
+	    model.insertNodeInto(log, options, 2);
+	    // Expand the root node
+	    expandPath(new TreePath(options));
+	}
+	/** Any implementation of TreeSelectionListener must include this method to allow this listener to know when the selection has changed. Here we swap the options view depending on the selected OptionTreeNode.
+	 * @param event A <Strong>TreeSelectionEvent</strong> which contains all the information garnered when the event occured.
+	 * @see org.greenstone.gatherer.gui.CreatePane.OptionTreeNode
+	 */
+	public void valueChanged(TreeSelectionEvent event) {
+	    TreePath path = null;
+	    OptionTreeNode node = null;
+	    //if(event != null) {
+	    //path = event.getPath();
+	    path = getSelectionPath();
+	    if(path != null) {
+		node = (OptionTreeNode)path.getLastPathComponent();
+	    }
+				//}
+	    if(previous_pane != null) {
+		//target_pane.remove(previous_pane);
+		options_pane.update(previous_pane);
+		if(scroll_pane != null) {
+		    right.remove(scroll_pane);
+		}
+		else {
+		    right.remove(previous_pane);
+		}
+		previous_pane = null;
+		scroll_pane = null;
+	    }
+	    if(node != null && node.equals(building)) {
+		previous_pane = options_pane.buildBuild(null);
+		scroll_pane = new JScrollPane(previous_pane);
+		right.add(scroll_pane, BorderLayout.CENTER);
+		//target_pane.add(previous_pane, BorderLayout.CENTER);
+	    }
+	    else if(node != null && node.equals(importing)) {
+		previous_pane = options_pane.buildImport(null);
+		scroll_pane = new JScrollPane(previous_pane);
+		right.add(scroll_pane, BorderLayout.CENTER);
+		//target_pane.add(previous_pane, BorderLayout.CENTER);
+	    }
+	    else {
+		if (options_pane != null) {
+		    previous_pane = options_pane.buildLog();
+		    right.add(previous_pane, BorderLayout.CENTER);
+		    right.updateUI(); // we need to repaint the log pane, cos it hasn't changed since last time
+		    ///ystem.err.println("I've added the log back to the right pane again.");
+		    //target_pane.add(previous_pane, BorderLayout.CENTER);
+		}
+	    }
+				//scroll_pane.setViewportView(previous_pane);
+	    //previous_pane.validate();
+	    right.validate();
+	    //System.err.println("Current pane size: " + previous_pane.getSize());
+	    //System.err.println("While its preferred size is: " + previous_pane.getPreferredSize());
+	}
+    }
+
+    /** The <strong>OptionTree</strong> is built from these nodes, each of which has several methods used in creating the option panes.
+     */
+    private class OptionTreeNode
+	extends DefaultMutableTreeNode
+	implements Comparable {
+	/** The text label given to this node in the tree. */
+	private String title = null;
+	/** a tool tip to be used for this node in the tree */
+	private String tool_tip = null;
+	/** Constructor.
+	 * @param title The <strong>String</strong> which serves as this nodes title.
+	 */
+	public OptionTreeNode(String title) {
+	    super();
+	    this.title = title;
+	}
+
+	/** This method compares two nodes for ordering.
+	 * @param obj The <strong>Object</strong> to compare to.
+	 * @return An <strong>int</strong> of one of the possible values -1, 0 or 1 indicating if this node is less than, equal to or greater than the target node respectively.
+	 */
+	public int compareTo(Object obj) {
+	    return title.compareTo(obj.toString());
+	}
+
+	/** This method checks if two nodes are equivalent.
+	 * @param obj The <strong>Object</strong> to be tested against.
+	 * @return A <strong>boolean</strong> which is <i>true</i> if the objects are equal, <i>false</i> otherwise.
+	 */
+	public boolean equals(Object obj) {
+	    if(compareTo(obj) == 0) {
+		return true;
+	    }
+	    return false;
+	}
+
+	/** get the tool tip */
+	public String getToolTipText() {
+	    return tool_tip;
+	}
+
+	/** set the tool tip */
+	public void setToolTipText(String tip) {
+	    tool_tip = tip;
+	}
+	
+	/** Method to translate this node into a textual representation.
+	 * @return A <strong>String</strong> which in this case is the title.
+	 */
+	public String toString() {
+	    return title;
+	}
+    }
+
+    // Adds tooltips to the tree nodes
+    private class ToolTipTreeCellRenderer
+	extends DefaultTreeCellRenderer {
+
+	public Component getTreeCellRendererComponent(JTree tree,
+						      Object value,
+						      boolean sel,
+						      boolean expanded,
+						      boolean leaf,
+						      int row,
+						      boolean hasFocus) {
+
+	    super.getTreeCellRendererComponent(tree, value, sel,
+					       expanded, leaf, row,
+					       hasFocus);
+	    if (value instanceof OptionTreeNode) {
+		String tip = ((OptionTreeNode) value).getToolTipText();
+		if (tip != null) {
+		    setToolTipText(tip);
+		}
+	    }
+	    return this;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DeleteCollectionPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DeleteCollectionPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DeleteCollectionPrompt.java	(revision 21112)
@@ -0,0 +1,328 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.file.WorkspaceTree;  // !!! Don't like this here
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/** This class provides the functionality to delete current collections from the GSDLHOME/collect/ directory. The user chooses the collection from a list, where each entry also displays details about itself, confirms the delete of a collection by checking a checkbox then presses the ok button to actually delete the collection.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class DeleteCollectionPrompt
+    extends ModalDialog {
+    /** The currently selected collection for deletion. */
+    private BasicCollectionConfiguration collection = null;
+    /** The model behind the list. */
+    private DefaultListModel list_model = null;
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private DeleteCollectionPrompt prompt = null;
+    /** The close button, which exits the prompt without deleting anything. */
+    private JButton close_button = null;
+    /** The ok button which causes the selected collection to be deleted. */
+    private JButton ok_button = null;
+    /** The confirmation check box. */
+    private JCheckBox confirmation = null;
+    /** The label above details. */
+    private JLabel details_label = null;
+    /** The label above the list. */
+    private JLabel list_label = null;
+    /** The list of available collections. */
+    private JList list = null;
+    /** The text area used to display details about the collection selected. */
+    private JTextArea details = null;
+    /** A string array used to pass arguments to the phrase retrieval method. */
+    private String args[] = null;
+
+    private boolean current_coll_deleted = false;
+    /** The size of the delete prompt screen. */
+    public static final Dimension SIZE = new Dimension(500, 500);
+
+    /** Constructor.
+     */
+    public DeleteCollectionPrompt() {
+	super(Gatherer.g_man);
+	close_button = new GLIButton();
+	close_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(close_button, "General.Close", "General.Close_Tooltip");
+	confirmation = new JCheckBox();
+	Dictionary.setText(confirmation, "DeleteCollectionPrompt.Confirm_Delete");
+	details = new JTextArea();
+	details.setEditable(false);
+	Dictionary.setText(details, "DeleteCollectionPrompt.No_Collection");
+	details_label = new JLabel();
+	Dictionary.setText(details_label, "DeleteCollectionPrompt.Collection_Details");
+	list = new JList();
+	list_label = new JLabel();
+	Dictionary.setText(list_label, "DeleteCollectionPrompt.Collection_List");
+	list_model = new DefaultListModel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_D);
+	Dictionary.setBoth(ok_button, "DeleteCollectionPrompt.Delete", "DeleteCollectionPrompt.Delete_Tooltip");
+	prompt = this;
+	setModal(true);
+	setSize(SIZE);
+	Dictionary.setText(this, "DeleteCollectionPrompt.Title");
+
+	setJMenuBar(new SimpleMenuBar("0")); // need to find an appropriate help page to open at
+	close_button.addActionListener(new CloseButtonListener());
+	confirmation.addActionListener(new ConfirmationCheckBoxListener());
+	confirmation.setEnabled(false);
+	confirmation.setSelected(false);
+	list.addListSelectionListener(new CollectionListListener());
+	list.clearSelection();
+	list.getSelectionModel().setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	list.setModel(list_model);
+	ok_button.addActionListener(new OKButtonListener());
+	ok_button.setEnabled(false);
+	scanForCollections();
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	list_model.clear();
+	list_model = null;
+	close_button = null;
+	confirmation = null;
+	details = null;
+	details_label = null;
+	list = null;
+	ok_button = null;
+	prompt = null;
+    }
+
+    /** This method causes the modal prompt to be displayed. 
+     * returns true if it has deleted the collection that is currently open */
+    public boolean display() {
+	// Central pane
+	JPanel list_pane = new JPanel(new BorderLayout());
+	list_pane.add(list_label, BorderLayout.NORTH);
+	list_pane.add(new JScrollPane(list), BorderLayout.CENTER);
+	list_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 5, 0));
+
+	JPanel details_pane = new JPanel(new BorderLayout());
+	details_pane.add(details_label, BorderLayout.NORTH);
+	details_pane.add(new JScrollPane(details), BorderLayout.CENTER);
+	details_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel central_pane = new JPanel(new GridLayout(2, 1));
+	central_pane.add(list_pane);
+	central_pane.add(details_pane);
+	central_pane.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
+
+	// Lower pane
+	JPanel confirmation_pane = new JPanel(new BorderLayout());
+	confirmation_pane.add(confirmation, BorderLayout.CENTER);
+	confirmation_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+
+	JPanel button_pane = new JPanel(new GridLayout(1, 2));
+	button_pane.add(ok_button);
+	button_pane.add(close_button);
+	button_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel lower_pane = new JPanel(new BorderLayout());
+	lower_pane.add(confirmation_pane, BorderLayout.NORTH);
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(0, 5, 5, 5));
+
+	// Final.
+	JPanel content_pane = (JPanel)this.getContentPane();
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(central_pane, BorderLayout.CENTER);
+	content_pane.add(lower_pane, BorderLayout.SOUTH);
+
+	// Center and display.
+	Dimension screen_size = Configuration.screen_size;
+	this.setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	this.setVisible(true); // blocks until the dialog is killed
+	return current_coll_deleted;
+    }
+
+
+    /** Shows a delete complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully deleted.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    public void resultPrompt(boolean success) {
+	args = new String[1];
+	args[0] = collection.getName();
+	if (success) {
+	    JOptionPane.showMessageDialog(prompt,Dictionary.get("DeleteCollectionPrompt.Successful_Delete", args),Dictionary.get("DeleteCollectionPrompt.Successful_Title"),JOptionPane.INFORMATION_MESSAGE);
+	}
+	else {
+	    JOptionPane.showMessageDialog(prompt,Dictionary.get("DeleteCollectionPrompt.Failed_Delete", args),Dictionary.get("DeleteCollectionPrompt.Failed_Title"),JOptionPane.WARNING_MESSAGE);
+	}
+    }
+
+    /** Method to scan the collect directory retrieving and reloading each collection it finds, while building the list of known collections.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.util.ArrayTools
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void scanForCollections() {
+	// Start at the collect dir.
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	if (collect_directory.exists()) {
+	    // Now for each child directory see if it contains a .col file and
+	    // if so try to load it..
+	    File collections[] = collect_directory.listFiles();
+	    ArrayTools.sort(collections);
+	    for(int i = 0; collections != null && i < collections.length; i++) {
+		if(collections[i].isDirectory() && !collections[i].getName().equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+		    File config_file = new File(collections[i], Utility.CONFIG_FILE);
+		    if (config_file.exists()) {
+			BasicCollectionConfiguration config = new BasicCollectionConfiguration(config_file);
+			list_model.addElement(config);
+			config = null;
+		    }
+		}
+	    }
+	}
+	// Otherwise the collect directory doesn't actually exist, so there ain't much we can do.
+    }
+
+    /** A button listener implementation, which listens for actions on the close button and disposes of the dialog when detected. */
+    private class CloseButtonListener 
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    // Done
+	    prompt.dispose();
+	}
+    }
+
+    /** This private class listens for selection events in from the list and then displays the appropriate details for that collection.
+     */
+    private class CollectionListListener 
+	implements ListSelectionListener {
+	/** Any implementation of ListSelectionListener must include this method so we can be informed when the list selection changes.
+	 * @param  event a <strong>ListSelectionEvent</strong> containing all the relevant information garnered from the event itself
+	 */
+	public void valueChanged(ListSelectionEvent event) {
+	    ok_button.setEnabled(false);
+	    confirmation.setSelected(false);
+	    if(!list.isSelectionEmpty()) {
+		confirmation.setEnabled(true);
+		collection = (BasicCollectionConfiguration) list.getSelectedValue();
+		args = new String[3];
+		args[0] = collection.getCreator();
+		args[1] = collection.getMaintainer();
+		args[2] = collection.getDescription();
+		Dictionary.setText(details, "DeleteCollectionPrompt.Details", args);
+		details.setCaretPosition(0);
+	    }
+	    else {
+		confirmation.setEnabled(false);
+		Dictionary.setText(details, "DeleteCollectionPrompt.No_Collection");
+	    }
+	}
+    }
+
+    /** A check box listener so we can tell if the user has confirmed the deletion */
+    private class ConfirmationCheckBoxListener
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    // OK button is only enabled if the confirmation check box is ticked
+	    ok_button.setEnabled(confirmation.isSelected());
+	}
+    }
+
+    /** The OK button listener implementation. */
+    private class OKButtonListener 
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 * @see org.greenstone.gatherer.Configuration
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.util.Utility
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    // Delete the selected collection.
+	    // !! This should all go into CollectionManager !!
+
+	    // First we must release it from the local library
+	    if (LocalLibraryServer.isRunning() == true) {
+		LocalLibraryServer.releaseCollection(collection.getShortName());
+	    }
+
+	    if (Gatherer.isGsdlRemote) {
+		RemoteGreenstoneServer.deleteCollection(collection.getShortName());
+	    }
+
+	    if (Gatherer.c_man.deleteCollection(collection.getShortName())) {
+		// Refresh the collections shown in the workspace tree
+		Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.LIBRARY_CONTENTS_CHANGED);
+
+		if (Gatherer.c_man.getCollection() != null && collection.getShortName().equals(Gatherer.c_man.getCollection().getName())) {
+		    current_coll_deleted = true;
+		}
+		list_model.removeElement(collection);
+
+		resultPrompt(true);
+		Dictionary.setText(details, "DeleteCollectionPrompt.No_Collection");
+		confirmation.setEnabled(false);
+		confirmation.setSelected(false);
+		ok_button.setEnabled(false);
+		collection = null;
+	    }
+	    else {
+		resultPrompt(false);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DesignPane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DesignPane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DesignPane.java	(revision 21112)
@@ -0,0 +1,125 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+/** This serves as the Collection configuration pane and came in two flavors.<BR>
+ * The lower tech version has now been made redundant.<BR>
+ * The higher tech, and finally implemented, version uses a tree to allow a user to navigate through the various regions of the config file and further allows them to interactively design and modify such gsdl elements as classifiers and plugins.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.2
+ */
+public class DesignPane 
+    extends JPanel {
+    /** The collection manager is responsible for parsing in, and allowing the editing of, a single collections configuration file. */
+    private CollectionDesignManager cdm = null;
+
+    /** The constructor. */
+    public DesignPane() {
+    }
+
+    /** Called to cause the components to lay themselves out and be displayed.
+     */
+    public void display() {
+	// Layout
+	this.setLayout(new BorderLayout());
+    }
+
+    public void gainFocus() {
+	if (cdm == null && Gatherer.c_man.ready()) {
+	    // Retrieve the new config manager.
+	    cdm = Gatherer.c_man.getCollection().cdm;
+	    // Display it.
+	    cdm.display(this);
+	}
+	if (cdm != null) {
+	    cdm.gainFocus();
+	}
+    }
+
+    public ArrayList getLanguageCodes() {
+	return cdm.language_manager.getLanguageCodes();
+    }
+
+    public boolean canSave() {
+	return cdm.canSave();
+    }
+
+    public void loseFocus() {
+	if (cdm != null) {
+	    cdm.save(false);
+	}
+    }
+
+    /** Called whenever the detail mode changes to ensure the filters are at an appropriate level (ie only editable by those that understand regular expression matching)
+     * @param mode the mode level as an int
+     */
+    public void modeChanged(int mode) {
+	if (cdm != null) {
+	    cdm.modeChanged(mode);
+	}
+    }
+
+    /** This method is called whenever the state of the current collection changes.
+     */
+    public void refresh(int refresh_reason, boolean ready)
+    {
+	if (ready) {
+	    // Retrieve the new config manager.
+	    cdm = Gatherer.c_man.getCollection().cdm;
+	    // Display it.
+	    cdm.display(this);
+	}
+	else {
+	    if (cdm != null) {
+		cdm.destroy();
+		cdm = null;
+	    }
+	}
+    }
+
+    public void saveConfiguration() {
+	if (cdm != null) {
+	    cdm.save(true);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadPane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadPane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadPane.java	(revision 21112)
@@ -0,0 +1,386 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.WGet;
+import org.greenstone.gatherer.file.WorkspaceTree;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.1
+ */
+public class DownloadPane
+    extends JPanel {
+
+    /** The threshold for when the advanced options get added */
+    static private final int THRESHOLD = Configuration.SYSTEMS_MODE;
+
+    private boolean download_button_enabled = false;
+    private boolean ready = false;
+    private int current_mode;
+    private JButton clear_cache_button;
+    private JButton download_button;
+    private JCheckBox higher_checkbox;
+    private JCheckBox requisite_checkbox;
+    private JCheckBox same_host_checkbox;
+    private JScrollPane list_scroll;
+    private JComboBox depth_combobox;
+    private JPanel options_pane;
+    private JTextField url_field;
+    private Vector depth_model;
+    private WGet getter;
+
+    public DownloadPane() {
+	super();
+	// Create a new wget thread
+        getter = new WGet();
+        getter.start();
+	// And retrieve its list of pending jobs
+	list_scroll = getter.getDownloadJobList();
+
+	current_mode = Configuration.getMode();
+	// String user_dir = System.getProperty("user.dir"); // ****
+	// System.err.println("**** user dir = "+user_dir);
+	// Create
+	JPanel edit_pane = new JPanel();
+	JPanel details_pane = new JPanel();
+	options_pane = new JPanel();
+
+	JPanel url_pane = new JPanel();
+	JLabel url_label = new JLabel();
+	Dictionary.registerText(url_label, "Mirroring.Source_URL");
+	url_field = new JTextField();
+	Dictionary.registerTooltip(url_field, "Mirroring.Source_URL_Tooltip");
+
+	depth_model = new Vector();
+	depth_model.add(new DepthEntry(0, Dictionary.get("Mirroring.Download_Depth.Zero")));
+	/* @todo - add to dictionary */
+	depth_model.add(new DepthEntry(1, String.valueOf(1)));
+	/* @todo - add to dictionary */
+	depth_model.add(new DepthEntry(2, String.valueOf(2)));
+	/* @todo - add to dictionary */
+	depth_model.add(new DepthEntry(3, String.valueOf(3)));
+	/* @todo - add to dictionary */
+	depth_model.add(new DepthEntry(4, String.valueOf(4)));
+	/* @todo - add to dictionary */
+	depth_model.add(new DepthEntry(5, String.valueOf(5)));
+	depth_model.add(new DepthEntry(-1, Dictionary.get("Mirroring.Download_Depth.Unlimited")));
+	JPanel depth_pane = new JPanel();
+	JLabel depth_label = new JLabel();
+	Dictionary.registerText(depth_label, "Mirroring.Download_Depth");
+	depth_combobox = new JComboBox(depth_model);
+	Dictionary.registerTooltip(depth_combobox, "Mirroring.Download_Depth_Tooltip");
+
+	requisite_checkbox = new JCheckBox();
+	Dictionary.registerText(requisite_checkbox, "Mirroring.Download_Embedded");
+	
+
+	higher_checkbox = new JCheckBox();
+	Dictionary.registerText(higher_checkbox, "Mirroring.Higher_Directories");
+	higher_checkbox.addActionListener(new CheckboxClickListener());
+	
+	same_host_checkbox = new JCheckBox();
+	same_host_checkbox.setSelected(true);
+	Dictionary.registerText(same_host_checkbox, "Mirroring.Same_Host");
+
+	JPanel button_pane = new JPanel();
+
+	JButton preferences_button = new GLIButton();
+	preferences_button.setEnabled(true);
+	preferences_button.setMnemonic(KeyEvent.VK_P);
+	Dictionary.registerBoth(preferences_button, "Mirroring.Preferences", "Mirroring.Preferences_Tooltip");
+	
+	clear_cache_button = new GLIButton();
+	clear_cache_button.setEnabled(true);
+	clear_cache_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(clear_cache_button, "Mirroring.ClearCache", "Mirroring.ClearCache_Tooltip");
+
+	download_button = new GLIButton();
+	download_button.setEnabled(true);
+	download_button.setMnemonic(KeyEvent.VK_D);
+	Dictionary.registerBoth(download_button, "Mirroring.Download", "Mirroring.Download_Tooltip");
+	// Connect
+	clear_cache_button.addActionListener(new ClearCacheListener());
+	download_button.addActionListener(new DownloadButtonListener());
+	preferences_button.addActionListener(new PreferencesButtonActionListener());
+
+	// Layout
+	url_pane.setLayout(new BorderLayout(5,0));
+	url_pane.add(url_label, BorderLayout.WEST);
+	url_pane.add(url_field, BorderLayout.CENTER);
+
+	depth_pane.setLayout(new GridLayout(1,3));
+	depth_pane.add(depth_label);
+	depth_pane.add(new JPanel());
+	depth_pane.add(depth_combobox);
+
+	details_pane.setLayout(new GridLayout(2,1,0,5));
+	details_pane.setBorder(BorderFactory.createEmptyBorder(2,2,5,2));
+	details_pane.add(url_pane);
+	details_pane.add(depth_pane);
+
+	options_pane.setLayout(new GridLayout(3,1,0,5));
+	
+	options_pane.add(higher_checkbox);
+	options_pane.add(same_host_checkbox);
+	if (current_mode >= THRESHOLD) {
+	    options_pane.add(requisite_checkbox);
+	}
+
+	button_pane.setLayout(new GridLayout(3,1));
+	button_pane.add(preferences_button);
+	button_pane.add(clear_cache_button);
+	button_pane.add(download_button);
+
+	edit_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(2,0,0,0), BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder(Dictionary.get("Mirroring.Download_Controls")), BorderFactory.createEmptyBorder(2,2,2,2))));
+	edit_pane.setLayout(new BorderLayout());
+	edit_pane.add(details_pane, BorderLayout.NORTH);
+	edit_pane.add(button_pane, BorderLayout.EAST);
+	edit_pane.add(options_pane, BorderLayout.CENTER);
+
+	setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	setLayout(new BorderLayout());
+	add(edit_pane, BorderLayout.NORTH);
+	add(list_scroll, BorderLayout.CENTER);
+    }
+
+    public void afterDisplay() {
+	ready = true;
+    }
+
+
+    /** This method is called whenever the Download pane is brought into focus and is a good time to display a warning message if WGet is not available or of an older, problematic, version.
+     */
+    public void gainFocus() {
+	if(!ready) {
+	    return;
+	}
+	// Lets see what warning message we should display, if any.
+	String wget_version_str = Configuration.getWGetVersion();
+	if(wget_version_str.equals(StaticStrings.NO_WGET_STR)) {
+	    // If there was no WGet available then downloading is disabled entirely
+	    download_button_enabled = false;
+	    // And we tell the user why.
+	    Gatherer.missingWGET();
+	}
+	else if(wget_version_str.equals(StaticStrings.WGET_OLD_STR)) {
+	    // Downloading is enabled
+	    download_button_enabled = true;
+	    // But we display a preventable warning message about the path problems.
+	    Gatherer.oldWGET();
+	}
+	// Otherwise version must be ok
+	else {
+	    download_button_enabled = true;
+	}
+	// It is also a good time to determine if the download should be enabled - ie if its allowed to be enabled and a valid URL is present in the field.
+	download_button.setEnabled(download_button_enabled);
+    }
+
+    public void modeChanged(int mode) {
+	int old_mode = current_mode;
+	current_mode = mode;
+	if (old_mode >= THRESHOLD && current_mode >= THRESHOLD) {
+	    return;
+	}
+	if (old_mode < THRESHOLD && current_mode < THRESHOLD) {
+	    return;
+	}
+	if (current_mode >= THRESHOLD) {
+	    options_pane.add(requisite_checkbox);
+	}
+	else {
+	    options_pane.remove(requisite_checkbox);
+	    requisite_checkbox.setSelected(false);
+	}
+    }
+
+
+    public void refresh(int refresh_reason, boolean ready)
+    {
+    }
+
+
+    /** A DepthEntry contains a depth value, as an int, and a representitive text string. This allows for a more meaningful appearance than '0' for instance. */
+    private class DepthEntry 
+	implements Comparable {
+	private int value;
+	private String text;
+	/** Default constructor. 
+	 * @param value the depth value as an int
+	 * @param text how this entry show represent itself as a String
+	 */
+	public DepthEntry(int value, String text) {
+	    this.text = text;
+	    this.value = value;
+	}
+	/** Determines the natural ordering of this DepthEntry and some other object
+	 * @param object the Object to test for ordering
+	 * @return >0 if the object is before this one, 0 if they are equal, or <0 if the object is after this one
+	 */
+	public int compareTo(Object object) {
+	    // If the object is a DepthEntry then this is easy pesy.
+	    if(object instanceof DepthEntry) {
+		int target_value = ((DepthEntry) object).getValue();
+		return value - target_value;
+	    }
+	    // If the object is a String, then we should try parsing an int from it
+	    if(object instanceof String) {
+		try {
+		    int target_value = Integer.parseInt((String)object);
+		    return value - target_value;
+		}
+		catch(Exception exception) {
+		}
+	    }
+	    // Otherwise we'll try a string comparison between our text and the toString of the object
+	    return text.compareTo(object.toString());
+	}
+
+	/** Determine if this DepthEntry is equal to some other object. Given that there is significant processing involved we push the resposiblility for this onto compareTo, so we only have the complex code once.
+	 * @param object the Object to test for equality
+	 * @return true if the object is equal to this depth entry, false otherwise
+	 */
+	public boolean equals(Object object) {
+	    return (compareTo(object) == 0);
+	}
+
+	public int getValue() {
+	    return value;
+	}
+
+	public String toString() {
+	    return text;
+	}
+    }
+
+    private class ClearCacheListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    // Retrieve the cache folder and delete it.
+	    Utility.delete(Gatherer.getGLIUserCacheDirectoryPath());
+	    // ...and refresh the node in the workspace tree to show it's all gone
+	    Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
+	}
+    }
+
+    private class CheckboxClickListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    if (higher_checkbox.isSelected()) {
+		same_host_checkbox.setEnabled(false);
+	    } else {
+		same_host_checkbox.setEnabled(true);
+	    }
+	}
+    }
+    
+    private class DownloadButtonListener
+	implements ActionListener {
+
+	public void actionPerformed(ActionEvent event) {
+	    // Retrieve the current url and confirm it is valid
+	    String url_str = url_field.getText();
+	    URL url = null;
+	    try {
+		url = new URL(url_str);
+	    }
+	    catch(MalformedURLException error) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Mirroring.Invalid_URL"), Dictionary.get("Mirroring.Invalid_URL_Title"), JOptionPane.ERROR_MESSAGE);
+	    }
+
+	    // Otherwise construct a new download job
+	    Object depth_object = depth_combobox.getSelectedItem();
+	    int depth_value = 0;
+	    if(depth_object instanceof DepthEntry) {
+		depth_value = ((DepthEntry)depth_object).getValue();
+	    }
+	    else if(depth_object instanceof String) {
+		String depth_string = (String) depth_object;
+		try {
+		    depth_value = Integer.parseInt(depth_string);
+		    // Create a new entry for the purposes of comparison, and if needs be, addition
+		    DepthEntry new_entry = new DepthEntry(depth_value, depth_string);
+		    // Try to select the correct item from the combobox
+		    depth_combobox.setSelectedItem(null);
+		    depth_combobox.setSelectedItem(new_entry);
+		    // We can see if that worked, and if it didn't add the entry
+		    if(depth_combobox.getSelectedItem() == null) {
+			depth_combobox.addItem(new_entry);
+			depth_combobox.setSelectedItem(new_entry);
+		    }
+		    new_entry = null;
+		}
+		catch(Exception exception) {
+		    /* @todo - add to dictionary */
+		    JOptionPane.showMessageDialog(Gatherer.g_man, "Mirroring.Mirror_Depth.Invalid_Depth", "Mirroring.Mirror_Depth.Invalid_Depth_Title", JOptionPane.ERROR_MESSAGE);
+		    return;
+		}
+		depth_string = null;
+	    }
+	    depth_object = null;
+    
+	    String destination_file_path = Gatherer.getGLIUserCacheDirectoryPath();
+	    (new File(destination_file_path)).mkdirs(); // If they aren't already
+
+	    if(url != null) {
+		getter.newDownloadJob(higher_checkbox.isSelected(), (higher_checkbox.isSelected()? false : !same_host_checkbox.isSelected()), !requisite_checkbox.isSelected(), url, depth_value, destination_file_path);
+	    }
+	}
+    }
+
+    private class PreferencesButtonActionListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    new Preferences(Preferences.CONNECTION_PREFS);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadProgressBar.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadProgressBar.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/DownloadProgressBar.java	(revision 21112)
@@ -0,0 +1,402 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.DownloadJob;
+import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
+
+
+public class DownloadProgressBar
+    extends JPanel 
+    implements ActionListener {
+
+    static final private Dimension MINIMUM_BUTTON_SIZE = new Dimension(100, 25);
+    static final private Dimension PROGRESS_BAR_SIZE = new Dimension(580,65);
+
+    private boolean simple = false;
+
+    private Dimension bar_size = new Dimension(520, 15);
+
+    private int current_action;
+    private int err_count;
+    private int file_count;
+    private int total_count;
+    private int warning_count;
+
+    private JLabel current_status;
+    private JLabel main_status;
+    private JLabel results_status;
+	 
+    private JPanel center_pane;
+    private JPanel inner_pane;
+
+    private JProgressBar progress;
+
+    private long file_size;
+    private long total_size;
+
+    private String current_url;
+    private String initial_url;
+
+    private DownloadJob owner;
+
+    public JButton stop_start_button;
+    public JButton log_button;
+    public JButton close_button;
+
+    public DownloadProgressBar(DownloadJob owner, String initial_url, boolean simple) {
+	this.owner = owner;
+	this.current_url = null;
+	this.err_count = 0;
+	this.initial_url = initial_url;
+	this.file_count = 0;
+	this.file_size = 0;
+	this.simple = simple;
+	this.total_count = 0;
+	this.total_size = 0;
+	this.warning_count = 0;
+
+	this.setLayout(new BorderLayout());
+	this.setBorder(BorderFactory.createBevelBorder(BevelBorder.RAISED));
+
+	this.current_action = DownloadJob.STOPPED;
+
+	inner_pane = new JPanel(new BorderLayout());
+	inner_pane.setBorder(BorderFactory.createEmptyBorder(2,2,2,2));
+
+	center_pane = new JPanel(new GridLayout(3,1));
+
+	main_status = new JLabel();
+
+	current_status = new JLabel();
+		  
+	results_status = new JLabel();
+
+	progress = new JProgressBar();
+	progress.setStringPainted(true);
+	progress.setMinimum(0);
+	progress.setMaximum(0);
+	progress.setEnabled(false);
+	Dictionary.setText(progress, "Mirroring.DownloadJob.Waiting");
+	inner_pane.add(progress, BorderLayout.CENTER);
+
+	center_pane.add(main_status);
+	center_pane.add(current_status);
+	center_pane.add(results_status);
+
+	JPanel button_pane = new JPanel();
+
+	stop_start_button = new GLIButton();
+	stop_start_button.addActionListener(this);
+	stop_start_button.addActionListener(owner);
+	stop_start_button.setMinimumSize(MINIMUM_BUTTON_SIZE);
+	stop_start_button.setMnemonic(KeyEvent.VK_P);
+	stop_start_button.setEnabled(true);
+	Dictionary.registerBoth(stop_start_button, "Mirroring.DownloadJob.Pause", "Mirroring.DownloadJob.Pause_Tooltip");
+
+	log_button = new GLIButton();
+	log_button.addActionListener(owner);
+	log_button.addActionListener(this);
+	log_button.setEnabled(false);
+	log_button.setMinimumSize(MINIMUM_BUTTON_SIZE);
+	log_button.setMnemonic(KeyEvent.VK_L);
+	Dictionary.registerBoth(log_button, "Mirroring.DownloadJob.Log", "Mirroring.DownloadJob.Log_Tooltip");
+
+	close_button = new GLIButton();
+	close_button.addActionListener(owner);
+	close_button.addActionListener(this);
+	close_button.setMinimumSize(MINIMUM_BUTTON_SIZE);
+	close_button.setMnemonic(KeyEvent.VK_C);
+	close_button.setEnabled(true);
+	Dictionary.registerBoth(close_button, "Mirroring.DownloadJob.Close", "Mirroring.DownloadJob.Close_Tooltip");
+
+	// Layout - or at least some of it
+
+	inner_pane.add(center_pane, BorderLayout.NORTH);
+
+	button_pane.setLayout(new GridLayout(3,1));
+	button_pane.add(stop_start_button);
+	button_pane.add(log_button);
+	button_pane.add(close_button);
+
+	this.add(inner_pane, BorderLayout.CENTER);
+	this.add(button_pane, BorderLayout.EAST);
+
+	// Make the labels, etc update.
+	refresh();
+    }
+
+    public void actionPerformed(ActionEvent event) {
+	Object source = event.getSource();
+	if(source == stop_start_button) {
+	    // If we are running, stop.
+	    if (current_action == DownloadJob.RUNNING) {
+		current_action = DownloadJob.STOPPED;
+		Dictionary.registerBoth(stop_start_button, "Mirroring.DownloadJob.Resume", "Mirroring.DownloadJob.Resume_Tooltip");
+		Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Stopped");
+		progress.setIndeterminate(false);
+	    } else {
+		current_action = DownloadJob.RUNNING;
+		// we are stopped, so restart
+		Dictionary.registerBoth(stop_start_button, "Mirroring.DownloadJob.Pause", "Mirroring.DownloadJob.Pause_Tooltip");
+		Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Progress");
+		progress.setIndeterminate(true);
+	    }
+	}
+	else if(source == close_button) {
+	    // If we are running, stop.
+	    if (current_action == DownloadJob.RUNNING) {
+		current_action = DownloadJob.STOPPED;
+		Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Stopped");
+		progress.setIndeterminate(false);
+	    }
+	}
+	else if(source == log_button) {
+	    LogDialog dialog = new LogDialog(owner.getLogDocument());
+	    dialog.setVisible(true);
+	    dialog = null;
+	}
+    }
+
+    /** This method is called when a new download is begun. The
+     * details of the download are updated and a new JProgressBar
+     * assigned to track the download.
+     * @param url The url String of the file that is being downloaded.
+     */
+    public void addDownload(String url) {
+	current_url = url;
+	file_size = 0;
+	refresh();
+    }
+
+    /** When the download of the current url is completed, this method
+     * is called to enlighten the DownloadProgressBar of this fact.
+     */
+    public void downloadComplete() {
+	current_url = null;
+	file_count++;
+	if(total_count < (file_count + err_count + warning_count)) {
+	    total_count = (file_count + err_count + warning_count);
+	}
+	progress.setValue(progress.getMaximum());
+	//Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Complete");
+	refresh();
+    }
+
+    public void downloadFailed() {
+	err_count++;
+	if(total_count < (file_count + err_count + warning_count)) {
+	    total_count = (file_count + err_count + warning_count);
+	}
+	refresh();
+    }
+
+    public void downloadWarning() {
+	warning_count++;
+	if(total_count < (file_count + err_count + warning_count)) {
+	    total_count = (file_count + err_count + warning_count);
+	}
+	refresh();
+    }
+
+    public Dimension getPreferredSize() {
+	return PROGRESS_BAR_SIZE;
+    }
+
+    /** When a link to be downloaded is located, the increaseTotalCount
+     * method is called. In this way the total count shows the most
+     * accurate remaining number of files to be downloaded.
+     */
+    public void increaseFileCount() {
+	total_count++;
+	refresh();
+    }
+
+    /** When a mirroring task is first initiated this function is called
+     * to set initial values for the variables if necessary and to 
+     * fiddle visual components such as the tool tip etc.
+     * @param reset A Boolean specifying whether the variables should be
+     * reset to zero.
+     */
+    public void mirrorBegun(boolean reset, boolean simple) {
+	if(reset) {
+	    this.file_count = 0;
+	    this.file_size = 0;
+	    this.total_count = 0;
+	    this.total_size = 0;
+	    this.err_count = 0;
+	    this.warning_count = 0;
+	}
+	current_action = DownloadJob.RUNNING;
+	stop_start_button.setEnabled(true);
+	log_button.setEnabled(true);
+	if(simple) {
+	    progress.setIndeterminate(true);
+	    Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Progress");
+	}
+    }	
+
+    /** Once a mirroring task is complete, is the DownloadJob returns from the
+     * native call but the status is still running, then this method
+     * is called to once again tinker with the pritty visual 
+     * components.
+     */
+    public void mirrorComplete() {
+	current_action = DownloadJob.COMPLETE;
+	current_url = null;
+	if(simple) {
+	    progress.setIndeterminate(false);
+	}
+	progress.setValue(progress.getMaximum());
+	Dictionary.setText(progress, "Mirroring.DownloadJob.Download_Complete");
+
+	stop_start_button.setEnabled(false);
+	this.updateUI();
+    }
+
+    /** When called this method updates the DownloadProgressBar to reflect
+     * the ammount of the current file downloaded.
+     */
+    public void updateProgress(long current, long expected) {
+	file_size = file_size + current;
+	if(!progress.isIndeterminate()) {
+	    // If current is zero, then this is the 'precall' before the
+	    // downloading actually starts.
+	    if(current == 0) {
+		// Remove the old progress bar, then deallocate it.
+		inner_pane.remove(progress);
+		progress = null;
+		if(expected == 0) {
+		    // We don't have a content length. This bar will go from 0 to 100 only!
+		    progress = new JProgressBar(0, 100);
+		}
+		else {
+		    // Assign a new progress bar of length expected content length.
+		    progress = new JProgressBar(0, (new Long(expected)).intValue());
+		}
+		progress.setEnabled(true);
+		// Add the new progress bar.
+		inner_pane.add(progress, BorderLayout.CENTER);
+		inner_pane.updateUI();
+	    }
+	    // Otherwise if expected is not zero move the progress bar and
+	    // update percent complete.
+	    else if (expected != 0) {
+		progress.setValue((new Long(file_size)).intValue());
+		int p_c = (new Double(progress.getPercentComplete() * 100)).intValue();
+		progress.setString(p_c + "%");
+		progress.setStringPainted(true);
+	    }
+	    // Finally, in the case we have no content length, we'll instead
+	    // write the current number of bytes downloaded again.
+	    else {
+		progress.setString(file_size + " b");
+		progress.setStringPainted(true);
+	    }
+	}
+	refresh();
+    }
+
+    /** Causes the two labels associated with this DownloadProgressBar object to
+     * update, thus reflecting the progression of the download. This 
+     * method is called by any of the other public setter methods in this 
+     * class.
+     */
+    private void refresh() {
+	// Refresh the contents of main label.
+	String args1[] = new String[1];
+	args1[0] = initial_url.toString();
+	Dictionary.setText(main_status, "Mirroring.DownloadJob.Downloading", args1);
+
+	if (current_url != null) {
+	    // Refresh the current label contents.
+	    String args2[] = new String[1];
+	    args2[0] = current_url;
+	    Dictionary.setText(current_status, "Mirroring.DownloadJob.Downloading", args2);
+	}
+	else if (current_action == DownloadJob.STOPPED || current_action == DownloadJob.PAUSED) {
+	    Dictionary.setText(current_status, "Mirroring.DownloadJob.Waiting_User");
+	}
+	else {
+	    Dictionary.setText(current_status, "Mirroring.DownloadJob.Download_Complete");
+	}
+
+	// Refresh the contents of results label
+	String args3[] = new String[4]; 
+	args3[0] = file_count + "";
+	args3[1] = total_count + "";
+	args3[2] = warning_count + "";
+	args3[3] = err_count + "";
+	Dictionary.setText(results_status, "Mirroring.DownloadJob.Status", args3);
+
+	this.updateUI();
+    }
+
+    static final private Dimension DIALOG_SIZE = new Dimension(640,480);
+
+    private class LogDialog
+	extends JDialog {
+	
+	public LogDialog(AppendLineOnlyFileDocument document) {
+	    super(Gatherer.g_man, "Mirroring.DownloadJob.Download_Log_Dialog_Title");
+	    setSize(DIALOG_SIZE);
+	    JPanel content_pane = (JPanel) getContentPane();
+	    JTextArea text_area = new JTextArea(document);
+	    JButton button = new GLIButton(Dictionary.get("General.Close"));
+	    // Connection
+	    button.addActionListener(new CloseActionListener());
+	    // Layout
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(new JScrollPane(text_area), BorderLayout.CENTER);
+	    content_pane.add(button, BorderLayout.SOUTH);
+	}
+
+	private class CloseActionListener
+	    implements ActionListener {
+	    public void actionPerformed(ActionEvent event) {
+		LogDialog.this.dispose();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EditorDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EditorDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EditorDialog.java	(revision 21112)
@@ -0,0 +1,151 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+
+/** A class that extends a JDialog into a editor for editing large block of text for the metadata value.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+final public class EditorDialog
+    extends ModalDialog 
+    implements ActionListener {
+   /** Is this dialog editable? */
+   private boolean editable = true;
+   /** The cancel, and I don't want the text I've typed, button. */
+    private JButton cancel = null;
+    /** The ok, I'll save what I've just typed in, button. */
+    private JButton ok = null;
+    /** The area in which we type. */
+    private JTextArea text = null;
+    /** And what result should be passed back to our caller. */
+    private String result = null;
+    /** The size of the edit pop-up. */
+    final static private Dimension SIZE = new Dimension(400, 300);
+
+    /** Constructor */
+    public EditorDialog() {
+	super(Gatherer.g_man);
+    }
+
+    /** Any implementation of ActionListener must include this method so we can be informed when an action has been performed on one of our target controls. In this case we generate a pop-up window to edit in.
+     * @param event An <strong>ActionEvent</strong> containing information about the event.
+     */
+    public void actionPerformed(ActionEvent event) {
+	if (event.getSource() == ok) {
+	    result = text.getText();
+	}
+	dispose();
+    }
+
+    /** Method to display the editing box on screen.
+     * @param value The initial text to be displayed in the editing area, as a <strong>String</strong>.
+     * @return The new value for the metadata value as a <strong>String</strong> or <i>null</i> if the user has pressed cancel.
+     */
+    public String display(String value) {
+	setModal(true);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("theenrichview"));
+	if (editable) {
+	    Dictionary.setText(this, "General.Edit");
+	} else {
+	    Dictionary.setText(this, "General.View");
+	}
+	// Create
+	text = new JTextArea(value);
+	text.setCaretPosition(value.length());
+	text.setEditable(editable);
+	text.setLineWrap(true);
+	text.setWrapStyleWord(false);
+	if (editable) {
+	    Dictionary.setTooltip(text, "EnrichPane.Value_Field_Tooltip");
+	} else {
+	    Dictionary.setTooltip(text, "EnrichPane.Value_Field_Tooltip_Uneditable");
+	}
+
+	cancel = new GLIButton();
+	cancel.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	ok = new GLIButton();
+	ok.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok, "General.OK", "General.OK_Tooltip");
+
+	// Listeners
+	cancel.addActionListener(this);
+	ok.addActionListener(this);
+
+	// Layout
+	JPanel button_pane = new JPanel();
+	button_pane.setLayout(new GridLayout(1,2));
+	if(editable) {
+	   button_pane.add(ok);
+	   button_pane.add(cancel);
+	}
+	else {
+	   button_pane.add(new JPanel());
+	   button_pane.add(ok);
+	}
+		  
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JScrollPane(text), BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+		  
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+	return result;
+    }
+
+    /** Specify if this text dialog should be editable or readonly
+     * @param editable true to allow editing, false otherwise
+     */
+    public void setEditable(boolean editable) {
+      if(text == null) {
+	 this.editable = editable;
+      }
+      else {
+	 text.setEditable(editable);
+      }
+   }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EmailField.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EmailField.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EmailField.java	(revision 21112)
@@ -0,0 +1,67 @@
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.util.regex.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+
+public class EmailField
+    extends JTextField 
+    implements DocumentListener {
+
+    static final public Pattern EMAIL_PATTERN = Pattern.compile("([^()\\-<> @,;:\"][^()<> @,;:\"]*|\"[^()<>@,;:\"]+\")@([A-Za-z](-*[A-Za-z0-9])*(\\.[A-Za-z](-*[A-Za-z0-9])*)*)");
+
+    private boolean invalid = false;
+    private Color background;
+    private Color invalid_background;
+
+    public EmailField(Color invalid_background) {
+	super();
+	this.invalid_background = invalid_background;
+    }
+
+    public EmailField(String email, Color invalid_background) {
+	super(email);
+	this.invalid_background = invalid_background;
+    }
+
+    /** Gives notification that an attribute or set of attributes changed.
+     * @param e
+     */
+    public void changedUpdate(DocumentEvent e) {
+	validateEmail();
+    }
+          
+    /** Gives notification that there was an insert into the document.
+     * @param e
+     */
+    public void insertUpdate(DocumentEvent e) {
+	validateEmail();
+    }
+          
+    /** Gives notification that a portion of the document has been removed. 
+     * @param e
+     */
+    public void removeUpdate(DocumentEvent e) {
+	validateEmail();
+    }
+
+    private void validateEmail() {
+	Matcher m = EMAIL_PATTERN.matcher(getText());
+	if(m.matches()) {
+	    // It was invalid, but now its valid again
+	    if(invalid) {
+		setBackground(background);
+		invalid = false;
+	    }
+	    // Otherwise nothings changed so why do anything
+	}
+	else {
+	    background = getBackground();
+	    setBackground(invalid_background);
+	    invalid = true;
+	}
+	m = null;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EnrichPane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EnrichPane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/EnrichPane.java	(revision 21112)
@@ -0,0 +1,635 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ * Based on code by John Thompson
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTree;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataValue;
+import org.greenstone.gatherer.metadata.MetadataValueTableEntry;
+import org.greenstone.gatherer.metadata.MetadataValueTreeNode;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+import org.greenstone.gatherer.util.DragGroup;
+import org.greenstone.gatherer.util.TreeSynchronizer;
+
+
+/** Provides a view of controls for the editing of metadata.
+ */
+public class EnrichPane
+    extends JPanel
+    implements TreeSelectionListener
+{
+    static private Dimension MINIMUM_SIZE = new Dimension(100, 100);
+    static private Dimension COLLECTION_TREE_SIZE = new Dimension(250, 500);
+
+    /** The collection tree. */
+    private CollectionTree collection_tree = null;
+    /** The currently reported selection. */
+    private CollectionTreeNode[] file_nodes = null;
+    /** Used to dynamically filter the collection tree. Synchronized with the same filter in the Gather view. */
+    private Filter collection_filter = null;
+    /** The label at the top of the collection tree, which shows the collection name. */
+    private JLabel collection_label;
+    /** The splitpane dividing the collection tree and the metadata editing controls. */
+    private JSplitPane external_split;
+    /** The metadata value table shows the metadata values that are currently assigned to a file. */
+    private MetadataValueTablePane metadata_value_table_pane = null;
+    /** The metadata value tree shows the metadata values that are currently assigned to a metadata element. */
+    private MetadataValueTreePane metadata_value_tree_pane = null;
+    /** Provide synchronization between the collection trees in this view and the collection pane view. */
+    private TreeSynchronizer collection_tree_sync = null;
+
+
+    /** Constructor.
+     * @param tree_sync The <strong>TreeSynchronizer</strong> to be used on the collection tree
+     */
+    public EnrichPane(TreeSynchronizer collection_tree_sync)
+    {
+	this.collection_tree_sync = collection_tree_sync;
+
+	// Create the metadata value tree pane
+	metadata_value_tree_pane = new MetadataValueTreePane();
+	metadata_value_tree_pane.addMetadataValueTreeSelectionListener(new MetadataValueTreeSelectionListener());
+
+	// Create metadata value table pane
+	metadata_value_table_pane = new MetadataValueTablePane();
+	metadata_value_table_pane.addMetadataValueTableListSelectionListener(new MetadataValueTableListSelectionListener());
+	metadata_value_table_pane.addMetadataValueTableMouseListener(new MetadataValueTableMouseListener());
+	metadata_value_table_pane.addMetadataValueTextFieldDocumentListener(new MetadataValueTextFieldDocumentListener());
+	metadata_value_table_pane.addMetadataValueTextFieldKeyListener(new MetadataValueTextFieldKeyListener());
+    }
+
+
+    /** Some actions can only occur after this panel has been displayed on-screen, so this method is provided to do exactly that. Such actions include the proportioning of the split panes and the setting of table column widths.
+     */
+    public void afterDisplay()
+    {
+	external_split.setDividerLocation(0.3);
+    }
+
+
+    /** Used to create, connect and layout the components to be shown on this control panel.
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.file.FileOpenActionListener
+     * @see org.greenstone.gatherer.gui.Filter
+     */
+    public void display()
+    {
+	// Creation
+	JPanel collection_pane = new JPanel(new BorderLayout());
+	collection_pane.setMinimumSize(MINIMUM_SIZE);
+	collection_pane.setPreferredSize(COLLECTION_TREE_SIZE);
+
+	external_split = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
+
+	collection_label = new JLabel();
+	Dictionary.registerText(collection_label, "Collection.No_Collection");
+	collection_label.setOpaque(true);
+
+	DragGroup group = new DragGroup();
+	collection_tree = new CollectionTree("Enrich", Gatherer.c_man.getCollectionTreeModel(), false);
+	group.add(collection_tree);
+	collection_tree.getSelectionModel().setSelectionMode(TreeSelectionModel.DISCONTIGUOUS_TREE_SELECTION);
+	collection_tree.putClientProperty("JTree.lineStyle", "Angled");
+	collection_tree.addMouseListener(Gatherer.g_man.foa_listener);
+	collection_tree.addMouseListener(new RightButtonListener());
+	collection_tree.addTreeSelectionListener(this);
+	collection_tree.addTreeExpansionListener(Gatherer.g_man.foa_listener);
+	collection_tree.setBackgroundNonSelectionColor(Configuration.getColor("coloring.collection_tree_background", false));
+	collection_tree.setTextNonSelectionColor(Configuration.getColor("coloring.collection_tree_foreground", false));
+	collection_tree.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	collection_tree.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	collection_tree.setRootVisible(false);
+
+	KeyListenerImpl key_listener = new KeyListenerImpl();
+	collection_tree.addKeyListener(key_listener);
+	JScrollPane collection_scroll = new JScrollPane(collection_tree);
+
+	collection_filter = Gatherer.g_man.getFilter(collection_tree);
+	collection_filter.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	collection_filter.setEditable(Configuration.getMode() > Configuration.LIBRARIAN_MODE);
+	Dictionary.registerTooltip(collection_filter.getComboBox(), "Collection.Filter_Tooltip");
+
+	// Layout
+	collection_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(3,3,3,3), BorderFactory.createLoweredBevelBorder()));
+	collection_pane.setMinimumSize(MINIMUM_SIZE);
+	collection_pane.setPreferredSize(new Dimension(Gatherer.g_man.getSize().width / 3, Gatherer.g_man.getSize().height));
+
+	// Collection Pane
+	collection_pane.add(collection_label, BorderLayout.NORTH);
+	collection_pane.add(collection_scroll, BorderLayout.CENTER);
+	collection_pane.add(collection_filter, BorderLayout.SOUTH);
+
+	JSplitPane metadata_editing_pane = new JSplitPane(JSplitPane.VERTICAL_SPLIT);
+	metadata_editing_pane.setBorder(BorderFactory.createEmptyBorder(0,5,5,5));
+	metadata_editing_pane.setDividerSize(8);
+
+	metadata_editing_pane.add(metadata_value_table_pane, JSplitPane.TOP);
+	metadata_editing_pane.add(metadata_value_tree_pane, JSplitPane.BOTTOM);
+	metadata_editing_pane.setDividerLocation(250);
+
+	external_split.add(collection_pane, JSplitPane.LEFT);
+	external_split.add(metadata_editing_pane, JSplitPane.RIGHT);
+
+	this.setLayout(new BorderLayout());
+	this.add(external_split, BorderLayout.CENTER);
+    }
+
+
+    /** Called to inform this pane that it has just gained focus as an effect of the user clicking on its tab
+     */
+    public void gainFocus()
+    {
+	// If there is nothing in the collection tree there is nothing to do
+	if (collection_tree == null || collection_tree.getRowCount() == 0) {
+	    return;
+	}
+
+	// Select the first node in the tree if nothing is already selected
+	if (collection_tree.getSelectionPaths() == null) {
+ 	    collection_tree.setImmediate(true);
+ 	    collection_tree.setSelectionRow(0);
+ 	    collection_tree.setImmediate(false);
+	    return;
+	}
+
+	// Force all of the controls to be updated
+	valueChanged(null);
+    }
+
+
+    /** Called to inform this pane that it has just lost focus as an effect of the user clicking on another tab
+     */
+    public void loseFocus()
+    {
+	// Very important: make sure metadata value is saved before leaving the pane
+	metadata_value_table_pane.stopEditingAndRebuild(file_nodes);
+
+	// Upload the modified metadata.xml files to the server now, if we're using one
+	MetadataXMLFileManager.uploadModifiedMetadataXMLFiles();
+    }
+
+
+    /** Called whenever the detail mode changes to ensure the filters are at an appropriate level (ie only editable by those that understand regular expression matching)
+     * @param mode the mode level as an int
+     */
+    public void modeChanged(int mode)
+    {
+	collection_filter.setEditable(mode > Configuration.LIBRARIAN_MODE);
+    }
+
+
+    /** Refresh this pane, depending on what has just happened (refresh_reason). */
+    public void refresh(int refresh_reason, boolean collection_loaded)
+    {
+	if (collection_loaded) {
+	    // Update collection label
+	    Dictionary.registerText(collection_label, "Collection.Collection");
+	    collection_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	    collection_label.setForeground(Configuration.getColor("coloring.collection_heading_foreground", false));
+
+	    // Update collection tree
+	    if (refresh_reason == Gatherer.COLLECTION_OPENED) {
+		collection_tree.setModel(Gatherer.c_man.getCollectionTreeModel());
+	    }
+
+	    // Update collection filter
+	    collection_filter.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	}
+	else {
+	    // Update collection label
+	    Dictionary.registerText(collection_label, "Collection.No_Collection");
+	    collection_label.setBackground(Color.lightGray);
+	    collection_label.setForeground(Color.black);
+
+	    // Update collection tree
+	    collection_tree.setModel(new DefaultTreeModel(new DefaultMutableTreeNode("Error")));
+
+	    // Update collection filter
+	    collection_filter.setBackground(Color.lightGray);
+	}
+
+	// Enable or disable the controls
+	collection_tree.setEnabled(collection_loaded);
+	collection_filter.setEnabled(collection_loaded);
+
+	// Ensure that this collection tree view is synchronized with all others
+	collection_tree_sync.add(collection_tree);
+
+	// Force the metadata table to be rebuilt (for switching extracted metadata on or off)
+	if (refresh_reason == Gatherer.PREFERENCES_CHANGED) {
+	    metadata_value_table_pane.stopEditingAndRebuild(file_nodes);
+	}
+    }
+
+
+    /** Called whenever the collection tree selection changes. This causes the metadata value table to be rebuilt. */
+    public void valueChanged(TreeSelectionEvent event)
+    {
+	// Nothing selected in the collection tree
+	if (collection_tree.getSelectionCount() == 0) {
+	    file_nodes = null;
+	}
+
+	// Some files selected in the collection tree
+	else {
+	    TreePath paths[] = collection_tree.getSelectionPaths();
+	    file_nodes = new CollectionTreeNode[paths.length];
+	    for (int i = 0; i < paths.length; i++) {
+		file_nodes[i] = (CollectionTreeNode) paths[i].getLastPathComponent();
+	    }
+	}
+
+	// Update the metadata value table (and consequently, the metadata value tree)
+	metadata_value_table_pane.stopEditingAndRebuild(file_nodes);
+    }
+
+
+    private class MetadataValueTableListSelectionListener
+	implements ListSelectionListener
+    {
+	public void valueChanged(ListSelectionEvent list_selection_event)
+	{
+	    // We only want to handle one event per selection, so wait for the value to stabilise
+	    if (list_selection_event.getValueIsAdjusting()) {
+		return;
+	    }
+
+	    // Update the metadata value tree for the current table selection
+	    metadata_value_tree_pane.rebuild(metadata_value_table_pane.getSelectedMetadataValueTableEntry());
+	}
+    }
+
+
+    private class MetadataValueTableMouseListener
+	extends MouseAdapter
+    {
+	public void mouseClicked(MouseEvent mouse_event)
+	{
+	    // We're only interested in clicks on the inherited column
+	    if (metadata_value_table_pane.isMouseEventForInheritedMetadataValueTableColumn(mouse_event) == false) {
+		return;
+	    }
+
+	    // If the selected metadata is inherited, switch to the folder it came from
+	    MetadataValueTableEntry selected_metadata_value_table_entry = metadata_value_table_pane.getSelectedMetadataValueTableEntry();
+	    if (selected_metadata_value_table_entry.isInheritedMetadata()) {
+		collection_tree.setSelection(selected_metadata_value_table_entry.getFolderMetadataInheritedFrom());
+	    }
+	}
+    }
+
+
+    private class MetadataValueTextFieldDocumentListener
+	implements DocumentListener
+    {
+	/** Gives notification that an attribute or set of attributes changed */
+	public void changedUpdate(DocumentEvent document_event) {
+	    validate(document_event);
+	}
+
+	/** Gives notification that there was an insert into the document */
+	public void insertUpdate(DocumentEvent document_event) {
+	    validate(document_event);
+	}
+
+	/** Gives notification that a portion of the document has been removed */
+	public void removeUpdate(DocumentEvent document_event) {
+	    validate(document_event);
+	}
+
+
+	/** Ensures that the value tree is updated in response to changes in the value text field */
+	private void validate(DocumentEvent document_event)
+	{
+	    try {
+		Document document = document_event.getDocument();
+		String metadata_value_string = document.getText(0, document.getLength());
+		metadata_value_tree_pane.selectBestPathForMetadataValue(metadata_value_string);
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+    }
+
+
+    private class MetadataValueTextFieldKeyListener
+	extends KeyAdapter
+    {
+	/** Gives notification of key events on the text field */
+	public void keyPressed(KeyEvent key_event)
+	{
+	    // Tab: Auto-complete what is selected in the metadata value tree
+	    if (key_event.getKeyCode() == KeyEvent.VK_TAB) {
+		MetadataValueTreeNode selected_metadata_value_tree_node = metadata_value_tree_pane.getSelectedMetadataValueTreeNode();
+		if (selected_metadata_value_tree_node != null) {
+		    metadata_value_table_pane.setMetadataValueTextFieldValue(selected_metadata_value_tree_node.getFullValue());
+		}
+
+		// We do not want this event to be processed by the table also
+		key_event.consume();
+	    }
+
+	    // Enter: save the current value then add a blank row for the selected metadata element
+	    if (key_event.getKeyCode() == KeyEvent.VK_ENTER) {
+		metadata_value_table_pane.stopEditingAndAddBlankRowForSelectedMetadataElement();
+	    }
+	}
+    }
+
+
+    private class MetadataValueTreeSelectionListener
+	implements TreeSelectionListener
+    {
+	public void valueChanged(TreeSelectionEvent tree_selection_event)
+	{
+	    // When a node is selected in the tree, fill the metadata value text field with the selected node's value
+	    MetadataValueTreeNode selected_metadata_value_tree_node = metadata_value_tree_pane.getSelectedMetadataValueTreeNode();
+	    if (selected_metadata_value_tree_node != null) {
+		metadata_value_table_pane.setMetadataValueTextFieldValue(selected_metadata_value_tree_node.getFullValue());
+	    }
+	}
+    }
+
+
+    /** This class listens for certain key presses, such as [Up] or [Down], -copied from Gather Pane */
+    private class KeyListenerImpl
+	extends KeyAdapter {
+	private boolean vk_left_pressed = false;
+	/** Called whenever a key that was pressed is released, it is this action that will cause the desired effects (this allows for the key event itself to be processed prior to this listener dealing with it). */
+	public void keyReleased(KeyEvent event) {
+	    ///ystem.err.println("Key Release detected. " + event.getKeyCode());
+	    if (event.getKeyCode() == KeyEvent.VK_UP || event.getKeyCode() == KeyEvent.VK_DOWN) {
+		DragTree tree = (DragTree)event.getSource();
+		// we need to manually do the up and down selections
+		boolean up = (event.getKeyCode() == KeyEvent.VK_UP);
+		int current_row = tree.getLeadSelectionRow();
+		tree.setImmediate(true);
+		if (up) {
+		    if (current_row > 0) {
+			tree.clearSelection();
+			tree.setSelectionRow(current_row-1);
+		    }
+		} else {
+		    if (current_row < tree.getRowCount()-1){
+			tree.clearSelection();
+			tree.setSelectionRow(current_row+1);
+		    }
+		}
+		tree.setImmediate(false);
+	    }
+	    else if (event.getKeyCode() == KeyEvent.VK_LEFT) {
+		// left key on a file shifts the selection to the parent folder
+		DragTree tree = (DragTree)event.getSource();
+		TreePath path = tree.getLeadSelectionPath();
+		if(path != null) {
+		    File file = ((CollectionTreeNode) path.getLastPathComponent()).getFile();
+		    if(file != null) {
+			if (file.isFile() || vk_left_pressed) {
+			    vk_left_pressed = false;
+			    TreePath parent_path = path.getParentPath();
+			    if (parent_path != null && parent_path.getParentPath() != null) {
+				// if this file is in the top level folder, don't move the focus 
+				tree.setImmediate(true);
+				tree.clearSelection();
+				tree.setSelectionPath(parent_path);
+				tree.setImmediate(false);
+			    }
+			}
+		    }
+		}
+	    }	    
+	}
+
+	// we need to watch for left clicks on an unopened folder - should shift the focus to teh parent folder. But because there is some other mysterious key listener that does opening and closing folders on right and left clicks, we must detect the situation before the other handler has done its job, and process it after. 
+	public void keyPressed(KeyEvent event) {
+	    if (event.getKeyCode() == KeyEvent.VK_LEFT) {
+		// left key on  closed directory shifts the selection to the parent folder
+		DragTree tree = (DragTree)event.getSource();
+		TreePath path = tree.getLeadSelectionPath();
+		if(path == null) return; 
+		File file = ((CollectionTreeNode) path.getLastPathComponent()).getFile();
+		if(file == null) return;
+		
+		if (file.isDirectory() && tree.isCollapsed(path)) {
+		    vk_left_pressed = true;
+		}		
+	    }
+	}
+    }
+
+
+    /** A listener for right mouse button clicks over the collection tree. */
+    private class RightButtonListener
+	extends MouseAdapter {
+	/** Called whenever a mouse click occurs (right or left) over a target component.
+	 * @param event A <strong>MouseEvent</strong> containing further information about the mouse click action.
+	 * @see org.greenstone.gatherer.gui.EnrichPane.RightButtonMenu
+	 */
+	public void mouseClicked(MouseEvent event) {
+	    if (SwingUtilities.isRightMouseButton(event)) {
+		new RightButtonMenu(event);
+	    }
+	}
+    }
+
+
+    /** When a user right-clicks within the collection tree they are presented with a small popup menu of context based options. This class provides such functionality.
+     */
+    private class RightButtonMenu
+	extends JPopupMenu
+	implements ActionListener {
+
+	/** The tree over which the right click action occurred. */
+	private DragTree tree = null;
+	/** The tree nodes selected when the right click action occurred. */
+	private TreePath[] selection_paths = null;
+	/** The file record over which the right click action occurred. */
+	private CollectionTreeNode node = null;
+
+	private JMenuItem collapse_folder = null;
+	private JMenuItem expand_folder = null;
+	private JMenuItem metaaudit = null;
+	private JMenuItem open_externally = null;
+
+
+	private RightButtonMenu(MouseEvent event)
+	{
+	    super();
+	    this.tree = collection_tree;
+
+	    // Note we have to use setImmediate() with the set selction paths
+	    // otherwise the selection doesn't get updated until after the 
+	    // popup comes up.
+
+	    // the right click position
+	    TreePath right_click_path = tree.getPathForLocation(event.getX(), event.getY());
+	    if (right_click_path == null) {
+		// user has clicked outside of the tree, clear the selection
+		selection_paths = null;
+		tree.setImmediate(true);
+		tree.clearSelection();
+		tree.setImmediate(false);
+	    } else {
+		// Get the paths currently selected in the tree
+		selection_paths = tree.getSelectionPaths();
+		if (selection_paths == null) {
+		    // nothing currently selected - we shift the selection to 
+		    // the node that was right clicked on
+		    selection_paths = new TreePath[1];
+		    selection_paths[0] = right_click_path;
+		    tree.setImmediate(true);
+		    tree.setSelectionPath(right_click_path);
+		    tree.setImmediate(false);
+		} else if (selection_paths.length == 1 && ! selection_paths[0].equals( right_click_path)) {
+		    tree.setImmediate(true);
+		    tree.clearSelection();
+		    tree.setSelectionPath(right_click_path);
+		    tree.setImmediate(false);
+		    selection_paths[0] = right_click_path;
+		} else {
+		    // we had multiply selected paths in the tree.
+		    // if we clicked on one of those paths, then use all the 
+		    // current selection, otherwise clear the selection and 
+		    // select the one we right clicked on
+		    boolean clicked_in_selection = false;
+		    for (int i=0; i<selection_paths.length; i++) {
+			if (selection_paths[i].equals(right_click_path)) {
+			    clicked_in_selection = true;
+			    break;
+			}
+		    }
+		    if (!clicked_in_selection) {
+			// want the tree to update right away
+			tree.setImmediate(true);
+			tree.clearSelection();
+			tree.setSelectionPath(right_click_path);
+			tree.setImmediate(false);
+			selection_paths = new TreePath[1];
+			selection_paths[0] = right_click_path;
+		    }
+		}
+	    }
+	    // finally we have the correct selection paths!
+
+	    // Create an appropriate context menu, based on what is selected
+	    buildContextMenu(selection_paths);
+
+	    // Show the popup menu on screen
+	    show(tree, event.getX(), event.getY());
+	}
+
+
+	private void buildContextMenu(TreePath[] selection_paths)
+	{
+	    // If nothing is selected, no options are available
+	    if (selection_paths == null) {
+		return;
+	    }
+
+	    DebugStream.println("Number of files/folders selected: " + selection_paths.length);
+
+	    // Always have meta-audit option
+	    metaaudit = new JMenuItem(Dictionary.get("Menu.Metadata_View", collection_tree.getSelectionDetails()), KeyEvent.VK_A);
+	    metaaudit.addActionListener(this);
+	    add(metaaudit);
+
+	    // Only meta-audit is available if multiple items are selected...
+	    if (selection_paths.length > 1) {
+		return;
+	    }
+
+	    TreePath path = selection_paths[0];
+	    node = (CollectionTreeNode) path.getLastPathComponent();
+
+	    // ---- Options for file nodes ----
+	    if (node.isLeaf()) {
+		// Open the file in an external program
+		open_externally = new JMenuItem(Dictionary.get("Menu.Open_Externally"), KeyEvent.VK_O);
+		open_externally.addActionListener(this);
+		add(open_externally);
+		return;
+	    }
+
+	    // ---- Options for folder nodes ----
+	    // Collapse or expand, depending on current status
+	    if (tree.isExpanded(path)) {
+		collapse_folder = new JMenuItem(Dictionary.get("Menu.Collapse"), KeyEvent.VK_C);
+		collapse_folder.addActionListener(this);
+		add(collapse_folder);
+	    }
+	    else {
+		expand_folder = new JMenuItem(Dictionary.get("Menu.Expand"), KeyEvent.VK_O);
+		expand_folder.addActionListener(this);
+		add(expand_folder);
+	    }
+	}
+
+
+	/** Called whenever one of the menu items is clicked, this method then causes the appropriate effect. */
+	public void actionPerformed(ActionEvent event)
+	{
+	    Object source = event.getSource();
+
+	    // Collapse folder
+	    if (source == collapse_folder) {
+		tree.collapsePath(selection_paths[0]);
+	    }
+
+	    // Expand folder
+	    else if (source == expand_folder) {
+		tree.expandPath(selection_paths[0]);
+	    }
+
+	    // Meta-audit
+	    else if (source == metaaudit) {
+		Gatherer.g_man.showMetaAuditBox();
+	    }
+
+	    // Open in external program
+	    else if (source == open_externally) {
+		Gatherer.f_man.openFileInExternalApplication(node.getFile());
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExplodeMetadataPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExplodeMetadataPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExplodeMetadataPrompt.java	(revision 21112)
@@ -0,0 +1,311 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Katherine Don, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import java.io.*;
+import java.util.ArrayList;
+
+import org.w3c.dom.Document;
+
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.cdm.Argument;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.cdm.Plugin;
+import org.greenstone.gatherer.collection.ScriptOptions;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.shell.GShellEvent;
+import org.greenstone.gatherer.shell.GShellListener;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.XMLTools;
+
+public class ExplodeMetadataPrompt	
+    extends ModalDialog
+    implements GShellListener
+{
+    /** The size of this new collection dialog box. */
+    static private Dimension SIZE = new Dimension(600, 350);
+    private JDialog self;
+    
+    /** the file we wil be exploding */
+    private File metadata_file = null;
+    /** the name of the plugin to be used */
+    private String plugin_name = null;
+    /** holds all the available options for the exploding script */
+    private ScriptOptions options = null;
+    /** the pane containing the options */
+    private JPanel options_pane = null;
+    /** the error message if any */
+    private StringBuffer error_message = null;
+    /** whether we were successful or not */
+    private boolean successful;
+
+
+    public ExplodeMetadataPrompt(File source_file) {
+	super(Gatherer.g_man, true);
+	this.self = this;
+	this.metadata_file = source_file;
+
+	// check that we actually have an explodable file
+	Plugin plugin = CollectionDesignManager.plugin_manager.getExploderPlugin(source_file);
+	if (plugin==null) {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("ExplodeMetadataPrompt.NotExplodable"), Dictionary.get("ExplodeMetadataPrompt.Title"), JOptionPane.ERROR_MESSAGE);
+	    return;
+	}
+	plugin_name = plugin.getName();
+	setJMenuBar(new SimpleMenuBar("explodingmetadata"));
+	setSize(SIZE);
+	Dictionary.setText(this, "ExplodeMetadataPrompt.Title");
+
+	// set up the script options
+	// we have empty initial values
+	String dom_string = "<Options/>";
+	Document doc = XMLTools.parseXML(new StringReader(dom_string));
+	options = new ScriptOptions(doc.getDocumentElement(), "explode_metadata_database.pl", false);
+	
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setOpaque(true);
+	
+	options_pane = new JPanel();
+	addScriptOptions(options_pane);
+	JScrollPane middle_pane = new JScrollPane(options_pane);
+
+	JTextArea instructions_area = new JTextArea();
+	instructions_area.setEditable(false);
+	instructions_area.setLineWrap(true);
+	instructions_area.setRows(5);
+	instructions_area.setWrapStyleWord(true);
+	Dictionary.setText(instructions_area, "ExplodeMetadataPrompt.Instructions");
+
+	JPanel button_pane = new JPanel();
+	JButton explode_button = new GLIButton();
+	Dictionary.setBoth(explode_button, "ExplodeMetadataPrompt.Explode", "ExplodeMetadataPrompt.Explode_Tooltip");
+	JButton cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+	
+	// Connection
+	cancel_button.addActionListener(new CancelListener());
+	explode_button.addActionListener(new ExplodeListener());
+
+	// Layout
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2));
+	button_pane.add(explode_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(instructions_area, BorderLayout.NORTH);
+	content_pane.add(middle_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Final dialog setup & positioning.
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+
+    public void destroy() 
+    {
+       	
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed of messages from the GShell.
+     * @param event A <strong>GShellEvent</strong> that contains, amoung other things, the message.
+     */
+    public synchronized void message(GShellEvent event) {
+	String message = event.getMessage();
+	if (message.startsWith("explode_metadata_database.pl>")) {
+	    message = message.substring(6);
+	    error_message.append(message);
+	    error_message.append("\n");
+	}
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell begins its task. Implementation side-effect, not actually used.
+     * @param event A <strong>GShellEvent</strong> that contains details of the initial state of the <strong>GShell</strong> before task comencement.
+     */
+    public synchronized void processBegun(GShellEvent event) {
+	// We don't care. 
+    }
+    
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell completes its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the final state of the <strong>GShell</strong> after task completion.
+     */
+    public synchronized void processComplete(GShellEvent event) {
+	successful = false;
+	if(event.getStatus() == GShell.OK) {
+	    successful = true;
+	}
+    }
+
+    private void addScriptOptions(JPanel options_pane) {
+
+	options_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	options_pane.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	options_pane.setLayout(new BoxLayout(options_pane, BoxLayout.Y_AXIS));
+
+	int current_mode = Configuration.getMode();
+	int total_argument_count = options.getArgumentCount();
+       
+	for(int i = 0; i < total_argument_count; i++) {
+	    // Retrieve the argument so we know how to format the control.
+	    Argument argument = options.getArgument(i);
+	    if(!argument.isHiddenGLI() && argument.getModeLevel() <= current_mode) {
+		// by default, all args are disabled, and no value
+		ArgumentControl argument_control = new ArgumentControl(argument,false, null);
+		// make sure they are coloured the way we want - this is not the standard arg control coloring
+		argument_control.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+		options_pane.add((JComponent)argument_control);
+	    }
+	}
+    }
+
+    private void updateScriptOptions() {
+	for(int i = 0; i < options_pane.getComponentCount(); i++) {
+	    Component component = options_pane.getComponent(i);
+	    if(component instanceof ArgumentControl) {
+		ArgumentControl ac = (ArgumentControl)component;
+		String name = ac.getName();
+		String value = ac.getValue();
+		boolean enabled = ac.isEnabled();
+		if (!enabled && value==null) {
+		    // flag type, remove from options altogether
+		    options.removeValue(name);
+		} else {
+		    if (value.equals("")) {
+			// if the value is empty, we don't want it to be passed to the script
+			options.setValue(name, false, value);
+		    } else {
+			options.setValue(name, enabled, value);
+		    }
+		}
+
+	    }
+	}
+    }
+
+    private int explodeMetadata()
+    {
+	if (Gatherer.isGsdlRemote) {
+	    // !! TO DO
+	    return -1;
+	}
+
+	// Generate the explode_metadata_database.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if (Utility.isWindows() && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "explode_metadata_database.pl");
+	command_parts_list.add("-plugin");
+	command_parts_list.add(plugin_name);
+
+	// Add in all the options from the user
+	String[] explode_options = options.getValues();
+	for (int i = 0; i < explode_options.length; i++) {
+	    command_parts_list.add(explode_options[i]);
+	}
+
+	// Add in the filename
+	command_parts_list.add(metadata_file.getPath());
+
+	// Run the explode_metadata_database.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	this.error_message = new StringBuffer();
+	GShell process = new GShell(command_parts, GShell.EXPLODE, 3, this, null, GShell.GSHELL_EXPLODE);
+	//process.start();
+	process.run();
+	
+	if (successful) {
+	    return 0;
+	} 
+	return -1;
+    }
+
+    private void resultPrompt(boolean success, String message)
+    {
+	if (success) {
+	    // !!! TO DO: Get explode_metadata_database.pl strings translated and use SimpleResultDialog below
+	    JOptionPane.showMessageDialog(null, Dictionary.get("ExplodeMetadataPrompt.Successful_Explode", metadata_file.getName()), Dictionary.get("ExplodeMetadataPrompt.Successful_Title"), JOptionPane.INFORMATION_MESSAGE);
+	}
+	else {
+	    String title = Dictionary.get("ExplodeMetadataPrompt.Failed_Title");
+	    String label = Dictionary.get("ExplodeMetadataPrompt.Failed_Explode", metadata_file.getName());
+	    SimpleResultDialog result_dialog = new SimpleResultDialog(title, label, message);
+	    result_dialog.setVisible(true); // Blocks
+	    result_dialog.dispose();
+	    result_dialog = null;
+	}
+    }
+
+    private class CancelListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    self.dispose();
+	}
+    }
+    
+    private class ExplodeListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent event) {
+	    // update the options
+	    updateScriptOptions();
+	    int exit_value = explodeMetadata();
+	    if (exit_value == 0) { // success
+		String new_dir = metadata_file.getPath();
+		// remove the extension to get the directory name
+		new_dir = new_dir.substring(0, new_dir.lastIndexOf('.'));
+		MetadataXMLFileManager.loadMetadataXMLFiles(new File(new_dir));
+		Gatherer.g_man.refreshCollectionTree(DragTree.COLLECTION_CONTENTS_CHANGED);
+		resultPrompt(true, error_message.toString());
+	    } else { // failure
+		resultPrompt(false, error_message.toString());
+	    }
+	    error_message = null;
+	    self.dispose();  
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExportAsPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExportAsPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExportAsPrompt.java	(revision 21112)
@@ -0,0 +1,575 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.shell.GShellEvent;
+import org.greenstone.gatherer.shell.GShellListener;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.CheckListEntry;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/** This class provides the functionality to export a set of current
+ * collections from the GSDLHOME/collect/ directory to various formats
+ * (hence ExportAs) using export.pl. The user chooses the collection from a
+ * list, where each entry also displays details about itself, confirms the
+ * delete of a collection by checking a checkbox then presses the ok button
+ * to actually delete the collection.  
+ * Copied from WriteCDImagePrompt
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class ExportAsPrompt
+    extends ModalDialog 
+    implements GShellListener {
+
+    static final private Dimension LABEL_SIZE = new Dimension(120, 25);
+    
+    private OKButtonListener ok_button_listener;
+
+    private JLabel saveas_label = null;
+    private JComboBox saveas_combobox = null;
+
+    private ArrayList all_collections = null;
+    private ArrayList selected_collections = null;
+    /** The list of collections to include in exported cd-rom/dvd image */
+    private CheckList list = null;
+    /** The currently selected collection for deletion. */
+    private BasicCollectionConfiguration collection = null;
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private ExportAsPrompt prompt = null;
+    /** The close button, which exits the prompt without deleting anything. */
+    private JButton cancel_button = null;
+    /** The ok button which causes the selected collection to be deleted. */
+    private JButton ok_button = null;
+    /** The label above details. */
+    private JLabel details_label = null;
+    /** The label above the list. */
+    private JLabel list_label = null;
+    /** The text area used to display details about the collection selected. */
+    private JTextArea details_textarea = null;
+    /** The text area used to display instructions for the cd-rom/dvd export */
+    private JTextArea instructions_textarea;
+    /** A string array used to pass arguments to the phrase retrieval method. */
+    private JTextField title_field = null;
+    private JLabel title_label = null;
+    private String args[] = null;
+    private String cd_title = null;
+    /** whether the exporting was successful or not */
+    private boolean successful = false;
+    /** whether we are trying to export or not */
+    private boolean exporting = false;
+    /** the error message if any */
+    private StringBuffer error_message = null;
+    /** The size of the export prompt screen. */
+    public static final Dimension SIZE = new Dimension(500, 500);
+
+    /** Constructor.
+     * @see org.greenstone.gatherer.collection.ExportAsPrompt.CancelButtonListener
+     * @see org.greenstone.gatherer.collection.ExportAsPrompt.CollectionListListener
+     * @see org.greenstone.gatherer.collection.ExportAsPrompt.OKButtonListener
+     */
+    public ExportAsPrompt() {
+	super(Gatherer.g_man, true);
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Close", "General.Close_Tooltip");
+	details_textarea = new JTextArea();
+	details_textarea.setEditable(false);
+	Dictionary.setText(details_textarea, "DeleteCollectionPrompt.No_Collection");
+	details_label = new JLabel();
+	Dictionary.setText(details_label, "DeleteCollectionPrompt.Collection_Details");
+
+	instructions_textarea = new JTextArea();
+	instructions_textarea.setCaretPosition(0);
+	instructions_textarea.setEditable(false);
+	instructions_textarea.setLineWrap(true);
+	instructions_textarea.setRows(4);
+	instructions_textarea.setWrapStyleWord(true);
+	Dictionary.registerText(instructions_textarea, "ExportAsPrompt.Instructions");
+
+        // Save As
+	ArrayList saveas_formats = new ArrayList();
+	saveas_formats.add("METS");
+	saveas_formats.add("DSpace");
+
+        saveas_label = new JLabel();
+        //saveas_label.setPreferredSize(LABEL_SIZE);
+        Dictionary.registerText(saveas_label, "ExportAsPrompt.SaveAs");
+        saveas_combobox = new JComboBox(saveas_formats.toArray());
+        Dictionary.registerTooltip(saveas_combobox, "ExportAsPrompt.SaveAs_Tooltip");
+
+	all_collections = new ArrayList();
+	list = new CheckList(true);
+	list_label = new JLabel();
+	Dictionary.setText(list_label, "DeleteCollectionPrompt.Collection_List");
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_D);
+	Dictionary.setBoth(ok_button, "ExportAsPrompt.Export", "ExportAsPrompt.Export_Tooltip");
+
+	title_field = new JTextField();
+	// Dictionary.setTooltip(title_field, "ExportAsPrompt.Export_Name_Tooltip");
+	title_label = new JLabel();
+	Dictionary.setText(title_label, "ExportAsPrompt.Export_Name");
+	scanForCollections();
+	list.setListData(all_collections);
+
+	prompt = this;
+	setSize(SIZE);
+	Dictionary.setText(this, "ExportAsPrompt.Title");
+
+	setJMenuBar(new SimpleMenuBar("0")); // need to find an appropriate help page to open at
+	cancel_button.addActionListener(new CancelButtonListener());
+	list.addListSelectionListener(new CollectionListListener());
+	list.clearSelection();
+	list.getSelectionModel().setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	ok_button_listener = new OKButtonListener();
+	ok_button.addActionListener(ok_button_listener);
+	ok_button.setEnabled(false);
+	//title.getDocument().addDocumentListener(new DocumentListener());
+    }
+
+    /** Destructor. */
+    public void destroy() {
+        saveas_label = null;
+	saveas_combobox = null;
+	all_collections.clear();
+	all_collections = null;
+	cancel_button = null;
+	details_textarea = null;
+	details_label = null;
+	list = null;
+	ok_button = null;
+	prompt = null;
+	if (selected_collections!=null) {
+	    selected_collections.clear();
+	    selected_collections = null;
+	}
+	title_field = null;
+	title_label = null;
+    }
+
+    /** This method causes the modal prompt to be displayed. 
+     * returns true if it has exported the collections that are currently selected */
+    public boolean display() {
+	// Top pane
+	JPanel instructions_pane = new JPanel(new BorderLayout());
+	instructions_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	instructions_pane.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+	
+	title_label.setBorder(BorderFactory.createEmptyBorder(0,5,0,15));
+
+	JPanel title_pane = new JPanel(new BorderLayout());
+	title_pane.add(title_label, BorderLayout.WEST);
+	title_pane.add(title_field, BorderLayout.CENTER);
+	instructions_pane.add(title_pane, BorderLayout.CENTER);
+
+	// Save as pane
+	JPanel saveas_pane = new JPanel(new BorderLayout());
+	saveas_pane.setLayout(new BorderLayout());
+	saveas_pane.add(saveas_label, BorderLayout.WEST);
+	saveas_pane.add(saveas_combobox, BorderLayout.CENTER);
+	instructions_pane.add(saveas_pane, BorderLayout.SOUTH);
+
+	// Central pane
+	JPanel list_pane = new JPanel(new BorderLayout());
+	list_pane.add(list_label, BorderLayout.NORTH);
+	list_pane.add(new JScrollPane(list), BorderLayout.CENTER);
+	list_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 5, 0));
+
+	JPanel details_pane = new JPanel(new BorderLayout());
+	details_pane.add(details_label, BorderLayout.NORTH);
+	details_pane.add(new JScrollPane(details_textarea), BorderLayout.CENTER);
+	details_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel central_pane = new JPanel(new GridLayout(2, 1));
+	central_pane.add(list_pane);
+	central_pane.add(details_pane);
+	central_pane.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
+
+	// Lower pane
+	JPanel button_pane = new JPanel(new GridLayout(1, 2));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+	button_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel lower_pane = new JPanel(new BorderLayout());
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(0, 5, 5, 5));
+
+	// Final.
+	JPanel content_pane = (JPanel)this.getContentPane();
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(instructions_pane, BorderLayout.NORTH);
+	content_pane.add(central_pane, BorderLayout.CENTER);
+	content_pane.add(lower_pane, BorderLayout.SOUTH);
+
+	// Center and display.
+	Dimension screen_size = Configuration.screen_size;
+	this.setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	this.setVisible(true); // blocks until the dialog is killed
+	return true;
+	
+    }
+
+    
+    /** This method calls the builcol.pl scripts via a GShell so as to not lock up the processor.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.Collection
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.shell.GShell
+     * @see org.greenstone.gatherer.shell.GShellListener
+     * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void exportAsCollections() {
+	DebugStream.println("ExportAsPrompt.exportAsCollections()");
+
+	int num_collections = selected_collections.size();
+	if (num_collections == 0) return;
+	cd_title = title_field.getText();
+	cd_title = cd_title.trim();
+	cd_title = cd_title.replaceAll("\"", "");
+
+	String export_type = (String) saveas_combobox.getSelectedItem();
+
+	// Generate the export.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if (Utility.isWindows() && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "export.pl");
+	command_parts_list.add("-gli");
+	command_parts_list.add("-language");
+	command_parts_list.add(Configuration.getLanguage());
+	command_parts_list.add("-removeold");
+	command_parts_list.add("-saveas");
+	command_parts_list.add(export_type);
+	command_parts_list.add("-exportdir");
+	command_parts_list.add(LocalGreenstone.getTmpDirectoryPath());
+
+	if (cd_title.equals("")) {
+	    command_parts_list.add("exported_" + export_type);
+	}
+	else {
+	    command_parts_list.add("exported_" + cd_title.replaceAll("\\s", ""));
+	}
+	
+	for (int i = 0; i < num_collections; i++) {
+	    command_parts_list.add(((BasicCollectionConfiguration) selected_collections.get(i)).getShortName());
+	}
+
+	DebugStream.print("export command = ");
+	for (int i = 0; i < command_parts_list.size(); i++) {
+	    DebugStream.print(command_parts_list.get(i) + " ");
+	    System.err.print(command_parts_list.get(i) + " ");
+	}
+	DebugStream.println("");
+	System.err.println("");
+
+	// Run the export.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell process = new GShell(command_parts, GShell.EXPORTAS, 3, this, null, GShell.GSHELL_EXPORTAS);
+	process.start();
+	//process.run();
+	DebugStream.println("ExportAsPrompt.exportAsCollections().return");
+	
+    }
+
+    /** Shows an export complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully deleted.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    public void resultPrompt(boolean success, String extra) {
+	args = new String[2];
+	StringBuffer coll_names = new StringBuffer();
+	for (int i=0; i<selected_collections.size();i++) {
+	    if (i>0) {
+		coll_names.append(", ");
+	    }
+	    BasicCollectionConfiguration complete_collection = (BasicCollectionConfiguration)selected_collections.get(i);
+	    coll_names.append(complete_collection.getName() + StaticStrings.SPACE_CHARACTER + StaticStrings.OPEN_PARENTHESIS_CHARACTER + complete_collection.getShortName() + StaticStrings.CLOSE_PARENTHESIS_CHARACTER);
+	    complete_collection = null;
+	}
+	
+	args[0] = coll_names.toString();
+
+	args[1] = LocalGreenstone.getTmpDirectoryPath();
+	if(cd_title.equals("")) {
+	    String export_type = (String)saveas_combobox.getSelectedItem();
+	    args[1] += "exported_" + export_type;
+	} else {
+	    args[1] += "exported_"+cd_title.replaceAll("\\s","");
+	}
+
+	String title;
+	String label;
+	String details;
+
+	if (success) {
+	    String successMessage
+		= (selected_collections.size()==1) 
+		? "ExportAsPrompt.Successful_ExportOne"
+		: "ExportAsPrompt.Successful_ExportMany";
+
+	    title = Dictionary.get("ExportAsPrompt.Successful_Title");
+	    label = Dictionary.get(successMessage, args);
+	    details = Dictionary.get("ExportAsPrompt.Successful_Details", args);
+	} else {
+	    String failedMessage
+		= (selected_collections.size()==1) 
+		? "ExportAsPrompt.Failed_ExportOne"
+		: "ExportAsPrompt.Failed_ExportMany";
+
+	    title = Dictionary.get("ExportAsPrompt.Failed_Title");
+	    label = Dictionary.get(failedMessage, args);
+	    details = Dictionary.get("ExportAsPrompt.Failed_Details", args);
+	}
+	SimpleResultDialog result_dialog = new SimpleResultDialog(title, label, details);
+	result_dialog.setVisible(true); // Blocks
+	result_dialog.dispose();
+	result_dialog = null;
+    }
+
+    /** Method to scan the collect directory retrieving and reloading each collection it finds, while building the list of known collections.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.util.ArrayTools
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void scanForCollections() {
+	// Start at the collect dir.
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	if (collect_directory.exists()) {
+	    // Now for each child directory see if it contains a .col file and
+	    // if so try to load it..
+	    File collections[] = collect_directory.listFiles();
+	    ArrayTools.sort(collections);
+	    for(int i = 0; collections != null && i < collections.length; i++) {
+		if(collections[i].isDirectory() && !collections[i].getName().equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+		    File config_file = new File(collections[i], Utility.CONFIG_FILE);
+		    if (config_file.exists()) {
+			BasicCollectionConfiguration config = new BasicCollectionConfiguration(config_file);
+			all_collections.add(config);		        
+			config = null;
+		    }
+		}
+	    }
+	}
+	// Otherwise the collect directory doesn't actually exist, so there ain't much we can do.
+    }
+
+
+    /** All implementation of GShellListener must include this method so the listener can be informed of messages from the GShell.
+     * @param event A <strong>GShellEvent</strong> that contains, amoung other things, the message.
+     */
+    public synchronized void message(GShellEvent event) {
+	// Ignore the messages from RecPlug with 'show_progress' set (used for progress bars)
+	String message = event.getMessage();
+	if (message.startsWith("export.pl>")) {
+	    message = message.substring(13);
+	    //DebugStream.println("message = "+event.getMessage());
+	    error_message.append(message);
+	    error_message.append("\n");
+	}
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell begins its task. Implementation side-effect, not actually used.
+     * @param event A <strong>GShellEvent</strong> that contains details of the initial state of the <strong>GShell</strong> before task comencement.
+     */
+    public synchronized void processBegun(GShellEvent event) {
+	// We don't care. 
+    }
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell completes its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the final state of the <strong>GShell</strong> after task completion.
+     */
+    public synchronized void processComplete(GShellEvent event) {
+	successful = false;
+	if(event.getStatus() == GShell.OK) {
+	    if(event.getType() == GShell.EXPORTAS) {
+		successful = true;
+	    }
+	}
+	ok_button_listener.processComplete();
+    }
+
+    /** A button listener implementation, which listens for actions on the close button and disposes of the dialog when detected. */
+    private class CancelButtonListener 
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    prompt.dispose();
+	}
+    }
+
+    /** This private class listens for selection events in from the list and then displays the appropriate details for that collection.
+     */
+    private class CollectionListListener 
+	implements ListSelectionListener
+    {
+	/** Any implementation of ListSelectionListener must include this method so we can be informed when the list selection changes.
+	 * @param  event a <strong>ListSelectionEvent</strong> containing all the relevant information garnered from the event itself
+	 */
+	public void valueChanged(ListSelectionEvent event)
+	{
+	    // Can only export when something is ticked
+	    ok_button.setEnabled(!list.isNothingTicked());
+
+	    if (list.isSelectionEmpty()) {
+		// This only happens when the dialog is first entered
+		Dictionary.setText(details_textarea, "DeleteCollectionPrompt.No_Collection");
+		return;
+	    }
+
+	    collection = (BasicCollectionConfiguration) ((CheckListEntry) list.getSelectedValue()).getObject();
+	    args = new String[3];
+	    args[0] = collection.getCreator();
+	    args[1] = collection.getMaintainer();
+	    args[2] = collection.getDescription();
+	    Dictionary.setText(details_textarea, "DeleteCollectionPrompt.Details", args);
+	    details_textarea.setCaretPosition(0);
+	}
+    }
+
+
+    /** The OK button listener implementation. */
+    private class OKButtonListener 
+	implements ActionListener {
+	private Component glass_pane;
+	private MouseListener mouse_blocker_listener;
+	private ProgressDialog progress_dialog;
+
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 * @see org.greenstone.gatherer.Configuration
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.util.Utility
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    ///ystem.err.println("OK Clicked");
+	    // Make sure there are some colls specified
+	    selected_collections = list.getTicked();
+	    error_message = new StringBuffer();
+	    
+	    // Set the cursor to hourglass
+	    glass_pane = getGlassPane();
+	    mouse_blocker_listener = new MouseAdapter() {};
+	    glass_pane.addMouseListener(mouse_blocker_listener);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	    glass_pane.setVisible(true);
+
+	    // Export the selected collection.
+	    ///ystem.err.println("Exporting As for named collections");
+	    exportAsCollections();
+
+	    // Show progress dialog
+	    ///ystem.err.println("Showing progress dialog");
+	    progress_dialog = new ProgressDialog();
+	    progress_dialog.setVisible(true);
+	}
+
+	public void processComplete() {
+	    ///ystem.err.println("Process complete");
+	    // Dispose of progress dialog
+	    progress_dialog.setVisible(false);
+	    progress_dialog.dispose();
+	    progress_dialog = null;
+
+	    // unset the cursor
+	    glass_pane.setVisible(false);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    glass_pane.removeMouseListener(mouse_blocker_listener);
+	    glass_pane = null;
+	    mouse_blocker_listener= null;
+	    
+	    if (successful) {
+		resultPrompt(true, error_message.toString());
+	    } else {
+		resultPrompt(false, error_message.toString());
+	    }
+	    error_message = null;
+	}
+
+	private class ProgressDialog 
+	    extends ModalDialog {
+	    
+	    private Dimension size = new Dimension(400,65);
+	    
+	    public ProgressDialog() {
+		super(Gatherer.g_man, Dictionary.get("ExportAsPrompt.Title"), true);
+		setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+		setSize(size);
+		JPanel content_pane = (JPanel) getContentPane();
+		JLabel progress_label = new JLabel();
+		Dictionary.setText(progress_label, "ExportAsPrompt.Progress_Label");
+		JProgressBar progress_bar = new JProgressBar();
+		progress_bar.setIndeterminate(true);
+		content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+		content_pane.setLayout(new BorderLayout());
+		content_pane.add(progress_label, BorderLayout.NORTH);
+		content_pane.add(progress_bar, BorderLayout.CENTER);
+		// Position
+		Rectangle frame_bounds = Gatherer.g_man.getBounds();
+		setLocation(frame_bounds.x + (frame_bounds.width - size.width) / 2, frame_bounds.y + (frame_bounds.height - size.height) / 2);
+	    }
+	}
+    }
+
+
+}
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExternalCollectionPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExternalCollectionPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ExternalCollectionPrompt.java	(revision 21112)
@@ -0,0 +1,178 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Project, NZDL, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 2003 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+
+// this has been changed to not offer the choice of a new metadata set, cos I haven't made this work properly.
+/** Prompt used when opening a non-gatherer built collection. We tell the user that they need to select metadata sets, and (in future) give them a choice between selecting an existing set or creating a new one */
+public class ExternalCollectionPrompt
+    extends ModalDialog {
+
+    public static int ERROR = 0;
+    public static int NEW_META_SET = 1;
+    public static int EXISTING_META_SET = 2;
+
+    private boolean cancelled = true;
+    private JDialog self;
+    private JRadioButton new_mds_button;
+    private JRadioButton existing_mds_button;
+    
+
+    static private Dimension size = new Dimension(600, 250);
+
+    public ExternalCollectionPrompt() {
+	super(Gatherer.g_man, true);
+	this.self = this;
+	setJMenuBar(new SimpleMenuBar("loadinganexternalcollection"));
+	setModal(true);
+	setSize(size);
+	Dictionary.setText(this, "ExternalCollectionPrompt.Title");
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel center_pane = new JPanel();
+
+	JPanel instructions_panel = new JPanel();
+	JTextArea instructions_textarea;
+	instructions_textarea = new JTextArea();
+	instructions_textarea.setCaretPosition(0);
+	instructions_textarea.setEditable(false);
+	instructions_textarea.setLineWrap(true);
+	instructions_textarea.setRows(6);
+	instructions_textarea.setWrapStyleWord(true);
+	Dictionary.registerText(instructions_textarea, "ExternalCollectionPrompt.Instructions_1"); // change to "ExternalCollectionPrompt.Instructions_2" when add the other choice back in
+
+	/*	new_mds_button = new JRadioButton();
+	new_mds_button.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	//new_mds_button.setMnemonic(KeyEvent.VK_I);
+	new_mds_button.setOpaque(false);
+	Dictionary.registerText(new_mds_button, "ExternalCollectionPrompt.NewMDS");
+	existing_mds_button = new JRadioButton();
+	existing_mds_button.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	//existing_mds_button.setMnemonic(KeyEvent.VK_X);
+	existing_mds_button.setOpaque(false);
+	Dictionary.registerText(existing_mds_button, "ExternalCollectionPrompt.ExistingMDS");
+
+	ButtonGroup bg = new ButtonGroup();
+	bg.add(new_mds_button);
+	bg.add(existing_mds_button);
+	new_mds_button.setSelected(true);
+
+	JPanel selection_pane = new JPanel();
+	selection_pane.setLayout(new GridLayout(2,1));
+	selection_pane.add(new_mds_button);
+	selection_pane.add(existing_mds_button);
+	*/
+	JPanel button_pane = new JPanel();
+	JButton ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	JButton cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(new OKButtonListener());
+	cancel_button.addActionListener(new CancelButtonListener());
+	//sets_list.addListSelectionListener(new MetadataListSelectionListener());
+
+	// Display
+	instructions_panel.setLayout(new GridLayout(1,1));
+	instructions_panel.add(instructions_textarea);
+
+
+	//center_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	//center_pane.setLayout(new GridLayout(2,1,0,5));
+	//center_pane.add(selection_pane);
+	
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2,5,0));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(instructions_panel, BorderLayout.NORTH);
+	//content_pane.add(center_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Show
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - size.width) / 2, (screen_size.height - size.height) / 2);
+	setVisible(true);
+    }
+
+    public int getMetadataChoice() {
+	return NEW_META_SET;
+	
+	/*if (existing_mds_button.isSelected()) {
+	    return EXISTING_META_SET;
+	}
+	if (new_mds_button.isSelected()) {
+	    return NEW_META_SET;
+	}
+	// something wrong
+	return ERROR;*/
+    }
+    public boolean isCancelled() {
+	return cancelled;
+    }
+    private class CancelButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    cancelled = true;
+	    self.dispose();
+	}
+    }
+    public class OKButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    cancelled = false;
+	    self.dispose();
+		
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/FileAssociationDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/FileAssociationDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/FileAssociationDialog.java	(revision 21112)
@@ -0,0 +1,490 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.filechooser.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.file.FileAssociationManager;
+import org.greenstone.gatherer.util.TableUtils;
+import org.greenstone.gatherer.util.Utility;
+
+/** The file association allows the entry of new file associations and the modification of existing ones.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class FileAssociationDialog
+    extends ModalDialog {
+
+    /** The default size for the dialog. */
+    static final private Dimension SIZE = new Dimension(600, 450);
+
+    private boolean ignore = false;
+    private FileAssociationDialog self;
+    private FileAssociationManager file_association_manager;
+    private JButton add_button;
+    private JButton browse_button;
+    private JButton close_button;
+    private JButton remove_button;
+    private JButton replace_button;
+    private JTable existing_associations_table;
+    private JTextField command_field;
+    private JTextField extension_field;
+
+    /** Create a new file association dialog.
+     * @param file_association_manager A reference to the <strong>FileAssociationManager</strong> so we can determine what extensions are already associated.
+     */
+    public FileAssociationDialog(FileAssociationManager file_association_manager) {
+	super(Gatherer.g_man);
+	this.file_association_manager = file_association_manager;
+	this.self = this;
+		  
+	// Creation
+	setModal(true);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("fileassociations"));
+
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+
+	JTextArea instructions_area = new JTextArea();
+	instructions_area.setEditable(false);
+	instructions_area.setLineWrap(true);
+	instructions_area.setRows(5);
+	instructions_area.setWrapStyleWord(true);
+	Dictionary.setText(instructions_area, "FileAssociationDialog.Instructions");
+
+	JPanel control_pane = new JPanel();
+	JLabel existing_associations_label = new JLabel();
+	existing_associations_table = new JTable(file_association_manager);
+	existing_associations_table.setAutoResizeMode(JTable.AUTO_RESIZE_NEXT_COLUMN);
+	existing_associations_table.setColumnSelectionAllowed(false);
+	existing_associations_table.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	JPanel lower_pane = new JPanel();
+	JPanel details_pane = new JPanel();
+
+	JPanel extension_pane = new JPanel();
+	JLabel extension_label = new JLabel();
+	extension_field = new NonWhitespaceField();
+
+	JLabel command_label = new JLabel();
+	JPanel command_pane = new JPanel();
+	command_field = new JTextField();
+	browse_button = new GLIButton();
+	browse_button.setEnabled(!Utility.isMac());
+	browse_button.setMnemonic(KeyEvent.VK_B);
+
+	JPanel button_pane = new JPanel();
+	
+	add_button = new GLIButton();
+	add_button.setEnabled(false);
+	add_button.setMnemonic(KeyEvent.VK_A);
+
+	replace_button = new GLIButton();
+	replace_button.setEnabled(false);
+	replace_button.setMnemonic(KeyEvent.VK_P);
+
+	remove_button = new GLIButton();
+	remove_button.setEnabled(false);
+	remove_button.setMnemonic(KeyEvent.VK_R);
+
+	close_button = new GLIButton();
+	close_button.setMnemonic(KeyEvent.VK_C);
+
+	// Connection
+	add_button.addActionListener(new AddButtonListener());
+	browse_button.addActionListener(new BrowseButtonListener());
+	close_button.addActionListener(new CloseButtonListener());
+	remove_button.addActionListener(new RemoveButtonListener());
+	replace_button.addActionListener(new ReplaceButtonListener());
+	CommandOrExtensionFieldListener coefl = new CommandOrExtensionFieldListener();
+	command_field.getDocument().addDocumentListener(coefl);
+	extension_field.getDocument().addDocumentListener(coefl);
+	coefl = null;
+	existing_associations_table.getSelectionModel().addListSelectionListener(new ExistingAssociationsTableListener());
+	Dictionary.setBoth(add_button, "FileAssociationDialog.Add", "FileAssociationDialog.Add_Tooltip");
+	if(Utility.isMac()) {
+	    Dictionary.setBoth(browse_button, "FileAssociationDialog.Browse", "FileAssociationDialog.Browse_Tooltip_Mac");
+	}
+	else {
+	    Dictionary.setBoth(browse_button, "FileAssociationDialog.Browse", "FileAssociationDialog.Browse_Tooltip");
+	}
+	Dictionary.setBoth(close_button, "FileAssociationDialog.Close", "FileAssociationDialog.Close_Tooltip");
+	Dictionary.setBoth(remove_button, "FileAssociationDialog.Remove", "FileAssociationDialog.Remove_Tooltip");
+	Dictionary.setBoth(replace_button, "FileAssociationDialog.Replace", "FileAssociationDialog.Replace_Tooltip");
+	Dictionary.setText(command_label, "FileAssociationDialog.Command");
+	Dictionary.setTooltip(command_field, "FileAssociationDialog.Command_Tooltip");
+	Dictionary.setText(extension_label, "FileAssociationDialog.Extension");
+	Dictionary.setTooltip(extension_field, "FileAssociationDialog.Extension_Tooltip");
+	Dictionary.setText(existing_associations_label, "FileAssociationDialog.Existing_Associations");
+	Dictionary.setText(this, "FileAssociationDialog.Title");
+
+	// Layout
+	extension_pane.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	extension_pane.setLayout(new BorderLayout(5,0));
+	extension_pane.add(extension_label, BorderLayout.WEST);
+	extension_pane.add(extension_field, BorderLayout.CENTER);
+
+	command_pane.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	command_pane.setLayout(new BorderLayout());
+	command_pane.add(command_field, BorderLayout.CENTER);
+	command_pane.add(browse_button, BorderLayout.EAST);
+
+	details_pane.setBorder(BorderFactory.createTitledBorder(Dictionary.get("FileAssociationDialog.Details")));
+	details_pane.setLayout(new GridLayout(3,1));
+	details_pane.add(extension_pane);
+	details_pane.add(command_label);
+	details_pane.add(command_pane);
+
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	button_pane.setLayout(new GridLayout(2,3));
+	button_pane.add(add_button);
+	button_pane.add(replace_button);
+	button_pane.add(remove_button);
+	button_pane.add(new JPanel());
+	button_pane.add(new JPanel());
+	button_pane.add(close_button);
+
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	lower_pane.setLayout(new BorderLayout());
+	lower_pane.add(details_pane, BorderLayout.CENTER);
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+
+	control_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	control_pane.setLayout(new BorderLayout());
+	control_pane.add(existing_associations_label, BorderLayout.NORTH);
+	control_pane.add(new JScrollPane(existing_associations_table), BorderLayout.CENTER);
+	control_pane.add(lower_pane, BorderLayout.SOUTH);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JScrollPane(instructions_area), BorderLayout.NORTH);
+	content_pane.add(control_pane, BorderLayout.CENTER);
+
+	Rectangle screen = Gatherer.g_man.getBounds(null);
+	setLocation((int)(screen.getX() + (screen.getWidth() - SIZE.width) / 2), (int)(screen.getY() + (screen.getHeight() - SIZE.height) / 2));
+	screen = null;
+    }
+
+    public void destroy() {
+	// Disconnect
+	// Clean up
+	file_association_manager = null;
+	self = null;
+    }
+
+    /** Redisplay the dialog, ensuring that the details (or lack thereof) for a certain extension is apparent. In fact if an extension is provided force the user to add it or cancel.
+     * @param new_extension The extension code as a <strong>String</strong>.
+     */
+    public String display(String new_extension) {
+	// Clear the current selection
+	existing_associations_table.clearSelection();
+	if(new_extension != null) {
+	    extension_field.setText(new_extension);
+	}
+	setVisible(true);
+	if(new_extension != null) {
+	    return file_association_manager.getCommandString(new_extension);
+	}
+	return null;
+    }
+
+    protected void processWindowEvent(WindowEvent event) {
+	if(event.getID() == WindowEvent.WINDOW_ACTIVATED) {
+	    TableUtils.fixColumnToPreferredWidth(existing_associations_table, 0); 
+	}
+	else {
+	    super.processWindowEvent(event);
+	}
+    }
+
+    private class AddButtonListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent event) {
+	    String extension_str = extension_field.getText();
+	    file_association_manager.setCommand(extension_str, command_field.getText());
+	    // Select the appropriate entry in the table
+	    int index = -1;
+	    for(int i = 0; index == -1 && i < file_association_manager.size(); i++) {
+		if(file_association_manager.getExtension(i).equals(extension_str)) {
+		    index = i;
+		}
+	    }
+	    if(index > -1) {
+		existing_associations_table.setRowSelectionInterval(index, index);
+	    }
+	    // And update buttons
+	    add_button.setEnabled(false);
+	    remove_button.setEnabled(true);
+	}
+    }
+
+    /** Batch filter shows only files ending in bat. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class BatchFileFilter
+	extends FileFilter {
+
+	/** Accept all .exe files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".bat");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Batch_File");
+	}
+    }
+
+    /** Whenever the user clicks the browse button, we should open up a file browser to allow them to select an executable file from somewhere in the file system. */
+    private class BrowseButtonListener 
+	implements ActionListener {
+	/** Open up a simple JFileChooser when the user clicks the button. 
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    JFileChooser chooser = new JFileChooser(new File(Gatherer.getGLIUserDirectoryPath()));
+	    OpenCollectionDialog.disableRename(chooser);
+	    chooser.setDialogTitle(Dictionary.get("FileAssociationDialog.Browse_Title"));
+	    chooser.setFileFilter(new BatchFileFilter());
+	    chooser.setFileFilter(new CoreObjectModelFileFilter());
+	    chooser.setFileFilter(new ExecutableFileFilter());
+	    chooser.setAcceptAllFileFilterUsed(true);
+	    if(chooser.showOpenDialog(Gatherer.g_man) == JFileChooser.APPROVE_OPTION) {
+		command_field.setText(chooser.getSelectedFile().getAbsolutePath());
+	    }
+	}
+    }
+
+    private class CloseButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    existing_associations_table.clearSelection();
+	    add_button.setEnabled(false);
+	    remove_button.setEnabled(false);
+	    replace_button.setEnabled(false);
+	    self.dispose();
+	}
+    }
+
+    /** Command filter shows only files ending in com. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class CoreObjectModelFileFilter
+	extends FileFilter {
+
+	/** Accept all .exe files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".com");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Command_File");
+	}
+    }
+
+    private class CommandOrExtensionFieldListener
+	implements DocumentListener {
+	/** Gives notification that an attribute or set of attributes changed. */
+	public void changedUpdate(DocumentEvent e) {
+	    changed();
+	}
+	/** Gives notification that there was an insert into the document. */
+	public void insertUpdate(DocumentEvent e) {
+	    changed();
+	}
+	/** Gives notification that a portion of the document has been removed. */
+	public void removeUpdate(DocumentEvent e) {
+	    changed();
+	} 
+
+	private void changed() {
+	    ignore = true;
+	    String extension_str = extension_field.getText();
+	    String command_str = command_field.getText();
+	    // Determine if there is currently an association selected
+	    int selected_index = -1;
+	    if((selected_index = existing_associations_table.getSelectionModel().getMinSelectionIndex()) != -1) {
+		String current_extension_str = file_association_manager.getExtension(selected_index);
+		String current_command_str = file_association_manager.getCommandString(current_extension_str);
+		// What buttons are enabled depends a lot on the file extension, taken to be the 'key'
+		if(extension_str.equals(current_extension_str)) {
+		    // If the extensions are the same then remove is enabled
+		    remove_button.setEnabled(true);
+		    // But if the command is different, then enable replace
+		    if(current_command_str.length() == 0) {
+			add_button.setEnabled(true);
+			replace_button.setEnabled(false);
+		    }
+		    else if(!command_str.equals(current_command_str)) {
+			add_button.setEnabled(false);
+			replace_button.setEnabled(true);
+		    }
+		    else {
+			add_button.setEnabled(false);
+			replace_button.setEnabled(false);
+		    }
+		}
+		// We can add it, but we first clear the selection
+		else {
+		    existing_associations_table.clearSelection();
+		    add_button.setEnabled(true);
+		    remove_button.setEnabled(false);
+		    replace_button.setEnabled(false);
+		}
+	    }
+	    // If not, we must first test if the extension is in use
+	    else {
+		int index = -1;
+		for(int i = 0; index == -1 && i < file_association_manager.size(); i++) {
+		    String existing_extension_str = file_association_manager.getExtension(i);
+		    if(existing_extension_str.equals(extension_str)) {
+			index = i;
+		    }
+		}
+		if(index != -1) {
+		    // Selection that index
+		    existing_associations_table.setRowSelectionInterval(index, index);
+		    // Set replace and remove enabled
+		    add_button.setEnabled(false);
+		    remove_button.setEnabled(true);
+		    replace_button.setEnabled(true);
+		}
+		// Otherwise if there is some text in both extension and command, enable add only
+		else if(extension_str.length() > 0 && command_str.length() > 0) {
+		    add_button.setEnabled(true);
+		    remove_button.setEnabled(false);
+		    replace_button.setEnabled(false);
+		}
+		// Otherwise disable everything
+		else {
+		    add_button.setEnabled(false);
+		    remove_button.setEnabled(false);
+		    replace_button.setEnabled(false);
+		}
+	    }
+	    ignore = false;
+	}
+    }
+
+    /** Executable filter shows only files ending in exe. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class ExecutableFileFilter
+	extends FileFilter {
+
+	/** Accept all .exe files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".exe");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Executable_File");
+	}
+    }
+
+    private class ExistingAssociationsTableListener
+	implements ListSelectionListener {
+
+	public void valueChanged(ListSelectionEvent event) {
+	    if(!event.getValueIsAdjusting() && !ignore) {
+		int selected_index = -1;
+		if((selected_index = existing_associations_table.getSelectionModel().getMinSelectionIndex()) != -1) {
+		    // Propagate the details of the selection down into the fields
+		    String extension_str = file_association_manager.getExtension(selected_index);
+		    extension_field.setText(extension_str);
+		    command_field.setText(file_association_manager.getCommandString(extension_str));
+		    extension_str = null;
+		    remove_button.setEnabled(true);
+		}
+		else {
+		    // Clear the fields
+		    extension_field.setText("");
+		    command_field.setText("");
+		    remove_button.setEnabled(false);
+		}
+		add_button.setEnabled(false);
+		replace_button.setEnabled(false);
+	    }
+	}
+    }
+
+    private class RemoveButtonListener 
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    file_association_manager.setCommand(extension_field.getText(), null);
+	    // And update buttons
+	    add_button.setEnabled(true);
+	    remove_button.setEnabled(false);
+	}
+    }
+
+    private class ReplaceButtonListener 
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    file_association_manager.setCommand(extension_field.getText(), command_field.getText());
+	    // And update buttons
+	    replace_button.setEnabled(false);
+	}
+    }
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Filter.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Filter.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Filter.java	(revision 21112)
@@ -0,0 +1,335 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import java.util.regex.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.file.FileSystemModel;
+import org.greenstone.gatherer.gui.tree.DragTree;
+
+/** <p>This object allows the user to set a filter on one of the workspace trees, specifying a preset type, or a regular expression that a files must match to be in the tree. Note that all directories are included. This class includes the controls for editing the filter. The trick is that several instances of the Filter class can share the same internal data (termed a 'run' of filters), so that the filter set on the GatherPane and the EnrichPane are virtually the same.</p>
+ * <p>The regular expression typed uses '*' as a wildcard character (equivalent to '.*'), and does not use '.' to match any single character (use '?' instead).</p>
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class Filter
+    extends JPanel {
+    /** The other filters in this run of filters, used to ensure they all show the same thing. */
+    private ArrayList others = null;
+    /** Is this the first filter of this run of filters created (later filters will share the same information). */
+    private boolean first = true;
+    /** Prevent any changes we make in the class from causing events which we then process causing events... */
+    private boolean ignore = false;
+    /** A reference to ourselves so inner classes can refer to us. */
+    private Filter this_filter = null;
+    /** The check box to enable/disable filter. */
+    private JCheckBox checkbox = null;
+    /** The editable combobox where you either choose a predefined filter, or type a new pseudo-regular expression. */
+    private GComboBox combobox = null;
+    /** The label shown on the filter controls. */
+    private JLabel label = null;
+    /** A reference to the tree this filter is being applied to. */
+    private DragTree tree = null;
+    /** The default size for the label. */
+    static final private Dimension SIZE = new Dimension(100,30);
+    /** Preprogrammed default filters. */
+    static final private String DEFAULTS[] = {"^.*\\.html?$", "^.*\\.xml$", "^.*\\.txt$", "(^.*\\.jpe?g$)|(^.*\\.png$)|(^.*\\.gif$)|(^.*\\.bmp$)|(^.*\\.tiff?$)"};
+
+    /** Constructor.
+     * @param tree A reference to the <strong>JTree</strong> being affected.
+     */
+    public Filter(DragTree tree) {
+	this(tree, null);
+    }
+
+    /** Constructor.
+     * @param tree A reference to the <strong>JTree</strong> being affected.
+     * @param others An <strong>ArrayList</strong> of the other Filters already in this run.
+     */
+    public Filter(DragTree tree, ArrayList others) {
+	super();
+	if (others == null) {
+	    others = new ArrayList();
+	}
+	this.others = others;
+	this.others.add(this);
+	this.this_filter = this;
+	this.tree = tree;
+	// Create components.
+	combobox = new GComboBox(true);
+	combobox.add(new Entry());
+	for(int i = 0; i < DEFAULTS.length; i++) {
+	    try {
+		Entry entry = new Entry(Dictionary.get("Filter." + i), Pattern.compile(DEFAULTS[i]));
+		combobox.add(entry);
+	    }
+	    catch (Exception error) {
+		error.printStackTrace();
+	    }
+	}
+	label = new JLabel();
+	Dictionary.setText(label, "Filter.Filter_Tree");
+	// Add listeners.
+	combobox.addActionListener(new ComboBoxListener());
+	// Layout.
+	label.setBorder(BorderFactory.createEmptyBorder(0,0,0,5));
+
+	setBorder(BorderFactory.createEmptyBorder(2,2,2,2));
+	setLayout(new BorderLayout());
+	add(label, BorderLayout.WEST);
+	add(combobox, BorderLayout.CENTER);
+    }
+    /** Retrieve the combobox associated with this filter.
+     * @return A <strong>GComboBox</strong>.
+     */
+    public GComboBox getComboBox() {
+	return combobox;
+    }
+
+    /** Used to restore the filter state to enabled, the normal state during collection editing.
+     * @param state The new state for the filter. <i>true</i> for enabled, <i>false</i> otherwise.
+     */
+    public void setEnabled(boolean state) {
+	ignore = true;
+	combobox.setEnabled(state);
+	ignore = false;
+    }
+
+    /** Set the combobox model for this filter.
+     * @param model The new <strong>ComboBoxModel</strong> to use.
+     */
+    public void setComboBoxModel(ComboBoxModel model) {
+	combobox.setModel(model);
+    }
+
+    /** Ensure that a certain entry is selected from the combobox.
+     * @param selection The <strong>Entry</strong> that should be selected.
+     */
+    public void setComboBoxSelection(Entry selection) {
+	ignore = true;
+	combobox.setSelectedItem(selection);
+	ignore = false;
+    }
+
+    /** Sets whether this combobox is editable or not
+     * @param value true to make the control editable, false otherwise
+     */
+    public void setEditable(boolean value) {
+	combobox.setEditable(value);	    
+    }
+
+    /** Set to signify that this filter is the first in a new run of filters.
+     * @param first <i>true</i> if this is the first filter in a run, <i>false</i> if it will just be added to the current run.
+     */
+    public void setFirst(boolean first) {
+	this.first = first;
+    }
+
+    /** Spawn produces a copy of this filter, which has new controls, but shares listeners with this filter, and vice versa. Thus we can have two synchronized, but differing sets of controls.
+     * @param tree_spawn The <strong>JTree</strong> this filter will affect.
+     * @return A new <strong>Filter</strong> which is in the same run as this filter.
+     */
+    public Filter spawn(DragTree tree_spawn) {
+	Filter filter = new Filter(tree_spawn, others);
+	filter.setFirst(false); // No spawned copies should generate error messages, but fail silently.
+	filter.setComboBoxModel(combobox.getModel());
+	filter.setComboBoxSelection((Entry)combobox.getSelectedItem());
+	return filter;
+    }
+
+    /** Encode an expression in pseudo-regular expression into regular expression.
+     * @param raw The pseudo-regular expression <strong>String</strong> which includes several characters which differ in meaning from regular expression queries.
+     * @return A proper regular expression as a <strong>String</strong>.
+     */
+    private String encode(String raw) {
+	StringBuffer working = new StringBuffer();
+	for(int i = 0; i < raw.length(); i++) {
+	    char c = raw.charAt(i);
+	    switch(c) {
+	    case '.':
+		working.append("\\.");
+		break;
+	    case '*':
+		working.append(".*");
+		break;
+	    case '?':
+		working.append(".");
+		break;
+	    default:
+		working.append(Character.toLowerCase(c));
+	    }
+	}
+	return working.toString();
+    }
+    /** This method applies the given pattern to the tree registered as belonging to this filter.*/
+    private void setFilter(Pattern pattern) {
+	// Show busy cursor.
+	Gatherer.g_man.wait(true);
+	FileSystemModel model = (FileSystemModel) tree.getModel();
+	// Apply filter
+	if(pattern != null) {
+	    model.setFilter(pattern.pattern());
+	}
+	else {
+	    model.setFilter(null);
+	}
+	// Ask tree to completely refresh
+	tree.refresh(null);
+	// Restore cursor
+	Gatherer.g_man.wait(false);
+    }
+
+    /** Listens for changes in the combobox as when one is detected attempts to compile a regular expression from whatever text was entered. If successful, or if the item chosen was a predefined filter, it then applies the filter to the target tree. */
+    private class ComboBoxListener
+	implements ActionListener {
+	/** Called when a new item is selected from the filter combobox, we treat the new entry as a pseudo-regular expression, compile it and then apply it to the tree.
+	 * @param event An <strong>ActionEvent</strong> containing more information about the change performed.
+	 * @see org.greenstone.gatherer.gui.Filter.Entry
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    try {
+		Object temp = combobox.getSelectedItem();
+		Entry entry = null;
+		if(temp instanceof String) {
+		    String temp_str = (String) temp;
+		    ///ystem.err.println("Filter = " + temp_str);
+
+		    // Ignore any string which matches a predefined filter
+		    if(temp_str.equals(Dictionary.get("Filter.All_Files"))) {
+		    }
+		    // HTM & HTML
+		    else if(temp_str.equals(Dictionary.get("Filter.0"))) {
+		    }
+		    // XML
+		    else if(temp_str.equals(Dictionary.get("Filter.1"))) {
+		    }
+		    // Text files
+		    else if(temp_str.equals(Dictionary.get("Filter.2"))) {
+		    }
+		    // Images
+		    else if(temp_str.equals(Dictionary.get("Filter.3"))) {
+		    }
+		    else {
+			// Make sure the filter isn't already in the list
+			boolean already_exists = false;
+			for (int i = 0; i < combobox.getItemCount(); i++) {
+			    if (temp_str.equals(combobox.getItemAt(i).toString())) {
+				already_exists = true;
+				entry = (Entry) combobox.getItemAt(i);
+				break;
+			    }
+			}
+
+			if (already_exists == false) {
+			    entry = new Entry(temp_str, Pattern.compile(encode(temp_str)));
+			    int position = combobox.getItemCount();
+			    combobox.insertItemAt(entry, position);
+			    combobox.setSelectedIndex(position);
+			}
+		    }
+		}
+		else {
+		    ///ystem.err.println("Custom Filter");
+		    entry = (Entry) temp;
+		}
+		if(entry != null) {
+		    setFilter(entry.getPattern());
+		}
+		// Else we ignore this event as being one of the painfully erroneous events we receive because we've been silly enough to have an editable combobox.
+	    }
+	    catch (PatternSyntaxException error) {
+		if(first) {
+		    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Filter.Invalid_Pattern"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+		}
+	    }
+	}
+    }
+    /** An object that holds a filter entry. This is string used for the filter pattern and, if not custom built, its name. */
+    private class Entry
+	implements Comparable {
+	/** The compiled pattern created from a regular expression. */
+	private Pattern pattern = null;
+	/** The name of this filter entry. */
+	private String name = null;
+	/** Constructor. */
+	public Entry() {
+	}
+	/** Constructor.
+	 * @param name The name of this entry as a <strong>String</strong>.
+	 * @param pattern The compiled regular expression as a <strong>Pattern</strong>.
+	 */
+	public Entry(String name, Pattern pattern) {
+	    this.name = name;
+	    this.pattern = pattern;
+	}
+	/** Compare two Entrys for ordering.
+	 * @param object The other Entry to compare to, as an <strong>Object</strong>.
+	 * @return An <i>int</i> indicating the respective ordering, as defined in java.lang.String#compareTo
+	 */
+	public int compareTo(Object object) {
+	    return toString().compareTo(object.toString());
+	}
+	/** Retrieve the pattern associated with this entry.
+	 * @return The <strong>Pattern</strong>.
+	 */
+	public Pattern getPattern() {
+	    return pattern;
+	}
+	/** Translate this entry into a textual representation.
+	 * @return A <strong>String</strong> containing the representation.
+	 */
+	public String toString() {
+	    String result = null;
+	    if (name != null) {
+		result = name;
+	    }
+	    else if (pattern == null) {
+		result = Dictionary.get("Filter.All_Files");
+	    }
+	    else {
+		result = pattern.pattern();
+	    }
+	    return result;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GComboBox.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GComboBox.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GComboBox.java	(revision 21112)
@@ -0,0 +1,351 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.basic.*;
+import org.greenstone.gatherer.Configuration;
+
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class GComboBox
+    extends JComboBox {
+
+    private Color background = null;
+    private Color foreground = null;
+    private Color editable_background = null;
+    private Color editable_foreground = null;
+    private Color selection_background = null;
+    private Color selection_foreground = null;
+
+    public GComboBox() {
+	super();
+	init();
+    }
+
+    public GComboBox(boolean editable) {
+	super();
+	setEditable(editable);
+	init();
+    }
+
+    public GComboBox(ArrayList data) {
+	super(data.toArray());
+	init();
+    }
+
+    public GComboBox(ArrayList data, boolean editable) {
+	super(data.toArray());
+	setEditable(editable);
+	init();
+    }
+
+    public GComboBox(ComboBoxModel model) {
+	super(model);
+	init();
+    }
+
+    public GComboBox(ComboBoxModel model, boolean editable) {
+	super(model);
+	setEditable(editable);
+	init();
+    }
+
+    public GComboBox(Object data[]) {
+	super(data);
+	init();
+    }
+
+    public GComboBox(Object data[], boolean editable) {
+	super(data);
+	setEditable(editable);
+	init();
+    }
+
+    public GComboBox(Vector data) {
+	super(data);
+	init();
+    }
+
+    public GComboBox(Vector data, boolean editable) {
+	super(data);
+	setEditable(editable);
+	init();
+    }
+
+    public int add(Object object) {
+	if (dataModel instanceof Model) {
+	    return ((Model) dataModel).add(object);
+	}
+	else {
+	    return -1;
+	}
+    }
+
+    public Object get(int index) {
+	return dataModel.getElementAt(index);
+    }
+
+    public void clear() {
+	if (dataModel instanceof Model) {
+	    ((Model) dataModel).clear();
+	}
+    }
+
+    public void init() {
+	Model model = new Model();
+	ComboBoxModel old_model = (ComboBoxModel) getModel();
+	setModel(model);
+	setOpaque(true);
+
+	// Restore any data given into our model
+	for(int i = 0; i < old_model.getSize(); i++) {
+	    model.add(old_model.getElementAt(i));
+	}
+
+	// Change component
+	UI ui = new UI();
+	setUI(ui);
+	ui.setButtonBackground();
+
+	// Initialization
+	this.background = Configuration.getColor("coloring.collection_tree_background", false);
+	this.foreground = Configuration.getColor("coloring.collection_tree_foreground", false);
+	this.editable_background = Configuration.getColor("coloring.editable_background", false);
+	this.editable_foreground = Configuration.getColor("coloring.editable_foreground", false);
+	this.selection_background = Configuration.getColor("coloring.collection_selection_background", false);
+	this.selection_foreground = Configuration.getColor("coloring.collection_selection_foreground", false);
+	if (isEditable()) {
+	    this.setBackground(editable_background);
+	    this.setForeground(editable_foreground);
+	}
+	else {
+	    this.setBackground(background);
+	    this.setForeground(foreground);
+	}
+	setBorder(BorderFactory.createLoweredBevelBorder());
+	setRenderer(new Renderer());
+    }
+
+    /** Overridden to do nothing. 
+     * @param background
+     */
+    public void setBackground(Color background) {
+    }
+
+    public void setBackgroundEditableColor(Color editable_background) {
+	this.editable_background = editable_background;
+    }
+
+    public void setBackgroundNonSelectionColor(Color background) {
+	this.background = background;
+    }
+
+    public void setBackgroundSelectionColor(Color selection_background) {
+	this.selection_background = selection_background;
+    }
+
+    public void setEditable(boolean editable) {
+	setEditor(new Editor());
+	super.setEditable(editable);
+	if (isEditable()) {
+	    this.setBackground(editable_background);
+	    this.setForeground(editable_foreground);
+	}
+	else {
+	    this.setBackground(background);
+	    this.setForeground(foreground);
+	}
+    }
+
+    public void setTextEditableColor(Color editable_foreground) {
+	this.editable_foreground = editable_foreground;
+    }
+
+    public void setTextNonSelectionColor(Color foreground) {
+	this.foreground = foreground;
+    }
+
+    public void setTextSelectionColor(Color selection_foreground) {
+	this.selection_foreground = selection_foreground;
+    }
+
+
+    private class Editor
+	extends JTextField
+	implements ComboBoxEditor {
+	public Editor() {
+	    setOpaque(true);
+	    if (isEditable()) {
+		setBackground(editable_background);
+		setForeground(editable_foreground);
+	    }
+	    else {
+		setBackground(background);
+		setForeground(foreground);
+	    }
+	    setSelectionColor(selection_background);
+	    setSelectedTextColor(selection_foreground);
+	}
+
+	public Component getEditorComponent() {
+	    return this;
+	}
+
+	public Object getItem() {
+	    return getText();
+	}
+
+	public void setItem(Object item) {
+	    if(item != null) {
+		setText(item.toString());
+	    }
+	    else {
+		setText("");
+	    }
+	}
+    }
+
+    private class Model
+	extends DefaultComboBoxModel {
+
+	public int add(Object extension) {
+	    int position = 0;
+	    String extension_str = extension.toString().toLowerCase();
+	    while(extension != null && position < getSize()) {
+		String sibling = getElementAt(position).toString().toLowerCase();
+		int order = extension_str.compareTo(sibling);
+		// If we are now less than the sibling, insert here.
+		if(order < 0) {
+		    insertElementAt(extension, position);
+		    extension = null;
+		}
+		// We are equal to the sibling, thus we already exist in list. Done.
+		else if(order == 0) {
+		    extension = null;
+		}
+		// Continue searching.
+		else {
+		    position++;
+		}
+	    }
+	    if(extension != null) {
+		position = getSize();
+		addElement(extension);
+	    }
+	    return position;
+	}
+
+	public void clear() {
+	    removeAllElements();
+	}
+    }
+
+    private class Renderer
+	extends JLabel
+	implements ListCellRenderer {
+	public Renderer() {
+	    super("");
+	    this.setOpaque(true);
+	}
+	public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+	    if(value != null) {
+		this.setText(value.toString());
+	    }
+	    else {
+		this.setText("");
+	    }
+	    if(isSelected) {
+		this.setBackground(selection_background);
+		this.setForeground(selection_foreground);
+	    }
+	    else if (isEditable()) {
+		this.setBackground(editable_background);
+		this.setForeground(editable_foreground);
+	    }
+	    else {
+		this.setBackground(background);
+		this.setForeground(foreground);
+	    }
+	    return this;
+	}
+    }
+
+    private class UI
+	extends BasicComboBoxUI {
+	public UI() {
+	    super();
+	}
+
+	protected ComboPopup createPopup() {
+	    BasicComboPopup popup = new BasicComboPopup(comboBox);
+	    // ---- I don't know why this code is here... maybe it is needed for the Mac? ----
+// 	    {
+// 		    public void show() {
+// 			if(comboBox.getMaximumRowCount() > 0) {
+// 			    Dimension popupSize = new Dimension( comboBox.getWidth(), getPopupHeightForRowCount( comboBox.getMaximumRowCount()));
+// 			    Rectangle popupBounds = new Rectangle( 0, 0, popupSize.width, popupSize.height);
+// 			    scroller.setMaximumSize( popupBounds.getSize() );
+// 			    scroller.setPreferredSize( popupBounds.getSize() );
+// 			    scroller.setMinimumSize( popupBounds.getSize() );
+// 			    list.invalidate();
+// 			    int selectedIndex = comboBox.getSelectedIndex();
+// 			    if ( selectedIndex == -1 ) {
+// 				list.clearSelection();
+// 			    }
+// 			    else {
+// 				list.setSelectedIndex( selectedIndex );
+// 			    }
+// 			    list.ensureIndexIsVisible( list.getSelectedIndex() );
+// 			    setLightWeightPopupEnabled( comboBox.isLightWeightPopupEnabled() );
+// 			    show(arrowButton, arrowButton.getSize().width - (popupSize.width + 4), arrowButton.getSize().height);
+// 			}
+// 		    }
+// 		};
+	    popup.getAccessibleContext().setAccessibleParent(comboBox);
+	    return popup;
+	}
+
+	public void setButtonBackground() {
+		arrowButton.setBackground(Configuration.getColor("coloring.button_background", false));
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GLIButton.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GLIButton.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GLIButton.java	(revision 21112)
@@ -0,0 +1,62 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import javax.swing.ImageIcon;
+import javax.swing.JButton;
+import org.greenstone.gatherer.util.Utility;
+
+/** A button that knows whether it should paint itself as opaque or otherwise. */
+public class GLIButton
+    extends JButton {
+
+    public GLIButton() {
+	super();
+	setOpaque(!Utility.isMac());
+    }
+
+    public GLIButton(ImageIcon icon) {
+	super(icon);
+	setOpaque(!Utility.isMac());
+    }
+
+    public GLIButton(String title) {
+	super(title);
+	setOpaque(!Utility.isMac());
+    }
+}
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GUIManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GUIManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GUIManager.java	(revision 21112)
@@ -0,0 +1,948 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.datatransfer.*;
+import java.awt.event.*;
+import java.io.File;
+import java.lang.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.filechooser.*;
+import javax.swing.plaf.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.Collection;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.file.FileOpenActionListener;
+import org.greenstone.gatherer.file.WorkspaceTree;
+import org.greenstone.gatherer.gui.metaaudit.MetaAuditFrame;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.TreeSynchronizer;
+import org.greenstone.gatherer.util.Utility;
+
+/** The GUIManager is in charge of creating the Gatherer window frame then filling it with the goodness of the view panes. GUIManager not only creates these panes, but allows some messaging between them. Furthermore GUIManager includes functionality from menu driven choices, simply as it was easier to put it here once and have it accessible from all pane children. */
+public class GUIManager
+    extends JFrame
+    implements ActionListener, ChangeListener {
+    /** The download pane contains controls for downloading internet sites. */
+    public DownloadPane download_pane = null;
+    /** The gather pane is more like a file manager where you drag files from one tree to another. */
+    private GatherPane gather_pane = null;
+    /** The enrich pane is used to assign, edit and remove metadata from files within the collection. */
+    public EnrichPane enrich_pane = null;
+    /** The design pane allows you to edit the design of the library in terms of the collection configuration file. */
+    public DesignPane design_pane = null;
+    /** The create pane contains scripting options for importing and building collections into libraries. */
+    public CreatePane create_pane = null;
+
+    public FileOpenActionListener foa_listener = new FileOpenActionListener();
+
+    /** A reference to the currently instantiated help window, if any. */
+    private HelpFrame help = null;
+    /** The menu bar. */
+    public MenuBar menu_bar = null;
+    public MetaAuditFrame meta_audit;
+    /** Are certain panes currently locked? */
+    private boolean locked = false;
+    /** The size of the Gatherer window. */
+    private Dimension size = null;
+    /** The filters used to dynamically filter the trees at user request. */
+    private HashMap filters = new HashMap();
+    /** The panel within the window that other components are placed on. */
+    private JPanel content_pane = null;
+    /** The last view pane selected. */
+    private JPanel previous_pane;
+    /** The main tab pane containing the different views, available here to trap view change events. */
+    private JTabbedPane tab_pane = null;
+    /** A threaded tab changer to try and avoid NPE on exit. */
+    private TabUpdater tab_updater = null;
+    /** Ensures that expansion events between like collection trees are synchronized. */
+    private TreeSynchronizer collection_tree_sync = null;
+
+
+    /**Constructor. Enable window events and arranges all other components.
+     * @param size The intial <strong>Dimension</strong> of the screen.
+     */
+    public GUIManager(Dimension size) {
+	super();
+	// Initialization
+	this.help = new HelpFrame();
+	this.size = size;
+	this.collection_tree_sync = new TreeSynchronizer();
+	this.meta_audit = new MetaAuditFrame(collection_tree_sync, null);
+	
+	this.setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+
+	// Add a focus listener to ourselves. Thus if we gain focus when a Modal Dialog should instead have it, we can try to bring the modal dialog to the fore.
+	this.addFocusListener(new GLIGUIFocusListener());
+
+	// Make the Tool tip hang around for a rediculous amount of time.
+	ToolTipManager.sharedInstance().setDismissDelay(10000);
+
+	// Set up some other UI stuff. (fonts handled in Gatherer.main())
+	UIManager.put("FileChooser.lookInLabelText", Dictionary.get("SaveCollectionBox.Look_In"));
+	UIManager.put("FileChooser.filesOfTypeLabelText", Dictionary.get("SaveCollectionBox.Files_Of_Type"));
+	UIManager.put("FileChooser.fileNameLabelText", Dictionary.get("SaveCollectionBox.File_Name"));
+    }
+
+    private class GLIGUIFocusListener
+	extends FocusAdapter {
+	public void focusGained(FocusEvent e) {
+	    if (ModalDialog.current_modal != null) {
+		ModalDialog.current_modal.makeVisible();
+		ModalDialog.current_modal.toFront();
+	    }
+	}
+    }
+
+    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured. In this case we are listening to actions from the menu-bar, and should react appropriately.
+     * @param event An <strong>ActionEvent</strong> containing information about the action that has occured.
+     */
+    public void actionPerformed(ActionEvent event) {
+	Object esrc = event.getSource();
+	// *************
+	// File Options.
+	// *************
+	if (esrc == menu_bar.file_associations) {
+	    Gatherer.assoc_man.edit();
+	}
+	else if (esrc == menu_bar.file_close) {
+	    saveThenCloseCurrentCollection();
+	}
+	else if (esrc == menu_bar.file_delete) {
+	    DeleteCollectionPrompt dcp = new DeleteCollectionPrompt();
+	    if (dcp.display()) {
+		saveThenCloseCurrentCollection();
+	    }
+	    dcp.destroy();
+	    dcp = null;
+	    System.gc();
+	}
+	else if (esrc == menu_bar.file_cdimage) {
+	    WriteCDImagePrompt wcdip = new WriteCDImagePrompt();
+	    wcdip.display();
+	    wcdip.destroy();
+	    wcdip = null;
+	}
+	else if (esrc == menu_bar.file_exportas) {
+	    ExportAsPrompt eap = new ExportAsPrompt();
+	    eap.display();
+	    eap.destroy();
+	    eap = null;
+	}
+	else if (esrc == menu_bar.file_exit) {
+	    exit();
+	}
+        else if (esrc == menu_bar.file_new) {
+	    new NewCollectionTask().start();
+	}
+	else if (esrc == menu_bar.file_open) {
+	    new OpenCollectionTask().start();
+	}
+	else if (esrc == menu_bar.file_options) {
+	    // Just incase the user has edited the GeneralSettings of a collection without losing focus afterwards. Well I'm forever losing foc... ooh shiney.
+	    design_pane.loseFocus();
+	    // And spawn a new preferences.
+	    new Preferences();
+	}
+	else if (esrc == menu_bar.file_save) {
+	    Gatherer.c_man.saveCollection();
+	}
+
+	// *************
+	// Edit Options.
+	// *************
+	else if(esrc == menu_bar.edit_copy) {
+	    try {
+		KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		// Get the component with selected text as a JTextComponent
+		JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();//getFocusOwner();
+		text.copy();
+	    }
+	    catch (Exception cce) {
+		// If the component is not a text component ignore the copy command
+		DebugStream.println(cce.toString());
+	    }
+	}
+	else if(esrc == menu_bar.edit_cut) {
+	    try {
+		KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		// Get the component with selected text as a JTextComponent
+		JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();
+		// Cut the text to the clipboard
+		text.cut();
+	    }
+	    catch (ClassCastException cce) {
+		// If the component is not a text component ignore the cut command
+		DebugStream.println(cce.toString());
+	    }
+	}
+	else if(esrc == menu_bar.edit_paste) {
+	    try {
+		KeyboardFocusManager kfm = KeyboardFocusManager.getCurrentKeyboardFocusManager();
+		// Get the component with selected text as a JTextComponent
+		JTextComponent text = (JTextComponent) kfm.getPermanentFocusOwner();
+		// Cut the text to the clipboard
+		text.paste();
+	    }
+	    catch (ClassCastException cce) {
+		// If the component is not a text component ignore the paste command
+		DebugStream.println(cce.toString());
+	    }
+	}
+
+	// *************
+	// Help Options.
+	// *************
+	else if (esrc == menu_bar.help_general) {
+	    HelpFrame.setView("introduction");
+	}
+	else if (esrc == menu_bar.help_download) {
+	    HelpFrame.setView("downloadingfiles");
+	}
+	else if (esrc == menu_bar.help_gather) {
+	    HelpFrame.setView("collectingfiles");
+	}
+	else if (esrc == menu_bar.help_enrich) {
+	    HelpFrame.setView("enrichingacollection");
+	}
+	else if (esrc == menu_bar.help_design) {
+	    HelpFrame.setView("designingacollection");
+	}
+	else if (esrc == menu_bar.help_create) {
+	    HelpFrame.setView("producingthecollection");
+	}
+	else if (esrc == menu_bar.help_about) {
+	    new AboutDialog(this);
+	}
+    }
+
+
+    /** Any actions that should happen after the display of the Gatherer window can be called here. Currently only updates the browser pane if it is active to work around bug in Mozilla renderer implementation.
+     */
+    public void afterDisplay() {
+	if (download_pane != null) {
+	    download_pane.afterDisplay();
+	}
+	enrich_pane.afterDisplay();
+    }
+
+
+    public void saveThenCloseCurrentCollection()
+    {
+	tab_pane.setSelectedComponent(gather_pane);
+	Gatherer.c_man.saveCollection();
+	Gatherer.c_man.closeCollection();
+    }
+
+
+    /** This is called when we're absolutely finished with the GLI. It is *not* called when the applet is suspended.
+     */
+    public void destroy()
+    {
+	// Destroying create pane ensures the latest log has been closed
+	if (create_pane != null) {
+	    create_pane.destroy();
+	}
+
+	// Deal to help
+	if (help != null) {
+	    help.destroy();
+	    help = null;
+	}
+    }
+
+
+    /** Enabled events on the window to be trapped, creates all the visual components, then builds the tab and other layouts.
+     */
+    public void display() {
+	content_pane = (JPanel) this.getContentPane();
+	// Enable window-type events to be fired.
+	enableEvents(AWTEvent.WINDOW_EVENT_MASK);
+	// Initialise and layout sub-components, plus other window dressing.
+	try {
+	    this.setSize(size);
+
+	    // Set the title
+	    String collection_title = null;
+	    String collection_name = null;
+	    if (Gatherer.c_man.ready()) {
+		Collection collection = Gatherer.c_man.getCollection();
+		collection_title = collection.getTitle();
+		collection_name = collection.getName();
+		collection = null;
+	    }
+	    setTitle(collection_title, collection_name);
+	    collection_title = null;
+	    collection_name = null;
+
+	    // Pretty corner icon
+	    this.setIconImage(JarTools.getImage("gatherer_small.gif").getImage());
+	    // BorderLayout for the main screen. I'll try my best to avoid these in subcomponents as they're space greedy.
+	    content_pane.setLayout(new BorderLayout());
+	    // Create the menu-bar and stick it up the top.
+	    menu_bar = new MenuBar(new MenuListenerImpl());
+
+	    //feedback changes
+	    //content_pane.add(menu_bar, BorderLayout.NORTH);
+	    this.setJMenuBar(menu_bar);
+	    // end feedback changes
+
+	    // Create the tabbed pane and plop it in the center where it will
+	    // expand to consume all available space like any good gas would.
+	    tab_pane = new JTabbedPane();
+	    tab_pane.addChangeListener(this);
+	    tab_pane.setFont(Configuration.getFont("general.font", false));
+
+	    if (Configuration.get("workflow.download", true)) {
+		download_pane = new DownloadPane();
+		// "GUI.Download_Tooltip" is used automatically
+		tab_pane.addTab("GUI.Download", JarTools.getImage("download.gif"), download_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(download_pane), Configuration.get("workflow.download", false));
+	    }
+
+	    gather_pane = new GatherPane(collection_tree_sync);
+	    gather_pane.display();
+	    if (Configuration.get("workflow.gather", true)) {
+		// "GUI.Gather_Tooltip" is used automatically
+		tab_pane.addTab("GUI.Gather", JarTools.getImage("gather.gif"), gather_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(gather_pane), Configuration.get("workflow.gather", false));
+	    }
+
+	    enrich_pane = new EnrichPane(collection_tree_sync);
+	    enrich_pane.display();
+	    if (Configuration.get("workflow.enrich", true)) {
+		// "GUI.Enrich_Tooltip" is used automatically
+		tab_pane.addTab("GUI.Enrich", JarTools.getImage("enrich.gif"), enrich_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(enrich_pane), false);
+	    }
+
+	    design_pane = new DesignPane();
+	    design_pane.display();
+	    if (Configuration.get("workflow.design", true)) {
+		// "GUI.Design_Tooltip" is used automatically
+		tab_pane.addTab("GUI.Design", JarTools.getImage("design.gif"), design_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(design_pane), false);
+	    }
+
+	    create_pane = new CreatePane();
+	    create_pane.display();
+	    if (Configuration.get("workflow.create", true)) {
+		// "GUI.Create_Tooltip" is used automatically
+		tab_pane.addTab("GUI.Create", JarTools.getImage("create.gif"), create_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(create_pane), false);
+	    }
+
+	    // Select the collect pane if it is available
+	    if (tab_pane.indexOfComponent(gather_pane) != -1) {
+		tab_pane.setSelectedComponent(gather_pane);
+	    }
+	    // Otherwise find the first tab that is enabled and select that.
+	    else {
+		for (int i = 0; i < tab_pane.getTabCount(); i++) {
+		    if (tab_pane.isEnabledAt(i)) {
+			tab_pane.setSelectedIndex(i);
+			break;
+		    }
+		}
+	    }
+
+	    Dictionary.register(tab_pane);
+	    content_pane.add(tab_pane, BorderLayout.CENTER);
+
+	    // Add an extra progress bar at the bottom of every screen when using a remote Greenstone server
+	    if (Gatherer.isGsdlRemote) {
+		JPanel remote_greenstone_server_progress_panel = new JPanel();
+		JLabel remote_greenstone_server_progress_label = new JLabel(Dictionary.get("RemoteGreenstoneServer.Progress"));
+		remote_greenstone_server_progress_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+		remote_greenstone_server_progress_panel.setLayout(new BorderLayout());
+		remote_greenstone_server_progress_panel.add(remote_greenstone_server_progress_label, BorderLayout.WEST);
+		remote_greenstone_server_progress_panel.add(RemoteGreenstoneServer.getProgressBar(), BorderLayout.CENTER);
+		content_pane.add(remote_greenstone_server_progress_panel, BorderLayout.SOUTH);
+	    }
+
+	    // Call refresh to update all controls to reflect current collection status.
+	    refresh(-1, Gatherer.c_man.ready());
+	}
+	catch (Exception e) {
+	    DebugStream.printStackTrace(e);
+	    // The GUI failing to build is an app killer
+	    e.printStackTrace();
+	    System.exit(1);
+	}
+    }
+
+
+    /** This method ensures that all the things needing saving are saved before Gatherer.exit() is called.
+     */
+    public void exit()
+    {
+	// If we have a collection open remember it for next time, then save it and close it
+	if (Gatherer.c_man.ready()) {
+	    Configuration.setString("general.open_collection", true, Gatherer.c_man.getCollectionFilePath());
+	    saveThenCloseCurrentCollection();
+	}
+	else {
+	    Configuration.setString("general.open_collection", true, null);
+	}
+
+	// Store the current position and size of the GLI for next time
+	Configuration.setBounds("general.bounds", true, getBounds());
+
+	// Save configuration
+	Configuration.save();
+
+	// Hide the main window
+	setVisible(false);
+
+	// If we're running as an applet we don't quit here (we quit when the browser calls GathererApplet.destroy())
+	if (!Gatherer.isApplet) {
+	    Gatherer.exit();
+	}
+    }
+
+
+    /** Retrieve the filter, or if one already exists, spawn a linked copy. */
+    public Filter getFilter(DragTree tree) {
+	Filter filter = (Filter) filters.get(tree.getModel());
+	if (filter == null) {
+	    filter = new Filter(tree, null);
+	    filters.put(tree.getModel(), filter);
+	    return filter;
+	}
+	return filter.spawn(tree);
+    }
+
+
+    /** This method is called when the collection is being built, and is used to disable all controls in all pane which could change the state of the collection.
+     */
+    public void lockCollection(boolean import_stage, boolean lock)
+    {
+	locked = lock;
+
+	if (import_stage) {
+	    int gather_pos = tab_pane.indexOfComponent(gather_pane);
+	    if (gather_pos != -1) {
+		tab_pane.setEnabledAt(gather_pos, !lock);
+	    }
+	    int enrich_pos = tab_pane.indexOfComponent(enrich_pane);
+	    if (enrich_pos != -1) {
+		tab_pane.setEnabledAt(enrich_pos, !lock);
+	    }
+	}
+
+	int design_pos = tab_pane.indexOfComponent(design_pane);
+	if (design_pos != -1) {
+	    tab_pane.setEnabledAt(design_pos, !lock);
+	}
+    }
+
+
+    public void modeChanged(int mode) {
+	// Set the title
+	String collection_title = null;
+	String collection_name = null;
+	if (Gatherer.c_man.ready()) {
+	    Collection collection = Gatherer.c_man.getCollection();
+	    collection_title = collection.getTitle();
+	    collection_name = collection.getName();
+	    collection = null;
+	}
+	setTitle(collection_title, collection_name);
+	collection_title = null;
+	collection_name = null;
+	// Now pass on the message to anyone who cares
+	if (download_pane != null) {
+	    download_pane.modeChanged(mode);
+	}
+	if (gather_pane != null) {
+	    gather_pane.modeChanged(mode);
+	}
+	if (enrich_pane != null) {
+	    enrich_pane.modeChanged(mode);
+	}
+	if (design_pane != null) {
+	    design_pane.modeChanged(mode);
+	}
+	if (create_pane != null) {
+	    create_pane.modeChanged(mode);
+	}
+    }
+
+
+    public void refresh(int refresh_reason, boolean collection_loaded)
+    {
+	// Set the collection information in the title bar
+	if (collection_loaded) {
+	    Collection collection = Gatherer.c_man.getCollection();
+	    setTitle(collection.getTitle(), collection.getName());
+	}
+	else {
+	    setTitle(null, null);
+	}
+
+	// Update the menu bar
+	menu_bar.refresh(refresh_reason, collection_loaded);
+
+	// Update the loaded panes
+	if (download_pane != null) {
+	    download_pane.refresh(refresh_reason, collection_loaded);
+	}
+	if (gather_pane != null) {
+	    gather_pane.refresh(refresh_reason, collection_loaded);
+	}
+	if (enrich_pane != null) {
+	    enrich_pane.refresh(refresh_reason, collection_loaded);
+	}
+	if (design_pane != null) {
+	    design_pane.refresh(refresh_reason, collection_loaded);
+	}
+	if (create_pane != null) {
+	    create_pane.refresh(refresh_reason, collection_loaded);
+	}
+
+	// Now enable tabs as necessary. Do this on event queue to prevent crazy NPEs
+	if (!locked) {
+	    if (tab_updater == null) {
+		tab_updater = new TabUpdater(tab_pane, collection_loaded);
+	    }
+	    else {
+		tab_updater.setReady(collection_loaded);
+	    }
+	    SwingUtilities.invokeLater(tab_updater);
+	}
+    }
+
+
+    public void refreshCollectionTree(int refresh_reason)
+    {
+	if (gather_pane != null) {
+	    gather_pane.refreshCollectionTree(refresh_reason);
+	}
+    }
+
+
+    public void refreshWorkspaceTree(int refresh_reason)
+    {
+	if (gather_pane != null) {
+	    gather_pane.refreshWorkspaceTree(refresh_reason);
+	}
+    }
+
+
+    /** Returns to some "initial" pane (when no collection is loaded) */
+    public void returnToInitialPane()
+    {
+	if (gather_pane != null) {
+	    tab_pane.setSelectedComponent(gather_pane);
+	}
+    }
+
+
+    /** Specifies whether a certain tab is enabled or not. */
+    private void setTabEnabled(String rawname, boolean state) {
+	// Retrieve the dictionary based name.
+	String name = Dictionary.get("GUI." + rawname);
+	int index = tab_pane.indexOfTab(name);
+	// Of course we may not have this tab available.
+	if(index != -1) {
+	    // Some tabs are also dependant on if a collection is ready
+	    Component component = tab_pane.getComponentAt(index);
+	    if(component == enrich_pane || component == design_pane || component == create_pane) {
+		tab_pane.setEnabledAt(index, state && Gatherer.c_man != null && Gatherer.c_man.ready());
+	    }
+	    else {
+		tab_pane.setEnabledAt(index, state);
+	    }
+	    // If this was the currently selected tab and it is now disabled, change the view to the first enabled tab.
+	    if(tab_pane.getSelectedIndex() == index && !state) {
+		boolean found = false;
+		for(int i = 0; !found && i < tab_pane.getTabCount(); i++) {
+		    if(tab_pane.isEnabledAt(i)) {
+			tab_pane.setSelectedIndex(i);
+			found = true;
+		    }
+		}
+		// If there are no tabs enabled, which should be impossible, then select the first tab
+		if(!found) {
+		    tab_pane.setSelectedIndex(0);
+		}
+	    }
+	}
+    }
+
+    /** Change the string shown in the title bar of the main gui frame. If either value is null, the 'No Collection' string is shown instead.
+     * @param title
+     * @param name
+     */
+    public void setTitle(String title, String name) {
+	// Finally display the collection name in the title bar.
+	StringBuffer title_buffer = new StringBuffer(Utility.PROGRAM_NAME);
+	title_buffer.append(StaticStrings.SPACE_CHARACTER);
+	title_buffer.append(StaticStrings.SPACE_CHARACTER);
+	// Describe the current user mode
+	title_buffer.append(StaticStrings.MODE_STR);
+	title_buffer.append(Configuration.getModeAsString());
+	title_buffer.append(StaticStrings.SPACE_CHARACTER);
+	title_buffer.append(StaticStrings.SPACE_CHARACTER);
+	// Now for the current collection
+	title_buffer.append(StaticStrings.COLLECTION_STR);
+	if (title != null && name != null) {
+	    title_buffer.append(title);
+	    title_buffer.append(StaticStrings.SPACE_CHARACTER);
+	    title_buffer.append(StaticStrings.OPEN_PARENTHESIS_CHARACTER);
+	    title_buffer.append(name);
+	    title_buffer.append(StaticStrings.CLOSE_PARENTHESIS_CHARACTER);
+	}
+	else {
+	    title_buffer.append(Dictionary.get("Collection.No_Collection"));
+	}
+	this.setTitle(title_buffer.toString());
+	title_buffer = null;
+    }
+
+
+    private class OpenCollectionTask
+        extends Thread
+    {
+	public void run()
+	{
+	    // If using a remote Greenstone check if the collection configurations have been downloaded; do so if not
+	    if (Gatherer.isGsdlRemote && !CollectionManager.downloaded_collection_configurations) {
+		if (RemoteGreenstoneServer.downloadCollectionConfigurations().equals("")) {
+		    // Something went wrong downloading the collection configurations
+		    return;
+		}
+
+		CollectionManager.downloaded_collection_configurations = true;
+	    }
+
+	    String collection_file_path = showOpenCollectionDialog();
+
+	    // User has selected a collection to open
+	    if (collection_file_path != null) {
+		// If there is already a collection open, save and close it
+		if (Gatherer.c_man.ready()) {
+		    saveThenCloseCurrentCollection();
+		}
+
+		// Open the selected collection
+		Gatherer.c_man.loadCollection(collection_file_path);
+	    }
+	}
+    }
+
+
+    /** When the load collection option is chosen this method is called to produce the modal file load prompt.
+     */
+    private String showOpenCollectionDialog()
+    {
+	// We first try the simple open collection dialog
+	SimpleOpenCollectionDialog dialog = new SimpleOpenCollectionDialog();
+	int user_choice = dialog.display();
+
+	// Used simple collection list
+	if (user_choice == SimpleOpenCollectionDialog.OK_OPTION) {
+	    return dialog.getFileName();
+	}
+
+	// Chosen to use the advanced 'browse' dialog
+	File collect_directory = null;
+	if (user_choice == SimpleOpenCollectionDialog.BROWSE_OPTION) {
+	    if (Gatherer.GS3) {
+		collect_directory = new File(Utility.getSitesDir(Configuration.gsdl3_path));
+	    }
+	    else {
+		collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	    }
+	    OpenCollectionDialog chooser = new OpenCollectionDialog(collect_directory);
+	    return chooser.getFileName();
+	}
+
+	// User must have cancelled the action
+	return null;
+    }
+
+
+    /** When called this method causes the MetadataAuditTable to display a nice dialog box which contains all the metadata assigned in the collection.
+     */
+    public void showMetaAuditBox() {
+	wait(true);
+	meta_audit.display();
+	wait(false);
+    }
+
+
+    private class NewCollectionTask
+	extends Thread
+    {
+	public void run()
+	{
+	    // If using a remote Greenstone check if the collection configurations have been downloaded; do so if not
+	    if (Gatherer.isGsdlRemote && !CollectionManager.downloaded_collection_configurations) {
+		if (RemoteGreenstoneServer.downloadCollectionConfigurations().equals("")) {
+		    // Something went wrong downloading the collection configurations
+		    return;
+		}
+
+		CollectionManager.downloaded_collection_configurations = true;
+	    }
+
+	    // Create the collection details prompt from new collection prompt
+	    NewCollectionMetadataPrompt ncm_prompt = null;
+	    NewCollectionDetailsPrompt ncd_prompt = new NewCollectionDetailsPrompt();
+	    // If no previous collection was indicated as a model design, then show the metadata selection prompt from new collection prompt
+	    if(!ncd_prompt.isCancelled() && (ncd_prompt.getBase() == null)) {
+		ncm_prompt = new NewCollectionMetadataPrompt();
+	    }
+	    // Create the new collection (if not cancelled) in a new thread.
+	    if (!ncd_prompt.isCancelled() && (ncm_prompt == null || !ncm_prompt.isCancelled())) {
+		// If there is already a collection open, save and close it.
+		if (Gatherer.c_man.ready()) {
+		    saveThenCloseCurrentCollection();
+		}
+
+		// Create new collection.
+		if (ncm_prompt == null) {
+		    Gatherer.c_man.createCollection(ncd_prompt.getDescription(), Configuration.getEmail(), ncd_prompt.getName(), ncd_prompt.getTitle(), ncd_prompt.getBase(), null);
+		}
+		else {
+		    Gatherer.c_man.createCollection(ncd_prompt.getDescription(), Configuration.getEmail(), ncd_prompt.getName(), ncd_prompt.getTitle(), null, ncm_prompt.getSets());
+		    ncm_prompt.dispose();
+		}
+
+		ncd_prompt.dispose();
+
+		// Return to some initial pane (Gather)
+		returnToInitialPane();
+
+		// Refresh the workspace tree to allow for the new collection
+		refreshWorkspaceTree(WorkspaceTree.LIBRARY_CONTENTS_CHANGED);
+
+		// Refresh the rest of the GLI
+		Gatherer.refresh(Gatherer.COLLECTION_OPENED);
+	    }
+
+	    // Done
+	    ncd_prompt = null;
+	    ncm_prompt = null;
+	}
+    }
+
+
+    /** Any implementation of ChangeListener must include this method so we can be informed when the state of one of the registered objects changes. In this case we are listening to view changes within the tabbed pane.
+     * @param event A ChangeEvent containing information about the event that fired this call.
+     */
+    public void stateChanged(ChangeEvent event)
+    {
+	// Tell the previous pane that it has lost focus
+	if (previous_pane != null) {
+	    if (previous_pane == enrich_pane) {
+		enrich_pane.loseFocus();
+	    }
+	    else if (previous_pane == design_pane) {
+		design_pane.loseFocus();
+	    }
+	    else if (previous_pane == create_pane) {
+		create_pane.loseFocus();
+	    }
+	}
+
+	menu_bar.tabSelected(tab_pane.getSelectedIndex());
+	int selected_index = tab_pane.getSelectedIndex();
+	if (selected_index == tab_pane.indexOfComponent(download_pane)) {
+	    download_pane.gainFocus();
+	}
+	else if (selected_index == tab_pane.indexOfComponent(gather_pane)) {
+	    gather_pane.gainFocus();
+	}
+	else if (selected_index == tab_pane.indexOfComponent(enrich_pane)) {
+	    enrich_pane.gainFocus();
+	}
+	else if (selected_index == tab_pane.indexOfComponent(design_pane)) {
+	    design_pane.gainFocus();
+	}
+	else if (selected_index == tab_pane.indexOfComponent(create_pane)) {
+	    create_pane.gainFocus();
+	}
+
+	previous_pane = (JPanel) tab_pane.getSelectedComponent();
+    }
+
+
+    private MouseListener mouse_blocker_listener = new MouseAdapter() {};
+
+    public void updateUI()
+    {
+	JPanel pane = (JPanel) getContentPane();
+	pane.updateUI();
+	// Also update all of the tabs according to workflow.
+	workflowUpdate("Download", Configuration.get("workflow.download", false));
+	workflowUpdate("Gather", Configuration.get("workflow.gather", false));
+	workflowUpdate("Enrich", Configuration.get("workflow.enrich", false));
+	workflowUpdate("Design", Configuration.get("workflow.design", false));
+	workflowUpdate("Create", Configuration.get("workflow.create", false));
+    }
+
+    public void wait(boolean waiting) {
+	Component glass_pane = getGlassPane();
+	if(waiting) {
+	    // Show wait cursor.
+	    glass_pane.addMouseListener(mouse_blocker_listener);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	    glass_pane.setVisible(true);
+	}
+	else {
+	    // Hide wait cursor.
+	    glass_pane.setVisible(false);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    glass_pane.removeMouseListener(mouse_blocker_listener);
+	}
+	glass_pane = null;
+    }
+
+    public void workflowUpdate(String raw, boolean state) {
+	WorkflowUpdater task = new WorkflowUpdater(raw, state);
+	SwingUtilities.invokeLater(task);
+	task = null;
+    }
+
+
+    /**Overridden from JFrame so we can exit safely when window is closed (or destroyed).
+     * @param event A <strong>WindowEvent</strong> containing information about the event that fired this call.
+     */
+    protected void processWindowEvent(WindowEvent event) {
+	if(event.getID() == WindowEvent.WINDOW_CLOSING) {
+	    exit();
+	}
+    }
+
+
+    /** Listens to actions upon the menu bar, and if it detects a click over the help menu brings the help window to the front if it has become hidden.
+     */
+    private class MenuListenerImpl
+	implements MenuListener {
+	/** Called whenever a popup menu is hidden, but we don't care.
+	 * @param e Some <strong>MenuEvent</strong> that we could care less about.
+	 */
+	public void menuCanceled(MenuEvent e) {
+	}
+	/** Called whenever a menu header (ie button) becomes unselected, but we don't care.
+	 * @param e Some <strong>MenuEvent</strong> that we could care less about.
+	 */
+	public void menuDeselected(MenuEvent e) {
+	}
+	/** This method, when a menu is first opened, is the only one we respond to by bringing the help window to the front if possible, but only if there is a help window and the help menu is the one opening.
+	 * @param e The <strong>MenuEvent</strong> whose source is checked.
+	 */
+	public void menuSelected(MenuEvent e) {
+	    if(e.getSource() == menu_bar.help) {
+		if(menu_bar.help.isSelected()) {
+		    menu_bar.help.doClick(10);
+		}
+	    }
+	}
+    }
+
+    private class TabUpdater
+	implements Runnable {
+	private boolean ready = false;
+	private int download_pos = -1;
+	private int enrich_pos = -1;
+	private int design_pos = -1;
+	private int create_pos = -1;
+	private int export_pos = -1;
+	private JTabbedPane tab_pane = null;
+
+	public TabUpdater(JTabbedPane tab_pane, boolean ready) {
+	    this.ready = ready;
+	    this.tab_pane = tab_pane;
+	    download_pos = tab_pane.indexOfComponent(download_pane);
+	    enrich_pos = tab_pane.indexOfComponent(enrich_pane);
+	    design_pos = tab_pane.indexOfComponent(design_pane);
+	    create_pos = tab_pane.indexOfComponent(create_pane);
+	}
+
+	public void run()
+	{
+	    if (download_pos != -1) {
+		if (ready) {
+		    tab_pane.setEnabledAt(download_pos, Configuration.get("workflow.download", false));
+		}
+		else {
+		    tab_pane.setEnabledAt(download_pos, Configuration.get("workflow.download", true));
+		}
+	    }
+	    if (enrich_pos != -1) {
+		tab_pane.setEnabledAt(enrich_pos, ready && Configuration.get("workflow.enrich", false));
+	    }
+	    if (design_pos != -1) {
+		tab_pane.setEnabledAt(design_pos, ready && Configuration.get("workflow.design", false) && Configuration.getMode() > Configuration.ASSISTANT_MODE);
+	    }
+	    if (create_pos != -1) {
+		tab_pane.setEnabledAt(create_pos, ready && Configuration.get("workflow.create", false));
+	    }
+	}
+
+	public void setReady(boolean ready) {
+	    this.ready = ready;
+	}
+    }
+
+    private class WorkflowUpdater
+	implements Runnable {
+	private boolean state;
+	private String raw;
+	public WorkflowUpdater(String raw, boolean state) {
+	    this.raw = raw;
+	    this.state = state;
+	}
+	public void run() {
+	    setTabEnabled(raw, state);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GatherPane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GatherPane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/GatherPane.java	(revision 21112)
@@ -0,0 +1,933 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTree;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.file.FileNode;
+import org.greenstone.gatherer.file.FileOpenActionListener;
+import org.greenstone.gatherer.file.FileQueue;
+import org.greenstone.gatherer.file.FileSystemModel;
+import org.greenstone.gatherer.file.RecycleBin;
+import org.greenstone.gatherer.file.WorkspaceTree;
+import org.greenstone.gatherer.file.WorkspaceTreeNode;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.gui.ExplodeMetadataPrompt;
+import org.greenstone.gatherer.util.DragComponent;
+import org.greenstone.gatherer.util.DragGroup;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.TreeSynchronizer;
+import org.greenstone.gatherer.util.Utility;
+
+/** The collection pane is analogous with a file manager. It is there that the user chooses which files to include in their collection and what structure the file hierarchy should take. The later aspect is not important for the Greenstone Suite, but is usefull for grouping files for ease of metadata markup. The view essentially consists of two file trees, one denoting the entire source workspace and the other the files within your collection. The trees themselves have a title bar at the top, a filter control at the bottom, and are coloured to indicate activity (grey for disabled). The remainder of the screen is taken by a status area, to indicate current file job progress during copying etc, and three buttons for controlling features of the view.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class GatherPane
+    extends JPanel
+    implements ActionListener, FocusListener {
+    /** The group encompassing all of the components available as drop targets for drag and drop actions. Required so that only one component renders the ghost and higlights itself as a target, which the other members are restored to their original, pristine, condition. */
+    private DragGroup group				  =  null;
+    /** The tree showing the files within the collection. */
+    private CollectionTree collection_tree = null;
+    /** The threaded queue that handles the actually movement of files, so that the gui remains responsive. */
+    private FileQueue file_queue		  =  null;
+    /** The filter currently applied to the collection tree. */
+    private Filter collection_filter = null;
+    /** The filter currently applied to the workspace tree. */
+    private Filter workspace_filter = null;
+    /** The button used to cancel all pending file queue jobs. */
+    private JButton stop_action		   =  null;
+    /** The button used to create a new folder in the collection tree. */
+    private JButton new_folder			=  null;
+    /** The label shown at the top of the collection tree. */
+    private JLabel collection_label	   =  null;
+    /** The label shown in the status area explaining the file apon which action is taking place. */
+    private JLabel filename_label		 =  null;
+    /** The label shown explaining the current state of the file queue thread. */
+    private JLabel status_label		   =  null;
+    /** The label at the top of the workspace tree. */
+    private JLabel workspace_label		=  null;
+    /** The panel that contains the collection tree. */
+    private JPanel collection_pane		=  null;
+    /** The panel that contains the various controls including the status area. */
+    private JPanel control_pane		   =  null;
+    /** The panel that contains the workspace tree. */
+    private JPanel workspace_pane		 =  null;
+    /** The scrollable area into which the collection tree is placed. */
+    private JScrollPane collection_scroll =  null;
+    /** The scrollable area into which the workspace tree is placed. */
+    private JScrollPane workspace_scroll  =  null;
+    /** A split pane seperating the two trees, allowing for the screen real-estate for each to be changed. */
+    private JSplitPane tree_pane		  =  null;
+    /** Text fragment arguments used to fill in phrases returned from the dictionary. */
+    private String args[]				 =  null;
+    /** Ensures that expansion and selection events between collection trees based on the same model are synchronized. */
+    private TreeSynchronizer collection_tree_sync = null;
+    /** The button used to delete files, which also doubles as a drop target for files from the Trees. */
+    private RecycleBin bin_button			=  null;
+    /** The default size of a special mapping dialog. */
+    static final Dimension DIALOG_SIZE = new Dimension(400, 120);
+    /** The minimum size a gui component can become. */
+    static private Dimension MIN_SIZE	= new Dimension( 90,  90);
+    /** The default size of the status area. */
+    static private Dimension STATUS_SIZE = new Dimension(450, 120);
+    /** The initial size of the trees. */
+    static private Dimension TREE_SIZE   = new Dimension(400, 430);
+
+    /** The tree showing the available source workspace. */
+    private WorkspaceTree workspace_tree = null;
+
+
+    /* Constructor.
+     * @param tree_sync Ensures that expansion events between like trees are synchronized.
+     * @see org.greenstone.gatherer.file.FileManager
+     * @see org.greenstone.gatherer.file.FileQueue
+     */
+    public GatherPane(TreeSynchronizer collection_tree_sync) {
+	this.group = new DragGroup();
+	this.file_queue = Gatherer.f_man.getQueue();
+	this.collection_tree_sync = collection_tree_sync;
+
+	// Create components.
+	stop_action = new GLIButton();
+	stop_action.addActionListener(this);
+	stop_action.setEnabled(false);
+	stop_action.setMnemonic(KeyEvent.VK_S);
+	file_queue.registerStopButton(stop_action);
+	Dictionary.registerBoth(stop_action, "Collection.Stop", "Collection.Stop_Tooltip");
+	
+	new_folder = new GLIButton(JarTools.getImage("folder.gif"));
+	new_folder.addActionListener(this);
+	new_folder.setEnabled(false);
+	new_folder.setMinimumSize(MIN_SIZE);
+	new_folder.setMnemonic(KeyEvent.VK_N);
+	new_folder.setPreferredSize(MIN_SIZE);
+	Dictionary.registerTooltip(new_folder, "Collection.New_Folder_Tooltip");
+    }
+
+    /** Any implementation of ActionListener requires this method so that when an action is performed the appropriate effect can occur. In this case there are three valid possibilities. If the action occured on the recycle bin, then delete the current selection from the collection tree. If the action instead occured on the new folder button, then create a new folder under the current (single) selection in the collection tree. And finally if the cancel button was pressed, cancel the current, and remaining, jobs on the file queue. */
+    public void actionPerformed(ActionEvent event) {
+	// If a user has clicked on the bin button directly remove whatever
+	// files are selected in the active tree.
+	if(event.getSource() == bin_button) {
+	    if(!bin_button.ignore()) {
+		// Find the active tree (you've made selections in).
+		DragTree tree = (DragTree) group.getActive();
+		// Fudge things a bit
+		group.setSource(tree);
+		// Determine the selection.
+		TreePath paths[] = tree.getSelectionPaths();
+		if(paths != null) {
+		    FileNode[] source_nodes = new FileNode[paths.length];
+		    for(int i = 0; i < paths.length; i++) {
+			source_nodes[i] = (FileNode)(paths[i].getLastPathComponent());
+		    }
+		    Gatherer.f_man.action(tree, source_nodes, bin_button, null);
+		}
+	    }
+	}
+	// If a user has clicked on new_folder create a new folder under
+	// whatever node is selected.
+	else if(event.getSource() == new_folder && collection_tree != null) {
+	    int count = collection_tree.getSelectionCount();
+	    boolean error = false;
+	    if(count == 1) {
+		TreePath path = collection_tree.getSelectionPath();
+		CollectionTreeNode node = (CollectionTreeNode) path.getLastPathComponent();
+		if (node.getAllowsChildren()) {
+		    Gatherer.f_man.newFolder(collection_tree, node);
+		}
+		else {
+		    // try the parent
+		    CollectionTreeNode parent = (CollectionTreeNode) node.getParent();
+		    if (parent!=null && parent.getAllowsChildren()) {
+			Gatherer.f_man.newFolder(collection_tree, parent);
+		    } else {
+			error = true;
+		    }
+		}
+	    }
+	    else {
+		error = true;
+	    }
+	    if(error) {
+		// instead of an error, we now create a new folder at the root
+		CollectionTreeNode node = (CollectionTreeNode) collection_tree.getModel().getRoot();
+		Gatherer.f_man.newFolder(collection_tree, node);
+	    }
+	}
+	else if(event.getSource() == stop_action) {
+	    file_queue.cancelAction();
+	}
+    }
+
+
+    /** Generates the pane on controls used to 'collect' files into the collection. Resposible for creating, connecting and laying out these controls. */
+    public void display() {
+	// Create Components.
+	KeyListenerImpl key_listener = new KeyListenerImpl();
+	MouseListenerImpl mouse_listener = new MouseListenerImpl();
+	this.addKeyListener(key_listener);
+
+	// Workspace Tree
+	workspace_pane = new JPanel();
+	workspace_pane.setMinimumSize(MIN_SIZE);
+	workspace_pane.setPreferredSize(TREE_SIZE);
+	workspace_pane.setSize(TREE_SIZE);
+
+	workspace_label = new JLabel();
+	workspace_label.setOpaque(true);
+	workspace_label.setBackground(Configuration.getColor("coloring.workspace_heading_background", false));
+	workspace_label.setForeground(Configuration.getColor("coloring.workspace_heading_foreground", false));
+	Dictionary.registerText(workspace_label, "Collection.Workspace");
+
+	workspace_tree = new WorkspaceTree(Utility.WORKSPACE_TREE);
+	group.add(workspace_tree);
+	workspace_tree.addFocusListener(this);
+	workspace_tree.addKeyListener(key_listener);
+	workspace_tree.addMouseListener(mouse_listener);
+	workspace_tree.addMouseListener(Gatherer.g_man.foa_listener);
+	workspace_tree.addTreeExpansionListener(Gatherer.g_man.foa_listener);
+	workspace_tree.putClientProperty("JTree.lineStyle", "Angled");
+	workspace_tree.setBackgroundNonSelectionColor(Configuration.getColor("coloring.workspace_tree_background", false));
+	workspace_tree.setTextNonSelectionColor(Configuration.getColor("coloring.workspace_tree_foreground", false));
+	workspace_tree.setBackgroundSelectionColor(Configuration.getColor("coloring.workspace_selection_background", false));
+	workspace_tree.setTextSelectionColor(Configuration.getColor("coloring.workspace_selection_foreground", false));
+	workspace_tree.setRootVisible(false);
+
+	workspace_scroll = new JScrollPane(workspace_tree);
+
+	workspace_filter = Gatherer.g_man.getFilter(workspace_tree);
+	workspace_filter.setBackground(Configuration.getColor("coloring.workspace_heading_background", false));
+	workspace_filter.setEditable(Configuration.getMode() > Configuration.LIBRARIAN_MODE);
+	Dictionary.registerTooltip(workspace_filter.getComboBox(), "Collection.Filter_Tooltip");
+
+	// Collection Tree
+	collection_pane = new JPanel();
+	collection_pane.setMinimumSize(MIN_SIZE);
+	collection_pane.setPreferredSize(TREE_SIZE);
+	collection_pane.setSize(TREE_SIZE);
+
+	collection_label = new JLabel();
+	collection_label.setOpaque(true);
+	Dictionary.registerText(collection_label, "Collection.No_Collection");
+
+	collection_tree = new CollectionTree(Utility.COLLECTION_TREE, Gatherer.c_man.getCollectionTreeModel(), true);
+	collection_tree.setEnabled(Gatherer.c_man.getCollectionTreeModel() != null);
+	group.add(collection_tree);
+	collection_tree.addFocusListener(this);
+	collection_tree.addKeyListener(key_listener);
+	collection_tree.addMouseListener(mouse_listener);
+	collection_tree.addMouseListener(Gatherer.g_man.foa_listener);
+	collection_tree.addTreeExpansionListener(Gatherer.g_man.foa_listener);
+	collection_tree.putClientProperty("JTree.lineStyle", "Angled");
+	collection_tree.setBackgroundNonSelectionColor(Configuration.getColor("coloring.collection_tree_background", false));
+	collection_tree.setTextNonSelectionColor(Configuration.getColor("coloring.collection_tree_foreground", false));
+	collection_tree.setBackgroundSelectionColor(Configuration.getColor("coloring.collection_selection_background", false));
+	collection_tree.setTextSelectionColor(Configuration.getColor("coloring.collection_selection_foreground", false));
+	collection_tree.setRootVisible(false);
+
+	collection_scroll = new JScrollPane(collection_tree);
+
+	collection_filter = Gatherer.g_man.getFilter(collection_tree);
+	collection_filter.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	collection_filter.setEditable(Configuration.getMode() > Configuration.LIBRARIAN_MODE);
+	Dictionary.registerTooltip(collection_filter.getComboBox(), "Collection.Filter_Tooltip");
+
+	tree_pane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
+
+	// Status pane
+	control_pane = new JPanel();
+
+	JPanel inner_pane = new JPanel();
+	inner_pane.setSize(STATUS_SIZE);
+
+	JPanel file_pane = new JPanel();
+	JPanel progress_pane = new JPanel();
+	JLabel file_status = file_queue.getFileStatus();
+
+	JProgressBar progress_bar = file_queue.getProgressBar();
+
+	JPanel button_pane = new JPanel();
+
+	bin_button = new RecycleBin();
+	bin_button.addActionListener(this);
+	bin_button.setEnabled(false);
+	bin_button.setMinimumSize(MIN_SIZE);
+	bin_button.setPreferredSize(MIN_SIZE);
+	Dictionary.registerTooltip(bin_button, "Collection.Delete_Tooltip");
+	group.add(bin_button);
+
+	// Layout Components.
+	workspace_pane.setLayout(new BorderLayout());
+	workspace_pane.add(workspace_label, BorderLayout.NORTH);
+	workspace_pane.add(workspace_scroll, BorderLayout.CENTER);
+	workspace_pane.add(workspace_filter, BorderLayout.SOUTH);
+
+	collection_pane.setLayout(new BorderLayout());
+	collection_pane.add(collection_label, BorderLayout.NORTH);
+	collection_pane.add(collection_scroll, BorderLayout.CENTER);
+	collection_pane.add(collection_filter, BorderLayout.SOUTH);
+
+	tree_pane.add(workspace_pane, JSplitPane.LEFT);
+	tree_pane.add(collection_pane, JSplitPane.RIGHT);
+	tree_pane.setDividerLocation(TREE_SIZE.width - 10);
+
+	file_pane.setBorder(BorderFactory.createEmptyBorder(2,2,2,2));
+	file_pane.setLayout(new BorderLayout());
+	file_pane.add(file_status, BorderLayout.CENTER);
+	file_pane.add(stop_action, BorderLayout.EAST);
+
+	progress_pane.setBorder(BorderFactory.createEmptyBorder(2,2,2,2));
+	progress_pane.setLayout(new BorderLayout());
+	progress_pane.add(progress_bar, BorderLayout.CENTER);
+
+	inner_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(10,10,10,10), BorderFactory.createLoweredBevelBorder()));
+	inner_pane.setLayout(new GridLayout(2,1));
+	inner_pane.add(file_pane);
+	inner_pane.add(progress_pane);
+
+	button_pane.add(new_folder);
+	button_pane.add(bin_button);
+
+	control_pane.setLayout(new BorderLayout());
+	control_pane.add(inner_pane, BorderLayout.CENTER);
+	control_pane.add(button_pane, BorderLayout.EAST);
+
+	this.setLayout(new BorderLayout());
+	this.add(tree_pane, BorderLayout.CENTER);
+	this.add(control_pane, BorderLayout.SOUTH);
+    }
+    /** This method ensures that a certain tree path is visible and selected within the collection tree, expanding nodes if necessary. If the method is successful the bounds of the new selection are returned. */
+    public Rectangle expandPath(TreePath path) {
+	collection_tree.setImmediate(true);
+	collection_tree.scrollPathToVisible(path);
+	collection_tree.setSelectionPath(path);
+	collection_tree.setImmediate(false);
+	return collection_tree.getRowBounds(collection_tree.getRowForPath(path));
+    }
+    /** Called whenever this pane gains focus, this method ensures that the various tree renderers are correctly colouring the tree (as these settings sometimes get lost).
+     * @param event A <strong>FocusEvent</strong> containing details about the focus action performed.
+     */
+    public void focusGained(FocusEvent event) {
+	DefaultTreeCellRenderer def = new DefaultTreeCellRenderer();
+	DefaultTreeCellRenderer w = (DefaultTreeCellRenderer)workspace_tree.getCellRenderer();
+	DefaultTreeCellRenderer c = (DefaultTreeCellRenderer)collection_tree.getCellRenderer();
+	if(event.getSource() == workspace_tree) {
+	    w.setBackgroundSelectionColor(def.getBackgroundSelectionColor());
+	    c.setBackgroundSelectionColor(Color.lightGray);
+	}
+	else if(event.getSource() == collection_tree) {
+	    c.setBackgroundSelectionColor(def.getBackgroundSelectionColor());
+	    w.setBackgroundSelectionColor(Color.lightGray);
+	}
+	repaint();
+    }
+    /** Implementation side-effect, not used in any way.
+     * @param event A <strong>FocusEvent</strong> containing details about the focus action performed.
+     */
+    public void focusLost(FocusEvent event) {
+    }
+
+    /** Called to inform this control panel that it has just gained focus as an effect of the user clicking on its tab.
+     */
+    public void gainFocus() {
+	// Update the meta-audit view to show the current selection, if any.
+	Gatherer.g_man.meta_audit.setRecords(getCollectionTreeSelection());
+    }
+
+    /** Retrieve a list of the currently selected file records in the collection tree. */
+    private CollectionTreeNode[] getCollectionTreeSelection()
+    {
+	TreePath paths[] = collection_tree.getSelectionPaths();
+	CollectionTreeNode records[] = null;
+	if (paths != null) {
+	    records = new CollectionTreeNode[paths.length];
+	    for (int i = 0; i < records.length; i++) {
+		records[i] = (CollectionTreeNode) paths[i].getLastPathComponent();
+	    }
+	}
+	return records;
+    }
+
+    
+    /** Called whenever the detail mode changes to ensure the filters are at an appropriate level (ie only editable by those that understand regular expression matching)
+     * @param mode the mode level as an int
+     */
+    public void modeChanged(int mode) {
+	collection_filter.setEditable(mode > Configuration.LIBRARIAN_MODE);
+	workspace_filter.setEditable(mode > Configuration.LIBRARIAN_MODE);
+    }
+
+
+    /** Refresh this pane, depending on what has just happened (refresh_reason). */
+    public void refresh(int refresh_reason, boolean collection_loaded)
+    {
+	if (collection_loaded) {
+	    // Update collection label
+	    Dictionary.registerText(collection_label, "Collection.Collection");
+	    collection_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	    collection_label.setForeground(Configuration.getColor("coloring.collection_heading_foreground", false));
+
+	    // Update collection tree
+	    if (refresh_reason == Gatherer.COLLECTION_OPENED) {
+		collection_tree.setModel(Gatherer.c_man.getCollectionTreeModel());
+	    }
+
+	    // Update collection filter
+	    collection_filter.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+	}
+	else {
+	    // Update collection label
+	    String[] args = new String[1];
+	    args[0] = Dictionary.get("Collection.No_Collection");
+	    Dictionary.registerText(collection_label, "Collection.Collection", args);
+	    collection_label.setBackground(Color.lightGray);
+	    collection_label.setForeground(Color.black);
+
+	    // Update collection tree
+	    collection_tree.setModel(new DefaultTreeModel(new DefaultMutableTreeNode("Error")));
+
+	    // Update collection filter
+	    collection_filter.setBackground(Color.lightGray);
+	}
+
+	// Enable or disable the controls
+	workspace_tree.setEnabled(true);
+	collection_tree.setEnabled(collection_loaded);
+	collection_filter.setEnabled(collection_loaded);
+	bin_button.setEnabled(collection_loaded);
+	new_folder.setEnabled(collection_loaded);
+
+	// Ensure that this collection tree view is synchronized with all others
+	collection_tree_sync.add(collection_tree);
+    }
+
+
+    public void refreshCollectionTree(int refresh_reason) {
+	collection_tree.refresh(null);
+    }
+
+
+    public void refreshWorkspaceTree(int refresh_reason) {
+	workspace_tree.refresh(refresh_reason);
+    }
+    
+
+    /** When a user right-clicks within the workspace and collection trees they are presented with a small popup menu of context based options. This class provides such functionality.
+     */
+    private class RightClickMenu
+	extends JPopupMenu
+	implements ActionListener {
+
+	/** The tree over which the right click action occurred. */
+	private DragTree tree = null;
+	/** The tree nodes selected when the right click action occurred. */
+	private TreePath[] selection_paths = null;
+	/** The file record over which the right click action occurred. */
+	private FileNode node = null;
+
+	private JMenuItem create_shortcut = null;
+	private JMenuItem delete_shortcut = null;
+	private JMenuItem collapse_folder = null;
+	private JMenuItem expand_folder = null;
+	private JMenuItem explode_metadata_database = null;
+	private JMenuItem delete = null;
+	private JMenuItem metaaudit = null;
+	private JMenuItem new_folder = null;
+	private JMenuItem new_dummy_doc = null;
+	private JMenuItem open_externally = null;
+
+
+	private RightClickMenu(DragTree tree, MouseEvent event)
+	{
+	    super();
+	    this.tree = tree;
+
+	    // Note we have to use setImmediate() with the set selction paths
+	    // otherwise the selection doesn't get updated until after the 
+	    // popup comes up.
+
+	    // the right click position
+	    TreePath right_click_path = tree.getPathForLocation(event.getX(), event.getY());
+	    if (right_click_path == null) {
+		// user has clicked outside of the tree, clear the selection
+		selection_paths = null;
+		tree.setImmediate(true);
+		tree.clearSelection();
+		tree.setImmediate(false);
+	    } else {
+		// Get the paths currently selected in the tree
+		selection_paths = tree.getSelectionPaths();
+		if (selection_paths == null) {
+		    // nothing currently selected - we shift the selection to 
+		    // the node that was right clicked on
+		    selection_paths = new TreePath[1];
+		    selection_paths[0] = right_click_path;
+		    tree.setImmediate(true);
+		    tree.setSelectionPath(right_click_path);
+		    tree.setImmediate(false);
+		} else if (selection_paths.length == 1 && ! selection_paths[0].equals( right_click_path)) {
+		    tree.setImmediate(true);
+		    tree.clearSelection();
+		    tree.setSelectionPath(right_click_path);
+		    tree.setImmediate(false);
+		    selection_paths[0] = right_click_path;
+		} else {
+		    // we had multiply selected paths in the tree.
+		    // if we clicked on one of those paths, then use all the 
+		    // current selection, otherwise clear the selection and 
+		    // select the one we right clicked on
+		    boolean clicked_in_selection = false;
+		    for (int i=0; i<selection_paths.length; i++) {
+			if (selection_paths[i].equals(right_click_path)) {
+			    clicked_in_selection = true;
+			    break;
+			}
+		    }
+		    if (!clicked_in_selection) {
+			// want the tree to update right away
+			tree.setImmediate(true);
+			tree.clearSelection();
+			tree.setSelectionPath(right_click_path);
+			tree.setImmediate(false);
+			selection_paths = new TreePath[1];
+			selection_paths[0] = right_click_path;
+		    }
+		}
+	    }
+	    // finally we have the correct selection paths!
+	    
+	    // Create an appropriate context menu, based on what is selected
+	    buildContextMenu(selection_paths);
+
+	    // Show the popup menu on screen
+	    show(tree, event.getX(), event.getY());
+	}
+
+
+	private void buildContextMenu(TreePath[] selection_paths)
+	{
+	    // If nothing is selected, only the new folder/dummy doc options are available...
+	    if (selection_paths == null) {
+		// ... but only in the collection tree!
+		if (tree == collection_tree) {
+		    new_folder = new JMenuItem(Dictionary.get("CollectionPopupMenu.New_Folder"), KeyEvent.VK_N);
+		    new_folder.addActionListener(this);
+		    add(new_folder);
+
+		    new_dummy_doc = new JMenuItem(Dictionary.get("CollectionPopupMenu.New_Dummy_Doc"));
+		    new_dummy_doc.addActionListener(this);
+		    add(new_dummy_doc);
+
+		    node = (CollectionTreeNode) tree.getModel().getRoot();
+		}
+
+		return;
+	    }
+
+	    DebugStream.println("Number of files/folders selected: " + selection_paths.length);
+
+	    // Collection tree: display meta-audit option and delete options
+	    if (tree == collection_tree) {
+		String[] args = new String[1];
+		args[0] = collection_tree.getSelectionDetails();
+		metaaudit = new JMenuItem(Dictionary.get("Menu.Metadata_View", args), KeyEvent.VK_A);
+		metaaudit.addActionListener(this);
+		add(metaaudit);
+
+		delete = new JMenuItem(Dictionary.get("CollectionPopupMenu.Delete"), KeyEvent.VK_D);
+		delete.addActionListener(this);
+		add(delete);
+	    }
+
+	    // Only meta-audit and delete are available if multiple items are selected...
+	    if (selection_paths.length > 1) {
+		return;
+	    }
+
+	    TreePath path = selection_paths[0];
+	    node = (FileNode) path.getLastPathComponent();
+
+	    // ---- Options for file nodes ----
+	    if (node.isLeaf()) {
+		// Explode metadata databases, for collection tree only, and for explodable files only
+		if (tree == collection_tree && ((CollectionTreeNode) node).isExplodable()) {
+		    explode_metadata_database = new JMenuItem(Dictionary.get("Menu.Explode_Metadata_Database"), KeyEvent.VK_E);
+		    explode_metadata_database.addActionListener(this);
+		    explode_metadata_database.setEnabled(!Gatherer.isGsdlRemote);
+		    add(explode_metadata_database);
+		}
+
+		// Open the file in an external program
+		open_externally = new JMenuItem(Dictionary.get("Menu.Open_Externally"), KeyEvent.VK_O);
+		open_externally.addActionListener(this);
+		add(open_externally);
+
+		return;
+	    }
+
+	    // ---- Options for folder nodes ----
+	    // Collapse or expand, depending on current status
+	    if (tree.isExpanded(path)) {
+		collapse_folder = new JMenuItem(Dictionary.get("Menu.Collapse"), KeyEvent.VK_C);
+		collapse_folder.addActionListener(this);
+		add(collapse_folder);
+	    }
+	    else {
+		expand_folder = new JMenuItem(Dictionary.get("Menu.Expand"), KeyEvent.VK_O);
+		expand_folder.addActionListener(this);
+		add(expand_folder);
+	    }
+
+	    // New folder/dummy doc options, for collection tree only
+	    if (tree == collection_tree && !node.isReadOnly()) {
+		new_folder = new JMenuItem(Dictionary.get("CollectionPopupMenu.New_Folder"), KeyEvent.VK_N);
+		new_folder.addActionListener(this);
+		add(new_folder);
+		new_dummy_doc = new JMenuItem(Dictionary.get("CollectionPopupMenu.New_Dummy_Doc"));
+		new_dummy_doc.addActionListener(this);
+		add(new_dummy_doc);
+	    }
+
+	    // Create/remove shortcut option, for workspace tree only
+	    if (tree == workspace_tree) {
+		// The "built-in" folders can't be modified
+		String node_name = node.toString();
+		if (node_name.equals(Dictionary.get("Tree.World")) || node_name.equals(Dictionary.get("Tree.Root")) || node_name.equals(Dictionary.get("Tree.DownloadedFiles"))) {
+		    return;
+		}
+
+		// You can unmap 1st level nodes
+		WorkspaceTreeNode root = (WorkspaceTreeNode) tree.getModel().getRoot();
+		if (root.getIndex(node) != -1) {
+		    delete_shortcut = new JMenuItem(Dictionary.get("MappingPrompt.Unmap"), KeyEvent.VK_R);
+		    delete_shortcut.addActionListener(this);
+		    add(delete_shortcut);
+		}
+		// Or map any other level directories
+		else {
+		    create_shortcut = new JMenuItem(Dictionary.get("MappingPrompt.Map"), KeyEvent.VK_S);
+		    create_shortcut.addActionListener(this);
+		    add(create_shortcut);
+		}
+	    }
+	}
+
+
+	/** Called whenever one of the menu items is clicked, this method then causes the appropriate effect. */
+	public void actionPerformed(ActionEvent event)
+	{
+  	    Object source = event.getSource();
+
+	    // Create shortcut
+	    if (source == create_shortcut) {
+ 		MappingPrompt mp = new MappingPrompt(node.getFile());
+ 		mp.destroy();
+	    }
+
+	    // Delete shortcut
+  	    else if (source == delete_shortcut) {
+  		workspace_tree.removeDirectoryMapping((WorkspaceTreeNode) node);
+  	    }
+
+	    // Collapse folder
+	    else if (source == collapse_folder) {
+		tree.collapsePath(selection_paths[0]);
+	    }
+
+	    // Expand folder
+	    else if (source == expand_folder) {
+		tree.expandPath(selection_paths[0]);
+	    }
+
+	    // Explode metadata database
+	    else if (source == explode_metadata_database) {
+		ExplodeMetadataPrompt emp = new ExplodeMetadataPrompt(node.getFile());
+		//emp.destroy();
+	    }
+
+	    // Delete
+	    else if (source == delete) {
+		FileNode[] source_nodes = new FileNode[selection_paths.length];
+		for (int i = 0; i < selection_paths.length; i++) {
+		    source_nodes[i] = (FileNode) selection_paths[i].getLastPathComponent();
+		}
+
+		// Fire a delete action
+		Gatherer.f_man.action(tree, source_nodes, bin_button, null);
+	    }
+
+	    // Meta-audit
+	    else if (source == metaaudit) {
+ 		Gatherer.g_man.showMetaAuditBox();
+ 	    }
+
+	    // New folder
+  	    else if (source == new_folder) {
+ 		Gatherer.f_man.newFolder(tree, (CollectionTreeNode) node);
+ 	    }
+	    // New dummy doc
+	    else if (source == new_dummy_doc) {
+		Gatherer.f_man.newDummyDoc(tree, (CollectionTreeNode) node);
+	    }
+	    // Open in external program
+	    else if (source == open_externally) {
+		Gatherer.f_man.openFileInExternalApplication(node.getFile());
+	    }
+  	}
+    }
+
+
+    /** This class listens for certain key presses, such as [Enter] or [Delete], and responds appropriately. */
+    private class KeyListenerImpl
+	extends KeyAdapter {
+	private boolean vk_left_pressed = false;
+	/** Called whenever a key that was pressed is released, it is this action that will cause the desired effects (this allows for the key event itself to be processed prior to this listener dealing with it). */
+	public void keyReleased(KeyEvent event) {
+	    ///ystem.err.println("Key Release detected. " + event.getKeyCode());
+	    if(event.getKeyCode() == KeyEvent.VK_DELETE) {
+		// Get the selected files from the tree and removal them using the default dnd removal method.
+		// Find the active tree (you've made selections in).
+		DragTree tree = (DragTree) group.getActive();
+		// Fudge things a bit
+		group.setSource(tree);
+		// Determine the selection.
+		TreePath paths[] = tree.getSelectionPaths();
+		if(paths != null) {
+		    FileNode[] source_nodes = new FileNode[paths.length];
+		    for(int i = 0; i < source_nodes.length; i++) {
+			source_nodes[i] = (FileNode) paths[i].getLastPathComponent();
+		    }
+		    Gatherer.f_man.action(tree, source_nodes, bin_button, null);
+		    source_nodes = null;
+		}
+	    }
+	    else if(event.getKeyCode() == KeyEvent.VK_ENTER) {
+		// Get the first selected file.
+		DragTree tree = (DragTree)event.getSource();
+		TreePath path = tree.getSelectionPath();
+		if(path != null) {
+		    File file = ((FileNode)path.getLastPathComponent()).getFile();
+		    if (file != null && file.isFile()) {
+			Gatherer.f_man.openFileInExternalApplication(file);
+		    }
+		    else {
+			if(!tree.isExpanded(path)) {
+			    tree.expandPath(path);
+			}
+			else {
+			    tree.collapsePath(path);
+			}
+		    }
+		}
+	    } else if (event.getKeyCode() == KeyEvent.VK_UP || event.getKeyCode() == KeyEvent.VK_DOWN) {
+		DragTree tree = (DragTree)event.getSource();
+		// we need to manually do the up and down selections
+		boolean up = (event.getKeyCode() == KeyEvent.VK_UP);
+		int current_row = tree.getLeadSelectionRow();
+		tree.setImmediate(true);
+		if (up) {
+		    if (current_row > 0) {
+			tree.clearSelection();
+			tree.setSelectionRow(current_row-1);
+		    }
+		} else {
+		    if (current_row < tree.getRowCount()-1){
+			tree.clearSelection();
+			tree.setSelectionRow(current_row+1);
+		    }
+		}
+		tree.setImmediate(false);
+	    } else if (event.getKeyCode() == KeyEvent.VK_LEFT) {
+		// left key on a file shifts the selection to the parent folder
+		DragTree tree = (DragTree)event.getSource();
+		TreePath path = tree.getLeadSelectionPath();
+		if(path != null) {
+		    File file = ((FileNode)path.getLastPathComponent()).getFile();
+		    if(file != null) {
+			if (file.isFile() || vk_left_pressed) {
+			    vk_left_pressed = false;
+			    TreePath parent_path = path.getParentPath();
+			    if (parent_path != null && parent_path.getParentPath() != null) {
+				// if this file is in the top level folder, don't move the focus 
+				tree.setImmediate(true);
+				tree.clearSelection();
+				tree.setSelectionPath(parent_path);
+				tree.setImmediate(false);
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// we need to watch for left clicks on an unopened folder - should shift the focus to teh parent folder. But because there is some other mysterious key listener that does opening and closing folders on right and left clicks, we must detect the situation before the other handler has done its job, and process it after. 
+	public void keyPressed(KeyEvent event) {
+	    if (event.getKeyCode() == KeyEvent.VK_LEFT) {
+		// left key on  closed directory shifts the selection to the parent folder
+		DragTree tree = (DragTree)event.getSource();
+		TreePath path = tree.getLeadSelectionPath();
+		if(path == null) return; 
+		File file = ((FileNode)path.getLastPathComponent()).getFile();
+		if(file == null) return;
+		
+		if (file.isDirectory() && tree.isCollapsed(path)) {
+		    vk_left_pressed = true;
+		}
+	    }
+	}	
+    }
+    
+    /** This provides a small prompt for gathering addition details about a special directory mapping such as its symbolic name. 
+     * Used when creating a new shortcut to a folder */
+    private class MappingPrompt
+	extends JDialog
+	implements ActionListener, KeyListener {
+	private boolean cancelled = false;
+	private JButton cancel_button = null;
+	private JButton ok_button = null;
+	private JTextField name_field = null;
+	public MappingPrompt(File file) {
+	    super(Gatherer.g_man);
+	    setModal(true);
+	    setSize(DIALOG_SIZE);
+	    Dictionary.setText(this, "MappingPrompt.Title");
+
+	    // Creation
+	    JPanel content_pane = (JPanel) getContentPane();
+	    JPanel center_pane = new JPanel();
+	    JPanel labels_pane = new JPanel();
+	    JPanel fields_pane = new JPanel();
+
+	    JLabel file_label = new JLabel();
+	    Dictionary.setText(file_label, "MappingPrompt.File");
+	    JLabel file_field = new JLabel(file.getAbsolutePath());
+
+	    JLabel name_label = new JLabel();
+	    Dictionary.setText(name_label, "MappingPrompt.Name");
+	    name_field = new JTextField(file.getName());
+
+	    JPanel button_pane = new JPanel();
+	    ok_button = new GLIButton();
+	    ok_button.setEnabled(name_field.getText().length() > 0);
+	    ok_button.setMnemonic(KeyEvent.VK_O);
+	    Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	    cancel_button = new GLIButton();
+	    cancel_button.setMnemonic(KeyEvent.VK_C);
+	    Dictionary.setBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+
+	    // Connection
+	    cancel_button.addActionListener(this);
+	    ok_button.addActionListener(this);
+	    name_field.addKeyListener(this);
+
+	    // Layout
+	    labels_pane.setLayout(new GridLayout(2,1,5,0));
+	    labels_pane.add(file_label);
+	    labels_pane.add(name_label);
+	    
+	    fields_pane.setLayout(new GridLayout(2,1,5,0));
+	    fields_pane.add(file_field);
+	    fields_pane.add(name_field);
+
+	    center_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	    center_pane.setLayout(new BorderLayout(5,0));
+	    center_pane.add(labels_pane, BorderLayout.WEST);
+	    center_pane.add(fields_pane, BorderLayout.CENTER);
+
+	    button_pane.setLayout(new GridLayout(1,2,5,5));
+	    button_pane.add(ok_button);
+	    button_pane.add(cancel_button);
+
+	    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(center_pane, BorderLayout.CENTER);
+	    content_pane.add(button_pane, BorderLayout.SOUTH);
+	    // Display
+	    Dimension screen_size = Configuration.screen_size;
+	    setLocation((screen_size.width - DIALOG_SIZE.width) / 2, (screen_size.height - DIALOG_SIZE.height) / 2);
+	    setVisible(true);
+	    // If not cancelled create mapping.
+	    if (!cancelled) {
+		workspace_tree.addDirectoryMapping(name_field.getText(), file);
+	    }
+	}
+	public void actionPerformed(ActionEvent event) {
+	    if(event.getSource() == cancel_button) {
+		cancelled = true;
+	    }
+	    dispose();
+	}
+	public void destroy() {
+	    cancel_button = null;
+	    ok_button = null;
+	    name_field = null;
+	}
+	public void keyPressed(KeyEvent event) {
+	}
+	public void keyReleased(KeyEvent event) {
+	    ok_button.setEnabled(name_field.getText().length() > 0);
+
+	}
+	public void keyTyped(KeyEvent event) {
+	}
+    }
+
+    /** This class listens for mouse clicks and responds right mouse button clicks (popup menu). */
+    private class MouseListenerImpl
+	extends MouseAdapter {
+	/** Any subclass of MouseAdapter can override this method to respond to mouse click events. In this case we want to open a pop-up menu if we detect a right mouse click over one of our registered components, and start an external application if someone double clicks on a certain file record. */
+	public void mouseClicked(MouseEvent event) {
+	    if (SwingUtilities.isRightMouseButton(event)) {
+		new RightClickMenu((DragTree) event.getSource(), event);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/HelpFrame.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/HelpFrame.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/HelpFrame.java	(revision 21112)
@@ -0,0 +1,366 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Principal Author: John Thompson, NZDL Project, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/**
+ * This class provides a nice help facility. It is a separate frame that can be positioned
+ * as the user wishes, and provides a contents page and the help information.
+ * @author Michael Dewsnip, NZDL Project
+ */
+public class HelpFrame 
+    extends JFrame
+{
+    /** The size of the frame */
+    static private final Dimension SIZE = new Dimension(760, 560);
+
+    static private HelpFrame self = null;
+
+    /** The help view at the bottom of the frame. */
+    static private JEditorPane help_pane = null;
+    /** The help contents tree at the top of the frame. */
+    static private JTree help_contents_tree = null;
+    /** The help contents tree model. */
+    static private HelpContentsTreeModel help_contents_tree_model = null;
+    /** The split between the contents tree and the page view. */
+    static private JSplitPane split_pane = null;
+
+
+    public HelpFrame()
+    {
+	setDefaultCloseOperation(HIDE_ON_CLOSE);
+	setSize(SIZE);
+	Dictionary.registerText(this, "Help.Title");
+
+	help_pane = new JEditorPane();
+	help_pane.setEditable(false);
+	help_pane.addHyperlinkListener(new HelpPaneHyperlinkListener());
+
+	HelpContentsTreeNode help_tree_root_node = new HelpContentsTreeNode(null, Dictionary.get("Help.Contents"));
+	help_contents_tree_model = new HelpContentsTreeModel(help_tree_root_node);
+	help_contents_tree = new JTree((DefaultTreeModel) help_contents_tree_model);
+	help_contents_tree.addTreeSelectionListener(new HelpContentsTreeSelectionListener());
+	help_contents_tree.setExpandsSelectedPaths(true);
+
+	// Creation
+	JPanel content_pane = (JPanel) this.getContentPane();
+	split_pane = new JSplitPane(JSplitPane.VERTICAL_SPLIT);
+	JScrollPane help_contents_tree_scroll = new JScrollPane(help_contents_tree);
+	JScrollPane help_pane_scroll = new JScrollPane(help_pane);
+
+	// Layout
+	split_pane.add(help_contents_tree_scroll, JSplitPane.LEFT);
+	split_pane.add(help_pane_scroll, JSplitPane.RIGHT);
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(split_pane, BorderLayout.CENTER);
+
+	// Center
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+
+	// Pretty corner icon
+	this.setIconImage(JarTools.getImage("gatherer_small.gif").getImage());
+
+	self = this;
+    }
+
+
+    public void destroy()
+    {
+	help_contents_tree = null;
+	help_pane = null;
+    }
+
+
+    /** Retrieve the relative file path to the language-specific help index xml file. */
+    private String getHelpFolder()
+    {
+	String help_folder = "help/" + Configuration.getLanguage() + "/";
+
+	// Applet case: get help files out of JAR file
+	if (Gatherer.isApplet && JarTools.getResource("/" + help_folder) != null) {
+	    return help_folder;
+	}
+
+	// Normal case: get help files out of GLI "help" folder
+	if (new File(help_folder).exists()) {
+	    return help_folder;
+	}
+
+	// Resort to English
+	return "help/" + StaticStrings.ENGLISH_LANGUAGE_STR  + "/";
+    }
+
+
+    private URL getURLForSection(String section_name)
+    {
+	// Form the path to the help file from the section name
+	if (section_name != null) {
+	    String help_file_path = getHelpFolder() + section_name + StaticStrings.HTM_FILE_EXTENSION;
+
+	    try {
+		// Applet case: get help file out of JAR file
+		if (Gatherer.isApplet) {
+		    return JarTools.getResource("/" + help_file_path);
+		}
+
+		// Normal case: get help file out of GLI "help" folder
+		return (new File(help_file_path)).toURL();
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+
+	return null;
+    }
+
+
+    static private void selectHelpContentsTreeNode(String section_name)
+    {
+	// Find the tree node with this section name, then select it
+	selectHelpContentsTreeNode(help_contents_tree_model.getHelpContentsTreeNodeNamed(section_name));
+    }
+
+
+    static private void selectHelpContentsTreeNode(HelpContentsTreeNode help_tree_node)
+    {
+	// Ensure the node in the help contents tree is selected and visible
+	TreePath help_tree_node_path = new TreePath(help_tree_node.getPath());
+	help_contents_tree.setSelectionPath(help_tree_node_path);
+	help_contents_tree.scrollPathToVisible(help_tree_node_path);
+    }
+
+
+    static public void setView(String section_name)
+    {
+	// Select the appropriate node in the help contents tree
+	selectHelpContentsTreeNode(section_name);
+
+	// Show the help page with the specified section name
+	self.showHelpPage(section_name);
+    }
+
+
+    private void showHelpPage(String section_name)
+    {
+	// Find the tree node with this section name, then show the page for the node
+	showHelpPage(getURLForSection(section_name));
+    }
+
+
+    private void showHelpPage(URL help_page_url)
+    {
+	// Display the selected page
+	if (help_page_url != null) {
+	    try {
+		help_pane.setPage(help_page_url);
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+
+	// Make the help frame visible
+	setVisible(true);
+	split_pane.setDividerLocation(0.2);
+    }
+
+
+    private class HelpContentsTreeModel
+	extends DefaultTreeModel
+    {
+	public HelpContentsTreeModel(MutableTreeNode help_tree_root_node)
+	{
+	    super(help_tree_root_node);
+
+	    // Load the XML help index file and build the contents structure from it
+	    try {
+		String help_index_file_path = getHelpFolder() + "help_index.xml";
+		Document document = XMLTools.parseXMLFile(help_index_file_path, true);
+
+		// Traverse the help hierarchy, building a tree representing its structure
+		Node document_node = document.getFirstChild();
+
+		// Add a help contents tree node for each major section
+		int section_number = 0;
+		NodeList children = document_node.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++) {
+		    Node child = children.item(i);
+		    if (child.getNodeName().equals(StaticStrings.SECTION_ELEMENT)) {
+			section_number++;
+			buildHelpContentsTree(help_tree_root_node, section_number + StaticStrings.EMPTY_STR, child);
+		    }
+		}
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+
+
+	private void buildHelpContentsTree(MutableTreeNode parent, String pos, Node node)
+	{
+	    // Determine the section name
+	    String section_name = ((Element) node).getAttribute(StaticStrings.NAME_ATTRIBUTE);
+
+	    // Determine the section title
+	    String section_title = "";
+	    NodeList children = node.getChildNodes();
+	    for (int i = 0; i < children.getLength(); i++) {
+		Node child = children.item(i);
+		if (child.getNodeName().equals(StaticStrings.TITLE_ELEMENT)) {
+		    section_title = pos + ": ";
+		    if (child.getFirstChild() != null) {
+			section_title += child.getFirstChild().getNodeValue();
+		    }
+		}
+	    }
+
+	    // Add the node into the tree
+	    HelpContentsTreeNode help_tree_node = new HelpContentsTreeNode(section_name, section_title);
+	    insertNodeInto(help_tree_node, parent, parent.getChildCount());
+
+	    // Apply recursively to the children of this node
+	    int section_number = 0;
+	    for (int i = 0; i < children.getLength(); i++) {
+		Node child = children.item(i);
+		if (child.getNodeName().equals(StaticStrings.SECTION_ELEMENT)) {
+		    section_number++;
+		    buildHelpContentsTree(help_tree_node, pos + StaticStrings.STOP_CHARACTER + section_number, child);
+		}
+	    }
+	}
+
+
+	private HelpContentsTreeNode getHelpContentsTreeNodeNamed(String section_name)
+	{
+	    // Find the node with name matching section name, somewhere in the tree
+	    if (section_name != null) {
+		Enumeration descendants = ((DefaultMutableTreeNode) root).preorderEnumeration();
+		while (descendants.hasMoreElements()) {
+		    HelpContentsTreeNode help_tree_node = (HelpContentsTreeNode) descendants.nextElement();
+		    if (section_name.equals(help_tree_node.section_name)) {
+			return help_tree_node;
+		    }
+		}
+	    }
+
+	    // Couldn't be found, so just return the root (Contents) node
+	    return (HelpContentsTreeNode) root;
+	}
+    }
+
+
+    /** This listener is used to listen for selection changes in the contents tree, and
+     * to show the appropriate page as required.
+     */
+    private class HelpContentsTreeSelectionListener
+	implements TreeSelectionListener
+    {
+	/** Any implementation of <i>TreeSelectionListener</i> must include this method so we can be informed when the tree selection changes.
+	 * @param event A <strong>TreeSelectionEvent</strong> containing al the relevant information about the event itself.
+	 */
+	public void valueChanged(TreeSelectionEvent event)
+	{
+	    HelpContentsTreeNode help_tree_node = (HelpContentsTreeNode) event.getPath().getLastPathComponent();
+	    selectHelpContentsTreeNode(help_tree_node);
+	    showHelpPage(help_tree_node.section_name);
+	}
+    }
+
+
+    /** This class provides a wrapper around a <strong>DefaultMutableTreeNode</strong> which
+     * provides the ability to set an html page to be loaded when this node is selected.
+     */
+    private class HelpContentsTreeNode 
+	extends DefaultMutableTreeNode
+    {
+	/** The unique name of the section (matches the HTML filename) */
+	public String section_name = null;
+	/** The title to be displayed for this tree node. */
+	public String section_title = null;
+
+	public HelpContentsTreeNode(String section_name, String section_title)
+	{
+	    this.section_name = section_name;
+	    this.section_title = section_title;
+	}
+
+	public String toString()
+	{
+	    return section_title;
+	}
+    }
+
+
+    private class HelpPaneHyperlinkListener
+	implements HyperlinkListener
+    {
+	public void hyperlinkUpdate(HyperlinkEvent e)
+	{
+	    if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED) {
+		// Determine which node in the help contents tree to select, then select it
+		String section_link = e.getURL().getFile();
+		if (section_link.endsWith(".htm")) {
+		    section_link = section_link.substring(section_link.lastIndexOf("/") + 1);
+		    section_link = section_link.substring(0, section_link.length() - ".htm".length());
+		    selectHelpContentsTreeNode(section_link);
+		}
+
+		// Change to the page specified by the link
+		showHelpPage(e.getURL());
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LockFileDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LockFileDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LockFileDialog.java	(revision 21112)
@@ -0,0 +1,227 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+/** A lock file dialog is shown whenever GLI detects an existing lock on a collection it is trying to open. The user can look at the details provided with the dialog, and decide whether to 'steal' the lock or to cancel the loading. */
+public class LockFileDialog
+    extends ModalDialog {
+
+    private int choice;
+    private Document document;
+    private JButton cancel_button;
+    private JButton ok_button;
+    private LockFileDialog self;
+
+    static final public int NO_OPTION  = 0;
+    static final public int YES_OPTION = 1;
+
+    static final private Dimension LABEL_SIZE = new Dimension(100, 25);
+    static final private Dimension SIZE = new Dimension(500, 250);
+    static final private int ICON_SIZE = 30;
+
+    public LockFileDialog(JFrame parent, String name, File lock_file) {
+	super(parent, Dictionary.get("LockFileDialog.Title"), true);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("openingacollection"));
+	this.self = this;
+
+	// Parse the lock file
+	document = XMLTools.parseXMLFile(lock_file);
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel upper_pane = new JPanel();
+	ImageIcon icon = JarTools.getImage("lcolicn.gif");
+	ImageIcon scaled_icon = new ImageIcon(icon.getImage().getScaledInstance(ICON_SIZE, ICON_SIZE, Image.SCALE_DEFAULT));
+	JLabel icon_label = new JLabel(scaled_icon);
+	JPanel title_pane = new JPanel();
+	JLabel title_one_label = new JLabel();
+	Dictionary.setText(title_one_label, "General.Warning"); 
+	JTextArea title_two_textarea = new JTextArea();
+	title_two_textarea.setEditable(false);
+	title_two_textarea.setLineWrap(true);
+	title_two_textarea.setOpaque(false);
+	title_two_textarea.setRows(3);
+	title_two_textarea.setWrapStyleWord(true);
+	Dictionary.setText(title_two_textarea, "LockFileDialog.Lockfile_Message_One");
+	JPanel central_pane = new JPanel();
+
+	JPanel name_pane = new JPanel();
+	JLabel name_label = new JLabel();
+	name_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.setText(name_label, "LockFileDialog.Name");
+	JTextField name_field = new JTextField(name);
+	name_field.setEditable(false);
+	name_field.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+
+	JPanel person_pane = new JPanel();
+	JLabel person_label = new JLabel();
+	person_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.setText(person_label, "LockFileDialog.User");
+	JTextField person_field = new JTextField(getValue("User"));
+	person_field.setEditable(false);
+	person_field.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+
+	JPanel machine_pane = new JPanel();
+	JLabel machine_label = new JLabel();
+	machine_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.setText(machine_label, "LockFileDialog.Machine");
+	JTextField machine_field = new JTextField(getValue("Machine"));
+	machine_field.setEditable(false);
+	machine_field.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+
+	JPanel created_pane = new JPanel();
+	JLabel created_label = new JLabel();
+	created_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.setText(created_label, "LockFileDialog.Date");
+	JTextField created_field = new JTextField(getValue("Date"));
+	created_field.setEditable(false);
+	created_field.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+
+	JLabel central_label = new JLabel();
+	Dictionary.setText(central_label, "LockFileDialog.Lockfile_Message_Two");
+	JPanel button_pane = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "LockFileDialog.OK_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "LockFileDialog.Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(new MyActionListener());
+	cancel_button.addActionListener(new MyActionListener());
+
+	// Layout
+	icon_label.setBorder(BorderFactory.createEmptyBorder(0,0,0,10));
+
+	title_pane.setLayout(new BorderLayout());
+	title_pane.add(title_one_label, BorderLayout.NORTH);
+	title_pane.add(title_two_textarea, BorderLayout.CENTER);
+
+	upper_pane.setLayout(new BorderLayout());
+	upper_pane.add(icon_label, BorderLayout.WEST);
+	upper_pane.add(title_pane, BorderLayout.CENTER);
+
+	name_pane.setLayout(new BorderLayout());
+	name_pane.add(name_label, BorderLayout.WEST);
+	name_pane.add(name_field, BorderLayout.CENTER);
+
+	person_pane.setLayout(new BorderLayout());
+	person_pane.add(person_label, BorderLayout.WEST);
+	person_pane.add(person_field, BorderLayout.CENTER);
+
+	machine_pane.setLayout(new BorderLayout());
+	machine_pane.add(machine_label, BorderLayout.WEST);
+	machine_pane.add(machine_field, BorderLayout.CENTER);
+
+	created_pane.setLayout(new BorderLayout());
+	created_pane.add(created_label, BorderLayout.WEST);
+	created_pane.add(created_field, BorderLayout.CENTER);
+
+	central_pane.setLayout(new GridLayout(5,1,0,5));
+	central_pane.add(name_pane);
+	central_pane.add(person_pane);
+	central_pane.add(machine_pane);
+	central_pane.add(created_pane);
+	central_pane.add(central_label);
+
+	button_pane.setLayout(new GridLayout(1,2,5,0));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(upper_pane, BorderLayout.NORTH);
+	content_pane.add(central_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Display
+	if (parent != null) {
+	    Rectangle frame_bounds = parent.getBounds();
+	    setLocation(frame_bounds.x + (frame_bounds.width - SIZE.width) / 2, frame_bounds.y + (frame_bounds.height - SIZE.height) / 2);
+	}
+	else {
+	    Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	    setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	}
+    }
+
+    public int getChoice() {
+	setVisible(true);
+	return choice;
+    }
+
+    private String getValue(String name) {
+	String value = "";
+	if(document != null) {
+	    try {
+		Element value_element = (Element) XMLTools.getNodeFromNamed(document.getDocumentElement(), name);
+		value = XMLTools.getValue(value_element);
+	    }
+	    catch (Exception error) {
+	    }
+	}
+	else {
+	    value = Dictionary.get("LockFileDialog.Error");
+	}
+	return value;
+    }
+
+    private class MyActionListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    if(event.getSource() == ok_button) {
+		choice = YES_OPTION;
+	    }
+	    else {
+		choice = NO_OPTION;
+	    }
+	    self.dispose();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LongProgressBar.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LongProgressBar.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/LongProgressBar.java	(revision 21112)
@@ -0,0 +1,245 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.math.BigInteger;
+import javax.swing.JProgressBar;
+import javax.swing.SwingUtilities;
+
+/** A progress bar that doesn't pack a sad when you feed it longs (such as the sizes of files) which get mangled into negative numbers. I know how it feels, as I often have negative progress. Also tries to be efficent when updating the JProgressBar, so only updates when the value actually changes, which become necessary when you move a large number of files with millions of bytes (where upon shifting a file of a thousand bytes would be pretty insignificant, maybe not even a percent, but would fire several progress updates). 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3c
+ */
+final public class LongProgressBar
+    extends JProgressBar {
+    private BigInteger cur_value = null;
+    private BigInteger max_value = null;
+    private BigInteger min_value = null;
+    /** The previous percentage value. */
+    private int previous = -1;
+    private SafeSetIndeterminate safe_set_indeterminate = null;
+    private SafeSetString safe_set_string = null;
+    private SafeSetValue safe_set_value = null;
+
+    public LongProgressBar() {
+	super(0, 100);
+	cur_value = BigInteger.ZERO;
+	min_value = BigInteger.ZERO;
+	max_value = BigInteger.ZERO;
+    }
+
+    public void addMaximum(int size) {
+	///ystem.err.println("Add maximum " + size);
+	if(size > 0) {
+	    max_value = max_value.add(new BigInteger((new Integer(size)).toString()));
+	    update();
+	}
+    }
+
+    public void addMaximum(long size) {
+	///ystem.err.println("Add maximum " + size);
+	if(size > 0) {
+	    max_value = max_value.add(new BigInteger((new Long(size)).toString()));
+	    update();
+	}
+    }
+
+    public void addValue(int size) {
+	///ystem.err.println("Add value " + size);
+	if(size > 0) {
+	    cur_value = cur_value.add(new BigInteger((new Integer(size)).toString()));
+	    update();
+	}
+    }
+
+    public void addValue(long size) {
+	///ystem.err.println("Add value " + size);
+	if(size > 0) {
+	    cur_value = cur_value.add(new BigInteger((new Long(size)).toString()));
+	    update();
+	}
+    }
+
+    public void clear()
+    {
+	previous = -1;
+	update();
+    }
+
+    public void reset() {
+	cur_value = BigInteger.ZERO;
+	max_value = BigInteger.ZERO;
+	previous = -1;
+	update();
+    }
+
+    public void setIndeterminate(boolean state) {
+	if(safe_set_indeterminate == null) {
+	    safe_set_indeterminate = new SafeSetIndeterminate(state);
+	}
+	else {
+	    safe_set_indeterminate.setState(state);
+	}
+	SwingUtilities.invokeLater(safe_set_indeterminate);		  
+    }
+
+    public void setMaximum(int size) {
+	///ystem.err.println("Add maximum " + size);
+	if(size > 0) {
+	    max_value = new BigInteger((new Integer(size)).toString());
+	    update();
+	}
+    }
+
+    public void setMaximum(long size) {
+	///ystem.err.println("Add maximum " + size);
+	if(size > 0) {
+	    max_value = new BigInteger((new Long(size)).toString());
+	    update();
+	}
+    }
+
+    public void setString(String label) {
+	safeSetString(label);
+    }
+	 
+    public void setValue(int size) {
+	///ystem.err.println("Set value " + size);
+	cur_value = new BigInteger((new Integer(size)).toString());
+	update();
+    }
+
+    public void setValue(long size) {
+	///ystem.err.println("Set value " + size);
+	cur_value = new BigInteger((new Long(size)).toString());
+	update();
+    }
+
+    /** This method is an unsafe way to set the progress bar to be indeterminate, -unless- it is called from the Event Queue. */
+    private void unsafeSetIndeterminate(boolean state) {
+	super.setIndeterminate(state);
+    }
+
+    private void unsafeSetString(String label) {
+	super.setString(label);
+    }
+
+    private void unsafeSetValue(int value) {
+	super.setValue(value);
+    }
+
+    private void update() {
+	int percentage = -1;
+	if(cur_value.equals(BigInteger.ZERO) || max_value.equals(BigInteger.ZERO)) {
+	    percentage = 0;
+	}
+	else {
+	    BigInteger tmp_value = cur_value.multiply(new BigInteger("100"));
+	    BigInteger per_value = tmp_value.divide(max_value);
+	    percentage = per_value.intValue();
+	    if(percentage > 100) {
+		percentage = 100;
+	    }
+	}
+	if(percentage != previous) {
+	    ///ystem.err.println("Progress bar status: value = " + getValue());
+	    previous = percentage;
+	    safeSetValue(percentage);
+	    safeSetString(percentage + "%");
+	}
+    }
+    /** Change the string displayed on the progress bar.
+     * @param label The name progress bar label as a <strong>String</strong>.
+     * Original code: Tad Frysinger (TeamB)
+     */
+    private void safeSetString(String label) {
+	if(safe_set_string == null) {
+	    safe_set_string = new SafeSetString(label);
+	}
+	else {
+	    safe_set_string.setLabel(label);
+	}
+	SwingUtilities.invokeLater(safe_set_string);  
+    }
+	 
+    /** Increase the value of the progress bar.
+     * @param ammount The ammount to increase as an <i>int</i>.
+     * Original code: Tad Frysinger (TeamB)
+     */
+    private void safeSetValue(final int ammount) {
+	// Update progress bar,
+	if(safe_set_value == null) {
+	    safe_set_value = new SafeSetValue(ammount);
+	}
+	else {
+	    safe_set_value.setValue(ammount);
+	}
+	SwingUtilities.invokeLater(safe_set_value);  
+    }
+
+    private class SafeSetIndeterminate 
+	implements Runnable {
+	private boolean value;
+	public SafeSetIndeterminate(boolean value) {
+	    this.value = value;
+	}
+	public void run() {
+	    unsafeSetIndeterminate(value);
+	}
+	synchronized public void setState(boolean value) {
+	    this.value = value;
+	}
+    }
+
+    private class SafeSetString 
+	implements Runnable {
+	private String label = null;
+	public SafeSetString(String label) {
+	    this.label = label;
+	}
+	public void run() {
+	    unsafeSetString(label);
+	}
+	synchronized public void setLabel(String label) {
+	    this.label = label;
+	}
+    }
+
+    private class SafeSetValue 
+	implements Runnable {
+	private int value = -1;
+	public SafeSetValue(int value) {
+	    this.value = value;
+	}
+	public void run() {
+	    unsafeSetValue(value);
+	}
+	synchronized public void setValue(int value) {
+	    this.value = value;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MenuBar.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MenuBar.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MenuBar.java	(revision 21112)
@@ -0,0 +1,283 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.JarTools;
+
+/** The menu bar for the Gatherer main GUI. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.2
+ */
+public class MenuBar
+    extends JMenuBar
+{
+    static public ImageIcon BLANK_ICON = JarTools.getImage("blank.gif");
+    static public ImageIcon HELP_ICON = JarTools.getImage("help.gif");
+
+    /** The icon to be displayed alongside the context choosen help file. */
+    private int current_tab          = -1;
+
+    private JMenu file               = null;
+    private JMenu edit               = null;
+    public JMenu help                = null;
+    public JMenuItem file_associations;
+    public JMenuItem file_cdimage    = null;
+    public JMenuItem file_close      = null;
+    public JMenuItem file_delete     = null;
+    public JMenuItem file_exit       = null;
+    public JMenuItem file_exportas   = null;
+    public JMenuItem file_new        = null;
+    public JMenuItem file_open       = null;
+    public JMenuItem file_options    = null;
+    public JMenuItem file_save       = null;
+    public JMenuItem edit_copy;
+    public JMenuItem edit_cut;
+    public JMenuItem edit_paste;
+    public JMenuItem help_general;
+    public JMenuItem help_download;
+    public JMenuItem help_gather;
+    public JMenuItem help_enrich;
+    public JMenuItem help_design;
+    public JMenuItem help_create;
+    public JMenuItem help_about;
+
+
+    public MenuBar(MenuListener menu_listener)
+    {
+	file = new JMenu();
+	file.setMnemonic(KeyEvent.VK_F);
+	Dictionary.registerText(file, "Menu.File");
+
+	file_associations = new JMenuItem();
+	file_associations.addActionListener(Gatherer.g_man);	  
+	file_associations.setMnemonic(KeyEvent.VK_A);
+	Dictionary.registerText(file_associations, "Menu.File_Associations");
+
+	file_cdimage = new JMenuItem();
+	file_cdimage.addActionListener(Gatherer.g_man);
+	file_cdimage.setEnabled(!Gatherer.isGsdlRemote);
+	file_cdimage.setMnemonic(KeyEvent.VK_I);
+	Dictionary.registerText(file_cdimage, "Menu.File_CDimage");
+
+	file_close = new JMenuItem();
+	file_close.addActionListener(Gatherer.g_man);
+	file_close.setEnabled(false);
+	file_close.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerText(file_close, "Menu.File_Close");
+
+	file_delete = new JMenuItem();
+	file_delete.addActionListener(Gatherer.g_man);
+	file_delete.setMnemonic(KeyEvent.VK_D);
+	Dictionary.registerText(file_delete, "Menu.File_Delete");
+
+	file_exit = new JMenuItem();
+	file_exit.addActionListener(Gatherer.g_man);
+	file_exit.setMnemonic(KeyEvent.VK_X);
+	Dictionary.registerText(file_exit, "Menu.File_Exit");
+
+	file_exportas = new JMenuItem();
+	file_exportas.addActionListener(Gatherer.g_man);
+	file_exportas.setEnabled(!Gatherer.isGsdlRemote);
+	file_exportas.setMnemonic(KeyEvent.VK_E);
+	Dictionary.registerText(file_exportas, "Menu.File_ExportAs");
+
+	file_new = new JMenuItem();
+	file_new.addActionListener(Gatherer.g_man);
+	file_new.setMnemonic(KeyEvent.VK_N);
+	Dictionary.registerText(file_new, "Menu.File_New");
+
+	file_open = new JMenuItem();
+	file_open.addActionListener(Gatherer.g_man);
+	file_open.setMnemonic(KeyEvent.VK_O);
+	Dictionary.registerText(file_open, "Menu.File_Open");
+
+	file_options = new JMenuItem();
+	file_options.addActionListener(Gatherer.g_man);
+	file_options.setMnemonic(KeyEvent.VK_P);
+	Dictionary.registerText(file_options, "Menu.File_Options");
+
+	file_save = new JMenuItem();
+	file_save.addActionListener(Gatherer.g_man);
+	file_save.setEnabled(false);
+	file_save.setMnemonic(KeyEvent.VK_S);
+	Dictionary.registerText(file_save, "Menu.File_Save");
+
+	// Layout (file menu)
+	file.add(file_new);
+	file.add(file_open);
+	file.add(file_save);
+	file.add(file_close);
+	file.add(new JSeparator());
+	file.add(file_delete);
+	file.add(file_exportas);
+	file.add(file_cdimage);
+	file.add(new JSeparator());
+	file.add(file_associations);
+	file.add(file_options);
+	file.add(new JSeparator());
+	file.add(file_exit);
+
+	// Edit menu
+	edit = new JMenu();
+	edit.setMnemonic(KeyEvent.VK_E);
+	Dictionary.registerText(edit, "Menu.Edit");
+		  
+	edit_cut = new JMenuItem();
+	edit_cut.addActionListener(Gatherer.g_man);
+	edit_cut.setMnemonic(KeyEvent.VK_X);
+	Dictionary.registerText(edit_cut, "Menu.Edit_Cut");
+
+	edit_copy = new JMenuItem();
+	edit_copy.addActionListener(Gatherer.g_man);
+	edit_copy.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerText(edit_copy, "Menu.Edit_Copy");
+
+	edit_paste = new JMenuItem();
+	edit_paste.addActionListener(Gatherer.g_man);
+	edit_paste.setMnemonic(KeyEvent.VK_V);
+	Dictionary.registerText(edit_paste, "Menu.Edit_Paste");
+
+	// Layout (edit menu)
+	edit.add(edit_cut);
+	edit.add(edit_copy);
+	edit.add(edit_paste);
+
+	// Help menu
+	help = new JMenu();
+	help.setIcon(HELP_ICON);
+	Dictionary.setText(help, "Menu.Help");
+
+	help_general = new JMenuItem();
+	help_general.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_general, "Source.General");
+
+	help_download = new JMenuItem(BLANK_ICON);
+	help_download.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_download, "GUI.Download");
+
+	help_gather = new JMenuItem(BLANK_ICON);
+	help_gather.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_gather, "GUI.Gather");
+
+	help_enrich = new JMenuItem(BLANK_ICON);
+	help_enrich.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_enrich, "GUI.Enrich");
+
+	help_design = new JMenuItem(BLANK_ICON);
+	help_design.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_design, "GUI.Design");
+
+	help_create = new JMenuItem(BLANK_ICON);
+	help_create.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_create, "GUI.Create");
+
+	help_about = new JMenuItem();
+	help_about.addActionListener(Gatherer.g_man);
+	Dictionary.registerText(help_about, "Menu.Help_About");
+
+	// Layout (help menu)
+	help.add(help_general);
+	help.add(new JSeparator());
+	if (Configuration.get("workflow.download", true)) {
+	    help.add(help_download);
+	}
+	if (Configuration.get("workflow.gather", true)) {
+	    help.add(help_gather);
+	}
+	if (Configuration.get("workflow.enrich", true)) {
+	    help.add(help_enrich);
+	}
+	if (Configuration.get("workflow.design", true)) {
+	    help.add(help_design);
+	}
+	if (Configuration.get("workflow.create", true)) {
+	    help.add(help_create);
+	}
+	help.add(new JSeparator());
+	help.add(help_about);
+
+	// Layout (menu bar)
+	this.add(file);
+	this.add(Box.createHorizontalStrut(15));
+	this.add(edit);
+	this.add(Box.createHorizontalGlue());
+	this.add(help);
+    }
+
+    /** Once a quit has been requested by the user, prevent any further menu selections. */
+    public void exit() {
+	file.setEnabled(false);
+	edit.setEnabled(false);
+	help.setEnabled(false);
+    }
+
+    public void refresh(int refresh_reason, boolean ready)
+    {
+	file_close.setEnabled(ready);
+	file_save.setEnabled(ready);
+    }
+
+
+    /** In order to provide context aware help advice we keep track of which
+     *  tab the user has open, and then highlight that help menu item with
+     *  separators.
+     *  @param tab_index The index of the selected tab (0-7).
+     */
+    public void tabSelected(int tab_index) {
+	JMenuItem selected;
+	if(current_tab != -1) {
+	    // Remove the image
+	    selected = help.getItem(current_tab);
+	    if(selected != null) {
+		selected.setIcon(BLANK_ICON);
+	    }
+	}
+	current_tab = tab_index + 2;
+	selected = help.getItem(current_tab);
+	if(selected != null) {
+	    selected.setIcon(HELP_ICON);
+	}
+	selected = null;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataElementListCellRenderer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataElementListCellRenderer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataElementListCellRenderer.java	(revision 21112)
@@ -0,0 +1,82 @@
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataTools;
+
+
+public class MetadataElementListCellRenderer
+    extends JLabel
+    implements ListCellRenderer
+{
+    private byte previous_directionality = Character.DIRECTIONALITY_UNDEFINED;
+
+
+    public byte getDirectionality()
+    {
+	if (previous_directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT || previous_directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC || previous_directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING || previous_directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE) {
+	    return Character.DIRECTIONALITY_RIGHT_TO_LEFT;
+	}
+	else {
+	    return Character.DIRECTIONALITY_LEFT_TO_RIGHT;
+	}
+    }
+
+
+    public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus)
+    {
+	if (value instanceof MetadataElement) {
+	    MetadataElement metadata_element = (MetadataElement) value;
+
+	    String interface_language_code = Configuration.getLanguage();
+	    String metadata_element_display = "";
+
+	    String metadata_element_display_name = metadata_element.getDisplayName();
+	    if (metadata_element_display_name != null) {
+		metadata_element_display += metadata_element_display_name + ":";
+	    }
+	    else {
+		metadata_element_display += metadata_element.getName() + ":";
+	    }
+
+	    String metadata_element_definition = MetadataTools.getMetadataElementAttribute(metadata_element, "definition", interface_language_code, "en");
+	    if (metadata_element_definition != null) {
+		metadata_element_display += " " + metadata_element_definition;
+	    }
+
+	    String metadata_element_comment = MetadataTools.getMetadataElementAttribute(metadata_element, "comment", interface_language_code, "en");
+	    if (metadata_element_comment != null) {
+		metadata_element_display += " " + metadata_element_comment;
+	    }
+
+	    setText(metadata_element_display);
+
+// 	    // Determine if the text should be left aligned or right aligned
+// 	    String text = metadata_element_display;
+// 	    int text_index = 0;
+// 	    byte directionality = Character.DIRECTIONALITY_UNDEFINED;
+// 	    while (directionality == Character.DIRECTIONALITY_UNDEFINED && text_index < text.length()) {
+// 		directionality = Character.getDirectionality(text.charAt(text_index));
+// 		text_index++;
+// 	    }
+// 	    if (directionality != previous_directionality) {
+// 		previous_directionality = directionality;
+// 		if (directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT || directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC || directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING || directionality == Character.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE) {
+// 		    setHorizontalAlignment(JLabel.TRAILING);
+// 		    setHorizontalTextPosition(JLabel.TRAILING);
+// 		    setComponentOrientation(ComponentOrientation.RIGHT_TO_LEFT);
+// 		}
+// 		else {
+// 		    setHorizontalAlignment(JLabel.LEADING);
+// 		    setHorizontalTextPosition(JLabel.LEADING);
+// 		    setComponentOrientation(ComponentOrientation.LEFT_TO_RIGHT);
+// 		}
+// 	    }
+	}
+
+	return this;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataImportMappingPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataImportMappingPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataImportMappingPrompt.java	(revision 21112)
@@ -0,0 +1,178 @@
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.metadata.MetadataTools;
+
+
+public class MetadataImportMappingPrompt
+    implements ActionListener
+{
+    final static public int ADD_BUTTON_PRESSED = 0;
+    final static public int MERGE_BUTTON_PRESSED = 1;
+    final static public int IGNORE_BUTTON_PRESSED = 2;
+    final static private Dimension DIALOG_SIZE = new Dimension(640, 180);
+
+    private int result;
+    private String metadata_element_name_full = null;
+    private GComboBox metadata_sets_combobox = null;
+    private GComboBox metadata_elements_combobox = null;
+    private JButton add_button = null;
+    private JButton merge_button = null;
+    private JButton ignore_button = null;
+    private JDialog on_screen = null;
+
+
+    public MetadataImportMappingPrompt(String metadata_element_name_full)
+    {
+	this.metadata_element_name_full = metadata_element_name_full;
+
+	// Construction and configuration
+	JDialog dialog = new ModalDialog(Gatherer.g_man);
+	dialog.setModal(true);
+	dialog.setSize(DIALOG_SIZE);
+	dialog.setJMenuBar(new SimpleMenuBar("importingpreviouslyassignedmetadata"));
+	Dictionary.setText(dialog, "MIMP.Title");
+
+	// All the loaded metadata sets except the extracted metadata set are applicable
+	ArrayList metadata_sets = MetadataSetManager.getMetadataSets();
+	for (int i = metadata_sets.size() - 1; i >= 0; i--) {
+	    if (((MetadataSet) metadata_sets.get(i)).getNamespace().equals(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE)) {
+		metadata_sets.remove(i);
+	    }
+	}
+
+  	add_button = new GLIButton();
+ 	add_button.addActionListener(this);
+  	add_button.setMnemonic(KeyEvent.VK_A);
+  	Dictionary.registerBoth(add_button, "MIMP.Add", "MIMP.Add_Tooltip");
+
+  	merge_button = new GLIButton();
+  	merge_button.addActionListener(this);
+  	merge_button.setEnabled(true);
+  	merge_button.setMnemonic(KeyEvent.VK_M);
+  	Dictionary.registerBoth(merge_button, "MIMP.Merge", "MIMP.Merge_Tooltip");
+
+  	ignore_button = new GLIButton();
+  	ignore_button.addActionListener(this);
+  	ignore_button.setEnabled(true);
+  	ignore_button.setMnemonic(KeyEvent.VK_S);
+  	Dictionary.registerBoth(ignore_button, "MIMP.Ignore", "MIMP.Ignore_Tooltip");
+
+	// !! Need to add instructions: "MIMP.Instructions", args: source elem
+	metadata_elements_combobox = new GComboBox();
+
+	metadata_sets_combobox = new GComboBox(metadata_sets);
+	metadata_sets_combobox.addActionListener(new MetadataSetListSelectionListener());
+	metadata_sets_combobox.setSelectedIndex(0);
+
+	// Layout
+	JPanel left_pane = new JPanel();
+	left_pane.setLayout(new GridLayout(3,1));
+	left_pane.add(new JLabel(Dictionary.get("MIMP.Source_Element")));
+	left_pane.add(new JLabel(Dictionary.get("MIMP.Target_Set")));
+	left_pane.add(new JLabel(Dictionary.get("MIMP.Target_Element")));
+
+	JPanel right_pane = new JPanel();
+	right_pane.setLayout(new GridLayout(3,1));
+	right_pane.add(new JLabel(metadata_element_name_full));
+	right_pane.add(metadata_sets_combobox);
+	right_pane.add(metadata_elements_combobox);
+
+	JPanel button_pane = new JPanel();
+	button_pane.setBorder(BorderFactory.createEmptyBorder(0,5,5,5));
+	button_pane.setLayout(new GridLayout(1,3));
+	button_pane.add(add_button);
+	button_pane.add(merge_button);
+	button_pane.add(ignore_button);
+
+	JPanel content_pane = (JPanel) dialog.getContentPane();
+	content_pane.setLayout(new BorderLayout());
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.add(left_pane, BorderLayout.WEST);
+	content_pane.add(right_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Display
+	Dimension screen_size = Configuration.screen_size;
+	dialog.setLocation((screen_size.width - DIALOG_SIZE.width) / 2, (screen_size.height - DIALOG_SIZE.height) / 2);
+	on_screen = dialog;
+	on_screen.setVisible(true); // Blocks until hidden.
+	on_screen.dispose();
+	on_screen = null;
+    }
+
+
+    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed when an action has occured.
+     * @param event An <strong>ActionEvent</strong> containing information gatherer when this event occured.
+     */
+    public void actionPerformed(ActionEvent event)
+    {
+	Object esrc = event.getSource();
+
+	if (esrc == add_button) {
+	    result = ADD_BUTTON_PRESSED;
+	}
+	if (esrc == merge_button) {
+	    result = MERGE_BUTTON_PRESSED;
+	}
+	if (esrc == ignore_button) {
+	    result = IGNORE_BUTTON_PRESSED;
+	}
+
+	on_screen.setVisible(false);
+    }
+
+
+    public MetadataElement getSelectedMetadataElement()
+    {
+	return (MetadataElement) metadata_elements_combobox.getSelectedItem();
+    }
+
+
+    public MetadataSet getSelectedMetadataSet()
+    {
+	return (MetadataSet) metadata_sets_combobox.getSelectedItem();
+    }
+
+
+    public int getResult()
+    {
+	return result;
+    }
+
+
+    private class MetadataSetListSelectionListener
+	implements ActionListener
+    {
+	public void actionPerformed(ActionEvent event)
+	{
+	    boolean enable_add_button = true;
+	    String metadata_element_name = MetadataTools.getMetadataElementName(metadata_element_name_full);
+
+	    MetadataSet metadata_set = (MetadataSet) metadata_sets_combobox.getSelectedItem();
+	    ArrayList metadata_elements = metadata_set.getMetadataSetElements();
+
+	    metadata_elements_combobox.removeAllItems();
+	    for (int i = 0; i < metadata_elements.size(); i++) {
+		MetadataElement metadata_element = (MetadataElement) metadata_elements.get(i);
+		metadata_elements_combobox.addItem(metadata_element);
+		if (metadata_element.getName().equals(metadata_element_name)) {
+		    metadata_elements_combobox.setSelectedIndex(i);
+		    enable_add_button = false;
+		}
+	    }
+
+	    add_button.setEnabled(enable_add_button);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTablePane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTablePane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTablePane.java	(revision 21112)
@@ -0,0 +1,417 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.metadata.MetadataChangedListener;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataTools;
+import org.greenstone.gatherer.metadata.MetadataValue;
+import org.greenstone.gatherer.metadata.MetadataValueTableEntry;
+import org.greenstone.gatherer.metadata.MetadataValueTableModel;
+import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.Utility;
+
+
+public class MetadataValueTablePane
+    extends JPanel
+{
+    private MetadataValueTable metadata_value_table = null;
+    /** Used to display either the MetadataValueTable (when files are selected) or a placeholder panel */
+    private CardLayout card_layout = null;
+    /** The name of the panel containing the metadata value table */
+    private String METADATA_VALUE_TABLE_CARD = "";
+    /** The name of the panel containing the "no metadata available" placeholder */
+    private String NO_METADATA_AVAILABLE_CARD = "No metadata available";
+    /** The name of the panel containing the "no files selected" placeholder */
+    private String NO_FILES_SELECTED_CARD = "No files selected";
+
+
+    public MetadataValueTablePane()
+    {
+	super();
+
+	metadata_value_table = new MetadataValueTable();
+
+	JScrollPane metadata_value_table_scroll = new JScrollPane(metadata_value_table);
+	metadata_value_table_scroll.getViewport().setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	metadata_value_table_scroll.setOpaque(true);
+
+	JPanel metadata_value_table_pane = new JPanel();
+	metadata_value_table_pane.setLayout(new BorderLayout());
+	metadata_value_table_pane.add(metadata_value_table_scroll, BorderLayout.CENTER);
+
+	JLabel no_metadata_available_label = new JLabel();
+	no_metadata_available_label.setHorizontalAlignment(JLabel.CENTER);
+	no_metadata_available_label.setOpaque(false);
+	no_metadata_available_label.setVerticalAlignment(JLabel.CENTER);
+	Dictionary.registerText(no_metadata_available_label, "EnrichPane.No_Metadata");
+
+	JPanel no_metadata_available_pane = new JPanel();
+	no_metadata_available_pane.setLayout(new BorderLayout());
+	no_metadata_available_pane.add(no_metadata_available_label, BorderLayout.CENTER);
+
+	JLabel no_files_selected_label = new JLabel();
+	no_files_selected_label.setHorizontalAlignment(JLabel.CENTER);
+	no_files_selected_label.setOpaque(false);
+	no_files_selected_label.setVerticalAlignment(JLabel.CENTER);
+	Dictionary.registerText(no_files_selected_label, "EnrichPane.No_File");
+
+	JPanel no_files_selected_pane = new JPanel();
+	no_files_selected_pane.setLayout(new BorderLayout());
+	no_files_selected_pane.add(no_files_selected_label, BorderLayout.CENTER);
+
+	card_layout = new CardLayout();
+
+	this.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	this.setFont(Configuration.getFont("general.font", false));
+	this.setLayout(card_layout);
+	this.add(metadata_value_table_pane, METADATA_VALUE_TABLE_CARD);
+	this.add(no_metadata_available_pane, NO_METADATA_AVAILABLE_CARD);
+	this.add(no_files_selected_pane, NO_FILES_SELECTED_CARD);
+    }
+
+
+    public void addMetadataValueTableListSelectionListener(ListSelectionListener list_selection_listener)
+    {
+	metadata_value_table.getSelectionModel().addListSelectionListener(list_selection_listener);
+    }
+
+
+    public void addMetadataValueTableMouseListener(MouseListener mouse_listener)
+    {
+	metadata_value_table.addMouseListener(mouse_listener);
+    }
+
+
+    public void addMetadataValueTextFieldDocumentListener(DocumentListener document_listener)
+    {
+	metadata_value_table.getMetadataValueTextField().getDocument().addDocumentListener(document_listener);
+    }
+
+
+    public void addMetadataValueTextFieldKeyListener(KeyListener key_listener)
+    {
+	metadata_value_table.getMetadataValueTextField().addKeyListener(key_listener);
+    }
+
+
+    public MetadataValueTableEntry getSelectedMetadataValueTableEntry()
+    {
+	return metadata_value_table.getSelectedMetadataValueTableEntry();
+    }
+
+
+    public boolean isMouseEventForInheritedMetadataValueTableColumn(MouseEvent mouse_event)
+    {
+	return metadata_value_table.isMouseEventForInheritedMetadataValueTableColumn(mouse_event);
+    }
+
+
+    public void setMetadataValueTextFieldValue(String metadata_value_string)
+    {
+	metadata_value_table.setMetadataValueTextFieldValue(metadata_value_string);
+    }
+
+
+    public void stopEditingAndAddBlankRowForSelectedMetadataElement()
+    {
+	metadata_value_table.stopEditing();
+	metadata_value_table.addBlankRowForSelectedMetadataElement();
+    }
+
+
+    public void stopEditingAndRebuild(CollectionTreeNode[] file_nodes)
+    {
+	metadata_value_table.stopEditing();
+	metadata_value_table.rebuild(file_nodes);
+
+	// If no files are selected display the "no file selected" card
+	if (file_nodes == null) {
+	    card_layout.show(this, NO_FILES_SELECTED_CARD);
+	}
+	// If the metadata value table is empty display the "no metadata available" card
+	else if (metadata_value_table.getRowCount() == 0) {
+	    card_layout.show(this, NO_METADATA_AVAILABLE_CARD);
+	}
+	// Otherwise display the card with the metadata value table
+	else {
+	    card_layout.show(this, METADATA_VALUE_TABLE_CARD);
+	}
+    }
+
+
+    private class MetadataValueTable
+	extends JTable
+	implements MetadataChangedListener
+    {
+	private int MINIMUM_TABLE_HEADER_SIZE = 15;
+
+	private MetadataValueTableModel metadata_value_table_model = null;
+	private JTextField metadata_value_text_field = new JTextField();
+
+
+	public MetadataValueTable()
+	{
+	    // Create the model for the table
+	    metadata_value_table_model = new MetadataValueTableModel();
+	    setModel(metadata_value_table_model);
+
+	    // We allow only one row in the table to be selected at a time
+	    setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	    // We use our own editor for the value column so we have easy access to the underlying text field
+	    setDefaultEditor(String.class, new DefaultCellEditor(metadata_value_text_field));
+
+	    // We need to listen for double clicks on the text field to open the editor dialog
+	    metadata_value_text_field.addMouseListener(new MetadataValueTextFieldMouseListener());
+	    metadata_value_text_field.setBorder(null);
+
+	    // We need to listen for key presses so we can catch Enter presses
+	    addKeyListener(new MetadataValueTableKeyListener());
+
+	    // We need to listen for metadata changes so we can rebuild the table
+	    MetadataXMLFileManager.addMetadataChangedListener(this);
+
+	    // We also have to ensure that the table column header hasn't gone on a severe Jenny Craig binge and has somehow lost 7/8th of its component size
+	    JTableHeader table_header = getTableHeader();
+	    Dimension table_header_preferred_size = table_header.getPreferredSize();
+	    if (table_header_preferred_size.height < MINIMUM_TABLE_HEADER_SIZE) {
+		table_header_preferred_size.setSize(table_header_preferred_size.width, MINIMUM_TABLE_HEADER_SIZE);
+		table_header.setPreferredSize(table_header_preferred_size);
+	    }
+
+	    // Set the size of the table columns: 2/7 for element, 5/7 for value (actual numbers don't matter) 
+	    TableColumnModel column_model = getColumnModel();
+	    TableColumn inherited_column = column_model.getColumn(0);
+	    inherited_column.setMinWidth(25);
+	    inherited_column.setPreferredWidth(25);
+	    inherited_column.setMaxWidth(25);
+	    TableColumn element_column = column_model.getColumn(1);
+	    element_column.setPreferredWidth(200);
+	    TableColumn value_column = column_model.getColumn(2);
+	    value_column.setPreferredWidth(500);
+
+	    // Use our own renderer for the table cells
+	    MetadataValueTableCellRenderer metadata_value_table_cell_renderer = new MetadataValueTableCellRenderer();
+	    inherited_column.setCellRenderer(metadata_value_table_cell_renderer);
+	    element_column.setCellRenderer(metadata_value_table_cell_renderer);
+	    value_column.setCellRenderer(metadata_value_table_cell_renderer);
+	}
+
+
+	private void addBlankRowForSelectedMetadataElement()
+	{
+	    // Add a blank entry for the selected metadata element, then switch to it
+	    MetadataElement selected_metadata_element = getSelectedMetadataValueTableEntry().getMetadataElement();
+	    int blank_row = metadata_value_table_model.addBlankRowForMetadataElement(selected_metadata_element);
+	    changeSelection(blank_row, 2, false, false);
+	}
+
+
+	private JTextField getMetadataValueTextField()
+	{
+	    return metadata_value_text_field;
+	}
+
+
+	private MetadataValueTableEntry getSelectedMetadataValueTableEntry()
+	{
+	    return metadata_value_table_model.getMetadataValueTableEntry(getSelectedRow());
+	}
+
+
+	private boolean isMouseEventForInheritedMetadataValueTableColumn(MouseEvent mouse_event)
+	{
+	    return (columnAtPoint(mouse_event.getPoint()) == 0);
+	}
+
+
+	public void metadataChanged(CollectionTreeNode[] file_nodes)
+	{
+	    rebuild(file_nodes);
+	}
+
+
+	private void rebuild(CollectionTreeNode[] file_nodes)
+	{
+	    // Note the metadata value table entry currently selected
+	    MetadataValueTableEntry selected_metadata_value_table_entry = getSelectedMetadataValueTableEntry();
+
+	    // We don't want a lot of ListSelectionEvents while the table is rebuilding
+	    clearSelection();
+
+	    // Rebuild the metadata value table model
+	    metadata_value_table_model.rebuild(file_nodes);
+
+	    // Restore the metadata value table entry selection
+	    if (selected_metadata_value_table_entry != null) {
+		int row_to_select = metadata_value_table_model.findMetadataValueTableEntryToSelect(selected_metadata_value_table_entry);
+		changeSelection(row_to_select, 2, false, false);
+	    }
+	}
+
+
+	private void setMetadataValueTextFieldValue(String metadata_value_string)
+	{
+	    metadata_value_text_field.setText(metadata_value_string);
+	    metadata_value_text_field.requestFocus();
+	}
+
+
+	private void stopEditing()
+	{
+	    // Save the current value in the text field, then remove the editor so it doesn't get saved again
+	    TableCellEditor table_cell_editor = getCellEditor();
+	    if (table_cell_editor != null) {
+		Object new_metadata_value = table_cell_editor.getCellEditorValue();
+		setValueAt(new_metadata_value, editingRow, editingColumn);
+		removeEditor();
+	    }
+	}
+
+
+	private class MetadataValueTableCellRenderer
+	    extends DefaultTableCellRenderer
+	{
+	    /** Returns the default table cell renderer.
+	     * @param table The <strong>JTable</strong>.
+	     * @param value The value to assign to the cell at [row, column] as an <strong>Object</strong>.
+	     * @param isSelected <i>true</i> if cell is selected.
+	     * @param hasFocus <i>true</i> if cell has focus.
+	     * @param row The row of the cell to render as an <i>int</i>.
+	     * @param column The column of the cell to render as an <i>int</i>.
+	     * @return The default table cell renderer <strong>Component</strong>.
+	     */
+	    public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column)
+	    {
+		JComponent component = (JLabel) super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column);
+
+		// First column: inherited metadata icon
+		if (column == 0 && value != null) {
+		    component = new JLabel(JarTools.getImage("upfolder.gif"));
+		}
+
+		// Make sure the focus always stay in the value cell of the selected row
+		if (column == 2 && isSelected) {
+		    table.editCellAt(row, column);
+		    if (table.isEditing()) {
+			table.getEditorComponent().requestFocus();
+		    }
+		}
+
+		// Set up the component
+		component.setOpaque(true);
+
+		// Foreground
+		if (metadata_value_table_model.isCommon(row)) {
+		    component.setForeground(Color.black);
+		}
+		else {
+		    component.setForeground(Color.gray);
+		}
+
+		// Background
+		if (isSelected) {
+		    component.setBackground(Configuration.getColor("coloring.workspace_heading_background", true));
+		}
+		else {
+		    if (column < 2) {
+			component.setBackground(Configuration.getColor("coloring.collection_heading_background", true));
+		    }
+		    else {
+			component.setBackground(Configuration.getColor("coloring.collection_tree_background", true));
+		    }
+		}
+
+		// The value column of cells never paints focus
+		if (column == 2) {
+		    component.setBorder(BorderFactory.createEmptyBorder(0,2,0,0));
+		}
+
+		// We set a tooltip over the element column containing the definition of the element
+		if (value instanceof MetadataElement) {
+		    String interface_language_code = Configuration.getLanguage();
+		    String metadata_element_definition = MetadataTools.getMetadataElementAttribute((MetadataElement) value, "definition", interface_language_code, "en");
+		    if (metadata_element_definition != null) {
+			component.setToolTipText(Utility.formatHTMLWidth(metadata_element_definition, 60));
+		    }
+		}
+
+		return component;
+	    }
+	}
+
+
+	private class MetadataValueTableKeyListener
+	    extends KeyAdapter
+	{
+	    /** Gives notification of key events on the text field */
+	    public void keyPressed(KeyEvent key_event)
+	    {
+		// Enter: save the current value then add a blank row for the selected metadata element
+		if (key_event.getKeyCode() == KeyEvent.VK_ENTER) {
+		    // ...but not for extracted metadata elements of course
+		    MetadataValueTableEntry metadata_value_table_entry = getSelectedMetadataValueTableEntry();
+		    if (!metadata_value_table_entry.getMetadataElement().isExtractedMetadataElement()) {
+			addBlankRowForSelectedMetadataElement();
+		    }
+
+		    // We do not want this event to be processed by the table also
+		    key_event.consume();
+		}
+	    }
+	}
+
+
+	private class MetadataValueTextFieldMouseListener
+	    extends MouseAdapter
+	{
+	    public void mouseClicked(MouseEvent mouse_event)
+	    {
+		// Double-click: pop up an editor dialog for large metadata values
+		if (mouse_event.getClickCount() == 2) {
+ 		    EditorDialog editor_dialog = new EditorDialog();
+ 		    String new_metadata_value_string = editor_dialog.display(metadata_value_text_field.getText());
+ 		    if (new_metadata_value_string != null) {
+ 			setMetadataValueTextFieldValue(new_metadata_value_string);
+ 		    }
+		}
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTreePane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTreePane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/MetadataValueTreePane.java	(revision 21112)
@@ -0,0 +1,298 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.metadata.MetadataElement;
+import org.greenstone.gatherer.metadata.MetadataValue;
+import org.greenstone.gatherer.metadata.MetadataValueTreeNode;
+import org.greenstone.gatherer.util.PatternTokenizer;
+
+
+/**
+ * This class is a little bit complex, a little bit subtle, and a little bit odd.
+ * The strange interaction model is due to the fact that it is very tightly
+ * coupled to the EnrichPane.
+ *
+ * The interaction is complex because there are three separate controls that the
+ * user may interact with, each of which can affect the other two. The three
+ * controls are:
+ *   - The "assigned metadata" table, at the top right of the meta edit pane.
+ *   - The "metadata value" text field, where users can type in new values.
+ *   - The "metadata value tree", which shows other values that have been
+ *     assigned to the selected metadata element.
+ *
+ * The interactions are:
+ *   - The "assigned metadata" table
+ *     Users may select one (and only one) row in this table. Selecting a row
+ *     chooses one metadata element. The text field will be set to the current
+ *     value of the metadata element. This value will also be selected in the
+ *     metadata value tree.
+ *
+ *   - The "metadata value" text field
+ *     If the value the user has typed in this is a prefix of an entry in the
+ *     value tree, this value will be selected in the value tree. In this case,
+ *     pressing "Tab" will complete the value (ie. make the value in the text
+ *     field the same as the value in the tree). This is to allow users to
+ *     quickly and easily assign existing metadata values to new documents.
+ *
+ *   - The "metadata value tree"
+ *     Selecting a value in the tree will set the text field to this value.
+ */
+public class MetadataValueTreePane
+    extends JPanel
+{
+    private boolean ignore_tree_selection_event = false;
+    /** The metadata value that the tree pane is built on */
+    private MetadataValue metadata_value = null;
+
+    /** Used to display either the MetadataValueTree (when metadata is selected) or a placeholder panel */
+    private CardLayout card_layout = null;
+    /** The name of the panel containing the metadata value tree */
+    private String METADATA_VALUE_TREE_CARD = "";
+    /** The name of the panel containing the "extracted metadata element selected" placeholder */
+    private String EXTRACTED_METADATA_ELEMENT_SELECTED_CARD = "Extracted metadata element selected";
+    /** The name of the panel containing the "inherited metadata selected" placeholder */
+    private String INHERITED_METADATA_SELECTED_CARD = "Inherited metadata selected";
+    /** The name of the panel containing the "no metadata element selected" placeholder */
+    private String NO_METADATA_ELEMENT_SELECTED_CARD = "No metadata element selected";
+
+    private JLabel metadata_value_tree_label = new JLabel();
+    private JTextArea extracted_metadata_element_selected_message;
+    private JTree metadata_value_tree;
+
+
+    public MetadataValueTreePane()
+    {
+	super();
+
+	// Card containing the metadata value tree
+	metadata_value_tree = new JTree();
+	metadata_value_tree.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+	metadata_value_tree.setModel(null);
+	metadata_value_tree.setRootVisible(false);
+	metadata_value_tree.putClientProperty("JTree.lineStyle", "Angled");
+
+	JPanel metadata_value_tree_pane = new JPanel();
+	metadata_value_tree_pane.setLayout(new BorderLayout());
+	metadata_value_tree_pane.add(metadata_value_tree_label, BorderLayout.NORTH);
+	metadata_value_tree_pane.add(new JScrollPane(metadata_value_tree), BorderLayout.CENTER);
+
+	// Card containing the "extracted metadata element selected" message
+	extracted_metadata_element_selected_message = new JTextArea("");
+	extracted_metadata_element_selected_message.setEditable(false);
+	extracted_metadata_element_selected_message.setLineWrap(true);
+	extracted_metadata_element_selected_message.setOpaque(false);
+	extracted_metadata_element_selected_message.setWrapStyleWord(true);
+
+	JPanel extracted_metadata_element_selected_pane = new JPanel();
+	extracted_metadata_element_selected_pane.setBorder(BorderFactory.createEmptyBorder(25,0,0,0));
+	extracted_metadata_element_selected_pane.setLayout(new BorderLayout());
+	extracted_metadata_element_selected_pane.add(extracted_metadata_element_selected_message, BorderLayout.CENTER);
+
+	// Card containing the "inherited metadata selected" message
+	JTextArea inherited_metadata_selected_message = new JTextArea("");
+	inherited_metadata_selected_message.setEditable(false);
+	inherited_metadata_selected_message.setLineWrap(true);
+	inherited_metadata_selected_message.setOpaque(false);
+	inherited_metadata_selected_message.setWrapStyleWord(true);
+	Dictionary.registerText(inherited_metadata_selected_message, "EnrichPane.InheritedMetadataSelected");
+
+	JPanel inherited_metadata_selected_pane = new JPanel();
+	inherited_metadata_selected_pane.setBorder(BorderFactory.createEmptyBorder(25,0,0,0));
+	inherited_metadata_selected_pane.setLayout(new BorderLayout());
+	inherited_metadata_selected_pane.add(inherited_metadata_selected_message, BorderLayout.CENTER);
+
+	// Card containing the "no metadata element selected" message
+	JLabel no_metadata_element_selected_label = new JLabel();
+	no_metadata_element_selected_label.setHorizontalAlignment(JLabel.CENTER);
+	no_metadata_element_selected_label.setOpaque(false);
+	no_metadata_element_selected_label.setVerticalAlignment(JLabel.CENTER);
+	Dictionary.registerText(no_metadata_element_selected_label, "EnrichPane.No_Metadata_Element");
+
+	JPanel no_metadata_element_selected_pane = new JPanel();
+	no_metadata_element_selected_pane.setLayout(new BorderLayout());
+	no_metadata_element_selected_pane.add(no_metadata_element_selected_label, BorderLayout.CENTER);
+
+	card_layout = new CardLayout();
+
+	this.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	this.setFont(Configuration.getFont("general.font", false));
+	this.setLayout(card_layout);
+	this.add(no_metadata_element_selected_pane, NO_METADATA_ELEMENT_SELECTED_CARD);
+	this.add(extracted_metadata_element_selected_pane, EXTRACTED_METADATA_ELEMENT_SELECTED_CARD);
+	this.add(inherited_metadata_selected_pane, INHERITED_METADATA_SELECTED_CARD);
+	this.add(metadata_value_tree_pane, METADATA_VALUE_TREE_CARD);
+    }
+
+
+    public void addMetadataValueTreeSelectionListener(TreeSelectionListener tree_selection_listener)
+    {
+	metadata_value_tree.addTreeSelectionListener(tree_selection_listener);
+    }
+
+
+    /** Returns a TreePath for the node most closely matching the metadata value. */
+    private TreePath getClosestPath(String metadata_value_string)
+    {
+	if (metadata_value_tree.getModel() == null) {
+	    return null;
+	}
+
+	// Start at the root of the tree
+	MetadataValueTreeNode tree_node = (MetadataValueTreeNode) metadata_value_tree.getModel().getRoot();
+
+	// Separate hierarchical values
+	PatternTokenizer tokenizer = new PatternTokenizer(metadata_value_string, MetadataValueTreeNode.METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN);
+	while (tokenizer.hasMoreTokens()) {
+	    String token = tokenizer.nextToken();
+
+	    // All components except the last must match exactly
+	    if (tokenizer.hasMoreTokens()) {
+		for (int i = 0; i < tree_node.getChildCount(); i++) {
+		    MetadataValueTreeNode child_node = (MetadataValueTreeNode) tree_node.getChildAt(i);
+		    if (child_node.getValue().equals(token)) {
+			// The matching node has been found, so move onto the next token
+			tree_node = child_node;
+			break;
+		    }
+		}
+	    }
+
+	    // The last component may match partially
+	    else {
+		for (int i = 0; i < tree_node.getChildCount(); i++) {
+		    MetadataValueTreeNode child_node = (MetadataValueTreeNode) tree_node.getChildAt(i);
+		    if (child_node.getFullValue().startsWith(metadata_value_string)) {
+			// The closest node has been found, so return its path
+			return new TreePath(child_node.getPath());
+		    }
+		}
+
+		// Not even a partial match
+		return null;
+	    }
+	}
+
+	// If nothing else, return the path of the root node
+	return new TreePath(tree_node.getPath());
+    }
+
+
+    public MetadataValueTreeNode getSelectedMetadataValueTreeNode()
+    {
+	if (metadata_value_tree.getSelectionCount() == 0 || ignore_tree_selection_event) {
+	    return null;
+	}
+
+	return (MetadataValueTreeNode) metadata_value_tree.getSelectionPath().getLastPathComponent();
+    }
+
+
+    public void rebuild(MetadataValue new_metadata_value)
+    {
+	// If the metadata value hasn't changed there is nothing to do
+	if (new_metadata_value == metadata_value) {
+	    return;
+	}
+
+	MetadataElement metadata_element = ((metadata_value != null) ? metadata_value.getMetadataElement() : null);
+	metadata_value = new_metadata_value;
+
+	// Selection cleared, so display "no metadata element selected" card and we're done
+	if (new_metadata_value == null) {
+	    metadata_value_tree.setModel(null);
+	    card_layout.show(this, NO_METADATA_ELEMENT_SELECTED_CARD);
+	    return;
+	}
+
+	// If a piece of inherited metadata is selected, display "inherited metadata selected" card and we're done
+	if (new_metadata_value.isInheritedMetadata()) {
+	    card_layout.show(this, INHERITED_METADATA_SELECTED_CARD);
+	    return;
+	}
+
+	// If an extracted metadata element is selected, display "extracted metadata element selected" card
+	MetadataElement new_metadata_element = new_metadata_value.getMetadataElement();
+	if (new_metadata_element.isExtractedMetadataElement()) {
+	    String[] args = new String[1];
+	    args[0] = new_metadata_element.getDisplayName();
+	    Dictionary.registerText(extracted_metadata_element_selected_message, "EnrichPane.AutoMessage", args);
+	    card_layout.show(this, EXTRACTED_METADATA_ELEMENT_SELECTED_CARD);
+	    return;
+	}
+
+	// Display the value tree for the selected metadata element
+	String[] args = new String[1];
+	args[0] = new_metadata_element.getDisplayName();
+	Dictionary.registerText(metadata_value_tree_label, "EnrichPane.ExistingValues", args);
+
+	metadata_value_tree.setModel(new_metadata_element.getMetadataValueTreeModel());
+	card_layout.show(this, METADATA_VALUE_TREE_CARD);
+	selectBestPathForMetadataValue(new_metadata_value.getFullValue());
+    }
+
+
+    public void selectBestPathForMetadataValue(String metadata_value_string)
+    {
+	TreePath best_path = getClosestPath(metadata_value_string);
+
+	// Select the new path in the tree
+	// The tree selection event this causes must be ignored, since it alters the metadata text field value
+	ignore_tree_selection_event = true;
+	metadata_value_tree.setSelectionPath(best_path);
+	ignore_tree_selection_event = false;
+
+	// If a folder has been specified, make sure it is expanded
+	if (metadata_value_string.endsWith(MetadataValueTreeNode.METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN) && !metadata_value_tree.isExpanded(best_path)) {
+	    metadata_value_tree.expandPath(best_path);
+	}
+
+	// Make sure the tree path is expanded
+	metadata_value_tree.makeVisible(best_path);
+
+	// Make sure the tree path is visible on screen
+	final Rectangle bounds = metadata_value_tree.getPathBounds(best_path);
+	if (bounds != null) {
+	    bounds.x = 0;  // Want the tree to be always flushed left
+
+	    // Scrolling the path to be visible must be done on the GUI thread
+	    Runnable task = new Runnable() {
+		    public void run() {
+			metadata_value_tree.scrollRectToVisible(bounds);
+		    }
+		};
+	    SwingUtilities.invokeLater(task);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalDialog.java	(revision 21112)
@@ -0,0 +1,208 @@
+/*#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import org.greenstone.gatherer.DebugStream;
+
+
+/** An extension of the JDialog that overrides the JVM's typical modal behaviour. This typical behaviour is that when a modal dialog is opened, all other windows cease to respond to user events until the modal dialog is disposed. However this prevents us opening the help documents property whenever a modal dialog is open. Thus we override the modal behaviour so that only the owner frame or dialog is blocked. 
+ * Note that because we always call the super constructor with modal set to false, this should be made visible with setVisible(true) rather than show() which will return straight away. */
+// feedback note: veronika had changed all the super constructor calls to 
+// use modal instead of false - not sure if this is needed so I have not 
+// put that in. --kjdon
+public class ModalDialog
+    extends JDialog {
+
+    /** The current modal dialog being shown on screen, if any. */
+    static public ModalDialog current_modal = null;
+
+    /** true if this dialog should be modal, ie block user actions to its owner window, false otherwise. */
+    protected boolean modal = false;
+    /** true if this dialog is currently waiting some thread. */
+    protected boolean waiting = false;
+
+    /** Constructor.
+     */
+    public ModalDialog() {
+	super((Frame)null, "", false);
+    }
+
+    /** Constructor.
+     * @param parent the Dialog which is the owener of this dialog.
+    */
+    public ModalDialog(Dialog parent) {
+	super(parent, "", false);
+    }
+
+    /** Constructor.
+     * @param parent the Dialog which is the owener of this dialog.
+     * @param modal true if this dialog should be modal, ie block user actions to its owner window, false otherwise.
+     */
+    public ModalDialog(Dialog parent, boolean modal) {
+	super(parent, "", false);
+	this.modal = modal;
+    }
+    
+    /** Constructor.
+     * @param parent the Dialog which is the owner of this dialog.
+     * @param title the String to use as the title for this dialog.
+     */
+    public ModalDialog(Dialog parent, String title) {
+	super (parent, title, false);
+	this.modal = false;
+    }
+
+    /** Constructor.
+     * @param parent the Dialog which is the owener of this dialog.
+     * @param title the String to use as the title for this dialog.
+     * @param modal true if this dialog should be modal, ie block user actions to its owner window, false otherwise.
+     */
+    public ModalDialog(Dialog parent, String title, boolean modal) {
+	super (parent, title, false);
+	this.modal = modal;
+    }
+
+   /** Constructor.
+     * @param parent the Frame which is the owener of this dialog.
+     */
+    public ModalDialog(Frame parent) {
+	super(parent, "", false);
+    }
+
+    /** Constructor.
+     * @param parent the Frame which is the owener of this dialog.
+     * @param modal whether this dialog is modal or not
+     */
+    public ModalDialog(Frame parent, boolean modal) {
+	super(parent, "", false);
+	this.modal = modal;
+    }
+    
+    /** Constructor.
+     * @param parent the Frame which is the owner of this dialog.
+     * @param title the String to use as the title for this dialog.
+     */
+    public ModalDialog(Frame parent, String title) {
+	super (parent, title, false);
+    }
+
+    /** Constructor.
+     * @param parent the Frame which is the owener of this dialog.
+     * @param title the String to use as the title for this dialog.
+     * @param modal true if this dialog should be modal, ie block user actions to its owner window, false otherwise.
+     */
+    public ModalDialog(Frame parent, String title, boolean modal) {
+	super (parent, title, false);
+	this.modal = modal;
+    }
+
+    public void dispose() {
+	super.dispose();
+    }
+
+    /** Ensures the current dialog is visible. */
+    public void makeVisible() {
+	super.setVisible(true);
+    }
+
+    /** The set visible method is overriden to provide modal functionality. It essentially hijacks control of the event dispatch thread while the dialog is open, only allowing non-user events to be passed to the parent dialog. Furthermore it only has this effect within the current AWT component tree by utilitizing the TreeLock.
+     * @param visible true if this dialog should be painted on-screen, false otherwise.
+     */
+    public void setVisible(boolean visible)
+    {
+	if (visible) {
+	    current_modal = this;
+	}
+	else {
+	    current_modal = null;
+	}
+
+	// If we are in the AWT Dispatch thread then it is all good.
+ 	if (SwingUtilities.isEventDispatchThread()) {
+ 	    super.setVisible(visible);
+	    if (modal && visible) {
+		EventQueue theQueue = getToolkit().getSystemEventQueue();
+		while (isVisible()) {
+		    try {
+			AWTEvent event = theQueue.getNextEvent();
+			Object src = event.getSource();
+
+			// Block all keyboard and mouse events to the parent component
+			if (src.equals(getParent())) {
+			    if (event instanceof KeyEvent || event instanceof MouseEvent) {
+				// System.err.println("Event to parent component blocked.");
+				continue;
+			    }
+			}
+
+			// Re-dispatch other events
+			if (event instanceof ActiveEvent) {
+  			    ((ActiveEvent) event).dispatch();
+  			}			    
+			else if (src instanceof Component) {
+			    ((Component) src).dispatchEvent(event);
+			}
+		    }
+		    catch (Exception exception) {
+			DebugStream.printStackTrace(exception);
+		    }
+		}
+	    }
+ 	}
+ 	else {
+ 	    try {
+ 		SwingUtilities.invokeAndWait(new MakeDialogVisibleTask(this, visible));
+ 	    }
+ 	    catch (Exception exception) {
+ 		DebugStream.printStackTrace(exception);
+ 	    }
+ 	}
+    }
+    
+    private class MakeDialogVisibleTask
+	implements Runnable {
+	private boolean make_visible;
+	private ModalDialog dialog;
+	public MakeDialogVisibleTask(ModalDialog dialog, boolean make_visible) {
+	    this.dialog = dialog;
+	    this.make_visible = make_visible;
+	}
+	public void run() {
+	    // Blocks until the user dismisses the dialog
+	    dialog.setVisible(make_visible);
+	}
+    }
+
+    /** Overridden method so we can control modality and not rely on the Dialog default.
+     * @param modal true if this dialog should be modal, ie block user actions to its owner window, false otherwise.
+     */
+    public void setModal (boolean modal) {
+	this.modal = modal;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalProgressPopup.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalProgressPopup.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/ModalProgressPopup.java	(revision 21112)
@@ -0,0 +1,84 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.gui;
+
+
+import java.awt.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Gatherer;
+
+
+public class ModalProgressPopup
+    extends ModalDialog
+{
+    /** The size of the popup */
+    final static private Dimension SIZE = new Dimension(300, 75);
+
+    private String message = null;
+
+
+    public ModalProgressPopup(String title, String message)
+    {
+	super(Gatherer.g_man, title);
+	this.message = message;
+    }
+
+
+    /** Method to close the popup. */
+    public void close()
+    {
+	setVisible(false);
+	Gatherer.g_man.wait(false);
+    }
+
+
+    /** Method to display the popup on screen. */
+    public void display()
+    {
+	setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+	setSize(SIZE);
+
+	// Create
+	JProgressBar progress_bar = new JProgressBar();
+	progress_bar.setIndeterminate(true);
+
+	// Layout
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JLabel(message), BorderLayout.NORTH);
+	content_pane.add(progress_bar, BorderLayout.CENTER);
+
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+
+	// Lock the rest of the GLI interface until close() is called
+	Gatherer.g_man.wait(true);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionDetailsPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionDetailsPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionDetailsPrompt.java	(revision 21112)
@@ -0,0 +1,401 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Project, NZDL, University of Waikato
+ *
+ * Copyright (C) 2003 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+public class NewCollectionDetailsPrompt
+    extends ModalDialog {
+
+    static public boolean titleClashes(String title, File current_config_file) {
+	// An empty collection title never clashes with anything. It may look ugly in the final collection but there is nothing wrong with having no title for a particular language.
+	if(title == null || title.length() == 0) {
+	    return false;
+	}
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	File children[] = collect_directory.listFiles();
+	for(int i = 0; children != null && i < children.length; i++) {
+	    if(children[i].isDirectory()) {
+		File config_file = new File(children[i], Utility.CONFIG_FILE);
+		if(current_config_file == null || !config_file.equals(current_config_file)) {
+		    BasicCollectionConfiguration other_collection = new BasicCollectionConfiguration(config_file);
+		    if(other_collection.getName().equalsIgnoreCase(title)) {  
+			return true;
+		    }
+		    other_collection = null;
+		}
+		config_file = null;
+	    }
+	}
+	return false;
+    }
+
+    private boolean cancelled;
+    private File base_final;
+    private JButton create_button;
+    private JComboBox base_collection;
+    private JDialog self;
+    private JTextArea description;
+    private JTextField title;
+    private String description_final;
+    private String title_final;
+    static private Dimension COMPONENT_SIZE = new Dimension(230, 25);
+    /** The size of this new collection dialog box. */
+    static private Dimension SIZE = new Dimension(600, 280);
+    static private int FILENAME_SIZE = 8;
+
+    /** Constructor.
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public NewCollectionDetailsPrompt() {
+	super(Gatherer.g_man, true);
+	this.cancelled = true;
+	this.self = this;
+	// Setup
+	setJMenuBar(new SimpleMenuBar("creatingacollection"));
+	setSize(SIZE);
+	Dictionary.setText(this, "NewCollectionPrompt.Title");
+
+	// Model building. Build a model of all of the collections in the gsdl collect directory with the appropriate directories.
+	Vector base_collection_model = new Vector();
+	// need to modify this to base a coll on any collection from any site
+	if (Gatherer.GS3) {
+	    File sites_dir = new File(Gatherer.getSitesDirectoryPath());
+	    File [] sites = sites_dir.listFiles();
+	    for (int i=0; i<sites.length; i++) {
+		File collect_directory = new File(sites_dir + sites[i].getName() + File.separator + "collect");
+		if (collect_directory.exists()) {
+		    addCollectionsToModel(base_collection_model, collect_directory, sites[i].getName());
+		}
+	    }
+	} else {
+	   File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	   addCollectionsToModel(base_collection_model, collect_directory, null);
+	}
+
+	// Sort the result.
+	Collections.sort(base_collection_model);
+	base_collection_model.add(0, new Item(null, Dictionary.get("NewCollectionPrompt.NewCollection")));
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setOpaque(true);
+	JPanel upper_pane = new JPanel();
+	JLabel instructions_label = new JLabel();
+	Dictionary.setText(instructions_label, "NewCollectionPrompt.Instructions");
+	JPanel title_pane = new JPanel();
+	JLabel title_label = new JLabel();
+	Dictionary.setText(title_label, "CDM.General.Collection_Name");
+	title = new JTextField();
+	title.setPreferredSize(COMPONENT_SIZE);
+	Dictionary.setTooltip(title, "CDM.General.Collection_Name_Tooltip");
+	JPanel name_pane = new JPanel();
+	JLabel name_label = new JLabel();
+	Dictionary.setText(name_label, "NewCollectionPrompt.Collection_Name");
+
+	JPanel center_pane = new JPanel();
+	JPanel description_pane = new JPanel();
+	JLabel description_label = new JLabel();
+	Dictionary.setText(description_label, "NewCollectionPrompt.Collection_Description");
+	description = new JTextArea();
+	description.setBackground(Configuration.getColor("coloring.editable_background", false));
+	description.setForeground(Configuration.getColor("coloring.editable_foreground", false));
+	description.setRows(5);
+	Dictionary.setTooltip(description, "CDM.General.Collection_Extra_Tooltip");
+
+	JPanel bottom_pane = new JPanel();
+	// Base Collection
+	JPanel base_collection_pane = new JPanel();
+	JLabel base_collection_label = new JLabel();
+	Dictionary.setText(base_collection_label, "NewCollectionPrompt.Base_Collection");
+	base_collection = new JComboBox(base_collection_model);
+	Dictionary.setTooltip(base_collection, "NewCollectionPrompt.Base_Collection_Tooltip");
+
+	JPanel button_pane = new JPanel();
+	create_button = new GLIButton();
+	create_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(create_button, "General.OK", "General.OK_Tooltip");
+	JButton cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+	// Connection
+	cancel_button.addActionListener(new CancelListener());
+	create_button.addActionListener(new CreateListener());
+	description.addKeyListener(new ColorListener(description));
+	description.addKeyListener(new DescriptionListener());
+	title.addKeyListener(new ColorListener(title));
+
+	// Layout
+	title_pane.setLayout(new BorderLayout(5,0));
+	title_pane.add(title_label, BorderLayout.WEST);
+	title_pane.add(title, BorderLayout.CENTER);
+
+	upper_pane.setLayout(new GridLayout(2,1));
+	upper_pane.add(instructions_label);
+	upper_pane.add(title_pane);
+
+	description_pane.setLayout(new BorderLayout());
+	description_pane.add(description_label, BorderLayout.NORTH);
+	description_pane.add(new JScrollPane(description), BorderLayout.CENTER);
+
+	base_collection_pane.setLayout(new BorderLayout(5,0));
+	base_collection_pane.add(base_collection_label, BorderLayout.WEST);
+	base_collection_pane.add(base_collection, BorderLayout.CENTER);
+
+	center_pane.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	center_pane.setLayout(new BorderLayout());
+	center_pane.add(description_pane, BorderLayout.CENTER);
+
+	bottom_pane.setLayout(new BorderLayout());
+	bottom_pane.add(base_collection_pane, BorderLayout.CENTER);
+	bottom_pane.add(button_pane, BorderLayout.SOUTH);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2));
+	button_pane.add(create_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(upper_pane, BorderLayout.NORTH);
+	content_pane.add(center_pane, BorderLayout.CENTER);
+	content_pane.add(bottom_pane, BorderLayout.SOUTH);
+	// Final dialog setup & positioning.
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+
+    public boolean isCancelled() {
+	return cancelled;
+    }
+
+    public File getBase() {
+	return base_final;
+    }
+
+    public String getDescription() {
+	return description_final;
+    }
+ 
+    /** Generates the collection short filename by taking the first eight characters of the title (spaces removed) and then adjusting by further truncating and then adding an unique suffix as necessary.
+     * @return the filename as a String
+     */
+    public String getName() {
+	// Retrieve the first 8 non-whitespace characters of title_final.
+	StringBuffer name_buffer = new StringBuffer("");
+
+	int i = 0;
+	int u = 0;
+	while(i < title_final.length() && u < 8) {
+	    char c = title_final.charAt(i);
+	    // Arg. We need a few more tests than just whitespace
+	    if(Character.isLetterOrDigit(c)) {
+		name_buffer.append(Character.toLowerCase(c));
+		u++;
+	    }
+	    i++;
+	}
+	// We need to ensure the filename is unique
+	int counter = 0;
+	StringBuffer new_name_buffer = new StringBuffer(name_buffer.toString());
+	while(filenameClashes(new_name_buffer.toString())) {
+	    new_name_buffer = new StringBuffer(name_buffer.toString());
+	    counter++;
+	    String suffix = String.valueOf(counter);
+	    // If we have to truncate the namestring so as to fit the suffix
+	    if(suffix.length() + new_name_buffer.length() > 8) {
+		new_name_buffer.replace(new_name_buffer.length() - suffix.length(), new_name_buffer.length(), suffix);
+	    }
+	    // Or just append it if that isn't necessary
+	    else {
+		new_name_buffer.append(suffix);
+	    }
+	    
+	}
+	// All done
+	return new_name_buffer.toString();
+    }
+    
+    private boolean filenameClashes(String filename)
+    {
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	File children[] = collect_directory.listFiles();
+	for(int i = 0; children != null && i < children.length; i++) {
+	    if(children[i].getName().equals(filename)) {
+		return true;
+	    }
+	}
+	return false;
+    }
+
+    public String getTitle() {
+	return title_final;
+    }
+
+    private void addCollectionsToModel(Vector base_collection_model, File collect_directory, String site) {
+	File[] possible_collections = collect_directory.listFiles();
+	for (int i = 0; possible_collections != null && i < possible_collections.length; i++) {
+	    // If the directory has a etc/collect.cfg file then it looks like a collection
+	    File collect_cfg_file = new File(possible_collections[i], Utility.CONFIG_FILE);
+	    if (collect_cfg_file.exists()) {
+		// If the directory has a metadata/ then the collection can be used as a base
+		File metadata_directory = new File(possible_collections[i], Utility.META_DIR);
+		if (metadata_directory.exists()) {
+		    // Add to model
+		    BasicCollectionConfiguration collect_cfg = new BasicCollectionConfiguration(collect_cfg_file);
+		    if (Gatherer.GS3 && site != null) {
+			collect_cfg.setSite(site);
+		    }
+		    Item item = new Item(possible_collections[i], collect_cfg);
+		    if (!base_collection_model.contains(item)) {
+			base_collection_model.add(item);
+		    }
+		}
+	    }
+	}
+       
+    }
+
+    private class CancelListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    cancelled = true;
+	    self.dispose();
+	}
+    }
+
+    private class ColorListener
+	extends KeyAdapter {
+	private JTextComponent component1;
+	private JTextComponent component2;
+	public ColorListener(JTextComponent component) {
+	    this.component1 = component;
+	    this.component2 = null;
+	}
+	public ColorListener(JTextComponent component1, JTextComponent component2) {
+	    this.component1 = component1;
+	    this.component2 = component2;
+	}
+	public void keyPressed(KeyEvent event) {
+	    component1.setForeground(Configuration.getColor("coloring.editable_foreground", false));
+	    component1.setBackground(Configuration.getColor("coloring.editable_background", false));
+	    if(component2 != null) {
+		component2.setForeground(Configuration.getColor("coloring.editable_foreground", false));
+		component2.setBackground(Configuration.getColor("coloring.editable_background", false));
+	    }
+	}
+    }
+    
+    private class CreateListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent event) {
+	    // Validate.
+	    title_final = title.getText();
+	    if(title_final.length() == 0) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("NewCollectionPrompt.Title_Error"), Dictionary.get("NewCollectionPrompt.Error"), JOptionPane.ERROR_MESSAGE);
+		return;
+	    }
+	    // We must ensure that the collection title is unique. This is a pain in the nether regions as we are forced to load the collect.cfg of each other collection in turn looking for a conflicting title
+	    else {
+		if(titleClashes(title_final, null)) {
+		    if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("NewCollectionPrompt.Title_Clash"), Dictionary.get("General.Warning"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
+			return;
+		    }
+		}
+	    }
+
+	    description_final = description.getText();
+
+	    // If we got this far there are no errors.
+	    Item item_final = (Item) base_collection.getSelectedItem();
+	    base_final = item_final.getFile();
+	    
+	    cancelled = false;
+	    
+	    self.dispose();
+	}
+    }
+    
+    private class DescriptionListener
+	extends KeyAdapter {
+	public void keyPressed(KeyEvent event) {
+	    if(event.getKeyCode() == KeyEvent.VK_TAB) {
+		event.consume();
+		base_collection.grabFocus();
+	    }
+	}
+    }
+
+    private class Item
+	implements Comparable {
+	private BasicCollectionConfiguration config;
+	private File file;
+	private String name;
+	public Item(File file, BasicCollectionConfiguration config) {
+	    this.config = config;
+	    this.file = file;
+	    this.name = null;
+	}
+	public Item(File file, String name) {
+	    this.config = null;
+	    this.file = file;
+	    this.name = name;
+	}
+	public int compareTo(Object other) {
+	    return toString().toLowerCase().compareTo(other.toString().toLowerCase());
+	}
+	public boolean equals(Object other) {
+	    return compareTo(other) == 0;
+	}
+	public File getFile() {
+	    return file;
+	}
+	public String toString() {
+	    if(name == null && config != null) {
+		name = config.toString();
+	    }
+	    return name;
+	}
+    }
+}
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionMetadataPrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionMetadataPrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewCollectionMetadataPrompt.java	(revision 21112)
@@ -0,0 +1,225 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.metadata.MetadataSet;
+import org.greenstone.gatherer.metadata.MetadataSetManager;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.CheckListEntry;
+
+
+public class NewCollectionMetadataPrompt
+    extends ModalDialog
+{
+    private boolean cancelled = true;
+    private JDialog self;
+    private JList elements_list;
+    private CheckList metadata_sets_list;
+    static private Dimension size = new Dimension(700, 380);
+
+    public NewCollectionMetadataPrompt() {
+	this(false);
+    }
+    public NewCollectionMetadataPrompt(boolean existing_coll) {
+	super(Gatherer.g_man, true);
+	this.self = this;
+	if (existing_coll) {
+	    setJMenuBar(new SimpleMenuBar("openingacollection"));
+	    Dictionary.setText(this, "NewCollectionPrompt.Metadata_Title_Existing");
+	} else {
+	    setJMenuBar(new SimpleMenuBar("creatingacollection"));
+	    Dictionary.setText(this, "NewCollectionPrompt.Title");
+	}
+	setModal(true);
+	setSize(size);
+
+	// Show the metadata sets (except extracted!) available in the GLI "metadata" folder 
+	ArrayList metadata_sets = MetadataSetManager.listMetadataSets(new File(Gatherer.getGLIMetadataDirectoryPath()));
+	ArrayList metadata_set_list_entries = new ArrayList();
+	for (int i = 0; i < metadata_sets.size(); i++) {
+	    MetadataSet metadata_set = (MetadataSet) metadata_sets.get(i);
+
+	    // Don't show the extracted metadata set
+	    if (metadata_set.getNamespace().equals(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE)) {
+		continue;
+	    }
+
+	    metadata_set_list_entries.add(metadata_set);
+	}
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+
+	JPanel instructions_panel = new JPanel();
+	JLabel instructions_label1 = new JLabel();
+	Dictionary.setText(instructions_label1, "NewCollectionPrompt.Metadata_Instructions1");
+	JLabel instructions_label2 = new JLabel();
+	Dictionary.setText(instructions_label2, "NewCollectionPrompt.Metadata_Instructions2");
+
+	JPanel center_pane = new JPanel();
+
+	JPanel sets_list_pane = new JPanel();
+	JLabel sets_list_label = new JLabel();
+	sets_list_label.setOpaque(false);
+	Dictionary.setText(sets_list_label, "NewCollectionPrompt.Select_MDS");
+	metadata_sets_list = new CheckList(true);
+	metadata_sets_list.addListSelectionListener(new MetadataSetListSelectionListener());
+	metadata_sets_list.setListData(metadata_set_list_entries);
+	metadata_sets_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+
+	JPanel elements_list_pane = new JPanel();
+	JLabel elements_list_label = new JLabel();
+	Dictionary.setText(elements_list_label, "NewCollectionPrompt.Metadata_Elements");
+	elements_list = new JList();
+	elements_list.setCellRenderer(new MetadataElementListCellRenderer());
+	elements_list.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	elements_list.setForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
+	elements_list.setSelectionBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	elements_list.setSelectionForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
+
+	// Set Dublin Core to be ticked initially, at Ian's request
+	for (int i = 0; i < metadata_set_list_entries.size(); i++) {
+	    MetadataSet metadata_set = (MetadataSet) metadata_set_list_entries.get(i);
+	    if (metadata_set.getMetadataSetFile().getName().equals("dublin.mds")) {
+		metadata_sets_list.setTickedObjects(new Object[] { metadata_set });
+	    }
+	}
+
+	JPanel button_pane = new JPanel();
+	JButton ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	JButton cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(new OKButtonListener());
+	cancel_button.addActionListener(new CancelButtonListener());
+
+	// Display
+	instructions_panel.setLayout(new GridLayout(2,1));
+	instructions_panel.add(instructions_label1);
+	instructions_panel.add(instructions_label2);
+
+	sets_list_pane.setLayout(new BorderLayout());
+	sets_list_pane.add(sets_list_label, BorderLayout.NORTH);
+	sets_list_pane.add(new JScrollPane(metadata_sets_list), BorderLayout.CENTER);
+
+	elements_list_pane.setLayout(new BorderLayout());
+	elements_list_pane.add(elements_list_label, BorderLayout.NORTH);
+	elements_list_pane.add(new JScrollPane(elements_list), BorderLayout.CENTER);
+
+	center_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	center_pane.setLayout(new GridLayout(2,1,0,5));
+	center_pane.add(sets_list_pane);
+	center_pane.add(elements_list_pane);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2,5,0));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(instructions_panel, BorderLayout.NORTH);
+	content_pane.add(center_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Show
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - size.width) / 2, (screen_size.height - size.height) / 2);
+	setVisible(true);
+    }
+
+    public ArrayList getSets() {
+	return metadata_sets_list.getTicked();
+    }
+
+    public boolean isCancelled() {
+	return cancelled;
+    }
+
+
+    private class CancelButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    cancelled = true;
+	    self.dispose();
+	}
+    }
+
+
+    private class MetadataSetListSelectionListener 
+	implements ListSelectionListener
+    {
+	public void valueChanged(ListSelectionEvent event)
+	{
+	    if (!metadata_sets_list.isSelectionEmpty()) {
+		// Retrieve the selected set
+		MetadataSet metadata_set = (MetadataSet) ((CheckListEntry) metadata_sets_list.getSelectedValue()).getObject();
+		elements_list.setListData(new Vector(metadata_set.getMetadataSetElements()));
+	    }
+	    else {
+		elements_list.setListData(new String[0]);
+	    }
+	}
+    }
+
+
+    private class OKButtonListener
+	implements ActionListener
+    {
+	public void actionPerformed(ActionEvent event)
+	{
+	    // See if the user has selected no metadata sets, and if so confirm thats what they really want.
+	    cancelled = false;
+	    if (metadata_sets_list.isNothingTicked()) {
+		WarningDialog dialog = new WarningDialog("warning.NoMetadataSetsSelected", "NoMetadataSetsSelected.Title", Dictionary.get("NoMetadataSetsSelected.Message"), null, true);
+		if (dialog.display() == JOptionPane.OK_OPTION) {
+		    // Otherwise we are free to go
+		    self.dispose();
+		}
+		dialog.dispose();
+		dialog = null;
+	    }
+	    else {
+		self.dispose();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewFolderOrFilePrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewFolderOrFilePrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NewFolderOrFilePrompt.java	(revision 21112)
@@ -0,0 +1,175 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.file.FileManager;
+
+public class NewFolderOrFilePrompt
+    extends JDialog 
+    implements ActionListener {
+    private CollectionTreeNode node;
+    private JButton cancel_button;
+    private JButton ok_button;
+    private JTextField name_field;
+    private String name;
+
+    private int type;
+    private String extension="";
+    static final private Dimension SIZE = new Dimension(350,115);
+
+    public NewFolderOrFilePrompt(CollectionTreeNode node, int type, String extension) {
+	super(Gatherer.g_man, true);
+	if (type == FileManager.FILE_TYPE) {
+	    setTitle(Dictionary.get("NewFolderOrFilePrompt.Title_File"));
+	} else {
+	    setTitle(Dictionary.get("NewFolderOrFilePrompt.Title_Folder"));
+	}
+	this.type = type;
+	if (extension != null) {
+	    this.extension = extension;
+	}
+	this.node = node;
+    }
+
+    public void actionPerformed(ActionEvent event) {
+	if (event.getSource() == ok_button) {
+	    name = name_field.getText()+extension;
+	}
+	else if(event.getSource() == cancel_button) {
+	    name = null;
+	}
+	dispose();
+    }
+
+    public String display() {
+	setSize(SIZE);
+	JPanel content_pane = (JPanel) getContentPane();
+
+	JPanel labels_pane = new JPanel();
+	JPanel fields_pane = new JPanel();
+	JPanel info_pane = new JPanel();
+
+	JLabel destination_label = new JLabel();
+	Dictionary.setText(destination_label, "NewFolderOrFilePrompt.Destination_Name");
+	JTextField destination_textfield = new JTextField(node.getFile().getName());
+	destination_textfield.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	destination_textfield.setEditable(false);
+	
+	JLabel name_label = new JLabel();
+	name_field = new JTextField(getAutomaticName());
+	if (type == FileManager.FILE_TYPE) {
+	    Dictionary.setText(name_label, "NewFolderOrFilePrompt.File_Name");
+	    Dictionary.setTooltip(name_field, "NewFolderOrFilePrompt.File_Name_Tooltip");
+	} else {
+	    Dictionary.setText(name_label, "NewFolderOrFilePrompt.Folder_Name");
+	    Dictionary.setTooltip(name_field, "NewFolderOrFilePrompt.Folder_Name_Tooltip");
+	}
+	JPanel button_pane = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	// Connection
+	cancel_button.addActionListener(this);
+	ok_button.addActionListener(this);
+
+	// Layout
+	labels_pane.setLayout(new GridLayout(2,1, 5,0));
+	labels_pane.add(destination_label);
+	labels_pane.add(name_label);
+
+	fields_pane.setLayout(new GridLayout(2,1,0,5));
+	fields_pane.add(destination_textfield);
+	fields_pane.add(name_field);
+	
+	info_pane.setLayout(new BorderLayout(5,0));
+	info_pane.add(labels_pane, BorderLayout.WEST);
+	info_pane.add(fields_pane, BorderLayout.CENTER);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1,2,0,5));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(info_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	// Display
+	Rectangle frame_bounds = Gatherer.g_man.getBounds();
+	setLocation(frame_bounds.x + (frame_bounds.width - SIZE.width) / 2, frame_bounds.y + (frame_bounds.height - SIZE.height) / 2);
+	setVisible(true);
+	return name;
+    }
+
+    private String getAutomaticName()
+    {
+	File file = node.getFile();
+	String default_name;
+	if (type == FileManager.FILE_TYPE) {
+	    default_name = Dictionary.get("NewFolderOrFilePrompt.Default_File_Name");
+	} else {
+	    default_name = Dictionary.get("NewFolderOrFilePrompt.Default_Folder_Name");
+	}
+	
+	File temp_file = new File(file, default_name+extension);
+	int count = 1;
+	while (temp_file.exists()) {
+	    temp_file = new File(file, default_name + " " + count+extension);
+	    count++;
+	}
+	if (extension.equals("")) {
+	    return temp_file.getName();
+	} 
+	String name = temp_file.getName();
+	// remove the extension
+	return name.substring(0, name.length()-extension.length());
+	
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NonWhitespaceField.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NonWhitespaceField.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/NonWhitespaceField.java	(revision 21112)
@@ -0,0 +1,73 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+/**************************************************************************************
+ * Written:     04/07/03
+ * Revised:     
+ **************************************************************************************/
+import javax.swing.*;
+import javax.swing.text.*;
+/** A JTextField that doesn't allow whitespace characters. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.4
+ */
+public class NonWhitespaceField 
+    extends JTextField {
+ 
+    public NonWhitespaceField() {
+	super();
+    }
+
+    public NonWhitespaceField(String text) {
+	super(text);
+    }
+ 
+    protected Document createDefaultModel() {
+	return new NonWhitespaceDocument();
+    }
+ 
+    static class NonWhitespaceDocument 
+	extends PlainDocument {
+		  
+	public void insertString(int offs, String str, AttributeSet a) 
+	    throws BadLocationException {
+				
+	    if (str == null) {
+		return;
+	    }
+	    char[] raw = str.toCharArray();
+	    StringBuffer result = new StringBuffer("");
+	    for (int i = 0; i < raw.length; i++) {
+		if(!Character.isWhitespace(raw[i])) {
+		    result.append(raw[i]);
+		}
+	    }
+	    super.insertString(offs, result.toString(), a);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OpenCollectionDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OpenCollectionDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OpenCollectionDialog.java	(revision 21112)
@@ -0,0 +1,273 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.beans.*;
+import java.io.File;
+import java.io.IOException;
+import java.text.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.filechooser.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.util.JarTools;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+public class OpenCollectionDialog
+    extends JFileChooser {
+    static final private ImageIcon LOCKED_COLLECTION_ICON = JarTools.getImage("lcolicn.gif");
+    static final private ImageIcon NORMAL_COLLECTION_ICON = JarTools.getImage("ncolicn.gif");
+    /** The name of the mouse listener that initiates editing on a double click. */
+    static final private String SINGLE_CLICK_LISTENER = "SingleClickListener";
+
+    /** Constructor */
+    public OpenCollectionDialog(File file) {
+	super(file);
+	///ystem.err.println("Size = " + getSize());
+	// Other initialization
+	setAcceptAllFileFilterUsed(false);
+	setDialogTitle(Dictionary.get("OpenCollectionDialog.Title"));
+	setFileFilter(new GathererFilter());
+	setFileSystemView(new GathererFileSystemView());
+	setFileView(new GathererFileView());
+	setSize(400,300);
+	// Stop the annoying renaming
+	disableRename(this);
+	// Description accessory
+	DescriptionPreview accessory = new DescriptionPreview(this);
+	setAccessory(accessory);
+	// Move accessory
+	JPanel accessory_pane = (JPanel) accessory.getParent();
+	JPanel center_pane = (JPanel) accessory_pane.getParent();
+	center_pane.add(accessory_pane, BorderLayout.SOUTH);
+    }
+
+    public void destroy() {
+    }
+
+    public String getFileName() {
+	// Continually loop until the use chooses an appropriate file or cancels.
+	while(true) {
+	    int return_val = showOpenDialog(null);
+	    if(return_val == JFileChooser.APPROVE_OPTION) {
+		File file = getSelectedFile();
+		if(file != null && file.getName().endsWith(".col")) {
+		    return file.getAbsolutePath();
+		}
+	    }
+	    else {
+		return null;
+	    }
+	}
+    }
+
+    /** Neat method to disable file renaming in filechooser.
+     * Thanks to: vladi21 from www.experts-exchange.com
+     */
+    static public void disableRename(Component c) {
+	if (c instanceof JList){
+	    EventListener[] listeners=c.getListeners(MouseListener.class);
+	    for(int i=0; listeners != null && i < listeners.length; i++) {
+		if (listeners[i].toString().indexOf(SINGLE_CLICK_LISTENER) != -1) {
+		    c.removeMouseListener((MouseListener)listeners[i]);
+		}
+	    }
+	    return;
+	}
+	if (c instanceof Container) {
+	    Component[] children = null;
+	    children = ((Container)c).getComponents();
+	    if (children != null) {
+		for(int i = 0; children != null && i < children.length; i++) {
+		    disableRename(children[i]);
+		}
+	    }
+	}
+    }
+
+
+    /* The DescriptionPreview accessory is adapted from the ImagePreview.java (an example used by FileChooserDemo2.java). */
+    private class DescriptionPreview
+	extends JPanel
+	implements PropertyChangeListener {
+	private File file = null;
+	private JTextArea text;
+
+	public DescriptionPreview(JFileChooser fc) {
+	    setName("Description Preview");
+	    setPreferredSize(new Dimension(100, 75)); // About three rows.
+
+	    text = new JTextArea();
+	    Dictionary.setText(text, "OpenCollectionDialog.No_Description");
+
+	    setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	    setLayout(new BorderLayout());
+	    add(new JScrollPane(text), BorderLayout.CENTER);
+
+	    fc.addPropertyChangeListener(this);
+	}
+
+	/** Whenever a new file or folder is selected we determine what description should be displayed. */
+	public void propertyChange(PropertyChangeEvent e) {
+	    boolean update = false;
+	    String prop = e.getPropertyName();
+	    // If the directory changed, don't show an image.
+	    if (JFileChooser.DIRECTORY_CHANGED_PROPERTY.equals(prop)) {
+		file = null;
+		update = true;
+	    }
+	    // If a file became selected, find out which one.
+	    else if (JFileChooser.SELECTED_FILE_CHANGED_PROPERTY.equals(prop)) {
+		file = (File) e.getNewValue();
+		update = true;
+	    }
+
+	    // Update the description accordingly.
+	    if (update) {
+		if(file == null) {
+		    Dictionary.setText(text, "OpenCollectionDialog.No_Description");
+		}
+		else {
+		    // Build a wrapper around the collection configuration file.
+		    File config_file = new File(file.getParentFile(), Utility.CONFIG_FILE);
+		    if(config_file.exists()) {
+			BasicCollectionConfiguration collect_cfg = new BasicCollectionConfiguration(config_file);
+			text.setText(StaticStrings.SPEECH_CHARACTER + collect_cfg.getName() + StaticStrings.SPEECH_CHARACTER + StaticStrings.NEW_LINE_CHAR);
+			text.setText(collect_cfg.getDescription());
+			text.setCaretPosition(0);
+		    }
+		}
+	    }
+	}
+    }
+
+    /** ImageFilter.java is a 1.4 example used by FileChooserDemo2.java. */
+    private class GathererFilter
+	extends FileFilter {
+
+	// Accept all directories and all .col files
+	public boolean accept(File f) {
+	    return (f.isDirectory() || f.getName().toLowerCase().endsWith(".col"));
+	}
+
+	// The description of this filter
+	public String getDescription() {
+	    return Dictionary.get("OpenCollectionDialog.Collection");
+	}
+    }
+
+    private class GathererFileSystemView
+	extends FileSystemView {
+
+	private FileSystemView default_system_view = FileSystemView.getFileSystemView();
+
+	/** Creates a new folder with a default folder name. */
+	public File createNewFolder(File containingDir)
+	    throws IOException {
+	    return default_system_view.createNewFolder(containingDir);
+	}
+
+	/** Gets the list of shown (i.e. */
+	public File[] getFiles(File dir, boolean useFileHiding) {
+	    // Retrieve the files usually returned by the platform specific file system view.
+	    File[] files = default_system_view.getFiles(dir, useFileHiding);
+	    // Determine if the current dir actually contains a valid greenstone collection. To do this we check for the presence of the file <dir>/etc/collect.cfg and a directory named import.
+	    File collect_cfg_file = new File(dir, Utility.CONFIG_FILE);
+	    File import_dir_file = new File(dir, Utility.IMPORT_DIR);
+	    if(collect_cfg_file.exists() && import_dir_file.exists()) {
+		// Create a new dummy collection file.
+		String name = dir.getName();
+		File collection_file = new File(dir, name + ".col");
+		// If this file doesn't already exist.
+		if(!collection_file.exists()) {
+		    File[] temp = new File[files.length + 1];
+		    System.arraycopy(files, 0, temp, 0, files.length);
+		    temp[files.length] = collection_file;
+		    files = temp;
+		    temp = null;
+		}
+		collection_file = null;
+		name = null;
+	    }
+	    import_dir_file = null;
+	    collect_cfg_file = null;
+	    return files;
+	}
+    }
+
+    /** A FileView which returns differing icons depending on whether a lock file is present within the collection folder. */
+    private class GathererFileView
+	extends FileView {
+
+	public String getDescription(File file) {
+	    String description = null;
+	    if (file.getName().endsWith(".col")) {
+		if (!lockExists(file)) {
+		    description = Dictionary.get("OpenCollectionDialog.Normal_Collection");
+		}
+		else {
+		    description = Dictionary.get("OpenCollectionDialog.Locked_Collection");
+		}
+	    }
+	    return description;
+	}
+
+	public Icon getIcon(File file) {
+	    Icon icon = null;
+	    if (file.getName().endsWith(".col")) {
+		if (!lockExists(file)) {
+		    icon = NORMAL_COLLECTION_ICON;
+		}
+		else {
+		    icon = LOCKED_COLLECTION_ICON;
+		}
+	    }
+	    return icon;
+	}
+
+	/** Determine if a lock file exists. */
+	private boolean lockExists(File file) {
+	    File lock_file = new File(file.getParentFile(), CollectionManager.LOCK_FILE);
+	    return lock_file.exists();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OptionsPane.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OptionsPane.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/OptionsPane.java	(revision 21112)
@@ -0,0 +1,544 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.Argument;
+import org.greenstone.gatherer.collection.ScriptOptions;
+import org.greenstone.gatherer.collection.Collection;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
+import org.greenstone.gatherer.util.AppendLineOnlyFileDocumentOwner;
+
+/** This class serves as the data holder for all subclasses of option panes, such as Import options or All options. It also contains methods for creating each of the option lines as they would appear in the subpane. Futhermore it has a method for considering all the arguments and generating a <strong>String[]</strong> to allow you to pass them to the <strong>GShell</strong>.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.2
+ */
+public class OptionsPane
+    extends JPanel 
+    implements AppendLineOnlyFileDocumentOwner, MouseListener {
+
+    static final public char SUCCESSFUL = 's';
+    static final public char UNSUCCESSFUL = 'u';
+    static final public char CANCELLED = 'c';
+    static final public char UNKNOWN = 'x';
+
+    static private int BUILD = 0;
+    static private int IMPORT = 1;
+    static private int MINIMUM_ROWS = 11;
+ 
+    /** All process messages are written to this log text area. */
+    public JTextArea log_textarea = null;
+
+    private ArrayList current_controls;
+
+    /** The <strong>ScriptOptions</strong> data object contains all the option settings we wish to persist between Gatherer sessions (and thus is stored in <strong>Collection</strong>). */
+    private ScriptOptions build_options = null;
+    private ScriptOptions import_options = null;
+
+    private FileEntry file_entry = null;
+
+    /** the log pane - we only create it once now, not each time */
+    private JPanel log_pane = null;
+    /** the list of previous log messages */
+    private JList log_list = null;
+    private Vector writing_documents;
+
+
+    /** The default constructor creates the few session length options, but either retrieves the rest from the current collection, or creates a default set of options. */
+    public OptionsPane(ScriptOptions import_options, ScriptOptions build_options) {
+	this.build_options = build_options;
+	this.import_options = import_options;
+	this.current_controls = new ArrayList();
+	this.writing_documents = new Vector();
+
+	// Have to do this here, not in display, as the message log view may not have been displayed yet.
+	log_textarea = new JTextArea();
+	log_textarea.setEditable(false);
+    }
+
+    /** This method creates the panel with all the build only options on it.
+     * @param pane a JPanel which already has previous arguments available on it, to allow for lower moes to concatenate all of the arguments together
+     * @return a JPanel which can be used to display all the build only options
+     * @see org.greenstone.gatherer.Configuration#EXPERT_MODE
+     * @see org.greenstone.gatherer.Configuration#getColor
+     * @see org.greenstone.gatherer.Configuration#getMode
+     * @see org.greenstone.gatherer.Gatherer#config
+     * @see org.greenstone.gatherer.cdm.Argument
+     * @see org.greenstone.gatherer.collection.BuildOptions#getBuildArgument
+     * @see org.greenstone.gatherer.collection.BuildOptions#getBuildArgumentCount
+     * @see org.greenstone.gatherer.collection.BuildOptions#getBuildValue
+     * @see org.greenstone.gatherer.collection.BuildOptions#getBuildValueEnabled
+     * @see org.greenstone.gatherer.gui.OptionsPane.MyArgumentControl
+     */
+    public JPanel buildBuild(JPanel pane) {
+	// Reset the arguments
+	if(pane == null) {
+	    current_controls.clear();
+	}
+	ArrayList build_arguments = new ArrayList();
+	int current_mode = Configuration.getMode();
+	int total_build_argument_count = build_options.getArgumentCount();
+       
+	for(int i = 0; i < total_build_argument_count; i++) {
+	    // Retrieve the argument so we know how to format the control.
+	    Argument argument = build_options.getArgument(i);
+	    if(!argument.isHiddenGLI() && argument.getModeLevel() <= current_mode) {
+		// Now attempt to retrieve any existing value for this argument.
+		boolean enabled = build_options.getValueEnabled(argument.getName());
+		String value = build_options.getValue(argument.getName());
+		MyArgumentControl argument_control = new MyArgumentControl(BUILD, argument, enabled, value);
+		build_arguments.add(argument_control);
+	    }
+	}
+	current_controls.addAll(build_arguments);
+	// Now that we know how many arguments there are we can build the pane to view them on. Modes lower than EXPERT can provide a previous pane on which to add the arguments.
+	if(pane == null || current_mode >= Configuration.EXPERT_MODE) {
+	    pane = new JPanel();
+	    pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    pane.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    int argument_count = build_arguments.size();
+	    // If in any of the higher detail modes, and assuming we don't want super phat argument controls, we better ensure there is a minimum number or lines in the grid layout
+	    if(current_mode >= Configuration.EXPERT_MODE) {
+		if(argument_count < MINIMUM_ROWS) {
+		    argument_count = MINIMUM_ROWS;
+		}
+		pane.setLayout(new GridLayout(argument_count, 1, 5, 5));
+	    }
+	    // Otherwise we're just going to throw them on one after another and chuck it in a scroll pane anyway
+	    else {
+		pane.setLayout(new BoxLayout(pane, BoxLayout.Y_AXIS));
+	    }
+	}
+	for(int j = 0; j < build_arguments.size(); j++) {
+	    pane.add((JComponent)build_arguments.get(j));
+	}
+	pane.addMouseListener(this);
+	build_arguments = null;
+	return pane;
+    }
+
+    /** This method creates the panel with all the import only options on it.
+     * @param pane a JPanel which already has previous arguments available on it, to allow for lower moes to concatenate all of the arguments together
+     * @return a JPanel which can be used to display all the build only options
+     * @see org.greenstone.gatherer.Configuration#EXPERT_MODE
+     * @see org.greenstone.gatherer.Configuration#getColor
+     * @see org.greenstone.gatherer.Configuration#getMode
+     * @see org.greenstone.gatherer.Gatherer#config
+     * @see org.greenstone.gatherer.cdm.Argument
+     * @see org.greenstone.gatherer.collection.BuildOptions#getImportArgument
+     * @see org.greenstone.gatherer.collection.BuildOptions#getImportArgumentCount
+     * @see org.greenstone.gatherer.collection.BuildOptions#getImportValue
+     * @see org.greenstone.gatherer.collection.BuildOptions#getImportValueEnabled
+     * @see org.greenstone.gatherer.gui.OptionsPane.ArgumentControl
+     */
+    public JPanel buildImport(JPanel pane) {
+	// Reset the arguments
+	if(pane == null) {
+	    current_controls.clear();
+	}
+	ArrayList import_arguments = new ArrayList();
+	int current_mode = Configuration.getMode();
+	int total_import_argument_count = import_options.getArgumentCount();       
+	for(int i = 0; i < total_import_argument_count; i++) {
+	    // Retrieve the argument so we know how to format the control.
+	    Argument argument = import_options.getArgument(i);
+	    if(!argument.isHiddenGLI() && argument.getModeLevel() <= current_mode) {
+		// Now attempt to retrieve any existing value for this argument.
+		boolean enabled = import_options.getValueEnabled(argument.getName());
+		String value = import_options.getValue(argument.getName());
+		MyArgumentControl argument_control = new MyArgumentControl(IMPORT, argument, enabled, value);
+		import_arguments.add(argument_control);
+	    }
+	}
+	current_controls.addAll(import_arguments);
+	// Now that we know how many arguments there are we can build the pane to view them on. Modes lower than EXPERT can provide a previous pane on which to add the arguments.
+	if(pane == null || current_mode >= Configuration.EXPERT_MODE) {
+	    pane = new JPanel();
+	    pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	    pane.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    int argument_count = import_arguments.size();
+	    // If in any of the higher detail modes, and assuming we don't want super phat argument controls, we better ensure there is a minimum number or lines in the grid layout
+	    if(current_mode >= Configuration.EXPERT_MODE) {
+		if(argument_count < MINIMUM_ROWS) {
+		    argument_count = MINIMUM_ROWS;
+		}
+		pane.setLayout(new GridLayout(argument_count, 1, 5, 5));
+	    }
+	    // Otherwise we're just going to throw them on one after another and chuck it in a scroll pane anyway
+	    else {
+		pane.setLayout(new BoxLayout(pane, BoxLayout.Y_AXIS));
+	    }
+	}
+	for(int j = 0; j < import_arguments.size(); j++) {
+	    pane.add((JComponent)import_arguments.get(j));
+	}
+	pane.addMouseListener(this);
+	import_arguments = null;
+	return pane;
+    }
+
+    /** This method is used to build a panel based on the message log, which is nothing like any of the other panels.
+     * @return A <strong>JPanel</strong> containing a scrollable text area which represents the shell process message log.
+     */
+    public JPanel buildLog() {
+	// we now save the log pane
+	if (log_pane == null) {
+	    log_pane = new JPanel(new BorderLayout());
+
+	    // Build a list of the log files available, ordering by last modified. Log files are like build_log.date.txt
+	    DefaultListModel contents = new DefaultListModel();
+	    File log_directory = new File(Gatherer.c_man.getCollectionLogDirectoryPath());
+	    File children[] = log_directory.listFiles();
+	    for(int i = 0; children != null && i < children.length; i++) {
+		String filename = children[i].getName();
+		if(filename.startsWith("build_log.") && filename.endsWith(".txt") ) {
+		    String datestamp = filename.substring(filename.indexOf(".") + 1, filename.lastIndexOf(".")).toLowerCase();
+		    if(datestamp.indexOf("s") == -1 && datestamp.indexOf("u") == -1 && datestamp.indexOf("c") == -1 && datestamp.indexOf("x") == -1) {
+			FileEntry entry = new FileEntry(children[i].getName(), children[i].getAbsolutePath());
+			// We are about to insert it. But where.
+			boolean found = false;
+			for(int j = 0; !found && j < contents.size(); j++) {
+			    FileEntry sibling = (FileEntry) contents.getElementAt(j);
+			    int order = entry.compareTo(sibling);
+			    if(order > 0) {
+				contents.insertElementAt(entry, j);
+				found = true;
+			    }
+			}
+			if(!found) {
+			    contents.addElement(entry);
+			}
+		    }
+		}
+	    }
+
+	    log_list = new JList(contents);
+	    log_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	    log_list.setLayoutOrientation(JList.VERTICAL);
+	    log_list.setPreferredSize(new Dimension(600, 100));
+	    log_list.setVisibleRowCount(3);
+	    log_list.addListSelectionListener(new LogListListener());
+
+	    JLabel log_history_label = new JLabel();
+	    Dictionary.registerText(log_history_label, "OptionsPane.LogHistory");
+	    JPanel log_history_pane = new JPanel();
+	    log_history_pane.setPreferredSize(new Dimension(600, 100));
+	    log_history_pane.setLayout(new BorderLayout());
+	    log_history_pane.add(log_history_label, BorderLayout.NORTH);
+	    log_history_pane.add(new JScrollPane(log_list), BorderLayout.CENTER);
+	    
+	    log_pane.add(new JScrollPane(log_textarea), BorderLayout.CENTER);
+	    log_pane.add(log_history_pane, BorderLayout.SOUTH);
+	}
+	return log_pane;
+    }
+
+    public AppendLineOnlyFileDocument createNewLogDocument() {
+	long time = System.currentTimeMillis();
+	StringBuffer name = new StringBuffer();
+	name.append("build_log.");
+	name.append(time);
+	name.append(".txt");
+	// just in case there is no log directory
+	File file = new File(Gatherer.c_man.getCollectionLogDirectoryPath() + name.toString());
+	File parent_file = file.getParentFile();
+	parent_file.mkdirs();
+	parent_file = null;
+	// create the file entry and add it to the list at pos 0 - it will always be the newest one created
+	file_entry = new FileEntry(name.toString(), file.getAbsolutePath());
+	((DefaultListModel)log_list.getModel()).add(0, file_entry);
+	log_list.setSelectedIndex(0);
+	// Finally retrieve and return the document associated with this file entry
+	return file_entry.getDocument();
+    }
+
+
+    /** Attempts to discover the latest document count.
+     * @return An <strong>int</strong> detailing the number of documents in this collection.
+     */
+    public int getDocumentCount() {
+	if(Gatherer.c_man.ready()) {
+	    int count = Gatherer.c_man.getCollection().getDocumentCount();
+	    if(count != 0) {
+		return count;
+	    }
+	}
+	return 1;
+    }
+
+    /** Called by our magic log documents after they have finished writing themselves to file, whereapon it is no longer necessary to hold a reference to them. */
+    public void remove(AppendLineOnlyFileDocument document) {
+	writing_documents.remove(document);
+    }
+
+    public void resetFileEntry() {
+	if(file_entry != null) {
+	    file_entry.reset();
+	}
+    }
+
+    /** Given a panel containing ArgumentControls, update the values associated with them. */
+    public void update(JPanel panel) {
+	if(panel == log_pane) {
+	    return;
+	}
+
+	for(int i = 0; i < panel.getComponentCount(); i++) {
+	    Component component = panel.getComponent(i);
+	    if(component instanceof MyArgumentControl) {
+		((MyArgumentControl)component).update();
+	    }
+	}
+    }
+
+    /** Implementation side-effect
+     * @param e a MouseEvent
+     */
+    public void mouseClicked(MouseEvent e) {}
+
+    /** Implementation side-effect
+     * @param e a MouseEvent
+     */
+    public void mouseEntered(MouseEvent e) {}
+
+    /** Implemented to ensure that, by the time the mouse pointer leaves the current build options screen, ang changes to the value the JSpinners have been commited
+     * @param e a MouseEvent
+     */
+    public void mouseExited(MouseEvent e) {
+	// Loop through the controls, and if the current control is a JSpinner, commit its current editing
+	for(int i = 0; i < current_controls.size(); i++) {
+	    MyArgumentControl control = (MyArgumentControl) current_controls.get(i);
+	    JComponent value_control = control.getValueControl();
+	    if(value_control instanceof JSpinner) {
+		try {
+		    ((JSpinner)value_control).commitEdit();
+		}
+		catch(Exception exception) {
+		    DebugStream.println("Exception in OptionsPane.mouseExited() - unexpected");
+		    DebugStream.printStackTrace(exception);
+		}
+	    }
+	    value_control = null;
+	    control = null;
+	}
+    }
+
+    /** Implementation side-effect
+     * @param e a MouseEvent
+     */
+    public void mousePressed(MouseEvent e) {}
+
+    /** Implementation side-effect
+     * @param e a MouseEvent
+     */
+    public void mouseReleased(MouseEvent e) {}
+
+    private class MyArgumentControl
+	extends ArgumentControl {
+	private int type;
+
+	public MyArgumentControl(int type, Argument argument, boolean enable, String value) {
+	    super(argument, enable, value);
+	    this.type = type;
+	}
+
+	/** Update the values stored in the collection so as to remember the current state of this argument. */
+	public void update() {
+	    String name = getName();
+	    boolean enable = isEnabled();
+	    String value = getValue();
+	    // If this argument was a flag, but is now disabled, remove from the build options altogether
+	    if(!enable && value == null) {
+		if(type == BUILD) {
+		    build_options.removeValue(name);
+		}
+		else {
+		    import_options.removeValue(name);
+		}
+	    }
+	    // Otherwise update the argument value
+	    else {
+		if(type == BUILD) {
+		    build_options.setValue(name, enable, value);
+		}
+		else {
+		    import_options.setValue(name, enable, value);
+		}
+	    }
+	}
+    }
+
+    /** Holds a File which has a particular naming convention build_log.date.txt also keeps a Date corresponding to the date in its name*/
+    private class FileEntry {
+
+       private AppendLineOnlyFileDocument current_document;
+       private Date date;
+       private long last_modified;
+       private String display;
+       private String filename;
+       private String filepath;
+       
+       public FileEntry(String filename, String filepath) {
+	  this.date = null;
+	  this.display = null;
+	  this.filename = filename;
+	  this.filepath = filepath;
+	  this.last_modified = 0L;
+       }
+
+	/** returns 0 if the dates are the same, -ve number if the current FileEntry is earlier than the fe FileEntry ...*/
+	public int compareTo(FileEntry file_entry) {
+	    Date our_date = getDate();
+	    Date other_date = file_entry.getDate();
+	    return our_date.compareTo(other_date);
+	}
+
+	public Date getDate() {
+	    if(date == null) {
+		// Need to exclude first '.'
+		int first_index = filename.indexOf(".") + 1;
+		// Need to exclude the last '.'
+		int last_index = filename.lastIndexOf(".");
+		if(first_index > 0 && last_index > 0 && first_index < last_index) {
+		    String date_string = filename.substring(first_index, last_index);
+		    date = new Date(Long.parseLong(date_string));
+		}
+		else {
+		    date = new Date(); // Current date
+		}
+	    }
+	    return date;
+	}
+
+       public AppendLineOnlyFileDocument getDocument() {
+	  if(current_document == null) {
+	     current_document = new AppendLineOnlyFileDocument(filepath);
+	  }
+	  return current_document;
+	}
+
+	public void reset() {
+	    display = null;
+	}
+
+	/** we only want the date out of the file name, not the whole path */
+	public String toString() {
+	    File file = new File(filename);
+	    if(display == null) {
+		last_modified = file.lastModified();
+		StringBuffer d = new StringBuffer();
+		Date date = getDate();
+		d.append(date.toString());
+		char success = UNKNOWN;
+		File the_file = new File(filepath);
+		if(the_file.exists()) {
+		   try {
+		      FileInputStream in = new FileInputStream(the_file);
+		      success = (char) in.read();
+		      in.close();
+		      in = null;
+		   }
+		   catch(Exception error) {
+		      ///ystem.err.println("Log '" + filepath + "' not found!");
+		      ///atherer.printStackTrace(error);
+		   }
+		}
+		the_file = null;
+		switch (success) {
+		case SUCCESSFUL:
+		    d.append(Dictionary.get("OptionsPane.Successful"));
+		    break;
+		case UNSUCCESSFUL:
+		    d.append(Dictionary.get("OptionsPane.Unsuccessful"));
+		    break;
+		case CANCELLED:
+		    d.append(Dictionary.get("OptionsPane.Cancelled"));
+		    break;
+		default:
+		    d.append(Dictionary.get("OptionsPane.Unknown"));
+		}
+		display = d.toString();
+	    }
+	    return display;
+	}
+    }
+
+    /** a ListSelectionListener that triggers the load of a newly selected log */
+    private class LogListListener implements ListSelectionListener {
+
+	public void valueChanged(ListSelectionEvent e) {
+	    if (!e.getValueIsAdjusting()) { // we get two events for one change in list selection - use the false one ( the second one)
+		///ystem.err.println("Log change detected.");
+		JList source  = (JList)e.getSource();
+		file_entry = (FileEntry) source.getSelectedValue();
+		// First we determine if the old log has been completely written to file
+		Document document = log_textarea.getDocument();
+		///ystem.err.println(" * current document: " + document);
+		///ystem.err.println(" * new document:     " + file_entry.getDocument());
+		// If we are dealing with the same document don't do anything.
+		if(document != file_entry.getDocument()) {
+		    if(document instanceof AppendLineOnlyFileDocument) {
+			AppendLineOnlyFileDocument append_line_only_file_document = (AppendLineOnlyFileDocument) document;
+			if(append_line_only_file_document.isStillWriting()) {
+			    ///ystem.err.println("Current log is still active... finishing.");
+			    writing_documents.add(append_line_only_file_document); // We have to maintain a reference until they are all done.
+			    append_line_only_file_document.setOwner(OptionsPane.this);
+			    append_line_only_file_document.setExit();
+			}
+			else {
+			    ///ystem.err.println("Current log is complete. Nothing to do.");
+			}
+		    }
+		    // Load the new log
+		    log_textarea.setDocument(file_entry.getDocument());
+		}
+	    }
+	}
+    }
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Preferences.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Preferences.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/Preferences.java	(revision 21112)
@@ -0,0 +1,935 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.plaf.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.ClassifierManager;
+import org.greenstone.gatherer.cdm.LanguageManager;
+import org.greenstone.gatherer.cdm.PluginManager;
+import org.greenstone.gatherer.collection.Collection;
+import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.util.ArrayTools; // just for debug
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.CheckListEntry;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+public class Preferences
+    extends ModalDialog
+{
+    static final public String CONNECTION_PREFS = "connection";
+    static final public String GENERAL_PREFS = "general";
+
+    static final private Dimension LABEL_SIZE = new Dimension(280, 25);
+    static final private Dimension ROW_SIZE = new Dimension(640, 25);
+    static final private Dimension SIZE = new Dimension(640, 345);
+    static final private String TRUE = "true";
+
+    private CheckList warning_preferences_check_list;
+    private EmailField email_field;
+    private JButton apply_button;
+    private JButton cancel_button;
+    private JButton ok_button;
+    private JCheckBox show_file_size_checkbox;
+    private JCheckBox use_proxy_checkbox;
+    private JCheckBox view_extracted_metadata_checkbox;
+    private JCheckBox workflow_download;
+    private JCheckBox workflow_gather;
+    private JCheckBox workflow_enrich;
+    private JCheckBox workflow_design;
+    private JCheckBox workflow_create;
+    private JComboBox language_combobox;
+    private JComboBox servlet_combobox; // GS3
+    private JComboBox site_combobox; // GS3
+    private JRadioButton assistant_mode_radio_button;
+    private JRadioButton expert_mode_radio_button;
+    private JRadioButton librarian_mode_radio_button;
+    private JRadioButton systems_mode_radio_button;
+    private JSpinner proxy_port_field;
+    private JTabbedPane tab_pane;
+    private JTextArea mode_description_textarea;
+    private JTextField font_field;
+    private JTextField gliserver_url_field;
+    private JTextField library_path_field;
+    private JTextField program_field;
+    private JTextField proxy_host_field;
+    private Preferences self;
+
+    private String current_site_selection;
+
+    public Preferences() {
+	this(GENERAL_PREFS);
+    }
+
+    public Preferences(String initial_view) {
+	// Initialize
+	super(Gatherer.g_man, true);
+	this.self = this;
+	setSize(SIZE);
+	Dictionary.registerText(this, "Preferences");
+	setJMenuBar(new SimpleMenuBar("preferences"));
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	tab_pane = new JTabbedPane();
+	JPanel general_preferences = createGeneralPreferences();
+	tab_pane.add("Preferences.General", general_preferences); // Also uses "Preferences.General_Tooltip"
+	tab_pane.add("Preferences.Mode", createModePreferences()); // Also uses "Preferences.Mode_Tooltip"
+	tab_pane.add("Preferences.Workflow", createWorkflowPreferences()); // Also uses "Preferences.Workflow_Tooltip"
+	JPanel connection_preferences = createConnectionPreferences();
+	tab_pane.add("Preferences.Connection", connection_preferences); // Also uses "Preferences.Connection_Tooltip"
+	tab_pane.add("Preferences.Warnings", createWarningPreferences()); // Also uses "Preferences.Warnings_Tooltip"
+	Dictionary.register(tab_pane);
+
+	JPanel button_pane = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.registerBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	apply_button = new GLIButton();
+	apply_button.setMnemonic(KeyEvent.VK_A);
+	Dictionary.registerBoth(apply_button, "General.Apply", "General.Apply_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(cancel_button, "General.Cancel", "General.Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(new OKButtonListener(true));
+	apply_button.addActionListener(new OKButtonListener(false));
+	cancel_button.addActionListener(new CancelButtonListener());
+
+	// Layout
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,2,2,2));
+	button_pane.setLayout(new GridLayout(1,3,0,5));
+	button_pane.add(ok_button);
+	button_pane.add(apply_button);
+	button_pane.add(cancel_button);
+
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(tab_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	Dimension frame_size = Gatherer.g_man.getSize();
+	Point frame_location = Gatherer.g_man.getLocation();
+	setLocation(((frame_size.width - SIZE.width) / 2), ((frame_size.height - SIZE.height)));
+
+	// Bring the desired pane to the fore
+	if (initial_view.equals(CONNECTION_PREFS)) {
+	    tab_pane.setSelectedComponent(connection_preferences);
+	}
+	else {
+	    tab_pane.setSelectedComponent(general_preferences);
+	}
+	
+	// Clean up
+	general_preferences = null;
+	connection_preferences = null;
+	frame_location = null;
+	frame_size = null;
+	cancel_button = null;
+	ok_button = null;
+	button_pane = null;
+	tab_pane = null;
+	content_pane = null;
+
+	setVisible(true);
+    }
+
+    private JPanel createConnectionPreferences() {
+	JPanel program_pane = new JPanel();
+	program_pane.setPreferredSize(ROW_SIZE);
+	JLabel program_label = new JLabel();
+	program_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(program_label, "Preferences.Connection.ProgramCommand");
+	program_field = new JTextField(Configuration.getPreviewCommand());
+	program_field.setCaretPosition(0);
+	Dictionary.registerTooltip(program_field, "Preferences.Connection.ProgramCommand_Tooltip");
+
+	JPanel library_path_pane = new JPanel();
+	library_path_pane.setPreferredSize(ROW_SIZE);
+	JLabel library_path_label = new JLabel();
+	library_path_label.setPreferredSize(LABEL_SIZE);
+	String library_url_string = "";
+	if (Configuration.library_url != null) {
+	    library_url_string = Configuration.library_url.toString();
+	}
+	library_path_field = new JTextField(library_url_string);
+	library_path_field.setCaretPosition(0);
+	if (Gatherer.GS3) {
+	    Dictionary.registerText(library_path_label, "Preferences.Connection.Library_Path_GS3");
+	    Dictionary.registerTooltip(library_path_field, "Preferences.Connection.Library_Path_Tooltip_GS3");
+	} else {
+	    Dictionary.registerText(library_path_label, "Preferences.Connection.Library_Path");
+	    Dictionary.registerTooltip(library_path_field, "Preferences.Connection.Library_Path_Tooltip");
+	}
+
+	JPanel gliserver_url_pane = null;
+	JLabel gliserver_url_label = null;
+	if (Gatherer.isGsdlRemote) {
+	    gliserver_url_pane = new JPanel();
+	    gliserver_url_pane.setPreferredSize(ROW_SIZE);
+	    gliserver_url_label = new JLabel();
+	    gliserver_url_label.setPreferredSize(LABEL_SIZE);
+	    String gliserver_url_string = "";
+	    if (Configuration.gliserver_url != null) {
+		gliserver_url_string = Configuration.gliserver_url.toString();
+	    }
+	    gliserver_url_field = new JTextField(gliserver_url_string);
+	    gliserver_url_field.setCaretPosition(0);
+	    Dictionary.registerText(gliserver_url_label, "Preferences.Connection.GLIServer_URL");
+	    Dictionary.registerTooltip(gliserver_url_field, "Preferences.Connection.GLIServer_URL_Tooltip");
+	}
+
+	JPanel site_pane = null;
+	JLabel site_label = null;
+	JPanel servlet_pane = null;
+	JLabel servlet_label = null;
+	if (Gatherer.GS3) {
+	    site_pane = new JPanel();
+	    site_pane.setPreferredSize(ROW_SIZE);
+	    site_label = new JLabel();
+	    Dictionary.registerText(site_label, "Preferences.Connection.Site");
+	    site_label.setPreferredSize(LABEL_SIZE);
+	    // what should we do if Gatherer.servlet_config.getSites() is null?
+	    site_combobox = new JComboBox(Gatherer.servlet_config.getSites().toArray());
+	    Dictionary.registerTooltip(site_combobox, "Preferences.Connection.Site_Tooltip");
+	    servlet_pane = new JPanel();
+	    servlet_pane.setPreferredSize(ROW_SIZE);
+	    servlet_label = new JLabel();
+	    Dictionary.registerText(servlet_label, "Preferences.Connection.Servlet");
+	    servlet_label.setPreferredSize(LABEL_SIZE);
+	    servlet_combobox = new JComboBox();
+	    // try to locate and select the current site
+	    String this_site = Configuration.site_name;
+	    for(int b = 0; b < site_combobox.getItemCount(); b++) {
+		String entry = (String) site_combobox.getItemAt(b);
+		if(this_site.equals(entry)) {
+		    site_combobox.setSelectedIndex(b);
+		    break;
+		}
+	    }
+
+	    // just in case its not the current one?
+	    current_site_selection = (String)site_combobox.getSelectedItem();
+
+	    ArrayList servlet_options = Gatherer.servlet_config.getServletsForSite(current_site_selection);
+	    if (servlet_options == null) {
+		//servlet_combobox.setModel(new DefaultComboBoxModel());
+		Dictionary.registerTooltip(servlet_combobox, "Preferences.Connection.Servlet_Tooltip2");
+		servlet_combobox.setEnabled(false);
+	    } else {
+		System.err.println(ArrayTools.objectArrayToString(servlet_options.toArray()));
+
+		servlet_combobox.setModel(new DefaultComboBoxModel(servlet_options.toArray()));
+		Dictionary.registerTooltip(servlet_combobox, "Preferences.Connection.Servlet_Tooltip");
+		servlet_combobox.setEnabled(true);
+		// try to locate and select the current servlet
+		String this_servlet = Configuration.getServletPath();
+		for(int b = 0; b < servlet_combobox.getItemCount(); b++) {
+		    String entry = (String) servlet_combobox.getItemAt(b);
+		    if(this_servlet.equals(entry)) {
+			servlet_combobox.setSelectedIndex(b);
+			break;
+		    }
+		}
+		
+	    }
+	}
+	
+	boolean currently_enabled = Configuration.get("general.use_proxy", true);
+	// Creation
+	JPanel connection_pane = new JPanel();
+	use_proxy_checkbox = new JCheckBox();
+	use_proxy_checkbox.setSelected(currently_enabled);
+	Dictionary.registerText(use_proxy_checkbox, "Preferences.Connection.Use_Proxy");
+
+	use_proxy_checkbox.setPreferredSize(ROW_SIZE);
+	JPanel proxy_host_pane = new JPanel();
+	proxy_host_pane.setPreferredSize(ROW_SIZE);
+	JLabel proxy_host_label = new JLabel();
+	proxy_host_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(proxy_host_label, "Preferences.Connection.Proxy_Host");
+	proxy_host_field = new JTextField(Configuration.getString("general.proxy_host", true));
+	proxy_host_field.setEnabled(currently_enabled);
+	Dictionary.registerTooltip(proxy_host_field, "Preferences.Connection.Proxy_Host_Tooltip");
+	JPanel proxy_port_pane = new JPanel();
+	proxy_port_pane.setPreferredSize(ROW_SIZE);
+	JLabel proxy_port_label = new JLabel();
+	proxy_port_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(proxy_port_label, "Preferences.Connection.Proxy_Port");
+	String port_value = Configuration.getString("general.proxy_port", true);
+	if(port_value.length() > 0) {
+	   proxy_port_field = new JSpinner(new SpinnerNumberModel((new Integer(port_value)).intValue(), 0, 65535, 1));
+	}
+	else {
+	   proxy_port_field = new JSpinner(new SpinnerNumberModel(0, 0, 65535, 1));
+	}
+	proxy_port_field.setEnabled(currently_enabled);
+	Dictionary.registerTooltip(proxy_port_field, "Preferences.Connection.Proxy_Port_Tooltip");
+
+	// Connection
+	use_proxy_checkbox.addActionListener(new UseProxyListener());
+	if (Gatherer.GS3) {
+	    site_combobox.addActionListener(new SiteComboboxListener());
+	}
+
+	// Layout
+	program_pane.setLayout(new BorderLayout());
+	program_pane.add(program_label, BorderLayout.WEST);
+	program_pane.add(program_field, BorderLayout.CENTER);
+
+	library_path_pane.setLayout(new BorderLayout());
+	library_path_pane.add(library_path_label, BorderLayout.WEST);
+	library_path_pane.add(library_path_field, BorderLayout.CENTER);
+
+	if (Gatherer.isGsdlRemote) {
+	    gliserver_url_pane.setLayout(new BorderLayout());
+	    gliserver_url_pane.add(gliserver_url_label, BorderLayout.WEST);
+	    gliserver_url_pane.add(gliserver_url_field, BorderLayout.CENTER);
+	}
+
+	if (Gatherer.GS3) {
+	    site_pane.setLayout(new BorderLayout());
+	    site_pane.add(site_label, BorderLayout.WEST);
+	    site_pane.add(site_combobox, BorderLayout.CENTER);
+
+	    servlet_pane.setLayout(new BorderLayout());
+	    servlet_pane.add(servlet_label, BorderLayout.WEST);
+	    servlet_pane.add(servlet_combobox, BorderLayout.CENTER);
+	}
+
+	proxy_host_pane.setLayout(new BorderLayout());
+	proxy_host_pane.add(proxy_host_label, BorderLayout.WEST);
+	proxy_host_pane.add(proxy_host_field, BorderLayout.CENTER);
+
+	proxy_port_pane.setLayout(new BorderLayout());
+	proxy_port_pane.add(proxy_port_label, BorderLayout.WEST);
+	proxy_port_pane.add(proxy_port_field, BorderLayout.CENTER);
+
+	connection_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	connection_pane.setLayout(new GridLayout(8,1,0,2));
+	connection_pane.add(program_pane);
+	connection_pane.add(library_path_pane);
+	if (Gatherer.isGsdlRemote) {
+	    connection_pane.add(gliserver_url_pane);
+	}
+	if (Gatherer.GS3) {
+	    connection_pane.add(site_pane);
+	    connection_pane.add(servlet_pane);
+	}
+	connection_pane.add(use_proxy_checkbox);
+	connection_pane.add(proxy_host_pane);
+	connection_pane.add(proxy_port_pane);
+
+	return connection_pane;
+    }
+
+    private JPanel createGeneralPreferences() {
+	JPanel general_pane = new JPanel();
+
+	// Build the model of available languages
+	ArrayList dictionary_model = new ArrayList();
+
+	// The new method makes use of the successor to the languages.dat file, classes/xml/languages.xml
+	NodeList language_elements = LanguageManager.LANGUAGES_DOCUMENT.getDocumentElement().getElementsByTagName(StaticStrings.LANGUAGE_ELEMENT);
+	for(int i = 0; i < language_elements.getLength(); i++) {
+	    Element language_element = (Element) language_elements.item(i);
+	    if((language_element.getAttribute(StaticStrings.GLI_ATTRIBUTE)).equalsIgnoreCase(StaticStrings.TRUE_STR) || (language_element.getAttribute(StaticStrings.MDS_ATTRIBUTE)).equalsIgnoreCase(StaticStrings.TRUE_STR)) {
+		Locale locale = new Locale(language_element.getAttribute(StaticStrings.CODE_ATTRIBUTE));
+		String description = language_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
+		DictionaryEntry entry = new DictionaryEntry(description, locale);
+		if(!dictionary_model.contains(entry)) {
+		    dictionary_model.add(entry);
+		}
+		entry = null;
+		description = null;
+		locale = null;
+	    }
+	    language_element = null;
+	}
+	language_elements = null;
+
+	// Users email
+	JPanel email_pane = new JPanel();
+	JLabel email_label = new JLabel();
+	email_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(email_label, "Preferences.General.Email");
+	email_field = new EmailField(Configuration.getColor("coloring.error_background", false));
+	email_field.setText(Configuration.getEmail());
+	Dictionary.registerTooltip(email_field, "Preferences.General.Email_Tooltip");
+
+	// Font selection
+	JPanel font_pane = new JPanel();
+	JLabel font_label = new JLabel();
+	font_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(font_label, "Preferences.General.Font");
+	font_field = new JTextField(Configuration.getString("general.font", true));
+	Dictionary.registerTooltip(font_field, "Preferences.General.Font_Tooltip");
+
+	// Extracted metadata
+	view_extracted_metadata_checkbox = new JCheckBox();
+	view_extracted_metadata_checkbox.setSelected(false);
+	if (Configuration.get("general.view_extracted_metadata", Configuration.COLLECTION_SPECIFIC)) {
+	    view_extracted_metadata_checkbox.setSelected(true);
+	}
+	Dictionary.registerBoth(view_extracted_metadata_checkbox, "Preferences.General.View_Extracted_Metadata", "Preferences.General.View_Extracted_Metadata_Tooltip");
+
+	// Show file sizes
+	show_file_size_checkbox = new JCheckBox();
+	show_file_size_checkbox.setSelected(false);
+	if (Configuration.get("general.show_file_size", Configuration.COLLECTION_SPECIFIC)) {
+	    show_file_size_checkbox.setSelected(true);
+	}
+	Dictionary.registerBoth(show_file_size_checkbox, "Preferences.General.Show_File_Size", "Preferences.General.Show_File_Size_Tooltip");
+
+	// Language
+	JPanel language_pane = new JPanel();
+	JLabel language_label = new JLabel();
+	language_label.setPreferredSize(LABEL_SIZE);
+	Dictionary.registerText(language_label, "Preferences.General.Interface_Language");
+	language_combobox = new JComboBox(dictionary_model.toArray());
+	Dictionary.registerTooltip(language_combobox, "Preferences.General.Interface_Language_Tooltip");
+	// Try to locate and select the current language
+	String language_code = Configuration.getLanguage();
+	for (int b = 0; b < language_combobox.getItemCount(); b++) {
+	    DictionaryEntry entry = (DictionaryEntry) language_combobox.getItemAt(b);
+	    if (language_code.equalsIgnoreCase(entry.getLocale().getLanguage())) {
+		language_combobox.setSelectedIndex(b);
+		break;
+	    }
+	}
+
+	// Layout
+	email_pane.setLayout(new BorderLayout());
+	email_pane.add(email_label, BorderLayout.WEST);
+	email_pane.add(email_field, BorderLayout.CENTER);
+
+	language_pane.setLayout(new BorderLayout());
+	language_pane.add(language_label, BorderLayout.WEST);
+	language_pane.add(language_combobox, BorderLayout.CENTER);
+
+	font_pane.setLayout(new BorderLayout());
+	font_pane.add(font_label, BorderLayout.WEST);
+	font_pane.add(font_field, BorderLayout.CENTER);
+
+	general_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	general_pane.setLayout(new GridLayout(5,1,0,5));
+	general_pane.add(email_pane);
+	general_pane.add(language_pane);
+	general_pane.add(font_pane);
+	general_pane.add(view_extracted_metadata_checkbox);
+	general_pane.add(show_file_size_checkbox);
+
+	return general_pane;
+    }
+
+    /** Generate the controls for altering the detail mode.
+     * @return a JPanel of controls
+     */
+    private JPanel createModePreferences() {
+	// Create Controls
+	JPanel mode_panel = new JPanel();
+	JPanel button_panel = new JPanel();
+	ButtonGroup mode_button_group = new ButtonGroup();
+	assistant_mode_radio_button = new JRadioButton(Dictionary.get("Preferences.Mode.Assistant"));
+	assistant_mode_radio_button.setOpaque(false);
+	mode_button_group.add(assistant_mode_radio_button);
+	expert_mode_radio_button = new JRadioButton(Dictionary.get("Preferences.Mode.Expert"));
+	expert_mode_radio_button.setOpaque(false);
+	mode_button_group.add(expert_mode_radio_button);
+	librarian_mode_radio_button = new JRadioButton(Dictionary.get("Preferences.Mode.Librarian"));
+	librarian_mode_radio_button.setOpaque(false);
+	mode_button_group.add(librarian_mode_radio_button);
+	systems_mode_radio_button = new JRadioButton(Dictionary.get("Preferences.Mode.Systems"));
+	systems_mode_radio_button.setOpaque(false);
+	mode_button_group.add(systems_mode_radio_button);
+	mode_description_textarea = new JTextArea();
+	mode_description_textarea.setEditable(false);
+	mode_description_textarea.setLineWrap(true);
+	mode_description_textarea.setWrapStyleWord(true);
+	// Determine which value is already selected
+	switch(Configuration.getMode()) {
+	case Configuration.ASSISTANT_MODE:
+	    assistant_mode_radio_button.setSelected(true);
+	    mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Assistant_Description"));
+	    break;
+	case Configuration.SYSTEMS_MODE:
+	    systems_mode_radio_button.setSelected(true);
+	    mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Systems_Description"));
+	    break;
+	case Configuration.EXPERT_MODE:
+	    expert_mode_radio_button.setSelected(true);
+	    mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Expert_Description"));
+	    break;
+	default:
+	    librarian_mode_radio_button.setSelected(true);
+	    mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Librarian_Description"));
+	}
+	// Connection - when a radio button is selected we have to change the description
+	ModeRadioButtonListener listener = new ModeRadioButtonListener();
+	assistant_mode_radio_button.addActionListener(listener);
+	expert_mode_radio_button.addActionListener(listener);
+	librarian_mode_radio_button.addActionListener(listener);
+	systems_mode_radio_button.addActionListener(listener);
+	listener = null;
+	// Layout
+	button_panel.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	button_panel.setLayout(new GridLayout(4,1,2,2));
+	button_panel.add(assistant_mode_radio_button);
+	button_panel.add(librarian_mode_radio_button);
+	button_panel.add(systems_mode_radio_button);
+	button_panel.add(expert_mode_radio_button);
+
+	mode_panel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	mode_panel.setLayout(new BorderLayout());
+	mode_panel.add(button_panel, BorderLayout.NORTH);
+	mode_panel.add(new JScrollPane(mode_description_textarea), BorderLayout.CENTER);
+
+	return mode_panel;
+    }
+
+
+    /** The warning preferences are controlled through a checklist. */
+    private JPanel createWarningPreferences()
+    {
+	warning_preferences_check_list = new CheckList(false);
+
+	// Read all the warnings from the general xml/config.xml file, and their values from the user config.xml file
+	Document general_config_xml_file_document = XMLTools.parseXMLFile("xml/config.xml", true);
+	NodeList argument_elements_nodelist = general_config_xml_file_document.getDocumentElement().getElementsByTagName("Argument");
+	for (int i = 0; i < argument_elements_nodelist.getLength(); i++) {
+	    Element argument_element = (Element) argument_elements_nodelist.item(i);
+	    String argument_element_name = argument_element.getAttribute("name");
+	    if (argument_element_name.startsWith("warning.")) {
+		String warning_title = Dictionary.get(argument_element_name.substring("warning.".length()) + ".Title");
+		boolean warning_enabled = Configuration.get(argument_element_name, true);
+		CheckListEntry warning_entry = new CheckListEntry(warning_title, warning_enabled);
+		warning_entry.setProperty(argument_element_name);
+		warning_preferences_check_list.addEntry(warning_entry);
+	    }
+	}
+
+	JPanel warning_preferences_pane = new JPanel();
+	warning_preferences_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	warning_preferences_pane.setLayout(new BorderLayout());
+	warning_preferences_pane.add(new JScrollPane(warning_preferences_check_list), BorderLayout.CENTER);
+	return warning_preferences_pane;
+    }
+
+
+    private JPanel createWorkflowPreferences() {
+	// Read in the predefined configurations file
+	Vector predefined = new Vector();
+	Document predefined_document = XMLTools.parseXMLFile("xml/workflows.xml", true);
+	Element predefined_element = predefined_document.getDocumentElement();
+	NodeList workflow_elements = predefined_element.getElementsByTagName("Workflow");
+	for(int i = 0; i < workflow_elements.getLength(); i++) {
+	    predefined.add(new WorkflowElementWrapper((Element)workflow_elements.item(i)));
+	}
+
+	// Creation
+	JPanel workflow_preferences_pane = new JPanel();
+	JPanel checklist_pane = new JPanel();
+	JLabel title_label = new JLabel();
+	title_label.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(title_label, "Preferences.Workflow.Title");
+
+	workflow_download = new JCheckBox();
+	workflow_download.setSelected(Configuration.get("workflow.download", false));
+	workflow_download.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(workflow_download, "Preferences.Workflow.Download");
+
+	workflow_gather = new JCheckBox();
+	workflow_gather.setSelected(Configuration.get("workflow.gather", false));
+	workflow_gather.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(workflow_gather, "Preferences.Workflow.Gather");
+
+	workflow_enrich = new JCheckBox();
+	workflow_enrich.setSelected(Configuration.get("workflow.enrich", false));
+	workflow_enrich.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(workflow_enrich, "Preferences.Workflow.Enrich");
+
+	workflow_design = new JCheckBox();
+	workflow_design.setSelected(Configuration.get("workflow.design", false));
+	workflow_design.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(workflow_design, "Preferences.Workflow.Design");
+
+	workflow_create = new JCheckBox();
+	workflow_create.setSelected(Configuration.get("workflow.create", false));
+	workflow_create.setPreferredSize(ROW_SIZE);
+	Dictionary.registerText(workflow_create, "Preferences.Workflow.Create");
+
+	JPanel predefined_pane = new JPanel();
+	JLabel predefined_label = new JLabel();
+	Dictionary.registerText(predefined_label, "Preferences.Workflow.Predefined.Label");
+	JComboBox predefined_combobox = new JComboBox(predefined);
+
+	// Connection
+	predefined_combobox.addActionListener(new PredefinedActionListener());
+
+	// Layout
+	checklist_pane.setLayout(new BoxLayout(checklist_pane, BoxLayout.Y_AXIS));
+	checklist_pane.add(title_label);
+	if (Configuration.get("workflow.download", true)) {
+	    checklist_pane.add(workflow_download);
+	}
+	if (Configuration.get("workflow.gather", true)) {
+	    checklist_pane.add(workflow_gather);
+	}
+	if (Configuration.get("workflow.enrich", true)) {
+	    checklist_pane.add(workflow_enrich);
+	}
+	if (Configuration.get("workflow.design", true)) {
+	    checklist_pane.add(workflow_design);
+	}
+	if (Configuration.get("workflow.create", true)) {
+	    checklist_pane.add(workflow_create);
+	}
+
+	predefined_pane.setLayout(new BorderLayout(5,0));
+	predefined_pane.add(predefined_label, BorderLayout.WEST);
+	predefined_pane.add(predefined_combobox, BorderLayout.CENTER);
+
+	workflow_preferences_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	workflow_preferences_pane.setLayout(new BorderLayout());
+	workflow_preferences_pane.add(checklist_pane, BorderLayout.CENTER);
+	workflow_preferences_pane.add(predefined_pane, BorderLayout.SOUTH);
+
+	return workflow_preferences_pane;
+    }
+
+
+    private class OKButtonListener
+	implements ActionListener {
+	private boolean close;
+	public OKButtonListener(boolean close) {
+	    this.close = close;
+	}
+	public void actionPerformed(ActionEvent event)
+	{
+	    // Connection preferences
+	    String program_str = program_field.getText();
+	    if (program_str.length() > 0 && program_str.indexOf("%1") == -1) {
+		program_str = program_str + " %1";
+	    }
+	    Configuration.setPreviewCommand(program_str);
+
+	    String library_url_string = library_path_field.getText();
+	    if (library_url_string.equals("")) {
+		Configuration.library_url = null;
+	    }
+	    else {
+		try {
+		    Configuration.library_url = new URL(library_url_string);
+		}
+		catch (MalformedURLException exception) {
+		    DebugStream.printStackTrace(exception);
+		}
+	    }
+	    Configuration.setString("general.library_url", true, library_url_string);
+
+	    if (Gatherer.isGsdlRemote) {
+		String gliserver_url_string = gliserver_url_field.getText();
+		if (gliserver_url_string.equals("")) {
+		    Configuration.gliserver_url = null;
+		}
+		else {
+		    try {
+			Configuration.gliserver_url = new URL(gliserver_url_string);
+		    }
+		    catch (MalformedURLException exception) {
+			DebugStream.printStackTrace(exception);
+		    }
+		}
+		Configuration.setString("general.gliserver_url", true, gliserver_url_string);
+	    }
+
+	    boolean site_changed = false;
+	    if (Gatherer.GS3) {
+		String current_site = Configuration.site_name;
+		String new_site =(String)site_combobox.getSelectedItem() ;
+		if (!new_site.equals(current_site)) {
+		    site_changed = true;
+		}
+		Configuration.setSiteAndServlet(new_site, (String)servlet_combobox.getSelectedItem());
+	    }
+
+	    Configuration.set("general.use_proxy", true, use_proxy_checkbox.isSelected());
+	    Configuration.setString("general.proxy_host", true, proxy_host_field.getText());
+	    Configuration.setString("general.proxy_port", true, proxy_port_field.getValue() + "");
+	    Gatherer.setProxy();
+
+	    // General preferences
+	    Configuration.setEmail(email_field.getText());
+	    Configuration.set("general.show_file_size", Configuration.COLLECTION_SPECIFIC, show_file_size_checkbox.isSelected());
+	    Configuration.set("general.view_extracted_metadata", Configuration.COLLECTION_SPECIFIC, view_extracted_metadata_checkbox.isSelected());
+
+	    // Two options requiring restarting the GLI to apply: interface font, and interface language
+	    boolean restart_required = false;
+
+	    // GLI interface font
+	    String current_font = Configuration.getString("general.font", true);
+	    if (!current_font.equals(font_field.getText())) {
+		Configuration.setString("general.font", true, font_field.getText());
+		restart_required = true;
+	    }
+
+	    // GLI interface language
+	    String current_lang = Configuration.getLanguage();
+	    if (!current_lang.equals(((DictionaryEntry) language_combobox.getSelectedItem()).getLocale().getLanguage())) {
+		Configuration.setLocale("general.locale", Configuration.GENERAL_SETTING, ((DictionaryEntry) language_combobox.getSelectedItem()).getLocale());
+		restart_required = true;
+
+		// Delete the plugins.dat and classifiers.dat files
+		PluginManager.clearPluginCache();
+		ClassifierManager.clearClassifierCache();
+	    }
+
+	    // Inform the user that a restart is required, if necessary
+	    if (restart_required) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Preferences.General.Restart_Required"), Dictionary.get("General.Warning"), JOptionPane.WARNING_MESSAGE);
+	    }
+
+	    // Mode preferences
+	    int current_mode = Configuration.getMode();
+	    int new_mode;
+	    if (assistant_mode_radio_button.isSelected()) {
+		new_mode = Configuration.ASSISTANT_MODE;
+	    }
+	    else if (systems_mode_radio_button.isSelected()) {
+		new_mode = Configuration.SYSTEMS_MODE;
+	    }
+	    else if (expert_mode_radio_button.isSelected()) {
+		new_mode = Configuration.EXPERT_MODE;
+	    }
+	    else {
+		new_mode = Configuration.LIBRARIAN_MODE;
+	    }
+
+	    // If there has been a change in modes, update the config, and also inform the persistant gui views that have a need to know
+	    if (new_mode != current_mode) {
+		Configuration.setMode(new_mode);
+		Collection collection = Gatherer.c_man.getCollection();
+		if (collection != null) {
+		    collection.cdm.modeChanged(new_mode);
+		}
+		Gatherer.g_man.modeChanged(new_mode);
+	    }
+
+	    // Warning preferences
+	    ListModel warning_preferences_check_list_model = warning_preferences_check_list.getModel();
+	    for (int i = 0; i < warning_preferences_check_list_model.getSize(); i++) {
+		CheckListEntry entry = (CheckListEntry) warning_preferences_check_list_model.getElementAt(i);
+		Configuration.set(entry.getProperty(), true, entry.isSelected());
+	    }
+
+	    if (Gatherer.GS3 && site_changed && Gatherer.c_man.getCollection() != null) {
+		// shut down the collection
+		System.err.println("shutting down teh collection");
+		Gatherer.g_man.saveThenCloseCurrentCollection();
+	    }
+
+	    // Workflow preferences
+	    Configuration.set("workflow.download", false, workflow_download.isSelected());
+	    Configuration.set("workflow.gather", false, workflow_gather.isSelected());
+	    Configuration.set("workflow.enrich", false, workflow_enrich.isSelected());
+	    Configuration.set("workflow.design", false, workflow_design.isSelected());
+	    Configuration.set("workflow.create", false, workflow_create.isSelected());
+	    Gatherer.g_man.workflowUpdate("Download", workflow_download.isSelected());
+	    Gatherer.g_man.workflowUpdate("Gather", workflow_gather.isSelected());
+	    Gatherer.g_man.workflowUpdate("Enrich", workflow_enrich.isSelected());
+	    Gatherer.g_man.workflowUpdate("Design", (workflow_design.isSelected() && Configuration.getMode() > Configuration.ASSISTANT_MODE));
+	    Gatherer.g_man.workflowUpdate("Create", workflow_create.isSelected());
+
+	    // Always save configuration changes immediately (in case the GLI crashes)
+	    Configuration.save();
+
+	    // Refresh the GLI to account for the configuration changes
+	    Gatherer.refresh(Gatherer.PREFERENCES_CHANGED);
+
+	    // Hide dialog
+	    if (close) {
+		self.dispose();
+	    }
+	}
+    }
+
+    private class CancelButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    self.dispose();
+	}
+    }
+
+    private class DictionaryEntry
+	implements Comparable {
+	private Locale locale;
+	private String description;
+	public DictionaryEntry(Locale locale) {
+	    this.description = null;
+	    this.locale = locale;
+	}
+	public DictionaryEntry(String description, Locale locale) {
+	    this.description = description;
+	    this.locale = locale;
+	}
+	public int compareTo(Object object) {
+	    return toString().compareTo(object.toString());
+	}
+	public boolean equals(Object object) {
+	    return toString().equals(object.toString());
+	}
+	public Locale getLocale() {
+	    return locale;
+	}
+	public String toString() {
+	    if(description != null) {
+		return description;
+	    }
+	    else {
+		return locale.getDisplayName();
+	    }
+	}
+    }
+
+    /** This listener updates the mode description depending on what mode is selected. */
+    private class ModeRadioButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    Object source = event.getSource();
+	    if(source == assistant_mode_radio_button) {
+		mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Assistant_Description"));
+	    }
+	    else if(source == expert_mode_radio_button) {
+		mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Expert_Description"));
+	    }
+	    else if(source == systems_mode_radio_button) {
+		mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Systems_Description"));
+	    }
+	    else {
+		mode_description_textarea.setText(Dictionary.get("Preferences.Mode.Librarian_Description"));
+	    }
+	    source = null;
+	}
+    }
+
+    private class PredefinedActionListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    JComboBox cb = (JComboBox) event.getSource();
+	    WorkflowElementWrapper element = (WorkflowElementWrapper) cb.getSelectedItem();
+	    CheckboxUpdater task = new CheckboxUpdater(element);
+	    SwingUtilities.invokeLater(task);
+	}
+
+	private class CheckboxUpdater
+	    implements Runnable {
+	    private WorkflowElementWrapper element;
+	    public CheckboxUpdater(WorkflowElementWrapper element) {
+		this.element = element;
+	    }
+	    public void run() {
+		workflow_download.setSelected(element.getEnabled("download"));
+		workflow_gather.setSelected(element.getEnabled("gather"));
+		workflow_enrich.setSelected(element.getEnabled("enrich"));
+		workflow_design.setSelected(element.getEnabled("design"));
+		workflow_create.setSelected(element.getEnabled("create"));
+	    }
+	}
+    }
+
+
+    private class SiteComboboxListener
+	implements ActionListener {
+	private boolean ignore_event=false;
+	public void actionPerformed(ActionEvent event) {
+	    System.err.println("event occurred "+event.paramString());
+	    String site = (String) site_combobox.getSelectedItem();
+	    System.err.println("new site = "+site);
+	    if (!site.equals(current_site_selection)) {
+		current_site_selection = site;
+		System.err.println("changed the current selection");
+		ArrayList servlet_options = Gatherer.servlet_config.getServletsForSite(current_site_selection);
+		if (servlet_options == null) {
+		    System.err.println("no servlets for this site");
+		    servlet_combobox.setModel(new DefaultComboBoxModel());
+		    Dictionary.registerTooltip(servlet_combobox, "Preferences.Connection.Servlet_Tooltip2");
+		    servlet_combobox.setEnabled(false);
+
+		} else {
+		    System.err.println(ArrayTools.objectArrayToString(servlet_options.toArray()));
+		    servlet_combobox.setModel(new DefaultComboBoxModel(servlet_options.toArray()));
+		    Dictionary.registerTooltip(servlet_combobox, "Preferences.Connection.Servlet_Tooltip");
+		    servlet_combobox.setEnabled(true);
+		}
+	    }
+	}    
+    }
+
+    private class UseProxyListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    boolean enabled = use_proxy_checkbox.isSelected();
+	    Configuration.set("general.use_proxy", true, enabled);
+	    // Fortunately this is already driven by the event thread.
+	    proxy_host_field.setEnabled(enabled);
+	    proxy_port_field.setEnabled(enabled);
+	}
+    }
+
+    private class WorkflowElementWrapper {
+	private Element element;
+	private String text;
+	public WorkflowElementWrapper(Element element) {
+	    this.element = element;
+	}
+	public boolean getEnabled(String name) {
+	    boolean result = true;
+	    if(element.getAttribute(name).equalsIgnoreCase("false")) {
+		result = false;
+	    }
+	    return result;
+	}
+	public String toString() {
+	    if (text == null) {
+		text = Dictionary.get(element.getFirstChild().getNodeValue());
+	    }
+	    return text;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/PreviewCommandDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/PreviewCommandDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/PreviewCommandDialog.java	(revision 21112)
@@ -0,0 +1,267 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.filechooser.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.Utility;
+
+/** This dialog allows the user to choose which browser to use for the preview command 
+ * @author Katherine Don, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class PreviewCommandDialog
+    extends ModalDialog {
+
+    /** The default size for the dialog. */
+    static final private Dimension SIZE = new Dimension(500, 195);
+
+    private PreviewCommandDialog self;
+    private JButton browse_button;
+    private JButton cancel_button;
+    private JButton ok_button;
+    private JTextField command_field;
+    private String preview_command=null;
+    /** Create a new PreviewCommandDialog
+     */
+    public PreviewCommandDialog() {
+	super(Gatherer.g_man);
+	this.self = this;
+		  
+	// Creation
+	setModal(true);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("thepreviewview"));
+
+	JPanel content_pane = (JPanel) getContentPane();
+	content_pane.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
+
+	JTextArea instructions_area = new JTextArea();
+	instructions_area.setEditable(false);
+	instructions_area.setLineWrap(true);
+	instructions_area.setRows(5);
+	instructions_area.setWrapStyleWord(true);
+	Dictionary.setText(instructions_area, "PreviewCommandDialog.Instructions");
+
+	JPanel button_pane = new JPanel();
+	JPanel lower_pane = new JPanel();
+
+	JPanel command_pane = new JPanel();
+	command_field = new JTextField();
+	browse_button = new GLIButton();
+	browse_button.setEnabled(!Utility.isMac());
+	browse_button.setMnemonic(KeyEvent.VK_B);
+
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+
+	// Connection
+	browse_button.addActionListener(new BrowseButtonListener());
+	ok_button.addActionListener(new OkButtonListener());
+	cancel_button.addActionListener(new CancelButtonListener());
+	if(Utility.isMac()) {
+	    Dictionary.setBoth(browse_button, "FileAssociationDialog.Browse", "FileAssociationDialog.Browse_Tooltip_Mac");
+	}
+	else {
+	    Dictionary.setBoth(browse_button, "FileAssociationDialog.Browse", "FileAssociationDialog.Browse_Tooltip");
+	}
+
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	Dictionary.setBoth(ok_button, "General.OK", "FileAssociationDialog.Close_Tooltip");	
+	Dictionary.setText(this, "PreviewCommandDialog.Title");
+
+	// Layout
+	command_pane.setBorder(BorderFactory.createEmptyBorder(2,0,2,0));
+	command_pane.setLayout(new BorderLayout());
+	command_pane.add(command_field, BorderLayout.CENTER);
+	command_pane.add(browse_button, BorderLayout.EAST);
+
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+
+	button_pane.setLayout(new GridLayout(1,2,5,0));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(2,0,0,0));
+	lower_pane.setLayout(new BorderLayout());
+	lower_pane.add(command_pane, BorderLayout.CENTER);
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JScrollPane(instructions_area), BorderLayout.NORTH);
+	content_pane.add(lower_pane, BorderLayout.CENTER);
+
+	Rectangle screen = Gatherer.g_man.getBounds(null);
+	setLocation((int)(screen.getX() + (screen.getWidth() - SIZE.width) / 2), (int)(screen.getY() + (screen.getHeight() - SIZE.height) / 2));
+	screen = null;
+    }
+
+    public void destroy() {
+	// Disconnect
+	// Clean up
+	self = null;
+    }
+
+    /** Redisplay the dialog
+     */
+    public String display() {
+	setVisible(true);
+	return preview_command;
+    }
+
+
+    private class OkButtonListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent event) {
+	    preview_command = command_field.getText();
+	    self.dispose();
+	    
+	}
+    }
+
+
+    /** Whenever the user clicks the browse button, we should open up a file browser to allow them to select an executable file from somewhere in the file system. */
+    private class BrowseButtonListener 
+	implements ActionListener {
+	/** Open up a simple JFileChooser when the user clicks the button. 
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    JFileChooser chooser = new JFileChooser(new File(Gatherer.getGLIUserDirectoryPath()));
+	    OpenCollectionDialog.disableRename(chooser);
+	    chooser.setDialogTitle(Dictionary.get("FileAssociationDialog.Browse_Title"));
+	    chooser.setFileFilter(new BatchFileFilter());
+	    chooser.setFileFilter(new CoreObjectModelFileFilter());
+	    chooser.setFileFilter(new ExecutableFileFilter());
+	    chooser.setAcceptAllFileFilterUsed(true);
+	    if(chooser.showOpenDialog(Gatherer.g_man) == JFileChooser.APPROVE_OPTION) {
+		command_field.setText(chooser.getSelectedFile().getAbsolutePath());
+	    }
+	}
+    }
+
+    
+    private class CancelButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    self.dispose();
+	}
+    }
+
+    /** Batch filter shows only files ending in bat. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class BatchFileFilter
+	extends FileFilter {
+
+	/** Accept all .exe files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".bat");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Batch_File");
+	}
+    }
+
+    /** Command filter shows only files ending in com. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class CoreObjectModelFileFilter
+	extends FileFilter {
+
+	/** Accept all .com files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".com");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Command_File");
+	}
+    }
+
+
+    /** Executable filter shows only files ending in exe. Based on ImageFilter.java which is a 1.4 example used by FileChooserDemo2.java. */
+    private class ExecutableFileFilter
+	extends FileFilter {
+
+	/** Accept all .exe files
+	 * @param file a File
+	 * @return true is this file should be shown, false otherwise
+	 */
+	public boolean accept(File file) {
+	    return file.getName().toLowerCase().endsWith(".exe");
+	}
+
+	/** The description of this filter
+	 * @return a String
+	 */
+	public String getDescription() {
+	    return Dictionary.get("FileAssociationDialog.Executable_File");
+	}
+    }
+
+
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleMenuBar.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleMenuBar.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleMenuBar.java	(revision 21112)
@@ -0,0 +1,34 @@
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.JarTools;
+
+public class SimpleMenuBar
+    extends JMenuBar
+    implements ActionListener {
+
+    private String page_name;
+
+    public SimpleMenuBar(String page_name) {
+	this.page_name = page_name;
+	JMenu help = new JMenu();
+	help.setIcon(JarTools.getImage("help.gif"));
+	Dictionary.setText(help, "Menu.Help");
+
+	JMenuItem help_help = new JMenuItem();
+	help_help.addActionListener(this);
+	Dictionary.setText(help_help, "Menu.Help");
+
+	help.add(help_help);
+
+	add(Box.createHorizontalGlue());
+	add(help);
+    }
+
+    public void actionPerformed(ActionEvent event) {
+	HelpFrame.setView(page_name);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleOpenCollectionDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleOpenCollectionDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleOpenCollectionDialog.java	(revision 21112)
@@ -0,0 +1,330 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/** A dialog which provides a straight-forward access to the currently installed collections. It also will contain the ability to continue through to the original OpenCollectionDialog if your source collection is located somewhere other than the gsdl collect folder. */
+public class SimpleOpenCollectionDialog
+    extends ModalDialog {
+
+    static final public int OK_OPTION = 0;
+    static final public int CANCEL_OPTION = 1;
+    static final public int BROWSE_OPTION = 2;
+
+    static final private Dimension SIZE = new Dimension(640,480);
+    static final private String BLANK = "b";
+    static final private String DESCRIPTION = "d";
+
+    private CardLayout card_layout;
+    private int result;
+    private JButton advanced_button;
+    private JButton cancel_button;
+    private JButton open_button;
+    private JList collection_list;
+    private JTextArea description_textarea;
+    private JPanel description_pane;
+    private String filename;
+
+    public SimpleOpenCollectionDialog() {
+	super(Gatherer.g_man, "", true);
+	setSize(SIZE);
+	Dictionary.setText(this, "OpenCollectionDialog.Title");
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+
+	JPanel center_pane = new JPanel();
+
+	JPanel collection_list_pane = new JPanel();
+	JLabel collection_list_label = new JLabel();
+	Dictionary.setText(collection_list_label, "OpenCollectionDialog.Available_Collections");
+	collection_list = new JList(new CollectionListModel());
+	description_pane = new JPanel();
+	card_layout = new CardLayout();
+
+	JPanel blank_pane = new JPanel();
+
+	JPanel description_textarea_pane = new JPanel();
+	JLabel description_textarea_label = new JLabel();
+	Dictionary.setText(description_textarea_label, "OpenCollectionDialog.Description");
+	description_textarea = new JTextArea();
+
+	JPanel button_pane = new JPanel();
+	open_button = new GLIButton();
+	open_button.setEnabled(false);
+	open_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(open_button, "OpenCollectionDialog.Open", "OpenCollectionDialog.Open_Tooltip");
+	advanced_button = new GLIButton();
+	advanced_button.setMnemonic(KeyEvent.VK_B);
+	Dictionary.setBoth(advanced_button, "OpenCollectionDialog.Browse", "OpenCollectionDialog.Browse_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	// Connection
+	advanced_button.addActionListener(new AdvancedListener());
+	cancel_button.addActionListener(new CancelListener());
+	open_button.addActionListener(new OpenListener());
+	CollectionListSelectionListener clsl = new CollectionListSelectionListener();
+	collection_list.addListSelectionListener(clsl);
+	collection_list.addMouseListener(clsl);
+	clsl = null;
+
+	// Layout
+	collection_list_pane.setLayout(new BorderLayout());
+	collection_list_pane.add(collection_list_label, BorderLayout.NORTH);
+	collection_list_pane.add(new JScrollPane(collection_list), BorderLayout.CENTER);
+
+	description_textarea_pane.setLayout(new BorderLayout());
+	description_textarea_pane.add(description_textarea_label, BorderLayout.NORTH);
+	description_textarea_pane.add(new JScrollPane(description_textarea), BorderLayout.CENTER);
+
+	description_pane.setLayout(card_layout);
+	description_pane.add(description_textarea_pane, DESCRIPTION);
+	description_pane.add(blank_pane, BLANK);
+
+	center_pane.setLayout(new GridLayout(2,1,0,5));
+	center_pane.add(collection_list_pane);
+	center_pane.add(description_pane);
+
+	button_pane.setLayout(new GridLayout(1,4));
+	button_pane.add(new JPanel());
+	button_pane.add(open_button);
+	if (!Gatherer.GS3) { // Temporarily disable Browse for GS3
+	    button_pane.add(advanced_button);
+	}
+	button_pane.add(cancel_button);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(center_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	screen_size = null;
+    }
+
+    public void destroy() {
+    }
+
+    public int display() {
+	setVisible(true);
+	return result;
+    }
+
+    public String getFileName() {
+	return this.filename;
+    }
+    private class AdvancedListener
+	implements ActionListener {
+
+	public void actionPerformed(ActionEvent event) {
+	    result = BROWSE_OPTION;
+	    SimpleOpenCollectionDialog.this.dispose();
+	}
+    }
+
+    private class CancelListener
+	implements ActionListener {
+
+	public void actionPerformed(ActionEvent event) {
+	    result = CANCEL_OPTION;
+	    SimpleOpenCollectionDialog.this.dispose();
+	}
+    }
+
+    private class CollectionListSelectionListener
+	extends MouseAdapter
+	implements ListSelectionListener {
+
+	public void mouseClicked(MouseEvent event) {
+	    ///ystem.err.println("Mouse clicked");
+	    if(event.getClickCount() >= 2) {
+		Point location = event.getPoint();
+		int index = collection_list.locationToIndex(location);
+		collection_list.setSelectedIndex(index);
+		location = null;
+		open_button.doClick();
+	    }
+	}
+
+	public void valueChanged(ListSelectionEvent event) {
+	    if(collection_list.isSelectionEmpty()) {
+		card_layout.show(description_pane, BLANK);
+		open_button.setEnabled(false);
+	    }
+	    else {
+		BasicCollectionConfiguration collection_configuration = (BasicCollectionConfiguration) collection_list.getSelectedValue();
+		description_textarea.setText(collection_configuration.getDescription());
+		description_textarea.setCaretPosition(0);
+		card_layout.show(description_pane, DESCRIPTION);
+		open_button.setEnabled(true);
+	    }
+	}
+    }
+
+    // use this if we ever go back to viewing all colls at once
+    /*    private class GS3CollectionListModel
+	extends AbstractListModel {
+
+	private TreeSet data;
+
+	public GS3CollectionListModel() {
+	    data = new TreeSet();
+	    File sites_folder = new File(Utility.getSitesDir(Configuration.gsdl3_path));
+	    File sites[] = sites_folder.listFiles();
+	    for (int s=0; s<sites.length; s++) {
+		String site_name = sites[s].getName();
+		System.err.println("found site "+site_name);
+		File collect_folder = new File(Utility.getCollectDir(Configuration.gsdl3_path, site_name) );
+		if (!collect_folder.exists()) {
+		    continue;
+		}
+		File[] collection_folders = collect_folder.listFiles();
+		for(int i = 0; i < collection_folders.length; i++) {
+		    File collection_folder = collection_folders[i];
+		    String collection_foldername = collection_folder.getName();
+		    if(!collection_folder.isFile() && !collection_foldername.equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+			BasicCollectionConfiguration collection_configuration = new BasicCollectionConfiguration(new File(collection_folder, Utility.CONFIG_FILE), site_name);
+			if(!collection_configuration.getName().equals(StaticStrings.ERROR_STR)) {
+			    data.add(collection_configuration);
+			}
+		    }
+		    collection_foldername = null;
+		    collection_folder = null;
+		} // for each collection
+		collection_folders = null;
+		collect_folder = null;
+	    } // for each site
+	    sites_folder = null;
+	    sites = null;
+	}
+
+	public Object getElementAt(int index) {
+	    Iterator iterator = data.iterator();
+	    for(int i = 0; iterator.hasNext(); i++) {
+		Object object = iterator.next();
+		if(i == index) {
+		    iterator = null;
+		    return object;
+		}
+		object = null;
+	    }
+	    iterator = null;
+	    return null;
+	}
+
+	public int getSize() {
+	    return data.size();
+	}
+    }
+    */	
+    private class CollectionListModel
+	extends AbstractListModel {
+
+	private TreeSet data;
+
+	public CollectionListModel() {
+	    data = new TreeSet();
+	    File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	    if (!collect_directory.exists()) {
+		collect_directory = null;
+		return;
+	    }
+	    File[] collection_folders = collect_directory.listFiles();
+	    for(int i = 0; i < collection_folders.length; i++) {
+		File collection_folder = collection_folders[i];
+		String collection_foldername = collection_folder.getName();
+		if(collection_folder.isDirectory() && !collection_foldername.equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+		    File config_file = new File(collection_folder, Utility.CONFIG_FILE);
+		    if (config_file.exists()) {
+			BasicCollectionConfiguration collection_configuration = new BasicCollectionConfiguration(config_file);
+			data.add(collection_configuration);
+		    }
+		    config_file = null;
+		}
+		collection_foldername = null;
+		collection_folder = null;
+	    }
+	    collection_folders = null;
+	    collect_directory = null;
+	}
+
+	public Object getElementAt(int index) {
+	    Iterator iterator = data.iterator();
+	    for(int i = 0; iterator.hasNext(); i++) {
+		Object object = iterator.next();
+		if(i == index) {
+		    iterator = null;
+		    return object;
+		}
+		object = null;
+	    }
+	    iterator = null;
+	    return null;
+	}
+
+	public int getSize() {
+	    return data.size();
+	}
+    }
+
+    private class OpenListener
+	implements ActionListener {
+
+	public void actionPerformed(ActionEvent event) {
+	    result = OK_OPTION;
+	    Object selected_object = collection_list.getSelectedValue();
+	    if (selected_object instanceof BasicCollectionConfiguration) {
+		BasicCollectionConfiguration collection_configuration = (BasicCollectionConfiguration)selected_object; //(BasicCollectionConfiguration) collection_list.getSelectedValue();
+		//*************888
+		File collect_cfg_file = collection_configuration.getFile(); // returns the collect.cfg file
+		File etc_folder = collect_cfg_file.getParentFile();
+		File collection_folder = etc_folder.getParentFile();
+		filename = collection_folder.getAbsolutePath() + File.separator + collection_folder.getName() + Utility.GLI_EXTENSION;
+		collection_folder = null;
+		etc_folder = null;
+		collect_cfg_file = null;
+		collection_configuration = null;
+		SimpleOpenCollectionDialog.this.dispose();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleResultDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleResultDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/SimpleResultDialog.java	(revision 21112)
@@ -0,0 +1,104 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.event.*;
+
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.Dictionary;
+
+public class SimpleResultDialog
+    extends ModalDialog {
+    
+    private Dimension size = new Dimension(600,300);
+    private SimpleResultDialog self;
+    
+    public SimpleResultDialog(String title, String label, String results) {
+	super(Gatherer.g_man, title, true);
+	this.self = this;
+	
+	setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+	setSize(size);
+	
+	JPanel content_pane = (JPanel) getContentPane();
+	JLabel result_label = new JLabel(label);
+	
+	JPanel button_pane = new JPanel();
+	JButton close_button = new GLIButton(Dictionary.get("General.Close"));
+	close_button.addActionListener(new CloseButtonListener());
+	
+	JPanel output_pane = new JPanel();
+	JLabel output_label = new JLabel();
+	Dictionary.registerText(output_label, "General.Review_Output");
+	
+	JTextArea output_textarea = new JTextArea(results);
+	output_textarea.setCaretPosition(0);
+	output_textarea.setEditable(false);
+	output_textarea.setLineWrap(true);
+	output_textarea.setWrapStyleWord(true);
+	
+	button_pane.setLayout(new BorderLayout());
+	button_pane.add(close_button, BorderLayout.EAST);
+	
+	output_pane.setBorder(BorderFactory.createEmptyBorder(5,0,5,0));
+	output_pane.setLayout(new BorderLayout());
+	output_pane.add(output_label, BorderLayout.NORTH);
+	output_pane.add(new JScrollPane(output_textarea), BorderLayout.CENTER);
+	
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(result_label, BorderLayout.NORTH);
+	content_pane.add(output_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+	
+	// Position
+	Rectangle frame_bounds = Gatherer.g_man.getBounds();
+	setLocation(frame_bounds.x + (frame_bounds.width - size.width) / 2, frame_bounds.y + (frame_bounds.height - size.height) / 2);
+    }
+    
+    private class CloseButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    self.setVisible(false);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/URLField.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/URLField.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/URLField.java	(revision 21112)
@@ -0,0 +1,36 @@
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.net.*;
+import javax.swing.*;
+
+
+public class URLField
+    extends JTextField
+{
+    public URLField(Color foreground, Color background)
+    {
+	super();
+	setBackground(background);
+	setForeground(foreground);
+    }
+
+
+    public boolean validateURL()
+    {
+	String url_string = getText();
+	if (!url_string.equals("")) {
+	    // Check the URL string is valid by trying to create a URL object from it
+	    try {
+		new URL(url_string);
+	    }
+	    catch (MalformedURLException exception) {
+		// URL string is invalid
+		return false;
+	    }
+	}
+
+	// URL string is valid
+	return true;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WarningDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WarningDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WarningDialog.java	(revision 21112)
@@ -0,0 +1,259 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Project, NZDL, University of Waikato
+ *
+ * Copyright (C) 2003 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.JarTools;
+
+
+/** A dialog that warns about something, and allows the user to turn off the warning in the future. */
+public class WarningDialog
+    extends ModalDialog 
+    implements ActionListener, KeyListener
+{
+    static final private Dimension NORMAL_SIZE = new Dimension(450, 160);
+    static final private Dimension SETTING_SIZE = new Dimension(450, 200);
+
+    private int result = JOptionPane.CANCEL_OPTION;
+    private JButton cancel_button;
+    private JButton ok_button;
+    private JCheckBox show_check;
+    private JTextField value_field;
+    private JPanel value_panel;
+    private String affected_property;
+    private String full_property;
+
+
+    public WarningDialog(String warning_name, String warning_title_key, String warning_message, String affected_property, boolean can_cancel)
+    {
+	super(Gatherer.g_man, "Warning", true);
+
+	// Determine the name of this prompt.
+	this.affected_property = affected_property;
+	this.full_property = warning_name;
+
+	// Now build dialog.
+	if (affected_property != null) {
+	    setSize(SETTING_SIZE);
+	}
+	else {
+	    setSize(NORMAL_SIZE);
+	}
+	Dictionary.setText(this, warning_title_key);
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel text_pane = new JPanel();
+	JLabel icon_label = new JLabel(JarTools.getImage("gatherer_medium.gif"));
+
+	JTextArea text_area = new JTextArea();
+	text_area.setCaretPosition(0);
+	text_area.setEditable(false);
+	text_area.setLineWrap(true);
+	text_area.setText(warning_message);
+	text_area.setWrapStyleWord(true);
+
+	value_panel = new JPanel();
+	JLabel value_label = new JLabel();
+	Dictionary.setText(value_label, "WarningDialog.Value");
+
+	value_field = new JTextField();
+	JPanel bottom_pane = new JPanel();
+	show_check = new JCheckBox();
+	Dictionary.setText(show_check, "WarningDialog.Dont_Show_Again");
+	JPanel control_pane = new JPanel();
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_O);
+	Dictionary.setBoth(ok_button, "General.OK", "General.OK_Tooltip");
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+
+	// Connection
+	ok_button.addActionListener(this);
+	cancel_button.addActionListener(this);
+	ok_button.addKeyListener(this);
+	cancel_button.addKeyListener(this);
+	getRootPane().setDefaultButton(ok_button);
+
+	// Layout
+	icon_label.setBorder(BorderFactory.createEmptyBorder(0,0,0,5));
+
+	value_label.setBorder(BorderFactory.createEmptyBorder(0, icon_label.getPreferredSize().width, 0, 0));
+
+	value_panel.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	value_panel.setLayout(new BorderLayout(5,0));
+	value_panel.add(value_label, BorderLayout.WEST);
+	value_panel.add(value_field, BorderLayout.CENTER);
+
+	text_pane.setLayout(new BorderLayout());
+	text_pane.add(icon_label, BorderLayout.WEST);
+	text_pane.add(new JScrollPane(text_area), BorderLayout.CENTER);
+	if(affected_property != null) {
+	    text_pane.add(value_panel, BorderLayout.SOUTH);
+	}
+
+	if(can_cancel) {
+	    control_pane.setLayout(new GridLayout(1,2,5,0));
+	    control_pane.add(ok_button);
+	    control_pane.add(cancel_button);
+	}
+	else {
+	    control_pane.setLayout(new BorderLayout());
+	    control_pane.add(ok_button, BorderLayout.EAST);
+	}
+
+	bottom_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	bottom_pane.setLayout(new BorderLayout());
+	bottom_pane.add(show_check, BorderLayout.CENTER);
+	bottom_pane.add(control_pane, BorderLayout.EAST);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(text_pane, BorderLayout.CENTER);
+	content_pane.add(bottom_pane, BorderLayout.SOUTH);
+
+	// Position
+	Dimension size = getSize();
+	if (Gatherer.g_man != null) {
+	    Rectangle frame_bounds = Gatherer.g_man.getBounds();
+	    setLocation(frame_bounds.x + (frame_bounds.width - size.width) / 2, frame_bounds.y + (frame_bounds.height - size.height) / 2);
+	}
+	else {
+	    Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	    setLocation((screen_size.width - size.width) / 2, (screen_size.height - size.height) / 2);
+	}
+    }
+
+    public void actionPerformed(ActionEvent event) {
+	boolean bad_value = false;
+	if(event.getSource() == ok_button) {
+	    if(affected_property != null && Configuration.self != null) {
+		String value = value_field.getText();
+		if(value.length() > 0) {
+		   if(value_field instanceof URLField) {
+		      bad_value = !((URLField)value_field).validateURL();
+		   }
+		   if(!bad_value) {
+		      // Store the value of the property
+		      Configuration.setString(affected_property, true, value_field.getText());
+		   }
+		}
+	    }
+	    if(!bad_value) {
+		result = JOptionPane.OK_OPTION;
+	    }
+	}
+	if(!bad_value) {
+	    if (Configuration.self != null) {
+		// Store the state of the show message checkbox.
+		Configuration.set(full_property, true, !show_check.isSelected());
+	    }
+	    // Done.
+	    setVisible(false);
+	}
+	else {
+	    JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("WarningDialog.Invalid_Value"), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
+	}
+    }
+
+    /** Gives notification of key events on the buttons */
+    public void keyPressed(KeyEvent e) {
+	if (e.getKeyCode() == KeyEvent.VK_ENTER) {
+	    // Enter: Click the button in focus
+	    Object source = e.getSource();
+	    if (source instanceof AbstractButton) {
+		((AbstractButton) source).doClick();
+	    }
+	}
+    }
+
+    public void keyReleased(KeyEvent e) { }
+
+    public void keyTyped(KeyEvent e) { }
+
+
+    public int display() {
+	if (Configuration.self == null || Configuration.get(full_property, false)) {
+	    setVisible(true);
+	}
+	// We are no longer showing this dialog, so result must always be true.
+	else {
+	    result = JOptionPane.OK_OPTION;
+	}
+	// Ask the parent to repaint, just to be sure
+	if(Gatherer.g_man != null) {
+	    Gatherer.g_man.repaint();
+	}
+	return result;
+    }
+
+    /** Allows you to specify whether this dialog is a warning dialog, or only a message dialog.
+     * @param message_only true if this dialog shows a message, false for a warning
+     */
+    public void setMessageOnly(boolean message_only) {
+	// If this is a message then change the checkbox
+	if(message_only) {
+	    Dictionary.setText(show_check, "WarningDialog.Dont_Show_Again_Message");
+	}
+	// And if its a warning, change them back
+	else {
+	    Dictionary.setText(show_check, "WarningDialog.Dont_Show_Again");
+	}
+    }
+
+    /** Allows you to replace the generic text field control with a JTextField subclass with further functionality. For instance you might provide a URLField to allow only valid URLs to be accepted.
+     * @param  control the JTextField subclass you want to use for the control
+     */
+    public void setValueField(JTextField control) {
+	// Remove the current control
+	value_panel.remove(value_field);
+	// Replace with the new one
+	value_field = control;
+	// Re-add
+	value_panel.add(value_field, BorderLayout.CENTER);
+    }
+
+    protected void processWindowEvent(WindowEvent event) {
+	if(event.getID() == WindowEvent.WINDOW_ACTIVATED) {
+	    if(affected_property != null) {
+		value_field.requestFocus();
+	    }
+	    else {
+		ok_button.requestFocus();
+	    }
+	}
+	else {
+	    super.processWindowEvent(event);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WriteCDImagePrompt.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WriteCDImagePrompt.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/WriteCDImagePrompt.java	(revision 21112)
@@ -0,0 +1,590 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.event.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalGreenstone;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.shell.GShellEvent;
+import org.greenstone.gatherer.shell.GShellListener;
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.CheckList;
+import org.greenstone.gatherer.util.CheckListEntry;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+/** This class provides the functionality to write a CD-ROM/DVD image of current collections from the GSDLHOME/collect/ directory to CDROM. The user chooses the collection from a list, where each entry also displays details about itself, confirms the delete of a collection by checking a checkbox then presses the ok button to actually delete the collection.
+    Copied from DeleteCollectionPrompt
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class WriteCDImagePrompt
+    extends ModalDialog 
+    implements GShellListener {
+    
+    private OKButtonListener ok_button_listener;
+    private ArrayList all_collections = null;
+    private ArrayList selected_collections = null;
+    /** The list of collections to include in exported cd-rom/dvd image */
+    private CheckList list = null;
+    /** The currently selected collection for deletion. */
+    private BasicCollectionConfiguration collection = null;
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private WriteCDImagePrompt prompt = null;
+    /** The close button, which exits the prompt without deleting anything. */
+    private JButton cancel_button = null;
+    /** The ok button which causes the selected collection to be deleted. */
+    private JButton ok_button = null;
+    /** The label above details. */
+    private JLabel details_label = null;
+    /** The label above the list. */
+    private JLabel list_label = null;
+    /** The text area used to display details about the collection selected. */
+    private JTextArea details_textarea = null;
+    /** The text area used to display instructions for the cd-rom/dvd export */
+    private JTextArea instructions_textarea;
+    /** A string array used to pass arguments to the phrase retrieval method. */
+    private JTextField title_field = null;
+    private JTextField estimated_size_field = null;
+    private JLabel title_label = null;
+    private JLabel estimated_size_label = null;
+    private String args[] = null;
+    private String cd_title = null;
+    /** whether the exporting was successful or not */
+    private boolean successful = false;
+    /** whether we are trying to export or not */
+    private boolean exporting = false;
+    /** the error message if any */
+    private StringBuffer error_message = null;
+    /** This is the size of the exported stuff on its own (no collections). */
+    private long total_exported_size = 52000000;
+    /** The size of the export prompt screen. */
+    public static final Dimension SIZE = new Dimension(500, 500);
+
+    /** Constructor.
+     * @see org.greenstone.gatherer.collection.WriteCDImagePrompt.CancelButtonListener
+     * @see org.greenstone.gatherer.collection.WriteCDImagePrompt.CollectionListListener
+     * @see org.greenstone.gatherer.collection.WriteCDImagePrompt.OKButtonListener
+     */
+    public WriteCDImagePrompt() {
+	super(Gatherer.g_man, true);
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Close", "General.Close_Tooltip");
+	details_textarea = new JTextArea();
+	details_textarea.setEditable(false);
+	Dictionary.setText(details_textarea, "DeleteCollectionPrompt.No_Collection");
+	details_label = new JLabel();
+	Dictionary.setText(details_label, "DeleteCollectionPrompt.Collection_Details");
+
+	instructions_textarea = new JTextArea();
+	instructions_textarea.setCaretPosition(0);
+	instructions_textarea.setEditable(false);
+	instructions_textarea.setLineWrap(true);
+	instructions_textarea.setRows(4);
+	instructions_textarea.setWrapStyleWord(true);
+	Dictionary.registerText(instructions_textarea, "WriteCDImagePrompt.Instructions");
+
+	all_collections = new ArrayList();
+	list = new CheckList(true);
+	list_label = new JLabel();
+	Dictionary.setText(list_label, "DeleteCollectionPrompt.Collection_List");
+	ok_button = new GLIButton();
+	ok_button.setMnemonic(KeyEvent.VK_D);
+	Dictionary.setBoth(ok_button, "WriteCDImagePrompt.Export", "WriteCDImagePrompt.Export_Tooltip");
+
+	title_field = new JTextField();
+	// Dictionary.setTooltip(title_field, "WriteCDImagePrompt.CD_Name_Tooltip");
+	title_label = new JLabel();
+	Dictionary.setText(title_label, "WriteCDImagePrompt.CD_Name");
+
+	estimated_size_field = new JTextField();
+	estimated_size_field.setEditable(false);
+	estimated_size_field.setText(Utility.formatFileLength(total_exported_size));
+	estimated_size_field.setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	estimated_size_label = new JLabel();
+	Dictionary.setText(estimated_size_label, "WriteCDImagePrompt.Size_Label");
+
+	scanForCollections();
+	list.setListData(all_collections);
+
+	prompt = this;
+	setSize(SIZE);
+	Dictionary.setText(this, "WriteCDImagePrompt.Title");
+
+	setJMenuBar(new SimpleMenuBar("0")); // need to find an appropriate help page to open at
+	cancel_button.addActionListener(new CancelButtonListener());
+	list.addListSelectionListener(new CollectionListListener());
+	list.clearSelection();
+	list.getSelectionModel().setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	ok_button_listener = new OKButtonListener();
+	ok_button.addActionListener(ok_button_listener);
+	ok_button.setEnabled(false);
+	//title.getDocument().addDocumentListener(new DocumentListener());
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	all_collections.clear();
+	all_collections = null;
+	cancel_button = null;
+	details_textarea = null;
+	details_label = null;
+	list = null;
+	ok_button = null;
+	prompt = null;
+	if (selected_collections!=null) {
+	    selected_collections.clear();
+	    selected_collections = null;
+	}
+	title_field = null;
+	title_label = null;
+    }
+
+    /** This method causes the modal prompt to be displayed. 
+     * returns true if it has exported the collections that are currently selected */
+    public boolean display() {
+	// Top pane
+	JPanel instructions_pane = new JPanel(new BorderLayout());
+	instructions_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	instructions_pane.add(new JScrollPane(instructions_textarea), BorderLayout.CENTER);
+	
+	title_label.setBorder(BorderFactory.createEmptyBorder(0,5,0,15));
+
+	JPanel title_pane = new JPanel(new BorderLayout());
+	title_pane.add(title_label, BorderLayout.WEST);
+	title_pane.add(title_field, BorderLayout.CENTER);
+	instructions_pane.add(title_pane, BorderLayout.SOUTH);
+
+	// Central pane
+	JPanel list_pane = new JPanel(new BorderLayout());
+	list_pane.add(list_label, BorderLayout.NORTH);
+	list_pane.add(new JScrollPane(list), BorderLayout.CENTER);
+	list_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 5, 0));
+
+	JPanel details_pane = new JPanel(new BorderLayout());
+	details_pane.add(details_label, BorderLayout.NORTH);
+	details_pane.add(new JScrollPane(details_textarea), BorderLayout.CENTER);
+	details_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel central_pane = new JPanel(new GridLayout(2, 1));
+	central_pane.add(list_pane);
+	central_pane.add(details_pane);
+	central_pane.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
+
+	// Lower pane
+	JPanel estimated_size_pane = new JPanel(new BorderLayout());
+	estimated_size_pane.add(estimated_size_label, BorderLayout.WEST);
+	estimated_size_pane.add(estimated_size_field, BorderLayout.CENTER);
+	estimated_size_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 5, 0));
+
+	JPanel button_pane = new JPanel(new GridLayout(1, 2));
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+	button_pane.setBorder(BorderFactory.createEmptyBorder(0, 0, 0, 0));
+
+	JPanel lower_pane = new JPanel(new BorderLayout());
+	lower_pane.add(estimated_size_pane, BorderLayout.NORTH);
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+	lower_pane.setBorder(BorderFactory.createEmptyBorder(0, 5, 5, 5));
+
+	// Final.
+	JPanel content_pane = (JPanel)this.getContentPane();
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(instructions_pane, BorderLayout.NORTH);
+	content_pane.add(central_pane, BorderLayout.CENTER);
+	content_pane.add(lower_pane, BorderLayout.SOUTH);
+
+	// Center and display.
+	Dimension screen_size = Configuration.screen_size;
+	this.setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	this.setVisible(true); // blocks until the dialog is killed
+	return true;
+	
+    }
+
+    
+    /** This method calls the builcol.pl scripts via a GShell so as to not lock up the processor.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.collection.Collection
+     * @see org.greenstone.gatherer.gui.BuildOptions
+     * @see org.greenstone.gatherer.shell.GShell
+     * @see org.greenstone.gatherer.shell.GShellListener
+     * @see org.greenstone.gatherer.shell.GShellProgressMonitor
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    public void writeCDImageCollections()
+    {
+	DebugStream.println("WriteCDImagePrompt.writeCDImageCollections()");
+
+	int num_collections = selected_collections.size();
+	if (num_collections == 0) return;
+	cd_title = title_field.getText();
+	cd_title = cd_title.trim();
+	cd_title = cd_title.replaceAll("\"","");
+
+	// Generate the exportcol.pl command
+	ArrayList command_parts_list = new ArrayList();
+	if (Utility.isWindows() && (!Gatherer.isGsdlRemote)) {
+	    command_parts_list.add(Configuration.perl_path);
+	    command_parts_list.add("-S");
+	}
+	command_parts_list.add(LocalGreenstone.getBinScriptDirectoryPath() + "exportcol.pl");
+
+	if (Gatherer.isGsdlRemote) {
+	    command_parts_list.add("-nonetscape");
+	}
+
+	command_parts_list.add("-gli");
+	command_parts_list.add("-language");
+	command_parts_list.add(Configuration.getLanguage());
+
+	if (cd_title.equals("")) {
+	    command_parts_list.add("-cdname");
+	    command_parts_list.add("Greenstone Collections");
+
+	    command_parts_list.add("-cddir");
+	    command_parts_list.add("exported_collections");
+	}
+	else {
+	    command_parts_list.add("-cdname");
+	    command_parts_list.add(cd_title);
+
+	    String cd_dir = "exported_"+cd_title.replaceAll("\\s","");
+	    command_parts_list.add("-cddir");
+	    command_parts_list.add(cd_dir);
+	}
+
+	for (int i = 0; i < num_collections; i++) {
+	    command_parts_list.add(((BasicCollectionConfiguration) selected_collections.get(i)).getShortName());
+	}
+
+	DebugStream.print("export command = ");
+	for (int i = 0; i < command_parts_list.size(); i++) {
+	    DebugStream.print(command_parts_list.get(i) + " ");
+	}
+	DebugStream.println("");
+
+	// Run the exportcol.pl command
+	String[] command_parts = (String[]) command_parts_list.toArray(new String[0]);
+	GShell process = new GShell(command_parts, GShell.CDIMAGE, 3, this, null, GShell.GSHELL_CDIMAGE);
+	process.start();
+	//process.run();
+	DebugStream.println("WriteCDImagePrompt.writeCDImageCollections().return");
+	
+    }
+
+    /** Shows an export complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully deleted.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    public void resultPrompt(boolean success, String extra) {
+	args = new String[2];
+	StringBuffer coll_names = new StringBuffer();
+	for (int i=0; i<selected_collections.size();i++) {
+	    if (i>0) {
+		coll_names.append(", ");
+	    }
+	    BasicCollectionConfiguration complete_collection = (BasicCollectionConfiguration)selected_collections.get(i);
+	    coll_names.append(complete_collection.getName() + StaticStrings.SPACE_CHARACTER + StaticStrings.OPEN_PARENTHESIS_CHARACTER + complete_collection.getShortName() + StaticStrings.CLOSE_PARENTHESIS_CHARACTER);
+	    complete_collection = null;
+	}
+	
+	args[0] = coll_names.toString();
+	args[1] = LocalGreenstone.getTmpDirectoryPath();
+	if(cd_title.equals("")) {
+	    args[1] += "exported_collections";
+	} else {
+	    args[1] += "exported_"+cd_title.replaceAll("\\s","");
+	}
+	String title;
+	String label;
+	if (success) {
+	    title = Dictionary.get("WriteCDImagePrompt.Successful_Title");
+	    label = Dictionary.get("WriteCDImagePrompt.Successful_Export", args);
+	} else {
+	    title = Dictionary.get("WriteCDImagePrompt.Failed_Title");
+	    label = Dictionary.get("WriteCDImagePrompt.Failed_Export", args);
+	}
+	SimpleResultDialog result_dialog = new SimpleResultDialog(title, label, extra);
+	result_dialog.setVisible(true); // Blocks
+	result_dialog.dispose();
+	result_dialog = null;
+    }
+
+    /** Method to scan the collect directory retrieving and reloading each collection it finds, while building the list of known collections.
+     * @see org.greenstone.gatherer.Configuration
+     * @see org.greenstone.gatherer.Gatherer
+     * @see org.greenstone.gatherer.util.ArrayTools
+     * @see org.greenstone.gatherer.util.Utility
+     */
+    private void scanForCollections() {
+	// Start at the collect dir.
+	File collect_directory = new File(Gatherer.getCollectDirectoryPath());
+	if (collect_directory.exists()) {
+	    // Now for each child directory see if it contains a .col file and
+	    // if so try to load it..
+	    File collections[] = collect_directory.listFiles();
+	    ArrayTools.sort(collections);
+	    for(int i = 0; collections != null && i < collections.length; i++) {
+		if(collections[i].isDirectory() && !collections[i].getName().equals(StaticStrings.MODEL_COLLECTION_NAME)) {
+		    File config_file = new File(collections[i], Utility.CONFIG_FILE);
+		    if (config_file.exists()) {
+			BasicCollectionConfiguration config = new BasicCollectionConfiguration(config_file);
+			all_collections.add(config);		        
+			config = null;
+		    }
+		}
+	    }
+	}
+	// Otherwise the collect directory doesn't actually exist, so there ain't much we can do.
+    }
+
+
+    /** All implementation of GShellListener must include this method so the listener can be informed of messages from the GShell.
+     * @param event A <strong>GShellEvent</strong> that contains, amoung other things, the message.
+     */
+    public synchronized void message(GShellEvent event) {
+	// Ignore the messages from RecPlug with 'show_progress' set (used for progress bars)
+	String message = event.getMessage();
+	if (message.startsWith("exportcol.pl>")) {
+	    message = message.substring(13);
+	    //DebugStream.println("message = "+event.getMessage());
+	    error_message.append(message);
+	    error_message.append("\n");
+	}
+    }
+
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell begins its task. Implementation side-effect, not actually used.
+     * @param event A <strong>GShellEvent</strong> that contains details of the initial state of the <strong>GShell</strong> before task comencement.
+     */
+    public synchronized void processBegun(GShellEvent event) {
+	// We don't care. 
+    }
+    /** All implementation of GShellListener must include this method so the listener can be informed when a GShell completes its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the final state of the <strong>GShell</strong> after task completion.
+     */
+    public synchronized void processComplete(GShellEvent event) {
+	successful = false;
+	if(event.getStatus() == GShell.OK) {
+	    if(event.getType() == GShell.CDIMAGE) {
+		successful = true;
+	    }
+	}
+	ok_button_listener.processComplete();
+    }
+
+    /** A button listener implementation, which listens for actions on the close button and disposes of the dialog when detected. */
+    private class CancelButtonListener 
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    prompt.dispose();
+	}
+    }
+
+
+    /** This private class listens for selection events in from the list and then displays the appropriate details for that collection.
+     */
+    private class CollectionListListener 
+	implements ListSelectionListener
+    {
+	/** Any implementation of ListSelectionListener must include this method so we can be informed when the list selection changes.
+	 * @param  event a <strong>ListSelectionEvent</strong> containing all the relevant information garnered from the event itself
+	 */
+	public void valueChanged(ListSelectionEvent event)
+	{
+	    // Wait for things to settle down a bit
+	    if (event.getValueIsAdjusting()) {
+		return;
+	    }
+
+	    // Can only export when something is ticked
+	    ok_button.setEnabled(!list.isNothingTicked());
+
+	    if (list.isSelectionEmpty()) {
+		// This only happens when the dialog is first entered
+		Dictionary.setText(details_textarea, "DeleteCollectionPrompt.No_Collection");
+		return;
+	    }
+
+	    collection = (BasicCollectionConfiguration) ((CheckListEntry) list.getSelectedValue()).getObject();
+	    args = new String[3];
+	    args[0] = collection.getCreator();
+	    args[1] = collection.getMaintainer();
+	    args[2] = collection.getDescription();
+	    Dictionary.setText(details_textarea, "DeleteCollectionPrompt.Details", args);
+	    details_textarea.setCaretPosition(0);
+
+	    // Find the size of the "etc", "images" and "index" directories of the collection
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection.getShortName());
+	    File etc_directory = new File(collection_directory_path + "etc");
+	    File images_directory = new File(collection_directory_path + "images");
+	    File index_directory = new File(collection_directory_path + "index");
+	    long collection_size_built = getFileSize(etc_directory) + getFileSize(images_directory) + getFileSize(index_directory);
+
+	    // Add/subtract it from the total, depending on whether the collection has just been ticked/unticked
+	    if (((CheckListEntry) list.getSelectedValue()).isSelected()) {
+		total_exported_size += collection_size_built;
+	    }
+	    else {
+		total_exported_size -= collection_size_built;
+	    }
+
+	    // Update the size field
+	    estimated_size_field.setText(Utility.formatFileLength(total_exported_size));
+	}
+
+
+	private long getFileSize(File file)
+	{
+	    long file_size = 0;
+
+	    // Directory case
+	    if (file.isDirectory()) {
+		File files[] = file.listFiles();
+		for (int i = 0; i < files.length; i++) {
+		    file_size += getFileSize(files[i]);
+		}
+	    }
+	    // File case
+	    else {
+		file_size = file.length();
+	    }
+
+	    return file_size;
+	}
+    }
+
+
+    /** The OK button listener implementation. */
+    private class OKButtonListener 
+	implements ActionListener {
+	private Component glass_pane;
+	private MouseListener mouse_blocker_listener;
+	private ProgressDialog progress_dialog;
+
+	/** Any implementation of ActionListener must include this method so we can be informed when the button is actioned.
+	 * @param event An <strong>ActionEvent</strong> containing all the relevant information garnered from the event itself.
+	 * @see org.greenstone.gatherer.Configuration
+	 * @see org.greenstone.gatherer.Gatherer
+	 * @see org.greenstone.gatherer.util.Utility
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    ///ystem.err.println("OK Clicked");
+	    // Make sure there are some colls specified
+	    selected_collections = list.getTicked();
+	    error_message = new StringBuffer();
+	    
+	    // Set the cursor to hourglass
+	    glass_pane = getGlassPane();
+	    mouse_blocker_listener = new MouseAdapter() {};
+	    glass_pane.addMouseListener(mouse_blocker_listener);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	    glass_pane.setVisible(true);
+
+	    // Export the selected collection.
+	    ///ystem.err.println("Write CD-ROM/DVD image for named collections");
+	    writeCDImageCollections();
+
+	    // Show progress dialog
+	    ///ystem.err.println("Showing progress dialog");
+	    progress_dialog = new ProgressDialog();
+	    progress_dialog.setVisible(true);
+	}
+
+	public void processComplete() {
+	    ///ystem.err.println("Process complete");
+	    // Dispose of progress dialog
+	    progress_dialog.setVisible(false);
+	    progress_dialog.dispose();
+	    progress_dialog = null;
+
+	    // unset the cursor
+	    glass_pane.setVisible(false);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    glass_pane.removeMouseListener(mouse_blocker_listener);
+	    glass_pane = null;
+	    mouse_blocker_listener= null;
+	    
+	    if (successful) {
+		resultPrompt(true, error_message.toString());
+	    } else {
+		resultPrompt(false, error_message.toString());
+	    }
+	    error_message = null;
+	}
+
+	private class ProgressDialog 
+	    extends ModalDialog {
+	    
+	    private Dimension size = new Dimension(400,65);
+	    
+	    public ProgressDialog() {
+		super(Gatherer.g_man, Dictionary.get("WriteCDImagePrompt.Title"), true);
+		setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+		setSize(size);
+		JPanel content_pane = (JPanel) getContentPane();
+		JLabel progress_label = new JLabel();
+		Dictionary.setText(progress_label, "WriteCDImagePrompt.Progress_Label");
+		JProgressBar progress_bar = new JProgressBar();
+		progress_bar.setIndeterminate(true);
+		content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+		content_pane.setLayout(new BorderLayout());
+		content_pane.add(progress_label, BorderLayout.NORTH);
+		content_pane.add(progress_bar, BorderLayout.CENTER);
+		// Position
+		Rectangle frame_bounds = Gatherer.g_man.getBounds();
+		setLocation(frame_bounds.x + (frame_bounds.width - size.width) / 2, frame_bounds.y + (frame_bounds.height - size.height) / 2);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Autofilter.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Autofilter.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Autofilter.java	(revision 21112)
@@ -0,0 +1,379 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+
+import java.util.ArrayList;
+import org.greenstone.gatherer.util.StaticStrings;
+
+/** An Autofilter object stores the filters set on a single column, and provides a method for determining if a certain value passes the filter.
+ * @author John Thompson
+ * @version 2.3
+ */
+public class Autofilter {
+
+    static public void main(String[] args) {
+	if(args.length != 2) {
+	    System.err.println("Usage: java Autofilter <string> <pattern>");
+	}
+	else {
+	    switch(compareToPattern(args[0], args[1])) {
+	    case LESS_THAN:
+		System.err.print(args[0] + " is less than");
+		break;
+	    case GREATER_THAN:
+		System.err.print(args[0] + " is greater than");
+		break;
+	    case EQUAL:
+		System.err.print(args[0] + " is equal to");
+		break;
+	    default:
+		System.err.print(args[0] + " is something to");
+	    }
+	    System.err.println(" " + args[1]);
+	}
+    }
+
+    /** A String comparison which also recognises the WildCard character. The basic idea is that '*' matches zero or more of any characters, and if we limited patterns to be like "a*" then as soon as we hit the star we could return a match. The problem is that we want patterns like "b*y" which would match "boy" but not "bee" nor "bus" (but should return less than or greater than for each of these respectively). Thus our testing has to become non-deterministic so the matches are conducted like so: 
+     * Given p = [ b * y ]
+     * Test  s = [ b e e ]  Result: p0=s0,{(p2>s1=>ERROR),(p1=s1,{(p1>s2=>ERROR),(p2>s2=>LESS_THAN)})} => LESS_THAN
+     * Test  s = [ b o y ]  Result: p0=s0,{(p2>s1=>ERROR),(p1=s1,{(p1>s2=>ERROR),(p2=s2=>EQUAL)})} => EQUAL
+     * Test  s = [ b u s ]  Result: p0=s0,{(p2>s1=>ERROR),(p1=s1,{(p1>s2=>ERROR),(p2<s2=>GREATER_THAN)})} => GREATER_THAN
+     * where the two () phrases within a {} represent two different 'threads' of processing. Errors are generated when one string runs out of characters before the other. Only non-error results are propogated. I've decided to implement this non-deterministic matching as a recursive function call.
+     * If you analyse it you can see there are three possible actions when a '*' is detected.
+     * 1. * is an empty string, ignore it in pattern and try to compare the next character in p with the current character in s. This case only occurs if there are more characters in p (otherwise apple would be greater than a*)
+     * 2. * matches exactly the current character in s, try to match the next character in p with the next character in s
+     * 3. * matches some string of characters including the current character in s, try to match '*' against the next character in s
+     * @param  s the String being matched
+     * @param  p the Pattern to match it against
+     * @return -1 if s is less than p, 0 if they are equal, 1 if s is greater than p
+     */
+    static private int compareToPattern(String s, String p) {
+	if(p.indexOf(StaticStrings.STAR_CHAR) == -1) {
+	    return compareToPattern(s, p, 0, 0, false);
+	}
+	else {
+	    return compareToPattern(s, p, 0, 0, true);
+	}
+    }
+
+    static final private int LESS_THAN = -1;
+    static final private int EQUAL = 0;
+    static final private int GREATER_THAN = 1;
+    static final private int ERROR = 42;
+
+    static private int compareToPattern(String s, String p, int s_index, int p_index, boolean wildcard_matching) {
+	// Lets do the simple cases first.
+	// Both out of characters at the same time
+	if(s_index >= s.length() && p_index >= p.length()) {
+	    ///ystem.err.println("Both out of characters. Equal.");
+	    return EQUAL;
+	}
+	// s out of characters first
+	if(s_index >= s.length()) {
+	    // Not allowed to run out if matching wildcard
+	    if(wildcard_matching) {
+		return ERROR;
+	    }
+	    // Remember that wildcard matches the empty string too, so as long as there are only '*' characters left in the pattern we can say they are equal as well.
+	    else {
+		while(p_index < p.length()) {
+		    if(p.charAt(p_index) == StaticStrings.STAR_CHAR) {
+			p_index++;
+		    }
+		    else {
+			///ystem.err.println("S out of characters. Less Than.");
+			return LESS_THAN;
+		    }
+		}
+		// We've run out of pattern and it only contained '*' so we're still equal
+		///ystem.err.println("Both out of characters. Equal.");
+		return EQUAL;
+	    }
+	}
+	// p out of characters first
+	if(p_index >= p.length()) {
+	    // Not allowed to run out if matching wildcard
+	    if(wildcard_matching) {
+		return ERROR;
+	    }
+	    // so it is greater than
+	    else {
+		///ystem.err.println("P out of characters. Greater Than.");
+		return GREATER_THAN;
+	    }
+	}
+	char s_char = s.charAt(s_index);
+	char p_char = p.charAt(p_index);
+	///ystem.err.println("Comparing " + s_char + " against pattern character " + p_char);
+	// Equivelent characters
+	// Now the tricky-dicky bit - WildCard matching.
+	if(p_char == StaticStrings.STAR_CHAR) {
+	    int result;
+	    // Case 1
+	    ///ystem.err.println("WildCard Case 1");
+	    if(p_index + 1 < p.length()) {
+		if((result = compareToPattern(s, p, s_index, p_index + 1, wildcard_matching)) != ERROR) {
+		    return result;
+		}
+	    }
+	    // Case 2
+	    ///ystem.err.println("WildCard Case 2");
+	    if((result = compareToPattern(s, p, s_index + 1, p_index + 1, wildcard_matching)) != ERROR) {
+		return result;
+	    } 
+	    // Case 3
+	    ///ystem.err.println("WildCard Case 3");
+	    if((result = compareToPattern(s, p, s_index + 1, p_index, wildcard_matching)) != ERROR) {
+		return result;
+	    }
+	}
+	if(s_char == p_char) {
+	    return compareToPattern(s, p, s_index + 1, p_index + 1, wildcard_matching);
+	}
+	// A preceeding character
+	if(s_char < p_char) {
+	    ///ystem.err.println("s char is less than p char");
+	    if(wildcard_matching) {
+		///ystem.err.println("Because we are wildcard matching we still have to match the rest of s incase of errors.");
+		if(compareToPattern(s, p, s_index + 1, p_index + 1, wildcard_matching) != ERROR) {
+		    ///ystem.err.println("No error. Less than.");
+		    return LESS_THAN;
+		}
+		else {
+		    ///ystem.err.println("Error detected.");
+		    return ERROR;
+		}
+	    }
+	    else {
+		///ystem.err.println("Less than.");
+		return LESS_THAN;
+	    }
+	}
+	// A succeeding character
+	if(s_char > p_char) {
+	    ///ystem.err.println("s char is greater than p char");
+	    if(wildcard_matching) {
+		///ystem.err.println("Because we are wildcard matching we still have to match the rest of s incase of errors.");
+		if(compareToPattern(s, p, s_index + 1, p_index + 1, wildcard_matching) != ERROR) {
+		    ///ystem.err.println("No error. Greater than.");
+		    return GREATER_THAN;
+		}
+		else {
+		    ///ystem.err.println("Error detected.");
+		    return ERROR;
+		}
+	    }
+	    else {
+		///ystem.err.println("Greater than.");
+		return GREATER_THAN;
+	    }
+	}
+	// Oh-No. Well, we'll just assume that string is less than pattern
+	return LESS_THAN;
+    }
+
+    /** <i>true</i> if the filter should be applied, <i>false</i> to indicate the filter is turned off. */
+    public boolean active;
+    /** <i>true</i> if the matching for the first expression should be case sensitive, <i>false</i> otherwise. */
+    public boolean casesense_one;
+    /** <i>true</i> if the matching for the second expression should be case sensitive, <i>false</i> otherwise. */
+    public boolean casesense_two;
+    /** Used to determine the operation intended when applying two filters, and set using the values of the OPERATION_TYPE enumeration. */
+    public boolean operation;
+    /** Used to determine how the column this filter is applied to should be sorted, and set using the values of the SORT_TYPE enumeration. */
+    public boolean sort;
+    /** The method to be used for the first filter expression, set from the values of the METHOD_LIST enumeration. */
+    public int method_one;
+    /** The method to be used for the second filter expression, set from the values of the METHOD_LIST enumeration. */
+    public int method_two;
+    /** The value to be matched against for the first expression. */
+    public String value_one;
+    /** The value to be matched against for the second expression. */
+    public String value_two;
+    /** An element of the SORT_TYPE enumeration, indicates lowest to highest value column ordering. */
+    public static final boolean ASCENDING = true;
+    /** An element of the OPERATION_TYPE enumeration, indicates that both filter expressions must be met (conjunction). */
+    public static final boolean AND = true;
+    /** An element of the SORT_TYPE enumeration, indicates highest to lowest value column ordering. */
+    public static final boolean DESCENDING = false;
+    /** An element of the OPERATION_TYPE enumeration, indicates that either (or both) filter expressions must be met (disjunction). */
+    public static final boolean OR = false;
+    /** An enumeration of symbolic names of various matching methods. */
+    public static final String METHOD_LIST[] = { "Autofilter.eqeq", "Autofilter.!eq", "Autofilter.<", "Autofilter.<eq", "Autofilter.>", "Autofilter.>eq", "Autofilter.^", "Autofilter.!^", "Autofilter.$", "Autofilter.!$", "Autofilter.?", "Autofilter.!?" };
+
+    /** Default Constructor. */
+    public Autofilter() {
+	operation = OR;
+	sort = ASCENDING;
+    }
+    /** Determine if this filter is currently active.
+     * @return <i>true</i> if it is active, <i>false</i> otherwise.
+     */
+    public boolean active() {
+	return active;
+    }
+    /** Determine if this list of values (for a certain cell) passes the filter.
+     * @param values An <strong>ArrayList</strong> of values sourced from a single cell in the associated column.
+     * @return <i>true</i> if the values match and should be displayed, <i>false</i> otherwise.
+     */
+    public boolean filter(ArrayList values) {
+	boolean result = false;
+	if(value_one != null) {
+	    result = filter(values, method_one, value_one, casesense_one);
+	    if(result) {
+		if(operation == AND && value_two != null) {
+		    result = filter(values, method_two, value_two, casesense_two);
+		}
+	    }
+	    else if(operation == OR && value_two != null) {
+		result = filter(values, method_two, value_two, casesense_two);
+	    }
+	}
+	return result;
+    }
+    /** Set the current activity state of this filter.
+     * @param active The new state of this filter, <i>true</i> to activate, <i>false</i> otherwise.
+     */
+    public void setActive(boolean active) {
+	this.active = active;
+    }
+    /** Set one of the filter expressions using the given information.
+     * @param number The number of the filter you wish to set as an <i>int</i>. Either 1 or 2.
+     * @param method An <i>int</i> indicating the method to be used when matching.
+     * @param value The <strong>String</strong> to be matched against.
+     * @param casesense <i>true</i> if this expression should be case sensitive, <i>false</i> otherwise.
+     */
+    public void setFilter(int number, int method, String value, boolean casesense) {
+	if(!casesense && value != null) {
+	    value = value.toLowerCase();
+	}
+	if(number == 1) {
+	    casesense_one = casesense;
+	    method_one = method;
+	    value_one = value;
+	}
+	else {
+	    casesense_two = casesense;
+	    method_two = method;
+	    value_two = value;
+	}
+    }
+    /** Set the operation to be used to join the two filters (if a second filter is set).
+     * @param operation <i>true</i> for conjunct filters, <i>false</i> for disjunct.
+     */ 
+    public void setOperation(boolean operation) {
+	this.operation = operation;
+    }
+    /** Set the sort order of this column.
+     * @param sort <i>true</i> for ascending sort, <i>false</i> for descending.
+     */
+    public void setSort(boolean sort) {
+	this.sort = sort;
+    }
+    /** Decide whether a row should be displayed or filtered. The result depends on the selector set with setFilterType.
+     * @param values An <strong>ArrayList</strong> of values to be checked against the filter.
+     * @param method The method of matching to be used, as an <i>int</i>.
+     * @param target The <strong>String</Strong> to match against.
+     * @param casesense <i>true</i> if the match is to be case sensitive, <i>false</i> otherwise.
+     * @return <i>true</i> to display the row, <i>false</i> to hide it.
+     */
+    public boolean filter(ArrayList values, int method, String target, boolean casesense) {
+	boolean result = false;
+	// There are several special cases when the filter always returns turn, such as when the target is the wildcard character.
+	if(target == null || target.length() == 0 || target.equals("*")) {
+	    result = true;
+	}
+	else {
+	    // For each value in the list...
+	    for(int i = 0; i < values.size(); i++) {
+		boolean pass;
+		String source;
+		// Account for case sensitivity.
+		if(casesense) {
+		    source = values.get(i).toString();
+		}
+		else {
+		    source = values.get(i).toString().toLowerCase();
+		}
+		///ystem.err.println("Testing " + source + " against pattern " + target);
+		// Perform the match, based on the selected method.
+		switch(method) {
+		case 1: // !EQ
+		    pass = (compareToPattern(source, target) != 0);
+		    break;
+		case 2: // <
+		    pass = (compareToPattern(source, target) < 0);
+		    break;
+		case 3: // <eq
+		    pass = (compareToPattern(source, target) <= 0);
+		    break;
+		case 4: // >
+		    pass = (compareToPattern(source, target) > 0);
+		    break;
+		case 5: // >eq
+		    pass = (compareToPattern(source, target) >= 0);
+		    break;
+		case 6: // ^
+		    pass = source.startsWith(target);
+		    break;
+		case 7: // !^
+		    pass = !(source.startsWith(target));
+		    break;
+		case 8: // $
+		    pass = source.endsWith(target);
+		    break;
+		case 9: // !$
+		    pass = !(source.endsWith(target));
+		    break;
+		case 10: // ?
+		    pass = (source.indexOf(target) != -1);
+		    break;
+		case 11: // !?
+		    pass = (source.indexOf(target) == -1);
+		    break;
+		default: // EQEQ
+		    pass = (compareToPattern(source, target) == 0);
+		    break;
+		}
+		result = result || pass;
+	    }
+	}
+	return result;		 
+    }
+    /** Produce a textual representation of this autofilter.
+     * @return A <strong>String</strong> displaying details of this autofilter.
+     */
+    public String toString() {
+	String result = "One: " + method_one + " - " + value_one + " - " + casesense_one;
+	if(value_two != null) {
+	    result = result + "\n" + "Operation: " + (operation?"AND":"OR") + "\n";
+	    result = result + "Two: " + method_two + " - " + value_two + " - " + casesense_two;
+	}
+	return result;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/AutofilterDialog.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/AutofilterDialog.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/AutofilterDialog.java	(revision 21112)
@@ -0,0 +1,422 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.SimpleMenuBar;
+import org.greenstone.gatherer.gui.ModalDialog;
+
+/** The autofilter concept comes from Microsoft Excel spreadsheets that use autofilters to filter to the sheet. When you click on the heading of a column, a new prompt allows you to specify what filter should be acting apon the selected column. Any new filter is conjoined with any previous filters to provide a sheet containing only rows that match all current filters. Each column must also provide an indicator for determining if a filter is set (in this case a special icon) and and a method for removing a filter (use the clear filter button within the autofilter dialog). Having recently discovered that most JVMs aren't very good at recoving memory used by dialog and frame windows, special care must be made to deallocate all references properly, as a user may open several dozen autofilter prompts over the lifetime of a session.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public final class AutofilterDialog
+    extends ModalDialog {
+    /** The filter being edited. */
+    private Autofilter filter;
+    /** A reference to ourselves so our inner classes can reference us. */
+    private AutofilterDialog self;
+
+    private boolean cancelled;
+    /** The value returned from the filter dialog prompt. Used to determine if a prompt was set or unset, and what subsequent action to take. */
+    private byte return_value = 0;
+    /** The button used to cancel the prompt. */
+    private JButton cancel_button = null;
+    private JButton remove_button = null;
+    private JButton set_button = null;
+    /** Used to specify that the given filters should be applied conjunctly. */
+    private JRadioButton and_radiobutton = null;
+    /** The check box used to specify whether the first filter is case sensitive. */
+    private JCheckBox first_case = null;
+    /** Used to disable the second filter. */
+    private JRadioButton none_radiobutton = null;
+    /** Used to specify that the given filters should be applied disjunctly. */
+    private JRadioButton or_radiobutton = null;
+    /** The check box used to specify whether the second filter is case sensitive. */
+    private JCheckBox second_case = null;
+    /** Used to specify the order of the resulting set: Ascending or Descending. */
+    private JRadioButton ascending_radiobutton = null;
+    private JRadioButton descending_radiobutton = null;
+    /** The method used to match the first filter: Contains, Doesn't contain etc. */
+    private JComboBox first_method = null;
+    /** The value to be matched for the first filter. */
+    private JComboBox first_value = null;
+    /** The method used to match the first filter. Options as for the first method. */
+    private JComboBox second_method = null;
+    /** The value to be matched for the second filter. */
+    private JComboBox second_value = null;
+    /** Used for the most basic filter where an Equals, Case sensitive method is automatically used. */
+    private JComboBox value = null;
+    /** The label which displays the name of the currently selected column (the column that will be associated with the autofilter this dialog produces). */
+    private JLabel name;
+    /** The autofilter prompt contains two different panes, one for basic functionality and one for advanced. This containiner component is used to allow access to each via a 'tabbed' interface. */
+    private JTabbedPane control = null;
+    /** A reference back to the MetaAudit dialog that spawned this prompt. Used to make sure that any open dialog window is always in front of the audit pane. */
+    private MetaAuditFrame dialog;
+    /** The default size for the autofilter control. */
+    static final private Dimension SIZE = new Dimension(640, 245);
+
+    /** Constructor.
+     * @param dialog A reference to the <strong>MetaAuditFrame</strong> that spawned this dialog.
+     * @see org.greenstone.gatherer.gui.metaaudit.Autofilter
+     * @see org.greenstone.gatherer.gui.metaaudit.AutofilterDialog.ButtonListener
+     * @see org.greenstone.gatherer.gui.metaaudit.AutofilterDialog.CheckListener
+     */
+    public AutofilterDialog(MetaAuditFrame dialog) {
+	super(Gatherer.g_man);
+	this.dialog = dialog;
+	this.self = this;
+	setModal(true);
+	setJMenuBar(new SimpleMenuBar("reviewingmetadata")); 
+	setSize(SIZE);
+	Dictionary.setText(this, "Autofilter.Title");
+
+	// Creation
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel name_pane = new JPanel();
+	JLabel name_label = new JLabel();
+	Dictionary.setText(name_label, "Autofilter.Name");
+	JTextField name_template = new JTextField();
+	name = new JLabel();
+	name.setBorder(name_template.getBorder());
+	control = new JTabbedPane();
+	JPanel value_pane = new JPanel();
+	JPanel inner_value_pane = new JPanel();
+	JLabel value_label = new JLabel();
+	Dictionary.setText(value_label, "Autofilter.eqeq"); 
+	value = new JComboBox();
+	value.setEditable(false);
+	JPanel custom_pane = new JPanel();
+
+	JPanel first_pane = new JPanel();
+	first_method = new JComboBox();
+	first_value = new JComboBox();
+	first_value.setEditable(true);
+	first_value.addItem("");
+	first_value.setSelectedItem("");
+	first_case = new JCheckBox();
+	Dictionary.setText(first_case, "Autofilter.Case_Sensitive");
+	JPanel operator_pane = new JPanel();
+	JLabel operator_label = new JLabel();
+	Dictionary.setText(operator_label, "Autofilter.Operator");
+	ButtonGroup operator_group = new ButtonGroup();
+	and_radiobutton = new JRadioButton();
+	and_radiobutton.setOpaque(false);
+	Dictionary.setText(and_radiobutton, "Autofilter.AND");
+	none_radiobutton = new JRadioButton();
+	none_radiobutton.setOpaque(false);
+	none_radiobutton.setSelected(true);
+	Dictionary.setText(none_radiobutton, "Autofilter.None");
+	or_radiobutton = new JRadioButton();
+	or_radiobutton.setOpaque(false);
+	Dictionary.setText(or_radiobutton, "Autofilter.OR");
+	operator_group.add(none_radiobutton);
+	operator_group.add(and_radiobutton);
+	operator_group.add(or_radiobutton);
+
+	JPanel second_pane = new JPanel();
+	second_method = new JComboBox();
+	second_method.setEnabled(false);
+	second_value = new JComboBox();
+	second_value.setEditable(true);
+	second_value.setEnabled(false);
+	second_value.addItem("");
+	second_value.setSelectedItem("");
+	second_case = new JCheckBox();
+	Dictionary.setText(second_case, "Autofilter.Case_Sensitive"); 
+	JPanel lower_pane = new JPanel();
+	JPanel order_pane = new JPanel();
+	Dictionary.setTooltip(order_pane, "Autofilter.Order_Tooltip");
+	JLabel order_label = new JLabel();
+	Dictionary.setText(order_label, "Autofilter.Order");
+
+	ButtonGroup order_group = new ButtonGroup();
+	ascending_radiobutton = new JRadioButton();
+	ascending_radiobutton.setOpaque(false);
+	ascending_radiobutton.setSelected(true);
+	Dictionary.setText(ascending_radiobutton, "Autofilter.Ascending");
+	descending_radiobutton = new JRadioButton();
+	descending_radiobutton.setOpaque(false);
+	Dictionary.setText(descending_radiobutton, "Autofilter.Descending");
+	order_group.add(ascending_radiobutton);
+	order_group.add(descending_radiobutton);
+
+	// Assign values to method comboboxes.
+	for(int i = 0; i < Autofilter.METHOD_LIST.length; i++) {
+	    first_method.addItem(Dictionary.get(Autofilter.METHOD_LIST[i]));
+	    second_method.addItem(Dictionary.get(Autofilter.METHOD_LIST[i]));
+	}
+	JPanel button_pane = new JPanel();
+	cancel_button = new GLIButton();
+	cancel_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.setBoth(cancel_button, "General.Cancel", "General.Pure_Cancel_Tooltip");
+	remove_button = new GLIButton();
+	remove_button.setMnemonic(KeyEvent.VK_R);
+	Dictionary.setBoth(remove_button, "Autofilter.Remove", "Autofilter.Remove_Tooltip");
+	set_button = new GLIButton();
+	set_button.setMnemonic(KeyEvent.VK_S);
+	Dictionary.setBoth(set_button, "Autofilter.Set", "Autofilter.Set_Tooltip");
+
+	// Connection
+	and_radiobutton.addActionListener(new CheckListener(true));
+	none_radiobutton.addActionListener(new CheckListener(false));
+	or_radiobutton.addActionListener(new CheckListener(true));
+	cancel_button.addActionListener(new ButtonListener()); // It returns any currently set filter
+	remove_button.addActionListener(new ButtonListener(false));
+	set_button.addActionListener(new ButtonListener(true));
+
+	// Layout		  
+	name_label.setBorder(BorderFactory.createEmptyBorder(0,0,0,5));
+
+	name_pane.setBorder(BorderFactory.createEmptyBorder(0,0,5,0));
+	name_pane.setLayout(new BorderLayout());
+	name_pane.add(name_label, BorderLayout.WEST);
+	name_pane.add(name, BorderLayout.CENTER);
+
+	value_label.setBorder(BorderFactory.createEmptyBorder(0,0,0,5));
+
+	inner_value_pane.setLayout(new BorderLayout());
+	inner_value_pane.add(value_label, BorderLayout.WEST);
+	inner_value_pane.add(value, BorderLayout.CENTER);
+
+	value_pane.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+	value_pane.setLayout(new BorderLayout());
+	value_pane.add(inner_value_pane, BorderLayout.NORTH);
+
+	first_pane.setLayout(new BorderLayout());
+	first_pane.add(first_method, BorderLayout.WEST);
+	first_pane.add(first_value, BorderLayout.CENTER);
+	first_pane.add(first_case, BorderLayout.EAST);
+
+	operator_pane.setLayout(new GridLayout(1,4));
+	operator_pane.add(operator_label);
+	operator_pane.add(none_radiobutton);
+	operator_pane.add(and_radiobutton);
+	operator_pane.add(or_radiobutton);
+
+	second_pane.setLayout(new BorderLayout());
+	second_pane.add(second_method, BorderLayout.WEST);
+	second_pane.add(second_value, BorderLayout.CENTER);
+	second_pane.add(second_case, BorderLayout.EAST);
+
+	order_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	order_pane.setLayout(new GridLayout(1,2));
+	order_pane.add(order_label);
+	order_pane.add(ascending_radiobutton);
+	order_pane.add(descending_radiobutton);
+
+	custom_pane.setBorder(BorderFactory.createEmptyBorder(2,2,2,2));
+	custom_pane.setLayout(new GridLayout(3,1));
+	custom_pane.add(first_pane);
+	custom_pane.add(operator_pane);
+	custom_pane.add(second_pane);
+
+	control.add(Dictionary.get("Autofilter.Filter_By_Value"), value_pane);
+	control.add(Dictionary.get("Autofilter.Custom_Filter"), custom_pane);
+
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,0,0,0));
+	button_pane.setLayout(new GridLayout(1, 3));
+	button_pane.add(set_button);
+	button_pane.add(remove_button);
+	button_pane.add(cancel_button);
+
+	lower_pane.setLayout(new BorderLayout());
+	lower_pane.add(order_pane, BorderLayout.CENTER);
+	lower_pane.add(button_pane, BorderLayout.SOUTH);
+
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(name_pane, BorderLayout.NORTH);
+	content_pane.add(control, BorderLayout.CENTER);
+	content_pane.add(lower_pane, BorderLayout.SOUTH);
+
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	screen_size = null;
+    }
+
+    public boolean cancelled() {
+	return cancelled;
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	dispose();
+	first_case = null;
+	second_case = null;
+	first_method = null;
+	first_value = null;
+	second_method = null;
+	second_value = null;
+	value = null;
+	control = null;
+	dialog = null;
+	self = null;
+	name = null;
+    }
+    /** Display the modal dialog box, allowing the user to define the filter. When the user presses one of the buttons, dispose and return to the caller providing an indication of which button was pressed.
+     * @return An <i>int</i> which indicates which button was pressed to dismiss the dialog.
+     */
+    public Autofilter display(Autofilter filter, ArrayList raw_values, String column_name) {
+	this.cancelled = false;
+	this.filter = filter;
+	name.setText(column_name);
+	// Prune values so it only contains unique entries, then order.
+	TreeSet values = new TreeSet(raw_values);
+	value.setModel(new DefaultComboBoxModel(values.toArray()));
+	String star = "*";
+	value.insertItemAt(star, 0);
+	value.setSelectedItem(star);
+	first_value.setModel(new DefaultComboBoxModel(values.toArray()));
+	second_value.setModel(new DefaultComboBoxModel(values.toArray()));
+	// Restore previous values.
+	if(filter != null && filter.value_one != null) {
+	    value.setSelectedItem(filter.value_one);
+	    first_method.setSelectedIndex(filter.method_one);
+	    first_value.setSelectedItem(filter.value_one);
+	    first_case.setSelected(filter.casesense_one);
+	    if (filter.value_two == null) {
+		none_radiobutton.setSelected(true);
+	    }
+	    else {
+		if (filter.operation) {
+		    and_radiobutton.setSelected(true);
+		}
+		else {
+		    or_radiobutton.setSelected(true);
+		}
+		second_method.setSelectedIndex(filter.method_two);
+		second_value.setSelectedItem(filter.value_two);
+		second_case.setSelected(filter.casesense_two);
+	    }
+	    if (filter.sort) {
+		ascending_radiobutton.setSelected(true);
+		// order.setSelectedIndex(0);
+	    }
+	    else {
+		descending_radiobutton.setSelected(true);
+		// order.setSelectedIndex(1);
+	    }
+	}
+	// Display	  
+	setVisible(true);
+	dialog.toFront();
+	return this.filter;
+    }
+
+    /** Listens for actions on the button it is attached to, and when notified sets the return_value and disposes of the dialog. */
+    private final class ButtonListener
+	implements ActionListener {
+	private boolean cancel_button;
+	private boolean return_filter;
+	
+	/** Default constructor. Returns any filter that was already set */
+	public ButtonListener() {
+	    this.cancel_button = true;
+	}
+
+	/** Does an action on this button cause a filter to be returned. */
+	/** Constructor takes an associated return value as an argument.
+	 * @param return_filter <i>true</i> if we update then return the filter, <i>false</i> to clear existing filter.
+	 */
+	public ButtonListener(boolean return_filter) {
+	    this.cancel_button = false;
+	    this.return_filter = return_filter;
+	}
+	/** When any registered component is actioned apon, set the value and hide the dialog. We hide rather than dispose, because hide assures the data values will be retained. 
+	 * @param event An <strong>ActionEvent</strong> containing information about the action that caused this method call.
+	 * @see org.greenstone.gatherer.gui.metaaudit.Autofilter
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    if(cancel_button) {
+		cancelled = true;
+	    }
+	    else if(return_filter) {
+		if(control.getSelectedIndex() == 0) {
+		    filter.setFilter(1, 0, (String)value.getSelectedItem(), true);
+		    filter.setFilter(2, 0, null, true);
+		}
+		else {
+		    filter.setFilter(1, first_method.getSelectedIndex(), (String)first_value.getSelectedItem(), first_case.isSelected());
+		    if(!none_radiobutton.isSelected()) {
+			if (and_radiobutton.isSelected()) {
+			    filter.setOperation(Autofilter.AND);
+			}
+			else {
+			    filter.setOperation(Autofilter.OR);
+			}
+			filter.setFilter(2, second_method.getSelectedIndex(), (String)second_value.getSelectedItem(), second_case.isSelected());
+		    }
+		}
+		if (ascending_radiobutton.isSelected()) {
+		    filter.setSort(Autofilter.ASCENDING);
+		}
+		else {
+		    filter.setSort(Autofilter.DESCENDING);
+		}
+	    }
+	    else {
+		filter = null;
+	    }
+	    setVisible(false);
+	}
+    }
+    /** Listens for actions on the check box it is attached to, and when notified sets the state of the second method and value to the specified state. */
+    private final class CheckListener
+	implements ActionListener {
+	/** The default desire state any check button this listens to. */
+	private boolean desired_state = false;
+	/** The constructor takes an associated desired state.
+	 * @param desired_state The state that should be set when this is actioned, as a <i>boolean</i>.
+			*/
+	public CheckListener(boolean desired_state) {
+	    this.desired_state = desired_state;
+	}
+	/** Whenever our registered components are actioned apon, set the state of the second method and value to the desired state. */
+	public void actionPerformed(ActionEvent event) {
+	    second_method.setEnabled(desired_state);
+	    second_value.setEnabled(desired_state);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Filter.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Filter.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/Filter.java	(revision 21112)
@@ -0,0 +1,151 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+/**************************************************************************************
+ * Title:        Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Company:      The University of Waikato
+ * Written:      03/09/02
+ * Revised:      04/10/02 - Commented
+ **************************************************************************************/
+import de.qfs.lib.gui.TableModelFilter;
+import de.qfs.lib.gui.TableModelFilterEvent;
+import de.qfs.lib.gui.TableModelFilterListener;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.Vector;
+import javax.swing.table.TableModel;
+
+/** This class essentially manages the autofilters applied to the metaaudit table.
+ * @author John Thompson
+ * @version 2.3
+ */
+public class Filter
+    implements TableModelFilter {
+    /** An array of autofilters. */
+    private Autofilter filters[];
+    /** The registered TableModelFilterListeners. */
+    private Vector listeners = new Vector ();
+    /** Constructor. 
+     * @param columns The number of columns this Filter will be expected to filter, as an <i>int</i>. 
+     */
+    public Filter(int columns) {
+	filters = new Autofilter[columns];
+    }
+    /** Add a TableModelFilterListener to the TableModelFilter. 
+     * @param listener The <strong>TableModelFilterListener</strong> to add, if it isn't already registered.
+     */
+    public synchronized void addTableModelFilterListener(TableModelFilterListener listener) {
+	if (!listeners.contains(listener)) {
+	    listeners.addElement(listener);
+	}
+    }
+    /** Remove a filter from a column.
+     * @param column The column number to remove the filter from, as an <i>int</i>.
+     */
+    public void clearFilter(int column) {
+	if(filters[column] != null) {
+	    filters[column].setActive(false);
+	}
+    }
+
+
+    /** Determine if a certain row should be shown in the table by checking it against all the current filters.
+     * @param model The <strong>TableModel</strong> the row is from.
+     * @param row The row number as an <i>int</i>
+     * @return <i>true</i> if the rows data matches all autofilters set and should be displayed, <i>false</i> otherwise.
+     */
+    public boolean filter(TableModel model, int row)
+    {
+	for (int i = 0; i < filters.length; i++) {
+	    if (filters[i] != null && filters[i].active()) {
+		ArrayList values = (ArrayList) model.getValueAt(row, i);
+		if (filters[i].filter(values) == false) {
+		    return false;
+		}
+	    }
+	}
+
+	return true;
+    }
+
+
+    /** Retrieve the autofilter associated with a certain column.
+     * @param column The column number as an <i>int</i>.
+     * @return The <strong>Autofilter</strong> assigned to that column.
+     */
+    public Autofilter getFilter(int column) {
+	Autofilter filter = filters[column];
+	if(filter == null) {
+	    filter = new Autofilter();
+	    filters[column] = filter;
+	}
+	return filter;
+    }
+    /** Determine if a certain column is filtered.
+     * @param column The column number as an <i>int</i>.
+     * @return <i>true</i> if there is an active autofilter assigned to this column, <i>false</i> otherwise.
+     */
+    public boolean isFiltered(int column) {
+	boolean result = false;
+	if(filters[column] != null) {
+	    result = filters[column].active();
+	}
+	return result;
+    }
+    /** Remove a TableModelFilterListener from the TableModelFilter.
+     * @param listener The <strong>TableModelFilterListener</strong> to remove.
+     */
+    public synchronized  void removeTableModelFilterListener(TableModelFilterListener listener) {
+	listeners.removeElement(listener);
+    }
+    /** Called whenever the filters assigned changes significantly, thus prompting a reload of the table model data. */
+    public void fireFilterChanged() {
+	///ystem.err.println("Filter Changed!");
+	Vector tmp;
+	synchronized(this) {
+	    tmp = (Vector) listeners.clone();
+	}
+		  
+	TableModelFilterEvent event = new TableModelFilterEvent (this);
+	Enumeration enumeration = tmp.elements();
+	while (enumeration.hasMoreElements()) {
+	    ((TableModelFilterListener) enumeration.nextElement()).filterChanged(event);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderListener.java	(revision 21112)
@@ -0,0 +1,79 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+
+import de.qfs.lib.gui.TableModelSorter;
+import java.awt.event.MouseAdapter;
+import java.awt.event.MouseEvent;
+import java.util.ArrayList;
+import org.greenstone.gatherer.metadata.MetadataAuditTableModel;
+
+public class HeaderListener
+    extends MouseAdapter {
+    private MetaAuditFrame parent_frame;
+    private MetaAuditTable table;
+    public HeaderListener(MetaAuditFrame parent_frame, MetaAuditTable table) {
+	this.parent_frame = parent_frame;
+	this.table = table;
+    }
+
+    public void mouseClicked(MouseEvent event) {
+	// Determine the column this was clicked on.
+	int column = table.getColumnModel().getColumnIndexAtX(event.getX());
+	int clicked_column = table.convertColumnIndexToModel(column);
+	// Display the currently assigned filter.
+	Filter filter_model = table.getFilter();
+	Autofilter filter = filter_model.getFilter(clicked_column);
+	MetadataAuditTableModel model = table.getOriginalModel();
+	ArrayList default_values = model.getColumnValues(clicked_column);
+	String column_name = model.getColumnName(clicked_column);
+	filter = parent_frame.autofilter_dialog.display(filter, default_values, column_name);
+	if(!parent_frame.autofilter_dialog.cancelled()) {
+	    if(filter == null) {
+		filter_model.clearFilter(clicked_column);
+	    }
+	    else {
+		filter.setActive(true);
+		TableModelSorter sorter = table.getSorter();
+		sorter.setSortColumn(clicked_column);
+		sorter.setSortAscending(filter.sort);
+	    }
+	    filter_model.fireFilterChanged();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderRenderer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderRenderer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/HeaderRenderer.java	(revision 21112)
@@ -0,0 +1,79 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.TableCellRenderer;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.JarTools;
+
+/** This custom button renderer replaces the boring old column headers with clickable versions which are based on JToggleButtons.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class HeaderRenderer 
+    extends JToggleButton
+    implements TableCellRenderer {
+
+    /** The constructor sets up a margin around the button, and adds the filter icons.
+     */
+    public HeaderRenderer() {
+	setMargin(new Insets(0,0,0,0));
+	setIcon(JarTools.getImage("filter.gif"));
+	setSelectedIcon(JarTools.getImage("filter-on.gif")); 
+    }
+
+    /** Called to create the component to be used as the 'rubber-stamp' for the table cell given the follwing arguments.
+     * @param table The <strong>JTable</strong> that is asking the renderer to draw; can be <i>null</i>.
+     * @param value The value of the cell to be rendered as an <strong>Object</strong>. It is up to the specific renderer to interpret and draw the value. For example, if value is the string "true", it could be rendered as a string or it could be rendered as a check box that is checked. null is a valid value.
+     * @param isSelected <i>true</i> if the cell is to be rendered with the selection highlighted; otherwise <i>false</i>.
+     * @param hasFocus If <i>true</i>, render cell appropriately. For example, put a special border on the cell, if the cell can be edited, render in the color used to indicate editing.
+     * @param row The row index of the cell being drawn as an <i>int</i>. When drawing the header, the value of row is -1.
+     * @param column The column index of the cell being drawn as an <i>int</i>.
+     * @see org.greenstone.gatherer.gui.metaaudit.MetaAuditTable
+     */
+    public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) {
+	setText(value.toString());
+	setToolTipText(Dictionary.get("Autofilter.Table_Header_Tooltip", value.toString()));
+	MetaAuditTable temp = (MetaAuditTable) table;
+	if (temp != null) { // And it may if this is called during table init.
+	    setSelected(temp.isFiltered(column));
+	}
+	return this;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditFrame.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditFrame.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditFrame.java	(revision 21112)
@@ -0,0 +1,225 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.gui.GLIButton;
+import org.greenstone.gatherer.gui.SimpleMenuBar;
+import org.greenstone.gatherer.gui.ModalDialog;
+import org.greenstone.gatherer.util.TreeSynchronizer;
+
+/** The MetaAuditFrame provides a table view of all of the metadata assigned to a selection of CollectionTreeNodes. All values for a certain file and a certain metadata element appear in the same cell. This table can be sorted by any column, and also has a MS Excel-like AutoFilter allowing you to restrict the rows visible to only those that match a certain set of criteria (applied to each column, and then 'ANDED', or cojoined, to determine the filter). Finally this dialog does not block the Gatherer tool, so the file selection can be changed and the dialog will just generate a new table dynamically.<BR>
+ * Much effort has gone into optimizing this table, as it quickly becomes slow and unresponsive to build/filter/sort when the number of records selected is high. However its performance is still nowhere as good as the Excel spreadsheet, and selections of 1000+ records can cause some serious waiting problems. Performance progression, shown in terms of time taken to perform action, are shown below. Note that building includes both the creation of the data model, and/or the time taken to lay out row and column sizes (due to this using multiple entry cells). Also all tables, by default, must be sorted by one column, which is initially the first column in ascending order:<BR><BR>
+ * <TABLE border=1 cellspacing=0 cellpadding=5>
+ * <TR><TD align=center colspan=5>Time Taken with 1000 records (ms)</TD></TR>
+ * <TR><TD>Action</TD><TD>Original</TD><TD>Revision 1</TD><TD>Revision 2</TD><TD>Revision 3</TD></TR>
+ * <TR><TD>Build</TD><TD align=right>113013</TD><TD align=right>1500</TD><TD align=right>1280</TD><TD align=right>665</TD></TR>
+ * <TR><TD>Sort</TD><TD align=right>34228</TD><TD align=right>353</TD><TD align=right>341</TD><TD align=right>338</TD></TR>
+ * <TR><TD>Filter</TD><TD align=right>57110</TD><TD align=right>485</TD><TD align=right>485</TD><TD align=right>485</TD></TR>
+ * </TABLE>
+ * <BR><BR>
+ * <TABLE border=1 cellspacing=0 cellpadding=5>
+ * <TR><TD align=center colspan=5>Time Taken with 10,000 records (ms) [hdlsub]</TD></TR>
+ * <TR><TD>Action</TD><TD>Original</TD><TD>Revision 1</TD><TD>Revision 2</TD><TD>Revision 3</TD></TR>
+ * <TR><TD>Build</TD><TD align=right>DNF</TD><TD align=right>8667</TD><TD align=right>6784</TD><TD align=right>3081</TD></TR>
+ * <TR><TD>Sort</TD><TD align=right>DNF</TD><TD align=right>236</TD><TD align=right>241</TD><TD align=right>228</TD></TR>
+ * <TR><TD>Filter</TD><TD align=right>DNF</TD><TD align=right>59272</TD><TD align=right>59767</TD><TD align=right>59614</TD></TR>
+ * </TABLE>
+ * <BR><BR>
+ * <TABLE border=1 cellspacing=0 cellpadding=5>
+ * <TR><TD align=center colspan=5>Time Taken with 30,000 records (ms) [hdl]</TD></TR>
+ * <TR><TD>Action</TD><TD>Original</TD><TD>Revision 1</TD><TD>Revision 2</TD><TD>Revision 3</TD></TR>
+ * <TR><TD>Build</TD><TD align=right>DNF</TD><TD align=right>19037</TD><TD align=right>16478</TD><TD align=right>8111</TD></TR>
+ * <TR><TD>Sort</TD><TD align=right>DNF</TD><TD align=right>314</TD><TD align=right>301</TD><TD align=right>385</TD></TR>
+ * <TR><TD>Filter</TD><TD align=right>DNF</TD><TD align=right>[300000]</TD><TD align=right>[300000]</TD><TD align=right>[300000]</TD></TR>
+ * </TABLE><BR><BR>
+ * It is easy to see that while building and sorting have become quite fast and reasonable, sorting is still unacceptable in terms of performance. The difficulty is this, that while sorting is easy (and fast) to implement via the Decorator pattern (making the overall table model a MDVC one) it requires the entire contents of a column to be available. Thus no matter the speed of your sort algorithm you will end up making N calls to getValueAt() in the table data model, for N records. This is not a problem for the initial model, hence the small sort times. However when you attempt to place another deocrator model (or modify the existing one) in order to filter only selected rows, even if the filtering itself takes little time, this little time quickly grows to unacceptable levels. In fact testing has shown it to grow much faster than linear time (though I'm not sure where). Doubling the number of records to filter increases the processing time by 5-7 times.<BR>
+ * In attempting to solve this problem, I have found many resources that mention a similar problem when trying to build visual representations of database tables, especially over networks or other temporally-non-deterministic connections. The common solution is to 'page' only those records necessary into memory on demand, thus making initial response significantly faster, while paying a small price for subsequent page seeks. There are even studies and algorithms for determining pages loads etc, I would imagine garnered from the memory management community. However none of these really help me usless I can get around the problem of having to have the entire columns population immeditaly accessable for sorting purposes.<BR>
+ * I believe with more time is would be possible to make a hybrid table model with properties such as you would require for the aforemenetioned database application, but which also provides fast methods for determining the highest and lowest records perhaps by building traversable heaps in the original model. Such a solution would incur a build penalty, and could consume significant memory, but would provide ordered and sequential access to the data within.<BR>
+ * The discussion aside, I have run out of time to do anything further on this, so will include as is, and provide a warning dialog whenever the number of records might suffer slow processing (1000+ records).
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class MetaAuditFrame
+    extends ModalDialog 
+    implements TreeSelectionListener {
+    public AutofilterDialog autofilter_dialog;
+    /** Whether the selection has changed since the last time we built the model (because it has been hidden and theres no point refreshing a model you can't see!). */
+    private boolean invalid = true;
+    /** An array holding the most recent list of paths selected (plus some nulls for those paths removed). */
+    private CollectionTreeNode records[];
+    /** A reference to ourselves so that inner classes can interact with us. */
+    private MetaAuditFrame self;
+    /** The table in which the metadata is shown. */
+    private MetaAuditTable table;
+    /** The default size for the metaaudit dialog. */
+    static final private Dimension SIZE = new Dimension(640,505);
+
+    /** Constructor.*/
+    public MetaAuditFrame(TreeSynchronizer tree_sync, CollectionTreeNode records[]) {
+	super(Gatherer.g_man);
+
+	// Arguments
+	this.autofilter_dialog = new AutofilterDialog(this);
+	this.records = records;
+	this.self = this;
+	this.table = new MetaAuditTable(this);
+
+	// Creation
+	setDefaultCloseOperation(JDialog.HIDE_ON_CLOSE);
+	setSize(SIZE);
+	setJMenuBar(new SimpleMenuBar("reviewingmetadata")); 
+	Dictionary.registerText(this, "MetaAudit.Title");
+	JPanel content_pane = (JPanel) getContentPane();
+	JPanel button_pane = new JPanel();
+
+	JButton close_button = new GLIButton();
+	close_button.setMnemonic(KeyEvent.VK_C);
+	Dictionary.registerBoth(close_button, "MetaAudit.Close", "MetaAudit.Close_Tooltip");
+
+	// Connection
+	close_button.addActionListener(new CloseListener());
+	tree_sync.addTreeSelectionListener(this);
+
+	// Layout
+	button_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	button_pane.setLayout(new BorderLayout());
+	button_pane.add(close_button, BorderLayout.CENTER);
+		 
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	content_pane.setLayout(new BorderLayout());
+	content_pane.add(new JScrollPane(table), BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+		  
+	Dimension screen_size = Configuration.screen_size;
+	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
+	screen_size = null;
+	close_button = null;
+	button_pane = null;
+	content_pane = null;
+    }
+
+    /** Destructor. */
+    public void destroy() {
+	records = null;
+	self = null;
+	table = null;
+    }
+
+    /** Display the dialog on screen. */
+    public void display() {
+	if(invalid) {
+	    rebuildModel();
+	}
+	setVisible(true);
+    }
+
+    public void setRecords(CollectionTreeNode[] records) {
+	this.records = records;
+	if(isVisible()) {
+	    rebuildModel();
+	}
+	else {
+	    invalid = true;
+	}
+    }
+
+    /** This method is called whenever the selection within the collection tree changes. This causes the table to be rebuilt with a new model.
+     * @param event A <strong>TreeSelectionEvent</strong> containing information about the event.
+     */
+    public void valueChanged(TreeSelectionEvent event) {
+	Object source = event.getSource();
+	if(source instanceof JTree) {
+	    TreePath paths[] = ((JTree)source).getSelectionPaths();
+	    if(paths != null) {
+		records = new CollectionTreeNode[paths.length];
+		for(int i = 0; i < paths.length; i++) {
+		    records[i] = (CollectionTreeNode) paths[i].getLastPathComponent();
+		}
+		if(isVisible()) {
+		    rebuildModel();
+		}
+		else {
+		    invalid = true;
+		}
+	    }
+	}
+    }
+
+    public void wait(boolean waiting) {
+	Component glass_pane = getGlassPane();
+	if(waiting) {
+	    // Show wait cursor.
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	    glass_pane.setVisible(true);
+	}
+	else {
+	    // Hide wait cursor.
+	    glass_pane.setVisible(false);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	}
+	glass_pane = null;
+    }
+
+    /** Rebuild the metaaudit table model using the current collection tree selection. 
+     */
+    private void rebuildModel() {
+	// Build and set model
+	table.newModel(records);
+	// Done.
+	invalid = false;
+    }
+
+    /** Listens for actions upon the close button, and if detected closes the dialog */
+    private class CloseListener
+	implements ActionListener {
+	/** Any implementation of ActionListener must include this method so that we can be informed when an action has been performed on our target control(s).
+	 * @param event An <strong>ActionEvent</strong> containing information about the event.
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    self.setVisible(false);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditRenderer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditRenderer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditRenderer.java	(revision 21112)
@@ -0,0 +1,63 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+import java.awt.*;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.table.DefaultTableCellRenderer;
+
+public class MetaAuditRenderer
+    extends DefaultTableCellRenderer {
+
+    public Component getTableCellRendererComponent(JTable table, Object value, boolean is_selected, boolean has_focus, int row, int column) {
+	ArrayList array = (ArrayList) value;
+	JLabel label = (JLabel) super.getTableCellRendererComponent(table, "", is_selected, has_focus, row, column);
+	JPanel panel = new JPanel();
+	panel.setBackground(label.getBackground());
+	panel.setBorder(label.getBorder());
+	if(array != null) {
+	    panel.setLayout(new GridLayout(array.size(), 1));
+	    for(int i = 0; i < array.size(); i++) {
+		JLabel entry = new JLabel(array.get(i).toString());
+		entry.setFont(label.getFont());
+		panel.add(entry);
+	    }
+	}
+	return panel;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditTable.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditTable.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/metaaudit/MetaAuditTable.java	(revision 21112)
@@ -0,0 +1,198 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.metaaudit;
+
+import de.qfs.lib.gui.*;
+import java.awt.*;
+import java.util.ArrayList;
+import javax.swing.JTable;
+import javax.swing.event.TableModelEvent;
+import javax.swing.table.*;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.metadata.MetadataAuditTableModel;
+
+
+/**
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class MetaAuditTable
+    extends JTable 
+    implements TableModelSorterListener {
+
+    private boolean initial_state;
+
+    private Filter filter;
+
+    private HeaderRenderer header_renderer;
+
+    private MetaAuditFrame parent_frame;
+
+    private MetaAuditRenderer cell_renderer;
+
+    private MetadataAuditTableModel metadata_audit_table_model;
+
+    private SortedTableHelper helper;
+
+    static final private int MARGIN = 10;
+
+    public MetaAuditTable(MetaAuditFrame parent_frame) {
+	super();
+	this.cell_renderer = new MetaAuditRenderer();
+	this.header_renderer = new HeaderRenderer();
+	this.parent_frame = parent_frame;
+	getTableHeader().addMouseListener(new HeaderListener(parent_frame, this));
+	setAutoResizeMode(AUTO_RESIZE_OFF);
+    }
+
+    public Filter getFilter() {
+	return filter;
+    }
+
+
+   public MetadataAuditTableModel getOriginalModel()
+   {
+      return metadata_audit_table_model;
+   }
+
+
+    public TableModelSorter getSorter() {
+	return helper.getTableModelSorter();
+    }
+
+    public boolean isFiltered(int column) {
+	if(filter == null) {
+	    return false;
+	}
+	return filter.isFiltered(column);
+    }
+
+    public void newModel(CollectionTreeNode records[]) {
+	if(records == null || records.length == 0) {
+	    setModel(new DefaultTableModel());
+	    return;
+	}
+	wait(true);
+	initial_state = true;
+	// System.err.println("Build new Model - should only be called once per click!");
+	metadata_audit_table_model = new MetadataAuditTableModel();
+	metadata_audit_table_model.rebuild(records);
+	setModel(metadata_audit_table_model);
+	filter = new Filter(metadata_audit_table_model.getColumnCount());
+	helper = new SortedTableHelper (this, filter, new DefaultTableModelSorter(), header_renderer, null);
+	helper.prepareTable();
+	// Add renderer to columns.
+	TableColumnModel column_model = getColumnModel();
+	for(int i = 0; i < column_model.getColumnCount(); i++) {
+	    column_model.getColumn(i).setCellRenderer(cell_renderer);
+	    column_model.getColumn(i).setHeaderRenderer(header_renderer);
+	}
+	wait(false);
+	initial_state = false;
+    }
+
+    public void tableChanged(TableModelEvent e) {
+	wait(true);
+	super.tableChanged(e);
+
+	// Create storage area
+	ArrayList heights[] = new ArrayList[dataModel.getRowCount()];
+	int widths_value[] = new int[dataModel.getColumnCount()];
+	ArrayList widths[] = new ArrayList[dataModel.getColumnCount()];
+
+	// Iterate through cell values finding tallest and widest for each row and column respectively.
+	for (int j = 0; j < dataModel.getRowCount(); j++) {
+	    for (int k = 0; k < dataModel.getColumnCount(); k++) {
+		Object object = dataModel.getValueAt(j, k);
+		if (object instanceof ArrayList) {
+		    ArrayList data = (ArrayList) object;
+		    if (heights[j] == null || data.size() > heights[j].size()) {
+			heights[j] = data;
+		    }
+		    String longest = "";
+		    for (int i = 0; i < data.size(); i++) {
+			String temp = data.get(i).toString();
+			if (temp.length() > longest.length()) {
+			    longest = temp;
+			}
+		    }
+		    if (widths[k] == null || longest.length() > widths_value[k]) {
+			widths_value[k] = longest.length();
+			widths[k] = data;
+		    }
+		}
+	    }
+	}
+
+	// Now assign the dimensions we have determined to be the largest.
+	for (int l = 0; l < heights.length; l++) {
+	    if (heights[l] != null) {
+		Component component = cell_renderer.getTableCellRendererComponent(this, heights[l], false, false, 0, 0);
+		Dimension size = component.getPreferredSize();
+		setRowHeight(l, size.height);
+	    }
+	}
+	for (int m = 0; m < widths.length; m++) {
+	    if (widths[m] != null) {
+		Component component = cell_renderer.getTableCellRendererComponent(this, widths[m], false, false, 0, 0);
+		Dimension size = component.getPreferredSize();
+		int width = size.width + MARGIN;
+		if (width > (2 * parent_frame.getSize().width) / 3) {
+		    width = (2 * parent_frame.getSize().width) / 3;
+		}
+
+		Component header = header_renderer.getTableCellRendererComponent(this, dataModel.getColumnName(m), false, false, 0, 0);
+		if (header.getPreferredSize().width + MARGIN > width) {
+		    width = header.getPreferredSize().width + MARGIN;
+		}
+		columnModel.getColumn(m).setPreferredWidth(width);
+	    }
+	}
+
+	wait(initial_state);
+    }
+
+
+    public void sortOrderChanged(TableModelSorterEvent event) {
+    }
+
+    private void wait(boolean waiting) {
+	if(parent_frame != null) {
+	    parent_frame.wait(waiting);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTree.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTree.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTree.java	(revision 21112)
@@ -0,0 +1,691 @@
+package org.greenstone.gatherer.gui.tree;
+
+import java.awt.*;
+import java.awt.datatransfer.*;
+import java.awt.dnd.*;
+import java.awt.geom.AffineTransform;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.plaf.basic.BasicTreeUI;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.file.FileNode;
+import org.greenstone.gatherer.file.FileSystemModel;
+import org.greenstone.gatherer.util.DragComponent;
+import org.greenstone.gatherer.util.DragGroup;
+import org.greenstone.gatherer.util.DragTreeSelectionModel;
+import org.greenstone.gatherer.util.Utility;
+
+public abstract class DragTree
+    extends JTree
+    implements Autoscroll, DragGestureListener, DragSourceListener, DropTargetListener, DragComponent, TreeSelectionListener {
+    /** The normal background color. */
+    private Color background_color;
+    /** The normal foreground color. */
+    private Color foreground_color;
+    /** The Group this component belongs to. */
+    private DragGroup group;
+    /** The image to use for the disabled background. */
+    private ImageIcon disabled_background;
+    /** The image to use for a normal background. */
+    private ImageIcon normal_background;
+    /** The icon to use for multiple node drag'n'drops. We decided against using the windows paradigm or a block of x horizontal lines for x files. */
+    private ImageIcon multiple_icon = new ImageIcon("resource"+File.separator+"multiple.gif");
+    /** The default drag action, although its not that important as we provide custom icons during drags. */
+    private int drag_action = DnDConstants.ACTION_MOVE;
+    /** The location of the last ghost drawn, so that we can repair the 'spoilt' area. */
+    private Point pt_last = null;
+    /** The region borderer by the lower cue line. */
+    private Rectangle lower_cue_line;
+    /** The region covered by the drag ghost icon. */
+    private Rectangle ra_ghost = new Rectangle();
+    /** The region borderer by the upper cue line. */
+    private Rectangle upper_cue_line;
+    /** The identifying name of this Tree. */
+    private String name;
+    /** The last tree path the drag was hovered over. */
+    private TreePath previous_path = null;
+
+    static private Cursor NO_DRAG_CURSOR = null;
+
+    static private final Color TRANSPARENT_COLOR = new Color(0,0,0,0);
+    /** The distance from the edge of the current view within the scroll bar which if entered causes the view to scroll. */
+    static private final int AUTOSCROLL_MARGIN = 12;
+
+    static public final int TREE_DISPLAY_CHANGED        = 0;
+    static public final int LOADED_COLLECTION_CHANGED   = 1;
+    static public final int COLLECTION_CONTENTS_CHANGED = 2;
+
+
+    public DragTree(String name, TreeModel model, boolean mixed_selection)
+    {
+	super();
+	init(name, mixed_selection);
+	if (model != null) {
+	    setModel(model);
+	}
+    }
+
+    public void init(String name, boolean mixed_selection) {
+	if (NO_DRAG_CURSOR == null) {
+	    NO_DRAG_CURSOR = DragSource.DefaultMoveNoDrop;
+	}
+
+	// Init
+	this.name = name;
+
+	// Creation
+	this.putClientProperty("JTree.lineStyle", "Angled");
+	this.setAutoscrolls(true);
+	this.setEditable(false);
+	this.setLargeModel(true);
+	this.setOpaque(true);
+	this.setSelectionModel(new DragTreeSelectionModel(this, mixed_selection));
+	this.setShowsRootHandles(true);
+	this.setUI(new BasicTreeUI());
+	// Connection
+	addTreeSelectionListener(this);
+
+	DragTreeCellRenderer renderer = new DragTreeCellRenderer();
+	//make the renderer paint nodes as transparent when not selected
+	//renderer.setBackgroundNonSelectionColor(new Color(0,0,0,0));
+	setCellRenderer(renderer);
+	// It turns out VariableHeightLayoutCache is buggy under MacOS, so I'll force it to use FixedHeightLayoutCache. To do that I have to set a cell height, ala below, and set the large model property to true, ala above. And buggy VariableHeightLayoutCache goes away. Plus this actually provides a minor performance boost as the layout manager doesn't have to calculate new bounds each time (well... I suppose any gain is actually pretty much eclipsed by the time taken for file access while we determine what a certain nodes children are).
+	// And once we have the cell renderer, use it to determine a fixed height for the rows
+	Component prototype_row = renderer.getTreeCellRendererComponent(this, "Prototype", true, true, false, 0, true);
+	this.setRowHeight(prototype_row.getSize().height);
+	prototype_row = null;
+
+
+	// Drag'n'drop Setup
+	// Drag source setup.
+	DragSource.getDefaultDragSource().createDefaultDragGestureRecognizer(this, drag_action, this);
+	// Drop destination setup.
+	new DropTarget(this, drag_action, this, true);
+    }
+
+    // Autoscroll Interface - Scroll because the mouse cursor is in our scroll zone.<br>
+    // The following code was borrowed from the book:<br>
+    //              Java Swing<br>
+    //              By Robert Eckstein, Marc Loy & Dave Wood<br>
+    //              Paperback - 1221 pages 1 Ed edition (September 1998)<br>
+    //              O'Reilly & Associates; ISBN: 156592455X<br>
+    // The relevant chapter of which can be found at:<br>
+    //              http://www.oreilly.com/catalog/jswing/chapter/dnd.beta.pdf<br>
+    // But I've probably tortured it beyond all recognition anyway.
+    public void autoscroll(Point pt) {
+	// Figure out which row we're on.
+	int row = getRowForLocation(pt.x, pt.y);
+	// If we are not on a row then ignore this autoscroll request
+	if (row < 0) return;
+	Rectangle bounds = getBounds();// Yes, scroll up one row
+	// Now decide if the row is at the top of the screen or at the bottom. We do this to make the previous row (or the next row) visible as appropriate. If we're at the absolute top or bottom, just return the first or last row respectively.
+	// Is row at top of screen?
+	if(pt.y + bounds.y <= AUTOSCROLL_MARGIN) {
+	    // Yes, scroll up one row
+	    if(row <= 0) {
+		row = 0;
+	    }
+	    else {
+		row = row - 1;
+	    }
+	}
+	else {
+	    // No, scroll down one row
+	    if(row < getRowCount() - 1) {
+		row = row + 1;
+	    }
+	}
+	this.scrollRowToVisible(row);
+    }
+
+    /** In order for the appearance to be consistant, given we may be in the situation where the pointer has left our focus but the ghost remains, this method allows other members of the GGroup to tell this component to clear its ghost.
+     */
+    public void clearGhost() {
+	// Erase the last ghost image and cue line
+	paintImmediately(ra_ghost.getBounds());
+    }
+
+    /** Any implementation of DragSourceListener must include this method so we can be notified when the drag event ends (somewhere else), which will in turn remove actions.
+     * @param event A <strong>DragSourceDropEvent</strong> containing all the information about the end of the drag event.
+     */
+    public void dragDropEnd(DragSourceDropEvent event) {
+	if(event.getDropSuccess()) {
+	    // Do whatever I do when the drop is successful.
+	}
+    }
+    /** Any implementation of DragSourceListener must include this method so we can be notified when the drag focus enters this component.
+     * @param event A <strong>DragSourceDragEvent</strong> containing all the information
+     * about the drag event.
+     */
+    public void dragEnter(DragSourceDragEvent event) {
+	// Handled elsewhere.
+    }
+    /** Any implementation of DropTargetListener must include this method so we can be notified when the drag focus enters this component, which in this case is to grab focus from within our group.
+     * @param event A <strong>DropTargetDragEvent</strong> containing all the information about the drag event.
+     */
+    public void dragEnter(DropTargetDragEvent event) {
+	group.grabFocus(this);
+    }
+    /** Any implementation of DragSourceListener must include this method so we can be notified when the drag focus leaves this component.
+     * @param event A <strong>DragSourceEvent</strong> containing all the information about the drag event.
+     */
+    public void dragExit(DragSourceEvent event) {
+	clearGhost();
+    }
+
+    /** Any implementation of DropTargetListener must include this method
+     * so we can be notified when the drag focus leaves this component.
+     * @param event A DropTargetEvent containing all the information
+     * about the drag event.
+     */
+    public void dragExit(DropTargetEvent event) {
+	clearGhost();
+    }
+
+    /** Any implementation of DragGestureListener must include this method
+     * so we can be notified when a drag action has been noticed, thus a
+     * drag action has begun.
+     * @param event A DragGestureEvent containing all the information about
+     * the drag event.
+     */
+    public void dragGestureRecognized(DragGestureEvent event) {
+	// Can never drag from Enrich tree
+	if(name.equals("Enrich")) {
+	    return;
+	}
+	// Disable editing, unless you want to have the edit box pop-up part way through dragging.
+	this.setEditable(false);
+	// We need this to find one of the selected nodes.
+	Point origin = event.getDragOrigin();
+	TreePath path = this.getPathForLocation(origin.x, origin.y);
+	// Taking into account our delayed model of selection, it is possible the user has performed a select and drag in one click. Here we utilize the Windows paradigm like so: If the node at the origin of the drag and drop is already in our selection then we perform multiple drag and drop. Otherwise we recognise that this is a distinct drag-drop and move only the origin node.
+	if(!isPathSelected(path)) {
+	    ((DragTreeSelectionModel)selectionModel).setImmediate(true);
+	    setSelectionPath(path);
+	    ((DragTreeSelectionModel)selectionModel).setImmediate(false);
+	}
+	if(path == null) {
+	    return;
+	}
+	if (!isValidDrag()) {
+	    try {
+		event.startDrag(NO_DRAG_CURSOR, new StringSelection("dummy"), this);
+		//dragging = true;
+	    }
+	    catch(Exception error) {
+		error.printStackTrace();
+	    }
+	    return;
+	}
+	// Now update the selection stored as far as the group is concerned.
+	group.setSelection(getSelectionPaths());
+	group.setSource(this);
+	// First grab ghost.
+	group.grabFocus(this);
+	// Ghost Image stuff.
+	Rectangle rect = this.getPathBounds(path);
+	group.mouse_offset = new Point(origin.x - rect.x, origin.y - rect.y);
+	// Create the ghost image.
+	// Retrieve the selected files.
+	int selection_count = getSelectionCount();
+	if(selection_count > 0) {
+	    JLabel label;
+	    if(selection_count == 1) {
+		TreePath node_path = getSelectionPath();
+		FileNode node = (FileNode) path.getLastPathComponent();
+		label = new JLabel(node.toString(), ((DefaultTreeCellRenderer)getCellRenderer()).getLeafIcon(), JLabel.CENTER);
+	    }
+	    else {
+		String title = getSelectionCount() + Dictionary.get("Tree.Files");
+		label = new JLabel(title, ((DefaultTreeCellRenderer)getCellRenderer()).getClosedIcon(), JLabel.CENTER);
+		title = null;
+	    }
+	    // The layout manager normally does this.
+	    Dimension label_size = label.getPreferredSize();
+	    label.setSize(label_size);
+	    label.setBackground(TRANSPARENT_COLOR);
+	    label.setOpaque(true);
+	    // Get a buffered image of the selection for dragging a ghost image.
+	    group.image_ghost = new BufferedImage(label_size.width, label_size.height, BufferedImage.TYPE_INT_ARGB_PRE);
+	    label_size = null;
+	    // Get a graphics context for this image.
+	    Graphics2D g2 = group.image_ghost.createGraphics();
+	    // Make the image ghostlike
+	    g2.setComposite(AlphaComposite.getInstance (AlphaComposite.SRC, 0.5f));
+	    // Ask the cell renderer to paint itself into the BufferedImage
+	    label.paint(g2);
+	    g2 = null;
+	    label = null;
+	    try {
+		event.startDrag(new Cursor(Cursor.DEFAULT_CURSOR), group.image_ghost, group.mouse_offset, new StringSelection("dummy"), this);
+		//dragging = true;
+	    }
+	    catch(Exception error) {
+		error.printStackTrace();
+	    }
+	}
+
+    }
+
+    /** Implementation side-effect.
+     * @param event A DragSourceDragEvent containing all the information about the drag event.
+     */
+    public void dragOver(DragSourceDragEvent event) {
+    }
+
+    /** Any implementation of DropTargetListener must include this method
+     * so we can be notified when the drag moves in this component.
+     * @param event A DropTargetDragEvent containing all the information
+     * about the drag event.
+     */
+    public void dragOver(DropTargetDragEvent event) {
+	// Draw the mouse ghost
+	Graphics2D g2 = (Graphics2D) getGraphics();
+	Point pt = event.getLocation();
+	if(pt_last != null && pt.equals(pt_last)) {
+	    return;
+	}
+	pt_last = pt;
+	if(!DragSource.isDragImageSupported() && group.image_ghost != null) {
+	    // Erase the last ghost image and cue line
+	    paintImmediately(ra_ghost.getBounds());
+	    // Remember where you are about to draw the new ghost image
+	    ra_ghost.setRect(pt.x - group.mouse_offset.x, pt.y - group.mouse_offset.y, group.image_ghost.getWidth(), group.image_ghost.getHeight());
+	    // Draw the ghost image
+	    g2.drawImage(group.image_ghost, AffineTransform.getTranslateInstance(ra_ghost.getX(), ra_ghost.getY()), null);
+	}
+	// Now we highlight the target node if it is a valid drop target. Of course we don't bother if we are still over a node which has already been identified as to whether its a drop target
+	TreePath target_path = this.getPathForLocation(pt.x, pt.y);
+	
+	if (target_path == null) {
+	    // the user has moved the mouse into background area - need to remove any existing cue lines
+	    if(upper_cue_line != null && lower_cue_line != null) {
+		paintImmediately(upper_cue_line.getBounds());
+		paintImmediately(lower_cue_line.getBounds());
+		previous_path = null;
+	    }
+	    // don't need to display anything
+	    return;
+	} 
+	
+	if(previous_path == null || target_path != null && !target_path.equals(previous_path)) {
+	    // Immediately clear the old cue lines.
+	    if(upper_cue_line != null && lower_cue_line != null) {
+		paintImmediately(upper_cue_line.getBounds());
+		paintImmediately(lower_cue_line.getBounds());
+	    }
+	    if(isValidDrop(target_path)) {
+		// Get the drop target's bounding rectangle
+		Rectangle raPath = getPathBounds(target_path);
+		// Cue line bounds (2 pixels beneath the drop target)
+		upper_cue_line = new Rectangle(0, raPath.y + (int)raPath.getHeight(), getWidth(), 2);
+		lower_cue_line = new Rectangle(0, raPath.y, getWidth(), 2);
+		g2.setColor(((DefaultTreeCellRenderer)cellRenderer).getBackgroundSelectionColor()); // The cue line color
+		g2.fill(upper_cue_line);         // Draw the cue line
+		g2.fill(lower_cue_line);
+	    }
+	    else {
+		upper_cue_line = null;
+		lower_cue_line = null;
+	    }
+	}
+    }
+
+    /** Any implementation of DropTargetListener must include this method
+	  * so we can be notified when the drag ends, ie the transferable is
+	  * dropped.
+	  * @param event A DropTargetDropEvent containing all the information
+	  * about the end of the drag event.
+	  */
+    public void drop(DropTargetDropEvent event) {
+	///start = System.currentTimeMillis();
+	///ystem.err.println("Drop target drop: " + this);
+	event.acceptDrop(drag_action);
+	if(!name.equals(Utility.WORKSPACE_TREE)) {
+	    // Determine what node we dropped over.
+	    Point pt = event.getLocation();
+	    TreePath target_path = this.getPathForLocation(pt.x, pt.y);
+	    FileNode target = null;
+	    if(target_path != null) {
+		if(isValidDrop(target_path)) {
+		    target = (FileNode) target_path.getLastPathComponent();
+		} else if (isValidDropOntoFile(target_path)) {
+		    target = (FileNode) target_path.getParentPath().getLastPathComponent();
+		}
+	    }
+	    else {
+		TreeModel m = getModel();
+		// check that we have a model and is a FileSystemModel (ie check that a collection is loaded
+		if (m != null && m instanceof FileSystemModel) {
+		    target = (FileNode) m.getRoot();
+		}
+	    }
+	    target_path = null;
+	    pt = null;
+	    if(target != null) {
+		///ystem.err.println("Valid drop.");
+		TreePath[] selection = group.getSelection();
+		if(target != null && selection != null) {
+		    FileNode[] source_nodes = new FileNode[selection.length];
+		    for(int i = 0; i < source_nodes.length; i++) {
+			source_nodes[i] = (FileNode) selection[i].getLastPathComponent();
+		    }
+		    Gatherer.f_man.action(group.getSource(), source_nodes, this, target);
+		    source_nodes = null;
+		}
+		group.setSelection(null);
+		group.setSource(null);
+		selection = null;
+		target = null;
+	    }
+	}
+	// Clear up the group.image_ghost
+	paintImmediately(ra_ghost.getBounds());
+	event.getDropTargetContext().dropComplete(true);
+	group.image_ghost = null;
+    }
+
+
+    /** Any implementation of DragSourceListener must include this method
+     * so we can be notified when the action to be taken upon drop changes.
+     * @param event A DragSourceDragEvent containing all the information
+     * about the drag event.
+     */
+    public void dropActionChanged(DragSourceDragEvent event) {
+    }
+
+    /** Any implementation of DropTargetListener must include this method
+     * so we can be notified when the action to be taken upon drop changes.
+     * @param event A DropTargetDragEvent containing all the information
+     * about the drag event.
+     */
+    public void dropActionChanged(DropTargetDragEvent event) {
+    }
+
+    /** Used to notify this component that it has gained focus. It should
+     * make some effort to inform the user of this.
+     */
+    public void gainFocus() {
+	///ystem.err.println("Gained focus: " + this);
+	((DragTreeCellRenderer)cellRenderer).gainFocus();
+	repaint();
+    }
+
+    /** Autoscroll Interface...
+     * The following code was borrowed from the book:
+     *              Java Swing
+     *              By Robert Eckstein, Marc Loy & Dave Wood
+     *              Paperback - 1221 pages 1 Ed edition (September 1998)
+     *              O'Reilly & Associates; ISBN: 156592455X
+     *
+     * The relevant chapter of which can be found at:
+     *              http://www.oreilly.com/catalog/jswing/chapter/dnd.beta.pdf
+     * Calculate the insets for the *JTREE*, not the viewport
+     * the tree is in. This makes it a bit messy.
+     */
+    public Insets getAutoscrollInsets()
+    {
+	Rectangle raOuter = this.getBounds();
+	Rectangle raInner = this.getParent().getBounds();
+	return new Insets(raInner.y - raOuter.y + AUTOSCROLL_MARGIN,
+			  raInner.x - raOuter.x + AUTOSCROLL_MARGIN,
+			  raOuter.height - raInner.height - raInner.y + raOuter.y + AUTOSCROLL_MARGIN,
+			  raOuter.width - raInner.width - raInner.x + raOuter.x + AUTOSCROLL_MARGIN);
+    }
+
+    public String getSelectionDetails() {
+	return ((DragTreeSelectionModel)selectionModel).getDetails();
+    }
+
+    public FileSystemModel getTreeModel() {
+	return (FileSystemModel) getModel();
+    }
+
+    /** This method is used to inform this component when it loses focus,
+     * and should indicate this somehow.
+     */
+    public void loseFocus() {
+	///ystem.err.println("Lost focus: " + this);
+	((DragTreeCellRenderer)cellRenderer).loseFocus();
+	repaint();
+    }
+
+
+    public void paint(Graphics g) {
+	if(disabled_background != null) {
+	    int height = getSize().height;
+	    int offset = 0;
+	    ImageIcon background;
+	    if(isEnabled()) {
+		background = normal_background;
+	    }
+	    else {
+		background = disabled_background;
+	    }
+	    while((height - offset) > 0) {
+		g.drawImage(background.getImage(), 0, offset, null);
+		offset = offset + background.getIconHeight();
+	    }
+	    background = null;
+	}
+	super.paint(g);
+    }
+
+    public void refresh(TreePath path) {
+	if (treeModel instanceof FileSystemModel) {
+	    ((FileSystemModel)treeModel).refresh(path);
+	}
+	else {
+	    // System.err.println("DragTree::refresh - Tree model is " + treeModel);
+	}
+    }
+
+    public void setBackgroundNonSelectionColor(Color color) {
+	background_color = color;
+	if(isEnabled()) {
+	    setBackground(color);
+	    ((DefaultTreeCellRenderer)cellRenderer).setBackgroundNonSelectionColor(color);
+	}
+	else {
+	    setBackground(Color.lightGray);
+	    ((DefaultTreeCellRenderer)cellRenderer).setBackgroundNonSelectionColor(Color.lightGray);
+	}
+	repaint();
+    }
+
+    public void setBackgroundSelectionColor(Color color) {
+	((DefaultTreeCellRenderer)cellRenderer).setBackgroundSelectionColor(color);
+	repaint();
+    }
+
+    /** Override the normal setEnabled so the Tree exhibits a little more
+     * change, which in this instance is the background colour changing.
+     * @param state Whether this GTree should be in an enabled state.
+     */
+    public void setEnabled(boolean state) {
+	super.setEnabled(state);
+
+	// Change some colors
+	if(state) {
+	    setBackground(background_color);
+	    ((DefaultTreeCellRenderer)cellRenderer).setBackgroundNonSelectionColor(background_color);
+	    ((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(foreground_color);
+	}
+	else {
+	    setBackground(Color.lightGray);
+	    ((DefaultTreeCellRenderer)cellRenderer).setBackgroundNonSelectionColor(Color.lightGray);
+	    ((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(Color.black);
+	}
+	repaint();
+    }
+
+    public void setGroup(DragGroup group) {
+	this.group = group;
+    }
+
+    /** Determines whether the following selection attempts should go through the normal delayed selection model, or should happen immediately.*/
+    public void setImmediate(boolean state) {
+	((DragTreeSelectionModel)selectionModel).setImmediate(state);
+    }
+
+    public void setModel(TreeModel model) {
+	super.setModel(model);
+	if(model instanceof FileSystemModel) {
+	    FileSystemModel file_system_model = (FileSystemModel) model;
+	    file_system_model.setTree(this);
+	    removeTreeExpansionListener(file_system_model);
+	    addTreeExpansionListener(file_system_model);
+	    removeTreeWillExpandListener(file_system_model);
+	    addTreeWillExpandListener(file_system_model);
+	    file_system_model = null;
+	}
+    }
+
+    /** Ensure that that file node denoted by the given file is selected. */
+    public void setSelection(File file) {
+	// We know the file exists, and thus that it must exists somewhere in our tree hierarchy.
+	// 1. Retrieve the root node of our tree.
+	FileNode current = (FileNode) getModel().getRoot();
+	// 2. Find that node in the file parents, keeping track of each intermediate file.
+	ArrayList files = new ArrayList();
+	while(file != null && !current.toString().equals(file.getName())) {
+	    files.add(0, file);
+	    file = file.getParentFile();
+	}
+	if(file == null) {
+	    return;
+	}
+	// 3. While there are still remaining intermediate files.
+	while(files.size() > 0) {
+	    file = (File) files.remove(0);
+				// 3a. Find the next file in the current nodes children.
+	    boolean found = false;
+	    current.map();
+	    for(int i = 0; !found && i < current.getChildCount(); i++) {
+		FileNode child = (FileNode) current.getChildAt(i);
+		if(child.toString().equals(file.getName())) {
+		    // 3b. Make the current node this node (if found) and continue.
+		    found = true;
+		    current = child;
+		}
+	    }
+	    // 3c. If not found then return as this node can't exists somehow.
+	    if(!found) {
+		return;
+	    }
+	}
+	// 4. We should now have the desired node. Remember to make the selection immediate.
+	TreePath path = new TreePath(current.getPath());
+	setImmediate(true);
+	setSelectionPath(path);
+	setImmediate(false);
+    }
+
+    public void setTextNonSelectionColor(Color color) {
+	foreground_color = color;
+	if(isEnabled()) {
+	    ((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(color);
+	}
+	else {
+	    ((DefaultTreeCellRenderer)cellRenderer).setTextNonSelectionColor(Color.black);
+	}
+	repaint();
+    }
+
+    public void setTextSelectionColor(Color color) {
+	((DefaultTreeCellRenderer)cellRenderer).setTextSelectionColor(color);
+	repaint();
+    }
+
+    public String toString() {
+	return name;
+    }
+
+    public void valueChanged(TreeSelectionEvent event) {
+	if(group == null) {
+	    ///ystem.err.println("Oh my god, this tree has no group: " + this);
+	}
+	else {
+	    group.grabFocus(this);
+	}
+    }
+
+    /** returns false for dummy nodes (ones without files), and system root
+     * nodes */
+    private boolean isValidDrag() {
+	//because you cant select nodes that are children of another selection, and we use a contiguous selection model, we just test the first selection path
+	TreePath node_path = getSelectionPath();
+	FileNode node = (FileNode) node_path.getLastPathComponent();
+
+	if (node.getFile() == null) {
+	    return false;
+	}
+	if (node.isFileSystemRoot()) {
+	    return false;
+	}
+	// We also don't allow the user to select files that reside within the currently loaded collection
+	TreePath[] paths = getSelectionPaths();
+	for(int i = 0; i < paths.length; i++) {
+	    FileNode child = (FileNode) paths[i].getLastPathComponent();
+	    if (child.isInLoadedCollection()) {
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    private boolean isValidDropOntoFile(TreePath target_path) {
+	boolean valid = false;
+	if(target_path != null) {
+	    FileNode target_node = (FileNode) target_path.getLastPathComponent();
+	    if (target_node.isLeaf()) {
+		// check the parent node for being a valid drop
+		return isValidDrop(target_path.getParentPath());
+	    }
+	}
+	return false;
+    }
+
+		
+    private boolean isValidDrop(TreePath target_path) {
+	boolean valid = false;
+	if(target_path == null) {
+	    previous_path = null;
+	} 
+	else {
+	    FileNode target_node = (FileNode) target_path.getLastPathComponent();
+	    // We can only continue testing if the node is a folder.
+	    if(!target_node.isLeaf()) {
+		// Now we check if the node is readonly.
+		if(!target_node.isReadOnly()) {
+		    // Finally we check the target path against the paths in the selection to ensure we are not adding to our own ancestors!
+		    TreePath[] selection = group.getSelection();
+		    boolean failed = false;
+		    for(int i = 0; !failed && selection != null && i < selection.length; i++) {
+			failed = selection[i].isDescendant(target_path);
+		    }
+		    // Having finally completed all the tests, we can highlight the drop target.
+		    if(!failed) {
+			valid = true;
+		    }
+		    else {
+			///ystem.err.println("Invalid. Target is descendant of itself.");
+		    }
+		}
+		else {
+		    ///ystem.err.println("Read only.");
+		}
+	    }
+	    else {
+		///ystem.err.println("Leaf node. Children not allowed.");
+	    }
+	    previous_path = target_path;
+	}
+	return valid;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTreeCellRenderer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTreeCellRenderer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/gui/tree/DragTreeCellRenderer.java	(revision 21112)
@@ -0,0 +1,89 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.gui.tree;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.tree.DefaultTreeCellRenderer;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.file.FileNode;
+import org.greenstone.gatherer.util.StaticStrings;
+import org.greenstone.gatherer.util.Utility;
+
+public class DragTreeCellRenderer
+    extends DefaultTreeCellRenderer
+{
+    private Color selection_background;
+    private Color selection_foreground;
+
+    public DragTreeCellRenderer() {
+	super();
+	// Have to wait until this image is loaded
+	selection_background = getBackgroundSelectionColor();
+	selection_foreground = getTextSelectionColor();
+    }
+
+    public void gainFocus() {
+	setBackgroundSelectionColor(selection_background);
+	setTextSelectionColor(selection_foreground);
+    }
+
+    /** Configures the renderer based on the passed in components. */
+    public Component getTreeCellRendererComponent(JTree tree, Object value, boolean sel, boolean expanded, boolean leaf, int row, boolean hasFocus)
+    {
+	// Rendering FileNodes
+	if (value instanceof FileNode) {
+	    FileNode node = (FileNode) value;
+	    String node_text = node.toString();
+
+	    // Add file sizes if necessary
+	    if (Configuration.get("general.show_file_size", Configuration.COLLECTION_SPECIFIC) && node.getFile() != null && !node.getAllowsChildren()) {
+		node_text = node_text + StaticStrings.SPACE_CHARACTER + StaticStrings.LBRACKET_CHARACTER + Utility.formatFileLength(node.getFile().length()) + StaticStrings.RBRACKET_CHARACTER;
+	    }
+
+	    return (JLabel) super.getTreeCellRendererComponent(tree, node_text, sel, expanded, leaf, row, hasFocus);
+	}
+
+	// Use default renderer for anything else
+	return (JLabel) super.getTreeCellRendererComponent(tree, value, sel, expanded, leaf, row, hasFocus);
+    }
+
+    public void loseFocus() {
+	setBackgroundSelectionColor(Color.lightGray);
+	setTextSelectionColor(Color.black);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java	(revision 21112)
@@ -0,0 +1,273 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.Utility;
+
+
+/** This class represents one doc.xml file */
+public class DocXMLFile
+    extends File
+{
+    private HashMap source_file_name_to_description_elements_mapping = new HashMap();
+
+
+    public DocXMLFile(String doc_xml_file_path)
+    {
+	super(doc_xml_file_path);
+    }
+
+
+    public ArrayList getMetadataExtractedFromFile(File file)
+    {
+	// Build up a list of metadata extracted from this file
+	ArrayList metadata_values = new ArrayList();
+
+	String file_relative_path = file.getAbsolutePath();
+	int import_index = file_relative_path.indexOf("import");
+	if (import_index != -1) {
+	    file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
+	}
+
+	// Check whether this doc.xml file contains extracted metadata for the specified file
+	ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
+	if (description_elements_list == null) {
+	    // ...it doesn't
+	    return metadata_values;
+	}
+
+ 	MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+	// Parse the doc.xml file
+	DebugStream.println("Applicable doc.xml file: " + this);
+	try {
+	    BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+
+	    int description_element_num = 0;
+	    int next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+	    boolean in_relevant_description_element = false;
+
+	    String line = null;
+	    for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+		// Check if this line contains the start of a relevant Description element
+		if (line_num == next_description_element_start) {
+		    in_relevant_description_element = true;
+		    continue;
+		}
+
+		// If we're not in a relevant Description element we don't care about anything
+		if (in_relevant_description_element == false) {
+		    continue;
+		}
+
+		// Check if this line contains the end of the relevant Description element
+		if (line.indexOf("</Description>") != -1) {
+		    description_element_num++;
+		    if (description_element_num == description_elements_list.size()) {
+			break;
+		    }
+
+		    next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+		    in_relevant_description_element = false;
+		    continue;
+		}
+
+		// If this line doesn't contain a complete Metadata element, we're not interested
+		if (line.indexOf("<Metadata ") == -1 || line.indexOf("</Metadata>") == -1) {
+		    continue;
+		}
+
+		// Extract the metadata element name
+		int name_index = line.indexOf(" name=\"") + " name=\"".length();
+		String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+		// If the metadata has a namespace it isn't extracted metadata, so we're not interested
+		String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+		if (!metadata_set_namespace.equals("")) {
+		    continue;
+		}
+
+		// Extracted metadata!
+		String metadata_element_name = metadata_element_name_full;
+
+		// We completely ignore bibliographic data
+		if (metadata_element_name.equals("SourceSegment")) {
+		    return new ArrayList();
+		}
+
+		// Ignore lower-case metadata elements (gsdlsourcefilename, gsdlassocfile etc.)
+		//   and those starting with '/' (/srclink)
+		char first_character = metadata_element_name.charAt(0);
+		if (Character.isLowerCase(first_character) || first_character == '/') {
+		    continue;
+		}
+
+		MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+
+		// Value trees are not stored for extracted metadata, so create a new value tree node now
+		int value_index = line.indexOf(">", name_index) + ">".length();
+		String metadata_element_value = line.substring(value_index, line.lastIndexOf("</Metadata>"));
+
+		metadata_element.addMetadataValue(metadata_element_value);
+		MetadataValueTreeNode metadata_value_tree_node = metadata_element.getMetadataValueTreeNode(metadata_element_value);
+
+		// Add the new metadata value to the list
+		MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+		metadata_values.add(metadata_value);
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+
+	return metadata_values;
+    }
+
+
+    /**
+     * Every doc.xml file must be skimmed when a collection is opened, for two reasons:
+     *   - To build a mapping from source file to its corresponding doc.xml file
+     *   - To get a complete list of all extracted metadata elements
+     */
+    public void skimFile()
+    {
+	MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+	// Skim the doc.xml file as quickly as possible (don't parse as XML), looking at the Metadata elements
+	DebugStream.println("Skimming doc.xml file " + this + "...");
+	try {
+	    BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+	    int description_element_start = -1;
+
+	    String line = null;
+	    for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+		// This line contains the start of a Description element
+		if (line.indexOf("<Description>") != -1) {
+		    if (description_element_start != -1) {
+			System.err.println("Parse error: previous Description element unfinished!");
+		    }
+		    description_element_start = line_num;
+		    continue;
+		}
+
+		// This line contains the end of a Description element
+		if (line.indexOf("</Description>") != -1) {
+		    if (description_element_start == -1) {
+			System.err.println("Parse error: Description element unstarted!");
+		    }
+		    description_element_start = -1;
+		    continue;
+		}
+
+		// If we're not in a Description element there shouldn't be any Metadata elements
+		if (description_element_start == -1) {
+		    continue;
+		}
+
+		// This line doesn't contain a Metadata element, so we're not interested
+		if (line.indexOf("<Metadata ") == -1) {
+		    DebugStream.println("Warning: Description element line doesn't contain Metadata element.");
+		    continue;
+		}
+
+		// Extract the metadata element name
+		int name_index = line.indexOf(" name=\"") + " name=\"".length();
+		String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+		// If the metadata has a namespace it isn't extracted metadata, so we're not interested
+		String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+		if (!metadata_set_namespace.equals("")) {
+		    continue;
+		}
+
+		// Extracted metadata!
+		String metadata_element_name = metadata_element_name_full;
+
+		// Note which file this doc.xml is for
+		if (metadata_element_name.equals("gsdlsourcefilename")) {
+		    // Extract the gsdlsourcefilename element value
+		    int value_index = line.indexOf(">", name_index) + ">".length();
+		    String gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
+
+		    // We're only interested in the path relative to the import folder
+		    int import_index = gsdlsourcefilename_value.indexOf("import");
+		    if (import_index != -1) {
+			gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
+
+			boolean is_unix_path = gsdlsourcefilename_value.startsWith("/");
+			gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
+
+			// Make sure the path matches the OS that is running
+			if (is_unix_path && Utility.isWindows()) {
+			    // Convert path from Unix to Windows
+			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
+			}
+			else if (!is_unix_path && !Utility.isWindows()) {
+			    // Convert path from Windows to Unix
+			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
+			}
+
+			// Remember this for quick access later
+			if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
+			    source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, new ArrayList());
+			}
+
+			((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
+		    }
+
+		    // Warn about an odd gsdlsourcefilename, except if it is the Greenstone "tmp" directory
+		    // This is true when the source files come from a zip file processed by ZIPPlug, for example
+		    else if (gsdlsourcefilename_value.indexOf("tmp") == -1) {
+			// We don't really know what is going on...
+			System.err.println("Warning: Could not understand gsdlsourcefilename " + gsdlsourcefilename_value);
+		    }
+		}
+
+		// Ignore lower-case metadata elements (gsdlsourcefilename, gsdlassocfile etc.)
+		//   and those starting with '/' (/srclink)
+		char first_character = metadata_element_name.charAt(0);
+		if (Character.isLowerCase(first_character) || first_character == '/') {
+		    continue;
+		}
+
+		MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+		if (metadata_element == null) {
+		    // This element isn't defined in ex.mds, so create it for this session
+		    DebugStream.println("Extracted metadata element not defined: " + metadata_element_name);
+		    extracted_metadata_set.addMetadataElementForThisSession(metadata_element_name);
+		}
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java	(revision 21112)
@@ -0,0 +1,88 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+
+
+/** This class is a static class that manages the doc.xml files */
+public class DocXMLFileManager
+{
+    static private ArrayList doc_xml_files = new ArrayList();
+
+
+    static public void clearDocXMLFiles()
+    {
+	doc_xml_files.clear();
+    }
+
+
+    static public ArrayList getMetadataExtractedFromFile(File file)
+    {
+	// Build up a list of metadata values extracted from this file
+	ArrayList metadata_values = new ArrayList();
+
+	// Look at each loaded doc.xml file to see if any have extracted metadata for this file
+	for (int i = 0; i < doc_xml_files.size(); i++) {
+	    DocXMLFile doc_xml_file = (DocXMLFile) doc_xml_files.get(i);
+	    metadata_values.addAll(doc_xml_file.getMetadataExtractedFromFile(file));
+	}
+
+	return metadata_values;
+    }
+
+
+    static public void loadDocXMLFiles(File directory)
+    {
+	// Make sure the directory (archives) exists
+	if (directory.exists() == false) {
+	    return;
+	}
+
+	// Look recursively at each subfile of the directory for doc.xml files
+	File[] directory_files = directory.listFiles();
+	for (int i = 0; i < directory_files.length; i++) {
+	    File child_file = directory_files[i];
+	    if (child_file.isDirectory()) {
+		loadDocXMLFiles(child_file);
+	    }
+	    else if (child_file.getName().equals("doc.xml")) {
+		loadDocXMLFile(child_file);
+	    }
+	}
+    }
+
+
+    static private void loadDocXMLFile(File doc_xml_file_file)
+    {
+	DocXMLFile doc_xml_file = new DocXMLFile(doc_xml_file_file.getAbsolutePath());
+	doc_xml_file.skimFile();
+	doc_xml_files.add(doc_xml_file);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataAuditTableModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataAuditTableModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataAuditTableModel.java	(revision 21112)
@@ -0,0 +1,237 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+
+
+public class MetadataAuditTableModel
+    extends AbstractTableModel
+{
+    /** The CollectionTreeNodes this model is built for */
+    private CollectionTreeNode[] file_nodes = null;
+    /** The list of MetadataElements in the table (columns) */
+    private ArrayList metadata_elements = new ArrayList();
+    /** The list of MetadataAuditTableEntries in the table (rows) */
+    private ArrayList metadata_audit_table_entries = new ArrayList();
+
+
+    /** Returns the number of columns in this table. */
+    public int getColumnCount()
+    {
+	return 1 + metadata_elements.size();
+    }
+
+
+    /** Returns the name of the specified column. */
+    public String getColumnName(int col)
+    {
+	// First column: filename
+	if (col == 0) {
+	    return Dictionary.get("AuditTable.File");
+	}
+
+	int metadata_element_index = col - 1;
+	return metadata_elements.get(metadata_element_index).toString();
+    }
+
+
+    /** Returns the values in a particular column. */
+    public ArrayList getColumnValues(int col)
+    {
+	// First column: filename
+	if (col == 0) {
+	    ArrayList column_values = new ArrayList();
+	    for (int i = 0; i < metadata_audit_table_entries.size(); i++) {
+		MetadataAuditTableEntry metadata_audit_table_entry = (MetadataAuditTableEntry) metadata_audit_table_entries.get(i);
+		column_values.add(metadata_audit_table_entry.getFilename());
+	    }
+
+	    return column_values;
+	}
+
+	// Check values are reasonable
+	int metadata_element_index = col - 1;
+	if (metadata_element_index < 0 || metadata_element_index >= metadata_elements.size()) {
+	    return null;
+	}
+
+	ArrayList column_values = new ArrayList();
+	for (int i = 0; i < metadata_audit_table_entries.size(); i++) {
+	    MetadataAuditTableEntry metadata_audit_table_entry = (MetadataAuditTableEntry) metadata_audit_table_entries.get(i);
+	    MetadataValueTreeNode[] metadata_value_tree_nodes = metadata_audit_table_entry.getMetadataValueTreeNodes(metadata_element_index);
+	    if (metadata_value_tree_nodes != null) {
+		for (int j = 0; j < metadata_value_tree_nodes.length; j++) {
+		    column_values.add(metadata_value_tree_nodes[j].toString());
+		}
+	    }
+	}
+
+	return column_values;
+    }
+
+
+    /** Returns the number of rows in this table. */
+    public int getRowCount()
+    {
+	return metadata_audit_table_entries.size();
+    }
+
+
+    /** Returns the cell value at a given row and column as an Object. */
+    public Object getValueAt(int row, int col)
+    {
+	// Check values are reasonable
+	if (row < 0 || row >= metadata_audit_table_entries.size()) {
+	    return null;
+	}
+
+	// First column: filename
+	if (col == 0) {
+	    ArrayList cell_values = new ArrayList();
+	    cell_values.add(((MetadataAuditTableEntry) metadata_audit_table_entries.get(row)).getFilename());
+	    return cell_values;
+	}
+
+	// Check values are reasonable
+	int metadata_element_index = col - 1;
+	if (metadata_element_index < 0 || metadata_element_index >= metadata_elements.size()) {
+	    return null;
+	}
+
+	ArrayList cell_values = new ArrayList();
+	MetadataAuditTableEntry metadata_audit_table_entry = (MetadataAuditTableEntry) metadata_audit_table_entries.get(row);
+	MetadataValueTreeNode[] metadata_value_tree_nodes = metadata_audit_table_entry.getMetadataValueTreeNodes(metadata_element_index);
+	if (metadata_value_tree_nodes != null) {
+	    for (int j = 0; j < metadata_value_tree_nodes.length; j++) {
+		cell_values.add(metadata_value_tree_nodes[j].toString());
+	    }
+	}
+
+	return cell_values;
+    }
+
+
+    public void rebuild(CollectionTreeNode[] file_nodes)
+    {
+	metadata_elements = MetadataSetManager.getEveryMetadataSetElement();
+	metadata_audit_table_entries.clear();
+
+	this.file_nodes = file_nodes;
+	if (file_nodes != null && file_nodes.length > 0) {
+	    // Create model builder
+	    MetadataAuditTableModelBuilder builder = new MetadataAuditTableModelBuilder();
+	    builder.run();
+	}
+    }
+
+
+    private class MetadataAuditTableModelBuilder
+    {
+	// Build a list of MetadataAuditTableEntries that represent the metadata asssigned to the selected files
+	public void run()
+	{
+	    // Process each of the selected files in turn
+	    for (int i = 0; i < file_nodes.length; i++) {
+		File current_file = file_nodes[i].getFile();
+
+		// Get the metadata assigned to this file
+		ArrayList current_file_metadata = MetadataXMLFileManager.getMetadataAssignedToFile(current_file);
+
+		// Get the extracted metadata for this file, if desired
+		if (Configuration.get("general.view_extracted_metadata", Configuration.COLLECTION_SPECIFIC) == true) {
+		    current_file_metadata.addAll(DocXMLFileManager.getMetadataExtractedFromFile(current_file));
+		}
+
+		MetadataAuditTableEntry metadata_audit_table_entry = new MetadataAuditTableEntry(current_file, current_file_metadata);
+		metadata_audit_table_entries.add(metadata_audit_table_entry);
+	    }
+	}
+    }
+
+
+    /** This class represents one (file, metadata) pair */
+    private class MetadataAuditTableEntry
+    {
+	private File file = null;
+	private ArrayList[] metadata_values = null;
+
+
+	public MetadataAuditTableEntry(File file, ArrayList metadata_values_list)
+	{
+	    this.file = file;
+
+	    metadata_values = new ArrayList[metadata_elements.size()];
+	    for (int i = 0; i < metadata_values.length; i++) {
+		metadata_values[i] = new ArrayList();
+	    }
+
+	    for (int i = 0; i < metadata_values_list.size(); i++) {
+		MetadataValue metadata_value = (MetadataValue) metadata_values_list.get(i);
+		MetadataElement metadata_element = metadata_value.getMetadataElement();
+
+		// Find the right column to put the value in
+		for (int j = 0; j < metadata_elements.size(); j++) {
+		    if (metadata_element.equals(metadata_elements.get(j))) {
+			metadata_values[j].add(metadata_value);
+			break;
+		    }
+		}
+	    }
+	}
+
+
+	public String getFilename()
+	{
+	    return file.getName();
+	}
+
+
+ 	public MetadataValueTreeNode[] getMetadataValueTreeNodes(int metadata_element_index)
+ 	{
+	    ArrayList metadata_values_list = metadata_values[metadata_element_index];
+ 	    if (metadata_values_list == null) {
+ 		return null;
+ 	    }
+
+	    MetadataValueTreeNode[] metadata_value_tree_nodes = new MetadataValueTreeNode[metadata_values_list.size()];
+	    for (int i = 0; i < metadata_values_list.size(); i++) {
+		metadata_value_tree_nodes[i] = ((MetadataValue) metadata_values_list.get(i)).getMetadataValueTreeNode();
+	    }
+
+ 	    return metadata_value_tree_nodes;
+ 	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataChangedListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataChangedListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataChangedListener.java	(revision 21112)
@@ -0,0 +1,37 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+
+
+public interface MetadataChangedListener
+{
+    /** Notification that the metadata assigned to the collection has changed. */
+    public void metadataChanged(CollectionTreeNode[] file_nodes);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataElement.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataElement.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataElement.java	(revision 21112)
@@ -0,0 +1,184 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.File;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class represents one Metadata element */
+public class MetadataElement
+{
+    // Language-independent metadata element name, with namespace (eg. dc.Subject)
+    private String metadata_element_name_full = null;
+    private Element metadata_element_element = null;
+    private MetadataValueTreeModel metadata_value_tree_model = null;
+
+
+    public MetadataElement(String metadata_element_name_full, Element metadata_element_element)
+    {
+	this.metadata_element_name_full = metadata_element_name_full;
+	this.metadata_element_element = metadata_element_element;
+	this.metadata_value_tree_model = new MetadataValueTreeModel(this);
+    }
+
+
+    public MetadataValueTreeNode addMetadataValue(String metadata_value)
+    {
+	return metadata_value_tree_model.addMetadataValue(metadata_value);
+    }
+
+
+    public String getAttribute(String attribute_name, String language_code)
+    {
+	// Some extracted metadata elements may not be in ex.mds, and so have no attributes
+	if (metadata_element_element == null) {
+	    return null;
+	}
+
+	Element parent_element = metadata_element_element;
+	boolean language_matches = false;
+
+	NodeList language_elements_nodelist = metadata_element_element.getElementsByTagName("Language");
+	for (int i = 0; i < language_elements_nodelist.getLength(); i++) {
+	    Element current_language_element = (Element) language_elements_nodelist.item(i);
+	    String current_language_element_code = current_language_element.getAttribute("code");
+	    if (current_language_element_code.equals(language_code)) {
+		// Found language; use it for the next step
+		parent_element = current_language_element;
+		language_matches = true;
+		break;
+	    }
+	}
+
+	NodeList attribute_elements_nodelist = parent_element.getElementsByTagName("Attribute");
+	for (int i = 0; i < attribute_elements_nodelist.getLength(); i++) {
+	    Element current_attribute_element = (Element) attribute_elements_nodelist.item(i);
+	    String current_attribute_element_name = current_attribute_element.getAttribute("name");
+	    if (!current_attribute_element_name.equals(attribute_name)) {
+		// Move on to the next Attribute element if the name doesn't match
+		continue;
+	    }
+
+	    // Check if the language matches
+	    String current_attribute_element_language = current_attribute_element.getAttribute("language");
+	    if (language_matches || current_attribute_element_language.equals(language_code)) {
+		// Found attribute; returns its value
+		return XMLTools.getElementTextValue(current_attribute_element);
+	    }
+	}
+
+	return null;
+    }
+
+
+    /** Returns language-dependent metadata element name, with namespace (eg. dc.Subject and Keywords) */ 
+    public String getDisplayName()
+    {
+	String metadata_element_display_name = MetadataTools.getMetadataElementAttribute(this, "identifier", Configuration.getLanguage(), "en");
+
+	// Root case (no subelements)
+	if (metadata_element_name_full.indexOf(MetadataTools.SUBELEMENT_SEPARATOR) == -1) {
+	    if (metadata_element_display_name != null) {
+		return getNamespace() + "." + metadata_element_display_name;
+	    }
+
+	    return metadata_element_name_full;
+	}
+
+	// Recursive case (hierarchical metadata elements)
+	else {
+	    String parent_metadata_element_name_full = metadata_element_name_full.substring(0, metadata_element_name_full.lastIndexOf(MetadataTools.SUBELEMENT_SEPARATOR));
+	    MetadataElement parent_metadata_element = MetadataTools.getMetadataElementWithName(parent_metadata_element_name_full);
+
+	    String parent_metadata_element_display_name = parent_metadata_element_name_full;
+	    if (parent_metadata_element != null) {
+		parent_metadata_element_display_name = parent_metadata_element.getDisplayName();
+	    }
+
+	    if (metadata_element_display_name != null) {
+		return parent_metadata_element_display_name + MetadataTools.SUBELEMENT_SEPARATOR + metadata_element_display_name;
+	    }
+
+	    return parent_metadata_element_display_name + metadata_element_name_full.substring(parent_metadata_element_name_full.length());
+	}
+    }
+
+
+    public MetadataValueTreeModel getMetadataValueTreeModel()
+    {
+	return metadata_value_tree_model;
+    }
+
+
+    public MetadataValueTreeNode getMetadataValueTreeNode(String metadata_value)
+    {
+	return metadata_value_tree_model.getMetadataValueTreeNode(metadata_value);
+    }
+
+
+    /** Returns language-independent metadata element name, with namespace (eg. dc.Subject) */
+    public String getFullName()
+    {
+	return metadata_element_name_full;
+    }
+
+
+    public String getNamespace()
+    {
+	return MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+    }
+
+
+    /** Returns language-independent metadata element name, *without* namespace (eg. Subject) */
+    public String getName()
+    {
+	return MetadataTools.getMetadataElementName(metadata_element_name_full);
+    }
+
+
+    public boolean isExtractedMetadataElement()
+    {
+	return getNamespace().equals(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+    }
+
+
+    public boolean isSubelement()
+    {
+	return (metadata_element_name_full.indexOf(MetadataTools.SUBELEMENT_SEPARATOR) != -1);
+    }
+
+
+    /** This is used to display the element in the metadata value table */
+    public String toString()
+    {
+	return getDisplayName();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSet.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSet.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSet.java	(revision 21112)
@@ -0,0 +1,218 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.File;
+import java.util.*;
+import javax.swing.filechooser.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class represents one metadata set */
+public class MetadataSet
+{
+    private File metadata_set_file = null;
+    private ArrayList metadata_set_elements = new ArrayList();
+    private String metadata_set_namespace = null;
+
+
+    public MetadataSet(File metadata_set_file)
+    {
+	this.metadata_set_file = metadata_set_file;
+
+	// Parse the specified metadata set file
+	Document document = XMLTools.parseXMLFile(metadata_set_file);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata set file " + metadata_set_file.getAbsolutePath());
+	    return;
+	}
+
+	// Read the metadata set's namespace
+	metadata_set_namespace = document.getDocumentElement().getAttribute("namespace");
+
+	// Read the Element elements in the metadata set
+	NodeList element_elements_nodelist = document.getElementsByTagName("Element");
+	for (int i = 0; i < element_elements_nodelist.getLength(); i++) {
+	    Element metadata_element_element = (Element) element_elements_nodelist.item(i);
+	    String metadata_element_name = metadata_element_element.getAttribute("name");
+
+	    // Deal with hierarchical metadata elements
+	    Element parent_metadata_element_element = (Element) metadata_element_element.getParentNode();
+	    while (parent_metadata_element_element.getTagName().equals("Element")) {
+		String parent_metadata_element_name = parent_metadata_element_element.getAttribute("name");
+		metadata_element_name = parent_metadata_element_name + MetadataTools.SUBELEMENT_SEPARATOR + metadata_element_name;
+		parent_metadata_element_element = (Element) parent_metadata_element_element.getParentNode();
+	    }
+
+	    String metadata_element_name_full = metadata_set_namespace + MetadataTools.NAMESPACE_SEPARATOR + metadata_element_name;
+	    metadata_set_elements.add(new MetadataElement(metadata_element_name_full, metadata_element_element));
+	}
+    }
+
+
+    public MetadataElement addMetadataElementForThisSession(String metadata_element_name)
+    {
+	String metadata_element_name_full = metadata_set_namespace + MetadataTools.NAMESPACE_SEPARATOR + metadata_element_name;
+	MetadataElement metadata_element = new MetadataElement(metadata_element_name_full, null);
+	metadata_set_elements.add(metadata_element);
+	return metadata_element;
+    }
+
+
+    public int compareMetadataElements(MetadataElement metadata_element_a, MetadataElement metadata_element_b)
+    {
+	// Extracted metadata is sorted alphabetically
+	if (metadata_set_namespace.equals(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE)) {
+	    return metadata_element_a.getName().compareTo(metadata_element_b.getName());
+	}
+
+	// Elements in metadata sets are sorted according to the other they are specified in the .mds file
+	return metadata_set_elements.indexOf(metadata_element_a) - metadata_set_elements.indexOf(metadata_element_b);
+    }
+
+
+    public String getAttribute(String attribute_name, String language_code)
+    {
+	// Parse the specified metadata set file
+	Document document = XMLTools.parseXMLFile(metadata_set_file);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata set file " + metadata_set_file.getAbsolutePath());
+	    return null;
+	}
+
+	Element document_element = document.getDocumentElement();
+	Element parent_element = document_element;
+	boolean language_matches = false;
+
+	NodeList setlanguage_elements_nodelist = document_element.getElementsByTagName("SetLanguage");
+	for (int i = 0; i < setlanguage_elements_nodelist.getLength(); i++) {
+	    Element current_setlanguage_element = (Element) setlanguage_elements_nodelist.item(i);
+	    String current_setlanguage_element_code = current_setlanguage_element.getAttribute("code");
+	    if (current_setlanguage_element_code.equals(language_code)) {
+		// Found language; use it for the next step
+		parent_element = current_setlanguage_element;
+		language_matches = true;
+		break;
+	    }
+	}
+
+	NodeList attribute_elements_nodelist = parent_element.getElementsByTagName(attribute_name);
+	for (int i = 0; i < attribute_elements_nodelist.getLength(); i++) {
+	    Element current_attribute_element = (Element) attribute_elements_nodelist.item(i);
+
+	    // Check if the language matches
+	    String current_attribute_element_language = current_attribute_element.getAttribute("language");
+	    if (language_matches || current_attribute_element_language.equals(language_code)) {
+		// Found attribute; returns its value
+		return XMLTools.getElementTextValue(current_attribute_element);
+	    }
+	}
+
+	return null;
+    }
+
+
+    public MetadataElement getMetadataElementWithDisplayName(String metadata_element_display_name_full)
+    {
+	// Find the desired metadata element in the list
+	for (int i = 0; i < metadata_set_elements.size(); i++) {
+	    MetadataElement metadata_element = (MetadataElement) metadata_set_elements.get(i);
+	    if (metadata_element.getDisplayName().equals(metadata_element_display_name_full)) {
+		return metadata_element;
+	    }
+	}
+
+	// Not found
+	return null;
+    }
+
+
+    public MetadataElement getMetadataElementWithName(String metadata_element_name)
+    {
+	// Find the desired metadata element in the list
+	for (int i = 0; i < metadata_set_elements.size(); i++) {
+	    MetadataElement metadata_element = (MetadataElement) metadata_set_elements.get(i);
+	    if (metadata_element.getName().equals(metadata_element_name)) {
+		return metadata_element;
+	    }
+	}
+
+	// Not found
+	return null;
+    }
+
+
+    public ArrayList getMetadataSetElements()
+    {
+	// We don't want anyone to modify the list, so return a copy of it
+	return (ArrayList) metadata_set_elements.clone();
+    }
+
+
+    public File getMetadataSetFile()
+    {
+	return metadata_set_file;
+    }
+
+
+    public String getNamespace()
+    {
+	return metadata_set_namespace;
+    }
+
+
+    /** This is used to display the metadata set in the metadata import mapping and explode metadata prompts */
+    public String toString()
+    {
+	return MetadataTools.getMetadataSetAttribute(this, "Name", Configuration.getLanguage(), "en") + " (" + metadata_set_namespace + ")";
+    }
+
+
+    static public class MetadataSetFileFilter
+	extends FileFilter
+    {
+	public boolean accept(File file)
+	{
+	    String file_name = file.getName().toLowerCase();
+	    if (file_name.endsWith(".mds") || file_name.indexOf(".") == -1) {
+		return true;
+	    }
+
+	    return false;
+	}
+
+
+	public String getDescription()
+	{
+	    return Dictionary.get("MetadataSet.Files");
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSetManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSetManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataSetManager.java	(revision 21112)
@@ -0,0 +1,207 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.gui.MetadataImportMappingPrompt;
+
+
+/** This class is a static class that manages the metadata sets */
+public class MetadataSetManager
+{
+    static public final String EXTRACTED_METADATA_NAMESPACE = "ex";
+
+    static private ArrayList metadata_sets = new ArrayList();
+
+
+    static public void clearMetadataSets()
+    {
+	metadata_sets.clear();
+    }
+
+
+    static public int compareMetadataElements(MetadataElement metadata_element_a, MetadataElement metadata_element_b)
+    {
+	// First compare the namespaces of the two metadata elements
+	if (!metadata_element_a.getNamespace().equals(metadata_element_b.getNamespace())) {
+	    return metadata_element_a.getNamespace().compareTo(metadata_element_b.getNamespace());
+	}
+
+	// Otherwise compare the two elements within the one set
+	MetadataSet metadata_set = getMetadataSet(metadata_element_a.getNamespace());
+	return metadata_set.compareMetadataElements(metadata_element_a, metadata_element_b);
+    }
+
+
+    static public ArrayList getEveryMetadataSetElement()
+    {
+	ArrayList every_metadata_set_element = new ArrayList();
+
+	for (int i = 0; i < metadata_sets.size(); i++) {
+	    MetadataSet metadata_set = (MetadataSet) metadata_sets.get(i);
+	    every_metadata_set_element.addAll(metadata_set.getMetadataSetElements());
+	}
+
+	return every_metadata_set_element;
+    }
+
+
+    static public MetadataSet getMetadataSet(String metadata_set_namespace)
+    {
+	// Find the desired metadata set in the list
+	for (int i = 0; i < metadata_sets.size(); i++) {
+	    MetadataSet metadata_set = (MetadataSet) metadata_sets.get(i);
+	    if (metadata_set.getNamespace().equals(metadata_set_namespace)) {
+		return metadata_set;
+	    }
+	}
+
+	// Not found
+	return null;
+    }
+
+
+    static public ArrayList getMetadataSets()
+    {
+	// We don't want anyone to modify the list, so return a copy of it
+	return (ArrayList) metadata_sets.clone();
+    }
+
+
+    static public ArrayList listMetadataSets(File directory)
+    {
+	// Make sure the directory (metadata) exists
+	if (directory.exists() == false) {
+	    return null;
+	}
+
+	// Load just those .mds files in this directory, and return them
+	ArrayList directory_metadata_sets = new ArrayList();
+	File[] directory_files = directory.listFiles();
+	for (int i = 0; i < directory_files.length; i++) {
+	    File child_file = directory_files[i];
+	    if (!child_file.isDirectory() && child_file.getName().endsWith(".mds")) {
+		MetadataSet metadata_set = new MetadataSet(child_file);
+		directory_metadata_sets.add(metadata_set);
+	    }
+	}
+
+	return directory_metadata_sets;
+    }
+
+
+    static public void loadMetadataSets(File directory)
+    {
+	// Make sure the directory (metadata) exists
+	if (directory.exists() == false) {
+	    return;
+	}
+
+	// Look recursively at each subfile of the directory for .mds files
+	File[] directory_files = directory.listFiles();
+	for (int i = 0; i < directory_files.length; i++) {
+	    File child_file = directory_files[i];
+	    if (child_file.isDirectory()) {
+		loadMetadataSets(child_file);
+	    }
+	    else if (child_file.getName().endsWith(".mds")) {
+		loadMetadataSet(child_file);
+	    }
+	}
+    }
+
+
+    static public void loadMetadataSet(File metadata_set_file)
+    {
+	MetadataSet metadata_set = new MetadataSet(metadata_set_file);
+
+	// Ignore legacy crap
+	if (metadata_set.getNamespace().equals("hidden")) {
+	    return;
+	}
+
+	metadata_sets.add(metadata_set);
+    }
+
+
+    static public String mapUnloadedMetadataElement(String metadata_element_name_full)
+    {
+	// Check if we have an import mapping for this metadata element
+	String target_metadata_element_name_full = ProfileXMLFileManager.getMetadataElementFor(metadata_element_name_full);
+	if (target_metadata_element_name_full != null) {
+	    // Yes, so return it
+	    return target_metadata_element_name_full;
+	}
+
+	// If there are no metadata sets (except extracted) loaded then there is nothing to map the element into
+	if (metadata_sets.size() <= 1) {
+	    return null;
+	}
+
+	// Ask the user how they want to deal with this element
+	MetadataImportMappingPrompt metadata_import_mapping_prompt = new MetadataImportMappingPrompt(metadata_element_name_full);
+	int result = metadata_import_mapping_prompt.getResult();
+
+	// Add the element into an existing metadata set
+	if (result == MetadataImportMappingPrompt.ADD_BUTTON_PRESSED) {
+	    MetadataSet target_metadata_set = metadata_import_mapping_prompt.getSelectedMetadataSet();
+	    String metadata_element_name = MetadataTools.getMetadataElementName(metadata_element_name_full);
+	    target_metadata_set.addMetadataElementForThisSession(metadata_element_name);
+	    target_metadata_element_name_full = target_metadata_set.getNamespace() + "." + metadata_element_name;
+	}
+
+	// Replace the element with an element in an existing metadata set
+	if (result == MetadataImportMappingPrompt.MERGE_BUTTON_PRESSED) {
+	    MetadataElement target_metadata_element = metadata_import_mapping_prompt.getSelectedMetadataElement();
+	    target_metadata_element_name_full = target_metadata_element.getFullName();
+	}
+
+	// Ignore the element
+	if (result == MetadataImportMappingPrompt.IGNORE_BUTTON_PRESSED) {
+	    target_metadata_element_name_full = "";
+	}
+
+	// Store this import mapping for future elements with the same name
+	ProfileXMLFileManager.mapElement(metadata_element_name_full, target_metadata_element_name_full);
+
+	return target_metadata_element_name_full;
+    }
+
+
+    static public void unloadMetadataSet(MetadataSet metadata_set)
+    {
+	// Find the metadata set in the list of loaded sets, and remove it
+	for (int i = 0; i < metadata_sets.size(); i++) {
+	    if (metadata_set == (MetadataSet) metadata_sets.get(i)) {
+		metadata_sets.remove(i);
+		break;
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataTools.java	(revision 21112)
@@ -0,0 +1,137 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+/** This class is a static class containing useful metadata functions */
+public class MetadataTools
+{
+    static final public String NAMESPACE_SEPARATOR = ".";
+    static final public String SUBELEMENT_SEPARATOR = "^";
+
+
+    static public String getDisplayNameForMetadataElementWithName(String metadata_element_name_full)
+    {
+	MetadataElement metadata_element = getMetadataElementWithName(metadata_element_name_full);
+	if (metadata_element != null) {
+	    return metadata_element.getDisplayName();
+	}
+
+	return metadata_element_name_full;
+    }
+
+
+    static public String getMetadataElementAttribute(MetadataElement metadata_element, String attribute_name, String language_code, String fallback_language_code)
+    {
+	String metadata_element_attribute = metadata_element.getAttribute(attribute_name, language_code);
+
+	// If the attribute isn't defined in the desired language, resort to the fallback
+	if (metadata_element_attribute == null && !language_code.equals(fallback_language_code)) {
+	    metadata_element_attribute = metadata_element.getAttribute(attribute_name, fallback_language_code);
+	}
+
+	return metadata_element_attribute;
+    }
+
+
+    static public String getMetadataElementName(String metadata_element_name_full)
+    {
+	// Full element name contains namespace
+	if (metadata_element_name_full.indexOf(NAMESPACE_SEPARATOR) != -1) {
+	    return metadata_element_name_full.substring(metadata_element_name_full.indexOf(NAMESPACE_SEPARATOR) + 1);
+	}
+
+	// No namespace
+	return metadata_element_name_full;      
+    }
+
+
+    static public MetadataElement getMetadataElementWithDisplayName(String metadata_element_display_name_full)
+    {
+	String metadata_set_namespace = getMetadataSetNamespace(metadata_element_display_name_full);
+	MetadataSet metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+	if (metadata_set == null) {
+	    return null;
+	}
+
+	return metadata_set.getMetadataElementWithDisplayName(metadata_element_display_name_full);
+    }
+
+
+    static public MetadataElement getMetadataElementWithName(String metadata_element_name_full)
+    {
+	String metadata_set_namespace = getMetadataSetNamespace(metadata_element_name_full);
+	MetadataSet metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+	if (metadata_set == null) {
+	    return null;
+	}
+
+	String metadata_element_name = getMetadataElementName(metadata_element_name_full);
+	return metadata_set.getMetadataElementWithName(metadata_element_name);
+    }
+
+
+    static public String getMetadataSetAttribute(MetadataSet metadata_set, String attribute_name, String language_code, String fallback_language_code)
+    {
+	String metadata_set_attribute = metadata_set.getAttribute(attribute_name, language_code);
+
+	// If the attribute isn't defined in the desired language, resort to the fallback
+	if (metadata_set_attribute == null && !language_code.equals(fallback_language_code)) {
+	    metadata_set_attribute = metadata_set.getAttribute(attribute_name, fallback_language_code);
+	}
+
+	return metadata_set_attribute;
+    }
+
+
+    static public String getMetadataSetNamespace(String metadata_element_name_full)
+    {
+	// Full element name contains namespace
+	if (metadata_element_name_full.indexOf(NAMESPACE_SEPARATOR) != -1) {
+	    return metadata_element_name_full.substring(0, metadata_element_name_full.indexOf(NAMESPACE_SEPARATOR));
+	}
+
+	// No namespace
+	return "";
+    }
+
+
+    static public String getRegularExpressionThatMatchesFilePath(String file_path)
+    {
+	// Convert the file path into a regular expression that will match it
+	String file_path_regexp = file_path;
+	file_path_regexp = file_path_regexp.replaceAll("\\.", "\\\\.");
+	file_path_regexp = file_path_regexp.replaceAll("\\(", "\\\\(");
+	file_path_regexp = file_path_regexp.replaceAll("\\)", "\\\\)");
+	file_path_regexp = file_path_regexp.replaceAll("\\[", "\\\\[");
+	file_path_regexp = file_path_regexp.replaceAll("\\]", "\\\\]");
+	file_path_regexp = file_path_regexp.replaceAll("\\{", "\\\\{");
+	file_path_regexp = file_path_regexp.replaceAll("\\}", "\\\\}");
+	file_path_regexp = file_path_regexp.replaceAll("\\+", "\\\\+");
+	return file_path_regexp;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValue.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValue.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValue.java	(revision 21112)
@@ -0,0 +1,118 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+
+
+/** This class represents one metadata (element, value) pair */
+public class MetadataValue
+    implements Comparable
+{
+    /** The folder that this piece of metadata was inherited from, if applicable */
+    private File folder_metadata_inherited_from = null;
+    private MetadataElement metadata_element = null;
+    private MetadataValueTreeNode metadata_value_tree_node = null;
+    private boolean is_accumulating_metadata = false;
+
+
+    public MetadataValue(MetadataElement metadata_element, MetadataValueTreeNode metadata_value_tree_node)
+    {
+	this.metadata_element = metadata_element;
+	this.metadata_value_tree_node = metadata_value_tree_node;
+    }
+
+
+    public int compareTo(Object metadata_value_object)
+    {
+	MetadataValue metadata_value = (MetadataValue) metadata_value_object;
+
+	// First compare the metadata elements
+	int c = MetadataSetManager.compareMetadataElements(this.getMetadataElement(), metadata_value.getMetadataElement());
+	if (c != 0) {
+	    return c;
+	}
+
+	// Then use the metadata values
+	return this.getFullValue().compareTo(metadata_value.getFullValue());
+    }
+
+
+    public File getFolderMetadataInheritedFrom()
+    {
+	return folder_metadata_inherited_from;
+    }
+
+
+    public MetadataElement getMetadataElement()
+    {
+	return metadata_element;
+    }
+
+
+    public String getFullValue()
+    {
+	return metadata_value_tree_node.getFullValue();
+    }
+
+
+    public String getValue()
+    {
+	return metadata_value_tree_node.getValue();
+    }
+
+
+    public MetadataValueTreeNode getMetadataValueTreeNode()
+    {
+	return metadata_value_tree_node;
+    }
+
+
+    public void inheritsMetadataFromFolder(File folder_metadata_inherited_from)
+    {
+	this.folder_metadata_inherited_from = folder_metadata_inherited_from;
+    }
+
+
+    public boolean isAccumulatingMetadata()
+    {
+	return is_accumulating_metadata;
+    }
+
+
+    public boolean isInheritedMetadata()
+    {
+	return (folder_metadata_inherited_from != null);
+    }
+
+
+    public void setIsAccumulatingMetadata(boolean is_accumulating_metadata)
+    {
+	this.is_accumulating_metadata = is_accumulating_metadata;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableEntry.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableEntry.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableEntry.java	(revision 21112)
@@ -0,0 +1,68 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+/** This class represents one entry in the metadata value table. */
+public class MetadataValueTableEntry
+    extends MetadataValue
+{
+    private int occurrences = 0;
+
+
+    public MetadataValueTableEntry(MetadataElement metadata_element, MetadataValueTreeNode metadata_value_tree_node)
+    {
+	super(metadata_element, metadata_value_tree_node);
+
+	occurrences = 1;
+    }
+
+
+    public MetadataValueTableEntry(MetadataValue metadata_value)
+    {
+	this(metadata_value.getMetadataElement(), metadata_value.getMetadataValueTreeNode());
+	inheritsMetadataFromFolder(metadata_value.getFolderMetadataInheritedFrom());
+    }
+
+
+    public void anotherOccurrence()
+    {
+	occurrences++;
+    }
+
+
+    public int getOccurrences()
+    {
+	return occurrences;
+    }
+
+
+    public void setOccurrences(int occurrences)
+    {
+	this.occurrences = occurrences;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableModel.java	(revision 21112)
@@ -0,0 +1,463 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.table.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.gui.WarningDialog;
+
+
+public class MetadataValueTableModel
+    extends AbstractTableModel
+{
+    /** The CollectionTreeNodes this model is built for */
+    private CollectionTreeNode[] file_nodes = null;
+    /** The list of MetadataValueTableEntries in the table */
+    private ArrayList metadata_value_table_entries = new ArrayList();
+
+    static final private String[] COLUMN_NAMES = { "", Dictionary.get("Metadata.Element"), Dictionary.get("Metadata.Value") };
+
+
+    public int addBlankRowForMetadataElement(MetadataElement metadata_element)
+    {
+	MetadataValueTableModelBuilder metadata_value_table_model_builder = new MetadataValueTableModelBuilder();
+	MetadataValue blank_metadata_value = new MetadataValue(metadata_element, new MetadataValueTreeNode(""));
+	return metadata_value_table_model_builder.insertMetadataValue(blank_metadata_value);
+    }
+
+
+    public int findMetadataValueTableEntryToSelect(MetadataValueTableEntry metadata_value_table_entry)
+    {
+	MetadataElement metadata_element = metadata_value_table_entry.getMetadataElement();
+
+	// Find the correct entry to select
+	for (int i = 0; i < metadata_value_table_entries.size(); i++) {
+	    MetadataValueTableEntry current_metadata_value_table_entry = (MetadataValueTableEntry) metadata_value_table_entries.get(i);
+	    int element_comparison = MetadataSetManager.compareMetadataElements(current_metadata_value_table_entry.getMetadataElement(), metadata_element);
+
+	    // We've found the right element, so check if the value already exists
+	    if (element_comparison == 0) {
+		int value_comparison = current_metadata_value_table_entry.compareTo(metadata_value_table_entry);
+		if (value_comparison == 0) {
+		    // Entry found!
+		    return i;
+		}
+	    }
+
+	    // We've just gone past the correct entry to select
+	    if (element_comparison > 0) {
+		return i - 1;
+	    }
+	}
+
+	// Have to select the last entry
+	return metadata_value_table_entries.size() - 1;
+    }
+
+
+    public Class getColumnClass(int col)
+    {
+	return getValueAt(0, col).getClass();
+    }
+
+
+    /** Returns the number of columns in this table. */
+    public int getColumnCount()
+    {
+	return COLUMN_NAMES.length;
+    }
+
+
+    /** Retrieves the name of the specified column. */
+    public String getColumnName(int col)
+    {
+	return COLUMN_NAMES[col];
+    }
+
+
+    /* Called to retrieve the MetadataValueTableEntry at a certain row. Usually caused by the user selecting a row in the table. It is synchronized so that the model doesn't up and change while we're trying to retrieve the indicated element. */
+    public synchronized MetadataValueTableEntry getMetadataValueTableEntry(int row)
+    {
+	if (row >= 0 && row < metadata_value_table_entries.size()) {
+	    return (MetadataValueTableEntry) metadata_value_table_entries.get(row);
+	}
+
+	return null;
+    }
+
+
+    /** Returns the number of rows in this table. */
+    public int getRowCount()
+    {
+	return metadata_value_table_entries.size();
+    }
+
+
+    /** Returns the cell value at a given row and column as an Object. */
+    public Object getValueAt(int row, int col)
+    {
+	// Check values are reasonable
+	if (row < 0 || row >= metadata_value_table_entries.size() || col < 0 || col >= COLUMN_NAMES.length) {
+	    return null;
+	}
+
+	MetadataValueTableEntry metadata_value_table_entry = (MetadataValueTableEntry) metadata_value_table_entries.get(row);
+	if (col == 0 && metadata_value_table_entry.isInheritedMetadata()) {
+	    return metadata_value_table_entry.getFolderMetadataInheritedFrom();
+	}
+
+	if (col == 1) {
+	    return metadata_value_table_entry.getMetadataElement();
+	}
+
+	if (col == 2) {
+	    return metadata_value_table_entry.getFullValue();
+	}
+
+	return null;
+    }
+
+
+    public boolean isCellEditable(int row, int col)
+    {
+	// The inherited and element columns are never editable
+	if (col < 2) {
+	    return false;
+	}
+
+	// Extracted and inherited metadata is not editable
+	MetadataValueTableEntry metadata_value_table_entry = (MetadataValueTableEntry) metadata_value_table_entries.get(row);
+	if (metadata_value_table_entry.getMetadataElement().isExtractedMetadataElement() || metadata_value_table_entry.isInheritedMetadata()) {
+	    return false;
+	}
+
+	return true;
+    }
+
+
+    /** Determine if the given metadata is common to all selected file nodes. */
+    public boolean isCommon(MetadataValueTableEntry metadata_value_table_entry)
+    {
+	return (file_nodes != null && metadata_value_table_entry.getOccurrences() == file_nodes.length);
+    }
+
+
+    /** Determine if the given metadata is common to all selected file nodes. */
+    public boolean isCommon(int row)
+    {
+	if (row >= 0 && row < metadata_value_table_entries.size()) {
+	    return isCommon((MetadataValueTableEntry) metadata_value_table_entries.get(row));
+	}
+
+	return false;
+    }
+
+
+    public void rebuild(CollectionTreeNode[] file_nodes)
+    {
+	this.file_nodes = file_nodes;
+	metadata_value_table_entries.clear();
+
+	// Collection is in a state of flux
+	if (!Gatherer.c_man.ready()) {
+	    return;
+	}
+
+	if (file_nodes == null || file_nodes.length == 0) {
+	    return;
+	}
+
+	// Create model builder
+	MetadataValueTableModelBuilder builder = new MetadataValueTableModelBuilder();
+	builder.run();
+    }
+
+
+    public void setValueAt(Object new_metadata_value, int row, int col)
+    {
+	MetadataValueTableEntry metadata_value_table_entry = getMetadataValueTableEntry(row);
+
+	// If nothing has changed no action is necessary
+	String old_metadata_value = metadata_value_table_entry.getFullValue();
+	if (new_metadata_value.equals(old_metadata_value)) {
+	    return;
+	}
+
+	// Lock the interface so nothing can be changed while the metadata edit is being processed
+	Gatherer.g_man.wait(true);
+
+	// Metadata value added
+	if (old_metadata_value.equals("") && !new_metadata_value.equals("")) {
+ 	    // If we're adding metadata to folders display the warning
+ 	    if (!file_nodes[0].isLeaf()) {
+ 		WarningDialog dialog = new WarningDialog("warning.DirectoryLevelMetadata", "DirectoryLevelMetadata.Title", Dictionary.get("DirectoryLevelMetadata.Message"), null, true);
+ 		int dialog_result = dialog.display();
+ 		dialog.dispose();
+ 		if (dialog_result != JOptionPane.OK_OPTION) {
+		    Gatherer.g_man.wait(false);
+ 		    return;
+ 		}
+ 	    }
+
+	    MetadataElement metadata_element = metadata_value_table_entry.getMetadataElement();
+	    MetadataValueTreeNode metadata_value_tree_node = metadata_element.addMetadataValue((String) new_metadata_value);
+	    MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+	    metadata_value.setIsAccumulatingMetadata(true);
+	    (new AppendMetadataTask(metadata_value)).run();
+	}
+
+	// Metadata value removed
+	else if (!old_metadata_value.equals("") && new_metadata_value.equals("")) {
+	    (new RemoveMetadataTask(metadata_value_table_entry)).run();
+	}
+
+	// Metadata value replaced
+	else {
+	    MetadataElement metadata_element = metadata_value_table_entry.getMetadataElement();
+	    MetadataValueTreeNode metadata_value_tree_node = metadata_element.addMetadataValue((String) new_metadata_value);
+	    MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+	    metadata_value.setIsAccumulatingMetadata(!metadata_value_table_entry.isInheritedMetadata());
+	    (new ReplaceMetadataTask(metadata_value_table_entry, metadata_value)).run();
+	}
+    }
+
+
+    private class AppendMetadataTask
+	// extends Thread
+    {
+	private MetadataValue metadata_value = null;
+
+	private AppendMetadataTask(MetadataValue metadata_value)
+	{
+	    this.metadata_value = metadata_value;
+	}
+
+	public void run()
+	{
+	    try {
+		// Edit metadata.xml files to add the metadata
+		MetadataXMLFileManager.addMetadata(file_nodes, metadata_value);
+	    }
+	    catch (Exception exception) {
+		// We need to catch any exceptions here so the interface is unlocked below
+		DebugStream.printStackTrace(exception);
+	    }
+
+	    // Operation finished, so turn busy cursor off and unlock interface
+	    Gatherer.g_man.wait(false);
+	}
+    }
+
+
+    private class ReplaceMetadataTask
+	// extends Thread
+    {
+	private MetadataValueTableEntry selected_metadata_value_table_entry = null;
+	private MetadataValue metadata_value = null;
+
+	private ReplaceMetadataTask(MetadataValueTableEntry selected_metadata_value_table_entry, MetadataValue metadata_value)
+	{
+	    this.selected_metadata_value_table_entry = selected_metadata_value_table_entry;
+	    this.metadata_value = metadata_value;
+	}
+
+	public void run()
+	{
+	    try {
+		// Edit metadata.xml files to replace the metadata
+		MetadataXMLFileManager.replaceMetadata(file_nodes, selected_metadata_value_table_entry, metadata_value);
+	    }
+	    catch (Exception exception) {
+		// We need to catch any exceptions here so the interface is unlocked below
+		DebugStream.printStackTrace(exception);
+	    }
+
+	    // Operation finished, so turn busy cursor off and unlock interface
+	    Gatherer.g_man.wait(false);
+	}
+    }
+
+
+    private class RemoveMetadataTask
+	// extends Thread
+    {
+	private MetadataValueTableEntry selected_metadata_value_table_entry = null;
+
+	private RemoveMetadataTask(MetadataValueTableEntry selected_metadata_value_table_entry)
+	{
+	    this.selected_metadata_value_table_entry = selected_metadata_value_table_entry;
+	}
+
+	public void run()
+	{
+	    try {
+		// Edit metadata.xml files to remove the metadata
+		MetadataXMLFileManager.removeMetadata(file_nodes, selected_metadata_value_table_entry);
+	    }
+	    catch (Exception exception) {
+		// We need to catch any exceptions here so the interface is unlocked below
+		DebugStream.printStackTrace(exception);
+	    }
+
+	    // Operation finished, so turn busy cursor off and unlock interface
+	    Gatherer.g_man.wait(false);
+	}
+    }
+
+
+    private class MetadataValueTableModelBuilder
+    {
+	public void run()
+	{
+	    // System.err.println("Building MetadataValueTableModel...");
+
+	    // Build a list of MetadataValueTableEntries that represent the metadata asssigned to the selected files
+	    boolean hid_extracted_metadata = false;
+	    ArrayList metadata_elements_seen = new ArrayList();
+
+	    // Process each of the selected files in turn
+	    for (int i = 0; i < file_nodes.length; i++) {
+		File current_file = file_nodes[i].getFile();
+
+		// Get the metadata assigned to this file
+		ArrayList assigned_metadata = MetadataXMLFileManager.getMetadataAssignedToFile(current_file);
+		for (int j = 0; j < assigned_metadata.size(); j++) {
+		    MetadataValue metadata_value = (MetadataValue) assigned_metadata.get(j);
+		    MetadataElement metadata_element = metadata_value.getMetadataElement();
+
+		    // Insert this metadata value into the table, unless it already exists (in which case increment its count)
+		    insertMetadataValue(metadata_value);
+
+		    // Remember we have seen this metadata element
+		    if (metadata_elements_seen.contains(metadata_element) == false) {
+			metadata_elements_seen.add(metadata_element);
+		    }
+		}
+
+		// Get the extracted metadata for this file, if desired
+		if (Configuration.get("general.view_extracted_metadata", Configuration.COLLECTION_SPECIFIC) == true) {
+		    ArrayList extracted_metadata = DocXMLFileManager.getMetadataExtractedFromFile(current_file);
+		    for (int k = 0; k < extracted_metadata.size(); k++) {
+			MetadataValue metadata_value = (MetadataValue) extracted_metadata.get(k);
+
+			// Insert this metadata value into the table, unless it already exists (in which case increment its count)
+			insertMetadataValue(metadata_value);
+		    }
+		}
+	    }
+
+	    // Make sure each non-extracted metadata element appears in the table (even if blank)
+	    ArrayList every_metadata_set_element = MetadataSetManager.getEveryMetadataSetElement();
+	    for (int i = 0; i < every_metadata_set_element.size(); i++) {
+		MetadataElement metadata_element = (MetadataElement) every_metadata_set_element.get(i);
+
+		// If we haven't seen this metadata element and it isn't extracted, add it now
+		if (!metadata_elements_seen.contains(metadata_element) && !metadata_element.isExtractedMetadataElement()) {
+		    MetadataValueTableEntry metadata_value_table_entry = new MetadataValueTableEntry(metadata_element, new MetadataValueTreeNode(""));
+
+		    // Blank metadata is common to all selected files (otherwise it wouldn't be blank!)
+		    metadata_value_table_entry.setOccurrences(file_nodes.length);
+
+		    // Add it to the table
+		    insertMetadataValueTableEntry(metadata_value_table_entry);
+		}
+	    }
+
+	    // If extracted metadata was hidden, display the warning
+	    if (hid_extracted_metadata) {
+		showExtractedMetadataWarning();
+	    }
+	}
+
+
+	/** Inserts the new metadata value into the table */
+	private int insertMetadataValue(MetadataValue metadata_value)
+	{
+	    return insertMetadataValueTableEntry(new MetadataValueTableEntry(metadata_value));
+	}
+
+
+	/** Inserts the new metadata value table entry into the table */
+	private int insertMetadataValueTableEntry(MetadataValueTableEntry metadata_value_table_entry)
+	{
+	    MetadataElement metadata_element = metadata_value_table_entry.getMetadataElement();
+
+	    // Find the correct place to insert the table entry
+	    for (int i = 0; i < metadata_value_table_entries.size(); i++) {
+		MetadataValueTableEntry current_metadata_value_table_entry = (MetadataValueTableEntry) metadata_value_table_entries.get(i);
+		int element_comparison = MetadataSetManager.compareMetadataElements(current_metadata_value_table_entry.getMetadataElement(), metadata_element);
+
+		// We've found the right element, so check if the value already exists
+		if (element_comparison == 0) {
+		    int value_comparison = current_metadata_value_table_entry.compareTo(metadata_value_table_entry);
+		    if (value_comparison == 0) {
+			// Entry already exists, so increment count (except for blank entries)
+			if (!metadata_value_table_entry.getFullValue().equals("")) {
+			    current_metadata_value_table_entry.anotherOccurrence();
+			}
+			return i;
+		    }
+		}
+
+		// Found insertion point
+		if (element_comparison > 0) {
+		    metadata_value_table_entries.add(i, metadata_value_table_entry);
+		    fireTableRowsInserted(i, i);
+		    return i;
+		}
+	    }
+
+	    // Must go at the end of the table
+	    metadata_value_table_entries.add(metadata_value_table_entry);
+	    fireTableRowsInserted(metadata_value_table_entries.size() - 1, metadata_value_table_entries.size() - 1);
+	    return metadata_value_table_entries.size() - 1;
+	}
+
+
+	private void showExtractedMetadataWarning()
+	{
+	    Runnable task = new Runnable() {
+		    public void run() {
+			WarningDialog dialog = new WarningDialog("warning.ExtractedMetadata", "ExtractedMetadata.Title", Dictionary.get("ExtractedMetadata.Message"), null, false);
+			dialog.display();
+			dialog.dispose();
+			dialog = null;
+		    }
+		};
+	    SwingUtilities.invokeLater(task);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeModel.java	(revision 21112)
@@ -0,0 +1,137 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import javax.swing.tree.*;
+
+
+/** This class represents all the metadata values for one metadata element */
+public class MetadataValueTreeModel
+    extends DefaultTreeModel
+{
+    public MetadataValueTreeModel(MetadataElement metadata_element)
+    {
+	// The root node of the value tree is the MetadataElement it represents
+	super(new MetadataValueTreeNode(metadata_element));
+    }
+
+
+    public MetadataValueTreeNode addMetadataValue(String metadata_value)
+    {
+	return addMetadataValueInternal((MetadataValueTreeNode) root, metadata_value);
+    }
+
+
+    private MetadataValueTreeNode addMetadataValueInternal(MetadataValueTreeNode parent_metadata_value_tree_node, String metadata_value_remaining)
+    {
+	// Split the metadata value into a hierarchy
+	int split_point = metadata_value_remaining.indexOf(MetadataValueTreeNode.METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN);
+	String metadata_value = ((split_point == -1) ? metadata_value_remaining : metadata_value_remaining.substring(0, split_point));
+	metadata_value_remaining = ((split_point == -1) ? "" : metadata_value_remaining.substring(split_point + 1));
+
+	// Add the value into the tree in the correct place (sorted)
+	MetadataValueTreeNode metadata_value_tree_node = null;
+	for (int i = 0; i < parent_metadata_value_tree_node.getChildCount(); i++) {
+	    MetadataValueTreeNode child_metadata_value_tree_node = (MetadataValueTreeNode) parent_metadata_value_tree_node.getChildAt(i);
+	    int c = metadata_value.compareTo(child_metadata_value_tree_node.getValue());
+
+	    // Insert node before existing value
+	    if (c < 0) {
+		metadata_value_tree_node = new MetadataValueTreeNode(metadata_value);
+		insertNodeInto(metadata_value_tree_node, parent_metadata_value_tree_node, i);
+		break;
+	    }
+
+	    // Node already exists (don't add again)
+	    if (c == 0) {
+		metadata_value_tree_node = child_metadata_value_tree_node;
+		break;
+	    }
+	}
+
+	// If no node has been added yet, it must go at the end of the list
+	if (metadata_value_tree_node == null) {
+	    metadata_value_tree_node = new MetadataValueTreeNode(metadata_value);
+	    insertNodeInto(metadata_value_tree_node, parent_metadata_value_tree_node, parent_metadata_value_tree_node.getChildCount());
+	}
+
+	// If there is some of the metadata value remaining, add that hierarchically
+	if (!metadata_value_remaining.equals("")) {
+	    metadata_value_tree_node = addMetadataValueInternal(metadata_value_tree_node, metadata_value_remaining);
+	}
+
+	return metadata_value_tree_node;
+    }
+
+
+    public MetadataValueTreeNode getMetadataValueTreeNode(String metadata_value)
+    {
+	return getMetadataValueTreeNodeInternal((MetadataValueTreeNode) root, metadata_value);
+    }
+
+
+    private MetadataValueTreeNode getMetadataValueTreeNodeInternal(MetadataValueTreeNode parent_metadata_value_tree_node, String metadata_value_remaining)
+    {
+	// Split the metadata value into a hierarchy
+	int split_point = metadata_value_remaining.indexOf(MetadataValueTreeNode.METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN);
+	String metadata_value = ((split_point == -1) ? metadata_value_remaining : metadata_value_remaining.substring(0, split_point));
+	metadata_value_remaining = ((split_point == -1) ? "" : metadata_value_remaining.substring(split_point + 1));
+
+	// Find the value in the tree in the correct place (sorted)
+	MetadataValueTreeNode metadata_value_tree_node = null;
+	for (int i = 0; i < parent_metadata_value_tree_node.getChildCount(); i++) {
+	    MetadataValueTreeNode child_metadata_value_tree_node = (MetadataValueTreeNode) parent_metadata_value_tree_node.getChildAt(i);
+	    int c = metadata_value.compareTo(child_metadata_value_tree_node.getValue());
+
+	    // Node doesn't exist in the tree
+	    if (c < 0) {
+		return null;
+	    }
+	 
+	    // Node exists
+	    if (c == 0) {
+		metadata_value_tree_node = child_metadata_value_tree_node;
+		break;
+	    }
+	}
+
+	// If no node has been found, it doesn't exist
+	if (metadata_value_tree_node == null) {
+	    return null;
+	}
+
+	// If there is some of the metadata value remaining, find that recursively
+	if (!metadata_value_remaining.equals("")) {
+	    metadata_value_tree_node = getMetadataValueTreeNodeInternal(metadata_value_tree_node, metadata_value_remaining);
+	}
+
+	return metadata_value_tree_node;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeNode.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeNode.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataValueTreeNode.java	(revision 21112)
@@ -0,0 +1,79 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import javax.swing.tree.*;
+
+
+/** This class represents one Metadata value */
+public class MetadataValueTreeNode
+    extends DefaultMutableTreeNode
+{
+    static public final String METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN = "|";
+
+    private String value = null;
+
+
+    public MetadataValueTreeNode(Object object)
+    {
+	super(object);
+    }
+
+
+    public MetadataValueTreeNode(String value)
+    {
+	this.value = value;
+    }
+
+
+    public String getFullValue()
+    {
+	if (parent == null) {
+	    return "";
+	}
+
+	if (((MetadataValueTreeNode) parent).isRoot()) {
+	    return value;
+	}
+
+	return ((MetadataValueTreeNode) parent).getFullValue() + METADATA_VALUE_TREE_NODE_HIERARCHY_TOKEN + value;
+    }
+
+
+    public String getValue()
+    {
+	return value;
+    }
+
+
+    /** This is used to display the node in the value tree */
+    public String toString()
+    {
+	return value;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java	(revision 21112)
@@ -0,0 +1,556 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class represents one metadata.xml file */
+public class MetadataXMLFile
+    extends File
+{
+    static final private String DESCRIPTION_ELEMENT = "Description";
+    static final private String DIRECTORY_FILENAME = ".*";
+    static final private String FILENAME_ELEMENT = "FileName";
+    static final private String FILESET_ELEMENT = "FileSet";
+    static final private String METADATA_ELEMENT = "Metadata";
+
+
+    public MetadataXMLFile(String metadata_xml_file_path)
+    {
+	super(metadata_xml_file_path);
+    }
+
+
+    public void addMetadata(File file, ArrayList metadata_values)
+    {
+	// Parse the metadata.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata.xml file " + getAbsolutePath());
+	    return;
+	}
+
+	// Determine the file's path relative to the location of the metadata.xml file
+	String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
+	String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
+	if (file_relative_path.startsWith(File.separator)) {
+	    file_relative_path = file_relative_path.substring(File.separator.length());
+	}
+
+	// Form a regular expression that specifies the scope of the metadata
+	String file_path_regexp;
+	if (file_relative_path.equals("")) {
+	    // Special case for matching all files in the directory
+	    file_path_regexp = DIRECTORY_FILENAME;
+	}
+	else {
+	    // Convert the file path into a regular expression that will match it
+	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
+	}
+
+	// Find the appropriate FileSet element for this file
+	Element appropriate_fileset_element = null;
+
+	// Read all the FileSet elements in the file
+	NodeList fileset_elements_nodelist = document.getElementsByTagName(FILESET_ELEMENT);
+	for (int i = 0; i < fileset_elements_nodelist.getLength(); i++) {
+	    Element current_fileset_element = (Element) fileset_elements_nodelist.item(i);
+
+	    // Check the FileName elements of the FileSet to see if we have a match
+	    NodeList filename_elements_nodelist = current_fileset_element.getElementsByTagName(FILENAME_ELEMENT);
+	    for (int j = 0; j < filename_elements_nodelist.getLength(); j++) {
+		Element current_filename_element = (Element) filename_elements_nodelist.item(j);
+		String current_filename_element_value = XMLTools.getElementTextValue(current_filename_element);
+
+		// Only exact matches can be extended with new metadata
+		if (current_filename_element_value.equals(file_path_regexp)) {
+		    appropriate_fileset_element = current_fileset_element;
+		    break;
+		}
+	    }
+	}
+
+	// If no appropriate FileSet element exists create a new one for this file
+	if (appropriate_fileset_element == null) {
+	    DebugStream.println("Creating new FileSet element for file since none exists...");
+	    appropriate_fileset_element = document.createElement(FILESET_ELEMENT);
+
+	    Element new_filename_element = document.createElement(FILENAME_ELEMENT);
+	    new_filename_element.appendChild(document.createTextNode(file_path_regexp));
+	    appropriate_fileset_element.appendChild(new_filename_element);
+
+	    Element new_description_element = document.createElement(DESCRIPTION_ELEMENT);
+	    appropriate_fileset_element.appendChild(new_description_element);
+
+	    document.getDocumentElement().appendChild(appropriate_fileset_element);
+	}
+
+	// Add each of the metadata values to the FileSet's Description element
+	Element description_element = (Element) appropriate_fileset_element.getElementsByTagName(DESCRIPTION_ELEMENT).item(0);
+	for (int i = 0; i < metadata_values.size(); i++) {
+	    MetadataValue metadata_value = (MetadataValue) metadata_values.get(i);
+	    String metadata_element_name_full = metadata_value.getMetadataElement().getFullName();
+
+	    // Remove any characters that are invalid in XML
+	    String metadata_value_string = XMLTools.removeInvalidCharacters(metadata_value.getFullValue());
+
+	    // Square brackets need to be escaped because they are a special character in Greenstone
+	    metadata_value_string = metadata_value_string.replaceAll("\\[", "&#091;");
+	    metadata_value_string = metadata_value_string.replaceAll("\\]", "&#093;");
+
+	    // Check if this piece of metadata has already been assigned to this FileSet element
+	    boolean metadata_already_assigned = false;
+	    NodeList metadata_elements_nodelist = description_element.getElementsByTagName(METADATA_ELEMENT);
+	    for (int k = 0; k < metadata_elements_nodelist.getLength(); k++) {
+		Element current_metadata_element = (Element) metadata_elements_nodelist.item(k);
+
+		// Check if the metadata element name matches
+		String current_metadata_element_name_full = current_metadata_element.getAttribute("name");
+		if (current_metadata_element_name_full.equals(metadata_element_name_full)) {
+		    // Check if the metadata element value matches
+		    String current_metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
+		    if (current_metadata_value_string.equals(metadata_value_string)) {
+			// Metadata already assigned
+			metadata_already_assigned = true;
+			break;
+		    }
+		}
+	    }
+
+	    // If the piece of metadata hasn't already been assigned, add it now
+	    if (!metadata_already_assigned) {
+		// Create a new Metadata element to record this metadata
+		Element new_metadata_element = document.createElement(METADATA_ELEMENT);
+		new_metadata_element.setAttribute("name", metadata_value.getMetadataElement().getFullName());
+		new_metadata_element.setAttribute("mode", (metadata_value.isAccumulatingMetadata() ? "accumulate" : "override"));
+		new_metadata_element.appendChild(document.createTextNode(metadata_value_string));
+
+		// Accumulating metadata: add at the end
+		if (metadata_value.isAccumulatingMetadata()) {
+		    description_element.appendChild(new_metadata_element);
+		}
+		// Override metadata: add at the start (so it overrides inherited metadata without affecting other assigned metadata)
+		else {
+		    description_element.insertBefore(new_metadata_element, description_element.getFirstChild());
+		}
+	    }
+	}
+
+	// Rewrite the metadata.xml file
+	XMLTools.writeXMLFile(this, document);
+    }
+
+
+    public ArrayList getMetadataAssignedToFile(File file)
+    {
+	// Parse the metadata.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata.xml file " + getAbsolutePath());
+	    return new ArrayList();
+	}
+
+	// Determine the file's path relative to the location of the metadata.xml file
+	File metadata_xml_file_directory = getParentFile();
+	String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory.getAbsolutePath().length());
+	if (file_relative_path.startsWith(File.separator)) {
+	    file_relative_path = file_relative_path.substring(File.separator.length());
+	}
+
+	// Build up a list of metadata assigned to this file
+	ArrayList metadata_values = new ArrayList();
+
+	// Read all the FileSet elements in the file
+	NodeList fileset_elements_nodelist = document.getElementsByTagName(FILESET_ELEMENT);
+	for (int i = 0; i < fileset_elements_nodelist.getLength(); i++) {
+	    Element current_fileset_element = (Element) fileset_elements_nodelist.item(i);
+	    boolean current_fileset_matches = false;
+	    File folder_metadata_inherited_from = null;
+
+	    // Check the FileName elements of the FileSet to see if we have a match
+	    NodeList filename_elements_nodelist = current_fileset_element.getElementsByTagName(FILENAME_ELEMENT);
+	    for (int j = 0; j < filename_elements_nodelist.getLength(); j++) {
+		Element current_filename_element = (Element) filename_elements_nodelist.item(j);
+		String current_filename_element_value = XMLTools.getElementTextValue(current_filename_element);
+
+		// This fileset specifies metadata for the file
+		if (file_relative_path.matches(current_filename_element_value)) {
+		    current_fileset_matches = true;
+		    if (!file_relative_path.equals("") && current_filename_element_value.equals(DIRECTORY_FILENAME)) {
+			folder_metadata_inherited_from = metadata_xml_file_directory;
+		    }
+		    break;
+		}
+
+		// This fileset specifies metadata for the folder the file is in
+		if (file_relative_path.startsWith(current_filename_element_value + File.separator)) {
+		    current_fileset_matches = true;
+		    folder_metadata_inherited_from = new File(metadata_xml_file_directory, current_filename_element_value);
+		    break;
+		}
+	    }
+
+	    // The FileSet doesn't apply, so move onto the next one
+	    if (current_fileset_matches == false) {
+		continue;
+	    }
+
+	    // Read all the Metadata elements in the fileset
+	    NodeList metadata_elements_nodelist = current_fileset_element.getElementsByTagName(METADATA_ELEMENT);
+	    for (int k = 0; k < metadata_elements_nodelist.getLength(); k++) {
+		Element current_metadata_element = (Element) metadata_elements_nodelist.item(k);
+		String metadata_element_name_full = current_metadata_element.getAttribute("name");
+		String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+
+		// Ignore legacy crap
+		if (metadata_set_namespace.equals("hidden")) {
+		    continue;
+		}
+
+		MetadataSet metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+		if (metadata_set == null) {
+		    // The metadata set isn't loaded, so give the option of mapping the element into a loaded set
+		    String target_metadata_element_name_full = MetadataSetManager.mapUnloadedMetadataElement(metadata_element_name_full);
+		    if (target_metadata_element_name_full == null || target_metadata_element_name_full.equals("")) {
+			// Skip this element if we still don't have a loaded element for it
+			continue;
+		    }
+
+		    metadata_element_name_full = target_metadata_element_name_full;
+		    metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+		    metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+		}
+
+		MetadataElement metadata_element = MetadataTools.getMetadataElementWithName(metadata_element_name_full);
+
+		// If the element doesn't exist in the metadata set, we're not interested
+		if (metadata_element == null) {
+		    continue;
+		}
+
+		// Square brackets need to be escaped because they are a special character in Greenstone
+		String metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
+		metadata_value_string = metadata_value_string.replaceAll("&#091;", "[");
+		metadata_value_string = metadata_value_string.replaceAll("&#093;", "]");
+
+		MetadataValueTreeNode metadata_value_tree_node = metadata_element.getMetadataValueTreeNode(metadata_value_string);
+
+		// If there is no metadata value tree node for this value, create it
+		if (metadata_value_tree_node == null) {
+		    DebugStream.println("Note: No value tree node for metadata value \"" + metadata_value_string + "\"");
+		    metadata_element.addMetadataValue(metadata_value_string);
+		    metadata_value_tree_node = metadata_element.getMetadataValueTreeNode(metadata_value_string);
+		}
+
+		MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+		metadata_value.inheritsMetadataFromFolder(folder_metadata_inherited_from);
+
+		// Is this accumulating metadata?
+		if (current_metadata_element.getAttribute("mode").equals("accumulate")) {
+		    metadata_value.setIsAccumulatingMetadata(true);
+		}
+
+		// Add the new metadata value to the list
+		metadata_values.add(metadata_value);
+	    }
+	}
+
+	return metadata_values;
+    }
+
+
+    public void removeMetadata(File file, ArrayList metadata_values)
+    {
+	// Parse the metadata.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata.xml file " + getAbsolutePath());
+	    return;
+	}
+
+	// Determine the file's path relative to the location of the metadata.xml file
+	String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
+	String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
+	if (file_relative_path.startsWith(File.separator)) {
+	    file_relative_path = file_relative_path.substring(File.separator.length());
+	}
+
+	// Form a regular expression that specifies the scope of the metadata
+	String file_path_regexp;
+	if (file_relative_path.equals("")) {
+	    // Special case for matching all files in the directory
+	    file_path_regexp = DIRECTORY_FILENAME;
+	}
+	else {
+	    // Convert the file path into a regular expression that will match it
+	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
+	}
+
+	// Find the appropriate FileSet element for this file
+	Element appropriate_fileset_element = null;
+
+	// Read all the FileSet elements in the file
+	NodeList fileset_elements_nodelist = document.getElementsByTagName(FILESET_ELEMENT);
+	for (int i = 0; i < fileset_elements_nodelist.getLength(); i++) {
+	    Element current_fileset_element = (Element) fileset_elements_nodelist.item(i);
+
+	    // Check the FileName elements of the FileSet to see if we have a match
+	    NodeList filename_elements_nodelist = current_fileset_element.getElementsByTagName(FILENAME_ELEMENT);
+	    for (int j = 0; j < filename_elements_nodelist.getLength(); j++) {
+		Element current_filename_element = (Element) filename_elements_nodelist.item(j);
+		String current_filename_element_value = XMLTools.getElementTextValue(current_filename_element);
+
+		// Only exact matches can be extended with new metadata
+		if (current_filename_element_value.equals(file_path_regexp)) {
+		    appropriate_fileset_element = current_fileset_element;
+		    break;
+		}
+	    }
+	}
+
+	// If no appropriate FileSet element exists we have no idea where the metadata is
+	if (appropriate_fileset_element == null) {
+	    System.err.println("Error: No appropriate FileSet element found when removing metadata...");
+	    return;
+	}
+
+	// Remove each of the metadata values from the FileSet's Description element
+	Element description_element = (Element) appropriate_fileset_element.getElementsByTagName(DESCRIPTION_ELEMENT).item(0);
+	for (int i = 0; i < metadata_values.size(); i++) {
+	    MetadataValue metadata_value = (MetadataValue) metadata_values.get(i);
+
+	    // Remove any characters that are invalid in XML
+	    String metadata_value_string = XMLTools.removeInvalidCharacters(metadata_value.getFullValue());
+
+	    // Square brackets need to be escaped because they are a special character in Greenstone
+	    metadata_value_string = metadata_value_string.replaceAll("\\[", "&#091;");
+	    metadata_value_string = metadata_value_string.replaceAll("\\]", "&#093;");
+
+	    // Find the Metadata element to delete from the fileset
+	    String metadata_element_name_full = metadata_value.getMetadataElement().getFullName();
+	    NodeList metadata_elements_nodelist = appropriate_fileset_element.getElementsByTagName(METADATA_ELEMENT);
+	    for (int k = 0; k < metadata_elements_nodelist.getLength(); k++) {
+		Element current_metadata_element = (Element) metadata_elements_nodelist.item(k);
+
+		// Check the metadata element name matches
+		String current_metadata_element_name_full = current_metadata_element.getAttribute("name");
+		if (current_metadata_element_name_full.equals(metadata_element_name_full)) {
+		    // Check the metadata element value matches
+		    String current_metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
+		    if (current_metadata_value_string.equals(metadata_value_string)) {
+			// Remove this Metadata element
+			current_metadata_element.getParentNode().removeChild(current_metadata_element);
+			break;
+		    }
+		}
+	    }
+	}
+
+	// Rewrite the metadata.xml file
+	XMLTools.writeXMLFile(this, document);
+    }
+
+
+    public void replaceMetadata(File file, MetadataValue old_metadata_value, MetadataValue new_metadata_value)
+    {
+	// Parse the metadata.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata.xml file " + getAbsolutePath());
+	    return;
+	}
+
+	// Determine the file's path relative to the location of the metadata.xml file
+	String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
+	String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
+	if (file_relative_path.startsWith(File.separator)) {
+	    file_relative_path = file_relative_path.substring(File.separator.length());
+	}
+
+	// Form a regular expression that specifies the scope of the metadata
+	String file_path_regexp;
+	if (file_relative_path.equals("")) {
+	    // Special case for matching all files in the directory
+	    file_path_regexp = DIRECTORY_FILENAME;
+	}
+	else {
+	    // Convert the file path into a regular expression that will match it
+	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
+	}
+
+	// Remove any characters that are invalid in XML
+	String old_metadata_value_string = XMLTools.removeInvalidCharacters(old_metadata_value.getFullValue());
+	String new_metadata_value_string = XMLTools.removeInvalidCharacters(new_metadata_value.getFullValue());
+
+	// Square brackets need to be escaped because they are a special character in Greenstone
+	old_metadata_value_string = old_metadata_value_string.replaceAll("\\[", "&#091;");
+	old_metadata_value_string = old_metadata_value_string.replaceAll("\\]", "&#093;");
+	new_metadata_value_string = new_metadata_value_string.replaceAll("\\[", "&#091;");
+	new_metadata_value_string = new_metadata_value_string.replaceAll("\\]", "&#093;");
+
+	// Read all the FileSet elements in the file
+	NodeList fileset_elements_nodelist = document.getElementsByTagName(FILESET_ELEMENT);
+	for (int i = 0; i < fileset_elements_nodelist.getLength(); i++) {
+	    Element current_fileset_element = (Element) fileset_elements_nodelist.item(i);
+	    boolean current_fileset_matches = false;
+
+	    // Check the FileName elements of the FileSet to see if we have a match
+	    NodeList filename_elements_nodelist = current_fileset_element.getElementsByTagName(FILENAME_ELEMENT);
+	    for (int j = 0; j < filename_elements_nodelist.getLength(); j++) {
+		Element current_filename_element = (Element) filename_elements_nodelist.item(j);
+		String current_filename_element_value = XMLTools.getElementTextValue(current_filename_element);
+
+		// Only exact matches can be edited
+		if (current_filename_element_value.equals(file_path_regexp)) {
+		    current_fileset_matches = true;
+		    break;
+		}
+	    }
+
+	    // The FileSet doesn't apply, so move onto the next one
+	    if (current_fileset_matches == false) {
+		continue;
+	    }
+
+	    // Each metadata value is only allowed to be assigned once
+	    boolean new_metadata_value_already_exists = false;
+	    Element metadata_element_to_edit = null;
+
+	    // Find the Metadata element to replace in the fileset
+	    String metadata_element_name_full = old_metadata_value.getMetadataElement().getFullName();
+	    NodeList metadata_elements_nodelist = current_fileset_element.getElementsByTagName(METADATA_ELEMENT);
+	    for (int k = 0; k < metadata_elements_nodelist.getLength(); k++) {
+		Element current_metadata_element = (Element) metadata_elements_nodelist.item(k);
+
+		// Check the metadata element name matches
+		String current_metadata_element_name_full = current_metadata_element.getAttribute("name");
+		if (!current_metadata_element_name_full.equals(metadata_element_name_full)) {
+		    continue;
+		}
+
+		// Check the new metadata value doesn't already exist
+		String current_metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
+		if (current_metadata_value_string.equals(new_metadata_value_string)) {
+		    new_metadata_value_already_exists = true;
+		}
+
+		// Check the metadata element value matches
+		if (current_metadata_value_string.equals(old_metadata_value_string)) {
+		    metadata_element_to_edit = current_metadata_element;
+		}
+	    }
+
+	    // If the new metadata value already existed, remove the original value
+	    if (new_metadata_value_already_exists) {
+		metadata_element_to_edit.getParentNode().removeChild(metadata_element_to_edit);
+	    }
+	    // Otherwise replace the old value with the new value
+	    else {
+		XMLTools.setElementTextValue(metadata_element_to_edit, new_metadata_value_string);
+	    }
+	}
+
+	// Rewrite the metadata.xml file
+	XMLTools.writeXMLFile(this, document);
+    }
+
+
+    /**
+     * Every metadata.xml file must be skimmed when a collection is opened, for two reasons:
+     *   - To build complete and accurate metadata value trees (needed for Enrich pane and hierarchy files)
+     *   - To handle any non-namespaced metadata in the file
+     */
+    public void skimFile()
+    {
+	boolean file_changed = false;
+
+	// Parse the metadata.xml file
+	DebugStream.println("Skimming metadata.xml file " + this + "...");
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse metadata.xml file " + getAbsolutePath());
+	    return;
+	}
+
+	// Read all the Metadata elements in the file
+	NodeList metadata_elements_nodelist = document.getElementsByTagName(METADATA_ELEMENT);
+	for (int i = 0; i < metadata_elements_nodelist.getLength(); i++) {
+	    Element current_metadata_element = (Element) metadata_elements_nodelist.item(i);
+	    String metadata_element_name_full = current_metadata_element.getAttribute("name");
+	    String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+
+	    // Ignore legacy crap
+	    if (metadata_set_namespace.equals("hidden")) {
+		continue;
+	    }
+
+	    MetadataSet metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+	    if (metadata_set == null) {
+		// The metadata set isn't loaded, so give the option of mapping the element into a loaded set
+		String target_metadata_element_name_full = MetadataSetManager.mapUnloadedMetadataElement(metadata_element_name_full);
+		if (target_metadata_element_name_full == null || target_metadata_element_name_full.equals("")) {
+		    // Skip this element if we still don't have a loaded element for it
+		    continue;
+		}
+
+		// Update the metadata.xml file to have the new element name
+		current_metadata_element.setAttribute("name", target_metadata_element_name_full);
+		file_changed = true;
+
+		metadata_element_name_full = target_metadata_element_name_full;
+		metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+		metadata_set = MetadataSetManager.getMetadataSet(metadata_set_namespace);
+	    }
+
+	    String metadata_element_name = MetadataTools.getMetadataElementName(metadata_element_name_full);
+	    MetadataElement metadata_element = metadata_set.getMetadataElementWithName(metadata_element_name);
+
+	    // If the element doesn't exist in the metadata set, add it
+	    if (metadata_element == null) {
+		metadata_element = metadata_set.addMetadataElementForThisSession(metadata_element_name);
+	    }
+
+	    // Square brackets need to be escaped because they are a special character in Greenstone
+	    String metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
+	    metadata_value_string = metadata_value_string.replaceAll("&#091;", "[");
+	    metadata_value_string = metadata_value_string.replaceAll("&#093;", "]");
+
+	    metadata_element.addMetadataValue(metadata_value_string);
+	}
+
+	// Rewrite the metadata.xml file if it has changed
+	if (file_changed) {
+	    XMLTools.writeXMLFile(this, document);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java	(revision 21112)
@@ -0,0 +1,378 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.XMLTools;
+
+
+/** This class is a static class that manages the metadata.xml files */
+public class MetadataXMLFileManager
+{
+    static private ArrayList metadata_xml_files = new ArrayList();
+    /** The objects listening for MetadataChanged events. */
+    static private ArrayList metadata_changed_listeners = new ArrayList();
+    /** Keep track of which metadata.xml files have been modified so we can upload them to the server */
+    static private ArrayList modified_metadata_xml_files = new ArrayList();
+
+
+    static public void addMetadata(CollectionTreeNode file_node, ArrayList metadata_values)
+    {
+	addMetadata(new CollectionTreeNode[] { file_node }, metadata_values);
+    }
+
+
+    static public void addMetadata(CollectionTreeNode[] file_nodes, MetadataValue metadata_value)
+    {
+	ArrayList metadata_values = new ArrayList();
+	metadata_values.add(metadata_value);
+	addMetadata(file_nodes, metadata_values);
+    }
+
+
+    static public void addMetadata(CollectionTreeNode[] file_nodes, ArrayList metadata_values)
+    {
+	// Check the list of metadata values is non-empty
+	if (metadata_values.isEmpty()) {
+	    return;
+	}
+
+	// Add the metadata to each file node in turn
+	for (int i = 0; i < file_nodes.length; i++) {
+	    File current_file = file_nodes[i].getFile();
+	    DebugStream.println("Adding metadata to " + current_file.getAbsolutePath());
+
+	    // Find which metadata.xml file needs editing
+	    boolean applicable_metadata_xml_file_found = false;
+	    File current_file_directory = (current_file.isDirectory() ? current_file : current_file.getParentFile());
+	    String current_file_directory_path = current_file_directory.getAbsolutePath();
+	    for (int j = 0; j < metadata_xml_files.size(); j++) {
+		MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(j);
+
+		// This metadata.xml file is only applicable if it is at the same level as the file
+		if (current_file_directory_path.equals(metadata_xml_file.getParentFile().getAbsolutePath())) {
+		    applicable_metadata_xml_file_found = true;
+		    metadata_xml_file.addMetadata(current_file, metadata_values);
+		    if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
+			modified_metadata_xml_files.add(metadata_xml_file);
+		    }
+		}
+	    }
+
+	    // If no applicable metadata.xml file exists, we have to create a new one
+	    if (!applicable_metadata_xml_file_found) {
+		// Create a new (empty) metadata.xml file in the file's directory...
+		File new_metadata_xml_file_file = new File(current_file_directory, "metadata.xml");
+		XMLTools.writeXMLFile(new_metadata_xml_file_file, XMLTools.parseXMLFile("xml/metadata.xml", true));
+
+		// ...load it...
+		MetadataXMLFile new_metadata_xml_file = loadMetadataXMLFile(new_metadata_xml_file_file);
+
+		// ...and add the metadata
+		new_metadata_xml_file.addMetadata(current_file, metadata_values);
+		if (!modified_metadata_xml_files.contains(new_metadata_xml_file)) {
+		    modified_metadata_xml_files.add(new_metadata_xml_file);
+		}
+	    }
+	}
+
+	// Let any listeners know that the metadata has changed
+	fireMetadataChangedEvent(file_nodes);
+    }
+
+
+    static public void addMetadataChangedListener(MetadataChangedListener metadata_changed_listener)
+    {
+	metadata_changed_listeners.add(metadata_changed_listener);
+    }
+
+
+    static public void clearMetadataXMLFiles()
+    {
+	metadata_xml_files.clear();
+    }
+
+
+    static private void fireMetadataChangedEvent(CollectionTreeNode[] file_nodes)
+    {
+	// Send the event off to all the MetadataChangedListeners
+	for (int i = 0; i < metadata_changed_listeners.size(); i++) {
+	    ((MetadataChangedListener) metadata_changed_listeners.get(i)).metadataChanged(file_nodes);
+	}
+    }
+
+
+    /** Returns the metadata assigned to a file outside the collection, excluding folder-level/inherited metadata. */
+    static public ArrayList getMetadataAssignedDirectlyToExternalFile(File file)
+    {
+	DebugStream.println("Getting metadata assigned directly to external file " + file + "...");
+
+	// Build up a list of applicable metadata.xml files
+	ArrayList applicable_metadata_xml_files = new ArrayList();
+
+	File directory = (file.isDirectory() ? file : file.getParentFile());
+	while (directory != null) {
+	    File metadata_xml_file = new File(directory, "metadata.xml");
+	    if (metadata_xml_file.exists() && !metadata_xml_file.isDirectory()) {
+		// It is very important that shallower files come before deeper ones
+		applicable_metadata_xml_files.add(0, new MetadataXMLFile(metadata_xml_file.getAbsolutePath()));
+	    }
+
+	    directory = directory.getParentFile();
+	}
+
+	// Get the metadata assigned to the specified file from the applicable metadata.xml files
+	ArrayList assigned_metadata = getMetadataAssignedToFile(file, applicable_metadata_xml_files);
+
+	// Remove any folder-level metadata
+	for (int i = assigned_metadata.size() - 1; i >= 0; i--) {
+	    if (((MetadataValue) assigned_metadata.get(i)).isInheritedMetadata()) {
+		assigned_metadata.remove(i);
+	    }
+	}
+
+	return assigned_metadata;
+    }
+
+
+    /** Returns the metadata assigned to a file inside the collection, excluding folder-level/inherited metadata. */
+    static public ArrayList getMetadataAssignedDirectlyToFile(File file)
+    {
+	// Get all the metadata assigned to the specified file...
+	ArrayList assigned_metadata = getMetadataAssignedToFile(file);
+
+	// ...then remove any folder-level metadata
+	for (int i = assigned_metadata.size() - 1; i >= 0; i--) {
+	    if (((MetadataValue) assigned_metadata.get(i)).isInheritedMetadata()) {
+		assigned_metadata.remove(i);
+	    }
+	}
+
+	return assigned_metadata;
+    }
+
+
+    /** Returns all the metadata assigned to a file inside the collection. */
+    static public ArrayList getMetadataAssignedToFile(File file)
+    {
+	// Build up a list of applicable metadata.xml files
+	ArrayList applicable_metadata_xml_files = new ArrayList();
+
+	// Look at each loaded metadata.xml file to see if it is potentially applicable
+	String file_directory_path = (file.isDirectory() ? file : file.getParentFile()).getAbsolutePath() + File.separator;
+	for (int i = 0; i < metadata_xml_files.size(); i++) {
+	    MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(i);
+
+	    // This metadata.xml file is only potentially applicable if it is above or at the same level as the file
+	    if (file_directory_path.startsWith(metadata_xml_file.getParentFile().getAbsolutePath() + File.separator)) {
+		applicable_metadata_xml_files.add(metadata_xml_file);
+	    }
+	}
+
+	// Return the metadata assigned to the specified file from the applicable metadata.xml files
+	return getMetadataAssignedToFile(file, applicable_metadata_xml_files);
+    }
+
+
+    static private ArrayList getMetadataAssignedToFile(File file, ArrayList applicable_metadata_xml_files)
+    {
+	// Build up a list of metadata values assigned to this file
+	ArrayList metadata_values_all = new ArrayList();
+
+	// Look at each applicable metadata.xml file to see if it assigns metadata to this file
+	for (int i = 0; i < applicable_metadata_xml_files.size(); i++) {
+	    MetadataXMLFile metadata_xml_file = (MetadataXMLFile) applicable_metadata_xml_files.get(i);
+	    DebugStream.println("Applicable metadata.xml file: " + metadata_xml_file);
+
+	    ArrayList metadata_values = metadata_xml_file.getMetadataAssignedToFile(file);
+	    for (int j = 0; j < metadata_values.size(); j++) {
+		MetadataValue metadata_value = (MetadataValue) metadata_values.get(j);
+
+		// Overriding metadata: remove any values with this metadata element
+		if (metadata_value.isAccumulatingMetadata() == false) {
+		    for (int k = metadata_values_all.size() - 1; k >= 0; k--) {
+			if (((MetadataValue) metadata_values_all.get(k)).getMetadataElement().equals(metadata_value.getMetadataElement())) {
+			    metadata_values_all.remove(k);
+			}
+		    }
+		}
+
+		metadata_values_all.add(metadata_value);
+	    }
+	}
+
+	return metadata_values_all;
+    }
+
+
+    static public void loadMetadataXMLFiles(File directory)
+    {
+	// Make sure the directory (import) exists
+	if (directory.exists() == false) {
+	    return;
+	}
+
+	// Look recursively at each subfile of the directory for metadata.xml files
+	File[] directory_files = directory.listFiles();
+	for (int i = 0; i < directory_files.length; i++) {
+	    File child_file = directory_files[i];
+	    if (child_file.isDirectory()) {
+		loadMetadataXMLFiles(child_file);
+	    }
+	    else if (child_file.getName().equals("metadata.xml")) {
+		loadMetadataXMLFile(child_file);
+	    }
+	}
+    }
+
+
+    static private MetadataXMLFile loadMetadataXMLFile(File metadata_xml_file_file)
+    {
+	MetadataXMLFile metadata_xml_file = new MetadataXMLFile(metadata_xml_file_file.getAbsolutePath());
+	metadata_xml_file.skimFile();
+	metadata_xml_files.add(metadata_xml_file);
+	return metadata_xml_file;
+    }
+
+
+    static public void removeMetadata(CollectionTreeNode file_node, ArrayList metadata_values)
+    {
+	removeMetadata(new CollectionTreeNode[] { file_node }, metadata_values);
+    }
+
+
+    static public void removeMetadata(CollectionTreeNode[] file_nodes, MetadataValue metadata_value)
+    {
+	ArrayList metadata_values = new ArrayList();
+	metadata_values.add(metadata_value);
+	removeMetadata(file_nodes, metadata_values);
+    }
+
+
+    static public void removeMetadata(CollectionTreeNode[] file_nodes, ArrayList metadata_values)
+    {
+	// Check the list of metadata values is non-empty
+	if (metadata_values.isEmpty()) {
+	    return;
+	}
+
+	// Remove the metadata from each file node in turn
+	for (int i = 0; i < file_nodes.length; i++) {
+	    File current_file = file_nodes[i].getFile();
+	    DebugStream.println("Removing metadata from " + current_file.getAbsolutePath());
+
+	    // Find which metadata.xml file needs editing
+	    File current_file_directory = (current_file.isDirectory() ? current_file : current_file.getParentFile());
+	    String current_file_directory_path = current_file_directory.getAbsolutePath();
+	    for (int j = 0; j < metadata_xml_files.size(); j++) {
+		MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(j);
+
+		// This metadata.xml file is only potentially applicable if it is above or at the same level as the file
+		if (current_file_directory_path.startsWith(metadata_xml_file.getParentFile().getAbsolutePath())) {
+		    metadata_xml_file.removeMetadata(current_file, metadata_values);
+		    if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
+			modified_metadata_xml_files.add(metadata_xml_file);
+		    }
+		}
+	    }
+	}
+
+	// Let any listeners know that the metadata has changed
+	fireMetadataChangedEvent(file_nodes);
+    }
+
+
+    static public void removeMetadataChangedListener(MetadataChangedListener metadata_changed_listener)
+    {
+	metadata_changed_listeners.remove(metadata_changed_listener);
+    }
+
+
+    static public void replaceMetadata(CollectionTreeNode[] file_nodes, MetadataValue old_metadata_value, MetadataValue new_metadata_value)
+    {
+	// Replace the metadata in each file node in turn
+	for (int i = 0; i < file_nodes.length; i++) {
+	    File current_file = file_nodes[i].getFile();
+	    DebugStream.println("Replacing metadata in " + current_file.getAbsolutePath());
+
+	    // Find which metadata.xml file needs editing
+	    File current_file_directory = (current_file.isDirectory() ? current_file : current_file.getParentFile());
+	    String current_file_directory_path = current_file_directory.getAbsolutePath();
+	    for (int j = 0; j < metadata_xml_files.size(); j++) {
+		MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(j);
+
+		// This metadata.xml file is only applicable if it is at the same level as the file
+		if (current_file_directory_path.equals(metadata_xml_file.getParentFile().getAbsolutePath())) {
+		    metadata_xml_file.replaceMetadata(current_file, old_metadata_value, new_metadata_value);
+		    if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
+			modified_metadata_xml_files.add(metadata_xml_file);
+		    }
+		}
+	    }
+	}
+
+	// Let any listeners know that the metadata has changed
+	fireMetadataChangedEvent(file_nodes);
+    }
+
+
+    static public void unloadMetadataXMLFile(File metadata_xml_file_file)
+    {
+	DebugStream.println("Unloading metadata.xml file " + metadata_xml_file_file);
+
+	// Find the metadata.xml file in the list of loaded files, and remove it
+	for (int i = 0; i < metadata_xml_files.size(); i++) {
+	    MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(i);
+	    if (metadata_xml_file_file.getAbsolutePath().equals(metadata_xml_file.getAbsolutePath())) {
+		metadata_xml_files.remove(i);
+		break;
+	    }
+	}
+    }
+
+
+    static public void uploadModifiedMetadataXMLFiles()
+    {
+	// This is only necessary when the collection is stored on a remote server
+	if (Gatherer.isGsdlRemote) {
+	    if (modified_metadata_xml_files.isEmpty()) {
+		DebugStream.println("No modified metadata.xml files to upload.");
+		return;
+	    }
+
+	    // Upload the files modified since last time, then reset the list
+	    RemoteGreenstoneServer.uploadCollectionFiles(Gatherer.c_man.getCollection().getName(), (File[]) modified_metadata_xml_files.toArray(new File[0]));
+	    modified_metadata_xml_files.clear();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFile.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFile.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFile.java	(revision 21112)
@@ -0,0 +1,110 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.util.XMLTools;
+import org.w3c.dom.*;
+
+
+/** This class represents one profile.xml file */
+public class ProfileXMLFile
+    extends File
+{
+    private HashMap metadata_mapping = null;
+
+
+    public ProfileXMLFile(String profile_xml_file_path)
+    {
+	super(profile_xml_file_path);
+    }
+
+
+    public String getMetadataElementFor(String metadata_element_name_full)
+    {
+	if (getMetadataMapping() == null) {
+	    return null;
+	}
+
+	return (String) metadata_mapping.get(metadata_element_name_full);
+    }
+
+
+    public HashMap getMetadataMapping()
+    {
+	// If we have already loaded the metadata mapping, use it
+	if (metadata_mapping != null) {
+	    return metadata_mapping;
+	}
+
+	// Build the metadata mapping from the profile.xml file
+	metadata_mapping = new HashMap();
+
+	// Parse the profile.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse profile.xml file " + getAbsolutePath());
+	    return null;
+	}
+
+	// Read all the Action elements in the file
+	NodeList action_elements_nodelist = document.getElementsByTagName("Action");
+	for (int i = 0; i < action_elements_nodelist.getLength(); i++) {
+	    Element current_action_element = (Element) action_elements_nodelist.item(i);
+	    String source_metadata_element_name_full = current_action_element.getAttribute("source");
+	    String target_metadata_element_name_full = current_action_element.getAttribute("target");
+	    metadata_mapping.put(source_metadata_element_name_full, target_metadata_element_name_full);
+	}
+
+	return metadata_mapping;
+    }
+
+
+    public void mapElement(String source_metadata_element_name_full, String target_metadata_element_name_full)
+    {
+	// Parse the profile.xml file
+	Document document = XMLTools.parseXMLFile(this);
+	if (document == null) {
+	    System.err.println("Error: Could not parse profile.xml file " + getAbsolutePath());
+	    return;
+	}
+
+	// Create a new Action element to record this mapping
+	Element new_action_element = document.createElement("Action");
+	new_action_element.setAttribute("source", source_metadata_element_name_full);
+	new_action_element.setAttribute("target", target_metadata_element_name_full);
+	document.getDocumentElement().appendChild(new_action_element);
+
+	// Rewrite the profile.xml file
+	XMLTools.writeXMLFile(this, document);
+
+	// Invalidate the metadata mapping
+	metadata_mapping = null;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFileManager.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFileManager.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/metadata/ProfileXMLFileManager.java	(revision 21112)
@@ -0,0 +1,90 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.metadata;
+
+
+import java.io.*;
+import java.util.*;
+import org.greenstone.gatherer.util.XMLTools;
+
+
+/** This class is a static class that manages the profile.xml file */
+public class ProfileXMLFileManager
+{
+    static private ProfileXMLFile profile_xml_file = null;
+
+
+    static public void clearProfileXMLFile()
+    {
+	profile_xml_file = null;
+    }
+
+
+    static public void loadProfileXMLFile(File directory)
+    {
+	// Make sure the directory (metadata) exists
+	if (directory.exists() == false) {
+	    directory.mkdir();
+	}
+
+	File profile_xml_file_file = new File(directory, "profile.xml");
+	if (!profile_xml_file_file.exists()) {
+	    // Create a new (empty) profile.xml file in this directory...
+	    XMLTools.writeXMLFile(profile_xml_file_file, XMLTools.parseXMLFile("xml/profile.xml", true));
+	}
+
+	profile_xml_file = new ProfileXMLFile(profile_xml_file_file.getAbsolutePath());
+    }
+
+
+    static public String getMetadataElementFor(String metadata_element_name_full)
+    {
+	if (profile_xml_file == null) {
+	    return null;
+	}
+
+	return profile_xml_file.getMetadataElementFor(metadata_element_name_full);
+    }
+
+
+    static public HashMap getMetadataMapping()
+    {
+	if (profile_xml_file == null) {
+	    return null;
+	}
+
+	return profile_xml_file.getMetadataMapping();
+    }
+
+
+    static public void mapElement(String metadata_element_name_full, String target_metadata_element_name_full)
+    {
+	if (profile_xml_file != null) {
+	    profile_xml_file.mapElement(metadata_element_name_full, target_metadata_element_name_full);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/RemoteGreenstoneServer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/RemoteGreenstoneServer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/RemoteGreenstoneServer.java	(revision 21112)
@@ -0,0 +1,1138 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.remote;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.zip.*;
+import javax.swing.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.GAuthenticator;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.collection.CollectionManager;
+import org.greenstone.gatherer.shell.GShell;
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+public class RemoteGreenstoneServer
+{
+    // ----------------------------------------------------------------------------------------------------
+    //   PUBLIC LAYER
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static public String deleteCollection(String collection_name)
+    {
+	return performAction(new RemoteGreenstoneServerDeleteCollectionAction(collection_name));
+    }
+
+
+    static public String deleteCollectionFile(String collection_name, File collection_file)
+    {
+	return performAction(new RemoteGreenstoneServerDeleteCollectionFileAction(collection_name, collection_file));
+    }
+
+
+    static public String downloadCollection(String collection_name)
+    {
+	return performAction(new RemoteGreenstoneServerDownloadCollectionAction(collection_name));
+    }
+
+
+    static public String downloadCollectionArchives(String collection_name)
+    {
+	return performAction(new RemoteGreenstoneServerDownloadCollectionArchivesAction(collection_name));
+    }
+
+
+    static public String downloadCollectionConfigurations()
+    {
+	return performAction(new RemoteGreenstoneServerDownloadCollectionConfigurationsAction());
+    }
+
+
+    static public String downloadCollectionFile(String collection_name, File collection_file)
+    {
+	return performAction(new RemoteGreenstoneServerDownloadCollectionFileAction(collection_name, collection_file));
+    }
+
+
+    static public String getScriptOptions(String script_name, String script_arguments)
+    {
+	return performAction(new RemoteGreenstoneServerGetScriptOptionsAction(script_name, script_arguments));
+    }
+
+
+    static public String moveCollectionFile(String collection_name, File source_collection_file, File target_collection_file)
+    {
+	return performAction(new RemoteGreenstoneServerMoveCollectionFileAction(collection_name, source_collection_file, target_collection_file));
+    }
+
+
+    static public String newCollectionDirectory(String collection_name, File new_collection_directory)
+    {
+	return performAction(new RemoteGreenstoneServerNewCollectionDirectoryAction(collection_name, new_collection_directory));
+    }
+
+
+    static public String runScript(String collection_name, String script_name, String script_arguments, GShell shell)
+    {
+	return performAction(new RemoteGreenstoneServerRunScriptAction(collection_name, script_name, script_arguments, shell));
+    }
+
+
+    static public String uploadCollectionFile(String collection_name, File collection_file)
+    {
+	return performAction(new RemoteGreenstoneServerUploadCollectionFilesAction(collection_name, new File[] { collection_file }));
+    }
+
+
+    static public String uploadCollectionFiles(String collection_name, File[] collection_files)
+    {
+	return performAction(new RemoteGreenstoneServerUploadCollectionFilesAction(collection_name, collection_files));
+    }
+
+
+    static public String uploadFilesIntoCollection(String collection_name, File[] source_files, File target_collection_directory)
+    {
+	return performAction(new RemoteGreenstoneServerUploadFilesIntoCollectionAction(collection_name, source_files, target_collection_directory));
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static public void exit()
+    {
+	System.err.println("Exiting, number of jobs on queue: " + remote_greenstone_server_action_queue.size());
+
+	// If there are still jobs on the queue we must wait for the jobs to finish
+	while (remote_greenstone_server_action_queue.size() > 0) {
+	    synchronized (remote_greenstone_server_action_queue) {
+		try {
+		    DebugStream.println("Waiting for queue to become empty...");
+		    remote_greenstone_server_action_queue.wait(500);
+		}
+		catch (InterruptedException exception) {}
+	    }
+	}
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   QUEUE LAYER
+    // ----------------------------------------------------------------------------------------------------
+
+
+    /** Returns null if we cannot wait for the action to finish, "" if the action failed, or the action output. */
+    static private String performAction(RemoteGreenstoneServerAction remote_greenstone_server_action)
+    {
+	// Add the action to the queue
+	remote_greenstone_server_action_queue.addAction(remote_greenstone_server_action);
+
+	// If we're running in the GUI thread we must return immediately
+	// We cannot wait for the action to complete because this will block any GUI updates
+	if (SwingUtilities.isEventDispatchThread()) {
+	    System.err.println("WARNING: In event dispatch thread, returning immediately...");
+	    return null;
+	}
+
+	// Otherwise wait until the action is processed
+	while (!remote_greenstone_server_action.processed) {
+	    synchronized (remote_greenstone_server_action) {
+		try {
+		    DebugStream.println("Waiting for action to complete...");
+		    remote_greenstone_server_action.wait(500);
+		}
+		catch (InterruptedException exception) {}
+	    }
+	}
+
+	// Return "" if the action failed
+	if (!remote_greenstone_server_action.processed_successfully) {
+	    return "";
+	}
+	// Otherwise return the action output
+	return remote_greenstone_server_action.action_output;
+    }
+
+
+    static private RemoteGreenstoneServerActionQueue remote_greenstone_server_action_queue = new RemoteGreenstoneServerActionQueue();
+
+
+    static private class RemoteGreenstoneServerActionQueue
+	extends Thread
+    {
+	/** The queue of waiting jobs. */
+	private ArrayList queue = null;
+
+
+	public RemoteGreenstoneServerActionQueue()
+	{
+	    if (Gatherer.isGsdlRemote) {
+		queue = new ArrayList();
+		start();
+	    }
+	}
+
+
+	synchronized public void addAction(RemoteGreenstoneServerAction remote_greenstone_server_action)
+	{
+	    queue.add(remote_greenstone_server_action);
+	    notifyAll();
+	}
+
+
+	public int size()
+	{
+	    return queue.size();
+	}
+
+
+	public void run()
+	{
+	    while (true) {
+		// If there are jobs on the queue, get the next in line and process it
+		if (queue.size() > 0) {
+		    RemoteGreenstoneServerAction remote_greenstone_server_action = (RemoteGreenstoneServerAction) queue.get(0);
+
+		    try {
+			remote_greenstone_server_action.perform();
+
+			// No exceptions were thrown, so the action was successful
+			remote_greenstone_server_action.processed_successfully = true;
+		    }
+		    catch (RemoteGreenstoneServerActionCancelledException exception) {
+			remote_greenstone_server_action.processed_successfully = false;
+		    }
+		    catch (Exception exception) {
+			DebugStream.printStackTrace(exception);
+			JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", exception.getMessage()), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
+			remote_greenstone_server_action.processed_successfully = false;
+		    }
+
+		    // We're done with this action, for better or worse
+		    remote_greenstone_server_action.processed = true;
+		    progress_bar.setAction(null);
+		    queue.remove(0);
+		}
+
+		// Otherwise the queue is empty
+		else {
+		    // Wait until we are notify()ed by addAction that there is a new job on the queue
+		    synchronized (this) {
+			try {
+			    wait();
+			}
+			catch (InterruptedException exception) { }
+		    }
+		}
+	    }
+	}
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   PROGRESS BAR
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static private RemoteGreenstoneServerProgressBar progress_bar = new RemoteGreenstoneServerProgressBar();
+
+
+    static private class RemoteGreenstoneServerProgressBar
+	extends JProgressBar
+    {
+	public RemoteGreenstoneServerProgressBar()
+	{
+	    setBackground(Configuration.getColor("coloring.collection_tree_background", false));
+	    setForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
+	    setString(Dictionary.get("FileActions.No_Activity"));
+	    setStringPainted(true);
+	}
+
+
+	public void setAction(String action)
+	{
+	    if (action != null) {
+		DebugStream.println(action);
+	    }
+
+	    // We cannot call this from the GUI thread otherwise the progress bar won't start
+	    if (SwingUtilities.isEventDispatchThread()) {
+		System.err.println("ERROR: RemoteGreenstoneServerProgressBar.setAction() called from event dispatch thread!");
+		return;
+	    }
+
+	    // Set the string on the progress bar, and start or stop it
+	    if (action == null) {
+		setString(Dictionary.get("FileActions.No_Activity"));
+		setIndeterminate(false);
+	    }
+	    else {
+		setString(action);
+		setIndeterminate(true);
+	    }
+	}
+    }
+
+
+    static public RemoteGreenstoneServerProgressBar getProgressBar()
+    {
+	return progress_bar;
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   ACTIONS
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static private abstract class RemoteGreenstoneServerAction
+    {
+	public String action_output = null;
+	public boolean processed = false;
+	public boolean processed_successfully;
+
+	abstract public void perform()
+	    throws Exception;
+    }
+
+
+    static private class RemoteGreenstoneServerActionCancelledException
+	extends Exception
+    {
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DELETE COLLECTION
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDeleteCollectionAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+
+	public RemoteGreenstoneServerDeleteCollectionAction(String collection_name)
+	{
+	    this.collection_name = collection_name;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Deleting collection " + collection_name + "...");
+
+	    String delete_collection_command = "cmd=delete-collection";
+	    delete_collection_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    action_output = sendCommandToServer(delete_collection_command, null);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DELETE COLLECTION FILE
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDeleteCollectionFileAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File collection_file;
+
+	public RemoteGreenstoneServerDeleteCollectionFileAction(String collection_name, File collection_file)
+	{
+	    this.collection_name = collection_name;
+	    this.collection_file = collection_file;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String collection_file_relative_path = getPathRelativeToDirectory(collection_file, collection_directory_path);
+	    collection_file_relative_path = collection_file_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    progress_bar.setAction("Deleting collection file " + collection_file_relative_path + "...");
+
+	    String delete_collection_file_command = "cmd=delete-collection-file";
+	    delete_collection_file_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    delete_collection_file_command += "&file=" + URLEncoder.encode(collection_file_relative_path, "UTF-8");
+	    action_output = sendCommandToServer(delete_collection_file_command, null);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DOWNLOAD COLLECTION
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDownloadCollectionAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+
+	public RemoteGreenstoneServerDownloadCollectionAction(String collection_name)
+	{
+	    this.collection_name = collection_name;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Downloading remote collection " + collection_name + "...");
+
+	    String download_collection_command = "cmd=download-collection";
+	    download_collection_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    String zip_file_path = Gatherer.getCollectDirectoryPath() + collection_name + ".zip";
+	    action_output = downloadFile(download_collection_command, zip_file_path);
+
+	    // Delete the existing (local) collection
+	    Gatherer.c_man.deleteCollection(collection_name);
+
+	    // Unzip the collection just downloaded
+	    ZipTools.unzipFile(zip_file_path, Gatherer.getCollectDirectoryPath());
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DOWNLOAD COLLECTION ARCHIVES
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDownloadCollectionArchivesAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+
+	public RemoteGreenstoneServerDownloadCollectionArchivesAction(String collection_name)
+	{
+	    this.collection_name = collection_name;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Downloading collection archives for " + collection_name + "...");
+
+	    String download_collection_archives_command = "cmd=download-collection-archives";
+	    download_collection_archives_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    String zip_file_path = Gatherer.getCollectDirectoryPath() + collection_name + "-archives.zip";
+	    action_output = downloadFile(download_collection_archives_command, zip_file_path);
+
+	    // Delete the existing (local) collection archives
+	    Utility.delete(new File(Gatherer.c_man.getCollectionArchivesDirectoryPath()));
+
+	    // Unzip the collection archives just downloaded
+	    ZipTools.unzipFile(zip_file_path, Gatherer.getCollectDirectoryPath());
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DOWNLOAD COLLECTION CONFIGURATIONS
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDownloadCollectionConfigurationsAction
+	extends RemoteGreenstoneServerAction
+    {
+	public RemoteGreenstoneServerDownloadCollectionConfigurationsAction()
+	{
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Downloading collection configurations...");
+
+	    // Delete the existing (local) collect directory
+	    Utility.delete(new File(Gatherer.getCollectDirectoryPath()));
+	    new File(Gatherer.getCollectDirectoryPath()).mkdirs();
+
+	    String download_collection_configurations_command = "cmd=download-collection-configurations";
+	    String zip_file_path = Gatherer.getCollectDirectoryPath() + "collections.zip";
+	    action_output = downloadFile(download_collection_configurations_command, zip_file_path);
+
+	    // Unzip the collection configurations just downloaded
+	    ZipTools.unzipFile(zip_file_path, Gatherer.getCollectDirectoryPath());
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    DOWNLOAD COLLECTION FILE
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerDownloadCollectionFileAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File collection_file;
+
+	public RemoteGreenstoneServerDownloadCollectionFileAction(String collection_name, File collection_file)
+	{
+	    this.collection_name = collection_name;
+	    this.collection_file = collection_file;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String collection_file_relative_path = getPathRelativeToDirectory(collection_file, collection_directory_path);
+	    collection_file_relative_path = collection_file_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    progress_bar.setAction("Downloading collection file " + collection_file_relative_path + "...");
+
+	    String download_collection_file_command = "cmd=download-collection-file";
+	    download_collection_file_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    download_collection_file_command += "&file=" + URLEncoder.encode(collection_file_relative_path, "UTF-8");
+	    String zip_file_name = collection_name + "-" + collection_file.getName() + ".zip";
+	    String zip_file_path = collection_directory_path + zip_file_name;
+	    action_output = downloadFile(download_collection_file_command, zip_file_path);
+
+	    // Unzip the collection file just downloaded
+	    ZipTools.unzipFile(zip_file_path, collection_directory_path);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    GET SCRIPT OPTIONS
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerGetScriptOptionsAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String script_name;
+	private String script_arguments;
+
+	public RemoteGreenstoneServerGetScriptOptionsAction(String script_name, String script_arguments)
+	{
+	    this.script_name = script_name;
+	    this.script_arguments = script_arguments;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Getting options for " + script_name + "...");
+
+	    String get_script_options_command = "cmd=get-script-options";
+	    get_script_options_command += "&script=" + script_name;
+	    get_script_options_command += "&xml=";
+	    get_script_options_command += "&language=" + Configuration.getLanguage();
+	    get_script_options_command += script_arguments;
+	    action_output = sendCommandToServer(get_script_options_command, null);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    MOVE COLLECTION FILE
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerMoveCollectionFileAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File source_collection_file;
+	private File target_collection_file;
+
+	public RemoteGreenstoneServerMoveCollectionFileAction(String collection_name, File source_collection_file, File target_collection_file)
+	{
+	    this.collection_name = collection_name;
+	    this.source_collection_file = source_collection_file;
+	    this.target_collection_file = target_collection_file;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String source_collection_file_relative_path = getPathRelativeToDirectory(source_collection_file, collection_directory_path);
+	    source_collection_file_relative_path = source_collection_file_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    String target_collection_file_relative_path = getPathRelativeToDirectory(target_collection_file, collection_directory_path);
+	    target_collection_file_relative_path = target_collection_file_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    progress_bar.setAction("Moving file " + source_collection_file_relative_path + " -> " + target_collection_file_relative_path + "...");
+
+	    String move_collection_file_command = "cmd=move-collection-file";
+	    move_collection_file_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    move_collection_file_command += "&source=" + URLEncoder.encode(source_collection_file_relative_path, "UTF-8");
+	    move_collection_file_command += "&target=" + URLEncoder.encode(target_collection_file_relative_path, "UTF-8");
+	    action_output = sendCommandToServer(move_collection_file_command, null);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    NEW COLLECTION DIRECTORY
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerNewCollectionDirectoryAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File new_collection_directory;
+
+	public RemoteGreenstoneServerNewCollectionDirectoryAction(String collection_name, File new_collection_directory)
+	{
+	    this.collection_name = collection_name;
+	    this.new_collection_directory = new_collection_directory;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String new_collection_directory_relative_path = getPathRelativeToDirectory(new_collection_directory, collection_directory_path);
+	    new_collection_directory_relative_path = new_collection_directory_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    progress_bar.setAction("Creating new directory " + new_collection_directory_relative_path + "...");
+
+	    String new_collection_directory_command = "cmd=new-collection-directory";
+	    new_collection_directory_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    new_collection_directory_command += "&directory=" + URLEncoder.encode(new_collection_directory_relative_path, "UTF-8");
+	    action_output = sendCommandToServer(new_collection_directory_command, null);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    RUN SCRIPT
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerRunScriptAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private String script_name;
+	private String script_arguments;
+	private GShell shell;
+
+	public RemoteGreenstoneServerRunScriptAction(String collection_name, String script_name, String script_arguments, GShell shell)
+	{
+	    this.collection_name = collection_name;
+	    this.script_name = script_name;
+	    this.script_arguments = script_arguments;
+	    this.shell = shell;
+	}
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Running " + script_name + "...");
+
+	    String run_script_command = "cmd=run-script";
+	    run_script_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    run_script_command += "&script=" + script_name;
+	    run_script_command += "&language=" + Configuration.getLanguage();
+	    run_script_command += script_arguments;
+	    action_output = sendCommandToServer(run_script_command, shell);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    UPLOAD COLLECTION FILE
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerUploadCollectionFilesAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File[] collection_files;
+
+
+	public RemoteGreenstoneServerUploadCollectionFilesAction(String collection_name, File[] collection_files)
+	{
+	    this.collection_name = collection_name;
+	    this.collection_files = collection_files;
+	}
+
+
+	public void perform()
+	    throws Exception
+	{
+	    progress_bar.setAction("Uploading collection files...");
+
+	    // Determine the file paths relative to the collection directory
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String[] collection_file_relative_paths = new String[collection_files.length];
+	    for (int i = 0; i < collection_files.length; i++) {
+		collection_file_relative_paths[i] = getPathRelativeToDirectory(collection_files[i], collection_directory_path);
+	    }
+
+	    // Zip up the files to send to the server
+	    String zip_file_name = collection_name + "-" + System.currentTimeMillis() + ".zip";
+	    String zip_file_path = collection_directory_path + zip_file_name;
+	    ZipTools.zipFiles(zip_file_path, collection_directory_path, collection_file_relative_paths);
+
+	    // Upload the zip file
+	    String upload_collection_file_command = "cmd=upload-collection-file";
+	    upload_collection_file_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    upload_collection_file_command += "&file=" + URLEncoder.encode(zip_file_name, "UTF-8");
+	    upload_collection_file_command += "&directory=";
+	    upload_collection_file_command += "&zip=true";
+	    action_output = uploadFile(upload_collection_file_command, zip_file_path);
+	}
+    }
+
+
+    /**
+     * --------------------------------------------------------------------------------------------
+     *    UPLOAD FILES INTO COLLECTION
+     * --------------------------------------------------------------------------------------------
+     */
+    static private class RemoteGreenstoneServerUploadFilesIntoCollectionAction
+	extends RemoteGreenstoneServerAction
+    {
+	private String collection_name;
+	private File[] source_files;
+	private File target_collection_directory;
+
+
+	public RemoteGreenstoneServerUploadFilesIntoCollectionAction(String collection_name, File[] source_files, File target_collection_directory)
+	{
+	    this.collection_name = collection_name;
+	    this.source_files = source_files;
+	    this.target_collection_directory = target_collection_directory;
+	}
+
+
+	public void perform()
+	    throws Exception
+	{
+	    String collection_directory_path = CollectionManager.getCollectionDirectoryPath(collection_name);
+	    String target_collection_directory_relative_path = getPathRelativeToDirectory(target_collection_directory, collection_directory_path);
+	    target_collection_directory_relative_path = target_collection_directory_relative_path.replaceAll((Utility.isWindows() ? "\\\\" : "\\/"), "|");
+	    progress_bar.setAction("Uploading files into collection...");
+
+	    String zip_file_name = collection_name + "-" + System.currentTimeMillis() + ".zip";
+	    String zip_file_path = Gatherer.getCollectDirectoryPath() + zip_file_name;
+	    DebugStream.println("Zip file path: " + zip_file_path);
+
+	    String base_directory_path = source_files[0].getParentFile().getAbsolutePath();
+	    DebugStream.println("Base directory path: " + base_directory_path);
+	    String[] source_file_relative_paths = new String[source_files.length];
+	    for (int i = 0; i < source_files.length; i++) {
+		DebugStream.println("Source file path: " + source_files[i]);
+		source_file_relative_paths[i] = getPathRelativeToDirectory(source_files[i], base_directory_path);
+	    }
+
+	    ZipTools.zipFiles(zip_file_path, base_directory_path, source_file_relative_paths);
+
+	    String upload_collection_file_command = "cmd=upload-collection-file";
+	    upload_collection_file_command += "&c=" + URLEncoder.encode(collection_name, "UTF-8");
+	    upload_collection_file_command += "&file=" + URLEncoder.encode(zip_file_name, "UTF-8");
+	    upload_collection_file_command += "&directory=" + URLEncoder.encode(target_collection_directory_relative_path, "UTF-8");
+	    upload_collection_file_command += "&zip=true";
+	    action_output = uploadFile(upload_collection_file_command, zip_file_path);
+	}
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   AUTHENTICATION LAYER
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static private PasswordAuthentication remote_greenstone_server_authentication = null;
+    // static private PasswordAuthentication remote_greenstone_server_authentication = new PasswordAuthentication(System.getProperty("user.name"), new char[] { });
+
+
+    static private class RemoteGreenstoneServerAuthenticateTask
+	extends Thread
+    {
+	public void run()
+	{
+	    remote_greenstone_server_authentication = new RemoteGreenstoneServerAuthenticator().getAuthentication();
+	}
+
+
+	static private class RemoteGreenstoneServerAuthenticator
+	    extends GAuthenticator
+	{
+	    public PasswordAuthentication getAuthentication()
+	    {
+		return getPasswordAuthentication();
+	    }
+
+	    protected String getMessageString()
+	    {
+		return Dictionary.get("RemoteGreenstoneServer.Authentication_Message");
+	    }
+ 	}
+    }
+
+
+    static private void authenticateUser()
+	throws RemoteGreenstoneServerActionCancelledException
+    {
+	// If we don't have any authentication information then ask for it now
+	if (remote_greenstone_server_authentication == null) {
+	    try {
+		// We have to do this on the GUI thread
+		SwingUtilities.invokeAndWait(new RemoteGreenstoneServerAuthenticateTask());
+	    }
+	    catch (Exception exception) {
+		DebugStream.printStackTrace(exception);
+	    }
+
+	    // If it is still null then the user has cancelled the authentication, so the action is cancelled
+	    if (remote_greenstone_server_authentication == null) {
+		throw new RemoteGreenstoneServerActionCancelledException();
+	    }
+	}
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   REQUEST LAYER
+    // ----------------------------------------------------------------------------------------------------
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String downloadFile(String gliserver_args, String file_path)
+	throws Exception
+    {
+	while (true) {
+	    // Check that Configuration.gliserver_url is set
+	    if (Configuration.gliserver_url == null) {
+		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
+	    }
+
+	    // Ask for authentication information (if necessary), then perform the action
+	    authenticateUser();
+	    String gliserver_url_string = Configuration.gliserver_url.toString();
+	    String command_output = downloadFileInternal(gliserver_url_string, gliserver_args, file_path);
+
+	    // Check the first line to see if authentication has failed; if so, go around the loop again
+	    if (command_output.startsWith("ERROR: Authentication failed:")) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
+		remote_greenstone_server_authentication = null;
+		continue;
+	    }
+	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
+	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
+		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
+		    // The user has decided to cancel the action
+		    throw new RemoteGreenstoneServerActionCancelledException();
+		}
+
+		// The user has decided to steal the lock... rerun the command with "&steal_lock="
+		gliserver_args += "&steal_lock=";
+		continue;
+	    }
+	    // Handle other types of errors by throwing an exception
+	    else if (command_output.startsWith("ERROR: ")) {
+		throw new Exception(command_output.substring("ERROR: ".length()));
+	    }
+
+	    // There were no exceptions thrown so the action must have succeeded
+	    return command_output;
+	}
+    }
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String sendCommandToServer(String gliserver_args, GShell shell)
+	throws Exception
+    {
+	while (true) {
+	    // Check that Configuration.gliserver_url is set
+	    if (Configuration.gliserver_url == null) {
+		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
+	    }
+
+	    // Ask for authentication information (if necessary), then perform the action
+	    authenticateUser();
+	    String gliserver_url_string = Configuration.gliserver_url.toString();
+	    String command_output = sendCommandToServerInternal(gliserver_url_string, gliserver_args, shell);
+	    // System.err.println("Command output: " + command_output);
+
+	    // Check the first line to see if authentication has failed; if so, go around the loop again
+	    if (command_output.startsWith("ERROR: Authentication failed:")) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
+		remote_greenstone_server_authentication = null;
+		continue;
+	    }
+	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
+	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
+		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
+		    // The user has decided to cancel the action
+		    throw new RemoteGreenstoneServerActionCancelledException();
+		}
+
+		// The user has decided to steal the lock... rerun the command with "&steal_lock="
+		gliserver_args += "&steal_lock=";
+		continue;
+	    }
+	    // Handle other types of errors by throwing an exception
+	    else if (command_output.startsWith("ERROR: ")) {
+		throw new Exception(command_output.substring("ERROR: ".length()));
+	    }
+
+	    // There were no exceptions thrown so the action must have succeeded
+	    return command_output;
+	}
+    }
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String uploadFile(String gliserver_args, String file_path)
+	throws Exception
+    {
+	while (true) {
+	    // Check that Configuration.gliserver_url is set
+	    if (Configuration.gliserver_url == null) {
+		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
+	    }
+
+	    // Ask for authentication information (if necessary), then perform the action
+	    authenticateUser();
+	    String gliserver_url_string = Configuration.gliserver_url.toString();
+	    String command_output = uploadFileInternal(gliserver_url_string, gliserver_args, file_path);
+	    // System.err.println("Command output: " + command_output);
+
+	    // Check the first line to see if authentication has failed; if so, go around the loop again
+	    if (command_output.startsWith("ERROR: Authentication failed:")) {
+		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
+		remote_greenstone_server_authentication = null;
+		continue;
+	    }
+	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
+	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
+		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
+		    // The user has decided to cancel the action
+		    throw new RemoteGreenstoneServerActionCancelledException();
+		}
+
+		// The user has decided to steal the lock... rerun the command with "&steal_lock="
+		gliserver_args += "&steal_lock=";
+		continue;
+	    }
+	    // Handle other types of errors by throwing an exception
+	    else if (command_output.startsWith("ERROR: ")) {
+		throw new Exception(command_output.substring("ERROR: ".length()));
+	    }
+
+	    // There were no exceptions thrown so the action must have succeeded
+	    return command_output;
+	}
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   NETWORK LAYER
+    // ----------------------------------------------------------------------------------------------------
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String downloadFileInternal(String download_cgi, String cgi_args, String file_path)
+	throws Exception
+    {
+	DebugStream.println("gliserver URL: " + download_cgi);
+	System.err.println("gliserver args: " + cgi_args);
+
+	// Add username and password
+	cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
+	cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
+
+	URL download_url = new URL(download_cgi);
+	URLConnection dl_connection = download_url.openConnection();
+	dl_connection.setDoOutput(true);
+	OutputStream dl_os = dl_connection.getOutputStream();
+
+	PrintWriter dl_out = new PrintWriter(dl_os);
+	dl_out.println(cgi_args);
+	dl_out.close();
+
+	// Download result from running cgi script
+	InputStream dl_is = dl_connection.getInputStream();
+	BufferedInputStream dl_bis = new BufferedInputStream(dl_is);
+	DataInputStream dl_dbis = new DataInputStream(dl_bis);
+
+	String first_line = "";
+	byte[] buf = new byte[1024];
+	int len = dl_dbis.read(buf);
+	if (len >= 0) {
+	    String first_chunk = new String(buf, 0, len);
+	    first_line = first_chunk.substring(0, ((first_chunk.indexOf("\n") != -1) ? first_chunk.indexOf("\n") : len));
+
+	    // Save the data to file
+	    FileOutputStream zip_fos = new FileOutputStream(file_path);
+	    BufferedOutputStream zip_bfos = new BufferedOutputStream(zip_fos);
+
+	    while (len >= 0) {
+		zip_bfos.write(buf, 0, len);
+		len = dl_dbis.read(buf);
+	    }
+
+	    zip_bfos.close();
+	    zip_fos.close();
+	}
+
+	dl_dbis.close();
+	dl_bis.close();
+	dl_is.close();
+	return first_line;
+    }
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String sendCommandToServerInternal(String gliserver_url_string, String cgi_args, GShell shell)
+	throws Exception
+    {
+	DebugStream.println("gliserver URL: " + gliserver_url_string);
+	System.err.println("gliserver args: " + cgi_args);
+
+	// Add username and password
+	cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
+        cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
+
+	URL gliserver_url = new URL(gliserver_url_string + "?" + cgi_args);
+	URLConnection gliserver_connection = gliserver_url.openConnection();
+
+	// Read the output of the command from the server, and return it
+	StringBuffer command_output_buffer = new StringBuffer(2048);
+	InputStream gliserver_is = gliserver_connection.getInputStream();
+	BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
+	String gliserver_output_line = gliserver_in.readLine();
+	while (gliserver_output_line != null) {
+	    if (shell != null) {
+		shell.fireMessage(gliserver_output_line);
+		if (shell.hasSignalledStop()) {
+		    throw new RemoteGreenstoneServerActionCancelledException();
+		}
+	    }
+	    command_output_buffer.append(gliserver_output_line + "\n");
+	    gliserver_output_line = gliserver_in.readLine();
+	}
+	gliserver_in.close();
+
+	return command_output_buffer.toString();
+    }
+
+
+    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
+    static private String uploadFileInternal(String upload_cgi, String cgi_args, String file_path)
+	throws Exception
+    {
+	DebugStream.println("gliserver URL: " + upload_cgi);
+	System.err.println("gliserver args: " + cgi_args);
+
+	// Add username and password
+	cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
+	cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
+
+	// Open a HTTP connection to the URL
+	URL url = new URL(upload_cgi);
+	HttpURLConnection gliserver_connection = (HttpURLConnection) url.openConnection();
+
+	gliserver_connection.setDoInput(true);         // Allow Inputs
+	gliserver_connection.setDoOutput(true);        // Allow Outputs
+	gliserver_connection.setUseCaches(false);      // Don't use a cached copy.
+
+	gliserver_connection.setRequestProperty("Connection", "Keep-Alive");	    
+
+	DataOutputStream dos = new DataOutputStream(gliserver_connection.getOutputStream());
+	dos.writeBytes(cgi_args + "\n");
+
+	// Send zip file to server
+	File file = new File(file_path);
+	FileInputStream fileInputStream = new FileInputStream(file);
+
+	// create a buffer of maximum size
+	final int maxBufferSize = 1024;
+	int bytesAvailable = fileInputStream.available();
+	int bufferSize = Math.min(bytesAvailable, maxBufferSize);
+	byte[] buffer = new byte[bufferSize];
+
+	// read file and write it into form...
+	int bytesRead = fileInputStream.read(buffer, 0, bufferSize);
+	while (bytesRead > 0) {
+	    dos.write(buffer, 0, bufferSize);
+	    bytesAvailable = fileInputStream.available();
+	    bufferSize = Math.min(bytesAvailable, maxBufferSize);
+	    bytesRead = fileInputStream.read(buffer, 0, bufferSize);
+	}
+
+	// close streams
+	fileInputStream.close();
+	dos.flush();
+	dos.close();
+
+	// Read the output of the command from the server, and return it
+	String command_output = "";
+	InputStream gliserver_is = gliserver_connection.getInputStream();
+	BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
+	String gliserver_output_line = gliserver_in.readLine();
+	while (gliserver_output_line != null) {
+	    command_output += gliserver_output_line + "\n";
+	    gliserver_output_line = gliserver_in.readLine();
+	}
+	gliserver_in.close();
+
+	return command_output;
+    }
+
+
+    // ----------------------------------------------------------------------------------------------------
+    //   UTILITIES
+    // ----------------------------------------------------------------------------------------------------
+
+
+    static private String getPathRelativeToDirectory(File file, String directory_path)
+    {
+	String file_path = file.getAbsolutePath();
+	if (!file_path.startsWith(directory_path)) {
+	    System.err.println("ERROR: File path " + file_path + " is not a child of " + directory_path);
+	    return file_path;
+	}
+
+	String relative_file_path = file_path.substring(directory_path.length());
+	if (relative_file_path.startsWith(File.separator)) {
+	    relative_file_path = relative_file_path.substring(File.separator.length());
+	}
+	return relative_file_path;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionArchives.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionArchives.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionArchives.java	(revision 21112)
@@ -0,0 +1,148 @@
+package org.greenstone.gatherer.remote;
+
+import java.io.*;
+import java.util.zip.*;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/**
+ */
+public class ZipCollectionArchives
+{
+    static public void main(String[] args)
+    {
+	if (args.length != 3) {
+	    System.err.println("Usage: ZipCollectionArchives <zip-file> <collect-directory-path> <collection-name>");
+	    return;
+	}
+
+	String zip_file_path = args[0];
+	String collect_directory_path = args[1];
+	String collection_name = args[2];
+
+	if (!collect_directory_path.endsWith(File.separator)) {
+	    collect_directory_path += File.separator;
+	}
+
+ 	try {
+	    ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip_file_path));
+
+	    // Add archives directory, with doc.xml files only
+	    String archives_relative_path = collection_name + File.separator + "archives";
+	    ZipTools.ZipFilter collection_archives_zip_filter = new CollectionArchivesZipFilter();
+	    ZipTools.addFileToZip(zos, collect_directory_path, archives_relative_path, collection_archives_zip_filter);
+
+	    zos.close();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    static public class CollectionArchivesZipFilter
+	extends ZipTools.NullZipFilter
+    {
+	private byte[] bytes_remaining = new byte[0];
+	private int section_depth = 0;
+	private boolean in_section_content = false;
+
+
+	public boolean shouldIncludeFile(String relative_file_path)
+	{
+	    // Only doc.xml files are included
+	    return (relative_file_path.equals("doc.xml") || relative_file_path.endsWith(File.separator + "doc.xml"));
+	}
+
+
+	public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read)
+	{
+	    try {
+		// Read the next chunk of bytes from the file
+		byte[] bytes_read_this_time = new byte[bytes_to_read];
+		int num_bytes_read_this_time = bis.read(bytes_read_this_time, 0, bytes_to_read);
+		if (num_bytes_read_this_time == -1) {
+		    // We must be at the end of the file
+		    return -1;
+		}
+
+		// Add this chunk to the bytes remaining from last time
+		byte[] bytes_unprocessed = new byte[bytes_remaining.length + num_bytes_read_this_time];
+		for (int i = 0; i < bytes_remaining.length; i++) {
+		    bytes_unprocessed[i] = bytes_remaining[i];
+		}
+		for (int i = 0; i < num_bytes_read_this_time; i++) {
+		    bytes_unprocessed[bytes_remaining.length + i] = bytes_read_this_time[i];
+		}
+
+		// Split the chunk into lines and filter them
+		int num_bytes_processed = 0;
+		int num_bytes_unfiltered = 0;
+		int last_line_start = 0;
+		for (int i = 0; (i < bytes_unprocessed.length && num_bytes_unfiltered < bytes_to_read); i++) {
+		    // Found the end of a line
+		    if (bytes_unprocessed[i] == '\n') {
+			int line_length = i - last_line_start + 1;
+			String line = new String(bytes_unprocessed, last_line_start, line_length);
+
+			// If this line isn't filtered, write it to the data array
+			if (!filterFileLine(line)) {
+			    for (int j = 0; (j < line_length && num_bytes_unfiltered < bytes_to_read); j++) {
+				data[num_bytes_unfiltered] = bytes_unprocessed[last_line_start + j];
+				num_bytes_unfiltered++;
+				num_bytes_processed++;
+			    }
+			}
+			// Otherwise ignore it
+			else {
+			    num_bytes_processed = i + 1;
+			}
+
+			last_line_start = i + 1;
+		    }
+		}
+
+		// Create a new byte array containing any bytes remaining (these will be processed next time)
+		int num_bytes_remaining = bytes_unprocessed.length - num_bytes_processed;
+		bytes_remaining = new byte[num_bytes_remaining];
+		for (int i = 0; i < num_bytes_remaining; i++) {
+		    bytes_remaining[i] = bytes_unprocessed[num_bytes_processed + i];
+		}
+
+		return num_bytes_unfiltered;
+	    }
+	    catch (Exception exception) {
+		exception.printStackTrace();
+	    }
+
+	    return -1;
+	}
+
+
+	private boolean filterFileLine(String line)
+	{
+	    boolean filter_line = false;
+
+	    if (line.indexOf("<Section>") != -1) {
+		section_depth++;
+	    }
+	    if (line.indexOf("<Content>") != -1) {
+		in_section_content = true;
+	    }
+
+	    // If we're in a subsection or in a content element, filter this line
+	    if (section_depth > 1 || in_section_content) {
+		filter_line = true;
+	    }
+
+	    if (line.indexOf("</Content>") != -1) {
+		in_section_content = false;
+	    }
+	    if (line.indexOf("</Section>") != -1) {
+		section_depth--;
+	    }
+
+	    return filter_line;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionConfigurations.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionConfigurations.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionConfigurations.java	(revision 21112)
@@ -0,0 +1,72 @@
+package org.greenstone.gatherer.remote;
+
+import java.io.*;
+import java.util.zip.*;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/**
+ */
+public class ZipCollectionConfigurations
+{
+    static public void main(String[] args)
+    {
+	if (args.length != 2) {
+	    System.err.println("Usage: ZipCollectionConfigurations <zip-file> <collect-directory-path>");
+	    return;
+	}
+
+	String zip_file_path = args[0];
+	String collect_directory_path = args[1];
+
+	if (!collect_directory_path.endsWith(File.separator)) {
+	    collect_directory_path += File.separator;
+	}
+	File collect_directory = new File(collect_directory_path);
+
+ 	try {
+	    ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip_file_path));
+	    ZipTools.ZipFilter collection_etc_zip_filter = new CollectionEtcZipFilter();
+	    ZipTools.ZipFilter collection_metadata_zip_filter = new CollectionMetadataZipFilter();
+
+	    // Add the etc and metadata directories from each of the collections
+	    File[] collection_directories = collect_directory.listFiles();
+	    for (int i = 0; i < collection_directories.length; i++) {
+		if (collection_directories[i].isDirectory()) {
+		    String etc_relative_path = collection_directories[i].getName() + File.separator + "etc";
+		    ZipTools.addFileToZip(zos, collect_directory_path, etc_relative_path, collection_etc_zip_filter);
+
+		    String metadata_relative_path = collection_directories[i].getName() + File.separator + "metadata";
+		    ZipTools.addFileToZip(zos, collect_directory_path, metadata_relative_path, collection_metadata_zip_filter);
+		}
+	    }
+
+	    zos.close();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    static private class CollectionEtcZipFilter
+	extends ZipTools.NullZipFilter
+    {
+	public boolean shouldIncludeFile(String relative_file_path)
+	{
+	    // Only collect.cfg files are included
+	    return (relative_file_path.equals("collect.cfg") || relative_file_path.endsWith(File.separator + "collect.cfg"));
+	}
+    }
+
+
+    static private class CollectionMetadataZipFilter
+	extends ZipTools.NullZipFilter
+    {
+	public boolean shouldIncludeFileContent(String relative_file_path)
+	{
+	    // We don't include content for any of the metadata set files
+	    return false;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionShell.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionShell.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipCollectionShell.java	(revision 21112)
@@ -0,0 +1,74 @@
+package org.greenstone.gatherer.remote;
+
+import java.io.*;
+import java.util.zip.*;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/**
+ */
+public class ZipCollectionShell
+{
+    static public void main(String[] args)
+    {
+	if (args.length != 3) {
+	    System.err.println("Usage: ZipCollectionShell <zip-file> <collect-directory-path> <collection-name>");
+	    return;
+	}
+
+	String zip_file_path = args[0];
+	String collect_directory_path = args[1];
+	String collection_name = args[2];
+
+	if (!collect_directory_path.endsWith(File.separator)) {
+	    collect_directory_path += File.separator;
+	}
+
+ 	try {
+	    ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip_file_path));
+	    ZipTools.ZipFilter null_zip_filter = new ZipTools.NullZipFilter();
+
+	    // Add archives directory, with doc.xml files only
+	    String archives_relative_path = collection_name + File.separator + "archives";
+	    ZipTools.ZipFilter collection_archives_zip_filter = new ZipCollectionArchives.CollectionArchivesZipFilter();
+	    ZipTools.addFileToZip(zos, collect_directory_path, archives_relative_path, collection_archives_zip_filter);
+
+	    // Add etc/collect.cfg file
+	    String collect_cfg_relative_path = collection_name + File.separator + "etc" + File.separator + "collect.cfg";
+	    ZipTools.addFileToZip(zos, collect_directory_path, collect_cfg_relative_path, null_zip_filter);
+
+	    // Add images directory
+	    String images_relative_path = collection_name + File.separator + "images";
+	    ZipTools.addFileToZip(zos, collect_directory_path, images_relative_path, null_zip_filter);
+
+	    // Add import directory, with zero-length files except for metadata.xml
+	    String import_relative_path = collection_name + File.separator + "import";
+	    ZipTools.ZipFilter collection_import_zip_filter = new CollectionImportZipFilter();
+	    ZipTools.addFileToZip(zos, collect_directory_path, import_relative_path, collection_import_zip_filter);
+
+	    // Add index/build.cfg file
+	    String build_cfg_relative_path = collection_name + File.separator + "index" + File.separator + "build.cfg";
+	    ZipTools.addFileToZip(zos, collect_directory_path, build_cfg_relative_path, null_zip_filter);
+
+	    // Add metadata directory
+	    String metadata_relative_path = collection_name + File.separator + "metadata";
+	    ZipTools.addFileToZip(zos, collect_directory_path, metadata_relative_path, null_zip_filter);
+
+	    zos.close();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    static private class CollectionImportZipFilter
+	extends ZipTools.NullZipFilter
+    {
+	public boolean shouldIncludeFileContent(String relative_file_path)
+	{
+	    // We only include file content for metadata.xml files
+	    return (relative_file_path.equals("metadata.xml") || relative_file_path.endsWith(File.separator + "metadata.xml"));
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipFiles.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipFiles.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/remote/ZipFiles.java	(revision 21112)
@@ -0,0 +1,29 @@
+package org.greenstone.gatherer.remote;
+
+import java.io.File;
+import java.util.ArrayList;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/**
+ */
+public class ZipFiles
+{
+    static public void main(String[] args)
+    {
+	if (args.length < 3) {
+	    System.err.println("Usage: ZipFiles <zip-file> <base-directory-path> (<files> ...)");
+	    return;
+	}
+
+	String zip_file_path = args[0];
+	String base_directory_path = args[1];
+	ArrayList relative_file_paths_list = new ArrayList();
+	for (int i = 2; i < args.length; i++) {
+	    relative_file_paths_list.add(args[i]);
+	}
+
+	String[] relative_file_paths = (String[]) relative_file_paths_list.toArray(new String[0]);
+	ZipTools.zipFiles(zip_file_path, base_directory_path, relative_file_paths);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBasicProgressMonitor.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBasicProgressMonitor.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBasicProgressMonitor.java	(revision 21112)
@@ -0,0 +1,128 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+
+import java.awt.Component;
+import java.util.ArrayList;
+import javax.swing.JProgressBar;
+
+/** This is the most basic implementation of <i>GShellProgressMonitor</i> in that it doesn't even attempt to parse the process messages. Instead it has one graphic which is displayed once the start() method is called, and another which appears after the stop().
+ */
+public class GBasicProgressMonitor 
+    extends JProgressBar 
+    implements GShellProgressMonitor {
+    /** Whether the process has been asked to stop. */
+    private boolean stop = false;
+    /** Constructor **/
+    public GBasicProgressMonitor() {
+	super();
+    }
+    /** Don't do anything if we are given a progress bar, we have our own.
+     * @param progress_bar A JProgressBar
+     */
+    public void addProgressBar(JProgressBar progress_bar) {
+    }
+    /** Determine the script exit value according to the progress monitor. This gets around a problem where several script failures actually result with a successful exit value.
+     * @return A <i>int</i> with a value of zero if and only if the script was successful.
+     */
+    public int exitValue() {
+	// Always return true as we don't know any better.
+	return 0;
+    }
+    /** As we are probably accessing this via an interface we need to gain
+     * access to the component itself.
+     */
+    public Component getProgress() {
+	return this;
+    }
+
+    public JProgressBar getSharedProgress() {
+	return null;
+    }
+
+    public void messageOnProgressBar(String message)
+    {
+	if ((message!=null) & (message!="")) {
+	    setString(message);
+	}
+	else {
+	    setString(null);
+	}
+    }
+
+
+    /** Method to determine the state of the stop flag, which may be set by the visual component that created this monitor.
+     * @return A <strong>boolean</strong> indicating if the process has been asked to stop.
+     */
+    public boolean hasSignalledStop() {
+	return stop;
+    }
+    /** Inform the progress bar that it should programatically increment progress by one step. */
+    public void increment() {
+	// We can't do anything about this.
+    }
+    /** The parse method is how more complex progress bars figure out how far they have progressed. However we don't.
+     * @param queue a queue which at the moment should contain a single GShellEvent
+     */
+    public void process(ArrayList queue) {
+    }
+
+    public void reset() {};
+
+    public void saving() {
+    }
+
+    /** Since the creator of this process monitor is actually in the GUI, this class provides the simpliest way to send a cancel process message between the two.
+     * @param state The desired state of the stop flag as a <strong>boolean</strong>.
+     */
+    public void setStop(boolean state) {
+	stop = state;
+    }
+    /** This method tells this class that the job it is showing the progress
+     * for has begun.
+     */
+    public void start() {
+	setIndeterminate(true);
+    }
+    /** This method tells this class that the job it is showing the progress
+     * for has come to an end.
+     */
+    public void stop() {
+	setIndeterminate(false);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBuildProgressMonitor.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBuildProgressMonitor.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GBuildProgressMonitor.java	(revision 21112)
@@ -0,0 +1,598 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+import java.awt.Component;
+import java.util.ArrayList;
+import javax.swing.JProgressBar;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.util.StaticStrings;
+/** This implementation of <i>GShellProgressMonitor</i> is designed to parse and translate the progress of a buildcol.pl call.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class GBuildProgressMonitor
+    implements GShellProgressMonitor {
+    /** The minimum value = 0 */
+    static final private int MIN = 0;
+    /** In order to get a smoothish progress bar, set the maximum to a large number = 1000000 */
+    static final private int MAX = 1000000;
+    /** The various states the build progress state machine can be in, starting at the base state. */
+    static final int BASE                          = -1;
+    static final int BUILD                         =  0;
+    static final int COMPRESSTEXT                  = 10;
+    static final int COMPRESSTEXT_COLLECTTEXTSTATS = 11;
+    static final int COMPRESSTEXT_CREATINGCOMPRESS = 12;
+    static final int COMPRESSTEXT_COMPRESSINGTEXT  = 13;
+    static final int INDEX                         = 20;
+    static final int INDEX_CREATINGINDEXDIC        = 21;
+    static final int INDEX_INVERTINGTEXT           = 22;
+    static final int INDEX_CREATETHEWEIGHTS        = 23;
+    static final int INDEX_CREATESTEMMEDDIC        = 24;
+    static final int INDEX_CREATINGSTEMINDX        = 25;
+    static final int CREATEINFODATA                = 30;
+    static final int PHIND                         = 40;
+    static final int PHIND_EXTRACTINGVOCAB         = 41;
+    static final int PHIND_EXTRACTINGPHRASE        = 42;
+    static final int PHIND_SORTANDRENUMBER         = 43;
+    static final int PHIND_PHRASEDATABASES         = 44;
+    static final int PHIND_WORDLEVELINDEXES        = 45;
+    static final int PHIND_DOCINFODATABASES        = 46;
+    static final int CREATINGAUXILARY              = 50;
+    /** Now the language independant sentinel strings. */
+    static final String BADARGUMENT_ELEMENT    = "BadArgument";
+    static final String BADARGUMENTVALUE_ELEMENT = "BadArgumentValue";
+    static final String BADCLASSIFIER_ELEMENT = "BadClassifier";
+    static final String BADPLUGIN_ELEMENT = "BadPlugin";
+    static final String BUILD_ELEMENT          = "Build";
+    static final String COLLECTTEXTSTATS_VALUE = "CollectTextStats";
+    static final String COMPRESSINGTEXT_VALUE  = "CompressingText";
+    static final String COMPRESSTEXT_VALUE     = "CompressText";
+    static final String CREATINGAUXILARY_VALUE = "CreatingAuxilary";
+    static final String CREATINGCOMPRESS_VALUE = "CreatingCompress";
+    static final String CREATINGINDEXDIC_VALUE = "CreatingIndexDic";
+    static final String CREATINGSTEMINDX_VALUE = "CreatingStemIndx";
+    static final String CREATEINFODATA_VALUE   = "CreateInfoData";
+    static final String CREATESTEMMEDDIC_VALUE = "CreateStemmedDic";
+    static final String CREATETHEWEIGHTS_VALUE = "CreateTheWeights";
+    static final String DOCINFODATABASES_VALUE = "DocInfoDatabases";
+    static final String EXTRACTINGPHRASE_VALUE = "ExtractingPhrase";
+    static final String EXTRACTINGVOCAB_VALUE  = "ExtractingVocab";
+    static final String FATALERROR_ELEMENT     = "FatalError";
+    static final String INDEX_VALUE            = "Index";
+    static final String INVERTINGTEXT_VALUE    = "InvertingText";
+    static final String NAME_ATTRIBUTE         = "name";
+    static final String PHASE_ELEMENT          = "Phase";
+    static final String PHIND_VALUE            = "Phind";
+    static final String PHRASEDATABASES_VALUE  = "PhraseDatabases";
+    static final String SKIPCREATINGCOMP_VALUE = "SkipCreatingComp";
+    static final String SORTANDRENUMBER_VALUE  = "SortAndRenumber";
+    static final String SOURCE_ATTRIBUTE       = "source";
+    static final String STAGE_ELEMENT          = "Stage";
+    static final String WARNING_ELEMENT        = "Warning";   
+    static final String WORDLEVELINDEXES_VALUE = "WordLevelIndexes";
+
+    static final private String ARGUMENT_ATTRIBUTE = "a";
+    static final private String CLASSIFIER_ATTRIBUTE = "c";
+    static final private String PLUGIN_ATTRIBUTE = "p";
+
+    /** Indicates if the GUI has asked the process this object monitors to stop. */
+    private boolean stop = false;
+    /** The current number of stages we have completed. */
+    private int current_stages = 0;
+    /** The number of stages we are expecting to complete. */
+    private int expected_stages = 0;
+
+    private int threshold = Configuration.SYSTEMS_MODE;
+    /** The current state we are in. The state will change depending on the next flag recieved. */
+    private int state = BASE;
+    /** The progress bar this monitor updates. */
+    private JProgressBar progress_bar;
+    /** A progress bar that is shared between this this listener and the Import monitor. */
+    private JProgressBar shared_progress_bar;
+
+    /** Construct a new GBuildProgressMonitor. */
+    public GBuildProgressMonitor(JProgressBar shared_progress_bar) {
+	this.shared_progress_bar = shared_progress_bar;
+	progress_bar = new JProgressBar();
+	progress_bar.setMaximum(MAX);
+	progress_bar.setMinimum(MIN);
+	progress_bar.setString(null);
+	progress_bar.setStringPainted(true);
+	setValue(MIN);
+    }
+
+    /** Method to register a new progress bar with this monitor.
+     * @param progress_bar The new <strong>JProgressBar</strong>.
+     */	 
+    public void addProgressBar(JProgressBar progress_bar) {
+	this.progress_bar = progress_bar;
+	progress_bar.setMaximum(MAX);
+	progress_bar.setMinimum(MIN);
+	progress_bar.setStringPainted(false);
+	progress_bar.setString(null);
+	setValue(MIN);
+    }
+
+    /** Determine the script exit value according to the progress monitor. This gets around a problem where several script failures actually result with a successful exit value.
+     * @return A <i>int</i> with a value of zero if and only if the script was successful.
+     */
+    public int exitValue() {
+	if(state == BASE) {
+	    return 0;
+	}
+	return 1;
+    }
+    /** Method to retrieve whatever control is being used as the progress indicator. Usually a <strong>JProgressBar</strong> but there may be others implemented later.
+     * @return A <strong>Component</strong> on which the progress of the process is being displayed.
+     */
+    public Component getProgress() {
+	return progress_bar;
+    }
+
+    public JProgressBar getSharedProgress() {
+	return shared_progress_bar;
+    }
+
+    public void messageOnProgressBar(String message)
+    {
+	if ((message!=null) & (message!="")) {
+	    progress_bar.setString(message);
+	    shared_progress_bar.setString(message);
+	}
+	else {
+	    progress_bar.setString(null);
+	    shared_progress_bar.setString(null);
+	}
+    }
+
+    /** Method to determine the state of the stop flag, which may be set by the visual component that created this monitor.
+     * @return A <strong>boolean</strong> indicating if the process has been asked to stop.
+     */
+    public synchronized boolean hasSignalledStop() {
+	return stop;
+    }
+    /** Inform the progress bar that it should programatically increment progress by one step. */
+    public void increment() {
+	// There is no reason this should be called for a build.
+    }
+
+    /** This method is used to 'feed in' a line of text captured from the process.
+     * @param queue a queue which at the moment should contain a single GShellEvent
+     */
+    public void process(ArrayList queue) {
+	GShellEvent event = (GShellEvent) queue.get(0);
+	// Remove 'build.pl> ' bit
+	String line = event.getMessage();
+	line = line.substring(line.indexOf(StaticStrings.GREATER_THAN_CHARACTER) + 1);
+	line = line.trim();
+	// We are only really interested in processing pseudo-XML tags
+	if(line.startsWith(StaticStrings.LESS_THAN_CHARACTER) && line.endsWith(StaticStrings.GREATER_THAN_CHARACTER)) {
+	    // All events are vetoed
+	    event.veto();
+	    
+	    // Create a new element from it
+	    GShellElement element = new GShellElement(line);
+	    // Now change states within the state machine as appropriate. A state change may subsequently incur a progress update.
+	    String name = null;
+	    String value = null;
+	    switch(state) {
+	    case BASE:
+		// The only thing we would expect is a build message
+		name = element.getElementName();
+		if(name.equals(BUILD_ELEMENT)) {
+		    progress_bar.setIndeterminate(false);
+		    state = BUILD;
+		    // Produce a lower detail message if required
+		    if(Configuration.getMode() <= threshold && !stop) {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.BuildBegun1"), event.getStatus()));
+		    }
+		}
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true);
+		}
+		name = null;
+		break;
+	    case BUILD:
+		// The only thing we would expect is a stage element or a 
+		// bad classifier arg name
+		name = element.getElementName();
+
+		if(name.equals(STAGE_ELEMENT)) {
+		    progress_bar.setIndeterminate(false);
+		    // Now we determine what stage we are moving to
+		    value = element.getAttribute(NAME_ATTRIBUTE);
+		    if(value.equals(COMPRESSTEXT_VALUE)) {
+			state = COMPRESSTEXT;
+			if(Configuration.getMode() <= threshold) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.CompressText"), event.getStatus()));
+			}
+		    }
+		    else if(value.equals(INDEX_VALUE)) {
+			state = INDEX;
+			if(Configuration.getMode() <= threshold) {
+			    String args[] = new String[1];
+			    args[0] = element.getAttribute(SOURCE_ATTRIBUTE);
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.Index", args), event.getStatus()));
+			    args = null;
+			}
+		    }
+		    else if(value.equals(CREATEINFODATA_VALUE)) {
+			state = CREATEINFODATA;
+			if(Configuration.getMode() <= threshold) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.InfoDatabase"), event.getStatus()));
+			}
+		    }
+		    else if(value.equals(PHIND_VALUE)) {
+			state = PHIND;
+			if(Configuration.getMode() <= threshold) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.Phind"), event.getStatus()));
+			}
+		    }
+		    else if(value.equals(CREATINGAUXILARY_VALUE)) {
+			state = CREATINGAUXILARY;
+			if(Configuration.getMode() <= threshold) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.Auxilary"), event.getStatus()));
+			}
+		    }
+		    else {
+			// Unknown command, go indeterminate
+			DebugStream.println("Unknown value: " + value);
+			progress_bar.setIndeterminate(true);
+		    }
+		    value = null;
+		}
+		// The build end message indicates a successfulish build.
+		else if(name.equals(BUILD_ELEMENT) && element.isClosed()) {
+		    progress_bar.setIndeterminate(false);
+		    setValue(MAX);
+		    state = BASE;
+		    if(Configuration.getMode() <= threshold) {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.BuildComplete1"), event.getStatus()));
+		    }
+		}
+		// We had a bad argument to a plugin/classifier
+		else if (name.equals(BADARGUMENT_ELEMENT)) {
+		    if(Configuration.getMode() <= threshold) {
+			String args[] = new String[1];
+			args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgument", args), event.getStatus()));
+			args = null;
+		    }
+		}
+		// We had a bad argument value to a plugin/clasifier
+		else if (name.equals(BADARGUMENTVALUE_ELEMENT)) {
+		    if(Configuration.getMode() <= threshold) {
+			String args[] = new String[1];
+			args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgumentValue", args), event.getStatus()));
+			args = null;
+		    }
+		}
+		// And this one tellsa us the plugin
+		else if (name.equals(BADPLUGIN_ELEMENT)) {
+		    if(Configuration.getMode() <= threshold) {
+			String args[] = new String[1];
+			args[0] = element.getAttribute(PLUGIN_ATTRIBUTE);    
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadPluginOptions", args), event.getStatus()));
+			args = null;
+		    }
+		}
+		// or the classifier
+		else if (name.equals(BADCLASSIFIER_ELEMENT)) {
+		    if(Configuration.getMode() <= threshold) {
+			String args[] = new String[1];
+			args[0] = element.getAttribute(CLASSIFIER_ATTRIBUTE);    
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadClassifierOptions", args), event.getStatus()));
+			args = null;
+		    }
+		}
+		
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true);
+		}
+		name = null;
+		break;
+	    case COMPRESSTEXT:
+		// We are either dealing with a phase, aa fatal error or an end stage
+		name = element.getElementName();
+		if(name.equals(PHASE_ELEMENT)) {
+		    // There are three phases within compressing text (or of four possible ones)
+		    value = element.getAttribute(NAME_ATTRIBUTE);
+		    if(value.equals(COLLECTTEXTSTATS_VALUE)) {
+			// Nothing to do
+		    }
+		    else if(value.equals(CREATINGCOMPRESS_VALUE) || value.equals(SKIPCREATINGCOMP_VALUE)) {
+			// One third complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.3d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(COMPRESSINGTEXT_VALUE)) {
+			// Two thirds complete
+			progress_bar.setIndeterminate(false);
+		        setValue((int)(((double)current_stages + 0.6d) * MAX) / expected_stages);
+		    }
+		    else {
+			// Unknown command, go indeterminate
+			DebugStream.println("Unknown value: " + value);
+			progress_bar.setIndeterminate(true);
+		    }
+		    value = null;
+		}
+		// Note that a fatal error is always followed by an end-stage, but not an end build. Thus the final state will be build (not base) and calls to exitValue() with indicate failure.
+		else if(name.equals(FATALERROR_ELEMENT)) {
+		    // On simplier modes produce a meaningful message
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Build.FatalError"), event.getStatus()));
+		}
+		else if(name.equals(STAGE_ELEMENT) && element.isClosed()) {
+		    // Increase progress so that stage is complete.
+		    current_stages++;
+		    progress_bar.setIndeterminate(false);
+		    setValue((current_stages * MAX) / expected_stages);
+		    state = BUILD;
+		}
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true);
+		}
+		name = null;
+		break;
+	    case INDEX:
+		/** @todo - If on lower modes output a meaningful message. */
+		name = element.getElementName();
+		if(name.equals(PHASE_ELEMENT)) {
+		    value = element.getAttribute(NAME_ATTRIBUTE);
+		    if(value.equals(CREATINGINDEXDIC_VALUE)) {
+			// Nothing to do
+		    }
+		    else if(value.equals(INVERTINGTEXT_VALUE)) {
+			// One fifth complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.2d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(CREATETHEWEIGHTS_VALUE)) {
+			// Two fifths complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.4d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(CREATESTEMMEDDIC_VALUE)) {
+			// Three fifths complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.6d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(CREATINGSTEMINDX_VALUE)) {
+			// Four fifths complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.8d) * MAX) / expected_stages);
+		    }
+		    else {
+			// Unknown command, go indeterminate
+			DebugStream.println("Unknown value: " + value);
+			progress_bar.setIndeterminate(true); 
+		    }
+		    value = null;
+		}
+		else if(name.equals(WARNING_ELEMENT)) {
+		    /** @todo - If on lower modes output a meaningful message. */
+		}
+		// Note that a fatal error is always followed by an end-stage, but not an end build. Thus the final state will be build (not base) and calls to exitValue() with indicate failure.
+		else if(name.equals(FATALERROR_ELEMENT)) {
+		    // On lower modes output a meaningful message.
+		    
+		}
+		else if(name.equals(STAGE_ELEMENT) && element.isClosed()) {
+		    // Increase progress so that stage is complete.
+		    current_stages++;
+		    progress_bar.setIndeterminate(false);
+		    setValue((current_stages * MAX) / expected_stages);
+		    state = BUILD;
+		}
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true); 
+		}
+		name = null;
+		break;
+	    case PHIND:
+		name = element.getElementName();
+		if(name.equals(PHASE_ELEMENT)) {
+		    value = element.getAttribute(NAME_ATTRIBUTE);
+		    if(value.equals(EXTRACTINGVOCAB_VALUE)) {
+			// Nothing to do
+		    }
+		    else if(value.equals(EXTRACTINGPHRASE_VALUE)) {
+			// One sixth complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.16d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(SORTANDRENUMBER_VALUE)) {
+			// Two sixths complete
+			progress_bar.setIndeterminate(false);
+		        setValue((int)(((double)current_stages + 0.33d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(PHRASEDATABASES_VALUE)) {
+			// Three sixths complete
+			progress_bar.setIndeterminate(false);
+		        setValue((int)(((double)current_stages + 0.5d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(WORDLEVELINDEXES_VALUE)) {
+			// Four sixths complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.66d) * MAX) / expected_stages);
+		    }
+		    else if(value.equals(DOCINFODATABASES_VALUE)) {
+			// Five sixths complete
+			progress_bar.setIndeterminate(false);
+			setValue((int)(((double)current_stages + 0.83d) * MAX) / expected_stages);
+		    }
+		    else {
+			// Unknown command, go indeterminate
+			DebugStream.println("Unknown value: " + value);
+			progress_bar.setIndeterminate(true); 
+		    }
+		    value = null;
+		}
+		else if(name.equals(STAGE_ELEMENT) && element.isClosed()) {
+		    // Increase progress so that stage is complete.
+		    current_stages++;
+		    progress_bar.setIndeterminate(false);
+		    setValue((current_stages * MAX) / expected_stages);
+		    state = BUILD;
+		}
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true); 
+		}
+		name = null;
+		break;
+	    case CREATEINFODATA:
+	    case CREATINGAUXILARY:
+		name = element.getElementName();
+		if(name.equals(STAGE_ELEMENT) && element.isClosed()) {
+		    // Another stage complete
+		    current_stages++;
+		    progress_bar.setIndeterminate(false);
+		    setValue((current_stages * MAX) / expected_stages);
+		    state = BUILD;
+		}
+		else {
+		    // Unknown command, go indeterminate
+		    DebugStream.println("Unknown name: " + name);
+		    progress_bar.setIndeterminate(true);
+		}
+		name = null;
+		break;
+	    default:
+		// Unknown command, go indeterminate
+		DebugStream.println("Unknown name: " + name);
+		progress_bar.setIndeterminate(true);
+	    }
+	}
+	// If we are dealing with lower detail modes then veto any other messages. We do this here as the progress monitors are garuenteed to recieve this message before anything else
+	else if(Configuration.getMode() <= threshold) {
+	    event.veto();
+	}
+    }
+
+    public void reset() {
+	current_stages = 0;
+	expected_stages = 0;
+	progress_bar.setIndeterminate(false);
+	progress_bar.setString(null);
+	setValue(MIN);
+	state = BASE;
+	progress_bar.updateUI();
+    }
+
+    public void saving() {
+    }
+
+    public void setSharedProgressBar(JProgressBar shared_progress_bar) {
+	this.shared_progress_bar = shared_progress_bar;
+    }
+
+    /** Since the creator of this process monitor is actually in the GUI, this class provides the simpliest way to send a cancel process message between the two.
+     * @param state The desired state of the stop flag as a <strong>boolean</strong>.
+     */
+    public synchronized void setStop(boolean state) {
+	this.stop = state;
+	progress_bar.setIndeterminate(false);
+    }
+    /** This method resets this monitor to the start, reseting the process parsing and progress bar.
+     * TODO Everthing.
+     */
+    public void start() {
+	stop = false;
+	setValue(MIN);
+	expected_stages = 3; // Always compress text, create info database and auxiliary creation
+	// Retrieve cdm.
+	CollectionDesignManager cdm = Gatherer.c_man.getCollection().cdm;
+	// If we are using MGPP the number of indexes is always 1
+	if(cdm.searchtype_manager.isMGPP()) {
+	    expected_stages = expected_stages + 1;
+	} else if (cdm.searchtype_manager.isLucene()) {
+	    // lucene no of indexes == number of levels
+	    int num_levels = cdm.index_manager.getNumLevels();
+	    if (num_levels == 0) num_levels = 1;
+	    expected_stages = expected_stages + num_levels;
+	}
+	else {
+	    int num_indexes = cdm.index_manager.getSize();
+	    int num_partitions = cdm.subcollectionindex_manager.getSize();
+	    if(num_partitions > 0) {
+		num_indexes = num_indexes * num_partitions;
+	    }
+	    int num_languages = cdm.language_manager.getSize();
+	    if(num_languages > 0) {
+		num_indexes = num_indexes * num_languages;
+	    }
+	    expected_stages = expected_stages + num_indexes;
+	}
+	// And if we have a phind classifier, we have one further index
+	if(cdm.classifier_manager.isPhindClassifierAssigned()) {
+	    expected_stages = expected_stages + 1;
+	}
+	cdm = null;
+    }
+    /** This method indicates the process is complete. 
+     * TODO Everthing.
+     */
+    public void stop() {
+	progress_bar.setIndeterminate(false);
+	setValue(MAX);
+    }
+
+    private void setValue(int value) {
+	progress_bar.setValue(value);
+	// Incrementing the shared progress bar is a little more problematic as we may very well be trying to set it to MIN but instead set it to halfway if we're not careful.
+	if(value > MIN) {
+	    shared_progress_bar.setValue(MAX + value);
+	}
+    }
+}
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GImportProgressMonitor.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GImportProgressMonitor.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GImportProgressMonitor.java	(revision 21112)
@@ -0,0 +1,453 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+import java.awt.Component;
+import java.util.*;
+import javax.swing.JProgressBar;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.util.StaticStrings;
+
+/** This implementation of <i>GShellProgressMonitor</i> is designed to parse and translate the progress of a import.pl call.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.1
+ */
+public class GImportProgressMonitor
+    implements GShellProgressMonitor {
+    /** Indicates if the progress bar is currently showing a string. */
+    private boolean showing_string = false;
+    /** Indicates if the GUI has asked the process this object monitors to stop. */
+    private boolean stop = false;
+    /** A count of the extracted files processed so far. */
+    private int extracted_file_count;
+    /** The number of documents processed (or rejected) so far. */
+    private int file_count;
+    /** The next value to be set for the progress bar - I use this rather than a compounding progress measure to try to limit rounding inaccuracies (nothing looks worse than the progress bar having to jump the last 10-15%) */
+    private int next_progress_value;
+    /** The number of files expected to be scanned by this import process. */
+    private int num_expected_docs;
+    /** This holds the number of documents actually processed by the import command, as garnered from the final block of text output. */
+    private int num_docs;
+
+    private int threshold = Configuration.SYSTEMS_MODE;
+    /** The progress bar this monitor updates. */
+    private JProgressBar progress_bar;
+
+    /** The progress bar that is shared with build monitor. */
+    private JProgressBar shared_progress_bar;
+
+    /** */
+    static final private String BLOCKED_ATTRIBUTE = "blocked";
+    static final private String CONSIDERED_ATTRIBUTE = "considered";
+    static final private String FAILED_ATTRIBUTE = "failed";
+    static final private String IGNORED_ATTRIBUTE = "ignored";
+    static final private String IMPORT_ELEMENT = "Import";
+    static final private String PROCESSED_ATTRIBUTE = "processed";  
+
+    static final private String ARGUMENT_ATTRIBUTE = "a";
+    /** */
+    static final private String NAME_ATTRIBUTE = "n";
+    /** */
+    static final private String PLUGIN_ATTRIBUTE = "p";
+    static final private String REASON_ATTRIBUTE = "r";
+    /** The fixed portion of the progress bar used for the calculating of file size and other pre-import functions. */
+    static final private int CALCULATION = 50000;
+    /** The fixed portion of the progress bar used for extracted metadata. */
+    static final private int EXTRACTED = 200000;
+    /** The minimum value for this progress bar. */
+    static final private int MIN = 0;
+    /** The fixed portion of the progress bar used for processed documents. */
+    static final private int PROCESSED = 750000;
+    /** The element name of a file detected message. */
+    static final private String FILE_ELEMENT = "File";
+    /** The element name of a file processing message. */
+    static final private String PROCESSING_ELEMENT = "Processing";
+    /** The element name of a processing error message. */
+    static final private String PROCESSINGERROR_ELEMENT = "ProcessingError";
+    /** The element name of an import complete message. */
+    static final private String IMPORTCOMPLETE_ELEMENT = "ImportComplete";
+    /** The element name of a file not processed. */
+    static final private String NONPROCESSEDFILE_ELEMENT = "NonProcessedFile";
+    /** The element name of a file not recognised. */
+    static final private String NONRECOGNISEDFILE_ELEMENT = "NonRecognisedFile";
+    /** The element name of a BadArgument message */
+    static final private String BADARGUMENT_ELEMENT = "BadArgument";
+    static final private String BADARGUMENTVALUE_ELEMENT = "BadArgumentValue";
+    static final private String BADPLUGIN_ELEMENT = "BadPlugin";
+
+    public GImportProgressMonitor() {
+	progress_bar = new JProgressBar();
+	progress_bar.setIndeterminate(false);
+	progress_bar.setMaximum(CALCULATION + PROCESSED + EXTRACTED);
+	progress_bar.setMinimum(MIN);
+	progress_bar.setString(null);
+	progress_bar.setStringPainted(true);
+	next_progress_value = CALCULATION;
+	shared_progress_bar = new JProgressBar();
+	shared_progress_bar.setIndeterminate(false);
+	shared_progress_bar.setMaximum((CALCULATION + PROCESSED + EXTRACTED) * 2);
+	shared_progress_bar.setMinimum(MIN);
+	shared_progress_bar.setString(null); // Default string
+	shared_progress_bar.setStringPainted(true);
+	setValue(MIN);
+    }
+
+    /** Method to register a new progress bar with this monitor.
+     * @param progress_bar The new <strong>JProgressBar</strong>.
+     */
+    public void addProgressBar(JProgressBar progress_bar) {
+	this.progress_bar = progress_bar;
+	progress_bar.setMaximum(CALCULATION + PROCESSED + EXTRACTED);
+	progress_bar.setMinimum(MIN);
+	setValue(MIN);
+	next_progress_value = CALCULATION;
+    }
+
+    /** Determine the script exit value according to the progress monitor. This gets around a problem where several script failures actually result with a successful exit value.
+     * @return A <i>int</i> with a value of zero if and only if the script was successful.
+     */
+    public int exitValue() {
+	if(num_docs > 0) {
+	    return 0;
+	}
+	return 1;
+    }
+
+    /** Retrieve the number of documents recorded by the import monitor during the execution of the import scripts.
+     * @return An <i>int</i> giving the document number.
+     */
+    public int getNumberOfDocuments() {
+	return num_docs;
+    }
+
+    /** Method to retrieve whatever control is being used as the progress indicator. Usually a <strong>JProgressBar</strong> but there may be others implemented later.
+     * @return A <strong>Component</strong> on which the progress of the process is being displayed.
+     */
+    public Component getProgress() {
+	return progress_bar;
+    }
+
+    public JProgressBar getSharedProgress() {
+	return shared_progress_bar;
+    }
+
+    public void messageOnProgressBar(String message)
+    {
+	if ((message!=null) & (message!="")) {
+	    progress_bar.setString(message);
+	    shared_progress_bar.setString(message);
+	    showing_string = true;
+	}
+	else {
+	    progress_bar.setString(null);
+	    shared_progress_bar.setString(null);
+	    showing_string = false;
+	}
+    }
+
+    /** Method to determine the state of the stop flag, which may be set by the visual component that created this monitor.
+     * @return A <strong>boolean</strong> indicating if the process has been asked to stop.
+     */
+    public synchronized boolean hasSignalledStop() {
+	return stop;
+    }
+
+    /** Inform the progress bar that it should programatically increment progress by one step. This is only called during the metadata archive extraction so each step should be (1000000 / 5) / num_docs. */
+    public void increment() {
+	extracted_file_count++;
+	// The current progress is calculated to be:
+	// The fixed calculation value plus the fixed processed value plus some portion of the fixed extracted value. This portion is the extracted_file_count over the total number of documents available. Note that this breaks badly for bibliographical files (for now).
+	setValue(CALCULATION + PROCESSED + ((EXTRACTED * extracted_file_count) / num_docs));
+    }
+
+    /** This method is used to 'feed in' a line of text captured from the process.
+     * @param queue a queue which at the moment should contain a single GShellEvent
+     */
+    public void process(ArrayList queue) {
+	// Retrieve the first event.
+	GShellEvent event = (GShellEvent) queue.get(0);
+	// Remove 'import.pl> ' bit
+	String line = event.getMessage();
+	line = line.substring(line.indexOf(StaticStrings.GREATER_THAN_CHARACTER) + 1);
+	line = line.trim();
+	// System.err.println("**** line = " + line);
+
+	///ystem.err.println("Process: " + line);
+	if(line.startsWith(StaticStrings.LESS_THAN_CHARACTER) && line.endsWith(StaticStrings.GREATER_THAN_CHARACTER)) {
+	    // No original event is passed on, even in the lower modes
+	    event.veto();
+	    // Create a new element from it
+	    GShellElement element = new GShellElement(line);
+	    String name = element.getElementName();
+	    ///ystem.err.println("Gatherer tag: " + name);
+	    // We may be reading a file. Remember we have to offset process as we recieve this message 'before' a document is processed. Hence the use of 'next_progress_value'
+	    if(name.equals(IMPORT_ELEMENT)) {
+		///ystem.err.println("#Import");
+		// We're into parsing output, so we don't need the 'calculating file size' etc string.
+		if(showing_string) {
+		    progress_bar.setString(null);
+		    showing_string = false;
+		}
+		if(Configuration.getMode() <= threshold) {
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportBegun1"), event.getStatus()));
+		}
+	    }
+	    else if(name.equals(FILE_ELEMENT)) {
+		///ystem.err.println("#File");
+		file_count++;
+		// Set the next progress
+		if(next_progress_value > 0) {
+		    setValue(next_progress_value);
+		}
+		// Now we calculate the next progress value
+		next_progress_value = CALCULATION + ((PROCESSED * file_count) / num_expected_docs);
+		event.veto(); // Unconditional veto
+	    }
+	    // Or we're being told what plugin is actually processing the file
+	    else if(name.equals(PROCESSING_ELEMENT)) {
+		///ystem.err.println("#FileProcessing");
+		// If we are at lower mode settings fire a new 'dumbed down' event
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[2];
+		    args[0] = element.getAttribute(NAME_ATTRIBUTE);
+		    args[1] = element.getAttribute(PLUGIN_ATTRIBUTE);
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessing", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // processing error
+	    else if (name.equals(PROCESSINGERROR_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] = element.getAttribute(NAME_ATTRIBUTE);
+		    String reason = element.getAttribute(REASON_ATTRIBUTE);
+		    if (reason == null || reason.equals("")) {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessingError", args), event.getStatus()));
+		    }
+		    else {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessingError", args) + " (" + reason + ")", event.getStatus()));
+		    }
+		    args = null;
+		}
+	    }
+	    // unrecognised file
+	    else if (name.equals(NONRECOGNISEDFILE_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] =element.getAttribute(NAME_ATTRIBUTE);
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileNotRecognised", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // unprocessed file
+	    else if (name.equals(NONPROCESSEDFILE_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] =element.getAttribute(NAME_ATTRIBUTE);		    
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileNotProcessed", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // We had a bad argument to a plugin
+	    else if (name.equals(BADARGUMENT_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgument", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // We had a bad argument value to a plugin
+	    else if (name.equals(BADARGUMENTVALUE_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgumentValue", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // And this one tellsa us the plugin
+	    else if (name.equals(BADPLUGIN_ELEMENT)) {
+		if(Configuration.getMode() <= threshold) {
+		    String args[] = new String[1];
+		    args[0] = element.getAttribute(PLUGIN_ATTRIBUTE);    
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadPluginOptions", args), event.getStatus()));
+		    args = null;
+		}
+	    }
+	    // Or the import complete element
+	    else if(name.equals(IMPORTCOMPLETE_ELEMENT)) {
+		// Set the next progress
+		setValue(next_progress_value);
+		// If we are at lower mode settings fire a new 'dumbed down' event
+		if(Configuration.getMode() <= threshold) {
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete1"), event.getStatus()));
+		    String considered_str = element.getAttribute(CONSIDERED_ATTRIBUTE);
+
+		    if (considered_str.equals(StaticStrings.ONE_CHARACTER)) {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete2.1"), event.getStatus()));	
+		    } else {
+			queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete2", considered_str), event.getStatus()));
+		    }
+		    considered_str = null;
+		    // The number of documents processed
+		    String processed_str = element.getAttribute(PROCESSED_ATTRIBUTE);
+		    if(!processed_str.equals(StaticStrings.ZERO_CHARACTER)) {
+			if(processed_str.equals(StaticStrings.ONE_CHARACTER)) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Processed.1"), event.getStatus()));
+			}
+			else {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Processed", processed_str), event.getStatus()));
+			}
+		    }
+		    // Try to set num docs
+		    try {
+			num_docs = Integer.parseInt(processed_str);
+		    }
+		    catch(Exception exception) {
+			num_docs = 0;
+		    }  
+		    processed_str = null;
+
+		    // The number of documents blocked
+		    String blocked_str = element.getAttribute(BLOCKED_ATTRIBUTE);
+		    if(!blocked_str.equals(StaticStrings.ZERO_CHARACTER)) {
+			if (blocked_str.equals(StaticStrings.ONE_CHARACTER)) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Blocked.1"), event.getStatus()));
+			} else {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Blocked", blocked_str), event.getStatus()));
+			}
+		    }
+		    blocked_str = null;
+		    
+		    // The number of documents ignored
+		    String ignored_str = element.getAttribute(IGNORED_ATTRIBUTE);
+		    if(!ignored_str.equals(StaticStrings.ZERO_CHARACTER)) {
+			if(ignored_str.equals(StaticStrings.ONE_CHARACTER)) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Ignored.1"), event.getStatus()));
+			} else {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Ignored", ignored_str), event.getStatus()));
+			}
+		    }
+		    ignored_str = null;
+		    
+		    // The number of documents failed
+		    String failed_str = element.getAttribute(FAILED_ATTRIBUTE);
+		    if(!failed_str.equals(StaticStrings.ZERO_CHARACTER)) {
+			if(failed_str.equals(StaticStrings.ONE_CHARACTER)) {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Failed.1"), event.getStatus()));
+			} else {
+			    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Failed", failed_str), event.getStatus()));
+			}
+		    }
+		    failed_str = null;
+		    // Message is finally ready
+		    queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete3"), event.getStatus()));
+		} // if mode below threshhold
+		else {
+		    // Try to set num docs
+		    String processed_str = element.getAttribute(PROCESSED_ATTRIBUTE);
+		    try {
+			num_docs = Integer.parseInt(processed_str);
+		    }
+		    catch(Exception exception) {
+			num_docs = 0;
+		    }
+		}
+	    }
+	    else {
+		// Veto it
+		event.veto();
+	    }
+	} // GLI output
+	else if(Configuration.getMode() <= threshold) {
+	    event.veto();
+	}
+    }
+
+    public void reset() {
+	setValue(MIN);
+	progress_bar.setString(null);
+	progress_bar.updateUI();
+    }
+
+    public void saving() {
+	progress_bar.setString(Dictionary.get("SaveProgressDialog.Title"));
+	showing_string = true;
+    }
+
+    /** Since the creator of this process monitor is actually in the GUI, this class provides the simpliest way to send a cancel process message between the two.
+     * @param state The desired state of the stop flag as a <strong>boolean</strong>.
+     */
+    public synchronized void setStop(boolean state) {
+	this.stop = state;
+    }
+
+    /** This method resets this monitor to the start, reseting the process parsing and progress bar.
+     */
+    public void start() {
+	stop = false;
+	setValue(MIN);
+	progress_bar.setString(Dictionary.get("FileActions.Calculating_Size"));
+	showing_string = true;
+	extracted_file_count = 0;
+	file_count = 0;
+	next_progress_value = -1;
+	num_docs = -1;
+	num_expected_docs = Gatherer.c_man.getCollection().getCount();
+    }
+
+    /** This method indicates the process is complete. 
+     */
+    public void stop() {
+	setValue(CALCULATION + PROCESSED + EXTRACTED);
+    }
+
+    int previous_value = -1;
+    private void setValue(int value) {
+	progress_bar.setValue(value);
+	shared_progress_bar.setValue(value);
+	if(value != 0 && value < previous_value) {
+	    DebugStream.println("Progress is decreasing! " + previous_value + " -> " + value);
+	}
+	previous_value = value;
+    }
+}
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShell.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShell.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShell.java	(revision 21112)
@@ -0,0 +1,640 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.regex.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.Configuration;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.cdm.CollectionConfiguration;
+import org.greenstone.gatherer.cdm.CollectionDesignManager;
+import org.greenstone.gatherer.cdm.CollectionMetaManager;
+import org.greenstone.gatherer.cdm.CollectionMeta;
+import org.greenstone.gatherer.metadata.DocXMLFileManager;
+import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
+import org.greenstone.gatherer.util.StaticStrings; 
+import org.greenstone.gatherer.util.Utility;
+import org.greenstone.gatherer.util.ZipTools;
+
+
+/** The <strong>GShell</strong> is reponsible for running a separately threaded process in the command shell. This is necessary for executing the Perl Scripts and also for other system related funcitonality.
+ */
+public class GShell
+    extends Thread {
+    /** A flag used to determine if this process has been asked to cancel. */
+    private boolean cancel = false;
+    private BufferedOutputStream buffered_output_stream = null;
+    /** The list of listeners associated with this class. */
+    private EventListenerList listeners = null;
+    /** The current status of this shell process. */
+    private int status = -1;
+    /** The type of message being sent. */
+    private int msg_type = -1;
+    /** The type of shell process. */
+    private int type = -1;
+    /** The caller of this process, and thus the class most interested in messages. */
+    private GShellListener caller = null;
+    /** The progress monitor associated with this process. */
+    private GShellProgressMonitor progress = null;
+    /** Arguments to be given to the process (including the executable you are calling. */
+    private String args[] = null;
+    /** Elements in process type enumeration. */
+    static final public int BUILD = 0;
+    static final public int IMPORT = 1;
+    static final public int NEW = 2;
+    static final public int EXPORTAS = 3;
+    static final public int CDIMAGE = 4;
+    static final public int CONVERT = 5;
+    static final public int EXPLODE = 6;
+
+    /** Elements in status type enumeration. */
+    static final public int ERROR = 0;
+    static final public int OK = 1;
+    static final public int CANCELLED = 2;
+
+    /** Elements in process type name enumeration. */
+    static public String GSHELL_BUILD = "gshell_build";
+    static public String GSHELL_IMPORT = "gshell_import";
+    static public String GSHELL_NEW = "gshell_new";
+    static public String GSHELL_EXPORTAS = "gshell_exportas";
+    static public String GSHELL_CDIMAGE = "gshell_cdimage";
+    static public String GSHELL_CONVERT = "gshell_convert";
+    static public String GSHELL_EXPLODE = "gshell_explode";
+
+    /** Determine if the given process is still executing. It does this by attempting to throw an exception - not the most efficient way, but the only one as far as I know
+     * @param process the Process to test
+     * @return true if it is still executing, false otherwise
+     */
+    static public boolean processRunning(Process process) {
+	boolean process_running = false;
+
+	try {
+	    process.exitValue(); // This will throw an exception if the process hasn't ended yet.
+	}
+	catch(IllegalThreadStateException itse) {
+	    process_running = true;
+	}
+	catch(Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	return process_running;
+    }
+
+    /** Constructor gatherer all the data required to create a new process, and emit meaningfull messages.
+     * @param args A <strong>String[]</strong> containing the arguments to the process thread, including the name of the executable.
+     * @param type An <strong>int</strong> that indicates what group of processes this process belongs to, as some are treated slightly differently (i.e an IMPORT type process is always followed by a BUILD one).
+     * @param msg_type As process threads may be background (like a makecol.pl call) or important processes in their own right (such as an IMPORT-BUILD) we must be able to set what level messages posted by this class will have by usings this <strong>int</strong>.
+     * @param caller The default <i>GShellListener</i> that is interested in the progress of this process.
+     * @param progress The <i>GShellProgressMonitor</i> associated with this process.
+     * @param name A <strong>String</strong> identifier given to the process, for convience and debug reasons.
+     */
+    public GShell(String args[], int type, int msg_type, GShellListener caller, GShellProgressMonitor progress, String name) {
+	super(name);
+	this.args = args;
+	this.msg_type = msg_type;
+	this.type = type;
+	this.caller = caller;
+	this.progress = progress;
+	this.status = 0;
+	// Lower this jobs priority
+	this.setPriority(Thread.MIN_PRIORITY);
+	listeners = new EventListenerList();
+	listeners.add(GShellListener.class, caller);
+    }
+    /** This method adds another shell listener to this process.
+     * @param listener The new <i>GShellListener</i>.
+     */
+    public void addGShellListener(GShellListener listener) {
+	listeners.add(GShellListener.class, listener);
+    }
+    /** This method removes a certain shell listener from this process.
+     * @param listener The <i>GShellListener</i> to be removed.
+     */
+    /* private void removeGShellListener(GShellListener listener) {
+       listeners.remove(GShellListener.class, listener);
+       } */
+
+
+
+//     protected boolean got_stream_char(InputStreamReader isr, StringBuffer line_buffer,
+// 				      BufferedOutputStream bos) throws IOException 
+//     {
+// 	// Hopefully this doesn't block if the process is trying to write to STDOUT/STDERR.
+
+// 	boolean input_status = false;
+
+// 	if(isr.ready()) {
+// 	    input_status = true;
+// 	    int c = isr.read();
+// 	    if(c == '\n' || c == '\r') {
+// 		if(line_buffer.length() > 0) {
+// 		    String line = line_buffer.toString();
+// 		    DebugStream.println("* " + line + " *");
+// 		    fireMessage(type, typeAsString(type) + "> " + line, status, bos);
+// 		    line = null;
+// 		    line_buffer = new StringBuffer();
+// 		}
+// 	    }
+// 	    else {
+// 		line_buffer.append((char)c);
+// 	    }
+// 	}
+
+// 	return input_status;
+//     }
+
+
+    protected StringBuffer get_stream_char(InputStreamReader isr, StringBuffer line_buffer,
+				   BufferedOutputStream bos) throws IOException 
+    {
+	int c = isr.read();
+	///atherer.println("isr: '" + (char) c + "'");
+	if(c == '\n' || c == '\r') {
+	    if(line_buffer.length() > 0) {
+		String line = line_buffer.toString();
+		// DebugStream.println("* " + line + " *");
+		fireMessage(type, typeAsString(type) + "> " + line, status, bos);
+		line_buffer = new StringBuffer();
+	    }
+	}
+	else {
+	    line_buffer.append((char)c);
+	}
+
+	return line_buffer;
+    }
+
+
+    protected int check_for_error(String line, StringBuffer error_list, 
+				  int error_count)
+    {
+	if (line.startsWith("ERROR:")) {
+	    error_count++;
+	    if (error_count<10) {
+		error_list.append(line);
+	    }
+	    else if (error_count==10) {
+		error_list.append("...\n");
+	    }
+	}
+	return error_count;
+    }
+
+
+    private void runRemote(String[] args, BufferedOutputStream bos)
+    {
+	// Make sure the process hasn't been cancelled
+	if (hasSignalledStop()) {
+	    return;
+	}
+
+	try {
+	    int directory_name_end = args[0].lastIndexOf(File.separator);
+	    String script_name = ((directory_name_end != -1) ? args[0].substring(directory_name_end + 1) : args[0]);
+	    System.err.println("Script name: " + script_name);
+	    String collection_name = args[args.length - 1];
+	    System.err.println("Collection name: " + collection_name);
+
+	    String script_args = "";
+	    for (int i = 1; i < (args.length - 1); i++) {
+		// Skip arguments that won't make sense on the server
+		if (args[i].equals("-collectdir") || args[i].equals("-importdir") || args[i].equals("-builddir")) {
+		    i++;
+		    continue;
+		}
+
+		// Script arguments get changed to CGI arguments
+		if (args[i].startsWith("-")) {
+		    script_args += "&" + args[i].substring(1) + "=";
+		    if ((i + 1) < (args.length - 1) && !args[i + 1].startsWith("-")) {
+			script_args += URLEncoder.encode(args[i + 1], "UTF-8");
+			i++;
+		    }
+		}
+	    }
+
+	    System.err.println("Script args: " + script_args);
+	    buffered_output_stream = bos;
+	    String command_output = RemoteGreenstoneServer.runScript(collection_name, script_name, script_args, this);
+	    status = (command_output.equals("") ? CANCELLED : OK);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	    status = ERROR;
+	}
+    }
+
+
+    private void runLocal(String[] args, BufferedOutputStream bos) 
+    {
+	try {
+	    String command = "";
+	    for(int i = 0; i < args.length; i++) {
+		command = command + args[i] + " ";
+	    }
+	    
+	    System.err.println("Command: "+command);
+	    ///ystem.err.println("Command: " + command);
+	    fireMessage(type, Dictionary.get("GShell.Command") + ": " + command, status, null);
+	    
+	    Runtime rt = Runtime.getRuntime();
+	    Process prcs = rt.exec(args);
+	    
+	    InputStreamReader eisr = new InputStreamReader( prcs.getErrorStream(), "UTF-8" );
+	    InputStreamReader stdisr = new InputStreamReader( prcs.getInputStream(), "UTF-8"  );
+	
+	    StringBuffer eline_buffer = new StringBuffer();
+	    StringBuffer stdline_buffer = new StringBuffer();
+	
+	    while(type != GShell.NEW && processRunning(prcs) && !hasSignalledStop()) {
+		// Hopefully this doesn't block if the process is trying to write to STDOUT.
+		if((eisr!=null) && eisr.ready()) { 
+		    eline_buffer = get_stream_char(eisr,eline_buffer,bos);
+		}
+		// Hopefully this won't block if the process is trying to write to STDERR
+		else if(stdisr.ready()) {
+		    stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
+		}
+		else {
+		    try {
+			sleep(100);
+		    }
+		    catch(Exception exception) {
+		    }
+		}
+	    }
+	
+	    if(!hasSignalledStop()) {
+		// Of course, just because the process is finished doesn't
+		// mean the incoming streams are empty. Unfortunately I've
+		// got no chance of preserving order, so I'll process the
+		// error stream first, then the out stream
+		while(eisr.ready()) {
+		    eline_buffer = get_stream_char(eisr,eline_buffer,bos);
+		}
+	    
+		while(stdisr.ready()) {
+		    stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
+		}
+	    
+		// Ensure that any messages still remaining in the string buffers are fired off.
+		if(eline_buffer.length() > 0) {
+		    String eline = eline_buffer.toString();
+		    //DebugStream.println("Last bit of eline: " + eline);
+		    fireMessage(type, typeAsString(type) + "> " + eline, status, bos);
+		    eline = null;
+		}
+	    
+		if(stdline_buffer.length() > 0) {
+		    String stdline = stdline_buffer.toString();
+		    //DebugStream.println("Last bit of stdline: " + stdline);
+		    fireMessage(type, typeAsString(type) + "> " + stdline, status, null);
+		    stdline = null;
+		}
+	    }
+	    else {
+		System.err.println("We've been asked to stop.");
+	    }
+	
+	
+	    if(!hasSignalledStop()) {
+		// Now display final message based on exit value
+	    
+		prcs.waitFor(); 
+
+		if(prcs.exitValue() == 0) {
+		    status = OK;
+		    fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Success"), status, null);
+		}
+		else {
+		    status = ERROR;
+		    fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Failure"), status, null);
+		}
+	    
+		eisr.close();
+		stdisr.close();
+	    }
+	    else {
+		// I need to somehow kill the child process. Unfortunately
+		// Thread.stop() and Process.destroy() both fail to do
+		// this. But now, thankx to the magic of Michaels 'close the
+		// stream suggestion', it works fine (no it doesn't!)
+		prcs.getInputStream().close();
+		prcs.getErrorStream().close();
+		prcs.getOutputStream().close();
+		prcs.destroy();
+		status = CANCELLED;
+	    }
+	}
+	// Exception
+	catch (Exception exception) {
+	    DebugStream.println("Exception in GShell.runLocal() - unexpected");
+	    DebugStream.printStackTrace(exception);
+	    status = ERROR;
+	}
+    }
+    
+    
+
+    /** Any threaded class must include this method to allow the thread body to be run. */
+    public void run() {
+	String col_name = args[args.length-1];		
+
+	// Determine if the user has asked for an outfile.
+	String out_name = null;
+	BufferedOutputStream bos = null;
+	if(type == IMPORT || type == BUILD) {
+	    if(type == IMPORT) {
+		out_name = (String) Gatherer.c_man.getCollection().import_options.getValue("out");
+	    }
+	    else {
+		out_name = (String) Gatherer.c_man.getCollection().build_options.getValue("out");
+	    }
+	    if(out_name != null && out_name.length() > 0) {
+		try {
+		    bos = new BufferedOutputStream(new FileOutputStream(new File(out_name), true));
+		}
+		catch (Exception error) {
+		    DebugStream.printStackTrace(error);
+		}
+	    }
+	}
+
+	// Issue a processBegun event
+	System.err.println("\nFiring process begun for " + type + "...");
+	fireProcessBegun(type, status);
+	System.err.println("Done process begun.");
+	if (Gatherer.isGsdlRemote) {
+	    runRemote(args,bos);
+	}
+	else {
+	    runLocal(args,bos);
+	}
+	System.err.println("Done runLocal().");
+
+	if(status == OK) {
+	    if (type == NEW) {
+		if (Gatherer.isGsdlRemote) {
+		    RemoteGreenstoneServer.downloadCollection(col_name);
+		}
+	    }
+	    else if(type == IMPORT) {
+		
+		// download the archives directory (if gsdl server is remote)			
+		if (Gatherer.isGsdlRemote) {
+
+		    if (progress!=null) {
+			progress.messageOnProgressBar("Downloading archive data from server");
+		    }
+
+		    RemoteGreenstoneServer.downloadCollectionArchives(col_name);
+
+		    if (progress!=null) {
+			progress.messageOnProgressBar("");
+		    }      
+		}
+
+		// Refresh the DocXMLFileManager
+		fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Start"), status, null);
+		DocXMLFileManager.clearDocXMLFiles();
+		DocXMLFileManager.loadDocXMLFiles(new File(Gatherer.c_man.getCollectionArchivesDirectoryPath()));
+		fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Complete"), status, null);
+	    }
+
+ 	    else if(type == BUILD) {
+ 		// download the building directory (if gsdl server is remote)
+ 		if (Gatherer.isGsdlRemote) {
+ 		    if (progress!=null) {
+ 			progress.messageOnProgressBar("Downloading index data from server");
+ 		    }
+
+		    // Only need to download build.cfg file
+		    File build_cfg_file = new File(Gatherer.c_man.getCollectionBuildingDirectoryPath(), "build.cfg");
+		    RemoteGreenstoneServer.downloadCollectionFile(col_name, build_cfg_file);
+		    
+ 		    if (progress!=null) {
+ 			progress.messageOnProgressBar("");
+ 		    }
+ 		}
+ 	    }
+	    else if(type == CDIMAGE) {
+		// download exported files from tmp folder (if gsdl server is remote)			
+		if (Gatherer.isGsdlRemote) {
+		    if (progress!=null) {
+			progress.messageOnProgressBar("Downloading CD-ROM data from server");
+		    }
+
+		    // !! TO DO
+		    
+ 		    if (progress!=null) {
+ 			progress.messageOnProgressBar("");
+ 		    }
+		}
+	    }
+	}
+
+	// We're done.	
+	System.err.println("Firing process complete for " + type + "...");
+	fireProcessComplete(type, status);
+	// Close bos
+	if(bos != null) {
+	    try {
+		bos.close();
+		bos = null;
+	    }
+	    catch(Exception error) {
+		DebugStream.printStackTrace(error);
+	    }
+	}
+    }
+
+
+    public void fireMessage(String message)
+    {
+	fireMessage(type, typeAsString(type) + "> " + message, status, buffered_output_stream);
+    }
+
+
+    /** Method for firing a message to all interested listeners.
+     * @param type An <strong>int</strong> indicating the process type.
+     * @param message The message as a <strong>String</strong>.
+     * @param status An <strong>int</strong> specifying the current status of the process.
+     */
+    public void fireMessage(int type, String message, int status, BufferedOutputStream bos) {
+	GShellEvent event = new GShellEvent(this, 0, type, message, status);
+	// If there is a progress monitor attached, pass the event to it first. Note that we pass a queue of messages as the processing may cause one message to be split into several.
+	ArrayList message_queue = new ArrayList();
+	message_queue.add(event);
+	if(progress != null) {
+	    progress.process(message_queue);
+	}
+	for(int j = 0; j < message_queue.size(); j++) {
+	    GShellEvent current_event = (GShellEvent) message_queue.get(j);
+	    // If the event hasn't been vetoed, pass it on to other listeners
+	    if(!current_event.isVetoed()) {
+		Object[] concerned = listeners.getListenerList();
+		for(int i = 0; i < concerned.length ; i++) {
+		    if(concerned[i] == GShellListener.class) {
+			((GShellListener)concerned[i+1]).message(current_event);
+		    }
+		}
+		concerned = null;
+	    }
+	}
+	// And if we have a buffered output stream from error messages, send the message there
+	if(bos != null) {
+	    try {
+		bos.write(message.getBytes(), 0, message.length());
+	    }
+	    catch(Exception exception) {
+		DebugStream.println("Exception in GShell.fireMessage() - unexpected");
+		DebugStream.printStackTrace(exception);
+	    }
+	}
+	message_queue = null;
+	event = null;
+    }
+
+    /** Method for firing a process begun event which is called, strangly enough, when the process begins.
+     * @param type An <strong>int</strong> indicating the process type.
+     * @param status An <strong>int</strong> specifying the current status of the process.
+     */
+    protected void fireProcessBegun(int type, int status) {
+	// Start the progres monitor if available
+	if(progress != null) {
+	    System.err.println("About to call progress.start().");
+	    progress.start();
+	    System.err.println("Called progress.start().");
+	}
+	// Fire an event
+	GShellEvent event = new GShellEvent(this, 0, type, "", status);
+	Object[] concerned = listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i++) {
+	    if(concerned[i] == GShellListener.class) {
+		((GShellListener)concerned[i+1]).processBegun(event);
+	    }
+	}
+    }
+
+
+    /** Method for firing a process complete event which is called, no surprise here, when the process ends.
+     * @param type An <strong>int</strong> indicating the process type.
+     * @param status An <strong>int</strong> specifying the current status of the process.
+     */
+    protected void fireProcessComplete(int type, int status) {
+	// Tidy up by stopping the progress bar. If it was cancelled then the cancel command has arrived via the progress bars and they don't need to be told again (it actually causes problems).
+	if(progress != null && status != CANCELLED) {
+	    progress.stop();
+	}
+	
+	// If we were cancelled, and we are lower details modes, fire off one last message.
+	if(status == CANCELLED && Configuration.getMode() <= Configuration.SYSTEMS_MODE) {
+	    GShellEvent current_event = new GShellEvent(this, 0, type, Dictionary.get("GShell.Build.BuildCancelled"), status);
+	    Object[] concerned = listeners.getListenerList();
+	    for(int i = 0; i < concerned.length ; i++) {
+		if(concerned[i] == GShellListener.class) {
+		    ((GShellListener)concerned[i+1]).message(current_event);
+		}
+	    }
+	    concerned = null;
+	}
+	// And firing off an event
+	GShellEvent event = new GShellEvent(this, 0, type, "", status);
+	Object[] concerned = listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i++) {
+	    if(concerned[i] == GShellListener.class) {
+		((GShellListener)concerned[i+1]).processComplete(event);
+	    }
+	}
+    }
+
+    /** Method to determine if the user, via the progress monitor, has signalled stop.
+     * @return A <strong>boolean</strong> indicating if the user wanted to stop.
+     */
+    public boolean hasSignalledStop() {
+	boolean has_signalled_stop = false;
+	if(progress != null) {
+	    has_signalled_stop = progress.hasSignalledStop();
+	}
+	if(has_signalled_stop) {
+	    status = CANCELLED;
+	}
+	return has_signalled_stop;
+    }
+
+    /** Converts a type into a text representation.
+     * @param type An <strong>int</strong> which maps to a shell process type.
+     * @return A <strong>String</strong> which is the thread process's text name.
+     */
+    public String typeAsString(int type) {
+	String name = null;
+	switch(type) {
+	case BUILD:
+	    name = "buildcol.pl";
+	    break;
+	case IMPORT:
+	    name = "import.pl";
+	    break;
+	case NEW:
+	    name = "mkcol.pl";
+	    break;
+	case EXPORTAS:
+	    name = "export.pl";
+	    break;
+	case CDIMAGE:
+	    name = "exportcol.pl";
+	    break;
+	case CONVERT:
+	    name = "convert_coll_from_gs2.pl";
+	    break;
+	case EXPLODE:
+	    name = "explode_metadata_database.pl";
+	    break;
+	default:
+	    name = "";
+	}
+	return name;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellElement.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellElement.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellElement.java	(revision 21112)
@@ -0,0 +1,106 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, New Zealand Digital Library, University of Waikato
+ *
+ * Copyright (C) 2004 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+import java.util.HashMap;
+import java.util.StringTokenizer;
+import org.greenstone.gatherer.util.StaticStrings;
+
+/** This class wraps around a pseudo-XML message we have recieved from an external script, and allows access to the element name, attributes etc via method calls. */
+public class GShellElement {
+    private boolean is_closed = false;
+    private HashMap attributes;
+    private String element_name;
+    /** Create a new element around the message string. The string is expected to start with < and end with > */
+    /** this is a hack - why aren't we using proper XML parsing?? */
+    public GShellElement(String message) {
+	// Remove the angle brackets
+	message = message.substring(1, message.length() - 1);
+	// Tokenize the message
+	StringTokenizer tokenizer = new StringTokenizer(message);
+	// The first token is the element name - be aware it may start with the close tag indicator
+	element_name = tokenizer.nextToken();
+	if(element_name.startsWith(StaticStrings.URL_SEPARATOR_CHARACTER)) {
+	    element_name = element_name.substring(1);
+	    is_closed = true;
+	}
+	// Now while we have remaining tokens
+	while(tokenizer.hasMoreTokens()) {
+	    int index = -1;
+	    // The next token might be a attribute=value pair or it could be the close tag indicator
+	    String token = tokenizer.nextToken();
+	    if(token.equals(StaticStrings.URL_SEPARATOR_CHARACTER)) {
+		is_closed = true;
+	    }
+	    else if(token.length() > 3 && (index = token.indexOf(StaticStrings.EQUALS_CHARACTER)) != -1) {
+		String attribute_name = token.substring(0, index);
+		String attribute_value = token.substring(index + 1);
+		// Value may end in a /
+		if(attribute_value.endsWith(StaticStrings.URL_SEPARATOR_CHARACTER)) {
+		    is_closed = true;
+		    attribute_value = attribute_value.substring(0, attribute_value.length() - 1);
+		}
+		// We may have to strip ' marks off value - if it starts with ' and doesn't end with ' we read in more tokens until we get an ending '
+		if(attribute_value.startsWith(StaticStrings.SINGLE_QUOTE_CHARACTER)) {
+		    while (!attribute_value.endsWith(StaticStrings.SINGLE_QUOTE_CHARACTER) && tokenizer.hasMoreTokens()) {
+			attribute_value += " "+tokenizer.nextToken();
+		    }
+		    if (attribute_value.endsWith(StaticStrings.SINGLE_QUOTE_CHARACTER)) {
+			attribute_value = attribute_value.substring(1, attribute_value.length() - 1);
+		    }
+		}
+		if(attributes == null) {
+		    attributes = new HashMap();
+		}
+		attributes.put(attribute_name, attribute_value);
+		attribute_value = null;
+		attribute_name = null;
+	    }
+	    token = null;
+	}
+	tokenizer = null;
+    }
+
+    public String getAttribute(String attribute_name) {
+	String result = null;
+	if(attributes != null) {
+	    result = (String) attributes.get(attribute_name);
+	}
+	if(result == null) {
+	    result = "";
+	}
+	return result;
+    }
+
+    public String getElementName() {
+	return element_name;
+    }
+
+    public boolean isClosed() {
+	return is_closed;
+    }
+}
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellEvent.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellEvent.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellEvent.java	(revision 21112)
@@ -0,0 +1,120 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+/** Title:        The Gatherer<br>
+ * Description:  The Gatherer: a tool for gathering and enriching digital collections.<br>
+ * Copyright:    Copyright (c) 2001<br>
+ * Company:      The University of Waikato<br>
+ * Written:        /  /01<br>
+ * Revised:      12/05/02 - Commented<br>
+ *               29/05/02 - Moved into correct package<br>
+ * @author John Thompson, Greenstone Digital Libraries
+ * @version 2.1 */
+import java.awt.AWTEvent;
+
+/** This class encapsulates all the information created by an event within a <strong>GShell</strong> process. */
+public class GShellEvent 
+    extends AWTEvent {
+    /** Allows some part of the program to prevent this message being processed by other parts. */
+    private boolean vetoed = false;
+    /** The status of the process at the completion of event. */
+    private int status = -1;
+    /** The process type (such as COPY, BUILD or IMPORT). */
+    private int type = -1;
+    /** Any message associated with this event. */
+    private String message = null;
+
+    /* Constructor.
+     * @param source The <strong>GShell</strong> that fired this message.
+     * @param id The event identifier as an <strong>int</strong>.
+     * @param type The process type as an <strong>int</strong>.
+     * @param message A <strong>String</strong> representing any message attatched with this event.
+     * @param status The status of the process post event, as an <strong>int</strong>.
+     */
+    public GShellEvent(Object source, int id, int type, String message, int status) {
+	super(source, id);
+	this.message = message;
+	this.status = status;
+	this.type = type;
+    }
+
+    /** Gets the message associated with this event.
+     * @return The message as a <strong>String</strong> or <i>null</i>.
+     */
+    public String getMessage() {
+	return message;
+    }
+
+    /** Gets the status associated with this event. This status can then be matched back to the constants in <strong>GShell</strong>.
+     * @return An <strong>int</strong> signifying the process status.
+     */ 
+    public int getStatus() {
+	return status;
+    }
+
+    /** Gets the type associated with this event. This type can then be matched back to the constants in <strong>GShell</strong>.
+     * @return An <strong>int</strong> signifying the process type.
+     */ 
+    public int getType() {
+	return type;
+    }
+
+    /** Determine if this event has been vetoed by some other part of the GLI
+     * @return true if the event has, and thus shouldn't be further processed, false otherwise.
+     */
+    public boolean isVetoed() {
+	return vetoed;
+    }
+
+    /** Change the message to be displayed by this event - usually from something technical so something easier to understand.
+     * @param message the new message as a String
+     */
+    public void setMessage(String message) {
+	this.message = message;
+    }
+
+    public String toString() {
+	return "org.greenstone.gatherer.shell.GShellEvent[" + message + "," + status + "," + type + "]";
+    }
+
+    /** Prevent this event being processed by other parts of the GLI.
+     */
+    public void veto() {
+	vetoed = true;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellListener.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellListener.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellListener.java	(revision 21112)
@@ -0,0 +1,70 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+/**
+ * Title:        The Gatherer<br>
+ * Description:  The Gatherer: a tool for gathering and enriching digital collections.<br>
+ * Copyright:    Copyright (c) 2001<br>
+ * Company:      The University of Waikato<br>
+ * Written:        /  /01<br>
+ * Revised:      12/05/02 - Commented<br>
+ *               29/05/02 - Moved into correct package<br>
+ * @author John Thompson, Greenstone Digital Libraries
+ * @version 2.1
+ */
+import java.util.EventListener;
+
+/** This interface details the methods required of a class that wishes to listen to a <strong>GShell</strong>.
+ */
+public interface GShellListener 
+    extends EventListener {
+    /** All implementation of <i>GShellListener</i> must include this method so the listener can be informed of messages from the <strong>GShell</strong>.
+     * @param event A <strong>GShellEvent</strong> that contains, amoung other things, the message.
+     */
+    public void message(GShellEvent event);
+
+    /** All implementation of <i>GShellListener</i> must include this method so the listener can be informed when a <strong>GShell</strong> begins its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the initial state of the <strong>GShell</strong> before task comencement.
+     */
+    public void processBegun(GShellEvent event);
+
+    /** All implementation of <i>GShellListener</i> must include this method so the listener can be informed when a <strong>GShell</strong> completes its task.
+     * @param event A <strong>GShellEvent</strong> that contains details of the final state of the <strong>GShell</strong> after task completion.
+     */
+    public void processComplete(GShellEvent event);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellProgressMonitor.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellProgressMonitor.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/shell/GShellProgressMonitor.java	(revision 21112)
@@ -0,0 +1,105 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.shell;
+
+/**
+ * Title:        The Gatherer<br>
+ * Description:  The Gatherer: a tool for gathering and enriching digital collections.<br>
+ * Copyright:    Copyright (c) 2001<br>
+ * Company:      The University of Waikato<br>
+ * Written:        /  /01<br>
+ * Revised:      12/05/02 - Commented<br>
+ *               29/05/02 - Moved into correct package<br>
+ * @author John Thompson, Greenstone Digital Libraries
+ * @version 2.1
+ */
+import java.awt.Component;
+import java.util.ArrayList;
+import javax.swing.JProgressBar;
+
+/** When implemented, this interface allows another class to monitor the progress of a <strong>GShell</strong> process. Specifically implementing classes should be designed to take the textual output from the process and then translate that message into some quanitive measure of progress which can then be shown on the progress bar.
+ */
+public interface GShellProgressMonitor {
+    /** Method to register a new progress bar with this monitor.
+     * @param progress_bar The new <strong>JProgressBar</strong>.
+     */
+    public void addProgressBar(JProgressBar progress_bar);
+
+    /** Determine the script exit value according to the progress monitor. This gets around a problem where several script failures actually result with a successful exit value.
+     * @return A <i>int</i> with a value of zero if and only if the script was successful.
+     */
+    public int exitValue();
+
+    public JProgressBar getSharedProgress();
+
+    /** Method to retrieve whatever control is being used as the progress indicator. Usually a <strong>JProgressBar</strong> but there may be others implemented later.
+     * @return A <strong>Component</strong> on which the progress of the process is being displayed.
+     */
+    public Component getProgress();
+
+    public void messageOnProgressBar(String message);
+
+    /** Method to determine the state of the stop flag, which may be set by the visual component that created this monitor.
+     * @return A <strong>boolean</strong> indicating if the process has been asked to stop.
+     */
+    public boolean hasSignalledStop();
+
+    /** Inform the progress bar that it should programatically increment progress by one step. */
+    public void increment();
+
+    /** This method is used to 'feed in' a line of text captured from the process.
+     * @param queue what will be used as a queue of events, but what, at the moment, should only have one event in it
+     */
+    public void process(ArrayList queue);
+
+    public void reset();
+
+    public void saving();
+
+    /** Since the creator of this process monitor is actually in the GUI, this class provides the simpliest way to send a cancel process message between the two.
+     * @param state The desired state of the stop flag as a <strong>boolean</strong>.
+     */
+    public void setStop(boolean state);
+
+    /** This method resets this monitor to the start, reseting the process parsing and progress bar.
+     */
+    public void start();
+
+    /** This method indicates the process is complete. 
+     */
+    public void stop();
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocument.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocument.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocument.java	(revision 21112)
@@ -0,0 +1,987 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Greenstone Librarian Interface (GLI) application, 
+ * part of the Greenstone digital library software suite from the New 
+ * Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
+ *
+ * Author: John Thompson, Greenstone Project, New Zealand Digital Library, 
+ *         University of Waikato
+ *         http://www.nzdl.org
+ *
+ * Copyright (C) 2004 New Zealand Digital Library, University of Waikato
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.text.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.gui.GLIButton;
+
+/** A Document whose underlying data is stored in a RandomAccessFile, and whose Element implementations lack the memory hogging problems associated with anything that extends the AbstractDocument class. This Document, for reasons of time constraints and sanity, only provides an editting ability of appending new lines to the end of the current document, perfect for our logging needs, completely useless for text editing purposes. Furthermore, since the append actions tend to somewhat swamp the IO, I'll temporarily store strings in the structure model, and write them out using a separate thread. 
+ * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+ * @version 2.41 final
+ */
+public class AppendLineOnlyFileDocument
+    implements Document {
+    /** A default string to append at the start of each log file. Of special note is the beginning '.' character that will, upon completion of the import/build process, be replaced with a letter indicating the final status of this build attempt. */
+    static final private String GLI_HEADER_STR = ".";
+    /** The root element of the document model. */
+    private AppendLineOnlyFileDocumentElement root_element;
+    /** The current owner of this document, useful for callbacks when certain tasks are complete. */
+    private AppendLineOnlyFileDocumentOwner owner;
+    /** Is this document a build log, and hence has some extra functionality, of just a straight log. */
+    private boolean build_log = false;
+    /** A list of listeners attached to this document. */
+    private EventListenerList listeners_list;
+    /** This properties hash map allows you to store any metadata about this document for later review. */
+    private HashMap properties;
+    /** Indicates the current length of the underlying file (which may not equal the current number of characters in the document). */
+    private long length;
+    /** The random access file which will back this document model. */
+    private RandomAccessFile file;
+    /** The filename given to the underlying file. */
+    private String filename;
+    /** An independant thread responsible for writing the 'in memory' contents of the document model to the random access file as appropriate. This is done so IO lag does not effect the gui as much as it used to. */
+    private WriterThread writer;
+
+    
+    public AppendLineOnlyFileDocument(String filename) {
+	this(filename, true);
+    }
+
+    /** The constructor is responsible for creating several necessary data structures as well as opening the random access file. When it creates the new file, as it is wont to do, it will immediately add the special header line defined above.
+     * @param filename the absolute path name of the file as a String
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread
+     * @see org.greenstone.gatherer.util.StaticStrings#NEW_LINE_CHAR
+     */
+    public AppendLineOnlyFileDocument(String filename, boolean build_log) {
+	DebugStream.println("Creating log: " + filename);
+	// Initialization 
+	this.build_log = build_log;
+	this.filename = filename;
+	this.listeners_list = new EventListenerList();
+	this.properties = new HashMap();
+	this.writer = new WriterThread();
+	writer.start();
+	// Open underlying file
+	try {
+	    file = new RandomAccessFile(filename, "rw");
+	    // Create the root element.
+	    length = file.length();
+	    root_element = new AppendLineOnlyFileDocumentElement();
+	    // Now quickly read through the underlying file, building an Element for each line.
+	    long start_offset = 0L;
+	    file.seek(start_offset);
+	    int character = -1;
+	    while((character = file.read()) != -1) {
+		if(character == StaticStrings.NEW_LINE_CHAR) {
+		    long end_offset = file.getFilePointer();
+		    Element child_element = new AppendLineOnlyFileDocumentElement(start_offset, end_offset);
+		    root_element.add(child_element);
+		    child_element = null;
+		    start_offset = end_offset;
+		}
+	    }
+	    // If there we no lines found, and we are a build log, then append the file header.
+	    if(build_log && root_element.getElementCount() == 0) {
+		appendLine(GLI_HEADER_STR);
+	    }
+	}
+	catch (Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+    /** Adds a document listener for notification of any changes. 
+     * @param listener the new DocumentListener to keep track of
+     */
+    public void addDocumentListener(DocumentListener listener) {
+	///ystem.err.println("addDocumentListener(" + listener + ")");
+	listeners_list.add(DocumentListener.class, listener);
+    }
+
+    /** Append some content after the document. 
+     * @param str the String to add as a new line in the document
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentEvent
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread#queue
+     */
+    public void appendLine(String str) {
+	// Ensure the string ends in a newline
+	try {
+	    if (!str.endsWith("\n")) {
+		str = str + "\n";
+	    }
+	    int str_length = str.getBytes("UTF-8").length;
+	    long start_offset = length;
+	    long end_offset = start_offset + (long) str_length;
+	    length = length + str_length;
+	    // Create a new element to represent this line
+	    AppendLineOnlyFileDocumentElement new_line_element = new AppendLineOnlyFileDocumentElement(start_offset, end_offset, str);
+	    root_element.add(new_line_element);
+	    // Queue the content to be written.
+	    writer.queue(new_line_element);
+	    // Now fire an event so everyone knows the content has changed.
+	    DocumentEvent event = new AppendLineOnlyFileDocumentEvent(new_line_element, (int)start_offset, str_length, DocumentEvent.EventType.INSERT);
+	    Object[] listeners = listeners_list.getListenerList();
+	    for (int i = listeners.length - 2; i >= 0; i = i - 2) {
+		if (listeners[i] == DocumentListener.class) {
+		    ((DocumentListener)listeners[i+1]).insertUpdate(event);
+		}
+	    }
+	    listeners = null;
+	    event = null;
+	    new_line_element = null;
+	}
+	catch(Exception error) {
+	    DebugStream.printStackTrace(error);
+	}
+    }
+
+    /** Returns a position that will track change as the document is altered. 
+     * @param offs the position offset as an int
+     * @return the Position to be monitored
+     *  @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentPosition
+     */
+    public Position createPosition(int offs)  {
+	return new AppendLineOnlyFileDocumentPosition(offs);
+    }
+
+    /** The destructor is implemented to ensure the current log file has finished transferring from memory to disk, and that the random access file is properly closed, before GLI exits.
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread#finish
+     */
+    public void destroy() {
+	try {
+	    writer.finish();
+	    file.close();
+	}
+	catch(Exception exception) {
+	    DebugStream.println("Exception in AppendLineOnlyFileDocument.destroy() - unexpected");
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+
+    /** Gets the default root element for the document model. 
+     * @return the root Element of this document
+     */
+    public Element getDefaultRootElement() {
+	return root_element;
+    }
+
+    /** Returns the length of the data. 
+     * @return the length as an int (not a long)
+     */
+    public int getLength() {
+	return (int) length;
+    }
+
+    /** A version of get length which returns the offset to the start of the last line in the document.
+     * @return the offset length as an int
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement
+     */
+    public int getLengthToNearestLine() {
+	AppendLineOnlyFileDocumentElement last_element = (AppendLineOnlyFileDocumentElement)root_element.getElement(root_element.getElementCount() - 1);
+	if(last_element != null ) {
+	    return last_element.getStartOffset();
+	}
+	else {
+	    return (int) length; // The best we can do.
+	}
+    }
+	
+    /** Retrieve a property that has previously been set for this document.
+     * @param key the key String under which the vaue is stored
+     * @return the value which is also a String, or null if no such value
+     */
+    public Object getProperty(Object key) {
+	return properties.get(key);
+    }
+
+    /** Determine if the writer thread is still running, ie a log file which is currently open for writing
+     * @return true if the writer is still active, false otherwise
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread#isStillWriting
+     */
+    public boolean isStillWriting() {
+	return writer.isStillWriting();
+    }
+
+    /** Gets a sequence of text from the document. 
+     * @param offset the starting offset for the fragment of text required, as an int
+     * @param l the length of the text, also as an int
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement
+     * @see org.greenstone.gatherer.util.StaticStrings#EMPTY_STR
+     */
+    public String getText(int offset, int l) 
+	throws BadLocationException {
+
+	if (l == 0) {
+	    return "";
+	}
+
+	try {
+	    return read((long) offset, l);
+	}
+	catch (Exception ex) {
+	}
+
+	return "";
+
+//  	// System.err.println("AppendLineOnlyFileDocument::getText() request...");
+//  	String request = "getText(" + offset + ", " + l + ")";
+//  	///ystem.err.println(request);
+//  	String text = null;
+//  	if(text == null || text.length() < l) {
+//  	    try {
+//  		int file_length = (int) file.length();
+//  		///ystem.err.println("file_length = " + file_length + ", length = " + length);
+//  		if(l == 0) {
+//  		    text = StaticStrings.EMPTY_STR;
+//  		}
+//  		else if(0 <= offset && offset < length && (offset + l) <= length) {
+//  		    if(offset < file_length) {
+//  			text = read((long)offset, l);
+//  			if(text.length() != l) {
+//  			    ///ystem.err.println("Asked for " + l + " characters of text. But only read " + text.length());
+//  			    throw new BadLocationException("AppendLineOnlyDocument.getText(" + offset + ", " + l + ")", offset); 
+//  			}
+//  		    }
+//  		    else {
+//  			int index = root_element.getElementIndex(offset);
+//  			if(index < root_element.getElementCount()) {
+//  			    AppendLineOnlyFileDocumentElement element = (AppendLineOnlyFileDocumentElement) root_element.getElement(index);
+//  			    text = element.getContent();
+//  			}
+//  			else {
+//  			    ///ystem.err.println("Index is " + index + " but there are only " + root_element.getElementCount() + " children.");
+//  			    throw new BadLocationException("AppendLineOnlyDocument.getText(" + offset + ", " + l + ")", offset); 
+//  			}
+//  		    }
+		    
+//  		}
+//  	    }
+//  	    catch (IOException error) {
+//  		DebugStream.printStackTrace(error);
+//  	    }
+//  	    if(text == null) {
+//  		///ystem.err.println("Text is null.");
+//  		throw new BadLocationException("AppendLineOnlyDocument.getText(" + offset + ", " + l + ")", offset);
+//  	    }
+//  	}
+//  	request = null;
+//  	return text;
+    }
+
+    /** Fetches the text contained within the given portion of the document. 
+     * @param offset the starting offset of the desired text fragment, as an int
+     * @param length the length of the text also as an int
+     * @param txt the Segment into which the text fragment should be placed
+     */
+    public void getText(int offset, int length, Segment txt) 
+	throws javax.swing.text.BadLocationException  {
+	String str = getText(offset, length);
+	txt.array = str.toCharArray();
+	txt.count = str.length();
+	txt.offset = 0;
+	str = null;
+    }
+	
+    /** Set a property for this document.
+     * @param key the key to store the value under, as a String
+     * @param value the property value itself also as a String
+     */
+    public void putProperty(Object key, Object value) {
+	properties.put(key, value);
+    }
+
+    /** Determine if this document is ready by testing if it has an open descriptor to the random access file.
+     * @return true if the file exists thus the document is ready, false otherwise
+     */
+    public boolean ready() {
+	return (file != null);
+    }
+	
+    /** Removes a document listener.
+     * @param listener the Document we wish to remove
+     */
+    public void removeDocumentListener(DocumentListener listener) {
+	listeners_list.remove(DocumentListener.class, listener);
+    }
+
+    /** Whenever the user requests a change of the loaded log, this method is called to ensure the previous log has been completely written to file. Note that this only garuentees that text fragments currently queued for writing will be correctly written - while the behaviour is undefined for text fragments added after this call (some may write depending on several race conditions).
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread#exit
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.WriterThread#finish
+     */
+    public void setExit() {
+	writer.exit();
+	writer.finish();
+    }
+
+    /** Establish the current owner of this document.
+     * @param owner the current AppendLineOnlyFileDocumentOwner
+     */
+    public void setOwner(AppendLineOnlyFileDocumentOwner owner) {
+	this.owner = owner;
+    }
+
+    /** To record the final state of the logging process we reserve a single character at the start of the file, and then replace it when the build process is complete.
+     * @param character the final status char to replace the X at the start of the log
+     */
+    public synchronized void setSpecialCharacter(char character) {
+	if(build_log) {
+	    try {
+		file.seek(0L);
+		file.write((int)character);
+	    }
+	    catch (Exception error) {
+		DebugStream.printStackTrace(error);
+	    }
+	}
+    }
+
+    /** Request a text representation of this document which currently returns the filename.
+     * @return the text as a String
+     */
+    public String toString() {
+	return filename;
+    }
+
+    /** This method reads a fragment of text from the underlying random access file. It is synchronized so that the file isn't being written to at the same time.
+     * @param start_offset the offset within the file to set the read pointer at, as a long
+     * @param l the number of characters to read as an int
+     */
+    private synchronized String read(long start_offset, int l) 
+	throws IOException {
+	//print("read(" + start_offset + ", " + l + ")... ");
+	byte[] buffer = new byte[l];
+	file.seek(start_offset);
+	int result = file.read(buffer, 0, l);
+	return new String(buffer, "UTF-8");
+	//print("read() complete");
+    }
+
+    /** This methods writes a fragment of text to the underlying random access file. It is synchronized so that the file isn't being read from at the same time.
+     * @param start_offset the offset within the file to set the write pointer at, as a long
+     * @param end_offset the final offset of the new text fragment within the file, as a long. This is used to extend the file to be the correct length
+     * @param str the String to be written
+     * @param l the number of characters from str to be written, as an int
+     */
+    private synchronized void write(long start_offset, long end_offset, String str, int l) 
+	throws IOException {	
+	//print("write(" + start_offset + ", " + end_offset + ", " + str + ", " + l + ")");
+	file.setLength(end_offset);
+	file.seek(start_offset);
+	file.write(str.getBytes("UTF-8"), 0, l);
+	//print("write() complete");
+    }
+
+    /** This class provides the building-block for the document model. Each element is a line in the document, or has no content but contains several child elements. Due to the basic nature of what we are modelling only the root node has children. */
+    private class AppendLineOnlyFileDocumentElement
+	extends ArrayList
+	implements Element {
+	/** Our parent Element. */
+	private Element parent;
+	/** The offset to the end of our fragment(s) of text, in respect to the entire document. */
+	private long end;
+	/** The offset to the start of our fragment(s) of text, in respect to the entire document. */
+	private long start;
+	/** If we haven't been written to file yet, this contains the text fragment itself. */
+	private String content;
+
+	/** Construct a new root element, which can have no content, but calculates its start and end from its children. */
+	public AppendLineOnlyFileDocumentElement() {
+	    super();
+	    this.end = 0L;
+	    this.parent = null;
+	    this.start = 0L;
+	}
+
+	/** Construct a new element, whose content is found in bytes start to end - 1 within the random access file backing this document.
+	 * @param start the starting offset as a long
+	 * @param end the ending offset as a long
+	 */
+	public AppendLineOnlyFileDocumentElement(long start, long end) {
+	    super();
+	    this.end = end;
+	    this.parent = null;
+	    this.start = start;
+	}
+
+	/** Construct a new element, whose content is provided, but should at some later time be written to bytes start to end - 1 in the random access file backing this document.
+	 * @param start the starting offset as a long
+	 * @param end the ending offset as a long
+	 * @param content the text fragment String which we represent in the model
+	 */
+	public AppendLineOnlyFileDocumentElement(long start, long end, String content) {
+	    super();
+	    this.content = content;
+	    this.end = end;
+	    this.parent = null;
+	    this.start = start;
+	}
+
+	/** Add the given node as one of our children.
+	 * @param child the new child Element
+	 */
+	public void add(Element child) {
+	    super.add(child);
+	    ((AppendLineOnlyFileDocumentElement)child).setParent(this);
+	}
+
+	/** Having written the content to file this method removes it from the element. */ 
+	public void clearContent() {
+	    content = null;
+	}
+
+	/** This document does not allow content markup. 
+	 * @return always returns null
+	 */
+	public AttributeSet getAttributes() {
+	    return null;
+	}
+
+	/** Retrieve the current content of this element, which may be the text fragment this element represents.
+	 * @return either the text fragment as a String, or null if the fragment has already been written to disk
+	 */
+	public String getContent() {
+	    return content;
+	}
+
+	/** Fetches the document associated with this element. 
+	 * @return the AppendLineOnlyDocument containing this element
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument
+	 */
+	public Document getDocument() {
+	    return AppendLineOnlyFileDocument.this;
+	}
+
+	/** Fetches the child element at the given index. 
+	 * @param index the int index of the element to retrieve
+	 * @return the requested Element
+	 */
+	public Element getElement(int index) {
+	    Element element;
+	    if(0 <= index && index < size()) {
+		element = (Element) get(index);
+	    }
+	    else {
+		throw new IndexOutOfBoundsException("AppendLineOnlyDocument.AppendLineOnlyFileDocumentElement.getElement(" + index + ")");
+	    }
+	    return element;
+	}
+          
+	/** Gets the number of child elements contained by this element. 
+	 * @return an int
+	 */
+	public int getElementCount() {
+	    return size();
+	}
+
+	/** Gets the child element index closest to the given offset.
+	 * @param offset the offset within the text of the document, which due to the way the model is built must fll within the bounds of one of our child nodes
+	 * @return the closest index as an int
+	 */
+	public int getElementIndex(int offset) {
+	    int index = -1;
+	    if(parent != null) {
+		index = -1;
+	    }
+	    else if(offset < 0) {
+		index = 0;
+	    }
+	    else if(offset >= length) {
+		index = size() - 1;
+	    }
+	    else {
+		int size = size();
+		for(int i = 0; index == -1 && i < size; i++) {
+		    Element child = (Element) get(i);
+		    if(child.getStartOffset() <= offset && offset < child.getEndOffset()) {
+			index = i;
+		    }
+		    child = null;
+		}
+	    }
+	    return index;
+	}
+          
+	/** Fetches the offset from the beginning of the document that this element ends at. 
+	 * @return the offset as an int
+	 */
+	public int getEndOffset() {
+	    if(parent != null) {
+		return (int) end;
+	    }
+	    // Return the Documents length.
+	    else {
+		return (int) length;
+	    }
+	}
+          
+	/** This method retrieves the name of the element, however names are not important in this document so the name is always an empty string.
+	 * @return an empty String
+	 * @see org.greenstone.gatherer.util.StaticStrings#EMPTY_STR
+	 */
+	public String getName() {
+	    return StaticStrings.EMPTY_STR;
+	}
+          
+	/** Fetches the parent element. 
+	 * @return the parent Element
+	 */
+	public Element getParentElement() {
+	    return parent;
+	}
+          
+	/** Fetches the offset from the beginning of the document that this element begins at. 
+	 * @return the offset as an int
+	 */
+	public int getStartOffset() {
+	    if(parent != null) {
+		return (int) start;
+	    }
+	    else {
+		return 0;
+	    }
+	}
+          
+	/** Since this is a very simple model, only the root node can have children. All the children are leaves. 
+	 * @return true if this is the root node, false otherwise
+	 */
+	public boolean isLeaf() {
+	    return (parent != null);
+	} 
+
+	/** Establish the parent of this element
+	 * @param parent the parent Element
+	 */
+	public void setParent(Element parent) {
+	    this.parent = parent;
+	}
+    }
+    // AppendLineOnlyFileDocumentElement
+
+    /** This event is created to encapsulate the details of some change to the document. */
+    private class AppendLineOnlyFileDocumentEvent
+	implements DocumentEvent {
+	/** The type of change event. */
+	private DocumentEvent.EventType type;
+	/** The element this change occured to, or in. */
+	private AppendLineOnlyFileDocumentElement element;
+	/** Another encapsulating class which contains further detail about the change itself. */
+	private AppendLineOnlyFileDocumentElementChange element_change;
+	/** The length of the text fragment affected by the change. */
+	private int len;
+	/** The offset within the document of the start of the change. */
+	private int offset;
+	
+	/** Construct a new AppendLineOnlyFileDocumentEvent given the pertinant details.
+	 * @param element the AppendLineOnlyFileDocumentElement affected by this change
+	 * @param offset the offset within the document of the start of the change
+	 * @param len the length of the text fragment affected by the change
+	 * @param type the type of change event
+	 */
+	public AppendLineOnlyFileDocumentEvent(AppendLineOnlyFileDocumentElement element, int offset, int len, DocumentEvent.EventType type) {
+	    this.element = element;
+	    this.element_change = null;
+	    this.len = len;
+	    this.offset = offset;
+	    this.type = type;
+	}
+
+	/** Retrieve the docment which generated this event.
+	 * @return the Document
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument
+	 */
+	public Document getDocument() {
+	    return AppendLineOnlyFileDocument.this;
+	}
+
+	/** Retrieve the length of the change.
+	 * @return the length as an int
+	 */
+	public int getLength() {
+	    return len;
+	}
+
+	/** Retrieve the start offset of the change.
+	 * @return the offset as an int
+	 */
+	public int getOffset() {
+	    return offset;
+	}
+
+	/** Retrieve the type of change.
+	 * @return the type as a DocumentEvent.EventType
+	 */
+	public DocumentEvent.EventType getType() {
+	    return type;
+	}
+
+	/** Retrieve the element change object which further describes the changes. Of course given that we only allow the appending of new lines there really is only one type of event.
+	 * @param elem implementation side-effect
+	 * @return a DocumentEvent.ElementChange
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentEvent.AppendLineOnlyFileDocumentElementChange
+	 */
+	public DocumentEvent.ElementChange getChange(Element elem) {
+	    if(element_change == null) {
+		element_change = new AppendLineOnlyFileDocumentElementChange();
+	    }
+	    return element_change;
+	}
+
+	/** A pretty unexciting implementation of the document element change class. This usually describes the myriad of possible change events that can occur on a document. However we only allow line appends so there is really only one type of change. */
+	private class AppendLineOnlyFileDocumentElementChange
+	    implements DocumentEvent.ElementChange {
+	    /** This array always contains the one Element that was appended. */ 
+	    private Element[] children_added;
+	    /** This Element array is always empty. */
+	    private Element[] children_removed;
+	    /** The index of the affected element within its parents children. */
+	    private int index;
+	    /** Construct a new AppendLineOnlyFileDocumentElementChange with the default 'append line' details. */
+	    public AppendLineOnlyFileDocumentElementChange() {
+		children_added = new Element[1];
+		children_added[0] = element;
+		children_removed = new Element[0];
+		index = root_element.indexOf(element);
+	    }
+
+	    /** Gets the child element that was appended to the parent element. 
+	     * @return an Element[] containing the added element
+	     */
+	    public Element[] getChildrenAdded() {
+		return children_added;
+	    }
+
+	    /** This model does not allow elements to be removed.
+	     * @return an Element[] containing nothing
+	     */
+	    public Element[] getChildrenRemoved() {
+		return children_removed;
+	    }
+          
+	    /** Returns the root element, as our document structure is only two layers deep.
+	     * @return the root Element
+	     */
+	    public Element getElement() {
+		return root_element;
+	    }
+          
+	    /** Fetches the index within the element represented. 
+	     * @return an int specifying the index of change within the root element
+	     */
+	    public int getIndex() {
+		return index;
+	    }
+	}
+	// AppendLineOnlyFileDocumentElementChange
+    }
+    // AppendLineOnlyFileDocumentEvent
+    
+    /** This class denoted a position within our document by an offset. Quite possibly the last interesting class I've ever had to deal with, but you have to implement it so here goes... */
+    private class AppendLineOnlyFileDocumentPosition
+	implements Position {
+	/** The offset within our document. */
+	private int offset;
+	/** Construct a new position given an offset.
+	 * @param offset the offset as an int
+	 */
+	public AppendLineOnlyFileDocumentPosition(int offset) {
+	    this.offset = offset;
+	}
+	/** Retrieve the offset of this position.
+	 * @return the offset as an int
+	 */
+	public int getOffset() {
+	    return offset;
+	}
+    }
+    // AppendLineOnlyFileDocumentPosition
+    
+    /** This thread is responsible to writing the in memory text fragment contents of the document elements out to the random access file. The writing proceedure is implemented this way so that the gui doesn't take the performance hit of the IO. */
+    private class WriterThread
+	extends Thread {
+	/** When true was are being told to assume the contents of the queue are now static, and once completed the queue will be empty (and not expecting any new jobs). Used to finish the queue before the GLI exits. */
+	private boolean empty_queue;
+	/** Setting this flag to true tells the writer to stop regardless of whether there are items left on the queue. */
+	private boolean exit;
+	/** When the writer thread is busy writing content to the file this flag is set to true. */
+	private boolean running;
+	/** When the writer thread is running, but has emptied the current queue and so is waiting, this flag is set to true. */
+	private boolean waiting;
+	/** The fragment queue as a Vector (for thread reasons). */
+	private Vector queue;
+	/** Construct a new writer thread with an initially empty queue. */
+	public WriterThread() {
+	    super("WriterThread");
+	    empty_queue = false;
+	    exit = false;
+	    queue = new Vector();
+	    waiting = false;
+	}
+
+	/** This method is called to ask the writer thread to die after it finishes any current write proceceedure. */
+	public synchronized void exit() {
+	    //print("WriterThread.exit() start");
+	    exit = true;
+	    notify();
+	    //print("WriterThread.exit() complete");
+	}
+
+	/** When this returns there are no jobs waiting in the queue. */
+	public void finish() {
+	    if(!queue.isEmpty() && running) {
+		///ystem.err.println("Somethings gone wrong, as there are still " + queue.size() + " items to be written out and yet the thread seems to be waiting.");
+		empty_queue = true;
+		run();
+	    }
+	}
+
+	/** Determine if the writer is currently running.
+	 * @return true if it is, false if it has been exited or if it was never running in the first place (an old log)
+	 */
+	public boolean isStillWriting() {
+	    return running;
+	}
+
+	/** The run method of this thread checks if there are any document elements queued to be written to file, and then writes them as necessary.
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement
+	 */
+	public void run() {
+	    exit = false;
+	    running = true;
+	    while(!exit) {
+		if(!queue.isEmpty()) { 
+		    AppendLineOnlyFileDocumentElement element = (AppendLineOnlyFileDocumentElement) queue.remove(0);
+		    // Write the content to file.
+		    String content = element.getContent();
+		    if(content != null) {
+			try {
+			    write(element.getStartOffset(), element.getEndOffset(), content, content.getBytes("UTF-8").length);
+			}
+			catch(Exception error) {
+			    DebugStream.printStackTrace(error);
+			}
+			element.clearContent();
+		    }
+		}
+		else if(empty_queue) {
+		    exit = true;
+		}
+		else {
+		    synchronized(this) {
+			try {
+			    print("WriterThread.wait() start");
+			    waiting = true;
+			    wait();
+			}
+			catch(Exception exception) {
+			}
+			print("WriterThread.wait() complete");
+			waiting = false;
+		   }
+		}
+	    }
+	    print("WriterThread completely finished. Exiting.");
+	    running = false;
+	    if(owner != null) {
+		owner.remove(AppendLineOnlyFileDocument.this);
+	    }
+	}
+
+	/** Enqueue a document element to have its text content written to file.
+	 * @param element the Element needing its content written out
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.AppendLineOnlyFileDocumentElement#getContent
+	 */
+	public synchronized void queue(Element element) {
+	    print("WriterThread.queue(): " + ((AppendLineOnlyFileDocumentElement)element).getContent());
+	    queue.add(element);
+	    notify();
+	}
+    }
+    // WriterThread
+
+    // ***** METHODS WE ARE NOW IGNORING BECAUSE WE ARE VIRTUALLY READ-ONLY *****
+    
+    /** Not implemented. Add a new undo listener to this document.
+     * @param listener UndoableEditListener
+     */
+    public void addUndoableEditListener(UndoableEditListener listener) {}
+    	
+    /** Not implemented. Get the last position in the document.
+     * @return null
+     */
+    public Position getEndPosition() { return null; }
+
+    /** Not implemented. Gets all root elements defined.
+     * @return null
+     */
+    public Element[] getRootElements() { return null; }
+	
+    /** Not implemented. Retrieve the first position in the document.
+     * @return null
+     */
+    public Position getStartPosition() { return null; }
+    
+    /** Not implemented. Insert a text fragment into our document.
+     * @param offset int
+     * @param str String
+     * @param a AttributeSet
+     */
+    public void insertString(int offset, String str, AttributeSet a) {}
+
+    /** Not implemented. Removes some content from the document. 
+     * @param offs int
+     * @param len int
+     */
+    public void remove(int offs, int len) {}
+
+    /** Not implemented. Removes an undo listener. 
+     * @param listener UndoableEditListener
+     */
+    public void removeUndoableEditListener(UndoableEditListener listener) {}
+    
+    /** Not implemented. Renders a runnable apparently - whatever that means. 
+     * @param r Runnable
+     */
+    public void render(Runnable r) {}
+
+    // DEBUG ONLY
+
+    /** Print a message to the GLI debug file stream.
+     * @param message the message to be written as a String
+     */
+    static synchronized public void print(String message) {
+	if (message.endsWith("\n")) {
+	    DebugStream.print(message); 
+	}
+	else {
+	    DebugStream.println(message); 
+	}
+    }
+
+    /** Create a test document with the ability to add new lines of text as necessary.
+     * @param args the initial starting arguments as a String array
+     * @see org.greenstone.gatherer.gui.GLIButton
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument
+     * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.ReadButtonListener
+     */
+    static public void main(String[] args) {
+	JFrame frame = new JFrame("AppendLineOnlyFileDocument Test");
+	frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+	frame.setSize(640,480);
+	JPanel content = (JPanel) frame.getContentPane();
+	
+	//PlainDocument document = new PlainDocument();
+	//document.setAsynchronousLoadPriority(-1);
+	final AppendLineOnlyFileDocument document = new AppendLineOnlyFileDocument("temp.txt");
+
+	final JTextArea text_area = new JTextArea(document);
+
+	JButton read_button = new GLIButton("Read Huge File");
+	read_button.setMnemonic(KeyEvent.VK_R);
+	read_button.addActionListener(new ReadButtonListener(document));
+	content.setLayout(new BorderLayout());
+	content.add(new JScrollPane(text_area), BorderLayout.CENTER);
+	content.add(read_button, BorderLayout.SOUTH);
+
+	frame.setVisible(true);
+    }
+
+    /** Listens for actions on the read button, and if detected creates a new ReadTask to test the document. 
+     * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+     * @version 2.41 final
+     */
+    static private class ReadButtonListener
+	implements ActionListener {
+	/** The AppendLineOnlyFileDocument object we are testing. */
+	private AppendLineOnlyFileDocument document;
+
+	/** All the constructor does is take a copy of the document to test.
+	 * @param document the Document
+	 */
+	public ReadButtonListener(AppendLineOnlyFileDocument document) {
+	    this.document = document;
+	}
+
+	/** When the button is clicked this method is called to create the new task and run it.
+	 * @param event an ActionEvent containing further information about the button click
+	 * @see org.greenstone.gatherer.util.AppendLineOnlyFileDocument.ReadTask
+	 */
+	public void actionPerformed(ActionEvent event) {
+	    Thread task = new ReadTask(document);
+	    task.start();
+	}
+    }
+
+    /** This threaded task opens a large document, aptly named 'big.txt', and then bombards the document object we are testing with lines from the file. This file should be several megs (such as Alice or TREC) to fully test functionality, thread conditions etc. 
+     * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+     * @version 2.41 final
+     */ 
+    static private class ReadTask
+	extends Thread {
+	/** The AppendLineOnlyFileDocument object we are testing. */
+	private AppendLineOnlyFileDocument document;
+
+	/** Construct a new task which will perform testing on the given document.
+	 * @param document the AppendLineOnlyFileDocument to append lines to
+	 */
+	public ReadTask(AppendLineOnlyFileDocument document) {
+	    super("LoadHugeFileThread");
+	    this.document = document;
+	}
+
+	/** The runable part of this class opens the file, then reads it in a line at a time, appending these lines to the ever growing document. */
+	public void run() {
+	    // Load the specified document
+	    try {
+		BufferedReader in = new BufferedReader(new FileReader(new File("big.txt")));
+		String line;
+		
+		while ((line = in.readLine()) != null) {
+		    document.appendLine(line);
+		    try {
+			// Wait a random ammount of time.
+			synchronized(this) {
+			    //print("LoadHugeFileThread.wait() start");
+			    wait(100);
+			    //print("LoadHugeFileThread.wait() complete");
+			}
+		    }
+		    catch(Exception error) {
+			error.printStackTrace();
+		    }
+		}
+		
+	    } catch (Exception error) {
+		error.printStackTrace();
+	    }
+	}
+    }
+    // ReadTask
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocumentOwner.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocumentOwner.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/AppendLineOnlyFileDocumentOwner.java	(revision 21112)
@@ -0,0 +1,46 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Greenstone Librarian Interface (GLI) application, 
+ * part of the Greenstone digital library software suite from the New 
+ * Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
+ *
+ * Author: John Thompson
+ *         Greenstone Project, New Zealand Digital Library
+ *         University of Waikato
+ *         http://www.nzdl.org
+ *
+ * Copyright (C) 2004 New Zealand Digital Library, University of Waikato
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+
+/** Any class that wants to act as the owner of an AppendLineOnlyFileDocument has to implement this.
+ * @author John Thompson, Greenstone Project, New Zealand Digital Library, University of Waikato
+ * @version 2.41 final
+ */
+public interface AppendLineOnlyFileDocumentOwner {
+    /** Remove the given document from the owner. The is called by the document itself when it no longer needs a hanging reference in the owner class.
+     * @param document the AppendLineOnlyFileDocument to remove
+     */
+    public void remove(AppendLineOnlyFileDocument document);
+}
+
+
+
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ArrayTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ArrayTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ArrayTools.java	(revision 21112)
@@ -0,0 +1,353 @@
+/* GPL_HEADER */
+package org.greenstone.gatherer.util;
+
+/**************************************************************************************
+ * Title:        Gatherer
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.
+ * Company:      The University of Waikato
+ * Written:        /  /01
+ * Revised:      16/08/02 Improved
+ **************************************************************************************/
+import java.awt.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.filechooser.*;
+import org.w3c.dom.*;
+
+/** This utility class contains a series of static methods for common array type manipulations including appending, casting and changing between containers.
+ * @author John Thompson
+ * @version 2.3
+ */
+public class ArrayTools {
+
+    /** Append a Component onto an array of Components. */
+    static public Component[] add(Component[] a, Component b) {
+	Component[] c = null;
+	if(a != null && b != null) {
+	    c = new Component[a.length + 1];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    c[c.length - 1] = b;
+	}
+	else if(a == null && b != null) {
+	    c = new Component[1];
+	    c[0] = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficiently appends a new element onto the end of a element array.
+     * @param a The initial <strong>Element[]</strong>.
+     * @param b The new <strong>Element</strong>.
+     * @return An <strong>Element[]</strong> containing a followed by b.
+     */
+    static public Element[] add(Element a[], Element b) {
+	Element[] c = null;
+	if(a != null && b != null) {
+	    c = new Element[a.length + 1];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    c[c.length - 1] = b;
+	}
+	else if(a == null && b != null) {
+	    c = new Element[1];
+	    c[0] = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficently appends one element array onto the end of another.
+     * @param a The initial <strong>Element[]</strong>.
+     * @param b The <strong>Element[]</strong> to append.
+     * @return An <strong>Element[]</strong> containing a followed by b.
+     */
+    static public Element[] add(Element a[], Element b[]) {
+	Element[] c = null;
+	if(a != null && b != null) {
+	    c = new Element[a.length + b.length];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    System.arraycopy(b, 0, c, a.length, b.length);
+	}
+	else if(a == null && b != null) {
+	    c = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficiently appends a new file onto the end of a file array.
+     * @param a The initial <strong>File[]</strong>.
+     * @param b The new <strong>File</strong>.
+     * @return A <strong>File[]</strong> containing a followed by b.
+     */
+    static public File[] add(File a[], File b) {
+	File[] c = null;
+	if(a != null && b != null) {
+	    c = new File[a.length + 1];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    c[c.length - 1] = b;
+	}
+	else if(a == null && b != null) {
+	    c = new File[1];
+	    c[0] = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficently appends one file array onto the end of another.
+     * @param a The initial <strong>File[]</strong>.
+     * @param b The <strong>File[]</strong> to append.
+     * @return A <strong>File[]</strong> containing a followed by b.
+     */
+    static public File[] add(File a[], File b[]) {
+	File[] c = null;
+	if(a != null && b != null) {
+	    c = new File[a.length + b.length];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    System.arraycopy(b, 0, c, a.length, b.length);
+	}
+	else if(a == null && b != null) {
+	    c = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+
+
+    /** This method efficiently appends a new node onto the end of a node array.
+     * @param a The initial <strong>Node[]</strong>.
+     * @param b The new <strong>Node</strong>.
+     * @return A <strong>Node[]</strong> containing a followed by b.
+     */
+    static public Node[] add(Node a[], Node b) {
+	Node[] c = null;
+	if(a != null && b != null) {
+	    c = new Node[a.length + 1];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    c[c.length - 1] = b;
+	}
+	else if(a == null && b != null) {
+	    c = new Node[1];
+	    c[0] = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficently appends one node array onto the end of another.
+     * @param a The initial <strong>Node[]</strong>.
+     * @param b The <strong>Node[]</strong> to append.
+     * @return A <strong>Node[]</strong> containing a followed by b.
+     */
+    static public Node[] add(Node a[], Node b[]) {
+	Node[] c = null;
+	if(a != null && b != null) {
+	    c = new Node[a.length + b.length];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    System.arraycopy(b, 0, c, a.length, b.length);
+	}
+	else if(a == null && b != null) {
+	    c = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+	 
+    /** This method efficiently appends a new string onto the end of a string array.
+     * @param a The initial <strong>String[]</strong>.
+     * @param b The new <strong>String</strong>.
+     * @return A <strong>String[]</strong> containing a followed by b.
+     */
+    static public String[] add(String a[], String b) {
+	String[] c = null;
+	if(a != null && b != null) {
+	    c = new String[a.length + 1];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    c[c.length - 1] = b;
+	}
+	else if(a == null && b != null) {
+	    c = new String[1];
+	    c[0] = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method efficently appends one string array onto the end of another.
+     * @param a The initial <strong>String[]</strong>.
+     * @param b The <strong>String[]</strong> to append.
+     * @return A <strong>String[]</strong> containing a followed by b.
+     */
+    static public String[] add(String a[], String b[]) {
+	String[] c = null;
+	if(a != null && b != null) {
+	    c = new String[a.length + b.length];
+	    System.arraycopy(a, 0, c, 0, a.length);
+	    System.arraycopy(b, 0, c, a.length, b.length);
+	}
+	else if(a == null && b != null) {
+	    c = b;
+	}
+	else if(a != null && b == null) {
+	    c = a;
+	}
+	return c;
+    }
+    /** This method takes an array list and creates a string array.
+     * @param a An <strong>ArrayList</strong> containing hopefully <strong>String</strong> or else this will fail.
+     * @return A <strong>String[]</strong> or <i>null</i> if the array could not be created.
+     */
+    static public String[] arrayListToStringArray(ArrayList a) {
+	String array[] = new String[a.size()];
+	for(int i = 0; i < array.length; i++) {
+	    array[i] = (String)a.get(i);
+	}
+	return array;
+    }
+
+    static public File[] filter(File[] files, String pattern, boolean exclude) {
+	int write_ptr = 0;
+	///ystem.err.println("Filtering by '" + pattern + "', Exclude? " + exclude + " :");
+	for(int read_ptr = 0; read_ptr < files.length; read_ptr++) {
+	    File current = files[read_ptr];
+	    files[write_ptr] = current;
+	    ///ystem.err.print("Testing " + current.getName() + " -> ");
+	    // Determine whether we move the write pointer or not.
+	    if(current.getName().toLowerCase().matches(pattern)) {
+		if(!exclude) {
+		    ///ystem.err.println("Match, Exclude.");
+		    write_ptr++;  
+		}
+		else {
+		    ///ystem.err.println("Match, Include.");
+		}
+	    }
+	    else {
+		// You can't exclude folders with an inclusion filter!
+		if(exclude || current.isDirectory()) {
+		    ///ystem.err.println("Nonmatch, Exclude.");
+		    write_ptr++;
+		}
+		else {
+		    ///ystem.err.println("Nonmatch, Include.");
+		}
+	    }
+	}
+	File[] result = new File[write_ptr];
+	System.arraycopy(files, 0, result, 0, result.length);
+	pattern = null;
+	files = null;
+	return result;
+    }
+
+    /** Method to convert a list of nodes from the DOM model into a node array. This is quite useful as using a <strong>NodeList</strong> is problematic if you wish to traverse the tree while editing. Because they are 'live' representations of the model you are editing, it is easy to drop out of for loops early or run off the ends of arrays.
+     * @param node_list A <strong>NodeList</strong> representing the <strong>Node</strong>s we want to iterate through.
+     * @return A brand new <strong>Node[]</strong> with the first <strong>Node</strong> in the <strong>NodeList</strong> at the head.
+     * @see org.w3c.dom.Node
+     * @see org.w3c.dom.NodeList
+     */
+    static public Node[] nodeListToNodeArray(NodeList node_list) {
+	Node nodes[] = null;
+	for(int i = 0; i < node_list.getLength(); i++) {
+	    nodes = add(nodes, node_list.item(i));
+	}
+	return nodes;
+    }
+    /** Transforms an Object array into a single string of the form '[o1,o2,...,on]'.
+     * @param objects An <strong>Object[]</strong>. Note that the objects in this array must support toString() reasonably.
+     * @return A <strong>String</strong> representing the given array.
+     */
+    static public String objectArrayToString(Object objects[]) {
+	StringBuffer result = new StringBuffer("[");
+	for(int i = 0; i < objects.length; i++) {
+	    result.append(objects[i].toString());
+	    if(i < objects.length - 1) {
+		result.append(",");
+	    }
+	}
+	result.append("]");
+	return result.toString();
+    }
+
+
+    /** Sorts an array of files, putting non-files first. Case insensitive.
+     * @param files_to_sort The File[] to be sorted.
+     */
+    static public void sort(File[] files_to_sort)
+    {
+	// Return if there is nothing to sort
+	if (files_to_sort == null || files_to_sort.length <= 1) {
+	    return;
+	}
+
+	FileSystemView fileSystemView = FileSystemView.getFileSystemView();
+	FileComparator comparator = new FileComparator();
+
+	// Separate the file system roots and directories from the files, and sort separately
+	ArrayList files_list = new ArrayList((files_to_sort.length * 3) / 4);
+	ArrayList non_files_list = new ArrayList((files_to_sort.length) / 4);
+	for (int i = 0; i < files_to_sort.length; i++) {
+	    File file = files_to_sort[i];
+
+	    // File is a system root
+	    if (fileSystemView.isFileSystemRoot(file)) {
+		non_files_list.add(file);
+	    }
+	    // File is a directory
+	    else if (file.isDirectory()) {
+		non_files_list.add(file);
+	    }
+	    // File is an actual file
+	    else {
+		files_list.add(file);
+	    }
+	}
+
+	// Sort files
+	Object[] files = files_list.toArray();
+	Arrays.sort(files, comparator);
+
+	// Sort non-files
+	Object[] non_files = non_files_list.toArray();
+	Arrays.sort(non_files, comparator);
+
+	// Merge the results, putting non-files before files
+	int j = 0;
+	for (int i = 0; i < non_files.length; i++) {
+	    files_to_sort[j++] = (File) non_files[i];
+	}
+	for (int i = 0; i < files.length; i++) {
+	    files_to_sort[j++] = (File) files[i];
+	}
+    }
+
+
+    /** Comparator used to order files. */
+    static private class FileComparator
+	implements Comparator {
+
+	/** Compare two files in terms of ordering of their paths.
+	 * @param o1 The <strong>Object</strong> that represents the first file.
+	 * @param o2 The other <strong>Object</strong>, also a file.
+	 * @return An <i>int</i> which is <1, 0 or >1 if o1 is <o2, =o2 or >o2 respectively.
+	 */
+	public int compare(Object o1, Object o2)
+	{
+	    File f1 = (File) o1;
+	    File f2 = (File) o2;
+
+	    return f1.getName().compareToIgnoreCase(f2.getName());
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckList.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckList.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckList.java	(revision 21112)
@@ -0,0 +1,224 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.border.*;
+
+
+/** This class provides a visual component that has the form of a list, as provided by JList but uses JCheckBox for data selection. Thus several elements can be 'ticked' in the list, and this selection returned using the method getSelected().<BR>Parts of this code modified from Trevor Harmon's posting on www.dejanews.com.
+ * @author John Thompson
+ * @version 2.3
+ */ 
+public class CheckList
+    extends JList
+{
+    /** The border used when a list row is not in focus. */
+    static private Border noFocusBorder = new EmptyBorder(1, 1, 1, 1);
+
+    private boolean show_selected_row = true;
+
+
+    /** Constructor. */
+    public CheckList(boolean show_selected_row)
+    {
+	super();
+	this.show_selected_row = show_selected_row;
+
+	setCellRenderer(new CheckListCellRenderer());
+	setModel(new DefaultListModel());
+	setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	addMouseListener(new CheckListMouseListener());
+    }
+
+
+    public void addEntry(CheckListEntry entry)
+    {
+	DefaultListModel model = (DefaultListModel) getModel();
+
+	// Add the entry in alpabetical order
+	String name = entry.toString();
+	for (int i = 0; i < model.size(); i++) {
+	    Object sibling = model.getElementAt(i);
+	    if (name.compareTo(sibling.toString()) <= 0) {
+		model.add(i, entry);
+		return;
+	    }
+	}
+
+	model.addElement(entry);
+    }
+
+
+    public void clearTicked()
+    {
+	DefaultListModel model = (DefaultListModel) getModel();
+	for (int i = 0; i < model.size(); i++) {
+	    ((CheckListEntry) model.get(i)).setSelected(false);
+	}
+	updateUI();
+    }
+
+
+    /** Retrieve the currently ticked entries from this list.
+     * @return An <strong>ArrayList</strong> containing only those entries from the initial list that are checked.
+     * @see org.greenstone.gatherer.checklist.Entry
+     */
+    public ArrayList getTicked()
+    {
+	ArrayList result = new ArrayList();
+	DefaultListModel model = (DefaultListModel) getModel();
+	for (int i = 0; i < model.size(); i++) {
+	    CheckListEntry entry = (CheckListEntry) model.get(i);
+	    if (entry.isSelected()) {
+		result.add(entry.getObject());
+	    }
+	}
+	return result;
+    }
+
+
+    /** This is different from isSelectionEmpty! */
+    public boolean isNothingTicked()
+    {
+	DefaultListModel model = (DefaultListModel) getModel();
+	for (int i = 0; i < model.size(); i++) {
+	    if (((CheckListEntry) model.get(i)).isSelected()) {
+		return false;
+	    }
+	}
+
+	return true;
+    }
+
+
+    public void setListData(ArrayList list_data)
+    {
+ 	// Create a new model.
+	setModel(new DefaultListModel());
+
+	// Add the items from the list to the model
+ 	for (int i = 0; i < list_data.size(); i++) {
+ 	    Object list_object = list_data.get(i);
+ 	    if (list_object instanceof CheckListEntry) {
+ 		addEntry((CheckListEntry) list_object);
+ 	    }
+	    else {
+		addEntry(new CheckListEntry(list_object));
+	    }
+ 	}
+    }
+
+
+    public void setTickedObjects(Object[] objects)
+    {
+	if (objects == null) {
+	    return;
+	}
+
+	DefaultListModel model = (DefaultListModel) getModel();
+	for (int i = 0; i < model.size(); i++) {
+	    CheckListEntry entry = (CheckListEntry) model.get(i);
+	    for (int j = 0; j < objects.length; j++) {
+		if (entry.getObject().equals(objects[j])) {
+		    entry.setSelected(true);
+		}
+	    }
+	}
+	updateUI();
+    }
+
+
+    /** A custom list cell renderer for producing rows which contain clickable check boxes. */
+    protected class CheckListCellRenderer 
+	implements ListCellRenderer
+    {
+	/** Return a component that has been configured to display the specified value. That component's paint method is then called to "render" the cell. If it is necessary to compute the dimensions of a list because the list cells do not have a fixed size, this method is called to generate a component on which getPreferredSize  can be invoked.
+	 * @param list The </strong>JList</strong> we're painting.
+	 * @param value The value returned by list.getModel().getElementAt(index), as an <strong>Object</strong>.
+	 * @param index The cells index as an <i>int</i>.
+	 * @param is_selected <i>true</i> if the specified cell was selected, <i>false</i> otherwise.
+	 * @param cell_has_focus <i>true</i> if and only if the specified cell has the focus.
+	 * @return A <strong>Component</strong> whose paint() method will render the specified value.
+	 */
+	public Component getListCellRendererComponent(JList list, Object value, int index, boolean is_selected, boolean cell_has_focus) {
+	    JCheckBox checkbox = (JCheckBox) value;
+	    if (show_selected_row) {
+		checkbox.setBackground(is_selected ? list.getSelectionBackground() : list.getBackground());
+		checkbox.setForeground(is_selected ? list.getSelectionForeground() : list.getForeground());
+		checkbox.setBorderPainted(true);
+	    }
+	    else {
+		checkbox.setBackground(list.getBackground());
+		checkbox.setForeground(list.getForeground());
+		checkbox.setBorderPainted(false);
+	    }
+	    checkbox.setEnabled(list.isEnabled());
+	    checkbox.setFont(list.getFont());
+	    checkbox.setFocusPainted(false);
+	    checkbox.setBorder((is_selected) ? UIManager.getBorder("List.focusCellHighlightBorder") : noFocusBorder);
+	    return checkbox;
+	}
+    }
+
+
+    /** Listens for clicks apon the checks within the list, and updates as necessary. */
+    private class CheckListMouseListener 
+	extends MouseAdapter
+    {
+	/** Called whenever the mouse is clicked over our list. We find the nearest checkbox and change its state.
+	 * @param e A <strong>MouseEvent</strong> containing everything you ever wanted to know about the mouse event but were afraid to ask.
+	 */
+	public void mousePressed(MouseEvent e)
+	{
+	    JList list = (JList) e.getSource();
+	    int index = list.locationToIndex(e.getPoint());
+	    CheckListEntry checkbox = (CheckListEntry) list.getModel().getElementAt(index);
+	    if (!checkbox.isFixed()) {
+		checkbox.setSelected(!checkbox.isSelected());
+	    }
+	    checkbox.grabFocus();
+
+	    // We need to cause a ListSelectionEvent -- this is ugly but I can't find a better way quickly
+	    list.removeSelectionInterval(0, 0);
+	    list.setSelectionInterval(index, index);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckListEntry.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckListEntry.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/CheckListEntry.java	(revision 21112)
@@ -0,0 +1,75 @@
+package org.greenstone.gatherer.util;
+
+import javax.swing.JCheckBox;
+
+/** A CheckListEntry encapsulates a single row of the list, both its check box and the object the row represents. */
+public class CheckListEntry
+    extends JCheckBox 
+    implements Comparable {
+
+    /** Is this checkboxes state fixed. */
+    private boolean fixed = false;
+    /** The original object this row is based on, and whose toString() is used to generate the check box label. */
+    private Object object = null;
+    /** A sundry string for storing original values (rather than strings retrieved from dictionary). */
+    private String property;
+    /** Constructor. */
+    public CheckListEntry(Object object) {
+	super(object.toString());
+	this.object = object;
+    }
+
+    /** Constructor. */
+    public CheckListEntry(String text, boolean is_selected) {
+	super(text);
+	setSelected(is_selected);
+    }
+
+    /** Test this CheckListEntry against an Object for ordering. 
+     * @param obj the other Object.
+     * @return <0, 0 or >0 if this CheckListEntry is before, equal to or after the given object.
+     */
+    public int compareTo(Object obj) {
+	return toString().toLowerCase().compareTo(obj.toString().toLowerCase());
+    }
+    /** Test this CheckListEntry against an Object for equality. 
+     * @param obj the other Object.
+     * @return true if the two are equal, false otherwise.
+     */
+    public boolean equals(Object obj) {
+	return toString().equalsIgnoreCase(obj.toString());
+    }
+
+    /** Retrieve the object associated with this entry.
+     * @return An <strong>Object</strong>.
+     */
+    public Object getObject() {
+	if(object == null) {
+	    return getText();
+	}
+	return object;
+    }
+
+    public String getProperty() {
+	return property;
+    }
+
+    public boolean isFixed() {
+	return fixed;
+    }
+
+    public void setFixed(boolean fixed) {
+	this.fixed = fixed;
+    }
+
+    public void setProperty(String property) {
+	this.property = property;
+    }
+
+    public String toString() {
+	if(object == null) {
+	    return getText();
+	}
+	return object.toString();
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Codec.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Codec.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Codec.java	(revision 21112)
@@ -0,0 +1,379 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.util.*;
+
+/** Provides a standard, extensible way to convert from one format of string to another (given that each format has differing requirements regarding legal characters and escaped characters)
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3d
+ */
+public class Codec {
+    
+    static final public String DECODE_PATH = "DECODE_PATH";
+    static final public String DECODE_SQUARE_BRACKETS = "DECODE_SQUARE_BRACKETS";
+    static final public String DOM_TO_GREENSTONE = "DOM_TO_GREENSTONE";
+    static final public String DOM_TO_TEXT = "DOM_TO_TEXT";
+    static final public String ENCODE_PATH = "ENCODE_PATH";
+    static final public String ENCODE_SQUARE_BRACKETS = "ENCODE_SQUARE_BRACKETS";
+    static final public String GREENSTONE_TO_DOM = "GREENSTONE_TO_DOM";
+    static final public String GREENSTONE_TO_TEXT = "GREENSTONE_TO_TEXT";
+    static final public String TEXT_TO_DOM = "TEXT_TO_DOM";
+    static final public String TEXT_TO_GREENSTONE = "TEXT_TO_GREENSTONE";
+    static final public String TEXT_TO_REGEXP = "TEXT_TO_REGEXP";
+    static final public String TEXT_TO_SHELL_UNIX = "TEXT_TO_SHELL_UNIX";
+    static final public String TEXT_TO_SHELL_WINDOWS = "TEXT_TO_SHELL_WINDOWS";
+    
+    static final private int MAX_CACHE_SIZE = 100;
+    
+    static private HashMap TRANSFORMS;
+    static private HashMap3D CACHE;
+    
+    /** Static function called to construct TRANSFORMS mappings */
+    static {
+	TRANSFORMS = new HashMap();
+
+	String[] decode_path = {
+	    "\\|",    "\\\\",
+	    "&#124;", "\\|"
+	};
+	TRANSFORMS.put(DECODE_PATH, decode_path);
+	decode_path = null;
+
+	// Transform text into text, but without [ and ]
+	String[] decode_square_brackets = {
+	    "&#091;", "\\[",
+	    "&#093;", "\\]" 
+	};
+	TRANSFORMS.put(DECODE_SQUARE_BRACKETS, decode_square_brackets);
+	decode_square_brackets = null;
+
+	// Translate DOM encoded text into Greenstone encoding
+	String[] dom_to_greenstone = {
+	    "&apos;", "\\\\\'",
+	    "&gt;", ">",
+	    "&lt;", "<",
+	    "&quot;", "\\\\\"",
+	    "&amp;", "&"
+	};
+	//  removed "\n", "\\\\n", - config files are allowed new lines
+	// added "\\|", "\\\\"
+
+	TRANSFORMS.put(DOM_TO_GREENSTONE, dom_to_greenstone);
+	dom_to_greenstone = null;
+
+	// Transform DOM encoded text into plain text
+	String[] dom_to_text = {
+	   "&amp;#091;", "\\[",
+	   "&amp;#093;", "\\]",
+	   "&apos;", "\'",
+	   "&gt;", ">",
+	   "&lt;", "<",
+	   "&quot;", "\"",
+	   "&amp;", "&"
+	};
+	TRANSFORMS.put(DOM_TO_TEXT, dom_to_text);
+	dom_to_text = null;
+
+	// Transform text into a regular expression that will match it
+	String[] text_to_regexp = {
+	    "\\\\", "\\\\\\\\",
+	    "\\(", "\\\\(",
+	    "\\)", "\\\\)",
+	    "\\[", "\\\\[",
+	    "\\]", "\\\\]",
+	    "\\{", "\\\\{",
+	    "\\}", "\\\\}",
+	    "\\.", "\\\\."
+	};
+	TRANSFORMS.put(TEXT_TO_REGEXP, text_to_regexp);
+	text_to_regexp = null;
+
+	String[] encode_path = {
+	    "\\|",  "&#124;",
+	    "\\\\", "\\|"
+	};
+	TRANSFORMS.put(ENCODE_PATH, encode_path);
+	encode_path = null;
+
+	// Transform text into text, but without [ and ]
+	String[] encode_square_brackets = {
+	    "\\[", "&#091;",
+	    "\\]", "&#093;"
+	};
+	TRANSFORMS.put(ENCODE_SQUARE_BRACKETS, encode_square_brackets);
+	encode_square_brackets = null;
+
+	// Transform Greenstone encoded text to DOM encoding
+	String[] greenstone_to_dom = {
+	    "&", "&amp;",
+	    "<", "&lt;",
+	    ">", "&gt;",
+	    "\\\\\"", "&quot;",
+	    "\\\\\'", "&apos;",
+	    "\"", "&quot;",
+	    "\'", "&apos;"
+	};
+	// removed"\\\\n", "\n", added "\\\\", "\\|"
+
+	TRANSFORMS.put(GREENSTONE_TO_DOM, greenstone_to_dom);
+	greenstone_to_dom = null;
+
+	// Transform Greenstone encoded text to plain text
+	String[] greenstone_to_text = {
+	    "\\\\\"", "\"",
+	    "\\\\\'", "\'",
+	    "&quot;", "\"",
+	    "&apos;", "\'",
+	    "&#091;", "\\[",
+	    "&#093;", "\\]"
+	};
+	// removed    "\\\\n", "\n", "\\|", "\\\\"
+
+	TRANSFORMS.put(GREENSTONE_TO_TEXT, greenstone_to_text);
+	greenstone_to_text = null;
+
+	// Transform plain html text into something that can be placed in a DOM
+	String[] text_to_dom = {
+	    "&", "&amp;",
+	    "<", "&lt;",
+	    ">", "&gt;",
+	    "\"", "&quot;",
+	    "\'", "&apos;"
+	};
+	TRANSFORMS.put(TEXT_TO_DOM, text_to_dom);
+	text_to_dom = null;
+
+	// Transform plain html text into greenstone encoding
+	String[] text_to_greenstone = {
+
+	    "\\[", "&#091;",
+	    "\\]", "&#093;",
+	    "\"", "&quot;",
+	    "\n", "\\\\n"
+	};
+	// 	    "\'", "&apos;",
+	// removed "\\\\", "\\|",
+	TRANSFORMS.put(TEXT_TO_GREENSTONE, text_to_greenstone);
+	text_to_greenstone = null;
+
+	// Transform plain html text into something that can be placed in a shell command
+	String[] text_to_shell_unix = {
+	    "\"", "\\\\\"",
+	    "\'", "\\\\\'",
+	    "\n", "\\\\n"
+	};
+	TRANSFORMS.put(TEXT_TO_SHELL_UNIX, text_to_shell_unix);
+	text_to_shell_unix = null;
+
+	// Transform plain html text into something that can be placed in a shell command. Windows requires twice as many escaped for speech marks to be passed to underlying processes
+	String[] text_to_shell_windows = {
+	    "\"", "\\\\\\\\\\\\\"",
+	    "\'", "\\\\\'",
+	    "\n", "\\\\n"
+	};
+	TRANSFORMS.put(TEXT_TO_SHELL_WINDOWS, text_to_shell_windows);
+	text_to_shell_windows = null;
+
+	CACHE = new HashMap3D();
+    }
+
+    static public String transform(String raw, String transform) {
+	if(raw == null) {
+	    return raw;
+	}
+	// System.err.println("Transforming by "+transform+":\n" + raw);
+	String processed = (String) CACHE.get(transform, raw);
+	if(processed == null) {
+	    processed = raw;
+	    String[] transforms = (String[]) TRANSFORMS.get(transform);
+	    if(transforms != null) {
+		for(int i = 0; i < transforms.length; i = i + 2) {
+		    String target = transforms[i];
+		    String result = transforms[i+1];
+		    processed = processed.replaceAll(target, result);
+		}
+	    }
+	    //DebugStream.println("\n*** Transform: " + transform + " ***");
+	    //DebugStream.println("*** Raw      : '" + raw + "'");
+	    //DebugStream.println("*** Processed: '" + processed + "'");
+	    // If cache is at maximum size, empty it and start again
+	    if(CACHE.size() == MAX_CACHE_SIZE) {
+		CACHE.clear();
+	    }
+	    CACHE.put(transform, raw, processed);
+	}
+	return processed;
+    }
+
+    /** Transform either of the accepted unicode escape sequences styles from in the string into single characters */
+    static final private char AND_CHAR = '&';
+    static final private char ESCAPE_CHAR = '\\';
+    static final private char HASH_CHAR = '#';
+    static final private char LOWER_U_CHAR = 'u';
+    static final private char UPPER_U_CHAR = 'U';
+    static final private char SEMICOLON_CHAR = ';';
+
+    static public String transformUnicode(String raw) {
+	StringBuffer processed = new StringBuffer();
+	int index = 0;
+	int raw_length = raw.length();
+	while(index < raw_length) {
+	    char c0 = raw.charAt(index);
+	    switch(c0) {
+	    case AND_CHAR:
+		if(index + 1 < raw_length) {
+		    // First the HTML &#231; type
+		    char c1 = raw.charAt(index + 1);
+		    if(c1 == HASH_CHAR) {
+			StringBuffer number_str = new StringBuffer();
+			char c2;
+			int offset = 2;
+			while(index + offset < raw_length && (c2 = raw.charAt(index + offset)) != SEMICOLON_CHAR) {
+			    number_str.append(c2);
+			    offset++;
+			}
+			// We've either run out of characters or have parsed a number
+			if(index + offset < raw_length && raw.charAt(index + offset) == SEMICOLON_CHAR) {
+			    int number = Integer.parseInt(number_str.toString());
+			    processed.append((char)number);
+			    index = index + offset;
+			    number_str = null;
+			    break;
+			}
+			number_str = null;
+		    }
+		}
+		processed.append(c0);
+		break;
+	    case ESCAPE_CHAR:
+		// Now the \u00e7 type
+		if(index + 1 < raw_length) {
+		    char c3 = raw.charAt(index + 1);
+		    if((c3 == UPPER_U_CHAR || c3 == LOWER_U_CHAR) && index + 5 < raw_length) {
+			// We read four digits
+			String hex_str = raw.substring(index + 2, index + 6);
+			int number = Integer.parseInt(hex_str, 16);
+			hex_str = null;
+			processed.append((char)number);
+			index = index + 5;
+			break;
+		    }
+		}
+		processed.append(c0);
+		break;
+	    default:
+		processed.append(c0);
+	    }
+	    index++;
+	}
+	return processed.toString();
+    }
+
+    static public void main(String[] args) {
+	if(args.length < 2) {
+	    String processed;
+	    String raw;
+	    String transform;
+
+	    System.err.println("Running Test Suite");
+
+	    transform = "DOM_TO_GREENSTONE";
+	    System.err.println("Test " + transform);
+	    raw = "A &amp;lt;\nand a &lt;a href=&quot;here.html&quot;&gt;&lt;font size=&apos;2&apos;&gt;URL&lt;/font&gt;&lt;/a&gt;";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "DOM_TO_TEXT";
+	    System.err.println("Test " + transform);
+	    raw = "A &amp;lt;\nand a &lt;a href=&quot;here.html&quot;&gt;&lt;font size=&apos;2&apos;&gt;URL&lt;/font&gt;&lt;/a&gt;";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "GREENSTONE_TO_DOM";
+	    System.err.println("Test " + transform);
+	    raw = "A &lt;\\nand a <a href=\\\"here.html\\\"><font size=\\\'2\\\'URL</font></a>";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "GREENSTONE_TO_TEXT";
+	    System.err.println("Test " + transform);
+	    raw = "These \\[ \\] should be escaped, and so should \\\\ that. These &quot; &apos; \\n are encoded.";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "TEXT_TO_DOM";
+	    System.err.println("Test " + transform);
+	    raw = "A &lt;\nand a <a href=\"here.html\"><font size='2'>URL</font></a>";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "TEXT_TO_GREENSTONE";
+	    System.err.println("Test " + transform);
+	    raw = "These [ ] should be escaped, and so should \\ that. These \" \' \n are encoded.";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    transform = "TEXT_TO_SHELL";
+	    System.err.println("Test " + transform);
+	    if(Utility.isWindows()) {
+		System.err.println("[Windows Version]");
+		transform = "TEXT_TO_SHELL_WINDOWS";
+	    }
+	    else {
+		System.err.println("[Unix Version]");
+		transform = "TEXT_TO_SHELL_UNIX";
+	    }
+	    raw = "A &lt;\nand a <a href=\"here.html\"><font size='2'>URL</font></a>";
+	    System.err.println("Raw:       '" + raw + "'");
+	    processed = transform(raw, transform);
+	    System.err.println("Processed: '" + processed + "'");
+
+	    System.err.println("***** UNICODE TEST *****");
+	    System.err.println("\\u0030 => " + transformUnicode("\\u0030"));
+	    System.err.println("\\u0041 => " + transformUnicode("\\u0041"));
+	    System.err.println("\\u007a => " + transformUnicode("\\u007a"));
+	    System.err.println("\\u00e7 => " + transformUnicode("\\u00e7"));
+	    System.err.println("&#48;   => " + transformUnicode("&#48;"));
+	    System.err.println("&#65;   => " + transformUnicode("&#65;"));
+	    System.err.println("&#122;  => " + transformUnicode("&#122;"));
+	    System.err.println("&#231;  => " + transformUnicode("&#231;"));
+	}
+	else {
+	    System.err.println("Raw:       '" + args[0] + "'");
+	    System.err.println("Transform: " + args[1]);
+	    String processed = transform(args[0], args[1]);
+	    System.err.println("Processed: '" + processed + "'");
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DOMTree.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DOMTree.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DOMTree.java	(revision 21112)
@@ -0,0 +1,149 @@
+package org.greenstone.gatherer.util;
+
+import java.util.*;
+import javax.swing.*;
+import javax.swing.tree.*;
+import org.w3c.dom.*;
+
+public class DOMTree
+    extends JTree {
+
+    public DOMTree(Document document) {
+	super();
+	setDocument(document);
+    }
+
+    public void setDocument(Document document) {
+	setModel(new DefaultTreeModel(new DOMTreeNode(document.getDocumentElement(), null)));
+    }
+
+    private class DOMTreeNode 
+	implements TreeNode {
+
+	private Node node = null;
+	private NodeList children = null;
+	private String text = null;
+	private TreeNode parent = null;
+	private TreeNode self = null;
+
+	DOMTreeNode(Node node, TreeNode parent) {
+	    this.node = node;
+	    this.parent = parent;
+	    this.self = this;
+	}
+	
+	/** Returns the children of the receiver as an Enumeration. */
+	public Enumeration children() {
+	    if(children == null) {
+		children = node.getChildNodes();
+	    }
+	    return new NodeListEnumeration();
+	}
+          
+	/** Returns true if the receiver allows children. */
+	public boolean getAllowsChildren() {
+	    return true;
+	}
+          
+	/** Returns the child TreeNode at index childIndex. */
+	public TreeNode getChildAt(int childIndex) {
+	    if(children == null) {
+		children = node.getChildNodes();
+	    }
+	    return new DOMTreeNode(children.item(childIndex), this);
+	}
+          
+	/** Returns the number of children TreeNodes the receiver contains. */
+	public int getChildCount() {
+	    if(children == null) {
+		children = node.getChildNodes();
+	    }
+	    return children.getLength();
+	}
+          
+	/** Returns the index of node in the receivers children. */
+	public int getIndex(TreeNode find_node) {
+	    if(children == null) {
+		children = node.getChildNodes();
+	    }
+	    Node target_node = ((DOMTreeNode)find_node).getNode();
+	    int children_count = children.getLength();
+	    for(int i = 0; i < children_count; i++) {
+		Node test_node = children.item(i);
+		if(target_node == test_node) {
+		    test_node = null;
+		    target_node = null;
+		    return i;
+		}
+		test_node = null;
+	    }
+	    target_node = null;
+	    return -1;
+	}
+	    
+	public Node getNode() {
+	    return node;
+	}
+
+	/** Returns the parent TreeNode of the receiver. */
+	public TreeNode getParent() {
+	    return parent;
+	}
+	    
+	/** Returns true if the receiver is a leaf. */
+	public boolean isLeaf() {
+	    return !node.hasChildNodes();
+	}
+
+	public String toString() {
+	    if(text == null) {
+		StringBuffer temp = new StringBuffer(node.getNodeName());
+		String value = node.getNodeValue();
+		if(value != null && value.length() != 0) {
+		    temp.append("=\"");
+		    temp.append(value);
+		    temp.append("\"");
+		}
+		value = null;
+		if(node instanceof Element) {
+		    Element element = (Element) node;
+		    temp.append(": ");
+		    NamedNodeMap attributes = element.getAttributes();
+		    int attribute_count = attributes.getLength();
+		    for(int i = 0; i < attribute_count; i++) {
+			Node attribute = attributes.item(i);
+			temp.append(attribute.getNodeName());
+			temp.append("=\"");
+			temp.append(attribute.getNodeValue());
+			if(i < attribute_count - 1) {
+			    temp.append("\" ");
+			}
+			else {
+			    temp.append("\"");
+			}
+		    }
+		    attributes = null;
+		    element = null;
+		}
+		text = temp.toString();
+		temp = null;
+	    }
+	    return text;
+	}
+
+	private class NodeListEnumeration
+	    implements Enumeration {
+	    private int index = 0;
+	    /** Tests if this enumeration contains more elements. */
+	    public boolean hasMoreElements() {
+		return index < children.getLength();
+	    }
+	    /** Returns the next element of this enumeration if this enumeration object has at least one more element to provide. */
+	    public Object nextElement() {
+		int old_index = index;
+		index = index + 1;
+		return new DOMTreeNode(children.item(old_index), self);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragComponent.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragComponent.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragComponent.java	(revision 21112)
@@ -0,0 +1,65 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+/**
+ * Title:        Gatherer<br>
+ * Description:  The Gatherer: a tool for gathering and enriching a digital collection.<br>
+ * Copyright:    Copyright (c) 2001<br>
+ * Company:      The University of Waikato<br>
+ * Written:        /  /01<br>
+ * Revised:      21/06/02 - Moved into correct package and commented.<br>
+ * @author John Thompson, 9826509
+ * @version 2.1
+ */
+import java.awt.event.FocusListener;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.file.FileSystemModel;
+
+/** Any component that wants to act a GTree like drag'n'drop component must implement this. */
+public interface DragComponent {
+    public void addFocusListener(FocusListener listener);
+    /** In order for the appearance to be consistant, given we may be in the situation where the pointer has left our focus but the ghost remains, this method allows other members of the Group to tell this component to repair the 'spoilt' region left by its ghost. */
+    public void clearGhost();
+    /** Used to notify this component that it has gained focus by some method other that mouse focus. */
+    public void gainFocus();	 
+    /** Retrieve the model associated with this component. */
+    public FileSystemModel getTreeModel();
+    /** This method is used to inform this component when it loses focus by means other than a drag mouse event, and should indicate this somehow. */
+    public void loseFocus();
+    /** Set the components group. */
+    public void setGroup(DragGroup group);
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragGroup.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragGroup.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragGroup.java	(revision 21112)
@@ -0,0 +1,136 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.awt.Point;
+import java.awt.event.*;
+import java.awt.image.BufferedImage;
+import java.util.Vector;
+import javax.swing.JComponent;
+import javax.swing.tree.TreePath;
+import org.greenstone.gatherer.gui.tree.DragTree;
+
+
+/** This class acts as a linker between all the various drag and drop enabled DragComponent. It provides methods for ensuring only one component is the drop target, showing extra user feedback, and maintains a single point of focus. Moreover it provides a storage space for necessary shared variables such as mouse_offset of initial drag etc. */
+public class DragGroup 
+    implements FocusListener {
+    /** The image used for the ghost icon when dragging. */
+    public BufferedImage image_ghost = null;
+    /** The initial offset of the mouse from the selection within a DragComponent. Then as you drag the mouse, the ghost appears in a visually consistant place, and remains at the same offset throughout the dragging process. */
+    public Point mouse_offset = null;
+    /** The component currently in charge of drawing the ghost icon. Note that this isn't necessarily the only component that needs to repaint 'spoilt' screen real-estate as a side effect of the ghost. */
+    private DragComponent ghost_owner = null;
+    /** The component where the drag began, so that extra information necessary for the drag can be garnered by the drop target, or so that focus can return to the source component if the drop is rejected. */
+    private DragTree drag_source = null;
+    /** The selected nodes, as determined when the drag begun. */
+    private TreePath[] selection = null;
+    /** A list of DragComponents registered as being part of this group. */
+    private Vector registered = new Vector();
+
+    /** Register a DragComponent as begin part of this group.
+     * @param component The DragComponent to add.
+     */
+    public void add(DragComponent component) {
+	if(!registered.contains(component)) {
+	    registered.add(component);
+	    component.addFocusListener(this);
+	    component.setGroup(this);
+	}
+    }
+
+    /** Invoked when a component gains the keyboard focus. */
+    public void focusGained(FocusEvent event) {
+	((DragComponent)event.getComponent()).gainFocus();
+    }
+          
+    /** Invoked when a component loses the keyboard focus. */
+    public void focusLost(FocusEvent event) {
+	((DragComponent)event.getComponent()).loseFocus();
+    }          
+    /** Determines the current 'active' component, ie that component with focus.
+     * @return The DragComponent which is currently responsible for drawing the ghost, and thus is active.
+     */
+    public DragComponent getActive() {
+	return ghost_owner;
+    }
+    /** Retrieve the nodes selected at the beginning of this drag operation.
+     * @return A TreePath[].
+     */
+    public TreePath[] getSelection() {
+	return selection;
+    }
+    /** Retrieve the component which served as the source of this drag action.
+     * @return The DragComponent in question.
+     */
+    public DragTree getSource() {
+	return drag_source;
+    }
+    /** When called this method asserts that one of the registered GComponents just became the proud owner of focus, and is henceforth responsible for drawing the ghost, and acting like a component thats in focus.
+     * @param new_owner The DragComponent that has gained focus during a drag action.
+     */
+    public void grabFocus(DragComponent new_owner) {
+	// Tell the previous owner of ghost to clear itself.
+	if(ghost_owner != null) {
+	    ghost_owner.clearGhost();
+	}
+	// Assign new owner
+	ghost_owner = new_owner;
+	// Then tell all other GComponents to indicate they aren't in focus in whatever way they do that (ie selected items of a GTree go gray). Of course the new owners told it is in focus.
+	for(int i = 0; i < registered.size(); i++) {
+	    DragComponent comp = (DragComponent)registered.get(i);
+	    if(comp == new_owner) {
+		comp.gainFocus();
+	    }
+	    else {
+		comp.loseFocus();
+	    }
+	}
+    }
+    /** Sets the value of selected_nodes.
+     * @param selection The new value for selected_nodes as a TreePath[].
+     */
+    public void setSelection(TreePath[] selection) {
+	this.selection = selection;
+    }
+    /** Sets the value of drag_source.
+     * @param drag_source The new value for drag_source as a DragTree.
+     */
+    public void setSource(DragTree drag_source) {
+	this.drag_source = drag_source;
+	///ystem.err.println("The drag source is now " + drag_source);
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragTreeSelectionModel.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragTreeSelectionModel.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/DragTreeSelectionModel.java	(revision 21112)
@@ -0,0 +1,278 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.awt.Point;
+import java.awt.event.*;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.gui.tree.DragTree;
+
+public class DragTreeSelectionModel
+    extends DefaultTreeSelectionModel
+    implements MouseListener {
+
+    private boolean immediate = false;
+    private boolean mixed_selection = false;
+
+    private int type               = -1;
+
+    private TreePath temp_path     = null;
+    private TreePath temp_paths[]  = null;
+
+    static private final int NONE  = -1;
+    static private final int ADD   =  0;
+    static private final int SET   =  1;
+
+    /** Constructor.
+     * @param tree the DragTree which will use this selection model.
+     * @param mixed_selection true if this selection model allows selections to contain files and folders, false otherwise.
+     */
+    public DragTreeSelectionModel(DragTree tree, boolean mixed_selection) {
+	super();
+	this.mixed_selection = mixed_selection;
+	setSelectionMode(DISCONTIGUOUS_TREE_SELECTION);
+	tree.addMouseListener(this);
+    }
+
+    public void addSelectionPath(TreePath path) {
+	if(!immediate) {
+	    this.temp_path = path;
+	    this.type = this.ADD;
+	}
+	else if(isAppropriate(path)) {
+	    temp_path = null;
+	    super.addSelectionPath(path);
+	}
+    }
+
+    public void addSelectionPaths(TreePath paths[]) {
+	if(!immediate) {
+	    this.temp_paths = paths;
+	    this.type = this.ADD;
+	}
+	else if(isAppropriate(paths, true)) {
+	    temp_paths = null;
+	    super.setSelectionPaths(paths);
+	}
+    }
+
+    public String getDetails() {
+	String suffix = null;
+	int file_count = 0;
+	int folder_count = 0;
+	for(int i = 0; selection != null && i < selection.length; i++) {
+	    TreeNode node = (TreeNode) selection[i].getLastPathComponent();
+	    if(node.isLeaf()) {
+		file_count++;
+	    }
+	    else {
+		folder_count++;
+	    }
+	}
+
+	// Update the metaaudit_suffix
+	String args[] = null;
+	if(file_count > 0 && folder_count > 0) {
+	    if(file_count > 1 && folder_count > 1) {
+		args = new String[2];
+		args[0] = String.valueOf(file_count);
+		args[1] = String.valueOf(folder_count);
+		suffix = Dictionary.get("FileActions.Selected", args);
+	    }
+	    else if(file_count > 1) {
+		args = new String[1];
+		args[0] = String.valueOf(file_count);
+		suffix = Dictionary.get("FileActions.Files_And_Directory_Selected", args);
+	    }
+	    else if(folder_count > 1) {
+		args = new String[1];
+		args[0] = String.valueOf(folder_count);
+		suffix = Dictionary.get("FileActions.File_And_Directories_Selected", args);
+	    }
+	    else {
+		suffix = Dictionary.get("FileActions.File_And_Directory_Selected");
+	    }
+	}
+	else if(file_count > 0) {
+	    if(file_count > 1) {
+		args = new String[1];
+		args[0] = String.valueOf(file_count);
+		suffix = Dictionary.get("FileActions.Files_Selected", args);
+	    }
+	    else if(file_count == 1) {
+		suffix = Dictionary.get("FileActions.File_Selected");
+	    }
+	}
+	else if(folder_count > 0) {
+	    if(folder_count > 1) {
+		args = new String[1];
+		args[0] = String.valueOf(folder_count);
+		suffix = Dictionary.get("FileActions.Directories_Selected", args);
+	    }
+	    else {
+		suffix = Dictionary.get("FileActions.Directory_Selected");
+	    }
+	}
+	args = null;
+
+	return suffix;
+    }
+
+    /** Any implementation of MouseListener must include this method so
+     * we can be informed when the mouse is clicked.
+     * @param event A MouseEvent containing all the information about
+     * this mouse click.
+     */
+    public void mouseClicked(MouseEvent event) {
+    }
+
+    /** Any implementation of MouseListener must include this method so
+     * we can be informed when the mouse enters a component.
+     * @param event A MouseEvent containing all the information about
+     * this mouse action.
+     */
+    public void mouseEntered(MouseEvent event) {
+    }
+
+    /** Any implementation of MouseListener must include this method so
+     * we can be informed when the mouse exits a component.
+     * @param event A MouseEvent containing all the information about
+     * this mouse action.
+     */
+    public void mouseExited(MouseEvent event) {
+    }
+
+    /** Any implementation of MouseListener must include this method so
+     * we can be informed when the mouse is pressed (start of click).
+     * @param event A MouseEvent containing all the information about
+     * this mouse action.
+     */
+    public void mousePressed(MouseEvent event) {
+    }
+
+    /** Any implementation of MouseListener must include this method so
+     * we can be informed when the mouse is released (end of click).
+     * @param event A MouseEvent containing all the information about
+     * this mouse action.
+     */
+    public void mouseReleased(MouseEvent event) {
+	try {
+	switch(this.type) {
+	case 0: // this.ADD
+	    if(this.temp_path != null && isAppropriate(temp_path)) {
+		super.addSelectionPath(this.temp_path);
+		this.temp_path = null;
+	    }
+	    if(this.temp_paths != null && isAppropriate(temp_paths, true)) {
+		super.addSelectionPaths(this.temp_paths);
+		this.temp_paths = null;
+	    }
+	    this.type = this.NONE;
+	    break;
+	case 1: // this.SET
+	    if(this.temp_path != null) {
+		super.setSelectionPath(this.temp_path);
+		this.temp_path = null;
+	    }
+	    if(this.temp_paths != null && isAppropriate(temp_paths, false)) {
+		super.setSelectionPaths(this.temp_paths);
+		this.temp_paths = null;
+	    }
+	    this.type = this.NONE;
+	    break;
+	}
+	}
+	catch (Exception error) {
+		DebugStream.printStackTrace(error);
+	}
+    }
+
+    public void setImmediate(boolean value) {
+	immediate = value;
+    }
+
+    public void setSelectionPath(TreePath path) {
+	// Since this is only a single path we don't need to check whether its an appropriate selection.
+	if(!immediate) {
+	    this.temp_path = path;
+	    this.type = this.SET;
+	}
+	else {
+	    temp_path = null;
+	    super.setSelectionPath(path);
+	}
+    }
+
+    public void setSelectionPaths(TreePath paths[]) {
+	if(!immediate) {
+	    this.temp_paths = paths;
+	    this.type = this.SET;
+	}
+	else if(isAppropriate(paths, false)) {
+	    temp_paths = null;
+	    super.setSelectionPaths(paths);
+	}
+    }
+
+    /** Ensure that the given path is appropriate to add to the current selection, preventing mixed selections of files and folder if required. We also must check that no path is a ancestor/descendant of another.
+     */
+    private boolean isAppropriate(TreePath path) {
+	boolean appropriate = true;
+	TreeNode new_node = (TreeNode) path.getLastPathComponent();
+	// If there is a current selection
+	if(selection != null && selection.length > 0) {
+	    for(int i = 0; appropriate && i < selection.length; i++) {
+		TreeNode current_node = (TreeNode) selection[i].getLastPathComponent();
+		appropriate = appropriate && (mixed_selection || new_node.isLeaf() == current_node.isLeaf());
+		appropriate = appropriate && !path.isDescendant(selection[i]) && !selection[i].isDescendant(path);
+	    }
+	}
+	return appropriate;
+    }
+    /** Ensure that the given paths are appropriate to add to the current selection, preventing mixed selections of files and folder if required. We also must check that no path is a ancestor/descendant of another. One last detail to keep in mind is that adding selections depends upon the current selection, whereas set the selection paths doesn't (replaces them instead) and thus no check of the current paths is needed. */
+    private boolean isAppropriate(TreePath[] paths, boolean check_current) {
+	boolean appropriate = true;
+	if(check_current && paths != null && paths.length > 0) {
+	    for(int i = 0; appropriate && i < paths.length; i++) {
+		appropriate = appropriate && isAppropriate(paths[i]);
+	    }
+	}
+	return appropriate;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ExternalProgram.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ExternalProgram.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ExternalProgram.java	(revision 21112)
@@ -0,0 +1,107 @@
+package org.greenstone.gatherer.util;
+
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import org.greenstone.gatherer.DebugStream;
+
+
+/**
+ * This class provides a convenient method of running external programs and viewing/analysing
+ * their output. The external programs may be run with or without arguments. 
+ */
+public class ExternalProgram
+{
+    private Process programProcess = null;
+
+    private BufferedReader programOut = null;
+    private BufferedReader programErr = null;
+
+
+    /**
+     * Prepares to run a specified external program with the specified arguments.
+     *
+     * @param programName The name (and path, if necessary) of the program to run.
+     * @param programArgs The arguments to provide to the program when run.
+     */
+    public ExternalProgram(String programName, String programArgs)
+	throws IOException
+    {
+	// Run the program
+	programProcess = Runtime.getRuntime().exec(programName + " " + programArgs);
+
+	// Grab the program output
+	programOut = new BufferedReader(new InputStreamReader(programProcess.getInputStream()));
+	programErr = new BufferedReader(new InputStreamReader(programProcess.getErrorStream()));
+
+    }
+
+
+
+    /**
+     */
+    public int exitProgram()
+	throws IOException
+    {
+	// Finished, thank you
+	//programIn.close();
+
+	// Wait for the process to finish
+	try {
+	    programProcess.waitFor();
+	}
+	catch (InterruptedException ex) {
+	    throw new Error("Internal Error: Process interrupted.\n" + ex);
+	}
+
+	// Return the program exit value
+	return programProcess.exitValue();
+    }
+
+
+    /**
+     * Returns a single line of output sent by the program to standard output.
+     *
+     * @return a single line of output sent by the program to standard output.
+     */
+    public String getLineOfProgramOutput()
+    {
+	// Read in a line from the program output
+	try {
+	    if (programOut.ready())
+		return programOut.readLine();
+	    else
+		return null;
+	}
+	catch (IOException ex) {
+	    DebugStream.println("Error: Exception occurred while reading program output.");
+	    DebugStream.println("Exception: " + ex);
+	    return null;
+	}
+    }
+
+
+    /**
+     * Returns a single line of output sent by the program to standard error.
+     *
+     * @return a single line of output sent by the program to standard error.
+     */
+    public String getLineOfProgramError()
+    {
+	// Read in a line from the program error
+	try {
+	    if (programErr.ready())
+		return programErr.readLine();
+	    else
+		return null;
+	}
+	catch (IOException ex) {
+	    DebugStream.println("Error: Exception occurred while reading program error.");
+	    DebugStream.println("Exception: " + ex);
+	    return null;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GSDLSiteConfig.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GSDLSiteConfig.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GSDLSiteConfig.java	(revision 21112)
@@ -0,0 +1,205 @@
+package org.greenstone.gatherer.util;
+
+import java.io.*;
+import java.util.*;
+
+public class GSDLSiteConfig
+    extends LinkedHashMap {
+    private File gsdlsite_cfg;
+    private File glisite_cfg;
+    private String autoenter_initial;
+    private String start_browser_initial;
+    
+    static final private String AUTOENTER = "autoenter";
+    static final private String COLON = ":";
+    static final private String ENTERLIB = "enterlib";
+    static final private String FALSE = "0";
+    static final private String GLISITE_CFG = "glisite.cfg";
+    static final private String GSDL = "gsdl";
+    static final private String GSDLSITE_CFG = "gsdlsite.cfg";
+    static final private String LOCAL_HOST = "http://localhost"; 
+    static final private String PORTNUMBER = "portnumber";
+    static final private String SEPARATOR = "/";
+    static final private String SPECIFIC_CONFIG = "--config=";
+    static final private String STARTBROWSER = "start_browser";
+    static final private String TRUE = "1";
+    static final private String URL = "url";
+
+    public GSDLSiteConfig(File server_exe) {
+	debug("New GSDLSiteConfig for: " + server_exe.getAbsolutePath());
+	gsdlsite_cfg = new File(server_exe.getParentFile(), GSDLSITE_CFG);
+	glisite_cfg = new File(server_exe.getParentFile(), GLISITE_CFG);
+	autoenter_initial = null;
+	start_browser_initial = null;
+	if(gsdlsite_cfg.exists()) {
+	    debug("Load: " + gsdlsite_cfg.getAbsolutePath());
+	    clear();
+	    try {
+		BufferedReader in = new BufferedReader(new FileReader(gsdlsite_cfg));
+		String line = null;
+		while((line = in.readLine()) != null) {
+		    String key = null;
+		    String value = null;
+		    int index = -1;
+		    if((index = line.indexOf("=")) != -1 && line.length() >= index + 1) {
+			key = line.substring(0, index);
+			value = line.substring(index + 1);
+		    }
+		    else {
+			key = line;
+		    }
+		    put(key, value);
+		}
+	    }
+	    catch (Exception error) {
+		error.printStackTrace();
+	    }
+	}
+	else {
+	    debug("No GSDLsite.cfg file can be found!");
+	}
+    }
+
+    public boolean exists() {
+	return gsdlsite_cfg.exists();
+    }
+
+    public String getLocalHostURL() {
+	StringBuffer url = new StringBuffer(LOCAL_HOST);
+	url.append(COLON);
+	url.append((String)get(PORTNUMBER));
+	String enterlib = (String)get(ENTERLIB);
+	if(enterlib == null || enterlib.length() == 0) {
+	    // Use the default /gsdl and hope for the best.
+	    url.append(SEPARATOR);
+	    url.append(GSDL);
+	}
+	else {
+	    if(!enterlib.startsWith(SEPARATOR)) {
+		url.append(SEPARATOR);
+	    }
+	    url.append(enterlib);
+	}
+	enterlib = null;
+	debug("Found Local Library Address: " + url.toString());
+	return url.toString();
+    }
+
+    public String getSiteConfigFilename() {
+	return SPECIFIC_CONFIG + glisite_cfg.getAbsolutePath();
+    }
+
+    public String getURL() {
+	// URL is made from url and portnumber
+	String url = (String) get(URL);
+	if(url != null) {
+	    StringBuffer temp = new StringBuffer(url);
+	    temp.append(COLON);
+	    temp.append((String)get(PORTNUMBER));
+	    String enterlib = (String)get(ENTERLIB);
+	    if(enterlib == null || enterlib.length() == 0) {
+		// Use the default /gsdl and hope for the best.
+		temp.append(SEPARATOR);
+		temp.append(GSDL);
+	    }
+	    else {
+		if(!enterlib.startsWith(SEPARATOR)) {
+		    temp.append(SEPARATOR);
+		}
+		temp.append(enterlib);
+	    }
+	    enterlib = null;
+	    url = temp.toString();
+	}
+	debug("Found Local Library Address: " + url);
+	return url;
+    }
+
+    public void load() {
+	if(glisite_cfg.exists()) {
+	    debug("Load: " + glisite_cfg.getAbsolutePath());
+	    clear();
+	    try {
+		BufferedReader in = new BufferedReader(new FileReader(glisite_cfg));
+		String line = null;
+		while((line = in.readLine()) != null) {
+		    String key = null;
+		    String value = null;
+		    int index = -1;
+		    if((index = line.indexOf("=")) != -1 && line.length() >= index + 1) {
+			key = line.substring(0, index);
+			value = line.substring(index + 1);
+		    }
+		    else {
+			key = line;
+		    }
+		    put(key, value);
+		}
+	    }
+	    catch (Exception error) {
+		error.printStackTrace();
+	    }
+	}
+	else {
+	    debug("No GSDLsite.cfg file can be found!");
+	}
+    }
+
+    /** Restore the autoenter value to its initial value, and remove url if present. */
+    public void restore() {
+	if(glisite_cfg != null) {
+	    // Delete the file
+	    glisite_cfg.delete();
+	}
+	else {
+	    debug("Restore Initial Settings");
+	    put(AUTOENTER, autoenter_initial);
+	    put(STARTBROWSER, start_browser_initial);
+	    remove(URL);
+	    save();
+	}
+    }
+
+    public void set() {
+	debug("Set Session Settings");
+	if(autoenter_initial == null) {
+	    autoenter_initial = (String) get(AUTOENTER);
+	    debug("Remember autoenter was: " + autoenter_initial);
+	}
+	put(AUTOENTER, TRUE);
+	if(start_browser_initial == null) {
+	    start_browser_initial = (String) get(STARTBROWSER);
+	    debug("Remember start_browser was: " + start_browser_initial);
+	}
+	put(STARTBROWSER, FALSE);
+	save();
+    }
+
+    private void debug(String message) {
+	///ystem.err.println(message);
+    }
+
+    private void save() {
+	//debug("Save: " + gsdlsite_cfg.getAbsolutePath());
+	debug("Save: " + glisite_cfg.getAbsolutePath());
+	try {
+	    //BufferedWriter out = new BufferedWriter(new FileWriter(gsdlsite_cfg, false));
+	    BufferedWriter out = new BufferedWriter(new FileWriter(glisite_cfg, false));
+	    for(Iterator keys = keySet().iterator(); keys.hasNext(); ) {
+		String key = (String) keys.next();
+		String value = (String) get(key);
+		out.write(key, 0, key.length());
+		if(value != null) {
+		    out.write('=');
+		    out.write(value, 0, value.length());
+		}
+		out.newLine();
+	    }
+	    out.flush();
+	    out.close();
+	}
+	catch (Exception error) {
+	    error.printStackTrace();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GURL.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GURL.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/GURL.java	(revision 21112)
@@ -0,0 +1,183 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.io.*;
+import java.lang.*;
+import java.net.*;
+import java.util.*;
+
+/**
+ * Extends the standard URL class to include several new data elements and
+ * the ability to take a string and parse a valid url.
+ */
+public class GURL
+    implements Cloneable, Serializable {
+
+    private Integer type;
+
+    private URL url;
+
+    private String file;
+    private String host;
+    private String path;
+    private String protocol;
+    private String title = "empty";
+
+    /** This Vector maintains a record of the url links from within
+     * this resource, and their type (EXTERNAL vs INTERNAL). Internal
+     * links can be 'hidden' from the user if desired, or a user can
+     * opt to have all internal links automatically moved into a
+     * collection if the page that depends upon them is moved. <BR>
+     * Each entry in the vector is of the form.
+     * links[i] = String url
+     * links[i+1] = int type
+     */
+    private Vector links;
+
+    /* Static Integer identifiers for ease of use. */
+    static public Integer INTERNAL = new Integer(0);
+    static public Integer EXTERNAL = new Integer(1);
+
+    public GURL(String url_str) {
+	this(url_str, GURL.EXTERNAL);
+    }
+
+    public GURL(URL url) {
+	this(url, GURL.EXTERNAL);
+    }
+
+    public GURL(URL url, Integer type) {
+	this.type = type;
+	this.url = url;
+    }
+
+    public GURL(String url_str, Integer type) {
+	this.type = type;
+
+	try {
+	    if(!url_str.startsWith("file:") && url_str.indexOf("://") == -1) {
+		url_str = "http://" + url_str;
+	    }
+	    url = new URL(url_str);
+	}
+	catch (MalformedURLException e) {
+	    // Fix it
+	}
+    }
+
+    public void addLink(String url) {
+	addLink(url, GURL.EXTERNAL);
+    }
+
+    public void addLink(String url, Integer type) {
+	if(links == null) {
+	    links = new Vector();
+	}
+	links.add(url);
+	links.add(type);
+    }
+
+    /** Used by tree renderer to display name. So we'll only return the
+     * filename.
+     */
+    public String getFile() {
+	return parseFile(url.getFile());
+    }
+
+    public String getHost() {
+	return url.getHost();
+    }
+
+    /* private String getLocalFile() {
+	String local_file = getPath() + File.separator + getFile();
+	// Strip msdos roots if they're still there.
+	while(local_file.indexOf(":") != -1) {
+	    local_file = local_file.substring(0, local_file.indexOf(":")) + local_file.substring(local_file.indexOf(":") + 1);
+	}
+	return local_file;
+	} */
+
+    public String getPath() {
+	return getHost() + parsePath(url.getFile());
+    }
+
+    public int getPort() {
+	return url.getPort();
+    }
+
+    public String getProtocol() {
+	String protocol = url.getProtocol();
+	if(protocol.equals("file")) {
+	    return protocol + ":";
+	}
+	return protocol + "://";
+    }
+
+    public URL getURL() {
+	return url;
+    }
+
+    public String toString() {
+	return url.toString();
+    }
+
+    public boolean valid() {
+	return (url != null);
+    }
+
+    private String parseFile(String raw) {
+	if(raw != null && !raw.equals("")) {
+	    // Remove everything upto and including the last '/'
+	    if(raw.indexOf("/") != -1) {
+		return raw.substring(raw.lastIndexOf("/") + 1);
+	    }
+	}
+	return raw;
+    }
+
+    private String parsePath(String raw) {
+	if(raw != null && !raw.equals("")) {
+	    // Remove everything after the last "/"
+	    if(raw.indexOf("/") != -1) {
+		return raw.substring(0, raw.lastIndexOf("/"));
+	    }
+	}
+	return raw;
+    }
+
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HTMLStringTokenizer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HTMLStringTokenizer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HTMLStringTokenizer.java	(revision 21112)
@@ -0,0 +1,142 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+
+/** This class functions much like a <strong>StringTokenizer</strong> in that it tokenizes a long string into tokens, however this tokenizer cleverly notices HTML formatting tags. */
+public class HTMLStringTokenizer {
+    /** The current position in the source string. */
+    private int pos = 0;
+    /** The current token, usually created by the last nextToken call. */
+    private String current = null;
+    /** The previous token. */
+    private String previous = null;
+    /** The string to be tokenized, including any HTML markup. */
+    private String source = null;
+    /** Constructor.
+     * @param source The source <strong>String</strong> to be tokenized.
+     */
+    public HTMLStringTokenizer(String source) {
+	this.source = source;
+	// Parse the first token.
+	parseToken();
+    }
+
+    /** Determines if there are still tokens remaining unparsed in the source.
+     * @return A <strong>boolean</strong> which is <i>true</i> if there are more tokens.
+     */ 
+    public boolean hasMoreTokens() {
+	if(current != null && current.length() > 0) {
+	    return true;
+	}
+	return false;
+    } 
+
+    /** Determines if the tag currently being returned by sameToken is a tag.
+     * @return A <strong>boolean</strong> indicating if the token is a tag.
+     */
+    public boolean isTag() {
+	if(previous.startsWith("<") && previous.endsWith(">")) {
+	    return true;
+	}
+	return false;
+    }
+
+    /** Retrieves the next token. 
+     * @return A <strong>String</strong> representing the token.
+     */
+    public String nextToken() {
+	previous = current;
+	// Get the next token.
+	parseToken();
+	// Return previous.
+	return previous;
+    }
+
+    /** Parses the next token and stores it in current.
+     */
+    private void parseToken() {
+	boolean found = false;
+	boolean tag = false;
+	boolean text = false;
+	// Reset current
+	current = "";
+	// Parse away
+	dumpWhiteSpace();
+	while(pos < source.length() && !found) {
+	    char c = (char)source.charAt(pos);
+	    if(!tag && !text) {
+		if(c == '<') {
+		    tag = true;
+		}
+		else {
+		    text = true;
+		}
+		current = current + c;
+	    }
+				// Reading a tag. Watch only for '>'.
+	    else if(tag) {
+		if(c == '>') {
+		    found = true;
+		}
+		current = current + c;
+	    }
+				// Reading text. Watch for ' ' and '<'. Rollback '<'.
+	    else if(text) {
+		if(c == ' ') {
+		    found = true;
+		}
+		else if(c == '<') {
+		    found = true;
+		    pos--;
+		}
+		else {
+		    current = current + c;
+		}
+	    }
+	    pos++;
+	}
+    }
+
+    /** Method to ignore whitespace in the source.
+     */
+    private void dumpWhiteSpace() {
+	while(pos < source.length() && source.charAt(pos) == ' ') {
+	    pos++;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HashMap3D.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HashMap3D.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/HashMap3D.java	(revision 21112)
@@ -0,0 +1,138 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+/**************************************************************************************
+ * Written:      20/08/02
+ * Revised:      
+ **************************************************************************************/
+import java.util.HashMap;
+import java.util.Iterator;
+
+/** Provides a HashMap implementation that indexes by two keys. Perfect for the storage of metadata references based on their metadata element and assigned value. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class HashMap3D
+    extends HashMap {
+
+    private int capacity = 4;
+
+    private int size = 0;
+
+    private Object last_key_one = null;
+    private Object last_key_two = null;
+    private Object last_value = null;
+
+    /** Default constructor.
+     */
+    public HashMap3D() {
+	super();
+    }
+    /** Constructor with a specific initial capacity.
+     * @param capacity The initial capacity as an <i>int</i>.
+     */
+    public HashMap3D(int capacity) {
+	super(capacity);
+	this.capacity = capacity;
+    }
+    /** Completely remove the contents of this HashMap and its child HashMaps. */
+    public void clear() {
+	size = 0;
+	Iterator iterator = values().iterator();
+	while(iterator.hasNext()) {
+	    HashMap inner_mapping = (HashMap) iterator.next();
+	    inner_mapping.clear();
+	    inner_mapping = null;
+	}
+	iterator = null;
+	super.clear();
+    }
+    /** Determine if this hash map contains an entry for the given keys. Also cache this entry because theres a good chance the next get call will ask for this entry.
+     * @param key_one The first key as an <strong>Object</strong>.
+     * @param key_two The second key as an <strong>Object</strong>.
+     * @return <i>true</i> if such an entry exists, <i>false</i> otherwise.
+     */
+    public boolean contains(Object key_one, Object key_two) {
+	boolean result = false;
+	// Retrieve the hash mapping at key_one.
+	HashMap map = (HashMap) get(key_one);
+	// If there is such a map then retrieve the value at key_two.
+	if(map != null) {
+	    last_value = map.get(key_two);
+	    if(last_value != null) {
+		last_key_one = key_one;
+		last_key_two = key_two;
+		result = true;
+	    }
+	}
+	return result;
+    }
+
+    /** Retrieve an entry from this three dimensional hash mapping.
+     * @param key_one The first key as an <strong>Object</strong>.
+     * @param key_two The second key as an <strong>Object</strong>.
+     * @return The value <strong>Object</strong> located at (key_one, key_two), or <i>null</i> if no such value.
+     */
+    public Object get(Object key_one, Object key_two) {
+	Object result = null;
+	if(key_one.equals(last_key_one) && key_two.equals(last_key_two)) {
+	    result = last_value;
+	}
+	else {
+	    // Retrieve the hash mapping at key_one.
+	    HashMap map = (HashMap) get(key_one);
+	    // If there is such a map then retrieve the value at key_two.
+	    if(map != null) {
+		result = map.get(key_two);
+		if(result != null) {
+		    last_key_one = key_one;
+		    last_key_two = key_two;
+		    last_value = result;
+		}
+	    }
+	}
+	return result;
+    }
+
+    /** Put an entry into this three dimensional hash mapping.
+     * @param key_one The first key, to store this value under, as an <strong>Object</strong>.
+     * @param key_two The second key, to store this value under, as an <strong>Object</strong>.
+     * @param value The value <strong>Object</strong> itself.
+     */
+    public void put(Object key_one, Object key_two, Object value) {
+	// Retrieve the hash mapping at key_one, or if none exists create one.
+	HashMap map = (HashMap) get(key_one);
+	if(map == null) {
+	    map = new HashMap(capacity);
+	    put(key_one, map);
+	}
+	// Now add the value to this mapping.
+	map.put(key_two, value);
+	size++;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/JarTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/JarTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/JarTools.java	(revision 21112)
@@ -0,0 +1,132 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.util;
+
+
+import java.awt.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+
+
+public class JarTools
+{
+    static private Class root_class = null;
+    static private ImageIcon ERROR_ICON = null; 
+
+
+    static public void initialise(Object root_object)
+    {
+	root_class = root_object.getClass();
+	ERROR_ICON = getImage("error.gif");
+    }
+
+
+    static public void extractFromJar(String filename, String dst_dir, boolean must_be_present) 
+    {
+	try {
+	    // setup input stream for slurping out file
+	    InputStream fis = root_class.getResourceAsStream("/"+filename);
+	    BufferedInputStream fbis = new BufferedInputStream(fis); 
+	    DataInputStream fdbis = new DataInputStream(fbis);
+
+	    // set up output stream for writing to disk
+	    String ofname = dst_dir + filename;
+	    FileOutputStream fos = new FileOutputStream(ofname);
+	    BufferedOutputStream bfos = new BufferedOutputStream(fos);
+
+	    byte[] buf = new byte[1024];
+	    int len;
+	    int total_bytes = 0;
+	    while ((len = fdbis.read(buf)) >= 0) {
+		bfos.write(buf,0,len);
+		total_bytes += len;
+	    }
+
+	    fdbis.close();
+	    bfos.close();
+	}
+	catch (Exception exception) {
+	    if (must_be_present) {
+		exception.printStackTrace();
+	    }
+	}
+    }
+
+
+    /** Method to retrieve an image icon with the given filename found in classpath or the resouces directory.
+     *  @return The specified <strong>ImageIcon</strong>, or an error image replacement if no such images exists.
+     */
+    static public ImageIcon getImage(String filename) {
+	return getImage(filename, false);
+    }
+
+    static public ImageIcon getImage(String filename, boolean wait_until_complete)
+    {
+	ImageIcon image = new ImageIcon(root_class.getResource("/images/" + filename));
+	if (image == null) {
+	    image = ERROR_ICON;
+	}
+
+	if (wait_until_complete) {
+	    int load_status;
+	    do {
+		load_status = image.getImageLoadStatus();
+	    }
+	    while (load_status != MediaTracker.ABORTED && load_status != MediaTracker.ERRORED && load_status != MediaTracker.COMPLETE);
+	}
+
+	return image;
+    }
+
+
+    static public URL getResource(String resource_name)
+    {
+	return root_class.getResource(resource_name);
+    }
+
+
+    static public InputStream getResourceAsStream(String resource_name)
+    {
+	return root_class.getResourceAsStream(resource_name);
+    }
+
+
+    static public boolean isInJar(String filename) 
+    {
+	try {
+	    InputStream fis = root_class.getResourceAsStream("/" + filename);		    
+	    fis.close();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	    return false;
+	}
+
+	return true;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/PatternTokenizer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/PatternTokenizer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/PatternTokenizer.java	(revision 21112)
@@ -0,0 +1,72 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+public class PatternTokenizer {
+    private int index;
+    private String content;
+    private String pattern;
+    public PatternTokenizer(String content, String pattern) {
+	this.content = content;
+	this.index = 0;
+	this.pattern = pattern;
+    }
+    public boolean hasMoreTokens() {
+	return (index != -1 && content.length() - index > 0);
+    }
+    public String nextToken() {
+	// Special case when index is zero
+	if(index == 0) {
+	    index = content.indexOf(pattern);
+	    // No occurance of pattern, return whole string
+	    if(index == -1) {
+		return content;
+	    }
+	    // Return the fragment from 0 to index
+	    else {
+		return content.substring(0, index);
+	    }
+	}
+	// Otherwise
+	else {
+	    // We must first skip the pattern, as it should be at the head of our current view into content
+	    index = index + pattern.length();
+	    int end_index = content.indexOf(pattern, index);
+	    String fragment;
+	    // No occurance of pattern, return fragment from index to length
+	    if(end_index == -1) {
+		fragment = content.substring(index);
+	    }
+	    // Return the fragment from index to end index, then set index to end index
+	    else {
+		fragment = content.substring(index, end_index);
+	    }
+	    index = end_index;
+	    return fragment;
+	}
+    }    
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/StaticStrings.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/StaticStrings.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/StaticStrings.java	(revision 21112)
@@ -0,0 +1,245 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+
+/** Were you to guess that this is a class object choc-a-block full of static stringy goodness, you'd be right. They come in several flavours: Those ending _STR are strings you might find as values in XML, _ELEMENT are element names, _ATTRIBUTE are attribute names, _CHAR are particular characters while _CHARACTER are also characters but expressed as strings (for regex purposes ie startsWith, endsWith, indexOf and replaceAll); _PATTERN are strings which require extra escaping to put them through regex. Finally anything else is just a static string used within GLI. */
+public class StaticStrings {
+    static final public char   CLOSEBRACKET_CHAR                          = ']';
+    static final public char   DOUBLEQUOTE_CHAR                           = '\"';
+    static final public char   BACKSLASH_CHAR                             = '\\';
+    static final public char   FORWARDSLASH_CHAR                          = '/';
+    static final public char   NEW_LINE_CHAR                              = '\n';
+    static final public char   NS_SEP                                     = '.';
+    static final public char   OPENBRACKET_CHAR                           = '[';
+    static final public char   PIPE_CHAR                                  = '|';
+    static final public char   SINGLEQUOTE_CHAR                           = '\'';
+    static final public char   SPACE_CHAR                                 = ' ';
+    static final public char   STAR_CHAR                                  = '*';
+    static final public String ALLFIELDS_STR                              = "allfields";
+    static final public String ARCPLUG_STR                                = "ArcPlug";
+    static final public String ASSIGNED_ATTRIBUTE                         = "assigned";
+    static final public String ASSOCIATIONS_ELEMENT                       = "Associations";
+    static final public String AT_CHARACTER                               = "@";
+    static final public String ATTRIBUTE_ELEMENT                          = "Attribute";
+    static final public String BUILDTYPE_ELEMENT                              = "BuildType";
+    static final public String BUILDTYPE_STR                              = "buildtype";
+    static final public String CACHE_FOLDER                               = "cache";
+    static final public String CCS_STR                                    = "ccs";
+    static final public String CLASSES_FOLDER                             = "classes";
+    static final public String CLASSIFY_ELEMENT                           = "Classify";
+    static final public String CLASSIFY_STR                               = "classify";
+    static final public String CLOSE_PARENTHESIS_CHARACTER                = ")";
+    static final public String CODE_ATTRIBUTE                             = "code";
+    static final public String COLNAME_PATTERN                            = "<col_name>";
+    static final public String COLLECT_CFG                                = "collect.cfg";
+    static final public String COLLECTCONFIGURATION_XML                   = "CollectionConfig.xml";
+    static final public String COLLECTDIR_ARGUMENT                        = "-collectdir";
+    static final public String COLLECTION_ELEMENT                         = "Collection";
+    static final public String COLLECTION_STR                             = "Collection: ";
+    static final public String COLLECTIONMETADATA_COLLECTIONNAME_STR      = "collectionname";
+    static final public String COLLECTIONMETADATA_COLLECTIONEXTRA_STR     = "collectionextra";
+    static final public String COLLECTIONMETADATA_CREATOR_ELEMENT         = "Creator";
+    static final public String COLLECTIONMETADATA_CREATOR_STR             = "creator";
+    static final public String COLLECTIONMETADATA_ELEMENT                 = "CollectionMetadata";
+    static final public String COLLECTIONMETADATA_ICONCOLLECTION_STR      = "iconcollection";
+    static final public String COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR = "iconcollectionsmall";
+    static final public String COLLECTIONMETADATA_MAINTAINER_ELEMENT      = "Maintainer";
+    static final public String COLLECTIONMETADATA_MAINTAINER_STR          = "maintainer";
+    static final public String COLLECTIONMETADATA_PUBLIC_ELEMENT          = "Public";
+    static final public String COLLECTIONMETADATA_PUBLIC_STR              = "public";
+    static final public String COLLECTIONMETADATA_STR                     = "collectionmeta";
+    static final public String COLON_CHARACTER                            = ":";
+    static final public String COMMA_CHARACTER                            = ",";
+    static final public String COMMENT_VALUE                              = "comment";
+    static final public String COMMENTED_INDEXES_STR                      = "#indexes";
+    static final public String COMMENTED_INDEX_DEFAULT_STR                = "#defaultindex";
+    static final public String COMMENTED_LEVELS_STR                       = "#levels";
+    static final public String CONTENT_ATTRIBUTE                          = "content";
+    static final public String CONTENT_ELEMENT                            = "content";
+    static final public String CUSTOM_ATTRIBUTE                           = "custom";
+    static final public String DC_ATTRIBUTE                               = "dc";
+    static final public String DEBUG_ARGUMENT                             = "-debug";
+    static final public String DEFINITION_VALUE                           = "definition";
+    static final public String DESCRIPTION_ELEMENT                        = "Description";
+    static final public String DICTIONARY_FILENAME                        = "dictionary";
+    static final public String DIRECTORY_MAPPINGS_ELEMENT                 = "DirectoryMappings";
+    static final public String DOCUMENT_STR                               = "document";
+    static final public String ETC_FOLDER                                 = "etc";
+    static final public String EMPTY_STR                                  = "";
+    static final public String ENGLISH_LANGUAGE_STR                       = "en";
+    static final public String ENTRY_ELEMENT                              = "Entry";
+    static final public String ENUM_STR                                   = "enum";
+    static final public String EQUALS_CHARACTER                           = "=";
+    static final public String ESCAPE_PATTERN                             = "\\\\";
+    static final public String ESCAPE_STR                                 = "\\";
+    static final public String EXCLAMATION_CHARACTER                     = "!";
+    static final public String EXCLUDE_STR                                = "exclude";
+    static final public String EXTENSION_ATTRIBUTE                        = "extension";
+    static final public String EXTRACTED_METADATA_ELEMENT                 = "XMetadata";
+    static final public String EXTRACTED_NAMESPACE                        = "ex.";
+    static final public String FALSE_STR                                  = "false";
+    static final public String FEEDBACK_ARGUMENT                          = "-feedback";
+    static final public String FILE_ATTRIBUTE                             = "file";
+    static final public String FILENAME_STR                               = "Filename";
+    static final public String FLAG_STR                                   = "flag";
+    static final public String FORMAT_ELEMENT                             = "Format";
+    static final public String FORMAT_STR                                 = "format";
+    static final public String GLI_ATTRIBUTE                              = "gli";
+    static final public String GLISERVER_URL_ARGUMENT                     = "-gliserver_url";
+    static final public String GREATER_THAN_CHARACTER                     = ">";
+    static final public String GSDL_ARGUMENT                              = "-gsdl";
+    static final public String GSDL3_ARGUMENT                              = "-gsdl3";
+    static final public String GSDL3_SRC_ARGUMENT                         = "-gsdl3src";
+    static final public String HFILE_ARGUMENT                             = "-hfile";
+    static final public String HIDDENGLI_ELEMENT                          = "HiddenGLI";
+    static final public String HIERARCHY_ATTRIBUTE                        = "hierarchy";
+    static final public String HIERARCHY_CLASSIFIER                       = "Hierarchy";
+    static final public String HIERARCHY_STR                              = "hierarchy";
+    static final public String HTTP_PROTOCOL_STR                          = "http://";
+    static final public String HELP_ARGUMENT                              = "-help";
+    static final public String HELP_INDEX_FILENAME                        = "help_index.xml";
+    static final public String HTM_FILE_EXTENSION                         = ".htm";
+    static final public String IDENTIFIER_VALUE                           = "identifier";
+    static final public String IMAGES_PATH_RELATIVE_TO_GSDL_PREFIX        = "_httpprefix_/collect/<col_name>/images/";
+    static final public String IMPORT_FOLDER                              = "import";
+    static final public String INCLUDE_STR                                = "include";
+    static final public String INDEX_ELEMENT                              = "Index";
+    static final public String INDEX_DEFAULT_ELEMENT                      = "DefaultIndex";
+    static final public String INDEX_DEFAULT_STR                          = "defaultindex";
+    static final public String INDEX_STR                                  = "indexes";
+    static final public String INDEXES_ELEMENT                            = "Indexes";
+    static final public String INT_STR                                    = "int";
+    static final public String[] KEEP_PLUG                                = { "GAPlug", "METSPlug" };
+
+    static final public String LANGUAGE_ARGUMENT                          = "l=";
+    static final public String LANGUAGE_ATTRIBUTE                         = "language";
+    static final public String LANGUAGE_ELEMENT                           = "Language";
+    static final public String LANGUAGE_DEFAULT_ELEMENT                   = "DefaultLanguage";
+    static final public String LANGUAGE_DEFAULT_STR                       = "defaultlanguage";
+    static final public String LANGUAGE_STR                               = "language";
+    static final public String LANGUAGEDEPENDENT_ATTRIBUTE                = "language_dependent";
+    static final public String LANGUAGES_ELEMENT                          = "Languages";
+    static final public String LANGUAGES_STR                              = "languages";
+    static final public String LBRACKET_CHARACTER                         = "[";
+    static final public String LBRACKET_PATTERN                           = "\\[";
+    static final public String LESS_THAN_CHARACTER                        = "<";
+    static final public String LEVEL_ATTRIBUTE                            = "level";
+    static final public String LEVELS_ELEMENT                             = "Levels";
+    static final public String LEVELS_STR                                 = "levels";
+    static final public String LIBRARY_URL_ARGUMENT                       = "-library_url";
+    static final public String LOCAL_LIBRARY_ARGUMENT                     = "-local_library";
+    static final public String LOAD_ARGUMENT                              = "-load";
+    static final public String MAC_OPEN_COMMAND                           = "open %1";
+    static final public String MAPPING_ELEMENT                            = "Mapping";
+    static final public String MARC_EXTENSION                             = ".marc";
+    static final public String MDS_ATTRIBUTE                              = "mds";
+    static final public String METADATA_ARGUMENT                          = "-metadata";
+    static final public String METADATA_BAK                               = "~metadata.xml";
+    static final public String METADATA_ELEMENT                           = "Metadata";
+    static final public String METADATA_SET_EXTENSION                     = ".mds";
+    static final public String METADATA_STR                               = "metadata";
+    static final public String METADATA_TYPE_STR                          = "metadata";
+    static final public String METADATA_XML                               = "metadata.xml";
+    static final public String METADATUM_TYPE_STR                         = "metadatum";
+    static final public String MGPP_ATTRIBUTE                             = "mgpp_enabled";
+    static final public String MINUS_CHARACTER                            = "-";
+    static final public String MODE_STR                                   = "Mode: ";
+    static final public String MODEGLI_ELEMENT                            = "ModeGLI";
+    static final public String MODEL_COLLECTION_NAME                      = "modelcol";
+    static final public String NAME_ATTRIBUTE                             = "name";
+    static final public String NAME_ELEMENT                               = "Name";
+    static final public String NEWLINE_CHARACTER                          = "\\";
+    static final public String NEWLINE_ELEMENT                            = "NewLine";
+    static final public String NO_LOAD_ARGUMENT                           = "-no_load";
+    static final public String NO_WGET_STR                                = "NoWGet";
+    static final public String ONE_CHARACTER                              = "1";
+    static final public String OPEN_PARENTHESIS_CHARACTER                 = "(";
+    static final public String OPTION_ELEMENT                             = "Option";
+    static final public String OPTIONS_ATTRIBUTE                          = "options";
+    static final public String PARAGRAPH_STR                              = "paragraph";
+    static final public String PERL_ARGUMENT                              = "-perl";
+    static final public String PHIND_CLASSIFIER                           = "Phind";
+    static final public String PIPE_STR                                   = "|";
+    static final public String PLUGIN_ELEMENT                             = "Plugin";
+    static final public String PLUGIN_STR                                 = "plugin";
+    static final public String PROPERTIES_FILE_EXTENSION                  = ".properties";
+    static final public String RANGE_ELEMENT                              = "Range";
+    static final public String RBRACKET_CHARACTER                         = "]";
+    static final public String RBRACKET_PATTERN                           = "\\]";
+    static final public String RECPLUG_STR                                = "RecPlug";
+    static final public String REGEXP_STR                                 = "regexp";
+    static final public String SEARCHTYPE_ELEMENT                         = "SearchType";
+    static final public String SEARCHTYPE_STR                             = "searchtype";
+    static final public String SECTION_ELEMENT                            = "Section";
+    static final public String SECTION_STR                                = "section";
+    static final public String SEPARATOR_ATTRIBUTE                        = "separator";
+    static final public String SEPARATOR_CHARACTER                        = "/";
+    static final public String SERVLET_ARGUMENT                           ="-servlet";
+    static final public String SETLANGUAGE_ELEMENT                        = "SetLanguage";
+    static final public String SINGLE_ESCAPE_ONLY_PATTERN                 = "[^\\\\]+\\\\[^\\\\]+";
+    static final public String SINGLE_QUOTE_CHARACTER                     = "'";
+    static final public String SITE_ARGUMENT                              = "-site";
+    static final public String SOURCESEGMENT_VALUE                        = "SourceSegment";
+    static final public String SPACE_CHARACTER                            = " ";
+    static final public String SPECIAL_ATTRIBUTE                          = "special";
+    static final public String SPEECH_CHARACTER                           = "\"";
+    static final public String STOP_CHARACTER                             = ".";
+    static final public String STRING_STR                                 = "string";
+    static final public String SUBCOLLECTION_ELEMENT                      = "Subcollection";
+    static final public String SUBCOLLECTION_STR                          = "subcollection";
+    static final public String SUBCOLLECTION_DEFAULT_INDEX_ELEMENT        = "DefaultSubcollection";
+    static final public String SUBCOLLECTION_DEFAULT_INDEX_STR            = "defaultsubcollection";
+    static final public String SUBCOLLECTION_INDEX_STR                    = "indexsubcollections";
+    static final public String SUBCOLLECTION_INDEXES_ELEMENT              = "SubcollectionIndexes";
+    static final public String SUPERCOLLECTION_ELEMENT                    = "Supercollection";
+    static final public String SUPERCOLLECTION_STR                        = "supercollection";
+    static final public String TAB_CHARACTER                              = "\t";
+    static final public String TEXT_NODE                                  = "#text";
+    static final public String TEXT_STR                                   = "text";
+    static final public String TIMESTAMP_ARGUMENT                         = "&uq=";
+    static final public String TITLE_ELEMENT                              = "Title";
+    static final public String TRUE_STR                                   = "true";
+    static final public String TRUNCATED_STRING                           = "...";
+    static final public String TYPE_ATTRIBUTE                             = "type";
+    static final public String UNDERSCORE_CHARACTER                       = "_";
+    static final public String UNKNOWN_ELEMENT                            = "Unknown";
+    static final public String UNKNOWNPLUG_STR                             = "UnknownPlug";
+    static final public String USE_METADATA_FILES_ARGUMENT                = "use_metadata_files";
+    static final public String USE_REMOTE_GREENSTONE_ARGUMENT             = "-use_remote_greenstone";
+    static final public String URL_SEPARATOR_CHARACTER                    = "/";
+    static final public String VALUE_ATTRIBUTE                            = "value";
+    static final public String VALUE_TREE_ATTRIBUTE                       = "remember_values";
+    static final public String VERSION_ATTRIBUTE                          = "version";
+    static final public String WGET_ARGUMENT                              = "-wget";
+    static final public String WGET_OLD_STR                               = "<WGet1.9";
+    static final public String WGET_STR                                   = "WGet1.9";
+    static final public String WIN_9X_OPEN_COMMAND                        = "command.com /c start \"%1\"";
+    static final public String WIN_OPEN_COMMAND                           = "cmd.exe /c start \"\" \"%1\"";
+    static final public String YES_STR                                    = "yes";
+    static final public String ZERO_CHARACTER                             = "0";
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/SynchronizedTreeModelTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/SynchronizedTreeModelTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/SynchronizedTreeModelTools.java	(revision 21112)
@@ -0,0 +1,123 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.lang.Runnable;
+import javax.swing.SwingUtilities;
+import javax.swing.tree.*;
+import org.greenstone.gatherer.DebugStream;
+
+/** Due to the TreeModel objects not having any synchronization, certain assumptions, such as the model state remaining constant during a repaint, don't always hold - especially given that I'm changing the tree model on a different thread. In order to get around this I will use the latest swing paradigm wherein you flag a section of code to be executed by the AWT GUI Event queue, as soon as other gui tasks have finished. This way I shouldn't have tree redraws throwing NPEs because the array size of the children of a certain node has changed -while- the repaint call was made, i.e. repaint() calls getChildCount() = 13, removeNodeFromParent() called, repaint calls getChildAt(12) = ArrayIndexOutOfBoundsException. 
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3c
+ */
+public class SynchronizedTreeModelTools {
+    /** Adds an insertNodeInto model update onto the AWT Event queue. This gets around the lack of synchronization illustrated above. */
+    static final public Runnable insertNodeInto(DefaultTreeModel model, MutableTreeNode parent, MutableTreeNode target_node) {
+	return insertNodeInto(model, parent, target_node, true);
+    }
+
+    static final public Runnable insertNodeInto(final DefaultTreeModel model, final MutableTreeNode parent, final MutableTreeNode target_node, final boolean wait_allowed) {
+	final Runnable doInsertNodeInto = new Runnable() {
+		public void run() {
+		    ///ystem.err.print("Running task... ");
+		    DebugStream.println("insertNodeInto(" + model + ", " + parent + ", " + target_node + ", " + wait_allowed);
+		    int index = -1;
+		    int pos = 0;
+		    while(index == -1 && pos < parent.getChildCount()) {
+			TreeNode node = parent.getChildAt(pos);
+			int result = 0;
+			///ystem.err.println("Compare " + target_node + " to " + node);
+			if((target_node.isLeaf() && node.isLeaf()) || (!target_node.isLeaf() && !node.isLeaf())) {
+			    result = target_node.toString().toLowerCase().compareTo(node.toString().toLowerCase());
+			}
+			else if(target_node.isLeaf()) {
+			    result = -1;
+			}
+			else {
+			    result = 1;
+			}
+			if(result > 0) {
+			    ///ystem.err.println("Keep searching...");
+			    pos++;
+			}
+			else {
+			    ///ystem.err.println("Found!");
+			    index = pos;
+			}
+		    }
+		    if(index == -1) {
+			index = parent.getChildCount();
+		    }
+		    model.insertNodeInto(target_node, parent, index);
+		}
+	    };
+	///ystem.err.print("Queuing Task... ");
+	try {
+	    if(wait_allowed && !SwingUtilities.isEventDispatchThread()) {
+		///ystem.err.print("In another thread - invoke and wait... ");
+		SwingUtilities.invokeAndWait(doInsertNodeInto);
+	    }
+	    else {
+		///ystem.err.print("In Event Thread or wait not allowed - invoke later... ");
+		SwingUtilities.invokeLater(doInsertNodeInto);
+	    }
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	///ystem.err.print("Added Task... ");
+	return doInsertNodeInto;
+    }
+
+
+    /** Adds a removeNodeFromParent model update onto the AWT Event queue. This gets around the lack of synchronization illustrated above.
+     * @param model The <strong>GTreeModel</strong> we want to remove the node from.
+     * @param target_node The <strong>GTreeNode</strong> to remove.
+     */
+    static final public void removeNodeFromParent(final DefaultTreeModel model, final MutableTreeNode target_node) {
+	///ystem.err.println("Remove " + target_node + " from parent in model " + model);
+	final Runnable doRemoveNodeFromParent = new Runnable() {
+		public void run() {
+		    model.removeNodeFromParent(target_node);
+		}
+	    };
+	try {
+	    //SwingUtilities.invokeLater(doRemoveNodeFromParent);
+	    SwingUtilities.invokeAndWait(doRemoveNodeFromParent);
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+	// If we've thrown an error because we tried to invoke the runnable task and wait, when we are in the AWTEvent thread already, then try agin but with an invoke later.
+	catch (java.lang.Error error) {
+	    if(error.toString().equals("java.lang.Error: Cannot call invokeAndWait from the event dispatcher thread")) {
+		SwingUtilities.invokeLater(doRemoveNodeFromParent);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TableUtils.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TableUtils.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TableUtils.java	(revision 21112)
@@ -0,0 +1,92 @@
+package org.greenstone.gatherer.util;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.*;
+
+/**
+ * Utilites for manipulating JTables.
+ *
+ * @author Marc Hedlund
+ *   <a href="mailto:marc@precipice.org">&lt;marc@precipice.org&gt;</a>
+ * @version $Revision$
+ */
+
+public class TableUtils  {
+
+  // This is based on code found at
+  // <http://java.sun.com/docs/books/tutorial/ui/swingComponents/table.html>,
+  // somewhat modified.
+
+//    public static void setPreferredCellSizes(JTable table) {
+//      TableModel model = table.getModel();
+//      TableColumnModel colModel = table.getColumnModel();
+
+//      for (int i = 0; i < model.getColumnCount(); i++) {
+//        TableColumn column = colModel.getColumn(i);
+
+//        int longestCell = 0;
+//        int highestCell = 0;
+
+//        for (int j = 0; j < model.getRowCount(); j++) {
+//          Object value = model.getValueAt(j, i);
+//          if (value == null) continue;
+
+//          Component cell =
+//            table.getDefaultRenderer(model.getColumnClass(i)).
+//            getTableCellRendererComponent(table, value,
+//                                          false, false, j, i);
+
+//          int width = cell.getPreferredSize().width;
+//          int height = cell.getPreferredSize().height;
+
+//          if (width > longestCell) longestCell = width;         
+//          if (height > highestCell) highestCell = height;
+//        }
+
+//        Component headerComp = column.getHeaderRenderer().
+//          getTableCellRendererComponent(table, column.getHeaderValue(),
+//                                        false, false, 0, 0);
+
+//        int headerWidth = headerComp.getPreferredSize().width;
+//        int headerHeight = headerComp.getPreferredSize().height;
+
+//        column.setPreferredWidth(Math.max(headerWidth, longestCell));
+
+//        int currentHeight = table.getRowHeight();
+//        int preferredHeight = Math.max(headerHeight, highestCell);
+
+//        table.setRowHeight(Math.max(currentHeight, preferredHeight));
+//      }
+//    }                                                                   
+
+  /**
+   * Takes a column in an existing table and makes it fixed-width.
+   * Specifically, it sets the column's minimum and maximum widths to
+   * its preferred width, and disables auto-resize for the table as a
+   * whole.
+   *
+   * <p>
+   *
+   * Later on this should take a column array for efficiency.
+   *
+   * @param table JTable The table to modify
+   * @param colIndex int Which column to fix
+   * @return int The width of the column as it was fixed
+   */
+
+  public static int fixColumnToPreferredWidth(JTable table, int colIndex) {
+    table.setAutoResizeMode(JTable.AUTO_RESIZE_NEXT_COLUMN);
+
+    TableColumnModel tcm = table.getColumnModel();
+    TableColumn col = tcm.getColumn(colIndex);
+    int width = col.getPreferredWidth();
+
+    col.setMaxWidth(width);
+    col.setMinWidth(width);    
+
+    //table.setAutoResizeMode(JTable.AUTO_RESIZE_OFF);
+
+    return width;
+  }
+}  
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TreeSynchronizer.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TreeSynchronizer.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/TreeSynchronizer.java	(revision 21112)
@@ -0,0 +1,140 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.gatherer.util;
+
+import java.util.Vector;
+import javax.swing.JTree;
+import javax.swing.event.TreeExpansionEvent;
+import javax.swing.event.TreeExpansionListener;
+import javax.swing.event.TreeSelectionEvent;
+import javax.swing.event.TreeSelectionListener;
+import javax.swing.tree.TreeModel;
+import javax.swing.tree.TreePath;
+import org.greenstone.gatherer.gui.tree.DragTree;
+/** My latest diabolical class synchronizes the expansion state of two or more JTrees. Muh-hahahaha. Note that these tree should be based on the same model. If they aren't it won't work. So there.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.1
+ */
+final public class TreeSynchronizer 
+    extends Vector 
+    implements TreeExpansionListener, TreeSelectionListener {
+    /** <i>true</i> if we should temporarily ignore further events, most likely because we know our actions are causing them. */ 
+    private boolean ignore;
+    /** A list of tree selection listeners. */
+    private Vector selection_listeners = new Vector();
+    /** Add a new tree to the synchronization list of trees to be synchronized.
+     * @param tree The lastest victim, a <strong>JTree</strong>.
+     */
+    public void add(JTree tree) {
+	super.add(tree);
+	tree.addTreeExpansionListener(this);
+	tree.addTreeSelectionListener(this);
+    }
+
+    /** We allow the Gatherer to add tree listeners to this class, as it persists between collection changes transparently. Thus there is no need to reattach listeners everytime the collection changes. */
+    public void addTreeSelectionListener(TreeSelectionListener listener) {
+	if(!selection_listeners.contains(listener)) {
+	    selection_listeners.add(listener);
+	}
+    }
+
+    /** Called whenever an item in the tree has been collapsed.
+     * @param event A <strong>TreeExpansionEvent</strong> containing information about the event.
+     */
+    public void treeCollapsed(TreeExpansionEvent event) {
+	if(!ignore) {
+	    ignore = true;
+	    // Collapse that path in all registered trees.
+	    JTree tree = (JTree)event.getSource();
+	    TreePath path = event.getPath();
+	    for(int i = size(); i != 0; i--) {
+		JTree sibling = (JTree) get(i - 1);
+		if(!sibling.equals(tree)) {
+		    sibling.collapsePath(path);
+		}
+	    }
+	    ignore = false;
+	}
+    }
+    /** Called whenever an item in the tree has been expanded.
+     * @param event A <strong>TreeExpansionEvent</strong> containing information about the event.
+     */
+    public void treeExpanded(TreeExpansionEvent event) {
+	if(!ignore) {
+	    ignore = true;
+	    // Expand that path in all registered trees.
+	    JTree tree = (JTree)event.getSource();
+	    TreePath path = event.getPath();
+	    for(int i = size(); i != 0; i--) {
+		JTree sibling = (JTree) get(i - 1);
+		if(!sibling.equals(tree)) {
+		    sibling.expandPath(path);
+		}
+	    }
+	    ignore = false;
+	}
+    }
+    /** Called whenever the one of the trees selection changes.
+     * @param event A <strong>TreeSelectionEvent</strong> containing information about the event.
+     */
+    public void valueChanged(TreeSelectionEvent event) {
+	if(!ignore) {
+	    ignore = true;
+	    // Recover the tree that is the source.
+	    JTree tree = (JTree) event.getSource();
+	    // Brute Force approach.
+	    // Extract the currently selected paths.
+	    TreePath paths[] = tree.getSelectionPaths();
+	    // Then for every registered tree, that isn't this one, ensure those paths are selected.
+	    /*
+	    for(int i = size(); paths != null && i != 0; i--) {
+		JTree sibling = (JTree) get(i - 1);
+		if(!sibling.equals(tree)) {
+		    // One last thing to do. If this is actually a DragTree we are dealing with we have to tell it to set the selection values immediately, not wait for the clear until it is sure it is no the pre-cursor to a drag.
+		    if(sibling instanceof DragTree) {
+			DragTree gtree = (DragTree)sibling;
+			gtree.setImmediate(true);
+			gtree.setSelectionPaths(paths);
+			// I'm going to ensure that the last selected path is visible.
+			gtree.scrollPathToVisible(paths[paths.length - 1]);
+			gtree.setImmediate(false);
+		    }
+		    else {
+			sibling.setSelectionPaths(paths);
+			sibling.scrollPathToVisible(paths[paths.length - 1]);
+		    }
+		}
+	    }
+	    */
+	    // Pass on message to all listeners.
+	    for(int i = 0; i < selection_listeners.size(); i++) {
+		((TreeSelectionListener) selection_listeners.get(i)).valueChanged(event);
+	    }
+	    ignore = false;
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Unzip.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Unzip.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Unzip.java	(revision 21112)
@@ -0,0 +1,122 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: David Bainbridge, NZDL Project, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.util;
+
+import java.io.*;
+import java.util.*;
+import java.util.zip.*;
+
+
+public class Unzip 
+{
+
+    public static final void copyInputStream(InputStream in, OutputStream out)
+	throws IOException
+    {
+	byte[] buffer = new byte[1024];
+	int len;
+
+	while((len = in.read(buffer)) >= 0)
+	    out.write(buffer, 0, len);
+
+	in.close();
+	out.close();
+    }
+
+
+    static public void main(String[] args) 
+    {
+	if (args.length != 2) {
+	    System.err.println("Usage: Unzip <zip-file> <destination-directory>");
+	    return;
+	}
+
+	String zip_file_path = args[0];
+	// System.err.println("Zip file path: " + zip_file_path);
+	String destination_directory_path = args[1];
+	if (!destination_directory_path.endsWith(File.separator)) {
+	    destination_directory_path += File.separator;
+	}
+	// System.err.println("Destination directory path: " + destination_directory_path);
+
+	try {
+	    ZipFile zipFile = new ZipFile(zip_file_path);
+
+	    Enumeration entries = zipFile.entries();
+	    while (entries.hasMoreElements()) {
+		ZipEntry entry = (ZipEntry) entries.nextElement();
+
+		if (entry.isDirectory()) {
+		    // Assume directories are stored parents first then children.
+		    // System.err.println("Extracting directory: " + entry.getName());
+		    // This is not robust, just for demonstration purposes.
+		    String dir_name = entry.getName();
+		    String full_dir_name = destination_directory_path + dir_name;
+		    // System.err.println("Full directory name: " + full_dir_name);
+		    File dirFile = new File(full_dir_name);
+		    dirFile.mkdir();
+		}
+		else {
+		    // System.err.println("Extracting file: " + entry.getName());
+		    String file_name = entry.getName();
+		    String full_file_name = destination_directory_path + file_name;
+		    File directory = (new File(full_file_name)).getParentFile();
+		    // System.err.println("Full file name: " + full_file_name);
+		    if (!directory.exists() && !directory.mkdirs()) {
+			System.err.println("Error: Could not create directory " + directory + "!");
+		    }
+
+		    FileOutputStream fos = new FileOutputStream(full_file_name);
+		    BufferedOutputStream bos = new BufferedOutputStream(fos);
+
+		    copyInputStream(zipFile.getInputStream(entry),bos);
+		}
+	    }
+
+	    zipFile.close();
+	}
+	catch (ZipException error) {
+	    System.err.println("Error: Unable to open '" + zip_file_path + "'");
+	    System.err.println("This may be caused by the zip file being empty.");
+	    error.printStackTrace();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Utility.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Utility.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/Utility.java	(revision 21112)
@@ -0,0 +1,454 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.gatherer.util;
+
+
+// Don't even think about adding import java.awt.* here!
+// The functions in this class should not use any graphical classes. Put your function somewhere else buster!
+import java.io.*;
+import java.net.*;
+import java.util.*;
+// Don't even think about adding import javax.swing.* here!
+// The functions in this class should not use any graphical classes. Put your function somewhere else buster!
+import org.greenstone.gatherer.Dictionary;
+// Don't even think about adding import org.greenstone.gatherer.Gatherer in here!
+// The functions in this class should be independent of the Gatherer class. Put your function somewhere else buster!
+
+
+/** To provide a library of common methods, in a static context, for use in the Gatherer.
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3b
+ */
+public class Utility
+{
+    /** The size of the io buffer, in bytes. */
+    static final public int BUFFER_SIZE = 1024;
+    static final public String CFG_CLASSIFY = "classify";
+    static final public String CFG_CLASSIFY_BUTTONNAME = "-buttonname";
+    static final public String CFG_CLASSIFY_HFILE = "-hfile";
+    static final public String CFG_CLASSIFY_METADATA = "-metadata";
+    static final public String CFG_CLASSIFY_SORT = "-sort";
+    static final public String CFG_FORMAT = "format";
+    static final public String COLLECTION_TREE = "Collection";
+    /** Definition of an important directory name, in this case the file the collection configuration is expect to be in. */
+    static final public String CONFIG_FILE = "etc" + File.separator + "collect.cfg";
+    static final public String GLI_ARCHIVE = "GLI.jar";
+    static final public String GLI_EXTENSION = ".col";
+
+    /** Definition of an important directory name, in this case the import directory for the collection. */
+    static final public String IMPORT_DIR = "import" + File.separator;
+    /** Definition of an important directory name, in this case the backup import directory for the collection. */
+    static final public String IMPORT_BAK_DIR = "import.bak" + File.separator;
+    /** Definition of an important directory name, in this case the macros directory for the collection. */
+    static final public String MACROS_DIR = "macros" + File.separator;
+    /** Definition of an important directory name, in this case the location of the expected collection metadata sets.. */
+    static final public String META_DIR = "metadata" + File.separator; // Col. Copy
+    /** The default name of the perl executable under unix. */
+    static final public String PERL_EXECUTABLE_UNIX = "perl";
+    /** The default name of the perl executable under windows. */
+    static final public String PERL_EXECUTABLE_WINDOWS = "Perl.exe";
+    /** The name of the Gatherer. */
+    static final public String PROGRAM_NAME = "Greenstone Librarian Interface";
+    /** The current version of the Gatherer. */
+    static final public String PROGRAM_VERSION = "v2.62";
+    static final public String WORKSPACE_TREE = "Workspace";
+
+
+    /**
+     * Delete a file or directory
+     * @param file The <strong>File</strong> you want to delete.
+     * @return A <i>boolean</i> which is <i>true</i> if the file specified was successfully deleted, <i>false</i> otherwise.
+     */
+    static public boolean delete(File file)
+    {
+	// Nothing to do if it doesn't exist
+	if (!file.exists()) {
+	    return true;
+	}
+
+	return deleteInternal(file);
+    }
+
+
+    /** Convenience function. */
+    static public boolean delete(String filename)
+    {
+	return delete(new File(filename));
+    }
+
+
+    /** In Java you have to make sure a directory is empty before you delete it, so recursively delete. */
+    static private boolean deleteInternal(File file)
+    {
+	// If file is a directory, we have to recursively delete its contents first
+	if (file.isDirectory()) {
+	    File files[] = file.listFiles();
+	    for (int i = 0; i < files.length; i++) {
+		if (deleteInternal(files[i]) == false) {
+		    System.err.println("Error: Could not delete folder " + file);
+		    return false;
+		}
+	    }
+	}
+
+	// Delete file
+	if (file.delete() == false) {
+	    System.err.println("Error: Could not delete file " + file);
+	    return false;
+	}
+
+	return true;
+    }
+
+
+    /** Convert a long, detailing the length of a file in bytes, into a nice human readable string using b, kb, Mb and Gb. */
+    static final public String BYTE_SUFFIX = " b";
+    static final public long GIGABYTE = 1024000000l;
+    static final public String GIGABYTE_SUFFIX = " Gb";
+    static final public long KILOBYTE =       1024l;
+    static final public String KILOBYTE_SUFFIX = " kb";
+    static final public long MEGABYTE =    1024000l;
+    static final public String MEGABYTE_SUFFIX = " mb";
+    static final public String formatFileLength(long length) {
+	StringBuffer result = new StringBuffer("");
+	float number = 0f;
+	String suffix = null;
+	// Determine the floating point number and the suffix (radix) used.
+	if(length >= GIGABYTE) {
+	    number = (float) length / (float) GIGABYTE;
+	    suffix = GIGABYTE_SUFFIX;
+	}
+	else if(length >= MEGABYTE) {
+	    number = (float) length / (float) MEGABYTE;
+	    suffix = MEGABYTE_SUFFIX;
+	}
+	else if(length >= KILOBYTE) {
+	    number = (float) length / (float) KILOBYTE;
+	    suffix = KILOBYTE_SUFFIX;
+	}
+	else {
+	    // Don't need to do anything fancy if the file is smaller than a kilobyte
+	    return length + BYTE_SUFFIX;
+	}
+	// Create the formatted string remembering to round the number to 2.d.p. To do this copy everything in the number string from the start to the first occurance of '.' then copy two more digits. Finally search for and print anything that appears after (and including) the optional 'E' delimter.
+	String number_str = Float.toString(number);
+	char number_char[] = number_str.toCharArray();
+	int pos = 0;
+	// Print the characters up to the '.'
+	while(number_char != null && pos < number_char.length && number_char[pos] != '.') {
+	    result.append(number_char[pos]);
+	    pos++;
+	}
+	if(pos < number_char.length) {
+	    // Print the '.' and at most two characters after it
+	    result.append(number_char[pos]);
+	    pos++;
+	    for(int i = 0; i < 2 && pos < number_char.length; i++, pos++) {
+		result.append(number_char[pos]);
+	    }
+	    // Search through the remaining string for 'E'
+	    while(pos < number_char.length && number_char[pos] != 'E') {
+		pos++;
+	    }
+	    // If we still have string then we found an E. Copy the remaining string.
+	    while(pos < number_char.length) {
+		result.append(number_char[pos]);
+		pos++;
+	    }
+	}
+	// Add suffix
+	result.append(suffix);
+	// Done
+	return result.toString();
+    }
+
+    /** This method formats a given string, using HTML markup, so its width does not exceed the given width and its appearance if justified.
+     * @param text The <strong>String</strong> requiring formatting.
+     * @param width The maximum width per line as an <i>int</i>.
+     * @return A <strong>String</strong> formatted so as to have no line longer than the specified width.
+     * TODO Currently HTML formatting tags are simply removed from the text, as the effects of spreading HTML tags over a break are undetermined. To solve this we need to associate tags with a certain text token so if it gets broken on to the next line the tags go with it, or if the tags cover a sequence of words that are broken we need to close then reopen the tags. However all this is a major task and well beyond anything I have time to 'muck-round' on.
+     */
+    static public String formatHTMLWidth(String text, int width) {
+	if(text == null) {
+	    return "Error";
+	}
+	HTMLStringTokenizer html = new HTMLStringTokenizer(text);
+	int current_width = 0;
+	int threshold = width / 2;
+	Stack lines = new Stack();
+	String line = "";
+	while(html.hasMoreTokens()) {
+	    String token = html.nextToken();
+	    while(token != null) {
+		if(html.isTag()) {
+		    // Insert smart HTML tag code here.
+		    token = null;
+		}
+		else {
+		    // If the token is bigger than two thirds width, before we've even started break it down.
+		    if(current_width + 1 + token.length() > width && token.length() > threshold) {
+			if(width == current_width) {
+			    lines.push(line);
+			    line = token;
+			    current_width = token.length();
+			}
+			else {
+			    String prefix = token.substring(0, width - 1 - current_width);
+			    token = token.substring(prefix.length());
+			    if(current_width == 0) {
+				line = line + prefix;
+			    }
+			    else {
+				line = line + " " + prefix;
+			    }
+			    lines.push(line);
+			    line = "";
+			    current_width = 0;
+			}
+		    }
+		    // If adding the next token would push us over the maximum line width.
+		    else if(current_width + 1 + token.length() > width) {
+			lines.push(line);
+			line = token;
+			current_width = token.length();
+			token = null;
+		    }
+		    // Otherwise we should be able to just add the token, give or take.
+		    else {
+			if(current_width == 0) {
+			    line = line + token;
+			    current_width = token.length();
+			}
+			else {
+			    // Special case for standard punctuation which may exist after a tag like so:
+			    // My name is <scratchy>Slim Shady</scratchy>.   <-- Annoying punctuation.
+			    if(token.equals(".") || token.equals(",") || token.equals("!") || token.equals("?")) {
+				line = line + token;
+				current_width = current_width + 1;
+			    }
+			    else {
+				line = line + " " + token;
+				current_width = current_width + 1 + token.length();
+			    }
+			}
+			token = null;
+		    }
+		}
+	    }
+	}
+	String result = line;
+	while(!lines.empty()) {
+	    result = (String)lines.pop() + "<BR>" + result;
+	}
+	// Replace ' ' with "&nbsp;"
+	boolean tag = false;
+	int pos = 0;
+	while(pos < result.length()) {
+	    if(result.charAt(pos) == '<') {
+		tag = true;
+	    }
+	    else if(result.charAt(pos) == '>') {
+		tag = false;
+	    }
+	    else if(result.charAt(pos) == ' ' && !tag) {
+		String prefix = result.substring(0, pos);
+		String suffix = result.substring(pos + 1);
+		result = prefix + "&nbsp;" + suffix;
+	    }
+	    pos++;
+	}
+	result = "<HTML>" + result + "</HTML>";
+	return result;
+    }
+
+
+    static public String getDateString() {
+	Calendar current = Calendar.getInstance();
+	String day_name = null;
+	switch(current.get(Calendar.DAY_OF_WEEK)) {
+	case Calendar.MONDAY: day_name = "Dates.Mon"; break;
+	case Calendar.TUESDAY: day_name = "Dates.Tue"; break;
+	case Calendar.WEDNESDAY: day_name = "Dates.Wed"; break;
+	case Calendar.THURSDAY: day_name = "Dates.Thu"; break;
+	case Calendar.FRIDAY: day_name = "Dates.Fri"; break;
+	case Calendar.SATURDAY: day_name = "Dates.Sat"; break;
+	case Calendar.SUNDAY: day_name = "Dates.Sun"; break;
+	default: day_name = "";
+	}
+	String month_name = null;
+	switch(current.get(Calendar.MONTH)) {
+	case Calendar.JANUARY: month_name = "Dates.Jan"; break;
+	case Calendar.FEBRUARY: month_name = "Dates.Feb"; break;
+	case Calendar.MARCH: month_name = "Dates.Mar"; break;
+	case Calendar.APRIL: month_name = "Dates.Apr"; break;
+	case Calendar.MAY: month_name = "Dates.May"; break;
+	case Calendar.JUNE: month_name = "Dates.Jun"; break;
+	case Calendar.JULY: month_name = "Dates.Jul"; break;
+	case Calendar.AUGUST: month_name = "Dates.Aug"; break;
+	case Calendar.SEPTEMBER: month_name = "Dates.Sep"; break;
+	case Calendar.OCTOBER: month_name = "Dates.Oct"; break;
+	case Calendar.NOVEMBER: month_name = "Dates.Nov"; break;
+	case Calendar.DECEMBER: month_name = "Dates.Dec"; break;
+	default: month_name = "";
+	}
+	int day = current.get(Calendar.DAY_OF_MONTH);
+	int hour = current.get(Calendar.HOUR_OF_DAY);
+	int minute = current.get(Calendar.MINUTE);
+	int second = current.get(Calendar.SECOND);
+	int year = current.get(Calendar.YEAR);
+
+	return Dictionary.get(day_name) + " " + Dictionary.get(month_name) + " " + day + " " + year + " " + Utility.pad(String.valueOf(hour), 2, '0', true) + ":" + Utility.pad(String.valueOf(minute), 2, '0', true) + ":" + Utility.pad(String.valueOf(second), 2, '0', true);
+    }
+
+
+    /** Determine this machines name.
+     * @return The name as a <strong>String</strong>.
+     */
+    static public String getMachineName() {
+	try {
+	    return InetAddress.getLocalHost().getHostName();
+	}
+	catch(UnknownHostException ex) {
+	}
+	return "Unknown Machine";
+    }
+
+
+    static public String getSitesDir(String gsdl3_path) {
+	return gsdl3_path + "sites" + File.separator;
+
+    }
+
+
+    /** Method to determine if the host system is MacOS based.
+     * @return a boolean which is true if the platform is MacOS, false otherwise
+     */
+    public static boolean isMac() {
+	Properties props = System.getProperties();
+	String os_name = props.getProperty("os.name","");
+	if(os_name.startsWith("Mac OS")) {
+	    return true;
+	}
+	return false;
+    }
+
+   
+    /** Method to determine if the host system is Microsoft Windows based.
+     * @return A <i>boolean</i> which is <i>true</i> if the platform is Windows, <i>false</i> otherwise.
+     */
+    public static boolean isWindows() {
+	Properties props = System.getProperties();
+	String os_name = props.getProperty("os.name","");
+	if(os_name.startsWith("Windows")) {
+	    return true;
+	}
+	return false;
+    }
+
+    public static boolean isWindows9x() {
+	Properties props = System.getProperties();
+	String os_name = props.getProperty("os.name","");
+	if(os_name.startsWith("Windows") && os_name.indexOf("9") != -1) {
+	    return true;
+	}
+	return false;
+    }
+    /** Takes a string and a desired length and pads out the string to the length by adding spaces to the left.
+     * @param str The target <strong>String</strong> that needs to be padded.
+     * @param length The desired length of the string as an <i>int</i>.
+     * @return A <strong>String</strong> made from appending space characters with the string until it has a length equal to length.
+     */
+    static private String pad(String str_raw, int length, char fill, boolean end) {
+	StringBuffer str = new StringBuffer(str_raw);
+	while(str.length() < length) {
+	    if(end) {
+		str.insert(0, fill);
+	    }
+	    else {
+		str.append(fill);
+	    }
+	}
+	return str.toString();
+    }
+
+
+    static public StringBuffer readXMLStream(InputStream input_stream)
+    {
+	StringBuffer xml = new StringBuffer("");
+
+	try {
+	    InputStreamReader isr = new InputStreamReader(input_stream, "UTF-8");
+	    BufferedReader buffered_in = new BufferedReader(isr);
+	    
+	    String line = "";
+	    boolean xml_content = false;
+	    while((line = buffered_in.readLine()) != null) {
+		if(xml_content) {
+		    xml.append(line);
+		    xml.append("\n");
+		}
+		else if(line.trim().startsWith("<?xml")) {
+		    xml_content = true;
+		    xml.append(line);
+		    xml.append("\n");
+		}
+	    }
+	    buffered_in = null;
+	}
+	catch (Exception error) {
+	    System.err.println("Failed when trying to parse XML stream");
+	    error.printStackTrace();
+	}
+
+	return xml;
+    }
+
+
+    /** I think this works a bit better on Unicode strings. */
+    static public String stripNL(String raw_string)
+    {
+	String stripped_string = new String();
+	for (int i = 0; i < raw_string.length(); i++) {
+	    char raw_character = raw_string.charAt(i);
+	    if (raw_character != '\n' && raw_character != '\t') {
+		stripped_string = stripped_string + raw_character;
+	    }
+	}
+	return stripped_string;
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/XMLTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/XMLTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/XMLTools.java	(revision 21112)
@@ -0,0 +1,260 @@
+package org.greenstone.gatherer.util;
+
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import org.apache.xerces.parsers.*;
+import org.apache.xml.serialize.*;
+import org.greenstone.gatherer.DebugStream;
+import org.w3c.dom.*;
+import org.xml.sax.*;
+
+
+/** This class is a static class containing useful XML functions */
+public class XMLTools
+{
+    /** Remove all of the child nodes from a certain node. */
+    static final public void clear(Node node)
+    {
+	while (node.hasChildNodes()) {
+	    node.removeChild(node.getFirstChild());
+	}
+    }
+
+
+    static public ArrayList getChildElementsByTagName(Element parent_element, String element_name)
+    {
+	ArrayList child_elements = new ArrayList();
+
+	NodeList children_nodelist = parent_element.getChildNodes();
+	for (int i = 0; i < children_nodelist.getLength(); i++) {
+	    Node child_node = children_nodelist.item(i);
+	    if (child_node.getNodeType() == Node.ELEMENT_NODE && child_node.getNodeName().equals(element_name)) {
+		child_elements.add(child_node);
+	    }
+	}
+
+	return child_elements;
+    }
+
+
+    static public String getElementTextValue(Element element)
+    {
+	// Find the first text node child
+	NodeList children_nodelist = element.getChildNodes();
+	for (int i = 0; i < children_nodelist.getLength(); i++) {
+	    Node child_node = children_nodelist.item(i);
+	    if (child_node.getNodeType() == Node.TEXT_NODE) {
+		return child_node.getNodeValue();
+	    }
+	}
+
+	// None found
+	return "";
+    }
+
+
+    /** Method to retrieve the value of a given node.
+     * @param element The <strong>Element</strong> whose value we wish to find.
+     * Soon to be deprecated!
+     */
+    static final public String getValue(Node element) {
+	if (element == null) {
+	    return "";
+	}
+	// If we've been given a subject node first retrieve its value node.
+	if(element.getNodeName().equals("Subject")) {
+	    element = getNodeFromNamed(element, "Value");
+	}
+	// If we've got a value node, then reconstruct the text. Remember that DOM will split text over 256 characters into several text nodes
+	if(element != null && element.hasChildNodes()) {
+	    StringBuffer text_buffer = new StringBuffer();
+	    NodeList text_nodes = element.getChildNodes();
+	    for(int i = 0; i < text_nodes.getLength(); i++) {
+		Node possible_text = text_nodes.item(i);
+		if(possible_text.getNodeName().equals(StaticStrings.TEXT_NODE)) {
+		    text_buffer.append(possible_text.getNodeValue());
+		}
+	    }
+	    return text_buffer.toString();
+	}
+	return "";
+    }
+
+
+    /** Method to retrieve from the node given, a certain child node with the specified name.
+     * @param parent The <strong>Node</strong> whose children should be searched.
+     * @param name The required nodes name as a <strong>String</strong>.
+     * @return The requested <strong>Node</strong> if it is found, <i>null</i> otherwise.
+     * Soon to be deprecated!
+     */ 
+    static final public Node getNodeFromNamed(Node parent, String name) {
+	Node child = null;
+	for(Node i = parent.getFirstChild(); i != null && child == null; 
+	    i = i.getNextSibling()) {
+	    if(i.getNodeName().equals(name)) {
+		child = i;
+	    }
+	}
+	return child;
+    }
+
+
+    /** Parse an XML document from a given file path */
+    static public Document parseXMLFile(String xml_file_path, boolean use_class_loader)
+    {
+	if (use_class_loader == true) {
+	    InputStream is = JarTools.getResourceAsStream("/" + xml_file_path);
+	    if (is != null) {
+		return parseXML(is);
+	    }
+	}
+
+ 	// Try the file outside the classes directory
+ 	return parseXMLFile(new File(xml_file_path));
+    }
+
+
+    /** Parse an XML document from a given file */
+    static public Document parseXMLFile(File xml_file)
+    {
+	// No file? No point trying!
+	if (xml_file.exists() == false) {
+	    return null;
+	}
+
+	try {
+	    return parseXML(new FileInputStream(xml_file));
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	    return null;
+	}
+    }
+
+
+    /** Parse an XML document from a given input stream */
+    static public Document parseXML(InputStream xml_input_stream)
+    {
+	Document document = null;
+
+	try {
+	    InputStreamReader isr = new InputStreamReader(xml_input_stream, "UTF-8");
+	    Reader xml_reader = new BufferedReader(isr);
+	    document = parseXML(xml_reader);
+	    isr.close();
+	    xml_input_stream.close();
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+
+	return document;
+    }
+
+
+    /** Parse an XML document from a given reader */
+    static public Document parseXML(Reader xml_reader)
+    {
+	Document document = null;
+
+	try {
+	    InputSource isc       = new InputSource(xml_reader);
+	    DOMParser parser      = new DOMParser();
+	    parser.setFeature("http://xml.org/sax/features/validation", false);
+	    parser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+	    // May or may not be ignored, the documentation for Xerces is contradictory. If it works then parsing -should- be faster.
+	    parser.setFeature("http://apache.org/xml/features/dom/defer-node-expansion", true);
+	    parser.setFeature("http://apache.org/xml/features/dom/include-ignorable-whitespace", false);
+	    parser.parse(isc);
+	    document = parser.getDocument();
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+
+	return document;
+    }
+
+
+    /** Removes characters that are invalid in XML (see http://www.w3.org/TR/2000/REC-xml-20001006#charsets) */
+    static public String removeInvalidCharacters(String text)
+    {
+	char[] safe_characters = new char[text.length()];
+	int j = 0;
+
+	char[] raw_characters = new char[text.length()];
+	text.getChars(0, text.length(), raw_characters, 0);
+	for (int i = 0; i < raw_characters.length; i++) {
+	    char character = raw_characters[i];
+	    if ((character >= 0x20 && character <= 0xD7FF) || character == 0x09 || character == 0x0A || character == 0x0D || (character >= 0xE000 && character <= 0xFFFD) || (character >= 0x10000 && character <= 0x10FFFF)) {
+		safe_characters[j] = character;
+		j++;
+	    }
+	}
+
+	return new String(safe_characters, 0, j);
+    }
+
+
+    static public void setElementTextValue(Element element, String text)
+    {
+	// Remove all text node children
+	NodeList children_nodelist = element.getChildNodes();
+	for (int i = children_nodelist.getLength() - 1; i >= 0; i--) {
+	    Node child_node = children_nodelist.item(i);
+	    if (child_node.getNodeType() == Node.TEXT_NODE) {
+		element.removeChild(child_node);
+	    }
+	}
+
+	// Add a new text node
+	if (text != null) {
+	    element.appendChild(element.getOwnerDocument().createTextNode(text));
+	}
+    }
+
+
+    /** Set the #text node value of some element.
+     * @param element the Element whose value we wish to set
+     * @param value the new value for the element as a String
+     * Soon to be deprecated!
+     */
+    static final public void setValue(Element element, String value) {
+	// Remove any existing child node(s)
+	clear(element);
+	// Add new text node.
+	if (value != null) {
+	    element.appendChild(element.getOwnerDocument().createTextNode(value));
+	}
+    }
+
+
+    /** Write an XML document to a given file */
+    static public void writeXMLFile(File xml_file, Document document)
+    {
+	try {
+	    OutputStream os = new FileOutputStream(xml_file);
+	    // Create an output format for our document.
+	    OutputFormat f = new OutputFormat(document);
+	    f.setEncoding("UTF-8");
+	    f.setIndenting(true);
+	    f.setLineWidth(0); // Why isn't this working!
+	    f.setPreserveSpace(false);
+	    // Create the necessary writer stream for serialization.
+	    OutputStreamWriter osw = new OutputStreamWriter(os, "UTF-8");
+	    Writer w               = new BufferedWriter(osw);
+	    // Generate a new serializer from the above.
+	    XMLSerializer s        = new XMLSerializer(w, f);
+	    s.asDOMSerializer();
+	    // Finally serialize the document to file.
+	    s.serialize(document);
+	    // And close.
+	    os.close();
+	}
+	catch (Exception exception) {
+	    DebugStream.printStackTrace(exception);
+	}
+    }
+}
Index: /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ZipTools.java
===================================================================
--- /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ZipTools.java	(revision 21112)
+++ /main/tags/3.01/gli/src/org/greenstone/gatherer/util/ZipTools.java	(revision 21112)
@@ -0,0 +1,205 @@
+/**
+ *############################################################################
+ * A component of the Greenstone Librarian Interface, part of the Greenstone
+ * digital library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: David Bainbridge, NZDL Project, University of Waikato, NZ
+ *
+ * Copyright (C) 2005 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *############################################################################
+ */
+
+package org.greenstone.gatherer.util;
+
+
+import java.io.*;
+import java.util.*;
+import java.util.zip.*;
+import org.w3c.dom.*;
+
+
+public class ZipTools
+{
+    static public void unzipFile(String zip_file_path, String base_directory_path)
+    {
+	try {
+	    ZipFile zip_file = new ZipFile(new File(zip_file_path), ZipFile.OPEN_READ);
+	    
+	    Enumeration e = zip_file.entries(); 
+	    while (e.hasMoreElements()) {
+		ZipEntry zip_entry = (ZipEntry) e.nextElement();
+		File zip_entry_file = new File(base_directory_path + zip_entry.getName());
+		// System.err.println("    Unzipping: " + zip_entry_file.getAbsolutePath());
+
+		// Directory case
+		if (zip_entry.isDirectory()) {
+		    // Create named directory, if it doesn't already exist
+		    if (!zip_entry_file.exists() && !zip_entry_file.mkdirs()) {
+			System.err.println("Error: unable to create directory " + zip_entry_file);
+		    }
+		}
+
+		// File case
+		else {
+		    // Write out file to disk
+
+		    // Make sure its parent directory exists.
+		    File dir = new File(zip_entry_file.getParent());
+		    dir.mkdirs();
+
+		    // Set up input stream
+		    InputStream zis = zip_file.getInputStream(zip_entry); 
+		    BufferedInputStream bzis = new BufferedInputStream(zis); 
+		    DataInputStream dbzis = new DataInputStream(bzis);
+
+		    // Set up output stream
+		    FileOutputStream fzos = new FileOutputStream(zip_entry_file);
+		    BufferedOutputStream bfzos = new BufferedOutputStream(fzos);
+
+		    byte[] buf = new byte[1024];
+		    int len;
+		    while ((len = dbzis.read(buf)) >= 0) {
+			bfzos.write(buf,0,len);
+		    }
+
+		    dbzis.close();
+		    bzis.close();
+		    zis.close();
+
+		    bfzos.close();
+		    fzos.close();
+		}
+	    }
+
+	    zip_file.close();
+	} 
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    static public void zipFiles(String zip_file_path, String base_directory_path, String[] relative_file_paths)
+    {
+	try {
+	    ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zip_file_path));
+	    ZipFilter null_zip_filter = new NullZipFilter();
+
+	    // Add each file/directory in turn to the zip file
+	    for (int i = 0; i < relative_file_paths.length; i++) {
+		String relative_file_path = relative_file_paths[i];
+		addFileToZip(zos, base_directory_path, relative_file_path, null_zip_filter);
+	    }
+
+	    zos.close();
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    static public void addFileToZip(ZipOutputStream zos, String base_directory_path, String relative_file_path, ZipFilter zip_filter)
+    {
+	File file = new File(base_directory_path + File.separator + relative_file_path);
+
+	// Check that the file/directory exists
+	if (!file.exists()) {
+	    System.err.println("File " + file + " does not exist!");
+	    return;
+	}
+
+	try {
+	    // Directory case
+	    if (file.isDirectory()) {
+		// Add a zip entry for this directory
+		// Actually, don't -- this messes things up completely on Windows and doesn't seem to matter on Linux
+		// zos.putNextEntry(new ZipEntry(relative_file_path + File.separator));
+
+		// Apply recursively to each of the children of the directory
+		File[] child_files = file.listFiles();
+		for (int i = 0; i < child_files.length; i++) {
+		    addFileToZip(zos, base_directory_path, relative_file_path + File.separator + child_files[i].getName(), zip_filter);
+		}
+	    }
+
+	    // File case
+	    else {
+		// Add a zip entry for this file
+		if (zip_filter.shouldIncludeFile(relative_file_path)) {
+		    // Always use Unix style paths in zip files, even on Windows
+		    zos.putNextEntry(new ZipEntry(relative_file_path.replace('\\', '/')));
+
+		    if (zip_filter.shouldIncludeFileContent(relative_file_path)) {
+			BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
+			byte[] data = new byte[1024];
+			int bytes_read;
+			while ((bytes_read = zip_filter.filterFileContent(bis, data, 1024)) > -1) {
+			    zos.write(data, 0, bytes_read);
+			}
+			bis.close();
+		    }
+		}
+	    }
+	}
+	catch (Exception exception) {
+	    exception.printStackTrace();
+	}
+    }
+
+
+    public interface ZipFilter
+    {
+	public boolean shouldIncludeFile(String relative_file_path);
+
+	public boolean shouldIncludeFileContent(String relative_file_path);
+
+	public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read);
+    }
+
+
+    static public class NullZipFilter
+	implements ZipFilter
+    {
+	public boolean shouldIncludeFile(String relative_file_path)
+	{
+	    // All files are included
+	    return true;
+	}
+
+
+	public boolean shouldIncludeFileContent(String relative_file_path)
+	{
+	    // Content for all files is included
+	    return true;
+	}
+
+
+	public int filterFileContent(BufferedInputStream bis, byte[] data, int bytes_to_read)
+	{
+	    try {
+		return bis.read(data, 0, bytes_to_read);
+	    }
+	    catch (Exception exception) {
+		exception.printStackTrace();
+	    }
+
+	    return -1;
+	}
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/.cvsignore
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/.cvsignore	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/.cvsignore	(revision 21112)
@@ -0,0 +1,3 @@
+build
+dist
+test-classes
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/COPYING
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/COPYING	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/COPYING	(revision 21112)
@@ -0,0 +1,353 @@
+The files in this directory and all its subdirectories belong to the
+Greenstone Alerting Service. These files, with the exemption of the JAR files
+in the lib subdirectory, are Copyright (C) 2004, 2005 by Andrea Schweer
+(schweer@cs.waikato.ac.nz) and are licensed under the GNU General Public
+License Version 2 (see below). For the JAR files in the lib subdirectory, see
+the license information inside each JAR file or in the lib subdirectory.
+
+===============================================================================
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/build.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/build.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/build.xml	(revision 21112)
@@ -0,0 +1,517 @@
+<!--
+     General purpose build script for web applications and web services,
+     including enhanced support for deploying directly to a Tomcat 4
+     based server.
+
+     This build script assumes that the source code of your web application
+     is organized into the following subdirectories underneath the source
+     code directory from which you execute the build script:
+
+        docs                 Static documentation files to be copied to
+                             the "docs" subdirectory of your distribution.
+
+        src                  Java source code (and associated resource files)
+                             to be compiled to the "WEB-INF/classes"
+                             subdirectory of your web applicaiton.
+
+        web                  Static HTML, JSP, and other content (such as
+                             image files), including the WEB-INF subdirectory
+                             and its configuration file contents.
+
+     $Id$
+-->
+
+
+<!-- A "project" describes a set of targets that may be requested
+     when Ant is executed.  The "default" attribute defines the
+     target which is executed if no specific target is requested,
+     and the "basedir" attribute defines the current working directory
+     from which Ant executes the requested task.  This is normally
+     set to the current working directory.
+-->
+
+<project name="GSDL Alerting Service" default="compile" basedir=".">
+
+
+
+<!-- ===================== Property Definitions =========================== -->
+
+
+<!--
+
+  Each of the following properties are used in the build script.
+  Values for these properties are set by the first place they are
+  defined, from the following list:
+
+  * Definitions on the "ant" command line (ant -Dfoo=bar compile).
+
+  * Definitions from a "build.properties" file in the top level
+    source directory of this application.
+
+  * Definitions from a "build.properties" file in the developer's
+    home directory.
+
+  * Default definitions in this build.xml file.
+
+  You will note below that property values can be composed based on the
+  contents of previously defined properties.  This is a powerful technique
+  that helps you minimize the number of changes required when your development
+  environment is modified.  Note that property composition is allowed within
+  "build.properties" files as well as in the "build.xml" script.
+
+-->
+
+  <property file="build.properties"/>
+  <property file="${user.home}/build.properties"/>
+
+
+<!-- ==================== File and Directory Names ======================== -->
+
+
+<!--
+
+  These properties generally define file and directory names (or paths) that
+  affect where the build process stores its outputs.
+
+  app.name             Base name of this application, used to
+                       construct filenames and directories.
+                       Defaults to "myapp".
+
+  app.path             Context path to which this application should be
+                       deployed (defaults to "/" plus the value of the
+                       "app.name" property).
+
+  app.version          Version number of this iteration of the application.
+
+  build.home           The directory into which the "prepare" and
+                       "compile" targets will generate their output.
+                       Defaults to "build".
+
+  catalina.home        The directory in which you have installed
+                       a binary distribution of Tomcat 4.  This will
+                       be used by the "deploy" target.
+
+  dist.home            The name of the base directory in which
+                       distribution files are created.
+                       Defaults to "dist".
+
+  manager.password     The login password of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+  manager.url          The URL of the "/manager" web application on the
+                       Tomcat installation to which we will deploy web
+                       applications and web services.
+
+  manager.username     The login username of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+-->
+
+  <property name="app.name"      value="gsdl-as"/>
+  <property name="app.path"      value="/alerting"/>
+  <property name="app.version"   value="0.1-dev"/>
+  <property name="build.home"    value="${basedir}/build"/>
+  <property name="catalina.home" value="../../comms/jakarta/tomcat/"/>
+  <property name="dist.home"     value="${basedir}/dist"/>
+  <property name="docs.home"     value="${basedir}/docs"/>
+  <property name="docs.link"     value="http://java.sun.com/j2se/1.4.2/docs/api/"/>
+  <property name="manager.url"   value="http://localhost:8080/manager"/>
+  <property name="manager.username" value="admin"/>
+  <property name="manager.password" value="admin"/>
+  <property name="src.home"      value="${basedir}/src"/>
+  <property name="web.home"      value="${basedir}/web"/>
+
+
+<!-- ================== Custom Ant Task Definitions ======================= -->
+
+
+<!--
+
+  These properties define custom tasks for the Ant build tool that interact
+  with the "/manager" web application installed with Tomcat 4.  Before they
+  can be successfully utilized, you must perform the following steps:
+
+  - Copy the file "server/lib/catalina-ant.jar" from your Tomcat 4
+    installation into the "lib" directory of your Ant installation.
+
+  - Create a "build.properties" file in your application's top-level
+    source directory (or your user login home directory) that defines
+    appropriate values for the "manager.password", "manager.url", and
+    "manager.username" properties described above.
+
+  For more information about the Manager web application, and the functionality
+  of these tasks, see <http://localhost:8080/tomcat-docs/manager-howto.html>.
+
+-->
+
+  <taskdef name="install" classname="org.apache.catalina.ant.InstallTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="list"    classname="org.apache.catalina.ant.ListTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="reload"  classname="org.apache.catalina.ant.ReloadTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="remove"  classname="org.apache.catalina.ant.RemoveTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="false"/>
+  <property name="compile.optimize"    value="true"/>
+
+
+
+<!-- ==================== External Dependencies =========================== -->
+
+
+<!--
+
+  Use property values to define the locations of external JAR files on which
+  your application will depend.  In general, these values will be used for
+  two purposes:
+  * Inclusion on the classpath that is passed to the Javac compiler
+  * Being copied into the "/WEB-INF/lib" directory during execution
+    of the "deploy" target.
+
+  Because we will automatically include all of the Java classes that Tomcat 4
+  exposes to web applications, we will not need to explicitly list any of those
+  dependencies.  You only need to worry about external dependencies for JAR
+  files that you are going to include inside your "/WEB-INF/lib" directory.
+
+-->
+
+<!-- Dummy external dependency -->
+<!--
+  <property name="foo.jar"
+           value="/path/to/foo.jar"/>
+-->
+
+
+<!-- ==================== Compilation Classpath =========================== -->
+
+<!--
+
+  Rather than relying on the CLASSPATH environment variable, Ant includes
+  features that makes it easy to dynamically construct the classpath you
+  need for each compilation.  The example below constructs the compile
+  classpath to include the servlet.jar file, as well as the other components
+  that Tomcat makes available to web applications automatically, plus anything
+  that you explicitly added.
+
+-->
+
+  <path id="compile.classpath">
+
+    <!-- Include all JAR files that will be included in /WEB-INF/lib -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <fileset dir="lib">
+    	<include name="*.jar"/>
+    </fileset>
+    <!-- include greenstone files -->
+    <fileset dir="../../web/WEB-INF/lib/">
+      <include name="*.jar"/>
+    </fileset>
+    <!-- Include all elements that Tomcat exposes to applications -->
+    <pathelement location="${catalina.home}/common/classes"/>
+    <fileset dir="${catalina.home}/common/endorsed">
+      <include name="*.jar"/>
+    </fileset>
+    <fileset dir="${catalina.home}/common/lib">
+      <include name="*.jar"/>
+    </fileset>
+    <pathelement location="${catalina.home}/shared/classes"/>
+    <fileset dir="${catalina.home}/shared/lib">
+      <include name="*.jar"/>
+    </fileset>
+
+  </path>
+
+
+
+<!-- ==================== All Target ====================================== -->
+
+<!--
+
+  The "all" target is a shortcut for running the "clean" target followed
+  by the "compile" target, to force a complete recompile.
+
+-->
+
+  <target name="all" depends="clean,compile"
+   description="Clean build and dist directories, then compile"/>
+
+
+
+<!-- ==================== Clean Target ==================================== -->
+
+<!--
+
+  The "clean" target deletes any previous "build" and "dist" directory,
+  so that you can be ensured the application can be built from scratch.
+
+-->
+
+  <target name="clean"
+   description="Delete old build and dist directories">
+    <delete dir="${build.home}"/>
+    <delete dir="${dist.home}"/>
+  </target>
+
+
+
+<!-- ==================== Compile Target ================================== -->
+
+<!--
+
+  The "compile" target transforms source files (from your "src" directory)
+  into object files in the appropriate location in the build directory.
+  This example assumes that you will be including your classes in an
+  unpacked directory hierarchy under "/WEB-INF/classes".
+
+-->
+
+  <target name="compile" depends="prepare"
+   description="Compile Java sources">
+
+    <!-- Compile Java classes as necessary -->
+    <mkdir    dir="${build.home}/WEB-INF/classes"/>
+    <javac srcdir="${src.home}"
+          destdir="${build.home}/WEB-INF/classes"
+            debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+         optimize="${compile.optimize}">
+        <classpath refid="compile.classpath"/>
+    </javac>
+
+    <!-- Copy application resources -->
+    <copy  todir="${build.home}/WEB-INF/classes">
+      <fileset dir="${src.home}" excludes="**/*.java"/>
+    </copy>
+
+  </target>
+
+
+
+<!-- ==================== Dist Target ===================================== -->
+
+
+<!--
+
+  The "dist" target creates a binary distribution of your application
+  in a directory structure ready to be archived in a tar.gz or zip file.
+  Note that this target depends on two others:
+
+  * "compile" so that the entire web application (including external
+    dependencies) will have been assembled
+
+  * "javadoc" so that the application Javadocs will have been created
+
+-->
+
+  <target name="dist" depends="compile,javadoc"
+   description="Create binary distribution">
+
+    <!-- Copy documentation subdirectories -->
+    <mkdir   dir="${dist.home}/docs"/>
+    <copy    todir="${dist.home}/docs">
+      <fileset dir="${docs.home}"/>
+    </copy>
+
+    <!-- Create application JAR file -->
+    <jar jarfile="${dist.home}/${app.name}-${app.version}.war"
+         basedir="${build.home}"/>
+
+    <!-- Copy additional files to ${dist.home} as necessary -->
+
+  </target>
+
+
+
+<!-- ==================== Install Target ================================== -->
+
+<!--
+
+  The "install" target tells the specified Tomcat 4 installation to dynamically
+  install this web application and make it available for execution.  It does
+  *not* cause the existence of this web application to be remembered across
+  Tomcat restarts; if you restart the server, you will need to re-install all
+  this web application.
+
+  If you have already installed this application, and simply want Tomcat to
+  recognize that you have updated Java classes (or the web.xml file), use the
+  "reload" target instead.
+
+  NOTE:  This target will only succeed if it is run from the same server that
+  Tomcat is running on.
+
+  NOTE:  This is the logical opposite of the "remove" target.
+
+-->
+
+  <target name="install" depends="compile"
+   description="Install application to servlet container">
+
+    <install url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"
+            path="${app.path}"
+             war="file://${build.home}"/>
+
+  </target>
+
+
+<!-- ==================== Javadoc Target ================================== -->
+
+<!--
+
+  The "javadoc" target creates Javadoc API documentation for the Java
+  classes included in your application.  Normally, this is only required
+  when preparing a distribution release, but is available as a separate
+  target in case the developer wants to create Javadocs independently.
+
+-->
+
+  <target name="javadoc" depends="compile"
+   description="Create Javadoc API documentation">
+
+    <mkdir          dir="${dist.home}/docs/api"/>
+    <javadoc sourcepath="${src.home}"
+                destdir="${dist.home}/docs/api"
+    			   link="${docs.link}"
+    	        private="true"
+           packagenames="*">
+      <classpath refid="compile.classpath"/>
+    </javadoc>
+
+  </target>
+
+
+
+<!-- ====================== List Target =================================== -->
+
+<!--
+
+  The "list" target asks the specified Tomcat 4 installation to list the
+  currently running web applications, either loaded at startup time or
+  installed dynamically.  It is useful to determine whether or not the
+  application you are currently developing has been installed.
+
+-->
+
+  <target name="list"
+   description="List installed applications on servlet container">
+
+    <list    url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"/>
+
+  </target>
+
+
+<!-- ==================== Prepare Target ================================== -->
+
+<!--
+
+  The "prepare" target is used to create the "build" destination directory,
+  and copy the static contents of your web application to it.  If you need
+  to copy static files from external dependencies, you can customize the
+  contents of this task.
+
+  Normally, this task is executed indirectly when needed.
+
+-->
+
+  <target name="prepare">
+
+    <!-- Create build directories as needed -->
+    <mkdir  dir="${build.home}"/>
+    <mkdir  dir="${build.home}/WEB-INF"/>
+    <mkdir  dir="${build.home}/WEB-INF/classes"/>
+
+
+    <!-- Copy static content of this web application -->
+    <copy todir="${build.home}">
+      <fileset dir="${web.home}"/>
+    </copy>
+
+    <jar destfile="lib/templates.jar" basedir="lib/templates" includes="*.vm"/>
+
+    <!-- Copy external dependencies as required -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <mkdir  dir="${build.home}/WEB-INF/lib"/>
+    <copy todir="${build.home}/WEB-INF/lib">
+        <fileset dir="lib" includes="*.jar"/>
+    </copy>
+
+    <!-- Copy static files from external dependencies as needed -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+
+  </target>
+
+
+<!-- ==================== Reload Target =================================== -->
+
+<!--
+
+  The "reload" signals the specified application Tomcat 4 to shut itself down
+  and reload. This can be useful when the web application context is not
+  reloadable and you have updated classes or property files in the
+  /WEB-INF/classes directory or when you have added or updated jar files in the
+  /WEB-INF/lib directory.
+
+  NOTE: The /WEB-INF/web.xml web application configuration file is not reread
+  on a reload. If you have made changes to your web.xml file you must stop
+  then start the web application. 
+
+-->
+
+  <target name="reload" depends="compile"
+   description="Reload application on servlet container">
+
+    <reload url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+<!-- ==================== Remove Target =================================== -->
+
+<!--
+
+  The "remove" target tells the specified Tomcat 4 installation to dynamically
+  remove this web application from service.
+
+  NOTE:  This is the logical opposite of the "install" target.
+
+-->
+
+  <target name="remove"
+   description="Remove application on servlet container">
+
+    <remove url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+</project>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/create-tables.sql
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/create-tables.sql	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/create-tables.sql	(revision 21112)
@@ -0,0 +1,44 @@
+create table users (
+	username varchar(128) primary key, 
+	password text not null
+);
+
+create table subscriptions (
+	id int primary key auto_increment, 
+	name varchar(255), 
+	email text, 
+	rss bool not null default 0, 
+	page bool not null default 0, 
+	user varchar(128) not null references users.username, 
+	
+	unique(user,name)
+);
+
+create table predicates (
+	id int primary key auto_increment, 
+	field varchar(128) not null, 
+	value varchar(255), 
+	type enum("org.greenstone.gsdlas.profiles.EqualsPredicate", "org.greenstone.gsdlas.profiles.SubstringMatchPredicate", "org.greenstone.gsdlas.profiles.QueryPredicate") not null,
+	
+	unique(field,type,value)
+);
+
+create table subs_to_predicates (
+	subscription int not null references subscriptions.id, 
+	predicate int not null references predicates.id, 
+	
+	unique (subscription,predicate)
+);
+
+create table events (
+	id int primary key auto_increment,
+	timestamp datetime not null,
+	content text not null
+);
+
+create table events_to_subs (
+	event int not null references events.id,
+	subscription int not null references subscriptions.id,
+
+	unique (event,subscription)
+);
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/descriptor.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/descriptor.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/descriptor.xml	(revision 21112)
@@ -0,0 +1,7 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+</isd:service>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/.cvsignore
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/.cvsignore	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/.cvsignore	(revision 21112)
@@ -0,0 +1,1 @@
+templates.jar
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/error.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/error.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/error.vm	(revision 21112)
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Greenstone Digital Library Software</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>$title</p>
+			<p class="error">$message</p>
+			<p class="details">$details</p>
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/events.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/events.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/events.vm	(revision 21112)
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Events for subscription $subscription.name</p>
+<p class="description">$subscription</p>
+<p><a href="/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id">show as feed</a></p>
+#foreach ($event in $list)
+<p>$event</p>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/feed.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/feed.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/feed.vm	(revision 21112)
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--<?xml-stylesheet type="text/xml" href="/alerting/rss2html.xml"?>-->
+<rss version="2.0">
+   <channel>
+      <title>Events for Subscription $subscription.name</title>
+      <link>/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id</link>
+      <description>TODO</description>
+      <language>en</language>
+      <lastBuildDate>$lastBuildDate</lastBuildDate>
+      <generator>Alerting Service for Greenstone</generator>
+      <webMaster>$webmaster</webMaster>
+      
+      #foreach($event in $list)
+      <item>
+         <title>$event.document_title</title>
+         <link>$event.document_url</link>
+         <description>some description</description>
+         <pubDate>$event.timestamp</pubDate>
+         <guid>$event.document_url</guid>
+         <category>$event.type</category>
+      </item>
+      #end
+   </channel>
+</rss>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/general.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/general.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/general.vm	(revision 21112)
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+<p class="message">$message</p>
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/list.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/list.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/list.vm	(revision 21112)
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="list.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+#foreach($sub in $list)
+	<div class="subscription">
+		<p><span class="sub_id">Subscription $sub.Id</span><br/>
+		<a href="service?action=showEvents&subscriptionID=$sub.Id">show matching events</a> | <a href="service?action=editSubscription&subscriptionID=$sub.Id">edit subscription</a> | <a href="service?action=deleteSubscription&subscriptionID=$sub.Id">delete subscription</a></p>
+<!--		<table class="subscription">
+			<thead>
+				<tr><th>Field</th><th>Predicate</th></tr>
+			</thead>
+			<tfoot><tr><th/><th/></tr></tfoot>
+			<tbody>
+				#foreach($pred in $sub.Predicates)
+					<tr>
+						<td>$pred.Field</td>
+						<td>$pred.Value</td>
+					</tr>
+				#end
+			</tbody>
+		</table> // -->
+		<pre>
+		$sub
+		</pre>
+		<br/>
+	</div>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/login.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/login.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/login.vm	(revision 21112)
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Login</p>
+#if ($error) 
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=login" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="login"/>
+</form>
+<div class="description">Don't have an account yet? <a href="/alerting/service?action=showRegistrationForm">Register</a> now!</a></div>
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/register.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/register.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/register.vm	(revision 21112)
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Register</p>
+#if ($error)
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=register" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+<tr>
+<th><label for="password2">repeat password</label></th><td><input type="password" name="password2" id="password2"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="create account"/>
+</form>
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_collection.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_collection.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_collection.vm	(revision 21112)
@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (3/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="current">collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the collection(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which collection(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the collection(s) to monitor: 
+					<ol>
+					<li>You can choose one or more collections from the list below. Only events from the collections(s) you have chosen will be sent to you<br/>
+					<select multiple="multiple" name="hostCollectionID" size="7" style="margin:5px 0px;">
+						<option value="">any collection on any host</option>
+						#foreach($host in $hostnames)
+						<optgroup label="$host">
+							#foreach($collection in $collectionnames.get($host))
+							<option>$host|$collection</option>
+							#end
+							<option selected="selected" value="$host">any collection on host $host</option>
+						</optgroup>
+					#end
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a collection whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>collection name</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="collection_query" style="width:80%;" value="$!prefill.collection_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						<!-- collection description? creator? language? -->
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, events coming from <em>any collection</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the last step, you will be able to configure the the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="collection"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_host.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_host.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_host.vm	(revision 21112)
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (2/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td  class="current">host</td><td>collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the host(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which host(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the host(s) to monitor: 
+					<ol>
+					<li>You can choose one or more hosts from the list below. Only events from the host(s) you have chosen will be sent to you.<br/>
+					<select multiple="multiple" name="hostID" size="3" style="margin:5px 0px;">
+					#set($firstoption = true);
+					#foreach($host in $hostnames)
+					<option #if($firstoption) selected="selected" #set($firstoption = false) #end>$host</option>
+					#end
+					<option value="">any host</option>
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a host whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>host</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="host_query" style="width:80%;" value="$!prefill.host_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, only events coming from the <em>current host</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to choose the collection(s) to monitor, and you will be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="host"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_notification.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_notification.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_notification.vm	(revision 21112)
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (4/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="completed">collection</td><td class="current">notification</td>
+			</tr>
+			</table>
+			<p>Configure the way you will be notified of events </p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				<p class="explanation">You can <a href="preview.html" target="_new">preview</a> which documents currently match your criteria (following the link will open a new window).</p>
+				</div>
+				<div>
+					<h4>How to be notified of events matching your subscription</h4>
+					<div class="explanation">
+					You will be able to see the subscription in the list of all your subscriptions after logging in to the alerting service. <br/>
+					You can assign a name to your subscription that makes it easier for you to recognise it: 
+					<input name="subscription_name" type="text" value="$!prefill.subscription_name"/>
+					</div>
+					<br/>
+					<table>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="page" checked="checked" id="chck-page"/></td>
+							<td colspan="2">
+							<label for="chck-page">on my &quot;new since my last login&quot; page</label>
+							</td>
+						</tr>
+						<tr class="even">
+							<td><input name="way" type="checkbox" value="email" id="chck-email"/></td>
+							<td><label for="chck-email">via e-mail to</label>
+							</td>
+							<td>
+								<input type="text" name="email" value="$!prefill.email"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="rss" id="chck-rss"/></td>
+							<td colspan="2"><label for="chck-rss">with an <acronym title="Really Simple Syndication">RSS</acronym> feed</label> <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+						</tr>
+						</table>
+				</div>
+				<br/>
+				
+				<input type="hidden" name="current_page" value="notification"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next" disabled="disabled"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_type-details.vm
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_type-details.vm	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/lib/templates/sub_type-details.vm	(revision 21112)
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Create a subscription</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (1/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="current">type/details</td>
+			<td>host</td>
+			<td>collection</td>
+			<td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the event type and how to identify the document(s) to watch</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div>
+					<h4>Event type</h4>
+					<input name="type" type="checkbox" value="new_document"  checked="checked" id="chck-new-document"/> <label for="chck-new-document">new document(s)</label><br/>
+					<input name="type" type="checkbox" value="document_changed"  id="chck-doc-changed"/> <label for="chck-doc-changed">document(s) changed</label><br/>
+					<input name="type" type="checkbox" value="document_deleted"  id="chck-doc-deleted"/> <label for="chck-doc-deleted">document(s) deleted</label><br/>
+				</div>
+				<div>
+					<h4>How to identify the document(s)</h4>
+					<table>
+						<tr class="even">
+							<th>title</th>
+							<td width="25%">matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_title" style="width:80%;" value="$!prefill.document_title"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>content</th>
+							<td>matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_content" style="width:80%;" value="$!prefill.document_content"/>
+							</td>
+						</tr>
+						<tr class="even">
+							<th>metadata</th>
+							<td>has field <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists" title="metadata field exists"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_has_field" value="$!prefill.metadata_has_field" style="width:80%;"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>metadata</th>
+							<td>has value <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists and has value" title="metadata field exists and has value"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_field" value="$!prefill.metatada_field" style="width:40%;"/> is
+								<input type="text" name="metadata_value" value="$!prefill.metadata_value" style="width:40%;"/>
+							</td>
+						</tr>
+					</table>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to select the hosts and collections that you want to monitor for events. You will also be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="type-details"/>
+				<input disabled="disabled" type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input disabled="disabled" type="submit" name="next_page" value="finish"/>
+				<br/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 21112)
@@ -0,0 +1,547 @@
+/*
+ * Created on Nov 22, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.*;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.*;
+
+import org.apache.velocity.Template;
+import org.apache.velocity.context.Context;
+import org.apache.velocity.servlet.VelocityServlet;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.profiles.Predicate;
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.users.UserManagementException;
+import org.greenstone.gsdlas.users.UserManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingService extends VelocityServlet {
+    
+    /* (non-Javadoc)
+     * @see javax.servlet.Servlet#init(javax.servlet.ServletConfig)
+     */
+    public void init(ServletConfig config) throws ServletException {
+        super.init(config);
+        System.out.println("reloading subs");
+        ProfileStore.getInstance().restoreFromDatabase();
+        try {
+            Set subs = ProfileStore.getInstance().getAllSubscriptionsFor("andrea");
+            System.out.println("reloaded " + subs.size() + " subs for andrea: ");
+            for (Iterator iter = subs.iterator(); iter.hasNext();) {
+                System.out.println(iter.next());
+            }
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+    public static final String[] actions = new String[] {
+            "createSubscription",
+            "deleteSubscription",
+            "editSubscription",
+            "showEvents",
+            "showFeed",
+            "listSubscriptions",
+            "login",
+            "register",
+            "logout",
+            "showLoginForm",
+            "showRegistrationForm"
+    };
+    
+    
+    protected Template handleRequest(HttpServletRequest req,
+            HttpServletResponse res, Context context) {
+        
+        String action = req.getParameter(Constants.ACTION_PARAM);
+
+        Map args = req.getParameterMap();
+        
+        if (action != null && action.equals("receiveEvent")) {
+            receiveEvent(args);
+            return null;
+        }
+        
+        args = normalise(args);
+        
+        if (action == null || !ArrayHelper.contains(actions, action)) {
+            String title = "Unknown action";
+            String message = "I don't know how to " + action;
+            String details = "The only actions I know are " + ArrayHelper.toString(actions);
+            return showError(context, message, details);
+        }
+                
+        String templateString = "";
+        
+        try {
+            Method method = AlertingService.class.getDeclaredMethod(action, new Class[] {Map.class, Context.class});
+            templateString = (String) method.invoke(this, new Object[] {args, context});
+        } catch (Exception e) {
+            String message = "An error has occured, I couldn't do what you told me to do.";
+            String details = e.getMessage() + " (" + e.getClass().getName() + "); "
+            	+ e.getCause() 
+            	+ " ; action is " + action;
+            return showError(context, message, details);
+        }
+        
+        Template template = null;
+        try {
+            template = getTemplate(templateString);
+        } catch (Exception e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+            try {
+                error((HttpServletRequest)context.get(REQUEST), 
+                        (HttpServletResponse)context.get(RESPONSE),
+                        e);
+            } catch (Exception e2) {
+                // TODO Auto-generated catch block
+                e2.printStackTrace();
+            }
+        }
+        return template;
+    }
+    
+    /**
+     * @param arguments
+     * @param context
+     * @return the Velocity template to use
+     * @throws Exception
+     */
+    public String createSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "createSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            arguments.putAll(getPageArgsFromSession(session));
+            arguments.put("username", session.getAttribute("username"));
+            ProfileStore.getInstance().createSubscription(arguments);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, true);
+        }
+    }
+
+    public String deleteSubscription(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "deleteSubscription");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        try {
+            ProfileStore.getInstance().deleteSubscription(subscriptionID);
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String editSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "editSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            ProfileStore.getInstance().changeSubscription(arguments, session);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, false);
+        }
+    }
+    
+    public String showFeed(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showFeed");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        HttpServletResponse res = (HttpServletResponse) context.get(RESPONSE);
+        res.setContentType("text/xml");
+        return "feed.vm";
+    }
+    
+    public String showEvents(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showEvents");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        return "events.vm";
+    }
+    
+    public String showLoginForm(Map arguments, Context context) {
+        return "login.vm";
+    }
+    
+    public String showRegistrationForm(Map arguments, Context context) {
+        return "register.vm";
+    }
+
+    public String listSubscriptions(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "listSubscriptions");
+            return showLoginForm(arguments, context);
+        }
+        String username = (String) session.getAttribute("username");
+        context.put("title", "List of Subscriptions for " + username);
+        try {
+            Collection subscriptions = ProfileStore.getInstance().getAllSubscriptionsFor(username); 
+            context.put("list", subscriptions);
+            return "list.vm";
+        } catch (DatabaseException de) {
+            context.put("message", "couldn't get list of subscriptions for " + username);
+            context.put("details", de.getMessage());
+            return "error.vm";
+        }
+    }
+    
+    public String login(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().loginUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showLoginForm(arguments, context); 
+        }
+        if (session.getAttribute("next_action") != null) {
+            String nextAction = (String) session.getAttribute("next_action");
+            Method method = AlertingService.class.getDeclaredMethod(nextAction, new Class[] {Map.class, Context.class});
+            return (String) method.invoke(this, new Object[] {arguments, context});
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String register(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().createUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showRegistrationForm(arguments, context);
+        }
+        return login(arguments, context);
+    }
+    
+    public String logout(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        Enumeration atts = session.getAttributeNames();
+        while (atts.hasMoreElements()) {
+            session.removeAttribute((String) atts.nextElement());
+        }
+        session.invalidate();
+        return "general.vm";
+    }
+    
+    public void receiveEvent(Map rawEvent) {
+        Map event = new TreeMap();
+        for (Iterator iter = rawEvent.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (key.equals("action")) continue; // we don't want this
+            String[] value = (String[]) rawEvent.get(key);
+            event.put(key, value[0]);
+        }
+        System.out.println("receiving event " + event);
+        
+        GreenstoneCommunicator gsComm = null;
+        try {
+            String hostID = (String) event.get(Constants.HOST_ID_FIELD);
+            gsComm = new GreenstoneCommunicator(new URL(hostID));
+        } catch (Exception e) {
+            System.err.println("Can't communicate to Greenstone: " + e.getMessage());
+            e.printStackTrace();
+        }
+        
+        Set matchedSubscriptions = ProfileStore.getInstance().filter(event, gsComm);
+        
+        try {
+            EventStore.getInstance().add(event, matchedSubscriptions);
+            Notifier.getInstance().sendNotifications(event, matchedSubscriptions);
+        } catch (Exception e) {
+            System.err.println("Couldn't save events: " + e.getMessage());
+            e.printStackTrace();
+        }
+        System.out.println(matchedSubscriptions.size() + " matching subscriptions: " + matchedSubscriptions);
+    }
+
+    /**
+     * @param args
+     */
+    private Map normalise(Map args) {
+        Map result = new TreeMap();
+        for (Iterator iter = args.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (Predicate.isMultiValued(key) || key.equals("way")) {
+                // multi-valued attributes
+                String[] values = ((String[]) args.get(key));
+                result.put(key, Arrays.asList(values));
+            } else {
+                String firstValue = ((String[])args.get(key))[0];
+                result.put(key, firstValue);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * @param context
+     * @param message
+     * @param details
+     * @return
+     */
+    private Template showError(Context context, String message, String details) {
+        context.put("title", "Error");
+        context.put("message", message);
+        context.put("details", details);
+        try {
+            return getTemplate("error.vm");
+        } catch (Exception e) {
+            try {
+                super.error((HttpServletRequest)context.get("req"), 
+                        (HttpServletResponse)context.get("res"), 
+                        e);
+            } catch (Exception e2) {
+                e2.printStackTrace();
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @param arguments
+     * @param context
+     * @param create
+     * @return
+     * @throws Exception
+     */
+    private String showSubscriptionWizardPage(Map arguments, Context context, boolean create) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (create) {
+            context.put(Constants.ACTION_PARAM, "createSubscription");
+        } else {
+            // prefill from existing subscription
+            context.put(Constants.ACTION_PARAM, "editSubscription");
+            String subscriptionID = (String) arguments.get("subscriptionID");
+            int subID = Integer.parseInt(subscriptionID);
+            prefillFromSubscription(session, subID);
+        }
+        if (!arguments.containsKey("current_page")) {
+            return "sub_type-details.vm";
+        }
+        
+        String currentPage = (String) arguments.get("current_page");
+        String direction = (String) arguments.get("next_page");
+        
+        if (arguments.containsKey(Constants.HOST_QUERY_FIELD)) {
+            String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+            if (hostQuery != null && hostQuery.length() != 0)
+                arguments.remove(Constants.HOST_ID_FIELD);
+        }
+        if (arguments.containsKey(Constants.COLLECTION_QUERY_FIELD)) {
+            String collQuery = (String) arguments.get(Constants.COLLECTION_QUERY_FIELD);
+            if (collQuery != null && collQuery.length() != 0)
+                arguments.remove(Constants.COLLECTION_ID_FIELD);
+        }
+        
+        // save page arguments
+        savePageArgsToSession(currentPage, arguments, session);
+        
+        String nextPage = getNextPage(currentPage, direction);
+
+        // fill prefill
+        context.put("prefill", getPageArgsFromSession(nextPage, session));
+        
+        // fill preview
+        context.put("preview", getPagePreview(nextPage, session));
+        
+        // get page-specific stuff
+        if (nextPage.equals("host")) {
+            String[] hostNames;
+            try {
+                GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                hostNames = gsComm.getHostNames();
+            } catch (Exception e) {
+                hostNames = new String[] { "localhost" };
+            }
+            context.put("hostnames", hostNames);
+            session.setAttribute("hostnames", hostNames);
+        } else if (nextPage.equals("collection")) {
+            List hostNames = (List) arguments.get(Constants.HOST_ID_FIELD);
+            if (hostNames == null || hostNames.isEmpty()) {
+                hostNames = new Vector();
+                // no host names -> use host query
+                String[] hostsFromSession = (String[]) session.getAttribute("hostnames");
+                if (hostsFromSession == null || hostsFromSession.length == 0) {
+                    try {
+                        GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                        hostsFromSession = gsComm.getHostNames();
+                    } catch (Exception e) {
+                        hostsFromSession = new String[] { "localhost" };
+                    }
+                }
+                String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+                for (int i = 0; i < hostsFromSession.length; i++) {
+                    if (hostsFromSession[i] != null && hostsFromSession[i].indexOf(hostQuery) >= 0) {
+                        hostNames.add(hostsFromSession[i]);
+                    }
+                }
+            }
+            
+            Map collNames = new TreeMap();
+            for (Iterator iter = hostNames.iterator(); iter.hasNext();) {
+                String host = (String) iter.next();
+                Set collNamesForHost = new TreeSet();   
+                try {
+                    URL url = new URL("http://" + host + ":8080/soap/servlet/rpcrouter");
+                    GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+                    collNamesForHost.addAll(Arrays.asList(gsComm.getCollectionNames()));
+                } catch (Exception e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }	
+                collNames.put(host, collNamesForHost);
+            }
+            context.put("collectionnames", collNames);
+            context.put("hostnames", hostNames);
+        }
+        
+        return "sub_" + nextPage + ".vm";
+    }
+
+    /**
+     * @param session
+     * @param subID
+     */
+    private void prefillFromSubscription(HttpSession session, int subID) {
+        Subscription sub = ProfileStore.getInstance().getSubscription(subID);
+        Map typeArgs = new HashMap();
+        // TODO really fill stuff
+        savePageArgsToSession("type-details", typeArgs, session);
+        Map hostArgs = new HashMap();
+        savePageArgsToSession("host", hostArgs, session);
+        Map collArgs = new HashMap();
+        savePageArgsToSession("collection", collArgs, session);
+        Map notificationArgs = new HashMap();
+        savePageArgsToSession("notification", notificationArgs, session);
+    }
+
+    /**
+     * @param nextPage
+     * @param session
+     * @return
+     */
+    private Map getPagePreview(String nextPage, HttpSession session) {
+        Map preview = new TreeMap();
+        if (nextPage.equals("type-details")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("type-details", session));
+        if (nextPage.equals("host")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("host", session));
+        if (nextPage.equals("collection")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("collection", session));
+        return preview;
+    }
+
+    /**
+     * @param currentPage
+     * @param direction
+     * @return
+     * @throws Exception
+     */
+    private String getNextPage(String currentPage, String direction) throws Exception {
+        String nextPage;
+        if (currentPage.equals("host") && direction.equals("back")) {
+            nextPage = "type-details";
+        } else if (currentPage.equals("type-details") || (currentPage.equals("collection") && direction.equals("back"))) {
+            nextPage = "host";
+        } else if (currentPage.equals("host") || (currentPage.equals("notification") && direction.equals("back"))) {
+            nextPage = "collection";
+        } else if (currentPage.equals("collection")) {
+            nextPage = "notification";
+        } else {
+            throw new Exception("unknown combination of currentPage=" + currentPage + " and nextPage=" + direction);
+        }
+        return nextPage;
+    }
+
+    /**
+     * @param page
+     * @param session
+     * @return 
+     */
+    private Map getPageArgsFromSession(String page, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null || !pageArgs.containsKey(page))
+            return new TreeMap();
+        return (Map) pageArgs.get(page);
+    }
+
+    private Map getPageArgsFromSession(HttpSession session) {
+        Map result = new TreeMap();
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs != null) {
+            for (Iterator iter = pageArgs.values().iterator(); iter.hasNext();) {
+                Map args = (Map) iter.next();
+                result.putAll(args);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @param page
+     * @param arguments
+     * @param session
+     */
+    private void savePageArgsToSession(String page, Map arguments, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null) {
+            pageArgs = new TreeMap();
+        }
+        pageArgs.put(page, arguments);
+        session.setAttribute("page_args", pageArgs);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 21112)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 21, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Constants {
+    public static final String ACTION_PARAM = "action";
+    
+    public static final String HOST_QUERY_FIELD = "host_query";
+    public static final String COLLECTION_QUERY_FIELD = "collection_query";
+    public static final String HOST_ID_FIELD = "hostID";
+    public static final String COLLECTION_ID_FIELD = "collectionID";
+    public static final String HOST_COLLECTION_ID_FIELD = "hostCollectionID";
+    public static final String DOCUMENT_ID_FIELD = "documentID";
+    public static final String DOCUMENT_CONTENT_FIELD = "document_content";
+    public static final String DOCUMENT_TITLE_FIELD = "document_title";
+    public static final String TYPE_FIELD = "type";
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 21112)
@@ -0,0 +1,169 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Date;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class EventStore {
+
+    private static EventStore instance;
+    
+    private EventStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static EventStore getInstance() {
+        if (instance == null) instance = new EventStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) throws DatabaseException, SQLException, ParseException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+
+        int eventID = saveEventToDatabase(event);
+        
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            int subID = sub.getId();
+            String sqlString = "INSERT INTO events_to_subs (event,subscription) " +
+            		"VALUES (" + eventID + "," + subID + ");";
+            statement.executeUpdate(sqlString);
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     * @throws ParseException
+     */
+    private int saveEventToDatabase(Map event) throws DatabaseException, SQLException, ParseException {
+        String eventString = mapAsString(event);
+        String timestamp = (String) event.get("timestamp");
+        timestamp = convertToSQLDatetime(timestamp);
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT id FROM events WHERE timestamp = " + timestamp + 
+        	" AND content = '" + eventString + "';";
+        ResultSet results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        
+        String insert = "INSERT INTO events (timestamp,content) " +
+        		"VALUES (" + timestamp + ",'" + eventString + "');"; 
+        statement.executeUpdate(insert);
+        results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        throw new DatabaseException("couldn't save event to database");
+    }
+
+    /**
+     * converts a timestamp from an event to a format recognised by MySQL
+     * @param timestamp
+     * @return
+     * @throws ParseException
+     */
+    private String convertToSQLDatetime(String timestamp) throws ParseException {
+        Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z").parse(timestamp);
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+
+    /**
+     * @param event
+     */
+    private String encodeSpecialChars(String string) {
+        String encodedString = string.replaceAll("&", "&amp;");
+        encodedString = encodedString.replaceAll("<", "&lt;");
+        encodedString = encodedString.replaceAll(">", "&gt;");
+        encodedString = encodedString.replaceAll("\'", "&apos;");
+        encodedString = encodedString.replaceAll("\"", "&quot;");
+        return encodedString;
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public Set getEvents(Integer subscriptionID) throws DatabaseException, SQLException {
+        Set result = new HashSet(); // can't use TreeSet, because we put Maps in there, and Map isn't instanceof Comparable
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT content " +
+        		"FROM events e JOIN events_to_subs ets ON (e.id = ets.event) " +
+        		"WHERE ets.subscription = " + subscriptionID.intValue() + ";";
+        ResultSet results = statement.executeQuery(query);
+        while (results.next()) {
+            Map event = stringAsMap(results.getString("content"));
+            result.add(event);
+        }
+        return result;
+    }
+
+    private String mapAsString(Map map) {
+        StringBuffer buffer = new StringBuffer();
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) map.get(key);
+            String encodedValue = encodeSpecialChars(value);
+            buffer.append(key);
+            buffer.append(">"); // > has been encoded to &gt; in the original String
+            buffer.append(encodedValue);
+            if (iter.hasNext()) {
+                buffer.append(">>");
+            }
+        }
+        return buffer.toString();
+    }
+    
+    private Map stringAsMap(String string) {
+        Map map = new HashMap();
+        String[] lines = string.split(">>");
+        for (int i = 0; i < lines.length; i++) {
+            String line = lines[i];
+            String[] columns = line.split(">");
+            String key = columns[0];
+            String value = null;
+            if (columns.length > 1) {
+                value = columns[1];
+            }
+            map.put(key, value);
+        }
+        return map;
+    }
+    
+    public void cleanUp() {
+        // TODO implement method 
+        // delete all events that are too old (?)
+        // delete all events that don't match any subscriptions
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 21112)
@@ -0,0 +1,215 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.*;
+
+import org.apache.soap.Constants;
+import org.apache.soap.Fault;
+import org.apache.soap.rpc.*;
+import org.w3c.dom.*;
+
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicator {
+
+    private URL host;
+    private DocumentBuilder builder;
+
+    public GreenstoneCommunicator(URL host) throws Exception {
+        this.host = host;
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        try {
+            builder = factory.newDocumentBuilder();
+        } catch (ParserConfigurationException e) {
+            throw new Exception(e);
+        }
+
+    }
+
+    /**
+     * 
+     */
+    public GreenstoneCommunicator() {
+        
+        // TODO Auto-generated constructor stub
+    }
+
+    public String[] getHostNames() {
+        // TODO method implementation
+        return new String[] {"localhost"};
+    }
+    
+    public String[] getCollectionNames() throws Exception {
+        Document document = createSubsetDescriptionRequest("", "collectionList");
+
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("collection");
+        String[] result = new String[nodeList.getLength()];
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            Node name = node.getAttributes().getNamedItem("name");
+            result[i] = name.getNodeValue();
+        }
+        
+	System.out.println(ArrayHelper.toString(result));
+        
+        return result;
+    }
+    
+    public boolean supportsTextSearch(String collection) throws Exception {
+        Document document = createSubsetDescriptionRequest(collection, "serviceList");
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("service");
+        
+        boolean hasTextQuery = false;
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue(); 
+            if (name.equals("TextQuery")) {
+                hasTextQuery = true;
+                break;
+            }
+        }
+        
+        if (!hasTextQuery) return false; // doesn't support text queries at all
+        
+        document = createSubsetDescriptionRequest(collection + "/TextQuery", "paramList");
+        
+        response = sendToGreenstone(document.getDocumentElement());
+        
+        nodeList = response.getElementsByTagName("param");
+        
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue();
+            if (name == null) break;
+            String value = node.getAttributes().getNamedItem("type").getNodeValue();
+            if (value == null) break;
+            if (name.equals("query") && value.equals("string")) {
+                return true;
+            }
+        }
+        
+        return false;
+    }
+    
+    public Set fullTextSearch(String collection, String query) throws Exception {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "process");
+        requestElement.setAttribute("to", collection + "/TextQuery");
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "query");
+        paramElement.setAttribute("value", query);
+        // BTS 3 what if numReturnedDocs < numMatchedDocs
+        paramListElement.appendChild(paramElement);
+        
+        
+        Set result = new TreeSet();
+        //DOM2Writer.serializeAsXML(root, new OutputStreamWriter(System.out));
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList documentNodes = response.getElementsByTagName("documentNode");
+        
+        for(int i = 0; i < documentNodes.getLength(); i++) {
+            Node node = documentNodes.item(i);
+            String docID = node.getAttributes().getNamedItem("nodeID").getNodeValue();
+            Matcher matcher = Pattern.compile("(\\d{8}:\\d+).*").matcher(docID);
+            if (matcher.matches()) {
+                docID = matcher.group(1);
+            }
+            result.add(docID);
+        }
+        
+        return result;
+    }
+    
+    private Element sendToGreenstone(Element message) throws Exception {
+        Call call = new Call();
+        // TODO get the uri from a configuration file
+        call.setTargetObjectURI("localsite");
+        call.setMethodName("process");
+        // set Encoding Style to use literal XML
+        call.setEncodingStyleURI(Constants.NS_URI_LITERAL_XML);
+        // Set Method Parameters - use literal xml
+        Parameter param = new Parameter("message", 
+                org.w3c.dom.Element.class,
+                message, 
+                Constants.NS_URI_LITERAL_XML);
+
+        Vector paramList = new Vector();
+        paramList.addElement(param);
+        call.setParams(paramList);
+
+        // Invoke the Service
+        Response response = call.invoke(host, "");
+
+        // Check for Faults
+        if (!response.generatedFault()) { // no error
+            // Extract Return value
+            Parameter result = response.getReturnValue();
+            return (Element) result.getValue();
+        } else {
+            //  Extract Fault Code and String
+            Fault f = response.getFault();
+            String faultCode = f.getFaultCode();
+            String faultString = f.getFaultString();
+            throw new Exception(faultCode + ": Fault occured: " + faultString);
+        }
+    }
+
+    /**
+     * @return
+     */
+    private Document createSubsetDescriptionRequest(String object, String subsetType) {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "describe");
+        requestElement.setAttribute("to", object);
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "subset");
+        paramElement.setAttribute("value", subsetType);
+        paramListElement.appendChild(paramElement);
+        return document;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 21112)
@@ -0,0 +1,77 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class NotificationStore {
+
+    private static NotificationStore instance;
+    private Map eventsForSubscriptionID = new TreeMap();
+    
+    private NotificationStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static NotificationStore getInstance() {
+        if (instance == null) instance = new NotificationStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) {
+        encodeValues(event);
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            Integer subscriptionID = new Integer(sub.getId()); 
+            if (!eventsForSubscriptionID.containsKey(subscriptionID)) {
+                eventsForSubscriptionID.put(subscriptionID, new HashSet());
+            }
+            ((Set) eventsForSubscriptionID.get(subscriptionID)).add(event);
+        }
+    }
+
+    /**
+     * @param event
+     */
+    private void encodeValues(Map event) {
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            String encodedValue = value.replaceAll("&", "&amp;");
+            encodedValue = encodedValue.replaceAll("<", "&lt;");
+            encodedValue = encodedValue.replaceAll(">", "&gt;");
+            encodedValue = encodedValue.replaceAll("\'", "&apos;");
+            encodedValue = encodedValue.replaceAll("\"", "&quot;");
+            
+            event.put(key, encodedValue);
+        }
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     */
+    public Set getEvents(Integer subscriptionID) {
+        if (eventsForSubscriptionID.containsKey(subscriptionID)) {
+            return Collections.unmodifiableSet((Set) eventsForSubscriptionID.get(subscriptionID));
+        }
+        return new TreeSet();
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 21112)
@@ -0,0 +1,96 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+import java.util.Map;
+import java.util.Set;
+
+import javax.mail.MessagingException;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.util.Mailer;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ * 
+ * Singleton
+ */
+public class Notifier {
+
+    private static Notifier instance; 
+    
+    private Notifier () {
+        // hide default constructor
+    }
+    
+    /**
+     * @return
+     */
+    public static Notifier getInstance() {
+        if (instance == null) instance = new Notifier();
+        return instance;
+    }
+
+    /**
+     * @param event
+     * @param matchedSubscriptions
+     */
+    public void sendNotifications(Map event, Set matchedSubscriptions) {
+        // TODO read smtp host and sender from configuration file
+        Mailer mailer = new Mailer(System.getProperty("user.name") + "@localhost", "localhost");
+        String eventText = eventToEmail(event);
+        String subject = "Event for your subscription";
+        
+        for (Iterator iter = matchedSubscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            if (!sub.wantsEMailNotification()) continue;
+
+            StringBuffer message = new StringBuffer();
+            // TODO formatting of notification email via velocity?
+            message.append("You've created a subscription at the Greenstone Alerting Service running on " +
+            		"localhost:\n\n");
+            message.append(sub.toString());
+            message.append("\n");
+            message.append("An event has occurred that matches this subscription:\n\n");
+            message.append(eventText);
+            message.append("\n");
+            message.append("Go to http://localhost:8080/alerting/service?action=listSubscriptions to " +
+            		"change your subscriptions, or to http://localhost:8080/alerting/service?action=showEvents&subscriptionID=" + 
+            		sub.getId() + " to see all events that match the subscription.\n");
+            
+            String to = sub.getMailAddress();
+            
+            try {
+                mailer.sendMail(to, subject, message.toString());
+            } catch (MessagingException e) {
+                System.err.println("Couldn't notify for subscription " + sub.getId() + ": " + e.getMessage());
+                e.printStackTrace();
+            }
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     */
+    private String eventToEmail(Map event) {
+        StringBuffer message = new StringBuffer();
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            message.append(key + ": " + value + "\n");
+        }
+        return message.toString();
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 21112)
@@ -0,0 +1,416 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.*;
+
+/**
+ * Storage for all Profiles. Singleton.
+ * 
+ * @author schweer
+ *  
+ */
+public class ProfileStore {
+
+    private static ProfileStore instance;
+
+    private Set noEqualsSubscriptions = new TreeSet();
+    private Map dontCareSubscriptions = new TreeMap();
+    private Map subscriptions = new TreeMap();
+    
+
+    private ProfileStore() {
+        // hide default constructor
+    }
+
+    /**
+     * @return the one existing instance of this class.
+     */
+    public static ProfileStore getInstance() {
+        if (instance == null) {
+            instance = new ProfileStore();
+        }
+        return instance;
+    }
+
+    /**
+     * @param subscription
+     *            the subscription to add to the profile store.
+     */
+    private void addSubscription(Subscription subscription) {
+        List documentIdPredicate = subscription.getPredicateList(Constants.DOCUMENT_ID_FIELD);
+        List collectionIdPredicate = subscription.getPredicateList(Constants.COLLECTION_ID_FIELD);
+        List hostIdPredicate = subscription.getPredicateList(Constants.HOST_ID_FIELD);
+        List typePredicate = subscription.getPredicateList(Constants.TYPE_FIELD);
+        
+        Integer subId = new Integer(subscription.getId());
+        
+        if (documentIdPredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.DOCUMENT_ID_FIELD)) {
+                dontCareSubscriptions.put(Constants.DOCUMENT_ID_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.DOCUMENT_ID_FIELD);
+            list.add(subId);
+        }
+        if (hostIdPredicate.isEmpty()) {
+            if (collectionIdPredicate.isEmpty()) {
+                if (!dontCareSubscriptions.containsKey(Constants.COLLECTION_ID_FIELD)) {
+                    dontCareSubscriptions.put(Constants.COLLECTION_ID_FIELD, new TreeSet());
+                }
+                Set list = (Set) dontCareSubscriptions.get(Constants.COLLECTION_ID_FIELD);
+                list.add(subId);
+            }
+        }
+        if (typePredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.TYPE_FIELD)) {
+                dontCareSubscriptions.put(Constants.TYPE_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.TYPE_FIELD);
+            list.add(subId);
+        }
+        
+        if (documentIdPredicate.isEmpty() && collectionIdPredicate.isEmpty() 
+                && hostIdPredicate.isEmpty() && typePredicate.isEmpty()) {
+            noEqualsSubscriptions.add(subId);
+        }
+        subscriptions.put(subId, subscription);
+    }
+
+    /**
+     * @return
+     */
+    public Set getSubscriptionsWithoutEqualsPredicates() {
+        return Collections.unmodifiableSet(noEqualsSubscriptions);
+    }
+
+    /**
+     * Computes the list of equality subscriptions satisfied by the event.
+     * @param event
+     * @return
+     */
+    private Set getPartiallyMatchedSubscriptions(Map event) {
+        Set result = new TreeSet();
+        
+        EqualsPredicate predicate;
+        
+        // all subscriptions with matching type...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.TYPE_FIELD, 
+                (String)event.get(Constants.TYPE_FIELD));
+        if (predicate != null)
+            result.addAll(predicate.getSubscriptionIDs());
+        
+        // ...but only those subscriptions where the doc id matches...
+        Set docIDSubs = new TreeSet();
+        predicate = PredicateFactory.getEqualsPredicate(Constants.DOCUMENT_ID_FIELD, 
+                (String)event.get(Constants.DOCUMENT_ID_FIELD));
+        if (predicate != null)
+            docIDSubs.addAll(predicate.getSubscriptionIDs());
+        docIDSubs.addAll(getDontCareSubscriptions(Constants.DOCUMENT_ID_FIELD));
+        result.retainAll(docIDSubs);
+        
+        // ...and only those subscriptions where the host/collection id matches:
+        Set hostCollIDSubs = new TreeSet();
+        //   i.e.: host|coll is "don't care"...
+        hostCollIDSubs.addAll(getDontCareSubscriptions(Constants.COLLECTION_ID_FIELD));
+        //   ...or host matches (-> coll is "don't care" for this host)...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.HOST_ID_FIELD, 
+                (String)event.get(Constants.HOST_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        //   ...or host|coll matches
+        predicate = PredicateFactory.getEqualsPredicate(Constants.COLLECTION_ID_FIELD, 
+                (String)event.get(Constants.COLLECTION_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        
+        result.retainAll(hostCollIDSubs);
+        
+        return result;
+    }
+
+    /**
+     * 
+     * @param gsComm 
+     * @return
+     * @throws UnsupportedFieldException
+     */
+    private Set findMatchedQueryPredicates(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedPreds = new TreeSet();
+        
+        // iterate through all document content predicates
+        // execute the query stored in the predicate
+        // predicate is matched iff the docID occurs in the query result
+        
+        for (Iterator iter = PredicateFactory.getQueryPredicates(Constants.DOCUMENT_CONTENT_FIELD).iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            String collection = (String) event.get(Constants.COLLECTION_ID_FIELD);
+            String query = predicate.getValue();
+            Set results;
+            try {
+                results = gsComm.fullTextSearch(collection, query);
+                if (results.contains(event.get(Constants.DOCUMENT_ID_FIELD))) {
+                    matchedPreds.add(predicate);
+                }
+            } catch (Exception e) {
+            }
+        }
+        
+        return matchedPreds;
+    }
+
+    /**
+     * Computes the set of SubstringMatchPredicates satisfied by the event.
+     * @param event
+     * 
+     * @return
+     */
+    private Set findMatchedSubstringMatchPredicates(Map event) {
+        Set result = new TreeSet();
+        
+        for (Iterator iter = PredicateFactory.getAllSubstringMatchPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate.isSatisfied(event)) {
+                result.add(predicate);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Filters the event against the profiles. It uses the equality-preferred
+     * algorithm as described in Fabret, Llirbat, Pereira and Shasha:
+     * <em>Efficient Matching for Content-based Publish/Subscribe 
+     * Systems</em>.
+     * @param event
+     * @param gsComm 
+     */
+    public Set filter(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedSubscriptions = new TreeSet();
+    
+        Set partiallyMatchedSubscriptions 
+        	= getPartiallyMatchedSubscriptions(event);
+    
+        Set matchedPredicates = findMatchedSubstringMatchPredicates(event);
+        matchedPredicates.addAll(findMatchedQueryPredicates(event, gsComm));
+    
+        Map numMatchedPredicates = new TreeMap();
+        for (Iterator iter = matchedPredicates.iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            for (Iterator iterator = pred.getSubscriptionIDs().iterator(); iterator
+                    .hasNext();) {
+                Integer subId = (Integer) iterator.next();
+                Subscription sub = (Subscription) subscriptions.get(subId);
+                Integer count = (Integer) numMatchedPredicates.get(subId);
+                int newCountValue = (count == null ? 1 : count.intValue() + 1);
+                numMatchedPredicates.put(subId,
+                        new Integer(newCountValue));
+            }
+        }
+    
+        Set notUnmatchedSubscriptions = partiallyMatchedSubscriptions;
+        notUnmatchedSubscriptions.addAll(noEqualsSubscriptions);
+        for (Iterator iter = notUnmatchedSubscriptions.iterator(); iter
+                .hasNext();) {
+            Integer subId = (Integer) iter.next();
+            Subscription sub = (Subscription) subscriptions.get(subId);
+            int matchedPredicatesCount = 0;
+            if (numMatchedPredicates.get(subId) != null) {
+                matchedPredicatesCount = ((Integer) numMatchedPredicates.get(subId)).intValue();
+            }
+            if (matchedPredicatesCount == sub.getNumOfNonEqualsPredicates()) {
+                matchedSubscriptions.add(sub);
+            }
+        }
+        return matchedSubscriptions;
+    }
+
+
+    /**
+     * @param arguments
+     * @throws Exception
+     */
+    public void createSubscription(Map arguments) throws Exception {
+        Subscription sub = new Subscription(arguments);
+        addSubscription(sub);
+    }
+
+    
+    /**
+     * @param subscriptionID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void deleteSubscription(String subscriptionID) throws DatabaseException, SQLException {
+        Integer subID = new Integer(subscriptionID);
+        System.out.println("deleting subscription " + subscriptionID);
+        if (!subscriptions.containsKey(subID)) {
+            return;
+        }
+        Subscription sub = (Subscription) subscriptions.get(subID);
+        for (Iterator iter = dontCareSubscriptions.values().iterator(); iter.hasNext();) {
+            Set entry = (Set) iter.next();
+            if (entry.contains(subID))
+                entry.remove(subID);
+        }
+        if (noEqualsSubscriptions.contains(subID))
+            noEqualsSubscriptions.remove(subID);
+        for (Iterator iter = sub.getPredicates().iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            pred.removeSubscription(subID);
+        }
+
+        subscriptions.remove(subID);
+        
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        
+        Statement statement = conn.createStatement();
+        
+        String sqlString = "DELETE FROM subscriptions WHERE id = " + subscriptionID;
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT predicate FROM subs_to_predicates " +
+        		"WHERE subscription = " + subscriptionID;
+        ResultSet predicates = statement.executeQuery(sqlString);
+        while (predicates.next()) {
+            // if there aren't any other subscriptions using this predicate, delete predicate
+            int predicateID = predicates.getInt("predicate");
+            
+        }
+        
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     */
+    public void changeSubscription(Map arguments, HttpSession session) {
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        // BTS 14 really edit the subscription
+        
+    }
+
+    /**
+     * @param username
+     * @return
+     * @throws DatabaseException
+     */
+    public Set getAllSubscriptionsFor(String username) throws DatabaseException {
+        Set result = new TreeSet();
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT id FROM subscriptions " +
+            		"WHERE user like '" + username + "';";
+            ResultSet idList = statement.executeQuery(sqlString);
+            while (idList.next()) {
+                int id = idList.getInt("id");
+                result.add(subscriptions.get(new Integer(id)));
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        
+        return result;
+    }
+
+    /**
+     * @param string
+     * @return
+     */
+    private Set getDontCareSubscriptions(String field) {
+        Set result = new TreeSet();
+        if (dontCareSubscriptions.containsKey(field)) {
+            result = (Set) dontCareSubscriptions.get(field);
+        }
+        return result;
+    }
+
+    /**
+     * 
+     */
+    public void restoreFromDatabase() {
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT * FROM subscriptions";
+            ResultSet subs = statement.executeQuery(sqlString);
+            while (subs.next()) {
+                // construct map for all predicates
+                int id = subs.getInt("id");
+                Map map = new TreeMap();
+                map.put("username", subs.getString("user"));
+                map.put("email", subs.getString("email"));
+                map.put("subscription_name", subs.getString("name"));
+                Vector ways = new Vector();
+                if (subs.getInt("rss") != 0) {
+                    ways.add("rss");
+                }
+                if (subs.getInt("page") != 0) {
+                    ways.add("page");
+                }
+                map.put("way", ways);
+                Statement stmnt = conn.createStatement();
+                sqlString = "SELECT p.type, p.field, p.value from subs_to_predicates stp join predicates p on(stp.predicate = p.id) where stp.subscription = " + id + ";";
+                ResultSet predicates = stmnt.executeQuery(sqlString);
+                while (predicates.next()) {
+                    String type = predicates.getString(Constants.TYPE_FIELD);
+                    String field = predicates.getString("field");
+                    if (type.equals(SubstringMatchPredicate.class.getName())) {
+                        if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                            field = Constants.COLLECTION_QUERY_FIELD;
+                        } else if (field.equals(Constants.HOST_ID_FIELD)) {
+                            field = Constants.HOST_QUERY_FIELD;
+                        }
+                    }
+                    String value = predicates.getString("value");
+                    if (type.equals(EqualsPredicate.class.getName())) {
+                        if (!map.containsKey(field)) {
+                            map.put(field, new Vector());
+                        }
+                        ((List) map.get(field)).add(value);
+                    } else {
+                        map.put(field, value);
+                    }
+                }
+                try {
+                    createSubscription(map);
+                } catch (Exception e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+            }
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param i
+     * @return
+     */
+    public Subscription getSubscription(int id) {
+        return (Subscription) subscriptions.get(new Integer(id));
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 21112)
@@ -0,0 +1,21 @@
+/*
+ * Created on Nov 8, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UnsupportedFieldException extends Exception {
+    public UnsupportedFieldException(String msg) {
+        super(msg);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 21112)
@@ -0,0 +1,29 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseException extends Exception {
+    
+    public DatabaseException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * @param string
+     */
+    public DatabaseException(String message) {
+        super(message);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 21112)
@@ -0,0 +1,56 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseManager {
+
+    private static DatabaseManager instance;
+    private Connection database;
+
+    private DatabaseManager() {
+        // hide constructor
+    }
+    
+    public static DatabaseManager getInstance() {
+        if (instance == null) {
+            instance = new DatabaseManager();
+        }
+        return instance;
+    }
+    
+    public Connection getDatabaseConnection() throws DatabaseException {
+        if(database == null) {
+            String connectionString = "jdbc:mysql://localhost/alerting?user=gsdl3admin";
+            try {
+                Class.forName("com.mysql.jdbc.Driver").newInstance();
+                database = DriverManager.getConnection(connectionString);
+            } catch (Exception e) {
+                if (e instanceof SQLException) {
+                    SQLException sqle = (SQLException) e;
+                    System.err.println("SQLState: " + sqle.getSQLState());
+                }
+                throw new DatabaseException("Could not establish connection to database (" + connectionString + ")", e);
+            }
+        }
+        return database;
+    }
+    
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 21112)
@@ -0,0 +1,35 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class EqualsPredicate extends Predicate {
+
+    public EqualsPredicate(String field, String id) {
+        super.field = field;
+        super.value = id;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String value = (String) event.get(field);
+        return value != null && value.equals(value);
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 21112)
@@ -0,0 +1,26 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+/**
+ * Exception thrown when a Predicate cannot be created due to a syntax error.
+ * @author schweer
+ * 
+ */
+public class ParseException extends Exception {
+
+    /**
+     * @param message the exception message.
+     * @see Exception#Exception(String)
+     */
+    public ParseException(String message) {
+        super(message);
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 21112)
@@ -0,0 +1,175 @@
+/*
+ * Created on Nov 2, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Predicate implements Comparable {
+
+    protected Set subscriptions = new TreeSet();
+    protected String field;
+    protected String value;
+    protected int id;
+    
+
+    public static final String[] singleValueFields = new String[] {
+            Constants.DOCUMENT_TITLE_FIELD,
+            Constants.DOCUMENT_CONTENT_FIELD,
+            "metadata_has_field",
+            Constants.HOST_QUERY_FIELD,
+            Constants.COLLECTION_QUERY_FIELD };
+    public static final String[] multiValueFields = new String[] {
+            Constants.TYPE_FIELD,
+            Constants.HOST_ID_FIELD,
+            Constants.COLLECTION_ID_FIELD,
+            Constants.HOST_COLLECTION_ID_FIELD,
+            Constants.DOCUMENT_ID_FIELD };
+    
+    public void addSubscription(int subscriptionID) {
+        subscriptions.add(new Integer(subscriptionID));
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getSubscriptions()
+     */
+    public Set getSubscriptionIDs() {
+        return Collections.unmodifiableSet(subscriptions);
+    }
+
+    public int getID() {
+        return id;
+    }
+    
+    public void setID(int i) {
+        this.id = i;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getKey()
+     */
+    public String getField() {
+        return field;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getValue()
+     */
+    public String getValue() {
+        return value;
+    }
+
+    public String toString() {
+        return field + "=" + value;
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Predicate)) return false;
+        return id == ((Predicate)other).getID();
+    }
+    
+    public int compareTo(Object other) {
+        Predicate otherPredicate = (Predicate) other;
+        return new Integer(id).compareTo(new Integer(otherPredicate.getID()));
+    }
+    
+    public abstract boolean isSatisfied(Map event);
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isSingleValued(String key) {
+        return ArrayHelper.contains(singleValueFields, key);
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isMultiValued(String key) {
+        return ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    int saveToDatabase() throws SQLException, DatabaseException {
+        String query = "SELECT id FROM predicates " +
+        		"WHERE field = '" + field + 
+        		"' AND value = '" + value + 
+        		"' AND type = '" + this.getClass().getName() + "';";
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        System.out.println(query);
+        ResultSet result = statement.executeQuery(query);
+        if (result.next()) { // predicate already exists in database
+            int id = result.getInt("id"); 
+            System.out.println("predicate already exists with id " + id);
+            return id;
+        }
+        // predicate has been newly created
+        String insert = "INSERT INTO predicates (field, type, value) " +
+        "VALUES ('" + field + "', '" + this.getClass().getName() + "', '" + value + "');";
+        System.out.println(insert);
+        statement.execute(insert);
+        result = statement.executeQuery(query);
+        if (result.next()) {
+            return result.getInt("id");
+        } else {
+            throw new DatabaseException("could not save predicate");
+        }
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isFieldName(String key) {
+        return ArrayHelper.contains(singleValueFields, key) || ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @param subID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void removeSubscription(Integer subID) throws SQLException, DatabaseException {
+        subscriptions.remove(subID);
+
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        String sqlString = "DELETE FROM subs_to_predicates " +
+        	"WHERE subscription = " + subID + 
+        	" AND predicate = " + id;
+        statement.executeUpdate(sqlString);
+        
+        if (subscriptions.isEmpty()) {
+            PredicateFactory.deletePredicate(this);
+            sqlString = "DELETE FROM predicates WHERE id = " + id;
+            statement.executeUpdate(sqlString);
+        }
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 21112)
@@ -0,0 +1,245 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.SQLException;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public abstract class PredicateFactory {
+
+    private static int id = 0; 
+    
+    private static Map equalsPredicates = new TreeMap();
+    
+    // substring for collection_name -> SubstringMatchPredicates referring to it
+    private static Map collectionNameMatchPredicates = new TreeMap();
+    private static Map hostNameMatchPredicates = new TreeMap();
+    
+    // query -> QueryPredicates referring to it
+    private static Map documentTitleQueryPredicates = new TreeMap();
+    private static Map documentContentQueryPredicates = new TreeMap();
+    
+    
+    /**
+     * @param key
+     * @param value
+     * @return
+     * @throws ParseException
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public static Predicate createPredicate(String key, String value)
+            throws SQLException, DatabaseException {
+        System.out.println("creating predicate for key " + key + " and value " + value);
+        if (!Predicate.isSingleValued(key)) {
+            return null;
+        }
+        Predicate result = null;
+        if (value == null || value.length() == 0) {
+            return null;
+        }
+        
+        if (Predicate.isMultiValued(key)) {
+            List list = new ArrayList();
+            list.add(value);
+            return (Predicate) createPredicates(key, list).get(0);
+        } else if (key.startsWith("document")) {
+            result = createQueryPredicate(key, value);
+        } else if (key.endsWith("_query")) {
+            result = createSubstringMatchPredicate(key, value);
+        } else {
+        	return null;
+        }
+        result.setID(result.saveToDatabase());
+        return result;
+    }
+    
+    public static List createPredicates(String key, List values) throws SQLException, DatabaseException {
+        if (!Predicate.isMultiValued(key)) {
+            return null;
+        }
+        List results = createEqualsPredicates(key, values);
+        for (Iterator iter = results.iterator(); iter.hasNext();) {
+            EqualsPredicate pred = (EqualsPredicate) iter.next();
+
+            pred.setID(pred.saveToDatabase());
+        }
+        return results;
+    }
+
+    /**
+     * @param key
+     * @param value
+     * @return
+     */
+    private static SubstringMatchPredicate createSubstringMatchPredicate(String key, String value) {
+        SubstringMatchPredicate predicate = null;
+        if (key.equals(Constants.COLLECTION_QUERY_FIELD)) {
+            if (collectionNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) collectionNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.COLLECTION_ID_FIELD, value);
+                collectionNameMatchPredicates.put(value, predicate);
+            }
+        } else if (key.equals(Constants.HOST_QUERY_FIELD)) {
+            if (hostNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) hostNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.HOST_ID_FIELD, value);
+                hostNameMatchPredicates.put(value, predicate);
+            }
+        }
+        // TODO other fields?
+        return predicate;
+    }
+
+    /**
+     * @param key
+     * @param values
+     * @return
+     */
+    private static List createEqualsPredicates(String key, List values) {
+        List result = new Vector();
+        
+        Map keyToPredicates;
+        if (!equalsPredicates.containsKey(key)) {
+            keyToPredicates = new HashMap();
+            equalsPredicates.put(key, keyToPredicates);
+        }
+        keyToPredicates = (Map) equalsPredicates.get(key);
+        
+        for (Iterator iter = values.iterator(); iter.hasNext();) {
+            String value = (String) iter.next();
+            if (value == null || value.length() == 0) {
+                continue;
+            }
+
+            if (key.equals(Constants.HOST_ID_FIELD) && !(value.startsWith("http://"))) {
+                value = "http://" + value + ":8080/soap/servlet/rpcrouter";
+            }
+            
+            EqualsPredicate predicate = null;
+            if (keyToPredicates.containsKey(value)) {
+                predicate = (EqualsPredicate) keyToPredicates.get(value);
+            } else {
+                predicate = new EqualsPredicate(key, value);
+                keyToPredicates.put(value, predicate);
+            }
+            result.add(predicate);
+        }
+        return result;
+    }
+    
+    /**
+     * @param field
+     * @param query
+     * @return
+     */
+    private static QueryPredicate createQueryPredicate(String field,
+            String query) {
+        QueryPredicate predicate = null;
+        if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            if (!documentContentQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentContentQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentContentQueryPredicates.get(query);
+        } else if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            if (!documentTitleQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentTitleQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentTitleQueryPredicates.get(query);
+        }
+        return predicate;
+    }
+
+    public static Collection getAllQueryPredicates() {
+        Set result = new TreeSet();
+        result.addAll(documentTitleQueryPredicates.values());
+        result.addAll(documentContentQueryPredicates.values());
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getQueryPredicates(String field) {
+        Set result = new TreeSet();
+        if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            result.addAll(documentTitleQueryPredicates.values());
+        } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            result.addAll(documentContentQueryPredicates.values());
+        }
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getAllSubstringMatchPredicates() {
+        Collection result = new Vector();
+        result.addAll(hostNameMatchPredicates.values());
+        result.addAll(collectionNameMatchPredicates.values());
+        return Collections.unmodifiableCollection(result);
+    }
+
+    /**
+     * @param string
+     * @param string2
+     * @return
+     */
+    public static EqualsPredicate getEqualsPredicate(String key, String value) {
+        EqualsPredicate result = null; 
+        if (equalsPredicates.containsKey(key)) {
+            result = (EqualsPredicate) ((Map)equalsPredicates.get(key)).get(value);
+        }
+        return result;
+    }
+
+    /**
+     * @param predicate
+     */
+    public static void deletePredicate(Predicate predicate) {
+        String field = predicate.getField();
+        String value = predicate.getValue();
+        
+        if (predicate instanceof EqualsPredicate) {
+            Map map = (Map) equalsPredicates.get(field);
+            if (map.containsKey(value)) {
+                map.remove(value);
+            }
+        } else if (predicate instanceof QueryPredicate) {
+            if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+                documentTitleQueryPredicates.remove(value);
+            } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+                documentContentQueryPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else if (predicate instanceof SubstringMatchPredicate) {
+            if (field.equals(Constants.HOST_ID_FIELD)) {
+                hostNameMatchPredicates.remove(value);
+            } else if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                collectionNameMatchPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else {
+            System.err.println("problem while removing: unknown class " + predicate.getClass().getName());
+        }
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 21112)
@@ -0,0 +1,33 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.List;
+import java.util.Vector;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Profile {
+    private List subscriptions;
+
+    private String user;
+
+    public Profile(String user) {
+        this.user = user;
+        subscriptions = new Vector();
+    }
+
+    public void add(Subscription sub) {
+        subscriptions.add(sub);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 21112)
@@ -0,0 +1,37 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class QueryPredicate extends Predicate {
+    
+    private Set containsWords = new TreeSet();
+    private Set notContainsWords = new TreeSet();
+    
+    public QueryPredicate (String field, String query) {
+        super.field = field;
+        super.value = query;
+    }
+    public boolean isSatisfied(Map event) {
+        return true;
+    }
+    
+    public String toString() {
+        return field + " matches " + value;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 21112)
@@ -0,0 +1,300 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Subscription implements Comparable {
+    private Map map;
+    
+    private int id;
+    private String username;
+    private String name;
+    private String email;
+    private boolean rssNotification;
+    private boolean eventsSincePageNotification;
+    
+    private int numOfNonEqualsPredicates;
+    
+    public Subscription(Map valueMap) throws DatabaseException, SQLException {
+        valueMap = cleanHostCollectionIDs(valueMap);
+        map = new TreeMap();
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            
+            if (!Predicate.isFieldName(key)) {
+                continue;
+            }
+            
+            Object value = valueMap.get(key);
+            
+            if (value instanceof String) {
+                Predicate predicate = PredicateFactory.createPredicate(key, (String) value);  
+                map.put(key, predicate);
+
+                if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                    numOfNonEqualsPredicates++;
+                }
+            } else if (value instanceof List) {
+                List values = (List)value;
+                List predicates = PredicateFactory.createPredicates(key, values);
+                map.put(key, predicates);
+                for (Iterator iterator = predicates.iterator(); iterator
+                        .hasNext();) {
+                    Predicate predicate = (Predicate) iterator.next();
+                    
+                    if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                        numOfNonEqualsPredicates++;
+                    }
+                }
+            } 
+            
+        }
+        
+        System.out.println("finished creating predicates");
+        
+        username = (String) valueMap.get("username");
+        name = (String) valueMap.get("subscription_name");
+        email = (String) valueMap.get("email");
+        rssNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("rss");
+        eventsSincePageNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("page");
+        id = saveToDatabase();
+        
+        for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate != null) {
+                predicate.addSubscription(id);
+            }
+        }
+    }
+    
+    /**
+     * @param valueMap
+     * @return
+     */
+    private Map cleanHostCollectionIDs(Map valueMap) {
+        Map result = new TreeMap();
+        Pattern pattern = Pattern.compile("^(.*)\\|(.*)$");
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (!key.equals(Constants.HOST_COLLECTION_ID_FIELD)) {
+                result.put(key, valueMap.get(key));
+                continue;
+            }
+            if (key.equals(Constants.HOST_ID_FIELD)) { continue; }
+            
+            List values = (List) valueMap.get(key);
+            List hostIDs = new Vector();
+            List collIDs = new Vector();
+            for (Iterator iterator = values.iterator(); iterator.hasNext();) {
+                String id = (String) iterator.next();
+                Matcher matcher = pattern.matcher(id);
+                if (matcher.matches()) {
+                    String host = null;
+                    String coll = null;
+                    try {
+                        host = matcher.group(1);
+                        coll = matcher.group(2);
+                    } catch (IndexOutOfBoundsException e) { /*ignore*/ }
+                    if (host != null && host.length() > 0) {
+                        if (coll != null && coll.length() > 0) {
+                            if (!collIDs.contains(id)) collIDs.add(id);
+                        } else {
+                            if (!hostIDs.contains(host)) hostIDs.add(host);
+                        }
+                    }
+                }
+            }
+            result.put(Constants.HOST_ID_FIELD, hostIDs);
+            result.put(Constants.COLLECTION_ID_FIELD, collIDs);
+        }
+        return result;
+    }
+
+    public boolean containsKey(Object key) {
+        return map.containsKey(key);
+    }
+    
+    public boolean containsValue(Object value) {
+        return map.containsValue(value);
+    }
+    
+    public Predicate getPredicate(String field) {
+        if (map.get(field) instanceof Predicate) {
+            return (Predicate) map.get(field);
+        }
+        return null;
+    }
+    
+    public List getPredicateList(String field) {
+        if (map.get(field) instanceof List) {
+            return (List) map.get(field);
+        }
+        return new Vector();
+    }
+    
+    public Set keySet() {
+        return map.keySet();
+    }
+    
+    public Set getPredicates() {
+        Set result = new HashSet();
+        for (Iterator iter = map.values().iterator(); iter.hasNext();) {
+            Object next = iter.next();
+            if (next == null) {
+                continue;
+            } else if (next instanceof Predicate) {
+                result.add(next);
+            } else if (next instanceof List) {
+                result.addAll((Collection) next);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @return
+     */
+    public int getId() {
+        return id;
+    }
+    
+    public int getNumOfNonEqualsPredicates() {
+        return numOfNonEqualsPredicates;
+    }
+    
+    public String toString() {
+        StringBuffer result = new StringBuffer("Subscription ");
+        result.append(name != null ? name : "(no name) ");
+        result.append(" (id ");
+        result.append(id);
+        result.append("):\n");
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            Object value = map.get(key);
+            if (value instanceof Predicate) {
+                Predicate pred = (Predicate) value;
+                result.append(pred);
+            }
+            if (value instanceof List) {
+                List list = (List) value;
+                for (Iterator iterator = list.iterator(); iterator.hasNext();) {
+                    Predicate pred = (Predicate) iterator.next();
+                    result.append(pred);
+                    if (iterator.hasNext()) {
+                        result.append(" or ");
+                    }
+                }
+            }
+            result.append("\n");
+        }
+        return result.toString();
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Subscription)) return false;
+        return id == ((Subscription)other).id;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Comparable#compareTo(java.lang.Object)
+     */
+    public int compareTo(Object arg0) {
+        Subscription other = (Subscription) arg0;
+        return new Integer(id).compareTo(new Integer(other.id));
+    }
+    
+    private int saveToDatabase() throws DatabaseException, SQLException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String sqlString;
+        boolean initial = true;
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        ResultSet result = statement.executeQuery(sqlString);
+        if (result.next()) {
+            initial = false;
+            this.id = result.getInt("id");
+        }
+        
+        if (initial) {
+            sqlString = "INSERT INTO subscriptions (name, email, rss, page, user) " +
+            		"VALUES ('" + name + "','" + email + "'," + (rssNotification ? 1 : 0) +
+            		"," + (eventsSincePageNotification ? 1: 0)+ ", '" + username + "');";
+        } else {
+            sqlString = "UPDATE subscriptions SET name='" + name + "', email='" +
+            		email + "', rss=" + (rssNotification ? 1 : 0) + ",page=" + 
+            		(eventsSincePageNotification ? 1 : 0)+ " WHERE id=" + this.id + ";";
+            // cannot change user
+        }
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        System.out.println(sqlString);
+        result = statement.executeQuery(sqlString);
+        
+        if (result.next()) {
+            id = result.getInt("id");
+        } else {
+            throw new DatabaseException("Couldn't save subscription");
+        }
+        
+        if (initial) {
+           for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+               Predicate predicate = (Predicate) iter.next();
+               sqlString = "INSERT INTO subs_to_predicates (subscription, predicate) " +
+               		"VALUES (" + id + "," + predicate.getID() + ");";
+               statement.executeUpdate(sqlString);
+           }
+        }
+        return id;
+    }
+
+    /**
+     * @return
+     */
+    public boolean wantsEMailNotification() {
+        return email != null && email.length() > 1;
+    }
+
+    /**
+     * @return
+     */
+    public String getMailAddress() {
+        return email;
+    }
+    
+    public String getName() {
+        return name;
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 21112)
@@ -0,0 +1,39 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class SubstringMatchPredicate extends Predicate {
+
+    public SubstringMatchPredicate(String field, String substring) {
+        super.field = field;
+        super.value = substring;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String fieldValue = (String) event.get(field);
+        return fieldValue != null && fieldValue.indexOf(value) >= 0;
+    }
+
+    public String toString() {
+        return field + " contains " + value;
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 21112)
@@ -0,0 +1,21 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PasswordMismatchException extends Exception {
+    public PasswordMismatchException(String message) {
+        super(message);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 21112)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManagementException extends Exception {
+
+    /**
+     * @param e
+     */
+    public UserManagementException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    
+    public UserManagementException(String message) {
+        super(message);
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 21112)
@@ -0,0 +1,121 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+import java.sql.*;
+import java.util.Map;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManager {
+    private static UserManager instance;
+    
+    private UserManager() {
+        // hide constructor
+    }
+    
+    static public UserManager getInstance() {
+        if (instance == null) {
+            instance = new UserManager();
+        }
+        return instance;
+    }
+
+    /**
+     * @param session
+     * @return
+     */
+    public boolean isLoggedIn(HttpSession session) {
+        if (session.getCreationTime() - session.getLastAccessedTime() > session.getMaxInactiveInterval())
+            return false;
+        return session.getAttribute("username") != null;
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws PasswordMismatchException
+     * @throws UserManagementException
+     */
+    public void createUser(Map arguments, HttpSession session) throws PasswordMismatchException, UserManagementException {
+        if (!arguments.get("password").equals(arguments.get("password2"))) {
+            throw new PasswordMismatchException("The passwords don't match");
+        }
+        String username = (String) arguments.get("username");
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+//        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+//        } catch (NoSuchAlgorithmException e) {
+//            e.printStackTrace();
+//            throw new UserManagementException("could not create user", e);
+//        }
+        
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            statement.executeUpdate("INSERT INTO users (username, password) " +
+            		"VALUES ('" + username + "','" + new String(pwdHash) + "')");
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not create user", e);
+        }
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws UserManagementException
+     */
+    public void loginUser(Map arguments, HttpSession session) throws UserManagementException {
+        String username = (String) arguments.get("username");
+        
+        if (isLoggedIn(session) && session.getAttribute("username").equals(username)) {
+            return; // already logged in
+        }
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+            
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            ResultSet results = statement.executeQuery("SELECT password " +
+            		"FROM users WHERE username like '" + username + "';");
+            String pwdFromDB = "";
+            if(results.next()) {
+                pwdFromDB = results.getString("password");
+            }
+            if (!pwdFromDB.equals(new String(pwdHash))) {
+                throw new PasswordMismatchException("user " + username 
+                        + " is unknown, or the passwords don't match");
+            }
+            session.setAttribute("username", username);
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not login user", e);
+        }
+
+    }
+    
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 21112)
@@ -0,0 +1,40 @@
+/*
+ * Created on Dec 5, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Arrays;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ArrayHelper {
+    static public boolean contains(String[] array, String element) {
+        if (array == null || element == null) {
+            return false;
+        }
+        Arrays.sort(array);
+        int index = Arrays.binarySearch(array, element);
+        return index > -1 && index <= array.length;
+    }
+
+    static public String toString(String [] array) {
+	StringBuffer buffer = new StringBuffer();
+	buffer.append("[");
+	for (int i=0; i<array.length; i++) {
+	    buffer.append((String)array[i]);
+	    buffer.append(",");
+	}
+	buffer.append("]");
+	return buffer.toString();
+
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 21112)
@@ -0,0 +1,58 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Properties;
+
+import javax.mail.*;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class Mailer {
+    
+    private String fromAddress;
+    private String mailHost;
+    
+    public Mailer (String from, String host) {
+        fromAddress = from;
+        mailHost = host;
+    }
+    
+    public void sendMail(String to, String subject, String message) throws MessagingException {
+        // Set the host smtp address
+        Properties props = new Properties();
+        props.put("mail.smtp.host", mailHost);
+        
+        // create some properties and get the default Session
+        //Session session = Session.getDefaultInstance(props, null);
+        Session session = Session.getInstance(props);
+        
+        // create a message
+        Message msg = new MimeMessage(session);
+        
+        // set the from and to address
+        InternetAddress addressFrom = new InternetAddress(fromAddress);
+        msg.setFrom(addressFrom);
+        
+        InternetAddress[] addressTo = new InternetAddress[1];
+        addressTo[0] = new InternetAddress(to);
+        msg.setRecipients(Message.RecipientType.TO, addressTo);
+        
+        // Setting the Subject and Content Type
+        msg.setSubject(subject);
+        msg.setContent(message, "text/plain");
+        Transport.send(msg);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 21112)
@@ -0,0 +1,242 @@
+/*
+ * Created on Nov 11, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ * 
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.*;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PatriciaTree {
+	
+	private Node root;
+	
+	public PatriciaTree () {
+	}
+	
+	public void put(CharSequence key, Object value) {
+		root = put(key, value, root, 0);
+	}
+	
+	public Set get(CharSequence key) {
+		return get(key, root, 0);
+	}
+	
+	public String toString() {
+	    return toPreorderString(root).toString();
+	}
+	
+	private StringBuffer toPreorderString(Node root) {
+	    StringBuffer buffer = new StringBuffer("(\n");
+	    buffer.append(root);
+	    buffer.append("\nchildren:");
+	    if (root.hasChildren()) {
+	        for (Iterator iter = root.children.keySet().iterator(); iter.hasNext();) {
+	            Character key = (Character) iter.next();
+	            Node child = root.getChild(key);
+	            buffer.append("\n");
+	            buffer.append(key);
+	            buffer.append("->");
+	            buffer.append(toPreorderString(child));
+	        }
+	    }
+	    
+	    buffer.append("\n)");
+	    return buffer;
+	}
+	
+	/**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return the set of values associated with <code>key</code>
+	 */
+	private Set get(CharSequence key, Node root, int position) {
+		Set result = new HashSet();
+		
+		int i = findFirstMismatch(key, root, position);
+		
+		if (i == root.skipped.length()) {
+			Character nextChar = new Character(key.charAt(position + i));
+			if (key.length() > position + i + 1) { // nextChar is not last character
+				if (root.hasChild(nextChar)) {
+					result.addAll(get(key, root.getChild(nextChar), position + i + 1));
+				}	
+			} else {
+				result.addAll(root.getValues(nextChar));
+			}
+		}
+		
+		return result;
+	}
+
+	/**
+	 * @param key
+	 * @param value
+	 * @param root
+	 * @param position
+	 * @return the new root node
+	 */
+	private Node put(CharSequence key, Object value, Node root, int position) {
+	    if (root == null) {
+	        root = new Node();
+	        root.skipped = key.subSequence(position, key.length() - 1);
+	        root.addValue(new Character(key.charAt(key.length() - 1)), value);
+	        return root;
+	    }
+	    
+	    if (root.skipped == null || root.skipped.length() == 0) {
+	        addAsValueOrChild(key, value, root, position);
+	        return root;
+	    }
+	    
+	    int i = findFirstMismatch(key, root, position);
+	    if (i >= root.skipped.length() && position + i < key.length()) {
+	        addAsValueOrChild(key, value, root, position + i);
+	        return root;
+	    }
+	    
+	    if (i >= root.skipped.length() && isLastCharacter(key, position + i - 1)) {
+	        Node newRoot = new Node();
+	        newRoot.skipped = root.skipped.subSequence(0, i - 1);
+	        newRoot.addChild(new Character(root.skipped.charAt(i - 1)), root);
+	        try {
+		        root.skipped = root.skipped.subSequence(i, root.skipped.length());
+		    } catch (StringIndexOutOfBoundsException e) {
+		        root.skipped = "";
+		    }
+		    addAsValueOrChild(key, value, newRoot, position + i - 1);
+		    return newRoot;
+	    }
+	    
+	    Node newRoot = new Node();
+	    newRoot.skipped = root.skipped.subSequence(0, i);
+	    newRoot.addChild(new Character(root.skipped.charAt(i)), root);
+	    try {
+	        root.skipped = root.skipped.subSequence(i + 1, root.skipped.length());
+	    } catch (StringIndexOutOfBoundsException e) {
+	        root.skipped = "";
+	    }
+	    
+	    addAsValueOrChild(key, value, newRoot, position + i);
+	    
+	    return newRoot;
+	}
+
+    /**
+     * @param key
+     * @param value
+     * @param root
+     * @param position
+     */
+    private void addAsValueOrChild(CharSequence key, Object value, Node root, int position) {
+        Character nextChar = new Character(key.charAt(position)); 
+        if (isLastCharacter(key, position)) {
+            root.addValue(nextChar, value); 
+        } else {
+            Node newChild = put(key, value, root.getChild(nextChar), position + 1);
+            root.addChild(nextChar, newChild);
+        }
+    }
+
+    /**
+     * @param key
+     * @param position
+     * @return
+     */
+    private boolean isLastCharacter(CharSequence key, int position) {
+        return key.length() - position == 1;
+    }
+
+    /**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return
+	 */
+	private int findFirstMismatch(CharSequence key, Node root, int position) {
+		int i = 0;
+		while (i < root.skipped.length()) {
+			try {
+				if (root.skipped.charAt(i) == key.charAt(position + i)) {
+					i++;
+				} else {
+					return i;
+				}
+			} catch (StringIndexOutOfBoundsException e) {
+				return i;
+			}
+		}
+		return i;
+	}
+
+
+	private class Node {
+		private CharSequence skipped = "";
+		private Map children;
+		private Map values;
+		
+		/**
+		 * @return
+		 */
+		public Set getValues(Character key) {
+			if (values == null || !values.containsKey(key))
+				return new HashSet();
+			return (Set) values.get(key);
+		}
+
+		boolean hasChild(Character key) {
+			return children != null && children.containsKey(key);
+		}
+		
+		boolean hasChildren() {
+			return children != null && !children.isEmpty();
+		}
+		
+		void addChild(Character key, Node child) {
+			if (children == null ) {
+				children = new TreeMap();
+			}
+			children.put(key, child);
+		}
+		
+		Node getChild(Character key) {
+			if (children == null) return null;
+			return (Node) children.get(key);
+		}
+		
+		void addValue(Character key, Object value) {
+			if (values == null) {
+				values = new TreeMap();
+			}
+			Set valueSet;
+			if (values.containsKey(key)) {
+				valueSet = (Set) values.get(key);
+			} else {
+				valueSet = new HashSet();
+				values.put(key, valueSet);
+			}
+			valueSet.add(value);
+		}
+		
+		public String toString() {
+		    StringBuffer buffer = new StringBuffer();
+		    buffer.append(skipped);
+		    buffer.append("\nvalues:");
+		    if (values != null) {
+		        buffer.append(values);
+		    }
+		    return buffer.toString();
+		}
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 21112)
@@ -0,0 +1,68 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class AbstractClientResponse implements ClientResponse
+{ private boolean success;
+  private Map     values;
+
+  public AbstractClientResponse()
+  { this.success = true;
+    this.values  = new HashMap();
+  }
+  
+  public boolean isSuccessful()
+  { return this.success;
+  }
+  
+  public void setSuccessful(boolean success)
+  { this.success = success;
+  }
+  
+  public void addValue(String label, String value)
+  { if (this.values.get(label) == null)
+    { this.setValue(label, value);
+    }
+    else
+    { Vector set = (Vector) this.values.get(label);
+      set.addElement(value);
+    }
+  }
+  
+  public void setValue(String label, String value)
+  { Vector set = new Vector();
+    set.addElement(value);
+    this.values.put(label, set);
+  }
+  
+  public Vector getValues(String label)
+  { return (Vector) this.values.get(label);
+  }
+
+  public Set getLabels()
+  { return this.values.keySet();
+  }
+
+  public String toString()
+  { Iterator keys = this.values.keySet().iterator();
+
+    StringBuffer buffer = new StringBuffer();
+
+    while (keys.hasNext())
+    { String key = keys.next().toString();
+
+      buffer.append("  " +  key + ":\n");
+
+      Vector vals = (Vector) this.values.get(key);
+      Enumeration valist = vals.elements();
+      while (valist.hasMoreElements())
+	{ String nextVal = valist.nextElement().toString();
+
+	  buffer.append("    ");
+	  buffer.append(nextVal);
+	  buffer.append("\n");
+	}
+    }
+    return buffer.toString();
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 21112)
@@ -0,0 +1,71 @@
+package org.greenstone.p2p;
+
+import java.net.*;
+import java.util.*;
+
+public abstract class AbstractServerAction implements NameServerAction
+{ NameServer host;
+
+  public AbstractServerAction(NameServer host)
+  { this.host = host;
+  }
+
+  public abstract String getActionName();
+  
+  public void propagateAction(URL forwarder, Map parameters)
+  { Enumeration subServers = host.subServers();
+  
+    // poll all child servers
+    while (subServers.hasMoreElements())
+    { URL server = (URL) subServers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.propagateActionViaServer(server, parameters, forwarder);
+      }
+    }
+    
+    // poll upwards to parent
+    if (host.getParent() != null &&
+	!host.getParent().equals(forwarder))
+    { this.propagateActionViaServer(host.getParent(), parameters, forwarder);
+    }
+  }
+
+  public void doAction(URL forwarder, Map parameters)
+  { // start propagating the action outwards
+    this.propagateAction(forwarder, parameters);
+  
+    // now execute it on our children
+    Enumeration servers = host.servers();
+  
+    while (servers.hasMoreElements())
+    { URL server = (URL) servers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.doActionOnServer(server, parameters, forwarder);
+      }
+    }
+  }
+  
+  String propagateActionViaServer(URL server, Map paramters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to server, in a separate thread
+    // get responses
+    // forward those back
+    return "";
+  }
+  
+  String doActionOnServer(URL server, Map parameters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to the destination server (as a URL)
+    
+    // get response (as HTML)
+    
+    // forward it back...
+    return "";
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/Client.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 21112)
@@ -0,0 +1,40 @@
+import java.io.*;
+import java.net.*;
+
+public class Client {
+    public static String read() throws IOException {
+
+        Socket echoSocket = null;
+        PrintWriter out = null;
+        BufferedReader in = null;
+
+        try {
+            echoSocket = new Socket("taranis", 7);
+            out = new PrintWriter(echoSocket.getOutputStream(), true);
+            in = new BufferedReader(new InputStreamReader(
+                                        echoSocket.getInputStream()));
+        } catch (UnknownHostException e) {
+            System.err.println("Don't know about host: taranis.");
+            System.exit(1);
+        } catch (IOException e) {
+            System.err.println("Couldn't get I/O for "
+                               + "the connection to: taranis.");
+            System.exit(1);
+        }
+
+        BufferedReader stdIn = new BufferedReader(
+                                   new InputStreamReader(System.in));
+        String userInput;
+
+        while ((userInput = stdIn.readLine()) != null) {
+            out.println(userInput);
+            System.out.println("echo: " + in.readLine());
+        }
+
+        out.close();
+        in.close();
+        stdIn.close();
+        echoSocket.close();
+	return "";
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public interface ClientResponse
+{ boolean isSuccessful(); // obvious
+  Vector  getValues(String label); // get the values associated with a given
+                                   // label; which values are available varies
+                                   // from action to action
+  Set     getLabels();             // get the labels in this response
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 21112)
@@ -0,0 +1,13 @@
+package org.greenstone.p2p;
+
+public class DefaultServerAction extends AbstractServerAction
+{ public static final String DEFAULT_ACTION = "default";
+
+  public DefaultServerAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return DEFAULT_ACTION;
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 21112)
@@ -0,0 +1,31 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public interface GreenstoneClient
+{ /**
+   * This interface needs to be fulfilled by a Greenstone machine wanting to
+   * register and receive messages from the Greenstone Distributed Service.
+   *
+   */
+
+  /**
+   * This method is called by a GDS server when an action (message) has a response
+   * sent in reply.  The Greenstone server that sends the action will have this
+   * method called for each response received from the GDS network.
+   *
+   * @param: <code>NameServerAction</code> - at present, </code>null</code>.
+   * This method *must* be synchronized
+   * @param: <code>ClientResponse</code> - is a simple wrapper for the Map class.
+   * 
+   */
+  public void receiveResponse(NameServerAction action, ClientResponse response);
+
+  /**
+   *
+   */
+  public void receiveMessage(String message, Map parameters);
+  public String getIdentifier();
+  public URL    getURL();
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 21112)
@@ -0,0 +1,94 @@
+
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+/**
+ * This is the class that provides a set of functions for a Greenstone server to call
+ * in order to send information/messages into a GDS network.
+ */
+
+public class GreenstoneInterface
+{ private GreenstoneClient client;
+  private URL              server;
+
+  // temporary variables which will eventually disapper - ignore!
+  private static GreenstoneClient lastClient;
+  private static NameServer       nameServer = null;
+  // end of temporary variables
+
+  /**
+   * Make a new interface with a client that will receive the responses
+   * to any messages sent.
+   *
+   * @param: the client to receive responses to messages sent through this
+   *         interface.
+   */
+  public GreenstoneInterface(GreenstoneClient client)
+  { this.client = client;
+    lastClient = this.client;
+    //    /*this.*/nameServer = null;
+  }
+
+  /**
+   *  Set up a registration for this machine with a GDS server.
+   *
+   *  The Greenstone machine should take care to store the URL of the
+   *  server for later use - don't continually reregister the same
+   *  server!
+   *
+   *  @param: <code>URL</code> the location of the GDS server to register with
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean registerWithServer(URL server)
+  { // connect to server
+    // 
+    if (/*this.*/nameServer == null) {
+      /*this.*/nameServer = new NameServer(server.toString());
+    }
+    this.server = server;
+    /*this.*/nameServer.registerServer(this.client.getURL());
+
+    return true;
+  }
+
+  /**
+   *  Method to unregister from the current GDS server for this Greenstone
+   *  machine.
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean unregisterFromServer()
+  { /*this.*/nameServer = null;
+    return true;
+  }
+
+  /**
+   *  Send a message into the GDS system through the current GDS server.
+   *
+   *  Success does not guarantee that you will receive a response.
+   *
+   *  @param: <code>String</code> the identifier for this type of message
+   *  @param: <code>Map</code> a <code>Vector</code> item is associated with
+   *          each key in this map.
+   *
+   *  @return: <code>boolean</code> whether the sending was successful or not.
+   */
+  public boolean sendMessage(String message, Map parameters)
+  { if (/*this.*/nameServer != null)
+      { /*this.*/nameServer.sendMessage(this.client.getURL(), message, parameters);
+    }
+    return false;
+  }
+
+  /**
+   *  Receive a response to a message sent from this Greenstone machine and
+   *  forward it to the current client - won't be needed in the long run, it
+   *  is here at present to provide a route for returning responses.
+   */
+  static public void receiveResponse(URL greenstoneURL, ClientResponse response)
+  { lastClient.receiveResponse(null, response);
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+
+/**
+ *  Simple class to hold the location of a Greenstone server.
+ *
+ *  This class should be subclassed and methods overridden for 
+ *  Greenstone 2, 3 or any special installations such as the
+ *  Music DL.
+ */
+
+public class GreenstoneServer
+{ private URL server;
+  
+  public GreenstoneServer(URL url)
+  { this.server = url;
+  }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 21112)
@@ -0,0 +1,70 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+public class GreenstoneTest implements GreenstoneClient
+{ String name;
+  URL    url;
+  
+  public GreenstoneTest(String id, String url)
+  { try {
+      this.url = new URL(url);
+    }
+    catch (java.net.MalformedURLException malEx)
+      { this.url = null;
+      }
+    this.name = id;
+  }
+
+  public void receiveResponse(NameServerAction action, ClientResponse response)
+  {
+    System.out.println("\n----\nReceived a response:");
+    System.out.println(response.toString());
+  }
+
+  public void receiveMessage(String message, Map parameters)
+  {
+  }
+
+  public String getIdentifier()
+  { return this.name;
+  }
+
+  public URL getURL()
+  { return this.url;
+  }
+
+  /**
+   *  arg[0] => identifier for this machine
+   *  arg[1] => a url for this machine
+   *  arg[2] => a url for the GDS server
+   *
+   * e.g. run with:
+   * "java org.greenstone.p2p.GreenstoneTest localhost http://localhost http://www.nzdl.org"
+   */
+  public static void main(String args[])
+  { GreenstoneClient client = new GreenstoneTest(args[0], args[1]);
+    GreenstoneClient gsdl   = new GreenstoneTest("gsdl", "http://www.gsdl.org");
+    
+    URL server;
+    try
+      {
+	server = new URL(args[2]);
+      }
+    catch (java.net.MalformedURLException malEx)
+      { server = null;
+      }
+    
+    GreenstoneInterface gsdlIface = new GreenstoneInterface(gsdl);
+    gsdlIface.registerWithServer(server);
+
+    GreenstoneInterface iface = new GreenstoneInterface(client);
+    iface.registerWithServer(server);
+
+    Map parameters = new HashMap();
+    iface.sendMessage("ListServers", parameters);
+
+    iface.sendMessage("DocDelta", parameters);
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 21112)
@@ -0,0 +1,35 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public class ListServersAction extends AbstractServerAction
+{ public String LISTSERVERS = "ListServers";
+
+  public ListServersAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return LISTSERVERS;
+  }
+  
+  public void doAction(URL forwarder, Map parameters)
+  { this.propagateAction(forwarder, parameters);
+  
+    // Build a list of all servers
+    AbstractClientResponse response = new AbstractClientResponse();
+
+    Enumeration servers = this.host.servers();
+    while (servers.hasMoreElements())
+      { URL gsServer = (URL) servers.nextElement();
+	System.out.println("Adding listserver item " + gsServer);
+
+	response.addValue("Server", gsServer.toString());
+      }
+    response.setSuccessful(true);
+
+    // Send it back to the forwarder right now...
+    this.host.receiveResponse(forwarder, response);
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 21112)
@@ -0,0 +1,124 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServer
+{ ServerList list;
+  URL        parent;
+  Vector     childServers; // name servers
+  Vector     children;     // greenstone servers
+  String     identity;
+  Map        actionMap;
+ 
+  public NameServer(String identity)
+  { this.list     = new ServerList();
+    this.identity = identity;
+    this.parent   = null;
+    this.children = new Vector();
+    this.childServers = new Vector();
+
+    this.actionMap = new HashMap();
+
+    NameServerAction listServersAction = new ListServersAction(this);
+    this.addAction(listServersAction);
+    NameServerAction defaultAction = new DefaultServerAction(this);
+    this.addAction(defaultAction);
+    
+    // set up registration listener
+    // set up messaging broadcaster
+    // set up messaging listener
+  }
+
+  private void addAction(NameServerAction action)
+  { this.actionMap.put(action.getActionName(), action);
+  }
+
+  private void loadChildren(BufferedReader in)
+  { try 
+      { String child = in.readLine();
+      }
+    catch (IOException ex)
+      {
+      }
+  }
+
+  private void loadServers(BufferedReader in)
+  { try
+      { String child = in.readLine();
+      }
+    catch (IOException io)
+      {
+      }
+  }
+
+  public void sendMessage(URL origin, String actionName, Map parameters)
+  { NameServerAction action = (NameServerAction) this.actionMap.get(actionName);
+    if (action != null)
+      { action.doAction(origin, parameters);
+      }
+    else
+      { System.out.println("Sending action " + actionName + " through <default>");
+
+	// TODO: add action name to the parameter map so that it is not lost...
+	action = (NameServerAction) this.actionMap.get("Default");
+	if (action != null)
+	{ action.doAction(origin, parameters);
+	}
+      }
+    //NameServerAction action = NameServerActionFactory.createAction(action, parameters);
+    //action.doAction();
+  }
+
+  public void receiveResponse(URL responseTo, ClientResponse response)
+  { GreenstoneInterface.receiveResponse(responseTo, response);
+  }
+    
+  private void loadParent()
+  {
+  }
+
+  public String getIdentity()
+  { return this.identity;
+  }
+  
+  public void setParent(URL parent)
+  { this.parent = parent;
+  }
+  
+  public URL getParent()
+  { return this.parent;
+  }
+  
+  public void addSubServer(URL subserver)
+  { this.childServers.add(subserver);
+  }
+  
+  public void registerServer(URL child)
+  { System.out.println("Receiving registration from " + child.toString());
+    this.children.add(child);
+  }
+  
+  public Enumeration subServers()
+  { return this.childServers.elements();
+  }
+  
+  public Enumeration servers()
+  { return this.children.elements();
+  }
+
+  public static boolean pollServer(URL server)
+  { return false;
+  }
+
+  /**
+   *  
+   */
+  public static void main(String args[])
+  { NameServer server = new NameServer(args[0]);
+
+    // Load the action items
+    // Wait for signals
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 21112)
@@ -0,0 +1,10 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+import java.net.URL;
+
+public interface NameServerAction
+{ public String getActionName();
+  public void   doAction(URL forwarder, Map parameters);
+  public void   propagateAction(URL forwarder, Map parameters);
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+
+public class NameServerActionFactory
+{ public NameServerAction createAction(String actionName, Map parameters)
+  { return null;
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 21112)
@@ -0,0 +1,16 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServerRegistrationThread extends Thread
+{ public void run() 
+  { /*try
+    {
+    }
+    catch (IOException ex)
+    {
+    }*/
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 21112)
@@ -0,0 +1,19 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class ServerList
+{ Map map;
+
+  public ServerList()
+  { this.map = new HashMap();
+  }
+  
+  public void addServer(GreenstoneServer server)
+  { this.map.put(server, server);
+  }
+  
+  Iterator servers()
+  { return ((Collection)this.map.values()).iterator();
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 21112)
@@ -0,0 +1,163 @@
+/*
+ * Created on Nov 24, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.exception.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingServiceTest extends TestCase {
+
+    private AlertingService as;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(AlertingServiceTest.class);
+    }
+
+    public final void testSubscribe() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.DOCUMENT_CONTENT_FIELD, "albatros");
+            
+            String template = as.createSubscription(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testListSubscriptions() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            
+            String template = as.listSubscriptions(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testReceiveEvent() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.DOCUMENT_ID_FIELD, "20041124:1");
+            arguments.put(Constants.DOCUMENT_TITLE_FIELD, "keine albatrosse in sicht");
+            arguments.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            as.receiveEvent(arguments);
+            
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testShowFeed() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put("subscriptionID", "0");
+            
+            String template = as.showFeed(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+    protected void setUp() throws Exception {
+        as = new AlertingService();
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 21112)
@@ -0,0 +1,75 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicatorTest extends TestCase {
+
+    private GreenstoneCommunicator gsComm;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(GreenstoneCommunicatorTest.class);
+    }
+    
+    protected void setUp() throws Exception {
+        URL host;
+        try {
+            host = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            gsComm = new GreenstoneCommunicator(host);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testGetSiteList() {
+        String[] sites;
+        try {
+            sites = gsComm.getCollectionNames();
+            for (int i = 0; i < sites.length; i++) {
+                System.out.println(sites[i]);
+                
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testSupportsFullTextSearch() {
+        try {
+            assertTrue(gsComm.supportsTextSearch("gs2mgppdemo"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testFullTextSearch() {
+        try {
+            Set result = gsComm.fullTextSearch("gs2mgppdemo", "Burundi");
+            assertEquals(14, result.size());
+            assertTrue(result.contains("HASH4632a8a51d33c47a75c559.7.1"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 21112)
@@ -0,0 +1,87 @@
+/*
+ * Created on Nov 18, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ProfileStoreTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(ProfileStoreTest.class);
+    }
+
+    public final void testAddSubscription() {
+        try {
+            addSubscriptions();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    public final void testFilter() {
+        try {
+            addSubscriptions();
+            
+            URL url = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+            
+            Map event = new TreeMap();
+            
+            event.put(Constants.DOCUMENT_ID_FIELD, "20041118:5");
+            event.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+            event.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            Set result = ProfileStore.getInstance().filter(event, gsComm);
+            
+            System.out.println(result.size());
+            
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    private void addSubscriptions() throws Exception {
+        Map valueMap = new TreeMap();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Tomatensalat");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Water+Buffalo");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "gender+equality");
+        
+        addSubscription(valueMap);
+    }
+    
+    private void addSubscription(Map valueMap) throws Exception {  
+        ProfileStore.getInstance().createSubscription(valueMap);    
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 21112)
@@ -0,0 +1,35 @@
+/*
+ * Created on Jan 1, 2005
+ *
+ * TODO To change the template for this generated file go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+package org.greenstone.gsdlas.util;
+
+import javax.mail.MessagingException;
+
+import junit.framework.TestCase;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class MailerTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(MailerTest.class);
+    }
+
+    public final void testSendMail() {
+        Mailer mailer = new Mailer("andrea@localhost", "localhost");
+        try {
+            mailer.sendMail("andrea@localhost", "test", "test message testing...\n");
+        } catch (MessagingException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/WEB-INF/web.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/WEB-INF/web.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/WEB-INF/web.xml	(revision 21112)
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+
+
+    <!-- General description of your web application -->
+
+    <display-name>Alerting Service for Greenstone</display-name>
+    <description>
+	An alerting service for Greenstone, 
+	written by Andrea Schweer (schweer@cs.waikato.ac.nz).
+    </description>
+
+
+    <!-- Context initialization parameters that define shared
+         String constants used within your application, which
+         can be customized by the system administrator who is
+         installing your application.  The values actually
+         assigned to these parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletContext().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of context initialization
+         parameters, including zero.
+    -->
+
+    <context-param>
+      <param-name>webmaster</param-name>
+      <param-value>schweer@cs.waikato.ac.nz</param-value>
+      <description>
+        The EMAIL address of the administrator to whom questions
+        and comments about this application should be addressed.
+      </description>
+    </context-param>
+
+
+    <!-- Servlet definitions for the servlets that make up
+         your web application, including initialization
+         parameters.  With Tomcat, you can also send requests
+         to servlets not listed here with a request like this:
+
+           http://localhost:8080/{context-path}/servlet/{classname}
+
+         but this usage is not guaranteed to be portable.  It also
+         makes relative references to images and other resources
+         required by your servlet more complicated, so defining
+         all of your servlets (and defining a mapping to them with
+         a servlet-mapping element) is recommended.
+
+         Servlet initialization parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletConfig().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of servlets, including zero.
+    -->
+
+    <servlet>
+      <servlet-name>service</servlet-name>
+      <description>
+        An alerting service for Greenstone 3
+      </description>
+      <servlet-class>org.greenstone.gsdlas.AlertingService</servlet-class>
+      <init-param>
+            <param-name>properties</param-name>
+            <param-value>/velocity.properties</param-value>
+      </init-param>
+    </servlet>
+<!--
+    <servlet>
+      <servlet-name>subscribe</servlet-name>
+      <description>
+        This servlet creates new subscriptions.
+      </description>
+      <servlet-class>org.greenstone.gsdlas.SubscriptionServlet</servlet-class>
+    </servlet> -->
+
+	<!-- Define mappings that are used by the servlet container to
+         translate a particular request URI (context-relative) to a
+         particular servlet.  The examples below correspond to the
+         servlet descriptions above.  Thus, a request URI like:
+
+           http://localhost:8080/{contextpath}/graph
+
+         will be mapped to the "graph" servlet, while a request like:
+
+           http://localhost:8080/{contextpath}/saveCustomer.do
+
+         will be mapped to the "controller" servlet.
+
+         You may define any number of servlet mappings, including zero.
+         It is also legal to define more than one mapping for the same
+         servlet, if you wish to.
+    -->
+
+<!--    <servlet-mapping>
+      <servlet-name>receive</servlet-name>
+      <url-pattern>/receive</url-pattern>
+    </servlet-mapping>
+
+    <servlet-mapping>
+      <servlet-name>subscribe</servlet-name>
+      <url-pattern>/subscribe</url-pattern>
+    </servlet-mapping> -->
+    <servlet-mapping>
+      <servlet-name>service</servlet-name>
+      <url-pattern>/service</url-pattern>
+    </servlet-mapping>
+</web-app>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/about.html
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/about.html	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/about.html	(revision 21112)
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+      <title>Greenstone Version 3</title>
+			<link rel="stylesheet" type="text/css" href="default.css"/>
+			</head>
+  <body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+    <div id="content"><h3>Alerting Service for Greenstone &ndash; About</h3>
+		<p></p>
+	
+	
+	(description of what an alerting service does)<br/>
+	<a href="index.html">back</a> to the alerting service.<br/>
+	</div>
+  </body></html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/default.css
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/default.css	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/default.css	(revision 21112)
@@ -0,0 +1,52 @@
+body {
+	color:             black;
+	background-color:  white;
+	/*font-family:       "Trebuchet MS", Arial, helvetica, sans-serif;*/
+	text-align:        center;
+	margin:            0px;
+	background-image:  url("images/chalk.gif");
+	background-repeat: repeat-y;
+}
+
+#content {
+	background-image: url("images/gsdlhead.gif");
+	background-repeat: no-repeat;
+	background-position: center top;
+	padding-top:  88px;
+	width:        537px;
+	position:     relative;
+	text-align:   left;
+	margin-left:  auto;
+	margin-right: auto;
+	margin-top:   10px;
+}
+
+
+.error {
+        color: red;
+        font-weight: bold;
+}
+
+.details {
+        font-size: small;
+}
+
+table, thead {
+        border: 1pt solid black;
+}
+
+thead {
+        background-color: gray;
+}
+
+a:link {
+	color: #006666;
+}
+
+a:active {
+	color: #cc9900;
+}
+
+a:visited {
+	color: #666633;
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/descriptor.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/descriptor.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/descriptor.xml	(revision 21112)
@@ -0,0 +1,8 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+  <isd:faultListener>org.apache.soap.server.DOMFaultListener</isd:faultListener>
+</isd:service>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/feed.css
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/feed.css	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/feed.css	(revision 21112)
@@ -0,0 +1,45 @@
+@import url("default.css");
+
+#channelhead {
+	margin-bottom: 20px;
+}
+
+#channelhead a:link {
+	text-decoration: none;
+	color:           black;
+}
+
+#channelhead a:visited {
+	text-decoration: none;
+	color:            black;
+}
+
+
+#channelhead a:hover, a:focus, a:active {
+	color:            blue;
+	text-decoration:  underline;
+}
+
+#channelinfo {
+	border-top: 2px solid black;
+	font-size: small;
+	margin-top: 20px;
+}
+
+div.item {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+span.date, span.category {
+	font-style: italic;
+}
+
+div.description {
+	font-size: small;
+}
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/form.css
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/form.css	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/form.css	(revision 21112)
@@ -0,0 +1,72 @@
+@import url("default.css");
+
+form {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+td, th {
+	padding: 0px 5px;
+}
+
+thead tr {
+	background-color: #CCCCCC;
+}
+
+tr.even {
+	background-color: #CCCCCC;
+}
+
+.explanation {
+	font-size: small;
+}
+
+#progress {
+	font-size: x-small;
+	border: 0px solid black;
+	width: 80%;
+	margin: 5px auto;
+}
+
+#progress td {
+	width: 25%;
+	vertical-align: top;
+	text-align: center;
+	color: #555;
+	border-top: 10px solid #CCC;
+	font-family: "Trebuchet MS", Arial, helvetica, sans-serif;
+}
+
+#progress td.completed {
+	border-top-color: #9CBE9C;
+	color: #000;
+}
+
+#progress td.current {
+	background-color: #F0E68C;
+	color: #000;
+}
+
+table.subscription {
+	font-size: small;
+}
+
+.rssbutton {
+	background: #ff6600;
+	color: #ffffff;
+	border-left:   1px solid #cc9966;
+	border-top:    1px solid #ccaa99;
+	border-right:  1px solid #993300;
+	border-bottom: 1px solid #331100;
+	padding: 0px 0.5em 0px 0.5em;
+	font-family: helvetica, arial, sans-serif;
+	font-weight: bold;
+	font-size: xx-small;
+	margin-top: 1em;
+}
+
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/index.html
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/index.html	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/index.html	(revision 21112)
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="default.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>What do you want to do today?</p>
+			
+			<ul>
+				<li>
+					<a href="/alerting/service?action=showLoginForm">Log in</a></li>
+				</li>
+				<li>
+					<a href="subscribe.html">Create</a> a subscription.</li>
+				<li>
+					<a href="/alerting/service?action=listSubscriptions">View and edit</a> your existing subscriptions.</li>
+				<li>
+					<a href="about.html">Read more</a> about the alerting service.</li>
+				<li>
+					<a href="/alerting/service?action=logout">Log out</a></li?
+			</ul>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/list.css
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/list.css	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/list.css	(revision 21112)
@@ -0,0 +1,12 @@
+@import url("default.css");
+
+div.subscription {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/rss2html.xsl
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/rss2html.xsl	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/rss2html.xsl	(revision 21112)
@@ -0,0 +1,65 @@
+<?xml version="1.0" ?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+	<xsl:output method="xml" 
+  	doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" 
+	  doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN" indent="yes"/>
+	<!-- <xsl:output method="html"/> -->
+
+	<xsl:template match="/">
+		<html>
+			<xsl:attribute name="lang">
+				<xsl:value-of select="//channel/language"/>
+			</xsl:attribute>
+			<head>
+				<link rel="stylesheet" type="text/css" href="/alerting/feed.css"/>
+				<title>
+					<xsl:value-of select="//channel/title"/>
+				</title>
+			</head>
+			<body>
+				<div id="content">
+					<xsl:apply-templates/>
+				</div>
+   		</body>
+		</html>
+  </xsl:template>
+	
+	<xsl:template match="channel">
+		<div id="channelhead">
+      <h3>
+				<a>
+      		<xsl:attribute name="href">
+        		<xsl:value-of select="link"/>
+					</xsl:attribute>
+					<xsl:value-of select="title"/>
+				</a>
+			</h3>
+			<div class="description">
+        	<xsl:value-of select="description" disable-output-escaping="yes"/>
+			</div>
+		</div>
+    <xsl:apply-templates/>
+		<div id="channelinfo">
+			Last rebuilt <xsl:value-of select="lastBuildDate"/>.<br/>
+			This file has been automatically generated by <xsl:value-of select="generator"/>; please <a><xsl:attribute name="href">mailto:<xsl:value-of select="webMaster"/></xsl:attribute>contact the webmaster</a> if you have any questions.<br/>
+		</div>
+  </xsl:template>
+
+  <xsl:template match="item">
+		<div class="item">
+			<span class="date"><xsl:value-of select="pubDate"/></span>: 
+			<span class="category"><xsl:value-of select="category"/></span><br/>
+    	<a class="item">
+      	<xsl:attribute name="href">
+        	<xsl:value-of select="link"/>
+				</xsl:attribute>
+				<xsl:value-of select="title"/>
+      </a><br/>
+		</div>
+      <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="text()"/>
+	
+</xsl:stylesheet>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/subscribe.html
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/subscribe.html	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/subscribe.html	(revision 21112)
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="form.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>Create a subscription</p>
+			
+			You can subscribe to be notified of events about these elements:
+			<ul>
+				<li><a href="/alerting/service?action=createSubscription">documents</a> (new, modified or deleted documents)</li>
+				<li><a href="sub_docparts.html">parts of documents</a> (new, modified or deleted parts of documents)</li>
+				<li><a href="sub_metadata.html">document metadata</a> (new, modified or deleted metadata of documents)</li>
+				<li><a href="sub_collection.html">collections</a> (new, rebuilt or deleted collections)</li>
+				<li><a href="sub_classifiers.html">classifiers of collections</a> (new or deleted classifiers) <img alt="help" src="images/help.png" longdesc="this influences the ways you can browse a collection" title="this influences the ways you can browse a collection"/></li>
+				<li><a href="sub_indexes.html">indexes of a collection</a> (new or deleted indexes) <img alt="help" src="images/help.png" longdesc="this influences the ways you can search in a collection" title="this influences the ways you can search in a collection"/></li>
+			</ul>
+			
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/test.html
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/test.html	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/test.html	(revision 21112)
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Alerting Service for Greenstone</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+		<style type="text/css">
+		<!--
+		table { border:1pt solid black; padding:2px; background:#F0F0F0; }
+		th { text-align:left; background:lightblue; }
+		th,td { border-width:0px; }
+		-->
+		</style>
+	</head>
+	<body>
+		<div id="content">
+		<!-- <h1>Test</h1>
+		<form action="/alerting/service?test" method="post">
+			<input type="text" name="text"/><br/>
+			<input type="submit" value="submit"/>
+		</form>
+		<hr/> -->
+		
+		<h3>Subscribe</h3>
+		<form action="/alerting/service" method="post" name="subscribe">
+			<input type="hidden" name="action" value="subscribe"/>
+			<table border="1">
+				<tr>
+					<th>Event Type</th>
+					<td>
+						<input type="radio" name="type" value="new_document" checked="checked"/> new document
+						<input type="radio" name="type" value="document_deleted"/> document deleted
+						<input type="radio" name="type" value="document_content_changed" /> content changed
+						<input type="radio" name="type" value="document_md_changed"/> metadata changed
+					</td>
+				</tr>
+				<tr>
+					<th>Document ID</th>
+					<td>
+						<input type="text" name="documentID"/> 
+					</td>
+				</tr>
+				<tr>
+					<th>Document Title</th>
+					<td>
+						<input type="text" name="document_title"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Document Content</th>
+					<td>
+						<input type="text" name="document_content"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Collection ID</th>
+					<td>
+					 <input type="text" name="collectionID"/>
+					</td>
+				</tr>
+			</table>
+			<br/>
+			<input type="submit" value="submit"/>
+		</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/greenstone3-extensions/gsdl-as/web/velocity.properties
===================================================================
--- /main/tags/3.01/greenstone3-extensions/gsdl-as/web/velocity.properties	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/gsdl-as/web/velocity.properties	(revision 21112)
@@ -0,0 +1,155 @@
+#----------------------------------------------------------------------------
+# These are the default properties for the
+# Velocity Runtime. These values are used when
+# Runtime.init() is called, and when Runtime.init(properties)
+# fails to find the specificed properties file.
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+# R U N T I M E  L O G
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+#  default LogSystem to use: default: AvalonLogSystem
+#----------------------------------------------------------------------------
+
+# runtime.log.logsystem.class = org.apache.velocity.runtime.log.AvalonLogSystem,org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+runtime.log.logsystem.class = org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+#---------------------------------------------------------------------------
+# This is the location of the Velocity Runtime log.
+#----------------------------------------------------------------------------
+
+runtime.log = ../logs/velocity.log
+
+#----------------------------------------------------------------------------
+# This controls if Runtime.error(), info() and warn() messages include the
+# whole stack trace. The last property controls whether invalid references
+# are logged.
+#----------------------------------------------------------------------------
+
+runtime.log.error.stacktrace = true
+runtime.log.warn.stacktrace = false
+runtime.log.info.stacktrace = false
+runtime.log.invalid.reference = true
+
+#----------------------------------------------------------------------------
+# Configuration for the Log4JLogSystem.
+# You must define the runtime.log.logsystem.class property to be:
+#   org.apache.velocity.runtime.log.Log4JLogSystem
+#
+# You must also include Log4J's .jar files into your classpath. They are
+# included with the Velocity distribution in the build/lib directory.
+#
+# There are several different options that you can configure.
+# Uncomment the ones that you want and also define their settings.
+#----------------------------------------------------------------------------
+#runtime.log.logsystem.log4j.pattern=%d - %m%n
+#runtime.log.logsystem.log4j.file.size=100000
+#runtime.log.logsystem.log4j.file.backups=1
+#runtime.log.logsystem.log4j.syslogd.host=my.syslog.server.com
+#runtime.log.logsystem.log4j.syslogd.facility=LOG_DAEMON
+#runtime.log.logsystem.log4j.remote.host=my.remote.server.com
+#runtime.log.logsystem.log4j.remote.port=1099
+#runtime.log.logsystem.log4j.email.server=localhost
+#runtime.log.logsystem.log4j.email.from=root@localhost
+#runtime.log.logsystem.log4j.email.to=root@localhost
+#runtime.log.logsystem.log4j.email.subject=Velocity Error Report
+#runtime.log.logsystem.log4j.email.buffer.size=512
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  E N C O D I N G
+#----------------------------------------------------------------------------
+
+input.encoding=ISO-8859-1
+output.encoding=ISO-8859-1
+
+#----------------------------------------------------------------------------
+# F O R E A C H  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These properties control how the counter is accessed in the #foreach
+# directive. By default the reference $velocityCount will be available
+# in the body of the #foreach directive. The default starting value
+# for this reference is 1.
+#----------------------------------------------------------------------------
+
+directive.foreach.counter.name = velocityCount
+directive.foreach.counter.initial.value = 1
+
+#----------------------------------------------------------------------------
+# I N C L U D E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These are the properties that governed the way #include'd content
+# is governed.
+#----------------------------------------------------------------------------
+
+directive.include.output.errormsg.start = <!-- include error :
+directive.include.output.errormsg.end   =  see error log -->
+
+#----------------------------------------------------------------------------
+# P A R S E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+
+directive.parse.max.depth = 10
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  L O A D E R S
+#----------------------------------------------------------------------------
+#
+#
+#----------------------------------------------------------------------------
+
+# resource.loader = file
+# 
+# file.resource.loader.description = Velocity File Resource Loader
+# file.resource.loader.class = org.apache.velocity.runtime.resource.loader.FileResourceLoader
+# file.resource.loader.path = .
+# file.resource.loader.cache = false
+# file.resource.loader.modificationCheckInterval = 2
+resource.loader = class
+class.resource.loader.description = Velocity Class Resource Loader
+class.resource.loader.class = org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+
+#----------------------------------------------------------------------------
+# VELOCIMACRO PROPERTIES
+#----------------------------------------------------------------------------
+# global : name of default global library.  It is expected to be in the regular
+# template path.  You may remove it (either the file or this property) if
+# you wish with no harm.
+#----------------------------------------------------------------------------
+velocimacro.library = VM_global_library.vm
+
+velocimacro.permissions.allow.inline = true
+velocimacro.permissions.allow.inline.to.replace.global = false
+velocimacro.permissions.allow.inline.local.scope = false
+
+velocimacro.context.localscope = false
+
+#----------------------------------------------------------------------------
+# INTERPOLATION
+#----------------------------------------------------------------------------
+# turn off and on interpolation of references and directives in string
+# literals.  ON by default :)
+#----------------------------------------------------------------------------
+runtime.interpolate.string.literals = true
+
+
+#----------------------------------------------------------------------------
+# RESOURCE MANAGEMENT
+#----------------------------------------------------------------------------
+# Allows alternative ResourceManager and ResourceCache implementations
+# to be plugged in.
+#----------------------------------------------------------------------------
+resource.manager.class = org.apache.velocity.runtime.resource.ResourceManagerImpl
+resource.manager.cache.class = org.apache.velocity.runtime.resource.ResourceCacheImpl
+
+#----------------------------------------------------------------------------
+# PLUGGABLE INTROSPECTOR
+#----------------------------------------------------------------------------
+# Allows alternative introspection and all that can of worms brings
+#----------------------------------------------------------------------------
+
+runtime.introspector.uberspect = org.apache.velocity.util.introspection.UberspectImpl
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/README
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/README	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/README	(revision 21112)
@@ -0,0 +1,82 @@
+Vishnu package for ...
+
+If you are running this as part of Greenstone, just installing Greenstone will install Vishnu.
+
+If you want to run it standalone:
+
+You need to have Java 1.4 or higher and Ant installed. The environment variable JAVA_HOME needs to be set. 
+You need to have MG installed - this should be the version from Greenstone, and you need the mg.jar file in your CLASSPATH and the libmgjni.so in your LD_LIBRARY_PATH.
+You need the lucene 1.4 or higher jar file in your CLASSPATH.
+Other jar files needed in CLASSPATH: xercesImpl.jar, servlet-api.jar.
+These jar files all come as part of the Greenstone download. Some in gsdl3/lib/java, and servlet.jar in gsdl3/comms/jakarta/tomcat/lib (when unpacked).
+
+Edit the build.xml file and change the prefix property to be the current directory.
+
+Then run 'ant' (or "ant all').
+
+
+COLLECTION BUILDING:
+
+Make a directory called collections inside the main vishnu directory. Inside this, create a new directory for your collection. The name of this directory is the collection identifier. Inside the collection directory create a new direcotry called import. Inside here put all the documents that you want in your collection. Documents must be plain text, xml or html.
+
+Then you can build the collection:
+
+Command line: 
+java vishnu.builder.CollectionBuilder -collsHome (directory where collections live: path-to-vishnu/collections) -collection (directory name of collection) -name (descriptive name for collection) -description (collection description) -index ("mg" or "luc")
+
+If you want to use views, add the option "-use-views" to the above command. This will create a separate index for each sub-directory in the import folder, and you can search each sub-view independently.
+
+Using a GUI tool:
+java vishnu.builder.IndexerGUI
+
+Fill in the fields as appropriate, and click "Start indexing".
+
+Once the collection is built, you need to rename the building directory to index. Collections are built into the building directory, but served from the index directory. This allows you to serve a built collection while rebuilding.
+
+For collections in the Greenstone 3 context, you will need to create the files etc/collectionConfig.xml and index/buildConfig.xml.
+These should look like the following:
+
+etc/collectionConfig.xml:
+<collectionConfig>
+  <metadataList>
+    <metadata name="creator">creator's email address</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Collection name</displayItem>
+    <displayItem name='description' lang='en'>Collection description
+    </displayItem>
+  </displayItemList>
+</collectionConfig>
+
+index/buildConfig.xml:
+<buildConfig>
+  <metadataList/>
+  <serviceRackList>
+    <serviceRack name="Visualizer">
+      <engineType name="MG"/> <!-- or LUCENE, depending on how you built the collection -->
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
+
+Note that you cannot use views in the Greenstone context yet.
+
+RUNNING THE SERVLET:
+
+
+If you are running this standalone, you need to run a servlet container for VisServlet.
+We have provided an example configuration for use with Tomcat (version 4.1).
+
+The vishnu/web directory is setup to be the context directory for vishnu in Tomcat. 
+You need to edit Tomcat's conf/server.xml file, and add in the context for vishnu. Add the following element after the tomcat root context element:
+
+<Context path="/vishnu" docBase="full-path-to-vishnu/web" debug="1" reloadable="true"><Resources allowLinking='true'/></Context>
+(replacing "full path-to-vishnu" with the path to the vishnu directory.)
+Edit the vishnu/web/WEB-INF/web.xml file and put in the correct path to your collections directory.
+
+The vishnu.jar file needs to be in your CLASSPATH, and make sure that Tomcat is using your CLASSPATH.
+
+RUNNING THE APPLET:
+
+The file vishnu/web/index.html is an introductory page about vishnu, and contains the applet. Make sure that the library parameter has the correct address for your visualiser servlet. The default is http://localhost:8080/vishnu/server. This will be correct if you have installed Tomcat to use port 8080, and have set the vishnu servlet up as described above. But note, if you are using Mozilla, it may not like localhost here, so if you have problems, try putting the full name of the server running Tomcat in place of localhost.
+
+If you are running Tomcat as described above, you can view this index file at http://localhost:8080/vishnu/index.html
Index: /main/tags/3.01/greenstone3-extensions/vishnu/build.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/build.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/build.xml	(revision 21112)
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+
+<!-- ====================================================================== 
+     April 2005                                                    
+
+     Vishnu build script
+                   
+     kjdon                                                                
+     ====================================================================== -->
+<project name="vishnu" default="all" basedir=".">
+
+  <property environment="env"/>
+  <property name="build.dir" value="${basedir}/build"/>
+  <property name="src.dir" value="${basedir}/src"/>
+
+  <!-- if vishnu is installed as a package inside greenstone -->
+  <property name="prefix" value="../.."/>
+  <!-- use if vishnu is standalone -->
+  <property name="prefix" value="${basedir}"/>
+  <property name="catalina.home" value="${env.CATALINA_HOME}"/>
+  <property name="web.lib" value="${prefix}/web/WEB-INF/lib"/>
+  <property name="web.classes" value="${prefix}/web/WEB-INF/classes"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="true"/>
+  <property name="compile.optimize"    value="true"/>
+
+  <!-- classpath : this is for vishnu with Greenstone. For standalone vishnu,
+  make sure all the specified jar files are on the environment CLASSPATH -->
+  <path id="compile.classpath">
+    <pathelement location="${prefix}/lib/jni/mg.jar"/>
+    <pathelement location="${web.lib}/lucene-1.4.1.jar"/>
+    <pathelement location="${catalina.home}/common/lib/servlet-api.jar"/>
+    <pathelement location="${catalina.home}/common/endorsed/xercesImpl.jar"/>
+    <pathelement path="${env.CLASSPATH}"/>
+    <!-- if not usign this with greenstone, don't need this -->
+    <pathelement location="${web.lib}/gsdl3.jar"/>
+  </path>
+
+  <path id="ld.library.path">
+    <pathelement location="${prefix}/lib/jni"/>
+  </path>
+  <target name="all" depends="clean,compile"/>
+
+  <target name="clean" depends=""
+    description="Remove the compiled code">
+    <delete dir="${build.dir}"/>
+  </target>
+  
+  <target name="compile" depends=""
+    description="Compile the code">
+    <mkdir dir="${build.dir}"/>
+    <javac srcdir="${src.dir}"
+      destdir="${build.dir}"
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath>
+	<path refid="compile.classpath"/>
+      </classpath>
+      <!-- uncomment this if not in a greenstone context -->
+      <!--<exclude name="org/**"/>-->
+    </javac>
+    <jar destfile="${build.dir}/vishnu.jar">
+      <fileset dir="${build.dir}">
+	<include name="vishnu/**"/>
+	<include name="org/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>      
+    <copy file="${build.dir}/vishnu.jar" todir="${web.lib}"/>
+  </target>
+</project>
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 21112)
@@ -0,0 +1,295 @@
+/*
+ *    Visualiser.java
+ *    Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Text; 
+import java.util.Vector;
+import java.util.HashMap;
+import java.io.File;
+import java.io.*;
+import vishnu.server.*;
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.Base64;
+
+public class Visualizer
+    extends ServiceRack {
+    
+    // the services on offer
+    private static final String VIS_SERVICE = "VisApplet";
+
+    // other internal strings
+    private static final String ENGINE_TYPE_ELEM = "engineType";
+    private static final String LUCENE_ENGINE = "LUCENE";
+    private static final String MG_ENGINE = "MG";
+
+    private Element applet_description = null;
+    
+    private String engine_type = LUCENE_ENGINE; // lucene is default
+    private String collection_home = null;
+    private SearchInterface engine = null;
+    public Visualizer () {
+
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+	e.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+	short_service_info.appendChild(e);
+
+	applet_description = this.doc.createElement(GSXML.SERVICE_ELEM);
+        applet_description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+        applet_description.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+
+
+	String app_info = "<"+GSXML.APPLET_ELEM+" CODEBASE='lib' CODE='vishnu.testvis.visual.VishnuSingle.class' ARCHIVE='vishnu.jar,gsdl3.jar,xercesImpl.jar,xml-apis.jar' WIDTH='1000' HEIGHT='800'>";
+	app_info += "<PARAM NAME='library' VALUE=''/>"; // filled in by receptionist
+	app_info += "<PARAM NAME='viscgi' VALUE='?";
+	app_info += GSParams.ACTION +"=a&amp;"+GSParams.REQUEST_TYPE +"=r&amp;"+GSParams.SERVICE+"="+VIS_SERVICE+"&amp;"+GSParams.OUTPUT+"=xml&amp;"+GSParams.RESPONSE_ONLY+"=1'/>";
+	app_info += "<PARAM NAME='collection'   VALUE='" + this.cluster_name + "'/>";
+	app_info += "<PARAM NAME='engine'   VALUE='GSDLEngine' />";
+	// add view info if appropriate
+	app_info += "The visualization applet.</"+GSXML.APPLET_ELEM+">";
+	   
+	Document dom = converter.getDOM(app_info);
+	if (dom == null) {
+	    System.err.println("Visualizer.configure Error: Couldn't parse applet info");
+	    return false;
+	}
+	Element app_elem = dom.getDocumentElement();
+	applet_description.appendChild(this.doc.importNode(app_elem, true));
+
+	// get engine type from config file
+	Element engine_elem = (Element)GSXML.getChildByTagName(info, ENGINE_TYPE_ELEM);
+	if (engine_elem != null) {
+	    engine_type = engine_elem.getAttribute(GSXML.NAME_ATT);
+	    if (engine_type.equals("")) {
+		engine_type = LUCENE_ENGINE;
+	    }
+	}
+	
+	collection_home = this.site_home + File.separator + "collect"+ File.separator + this.cluster_name + File.separator;
+	if (engine_type.equals(LUCENE_ENGINE)) {
+	    // make the full path the arg to MGSearcher??
+	    engine = new LUCSearcher(collection_home, null);
+	} else if (engine_type.equals(MG_ENGINE)) {
+	    engine = new MGSearcher(collection_home, this.cluster_name, null);
+	} else {
+	    System.err.println("Visualiser: invalid engine type: "+engine_type);
+	    return false;
+	}
+	return true;
+    }	
+    
+    
+    
+    protected Element getServiceDescription(String service, String lang, String subset) {
+	if (!service.equals(VIS_SERVICE)) {
+	    return null;
+	}
+	Element describe = (Element) applet_description.cloneNode(true);
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  
+							    getTextString(VIS_SERVICE+".name", lang)));
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  
+							    getTextString(VIS_SERVICE+".description", lang)));
+	return describe;
+    }
+    
+    protected Element processVisApplet(Element request)
+    {
+
+	Element param_elem = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_elem, false);
+
+	String type = (String)params.get("type");
+
+	// the result element
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, VIS_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// applet result info must be in appletInfo element
+	Element applet_data = this.doc.createElement(GSXML.APPLET_DATA_ELEM);
+	result.appendChild(applet_data);
+	Element vis_data = this.doc.createElement("visData");
+	applet_data.appendChild(vis_data);
+
+	String results = "";
+	if (type.equals("search")) {
+
+	    //Element cluster_results = this.doc.createElement("cluster");
+	    //vis_data.appendChild(cluster_results);
+	    //Element description_results = this.doc.createElement("descriptions");
+	    //vis_data.appendChild(description_results);
+
+	    String query = (String)params.get("q");
+	    System.err.println("the query was "+query);
+	    
+	    Vector doc_nums = new Vector();
+	    Vector descriptions = new Vector();
+	    
+	    engine.search(query);
+	    doc_nums = engine.getDocIdentifiers();
+	    descriptions = engine.getDocDescriptions();
+	    
+	    DataBlock db = generateDataBlock(collection_home, null, doc_nums, descriptions);
+	    System.err.println("got back data, now converting to string");
+	    try {
+		results = Base64.encodeObject(db);
+	    } catch (Exception e) {
+		System.err.println("trying to base64 encode the datablock, but exception happened: "+e);
+	    }
+	    System.err.println("after converting to string");
+// 	    // serialise the data block
+// 	    try {
+// 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+// 		ObjectOutputStream oos = new ObjectOutputStream(baos);
+// 		oos.writeObject(db);
+// 		oos.close();
+		
+// 		//results = baos.toString("UTF-8");
+// 		results = Base64.encode(baos.toByteArray());
+// 	    } catch (Exception e) {
+// 		System.err.println("Visualizer serialise data block error: "+e);
+// 	    }
+	    //System.err.println("num docs = "+doc_nums.size());
+	    //Cluster c = new Cluster();
+	    //results = c.getCluster(collection_home,doc_nums);
+	    //results.trim(); 
+	    //System.err.println("results = "+results);
+
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+
+// 	    for (int i=0; i<descriptions.size(); i++) {
+// 		String d = (String)descriptions.get(i);
+// 		Element de = GSXML.createTextElement(this.doc, "desc", d);
+// 		description_results.appendChild(de);
+// 	    }
+	    System.err.println("end of search");
+	} else if (type.equals("fetch")) {
+	    String docNum = (String)params.get("d");
+	
+	    Vector doc = engine.getDocContent(Integer.parseInt(docNum));
+  	    for (int i=0; i<doc.size(); i++) {
+		results += (String)doc.get(i) +"\n";
+	    }
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+	} else {
+	    System.err.println("invalid type sent to Visualiser process: "+type);
+	    return result;
+	}
+	System.err.println("returning result");
+	return result;
+    }
+
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.err.println("Num descriptions: " + descriptions.size());
+	    System.err.println("Num docs: " + docs.length);
+	    System.err.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    System.err.println("end of try in generatedatablock");
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	System.err.println("returning data");
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CKIndexer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CKIndexer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CKIndexer.java	(revision 21112)
@@ -0,0 +1,395 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import java.util.Hashtable;
+import vishnu.util.*;
+
+public class CKIndexer extends Indexer
+{
+
+    static private String delimiter = "()<>.,?!'\" "; 
+
+
+    private static void usage()
+    {
+        System.out.println("java Indexer/CKIndexer \n -i InputDirectory [$COLL_HOME/import/]) \n -o OutputDirectory [$COLL_HOME/index/ck_index/java] \n -r root directory of collection \n -list (name of file listing the docs to be indexed)[docs.lst]\n -d string containing chars to be eliminated from words\n");
+	System.exit(0);
+    }
+    
+    
+    public static void main(String[] args)
+    {
+        String in = null;
+        String out = null;
+        //String COLL_HOME = null;
+	//String docList = null;
+	
+        if( args.length == 0 ){
+            usage();
+            System.exit(0);
+        }
+	
+        int a = 0;
+        while( a < args.length ){
+            if( args[a].equals("-i") )
+                in = args[++a];
+            else if( args[a].equals("-o") )
+                out = args[++a];
+            //else if( args[a].equals("-r") )
+            //    COLL_HOME = args[++a];
+	    //else if( args[a].equals("-list") )
+	    //	docList = args[++a];
+	    else if( args[a].equals("-d") )
+		delimiter = args[++a];
+            a++;
+        }
+	
+	CKIndexer cki = new CKIndexer();
+
+	if (out==null || in==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	//if( COLL_HOME == null ) usage();
+	//cki.setRootDirectory(COLL_HOME);
+	
+	//if( out == null ) out = COLL_HOME + "/index/ck_index/java/";
+	cki.setOutputDirectory(out);
+	
+	//if( in == null ) in = COLL_HOME + "/import/";
+	cki.setInputDirectory(in);
+	
+	//if( docList != null ) cki.setDocList(docList);
+	
+	cki.startIndexing();
+    }
+    
+    
+    public CKIndexer(){
+	super("CKIndexer");
+    }
+    
+    
+//     public void setDocList(String doclist)
+//     {
+// 	docList = doclist;
+//     }
+
+
+    public void startIndexing(){
+	
+	initialize();
+	/**** read in archive ****/
+	
+	Vector docs = getDocList(inputDir); //readDocList(docList);
+	
+	System.out.println(docs.size() + " documents read");
+	
+	/****  hash of ALL words of corpus + their document frequency ****/
+	
+	Hashtable corpus_hash = new Hashtable();
+	
+	
+	/**** readDocuments and populate hash ****/
+
+	System.out.println("First run: getting document frequencies of all words");
+	 
+	getDocFreqs(docs, corpus_hash);
+	
+	
+	/**** prune words to get candidate keywords ****/
+
+	int min = 3;
+	int max = (int)(0.3*docs.size());
+	int minLength = 4;
+	
+	int num_ck = pruneKeywords(corpus_hash, minLength, min, max);
+	
+	
+	/**** sort hashtable by value ****/
+	
+	Enumeration enumer = corpus_hash.keys();
+	String[] words_ck = new String[num_ck];
+	int[] freqs_ck = new int[num_ck];
+	
+	int count = 0;
+	while( enumer.hasMoreElements() ){
+	    words_ck[count] = (String)enumer.nextElement();
+	    freqs_ck[count] = -((Integer)corpus_hash.get(words_ck[count])).intValue();
+	    count++;
+	}
+	
+	Misc.quickSort(freqs_ck,words_ck);
+	
+	
+	/**** normalize document frequencies ****/
+	
+	float[] freqs_f = new float[num_ck];
+	
+	for( int i = 0; i < num_ck; i++ ){
+	    freqs_f[i] = (float)(-1*freqs_ck[i])/docs.size();
+	}
+	
+	
+	/**** save document frequencies and candidate keywords ****/
+	
+	printToFile(freqs_f,outputDir + "doc_freq");
+	printToFile(words_ck,outputDir + "ck_words");	
+	
+	
+	/**** create (ck_word, index hash) ****/
+	
+	Hashtable ck_hash = new Hashtable();
+	
+	for( int i = 0; i < words_ck.length; i++ ){
+	    ck_hash.put(words_ck[i],new Integer(i));
+	}
+	
+	
+	/**** read documents and store for each document the ck words it contains and their frequencies ****/
+
+	System.out.println("Second run: getting term frequencies of candidate keywords");
+	
+	Vector termFreq = getTermFreqs(docs,ck_hash);
+	
+	
+	/**** save term frequency vector ****/ 
+	
+	printToFile(termFreq,outputDir + "term_freq");
+    }
+    
+    
+    private int pruneKeywords(Hashtable corpus_hash, int minLength, int min, int max){
+	Set ck_set = corpus_hash.keySet();
+	Enumeration enumer = corpus_hash.keys();
+	
+	int count = 0;
+	int pcount = 0;
+      	
+	while( enumer.hasMoreElements() ) {
+	    
+	    String word = (String)enumer.nextElement();
+	    int freq = ((Integer)corpus_hash.get(word)).intValue();
+	    
+	    
+	    /**** if word is a number remove ****/
+	    
+	    try{
+		int i = Integer.parseInt(word);
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }catch(NumberFormatException ex){}
+
+
+	    /**** if word too rare or too freq remove ****/
+	    
+	    if( word.length() < minLength || freq < min || freq > max ){
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }
+	}
+	return corpus_hash.size();
+    }
+    
+    
+    private Vector getTermFreqs(Vector docs, Hashtable ck_hash)
+    {
+	Vector termFreqVector = new Vector();
+	
+	for( int i = 0; i < docs.size(); i++ ){
+
+	    Hashtable doc_ck_hash = new Hashtable();
+	    String doc = (String)docs.elementAt(i);
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			Integer index = null;
+			if( (index = (Integer)ck_hash.get(word)) != null ){
+			    Integer count;
+			    
+			    if( (count = (Integer)doc_ck_hash.get(index)) != null ){
+				int newCount = count.intValue() + 1;
+				doc_ck_hash.put(index,new Integer(newCount));
+			    }else{
+				doc_ck_hash.put(index,new Integer(1));				
+			    }
+			}
+		    }
+		}
+		dis.close();
+	    }catch(Exception ex){ex.printStackTrace();}
+	
+	
+	    
+	    termFreqVector.addElement(doc_ck_hash);
+	}
+	
+	return termFreqVector;
+    }
+    
+
+    private void getDocFreqs(Vector docs, Hashtable corpus_hash)
+    {
+	for( int i = 0; i < docs.size(); i++ ){
+	    
+	    String doc = (String)docs.elementAt(i);
+	    
+	    Hashtable doc_hash = new Hashtable();
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			
+			
+			if( (Object)doc_hash.get(word) == null ){
+			    
+			    
+			    /**** make a note that there is at least one occurrence ****/
+			    
+			    doc_hash.put(word,word);
+			    
+			    
+			    /**** add it to corpus_hash if necessary or increment the counter ****/
+			    
+			    Integer count;
+			    if( (count = (Integer)corpus_hash.get(word)) == null ){
+				
+				
+				/**** add it to corpus_hash ****/
+				
+				corpus_hash.put(word,new Integer(1));
+			    }
+			    else{
+				
+				/**** increment counter in doc_freq ****/
+				
+				Integer newCount = new Integer(count.intValue()+1);
+				corpus_hash.put(word,newCount);
+			    }
+			}
+		    }
+		}
+		dis.close();
+		
+	    } catch(IOException ex){ex.printStackTrace();}
+	}
+    }
+    
+    private Vector getDocList(String input_dir_name) {
+
+	File input_dir = new File(input_dir_name);
+	if (!input_dir.exists()) {
+	    System.err.println("CKIndexer Error: input directory "+input_dir_name+ " does not exist!");
+	    return null;
+	}
+
+	Vector file_vector = new Vector();
+	processDirectory(input_dir, file_vector);
+	return file_vector;
+    }
+    private void processDirectory(File dir, Vector file_list) {
+	File [] files = dir.listFiles();
+	for (int i=0; i<files.length; i++) {
+	    if (files[i].isDirectory()) {
+		processDirectory(files[i], file_list);
+	    } else {
+		file_list.add(files[i].getPath());
+	    }
+	}
+
+    }
+//     private Vector readDocList(String list)
+//     {
+// 	list = inputDir + "/" + "docs.lst";
+	
+// 	File docList = new File(list);
+	
+// 	if (!docList.exists()){
+// 	    System.out.println("No documents found");
+// 	    System.exit(0);
+// 	}
+	
+// 	Vector docs = new Vector();
+	
+// 	try{
+//             BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+// 	    String line = "";
+//             while( (line=dis.readLine())!=null ){
+// 		docs.addElement(new String(inputDir+"/" + line));
+// 	    }
+//             dis.close();
+// 	} catch(IOException ex){ex.printStackTrace();}
+	
+// 	return docs;
+//     }
+    
+    
+    private void printToFile(Object obj, String name){
+	
+	FileOutputStream fos = null;
+	ObjectOutputStream out = null;
+	
+	try{
+	    fos = new FileOutputStream(name);
+	    out = new ObjectOutputStream(fos);
+	    out.writeObject(obj);
+	    out.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+    }
+}
+
+
+	/*Hgram hg = new Hgram();
+	hg.setBinNumber(50);
+	hg.setLabel("Document frequency");
+	hg.setData(freqs_f,Hgram.RAW_DATA);
+	hg.update();*/
+
+ /*    private Map.Entry[] sort(Hashtable h)
+    {
+	Set set = h.entrySet();
+	Map.Entry [] entries = 
+	    (Map.Entry[])set.toArray(
+				     new Map.Entry[set.size()]);
+	Arrays.sort(entries, new Comparator() {
+		public int compare(Object o1, Object o2) {
+		    int i1 = ((Integer)((Map.Entry)o1).getValue()).intValue();
+		    int i2 = ((Integer)((Map.Entry)o2).getValue()).intValue();
+		    if( i1 < i2 ) return 0;
+		    else return 1;
+		}
+		public boolean equals(Object that) {
+		    return this==that;
+		}
+	    });
+	return entries;
+	}*/
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CollectionBuilder.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 21112)
@@ -0,0 +1,252 @@
+package vishnu.builder;
+
+import java.util.*;
+import java.io.*;
+
+public class CollectionBuilder
+{
+    //String site;
+    String collID = null; // directory name
+    String collHome = null; // full path to collections directory
+    String collName = null; // descriptive name
+    String collDesc = null; // descriptive description :-)
+    String indexer = null; // mg or lucene
+    boolean use_views = false;
+    long numDocs=0;
+    Vector views = null;
+    private static void usage()
+    {
+	System.out.println("java vishnu.builder.CollectionBuilder\n -collsHome (directory where collections live) \n -collection (directory name of collection)\n -name (descriptive name for collection) \n -description (collection description)\n -index (\"mg\" or \"luc\")\n [-use_views ]");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	String collID = null;
+	String collsHome = null;
+	String collName = "";
+	String collDesc = "";
+	String indexer = null;
+	boolean use_views = false;
+	int a = 0;
+	while( a < args.length ){
+	    if (args[a].equals("-collsHome"))
+		collsHome = args[++a];
+	    else if (args[a].equals("-collection"))
+		collID = args[++a];
+	    else if( args[a].equals("-name") )
+		collName = args[++a];
+	    else if( args[a].equals("-description") )
+		collDesc = args[++a];
+	    else if( args[a].equals("-index") )
+		indexer = args[++a];
+	    else if (args[a].equals("-use_views")) 
+		use_views = true;
+	    a++;
+	}
+	
+	if (collsHome==null || collID==null || indexer==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	if (collName.equals("")) {
+	    collName = collID;
+	}
+	if( !indexer.equals("mg") && !indexer.equals("luc") )
+	    indexer = "luc";
+	
+	if (!collsHome.endsWith(File.separator)) {
+	    collsHome += File.separator;
+	}
+	String coll_home = collsHome + collID + File.separator;
+	
+	CollectionBuilder cb = new CollectionBuilder(coll_home, collID, collName, collDesc, indexer, use_views);
+    }
+    
+    
+    
+    public CollectionBuilder(String coll_home, String coll_id, String coll_name,String coll_desc,  String ind, boolean use_views)
+    {
+	this.collHome = coll_home;
+	this.collID = coll_id;
+	this.collName = coll_name;
+	this.collDesc = coll_desc;
+	this.indexer = ind;
+	this.use_views = use_views;
+	
+	if (use_views) {
+	    views = new Vector();
+	}
+	if (!setupCollection()) {
+	    System.err.println("Couldn't setup the collection. Quitting build");
+	    return;
+	}
+	
+	/**** get text annotation ****/
+
+	FTIndexing();
+
+
+	/**** extract keywords ****/
+
+	CKIndexing();
+
+	/**** write out the config file ****/
+	printConfig();
+    }
+
+
+
+    private boolean setupCollection()
+    {
+	// check the import directory
+	File import_dir = new File(collHome + "import");
+	if (!import_dir.isDirectory()) {
+	    System.err.println("the import directory "+import_dir.getPath()+" doesn't exist. Quitting build");
+	    return false;
+	}
+	
+	
+	// create the building directory
+	try {
+	    File index_dir = new File(collHome + "building");
+	    if (!index_dir.exists()) {
+		index_dir.mkdir();
+	    } else if (!index_dir.isDirectory()) {
+		// index is present but is not a directory
+		System.err.println("Trying to create building dir but a file already exists at "+index_dir.getPath());
+		return false;
+	    }
+	} catch (Exception e) {
+	    System.err.println("CollectionBuilder.setupCollection Error: "+e);
+	    return false;
+	}
+
+	// create the view list
+	if (use_views) {
+	    // create a new view for each sub directory in import
+	    File [] sub_dirs = import_dir.listFiles();
+	    for (int i=0; i<sub_dirs.length; i++) {
+		if (sub_dirs[i].isDirectory()) {
+		    views.add(sub_dirs[i].getName());
+		}
+	    }
+	    if (views.size()==0) {
+		// we haven't found any subdirectories
+		System.err.println("no sub directories were found in the import dorectory ("+import_dir.getPath()+"), not using views");
+		use_views = false;
+
+	    }
+	}
+	return true;
+    }
+
+    private void printConfig() {
+
+	String index_type="";
+	if (indexer.equals("luc")) {
+	    index_type = "LUCENE";
+	} else if (indexer.equals("mg")) {
+	    index_type = "MG";
+	}
+	
+	try{
+	    File target = new File(collHome + "config.xml");
+	    PrintWriter out = new PrintWriter(new FileWriter(target),true);
+	
+	    StringBuffer buffer = new StringBuffer();
+	    buffer.append("<collection name=\"");
+	    buffer.append(collName);
+	    buffer.append("\" description=\"");
+	    buffer.append(collDesc);
+	    buffer.append("\" engine=\"");
+	    buffer.append(index_type);
+	    buffer.append("\" size=\"");
+	    buffer.append(String.valueOf(numDocs));
+	    buffer.append("\">\n");
+	    
+	    // add in views
+	    if (use_views) {
+		buffer.append("<viewList>\n");
+		for (int i=0; i<views.size(); i++) {
+		    buffer.append("<view name=\"");
+		    buffer.append(views.get(i));
+		    buffer.append("\"/>");
+		}
+		buffer.append("\n</viewList>\n");
+	    } 
+	    buffer.append("</collection>\n");
+		
+	    out.print(buffer.toString());
+	    out.close();
+
+	} catch (IOException e){ System.out.println("Error writing config file: " + e.toString());
+	}
+    }
+    
+
+    private void CKIndexing()
+    {
+	CKIndexer cki = new CKIndexer();
+	cki.setCollectionName(collID);
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator+"ck_index"+File.separator;
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("calculating candidate keywords for view "+view);
+		cki.setOutputDirectory(out+view+File.separator);
+		cki.setInputDirectory(in+view+File.separator);
+		cki.startIndexing();
+	    }
+	} else {
+	    System.out.println("calculating candidate keywords");
+	    // just do the import dir as a whole
+	    cki.setOutputDirectory(out);
+	    cki.setInputDirectory(in);
+	    cki.startIndexing();
+	}
+    }
+    
+
+
+    private void FTIndexing()
+    {
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator + indexer + "_index"+File.separator;
+	Indexer the_indexer;
+	if (indexer.equals("luc") ) {
+	    the_indexer = new LuceneWrapper();
+	} else {
+	    the_indexer = new MGWrapper();
+	}
+	the_indexer.setCollectionName(collID);
+
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("indexing the text for view "+view);
+		the_indexer.setOutputDirectory(out+view+File.separator);
+		the_indexer.setInputDirectory(in+view+File.separator);
+		the_indexer.startIndexing();
+	    }
+
+	} else {
+	    System.out.println("indexing the text");
+	    the_indexer.setOutputDirectory(out);
+ 	    the_indexer.setInputDirectory(in);
+ 	    the_indexer.startIndexing();
+	}
+
+    }
+}
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/FileDocument.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/FileDocument.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/FileDocument.java	(revision 21112)
@@ -0,0 +1,57 @@
+package vishnu.builder;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.FileInputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.DateField;
+
+/** A utility for making Lucene Documents from a File. */
+
+public class FileDocument {
+  /** Makes a document for a File.
+    <p>
+    The document has three fields:
+    <ul>
+    <li><code>path</code>--containing the pathname of the file, as a stored,
+    tokenized field;
+    <li><code>modified</code>--containing the last modified date of the file as
+    a keyword field as encoded by <a
+    href="lucene.document.DateField.html">DateField</a>; and
+    <li><code>contents</code>--containing the full contents of the file, as a
+    Reader field;
+    */
+  public static Document Document(File f, String relative_path)
+       throws java.io.FileNotFoundException {
+	 
+    // make a new, empty document
+    Document doc = new Document();
+
+    // Add the path of the file as a field named "path".  Use a Text field, so
+    // that the index stores the path, and so that the path is searchable
+    //doc.add(Field.Text("path", f.getPath()));
+    doc.add(Field.Text("path", relative_path));
+    // Add the last modified date of the file a field named "modified".  Use a
+    // Keyword field, so that it's searchable, but so that no attempt is made
+    // to tokenize the field into words.
+    doc.add(Field.Keyword("modified",
+			  DateField.timeToString(f.lastModified())));
+
+    // Add the contents of the file a field named "contents".  Use a Text
+    // field, specifying a Reader, so that the text of the file is tokenized.
+    // ?? why doesn't FileReader work here ??
+    FileInputStream is = new FileInputStream(f);
+    Reader reader = new BufferedReader(new InputStreamReader(is));
+    doc.add(Field.Text("contents", reader));
+
+    // return the document
+    return doc;
+  }
+
+  private FileDocument() {}
+}
+    
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/Indexer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/Indexer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/Indexer.java	(revision 21112)
@@ -0,0 +1,89 @@
+package vishnu.builder;
+
+import java.io.File;
+abstract public class Indexer
+{ 
+    
+    // views stuff
+//     public static final int DEFAULT = 0;
+//     public static final int DIP = 1;
+//     public static final int REG = 2;
+//     public static final int LEM = 3;
+//     public static final int SYN = 4;
+
+//     public static final String[] types = new String[]{"full","dip","reg","lem","syn"};
+    
+//     boolean use_views = false;
+//     int type = -1;
+    String name = null;
+    //String limiter = null;
+    String outputDir = null;
+    String inputDir = null;
+    String collName = null;
+
+    public Indexer(String n)
+    {
+	name = n;		
+    }
+    
+    
+//     public void setUseViews(boolean use_v) {
+// 	use_views = use_v;
+//     }
+    
+//     public void setIndexingType(int t)
+//     {
+// 	type = t;
+//     }    
+    public void setOutputDirectory(String s)
+    {
+	outputDir = s;
+    }
+    
+//     public void setLimiter(String s)
+//     {
+//         limiter = s;
+//     }
+    
+    public void setInputDirectory(String s)
+    {
+        inputDir = s;
+    }
+    
+     
+    public void setCollectionName(String c)
+    {
+	collName = c;
+    }
+
+    // this should be called at the start of startIndexing()
+    protected void initialize() {
+
+	if( inputDir == null || outputDir == null ){
+	    System.err.println("Input dir or output dir is null");
+	    System.exit(0);
+	}
+	
+	/* check if index dir exists */
+	File in = new File(inputDir);
+	if (!in.isDirectory()) {
+	    System.err.println("input directory ("+inputDir+") not a directory, exiting...");
+	    System.exit(0);
+	}
+
+	/**** check if output directory exists, if not create it ****/	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+	    try {
+		f.mkdirs();
+	    } catch (Exception e) {
+		System.err.println("Couldn't create output directory "+outputDir);
+		System.err.println(e);
+		System.exit(0);
+	    }
+	}
+	
+    }
+    
+    abstract public void startIndexing();
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/IndexerGUI.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 21112)
@@ -0,0 +1,126 @@
+package vishnu.builder;
+
+import java.awt.event.*;
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.io.File;
+import vishnu.util.SpringUtilities;
+
+public class IndexerGUI {
+    public final static boolean RIGHT_TO_LEFT = false;
+    public Toolkit toolkit = Toolkit.getDefaultToolkit();
+    private static JTextField[] textFields; 
+    private static JTextArea ja;
+
+    public static void addComponentsToPane(Container pane)
+    {
+	String [] labels = {"Collections directory: ", "Collection id (directory name): ", "Collection name: ","Collection description: ", "Indexer (\"mg\" or \"luc\"): ", "Use views (\"yes\" or \"no\"): "};
+        int numPairs = labels.length;
+        textFields = new JTextField[numPairs];
+	
+        pane.setLayout(new BoxLayout(pane,BoxLayout.Y_AXIS));
+	
+        JPanel p = new JPanel(new SpringLayout());
+	
+        for (int i = 0; i < numPairs; i++) {
+	    JLabel l = new JLabel(labels[i], JLabel.TRAILING);
+	    p.add(l);
+	    textFields[i] = new JTextField(30);
+	    l.setLabelFor(textFields[i]);
+	    p.add(textFields[i]);
+	}
+	
+	SpringUtilities.makeCompactGrid(p,numPairs, 2, 6, 6, //initX, initY
+                                6, 6);       //xPad, yPad
+	
+	pane.add(p);
+	
+	JPanel butPan = new JPanel();
+	butPan.setLayout(new BoxLayout(butPan,BoxLayout.X_AXIS));
+
+	Dimension minSize = new Dimension(10, 50);
+	Dimension prefSize = new Dimension(10, 50);
+	Dimension maxSize = new Dimension(Short.MAX_VALUE, 50);
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+	JButton button = new JButton(" Start indexing ");
+        button.setBorder(BorderFactory.createBevelBorder(0));
+	button.addActionListener(new ActionListener(){
+                public void actionPerformed(ActionEvent e){
+			build();
+                }
+            });
+        butPan.add(button);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+        butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	pane.add(butPan);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	ja = new JTextArea(5,5);
+        JScrollPane scrollingArea = new JScrollPane(ja);
+	scrollingArea.setBorder(BorderFactory.createBevelBorder(1));
+	pane.add(scrollingArea);
+     }
+
+    private static void build()
+    {
+	// get data
+	String colls_home = textFields[0].getText();
+	String coll_id = textFields[1].getText();
+	String coll_name = textFields[2].getText();
+	String coll_desc = textFields[3].getText();
+	String indexer = textFields[4].getText();
+	String use_views_str = textFields[5].getText();
+	String coll_home = colls_home;
+	if (!coll_home.endsWith(File.separator)) {
+	    coll_home += File.separator;
+	}
+	coll_home += coll_id + File.separator;
+
+	if (!indexer.equals("mg") || !indexer.equals("luc")) {
+	    indexer = "mg";
+	}
+	boolean use_views = false;
+	if (use_views_str.equals("yes")) {
+	    use_views = true;
+	}
+	String output = "Launching Collection Builder with \n collections home = "+colls_home + "\n collection id = "+coll_name+"\n indexer = "+indexer;
+	if (use_views) {
+	    output += "\n using views";
+	} else {
+	    output += "\n not using views";
+	}
+	ja.setText(output);
+
+	CollectionBuilder builder = new CollectionBuilder(coll_home, coll_id, coll_name, coll_desc, indexer, use_views); 
+	output += "\n Indexing Finished";
+	ja.setText(output);
+    }
+
+    private static void createAndShowGUI() {
+        JFrame.setDefaultLookAndFeelDecorated(true);
+        JFrame frame = new JFrame("Indexing Tool");
+        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+        addComponentsToPane(frame.getContentPane());
+
+        frame.pack();
+
+	Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+        int w = frame.getSize().width;
+        int h = frame.getSize().height;
+        int x = (dim.width-w)/2;
+        int y = (dim.height-h)/2;
+        frame.setLocation(x,y);
+
+	frame.setVisible(true);
+    }
+
+    public static void main(String[] args) {
+	createAndShowGUI();
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/LuceneWrapper.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 21112)
@@ -0,0 +1,162 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexWriter;
+
+
+public class LuceneWrapper extends Indexer
+{
+    
+    private static void usage()
+    {
+	System.out.println("java LucenceWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+
+	LuceneWrapper luc = new LuceneWrapper();
+	
+	out = out + "luc_index";
+	luc.setCollectionName(coll);
+        luc.setOutputDirectory(out);
+	luc.setInputDirectory(in);
+	luc.startIndexing();
+    }
+    
+    
+    public LuceneWrapper()
+    {
+	super("Lucene Indexer");
+    }
+    
+    public void startIndexing() {
+	initialize();
+	try {
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    // this is a bit of a hack putting import here
+	    indexFiles(writer, new File(inputDir), "import/");
+	    writer.optimize();
+	    writer.close();
+	} catch (Exception e) {
+	    System.err.println("Error creating the Lucene index: "+e);
+	    System.exit(0);
+	}
+
+    }
+
+    private void indexFiles(IndexWriter writer, File input_dir, String current_path) {
+	File [] doc_list = input_dir.listFiles();
+	for (int i=0; i<doc_list.length; i++) {
+	    File f = doc_list[i];
+	    if (f.isDirectory()) {
+		indexFiles(writer, f, current_path+f.getName()+File.separator);
+	    } else {
+		try {
+		    writer.addDocument(FileDocument.Document(f, current_path+f.getName()));	
+		} catch (Exception e) {
+		    System.err.println("Error indexing document "+current_path+f.getName()+": "+e);
+		}
+	    }
+	}
+
+    }
+
+	
+    public void startIndexingOld()
+    {
+	
+	if( inputDir == null || outputDir == null ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	
+	/**** check if output directory exists, if not create it ****/
+	
+	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+            System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/");
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }
+        }	
+	
+	File docList = new File(inputDir + "/docs.lst");
+	
+	if (!docList.exists()){
+	    System.out.println("No documents found");
+	    System.exit(0);
+	}
+	
+	Vector docs = new Vector();
+	
+	try{
+            BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+	    String line = "";
+            while( (line=dis.readLine())!=null ){
+		docs.addElement(new String(inputDir+"/" + line));
+	    }
+            dis.close();
+	} catch(IOException ex){ex.printStackTrace();}
+	
+	try{
+	    
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    
+	    for( int i = 0; i < docs.size(); i++ ){
+		File file = new File((String)docs.elementAt(i));		
+		//writer.addDocument(FileDocument.Document(file));
+	    }
+	    writer.optimize();
+	    writer.close();
+        }catch(Exception ex){ex.printStackTrace();}
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/MGWrapper.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/MGWrapper.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/builder/MGWrapper.java	(revision 21112)
@@ -0,0 +1,123 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+
+public class MGWrapper extends Indexer
+{
+    private static void usage()
+    {
+	System.out.println("java Indexer MGWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+	
+	out = out + "mg_index" + File.separator;
+	
+	MGWrapper mgw = new MGWrapper();
+	mgw.setCollectionName(coll);
+        mgw.setOutputDirectory(out);
+	mgw.setInputDirectory(in);
+	mgw.startIndexing();
+    }
+    
+    
+    public MGWrapper()
+    {
+	super("Managing Gigabytes Indexer");
+    }
+    
+    
+    public void startIndexing()
+    {
+
+	initialize();
+	// new mg requires a .mg_getrc to work
+	// lets put it in the output dir
+	File rc_file = new File(outputDir, ".mg_getrc");
+	if (!rc_file.exists()) {
+	    // if its there, assume it is correct
+	    try {
+		rc_file.createNewFile();
+		BufferedWriter rc_out = new BufferedWriter(new FileWriter(rc_file, true));
+		String data_line = collName+"\tDIR2\t"+inputDir;
+		rc_out.write(data_line, 0, data_line.length());
+		rc_out.newLine();
+		rc_out.flush();
+		rc_out.close();
+	    } catch (Exception ex) {
+		System.err.println(ex);
+		System.err.println("Unable to create file "+rc_file.getPath()+", cancelling build");
+		System.exit(0);
+	    }
+	}
+	/*
+	if( !f.exists() ){
+	    System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/"); 
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }	
+	}
+	*/
+	try{
+	    //String command = /*GSDL3_HOME +*/ "/packages/VIS/" + "MG/mg/mg-1.2.1/bin/mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    //String command = "mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    // requires .mg_getrc file
+	    String []env = new String [1];
+	    env[0] = "MG_GETRC="+outputDir+".mg_getrc";
+	    String command = "mgbuild -d " + outputDir + " " + collName;
+	    System.out.println("Executing.. " + command);
+	    Process proc = Runtime.getRuntime().exec(command, env);
+	    proc.waitFor();
+	    System.out.println(proc.exitValue());
+	}
+	catch(IOException ex){ex.printStackTrace();}
+	catch (InterruptedException ex){ System.out.println(ex); ex.printStackTrace();
+	} 
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/AveLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/AveLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/AveLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class AveLink extends LinkFunc
+{
+    float link(float a, float b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Cluster.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Cluster.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Cluster.java	(revision 21112)
@@ -0,0 +1,76 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int id;
+    public Cluster  parent = null;
+    public Cluster  child_1 = null;
+    public Cluster  child_2 = null;
+    
+    public double children_sim;
+    
+    // size of cluster
+    public int items;
+    
+    public int split;
+    
+    // [] -> centroid for each vector element ?
+    public float[] centroid=null;
+    
+    public Vector iter_items = null;
+    public int iter_items_num = 0;
+    
+    public int x = 0;
+    public int y = 0;
+    
+    public float[] getCentroid(){return centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster c1, Cluster c2, double children_sim, int items)
+    {
+	this.id               = id;
+	this.children_sim     = children_sim;
+	this.items            = items;
+	split                 = 0;
+	this.parent           = parent;
+        iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (c1 != null && c2 != null){
+	    if (c2.items > c1.items){
+		child_1 = c1;
+		child_2 = c2;
+	    }else{
+	        child_1 = c2;
+		child_2 = c1;
+	    }
+	}else{ // if either or both are null
+	    child_1 = c1;
+	    child_2 = c2;
+	}
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (id != -1)
+	    return 0;
+	else
+	    return Math.max(1 + child_1.depth(), 1 + child_2.depth());
+    }
+    
+    
+    void addChildrenToOutputVector(int []coding, Vector output)
+    {
+	if (id != -1){
+    	    Integer ii = new Integer(coding[id]);
+            output.addElement(ii);
+	}else{
+      	    child_1.addChildrenToOutputVector(coding,output);
+      	    child_2.addChildrenToOutputVector(coding,output);
+	}
+    }
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/ClusterArray.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(int num_of_clusters)
+    {
+	contents = new Vector(num_of_clusters);
+	
+	for(int i = 0; i < num_of_clusters; i++)
+	    contents.addElement(new Cluster(i, null, null, null, 0, 1));
+    }
+    
+    
+    // counts the number of clusters that are less dissimilar than disim
+    private void sweep_tree (Cluster cluster, double disim, int min_c_items, int[] clusters)
+    {
+	cluster.split = 0;
+	
+	if (cluster.id == -1){
+	    if (cluster.children_sim <= disim && cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }else{
+		sweep_tree (cluster.child_1, disim, min_c_items, clusters);
+		sweep_tree (cluster.child_2, disim, min_c_items, clusters);
+	    }
+    	}else{
+	    if (cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }
+	}
+    }
+
+
+    double split (Cluster cluster, double[] heights, int n, int target, int min_c_items, int[] clusters)
+    {
+	
+	/*
+	  Try to split 'cluster', made up of 'n' vectors, into 'target'
+	  clusters by incrementing iteratively the mimimum required
+	  similarity between the 'cluster\'s' children. The result is an
+	  array of clusters, 'cluster_ary' which contains 'clusters'
+	  clusters.  Obviously the final number of clusters may not be the
+	  same as the desired number of clusters. The function return the
+	  threshold similarity which resulted in the clusters being split
+	  in 'clusters' clusters.
+	*/
+	
+	int i;
+	
+	if (target < 2) target = 2;
+	
+	if (target > n) target = n;
+	
+	// height store minimum similarity values
+	// this is then the iterative incrementing loop
+	for (i = target - 2; i <= n - 1; i++){
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], min_c_items, clusters);
+	    
+	    if (clusters[0] >= target) break;
+	}
+	
+	// if target - 2 > n - 1, i.e. if the previous loop has not been 
+	// entered at all
+	if (i > n - 1){
+	    /* ignore the min_c_size requirement */
+	    i = target - 1;
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], 1, clusters);
+	}
+	
+	return heights[i];
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Clustering.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Clustering.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Clustering.java	(revision 21112)
@@ -0,0 +1,128 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+    public static final int BUCKSHOT = 0;
+    public static final int COMPLETE = 1;
+    
+    private Vector [] clusters;
+    private double [][] centroids;
+    
+    public Vector [] getClusters()
+    {
+        return clusters;
+    }
+
+    public double[][] getCentroids()
+    {
+        return centroids;
+    }
+    
+    
+    public Clustering (int linkage, float[][] KDMatrix, int min_c_items, int target_clusters, int limit)
+    {
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	
+        int wordCount = KDMatrix[0].length;
+	int hitDocs = KDMatrix.length;
+        
+	Cluster toplevel_cluster;
+        ClusterArray splitclusters = null;
+	int[] n_clusters = new int[]{0};
+	
+        double threshold;
+	
+        Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+
+        int[] coding = new int[hitDocs];
+        
+
+        for (int c = 0; c < hitDocs; c++) coding[c] = c;
+	
+	switch (linkage){
+	case COMPLETE_LINKAGE:
+	    linkage_func = new MaxLink();
+	    break;
+	case AVERAGE_LINKAGE:
+	    linkage_func = new AveLink();
+	    break;
+	case SINGLE_LINKAGE:
+	    linkage_func = new MinLink();
+	    break;
+	default:
+	    System.exit(0);
+	}
+	
+
+        /**** make sure upper bound does not exceed #docs ****/
+        
+	if(limit > hitDocs) limit = hitDocs;
+	limit = (int)hitDocs/2;
+	
+
+	/**** get similarity matrix of the first limit docs with each other ****/
+	
+        sim_matrix = new SimMatrix(KDMatrix, limit, sim_func);
+	
+	heights = new double[hitDocs];
+	
+        Hierarchical hierarchical = new Hierarchical(limit);
+        
+        toplevel_cluster = hierarchical.clustering(sim_matrix, limit, heights, linkage_func);
+	if( toplevel_cluster == null )
+	    return;
+	
+        System.out.println("Clustered items: " + toplevel_cluster.items);
+	Cluster c1 = toplevel_cluster.child_1;
+	Cluster c2 = toplevel_cluster.child_2;
+	System.out.println("Child1: " + c1.items);
+	System.out.println("Child2: " + c2.items);
+	
+	splitclusters = new ClusterArray(0);
+        
+	threshold = splitclusters.split(toplevel_cluster, heights, limit, target_clusters, min_c_items, n_clusters);
+	
+    	System.out.println(n_clusters[0] + " clusters");
+	System.out.println("Threshold: " + threshold);
+	
+        System.out.println("Target_clusters " + target_clusters);
+	System.out.println("Limit " + limit);
+	System.out.println("#docs: " + hitDocs);
+	
+        Iterative iter = new Iterative(wordCount);
+	
+	iter.clustering(splitclusters, n_clusters, KDMatrix, limit, hitDocs, sim_func);
+	
+	System.out.println("Finished iterative clustering");
+
+        clusters = new Vector[n_clusters[0]];
+        centroids = new double[n_clusters[0]][wordCount];
+
+	for (int i = 0; i < n_clusters[0]; i++){
+	    clusters[i]=new Vector();
+	    Cluster clust = (Cluster)splitclusters.contents.elementAt(i);
+	    clust.addChildrenToOutputVector(coding,clusters[i]);
+	    
+	    for (int j = 0; j < clust.iter_items_num; j++){
+		Integer ii = (Integer)clust.iter_items.elementAt(j);
+		Integer jj = new Integer (coding[ii.intValue()]);
+		clusters[i].addElement(jj);
+	    }
+	    
+	    float [] centroid = clust.getCentroid();
+	    
+	    for (int j = 0; j < wordCount; j++){
+		centroids[i][j]=centroid[j];
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Distance.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Distance.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Distance.java	(revision 21112)
@@ -0,0 +1,10 @@
+package vishnu.cluster;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract float get(float[] v1, float[] v2);
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 21112)
@@ -0,0 +1,15 @@
+package vishnu.cluster;
+
+public class EuclideanDistance extends Distance
+{
+    public float get(float[] v1, float[] v2)
+    {
+  	float sum = 0;
+
+	for(int c=0;c<v1.length;c++){
+      	    float x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return (float)Math.sqrt(sum);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Hierarchical.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 21112)
@@ -0,0 +1,234 @@
+package vishnu.cluster;
+
+public class Hierarchical
+{
+    public float[] minrow;
+    public int[] colind;
+    public TriangleIndex triangle;
+    
+    public Hierarchical(int matrix_rows){
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+    }
+    
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+	for (i = index_start + 1; i <= index_end; i++){
+	    if (m.matrix[i] < minrow[row]){
+		minrow[row] = m.matrix[i];
+		colind[row] = i;
+	    }
+	}
+    }
+    
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row){
+	int i;
+	int row;
+	float value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++){
+	    if (minrow[i] < value){
+		row = i;
+		value = minrow[i];
+	    }
+    	}
+	
+	return row;
+    }
+
+    
+    // num_of_rows is "limit" = max docs to be clustered in hierarchical clustering
+    Cluster clustering (SimMatrix sim_matrix, int num_of_rows, double[] heights, LinkFunc linkage_func)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+        int min_row, index, row, col, i;
+	double sim;
+	
+        int last_row_index, sim_matrix_size;
+	
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+	
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	cluster_ary = new ClusterArray (num_of_rows);
+	last_row_index = num_of_rows - 1;
+	
+	// minimum distance for each row of the sim_matrix
+	minrow = new float[num_of_rows];
+	
+	// column index for minimum distance entry
+	colind = new int[num_of_rows];
+	
+	for (i = 1; i < num_of_rows; i++) set_minrow(sim_matrix, i);
+	
+	// do all the hierarchical clustering by running over all rows, updating the matrix 
+	// as a new cluster is formed each time
+	for (i = 0; i < num_of_rows - 1; i++){
+
+	    // find the row with the lowest disimilarity
+	    min_row = get_minrow(last_row_index);
+	    sim = minrow[min_row];
+	    
+	    System.out.println("Min sim: " + sim);
+	    
+	    // the column with the lowest distance in that row
+	    index  = colind[min_row];
+	    
+	    // array of heights, DESCENDING disim, i.e. sim goes last
+	    heights[num_of_rows - 2 - i] = sim; 
+	    
+	    row = triangle.i_ind[index];
+	    col = triangle.j_ind[index];
+	    
+	    // merges rows and cols to create a smaller sim_matrix
+	    merge (row, col, sim_matrix, last_row_index, linkage_func);
+	    System.out.println("Merging: " + row + " and " + col);
+	    
+	    sim_matrix_size -= last_row_index;
+	    
+	    child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+	    child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+	    
+	    // parents get id = -1 to indicate that they are not representing single docs
+	    // all leaves have an id (1 - num_rows) that points to the doc they contain
+	    int sum = child_1.items + child_2.items;
+	    System.out.println("Create new parent with " + child_1.items + " + " + child_2.items + " leaves");
+	    parent = new Cluster (-1, null, child_1, child_2, sim, child_1.items + child_2.items);
+	    
+	    child_1.parent = parent;
+	    child_2.parent = parent;
+	    
+	    cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+	    cluster_ary.contents.setElementAt(parent,col);
+	    
+	    last_row_index--;
+	}
+	
+	return parent;
+    }
+    
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+    {
+	
+	/* i and j are the matrix elements with maximum similarity         */
+	/* matrix is the triangle similarity matrix with last row index n  */
+	/* linkage_func is a pointer to the linkage function               */
+	
+	/* i > j */
+	
+	int c, k, index1, index2;
+	
+        // go over all rows
+	for (k = 0; k <= n; k++){
+	    
+    	    // if k < j the affected elements are on row j
+	    if (k < j){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	    }else if (k > j && k < i){
+		/* if j < k < i then only one element is updated per row */
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if (k > i && k < n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		//printf ("3 - [%d, %d] = link ([%d, %d], [%d, %d]) = %f\n", k, j, k, i, k, j, matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		index1 = triangle.index (k, i);
+		index2 = triangle.index (k, n);
+		m.matrix[index1] = m.matrix[index2];
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if(k == n  && i != n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		for (c = 0; c < i; c++){
+		    index1 = triangle.index (c, i);
+		    index2 = triangle.index (c, n);
+		    m.matrix[index1] = m.matrix[index2];
+		}
+		
+	    }
+    	}
+	set_minrow(m, i);
+	set_minrow(m, j);
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Iterative.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Iterative.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/Iterative.java	(revision 21112)
@@ -0,0 +1,102 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int wordCount;
+    
+    Iterative(int vsize)
+    {
+	wordCount = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+	
+	for (i = 0; i < wordCount; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+
+
+    void cluster_center (Cluster cluster, float [][] vectors, float[] centroid)
+    {
+        int i;
+	
+      	if (cluster.id == -1){
+      	    cluster_center (cluster.child_1, vectors, centroid);
+      	    cluster_center (cluster.child_2, vectors, centroid);
+    	}else{
+            float[] doubleArray = vectors[cluster.id];
+      	    for (i = 0; i < doubleArray.length; i++)
+		centroid[i] += doubleArray[i];
+        }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, float [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+
+	/**** for each hierarchical cluster, compute centroid ****/
+	for (i = 0; i < num_of_clusters; i++){
+	    clust = (Cluster)clusters.contents.elementAt(i);
+	    clust.centroid = new float[wordCount];
+	    cluster_center (clust, vectors, clust.centroid);
+	    
+	    for (ssum = j = 0; j < wordCount; j++)
+		ssum += clust.centroid[j] * clust.centroid[j];
+	    
+	    if(ssum>0)                // normalise
+		for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < wordCount; j++)
+		    clust.centroid[j] *= ssum;
+	}
+    }
+    
+    
+    void place_item (ClusterArray clusters, int num_of_clusters, float [][] vectors, int item_id,  Distance sim_func)
+    {
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	double minDist = sim_func.get(clust.centroid,vectors[item_id]);	
+	int x_minDist = 0;
+	
+	for (int i = 1; i < num_of_clusters; i++){
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    double dist = sim_func.get(clust.centroid,vectors[item_id]);
+	    
+            if (dist < minDist){
+		x_minDist = i;
+		minDist = dist;
+	    }
+    	}
+	
+	//System.out.println("in " + x_minDist); 
+	clust = (Cluster)clusters.contents.elementAt(x_minDist);
+        clust.iter_items.addElement(new Integer(item_id));
+	clust.iter_items_num++;
+    }
+    
+    
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, int[] num_of_clusters, float[][] vectors, int start_index, int end_index,Distance sim_func)
+    {
+        int i;
+
+	System.out.println("Start: " + start_index + " Finish: " + end_index);
+	
+	if (start_index >= end_index)
+	    return;
+	
+	compute_cluster_centers(clusters, num_of_clusters[0], vectors);
+	
+	for (i = start_index; i < end_index; i++){
+	    System.out.println("Placing document " + i);
+	    place_item(clusters, num_of_clusters[0], vectors, i, sim_func);
+	}
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/LinkFunc.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 21112)
@@ -0,0 +1,8 @@
+package vishnu.cluster;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract float link(float a, float b);
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MaxLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MaxLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MaxLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MaxLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a > b? a*99+b: b*99+a)/100;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MinLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MinLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/MinLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MinLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/SimMatrix.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 21112)
@@ -0,0 +1,70 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class SimMatrix
+{
+    
+    public float[] matrix;
+    
+
+    public SimMatrix(float[][] KDMatrix, int limit, Distance simfunc){
+	
+	float iarray[];
+	
+	int hitDocs = KDMatrix.length;
+	int wordCount = KDMatrix[0].length;
+	int index = 0;
+	
+	// to exclude diagonal and conjugates
+	// - - - - -
+	//         |
+	// *       |
+	// * *     |
+	// * * *   |    
+	
+	int realSize = (hitDocs * (hitDocs - 1)/2);
+	matrix = new float[realSize];
+	
+	// this gets the sim matrix for all docs rather 
+	// than only the first limit - mistake?
+
+	for(int i = 1; i < hitDocs; i++){
+	    iarray = KDMatrix[i];
+	    for(int j = 0; j < i; j++){
+		matrix[index] = simfunc.get(iarray,KDMatrix[j]);
+	        index++;
+	    }
+	}
+    }
+
+    public void printMatrix()
+    {
+	int countA = 0;
+	int countB = 0;
+
+	for( int i = 0; i < matrix.length; i++ ){
+	    System.out.print(matrix[i] + " ");
+	    if( countA >= countB ){
+		System.out.println();
+		countB++;
+		countA = 0;
+	    }
+	    else countA++;
+	}
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheInt.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheInt.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheInt.java	(revision 21112)
@@ -0,0 +1,11 @@
+package vishnu.cluster;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheVectors.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheVectors.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TheVectors.java	(revision 21112)
@@ -0,0 +1,79 @@
+package vishnu.cluster;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TriangleIndex.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Collection.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Collection.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Collection.java	(revision 21112)
@@ -0,0 +1,23 @@
+package vishnu.datablock;
+
+import java.io.*;
+
+public class Collection implements Serializable
+{
+    public String dir;
+    public String name;
+    public String desc;
+    public int size;
+    public String engine;
+    public String [] views;
+    
+    public Collection(String di, String n, String d, int s, String e, String [] v)
+    {
+	dir = di;
+	name = n; 
+	desc = d; 
+	size = s;
+	engine = e;
+	views = v;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/DataBlock.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/DataBlock.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/DataBlock.java	(revision 21112)
@@ -0,0 +1,35 @@
+package vishnu.datablock;
+
+import java.io.*;
+import java.util.*;
+
+
+
+public class DataBlock implements Serializable
+{
+    public int [] docs = null;
+    public String [] descriptions = null;
+    public String [] words = null;
+    public SparseMatrix matrix = null;
+    public Vector [] clusters = null;
+    public Point2D [] sammon = null;
+
+    public DataBlock() {
+    }
+    public DataBlock(int[] docs,
+		     String [] descriptions,
+		     String [] words,
+		     SparseMatrix matrix,
+		     Vector [] clusters,
+		     Point2D [] sammon)
+    {
+	this.docs = docs;
+	this.descriptions = descriptions;
+        this.words = words;
+	this.matrix = matrix;
+	this.clusters = clusters;
+	this.sammon = sammon;
+    }
+    
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Node.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Node.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Node.java	(revision 21112)
@@ -0,0 +1,37 @@
+package vishnu.datablock;
+
+public class Node
+{
+    Node next;
+    int value;
+    
+    public Node(int i, Node n)
+    {
+        value = i;
+        next = n;
+    }
+    
+    public Node(int i)
+    {
+        value = i;
+        next = null;
+    }
+    
+    public Node push(Node n)
+	{
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+	public Node pop()
+	{
+		return next;
+	}
+	
+	public int getValue()
+	{
+	    return value;
+	}
+  
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Point2D.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Point2D.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Point2D.java	(revision 21112)
@@ -0,0 +1,21 @@
+package vishnu.datablock;
+import java.io.*;
+/** 
+* Point2D.java: Class for points in logical coordinates.
+* From Section 1.5 of
+* @author Ammeraal, L. (1998) Computer Graphics for Java Programmers,
+* Chichester: John Wiley.
+*/       
+public class Point2D  implements Serializable
+{
+   /** x cartesian coordinate */
+   public float x;
+   /** y cartesian coordinate */
+   public float y;
+   /**
+   * constructor
+   * @param cartesian x coordinate
+   * @param cartesian y coordinate
+   */
+   public Point2D(float x, float y){this.x = x; this.y = y;}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Query.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Query.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/Query.java	(revision 21112)
@@ -0,0 +1,69 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+
+package vishnu.datablock;
+import java.io.*;
+
+public class Query implements Serializable
+{
+    public String query = null;
+    public String collection = null;
+    public int docNum = -1;
+    public String action = null;
+
+    public Query(){}
+
+    public Query(String q, String c, String s)
+    {
+	query = q;
+	collection = c;
+    }
+
+    public void setCollection(String c)
+    {
+	collection = c;
+    }
+
+    public void setDocNum(int id)
+    {
+	docNum = id;
+    }
+
+    public void setQuery(String q)
+    {
+	query = q;
+    }
+
+    public void setAction(String a)
+    {
+	action = a;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/ShortFileException.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 21112)
@@ -0,0 +1,6 @@
+package vishnu.datablock;
+public class ShortFileException extends Exception
+{
+    public ShortFileException(){super();}
+    public ShortFileException(String s){super(s);}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseMatrix.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 21112)
@@ -0,0 +1,81 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseMatrix implements Serializable
+{
+    private SparseNode [] nodes;
+    private int n_cols;
+    private int n_rows;
+    
+    public SparseMatrix(int rows, int cols)
+    {
+        n_rows = rows;
+	n_cols = cols;
+	
+        nodes = new SparseNode[n_rows];
+	
+        for (int c = 0; c < n_rows; c++)
+	    nodes[c] = null;
+    }
+    
+    
+    public void addDataItem(int row, int column, float value)
+    {
+	SparseNode node; 
+	node = new SparseNode(value, column);
+	nodes[row] = node.push(nodes[row]);
+    }
+    
+    
+    public SparseNode getRow(int row)
+    {
+	return nodes[row];
+    }
+
+
+    public int getRowCount()
+    {
+	    return n_rows;
+    }
+
+
+    public int getColumnCount()
+    {
+	    return n_cols;
+    }
+    
+
+    public float[][] getExpanded()
+    {
+        float[][] full = new float[n_rows][n_cols];
+	
+	SparseNode n;
+	
+	for(int r = 0; r < n_rows; r++){
+	    for (int c = 0; c < n_cols; c++)
+		full[r][c]=0;
+	    
+	    n=nodes[r];
+	    
+	    while(n!= null){
+		full[r][n.getColumn()]=n.getValue();
+		n=n.getNext();
+	    }
+	}
+	return full;
+    }
+    
+    
+    public void print()
+    {
+	for( int i = 0; i < nodes.length; i++ ){
+	    SparseNode n = getRow(i);
+	    while(n != null ){
+		System.out.print(n.getColumn()+ " ");
+		n = n.getNext();
+	    }
+	    System.out.println();
+	}
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseNode.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseNode.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/datablock/SparseNode.java	(revision 21112)
@@ -0,0 +1,59 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseNode implements Serializable
+{
+    private float value;
+    
+    private int column;
+    private SparseNode next;
+    
+    
+    SparseNode(float f, int c)
+    {
+	value = f;
+	column = c;
+	next = null;
+    }
+    
+    
+    SparseNode(float f, int c, SparseNode n)
+    {
+	value = f;
+	column = c;
+	next = n;
+    }
+    
+
+    public SparseNode push(SparseNode n)
+    {
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+
+    public SparseNode pop()
+    {
+	return next;
+    }
+
+    
+    public SparseNode getNext()
+    {
+        return next;
+    }
+
+    
+    public int getColumn()
+    {
+        return column;
+    }
+    
+
+    public float getValue()
+    {
+        return value;
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKServer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKServer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKServer.java	(revision 21112)
@@ -0,0 +1,293 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Enumeration;
+import vishnu.util.Misc;
+import vishnu.util.Sammon;
+import vishnu.cluster.*;
+import vishnu.datablock.*;
+
+public class CKServer 
+{
+    private String index_directory = null;
+
+    private final static String FILE_DOC_FREQ = "doc_freq";
+    private final static String FILE_TERM_FREQ = "term_freq";
+    private final static String FILE_CK_WORDS = "ck_words";
+    
+    private float[] docFreq = null;
+    private String[] ck_words = null;
+    private Vector termFreq = null;
+
+
+    private int[] x_keywords;
+    private int[] x_hitdocs;
+    private float[][] KDMatrix;
+    private Vector[] clusters;
+    private double[][] centroids;
+    private Point2D[] sammon;
+    private Vector descriptions;
+    
+    public CKServer(String collection_home, String view) {
+	
+	this.index_directory = collection_home;
+	if (!this.index_directory.endsWith(File.separator)) {
+	    this.index_directory += File.separator;
+	}
+	this.index_directory += "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    this.index_directory += view + File.separator;
+	}
+	    
+	loadIndices();
+    }
+    
+    private void loadIndices(){
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	try {
+	    docFreq = (float[])getObject(this.index_directory  + FILE_DOC_FREQ);
+	    ck_words = (String[])getObject(this.index_directory + FILE_CK_WORDS);
+	    termFreq = (Vector)getObject(this.index_directory + FILE_TERM_FREQ);
+	} catch (Exception e) {
+	    System.err.println("CKServer Error: Couldn't load up ck index data: "+e);
+	}
+    }
+
+    
+    private Object getObject(String str)
+	throws IOException, FileNotFoundException, ClassNotFoundException {
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	fis = new FileInputStream(str);
+	in = new ObjectInputStream(fis);
+	o = in.readObject();
+	in.close();
+	return o;
+    }
+
+
+    public void setDescriptions(Vector desc)
+    {
+	descriptions = desc;
+    }
+    
+
+    public String[] computeKeywords(Vector docNum)
+    {
+	int hits = docNum.size();
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+	
+
+	/**** compute interesting weight for each keyword ****/
+	
+	Enumeration enumer = keyHash.keys();
+	
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+	
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+	    
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+	    
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+	    
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+	
+	
+	/**** store indices of up to 100 top ranked keywords ****/
+	
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	}
+	
+	
+	/**** sort the indices_kw array in increasing order of indices ****/
+	
+	Misc.quickSort(indices_kw);
+
+	x_keywords = indices_kw;
+	
+	String[] keywords = new String[x_keywords.length];
+	for( int i = 0; i < x_keywords.length; i++ ){
+	    keywords[i] = ck_words[x_keywords[i]];
+	}
+
+	return keywords;
+    }
+    
+    
+    public SparseMatrix getSparseMatrix(Vector docNums){
+		
+	int hits = docNums.size();
+	int wordCount = x_keywords.length;
+	int scale = 100; // length of document vector
+	
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+	
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNums.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(x_keywords[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[x_keywords[j]]));
+		    w = term_freq.intValue();
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+	    }
+	    
+	    
+	    /**** sqrt sum to obtain |z| ****/
+	    
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i])/scale; else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix and full matrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+	KDMatrix = new float[hits-emptyRows][wordCount];
+		
+	int ci = 0;
+
+	for( int i = hits-1; i >= 0; i-- ){
+	    if( sum_rows[i] == 0 ){
+		docNums.removeElementAt(i);
+		descriptions.removeElementAt(i);
+	    }
+	}
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 ) continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = m; //(float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    KDMatrix[ci][j] = norm_m; //(float)(norm_m);
+		}
+		else KDMatrix[ci][j] = 0;
+	    }
+	    ci++;
+	}
+	
+	x_hitdocs = new int[docNums.size()];
+	
+	for( int i = 0; i < docNums.size(); i++ ){
+	    x_hitdocs[i] = ((Integer)docNums.elementAt(i)).intValue();
+	}
+	
+	return matrix;
+    }
+
+
+    public Vector[] getClusters(){
+
+	int wordCount = KDMatrix[0].length;
+	int docHits = KDMatrix.length;
+	System.err.println("getClusters: word count = "+wordCount+", docHits = "+docHits);
+        int clusterNum =  docHits>20?10:docHits;
+      
+        int limit =  docHits/2;
+	
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,KDMatrix,1,clusterNum,limit);
+	
+        clusters = clustering.getClusters();
+	centroids= clustering.getCentroids();
+	
+	// what are these for??
+	//Sammon map = new Sammon(centroids);
+	//Point2D [] sammon = map.getMapping();
+	return clusters;
+    }
+    
+    
+    public int[] getHitDocuments()
+    {
+	return x_hitdocs;
+    }
+
+
+    public double[][] getCentroids()
+    {
+	return centroids;
+    }
+
+
+    public String[] getHitDescriptions()
+    {
+	String[] desc = new String[descriptions.size()];
+	for( int i = 0; i < desc.length; i++ ){
+	    desc[i] = (String)descriptions.elementAt(i);
+	}
+
+	return desc;
+    }
+    
+    public Point2D[] getSammonMap(double[][] centroids)
+    {
+	if (centroids == null) {
+	    return null;
+	}
+	Sammon sammon = new Sammon(centroids);
+	Point2D[] p = sammon.getMapping();
+	
+	return p;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKTester.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKTester.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/CKTester.java	(revision 21112)
@@ -0,0 +1,230 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.*;
+import vishnu.server.Search.*;
+import vishnu.util.*;
+import vishnu.datablock.*;
+
+public class CKTester
+{
+
+    public static void main(String[] args){
+	CKTester ckt = new CKTester(args);
+    }
+
+    
+    public CKTester(String[] args){
+	
+	Vector docNum = new Vector();
+	Vector descriptions = new Vector();
+	
+	String query = args[0];
+	String coll = args[1];
+	String vis_home = args[2];
+	String view = null;
+	if (args.length >= 4) {
+	    view = args[3];
+	}
+
+	/**** read in ck words, doc freq and term freq ****/
+
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	String root = vis_home;
+	if (!root.endsWith(File.separator)) {
+	    root += File.separator;
+	}
+	root += "collections" + File.separator+ coll + File.separator + "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    root += view+File.separator;
+	}
+	String fileDocFreq = "doc_freq";
+	String fileTermFreq = "term_freq";
+	String fileCKWords = "ck_words";
+	
+	float[] docFreq = (float[])getObject(root+fileDocFreq);
+	String[] ck_words = (String[])getObject(root + fileCKWords);
+	Vector termFreq = (Vector)getObject(root + fileTermFreq);
+
+
+	/**** get relevant docs ****/
+
+	LUCSearcher lus = new LUCSearcher(coll, view); // coll should be fullpath
+	//lus.setRootDirectory(vis_home);
+	lus.search(query);
+	docNum = lus.getDocIdentifiers();
+	descriptions = lus.getDocDescriptions();
+	
+	int hits = docNum.size();
+	
+
+	/**** pool the candidate keywords and note their frequencies ****/
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+
+	System.out.println("all keywords: " + count_ck_all);
+	System.out.println("pooled: " + keyHash.size());
+
+
+	/**** compute interesting weight for each keyword ****/
+
+	Enumeration enumer = keyHash.keys();
+
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+
+	    //System.out.println(set_freq_f+" "+docFreq[index_ck.intValue()]+" "+weight_ck[count]);
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+
+
+	/**** store indices of up to 100 top ranked keywords ****/
+
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	    System.out.println(indices_kw[i] + " " + ck + " " + weight + " ");
+	}
+
+
+	/**** sort the indices_kw array in increasing order of indices ****/
+
+	Misc.quickSort(indices_kw);
+
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(indices_kw[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[indices_kw[j]]));
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+		System.out.print(matrix_kd[i][j] + " ");
+	    }
+	    
+	    System.out.println();
+	    
+
+	    /**** sqrt sum to obtain |z| ****/
+
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i]); else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+
+	int ci = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 )
+		continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = (float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    System.out.print(j + " (" + norm_m + ") ");
+		}
+	    }
+
+	    ci++;
+	    System.out.println();
+	}
+    }
+    
+
+
+    private Object getObject(String str){
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	try{
+	    fis = new FileInputStream(str);
+	    in = new ObjectInputStream(fis);
+	    o = in.readObject();
+	    in.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+	catch( ClassNotFoundException ex){
+	    ex.printStackTrace();
+	}
+	return o;
+    }
+}
+
+
+class Node{
+    Node next = null;
+    float value = -1;
+    int docIndex = -1;
+    int kwIndex = -1;
+
+    public Node(int d, int k)
+    {
+	docIndex = d;
+	kwIndex = k;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Packaging.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Packaging.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Packaging.java	(revision 21112)
@@ -0,0 +1,376 @@
+package vishnu.server;
+
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+//import org.apache.xpath.XPathAPI;
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+import java.awt.*;
+
+public class Packaging
+{
+    static Document doc = null;
+    static boolean out = false;
+
+    static String matrixString = "";
+
+    public Packaging()
+    {
+    }
+
+
+    public static int[] inInts(StringTokenizer st)
+    {
+	String line;
+	int intCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new int[0];
+	}
+
+	intCount = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(intCount);
+
+	int[] intarray = new int[intCount];
+	for (int c = 0; c < intCount;c++)
+	    {
+		line = st.nextToken();
+		int n = Integer.parseInt(line);
+		intarray[c] = n;
+		if(out) System.out.println(n);
+	    }
+	return intarray;
+    }		
+    
+    
+    public static String [] inStrings(StringTokenizer st)
+    {
+	String line;
+	String [] instrings;
+	int strCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new String[0];
+	}
+	
+	strCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(strCount);
+	   
+	instrings = new String[strCount];
+	for (int c = 0; c < strCount;c++)
+	    {
+		instrings[c] = st.nextToken();
+	    }
+
+	return instrings;
+    }
+
+    public static SparseMatrix inMatrix(StringTokenizer st, int wordCount)
+    {
+	String line;
+	int rowCount,column,value;
+
+	if (!st.hasMoreTokens()) {
+	    return new SparseMatrix(0, wordCount);
+	}
+
+	rowCount = Integer.parseInt(st.nextToken());
+	matrixString += String.valueOf(rowCount) + "\n";
+	if(out) System.out.println(rowCount);
+	
+	SparseMatrix matrix = new SparseMatrix(rowCount,wordCount);
+	
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		matrixString += line + "\n";
+
+		if(out) System.out.println(line);
+		
+		if(line.length() > 1)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					matrix.addDataItem(c, column, value);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+	return matrix;
+    } // inMatrix
+
+    
+    public static Vector [] inCluster(StringTokenizer st, int docCount)
+    {
+	String line;
+	int clusterCount,value;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Vector[0];
+	}
+	clusterCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(clusterCount);
+	Vector [] clusters = new Vector[clusterCount];
+	
+	for (int c = 0; c < clusterCount;c++)
+	    {
+		Vector v = new Vector();
+
+		line = st.nextToken();
+		if(out) System.out.println(line);
+		if (line.length()>0)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				
+				value = Integer.parseInt(tokenString);
+				if(value >= 0 && value <= docCount)
+				    {
+					v.addElement(new Integer(value));
+				    } // if
+			    } // while
+		    } // if
+		
+		clusters[c]=v;
+	    } // for
+	return clusters;
+    }
+ 
+
+    public static Point2D[] inPoint2D(StringTokenizer st)
+    {
+	String line;
+	int count ;
+	float x = 0,y = 0;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Point2D[0];
+	}
+	
+	count = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(count);
+	
+	Point2D [] array = new Point2D[count];
+	
+	for (int c = 0; c < count;c++)
+	    {
+		line = st.nextToken();
+		if(out)	System.out.println(line);
+		StringTokenizer tokens = new StringTokenizer(line,",");
+
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			x = Float.parseFloat(tokenString);
+		    }
+			
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			y = Float.parseFloat(tokenString);
+		    } // if
+		    		
+		array[c]= new Point2D(x,y);
+	    } // for
+	return array;
+    }
+
+ 
+ public Element getXMLPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	Element response = doc.createElement("response");
+	response.setAttribute("from","Packaging");
+	
+	Element docList = doc.createElement("docNodeList");
+	response.appendChild(docList);
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		Element node = doc.createElement("docNode");
+		node.setAttribute("docNum", String.valueOf(docs[i]));
+		node.setAttribute("title",(String)desc.elementAt(i));
+		docList.appendChild(node);
+	    }
+
+	Element sammonList = doc.createElement("sammonList");
+	response.appendChild(sammonList);
+	for( int i = 0; i < sammon.length; i++ )
+	    {
+		float x = sammon[i].x;
+		float y = sammon[i].y;		
+		Element node = doc.createElement("sammonNode");
+		node.setAttribute("xpos",String.valueOf(x));
+		node.setAttribute("ypos",String.valueOf(y));
+		sammonList.appendChild(node);
+	    }
+
+	Element wordList = doc.createElement("wordList");
+	response.appendChild(wordList);
+	for( int i = 0; i < words.length; i++ )
+	    {
+		Element node = doc.createElement("wordNode");
+		node.setAttribute("term",words[i]);
+		wordList.appendChild(node);
+	    }
+
+	Element mat = doc.createElement("DocTermSparseMatrix");
+	response.appendChild(mat);
+
+	st = new StringTokenizer(matrixString);
+	String line;
+	int rowCount,column,value;
+
+	rowCount = Integer.parseInt(st.nextToken());
+	mat.setAttribute("rows",String.valueOf(rowCount));
+	mat.setAttribute("cols",String.valueOf(words.length));
+
+	Element rowList = doc.createElement("rowList");
+	mat.appendChild(rowList);
+
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		if(line.length() > 1)
+		    {
+			Element colList = doc.createElement("colList");
+			rowList.appendChild(colList);
+			colList.setAttribute("row",String.valueOf(c));
+			
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				Element entryElem = doc.createElement("entry");
+				colList.appendChild(entryElem);
+
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+
+				entryElem.setAttribute("docFreq",tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					entryElem.setAttribute("term",tokenString);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+
+	Element clusterList = doc.createElement("ClusterList");
+
+	response.appendChild(clusterList);
+
+	for( int i = 0; i < clusters.length; i++ )
+	    {
+		Element cluster = doc.createElement("Cluster");
+		clusterList.appendChild(cluster);
+
+		Vector vector = clusters[i];
+		for( int c = 0; c < vector.size(); c++ )
+		    {
+			Element entry = doc.createElement("leaf");
+			cluster.appendChild(entry);
+			
+			Integer I = (Integer)vector.elementAt(c);
+			entry.setAttribute("docNum",String.valueOf(I.intValue()));
+		    }
+	    }
+
+	for( int i = 0; i < docs.length; i++ ){
+            descriptions[i] = (String)desc.elementAt(i);
+	}
+
+	return response;
+    }
+
+
+   
+    public DataBlock getDataPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		descriptions[i] = (String)desc.elementAt(i);
+		//descriptions[i]	= "doc"+Integer.toString(i);
+	    }
+
+	DataBlock data = new DataBlock(docs,descriptions,words,matrix,clusters,sammon); 
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 21112)
@@ -0,0 +1,173 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+
+import java.util.Vector;
+import java.util.Hashtable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+
+public class LUCSearcher implements SearchInterface
+{
+    protected Document doc=null;
+    private String collection_home;
+    private String index_dir;
+    private Vector docIDs;
+    private Vector docDescs;
+    private Hits hits;
+    private Hashtable ht;
+    
+    public LUCSearcher(String coll_home, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	index_dir = collection_home + File.separator + "index" + 
+	    File.separator + "luc_index" + File.separator;
+	if (view != null) {
+	    index_dir += view + File.separator;
+	}
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+    
+    
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+    
+    
+    public Vector getDocContent(int doc_num)
+    {
+	System.err.println("in doc content, doc num="+doc_num);
+	Integer in = null;
+	if (ht==null) {
+	    System.err.println("ht = null");
+	}
+	try {
+	    in = (Integer)ht.get(new Integer(doc_num));
+	} catch (Exception e) {
+	    System.err.println("exception happened: "+e);
+	}
+	System.err.println("*");
+	int id = 1;
+	System.err.println("*");
+	if( in != null ) id = in.intValue();
+	System.err.println("hit id ="+id);
+	Vector v = new Vector();
+
+	try {
+	    Directory directory = FSDirectory.getDirectory(new File(index_dir), false);
+
+	    IndexReader reader = IndexReader.open(directory);
+	    
+	    Document doc = reader.document(doc_num);
+	    String path = doc.get("path");
+	    // make sure the slashes go the right way
+	    path = path.replaceAll("[\\\\\\/]", File.separator);
+	    String full_path = collection_home + path;
+	    File f = new File(full_path);
+	    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+	    String line;
+	    
+	    while( (line=dis.readLine()) != null ){
+		v.addElement(line);
+	    }
+	    dis.close();
+	} catch(IOException ex){v.addElement("Content of this document currently unavailable");}    
+	return v;
+    }
+
+    
+    public void search(String q)
+    {
+	docIDs = new Vector();
+	docDescs = new Vector();
+	ht = new Hashtable();
+	
+	try {
+	    Searcher searcher = new IndexSearcher(index_dir);
+	    Analyzer analyzer = new StandardAnalyzer();
+	    
+	    Query query = QueryParser.parse(q, "contents", analyzer);
+	    System.out.println("Searching for: " + query.toString("contents") + " in " + index_dir);
+	    
+	    hits = searcher.search(query);
+	    System.out.println(hits.length() + " total matching documents");
+	    int max_docs = Math.min(MAX_DOCS,hits.length());
+	    for (int i = 0; i < max_docs; i++) {
+
+		int id = hits.id(i);
+		ht.put(new Integer(id),new Integer(i));
+		docIDs.addElement(new Integer(id));
+		
+		Document doc = hits.doc(i);
+		String path = doc.get("path");
+		// make sure the slashes go the right way
+		path = path.replaceAll("[\\\\\\/]", File.separator);
+		String full_path = collection_home + path;
+		File f = new File(full_path);
+		
+		String line = "";
+		
+		try{
+		    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+		    while( line.length() < MAX_DESCRIPTION_LENGTH ){
+			String tmp = dis.readLine();
+			line += tmp;
+		    }
+		    docDescs.addElement(line.substring(0,MAX_DESCRIPTION_LENGTH));
+		    dis.close();
+		}catch(IOException ex){docDescs.addElement(new String("No description available"));}    
+	    }
+	    searcher.close();
+	    
+	} catch (Exception ex) {ex.printStackTrace();}
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/MGSearcher.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 21112)
@@ -0,0 +1,101 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+
+import org.greenstone.mg.*;
+
+public class MGSearcher implements SearchInterface
+{
+ 
+    private String collection_home;
+    private String collection_name;
+    private Vector docIDs;
+    private Vector docDescs;
+    private String mg_index_path;
+    private MGWrapper mg_src;
+    
+    public MGSearcher(String coll_home, String coll_name, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	collection_name = coll_name;
+	mg_src = new MGWrapper();
+	mg_index_path = "index"+File.separator+"mg_index"+ File.separator;
+	if (view != null) {
+	    mg_index_path += view + File.separator;
+	}
+	mg_index_path += collection_name+File.separator+collection_name;
+	mg_src.setIndex(mg_index_path);
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+
+
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+
+    // why do we want document in a vector ?? -- kjdon
+    public Vector getDocContent(int doc_num) {
+
+	Vector document = new Vector();
+	String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	String [] lines = doc_content.split("\n");
+	for (int i=0; i<lines.length; i++) {
+	    document.addElement(lines[i]);
+	}
+	return document;
+
+    }
+
+    public void search(String query) {
+	this.mg_src.runQuery(collection_home, mg_index_path, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	docIDs = new Vector();
+	docDescs = new Vector();
+	Vector doc_results =   mqr.getDocs();
+	int max_docs = Math.min(MAX_DOCS,doc_results.size());
+	for (int i=0; i<max_docs; i++) {
+	    int doc_num = (int)((MGDocInfo)doc_results.get(i)).num_;
+	    docIDs.addElement(new Integer(doc_num));
+	    String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	    docDescs.addElement(doc_content.substring(0,MAX_DESCRIPTION_LENGTH));
+	}	
+    }
+    
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/SearchInterface.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 21112)
@@ -0,0 +1,41 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import vishnu.datablock.*;
+import java.util.*;
+import org.w3c.dom.Element;
+
+public interface SearchInterface
+{
+
+    public static final int MAX_DOCS = 100;
+    public static final int MAX_DESCRIPTION_LENGTH = 200;
+
+    public void search(String query);
+
+    public Vector getDocIdentifiers();
+
+    public Vector getDocDescriptions();
+
+    public Vector getDocContent(int doc_num);
+
+    //public void setRootDirectory(String root);
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/VisServlet.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/VisServlet.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/server/VisServlet.java	(revision 21112)
@@ -0,0 +1,328 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server;
+
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.XMLUtil;
+
+import java.io.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.util.Vector;
+import java.util.Hashtable;
+
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+public class VisServlet extends HttpServlet
+{
+    final String DOCUMENT_FETCH_ACTION = "fetch";
+    final String DOCUMENT_SEARCH_ACTION = "search";
+    final String KEYWORDS_COMPUTE_ACTION = "keywords";
+    final String INITIALIZATION_ACTION = "initialise";
+    
+    protected String COLLS_HOME=null; 
+    
+    Hashtable collH;
+
+    public void init(ServletConfig config) throws ServletException {
+	// always call super.init;
+	super.init(config);
+
+	COLLS_HOME = config.getInitParameter("collectionsHome");
+	if (COLLS_HOME == null || COLLS_HOME.equals("")) {
+	    System.err.println("initialization parameter, collectionsHome, not set");
+	    System.exit(1);
+	}
+	if (!COLLS_HOME.endsWith(File.separator)) {
+	    COLLS_HOME += File.separator;
+	}
+    }
+
+    private Object receiveObject(HttpServletRequest req) throws Exception
+    {
+        ObjectInputStream in = new ObjectInputStream( req.getInputStream() );
+        Object obj = in.readObject();
+        in.close();
+        return obj;
+    }
+    
+    
+    private void sendObject(HttpServletResponse resp, Object obj) throws Exception
+    {
+        ObjectOutputStream out = new ObjectOutputStream( resp.getOutputStream() );
+        out.writeObject(obj);
+        out.close();
+    }
+    
+    public void doGet(HttpServletRequest req, HttpServletResponse resp) 
+	throws IOException {
+
+	resp.setContentType("text/html;charset=UTF-8");
+        PrintWriter out = resp.getWriter();
+	String output = "<html><head><title>Visualiser servlet</title></head>\n<body>The servlet is running, and accepting messages via POST</body></html>";
+	out.println(output);
+
+    }
+    public void doPost(HttpServletRequest req, HttpServletResponse resp)
+    {
+	Object obj;
+	
+        try{
+	    obj = receiveObject(req);
+	    Vector q = (Vector)obj;
+	    Vector response = new Vector();
+	    
+	    String action = (String)q.elementAt(0);
+	    System.out.println("Action.. " + action);
+	    
+ 	    if( action.equals(INITIALIZATION_ACTION) ){
+		Vector colls = activateCollections();
+		sendObject(resp,colls);
+	    }
+	    else if( action.equals(DOCUMENT_SEARCH_ACTION) ){
+		Vector docNums = new Vector();
+		Vector descriptions = new Vector();
+		String engine;
+		
+		String query = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		System.out.println("Requesting..." + collection);
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+		
+		System.out.println("Retrieving with " + engine);
+		
+		String collection_home = COLLS_HOME+collection+File.separator;
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		
+		search_engine.search(query);
+		docNums = search_engine.getDocIdentifiers();
+		descriptions = search_engine.getDocDescriptions();
+		System.out.println(docNums.size() + " documents in VisServlet");
+		DataBlock data = generateDataBlock(collection_home, view, docNums, descriptions);
+		if (data != null) {
+		    response.addElement(data);
+		}
+		sendObject(resp,response);
+		search_engine = null;
+		data = null;
+		docNums = null;
+		descriptions = null;
+		
+	    }
+	    else if( action.equals(DOCUMENT_FETCH_ACTION) ){
+		String docNum = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		String engine;
+		String collection_home = COLLS_HOME+collection+File.separator;
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+                System.out.println("Retrieving with " + engine);
+		
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		System.err.println("Searching for doc "+docNum);
+		Vector doc = search_engine.getDocContent(Integer.parseInt(docNum));
+		response.addElement(doc);			
+		sendObject(resp,response);
+		search_engine = null;
+	    }
+	} catch(Exception e){}
+    }
+    
+
+    public Vector connect(Vector q)
+    {
+	return null;
+    }
+
+
+    private Vector activateCollections()
+    {
+	System.err.println("activating collections");
+	Vector colls = new Vector();
+	collH = new Hashtable();
+	Document doc = null;
+	
+	File dir = new File(COLLS_HOME);
+	if( !dir.exists() ){
+	    System.err.println("collections directory, "+COLLS_HOME+", doesn't exist, can't activate collections");
+	    return null;
+	}
+	
+	File[] list = dir.listFiles();
+	
+	for( int i = 0; i < list.length; i++ ){
+	    File f = list[i];
+	    if( f.isDirectory()){
+		File config_file = new File(f, "config.xml");
+		if (!config_file.exists()) {
+		    System.err.println("Collection "+f.getName()+" has no config file, ignoring");
+		    // ignore this collection
+		    continue;
+		} else {
+		    System.err.println("Trying to init collection "+f.getName());
+		}
+		try {
+		    DocumentBuilderFactory doc_build_fac = DocumentBuilderFactory.newInstance();
+		    DocumentBuilder doc_build = doc_build_fac.newDocumentBuilder();
+		    doc = doc_build.parse(config_file);
+		} catch (Exception e) {
+		    System.err.println("Error parsing config file "+config_file.getPath()+", ignoring this collection");
+		    continue;
+		}
+		Element root = doc.getDocumentElement();
+		String name = root.getAttribute("name");
+		String desc = root.getAttribute("description");
+		String engine = root.getAttribute("engine");
+		int size = Integer.parseInt(root.getAttribute("size"));
+		String coll_id = f.getName();
+		Element view_elem = (Element)XMLUtil.getChildByTagName(root, "viewList");
+		String [] views = null;
+		if (view_elem != null) {
+		    NodeList view_list = view_elem.getElementsByTagName("view");
+		    int num_views = view_list.getLength();
+		    if (num_views > 0) {
+			views = new String[num_views];
+			for (int j=0; j<num_views; j++) {
+			    String v = ((Element)view_list.item(j)).getAttribute("name");
+			    views[j]=v;
+			}
+		    }
+		}
+		Collection c = new Collection(coll_id,name,desc,size,engine,views);
+		System.out.println("Hashing.." + coll_id);
+		colls.addElement(c);
+		collH.put(coll_id,engine);
+	    }			
+	}
+	doc = null;
+	return colls;
+    }
+    
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.out.println("Num descriptions: " + descriptions.size());
+	    System.out.println("Num docs: " + docs.length);
+	    System.out.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	return data;
+    }
+}
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/browser/Browser.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 21112)
@@ -0,0 +1,125 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.browser;
+
+import javax.swing.JFrame;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.applet.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.net.URL;
+import java.io.IOException;
+import java.util.*;
+
+public class Browser extends JFrame
+{
+  JScrollPane scrollPane;
+  JEditorPane editorPane;
+  JPanel panel;
+  public Browser(URL url)
+  {
+    jb_init(url);
+  }
+  
+  public Browser(Vector document, long docNumber)
+  {
+    jb_init(document);
+  }
+
+  private void jb_init(Vector document)
+  {
+     WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = new JEditorPane();
+    editorPane.setEditable(false);               
+    editorPane.setContentType("text/html");   
+    
+    String text = new String("");
+    
+    for( int i = 0; i < document.size(); i++ )
+    {
+        text += (String)document.elementAt(i) + "\n";
+    }
+    
+    editorPane.setText(text);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    
+    /*editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");*/
+    
+    setVisible(true);
+  }
+
+  private void jb_init(URL url)
+  {
+    WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    setVisible(true);
+  }
+
+  private JEditorPane createEditorPane(URL url)
+  {
+    JEditorPane edPane = new JEditorPane();
+    edPane.setEditable(false);
+    displayURL(url,edPane);
+    return edPane;
+  }
+
+  private void displayURL(URL url, JEditorPane edPane)
+  {
+    try
+    {
+      edPane.setPage(url);
+    }
+    catch(IOException e)
+    {
+      System.out.println("Bad URL: "+ url.toString());
+    }
+  }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class AveLink extends LinkFunc
+{
+    double link(double a, double b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 21112)
@@ -0,0 +1,120 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int _id;
+    public Cluster  _parent = null;
+    public Cluster  _child_1 = null;
+    public Cluster  _child_2 = null;
+    public Vector docVect;
+    public Vector docObjVect;
+    public int[] wordFreq;  	
+    
+    public double _children_sim;
+    
+    // probably the size of the cluster
+    public int _items;
+    
+    public int _split;
+    
+    
+    
+    // [] -> centroid for each vector element ?
+    public double[] _centroid=null;
+    
+    public Vector _iter_items = null;
+    public int _iter_items_num = 0;
+    
+    public int _x = 0;
+    public int _y = 0;
+    public double[] getCentroid(){return _centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster child_1, Cluster child_2, double children_sim, int items)
+    {
+	_id                    = id;
+	_children_sim          = children_sim;
+	_items                 = items;
+	_split                 = 0;
+	_parent                = parent;
+        _iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (child_1 != null && child_2 != null)
+	    {
+		if (child_2._items > child_1._items)
+		    {
+		        _child_1 = child_1;
+		        _child_2 = child_2;
+		    }
+    		else
+		    {
+		        _child_1 = child_2;
+		        _child_2 = child_1;
+		    }
+  	    }
+	else // if either or both are null
+	    {
+    		_child_1 = child_1;
+    		_child_2 = child_2;
+	    }
+    }
+    
+    public void setDocVect(Vector docs)
+    {
+	docVect = docs;
+    }
+    
+    public void setDocObjVect(Vector docObj)
+    {
+	docObjVect = docObj;
+    }
+    
+    public Vector getDocObjVect()
+    {
+	return docObjVect;
+    }
+    
+    public Vector getDocVect()
+    {
+	return docVect;
+    }
+    
+    public void setWordFreq(int[] freqs)
+    {
+	wordFreq = freqs;
+    }
+    
+    public int[] getWordFreq()
+    {
+	return wordFreq;
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (_id != -1)
+	    return 0;
+	else
+    		return Math.max(1 + _child_1.depth(), 1 + _child_2.depth());
+	}
+
+    void addChildrenToOutputVector(int []coding, Vector output)
+	{
+  	  if (_id != -1)
+          {
+    	    Integer ii = new Integer(coding[_id]);
+            output.addElement(ii);
+
+          }
+  	  else
+    	  {
+      	    _child_1.addChildrenToOutputVector(coding,output);
+      	    _child_2.addChildrenToOutputVector(coding,output);
+    	  }
+    }
+
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 21112)
@@ -0,0 +1,56 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(DataManager dm, Vector indices)
+    {
+	int i;
+	
+	contents = new Vector();
+
+	// num_of_clusters = number of docs
+	for (i = 0; i < indices.size(); i++)
+	    {
+		int id = ((Integer)indices.elementAt(i)).intValue();
+		Cluster c = new Cluster(i, null, null, null, 0, 1);
+		Vector docIndexVector = new Vector();
+		docIndexVector.addElement(new Integer(id));
+		c.setDocVect(docIndexVector);
+		
+		Vector docObjVector = new Vector();
+		DocObj docObj = dm.getDocObjAt(id);
+		docObjVector.addElement(docObj);
+		c.setDocObjVect(docObjVector);    
+		
+		DataBlock data = dm.getDataBlock();
+		int[] wordFreqs = new int[data.words.length];
+		for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+		
+		SparseNode row = data.matrix.getRow(id);
+		
+		while( row!= null)
+		    {
+			wordFreqs[row.getColumn()]++;
+			row = row.getNext();
+		    }
+		c.setWordFreq(wordFreqs);
+		
+    		contents.addElement(c);
+		
+		// i.e. id = i, similarity between kids = 0, #items = 1
+	    }
+    }
+    
+    
+}
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 21112)
@@ -0,0 +1,171 @@
+package vishnu.testvis.dendro;
+
+import java.awt.*;
+import vishnu.testvis.object.*;
+import java.util.Vector;
+
+public class ClusterObj
+
+{
+    Cluster cluster;
+    Point position;
+    int radius;
+    boolean inSubCluster;
+    private boolean orderStatus;
+    private boolean keywordStatus;
+    private String[] freqWords;
+    
+    DocObj[] _docs;		// list of DocObj ref
+    String _label;		// (top 3-5? related words)
+    DataManager _data;
+    
+    public ClusterObj(Cluster c, Point P, int r, DataManager data)
+    {
+        cluster = c;
+        position = P;
+        radius = r;
+        inSubCluster = false;
+        _data = data;
+        orderStatus = false;
+        keywordStatus = false;
+    }
+    
+    
+    public void setPosition(Point p)
+    {
+        position = p;
+    }
+    
+    public void setOrderStatus(boolean b)
+    {
+        orderStatus = b;
+    }
+    
+    public boolean getOrderStatus()
+    {
+        return orderStatus;
+    }
+    
+    public void setFreqWords(String[] str)
+    {
+        freqWords = str;
+        keywordStatus = true;
+    }
+    
+    public String[] getFreqWords()
+    {
+        return freqWords;
+    }
+    
+    public boolean getKeywordStatus()
+    {
+        return keywordStatus;
+    }
+    
+    public void setIsSubset(boolean b)
+    {
+        inSubCluster = b;
+    }
+    
+    public boolean getIsSubset()
+    {
+        return inSubCluster;
+    }
+    
+    public boolean inCircle(int x, int y)
+    {
+        double dist_sqr = (x-position.x)*(x-position.x) + (y-position.y)*(y-position.y);
+        return (dist_sqr <= (4*radius*radius)) ? true : false;
+    }
+    
+    
+    public void setDocObjs(DocObj[] docArray)
+    {
+        _docs = docArray;
+    }
+    
+    
+    public int getX()
+    {
+        return position.x;
+    }
+    
+    public int getY()
+    {
+        return position.y;
+    }
+    
+    public int getRad()
+    {
+        return radius;
+    }
+   
+   
+    public ClusterObj(DocObj[] d, String t, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _data = dm;
+    }
+  
+  
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+  
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    
+    Vector rwVec = new Vector(s);
+    
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+  
+  
+  public int getDocSize()
+  {
+        return cluster._items;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+        
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 21112)
@@ -0,0 +1,248 @@
+package vishnu.testvis.dendro;
+
+/* Related word hierarchical clustering using a given triangle similarity matrix */
+
+//import cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+
+    private Vector [] theResult;
+    private double [][] theCentroids;
+    private Cluster theRoot;
+    public DataManager dataManager;
+    private int method = 1;
+
+    public Vector [] getTheResults()
+    {
+        return theResult;
+    }
+    public double[][] getTheCentroids()
+    {
+        return theCentroids;
+    }
+    
+    public Cluster getRoot()
+    {
+        return theRoot;
+    }
+
+
+    public Clustering (DataManager dm, int linkage, float [][]m,
+		       int num_of_vectors,
+		       int arraySize, int min_c_items,
+		       int target_clusters, int limit)
+    {
+	
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	dataManager = dm;
+	
+	// number of keywords
+	int VECTOR_SIZE = arraySize;
+        
+	Cluster toplevel_cluster;
+	TheInt clusters = new TheInt(0);
+	
+	Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+	double [][] vectors = floatToDoubleMatrix(m);//intToDoubleMatrix(m);
+	//int [] coding = new int[m.length];
+        
+	// populate array with numbers from 0 to #docs-1
+	//for (int c=0;c<m.length;c++) coding[c]=c;
+	
+	int i, j;
+	
+	switch (linkage)
+	    {
+    	    case COMPLETE_LINKAGE:
+      		linkage_func = new MaxLink();
+      		break;
+    	    case AVERAGE_LINKAGE:
+      		linkage_func = new AveLink();
+      		break;
+    	    case SINGLE_LINKAGE:
+      		linkage_func = new MinLink();
+      		break;
+    	    default:
+      		System.out.println("clustering.c: main(): linkage type "+
+	                  linkage +" not defined.");
+      	        System.exit(0);
+    	}
+
+        long startTime = System.currentTimeMillis();
+        long readTime = System.currentTimeMillis();
+        
+        
+        // if fewer documents than upper cluster limit
+        // I think this is an error, should be vectors.length, viz. #documents
+        // which was never spotted as limit is always smaller
+        // than #keywords
+        // if(limit > vectors[0].length) limit = vectors[0].length;
+
+        // contains distance matrix between documents, built using
+        // the double vector[i][] list of keyword frequencies
+
+
+	Vector indices = null;
+	//randomly choose sqrt(k*n) documents
+
+	if( method == 0 )
+	    {
+
+		int sampleSize = 2*(int)Math.sqrt(vectors.length);
+		Vector sampleInt = new Vector();
+		Vector allDocs = new Vector();
+		
+		for( int s=0; s<vectors.length;s++ )
+		    {
+			allDocs.addElement(new Integer(s));
+		    }
+		
+		while( sampleInt.size() < sampleSize )
+		    {
+			int r = (int)(Math.random()*allDocs.size());
+			sampleInt.addElement(new Integer(r));
+			allDocs.removeElementAt(r);
+		    }
+		double[][] sample_matrix = new double[sampleSize][VECTOR_SIZE];
+		for(int s=0; s<sampleSize;s++)
+		    {
+			int docNum = ((Integer)sampleInt.elementAt(s)).intValue();
+			sample_matrix[s] = vectors[docNum];
+		    }
+		
+		indices = sampleInt;
+		sim_matrix = new SimMatrix(sample_matrix,limit,sim_func,VECTOR_SIZE);
+
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+		
+		for( int it = 0; it < allDocs.size(); it++ )
+		    {
+			int docNum = ((Integer)allDocs.elementAt(it)).intValue();
+			
+			double min_dist = 10000;
+			int min_index = -1;
+
+			for( int c = 0; c < content.size(); c++ )
+			    {
+				
+				Cluster clust = (Cluster)content.elementAt(c);
+
+				int docNums = ((Integer)clust.getDocVect().elementAt(0)).intValue();
+
+				double sim = sim_func.get(vectors[docNums],vectors[docNum], VECTOR_SIZE);
+
+				if (sim < min_dist)
+				    {
+					min_dist = sim;
+					min_index = c;
+				    }
+			    }
+			
+			Cluster clust = (Cluster)content.elementAt(min_index);
+			clust._iter_items.addElement(new TheInt(docNum));
+			clust._iter_items_num++;
+			clust.docVect.addElement(new Integer(docNum));
+			long iterativeTime = System.currentTimeMillis();
+			
+		    }
+	    }			 
+	else
+	    {
+   		Vector v = new Vector();
+		for( int l = 0; l < vectors.length; l++ )
+		    {
+			v.addElement(new Integer(l));
+		    }
+
+		indices = v;
+		sim_matrix = new SimMatrix(vectors, limit, sim_func, VECTOR_SIZE);
+	        
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+	    }
+	
+    }
+
+    static double[][] intToDoubleMatrix(int [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+    static double[][] floatToDoubleMatrix(float [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+
+
+    private void getLeaves(Cluster node, Vector content)
+    {
+	if( node._child_1 != null )
+	    {
+		getLeaves(node._child_1,content);
+		getLeaves(node._child_2,content);
+	    }
+	else
+	    {
+		content.addElement(node);
+		Vector v = node.docVect;
+		int i = ((Integer)v.elementAt(0)).intValue();
+	    }
+    }
+    
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 21112)
@@ -0,0 +1,102 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class DendroKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    DendroPanel parentPane;
+
+    public DendroKeywordPanel(Vishnu currentMain, DendroPanel parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+         _dataModel = new RW_DataModel();
+         _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                 refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+    }
+
+    public void refineEngine()
+    {
+        // the filter words are available from within _dataModel, so nothing
+       // is passed on
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+    }
+        
+
+    public void update(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 21112)
@@ -0,0 +1,61 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import java.util.*;
+
+/** Panel containing the dendro map interface */
+public class DendroPanel extends JPanel
+{
+
+  public final static int LIST_ON_LEFT = 0;
+  public final static int LIST_ON_RIGHT = 1;
+  
+  public JPanel picturePanel;
+  public TreePanel treePanel;
+  public DendroKeywordPanel keywordPanel;
+  public HLinkPanel hLinkPanel;
+  public Stack dataStack;
+  public DataManager dataMan=null;
+  public Vishnu currentMain;
+  public JPanel topPanel = new JPanel();
+
+  public DendroPanel(Vishnu parent)
+  {
+    currentMain = parent;
+    setUpGui();
+  }
+  
+  public void setUpGui()
+  {
+        setLayout(new BorderLayout(10,10));
+        setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+        treePanel = new TreePanel(this,currentMain);
+        keywordPanel = new DendroKeywordPanel(currentMain,this);	
+	Dimension d = currentMain.getSize();
+        hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+        picturePanel = new JPanel();
+        picturePanel.setLayout(new BorderLayout(0,3));
+              
+        picturePanel.add(treePanel,"Center");
+        add("East",  hLinkPanel.linkPane);
+        add("Center",   picturePanel);
+        add("West",   keywordPanel);
+        setVisible(true);
+  }
+  
+  public void update()
+  {
+    dataMan = currentMain.dataManager;
+    dataStack = new Stack(); 
+    treePanel.clear();
+    treePanel.update();
+    keywordPanel.clear();
+    hLinkPanel.clear();
+  }
+}
+  
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Distance.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 21112)
@@ -0,0 +1,10 @@
+package vishnu.testvis.dendro;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract double get(double[] v1, double[] v2, int size);
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 21112)
@@ -0,0 +1,16 @@
+package vishnu.testvis.dendro;
+
+public class EuclideanDistance extends Distance
+{
+    public double get(double[] v1, double[] v2, int size)
+    {
+  	double sum = 0, x;
+   //System.out.println("Array length "+ v1.length);
+	for(int c=0;c<v1.length;c++)
+	{
+      	    x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return Math.sqrt(sum);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 21112)
@@ -0,0 +1,80 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        //linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 21112)
@@ -0,0 +1,356 @@
+package vishnu.testvis.dendro;
+
+import vishnu.cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Hierarchical
+{
+    public double[] minrow;
+    public int[]    colind;
+    //public SimMatrix smatrix;
+    public TriangleIndex triangle;
+    public final double weight = 3.2;
+    public DataManager dataManager;
+
+    public Hierarchical(int matrix_rows)
+    {
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+	
+    }
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+    	for (i = index_start + 1; i <= index_end; i++)
+	    {
+      		if (m.matrix[i] < minrow[row])
+		    {
+			minrow[row] = m.matrix[i];
+			colind[row] = i;
+		    }
+	    }
+    }
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row)
+    {
+	int i;
+	int row;
+	double value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++)
+	    {
+      		if (minrow[i] < value)
+		    {
+			row = i;
+			value = minrow[i];
+		    }
+	    }
+	
+	return row;
+    }
+    
+    // num_of_rows is number of documents, or subsample for Buckshot
+    // indices is a vector of the doc ids (made necessary by Buckshot, which uses a subsample of all docs)
+    Cluster clustering (DataManager dm, SimMatrix sim_matrix, int num_of_rows,
+			double[] heights, LinkFunc linkage_func, Vector indices)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+	
+        int min_row, index, row, col, i;
+	double sim;
+
+        int last_row_index, sim_matrix_size;
+
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+  		
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	//	cluster_ary = new ClusterArray (dm, num_of_rows);
+
+	cluster_ary = new ClusterArray(dm,indices);
+	last_row_index = num_of_rows - 1;
+	
+	// a variable that is a double and exists for each row - what can this be?
+	// the lowest dissimilarity score?
+	minrow = new double[num_of_rows];
+	
+	// a variable that is an int and exists for each row - what can that be?
+	// the document index of the document with that lowest score?
+	colind = new int[num_of_rows];
+	
+  	    for (i = 1; i < num_of_rows; i++)
+        {
+            set_minrow(sim_matrix, i);
+        }
+        
+        // do all the hierarchical clustering by running over all rows, updating the matrix 
+        // as a new cluster is formed each time
+  	    for (i = 0; i < num_of_rows - 1; i++)
+        {
+                // find the row with the lowest disimilarity
+      	        min_row = get_minrow(last_row_index);
+      	        sim = minrow[min_row];
+      	        
+      	        // the column with the lowest dis... (sim) in that row
+      	        index  = colind[min_row];
+
+                // array of heights, DESCENDING disim, i.e. sim goes last
+      	        heights[num_of_rows - 2 - i] = sim; 
+
+      	        row = triangle.i_ind[index];
+      	        col = triangle.j_ind[index];
+      	        
+      	        newMerge(row,col,sim_matrix,last_row_index,linkage_func,cluster_ary);
+      	            
+      	        sim_matrix_size -= last_row_index;
+      	        
+      	        child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+      	        child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+
+      	        parent = new Cluster (-1, null, child_1, child_2, sim, child_1._items + child_2._items);
+      	        Vector vDoc = new Vector();
+      	        vDoc.addAll(child_1.getDocVect());
+      	        vDoc.addAll(child_2.getDocVect());
+      	        parent.setDocVect(vDoc);
+      	        
+      	        Vector vDocObj = new Vector();
+      	        vDocObj.addAll(child_1.getDocObjVect());
+      	        vDocObj.addAll(child_2.getDocObjVect());
+      	        parent.setDocObjVect(vDocObj);
+      	        
+      	        int[] wordFreq1 = child_1.getWordFreq();
+      	        int[] wordFreq2 = child_2.getWordFreq();
+      	        for( int z = 0; z < wordFreq1.length; z++ )
+      	        {
+      	            wordFreq1[z] += wordFreq2[z];
+      	        }
+      	        parent.setWordFreq(wordFreq1);
+      	        
+      	        child_1._parent = parent;
+      	        child_2._parent = parent;
+      	        cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+      	        cluster_ary.contents.setElementAt(parent,col);
+      	        last_row_index--;
+         	        
+    	    }
+
+  	    return parent;
+
+        }
+        
+    void newMerge(int r, int c, SimMatrix m, int lastRowIndex, LinkFunc linkage_func, ClusterArray clusterArr)
+    {
+        int n = lastRowIndex+1;
+        double newMatrix[][] = new double[n][n];
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( i != j ) newMatrix[i][j] = (double)((int)(m.matrix[triangle.index(i,j)]*100))/100;
+                    else newMatrix[i][j] = 0;
+            }
+        }
+        
+        for( int i = 0; i < n; i++ )
+        {
+            int sizeA =((Cluster)clusterArr.contents.elementAt(c))._items + ((Cluster)clusterArr.contents.elementAt(r))._items;
+            int sizeB = ((Cluster)clusterArr.contents.elementAt(i))._items;
+            // large if cluster differ in size
+            double scale = weight * (double)((sizeA+sizeB)/2*(sizeA+sizeB)/2)/(sizeA*sizeA+sizeB*sizeB);
+            
+            newMatrix[c][i] = scale*linkage_func.link(newMatrix[c][i],newMatrix[r][i]);            
+            
+        }
+        
+        newMatrix[c][c] = 0;
+        newMatrix[c][r] = newMatrix[c][n-1];
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][c] = newMatrix[c][i];
+        }
+        
+        
+        for( int i = 0; i < n; i++ )
+        {
+            if( i != c ) newMatrix[r][i] = newMatrix[n-1][i];
+        }
+        newMatrix[r][r] = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][r] = newMatrix[r][i];
+        }
+        
+        int index = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( j < i )
+                {
+                    m.matrix[index] = newMatrix[i][j];
+                    index++;
+                }
+          
+            }
+          
+       }
+       
+       for (int i = 1; i < n; i++)
+       {
+            set_minrow(m, i);
+       }
+        
+     
+    }
+
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+	{
+  /* i and j are the matrix elements with maximum similarity         */
+  /* matrix is the triangle similarity matrix with last row index n  */
+  /* linkage_func is a pointer to the linkage function               */
+
+  /* i > j */
+
+  	    int c, k, index1, index2;
+
+        // go over all rows
+  	    for (k = 0; k <= n; k++)
+    	{
+    	    // if k < j the affected elements are on row j
+      		if (k < j)
+		    {
+		        // index1 is the smaller of the two, as j < i
+	     	    index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+        	}
+      		else if (k > j && k < i)
+		    {
+	            // if j < k < i then only one element is updated per row
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+    	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind
+	      		    if (m.matrix[index1] < minrow[k])
+        			{
+		          	    minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      		else if (k > i && k < n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  
+          	    if (index1 == colind[k])
+	    	    {
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+	  		            minrow[k] = m.matrix[index1];
+                        colind[k] = index1;
+			        }
+	    	    }
+
+	  	        index1 = triangle.index (k, i);
+	  	        index2 = triangle.index (k, n);
+	  	        m.matrix[index1] = m.matrix[index2];
+	  	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row 
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+		  	            minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      	    else if (k == n  && i != n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  			index2 = triangle.index (k, i);
+	  			m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  			if (index1 == colind[k])
+	    		{
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      			set_minrow(m, k);
+	    		}
+	  			else
+	    		{
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      			if (m.matrix[index1] < minrow[k])
+					{
+		  				minrow[k] = m.matrix[index1];
+		  				colind[k] = index1;
+					}
+	    		}
+
+	   			for (c = 0; c < i; c++)
+	    		{
+	      			index1 = triangle.index (c, i);
+	      			index2 = triangle.index (c, n);
+	      			m.matrix[index1] = m.matrix[index2];
+	      		}
+			}
+						
+    	}
+  		set_minrow(m, i);
+  		set_minrow(m, j);
+	}
+
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 21112)
@@ -0,0 +1,111 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int VECTOR_SIZE;
+    
+    Iterative(int vsize)
+    {
+	VECTOR_SIZE = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+
+	for (i = 0; i < VECTOR_SIZE; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+    
+    
+    void cluster_center (Cluster cluster, double [][] vectors, double[] centroid)
+    {
+        int i;
+	
+      	if (cluster._id == -1)
+	    {
+		cluster_center (cluster._child_1, vectors, centroid);
+		cluster_center (cluster._child_2, vectors, centroid);
+	    }
+        else
+	    {
+		double[] doubleArray = vectors[cluster._id];
+		for (i = 0; i < doubleArray.length; i++)
+	            centroid[i] += doubleArray[i];
+	    }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, double [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+	for (i = 0; i < num_of_clusters; i++)
+	    {
+		clust = (Cluster)clusters.contents.elementAt(i);
+	        clust._centroid = new double[VECTOR_SIZE];
+		cluster_center (clust, vectors, clust._centroid);
+		for (ssum = j = 0; j < VECTOR_SIZE; j++)
+		    ssum += clust._centroid[j] * clust._centroid[j];
+		if(ssum>0)                // normalise
+		    for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < VECTOR_SIZE; j++)
+			clust._centroid[j] *= ssum;
+	    }
+    }
+    
+
+    void place_item (ClusterArray clusters,
+		     int num_of_clusters,
+		     double [][] vectors,
+		     int item_id,  Distance sim_func)
+    {
+	double max_sim, sim;
+	int max_sim_index;
+	
+	int i;
+	
+	max_sim_index = 0;
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	max_sim = sim_func.get(clust._centroid,
+			       vectors[item_id], VECTOR_SIZE);
+	
+  	    for (i = 1; i < num_of_clusters; i++)
+    	{
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    sim = sim_func.get(clust._centroid,
+			vectors[item_id], VECTOR_SIZE);
+
+            if (sim < max_sim)
+	        {
+	  	        max_sim_index = i;
+	  	        max_sim = sim;
+	        }
+    	}
+  	    
+  	    clust = (Cluster)clusters.contents.elementAt(max_sim_index);
+        clust._iter_items.addElement(new TheInt(item_id));
+	    clust._iter_items_num++;
+    }
+
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, TheInt num_of_clusters,
+                      double[][] vectors, int start_index, int end_index,
+                      Distance sim_func)
+    {
+        int i;
+    
+  	    if (start_index >= end_index)
+    	    	return;
+
+  	    compute_cluster_centers(clusters, num_of_clusters.val, vectors);
+
+  	    for (i = start_index; i < end_index; i++)
+      	    	place_item(clusters, num_of_clusters.val, vectors, i, sim_func);
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 21112)
@@ -0,0 +1,8 @@
+package vishnu.testvis.dendro;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract double link(double a, double b);
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MaxLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a > b) ? a : b;      //*99+b: b*99+a)/100;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MinLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 21112)
@@ -0,0 +1,76 @@
+package vishnu.testvis.dendro;
+
+import vishnu.testvis.object.*;
+
+public class QSortForDocObj
+{
+
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+  
+   void QuickSort(DocObj a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+           mid = a[ ( lo0 + hi0 ) / 2 ].getNumber();
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].getNumber() < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].getNumber() > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(DocObj a[], int i, int j)
+   {
+      DocObj D;
+      D = a[i];
+      a[i] = a[j];
+      a[j] = D;
+   }
+
+   public void sort(DocObj a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 21112)
@@ -0,0 +1,46 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+// kainof DistanceMatrix
+public class SimMatrix
+{
+  public double[] matrix;
+
+  // "vectors" is the expanded SparseMatrix, i.e. doc x keywords
+  // VECTOR_SIZE = #keywords
+  public SimMatrix(double[][] vectors,
+      int limit, Distance simfunc, int VECTOR_SIZE)
+  {
+    int i,j;
+    double iarray[];
+    
+    // size = #docs
+    int size=vectors.length, index=0;
+    
+    // to exclude diagonal and conjugates
+    // - - - - -
+    //         |
+    // *       |
+    // * *     |
+    // * * *   |    
+    int realSize = (size * (size - 1)/2);
+    matrix=new double[realSize];
+    System.out.println("Similarity matrix size: " + size);
+    
+    for(i = 1; i < size; i++)
+    {
+        iarray=vectors[i];
+        for(j = 0; j < i; j++)
+        {
+            matrix[index] = simfunc.get(iarray,
+                            vectors[j],
+                            VECTOR_SIZE);
+	        index++;
+        }
+    }
+  }
+
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 21112)
@@ -0,0 +1,17 @@
+package vishnu.testvis.dendro;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+	
+	public void setInt(int i)
+	{
+	    val = i;
+	}
+	
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 21112)
@@ -0,0 +1,79 @@
+package vishnu.testvis.dendro;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 21112)
@@ -0,0 +1,822 @@
+package vishnu.testvis.dendro;
+
+import java.awt.image.*;
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import vishnu.datablock.*;
+
+
+public class TreePanel extends JPanel
+{
+    
+    DendroPanel parent;
+    Vishnu currentMain;
+    DataManager dataManager;
+    DataBlock data;
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics offGraphics;
+    Graphics2D offGraphics2D;
+    Vector clusterVect = null;
+    Cluster root = null;
+    Cluster selectedRoot = null;
+    ClusterObj selectedCluster = null;
+    ClusterObj oldCluster = null;
+    
+    int selectedClusterID = -1; 
+    int oldClusterID = -1;
+    JPopupMenu popup;
+    boolean constructed = false;
+    int selectedDepth = -1;
+    int initBranchLength = 85;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    public TreePanel(DendroPanel parent, Vishnu frame)
+    {
+        currentMain = frame;
+        this.parent = parent;
+        dataManager = frame.dataManager;
+             
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+        
+        this.addComponentListener( new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+	    {
+		repaint();
+            }
+        }
+        );
+        
+        this.addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int x = mouseEvent.getX();
+                int y = mouseEvent.getY();
+                
+                if( clusterVect != null )
+                {
+                    
+                    oldClusterID = selectedClusterID;
+                    oldCluster = selectedCluster;
+                    selectedClusterID = -1;
+                    selectedCluster = null;
+                    
+                    boolean notFound = true;
+                    int i = 0;
+                    
+                    while( notFound && (i < clusterVect.size()) )
+                    {
+                       ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                       
+                       if( co.inCircle(x,y) )
+                       {
+                            selectedCluster = co;
+                            selectedClusterID = i;
+                                        
+                            if( selectedClusterID != oldClusterID )
+                            {
+                                 repaint();
+                            }
+                            notFound = false;
+                       }
+                       i++;
+                    }
+                    
+                    if( (oldClusterID != selectedClusterID) && ( oldClusterID != -1 ) )
+                    {
+                        popup.setVisible(false);
+                        repaint();
+			//update();
+                    }
+                    
+                    if( selectedClusterID != -1 )
+                    {
+                        if( selectedClusterID != oldClusterID )
+                        {
+                            popup.removeAll();
+                            int size = selectedCluster.cluster._items;
+                            String str =  size + ((size > 1) ? " documents" : " document");
+                            JMenuItem menuItem = new JMenuItem(str);
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent actionEvent)
+                                {
+                                    getDocObjs();
+                                    String[] drillWords = selectedCluster.getFreqWords();
+                                    String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+
+                                    updateHLinkPanel();                                
+                                    updateKeyWordTable();
+                                }
+                                
+                            });
+                            
+                            popup.add(menuItem);
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            
+                            int mode0 = -1;
+                            
+                            // if the currentRoot is selected, dont show "drilldown"
+                            if( selectedClusterID == 0 )
+                            {
+                                // if we are at the top, dont show "undrill"
+                                if( ((Object)selectedCluster.cluster).equals((Object)root) )
+                                {
+                                    str = "Top of hierarchy";
+                                    mode0 = 2;
+                                }
+                                else
+                                {
+                                    str = "Sequential undrill";
+                                    mode0 = 1;
+                                }
+                            }
+                            // if somewhere else in the tree
+                            else
+                            {
+                                if( selectedCluster.cluster._items > 1 )
+                                {
+                                    str = "Drill down";
+                                    mode0 = 0;
+                                }
+                                else
+                                {
+                                    str = "Outermost twigs";
+                                    mode0 = 3;
+                                }
+                            }
+                            
+                            final int mode = mode0;
+                            
+                            menuItem = new JMenuItem(str);
+                            if( (mode != 0 ) && (mode != 1) )
+                                menuItem.setEnabled(false);
+                                
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent evt)
+                                {
+                                    if( mode == 0 )
+                                    {
+                                        selectedRoot = selectedCluster.cluster;
+                                        String[] drillWords = selectedCluster.getFreqWords();
+                                        String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+                                        
+                                        // to establish new clusterobj vector
+                                        constructed = false;   
+                                        repaint();
+					//update();                                  
+                                    }
+                                    if( mode == 1 )
+                                    {
+                                        selectedRoot = selectedRoot._parent;
+                                        constructed = false;
+                                        repaint();
+					//update();
+                                    }
+                                }
+                            });
+                            popup.add(menuItem);
+                            
+                            if( mode == 1 )
+                            {
+                                menuItem = new JMenuItem("Total undrill");
+                                menuItem.addActionListener(new ActionListener()
+                                {
+                                    public void actionPerformed(ActionEvent evt)
+                                    {
+                                        selectedRoot = root;
+                                        constructed = false;
+                                        //update();
+					repaint();
+                                    }
+                                });
+                                popup.add(menuItem);
+                            }
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            int[] wordFreq = selectedCluster.cluster.getWordFreq();
+                            String[] words = getMostFrequentWords(wordFreq);
+                            
+                            selectedCluster.setFreqWords(words);
+                            
+                            menuItem = new JMenuItem(words[0]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[1]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[2]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            popup.validate();
+                            popup.setVisible(true);
+                        }
+                        
+                        popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                        mouseEvent.consume();
+                    }     
+                }  // of clustervect != null
+             } // of mouse moved
+        });
+        
+        setVisible(true);
+    }
+    
+    public void clear()
+    {
+        constructed = false;
+    }
+
+        
+    public String[] getMostFrequentWords(int[] wordFreq)
+    {
+        
+        
+        int max1Freq = 0;
+        int max1Index = 0;
+        int max2Freq = 0;
+        int max2Index = 0;
+        int max3Freq = 0;
+        int max3Index = 0;
+        
+        for( int i = 0; i < wordFreq.length; i++ )
+        {
+            if( wordFreq[i] > max3Freq )
+            {
+                if( wordFreq[i] > max2Freq )
+                {
+                    if( wordFreq[i] > max1Freq )
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = max1Freq;
+                        max2Index = max1Index;
+                        max1Freq = wordFreq[i];
+                        max1Index = i;
+                    }
+                    else
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = wordFreq[i];
+                        max2Index = i;
+                    }
+                }
+                else
+                {
+                    max3Freq = wordFreq[i];
+                    max3Index = i;
+                }
+            }
+                
+        }
+        
+        data = dataManager.getDataBlock();
+        String[] words = new String[3];
+        words[0] = data.words[max1Index];
+        words[1] = data.words[max2Index];
+        words[2] = data.words[max3Index];
+        return words;
+    }
+    
+    public void getDocObjs()
+    {
+       
+       Vector docVect = selectedCluster.cluster.getDocVect();
+       DocObj[] docs = new DocObj[docVect.size()];
+       
+       for( int i = 0; i < docVect.size(); i++ )
+       {
+            int index = ((Integer)docVect.elementAt(i)).intValue();
+            docs[i] = dataManager.getDocObjAt(index);
+       }    
+       
+       
+       if( !selectedCluster.getOrderStatus() )
+       {
+            QSortForDocObj qSort = new QSortForDocObj();
+            try
+            {   
+                 qSort.sort(docs);
+            }
+            catch (Exception e)
+            {}
+            
+            selectedCluster.setDocObjs(docs);
+            selectedCluster.setOrderStatus(true);
+       }
+       
+        
+    }
+    
+    public void updateHLinkPanel()
+    {
+       
+        String htmlDescription = selectedCluster.formatDocs();
+        parent.hLinkPanel.setText(htmlDescription);
+    }
+    
+    public void updateKeyWordTable()
+    {
+        parent.keywordPanel.update(selectedCluster);
+    }
+    
+    
+    public void setConstructed(boolean b)
+    {
+        constructed = b;
+    }
+    
+    
+    public void getData()
+    {
+        dataManager = currentMain.dataManager;
+        data = dataManager.getDataBlock();
+        SparseMatrix m = data.matrix;
+        
+        Vector idVect = new Vector();
+        Vector sammCluster =  dataManager.getSammonClusters();
+        
+        for( int i = 0; i < sammCluster.size(); i++ )
+        {
+            SammCluster cluster = (SammCluster)sammCluster.elementAt(i);
+            idVect.addAll(cluster.getIDs());
+        }
+        
+        for( int c = 0; c < idVect.size(); c++ )
+        {
+            Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        //int [][] full = m.getExpanded(); 
+        float [][] full = m.getExpanded(); 
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+               
+        int limit =  m.getRowCount();
+        
+        int clusterNum = 10;
+                      
+        Clustering clustering = new Clustering(dataManager,Clustering.COMPLETE_LINKAGE,
+					       full,m.getRowCount(),
+					       m.getColumnCount(),1,
+					       clusterNum,limit);
+          
+        root = clustering.getRoot();
+        selectedRoot = root;
+    }
+    
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	
+	d = getSize();
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics2D = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+		
+	if( !constructed )
+	    {
+		clusterVect = new Vector();
+		selectedCluster = null;
+		selectedClusterID = -1;
+	    }
+	
+	int initRad = 0;  
+	initBranchLength = (int)(Math.min(d.width,d.height)*0.12);
+	
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed ){
+	    ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+	    clusterVect.addElement(rootObj);            
+	}
+	else{
+	    ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+	    c.setPosition(center);
+	    clusterVect.setElementAt(c,0);
+	    c = (ClusterObj)clusterVect.elementAt(0);
+	}
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) )
+                {
+                    selected = true;                    
+                    selectedDepth = 0;
+                    offGraphics2D.setColor(Color.blue);
+                }
+
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+		}
+	    
+	    g2.drawImage(bi, 0, 0, this);
+	constructed = true;
+    }
+
+           
+    public void update()
+    {
+	
+	if( !constructed )
+            {
+                clusterVect = new Vector();
+                selectedCluster = null;
+                selectedClusterID = -1;
+            }
+	
+	Dimension d = getSize();
+	offScreenSize = d;
+	if( d.width <= 0 ) return;
+
+	offScreen = createImage(d.width,d.height);
+	offGraphics = offScreen.getGraphics();
+            
+	Graphics2D offGraphics2D = (Graphics2D)offGraphics;
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+	offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	offGraphics2D.setStroke(myStroke);	        
+	
+	int initRad = 0;  
+
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed )
+            {
+                ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+                clusterVect.addElement(rootObj);            
+            }
+	else
+            {
+                ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+                c.setPosition(center);
+                clusterVect.setElementAt(c,0);
+                c = (ClusterObj)clusterVect.elementAt(0);
+            }
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+	
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) ){
+		selected = true;                    
+		selectedDepth = 0;
+		offGraphics2D.setColor(Color.blue);
+	    }
+	    
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+		    {
+			int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+            }
+
+           repaint();
+           constructed = true;
+    }
+    
+    // theta is the angle in the polar representation
+    public void paintPoke(Graphics2D g, Cluster child, int x, int y, Point center, int d, double theta, double alpha,int r, boolean selected, Vector allSubNodeIDs, TheInt ii)
+    {
+	// translate parent cluster position to center
+	int x1 = x - center.x;
+	int y1 = y - center.y;
+	
+	// rotate cluster about origin by alpha to obtain intermediate tree point
+	int x_inter = (int)(x1 * Math.cos(alpha) + y1 * Math.sin(alpha)) + center.x;
+	int y_inter = (int)(x1 * - Math.sin(alpha) + y1 * Math.cos(alpha)) + center.y;
+
+	// rotate cluster about origin by theta
+	int x2 = (int)(x1 * Math.cos(-theta) + y1 * Math.sin(-theta));
+	int y2 = (int)(x1 * - Math.sin(-theta) + y1 * Math.cos(-theta));
+	
+	// extend radius
+	int dRad = (initBranchLength-d*10);
+	if( dRad <= 0 ) dRad = 10;
+	int x3 = x2 + dRad;
+	int y3 = y2;
+
+	// rotate back by (theta +/- alpha)
+	int x4 = (int)(x3 * Math.cos(theta+alpha) + y3 * Math.sin(theta+alpha));
+	int y4 = (int)(x3 * -Math.sin(theta+alpha) + y3 * Math.sin(theta+alpha));
+
+	// finally, add the center back again
+	int x_new = x4 + center.x;
+	int y_new = y4 + center.y;
+		
+	int startAngle =  (int)(180*theta/Math.PI);
+	int rangeAngle = (int)(180*alpha/Math.PI);
+			
+	boolean hitSelectedNode = false;
+	if( selectedClusterID != -1 )
+	{
+	    Object obj1 = (Object)child;
+	    ClusterObj selObj = (ClusterObj)clusterVect.elementAt(selectedClusterID);
+	    Object obj2 = (Object)selObj.cluster;
+	    hitSelectedNode = obj1.equals(obj2);
+	}
+	
+	int currentClusterID = -1;
+    
+	if( selectedClusterID != -1 )
+	    {
+		int w = 0;
+		boolean found = false;
+		while( w < clusterVect.size() && !found )
+		    {
+			Object obj1 = (Object)child;
+			ClusterObj obj = (ClusterObj)clusterVect.elementAt(w);
+			Object obj2 = (Object)obj.cluster;
+			if( obj1.equals(obj2) )
+			    {
+				currentClusterID = w;
+				found = true;
+			    }
+			w++;
+		    }
+	    }
+	
+	if( (d <= selectedDepth) && (!hitSelectedNode) )
+	    {	        
+		g.setColor(new Color( (float) 0.7, (float) 0.7, (float) 1)); 
+		selected = false;
+		
+	    }
+	
+	g.drawArc(center.x-r,center.y-r,2*r,2*r,startAngle,rangeAngle);
+	g.drawLine(x_inter,y_inter, x_new, y_new);
+	
+	if( hitSelectedNode )
+	    {	 
+		selectedDepth = d;
+		selected = true;
+		g.setColor(Color.blue);
+		Vector docVector = child.getDocVect();
+	    }
+	
+	g.setColor(selected ? Color.blue : new Color((float) 0.7, (float) 0.7, (float) 1) );
+	
+	int radius = (int)(Math.log(child._items))* 3;
+	if( radius < 2 ) radius = 2;
+
+	if( child._items == 1 )
+	    {
+		Vector docVector = child.getDocVect();
+		radius = Math.max(5,(int)Math.sqrt(docVector.size())*2);
+	    }
+	
+  	d++;
+	r = (int)(Math.sqrt((x_new-center.x)*(x_new-center.x) + (y_new-center.y)*(y_new-center.y)));
+	
+	Point P = new Point(x_new,y_new);
+	if( !constructed )
+	    {	    
+		ClusterObj cObj = new ClusterObj(child,P,radius,dataManager);
+		clusterVect.addElement(cObj);            
+	    }
+	else
+	    {
+		ClusterObj c = (ClusterObj)clusterVect.elementAt(ii.val);
+		c.setPosition(P);
+		clusterVect.setElementAt(c,ii.val);
+	    }
+	ii.val++;
+	
+   	double new_alpha = (double)alpha/2;
+   	
+   	if( selected )
+   	    allSubNodeIDs.addElement(new Integer(currentClusterID) );
+	
+	if( child._items > 1 && d < 6 ){
+	    paintPoke(g,child._child_1,x_new,y_new,center,d,theta+alpha,new_alpha,r,selected,allSubNodeIDs,ii);
+	    paintPoke(g,child._child_2,x_new,y_new,center,d,theta+alpha,-new_alpha,r,selected, allSubNodeIDs,ii);
+	}
+
+	// we have reached the outermost twigs
+	else{
+	    int[] wordFreq = child.getWordFreq();
+	    String[] words = getMostFrequentWords(wordFreq);
+	    
+	    Font f = new Font("Courier",Font.PLAIN,16);
+	    FontMetrics fm = g.getFontMetrics(f);
+	    int ascent = fm.getAscent();
+	    int height = fm.getHeight();
+	    int wordLength = fm.stringWidth(words[0]);
+	    g.setFont(f);
+	    
+	    if( x_new > center.x ){
+		g.drawString(words[0],x_new + Math.max(radius,5) + 10,y_new + Math.max(radius,5));
+	    }
+	    else{
+		g.drawString(words[0],x_new - Math.max(radius,5) - 10 - wordLength,y_new + Math.max(radius,5));
+	    }
+	}
+    }
+    
+    
+    public void paintTree(Cluster cc, Graphics g, int x, int y, int dist)
+    {
+        int dy = 10;		
+        int rad = 2;
+        int dmt = 4;
+	g.setColor(Color.white);
+	g.fillArc(x-rad,y-rad,dmt,dmt,0,360);
+	g.setColor(Color.blue);
+	g.drawOval(x-rad,y-rad,dmt,dmt);
+	String desc = String.valueOf(cc._items);
+	g.drawString(desc,x-(int)rad/2 - (desc.length()-2)*3,y + (int)rad/2 - 2);
+	
+	if( (cc._items > 1) )
+	    {
+		g.drawLine(x,y+rad,x-dist,y-rad+dy);
+		g.drawLine(x,y+rad,x+dist,y-rad+dy);
+		if( cc._child_1 != null ) paintTree(cc._child_1, g, x - dist, y + dy, (int)dist/2);
+		if( cc._child_2 != null ) paintTree(cc._child_2, g, x +  dist, y + dy, (int)dist/2);
+	    }
+    }
+    
+    public void paint(Graphics g)
+    {
+       update(g);
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.testvis.dendro;
+
+public class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 21112)
@@ -0,0 +1,107 @@
+
+/**
+ * Title:        Visualisation<p>
+ * Description:  An implementation of some search result visualisations
+ * <p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      <p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.evaluate;
+import vishnu.testvis.object.*;
+import vishnu.testvis.visual.*;
+import java.io.*;
+import java.util.*;
+import java.text.*;
+
+public class Logging
+{
+    private Vishnu displayFrame;
+    private String _id;
+    String filename;
+    PrintWriter outlog;
+    long startmilli;
+
+    public Logging(String id, String path, Vishnu d)
+    {
+        displayFrame = d;
+        DateFormat longTime = DateFormat.getDateTimeInstance(DateFormat.FULL,DateFormat.FULL);
+        DateFormat shortTime = DateFormat.getDateTimeInstance(DateFormat.SHORT,DateFormat.SHORT);
+        Date todayDate = new Date();
+        String node = replace_chars(
+            replace_chars(id + "_" + shortTime.format(new Date()) + ".xml"," ","_"),
+            "/:","-");
+        if (null==path)filename = node;
+        else filename = path + node;
+        System.out.println(filename);
+        //Properties p = d.getProperties();
+        _id = id;
+        long Time;
+        try
+        {
+            /*outlog =
+                new PrintWriter(
+                    new BufferedWriter(
+                        new FileWriter(filename)));*/
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    outlog = new PrintWriter(fw,true);
+		
+		    outlog.println("<LOG>");
+            startmilli=System.currentTimeMillis();
+            log("ID",_id);
+            log("DATE_AND_TIME",longTime.format(todayDate));
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    private String replace_chars(String target,String oldch, String newch)
+    {
+        StringTokenizer tokenizer = new StringTokenizer(target,oldch);
+        StringBuffer buff = new StringBuffer();
+
+        while(tokenizer.hasMoreElements())
+        {
+            if(buff.length()>0)
+                buff.append(newch);
+            buff.append(tokenizer.nextToken());
+        }
+        return buff.toString();
+    }
+    
+    public void log(String tag, String value)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\">" +  value + "</" + tag + ">";
+        outlog.println(out);
+    }
+    
+
+    public void log(String tag)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\"/>";
+        outlog.println(out);
+    }
+
+    public void endsection(String tag)
+    {
+        outlog.println("</"+tag+">");
+    }
+
+    public void startsection(String tag)
+    {
+        outlog.println("<"+tag+">");
+    }
+
+    public void logout()
+    {
+        log("LOGOUT",_id);
+        outlog.println("</LOG>");
+        outlog.close();
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 21112)
@@ -0,0 +1,338 @@
+package vishnu.testvis.evaluate;
+import java.io.*; 
+import java.util.*;
+
+public class Queries
+{
+    Vector queryID;
+    Vector queryString;
+    Vector relevance[];
+
+    public Queries()
+    {
+        getQueries();
+        
+        // only of the queries just read
+        getRelevanceData();
+        //printRelevance();
+    }
+    
+    private void getRelevanceData()
+    {
+        
+        relevance = new Vector[queryID.size()];
+        for( int i = 0; i < queryID.size(); i++ )
+        {
+            relevance[i] = new Vector();            
+        }
+        
+        try
+        {
+		    FileReader file = new FileReader("qrels.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    String id = st.nextToken();
+		    //System.out.println(id);
+		    int index = 0;
+		    if( queryID.contains(id) )
+		    {
+		        index = queryID.indexOf(id);
+		        //System.out.println("Contained at position " + index);
+		    }
+		    
+		    String docStr = st.nextToken();
+		    //System.out.println(docStr);
+		    relevance[index].addElement(docStr);
+		    boolean eof = false;
+		    			
+		    while( !eof ) 
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {
+			        st = new StringTokenizer(line);
+	    	    	id = st.nextToken();
+		            if( queryID.contains(id) )
+		            index = queryID.indexOf(id);
+		            docStr = st.nextToken();
+		            relevance[index].addElement(docStr);
+		            st = new StringTokenizer(line);
+		        }
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}    
+    }
+    
+    
+    private void getQueries()
+    {
+        queryID = new Vector();
+        queryString = new Vector();
+        
+        try
+		{
+		    FileReader file = new FileReader("queries.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    
+		    String keywords = new String();
+		    
+		    while( st.hasMoreTokens() )
+		    {
+		        String strTemp = st.nextToken();
+		        
+		        // check whether token is last token
+		        // assume last token is the query number
+		        if( !st.hasMoreTokens() )
+		        {
+		            queryID.addElement(strTemp);
+		            keywords = keywords.trim();
+		            queryString.addElement(keywords);
+		        }
+		        else
+		        {
+		            keywords += " " + strTemp;
+		        }
+		    }
+		   	    
+            boolean eof = false;
+			
+		    while( !eof )
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    //System.out.println(line);
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {   
+	    		    keywords = new String();
+		    
+		            st = new StringTokenizer(line);
+		            while( st.hasMoreTokens() )
+		            {
+		                String strTemp = st.nextToken();
+		                
+		                // check whether token is last token
+		                // assume last token is the query number
+		                if( !st.hasMoreTokens() )
+		                {
+		                    queryID.addElement(strTemp);
+		                    queryString.addElement(keywords);
+		                    //System.out.println(keywords);
+		                }
+		                else
+		                {
+		                    keywords += " " + strTemp;
+		                }
+		            }
+        		}
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}
+	}
+        
+    public Object[] getQueryObjectArray()
+    {
+        Object[] objArray = new Object[queryString.size()];
+	    for( int i = 0; i < queryString.size(); i++ )
+		{
+		    objArray[i] = queryString.elementAt(i);
+		}
+		return objArray;
+    }
+        
+    public Vector getQueryIDs()
+    {
+        return queryID;
+    }
+    
+    
+    public void printRelevance()
+    {
+        for( int i = 0; i < relevance.length; i++ )
+        {
+            for( int j = 0; j < relevance[i].size(); j++ )
+            {
+                System.out.print((String)relevance[i].elementAt(j));
+            }
+            System.out.println();
+            System.out.println();
+        }
+    }
+    
+    // returns vector with first element = total of relevant followed
+    // by precision at 10, 20...
+    public Vector getPerformance(String query, String rankedTrecIDs[])
+    {
+        int num = 0;
+        Vector precisionAt = new Vector();
+        int index = queryString.indexOf(query);
+        //System.out.println("Query number " + index + " selected");
+        if( index == -1 )
+            return precisionAt;  // return empty vector
+                
+        boolean relevant[] = new boolean[rankedTrecIDs.length];
+        //System.out.println("Size of ranked list: " + rankedTrecIDs.length);
+        
+        for( int i = 0; i < rankedTrecIDs.length; i++ )
+        {
+            // determine for each doc in the list whether it is relevant
+            if( relevance[index].contains(rankedTrecIDs[i]) )
+            {
+                relevant[i] = true;
+                num++;
+            }
+            else 
+                relevant[i] = false;
+                
+            // precision at 10,20,30,40,50    
+            if( (i+1) % 10 == 0 && (i+1)/10 < 6)
+            {
+                Integer prec = new Integer(Math.round(((float)num/(i+1)*100)));
+                
+                //System.out.println((double)num/(i+1));
+                precisionAt.addElement(prec);
+                
+            }    
+        }
+        
+        precisionAt.addElement(new Integer(relevance[index].size()));
+        precisionAt.addElement(new Integer(num));
+        //System.out.println(num + " relevant docs from a total of " + relevance[index].size() + " have been found");
+        
+        return precisionAt;
+  }
+    
+}
+
+/*		int totalRel = no_in_cat[categories.indexOf(queryId)]-1; // number of relevant
+		int relSoFar = 0;
+		
+		int pre_vs_rec[] = new int[11];
+			
+				
+		// trec method to compute average precision
+		// compute P for relevant images only
+		// P(r0) = max(P(r >= r0))
+
+		
+		// the following array stores pre/rec for every relevant, the first 
+		// entry is precision for the 0th item = 0.
+		// maximum size of array = items.length
+		// actual size of array = totalRel
+		
+		int recall = 0;
+		int precision = 0;
+		
+		
+		// begin of Rueger method
+		
+		relSoFar = totalRel;
+            
+	   	    // i  = 1 because target img is at zero!!!
+	   	    int maxPrecision = 0;
+	   	    int cutL = 10;
+		    for( int i = no_of_items -1; i > 0; i-- )
+			{
+				String listId = new
+			    String((String)cat_of_item.elementAt((int)list[i]));
+				if( listId.equals(queryId) )
+			    {
+			        recall = (int)(100*relSoFar)/totalRel;
+			        
+			        precision = (int)(100*relSoFar)/i;
+			        
+
+			        double cut = (double)(recall)/10;
+			        
+			        //if( r == 6 ) System.out.print(recall + " " + precision + ":");
+			        			        
+                    // if hit the next cut-off cutL, then set cut-off value to whatever 
+                    // is maximum
+			        if( cut == cutL )
+			        {
+			            //System.out.print("eq ");
+			            if( precision > maxPrecision )
+			            {
+			              //  System.out.print("eq-cm(" + precision + ") ");
+			                maxPrecision = precision;
+			            }
+                        
+                        //System.out.print("eq-ap(" + maxPrecision + ") ");
+			            pre_vs_rec[cutL] = maxPrecision;
+			            cutL--;
+			        }
+			        
+			        // if moved to the left of cut-off, the cut-off value (and those
+			        // lying between it and the last one set) is given the 
+			        // current maxPrecision
+			        else if( cut < cutL )   
+			        {
+			            for( int c = (int)cut+1; c < cutL+1; c++ )
+			            {
+			                pre_vs_rec[c] = maxPrecision;
+			            }
+			            
+			            
+			            if( precision > maxPrecision )
+			            {
+			                maxPrecision = precision;
+			            }
+			            
+			            cutL = (int)cut;
+    
+			        }
+		
+		            
+			        
+			        // just in case cut > cutL, we need still to update maxPrecision
+			        if( precision > maxPrecision )
+			        {
+			            maxPrecision = precision;
+			        }
+			        relSoFar--;
+			    
+			    }
+			    // if not relevant, nothing happens
+		    }
+		    
+		    for( int z = 0; z < cutL+1; z++ )
+		    {
+		        pre_vs_rec[z] = maxPrecision;
+		    }
+		    	    
+            // end of Rueger method
+		
+		// average precision for this query                                                     
+		double area = 0;
+		for(int j = 0; j < 10; j++ )
+		{
+			area += (pre_vs_rec[j]+pre_vs_rec[j+1])/2* 0.1;
+		}
+			
+		Performance p = new Performance(area, pre_vs_rec, queryId);
+		return p;
+  
+	}
+*/
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/CacheManager.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 21112)
@@ -0,0 +1,169 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+
+
+public class CacheManager
+{
+    
+    int cacheSize;
+    Vector file_names_dat;
+    Vector file_names_doc;
+        
+    public CacheManager()
+    {
+        //getCache();
+    }
+    
+    private void getCache()
+    {
+        Properties p = System.getProperties();
+        String currentDir = p.getProperty("user.dir");
+        File cache_dir = new File(currentDir);
+        file_names_dat = new Vector();
+        file_names_doc = new Vector();
+        
+        String fileArr[] = null;
+        if( cache_dir.exists() )
+        {
+        
+            fileArr = cache_dir.list(new FilenameFilter()
+            {
+                public boolean accept(File dir, String name)
+		        {
+		            return ( name.endsWith(".dat") || name.endsWith(".doc") );
+		        }
+	        });
+	    }
+	    else 
+	    {
+	        System.out.println("Directory does not exist");
+	        System.exit(0);
+	    }
+	    
+	    for( int i = 0; i < fileArr.length; i++ )
+	    {   
+	        if( fileArr[i].endsWith(".dat") )
+	            file_names_dat.addElement(fileArr[i]);
+	        if( fileArr[i].endsWith(".doc") )
+	            file_names_doc.addElement(fileArr[i]);
+	    }	    
+    }
+           
+    public void addDataItem(DataBlock data, String key)
+    {   
+        String fileName = new String(key +".dat");
+        file_names_dat.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(data);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+    
+     public void addDocItem(Vector doc, String key)
+    {   
+        String fileName = new String(key +".doc");
+        file_names_doc.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(doc);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+   
+        
+    public boolean containsData(String query)
+    {
+        String path = new String(query+".dat");
+        
+        if( file_names_dat.contains(path) )
+            return true;
+            else
+            return false;
+    }
+    
+    public boolean containsDoc(String docName)
+    {
+        String path = new String(docName+".doc");
+        
+        if( file_names_doc.contains(path) )
+            return true;
+            else
+            return false;
+    }
+  
+    
+    public Vector getDoc(String docName)
+    {
+        String filename = new String(docName + ".doc");
+        Vector doc = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            doc = (Vector)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return doc;   
+    }
+    
+    
+ 
+    public DataBlock getData(String query)
+    {
+        String filename = new String(query + ".dat");
+        DataBlock data = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            data = (DataBlock)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return data;   
+    }
+    
+    public int getSize()
+    {
+        return file_names_dat.size();
+    }
+    
+    public Vector listCache()
+    {
+        return file_names_dat;
+    }
+    
+    public void printMe()
+    {
+        for( int i = 0; i < file_names_dat.size(); i++ )
+        {
+            String object = (String)file_names_dat.elementAt(i);
+            System.out.println(object);
+        }
+    }
+  
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 21112)
@@ -0,0 +1,69 @@
+/**
+ * @(#)ClusterGroupObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+
+public class ClusterGroupObj
+{
+    ClusterObj[] clusters;  //-- list of ClusterObj ref
+    String label;  //-- a general description for the group (take the highest related word)
+    private int groupSize;  //-- total number of docs in the group
+    public ClusterGroupObj(ClusterObj[] c, String ls)
+    {
+        //System.out.println("ClusterGroupObject constructor called label " + ls);
+        //System.out.flush();
+        clusters=c; label=ls;
+        groupSize=0;
+        for (int i=0;i<clusters.length;i++)
+      		groupSize+=clusters[i].getDocsSize();
+	}
+
+
+ public String getLabels()
+ {
+   return label;
+ }
+
+
+  public String getLabels_toString()
+  {
+    return label;
+  }
+/*
+  	String tmp="";
+  	for (int i=0;i<label.length;i++) tmp = tmp + " " + label[i];
+  	return tmp;
+  }
+*/
+  /**
+   * @return the total number of documents in the group
+   */
+  public int getGroupSize(){return groupSize;}
+
+  /**
+   * @return the number of clusters in the group
+   */
+  public int getClustersSize(){return clusters.length;}
+
+
+  /**
+   * @return the clusters list
+   */
+  public ClusterObj[] getClusters(){return clusters;}
+
+
+  public String getInfo(){
+  	return "No of clusters: "+ String.valueOf(clusters.length)+ " totally contains: "+
+  		String.valueOf(getGroupSize())+" docs\nGroup Label: "+getLabels_toString()+"\n";
+  }
+
+  public void showInfo(){
+    System.out.print(getInfo());
+    for (int i = 0; i<getClustersSize(); i++)
+      System.out.print( i+")  "+clusters[i].getInfo());
+  }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 21112)
@@ -0,0 +1,77 @@
+/**
+ * @(#)ClusterObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.Vector;
+
+public class ClusterObj
+{
+    DocObj[] _docs;		//-- list of DocObj ref
+    String _label;		//-- (top 3-5? related words)
+    Point2D [] _sammon;			//-- 2D mapping for the centroids
+    DataManager _data;
+    public ClusterObj(DocObj[] d, String t, Point2D [] sammon, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _sammon = sammon;
+        _data = dm;
+    }
+  /**
+   * @return the number of documents in the cluster
+   */
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    Vector rwVec = new Vector(s);
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DataManager.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 21112)
@@ -0,0 +1,627 @@
+package vishnu.testvis.object;
+
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.dendro.*;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.io.*;
+import java.net.*;
+
+import vishnu.testvis.sammon.*;
+import vishnu.util.Sammon;
+import vishnu.cluster.Clustering;
+import vishnu.testvis.browser.*;
+
+public class DataManager
+{
+    private Vishnu displayFrame;
+    private DataBlock data = null;
+    private CacheManager cache = null;
+    public Vector collections;
+    public Collection currentCollection = null;
+    public String currentView = null;
+    /** Labels for the clusters needed by tree map,
+     *  generated in this class */
+    private String [] clusterLabels = null;
+    /** vertical array of vectors representing
+     *  the documents that word x appears in */
+    private int [] clustersBigWord;
+    //private Vector [] wordDocuments;
+    /** ClusterObjs as used in Tree map */
+    private DocObj [] docObjs;
+    private ClusterObj [] clusterObjs = null;
+    private ClusterGroupObj [] clusterGroups = null;
+    private int[][] sommClusterWords;
+    private Vector sammonClusters;
+
+    private String engine_type = "Engine";
+    
+    /** Set gsdl state */
+    public void setEngineType(String engine) {
+	engine_type = engine;
+    }
+    /** Generate labels for the clusters needed by tree map,
+     *  pre: data is initialised
+     */
+
+    private void setUpClusterLabels()
+    {
+        int [] wordFreqs = new int [data.words.length];
+        StringBuffer sbuff= new StringBuffer(20);
+        clustersBigWord = new int[data.clusters.length];
+        clusterLabels= new String[data.clusters.length];
+        sommClusterWords= new int [data.clusters.length][4];
+
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+
+            for(int d=0; d < data.clusters[c].size(); d++)
+            {
+                Integer n = (Integer) data.clusters[c].elementAt(d);
+                int docNo = n.intValue();
+                SparseNode row = data.matrix.getRow(docNo);
+                while( row!= null)
+                {
+                    wordFreqs[row.getColumn()]++;
+                    row = row.getNext();
+                }
+            }
+
+            int col1 = -1, c1val = 0;
+            int col2 = -1, c2val = 0;
+            int col3 = -1, c3val = 0;
+            int col4 = -1, c4val = 0;
+            for(int d=0; d < data.words.length; d++)
+            {
+                if(c1val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=col1;
+                    c2val=c1val;
+                    col1=d;
+                    c1val=wordFreqs[d];
+                }
+                else if(c2val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=d;
+                    c2val=wordFreqs[d];
+                }
+                else if(c3val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=d;
+                    c3val=wordFreqs[d];
+                }
+                else if(c4val < wordFreqs[d])
+                {
+                    col4=d;
+                    c4val=wordFreqs[d];
+                }
+
+            } // for (each document in the cluster)
+           if(col1 > -1)
+           {
+                sbuff.append(data.words[col1]);
+                if(col2 > -1)
+                {
+                    sbuff.append(" ");
+                    sbuff.append(data.words[col2]);
+                    if(col3 > -1)
+                    {
+                        sbuff.append(" ");
+                        sbuff.append(data.words[col3]);
+                    }  // if
+                } // if
+                clusterLabels[c] = new String(sbuff);
+                sbuff.setLength(0);
+            } //if
+            else
+                clusterLabels[c] = new String("<empty>");
+
+            clustersBigWord[c] = col1;
+            sommClusterWords[c][0] = col1;
+            sommClusterWords[c][1] = col2;
+            sommClusterWords[c][2] = col3;
+            sommClusterWords[c][3] = col4;
+        } // for (each cluster)
+    } // end of setUpClusterLabels()
+    
+
+    private void setUpDocObjs()
+    {
+        docObjs = new DocObj[data.docs.length];
+        for(int c=0;c<data.docs.length;c++)
+            docObjs[c] = new DocObj(c, data.descriptions[c], data.matrix.getRow(c), data.docs[c]);
+    }
+    
+    public DocObj getDocObjAt(int index)
+    {
+        return docObjs[index];
+    }
+
+    public int getClusterGroupsSize()
+    {
+        return clusterGroups.length;
+    }
+
+    public ClusterGroupObj getClusterGroup(int i)
+    {
+        return clusterGroups[i];
+    }
+    private void setUpClusterObjs()
+    {
+        clusterObjs = new ClusterObj[data.clusters.length];
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            DocObj [] dObjArray = new DocObj[data.clusters[c].size()];
+            for(int d=0; d<data.clusters[c].size();d++)
+            {
+                Integer n = (Integer)data.clusters[c].elementAt(d);
+                dObjArray[d] = docObjs[n.intValue()];
+            }
+            clusterObjs[c]= new ClusterObj(dObjArray,clusterLabels[c],data.sammon,this);
+        }
+        //ClusterObj(DocObj[] d, String t, Point2D sammon, DataManager dm)
+        // !!!!remember to comment out this loop when it works
+        //for(int c=0;c<data.clusters.length;c++)
+        //    System.out.println("In cluster " + c  + clusterObjs[c].getDocsSize() + " documents ");
+    }
+    /** This for tree map as it needs clusters of clusters
+     * this is a cheap and probably not very satisfactory solution
+     * as all I am doing is taking clusters with the common most significant
+     * word and putting them in groups.
+     */
+    private void setUpClusterGroupObjs()
+    {
+        int count = data.clusters.length;
+        int num=0, i,j;
+        int [] sizes = new int[count];
+        for(i=0;i < count;i++)
+            sizes[i]=1;
+
+        // contains #supercluster clusterList objects, each a list of clusters
+        Vector groupList = new Vector(count);
+        
+        // contains #supercluster BigWords for each supercluster
+        Vector wordList = new Vector(count);
+        for(i=0;i < count;i++)
+        {
+            if(sizes[i]>0)
+            {
+                Vector clusterList = new Vector(count);
+                clusterList.addElement(clusterObjs[i]);
+                int bigWord = clustersBigWord[i];
+                for (j = i+1; j < count; j++)
+                    if (clustersBigWord[j] == bigWord)
+                    {
+                        sizes[i]+=sizes[j];
+                        if(sizes[j]>0)
+                            clusterList.addElement(clusterObjs[j]);
+                        sizes[j]=0;
+                    }
+                groupList.addElement(clusterList);
+                wordList.addElement(new Integer(bigWord));
+            }
+        }
+
+        clusterGroups = new ClusterGroupObj[groupList.size()];
+
+        for(i=0;i < groupList.size();i++)
+        {
+            Vector clusterList = (Vector)groupList.elementAt(i);
+            if(clusterList == null)
+                System.out.println("Cluster list "+ i + " is null");
+            else
+            {
+                //ClusterObj [] clusterObjsArray = clusterList.toArray();
+                int groupSize = clusterList.size();
+                ClusterObj [] clusterObjsArray = new ClusterObj [groupSize];
+                for (j=0; j < groupSize;j++)
+                    clusterObjsArray[j] = (ClusterObj) clusterList.elementAt(j);
+                //System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                Integer bigWord = (Integer)wordList.elementAt(i);
+                int bw = bigWord.intValue();
+                String word;
+                if (bw <0)
+                {
+                    System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                    System.out.println("bigword is empty");
+                    word = "< Empty >";
+                }
+                else
+                    word = data.words[bigWord.intValue()];
+                clusterGroups[i] = new ClusterGroupObj(clusterObjsArray,word);
+            }
+        }
+    }
+    
+    
+    private void printData(String[] str, String filename)
+    {
+        try
+	    {
+	        filename += ".txt";
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    PrintWriter out = new PrintWriter(fw,true);
+		    String line = new String();
+		   			
+			    for( int j = 0; j < str.length; j++ )
+			    {
+			        line = str[j] + " ";
+			        out.println(j + " " + line);
+			        System.out.println(j + " " + line);
+		        }
+		    
+		    out.close();
+	    } catch (IOException e){ System.out.println("Error " + e.toString());
+	    } catch (SecurityException se) { System.out.println("Error " + 	se.toString());}
+	}
+	
+	public void setUpDendroClusters()
+	{
+	    displayFrame.dendroPanel.treePanel.getData();
+	}
+    
+       
+    
+    public int query(String queryStr)
+    {
+        if( !displayFrame.refine )
+        {
+            if( displayFrame.firstSearch )
+            {
+                displayFrame.firstSearch = false;
+            }
+        }
+                
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return 0;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine theEngine = new GSDLEngine(displayFrame);
+	System.out.println("Searching for " + queryStr);
+	DataBlock db =  theEngine.getData(queryStr, currentCollection.dir, currentView);
+
+        if( db == null ) return 0;
+
+	data = db;
+	
+        int spaceIndex=0;
+        int count = 0;
+        int wordNum = 0;
+        int clusterNum=0;
+        StringBuffer qbuff = new StringBuffer();
+            
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+        if( !displayFrame.refine ) setUpDendroClusters();
+        
+	count = this.data.docs.length;
+	wordNum = this.data.words.length;
+	clusterNum = this.data.clusters.length;
+	
+        if( !displayFrame.refine )
+	    {
+		displayFrame.searchStarted = true;
+	    }
+        
+	displayFrame.update();
+        return count;
+    }
+
+
+    public void fetch(int index)
+    {
+        if(data==null) return ;
+        if(index < 0 || index >= data.docs.length) return;
+        int docNum = data.docs[index];
+        fetchDocNo(docNum);
+    }
+
+
+    public void fetchDocNo(long docNum)
+    {
+        System.out.println("Fetch document: " + Long.toString(docNum));
+
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine engine = new GSDLEngine(displayFrame);
+	Vector document = theEngine.getDocument(Long.toString(docNum),currentCollection.dir, currentView);
+	
+	// Fetch theFetch = new Fetch(displayFrame);
+        // document = theFetch.getDocument(docNum);
+        
+	Browser browser;
+	if (document.size() > 1) {
+	    String url = (String)document.elementAt(1);
+	    
+	    if( url.indexOf("http") != -1 )
+		{
+		    try{
+			displayFrame.getAppletContext().showDocument(new URL(url),"_blank");
+			// http://icarus.umkc.edu/cgi-bin/ptext?doc=perseus:text:1999.01.0154"),"_blank");
+		    }catch(IOException ex){ex.printStackTrace();};
+		}     
+	    else browser = new Browser(document,docNum);
+	} else { 
+	    browser = new Browser(document,docNum);
+	}
+    }
+    
+
+    public String [] getClusterLabels()
+    {
+        return clusterLabels;
+    }
+
+
+    public String [] getDescriptions()
+    {
+        if(data != null)
+            return data.descriptions;
+        else return new String[0];
+    }
+    
+    
+    public String [] getTrecIDs()
+    {
+        if(data != null)
+        {
+            String trecID[] = new String[data.descriptions.length];
+            for( int i = 0; i < data.descriptions.length; i++ )
+            {
+                StringTokenizer st = new StringTokenizer(data.descriptions[i]);
+                trecID[i] = st.nextToken();             
+            }
+            return trecID;
+        }
+        else return new String[0];
+    }
+    
+       
+    public String getDescriptionStr(Vector subset)
+    {
+        String returnStr = new String("");
+        for(int c = 0; c < subset.size();c++)
+        {
+            Integer n = (Integer)(subset.elementAt(c));
+            returnStr = returnStr + data.descriptions[n.intValue()] + "\n";
+        }
+        return returnStr;
+    }
+    public int [] getDocs()
+    {
+        if(data != null)
+            return data.docs;
+        else return new int[0];
+    }
+
+    public String [] getWords()
+    {
+        if(data != null)
+        {
+            if(data.words==null)
+            {
+                System.out.println("No words in data!");
+                return new String[0];
+            }
+            return data.words;
+        }
+        else return new String[0];
+    }
+
+    public SparseMatrix getMatrix()
+    {
+        if(data != null)
+            return data.matrix;
+        else return null;
+    }
+
+    public Vector [] getClusters()
+    {
+        if(data != null)
+            return data.clusters;
+        else return null;
+    }
+    public Point2D [] getSammon()
+    {
+        if(data != null)
+            return data.sammon;
+        else return new Point2D[0];
+    }
+
+    public DataManager(Vishnu d) {
+	this(d, null);
+    }
+    public DataManager(Vishnu d, String engine)
+    {
+        displayFrame = d;
+	if (engine != null && !engine.equals("")) {
+	    engine_type = engine;
+	}
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+
+	theEngine.setDisplay(displayFrame);
+	Vector q = new Vector();
+	q.addElement("initialise");
+	collections = theEngine.getCollections(q); 
+    }
+
+
+    // This constructor creates a smaller data manager from
+    // several clusters from a larger one
+    // it is intended to be used with sammon mapping drilling down
+    // The clustering code is lifted from the C version and could be
+    // improved
+    
+    public DataManager(DataBlock oldData, Vector idVect, Vishnu d)
+    {
+        displayFrame = d;
+        
+        // create new document/keyword matrix with only the 
+        // documents from the selected cluster, but keeping
+        // the same 100 keywords as columns
+        SparseMatrix m = new SparseMatrix(idVect.size(),
+                                          oldData.matrix.getColumnCount());
+
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+          
+          // ptr to the first document of ith row (linked list)
+          SparseNode row = oldData.matrix.getRow(i.intValue());
+          // add the row to new lean matrix
+          while(row != null)
+          {
+              m.addDataItem(c,row.getColumn(),row.getValue());
+              row=row.getNext();
+          }
+        }
+        
+        float [][] full = m.getExpanded(); 
+        
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+        int clusterNum =  m.getRowCount()>20?10:m.getRowCount()/2;
+        
+        // limit set to 20 if > 20 docs otherwise to 3/4 #docs
+        int limit =  m.getRowCount()>20?20:(m.getRowCount()/2)+
+                      (m.getRowCount()/4);
+                      
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,
+					       full,/*m.getRowCount(),
+						      m.getColumnCount(),*/1,
+					       clusterNum,limit);
+                                                    
+        Vector [] clusters = clustering.getClusters();
+        double [][] centroids= clustering.getCentroids();
+
+        Sammon map = new Sammon(centroids);
+        Point2D [] sammon = map.getMapping();
+        String [] words = oldData.words;
+        int [] newDocs = new int[idVect.size()];
+        String [] newDesc = new String[idVect.size()];
+        for (int c=0;c<idVect.size();c++)
+        {
+          Integer i = (Integer) idVect.elementAt(c);
+          int index = i.intValue();
+          newDocs[c] = oldData.docs[index];
+          newDesc[c] = oldData.descriptions[index];
+        }
+        data = new DataBlock(newDocs,newDesc,words,m,clusters,sammon);
+        
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+    }
+
+    public DataBlock getDataBlock()
+    {
+        return data;
+    }
+    
+    public int getWordCount()
+    {
+        if (data!=null)
+            return data.words.length;
+        else return 0;
+    }
+    public int getDocCount()
+    {
+        if (data!=null)
+            return data.docs.length;
+        return 0;
+    }
+
+    public Vector getSammonClusters()
+    {
+        if(data != null)
+            return sammonClusters;
+        else return null;
+    }
+
+    private void setUpSammClusters()
+    {
+        LengthTable [] table = new LengthTable [data.clusters.length];
+        
+        // create as many sammon clusters as there are clusters in the data object
+        // and store it in sammmonClusters
+        sammonClusters = new Vector(data.clusters.length);
+        
+        for (int c = 0; c < data.clusters.length; c++){
+	    table[c]= new LengthTable(c,data.clusters[c].size());
+        }
+        
+        
+        // sort by cluster size, I guess
+        QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+        try
+        {
+            qSort.sort(table);
+        }
+        catch (Exception e)
+        {}
+        
+        
+        for (int c = 0; c < data.clusters.length; c++)
+        {
+            int i = table[c].index;
+             SammCluster cluster = new SammCluster(data.clusters[i],
+                        data.sammon[i], sommClusterWords[i],
+                        clusterObjs[i]);
+            sammonClusters.addElement(cluster);
+        }
+    }
+
+    public void requestCollection()
+    {
+	// should reset the data
+	data = null;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DocObj.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 21112)
@@ -0,0 +1,89 @@
+/**
+ * @(#)DocObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ * Adapted to use sparce matrix Matthew Carey 7/7/2000
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.*;
+public class DocObj
+{
+    private int _number;
+    private int _mgNumber;
+    private String _description;
+//--!!! we don't need to store the whole number, as the actual value is not very important
+//--!!! but we need to keep track the doc contains particular related words or not
+//--!!! therefore, one bit will do
+    private SparseNode _weights, _sparceNode;
+
+
+    public DocObj(int num, String s,SparseNode w, int mgn)
+    {
+        _number = num;
+        _description=s;
+        //_fields=f;
+        _weights=w;
+        _mgNumber = mgn;
+    }
+
+  //public void setWeights(int[] w){_weights=w;}
+    public int getNumber()
+    {
+        return _number;
+    }
+    public String getDescription()
+    {
+        return _description;
+    }
+
+    public SparseNode getWeights()
+    {
+        return _weights;
+    }
+    public float getWeights(int i)
+    {
+        SparseNode test = _weights;
+        while(test != null)
+        {
+            if(test.getColumn() == i) return test.getValue();
+            test = test.getNext();
+        }
+        return 0;
+    }
+    static public String getANumberOfWords(String in, int num)
+    {
+        StringTokenizer st = new StringTokenizer(in);
+        StringBuffer buff = new StringBuffer();
+        for(int c=0;c<num && st.hasMoreTokens() ;c++)
+        {
+            if (c>0) buff.append(" ");
+            buff.append(st.nextToken());
+        }
+        String retString = new String(buff);
+        return retString;
+    }
+    
+    public String getTrecID()
+    {
+        StringTokenizer st = new StringTokenizer(_description);
+        return st.nextToken();             
+    }
+
+    public String format4Fields()
+    {
+        String s = "";
+        s += "<dl>\n";
+        s += "  <dt> <a HREF=\"" + _mgNumber +"\">"+ "Document: " + _mgNumber + "</a>\n";
+        //s += "  <dd> <b>"+ getANumberOfWords(_description,5) +"</b><br> \n";
+	String modDesc = _description.replaceAll("<"," ");
+	modDesc = modDesc.replaceAll(">"," ");
+        s += "    <dd> <font color=gray> " + modDesc + " </font>\n";
+        s += "</dl>\n";
+	System.out.println(s);
+        return s;
+  }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/Engine.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/Engine.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/Engine.java	(revision 21112)
@@ -0,0 +1,118 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+
+public class Engine
+{
+    Vishnu display;
+    String servlet;
+
+    public Engine() {
+    }
+
+    public Engine(Vishnu display)
+    {
+	this.setDisplay(display);
+	
+    }
+    
+    public void setDisplay(Vishnu display) {
+	this.display = display;
+	servlet = display.server_address;
+	System.out.println("servlet= "+servlet);
+
+    }
+    
+    public Vector getCollections(Vector q)
+    {
+	Vector c = null;// = new Vector();
+	try{
+	    URLConnection con = getConnection();
+            System.out.println("Sending request for collections");
+	    sendObject(con,q);
+	    c = (Vector)receiveObject(con);
+	} catch( Exception ex) {System.out.println(ex);}
+	return c;
+    }
+    
+    public DataBlock getData(String query, String collection, String view)
+    {
+        DataBlock data = null;
+        Vector q = new Vector();
+	q.addElement("search");
+        q.addElement(query);
+	q.addElement(collection);
+	q.addElement(view);
+	
+
+        try{
+            URLConnection con = getConnection();
+            sendObject(con,q);
+	    Vector v = (Vector)receiveObject(con);
+	    if( v == null || v.size()==0 ) {
+		System.out.println("Engine.getData: Null or empty result");
+	    } else {
+		data = (DataBlock)v.elementAt(0);
+	    }
+        } catch( Exception ex)  {
+	    System.out.println("Engine.getData: "+ex);
+	}
+	
+	return data;
+    }
+
+    
+    public Vector getDocument(String id, String collection, String view)
+    {
+	Vector q = new Vector();
+	Vector resp = new Vector();
+	q.addElement("fetch");
+	q.addElement(id);
+	q.addElement(collection);
+	q.addElement(view);
+
+	try{
+	    URLConnection con = getConnection();
+	    sendObject(con,q);
+	    resp = (Vector)receiveObject(con);
+	} catch(Exception ex) {ex.printStackTrace();}
+	return (Vector)resp.elementAt(0);
+    }
+
+
+    private URLConnection getConnection() throws IOException
+    {
+        URL u = new URL(servlet);
+        URLConnection con = u.openConnection();
+        con.setDoInput(true);
+        con.setDoOutput(true);
+        con.setUseCaches(false);
+	con.setRequestProperty("Content-type","application/octet-stream");
+        con.setAllowUserInteraction(false);
+        return con;
+    }
+    
+    
+    private void sendObject(URLConnection con, Object obj) throws IOException {
+        ObjectOutputStream out = new  ObjectOutputStream(con.getOutputStream());
+        if( obj != null )
+	    {
+                out.writeObject(obj);
+	    }
+        out.close();
+    }
+    
+    
+    private Object receiveObject(URLConnection con) throws Exception
+    {
+	Object obj;
+	InputStream is = con.getInputStream();
+	ObjectInputStream in = new ObjectInputStream(con.getInputStream() );
+	obj = in.readObject();
+	in.close();
+	return obj;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 21112)
@@ -0,0 +1,107 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.server.Packaging;
+import vishnu.util.XMLUtil;
+import vishnu.util.Base64;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+
+public class GSDLEngine extends Engine
+{
+    public GSDLEngine() {
+    }
+    
+    public GSDLEngine(Vishnu display)
+    {
+	this.setDisplay(display);
+    }
+        
+    public Vector getCollections(Vector q) {
+	return null;
+    }
+    
+    public DataBlock getData(String query_term, String collection, String view) {
+
+	
+	String query = servlet+"&type=search&q="+query_term;
+	if (view != null) {
+	    query += "&view="+view;
+	}
+	System.out.println("sending query: " + query_term);
+	DataBlock db = null;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    
+	    // unserialize the data block
+	    db = (DataBlock) Base64.decodeToObject(results);
+	    //byte [] bytes = results.getBytes("UTF-8");
+// 	    byte [] bytes = Base64.decode(results);
+// 	    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+// 	    ObjectInputStream ois = new ObjectInputStream(bais);
+// 	    db = (DataBlock)ois.readObject();
+//	    ois.close();
+// 	    Element cluster_results = (Element)XMLUtil.getChildByTagName(data_elem, "cluster");
+// 	    String results = XMLUtil.getNodeText(cluster_results);
+// 	    //System.out.println("recived results:" + results);
+// 	    Element description_results = (Element)XMLUtil.getChildByTagName(data_elem, "descriptions");
+// 	    NodeList descs = description_results.getElementsByTagName("desc");
+// 	    Vector desc_vector = new Vector();
+// 	    for (int i=0; i<descs.getLength(); i++) {
+// 		String d = XMLUtil.getNodeText((Element)descs.item(i));
+// 		desc_vector.add(d);
+// 	    }
+// 	    Packaging p = new Packaging();
+// 	    db = p.getDataPackage(results, desc_vector);
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return db;
+
+    }
+
+    public Vector getDocument(String id, String collection, String view) {
+	String query = servlet+"&type=fetch&d="+id;
+	if (view != null) {
+	    query += "&view="+view;
+	}	
+	System.out.println("sending fetch: " + id);
+	Vector doc;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    System.out.println("recived results:" + results);
+	    // ugly hack. make this nicer!!
+	    doc = new Vector(Arrays.asList(results.split("\n")));
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return doc;
+		
+    }
+    
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/LengthTable.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 21112)
@@ -0,0 +1,21 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.object;
+
+public class LengthTable
+{
+    public int index;
+    public int length;
+    public LengthTable(int ind, int len)
+    {
+        index= ind;
+        length = len;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 21112)
@@ -0,0 +1,450 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.awt.image.*;
+
+class ClusterPanel extends JPanel
+{
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    private SammCluster currentCluster;
+    private final int RADIUS = 30;
+    private SammCluster drawnCluster;
+    private GraphicalPane parentSammon;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    int locationID = -1;
+    JPopupMenu[] popups;
+    JPopupMenu popup;
+    Vishnu _currentMain;
+    DataManager dm; 
+
+    public ClusterPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.parentSammon = parent;
+        _currentMain=currentMain;
+	dm = _currentMain.dataManager;
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+
+        setOpaque(true);
+        setPreferredSize(new Dimension(600,600));
+        setBorder(BorderFactory.createCompoundBorder(
+            BorderFactory.createEmptyBorder(20,20,20,20),
+            BorderFactory.createLoweredBevelBorder()));
+            
+        addComponentListener(
+            new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+            {
+                if( parentSammon.dataMan != null && parentSammon.dataMan.getDocCount()>0){
+		    repaint();
+                }
+
+            }
+        }
+        );
+
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int docsInSelectedClusters=0;
+                
+                if( parentSammon.dataMan == null) return;
+                int oldLocationID = locationID;
+                
+                Vector clusters = parentSammon.dataMan.getSammonClusters();
+                if (clusters == null) return;
+                locationID = -1;
+                for (int i = clusters.size()-1 ; i >= 0 ; i--)
+                {
+
+                    SammCluster cluster = (SammCluster) clusters.elementAt(i);
+                    int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+                    int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) +RADIUS;
+
+                    if (inCircle(mouseEvent,x,y,cluster.getDisplayedRadius()))
+                    {
+                        locationID = i;
+                    }
+                    if(cluster.getSelected())
+                      docsInSelectedClusters+=cluster.documentIDs.size();
+                }
+
+                if ((oldLocationID != locationID) && (oldLocationID != -1))
+                    popup.setVisible(false);
+
+
+                if (locationID != -1)
+                {
+		    if (locationID != oldLocationID)
+                    {
+               
+                        popup.removeAll();
+                             
+                        String[] allWords = parentSammon.dataMan.getWords();
+                        
+                        currentCluster = (SammCluster) clusters.elementAt(locationID);
+                        String str =  currentCluster.getSize() + " documents";
+                        JMenuItem menuItem = new JMenuItem(str);
+                        popup.add(menuItem);
+
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent actionEvent)
+                            {
+                                drawnCluster = currentCluster;
+                                updateHLinkPanel();
+                                repaint();
+				//update
+                            }
+                        });
+
+                        popup.add(new JPopupMenu.Separator());
+                        menuItem = new JMenuItem("Drill down");
+
+                        if(!currentCluster.getSelected())
+                          docsInSelectedClusters+=currentCluster.documentIDs.size();
+                                                   
+                        if(docsInSelectedClusters<7)menuItem.setEnabled(false);
+                            else menuItem.setEnabled(true);
+                        
+                        // for drilldown
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                Vector idVector;
+                                SammCluster thisCluster = getCurrentCluster();
+                                                                
+                                int theIndex = thisCluster.getWords()[0];
+                                String drillWords = new String(parentSammon.dataMan.getWords()[theIndex]);
+                                for( int i = 1; i < thisCluster.getWords().length; i++ )
+                                {
+                                    theIndex = thisCluster.getWords()[i];
+                                    drillWords += " " + parentSammon.dataMan.getWords()[theIndex];
+                                }
+				
+				Vector clusterPl = parentSammon.dataMan.getSammonClusters();
+
+                                Vector idVect = (Vector)thisCluster.getIDs().clone();
+                                
+                                thisCluster.setSelected(true);
+                                for (int i = clusterPl.size()-1; i >= 0; i--)
+                                {
+
+                                  SammCluster cluster = (SammCluster) clusterPl.elementAt(i);
+                                  if (!cluster.equals(thisCluster) && cluster.getSelected())
+                                    idVect.addAll(cluster.getIDs());
+                                }
+                                if(idVect.size()>6)
+                                {
+                                
+                                  parentSammon.drillDown(idVect);
+                                  
+                                  currentCluster=null;
+                                  drawnCluster=null;
+				  //update();
+				  repaint();
+                                }
+                                
+                            }
+                        });
+                        popup.add(menuItem);
+                        if(!currentCluster.getSelected())
+                          menuItem = new JMenuItem("Select");
+                        else
+                          menuItem = new JMenuItem("Deselect");
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                SammCluster cluster = getCurrentCluster();
+                                cluster.setSelected(!cluster.getSelected());
+                                //update();
+				repaint();
+                                
+                            }
+                        });
+
+                        popup.add(menuItem);
+                        int[] words = currentCluster.getWords();
+                        popup.add(new JPopupMenu.Separator());
+                        for(int c=0;c<4;c++)
+                        {
+                          int index = words[c];
+                          if (index>= 0)menuItem = new JMenuItem(allWords[index]);
+                          popup.add(menuItem);
+                          menuItem.setEnabled(false);
+                        }
+
+                        popup.validate();
+                        popup.setVisible(true);
+			
+                    }
+                    popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                    mouseEvent.consume();
+                }
+		
+            }
+	    });
+	
+        setVisible(true);
+    }
+
+    
+    public SammCluster getCurrentCluster() {return currentCluster;}
+    
+    
+    public void paint(Graphics g)
+    {
+	update(g);
+    }
+    
+
+    public boolean inCircle(MouseEvent mouseEvent, int x, int y, int radius)
+    {
+        int dx2 = (mouseEvent.getX()-x)*(mouseEvent.getX()-x);
+        int dy2 = (mouseEvent.getY()-y)*(mouseEvent.getY()-y);
+
+        return (dx2 + dy2 < radius*radius);
+    }
+    
+    
+    private void drawFlag(Graphics g,int x, int y)
+    {
+	g.drawLine(x,y,x,y-12);
+	int [] xs = new int[3];
+	int [] ys = new int[3];
+	xs[0]=x;xs[1]= x+10; xs[2]=x;
+	ys [0]=y-12; ys[1]=y-7;ys[2]=y-2;
+	g.fillPolygon(xs,ys,3);
+    }
+
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	Dimension d = getSize();
+	offScreenSize = d;
+	
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics.setColor(Color.white);
+	/*offGraphics.clearRect(0, 0, area.width, area.height);
+	offGraphics.draw(area);
+	offGraphics.fill(area);*/
+	offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	if (clusters.size()==0) return;
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+	    myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+	    }
+	
+	g2.drawImage(bi,0,0,this);
+    }
+    
+    
+    public void update()
+    {
+	/*        Dimension d = getSize();
+        offScreenSize = d;
+        offScreen = createImage(d.width,d.height);
+        offGraphics = (Graphics2D)offScreen.getGraphics();
+	
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            //offGraphics.setColor(new Color((float) (1 - 0.7*clusterWeight),(float) 0, (float) 0));
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+            BasicStroke myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+		}*/
+        repaint();
+    }
+
+
+    private void updateHLinkPanel()
+    {
+        if( parentSammon.dataMan==null) return;
+        
+        if (locationID != -1)
+	    {
+		
+		Vector clusters = parentSammon.dataMan.getSammonClusters();
+		
+		SammCluster selectedCluster = (SammCluster) clusters.elementAt(locationID);
+		ClusterObj cObj = selectedCluster.getClusterObj();
+		String htmlDescription = cObj.formatDocs();
+		
+		parentSammon.hLinkPanel.setText(htmlDescription);
+		parentSammon.keywordPanel.update(cObj);
+	    }
+    }
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 21112)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+
+/** Panel containing the sammon map interface */
+public class GraphicalPane extends JPanel
+{
+    public JPanel picturePanel;
+    public ClusterPanel clusterPanel;
+    public SammKeywordPanel keywordPanel;
+    public HLinkPanel hLinkPanel;
+    public Stack dataStack;
+    
+    public DataManager dataMan=null;
+    public Vishnu currentMain;
+    public JLabel pictureLabel = new JLabel("Level 0");
+    public JButton pictureButton = new JButton("Back");
+    public JPanel topPanel = new JPanel();
+    private int largeFont = 16;
+    private int smallFont = 14;
+    private int level = 0;
+    
+    public GraphicalPane(Vishnu currentMain)
+    {
+	this.currentMain = currentMain;
+	dataMan = currentMain.dataManager;
+	
+	dataStack=new Stack();
+	level=0;
+	setName("Sammon Map");
+	setLayout(new BorderLayout(10,10));
+	setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+	clusterPanel = new ClusterPanel(currentMain,this);
+	keywordPanel = new SammKeywordPanel(currentMain,this);
+	Dimension d = currentMain.getSize();
+	hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+	picturePanel = new JPanel();
+	picturePanel.setLayout(new BorderLayout(0,3));
+	topPanel.setLayout(new FlowLayout(FlowLayout.LEFT,10,0));
+	topPanel.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
+	pictureLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	pictureButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	topPanel.add(pictureLabel);
+	topPanel.add(pictureButton);
+	pictureButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    unDrill();
+		}
+	    });
+	
+	pictureButton.setEnabled(false);
+	picturePanel.add(topPanel,"North");
+
+	picturePanel.add(clusterPanel,"Center");
+	add("East", hLinkPanel.linkPane);
+	add("Center", picturePanel);
+	add("West", keywordPanel);
+	setVisible(true);
+    }
+    
+    public void drillDown(Vector idVect)
+    {
+	System.out.println("Drilling");
+
+	// save old data manager and create new one with old data
+	// and vector of document ids of the selected cluster
+	dataStack.push(dataMan);
+	dataMan = new DataManager(dataMan.getDataBlock(), idVect, currentMain);
+	level++;
+	pictureButton.setEnabled(true);
+	keywordPanel.clear();
+	hLinkPanel.clear();
+	pictureLabel.setText("Level "+ level);
+    }
+    
+    public void unDrill()
+    {
+	if(!dataStack.empty()){
+		dataMan = (DataManager)dataStack.pop();
+		hLinkPanel.clear();
+		keywordPanel.clear();
+		level--;
+		clusterPanel.repaint();
+		pictureLabel.setText("Level "+ level);
+	    }
+	else level = 0;
+	if(level==0)
+	    pictureButton.setEnabled(false);
+    }
+  
+    public void update()
+    {
+	dataMan = currentMain.dataManager;
+	dataStack=new Stack(); // dispose of old stack - create new empty one
+	level=0;
+	clusterPanel.repaint();
+	keywordPanel.clear();
+	hLinkPanel.clear();
+    }
+  
+    
+    public void addNotify()
+    {
+        Dimension d = getSize();
+	//Record the size of the window prior to calling parents addNotify.
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    /** Used for addNotify check. */
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 21112)
@@ -0,0 +1,80 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 21112)
@@ -0,0 +1,142 @@
+/*
+ * @(#)QSortAlgorithm.java        1.6 96/12/06
+ *
+ * Copyright (c) 1994-1996 Sun Microsystems, Inc. All Rights Reserved.
+ *
+ * Sun grants you ("Licensee") a non-exclusive, royalty free, license to use,
+ * modify and redistribute this software in source and binary code form,
+ * provided that i) this copyright notice and license appear on all copies of
+ * the software; and ii) Licensee does not utilize the software in a manner
+ * which is disparaging to Sun.
+ *
+
+ * This software is provided "AS IS," without a warranty of any kind. ALL
+ * EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, INCLUDING ANY
+ * IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE OR
+ * NON-INFRINGEMENT, ARE HEREBY EXCLUDED. SUN AND ITS LICENSORS SHALL NOT BE
+ * LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING
+ * OR DISTRIBUTING THE SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL SUN OR ITS
+ * LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT,
+
+ * INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER
+ * CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF
+ * OR INABILITY TO USE SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ *
+ * This software is not designed or intended for use in on-line control of
+ * aircraft, air traffic, aircraft navigation or aircraft communications; or in
+ * the design, construction, operation or maintenance of any nuclear
+
+ * facility. Licensee represents and warrants that it will not use or
+ * redistribute the Software for such purposes.
+ */
+
+/**
+ * A quick sort demonstration algorithm
+ * SortAlgorithm.java
+ *
+ * @author James Gosling
+ * @author Kevin A. Smith
+ * @version         @(#)QSortAlgorithm.java        1.3, 29 Feb 1996
+ */
+
+package vishnu.testvis.sammon;
+
+import vishnu.testvis.object.*;
+
+public class QSortAlgorithmTableLength
+{
+
+    /**
+     * A version of pause() that makes it easier to ensure that we pause
+     * exactly the right number of times.
+
+     */
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+
+   /** This is a generic version of C.A.R Hoare's Quick Sort
+    * algorithm.  This will handle arrays that are already
+    * sorted, and arrays with duplicate keys.<BR>
+    *
+    * If you think of a one dimensional array as going from
+    * the lowest index on the left to the highest index on the right
+    * then the parameters to this function are lowest index or
+
+    * left and highest index or right.  The first time you call
+    * this function it will be with the parameters 0, a.length - 1.
+    *
+    * @param a       an integer array
+    * @param lo0     left boundary of array partition
+    * @param hi0     right boundary of array partition
+    */
+   void QuickSort(LengthTable a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+         /* Arbitrarily establishing partition element as the midpoint of
+
+          * the array.
+          */
+         mid = a[ ( lo0 + hi0 ) / 2 ].length;
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].length < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].length > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(LengthTable a[], int i, int j)
+   {
+      LengthTable T;
+      T = a[i];
+      a[i] = a[j];
+      a[j] = T;
+   }
+
+   public void sort(LengthTable a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 21112)
@@ -0,0 +1,86 @@
+package vishnu.testvis.sammon;
+
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.object.ClusterObj;
+
+public class SammCluster {
+
+    // The co-ordinates of the centroid in the keyword space
+    //public Vector centroid  = null;
+
+    Point2D sammon;
+
+    // The documents ID contained in this cluster
+    public Vector documentIDs = new Vector();
+    public int[] sigWords;
+    private boolean selected =false;
+    private ClusterObj cObj;
+    public int getSize() { return documentIDs.size();}
+
+    // When looking for a particular cluster in a vector
+    /*public boolean equals(Object obj) {
+
+      System.out.println("Equals " + obj.getClass());
+
+        if (obj instanceof MyInteger)
+        return (documentIDs.size() == ((MyInteger) obj).value);
+
+          return super.equals(obj);
+
+            }
+    */
+    // Co-ordinates of the cluster centroid in the 2 dimension space
+    private double x;
+
+    private double y;
+
+    // Displayed size of the cluster
+    private int displayedRadius;
+
+    public void setDisplayedRadius(int value) {displayedRadius = value;}
+
+    public int getDisplayedRadius() {return displayedRadius;}
+
+
+    //private int numDocs;
+
+    // TEMPORARY: For random generator
+
+    public SammCluster(Vector theDocs, Point2D sammon,
+          int [] sigWords, ClusterObj c)
+    {
+        documentIDs = theDocs;
+        this.sammon = sammon;
+        this.sigWords=sigWords;
+        this.cObj=c;
+    }
+
+    public boolean getSelected() {return selected;}
+    public void setSelected(boolean state){selected=state;}
+
+    double getX() {return sammon.x;}
+
+    double getY() {return sammon.y;}
+
+    public ClusterObj getClusterObj(){ return cObj;}
+
+    //int getNumDocs() {return numDocs;}
+
+    public Vector getIDs() {return documentIDs;}
+
+    void addDocumentID(Integer docID) {
+        documentIDs.addElement(docID);
+    }
+
+   // void setClusterCentroid(Vector centroid) {
+   //     this.centroid = centroid;
+
+    int [] getWords() { return sigWords;}
+
+    void dump()
+    {
+        //System.out.println("Centroid : " + centroid.toString());
+        //System.out.println("Doc IDs: " + documentIDs.toString());
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 21112)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class SammKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    //SammRWTable _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    GraphicalPane parentPane;
+
+    public SammKeywordPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+         //setBackground(Color.white);
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+        //JLabel title = new JLabel("Related keywords");
+        //title.setFont(theFont);
+        //add(title,"North");
+
+        _dataModel = new RW_DataModel();
+        //_dataModel = new SammRWTable();
+        _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                // what does this do Shalini?
+                // Call the refine engine
+                refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+
+
+    }
+    public void refineEngine()
+    {
+       currentMain.log.startsection("FILTERING");
+       
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+      
+       currentMain.log.log("FILTER_WORDS",s);
+       currentMain.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+       Vector performance = currentMain.queries.getPerformance(currentMain.query, trec_id);
+       
+       currentMain.log.startsection("PERFORMANCE");
+
+       if( performance.size() > 0 )
+       {
+            Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+            Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+            currentMain.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+            currentMain.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+            for( int i = 0; i < performance.size()-2; i++ )
+            {
+                Integer perf = (Integer)performance.elementAt(i);
+                String str = "PRECISION_" + Integer.toString((i+1)*10);
+                currentMain.log.log(str,Integer.toString(perf.intValue()));
+            }
+       }
+       else
+       {
+            currentMain.log.log("INFO","No relevance data available");
+       }
+     
+                   
+       currentMain.log.endsection("PERFORMANCE");
+       currentMain.log.endsection("FILTERING");
+    }
+        
+    // have dendroclusterobj/sammonclusterobj etc as subclasses
+    // of some abstract clusterObj superclass with some 
+    // core fields (docs, hitdocFreq etc)
+    public void update(ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+    
+    public void update2(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 21112)
@@ -0,0 +1,183 @@
+/**
+ * @(#)ClusterGroupRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.*;
+import java.awt.*;
+import java.awt.geom.AffineTransform;
+
+public class ClusterGroupRect extends ZRect
+{
+  static final private Color __COLOUR=Color.black;
+  private ClusterGroupObj _data;
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,
+    Color c, ClusterGroupObj o)
+  {
+    super(x1,y1,x2,y2,c, o.getGroupSize());
+    _data = o;
+  }
+
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,ClusterGroupObj o)
+  {
+      super(x1,y1,x2,y2,__COLOUR, o.getGroupSize());
+      _data = o;
+  }
+
+
+
+  public void paint(Graphics oldg)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    g.draw(getBounds2D());
+  }
+
+
+  public void paint(Graphics oldg, float f)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    paintBorder(oldg, f);
+  }
+
+
+
+
+
+  /**
+   * !!! bug:- should check the boundary of the box and the length of the strings
+   * display at most two labels on the centre of the rectangle
+   * @param oldg is the display area
+   * @param c is the colour
+   */
+  private void centerText(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+
+    if (labels!=null){
+// Font f = new Font("Courier",Font.BOLD,12);
+      Font f = g.getFont();
+      FontMetrics fm = g.getFontMetrics(f);
+      float ascent = (float) fm.getAscent();
+      float height = (float) fm.getHeight();
+      float x=getUpperLeftX(), y=getUpperLeftY(),w=(float)getWidth(), h=(float)getHeight();
+      float width0=0F, width1=0F, x0=0F, x1=0F, y0=0F, y1=0F;
+
+        g.setColor(c);
+        width0 = (float) fm.stringWidth(labels);
+        x0 = x + (w-width0)/2F;
+
+// g.setFont(f);
+//   AffineTransform transform
+//     = AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,100.0,50.0);
+        g.drawString(labels,x0,y0);
+        /*
+        if (labels.length==1){
+          y0 = y + (h-height)/2F + ascent;
+          g.drawString(labels[0],x0,y0);
+        }
+        else
+        {
+          width1 = (float) fm.stringWidth(labels[1]);
+          x1 = x + (w-width1)/2F;
+          y0 = y + (h-(height*2F))/2F + ascent;
+          y1 = y0 + (height*1F);
+          g.drawString(labels[0],x0,y0);
+          g.drawString(labels[1],x1,y1);
+        }
+        */
+    }
+  }  //-- centerText
+
+
+
+AffineTransform transform;
+
+  //-- pre: _data.getLabels() has at least 2 elements
+  //-- post: draw two labels to the graphic
+  private void centerText2(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+    if (labels!=null)
+    {
+      Font f;
+      FontMetrics fm;
+      int ascent, height;
+      int x=(int) getUpperLeftX(), y=(int)getUpperLeftY(),
+          w=(int) getWidth(),      h=(int)getHeight();
+      int width0=0, width1=0, x0=0, x1=0, y0=0, y1=0;
+
+      g.setColor(c);
+
+      for (int fs=12; fs>=6; fs-=2)
+	  {
+        f = new Font("Dialog",Font.BOLD,fs);    // <--- Font for the labels
+
+        fm = g.getFontMetrics(f);
+        ascent = fm.getAscent();
+        height = fm.getHeight();
+        width0 = fm.stringWidth(labels);
+        /*
+        width0 = fm.stringWidth(labels[0]);
+        width1 = fm.stringWidth(labels[1]);
+        */
+        //-- paint horizontally
+        /*if (Math.max(width0,width1)<w  &&  2*height<h)*/
+        if (width0<w  &&  height<h)
+        {
+          x0 = x + (w-width0)/2;
+          //x1 = x + (w-width1)/2;
+          y0 = y + (h-(height)) + ascent;
+          //y1 = y0 + (height*1);
+          g.setFont(f);
+          g.drawString(labels,x0,y0);
+          //g.drawString(labels[1],x1,y1);
+ //System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    horizontal!");
+
+          break;
+        }
+        //-- paint vertically
+/*
+        else if (Math.max(width0,width1)<h  &&  2*height<w)
+        {
+          g.setFont(f);
+
+          x0 = x + (w-width0)/2;
+          x1 = x + (w-width1)/2;
+          y0 = y + (h-(height*2))/2 + ascent;
+          y1 = y0 + (height*1);
+          g.setFont(f);
+		  transform
+     		= AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,
+                (double)(x+w)/2.0,(double)(y+h)/2.0);
+		 g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         g.drawString(labels[0],x0,y0);
+         g.drawString(labels[1],x1,y1);
+         System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    vertical!");
+         System.err.println("x: "+x+"  y: "+y+"  w: "+w+"  h: "+h );
+
+         transform
+            = AffineTransform.getRotateInstance(360.0*Math.PI/180.0,
+               (double)(x+w)/2.0,(double)(y+h)/2.0);
+         g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         break;
+        }
+*/
+
+
+      }  // for
+
+    }  // if
+  }  // centerText2
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 21112)
@@ -0,0 +1,163 @@
+/**
+ * @(#)ClusterRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.ClusterObj;
+import java.awt.*;
+
+
+public class ClusterRect extends ZRect
+{
+    private ClusterObj _data;
+	public ClusterRect(float x1, float y1, float x2, float y2, Color c, ClusterObj o)
+    {
+		super(x1,y1,x2,y2,c,o.getDocsSize());
+        _data = o;
+	}
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill(getBounds2D());
+    g.setColor(Color.white);
+    g.draw(getBounds2D());
+  }
+
+  public ClusterObj getClusterObj(){return _data;}
+
+  public String getInfo(){return _data.getInfo();}
+
+
+  //------------------------------------------------------------------
+  private int _xRef=0, _yRef=0;   // used in paintLabels and createTipBox
+
+  /**
+   * paint the cluster's labels on the graphical pane
+   * @param oldg is the graphics to be pained
+   * @param w is the max width of the graphic
+   * @param h is the max heigth of the graphic
+   */
+  public void paintLabels(Graphics oldg, float w, float h){
+  Color tipBoxColor = new Color(255,255,180);
+  Graphics2D g = (Graphics2D) oldg;
+  Font f = new Font("TimesRoman",Font.PLAIN,12);
+  FontMetrics fm = g.getFontMetrics(f);
+  int ascent = fm.getAscent();
+  int height = fm.getHeight();
+  int boxWidth = 4 + Math.max(fm.stringWidth(_data.getLabel1()),
+                              fm.stringWidth(_data.getLabel2()) );
+  int boxHeight = 0 + height * 2;
+  Polygon poly;
+
+    g.setFont(f);
+
+    //-- setup the position of the tipbox
+    poly = createTipBox(boxWidth, boxHeight, w, h);
+
+    //-- paint the box
+    g.setColor(tipBoxColor);
+    g.fillPolygon(poly);
+    g.setColor(Color.black);
+    g.drawPolygon(poly);
+    //-- paint the text
+    g.drawString(_data.getLabel1(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent);
+    g.drawString(_data.getLabel2(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent + height);
+
+  }
+
+
+  private Polygon createTipBox(int boxW, int boxH, float maxW, float maxH){
+  int[] xs = new int[8];
+  int[] ys = new int[8];
+  int x=0, y=0,
+    arrowH=10,
+    arrowW=8,
+    arrowOffset=8,
+    gap=5; //-- gap is the space between the tipbox and the rectangle
+  float position=0.2F;  //-- set the box position
+
+    try{
+
+      //-- try RHS
+      x = (int) getLowerRightX() + gap;
+      y = (int) (getUpperLeftY() + getHeight()*position);
+      if ((x+boxW<=maxW) && (y+boxH<=maxH)){
+        xs[0]=xs[6]=xs[4]=xs[3]=xs[7] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x - arrowH;
+        ys[5]=ys[0]=ys[1]=ys[7] = y;
+        ys[3]=ys[2] = y + boxH;
+        ys[6] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try LHS
+      x = (int) getUpperLeftX() - gap - boxW;
+      if ((x>=0) && (y+boxH<=maxH)){
+        xs[0]=xs[7]=xs[6] = x;
+        xs[1]=xs[2]=xs[4]=xs[5] = x + boxW;
+        xs[3] = x + boxW + arrowH;
+        ys[7]=ys[0]=ys[1]=ys[3] = y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try top
+      x = (int) (getUpperLeftX() + getWidth()*position);
+      y = (int) getUpperLeftY() - boxH - gap;
+      if ((y>=0) && (x+boxW<=maxW)){
+        xs[0]=xs[7]=xs[6]=xs[4] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1] = y;
+        ys[6]=ys[5]=ys[3]=ys[2] = y + boxH;
+        ys[4] = y + boxH + arrowH;
+        throw new Exception();
+      }
+
+      //-- try bottom
+      y = (int) getLowerRightY() + gap;
+      if ((y+boxH<=maxH) && (x+boxW<=maxW)){
+        xs[7]=xs[0]=xs[6]=xs[2] = x;
+        xs[4]=xs[5] = x + boxW;
+        xs[1] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1]=ys[3]=ys[4]= y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y - arrowH;
+        throw new Exception();
+      }
+
+      //-- try inside the rectangle
+      x = (int) getUpperLeftX() + 5;
+      y = (int) getUpperLeftY() + 5 + 5;
+      xs[7]=xs[0]=xs[6]=xs[2] = x;
+      xs[4]=xs[5] = x + boxW;
+      xs[1] = x + 3;
+      xs[3] = x + 3 + 5;
+      ys[7]= ys[0]=ys[1]=ys[3]=ys[4] = y;
+      ys[6]= ys[5]   = y + boxH;
+      ys[2] = y - 5;
+
+    }
+    catch(Exception e){
+    }
+
+    _xRef = x;
+    _yRef = y;
+    return (new  Polygon(xs,ys,xs.length));
+  }
+
+
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 21112)
@@ -0,0 +1,119 @@
+/**
+ * @(#)Rect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.geom.Rectangle2D;
+import java.awt.*;
+
+/**
+ * The Rect class contains direction and colour, the direction is calculated
+ * based on the initial points.
+ */
+abstract class Rect extends Rectangle2D.Float{
+//-- the rectangle will move to this direction during zooming
+final static protected int __NORTH=0, __EAST=1, __SOUTH=2, __WEST=3;
+private int _direction;
+//-- any two points to create a rectangle
+private float _left_x, _left_y, _right_x, _right_y;
+//-- color of the rectangle
+private Color _colour;
+
+  public Rect(float x1, float y1, float x2, float y2, Color c){
+    setup(x1, y1, x2, y2);
+    _colour = c;
+  }
+
+
+  public void showInfo(){
+    System.err.println( "x: "+getX() +"  y: "+getY()+"  w: "+getWidth()+"  h: "+getHeight());
+    System.err.println(   "    direction: "+getDirection_toString()+"  colour: "+
+        _colour.toString());
+  }
+
+
+  /**
+   * setup the direction and the corresponding coordination
+   */
+  private void setup(float x1, float y1, float x2, float y2){
+    _left_x=x1; _left_y=y1;
+    _right_x=x2; _right_y=y2;
+    //-- set up the direction
+    if (x1<x2)
+      if (y1<y2) _direction = __WEST;
+      else if (y1>y2) _direction = __SOUTH;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else if (x1>x2)
+      if (y1<y2) _direction = __NORTH;
+      else if (y1>y2) _direction = __EAST;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else
+      System.err.println("Error - Rectangle box cannot be zero");
+    //-- setup the corresponding coordination of the rectangle obj
+    switch (_direction) {
+      case __NORTH:
+        super.setRect(x2,y1,x1-x2,y2-y1);
+        break;
+      case __EAST:
+        super.setRect(x2,y2,x1-x2,y1-y2);
+        break;
+      case __SOUTH:
+        super.setRect(x1,y2,x2-x1,y1-y2);
+        break;
+      case __WEST:
+        super.setRect(x1,y1,x2-x1,y2-y1);
+        break;
+    }
+  }  //-- setup
+
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill( getBounds2D() );
+    g.setColor(Color.black);
+    g.draw( getBounds2D() );
+  }
+
+  public void paintBorder(Graphics oldg, float w){
+    Graphics2D g = (Graphics2D) oldg;
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getLowerRightX()-w,getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       (float) getWidth(),w));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getLowerRightY()-w,
+       (float) getWidth(),w));
+  }
+
+  public boolean isNorth(){return _direction==__NORTH;}
+  public boolean isEast(){return _direction==__EAST;}
+  public boolean isSouth(){return _direction==__SOUTH;}
+  public boolean isWest(){return _direction==__WEST;}
+
+  public Color getColor(){return _colour;}
+  public float getUpperLeftX(){return ((float) getX());}
+  public float getUpperLeftY(){return ((float) getY());}
+  public float getLowerRightX(){return ((float) (getX()+getWidth()));}
+  public float getLowerRightY(){return ((float) (getY()+getHeight()));}
+  public int getDirection(){return _direction;}
+  public int getOppositeDirection(){return (_direction+2)%4;}
+
+  /**
+   * @return the direction of the rectangule as a string
+   */
+  public String getDirection_toString(){
+    switch (_direction){
+      case __NORTH: return "NORTH";
+      case __EAST: return "EAST";
+      case __SOUTH: return "SOUTH";
+      default: return "WEST";
+    }
+  }
+
+
+}  //-- Rect
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 21112)
@@ -0,0 +1,157 @@
+/**
+ * @(#)ZRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.*;
+
+public class ZRect extends Rect{
+//-- no of doc that the rectangle represented
+private int _capacity=0;
+
+//-- the new points that the rectangle will be translated (java coordination)
+protected float _t_left_x=0f, _t_left_y=0f, _t_right_x=0f, _t_right_y=0f;
+//-- if the rectangle has been choosen, set it to true
+private boolean _chosen=false;
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c){
+    super(x1, y1, x2, y2, c);
+  }
+
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c, int s){
+    super(x1, y1, x2, y2, c);
+    _capacity = s;
+  }
+
+  public int getCapacity(){return _capacity;}
+  public void setCapacity(int i){_capacity=i;}
+
+  public boolean isChosen(){return _chosen;}
+  public void setChosen(){_chosen=true;}
+  public void resetChosen(){_chosen=false;}
+
+  /**
+   * set the new coordinate, that the rectangle will move to
+   * pre: the coordinate is based on Java graphic coordination
+   *      x1,y1 is the left upper corner and
+   *      x2,y2 is the right lower corner
+   */
+  public void translateTo(float x1, float y1, float x2, float y2){
+    _t_left_x=x1; _t_left_y=y1;
+    _t_right_x=x2; _t_right_y=y2;
+  }
+
+
+
+  private static boolean almostEqual(float a, float b){
+  float tolerant = 0.00001F;
+    return  (Math.abs(a-b) <= tolerant);
+  }
+
+
+  /**
+   * @param x is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return y, based on x
+   */
+  private static float newY(
+    float x,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//    return (x1==x2)? y2: ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+    if (almostEqual(x1,x2))
+      return ((y2-y1)/(float) totalFrame) * (float) (currentFrame+1) + y1;
+    else
+      return ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+  }
+
+  /**
+   * @param y is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return x, based on y
+   */
+  private static float newX(
+    float y,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//   return (y1==y2)? x2: ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+    if (almostEqual(y1,y2))
+      return ((x2-x1)/(float) totalFrame) * (float) (currentFrame+1) + x1;
+    else
+      return ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+  }
+
+
+  /**
+   * pre: translateTo is called
+   * @param how many animated pictures u want to create
+   * @retrun a list of animation picture (rectangles with diff positions)
+   */
+  public ZRect[] zooming(int noOfFrame){
+  ZRect[] vec = new ZRect[noOfFrame];
+  float dist1, dist2, t1, t2;
+
+    switch (getDirection()){
+      case __NORTH: case __SOUTH:
+        dist1 = (_t_left_y - getUpperLeftY())/(float)noOfFrame;
+        dist2 = (_t_right_y - getLowerRightY())/(float)noOfFrame;
+        t1 = getUpperLeftY();
+        t2 = getLowerRightY();
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            newX(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t1,
+            newX(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            t2,
+            getColor());
+        }
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+        break;
+      default:
+        //-- for WEST and EAST
+        dist1 = (_t_left_x - getUpperLeftX())/(float)noOfFrame;
+        dist2 = (_t_right_x - getLowerRightX())/(float)noOfFrame;
+        t1 = getUpperLeftX();
+        t2 = getLowerRightX();
+
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            t1,
+            newY(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t2,
+            newY(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            getColor());
+        }
+        /**
+         * create the final picture here, the reason we don't do in the above
+         * for loop is because we avoid the 'tolerant' error from the floating
+         * point computation.
+         */
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+
+    }  //-- switch
+
+    return vec;
+  }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 21112)
@@ -0,0 +1,56 @@
+/**
+ * @(#)BarRenderer.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.TableCellRenderer;
+
+//!!! the % in getString and getColor may WRONG !!!
+
+public class BarRenderer extends JProgressBar implements TableCellRenderer
+{
+
+  public BarRenderer(){
+    super(JProgressBar.HORIZONTAL);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+
+  public BarRenderer(int min, int max){
+    super(min,max);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+  public String getString(){
+    if (getValue()==0) return "";
+    else return String.valueOf(getValue())+"%";
+  }
+
+  public Component getTableCellRendererComponent(JTable table, Object value,
+    boolean isSelected, boolean hasFocus, int row, int column){
+  int n = 0;
+    if (value instanceof Number){
+      n = ((Number) value).intValue();
+    } else{
+      System.err.println("Error - item in bar render is not a number");
+    }
+    setValue(n);
+    setForeground(getColor(n));
+    return this;
+  }
+
+  private Color getColor(int value){
+    if (value < 40) return Color.green;
+    else if (value <80) return Color.yellow;
+    else
+      return Color.red;
+  }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 21112)
@@ -0,0 +1,289 @@
+/**
+ * @(#)RW_DataModel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DocObj;
+import javax.swing.table.DefaultTableModel;
+import java.util.*;
+import vishnu.testvis.sammon.QSortAlgorithmTableLength;
+import vishnu.testvis.object.LengthTable;
+
+/**
+ * This table contains 3 columns:
+ *   - a tick box
+ *   - a related word
+ *   - and the hit document frequency
+ */
+public class RW_DataModel extends DefaultTableModel{
+static final private String[] __COLNAMES={"","Related Word","Hit Doc Freq"};
+private String[] _related_words;
+private ClusterObj _cObj;
+private Vector _colnames = new Vector(__COLNAMES.length);
+
+private Vector _docs = null;  // this is created inside 'setData'
+
+
+  public RW_DataModel()
+  {
+    //-- setup the column's names
+    for (int i=0;i<__COLNAMES.length;i++)
+      _colnames.addElement(__COLNAMES[i]);
+    clear();
+  }
+
+
+  //-- clear all data variables in this data model
+  public void clear()
+  {
+    _related_words = null;
+    _cObj = null;
+    _docs = null;
+    setDataVector(new Vector(0), _colnames);
+  }
+
+
+  //--------------------------------------------------------------------------
+  /**
+   * Set up the cluster data in the data model
+   * pre: the related-words are new to the data model
+   * @param rw is a list of related words
+   * @param c is the data of the cluster
+   */
+  public void setData(String[] rw, ClusterObj c)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+    _cObj = c;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+  
+  // overloaded, this one for dendro
+  // we pass on freq vector instead of obj
+  public void setData(String[] rw, vishnu.testvis.dendro.ClusterObj _cObj)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+
+
+  public String[] getRefinedTrecIDs()
+  {
+      Vector sVec = new Vector();
+      HashSet set = new HashSet();  //-- this can avoid duplicated items !
+      boolean isNewItem;
+      
+      for(int i=0;i<getDataVector().size();i++)
+      {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                
+                Vector docs = (Vector) _docs.elementAt(i);
+
+                for (int j=0;j<docs.size();j++)
+                {
+                    isNewItem = set.add((DocObj) docs.elementAt(j));
+                    if (isNewItem)
+                    {
+                        sVec.addElement(((DocObj)docs.elementAt(j)).getTrecID());
+                        //System.out.println(((DocObj) docs.elementAt(j)).getTrecID());
+                    }
+                }
+            }
+      }
+      
+      String[] sStr = new String[sVec.size()];
+      for( int i = 0; i < sVec.size(); i++ )
+      {
+          sStr[i] = new String((String)sVec.elementAt(i));
+      }
+      return sStr;
+  }
+  
+  public String[] getFilterWords()
+  {
+        Vector sVect = new Vector();
+        
+        for(int i=0;i<getDataVector().size();i++)
+        {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                sVect.addElement((String)getValueAt(i,1));
+            }   
+        }
+        
+        String sStr[] = new String[sVect.size()];
+        
+        for( int i = 0; i < sVect.size(); i++ )
+        {
+            sStr[i] = (String)sVect.elementAt(i);    
+        }
+        return sStr;
+  }
+
+  public String getRefinedDocs(){
+  String s = "";
+  HashSet set = new HashSet();  //-- this can avoid duplicated items !
+  boolean isNewItem;
+    for(int i=0;i<getDataVector().size();i++){
+
+      if (((Boolean) getValueAt(i,0)).booleanValue()) {
+       Vector docs = (Vector) _docs.elementAt(i);
+
+        for (int j=0;j<docs.size();j++){
+          isNewItem = set.add((DocObj) docs.elementAt(j));
+          if (isNewItem)
+          {
+            s += ((DocObj) docs.elementAt(j)).format4Fields();
+            //System.out.println(((DocObj) docs.elementAt(j)).format4Fields());
+          }
+            
+        }
+
+      }
+
+    }
+    s = "<html><font face=Arial size=+10> " + set.size()  + " documents found\n" + s +
+        "</font></html>\n";
+    return s;
+  }
+
+
+
+
+  //--------------------------------------------------------------------------
+  public Class getColumnClass(int col)
+  {
+    switch (col)
+    {
+      case 0: return Boolean.class;
+      case 1: return String.class;
+      case 2: return Integer.class;
+      default: return Object.class;
+    }
+  }
+
+  public int getColumnCount(){return __COLNAMES.length;}
+
+  public boolean isCellEditable(int row, int col){
+    return ((col==0) ? true : false);
+  }
+  public void setValueAt(Object value, int row, int col){
+    super.setValueAt(value,row,col);
+  }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 21112)
@@ -0,0 +1,189 @@
+/**
+ * @(#)NavigatorPanel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.visual.Vishnu;
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.table.*;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.util.Vector;
+
+
+public class NavigatorPanel extends JComponent
+{
+    private JTable _table;
+    private JButton _showAllButton, _refineButton;
+    private JEditorPane _display;
+    private TMNavigator _outerFrame;
+    private ClusterObj _cObj;
+//public String _codeBase;
+
+    private RW_DataModel _dataModel;  //-- this is created within this class
+
+
+    public NavigatorPanel(JTable t, /*JButton b1,*/ JButton b2,
+        JEditorPane d, TMNavigator f)
+    {
+        _table = t;
+        /*_showAllButton = b1;
+        _showAllButton.setActionCommand("showAllDoc");
+        _showAllButton.setToolTipText("show ALL documents within the selected cluster");
+        _showAllButton.addActionListener(new ButtonHandler());*/
+
+        _refineButton = b2;
+        _refineButton.setActionCommand("refineDoc");
+        _refineButton.setToolTipText("show documents with any of the selected related-words");
+        _refineButton.addActionListener(new ButtonHandler());
+
+        _display = d;
+        _outerFrame = f;
+
+        _display.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        System.out.println("hyperlink result: " + index);
+                        if (index > 0)
+                            _outerFrame._displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+        _dataModel = new RW_DataModel();
+        _table.setModel(_dataModel);
+
+
+
+        _cObj = null;
+        unableButtons();
+        setupRenderer();
+    }
+
+    private void unableButtons()
+    {
+        //_showAllButton.setEnabled(false);
+        _refineButton.setEnabled(false);
+    }
+    private void enableButtons()
+    {
+        //_showAllButton.setEnabled(true);
+        _refineButton.setEnabled(true);
+    }
+
+    public void clear()
+    {
+        _cObj = null;
+        _dataModel.clear();
+        _display.setText("");
+        unableButtons();
+    }
+
+
+
+    public void setData(String[] rw, ClusterObj c)
+    {
+        _cObj = c;
+        _dataModel.setData(rw, _cObj);
+        enableButtons();
+        setupRenderer();
+    }
+
+
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+    
+    public void showDocuments()
+    {
+        if( _cObj != null )
+        {
+            _display.setText(_cObj.formatDocs());
+        }
+    }
+     
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            if (_cObj!=null)
+            {
+                String action = event.getActionCommand();
+                if (action.equals("showAllDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(_cObj.formatDocs());
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+                else if (action.equals("refineDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(   _dataModel.getRefinedDocs()   );
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                    
+                    _outerFrame._displayFrame.log.startsection("FILTERING");
+       
+                    String trec_id[] = _dataModel.getRefinedTrecIDs();
+                    String filter_Words[] = _dataModel.getFilterWords();
+                    String s = new String("");
+                    for( int i = 0; i < filter_Words.length; i++ )
+                    {
+                            s += filter_Words[i] + " ";
+                    }
+      
+                   _outerFrame._displayFrame.log.log("FILTER_WORDS",s);
+                   _outerFrame._displayFrame.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+                   Vector performance = _outerFrame._displayFrame.queries.getPerformance(_outerFrame._displayFrame.query, trec_id);
+                   
+                   _outerFrame._displayFrame.log.startsection("PERFORMANCE");
+                    
+                   if( performance.size() > 0 )
+                   {
+                        Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+                        Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+                        _outerFrame._displayFrame.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+                        _outerFrame._displayFrame.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+                        for( int i = 0; i < performance.size()-2; i++ )
+                        {
+                            Integer perf = (Integer)performance.elementAt(i);
+                            String str = "PRECISION_" + Integer.toString((i+1)*10);
+                            _outerFrame._displayFrame.log.log(str,Integer.toString(perf.intValue()));
+                        }
+                   }
+                   else
+                   {
+                        _outerFrame._displayFrame.log.log("INFO","No relevance data available");
+                   }
+                
+                    _outerFrame._displayFrame.log.endsection("PERFORMANCE");
+                    _outerFrame._displayFrame.log.endsection("FILTERING");
+                    
+                }
+            }
+        }
+    }  //-- end ButtonHandler
+
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 21112)
@@ -0,0 +1,159 @@
+/**
+ * @(#)Navigator.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.visual.Vishnu;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.border.BevelBorder;
+
+
+public class TMNavigator extends JPanel
+{
+    final private int _TREEMAPWIDTH=300, _TREEMAPHEIGHT=350;
+    public final static int LIST_ON_LEFT = 0;
+    public final static int LIST_ON_RIGHT = 1;
+    private int listPos;
+
+//    JTextField _searchTF = new JTextField();
+//    JButton _searchButton = new JButton("Search");
+    public Vishnu _displayFrame;
+    JScrollPane _displayJSP = new JScrollPane();
+    JEditorPane _displayJEP = new JEditorPane();
+    JTable _table = new JTable();
+    JButton _showAllButton = new JButton("Show All Documents");
+    JButton _refineButton = new JButton("Filter");
+    NavigatorPanel navigator;
+//  = new NavigatorPanel(_table, _showAllButton, _refineButton, _displayJEP, this,_codeBase);
+
+    JButton _backButton = new JButton("<<   Back");
+    TreemapPanel _treemapPanel;
+//  = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _codeBase);
+
+
+
+	public TMNavigator(Vishnu displayFrame, int listPos)
+    {
+   	    _displayFrame = displayFrame;
+   	    this.listPos = listPos;
+   	    
+        navigator = new NavigatorPanel(_table, /*_showAllButton*/ _refineButton, _displayJEP, this);
+        _treemapPanel = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _displayFrame);
+        
+        buildGUI();
+        
+	    setSize(_TREEMAPWIDTH*2+40,_TREEMAPHEIGHT+340);
+        //setVisible(false);
+	    //setupEventHandler();
+	}
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        _treemapPanel.setVisible(f);
+    }
+
+    void buildGUI()
+    {
+        JPanel tmpPanel;
+        JScrollPane tmpJSP;
+
+        //this.setLayout(new FlowLayout(FlowLayout.LEFT,10,10));
+        this.setLayout(null);
+//        _searchTF.setFont(new Font("Courier",Font.BOLD,16));
+//        _searchTF.setBounds(10,10,400,30);
+//        add(_searchTF,null);
+
+//        _searchButton.setActionCommand("search");
+//        _searchButton.setBounds(10+400+10,10,100,30);
+//        add(_searchButton,null);
+        _backButton.setBorder(BorderFactory.createEtchedBorder());
+        _backButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _backButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        add(_backButton,null);
+
+        if( listPos == 0 ) // list on left
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 ) // list on right
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10+_TREEMAPWIDTH+10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+
+
+        _displayJEP.setEditorKit(new HTMLEditorKit());
+        _displayJEP.setEditable(false);
+        Dimension sSize=this.getToolkit().getScreenSize();
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_displayJEP);
+        tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10+_TREEMAPHEIGHT+10+30+10, _TREEMAPWIDTH*2+10,
+            sSize.height>700?200:85);
+        add(tmpPanel,null);
+
+        //-- manually creating the columns
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(_TREEMAPWIDTH*2/3);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(_TREEMAPWIDTH/3);
+        _table.setRowSelectionAllowed(false);
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_table);
+        
+        if( listPos == 0 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10+_TREEMAPWIDTH+10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        
+        
+        _showAllButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _showAllButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        _showAllButton.setBorder(BorderFactory.createEtchedBorder());
+        //add(_showAllButton,null);
+
+        _refineButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _refineButton.setBounds(10+_TREEMAPWIDTH+10+200,10+_TREEMAPHEIGHT+10,100,30);
+        _refineButton.setBorder(BorderFactory.createEtchedBorder());
+        add(_refineButton,null);
+
+    }
+
+
+    private JPanel newJPanel(Component c, Color bg,int x, int y, int w, int h)
+    {
+        JPanel p = new JPanel();
+        p.setBorder(new BevelBorder(BevelBorder.LOWERED));
+        p.setLayout(new BorderLayout());
+        p.setBounds(x,y,w,h);
+        p.setBackground(bg);
+        p.add(c,BorderLayout.CENTER);
+        return p;
+  }
+
+    public void update()
+    {
+        _treemapPanel.newSearch();
+    }
+
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 21112)
@@ -0,0 +1,539 @@
+/**
+* name      TreemapPanel.java 
+* author    Peter Au
+* modified  Daniel Heesch (29.04.02)
+*/
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.rectangle.ClusterRect;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.Stack;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import java.util.*;
+
+public class TreemapPanel extends Canvas
+{
+
+    private int _width, _height;
+    private JButton _popUpButton;
+    private NavigatorPanel _navigator;
+    private String _newTerms;
+    Vishnu _displayFrame;
+    Image _offScreenImage=null;
+    boolean _init_done=false, _drillDownFlag=false, _newSearchFlag=false, _popUpFlag=false;
+    Stack _visualObjs;
+    
+    JPopupMenu[] popups;
+    PopupMenu popup;
+
+    public TreemapPanel(int w, int h, JButton bb, NavigatorPanel n, Vishnu displayFrame)
+    {
+        _displayFrame = displayFrame;
+        _width=w; _height=h;
+        _popUpButton = bb;
+        _navigator = n;
+        _displayFrame = displayFrame;
+        setSize(_width,_height);
+        _visualObjs = new Stack();
+        addMouseListener(new MouseHandler());
+        _popUpButton.setActionCommand("popUp");
+        _popUpButton.setEnabled(false);
+        _popUpButton.setToolTipText("Go back to the previous level");
+        _popUpButton.addActionListener(new ButtonHandler());
+                   
+        popup = new PopupMenu();
+        
+        add(popup);
+
+    }
+
+    class MouseHandler extends MouseAdapter{}
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            String action = event.getActionCommand();
+            String st;
+            if (action.equals("popUp")){
+                processPopUp();
+            }
+        }
+    }
+    
+    public void update(Graphics oldg)
+    {
+        Graphics2D g = (Graphics2D) oldg;
+        Visualiser currentVObj;
+        
+        if (_newSearchFlag)
+        {
+            _visualObjs.push(new Visualiser(_displayFrame.dataManager, 0F,0F, _width, _height));
+            currentVObj = ((Visualiser) _visualObjs.peek());
+            currentVObj.createTreemapView(createImage(_width, _height));
+            
+            //-- draw the treemap view on the offscreen-buffer
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            
+            _newSearchFlag = false;
+        }
+        else if (_drillDownFlag)
+        {
+            _displayFrame.refine = true;
+            currentVObj = (Visualiser) _visualObjs.peek();
+            
+            //-- create a list of image buffer for the animated pictures
+            Image iBuffer[]=new Image[Visualiser.__FRAME];
+            
+            for(int i=0;i<iBuffer.length;i++)
+                iBuffer[i]=createImage(_width,_height);
+
+            //-- prepare the zooming pictures of the current visaul object
+            currentVObj.prepare_zooming(iBuffer, this);
+
+            int s = _visualObjs.size();
+            float x = Visualiser.__borderwidth*s,
+                y = Visualiser.__borderheight*s;
+         
+            DataManager dm = new DataManager(_displayFrame);
+            dm.query(_newTerms);
+            _visualObjs.push(new Visualiser(dm,x, y, _width-x*2, _height-y*2));
+
+                /**
+                * create a treemap view for the new data
+                * before doing so, we have to copy the last zooming picture to the treemap
+                * this is because the zooming picture contains the borders
+                */
+                
+            Image tmpImg = createImage(_width, _height);
+            Graphics2D tmpGph = (Graphics2D) tmpImg.getGraphics();
+            tmpGph.drawImage(currentVObj.getLastZoomingImage(),0,0,this);
+            
+            //-- now create the treemap view
+            ((Visualiser) _visualObjs.peek()).createTreemapView(tmpImg);
+
+            //-- draw the treemap view (top element of the stack) on the offscreen buffer
+            ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+
+            //-- when everyting is done, we then show the animated pictures
+            //-- we use the 2nd element of the stack, not the top one
+            currentVObj.show_zoomIn(g,this);
+
+            _drillDownFlag = false;
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+        }
+        else if (_popUpFlag)
+        {
+
+            currentVObj = (Visualiser) _visualObjs.peek();
+            //-- redraw the treemap view on the offscreen buffer, as the previous
+            //-- level is gone
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            //-- show the animated pictures
+            currentVObj.show_zoomOut(g,this);
+            _popUpFlag = false;
+        }
+        paint(g);
+    }
+
+
+    public void clearDisplay()
+    {
+        _navigator.clear();
+    }
+
+
+    public void processPopUp()
+    {
+        if (_visualObjs.size() > 1)
+        {
+            _displayFrame.log.log("UNREFINE_QUERY");
+            //-- get rid of the top obj in the stack
+            _visualObjs.pop();
+            //-- reset the necessary instance variable back to the default values
+            ((Visualiser) _visualObjs.peek()).setDefault();
+            //-- clear the table and display
+            clearDisplay();
+            if (_visualObjs.size()==1) _popUpButton.setEnabled(false);
+            _popUpFlag = true;
+            repaint();
+        }
+    }
+
+    public void newSearch()
+    {
+        clearDisplay();
+        _popUpButton.setEnabled(false);
+        _visualObjs = new Stack();
+        _newSearchFlag = true;
+        repaint();
+    }
+
+
+    private void processDrillDown(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+        //-- chose which cluster to drill down
+        if (r!=null){
+            //-- do sth on 'r'??? send it back to the server and do the clustering again
+            _popUpButton.setEnabled(true);
+            clearDisplay();
+            _drillDownFlag = true;
+            _newTerms = r.getClusterObj().getLabel();
+            _navigator.clear();  //-- clear the table and the display
+            repaint();
+
+        }
+        else
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+
+
+
+    private void processInspect(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(x,y);
+        if (r!=null){
+            //-- draw a corner on the selected cluster
+            ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                this, x, y, _width, _height);
+            //-- setup the data model in the table
+            _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                r.getClusterObj());
+                
+            // and show all documents!
+            _navigator.showDocuments();            
+                
+            repaint();
+        }
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+    
+    protected void processMouseEvent(MouseEvent e)
+    {
+        if (e.getID()==MouseEvent.MOUSE_CLICKED)
+        {       
+            
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                       
+                    if (r!=null)
+                    {
+                        
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        
+                        String str = r.getClusterObj().getLabel();
+                        _displayFrame.log.log("SELECTED_CLUSTER",str);
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            /*img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            System.out.println("Changed?: " + img_hasChanged);
+            if (img_hasChanged)*/  repaint();
+        }
+        
+        }  // if mouseclick
+    }
+
+
+    //------------------------------------------------------------------
+
+    public void paint(Graphics oldg)
+    {
+        //System.out.println("Tree map paint called");
+        Graphics2D tmpGph, g = (Graphics2D) oldg;
+        Image tmpImg;
+        /**
+        * create off-scrren images for double-buffering
+        * There is a known deficiency bug (Java itself) with createImage in java applications.
+        * Workaround: to create necessary image buffers inside paint!
+        */
+        if (!_init_done){
+            _offScreenImage = createImage(_width, _height);
+            _init_done = true;
+        }
+
+        /*if(this.isVisible())*/ g.drawImage(_offScreenImage,0,0,this);
+
+    }  //-- paint
+
+
+
+}  //-- TreemapPanel
+
+
+        /*
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent e)
+            {
+                if (!_visualObjs.empty())
+                {
+                    final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+                    ClusterObj cObj = r.getClusterObj();
+                    int numDocs = cObj.getDocsSize();
+                    System.out.print("Mouse moved to ");           
+                    System.out.println(e.getX() + " " + e.getY());
+                    
+                    boolean img_hasChanged;
+            
+                    popup.removeAll();
+                    MenuItem menuItem = new MenuItem(numDocs + " documents");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Show documents");
+                    
+                            if (r!=null)
+                            {
+                                //-- draw a corner on the selected cluster
+                                //((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                                // this, x, y, _width, _height);
+                                //-- setup the data model in the table
+                                _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                                r.getClusterObj());
+                
+                                // and show all documents!
+                                _navigator.showDocuments();            
+                
+                                 repaint();
+                            }
+         
+                        }
+                    });
+            
+                    popup.add(menuItem);
+                
+                    menuItem = new MenuItem("Drill down");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Drill down clicked");
+                        }
+                    });
+            
+                    popup.add(menuItem);
+            
+                    popup.show(this,e.getX(),e.getY());
+            
+                    img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                    _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+                    System.out.println("Changed?: " + img_hasChanged);
+                    if (img_hasChanged)  repaint();
+                }
+            }
+        });*/
+        
+        /*  protected void processMouseEvent(MouseEvent e)
+    {
+
+        if (!_visualObjs.empty())
+        {
+            //-- if the mouse points outside the obj, then copy the original treemap view
+            //-- without the tipbox to the screen
+            if (e.getID()==MouseEvent.MOUSE_EXITED){
+                //-- get the most update view of the treemap
+                ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+                repaint();
+            }
+            else if (e.getID()==MouseEvent.MOUSE_CLICKED){
+
+                //-- if the user pressed the first mouse button
+                if ((e.getModifiers() & InputEvent.BUTTON1_MASK) == InputEvent.BUTTON1_MASK)
+                {
+                    System.out.println("Mouse clicked");
+                    processInspect(e.getX(),e.getY());
+                    
+                }
+                //-- if the user pressed the third mouse button
+                else if ((e.getModifiers() & InputEvent.BUTTON3_MASK) == InputEvent.BUTTON3_MASK)
+                    processDrillDown(e.getX(),e.getY());
+
+            }
+        }
+
+    }*/
+
+    //------------------------------------------------------------------
+/*    protected void processMouseMotionEvent(MouseEvent e)
+    {
+        System.out.println("Outer Mouse in motion");
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            System.out.println("Inner Mouse in motion");
+            System.out.println(e.getX() + " " + e.getY());
+            //System.out.println("Mouse in motion");
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    System.out.println("Show documents");
+                    
+                    if (r!=null)
+                    {
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            //img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+            //   _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            //System.out.println("Changed?: " + img_hasChanged);
+            //if (img_hasChanged)
+            repaint();
+        }
+    }*/
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 21112)
@@ -0,0 +1,671 @@
+/**
+ * @(#)Visualiser.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.*;
+import vishnu.testvis.treemap.rectangle.*;
+import java.lang.Exception;
+import java.awt.*;
+import java.awt.image.ImageObserver;
+import java.awt.geom.GeneralPath;
+
+public class Visualiser extends Canvas implements Runnable
+{
+
+  static private final int __FRAMEDELAY = 100;
+
+  static final int __FRAME=10;
+  static float __borderwidth=5F, __borderheight=5F;
+//-- the following variables specify the actual drawing area
+  final private float _upperX, _upperY, _width, _height;
+
+  private DataManager _data;
+  private Image _treemapView = null;  //-- static treemap view
+  private Image[] _zoomingImages;
+
+//-- rectangles represent the cluster-groups and the clusters
+  private ClusterGroupRect[] _gpRects = null;
+  private ClusterRect[][] _cRects = null;
+
+//-- the rectangle the mouse pointed to, used for tipbox
+  private ClusterRect _previousRect = null;
+
+//-- to identify which rectangle has a corner-paint
+  private ClusterRect _cornerRect = null;
+
+
+  /**
+   * Constructor: visualise the data into the specific size of pane
+   * @param d is the data obj
+   * @param x is the left upper x-coordinate of the display pane
+   * @param y is the left upper y-coordinate of the display pane
+   * @param w is the width of the display pane
+   * @param h is the height of the display pane
+   */
+  public Visualiser(DataManager d, float x, float y, float w, float h)
+  {
+    int noOfGroup;
+    _data = d;
+    _upperX=x; _upperY=y; _width=w; _height=h;
+
+    //-- allocate memory for _cRects and _gpRects
+    noOfGroup = _data.getClusterGroupsSize();
+    _gpRects = new ClusterGroupRect[noOfGroup];
+    _cRects = new ClusterRect[noOfGroup][];
+    for (int i=0; i<noOfGroup; i++)
+      _cRects[i] = new ClusterRect [_data.getClusterGroup(i).getClustersSize()];
+    // create rectangles for the corresponding groups and clusters
+    createRects();
+  }
+
+
+  static public void setBorder(float w, float h)
+  {
+    __borderwidth = w;
+    __borderheight = h;
+  }
+
+
+  public DataManager getDataManager(){return _data;}
+
+  //---------------------------------------------------------------------
+  /**
+   * set different variables back to default values
+   */
+  public void setDefault()
+  {
+    _previousRect = null;
+    _cornerRect = null;
+    try
+    {
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].isChosen())
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].isChosen())
+	    {
+              _gpRects[i].resetChosen();
+              _cRects[i][j].resetChosen();
+              throw new Exception();
+            }  //-- if
+    }
+    catch(Exception e)
+    {
+    }  //-- catch
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * @return a snapshot of the tree map
+   */
+  public Image getTreemapView(){
+    return _treemapView;
+  }
+
+
+  /**
+   * create a treemap view of the obj, only call once
+   * @param img
+   */
+  public void createTreemapView(Image img)
+  {
+  	final float GP_BORDERWIDTH = 1F; // <-- set the width of the border of each group
+    _treemapView = img;
+    Graphics2D g = (Graphics2D) _treemapView.getGraphics();
+    for (int i=0;i<_gpRects.length; i++)
+	{
+      for(int j=0;j<_cRects[i].length;j++)
+	  {
+        _cRects[i][j].paint(g);
+      }
+      _gpRects[i].paint(g);
+    } //-- for
+    //-- paint the border of each group
+    for (int i=0;i<_gpRects.length; i++)
+       _gpRects[i].paintBorder(g,GP_BORDERWIDTH);
+    // paint the outer border
+    ZRect fooRect = new ZRect(_upperX,_upperY,_upperX+_width,_upperY+_height,null);
+    g.setColor(Color.black);
+    fooRect.paintBorder(g, GP_BORDERWIDTH);
+
+  }  //-- create_pict
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the treemap view on the image, together with the corner triangle
+   */
+  public void drawTreemapView(Image img, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) img.getGraphics();
+    g.drawImage(_treemapView,0,0,observer);
+    drawCorner(g);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect and make
+   * sure that the tipbox is drawn as well
+   */
+  public void drawCorner(ClusterRect r, Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    _cornerRect = r;
+    _previousRect = null; //-- force the tipbox to be repainted
+    drawTipBox(offScreenImg, observer, x, y, w, h);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect
+   */
+  private void drawCorner(Graphics2D g)
+  {
+    if (_cornerRect!=null)
+    {
+      GeneralPath p;
+      float x=_cornerRect.getLowerRightX(), y=_cornerRect.getUpperLeftY(), w=8F;
+      p = new GeneralPath(GeneralPath.WIND_NON_ZERO);
+      if (Math.min(_cornerRect.getWidth(),_cornerRect.getHeight())<w)
+        w = (float) Math.min(_cornerRect.getWidth(),_cornerRect.getHeight());
+      p.moveTo(x-0.7F,   y+0.9F);
+      p.lineTo(x-0.7F-w, y+0.9F);
+      p.lineTo(x-0.7F,   y+0.9F+w);
+      p.closePath();
+      g.setColor(Color.black);
+      g.fill(p);
+    }
+  }
+
+
+  /**
+   * signal the caller (by returning ture) to update the screen, a tipbox
+   * with the label of the cluster is drawn on the offscreen-image accordingly
+   * @param offScreenImg is the image where the tip box to be drawn on
+   * @param observer
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param w is the max width of the image to be used
+   * @param h is the max height of the image to be used
+   * @return true if the tipbox is drawn on the offScreenImg, otherwise false
+   */
+  public boolean drawTipBox(Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    Graphics2D g = (Graphics2D) offScreenImg.getGraphics();
+    ClusterRect tmpRect=null;
+
+    //-- find out which rectangle the mouse points to
+    try{
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].contains(x,y))
+            {
+              tmpRect = _cRects[i][j];
+              throw new Exception();
+            }  //-- if
+
+      //-- the mouse is not pointed to any of the rectangles
+      if (_previousRect==null) return false;
+      else
+      {
+        //-- the previous pict has a tip box => draw the orginal pict
+        _previousRect = null;
+        drawTreemapView(offScreenImg, observer);
+        return true;
+      }
+    }
+    catch(Exception e){
+      //-- the mouse is pointed to a cluster's rectangle
+
+      if (_previousRect==tmpRect)
+      {
+        //-- if the tip box is already on the graphic, we don't need to update
+        return false;
+      }
+      else
+      {
+        _previousRect = tmpRect;
+        drawTreemapView(offScreenImg, observer);
+        if (_previousRect!=null)
+        {
+          _previousRect.paintLabels(g, w, h);
+        }
+        return true;
+      }
+
+    }  //-- catch
+  }  //-- drawTipBox
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * find out which rectangle the mouse point to (for inspect)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect inspect(int x, int y)
+  {
+    return findClusterRect(x, y, false);
+  }
+
+  /**
+   * find out which rectangle the mouse point to (for drill down)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect drillDown(int x, int y)
+  {
+    return findClusterRect(x, y, true);
+  }
+
+  /**
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param setChosenFlag
+   * @return the cluster rectangle that the mouse points to, and if the
+   *   'setChosenFlag' is true, then set up the corresponding flag in
+   *   _gpRect and _cRect
+   */
+  private ClusterRect findClusterRect(int x, int y, boolean setChosenFlag)
+  {
+    //-- find out which rectangle the mouse points to
+    for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+            for(int j=0;j<_cRects[i].length;j++)
+                if (_cRects[i][j].contains(x,y))
+                {
+                    if (setChosenFlag)
+                    {
+                        _gpRects[i].setChosen();
+                        _cRects[i][j].setChosen();
+                    }
+                    return _cRects[i][j];
+                }  //-- if
+    return null;
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * !!! BUG - Color is not fixed !!!
+   * initialise _gpRects and _cRects
+   * create_sub_rects is called
+   */
+  public void createRects()
+  {
+    float area = _width*_height;  //-- the area of the display
+    int noOfGroup = _data.getClusterGroupsSize();
+    float totalDocs = (float) _data.getDocCount();
+    float lx=_upperX, ly=_upperY, rx=_upperX+_width, ry=_upperY+_height, tmp=0F;
+    Color baseColor = Color.yellow;
+    ClusterGroupObj gpObj;
+    int gpSize;
+
+    for (int i=0;i<noOfGroup; i++)
+    {
+        gpObj = _data.getClusterGroup(i);
+        gpSize = gpObj.getGroupSize();
+
+    //??? Color is not fixed !!!
+        if (i==0) baseColor=Color.cyan;
+        if (i==1) baseColor=Color.green;
+        if (i==2) baseColor=Color.blue;
+        if (i==3) baseColor=Color.orange;
+        if (i==4) baseColor=Color.pink;
+        if (i==5) baseColor=Color.red;
+
+      //-- West
+        if (rx>lx && ry>ly)
+        {
+            tmp = rx;
+            rx = ((float)gpSize/totalDocs)*area/(ry-ly)+lx;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- North
+        else if (lx>rx && ry>ly)
+        {
+            tmp = ry;
+            ry = ((float)gpSize/totalDocs)*area/(lx-rx)+ly;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+      //-- East
+        else if (lx>rx && ly>ry)
+        {
+            tmp = rx;
+            rx = lx - ((float)gpSize/totalDocs)*area/(ly-ry);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+                gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- South
+        else if (rx>lx && ly>ry)
+        {
+            tmp = ry;
+            ry = ly - ((float)gpSize/totalDocs)*area/(rx-lx);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+    } //-- for
+  }  //-- createRects
+
+
+  /**
+   * !!! BUG: Color hasn't fixed !!!
+   * create the rectangles for the clusters in the specified group
+   * @param gpNo is the index of the Group currently working on
+   * @param lx is the initial left x-coordinate (based on Rect coordination)
+   * @param ly is the initial left y-coordinate (ditto)
+   * @param rx is the initial right x-coordinate (ditto)
+   * @param ry is the initial right y-coordinate (ditto)
+   * @param cs is the list of clusters in the group
+   * @param total is the total number of documents found in the group
+   * @param col is the base colour of the group
+   */
+  private void create_sub_rects(int gpNo, float lx, float ly, float rx, float ry,
+                             ClusterObj[] cs, int total, Color col)
+  {
+
+    ZRect dummy = new ZRect(lx,ly,rx,ry,null);
+    float tmp, area = (float) (dummy.getWidth()*dummy.getHeight());
+
+    for(int i=0;i<cs.length;i++)
+    {
+
+     if (rx>lx && ry>ly)
+     {
+       tmp = rx;
+        rx = ((float)cs[i].getDocsSize()/(float)total)*area/(ry-ly)+lx;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- North
+      else if (lx>rx && ry>ly)
+      {
+        tmp = ry;
+        ry = ((float)cs[i].getDocsSize()/(float)total)*area/(lx-rx)+ly;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+      //-- East
+      else if (lx>rx && ly>ry)
+      {
+        tmp = rx;
+        rx = lx - ((float)cs[i].getDocsSize()/(float)total)*area/(ly-ry);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- South
+      else if (rx>lx && ly>ry)
+      {
+        tmp = ry;
+        ry = ly - ((float)cs[i].getDocsSize()/(float)total)*area/(rx-lx);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+
+//??? Color hasn't fixed !!!
+      col = darker(col);
+
+    }
+  }  //-- create_sub_rects
+
+
+    /**
+     * @see        java.awt.Color#darker
+     */
+    private Color darker(Color c) {
+    double FACTOR = 0.85;
+      return new Color(Math.max((int)(c.getRed()  *FACTOR), 0),
+       Math.max((int)(c.getGreen()*FACTOR), 0),
+       Math.max((int)(c.getBlue() *FACTOR), 0));
+    }
+
+
+
+  //---------------------------------------------------------------------
+  //-- pre: _zoomImages can't be null
+  public Image getLastZoomingImage()
+  {
+    return _zoomingImages[_zoomingImages.length-1];
+  }
+
+  public Image getZoomingImage(int i){return _zoomingImages[i];}
+
+
+  public void show_zoomIn(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=0;i<_zoomingImages.length;i++)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+  public void show_zoomOut(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=_zoomingImages.length-1;i>=0;i--)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+
+  private void timeDelay()
+  {
+    Thread delayThread = new Thread(this);
+    delayThread.start();
+    try{
+      delayThread.join();
+    }
+    catch (InterruptedException ex)
+    {
+    }
+  }
+
+
+  public void run()
+  {
+    long lastTime=0, time=0;
+    do
+    {
+      time = System.currentTimeMillis();
+      try{
+        Thread.sleep(__FRAMEDELAY);
+      }
+      catch (InterruptedException ex)
+      {
+      }
+      lastTime = time;
+    } while (time - lastTime > __FRAMEDELAY);
+
+  }
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the animated zooming pictures on the buffer
+   * @param a list of images that is used for animation pictures
+   * @param observer
+   */
+  public void prepare_zooming(Image[] buffer, ImageObserver observer)
+  {
+    _zoomingImages = buffer;
+    prepare_zooming_buffer(observer);
+    create_zooming_picts();
+  }
+
+  /**
+   * draw the border on each buffer
+   * @param a list of buffer
+   */
+  private void prepare_zooming_buffer(ImageObserver observer)
+  {
+    Graphics2D g;
+    if (_zoomingImages==null)
+    {
+      System.err.println("Error - zooming buffer cannot be empty!!!");
+    }
+    else
+    {
+      //-- copy the static view to one buffer
+      g = (Graphics2D) _zoomingImages[0].getGraphics();
+      g.drawImage(_treemapView,0,0,observer);
+      //-- clear the main treemap area and leave the border
+      g.setColor(Color.white);
+      g.fill(new Rectangle.Float(_upperX, _upperY, _width, _height));
+      //-- copy that buffer to other buffers
+      for(int k=1;k<_zoomingImages.length;k++)
+      {
+        g = (Graphics2D) _zoomingImages[k].getGraphics();
+        g.drawImage(_zoomingImages[0],0,0,observer);
+      }
+    }
+  }
+
+
+  private void create_zooming_picts()
+  {
+    int[] edgeSize = {0,0,0,0};
+    float w_y, e_y, n_x, s_x, portion, tmp;
+    ZRect[] tmpRects;
+    Graphics tmpGph;
+    /**
+     * pre-calculate the edgeSize
+     * the edgeSize represents the total number of docs on each border
+     */
+    for(int i=0;i<_gpRects.length;i++)
+      if (!_gpRects[i].isChosen())
+        edgeSize[_gpRects[i].getDirection()] += _gpRects[i].getCapacity();
+      else
+        for(int j=0;j<_cRects[i].length;j++)
+          if(!_cRects[i][j].isChosen())
+            edgeSize[_gpRects[i].getDirection()] += _cRects[i][j].getCapacity();
+
+    //-- initise the starting points on each side
+    w_y=_upperY;                e_y=_upperY+__borderheight;
+    n_x=_upperX+__borderwidth;  s_x=_upperX;
+
+
+    for(int i=0;i<_gpRects.length;i++)
+    {
+      //-- prepare zooming the whole group, if the user has NOT chosen a cluster
+      //-- within the group
+      if (!_gpRects[i].isChosen())
+      {
+        portion = ((float)_gpRects[i].getCapacity()) /
+                  ((float) edgeSize[_gpRects[i].getDirection()]);
+        if (_gpRects[i].isWest())
+        {
+          tmp = w_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+          w_y = tmp;
+        }
+        else if (_gpRects[i].isEast())
+        {
+          tmp = e_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+          e_y = tmp;
+        }
+        else if (_gpRects[i].isNorth())
+        {
+          tmp = n_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+          n_x = tmp;
+
+        }
+        else
+        {
+          tmp = s_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+          s_x = tmp;
+        }
+
+        //-- generate a list of animation picture
+        tmpRects = _gpRects[i].zooming(_zoomingImages.length);
+
+        //-- draw the rectangles on the offsreen buffer
+        for(int j=0;j<_zoomingImages.length;j++)
+          tmpRects[j].paint(_zoomingImages[j].getGraphics());
+
+      }
+      else
+        //-- prepare zooming each cluster within the group
+        for (int j=0;j<_cRects[i].length;j++)
+          if (!_cRects[i][j].isChosen())
+          {
+            portion = ((float) _cRects[i][j].getCapacity()) /
+                      ((float) edgeSize[_gpRects[i].getDirection()]);
+            if (_gpRects[i].isWest())
+            {
+              tmp = w_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+              w_y = tmp;
+            }
+            else if (_gpRects[i].isEast())
+            {
+              tmp = e_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+              e_y = tmp;
+            }
+            else if (_gpRects[i].isNorth())
+            {
+              tmp = n_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+              n_x = tmp;
+            }
+            else
+            {
+              tmp = s_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+              s_x = tmp;
+            }  //-- if
+
+            //-- generate a list of animation picture
+            tmpRects = _cRects[i][j].zooming(_zoomingImages.length);
+
+            //-- draw the rectangles on the offsreen buffer
+            for(int k=0;k<_zoomingImages.length;k++)
+              tmpRects[k].paint(_zoomingImages[k].getGraphics());
+          }  //-- if
+    }  //-- for
+  }  //-- prepare_zooming
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Cluster.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 21112)
@@ -0,0 +1,117 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class Cluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    static int data[][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+    static int finalCluster;
+
+    static void buildMatrix ()
+    {
+	float temp;
+	float theDiff;
+	int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	    keyword1 = keywords[word1];
+	    for (int word2=0; word2<MAX;word2++)
+            {
+		keyword2 = keywords[word2];
+                theDiff = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {	
+		    temp= data[doc][keyword1]-data[doc][keyword2];
+                    theDiff+= temp*temp;
+                }
+                temp = (int)Math.sqrt(theDiff);
+                matrix[word1][word2]=matrix[word2][word1]=(int)temp;
+            }
+         }
+	for (int row = 0;row <MAX; row++)
+	   matrix[row][row] = NOVALUE;
+    } 
+
+    static void findSmallest ()
+    {
+	int group = matrix[0].length-1;
+	int smallest = NOVALUE;
+	int smallRow = NOVALUE;
+	int smallCol = NOVALUE;
+	int row,col;
+	for (row =0; row<MAX; row++)
+	    for (col=0; col<MAX; col++)
+	       if (smallest > matrix[row][col])
+	        {
+		   smallest = matrix[row][col];
+		   smallRow = row;	
+		   smallCol = col;
+ 	        }
+	for (col=0; col<MAX; col++)
+	{
+	    if (matrix[smallRow][col] < matrix[smallCol][col])
+		matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = NOVALUE;
+	}
+	addClusters(smallRow, smallCol);
+	for (row =0; row<MAX; row++)
+        {
+	    matrix[row][smallCol] = NOVALUE;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	} 	
+    }
+
+    static void addClusters(int destination, int source)
+    {
+	int i;
+	int temp[] = new int[clusters[destination].length+clusters[source].length];
+	for (i = 0; i<clusters[destination].length;i++)
+	    temp[i] = clusters[destination][i];
+	for (i = 0; i<clusters[source].length;i++)
+	    temp[i+clusters[destination].length] = clusters[source][i];
+	clusters[destination] = temp;
+	clusters[source] = EMPTYARRAY;
+	finalCluster = destination;
+    }
+
+    static void preClusters()
+    {
+	clusters = new int[MAX][1];
+	for (int row =0; row <MAX; row++)
+	{
+	    clusters[row][0] = row;
+	}
+    }
+
+    static int[] maximunDistance (int[][] newData, int[] newKeywords, int documentCount)
+    {
+	data = newData;
+	keywords = newKeywords;
+	maxDocuments = documentCount;
+	MAX = keywords.length;
+	buildMatrix();
+	preClusters();
+	for (int i=1;i<MAX;i++)
+	{
+ 	    findSmallest();
+        }
+	int temp[] = new int[MAX];
+	for (int i=1;i<MAX;i++)
+	    temp[i] = keywords[clusters[finalCluster][i]];
+	return temp;
+    }
+}
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 21112)
@@ -0,0 +1,1124 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.applet.*;
+import java.awt.event.*;
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.awt.Cursor;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class CvManyPoints extends Canvas
+{
+    protected Vishnu displayFrame;
+    protected RadVizPanel radVizPanel;
+    protected DataManager dataManager;
+    protected int [] docs;
+    
+    /** constant square shape flag */
+    static final int SQUARESHAPE = 0;
+    /** constant round shape flag */
+    static final int ROUNDSHAPE = 1;
+    /** constant diamond shape flag */
+    static final int DIAMONDSHAPE = 2;
+    /** constant maximum keywords allowed */
+    static final int ABSMAXKEYWORD = 1000;
+    /** constant maximum documents allowed */
+    static final int ABSMAXDOC = 1000;
+    /** count of the documents */
+    protected int documentCount = 1000;
+    /** circle array size */
+    protected int maxCircleCount = 40;
+    /** number of circles */
+    protected int circleCount = 40;
+    /** number of spokes */
+    protected int spokeCount = 16;
+    /** radius of the display in fp units */
+    protected float r = 10.0F;
+    /** locked flag */
+    protected boolean isLocked = false;
+    /** projected flag */
+    protected boolean isProjected = true;
+    /** document selected flag */
+    protected boolean atDoc = false;
+    /** grid flag */
+    protected boolean displayGrid = false;
+    /** rim flag */
+    protected boolean displayRim = true;
+    /** doc point shape flag */
+    protected int docsShape = SQUARESHAPE;
+    /** edge node shape flag */
+    protected int nodesShape = ROUNDSHAPE;
+    /** colour of nodes */
+    protected Color defNodesColor = Color.black;
+    /** colour of docs */
+    protected Color defDocsColor = Color.orange;
+    /** colour of highlighted nodes */
+    protected Color defHighlight = Color.blue;
+    /** colour of selected documents */
+    protected Color defSelect = Color.red;
+    /** Radius divided by radians */
+    protected float magic = (float)((double)r / (Math.PI / 2.0) );
+    /** projection factor */
+    protected float factor = 3F;
+    /** image object used for double buffering graphics */
+    protected Image image=null;
+    /** graphics used for double buffering graphics */
+    protected Graphics gImage=null;
+    /** double buffered image size */
+    protected int imgW,imgH;
+    /** dimensions of the canvas */
+    protected Dimension dim;
+    /** the integer location of 0,0 fp axis */
+    protected int centerX, centerY;
+    /** virtual pixel size and canvas fp dimensions */
+    protected float pixelSize, rWidth = 25.0F, rHeight = 25.0F;
+    /** array of edge node locations */
+    protected Point2D[] points= null;
+    /** selected edge node */
+    protected Point2D tp =null;
+    /** selected edge node index */
+    protected int tpIndex = 0;
+    /** angle stored for rotating edge nodes */
+    protected float lastAngle;
+    /** Full 2D linked list of data */
+    
+    protected Pt2DNode matrix= null;
+    /** Array of partial lists for each key word */
+    protected Pt2DNode columns[] = null;
+    /** The currently selected document if any */
+    protected Pt2DNode selectDoc = null;
+    /** A class that stores the keywords and whether they are displayed */
+    protected KeyWordArray keyWordArray;
+    /** Array of linked lists that stores the data to be displayed */
+    protected SparseMatrix theDataMatrix;
+    /** A class that maps displayed keywords to stored keywords */
+    protected MapKeyWords mapping;
+    /** A two dimensional array of integers used for clustering */
+    //protected int [][] fullMatrix;
+    protected float [][] fullMatrix;
+    /** array of data to draw target */
+    protected int circles[][] = new int[maxCircleCount][4];
+    /** array of data to draw rim */
+    protected int rim[] = new int[4];
+    /** array of data to draw spokes */
+    protected int spokes[][] = new int [spokeCount][2];
+    /** font for the mouse 'balloon' */
+    protected Font balloonFont = null;
+    /** metrics of the font for the mouse 'balloon' */
+    protected FontMetrics balloonFontMetrics = null;
+    /** Flag if drawing needs to just pasted from the buffer or redrawn */
+    protected boolean isDirty = true;
+    /** Scale the non-projected output if it goes off the page */
+    protected float scale=1.0F;
+    /** Last scale used when circles calculated (Flat mode only) */
+    protected float lastCircleScale = 1.0F;
+
+    /**
+     *  Rotate a Point2D around the 0.0,0.0 axis by phi angle (in radians)
+     *  @param Point2D around the 0.0,0.0 axis by
+     *  @param phi angle (in radians)
+     */
+    protected void rotatePt(Point2D pt,float phi)
+    {
+        float c = (float)Math.cos(phi), s = (float)Math.sin(phi),
+            r11 = c, r12 = s, r21 = -s, r22 = c;
+        float xNew = pt.x * r11 + pt.y * r21,
+            yNew = pt.x * r12 + pt.y * r22;
+        pt.x = xNew; pt.y = yNew;
+	
+    }
+    /**
+    * This is a public gateway for export of the keyWordsArray class
+    */
+    public KeyWordArray getKeyWords()
+    {
+        return keyWordArray;
+    }
+
+
+    /**
+    *  Fill the integer array circles[circleCount][4] with the correct numbers for
+    *  circleCount concentric circles about the 0.0,0.0 axis
+    */
+
+    protected void setCircles()
+    {
+        float  gap =r/4.0F, t = r/4.0F;
+        rim[0]=iX(-r); rim[1]=iY(r);
+        rim[2]= (int) (2.0F * r / pixelSize);
+        rim[3] = rim[2];
+        for (int count=0;count < circleCount; count++)
+        {
+            Point2D pc = new Point2D(-t,0.0F);
+            projection(pc);
+            circles[count][0]=iX(pc.x);
+            circles[count][2]=(int)((-pc.x) * 2.0F /pixelSize);
+            pc.x=0.0F; pc.y= t;
+            projection(pc);
+            circles[count][1]=iY(pc.y);
+            circles[count][3]=(int)(pc.y * 2.0F /pixelSize);
+            t += gap;
+        }
+        if (!isProjected) lastCircleScale=scale;
+    }
+
+    /** Create an array of linked lists that point only to the tuples affected by
+    *   each particular node, thus allowing faster update of the the locations
+    *   and easy display of only the tuples on that column of data, the membership
+    *   of these lists is subset of the larger list at 'matrix'
+    */
+    protected void setColumns()
+    {
+        int keyWordCount=keyWordArray.getNumWords();
+        columns = new Pt2DNode[keyWordCount];
+        Pt2DNode tails[] = new Pt2DNode[keyWordCount];
+        Pt2DNode k = null;
+        for (int i = 0; i< keyWordCount;i++) tails[i] = columns[i]=null;
+
+        if(keyWordCount==0) return;
+        Pt2DNode m = matrix;
+        SparseNode j=null;
+
+        // This may appear a bit obsure;
+
+        // columns[] is the start of an array of linked lists representing
+        // each of the keywords. tails[] is a parallel linked list representing the
+        // last added node to the columns[] lists;
+
+        // m loops through the rows and j iterates through the columns of each row
+        // when j finds a reference to a particular column[index] row m is at is added to
+        // the linked list for that column[index]
+
+        while (m != null)
+        {
+            j = m.tuple.row;
+            while (j != null)
+            {
+                k = new Pt2DNode(m.tuple);
+                int column = j.getColumn();
+                if(column > keyWordCount)
+                    System.out.println("col "+ column + " of " + keyWordCount);
+                else
+                {
+                    if (tails[column]!=null)
+                        tails[column].setNext(k);
+                    else columns[column] = k;
+                    tails[column]=k;
+                }
+                j=j.getNext();
+            }
+            m=m.getNext();
+        }
+    }
+    
+    /** Build a linked list of Points and tuples down connected to lists of nodes in the
+     * tuples across.
+     * @param dataItems the list
+     * @param refs[] the document titles
+     * @param docCount the number of documents
+     * @param wordCount the number of visible words
+     */
+    protected void setDatalists(SparseMatrix dataItems, String refs[],int docCount, int wordCount)
+    {
+        Pt2DNode m = null, n=null;
+        boolean state;
+        for (int co=0;co<docCount;co++)
+        {
+            SparseNode start = dataItems.getRow(co);
+            Point2D pn = new Point2D(0.0F,0.0F);
+            state = dataItemXY(pn,start);
+
+            n = new Pt2DNode(pn.x,pn.y,co,start,refs[co]);
+            if (state) projection(n.tuple.pnt);
+            n.tuple.state =state;
+            if (state)
+            {
+                n.tuple.iPnt.x=iX(n.tuple.pnt.x);
+                n.tuple.iPnt.y=iY(n.tuple.pnt.y);
+            }
+            if (m != null) m.setNext(n);
+            else matrix = n;
+            m=n;
+        }
+        setColumns();
+    }
+
+    /** Travers the linked lists setting new location values for the document points
+     * @param ma a linked list of document nodes
+     */
+    protected boolean setDocPoints(Pt2DNode ma)
+    {
+        while (ma != null)
+        {
+            ma.tuple.state = dataItemXY(ma.tuple.pnt,ma.tuple.row);
+            if (ma.tuple.state)
+            {
+                //c++;
+                projection(ma.tuple.pnt);
+                ma.tuple.iPnt.x=iX(ma.tuple.pnt.x);
+                ma.tuple.iPnt.y=iY(ma.tuple.pnt.y);
+                if (ma.tuple.iPnt.x<0 || ma.tuple.iPnt.x>dim.width ||
+                    ma.tuple.iPnt.y<0 || ma.tuple.iPnt.y>dim.height)
+                    return  true;
+            }
+            ma=ma.getNext();
+        }
+        return false;
+    }
+
+    /** Fill the integer array spokes[16][2] with the correct numbers for
+    *   16 radial points on the outermost circle, by defining a point on
+    *  the circle and rotating it by PI/8 radians 16 times
+    */
+    protected void setSpokes()
+    {
+        float phi=(float)Math.PI/(float)(spokeCount/2);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < spokeCount;count++)
+        {
+            spokes[count][0]=iX(pt.x);
+            spokes[count][1]=iY(pt.y);
+            rotatePt(pt,phi);
+        }
+    }
+
+    /** set the key word nodes in an even circle round the rim */
+    protected void setKeyNodes()
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        float phi=(float)Math.PI/((float)keyWordCount/2.0F);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < keyWordCount;count++)
+	    {
+		points[count] = new Point2D(pt.x,pt.y);
+		rotatePt(pt,phi);
+	    }
+    }
+    
+    /** find the x y coordinate of a tuple of data
+    *   @param theP point to be set
+    *   @param tuple linked list of data
+    *   @return true if tuple contains a reference to a visible keynode
+    */
+    protected boolean dataItemXY(Point2D theP, SparseNode tuple)
+    {
+        int index;
+        boolean result=false;
+        theP.x=0.0F; theP.y = 0.0F;
+        while (tuple != null)
+        {
+            if (keyWordArray.getState(tuple.getColumn()))
+            {
+                result = true;
+                index = mapping.mapWordsToNodes(tuple.getColumn());
+                theP.x += tuple.getValue()/r * points[index].x;
+                theP.y += tuple.getValue()/r * points[index].y;
+            }
+            tuple=tuple.getNext();
+        }
+        return result;
+    }
+
+    /** set the text balloon label at the correct location, to the correct size
+    *   and with the correct text
+    *   @param theString the text to display
+    *   @param x the x coordinate of the mouse
+    *   @param y the y coordinate of the mouse
+    */
+    void setBalloon(String theString,int x,int y)
+    {
+        Rectangle bounds1 = getBounds();
+	
+        x-=(bounds1.x)-30; y+=(bounds1.y)+20;
+	
+        radVizPanel.docLabel.setText(theString);
+        int width = balloonFontMetrics.stringWidth(theString)+2;
+        int height = balloonFontMetrics.getMaxAscent()+2;
+        radVizPanel.docLabel.setBounds(x,y,width,height);
+    }
+
+    /**  recalculate edge points and document positions in the light of
+    *   of changes to the keywords
+    */
+    public void recalc()
+    {
+        mapping = new MapKeyWords(keyWordArray);
+        tp =null;
+        scale=1.0F;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+        repaint();
+    }
+
+    /**
+    * constructor sets everything up gets creates an instance of DataLink and
+    * gets the data matrix and the keywords calls code to initialise display
+    * @param DisplayFrame the central object of the display
+    * @param width of canvas
+    * @param height of canvas
+    * @throws IOException
+    */
+    public CvManyPoints(Vishnu df) throws IOException
+    {
+        setVisible(false);
+       	this.displayFrame = df;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+	
+       	documentCount=0;
+       	int keyWordCount=0;
+        keyWordArray = new KeyWordArray();
+	
+       	initgr();
+       	setCircles();
+       	setSpokes();
+       	/**
+       	* add an anonymous class to listen for size changed events
+       	*/
+	
+        addComponentListener(new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt)
+		{
+		    initgr();
+		    setCircles();
+		    setSpokes();
+		    while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+		    repaint();
+		}
+	    });
+	
+        // From Java in a Nutshell
+        // listen for mouse button pressed event and if on a node point
+        // record the current angle and set the value of tp
+        /**
+	 * add an anonomous class to listen for mouse pressed events
+	 */
+        addMouseListener(new MouseAdapter()
+	    {
+		public void mousePressed(MouseEvent evt)
+		{
+		    int intX = evt.getX();
+		    int intY = evt.getY();
+		    float x = fx(intX), y = fy(intY);
+		    tp = findPoint(x,y);
+		    if (tp == null){
+			selectDoc = findDoc(intX, intY);
+			if (selectDoc != null){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			}   
+			//else theOuterFrame.setDocSelectMenu("No Document Selected      ",-1);
+			repaint();
+			return;
+		    }
+		    lastAngle=(float)Math.atan2(x,y);
+		    repaint();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int intX = event.getX();
+		    int intY = event.getY();
+		    
+		    selectDoc = findDoc(intX, intY);
+		    if (selectDoc != null){
+			if(event.getClickCount()>1){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			    dataManager.fetch(selectDoc.tuple.index);
+			}
+		    }
+		}
+		
+		public void mouseReleased(MouseEvent event)
+		{
+		    Cursor releaseCursor = new Cursor(Cursor.DEFAULT_CURSOR);
+		    displayFrame.radVizPanel.setCursor(releaseCursor);
+		}
+	    });
+	
+        // listen for mouse button dragged event and if tp is set
+        // work out a new position for tp and the data points
+        /**
+	 * add an anonomous class to listen for mouse motion events
+	 */
+        addMouseMotionListener(new MouseMotionAdapter(){
+		public void mouseDragged(MouseEvent evt)
+		{
+		    Cursor dragCursor = new Cursor(Cursor.HAND_CURSOR);
+		    displayFrame.radVizPanel.setCursor(dragCursor);
+		    
+		    float x = fx(evt.getX()), y = fy(evt.getY());
+		    if (tp!=null){
+			float thisAngle = (float)Math.atan2(x,y);
+			if(isLocked){
+			    float phi = lastAngle-thisAngle;
+			    rotatePt(tp, phi);
+			}
+			else{
+			    tp.x = x;
+			    tp.y = y;
+			}
+			if (columns!= null){
+			    int index = mapping.mapNodesToWords(tpIndex);
+			    if (columns[index]!= null){
+				if(setDocPoints( columns[index] ) && !isProjected){
+				    do{
+					scale*=0.9;
+				    } while(setDocPoints( matrix ));
+				}
+				repaint();
+			    }
+			}
+			lastAngle=thisAngle;
+		    }
+		}
+		
+		
+		
+		/** listen for mouse movement and try and find a document
+		 * report its presence with a balloon label or ensure
+		 * the balloon label is hidden
+		 * @param evt the mouse event
+		 */
+		public void mouseMoved(MouseEvent evt)
+		{
+		    System.out.println("Moving");
+		    /* int intX = evt.getX();
+		    int intY = evt.getY();
+		    Pt2DNode theNode = findDoc(intX, intY);
+		    
+		    if (theNode != null){
+			if (atDoc) radVizPanel.docLabel.setVisible(false);
+			String theString = new String("Doc: " + theNode.tuple.ref);
+			setBalloon(theNode.tuple.ref,intX,intY);
+			radVizPanel.docLabel.setVisible(true);
+			atDoc = true;
+		    }
+		    else{
+			if (atDoc){
+			    radVizPanel.docLabel.setVisible(false);
+			}
+			atDoc = false;
+			}*/
+		    return;
+		}
+	    });
+    }
+
+    /* the Data loading happens in here now */
+    
+    public void update()
+    {
+        int maxWordsCount;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+
+        //if(dataManager.getDocCount()==0)return;
+        selectDoc = null;
+        tp =null;
+       	docs = dataManager.getDocs();
+       	documentCount = docs.length;
+        String [] words = dataManager.getWords();
+        maxWordsCount = words.length;
+        //maxWordsCount= in.setKeywordCount(maxWordsCount);
+        int keyWordCount = Math.min(maxWordsCount,12);
+       	keyWordArray = new KeyWordArray(words,words.length);
+        maxWordsCount = keyWordArray.getNumWords();
+
+
+       	mapping = new MapKeyWords(keyWordArray);
+
+        String titles[] = dataManager.getDescriptions();
+        if(titles==null) System.out.println("no descriptions read in");
+        points= new Point2D[keyWordCount];
+        balloonFont = new Font("Dialog", Font.PLAIN, 16);
+        balloonFontMetrics = getFontMetrics(balloonFont);
+        theDataMatrix = dataManager.getMatrix();
+        fullMatrix = theDataMatrix.getExpanded();
+        setKeyNodes();
+        setDatalists(theDataMatrix,titles,documentCount,maxWordsCount);
+        recalc();
+    }
+    /** Cluster the words round the edge
+    * @param type the type of clustering
+    */
+    public void cluster()
+    {
+        if(documentCount ==0) return;
+        int keyWordCount=keyWordArray.getNumEnabled();
+        int keylist[] = new int[keyWordCount];
+
+        for (int c=0;c<keylist.length;c++)
+        {
+            keylist[c]=mapping.mapNodesToWords(c);
+        }
+        if(fullMatrix==null) System.out.println("full matrix is null");
+        else
+        {
+            if(fullMatrix[0]==null) System.out.println("full matrix[0] is null");
+        }
+
+        keylist = HitCluster.maximunDistance (fullMatrix, keylist, documentCount);
+
+        mapping.reMapKeyWords(keylist,keyWordArray);
+
+        tp =null;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale *=0.9;
+        repaint();
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new dimensions
+    */
+    public void setSize(Dimension d)
+    {
+        super.setSize(d);
+        initgr();
+        setCircles();
+        setSpokes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new width
+    * @param new height
+    */
+    public void setSize(int x, int y)
+    {
+        Dimension d = new Dimension(x,y);
+        setSize(d);
+    }
+
+
+
+    /**
+    * Tidy up the image before canvas is destroyed a bit like a destructor in C
+    */
+    public void dispose()
+    {
+        gImage.dispose();
+    }
+
+
+    /** Set the shape of the edge word nodes
+    * @param state selects the shape
+    */
+    void setNodeShape(int state)
+    {
+        nodesShape = state;
+        repaint();
+    }
+
+    /** Set the shape of the centre document nodes
+    *  @param state selects the shape
+    */
+    void setDocShape(int state)
+    {
+        docsShape = state;
+        repaint();
+    }
+
+    /** switches the projection on and off
+    * @param state is the switch
+    */
+    public void projection(boolean state)
+    {
+        // to do: code goes here.
+        isProjected = state;
+        circleCount = state?40:4;
+        setCircles();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Display or not the grid
+    * @param switches on and off
+    */
+
+    public void grid(boolean state)
+    {
+        displayGrid=state;
+        repaint();
+    }
+
+    /**
+    * Display or not the rim
+    * @param switches on and off
+    */
+    public void rim(boolean state)
+    {
+        displayRim=state;
+        repaint();
+    }
+
+    /**
+    * Lock or not the edge nodes so they can only be moved in a circle
+    * @param switches on and off
+    */
+    public void lock(boolean state)
+    {
+        isLocked = state;
+    }
+
+
+    /**
+    * Flattens the projection slightly
+    *
+    */
+    public void flatter()
+    {
+        if(!isProjected)
+            scale/=0.9;
+        else
+        {
+            factor *=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix)&& !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Heightens the projection slightly
+    *
+    */
+    public void higher()
+    {
+        if(!isProjected)
+            scale*=0.9;
+        else
+        {
+            factor /=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+
+
+
+    /**
+    * Translate to integer raster coordinate X from
+    * floating point coord X based around a notional centre
+    * @param x floating point x coordinate
+    * @return integer X coordinate
+    */
+    int iX(float x){return Math.round(centerX + x/pixelSize);}
+    /**
+    * Translate to integer raster coordinate Y from
+    * floating point coord Y based around a notional centre
+    * @param y floating point y coordinate
+    * @return integer Y coordinate
+    */
+    int iY(float y){return Math.round(centerY - y/pixelSize);}
+    /**
+    * Translate from integer raster coordinate X to
+    * floating point coord X based around a notional centre
+    * @param x integer x coordinate
+    * @return floating point X coordinate
+    */
+    float fx(int X){return (X - centerX) * pixelSize;}
+    /**
+    * Translate from integer raster coordinate Y to
+    * floating point coord Y based around a notional centre
+    * @param y integer y coordinate
+    * @return floating point Y coordinate
+    */
+    float fy(int Y){return (centerY - Y) * pixelSize;}
+
+    /**
+    * find if an edge keyword node is at the x y location
+    * @param x fp coordinate
+    * @param y fp coordinate
+    * @return fp Point at location or null
+    */
+    public Point2D findPoint(float x, float y)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        for (int i=0;i<keyWordCount;i++)
+        {
+            int intX = iX(x), intY = iY(y),
+                lX = iX(points[i].x)-3, hX = lX+6,
+                lY = iY(points[i].y)-3, hY = lY+6;
+
+            if (intX > lX && intX < hX && intY > lY && intY < hY)
+            {
+
+                tpIndex = i;
+                return points[i];
+            }
+        }
+        return null;
+    }
+
+    /**
+    * find if a Doc node is at the x y location
+    * @param x int coordinate
+    * @param y int coordinate
+    * @return fp Point at location or null
+    */
+    public Pt2DNode findDoc(int intX, int intY)
+    {
+        Pt2DNode thePnt=matrix;
+        while (thePnt!= null)
+        {
+            if (thePnt.tuple.state)
+            {
+                int   lX = thePnt.tuple.iPnt.x-3, hX = lX+6,
+                    lY = thePnt.tuple.iPnt.y-3, hY = lY+6;
+                if (intX > lX && intX < hX && intY > lY && intY < hY) return thePnt;
+            }
+            thePnt = thePnt.getNext();
+        }
+        return null;
+    }
+    /** project the point radially from the centre using the arctan function
+    *  if isProjected is true or scale them it false
+    *  @param point to be projected
+    *  @param ingore scaling flag
+    */
+    protected void projection(Point2D p)
+    {
+        if (!isProjected)
+        {
+            if(scale!=1.0)
+            {
+                p.x*=scale;
+                p.y*=scale;
+            }
+            return;
+        }
+        float newRad;
+        float rad = (float)Math.sqrt(p.x*p.x+p.y*p.y);
+        float sine = p.y/rad;
+        float cosine = p.x/rad;
+        newRad =(float)Math.atan(rad/(factor*Math.PI))*magic;
+        p.x = newRad * cosine;
+        p.y = newRad * sine;
+    }
+
+    /** set up graphic settings such as centre and pixelsize
+    */
+    protected void initgr()
+    {
+        dim = getSize();
+        int maxX = dim.width - 1, maxY = dim.height - 1;
+        pixelSize = Math.max(rWidth/maxX, rHeight/maxY);
+        centerX = maxX/2; centerY = maxY/2;
+    }
+
+
+    /////////Athina has been here!!!!!///////////////////////////////////////
+
+    /** draw a for document node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    protected void drawItemPoint(int x, int y, Graphics g)
+    {
+
+        if (docsShape == SQUARESHAPE)
+            oval(g,x,y);
+        else if (docsShape == ROUNDSHAPE)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+    /**
+    *   draw a for keyword node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    void drawNodes(int x, int y, Graphics g)
+    {
+        if (nodesShape == 0)
+            oval(g,x,y);
+        else if (nodesShape == 1)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+
+    /**
+    *  draw the key name near the edgenode
+    *  @param g Graphics class
+    *  @param count index of the name to draw
+    *  @param select if the node is selected.
+    */
+    void drawKeyName(Graphics g, int count, boolean select)
+    {
+        FontMetrics fontMetrics = g.getFontMetrics();
+        Font font = new Font("arial", Font.PLAIN, 16);
+        String word = keyWordArray.getWord(mapping.mapNodesToWords(count));
+        int posX= iX(points[count].x)-2, posY= iY(points[count].y)-2;
+        int width = fontMetrics.stringWidth(word);
+        if (points[count].x < 0.0)
+            posX -= width - (int) (points[count].x/2.00);
+        else
+            posX += (int) (points[count].x/2.00);
+
+        if (points[count].y < 0.0)
+            posY += fontMetrics.getMaxAscent()- (int)(points[count].y/2.00);
+        else
+            posY -= (int) (points[count].y/2.00);
+        g.setFont(font);
+        g.drawString(word,posX,posY);
+        if (select){ g.drawRect(posX-2,posY-15,width+4,20);}
+    }
+
+    /**
+    * draw a box at the edge nodes
+    * @param g Graphics object
+    */
+    protected void drawEdgeNodes(Graphics g)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+
+        for (int count=0;count < keyWordCount; count++)
+        {
+            drawNodes(iX(points[count].x), iY(points[count].y), g);
+            drawKeyName(g,count,false);
+
+        }
+    }
+    /**
+    * Draw the documents that have the selected key word in them
+    * @param g Graphics object
+    */
+    protected void drawSelect(Graphics g)
+    {
+        if (tp!=null)
+        {
+            int x = iX(points[tpIndex].x), y = iY(points[tpIndex].y);
+            g.setColor(defSelect);
+            drawNodes(x, y, g);
+
+            Pt2DNode m = columns[mapping.mapNodesToWords(tpIndex)];
+            while (m != null)
+            {
+                x=m.tuple.iPnt.x;
+                y=m.tuple.iPnt.y;
+
+                drawItemPoint(x, y, g);
+
+                m=m.getNext();
+            }
+            g.setColor(defDocsColor);
+
+        }
+    }
+    /**
+    * draw the key word nodes that feature in the selected document
+    */
+    protected void drawDocSelect(Graphics g)
+    {
+        if (selectDoc==null) return;
+        g.setColor(defHighlight);
+        int x =selectDoc.tuple.iPnt.x, y =selectDoc.tuple.iPnt.y;
+        drawItemPoint(x, y, g);
+
+        SparseNode fn = selectDoc.tuple.row;
+        while (fn != null)
+        {
+            if (keyWordArray.getState(fn.getColumn()))
+            {
+                int index = mapping.mapWordsToNodes(fn.getColumn());
+                drawNodes(iX(points[index].x), iY(points[index].y), g);
+                drawKeyName(g,index,true);
+            }
+            fn=fn.getNext();
+
+        }
+        g.setColor(defNodesColor);
+
+    }
+
+    /**
+    * draw all the document crosses
+    * @param g Graphics object
+    */
+
+    protected void drawDocXs(Graphics g)
+    {
+        Pt2DNode m = matrix;
+        while (m != null)
+        {
+            if (m.tuple.state) drawItemPoint(m.tuple.iPnt.x, m.tuple.iPnt.y, g);
+            m=m.getNext();
+        }
+    }
+
+    /**
+    * draw the target
+    * @param g Graphics object
+    */
+
+    protected void drawTarget(Graphics g)
+    {
+        if (!isProjected && lastCircleScale!=scale)
+            setCircles();
+
+        if (displayRim)
+        {
+            g.setColor(Color.darkGray);
+            g.drawOval(rim[0],rim[1],rim[2],rim[3]);
+        }
+        if (displayGrid)
+        {
+            g.setColor(Color.lightGray);
+            for (int count=0;count < circleCount; count++)
+                g.drawOval(circles[count][0],circles[count][1],circles[count][2],circles[count][3]);
+            for (int count=0;count < spokeCount; count++)
+                g.drawLine(centerX, centerY,spokes[count][0],spokes[count][1]);
+        }
+    }
+
+    /**
+    * Repaint the screen overides canvas.repaint()
+    */
+
+    public void repaint()
+    {
+        isDirty=true;
+        super.repaint();
+    }
+
+    /**
+    * Repaint the screen overides canvas.update()
+    * to stop flickering
+    * @param g Graphics object
+    */
+
+    public void update(Graphics g)
+    {
+        paint(g);
+    }
+
+    /**
+    * draw a filled square with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+
+    public void rect(Graphics g, int x, int y)
+    {
+        g.drawRect(x-2,y-2,4,4);
+        g.fillRect(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled circle with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void oval(Graphics g, int x, int y)
+    {
+        g.drawOval(x-2,y-2,4,4);
+        g.fillOval(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled diamond with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void diamond(Graphics g, int x, int y)
+    {
+        int numpoints = 4;
+        int[] xpoints = new int[numpoints+1];
+        int[] ypoints = new int[numpoints+1];
+        for(int i=0; i < numpoints; i++) {
+            double angle = 2*Math.PI * i / numpoints;
+            xpoints[i] = (int)(x + 3*Math.cos(angle));
+            ypoints[i] = (int)(y - 3*Math.sin(angle));
+        }
+
+        g.fillPolygon(xpoints, ypoints, numpoints);
+        xpoints[numpoints] = xpoints[0];  ypoints[numpoints] = ypoints[0];
+        g.drawPolygon(xpoints, ypoints, numpoints+1);
+        repaint();
+    }
+
+    /**
+    * paint the whole thing
+    * @param g Graphics object
+    */
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Repaint CvManyPoints");
+        if(!isVisible()) return;
+        if (isDirty==true)
+        {
+            if (imgW != dim.width || imgH != dim.height)
+            {
+                if( gImage != null ) gImage.dispose();
+                imgW = dim.width; imgH = dim.height;
+                image = createImage(imgW, imgH);
+                gImage = image.getGraphics();
+            }
+            gImage.setColor(Color.white);
+            gImage.fillRect(0, 0, imgW, imgH);
+
+            drawTarget(gImage);
+            gImage.setColor(defNodesColor);
+            drawEdgeNodes(gImage);
+            gImage.setColor(defDocsColor);
+            drawDocXs(gImage);
+
+            if (tp!=null)
+                drawSelect(gImage);
+            if (selectDoc!=null)
+                drawDocSelect(gImage);
+
+            //gImage.drawRect(1,1,dim.width-1,dim.height-1);
+        }
+        g.drawImage(image, 0, 0, null);
+
+        isDirty=false;
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 21112)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class HitCluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    // static int data[][];
+    static float data [][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+
+    static void buildMatrix ()
+    {
+	    int count;
+	    int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	        keyword1 = keywords[word1];
+	        for (int word2=word1+1; word2<MAX;word2++)
+            {
+ 		        keyword2 = keywords[word2];
+                count = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {
+		            if ( (data[doc][keyword1] > 0) && (data[doc][keyword2] > 0))
+			            count++;
+	            }
+                matrix[word1][word2]=matrix[word2][word1]=count;
+            }
+        }
+	    for (int row = 0;row <MAX; row++)
+	    {
+	        matrix[row][row] = NOVALUE;
+	        for (int col = 0; col <row; col++)
+		    matrix[row][col] = matrix[col][row];
+	    }
+    }
+
+    static void findLargest ()
+    {
+	    int group = matrix[0].length-1;
+	    int largest = -1;
+	    int smallRow = NOVALUE;
+	    int smallCol = NOVALUE;
+	    int row,col;
+	    for (row =0; row<MAX; row++)
+	        for (col=row+1; col<MAX; col++)
+	            if (largest < matrix[row][col])
+	            {
+		            largest = matrix[row][col];
+		            smallRow = row;
+		            smallCol = col;
+ 	            }
+	    for (col=0; col<MAX; col++)
+	    {
+	        if (matrix[smallRow][col] > matrix[smallCol][col])
+		        matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = -1;
+	    }
+	    clusters[smallRow] = addClusters(smallRow, smallCol);
+	    clusters[smallCol] = EMPTYARRAY;
+	    for (row =0; row<MAX; row++)
+        {
+	        matrix[row][smallCol] = -1;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	    }
+    }
+
+    static int[] addClusters(int c1, int c2)
+    {
+	    int i;
+	    int temp[] = new int[clusters[c1].length+clusters[c2].length];
+	    if (keywords[clusters[c1][0]] < keywords[clusters[c2][0]])
+	    {
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i] = clusters[c1][i];
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i+clusters[c1].length] = clusters[c2][i];
+	    }
+	    else
+	    {
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i] = clusters[c2][i];
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i+clusters[c2].length] = clusters[c1][i];
+	    }
+	    return temp;
+    }
+
+    static void preClusters()
+    {
+	    clusters = new int[MAX][1];
+	    for (int row =0; row <MAX; row++)
+	    {
+	        clusters[row][0] = row;
+	    }
+    }
+
+    static int[] maximunDistance (float[][] newData, int[] newKeywords, int documentCount)
+    {
+	    data = newData;
+	    keywords = newKeywords;
+	    maxDocuments = documentCount;
+	    if (documentCount > data.length)
+	    {
+	        System.out.println ("WARNING matrix sent is shorter the document Count!");
+	        System.out.println ("I will use the shorter length only.");
+	        documentCount = data.length;
+        }
+	    MAX = keywords.length;
+	    buildMatrix();
+	    preClusters();
+	    for (int i=1;i<MAX;i++)
+	    {
+ 	        findLargest();
+        }
+	    int temp[] = new int[MAX];
+	    for (int i=0;i<MAX;i++)
+	    {
+	        temp[i] = keywords[clusters[0][i]];
+//System.out.println (i+" = "+keywords[i]+" = "+clusters[0][i]);
+    	}
+	    return temp;
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Hlink.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 21112)
@@ -0,0 +1,118 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.JEditorPane;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+
+public class Hlink extends JEditorPane
+{
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+    public int numDisplayedDocs = 15;
+    private int width, length;
+
+    public Hlink(int width, int length , Vishnu currentMain)
+    {
+        this.currentMain=currentMain;
+        this.width=width;
+        this.length=length;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public Hlink()
+    {
+        this.currentMain=null;
+        this.width=500;
+        this.length=300;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public void jbInit() throws Exception
+    {
+        linkPane = new JScrollPane(this);
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener
+    {
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+                System.out.println("Mouse ACTIVATED -> " + he.getURL().toString());
+        }
+    }
+
+    public void update()
+    {
+       int [] docIDs = currentMain.dataManager.getDocs();
+        Vector subSet = new Vector();
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+
+            // Check that the element exists
+            if (docIDs.length <= i) break;
+
+            subSet.addElement(new Integer(docIDs[i]));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDescriptionStr(subSet);
+
+        setText(htmlDescription);
+
+    }
+
+    public void clear()
+    {
+        setText("");
+    }
+
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 21112)
@@ -0,0 +1,17 @@
+package vishnu.testvis.visual;
+/** Integer point class 
+* (minimal)
+* @author Matthew Carey */
+
+public class Ipoint2D
+{
+   /** x value */
+   public int x;
+   /** y value */
+   public int y;
+   /** constructor 
+   * @param horizontal value
+   * @param vertical value
+   */
+   public Ipoint2D(int x, int y){this.x = x; this.y = y;}
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 21112)
@@ -0,0 +1,106 @@
+/** Class to store the keywords and their visible states
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+public class KeyWordArray
+{
+    /** constant default number enabled */
+    public static final int DEFAULT_ENABLED = 12;
+    /** the word array */
+    private String words[];
+    /** the enabled array */
+    private boolean enabled[];
+    /** the count */
+    private int count;
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    * @param number enabled
+    */
+    public KeyWordArray(String words[],int count,int numEnabled)
+    {
+        this.words = words;
+        this.count = count;
+        enabled = new boolean[count];
+        int theNumEnabled = count<numEnabled?count:numEnabled;
+        for (int c = 0; c < numEnabled && c < count; c++)
+            enabled[c]=true;
+        if (count > theNumEnabled)
+            for (int c = theNumEnabled; c < count; c++)
+                enabled[c]=false;
+    }
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    */
+    public KeyWordArray(String words[],int count)
+    {
+        this(words,count,DEFAULT_ENABLED);
+    }
+
+    public KeyWordArray()
+    {
+        this.count = 0;
+        this.words = null;
+        this.enabled = null;
+    }
+
+    /**
+    * Find the size of the array
+    * @return size of the array
+    */
+    public int getNumWords()
+    {
+        return count;
+    }
+
+    /**
+    * Find the word at index
+    * @param index of word
+    * @return word at index
+    */
+    public String getWord(int index)
+    {
+        if (index >=0 && index < count)
+           return this.words[index];
+        return "Error out of index";
+    }
+
+    /**
+    * Get state of word at index
+    * @param index of word
+    * @return state of word
+    */
+    public boolean getState(int index)
+    {
+        if (index >=0 && index < count)
+           return this.enabled[index];
+        return false;
+    }
+
+    /**
+    * Set state of word at index
+    * @param index of word
+    * @param new state of word
+    */
+    public void setState(int index, boolean state)
+    {
+        if (index >=0 && index < count)
+           this.enabled[index]=state;
+    }
+
+    /**
+    * Get number of enabled words
+    * @return number of enabled words
+    */
+    public int getNumEnabled()
+    {
+        int result = 0;
+        for (int c = 0; c < count; c++)
+            if (enabled[c]) result++;
+        return result;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 21112)
@@ -0,0 +1,188 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import java.awt.List;
+import java.applet.*;
+import javax.swing.*;
+
+/** A panel that is added to the frame containing an on screen list of keywords to select
+*   and deselect
+*   @author Athina Economou
+*/
+public class KeywordPanel extends JPanel
+{
+    /** the base frame */
+    protected RadVizPanel f;
+    protected KeyWordArray keyWordArray;
+    protected int wordCount;
+    protected Vector  selArrayIndex;
+    protected Vector  listArrayIndex;
+    protected List available;
+    protected List selected;
+    protected int largeFont = 16;
+    protected int smallFont = 14;
+
+
+    /** Constructor
+    * @param drawing canvas
+    * @param base frame
+    */
+    public KeywordPanel(final CvManyPoints canvas1, RadVizPanel f)
+    {
+        this.f = f;
+
+        keyWordArray = canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selArrayIndex = new Vector();
+        listArrayIndex = new Vector();
+        available = new List(12, false);
+        selected = new List(12, false);
+        // mc
+        SymAction lSymAction = new SymAction();
+
+//        Panel base = new Panel();
+        setLayout(new BorderLayout(10,0));
+        //setSize(50,400);
+        JPanel labels = new JPanel();
+        Font font1 = new Font("Monospaced", Font.PLAIN, largeFont);
+        labels.setFont(font1);
+        labels.setForeground(Color.black);
+        labels.add(new Label("Available           Displayed"));
+
+        JPanel rightPanel = new JPanel();
+        Font font = new Font("Dialog", Font.PLAIN, smallFont);
+        rightPanel.setFont(font);
+        JPanel leftPanel = new JPanel();
+        leftPanel.setFont(font);
+
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+        available.addActionListener(lSymAction);
+        available.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    if(event==null) return;
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)listArrayIndex.elementAt(index);
+                            selected.add(keywords);
+                            selArrayIndex.addElement(result);
+
+                            available.remove(keywords);
+                            listArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                             keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } //try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    } // catch
+                } //itemstate changed
+            }); // ItemListener
+            //mc
+            selected.addActionListener(lSymAction);
+            selected.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    //System.out.println("calling getSeleced index");
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                    //System.out.println("Index");
+                    //System.out.println(index);
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)selArrayIndex.elementAt(index);
+
+                            available.add(keywords);
+                            listArrayIndex.addElement(result);
+
+                            selected.remove(keywords);
+                            selArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                                keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } // try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    }
+                }
+            });
+
+            leftPanel.add(available);
+            rightPanel.add(selected);
+            add(labels, "North");
+            add(rightPanel, "East");
+            add(leftPanel, "West");
+            //base.add(text, "South");
+            setVisible(true);
+
+    }
+    public void update()
+    {
+        keyWordArray = f.canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selected.removeAll();
+        available.removeAll();
+        selArrayIndex.removeAllElements();
+        listArrayIndex.removeAllElements();
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+    }
+
+
+    /** A dummy action listener to absorb double mouse clicks */
+    class SymAction implements java.awt.event.ActionListener
+    {
+        public void actionPerformed(java.awt.event.ActionEvent event)
+        {
+            Object object = event.getSource();
+            //System.out.println("Action event");
+        }
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 21112)
@@ -0,0 +1,127 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class ListPanel extends JPanel
+{
+    JList documentsList = new JList();
+    BorderLayout borderLayout1 = new BorderLayout();
+    Border border1;
+    JLabel descriptionLabel = new JLabel();
+    Vishnu displayFrame;
+    JPanel southPanel = new JPanel();
+    JButton fetchButton = new JButton();
+
+    public ListPanel()
+    {
+        displayFrame=null;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public ListPanel(Vishnu d)
+    {
+        displayFrame=d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border1);
+        descriptionLabel.setFont(new java.awt.Font("Monospaced", 0, 14));
+        descriptionLabel.setText("Document descriptions");
+        documentsList.setFont(new java.awt.Font("Monospaced", 0, 12));
+        documentsList.setToolTipText("");
+        documentsList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+        documentsList.setVisibleRowCount(10);
+        documentsList.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                documentsList_mouseClicked(e);
+            }
+        });
+        fetchButton.setFont(new java.awt.Font("Monospaced", 0, 16));
+        fetchButton.setBorder(BorderFactory.createEtchedBorder());
+        fetchButton.setText("Fetch");
+        fetchButton.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                fetchButton_mouseClicked(e);
+            }
+        });
+        this.add(documentsList, BorderLayout.CENTER);
+        this.add(descriptionLabel, BorderLayout.NORTH);
+        this.add(southPanel, BorderLayout.SOUTH);
+        southPanel.add(fetchButton, null);
+    }
+
+    public void update()
+    {
+       documentsList.setListData(displayFrame.dataManager.getDescriptions());
+       repaint();
+    }
+
+    void documentsList_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseEntered(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseExited(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mousePressed(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseReleased(MouseEvent e)
+    {
+
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 21112)
@@ -0,0 +1,78 @@
+package vishnu.testvis.visual;
+ /** This class is maps the display array of keywords onto the stored.
+  *  array
+  *  @author Matthew Carey</a>
+  */
+public class MapKeyWords
+{
+   /** array map for display to stored */
+   protected int nodesToWord[] = null;
+   /** array map for stored to display */
+   protected int wordsToNodes[] = null;
+   /**
+   * map for display to stored
+   * @param display index
+   * @return stored index
+   */
+   public int mapNodesToWords(int index)
+   {
+        return nodesToWord[index];
+   }
+   /**
+   * map for stored to display
+   * @param stored index
+   * @return the display index
+   */
+   public int mapWordsToNodes(int index)
+   {
+        return wordsToNodes[index];
+   }
+   /**
+   * constructor
+   * @param keywordarray instance containing list of key words and states
+   */
+   public MapKeyWords(KeyWordArray keyWordArray)
+   {
+        int c, n=0;
+        nodesToWord = null;
+        wordsToNodes = null;
+        nodesToWord = new int [keyWordArray.getNumEnabled()];
+        wordsToNodes = new int [keyWordArray.getNumWords()];
+        int totalKeyWord=keyWordArray.getNumWords();
+        for (c=0;c < totalKeyWord; c++)
+        {
+             if (keyWordArray.getState(c))
+             {
+                wordsToNodes[c]=n;
+                nodesToWord[n]=c;
+                n++;
+             }
+             else
+                wordsToNodes[c]=-1;
+        }
+   }
+   /**
+   * re-Map the KeyWords with a new vector
+   * @param array the new vector
+   * @param keywordarray instance containing list of key words and states
+   */
+   public void reMapKeyWords(int array[],KeyWordArray keyWordArray)
+   {
+        if (array.length != keyWordArray.getNumEnabled())
+            return;
+        int c;
+        for (c=0;c<array.length;c++)
+        {
+            if (!keyWordArray.getState(array[c]))
+               return;
+        }
+
+        for (c=0;c<array.length;c++)
+        {
+                wordsToNodes[array[c]]=c;
+                nodesToWord[c]=array[c];
+        }
+   }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 21112)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.util.*;
+
+/**
+ * A custom component that displays multiple lines of text with specified
+ * margins and alignment.  In Java 1.1, we could extend Component instead
+ * of Canvas, making this a more efficient "Lightweight component"
+ * from Java Examples in a Nutshell,
+ * @author Flanagan, David - O'Rielly 1997 NY
+ */
+public class MultiLineLabel extends Canvas {
+  // User-specified attributes
+  protected String label;             // The label, not broken into lines
+  protected int margin_width;         // Left and right margins
+  protected int margin_height;        // Top and bottom margins
+  protected int alignment;            // The alignment of the text.
+  public static final int LEFT = 0, CENTER = 1, RIGHT = 2; // alignment values
+  // Computed state values
+  protected int num_lines;            // The number of lines
+  protected String[] lines;           // The label, broken into lines
+  protected int[] line_widths;        // How wide each line is
+  protected int max_width;            // The width of the widest line
+  protected int line_height;          // Total height of the font
+  protected int line_ascent;          // Font height above baseline
+  protected boolean measured = false; // Have the lines been measured?
+
+  // Here are five versions of the constructor
+  public MultiLineLabel(String label, int margin_width,
+                        int margin_height, int alignment) {
+    this.label = label;                 // Remember all the properties
+    this.margin_width = margin_width;
+    this.margin_height = margin_height;
+    this.alignment = alignment;
+    newLabel();                         // Break the label up into lines
+  }
+  public MultiLineLabel(String label, int margin_width, int margin_height) {
+    this(label, margin_width, margin_height, LEFT);
+  }
+  public MultiLineLabel(String label, int alignment) {
+    this(label, 10, 10, alignment);
+  }
+  public MultiLineLabel(String label) { this(label, 10, 10, LEFT); }
+  public MultiLineLabel() { this(""); }
+
+  // Methods to set and query the various attributes of the component
+  // Note that some query methods are inherited from the superclass.
+  public void setLabel(String label) {
+    this.label = label;
+    newLabel();               // Break the label into lines
+    measured = false;         // Note that we need to measure lines
+    repaint();                // Request a redraw
+  }
+  public void setFont(Font f) {
+    super.setFont(f);         // tell our superclass about the new font
+    measured = false;         // Note that we need to remeasure lines
+    repaint();                // Request a redraw
+  }
+  public void setForeground(Color c) {
+    super.setForeground(c);   // tell our superclass about the new color
+    repaint();                // Request a redraw (size is unchanged)
+  }
+  public void setAlignment(int a) { alignment = a; repaint(); }
+  public void setMarginWidth(int mw) { margin_width = mw; repaint(); }
+  public void setMarginHeight(int mh) { margin_height = mh; repaint(); }
+  public String getLabel() { return label; }
+  public int getAlignment() { return alignment; }
+  public int getMarginWidth() { return margin_width; }
+  public int getMarginHeight() { return margin_height; }
+
+  /**
+   * This method is called by a layout manager when it wants to
+   * know how big we'd like to be.  In Java 1.1, getPreferredSize() is
+   * the preferred version of this method.  We use this deprecated version
+   * so that this component can interoperate with 1.0 components.
+   */
+  public Dimension getPreferredSize() {
+    if (!measured) measure();
+    return new Dimension(max_width + 2*margin_width,
+                         num_lines * line_height + 2*margin_height);
+  }
+
+  /**
+   * This method is called when the layout manager wants to know
+   * the bare minimum amount of space we need to get by.
+   * For Java 1.1, we'd use getMinimumSize().
+   */
+  public Dimension getMinimumSize() { return getPreferredSize(); }
+
+  /**
+   * This method draws the label (same method that applets use).
+   * Note that it handles the margins and the alignment, but that
+   * it doesn't have to worry about the color or font--the superclass
+   * takes care of setting those in the Graphics object we're passed.
+   */
+  public void paint(Graphics g) {
+    int x, y;
+    Dimension size = this.getSize();  // use getSize() in Java 1.1
+    if (!measured) measure();
+    y = line_ascent + (size.height - num_lines * line_height)/2;
+    for(int i = 0; i < num_lines; i++, y += line_height) {
+      switch(alignment) {
+      default:
+      case LEFT:    x = margin_width; break;
+      case CENTER:  x = (size.width - line_widths[i])/2; break;
+      case RIGHT:   x = size.width - margin_width - line_widths[i]; break;
+      }
+      g.drawString(lines[i], x, y);
+    }
+  }
+
+  /** This internal method breaks a specified label up into an array of lines.
+   *  It uses the StringTokenizer utility class. */
+  protected synchronized void newLabel() {
+    StringTokenizer t = new StringTokenizer(label, "\n");
+    num_lines = t.countTokens();
+    lines = new String[num_lines];
+    line_widths = new int[num_lines];
+    for(int i = 0; i < num_lines; i++) lines[i] = t.nextToken();
+  }
+
+  /** This internal method figures out how the font is, and how wide each
+   *  line of the label is, and how wide the widest line is. */
+  protected synchronized void measure() {
+    FontMetrics fm = this.getFontMetrics(this.getFont());
+    line_height = fm.getHeight();
+    line_ascent = fm.getAscent();
+    max_width = 0;
+    for(int i = 0; i < num_lines; i++) {
+      line_widths[i] = fm.stringWidth(lines[i]);
+      if (line_widths[i] > max_width) max_width = line_widths[i];
+    }
+    measured = true;
+  }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 21112)
@@ -0,0 +1,153 @@
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Date;
+
+import javax.swing.JPanel;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class PlainPanel extends JPanel
+{
+    Vishnu displayFrame;
+    BorderLayout borderLayout1;
+    Border border1;
+    JScrollPane theScrollPane;
+    JEditorPane theEditorPane;
+
+
+  // This method returns an HTML table
+    public String makeHTMLTable()
+    {
+        String [] descriptions = displayFrame.dataManager.getDescriptions();
+        int [] docNums = displayFrame.dataManager.getDocs();
+        // Set up an output stream we can print the table to.
+        // This is easier than concatenating strings all the time.
+        int number;
+        if (docNums==null) number = 0;
+        else number = docNums.length;
+
+        StringWriter sout = new StringWriter();
+        PrintWriter out = new PrintWriter(sout);
+
+        // Print out the table
+	Dimension d = displayFrame.getSize();
+	int totalWidth = d.width-40;
+	int textWidth = totalWidth - 100;
+	
+        out.println("<TABLE BORDER= 2 WIDTH=" + totalWidth + "><TR>");
+
+        out.println("<TH WIDTH= 100>Document Number</TH>");
+        out.println("<TH WIDTH = " + textWidth + ">Sample Text</TH></TR>");
+
+        for(int i=0; i < docNums.length; i++)
+        {
+            out.print("<TR><TD><A HREF=\"");
+            out.print(docNums[i]);
+            out.print("\">");
+            out.print(docNums[i]);
+            out.print("</A></TD><TD>");
+            out.print(descriptions [i]);
+            out.println("</TD></TR>");
+        }
+        out.println("</TABLE>");
+        out.close();
+
+        // Get the string of HTML from the StringWriter and return it.
+        return sout.toString();
+    }
+
+    public PlainPanel()
+    {
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    public PlainPanel(Vishnu d)
+    {
+        displayFrame = d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+// The sychronized qualifier is placed here because when two plain
+// panels are created in different threads almost simulataniously
+// we seem to get interference happening
+    public synchronized void jbInit() throws Exception
+    {
+        borderLayout1 = new BorderLayout();
+        theEditorPane = new JEditorPane();
+
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        theEditorPane.setEditable(false);               // we're browsing not editing
+        theEditorPane.setContentType("text/html");      // must specify HTML text
+        theEditorPane.setText("<TABLE BORDER= 2 WIDTH=640><TR>"+
+                                "<TH WIDTH= 100>Document Number</TH>" +
+                                "<TH WIDTH = 520>Sample Text</TH></TR>" +
+                                "</TABLE>");  // specify the text to display
+        theEditorPane.setFont(new java.awt.Font("Monospaced", 0, 12));
+    // Set up the JEditorPane to handle clicks on hyperlinks
+        theEditorPane.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        if (index > 0)
+                            displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+
+        // Put the JEditorPane in a scrolling window and display it.
+
+        this.setLayout(borderLayout1);
+        this.setFont(new java.awt.Font("Monospaced", 0, 12));
+        this.setBorder(border1);
+        theScrollPane = new JScrollPane();
+        //theScrollPane.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_ALWAYS);
+        this.add(theScrollPane, BorderLayout.CENTER);
+        theScrollPane.getViewport().add(theEditorPane, null);
+        setSize(600, 400);
+        setVisible(true);
+    }
+
+    public void update()
+    {
+       theEditorPane.setText(makeHTMLTable());
+    }
+
+    public void paint(Graphics g)
+    {
+	update();
+    }
+
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 21112)
@@ -0,0 +1,42 @@
+/**
+* The head of the rows of floating point lists contains drawing and
+* title information about a document as well as the document index
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+import vishnu.datablock.*;
+
+public class Pnt2DTupleNode
+{
+    /** index of the document */
+    public int index;
+    /** start of linked list */
+    public SparseNode row;
+    /** cartesian floating point location */
+    public Point2D pnt;
+    /** raster integer locatoin */
+    public Ipoint2D iPnt;
+    /** drawn or not */
+    public boolean state;
+    /** document title */
+    public String ref;
+    /**
+    * Constructor
+    * @param cartesian x
+    * @param cartesain y
+    * @param index
+    * @param linked list of data
+    * @param title
+    */
+    Pnt2DTupleNode(float x, float y ,int i, SparseNode fn, String reference)
+    {
+        state = false;
+        index = i;
+        pnt = new Point2D(x,y);
+        iPnt = new Ipoint2D(0,0);
+        ref=reference;
+        if (reference==null) ref = "Not defined";
+        row=fn;
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 21112)
@@ -0,0 +1,53 @@
+package vishnu.testvis.visual;
+import vishnu.datablock.*;
+/**
+* Downward linked list wrapper class enables several lists to share
+* same row nodes
+* @author Matthew Carey
+*/
+
+public class Pt2DNode
+{
+     /** next node */
+     Pt2DNode next;
+     /** row data */
+     Pnt2DTupleNode tuple;
+     /** constructor
+     * @param another node to link to
+     */
+     Pt2DNode(Pnt2DTupleNode t)
+     {
+        tuple = t;
+        next = null;
+     }
+     /**
+     * constructor
+     * @param cartesian x
+     * @param cartesian y
+     * @param document index
+     * @param data row
+     * @param document title
+     */
+     Pt2DNode(float x, float y ,int i, SparseNode t, String s)
+     {
+        tuple = new Pnt2DTupleNode(x,y,i,t,s);
+        next = null;
+     }
+     /**
+     * Get next node
+     * @return next node
+     */
+     Pt2DNode getNext()
+     {
+        return next;
+     }
+     /**
+     * Set next node
+     * @param another node to be next
+     */
+     public void setNext(Pt2DNode n)
+     {
+        n.next=next;
+        next = n;
+     }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 21112)
@@ -0,0 +1,300 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class RadVizPanel extends JPanel
+{
+    protected CvManyPoints canvas1;
+    public java.awt.Label docLabel;
+    protected KeywordPanel keyWordPanel;
+    protected Panel south = new Panel();
+
+    protected Border border1;
+
+    protected JPanel buttonPanel = new JPanel();
+    protected JPanel buttonBasePanel = new JPanel();
+    protected JPanel labelPanel = new JPanel();
+    protected JLabel buttonLabel = new JLabel();
+    protected JButton zoomInButton = new JButton();
+    protected JButton zoomOutButton = new JButton();
+    protected JButton resetButton = new JButton();
+    protected JButton clusterButton = new JButton();
+
+    protected JCheckBox showGridCheckBox = new JCheckBox();
+    protected JCheckBox lockCheckBox = new JCheckBox();
+    protected JCheckBox showRimCheckBox = new JCheckBox();
+    protected JCheckBox projectCheckBox = new JCheckBox();
+
+    protected int smallFont = 14;
+    protected int largeFont = 16;
+    protected Border border3;
+    protected BorderLayout borderLayout1 = new BorderLayout(10,0);
+    protected Vishnu parent;
+
+    public RadVizPanel(Vishnu p)
+    {
+        parent = p;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(5,5,5,5);
+        border3 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border3);
+        this.setPreferredSize(new Dimension(600, 400));
+        canvas1 = new CvManyPoints (parent);
+	
+	// This is used as a balloon by the canvas1
+        docLabel = new java.awt.Label("  ");
+        docLabel.setVisible(false);
+
+	// place it out of view if does become visible
+        docLabel.setBounds(12,-20,10,10); 
+        docLabel.setBackground(new Color(16776895));
+        docLabel.setFont(new Font("Dialog", Font.PLAIN, largeFont));
+	
+        add(docLabel);
+        add(canvas1, "Center");
+	
+        keyWordPanel = new KeywordPanel(canvas1, this);
+	keyWordPanel.setVisible(false);
+	
+        buttonBasePanel.setLayout(new BorderLayout(10,10));
+        buttonPanel.setLayout(new GridLayout(0,2,10,10));
+        buttonLabel.setText("Settings");
+        buttonLabel.setForeground(Color.black);
+        buttonLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	
+        zoomInButton.setText("Zoom in");
+        zoomInButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        zoomInButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomInButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomInButton_mouseClicked(e);
+		}
+	    });
+        zoomOutButton.setText("Zoom Out");
+        zoomOutButton.setFont(new java.awt.Font("Monospaced", 0,largeFont));
+        zoomOutButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomOutButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomOutButton_mouseClicked(e);
+		}
+	    });
+        resetButton.setText("Reset");
+        resetButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        resetButton.setBorder(BorderFactory.createEtchedBorder());
+        resetButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    resetButton_mouseClicked(e);
+		}
+	    });
+        clusterButton.setText("Cluster");
+        clusterButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        clusterButton.setBorder(BorderFactory.createEtchedBorder());
+        clusterButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    clusterButton_mouseClicked(e);
+		}
+	    });
+	
+        showGridCheckBox.setText("Show Grid");
+        showGridCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showGridCheckBox.setSelected(false);
+        showGridCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showGridCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showGridCheckBox_itemStateChanged(e);
+		}
+	    });
+        showRimCheckBox.setText("Show Rim");
+        showRimCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showRimCheckBox.setSelected(true);
+        showRimCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showRimCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showRimCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+        projectCheckBox.setText("Projection");
+        projectCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        projectCheckBox.setSelected(true);
+        projectCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        projectCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    projectCheckBox_itemStateChanged(e);
+		}
+	    });
+        lockCheckBox.setText("Lock Radius");
+        lockCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        lockCheckBox.setSelected(false);
+        lockCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        lockCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    lockCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+	
+        buttonPanel.add(zoomInButton);
+        buttonPanel.add(zoomOutButton);
+        buttonPanel.add(resetButton);
+        buttonPanel.add(clusterButton);
+        buttonPanel.add(showGridCheckBox);
+        buttonPanel.add(showRimCheckBox);
+        buttonPanel.add(projectCheckBox);
+        buttonPanel.add(lockCheckBox);
+	
+        labelPanel.add(buttonLabel);
+        buttonBasePanel.add(buttonPanel,"Center");
+        buttonBasePanel.add(labelPanel,"North");
+        south.setLayout(new BorderLayout(10,0));
+	south.add(keyWordPanel,"West");
+        south.add(buttonBasePanel,"East");
+        south.setVisible(false);
+        add(south,"South");
+    }
+    
+    
+    public void setBothVisible(boolean f)
+    {
+        keyWordPanel.setVisible(f);
+        canvas1.setVisible(f);
+        south.setVisible(f);
+    }
+    
+    
+    public void addNotify()
+    {
+	//Record the size of the window prior to calling parents addNotify.
+        Dimension d = getSize();
+
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    
+    void  zoomOutButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.higher();
+    }
+
+
+    void  zoomInButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.flatter();
+    }
+
+
+    void  resetButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.recalc();
+    }
+
+
+    void  clusterButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.cluster();
+    }
+
+
+    void showGridCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.grid(showGridCheckBox.isSelected());
+    }
+
+
+    void showRimCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.rim(showRimCheckBox.isSelected());
+    }
+
+
+    void projectCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.projection(projectCheckBox.isSelected());
+    }
+
+
+    void lockCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.lock(lockCheckBox.isSelected());
+    }
+
+
+    public void update()
+    {
+        canvas1.update();
+        keyWordPanel.update();
+    }
+
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        setBothVisible(f);
+    }
+
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Updating RadVizPanel");
+	//canvas1.update();
+	keyWordPanel.update();
+    }
+    
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 21112)
@@ -0,0 +1,205 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import javax.swing.plaf.basic.*;
+import java.awt.*;
+import java.awt.event.*;
+
+
+/** Panel containing the graphical search interface */
+public class SimplePanel extends JPanel {
+
+    JTextField textField;
+    Vishnu currentMain;
+
+
+    public class InfoLabel extends JLabel {
+        void update() {
+            int startIndex = localHLinkPanel.index +1;
+            int endIndex = localHLinkPanel.index + localHLinkPanel.numDisplayedDocs;
+            int numDocs = currentMain.dataManager.getDocCount();
+            if (numDocs < endIndex) endIndex = numDocs;
+
+            setText("Documents "+ startIndex + "-" + endIndex + " out of " + numDocs);
+        }
+
+    }
+
+    InfoLabel infoLabel = new InfoLabel();
+
+    Hlink localHLinkPanel;
+
+    JButton backButton, nextButton;
+    public SimplePanel(Vishnu theMain)
+    {
+        this.currentMain = theMain;
+        localHLinkPanel = new Hlink(600,400,currentMain);
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public SimplePanel()
+
+    {
+        this.currentMain = null;
+        localHLinkPanel = new Hlink();
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    private void jbInit() throws Exception
+    {
+        setName("Simple ");
+
+
+        backButton = new JButton("<<");
+        backButton.setBorder(BorderFactory.createEtchedBorder());
+        backButton.setEnabled(false);
+        backButton.addMouseListener( new MouseAdapter() {
+            public void mouseClicked(MouseEvent e) {
+
+                if (!backButton.isEnabled()) return;
+
+                nextButton.setEnabled(true);
+
+                localHLinkPanel.index -= localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index == 0) {
+                    backButton.setEnabled(false);
+                }
+            }
+        });
+        nextButton = new JButton(">>");
+        nextButton.setBorder(BorderFactory.createEtchedBorder());
+        nextButton.setEnabled(false);
+        nextButton.addMouseListener( new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+
+                if (!nextButton.isEnabled()) return;
+
+                backButton.setEnabled(true);
+
+                localHLinkPanel.index += localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index + localHLinkPanel.numDisplayedDocs >= currentMain.dataManager.getDocCount())
+                {
+                    nextButton.setEnabled(false);
+                }
+            }
+        });
+
+        //
+
+        // Set the new layout
+        GridBagLayout gridBag = new GridBagLayout();
+        setLayout(gridBag);
+
+        GridBagConstraints constraints = new GridBagConstraints();
+
+        constraints.fill = GridBagConstraints.BOTH;
+        constraints.insets = new Insets(10,3,10,3);
+        constraints.ipadx = 0;
+        constraints.ipady = 8;
+
+
+
+        // Link panel
+        constraints.ipady = 4;
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = GridBagConstraints.REMAINDER;
+        constraints.gridheight = 8;
+
+        gridBag.setConstraints(localHLinkPanel.linkPane,constraints);
+        add(localHLinkPanel.linkPane);
+
+        // Info label
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = 6;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        infoLabel.setMinimumSize(new Dimension(500,20));
+        infoLabel.setMaximumSize(new Dimension(500,20));
+        gridBag.setConstraints(infoLabel,constraints);
+        add(infoLabel);
+
+        // Back button
+
+        constraints.fill = GridBagConstraints.NONE;
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        Dimension buttonDimension = new Dimension(55,30);
+        backButton.setMaximumSize(buttonDimension);
+        backButton.setPreferredSize(buttonDimension);
+        backButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(backButton,constraints);
+        add(backButton);
+
+        // Next button
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+
+        nextButton.setMaximumSize(buttonDimension);
+        nextButton.setPreferredSize(buttonDimension);
+        nextButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(nextButton,constraints);
+        add(nextButton);
+
+        // Global layout
+        setSize(600,400);
+        setPreferredSize(new Dimension(600,400));
+        setMinimumSize(new Dimension(600,400));
+
+        setVisible(true);
+    }
+
+
+    public void update() {
+
+        localHLinkPanel.index = 0;
+
+        infoLabel.update();
+        localHLinkPanel.update();
+
+        if (currentMain.dataManager.getDocCount() > localHLinkPanel.numDisplayedDocs) {
+            nextButton.setEnabled(true);
+        }
+
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 21112)
@@ -0,0 +1,445 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class Vishnu extends JApplet
+{
+    public JTabbedPane theTabbedPane = null;
+    public PlainPanel plainPanel = null;
+    public GraphicalPane sammonPanel = null;
+    public RadVizPanel radVizPanel = null;
+    public DendroPanel dendroPanel = null;
+    public JButton findButton = null;
+    
+    public DataManager dataManager = null;
+    
+    public Logging log = null;
+    public Queries queries = null;
+    public boolean searchStarted = false;
+    public boolean firstSearch = true;
+    public boolean refine = false;
+    
+    public JTextField searchBox = null;
+    public JComboBox collCombo = null;
+    public JComboBox viewCombo = null;
+    protected int largeFont = 16;
+    
+    public String query = null;
+    public String coll = null;
+    public String view = null;
+    
+    public String server_address = null;
+    
+    protected Dimension size = null;
+        
+    public void init()
+    {
+	String library = getParameter("library");
+	if (library == null ) {
+	    System.err.println("Error: library parameter is null!!");
+	    // what should we do?
+	    return;
+	}
+	String viscgi = getParameter("viscgi");
+	
+	query = getParameter("query");
+        coll = getParameter("collection");
+	view = getParameter("view");
+	
+	server_address = library;
+	if (viscgi != null) {
+	    // if viscgi parameter is set (even if empty) we assume that
+	    // the server is running  cgi-like with arguments in the url
+	    server_address += viscgi;
+	    server_address = tidy_URL(server_address, true);
+	} else {
+	    server_address = tidy_URL(server_address, false);
+	}
+	
+	// What kind of engine the dataManager should use
+	String engine = getParameter("engine");
+ 
+	String w = getParameter("width");
+	String h = getParameter("height");
+	
+	if( (w != null) && (h != null) )
+	    size = new Dimension(Integer.parseInt(w),Integer.parseInt(h));
+        else size = new Dimension(1000,800);
+	
+	dataManager = new DataManager(this, engine);
+	
+        this.setSize(size); 
+        
+	theTabbedPane = new JTabbedPane();
+
+	radVizPanel = new RadVizPanel(this);
+        radVizPanel.setName("RadialVis");
+        
+        plainPanel = new PlainPanel(this);
+        plainPanel.setName("PlainVis");
+        
+        sammonPanel = new GraphicalPane(this);
+        sammonPanel.setName("SammonVis");
+        
+        dendroPanel = new DendroPanel(this);
+        dendroPanel.setName("DendroVis");
+	
+	findButton = new JButton();
+        try {
+	    jbInit();
+	} catch(Exception e){e.printStackTrace();}
+	
+	initCollections();
+	
+	if( query != null && !query.equals("null") ){
+	    searchBox.setText(query);
+	    findQuery();
+	}
+    }
+
+    protected void initCollections() {
+
+	Collection current_collection=null;
+	String current_view = null;
+	if( coll == null || coll.equals("") || coll.equals("null") ) {
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	} else {
+	    // try to find the one specified in the parameters
+	    for( int i = 0; i < dataManager.collections.size(); i++ ){
+		Collection c = (Collection)dataManager.collections.elementAt(i);
+		if( c.dir.equals(coll) ){
+		    current_collection = c;
+		    current_view = view; // use the one from the parameters
+		    break;
+		}
+	    }
+	}
+	if (current_collection == null) {
+	    // could still be null if we specified an invalid collection in the parameters
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	}
+	if (current_view == null) {
+	    if (current_collection.views != null) {
+		current_view = (String)current_collection.views[0];
+	    }
+	}
+	dataManager.currentCollection = current_collection;
+	dataManager.currentView = current_view;
+	this.view = current_view;
+	dataManager.requestCollection();
+	System.out.println("Requesting collection " + current_collection.name);
+	collCombo.setSelectedItem(current_collection);
+	if (viewCombo != null && current_view != null) {
+	    viewCombo.setSelectedItem(current_view);
+	    
+	}
+	    
+    }
+
+    protected void jbInit() throws Exception
+    {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+	JPanel collPanel = new JPanel();
+	collPanel.setLayout(new BorderLayout(5,5));
+	collPanel.setBackground(SystemColor.control);
+
+	collCombo = new JComboBox(dataManager.collections);
+	collCombo.setEditable(false);
+	collCombo.addItemListener(new java.awt.event.ItemListener() {
+		public void itemStateChanged(ItemEvent i) {
+		    if (i.getStateChange()==ItemEvent.SELECTED){
+			dataManager.currentCollection = (Collection)i.getItem();
+			collCombo.hidePopup();
+			coll = dataManager.currentCollection.dir;
+			if (dataManager.currentCollection.views != null) {
+			    viewCombo.setModel(new DefaultComboBoxModel(dataManager.currentCollection.views));
+			    viewCombo.setEnabled(true);
+			    view = (String)viewCombo.getItemAt(0);
+			    dataManager.currentView = view;
+
+			} else {
+			    viewCombo.setModel(new DefaultComboBoxModel());
+			    viewCombo.setEnabled(false);
+			    view = null;
+			    dataManager.currentView = view;
+			}
+			dataManager.requestCollection();
+
+			update();
+			
+		    }
+		    
+		}
+		});
+	collCombo.setRenderer(new javax.swing.DefaultListCellRenderer() {
+		public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus)
+		{
+		    setText(((Collection)value).name);
+		    setFont(list.getFont());
+		    return this;
+		}
+	    });
+
+	JLabel collLabel = new JLabel();           
+	collLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        collLabel.setForeground(Color.black); 
+        collLabel.setText("Collection:");
+	
+	collPanel.add(collLabel, BorderLayout.WEST);
+	collPanel.add(collCombo, BorderLayout.CENTER);
+
+	JPanel viewPanel = new JPanel();
+	viewPanel.setLayout(new BorderLayout(5,5));
+	viewPanel.setBackground(SystemColor.control);
+
+	viewCombo = new JComboBox();
+	viewCombo.setEnabled(false);
+
+	JLabel viewLabel = new JLabel();           
+	viewLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        viewLabel.setForeground(Color.black); 
+        viewLabel.setText("View:");
+	
+	viewPanel.add(viewLabel, BorderLayout.WEST);
+	viewPanel.add(viewCombo, BorderLayout.CENTER);
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	JPanel boxesPanel = new JPanel();
+	boxesPanel.setLayout(new BorderLayout(5,5));
+	boxesPanel.setBackground(SystemColor.control);
+	boxesPanel.add(collPanel, BorderLayout.WEST);
+	boxesPanel.add(viewPanel, BorderLayout.EAST);
+	
+	topPanel.add(boxesPanel, BorderLayout.WEST);
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+    public void update()
+    {
+	plainPanel.update();
+        radVizPanel.update();
+        dendroPanel.update();        
+        sammonPanel.update();
+    }
+    
+        
+    void findQuery()
+    {
+	refine = false;
+	
+        int docCount;
+	String text = null;
+	
+	text = (String)searchBox.getText();
+	if(text.trim().length() < 1) return;
+	query = text.trim();
+	
+	if (viewCombo != null) {
+	    String view1 = null;
+	    if (viewCombo.isEnabled()) {
+		view1 = (String)viewCombo.getSelectedItem();	    
+	    }
+	    if (!view1.equals(this.view)) {
+		this.view = view1;
+		dataManager.currentView = view1;
+		dataManager.requestCollection(); // resets previous data
+	    }
+	}
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	
+	int count = 1;
+	String q = query;
+	docCount = dataManager.query(q);
+	if (docCount == 0 ) {
+      	    JOptionPane.showMessageDialog(this,"No documents found related to " + text);
+	    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    return;
+	}
+
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+
+    void theTabbedPane_stateChanged(ChangeEvent e)
+    {
+	for( int i = 0; i < theTabbedPane.getTabCount(); i++ )
+        {
+            theTabbedPane.setForegroundAt(i,Color.black);
+        }
+        
+        int tab = theTabbedPane.getSelectedIndex();
+        
+        theTabbedPane.setForegroundAt(tab,Color.blue);
+    }
+
+
+  void findButton_keyPressed(KeyEvent e)
+  {
+      int key = e.getKeyCode();
+      if (key==10)
+	  {
+	      findButton.setSelected(true);
+	      findQuery();
+	      findButton.setSelected(false);
+	  }
+  }
+    
+    void findButton_mouseClicked(MouseEvent e)
+    {
+	findQuery();
+    }
+    
+   
+    // Tidy up URLs
+    //
+    // Ensure a URL address (as string) has a protocol, host, and file.
+    //
+    // If the URL is a CGI script URL, it should be tidied up so that it is
+    // appropriate to tage attrib=value pairs on the end.  This means it
+    // must either end with a "?" or (if it contains a question-mark
+    // internally) end with a "&".
+    // taken from Phind.java, Greenstone digital library.
+    String tidy_URL(String address, boolean isCGI) {
+
+	// System.err.println("tidy URL: " + address);
+	
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+	    
+	}
+	
+	// if the URL is a cgi script, make sure it has a "?" in ti,
+	// and that it ends with a "?" or "&"
+	if (isCGI) {
+	    if (address.indexOf((int) '?') == -1) { 
+		address = address + "?";
+	    } else if (!address.endsWith("?")) {
+		address = address + "&";
+	    }
+	}
+
+	return address;
+    }
+
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 21112)
@@ -0,0 +1,118 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class VishnuSingle extends Vishnu
+{
+
+    protected void initCollections() {
+	// put in a dummy Collection
+	Collection c = new Collection(coll, coll, coll, 0, coll, null);
+	dataManager.currentCollection = c;
+    }
+
+    // overwrite this to remove the collection stuff - cos we are working within one collection only
+    protected void jbInit() throws Exception {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+}
+
+
+
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Base64.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Base64.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Base64.java	(revision 21112)
@@ -0,0 +1,1281 @@
+package vishnu.util;
+
+import java.io.*;
+/**
+ * Encodes and decodes to and from Base64 notation.
+ *
+ * <p>
+ * Change Log:
+ * </p>
+ * <ul>
+ *  <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the
+ *   encoded data was a single byte.</li>
+ *  <li>v2.0 - I got rid of methods that used booleans to set options. 
+ *   Now everything is more consolidated and cleaner. The code now detects
+ *   when data that's being decoded is gzip-compressed and will decompress it
+ *   automatically. Generally things are cleaner. You'll probably have to
+ *   change some method calls that you were making to support the new
+ *   options format (<tt>int</tt>s that you "OR" together).</li>
+ *  <li>v1.5.1 - Fixed bug when decompressing and decoding to a             
+ *   byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>.      
+ *   Added the ability to "suspend" encoding in the Output Stream so        
+ *   you can turn on and off the encoding if you need to embed base64       
+ *   data in an otherwise "normal" stream (like an XML file).</li>  
+ *  <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself.
+ *      This helps when using GZIP streams.
+ *      Added the ability to GZip-compress objects before encoding them.</li>
+ *  <li>v1.4 - Added helper methods to read/write files.</li>
+ *  <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li>
+ *  <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream
+ *      where last buffer being read, if not completely full, was not returned.</li>
+ *  <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li>
+ *  <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li>
+ * </ul>
+ *
+ * <p>
+ * I am placing this code in the Public Domain. Do with it as you will.
+ * This software comes with no guarantees or warranties but with
+ * plenty of well-wishing instead!
+ * Please visit <a href="http://iharder.net/xmlizable">http://iharder.net/base64</a>
+ * periodically to check for updates or to contribute improvements.
+ * </p>
+ *
+ * @author Robert Harder
+ * @author rob@iharder.net
+ * @version 2.0
+ */
+public class Base64
+{
+    
+/* ********  P U B L I C   F I E L D S  ******** */   
+    
+    
+    /** No options specified. Value is zero. */
+    public final static int NO_OPTIONS = 0;
+    
+    /** Specify encoding. */
+    public final static int ENCODE = 1;
+    
+    
+    /** Specify decoding. */
+    public final static int DECODE = 0;
+    
+    
+    /** Specify that data should be gzip-compressed. */
+    public final static int GZIP = 2;
+    
+    
+    /** Don't break lines when encoding (violates strict Base64 specification) */
+    public final static int DONT_BREAK_LINES = 8;
+    
+    
+/* ********  P R I V A T E   F I E L D S  ******** */  
+    
+    
+    /** Maximum line length (76) of Base64 output. */
+    private final static int MAX_LINE_LENGTH = 76;
+    
+    
+    /** The equals sign (=) as a byte. */
+    private final static byte EQUALS_SIGN = (byte)'=';
+    
+    
+    /** The new line character (\n) as a byte. */
+    private final static byte NEW_LINE = (byte)'\n';
+    
+    
+    /** The 64 valid Base64 values. */
+    private final static byte[] ALPHABET =
+    {
+        (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G',
+        (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N',
+        (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', 
+        (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z',
+        (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g',
+        (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n',
+        (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', 
+        (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z',
+        (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', 
+        (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/'
+    };
+    
+    /** 
+     * Translates a Base64 value to either its 6-bit reconstruction value
+     * or a negative number indicating some other meaning.
+     **/
+    private final static byte[] DECODABET =
+    {   
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,                 // Decimal  0 -  8
+        -5,-5,                                      // Whitespace: Tab and Linefeed
+        -9,-9,                                      // Decimal 11 - 12
+        -5,                                         // Whitespace: Carriage Return
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 14 - 26
+        -9,-9,-9,-9,-9,                             // Decimal 27 - 31
+        -5,                                         // Whitespace: Space
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,              // Decimal 33 - 42
+        62,                                         // Plus sign at decimal 43
+        -9,-9,-9,                                   // Decimal 44 - 46
+        63,                                         // Slash at decimal 47
+        52,53,54,55,56,57,58,59,60,61,              // Numbers zero through nine
+        -9,-9,-9,                                   // Decimal 58 - 60
+        -1,                                         // Equals sign at decimal 61
+        -9,-9,-9,                                      // Decimal 62 - 64
+        0,1,2,3,4,5,6,7,8,9,10,11,12,13,            // Letters 'A' through 'N'
+        14,15,16,17,18,19,20,21,22,23,24,25,        // Letters 'O' through 'Z'
+        -9,-9,-9,-9,-9,-9,                          // Decimal 91 - 96
+        26,27,28,29,30,31,32,33,34,35,36,37,38,     // Letters 'a' through 'm'
+        39,40,41,42,43,44,45,46,47,48,49,50,51,     // Letters 'n' through 'z'
+        -9,-9,-9,-9                                 // Decimal 123 - 126
+        /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 127 - 139
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 140 - 152
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 153 - 165
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 166 - 178
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 179 - 191
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 192 - 204
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 205 - 217
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 218 - 230
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 231 - 243
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9         // Decimal 244 - 255 */
+    };
+    
+    private final static byte BAD_ENCODING    = -9; // Indicates error in encoding
+    private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding
+    private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding
+
+    
+    /** Defeats instantiation. */
+    private Base64(){}
+    
+    
+    
+/* ********  E N C O D I N G   M E T H O D S  ******** */    
+    
+    
+    /**
+     * Encodes the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     *
+     * @param threeBytes the array to convert
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes )
+    {   
+        return encode3to4( threeBytes, 3 );
+    }   // end encodeToBytes
+    
+    
+    
+    /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     *
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes, int numSigBytes )
+    {  
+        byte[] dest = new byte[4];
+        encode3to4( threeBytes, 0, numSigBytes, dest, 0 );
+        return dest;
+    }
+    
+     /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     * Code can reuse a byte array by passing a four-byte array as <var>b4</var>.
+     *
+     * @param b4 A reusable byte array to reduce array instantiation
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.5.1
+     */
+    private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes )
+    {
+        encode3to4( threeBytes, 0, numSigBytes, b4, 0 );
+        return b4;
+    }   // end encode3to4
+
+    
+    /**
+     * Encodes up to three bytes of the array <var>source</var>
+     * and writes the resulting four Base64 bytes to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 3 for
+     * the <var>source</var> array or <var>destOffset</var> + 4 for
+     * the <var>destination</var> array.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param numSigBytes the number of significant bytes in your array
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the <var>destination</var> array
+     * @since 1.3
+     */
+    private static byte[] encode3to4( 
+     byte[] source, int srcOffset, int numSigBytes,
+     byte[] destination, int destOffset )
+    {
+        //           1         2         3  
+        // 01234567890123456789012345678901 Bit position
+        // --------000000001111111122222222 Array position from threeBytes
+        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
+        //          >>18  >>12  >> 6  >> 0  Right shift necessary
+        //                0x3f  0x3f  0x3f  Additional AND
+        
+        // Create buffer with zero-padding if there are only one or two
+        // significant bytes passed in the array.
+        // We have to shift left 24 in order to flush out the 1's that appear
+        // when Java treats a value as negative that is cast from a byte to an int.
+        int inBuff =   ( numSigBytes > 0 ? ((source[ srcOffset     ] << 24) >>>  8) : 0 )
+                     | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 )
+                     | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 );
+
+        switch( numSigBytes )
+        {
+            case 3:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = ALPHABET[ (inBuff       ) & 0x3f ];
+                return destination;
+                
+            case 2:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            case 1:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = EQUALS_SIGN;
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            default:
+                return destination;
+        }   // end switch
+    }   // end encode3to4
+    
+    
+    
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * The object is not GZip-compressed before being encoded.
+     *
+     * @param serializableObject The object to encode
+     * @return The Base64-encoded object
+     * @since 1.4
+     */
+    public static String encodeObject( java.io.Serializable serializableObject )
+    {
+        return encodeObject( serializableObject, NO_OPTIONS );
+    }   // end encodeObject
+    
+
+
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     * @param serializableObject The object to encode
+     * @param options Specified options
+     * @return The Base64-encoded object
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeObject( java.io.Serializable serializableObject, int options )
+    {
+        // Streams
+        java.io.ByteArrayOutputStream  baos  = null; 
+        java.io.OutputStream           b64os = null; 
+        java.io.ObjectOutputStream     oos   = null; 
+        java.util.zip.GZIPOutputStream gzos  = null;
+        
+        // Isolate options
+        int gzip           = (options & GZIP);
+        int dontBreakLines = (options & DONT_BREAK_LINES);
+        
+        try
+        {
+            // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream
+            baos  = new java.io.ByteArrayOutputStream();
+            b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+    
+            // GZip?
+            if( gzip == GZIP )
+            {
+                gzos = new java.util.zip.GZIPOutputStream( b64os );
+                oos  = new java.io.ObjectOutputStream( gzos );
+            }   // end if: gzip
+            else
+                oos   = new java.io.ObjectOutputStream( b64os );
+            
+            oos.writeObject( serializableObject );
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            return null;
+        }   // end catch
+        finally
+        {
+            try{ oos.close();   } catch( Exception e ){}
+            try{ gzos.close();  } catch( Exception e ){}
+            try{ b64os.close(); } catch( Exception e ){}
+            try{ baos.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return new String( baos.toByteArray() );
+    }   // end encode
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source )
+    {
+        return encodeBytes( source, 0, source.length, NO_OPTIONS );
+    }   // end encodeBytes
+    
+
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int options )
+    {   
+        return encodeBytes( source, 0, source.length, options );
+    }   // end encodeBytes
+    
+    
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source, int off, int len )
+    {
+        return encodeBytes( source, off, len, NO_OPTIONS );
+    }   // end encodeBytes
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int off, int len, int options )
+    {
+        // Isolate options
+        int dontBreakLines = ( options & DONT_BREAK_LINES );
+        int gzip           = ( options & GZIP   );
+        
+        // Compress?
+        if( gzip == GZIP )
+        {
+            java.io.ByteArrayOutputStream  baos  = null;
+            java.util.zip.GZIPOutputStream gzos  = null;
+            Base64.OutputStream            b64os = null;
+            
+    
+            try
+            {
+                // GZip -> Base64 -> ByteArray
+                baos = new java.io.ByteArrayOutputStream();
+                b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+                gzos  = new java.util.zip.GZIPOutputStream( b64os ); 
+            
+                gzos.write( source, off, len );
+                gzos.close();
+            }   // end try
+            catch( java.io.IOException e )
+            {
+                e.printStackTrace();
+                return null;
+            }   // end catch
+            finally
+            {
+                try{ gzos.close();  } catch( Exception e ){}
+                try{ b64os.close(); } catch( Exception e ){}
+                try{ baos.close();  } catch( Exception e ){}
+            }   // end finally
+
+            return new String( baos.toByteArray() );
+        }   // end if: compress
+        
+        // Else, don't compress. Better not to use streams at all then.
+        else
+        {
+            // Convert option to boolean in way that code likes it.
+            boolean breakLines = dontBreakLines == 0;
+            
+            int    len43   = len * 4 / 3;
+            byte[] outBuff = new byte[   ( len43 )                      // Main 4:3
+                                       + ( (len % 3) > 0 ? 4 : 0 )      // Account for padding
+                                       + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines      
+            int d = 0;
+            int e = 0;
+            int len2 = len - 2;
+            int lineLength = 0;
+            for( ; d < len2; d+=3, e+=4 )
+            {
+                encode3to4( source, d+off, 3, outBuff, e );
+
+                lineLength += 4;
+                if( breakLines && lineLength == MAX_LINE_LENGTH )
+                {   
+                    outBuff[e+4] = NEW_LINE;
+                    e++;
+                    lineLength = 0;
+                }   // end if: end of line
+            }   // en dfor: each piece of array
+
+            if( d < len )
+            {
+                encode3to4( source, d+off, len - d, outBuff, e );
+                e += 4;
+            }   // end if: some padding needed
+
+            return new String( outBuff, 0, e );
+        }   // end else: don't compress
+        
+    }   // end encodeBytes
+    
+
+    
+    
+    
+/* ********  D E C O D I N G   M E T H O D S  ******** */
+    
+    
+    /**
+     * Decodes the first four bytes of array <var>fourBytes</var>
+     * and returns an array up to three bytes long with the
+     * decoded values.
+     *
+     * @param fourBytes the array with Base64 content
+     * @return array with decoded values
+     * @since 1.3
+     */
+    private static byte[] decode4to3( byte[] fourBytes )
+    {
+        byte[] outBuff1 = new byte[3];
+        int    count    = decode4to3( fourBytes, 0, outBuff1, 0 );
+        byte[] outBuff2 = new byte[ count ];
+        
+        for( int i = 0; i < count; i++ )
+            outBuff2[i] = outBuff1[i];
+        
+        return outBuff2;
+    }
+    
+    
+    
+    
+    /**
+     * Decodes four bytes from array <var>source</var>
+     * and writes the resulting bytes (up to three of them)
+     * to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 4 for
+     * the <var>source</var> array or <var>destOffset</var> + 3 for
+     * the <var>destination</var> array.
+     * This method returns the actual number of bytes that 
+     * were converted from the Base64 encoding.
+     * 
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the number of decoded bytes converted
+     * @since 1.3
+     */
+    private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset )
+    {
+        // Example: Dk==
+        if( source[ srcOffset + 2] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 );
+            
+            destination[ destOffset ] = (byte)( outBuff >>> 16 );
+            return 1;
+        }
+        
+        // Example: DkL=
+        else if( source[ srcOffset + 3 ] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6 );
+            
+            destination[ destOffset     ] = (byte)( outBuff >>> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>>  8 );
+            return 2;
+        }
+        
+        // Example: DkLE
+        else
+        {
+            try{
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 )
+            //              | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6)
+                          | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF )      );
+
+            
+            destination[ destOffset     ] = (byte)( outBuff >> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>  8 );
+            destination[ destOffset + 2 ] = (byte)( outBuff       );
+
+            return 3;
+            }catch( Exception e){
+                System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset     ] ]  ) );
+                System.out.println(""+source[srcOffset+1]+  ": " + ( DECODABET[ source[ srcOffset + 1 ] ]  ) );
+                System.out.println(""+source[srcOffset+2]+  ": " + ( DECODABET[ source[ srcOffset + 2 ] ]  ) );
+                System.out.println(""+source[srcOffset+3]+  ": " + ( DECODABET[ source[ srcOffset + 3 ] ]  ) );
+                return -1;
+            }   //e nd catch
+        }
+    }   // end decodeToBytes
+    
+    
+    
+    
+    /**
+     * Very low-level access to decoding ASCII characters in
+     * the form of a byte array. Does not support automatically
+     * gunzipping or any other "fancy" features.
+     *
+     * @param source The Base64 encoded data
+     * @param off    The offset of where to begin decoding
+     * @param len    The length of characters to decode
+     * @return decoded data
+     * @since 1.3
+     */
+    public static byte[] decode( byte[] source, int off, int len )
+    {
+        int    len34   = len * 3 / 4;
+        byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output
+        int    outBuffPosn = 0;
+        
+        byte[] b4        = new byte[4];
+        int    b4Posn    = 0;
+        int    i         = 0;
+        byte   sbiCrop   = 0;
+        byte   sbiDecode = 0;
+        for( i = off; i < off+len; i++ )
+        {
+            sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits
+            sbiDecode = DECODABET[ sbiCrop ];
+            
+            if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better
+            {
+                if( sbiDecode >= EQUALS_SIGN_ENC )
+                {
+                    b4[ b4Posn++ ] = sbiCrop;
+                    if( b4Posn > 3 )
+                    {
+                        outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn );
+                        b4Posn = 0;
+                        
+                        // If that was the equals sign, break out of 'for' loop
+                        if( sbiCrop == EQUALS_SIGN )
+                            break;
+                    }   // end if: quartet built
+                    
+                }   // end if: equals sign or better
+                
+            }   // end if: white space, equals sign or better
+            else
+            {
+                System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" );
+                return null;
+            }   // end else: 
+        }   // each input character
+                                   
+        byte[] out = new byte[ outBuffPosn ];
+        System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); 
+        return out;
+    }   // end decode
+    
+    
+    
+    
+    /**
+     * Decodes data from Base64 notation, automatically
+     * detecting gzip-compressed data and decompressing it.
+     *
+     * @param s the string to decode
+     * @return the decoded data
+     * @since 1.4
+     */
+    public static byte[] decode( String s )
+    {   
+        byte[] bytes = s.getBytes();
+        bytes = decode( bytes, 0, bytes.length );
+        
+        // Check to see if it's gzip-compressed
+        // GZIP Magic Two-Byte Number: 0x8b1f (35615)
+        if( bytes.length >= 2 )
+        {
+            
+            int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);       
+            if(
+            bytes != null &&      // In case decoding returned null
+            bytes.length >= 4  && // Don't want to get ArrayIndexOutOfBounds exception
+            java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) 
+            {
+                java.io.ByteArrayInputStream  bais = null;
+                java.util.zip.GZIPInputStream gzis = null;
+                java.io.ByteArrayOutputStream baos = null;
+                byte[] buffer = new byte[2048];
+                int    length = 0;
+
+                try
+                {
+                    baos = new java.io.ByteArrayOutputStream();
+                    bais = new java.io.ByteArrayInputStream( bytes );
+                    gzis = new java.util.zip.GZIPInputStream( bais );
+
+                    while( ( length = gzis.read( buffer ) ) >= 0 )
+                    {
+                        baos.write(buffer,0,length);
+                    }   // end while: reading input
+
+                    // No error? Get new bytes.
+                    bytes = baos.toByteArray();
+
+                }   // end try
+                catch( java.io.IOException e )
+                {
+                    // Just return originally-decoded bytes
+                }   // end catch
+                finally
+                {
+                    try{ baos.close(); } catch( Exception e ){}
+                    try{ gzis.close(); } catch( Exception e ){}
+                    try{ bais.close(); } catch( Exception e ){}
+                }   // end finally
+
+            }   // end if: gzipped
+        }   // end if: bytes.length >= 2
+        
+        return bytes;
+    }   // end decode
+
+
+    
+
+    /**
+     * Attempts to decode Base64 data and deserialize a Java
+     * Object within. Returns <tt>null</tt> if there was an error.
+     *
+     * @param encodedObject The Base64 data to decode
+     * @return The decoded and deserialized object
+     * @since 1.5
+     */
+    public static Object decodeToObject( String encodedObject )
+    {
+        // Decode and gunzip if necessary
+        byte[] objBytes = decode( encodedObject );
+        
+        java.io.ByteArrayInputStream  bais = null;
+        java.io.ObjectInputStream     ois  = null;
+        Object obj = null;
+        
+        try
+        {
+            bais = new java.io.ByteArrayInputStream( objBytes );
+            ois  = new java.io.ObjectInputStream( bais );
+        
+            obj = ois.readObject();
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        catch( java.lang.ClassNotFoundException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        finally
+        {
+            try{ bais.close(); } catch( Exception e ){}
+            try{ ois.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return obj;
+    }   // end decodeObject
+    
+    
+    /* ********  I N N E R   C L A S S   I N P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * It will read data from another
+     * {@link java.io.InputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterInputStream
+     * @since 1.3
+     */
+    public static class InputStream extends java.io.FilterInputStream
+    {
+        private int     options;        // Options specified
+        private boolean encode;         // Encoding or decoding
+        private int     position;       // Current position in the buffer
+        private byte[]  buffer;         // Small buffer holding converted data
+        private int     bufferLength;   // Length of buffer (3 or 4)
+        private int     numSigBytes;    // Number of meaningful bytes in the buffer
+        private int     lineLength;
+        private boolean breakLines;     // Break lines at less than 80 characters
+        
+        
+        /**
+         * Constructs a InputStream in DECODE mode.
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @since 1.3
+         */
+        public InputStream( java.io.InputStream in )
+        {   
+            this( in, DECODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a InputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code>
+         *
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @param options Specified options
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 2.0
+         */
+        public InputStream( java.io.InputStream in, int options )
+        {   
+            super( in );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.breakLines = breakLines;
+            this.encode = encode;
+            this.bufferLength = encode ? 4 : 3;
+            this.buffer   = new byte[ bufferLength ];
+            this.position = -1;
+            this.lineLength = 0;
+        }   // end constructor
+        
+        /**
+         * Reads enough of the input stream to convert
+         * to/from Base64 and returns the next byte.
+         *
+         * @return next byte
+         * @since 1.3
+         */
+        public int read() throws java.io.IOException 
+        { 
+            // Do we need to get data?
+            if( position < 0 )
+            {
+                if( encode )
+                {
+                    byte[] b3 = new byte[3];
+                    int numBinaryBytes = 0;
+                    for( int i = 0; i < 3; i++ )
+                    {
+                        try
+                        { 
+                            int b = in.read();
+                            
+                            // If end of stream, b is -1.
+                            if( b >= 0 )
+                            {
+                                b3[i] = (byte)b;
+                                numBinaryBytes++;
+                            }   // end if: not end of stream
+                            
+                        }   // end try: read
+                        catch( java.io.IOException e )
+                        {   
+                            // Only a problem if we got no data at all.
+                            if( i == 0 )
+                                throw e;
+                            
+                        }   // end catch
+                    }   // end for: each needed input byte
+                    
+                    if( numBinaryBytes > 0 )
+                    {
+                        encode3to4( b3, 0, numBinaryBytes, buffer, 0 );
+                        position = 0;
+                        numSigBytes = 4;
+                    }   // end if: got data
+                    else
+                    {
+                        return -1;
+                    }   // end else
+                }   // end if: encoding
+                
+                // Else decoding
+                else
+                {
+                    byte[] b4 = new byte[4];
+                    int i = 0;
+                    for( i = 0; i < 4; i++ )
+                    {
+                        // Read four "meaningful" bytes:
+                        int b = 0;
+                        do{ b = in.read(); }
+                        while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC );
+                        
+                        if( b < 0 )
+                            break; // Reads a -1 if end of stream
+                        
+                        b4[i] = (byte)b;
+                    }   // end for: each needed input byte
+                    
+                    if( i == 4 )
+                    {
+                        numSigBytes = decode4to3( b4, 0, buffer, 0 );
+                        position = 0;
+                    }   // end if: got four characters
+                    else if( i == 0 ){
+                        return -1;
+                    }   // end else if: also padded correctly
+                    else
+                    {
+                        // Must have broken out from above.
+                        throw new java.io.IOException( "Improperly padded Base64 input." );
+                    }   // end 
+                    
+                }   // end else: decode
+            }   // end else: get data
+            
+            // Got data?
+            if( position >= 0 )
+            {
+                // End of relevant data?
+                if( /*!encode &&*/ position >= numSigBytes )
+                    return -1;
+                
+                if( encode && breakLines && lineLength >= MAX_LINE_LENGTH )
+                {
+                    lineLength = 0;
+                    return '\n';
+                }   // end if
+                else
+                {
+                    lineLength++;   // This isn't important when decoding
+                                    // but throwing an extra "if" seems
+                                    // just as wasteful.
+                    
+                    int b = buffer[ position++ ];
+
+                    if( position >= bufferLength )
+                        position = -1;
+
+                    return b & 0xFF; // This is how you "cast" a byte that's
+                                     // intended to be unsigned.
+                }   // end else
+            }   // end if: position >= 0
+            
+            // Else error
+            else
+            {   
+                // When JDK1.4 is more accepted, use an assertion here.
+                throw new java.io.IOException( "Error in Base64 code reading stream." );
+            }   // end else
+        }   // end read
+        
+        
+        /**
+         * Calls {@link #read} repeatedly until the end of stream
+         * is reached or <var>len</var> bytes are read.
+         * Returns number of bytes read into array or -1 if
+         * end of stream is encountered.
+         *
+         * @param dest array to hold values
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @return bytes read into array or -1 if end of stream is encountered.
+         * @since 1.3
+         */
+        public int read( byte[] dest, int off, int len ) throws java.io.IOException
+        {
+            int i;
+            int b;
+            for( i = 0; i < len; i++ )
+            {
+                b = read();
+                
+                //if( b < 0 && i == 0 )
+                //    return -1;
+                
+                if( b >= 0 )
+                    dest[off + i] = (byte)b;
+                else if( i == 0 )
+                    return -1;
+                else
+                    break; // Out of 'for' loop
+            }   // end for: each byte read
+            return i;
+        }   // end read
+        
+    }   // end inner class InputStream
+    
+    
+    
+    
+    
+    
+    /* ********  I N N E R   C L A S S   O U T P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * A OutputStream will write data to another
+     * {@link java.io.OutputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterOutputStream
+     * @since 1.3
+     */
+    public static class OutputStream extends java.io.FilterOutputStream
+    {
+        private int     options;
+        private boolean encode;
+        private int     position;
+        private byte[]  buffer;
+        private int     bufferLength;
+        private int     lineLength;
+        private boolean breakLines;
+        private byte[]  b4; // Scratch used in a few places
+        private boolean suspendEncoding;
+        
+        /**
+         * Constructs a OutputStream in ENCODE mode.
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out )
+        {   
+            this( out, ENCODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a OutputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code>
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @param options Specified options.
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out, int options )
+        {   
+            super( out );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.bufferLength = encode ? 3 : 4;
+            this.buffer       = new byte[ bufferLength ];
+            this.position     = 0;
+            this.lineLength   = 0;
+            this.suspendEncoding = false;
+            this.b4           = new byte[4];
+        }   // end constructor
+        
+        
+        /**
+         * Writes the byte to the output stream after
+         * converting to/from Base64 notation.
+         * When encoding, bytes are buffered three
+         * at a time before the output stream actually
+         * gets a write() call.
+         * When decoding, bytes are buffered four
+         * at a time.
+         *
+         * @param theByte the byte to write
+         * @since 1.3
+         */
+        public void write(int theByte) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theByte );
+                return;
+            }   // end if: supsended
+            
+            // Encode?
+            if( encode )
+            {
+                buffer[ position++ ] = (byte)theByte;
+                if( position >= bufferLength )  // Enough to encode.
+                {
+                    out.write( encode3to4( b4, buffer, bufferLength ) );
+
+                    lineLength += 4;
+                    if( breakLines && lineLength >= MAX_LINE_LENGTH )
+                    {
+                        out.write( NEW_LINE );
+                        lineLength = 0;
+                    }   // end if: end of line
+
+                    position = 0;
+                }   // end if: enough to output
+            }   // end if: encoding
+
+            // Else, Decoding
+            else
+            {
+                // Meaningful Base64 character?
+                if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC )
+                {
+                    buffer[ position++ ] = (byte)theByte;
+                    if( position >= bufferLength )  // Enough to output.
+                    {
+                        int len = Base64.decode4to3( buffer, 0, b4, 0 );
+                        out.write( b4, 0, len );
+                        //out.write( Base64.decode4to3( buffer ) );
+                        position = 0;
+                    }   // end if: enough to output
+                }   // end if: meaningful base64 character
+                else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC )
+                {
+                    throw new java.io.IOException( "Invalid character in Base64 data." );
+                }   // end else: not white space either
+            }   // end else: decoding
+        }   // end write
+        
+        
+        
+        /**
+         * Calls {@link #write} repeatedly until <var>len</var> 
+         * bytes are written.
+         *
+         * @param theBytes array from which to read bytes
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @since 1.3
+         */
+        public void write( byte[] theBytes, int off, int len ) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theBytes, off, len );
+                return;
+            }   // end if: supsended
+            
+            for( int i = 0; i < len; i++ )
+            {
+                write( theBytes[ off + i ] );
+            }   // end for: each byte written
+            
+        }   // end write
+        
+        
+        
+        /**
+         * Method added by PHIL. [Thanks, PHIL. -Rob]
+         * This pads the buffer without closing the stream.
+         */
+        public void flushBase64() throws java.io.IOException 
+        {
+            if( position > 0 )
+            {
+                if( encode )
+                {
+                    out.write( encode3to4( b4, buffer, position ) );
+                    position = 0;
+                }   // end if: encoding
+                else
+                {
+                    throw new java.io.IOException( "Base64 input not properly padded." );
+                }   // end else: decoding
+            }   // end if: buffer partially full
+
+        }   // end flush
+
+        
+        /** 
+         * Flushes and closes (I think, in the superclass) the stream. 
+         *
+         * @since 1.3
+         */
+        public void close() throws java.io.IOException
+        {
+            // 1. Ensure that pending characters are written
+            flushBase64();
+
+            // 2. Actually close the stream
+            // Base class both flushes and closes.
+            super.close();
+            
+            buffer = null;
+            out    = null;
+        }   // end close
+        
+        
+        
+        /**
+         * Suspends encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void suspendEncoding() throws java.io.IOException 
+        {
+            flushBase64();
+            this.suspendEncoding = true;
+        }   // end suspendEncoding
+        
+        
+        /**
+         * Resumes encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void resumeEncoding()
+        {
+            this.suspendEncoding = false;
+        }   // end resumeEncoding
+        
+        
+        
+    }   // end inner class OutputStream
+    
+    
+}   // end class Base64
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Hgram.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Hgram.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Hgram.java	(revision 21112)
@@ -0,0 +1,378 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.imageio.*;
+import javax.imageio.stream.*;
+import java.io.*;
+import java.util.*;
+import java.awt.image.*;
+
+public class Hgram extends JFrame
+{
+    public static int TOP = 0;
+    public static int LARGEST = 1;
+
+    // data is an array with each entry being the frequency of a bin
+    public static int BIN_DATA = 0;
+
+    // data comes in raw format and needs to be allocated to bins
+    public static int RAW_DATA = 1;
+    
+    double[] data;
+    
+    // default size of frame
+    int width = 400;
+    int height = 400;
+
+    // axis limits of histogram
+    double xmin;
+    double xmax;
+
+    // number of bins, bin frequencies and width
+    int noBins;
+    double bins[];
+    double binWidth;
+    
+    BufferedImage offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    
+    String label = null;
+    
+    double median = -1;
+    int med_bin = -1;
+    int mode = -1;
+    
+    
+    public Hgram()
+    {
+	init();
+    }
+
+    
+    public void setData(double[] d, int m){
+	data = d; 
+	mode = m;
+	xmin = getMinimum(data);
+	xmax = getMaximum(data);
+	createBins(data,noBins,xmin, xmax);
+    }
+
+      
+    public void setBins(double[] b){ bins = b; }
+
+
+    public void setBinNumber(int i){ noBins = i; }
+
+    
+    public void setMaximum(double i){ xmax = i; }
+    
+    
+    public void setMinimum(double i){ xmin = i; }
+
+
+    public void setBinWidth(double i){ binWidth = i; }
+    
+    
+    public void setLabel(String l){ label = l; }
+  
+
+    public void init()
+    {
+	this.setDefaultLookAndFeelDecorated(true);
+        this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+	this.setSize(width,height);
+	this.setLocation(0,0);
+	this.setVisible(true);
+	
+	this.addComponentListener( new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt){
+		    update();
+		}
+	    });
+	
+	
+	this.addMouseListener(new MouseAdapter(){
+		public void mousePressed(MouseEvent evt)
+		{
+    		    int intX = evt.getX();
+	    	    int intY = evt.getY();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    
+		    if( event.getClickCount() == 1 )
+			{
+			}
+		}
+	    });
+	
+	
+	this.addMouseMotionListener(new MouseMotionAdapter()
+	    {
+		public void mouseMoved(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    getBin(x,y);
+		}
+	    });
+    }
+
+    
+    public int getBin(int x, int y){ return -1; }
+    
+    
+    private int getMaxBin()
+    {
+	int maxIndex = -1;
+	double max = 0;
+	
+	for( int i = 0; i < bins.length; i++ ){
+	    if( bins[i] > max ){
+		max = bins[i];
+		maxIndex = i;
+	    }
+	}
+	
+	return maxIndex;
+    }
+    
+    
+    public void plot()
+    {
+	update();
+    }
+    
+      
+    public double getMaximum(double[] d)
+    {
+	double max = 0; 
+
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max ) max = d[i];
+	}
+	return max;
+    }
+ 
+
+
+    public double getMinimum(double[] d)
+    {
+	double min = d[0]*d[0]*1000;
+
+	for( int i = 0; i < d.length; i++ )
+	    {
+		if( d[i] < min ) min = d[i];
+	    }
+	return min;
+    }
+
+
+    public double getFirstMaximum(double[] d)
+    {
+	double max = 0;
+	for( int i = 0; i < bins.length; i++ )
+	    {
+		if( bins[i] > max )
+		    {
+			max = bins[i];
+		    }
+	    }
+	return max;
+    }
+
+
+    public double getSecondMaximum(double[] d)
+    {
+	double max_1 = 0;
+	double max_2 = 0;
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max_1 ){
+		max_2 = max_1;
+		max_1 = d[i];
+	    }
+	    else if( d[i] > max_2 ){
+		max_2 = d[i];
+	    }
+	}
+	return max_2;
+    }
+
+ 
+    public void createBins(double[] data, int noBins, double xmin, double xmax)
+    {
+	bins = new double[noBins];
+	binWidth = (double)(xmax-xmin)/noBins;
+	
+	for( int i = 0; i < data.length; i++ ){
+	    int bin = (int)Math.round((double)(data[i]-xmin)/binWidth);
+	    if( bin >= noBins ) bin = noBins-1;
+	    bins[bin]++;
+	}
+	
+	for( int i = 0; i < noBins; i++ ){
+	    bins[i] = (double)bins[i]/data.length;
+	}
+    }
+    
+    
+    public void update()
+    {
+	if( bins == null ) return;
+	
+	Dimension d = this.getSize();
+	width = d.width;
+	height = d.height;
+        offScreen = new BufferedImage( width, height, BufferedImage.TYPE_INT_ARGB ); 
+	offGraphics = (Graphics2D)offScreen.getGraphics();
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,width,height);
+    	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+        offGraphics.setStroke(myStroke);	
+	
+	int xaxis = (int)(0.8 * width); 
+	int xBuffer = (int)(0.1 * width);
+	
+	int yaxis = (int)(0.8 * d.height);
+	int yBuffer = (int)(0.1 * d.height);
+	
+	double maxBinLength = (double)xaxis/bins.length;
+	int effBinLength = (int)(0.8 * maxBinLength);
+	int binBuffer = (int)(0.1 * effBinLength);
+	
+	double max_1 = getFirstMaximum(bins);
+	double max_2 = getSecondMaximum(bins);
+	double max = max_1;
+	boolean cut = false;
+	
+	if( max_1 > 1.5 * max_2 ){
+	    cut = true;
+	    max = max_2;
+	}
+	
+	double pixelPerUnit = (double)yaxis/max; 
+	
+
+	/**** draw the bins ****/
+
+	for( int i = 0; i < bins.length; i++ ){
+	    int level = (int)(bins[i]* pixelPerUnit);
+	    double bound = (double)((int)(100 * (i * (double)(xmax-xmin)/bins.length + xmin)))/100;
+	    offGraphics.setColor(Color.blue);
+	    offGraphics.fillRect(xBuffer + (int)(i*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	}
+
+	if( label != null ){
+	    FontMetrics fm   = offGraphics.getFontMetrics(offGraphics.getFont());
+	    int x = width/2 - fm.stringWidth(label)/2;		
+	    offGraphics.drawString(label,x,yBuffer);
+	}
+
+
+	/**** draw axes and ticks ****/
+
+	offGraphics.setColor(Color.blue);
+	
+	int x = xBuffer + binBuffer;
+	int y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	
+	String tickStr = String.valueOf(xmin);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+	x = xBuffer + binBuffer+(int)(bins.length*maxBinLength)+binBuffer;
+	y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	tickStr = String.valueOf(xmax);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+        repaint();
+    }
+    
+    
+    public void paint(Graphics g)
+    {
+        Graphics2D g2 = (Graphics2D)g;
+	
+	if( offScreen!=null )
+	    g2.drawImage(offScreen, 0, 0, null);
+    }
+    
+    
+    public void printBins()
+    {
+	for( int i = 0; i < bins.length; i++ ){
+	    double low = i*binWidth;
+	    double up = (i+1)*binWidth;
+	    System.out.println("Bin.." + i + " " + low + " " + up + " " + bins[i]); 
+	}
+    }
+
+
+    public void save(String name, String type)
+    {
+	
+	try{
+	    Iterator imageWriters =
+		ImageIO.getImageWritersByFormatName(type);
+	    ImageWriter imageWriter = (ImageWriter)imageWriters.next();
+	    File file = new File( name + "." + type );
+	    ImageOutputStream ios =
+		ImageIO.createImageOutputStream( file );
+	    imageWriter.setOutput( ios );
+	    imageWriter.write( offScreen );
+	} catch( IOException ie ) {
+	    ie.printStackTrace();
+	}
+    }
+
+}
+
+ // accepts raw data
+    /*public Hgram(double[] data, int noBins, double min, double max)
+    {
+	super("JHistogram");
+	init();
+	createBins(data,noBins,min,max);
+	update();
+	}*/
+    
+    
+    // accepts both raw and binned data
+    /*public Hgram(double[] data, int noBins, int mode)
+    {
+	super("JHistogram");
+	this.mode = mode;
+	init();
+	createBins(data,noBins,getMinimum(data),getMaximum(data));
+	update();
+	}*/
+    
+
+
+/*
+	int level = (int)(bins[med_bin]* pixelPerUnit);
+	offGraphics.fillRect(xBuffer + (int)(med_bin*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	String med_string = String.valueOf(median);
+	med_string = med_string.substring(0,med_string.indexOf('.')+3);
+	offGraphics.drawString(med_string, xBuffer + (int)(med_bin*maxBinLength)+binBuffer-10,height-yBuffer-level-10);
+	    
+	//saveHistogram(offScreen);
+	
+	repaint();
+	    }
+	    }*/
+
+  /*
+    public Hgram(double[] data, int noBins)
+    {
+	init();
+	createBins(data,noBins,getMinimum(data),getFirstMaximum(data));
+	update();
+	}*/
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Misc.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Misc.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Misc.java	(revision 21112)
@@ -0,0 +1,240 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+
+public class Misc {
+
+    // quickSort
+    private static void qs(int l, int r, double v[][])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[1][(l+r)/2];
+		do
+		{
+
+			while( v[1][i] < pivot )
+			{
+				i++;
+			} 
+			while( v[1][j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[1][i];	
+					double t0 = v[0][i];
+					v[1][i] = v[1][j];
+					v[1][j] = t1;
+					v[0][i] = v[0][j];
+					v[0][j] = t0;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+    
+    
+    // quickSort
+    private static void qs(int l, int r, double v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		int pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					int t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int[] v, Object[] objArr)
+    {
+        if( l < r ){
+            int i = l;
+            int j = r;
+
+            int pivot = v[(l+r)/2];
+
+            do{
+                while( v[i] < pivot ){
+                    i++;
+                }
+                while( v[j] > pivot ){
+                    j--;
+                }
+
+                if( i <= j ){
+                    if( i < j ){
+                        int t1 = v[i];
+                        v[i] = v[j];
+                        v[j] = t1;
+
+                        Object tobj = objArr[i];
+                        objArr[i] = objArr[j];
+                        objArr[j] = tobj;
+ }
+                    i++;
+                    j--;
+                }
+
+            }
+            while( i <= j );
+            qs(l,j,v,objArr);
+            qs(i,r,v,objArr);
+        }
+    }
+
+
+    // quickSort
+    private static void qs(int l, int r, float[] v, Object[] objArr)
+    {
+	if( l < r ){
+	    int i = l;
+	    int j = r;
+	    
+	    float pivot = v[(l+r)/2];
+	    
+	    do{
+		while( v[i] < pivot ){
+		    i++;
+		} 
+		while( v[j] > pivot ){
+		    j--;
+		}
+		
+		if( i <= j ){
+		    if( i < j ){
+			float t1 = v[i];	
+			v[i] = v[j];
+			v[j] = t1;
+			
+			Object tobj = objArr[i];
+			objArr[i] = objArr[j];
+			objArr[j] = tobj;
+		    }
+		    i++;
+		    j--;
+		}
+		
+	    }
+	    while( i <= j );
+	    qs(l,j,v,objArr);
+	    qs(i,r,v,objArr);
+	}
+    }
+    
+
+    public static void quickSort(int[] v)
+    {
+	qs(0,v.length-1,v);
+    }
+
+    public static void quickSort(float[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(int[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(double v[])
+    {
+	qs(0,v.length-1,v);
+    }
+
+
+    public static void quickSort(double v[][])
+    {
+	qs(0,v[0].length-1,v);
+    }
+}
+    
+    
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Sammon.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Sammon.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/Sammon.java	(revision 21112)
@@ -0,0 +1,247 @@
+package vishnu.util;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.Point2D;
+
+public class Sammon
+{
+
+    public static final double MF =       0.3;		/* Magic Factor          */
+    public static final double _MAX_Err = 0.00005;	/* Max Mapping Error     */
+    public static final int _MAX_M =  1000;		/* Max Iteration Number  */
+    public static final int _MAX_N =  1000;		/* Max Number of Entries */
+    public static final int _MAX_L =  128;		/* Max Dimension for X   */
+
+
+    double[][] Xs;
+    double[][] Ys;
+    int number;
+    int dimension;
+    double total_Xs_dists;
+
+/* SS298: Added function for scaling values between 0 and 1
+* n : number of entries
+*/
+    void scale_values(int n)
+    {
+
+	// get the maximum and the minimum of Ys array
+
+    // get the maximum and the minimum of Ys array
+
+        double maxX = Ys[0][0];
+        double minX = Ys[0][0];
+        double maxY = Ys[0][1];
+        double minY = Ys[0][1];
+
+        int i;
+        for (i = 1 ; i < n ; i++)
+        {
+
+            if (maxX < Ys[i][0]) maxX = Ys[i][0];
+            if (minX > Ys[i][0]) minX = Ys[i][0];
+            if (maxY < Ys[i][1]) maxY = Ys[i][1];
+            if (minY > Ys[i][1]) minY = Ys[i][1];
+        }
+        maxY -= minY;
+        maxX -= minX;
+        for (i = 0 ; i < n ; i++)
+        {
+            if (maxX > 0)
+            {
+                Ys[i][0] -= minX;
+                Ys[i][0] /= maxX;
+                Ys[i][0] *=0.8;
+                Ys[i][0] +=0.1;
+            }
+            else Ys[i][0]=0.5;
+            if (maxY > 0)
+            {
+                Ys[i][1] -= minY;
+                Ys[i][1] /= maxY;
+                Ys[i][1] *= 0.8;
+                Ys[i][1] += 0.1;
+            }
+            else Ys[i][1]=0.5;
+        }
+
+    }
+
+
+
+/* return a Euclidean distance between two 'd'-dimensional vectors 'xs' and 'ys'
+*/
+    double eucl_dist(int d, double[] xs, double[] ys)
+    {
+        double ans=0.0;
+        double tmp;
+        int i;
+        for(i=0;i<d;i++)
+        {
+            tmp = xs[i]-ys[i];
+            ans += tmp * tmp;
+        }
+        return Math.sqrt(ans);
+    }
+
+
+/* n - number of entries
+* d - dimension
+* post:
+*   return an array of eucidean distances for X
+*   and assign its total distance (will be used in the mapping error) to 'total'
+*/
+    double[] compute_Xs_eucl_dists()
+    {
+        double tmp[];
+        int i,j,k;
+        tmp = new double[number*(number-1)/2];
+
+        total_Xs_dists=0.0;
+        k=0;
+        for(i=1;i<number;i++)
+            for(j=0;j<i;j++)
+            {
+                tmp[k] = eucl_dist(dimension, Xs[i], Xs[j]);
+                total_Xs_dists+=tmp[k];
+                k++;
+            }
+            return tmp;
+    }
+    
+    
+    public Point2D [] getMapping()
+    {
+        Point2D [] sam = new Point2D[Ys.length];
+        for (int c=0;c<Ys.length;c++)
+	    {
+            float fx = (float) Ys[c][0];
+            float fy = (float) Ys[c][1];
+            sam[c]=new Point2D(fx,fy);
+        }
+        return sam;
+    }
+
+
+
+/*----------------------------------------------------------------*/
+    
+    public Sammon(double [][] centroids)
+    {
+	
+        int i, j, m, p;
+        double[] Xs_dists;
+
+        double dist_star,dist, alpha, beta, gamma, zeta, err;
+        double uppDer0, uppDer1, lowDer0, lowDer1;
+        double maxX=0.0;
+        double minX=0.0;
+        double maxY=0.0;
+        double minY=0.0;
+
+        number = centroids.length;
+        dimension = centroids[0].length;
+
+        Xs = centroids;
+        Ys = new double[number][2];
+
+        Xs_dists = compute_Xs_eucl_dists();
+
+        p=0;
+        for (i=0;i<number;i++)
+        {
+            Ys[i][0] =  Xs[i][0];
+            double value = Xs_dists.length>0?Xs_dists[(i==(number-1))?i:p]:
+                          0.0;
+            Ys[i][1] =  Xs[i][0]+value;
+            p+=(number-(i+1));
+        }
+
+        m=0;  /* iteration counter */
+        do{
+
+            for(p=0;p<number;p++)
+            {
+                uppDer0=uppDer1=lowDer0=lowDer1=0.0;
+
+
+                /* calcuate partial derivatives */
+
+                for(j=0;j<number;j++)
+                {
+                    if (p==j) continue;
+                    dist_star = Xs_dists[(j>p) ? (j*(j-1)/2+p) : (p*(p-1)/2+j)];
+
+                    dist = eucl_dist(2,Ys[p],Ys[j]);
+                    if (dist == 0) dist = 0.00001;
+                    alpha = dist_star-dist;
+                    beta = dist_star*dist;
+                    if (beta == 0) beta = 0.00000000000001;
+                    gamma = alpha/beta;
+
+                    /* 1st dimension */
+                    zeta = Ys[p][0]-Ys[j][0];
+                    uppDer0 += gamma * zeta;
+                    lowDer0 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+
+                    /* 2nd dimension */
+                    zeta = Ys[p][1]-Ys[j][1];
+                    uppDer1 += gamma * zeta;
+                    lowDer1 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+                }  /* for */
+
+                Ys[p][0] = Ys[p][0]+MF*uppDer0/Math.abs(lowDer0);
+                Ys[p][1] = Ys[p][1]+MF*uppDer1/Math.abs(lowDer1);
+                if(p==0)
+                {
+                    maxX = Ys[0][0];
+                    minX = Ys[0][0];
+                    maxY = Ys[0][1];
+                    minY = Ys[0][1];
+                }
+                else
+                {
+                  if (maxX < Ys[p][0]) maxX = Ys[p][0];
+                  if (minX > Ys[p][0]) minX = Ys[p][0];
+                  if (maxY < Ys[p][1]) maxY = Ys[p][1];
+                  if (minY > Ys[p][1]) minY = Ys[p][1];
+                }
+
+
+            }  /* for */
+
+
+        /* calcuate the mapping error */
+
+            err=0.0;
+            p=0;
+            for(j=1;j<number;j++)
+                for(i=0;i<j;i++)
+                {
+                    dist_star = Xs_dists[p++];
+                    alpha = dist_star - eucl_dist(2,Ys[j],Ys[i]);
+                    err += (alpha*alpha/dist_star);
+                }
+            err=err / total_Xs_dists;
+
+            m++;
+        } while ((err>_MAX_Err) && (m!=_MAX_M));
+
+
+        if(maxY==minY)
+        {
+          System.out.println("Warning: simulated 'y' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][1]=Math.random();
+        }
+        if(maxX==minX)
+        {
+          System.out.println("Warning: simulated 'x' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][0]=Math.random();
+        }
+
+        scale_values(number);
+    }
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/SpringUtilities.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/SpringUtilities.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/SpringUtilities.java	(revision 21112)
@@ -0,0 +1,194 @@
+package vishnu.util;
+
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.awt.*;
+
+/**
+ * A 1.4 file that provides utility methods for
+ * creating form- or grid-style layouts with SpringLayout.
+ * These utilities are used by several programs, such as
+ * SpringBox and SpringCompactGrid.
+ */
+public class SpringUtilities {
+    /**
+     * A debugging utility that prints to stdout the component's
+     * minimum, preferred, and maximum sizes.
+     */
+    public static void printSizes(Component c) {
+        System.out.println("minimumSize = " + c.getMinimumSize());
+        System.out.println("preferredSize = " + c.getPreferredSize());
+        System.out.println("maximumSize = " + c.getMaximumSize());
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component is as big as the maximum
+     * preferred width and height of the components.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeGrid(Container parent,
+                                int rows, int cols,
+                                int initialX, int initialY,
+                                int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeGrid must use SpringLayout.");
+            return;
+        }
+
+        Spring xPadSpring = Spring.constant(xPad);
+        Spring yPadSpring = Spring.constant(yPad);
+        Spring initialXSpring = Spring.constant(initialX);
+        Spring initialYSpring = Spring.constant(initialY);
+        int max = rows * cols;
+
+        //Calculate Springs that are the max of the width/height so that all
+        //cells have the same size.
+        Spring maxWidthSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        Spring maxHeightSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        for (int i = 1; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            maxWidthSpring = Spring.max(maxWidthSpring, cons.getWidth());
+            maxHeightSpring = Spring.max(maxHeightSpring, cons.getHeight());
+        }
+
+        //Apply the new width/height Spring. This forces all the
+        //components to have the same size.
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            cons.setWidth(maxWidthSpring);
+            cons.setHeight(maxHeightSpring);
+        }
+
+        //Then adjust the x/y constraints of all the cells so that they
+        //are aligned in a grid.
+        SpringLayout.Constraints lastCons = null;
+        SpringLayout.Constraints lastRowCons = null;
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                                 parent.getComponent(i));
+            if (i % cols == 0) { //start of new row
+                lastRowCons = lastCons;
+                cons.setX(initialXSpring);
+            } else { //x position depends on previous component
+                cons.setX(Spring.sum(lastCons.getConstraint(SpringLayout.EAST), xPadSpring));
+            }
+
+            if (i / cols == 0) { //first row
+                cons.setY(initialYSpring);
+            } else { //y position depends on previous row
+                cons.setY(Spring.sum(lastRowCons.getConstraint(SpringLayout.SOUTH),
+                                     yPadSpring));
+            }
+            lastCons = cons;
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH,
+                            Spring.sum(
+                                Spring.constant(yPad),
+                                lastCons.getConstraint(SpringLayout.SOUTH)));
+        pCons.setConstraint(SpringLayout.EAST,
+                            Spring.sum(
+                                Spring.constant(xPad),
+                                lastCons.getConstraint(SpringLayout.EAST)));
+    }
+
+    /* Used by makeCompactGrid. */
+    private static SpringLayout.Constraints getConstraintsForCell(
+                                                int row, int col,
+                                                Container parent,
+                                                int cols) {
+        SpringLayout layout = (SpringLayout) parent.getLayout();
+        Component c = parent.getComponent(row * cols + col);
+        return layout.getConstraints(c);
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component in a column is as wide as the maximum
+     * preferred width of the components in that column;
+     * height is similarly determined for each row.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeCompactGrid(Container parent,
+                                       int rows, int cols,
+                                       int initialX, int initialY,
+                                       int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeCompactGrid must use SpringLayout.");
+            return;
+        }
+
+        //Align all cells in each column and make them the same width.
+        Spring x = Spring.constant(initialX);
+        for (int c = 0; c < cols; c++) {
+            Spring width = Spring.constant(0);
+            for (int r = 0; r < rows; r++) {
+                width = Spring.max(width,
+                                   getConstraintsForCell(r, c, parent, cols).
+                                       getWidth());
+            }
+            for (int r = 0; r < rows; r++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setX(x);
+                constraints.setWidth(width);
+            }
+            x = Spring.sum(x, Spring.sum(width, Spring.constant(xPad)));
+        }
+
+        //Align all cells in each row and make them the same height.
+        Spring y = Spring.constant(initialY);
+        for (int r = 0; r < rows; r++) {
+            Spring height = Spring.constant(0);
+            for (int c = 0; c < cols; c++) {
+                height = Spring.max(height,
+                                    getConstraintsForCell(r, c, parent, cols).
+                                        getHeight());
+            }
+            for (int c = 0; c < cols; c++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setY(y);
+                constraints.setHeight(height);
+            }
+            y = Spring.sum(y, Spring.sum(height, Spring.constant(yPad)));
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH, y);
+        pCons.setConstraint(SpringLayout.EAST, x);
+    }
+}
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/XMLUtil.java
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/XMLUtil.java	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/src/vishnu/util/XMLUtil.java	(revision 21112)
@@ -0,0 +1,71 @@
+/*********************************************************************
+ Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class XMLUtil {
+    
+    /** returns the (first) child element with the given name */
+    public static Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+    /** takes a list of elements, and returns an array of strings 
+     * of the values of attribute att_name */
+    public static String [] getAttributeValuesFromList(Element list,
+						       String att_name) {
+	
+	NodeList children = list.getChildNodes();
+	
+	int num_nodes = children.getLength();
+	String []ids = new String[num_nodes];
+	for (int i=0; i<num_nodes; i++) {
+	    Element e = (Element)children.item(i);
+	    String id = e.getAttribute(att_name);	    
+	    ids[i] = id;	    
+	}
+	
+	return ids;
+    }
+
+    /** extracts the text out of a node */
+    public static String getNodeText(Element param) {
+	param.normalize();
+	Node n = param.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	if (n == null) {
+	    return "";
+	}
+	return n.getNodeValue();
+    }
+
+}
Index: /main/tags/3.01/greenstone3-extensions/vishnu/web/WEB-INF/web.xml
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/web/WEB-INF/web.xml	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/web/WEB-INF/web.xml	(revision 21112)
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<!-- this is based on a sample web.xml file provided by 
+http://jakarta.apache.org/tomcat/tomcat-4.0-doc/appdev/web.xml.txt
+-->
+<web-app>
+
+  
+  <!-- General description of your web application -->
+  
+  <display-name>VISHNU</display-name>
+  <description>
+    Collection server for Imperial College's new Vishnu Visualisation tool.
+  </description>
+  
+  <context-param>
+    <param-name>webmaster</param-name>
+    <param-value>enter your email address here</param-value>
+    <description>
+      The EMAIL address of the administrator to whom questions
+      and comments about this application should be addressed.
+    </description>
+  </context-param>
+  
+
+  <!-- Servlet definitions for the servlets that make up
+  your web application, including initialization
+  parameters.  
+  -->
+  <servlet>
+    <servlet-name>visualiser</servlet-name>
+    <description>imperial college vishnu servlet</description>
+    <servlet-class>vishnu.server.VisServlet</servlet-class>
+    <init-param>
+      <param-name>collectionsHome</param-name>
+      <!-- edit this to provide the full path to the collections directory-->
+      <!-- make sure the path separators are correct for your platform -->
+      <param-value>path-to-vishnu/collections</param-value>
+    </init-param>
+  </servlet>
+
+  <!-- Define mappings that are used by the servlet container to
+  translate a particular request URI (context-relative) to a
+  particular servlet.  The examples below correspond to the
+  servlet descriptions above.  -->
+  <servlet-mapping>
+    <servlet-name>visualiser</servlet-name>
+    <url-pattern>/server</url-pattern>
+  </servlet-mapping>
+  <!-- Define the default session timeout for your application,
+  in minutes.  From a servlet or JSP page, you can modify
+  the timeout for a particular session dynamically by using
+  HttpSession.getMaxInactiveInterval(). -->
+  
+  <session-config>
+    <session-timeout>30</session-timeout>    <!-- 30 minutes -->
+  </session-config>
+  
+  
+</web-app>
+
+
Index: /main/tags/3.01/greenstone3-extensions/vishnu/web/index.html
===================================================================
--- /main/tags/3.01/greenstone3-extensions/vishnu/web/index.html	(revision 21112)
+++ /main/tags/3.01/greenstone3-extensions/vishnu/web/index.html	(revision 21112)
@@ -0,0 +1,64 @@
+<html>
+  <head>
+    <title>Imperial College Visualizer</title>
+  </head>
+  <body>
+<h1>VISHNU:</h1>
+
+<h2>Overview</h2>
+This is a document search engine with several new visualization front
+ends that aid navigation through the set of documents returned by a
+query. Keywords from the document set are identified and selected on
+the fly. These keywords are then used to compute document clusters. The
+clusters can be displayed and explored using three visualizations:
+Sammon cluster view, Dendro map or radial interactive visualization.<br>      
+
+<h2>Quick start</h2>
+<p>Choose a collection from the colection menu. Type a query into the search box and click the Find button. Each pane will then display the query
+results in a different manner.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight:bold;">PlainVis</span> ---&nbsp; This shows a ranked list of documents
+retrieved. Clicking on the link for a document (in the left most
+column) will bring up a new window containing the document.<br>
+</p>
+
+<p style="margin-left: 40px;"><span style="font-weight: bold;">RadialVis</span> --- This shows the
+key words from the retrieved document set around a circle with the
+documents represented by dots within the circle. The dots are arranged
+closest to the words that contain them. Clicking on a word will
+highlight the documents containing that word. Correspondingly, clicking
+on a dot will show the words that it contains. Words can be dragged
+which will move the dots, thus changing the influence of that particular
+word on the arrangement.<br>
+At the bottom part of the pane key words can be selected and de-selected.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">SammonVis</span> --- This displays
+documents clustered by key words. As you mouse over a cluster a menu is
+displayed. This shows the number of documents in the cluster, and, at
+the bottom of the menu, the top ranked key words used to form the
+cluster. Clicking on the number of documents brings up the key words
+and their percentages in the left hand frame and a summary of the
+documents in the right hand frame. It is possible to drill down two
+levels through this menu. To return to a higher level use the button at
+the top of the frame.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">DendroVis</span> ---&nbsp; This
+displays the document clusters arranged in a hierarchical format. As
+you mouse over nodes in the display a menu similar to the SammonVis
+menu is displayed. You can drill down through the hierarchy to examine
+deeper nodes. The key words are displayed at each node. The menu
+indicates if these are from the outer twigs (bottom) of the hierarchy.
+Clicking on the number of documents in the node displays the keywords
+and document summaries.<br>
+</p>
+
+      
+      <p>
+	<center>
+	<applet codebase='lib' code='vishnu.testvis.visual.Vishnu.class' archive='vishnu.jar' width='900' height='700'>
+	  <param name='library' value='http://localhost:8080/vishnu/server'/>
+	</applet>
+      </p>
+    </center>
+  </body>
+</html>
Index: /main/tags/3.01/gs2build-extra/Makefile.in
===================================================================
--- /main/tags/3.01/gs2build-extra/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/Makefile.in	(revision 21112)
@@ -0,0 +1,106 @@
+###########################################################################
+#
+# Makefile -- gsdl
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+srcdir = @srcdir@
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+@SET_MAKE@
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@
+CXXFLAGS = @CXXFLAGS@
+LIBS   = @LIBS@
+MDEFINES =
+
+GSDLOS = @gsdlos@
+
+COMPILE = $(CC) -c
+
+SOURCES = 
+
+OBJECTS = 
+
+ALLFILES = $(OBJECTS)
+
+#INSTALLDIRS = src/recpt src/txt2db src/db2txt src/hashfile src/getpw packages/mg src/phind/generate packages src/mgpp src/oaiservr
+INSTALLDIRS = src/txt2db src/db2txt src/hashfile packages/mg src/phind/generate packages src/mgpp 
+
+MODULEDIRS = @MODULEDIRS@
+
+#OTHERDIRS = packages packages/mg lib src/mgpp src/colservr src/recpt src/txt2db src/db2txt src/hashfile src/getpw src/phind/generate src/oaiservr
+
+OTHERDIRS = packages packages/mg lib src/mgpp src/txt2db src/db2txt src/hashfile src/phind/generate
+all:
+	for odir in $(OTHERDIRS) $(MODULEDIRS); do \
+	  echo making $@ in $$odir; \
+	  (cd $$odir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+install: all
+
+	for idir in $(INSTALLDIRS); do \
+	  echo installing $$idir; \
+	  (cd $$idir && $(MAKE) install) || exit 1; \
+	done
+
+# added test to check existence of Makefile, as src/recpt is in
+# both INSTALLDIRS and OTHERDIRS - can't make distclean twice!
+distclean: clean
+	rm -f Makefile config.cache config.h config.log stamp-h config.status
+	for subdir in $(INSTALLDIRS) $(OTHERDIRS); do \
+	  echo cleaning $@ in $$subdir; \
+	  (cd $$subdir && if test -e Makefile; then $(MAKE) distclean;fi) || exit 1; \
+	done
+
+clean:
+	for subdir in $(INSTALLDIRS) $(OTHERDIRS); do \
+	  echo cleaning $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) clean) || exit 1; \
+	done
+
+# commented out - jrm21 Dec 2004 - can get confused by cvs timestamps
+# (and might run the wrong version of autoconf)
+#
+## For an explanation of the following Makefile rules, see node
+## `Automatic Remaking' in GNU Autoconf documentation.
+#Makefile: Makefile.in config.status
+#	CONFIG_FILES=$@ CONFIG_HEADERS= ./config.status
+#config.status: configure
+#	./config.status --recheck
+#configure: configure.in aclocal.m4
+#	cd $(srcdir) && autoconf
+#
+#config.h: stamp-h
+#stamp-h: config.h.in config.status
+#	CONFIG_FILES= CONFIG_HEADERS=config.h ./config.status
+#config.h.in: stamp-h.in
+#stamp-h.in: configure.in aclocal.m4 acconfig.h
+#	cd $(srcdir) && autoheader
+#	$(TOUCH) $(srcdir)/stamp-h.in
+
+
+
Index: /main/tags/3.01/gs2build-extra/configure
===================================================================
--- /main/tags/3.01/gs2build-extra/configure	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/configure	(revision 21112)
@@ -0,0 +1,8545 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="lib/text_t.h"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subdirs_all="$ac_subdirs_all packages"
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CFLAGS CC LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT CXX CXXFLAGS ac_ct_CXX AWK YACC build build_cpu build_vendor build_os host host_cpu host_vendor host_os target target_cpu target_vendor target_os INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS STATIC GDBM_LIBPATH GDBM_INCLUDE gsdlos MODULEDIRS subdirs LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+System types:
+  --build=BUILD     configure for building on BUILD [guessed]
+  --host=HOST       cross-compile to build programs to run on HOST [BUILD]
+  --target=TARGET   configure for building compilers for TARGET [HOST]
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-gdbm=dir         Use local gdbm library and headers
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+
+Some influential environment variables:
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+
+PACKAGE=gsdl
+VERSION=2.1
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+
+# Check whether --with-gdbm or --without-gdbm was given.
+if test "${with_gdbm+set}" = set; then
+  withval="$with_gdbm"
+  gdbmdir=$withval
+fi;
+
+LDFLAGS=
+if test -z "$CFLAGS"; then
+CFLAGS=-O2
+fi
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C compiler default output file name" >&5
+echo $ECHO_N "checking for C compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C compiler works" >&5
+echo $ECHO_N "checking whether the C compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Make sure we can run config.sub.
+$ac_config_sub sun4 >/dev/null 2>&1 ||
+  { { echo "$as_me:$LINENO: error: cannot run $ac_config_sub" >&5
+echo "$as_me: error: cannot run $ac_config_sub" >&2;}
+   { (exit 1); exit 1; }; }
+
+echo "$as_me:$LINENO: checking build system type" >&5
+echo $ECHO_N "checking build system type... $ECHO_C" >&6
+if test "${ac_cv_build+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_build_alias=$build_alias
+test -z "$ac_cv_build_alias" &&
+  ac_cv_build_alias=`$ac_config_guess`
+test -z "$ac_cv_build_alias" &&
+  { { echo "$as_me:$LINENO: error: cannot guess build type; you must specify one" >&5
+echo "$as_me: error: cannot guess build type; you must specify one" >&2;}
+   { (exit 1); exit 1; }; }
+ac_cv_build=`$ac_config_sub $ac_cv_build_alias` ||
+  { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_build_alias failed" >&5
+echo "$as_me: error: $ac_config_sub $ac_cv_build_alias failed" >&2;}
+   { (exit 1); exit 1; }; }
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_build" >&5
+echo "${ECHO_T}$ac_cv_build" >&6
+build=$ac_cv_build
+build_cpu=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'`
+build_vendor=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'`
+build_os=`echo $ac_cv_build | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'`
+
+
+echo "$as_me:$LINENO: checking host system type" >&5
+echo $ECHO_N "checking host system type... $ECHO_C" >&6
+if test "${ac_cv_host+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_host_alias=$host_alias
+test -z "$ac_cv_host_alias" &&
+  ac_cv_host_alias=$ac_cv_build_alias
+ac_cv_host=`$ac_config_sub $ac_cv_host_alias` ||
+  { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_host_alias failed" >&5
+echo "$as_me: error: $ac_config_sub $ac_cv_host_alias failed" >&2;}
+   { (exit 1); exit 1; }; }
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_host" >&5
+echo "${ECHO_T}$ac_cv_host" >&6
+host=$ac_cv_host
+host_cpu=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'`
+host_vendor=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'`
+host_os=`echo $ac_cv_host | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'`
+
+
+echo "$as_me:$LINENO: checking target system type" >&5
+echo $ECHO_N "checking target system type... $ECHO_C" >&6
+if test "${ac_cv_target+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_target_alias=$target_alias
+test "x$ac_cv_target_alias" = "x" &&
+  ac_cv_target_alias=$ac_cv_host_alias
+ac_cv_target=`$ac_config_sub $ac_cv_target_alias` ||
+  { { echo "$as_me:$LINENO: error: $ac_config_sub $ac_cv_target_alias failed" >&5
+echo "$as_me: error: $ac_config_sub $ac_cv_target_alias failed" >&2;}
+   { (exit 1); exit 1; }; }
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_target" >&5
+echo "${ECHO_T}$ac_cv_target" >&6
+target=$ac_cv_target
+target_cpu=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\1/'`
+target_vendor=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\2/'`
+target_os=`echo $ac_cv_target | sed 's/^\([^-]*\)-\([^-]*\)-\(.*\)$/\3/'`
+
+
+# The aliases save the names the user supplied, while $host etc.
+# will get canonicalized.
+test -n "$target_alias" &&
+  test "$program_prefix$program_suffix$program_transform_name" = \
+    NONENONEs,x,x, &&
+  program_prefix=${target_alias}-
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lg" >&5
+echo $ECHO_N "checking for main in -lg... $ECHO_C" >&6
+if test "${ac_cv_lib_g_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lg  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_g_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_g_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_g_main" >&5
+echo "${ECHO_T}$ac_cv_lib_g_main" >&6
+if test $ac_cv_lib_g_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBG 1
+_ACEOF
+
+  LIBS="-lg $LIBS"
+
+fi
+ac_cv_lib_g=ac_cv_lib_g_main
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+ac_cv_lib_m=ac_cv_lib_m_main
+
+
+echo "$as_me:$LINENO: checking for main in -lcrypt" >&5
+echo $ECHO_N "checking for main in -lcrypt... $ECHO_C" >&6
+if test "${ac_cv_lib_crypt_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcrypt  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_crypt_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_crypt_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_crypt_main" >&5
+echo "${ECHO_T}$ac_cv_lib_crypt_main" >&6
+if test $ac_cv_lib_crypt_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBCRYPT 1
+_ACEOF
+
+  LIBS="-lcrypt $LIBS"
+
+fi
+ac_cv_lib_crypt=ac_cv_lib_crypt_main
+
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+echo "$as_me:$LINENO: checking for textdomain" >&5
+echo $ECHO_N "checking for textdomain... $ECHO_C" >&6
+if test "${ac_cv_func_textdomain+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define textdomain to an innocuous variant, in case <limits.h> declares textdomain.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define textdomain innocuous_textdomain
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char textdomain (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef textdomain
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char textdomain ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_textdomain) || defined (__stub___textdomain)
+choke me
+#else
+char (*f) () = textdomain;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != textdomain;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_textdomain=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_textdomain=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_textdomain" >&5
+echo "${ECHO_T}$ac_cv_func_textdomain" >&6
+if test $ac_cv_func_textdomain = yes; then
+  cat >>confdefs.h <<\_ACEOF
+#define ENABLE_NLS  1
+_ACEOF
+
+fi
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+# we cannot generate static libraries under Darwin
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  STATIC=""
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  STATIC="-static "
+fi
+
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+fi
+
+fi
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+
+
+echo "$as_me:$LINENO: checking that the GDBM library is available" >&5
+echo $ECHO_N "checking that the GDBM library is available... $ECHO_C" >&6
+if test ! -z "$gdbmdir" ; then
+  # look in given directory for header file
+  if test -r $gdbmdir/include/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir/include" ;
+  elif test -r $gdbmdir/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir" ;
+  fi
+
+  # look in given directory for library
+  if test -r $gdbmdir/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir" ;
+  elif test -r $gdbmdir/lib/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/lib" ;
+  elif test -r $gdbmdir/.libs/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/.libs" ;
+  fi
+
+  if test ! -z "$GDBM_LIBPATH" -a ! -z "$GDBM_INCLUDE" ; then
+    echo "$as_me:$LINENO: result: $gdbmdir" >&5
+echo "${ECHO_T}$gdbmdir" >&6 ;
+  else
+    echo "$as_me:$LINENO: result: can't find gdbm.h and libgdbm.a in $gdbmdir" >&5
+echo "${ECHO_T}can't find gdbm.h and libgdbm.a in $gdbmdir" >&6;
+    echo "$as_me:$LINENO: checking looking for GDBM library in default locations" >&5
+echo $ECHO_N "checking looking for GDBM library in default locations... $ECHO_C" >&6
+  fi
+fi
+
+# look in default path and in /usr/local
+if test -z "$GDBM_INCLUDE" ; then
+  # try in system directory
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <gdbm.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+success=no
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+  if test $success = "no"; then
+# last chance - look in /usr/local (eg FreeBSD)
+    GDBM_LIBPATH=-L/usr/local/lib
+    GDBM_INCLUDE=-I/usr/local/include
+    ac_save_CXXFLAGS="$CXXFLAGS"
+    ac_save_LIBS="$LIBS"
+      CXXFLAGS="$CXXFLAGS -I/usr/local/include"
+      LIBS="$LIBS -L/usr/local/lib -lgdbm"
+      cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <gdbm.h>
+#include <stdio.h>
+
+int
+main ()
+{
+printf("%s",gdbm_version);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success=/usr/local
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+success=no
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+      CXXFLAGS=$ac_save_CXXFLAGS
+      LIBS=$ac_save_LIBS
+  fi
+
+  echo "$as_me:$LINENO: result: $success" >&5
+echo "${ECHO_T}$success" >&6
+  if test $success = "no"; then
+  { { echo "$as_me:$LINENO: error: GDBM Library not available - cannot install" >&5
+echo "$as_me: error: GDBM Library not available - cannot install" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+fi
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_LIBGDBM  1
+_ACEOF
+
+
+
+
+
+echo "$as_me:$LINENO: checking that Perl 5 is available" >&5
+echo $ECHO_N "checking that Perl 5 is available... $ECHO_C" >&6
+success="no"
+pl_path="$PATH"
+IFS="${IFS=   }"; pl_save_ifs="$IFS"; IFS=":"
+for pl_dir in $pl_path; do
+  test -z "$pl_dir" && pl_dir=.
+  if test -x $pl_dir/perl; then
+    success="yes"
+    break
+  fi
+done
+IFS="$pl_save_ifs"
+
+if test $success = "yes"; then
+try=`perl configtest.pl` || try="0"
+try=`expr $try \> 5.000`
+if test $try = "1"; then
+echo "$as_me:$LINENO: result: \"yes\"" >&5
+echo "${ECHO_T}\"yes\"" >&6
+else
+success="no"
+fi
+fi
+
+if test $success = "no"; then
+echo "$as_me:$LINENO: result: \"no\"" >&5
+echo "${ECHO_T}\"no\"" >&6
+{ { echo "$as_me:$LINENO: error: \"Perl 5 not available - cannot install\"" >&5
+echo "$as_me: error: \"Perl 5 not available - cannot install\"" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+success=no
+echo "$as_me:$LINENO: checking \"whether STL library has known faults\"" >&5
+echo $ECHO_N "checking \"whether STL library has known faults\"... $ECHO_C" >&6
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <vector>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $success = "no"; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <vector.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success="yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+if test $success = "no"; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ospace\\std\\vector>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success="yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+if test $success = yes; then
+echo "$as_me:$LINENO: result: \"no\"" >&5
+echo "${ECHO_T}\"no\"" >&6
+else
+echo "$as_me:$LINENO: result: \"yes\"" >&5
+echo "${ECHO_T}\"yes\"" >&6
+{ { echo "$as_me:$LINENO: error: \"STL Broken - Obtain newer version of GNU C Compiler\"" >&5
+echo "$as_me: error: \"STL Broken - Obtain newer version of GNU C Compiler\"" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+gsdlos=`uname | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+# check for running bash under cygwin
+ if test "`echo $gsdlos | sed 's/cygwin//'`" != "$gsdlos" ;
+ then
+  gsdlos=windows
+ fi
+
+
+# check for endianness
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.o.$ac_objext"   | \
+  *" xmalloc.o.$ac_objext"   | \
+    "xmalloc.o.$ac_objext "* | \
+  *" xmalloc.o.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.o.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+# ---------------------------------------------------------------------------
+
+moduleDirs=""
+MODULEDIRS=""
+for x in `find modules -mindepth 1 -maxdepth 2`; do
+  if test -d "$x"; then
+    d="$x/Makefile.in"
+    if test -e "$d"; then
+      moduleDirs="$moduleDirs $x/Makefile"
+      MODULEDIRS="$MODULEDIRS $x"
+    fi
+  fi
+done
+echo "Making$moduleDirs in modules directory"
+
+
+# ---------------------------------------------------------------------------
+
+
+subdirs="$subdirs packages"
+
+
+# the list of folders in the src folder
+srclist="src/mgpp/lib/Makefile src/mgpp/text/Makefile src/mgpp/Makefile src/hashfile/Makefile src/txt2db/Makefile src/db2txt/Makefile src/phind/generate/Makefile "
+
+                                                                                          ac_config_files="$ac_config_files packages/mg/Makefile packages/mg/src/text/Makefile packages/mg/src/images/Makefile packages/mg/src/scripts/Makefile packages/mg/lib/Makefile lib/Makefile Makefile $srclist $moduleDirs"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "packages/mg/Makefile" ) CONFIG_FILES="$CONFIG_FILES packages/mg/Makefile" ;;
+  "packages/mg/src/text/Makefile" ) CONFIG_FILES="$CONFIG_FILES packages/mg/src/text/Makefile" ;;
+  "packages/mg/src/images/Makefile" ) CONFIG_FILES="$CONFIG_FILES packages/mg/src/images/Makefile" ;;
+  "packages/mg/src/scripts/Makefile" ) CONFIG_FILES="$CONFIG_FILES packages/mg/src/scripts/Makefile" ;;
+  "packages/mg/lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES packages/mg/lib/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "$srclist" ) CONFIG_FILES="$CONFIG_FILES $srclist" ;;
+  "$moduleDirs" ) CONFIG_FILES="$CONFIG_FILES $moduleDirs" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@CC@,$CC,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@build@,$build,;t t
+s,@build_cpu@,$build_cpu,;t t
+s,@build_vendor@,$build_vendor,;t t
+s,@build_os@,$build_os,;t t
+s,@host@,$host,;t t
+s,@host_cpu@,$host_cpu,;t t
+s,@host_vendor@,$host_vendor,;t t
+s,@host_os@,$host_os,;t t
+s,@target@,$target,;t t
+s,@target_cpu@,$target_cpu,;t t
+s,@target_vendor@,$target_vendor,;t t
+s,@target_os@,$target_os,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@STATIC@,$STATIC,;t t
+s,@GDBM_LIBPATH@,$GDBM_LIBPATH,;t t
+s,@GDBM_INCLUDE@,$GDBM_INCLUDE,;t t
+s,@gsdlos@,$gsdlos,;t t
+s,@MODULEDIRS@,$MODULEDIRS,;t t
+s,@subdirs@,$subdirs,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
+#
+# CONFIG_SUBDIRS section.
+#
+if test "$no_recursion" != yes; then
+
+  # Remove --cache-file and --srcdir arguments so they do not pile up.
+  ac_sub_configure_args=
+  ac_prev=
+  for ac_arg in $ac_configure_args; do
+    if test -n "$ac_prev"; then
+      ac_prev=
+      continue
+    fi
+    case $ac_arg in
+    -cache-file | --cache-file | --cache-fil | --cache-fi \
+    | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+      ac_prev=cache_file ;;
+    -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+    | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* \
+    | --c=*)
+      ;;
+    --config-cache | -C)
+      ;;
+    -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+      ac_prev=srcdir ;;
+    -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+      ;;
+    -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+      ac_prev=prefix ;;
+    -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+      ;;
+    *) ac_sub_configure_args="$ac_sub_configure_args $ac_arg" ;;
+    esac
+  done
+
+  # Always prepend --prefix to ensure using the same prefix
+  # in subdir configurations.
+  ac_sub_configure_args="--prefix=$prefix $ac_sub_configure_args"
+
+  ac_popdir=`pwd`
+  for ac_dir in : $subdirs; do test "x$ac_dir" = x: && continue
+
+    # Do not complain, so a configure script can configure whichever
+    # parts of a large source tree are present.
+    test -d $srcdir/$ac_dir || continue
+
+    { echo "$as_me:$LINENO: configuring in $ac_dir" >&5
+echo "$as_me: configuring in $ac_dir" >&6;}
+    { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+    cd $ac_dir
+
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      ac_sub_configure="$SHELL '$ac_srcdir/configure.gnu'"
+    elif test -f $ac_srcdir/configure; then
+      ac_sub_configure="$SHELL '$ac_srcdir/configure'"
+    elif test -f $ac_srcdir/configure.in; then
+      ac_sub_configure=$ac_configure
+    else
+      { echo "$as_me:$LINENO: WARNING: no configuration information is in $ac_dir" >&5
+echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2;}
+      ac_sub_configure=
+    fi
+
+    # The recursion is here.
+    if test -n "$ac_sub_configure"; then
+      # Make the cache file name correct relative to the subdirectory.
+      case $cache_file in
+      [\\/]* | ?:[\\/]* ) ac_sub_cache_file=$cache_file ;;
+      *) # Relative path.
+	ac_sub_cache_file=$ac_top_builddir$cache_file ;;
+      esac
+
+      { echo "$as_me:$LINENO: running $ac_sub_configure $ac_sub_configure_args --cache-file=$ac_sub_cache_file --srcdir=$ac_srcdir" >&5
+echo "$as_me: running $ac_sub_configure $ac_sub_configure_args --cache-file=$ac_sub_cache_file --srcdir=$ac_srcdir" >&6;}
+      # The eval makes quoting arguments work.
+      eval $ac_sub_configure $ac_sub_configure_args \
+	   --cache-file=$ac_sub_cache_file --srcdir=$ac_srcdir ||
+	{ { echo "$as_me:$LINENO: error: $ac_sub_configure failed for $ac_dir" >&5
+echo "$as_me: error: $ac_sub_configure failed for $ac_dir" >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+
+    cd $ac_popdir
+  done
+fi
+
Index: /main/tags/3.01/gs2build-extra/configure.in
===================================================================
--- /main/tags/3.01/gs2build-extra/configure.in	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/configure.in	(revision 21112)
@@ -0,0 +1,339 @@
+dnl Process this file with autoconf to produce a configure script.
+dnl Use autoconf-2.59 or later... it might not work anymore with
+dnl the older 2.1x versions - jrm21 12/2004
+AC_INIT(lib/text_t.h)
+AC_CONFIG_HEADER(config.h)
+
+PACKAGE=gsdl
+VERSION=2.1
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl
+dnl look for libgdbm if not in standard location
+dnl
+AC_ARG_WITH(gdbm, [  --with-gdbm=dir         Use local gdbm library and headers], gdbmdir=$withval)
+
+dnl Checks for programs.
+LDFLAGS=
+dnl only set CFLAGS if not already set in environment
+if test -z "$CFLAGS"; then
+CFLAGS=-O2
+fi
+AC_SUBST(CFLAGS)
+
+AC_PROG_CC
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+dnl get $target_cpu, $target_vendor, and $target_os
+dnl (as well as host_*)
+AC_CANONICAL_SYSTEM
+dnl do we actually want this? It adds a prefix based on host, system etc
+dnl AC_ARG_PROGRAM
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl check for -lg:
+AC_HAVE_LIBRARY(g)
+dnl check for -lm:
+AC_HAVE_LIBRARY(m)
+dnl check for -lcrypt:
+AC_HAVE_LIBRARY(crypt)
+
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+dnl see if gettext is in libc. (try the textdomain() function).
+AC_CHECK_FUNC(textdomain, AC_DEFINE(ENABLE_NLS ,1) )
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ([gmalloc])
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+# we cannot generate static libraries under Darwin
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  STATIC=""
+else
+  AC_MSG_RESULT(no)
+  STATIC="-static "
+fi
+AC_SUBST(STATIC)
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ([xmalloc])
+  AC_LIBOBJ([error])
+fi
+
+
+# ---------------------------------------------------------------------------
+
+dnl
+dnl Switch into C++
+dnl
+AC_LANG_CPLUSPLUS
+
+dnl
+dnl Check that the GDBM library is available
+dnl
+dnl check libgdbm manually, in case it's not in the standard location.
+
+AC_MSG_CHECKING(that the GDBM library is available)
+if test ! -z "$gdbmdir" ; then
+  # look in given directory for header file
+  if test -r $gdbmdir/include/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir/include" ;
+  elif test -r $gdbmdir/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir" ;
+  fi
+
+  # look in given directory for library
+  if test -r $gdbmdir/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir" ;
+  elif test -r $gdbmdir/lib/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/lib" ;
+  elif test -r $gdbmdir/.libs/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/.libs" ;
+  fi
+
+  if test ! -z "$GDBM_LIBPATH" -a ! -z "$GDBM_INCLUDE" ; then
+    AC_MSG_RESULT($gdbmdir) ;
+  else
+    AC_MSG_RESULT(can't find gdbm.h and libgdbm.a in $gdbmdir);
+    AC_MSG_CHECKING(   looking for GDBM library in default locations)
+  fi
+fi
+
+# look in default path and in /usr/local
+if test -z "$GDBM_INCLUDE" ; then
+  # try in system directory 
+  AC_TRY_COMPILE(
+  [#include <gdbm.h>
+],
+  [],
+  success=yes,
+  success=no
+  )
+
+  if test $success = "no"; then
+# last chance - look in /usr/local (eg FreeBSD)
+    GDBM_LIBPATH=-L/usr/local/lib
+    GDBM_INCLUDE=-I/usr/local/include
+    ac_save_CXXFLAGS="$CXXFLAGS"
+    ac_save_LIBS="$LIBS"
+      CXXFLAGS="$CXXFLAGS -I/usr/local/include"
+      LIBS="$LIBS -L/usr/local/lib -lgdbm"
+      AC_TRY_LINK(
+  	[#include <gdbm.h>
+#include <stdio.h>
+	],
+  	[printf("%s",gdbm_version);],
+  	success=/usr/local,
+  	success=no
+      )
+      CXXFLAGS=$ac_save_CXXFLAGS
+      LIBS=$ac_save_LIBS
+  fi
+
+  AC_MSG_RESULT($success)
+  if test $success = "no"; then
+  AC_MSG_ERROR(GDBM Library not available - cannot install)
+  fi
+fi
+dnl I don't think anything actually checks this define, but...
+AC_DEFINE(HAVE_LIBGDBM , 1)
+
+AC_SUBST(GDBM_LIBPATH)
+AC_SUBST(GDBM_INCLUDE)
+
+dnl
+dnl Now check that Perl is actually around, and can be found
+dnl
+AC_MSG_CHECKING(that Perl 5 is available)
+success="no"
+pl_path="$PATH"
+IFS="${IFS=   }"; pl_save_ifs="$IFS"; IFS=":"
+for pl_dir in $pl_path; do
+  test -z "$pl_dir" && pl_dir=.
+  if test -x $pl_dir/perl; then
+    success="yes"
+    break
+  fi
+done
+IFS="$pl_save_ifs"
+
+dnl
+dnl Perl actually can be run; now we see if it is version 5 by
+dnl running a sample script (which returns a "clean" version number
+dnl and then tests the result using expr.  If perl errors in some
+dnl way (eg. broken Perl executable, missing script), we set a failure
+dnl value ("0") and then proceed as best we can
+dnl
+if test $success = "yes"; then
+try=`perl configtest.pl` || try="0"
+try=`expr $try \> 5.000`
+if test $try = "1"; then
+AC_MSG_RESULT("yes")
+else
+success="no"
+fi
+fi
+
+if test $success = "no"; then
+AC_MSG_RESULT("no")
+AC_MSG_ERROR("Perl 5 not available - cannot install")
+fi
+
+dnl
+dnl Check if the STL library has any known faults;
+dnl In GCC 2.7.2.x series, some STL library headers are missing
+dnl a file called "new" which is referred to by other headers
+dnl (eg. vector/vector.h); if this applies, then all use of the
+dnl STL will cause a compile fail, so we'd better let the user
+dnl know now, before things really go wrong!
+dnl
+success=no
+AC_MSG_CHECKING("whether STL library has known faults")
+
+dnl - first shot, try simple good old vector on its own
+AC_TRY_COMPILE(
+[#include <vector>
+], 
+[],
+success=yes,
+)
+
+dnl - try using the .h equivalent
+if test $success = "no"; then
+AC_TRY_COMPILE(
+[#include <vector.h>
+], 
+[],
+success="yes",
+)
+fi
+
+dnl - now try for the objectspace headers
+if test $success = "no"; then
+AC_TRY_COMPILE(
+[#include <ospace\\std\\vector>
+], 
+[],
+success="yes",
+)
+fi
+
+if test $success = yes; then
+AC_MSG_RESULT("no")
+else
+AC_MSG_RESULT("yes")
+AC_MSG_ERROR("STL Broken - Obtain newer version of GNU C Compiler")
+fi
+
+gsdlos=`uname | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+# check for running bash under cygwin
+ if test "`echo $gsdlos | sed 's/cygwin//'`" != "$gsdlos" ;
+ then
+  gsdlos=windows
+ fi
+AC_SUBST(gsdlos)
+
+# check for endianness
+AC_C_BIGENDIAN
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ([xmalloc.o])
+  AC_LIBOBJ([error])
+fi
+
+# ---------------------------------------------------------------------------
+
+dnl
+dnl Locate all children of the modules folder
+dnl
+moduleDirs=""
+MODULEDIRS=""
+for x in `find modules -mindepth 1 -maxdepth 2`; do
+  if test -d "$x"; then
+    d="$x/Makefile.in"
+    if test -e "$d"; then
+dnl      echo $d
+      moduleDirs="$moduleDirs $x/Makefile"
+      MODULEDIRS="$MODULEDIRS $x"
+    fi
+  fi
+done
+echo "Making$moduleDirs in modules directory"
+AC_SUBST(MODULEDIRS)
+
+# ---------------------------------------------------------------------------
+dnl for Darwin, as our packages don't (yet) correctly guess our system type.
+dnl not needed any more?
+dnl ac_configure_args="$ac_configure_args --build=$build --target=$target"
+AC_CONFIG_SUBDIRS(packages)
+
+# the list of folders in the src folder
+srclist="src/mgpp/lib/Makefile src/mgpp/text/Makefile src/mgpp/Makefile src/hashfile/Makefile src/txt2db/Makefile src/db2txt/Makefile src/phind/generate/Makefile "
+
+AC_OUTPUT(packages/mg/Makefile packages/mg/src/text/Makefile packages/mg/src/images/Makefile packages/mg/src/scripts/Makefile packages/mg/lib/Makefile lib/Makefile Makefile $srclist $moduleDirs)
Index: /main/tags/3.01/gs2build-extra/lib.Makefile.in
===================================================================
--- /main/tags/3.01/gs2build-extra/lib.Makefile.in	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/lib.Makefile.in	(revision 21112)
@@ -0,0 +1,80 @@
+###########################################################################
+#
+# Makefile -- lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+CC = @CC@
+CPP = @CXX@
+#CCFLAGS = @CFLAGS@ -Wall -Wunused -pedantic -W -Woverloaded-virtual
+CCFLAGS = @CFLAGS@
+DEFS = @DEFS@
+INCLUDES = -I../packages/mg/lib -I..
+RANLIB = @RANLIB@
+
+COMPILE = $(CC) -c $(CCFLAGS) $(DEFS) $(INCLUDES)
+CPPCOMPILE = $(CPP) -c $(CCFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CPPCOMPILE) $< 
+
+HEADERS = \
+ text_t.h gsdlconf.h 
+
+SOURCES = \
+  text_t.cpp 
+
+OBJECTS = \
+  text_t.o  
+
+
+all: gsdllib.a
+
+GSDLLIBOBJECTS = $(OBJECTS)
+
+gsdllib.a: $(GSDLLIBOBJECTS)
+	rm -f gsdllib.a
+	$(AR) cru gsdllib.a $(GSDLLIBOBJECTS)
+	$(RANLIB) gsdllib.a
+
+clean:
+	rm -f $(OBJECTS) gsdllib.a 
+
+distclean: clean
+	rm -f Makefile
+install:
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+
+
Index: /main/tags/3.01/gs2build-extra/lib.win32.mak
===================================================================
--- /main/tags/3.01/gs2build-extra/lib.win32.mak	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/lib.win32.mak	(revision 21112)
@@ -0,0 +1,81 @@
+###########################################################################
+#
+# win32 makefile -- lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..
+USE_STLPORT = 0
+DLL = 0
+DLLDEBUG = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+AR = lib
+CC = cl
+
+CPPFLAGS = -GX
+!IF $(DLL)
+CPPFLAGS = $(CPPFLAGS) -MD
+!ELSE
+!IF ($(DLLDEBUG))
+CPPFLAGS = $(CPPFLAGS) -MDd
+!ENDIF
+!ENDIF
+
+DEFS = -D__WIN32__ -DHAVE_CONFIG_H -DPARADOCNUM -D_LITTLE_ENDIAN \
+       -DSHORT_SUFFIX
+INCLUDES = -I"$(GSDLHOME)" -I"$(GSDLHOME)\packages\mg\lib"
+!IF $(USE_STLPORT)
+DEFS = $(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .obj
+.cpp.obj:
+	$(COMPILE) $< 
+
+HEADERS = text_t.h gsdlconf.h
+
+SOURCES = text_t.cpp 
+
+OBJECTS = text_t.obj 
+
+all: gsdllib.lib
+
+gsdllib.lib : $(OBJECTS)
+	if EXIST gsdllib.lib del gsdllib.lib
+	$(AR) /out:gsdllib.lib $(OBJECTS)
+
+clean:
+	del *.obj 
+	del gsdllib.lib
+
+install:
+
Index: /main/tags/3.01/gs2build-extra/packages.Makefile
===================================================================
--- /main/tags/3.01/gs2build-extra/packages.Makefile	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/packages.Makefile	(revision 21112)
@@ -0,0 +1,58 @@
+###########################################################################
+#
+# Makefile -- Makefile for packages included with Greenstone
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# PACKAGEDIRS contains the packages that we want to make and install.  It
+# is assumed that each package will have at least four rules: all, install,
+# clean, and distclean.
+
+PACKAGEDIRS =	wv/wv-gs rtftohtml/rtftohtml_src pdftohtml/pdftohtml_gs \
+		wget/wget-1.9 expat/ cpan/XML-Parser-2.34 \
+		xlhtml/xlhtml-0.4.9.0 isis-gdl
+
+all:
+	for odir in $(PACKAGEDIRS); do \
+          echo making $@ in $$odir; \
+          (cd $$odir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+        done
+
+install:
+	for idir in $(PACKAGEDIRS); do \
+          echo installing $$idir; \
+          (cd $$idir && $(MAKE) install) || exit 1; \
+        done
+
+clean:
+	for subdir in $(PACKAGEDIRS); do \
+          echo cleaning $@ in $$subdir; \
+          (cd $$subdir && $(MAKE) clean || exit 0) ; \
+        done
+
+distclean:
+	for subdir in $(PACKAGEDIRS); do \
+          echo cleaning $@ in $$subdir; \
+          (cd $$subdir && $(MAKE) distclean || exit 0) ; \
+        done
Index: /main/tags/3.01/gs2build-extra/packages.configure
===================================================================
--- /main/tags/3.01/gs2build-extra/packages.configure	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/packages.configure	(revision 21112)
@@ -0,0 +1,167 @@
+#! /bin/sh
+
+PACKAGES=`pwd`
+cd ..
+GSDLHOME=`pwd`
+cd $PACKAGES
+
+GSDLOS=`uname -s |tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+# check for running bash under cygwin
+if test "`echo $GSDLOS | sed 's/cygwin//'`" != "$GSDLOS" ;
+then
+ GSDLOS=windows
+fi
+
+prefix=""
+bindir=""
+HOSTTARGETFLAGS=""
+
+# parse any arguments given from toplevel configure
+while test $# -ne 0; do
+    case "$1" in
+	--cache-file=*)
+	    cache_filename=`echo $1 | sed s/--cache-file=//`
+	    # turn relative path into an absolute path
+	    cache_dir=`echo $cache_filename | sed 's@/[^/]*$@@'`
+	    cache_dir=`cd $cache_dir 2>/dev/null && pwd`
+
+	    cache_basefile=`echo $cache_filename | sed 's@.*/@@'`
+	    cache_file="$cache_dir/$cache_basefile"
+	    if test ! -f "$cache_file"; then
+		CACHE_FILE=""
+	    else
+		CACHE_FILE="--cache-file=$cache_file"
+	    fi
+	    ;;
+	--prefix=*)
+	    prefix=`echo $1 | sed s/--prefix=//`
+	    ;;
+	--bindir=*)
+	    # where to install binaries to
+	    bindir=`echo $1 | sed s/--bindir=//`
+	    # turn relative path into an absolute path
+	    if test -d "$bindir" ; then
+	      bindir=`cd "$bindir" ; pwd `
+	      cd "$PACKAGES"
+	    elif test -z `echo $bindir | sed '/^\// p;D'` ; then
+	      rel_bindir="`echo $bindir | sed s@/.*@@`"
+	      cd "$rel_bindir"
+	      bindir="`pwd`/`echo $bindir | sed s@^[^/]*/@@`"
+	      cd "$PACKAGES"
+	    else
+	      # bindir doesn't exist, and isn't relative
+	      true
+	    fi
+	    ;;
+	--host=* | --target=*)
+	    HOSTTARGETFLAGS="$HOSTTARGETFLAGS $1"
+	    ;;
+	*)
+	    # ignore all other options for now
+	    ;;
+    esac
+    shift
+done
+
+# note! Our GSDL package currently always installs into it's source
+# directory, ignoring any --prefix given to configure.
+# When this is fixed, uncomment the relevant lines below...
+
+##if test -z "$prefix" ; then
+prefix="${GSDLHOME}"
+##fi
+
+##if test -z "$bindir" ; then
+bindir="${prefix}/bin/$GSDLOS"
+##fi
+
+# This loads the top configure's cache file, with variables already set.
+if test -f "$cache_file" ; then
+  . $cache_file
+fi
+
+# we assume that gzip and tar are on the search path.
+# non-GNU versions of tar don't take the -z option.
+
+# configure the wvWare Word document converter
+echo ""
+echo "Configuring wvWare"
+echo ""
+
+cd "$PACKAGES/wv"
+gzip -dc wv-0.7.1-gs.tar.gz | tar -xf -
+# datadir is for default location of wvHtml.xml config file
+cd wv-gs && \
+./configure $CACHE_FILE --prefix=$PACKAGES/wv --bindir="$bindir" --datadir="$prefix/etc/packages" $HOSTTARGETFLAGS
+
+# configure the pdftohtml converter
+echo ""
+echo "Configuring pdftohtml"
+echo ""
+
+cd "$PACKAGES/pdftohtml"
+gzip -dc pdftohtml_0_36_gs.tar.gz | tar -xf -
+cd pdftohtml_gs && \
+./configure $CACHE_FILE --prefix="$prefix" --bindir="$bindir" $HOSTTARGETFLAGS
+
+# rtftohtml converter
+echo ""
+echo "Configuring rtftohtml"
+echo ""
+
+cd "$PACKAGES/rtftohtml"
+gzip -dc rtftohtml-gs.tar.gz | tar -xf -
+cd rtftohtml_src && \
+./configure $CACHE_FILE --prefix="$prefix" --bindir="$bindir" $HOSTTARGETFLAGS
+
+# configure wget
+echo ""
+echo "Configuring wget"
+echo ""
+cd "$PACKAGES/wget"
+gzip -dc wget.tar.gz | tar -xf -
+cd "$PACKAGES/wget/wget-1.9" && \
+./configure $CACHE_FILE --prefix=$PACKAGES/wget --bindir="$bindir" $HOSTTARGETFLAGS
+
+# configure expat (needs to be done before XML::Parser
+echo ""
+echo "Configuring expat"
+echo ""
+cd "$PACKAGES/expat"
+gzip -dc expat-1.95.8.tar.gz | tar -xf -
+cd "$PACKAGES/expat/expat-1.95.8" && \
+./configure $CACHE_FILE --prefix=$PACKAGES/expat --bindir="$bindir" $HOSTTARGETFLAGS
+
+# configure XML::Parser perl module
+echo ""
+echo "Configuring XML::Parser"
+echo ""
+cd "$PACKAGES/cpan"
+perl_dir=`echo 'my $vn = $]; $vn =~ s/5\.00(\d).*$/perl-5.$1/; print "$vn" ' | perl`
+gzip -dc XML-Parser-2.34.tar.gz | tar -xf -
+cd "$PACKAGES/cpan/XML-Parser-2.34" && \
+perl Makefile.PL \
+  INSTALLSITELIB="$GSDLHOME/perllib/cpan/$perl_dir" \
+  PREFIX="$GSDLHOME/perllib/cpan/XML" \
+  INSTALLMAN1DIR="$GSDLHOME/perllib/cpan/$perl_dir" \
+  INSTALLMAN3DIR="$GSDLHOME/perllib/cpan/$perl_dir" \
+  SITEPREFIX="$GSDLHOME/perllib/cpan" \
+  EXPATINCPATH="$GSDLHOME/packages/expat/include" \
+  EXPATLIBPATH="$GSDLHOME/packages/expat/lib" 
+
+# configure powerpoint converter
+echo ""
+echo "Configuring xlhtml (powerpoint & excel converter)"
+echo ""
+cd "$PACKAGES/xlhtml"
+gzip -dc xlhtml-0.4.9.0-gs.tar.gz | tar -xf -
+cd xlhtml-0.4.9.0 && \
+./configure $CACHE_FILE --prefix="$prefix" --bindir="$bindir" $HOSTTARGETFLAGS
+
+
+# configure IsisGdl program
+echo ""
+echo "Configuring IsisGdl (CDS/ISIS database reader)"
+echo ""
+cd "$PACKAGES/isis-gdl" && \
+./configure $CACHE_FILE --prefix="$prefix" --bindir="$bindir" $HOSTTARGETFLAGS
Index: /main/tags/3.01/gs2build-extra/setup.bat
===================================================================
--- /main/tags/3.01/gs2build-extra/setup.bat	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/setup.bat	(revision 21112)
@@ -0,0 +1,51 @@
+@echo off
+set GSDLLANG=en
+
+if "%OS%" == "Windows_NT" goto WinNT
+if "%OS%" == "" goto Win95
+if "%GSDLLANG%" == "en" echo Setup failed - your PATH has not been set
+if "%GSDLLANG%" == "es" echo No se pudo realizar la configuraci¢n - no se ha establecido la RUTA.
+if "%GSDLLANG%" == "fr" echo Echc de l'installation - votre variable PATH n'a pas t ajuste
+if "%GSDLLANG%" == "ru" echo áâ ­®¢ª  ­¥ ã€ « áì -  ­¥ ¡ë« ãáâ ­®¢«¥­
+goto End
+
+:WinNT
+set GSDLHOME=@gsdlhome@
+set GSDLOS=windows
+set PATH=%GSDLHOME%\bin\windows\perl\bin;%GSDLHOME%\bin\windows;%GSDLHOME%\bin\script;%PATH%
+goto Success
+
+:Win95
+if "%1" == "SetEnv" goto Win95Env
+REM We'll invoke a second copy of the command processor to make
+REM sure there's enough environment space
+COMMAND /E:2048 /K %0 SetEnv
+goto End
+
+:Win95Env
+set GSDLHOME=@gsdlhome@
+set GSDLOS=windows
+set PATH="%GSDLHOME%\bin\windows\perl\bin";"%GSDLHOME%\bin\windows";"%GSDLHOME%\bin\script";"%PATH%"
+
+:Success
+if "%GSDLLANG%" == "en" echo Your environment has successfully been set up to run Greenstone.
+if "%GSDLLANG%" == "en" echo Note that these settings will only have effect within this MS-DOS
+if "%GSDLLANG%" == "en" echo session. You will therefore need to rerun setup.bat if you want
+if "%GSDLLANG%" == "en" echo to run Greenstone programs from a different MS-DOS session.
+
+if "%GSDLLANG%" == "es" echo Su ambiente ha sido configurado para correr los programas Greenstone.
+if "%GSDLLANG%" == "es" echo Recuerde que estos ajustes £nicamente tendr n efecto dentro de esta sesi¢n
+if "%GSDLLANG%" == "es" echo MS-DOS. Por lo tanto deber  ejecutar nuevamente setup.bat si desea
+if "%GSDLLANG%" == "es" echo correr los programas de Greenstone desde una sesi¢n MS-DOS diferente.
+
+if "%GSDLLANG%" == "fr" echo Votre environnement a t configure avec succs pour excuter Greenstone
+if "%GSDLLANG%" == "fr" echo Notez que ces paramtrages n'auront d'effet que dans cette session MS-DOS.
+if "%GSDLLANG%" == "fr" echo Vous devrez par consquent rexcuter setup.bat si vous voulez faire
+if "%GSDLLANG%" == "fr" echo lancer des programmes Greenstone dans une autre session MS-DOS.
+
+if "%GSDLLANG%" == "ru" echo  è¥ ®ªàãŠ¥­š¥ ¡ë«® ãá¯¥è­® ­ áâà®¥­®, çâ®¡ë ãáâ ­®¢šâì Greenstone ¡à âšâ¥
+if "%GSDLLANG%" == "ru" echo ¢­š¬ ­š¥, çâ® íâš ­ §­ ç¥­šï ¡ã€ãâ â®«ìª® š¬¥âì íää¥ªâ ¢ ¯à¥€¥« å íâ®£® MS DOS
+if "%GSDLLANG%" == "ru" echo á¥áášï. ë ¡ã€¥â¥ ¯®íâ®¬ã €®«Š­ë ¯®¢â®à­® ã¯à ¢«ïâì setup.bat, ¥á«š ë å®âšâ¥
+if "%GSDLLANG%" == "ru" echo ã¯à ¢«ïâì ¯à®£à ¬¬ ¬š ¥«ñ­ëå š§¢¥àŠ¥­­ëå ¯®à®€ ®â à §«šç­®© á¥áášš MS DOS.
+
+:End
Index: /main/tags/3.01/gs2build-extra/win32.mak
===================================================================
--- /main/tags/3.01/gs2build-extra/win32.mak	(revision 21112)
+++ /main/tags/3.01/gs2build-extra/win32.mak	(revision 21112)
@@ -0,0 +1,97 @@
+###########################################################################
+#
+# win32 makefile -- gsdl
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+AR = lib
+CC = cl
+CPPFLAGS =
+
+GSDLHOME = $(MAKEDIR)
+USE_STLPORT = 0
+DLL = 0
+DLLDEBUG = 0
+
+MDEFINES = /f win32.mak
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+SOURCES = 
+
+OBJECTS = 
+
+OTHERDIRS = packages\windows\gdbm\gdbm packages\windows\crypt\crypt \
+            packages\mg\lib src\mgpp\lib lib 
+
+INSTALLDIRS = packages\mg\src\text src\mgpp\text
+
+MAKECMD = $(MAKE) $(MDEFINES) GSDLHOME="$(GSDLHOME)"
+!IF $(USE_STLPORT)
+MAKECMD = $(MAKECMD) USE_STLPORT=1
+!ENDIF
+!IF $(DLL)
+MAKECMD = $(MAKECMD) DLL=1
+!ENDIF
+!IF $(DLLDEBUG)
+MAKECMD = $(MAKECMD) DLLDEBUG=1
+!ENDIF
+
+all: win32.mak $(OTHERDIRS) $(INSTALLDIRS)
+
+install: win32.mak $(INSTALLDIRS)
+
+$(OTHERDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	cd "$(GSDLHOME)"
+
+$(INSTALLDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	$(MAKECMD) install
+	cd "$(GSDLHOME)"
+
+FORCE:
+
+clean: win32.mak 
+	cd packages\windows\gdbm\gdbm 
+	$(MAKECMD) clean
+	cd ..\..\..\..
+        cd packages\windows\crypt\crypt 
+	$(MAKECMD) clean
+	cd ..\..\..\..
+	cd packages\mg\lib
+	$(MAKECMD) clean
+	cd ..\..\..
+	cd packages\mg\src\text
+	$(MAKECMD) clean
+	cd ..\..\..\..
+	cd src\mgpp\lib
+	$(MAKECMD) clean
+	cd ..\..\..
+ 	cd src\mgpp\text
+	$(MAKECMD) clean
+	cd ..\..\..
+	cd lib
+	$(MAKECMD) clean
+	 cd ..
Index: /main/tags/3.01/gsdl/WIN32cfg.h
===================================================================
--- /main/tags/3.01/gsdl/WIN32cfg.h	(revision 21112)
+++ /main/tags/3.01/gsdl/WIN32cfg.h	(revision 21112)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /main/tags/3.01/gsdl/acconfig.h
===================================================================
--- /main/tags/3.01/gsdl/acconfig.h	(revision 21112)
+++ /main/tags/3.01/gsdl/acconfig.h	(revision 21112)
@@ -0,0 +1,50 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define FASTCGI if needsbe */
+#undef USE_FASTCGI
+
+/* Define USE_CORBA if needsbe */
+#undef USE_CORBA
+
+/* Define MICO_DIR if needsbe */
+#undef MICO_DIR
+
+/* Define MICO_VERSION if required */
+#undef MICO_VERSION
+
+
+
Index: /main/tags/3.01/gsdl/aclocal.m4
===================================================================
--- /main/tags/3.01/gsdl/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/gsdl/aclocal.m4	(revision 21112)
@@ -0,0 +1,140 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ([regex])
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , [AC_LIBOBJ([rx])])
+fi
+])
Index: /main/tags/3.01/gsdl/bin/script/anonymiselog.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/anonymiselog.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/anonymiselog.pl	(revision 21112)
@@ -0,0 +1,120 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# anonymiselog.pl -- anonymise a log file by MD5 hashing all IP addresses
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+use MD5;
+
+
+sub main
+{
+    # Get the name of the log file to process
+    local $logfilename = shift(@_);
+
+    # Check that the necessary arguments were supplied
+    if (!$logfilename) {
+	print STDERR "Usage: anonymiselog.pl <logfile-name>\n";
+	die "Error: Required argument missing.\n";
+    }
+
+    # Open the log file
+    open(LOG_FILE, "<$logfilename") or die "Error: Could not open log file $logfilename.\n";
+
+    # Open the output file
+    local $outfilename = $logfilename . ".anon";
+    open(OUT_FILE, ">$outfilename") or die "Error: Could not write file $outfilename.\n"; 
+
+    # Create a new MD5 (RSA Data Security Inc. MD5 Message Digest) object
+    local $md5 = new MD5;
+
+    # Process the log, one line at a time
+    local $entry = "";
+    while (<LOG_FILE>) {
+	local $line = $_;
+	# print "Line: $line";
+
+	# If this line starts a new entry, process the previous one
+	if ($line =~ /^\//) {
+	    print OUT_FILE &anonymise_log_entry($entry);
+	    $entry = "";
+	}
+
+	# Remove trailing whitespace, and skip blank lines
+	$line =~ s/(\s*)$//;
+	next if ($line =~ /^$/);
+
+	$entry = $entry . $line;
+    }
+
+    # Process the last entry
+    print OUT_FILE &anonymise_log_entry($entry);
+
+    # All done
+    close(LOG_FILE);
+    close(OUT_FILE);
+}
+
+
+sub anonymise_log_entry
+{
+    local $entry = shift(@_);
+    return "" if ($entry eq "");
+
+    # Parse the IP address from the entry
+    $entry =~ /^\S+\s((\w|-|\.)+)\s\[/;
+    if (!defined($1)) {
+	print STDERR "Could not extract IP address from entry: $entry\n";
+	return "";
+    }
+
+    # Casefold the IP address, hash using MD5, and take the last 16 characters
+    local $ipaddress = $1;
+    $ipaddress =~ tr/A-Z/a-z/;
+    local $hashedaddress = substr($md5->hexhash($ipaddress), -16);
+
+    # Replace the IP address with the hashed value
+    $entry =~ s/$ipaddress/$hashedaddress/ig;
+
+    # Parse the Greenstone user identifier (z variable) from the entry
+    $entry =~ /\sz=((\w|-|\.)+)/;
+    if (!defined($1)) {
+	print STDERR "No z variable in entry: $entry\n";
+	return "";
+    }
+
+    # Casefold the Greenstone user ID, hash using MD5, and take the last 16 characters
+    local $gsuserid = $1;
+    $gsuserid =~ tr/A-Z/a-z/;
+    local $hasheduserid = substr($md5->hexhash($gsuserid), -16);
+
+    # Replace the Greenstone user ID with the hashed value
+    $entry =~ s/$gsuserid/$hasheduserid/ig;
+    return $entry . "\n";
+}
+
+
+&main(@ARGV);
Index: /main/tags/3.01/gsdl/bin/script/build
===================================================================
--- /main/tags/3.01/gsdl/bin/script/build	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/build	(revision 21112)
@@ -0,0 +1,625 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# build --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This perl script may be called directly or by running build.bat on
+# windows (build.bat is in bin\windows)
+
+# Note that this script has grown over time and now has many options for
+# use when called from within the collector. If it appears to
+# over-complicate things a little, that's why.
+
+package build;
+
+use FileHandle;
+use File::Copy;
+
+BEGIN {
+
+    die "GSDLHOME not set - did you remember to source setup.bash (unix) or " .
+	"run setup.bat (windows)?\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set - did you remember to source setup.bash (unix) or " .
+	"run setup.bat (windows)?\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+
+    STDOUT->autoflush(1);
+    STDERR->autoflush(1);
+}
+
+use lib qq($ENV{'GSDLHOME'}/perllib/cpan);
+use Mail::Sendmail;
+use parsargv;
+use util;
+use cfgread;
+
+# set up path - this allows for paths not to be supplied to system calls
+# and overcomes problems when GSDLHOME contains spaces (double quoting
+# the call doesn't work on win2k and probably other variants of winnt)
+my $path_separator = ":";
+$path_separator = ";" if $ENV{'GSDLOS'} =~ /^windows$/;
+$ENV{'PATH'} = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}) .
+    $path_separator . &util::filename_cat($ENV{'GSDLHOME'}, "bin", "script") .
+    $path_separator . $ENV{'PATH'};
+
+&parse_args (\@ARGV);
+
+my ($collection) = @ARGV;
+
+if (!defined $collection || $collection !~ /\w/) {
+    print STDERR "You must specify a collection to build\n";
+    &print_usage();
+    die "\n";
+}
+
+if ($optionfile =~ /\w/) {
+    open (OPTIONS, $optionfile) || die "Couldn't open $optionfile\n";
+    my $line = [];
+    my $options = [];
+    while (defined ($line = &cfgread::read_cfg_line ('build::OPTIONS'))) {
+	push (@$options, @$line);
+    }
+    close OPTIONS;
+    &parse_args ($options);
+}
+
+if ($maxdocs == -1) {
+    $maxdocs = "";
+} else {
+    $maxdocs = "-maxdocs $maxdocs";
+}
+
+my $cdir = $collectdir;
+$cdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect") unless $collectdir =~ /\w/;
+my $importdir = &util::filename_cat ($cdir, $collection, "import");
+my $archivedir = &util::filename_cat ($cdir, $collection, "archives");
+my $buildingdir = &util::filename_cat ($cdir, $collection, "building");
+my $indexdir = &util::filename_cat ($cdir, $collection, "index");
+my $bindir = &util::filename_cat ($ENV{'GSDLHOME'}, "bin");
+
+my $use_out = 0;
+my $outfile = $out;
+if ($out !~ /^(STDERR|STDOUT)$/i) {
+    open (OUT, ">$out") || die "Couldn't open output file $out\n";
+    $out = "OUT";
+    
+    # delete any existing .final file
+    &util::rm ("$outfile.final") if -e "$outfile.final";
+
+    $use_out = 1;
+}
+$out->autoflush(1);
+
+# delete any .kill file left laying around from a previously aborted build
+if (-e &util::filename_cat ($cdir, $collection, ".kill")) {
+    &util::rm (&util::filename_cat ($cdir, $collection, ".kill"));
+}
+
+# get maintainer email address from main.cfg
+my $maintainer = "NULL";
+my $main_cfg = &util::filename_cat ($ENV{'GSDLHOME'}, "etc", "main.cfg");
+my $cfgdata = &cfgread::read_cfg_file ($main_cfg, "maintainer");
+if (defined $cfgdata->{'maintainer'} && $cfgdata->{'maintainer'} =~ /\w/) {
+    $maintainer = $cfgdata->{'maintainer'};
+}
+# if maintainer is "NULL" email_events should be disabled
+if ($maintainer =~ /^NULL$/i) {
+    $email_events = "";
+}
+
+&main();
+
+if ($use_out) {
+    close OUT;
+
+    # if we've created a build log we'll copy it to the collection's etc
+    # directory
+    my ($etcdir);
+    if ($dontinstall) {
+	$etcdir = &util::filename_cat($collectdir, "etc", "build.log");
+    } else {
+	$etcdir = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection, "etc", "build.log");
+    }
+    
+    &util::cp($outfile, $etcdir);
+}
+
+sub print_usage {
+    print STDOUT "\n";
+    print STDOUT "build: Builds a Greenstone collection (i.e. runs import.pl and buildcol.pl\n";
+    print STDOUT "       then copies the resulting indexes to the correct place).\n\n";
+    print STDOUT "  usage: $0 [options] collection-name\n\n";
+    print STDOUT "  options:\n";
+    print STDOUT "   -optionfile file        Get options from file, useful on systems where\n";
+    print STDOUT "                           long command lines may cause problems\n";
+    print STDOUT "   -append                 Add new files to existing collection\n";
+    print STDOUT "   -remove_archives        Remove archives directory after successfully\n";
+    print STDOUT "                           building the collection.\n";
+    print STDOUT "   -remove_import          Remove import directory after successfully\n";
+    print STDOUT "                           importing the collection.\n";
+    print STDOUT "   -buildtype build|import If 'build' attempt to build directly\n";
+    print STDOUT "                           from archives directory (bypassing import\n";
+    print STDOUT "                           stage). Defaults to 'import'\n";
+    print STDOUT "   -maxdocs number         Maximum number of documents to build\n";
+    print STDOUT "   -download directory     Directory (or file) to get import documents from.\n";
+    print STDOUT "                           There may be multiple download directories and they\n";
+    print STDOUT "                           may be of type http://, ftp://, or file://\n";
+    print STDOUT "                           Note that any existing import directory will be\n";
+    print STDOUT "                           deleted to make way for the downloaded data if\n";
+    print STDOUT "                           a -download option is supplied\n";
+    print STDOUT "   -collectdir directory   Collection directory (defaults to " .
+	&util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n";
+    print STDOUT "   -dontinstall            Only applicable if -collectdir is set to something\n";
+    print STDOUT "                           other than the default. -dontinstall will suppress\n";
+    print STDOUT "                           the default behaviour which is to install the\n";
+    print STDOUT "                           collection to the gsdl/collect directory once it has\n";
+    print STDOUT "                           been built.\n";
+    print STDOUT "   -save_archives          Create a copy of the existing archives directory\n";
+    print STDOUT "                           called archives.org\n";
+    print STDOUT "   -out                    Filename or handle to print output status to.\n";
+    print STDOUT "                           The default is STDERR\n";
+    print STDOUT "   -statsfile name         Filename or handle to print import statistics to.\n";
+    print STDOUT "                           The default is STDERR\n";
+    print STDOUT "   -make_writable          If set build will make the collection and any\n";
+    print STDOUT "                           temporary files it created globally writable after\n";
+    print STDOUT "                           it finishes\n";
+    print STDOUT "   -log_events             Log important events (collection built successfully\n";
+    print STDOUT "                           etc.) to event_log_file\n";
+    print STDOUT "   -event_log_file file    File to append important events to (defaults to\n";
+    print STDOUT "                           " . &util::filename_cat ($ENV{'GSDLHOME'}, "etc", "events.txt") . "\n";
+    print STDOUT "   -email_events addr      Comma separated list of email addresses to mail\n";
+    print STDOUT "                           details of important collection building events\n";
+    print STDOUT "   -mail_server server     The outgoing (SMTP) mail server to be used by\n";
+    print STDOUT "                           email_events. email_events will be disabled if\n";
+    print STDOUT "                           mail_server isn't set\n";
+    print STDOUT "   -event_header file      File containing a header to go on any event\n";
+    print STDOUT "                           messages. If not specified build will create a\n";
+    print STDOUT "                           generic header\n\n";
+    print STDOUT "  [Type \"build | more\" if this help text scrolled off your screen]";
+    print STDOUT "\n" unless $ENV{'GSDLOS'} =~ /^windows$/i;
+}
+
+sub main {
+
+    if ($save_archives && -d $archivedir) {
+	print $out "caching original archives to ${archivedir}.org\n";
+	&util::cp_r ($archivedir, "${archivedir}.org");
+    }
+
+    # do the download thing if we have any -download options
+    if (scalar (@download)) {
+	# remove any existing import data
+	if (&has_content ($importdir)) {
+	    print $out "build: WARNING: removing contents of $importdir\n";
+	    &util::rm_r ($importdir);
+	}
+	
+	foreach $download_dir (@download) {
+
+	    # remove any leading or trailing whitespace from filenames (just in case)
+	    $download_dir =~ s/^\s+//;
+	    $download_dir =~ s/\s+$//;
+	    
+	    if ($download_dir =~ /^(http|ftp):\/\//) {
+		# use wget to mirror http or ftp urls
+		# options used are:
+		#  -P = the directory to download documents to
+		#  -np = don't ascend to parent directories. this means that only documents
+		#        that live in the same directory or below on the same server as
+		#        the given url will be downloaded
+		#  -nv = not too verbose
+		#  -r = recursively mirror
+		#  -N = use time-stamping to see if an up-to-date local copy of each 
+		#       file already exists. this may be useful if wget fails and 
+		#       is restarted
+		#  -l inf = infinite recursion depth
+		#  -R "*\?*" = don't download cgi based urls
+		#  -o = the output file to write download status to (only used if the -out
+		#       option was given to build)
+
+		my $download_cmd = "perl -S gsWget.pl -P \"$importdir\" -np -nv";
+		$download_cmd .= " -r -N -l inf -R \"*\\?*\"";
+		$download_cmd .= " -o \"$outfile.download\"" if $use_out;
+		$download_cmd .= " \"$download_dir\"";
+		system ($download_cmd);
+
+		# note that wget obeys the robot rules. this means that it will have
+		# downloaded a robots.txt file if one was present. since it's unlikely 
+		# anyone really wants to include it in a collection we'll delete it.
+		# robots.txt shouldn't be more than two directories deep (I think it will 
+		# always be exactly two deep but will look for it in the top directory too)
+		# so that's as far as we'll go looking for it.
+		if (opendir (DIR, $importdir)) {
+		    my @files = readdir DIR;
+		    closedir DIR;
+		    foreach my $file (@files) {
+			next if $file =~ /^\.\.?$/;
+			if ($file =~ /^robots.txt$/i) {
+			    &util::rm (&util::filename_cat ($importdir, $file));
+			    last;
+			} else {
+			    $file = &util::filename_cat ($importdir, $file);
+			    if (-d $file) {
+				if (opendir (DIR, $file)) {
+				    my @files2 = readdir DIR;
+				    closedir DIR;
+				    foreach my $file2 (@files2) {
+					if ($file2 =~ /^robots.txt$/i) {
+					    &util::rm (&util::filename_cat ($file, $file2));
+					    last;
+					}
+				    }
+				}
+			    }
+			}
+		    }
+		}
+
+		# if using output directory append the file download output to it
+		&append_file ($out, "$outfile.download");
+		
+	    } else {
+		# we assume anything not beginning with http:// or ftp://
+		# is a file or directory on the local file system.
+		$download_dir =~ s/^file:(\/\/)?//;
+		$download_dir =~ s/^\s+//; # may be whitespace between "file://" and the rest
+		
+		if (-e $download_dir) {
+		    # copy download_dir and all it contains to the import directory
+		    my $download_cmd = "perl -S filecopy.pl";
+		    $download_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
+		    $download_cmd .= " -out \"$outfile.download\"" if $use_out;
+		    $download_cmd .= " \"" . $download_dir . "\" " . $collection;
+		    system ($download_cmd);
+		    # if using output directory append the file download output to it
+		    &append_file ($out, "$outfile.download");
+		} else {
+		    print $out "WARNING: '$download_dir' does not exist\n";
+		}
+	    }
+	}
+    }
+
+    if (-e &util::filename_cat ($archivedir, "archives.inf")) {
+	if (&has_content ($importdir)) {
+	    if ($buildtype eq "build") {
+		&gsdl_build();
+	    } else {
+		&gsdl_import();
+		&gsdl_build();
+	    }
+	} else {
+	    # there are archives but no import, build directly from archives
+	    print $out "build: no import material was found, building directly\n";
+	    print $out "       from archives\n";
+	    &gsdl_build();
+	}
+    } else {
+	if (&has_content ($importdir)) {
+	    if ($buildtype eq "build") {
+		print $out "build: can't build directly from archives as no\n";
+		print $out "       imported archives exist (did you forget to\n";
+		print $out "       move the contents of $collection/import to\n";
+		print $out "       collection/archives?)\n";
+	    }
+	    &gsdl_import();
+	    if (&has_content ($archivedir, "^archives.inf\$")) {
+		&gsdl_build();
+	    } else {
+		my $msg = "build: ERROR: The collection could not be built as no\n";
+		$msg .=   "       valid data was imported. Are at least some of\n";
+		$msg .=   "       the files you imported in a format that can be\n";
+		$msg .=   "       processed by the specified Greenstone plugins?\n";
+		print $out $msg;
+		&log_event ($msg);
+		&final_out (6) if $use_out;
+		die "\n";
+	    }
+	} else {
+	    # no import or archives
+	    my $msg = "build: ERROR: The collection could not be built as it contains no data.\n";
+	    print $out $msg;
+	    &log_event ($msg);
+	    &final_out (1) if $use_out;
+	    die "\n";
+	}
+    }
+
+    if ($collectdir ne "" && !$dontinstall) {
+	my $install_collectdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect");
+	if (!&util::filenames_equal ($collectdir, $install_collectdir)) {
+	    
+	    # install collection to gsdl/collect
+	    print $out "installing the $collection collection\n";
+	    my $newdir = &util::filename_cat ($install_collectdir, $collection);
+	    my $olddir = &util::filename_cat ($collectdir, $collection);
+	    if (-d $newdir) {
+		my $msg = "build: Could not install collection as $newdir\n" .
+		    "       already exists. Collection will remain at\n$olddir\n";
+
+		print $out $msg;
+		&log_event ($msg);
+		&final_out (4) if $use_out;
+		die "\n";
+	    }
+	    if (!&File::Copy::move ($olddir, $newdir)) {
+		my $msg = "build: Failed to install collection to $newdir\n" .
+		    "       Collection will remain at $olddir\n";
+		print $out $msg;
+		&log_event ($msg);
+		&final_out (5) if $use_out;
+		die "\n";
+	    }
+	}
+    }
+
+    &log_event ("The $collection collection was built successfully\n");
+    &final_out (0) if $use_out;
+}
+
+sub gsdl_import {
+
+    print $out "importing the $collection collection\n\n";
+
+    my $import_cmd = "perl -S import.pl";
+    $import_cmd .= " -out \"$outfile.import\"" if $use_out;
+    $import_cmd .= " -removeold" unless $append;
+    $import_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
+    $import_cmd .= " -statsfile \"$statsfile\"" if $statsfile =~ /\w/;
+    $import_cmd .= " $maxdocs $collection";
+
+    system ($import_cmd);
+    # if using output directory append the import output to it
+    &append_file ($out, "$outfile.import");
+
+    if (-e &util::filename_cat ($archivedir, "archives.inf")) {
+	print $out "$collection collection imported successfully\n\n";
+	if ($remove_import) {
+	    print $out "removing import directory ($importdir)\n";
+	    &util::rm_r ($importdir);
+	}
+    } else {
+	my $msg = "build: ERROR: import.pl failed\n";
+	print $out "\n$msg";
+	&log_event ($msg);
+	&final_out (2) if $use_out;
+	die "\n";
+    }
+}
+
+sub gsdl_build {
+
+    print $out "building the $collection collection\n\n";
+
+    my $build_cmd = "perl -S buildcol.pl";
+    $build_cmd .= " -out \"$outfile.build\"" if $use_out;
+    $build_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
+    $build_cmd .= " $maxdocs $collection";
+    system ($build_cmd);
+    # if using output directory append the buildcol output to it
+    &append_file ($out, "$outfile.build");
+
+    if (-e &util::filename_cat ($buildingdir, "text", "$collection.ldb") ||
+	-e &util::filename_cat ($buildingdir, "text", "$collection.bdb")) {
+	print $out "$collection collection built successfully\n\n";
+	if ($remove_archives) {
+	    print $out "removing archives directory ($archivedir)\n";
+	    &util::rm_r ($archivedir);
+	}
+    } else {
+	my $msg = "build: ERROR: buildcol.pl failed\n";
+	print $out "\n$msg";
+	&log_event ($msg);
+	&final_out (3) if $use_out;
+	die "\n";
+    }
+
+    # replace old indexes with new ones
+    if (&has_content ($indexdir)) {
+	print $out "removing old indexes\n";
+	&util::rm_r ($indexdir);
+    }
+    rmdir ($indexdir) if -d $indexdir;
+    &File::Copy::move ($buildingdir, $indexdir);
+
+    # remove the cached arhives
+    if ($save_archives && -d "${archivedir}.org") {
+	&util::rm_r ("${archivedir}.org");
+    }
+}
+
+# return 1 if $dir directory contains any files or sub-directories (other
+# than those specified in the $ignore regular expression)
+sub has_content {
+    my ($dir, $ignore) = @_;
+
+    if (!-d $dir) {return 0;}
+    
+    opendir (DIR, $dir) || return 0;
+    my @files = readdir DIR;
+    closedir DIR;
+    
+    foreach my $file (@files) {
+	if ($file !~ /^\.{1,2}$/) {
+	    return 1 unless (defined $ignore && $file =~ /$ignore/);
+	}
+    }
+    return 0;
+}
+
+sub append_file {
+    my ($handle, $file) = @_;
+
+    open (FILE, $file) || return;
+    undef $/;
+    print $handle <FILE>;
+    $/ = "\n";
+    close FILE;
+    &util::rm ($file);
+}
+
+# creates a file called $outfile.final and writes an output code to it.
+# An output code of 0 specifies that there was no error
+sub final_out {
+    my ($exit_code) = @_;
+
+    if ($use_out && (!-e "$outfile.final")) {
+
+	if (open (FINAL, ">$outfile.final")) {
+	    print FINAL $exit_code;
+	    close FINAL;
+	}
+    }
+}
+
+sub log_event {
+    my ($msg) = @_;
+
+    return unless ($log_events || $email_events);
+
+    # get the event header
+    my $eheader = "[Build Event]\n";
+    $eheader .= "Date: " . scalar localtime() . "\n";
+    if ($event_header ne "" && open (HEADER, $event_header)) {
+	undef $/;
+	$eheader .= <HEADER>;
+	$/ = "\n";
+	close HEADER;
+    } else {
+	$eheader .= "Collection: $collection\n";
+	$eheader .= "GSDLHOME: $ENV{'GSDLHOME'}\n";
+	$eheader .= "Build Location: $collectdir\n";
+    }
+    
+    if ($log_events) {
+	my $fail = 0;
+	# append the event to the event log file
+	if ($event_log_file eq "" || !open (LOG, ">>$event_log_file")) {
+	    # log file defaults to $GSDLHOME/etc/events.txt
+	    $event_log_file = &util::filename_cat ($ENV{'GSDLHOME'}, "etc", "events.txt");
+	    if (!open (LOG, ">>$event_log_file")) {
+		print $out "build: ERROR: Couldn't open event log file $event_log_file\n";
+		$fail = 1;
+	    }
+	}
+	if (!$fail) {
+	    print LOG $eheader;
+	    print LOG $msg;
+	    print LOG "\n";
+	    close LOG;
+	}
+    }
+    
+    if ($email_events) {
+	# if mail_server isn't set email_events does nothing
+	if ($mail_server eq "") {
+	    print $out "build: WARNING: mail_server was not set - email_events option was ignored\n";
+	    return;
+	}
+	
+	my %mail = ('SMTP' => $mail_server, 
+		    'To' => $email_events,
+		    'From' => $maintainer,
+		    'Subject' => 'Greenstone Build Event'
+		    );
+	$mail{'Message'} = $eheader . $msg;
+	
+	if (!sendmail %mail) {
+	    print $out "build: ERROR sending mail to $email_events\n";
+	    print $out "'$Mail::Sendmail::error'\n";
+	}
+    }
+}
+
+
+sub parse_args {
+    my ($argref) = @_;
+
+    if (!parsargv::parse($argref, 
+			 'optionfile/.*/', \$optionfile,
+			 'append', \$append,
+			 'remove_archives', \$remove_archives,
+			 'remove_import', \$remove_import,
+			 'buildtype/^(build|import)$/import', \$buildtype,
+			 'maxdocs/^\-?\d+/-1', \$maxdocs,
+			 'download/.+', \@download,
+			 'collectdir/.*/', \$collectdir,
+			 'dontinstall', \$dontinstall,
+			 'save_archives', \$save_archives,
+			 'out/.*/STDERR', \$out,
+			 'make_writable', \$make_writable,
+			 'log_events', \$log_events,
+			 'event_log_file/.*/', \$event_log_file,
+			 'email_events/.*/', \$email_events,
+			 'mail_server/.*/', \$mail_server,
+			 'statsfile/.*/STDERR', \$statsfile,
+			 'event_header/.*/', \$event_header)) {
+	
+	&print_usage();
+	die "\n";
+    }
+}
+
+
+END {
+
+    if ($make_writable) {
+	# chmod a+rw new collection
+	my $installed_collection = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection);
+	&recursive_chmod($installed_collection);
+
+	# chmod a+rw anything we've left laying about in the tmp directory
+	if (($collectdir ne "") && 	
+	    (!&util::filenames_equal ($collectdir, &util::filename_cat($ENV{'GSDLHOME'}, "collect")))) {
+	    &recursive_chmod($collectdir);
+	}
+    }
+
+    # this will produce a .final file if one doesn't exist yet - that
+    # should only happen if there's been an error somewhere in the perl
+    # code
+    &final_out(7);
+
+    sub recursive_chmod {
+	my ($dir) = @_;
+	return unless -d $dir;
+
+	chmod (0777, $dir);
+
+	opendir (DIR, $dir) || die;
+	my @files = readdir DIR;
+	closedir DIR;
+	
+	foreach my $file (@files) {
+	    next if $file =~ /^\.\.?$/;
+	    $file = &util::filename_cat($dir, $file);
+	    if (-d $file) {
+		&recursive_chmod ($file);
+	    } else {
+		chmod (0777, $file);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl/bin/script/buildcol.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/buildcol.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/buildcol.pl	(revision 21112)
@@ -0,0 +1,548 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# buildcol.pl -- This program will build a particular collection
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# 11/04/03 Added usage datastructure - John Thompson
+
+package buildcol;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use colcfg;
+use util;
+use scriptutil;
+use FileHandle;
+use gsprintf;
+use printusage;
+use parse2;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and vice versa
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+my $mode_list =
+    [ { 'name' => "all",
+        'desc' => "{buildcol.mode.all}" },
+      { 'name' => "compress_text",
+        'desc' => "{buildcol.mode.compress_text}" },
+      { 'name' => "build_index",
+        'desc' => "{buildcol.mode.build_index}" },
+      { 'name' => "infodb",
+        'desc' => "{buildcol.mode.infodb}" } ];
+
+my $sec_index_list = 
+    [ {'name' => "never",
+       'desc' => "{buildcol.sections_index_document_metadata.never}" },
+      {'name' => "always",
+       'desc' => "{buildcol.sections_index_document_metadata.always}" },
+      {'name' => "unless_section_metadata_exists",
+       'desc' => "{buildcol.sections_index_document_metadata.unless_section_metadata_exists}" }
+      ];
+
+my $arguments =
+    [ { 'name' => "remove_empty_classifications",
+	'desc' => "{buildcol.remove_empty_classifications}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "archivedir",
+	'desc' => "{buildcol.archivedir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "builddir",
+	'desc' => "{buildcol.builddir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+#     { 'name' => "cachedir",
+#	'desc' => "{buildcol.cachedir}",
+#	'type' => "string",
+#	'reqd' => "no" },
+      { 'name' => "collectdir",
+	'desc' => "{buildcol.collectdir}",
+	'type' => "string",
+	# parsearg left "" as default
+	#'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "create_images",
+	'desc' => "{buildcol.create_images}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "debug",
+	'desc' => "{buildcol.debug}",
+	'type' => "flag",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "faillog",
+	'desc' => "{buildcol.faillog}",
+	'type' => "string",
+	# parsearg left "" as default
+	#'deft' => &util::filename_cat("<collectdir>", "colname", "etc", "fail.log"),
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "index",
+	'desc' => "{buildcol.index}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "keepold",
+	'desc' => "{buildcol.keepold}",
+	'type' => "flag",
+	'reqd' => "no",
+        'modegli' => "3" },
+      { 'name' => "removeold",
+	'desc' => "{buildcol.removeold}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "maxdocs",
+	'desc' => "{buildcol.maxdocs}",
+	'type' => "int",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "mode",
+	'desc' => "{buildcol.mode}",
+	'type' => "enum",
+	'list' => $mode_list,
+	# parsearg left "" as default
+#	'deft' => "all",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "no_strip_html",
+	'desc' => "{buildcol.no_strip_html}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "no_text",
+	'desc' => "{buildcol.no_text}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "sections_index_document_metadata",
+	'desc' => "{buildcol.sections_index_document_metadata}",
+	'type' => "enum",
+	'list' => $sec_index_list,
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "out",
+	'desc' => "{buildcol.out}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "verbosity",
+	'desc' => "{buildcol.verbosity}",
+	'type' => "int",
+	# parsearg left "" as default
+	#'deft' => "2",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "gli",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" } ];
+
+my $options = { 'name' => "buildcol.pl",
+		'desc' => "{buildcol.desc}",
+		'args' => $arguments };
+
+
+# globals
+my $collection;
+my $configfilename;
+my $out;
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+
+&main();
+
+sub main
+{
+    # command line args
+    my ($verbosity, $archivedir, $cachedir, $builddir, $maxdocs, 
+	$debug, $mode, $indexname, $removeold, $keepold, $remove_empty_classifications,
+	$create_images, $collectdir, $build, $type, $textindex,
+	$no_strip_html, $no_text, $faillog, $gli, $index, $language,
+	$sections_index_document_metadata);
+
+    my $xml = 0;
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # general options available to all plugins
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+    # If there are more than one argument left after parsing, it mean user input too many arguments.
+    # Error occoured  will return 0
+    if($intArgLeftinAfterParsing > 1)
+    {
+	$blnParseFailed = "true"; 
+    }
+    if($blnParseFailed eq "true")
+    {
+	print "";
+	&PrintUsage::print_txt_usage($options, "{buildcol.params}");
+	die "\n";
+    }
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language && $language =~ /\S/) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	print "\n";
+	return;
+    }
+
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+
+    $textindex = "";
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") ||
+	    (&gsprintf(STDERR, "{common.cannot_open_output_file}\n", $out) && die);
+	$out = "buildcol::OUT";
+	$close_out = 1;
+    }
+    $out->autoflush(1);
+
+    # get and check the collection
+    if (($collection = &util::use_collection(@ARGV, $collectdir)) eq "") {
+	&PrintUsage::print_txt_usage($options, "{buildcol.params}");
+	die "\n";
+    }
+
+    if ($faillog eq "") {
+	$faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
+    }
+    # note that we're appending to the faillog here (import.pl clears it each time)
+    # this could potentially create a situation where the faillog keeps being added 
+    # to over multiple builds (if the import process is being skipped)
+    open (FAILLOG, ">>$faillog") ||
+	(&gsprintf(STDERR, "{common.cannot_open_fail_log}\n", $faillog) && die);
+    $faillog = 'buildcol::FAILLOG';
+    $faillog->autoflush(1);
+
+    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
+
+    # read the configuration file
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+    my ($collectcfg, $buildtype); 
+
+    if (!-e $configfilename) {
+	&gsprintf($out, "{common.cannot_find_cfg_file}\n", $configfilename) && die;
+    }
+    
+    $collectcfg = &colcfg::read_collect_cfg ($configfilename);
+    
+    if ($verbosity !~ /\d+/) {
+	if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
+	    $verbosity = $collectcfg->{'verbosity'};
+	} else {
+	    $verbosity = 2; # the default
+	}
+    }
+    # we use searchtype for determining buildtype, but for old versions, use buildtype
+    if (defined $collectcfg->{'buildtype'}) {
+	$buildtype = $collectcfg->{'buildtype'};
+    } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
+	$buildtype = "mgpp";
+    } else {
+	$buildtype = "mg"; #mg is the default
+    }
+    if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	$archivedir = $collectcfg->{'archivedir'};
+    }
+    if (defined $collectcfg->{'cachedir'} && $cachedir eq "") {
+	$cachedir = $collectcfg->{'cachedir'};
+    }
+    if (defined $collectcfg->{'builddir'} && $builddir eq "") {
+	$builddir = $collectcfg->{'builddir'};
+    }
+    if ($maxdocs !~ /\-?\d+/) {
+	if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
+	    $maxdocs = $collectcfg->{'maxdocs'};
+	} else {
+	    $maxdocs = -1; # the default
+	}
+    }
+    if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
+	$debug = 1;
+    }
+    if ($mode !~ /^(all|compress_text|build_index|infodb)$/) {
+	if (defined $collectcfg->{'mode'} && $collectcfg->{'mode'} =~ /^(all|compress_text|build_index|infodb)$/) {
+	    $mode = $collectcfg->{'mode'};
+	} else {
+	    $mode = "all"; # the default
+	}
+    }
+    if (defined $collectcfg->{'index'} && $indexname eq "") {
+	$indexname = $collectcfg->{'index'};
+    }
+    if (defined $collectcfg->{'no_text'} && $no_text == 0) {
+	if ($collectcfg->{'no_text'} =~ /^true$/i) {
+	    $no_text = 1;
+	}
+    }
+    if (defined $collectcfg->{'no_strip_html'} && $no_strip_html == 0) {
+	if ($collectcfg->{'no_strip_html'} =~ /^true$/i) {
+	    $no_strip_html = 1;
+	}
+    }
+    if (defined $collectcfg->{'remove_empty_classifications'} && $remove_empty_classifications == 0) {
+	if ($collectcfg->{'remove_empty_classifications'} =~ /^true$/i) {
+	    $remove_empty_classifications = 1;
+	}
+    }
+    
+    if (defined $collectcfg->{'create_images'} && $collectcfg->{'create_images'} =~ /^true$/i) {
+	$create_images = 1;
+    }
+    if ($buildtype eq "mgpp" && defined $collectcfg->{'textcompress'}) {
+	$textindex = $collectcfg->{'textcompress'};
+    }
+    if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
+	$gli = 1;
+    }
+
+    if ($sections_index_document_metadata !~ /\S/ && defined $collectcfg->{'sections_index_document_metadata'}) {
+	$sections_index_document_metadata = $collectcfg->{'sections_index_document_metadata'};
+    }
+    
+    if ($sections_index_document_metadata !~ /^(never|always|unless_section_metadata_exists)$/) {
+	$sections_index_document_metadata = "never";
+    }
+    
+    ($removeold, $keepold) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, "building", $collectcfg);
+ 
+    $gli = 0 unless defined $gli;
+
+    print STDERR "<Build>\n" if $gli;
+
+    #set the text index
+    if (($buildtype eq "mgpp") || ($buildtype eq "lucene")) {
+	if ($textindex eq "") {
+	    $textindex = "text";
+	}
+    }
+    else {
+	$textindex = "section:text";
+    }
+
+    # create default images if required
+    if ($create_images) {
+	my $collection_name = $collection;
+	$collection_name = $collectcfg->{'collectionmeta'}->{'collectionname'}->{'default'}
+	if defined $collectcfg->{'collectionmeta'}->{'collectionname'}->{'default'};
+	&create_images ($collection_name);
+    }
+
+    # fill in the default archives and building directories if none
+    # were supplied, turn all \ into / and remove trailing /
+
+    my ($realarchivedir, $realbuilddir);  
+    $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives") if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+    $builddir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "building") if $builddir eq "";
+    $builddir =~ s/[\\\/]+/\//g;
+    $builddir =~ s/\/$//;
+
+    # update the archive cache if needed
+    if ($cachedir) {
+	&gsprintf($out, "{buildcol.updating_archive_cache}\n")
+	    if ($verbosity >= 1);
+
+	$cachedir =~ s/[\\\/]+$//;
+	$cachedir .= "/collect/$collection" unless 
+	    $cachedir =~ /collect\/$collection/;
+
+	$realarchivedir = "$cachedir/archives";
+	$realbuilddir = "$cachedir/building";
+	&util::mk_all_dir ($realarchivedir);
+	&util::mk_all_dir ($realbuilddir);
+	&util::cachedir ($archivedir, $realarchivedir, $verbosity);
+
+    } else {
+	$realarchivedir = $archivedir;
+	$realbuilddir = $builddir;
+    }
+
+    # build it in realbuilddir
+    &util::mk_all_dir ($realbuilddir);
+
+    my ($buildertype, $builderdir,  $builder);
+    # if a builder class has been created for this collection, use it
+    # otherwise, use the mg or mgpp builder
+    if (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}builder.pm") {
+	$builderdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
+	$buildertype = "${collection}builder";
+    } else {
+	$builderdir = "$ENV{'GSDLHOME'}/perllib";
+	if ($buildtype eq "lucene") {
+	    $buildertype = "lucenebuilder";
+	}
+	elsif ($buildtype eq "mgpp") {
+	    $buildertype = "mgppbuilder";
+	}
+	else {
+	    $buildertype = "mgbuilder";
+	}
+    }
+	
+    require "$builderdir/$buildertype.pm";
+
+    eval("\$builder = new $buildertype(\$collection, " .
+	 "\$realarchivedir, \$realbuilddir, \$verbosity, " .
+	 "\$maxdocs, \$debug, \$keepold, \$remove_empty_classifications, " .
+	 "\$out, \$no_text, \$faillog, \$gli)");
+    die "$@" if $@;
+
+    $builder->init();
+
+    if (($buildertype eq "mgppbuilder") && $no_strip_html) {
+	$builder->set_strip_html(0);
+    }
+    if ($sections_index_document_metadata ne "never") {
+	$builder->set_sections_index_document_metadata($sections_index_document_metadata);
+    }
+
+    if ($mode =~ /^all$/i) {
+	$builder->compress_text($textindex);
+	$builder->build_indexes($indexname);
+	$builder->make_infodatabase();
+	$builder->collect_specific();
+    } elsif ($mode =~ /^compress_text$/i) {
+	$builder->compress_text($textindex);
+    } elsif ($mode =~ /^build_index$/i) {
+	$builder->build_indexes($indexname);	
+    } elsif ($mode =~ /^infodb$/i) {
+	$builder->make_infodatabase();
+    } else {
+	(&gsprintf(STDERR, "{buildcol.unknown_mode}\n", $mode) && die);
+    }
+
+    $builder->make_auxiliary_files() if !$debug;
+    $builder->deinit();
+    
+    if (($realbuilddir ne $builddir) && !$debug) {
+	&gsprintf($out, "{buildcol.copying_back_cached_build}\n")
+	    if ($verbosity >= 1);
+	&util::rm_r ($builddir);
+	&util::cp_r ($realbuilddir, $builddir);
+    }
+
+    close OUT if $close_out;
+    close FAILLOG;
+
+    print STDERR "</Build>\n" if $gli;
+}
+
+sub create_images {
+    my ($collection_name) = @_;
+
+    my $image_script = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "script", "gimp", "title_icon-1.2.pl");
+    if (!-e $image_script) {
+	&gsprintf($out, "{buildcol.no_image_script}", $image_script);
+	&gsprintf($out, "{buildcol.no_default_images}\n\n");
+	return;
+    }
+
+    my $imagedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "images");
+
+    &util::mk_all_dir ($imagedir);
+
+    # create the images
+    system ("$image_script -size 1.5 -image_dir \"$imagedir\" -filename $collection.gif -text \"$collection_name\"");
+    system ("$image_script -image_dir \"$imagedir\" -filename ${collection}sm.gif -text \"$collection_name\"");
+
+    # update the collect.cfg configuration file (this will need
+    # to be changed when the config file format changes)
+    if (!open (CFGFILE, $configfilename)) {
+	&gsprintf($out, "{buildcol.cannot_open_cfg_file}\n", $configfilename);
+	&gsprintf($out, "{buildcol.unlinked_col_images}\n");
+	return;
+    }
+
+    my $line = ""; my $file = "";
+    my $found = 0; my $foundsm = 0;
+    while (defined ($line = <CFGFILE>)) {
+	if ($line =~ /collectionmeta\s+iconcollection\s+/) {
+	    $line = "collectionmeta iconcollection _httpprefix_/collect/$collection/images/$collection.gif\n";
+	    $found = 1;
+	} elsif ($line =~ /collectionmeta\s+iconcollectionsmall\s+/) {
+	    $line = "collectionmeta iconcollectionsmall _httpprefix_/collect/$collection/images/${collection}sm.gif\n";
+	    $foundsm = 1;
+	}
+	$file .= $line;
+    }
+    close CFGFILE;
+
+    $file .= "collectionmeta iconcollection _httpprefix_/collect/$collection/images/$collection.gif\n" if !$found;
+    $file .= "collectionmeta iconcollectionsmall _httpprefix_/collect/$collection/images/${collection}sm.gif\n" if !$foundsm;
+
+    if (!open (CFGFILE, ">$configfilename")) {
+	&gsprintf($out, "{buildcol.cannot_open_cfg_file}\n", $configfilename);
+	&gsprintf($out, "{buildcol.unlinked_col_images}\n");
+	return;
+    }
+    print CFGFILE $file;
+    close CFGFILE;
+}
Index: /main/tags/3.01/gsdl/bin/script/cancel_build.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/cancel_build.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/cancel_build.pl	(revision 21112)
@@ -0,0 +1,123 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# cancel_build.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will create a .kill file within the GSDLCOLLECTDIR/collect-name
+# directory which will cause any import.pl or buildcol.pl processes running
+# on that collection to abort.
+# It then cleans any mess left laying around the collection (including the
+# import directory !!).
+# cancel_build.pl is really only intended to be called from within the 
+# library's collectoraction and isn't much use for anything else. As such it
+# should be rewritten in C++ one day and included in the collectoraction 
+# itself.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use util;
+use parsargv;
+use File::Copy;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "cancel_build.pl: Cancel a build in progress. This script is\n";
+    print STDERR "                 called from the collector and is not intended\n";
+    print STDERR "                 for general use.\n\n";
+    print STDERR "  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -collectdir directory  Collection directory (defaults to " .
+	&util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n\n";
+}
+
+&main();
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 'collectdir/.*/', \$collectdir)) {
+	&print_usage();
+	die "\n";
+    }
+
+    my $collection = pop @ARGV;
+    if (!defined $collection || $collection !~ /\w/) {
+	print STDERR "no collection specified\n";
+	&print_usage();
+	die "\n";
+    }
+
+    my $cdir = &util::filename_cat ($collectdir, $collection);
+    if ($collectdir eq "") {
+	$collectdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect");
+	$cdir = &util::filename_cat ($collectdir, $collection);
+    } elsif (!-d $cdir) {
+	$cdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $collection);
+    }
+
+    if (! -d $cdir) {
+	print STDERR "Collection does not exist ($cdir)\n";
+	exit (1);
+    }
+
+    # create .kill file
+    my $killfile = &util::filename_cat ($cdir, ".kill");
+    if (!open (KILLFILE, ">$killfile")) {
+	print STDERR "Couldn't create .kill file ($killfile)\n";
+	exit (1);
+    }
+    print KILLFILE "kill $collection\n";
+    close KILLFILE;
+
+    sleep (2); # just give it a chance to die gracefully;
+
+    # remove archives, building, and import directories
+    &util::rm_r (&util::filename_cat ($cdir, "import"));
+    &util::rm_r (&util::filename_cat ($cdir, "building"));
+    &util::rm_r (&util::filename_cat ($cdir, "archives"));
+
+    # remove any other temporary files that may have been left laying around
+    # by the build
+    my $buildfile = &util::filename_cat ($collectdir, ".build");
+    &util::rm ($buildfile) if -e $buildfile;
+    my $bldfile = &util::filename_cat ($collectdir, "$collection.bld");
+    &util::rm ($bldfile) if -e $bldfile;
+    my $bldfile_d = &util::filename_cat ($collectdir, "$collection.bld.download");
+    &util::rm ($bldfile_d) if -e $bldfile_d;
+    my $bldfile_i = &util::filename_cat ($collectdir, "$collection.bld.import");
+    &util::rm ($bldfile_i) if -e $bldfile_i;
+    my $bldfile_b = &util::filename_cat ($collectdir, "$collection.bld.build");
+    &util::rm ($bldfile_b) if -e $bldfile_b;
+    my $bldfile_f = &util::filename_cat ($collectdir, "$collection.bld.final");
+    &util::rm ($bldfile_f) if -e $bldfile_f;
+    
+    # if there's an archives.org directory, rename it back to archives
+    if (-d &util::filename_cat ($cdir, "archives.org")) {
+	&File::Copy::move (&util::filename_cat ($cdir, "archives.org"),
+			   &util::filename_cat ($cdir, "archives"));
+    }
+}
Index: /main/tags/3.01/gsdl/bin/script/classinfo.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/classinfo.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/classinfo.pl	(revision 21112)
@@ -0,0 +1,170 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# classinfo.pl -- provide information about classifiers
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+use strict;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use classify;
+use util;
+use gsprintf;
+use printusage;
+
+use parse2;
+
+my $arguments =
+    [ { 'name' => "collect",
+	'desc' => "{classinfo.collect}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "listall",
+	'desc' => "{scripts.listall}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no" } ];
+
+my $options = { 'name' => "classinfo.pl",
+		'desc' => "{classinfo.desc}",
+		'args' => $arguments };
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+sub main {
+    my $collect = "";
+    my $xml = 0;
+    my $listall = 0;
+    my $language;
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # general options available to all plugins
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+    # If there are more than one argument left after parsing, it mean user input too many arguments.
+    # Error occoured  will return 0
+    if($intArgLeftinAfterParsing > 1)
+    {
+	&PrintUsage::print_txt_usage($options, "{classinfo.params}");
+	die "\n";
+    }
+
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    # Get classifier
+    my $classifier = shift (@ARGV);
+    if (defined $classifier) {
+	$classifier =~ s/\.pm$//; # allow xxx.pm as the argument
+    }
+    if (($listall == 0) && (!defined $classifier || $classifier eq "")) {
+	&gsprintf(STDERR, "{classinfo.no_classifier_name}\n\n");
+	&PrintUsage::print_txt_usage($options, "{classinfo.params}", 1);
+	die "\n";
+    }
+
+    # make sure the classifier is loaded from the correct location - a hack.
+    if ($collect ne "") {
+	$ENV{'GSDLCOLLECTDIR'} = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $collect);
+    } else {
+	$ENV{'GSDLCOLLECTDIR'} = $ENV{'GSDLHOME'};
+    }
+ 
+    if ($listall) {
+	my $class_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib","classify");
+	
+	if (!opendir (INDIR, $class_dir)) {
+	    print STDERR "classinfo.pl: could not open directory $class_dir\n";
+	} else {
+	    my @class_list = grep (/\.pm$/, readdir (INDIR));
+	    closedir (INDIR);
+
+	    if ($xml) {
+		my $num_class = scalar(@class_list);
+
+		print STDERR "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+		print STDERR "<!DOCTYPE ClassifyList [\n";
+		print STDERR "  <!ELEMENT ClassifyList (ClassifyName*)>\n";
+		print STDERR "  <!ELEMENT ClassifyName (#PCDATA)>\n";
+		print STDERR "  <!ATTLIST ClassifyList\n";
+		print STDERR "            length CDATA #REQUIRED>\n";
+		print STDERR "]>\n";
+
+		print STDERR "<ClassifyList length=\"$num_class\">\n";
+		map { print STDERR "  <ClassifyName>$_</ClassifyName>\n"; } @class_list;
+		print STDERR "</ClassifyList>\n";
+
+	    }
+	    else {
+		print STDERR join(" ",@class_list), "\n";
+	    }
+	}
+
+    }
+    else {
+	my $classobj = &classify::load_classifier_for_info ($classifier);
+	if ($xml) {
+	    $classobj->print_xml_usage();
+	}
+	else {
+	    &gsprintf(STDERR, "\n{classinfo.passing_options}\n\n");
+	    &gsprintf(STDERR, "{classinfo.option_types}:\n\n");
+	    &gsprintf(STDERR, "{classinfo.specific_options}\n\n");
+	    &gsprintf(STDERR, "{classinfo.general_options}\n\n");
+	    &gsprintf(STDERR, "$classifier {classinfo.info}:\n\n");
+	    
+	    $classobj->print_txt_usage();
+	}
+    }
+}
+
+
+&main ();
Index: /main/tags/3.01/gsdl/bin/script/cleantmp.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/cleantmp.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/cleantmp.pl	(revision 21112)
@@ -0,0 +1,80 @@
+#!/usr/bin/perl
+
+###########################################################################
+#
+# cleantmp.pl
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# cleantmp.pl cleans up the gsdl/tmp directory
+# it's intended mainly for use by the collectoraction
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "cleantmp.pl: Cleans up the Greenstone tmp directory by deleting anything\n";
+    print STDERR "             more than -expire_days days old.\n\n";
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -expire_days number   The number of days old information in the tmp\n";
+    print STDERR "                         directory must be before it is removed (defaults\n";
+    print STDERR "                         to 7 days)\n\n";
+}
+
+&main ();
+
+sub main {
+    
+    if (!parsargv::parse(\@ARGV, 'expire_days/\d/7', \$expire_days)) {
+	&print_usage();
+	die "\n";
+    }
+
+    my $current_time = time;
+    my $expire_time = ($current_time - ($expire_days*60*60*24));
+
+    my $tmpdir = &util::filename_cat ($ENV{'GSDLHOME'}, "tmp");
+
+    opendir (TMP, $tmpdir) || die "cleantmp.pl: couldn't open $tmpdir\n";
+    my @files = readdir TMP;
+    closedir TMP;
+
+    foreach $file (@files) {
+	next if ($file =~ /^\.\.$/);
+
+	if ($file =~ /^tbuild/) {
+	    my $thisfile = &util::filename_cat ($tmpdir, $file);
+	    my @stats = stat ($thisfile);
+	    if ($stats[9] < $expire_time) {
+		&util::rm_r ($thisfile);
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl/bin/script/convert_collection.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/convert_collection.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/convert_collection.pl	(revision 21112)
@@ -0,0 +1,112 @@
+#! /usr/bin/perl -w
+
+# converts an old style humanity collection which uses an index.txt file to
+# use a single metadata.xml file instead
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use util;
+use cfgread;
+
+my $collection = $ARGV[0];
+my $collectdir = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection);
+my $importdir = &util::filename_cat($collectdir, "import");
+
+die unless -d $importdir;
+
+# new import structure will be created in "import.new" directory
+my $importnewdir = $importdir . ".new";
+`mkdir $importnewdir`;
+
+
+# read in index.txt file and generate metadata.xml, in the process
+# converting the html files and copying them across to the import.new
+# directory
+
+my $metadata_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n";
+$metadata_xml .= "<!DOCTYPE DirectoryMetadata SYSTEM " .
+    "\"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">\n";
+$metadata_xml .= "<DirectoryMetadata>\n\n";
+
+my $index_txt = &util::filename_cat($importdir, "index.txt");
+
+open (INDEXTXT, $index_txt) || die;
+
+my $line = [];
+my @fields = ();
+my $count = 0;
+while (defined ($line = cfgread::read_cfg_line("main::INDEXTXT"))) {
+
+#    last if $count > 10;
+
+    if ($line->[0] eq "key:") {
+	shift @$line;
+	@fields = @$line;
+    } else {
+	
+	my $jobnumber = shift @$line;
+	&new_document($jobnumber);
+	$count ++;
+
+	my $i = 0;
+	for ($i = 0; $i < scalar(@$line); $i++) {
+	    if ($line->[$i] =~ /^<([^>]+)>(.*)$/) {
+		&set_metadata($1, $2);
+	    } else {
+		if (defined ($fields[$i])) {
+		    &set_metadata($fields[$i], $line->[$i]);
+		} else {
+		    print STDERR "error 1\n";
+		}
+	    }
+	}
+
+	$metadata_xml .= "    </Description>\n";
+	$metadata_xml .= "  </FileSet>\n\n";
+    }
+}
+close INDEXTXT;
+
+$metadata_xml .= "</DirectoryMetadata>\n";
+
+my $metafile = &util::filename_cat($importnewdir, "metadata.xml");
+open (META, ">$metafile") || die;
+print META $metadata_xml;
+close META;
+
+
+sub new_document {
+    my ($jobnumber) = @_;
+
+    print STDERR "creating new document ($jobnumber)\n";
+
+    my $docdir = &util::filename_cat($importdir, $jobnumber);
+    die unless -d $docdir;
+
+    # copy whole directory across to import.new
+    $jobnumber =~ s/^.*?\///;
+    my $newdocdir = &util::filename_cat($importnewdir, $jobnumber);
+    die if -e $newdocdir;
+    `cp -r $docdir $newdocdir`;
+
+    # convert the htm file to use the new syntax
+    my $htmfile = &util::filename_cat($newdocdir, "$jobnumber.htm");
+    die unless -e $htmfile;
+    `convert_toc.pl < $htmfile > $htmfile.new`;
+    `mv $htmfile.new $htmfile`;
+
+    # update metadata.xml
+    $metadata_xml .= "  <FileSet>\n";
+    $metadata_xml .= "    <FileName>$jobnumber</FileName>\n";
+    $metadata_xml .= "    <Description>\n";
+}
+
+sub set_metadata {
+    my ($key, $value) = @_;
+
+    $metadata_xml .= "      <Metadata name=\"$key\" mode=\"accumulate\">$value</Metadata>\n";
+}
Index: /main/tags/3.01/gsdl/bin/script/convert_toc.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/convert_toc.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/convert_toc.pl	(revision 21112)
@@ -0,0 +1,124 @@
+#! /usr/bin/perl -w
+
+# convert_toc.pl converts old <<TOC>> marked up files to use the new
+# <Section> syntax (suitable for processing by HTMLPlug -description_tags
+
+use File::Basename;
+
+my $level = 0;
+
+sub main {
+  open STDIN, "<$ARGV[0]" or die "$ARGV[0]: $!\n";
+  open STDOUT, ">$ARGV[1]" or die "$ARGV[1]: $!\n";
+
+  my $dirname = &File::Basename::dirname($ARGV[0]);
+
+  my $content = "";
+  
+  while (<STDIN>) {
+    $_ =~ s/[\cJ\cM]+$/\n/;
+    $content .= $_;
+  }
+
+  # fix images
+  $content =~ s/&lt;&lt;I&gt;&gt;\s*(\w+\.(?:png|jpe?g|gif))\s*(.*?)<\/p>/rename_image_old($1, $2, $dirname)/iegs;
+  $content =~ s/<img\s+src=\"?(\w+\.(?:png|jpe?g|gif))\"?/rename_image($1, $dirname)/iegs;
+
+  # process section title
+  $content =~ s/(?:<P[^>]*>|<BR>)(?:\s*<\/?[^>]*>)*?&lt;&lt;TOC(\d+)&gt;&gt;(.*?)(?=<\/?P[^>]*>|<BR>)/
+      process_toc ($1, $2)/sige;
+
+  # close the remaining sections
+  my $section_ending = "<!--\n";
+  for (my $j = 0; $j < $level; $j++) {
+    $section_ending .= "</Section>\n";
+  }
+  $section_ending .= "-->\n";
+  $content =~ s/(<\/body>)/$section_ending.$1/sie;
+
+  print STDOUT $content;
+  close STDOUT; close STDIN;
+}
+
+sub process_toc {
+  my ($thislevel, $title) = @_;
+  my $toc = '';
+ 
+  $title =~ s/<[^>]+>//sg;
+  $title =~ s/^\s+//s;
+  $title =~ s/\s+$//s;
+  $title =~ s/\n/ /sg;
+
+  $toc .= "\n<!--\n";
+
+  if ($thislevel <= $level) {
+    for (my $i = 0; $i < ($level-$thislevel+1); $i++) {
+	$toc .= "</Section>\n";
+    }
+  }
+
+  $toc .= "<Section>\n".
+	"  <Description>\n".
+	"    <Metadata name=\"Title\">$title</Metadata>\n".
+	"  </Description>\n".
+	"-->\n";
+
+  $level = $thislevel;
+
+  return $toc;
+}
+
+sub rename_image_old {
+    my ($image_name, $following_text, $dirname) = @_;
+
+    &rename_image($image_name, $dirname);
+
+    return "<center><img src=\"$image_name\"><br>$following_text<\/center><\/p>\n";
+}
+
+
+# may need to rename image files if case isn't consistent
+# (i.e. collections prepared on windows may have images named
+# AAA.GIF which are linked into the HTML as aaa.gif)
+sub rename_image {
+    my ($image_name, $dirname) = @_;
+
+
+    if (!-e "$dirname/$image_name") {
+
+	my ($pre, $ext) = $image_name =~ /^([^\.]+)\.(.*)$/;
+	my $image_name_uc = uc($image_name);
+	my $image_name_lc = lc($image_name);
+	my $image_name_ul = uc($pre) . "." . lc($ext);
+	my $image_name_lu = lc($pre) . "." . uc($ext);
+
+	if (-e "$dirname/$image_name_uc") {
+	    print STDERR "renaming $dirname/$image_name_uc --> $dirname/$image_name\n";
+	    rename ("$dirname/$image_name_uc", "$dirname/$image_name");
+
+	} elsif (-e "$dirname/$image_name_lc") {
+	    print STDERR "renaming $dirname/$image_name_lc --> $dirname/$image_name\n";
+	    rename ("$dirname/$image_name_lc", "$dirname/$image_name");
+	
+	} elsif (-e "$dirname/$image_name_ul") {
+	    print STDERR "renaming $dirname/$image_name_ul --> $dirname/$image_name\n";
+	    rename ("$dirname/$image_name_ul", "$dirname/$image_name");
+	
+	} elsif (-e "$dirname/$image_name_lu") {
+	    print STDERR "renaming $dirname/$image_name_lu --> $dirname/$image_name\n";
+	    rename ("$dirname/$image_name_lu", "$dirname/$image_name");
+	
+	} else {
+	    print STDERR "ERROR**** $dirname/$image_name could not be found\n";
+	    if (open (ERROR, ">>error.txt")) {
+		print ERROR "ERROR**** $dirname/$image_name could not be found\n";
+		close ERROR;
+	    }
+	}
+    }
+    return "<img src=\"$image_name\"";
+}
+
+
+
+&main;
Index: /main/tags/3.01/gsdl/bin/script/convert_toc_dir.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/convert_toc_dir.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/convert_toc_dir.pl	(revision 21112)
@@ -0,0 +1,41 @@
+#! /usr/bin/perl -w
+
+# run convert_toc.pl on any html files found in the directory passed in on
+# the command line (or any subdirs).
+
+die "\$GSDLOS not set !\n" if (!defined $ENV{GSDLOS});
+my ($move, $separator);
+if ($ENV{GSDLOS} eq "windows") {
+  $move = "move";
+  $separator = "\\";
+} else {
+  $move = "mv";
+  $separator = "/";
+}
+
+&recurse($ARGV[0]);
+
+sub recurse {
+    my ($dir) = @_;
+
+    if (!-d $dir) {
+	print STDERR "ERROR: $dir isn't a directory\n";
+	return;
+    }
+
+    opendir (DIR, $dir) || die;
+    my @files = readdir DIR;
+    closedir DIR;
+
+    foreach my $file (@files) {
+	next if $file =~ /^\.\.?$/;
+	my $fullpath = "$dir$separator$file";
+	if (-d $fullpath) {
+	    &recurse($fullpath);
+	} elsif ($file =~ /\.html?$/i) {
+	    print STDERR "converting $fullpath\n";
+	    `convert_toc.pl $fullpath $fullpath.new`;
+	    `$move $fullpath.new $fullpath`;
+	}
+    }
+}
Index: /main/tags/3.01/gsdl/bin/script/delcol.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/delcol.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/delcol.pl	(revision 21112)
@@ -0,0 +1,87 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# delcol.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use util;
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "delcol.pl: Deletes (forever) a collection.\n\n";
+    print STDERR "  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -f                     Force deletion of collection without prompting\n\n";
+}
+
+&main();
+
+sub main {
+    my $force = 0;
+    if (!parsargv::parse(\@ARGV, 'f', \$force)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	exit (1);
+    }
+
+    if (!$force) {
+	print STDOUT "Are you sure you want to completely remove the $collection collection?\n";
+	print STDOUT "(the $ENV{'GSDLCOLLECTDIR'} directory will be deleted). [y/n]\n";
+	my $in = <STDIN>;
+	if ($in !~ /^y(es)?$/i) {
+	    print STDOUT "Deletion of $collection collection cancelled\n";
+	    exit (0);
+	}
+    }
+    # delete any temporary files that are laying around
+    my $tmpdir = &util::filename_cat ($ENV{'GSDLHOME'}, "tmp");
+    if (opendir (TMPDIR, $tmpdir)) {
+	my @tmpfiles = readdir (TMPDIR);
+	closedir TMPDIR;
+	map { &util::rm(&util::filename_cat($tmpdir, $_)) if $_ =~ /^$collection/; } @tmpfiles;
+    }
+
+    &util::rm_r ($ENV{'GSDLCOLLECTDIR'});
+
+    # check if everything was deleted successfully
+    if (-d $ENV{'GSDLCOLLECTDIR'}) {
+	print STDERR "\ndelcol.pl WARNING: Not all files in $ENV{'GSDLCOLLECTDIR'} could\n";
+	print STDERR "be deleted\n";
+	exit (2);
+    }
+
+    print STDERR "$collection collection successfully deleted\n";
+    exit (0);
+}
Index: /main/tags/3.01/gsdl/bin/script/expand_macros.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/expand_macros.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/expand_macros.pl	(revision 21112)
@@ -0,0 +1,1267 @@
+#! /usr/bin/perl
+
+##################################################################################
+#                                                                                #
+# expand_macros.pl -- recursively expand greenstone macros / 1080805 - 1050905   #
+#                                                                                #
+# Copyright (C) 2005 Jens Wille <j_wille at gmx.net>                             #
+#                                                                                #
+# This program is free software; you can redistribute it and/or                  #
+# modify it under the terms of the GNU General Public License                    #
+# as published by the Free Software Foundation; either version 2                 #
+# of the License, or (at your option) any later version.                         #
+#                                                                                #
+# This program is distributed in the hope that it will be useful,                #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of                 #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                  #
+# GNU General Public License for more details.                                   #
+#                                                                                #
+# You should have received a copy of the GNU General Public License              #
+# along with this program; if not, write to the Free Software                    #
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.    #
+#                                                                                #
+##################################################################################
+
+#
+# expand_macros.pl reads in specified greenstone macro files and prints the
+# definitions of requested macros and recursively the definitions of macros
+# used within these definitions.
+#
+#
+# FEATURES:
+#
+# - generic:
+#
+#   - additional/collection-specific macro files can be included
+#
+#   - macros set within the server can be included (though this might not be
+#     of much help without reading the respective source file)
+#
+# - "interactive browse mode" where you can select which macro (and from which
+#   package) to display next.
+#
+#   - readline support and persistent history
+#
+#   - interactive commands
+#
+#   - read files from within interactive browse mode
+#
+# - batch mode only:
+#
+#   - search for macros that use certain macros ("reverse" search)
+#
+#   - search for strings (regular expressions) in macro definitions
+#
+#
+# TODO:
+#
+# - add "reverse search" and "string search" for browse mode
+#
+# - handle macro options better (v, c, l, ...?)
+#
+#
+# KNOWN BUGS:
+#
+# - for a sufficiently large file (> 12288 bytes == 12 k) (paged-)read will quit
+#   the program if not scrolled until the end of the file => SIGPIPE: broken pipe!
+#   SOLVED: the PIPE signal will simply be IGNOREd
+#
+
+use strict;
+use warnings;
+
+use Getopt::Long          qw(GetOptions);
+
+use File::Basename        qw(basename dirname);
+use File::Spec::Functions qw(catdir catfile tmpdir);
+
+use IO::Handle            qw(autoflush);
+STDOUT->autoflush(1);
+
+use Term::ReadLine;
+
+
+### progname and version
+
+my $NAME    = basename $0;
+my $VERSION = '0.21';
+
+
+### global patterns
+
+# my understanding of greenstone macro names:
+#   - enclosed in underscores ('_')
+#   - starts with a letter ([:alpha:])
+#   - followed by alphanumeric characters ([:alnum:])
+#     (consequently, it doesn't start with a number and
+#     particularly isn't a macro parameter, i.e. something like _1_)
+#   - also we might need to take care of escaped underscores ('\_')
+#
+#   => does this fit??? (see '<gsdl-source>/src/lib/display.cpp' for details,
+#      in particular the 'displayclass::loadparammacros' method)
+my $MACRO_PATTERN       = '[[:alpha:]][[:alnum:]]*';
+
+# package names: letters
+my $PACKAGE_NAME        = '[[:alpha:]]+';
+
+# package specification: package name followed by a colon (':')
+my $PACKAGE_PATTERN     = $PACKAGE_NAME . ':';
+
+# beginning/end of macro specification: underscore ('_'), not escaped,
+# i.e. not preceded by a backslash ('\')
+# (need to double-escape backslash here!)
+my $MACRO_AFFIX         = '(?<!\\\)_';
+
+# beginning of macro definition: opening curly bracket ('{')
+my $DEFINITION_START    = '\{';
+
+# end of macro definition: closing curly bracket ('{'), not escaped,
+# i.e. not preceded by a backslash ('\')
+# (need to double-escape backslash here!)
+my $DEFINITION_END      = '(?<!\\\)\}';
+
+# package declaration: 'package' (plus package name)
+my $PACKAGE_DECLARATION = 'package';
+
+
+### command line arguments
+
+# variable initialisation and default values
+my %ARG = (
+  'verbose'     => 0,
+  'version'     => 0,
+  'depth'       => 0,
+  'short'       => 0,
+  'reverse'     => 0,
+  'interactive' => 0,
+  'paged'       => 0,
+  'pager'       => $ENV{'PAGER'} || 'less',
+  'histfile'    => catfile($ENV{'GSDLTMPDIR'} || tmpdir, 'expand_macros.hist'),
+  'histsize'    => 100,
+  'macro_dirs'  => [catdir($ENV{'GSDLHOME'}, 'macros')],
+  'source_dir'  => $ENV{'GSDLSOURCE'} || '',
+  'args'        => []
+);
+
+# global vars
+my $TERM         = '';
+my $CURRENT_FILE = '';
+
+# usage information and help text
+my $USAGE = <<HERE_USAGE;
+usage:
+    $NAME [generic-options] [-s] [-d <depth>] [-r] {_[<package>:]<macro>_|<query>} ...
+    $NAME [generic-options] {-b|-i} [-p]
+    $NAME [-h|-?|--help]
+
+    generic options are: [-v] [-e <directory>,...] [-n <version>]
+HERE_USAGE
+
+my $HELP = <<HERE_HELP;
+$NAME: recursively expand greenstone macros (v$VERSION)
+
+$USAGE
+
+generic options:
+
+    -h, -?, --help                   display this help and exit
+    -v,     --verbose                output some extra information/warnings
+
+    {-e|--extra} <directory>,...     paths to extra macro directories, comma-separated list
+                                     [default directory: '$ARG{'macro_dirs'}[0]']
+
+            --source <directory>     path to greenstone source directory, so that macros which are
+                                     set within the server will be included
+                                     [default: '$ARG{'source_dir'}']
+                                     NOTE: you can set an environment variable GSDLSOURCE instead
+
+    {-n|--show-version} <version>    print only macro definitions for specified version (0=graphic/1=text)
+                                     [default: '$ARG{'version'}', set to '-1' for 'all']
+
+    -s,     --short                  short output, i.e. only macro names, no content
+
+
+batch mode:
+
+    {-d|--depth} <depth>             how deep to recurse through macros
+                                     [default: '$ARG{'depth'}', set to '-1' for 'unlimited']
+
+    -r,     --reverse                reverse search, recursively outputs the macros which use the specified macro
+
+    all non-option arguments will be treated as
+      - macro names (denoted by surrounding underscores '_')
+        or
+      - regular expressions to search for in macro definitions (otherwise)
+
+    (you can restrict your macro name query to a certain package by prepending the macro name with '<package-name>:')
+
+
+interactive browse mode:
+
+    -b, -i, --browse                 interactive browse mode, allows you to select what to display next
+
+    -p,     --paged                  data output will be passed to a pager
+                                     [default: '$ARG{'pager'}']
+            --pager <pager>          pass paged output to specified pager instead of above named default
+
+            --histfile <file>        path to history file to keep history between sessions
+                                     [default: '$ARG{'histfile'}']
+            --histsize <num>         maximum number of lines to keep in histfile
+                                     [default: '$ARG{'histsize'}']
+                                     NOTE: in case you don\'t want the history to be stored you may set
+                                           <histfile> to '' or <histsize> to '0'
+                                           (however, this does not remove any existing history files)
+
+
+NOTE: for this script to run your greenstone environment needs to be set up (GSDLHOME set)
+HERE_HELP
+
+my $INTERACTIVE_HELP = <<HERE_HELP;
+$NAME: expand greenstone macros in ***interactive browse mode*** (v$VERSION)
+
+
+usage instructions:
+
+    - commands are equal to command line options, except that they start with a dot '.'
+      (NOTE: not all command line options are available as command, see list below)
+
+    - commands that take an optional argument ([...]) will print their current value if
+      that argument is omitted (you can also use '.c' or '.config' to get a full overview)
+
+    - you can run several commands at once by separating them with semicolons
+
+    - you can quit the program at any time by hitting <ctrl>+d (<ctrl>+z on windows), or
+      by typing '.q', or '.quit'
+
+
+commands:
+
+    .h, .?, .help                    display this help
+
+    .q,     .quit                    exit program
+
+    .                                redisplay current stage
+
+    ..                               return to previous stage (same as leaving empty)
+
+    ..., .t, .top                    return to top to enter new macro name
+
+    {.n|.show-version} [<version>]   print only macro definitions for specified version (0=graphic/1=text)
+                                     [default: '0', set to '-1' for 'all']
+
+    .s,     .short                   short output, i.e. only macro names, no content
+
+    .p,     .paged                   data output will be passed to a pager
+                                     [default: '$ARG{'pager'}']
+            .pager [<pager>]         pass paged output to specified pager instead of above named default
+
+    .r,     .read [<file>]           display the contents of the specified file (by default the last file we came across)
+    .pr,    .paged-read [<file>]     same, but paged (without turning on paged mode permanently)
+
+    .c,     .config                  display current configuration
+HERE_HELP
+
+# allow for bundling of options
+Getopt::Long::Configure ("bundling");
+
+# parse arguments
+GetOptions( 'help|h|?'         => sub { print "$HELP\n"; exit 0 },
+            'verbose|v'        => \$ARG{'verbose'},
+            'only-version|n=i' => \$ARG{'version'},
+            'extra|e=s'        => \@{$ARG{'macro_dirs'}},
+            'source=s'         => \$ARG{'source_dir'},
+            'depth|d=i'        => \$ARG{'depth'},
+            'short|s'          => \$ARG{'short'},
+            'reverse|r'        => \$ARG{'reverse'},
+            'browse|b|i'       => \$ARG{'interactive'},
+            'paged|p'          => \$ARG{'paged'},
+            'pager=s'          => \$ARG{'pager'},
+            'histfile=s'       => \$ARG{'histfile'},
+            'histsize=i'       => \$ARG{'histsize'},
+            '<>'               => sub { push(@{$ARG{'args'}} => @_) } )
+  or die "$USAGE\n";
+
+
+### some sanity checks
+
+# need one of our "actions": batch, query or interactive
+# ("batch" requiring at least one macro name or regexp specified)
+die "$USAGE\n"
+  unless @{$ARG{'args'}} || $ARG{'interactive'};
+
+# need GSDLHOME for default macro directory
+# (does also allow to have the script in gsdl bin path)
+die "GSDLHOME not set! please change into the directory where greenstone has been installed and run/source the appropriate setup script.\n"
+  unless $ENV{'GSDLHOME'};
+
+
+### action!
+
+# build hash of macro information
+my %macro_db = build_db();
+die "macro db empty!\n"
+  unless %macro_db;
+
+unless ($ARG{'interactive'}) {
+  # batch mode
+
+  my $n = 0;
+  foreach my $arg (@{$ARG{'args'}}) {
+    if ($arg =~ s/^$MACRO_AFFIX((?:$PACKAGE_PATTERN)?$MACRO_PATTERN)$MACRO_AFFIX$/$1/) {
+      # macro
+
+      print "*** macro: $arg", ($ARG{'reverse'} ? ' (reverse) ' : ' '), "***\n\n";
+
+      unless ($ARG{'reverse'}) {
+        # "normal" search
+
+        get_macro($arg);
+      }
+      else {
+        # "reverse" search
+
+        # get the macros that use the specified macro
+        my @refs = get_r_macros($arg);
+        print "no macro referencing '$arg'\n\n"
+          unless @refs;
+
+        # now recurse those macros
+        foreach my $m (@refs) {
+          get_macro($m);
+        }
+      }
+    }
+    else {
+      # query
+
+      print "*** query: $arg", ($ARG{'reverse'} ? ' (reverse) ' : ' '), "***\n\n";
+
+      # get the macros that match the specified query
+      my @macros = get_r_macros($arg, 1);
+      print "no matches for '$arg'\n", ($ARG{'short'} ? '' : "\n")
+        unless @macros;
+
+      # now print those macros
+      foreach my $m (@macros) {
+        get_macro($m);
+      }
+    }
+
+    # print separator _between_ requested macros (i.e. everytime but the last time)
+    # (need to add extra newline for short display)
+    print(($ARG{'short'} ? "\n" : ''), '-' x 80, "\n\n")
+      unless ++$n >= @{$ARG{'args'}};
+  }
+}
+else {
+  # interactive browse mode
+
+  # ignore 'broken pipe' error
+  $SIG{'PIPE'} = 'IGNORE';
+
+  # get the pager executable
+  # no need to test if not in "paged" mode
+  get_pager()
+    if $ARG{'paged'};
+
+  # create new Term::ReadLine object
+  $TERM = Term::ReadLine->new($NAME);
+
+  # don't want the prompt underlined
+  $TERM->ornaments(0);
+  # don't want autohistory (can't set autohistory explicitly, so use this "workaround")
+  $TERM->MinLine(undef);
+
+  # restore history
+  readhist();
+
+  # print help hint
+  print <<HERE_HINT;
+entered '$NAME' in ***interactive browse mode*** (v$VERSION)
+[you can get help at any time by typing '.h', '.?', or '.help']
+
+HERE_HINT
+
+  # repeat until explicit exit
+  while (1) {
+    my $macro = prompt("enter macro name (without package specification) [leave empty to quit]\n> ");
+
+    # remove surrounding '_'
+    $macro =~ s/^_//;
+    $macro =~ s/_$//;
+
+    exit 0 unless length $macro;                    # normal exit
+    next   if     $macro eq '0' || $macro eq '-1';  # a command was executed
+
+    # now get all packages for given macro, and begin recursion...
+    recurse_packages($macro);
+  }
+
+  # can't expect anything down here to be executed
+}
+
+END {
+  if ($ARG{'interactive'}) {
+    # save history
+    savehist();
+  }
+}
+
+
+### that's it ;-)
+
+exit 0;
+
+
+### subroutines
+
+# <sub build_db>
+# build hash of macro information ("macro db")
+#
+# hash structure:
+#   macro
+#   -> package
+#      -> {'0=graphic'|'1=text'}
+#         -> 'file'
+#         -> 'line'
+#         -> 'content'
+#
+# usage:
+#   %macro_db = build_db()
+#
+#  => macro_db: returned hash ("macro db")
+#
+sub build_db {
+  my %macro_db = ();
+  my @dm_list  = ();
+  my ($n, $m) = (0, 0);
+
+  # get all macro files (*.dm) from specified directories
+  foreach my $dir (@{$ARG{'macro_dirs'}}) {
+    opendir(DIR, "$dir")
+      or die "can't read macro directory '$dir': $!\n";
+
+    push(@dm_list => map { $_ = catfile($dir, $_) } grep { /\.dm$/ } readdir(DIR));
+
+    closedir DIR;
+  }
+
+  # now parse each macro file and build hash
+  foreach my $dm (sort @dm_list) {
+    open(DM, "< $dm")
+      or die "can't open macro file '$dm': $!\n";
+
+    my ($name, $content, $version, $curpkg, $contd)
+     = ('',    '',       '0',      '',      0);
+
+    while (my $line = <DM>) {
+      chomp($line);
+      next unless length $line;      # skip empty lines
+      next if     $line =~ /^\s*$/;  # skip "empty" lines
+      next if     $line =~ /^\s*#/;  # skip comments (i hope this doesn't
+                                     # affect cases we actually wanted to keep)
+
+      if    ($line =~ /^$PACKAGE_DECLARATION\s*($PACKAGE_NAME)/) {
+        # remember the current package we are in
+        $curpkg = $1;
+      }
+      elsif ($line =~ /$MACRO_AFFIX($MACRO_PATTERN)$MACRO_AFFIX\s*(\[v=1\])?\s*$DEFINITION_START\s*(.*)/) {
+        # start of macro definition
+        $n++;
+
+        $name    = $1;
+        $version = (defined $2 && $2 eq '[v=1]') ? '1' : '0';
+        $content = $3 || '';
+
+        # don't include unnecessary version, unless we're interactive (where version may change during session)
+        next if $ARG{'version'} ne '-1' && $version ne $ARG{'version'} && ! $ARG{'interactive'};
+
+        # is the macro definition already finished?
+        $contd = ($content =~ s/\s*$DEFINITION_END.*//) ? 0 : 1;
+
+        if (exists $macro_db{$name}->{$curpkg}->{$version}) {
+          # everytime a macro definition already exists, it's simply
+          # overwritten - but we can give a warning
+          # (this might also serve debugging purposes)
+          $m++;
+
+          warn <<HERE_WARN if $ARG{'verbose'};
+duplicate definition of macro '$curpkg:$name' [v=$version] at '$dm', line $.
+(previously defined at $macro_db{$name}->{$curpkg}->{$version}->{'file'}, line $macro_db{$name}->{$curpkg}->{$version}->{'line'})
+HERE_WARN
+        }
+
+        # store the information we got so far
+        $macro_db{$name}->{$curpkg}->{$version}->{'file'}    = $dm;
+        $macro_db{$name}->{$curpkg}->{$version}->{'line'}    = $.;
+        $macro_db{$name}->{$curpkg}->{$version}->{'content'} = [$content] if length $content;
+      }
+      elsif ($contd) {
+        # continuation of macro definition
+
+        # is the macro definition already finished?
+        $contd = ($line =~ s/\s*$DEFINITION_END.*//) ? 0 : 1;
+
+        # store additional content
+        push(@{$macro_db{$name}->{$curpkg}->{$version}->{'content'}} => $line);
+      }
+      else {
+        # something else...
+
+        ($name, $content) = ('', '');
+      }
+    }
+
+    close DM;
+  }
+
+  # get server macros (overwriting already read macros)
+  if (length $ARG{'source_dir'}) {
+    if (-r $ARG{'source_dir'}) {
+      my $recpt_dir = catdir($ARG{'source_dir'}, 'src', 'src', 'recpt');
+      my @cpp_list  = ();
+
+      opendir(DIR, "$recpt_dir")
+        or die "can't read receptionist's source directory '$recpt_dir': $!\n";
+
+      push(@cpp_list => map { $_ = catfile($recpt_dir, $_) } grep { /\.cpp$/ } readdir(DIR));
+
+      close DIR;
+
+      foreach my $cpp (@cpp_list) {
+        open(CPP, "< $cpp")
+          or die "can't open source file '$cpp': $!\n";
+
+        my $args  = '';
+        my $contd = 0;
+        while (my $line = <CPP>) {
+          next unless $line =~ /disp\.setmacro\s*\((.*)/ || $contd;
+
+          unless (defined $1) {
+            $contd = 1;
+            next;
+          }
+
+          my $string = $1;
+
+          if    ($string =~ s/\);\s*$//) {
+            $args .= $string;
+            my ($name, $package, $value) = split(/\s*,\s*/ => $args, 3);
+
+            $name    =~ s/^\s*["']?//;
+            $name    =~ s/["']?\s*$//;
+            $package =~ s/^\s*["']?//;
+            $package =~ s/["']?\s*$//;
+
+            $package = 'Global'
+              if $package eq 'displayclass::defaultpackage';
+
+            $macro_db{$name}->{$package}->{'0'}->{'file'}    = 'SERVER: ' . $cpp;
+            $macro_db{$name}->{$package}->{'0'}->{'line'}    = $.;
+            $macro_db{$name}->{$package}->{'0'}->{'content'} = [$value];
+
+            $args = '';
+            ++$n;
+            $contd = 0;
+          }
+          elsif ($contd) {
+            $args .= ' ' . $string;
+          }
+          else {
+            $contd = 1;
+          }
+        }
+
+        close CPP;
+      }
+    }
+    else {
+      warn "can't find source directory '$ARG{'source_dir'}'! server macros will not be included\n";
+    }
+  }
+
+  # print some statistics
+  print "$n total macro definitions, $m duplicates\n"
+    if $ARG{'verbose'};
+
+  # we stored all information there is so we can return it
+  return %macro_db;
+}
+# </sub build_db>
+
+# <sub get_macro>
+# recursively print macro information
+#
+# usage:
+#   get_macro($macro[, $level])
+#
+#   macro:    macro name (optionally including package specification)
+#   level:    recursion level (optional)
+#
+#   => VOID CONTEXT
+#
+sub get_macro {
+  my ($macro, $level) = @_;
+  $level ||= 0;
+
+  # indent output according to recursion level
+  my $indent  = '    ' x $level;
+
+  # get all the packages which our macro is defined in
+  ($macro, my @packages) = get_packages($macro, $indent);
+  return unless @packages;
+
+  # macro definitions may occur in several packages so we display them all
+  # (unless a certain package was explicitly specified)
+  foreach my $pkg (@packages) {
+    foreach my $version (sort keys %{$macro_db{$macro}->{$pkg}}) {
+      print "$indent* $pkg:$macro [v=$version] ($macro_db{$macro}->{$pkg}->{$version}->{'file'}, line $macro_db{$macro}->{$pkg}->{$version}->{'line'})\n";
+
+      my $content = '';
+      # some macros are defined, but don't have any content
+      if (defined $macro_db{$macro}->{$pkg}->{$version}->{'content'}) {
+        # for batch display we condense the output a little bit...
+        map { s/^\s*//; s/\s*$// } @{$macro_db{$macro}->{$pkg}->{$version}->{'content'}};
+        # ...and put it on a single line
+        $content = join(' ' => @{$macro_db{$macro}->{$pkg}->{$version}->{'content'}});
+      }
+      print "$indent  { $content }\n\n"
+        unless $ARG{'short'};
+        # short display only, i.e. no content
+        # of the macro's definition
+
+      # only go (deeper) into referenced macros if we
+      # haven't reached the specified recursion level
+      if ($ARG{'depth'} eq '-1' || $level < $ARG{'depth'}) {
+        # get (referencing|referenced) macros...
+        my @refs = $ARG{'reverse'}
+                 ? get_r_macros($macro)
+                 : get_macros($content);
+
+        # ...and recurse above them (with increased recursion level)
+        foreach my $ref (@refs) {
+          get_macro($ref, $level + 1);
+        }
+      }
+    }
+  }
+}
+# </sub get_macro>
+
+# <sub get_macros>
+# returns a list of macros extracted from a content string
+# or a boolean value if a macro name was specified
+#
+# usage:
+#   @macros  = get_macros($content)
+#   $boolean = get_macros($content, $macro)
+#
+#   content: content string
+#   macro:   macro name
+#
+#   => macros:  list of macros
+#   => boolean: boolean value (true = 1 / false = empty list)
+#
+sub get_macros {
+  my ($content, $macro) = @_;
+  my @macro_list = ();
+  my %seen       = ();
+
+  # get each macro reference in the string
+  # (for macro name considerations see above)
+  while ($content =~ /$MACRO_AFFIX((?:$PACKAGE_PATTERN)?$MACRO_PATTERN)$MACRO_AFFIX/g) {
+    my $m = $1;
+
+                           # we want to skip some macros that have no content anyway (defined
+                           # from within the server) - unless we're doing a "reverse" search
+    next if $seen{$m}++ || ($m =~ /^(cgiarg.*|if|httpimg|gwcgi|(decoded)?compressedoptions)$/i
+                            && ! $ARG{'reverse'});
+
+    if (defined $macro) {
+      # is this the macro we wanted? then the current
+      # macro uses it => return true
+      return 1 if $m =~ /^(?:$PACKAGE_PATTERN)?$macro$/;
+    }
+    else {
+      # add macro to our list
+      push(@macro_list => $m);
+    }
+  }
+
+  # return the list of used macros
+  # (this evaluates to false (empty list) if there are no further
+  # macro calls or if this macro doesn't use the sought-after macro)
+  return sort @macro_list;
+}
+# </sub get_macros>
+
+# <sub get_r_macros>
+# returns a list of macro names which reference ("use") the
+# specified macro or match the query
+#
+# usage:
+#   @macros = get_r_macros($macro)
+#   @macros = get_r_macros($query)
+#
+#   macro:    macro name
+#   query:    query string (regular expression)
+#
+#   => macros: list of macros
+#
+sub get_r_macros {
+  my ($arg, $query) = @_;
+  $query ||= 0;
+  my %refs = ();
+
+  # need to test each single macro's...
+  foreach my $m (sort keys %macro_db) {
+    # ...each single package
+    foreach my $p (sort keys %{$macro_db{$m}}) {
+      foreach my $v (sort keys %{$macro_db{$m}->{$p}}) {
+        my $pm = "$p:$m";  # include package information in the macro name!
+
+        # does this macro have any content?
+        if (defined $macro_db{$m}->{$p}->{$v}->{'content'}) {
+          # stringify content!
+          my $content = join(' ' => @{$macro_db{$m}->{$p}->{$v}->{'content'}});
+
+          if ($query) {
+            # search regexp
+            $refs{$pm}++ if $content =~ /$arg/;
+          }
+          else {
+            # search macro
+            $refs{$pm}++ if get_macros($content, $arg);
+          }
+        }
+      }
+    }
+  }
+
+  # now we have all the macros which use our sought-after
+  return sort keys %refs;
+}
+# </sub get_r_macros>
+
+# <sub recurse_packages>
+# recurse all packages for a given macro
+#
+# usage:
+#   recurse_packages($macro)
+#
+#   macro:    macro name (any package specification will be dropped)
+#
+#   => VOID CONTEXT
+#
+sub recurse_packages {
+  my ($macro) = @_;
+
+  # repeat until explicit break/exit
+  while (1) {
+    # get all the packages which our macro is defined in
+    #($macro, my @packages) = get_packages($macro);
+    #return unless @packages;
+    my @packages = ();
+
+    my $n = 0;
+    my $package = '';
+    # ask for user's selection...
+    do {
+      # get all the packages which our macro is defined in
+      ($macro, @packages) = get_packages($macro);
+      return unless @packages;
+
+      # ask for user's selection...
+      print "select package for macro '$macro' [leave empty to return]\n";
+      foreach my $pkg (@packages) {
+        printf "    [%d]%s %s\n", ++$n, " " x (4 - length $n), $pkg;
+      }
+      $package = prompt();
+      $n = 0;
+    # ...until we return...
+    } until ($package eq '' || $package eq '-1'
+         # ...or a valid number is provided
+         || ($package =~ /^\d+$/ && $package > 0 && $package <= @packages));
+
+    return unless length $package;          # return to previous stage
+    return '-1'   if     $package eq '-1';  # return to top
+
+    # set selected package
+    $package = $packages[$package - 1];
+
+    foreach my $version (sort keys %{$macro_db{$macro}->{$package}}) {
+                  # all versions
+      next unless $ARG{'version'} eq '-1'
+                  # desired version
+               || $version eq $ARG{'version'}
+                  # fallback to 'graphic'
+               || ($version eq '0' && ! exists $macro_db{$macro}->{$package}->{'1'});
+
+                    # some macros are defined, but don't have any content
+      my $content = defined $macro_db{$macro}->{$package}->{$version}->{'content'}
+                    # now we want to retain the original structure
+                  ? join("\n" => @{$macro_db{$macro}->{$package}->{$version}->{'content'}})
+                  : '';
+
+      ($CURRENT_FILE = $macro_db{$macro}->{$package}->{$version}->{'file'}) =~ s/^SERVER: //;
+
+      my $content_string = "* $package:$macro [v=$version] ($macro_db{$macro}->{$package}->{$version}->{'file'}, line $macro_db{$macro}->{$package}->{$version}->{'line'})\n";
+      $content_string   .= "{ $content }\n"
+        unless $ARG{'short'};
+
+      print_output($content_string);
+
+      # now on to the macros referenced within this one
+      my $return = recurse_macros($content);
+
+      return $return if defined $return;
+    }
+  }
+}
+# </sub recurse_packages>
+
+# <sub get_packages>
+# returns list of packages for specified macro, also returns
+# modified macro name (without surrounding '_' and package specification)
+#
+# usage:
+#   ($macro, @packages) = get_packages($macro)
+#
+#   macro:    macro name
+#
+#   => macro:    modified macro name
+#   => packages: list of packages
+#
+sub get_packages {
+  my ($macro, $indent) = @_;
+  $indent ||= '';
+
+  # save original macro name (including package specification)
+  my $omacro = $macro;
+
+  my @packages = ();
+
+  if ($macro =~ /^($PACKAGE_PATTERN)?$MACRO_PATTERN$/) {
+    # valid macro name
+
+    # strip off package specification
+    my $package = ($macro =~ s/^($PACKAGE_NAME)://) ? $1 : '';
+
+    if (exists $macro_db{$macro}) {
+      # valid/existing macro
+
+      unless ($ARG{'interactive'}) {
+        if (length $package) {
+          # account for package specification
+
+          @packages = ($package)
+            if exists $macro_db{$macro}->{$package};
+        }
+        else {
+          # get all packages otherwise
+
+          @packages = sort keys %{$macro_db{$macro}};
+        }
+      }
+      else {
+        foreach my $pkg (sort keys %{$macro_db{$macro}}) {
+          push(@packages => $pkg)
+               # all versions
+            if $ARG{'version'} eq '-1'
+               # desired version
+            || exists $macro_db{$macro}->{$pkg}->{$ARG{'version'}}
+               # fallback to 'graphic'
+            || exists $macro_db{$macro}->{$pkg}->{'0'};
+        }
+      }
+    }
+  }
+  else {
+    # invalid macro name
+
+    warn "invalid macro name '$macro'!\n";
+    return;  # skip it
+  }
+
+  # no packages - no definition
+  unless (@packages) {
+    print "$indent- $omacro\n$indent  no definition for macro!\n\n";
+    return;  # skip it
+  }
+
+  # return modified macro name and packages found
+  return $macro, sort @packages;
+}
+# </sub get_packages>
+
+# <sub recurse_macros>
+# recurse all macros for a given content string
+#
+# usage:
+#   recurse_macros($content)
+#
+#   content:  content string
+#
+#   => VOID CONTEXT
+#
+sub recurse_macros {
+  my ($content) = @_;
+
+  # repeat until explicit break/exit
+  while (1) {
+    # get all the macros referenced within the current one
+    my @macros = get_macros($content);
+    return unless @macros;
+
+    my $n = 0;
+    my $macro = ''; 
+    # ask for user's selection...
+    do {
+      print "select macro [leave empty to return]\n";
+      foreach my $m (@macros) {
+        printf "    [%d]%s %s\n", ++$n, " " x (4 - length $n), $m;
+      }
+      $macro = prompt();
+      $n = 0;
+    # ...until we return...
+    } until ($macro eq '' || $macro eq '-1'
+         # ...or a valid number is provided
+         || ($macro =~ /^\d+$/ && $macro > 0 && $macro <= @macros));
+
+    return unless length $macro;          # return to previous stage
+    return '-1'   if     $macro eq '-1';  # return to top
+
+    # set selected macro
+    $macro = $macros[$macro - 1];
+
+    # now we want all the macro's packages again
+    my $return = recurse_packages($macro);
+
+    return $return if defined $return;
+  }
+}
+# </sub recurse_macros>
+
+# <sub prompt>
+# prompt for user input
+#
+# usage:
+#   $line = prompt([$prompt])
+#
+#   prompt:  optional prompt (default: '> ')
+#
+#   => line: user input
+#
+sub prompt {
+  my $prompt = shift || '> ';
+  my $term   = $TERM;
+
+  # read user input
+  my $line = $term->readline($prompt);
+
+  if (defined $line) {
+    # add input to history, unless it's just a number
+    $term->addhistory($line)
+      if $line =~ /[[:alpha:]]/;
+
+    if ($line =~ s/^\s*["']*\s*\././) {
+      # execute command
+      my $return = parse_command($line);
+
+      return $return if defined $return;
+    }
+    else {
+      return $line;
+    }
+  }
+
+  # allow for exiting by hitting <ctrl>+d,
+  # or quitting by command (.q, .quit)
+  die "\n";
+}
+# </sub prompt>
+
+# <sub print_output>
+# print output, paged or not
+#
+# usage:
+#   print_output($output)
+#   print_output(@output)
+#
+#   output: text to print
+#
+#   => VOID CONTEXT
+#
+sub print_output {
+  my $output = join('' => @_);
+
+  if ($ARG{'paged'}) {
+    # pass output to pager
+    open(LESS, "| $ARG{'pager'}")
+      or die "can't open pipe to '$ARG{'pager'}': $!";
+
+    print LESS "$output";
+
+    close LESS;
+  }
+  else {
+    # print to standard out...
+    print "\n$output\n";
+
+    # ...and wait for user reaction to continue
+    wait_for_user();
+  }
+}
+# </sub print_output>
+
+# <sub wait_for_user>
+# wait for user reaction to continue
+#
+# usage:
+#   wait_for_user()
+#
+#   => VOID CONTEXT
+#
+sub wait_for_user {
+  print "[press <enter> to continue]";
+  print "\n" if <STDIN>;
+}
+# </sub wait_for_user>
+
+# <sub parse_command>
+# prompt for user input
+#
+# usage:
+#   parse_command($command_line)
+#
+#   command_line: command string
+#
+#   => VOID CONTEXT
+#
+sub parse_command {
+  my $command_line = shift;
+  my @commands = split(/\s*;\s*/ => $command_line);
+
+  my $return = 0;
+
+  foreach my $command (@commands) {
+    my $msg = "command executed: '$command'";
+
+    $command =~ s/^\.//;
+    $command =~ s/^(\w+)["']*/$1/;
+    $command =~ s/\s*$//;
+
+    if    ($command =~ /^(h|\?|help)$/) {
+      print "$INTERACTIVE_HELP\n";
+
+      # wait for user reaction to continue
+      wait_for_user();
+
+      next;
+    }
+    elsif ($command =~ /^(q|quit)$/) {
+      return undef;
+    }
+    elsif ($command =~ /^(\.)$/) {
+      $return = '';
+
+      next;
+    }
+    elsif ($command =~ /^(\..|t|top)$/) {
+      $return = '-1';
+
+      next;
+    }
+    elsif ($command =~ /^(n|show-version)(?:\s+["']*(0|1|-1)["']*)?$/) {
+      $ARG{'version'} = $2
+        if defined $2;
+
+      $msg = "'version' " . (defined $2 ? '' : 'is currently ') . "set to: '$ARG{'version'}'";
+    }
+    elsif ($command =~ /^(s|short)$/) {
+      $ARG{'short'} = ! $ARG{'short'};
+
+      $msg = "'short' output " . ($ARG{'short'} ? 'en' : 'dis') . "abled";
+    }
+    elsif ($command =~ /^(p|paged)$/) {
+      $ARG{'pager'} = get_pager();
+      $ARG{'paged'} = ! $ARG{'paged'}
+        if -x $ARG{'pager'};
+
+      $msg = "'paged' output " . ($ARG{'paged'} ? 'en' : 'dis') . "abled";
+    }
+    elsif ($command =~ /^(pager)(?:\s+["']*(\w+)["']*)?$/) {
+      $ARG{'pager'} = get_pager($2)
+        if defined $2;
+
+      $msg = "'pager' " . (defined $2 ? '' : 'is currently ') . "set to: '$ARG{'pager'}'";
+    }
+    elsif ($command =~ /^(p|paged-)?(r|read)(?:\s+(["']?.+["']?))?$/) {
+      my $paged = $1 || '';
+      my $file  = $3 || $CURRENT_FILE;
+      $CURRENT_FILE = $file;
+
+      if (-r $file) {
+        open(FILE, "< $file")
+          or die "can't open file '$file': $!\n";
+
+        my @lines = <FILE>;
+
+        close FILE;
+
+        my $previous_paged = $ARG{'paged'};
+        $ARG{'paged'}      = 1 if $paged;
+
+        #print_output("$file:\n\n", @lines);
+        print_output(@lines);
+
+        $ARG{'paged'}      = $previous_paged;
+
+        next;
+      }
+
+      $msg = "can't find file '$file'";
+    }
+    elsif ($command =~ /^(c|config)$/) {
+      my $short = $ARG{'short'} ? 'enabled' : 'disabled';
+      my $paged = $ARG{'paged'} ? 'enabled' : 'disabled';
+
+      $msg = <<HERE_MSG;
+current configuration for '$NAME - interactive browse mode':
+
+'version':         $ARG{'version'}
+'short' output:    $short
+'paged' output:    $paged
+'pager':           $ARG{'pager'}
+current file:      $CURRENT_FILE
+HERE_MSG
+    }
+    elsif (length $command) {
+      $msg = "invalid command: .$command";
+    }
+    else {
+      # probably the '.' command
+      $return = 0;
+
+      next;
+    }
+
+    print "% $msg\n";
+  }
+
+  return $return;
+}
+# </sub parse_command>
+
+# <sub readhist>
+# read history from histfile
+#
+# usage:
+#   readhist();
+#
+#   => VOID CONTEXT
+#
+sub readhist {
+  my $term = $TERM;
+
+  if (-r $ARG{'histfile'}) {
+    open(HIST, "< $ARG{'histfile'}")
+      or die "can't open histfile '$ARG{'histfile'}': $!\n";
+
+    while (<HIST>) {
+      chomp;
+      $term->AddHistory($_);
+    }
+
+    close HIST;
+
+    warn "history restored from '$ARG{'histfile'}'\n"
+      if $ARG{'verbose'};
+  }
+  else {
+    warn "history could not be restored (maybe no/wrong history file specified)\n"
+      if $ARG{'verbose'};
+  }
+}
+# </sub readhist>
+
+# <sub savehist>
+# save history to histfile
+#
+# usage:
+#   savehist();
+#
+#   => VOID CONTEXT
+#
+sub savehist {
+  return unless length $ARG{'histfile'} && $ARG{'histsize'};
+
+  my $term = $TERM;
+
+  return unless length $term;
+
+  if (-w $ARG{'histfile'} || (! -e $ARG{'histfile'} && -w dirname $ARG{'histfile'})) {
+    my @history = $term->GetHistory;
+
+    # drop (consecutive) duplicate entries
+    my @unified  = ();
+    my $previous = '';
+    foreach my $element (@history) {
+      push(@unified => $element)
+        unless $element eq $previous;
+      $previous = $element;
+    }
+    @history = @unified;
+
+    # cut history to specified maximum number of entries
+    splice(@history, 0, @history - $ARG{'histsize'})
+      if @history > $ARG{'histsize'};
+
+    open(HIST, "> $ARG{'histfile'}")
+      or die "can't open history file '$ARG{'histfile'}' for writing: $!\n";
+
+    {
+    local $, = "\n";
+
+    print HIST @history, "";
+    }
+
+    close HIST;
+
+    warn "history written to '$ARG{'histfile'}'\n"
+      if $ARG{'verbose'};
+  }
+  else {
+    warn "history could not be written (maybe no history file specified, or history file not writable)\n"
+      if $ARG{'verbose'};
+  }
+}
+# </sub savehist>
+
+# <sub get_pager>
+# get pager executable
+#
+# usage:
+#   $pager = get_pager([$candidate]);
+#
+#   canidate: candidate for pager executable (defaulting to $ARG{'pager'})
+#
+#   => pager: pager executable
+#
+sub get_pager {
+  my $candidate = shift || $ARG{'pager'};
+
+  return $candidate if -x $candidate;
+
+  # get first pager executable in PATH
+  foreach my $path (split(':' => $ENV{'PATH'})) {
+    return catfile($path, $candidate) if -x catfile($path, $candidate);
+  }
+
+  # still no executable!
+  warn "can't find pager '$candidate'! disabling 'paged' output\n";
+  $ARG{'paged'} = 0;
+
+  return '-1';
+}
+# </sub get_pager>
Index: /main/tags/3.01/gsdl/bin/script/explode_metadata_database.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/explode_metadata_database.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/explode_metadata_database.pl	(revision 21112)
@@ -0,0 +1,351 @@
+#!/usr/bin/perl
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use strict;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+no strict 'refs'; # allow filehandles to be variables and vice versa
+use printusage;
+use parse2;
+
+my $unicode_list =
+    [ { 'name' => "auto",
+	'desc' => "{BasPlug.input_encoding.auto}" },
+      { 'name' => "ascii",
+	'desc' => "{BasPlug.input_encoding.ascii}" },
+      { 'name' => "utf8",
+	'desc' => "{BasPlug.input_encoding.utf8}" },
+      { 'name' => "unicode",
+	'desc' => "{BasPlug.input_encoding.unicode}" } ];
+
+my $arguments = 
+    [ 
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "plugin",
+	'desc' => "{explode.plugin}",
+	'type' => "string",
+	'reqd' => "yes",
+	'hiddengli' => "yes"},
+      { 'name' => "input_encoding",
+	'desc' => "{explode.encoding}",
+	'type' => "enum",
+	'deft' => "auto",
+	'list' => $unicode_list,
+	'reqd' => "no" },
+      { 'name' => "metadata_set",
+	'desc' => "{explode.metadata_set}",
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+      { 'name' => "document_field",
+	'desc' => "{explode.document_field}",
+	'type' => "string",
+	'reqd' => "no"},
+       { 'name' => "document_prefix",
+	'desc' => "{explode.document_prefix}",
+	'type' => "string",
+	'reqd' => "no"},
+      { 'name' => "document_suffix",
+	'desc' => "{explode.document_suffix}",
+	'type' => "string",
+	'reqd' => "no"},
+      { 'name' => "filename_field",
+	'desc' => "{explode.filename_field}",
+	'type' => "string",
+	'reqd' => "no"},
+      { 'name' => "verbosity",
+	'desc' => "{import.verbosity}",
+	'type' => "int",
+	'range' => "0,",
+	'deft' => "1",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "xml",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" }
+      ];
+	
+my $options = { 'name' => "explode_metadata_database.pl",
+		'desc' => "{explode.desc}",
+		'args' => $arguments };
+
+	    
+sub main
+{
+    my ($language, $input_encoding, $metadata_set, $plugin, $filename_field, 
+	$document_field, $document_prefix, $document_suffix, $verbosity);
+
+    my $xml = 0;
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # parse the options
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language && $language =~ /\S/) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	print "\n";
+	return;
+    }
+
+    # There should one arg left after parsing
+    if($intArgLeftinAfterParsing > 1)
+    {
+	&PrintUsage::print_txt_usage($options, "{explode.params}");
+	die "\n";
+    }
+
+    # The metadata database filename is the first value that remains after the options have been parsed out
+    my $filename = $ARGV[0];
+    if (!defined $filename || $filename !~ /\w/) { 
+	&PrintUsage::print_txt_usage($options, "{explode.params}");
+	print STDERR "You need to specify a filename";
+	die "\n";
+    }
+    # check that file exists
+    if (!-e $filename) {
+	print STDERR "File $filename doesn't exist...\n";
+	die "\n";
+    }
+    # check required options
+    if (!defined $plugin || $plugin !~ /\w/) {
+	&PrintUsage::print_txt_usage($options, "{explode.params}");
+	print STDERR "You need to specify a plugin";
+	die "\n";
+    }
+    
+    # check metadata set
+    if (defined $metadata_set && $metadata_set =~ /\w/) {
+	$metadata_set .= ".";
+    } else {
+	$metadata_set = "";
+    }
+
+    #check filename field
+    if (defined $filename_field && $filename_field eq "") {
+	undef $filename_field;
+    }
+    my $plugobj;
+    require "$plugin.pm";
+    eval ("\$plugobj = new $plugin()");
+    die "$@" if $@;
+
+    # ...and initialize it
+    $plugobj->init(1, "STDERR", "STDERR");
+    
+    if ($input_encoding eq "auto") {
+	$plugobj->{'input_encoding'} = $input_encoding;   
+	($language, $input_encoding) = $plugobj->textcat_get_language_encoding ($filename);
+    }
+    my $text = "";
+    # Use the plugin's read_file function to avoid duplicating code
+    $plugobj->read_file($filename, $input_encoding, undef, \$text);
+
+    # Create a directory to store the document files...
+    my ($documents_directory) = ($filename =~ /(.*)\.[^\.]+$/);
+    if (-d $documents_directory) {
+	die "Error: document directory $documents_directory already exists (bailing).\n";
+    }
+    &util::mk_dir($documents_directory);
+
+    # ...and a metadata.xml file for the document metadata (extracted from the database)
+    my $documents_metadata_xml_file = &util::filename_cat($documents_directory, "metadata.xml");
+    if (-e $documents_metadata_xml_file) {
+	die "Error: document metadata.xml file $documents_metadata_xml_file already exists (bailing).\n";
+    }
+
+    # Start the metadata.xml file
+    open(METADATA_XML_FILE, ">$documents_metadata_xml_file");
+    print METADATA_XML_FILE
+	"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n" .
+	"<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">\n" .
+	"<DirectoryMetadata>\n";
+
+    # Split the text into records, using the plugin's split_exp
+    my $split_exp = $plugobj->{'split_exp'};
+    my @metadata_records = split(/$split_exp/, $text);
+    print STDERR "Number of records: " . @metadata_records . "\n";
+
+    # Write the metadata from each record to the metadata.xml file
+    my $record_number = 0;
+    foreach my $record_text (@metadata_records) {
+	
+	# Use the plugin's process function to avoid duplicating code
+	my $doc_obj = new doc($filename, "nonindexed_doc");
+	$plugobj->process(\$record_text, undef, undef, $filename, undef, $doc_obj, 0);
+	# Get all the metadata assigned to this record
+	my $record_metadata = $doc_obj->get_all_metadata($doc_obj->get_top_section());
+	my $document_file;
+	
+	# try to get a doc to attach the metadata to
+	if (defined $document_field) {
+	    foreach my $pair (@$record_metadata) {
+		my ($field, $value) = (@$pair);
+		
+		# Does this metadata element specify a document to obtain?
+		if ($field eq $document_field) {
+		    my $document_file_full = $document_prefix . $value . $document_suffix;
+		    $document_file = &obtain_document($document_file_full, $documents_directory, $verbosity);
+		}
+	    }
+	}
+	# do we need to create a dummy doc??
+	if (not defined $document_file) {
+	    # try to get a file name
+	    if (defined $filename_field) {
+		
+		my $meta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $filename_field);
+		if (defined $meta) {
+		    $meta =~ s/&\w{1,10};//g; # remove entities
+		    $document_file = "$meta.nul";
+		    my $num = 0;
+		    while (-e "$documents_directory/$document_file") {
+			$num++;
+			$document_file = "$meta$num.nul";
+		    }
+		} else {
+		    $record_number = $record_number + 1;
+		    $document_file = sprintf("%4.4d", $record_number) . ".nul";
+		}
+	    } else {
+		$record_number = $record_number + 1;
+		$document_file = sprintf("%4.4d", $record_number) . ".nul";
+	    }
+	    open(DUMMY_FILE, ">$documents_directory/$document_file");
+	    close(DUMMY_FILE);
+
+	}
+	
+
+	&write_metadata_xml_file_entry(METADATA_XML_FILE, $document_file, $record_metadata, $metadata_set);
+    }
+
+    # Finish and close the metadata.xml file
+    print METADATA_XML_FILE "\n</DirectoryMetadata>\n";
+    close(METADATA_XML_FILE);
+
+    # Explode means just that: the original file is deleted
+    &util::rm($filename);
+}
+
+
+sub write_metadata_xml_file_entry
+{
+    my $metadata_xml_file = shift(@_);
+    my $file_name = shift(@_);
+    my $record_metadata = shift(@_);
+    my $meta_prefix = shift(@_);
+    
+    # Make $file_name XML-safe
+    $file_name =~ s/</&lt;/g;
+    $file_name =~ s/>/&gt;/g;
+
+    # Convert $file_name into a regular expression that matches it
+    $file_name =~ s/\./\\\./g;
+    $file_name =~ s/\(/\\\(/g;
+    $file_name =~ s/\)/\\\)/g;
+    $file_name =~ s/\{/\\\{/g;
+    $file_name =~ s/\}/\\\}/g;
+    $file_name =~ s/\[/\\\[/g;
+    $file_name =~ s/\]/\\\]/g;
+    
+    print $metadata_xml_file
+	"\n" .
+        "  <FileSet>\n" .
+	"    <FileName>$file_name</FileName>\n" .
+	"    <Description>\n";
+
+    foreach my $pair (@$record_metadata) {
+	my ($field, $value) = (@$pair);
+
+	# We're only interested in metadata from the database
+	next if ($field eq "lastmodified");
+	next if ($field eq "gsdlsourcefilename");
+	next if ($field eq "gsdldoctype");
+	next if ($field eq "FileFormat");
+
+	# Ignore the ^all metadata, since it will be invalid if the source metadata is changed
+	next if ($field =~ /\^all$/);  # ISISPlug specific!
+
+	# Make $value XML-safe
+	$value =~ s/</&lt;/g;
+	$value =~ s/>/&gt;/g;
+
+	# we are not allowed & in xml except in entities. 
+	# if there are undefined entities then parsing will also crap out.
+	# should we be checking for them too?
+	# this may not get all possibilities
+	$value =~ s/&([^;\s]*(\s|$))/&amp;$1/g;
+
+	print $metadata_xml_file "      <Metadata mode=\"accumulate\" name=\"$meta_prefix$field\">$value</Metadata>\n";
+    }
+
+    print $metadata_xml_file
+	"    </Description>\n" .
+	    "  </FileSet>\n";
+}
+
+sub obtain_document
+{
+    my $document_file_full = shift(@_);
+    my $documents_directory = shift(@_);
+    my $verbosity = shift(@_);
+    
+    print STDERR "Obtaining document file $document_file_full...\n" if ($verbosity > 1);
+
+    my $document_file_name;
+    my $local_document_file;
+
+    # Document specified is on the web
+    if ($document_file_full =~ /^http:/ || $document_file_full =~ /^ftp:/) {
+	$document_file_full =~ /([^\/]+)$/;
+	$document_file_name = $1;
+	$local_document_file = &util::filename_cat($documents_directory, $document_file_name);
+
+	my $wget_options = "--quiet";
+	$wget_options = "--verbose" if ($verbosity > 2);
+	$wget_options .= " --timestamping";  # Only re-download files if they're newer
+	`wget $wget_options $document_file_full --output-document $local_document_file`;
+    }
+    # Document specified is on the disk
+    else {
+	my $dir_sep = &util::get_os_dirsep();
+	$document_file_full =~ /(.+$dir_sep)?(.*)$/;
+	$document_file_name = $2;
+	$local_document_file = &util::filename_cat($documents_directory, $document_file_name);
+
+	&util::cp($document_file_full, $documents_directory);
+    }
+
+    # Check the document was obtained successfully
+    if (!-e $local_document_file) {
+	print STDERR "WARNING: Could not obtain document file $document_file_full\n";
+	return undef;
+    }
+
+    return $document_file_name;
+}
+
+&main(@ARGV);
Index: /main/tags/3.01/gsdl/bin/script/export.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/export.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/export.pl	(revision 21112)
@@ -0,0 +1,489 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# export.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will export a particular collection into a specific Format (e.g. METS or DSpace)
+# Author: Chi-Yu Huang Date: 08-10-2004
+
+package export;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and vice versa
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docprint;
+use util;
+use scriptutil;
+use FileHandle;
+use gsprintf;
+use printusage;
+use parse2;
+
+
+my $oidtype_list = 
+    [ { 'name' => "hash",
+        'desc' => "{export.OIDtype.hash}" },
+      { 'name' => "incremental",
+        'desc' => "{export.OIDtype.incremental}" } ];
+
+#** define to use the METS format or DSpace format
+my $saveas_list = 
+    [ { 'name' => "DSpace",
+        'desc' => "{export.saveas.DSpace}" },
+      { 'name' => "METS",
+        'desc' => "{export.saveas.METS}" } ];
+
+
+# Possible attributes for each argument
+# name: The name of the argument
+# desc: A description (or more likely a reference to a description) for this argument
+# type: The type of control used to represent the argument. Options include: string, int, flag, regexp, metadata, metadatum, language, enum etc
+# reqd: Is this argument required?
+# hiddengli: Is this argument hidden in GLI?
+# modegli: The lowest detail mode this argument is visible at in GLI
+
+my $saveas_argument =       
+      { 'name' => "saveas",
+	'desc' => "{export.saveas}",
+	'type' => "enum",
+	'list' => $saveas_list,
+	'deft' => "METS",
+	'reqd' => "no",
+	'modegli' => "3" };
+
+
+my $arguments = 
+    [ 
+      $saveas_argument,
+      { 'name' => "saveas_version",
+	'desc' => "{export.saveas_version}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "exportdir",
+	'desc' => "{export.exportdir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "importdir",
+	'desc' => "{import.importdir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "collectdir",
+	'desc' => "{export.collectdir}",
+	'type' => "string",
+	'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "listall",
+	'desc' => "{scripts.listall}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "debug",
+	'desc' => "{export.debug}",
+	'type' => "flag",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "faillog",
+	'desc' => "{export.faillog}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no",
+        'modegli' => "4" },
+      { 'name' => "keepold",
+	'desc' => "{export.keepold}",
+	'type' => "flag",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "removeold",
+	'desc' => "{export.removeold}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "maxdocs",
+	'desc' => "{export.maxdocs}",
+	'type' => "int",
+	'reqd' => "no",
+	'range' => "1,",
+	'modegli' => "1" },
+      { 'name' => "out",
+	'desc' => "{export.out}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "statsfile",
+	'desc' => "{export.statsfile}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "verbosity",
+	'desc' => "{export.verbosity}",
+	'type' => "int",
+	'range' => "0,3",
+	'deft' => "2",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "gli",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" }
+      ];
+
+my $options = { 'name' => "export.pl",
+		'desc' => "{export.desc}",
+		'args' => $arguments };
+
+my $listall_options = { 'name' => "export.pl",
+		        'desc' => "{export.desc}",
+		        'args' => [ $saveas_argument ] };
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+&main();
+
+sub main {
+    # params
+    my ($language, $verbosity, $importdir, $exportdir, $keepold, $listall,
+	$removeold, $saveas, $saveas_version, $debug, 
+	$maxdocs, $statsfile, $out, $faillog, $collectdir, $gli);
+    my $xml = 0;
+    
+    # other vars
+    my ($configfilename, $collection, $export_info_filename, $export_info, $processor, $pluginfo);
+    my $service = "export";
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # general options available to all plugins
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+    # can have more than one collection name, so don't check num args left    
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+
+    
+    # these are options used by other things - we just set default values
+    # undef means will be set from config file if there
+    my $gzip = undef; 
+    my $groupsize = 1;
+    my $OIDtype = undef;
+    my $sortmeta = undef;
+
+    my $explicit_exportdir = (defined $exportdir) ? 1 : 0;
+
+    # save these command line settings. don't want config file settings in one
+    # coll used for other colls
+    # does this apply to other vars???
+    my $global_removeold = $removeold;
+    my $global_keepold = $keepold;
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($listall) {
+	if ($xml) {
+	    &PrintUsage::print_xml_usage($listall_options);
+	}
+	else
+	{
+	    &PrintUsage::print_txt_usage($listall_options,"{export.params}");
+	}
+	die "\n";
+    }
+    elsif ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	die "\n";
+    }
+
+    if (scalar(@ARGV) == 0) {
+	&PrintUsage::print_txt_usage($options, "{export.params}");
+	die "\n";
+    }
+
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") ||
+	    (&gsprintf(STDERR, "{common.cannot_open_output_file}\n", $out) && die);
+	$out = 'export::OUT';
+	$close_out = 1;
+    }
+    $out->autoflush(1);
+
+    while (scalar(@ARGV)>0) {
+	my $collect_name = shift @ARGV;
+	$ENV{'GSDLCOLLECTION'} = $collect_name;
+
+	eval {
+	    # get and check the collection name
+	    if (($collection = &util::use_collection($collect_name, $collectdir)) eq "") {
+		&PrintUsage::print_txt_usage($options, "{export.params}");
+		die "\n";
+	    }
+
+	    if ($faillog eq "") {
+		$faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
+	    }
+	    open (FAILLOG, ">$faillog") ||
+		(&gsprintf(STDERR, "{export.cannot_open_fail_log}\n", $faillog) && die);
+	    my $faillogname = $faillog;
+	    $faillog = 'export::FAILLOG';
+	    $faillog->autoflush(1);
+	    
+	    # check sortmeta
+	    $sortmeta = undef unless defined $sortmeta && $sortmeta =~ /\S/;
+	    if (defined $sortmeta && $groupsize > 1) {
+		&gsprintf($out, "{export.cannot_sort}\n\n");
+		$sortmeta = undef;
+	    }
+
+	    # dynamically load 'docsave' module so it can pick up on a collection
+	    # specific docsave.pm is specified.
+	    
+	    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
+	    require docsave;
+	    
+	    # get the list of plugins for this collection and set any options that
+	    # were specified in the collect.cfg (all export.pl options except
+	    # -collectdir, -out and -faillog may be specified in the collect.cfg (these
+	    # options must be known before we read the collect.cfg))
+	    my $plugins = [];
+	    my @global_opts = ();
+	    
+	    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+	    if (!-e $configfilename) {
+		(&gsprintf($out, "{common.cannot_find_cfg_file}\n", $configfilename) && die);
+	    }
+	    
+	    my $collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	    if (defined $collectcfg->{'plugin'}) {
+		$plugins = $collectcfg->{'plugin'};
+	    }
+	    
+	    if ($verbosity !~ /\d+/) {
+		if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
+		    $verbosity = $collectcfg->{'verbosity'};
+		} else {
+		    $verbosity = 2; # the default
+		}
+	    }
+	    if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+		$importdir = $collectcfg->{'importdir'};
+	    }
+	    if (defined $collectcfg->{'exportdir'} && $exportdir eq "") {
+		$exportdir = $collectcfg->{'exportdir'};
+	    }
+
+	    if (defined $collectcfg->{'gzip'} && !$gzip) {
+		if ($collectcfg->{'gzip'} =~ /^true$/i) {
+		    $gzip = 1;
+		}
+	    }
+	    if ($maxdocs !~ /\-?\d+/) {
+		if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
+		    $maxdocs = $collectcfg->{'maxdocs'};
+		} else {
+		    $maxdocs = -1; # the default
+		}
+	    }
+	    if ($groupsize == 1) {
+		if (defined $collectcfg->{'groupsize'} && $collectcfg->{'groupsize'} =~ /\d+/) {
+		    $groupsize = $collectcfg->{'groupsize'};
+		}
+	    }
+	    if (!defined $OIDtype || ($OIDtype !~ /^(hash|incremental)$/)) {
+		if (defined $collectcfg->{'OIDtype'} && $collectcfg->{'OIDtype'} =~ /^(hash|incremental)$/) {
+		    $OIDtype = $collectcfg->{'OIDtype'};
+		} else {
+		    $OIDtype = "hash"; # the default
+		}
+	    }
+	    if (defined $collectcfg->{'sortmeta'} && $sortmeta eq "") {
+		$sortmeta = $collectcfg->{'sortmeta'};
+	    }
+	    if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
+		$debug = 1;
+	    }
+	    if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
+		$gli = 1;
+	    }
+	    
+	    # global plugin stuff
+	    if (defined $collectcfg->{'separate_cjk'}&& $collectcfg->{'separate_cjk'} =~ /^true$/i) {
+		push @global_opts, "-separate_cjk";
+	    }
+	    
+	    ($removeold, $keepold) = &scriptutil::check_removeold_and_keepold($global_removeold, $global_keepold, "export", $collectcfg);
+
+	    $gli = 0 unless defined $gli;
+
+	    print STDERR "<export>\n" if $gli;
+	    
+	    # fill in the default import and export directories if none
+	    # were supplied, turn all \ into / and remove trailing /
+	    $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
+	    $importdir =~ s/[\\\/]+/\//g;
+	    $importdir =~ s/\/$//;
+	    $exportdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "export") if $exportdir eq "";
+	    $exportdir =~ s/[\\\/]+/\//g;
+	    $exportdir =~ s/\/$//;
+	    
+	    # load all the plugins
+	    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts);
+	    
+	    if (scalar(@$pluginfo) == 0) {
+		&gsprintf($out, "{import.no_plugins_loaded}\n");
+		die "\n";
+	    }
+	    
+	    # remove the old contents of the export directory if needed
+	    if ($removeold && -e $exportdir) {
+		&gsprintf($out, "{export.removing_export}\n");
+		&util::rm_r ($exportdir);
+	    }
+	    
+	    # read the export information file
+	    if (!$debug) {
+		# Export to DSpace Arhive format or METs format
+		# If saveas=DSpace, a "contents" file will be created, otherwise "export.inf"
+		
+		if ($saveas eq "DSpace"){
+		    $export_info_filename = &util::filename_cat ($exportdir, "contents");
+		} elsif ($saveas eq "METS") {
+		    $export_info_filename = &util::filename_cat ($exportdir, "export.inf");
+		}
+		
+		$export_info = new arcinfo();
+		$export_info -> load_info ($export_info_filename);	
+		
+		$processor = new docsave ($collection, $export_info, $verbosity, $gzip, $groupsize, $out, $service, $saveas);
+		
+		$processor->setoutputdir ($exportdir);
+		
+		$processor->set_sortmeta ($sortmeta) if defined $sortmeta;
+		$processor->set_OIDtype ($OIDtype);
+		$processor->set_saveas ($saveas);
+		$processor->set_saveas_version ($saveas_version);
+	    } else {
+		$processor = new docprint ();
+	    }
+	    
+	    &plugin::begin($pluginfo, $importdir, $processor, $maxdocs);
+	    
+	    # process the import directory
+	    &plugin::read ($pluginfo, $importdir, "", {}, $processor, $maxdocs,0, $gli);
+	    
+	    &plugin::end($pluginfo, $processor);
+
+ 	    &plugin::deinit($pluginfo, $processor);
+	    
+	    # write out the export information file
+	    if (!$debug) {
+		$processor->close_file_output() if $groupsize > 1;
+		if ($saveas eq "METS") {
+		    $export_info->save_info($export_info_filename);
+		}
+	    }
+	    
+	    # write out export stats
+	    my $close_stats = 0;
+	    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
+		if (open (STATS, ">$statsfile")) {
+		    $statsfile = 'import::STATS';
+		    $close_stats = 1;
+		} else {
+		    &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
+		    &gsprintf($out, "{import.stats_backup}\n");
+		    $statsfile = 'STDERR';
+		}
+	    }
+
+	    &gsprintf($out, "\n");
+	    &gsprintf($out, "*********************************************\n");
+
+	    &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
+	    if ($close_stats) {
+		close STATS;
+	    }
+
+	    &gsprintf($out, "*********************************************\n");
+
+	    close OUT if $close_out;
+
+	    close FAILLOG;
+	};
+
+	if ($@) {
+	    print STDERR $@;
+	}
+
+##	$ENV{'GSDLCOLLECTION'} = undef;
+	$importdir = "";
+	$removeold = 0 if ($explicit_exportdir);
+
+    } # while processing ARGV
+    
+    &gsprintf($out, "\n");
+    &gsprintf($out, "*********************************************\n");
+    &gsprintf($out, "* {export.complete}\n");
+    &gsprintf($out, "*********************************************\n");    
+
+}
Index: /main/tags/3.01/gsdl/bin/script/exportcol.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/exportcol.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/exportcol.pl	(revision 21112)
@@ -0,0 +1,372 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# exportcol.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and vice versa
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+use util;
+use parse2;
+use printusage;
+
+my $arguments = 
+    [ { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "out",
+	'desc' => "{exportcol.out}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no" },
+      { 'name' => "nonetscape",
+	'desc' => "{exportcol.nonetscape}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "cdname",
+	'desc' => "{exportcol.cdname}",
+	'type' => "string",
+	'deft' => "Greenstone Collections",
+	'reqd' => "no" }, 
+      { 'name' => "cddir",
+	'desc' => "{exportcol.cddir}",
+	'type' => "string",
+	'deft' => "exported_collections",
+	'reqd' => "no" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "gli",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+
+      ];
+
+my $options = { 'name' => "exportcol.pl",
+		'desc' => "{exportcol.desc}",
+		'args' => $arguments };
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+&main();
+
+sub main {
+    my ($language, $out, $cdname, $cddir, $gli);
+
+    my $xml = 0;
+    my $nonetscape = 0;
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # parse options
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+    # don't care how many args are left - can have multiple collection names
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+    
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language && $language =~ /\S/) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+    
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	print "\n";
+	return;
+    }
+    
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+    
+    my $use_netscape = ($nonetscape) ? 0 : 1;
+
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	die "\n";
+    }
+
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+
+    my @coll_list = @ARGV;
+    
+    if (not @coll_list) { # empty list
+	&PrintUsage::print_txt_usage($options, "{exportcol.params}");
+	exit(1);
+    }
+
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") || 
+	    (&gsprintf(STDERR, "{common.cannot_open_output_file}\n", $out) && die);
+	$out = 'main::OUT';
+	$close_out = 1;
+    }
+
+    # first we do a quick check to see if the export coll function has been 
+    # installed
+    my $gssetupexe = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "gssetup.exe"); 
+    if (!-e $gssetupexe) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.export_coll_not_installed}\n");
+	die "\n";
+    }
+
+    # check each collection
+    my @valid_coll_list = ();
+    foreach my $c (@coll_list) {
+	my $colldir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $c);
+	if (! -d $colldir) {
+	    &gsprintf($out, "{exportcol.coll_not_found}\n", $c, $colldir);
+	    next;
+	}
+	my $colindexdir = &util::filename_cat ($colldir, "index");
+	my $coletcdir = &util::filename_cat ($colldir, "etc");
+	if ((!-d $colindexdir) || (!-d $coletcdir)) {
+	    &gsprintf($out, "{exportcol.coll_dirs_not_found}\n", $c);
+	    &gsprintf($out, "  $colindexdir\n");
+	    &gsprintf($out, "  $coletcdir\n");
+	    next;
+	}
+	# the collection seems ok, we add it to the valid coll list
+	push @valid_coll_list, $c;
+    }
+
+    if (not @valid_coll_list) {
+	# no valid colls left
+	&gsprintf($out, "{exportcol.fail} {exportcol.no_valid_colls}\n");
+	die "\n";
+    }
+
+    # create exported directory
+    my $topdir = &util::filename_cat ($ENV{'GSDLHOME'}, "tmp", $cddir);
+    &util::mk_all_dir ($topdir);
+    if (!-d $topdir) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.couldnt_create_dir}\n", $topdir);
+	die "\n";
+    }
+
+    # make other directories (we'll assume that if we created topdir
+    # successfully there'll be no problems creating these)
+    my $gsdldir = &util::filename_cat ($topdir, "gsdl");
+    &util::mk_all_dir ($gsdldir);
+    my $collectdir = &util::filename_cat ($gsdldir, "collect");
+    &util::mk_all_dir ($collectdir);
+    my $etcdir = &util::filename_cat ($gsdldir, "etc");
+    &util::mk_all_dir ($etcdir);
+    my $binjavadir = &util::filename_cat ($gsdldir, "bin", "java");
+    &util::mk_all_dir ($binjavadir);
+
+    # create the install.cfg file
+    my $installcfg = &util::filename_cat ($topdir, "install.cfg");
+    if (!open (INSTALLCFG, ">$installcfg")) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.couldnt_create_file}\n", $installcfg);
+	die "\n";
+    }
+    print INSTALLCFG "CompanyName:New Zealand Digital Library\n";
+    print INSTALLCFG "CollectionName:$cdname\n";
+    print INSTALLCFG "CollectionDirName:$cdname\n";
+    print INSTALLCFG "CollectionVersion:1.0\n";
+    print INSTALLCFG "CollectionVolume:1\n";
+    print INSTALLCFG "ProgramGroupName:Greenstone\n";
+    close INSTALLCFG;
+
+    # create the manifest.cfg file
+    my $manifestcfg = &util::filename_cat ($topdir, "manifest.cfg");
+    if (!open (MANIFESTCFG, ">$manifestcfg")) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.couldnt_create_file}\n", $manifestcfg);
+	die "\n";
+    }
+    print MANIFESTCFG "all:\n";
+    print MANIFESTCFG "  {library} {collection}\n\n";
+    print MANIFESTCFG "library:\n";
+    print MANIFESTCFG "  net32 net16 server.exe\n\n";
+    print MANIFESTCFG "database:\n";
+    print MANIFESTCFG '  etc ';
+    foreach my $c (@valid_coll_list) {
+	print MANIFESTCFG "collect\\$c\\index\\text\\$c.ldb ";
+    }
+    print MANIFESTCFG "\n\n";
+    print MANIFESTCFG "collection:\n";
+    print MANIFESTCFG "  collect etc images macros mappings bin\n";
+    close MANIFESTCFG;
+
+    #create the autorun.inf file
+    my $autoruninf = &util::filename_cat ($topdir, "Autorun.inf");
+    if (!open (AUTORUNINF, ">$autoruninf")) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.couldnt_create_file}\n", $autoruninf);
+	die "\n";
+    }
+
+    print AUTORUNINF "[autorun]\n";
+    print AUTORUNINF "OPEN=Setup.exe\n";
+    close AUTORUNINF;
+
+    # copy the necessary stuff from GSDLHOME
+    my $imagesdir = &util::filename_cat ($ENV{'GSDLHOME'}, "images");
+    my $macrosdir = &util::filename_cat ($ENV{'GSDLHOME'}, "macros");
+    my $mappingsdir = &util::filename_cat ($ENV{'GSDLHOME'}, "mappings");
+    my $maincfg = &util::filename_cat ($ENV{'GSDLHOME'}, "etc", "main.cfg");
+    my $serverexe = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "server.exe");
+    #my $gssetupexe = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "gssetup.exe"); - have checked this earlier
+    my $setupexe = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "Setup.exe");
+    my $net32dir = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "net32");
+    my $net16dir = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "net16");
+    my $netscapedir = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "netscape");
+    my $win32sdir = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "windows", "Win32s");
+    my $phindjar = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "java", "Phind.jar");
+    my $gsdlcollageappletjar = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", "java", "GsdlCollageApplet.jar");
+
+    if ((!-d $imagesdir) || (!-d $macrosdir) || (!-d $mappingsdir) || (!-e $maincfg) ||
+	(!-e $serverexe) || (!-e $gssetupexe) || (!-e $setupexe) || (!-d $net32dir) ||
+	(!-d $net16dir) || ($use_netscape && (!-d $netscapedir)) || (!-d $win32sdir) || (!-e $phindjar) ||
+	(!-e $gsdlcollageappletjar)) {
+	&gsprintf($out, "{exportcol.fail} {exportcol.non_exist_files}\n");
+	&gsprintf($out, "  $imagesdir\n");
+	&gsprintf($out, "  $macrosdir\n");
+	&gsprintf($out, "  $mappingsdir\n");
+	&gsprintf($out, "  $maincfg\n");
+	&gsprintf($out, "  $serverexe\n");
+	&gsprintf($out, "  $gssetupexe\n");
+	&gsprintf($out, "  $setupexe\n");
+	&gsprintf($out, "  $net32dir\n");
+	&gsprintf($out, "  $net16dir\n");
+	&gsprintf($out, "  $netscapedir\n") if ($use_netscape);
+	&gsprintf($out, "  $win32sdir\n");
+	&gsprintf($out, "  $phindjar\n");
+	&gsprintf($out, "  $gsdlcollageappletjar\n");
+	die "\n";
+    }
+
+    &util::cp_r ($imagesdir, $gsdldir);
+    &util::cp_r ($macrosdir, $gsdldir);
+    &util::cp_r ($mappingsdir, $gsdldir);
+    &util::cp ($maincfg, $etcdir);
+    &util::cp ($serverexe, $gsdldir);
+    &util::cp ($gssetupexe, $topdir);
+    &util::cp ($setupexe, $topdir);
+    &util::cp_r ($net32dir, $gsdldir);
+    &util::cp_r ($net16dir, $gsdldir);
+    &util::cp_r ($netscapedir, $topdir) if ($use_netscape);
+    &util::cp_r ($win32sdir, $topdir);
+    &util::cp ($phindjar, $binjavadir);
+    &util::cp ($gsdlcollageappletjar, $binjavadir);
+
+    # now change the home.dm macro file to a simple version
+    my $newmacrodir = &util::filename_cat ($gsdldir, "macros");
+    my $exporthome = &util::filename_cat ($newmacrodir, "exported_home.dm");
+    my $oldhome = &util::filename_cat ($newmacrodir, "home.dm");
+    if (-e $exporthome) {
+	&util::rm($oldhome);
+	&util::mv($exporthome, $oldhome);
+    }
+
+    # copy the collections over 
+    foreach my $c (@valid_coll_list) {
+	#old directories
+	my $colldir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $c);
+	my $colindexdir = &util::filename_cat ($colldir, "index");
+	my $coletcdir = &util::filename_cat ($colldir, "etc");
+	my $colimagesdir = &util::filename_cat ($colldir, "images");
+	my $colmacrosdir = &util::filename_cat ($colldir, "macros");
+	# new coll directory
+	my $newcoldir = &util::filename_cat ($collectdir, $c);
+
+	&util::mk_all_dir ($newcoldir);
+	&util::cp_r ($colindexdir, $newcoldir);
+	&util::cp_r ($coletcdir, $newcoldir);
+	&util::cp_r ($colimagesdir, $newcoldir) if (-e $colimagesdir);
+	&util::cp_r ($colmacrosdir, $newcoldir) if (-e $colmacrosdir);
+
+	# now we need to check the collect.cfg file to make sure it's public
+	my $collectcfg = &util::filename_cat ($newcoldir, "etc", "collect.cfg");
+	open INFILE, "<$collectcfg";
+	open OUTFILE, ">$collectcfg.tmp";
+	my $line;
+	while ($line = <INFILE>) {
+	    if ($line =~ /^\s*public\s+false/) {
+		print OUTFILE "public\ttrue\n";
+		last; # stop matching once we have found the line
+	    } else {
+		print OUTFILE "$line";
+	    }
+	}
+	# continue with no checking
+	while ($line = <INFILE>) {
+	    print OUTFILE "$line";
+	}
+	close INFILE;
+	close OUTFILE;
+	&util::mv("$collectcfg.tmp", $collectcfg);
+    }
+    &gsprintf($out, "{exportcol.success}");
+
+    my $successcolls = "";
+    my $first = 1;
+    foreach my $c (@valid_coll_list) {
+	if ($first) {
+	    $first=0;
+	} else {
+	    $successcolls .=", ";
+	}
+	$successcolls .= "$c";
+    }
+
+    my $gsdl_home = $ENV{'GSDLHOME'};
+    my $portable_topdir = $topdir;
+    # Disabled this because it isn't currently useful (the GLI applet doesn't do exporting)
+    # It doesn't work on Windows, either
+    # $portable_topdir =~ s/$gsdl_home/\$GSDLHOME/g;
+
+    &gsprintf($out, "{exportcol.output_dir}\n", $successcolls, $portable_topdir);
+    &gsprintf($out, "exportcol.pl succeeded:{exportcol.instructions}\n");
+    close OUT if $close_out;
+}
+
Index: /main/tags/3.01/gsdl/bin/script/extract_text.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/extract_text.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/extract_text.pl	(revision 21112)
@@ -0,0 +1,147 @@
+#! /usr/bin/perl -w
+
+# extract_text.pl extracts the text from macro files, install scripts
+# etc. for translation.
+
+# output is currently to a tab separated list, suitable for importing into
+# excel.
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+}
+
+my $installsh = "$ENV{'GSDLHOME'}/Install.sh";
+my $macrosdir = "$ENV{'GSDLHOME'}/macros";
+my $installshield = 'C:/My Installations/is_gsdl_cdrom/String Tables/0009-English/value.shl';
+
+my $texthash = {};
+
+# first extract all the itextn variables from Install.sh
+open (INSTALLSH, $installsh) || die;
+undef $/;
+my $file = <INSTALLSH>;
+$/ = "\n";
+close INSTALLSH;
+
+$texthash->{'install_sh'} = {};
+while ($file =~ s/^(itext\d+)=\"(.*?)(?<!\\)\"//sm) {
+    my $key = $1;
+    my $value = $2;
+    $value =~ s/\n/\\n/gs;
+    $value =~ s/\t/\\t/gs;
+    if (defined ($texthash->{'install_sh'}->{$key})) {
+	print STDERR "ERROR: $1 already defined\n";
+    }
+    $texthash->{'install_sh'}->{$key} = $value;
+}
+
+
+# now grab the text from the macro files
+$texthash->{'macros'} = {};
+
+# english.dm
+&grab_macros("$macrosdir/english.dm");
+# english2.dm
+&grab_macros("$macrosdir/english2.dm");
+
+
+# text from installshield installation
+$texthash->{'installshield'} = {};
+open (ISHIELD, $installshield) || die;
+my $line = "";
+my $textserver = {};
+while (defined ($line = <ISHIELD>)) {
+    next unless ($line =~ /\w/);
+    last if $line =~ /^\[General\]/;
+    next if $line =~ /^\[/;
+
+    $line =~ s/\n/\\n/gs;
+    $line =~ s/\t/\\t/gs;
+
+    if ($line =~ /^(TEXT_SERVERTXT_\d+)=(.*)$/) {
+	$textserver->{$1} = $2;
+    } else {
+	$line =~ /^([^=]+)=(.*)$/;
+	$texthash->{'installshield'}->{$1} = $2;
+    }
+}
+close ISHIELD;
+my $tserver = "";
+foreach my $stext (sort keys %{$textserver}) {
+    $tserver .= $textserver->{$stext};
+}
+$texthash->{'installshield'}->{'TEXT_SERVERTXT'} = $tserver;
+
+# print out the result
+foreach my $type (keys (%$texthash)) {
+    print STDOUT "type:\t$type\n";
+    foreach my $key (sort keys (%{$texthash->{$type}})) {
+	print STDOUT "$key\t$texthash->{$type}->{$key}\n";
+    }
+}
+
+sub grab_macros {
+    my ($filename) = @_;
+
+    my $line = "";
+    my $pack = "Global";
+    my $macroname = "";
+    my $macrovalue = "";
+    open (FILE, $filename) || die;
+    while (defined ($line = <FILE>)) {
+	if ($line =~ /^package (\w+)/) {
+	    $pack = $1;
+	    $macroname = "";
+	    $macrovalue = "";
+	    next;
+	}
+
+	if ($line =~ /^_([^_\s]+)_\s*(?:\[[^\]]*\])?\s*\{(.*)$/s) {
+	    die "ERROR 1\n" if ($macroname ne "" || $macrovalue ne "");
+	    $macroname = $1;
+	    $macrovalue = $2 if defined $2;
+	    if ($macrovalue =~ s/^(.*?)(?<!\\)\}/$1/) {
+		&addmacro($pack, \$macroname, \$macrovalue);
+	    }		
+	} elsif ($macroname ne "") {
+	    if ($line =~ /^(.*?)(?<!\\)\}/) {
+		$macrovalue .= $1 if defined $1;
+		&addmacro($pack, \$macroname, \$macrovalue);
+	    } else {
+		$macrovalue .= $line;
+	    }
+	} elsif ($line =~ /^\#\# \"([^\"]+)\" \#\# \w+ \#\# (\w+) \#\#/) {
+	    my $imagename = "image:$2";
+	    my $imagetext = $1;
+	    &addmacro($pack, \$imagename, \$imagetext);
+	}
+    }
+    close FILE;
+}
+
+sub addmacro {
+    my ($pack, $nameref, $valref) = @_;
+
+    if ($$nameref !~ /^(httpicon|width|height)/) {
+
+	my $name = "$pack:$$nameref";
+	
+	if (defined ($texthash->{'macros'}->{$name})) {
+	    print STDERR "ERROR: $name already defined\n";	
+	}
+
+
+	$$valref =~ s/\n/\\n/gs;
+	$$valref =~ s/\t/\\t/gs;
+
+	$texthash->{'macros'}->{$name} = $$valref;
+    }
+
+    else {
+#	print STDERR "ignoring $$nameref\n";
+    }
+
+    $$nameref = "";
+    $$valref = "";
+}
Index: /main/tags/3.01/gsdl/bin/script/filecopy.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/filecopy.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/filecopy.pl	(revision 21112)
@@ -0,0 +1,184 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# filecopy.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will "download" the specified files/directories
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use util;
+use parsargv;
+use File::stat;
+use File::Basename;
+use FileHandle;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "filecopy.pl: Recursively copies files into a collections import directory.\n\n";
+    print STDERR "\n  usage: $0 [options] [directories] collection-name\n\n";
+
+    print STDERR "  options:\n";
+    print STDERR "   -follow_links           Follow symbolic links when recursing directory\n";
+    print STDERR "                           structure\n";
+    print STDERR "   -collectdir directory   Collection directory (defaults to " .
+	&util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n";
+    print STDERR "   -out                    Filename or handle to print output status to.\n";
+    print STDERR "                           The default is STDERR\n\n";
+}
+
+sub get_file_list {
+    my $dirhash = shift @_;
+    my $filehash = shift @_;
+    my $dirname = shift @_;
+
+    my $full_importname 
+	= &util::filename_cat($collectdir, $dirname, "import");
+
+    # check for .kill file
+    if (-e &util::filename_cat($collectdir, $dirname, ".kill")) {
+	print $out "filecopy.pl killed by .kill file\n";
+	die "\n";
+    }
+
+    foreach my $file (@_) {
+
+	$file =~ s/^\"//;
+	$file =~ s/\"$//;
+
+	if (!$follow_links && -l $file) {
+	    # do nothing as we don't want to follow symbolic links
+
+	} elsif (-d $file) {
+	    my $dst_dir = &get_dst_dir ($full_importname, $file);
+	    # add this directory to the list to be created
+	    $dirhash->{$dst_dir} = 1;
+
+	    # read in dir
+            if (!opendir (DIR, $file)) {
+                print $out "Error: Could not open directory $file\n";
+            } else {
+                my @sub_files = grep (!/^\.\.?$/, readdir (DIR));
+                closedir DIR;
+		map { $_ = &util::filename_cat($file, $_); } @sub_files;
+		&get_file_list($dirhash, $filehash, $dirname, @sub_files);
+	    }
+
+	} else {
+	    my $dst_file = &get_dst_dir ($full_importname, $file);
+
+	    # make sure files directory is included in dirhash
+	    $dirhash->{File::Basename::dirname($dst_file)} = 1;
+
+	    if (-e $dst_file) {
+		# if destination file exists already we'll only copy it if
+		# the source file is newer
+		my $src_stat = stat($file);
+		my $dst_stat = stat($dst_file);
+		$filehash->{$file} = $dst_file if ($src_stat->mtime > $dst_stat->mtime);
+	    } else {
+		$filehash->{$file} = $dst_file;
+	    }
+	}
+    }
+}
+
+
+sub main {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'follow_links', \$follow_links,
+			 'collectdir/.*/', \$collectdir,
+			 'out/.*/STDERR', \$out)) {
+	&print_usage();
+	die "\n";
+    }
+
+    if ($collectdir !~ /\w/) {
+	$collectdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect");
+    }
+
+    my $collection = pop @ARGV;
+
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") || die "Couldn't open output file $out\n";
+	$out = OUT;
+	$close_out = 1;
+    }
+    $out->autoflush(1);
+
+    # first compile a list of all the files we want to copy (we do it this
+    # way rather than simply copying the files as we recurse the directory
+    # structure to avoid nasty infinite recursion if the directory we're
+    # copying to happens to be a subdirectory of one of the directories
+    # we're copying from)
+    my $dirhash = {};
+    my $filehash = {};
+    &get_file_list($dirhash, $filehash, $collection, @ARGV);
+
+    # create all the required destination directories
+    my $count = 0;
+    foreach my $dir (keys %$dirhash) {
+	# check for .kill file
+	if (($count ++ % 20 == 0) &&
+	    (-e &util::filename_cat($collectdir, $collection, ".kill"))) {
+	    print $out "filecopy.pl killed by .kill file\n";
+	    die "\n";
+	}
+	&util::mk_all_dir($dir);
+    }
+
+    # copy all the files
+    foreach my $file (keys %$filehash) {
+	# check for .kill file
+	if (($count ++ % 20 == 0) &&
+	    (-e &util::filename_cat($collectdir, $collection, ".kill"))) {
+	    print $out "filecopy.pl killed by .kill file\n";
+	    die "\n";
+	}
+	print $out "copying $file --> $filehash->{$file}\n";
+	&util::cp($file, $filehash->{$file});
+    }
+
+    close OUT if $close_out;
+    return 0;
+}
+
+sub get_dst_dir {
+    my ($full_importname, $dir) = @_;
+
+    if ($ENV{'GSDLOS'} eq "windows") {
+	# don't want windows filenames like c:\gsdl\...\import\c:\dir
+	$dir =~ s/^[a-z]:[\\\/]//i;
+    }
+    return &util::filename_cat($full_importname, $dir);
+}
+
+&main();
Index: /main/tags/3.01/gsdl/bin/script/gimp/flash_button-1.2.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/flash_button-1.2.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/flash_button-1.2.pl	(revision 21112)
@@ -0,0 +1,382 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# flash_button.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Modified by Rachid Ben Kaddour <qben@oce.nl> for gimp1.2
+
+# flash_button.pl generates all the flashy javascript type
+# buttons used by Greenstone
+# these are the buttons described in macro files as:
+# top_nav_button
+# nav_bar_button
+# document_button
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp qw/:auto :DEFAULT/;
+use parsargv;
+use util;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_yellow = "#D2B464";
+my $black = "#000000";
+
+my ($cfg_file, $width, $height, $text, $filenamestem, $fixed_width,
+    $width_space, $whitespace, $dont_center, $bgcolor, $fontcolor, 
+    $fontsize, $fontfoundry, $fontname, $fontweight, $fontslant, $fontwidth, 
+    $fontspacing, $fontregistry, $fontencoding, $image_dir);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file        configuration file containing one or more\n";
+    print STDERR "                         sets of the following options - use to create\n";
+    print STDERR "                         batches of images\n";
+    print STDERR "   -image_dir directory  directory to create images in [`pwd`]\n";
+    print STDERR "                         this should be full path to existing directory\n";
+    print STDERR "   -width number         width of button [65]\n";
+    print STDERR "   -height number        height of button [30]\n";
+    print STDERR "   -text string          button text\n";
+    print STDERR "   -filenamestem string  filename - two images will be produced -\n";
+    print STDERR "                         filenamestemon.gif and filenamestemof.gif\n";
+    print STDERR "   -fixed_width          fix width of image - if this isn't set\n";
+    print STDERR "                         image will be cropped with with width_space\n";
+    print STDERR "                         space at each side of text\n";
+    print STDERR "   -width_space number   width in pixels of blank space to leave at left\n";
+    print STDERR "                         and right edges of text [1]\n";
+    print STDERR "   -whitespace           add 1 pixel of white space at left and right edges\n";
+    print STDERR "                         of image\n";
+    print STDERR "   -dont_center          don't center text horizontally (only applicable if\n";
+    print STDERR "                         image is fixed width\n";
+    print STDERR "   -bgcolor hex_value    background color of button [$gsdl_yellow]\n";
+    print STDERR "   -fontcolor hex_value  text color [$black]\n";
+    print STDERR "   -fontsize number      font point size [10]\n";
+    print STDERR "   -foundry string       [*]\n";
+    print STDERR "   -fontname string      [lucida]\n";
+    print STDERR "   -fontweight string    [medium]\n";
+    print STDERR "   -fontslant            [r]\n";
+    print STDERR "   -fontwidth            [*]\n";
+    print STDERR "   -fontspacing          [*]\n\n";
+    print STDERR "   -fontregistry         [*]\n\n";
+    print STDERR "   -fontencoding         [*]\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $width = 65;
+    $height = 30;
+    $text = "";
+    $filenamestem = "";
+    $fixed_width = 0;
+    $width_space = 1;
+    $whitespace = 0;
+    $dont_center = 0;
+    $bgcolor = $gsdl_yellow;
+    $fontcolor = $black;
+    $fontsize = 10;
+    $fontfoundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+    $fontregistry = "*";
+    $fontencoding = "*";
+}
+
+sub gsdl_flash_button {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'image_dir/.*/./', \$image_dir,
+			 'width/^\d+$/65', \$width,
+			 'height/^\d+$/30', \$height,
+			 'text/.*/', \$text,
+			 'filenamestem/.*', \$filenamestem,
+			 'fixed_width', \$fixed_width,
+			 'width_space/^\d+$/1', \$width_space,
+			 'whitespace', \$whitespace,
+			 'dont_center', \$dont_center,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_yellow", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/10', \$fontsize,
+			 'foundry/.*/*', \$fontfoundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontspacing/.*/*', \$fontspacing,
+			 'fontregistry/.*/*', \$fontregistry,
+			 'fontencoding/.*/*', \$fontencoding)) {
+	&print_usage();
+	die "flash_button.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+
+    # replace any '\n' occurring in text with carriage return
+    $text =~ s/\\n/\n/gi;
+    
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filenamestem !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_images (0);
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_images (0);
+
+    }
+}
+
+sub produce_images {
+    my ($off_img) = @_;
+    
+    # create image, set background color etc.
+    my ($image, $backlayer) = &create_image ($off_img);
+    
+    # set the text if there is any
+    if (length($text)) {
+
+	my $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				   $fontsize, PIXELS, $fontfoundry, $fontname, $fontweight, 
+				   $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding);
+
+	my $textwidth = gimp_drawable_width($textlayer);
+	my $textheight = gimp_drawable_height($textlayer);
+
+	# check that text fits within image
+	if ($textheight > $height) {
+	    die "'$text' at fontsize of $fontsize pixels does not fit within image\n" . 
+		"$height pixels high. Decrease fontsize or increase image height\n";
+	}
+
+	my $spacers = $width_space * 2;
+	$spacers += 2 if $whitespace;
+
+	if (!$fixed_width || $textwidth > $width) {
+
+	    if ($fixed_width) {
+		print STDERR "WARNING (flash_button.pl): '$text' does not fit within $width pixel fixed width ";
+		print STDERR "image. Image width was increased to ",$textwidth + $spacers, " pixels\n";
+	    }
+	    
+	    $width = $textwidth + $spacers;
+
+	    # recreate image in new size
+	    ($image, $backlayer) = &create_image ($off_img);
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				    $fontsize, PIXELS, $fontfoundry, $fontname, $fontweight, 
+				    $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding);
+	    
+	}
+
+	my $y_offset = ($height-$textheight)-int($fontsize/5);
+	my $halfdescenders = "";
+
+	# Russian half descenders (KOI8-R)
+	# -- Uncomment if generating Russian images using KOI8-R encoded text --
+	# $halfdescenders .= chr(195) . chr(253);  # Checked
+	# $halfdescenders .= chr(196) . chr(198) . chr(221) . chr(227) . chr(228); # Unchecked
+
+	# Kazakh (and Russian) half descenders (Helvetica Kazakh font)
+	# -- Uncomment if generating Kazakh or Russian images using text encoded to match
+	#      a Kazakh Helvetica font:
+        #        http://www.unesco.kz/ci/projects/greenstone/kazakh_fonts/helv_k.ttf
+	#    Mapping for this encoding is: mapping/from_uc/kazakh.ump --
+	# $halfdescenders .= chr(178) . chr(179) . chr(187);             # Kazakh specific
+	# $halfdescenders .= chr(196) . chr(214) . chr(217) . chr(228);  # Generic Russian
+	# $halfdescenders .= chr(244) . chr(246) . chr(249);
+
+	# -- Uncomment if generating images using a font with half descenders --
+	# if ($text =~ /[$halfdescenders]/) {
+	    # half descenders - put text at fontsize/10 pixels above bottom
+	    # $y_offset = ($height-$textheight)-int($fontsize/10);
+	# }
+
+	my $descenders = "";
+
+	# Russian descenders (KOI8-R)
+	# -- Uncomment if generating Russian images using KOI8-R encoded text --
+	# $descenders .= chr(210) . chr(213);
+
+	# Kazakh (and Russian) descenders (Helvetica Kazakh font)
+	# -- Uncomment if generating Kazakh or Russian images using text encoded to match
+	#      a Kazakh Helvetica font:
+        #        http://www.unesco.kz/ci/projects/greenstone/kazakh_fonts/helv_k.ttf
+	#    Mapping for this encoding is: mapping/from_uc/kazakh.ump --
+	# $descenders .= chr(189) . chr(190);  # Kazakh specific
+	# $descenders .= chr(240) . chr(243);  # Generic Russian
+
+	if ($text =~ /[gjpqyJ$descenders]/) { ## capital J is a descender in lucida font
+	    # descenders - put text at bottom of image, otherwise 
+	    # go for fontsize/5 pixels above bottom. This is kind of hacky 
+	    # and may need some playing with for different fonts/fontsizes
+	    $y_offset = $height-$textheight;
+	}
+
+	if ($fixed_width) {
+	    if ($dont_center) {
+		# don't center text horizontally (start it width_space pixels from left edge)
+		my $x_offset = $width_space;
+		$x_offset += 1 if $whitespace;
+		gimp_layer_set_offsets ($textlayer, $x_offset, $y_offset);
+	    } else {
+		gimp_layer_set_offsets ($textlayer, ($width-$textwidth)/2, $y_offset);
+	    }
+	} else {
+	    gimp_layer_set_offsets ($textlayer, $spacers / 2, $y_offset); 
+	}
+    }
+    
+    # add pixel of whitespace to each edge if required
+    if ($whitespace) {
+	gimp_rect_select ($image, 0, 0, 1, $height, 0, 0, 0);
+	gimp_edit_clear ($backlayer);
+	gimp_selection_none ($image);
+	gimp_rect_select ($image, $width-1, 0, 1, $height, 0, 0, 0);
+	gimp_edit_clear ($backlayer);
+	gimp_selection_none ($image);
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+    
+    # make indexed colour 
+    gimp_convert_indexed ($image, NO_DITHER, MAKE_PALETTE, 256, 0, 1,"");
+    
+    # save image
+    my $filename = $filenamestem . "on.gif";
+    $filename = $filenamestem . "of.gif" if $off_img;
+    $filename = &util::filename_cat ($image_dir, $filename);
+
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+
+    &produce_images (1) unless $off_img;    
+}
+
+sub create_image {
+    my ($off_img) = @_;
+
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+    
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+    
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+    
+    # set colour of background
+    gimp_palette_set_foreground ($bgcolor);
+    
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($backlayer);
+    gimp_selection_none ($image);
+    
+    # create the gradient background
+    gimp_blend ($backlayer, 0, NORMAL_MODE, LINEAR, 70, 0, REPEAT_NONE, 0, 0, 0, 5, $height-3, 5, 0);
+    
+    # adjust lightness of "off" gif
+    gimp_hue_saturation ($backlayer, 0, 0, 100, 0) if $off_img;
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+
+    return ($image, $backlayer);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_flash_button", "create flashy javascript buttons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_flash_button", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+Gimp::on_net { gsdl_flash_button; };
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/gimp/flash_button.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/flash_button.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/flash_button.pl	(revision 21112)
@@ -0,0 +1,351 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# flash_button.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# flash_button.pl generates all the flashy javascript type
+# buttons used by Greenstone
+# these are the buttons described in macro files as:
+# top_nav_button
+# nav_bar_button
+# document_button
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp;
+use parsargv;
+use util;
+use unicode;
+
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_yellow = "#D2B464";
+my $black = "#000000";
+
+my ($cfg_file, $width, $height, $text, $filenamestem, $fixed_width,
+    $width_space, $whitespace, $dont_center, $bgcolor, $fontcolor, 
+    $fontsize, $foundry, $fontname, $fontweight, $fontslant, $fontwidth, 
+    $fontspacing, $image_dir);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file        configuration file containing one or more\n";
+    print STDERR "                         sets of the following options - use to create\n";
+    print STDERR "                         batches of images\n";
+    print STDERR "   -image_dir directory  directory to create images in [`pwd`]\n";
+    print STDERR "                         this should be full path to existing directory\n";
+    print STDERR "   -width number         width of button [65]\n";
+    print STDERR "   -height number        height of button [30]\n";
+    print STDERR "   -text string          button text\n";
+    print STDERR "   -filenamestem string  filename - two images will be produced -\n";
+    print STDERR "                         filenamestemon.gif and filenamestemof.gif\n";
+    print STDERR "   -fixed_width          fix width of image - if this isn't set\n";
+    print STDERR "                         image will be cropped with with width_space\n";
+    print STDERR "                         space at each side of text\n";
+    print STDERR "   -width_space number   width in pixels of blank space to leave at left\n";
+    print STDERR "                         and right edges of text [1]\n";
+    print STDERR "   -whitespace           add 1 pixel of white space at left and right edges\n";
+    print STDERR "                         of image\n";
+    print STDERR "   -dont_center          don't center text horizontally (only applicable if\n";
+    print STDERR "                         image is fixed width\n";
+    print STDERR "   -bgcolor hex_value    background color of button [$gsdl_yellow]\n";
+    print STDERR "   -fontcolor hex_value  text color [$black]\n";
+    print STDERR "   -fontsize number      font point size [10]\n";
+    print STDERR "   -foundry string       [*]\n";
+    print STDERR "   -fontname string      [lucida]\n";
+    print STDERR "   -fontweight string    [medium]\n";
+    print STDERR "   -fontslant            [r]\n";
+    print STDERR "   -fontwidth            [*]\n";
+    print STDERR "   -fontspacing          [*]\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $width = 65;
+    $height = 30;
+    $text = "";
+    $filenamestem = "";
+    $fixed_width = 0;
+    $width_space = 1;
+    $whitespace = 0;
+    $dont_center = 0;
+    $bgcolor = $gsdl_yellow;
+    $fontcolor = $black;
+    $fontsize = 10;
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+}
+
+sub gsdl_flash_button {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'image_dir/.*/./', \$image_dir,
+			 'width/^\d+$/65', \$width,
+			 'height/^\d+$/30', \$height,
+			 'text/.*/', \$text,
+			 'filenamestem/.*', \$filenamestem,
+			 'fixed_width', \$fixed_width,
+			 'width_space/^\d+$/1', \$width_space,
+			 'whitespace', \$whitespace,
+			 'dont_center', \$dont_center,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_yellow", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/10', \$fontsize,
+			 'foundry/.*/*', \$foundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontspacing/.*/*', \$fontspacing)) {
+	&print_usage();
+	die "flash_button.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+
+    # replace any '\n' occurring in text with carriage return
+    $text =~ s/\\n/\n/gi;
+
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filenamestem !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_images (0);
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_images (0);
+
+    }
+}
+
+sub produce_images {
+    my ($off_img) = @_;
+    
+    # create image, set background color etc.
+    my ($image, $backlayer) = &create_image ($off_img);
+    
+    # set the text if there is any
+    if (length($text)) {
+
+	my $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				   $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				   $fontslant, $fontwidth, $fontspacing);
+	
+
+	my $textwidth = gimp_drawable_width($textlayer);
+	my $textheight = gimp_drawable_height($textlayer);
+
+	# check that text fits within image
+	if ($textheight > $height) {
+	    die "'$text' at fontsize of $fontsize pixels does not fit within image\n" . 
+		"$height pixels high. Decrease fontsize or increase image height\n";
+	}
+
+	my $spacers = $width_space * 2;
+	$spacers += 2 if $whitespace;
+
+	if (!$fixed_width || $textwidth > $width) {
+
+	    if ($fixed_width) {
+		print STDERR "WARNING (flash_button.pl): '$text' does not fit within $width pixel fixed width ";
+		print STDERR "image. Image width was increased to ",$textwidth + $spacers, " pixels\n";
+	    }
+	    
+	    $width = $textwidth + $spacers;
+
+	    # recreate image in new size
+	    ($image, $backlayer) = &create_image ($off_img);
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				    $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				    $fontslant, $fontwidth, $fontspacing);
+	    
+	}
+
+	my $y_offset = ($height-$textheight)-int($fontsize/5);
+
+	my $descenders = "";
+
+	# russian descenders (KOI8-R)
+	# $descenders .= chr(0xD2);
+	# $descenders .= chr(0xD5);
+
+	if ($text =~ /[gjpqyJ$descenders]/) { ## capital J is a descender in lucida font
+	    # descenders - put text at bottom of image, otherwise 
+	    # go for fontsize/5 pixels above bottom. This is kind of hacky 
+	    # and may need some playing with for different fonts/fontsizes
+	    $y_offset = $height-$textheight;
+	}
+
+	if ($fixed_width) {
+	    if ($dont_center) {
+		# don't center text horizontally (start it width_space pixels from left edge)
+		my $x_offset = $width_space;
+		$x_offset += 1 if $whitespace;
+		gimp_layer_set_offsets ($textlayer, $x_offset, $y_offset);
+	    } else {
+		gimp_layer_set_offsets ($textlayer, ($width-$textwidth)/2, $y_offset);
+	    }
+	} else {
+	    gimp_layer_set_offsets ($textlayer, $spacers / 2, $y_offset); 
+	}
+    }
+    
+    # add pixel of whitespace to each edge if required
+    if ($whitespace) {
+	gimp_rect_select ($image, 0, 0, 1, $height, 0, 0, 0);
+	gimp_edit_clear ($image, $backlayer);
+	gimp_selection_none ($image);
+	gimp_rect_select ($image, $width-1, 0, 1, $height, 0, 0, 0);
+	gimp_edit_clear ($image, $backlayer);
+	gimp_selection_none ($image);
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+    
+    # make indexed colour 
+    gimp_convert_indexed ($image, 0, 256);
+    
+    # save image
+    my $filename = $filenamestem . "on.gif";
+    $filename = $filenamestem . "of.gif" if $off_img;
+    $filename = &util::filename_cat ($image_dir, $filename);
+
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+
+    &produce_images (1) unless $off_img;    
+}
+
+sub create_image {
+    my ($off_img) = @_;
+
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+    
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+    
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+    
+    # set colour of background
+    gimp_palette_set_foreground ($bgcolor);
+    
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($image, $backlayer);
+    gimp_selection_none ($image);
+    
+    # create the gradient background
+    gimp_blend ($image, $backlayer, 0, NORMAL_MODE, LINEAR, 70, 0, REPEAT_NONE, 0, 0, 0, 5, $height-3, 5, 0);
+    
+    # adjust lightness of "off" gif
+    gimp_hue_saturation ($image, $backlayer, 0, 0, 100, 0) if $off_img;
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+
+    return ($image, $backlayer);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_flash_button", "create flashy javascript buttons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_flash_button", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+sub net {
+  gsdl_flash_button;
+}
+
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/gimp/green_bar-1.2.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/green_bar-1.2.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/green_bar-1.2.pl	(revision 21112)
@@ -0,0 +1,342 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# green_bar.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Modified by Rachid Ben Kaddour <qben@oce.nl> for gimp1.2
+
+# green_bar.pl generates all the black on green gradient background
+# images used by Greenstone. 
+# these are the icons described in macro files as:
+# green version of nav_bar_button
+# green_bar_left_aligned
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp qw/:auto :DEFAULT/;
+use parsargv;
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black = "#000000";
+
+my ($cfg_file, $width, $height, $text, $filename, $width_space, 
+    $dont_center, $bgcolor, $fontcolor, $fontsize, $foundry, $fontname, 
+    $fontweight, $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding, $image_dir);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] macrofile\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file        configuration file containing one or more\n";
+    print STDERR "                         sets of the following options - use to create\n";
+    print STDERR "                         batches of images\n";
+    print STDERR "   -image_dir directory  directory to create images in [`pwd`]\n";
+    print STDERR "                         this should be full path to existing directory\n";
+    print STDERR "   -width number         width of bar [87]\n";
+    print STDERR "   -height number        height of bar [17]\n";
+    print STDERR "   -text string          icon text\n";
+    print STDERR "   -filename string      filename of resulting image\n";
+    print STDERR "   -width_space number   width in pixels of blank space to leave at left\n";
+    print STDERR "                         and right edges of text [1]\n";
+    print STDERR "   -dont_center          don't center text horizontally\n";
+    print STDERR "   -bgcolor hex_value    background color of bar [$gsdl_green]\n";
+    print STDERR "   -fontcolor hex_value  text color [$black]\n";
+    print STDERR "   -fontsize number      font point size [17]\n";
+    print STDERR "   -foundry string       [*]\n";
+    print STDERR "   -fontname string      [lucida]\n";
+    print STDERR "   -fontweight string    [medium]\n";
+    print STDERR "   -fontslant            [r]\n";
+    print STDERR "   -fontwidth            [*]\n";
+    print STDERR "   -fontspacing          [*]\n";
+    print STDERR "   -fontregistry         [*]\n";
+    print STDERR "   -fontencoding         [*]\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $width = 87;
+    $height = 17;
+    $text = "";
+    $filename = "";
+    $width_space = 1;
+    $dont_center = 0;
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = 17;
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+    $fontregistry = "*";
+    $fontencoding = "*";
+}
+
+sub gsdl_green_bar {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'image_dir/.*/./', \$image_dir,
+			 'width/^\d+$/87', \$width,
+			 'height/^\d+$/17', \$height,
+			 'text/.*/', \$text,
+			 'filename/.*', \$filename,
+			 'width_space/^\d+$/1', \$width_space,
+			 'dont_center', \$dont_center,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/17', \$fontsize,
+			 'foundry/.*/*', \$foundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontregistry/.*/*', \$fontregistry,
+			 'fontencoding/.*/*', \$fontencoding,
+			 'fontspacing/.*/*', \$fontspacing)) {
+	&print_usage();
+	die "green_bar.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+    
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_image ();
+
+    }
+}
+
+sub produce_image {
+    
+    # create image, set background color etc.
+    my ($image, $backlayer) = &create_image ();
+    
+    # set the text if there is any
+    if (length($text)) {
+
+	my $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				   $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				   $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding);
+	
+
+	my $textwidth = gimp_drawable_width($textlayer);
+	my $textheight = gimp_drawable_height($textlayer);
+
+	# check that text fits within image
+	if ($textheight > $height) {
+	    die "'$text' at fontsize of $fontsize pixels does not fit within image\n" . 
+		"$height pixels high. Decrease fontsize or increase image height\n";
+	}
+
+	my $spacers = $width_space * 2;
+
+	if ($textwidth > $width) {
+
+	    print STDERR "WARNING (green_bar.pl): '$text' does not fit within $width pixel fixed width ";
+	    print STDERR "image. Image width was increased to ",$textwidth + $spacers, " pixels\n";
+	    
+	    $width = $textwidth + $spacers;
+
+	    # recreate image in new size
+	    ($image, $backlayer) = &create_image ();
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				    $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				    $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding);
+	    
+	}
+
+	my $y_offset = ($height-$textheight)-int($fontsize/5);
+
+	my $halfdescenders = "";
+
+	# Russian half descenders (KOI8-R)
+	# -- Uncomment if generating Russian images using KOI8-R encoded text --
+	# $halfdescenders .= chr(195) . chr(253);  # Checked
+	# $halfdescenders .= chr(196) . chr(198) . chr(221) . chr(227) . chr(228); # Unchecked
+
+	# Kazakh (and Russian) half descenders (Helvetica Kazakh font)
+	# -- Uncomment if generating Kazakh or Russian images using text encoded to match
+	#      a Kazakh Helvetica font:
+        #        http://www.unesco.kz/ci/projects/greenstone/kazakh_fonts/helv_k.ttf
+	#    Mapping for this encoding is: mapping/from_uc/kazakh.ump --
+	# $halfdescenders .= chr(178) . chr(179) . chr(187);             # Kazakh specific
+	# $halfdescenders .= chr(196) . chr(214) . chr(217) . chr(228);  # Generic Russian
+	# $halfdescenders .= chr(244) . chr(246) . chr(249);
+
+	# -- Uncomment if generating images using a font with half descenders --
+	# if ($text =~ /[$halfdescenders]/) {
+	    # half descenders - put text at fontsize/10 pixels above bottom
+	    # $y_offset = ($height-$textheight)-int($fontsize/10);
+	# }
+
+	my $descenders = "";
+
+	# Russian descenders (KOI8-R)
+	# -- Uncomment if generating Russian images using KOI8-R encoded text --
+	# $descenders .= chr(210) . chr(213);
+
+	# Kazakh (and Russian) descenders (Helvetica Kazakh font)
+	# -- Uncomment if generating Kazakh or Russian images using text encoded to match
+	#      a Kazakh Helvetica font:
+        #        http://www.unesco.kz/ci/projects/greenstone/kazakh_fonts/helv_k.ttf
+	#    Mapping for this encoding is: mapping/from_uc/kazakh.ump --
+	# $descenders .= chr(189) . chr(190);  # Kazakh specific
+	# $descenders .= chr(240) . chr(243);  # Generic Russian
+
+	if ($text =~ /[gjpqyJ$descenders]/) { ## capital J is a descender in lucida font
+	    # descenders - put text at bottom of image, otherwise 
+	    # go for fontsize/5 pixels above bottom. This is kind of hacky 
+	    # and may need some playing with for different fonts/fontsizes
+	    $y_offset = $height-$textheight;
+	}
+
+	if ($dont_center) {
+	    # don't center text horizontally (start it width_space pixels from left edge)
+	    my $x_offset = $width_space;
+	    gimp_layer_set_offsets ($textlayer, $x_offset, $y_offset);
+	} else {
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)/2, $y_offset);
+	}
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+    
+    # make indexed colour 
+    gimp_convert_indexed ($image, NO_DITHER, MAKE_PALETTE, 256, 0, 1,"");
+    
+    # save image
+    $filename = &util::filename_cat ($image_dir, $filename);
+
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+
+}
+
+sub create_image {
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+    
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+    
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+    
+    # set colour of background
+    gimp_palette_set_foreground ($bgcolor);
+    
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($backlayer);
+    gimp_selection_none ($image);
+    
+    # create the gradient background
+    gimp_blend ($backlayer, 0, NORMAL_MODE, LINEAR, 70, 0, REPEAT_NONE, 0, 0, 0, 5, $height-3, 5, 0);
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+
+    return ($image, $backlayer);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_green_bar", "create green bar icons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_green_bar", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+Gimp::on_net { gsdl_green_bar; };
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/gimp/green_bar.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/green_bar.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/green_bar.pl	(revision 21112)
@@ -0,0 +1,307 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# green_bar.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# green_bar.pl generates all the black on green gradient background
+# images used by Greenstone. 
+# these are the icons described in macro files as:
+# green version of nav_bar_button
+# green_bar_left_aligned
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp;
+use parsargv;
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black = "#000000";
+
+my ($cfg_file, $width, $height, $text, $filename, $width_space, 
+    $dont_center, $bgcolor, $fontcolor, $fontsize, $foundry, $fontname, 
+    $fontweight, $fontslant, $fontwidth, $fontspacing, $image_dir);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] macrofile\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file        configuration file containing one or more\n";
+    print STDERR "                         sets of the following options - use to create\n";
+    print STDERR "                         batches of images\n";
+    print STDERR "   -image_dir directory  directory to create images in [`pwd`]\n";
+    print STDERR "                         this should be full path to existing directory\n";
+    print STDERR "   -width number         width of bar [87]\n";
+    print STDERR "   -height number        height of bar [17]\n";
+    print STDERR "   -text string          icon text\n";
+    print STDERR "   -filename string      filename of resulting image\n";
+    print STDERR "   -width_space number   width in pixels of blank space to leave at left\n";
+    print STDERR "                         and right edges of text [1]\n";
+    print STDERR "   -dont_center          don't center text horizontally\n";
+    print STDERR "   -bgcolor hex_value    background color of bar [$gsdl_green]\n";
+    print STDERR "   -fontcolor hex_value  text color [$black]\n";
+    print STDERR "   -fontsize number      font point size [17]\n";
+    print STDERR "   -foundry string       [*]\n";
+    print STDERR "   -fontname string      [lucida]\n";
+    print STDERR "   -fontweight string    [medium]\n";
+    print STDERR "   -fontslant            [r]\n";
+    print STDERR "   -fontwidth            [*]\n";
+    print STDERR "   -fontspacing          [*]\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $width = 87;
+    $height = 17;
+    $text = "";
+    $filename = "";
+    $width_space = 1;
+    $dont_center = 0;
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = 17;
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+}
+
+sub gsdl_green_bar {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'image_dir/.*/./', \$image_dir,
+			 'width/^\d+$/87', \$width,
+			 'height/^\d+$/17', \$height,
+			 'text/.*/', \$text,
+			 'filename/.*', \$filename,
+			 'width_space/^\d+$/1', \$width_space,
+			 'dont_center', \$dont_center,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/17', \$fontsize,
+			 'foundry/.*/*', \$foundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontspacing/.*/*', \$fontspacing)) {
+	&print_usage();
+	die "green_bar.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+    
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_image ();
+
+    }
+}
+
+sub produce_image {
+    
+    # create image, set background color etc.
+    my ($image, $backlayer) = &create_image ();
+    
+    # set the text if there is any
+    if (length($text)) {
+
+	my $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				   $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				   $fontslant, $fontwidth, $fontspacing);
+	
+
+	my $textwidth = gimp_drawable_width($textlayer);
+	my $textheight = gimp_drawable_height($textlayer);
+
+	# check that text fits within image
+	if ($textheight > $height) {
+	    die "'$text' at fontsize of $fontsize pixels does not fit within image\n" . 
+		"$height pixels high. Decrease fontsize or increase image height\n";
+	}
+
+	my $spacers = $width_space * 2;
+
+	if ($textwidth > $width) {
+
+	    print STDERR "WARNING (green_bar.pl): '$text' does not fit within $width pixel fixed width ";
+	    print STDERR "image. Image width was increased to ",$textwidth + $spacers, " pixels\n";
+	    
+	    $width = $textwidth + $spacers;
+
+	    # recreate image in new size
+	    ($image, $backlayer) = &create_image ();
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				    $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				    $fontslant, $fontwidth, $fontspacing);
+	    
+	}
+
+	my $y_offset = ($height-$textheight)-int($fontsize/5);
+
+	my $descenders = "";
+
+	# russian descenders (KOI8-R)
+	# $descenders .= chr(0xD2);
+	# $descenders .= chr(0xD5);
+
+	if ($text =~ /[gjpqyJ$descenders]/) { ## capital J is a descender in lucida font
+	    # descenders - put text at bottom of image, otherwise 
+	    # go for fontsize/5 pixels above bottom. This is kind of hacky 
+	    # and may need some playing with for different fonts/fontsizes
+	    $y_offset = $height-$textheight;
+	}
+
+	if ($dont_center) {
+	    # don't center text horizontally (start it width_space pixels from left edge)
+	    my $x_offset = $width_space;
+	    gimp_layer_set_offsets ($textlayer, $x_offset, $y_offset);
+	} else {
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)/2, $y_offset);
+	}
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+    
+    # make indexed colour 
+    gimp_convert_indexed ($image, 0, 256);
+    
+    # save image
+    $filename = &util::filename_cat ($image_dir, $filename);
+
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+
+}
+
+sub create_image {
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+    
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+    
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+    
+    # set colour of background
+    gimp_palette_set_foreground ($bgcolor);
+    
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($image, $backlayer);
+    gimp_selection_none ($image);
+    
+    # create the gradient background
+    gimp_blend ($image, $backlayer, 0, NORMAL_MODE, LINEAR, 70, 0, REPEAT_NONE, 0, 0, 0, 5, $height-3, 5, 0);
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+
+    return ($image, $backlayer);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_green_bar", "create green bar icons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_green_bar", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+sub net {
+  gsdl_green_bar;
+}
+
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/gimp/green_bar_g12.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/green_bar_g12.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/green_bar_g12.pl	(revision 21112)
@@ -0,0 +1,309 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# green_bar.pl 
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# green_bar.pl 
+#
+# This script generates the black on green gradient background images
+# used by Greenstone.  these are the icons described in macro files as:
+# green version of nav_bar_button green_bar_left_aligned
+#
+# This version has been rewritten for the GIMP Version 1.2.  You
+# should be able to run it if you install gimp 1.2 with perl support.
+# In Debian this means running "apt-get install gimp1.2 gimp1.2perl"
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp ":auto";
+use Gimp::Fu; 
+
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black      = "#000000";
+my $white      = "#FFFFFF";
+
+my ($current_dir) = `pwd`;
+chomp($current_dir);
+
+my ($cfg_file, $width, $height, $text, $filename, $width_space, 
+    $alignment, $fgcolor, $bgcolor, $fontcolor, $fontsize, $foundry, $fontname, 
+    $fontweight, $fontslant, $fontwidth, $fontspacing, $image_dir);
+
+sub reset_options {
+    $image_dir = "./";
+    $width = 87;
+    $height = 17;
+    $text = "";
+    $filename = "";
+    $width_space = 1;
+    $alignment = "eft";
+    $fgcolor = $white;
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = 17;
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+}
+
+sub gsdl_green_bar {
+
+    ($text, $filename, $image_dir, $width, $height, $fgcolor, $bgcolor,
+    $fontcolor, $fontsize, $alignment, , $width_space, $foundry,
+    $fontname, $fontweight, $fontslant, $fontwidth, $fontspacing,
+    $cfg_file) = @_;
+
+    # Read a configuration file
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    }
+
+    # Produce an image based onlyon command-line parameters
+    else {
+
+	&produce_image ();
+
+    }
+}
+
+
+sub produce_image {
+
+    # Create background image
+    my ($image, $backlayer) = &create_image ();
+
+    # set the text if there is any
+    if (length($text)) {
+
+	my $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, 
+				   $fontsize, PIXELS, $foundry, $fontname, $fontweight, 
+				   $fontslant, $fontwidth, $fontspacing, "*", "*");
+	
+
+	my $textwidth = gimp_drawable_width($textlayer);
+	my $textheight = gimp_drawable_height($textlayer);
+
+	# check that text fits within image
+	if ($textheight > $height) {
+	    die "'$text' at fontsize of $fontsize pixels does not fit within image\n" . 
+		"$height pixels high. Decrease fontsize or increase image height\n";
+	}
+
+	my $spacers = $width_space * 2;
+
+	if ($textwidth > $width) {
+
+	    print STDERR "WARNING (green_bar.pl): '$text' does not fit within $width pixel fixed width ";
+	    print STDERR "image. Image width was increased to ",$textwidth + $spacers, " pixels\n";
+	
+	    $width = $textwidth + $spacers;
+
+	    # recreate image in new size
+	    ($image, $backlayer) = &create_image ();
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1,
+				    $fontsize, PIXELS, $foundry, $fontname, $fontweight,
+				    $fontslant, $fontwidth, $fontspacing, "*", "*");
+	
+	}
+
+	my $y_offset = ($height-$textheight)-int($fontsize/5);
+
+	my $descenders = "";
+
+	# russian descenders (KOI8-R)
+	# $descenders .= chr(0xD2);
+	# $descenders .= chr(0xD5);
+
+	if ($text =~ /[gjpqyJ$descenders]/) { ## capital J is a descender in lucida font
+	    # descenders - put text at bottom of image, otherwise 
+	    # go for fontsize/5 pixels above bottom. This is kind of hacky 
+	    # and may need some playing with for different fonts/fontsizes
+	    $y_offset = $height-$textheight;
+	}
+
+	if ($alignment =~ /^l/i) {
+	    # align text to the left
+	    my $x_offset = $width_space;
+	    gimp_layer_set_offsets ($textlayer, $x_offset, $y_offset);
+	} elsif ($alignment =~ /^r/i) {
+	    # right alignment
+	    gimp_layer_set_offsets ($textlayer, ($width-($textwidth+$width_space)), $y_offset);
+	} else {
+	    # center alignment (the default)
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)/2, $y_offset);
+	}
+    }
+
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+
+    # make indexed colour 
+    if ($filename =~ /\.gif$/i) {
+        # make indexed colour (may need to do this for 
+        # other formats as well as gif)
+        gimp_convert_indexed ($image, 0, MAKE_PALETTE, 256, 0, 0, "");
+    }
+
+    # save image
+    $filename = &util::filename_cat ($image_dir, $filename);
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+}
+
+
+sub create_image {
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+
+    # set colour of background
+    gimp_palette_set_foreground ($bgcolor);
+    gimp_palette_set_background ($fgcolor);
+
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($backlayer);
+    gimp_selection_none ($image);
+
+    # create the gradient background
+    gimp_blend ($backlayer, 0, NORMAL_MODE, LINEAR, 70, 0, REPEAT_NONE, 0, 0, 0, 5, $height-3, 5, 0);
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+
+    return ($image, $backlayer);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+
+
+register("gsdl_green_bar", 
+	 "Create green bar icons for gsdl (gimp version 1.2)",
+	 "",
+	 "Stefan Boddie (1.0) and Gordon Paynter (1.2)",
+	 "Copyright 2000 Stefan Boddie and The New Zealand Digital Library Project",
+	 "2000-07-05",
+	 "<Toolbox>/Xtns/gsdl_green_bar", 
+	 "*",
+	 [
+           [PF_STRING, "text",        "Text appearing on bar", "greenstone"],
+
+           [PF_STRING, "filename",    "Output file", "green_bar.png"],
+           [PF_STRING, "image_dir",   "Directory to create images in", $current_dir],
+
+           [PF_INT,    "width",      "Width of the bar",  640],
+           [PF_INT,    "height",     "Height of the bar", 17],
+           [PF_STRING, "fgcolor",   "Foreground colour (top) of bar", $white],
+           [PF_STRING, "bgcolor",   "Background colour (bottom) of bar", $gsdl_green],
+           [PF_STRING, "fontcolor", "Colour of text on bar", $black],
+           [PF_INT,    "fontsize",  "Font size",    17],
+
+           [PF_STRING, "alignment",     "Alignment of text on the bar (left, center or right)", "left"],
+           [PF_INT,    "padding_space", "Space around the text",                   5],
+
+           [PF_STRING, "foundry",     "Font foundry", "*"],
+           [PF_STRING, "fontname",    "Font name",    "lucida"],
+           [PF_STRING, "fontweight",  "Font weight",  "medium"],
+           [PF_STRING, "fontslant",   "Font slant",   "r"],
+           [PF_STRING, "fontwidth",   "Font width",   "*"],
+           [PF_STRING, "fontspacing", "Font spacing", "*"],
+
+           [PF_STRING, "cfg_file",    "Configuration file", ""]
+
+         ], 
+         \&gsdl_green_bar);
+
+
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/gimp/title_icon-1.2.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/title_icon-1.2.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/title_icon-1.2.pl	(revision 21112)
@@ -0,0 +1,407 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# title_icon.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Modified by Rachid Ben Kaddour <qben@oce.nl> for gimp1.2
+
+# title_icon.pl generates all the green_title type icons and 
+# collection icons for Greenstone
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp qw/:auto :DEFAULT/;
+use parsargv;
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black = "#000000";
+
+
+local ($cfg_file, $size, $imagefile, $width, $height, $imageheight, $stripecolor, $stripewidth, 
+       $stripe_alignment, $i_transparency, $text, $text_alignment, $filename, $textspace_x, 
+       $textspace_y, $bgcolor, $fontcolor, $fontsize, $minfontsize, $foundry, $fontname, 
+       $fontweight, $fontslant, $fontwidth, $fontspacing, $fontregistry, $fontencoding, $image_dir, $dont_wrap);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] macrofile\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file         configuration file containing one or more\n";
+    print STDERR "                          sets of the following options - use to create\n";
+    print STDERR "                          batches of images\n";
+    print STDERR "   -size number           the overall size ratio of the image (i.e. a size\n";
+    print STDERR "                          of 2 will create an image twice the default size)\n";
+    print STDERR "   -image_dir directory   directory to create images in [`pwd`]\n";
+    print STDERR "                          this should be full path to existing directory\n";
+    print STDERR "   -imagefile             filename of image to embed within new icon\n";
+    print STDERR "   -width number          width of icon [150]\n";
+    print STDERR "   -height number         height of icon [44]\n";
+    print STDERR "   -imageheight number    this is the height of the image if the image contains\n";
+    print STDERR "                          an image (like collection icons) [110]\n";
+    print STDERR "   -stripecolor hex_value color of vertical stripe [$gsdl_green]\n";
+    print STDERR "   -stripewidth           width of vertical stripe [40]\n";
+    print STDERR "   -stripe_alignment      alignment of vertical stripe (left or right) [left]\n";
+    print STDERR "   -i_transparency number transparency of image within icon (0 > transparent > 100) [60]\n"; 
+    print STDERR "   -text string           image text\n";
+    print STDERR "   -text_alignment        alignment of text (left or right) [left]\n";
+    print STDERR "   -filename string       filename of resulting image\n";
+    print STDERR "   -textspace_x number    space in pixels between left/right edge of image and\n";
+    print STDERR "                          left/right edge of text [3]\n";
+    print STDERR "   -textspace_y number    space in pixels between top of image and top of\n";
+    print STDERR "                          text [3]\n";
+    print STDERR "   -bgcolor hex_value     background color of icon [$gsdl_green]\n";
+    print STDERR "   -fontcolor hex_value   text color [$black]\n";
+    print STDERR "   -fontsize number       font point size [17]\n";
+    print STDERR "   -minfontsize number    minimum point size font will be reduced to fit image [10]\n";
+    print STDERR "   -foundry string        [*]\n";
+    print STDERR "   -fontname string       [lucida]\n";
+    print STDERR "   -fontweight string     [medium]\n";
+    print STDERR "   -fontslant             [r]\n";
+    print STDERR "   -fontwidth             [*]\n";
+    print STDERR "   -fontspacing           [*]\n";
+    print STDERR "   -fontregistry          [*]\n";
+    print STDERR "   -fontencoding          [*]\n";
+    print STDERR "   -dont_wrap             don't attempt to wrap text\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $imagefile = "";
+    $width = int (150 * $size);
+    $height = int (44 * $size);
+    $imageheight = int (110 * $size);
+    $stripecolor = $gsdl_green;
+    $stripewidth = int (40 * $size);
+    $stripe_alignment = "left";
+    $i_transparency = 60;
+    $text = "";
+    $text_alignment = "left";
+    $filename = "";
+    $textspace_x = int (3 * $size);
+    $textspace_y = int (3 * $size);
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = int (17 * $size);
+    $minfontsize = int (10 * $size);
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+    $fontregistry = "*";
+    $fontencoding = "*";
+}
+
+sub gsdl_title_icon {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'size/\d+/1', \$size,
+			 'image_dir/.*/./', \$image_dir,
+			 'imagefile/.*/', \$imagefile,
+			 'width/^\d+$/150', \$width,
+			 'height/^\d+$/44', \$height,
+			 'imageheight/^\d+$/110', \$imageheight,
+			 "stripecolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$stripecolor,
+			 'stripewidth/^\d+$/40', \$stripewidth,
+			 'stripe_alignment/^(left|right)$/left', \$stripe_alignment,
+			 'i_transparency/^\d+$/60', \$i_transparency,
+			 'text/.*/', \$text,
+			 'text_alignment/^(left|right)$/left', \$text_alignment,
+			 'filename/.*', \$filename,
+			 'textspace_x/^\d+$/3', \$textspace_x,
+			 'textspace_y/^\d+$/3', \$textspace_y,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/17', \$fontsize,
+			 'minfontsize/^\d+$/10', \$minfontsize,
+			 'foundry/.*/*', \$foundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontspacing/.*/*', \$fontspacing,
+			 'fontregistry/.*/*', \$fontregistry,
+			 'fontencoding/.*/*', \$fontencoding,
+			 'dont_wrap', \$dont_wrap)) {
+	&print_usage();
+	die "title_icon.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_image ();
+
+    }
+}
+
+sub produce_image {
+
+    &adjust_args ();
+    &wrap_text () unless $dont_wrap;
+
+    my $use_image = 0;
+    if ($imagefile =~ /\w/) {
+	if (!-r $imagefile) {
+	    print STDERR "WARNING (title_icon.pl): imagefile '$imagefile cannot be ";
+	    print STDERR "read - $filename will be created without the use of an image file\n";
+	} else {
+	    $use_image = 1;
+	    $height = $imageheight;
+	}
+    }
+
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+	
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+	
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+	
+    # set colour of stripe
+    gimp_palette_set_foreground ($stripecolor);
+	
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($backlayer);
+    gimp_selection_none ($image);
+	
+    # fill in stripe
+    if ($stripe_alignment eq "left") {
+	gimp_rect_select ($image, 0, 0, $stripewidth, $height, 0, 0, 0);
+    } else {
+	gimp_rect_select ($image, $width-$stripewidth, 0, $stripewidth, $height, 0, 0, 0);
+    }
+    gimp_bucket_fill ($backlayer, FG_BUCKET_FILL, NORMAL_MODE, 100, 0, 1, 0, 0);
+    gimp_selection_none ($image);
+
+    # get image file (image goes on opposite side to stripe)
+    if ($use_image) {
+	my $rimage = gimp_file_load (RUN_NONINTERACTIVE, $imagefile, $imagefile);
+	my $rdraw = gimp_image_active_drawable ($rimage);
+	gimp_scale ($rdraw, 1, 0, 0, $width-$stripewidth, $height);
+	gimp_edit_copy ($rdraw);
+	
+	my $imagelayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+					 "ImageLayer", $i_transparency, NORMAL_MODE);
+	
+	gimp_image_add_layer ($image, $imagelayer, 0);
+	
+	# clear the new layer
+	gimp_selection_all ($image);
+	gimp_edit_clear ($imagelayer);
+	gimp_selection_none ($image);
+	
+	my $flayer = gimp_edit_paste ($imagelayer, 1); 
+	if ($stripe_alignment eq "left") {
+	    gimp_layer_set_offsets($flayer, $stripewidth, 0);
+	    gimp_layer_set_offsets($imagelayer, $stripewidth, 0);
+	} else {
+	    gimp_layer_set_offsets($flayer, 0, 0);
+	    gimp_layer_set_offsets($imagelayer, 0, 0);
+	}
+
+	$backlayer = gimp_image_flatten ($image);
+    }
+	
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+	
+    # set the text if there is any
+    my ($textlayer, $textheight, $textwidth);
+    my $fsize = $fontsize;
+    if (length($text)) {
+	$text =~ s/\\n/\n/gi;
+
+	while (1) {
+
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, $fsize, 
+				    PIXELS, $foundry, $fontname, $fontweight, $fontslant, 
+				    $fontwidth, $fontspacing, $fontregistry, $fontencoding);
+	    
+	    # check that text fits within image
+	    $textwidth = gimp_drawable_width($textlayer);
+	    $textheight = gimp_drawable_height($textlayer);
+	    if ((($textwidth + $textspace_x) > $width) || 
+		(($textheight + $textspace_y) > $height)) {
+		if ($fsize < $minfontsize) {
+		    die "Error (title_icon.pl): text '$text' doesn't fit on ${width}x${height} image " .
+			"(minimum font size tried: $minfontsize\n";
+		} else {
+		    gimp_selection_all ($image);
+		    gimp_edit_clear ($textlayer);
+		    gimp_selection_none ($image);
+		    $fsize --;
+		    print STDERR "WARNING (title_icon.pl): '$text' doesn't fit: reducing font size to $fsize\n";
+		}
+	    } else {
+		last;
+	    }
+	}
+	
+	# align text
+	if ($text_alignment eq "left") {
+	    gimp_layer_set_offsets ($textlayer, $textspace_x, $textspace_y);
+	} else {
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)-$textspace_x, $textspace_y);
+	}
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+	
+    if ($filename =~ /\.gif$/i) {
+	# make indexed colour (may need to do this for 
+	# other formats as well as gif)
+	gimp_convert_indexed ($image, NO_DITHER, MAKE_PALETTE, 256, 0, 1, "");
+    }
+	
+    # save image
+    my $filename = &util::filename_cat ($image_dir, $filename);
+    if ($filename =~ /\.jpe?g$/i) {
+	# gimp_file_save doesn't appear to work properly for jpegs
+	file_jpeg_save (RUN_NONINTERACTIVE, $image, $finishedlayer, 
+			$filename, $filename, 0.8, 0, 1);
+    } else {
+	gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, 
+			$filename, $filename);
+    }
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+
+# adjust arguments that are effected by the size argument
+sub adjust_args {    
+
+    if ($size != 1) {
+	my @size_args = ('width', 'height', 'imageheight', 'stripewidth',
+			 'textspace_x', 'textspace_y', 'fontsize', 'minfontsize');
+	foreach $arg (@size_args) {
+	    $$arg = int ($$arg * $size);
+	}
+    }
+}
+
+sub wrap_text {
+
+    # don't wrap text if it already contains carriage returns
+    return if $text =~ /\n/;
+
+    # the following assumes that all words are less than $wrap_length long
+    my $wrap_length = 14;
+
+    my $new_text = "";
+    while (length ($text) >= $wrap_length) {
+	my $line = substr ($text, 0, $wrap_length);
+	$text =~ s/^\Q$line\E//;
+	$line =~ s/\s([^\s]*)$/\n/;
+	$text = $1 . $text;
+	$new_text .= $line;
+    }
+    $new_text .= $text;
+    $text = $new_text;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_title_icon", "create title icons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_title_icon", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+Gimp::on_net { gsdl_title_icon; };
+exit main;
+
Index: /main/tags/3.01/gsdl/bin/script/gimp/title_icon.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/title_icon.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/title_icon.pl	(revision 21112)
@@ -0,0 +1,399 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# title_icon.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# title_icon.pl generates all the green_title type icons and 
+# collection icons for Greenstone
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp;
+use parsargv;
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black = "#000000";
+
+
+local ($cfg_file, $size, $imagefile, $width, $height, $imageheight, $stripecolor, $stripewidth, 
+       $stripe_alignment, $i_transparency, $text, $text_alignment, $filename, $textspace_x, 
+       $textspace_y, $bgcolor, $fontcolor, $fontsize, $minfontsize, $foundry, $fontname, 
+       $fontweight, $fontslant, $fontwidth, $fontspacing, $image_dir, $dont_wrap);
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] macrofile\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -cfg_file file         configuration file containing one or more\n";
+    print STDERR "                          sets of the following options - use to create\n";
+    print STDERR "                          batches of images\n";
+    print STDERR "   -size number           the overall size ratio of the image (i.e. a size\n";
+    print STDERR "                          of 2 will create an image twice the default size)\n";
+    print STDERR "   -image_dir directory   directory to create images in [`pwd`]\n";
+    print STDERR "                          this should be full path to existing directory\n";
+    print STDERR "   -imagefile             filename of image to embed within new icon\n";
+    print STDERR "   -width number          width of icon [150]\n";
+    print STDERR "   -height number         height of icon [44]\n";
+    print STDERR "   -imageheight number    this is the height of the image if the image contains\n";
+    print STDERR "                          an image (like collection icons) [110]\n";
+    print STDERR "   -stripecolor hex_value color of vertical stripe [$gsdl_green]\n";
+    print STDERR "   -stripewidth           width of vertical stripe [40]\n";
+    print STDERR "   -stripe_alignment      alignment of vertical stripe (left or right) [left]\n";
+    print STDERR "   -i_transparency number transparency of image within icon (0 > transparent > 100) [60]\n"; 
+    print STDERR "   -text string           image text\n";
+    print STDERR "   -text_alignment        alignment of text (left or right) [left]\n";
+    print STDERR "   -filename string       filename of resulting image\n";
+    print STDERR "   -textspace_x number    space in pixels between left/right edge of image and\n";
+    print STDERR "                          left/right edge of text [3]\n";
+    print STDERR "   -textspace_y number    space in pixels between top of image and top of\n";
+    print STDERR "                          text [3]\n";
+    print STDERR "   -bgcolor hex_value     background color of icon [$gsdl_green]\n";
+    print STDERR "   -fontcolor hex_value   text color [$black]\n";
+    print STDERR "   -fontsize number       font point size [17]\n";
+    print STDERR "   -minfontsize number    minimum point size font will be reduced to fit image [10]\n";
+    print STDERR "   -foundry string        [*]\n";
+    print STDERR "   -fontname string       [lucida]\n";
+    print STDERR "   -fontweight string     [medium]\n";
+    print STDERR "   -fontslant             [r]\n";
+    print STDERR "   -fontwidth             [*]\n";
+    print STDERR "   -fontspacing           [*]\n";
+    print STDERR "   -dont_wrap             don't attempt to wrap text\n\n";
+}
+
+sub reset_options {
+    $image_dir = "./";
+    $imagefile = "";
+    $width = int (150 * $size);
+    $height = int (44 * $size);
+    $imageheight = int (110 * $size);
+    $stripecolor = $gsdl_green;
+    $stripewidth = int (40 * $size);
+    $stripe_alignment = "left";
+    $i_transparency = 60;
+    $text = "";
+    $text_alignment = "left";
+    $filename = "";
+    $textspace_x = int (3 * $size);
+    $textspace_y = int (3 * $size);
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = int (17 * $size);
+    $minfontsize = int (10 * $size);
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+}
+
+sub gsdl_title_icon {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'cfg_file/.*/', \$cfg_file,
+			 'size/\d+/1', \$size,
+			 'image_dir/.*/./', \$image_dir,
+			 'imagefile/.*/', \$imagefile,
+			 'width/^\d+$/150', \$width,
+			 'height/^\d+$/44', \$height,
+			 'imageheight/^\d+$/110', \$imageheight,
+			 "stripecolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$stripecolor,
+			 'stripewidth/^\d+$/40', \$stripewidth,
+			 'stripe_alignment/^(left|right)$/left', \$stripe_alignment,
+			 'i_transparency/^\d+$/60', \$i_transparency,
+			 'text/.*/', \$text,
+			 'text_alignment/^(left|right)$/left', \$text_alignment,
+			 'filename/.*', \$filename,
+			 'textspace_x/^\d+$/3', \$textspace_x,
+			 'textspace_y/^\d+$/3', \$textspace_y,
+			 "bgcolor/#[0-9A-Fa-f]{6}/$gsdl_green", \$bgcolor,
+			 "fontcolor/#[0-9A-Fa-f]{6}/$black", \$fontcolor,
+			 'fontsize/^\d+$/17', \$fontsize,
+			 'minfontsize/^\d+$/10', \$minfontsize,
+			 'foundry/.*/*', \$foundry,
+			 'fontname/.*/lucida', \$fontname,
+			 'fontweight/.*/medium', \$fontweight,
+			 'fontslant/.*/r', \$fontslant,
+			 'fontwidth/.*/*', \$fontwidth,
+			 'fontspacing/.*/*', \$fontspacing,
+			 'dont_wrap', \$dont_wrap)) {
+	&print_usage();
+	die "title_icon.pl: incorrect options\n";
+    }
+
+    # will create wherever gimp was started up from if we don't do this
+    if ($image_dir eq "./") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+    }
+
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+
+	close CONF;
+
+    } else {
+
+	&produce_image ();
+
+    }
+}
+
+sub produce_image {
+
+    &adjust_args ();
+    &wrap_text () unless $dont_wrap;
+
+    my $use_image = 0;
+    if ($imagefile =~ /\w/) {
+	if (!-r $imagefile) {
+	    print STDERR "WARNING (title_icon.pl): imagefile '$imagefile cannot be ";
+	    print STDERR "read - $filename will be created without the use of an image file\n";
+	} else {
+	    $use_image = 1;
+	    $height = $imageheight;
+	}
+    }
+
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+	
+    # background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+	
+    # add the background layer
+    gimp_image_add_layer ($image, $backlayer, 0);
+	
+    # set colour of stripe
+    gimp_palette_set_foreground ($stripecolor);
+	
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($image, $backlayer);
+    gimp_selection_none ($image);
+	
+    # fill in stripe
+    if ($stripe_alignment eq "left") {
+	gimp_rect_select ($image, 0, 0, $stripewidth, $height, 0, 0, 0);
+    } else {
+	gimp_rect_select ($image, $width-$stripewidth, 0, $stripewidth, $height, 0, 0, 0);
+    }
+    gimp_bucket_fill ($image, $backlayer, FG_BUCKET_FILL, NORMAL_MODE, 100, 0, 1, 0, 0);
+    gimp_selection_none ($image);
+	
+    # get image file (image goes on opposite side to stripe)
+    if ($use_image) {
+	my $rimage = gimp_file_load (RUN_NONINTERACTIVE, $imagefile, $imagefile);
+	my $rdraw = gimp_image_active_drawable ($rimage);
+	gimp_scale ($rimage, $rdraw, 1, 0, 0, $width-$stripewidth, $height);
+	gimp_edit_copy ($rimage, $rdraw);
+	
+	my $imagelayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+					 "ImageLayer", $i_transparency, NORMAL_MODE);
+	
+	gimp_image_add_layer ($image, $imagelayer, 0);
+	
+	# clear the new layer
+	gimp_selection_all ($image);
+	gimp_edit_clear ($image, $imagelayer);
+	gimp_selection_none ($image);
+	
+	my $flayer = gimp_edit_paste ($image, $imagelayer, 1); 
+	if ($stripe_alignment eq "left") {
+	    gimp_layer_set_offsets($flayer, $stripewidth, 0);
+	    gimp_layer_set_offsets($imagelayer, $stripewidth, 0);
+	} else {
+	    gimp_layer_set_offsets($flayer, 0, 0);
+	    gimp_layer_set_offsets($imagelayer, 0, 0);
+	}
+    }
+	
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+	
+    # set the text if there is any
+    my ($textlayer, $textheight, $textwidth);
+    my $fsize = $fontsize;
+    if (length($text)) {
+	$text =~ s/\\n/\n/gi;
+
+	while (1) {
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, $fsize, 
+				    PIXELS, $foundry, $fontname, $fontweight, $fontslant, 
+				    $fontwidth, $fontspacing);
+	    
+	    # check that text fits within image
+	    $textwidth = gimp_drawable_width($textlayer);
+	    $textheight = gimp_drawable_height($textlayer);
+	    if ((($textwidth + $textspace_x) > $width) || 
+		(($textheight + $textspace_y) > $height)) {
+		if ($fsize < $minfontsize) {
+		    die "Error (title_icon.pl): text '$text' doesn't fit on ${width}x${height} image " .
+			"(minimum font size tried: $minfontsize\n";
+		} else {
+		    gimp_selection_all ($image);
+		    gimp_edit_clear ($image, $textlayer);
+		    gimp_selection_none ($image);
+		    $fsize --;
+		    print STDERR "WARNING (title_icon.pl): '$text' doesn't fit: reducing font size to $fsize\n";
+		}
+	    } else {
+		last;
+	    }
+	}
+	
+	# align text
+	if ($text_alignment eq "left") {
+	    gimp_layer_set_offsets ($textlayer, $textspace_x, $textspace_y);
+	} else {
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)-$textspace_x, $textspace_y);
+	}
+    }
+    
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+	
+    if ($filename =~ /\.gif$/i) {
+	# make indexed colour (may need to do this for 
+	# other formats as well as gif)
+	gimp_convert_indexed ($image, 0, 256);
+    }
+	
+    # save image
+    my $filename = &util::filename_cat ($image_dir, $filename);
+    if ($filename =~ /\.jpe?g$/i) {
+	# gimp_file_save doesn't appear to work properly for jpegs
+	file_jpeg_save (RUN_NONINTERACTIVE, $image, $finishedlayer, 
+			$filename, $filename, 0.8, 0, 1);
+    } else {
+	gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, 
+			$filename, $filename);
+    }
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+
+# adjust arguments that are effected by the size argument
+sub adjust_args {    
+
+    if ($size != 1) {
+	my @size_args = ('width', 'height', 'imageheight', 'stripewidth',
+			 'textspace_x', 'textspace_y', 'fontsize', 'minfontsize');
+	foreach $arg (@size_args) {
+	    $$arg = int ($$arg * $size);
+	}
+    }
+}
+
+sub wrap_text {
+
+    # don't wrap text if it already contains carriage returns
+    return if $text =~ /\n/;
+
+    # the following assumes that all words are less than $wrap_length long
+    my $wrap_length = 14;
+
+    my $new_text = "";
+    while (length ($text) >= $wrap_length) {
+	my $line = substr ($text, 0, $wrap_length);
+	$text =~ s/^$line//;
+	$line =~ s/\s([^\s]*)$/\n/;
+	$text = $1 . $text;
+	$new_text .= $line;
+    }
+    $new_text .= $text;
+    $text = $new_text;
+}
+    
+sub query {
+
+  gimp_install_procedure("gsdl_title_icon", "create title icons for gsdl",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-10",
+                        "<Toolbox>/Xtns/gsdl_title_icon", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+sub net {
+  gsdl_title_icon;
+}
+
+exit main;
+
Index: /main/tags/3.01/gsdl/bin/script/gimp/title_icon_g12.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gimp/title_icon_g12.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gimp/title_icon_g12.pl	(revision 21112)
@@ -0,0 +1,380 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# title_icon.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# title_icon.pl
+#
+# This script generates collection title icons for greenstone.
+#
+# This version has been rewritten for the GIMP Version 1.2.  You
+# should be able to run it if you install gimp 1.2 with perl support.
+# In Debian this means running "apt-get install gimp1.2 gimp1.2perl"
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp ":auto";
+use Gimp::Fu; 
+
+use util;
+use unicode;
+
+# set trace level to watch functions as they are executed
+#Gimp::set_trace(TRACE_ALL);
+#Gimp::set_trace(TRACE_CALL);
+
+my $gsdl_green = "#96c19b";
+my $black = "#000000";
+my $white      = "#FFFFFF";
+
+my ($current_dir) = `pwd`;
+chomp($current_dir);
+
+
+my ($text, $filename, $image_dir, $size, $width, $height,
+    $stripecolor, $stripewidth, $stripe_alignment, $textspace_x,
+    $textspace_y, $dont_wrap, $imagefile, $imageheight, $i_transparency,
+    $bgcolor, $fontcolor, $fontsize, $minfontsize, $text_alignment,
+    $foundry, $fontname, $fontweight, $fontslant, $fontwidth,
+    $fontspacing, $cfg_file);
+
+sub reset_options {
+    $image_dir = "./";
+    $imagefile = "";
+    $width = int (150 * $size);
+    $height = int (44 * $size);
+    $imageheight = int (110 * $size);
+    $stripecolor = $gsdl_green;
+    $stripewidth = int (40 * $size);
+    $stripe_alignment = "left";
+    $i_transparency = 60;
+    $text = "";
+    $text_alignment = "left";
+    $filename = "";
+    $textspace_x = int (3 * $size);
+    $textspace_y = int (3 * $size);
+    $bgcolor = $gsdl_green;
+    $fontcolor = $black;
+    $fontsize = int (17 * $size);
+    $minfontsize = int (10 * $size);
+    $foundry = "*";
+    $fontname = "lucida";
+    $fontweight = "medium";
+    $fontslant = "r";
+    $fontwidth = "*";
+    $fontspacing = "*";
+}
+
+sub gsdl_title_icon {
+
+    ($text, $filename, $image_dir, $size, $width, $height,
+     $stripecolor, $stripewidth, $stripe_alignment, $textspace_x,
+     $textspace_y, $dont_wrap, $imagefile, $imageheight, $i_transparency,
+     $bgcolor, $fontcolor, $fontsize, $minfontsize, $text_alignment,
+     $foundry, $fontname, $fontweight, $fontslant, $fontwidth,
+     $fontspacing, $cfg_file) = @_;
+
+    # Create images using a configuration file
+    if ($cfg_file =~ /\w/) {
+	
+	open (CONF, $cfg_file) || die "couldn't open cfg_file $cfg_file\n";
+	while (1) {
+
+	    &reset_options ();
+	
+	    # read image configuration entry
+	    my $status = &read_config_entry (CONF);
+	    if ($filename !~ /\w/) {
+		if ($status) {last;}
+		else {next;}
+	    }
+	    &produce_image ();
+	    if ($status) {last;}
+	}
+	close CONF;
+    }
+
+    # Create a single image from the parameters we have
+    else {
+	&produce_image ();
+    }
+
+}
+
+sub produce_image {
+
+    &adjust_args ();
+    &wrap_text () unless $dont_wrap;
+
+    my $use_image = 0;
+    if ($imagefile =~ /\w/) {
+	if (!-r $imagefile) {
+	    print STDERR "WARNING (title_icon.pl): imagefile '$imagefile cannot be ";
+	    print STDERR "read - $filename will be created without the use of an image file\n";
+	} else {
+	    $use_image = 1;
+	    $height = $imageheight;
+	}
+    }
+
+    # create the image
+    my $image = gimp_image_new ($width, $height, RGB_IMAGE);
+	
+    # create and add the background layer
+    my $backlayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+				    "BGLayer", 100, NORMAL_MODE);
+    gimp_image_add_layer ($image, $backlayer, 0);
+	
+    # clear the background
+    gimp_selection_all ($image);
+    gimp_edit_clear ($backlayer);
+    gimp_selection_none ($image);
+	
+    # fill in stripe
+    if ($stripewidth > 0) {
+        gimp_palette_set_foreground ($stripecolor);
+	if ($stripe_alignment eq "left") {
+	  gimp_rect_select ($image, 0, 0, $stripewidth, $height, 0, 0, 0);
+	} else {
+	  gimp_rect_select ($image, $width-$stripewidth, 0, $stripewidth, $height, 0, 0, 0);
+	}
+	gimp_bucket_fill ($backlayer, FG_BUCKET_FILL, NORMAL_MODE, 100, 0, 1, 0, 0);
+	gimp_selection_none ($image);
+    }
+
+    # get image file (image goes on opposite side to stripe)
+    if ($use_image) {
+	my $rimage = gimp_file_load (RUN_NONINTERACTIVE, $imagefile, $imagefile);
+	my $rdraw = gimp_image_active_drawable ($rimage);
+	gimp_scale ($rdraw, 1, 0, 0, $width-$stripewidth, $height);
+	gimp_edit_copy ($rdraw);
+	
+	# add the background image layer
+	my $imagelayer = gimp_layer_new ($image, $width, $height, RGB_IMAGE, 
+					 "ImageLayer", $i_transparency, NORMAL_MODE);
+	gimp_image_add_layer ($image, $imagelayer, 0);
+	
+	# clear the new layer
+	gimp_selection_all ($image);
+	gimp_edit_clear ($imagelayer);
+	gimp_selection_none ($image);
+	
+	my $flayer = gimp_edit_paste ($imagelayer, 1); 
+	if ($stripe_alignment eq "left") {
+	    gimp_layer_set_offsets($flayer, $stripewidth, 0);
+	    gimp_layer_set_offsets($imagelayer, $stripewidth, 0);
+	} else {
+	    gimp_layer_set_offsets($flayer, 0, 0);
+	    gimp_layer_set_offsets($imagelayer, 0, 0);
+	}
+    }
+
+    # flatten the image (otherwise the text will be "behind" the image)
+    $backlayer = gimp_image_flatten ($image);
+
+    # set colour of text
+    gimp_palette_set_foreground ($fontcolor);
+	
+    # set the text if there is any
+    my ($textlayer, $textheight, $textwidth);
+    my $fsize = $fontsize;
+    if (length($text)) {
+	$text =~ s/\\n/\n/gi;
+
+	while (1) {
+	    $textlayer = gimp_text ($image, $backlayer, 0, 0, $text, 0, 1, $fsize, 
+				    PIXELS, $foundry, $fontname, $fontweight, $fontslant, 
+				    $fontwidth, $fontspacing, "*", "*");
+	
+	    # check that text fits within image
+	    $textwidth = gimp_drawable_width($textlayer);
+	    $textheight = gimp_drawable_height($textlayer);
+	    if ((($textwidth + $textspace_x) > $width) || 
+		(($textheight + $textspace_y) > $height)) {
+		if ($fsize < $minfontsize) {
+		    die "Error (title_icon.pl): text '$text' doesn't fit on ${width}x${height} image " .
+			"(minimum font size tried: $minfontsize\n";
+		} else {
+		    gimp_selection_all ($image);
+		    gimp_edit_clear ($image, $textlayer);
+		    gimp_selection_none ($image);
+		    $fsize --;
+		    print STDERR "WARNING (title_icon.pl): '$text' doesn't fit: reducing font size to $fsize\n";
+		}
+	    } else {
+		last;
+	    }
+	}
+	
+	# align text
+	if ($text_alignment eq "left") {
+	    gimp_layer_set_offsets ($textlayer, $textspace_x, $textspace_y);
+	} else {
+	    gimp_layer_set_offsets ($textlayer, ($width-$textwidth)-$textspace_x, $textspace_y);
+	}
+    }
+
+    # flatten the image
+    my $finishedlayer = gimp_image_flatten ($image);
+	
+    if ($filename =~ /\.gif$/i) {
+	# make indexed colour (may need to do this for 
+	# other formats as well as gif)
+	gimp_convert_indexed ($image, 0, 256);
+    }
+	
+    # save image
+    my $filename = &util::filename_cat ($image_dir, $filename);
+    gimp_file_save (RUN_NONINTERACTIVE, $image, $finishedlayer, $filename, $filename);
+}
+
+# returns 1 if this is the last entry,
+sub read_config_entry {
+    my ($handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	next unless $line =~ /\w/;
+	my @line = ();
+	if ($line =~ /^\-+/) {return 0;}
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute any environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+	if (scalar (@line) == 2 && defined ${$line[0]}) {
+	    ${$line[0]} = $line[1];
+	}
+    }
+    return 1;
+}
+
+# adjust arguments that are effected by the size argument
+sub adjust_args {
+
+    if ($size != 1) {
+        $width *= $size;
+        $height *= $size;
+	$imageheight *= $size;
+	$stripewidth *= $size;
+	$textspace_x *= $size;
+	$textspace_y *= $size;
+	$fontsize *= $size;
+	$minfontsize *= $size;
+    }
+}
+
+sub wrap_text {
+
+    # don't wrap text if it already contains carriage returns
+    return if $text =~ /\n/;
+
+    # the following assumes that all words are less than $wrap_length long
+    my $wrap_length = 14;
+
+    my $new_text = "";
+    while (length ($text) >= $wrap_length) {
+	my $line = substr ($text, 0, $wrap_length);
+	$text =~ s/^$line//;
+	$line =~ s/\s([^\s]*)$/\n/;
+	$text = $1 . $text;
+	$new_text .= $line;
+    }
+    $new_text .= $text;
+    $text = $new_text;
+}
+
+
+register("gsdl_title_icon", 
+	 "Create title icons for gsdl (gimp version 1.2)",
+	 "", 
+	 "Stefan Boddie (1.0) and Gordon Paynter (1.2)",
+	 "Copyright 1999-2001 The New Zealand Digital Library Project",
+	 "2000-03-10",
+	 "<Toolbox>/Xtns/gsdl_title_icon", 
+	 "*",
+	 [
+           [PF_STRING, "text",      "Text appearing on icon", "greenstone collection"],
+
+           [PF_STRING, "filename",  "Output file", "title_icon.png"],
+           [PF_STRING, "image_dir", "Directory to create images in", $current_dir],
+
+           [PF_INT,    "size_ratio", "Size factor: a factor of 2 doulbles image size",  1],
+           [PF_INT,    "width",      "Width of the icon",  150],
+           [PF_INT,    "height",     "Height of the icon", 44],
+
+           [PF_STRING, "stripecolor", "Colour of the stripe", $gsdl_green],
+           [PF_INT,    "stripewidth", "Width of the stripe",  40],
+           [PF_STRING, "stripealign", "Alignment of stripe (left or right)", "left"],
+
+           [PF_INT,    "text_x_offset",    "Distance from left of image to text", 3],
+           [PF_INT,    "text_y_offset",    "Distance from top of image to text",  3],
+           [PF_STRING, "dont_wrap",        "don't attempt to wrap text", ""],
+
+           [PF_STRING, "imagefile",   "Filename of background image", ""],
+           [PF_INT,    "imageheight", "Height of background image", 110],
+           [PF_INT,    "imagetransp", "Transparency of background image (1-100)", 60],
+
+#           [PF_STRING, "fgcolor",     "Foreground colour (top) of bar", $white],
+           [PF_STRING, "bgcolor",     "Background colour (bottom) of bar", $gsdl_green],
+           [PF_STRING, "fontcolor",   "Colour of text on bar", $black],
+           [PF_INT,    "fontsize",    "Font size",    17],
+           [PF_INT,    "minfontsize", "Minimum font size if scaling required",    10],
+
+           [PF_STRING, "alignment",     "Alignment of text on the icon (left or right)", "left"],
+
+           [PF_STRING, "foundry",     "Font foundry", "*"],
+           [PF_STRING, "fontname",    "Font name",    "lucida"],
+           [PF_STRING, "fontweight",  "Font weight",  "medium"],
+           [PF_STRING, "fontslant",   "Font slant",   "r"],
+           [PF_STRING, "fontwidth",   "Font width",   "*"],
+           [PF_STRING, "fontspacing", "Font spacing", "*"],
+
+           [PF_STRING, "cfg_file",    "Configuration file", ""]
+
+         ], 
+         \&gsdl_title_icon);
+
+
+exit main;
+
+
Index: /main/tags/3.01/gsdl/bin/script/grab_collection.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/grab_collection.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/grab_collection.pl	(revision 21112)
@@ -0,0 +1,570 @@
+#!/usr/bin/perl -w
+
+
+# Both this script and its associated process_html.pl were written by
+# Marcel ?, while a student at Waikato University. Unfortunately he
+# was very new to perl at the time so the code is neither as clean nor
+# as fast as it could be (I've cleaned up a few of the more serious
+# bottlenecks -- it could do with alot more work though).  The code
+# does work though, if a little slowly. It's not ready for primetime
+# however and is included in the Greenstone source tree mostly so that
+# I don't lose it. -- Stefan - 24 Jul 2001
+
+
+# This script will download an entire collection (and its associated pictures and files) 
+# and store them in a temporary directory ($outputdir). 
+# A second script (process_html.pl) can be then be used to 'rebuild' the collection and link all the
+# downloaded pages and pictures together into a usable static collection.
+
+# This script will generate a number of text files required by the second script
+# and for possible recovery, namely:
+# - links.txt        - contains all the http links that were downloaded
+# - images.txt       - contains a list of all the images that were downloaded  
+# - image_dirs.txt   - contains a list of all the image-prefixes that need to be wiped from the html files
+#                      (otherwise you won't get to see any pictures)
+
+# Both this script and the html processing script have a recovery feature built in, they can continue from wherever
+# they left off, but this only works if $outputdir and $finaldir are set to different values.
+   
+# This is where all the downloaded html files end up, eg. 'temp_html/'
+my $outputdir  = 'temp_html/';
+# This is where all the processed html files (and the pictures) will end up eg. 'my_static_collection/'
+my $finaldir   = 'envl_collection/';
+
+# This is where we start our mirroring
+$address = 'http://nowhere.com/cgi-bin/library?a=p&p=about&c=demo&u=1';
+
+
+# whatever is specified in $option will be attached to the end of each html link before it is downloaded.
+# eg. "&u=1" to disable various features in the greenstone collections that are not needed with a static
+#            collection.
+# another example : "&l=nl" to set the entire collection to dutch (NetherLands :)  
+my $option = "&u=1";
+
+# Most OS have a limit on the maximum amount of files per directory (or folder).
+# A static collection can easily contain >3000 html files. Putting all those files
+# into one single directory is just asking for trouble. It's also very unwieldy ;^)
+# Hence...this value will set how much html files will be stored in one directory. 
+# These directories themselves will be numbered, 
+# so if $dir_entries = 500 then the directories will be "0/", "500/", "1000/", "1500/", "2000/", etc. 
+my $dir_entries = 250;
+
+# Occasionally a page occurs which contains no data (because &cl is not set) This option fixes that. 
+my $fix_empty_pages = "&cl=CL1";
+
+# These are the files that wget will download.
+# more can be added if necessary.
+my @graphic_formats = ('.gif','.jpg','.bmp','.png','.pdf','.mov','.mpeg','.jpeg','.rm');
+
+# ---------------------------------------[ System specific options ]----------------------------------------------------
+
+# The lynx variable specifies the command line for the lynx web browser
+# -- This is what I use under dos/win32
+# my $lynx = 'e:\lynx_w32\lynx -cfg=e:\lynx_w32\lynx.cfg';
+
+# -- This is what I use under linux
+my $lynx = 'lynx';
+
+# and the same for the wget utility
+my $wget = 'wget';
+
+# NB: There is one other linux specific command all the way at the end of this script, where I've used 'cp' to copy a file.       
+
+# Another NB: When saving the dl-ed html files to disk, I've set lynx to dump the html-source to the standard output,
+#             which I then simply redirect to a target file, BUT
+#             this does not work under DOS/win32. Redirecting standard output in a script causes it to be displayed on
+#             the screen instead. The easiest way to get around this I found was by doing the actual redirection in a simple
+#             batch file (say grab.bat), which contains the following line:
+#             @e:\lynx_w32\lynx -cfg=e:\lynx_w32\lynx.cfg -dump -source "%1" > %2 
+#
+#             Then replace line nr 326 -> 'system ("$kommand > $target");' with 'system("grab.bat $address $target");'
+#             Not a very elegant solution, but it works :) 
+
+#------------------------------------------------------------------------------------------------------------------------
+
+my %image_list;
+my $image_pointer = 0;
+
+my %linkz_list;
+my %short_linkz_list;
+my $linkz_pointer = 0;
+
+my %image_dirs_list;
+my $image_dirs_pointer = 0;
+
+my $numberdir = 0;
+
+my $start_time = (times)[0];
+
+# check if directories exist and create them if necessary..
+if ((-e $outputdir)&&(-d $outputdir)) 
+{ 
+    print " ** ",$outputdir," directory already exists..\n"; 
+}
+else
+{
+    print " ** Creating ",$outputdir," directory..\n";
+    mkdir($outputdir, 0777) or die " Cannot create output directory: $!\n";
+}
+
+if ((-e $finaldir)&&(-d $finaldir))
+{ 
+    print " ** ",$finaldir," directory already exists..\n";
+}
+else
+{
+    print " ** Creating ",$finaldir," directory..\n";
+    mkdir($finaldir, 0777) or die " Cannot create final directory: $!\n";
+}
+
+#-----------------------------------------------------------------------------------------------
+# No need to start from scratch everytime, we can recover/continue from wherever we left off
+# simply by checking which html files have been created
+#-----------------------------------------------------------------------------------------------
+
+$linknumber    = 0;                    # used to name/number the dl-ed html files
+
+my $failed     = 0;
+while ($failed == 0)
+{
+    if ($linknumber % $dir_entries == 0) 
+	{
+	    if (!((-e $outputdir.$linknumber)&&(-d $outputdir.$linknumber)))
+	    {
+		$failed++;
+		mkdir($outputdir.$linknumber, 0777) or print " ** Cannot create ",$outputdir.$linknumber, "!: $!\n";
+	    }
+	    $numberdir = $linknumber;
+	}
+
+    $check_file = $outputdir.$numberdir."/".$linknumber.".html";
+    if ((-e $check_file)&&($failed == 0)) 
+    { 
+	$linknumber++;
+    } 
+    else
+    {
+	$failed++;
+	# I'm subtracting 1 from the starting link, 
+	# just in case it only loaded half the page ;^)
+	if($linknumber>0) 
+	{ 
+	    $linknumber--; 
+	}
+	print " Will start downloading at number $linknumber \n";
+    }
+}
+
+# if we're starting from scratch, then we might as well nuke the links file
+#if ($linknumber == 0) 
+#{ 
+#    print " Starting from scratch - clobbering the old text files...\n";
+#    if (-e 'links.txt')
+#    {
+#	print "   Removing links.txt...\n";   
+#	unlink <links.txt> or print " ** Cannot delete links textfile: $!\n";
+#    }
+#    if (-e 'images.txt')
+#    {
+#	print "   Removing images.txt...\n";
+#	unlink <images.txt> or print " ** Cannot delete images textfile: $!\n";
+#    }
+#    if (-e 'image_dirs.txt')
+#    {
+#	print "   Removing image_dirs.txt...\n";
+#	unlink <image_dirs.txt> or print " ** Cannot delete image_dirs textfile: $!\n";
+#    }
+#}
+
+# if we're NOT starting from scratch, then read in old links from links text file
+# and grab the old image-links as well...
+if ($linknumber != 0)
+{
+    # load the old links from links.txt, if it doesn't exist, then give up :(
+    my $this = "";
+    my $that = "";
+    open (CHECK, "links.txt") or die " ** Cannot find/open links.txt file!: $! **\n";
+    while(eof CHECK == 0)
+    {
+	while($this ne "\n")
+	{
+	    read CHECK, $this ,1;
+	    $that = $that.$this;   
+	}
+	$linkz_list[$linkz_pointer] = $that;
+	
+	for my $search(0 .. (length($that) - 3))
+	{
+	    if((substr($that, $search, 3) eq '?a=')||(substr($that, $search, 3) eq '&a='))
+	    {
+		$short_linkz_list[$linkz_pointer] = substr($that, $search);
+		last;
+	    }
+	}
+	$linkz_pointer++;
+	$that = ""; $this = "";
+    }
+    close(CHECK);
+    print "- I found ",($#linkz_list + 1)," links in links.txt -\n";
+    
+    #make sure that we start dl-ing the correct first page 
+    $address = $linkz_list[$linknumber];
+
+    # load the old image links from image.txt (if it doesn't exist, no big deal ;)
+    my $im_this = "";
+    my $im_that = "";
+    open (IMAGES, "images.txt") || print " ** Cannot find/open images.txt file! : $! **\n";
+    while(eof IMAGES == 0)
+    {
+	while($im_this ne "\n")
+	{
+	    read IMAGES, $im_this ,1;
+	    $im_that = $im_that.$im_this;   
+	}
+	$image_list[$image_pointer] = $im_that;
+	$image_pointer++;
+	$im_that = ""; $im_this = "";
+    }
+    close(IMAGES);
+    print "- I found ",($#image_list + 1)," picture-links in images.txt -\n";
+
+    #..and last but not least, load any image_dirs from image_dirs.txt
+    # again, if its not there, no big deal :)
+    my $imd_this = "";
+    my $imd_that = "";
+    open (IMAGE_DIR, "image_dirs.txt") || print " ** Cannot find/open image_dirs.txt file!: $! **\n";
+    while(eof IMAGE_DIR == 0)
+    {
+	while($imd_this ne "\n")
+	{
+	    read IMAGE_DIR, $imd_this ,1;
+	    $imd_that = $imd_that.$imd_this;   
+	}
+	$image_dirs_list[$image_dirs_pointer] = $imd_that;
+	$image_dirs_pointer++;
+	$imd_that = ""; $imd_this = "";
+    }
+    close(IMAGE_DIR);
+    print "- I found ",($#image_dirs_list + 1)," picture directories in image_dirs.txt -\n";
+}
+
+#  Just keep going till we can find no more new links
+while(($#linkz_list < 0)||($#linkz_list+1 > $linknumber)) 
+{
+
+    # This line specifies the command line for the lynx web browser
+    my $kommand = $lynx.' -dump -image_links "'.$address.'"';
+
+    # dump page into text-array and find starting-point of the references/links
+    chomp(@data=`$kommand`);
+    for my $i(0 .. $#data)
+    {
+	if ($data[$i] eq "References") {
+	    $here = $i;}
+    }
+    $here = $here+2;
+
+    # process references/links
+    for $i($here .. $#data){
+
+	$its_an_image = 0;	     	
+    
+	#chop-off refs leading number&spaces (eg. '1. http://www.cs.waikato.ac.nz')
+	#                                          ^^^
+	
+#	$temp = substr($data[$i],3);
+#	@temp = split(/ /, $temp, 2);
+    
+	#check if the last 4 characters of the link equal .gif .jpg .png .bmp .pdf .mov .mpeg etc etc
+    
+#	for my $g(0 .. $#graphic_formats)
+#	{
+#	    if(substr($temp[1],(length($graphic_formats[$g]) * -1)) eq $graphic_formats[$g])
+#	    {
+#		$its_an_image = 1;
+#	    }
+#	}
+
+	$data[$i] =~ s/^\s*\d+\.\s+//;
+	if ($data[$i] =~ /\.(gif|jpe?g|png|bmp|pdf|mov|mpe?g|rm)$/i) {
+	    $its_an_image = 1;
+	}
+
+	# ignore mailto urls
+	if ($data[$i] !~ /mailto:/i) {
+	
+	    #----------- the link is NOT an image ----------------		
+	    if ($its_an_image == 0)
+	    { 
+		&its_a_link($data[$i], $outputdir);
+	    }
+
+	    #----------- the link IS an image ----------------
+	    if ($its_an_image != 0)
+	    { 
+		&its_an_image($data[$i], $finaldir); 
+	    }
+	}
+    }	
+
+    # save the web page to disk (in the appropriate numbered directory)
+    $kommand = $lynx.' -dump -source "'.$address.'"';
+
+    if ($linknumber % $dir_entries == 0) 
+    {
+	if ((-e $outputdir.$linknumber)&&(-d $outputdir.$linknumber))
+	{
+	    print " ** ",$outputdir.$linknumber, " - Directory allready exists.\n";
+	}
+	else
+	{
+	    mkdir($outputdir.$linknumber, 0777) or print " ** Cannot create ",$outputdir.$linknumber, "!: $!\n";
+	    mkdir($finaldir.$linknumber, 0777)  or print " ** Cannot create ",$outputdir.$linknumber, "!: $!\n";
+	}
+	$numberdir = $linknumber;
+    }
+    my $target = $outputdir.$numberdir."/".$linknumber.".html";
+
+    #---------------------------------------------------------------------------------------------------------------
+    # NOTE: This next command will NOT work under win32/dos, as redirecting standard output in a script causes it to
+    #       be dumped straight to the screen as opposed to into the target file.
+    #---------------------------------------------------------------------------------------------------------------
+    system ("$kommand > $target");
+    #---------------------------------------------------------------------------------------------------------------
+
+    print " Saved $target\n";
+
+    $linknumber++;
+
+    $address = $linkz_list[$linknumber];
+}
+
+my $end_time = (times)[0];
+
+print "\n\n\n *----------------------------*\n";
+print " |  Whew! Task completed! :-D |\n";
+print " *----------------------------*\n";
+printf"  Script took %.2f CPU seconds to complete ;^)\n", $end_time - $start_time;
+print "\n\n"; 
+print "  Now execute the  process_html.pl  script to link the downloaded collection together.\n";
+print "  Please do make sure that it is executed with the same options as this script ;-)\n";
+
+sub its_a_link 
+{
+    local($found) = @_;
+#    local($ok = 0, $kommand);
+    local($kommand);
+    local $short_link = "";
+
+    return if ($found =~ /\#.*$/);
+
+    # attach the custom options
+    $found .= $option;
+    
+    #little bit of trickery here - check if there is a &d= option present in the link
+    #if there is, then wipe the &cl= option!
+    #This should cut down multiple copies by 75%!!
+    
+    #but, if there is no &d option, and the &cl option is not set, then we have to set the &cl option to something
+    #otherwise we get pages which contain no data :\
+
+    if ($found =~ /[&\?]a=d/) {
+	if ($found =~ /[&\?]d=/) {
+	    $found =~ s/[&\?]cl=[^&]*//;
+	} elsif ($found !~ /[&\?]cl=/) {
+	    $found .= $fix_empty_pages;
+	}
+    }
+
+    # we also want to sort out any xxx.pr OIDs that we come across
+    $found =~ s/([&\?](cl|d)=.*?)\.\d+\.pr/$1/g;
+
+    # attach the EOL character.
+    $found = $found."\n";
+
+
+    # the hard way !!!
+#    for my $search(0 .. (length($found) - 3))
+#    {
+#	if((substr($found, $search, 3) eq '?d=')||(substr($found, $search, 3) eq '&d='))
+#	{
+#	    for my $second_search(0 .. (length($found) - 4))  
+#	    {
+#		if((substr($found, $second_search, 4) eq '?cl=')||(substr($found, $second_search, 4) eq '&cl='))
+#		{
+#		    for my $third_search(($second_search + 3) .. (length($found) - 1))
+#		    {
+#			if((substr($found, $third_search, 1)) eq '&')
+#			{
+#			    substr($found, $second_search, $third_search - $second_search) = ""; 
+#			    last;
+#			}
+#		    }
+#		    last;
+#		}
+#	    } 
+#	    last;
+#	}
+#	else
+#	{
+#	    if( $search == (length($found) - 3))
+#	    {
+#		for my $second_search(0 .. (length($found) - 4))  
+#		{
+#		    if((substr($found, $second_search, 4) eq '?cl=')||(substr($found, $second_search, 4) eq '&cl='))
+#		    {
+#			for my $third_search(($second_search + 3) .. (length($found) - 1))
+#			{
+#			    if((substr($found, $third_search, 1)) eq '&')
+#			    {
+#				if (substr($found, $second_search, $third_search - $second_search) eq '&cl=')
+#				{
+#				    substr($found, $second_search, $third_search - $second_search) = $fix_empty_pages;	
+#				}
+#				last;
+#			    }
+#			}
+#			last;
+#		    }
+#		} 
+#	    }
+#	}
+#    }
+    
+    # grab the last part of the link (ignoring the start and the &e option)
+#    for my $search(0 .. (length($found) - 3))
+#    {
+#	if((substr($found, $search, 3) eq '?a=')||(substr($found, $search, 3) eq '&a='))
+#	{
+#	    $short_link = substr($found, $search);
+#	    last;
+#	}
+#    }
+
+    ($short_link) = $found =~ /\?(.*)$/;
+    $short_link =~ s/(^|&)e=[^&]*/$1/;
+
+    
+    # this filters out multiple copies of for example the help page, which has #something at the end of its links
+    # now do this first with regular expression above -- Stefan
+
+#    for my $search(0 .. length($found))
+#    {
+#	if ((substr($found, $search, 1)) eq '#')
+#	{
+#	    $ok++;
+#	    last;
+#	}
+#    }
+ 
+
+   
+    # compare the found link to the links we've stored in the arrays (compares both full link and partial link)
+    for my $search(0 .. $#linkz_list)
+    {
+	return if ($found eq $linkz_list[$search]);
+	return if ($short_link eq $short_linkz_list[$search]);
+    }
+		    
+    # if found link is not in links array, add it
+    open (DUMP, ">>links.txt") or die " ** Can't open links.txt!: $!\n";
+    print DUMP $found;
+    close(DUMP);
+    
+    $linkz_list[$linkz_pointer] = $found;
+    $short_linkz_list[$linkz_pointer] = $short_link;
+    $linkz_pointer++;
+}
+
+sub do_image_dirs
+{
+    local($found) = @_;
+    my $count = 0;
+    my @br_index;
+    my $image_dir = "";
+    my $new_dir = 0;
+
+    for my $search(1 .. (length($found) - 1 ))
+    {
+	$bracket = substr($found, ($search * - 1), 1);
+	if ($bracket eq '/')
+	{
+	    $count++;
+	    $br_index[$count] = $search;
+	}
+	if($count == 2) 
+	{
+	    $image_dir = substr($found, ($br_index[2] * -1) , ($br_index[2] - $br_index[1]));
+	}
+    }
+    
+    my $dirs_to_wipe = substr($found, $br_index[$#br_index - 2] * - 1, $br_index[$#br_index - 2] - $br_index[2] + 1)."\n";
+    
+    for my $counter(0 .. $#image_dirs_list)
+    {
+	if($dirs_to_wipe eq $image_dirs_list[$counter])
+	{
+	    $new_dir++;
+	}
+    }
+    
+    if ($new_dir == 0)
+    {
+	open (IMAGE_DIRS, ">>image_dirs.txt") or die " ** Can't open image_dirs.txt!: $!\n";
+	print IMAGE_DIRS $dirs_to_wipe;
+	close(IMAGE_DIRS);
+	$image_dirs_list[$image_dirs_pointer] = $dirs_to_wipe;
+	$image_dirs_pointer++;
+    }
+
+    print "   ",substr($finaldir, 0 ,length($finaldir) - 1).$image_dir.substr($found, ($br_index[1] * - 1), length($found) - (length($found) - $br_index[1])),"\n";	    
+    
+    return $image_dir;
+}
+
+sub its_an_image 
+{
+    local($found, $outpdir) = @_;
+    local($kommand);
+    my $new = 0;
+    
+    my $temp_found = $found . "\n";
+
+    # check if the image is in the list
+    for my $counter(0 .. $#image_list)
+    {
+	if($temp_found eq $image_list[$counter])
+	{
+	    $new++;
+	}
+    }
+    
+    # only download the image if its not in the list..
+    if($new == 0)
+    {    
+	my $image_dir = &do_image_dirs;
+	my $temp_outputdir = $outpdir;
+	if (substr($temp_outputdir, -1, 1) eq "/")
+	{
+	    substr($temp_outputdir, -1, 1) = "";
+	}
+	
+	# wget is set to 'q - quiet' and 'nc - dont clobber existing files'
+	$kommand = $wget.' -qnc --directory-prefix='.$temp_outputdir.$image_dir.' "'.$found.'"';
+	system ("$kommand");
+        
+	open (IMAGES, ">>images.txt") or die " ** Can't open images.txt!: $!\n";
+	print IMAGES $temp_found;
+	close(IMAGES);
+	
+	$image_list[$image_pointer] = $temp_found;
+	$image_pointer++;
+	
+        # grab corresponding ON pictures for navigation bar if we've just dl-ed the OFF picture
+	if(substr($found , -6) eq "of.gif") 
+	{
+	    substr($found, -6, 6) = "on.gif";
+	    &its_an_image($found, $outpdir);
+	}
+    }    
+}
Index: /main/tags/3.01/gsdl/bin/script/gsConvert.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gsConvert.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gsConvert.pl	(revision 21112)
@@ -0,0 +1,1288 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# gsConvert.pl -- convert documents to HTML or TEXT format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2002 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# gsConvert.pl converts documents in a range of formats to HTML or TEXT
+# by exploiting third-party programs.  The sources of these are usually found
+# in the $GSDLHOME/packages directory, and the executables should live in
+# $GSDLHOME/bin/$GSDLOS (which is on the search path).
+#
+# Currently, we can convert the following formats by using external
+# conversion utilities:
+# Microsoft Word (versions 2,6,7 [==95?], 8[==97?], 9[==2000?]), RTF,
+# Adobe PDF, PostScript, MS PowerPoint (95 and 97), and MS Excel (95 and 97).
+#
+# We can try to convert any file to text with a perl implementation of the
+# UNIX strings command.
+#
+# We try to convert Postscript files to text using "gs" which is often on
+# *nix machines. We fall back to performing weak text extraction by using
+# regular expressions.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use Cwd;
+use File::Basename;
+
+# Are we running on WinNT or Win2000 (or later)?
+my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
+if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
+
+my $use_strings;
+my $pdf_complex;
+my $pdf_nohidden;
+my $pdf_zoom;
+my $pdf_ignore_images;
+my $pdf_allow_images_only;
+my $windows_scripting;
+
+sub print_usage
+{
+    print STDERR "\n";
+    print STDERR "gsConvert.pl: Converts documents in a range of formats to html\n";
+    print STDERR "              or text using third-party programs.\n\n";
+    print STDERR "  usage: $0 [options] filename\n";
+    print STDERR "  options:\n\t-type\tdoc|dot|pdf|ps|ppt|rtf|xls\t(input file type)\n";
+    print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
+    print STDERR "\t-output\tauto|html|text|pagedimg-jpg|pagedimg-gif|pagedimg-png\t(output file type)\n";
+    print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
+    print STDERR "\t-use_strings\tuse strings to extract text if conversion fails\n";
+    print STDERR "\t-windows_scripting\tuse windows script when converting Microsoft Word and PPT via VB script\n";
+    print STDERR "\t-pdf_complex\tuse complex output when converting PDF to HTML\n";
+    print STDERR "\t-pdf_nohidden\tDon't attempt to extract hidden text from PDF files\n";
+    print STDERR "\t-pdf_ignore_images\tdon't attempt to extract images when\n";
+    print STDERR "\t\tconverting PDF to HTML\n";
+    print STDERR "\t-pdf_allow_images_only\tallow images only (continue even if no text is present when converting to HTML)\n";
+    print STDERR "\t-pdf_zoom\tfactor by which to zoom PDF (only useful if\n";
+    print STDERR "\t\t-pdf_complex is set\n";
+    exit(1);
+}
+
+my $faillogfile="";
+my $timeout=0;
+
+sub main
+{
+    my (@ARGV) = @_;
+    my ($input_type,$output_type,$verbose);
+
+    # read command-line arguments
+    if (!parsargv::parse(\@ARGV,
+			 'type/(doc|dot|pdf|ps|ppt|rtf|xls)/', \$input_type,
+			 '/errlog/.*/', \$faillogfile,
+			 'output/(auto|html|text|pagedimg).*/', \$output_type,
+			 'timeout/\d+/0',\$timeout,
+			 'verbose/\d+/0', \$verbose,
+			 'use_strings', \$use_strings,
+			 'windows_scripting',\$windows_scripting,
+			 'pdf_complex', \$pdf_complex,
+			 'pdf_ignore_images', \$pdf_ignore_images,
+			 'pdf_allow_images_only', \$pdf_allow_images_only,
+			 'pdf_nohidden', \$pdf_nohidden,
+			 'pdf_zoom/\d+/2', \$pdf_zoom
+			 ))
+    {
+	print_usage();
+    }
+    
+  
+    # Make sure the input file exists and can be opened for reading
+    if (scalar(@ARGV!=1)) {
+	print_usage();
+    }
+
+    my $input_filename = $ARGV[0];
+    if (!-r $input_filename) {
+	print STDERR "Error: unable to open $input_filename for reading\n";
+	exit(1);
+    }
+
+    # Deduce filenames
+    my ($tailname,$dirname,$suffix) 
+	= File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
+    my $output_filestem = &util::filename_cat($dirname, "$tailname");
+
+    if ($input_type eq "")
+    {
+	$input_type = lc (substr($suffix,1,length($suffix)-1));
+    }
+    
+    # Change to temporary working directory
+    my $stored_dir = cwd();
+    chdir ($dirname) || die "Unable to change to directory $dirname";
+
+    # Select convert utility
+    if (!defined $input_type) {
+	print STDERR "Error: No filename extension or input type defined\n";
+	exit(1);
+    } 
+    elsif ($input_type eq "doc" || $input_type eq "dot") {
+	print &convertDOC($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "rtf") {
+	print &convertRTF($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "pdf") {
+	print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "ps") {
+	print &convertPS($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "ppt") {
+	print &convertPPT($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "xls") {
+	print &convertXLS($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    else {
+	print STDERR "Error: Unable to convert type '$input_type'\n";
+	exit(1);
+    }
+    
+    # restore to original working directory
+    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
+
+}
+
+&main(@ARGV);
+
+
+
+# Document-type conversion functions
+#
+# The following functions attempt to convert documents from their
+# input type to the specified output type.  If no output type was
+# given, then they first attempt HTML, and then TEXT.
+#
+# Each returns the output type ("html" or "text") or "fail" if no 
+# conversion is possible.
+
+# Convert a Microsoft word document
+
+sub convertDOC {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    # Many .doc files are not in fact word documents!
+    my $realtype = &find_docfile_type($input_filename);
+
+    if ($realtype eq "word6" || $realtype eq "word7" || $realtype eq "word8") {
+	return &convertWord678($input_filename, $output_filestem, $output_type);
+    } elsif ($realtype eq "rtf") {
+	return &convertRTF($input_filename, $output_filestem, $output_type);
+    } else {
+	return &convertAnything($input_filename, $output_filestem, $output_type);
+    }
+}
+
+# Convert a Microsoft word 6/7/8 document
+
+sub convertWord678 {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+    if (!$output_type || ($output_type =~ /html/i)){
+	if ($windows_scripting) {
+	    $success = &native_doc_to_html($input_filename, $output_filestem);
+	}
+	else {
+	    $success = &doc_to_html($input_filename, $output_filestem);	   
+	}
+	if ($success) {
+	   return "html";
+	}
+    }
+
+    return &convertAnything($input_filename, $output_filestem, $output_type);
+}
+
+
+# Convert a Rich Text Format (RTF) file
+
+sub convertRTF {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+
+    # Attempt specialised conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &rtf_to_html($input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+# rtf is so ugly that's it's not worth running strings over.
+# One day I'll write some quick'n'dirty regexps to try to extract text - jrm21
+#    return &convertAnything($input_filename, $output_filestem, $output_type);
+    return "fail";
+}
+
+
+# Convert an unidentified file
+
+sub convertAnything {
+    ($input_filename, $output_filestem, $output_type) = @_;
+    
+    my $success = 0;
+  
+    # Attempt simple conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &any_to_html($input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+    # Convert to text
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = &any_to_text($input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+    return "fail";
+}
+
+
+
+# Convert an Adobe PDF document
+
+sub convertPDF {
+    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+    $output_type =~ s/.*\-(.*)/$1/i;
+    # Attempt coversion to Image
+    if ($output_type =~ /jp?g|gif|png/i) {
+	$success = &pdf_to_img($dirname, $input_filename, $output_filestem, $output_type);
+	if ($success){
+	    return "item";
+	}
+    }
+
+    # Attempt conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &pdf_to_html($dirname, $input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+    # Attempt conversion to TEXT
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = &pdf_to_text($dirname, $input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+
+    return "fail";
+
+}
+
+
+# Convert an Adobe PostScript document
+
+sub convertPS {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+    $output_type =~ s/.*\-(.*)/$1/i;
+    # Attempt coversion to Image
+    if ($output_type =~ /jp?g|gif|png/i) {
+	$success = &ps_to_img($dirname, $input_filename, $output_filestem, $output_type);
+	if ($success){
+	    return "item";
+	}
+    }
+
+    # Attempt conversion to TEXT
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = &ps_to_text($input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+    return "fail";
+}
+
+
+sub convertPPT {
+    my ($input_filename, $output_filestem, $output_type) = @_;
+    my $success = 0;
+
+    my $ppt_convert_type = "";
+    #if (!$output_type || $windows_scripting ||($output_type !~ /html/i) ||($output_type !~ /text/i)){
+    if ($windows_scripting && ($output_type !~ /html/i) && ($output_type !~ /text/i)){
+	if ($output_type =~ /gif/i) {
+	    $ppt_convert_type = "-g";
+	} elsif ($output_type =~ /jp?g/i){
+	    $ppt_convert_type = "-j";
+	} elsif ($output_type =~ /png/i){
+	    $ppt_convert_type = "-p";
+	}
+	my $vbScript = &util::filename_cat($ENV{'GSDLHOME'}, "bin",
+					   $ENV{'GSDLOS'}, "pptextract");
+	$vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ /^windows$/i);
+			
+	$cmd = "";
+	if ($timeout) {$cmd = "ulimit -t $timeout;";}
+	# if the converting directory has already existed
+	if (-d $output_filestem) {
+	    print STDERR "**The conversion directory has existed\n";
+	    return "item";
+	} else {
+	    $cmd .=  "$vbScript $ppt_convert_type \"$input_filename\" \"$output_filestem\"";
+	    $cmd .= " 2>\"$output_filestem.err\""
+		if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+	    if (system($cmd) !=0) {
+		print STDERR "Powerpoint VB Scripting convert failed\n";
+	    } else {
+		return "item";
+	    }
+	}
+    } elsif (!$output_type || ($output_type =~ /html/i)) {
+	# Attempt conversion to HTML
+	#if (!$output_type || ($output_type =~ /html/i)) {
+	# formulate the command
+	$cmd = "";
+	$cmd .= "perl -S ppttohtml.pl ";
+	$cmd .= " \"$input_filename\" \"$output_filestem.html\"";
+	$cmd .= " 2>\"$output_filestem.err\""
+	    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+
+	# execute the command
+	$!=0;
+	if (system($cmd)!=0)
+	{
+	    print STDERR "Powerpoint 95/97 converter failed $!\n";
+	} else {
+	    return "html";
+	}
+    } 
+
+    $success = &any_to_text($input_filename, $output_filestem);
+    if ($success) {
+	return "text";
+    }
+    
+    return "fail";
+}
+
+
+sub convertXLS {
+    my ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+
+    # Attempt conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	# formulate the command
+	$cmd = "";
+	$cmd .= "perl -S xlstohtml.pl ";
+	$cmd .= " \"$input_filename\" \"$output_filestem.html\"";
+	$cmd .= " 2>\"$output_filestem.err\""
+	    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+	
+	
+	# execute the command
+	$!=0;
+	if (system($cmd)!=0)
+	{
+	    print STDERR "Excel 95/97 converter failed $!\n";
+	} else {
+	    return "html";
+	}
+    }
+
+    $success = &any_to_text($input_filename, $output_filestem);
+    if ($success) {
+	return "text";
+    }
+
+    return "fail";
+}
+
+
+
+# Find the real type of a .doc file
+#
+# We seem to have a lot of files with a .doc extension that are .rtf 
+# files or Word 5 files.  This function attempts to tell the difference.
+sub find_docfile_type {
+    ($input_filename) = @_;
+    
+    open(CHK, "<$input_filename");
+    binmode(CHK);
+    my $line = "";
+    my $first = 1;
+
+    while (<CHK>) {
+    
+	$line = $_;
+
+	if ($first) {
+	    # check to see if this is an rtf file
+	    if ($line =~ /^\{\\rtf/) {
+		close(CHK);
+		return "rtf";
+	    }
+	    $first = 0;
+	}
+	
+	# is this is a word 6/7/8 document?
+	if ($line =~ /Word\.Document\.([678])/) {
+	    close(CHK);
+	    return "word$1";
+	}
+
+    }
+
+    return "unknown";
+}
+
+
+# Specific type-to-type conversions
+#
+# Each of the following functions attempts to convert a document from 
+# a specific format to another.  If they succeed they return 1 and leave 
+# the output document(s) in the appropriate place; if they fail they 
+# return 0 and delete any working files.
+
+
+# Attempt to convert a word document to html with the wv program
+sub doc_to_html {
+    ($input_filename, $output_filestem) = @_;
+
+    my $wvWare = &util::filename_cat($ENV{'GSDLHOME'}, "bin", 
+				     $ENV{'GSDLOS'}, "wvWare");
+
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly
+    $wvWare = "wvWare" if ($ENV{'GSDLOS'} =~ /^windows$/i);
+
+    my $wv_conf = &util::filename_cat($ENV{'GSDLHOME'}, "etc", 
+				      "packages", "wv", "wvHtml.xml");
+    
+    my $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    $cmd .= "$wvWare --charset utf-8 --config \"$wv_conf\"";
+    $cmd .= " \"$input_filename\" > \"$output_filestem.html\"";
+    
+    # redirecting STDERR is a bad idea on windows 95/98
+    $cmd .= " 2> \"$output_filestem.err\""
+	if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+    # execute the command
+    $!=0;
+    if (system($cmd)!=0)
+    {
+	print STDERR "Error executing wv converter:$!\n";
+	if (-s "$output_filestem.err") {
+	    open (ERRFILE, "<$output_filestem.err");
+
+	    my $write_to_fail_log=0;
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {$write_to_fail_log=1;}
+
+	    my $line;
+	    while ($line=<ERRFILE>) {
+		if ($line =~ /\w/) {
+		    print STDERR "$line";
+		    print FAILLOG "$line" if ($write_to_fail_log);
+		}
+		if ($line !~ m/startup error/) {next;}
+		print STDERR " (given an invalid .DOC file?)\n";
+		print FAILLOG " (given an invalid .DOC file?)\n"
+		if ($write_to_fail_log);
+		
+	    } # while ERRFILE
+	    close FAILLOG if ($write_to_fail_log);
+	}
+	return 0; # we can try any_to_text
+    }
+
+    # Was the conversion successful?
+
+    if (-s "$output_filestem.html") {
+	open(TMP, "$output_filestem.html");
+	$line = <TMP>;
+	close(TMP);
+	if ($line && $line =~ /DOCTYPE HTML/) {
+	    &util::rm("$output_filestem.err") if -e "$output_filestem.err";
+	    return 1;
+	}
+    }
+	
+    # If here, an error of some sort occurred
+    &util::rm("$output_filestem.html") if -e "$output_filestem.html";
+    if (-e "$output_filestem.err") {
+	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
+	    open (ERRLOG,"$output_filestem.err");
+	    while (<ERRLOG>) {print FAILLOG $_;}
+	    close FAILLOG;
+	    close ERRLOG;
+	}
+	&util::rm("$output_filestem.err");
+    }
+    
+    return 0;
+}
+
+
+# Attempt to convert a word document to html with the word2html scripting program
+sub native_doc_to_html {
+    ($input_filename, $output_filestem) = @_;
+
+    my $vbScript = &util::filename_cat($ENV{'GSDLHOME'}, "bin",
+				       $ENV{'GSDLOS'}, "word2html");
+
+    $vbScript = "word2html" if ($ENV{'GSDLOS'} =~ /^windows$/i);
+    if (-e "$output_filestem.html") {
+	print STDERR "*** The conversion file has existed\n";
+	return 1;
+    }
+
+    my $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    #$cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
+    #$cmd .=  "$vbScript $input_filename $output_filestem.html";
+    $cmd .=  "$vbScript \"$input_filename\" \"$output_filestem.html\"";
+
+    # redirecting STDERR
+    $cmd .= " 2> \"$output_filestem.err\""
+	if ($ENV {'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+    
+    # execute the command
+    $!=0;
+    if (system($cmd)!=0)
+    {
+	print STDERR "Error executing word2Html converter:$!\n";
+	if (-s "$output_filestem.err") {
+	    open (ERRFILE, "<$output_filestem.err");
+	    
+	    my $write_to_fail_log=0;
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {$write_to_fail_log=1;}
+
+	    my $line;
+	    while ($line=<ERRFILE>) {
+		if ($line =~ /\w/) {
+		    print STDERR "$line";
+		    print FAILLOG "$line" if ($write_to_fail_log);
+		}
+		if ($line !~ m/startup error/) {next;}
+		print STDERR " (given an invalid .DOC file?)\n";
+		print FAILLOG " (given an invalid .DOC file?)\n"
+		if ($write_to_fail_log);
+		
+	    } # while ERRFILE
+	    close FAILLOG if ($write_to_fail_log);
+	}
+	return 0; # we can try any_to_text
+    }
+
+    # Was the conversion successful?
+    if (-s "$output_filestem.html") {
+	open(TMP, "$output_filestem.html");
+	$line = <TMP>;
+	close(TMP);
+	if ($line && $line =~ /html/) {
+	    &util::rm("$output_filestem.err") if -e "$output_filestem.err";
+	    return 1;
+	}
+    }
+	
+    # If here, an error of some sort occurred
+    &util::rm("$output_filestem.html") if -e "$output_filestem.html";
+    if (-e "$output_filestem.err") {
+	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
+	    open (ERRLOG,"$output_filestem.err");
+	    while (<ERRLOG>) {print FAILLOG $_;}
+	    close FAILLOG;
+	    close ERRLOG;
+	}
+	&util::rm("$output_filestem.err");
+    }
+    return 0;
+}
+
+# Attempt to convert an RTF document to html with rtftohtml
+
+sub rtf_to_html {
+    my ($input_filename, $output_filestem) = @_;
+
+    # formulate the command
+    $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    $cmd .= "rtftohtml";
+    #$cmd .= "rtf-converter";
+
+    $cmd .= " -o \"$output_filestem.html\" \"$input_filename\"";
+
+    $cmd .= " 2>\"$output_filestem.err\""
+        if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
+
+
+    # execute the command
+    $!=0;
+    if (system($cmd)!=0)
+    {
+	print STDERR "Error executing rtf converter $!\n";
+	# don't currently bother printing out error log...
+	# keep going, in case it still created an HTML file...
+    }
+
+    # Was the conversion successful?
+    my $was_successful=0;
+    if (-s "$output_filestem.html") {
+	# make sure we have some content other than header
+	open (HTML, "$output_filestem.html"); # what to do if fail?
+	my $line;
+	my $past_header=0;
+	while ($line=<HTML>) {
+
+	    if ($past_header == 0) {
+		if ($line =~ /<body>/) {$past_header=1;}
+		next;
+	    }
+
+	    $line =~ s/<[^>]+>//g;
+	    if ($line =~ /\w/ && $past_header) {  # we found some content...
+		$was_successful=1;
+		last;
+	    }
+	}
+	close HTML;
+    }
+
+    if ($was_successful) {
+	&util::rm("$output_filestem.err")
+	    if (-e "$output_filestem.err");
+	# insert the (modified) table of contents, if it exists.
+	if (-e "${output_filestem}_ToC.html") {
+	    &util::mv("$output_filestem.html","$output_filestem.src");
+	    my $open_failed=0;
+	    open HTMLSRC, "$output_filestem.src" || ++$open_failed;
+	    open TOC, "${output_filestem}_ToC.html" || ++$open_failed;
+	    open HTML, ">$output_filestem.html" || ++$open_failed;
+	    
+	    if ($open_failed) {
+		close HTMLSRC;
+		close TOC;
+		close HTML;
+		&util::mv("$output_filestem.src","$output_filestem.html");
+		return 1;
+	    }
+
+	    # print out header info from src html.
+	    while (defined($_ = <HTMLSRC>) && $_ =~ /\w/) {
+		print HTML "$_";
+	    }
+
+	    # print out table of contents, making links relative
+	    <TOC>; <TOC>; # ignore first 2 lines
+	    print HTML scalar(<TOC>); # line 3 = "<ol>\n"
+	    my $line;
+	    while ($line=<TOC>) {
+		$line =~ s@</body></html>$@@ ; # only last line has this
+		# make link relative
+		$line =~ s@href=\"[^\#]+@href=\"@;
+		print HTML $line;
+	    }
+	    close TOC;
+
+	    # rest of html src
+	    while (<HTMLSRC>) {
+		print HTML $_;
+	    }
+	    close HTMLSRC;
+	    close HTML;
+
+	    &util::rm("${output_filestem}_ToC.html");
+	    &util::rm("${output_filestem}.src");
+	}
+	# we don't yet do anything with footnotes ($output_filestem_fn.html) :(
+	return 1; # success
+    }
+
+    if (-e "$output_filestem.err") {
+	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	{
+	    print FAILLOG "Error - rtftohtml - couldn't extract text\n";
+	    #print FAILLOG "Error - rtf-converter - couldn't extract text\n";
+	    print FAILLOG " (rtf file might be too recent):\n";
+	    open (ERRLOG, "$output_filestem.err");
+	    while (<ERRLOG>) {print FAILLOG $_;}
+	    close ERRLOG;
+	    close FAILLOG;
+	}
+	&util::rm("$output_filestem.err");
+    }
+
+    &util::rm("$output_filestem.html") if (-e "$output_filestem.html");
+
+    return 0;
+}
+
+
+# Convert a pdf file to html with the pdftohtml command
+
+sub pdf_to_html {
+    my ($dirname, $input_filename, $output_filestem) = @_;
+
+    $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    $cmd .= "perl -S pdftohtml.pl -zoom $pdf_zoom";
+    $cmd .= " -c" if ($pdf_complex);
+    $cmd .= " -i" if ($pdf_ignore_images);
+    $cmd .= " -a" if ($pdf_allow_images_only);
+    $cmd .= " -hidden" unless ($pdf_nohidden);
+    $cmd .= " \"$input_filename\" \"$output_filestem\"";
+    
+    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000) {
+	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+	$cmd .= " > \"$output_filestem.err\"";
+    }
+
+    $!=0;
+
+    my $retval=system($cmd);
+    if ($retval!=0)
+    {
+	print STDERR "Error executing pdftohtml.pl";
+	if ($!) {print STDERR ": $!";} 
+	print STDERR "\n";
+    }
+
+    # make sure the converter made something
+    if ($retval!=0 || ! -s "$output_filestem.html")
+    {
+	&util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+	# print out the converter's std err, if any
+	if (-s "$output_filestem.err") {
+	    open (ERRLOG, "$output_filestem.err") || die "$!";
+	    print STDERR "pdftohtml error log:\n";
+	    while (<ERRLOG>) {
+		print STDERR "$_";
+	    }
+	    close ERRLOG;
+	}
+	&util::rm("$output_filestem.html") if (-e "$output_filestem.html");
+	if (-e "$output_filestem.err") {
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {
+		open (ERRLOG, "$output_filestem.err");
+		while (<ERRLOG>) {print FAILLOG $_;}
+		close ERRLOG;
+		close FAILLOG;
+	    }	
+	    &util::rm("$output_filestem.err");
+	}
+	return 0;
+    }
+
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+    return 1;
+}
+
+# Convert a pdf file to various types of image with the convert command
+
+sub pdf_to_img {
+    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
+
+    # Check that ImageMagick is installed and available on the path (except for Windows 95/98)
+    if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
+	my $result = `identify 2>&1`;
+	if ($? == -1 || $? == 256) {  # Linux and Windows return different values for "program not found"
+	    #ImageMagick is not installed, thus the convert utility is not available. 
+	    print STDERR "*** ImageMagick is not installed, the convert utility is not available\n";
+	    return 0;
+	}
+    }
+
+    $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    $output_type =~ s/.*\_(.*)/$1/i;
+     $cmd .= "perl -S pdftoimg.pl -convert_to $output_type \"$input_filename\" \"$output_filestem\"";
+    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000) {
+	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+	$cmd .= " > \"$output_filestem.err\"";
+    }
+
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly
+    $!=0;
+    my $retval=system($cmd);
+    if ($retval!=0)
+    {
+	print STDERR "Error executing pdftoimg.pl";
+	if ($!) {print STDERR ": $!";} 
+	print STDERR "\n";
+    }
+
+    #make sure the converter made something
+    #if ($retval !=0) || ! -s "$output_filestem")
+    if ($retval !=0) 
+    {
+	&util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+	#print out the converter's std err, if any
+	if (-s "$output_filestem.err") {
+	    open (ERRLOG, "$output_filestem.err") || die "$!";
+	    print STDERR "pdftoimg error log:\n";
+	    while (<ERRLOG>) {
+		print STDERR "$_";
+	    }
+	    close ERRLOG;
+	}
+	#&util::rm("$output_filestem.html") if (-e "$output_filestem.html");
+	if (-e "$output_filestem.err") {
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {
+		open (ERRLOG, "$output_filestem.err");
+		while (<ERRLOG>) {print FAILLOG $_;}
+		close ERRLOG;
+		close FAILLOG;
+	   }	
+	    &util::rm("$output_filestem.err");
+	}
+	return 0;
+    }
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+    return 1;
+}
+
+# Convert a PDF file to text with the pdftotext command
+
+sub pdf_to_text {
+    my ($dirname, $input_filename, $output_filestem) = @_;
+
+    my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
+
+    if ($ENV{'GSDLOS'} !~ /^windows$/i) {
+	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+	$cmd .= " > \"$output_filestem.err\"";
+    }
+    
+    if (system($cmd)!=0)
+    {
+	print STDERR "Error executing $cmd: $!\n";
+	&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+    }
+
+    # make sure there is some extracted text.
+    if (-e "$output_filestem.text") {
+	open (EXTR_TEXT, "$output_filestem.text") || warn "open: $!";
+	binmode(EXTR_TEXT); # just in case...
+	my $line="";
+	my $seen_text=0;
+	while (($seen_text==0) && ($line=<EXTR_TEXT>)) {
+	    if ($line=~ /\w/) {$seen_text=1;}
+	}
+	close EXTR_TEXT;
+	if ($seen_text==0) { # no text was extracted
+	    print STDERR "Error: pdftotext found no text\n";
+	    &util::rm("$output_filestem.text");
+	}
+    }
+
+    # make sure the converter made something
+    if (! -s "$output_filestem.text")
+    {
+	# print out the converters std err, if any
+	if (-s "$output_filestem.err") {
+	    open (ERRLOG, "$output_filestem.err") || die "$!";
+	    print STDERR "pdftotext error log:\n";
+	    while (<ERRLOG>) {
+		print STDERR "$_";
+	    }
+	    close ERRLOG;
+	}
+	# does this converter create a .out file?
+	&util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+	&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	if (-e "$output_filestem.err") {
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {
+		open (ERRLOG,"$output_filestem.err");
+		while (<ERRLOG>) {print FAILLOG $_;}
+		close ERRLOG;
+		close FAILLOG;
+	    }
+	    &util::rm("$output_filestem.err");
+	}
+	return 0;
+    }
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    return 1;
+}
+
+# Convert a PostScript document to text
+# note - just using "ps2ascii" isn't good enough, as it
+# returns 0 for a postscript interpreter error. ps2ascii is just
+# a wrapper to "gs" anyway, so we use that cmd here.
+
+sub ps_to_text {
+    my ($input_filename, $output_filestem) = @_;
+
+    my $error = "";
+
+    # if we're on windows we'll fall straight through without attempting
+    # to use gs
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$error = "Windows does not support gs";
+
+    } else {
+	my $cmd = "";
+	if ($timeout) {$cmd = "ulimit -t $timeout; ";}
+	$cmd .= "gs -q -dNODISPLAY -dNOBIND -dWRITESYSTEMDICT -dSIMPLE -c save ";
+	$cmd .= "-f ps2ascii.ps \"$input_filename\" -c quit > \"$output_filestem.text\"";
+	#$cmd .= "pstotext -output \"$output_filestem.text\" $input_filename\"";
+	$cmd .= " 2> $output_filestem.err";
+	$!=0;
+
+	my $retcode=system($cmd);
+	$retcode = $? >> 8;  # see man perlfunc - system for this...
+	# if system returns -1 | 127 (couldn't start program), look at $! for message
+
+	if ($retcode!=0) {if ($!) {$error=$!;} else {$error="couldn't run.\n";}}
+	elsif (! -e "$output_filestem.text") {
+	    $error="did not create output file.\n";
+	}
+	else 
+	{   # make sure the interpreter didn't get an error. It is technically
+	    # possible for the actual text to start with this, but....
+	    open PSOUT, "$output_filestem.text";
+	    if (<PSOUT> =~ /^Error: (.*)/) {
+		$error="interpreter error - \"$1\"";
+	    }
+	    close PSOUT;
+	}
+    }
+
+    if ($error ne "")
+    {
+	print STDERR "Warning: Error executing gs: $error\n";
+	&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+
+	if ("$faillogfile" ne "" && defined(open (FAILLOG, ">>$faillogfile")))
+	{
+	    print FAILLOG "gs - $error\n";
+	    if (-e "$output_filestem.err") {
+		open(ERRLOG, "$output_filestem.err");
+		while (<ERRLOG>) {print FAILLOG $_;}
+		close ERRLOG;
+	    }
+	    close FAILLOG;
+	}
+	&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+
+
+	# Fine then. We'll just do a lousy job by ourselves...
+	# Based on 5-line regexp sed script found at:
+	# http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
+	# 
+	print STDERR "Stripping text from postscript\n";
+	my $errorcode=0;
+	open (IN, "$input_filename") 
+	    ||  ($errorcode=1, warn "Couldn't read file: $!");
+	open (OUT, ">$output_filestem.text") 
+	    ||  ($errorcode=1, warn "Couldn't write file: $!");
+	if ($errorcode) {print STDERR "errors\n";return 0;}
+	
+	my $text="";  # this is for whole .ps file...
+	$text = join('', <IN>); # see man perlport, under "System Resources"
+	close IN;
+
+	# Make sure this is a ps file...
+	if ($text !~ /^%!/) {
+	    print STDERR "Bad postscript header: not '%!'\n";
+	    if ($faillogfile ne "" && defined(open(FAILLOG, ">>$faillogfile")))
+	    {
+		print FAILLOG "Bad postscript header: not '%!'\n";
+		close FAILLOG;
+	    }
+	    return 0;
+	}
+
+	# if ps has Page data, then use it to delete all stuff before it.
+	$text =~ s/^.*?%%Page:.*?\n//s; # treat string as single line
+	
+	# remove all leading non-data stuff
+	$text =~ s/^.*?\(//s;
+
+	# remove all newline chars for easier processing
+	$text =~ s/\n//g;
+	
+	# Big assumption here - assume that if any co-ordinates are
+	# given, then we are at the end of a sentence.
+	$text =~ s/\)-?\d+\ -?\d+/\) \(\n\)/g;
+
+	# special characters--
+	$text =~ s/\(\|\)/\(\ - \)/g; # j -> em-dash?
+
+	# ? ps text formatting (eg italics?) ?
+	$text =~ s/Fn\(f\)/\(\{\)/g; # f -> {
+	$text =~ s/Fn\(g\)/\(\}\)/g; # g -> }
+	$text =~ s/Fn\(j\)/\(\|\)/g; # j -> |
+	# default - remove the rest
+	$text =~ s/\ ?F.\((.+?)\)/\($1\)/g;
+
+	# attempt to add whitespace between words... 
+	# this is based purely on observation, and may be completely wrong...
+	$text =~ s/([^F])[defghijkuy]\(/$1 \( /g;
+	# eg I notice "b(" is sometimes NOT a space if preceded by a 
+	# negative number.
+	$text =~ s/\)\d+ ?b\(/\) \( /g;
+
+	# change quoted braces to brackets
+	$text =~ s/([^\\])\\\(/$1\{/g;
+	$text =~ s/([^\\])\\\)/$1\}/g ;
+
+	# remove everything that is not between braces
+	$text =~ s/\)([^\(\)])+?\(//sg ;
+	
+	# remove any Trailer eof stuff.
+	$text =~ s/\)[^\)]*$//sg;
+
+	### ligatures have special characters...
+	$text =~ s/\\013/ff/g;
+	$text =~ s/\\014/fi/g;
+	$text =~ s/\\015/fl/g;
+	$text =~ s/\\016/ffi/g;
+	$text =~ s/\\214/fi/g;
+	$text =~ s/\\215/fl/g;
+	$text =~ s/\\017/\n\* /g; # asterisk?
+	$text =~ s/\\023/\023/g;  # e acute ('e)
+	$text =~ s/\\177/\252/g;  # u"
+#	$text =~ s/ ?? /\344/g;  # a"
+
+	print OUT "$text";
+	close OUT;
+    }
+    # wrap the text - use a minimum length. ie, first space after this length.
+    my $wrap_length=72;
+    &util::mv("$output_filestem.text", "$output_filestem.text.tmp");
+    open INFILE, "$output_filestem.text.tmp" ||
+	die "Couldn't open file: $!";
+    open OUTFILE, ">$output_filestem.text" ||
+	die "Couldn't open file for writing: $!";
+    my $line="";
+    while ($line=<INFILE>) {
+	while (length($line)>0) {
+	    if (length($line)>$wrap_length) {
+		$line =~ s/^(.{$wrap_length}[^\s]*)\s*//;
+		print OUTFILE "$1\n";
+	    } else {
+		print OUTFILE "$line";
+		$line="";
+	    }
+	}
+    }
+    close INFILE;
+    close OUTFILE;
+    &util::rm("$output_filestem.text.tmp");
+
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    return 1;
+}
+
+
+# Convert a PS file to various types of image with the convert utility
+sub ps_to_img {
+    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
+
+    # Check that ImageMagick is installed and available on the path (except for Windows 95/98)
+    if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
+	my $result = `identify 2>&1`;
+	if ($? == -1 || $? == 256) {  # Linux and Windows return different values for "program not found"
+	    #ImageMagick is not installed, thus the convert utility is not available. 
+	    print STDERR "*** ImageMagick is not installed, the convert utility is not available\n";
+	    return 0;
+	}
+    }
+
+    $cmd = "";
+    if ($timeout) {$cmd = "ulimit -t $timeout;";}
+    $output_type =~ s/.*\_(.*)/$1/i;
+    $cmd .= "perl -S pstoimg.pl -convert_to $output_type \"$input_filename\" \"$output_filestem\"";
+    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000) {
+	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+	$cmd .= " > \"$output_filestem.err\"";
+    }
+
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly
+    $!=0;
+    my $retval=system($cmd);
+    if ($retval!=0)
+    {
+	print STDERR "Error executing pstoimg.pl";
+	if ($!) {print STDERR ": $!";} 
+	print STDERR "\n";
+    }
+    
+    #make sure the converter made something
+    #if ($retval !=0) || ! -s "$output_filestem")
+    if ($retval !=0) 
+    {
+	&util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+	#print out the converter's std err, if any
+	if (-s "$output_filestem.err") {
+	    open (ERRLOG, "$output_filestem.err") || die "$!";
+	    print STDERR "pstoimg error log:\n";
+	    while (<ERRLOG>) {
+		print STDERR "$_";
+	    }
+	    close ERRLOG;
+	}
+	#&util::rm("$output_filestem.html") if (-e "$output_filestem.html");
+	if (-e "$output_filestem.err") {
+	    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+	    {
+		open (ERRLOG, "$output_filestem.err");
+		while (<ERRLOG>) {print FAILLOG $_;}
+		close ERRLOG;
+		close FAILLOG;
+	   }	
+	    &util::rm("$output_filestem.err");
+	}
+	return 0;
+    }
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+    return 1;
+}
+
+# Convert any file to HTML with a crude perl implementation of the
+# UNIX strings command.
+
+sub any_to_html {
+    ($input_filename, $output_filestem) = @_;
+
+    # First generate a text file
+    return 0 unless (&any_to_text($input_filename, $output_filestem));
+
+    # create an HTML file from the text file
+    open(TEXT, "<$output_filestem.text");
+    open(HTML, ">$output_filestem.html");
+
+    print HTML "<html><head>\n";
+    print HTML "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html\">\n";
+    print HTML "<META NAME=\"GENERATOR\" CONTENT=\"Greenstone any_to_html\">\n";
+    print HTML "</head><body>\n\n";
+
+    my $line;
+    while ($line=<TEXT>) { 
+	$line =~ s/</&lt;/g;
+	$line =~ s/>/&gt;/g;
+	if ($line =~ /^\s*$/) {
+	    print HTML "<p>";
+	} else {
+	    print HTML "<br> ", $line;
+	}
+    }
+    print HTML "\n</body></html>\n";
+
+    close HTML;
+    close TEXT;
+
+    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+    return 1;
+}
+
+# Convert any file to TEXT with a crude perl implementation of the
+# UNIX strings command.
+# Note - this assumes ascii charsets :(		(jrm21)
+
+sub any_to_text {
+    ($input_filename, $output_filestem) = @_;
+
+    if (!$use_strings) {
+      return 0;
+    }
+    
+    open(IN, "<$input_filename") || return 0;
+    binmode(IN);
+    open(OUT, ">$output_filestem.text") || return 0;
+
+    my ($line);
+    my $output_line_count = 0;
+    while (<IN>) { 
+	$line = $_;
+
+	# delete anything that isn't a printable character
+	$line =~ s/[^\040-\176]+/\n/sg;
+
+	# delete any string less than 10 characters long
+	$line =~ s/^.{0,9}$/\n/mg;
+	while ($line =~ /^.{1,9}$/m) {
+	    $line =~ s/^.{0,9}$/\n/mg;
+	    $line =~ s/\n+/\n/sg;
+	}
+
+	# remove extraneous whitespace
+	$line =~ s/\n+/\n/gs;
+	$line =~ s/^\n//gs;
+
+	# output whatever is left
+	if ($line =~ /[^\n ]/) {
+	    print OUT $line;
+	    ++$output_line_count;
+	}
+    }
+
+    close OUT;
+    close IN;
+
+    if ($output_line_count) { # try to protect against binary only formats
+	return 1;
+    }
+
+    &util::rm("$output_filestem.text");
+    return 0;
+
+}
Index: /main/tags/3.01/gsdl/bin/script/gsWget.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gsWget.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gsWget.pl	(revision 21112)
@@ -0,0 +1,72 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will download the specified urls (http:, ftp: and file:)
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use util;
+
+# wget should live in the Greenstone directory structure
+# we'll bail if we can't find it
+my $exe = &util::get_os_exe ();
+my $cmd = &util::filename_cat ($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+$cmd .= $exe;
+if (! -e "$cmd") {
+    die "gsWget.pl failed: $cmd doesn't exist\n";
+}
+
+# if on windows we expect wget to already be on the path -
+# this allows us to avoid problems when GSDLHOME contains spaces
+# (double quoting the call doesn't work on win2000)
+if ($ENV{'GSDLOS'} =~ /^windows$/) {
+    $cmd = "wget";
+}
+
+# command-line parameters
+my @quoted_argv = map { "\"$_\"" } @ARGV;
+my $args = join(' ', @quoted_argv);
+$cmd .= " $args";
+
+# run the command
+my $status = system($cmd);
+
+# We should check the error status of wget; unfortunately this
+# is set to 1 even when we exit successfully, so we ignore it.
+#
+#$status /= 256;
+#if ($status != 0) {
+#    print STDERR "Error executing $cmd: $!\n";
+#    exit($status);
+#}
+
+print "\nDone: $cmd\n";
Index: /main/tags/3.01/gsdl/bin/script/gsdlinfo.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gsdlinfo.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gsdlinfo.pl	(revision 21112)
@@ -0,0 +1,142 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# gsdlinfo.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This program will look to see what collections are installed
+# under this file system
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use colcfg;
+
+if (scalar(@ARGV) && $ARGV[0] eq "--help") {
+    print STDERR "\n";
+    print STDERR "gsdlinfo.pl: Prints information on any Greenstone collections\n";
+    print STDERR "             found on the local file system.\n\n";
+    exit;
+}
+
+
+# the collection information is stored as an array of hashes
+# (one hash for each collection). The hash contains all the 
+# information in the collect and build configuration files
+
+
+# read in all the collection configuration information
+
+opendir (DIR, "$ENV{'GSDLHOME'}/collect") ||
+    die "ERROR - couldn't read the directory $ENV{'GSDLHOME'}/collect\n";
+@dir = readdir (DIR);
+closedir (DIR);
+
+@collections = ();
+foreach $dir (@dir) {
+    if ($dir =~ /^(\..*|modelcol|CVS)$/) {
+	# do nothing
+
+    } else {
+	# read in the collection configuration file
+	my $collectcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/etc/collect.cfg") {
+	    $collectcfg = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/$dir/etc/collect.cfg");
+	}
+	$collectcfg->{'collection'} = $dir;
+
+	# read in the build configuration file
+	my $buildcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg") {
+	    $buildcfg = &colcfg::read_build_cfg ("$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg");
+	}
+
+	# add the merged configuration files to the current list of collections
+	push (@collections, {%$collectcfg, %$buildcfg});
+    }
+}
+
+
+# do any sorting requested
+
+
+# print out the collection information
+
+# {'creator'}->string
+# {'maintainer'}->array of strings
+# {'public'}->string
+# {'beta'}->string
+# {'key'}->string
+# {'indexes'}->array of strings
+# {'defaultindex'}->string
+# {'builddate'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+
+print "\n";
+foreach $info (@collections) {
+    if (defined $info->{'collection'}) {
+	$collection = $info->{'collection'};
+	if (defined $info->{'public'}) {
+	    $public = $info->{'public'};
+	} else {
+	    $public = "?";
+	}
+	if (defined $info->{'beta'}) {
+	    $beta = $info->{'beta'};
+	} else {
+	    $beta = "?";
+	}
+	if (defined $info->{'numdocs'}) {
+	    $numdocs = $info->{'numdocs'};
+	} else {
+	    $numdocs = "?";
+	}
+	if (defined $info->{'numbytes'}) {
+	    $numMbytes = int(($info->{'numbytes'}/1024.0/1024.0)*1000)/1000.0;
+	} else {
+	    $numMbytes = "?";
+	}
+	write;
+    }
+}
+print "\n";
+
+exit;
+
+format STDOUT_TOP = 
+  name   public  beta   documents  size (Mbytes)
+-------- ------ ------ ----------- -------------
+.
+
+format STDOUT = 
+@>>>>>>> @>>>>> @>>>>> @>>>>>>>>>> @>>>>>>>>>>>>
+$collection, $public, $beta, $numdocs, $numMbytes
+.
Index: /main/tags/3.01/gsdl/bin/script/gsw3mir.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gsw3mir.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gsw3mir.pl	(revision 21112)
@@ -0,0 +1,52 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# urldownload.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will download the specified urls (http:, ftp: and file:)
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/lib");
+    $ENV{'PATH'} = "$ENV{'GSDLHOME'}/perllib/cpan/bin:$ENV{'PATH'}";
+}
+
+my @quoted_argv = map { "\"$_\"" } @ARGV;
+
+my $args = join(' ', @quoted_argv);
+$cmd = "w3mir $args";
+
+my $status = system($cmd);
+$status /= 256;
+if ($status != 0)
+{
+    print STDERR "Error: failed to execute $cmd\n";
+    exit($status);
+}
+
Index: /main/tags/3.01/gsdl/bin/script/gti.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/gti.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/gti.pl	(revision 21112)
@@ -0,0 +1,1166 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# gti.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+
+use iso639;
+use strict;
+use util;
+
+
+my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
+my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
+my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
+my $source_language_code = "en";  # This is non-negiotable
+
+my $gti_translation_files =
+    [ # Greenstone macrofiles
+      { 'key' => "coredm",
+	'file_type' => "macrofile",
+	'source_file' => "macros/english.dm",
+	'target_file' => "macros/{bn:bengali;fa:farsi;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
+      { 'key' => "auxdm",
+	'file_type' => "macrofile",
+	'source_file' => "macros/english2.dm",
+	'target_file' => "macros/{bn:bengali;fa:farsi;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
+
+      # GLI dictionary
+      { 'key' => "glidict",
+	'file_type' => "resource_bundle",
+	'source_file' => "gli/classes/dictionary.properties",
+	'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
+
+      # Greenstone Perl modules
+      { 'key' => "perlmodules",
+	'file_type' => "resource_bundle",
+	'source_file' => "perllib/strings.rb",
+	'target_file' => "perllib/strings_{target_language_code}.rb" } ];
+
+
+sub main
+{
+    # Get the command to process, and any arguments
+    my $gti_command = shift(@_);
+    my @gti_command_arguments = @_;
+
+    # Open the GTI log file for appending, or write to STDERR if that fails
+    if (!open(GTI_LOG, ">>$gti_log_file")) {
+	open(GTI_LOG, ">&STDERR");
+    }
+
+    # Log the command that launched this script
+    &log_message("Command: $0 @ARGV");
+
+    # Check that a command was supplied
+    if (!$gti_command) {
+	&throw_fatal_error("Missing command.");
+    }
+
+    # Process the command
+    if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
+	print &get_first_n_chunks_requiring_work(@gti_command_arguments);
+    }
+    if ($gti_command =~ /^get-language-status$/i) {
+	print &get_language_status(@gti_command_arguments);
+    }
+    if ($gti_command =~ /^search-chunks$/i) {
+	print &search_chunks(@gti_command_arguments);
+    }
+    if ($gti_command =~ /^submit-translations$/i) {
+	# This command cannot produce any output since it reads input
+	&submit_translations(@gti_command_arguments);
+    }
+
+    # The command was not recognized
+    # &throw_fatal_error("Unknown command \"$gti_command\".");
+}
+
+
+sub throw_fatal_error
+{
+    my $error_message = shift(@_);
+
+    # Write an XML error response
+    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
+    print "<GTIResponse>\n";
+    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
+    print "</GTIResponse>\n";
+
+    # Log the error message, then die
+    &log_message("Error: $error_message");
+    die "\n";
+}
+
+
+sub log_message
+{
+    my $log_message = shift(@_);
+    print GTI_LOG time() . " -- " . $log_message . "\n";
+}
+
+
+sub get_first_n_chunks_requiring_work
+{
+    # The code of the target language (ensure it is lowercase)
+    my $target_language_code = lc(shift(@_));
+    # The key of the file to translate (ensure it is lowercase)
+    my $translation_file_key = lc(shift(@_));
+    # The number of chunks to return (defaults to one if not specified)
+    my $num_chunks_to_return = shift(@_) || "1";
+
+    # Check that the necessary arguments were supplied
+    if (!$target_language_code || !$translation_file_key) {
+	&throw_fatal_error("Missing command argument.");
+    }
+
+    # Get (and check) the translation configuration
+    my ($source_file, $target_file, $translation_file_type)
+	= &get_translation_configuration($target_language_code, $translation_file_key);
+
+    # Parse the source language and target language files
+    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
+    my @source_file_lines = &read_file_lines($source_file_path);
+    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
+
+    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
+    my @target_file_lines = &read_file_lines($target_file_path);
+    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
+
+    # Filter out any automatically translated chunks
+    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
+	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
+	    delete $source_file_key_to_line_mapping{$chunk_key};
+	    delete $target_file_key_to_line_mapping{$chunk_key};
+	}
+    }
+
+    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
+    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
+
+    # Determine the target file chunks requiring translation
+    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
+    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
+
+    # Determine the target file chunks requiring updating
+    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
+    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
+
+    # Form an XML response to the command
+    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
+    $xml_response .= "<GTIResponse>\n";
+    $xml_response .= "  <TranslationFile"
+	. " key=\"" . $translation_file_key . "\""
+	. " target_file_path=\"" . $target_file . "\""
+	. " num_chunks_translated=\"" . (scalar(keys(%source_file_key_to_text_mapping)) - scalar(@target_file_keys_requiring_translation)) . "\""
+	. " num_chunks_requiring_translation=\"" . scalar(@target_file_keys_requiring_translation) . "\""
+	. " num_chunks_requiring_updating=\"" . scalar(@target_file_keys_requiring_updating) . "\"\/>\n";
+
+    # Do chunks requiring translation first
+    if ($num_chunks_to_return > scalar(@target_file_keys_requiring_translation)) {
+	$xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@target_file_keys_requiring_translation) . "\">\n";
+    }
+    else {
+	$xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
+    }
+
+    foreach my $chunk_key (@target_file_keys_requiring_translation) {
+	last if ($num_chunks_to_return == 0);
+
+	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
+	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping{$chunk_key});
+
+	$xml_response .= "    <Chunk key=\"$chunk_key\">\n";
+	$xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
+	$xml_response .= "      <TargetFileText></TargetFileText>\n";
+	$xml_response .= "    </Chunk>\n";
+
+	$num_chunks_to_return--;
+    }
+
+    $xml_response .= "  </ChunksRequiringTranslation>\n";
+
+    # Then do chunks requiring updating
+    if ($num_chunks_to_return > scalar(@target_file_keys_requiring_updating)) {
+	$xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@target_file_keys_requiring_updating) . "\">\n";
+    }
+    else {
+	$xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
+    }
+
+    foreach my $chunk_key (@target_file_keys_requiring_updating) {
+	last if ($num_chunks_to_return == 0);
+
+	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
+	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping{$chunk_key});
+	my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
+	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
+
+	$xml_response .= "    <Chunk key=\"$chunk_key\">\n";
+	$xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
+	$xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
+	$xml_response .= "    </Chunk>\n";
+
+	$num_chunks_to_return--;
+    }
+
+    $xml_response .= "  </ChunksRequiringUpdating>\n";
+
+    $xml_response .= "</GTIResponse>\n";
+    return $xml_response;
+}
+
+
+sub get_language_status
+{
+    # The code of the target language (ensure it is lowercase)
+    my $target_language_code = lc(shift(@_));
+
+    # Check that the necessary arguments were supplied
+    if (!$target_language_code) {
+	&throw_fatal_error("Missing command argument.");
+    }
+
+    # Form an XML response to the command
+    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
+    $xml_response .= "<GTIResponse>\n";
+    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
+
+    foreach my $translation_file (@$gti_translation_files) {
+	# Get (and check) the translation configuration
+	my ($source_file, $target_file, $translation_file_type)
+	    = &get_translation_configuration($target_language_code, $translation_file->{'key'});
+
+	# Parse the source language and target language files
+	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
+	my @source_file_lines = &read_file_lines($source_file_path);
+	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
+
+	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
+	my @target_file_lines = &read_file_lines($target_file_path);
+	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
+
+	# Filter out any automatically translated chunks
+	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
+	    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
+		delete $source_file_key_to_line_mapping{$chunk_key};
+		delete $target_file_key_to_line_mapping{$chunk_key};
+	    }
+	}
+
+	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
+	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
+
+	# Determine the target file chunks requiring translation
+	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
+	&log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
+
+	# Determine the target file chunks requiring updating
+	my @target_file_keys_requiring_updating = ();
+	if (-e $target_file_path) {
+	    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+	    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+	    @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
+	    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
+	}
+
+	$xml_response .= "    <TranslationFile"
+	    . " key=\"" . $translation_file->{'key'} . "\""
+	    . " target_file_path=\"" . $target_file . "\""
+	    . " num_chunks_translated=\"" . (scalar(keys(%source_file_key_to_text_mapping)) - scalar(@target_file_keys_requiring_translation)) . "\""
+	    . " num_chunks_requiring_translation=\"" . scalar(@target_file_keys_requiring_translation) . "\""
+	    . " num_chunks_requiring_updating=\"" . scalar(@target_file_keys_requiring_updating) . "\"\/>\n";
+    }
+
+    $xml_response .= "  </LanguageStatus>\n";
+
+    $xml_response .= "</GTIResponse>\n";
+    return $xml_response;
+}
+
+
+sub search_chunks
+{
+    # The code of the target language (ensure it is lowercase)
+    my $target_language_code = lc(shift(@_));
+    # The key of the file to translate (ensure it is lowercase)
+    my $translation_file_key = lc(shift(@_));
+    # The query string
+    my $query_string = join(' ', @_);
+
+    # Check that the necessary arguments were supplied
+    if (!$target_language_code || !$translation_file_key || !$query_string) {
+	&throw_fatal_error("Missing command argument.");
+    }
+
+    # Get (and check) the translation configuration
+    my ($source_file, $target_file, $translation_file_type)
+	= &get_translation_configuration($target_language_code, $translation_file_key);
+
+    # Parse the source language and target language files
+    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
+    my @source_file_lines = &read_file_lines($source_file_path);
+    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
+
+    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
+    my @target_file_lines = &read_file_lines($target_file_path);
+    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
+
+    # Filter out any automatically translated chunks
+    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
+	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
+	    delete $source_file_key_to_line_mapping{$chunk_key};
+	    delete $target_file_key_to_line_mapping{$chunk_key};
+	}
+    }
+
+    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
+    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
+
+    # Determine the target file chunks matching the query
+    my @target_file_keys_matching_query = ();
+    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
+	my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
+	if ($target_file_text =~ /$query_string/i) {
+	    # &log_message("Chunk with key $chunk_key matches query.");
+	    push(@target_file_keys_matching_query, $chunk_key);
+	}
+    }
+
+    # Form an XML response to the command
+    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
+    $xml_response .= "<GTIResponse>\n";
+
+    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
+    foreach my $chunk_key (@target_file_keys_matching_query) {
+ 	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
+
+ 	$xml_response .= "    <Chunk key=\"$chunk_key\">\n";
+ 	$xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
+ 	$xml_response .= "    </Chunk>\n";
+    }
+    $xml_response .= "  </ChunksMatchingQuery>\n";
+
+    $xml_response .= "</GTIResponse>\n";
+    return $xml_response;
+}
+
+
+sub submit_translations
+{
+    # The code of the target language (ensure it is lowercase)
+    my $target_language_code = lc(shift(@_));
+    # The key of the file to translate (ensure it is lowercase)
+    my $translation_file_key = lc(shift(@_));
+    # Whether to submit a target chunk even if it hasn't changed
+    my $force_submission_flag = shift(@_);
+
+    # Check that the necessary arguments were supplied
+    if (!$target_language_code || !$translation_file_key) {
+	&log_message("Fatal error (but cannot be thrown): Missing command argument.");
+	die "\n";
+    }
+
+    # Get (and check) the translation configuration
+    my ($source_file, $target_file, $translation_file_type)
+	= &get_translation_configuration($target_language_code, $translation_file_key);
+
+    # Parse the source language and target language files
+    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
+    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
+    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+    my %source_file_key_to_comment_date_mapping = &build_key_to_comment_date_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
+    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
+
+    my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
+    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
+    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+    my %target_file_key_to_comment_date_mapping = &build_key_to_comment_date_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
+    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
+
+    # Submission date
+    my $day = (localtime)[3];
+    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
+    my $year = (localtime)[5] + 1900;
+    my $submission_date = "$day-$month-$year";
+
+    open(SUBMISSION, "-");
+    my @submission_lines = <SUBMISSION>;
+    close(SUBMISSION);
+
+    # Remove any nasty carriage returns
+    &log_message("Submission:");
+    foreach my $submission_line (@submission_lines) {
+	$submission_line =~ s/\r$//;
+	&log_message("  $submission_line");
+    }
+
+    my %source_file_key_to_submission_mapping = ();
+    my %target_file_key_to_submission_mapping = ();
+    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
+	# Read source file part of submission
+	if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
+	    my $chunk_key = $1;
+
+	    # Read the source file text
+	    my $source_file_chunk_text = "";
+	    $i++;
+	    while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
+		$source_file_chunk_text .= $submission_lines[$i];
+ 		$i++;
+	    }
+	    $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
+
+	    &log_message("Source file key: $chunk_key");
+	    &log_message("Source file text: $source_file_chunk_text");
+	    $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
+	}
+
+	# Read target file part of submission
+	if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
+	    my $chunk_key = $1;
+
+	    # Read the target file text
+	    my $target_file_chunk_text = "";
+	    $i++;
+	    while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
+		$target_file_chunk_text .= $submission_lines[$i];
+ 		$i++;
+	    }
+	    $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
+
+	    &log_message("Target file key: $chunk_key");
+	    &log_message("Target file text: $target_file_chunk_text");
+	    $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
+	}
+    }
+
+    # -----------------------------------------
+    #   Validate the translation submissions
+    # -----------------------------------------
+
+    # Check that the translations are valid
+    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
+	# Make sure the submitted chunk still exists in the source file
+	if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
+	    &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
+	    delete $source_file_key_to_submission_mapping{$chunk_key};
+	    delete $target_file_key_to_submission_mapping{$chunk_key};
+	    next;
+	}
+
+	# Make sure the submitted source chunk matches the source file chunk
+	if ($source_file_key_to_submission_mapping{$chunk_key} ne $source_file_key_to_text_mapping{$chunk_key}) {
+	    &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
+	    &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
+	    &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
+	    delete $source_file_key_to_submission_mapping{$chunk_key};
+	    delete $target_file_key_to_submission_mapping{$chunk_key};
+	    next;
+	}
+    }
+
+    # Apply the submitted translations
+    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
+	# Only apply the submission if it is a change, unless -force_submission has been specified
+	if ($force_submission_flag || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
+	    $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
+	    $target_file_key_to_comment_date_mapping{$chunk_key} = $submission_date;
+	}
+    }
+
+    eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_comment_date_mapping, \$target_language_code)";
+}
+
+
+sub get_translation_configuration
+{
+    # Get the code of the target language
+    my $target_language_code = shift(@_);
+    # Get the key of the file to translate
+    my $translation_file_key = shift(@_);
+
+    # Read the translation data from the gti.cfg file
+    my ($source_file, $target_file, $translation_file_type) =
+	&get_translation_data_for($target_language_code, $translation_file_key);
+
+    # Check that the file to translate is defined in the gti.cfg file
+    if (!$source_file || !$target_file || !$translation_file_type) {
+	&throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
+    }
+
+    # Check that the source file exists
+    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
+    if (!-e $source_file_path) {
+	&throw_fatal_error("Source file $source_file_path does not exist.");
+    }
+
+    # Check that the source file is up to date
+    # unless ($translation_file_is_not_in_cvs) {
+	my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file`;
+	if ($source_file_cvs_status =~ /^C /) {
+	    &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
+	}
+	if ($source_file_cvs_status =~ /^M /) {
+	    &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
+	}
+    # }
+
+    return ($source_file, $target_file, $translation_file_type);
+}
+
+
+sub get_translation_data_for
+{
+    my ($target_language_code, $translation_file_key) = @_;
+
+    foreach my $translation_file (@$gti_translation_files) {
+	# If this isn't the correct translation file, move onto the next one
+	next if ($translation_file_key ne $translation_file->{'key'});
+
+	# Resolve the target language file
+	my $target_language_file = $translation_file->{'target_file'};
+	if ($target_language_file =~ /(\{.+\;.+\})/) {
+	    my $unresolved_target_language_file_part = $1;
+
+	    # Check for a special case for the target language code
+	    if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
+		my $resolved_target_language_file_part = $2;
+		$target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
+	    }
+	    # Otherwise use the last part as the default value
+	    else {
+		my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
+		$target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;		    
+	    }
+	}
+
+	# Resolve instances of {iso_639_1_target_language_name}
+	my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
+	$iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
+	$target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
+
+	# Resolve instances of {target_language_code}
+	$target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
+
+	return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
+    }
+
+    return ();
+}
+
+
+sub read_file_lines
+{
+    my ($file_path) = @_;
+
+    if (!open(FILE_IN, "<$file_path")) {
+	&log_message("Note: Could not open file $file_path.");
+	return ();
+    }
+    my @file_lines = <FILE_IN>;
+    close(FILE_IN);
+
+    return @file_lines;
+}
+
+
+sub build_key_to_line_mapping
+{
+    my ($file_lines, $translation_file_type) = @_;
+    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
+}
+
+
+sub build_key_to_text_mapping
+{
+    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
+
+    my %key_to_text_mapping = ();
+    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
+	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
+	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
+
+	my $chunk_text = @$file_lines[$chunk_starting_line];
+	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
+	    $chunk_text .= @$file_lines[$l];
+	}
+
+	# Map from chunk key to text
+	eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
+    }
+
+    return %key_to_text_mapping;
+}
+
+
+sub build_key_to_last_update_date_mapping
+{
+    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
+
+    # If the files aren't in CVS then we can't tell anything about what needs updating
+    # return () if ($translation_file_is_not_in_cvs);
+
+    # Build a mapping from key to CVS date
+    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
+    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
+
+    # Build a mapping from key to comment date
+    my %key_to_comment_date_mapping = &build_key_to_comment_date_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
+
+    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
+    my %key_to_last_update_date_mapping = ();
+    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
+	# Use the CVS date as a starting point
+	my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
+	$key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
+
+	# If a comment date exists and it is after the CVS date, use that instead
+	my $chunk_comment_date = $key_to_comment_date_mapping{$chunk_key};
+	if (defined($chunk_comment_date) && (!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
+	    $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
+	}
+    }
+
+    return %key_to_last_update_date_mapping;
+}
+
+
+sub build_key_to_cvs_date_mapping
+{
+    my ($filename, $translation_file_type) = @_;
+
+    # Use CVS to annotate each line of the file with the date it was last edited
+    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
+    my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;
+    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
+
+    my @cvs_annotated_file_lines_date = ();
+    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
+	# Extract the date from the CVS annotation at the front
+	$cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
+	push(@cvs_annotated_file_lines_date, $1);
+    }
+
+    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
+    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
+
+    my %key_to_cvs_date_mapping = ();
+    foreach my $chunk_key (keys(%key_to_line_mapping)) {
+ 	my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
+ 	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
+
+ 	# Find the date this chunk was last edited, from the CVS annotation
+ 	my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
+ 	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
+ 	    if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
+ 		# This part of the chunk has been updated more recently
+ 		$chunk_date = $cvs_annotated_file_lines_date[$l];
+ 	    }
+ 	}
+
+ 	# Map from chunk key to CVS date
+ 	$key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
+    }
+
+    return %key_to_cvs_date_mapping;
+}
+
+
+sub build_key_to_comment_date_mapping
+{
+    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
+
+    my %key_to_comment_date_mapping = ();
+    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
+	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
+	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
+
+	my $chunk_text = @$file_lines[$chunk_starting_line];
+	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
+	    $chunk_text .= @$file_lines[$l];
+	}
+
+	# Map from chunk key to comment date
+	my $chunk_comment_date;
+	eval "\$chunk_comment_date = &get_${translation_file_type}_chunk_comment_date(\$chunk_text)";
+	$key_to_comment_date_mapping{$chunk_key} = $chunk_comment_date if (defined($chunk_comment_date));
+    }
+
+    return %key_to_comment_date_mapping;
+}
+
+
+sub determine_chunks_requiring_translation
+{
+    my $source_file_key_to_text_mapping = shift(@_);
+    my $target_file_key_to_text_mapping = shift(@_);
+
+    # Chunks needing translation are those in the source file with no translation in the target file
+    my @target_file_keys_requiring_translation = ();
+    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
+	if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
+	    # &log_message("Chunk with key $chunk_key needs translating.");
+	    push(@target_file_keys_requiring_translation, $chunk_key);
+	}
+    }
+
+    return @target_file_keys_requiring_translation;
+}
+
+
+sub determine_chunks_requiring_updating
+{
+    my $source_file_key_to_last_update_date_mapping = shift(@_);
+    my $target_file_key_to_last_update_date_mapping = shift(@_);
+
+    # Chunks needing updating are those in the target file that have been more recently edited in the source file
+    my @target_file_keys_requiring_updating = ();
+    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
+	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
+ 	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
+ 	if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
+	    # &log_message("Chunk with key $chunk_key needs updating.");
+	    push(@target_file_keys_requiring_updating, $chunk_key);
+ 	}
+    }
+
+    return @target_file_keys_requiring_updating;
+}
+
+
+sub is_chunk_automatically_translated
+{
+    my ($chunk_key, $translation_file_type) = @_;
+    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
+}
+
+
+sub make_text_xml_safe
+{
+    my $text = shift(@_);
+    $text =~ s/\&/\&amp\;/g;
+    $text =~ s/</\&lt\;/g;
+    $text =~ s/>/\&gt\;/g;
+    return $text;
+}
+
+
+# Returns 1 if $date1 is after $date2, 0 otherwise
+sub is_date_after
+{
+    my ($date1, $date2) = @_;
+    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
+		  "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
+
+    my @date1parts = split(/-/, $date1);
+    my @date2parts = split(/-/, $date2);
+
+    # Compare year - nasty because we have rolled over into a new century
+    my $year1 = $date1parts[2];
+    if ($year1 < 80) {
+	$year1 += 100;
+    }
+    my $year2 = $date2parts[2];
+    if ($year2 < 80) {
+	$year2 += 100;
+    }
+
+    # Compare year
+    if ($year1 > $year2) {
+	return 1;
+    }
+    elsif ($year1 == $year2) {
+	# Year is the same, so compare month
+	if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
+	    return 1;
+	}
+	elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
+	    # Month is the same, so compare day
+	    if ($date1parts[0] > $date2parts[0]) {
+		return 1;
+	    }
+	}
+    }
+
+    return 0;
+}
+
+
+# ==========================================================================================
+#   MACROFILE FUNCTIONS
+
+sub build_key_to_line_mapping_for_macrofile
+{
+    my (@file_lines) = @_;
+
+    my $macro_package;
+    my %chunk_key_to_line_mapping = ();
+    # Process the contents of the file, line by line
+    for (my $i = 0; $i < scalar(@file_lines); $i++) {
+	my $line = $file_lines[$i];
+	$line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
+
+	# Check if a new package is being defined
+	if ($line =~ m/^package\s+(.+)/) {
+	    $macro_package = $1;
+	}
+
+	# Line contains a macro name
+	elsif ($line =~ m/^(_\w+_)/) {
+	    my $macro_key = $1;
+	    $line =~ s/\s*([^\\]\#.+)?$//;  # Remove any comments and nasty whitespace
+
+	    # While there is still text of the macro to go...
+	    my $startline = $i;
+	    while ($line !~ /\}$/) {
+		$i++;
+		if ($i == scalar(@file_lines)) {
+		    &throw_fatal_error("Could not find end of macro $macro_key.");
+		}
+		$line = $file_lines[$i];
+		$line =~ s/\s*([^\\]\#.+)?$//;  # Remove any comments and nasty whitespace
+	    }
+
+	    # The chunk key consists of the package name and the macro key
+	    my $chunk_key = $macro_package . "." . $macro_key;
+	    # Map from chunk key to line
+	    $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
+	}
+
+	# Icon: line in format ## "image text" ## image_type ## macro_name ##
+	elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
+	    # The chunk key consists of package name and macro key
+	    my $chunk_key = $macro_package . "." . $1;
+	    # Map from chunk key to line
+	    $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
+	}
+    }
+
+    return %chunk_key_to_line_mapping;
+}
+
+
+sub import_chunk_from_macrofile
+{
+    my ($chunk_text) = @_;
+
+    # Is this an icon macro??
+    if ($chunk_text =~ /^\#\# (.*)/) {
+	# Extract image macro text
+	$chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
+	$chunk_text = $1;
+
+	# Remove enclosing quotes
+	$chunk_text =~ s/^\"//;
+	$chunk_text =~ s/\"$//;
+    }
+
+    # No, so it must be a text macro
+    else {
+	# Remove macro key
+	$chunk_text =~ s/^_([^_]+)_(\s*)//;
+
+	# Remove language specifier
+	$chunk_text =~ s/^\[l=.*\](\s*)//;
+
+	# Remove braces enclosing text
+	$chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
+    }
+
+    return $chunk_text;
+}
+
+
+sub get_macrofile_chunk_comment_date
+{
+    my ($chunk_text) = @_;
+
+    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
+    if ($chunk_text =~ /\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d)\s*$/i) {
+	return $1;
+    }
+
+    return undef;
+}
+
+
+sub is_macrofile_chunk_automatically_translated
+{
+    my ($chunk_key) = @_;
+
+    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
+    if ($chunk_key =~ /\._(httpicon|width|height)/) {
+	return 1;
+    }
+
+    return 0;
+}
+
+
+# Use the source file to generate a target file that is formatted the same
+sub write_translated_macrofile
+{
+    my $source_file = shift(@_);  # Not used
+    my $source_file_lines = shift(@_);
+    my $source_file_key_to_text_mapping = shift(@_);
+    my $target_file = shift(@_);
+    my $target_file_lines = shift(@_);  # Not used
+    my $target_file_key_to_text_mapping = shift(@_);
+    my $target_file_key_to_comment_date_mapping = shift(@_);
+    my $target_language_code = shift(@_);
+
+    # Build a mapping from source file line to chunk key
+    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@$source_file_lines);
+    my %source_file_line_to_key_mapping = ();
+    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
+	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
+    }
+    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
+    my $source_file_line_number = 0;
+
+    # Build a mapping from target file line to chunk key
+    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@$target_file_lines);
+    my %target_file_line_to_key_mapping = ();
+    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
+	$target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
+    }
+    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
+
+    # Write the new target file
+    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
+    if (!open(TARGET_FILE, ">$target_file_path")) {
+	&throw_fatal_error("Could not write target file $target_file_path.");
+    }
+
+    # Use the header from the target file, to keep language and author information
+    if (scalar(@target_file_line_keys) > 0) {
+	my $target_file_line_number = 0;
+ 	my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
+ 	while ($target_file_line_number < $target_file_chunk_starting_line_number) {
+	    my $target_file_line = @$target_file_lines[$target_file_line_number];
+	    last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
+ 	    print TARGET_FILE $target_file_line;
+ 	    $target_file_line_number++;
+ 	}
+
+	$source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
+    }
+
+    # Model the new target file on the source file, with the target file translations
+    foreach my $line_key (@source_file_line_keys) {
+ 	# Fill in the gaps before this chunk starts
+ 	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
+ 	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
+ 	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
+ 	    print TARGET_FILE @$source_file_lines[$source_file_line_number];
+ 	    $source_file_line_number++;
+ 	}
+ 	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
+
+	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
+	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
+	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
+
+	my $macrofile_key = $chunk_key;
+	$macrofile_key =~ s/^(.+?)\.//;
+
+	# If no translation exists for this chunk, show this, and move on
+	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
+	    print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
+	    next;
+	}
+
+ 	# Grab the source chunk text
+ 	my $source_file_chunk = @$source_file_lines[$source_file_chunk_starting_line_number];
+ 	for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
+ 	    $source_file_chunk .= @$source_file_lines[$l];
+ 	}
+
+ 	# Is this an icon macro??
+ 	if ($source_file_chunk =~ /^\#\# (.*)/) {
+ 	    # Escape any newline and question mark characters so the source text is replaced correctly
+ 	    $source_file_chunk_text =~ s/\\/\\\\/g;
+	    $source_file_chunk_text =~ s/\?/\\\?/g;
+
+	    # Build the new target chunk from the source chunk
+	    my $target_file_chunk = $source_file_chunk;
+	    $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
+	    $target_file_chunk =~ s/(\s)*$//;
+	    print TARGET_FILE "$target_file_chunk";
+ 	}
+
+ 	# No, it is just a normal text macro
+ 	else {
+ 	    print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
+ 	}
+
+	# Add the update date, if one exists
+	if ($target_file_key_to_comment_date_mapping->{$chunk_key}) {
+	    print TARGET_FILE "  # Updated " . $target_file_key_to_comment_date_mapping->{$chunk_key};
+	}
+	print TARGET_FILE "\n";
+    }
+
+    close(TARGET_FILE);
+}
+
+
+sub sort_by_line
+{
+    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
+}
+
+
+# ==========================================================================================
+#   RESOURCE BUNDLE FUNCTIONS
+
+sub build_key_to_line_mapping_for_resource_bundle
+{
+    my (@file_lines) = @_;
+
+    my %key_to_line_mapping = ();
+    for (my $i = 0; $i < scalar(@file_lines); $i++) {
+	my $line = $file_lines[$i];
+	$line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
+
+	# Line contains a dictionary string
+	if ($line =~ /^(\S+?):(.*)$/) {
+	    my $chunk_key = $1;
+
+	    # Map from chunk key to line
+	    $key_to_line_mapping{$chunk_key} = $i . "-" . $i;
+	}
+    }
+
+    return %key_to_line_mapping;
+}
+
+
+sub import_chunk_from_resource_bundle
+{
+    my ($chunk_text) = @_;
+
+    # Simple: just remove string key
+    $chunk_text =~ s/^(\S+?)://;
+    $chunk_text =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
+
+    return $chunk_text;
+}
+
+
+sub get_resource_bundle_chunk_comment_date
+{
+    my ($chunk_text) = @_;
+
+    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
+    if ($chunk_text =~ /\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d)\s*$/i) {
+	return $1;
+    }
+
+    return undef;
+}
+
+
+sub is_resource_bundle_chunk_automatically_translated
+{
+    # No resource bundle chunks are automatically translated
+    return 0;
+}
+
+
+sub write_translated_resource_bundle
+{
+    my $source_file = shift(@_);  # Not used
+    my $source_file_lines = shift(@_);
+    my $source_file_key_to_text_mapping = shift(@_);
+    my $target_file = shift(@_);
+    my $target_file_lines = shift(@_);  # Not used
+    my $target_file_key_to_text_mapping = shift(@_);
+    my $target_file_key_to_comment_date_mapping = shift(@_);
+    my $target_language_code = shift(@_);  # Not used
+
+    # Build a mapping from chunk key to source file line, and from source file line to chunk key
+    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@$source_file_lines);
+    my %source_file_line_to_key_mapping = ();
+    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
+	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
+    }
+
+    # Write the new target file
+    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
+    if (!open(TARGET_FILE, ">$target_file_path")) {
+	&throw_fatal_error("Could not write target file $target_file_path.");
+    }
+
+    # Model the new target file on the source file, with the target file translations
+    my $source_file_line_number = 0;
+    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
+ 	# Fill in the gaps before this chunk starts
+ 	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
+ 	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
+ 	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
+ 	    print TARGET_FILE @$source_file_lines[$source_file_line_number];
+ 	    $source_file_line_number++;
+ 	}
+ 	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
+
+	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
+	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
+	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
+
+	# If no translation exists for this chunk, show this, and move on
+	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
+	    print TARGET_FILE "# -- Missing translation: $chunk_key\n";
+	    next;
+	}
+
+	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
+	if ($target_file_key_to_comment_date_mapping->{$chunk_key}) {
+	    print TARGET_FILE "  # Updated " . $target_file_key_to_comment_date_mapping->{$chunk_key};
+	}
+	print TARGET_FILE "\n";
+    }
+
+    close(TARGET_FILE);
+}
+
+
+&main(@ARGV);
Index: /main/tags/3.01/gsdl/bin/script/import.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/import.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/import.pl	(revision 21112)
@@ -0,0 +1,485 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# import.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will import a number of files into a particular collection
+
+package import;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docprint;
+use util;
+use scriptutil;
+use FileHandle;
+use gsprintf 'gsprintf';
+use printusage;
+use parse2;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and vice versa
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+my $oidtype_list = 
+    [ { 'name' => "hash",
+        'desc' => "{import.OIDtype.hash}" },
+      { 'name' => "incremental",
+        'desc' => "{import.OIDtype.incremental}" },
+      { 'name' => "assigned",
+        'desc' => "{import.OIDtype.assigned}" },
+      { 'name' => "dirname",
+        'desc' => "{import.OIDtype.dirname}" } ];
+
+#** define to use the original GA format or METS format
+my $saveas_list = 
+    [ { 'name' => "GA",
+        'desc' => "{import.saveas.GA}" },
+      { 'name' => "METS",
+        'desc' => "{import.saveas.METS}" } ];
+
+
+# Possible attributes for each argument
+# name: The name of the argument
+# desc: A description (or more likely a reference to a description) for this argument
+# type: The type of control used to represent the argument. Options include: string, int, flag, regexp, metadata, metadatum, language, enum etc
+# reqd: Is this argument required?
+# hiddengli: Is this argument hidden in GLI?
+# modegli: The lowest detail mode this argument is visible at in GLI
+
+my $arguments = 
+    [ { 'name' => "archivedir",
+	'desc' => "{import.archivedir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "collectdir",
+	'desc' => "{import.collectdir}",
+	'type' => "string",
+	# parsearg left "" as default
+	#'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	'deft' => "",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "debug",
+	'desc' => "{import.debug}",
+	'type' => "flag",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "faillog",
+	'desc' => "{import.faillog}",
+	'type' => "string",
+	# parsearg left "" as default
+	#'deft' => &util::filename_cat("&lt;collectdir&gt;", "colname", "etc", "fail.log"),
+	'deft' => "",
+	'reqd' => "no",
+        'modegli' => "4" },
+      { 'name' => "groupsize",
+	'desc' => "{import.groupsize}",
+	'type' => "int",
+	'deft' => "1",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "gzip",
+	'desc' => "{import.gzip}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "importdir",
+	'desc' => "{import.importdir}",
+	'type' => "string",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "keepold",
+	'desc' => "{import.keepold}",
+	'type' => "flag",
+	'reqd' => "no",
+        'modegli' => "3" },
+      { 'name' => "removeold",
+	'desc' => "{import.removeold}",
+	'type' => "flag",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "maxdocs",
+	'desc' => "{import.maxdocs}",
+	'type' => "int",
+	'reqd' => "no",
+	# parsearg left "" as default
+	#'deft' => "-1",
+	'range' => "1,",
+	'modegli' => "1" },
+      { 'name' => "OIDtype",
+	'desc' => "{import.OIDtype}",
+	'type' => "enum",
+	'list' => $oidtype_list,
+	# parsearg left "" as default
+	#'deft' => "hash",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "out",
+	'desc' => "{import.out}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "saveas",
+	'desc' => "{import.saveas}",
+	'type' => "enum",
+	'list' => $saveas_list,
+	'deft' => "GA",
+	'reqd' => "no",
+	'modegli' => "3" },      
+      { 'name' => "sortmeta",
+	'desc' => "{import.sortmeta}",
+#	'type' => "metadatum",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "removeprefix",
+	'desc' => "{BasClas.removeprefix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no", 
+	'modegli' => "3" },
+      { 'name' => "removesuffix",
+	'desc' => "{BasClas.removesuffix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no",
+	'modegli' => "3" }, 
+      { 'name' => "statsfile",
+	'desc' => "{import.statsfile}",
+	'type' => "string",
+	'deft' => "STDERR",
+	'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "verbosity",
+	'desc' => "{import.verbosity}",
+	'type' => "int",
+	'range' => "0,",
+	# parsearg left "" as default
+	#'deft' => "2",
+	'reqd' => "no",
+	'modegli' => "4" },
+      { 'name' => "gli",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" }];
+
+my $options = { 'name' => "import.pl",
+		'desc' => "{import.desc}",
+		'args' => $arguments };
+
+
+&main();
+
+sub main {
+    my ($verbosity, $importdir, $archivedir, $keepold, 
+	$removeold, $saveas, $version,
+	$gzip, $groupsize, $OIDtype, $debug, 
+	$maxdocs, $collection, $configfilename, $collectcfg, 
+	$pluginfo, $sortmeta, $removeprefix, $removesuffix, 
+	$archive_info_filename, $statsfile,
+	$archive_info, $processor, $out, $faillog, $collectdir, $gli, $language);
+
+    my $xml = 0;
+
+    my $service = "import";
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # general options available to all plugins
+    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+    # If there is more than one argument left after parsing, it mean user input too many arguments.
+    if($intArgLeftinAfterParsing > 1)
+    {
+	&PrintUsage::print_txt_usage($options, "{import.params}");
+	die "\n";
+    }
+    
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+    }
+
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language && $language =~ /\S/) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+	print "\n";
+	return;
+    }
+
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") ||
+	    (&gsprintf(STDERR, "{common.cannot_open_output_file}\n", $out) && die);
+	$out = 'import::OUT';
+	$close_out = 1;
+    }
+    $out->autoflush(1);
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV, $collectdir)) eq "") {
+	&PrintUsage::print_txt_usage($options, "{import.params}");
+	die "\n";
+    }
+
+    if ($faillog eq "") {
+	$faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
+    }
+    open (FAILLOG, ">$faillog") ||
+	(&gsprintf(STDERR, "{import.cannot_open_fail_log}\n", $faillog) && die);
+
+    
+    my $faillogname = $faillog;
+    $faillog = 'import::FAILLOG';
+    $faillog->autoflush(1);
+
+    # check sortmeta
+    $sortmeta = undef unless defined $sortmeta && $sortmeta =~ /\S/;
+    if (defined $sortmeta && $groupsize > 1) {
+	&gsprintf($out, "{import.cannot_sort}\n\n");
+	$sortmeta = undef;
+    }
+    # dynamically load 'docsave' module so it can pick up on a collection
+    # specific docsave.pm is specified.
+
+    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
+    require docsave;
+
+    # get the list of plugins for this collection and set any options that
+    # were specified in the collect.cfg (all import.pl options except
+    # -collectdir, -out and -faillog may be specified in the collect.cfg (these
+    # options must be known before we read the collect.cfg))
+    my $plugins = [];
+    my @global_opts = ();
+    
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+    if (!-e $configfilename) {
+	(&gsprintf($out, "{common.cannot_find_cfg_file}\n", $configfilename) && die);
+    }
+    
+    $collectcfg = &colcfg::read_collect_cfg ($configfilename);
+    if (defined $collectcfg->{'plugin'}) {
+	$plugins = $collectcfg->{'plugin'};
+    }
+
+    if ($verbosity !~ /\d+/) {
+	if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
+	    $verbosity = $collectcfg->{'verbosity'};
+	} else {
+	    $verbosity = 2; # the default
+	}
+    }
+    if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+	$importdir = $collectcfg->{'importdir'};
+    }
+    if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	$archivedir = $collectcfg->{'archivedir'};
+    }
+
+    if (defined $collectcfg->{'gzip'} && !$gzip) {
+	if ($collectcfg->{'gzip'} =~ /^true$/i) {
+	    $gzip = 1;
+	}
+    }
+    if ($maxdocs !~ /\-?\d+/) {
+	if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
+	    $maxdocs = $collectcfg->{'maxdocs'};
+	} else {
+	    $maxdocs = -1; # the default
+	}
+    }
+    if ($groupsize == 1) {
+	if (defined $collectcfg->{'groupsize'} && $collectcfg->{'groupsize'} =~ /\d+/) {
+	    $groupsize = $collectcfg->{'groupsize'};
+	}
+    }
+    if ($OIDtype !~ /^(hash|incremental|assigned|dirname)$/) {
+	if (defined $collectcfg->{'OIDtype'} && $collectcfg->{'OIDtype'} =~ /^(hash|incremental|assigned|dirname)$/) {
+	    $OIDtype = $collectcfg->{'OIDtype'};
+	} else {
+	    $OIDtype = "hash"; # the default
+	}
+    }
+    if ($saveas !~ /^(GA|METS)$/) {
+	if (defined $collectcfg->{'saveas'} && $collectcfg->{'saveas'} =~ /^(GA|METS)$/) {
+	    $saveas = $collectcfg->{'saveas'};
+	} else {
+	    $saveas ="GA";
+	}
+    }
+    if (defined $collectcfg->{'sortmeta'} && (!defined $sortmeta || $sortmeta eq "")) {
+	$sortmeta = $collectcfg->{'sortmeta'};
+    }
+    
+    if (defined $collectcfg->{'removeprefix'} && $removeprefix eq "") {
+	$removeprefix = $collectcfg->{'removeprefix'};
+    }
+    
+    if (defined $collectcfg->{'removesuffix'} && $removesuffix eq "") {
+	$removesuffix = $collectcfg->{'removesuffix'};
+    }
+    if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
+	$debug = 1;
+    }
+    if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
+	$gli = 1;
+    }
+    
+    
+    # global plugin stuff
+    if (defined $collectcfg->{'separate_cjk'} && $collectcfg->{'separate_cjk'} =~ /^true$/i) {
+	push @global_opts, "-separate_cjk";
+    }
+    
+    # check keepold and removeold
+    ($removeold, $keepold) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, "archives", $collectcfg);
+ 
+    $gli = 0 unless defined $gli;
+
+    print STDERR "<Import>\n" if $gli;
+    
+    # fill in the default import and archives directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+    $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives") if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+
+    # load all the plugins
+    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts);
+    if (scalar(@$pluginfo) == 0) {
+	&gsprintf($out, "{import.no_plugins_loaded}\n");
+	die "\n";
+    }
+
+    # remove the old contents of the archives directory (and tmp directory) if needed
+    if ($removeold) {
+	if (-e $archivedir) {
+	    &gsprintf($out, "{import.removing_archives}\n");
+	    &util::rm_r ($archivedir);
+	}
+	my $tmpdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "tmp");
+	$tmpdir =~ s/[\\\/]+/\//g;
+	$tmpdir =~ s/\/$//;
+	if (-e $tmpdir) {
+	    &gsprintf($out, "{import.removing_tmpdir}\n");
+	    &util::rm_r ($tmpdir);
+	}
+    }
+
+    # read the archive information file
+    if (!$debug) {
+	$archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
+	$archive_info = new arcinfo ();
+	$archive_info->load_info ($archive_info_filename);
+
+	# create a docsave object to process the documents
+	$processor = new docsave ($collection, $archive_info, $verbosity, $gzip, $groupsize, $out, $service, $saveas);
+	$processor->setoutputdir ($archivedir);
+	$processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
+	$processor->set_OIDtype ($OIDtype);
+	$processor->set_saveas ($saveas);
+	$processor->set_saveas_version ($version);
+
+    } else {
+	$processor = new docprint ();
+    }
+
+    &plugin::begin($pluginfo, $importdir, $processor, $maxdocs);
+
+    # process the import directory
+    &plugin::read ($pluginfo, $importdir, "", {}, $processor, $maxdocs, 0, $gli);
+    
+    &plugin::end($pluginfo, $processor);
+
+    &plugin::deinit($pluginfo, $processor);
+
+    # write out the archive information file
+    if (!$debug) {
+	$processor->close_file_output() if $groupsize > 1;
+	$archive_info->save_info($archive_info_filename);
+    }
+    
+    # write out import stats
+    my $close_stats = 0;
+    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
+	if (open (STATS, ">$statsfile")) {
+	    $statsfile = 'import::STATS';
+	    $close_stats = 1;
+	} else {
+	    &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
+	    &gsprintf($out, "{import.stats_backup}\n");
+	    $statsfile = 'STDERR';
+	}
+    }
+
+    &gsprintf($out, "\n");
+    &gsprintf($out, "*********************************************\n");
+    &gsprintf($out, "{import.complete}\n");
+    &gsprintf($out, "*********************************************\n");
+
+    &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
+    if ($close_stats) {
+	close STATS;
+    }
+
+    close OUT if $close_out;
+    close FAILLOG;
+}
Index: /main/tags/3.01/gsdl/bin/script/importfrom.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/importfrom.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/importfrom.pl	(revision 21112)
@@ -0,0 +1,453 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# importfrom.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will contact the named DL server
+#  and export its metadata and (optionally) it documents.
+
+# Currently only designed for OAI exporting
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use colcfg;
+use util;
+use parsargv;
+use FileHandle;
+
+my $wgetopt = "";
+
+my $num_processed = 0;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -getdoc                Also download if source document if present\n";
+    print STDERR "   -importdir directory   Where the original material lives\n";
+    print STDERR "   -keepold               Will not destroy the current contents of the\n";
+    print STDERR "                          import directory (the default)\n";
+    print STDERR "   -removeold             Will remove the old contents of the import\n";
+    print STDERR "                          directory -- use with care\n";
+    print STDERR "   -gzip                  Use gzip to compress exported documents\n";
+    print STDERR "                          (don't forget to include ZIPPlug in your plugin\n";
+    print STDERR "   -maxdocs number        Maximum number of documents to import\n";
+    print STDERR "   -debug                 Print imported text to STDOUT\n";
+    print STDERR "   -collectdir directory  Collection directory (defaults to " .
+	&util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n";
+    print STDERR "   -out                   Filename or handle to print output status to.\n";
+    print STDERR "                          The default is STDERR\n\n";
+}
+
+
+
+sub xml_pretty_print
+{
+    my ($text,$out,$verbosity) = @_;
+
+    if (system("xmllint --version >/dev/null 2>&1")!=0) {
+	if ($verbosity>1) {
+	    print STDERR "Warning: Unable to find xmllint for pretty printing.\n";
+	    print STDERR "         XML will be shown verbatim.\n\n";
+	}
+	print $out $text;
+    }
+    else {
+
+	if (!open (PPOUT,"|xmllint --format -")) {
+	    print STDERR "Error running xmllint: $!\n\n";
+	    print $out $text;
+	    return;
+	}
+
+	print PPOUT $text;
+	close(PPOUT);
+    }
+}
+
+sub wget_oai_url 
+{
+    my ($wget_cmd,$out,$verbosity) = @_;
+
+    if ($verbosity>2) {
+	print $out "  $wget_cmd\n";
+    }
+
+    open (OAIIN,"$wget_cmd |")
+	|| die "wget request failed: $!\n";
+ 
+    my $li_record = "";
+
+    my $line;
+    while (defined($line=<OAIIN>))
+    {
+	$li_record .= $line;
+	# print $out $line;
+    }
+
+    close(OAIIN);
+
+    return $li_record;
+}
+
+sub oai_info
+{
+    my ($base_url,$out,$verbosity) = @_;
+
+    my $base_wget_cmd = "wget $wgetopt -q -O - \"$base_url?_OPTS_\"";
+ 
+    my $identify = "verb=Identify";
+    my $list_sets = "verb=ListSets";
+    my $list_md_formats = "ListMetadataFormats"; # not currently used
+
+    my $identify_cmd = $base_wget_cmd;
+    $identify_cmd =~ s/_OPTS_/$identify/;
+    print $out "-------------------\n";
+    print $out "General Information\n";
+    print $out "-------------------\n";
+    my $identify_text = wget_oai_url($identify_cmd,$out,$verbosity);
+    xml_pretty_print($identify_text,$out,$verbosity);
+
+    
+    my $list_sets_cmd = $base_wget_cmd;
+    $list_sets_cmd =~ s/_OPTS_/$list_sets/;
+    print $out "-------------------\n";
+    print $out "Set Information\n";
+    print $out "-------------------\n";
+    my $list_sets_text = wget_oai_url($list_sets_cmd,$out,$verbosity);
+    xml_pretty_print($list_sets_text,$out,$verbosity);
+}
+
+
+sub get_oai_ids
+{
+    my ($base_url, $set, $format, $out, $verbosity) = @_;
+
+    print $out "Requesting list of identifiers ...\n";
+    
+    my $base_wget_cmd = "wget $wgetopt -q -O - \"$base_url?_OPTS_\"";
+    my $identifiers_cmd = $base_wget_cmd;
+
+    my $identifiers_opts = "verb=ListIdentifiers&metadataPrefix=$format";
+
+    if (defined $set && ($set ne "")) {
+	$identifiers_opts .= "&set=$set";
+    }
+
+    $identifiers_cmd =~ s/_OPTS_/$identifiers_opts/;
+
+    my $li_record = wget_oai_url($identifiers_cmd,$out,$verbosity);
+
+    print $out "... Done.\n";
+
+    return $li_record;
+}
+
+sub parse_oai_ids
+{
+    my ($li_record, $out, $verbosity) = @_;
+
+    # extract identifier list
+    $li_record =~ s/^.*?<identifier>/<identifier>/s;
+    $li_record =~ s/^(.*<\/identifier>).*$/$1/s;
+
+    my @ids = ();
+
+    while ($li_record =~ m/<identifier>(.*?)<\/identifier>(.*)$/s)
+    {
+	$li_record = $2;
+	push(@ids,$1);
+    }
+
+    return \@ids;
+}
+
+
+sub dir_file_split
+{
+    my ($file) = @_;
+
+    my @dirs = split("/",$file);
+    my $local_file = pop(@dirs);
+    my $sub_dirs = join("/",@dirs);
+
+    return ($sub_dirs,$local_file);
+}
+
+sub get_oai_document
+{
+    my ($doc_url,$output_dir, $out) = @_;
+
+    my ($id_dir,$id_fname) = dir_file_split($doc_url);
+
+    print $out "Getting document $doc_url\n";
+
+    &util::mk_dir($output_dir)  if (!-e "$output_dir");
+
+    my $full_id_fname = &util::filename_cat($output_dir,$id_fname);
+
+    my $wget_cmd = "wget $wgetopt --quiet -O $full_id_fname \"$doc_url\"";
+
+    if (system($wget_cmd)!=0) {
+	print STDERR "Error: failed to execute $wget_cmd\n";
+	return 0;
+    }
+
+    return 1;    
+}
+
+sub get_oai_records
+{
+    my ($base_url,$format, $ids,$output_dir, $get_id, $maxdocs, $out) = @_;
+
+    my $doc_count = 0;
+
+    my $i;
+    foreach $i ( @$ids )
+    {
+	# wget it;
+	my $url = "$base_url?verb=GetRecord&metadataPrefix=$format";
+	$url .= "&identifier=$i";
+	print $out "Downloading metadata record for $i\n";
+
+        my $i_url = $i; #convert OAI set separators (:) to directory sep
+        $i_url =~ s/:/\//g;
+	my $file_i_url = "$output_dir/$i_url.oai";
+
+	my $ds = &util::get_dirsep();
+	my $i_os = $i; #convert OAI set separators (:) to OS dir sep
+	$i_os =~ s/:/$ds/g;
+	my $file_i = &util::filename_cat($output_dir,"$i_os.oai");
+
+	# obtain record
+	my $wget_cmd = "wget $wgetopt -q -O - \"$url\"";
+
+	open (OAIIN,"$wget_cmd|")
+	    || die "wget request failed: $!\n";
+	my $i_record = "";
+	
+	my $line;
+	while (defined($line=<OAIIN>))
+	{
+	    $i_record .= $line;
+	}
+
+	close(OAIIN);
+
+	$num_processed++;
+	
+	# prepare subdirectory for record (if needed)
+	my ($i_dir,$unused) = dir_file_split($file_i_url);
+
+	&util::mk_all_dir($i_dir);
+
+	# look out for identifier tag in metadata section
+	if ($i_record =~ m/<metadata>(.*)<\/metadata>/s)
+	{
+	    my $m_record = $1;
+
+	    if ($get_id)
+	    {
+		my $got_doc = 0;
+
+		my @url_matches = ($m_record =~ m/<(?:dc:)?identifier>(.*?)<\/(?:dc:)?identifier>/gs);
+		foreach my $doc_url (@url_matches) 
+		{
+		    if ($doc_url =~ m/^(http|ftp):/) {
+
+			my $revised_doc_url = $doc_url;
+##			$revised_doc_url =~ s/hdl\.handle\.net/mcgonagall.cs.waikato.ac.nz:8080\/dspace\/handle/;
+			
+			my $srcdocs_dir = &util::filename_cat($i_dir,"srcdocs");
+
+			if (get_oai_document($revised_doc_url,$srcdocs_dir, $out)) {			   
+
+			    $got_doc = 1;
+			    my ($id_dir,$id_fname) = dir_file_split($revised_doc_url);
+			    
+			    $i_record =~ s/<metadata>(.*?)<(dc:)?identifier>$doc_url<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigURL>$doc_url<\/OrigURL>\n   <identifier>srcdocs\/$id_fname<\/identifier>$4<\/metadata>/s;
+			    
+			}
+		    }
+
+		    if (!$got_doc) {
+			$i_record =~ s/<metadata>(.*?)<(dc:)?identifier>$doc_url<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigIdentifier>$doc_url<\/OrigIdentifier>$4<\/metadata>/s;
+		    }
+		}
+	    }
+	}
+
+	# save record 
+	open (OAIOUT,">$file_i")
+	    || die "Unable to save oai metadata record: $!\n";
+	print OAIOUT $i_record;
+	close(OAIOUT);
+
+	$doc_count++;
+	last if ($doc_count == $maxdocs);
+    }
+}
+
+
+sub main {
+    my ($verbosity, $importdir, $keepold, 
+	$getdoc, $acquire_info, $acquire_set,
+	$removeold, $gzip, $groupsize, $debug, $maxdocs, $collection,
+	$configfilename, $collectcfg,
+	$out, $collectdir);
+
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'getdoc', \$getdoc,
+			 'info', \$acquire_info,
+			 'importdir/.*/', \$importdir,
+			 'keepold', \$keepold,
+			 'removeold', \$removeold,
+			 'gzip', \$gzip,
+			 'debug', \$debug,
+			 'maxdocs/^\-?\d+/-1', \$maxdocs,
+			 'collectdir/.*/', \$collectdir,
+			 'out/.*/STDERR', \$out)) {
+	&print_usage();
+	die "\n";
+    }
+
+    my $close_out = 0;
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+	open (OUT, ">$out") || die "Couldn't open output file $out\n";
+	$out = 'import::OUT';
+	$close_out = 1;
+    }
+    $out->autoflush(1);
+
+    # set removeold to false if it has been defined
+    $removeold = 0 if ($keepold);
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV, $collectdir)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+
+
+    # get acquire list
+    my $acquire = [];
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'acquire'}) {
+	    $acquire = $collectcfg->{'acquire'};
+	}
+	if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+	    $importdir = $collectcfg->{'importdir'};
+	}
+	if (defined $collectcfg->{'removeold'}) {
+	    if ($collectcfg->{'removeold'} =~ /^true$/i && !$keepold) {
+		$removeold = 1;
+	    }
+	    if ($collectcfg->{'removeold'} =~ /^false$/i && !$removeold) {
+		$removeold = 0;
+	    }
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+    
+    # fill in the default import directory if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+
+    # remove the old contents of the import directory if needed
+    if ($removeold && -e $importdir) {
+	print $out "Warning - removing current contents of the import directory\n";
+	print $out "          in preparation for the acquire\n";
+	&util::rm_r ($importdir);
+    }
+
+    my $e;
+    foreach $e ( @$acquire )
+    {
+	my $acquire_type = shift @$e;
+	my $acquire_src = undef;
+
+	if ($acquire_type ne "OAI") {
+	    print STDERR "Warning: $acquire_type not currently supported. Skipping.\n";
+	    next;
+	}
+
+	my $store_getdoc = $getdoc;
+
+	if (!parsargv::parse($e, 
+			     'getdoc',  \$getdoc,
+			     'set/.*/', \$acquire_set,
+			     'format/.*/oai_dc', \$metadata_format,
+			     'src/.*/', \$acquire_src)) {
+	    &print_usage();
+	    die "\n";
+	}
+
+	if (!defined $acquire_src) {
+	    print STDERR "Warning: Not -src flag defined.  Skipping.\n";
+	    next;
+	}
+
+	if (defined $acquire_info && ($acquire_info)) {
+	    oai_info($acquire_src,$out,$verbosity);
+	    next;
+	}
+
+	print $out "$acquire_type Acquire: from $acquire_src\n";
+
+	my $li_record = get_oai_ids($acquire_src,$acquire_set,$metadata_format,
+				    $out,$verbosity);
+	my $ids = parse_oai_ids($li_record,$out,$verbosity);
+
+	get_oai_records($acquire_src,$metadata_format, $ids,$importdir, 
+			$getdoc, $maxdocs, $out);
+	$getdoc = $store_getdoc;
+    }
+
+    print "\nNumber of documents processed: $num_processed\n";
+
+    close OUT if $close_out;
+}
+
+
+&main();
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/Core.pm
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/Core.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/Core.pm	(revision 21112)
@@ -0,0 +1,895 @@
+#
+# PDF::Core.pm, version 1.11 February 2000 antro
+#
+# Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+#
+# Free usage under the same Perl Licence condition.
+#
+
+package PDF::Core;
+
+$PDF::Core::VERSION = "1.11";
+
+=pod
+
+=head1 NAME
+
+PDF::Core - Core Library for PDF library
+
+=head1 SYNOPSIS
+
+  use PDF::Core;
+
+  $pdf=PDF::Core->new ;
+  $pdf=PDF->new(filename);
+
+  $res= $pdf->GetObject($ref);
+
+  $name = UnQuoteName($pdfname);							  
+  $string = UnQuoteString($pdfstring);							  
+
+  $pdfname = QuoteName($name);							  
+  $pdfhexstring = QuoteHexString($string);
+  $pdfstring = QuoteString($string);
+
+  $obj = PDFGetPrimitive (filehandle, \$offset);
+  $line = PDFGetLine (filehandle, \$offset);
+  
+
+=head1 DESCRIPTION
+
+The main purpose of the PDF::Core library is to provide the data structure
+and the constructor for the more general PDF library.
+
+=cut
+
+require 5.005;
+use strict;
+use Carp;
+use Exporter ();
+
+use vars qw(@ISA @EXPORT_OK $UseObjectCache);
+
+@ISA = qw(Exporter);
+
+@EXPORT_OK = qw( GetObject );
+
+#
+# Object caching
+#
+# If this variable is true, all processed objects will be added to the
+# object cache. If only header information of a PDF are read or very
+# big PDF are processed, turning off the cache reduces the memory usage.
+#
+$UseObjectCache = 1;
+
+
+#################################################################
+#
+# Helper functions
+#
+#################################################################
+
+=pod
+
+=head1 Helper functions
+
+This functions are not part of the class, but perform useful services.
+
+=cut
+
+#
+# Modification by johi: 18.12.1999
+#
+
+#################################################################
+=pod
+
+=head2 UnQuoteName ( string )
+
+This function processes quoted characters in a PDF-name. PDF-names returned by
+B<GetObject> are already processed by this function.
+
+Returns a string.
+
+=cut
+
+sub UnQuoteName ($)
+	{
+	my $value = shift;
+	$value =~ s/#([\da-f]{2})/chr(hex($1))/ige;
+	return $value;
+	}
+
+#################################################################
+=pod
+
+=head2 UnQuoteString ( string )
+
+This function extracts the text from PDF-strings and PDF-hexstrings.
+It will process all quoted characters and remove the enclosing braces.
+
+WARNING: The current version doesn't handle unicode strings properly.
+
+Returns a string.
+
+=cut
+
+sub UnQuoteString ($)
+	{
+#
+# Translate quoted character. 
+#
+	my $param = shift;
+	my $value;
+	if (($value) = $param =~ m/^<(.*)>$/)
+		{
+		$value =~ tr/0-9A-Fa-f//cd;
+		$value .= "0" if (length ($value) % 2);
+		$value =~ s/([\da-f]{2})/chr(hex($1))/ige;
+		}
+	elsif (($value) = $param =~ m/^\((.*)\)$/)
+		{
+		my %quoted = ("n" => "\n", "r" => "\r",
+					  "t" => "\t", "b" => "\b",
+					  "f" => "\f", "\\" => "\\",
+					  "(" => "(", ")" => ")");
+		$value =~ s/\\([nrtbf\\()]|[0-7]{1,3})/
+		defined ($quoted{$1}) ? $quoted{$1} : chr(oct($1))/gex;
+		}
+	else
+		{
+		$value = $param;
+		}
+
+	return $value;
+	}
+
+#################################################################
+=pod
+
+=head2 QuoteName ( string )
+
+This function quotes problematic characters in a PDF-name. This
+function should be used before writing a PDF-name back to a PDF-file.
+
+Returns a string.
+
+=cut
+
+sub QuoteName ($)
+	{
+	my $value = shift;
+	$value =~ s/(?<!\A)([\x00-\x20\x7f-\xff%()\[\]<>\/{}#])/
+						 sprintf ("#%2.2X", ord($1))/gex;
+	return $value;
+	}
+
+#################################################################
+=pod
+
+=head2 QuoteHexString ( string )
+
+This function translates a string into a PDF-hexstring. 
+
+Returns a string.
+
+=cut
+
+sub QuoteHexString ($)
+	{
+	my $value = shift;
+
+	$value =~ s/(.)/sprintf ("%2.2X", ord($1))/ge;
+	return ("<" . $value . ">");
+	}
+
+#################################################################
+=pod
+
+=head2 QuoteString ( string )
+
+This function translates a string into a PDF-string. Problematic
+character will be quoted.
+
+WARNING: The current version doesn't handle unicode strings properly.
+
+Returns a string.
+
+=cut
+
+sub QuoteString ($)
+	{
+	#
+	# Only \character style quotes will be added. The really important
+	# characters to quote are: ()\
+	# 
+	my $value = shift;
+
+	my %quote = ("\n" => "\\n", "\r" => "\\r",
+				  "\t" => "\\t", "\b" => "\\b",
+				  "\f" => "\\f", "\\" => "\\\\",
+				  "(" => "\\(", ")" => "\\)");
+	$value =~ s/([\n\r\t\b\f\\()])/$quote{$1}/g;
+	return ("(" . $value . ")");
+	}
+
+#################################################################
+=pod
+
+=head2 PDFGetPrimitive ( filehandle, offset )
+
+This internal function is used while parsing a PDF-file. If you are
+not writing extentions for this library and are parsing some special
+parts of the PDF-file, stay away and use B<GetObject> instead.
+
+This function has many quirks and limitations. Check the source for details.
+
+=cut
+
+sub PDFGetPrimitive (*\$)
+	{
+	my $fd = shift;
+	my $offset = shift;
+
+	binmode $fd;
+	seek $fd, $$offset, 0;
+
+	my $state = 0;
+	my $buffer;
+	my @collector;
+	my $lastchar;
+
+	while ()
+		{
+		# File offset is positioned on start of stream.
+		last if ($state == -4);
+
+		$state = 0;
+
+		# Process last element
+		if ($#collector >= 0)
+			{
+			my $lastvalue = $collector[$#collector];
+			
+			if ($lastvalue eq "R")
+				{
+				# Process references
+				if ($#collector >= 2
+					&& $collector[$#collector - 1] =~ m/\d+/
+					&& $collector[$#collector - 2] =~ m/\d+/)
+					{
+					$collector[$#collector - 2] .= join (" ", 
+						"", @collector[$#collector - 1, $#collector]);
+					$#collector -= 2; 
+					}
+				else
+					{
+					carp "Bad reference at offset ", $$offset;
+					}
+				}
+			elsif ($lastvalue eq "endobj")
+				{
+				# End of object
+				last;
+				}
+			elsif ($lastvalue eq "stream")
+				{
+				# End of object
+				$state = -4;
+				}
+			}
+		
+		# Set state for next element
+		if ($buffer eq "[") 
+			{
+			# Read array
+			$buffer = "";
+			push @collector, [ PDFGetPrimitive ($fd, $offset) ];
+			}
+		elsif ($buffer eq "<<")
+			{
+			# Read dictionary
+			$buffer = "";
+			push @collector, { PDFGetPrimitive ($fd, $offset) };
+			}
+		elsif ($buffer eq "(") 
+			{
+			# Here comes a string
+			$state = 1;
+			$lastchar = "";
+			}
+		elsif ($buffer eq "<") 
+			{
+			# Here comes a hex string
+			$state = -1;
+			}
+		elsif ($buffer eq ">")
+			{
+			# Wait for next > to terminate dictionary
+			$state = -2;
+			}
+		elsif ($buffer eq "%")
+			{
+			# Skip comments
+			$state = -3;
+			$buffer = "";
+			}
+		elsif ($buffer eq "]")
+			{
+			last;
+			}
+		elsif ($buffer eq ">>")
+			{
+			last;
+			}
+
+		# Read next item
+		while (read ($fd, $_, 1))
+			{
+			$$offset++;
+
+			if ($state == 0)
+				{
+				# Normal mode
+				if (m/[^\x00-\x20\x7f-\xff%()\[\]<>\/]/)
+					{
+					# Normal character inside a name or number
+					$buffer .= $_;
+					}
+				elsif (m/[\/\(\[\]\<\>%]/)
+					{
+					if ($buffer ne "")
+						{
+						# A new item starts
+						if ($buffer =~ m/^\//)
+							{
+							push @collector, UnQuoteName ($buffer);
+							}
+						else
+							{
+							push @collector, $buffer;
+							}
+						}
+					$buffer = $_;
+					last;
+					}
+				elsif (m/\s/)
+					{
+					# All kind of whitespaces are ignored
+					if ($buffer ne "")
+						{
+						# The old item is done starts
+						if ($buffer =~ m/^\//)
+							{
+							push @collector, UnQuoteName ($buffer);
+							}
+						else
+							{
+							push @collector, $buffer;
+							}
+						$buffer = "";
+						last;
+						}
+					}
+				else
+					{
+					# Strange character. Should not exist.
+					# Complain and move on.
+					carp "Strange character '", $_, "' at offset ",
+					$$offset, " in mode ", $state, " detected";
+					$buffer .= $_;
+					}
+				}
+			elsif ($state > 0)
+				{
+				# We have a string
+
+				if ($lastchar =~ m/\\[\r\n]+/ && m/[^\r\n]/)
+					{
+					# Clean up after line continuation
+					$lastchar = "";
+					}
+
+				if ($lastchar =~ m/\\[\r\n]*/)
+					{
+					# Process character after backslash
+					if (m/[\r\n]/)
+						{
+						# end of line
+						$lastchar .= $_;
+						}
+					else
+						{
+						# Just a quote
+						$buffer .= $lastchar . $_;
+						$lastchar = "";
+						}
+					}
+				else
+					{
+					if ($_ eq "\\")
+						{
+						# Quoted string starts
+						$lastchar = $_;
+						}
+					elsif ($_ eq "(")
+						{
+						# Count braces
+						$buffer .= $_;
+						$state ++;
+						}
+					elsif ($_ eq ")")
+						{
+						# End of string
+						$buffer .= $_;
+						unless (-- $state)
+							{
+							push @collector, $buffer;
+							$buffer = "";
+							last;
+							}
+						}
+					else
+						{
+						$buffer .= $_;
+						}
+					}
+				}
+			elsif ($state == -1)
+				{
+				if (m/[0-9a-f\s]/i)
+					{
+					# Hex character
+					$buffer .= $_;
+					}
+				elsif ($_ eq ">")
+					{
+					# End of string
+					$buffer .= $_;
+					push @collector, $buffer;
+					$buffer = "";
+					last;
+					}
+				elsif ($_ eq "<" && $buffer eq "<")
+					{
+					# This is not a string, but a dictionary instead
+					$buffer .= $_;
+					last;
+					}
+				else
+					{
+					# Should not be there. Complain and add it to the $buffer
+					carp "Bad character '", $_ , "' in hex string";
+					$buffer .= $_;
+					}
+				}
+			elsif ($state == -2)
+				{
+				# Wait for second > to terminate dictionary
+
+				# Some sanity checks
+				carp "Character '", $_, "' appeared while waiting for '>'" 
+				if ($_ ne ">");
+				carp "Buffer contains '", $buffer, "' and not '>'" 
+				if ($buffer ne ">");
+
+				$buffer = ">>";
+				last;
+				}
+			elsif ($state == -3)
+				{
+				# Skip comments;
+				last if (m/[\r\n]/);
+				}
+			elsif ($state == -4)
+				{
+				# Wait for newline to start stream
+
+				if ($_ eq "\n")
+					{
+					# Some sanity checks
+					carp "Text '", $buffer, 
+					"' appeared while waiting for start of stream" 
+					if ($buffer ne "");
+
+					$buffer = "";
+					last;
+					}
+				elsif (m/\S/)
+					{
+					$buffer .= $_;
+					}
+				}
+			else
+				{
+				# Unhandled status. Complain and reset
+				carp "Unhandled status ", $state;
+				}
+			}
+		if ($_ eq "")
+			{
+			# Unhandled status. Complain and reset
+			carp "Premature end of file reached";
+			
+			if ($buffer ne "")
+				{
+				push @collector, $buffer;
+				$buffer = "";
+				}
+			last;
+			}
+		}
+
+	return @collector;
+	}
+
+#################################################################
+=pod
+
+=head2 PDFGetline ( filehandle, offset )
+
+This internal function was used to read a line from a PDF-file. It has
+many limitations and you should stay away from it, if you don't know
+what you are doing. Use B<GetObject> or B<PDFGetPrimitive> instead.
+
+=cut
+
+sub PDFGetline {
+#
+# BUG WARNING:
+#
+# This function returns only one line, which doesn't mean anything most of the
+# time. Except for the fileheader and the xref-table, linebreaks can (and will!)
+# occur everywhere in a PDF and are just whitespace. You may find only part of a
+# PDF-primitve on one line, or more than one of them.
+#
+# If you want to read PDF-Primitves, use the function PDFGetPrimitive instead.
+#
+    my $fd = shift;
+    my $offset=shift;
+
+    my $buffer;
+    my $endflag=1;
+
+    binmode $fd;
+    seek $fd, $$offset, 0;
+
+    read($fd,$buffer,2);
+    $buffer =~ s/^\r?\n?// ;
+
+    $$offset +=2;
+
+    while ($endflag) {
+      read($fd,$_,1);
+      $$offset++;
+      $endflag = 0 if ( $_ eq "\r" || $_ eq "\n");
+      $buffer = $buffer . $_ ;
+    }
+    return $buffer;
+	}
+
+#################################################################
+#
+# Constructors
+#
+#################################################################
+
+=pod
+
+=head1 Constructor
+
+=cut
+
+#################################################################
+=pod
+
+=head2 new ( [ filename ] )
+
+This is the constructor of a new PDF object. If the filename is
+missing, it returns an empty PDF descriptor ( can be filled with
+$pdf->TargetFile). Otherwise, It acts as the B<PDF::Parse::TargetFile>
+method.
+
+=cut
+
+sub new {
+
+	my %PDF_Fields = (
+		  File_Name => undef, # Name of file
+		  File_Handler => undef, # Open handle to file
+		  Header => undef, # Identification string
+
+		  Objects => [], # Offset of objects
+		  Gen_Num => [], # Genereation number of objects
+		  Object_Length => [],	# Length of processed objects
+		  Object_Cache => {}, # Cache for objects.
+		  Page => [], # Information about all pages. Useful.
+
+		  Updated => 0,	# Is the PDF updated 
+		  Last_XRef_Offset => undef, # File offset of active Xref table
+		  Trailer => {}, # Content of active trailer
+		  Info => {}, # Content of active info object
+		  Catalog => {}, # Content of catalog
+		  PageTree => {}, # Content of root page
+		  );
+my $that = shift;
+my $class=ref($that) || $that ;
+  my $self = \%PDF_Fields ;
+  my $buf2=bless $self, $class;
+  if ( @_ ) { 			# I have the filename
+    $buf2->TargetFile($_[0]) ; 
+  }
+  return bless $self, $class;
+};
+
+#################################################################
+sub DESTROY {
+#
+# Close the file if not empty
+#
+  my $self = shift;
+  close ( $self->{File_Handler} ) if $self->{File_Handler} ;
+}
+
+#################################################################
+#
+# Methods
+#
+#################################################################
+
+=pod
+
+=head1 Methods
+
+The available methods are:
+
+=cut
+
+#################################################################
+=pod
+
+=head2 GetObject (reference)
+
+This methods returns the PDF-object for B<reference>. The string
+B<reference> must match the regular expression /^\d+ \d+ R$/,
+where the first number is the object number, the second number the
+generation number.
+
+The return value is a PDF-primitive, the type depends on the content
+of the object:
+
+=over
+
+=item B<undef>
+
+The object could not be found or an error. Not all referenced objects
+need to be present in a PDF-file. This value can be ignored.
+
+=item B<Hash Reference>
+
+If (UNIVERSAL::isa ($retval, "HASH") is true, the object is a
+PDF-dictionary. The keys of the hash should be either a PDF name (eg:
+/MediaBox) or a generated value like Stream_Offset. Everything else is
+an error.
+
+The values of the hash can be any PDF-primitive, including PDF-arrays
+and other dictionaries.
+
+This is the most common value returned by GetObject. If the key
+Stream_Offset exists, the dictionary is followed by stream data,
+starting at the file offeset indicated by this value.
+
+=item B<Array Reference>
+
+If (UNIVERSAL::isa ($retval, "ARRAY") is true, the object is a
+PDF-array. Each element may be of a different type, and may contain
+further references to arrays or any other PDF-primitive.
+
+=item B<String matching /^\d+ \d+ R$/>
+
+This is a reference to another PDF-Object. This value can be passed to
+GetObject. This kind of value may appear instead of most other types.
+Some PDF-writing programs seem to have special fun writing references
+when a simple number is expected. If the final number is need, use
+code like this to resolve references:
+
+while ($len =~ m/^\d+ \d+ R$/) {$len = $self->GetObject ($len);	}
+
+Example: 22 0 R
+
+=item B<String matching /^\//>
+
+This is a Name in a PDF dictionary. This string is already processed
+by B<UnQuotName> and may differ from the value in the PDF-file. In
+some very old andstrange non-standard PDF-files, this may lead to
+confusion.
+
+Example: /MediaBox
+
+=item B<String matching /^\(.*\)$/>
+
+This is a string. It may contain newlines, quoted characters und other
+strange stuff. Use PDF::UnQuoteString to extract the text.
+
+Example: (This is\na string with two \(2\) lines.)
+
+=item B<String matching /^E<lt>.*E<gt>$/>
+
+This is a hex encoded string. Use PDF::UnQuoteString to extract the text.
+
+Example: E<lt>48 45 4c4C4 F1cE<gt>
+
+=item B<String matching /^[\d.\+\-]+$/>
+
+This is probably a number.
+
+Example: 611
+
+=item B<String matching none of the above>
+
+this is either a PDF bareword (eg. true, false, ...) or a value
+generated by this method like Stream_Offset.
+
+Example: true
+
+=back
+
+To improve performance GetObject uses an internal cache for objects.
+Repeated requests for the same objects are not read form the file but
+satisfied from the cache. With the Variable B<$PDF::Core::UseObjectCache>,
+the caching mechanism can be turned off.
+
+B<WARNING>
+
+Special care must be taken, when returned objects are modified. If the
+object contains sub-objects, the sub-objects are not duplicated and
+all changes affect all other copies of this object. Use your own copy,
+if you need to modify those values.
+
+=cut
+
+sub GetObject (\*$;$)
+	{
+	my $self = shift;
+	my $ref = shift;
+	my $force = shift;
+
+#
+# Is PDF file open?
+#
+	croak "PDF-file not open." unless ($self->{"File_Handler"});
+
+#
+# Check reference
+#
+	my ($ind, $gen);
+	unless (($ind,$gen) = $ref =~ m/^(\d+) (\d+) R$/)
+		{
+		carp "Bad object reference '", $_, "'";
+		return undef;
+		}
+	if ($ind > $#{$self->{"Gen_Num"}} || $self->{"Gen_Num"}[$ind] != $gen)
+		{
+		#
+		# The page does not exist. According to the PDF specification,
+		# this is not an error.
+		#
+		return undef;
+		}
+
+	# Remove leading zero for cache key.
+	$ind += 0;
+	# Check cache
+	if ($UseObjectCache && ! $force
+		&& defined($self->{"Object_Cache"}{$ind}))
+		{
+		return $self->{"Object_Cache"}{$ind};
+		}
+
+	my $offset = $self->{"Objects"}[$ind];
+	my @data = PDFGetPrimitive ($self->{"File_Handler"}, $offset);
+
+	unless ($#data == 4  && $data[0] == $ind 
+		&& $data[1] == $gen && $data[2] eq "obj")
+		{
+		carp "Object mismatch: Got '", join (" ", @data[0..2]),
+		"' instead of '", join (" ", $ind, $gen, "obj"), "'";
+		return;
+		}
+
+	#
+	# An object is not always a dictionary. In such cases,
+	# adding additional keys breaks the content.
+	#
+	if (UNIVERSAL::isa ($data[3], "HASH"))
+		{
+		if ($data[4] eq "stream")
+			{
+			#
+			# Find end of a stream object
+			#
+			$data[3]{"Stream_Offset"} = $offset;
+			my $len = $data[3]{"/Length"};
+
+			# Length can be a reference to another object. 
+			# Resolve references in this case till something else appears.
+			while ($len =~ m/^\d+ \d+ R$/)
+				{
+				$len = $self->GetObject ($len);
+				}
+
+			# Skip stream
+			if ($len =~ m/^\d+$/)
+				{
+				$offset += $len;
+				}
+			else
+				{
+				carp "Strange: /Length resolves to '", $len, "' in object ", 
+				join (" ", @data[0..2]);
+				}
+
+			my @enddata = PDFGetPrimitive ($self->{"File_Handler"}, $offset);
+			$data[4] = $enddata[$#enddata];
+			}
+		}
+
+	#
+	# Save length of object.
+	#
+	$self->{"Object_Length"}[$ind] = $offset - $self->{"Objects"}[$ind];
+
+	carp "Bad object termination '", $data[4], "' in object ", 
+	join (" ", @data[0..2]) if ($data[4] ne "endobj");
+	
+
+	# Update cache
+	$self->{"Object_Cache"}{$ind} = $data[3] if ($UseObjectCache);
+
+	return $data[3];
+	}
+
+#
+# End of Modification by johi: 18.12.1999
+#
+#################################################################
+
+
+1;
+__END__
+
+=pod
+
+=head1 Variables
+
+Available variables are:
+
+=over 4
+
+=item B<$PDF::Core::VERSION>
+
+Contains the version of the library installed
+
+=item B<$PDF::Core::UseObjectCache>
+
+If this variable is true, all processed objects will be added to the
+object cache. If only header information of a PDF are read or very big
+PDF are processed, turning off the cache reduces the memory usage.
+
+=back 4
+
+=head1 Copyright
+
+  Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 Availability
+
+The latest version of this library is likely to be available from:
+
+http://www.geocities.com/CapeCanaveral/Hangar/4794/
+
+=cut
Index: /main/tags/3.01/gsdl/bin/script/indexes/PDF.pm
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/PDF.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/PDF.pm	(revision 21112)
@@ -0,0 +1,153 @@
+#
+# PDF.pm, version 1.11 February 2000 antro
+#
+# Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+#
+# Free usage under the same Perl Licence condition.
+#
+
+package PDF;
+
+$PDF::VERSION = "1.11";
+
+require 5.005;
+
+#require PDF::Core;  
+#require PDF::Parse; 
+require Core;  
+require Parse;
+ 
+use strict;
+use Carp;
+use Exporter ();
+
+use vars qw(@ISA $Verbose);
+
+#
+# Verbose off by default
+#
+$Verbose = 0;
+
+@ISA = qw(Exporter PDF::Core PDF::Parse);
+
+1;
+
+__END__
+
+=head1 NAME
+
+PDF - Library for PDF access and manipulation in Perl
+
+=head1 SYNOPSIS
+
+  use PDF;
+
+  $pdf=PDF->new ;
+  $pdf=PDF->new(filename);
+
+  $result=$pdf->TargetFile( filename );
+
+  print "is a pdf file\n" if ( $pdf->IsaPDF ) ;
+  print "Has ",$pdf->Pages," Pages \n";
+  print "Use a PDF Version  ",$pdf->Version ," \n";
+  print "and it is crypted  " if ( $pdf->IscryptedPDF) ;
+
+  print "filename with title",$pdf->GetInfo("Title"),"\n";
+  print "and with subject ",$pdf->GetInfo("Subject"),"\n";
+  print "was written by ",$pdf->GetInfo("Author"),"\n";
+  print "in date ",$pdf->GetInfo("CreationDate"),"\n";
+  print "using ",$pdf->GetInfo("Creator"),"\n";
+  print "and converted with ",$pdf->GetInfo("Producer"),"\n";
+  print "The last modification occurred ",$pdf->GetInfo("ModDate"),"\n";
+  print "The associated keywords are ",$pdf->GetInfo("Keywords"),"\n";
+
+  my (startx,starty, endx,endy) = $pdf->PageSize ($page) ;
+
+=head1 DESCRIPTION
+
+The main purpose of the PDF library is to provide classes and
+functions that allow to read and manipulate PDF files with perl. PDF
+stands for Portable Document Format and is a format proposed by Adobe.
+A full description of this format can be found in the B<Portable
+Document Reference Manual> by B<Adobe Systems Inc.>. For more details
+about PDF, refer to:
+
+B<http://www.adobe.com/>
+
+The main idea is to provide some "basic" modules for access 
+the information contained in a PDF file. Even if at this
+moment is in an early development stage, the scripts in the 
+example directory show that it is usable. 
+
+B<is_pdf> script test a list of files in order divide the PDF file
+from the non PDF using the info provided by the files 
+themselves. It doesn't use the I<.pdf> extension, it uses the information
+contained in the file.
+
+B<pdf_version> returns the PDF level used for writing a file.
+
+B<pdf_pages> gives the number of pages of a PDF file. 
+
+B<pagedump.pl> prints some information about individual pages in a
+PDF-file. Although the information as such are not very useful, it
+demontrates well some more complex aspects of the library. Check the
+function B<doprint> in this program on how to handle all possible data
+occuring in a PDF.
+
+The library is now splitted in 2 section :
+
+B<PDF::Core> that contains the data structure, the constructor and low
+level access fuctions;
+
+B<PDF::Parse> all kind of functions to parse the PDF-files and
+provide information about the content.
+
+Check the help-files of these modules for more details.
+
+=head1 Variables
+
+There are 2 variables that can be accessed:
+
+=over 4
+
+`=item B<$PDF::VERSION>
+
+Contain the version of the library installed.
+
+=item B<$PDF::Verbose>
+
+This variable is false by default. Change the value if you want 
+more verbose output messages from library.
+
+=back 4
+
+=head1 Copyright
+
+  Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 Availability
+
+The latest version of this library is likely to be available from:
+
+http://www.geocities.com/CapeCanaveral/Hangar/4794/
+
+and at any CPAN mirror
+
+=head1 Greetings
+
+Fabrizio Pivari ( pivari@geocities.com ) for all the suggestions about life, the universe and everything.
+Brad Appleton ( bradapp@enteract.com ) for his suggestions about the module organization.
+Thomas Drillich for the iso latin1 support 
+Ross Moore ( ross@ics.mq.edu.au ) for ReadInfo fix
+
+=cut
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/Parse.pm
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/Parse.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/Parse.pm	(revision 21112)
@@ -0,0 +1,513 @@
+#
+# PDF::Parse.pm, version 1.11 February 2000 antro
+#
+# Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+#
+# Free usage under the same Perl Licence condition.
+#
+
+package PDF::Parse;
+
+$PDF::Parse::VERSION = "1.11";
+
+=pod
+
+=head1 NAME
+
+PDF::Parse - Library with parsing functions for PDF library
+
+=head1 SYNOPSIS
+
+  use PDF::Parse;
+
+  $pdf->TargetFile($filename);
+  $pdf->LoadPageInfo;
+
+  $version = $pdf->Version;
+  $bool = $pdf->IsaPDF;
+  $bool = $pdf->IscryptPDF;
+
+  $info = $pdf->GetInfo ($key);
+  $pagenum = $pdf->Pages;
+
+  @size = $pdf->PageSize ($page);
+  # or
+  @size = $pdf->PageSize;
+
+  $rotation = $pdf->PageRotation ($page);
+  # or
+  $rotation = $pdf->PageRotation;
+
+=head1 DESCRIPTION
+
+The main purpose of the PDF::Parse library is to provide parsing functions
+for the more general PDF library.
+
+=head1 Methods
+
+The available methods are:
+
+=cut
+
+require 5.005;
+#require PDF::Core;
+require Core;
+
+use strict;
+use Carp;
+use Exporter ();
+
+use vars qw(@ISA @EXPORT_OK);
+
+@ISA = qw(Exporter PDF::Core);
+
+@EXPORT_OK = qw( LoadPageInfo GetInfo TargetFile
+				 Pages PageSize PageRotation IsaPDF
+				 Version IscryptPDF );
+
+#################################################################
+sub ReadCrossReference_pass1 {
+  my $fd = shift;
+  my $offset=shift;
+  my $self=shift;
+
+  my $initial_number;
+  my $obj_counter=0;
+  my $global_obj_counter=0;
+  my $buf;
+
+  binmode $fd;
+
+  $_=PDF::Core::PDFGetline ($fd,\$offset);
+
+  die "Can't read cross-reference section, according to trailer\n" if ! /xref\r?\n?/  ;
+
+  while () {
+    $_=PDF::Core::PDFGetline ($fd,\$offset);
+    s/^\n//;
+    s/^\r//;
+    last if (m/\btrailer\b/) ;
+#
+# An Object
+#
+    /^\d+\s+\d+\s+n\r?\n?/ && do { my $buf =$_;
+	       my $ind = $initial_number + ($obj_counter++);
+               ( not defined $self->{Objects}[$ind] )&& 
+		  do { $self->{Objects}[$ind] = int substr($buf,0,10);
+		       $self->{Gen_Num}[$ind] = int substr($buf,11,5);
+		     };
+	       $_=$buf;
+	       s/^.{18}//; 
+	       next ;
+   }; 
+#
+# A Freed Object
+#
+    /^\d+\s+\d+\s+f\r?\n?/ && do { my $buf =$_;
+   	       my $objects_generation_nr = substr($buf,11,5);
+	       my $Num=substr($buf,0,10);
+	       my $ind = $initial_number + ($obj_counter++);
+	       # $ind = $ind . "_" . $objects_generation_nr;
+		       $self->{Objects}[$ind] = - $Num;
+		       $self->{Gen_Num}[$ind] = $objects_generation_nr;
+		       $_=$buf;
+		       s/^.{18}//; 
+		       next ;
+     };
+#
+# A subsection
+#
+    /^\d+\s+\d+\r?\n?/  && do { 
+ 	my $buf = $_ ; 
+ 	 $initial_number = $buf; 
+ 	 $initial_number=~ s/^(\d+)\s+\d+\r?\n?.*/$1/; 
+	 $global_obj_counter += $obj_counter;
+ 	 $obj_counter=0; 
+	 next ;
+    };
+  }
+
+  $global_obj_counter +=$obj_counter;
+#
+# Now the trailer for updates 
+#
+
+#
+# Skip to start of dictionary.
+#
+    until (m/<</)
+		{
+		$_=PDF::Core::PDFGetline ($fd,\$offset);
+		}
+
+#
+# Read the dictionary
+#
+    my %trailer = ( PDF::Core::PDFGetPrimitive ($fd, $offset) );
+
+    if ($self->{"Trailer"}{"/Root"} eq "")
+		{
+		$self->{"Trailer"} = \%trailer;
+		#
+		# This code is here for backward compatibility only. If the content
+		# of the root trailer is needed, use $self->{"Trailer"} instead.
+		#
+		$self->{"Cross_Reference_Size"} = $trailer{"/Size"};
+		$self->{"Root_Object"} = $trailer{"/Root"};
+		$self->{"Crypt_Object"} = $trailer{"/Encrypt"};
+		}
+	if ($trailer{"/Prev"} =~ m/^\d+$/)
+		{  
+  		$self->{"Updated"} = 1;
+		my $old_seek = tell $fd;
+		$global_obj_counter += ReadCrossReference_pass1 ($fd,
+            $trailer{"/Prev"}, $self );
+		seek $fd, $old_seek, 0;
+		}
+
+
+  return $global_obj_counter;
+}
+
+#################################################################
+sub LoadPageSubtree (\*$;%)
+	{
+	my $self = shift;
+	my $ref = shift;
+	my %inheritance = @_ ;
+
+	my $data = $self->GetObject ($ref);
+
+	# Check which attributes are inherited. Adobe did not add any new
+	# inherited attributes in version 1.2 or later, so this list is
+	# complete.
+
+	# Do simple values.
+	foreach my $key ("/Rotate", "/Dur", "/Hid", "/Trans", 
+			 "/MediaBox", "/CropBox") {
+
+	    if (defined ($data->{$key})){
+
+		# Check if it is an indirect reference
+		if ($data->{$key} =~ m/^\d+ \d+ R$/) {
+		    
+		    my $dataref = $data->{$key};
+
+		    do {
+
+			$dataref = $self->GetObject ($dataref);
+
+		    } while ($dataref =~ m/^\d+ \d+ R$/);
+
+		    if (UNIVERSAL::isa ($data, "ARRAY")){
+
+			$inheritance{$key} = [];
+
+			foreach my $i (@{$data}){
+			    # Each element may be a reference.
+			    while ($i =~ m/^\d+ \d+ R$/){
+				$i = $self->GetObject ($i);
+			    }
+			    
+			    push @{$inheritance{$key}}, $i;
+			}
+		    } else {
+			$inheritance{$key} = $dataref;
+		    }
+		} else { #not an indirect reference
+		    $inheritance{$key} = $data->{$key};
+		}
+	    }
+	}
+	
+	# If this objects contains ressources, replace information in inheritance
+	$inheritance{"Resource_Object"} = $data->{"/Resources"}
+	    if (defined ($data->{"/Resources"}));
+
+	if ($data->{"/Type"} eq "/Pages")
+		{
+		# It's just an intermediate Node
+		foreach my $kid (@{$data->{"/Kids"}})
+			{
+			$self->LoadPageSubtree ($kid, %inheritance);
+			}
+		}
+	elsif ($data->{"/Type"} eq "/Page")
+		{
+		# We have a real page!
+		$inheritance{"Page_Object"} = $ref;
+		push @{$self->{"Page"}}, +{ %inheritance };
+		}
+	else
+		{
+		# Strange stuff. Complain and discard.
+		carp "While loading pages got object of type '", $data->{"/Type"}, "'";
+		}
+	}
+
+#################################################################
+=pod
+
+=head2 TargetFile ( filename )
+
+This method links the filename to the pdf descriptor and parses all
+kind of header information.
+
+=cut
+
+sub TargetFile {
+  my $self = shift;
+  my $file = shift;
+
+  croak "Already linked to the file ",$self->{File_Name},"\n" 
+      if $self->{File_Name} ;
+  
+  my $offset;
+
+  if ( $file ) {
+    open(FILE, "< $file") or croak "can't open $file: $!";
+    binmode FILE;
+    $self->{File_Name} = $file ;
+    $self->{File_Handler} = \*FILE;
+    my $buf;
+    read(FILE,$buf,4);
+    if ( $buf ne "%PDF" ) {
+     print "File $_[0] is not PDF compliant !\n" if $PDF::Verbose ;
+     return 0 ;
+    }
+    read(FILE,$buf,4);
+    $buf =~ s/-//;
+    $self->{Header}= $buf;
+    seek FILE,-50,2;
+    read( FILE, $offset, 50 );
+    $offset =~ s/[^s]*startxref\r?\n?(\d*)\r?\n?%%EOF\r?\n?/$1/;
+
+	$self->{"Last_XRef_Offset"} = $offset;
+    ReadCrossReference_pass1 (\*FILE, $offset, $self);
+	$self->{"Info"} = $self->GetObject ($self->{"Trailer"}{"/Info"});
+	$self->{"Catalog"} = $self->GetObject ($self->{"Trailer"}{"/Root"});
+	$self->{"PageTree"} = $self->GetObject ($self->{"Catalog"}{"/Pages"});
+    return 1;
+  } else {
+    croak "I need a file name (!)";
+	}
+}
+
+#################################################################
+=pod
+
+=head2 LoadPageInfo
+
+This function loads the information for all pages. This process can
+take some time for big PDF-files.
+
+=cut
+
+sub LoadPageInfo (\*)
+	{
+	my $self = shift;
+
+	# Reset Page Array
+	$#{$self->{"Page"}} = -1;
+
+	# Recurse
+	$self->LoadPageSubtree ($self->{"Catalog"}{"/Pages"});
+	}								
+
+
+
+#################################################################
+=pod
+
+=head2 Version
+
+Returns the PDF version used for writing the object file.
+
+=cut
+
+sub Version { 
+  return ($_[0]->{Header}); 
+}
+
+#################################################################
+=pod
+
+=head2 IsaPDF
+
+Returns true, if the file could be parsed and is a PDF-file.
+
+=cut
+
+sub IsaPDF { 
+  return ($_[0]->{Header} != undef) ; 
+}
+
+#################################################################
+=pod
+
+=head2 IscryptPDF
+
+Returns true if the PDF contains a crypt object. This indicates that
+the data of the PDF-File is encrypted. In this case, not all function
+work as expected.
+
+=cut
+
+sub IscryptPDF { 
+  return ($_[0]->{Crypt_Object} != undef) ; 
+}
+
+#################################################################
+=pod
+
+=head2 GetInfo ( key )
+
+Returns the various information contained in the info section of a PDF
+file (if present). A PDF file can have:
+
+  a title ==> GetInfo ("Title")
+  a subject ==> GetInfo ("Subject")
+  an author ==> GetInfo("Author")
+  a creation date ==> GetInfo("CreationDate")
+  a creator ==> GetInfo("Creator")
+  a producer ==> GetInfo("Producer")
+  a modification date ==> GetInfo("ModDate")
+  some keywords ==> GetInfo("Keywords")
+
+=cut
+
+sub GetInfo (\*$)
+	{
+	my $self = shift;
+	my $type = shift;
+
+	return PDF::Core::UnQuoteString ($self->{"Info"}{"/" . $type})
+	}
+
+#################################################################
+=pod
+
+=head2 Pages
+
+Returns the number of pages of the PDF-file.
+
+=cut
+
+sub Pages 
+	{
+	my $self = shift;
+
+	return $self->{"PageTree"}{"/Count"};
+	}
+
+
+
+#################################################################
+=pod
+
+=head2 PageSize ( [ page ] )
+
+Returns the size of a page in the PDF-file. If no parameter is given,
+the default size of the root page will be returned. This value may be
+overridden for any page.
+
+If the size of an individual page is requested and the page data is
+not already loaded, the method B<LoadPageInfo> will be executed. This
+may take some time for large PDF-files. The size of the root page is
+always available and will never execute B<LoadPageInfo>.
+
+=cut
+
+sub PageSize (;$)
+	{
+	my $self = shift;
+	my $page = shift;
+
+	if ($page > 0)
+		{
+		return undef if ($page > $self->{"PageTree"}{"/Count"});
+		$self->LoadPageInfo unless ($#{$self->{"Page"}} >= 0);
+		
+		return @{$self->{"Page"}[$page - 1]{"/MediaBox"}}
+		if (defined $self->{"Page"}[$page - 1]{"/MediaBox"});
+		}
+	else
+		{
+		return @{$self->{"PageTree"}{"/MediaBox"}}
+		if (defined $self->{"PageTree"}{"/MediaBox"});
+		}
+
+	return undef;
+	}
+
+
+#################################################################
+=pod
+
+=head2 PageRotation ( [ page ] )
+
+Returns the rotation of a page in the PDF-file. If no parameter is given,
+the default rotation of the root page will be returned. This value may be
+overridden for any page.
+
+If the rotation of an individual page is requested and the page data is
+not already loaded, the method B<LoadPageInfo> will be executed. This
+may take some time for large PDF-files. The rotation of the root page is
+always available and will never execute B<LoadPageInfo>.
+
+=cut
+sub PageRotation (;$)
+	{
+	my $self = shift;
+	my $page = shift;
+
+	my $rotate = 0;
+
+	if ($page > 0)
+		{
+		return undef if ($page > $self->{"PageTree"}{"/Count"});
+		$self->LoadPageInfo unless ($#{$self->{"Page"}} >= 0);
+		
+		$rotate = $self->{"Page"}[$page - 1]{"/Rotate"};
+		}
+	else
+		{
+		$rotate = $self->{"PageTree"}{"/Rotate"};
+		}
+
+	print "Rotation ", 0 + $rotate if ($PDF::Verbose);
+
+	return 0 + $rotate;
+	}
+#################################################################
+1;
+__END__
+
+=head1 Variables
+
+The only available variable is :
+
+=over
+
+=item B<$PDF::Parse::VERSION>
+
+Contains the version of the library installed
+
+=back
+
+
+=head1 Copyright
+
+  Copyright (c) 1998 - 2000 Antonio Rosella Italy antro@tiscalinet.it, Johannes Blach dw235@yahoo.com 
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 Availability
+
+The latest version of this library is likely to be available from:
+
+http://www.geocities.com/CapeCanaveral/Hangar/4794/
+
+=cut
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/amend_pdf.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/amend_pdf.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/amend_pdf.pl	(revision 21112)
@@ -0,0 +1,682 @@
+#!/usr/bin/perl
+#Jessica Thurston
+#usage: perl amend_pdf.pl filename
+#This program takes a pdf file 'filename'
+#and modifys it to include outline entries
+#(bookmarks).  
+#This is achieved using a pdf library which
+#parses the pdf file into objects allowing 
+#this script to more easily manipulate the pdf
+#objects and then modify the file by means
+#of incremental updates. (modifys the necessary
+#pdf objects and appends them to the end of 
+#the file, then updates the xreference table and
+#trailer). This library can be downloaded at
+# http://www.sanface.com/PDF-lib/
+#A file url.txt stored in the same
+#directory is read containing two columns of data:
+#title of related pdf document     url of related pdf document
+#The pdf document is modified so that the title of the
+#related pdf documents are displayed as bookmarks and
+#when the user clicks on it the web browser opens at
+#the url stored in the same column of url.txt.
+#
+#This program was designed to be used with the Greenstone
+#digital library in the following ways:
+#-When a user has built a collection containing
+# pdf documents, they have a choice of either displaying
+# the page in the web browser in gml form or saving the
+# original pdf document to view in an acrobat pdf reader.
+#-The related document file url.txt can be downloaded
+# in the same way if the user has included in the format
+# string for that page (ie search list) in the collection 
+# configuration file [urllink] related documents [/urllink].  
+# This will create a html link 'related documents' which 
+# when clicked on will enable the user to save a file url.txt 
+# containing the related documents for that particular document. 
+#-This file has been written upon running the perl script
+# relationPDF.pl which will also amend the gml documents to 
+# include the 'urllink' information enabling the user to 
+# include the related document links in config file format 
+# strings.
+#-So once this related doc info has been included the user
+# can download both the pdf file (doc.pdf) and the related
+# document file for that particluar document (url.txt).
+#-perl amend_pdf.pl doc.pdf is then run in the command line
+# modifying the pdf document to the desired effect.  
+#bugs:
+#-cannot be run more than once on the same file
+#-doesn't make use of free objects (free objects
+# are quite unlikely though, I didn't come across
+# any while looking through pdfs)
+#-many other things I haven't bothered testing
+
+use Carp;
+use Getopt::Long;
+use PDF;
+
+$objects = 0;
+$offsets = 1;
+
+
+#for each command line argument add the outlines
+#contained in url.txt
+#foreach (@ARGV) {
+#    create_outlines($_); 
+#}
+
+if(defined $ARGV[0]){
+    $pdf_document = $ARGV[0];
+} else {
+    print STDERR "You must supply pdf document filename\n";
+    return;
+}
+if(defined $ARGV[1]){
+    $related_docs = $ARGV[1];
+} else {
+print STDERR "You must supply related document filename\n";
+return;
+}
+create_outlines($pdf_document); 
+
+exit(1);
+
+
+#this function first checks to see is the file is indeed
+#a pdf file - if not then exits. If it is a pdf then it
+#checks to see if it is encrypted. If it is then exits.
+#Next we check if the outlines dictionary exists in the
+#document catalog. 
+#-If the outline entry exists but there are no actual outlines
+# present then call add_outlines but pass on the existing
+# outline dictionary.
+#-otherwise call modify_outlines to modify the existing outline
+# structure to include new outlines.
+#If outlines dictionary doesn't exist call add_outlines.
+sub create_outlines {
+
+  my $filename = shift;
+
+  #parse pdf filename
+  my @filelist = split("/", $filename);
+  my $file = pop @filelist;
+
+  print STDERR "pdf file to be manipulated: $file\n";
+  print STDERR "pdf file to be saved: tmp_$file\n";
+
+  #make a copy of that file in this directory
+  print STDERR "cp $filename $file\n"; 
+  `cp $filename $file\n`; 
+  `cp $filename tmp_$file\n`; 
+
+  #create and parse a new 'pdf' object
+  my $PDFfile = PDF->new($file);  
+
+  #if it is a pdf file
+  if ($PDFfile->{"Header"}){
+
+      if($PDFfile->IscryptPDF){
+	  #an encrypted file so cannot continue with
+	  #adding the related doc outlines to it 
+	  print STDERR "file \"$file\" is encrypted \n"; 
+	  return;
+      }
+
+      #display some relevant information about the document
+      print STDERR "Author: ", $PDFfile->GetInfo ("Author"), "\n"; 
+      print STDERR "Title: ", $PDFfile->GetInfo ("Title"), "\n"; 
+      print STDERR "Subject: ", $PDFfile->GetInfo ("Subject"), "\n"; 
+      print STDERR "Updated: ", $PDFfile->{"Updated"}, "\n";
+     
+      #if the pdf document already includes an outline dictionary
+      if (defined $PDFfile->{"Catalog"}{"/Outlines"}) {
+
+	  #get the outline object from the indirect ref
+	  my $outline_data = $PDFfile->GetObject($PDFfile->{"Catalog"}{"/Outlines"});
+	  #obtain the number of existing outlines
+	  $PDFfile->{"Outlines"}{"/Count"} = $outline_data->{"/Count"};
+	  print STDERR "number of existing outlines: $PDFfile->{\"Outlines\"}{\"/Count\"}\n";
+
+	  #this means that the pdf file had an outline dictionary but
+	  #did not actually include any outlines.
+	  if($PDFfile->{"Outlines"}{"/Count"} == 0){
+	      #obtain object number of outline dictionary and pass to add_outlines
+	      my $dictionary = split(/\s/, $PDFfile->{"Catalog"}{"/Outlines"});
+	     
+	      add_outlines($PDFfile, $file, $dictionary);
+	  } else { 
+	      #collect other outline data to pass to modify_outlines
+	      $PDFfile->{"Outlines"}{"/First"} = $outline_data->{"/First"};
+	      $PDFfile->{"Outlines"}{"/Last"} = $outline_data->{"/Last"};
+
+	      #modify last outline entry
+	      modify_outlines($outline_data, $PDFfile, $file);
+	  }
+      } else { #there was no outline dictionary thus no outlines so add some
+	  print STDERR "no bookmarks in \"$file\" \n"; 
+	  add_outlines($PDFfile, $file, 0);
+      }
+  } else { #the file was not a pdf file
+      print STDERR "$file is not a pdf file!!\n"; 
+  }
+
+
+  #copy back the file we modified and the saved copy 
+  my $copypath = join("/", @filelist);
+  print STDERR "cp $file $filename\n"; 
+  print STDERR "cp tmp_$file $copypath/tmp_$file\n"; 
+  `cp $file $filename\n`; 
+  `cp tmp_$file $copypath/tmp_$file\n`; 
+
+  #remove the copy of that file in this directory
+  #print STDERR "rm $file\n"; 
+  `rm $file\n`;  
+  `rm tmp_$file\n`; 
+}
+
+
+#add_outlines function recieves following params
+#PDF file information, filename of pdf file, outline dictionary
+#(if outline dict. is 0 that means it must be created).
+#First the function decides if we are to create new dictionary
+#or not. Next read in the url file. Open up the pdf document.
+#Modify the catalog so that it includes the outline dictionary
+#(this is done even if the catalog doesn't need modifying 
+# so that the objects in the objects/offset table will not
+# be all mucked up when it comes time to write the xreference
+# table)
+#Then append outline dictionary to end of file (either modified
+#or created). Then append the top heirarchy related document
+#outline.  Then the new file url bookmarks. Call  xref_table
+#to write new xreference table. Call trailer to write new trailer.
+
+#table[0][0] - document catalog
+#table[0][1] - document catalog offset
+#table[1][0] - outline dictionary
+#table[1][1] - outline dictionary offset
+#table[2][0] - related document outline
+#table[2][1] - related document outline offset
+#table[3][0] etc - url outlines, read from file
+#table[3][1] etc - url outlines offset, read from file
+
+sub add_outlines (*\$){
+    
+ my $PDFfile = shift; 
+ my $file = shift; 
+
+ my @table;
+ $table[1][$objects] = shift;
+
+ #read in file of url's
+ my @urls = read_file(); 
+ my $url_num = $#urls + 1;
+ 
+ #if outline dictionary was not present in catalog
+ if ($table[1][$objects] == 0){ #get new object number for new dictionary
+     $table[1][$objects] = $PDFfile->{"Trailer"}{"/Size"}; #outline dictionary to be created
+     $table[2][$objects] = $table[1][$objects] + 1; #related document outline
+ } else { #else get object number for related doc outline to be appended
+     $table[2][$objects] = $PDFfile->{"Trailer"}{"/Size"}; #related document outline
+ }
+ 
+ #open up pdf file for appending
+ open(FILE, ">> $file") or croak "can't open $file: $!"; 
+ binmode \*FILE;
+ 
+ #obtain object number and offset for the document catalog
+ $table[0][$objects] = split(/\s/, $PDFfile->{"Trailer"}{"/Root"});
+ $table[0][$offsets] = tell \*FILE;
+
+ #print the modified or original catalog back to the file (appended)
+ print FILE "$table[0][$objects] 0 obj
+";
+ print FILE "<<
+";
+ print FILE "/Pages $PDFfile->{\"Catalog\"}{\"/Pages\"}
+";
+ print FILE "/Outlines $table[1][$objects] 0 R
+"; #only line actually added
+ print FILE "/Type /Catalog
+";
+ print FILE "/DefaultGray $PDFfile->{\"Catalog\"}{\"/DefaultGray\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/DefaultGray"}));
+ print FILE "/DefaultRGB $PDFfile->{\"Catalog\"}{\"/DefaultRGB\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/DefaultRGB"}));
+ print FILE "/PageLabels $PDFfile->{\"Catalog\"}{\"/PageLabels\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/PageLabels"}));
+ print FILE "/Names $PDFfile->{\"Catalog\"}{\"/Names\"}
+" 
+     if (defined ($PDFfile->{"Catalog"}{"/Names"}));
+ print FILE "/Dests $PDFfile->{\"Catalog\"}{\"/Dests\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/Dests"}));
+ print FILE "/ViewerPreferences $PDFfile->{\"Catalog\"}{\"/ViewerPreferences\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/ViewerPreferences"}));
+ print FILE "/PageLayout $PDFfile->{\"Catalog\"}{\"/PageLayout\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/PageLayout"}));
+ print FILE "/PageMode $PDFfile->{\"Catalog\"}{\"/PageMode\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/PageMode"}));
+ print FILE "/Threads $PDFfile->{\"Catalog\"}{\"/Threads\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/Threads"}));
+ print FILE "/OpenAction $PDFfile->{\"Catalog\"}{\"/OpenAction\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/OpenAction"}));
+ print FILE "/URI $PDFfile->{\"Catalog\"}{\"/URI\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/URI"}));
+ print FILE "/Acroform $PDFfile->{\"Catalog\"}{\"/Acroform\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/Acroform"}));
+ print FILE "/StructTreeRoot $PDFfile->{\"Catalog\"}{\"/StructTreeRoot\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/StructTreeRoot"}));
+ print FILE "/SpiderInfo $PDFfile->{\"Catalog\"}{\"/SpiderInfo\"}
+"
+     if (defined ($PDFfile->{"Catalog"}{"/SpiderInfo"}));
+ print FILE ">>
+";
+ print FILE "endobj
+";
+ 
+ #obtain offset for outline dictionary
+ $table[1][$offsets] = tell \*FILE;
+
+ #append newly created outline dictionary
+ print FILE "$table[1][$objects] 0 obj
+";
+ print FILE "<<
+";
+ print FILE "/Type /Outlines
+";
+ print FILE "/Count ", $url_num + 1, "
+";
+ print FILE "/First $table[2][$objects] 0 R
+";
+ print FILE "/Last $table[2][$objects] 0 R
+";
+ print FILE ">>
+";
+ print FILE "endobj
+";
+
+
+ #get the related document outline object num and offset
+ my $obj = $table[2][$objects]; 
+ $table[2][$offsets] = tell \*FILE;
+
+ #append the top heirarchy related document outline to file 
+ print FILE "$table[2][$objects] 0 obj
+";
+ print FILE "<<
+";
+ print FILE "/Title (Related Documents)
+";
+ print FILE "/Parent $table[1][$objects] 0 R
+";
+ print FILE "/Count ", $url_num, "
+";
+ print FILE "/First ", $obj + 1, " 0 R
+";
+ print FILE "/Last ", $table[2][$objects] + $url_num, " 0 R
+";
+ print FILE ">>
+";
+ print FILE "endobj
+";
+
+ my $ind = 3;
+ $obj++;
+
+ 
+ #store the object nums and offsets of the new related document
+ #outlines and write them to the file (must be outline with
+ #an action eg go to specific url)
+ for $i ( 0 .. $#urls ) {
+     $table[$ind][$offsets] = tell \*FILE; 
+     $table[$ind][$objects] = $obj; 
+     print FILE "$obj 0 obj
+";
+     print FILE "<<
+";
+     print FILE "/Title ($urls[$i][0])
+"; 
+     print FILE "/Parent $table[2][$objects] 0 R
+";
+     print FILE "/Next ", $obj + 1, " 0 R
+" if (($obj + 1) <= ($table[2][$objects] + $url_num)); 
+     print FILE "/Prev ", $obj - 1, " 0 R
+" if(($obj - 1) != ($table[2][$objects]));
+     print FILE "/A << /Type /Action
+";
+     print FILE "/S /URI
+"; 
+     print FILE "/URI ($urls[$i][1])
+"; 
+     print FILE ">>
+";
+     print FILE ">>
+";
+     print FILE "endobj
+";
+     $obj++;
+     $ind++; 
+ }
+
+
+ #append new X-reference table 
+ my $xref_offset = tell \*FILE;
+ xref_table(\*FILE, $url_num, @table);
+
+ #print trailer 
+ trailer(\*FILE, $PDFfile, $obj); 
+ print FILE "$xref_offset
+";
+ print FILE "%%EOF
+"; 
+
+ #close FILE;
+}
+
+#modify_outlines function recieves following params
+#PDF file information, outline data, pdf filename
+#First the function obtains the object data for the
+#last outline entry. Next read in the url file. 
+#Open up the pdf document.Then append the modified 
+#outline dictionary to end of file. Next append the
+#modified last outline.  Then append the 
+#top heirarchy related document outline.  Then the 
+#new file url bookmarks. Call  xref_table
+#to write new xreference table. 
+#Call trailer to write new trailer.
+
+#table[0][0] - outline dictionary
+#table[0][1] - outline dictionary offset
+#table[1][0] - last outline
+#table[1][1] - last outline offset
+#table[2][0] - related document outline
+#table[2][1] - related document outline offset
+#table[3][0] etc - url outlines, read from file
+#table[3][1] etc - url outlines offset, read from file
+
+sub modify_outlines (*\$){
+
+    my ($outline_data, $PDFfile, $file) = @_;
+
+    #collect the data for the last outline (which must be modified)
+    $PDFfile->{"Outlines"}{"/Last"} = $outline_data->{"/Last"};
+    my $last_data = $PDFfile->GetObject($PDFfile->{"Outlines"}{"/Last"});  
+
+    #read in file of url's
+    my @urls = read_file(); 
+    my $url_num = $#urls + 1;
+ 
+    my @table;
+
+    #This number is the number to use for the next created object
+    #eg the related doc heirarchy
+    $table[2][$objects] = $PDFfile->{"Trailer"}{"/Size"};
+  
+    #open up the pdf file for appending
+    open(FILE, ">> $file") or croak "can't open $file: $!"; 
+    binmode \*FILE;
+    
+    #store the object number and offset of the outline dictionary
+    $table[0][$objects] = split(/\s/, $PDFfile->{"Catalog"}{"/Outlines"});
+    $table[0][$offsets] = tell \*FILE;
+   
+    #write the outline dictionary back to the file (appending)
+    print FILE "$table[0][$objects] 0 obj
+";
+    print FILE "<<
+";
+    print FILE "/Type /Outlines
+";
+    print FILE "/Count ", $PDFfile->{"Outlines"}{"/Count"} + 1, "
+"
+	if (defined ($PDFfile->{"Outlines"}{"/Count"}));
+    print FILE "/First $PDFfile->{\"Outlines\"}{\"/First\"}
+"
+	if (defined ($PDFfile->{"Outlines"}{"/First"}));
+    print FILE "/Last $table[2][$objects] 0 R
+";
+    print FILE ">>
+";
+    print FILE "endobj
+";
+    
+    #store the last outline entry object num and file offset
+    my @last_entry = split(/\s/, $PDFfile->{"Outlines"}{"/Last"});
+    $table[1][$objects] = $last_entry[0]; 
+    $table[1][$offsets] = tell \*FILE; 
+
+    #append modified last outline entry
+    print FILE "$table[1][$objects] 0 obj
+";
+    print FILE "<<
+";
+    print FILE "/Title $last_data->{\"/Title\"}
+";
+    print FILE "/Dest $last_data->{\"/Dest\"}
+" if(defined $last_data->{"/Dest"});
+    print FILE "/Parent $last_data->{\"/Parent\"}
+";
+    print FILE "/Prev $last_data->{\"/Prev\"}
+";
+    print FILE "/Next $table[2][$objects] 0 R
+";
+    print FILE "/First $last_data->{\"/First\"}
+" if(defined $last_data->{"/First"});
+    print FILE "/Last $last_data->{\"/Last\"}
+" if(defined $last_data->{"/Last"});
+    print FILE "/Count $last_data->{\"/Count\"}
+"if(defined $last_data->{"/Count"});
+    print FILE "/A $last_data->{\"/A\"}
+" if(defined $last_data->{"/A"});
+    print FILE "/SE $last_data->{\"/SE\"}
+" if(defined $last_data->{"/SE"});
+    print FILE ">>
+";
+    print FILE "endobj
+";
+    
+    #store the object num and offset of the related
+    #document top level outline
+    my $obj = $table[2][$objects] + 1;
+    $table[2][$offsets] = tell \*FILE;
+
+    #append related document top level outline
+    print FILE "$table[2][$objects] 0 obj
+";
+    print FILE "<<
+";
+    print FILE "/Title (Related Documents)
+"; 
+    print FILE "/Parent $last_data->{\"/Parent\"}
+";
+    print FILE "/Count ", $url_num, "
+";
+    print FILE "/First $obj 0 R
+";
+    print FILE "/Last ", $table[2][$objects] + $url_num , " 0 R
+";
+    print FILE ">>
+";
+    print FILE "endobj
+";
+    
+   
+    my $ind = 3;
+
+    #store the object nums and offsets of the new related document
+    #outlines and write them to the file (must be outline with
+    #an action eg go to specific url)
+    for $i ( 0 .. $#urls ) { 
+	$table[$ind][$objects] = $obj;  
+	$table[$ind][$offsets] = tell \*FILE; 
+	print FILE "$obj 0 obj
+";
+	print FILE "<<
+";
+	print FILE "/Title ($urls[$i][0])
+"; 
+	print FILE "/Parent $table[2][$objects] 0 R
+";
+	print FILE "/Next ", $obj + 1, " 0 R
+" if (($obj + 1) <= ($table[2][$objects] + $url_num)); 
+	print FILE "/Prev ", $obj - 1, " 0 R
+" if (($obj - 1) != ($table[2][$objects]));
+	print FILE "/A << /Type /Action
+";
+	print FILE "/S /URI
+"; 
+	print FILE "/URI ($urls[$i][1])
+"; 
+	print FILE ">>
+";
+	print FILE ">>
+";
+	print FILE "endobj
+";
+	$obj++;
+	$ind++;
+    }
+    
+    #append new X-reference table
+    my $xref_offset = tell \*FILE;   
+    xref_table(\*FILE, $url_num, @table); 
+  
+    #print trailer  
+    trailer(\*FILE, $PDFfile, $obj);
+    print FILE "$xref_offset
+";
+    print FILE "%%EOF
+"; 
+
+    close FILE;
+
+}
+
+
+#This function reads a file 'url.txt' which contains
+#two columns of data in the following format:
+#related document title    related document url
+#each array of title, url is stored in an array
+#(to obtain the title of the first related  
+#document in the file)
+#eg table[1st related document][title] 
+#(to obtain the url of the second related 
+#document in the file)
+#eg table[2nd related document][url]
+#this table is then returned to the calling 
+#function.
+sub read_file {
+
+    #create two-dimensional array for urls
+    my @urls;
+
+    #open up url.txt to start reading
+    open(URL, "$related_docs") or croak "Can't open $related_docs: $!"; 
+
+    my $counter = 0;
+
+    #read in url.txt and store into the arry
+    while (<URL>){
+	chomp;
+	if(/([^\t]*)\t(.*)/){ 
+	    push @urls, [ split ]; #split on /s into arrays of arrays
+	}
+    }
+
+    close URL;
+
+    return (@urls);
+
+}
+
+ 
+#this function takes as parameters the filehandle
+#to the pdf document, the number of related documents
+#to this pdf doc and a table of object numbers and
+#their offsets. Using this information it writes
+#(appends) a new xreference table to the pdf 
+#document. 
+
+sub xref_table  (*\$) {
+    
+    my ($fd, $num, @table) = @_;
+    my $offset;
+
+    #print the new xref table (append to file)
+    print $fd "xref
+";
+    print $fd "0 1 
+";
+    print $fd "0000000000 65535 f 
+";
+    print $fd "$table[0][$objects] 1 
+";
+    $offset = '0'x(10-length($table[0][$offsets])).$table[0][$offsets];
+    print $fd "$offset 00000 n 
+";
+    print $fd "$table[1][$objects] 1 
+"; 
+    $offset = '0'x(10-length($table[1][$offsets])).$table[1][$offsets];
+    print $fd "$offset 00000 n 
+";
+    print $fd "$table[2][$objects] ", $num + 1, " 
+"; 
+    for $i ( 2 .. ($num + 2) ) { #add 2 on because already written 2 to file
+	$offset = '0'x(10-length($table[$i][$offsets])).$table[$i][$offsets];
+	print $fd "$offset 00000 n 
+";
+    }
+    
+}
+
+
+#This function recieves as parameters the filehandle
+#to the pdf document, parsed information about the
+#document and the new size (last object number + 1)
+#of the pdf file.  Using this information it appends
+#a new trailer to the end of the pdf document.
+
+sub trailer  (*\$) {
+
+    my ($fd, $PDFfile, $new_size) = @_;
+
+    #append the new trailer to the end of the file
+    print $fd "trailer
+";
+    print $fd "<<
+";
+    print $fd "/Size ", $new_size, "
+";
+    print $fd "/Root $PDFfile->{\"Trailer\"}{\"/Root\"}
+";
+    print $fd "/Info $PDFfile->{\"Trailer\"}{\"/Info\"}
+" 
+	if (defined ($PDFfile->{"Trailer"}{"/Info"}));
+    print $fd "/ID [$PDFfile->{\"Trailer\"}{\"/ID\"}[0]$PDFfile->{\"Trailer\"}{\"/ID\"}[1]]
+"
+	if (defined ($PDFfile->{"Trailer"}{"/ID"}));
+    print $fd "/Prev $PDFfile->{\"Last_XRef_Offset\"}
+";
+    print $fd "/Encrypt $PDFfile->{\"Trailer\"}{\"/Encrypt\"}
+"
+	if (defined ($PDFfile->{"Trailer"}{"/Encrypt"}));
+    print $fd ">>
+";
+    print $fd "startxref
+";
+   
+}
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/buildkpi.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/buildkpi.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/buildkpi.pl	(revision 21112)
@@ -0,0 +1,491 @@
+#! /user/bin/perl 
+
+#usage: perl buildkpi.pl [-R] [collection] [collection] etc
+#
+#-r or -R will remove previous index files so that you may build new ones
+#
+#The program performs the following tasks:
+#-gathers the specified collections on the command line OR
+#-gathers the directories of all the collections in the collect directory, this is all
+# the directories apart from CVS, modelcol, . and .. which are not collections.
+#-It then retrieves the archive.inf file from the archive directory of each collection
+# to obtain the unique file ID and filepath of every document in the collection
+#-Then parse through each doc.gml stored in filepath to gather information
+#-From each file collect kea phrases and/or stems
+#-Determine the number of kea phrases and stems for each document 
+#-Search for the kea phrases in the numbered phrase index. If a phrase is not there then 
+# the program will write the kea phrase to the phrase index . 
+#-Search for the kea phrases in the keyphrase to document index. If the phrase is there, 
+# it will increment the number of documents that the keyphrase appears in and replace that
+# then append the hash ID to the list of documents in the entry. If the phrase is not there 
+# then the program will write the kea phrase to the phrase index.
+#-Then write document ID, no of phrase, phrase number from index followed by number of times 
+# phrase appears into the document_keyphrase index
+
+$gsdlhome = $ENV{'GSDLHOME'};
+
+require "getopts.pl";
+&Getopts('R'); #process option arguments
+
+
+#collections may be specified in the command line
+#otherwise, all collections will be used to build
+#the indexes.  
+if(@ARGV){
+
+    @directories = @ARGV;
+
+} else { #open collect directory and get a list of all collections
+    opendir(DIR, "$gsdlhome/collect");
+    @directories = grep(!/(^\.|(CVS)|(modelcol))/, readdir(DIR));
+    closedir(DIR);
+}
+
+
+#directory to store indexes in using collection names
+$dirname = "";
+foreach $collection (sort @directories){
+    $dirname .= $collection."_";
+}
+$dirname .= "indexes";
+print STDERR "directory name: $dirname\n";
+
+#if option R remove all previous indexes
+if($opt_R == 1){ #remove indexes
+    print STDERR "\nremoving $gsdlhome/bin/script/indexes/$dirname/keyphrase_index.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt\n";
+    system("rm $gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt");   
+    system("rm $gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt"); 
+    system("rm $gsdlhome/bin/script/indexes/$dirname/keyphrase_index.txt");
+} 
+
+#create the new directory, display an error message if the directory
+#already exists.
+`mkdir --verbose $gsdlhome/bin/script/indexes/$dirname`;
+
+#for each collection specified to build indexes for
+foreach $collection (@directories){
+
+    print STDERR "\nBUILDING INDEXES FOR COLLECTION $collection\n\n";
+
+    my @filelist;
+
+    #archives.inf contains a list of unique hash ID's of each file and file paths
+    open(INFO, "$gsdlhome/collect/$collection/archives/archives.inf")
+	or die "$gsdlhome/collect/$collection/archives/archives.inf could not be opened.";
+
+    while(<INFO>){ #get each line of text from archives.inf (OID \t filepath)
+	chomp;
+	push(@filelist, $_);
+    }
+
+    foreach $file (@filelist){ #add each document to the indexes
+	build_index($file, $collection);
+    }
+}
+
+#This function opens the file in the filepath sent as an argument.  From this it obtains 
+#the kea and/or stem data, and then searches for these phrases in the file, counting and storing 
+#how many times each phrase appears. The data is then sent to function keyphrase_document 
+#with arguments hash ID, kea phrases and stem phrases to build the keyphrase_document index.
+#The function which builds the document_keyphrase index is then passed the hash ID, the kea 
+#phrases and/or the stemmed phrases and the array/s which hold the number of times each phrase
+#appears in the document so that the data it has collected can be written to document_ keyphrase
+#index.
+
+sub build_index {
+
+    my $args = shift(@_);
+    my $collection = shift(@_);
+    my ($ID, $filepath) = split(/\t/, $args);
+    my $keaS = "";
+    my $stemsS = "";
+    my @kea_phrase_counts = 0;
+    my @stem_phrase_counts = 0;
+    my $text = "";
+
+    print STDERR "\nID: $ID\n";
+    print STDERR "filepath: $filepath\n";
+
+    #open file to extract keyphrase information
+    open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+
+    #patterns to search for so that we can extract the kea information
+    my $kea_search = ".* kea=\"([^\"]*)\"";
+    my $stem_search = "stems=\"([^\"]*)\"";
+
+    while(<FILE>){ #get kea and stem data and store
+	chomp;
+	$keaS = $1 if (/$kea_search/);
+	$stemsS = $1 if (/$stem_search/);
+    }
+
+    close(FILE);
+    
+    print STDERR "Kea: $keaS\n";
+    print STDERR "stems: $stemsS\n";
+
+    my @kea = split(", ", $keaS);
+    my @stems = split(", ", $stemsS);
+
+    if(@kea && @stems){ #if the data exists
+	
+	#open the filepath to the current document
+	open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	    or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+	
+	while(<FILE>){ #get the text
+	    chomp;
+	    $text .= $_;
+	}
+ 
+	#chop out all things in angled brackets
+	$text =~ s/(<[^>]*>)//g;
+
+	#initilise counts
+	for($i=0; $i<=$#kea; $i++){
+	    $kea_phrase_counts[$i] = 0;
+	}
+	
+	for($i=0; $i<=$#stems; $i++){
+	    $stem_phrase_counts[$i] = 0;
+	}
+
+	print STDERR "counting number of kea phrases in document...\n";
+
+	#using regular expressions generated from kea-reg and stem-reg
+	#count how many of each phrase appear in the document
+	$text_copy = $text;
+	for($i=0; $i<=$#kea; $i++){ #search for text with kea phrases
+	    my $phrase = $kea[$i];
+	    $reg = &kea_reg(split(/\s+/, $phrase));
+	    while($text_copy =~ s/$reg//i){
+		$kea_phrase_counts[$i]++; #count the number of kea phrases 
+	    } 
+	    $text_copy = $text;
+	}  
+
+	print STDERR "counting number of stemmed phrases in document...\n";
+
+	$text_copy = $text;
+	for($i=0; $i<=$#stems; $i++){ #search for text with stem phrases
+	    my $stem = $stems[$i];
+	    $reg = &stem_reg(split(/\s+/, $stem));
+	    while($text_copy =~ s/$reg//i){
+		$stem_phrase_counts[$i]++; #count the number of stem phrases
+	    } 
+	    $text_copy = $text;
+	}
+
+	
+	#write data to keyphrase_document index
+	&keyphrase_document($ID, $keaS, $stemsS); 
+	
+	#write data to document_keyphrase index
+	$kea_counts = join(", ", @kea_phrase_counts);
+	$stem_counts = join(", ", @stem_phrase_counts);
+	&document_keyphrase($ID, $keaS, $stemsS, $kea_counts, $stem_counts);
+	
+    } else { 
+	print STDERR "No kea data was found in file $filepath\n";
+    }
+
+}
+
+#returns a regular expression designed to 
+#search for stems in text
+#eg 'agri cari'
+#    agri followed by 0 or more non-whitespace characters
+#         followed by one or more whitespace OR 0 or 1 non-whitespace characters
+#    cari followed by 0 or more non-whitespace characters
+#modified from original by Stephen Lundy 
+
+sub stem_reg {
+   
+    $regexp = "";
+
+    $l = @_;
+
+    if ($l > 0) {
+        $s = shift;
+        $regexp = "$s\\S*";
+
+        if ($l-1 > 0) {
+            foreach $s (@_) {
+                $regexp .= "(\\s+|\\S?)$s\\S*";
+            }
+        }
+    }
+
+    return $regexp;
+}
+
+#returns a regular expression designed to 
+#search for phrases in text
+#eg 'agris caris'
+#    agris followed by 0 or 1 non-whitespace characters OR
+#          followed by one or more whitespace 
+#    caris followed by 0 or 1 non-whitespace characters
+#modified from original by Stephen Lundy 
+
+sub kea_reg {
+    $regexp = "";
+
+    $l = @_;
+
+    if ($l > 0) {
+        $s = shift;
+        #$regexp = "$s(\\S?)";
+	$regexp = "$s(\\s+|\\S?)";
+
+        if ($l-1 > 0) {
+            foreach $s (@_) {
+                $regexp .= "$s(\\s+|\\S?)";
+            }
+        }
+    }
+
+    return $regexp;
+}
+
+
+#This function is passed as arguments a list of kea phrases and/or stems. Its purpose is to 
+#check in the keyphrase index file for each phrase and determine whether or not an entry has
+#been made for that phrase and an index number assigned to it. If there has not been an entry
+#made then an index number is assigned to the phrase and it is written to the file.  This 
+#function is called by document_keyphrase and keyphrase_document.  Each line in the file has
+#this form:
+#-phrase index number:phrase
+#This function then returns a table of pairs of the phrases that were sent as arguments to it
+#{phrase => phrase index number}.
+ 
+sub keyphrase_index_search {
+
+    my $phrases = shift(@_); 
+    my @phrases = split(", ", $phrases);
+    my %table;
+    my $index = 1;
+    my $create_new_index = 0;
+
+    print STDERR "searching keyphrase index...\n";
+
+    #initilise table of phrases and index numbers
+    foreach $phrase (@phrases){
+	$table{"$phrase"} = "0";
+    }
+
+    #open keyphrase index for appending data and for reading
+    open(INDEX_OUT, ">>$gsdlhome/bin/script/indexes/$dirname/keyphrase_index.txt");
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/$dirname/keyphrase_index.txt")
+	or $create_new_index = 1;
+
+    if($create_new_index == 0){
+	#if the index already exists read in the phrases
+	while(<INDEX_IN>){
+	    chomp;
+	    foreach $phrase (@phrases){
+		if(/(\d+):$phrase/){
+		    $index = $1;
+		    $table{"$phrase"} = "$index";
+		}
+	    }
+	    $index++; #new starting index (one + the last index)
+	}
+
+	close(INDEX_IN);
+
+    }
+
+    #add new phrases to the phrase index
+    foreach $phrase (keys %table){
+	if($table{"$phrase"} eq "0"){
+	    print INDEX_OUT "$index:$phrase\n";
+	    $table{"$phrase"} = "$index"; 
+	    $index++;
+	}
+    }
+
+    close(INDEX_OUT);
+    return %table;
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrases and/or stems
+#that exist for that particular file.  Its purpose is to write to the keyphrase_document
+#index a line for the document it has been sent:
+#-phrase index number:number of documents it appears in|ID
+sub keyphrase_document{
+
+    my ($ID, $kea, $stems) = @_;
+    my $text = ""; 
+    my @textlist;
+    my $create_new_index = 0;
+
+    print STDERR "writing to keyphrase_document.txt...\n";
+
+    #get table of phrases and phrase indexes 
+    my %table = keyphrase_index_search($kea.", ".$stems); 
+    
+  
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt")
+	or $create_new_index = 1;
+
+    #read in document if file exists
+    if($create_new_index == 0){
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    } 
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt");
+
+    if($create_new_index == 0){ #amend existing index
+	
+	foreach $line (@textlist){
+	    foreach $phrase (keys %table){
+		if($line =~ /(\d+):(\d+)(.*)/){ #all lines of this form
+		    $index = $1;
+		    if($table{"$phrase"} eq "$index") { #if phrase exists in index
+			$ids = $3; #get all doc IDs for that keyphrase
+			if($ids !~ /$ID/){ #if doc ID not already included
+			    $num_docs = $2;
+			    $num_docs++; #increment number of docs
+			    $line = "$index:$num_docs$3|$ID"; #line to append to index
+			    $table{"$phrase"} = "0"; 
+			}
+		    }
+		}
+	    }
+	    print INDEX_OUT "$line\n";
+	}
+    }
+
+    #add new phrases to the index
+    foreach $phrase (keys %table){ #write 'phrase index:1:file ID
+	if($table{"$phrase"} ne "0"){
+	    my $line = "$table{$phrase}:1:$ID";
+	    print INDEX_OUT "$line\n"; 
+	}
+    }
+
+    close(INDEX_OUT);
+
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrases and/or stems
+#that exist for that particular file and a list of the number of times each kea and/or stem
+#phrase appear in that document.  Its purpose is to write to the document_keyphrase
+#index a line for the document it has been sent:
+#-file ID:number of phrases and/or stems appear in the document
+#      |pairs of 'phrase index,number of times the phrase appears in the document' 
+sub document_keyphrase {
+
+    my ($ID, $keaS, $stemsS, $kea_c, $stem_c) = @_;
+    my $text = "";
+    my @textlist;
+    my %phrases;
+    my $create_new_index = 0;
+
+    print STDERR "writing to document_keyphrase.txt...\n";
+
+    #split phrase counts into arrays
+    my @kea_counts = split(", ", $kea_c);
+    my @stem_counts = split(", ", $stem_c);
+  
+    #get table of phrases and phrase indexes
+    my %table = keyphrase_index_search($keaS.", ".$stemsS);
+
+    #split phrases into arrays
+    my @kea = split(", ", $keaS);
+    my @stems = split(", ", $stemsS);
+
+    #build new phrases dictionary
+    for($i=0; $i<=$#kea; $i++){
+	my $phrase = $table{"$kea[$i]"};
+	if($kea_counts[$i] > 0){
+	    $phrases{"$phrase"} = "$kea_counts[$i]";
+	} else {
+	    $phrases{"$phrase"} = 1;
+	}
+    } 
+    for($i=0; $i<=$#stems; $i++){
+	my $phrase = $table{"$stems[$i]"};
+	if($stem_counts[$i] > 0){
+	    $phrases{"$phrase"} = "$stem_counts[$i]";
+	} else {
+	    $phrases{"$phrase"} = 1;
+	}
+    }   
+    my @num = keys %phrases;
+    my $phrasenum = $#num + 1; #number of phrases in doc
+
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt")
+	or $create_new_index = 1;
+
+     
+    if($create_new_index == 0){ #index doesn't need to be created
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+	
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    }  
+
+
+    #must write this line to the file
+    #document ID:num of phrases|phrase index, number of times phrases appears
+    my $newline = "$ID:$phrasenum";
+    foreach $phrase (keys %phrases){
+	$newline .= "|$phrase,$phrases{$phrase}";
+    }
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt");
+
+    if($create_new_index == 1){ #create a new index
+
+	print INDEX_OUT "$newline\n";
+
+    } else {
+
+	#if ID is already in the file write line overtop incase
+	#someone has modified the file.  Otherwise add the line
+	#to the end of the file
+	my $found = 0;
+
+	foreach $line (@textlist){
+	    if($line =~ /([^:]+):(.*)/){ #all lines should follow this pattern
+		$id = $1;
+		if($ID eq $id){ #id is already in the file
+		    print INDEX_OUT "$newline\n"; #print line overtop
+		    $found = 1;
+		} else {
+		    print INDEX_OUT "$line\n"; #print old line out
+		}
+	    }
+	}
+
+	print INDEX_OUT "$newline\n" if ($found == 0); #append new line to end of file 
+
+    }
+
+    close(INDEX_OUT);
+
+}
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/buildkpiK.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/buildkpiK.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/buildkpiK.pl	(revision 21112)
@@ -0,0 +1,411 @@
+#! /user/bin/perl 
+
+#usage: perl buildkpiK.pl [-R] [collection] [collection] etc
+#
+#-r or -R will remove previous index files so that you may build new ones
+#
+#The program performs the following tasks:
+#-gathers the specified collections on the command line OR
+#-gathers the directories of all the collections in the collect directory, this is all
+# the directories apart from CVS, modelcol, . and .. which are not collections.
+#-It then retrieves the archive.inf file from the archive directory of each collection
+# to obtain the unique file ID and filepath of every document in the collection
+#-Then parse through each doc.gml stored in filepath to gather information
+#-From each file collect kea phrases. 
+#-Determine the number of kea phrases for each document 
+#-Search for the kea phrases in the numbered phrase index. If a phrase is not there then 
+# the program will write the kea phrase to the phrase index . 
+#-Search for the kea phrases in the keyphrase to document index. If the phrase is there, 
+# it will increment the number of documents that the keyphrase appears in and replace that
+# then append the hash ID to the list of documents in the entry. If the phrase is not there 
+# then the program will write the kea phrase to the phrase index.
+#-Then write document ID, no of phrase, phrase number from index followed by number of times 
+# phrase appears into the document_keyphrase index
+
+$gsdlhome = $ENV{'GSDLHOME'};
+$collection;
+
+require "getopts.pl";
+&Getopts('R'); #process option arguments
+
+#if option R remove all previous indexes
+if($opt_R == 1){ #remove indexes
+    print STDERR "\nremoving $gsdlhome/bin/script/indexes/keyphrase_index.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/keyphrase_document.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/document_keyphrase.txt\n";
+    system("rm $gsdlhome/bin/script/indexes/keyphrase_document.txt");   
+    system("rm $gsdlhome/bin/script/indexes/document_keyphrase.txt"); 
+    system("rm $gsdlhome/bin/script/indexes/keyphrase_index.txt");
+}
+
+#collections may be specified in the command line
+#otherwise, all collections will be used to build
+#the indexes.  
+if(@ARGV){
+
+    @directories = @ARGV;
+
+} else { #open collect directory and get a list of all collections
+    opendir(DIR, "$gsdlhome/collect");
+    @directories = grep(!/(^\.|(CVS)|(modelcol))/, readdir(DIR));
+    closedir(DIR);
+}
+
+#for each collection specified to build indexes for
+foreach $collection (@directories){
+
+    my @filelist;
+
+    #archives.inf contains a list of unique hash ID's of each file and file paths
+    open(INFO, "$gsdlhome/collect/$collection/archives/archives.inf")
+	or die "$gsdlhome/collect/$collection/archives/archives.inf could not be opened.";
+
+    while(<INFO>){ #get each line of text from archives.inf (OID \t filepath)
+	chomp;
+	push(@filelist, $_);
+    }
+
+    foreach $file (@filelist){ #add each document to the indexes
+	build_index($file, $collection);
+    }
+}
+
+#This function opens the file in the filepath sent as an argument.  From this it obtains 
+#the kea data, and then searches for these phrases in the file, counting and storing 
+#how many times each phrase appears. The data is then sent to function keyphrase_document 
+#with arguments hash ID, kea phrases to build the keyphrase_document index.
+#The function which builds the document_keyphrase index is then passed the hash ID, the kea 
+#phrases and the array/s which hold the number of times each phrase
+#appears in the document so that the data it has collected can be written to document_ keyphrase
+#index.
+
+sub build_index {
+
+    my $args = shift(@_);
+    my $collection = shift(@_);
+    my ($ID, $filepath) = split(/\t/, $args);
+    my $keaS = "";
+    my @kea_phrase_counts = 0;
+    my $text = "";
+
+    print STDERR "\nID: $ID\n";
+    print STDERR "filepath: $filepath\n";
+
+    #open file to extract keyphrase information
+    open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+
+    #patterns to search for so that we can extract the kea information
+    my $kea_search = ".* kea=\"([^\"]*)\"";
+
+    while(<FILE>){ #get kea and stem data and store
+	chomp;
+	$keaS = $1 if (/$kea_search/);
+    }
+
+    close(FILE);
+    
+    print STDERR "Kea: $keaS\n";
+
+    my @kea = split(", ", $keaS);
+
+    if(@kea){ #if the data exists
+	
+	#open the filepath to the current document
+	open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	    or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+	
+	while(<FILE>){ #get the text
+	    chomp;
+	    $text .= $_;
+	}
+ 
+	#chop out all things in angled brackets
+	$text =~ s/(<[^>]*>)//g;
+
+	#initilise counts
+	for($i=0; $i<=$#kea; $i++){
+	    $kea_phrase_counts[$i] = 0;
+	}
+	
+	
+	#using regular expressions generated from kea-reg 
+	#count how many of each phrase appear in the document
+	$text_copy = $text;
+	for($i=0; $i<=$#kea; $i++){ #search for text with kea phrases
+	    my $phrase = $kea[$i];
+	    $reg = &kea_reg(split(/\s+/, $phrase));
+	    while($text_copy =~ s/$reg//i){
+		$kea_phrase_counts[$i]++; #count the number of kea phrases 
+	    } 
+	    $text_copy = $text;
+	}  
+	
+	#write data to keyphrase_document index
+	&keyphrase_document($ID, $keaS); 
+	
+	#write data to document_keyphrase index
+	$kea_counts = join(", ", @kea_phrase_counts);
+	&document_keyphrase($ID, $keaS, $kea_counts);
+	
+    } else { 
+	print STDERR "No kea data was found in file $filepath\n";
+    }
+
+}
+
+
+#returns a regular expression designed to 
+#search for phrases in text
+#eg 'agris caris'
+#    agris followed by 0 or 1 non-whitespace characters OR
+#          followed by one or more whitespace 
+#    caris followed by 0 or 1 non-whitespace characters
+#modified from original by Stephen Lundy 
+
+sub kea_reg {
+    $regexp = "";
+
+    $l = @_;
+
+    if ($l > 0) {
+        $s = shift;
+        #$regexp = "$s(\\S?)";
+	$regexp = "$s(\\s+|\\S?)";
+
+        if ($l-1 > 0) {
+            foreach $s (@_) {
+                $regexp .= "$s(\\s+|\\S?)";
+            }
+        }
+    }
+
+    return $regexp;
+}
+
+
+#This function is passed as arguments a list of kea phrases and/or stems. Its purpose is to 
+#check in the keyphrase index file for each phrase and determine whether or not an entry has
+#been made for that phrase and an index number assigned to it. If there has not been an entry
+#made then an index number is assigned to the phrase and it is written to the file.  This 
+#function is called by document_keyphrase and keyphrase_document.  Each line in the file has
+#this form:
+#-phrase index number:phrase
+#This function then returns a table of pairs of the phrases that were sent as arguments to it
+#{phrase => phrase index number}.
+ 
+sub keyphrase_index_search {
+
+    my $phrases = shift(@_);
+    my @phrases = split(", ", $phrases);
+    my %table;
+    my $index = 1;
+    my $create_new_index = 0;
+
+    print STDERR "searching keyphrase index...\n";
+
+    #initilise table of phrases and index numbers
+    foreach $phrase (@phrases){
+	$table{"$phrase"} = "0";
+    }
+
+    #open keyphrase index for appending data and for reading
+    open(INDEX_OUT, ">>$gsdlhome/bin/script/indexes/keyphrase_index.txt");
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/keyphrase_index.txt")
+	or $create_new_index = 1;
+
+    if($create_new_index == 0){
+	#if the index already exists read in the phrases
+	while(<INDEX_IN>){
+	    chomp;
+	    foreach $phrase (@phrases){
+		if(/(\d+):$phrase/){
+		    $index = $1;
+		    $table{"$phrase"} = "$index";
+		}
+	    }
+	    $index++; #new starting index (one + the last index)
+	}
+
+	close(INDEX_IN);
+
+    }
+
+    #add new phrases to the phrase index
+    foreach $phrase (keys %table){
+	if($table{"$phrase"} eq "0"){
+	    print INDEX_OUT "$index:$phrase\n";
+	    $table{"$phrase"} = "$index"; 
+	    $index++;
+	}
+    }
+
+    close(INDEX_OUT);
+    return %table;
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrases
+#that exist for that particular file.  Its purpose is to write to the keyphrase_document
+#index a line for the document it has been sent:
+#-phrase index number:number of documents it appears in|ID
+sub keyphrase_document{
+
+    my ($ID, $kea) = @_;
+    my $text = ""; 
+    my @textlist;
+    my $create_new_index = 0;
+
+    print STDERR "writing to keyphrase_document.txt...\n";
+
+    #get table of phrases and phrase indexes 
+    my %table = keyphrase_index_search($kea); 
+    
+  
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/keyphrase_document.txt")
+	or $create_new_index = 1;
+
+    #read in document if file exists
+    if($create_new_index == 0){
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    } 
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/keyphrase_document.txt");
+
+    if($create_new_index == 0){ #amend existing index
+	
+	foreach $line (@textlist){
+	    foreach $phrase (keys %table){
+		if($line =~ /(\d+):(\d+)(.*)/){ #all lines of this form
+		    $index = $1;
+		    if($table{"$phrase"} eq "$index") { #if phrase exists in index
+			$ids = $3; #get all doc IDs for that keyphrase
+			if($ids !~ /$ID/){ #if doc ID not already included
+			    $num_docs = $2;
+			    $num_docs++; #increment number of docs
+			    $line = "$index:$num_docs$3|$ID"; #line to append to index
+			    $table{"$phrase"} = "0"; 
+			}
+		    }
+		}
+	    }
+	    print INDEX_OUT "$line\n";
+	}
+    }
+
+    #add new phrases to the index
+    foreach $phrase (keys %table){ #write 'phrase index:1:file ID
+	if($table{"$phrase"} ne "0"){
+	    my $line = "$table{$phrase}:1:$ID";
+	    print INDEX_OUT "$line\n"; 
+	}
+    }
+
+    close(INDEX_OUT);
+
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrases and/or stems
+#that exist for that particular file and a list of the number of times each kea and/or stem
+#phrase appear in that document.  Its purpose is to write to the document_keyphrase
+#index a line for the document it has been sent:
+#-file ID:number of phrases and/or stems appear in the document
+#      |pairs of 'phrase index,number of times the phrase appears in the document' 
+sub document_keyphrase {
+
+    my ($ID, $keaS, $kea_c) = @_;
+    my $text = "";
+    my @textlist;
+    my %phrases;
+    my $create_new_index = 0;
+
+    print STDERR "writing to document_keyphrase.txt...\n";
+
+    #split phrase counts into arrays
+    my @kea_counts = split(", ", $kea_c);
+  
+    #get table of phrases and phrase indexes
+    my %table = keyphrase_index_search($keaS);
+
+    #split phrases into arrays
+    my @kea = split(", ", $keaS);
+
+    #build new phrases dictionary
+    for($i=0; $i<=$#kea; $i++){
+	my $phrase = $table{"$kea[$i]"};
+	$phrases{"$phrase"} = "$kea_counts[$i]";
+    } 
+   
+    my @num = keys %phrases;
+    my $phrasenum = $#num + 1; #number of phrases in doc
+
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/document_keyphrase.txt")
+	or $create_new_index = 1;
+
+     
+    if($create_new_index == 0){ #index doesn't need to be created
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+	
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    }  
+
+
+    #must write this line to the file
+    #'document ID:num of phrases|phrase index, number of times phrases appears
+    my $newline = "$ID:$phrasenum";
+    foreach $phrase (keys %phrases){
+	$newline .= "|$phrase,$phrases{$phrase}";
+    }
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/document_keyphrase.txt");
+
+    if($create_new_index == 1){ #create a new index
+
+	print INDEX_OUT "$newline\n";
+
+    } else {
+
+	#if ID is already in the file write line overtop incase
+	#someone has modified the file.  Otherwise add the line
+	#to the end of the file
+	my $found = 0;
+
+	foreach $line (@textlist){
+	    if($line =~ /([^:]+):(.*)/){ #all lines should follow this pattern
+		$id = $1;
+		if($ID eq $id){ #id is already in the file
+		    print INDEX_OUT "$newline\n"; #print line overtop
+		    $found = 1;
+		} else {
+		    print INDEX_OUT "$line\n"; #print old line out
+		}
+	    }
+	}
+
+	print INDEX_OUT "$newline\n" if ($found == 0); #append new line to end of file 
+
+    }
+
+    close(INDEX_OUT);
+
+}
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/buildkpiS.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/buildkpiS.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/buildkpiS.pl	(revision 21112)
@@ -0,0 +1,420 @@
+#! /user/bin/perl 
+
+#usage: perl buildkpiS.pl [-R] [collection] [collection] etc
+#
+#-r or -R will remove previous index files so that you may build new ones
+#
+#The program performs the following tasks:
+#-gathers the specified collections on the command line OR
+#-gathers the directories of all the collections in the collect directory, this is all
+# the directories apart from CVS, modelcol, . and .. which are not collections.
+#-It then retrieves the archive.inf file from the archive directory of each collection
+# to obtain the unique file ID and filepath of every document in the collection
+#-Then parse through each doc.gml stored in filepath to gather information
+#-From each file collect stems of kea keyphrases.
+#-Determine the number of keyphrase stems for each document 
+#-Search for the keyphrase stems in the numbered phrase index. If a phrase is not there then 
+# the program will write the keyphrase stem to the phrase index . 
+#-Search for the keyphrase stem in the keyphrase to document index. If the stem is there, 
+# it will increment the number of documents that the keyphrase stem appears in and replace that
+# then append the hash ID to the list of documents in the entry. If the stem is not there 
+# then the program will write the stem to the phrase index.
+#-Then write document ID, no of phrase, phrase number from index followed by number of times 
+# phrase appears into the document_keyphrase index
+
+$gsdlhome = $ENV{'GSDLHOME'};
+$collection;
+
+require "getopts.pl";
+&Getopts('R'); #process option arguments
+
+#if option R remove all previous indexes
+if($opt_R == 1){ #remove indexes
+    print STDERR "\nremoving $gsdlhome/bin/script/indexes/keyphrase_index.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/keyphrase_document.txt\n";
+    print STDERR "removing $gsdlhome/bin/script/indexes/document_keyphrase.txt\n";
+    system("rm $gsdlhome/bin/script/indexes/keyphrase_document.txt");   
+    system("rm $gsdlhome/bin/script/indexes/document_keyphrase.txt"); 
+    system("rm $gsdlhome/bin/script/indexes/keyphrase_index.txt");
+}
+
+#collections may be specified in the command line
+#otherwise, all collections will be used to build
+#the indexes.  
+if(@ARGV){
+
+    @directories = @ARGV;
+
+} else { #open collect directory and get a list of all collections
+    opendir(DIR, "$gsdlhome/collect");
+    @directories = grep(!/(^\.|(CVS)|(modelcol))/, readdir(DIR));
+    closedir(DIR);
+}
+
+#for each collection specified to build indexes for
+foreach $collection (@directories){
+
+    my @filelist;
+
+    #archives.inf contains a list of unique hash ID's of each file and file paths
+    open(INFO, "$gsdlhome/collect/$collection/archives/archives.inf")
+	or die "$gsdlhome/collect/$collection/archives/archives.inf could not be opened.";
+
+    while(<INFO>){ #get each line of text from archives.inf (OID \t filepath)
+	chomp;
+	push(@filelist, $_);
+    }
+
+    foreach $file (@filelist){ #add each document to the indexes
+	build_index($file, $collection);
+    }
+}
+
+#This function opens the file in the filepath sent as an argument.  From this it obtains 
+#the kea and/or stem data, and then searches for these phrases in the file, counting and storing 
+#how many times each phrase appears. The data is then sent to function keyphrase_document 
+#with arguments hash ID, kea phrases and stem phrases to build the keyphrase_document index.
+#The function which builds the document_keyphrase index is then passed the hash ID, the kea 
+#phrases and/or the stemmed phrases and the array/s which hold the number of times each phrase
+#appears in the document so that the data it has collected can be written to document_ keyphrase
+#index.
+
+sub build_index {
+
+    my $args = shift(@_);
+    my $collection = shift(@_);
+    my ($ID, $filepath) = split(/\t/, $args);
+    my $stemsS = "";
+    my @stem_phrase_counts = 0;
+    my $text = "";
+
+    print STDERR "\nID: $ID\n";
+    print STDERR "filepath: $filepath\n";
+
+    #open file to extract keyphrase information
+    open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+
+    #patterns to search for so that we can extract the kea information
+    my $stem_search = "stems=\"([^\"]*)\"";
+
+    while(<FILE>){ #get kea and stem data and store
+	chomp;
+	$stemsS = $1 if (/$stem_search/);
+    }
+
+    close(FILE);
+    
+    print STDERR "stems: $stemsS\n";
+
+    my @stems = split(", ", $stemsS);
+
+    if(@stems){ #if the data exists
+	
+	#open the filepath to the current document
+	open(FILE, "$gsdlhome/collect/$collection/archives/$filepath")
+	    or die "$gsdlhome/collect/$collection/archives/$filepath could not be opened.";
+	
+	while(<FILE>){ #get the text
+	    chomp;
+	    $text .= $_;
+	}
+ 
+	#chop out all things in angled brackets
+	$text =~ s/(<[^>]*>)//g;
+
+	#initilise counts
+	for($i=0; $i<=$#stems; $i++){
+	    $stem_phrase_counts[$i] = 0;
+	}
+	
+	#using regular expressions generated from stem-reg
+	#count how many of each phrase appear in the document
+	
+	$text_copy = $text;
+	for($i=0; $i<=$#stems; $i++){ #search for text with stem phrases
+	    my $stem = $stems[$i];
+	    $reg = &stem_reg(split(/\s+/, $stem));
+	    while($text_copy =~ s/$reg//i){
+		$stem_phrase_counts[$i]++; #count the number of stem phrases
+	    } 
+	    $text_copy = $text;
+	}
+
+	
+	#write data to keyphrase_document index
+	&keyphrase_document($ID, $stemsS); 
+	
+	#write data to document_keyphrase index
+	$stem_counts = join(", ", @stem_phrase_counts);
+	&document_keyphrase($ID, $stemsS, $stem_counts);
+	
+    } else { 
+	print STDERR "No stem data was found in file $filepath\n";
+    }
+
+}
+
+#returns a regular expression designed to 
+#search for stems in text
+#eg 'agri cari'
+#    agri followed by 0 or more non-whitespace characters
+#         followed by one or more whitespace OR 0 or 1 non-whitespace characters
+#    cari followed by 0 or more non-whitespace characters
+#modified from original by Stephen Lundy 
+
+sub stem_reg {
+   
+    $regexp = "";
+
+    $l = @_;
+
+    if ($l > 0) {
+        $s = shift;
+        $regexp = "$s\\S*";
+
+        if ($l-1 > 0) {
+            foreach $s (@_) {
+                $regexp .= "(\\s+|\\S?)$s\\S*";
+            }
+        }
+    }
+
+    return $regexp;
+}
+
+
+
+#This function is passed as arguments a list of kea phrase stems. Its purpose is to 
+#check in the keyphrase index file for each phrase and determine whether or not an entry has
+#been made for that phrase and an index number assigned to it. If there has not been an entry
+#made then an index number is assigned to the phrase and it is written to the file.  This 
+#function is called by document_keyphrase and keyphrase_document.  Each line in the file has
+#this form:
+#-phrase index number:phrase
+#This function then returns a table of pairs of the phrases that were sent as arguments to it
+#{phrase => phrase index number}.
+ 
+sub keyphrase_index_search {
+
+    my $phrases = shift(@_);
+    my @phrases = split(", ", $phrases);
+    my %table;
+    my $index = 1;
+    my $create_new_index = 0;
+
+    print STDERR "searching keyphrase index...\n";
+
+    #initilise table of phrases and index numbers
+    foreach $phrase (@phrases){
+	$table{"$phrase"} = "0";
+    }
+
+    #open keyphrase index for appending data and for reading
+    open(INDEX_OUT, ">>$gsdlhome/bin/script/indexes/keyphrase_index.txt");
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/keyphrase_index.txt")
+	or $create_new_index = 1;
+
+    if($create_new_index == 0){
+	#if the index already exists read in the phrases
+	while(<INDEX_IN>){
+	    chomp;
+	    foreach $phrase (@phrases){
+		if(/(\d+):$phrase/){
+		    $index = $1;
+		    $table{"$phrase"} = "$index";
+		}
+	    }
+	    $index++; #new starting index (one + the last index)
+	}
+
+	close(INDEX_IN);
+
+    }
+
+    #add new phrases to the phrase index
+    foreach $phrase (keys %table){
+	if($table{"$phrase"} eq "0"){
+	    print INDEX_OUT "$index:$phrase\n";
+	    $table{"$phrase"} = "$index"; 
+	    $index++;
+	}
+    }
+
+    close(INDEX_OUT);
+    return %table;
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrases and/or stems
+#that exist for that particular file.  Its purpose is to write to the keyphrase_document
+#index a line for the document it has been sent:
+#-phrase index number:number of documents it appears in|ID
+sub keyphrase_document{
+
+    my ($ID, $stems) = @_;
+    my $text = ""; 
+    my @textlist;
+    my $create_new_index = 0;
+
+    print STDERR "writing to keyphrase_document.txt...\n";
+
+    #get table of phrases and phrase indexes 
+    my %table = keyphrase_index_search($stems); 
+    
+  
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/keyphrase_document.txt")
+	or $create_new_index = 1;
+
+    #read in document if file exists
+    if($create_new_index == 0){
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    } 
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/keyphrase_document.txt");
+
+    if($create_new_index == 0){ #amend existing index
+	
+	foreach $line (@textlist){
+	    foreach $phrase (keys %table){
+		if($line =~ /(\d+):(\d+)(.*)/){ #all lines of this form
+		    $index = $1;
+		    if($table{"$phrase"} eq "$index") { #if phrase exists in index
+			$ids = $3; #get all doc IDs for that keyphrase
+			if($ids !~ /$ID/){ #if doc ID not already included
+			    $num_docs = $2;
+			    $num_docs++; #increment number of docs
+			    $line = "$index:$num_docs$3|$ID"; #line to append to index
+			    $table{"$phrase"} = "0"; 
+			}
+		    }
+		}
+	    }
+	    print INDEX_OUT "$line\n";
+	}
+    }
+
+    #add new phrases to the index
+    foreach $phrase (keys %table){ #write 'phrase index:1:file ID
+	if($table{"$phrase"} ne "0"){
+	    my $line = "$table{$phrase}:1:$ID";
+	    print INDEX_OUT "$line\n"; 
+	}
+    }
+
+    close(INDEX_OUT);
+
+}
+
+#This function is passed as arguments file hash ID and a list of kea phrase stems
+#that exist for that particular file and a list of the number of times each stem
+#phrase appear in that document.  Its purpose is to write to the document_keyphrase
+#index a line for the document it has been sent:
+#-file ID:number of phrases and/or stems appear in the document
+#      |pairs of 'phrase index,number of times the phrase appears in the document' 
+sub document_keyphrase {
+
+    my ($ID, $stemsS, $stem_c) = @_;
+    my $text = "";
+    my @textlist;
+    my %phrases;
+    my $create_new_index = 0;
+
+    print STDERR "writing to document_keyphrase.txt...\n";
+
+    #split phrase counts into arrays
+    my @stem_counts = split(", ", $stem_c);
+  
+    #get table of phrases and phrase indexes
+    my %table = keyphrase_index_search($stemsS);
+
+    #split phrases into arrays
+    my @stems = split(", ", $stemsS);
+
+    #build new phrases dictionary
+    for($i=0; $i<=$#stems; $i++){
+	my $phrase = $table{"$stems[$i]"};
+	$phrases{"$phrase"} = "$stem_counts[$i]";
+    }   
+    my @num = keys %phrases;
+    my $phrasenum = $#num + 1; #number of phrases in doc
+
+    #open index for reading
+    open(INDEX_IN, "$gsdlhome/bin/script/indexes/document_keyphrase.txt")
+	or $create_new_index = 1;
+
+     
+    if($create_new_index == 0){ #index doesn't need to be created
+
+	while(<INDEX_IN>){
+	    $text .= $_;
+	}
+
+	close(INDEX_IN);
+	
+	#split text into lines
+	@textlist = split(/\n/, $text);
+
+    }  
+
+
+    #must write this line to the file
+    #'document ID:num of phrases|phrase index, number of times phrases appears
+    my $newline = "$ID:$phrasenum";
+    foreach $phrase (keys %phrases){
+	$newline .= "|$phrase,$phrases{$phrase}";
+    }
+
+    #open index for output
+    open(INDEX_OUT, ">$gsdlhome/bin/script/indexes/document_keyphrase.txt");
+
+    if($create_new_index == 1){ #create a new index
+
+	print INDEX_OUT "$newline\n";
+
+    } else {
+
+	#if ID is already in the file write line overtop incase
+	#someone has modified the file.  Otherwise add the line
+	#to the end of the file
+	my $found = 0;
+
+	foreach $line (@textlist){
+	    if($line =~ /([^:]+):(.*)/){ #all lines should follow this pattern
+		$id = $1;
+		if($ID eq $id){ #id is already in the file
+		    print INDEX_OUT "$newline\n"; #print line overtop
+		    $found = 1;
+		} else {
+		    print INDEX_OUT "$line\n"; #print old line out
+		}
+	    }
+	}
+
+	print INDEX_OUT "$newline\n" if ($found == 0); #append new line to end of file 
+
+    }
+
+    close(INDEX_OUT);
+
+}
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/indexes/relation.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/indexes/relation.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/indexes/relation.pl	(revision 21112)
@@ -0,0 +1,414 @@
+#!/usr/bin/perl
+
+#data structures used:
+#-document_keyphrase
+#%document_phrases = ("hashID" => {"phrase" => "number of occurrences"})
+#-keyphrase_document 
+#%phrases_document = ("phrase" => "no. of docs phrase occurs in")
+
+#after calculation relational data, it will amend gml files
+#to include this data while writing to a file ID_relatedlinks.txt
+#the html links of the related files
+#if option p is set then the user wishes to add the related documents
+#as bookmarks to the pdf file
+
+require "getopts.pl";
+&Getopts('N:P'); #process option arguments
+
+
+$gsdlhome = $ENV{'GSDLHOME'};
+@directories; #list of directories for the collections
+%phrases_document; #hash storing phrase index => no. of documents with phrase in it
+%document_phrases; #hash of hashes ID => {phrase number => no. of phrases in ID}
+$N = 0; #number of documents in the index (presumably in the collection)
+%cosine_matrix; #2-dimensional matrix, for each pair of documents stores a cosine measure
+$number_of_files = 1; #by default the top related file is returned
+$do_pdfs = 0;
+
+if($opt_N =~ /(\d)+/){ #if flag is set then return top N docs
+    $number_of_files = $opt_N;
+}
+
+#get server name and httpprefix
+$httpprefix;
+$servername = "nzdl2.cs.waikato.ac.nz"; #should be localhost
+
+if($opt_P == 1){ #if flag is set 
+    $do_pdfs = 1;
+    
+    #get the httpprefix from gsdlsite.cfg
+    open(CFG, "$gsdlhome/cgi-bin/gsdlsite.cfg") 
+	or die "$gsdlhome/cgi-bin/gsdlsite.cfg could not be opened";
+    
+    while(<CFG>){
+	chomp;
+	if(/httpprefix \/(.+)/){
+	    $httpprefix = $1; 
+	}
+    }
+    close CFG;
+}
+
+if(@ARGV){
+
+    @directories = @ARGV;
+
+} else { #open collect directory and get a list of all collections
+    opendir(DIR, "$gsdlhome/collect");
+    @directories = grep(!/(^\.|(CVS)|(modelcol))/, readdir(DIR));
+    closedir(DIR);
+}
+
+
+
+#get the name of the directory the index will be stored in
+$dirname = "";
+foreach $collection (sort @directories){
+    $dirname .= $collection."_";
+}
+$dirname .= "indexes";
+print STDERR "directory name: $dirname\n";
+
+#open keyphrases_document.txt
+open(KEY_FILE, "$gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt") 
+    or die "$gsdlhome/bin/script/indexes/$dirname/keyphrase_document.txt could not be opened";
+
+print STDERR "\nreading keyphrase_document.txt...\n";
+
+#read a list of phrase indexes and number of documents that phrase appears in 
+while(<KEY_FILE>){
+    chomp;
+    if(/(\d+)\:(\d+)(.*)/){ #all lines should be of this form
+	my $phrase_index = $1;
+	my $doc_num = $2;
+	$phrases_document{"$phrase_index"} = "$doc_num"; #build keyphrase hash
+    }
+}
+
+close(KEY_FILE); 
+
+print STDERR "\nreading document_keyphrase.txt...\n";
+
+#open document_keyphrases.txt
+open(DOC_FILE, "$gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt") 
+    or die "$gsdlhome/bin/script/indexes/$dirname/document_keyphrase.txt could not be opened";
+
+#read a list of documents and the phrases that appear in them (& how many times they appear)
+while(<DOC_FILE>){
+    chomp;
+    my %document;
+    if(/(\w+)\:(\d+)((\|\d+,\d+)+)/){ #every line in the index of this form
+	my $ID = $1;
+	my $pairs = $3;
+	while ($pairs =~ s/\|(\d+),(\d+)//) { #this is the list of keyphrases and numbers
+	    my $phrase_index = $1;
+	    my $num_phrase = $2;
+	    $document{"$phrase_index"} = "$num_phrase"; #table to be stored in another table
+	}
+	$document_phrases{"$ID"} = ({%document});	  
+	$N++; #number of documents in the collection (which have kea metadata)
+    }
+}
+
+close(DOC_FILE);
+ 
+#we do not want to recalculate measures ie if we have already calculated
+#doc1 & doc2 we do not then want to calculate doc2 & doc1
+#to achieve this we must first sort the list of ids so that both lists
+#of documents will be in the same order (otherwise hash tables return values in 
+#no particular order). We then compare each document in the left 
+#column below against the documents in the right column:
+
+#doc0 -> 
+#doc1 -> doc0 
+#doc2 -> doc0, doc 1 
+#doc3 -> doc0, doc 1, doc 2 
+#doc4 -> doc0, doc 1, doc 2, doc 3 
+#doc5 -> doc0, doc 1, doc 2, doc 3, doc 4 
+#etc
+
+#this way we are not recalculating the same value and also not
+#comparing each document to itself (which would result in the
+#value 1) thus more than halving the processing time.
+
+print STDERR "\ncalculating cosine measures for $N documents...\n";
+
+$count = 0;
+@ids = sort keys(%document_phrases);
+
+#repeat for all pairs of documents
+foreach $id1 (sort keys(%document_phrases)) { 
+    my %table;
+   
+    for($i=0; $i<$count; $i++){ #calculate the cosine measure and store
+	my $cosine_measure = &compare_docs($id1, $ids[$i]); 
+	$table{"$ids[$i]"} = "$cosine_measure";  	
+    }
+    $cosine_matrix{"$id1"} = ({%table});
+    $count++; #each iteration the number of documents to compare against expands by 1
+}
+
+&addtogml(); #add the relational data we have just gathered to the documents gml files
+
+   
+#this function takes as an argument two document IDs and uses these to extract the
+#keyphrases from the document_keyphrase data structure.  It then calculates a 
+#'measure of relativity' [0-1] with which we can establish how similar the two
+#documents are to each other. (0-not related, 1-same document).  The equation 
+#used to establish this similarity:
+#
+#                     for all phrases 
+#                     in both d1, d2 (fd1p * loge(N/fp) (fd2p * loge(N/fp) 
+#  cosine(d1, d2) = --------------------------------------------------------------
+#                     sqrt(sum of phrases in d1(fd1p * loge(N/fp))) * 
+#                     sqrt(sum of phrases in d2(fd2p * loge(N/fp))) 
+#
+#where d1 and d2 are lists of keyphrase and represent documents
+#fd1p is the frequency that phrase p occurs in document d1
+#fd2p is the frequency that phrase p occurs in document d2
+#fp is the number of documents that have p as a keyphrase
+#N is the number of documents in the collection 
+sub compare_docs {
+
+  
+    my $ID1 = shift(@_); #ID for document one 
+    my $ID2 = shift(@_); #ID for document two
+  
+    my @phrases; #a list of phrases in document 1 and document 2
+    my @phrases1; #a list of phrases in document 1
+    my @phrases2; #a list of phrases in document 2
+    
+    
+    foreach $phrase (keys %{ $document_phrases{$ID1}}) { #list of phrases in doc1
+	push(@phrases1, $phrase);
+    } 
+    foreach $phrase (keys %{ $document_phrases{$ID2}}) { #list of phrases in doc2
+	push(@phrases2, $phrase);
+    }
+    foreach $phrase1 (@phrases1) { #list holds intersection of doc1 and doc2
+	foreach $phrase2 (@phrases2) {
+	    push(@phrases, $phrase1) if ($phrase1 == $phrase2);
+	}
+    }
+ 
+#COSINE MEASURE
+    my $wqtwdt= 0; 
+   
+    foreach $phrase (@phrases){ #for all phrases ocurring in d1 and d2
+	#the frequency that phrase occurs in document d1
+	$fd1p = $document_phrases{$ID1}{$phrase}; 
+	
+	#log base e(N/the number of documents that have phrase as a keyphrase)
+	$log_freq = log($N/$phrases_document{$phrase});
+	
+	#the frequency that phrase occurs in document d2
+	$fd2p = $document_phrases{$ID2}{$phrase};
+	$sum = ($fd1p * $log_freq) * ($fd2p * $log_freq);
+	$wqtwdt += $sum;
+    }
+  
+   
+    my $wd = 0; #stores the calculation for wd
+
+    foreach $phrase (@phrases1){ #for all phrases ocurring in d1 and d2
+	#the frequency that phrase occurs in document d1
+	$fd1p = $document_phrases{$ID1}{$phrase};
+	
+	#log base e($N/the number of documents that have phrase as a keyphrase)
+	$log_freq = log($N/$phrases_document{$phrase});
+	$sum = $fd1p * $log_freq;
+	$wd += ($sum * $sum); #sum squared
+    }
+
+   
+    my $wq = 0; #stores the calculation for wq
+ 
+    foreach $phrase (@phrases2){ #for all phrases ocurring in d2
+	#the frequency that phrase occurs in document d2
+	$fd2p = $document_phrases{$ID2}{$phrase};
+
+	#log base e($N/the number of documents that have phrase as a keyphrase)
+	$log_freq = log($N/$phrases_document{$phrase});
+	$sum = $fd2p * $log_freq;
+	$wq += ($sum * $sum); #sum squared 
+    }
+   
+    my $wdwq = sqrt($wq)  * sqrt($wd);  
+    my $cosine = $wqtwdt / $wdwq;
+    
+    return $cosine;
+	
+}
+   
+#this function adds the relational data we have collected to the gml files
+sub addtogml {
+
+ print STDERR "\nadding relational data to $N gml documents...\n\n";
+ my @doclist;
+ my %filetable;
+ my $pattern = "kea="; #pattern to search for to find where to insert relation data
+
+ #open archive info for each collecticollection
+ foreach $collection (@directories){
+
+     open(INFO, "$gsdlhome/collect/$collection/archives/archives.inf") 
+	 or die "$gsdlhome/collect/$collection/archives/archives.inf could not be opened";
+     
+     #read a list of ID's and pathnames into a file table
+     while(<INFO>){
+	 chomp;
+	 my %idtable;
+	 my $ID;
+	 if(/(\w+)(\s+)([\w\.\/]+)/){ #format of the line
+	     $ID = $1;
+	     $path = $3;
+	     $idtable{"$path"} = "$collection";
+	 }
+	 $filetable{"$ID"} = ({%idtable});
+     } 
+  
+     close(INFO);
+ }
+
+ 
+
+ #for each id in the matrix calculate a list of related documents
+ foreach $id (keys %cosine_matrix){    
+    
+     @doclist = &calculate_list($id); #gets list of documents with most relevant scores
+
+     my @path = keys %{$filetable{$id}}; #get filepath for document
+     my $collection = $filetable{$id}{$path[0]};
+     
+     #open gml file to amend for each collection
+     open(FILE, "$gsdlhome/collect/$collection/archives/$path[0]") 
+	 or die "$gsdlhome/collect/$collection/archives/$path[0] could not be opened";
+     
+     #read the gml file into text
+     $text = ""; 
+     while(<FILE>){
+	 $text .= $_;
+     }
+     
+     close(FILE);
+     
+     #delete previous relational data and urllinks to files storing rel doc links
+     $text =~ s/(\s)+relation=\"([^\"]*)\"//g;
+     $text =~ s/(\s)+urllink=\"([^\"]*)\"//g;
+     $text =~ s/(\s)+\/urllink=\"([^\"]*)\"//g;
+     
+     #if we want to insert each relation item as 'relation1="id" relation2="id"'
+     #$count = 1; #insert each item in the list
+     #foreach $item (@doclist){ #insert list into text
+     #	 $text =~ s/(\s)($pattern)/ relation$count=\"$item\" $2/g;
+     #	 $count++;
+     #} 
+     
+     #or relation="id, id, id"
+     #my $relation = join(",", @doclist);
+
+     #if pdf flag is set then delete pdf url document
+     if($do_pdfs == 1){ 
+	 my $dirpath = $path[0];
+	 $dirpath =~ s/(^(\/doc\.gml))*\/doc\.(gml)/$1/;
+ 	 `rm $gsdlhome/collect/$collection/index/assoc/$dirpath/url.txt`; 
+     }
+
+     my $title = "related document";
+     $title = $2 if($text =~ /(\s)+Title=\"([^\"]*)\"/g);
+   
+     #relation="collection,id collection,id"
+     my @relationlist;
+     my $relation = "";
+     foreach $doc (@doclist){
+	 my @p = keys %{$filetable{$doc}};
+	 my $collect = $filetable{$doc}{$p[0]};
+	 push(@relationlist, "$collect,$doc");
+
+	 #write pdf docs to file
+	 if($do_pdfs == 1){
+	     my @rel_path = keys %{$filetable{$doc}};
+	     &write_related_urls($title, $collection, $path[0], $collect, $rel_path[0]);
+	 }
+     }
+  
+     #modify the text to include relation data
+     $relation = join(" ", @relationlist); 
+     $text =~ s/($pattern)/ relation=\"$relation\" $1/g;
+
+     #open gml file to write back new text
+     open(FILE, ">$gsdlhome/collect/$collection/archives/$path[0]") or print STDERR "NO!!!!\n";
+     print FILE "$text"; #write back text
+     close(FILE);
+
+     #amend the pdf file 
+     if($do_pdfs == 1){
+	 my $pdf_path = $path[0];
+	 $pdf_path =~ s/(^(\/doc\.gml))*\/doc\.(gml)/$1/;
+	 `perl amend_pdf.pl $gsdlhome/collect/$collection/index/assoc/$pdf_path/doc.pdf $gsdlhome/collect/$collection/index/assoc/$pdf_path/url.txt\n`;
+     }
+ } 
+}
+
+#this function builds a list of cosine measures for each
+#document and sorts the list of ids each measure belongs to
+#in reverse order (ie docs with greatest cosine measure first)
+#returns the list calculated
+sub calculate_list {
+   
+    my $document = shift(@_);
+    my %measures;
+    my @doclist;
+    
+    
+    #find the top $number_of_files for $document  
+    foreach $id (keys %{$cosine_matrix{$document}}){ 
+	    $measures{"$cosine_matrix{$document}{$id}"} = "$id";
+    }
+    
+    my @list = reverse sort {$a<=>$b} keys %measures;
+
+    #list is as big as specified in command line (default is 1)
+    for($i = 0; $i<$number_of_files; $i++){ 
+	my $id = $measures{$list[$i]};
+	push(@doclist, $id) if($id ne "");
+    }
+
+    return @doclist;
+}
+
+
+sub write_related_urls {
+
+    
+ #need server name eg nzdl2.cs.waikato.ac.nz
+ #localhost should work but doesn't on this computer?
+ #open up config file to read httpprefix
+ #get collection name of related document
+ #get hash directory of related document
+ #write http://nzdl2.cs.waikato.ac.nz/httpprefix/collectionname/index/assoc/directory/doc.pdf
+
+ my ($title, $collection, $path, $collect, $rel_path) = @_;
+
+ $rel_path =~ s/(^(\/doc\.gml))*\/doc\.(gml)/$1/;
+ $path =~ s/(^(\/doc\.gml))*\/doc\.(gml)/$1/;
+ 
+ print STDERR "writing related pdf urls to file ";
+ print STDERR "$gsdlhome/collect/$collection/index/assoc/$path/url.txt...\n";
+ 
+ open(URL, ">>$gsdlhome/collect/$collection/index/assoc/$path/url.txt") 
+     or open(URL, ">$gsdlhome/collect/$collection/index/assoc/$path/url.txt")
+	 or print STDERR "This file $title is not a pdf file\n";
+
+ print URL "$title\t"; 
+ print URL "http://$servername/$httpprefix/collect/$collect/index/assoc/$rel_path/doc.pdf\n";
+ close URL;
+
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/lucene_passes.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/lucene_passes.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/lucene_passes.pl	(revision 21112)
@@ -0,0 +1,210 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# lucene_passes.pl -- perl wrapper, akin to mgpp_passes, for Lucene
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use util;
+use ghtml;
+
+
+sub open_java_lucene
+{
+    my ($doc_tag_level,$full_builddir,$indexdir,$create,$verbosity) = @_;
+
+    my $bin_java = &util::filename_cat($ENV{'GSDLHOME'},"bin","java");
+    my $classpath = &util::filename_cat($bin_java,"LuceneWrap.jar");
+
+    my $java_lucene = "java -classpath \"$classpath\" GS2LuceneIndexer";
+    my $cmd_options = "$create -verbosity $verbosity";
+    my $java_cmd = "$java_lucene $cmd_options $doc_tag_level \"$full_builddir\" $indexdir";
+
+    if (!open (PIPEOUT, "| $java_cmd")) {
+	die "$PROGNAME - couldn't run $java_cmd\n";
+    }
+}
+
+sub close_java_lucene
+{
+    close(PIPEOUT);
+}
+
+sub save_xml_doc
+{
+    my ($full_textdir,$output_filename,$doc_xml) = @_;
+    $dir_sep = &util::get_os_dirsep();
+
+    my $full_output_filename 
+	= &util::filename_cat($full_textdir,$output_filename);
+    my ($full_output_dir) = ($full_output_filename =~ m/^(.*$dir_sep)/x);
+    &util::mk_all_dir($full_output_dir);
+    
+    open(DOCOUT,">$full_output_filename") 
+	|| die "Unable to open $full_output_filename";
+
+    print DOCOUT $doc_xml;
+    close(DOCOUT);
+
+    my @secs =  ($doc_xml =~ m/<Sec\s+gs2:id="\d+"\s*>.*?<\/Sec>/sg);
+
+
+# Currently not used, but consult with DB before removing
+#    foreach my $sec (@secs) {
+#	my ($docnum,$sec_text) = ($sec =~ m/<Sec\s+gs2:id="(\d+)"\s*>(.*?)<\/Sec>/s);
+#	my $docnum_filename 
+#	    = &util::filename_cat($full_textdir,"$docnum.xml");
+
+#    
+#	open(SECOUT,">$docnum_filename") 
+#	    || die "Unable to open $docnum_filename";
+
+#	print SECOUT &ghtml::unescape_html($sec_text);
+#	close(SECOUT);
+#    }
+
+}
+
+sub compress_xml_doc
+{
+    my ($full_textdir,$output_filename) = @_;
+
+    my $full_output_filename 
+	= &util::filename_cat($full_textdir,$output_filename);
+
+    `gzip $full_output_filename`;
+}
+
+sub monitor_xml_stream
+{
+    my ($mode, $full_textdir) = @_;
+
+    my $doc_xml = "";
+    my $output_filename = "";
+
+    my $line;
+    while (defined ($line = <STDIN>)) {
+	$line =~ s/&nbsp;/&amp;nbsp;/g;
+	$doc_xml .= $line;
+	if ($line =~ m/^<Doc.+file=\"(.*?)\".*>$/) {
+	    $output_filename = $1;
+	    
+	}
+	
+	if ($line =~ m/^<\/Doc>$/) {
+	    if ($mode eq "text") {
+		save_xml_doc($full_textdir,$output_filename,$doc_xml);
+	    } elsif ($mode eq "index") {
+		# notify lucene indexer
+		# print STDERR $doc_xml;
+##	    print PIPEOUT "$output_filename\n";
+		print PIPEOUT "$doc_xml";
+		#save_xml_doc($full_textdir, "$output_filename.txt", $doc_xml);
+	    }
+	    # compress file
+###	    compress_xml_doc($full_textdir,$output_filename);
+
+	    $doc_xml = "";
+	    $output_filename = "";
+	}
+    }
+}
+
+sub main
+{
+    my (@argv) = @_;
+    my $argc = scalar(@argv);
+
+    my $create = "";
+    my $verbosity = 1;
+
+    my @filtered_argv = ();
+
+    my $i = 0;
+    while ($i<$argc) {
+	if ($argv[$i] =~ m/^-(.*)$/) {
+
+	    my $option = $1;
+
+	    # -create causes build to be incremental
+	    if ($option eq ("create")) {
+		$create = "-create";
+	    }
+
+	    # -verbosity num
+	    elsif ($option eq "verbosity") {
+		$i++;
+		if ($i<$argc) {
+		    $verbosity = $argv[$i];
+		}
+	    }
+	    else {
+		print STDERR "Unrecognised minus option: -$option\n";
+	    }
+	}
+	else {
+	    push(@filtered_argv,$argv[$i]);
+	}
+	$i++;
+    }
+
+    my $filtered_argc = scalar(@filtered_argv);
+
+    if ($filtered_argc < 4) {
+	print STDERR "Usage: $PROGNAME [-create|-verbosity num] \"text\"|\"index\" doc-tag-level build-dir index-name\n";
+	exit 1;
+    }
+
+    my $mode = $filtered_argv[0];
+    my $doc_tag_level = $filtered_argv[1];
+    my $full_builddir = $filtered_argv[2];
+    my $indexdir      = $filtered_argv[3];
+###    print STDERR "**** ARGS = ", join(" ", @argv), "\n";
+
+    my $full_textdir = &util::filename_cat($full_builddir,"text");
+
+    if ($mode eq "index") {
+	# don't need the lucene stuff if we are just storing the docs
+	open_java_lucene($doc_tag_level,$full_builddir,$indexdir,$create,$verbosity);
+    }
+    monitor_xml_stream($mode, $full_textdir);
+    if ($mode eq "index") {
+	close_java_lucene();
+    }
+}
+
+$PROGNAME = $0;
+$PROGNAME =~ s/^.*\/(.*)$/$1/;
+
+&main(@ARGV);
+
Index: /main/tags/3.01/gsdl/bin/script/lucene_query.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/lucene_query.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/lucene_query.pl	(revision 21112)
@@ -0,0 +1,121 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# lucene_query.pl -- perl wrapper to initiate query using Lucene
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use util;
+
+sub full_indexdir 
+{
+    my ($col_name,$indexdir) = @_;
+
+    my $full_indexdir
+	= &util::filename_cat($ENV{'GSDLHOME'},"collect",$col_name,"index",
+			      $indexdir);
+
+    return $full_indexdir;
+}
+
+
+sub open_java_lucene
+{
+    my ($full_indexdir, $out_file) = @_;
+    my $bin_java = &util::filename_cat($ENV{'GSDLHOME'},"bin","java");
+    my $classpath = &util::filename_cat($bin_java,"LuceneWrap.jar");
+
+    my $java_lucene = "java -classpath $classpath GS2LuceneQuery";
+    my $java_cmd = "$java_lucene";
+
+    if (defined($out_file)) {
+	$out_file = " > \"".$out_file. "\"";
+    } else {
+	$out_file = "";
+    }
+    if (!open (PIPEOUT, "| $java_cmd $full_indexdir $out_file")) {
+	die "$PROGNAME - couldn't run $java_cmd\n";
+    }
+
+}
+
+sub close_java_lucene
+{
+    close(PIPEOUT);
+}
+
+sub main
+{
+    my (@argv) = @_;
+    my $argc = scalar(@argv);
+
+    my $full_indexdir;
+    my $query = undef;
+    my $out_file = undef;
+    if ($argc==1) {
+	$full_indexdir = $argv[0];
+    }
+    elsif ($argc==2) {
+	$full_indexdir = $argv[0];
+	$query = $argv[1];
+    }
+    elsif  ($argc==3) {
+	$full_indexdir = $argv[0];
+	$query = $argv[1];
+	$out_file = $argv[2];
+    }
+    else {
+	print STDERR "Usage: $PROGNAME full-index-dir [query] [outfile]\n";
+	exit 1;
+    }
+
+
+    open_java_lucene($full_indexdir, $out_file);
+
+    if (defined $query) {
+	print PIPEOUT "$query\n";
+    }
+    else {
+	while (defined (my $line = <STDIN>)) {
+	    print PIPEOUT $line;
+	}
+    }
+
+    close_java_lucene();
+}
+
+$PROGNAME = $0;
+$PROGNAME =~ s/^.*\/(.*)$/$1/;
+
+&main(@ARGV);
+
Index: /main/tags/3.01/gsdl/bin/script/makemapfile.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/makemapfile.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/makemapfile.pl	(revision 21112)
@@ -0,0 +1,188 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# makemapfile.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+
+# %translations is of the form:
+#
+# encodings{encodingname-encodingname}->blocktranslation
+# blocktranslation->[[0-255],[256-511], ..., [65280-65535]]
+#
+# Any of the top translation blocks can point to an undefined
+# value. This data structure aims to allow fast translation and 
+# efficient storage.
+%translations = ();
+
+# @array256 is used for initialisation, there must be
+# a better way...
+@array256 = (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);
+
+&main();
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "makemapfile.pl: Creates unicode map (.ump) files from plain\n";
+    print STDERR "                text code pages.\n\n";
+    print STDERR "  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -encoding name\n";
+    print STDERR "   -mapfile  text file from which to create binary ump file\n\n";
+}
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 
+			 'encoding/.+', \$encoding,
+			 'mapfile/.+', \$mapfile)) {
+	&print_usage();
+	die "\n";
+    }
+
+    if (!&loadencoding ($encoding, $mapfile)) {
+	die "couldn't load encoding $encoding";
+    }
+
+    # write out map files
+    &writemapfile ("$encoding-unicode", $encoding, 1);
+    &writemapfile ("unicode-$encoding", $encoding, 0);
+}
+
+sub writemapfile {
+    my ($encoding, $filename, $tounicode) = @_;
+
+    $filename .= ".ump"; # unicode map file
+    if ($tounicode) {
+	$filename = &util::filename_cat ($ENV{'GSDLHOME'}, "mappings", "to_uc", $filename);
+    } else {
+	$filename = &util::filename_cat ($ENV{'GSDLHOME'}, "mappings", "from_uc", $filename);
+    }
+
+    die "translation not defined" if (!defined $translations{$encoding});
+    my $block = $translations{$encoding};
+
+    print "writing $filename\n";
+    open (MAPFILE, ">" . $filename) || die;
+    binmode (MAPFILE);
+
+    my ($i, $j);
+    for ($i=0; $i<256; $i++) {
+	if (ref ($block->[$i]) eq "ARRAY") {
+	    print MAPFILE pack ("C", $i);
+	    for ($j=0; $j<256; $j++) {
+		# unsigned short in network order
+		print MAPFILE pack ("CC", int($block->[$i]->[$j] / 256), 
+				    $block->[$i]->[$j] % 256);
+	    }
+	}
+    }
+    close (MAPFILE);
+}
+
+# loadencoding expects the mapfile to contain (at least) two
+# tab-separated fields. The first field is the mapped value
+# and the second field is the unicode value.
+#
+# It returns 1 if successful, 0 if unsuccessful
+sub loadencoding {
+    my ($encoding, $mapfile) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+	return 1;
+    }
+
+    return 0 unless open (MAPFILE, $mapfile);
+
+    my ($line, @line);
+    $translations{$to} = [@array256];
+    $translations{$from} = [@array256];
+    while (defined ($line = <MAPFILE>)) {
+	chomp $line;
+	# remove comments
+	$line =~ s/\#.*$//;
+	next unless $line =~ /\S/;
+
+	# split the line into fields and do a few
+	# simple sanity checks
+	@line = split (/\t/, $line);
+	next unless (scalar(@line) >= 2 &&
+		     $line[0] =~ /^0x/ &&
+		     $line[1] =~ /^0x/);
+
+	my $char = hex($line[0]);
+	my $unic = hex($line[1]);
+
+	# might need this for some versions of gb but not gbk
+#	$char = $char | 0x8080 unless ($encoding =~ /gbk/i);
+
+	&addchartrans ($translations{$to}, $char, $unic);
+	&addchartrans ($translations{$from}, $unic, $char);
+    }
+
+    close (MAPFILE);
+
+    return 1;
+}
+
+# addchartrans adds one character translation to a translation block.
+# It also simplifies the translation block if possible.
+sub addchartrans {
+    my ($block, $from, $to) = @_;
+    my $i = 0;
+
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	$block->[$high] = [@array256];
+    }
+    $block->[$high]->[$low] = $to;
+}
Index: /main/tags/3.01/gsdl/bin/script/mirror.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/mirror.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/mirror.pl	(revision 21112)
@@ -0,0 +1,146 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# mirror.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program uses w3mirror to mirror a web site.  It looks for a
+# mirror program configuration files in etc, and if it finds them then
+# it runs the mirroring software using this configuration file, and placing
+# the mirror in the import directory.
+#
+# mirror.pl can use the w3mirror program or the wget program if they are
+# installed.
+#
+# To use w3mirror, the configuration file must be in etc/w3mir.cfg.
+# To use GNU wget, the configuration file (i.e. a wgetrc file) must
+# be in etc/wget.cfg and a file of the URLs to read in etc/wget.url
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docprint;
+use util;
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "mirror.pl: Uses w3mir or wget to sync a collections import data\n";
+    print STDERR "           with a website.\n\n";
+    print STDERR "  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -importdir directory   Where to place the mirrored material\n";
+}
+
+
+&main ();
+
+sub main {
+    my ($verbosity, $importdir, $etcdir, 
+        $collection, $configfilename, $collectcfg);
+
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'importdir/.*/', \$importdir )) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+
+    # get the etc directory
+    $etcdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc");
+    
+    # check the collection configuration file for options
+    my $interval = 0;
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, 
+					   "etc", "collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+	    $importdir = $collectcfg->{'importdir'};
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+    
+    # fill in the default import directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = "$ENV{'GSDLCOLLECTDIR'}/import" if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+
+    # make sure there is an import directory
+    if (! -e "$importdir") {
+	&util::mk_dir($importdir);
+    }
+
+    # if w3mir.cfg exists, 
+    # then we are using w3mirror to mirror the remote site
+    if (-e "$etcdir/w3mir.cfg") {
+
+	# run the mirror program from the import directory
+	my  $cmd = "cd $importdir; ";
+	$cmd .= "perl -S gsw3mir.pl -cfgfile $etcdir/w3mir.cfg";
+	# print "\n$cmd\n";
+	`$cmd`;
+
+    } 
+
+    # if wget.cfg and wget.url both exist, 
+    # then we are using GNU wget to mirror the remote site
+    elsif ((-e "$etcdir/wget.cfg") && (-e "$etcdir/wget.url")) {
+	$ENV{WGETRC} = "$etcdir/wget.cfg";
+	my $cmd = "perl -S gsWget.pl --input-file=$etcdir/wget.url --directory-prefix=$importdir";
+	system($cmd);
+    }
+
+    # otherwise, there are no mirror copnfiguration files
+    else {
+	die "Couldn't find the mirror configuration files in $etcdir\n";
+    }
+
+
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/mkcol.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/mkcol.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/mkcol.pl	(revision 21112)
@@ -0,0 +1,305 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# mkcol.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will setup a new collection from a model one. It does this by
+# copying the model, moving files to have the correct names, and replacing
+# text within the files to match the parameters.
+
+package mkcol;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use cfgread;
+use gsprintf;
+use printusage;
+
+my $arguments =
+    [ { 'name' => "creator",
+	'desc' => "{mkcol.creator}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "optionfile",
+	'desc' => "{mkcol.optionfile}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "maintainer",
+	'desc' => "{mkcol.maintainer}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "collectdir",
+	'desc' => "{mkcol.collectdir}",
+	'type' => "string",
+	'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	'reqd' => "no" },
+      { 'name' => "public",
+	'desc' => "{mkcol.public}",
+	'type' => "string",
+	'deft' => "true",
+	'reqd' => "no" },
+      { 'name' => "title",
+	'desc' => "{mkcol.title}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "about",
+	'desc' => "{mkcol.about}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "plugin",
+	'desc' => "{mkcol.plugin}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "quiet",
+	'desc' => "{mkcol.quiet}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "win31compat",
+	'desc' => "{mkcol.win31compat}",
+	'type' => "string",
+	'deft' => "true",
+	'reqd' => "no" } ];
+
+my $options = { 'name' => "mkcol.pl",
+		'desc' => "{mkcol.desc}",
+		'args' => $arguments };
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+sub traverse_dir
+{
+    my ($modeldir, $coldir) = @_;
+    my ($newfile, @filetext);
+
+    if (!(-e $coldir)) {
+
+	my $store_umask = umask(0002);
+	my $mkdir_ok = mkdir ($coldir, 0777);
+	umask($store_umask);
+
+	if (!$mkdir_ok) 
+	{
+	    die "$!";
+	}
+    }
+
+    opendir(DIR, $modeldir) ||
+	(&gsprintf(STDERR, "{common.cannot_read}\n", $modeldir) && die);
+    my @files = grep(!/^(\.\.?|CVS)$/, readdir(DIR));
+    closedir(DIR);
+
+    foreach $file (@files)
+    {
+	my $thisfile = &util::filename_cat ($modeldir, $file);
+	if (-d $thisfile) {
+	    my $colfiledir = &util::filename_cat ($coldir, $file);
+	    traverse_dir ($thisfile, $colfiledir);
+
+	} else {
+	    my $destfile = $file;
+	    $destfile =~ s/^modelcol/$collection/;
+	    $destfile =~ s/^MODELCOL/$capcollection/;
+	    &gsprintf(STDOUT, "{mkcol.doing_replacements}\n", $destfile)
+		unless $quiet;
+	    $destfile = &util::filename_cat ($coldir, $destfile);
+
+	    open (INFILE, $thisfile) || 
+		(&gsprintf(STDERR, "{common.cannot_read_file}\n", $thisfile) && die);
+	    open (OUTFILE, ">$destfile") ||
+		(&gsprintf(STDERR, "{common.cannot_create_file}\n", $destfile) && die);
+
+	    while (defined ($line = <INFILE>)) {
+		$line =~ s/\*\*collection\*\*/$collection/g;
+		$line =~ s/\*\*COLLECTION\*\*/$capcollection/g;
+		$line =~ s/\*\*creator\*\*/$creator/g;
+		$line =~ s/\*\*maintainer\*\*/$maintainer/g;
+		$line =~ s/\*\*public\*\*/$public/g;
+		$line =~ s/\*\*title\*\*/$title/g;
+		$line =~ s/\*\*about\*\*/$about/g;
+		$line =~ s/\*\*plugins\*\*/$pluginstring/g;
+
+		print OUTFILE $line;
+	    }
+	    
+	    close (OUTFILE);
+	    close (INFILE);
+	}
+    }
+}
+
+# get and check options
+sub parse_args {
+    my ($argref) = @_;
+    if (!&parsargv::parse($argref,
+			  'language/.*/', \$language,
+			  'optionfile/.*/', \$optionfile,
+			  'collectdir/.*/', \$collectdir,
+			  'creator/\w+\@[\w\.]+/', \$creator,
+			  'maintainer/\w+\@[\w\.]+/', \$maintainer,
+			  'public/true|false/true', \$public,
+			  'title/.+/', \$title,
+			  'about/.+/', \$about,
+			  'plugin/.+', \@plugin,
+			  'quiet', \$quiet,
+			  'win31compat/true|false/true', \$win31compat,
+			  'gli', \$gli,
+			  q^xml^, \$xml
+			  )) {
+	&PrintUsage::print_txt_usage($options, "{mkcol.params}");
+	die "\n";
+    }
+}
+
+sub main {
+
+    &parse_args (\@ARGV);
+
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+	&gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+	&PrintUsage::print_xml_usage($options);
+	die "\n";
+    }
+
+    if ($gli) { # the gli wants strings to be in UTF-8
+	&gsprintf::output_strings_in_UTF8; 
+    }
+	
+    if ($optionfile =~ /\w/) {
+	open (OPTIONS, $optionfile) ||
+	    (&gsprintf(STDERR, "{common.cannot_open}\n", $optionfile) && die);
+	my $line = [];
+	my $options = [];
+	while (defined ($line = &cfgread::read_cfg_line ('mkcol::OPTIONS'))) {
+	    push (@$options, @$line);
+	}
+	close OPTIONS;
+	&parse_args ($options);
+    }
+
+    # load default plugins if none were on command line
+    if (!scalar(@plugin)) {
+	@plugin = (ZIPPlug,GAPlug,TEXTPlug,"HTMLPlug -smart_block","EMAILPlug",
+		   PDFPlug,RTFPlug,WordPlug,PSPlug,ImagePlug,NULPlug,ArcPlug,"RecPlug -use_metadata_files");
+    }
+
+    # get and check the collection name
+    ($collection) = @ARGV;
+    if (!defined($collection)) {
+	&gsprintf(STDOUT, "{mkcol.no_colname}\n");
+	&PrintUsage::print_txt_usage($options, "{mkcol.params}");
+	die "\n";
+    }
+
+    if (($win31compat eq "true") && (length($collection)) > 8) {
+	&gsprintf(STDOUT, "{mkcol.long_colname}\n");
+	die "\n";
+    }
+
+    if ($collection eq "modelcol") {
+	&gsprintf(STDOUT, "{mkcol.bad_name_modelcol}\n");
+	die "\n";
+    }
+
+    if ($collection eq "CVS") {
+	&gsprintf(STDOUT, "{mkcol.bad_name_cvs}\n");
+	die "\n";
+    }
+
+    # We don't want creator to be required anymore - John Thompson 28-11-2003
+    #if (!defined($creator) || $creator eq "") {
+    #   &gsprintf(STDOUT, "{mkcol.creator_undefined}\n");
+    #   die "\n";
+    #}
+
+    #Of course thats means we don't want to default maintainer unless a creator is provided
+    if (defined($creator) && (!defined($maintainer) || $maintainer eq "")) {
+	$maintainer = $creator;
+    }
+
+    $public = "true" unless defined $public;
+
+    if (!defined($title) || $title eq "") {
+	$title = $collection;
+    }
+
+    # get capitalised version of the collection
+    $capcollection = $collection;
+    $capcollection =~ tr/a-z/A-Z/;
+
+    # get the strings to include.
+    $pluginstring = "";
+    foreach $plugin (@plugin) {
+	$pluginstring .= "plugin         $plugin\n";
+    }
+
+    $mdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", "modelcol");
+    if (defined $collectdir && $collectdir =~ /\w/) {
+	if (!-d $collectdir) {
+	    &gsprintf(STDOUT, "{mkcol.no_collectdir}\n", $collectdir);
+	    die "\n";
+	}
+	$cdir = &util::filename_cat ($collectdir, $collection);
+    } else {
+	$cdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $collection);
+    }
+
+    # make sure the model collection exists
+    (&gsprintf(STDERR, "{mkcol.cannot_find_modelcol}\n", $mdir) && die) unless (-d $mdir);
+
+    # make sure this collection does not already exist
+    if (-e $cdir) {
+	&gsprintf(STDOUT, "{mkcol.col_already_exists}\n");
+	die "\n";
+    }
+
+    # start creating the collection
+    &gsprintf(STDOUT, "\n{mkcol.creating_col}...\n", $collection)
+	unless $quiet;
+    &traverse_dir ($mdir, $cdir);
+    &gsprintf(STDOUT, "\n{mkcol.success}\n", $cdir)
+	unless $quiet;
+}
+
+&main ();
Index: /main/tags/3.01/gsdl/bin/script/nightly.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/nightly.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/nightly.pl	(revision 21112)
@@ -0,0 +1,104 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# nightly.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will find all the collections under $GSDLHOME/collect
+# and use the update.pl script to update them all.  It should be run
+# every night to keep mirror collections up-to-date.
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docprint;
+use util;
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "nightly.pl: Attempts to update all Greenstone collections found on\n";
+    print STDERR "            the local filesystem by calling update.pl for each of\n";
+    print STDERR "            them.\n\n";
+    print STDERR "  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -importdir directory   Where to place the mirrored material\n";
+    print STDERR "   -archivedir directory  Where the converted material ends up\n";
+}
+
+
+&main ();
+
+sub main {
+    my ($verbosity, $collectdir, $name, $collection);
+
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity )) {
+	&print_usage();
+	die "\n";
+    }
+
+
+    # get the contents of the collect directory
+    $collectdir =  &util::filename_cat($ENV{'GSDLHOME'}, "collect");
+    opendir(CDIR, $collectdir) || die "Cannot open $collectdir: $!";
+    my @files = readdir(CDIR);
+    closedir CDIR;
+
+
+    # update each collection
+    foreach $name (@files) {
+
+	$collection = &util::filename_cat($collectdir, $name);
+
+	# igore entries for "." and ".."
+	next unless ($name =~ /[^\.]/);
+
+	# ignore modelcol
+	next if ($name eq "modelcol");
+
+	# ignore entries that are not directories
+	next unless (-d $collection);
+
+	# ignore directories that do not have a subdir called etc
+	next unless (-d &util::filename_cat($collection, "etc"));
+
+	print "Updating: $name\n";
+	print `update.pl $name`;
+	
+    }
+
+}
Index: /main/tags/3.01/gsdl/bin/script/pdftohtml.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/pdftohtml.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/pdftohtml.pl	(revision 21112)
@@ -0,0 +1,267 @@
+#!/usr/bin/perl -w
+
+
+###########################################################################
+#
+# pdftohtml.pl -- convert PDF documents to HTML format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# pdftohtml.pl is a wrapper for running pdftohtml utility which converts
+# PDF documents to HTML, and converts images to PNG format for display in
+# the HTML pages generated
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use Cwd;
+use File::Basename;
+
+sub print_usage {
+# note - we don't actually ever use most of these options...
+print STDERR  
+    ("pdftohtml.pl wrapper for pdftohtml.\n",
+     "Usage: pdftohtml [options] <PDF-file> <html-file>\n",
+     "Options:\n",
+     "\t-i\tignore images (don't extract)\n",
+     "\t-a\tallow images only (continue even if no text is present)\n",
+     "\t-c\tproduce complex output (requires ghostscript)\n",
+     "\t-hidden\tExtract hidden text\n",
+     "\t-zoom\tfactor by which to zoom the PDF (only useful if -c is set)\n"
+     );
+exit (1);
+}
+
+sub main {
+    my (@ARGV) = @_;
+    my ($allow_no_text, $ignore_images, $complex, $zoom, $hidden);
+    
+    # read command-line arguments so that
+    # you can change the command in this script
+    if (!parsargv::parse(\@ARGV,
+			 'a', \$allow_no_text,
+			 'i', \$ignore_images,
+			 'c', \$complex,
+			 'hidden', \$hidden,
+			 'zoom/\d+/2', \$zoom,
+			 ))
+    {
+	print_usage();
+    }
+
+    # Make sure the input file exists and can be opened for reading
+    if (scalar(@ARGV) != 2) {
+	print_usage();
+    }
+
+    my $input_filename = $ARGV[0];
+    my $output_filestem = $ARGV[1];
+
+    $output_filestem =~ s/\.html$//i; # pdftohtml adds this suffix
+
+    # test that the directories exist to create the output file, or
+    # we should exit immediately. (File:: is included by util.pm)
+    my $output_dir =  File::Basename::dirname($output_filestem);
+    if (! -d $output_dir || ! -w $output_dir) {
+	die "pdftohtml.pl: cannot write to directory $output_dir\n";
+    }
+
+    my @dir = split (/(\/|\\)/, $input_filename);
+    my $input_basename = pop(@dir);
+    $input_basename =~ s/\.pdf//i;
+    my $dir = join ("", @dir);
+
+    if (!-r $input_filename) {
+	print STDERR "Error: unable to open $input_filename for reading\n";
+	exit(1);
+    }
+
+    # Heuristical code removed due to pdftohtml being "fixed" to not
+    # create bitmaps for each char in some pdfs. However, this means we
+    # now create .html files even if we can't extract any text. We should
+    # check for that now instead someday...
+
+
+    # formulate the command
+    my $cmd = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "pdftohtml");
+
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly.
+    $cmd = "pdftohtml" if ($ENV{'GSDLOS'} =~ /^windows$/);
+
+    $cmd .= " -i" if ($ignore_images);
+    $cmd .= " -c" if ($complex);
+    $cmd .= " -hidden" if ($hidden);
+    $cmd .= " -zoom $zoom";
+    $cmd .= " -noframes -p -enc UTF-8 \"$input_filename\" \"$output_filestem.html\"";
+
+# system() returns -1 if it can't run, otherwise it's $cmds ret val.
+    # note we return 0 if the file is "encrypted"
+    $!=0;
+    if (system($cmd)!=0) {
+	print STDERR "pdftohtml error for $input_filename $!\n";
+	# leave these for gsConvert.pl...
+	#&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	#&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+	return 1;
+    }
+
+    if (! -e "$output_filestem.html") {
+	return 1;
+    }
+
+# post-process to remove </b><b> and </i><i>, as these break up
+# words, screwing up indexing and searching.
+# At the same time, check that our .html file has some textual content.
+    &util::mv("$output_filestem.html","$output_filestem.html.tmp");
+    $!=0;
+    open INFILE, "$output_filestem.html.tmp" ||
+	die "Couldn't open file: $!";
+    open OUTFILE, ">$output_filestem.html" ||
+	die "Couldn't open file for writing: $!";
+    my $line;
+    my $seen_textual_content=$allow_no_text;
+    # check for unicode byte-order marker at the start of the file
+    $line = <INFILE>;
+    $line =~ s#\376\377##g;
+    while ($line) {
+	$line =~ s#</b><b>##g;
+	$line =~ s#</i><i>##g;
+	$line =~ s#\\#\\\\#g; # until macro language parsing is fixed...
+# check for any extracted text
+	if ($seen_textual_content == 0) {
+	    my $tmp_line=$line;
+	    $tmp_line =~ s/<[^>]*>//g;
+	    $tmp_line =~ s/Page\s\d+//;
+	    $tmp_line =~ s/\s*//g;
+	    if ($tmp_line ne "") {
+		$seen_textual_content=1;
+	    }
+	    # special - added to remove the filename from the title
+	    # this should be in the header, before we see "textual content"
+	    if ($line =~ m@<title>(.*?)</title>@i) {
+		my $title=$1;
+		
+		# is this title the name of a filename?
+		if (-r "$title.pdf" || -r "$title.html") {
+		    # remove the title
+		    $line =~ s@<title>.*?</title>@<title></title>\n<META NAME=\"filename\" CONTENT=\"$title\">@i;
+		}
+	    }
+	}
+
+	# relative hrefs to own document...
+	$line =~ s@href=\"$input_basename\.html\#@href=\"\#@go;
+# escape underscores, but not if they're inside tags (eg img/href names)
+	my $inatag = 0; # allow multi-line tags
+	if ($line =~ /_/) {
+	    my @parts=split('_',$line);
+	    my $lastpart=pop @parts;
+	    foreach my $part (@parts) {
+		if ($part =~ /<[^>]*$/) { # if we're starting a tag...
+		    $inatag=1;
+		} elsif ($part =~ />[^<]*$/) { # closing a tag
+		    $inatag=0;
+		}
+		if ($inatag) {
+		    $part.='_';
+		} else {
+		    $part.="&#95;";
+		}
+	    }
+	    $line=join('',@parts,$lastpart);
+	}
+
+	print OUTFILE $line;
+	$line = <INFILE>;
+    }
+    close INFILE;
+    close OUTFILE;
+    &util::rm("$output_filestem.html.tmp");
+
+    # Need to convert images from PPM format to PNG format
+    my @images;
+
+    my $directory=$output_filestem;
+    $directory =~ s@[^\/]*$@@;    # assume filename has no embedded slashes...
+    # newer versions of pdftohtml don't seem to do images this way anymore?
+    if (open (IMAGES, "${directory}images.log") || 
+	open (IMAGES, "${directory}image.log")) {
+	while (<IMAGES>) {
+	    push (@images, $_);
+	}
+	close IMAGES;
+	&util::rm("${directory}image.log") if (-e "${directory}image.log");
+
+    }
+
+    # no need to go any further if there is no text extracted from pdf.
+    if ($seen_textual_content == 0) {
+	print STDERR "Error: PDF contains no extractable text\n";
+	# remove images...
+	for $image (@images) {
+	    chomp($image);
+	    &util::rm("${directory}$image");
+	}
+	return 1;
+    }
+
+
+
+    for $image (@images) {
+	chomp($image);
+	my $cmd = "";
+	if ($ENV{'GSDLOS'} =~ /^windows/i) {
+	    $cmd = "pnmtopng \"${directory}$image\"";
+	    if (system($cmd)!=0) {
+		print STDERR "Error executing $cmd\n";
+		#return 1; # not sure about whether to leave this one in or take it out
+		next;
+	    }
+	} else {
+	    my @nameparts = split(/\./, $image);
+	    my $image_base = shift(@nameparts);
+	    $cmd = "pnmtopng \"${directory}$image\" > \"${directory}$image_base.png\" 2>/dev/null";
+	    if (system($cmd)!=0) {
+		$cmd = "convert \"${directory}$image\" \"${directory}$image_base.png\" 2>/dev/null";
+		if (system($cmd)!=0) {
+		    print STDERR "Cannot convert $image into PNG format (tried `pnmtopng' and `convert')...\n";
+		    #return 1; # not sure about whether to leave this one in or take it out
+		    next;
+		}
+	    }
+	}
+	&util::rm($image);
+    }
+
+    return 0;
+}
+
+# indicate our error status, 0 = success
+exit (&main(@ARGV));
+
Index: /main/tags/3.01/gsdl/bin/script/pdftoimg.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/pdftoimg.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/pdftoimg.pl	(revision 21112)
@@ -0,0 +1,167 @@
+#!/usr/bin/perl -w
+
+
+###########################################################################
+#
+# pdftoimg.pl -- convert PDF documents to various types of Image format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# pdftoimg.pl is a wrapper for running convert utility which converts
+# PDF documents to various types of image (e.g. PNG, GIF, JPEG format, 
+# Then use PagedImgPlug to deal with the the images
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use Cwd;
+use File::Basename
+;
+sub print_usage {
+    # note - we don't actually ever use most of these options...
+    print STDERR  
+	("pdftoimg.pl wrapper for pdftoimg.\n",
+	 "Usage: pdftoimg [options] <PDF-file> <output>>\n",
+	 "Options:\n",
+	 "\t-convert_to\toutput image type for the PDF\n"
+	 );
+    exit (1);
+}
+
+sub main {
+    my (@ARGV) = @_;
+    my ($convert_to);
+    
+    # read command-line arguments so that
+    # you can change the command in this script
+    if (!parsargv::parse(\@ARGV,
+			 'convert_to/.*/^', \$convert_to,
+			 )) {
+    	print_usage();
+    }
+    
+    # Make sure the input file exists and can be opened for reading
+    if (scalar(@ARGV) != 2) {
+	print_usage();
+    }
+    
+    my $input_filename = $ARGV[0];
+    my $output_filestem = $ARGV[1];
+    
+    # test that the directories exist to create the output file, or
+    # we should exit immediately. (File:: is included by util.pm)
+    &util::mk_dir($output_filestem) if (!-e $output_filestem);
+	    
+    my @dir = split (/(\/|\\)/, $input_filename);
+    my $input_basename = pop(@dir);
+    $input_basename =~ s/\.pdf//i;
+    my $dir = join ("", @dir);
+
+    if (!-r $input_filename) {
+	print STDERR "Error: unable to open $input_filename for reading\n";
+	exit(1);
+    }
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly.
+    $cmd = "convert";
+    #if ($ENV{'GSDLOS'} =~ /^windows$/);
+    #Convert utility will convert the PDF to GIF Animation
+    my $output_filename = &util::filename_cat($output_filestem, $input_basename);
+    if ($convert_to eq "gif") {
+	$cmd .= " \"$input_filename\" \"$output_filename-%02d.$convert_to\"";
+    } else {
+	$cmd .= " \"$input_filename\" \"$output_filename.$convert_to\"";
+    }	
+ 
+    # system() returns -1 if it can't run, otherwise it's $cmds ret val.
+    # note we return 0 if the file is "encrypted"
+    $!=0;
+    if (system($cmd)!=0) {
+	print STDERR "Convert error for $input_filename $!\n";
+	# leave these for gsConvert.pl...
+	#&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	#&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+	return 1;
+    } else {
+	# command execute successfully
+	create_itemfile($output_filestem, $input_basename, $convert_to);
+    }
+    return 0;
+}
+
+sub create_itemfile
+{
+    my ($output_dir, $convert_basename, $convert_to) = @_;
+    opendir(DIR, $output_dir) || die "can't opendir $output_dir: $!";  
+    my $item_file = $output_dir."/".$convert_basename.".item";
+    open(FILE,">$item_file");
+    
+    print FILE "<PagedDocument>\n";
+
+    my $page_num = "";
+    @dir_files = grep {-f "$output_dir/$_"} readdir(DIR);
+
+    # Sort files in the directory by page_num    
+    sub page_num {
+	my ($dir) = @_;
+	my $pagenum = "";
+	if ($ENV{'GSDLOS'} =~ /^windows$/){
+	   ($pagenum) =($dir =~ m/^.*\.(\d+)$/i);   
+	} else {
+	   ($pagenum) =($dir =~ m/^.*-(\d+)\.(.*)$/i);
+	}
+	$pagenum = $pagenum || 1;
+	return $pagenum;
+    }
+
+    # sort the files in the directory in the order of page_num rather than lexically. 
+    @dir_files = sort { page_num($a) <=> page_num($b) } @dir_files;
+
+    foreach my $file (@dir_files){
+        if ($ENV{'GSDLOS'} =~ /^windows$/ && $convert_to ne "gif"){
+	  ($page_num) =($file =~ m/^.*\.(\d+)$/i); 
+	} else {
+	    ($page_num) =($file =~ m/^.*-(\d+)\.(.*)/i);   
+	}
+	# as the converter will convert the document to image files start from page 0
+	$page_num =$page_num + 1 if defined $page_num;
+	$page_num = 1 unless defined $page_num;
+	if ($file !~ /\.item/i){
+	    print FILE "   <Page pagenum=\"$page_num\" imgfile=\"$file\" txtfile=\"\"/>\n";
+	} 
+    }
+
+    print FILE "</PagedDocument>\n";
+    closedir DIR;
+    return "";
+}
+
+# indicate our error status, 0 = success
+exit (&main(@ARGV));
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/ping.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/ping.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/ping.pl	(revision 21112)
@@ -0,0 +1,74 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# ping.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/Ping");
+}
+
+
+require LWP::UserAgent;
+use parsargv;
+
+&main();
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "ping.pl: Ping http or ftp URL to see if it's accessable.\n\n";
+    print STDERR "  usage: $0 [options] URL\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -quiet   Quiet operation\n\n";
+}
+
+sub main {
+
+    my ($quiet);
+    if (!parsargv::parse(\@ARGV, 'quiet', \$quiet)) {
+	&print_usage();
+	die "\n";
+    }
+
+    if (!scalar(@ARGV)) {
+	print STDERR "ERROR: no URL was provided\n";
+	die "\n";
+    }
+
+    $ua = new LWP::UserAgent;
+    $ua->timeout(60);
+    $request = new HTTP::Request('HEAD', $ARGV[0]);
+    $response = $ua->request($request);
+    
+    if ($response->is_success) {
+	print STDERR "$ARGV[0] ping succeeded\n" unless $quiet;
+	exit 0;
+    } else {
+	print STDERR "$ARGV[0] ping failed\n" unless $quiet;
+	exit 1;
+    }
+}
Index: /main/tags/3.01/gsdl/bin/script/pluginfo.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/pluginfo.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/pluginfo.pl	(revision 21112)
@@ -0,0 +1,215 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# pluginfo.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This program will print info about a plugin
+
+use strict;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use plugin;
+use util;
+use gsprintf;
+use printusage;
+
+use parse2;
+
+my $arguments =
+    [ { 'name' => "collect",
+	'desc' => "{pluginfo.collect}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "xml",
+	'desc' => "{scripts.xml}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "listall",
+	'desc' => "{scripts.listall}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "language",
+	'desc' => "{scripts.language}",
+	'type' => "string",
+	'reqd' => "no" } ];
+
+my $options = { 'name' => "pluginfo.pl",
+		'desc' => "{pluginfo.desc}",
+		'args' => $arguments };
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+sub main {
+    my $collect = "";
+    my $xml = 0;
+    my $listall = 0;
+    
+    my ($language, $encoding);
+
+    my $hashParsingResult = {};
+    my $blnParseFailed = "false";
+    # general options available to all plugins
+    my $unparsed_args = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
+
+    foreach my $strVariable (keys %$hashParsingResult)
+    {
+	my $value = $hashParsingResult->{$strVariable};
+	# test to make sure the variable name is 'safe'
+	if ($strVariable !~ /^\w+$/) {
+ 	    die "variable name '$strVariable' isn't safe!";
+	}
+	eval "\$$strVariable = \$value";
+    }
+
+    # if language wasn't specified, see if it is set in the environment
+    # (LC_ALL or LANG)
+    if (!$language && ($_=$ENV{'LC_ALL'} or $_=$ENV{'LANG'})) {
+	m/^([^\.]+)\.?(.*)/;
+	$language=$1;
+	$encoding=$2; # might be undef...
+# gsprintf::load_language* thinks "fr" is completely different to "fr_FR"...
+	$language =~ s/_.*$//;
+    }
+
+    # If $language has been set, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+	gsprintf::load_language_specific_resource_bundle($language);
+	if ($encoding) {
+	    $encoding =~ tr/-/_/;
+	    $encoding = lc($encoding);
+	    $encoding =~ s/utf_8/utf8/; # special
+	    $gsprintf::specialoutputencoding=$encoding;
+	}
+    }
+
+    # If there are more than one argument left after parsing, it mean user input too many arguments.
+    # Error occoured  will return 0
+    if( $unparsed_args > 1 or (@ARGV && $ARGV[0] =~ /^\-+h/) )
+    {
+	PrintUsage::print_txt_usage($options, "{pluginfo.params}");
+	die "\n";
+    }
+
+    my $plugin = shift (@ARGV);
+    if (defined $plugin) {
+	$plugin =~ s/\.pm$//; # allow xxxPlug.pm as the argument
+    }
+    if (($listall == 0) && (!defined $plugin || $plugin eq "")) {
+	gsprintf(STDERR, "{pluginfo.no_plugin_name}\n\n");
+	PrintUsage::print_txt_usage($options, "{pluginfo.params}", 1);
+	die "\n";
+    }
+
+    if ($collect ne "") {
+	$ENV{'GSDLCOLLECTDIR'} = &util::filename_cat ($ENV{'GSDLHOME'}, "collect", $collect);
+    } else {
+	$ENV{'GSDLCOLLECTDIR'} = $ENV{'GSDLHOME'};
+    }
+
+    if ($listall) {
+	my $plugin_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib","plugins");
+	
+	if (!opendir (INDIR, $plugin_dir)) {
+	    print STDERR "pluginfo.pl: could not open directory $plugin_dir\n";
+	} else {
+	    my @plugin_list = grep (/Plug\.pm$/, readdir (INDIR));
+	    closedir (INDIR);
+
+
+	    if ($xml) {
+		my $num_plugins = scalar(@plugin_list);
+
+		print STDERR "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+		print STDERR "<!DOCTYPE PluginList [\n";
+		print STDERR "  <!ELEMENT PluginList (PluginName*)>\n";
+		print STDERR "  <!ELEMENT PlugName   (#PCDATA)>\n";
+		print STDERR "  <!ATTLIST PluginList\n";
+		print STDERR "            length CDATA #REQUIRED>\n";
+		print STDERR "]>\n";
+
+		print STDERR "<PluginList length=\"$num_plugins\">\n";
+		map { print STDERR "  <PluginName>$_</PluginName>\n"; } @plugin_list;
+		print STDERR "</PluginList>\n";
+
+	    }
+	    else {
+		print STDERR join(" ",@plugin_list), "\n";
+	    }
+	}
+
+    }
+    else {
+
+	#my $pluginfo = &plugin::load_plugins ([[$plugin]]);
+	#my $plugobj = shift @$pluginfo;
+	my $plugobj = &plugin::load_plugin_for_info ($plugin);
+	if ($xml) {
+	    $plugobj->print_xml_usage();
+	}
+	else {
+	    gsprintf(STDERR, "\n{pluginfo.passing_options}\n\n");
+	    gsprintf(STDERR, "{pluginfo.option_types}:\n\n");
+	    gsprintf(STDERR, "{pluginfo.specific_options}\n\n");
+	    gsprintf(STDERR, "{pluginfo.general_options}\n\n");
+	    gsprintf(STDERR, "$plugin {pluginfo.info}:\n\n");
+	    
+	    $plugobj->print_txt_usage();
+	}
+    }
+}
+
+
+# this causes us to automatically send output to a pager, if one is
+# set, AND our output is going to a terminal
+# active state perl on windows doesn't do open(handle, "-|");
+if ($ENV{'GSDLOS'} !~ /windows/ && -t STDOUT) {
+    my $pager = $ENV{"PAGER"};
+    if (! $pager) {$pager="(less || more)"}
+    my $pid = open(STDIN, "-|"); # this does a fork... see man perlipc(1)
+    if (!defined $pid) {
+	gsprintf(STDERR, "pluginfo.pl - can't fork: $!");
+    } else {
+	if ($pid != 0) { # parent (ie forking) process. child gets 0
+	    exec ($pager);
+	}
+    }
+    open(STDERR,">&STDOUT"); # so it's easier to pipe output
+}
+
+
+&main ();
Index: /main/tags/3.01/gsdl/bin/script/ppttohtml.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/ppttohtml.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/ppttohtml.pl	(revision 21112)
@@ -0,0 +1,68 @@
+eval 'exec perl -x -- $0 $@'
+    if 0;
+#! perl
+# line 6
+
+sub usage() {
+    print "$0 <input.ppt> <output.html>\n";
+}
+
+
+if (@ARGV != 2) {
+    usage();
+    exit(1);
+}
+my $input_ppt=shift;
+my $output_html=shift;
+
+# try to find ppthtml binary in GSDLHOME
+my $ppthtml_binary="ppthtml";
+my $GSDLOS=$ENV{'GSDLOS'};
+my $GSDLHOME=$ENV{'GSDLHOME'};
+if ($GSDLOS =~ /^windows$/i) {
+    $ppthtml_binary.=".exe";
+}
+
+# assume it is on the path if running under windows, in case GSDLHOME
+# has a space
+if ($GSDLOS !~ /windows/i && -x "$GSDLHOME/bin/$GSDLOS/$ppthtml_binary") {
+    $ppthtml_binary="$GSDLHOME/bin/$GSDLOS/$ppthtml_binary";
+}
+
+if (! -r $input_ppt) {
+    print STDERR "Unable to read file `$input_ppt'\n";
+    exit (1);
+}
+
+my $return_value=
+    system("$ppthtml_binary \"$input_ppt\" > \"$output_html\"");
+
+if ($return_value != 0) {
+    exit (1);
+}
+
+# Ok, we made an html file. Check to see if it has any content, and remove
+# the little nag link at the the bottom.
+my $html="";
+open (HTML, "$output_html") || die "Can't read file:$!";
+$html=join('', <HTML>);
+close HTML;
+
+$html =~ s@<hr><FONT SIZE=-1>Created with.*\n</BODY></HTML>$@</BODY></HTML>@s;
+
+# if we are using the file name as the title, then get rid of it,
+# as HTMLPlug will use the first 100 chars instead if there's no title.
+$html =~ s@<title>.*?\.ppt</title>@<title></title>@i;
+
+my $tmp=$html;
+$tmp =~ s/^.*?<BODY>//ms;
+$tmp =~ s/(&nbsp;)|\s//gims;
+if ($tmp !~ m/(>[^<]+<)/) {
+    print STDERR "No text found in extracted html file!\n";
+    exit(1);
+}
+open (NEWHTML, ">$output_html") || die "Can't create file:$!";
+print NEWHTML $html;
+close NEWHTML;
+exit (0);
+
Index: /main/tags/3.01/gsdl/bin/script/process_html.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/process_html.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/process_html.pl	(revision 21112)
@@ -0,0 +1,393 @@
+#!/usr/bin/perl -w
+
+
+# Both this script and its associated process_html.pl were written by
+# Marcel ?, while a student at Waikato University. Unfortunately he
+# was very new to perl at the time so the code is neither as clean nor
+# as fast as it could be (I've cleaned up a few of the more serious
+# bottlenecks -- it could do with alot more work though).  The code
+# does work though, if a little slowly. It's not ready for primetime
+# however and is included in the Greenstone source tree mostly so that
+# I don't lose it. -- Stefan - 24 Jul 2001
+
+
+# This script rebuilds the static collection by linking all the downloaded html files
+# back together. 
+# It searches through html files and replaces links it recognizes from the links.txt file 
+# with the apropriate html file name (eg 1.html, 2.html etc)
+# This script also updates the links for the pictures.
+
+# This is where all the dl-ed html files are located, eg. 'temp_html/'
+my $outputdir  = 'temp_html/';
+
+# This is where all the processed files end up (don't want to overwrite originals ;) eg. 'my_static_collection/'
+my $finaldir   = 'envl_collection/';
+
+# If any options where used (such as &u=1) when the html files where dl-ed then please specify them here.
+my $option = "&u=1";
+
+# Please ensure these two options match the settings used when downloading the collection :)
+my $dir_entries = 250;
+my $fix_empty_pages = '&cl=CL1';
+
+#-------------------------------------------------------------------------------------------------
+
+# global arrays used to store links, links-index & html-filenames
+my %filez;
+my %out_filez;
+my %linkz;
+my %short_linkz1;
+my %short_linkz2;
+my %short_linkz3;
+my %linkz_index;
+#my %remove_these;
+my $remove_these = "";
+
+sub processfiles
+{
+    local($start_here) = @_;
+
+    for my $file($start_here .. $#filez)
+    {
+	if((-e $filez[$file])&&(-s $filez[$file]))
+	{
+	    open (FILE, $filez[$file]) or die "can't open ", $filez[$file],": $! \n";
+	    
+	    print " $filez[$file] ";
+	    
+	    undef $/;
+	    my $content_of_file = <FILE>;
+	    $/ = "\n";
+	    close(FILE);
+	    
+	    #quick & nasty fix for the 'open book' link
+	    local $quick_fix1 = "&cl=\"";
+	    local $quick_fix2 = "&cl=\'";
+	    
+	    $content_of_file =~ s/$quick_fix1/$fix_empty_pages\"/g;
+	    $content_of_file =~ s/$quick_fix2/$fix_empty_pages\'/g;
+	    
+	    for my $link(0 .. $#linkz)
+	    {
+		my $new_link = $linkz_index[$link].".html";
+		
+		if($short_linkz3[$link] ne "")
+		{
+		    $content_of_file =~ s/$short_linkz1[$link].*?$short_linkz2[$link].*?${short_linkz3[$link]}[^\"\'\s\>]*/$new_link/g;
+		}
+		else
+		{
+			$content_of_file =~ s/$short_linkz1[$link].*${short_linkz2[$link]}[^\"\'\s\>]*/$new_link/g;
+		}
+	    }
+    
+            $content_of_file =~ s/(["'])$remove_these/$1..\//g;
+	    open (TEMP, ">temp.html") or die "can't open temp.html: $! \n";
+	    print TEMP $content_of_file;
+	    close(TEMP);
+	    rename("temp.html", $out_filez[$file]) or die "cannot create", $out_filez[$file],": $! \n";
+	    print " --> $out_filez[$file]";
+	    print "..done\n";
+	}
+	else
+	{
+	    last;    # bomb out of loop. Done.
+	}
+    }
+     print " *** Done, cannot find any more files to process ***\n";
+}
+
+# the switch variable there so that I can create a couple of additional arrays without having to write an entirely new function :-)
+# 0 = off, 1 = on (puts values into %linkz_index, %short_linkz1 and %short_linkz2)
+sub sort_array_by_length
+{
+    local (*foo, $switch) = @_;
+    my $total = $#foo;
+    my %temp_linkz;
+    my $shortest = 999999;
+    my $longest = 0;
+
+    if ($switch != 0)
+    {
+	print "Processing linkz (chopping, slicing, dicing and sorting :-)...";
+    }
+
+    for my $counter(0 .. $total)
+    {
+	if (length($foo[$counter]) < $shortest)
+	{
+	    $shortest = length($foo[$counter]);   
+	    $temp_linkz[$total] = $foo[$counter];
+	}
+	if (length($foo[$counter]) > $longest)
+	{
+	    $longest = length($foo[$counter]);   
+	}  
+    }        
+    
+    $backward = $total;
+    for my $l($shortest .. $longest)
+    {      
+	local $numberdir = 0;
+	for my $counter(0 .. $total)
+	{
+	    if ($counter % $dir_entries == 0)
+	    {
+		$numberdir = $counter;
+	    }
+
+	    if(length($foo[$counter]) == $l)
+	    {
+		$temp_linkz[$backward] = $foo[$counter];
+		if ($switch != 0)
+		{ 
+		    $linkz_index[$backward] = "../".$numberdir."/".$counter;
+		    my $d_offset = 0;
+		    for my $search(0 .. (length($foo[$counter]) - 3))
+		    {
+			if((substr($foo[$counter], $search, 3) eq '?e=')||(substr($foo[$counter], $search, 3) eq '&e='))
+			{
+			    $short_linkz1[$backward] = substr($foo[$counter], 0, $search);
+			}
+			
+			for my $second_search($search .. length($foo[$counter]))
+			{
+			    if((substr($foo[$counter], $second_search, 3) eq '?d=')||(substr($foo[$counter], $second_search, 3) eq '&d='))
+			    {
+				$short_linkz3[$backward] = substr($foo[$counter], $second_search);
+				$d_offset = $second_search;
+				last;
+			    }
+			}
+			
+			if(substr($foo[$counter], $search, 3) eq '?a=')
+			{
+			    $short_linkz1[$backward] = substr($foo[$counter], 0, $search);
+			    if($d_offset > 0)
+			    {
+				$short_linkz2[$backward] = substr($foo[$counter], $search, $d_offset - $search);
+			    }
+			    else
+			    {
+				$short_linkz2[$backward] = substr($foo[$counter], $search);
+			    }
+			}
+			
+			if(substr($foo[$counter], $search, 3) eq '&a=')
+			{
+			    if($d_offset > 0)
+			    {
+				$short_linkz2[$backward] = substr($foo[$counter], $search, $d_offset - $search);
+			    }
+			    else
+			    {
+				$short_linkz2[$backward] = substr($foo[$counter], $search);
+			    }
+			}
+		    } 
+		}
+		$backward--;
+	    }
+	}
+    }    
+    # copy the sorted temp_array over the original array (must be a better way of doing this :\ )
+    for my $counter(0 .. $total)
+    {    
+	$foo[$counter] = $temp_linkz[$counter];
+    }
+    if ($switch != 0)
+    {
+	print "done!\n";
+    }
+}
+
+sub how_much_to_chop
+{
+    local($link) = @_;
+    my $bracket_counter = 0;
+    my $chop_offset = 0;
+
+    for my $search(0 .. length($link))
+    {
+	if (substr($link, $search, 1) eq '/')
+	{
+	    $bracket_counter++;   
+	}
+	if ($bracket_counter == 2)
+	{
+	    $chop_offset = $search + 1;
+	} 
+    }
+    return $chop_offset;
+}
+
+my $start_time = (times)[0]; 
+
+#-----------------------------------------------------------------------------------------------
+# No need to start from scratch everytime, we can recover/continue from wherever we left off
+# simply by checking which html files have been created
+#-----------------------------------------------------------------------------------------------
+my $linknumber = 0;
+my $failed     = 0;
+my $check_file = "";
+my $numberdir = 0;
+
+if($outputdir ne $finaldir)
+{
+    while ($failed == 0)
+    {
+	if ($linknumber % $dir_entries == 0) 
+	{
+	    if (!((-e $finaldir.$linknumber)&&(-d $finaldir.$linknumber)))
+	    {
+		$failed++;
+		mkdir($finaldir.$linknumber, 0777) or die " ** Cannot create ",$finaldir.$linknumber, "!: $!\n";
+	    }
+	    $numberdir = $linknumber;
+	}
+	
+	$check_file = $finaldir.$numberdir."/".$linknumber.".html";
+	if ((-e $check_file)&&($failed == 0)) 
+	{ 
+	    $linknumber++;
+	} 
+	else
+	{
+	    $failed++;
+	    # I'm subtracting 1 from the starting link, 
+	    # just in case it only loaded half the page ;^)
+	    if($linknumber>0) 
+	    { 
+		$linknumber--; 
+	    }
+	    print " Will start processing at number $linknumber \n";
+	}
+    }
+}
+my $i = 0;
+my $that = "";
+my $offset = 0;
+
+#read in old links from links text file
+open (CHECK, "links.txt") || die " ** Cannot find/open links text file!: $!\n";
+while (defined ($that = <CHECK>)) {
+    
+    if ($i == 0)
+    {  
+	#chop off the first bit
+	$offset = &how_much_to_chop($that); 
+	print " Offset has been set to: ",$offset,"\n";
+	print " This next bit will be ignored for all links in the links.txt file:\n";
+	print "  -->",substr($that,0,$offset),"<--\n";
+    }
+    
+    $that = substr($that, $offset);
+    
+    #Wipe-out the EOL character
+#    if (substr($that, -1) eq "\n") { substr($that, -1) = ""; }
+    chomp $that;
+    
+    #this wipes the options
+#    if (length($option) != 0)
+#    {	    
+#	substr($that, (length($option)) * -1) = "";
+#    }
+    $that =~ s/$option//;
+    
+    $linkz[$i] = $that;
+    
+    $short_linkz1[$i] = "";
+    $short_linkz2[$i] = "";
+    $short_linkz3[$i] = "";
+    
+    for my $search(0 .. (length($that) - 3))
+    {
+	if((substr($that, $search, 3) eq '?e=')||(substr($that, $search, 3) eq '&e='))
+	{
+	    $short_linkz1[$i] = substr($that, 0, $search);
+	}
+	
+	if(substr($that, $search, 3) eq '?a=')
+	{
+	    $short_linkz1[$i] = substr($that, 0, $search);
+	    $short_linkz2[$i] = substr($that, $search);
+	}
+	if(substr($that, $search, 3) eq '&a=')
+	{
+	    $short_linkz2[$i] = substr($that, $search);
+	}
+    }
+    $i++;
+    
+    if ($i % $dir_entries == 0)
+    {
+	if (!((-e $finaldir.$i)&&(-d $finaldir.$i)))
+	{
+	    mkdir($finaldir.$i, 0777) or die " ** Cannot create ",$finaldir.$i, "!: $!\n";
+	}
+    }
+}
+close(CHECK);
+
+print " - I found ",$i, " links in the links text file -\n";
+
+&sort_array_by_length(*linkz, 1);
+
+$numberdir = 0;
+
+for my $z(0 .. ($i - 1))
+{
+    if($z % $dir_entries == 0)
+    {
+	$numberdir = $z;
+    }
+    $filez[$z] = $outputdir.$numberdir."/".$z.".html";
+    $out_filez[$z] = $finaldir.$numberdir."/".$z.".html";
+}
+
+# ..and last but not least, load any image_dirs from image_dirs.txt
+my $imd_that = "";
+#my $image_dirs_pointer = 0;
+
+my @tmp_arr = ();
+open (IMAGE_DIR, "image_dirs.txt") || die " ** HEY! Cannot find/open image_dirs.txt file! : $! **\n";
+while(defined ($imd_that = <IMAGE_DIR>))
+{
+    chomp $imd_that;
+    push(@tmp_array, $imd_that);
+}
+close IMAGE_DIR;
+
+$remove_these = "(" . join ("|", sort {length $b <=> length $a} @tmp_array) . ")"; 
+
+#print " - I found ",($#remove_these + 1)," picture directories in image_dirs.txt -\n";
+#&sort_array_by_length(*remove_these, 0);
+
+print "-" x 20, "\n";
+print "  Here we go...\n";
+print "-" x 20, "\n";
+
+&processfiles($linknumber);
+
+my $end_time = (times)[0];
+print "\n\n\n *----------------------------*\n";
+print " |  Whew! Task completed! :-D |\n";
+print " *----------------------------*\n";
+printf"  Script took %.2f CPU seconds to complete ;^)\n", $end_time - $start_time;
+print "\n\n"; 
+print " Now there's a few things left to do...load up ",$finaldir, "0/0.html in your webbrowser and\n";
+print " make sure everything works.\n"; 
+print " The grab_collection script will have generated 3 text files that can be removed, namely:\n";
+print " - links.txt \n";
+print " - images.txt \n";
+print " - image_dirs.txt \n\n";
+if ($outputdir ne $finaldir)
+{
+    print "And then finally you can also delete the ",$outputdir," directory.\n\n";
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/pstoimg.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/pstoimg.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/pstoimg.pl	(revision 21112)
@@ -0,0 +1,168 @@
+#!/usr/bin/perl -w
+
+
+###########################################################################
+#
+# pstoimg.pl -- convert PS documents to various types of Image format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+# 
+###########################################################################
+# pstoimg.pl is a wrapper for running convert utility which converts
+# PS documents to various types of image (e.g. PNG, GIF, JPEG format, 
+# Then use PagedImgPlug to deal with the the images
+# Chi-Yu Huang 08.2005
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use Cwd;
+use File::Basename
+;
+sub print_usage {
+    # note - we don't actually ever use most of these options...
+    print STDERR  
+	("pstoimg.pl wrapper for pstoimg.\n",
+	 "Usage: pstoimg [options] <PS-file> <output>\n",
+	 "Options:\n",
+	 "\t-convert_to\toutput image type for the PS\n"
+	 );
+    exit (1);
+}
+
+sub main {
+    my (@ARGV) = @_;
+    my ($convert_to);
+     
+    # read command-line arguments so that
+    # you can change the command in this script
+    if (!parsargv::parse(\@ARGV,
+			 'convert_to/.*/^', \$convert_to,
+			 )) {
+    	print_usage();
+    }
+ 
+    # Make sure the input file exists and can be opened for reading
+    if (scalar(@ARGV) != 2) {
+	print_usage();
+    }
+
+    my $input_filename = $ARGV[0];
+    my $output_filestem = $ARGV[1];
+    
+    # test that the directories exist to create the output file, or
+    # we should exit immediately. (File:: is included by util.pm)
+    &util::mk_dir($output_filestem) if (!-e $output_filestem);
+	    
+    my @dir = split (/(\/|\\)/, $input_filename);
+    my $input_basename = pop(@dir);
+    $input_basename =~ s/\.ps//i;
+    my $dir = join ("", @dir);
+    
+    if (!-r $input_filename) {
+	print STDERR "Error: unable to open $input_filename for reading\n";
+	exit(1);
+    }
+    # don't include path on windows (to avoid having to play about
+    # with quoting when GSDLHOME might contain spaces) but assume
+    # that the PATH is set up correctly.
+    $cmd = "convert";
+    #if ($ENV{'GSDLOS'} =~ /^windows$/);
+    #Convert utility will convert the PS to GIF Animation
+    my $output_filename = &util::filename_cat($output_filestem, $input_basename);
+    if ($convert_to eq "gif") {
+	$cmd .= " \"$input_filename\" \"$output_filename-%02d.$convert_to\"";
+    } else {
+	$cmd .= " \"$input_filename\" \"$output_filename.$convert_to\"";
+    }	
+
+    # system() returns -1 if it can't run, otherwise it's $cmds ret val.
+    # note we return 0 if the file is "encrypted"
+    $!=0;
+    if (system($cmd)!=0) {
+	print STDERR "Convert error for $input_filename $!\n";
+	# leave these for gsConvert.pl...
+	#&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	#&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+	return 1;
+    } else {
+	# command execute successfully
+	create_itemfile($output_filestem, $input_basename, $convert_to);
+    }
+    return 0;
+}
+
+sub create_itemfile
+{
+    my ($output_dir, $convert_basename, $convert_to) = @_;
+    opendir(DIR, $output_dir) || die "can't opendir $output_dir: $!";  
+    my $item_file = $output_dir."/".$convert_basename.".item";
+    open(FILE,">$item_file");
+    
+    print FILE "<PagedDocument>\n";
+
+    my $page_num = "";
+    @dir_files = grep {-f "$output_dir/$_"} readdir(DIR);
+
+    # Sort files in the directory by page_num    
+    sub page_num {
+	my ($dir) = @_;
+	my $pagenum = "";
+	if ($ENV{'GSDLOS'} =~ /^windows$/){
+	   ($pagenum) =($dir =~ m/^.*\.(\d+)$/i);   
+	} else {
+	   ($pagenum) =($dir =~ m/^.*-(\d+)\.(.*)$/i);
+	}
+	$pagenum = $pagenum || 1;
+	return $pagenum;
+    }
+
+    # sort the files in the directory in the order of page_num rather than lexically. 
+    @dir_files = sort { page_num($a) <=> page_num($b) } @dir_files;
+
+    foreach my $file (@dir_files){
+        if ($ENV{'GSDLOS'} =~ /^windows$/ && $convert_to ne "gif"){
+	  ($page_num) =($file =~ m/^.*\.(\d+)$/i); 
+	} else {
+	    ($page_num) =($file =~ m/^.*-(\d+)\.(.*)/i);   
+	}
+	# as the converter will convert the document to image files start from page 0
+	$page_num =$page_num + 1 if defined $page_num;
+	$page_num = 1 unless defined $page_num;
+	if ($file !~ /\.item/i){
+	    print FILE "   <Page pagenum=\"$page_num\" imgfile=\"$file\" txtfile=\"\"/>\n";
+	} 
+    }
+
+    print FILE "</PagedDocument>\n";
+    closedir DIR;
+    return "";
+}
+
+# indicate our error status, 0 = success
+exit (&main(@ARGV));
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/sendmail.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/sendmail.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/sendmail.pl	(revision 21112)
@@ -0,0 +1,89 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# sendmail.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# sendmail.pl is a simple wrapper around the Sendmail perl module written
+# by Milivoj Ivkovic <mi@alma.ch>
+
+# Input is either read from STDIN or, if the -msgfile option is set, read
+# from the given file.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use lib qq($ENV{'GSDLHOME'}/perllib/cpan);
+use Mail::Sendmail;
+use parsargv;
+
+if (!parsargv::parse(\@ARGV, 
+		     'to/@', \$to,
+		     'from/@', \$from,
+		     'subject/.*/', \$subject,
+		     'smtp/.*', \$smtp,
+		     'msgfile/.*/', \$msgfile)) {
+    print STDERR "\n";
+    print STDERR "sendmail.pl: A simple platform independant mail sending program.\n\n";
+    print STDERR "  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -to addr      Comma separated list of mail recipients\n";
+    print STDERR "   -from addr    Address of message sender\n";
+    print STDERR "   -subject str  Optional subject line\n";   
+    print STDERR "   -smtp server  The outgoing (SMTP) mail server\n";
+    print STDERR "   -msgfile file A file from which message to be sent is read.\n";
+    print STDERR "                 If not set message will be read in from STDIN\n";
+    die "\n";
+}
+
+my %mail = ('SMTP' => $smtp, 
+	    'To' => $to,
+	    'From' => $from,
+	    'Subject' => $subject,
+	    );
+
+my $msg = "";
+if ($msgfile ne "") {
+    if (!open (MSGFILE, $msgfile)) {
+	print STDERR "ERROR: Failed to open $msgfile. Message was not sent\n";
+	exit (-1);
+    }
+    undef $/;
+    $msg = <MSGFILE>;
+    $/ = "\n";
+    close MSGFILE;
+} else {
+    $msg = <STDIN>;
+}
+
+$mail{'Message'} = $msg;
+	
+if (!sendmail %mail) {
+    print STDERR "ERROR: Failed to send message\n";
+    print STDERR "'$Mail::Sendmail::error'\n";
+    exit (-1);
+}
Index: /main/tags/3.01/gsdl/bin/script/translate.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/translate.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/translate.pl	(revision 21112)
@@ -0,0 +1,488 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# translate.pl 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# translate.pl takes a translated macro file (filename passed in on command
+# line) and generates any images required by it. Check out english.dm for
+# an example of the format translate.pl expects
+
+# translate.pl uses gimp to generate images so needs gimp installed and set
+# up for scripting with perl
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use Gimp qw/:auto :DEFAULT/;
+use parsargv;
+use util;
+use unicode;
+
+# these html entities will be translated correctly when occurring in 
+# images. Any entities not in this list will not.
+my %rmap = ('auml' => chr (228),
+	    'euml' => chr (235),
+	    'iuml' => chr (239),
+	    'ouml' => chr (246),
+	    'uuml' => chr (252),
+	    'Auml' => chr (196),
+	    'Euml' => chr (203),
+	    'Iuml' => chr (207),
+	    'Ouml' => chr (214),
+	    'Uuml' => chr (220),
+	    'szlig' => chr (223),
+	    'aacute' => chr (225),
+	    'eacute' => chr (233),
+	    'iacute' => chr (237),
+	    'oacute' => chr (243),
+	    'uacute' => chr (250),
+	    'Aacute' => chr (193),
+	    'Eacute' => chr (201),
+	    'Iacute' => chr (205),
+	    'Oacute' => chr (211),
+	    'Uacute' => chr (218),
+	    'agrave' => chr (224),
+	    'egrave' => chr (232),
+	    'igrave' => chr (236),
+	    'ograve' => chr (242),
+	    'ugrave' => chr (249),
+	    'Agrave' => chr (192),
+	    'Egrave' => chr (200),
+	    'Igrave' => chr (204),
+	    'Ograve' => chr (210),
+	    'Ugrave' => chr (217),
+	    'ntilde' => chr (241),
+	    'Ntilde' => chr (209),
+	    'atilde' => chr (227),
+	    'Atilde' => chr (195),
+	    'otilde' => chr (245),
+	    'Otilde' => chr (213),
+	    'ccedil' => chr (231),
+	    'Ccedil' => chr (199),
+	    'ecirc' => chr (234),
+	    'Ecirc' => chr (202),
+	    'acirc' => chr (226),
+	    'Acirc' => chr (194),
+	    );
+
+my $hand_made = 0;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "translate.pl: Uses gimp to generate any images required by a\n";
+    print STDERR "              Greenstone macro file.\n\n";
+    print STDERR "  usage: $0 [options] macrofile\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -save_orig_file       edited macrofile will be written to\n";
+    print STDERR "                         macrofile.new leaving macrofile unchanged\n";
+    print STDERR "   -language_symbol      ISO abbreviation of language (e.g. German=de,\n";
+    print STDERR "                         French=fr, Maori=mi)\n";
+    print STDERR "   -image_dir directory  full path to directory in which to create images\n";
+    print STDERR "                         (defaults to `pwd`/images)\n\n";
+}
+
+sub gsdl_translate {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'save_orig_file', \$save_orig_file,
+			 'language_symbol/[A-Za-z]{2}', \$language_symbol,
+			 'image_dir/.*/images', \$image_dir)) {
+	&print_usage();
+	die "\n";
+    }
+
+    if ($image_dir eq "images") {
+	$image_dir = `pwd`;
+	chomp $image_dir;
+	$image_dir = &util::filename_cat ($image_dir, "images");
+    }
+
+    if (!defined $ARGV[0]) {
+	print STDERR "no macro file supplied\n\n";
+	&print_usage();
+	die "\n";
+    }
+    my $macrofile = $ARGV[0];
+    die "\nmacrofile $macrofile does not exist\n\n" unless -e $macrofile;
+
+    if (!-e $image_dir) {
+	mkdir ($image_dir, 511) || die "\ncouldn't create image_dir $image_dir\n\n";
+    }
+
+    open (INPUT, $macrofile) || die "\ncouldn't open $macrofile for reading\n\n";
+    open (OUTPUT, ">$macrofile.new") || die "\ncouldn't open temporary file $macrofile.new for writing\n\n";
+
+    &parse_file (INPUT, OUTPUT);
+
+    close OUTPUT;
+    close INPUT;
+
+    if (!$save_orig_file) {
+	`mv $macrofile.new $macrofile`;
+    }
+
+    print STDERR "\n\n";
+    print STDERR "translation of macro file $macrofile completed\n";
+    print STDERR "the translated macro file is $macrofile.new\n" if $save_orig_file;
+    print STDERR "\n";
+    if ($hand_made) {
+	print STDERR "$hand_made hand made images were found within $macrofile,\n";
+	print STDERR "these will need to be made by hand (grep $macrofile for 'hand_made'\n\n";
+    }
+    print STDERR "To add your new interface translation to Greenstone you'll need to:\n";
+    print STDERR "  1. Copy your new macro file to your GSDLHOME/macros directory\n";
+    print STDERR "  2. Add your new macro file to the macrofiles list in your\n";
+    print STDERR "     GSDLHOME/etc/main.cfg configuration file\n";
+    print STDERR "  3. Copy your newly created images from $image_dir to \n";
+    print STDERR "     GSDLHOME/images/$language_symbol/\n\n";
+    print STDERR "Access your new interface language by setting the language (l) cgi\n";
+    print STDERR "argument to '$language_symbol'\n\n";
+
+}
+
+sub parse_file {
+    my ($input, $output) = @_;
+
+    undef $/;
+    my $dmfile = <$input>;
+    $/ = "\n";
+
+    # process all the images
+
+    $dmfile =~ s/(?:^|\n)\#\#\s*\"([^\"]*)\"\s*\#\#\s*([^\s\#]*)\s*\#\#\s*([^\s\#]*)\s*\#\#(.*?)(?=(\n\#|\s*\Z))/&process_image ($1, $2, $3, $4)/esg;
+
+    # add language parameter to each macro
+    $dmfile =~ s/(\n\s*)(_[^_]*_)\s*(\[[^\]]*\])?\s*\{/$1 . &add_language_param ($2, $3)/esg;
+    
+    print $output $dmfile;
+}
+
+sub process_image {
+    my ($text, $image_type, $image_name, $image_macros) = @_;
+
+    my $origtext = $text;
+
+    $text =~ s/&(\d{3,4});/chr($1)/ge;
+    $text =~ s/&([^;]*);/$rmap{$1}/g;
+
+    # Default font is set by the individual scripts (flash_button-1.2.pl etc), usually lucida
+    my $fontchoice = "";
+
+    # special case for Kazakh images (can also handle Russian) - encode to match Kazakh font
+    if ($language_symbol eq "kz") {  # || $language_symbol eq "ru") {
+	$text = &unicode::unicode2singlebyte(&unicode::utf82unicode($text), "kazakh");
+	$fontchoice = " -foundry 2rebels -fontname \"helv kaz\"";
+    }
+    # special case for Russian images - font is koi8-r encoded
+    elsif ($language_symbol eq "ru") {
+	$text = &unicode::unicode2singlebyte(&unicode::utf82unicode($text), "koi8_r");
+	$fontchoice = " -foundry cronyx -fontname helvetica";
+    }
+    # special case for Thai images
+    elsif ($language_symbol eq "th") {
+	$text = &unicode::unicode2singlebyte(&unicode::utf82unicode($text), "windows_874");
+	$fontchoice = " -foundry monotype -fontname BrowalliaUPC -fontregistry tis620";
+    }
+    # special case for Ukrainian images - font is windows_1251 encoded 
+    elsif ($language_symbol eq "uk") {
+	$text = &unicode::unicode2singlebyte(&unicode::utf82unicode($text), "windows_1251");
+	$fontchoice = " -foundry rfx -fontname serene";
+    }
+
+    # edit image macros
+    $image_macros =~ s/(_httpimg_\/)(?:[^\/\}]*\/)?([^\}]*\.(?:gif|jpe?g|png))/$1$language_symbol\/$2/gs;
+
+    if ($image_type eq "top_nav_button") {
+
+	# generate images
+	my $options = "-text \"$text\" -filenamestem $image_name -image_dir $image_dir";
+	$options .= " -height 20 -whitespace";
+	$options .= $fontchoice;
+
+	# special case for Kazakh images (can also handle Russian)
+	if ($language_symbol eq "kz") {  # || $language_symbol eq "ru") {
+	    $options .= " -fontsize 10 -fontweight bold";
+	}
+	# special case for Russian images
+	elsif ($language_symbol eq "ru") {
+	    $options .= " -fontsize 10 -fontweight bold";
+	}
+	# special case for Thai images
+	elsif ($language_symbol eq "th") {
+	    $options .= " -fontsize 20 -fontweight bold";  # Browallia
+	}
+	# special case for Ukrainian images
+	elsif ($language_symbol eq "uk") {
+	    $options .= " -fontsize 10";
+	}
+	else {
+	    $options .= " -fontsize 12";
+	}
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# get width of new images and edit width macro
+	# my $fullfilename = &util::filename_cat ($image_dir, "${image_name}on.gif");
+	# &process_width_macro ($fullfilename, $image_name, \$image_macros);
+
+    }
+    elsif ($image_type eq "nav_bar_button") {
+
+	# generate on and off images
+	my $options = "-text \"$text\" -filenamestem $image_name -image_dir $image_dir";
+	$options .= " -height 17 -fixed_width -width 87";
+	$options .= $fontchoice;
+
+	# special case for Thai images
+	if ($language_symbol eq "th") {
+	    $options .= " -fontsize 18 -fontweight bold";  # Browallia
+	}
+	else {
+	    $options .= " -fontsize 17";
+	}
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# generate green image
+	$options = "-text \"$text\" -filenamestem ${image_name}gr -image_dir $image_dir";
+	$options .= " -height 17 -fixed_width -width 87";
+	$options .= " -bgcolor \"#96c19b\"";
+	$options .= $fontchoice;
+
+	# special case for Thai images
+	if ($language_symbol eq "th") {
+	    $options .= " -fontsize 18 -fontweight bold";  # Browallia
+	}
+	else {
+	    $options .= " -fontsize 17";
+	}
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# delete the unused light image
+	unlink(util::filename_cat($image_dir, "${image_name}gr" . "of.gif"));
+
+	# rename the dark image
+	&util::mv(util::filename_cat($image_dir, "${image_name}gr" . "on.gif"),
+		util::filename_cat($image_dir, "$image_name" . "gr.gif"));
+
+	# get width of new images and edit width macro
+	my $fullfilename = &util::filename_cat ($image_dir, "${image_name}on.gif");
+	&process_width_macro ($fullfilename, $image_name, \$image_macros);
+
+    }
+    elsif ($image_type eq "document_button") {
+
+	# generate on and off images
+	my $options = "-text \"$text\" -filenamestem $image_name -image_dir $image_dir";
+	$options .= " -fixed_width -whitespace";
+	$options .= $fontchoice;
+
+	# special case for Kazakh images (can also handle Russian)
+	if ($language_symbol eq "kz") {  # || $language_symbol eq "ru") {
+	    $options .= " -fontsize 8";
+	}
+	# special case for Russian images
+	elsif ($language_symbol eq "ru") {
+	    $options .= " -fontsize 8";
+	}
+	# special case for Thai images
+	elsif ($language_symbol eq "th") {
+	    $options .= " -fontsize 22 -fontweight bold";  # Browallia
+	}
+	# special case for Ukrainian images
+	elsif ($language_symbol eq "uk") {
+	    $options .= " -fontsize 8";
+	}
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# get width of new images and edit width macro
+	# my $fullfilename = &util::filename_cat ($image_dir, "${image_name}on.gif");
+	# &process_width_macro ($fullfilename, $image_name, \$image_macros);
+
+    }
+    elsif ($image_type eq "collector_bar_button") {
+
+	$text =~ s/\\n/\n/g;
+	if ($text !~ /\n/) {
+	    # Format the text so it is centered, one word per line
+	    local @textparts = split(/[ \t\n]+/, $text);
+	    local $maxlength = 0;
+	    for ($i = 0; $i < scalar(@textparts); $i++) {
+		if (length($textparts[$i]) > $maxlength) {
+		    $maxlength = length($textparts[$i]);
+		}
+	    }
+	    $text = "";
+	    for ($i = 0; $i < scalar(@textparts); $i++) {
+		if (length($textparts[$i]) < $maxlength) {
+		    $text .= ' ' x ((($maxlength - length($textparts[$i])) / 2) + 1);
+		}
+		$text .= $textparts[$i] . "\n";
+	    }
+	}
+
+	# generate on and off images (yellow)
+	my $options = "-text \"$text\" -filenamestem yc$image_name -image_dir $image_dir";
+	$options .= " -width 77 -height 26 -fixed_width -whitespace -fontsize 13";
+	$options .= $fontchoice;
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# generate on and off images (green)
+	$options = "-text \"$text\" -filenamestem gc$image_name -image_dir $image_dir";
+	$options .= " -width 77 -height 26 -fixed_width -whitespace -fontsize 13";
+	$options .= " -bgcolor \"#96c19b\"";
+	$options .= $fontchoice;
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# generate on and off images (grey) - only the light image (off) is used
+	$options = "-text \"$text\" -filenamestem nc$image_name -image_dir $image_dir";
+	$options .= " -width 77 -height 26 -fixed_width -whitespace -fontsize 13";
+	$options .= " -bgcolor \"#7E7E7E\" -fontcolor \"#a0a0a0\"";
+	$options .= $fontchoice;
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/flash_button-1.2.pl $options`;
+
+	# delete the unused dark image
+	unlink(util::filename_cat($image_dir, "nc$image_name" . "on.gif"));
+
+    }
+    elsif ($image_type eq "green_bar_left_aligned") {
+
+	# generate green bar image (we're assuming these bars are always 537
+	# pixels and are never stretched by excess text
+	my $options = "-text \"$text\" -filename ${image_name}.gif -image_dir $image_dir";
+	$options .= " -dont_center -width 537 -width_space 15";
+	$options .= $fontchoice;
+
+	# special case for Thai images
+	if ($language_symbol eq "th") {
+	    $options .= " -fontsize 18 -fontweight bold";  # Browallia
+	}
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/green_bar-1.2.pl $options`;
+
+    }
+    elsif ($image_type eq "green_title") {
+	
+	# read the width if it is specified in $image_macros
+        my ($width) = $image_macros =~ /_width${image_name}x?_\s*[^\{]*\{(\d+)\}/;
+        $width = 200 unless ($width);
+
+        # generate green title image
+	my $options = "-text \"$text\" -filename ${image_name}.gif -image_dir $image_dir";
+	$options .= " -width $width -height 57 -stripe_alignment right -text_alignment right";
+        $options .= $fontchoice;
+
+        # special case for Thai images
+        if ($language_symbol eq "th") {
+	    $options .= " -fontsize 50 -fontweight bold";  # Browallia
+	}
+        else {
+	    $options .= " -fontsize 26 -fontweight bold";
+	}
+
+	`$ENV{'GSDLHOME'}/bin/script/gimp/title_icon-1.2.pl $options`;
+
+	# get width of resulting image and edit _width..._ macro in $image_macros
+        # (no longer needed since we always resize to the width read from $image_macros.)
+        # my $fullfilename = &util::filename_cat ($image_dir, "${image_name}.gif");
+	# &process_width_macro ($fullfilename, $image_name, \$image_macros);
+
+    }
+    elsif ($image_type eq "hand_made") {
+
+	$hand_made ++;
+
+    }
+    else {
+	
+	print STDERR "WARNING (translate.pl): unknown image type found ($image_type)\n";
+
+    }
+
+    return "\n\#\# \"$origtext\" \#\# $image_type \#\# $image_name \#\#$image_macros";
+}
+
+sub process_width_macro {
+    my ($filename, $image_name, $image_macros) = @_;
+
+    my $img_info = &get_img_info ($filename);
+    $$image_macros =~ s/(_width${image_name}x?_\s*(?:\[[^\]]*\])?\s*\{)(\d+)(\})/$1$img_info->{'width'}$3/s;
+}
+
+sub add_language_param {
+    my ($macroname, $paramlist) = @_;
+
+    my $first = 1;
+    if (defined $paramlist) {
+	$paramlist =~ s/^\[//;
+	$paramlist =~ s/\]$//;
+	my @params = split /\,/, $paramlist;
+	$paramlist = "";
+	foreach $param (@params) {
+	    # remove any existing language parameter
+	    if ($param !~ /^l=/) {
+		$paramlist .= "," unless $first;
+		$paramlist .= $param;
+		$first = 0;
+	    }
+	}
+    }
+    $paramlist .= "," unless $first;
+    $paramlist .= "l=" . $language_symbol;
+    return "$macroname [$paramlist] {";
+}
+
+sub get_img_info {
+    my ($imagefile) = @_;
+    my %info = ();
+
+    if (!-r $imagefile) {
+	print STDERR "ERROR (translate.pl): couldn't open $imagefile to get dimensions\n";
+	$info{'width'} = 0;
+	$info{'height'} = 0;
+    } else {
+	my $image = gimp_file_load (RUN_NONINTERACTIVE, $imagefile, $imagefile);
+	$info{'width'} = gimp_image_width ($image);
+	$info{'height'} = gimp_image_height ($image);
+    }
+
+    return \%info;
+}
+
+sub query {
+
+  gimp_install_procedure("gsdl_translate", "translate macro files and create images",
+                        "", "Stefan Boddie", "Stefan Boddie", "2000-03-14",
+                        "<Toolbox>/Xtns/gsdl_translate", "*", &PROC_EXTENSION,
+                        [[PARAM_INT32, "run_mode", "Interactive, [non-interactive]"]], []);
+}
+
+Gimp::on_net {   gsdl_translate; };
+exit main;
Index: /main/tags/3.01/gsdl/bin/script/unbuildv1.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/unbuildv1.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/unbuildv1.pl	(revision 21112)
@@ -0,0 +1,745 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# unbuildv1.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# this program will decompress all the text from a built index
+# and return it to gml format - this version of the program is designed to
+# work on collections built with the first version of the greenstone software
+# it extracts the browse classifications from the gdbm file, and recreates 
+# them. It also creates a default collect.cfg file.
+
+# To run, this program needs the old archive directory (called archives),
+# the index directory with the text and one other index, not paragraphs!! -
+# use one of the section indexes. Also needs the build.cfg file (in the
+# index dir) - used in creating a collect.cfg file. If build.cfg is not
+# present, the parts of the collect.cfg file relating to indexes will not
+# be entered.
+
+# Stefan altered this script slightly August 2002 to use
+# $GSDLHOME/bin/$GSDLOS/mgquery_old instead of the old mgquery in
+# nzdl-1.2. mgquery_old is a statically linked binary compiled from
+# mg-1.3d. It was compiled with SHORT_SUFFIX set so doesn't need the index
+# file extensions to be altered. 
+# Also changed the command line options a little.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    $FileHandle = 'FH000';
+}
+
+use doc;
+use docsave;
+use util;
+use parsargv;
+use FileHandle;
+use English;
+use cfgread;
+use unicode;
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+
+# globals
+$collection = "";  # the collection name
+$index = "";  # the selected index (like stt/unu)
+$textdir = ""; # the textdir (like text/unu)
+$toplevelinfo = []; #list of document OIDs
+%infodb = (); #hash into GDBM file
+$classifyinfo = []; # list of classifications
+$doc_classif_info = {};  # hash of OIDs->classifications they belong to
+$collect_cfg = {}; #data for the configuration file
+$mgread = ++$FileHandle;
+$mgwrite = ++$FileHandle;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -old directory         The directory in which the old collection lives\n";
+    print STDERR "   -new directory         The directory in which to put the new collection\n";
+}
+
+&main ();
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'old/.*/', \$old,
+			 'new/.*/', \$new)) {
+	&print_usage();
+	die "\n";
+    }
+
+    die "$old does not exist\n\n" unless (-d $old);
+    my $old_indexdir = &util::filename_cat($old, "index");
+    my $old_archivedir = &util::filename_cat($old, "archives");
+
+    my $new_archivedir = &util::filename_cat($new, "archives");
+    my $new_etcdir = &util::filename_cat($new, "etc");
+    if (!-e $new_archivedir) {
+	&util::mk_all_dir($new_archivedir);
+    }
+    if (!(-d $new_etcdir)) {
+	&util::mk_all_dir($new_etcdir);
+    }
+
+    my $gdbmfile = &get_gdbmfile($old_indexdir); # sets $collection and $textdir
+    &set_index($old_indexdir);  # sets $index (just chooses one index)
+
+    my $buildcfgfile = &util::filename_cat($old_indexdir, "build.cfg");
+    my $colcfgfile = &util::filename_cat($new_etcdir, "collect.cfg");
+
+    # don't need this but giving it a value stops an annoying undefined
+    # value warning
+    $ENV{'GSDLCOLLECTDIR'} = 'tmp';
+
+    &add_default_cfg();
+    &add_index_cfg($buildcfgfile);
+
+    # work out all the classifications from the gdbm file, info for each doc
+    # (ie which classifications they belong to, are kept in $doc_classif_info
+    &get_classifyinfo($gdbmfile); #puts a list of classifications into classifyinfo
+    &get_toplevel_OID($gdbmfile); # puts a list of the top level document OIDs into $toplevelinfo
+
+    # read ldb file into %infodb
+    &read_gdbm($gdbmfile);
+
+    #this makes the files specifying the hierarchy of subjects, titles etc
+    foreach my $classify (@$classifyinfo) {
+	&make_info_file($new_etcdir, $classify);
+    }
+    
+    # write out the collect.cfg
+    &output_cfg_file($colcfgfile);
+
+    &openmg($old_indexdir);
+
+    # read the archive information file
+    my $archive_info_filename = &util::filename_cat ($new_archivedir, "archives.inf");
+    my $archive_info = new arcinfo ();
+
+    # create a docsave object to process the documents
+    my $processor = new docsave ("", $archive_info, $verbosity);
+    $processor->setarchivedir ($new_archivedir);
+
+    my ($doc_obj, $hashref, $children);
+    print STDERR "processing documents now\n" if $verbosity >=2;
+    foreach $oid (@$toplevelinfo) {
+	$value = $infodb{$oid};
+	$hashref={};
+	$children = [];
+	&get_metadata($value, $hashref);
+	$doc_obj = new doc ();
+	$doc_obj->set_OID($oid);
+	my ($olddir) = $hashref->{'archivedir'}; # old dir for this doc, where images are stored
+	$top = $doc_obj->get_top_section();
+	&add_section_content ($doc_obj, $top, $hashref, $olddir);
+	&add_classification_metadata($oid, $doc_obj, $top);
+	&add_cover_image($doc_obj, $old_archivedir, $olddir);
+	&get_children($hashref, $children);
+	&recurse_sections($doc_obj, $children, $oid, $top, $olddir) if (defined ($children));
+
+	#change OID to new format
+	$doc_obj->delete_OID();
+	$doc_obj->set_OID();
+	$processor->process($doc_obj);
+#	last;
+    }
+    print STDERR "\n";
+   
+    &closemg();
+
+    # write out the archive information file
+    $archive_info->save_info($archive_info_filename);
+   
+    
+}
+
+# returns the path to the gdbm info database - also
+# sets the $collection and $textdir global variable
+sub get_gdbmfile {
+    my ($indexdir) = @_;
+    my ($gdbmfile);
+
+    opendir (DIR, $indexdir) || die "Couldn't open directory $indexdir\n\n";
+    my @conts = readdir DIR;
+    close DIR;
+
+    foreach $file (@conts) {
+	if ($file =~ /text$/) {
+	    $textdir = $file;
+	    last;
+	}
+    }
+    die "No text directory found in $indexdir\n\n" 
+	unless defined $textdir && $textdir =~ /text$/;
+
+    $gdbmfile = &util::filename_cat ($indexdir, $textdir);
+
+    opendir (DIR, $gdbmfile) || die "Couldn't open directory $gdbmfile\n\n";
+    @conts = readdir DIR;
+    close DIR;
+
+    foreach my $file (@conts) {
+	if ($file =~ /^(.*?)\.(?:ldb|bdb)$/) {
+	    $collection = $1;
+	    $gdbmfile = &util::filename_cat ($gdbmfile, $file);
+	    last;
+	}
+    }
+    
+    if (defined $collection && $collection =~ /\w/) {
+	$textdir = &util::filename_cat ($textdir, $collection);
+    } else {
+	die "collection global wasn't set\n";
+    }
+    return $gdbmfile if (-e $gdbmfile);
+    die "Couldn't find gdbm info database in $indexdir\n\n";
+}
+
+
+#this function will probably need to be changed for any different file formats
+sub get_toplevel_OID {
+    my ($gdbmfile) = @_;
+
+    open (DB2TXT, "db2txt $gdbmfile |") || die "couldn't open pipe to db2txt\n";
+    print STDERR "Finding all top level sections from $gdbmfile\n" if $verbosity >= 2;
+
+    $/ = '-' x 70;
+    my $entry = "";
+    while (defined ($entry = <DB2TXT>)) {
+	next unless $entry =~ /\w/;  #ignore blank entries
+	$entry =~ s/\n+/\\n/g;  # replace multiple \n with single \n
+	my ($key, $value) = $entry =~ /\[([^\]]*)\](.*)/;
+
+	next unless ($key =~ /B\.\d*$/); # assumes top level OID is like
+	                                 # B.12
+
+	push( @$toplevelinfo, $key);
+   
+    }
+
+    $/ = "\n";
+}
+    
+# gets all the metadata from a gdbm file entry, and puts it into a hashref
+#this has changed for old style formats
+sub get_metadata {
+
+    my ($gdb_str_ref, $hashref) = @_;
+#    print STDERR $gdb_str_ref;
+    my @entries = split(/\n/, $gdb_str_ref);
+
+    my $hastext = 1;
+    foreach $entry (@entries) {
+	
+	my($key, $value) = ($entry =~ /^<([^>]*)>(.*?)$/ );
+
+	next if ($key eq "p");
+	next if ($key eq "j");
+	
+	$key = "Title" if $key eq "t";
+	$key = "docnum" if $key eq "d";	
+	if ($key eq "o") {
+	    $key = "archivedir";
+	    $value =~ s/HASH/hash/;
+	}
+	if ($key eq "c") {
+	    $key = "contains";
+	    $hastext=0;}
+
+	$$hashref{$key} .= $value;
+						  
+    }
+    
+    $$hashref{'hastxt'} = $hastext;
+
+}					  
+
+# takes a hashref containing the metadata for a gdbmfile entry, and
+# extracts the childrens numbers (from the 'contains' entry).  assumes
+# format is like B.14.1:B.24.2:B.24.3 returns a list like 1 2 3 4
+sub get_children {	
+    my ($hashref, $children) = @_; 
+
+    my $childs = $hashref->{'contains'};
+    if (defined ($childs)) {
+	@xchildren = split /\:/, $childs;
+	foreach $kid (@xchildren) {
+	    my ($child)= $kid =~ /\.([^\.]*)(?:\\n|$)/;
+	    push (@$children, $child);
+	}
+#	print STDERR "children are: ", @$children;
+    }
+    else {
+	$children = [];
+    }
+}
+
+#takes a hashref containing the metadata for a gdbmfile entry, and extracts 
+#the childrens numbers (from the 'contains' entry).	
+#assumes format is like B.14.1:B.24.2:B.24.3
+#returns a list with the full child name ie B.14.1 B.14.2 etc
+#used for classification stuff
+sub get_whole_children {	
+
+    my ($hashref, $children) = @_; 
+
+    my $childs = $hashref->{'contains'};
+    if (defined ($childs)) {
+	@$children = split /\:/, $childs;
+	#print STDERR "children are: ", @$children;
+    }
+    else {
+	$children = [];
+    }
+}
+    
+sub recurse_sections {
+    my ($doc_obj, $children, $parentoid, $parentsection, $olddir) = @_;
+
+    foreach $child (sort numerically @$children) {
+	$doc_obj->create_named_section("$parentsection.$child");
+	my $value = $infodb{"$parentoid.$child"};
+	my $hashref={};
+	&get_metadata($value, $hashref); # get childs metadata
+	my $newchildren = [];
+	&get_children($hashref, $newchildren); # get childs children
+	#add content fo rcurrent section
+	&add_section_content($doc_obj, "$parentsection.$child", $hashref, $olddir);
+	# process all the children if there are any
+	&recurse_sections($doc_obj, $newchildren, "$parentoid.$child", "$parentsection.$child", $olddir)
+	    if (defined ($newchildren));
+    }
+}					     
+
+sub add_section_content {
+    my ($doc_obj, $cursection, $hashref, $olddir) = @_;
+    
+    foreach $key (keys %$hashref) {
+	#dont need to store these metadata
+	next if $key =~ /(contains|docnum|hastxt|doctype|archivedir|classifytype)/i;
+	my @items = split /@/, $hashref->{$key};
+	map {$doc_obj->add_metadata ($cursection, $key, $_); } @items;
+	
+    }
+
+    my ($hastext) = $hashref->{'hastxt'} =~ /(0|1)/;
+    if ($hastext) {
+	
+	my ($docnum)= $hashref->{'docnum'} =~ /(\d*)/;
+	my $text = &get_text($docnum);
+	$doc_obj->add_text ($cursection, $text);
+	
+	my (@images) = $text =~ /<img.*?src=\"([^\"]*)\"[^>]*>/g;
+	
+	if (scalar(@images)>0) {
+	    
+	    foreach $img (@images) {
+		my ($assoc_file) = $img =~ /([^\/\\]*\..*)$/; #the name of the image
+		$img =~ s/_httpcollection_/$old/; #replace _httpcollection_ with path to old collection
+		$olddir =~ s/HASH/hash/;
+		$img =~ s/_thisOID_/$olddir/; #replace _thisOID_ with old archivedir name   #the path to the image
+		$img = lc $img; # doc names and path upper case in file, lower
+		# case in directory
+		if (-e $img) {
+		    $doc_obj->associate_file($img, $assoc_file);
+		} else {
+		    print STDERR "WARNING: $img does not exist\n";
+		}
+	    }
+	    
+	}
+    }
+}
+
+
+sub add_classification_metadata {
+
+    my ($oid, $doc_obj, $cursection) = @_;
+    
+    if (defined $doc_classif_info->{$oid}) {
+
+	$hashref = $doc_classif_info->{$oid};
+	
+	foreach $key (keys %$hashref) {
+	    my @items = @{$hashref->{$key}};
+	    map {$doc_obj->add_metadata ($cursection, $key, $_); } @items;
+	}
+    }
+}
+# picks up the cover image "cover.jpg" from the old archives directory.
+sub add_cover_image {
+    my ($doc_obj, $archivedir, $olddir) =  @_;
+    $assoc_file = "cover.jpg";
+    $img = &util::filename_cat($archivedir, $olddir, $assoc_file);
+
+    if (-e $img) {
+	$doc_obj->associate_file($img, $assoc_file);
+    } else {
+	print STDERR "WARNING: couldn't find $img\n";
+    }
+}
+
+sub set_index {
+    my ($indexdir) = @_;
+
+    # check that $collection has been set
+    die "collection global was not set\n"
+	unless defined $collection && $collection =~ /\w/;
+
+    # find an index (just use first non-text and non-paragraph directory we
+    # come across in $indexdir)
+    opendir (INDEXDIR, $indexdir) || die "couldn't open directory $indexdir\n";
+    my @indexes = readdir INDEXDIR;
+    close INDEXDIR;
+    foreach $i (@indexes) {
+	next if $i =~ /text$/i || $i =~ /\./ || $i =~ /^p/i;
+	$index = &util::filename_cat ($i, $collection);
+	last;
+    }
+}
+
+
+#########################################################################
+
+################ functions involving mg ################################
+sub get_text {
+    my ($docnum) = @_;
+
+    print STDERR "." if $verbosity >= 2;
+    &mgcommand ($docnum);
+
+   <$mgread>;	# eat the document separator
+
+    my $text = "";
+    my $line = "";
+
+    while (defined ($line = <$mgread>))
+    {
+	last if $line =~ /^<\/mg>/;
+	$text .= $line;
+    }
+
+    # Read in the last statement, which should be:
+    #  "dd documents retrieved."
+    <$mgread>;
+
+    return $text;
+}
+
+
+sub numerically {$a <=> $b;}
+
+
+
+# mg stuff
+
+sub openmg {
+    my ($indexdir) = @_;
+
+    die "Unable to start mgquery." unless
+	&openpipe($mgread, $mgwrite, "mgquery_old -d $indexdir -f $index -t $textdir");
+
+    $mgwrite->autoflush();
+    &mgcommand('.set expert true');
+    &mgcommand('.set terminator "</mg>\n"');
+    &mgcommand('.set mode text');
+    &mgcommand('.set query docnums');
+    &mgcommand('.set term_freq off');
+}
+
+sub closemg {
+    &mgcommand (".quit");
+    close($mgread);
+    close($mgwrite);
+}
+
+sub mgcommand {
+    my ($command) = @_;
+
+    return if $command =~ /^\s*$/;  #whitespace
+    #print STDERR "command: $command\n";
+    print $mgwrite "$command\n";
+
+    # eat up the command executed which is echoed
+    <$mgread>;
+}
+
+# openpipe(READ, WRITE, CMD)
+# 
+# Like open2, except CMD's stderr is also redirected.
+# 
+sub openpipe
+{
+    my ($read, $write, $cmd) = @_;
+    my ($child_read, $child_write);
+
+    $child_read = ++$FileHandle;
+    $child_write = ++$FileHandle;
+
+    pipe($read, $child_write) || die "Failed pipe($read, $child_write): $!";
+    pipe($child_read, $write) || die "Failed pipe($child_read, $write): $!";
+    my $pid;
+
+    if (($pid = fork) < 0) {
+        die "Failed fork: $!";
+    } elsif ($pid == 0) {
+        close($read);
+        close($write);
+        open(STDIN, "<&$child_read");
+        open(STDOUT, ">&$child_write");
+        open(STDERR, ">&$child_write");
+        exec($cmd);
+        die "Failed exec $cmd: $!";
+    }
+
+    close($child_read);
+    close($child_write);
+
+    $write->autoflush();
+    $read->autoflush();
+
+    return 1;
+}
+
+
+######################################################################
+
+############# functions to do with the classificaiton stuff ##########
+sub get_classifyinfo {
+    my ($gdbmfile) = @_;
+
+    open (DB2TXT, "db2txt $gdbmfile |") || die "couldn't open pipe to db2txt\n";
+    print STDERR "Finding all classification sections from $gdbmfile\n" ;
+
+    $/ = '-' x 70;
+    my $entry = "";
+    while (defined ($entry = <DB2TXT>)) {
+	next unless $entry =~ /\w/;  #ignore blank entries
+	$entry =~ s/\n+/\\n/g;  # replace multiple \n with single \n
+	my ($key, $value) = $entry =~ /\[([^\]]*)\](.*)/;
+
+	next unless ($key =~/^[A-Z]$/); # assumes classification OID is like
+	                                 # C or T etc
+
+	push( @$classifyinfo, $key);
+   
+    }
+
+    $/ = "\n";
+}
+
+#this creates the classification files needed for the hierarchy classifier
+#used for subjects, titles, orgs etc
+#also adds in entries to the collect_cfg hash    
+sub make_info_file {
+    my ($etcdir, $classifier) = @_;
+    my $info_file = "";
+
+    $info_file = &util::filename_cat($etcdir, "CL.${classifier}.txt");
+    $classifier_name = "CL.$classifier";
+    print STDERR "classification $classifier will be called CL.$classifier\n";
+
+    open (OUTDOC, ">$info_file" ) || die  "couldn't open file $info_file\n";
+	
+    my $entry = $infodb{$classifier};
+    #print STDERR "entry = $entry\n";
+    my $hashref = {};
+    &get_metadata($entry, $hashref);
+    my $children=[];
+    &get_whole_children($hashref, $children);
+    foreach $child (@$children) {
+	&process_entry(OUTDOC, $classifier_name, $child);
+    }
+  
+    close OUTDOC;
+    
+    &add_classify_cfg($classifier, $classifier_name, $info_file);
+    
+}
+
+
+sub process_entry {
+    my ($handle, $classifier_name, $classify_id) = @_;
+
+    my $value = $infodb{$classify_id};
+    my $hashref={};
+    &get_metadata($value, $hashref);
+    my $title = $hashref->{'Title'};
+
+    &add_line($handle, $classify_id, $title);
+    
+    my $children = [];
+    &get_whole_children($hashref, $children);
+    foreach $child (@$children) {
+	if (&is_document($child)) {
+	    &add_doc_metadata($child, $classifier_name, $classify_id);
+	}
+	else {
+	    &process_entry($handle, $classifier_name, $child);
+	}
+    }
+}
+
+sub add_doc_metadata {
+    my ($doc_id, $classifier_name, $classifier_id) = @_;
+
+    #add entry to doc database
+    #print STDERR "at doc level, docnum=$classify_id\n";
+    $doc_classif_info->{$doc_id}={} unless defined $doc_classif_info->{$doc_id};
+    $doc_classif_info->{$doc_id}->{$classifier_name}=[] unless
+	defined $doc_classif_info->{$doc_id}->{$classifier_name};
+    push (@{$doc_classif_info->{$doc_id}->{$classifier_name}}, $classifier_id); 
+}
+
+sub add_line {
+    my ($handle, $classify_id, $title) = @_;
+    $title = &unicode::ascii2utf8(\$title);
+    my ($num) = $classify_id =~ /^[A-Z]\.(.*)$/; #remove the C. from the front
+    print $handle "$classify_id\t$num\t\"$title\"\n";
+}
+
+sub is_document {
+    my ($oid) = @_;
+    return 1 if $oid =~ /^B\.\d/;
+    return 0;
+}
+    
+########################################################################
+
+########## stuff for producing collect.cfg file ###########################
+
+sub add_default_cfg {
+
+    $username=`whoami`;
+    $username=`logname` unless defined $username;
+    $username="a_user" unless defined $username;
+    $username =~ s/\n//;
+    $collect_cfg->{'creator'}="$username\@cs.waikato.ac.nz";
+    $collect_cfg->{'maintainer'}="$username\@cs.waikato.ac.nz";
+    $collect_cfg->{'public'}="true";
+    
+    $collect_cfg->{'plugin'}=[];
+    push (@{$collect_cfg->{'plugin'}}, ["GAPlug"]);
+    push (@{$collect_cfg->{'plugin'}}, ["ArcPlug"]);
+    push (@{$collect_cfg->{'plugin'}}, ["RecPlug"]);
+
+    $collect_cfg->{'format'}={};
+    $collect_cfg->{'format'}->{'DocumentImages'}="true";
+    $collect_cfg->{'format'}->{'DocumentText'} = 
+	"\"<h3>[Title]</h3>\\\\n\\\\n<p>[Text]\"";
+    $collect_cfg->{'format'}->{'SearchVList'} =
+	"\"<td valign=top>[link][icon][/link]</td><td>{If}{[parent(All': '):Title],[parent(All': '):Title]:}[link][Title][/link]</td>\"";
+
+    $collect_cfg->{'collectionmeta'}={};
+    $collect_cfg->{'collectionmeta'}->{'collectionname'}="\"$collection\"";
+    $collect_cfg->{'collectionmeta'}->{'iconcollection'}="\"_httpprefix_/collect/$collection/images/$collection.gif\"";
+    $collect_cfg->{'collectionmeta'}->{'iconcollectionsmall'}="\"_httpprefix_/collect/$collection/images/${collection}sm.gif\"";
+    $collect_cfg->{'collectionmeta'}->{'collectionextra'} = "\"This is a collection rebuilt from CDROM.\"";
+
+}
+
+sub add_index_cfg {
+  my ($buildfile) = @_;
+
+  my $data={};
+  $collect_cfg->{'indexes'}=[];
+  if (-e $buildfile) {
+      $data=&cfgread::read_cfg_file($buildfile, '^(this)$', '^(indexmap)$'); 
+      foreach my $i (@{$data->{'indexmap'}}) {
+          my ($thisindex, $abbrev)= split (/\-\>/, $i);
+          push (@{$collect_cfg->{'indexes'}}, $thisindex);
+	  $collect_cfg->{'defaultindex'} = $thisindex unless defined 
+	      $collect_cfg->{'defaultindex'};
+          $name=&get_index_name($thisindex);
+	  $thisindex=".$thisindex";
+          $collect_cfg->{'collectionmeta'}->{$thisindex} = "\"$name\"";
+      } 
+  }
+  else {
+      print STDERR "Couldn't read $buildfile, could not add index data to configuration file\n";
+  }
+
+}
+
+sub get_index_name {
+    my ($thisindex) = @_;
+    return "paragraphs" if $thisindex =~ /paragraph/;
+    return "chapters" if $thisindex =~ /section.*text/;
+    return "titles" if $thisindex =~ /Title/;
+    return "other";
+}
+
+sub add_classify_cfg {
+
+    my ($classify, $metadata, $file) = @_;
+    $collect_cfg->{'classify'} = [] unless defined $collect_cfg->{'classify'};
+
+    my ($title);
+    $title = "Howto" if $classify eq 'H';
+    $title = "Subject" if $classify eq 'C';
+    $title = "Organization" if $classify eq 'O';
+    $title = "Title" if $classify eq 'T';
+    $title = "Other" unless defined $title;
+
+    my ($filename) = $file =~ /\/([^\/]*)$/;
+    my $entry = "Hierarchy -hfile $filename -metadata $metadata -buttonname $title -sort Title";
+    $entry .= " -hlist_at_top" if $title eq "Title";
+    push (@{$collect_cfg->{'classify'}},[$entry]);
+   
+
+}
+
+sub output_cfg_file {
+
+    my ($collfile) = @_;
+    &cfgread::write_cfg_file($collfile, $collect_cfg, 
+			     '^(creator|maintainer|public|beta|defaultindex)$',
+                            '^(indexes)$', '^(format|collectionmeta)$',
+                            '^(plugin|classify)$');
+}
+
+sub read_gdbm {
+    my ($filename) = @_;
+
+    open (PIPEIN, "db2txt \"$filename\" |") || die "couldn't open pipe from db2txt\n";
+    my $line = ""; my $key = ""; my $value = "";
+    while (defined ($line = <PIPEIN>)) {
+	if ($line =~ /^\[([^\]]+)\]$/) {
+	    $key = $1;
+	} elsif ($line =~ /^-{70}$/) {
+	    $infodb{$key} = $value;
+	    $value = "";
+	    $key = "";
+	} else {
+	    $value .= $line;
+	}
+    }
+    close PIPEIN;
+}
Index: /main/tags/3.01/gsdl/bin/script/unbuildv2.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/unbuildv2.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/unbuildv2.pl	(revision 21112)
@@ -0,0 +1,786 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# unbuildv2.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# this program will decompress all the text from a built index
+# and return it to gml format -
+#this is based on the format current in Nov 1999. (version two)
+
+# Stefan updated unbuildv1.pl in August 2002 but unbuildv2.pl was not
+# updated. It probably needs some work done before using.
+
+# Katherine updated this but apparently it still doesn't work.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    $FileHandle = 'FH000';
+}
+
+use unbuildutil;
+use doc;
+use docsave;
+use util;
+use parsargv;
+use GDBM_File;
+use FileHandle;
+use English;
+use cfgread;
+use unicode;
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+
+# globals
+$collection = "";  # the collection name
+$index = "";  # the selected index (like stt/unu)
+$textdir = ""; # the textdir (like text/unu)
+$toplevelinfo = []; #list of document OIDs
+%infodb = (); #hash into GDBM file
+$classifyinfo = []; # list of classifications
+$doc_classif_info = {};  # hash of OIDs->classifications they belong to
+$collect_cfg = {}; #data for the configuration file
+
+$mgread = ++$FileHandle;
+$mgwrite = ++$FileHandle;
+
+
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -indexdir directory    The index to be decompressed (defaults to ./index)\n";
+    print STDERR "   -archivedir directory  Where the converted material ends up (defaults to ./archives.new\n";
+    print STDERR "   -removeold             Will remove the old contents of the archives\n";
+    print STDERR "                          directory -- use with care\n\n";
+}
+
+&main ();
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'indexdir/.*/index', \$indexdir,
+			 'archivedir/.*/archives.new', \$archivedir,
+			 'removeold', \$removeold)) {
+	&print_usage();
+	die "\n";
+    }
+
+    die "indexdir $indexdir does not exist\n\n" unless (-d $indexdir);
+    $indexdir =~ s/\/$//;
+    if (-d $archivedir) {
+	if ($removeold) {
+	    print STDERR "Warning - removing current contents of the archives directory $archivedir\n";
+	    print STDERR "          in preparation for the import\n";
+	    sleep(5); # just in case...
+	    &util::rm_r ($archivedir);
+	}
+    } else {
+	&util::mk_all_dir ($archivedir);
+    }
+
+    $etcdir = "./etc";
+    if (!(-d $etcdir)) {
+	&util::mk_all_dir ($etcdir);
+    }
+
+
+    my $gdbmfile = &get_gdbmfile ($indexdir); #sets $collection and $textdir
+    &set_index ();  # sets $index (just chooses one index)
+
+
+    my $buildcfgfile = &util::filename_cat($indexdir, "build.cfg");
+    my $colcfgfile = &util::filename_cat($etcdir, "collect.cfg");
+
+    &add_default_cfg();
+    &add_index_cfg($buildcfgfile);
+
+
+    #work out all the classifications from the gdbm file, info for each doc
+#(ie which classifications they belong to, are kept in $doc_classif_info
+    &get_classifyinfo ($gdbmfile); #puts a list of classifications into classifyinfo
+
+    &get_toplevel_OID ($gdbmfile); # puts a list of the top level document OIDs into $toplevelinfo
+
+    #tie (%infodb, "GDBM_File", $gdbmfile, 1, 0);
+    
+    #read ldb file into %infodb
+    &unbuildutil::read_gdbm($gdbmfile, \%infodb);
+
+    #this makes the files specifying the hierarchy of subjects, titles etc
+    foreach $classify (@$classifyinfo) {
+	
+	&make_info_file($classify);
+
+    }
+
+    
+    #write out the collect.cfg
+    &output_cfg_file($colcfgfile);
+
+    &openmg ();
+
+    # read the archive information file
+    my $archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
+    my $archive_info = new arcinfo ();
+
+    # create a docsave object to process the documents
+    my $processor = new docsave ("", $archive_info, $verbosity);
+    $processor->setarchivedir ($archivedir);
+
+    my ($doc_obj, $hashref, $children);
+    print STDERR "processing documents now\n" if $verbosity >=2;
+    foreach $oid (@$toplevelinfo) {
+	$value = $infodb{$oid};
+	$hashref={};
+	$children = [];
+	&get_metadata($value, $hashref);
+	$doc_obj = new doc ();
+	$doc_obj->set_OID($oid);
+	my ($olddir) = $hashref->{'archivedir'}; # old dir for this doc, where images are stored
+	$top = $doc_obj->get_top_section();
+	&add_section_content ($doc_obj, $top, $hashref, $olddir);
+	&add_classification_metadata($oid, $doc_obj, $top);
+	&add_cover_image($doc_obj, $olddir);
+	&get_children($hashref, $children);
+	&recurse_sections($doc_obj, $children, $oid, $top, $olddir);
+	$processor->process($doc_obj);
+	
+    }
+    print STDERR "\n";
+   
+    &closemg();
+
+    # write out the archive information file
+    $archive_info->save_info($archive_info_filename);
+}
+
+# returns the path to the gdbm info database - also
+# sets the $collection and $textdir global variable
+sub get_gdbmfile {
+    my ($indexdir) = @_;
+    my ($gdbmfile);
+
+    opendir (DIR, $indexdir) || die "Couldn't open directory $indexdir\n\n";
+    my @conts = readdir DIR;
+    close DIR;
+
+    foreach $file (@conts) {
+	if ($file =~ /text$/) {
+	    $textdir = $file;
+	    last;
+	}
+    }
+    die "No text directory found in $indexdir\n\n" 
+	unless defined $textdir && $textdir =~ /text$/;
+
+    $gdbmfile = &util::filename_cat ($indexdir, $textdir);
+
+    opendir (DIR, $gdbmfile) || die "Couldn't open directory $gdbmfile\n\n";
+    @conts = readdir DIR;
+    close DIR;
+
+    foreach $file (@conts) {
+	if ($file =~ /^(.*?)\.(?:ldb|bdb)$/) {
+	    $collection = $1;
+	    $gdbmfile = &util::filename_cat ($gdbmfile, $file);
+	    last;
+	}
+    }
+    
+    if (defined $collection && $collection =~ /\w/) {
+	$textdir = &util::filename_cat ($textdir, $collection);
+    } else {
+	die "collection global wasn't set\n";
+    }
+    return $gdbmfile if (-e $gdbmfile);
+    die "Couldn't find gdbm info database in $indexdir\n\n";
+}
+
+
+sub get_toplevel_OID {
+    my ($gdbmfile) = @_;
+
+    open (DB2TXT, "db2txt $gdbmfile |") || die "couldn't open pipe to db2txt\n";
+    print STDERR "Finding all top level sections from $gdbmfile\n" if $verbosity >= 2;
+
+    $/ = '-' x 70;
+    my $entry = "";
+    while (defined ($entry = <DB2TXT>)) {
+	next unless $entry =~ /\w/;  #ignore blank entries
+	$entry =~ s/\n+/\\n/g;  # replace multiple \n with single \n
+	my ($key, $value) = $entry =~ /\[([^\]]*)\](.*)/;
+
+	next if ($key =~ /\./); #ignore any lower level entries
+	next if ($key =~ /^CL/); #ignore classification entries
+	next if ($value =~ /<section>/); #ignore docnum->OID entries
+	next if ($value =~ /<docoid>/); #ignore strange s133->OID entries
+	next if ($key !~ /\d/); #ignore collection, browse entries
+
+	push( @$toplevelinfo, $key);
+   
+    }
+
+    $/ = "\n";
+    #print STDERR "toplevel sections are: ", join ("\n", @$toplevelinfo);
+    #print STDERR "\n";
+}
+    
+# gets all the metadata from a gdbm file entry, and puts it into a hashref
+sub get_metadata {
+    
+    my ($gdb_str_ref, $hashref) = @_;
+    my @entries = split(/\n/, $gdb_str_ref);
+    foreach $entry (@entries) {
+	my($key, $value) = ($entry =~ /^<([^>]*)>(.*?)$/ );
+	$$hashref{$key} .= '@' if defined $$hashref{$key};
+	$$hashref{$key} .= $value;
+	
+    }
+   
+    
+}					  
+
+#takes a hashref containing the metadata for a gdbmfile entry, and extracts 
+#the childrens numbers (from the 'contains' entry).	
+#assumes format is ".1;".2;".3
+sub get_children {	
+    my ($hashref, $children) = @_; 
+
+    $childs = $hashref->{'contains'};
+    if (defined ($childs)) {
+	$childs =~ s/\@$//;  #remove trailing @
+	$childs =~ s/^\"\.//; #remove initial ".
+	@$children = split /\;\"\./, $childs;
+	
+    }
+    else {
+	  $children = [];
+      }
+}
+	
+#takes a hashref containing the metadata for a gdbmfile entry, and extracts 
+#the childrens numbers (from the 'contains' entry).	
+#assumes format is ".1;".2;".3
+#returns a list with the full child name ie HASH0123...ac.1 HASH0123...ac.2 
+#etc
+#used for classification stuff
+sub get_whole_children {	
+
+    my ($parentoid, $hashref, $children) = @_; 
+    
+    my $childs = $hashref->{'contains'};
+    my @items;
+    if (defined ($childs)) {
+	$childs =~ s/\@$//;  #remove trailing @
+	@items = split /\;/, $childs; #split on ;
+	foreach $item (@items) {
+	    $item =~ s/^\"/$parentoid/; # replace " with parentoid
+	    push (@$children, "$item");
+	}
+    }
+    else {
+	  $children = [];
+      }
+}
+
+    
+sub recurse_sections {
+    my ($doc_obj, $children, $parentoid, $parentsection, $olddir) = @_;
+
+    foreach $child (sort numerically @$children) {
+	$doc_obj->create_named_section("$parentsection.$child");
+	my $value = $infodb{"$parentoid.$child"};
+	my $hashref={};
+	&get_metadata($value, $hashref); # get childs metadata
+	my $newchildren = [];
+	&get_children($hashref, $newchildren); # get childs children
+	#add content fo rcurrent section
+	&add_section_content($doc_obj, "$parentsection.$child", $hashref, $olddir);
+	# process all the children if there are any
+	&recurse_sections($doc_obj, $newchildren, "$parentoid.$child", "$parentsection.$child", $olddir)
+	    if (defined ($newchildren));
+    }
+
+
+}					     
+
+sub add_section_content {
+    my ($doc_obj, $cursection, $hashref, $olddir) = @_;
+ 
+    foreach $key (keys %$hashref) {
+	#dont need to store these metadata
+	next if $key =~ /(contains|docnum|hastxt|doctype|archivedir|classifytype)/i;
+	my @items = split /@/, $hashref->{$key};
+	map {$doc_obj->add_metadata ($cursection, $key, $_); } @items;
+
+    }
+    my ($docnum)= $hashref->{'docnum'} =~ /(\d*)/;
+    my ($hastext) =$hashref->{'hastxt'} =~ /(0|1)/;
+
+    my $images=[];
+    if ($hastext) {
+	my $text = &get_text($docnum);
+    
+	#my (@images) = $text =~ /<img.*?src=\"([^\"]*)\"[^>]*>/g;
+    
+	# in text replace path to image with _httpdocimg_/blah.gif
+	#while ($text =~ s/(<img.*?src=\")([^\"]*)(\"[^>]*>)/
+	#   $1.&get_img($2, \@images).$3/sgei) {
+	$text =~ s/(<img.*?src=\")([^\"]*)(\"[^>]*>)/
+	    $1.&get_img($2,$images).$3/sgei;
+    
+	$doc_obj->add_text ($cursection, $text);
+	
+	if (scalar(@$images)>0) {
+	    
+	    foreach $img (@$images) {
+		my ($assoc_file) = $img =~ /([^\/\\]*\..*)$/; #the name of the image
+		$img =~ s/_httpcollection_/\./; #replace _httpcollection_ with .
+		$img =~ s/_thisOID_/$olddir/; #replace _thisOID_ with old archivedir name
+		
+		$doc_obj->associate_file($img, $assoc_file);
+	    }
+	}
+    }
+}
+
+
+
+sub get_img {
+    my ($path, $images) = @_;
+    my $img = "_httpdocimg_/";
+    my ($imgname) = $path =~ /([^\/\\]*\..*)$/;
+    push (@$images, $path);
+    $img .= $imgname;
+    return $img;
+}
+    
+
+sub add_classification_metadata {
+
+    my ($oid, $doc_obj, $cursection) = @_;
+    
+    if (defined $doc_classif_info->{$oid}) {
+
+	my $hashref = $doc_classif_info->{$oid};
+	
+	foreach $key (keys %$hashref) {
+	    my @items = @{$hashref->{$key}};
+	    map {$doc_obj->add_metadata ($cursection, $key, $_); } @items;
+	}
+    }
+}
+
+# picks up the cover image "cover.jpg" from the old archives directory.
+sub add_cover_image {
+
+    my ($doc_obj, $olddir) =  @_;
+    $assoc_file = "cover.jpg";
+    $img = "archives/$olddir/$assoc_file";
+
+
+    if (-e $img) {
+	$doc_obj->associate_file($img, $assoc_file);
+    }
+}
+
+
+
+sub set_index {
+    # check that $collection has been set
+    die "collection global was not set\n"
+	unless defined $collection && $collection =~ /\w/;
+
+    # find an index (just use first non-text directory we come across in $indexdir)
+    opendir (INDEXDIR, $indexdir) || die "couldn't open directory $indexdir\n";
+    my @indexes = readdir INDEXDIR;
+    close INDEXDIR;
+    foreach $i (@indexes) {
+	next if $i =~ /text$/i || $i =~ /\./ || $i =~ /assoc$/i;
+	$index = &util::filename_cat ($i, $collection);
+	last;
+    }
+}
+
+
+#########################################################################
+
+################ functions involving mg ################################
+
+sub get_text {
+    my ($docnum) = @_;
+
+    print STDERR "." if $verbosity >= 2;
+    &mgcommand ($docnum);
+
+   <$mgread>;	# eat the document separator
+
+    my $text = "";
+    my $line = "";
+
+    while (defined ($line = <$mgread>))
+    {
+	last if $line =~ /^<\/mg>/;
+	$text .= $line;
+    }
+
+    # Read in the last statement, which should be:
+    #  "dd documents retrieved."
+    <$mgread>;
+
+    return $text;
+}
+
+sub numerically {$a <=> $b;}
+
+
+
+sub openmg {
+
+    #print STDERR "index: $index\n";
+
+    die "Unable to start mgquery." unless
+	&openpipe($mgread, $mgwrite,
+		 "mgquery -d $indexdir -f $index -t $textdir");
+
+    $mgwrite->autoflush();
+
+    &mgcommand('.set expert true');
+    &mgcommand('.set terminator "</mg>\n"');
+    &mgcommand('.set mode text');
+    &mgcommand('.set query docnums');
+    &mgcommand('.set term_freq off');
+    &mgcommand('.set briefstats off');
+    &mgcommand('.set memstats off');
+    &mgcommand('.set sizestats off');
+    &mgcommand('.set timestats off');
+}
+
+sub closemg {
+    &mgcommand (".quit");
+    close($mgread);
+    close($mgwrite);
+}
+
+sub mgcommand {
+    my ($command) = @_;
+
+    return if $command =~ /^\s*$/;  #whitespace
+    #print STDERR "command: $command\n";
+    print $mgwrite "$command\n";
+
+    # eat up the command executed which is echoed
+    <$mgread>;
+}
+
+# openpipe(READ, WRITE, CMD)
+# 
+# Like open2, except CMD's stderr is also redirected.
+# 
+sub openpipe
+{
+    my ($read, $write, $cmd) = @_;
+    my ($child_read, $child_write);
+
+    $child_read = ++$FileHandle;
+    $child_write = ++$FileHandle;
+
+    pipe($read, $child_write) || die "Failed pipe($read, $child_write): $!";
+    pipe($child_read, $write) || die "Failed pipe($child_read, $write): $!";
+    my $pid;
+
+    if (($pid = fork) < 0) {
+        die "Failed fork: $!";
+    } elsif ($pid == 0) {
+        close($read);
+        close($write);
+        open(STDIN, "<&$child_read");
+        open(STDOUT, ">&$child_write");
+        open(STDERR, ">&$child_write");
+        exec($cmd);
+        die "Failed exec $cmd: $!";
+    }
+
+    close($child_read);
+    close($child_write);
+
+    $write->autoflush();
+    $read->autoflush();
+
+    return 1;
+}
+
+
+
+
+######################################################################
+
+############# functions to do with the classification stuff ##########
+
+#returns the top level classification oids
+sub get_classifyinfo {
+    my ($gdbmfile) = @_;
+
+    open (DB2TXT, "db2txt $gdbmfile |") || die "couldn't open pipe to db2txt\n";
+    print STDERR "Finding all classification sections from $gdbmfile\n" ;
+
+    $/ = '-' x 70;
+    my $entry = "";
+    while (defined ($entry = <DB2TXT>)) {
+	next unless $entry =~ /\w/;  #ignore blank entries
+	$entry =~ s/\n+/\\n/g;  # replace multiple \n with single \n
+	my ($key, $value) = $entry =~ /\[([^\]]*)\](.*)/;
+
+	next unless ($key =~/^CL\d$/); # assumes classification OID is like
+	                                 # CL1 or CL2 etc
+
+	push( @$classifyinfo, $key);
+   
+    }
+
+    $/ = "\n";
+    #print STDERR "classifications are: ", join(", ", @$classifyinfo);
+    #print STDERR "\n";
+    
+}
+
+#this creates the classification files needed for the hierarchy classifier
+#used for subjects, titles, orgs etc
+#also adds in entries to the collect_cfg hash    
+sub make_info_file {
+
+    my ($classifier) = @_;
+    my $info_file = "";
+    my $entry = $infodb{$classifier};
+
+    my $hashref = {};
+    &get_metadata($entry, $hashref);
+   
+    my $classifier_name = "CL".$hashref->{'Title'}; #like CLSubject
+    $classifier_name =~ s/\@$//; #remove trailing @
+   
+    # check children - if there is a classifier node at this level, 
+    # use a hierarchy, otherwise use an AZList.
+    
+    my $children=[];
+    my $hierarchy = 0;
+    &get_whole_children($classifier, $hashref, $children); #returns a list of the child ids
+    foreach $child(@$children) {
+	if(not &is_document($child)) {
+	    $hierarchy = 1;
+	    last;
+	}
+    }
+
+    if (!$hierarchy) {
+	&add_classify_cfg_list($classifier_name);
+    }else { #there is a hierarchy so create a file
+	$info_file = "./etc/$classifier_name.txt";
+
+	print STDERR "classification $classifier will be called $classifier_name\n";
+
+	open (OUTDOC, "> $info_file" ) || die  "couldn't open file $info_file\n";
+	
+	foreach $child (@$children) {
+	    &process_entry(OUTDOC, $classifier_name, $child);
+	}
+	
+	close OUTDOC;
+    
+	&add_classify_cfg($classifier, $classifier_name, $info_file);
+    }
+}
+
+
+sub process_entry {
+    my ($handle, $classifier_name, $classify_id) = @_;
+    my $value = $infodb{$classify_id};
+    
+    my $hashref={};
+    &get_metadata($value, $hashref);
+    my $title = $hashref->{'Title'};
+    $title =~ s/\@$//; #remove trailing @
+    &add_line($handle, $classify_id, $title);
+    
+    my $children = [];
+    &get_whole_children($classify_id, $hashref, $children);
+    foreach $child (@$children) {
+	if (&is_document($child)) {
+	    &add_doc_metadata($child, $classifier_name, $title);
+	}else {
+	    &process_entry($handle, $classifier_name, $child);
+	}
+    }
+    
+}
+    
+
+
+sub add_doc_metadata {
+
+    my ($doc_id, $classifier_name, $classifier_id) = @_;
+
+	#add entry to doc database
+	#print STDERR "at doc level, docnum=$classify_id\n";
+
+    $doc_classif_info->{$doc_id}={} unless defined $doc_classif_info->{$doc_id};
+    $doc_classif_info->{$doc_id}->{$classifier_name}=[] unless
+	defined $doc_classif_info->{$doc_id}->{$classifier_name};
+    push (@{$doc_classif_info->{$doc_id}->{$classifier_name}}, $classifier_id); 
+
+}
+
+
+
+
+sub add_line {
+
+    my ($handle, $classify_id, $title) = @_;
+    #print STDERR "classify id= $classify_id, title= $title\n";
+    $title = &unicode::ascii2utf8(\$title);
+    my ($num) = $classify_id =~ /^CL\d\.(.*)$/; #remove the CL1. from the front
+    
+    print $handle "\"$title\"\t$num\t\"$title\"\n";
+
+
+}
+
+sub is_document {
+    my ($oid) = @_;
+    return 1 if $oid =~ /^HASH/i;
+    return 0;
+}
+    
+########################################################################
+
+########## stuff for producing collect.cfg file ###########################
+
+sub add_default_cfg {
+
+    $username=`whoami`;
+    $username=`logname` unless defined $username;
+    $username="a_user" unless defined $username;
+    $username =~ s/\n//;
+    $collect_cfg->{'creator'}="$username\@cs.waikato.ac.nz";
+    $collect_cfg->{'maintainer'}="$username\@cs.waikato.ac.nz";
+    $collect_cfg->{'public'}="true";
+    $collect_cfg->{'beta'}="true";
+    
+    $collect_cfg->{'plugin'}=[];
+    push (@{$collect_cfg->{'plugin'}}, ["GAPlug"]);
+    push (@{$collect_cfg->{'plugin'}}, ["ArcPlug"]);
+    push (@{$collect_cfg->{'plugin'}}, ["RecPlug"]);
+
+    $collect_cfg->{'format'}={};
+    $collect_cfg->{'format'}->{'DocumentImages'}="true";
+    $collect_cfg->{'format'}->{'DocumentText'} = 
+	"\"<h3>[Title]</h3>\\\\n\\\\n<p>[Text]\"";
+    $collect_cfg->{'format'}->{'SearchVList'} =
+	"\"<td valign=top>[link][icon][/link]</td><td>{If}{[parent(All': '):Title],[parent(All': '):Title]:}[link][Title][/link]</td>\"";
+
+    $collect_cfg->{'collectionmeta'}={};
+    $collect_cfg->{'collectionmeta'}->{'collectionname'}="\"$collection\"";
+    $collect_cfg->{'collectionmeta'}->{'iconcollection'}="\"_httpprefix_/collect/$collection/images/$collection.gif\"";
+    $collect_cfg->{'collectionmeta'}->{'iconcollectionsmall'}="\"_httpprefix_/collect/$collection/images/${collection}sm.gif\"";
+    $collect_cfg->{'collectionmeta'}->{'collectionextra'} = "\"This is a collection rebuilt from CDROM.\"";
+
+}
+
+sub add_index_cfg {
+  my ($buildfile) = @_;
+
+  my $data={};
+  $collect_cfg->{'indexes'}=[];
+  if (-e $buildfile) {
+      $data=&cfgread::read_cfg_file($buildfile, '^(this)$', '^(indexmap)$'); 
+      foreach $i (@{$data->{'indexmap'}}) {
+          ($thisindex, $abbrev)= split ("\-\>", $i);
+          push (@{$collect_cfg->{'indexes'}}, $thisindex);
+	  $collect_cfg->{'defaultindex'} = $thisindex unless defined 
+	      $collect_cfg->{'defaultindex'};
+          $name=&get_index_name($thisindex);
+	  $thisindex=".$thisindex";
+          $collect_cfg->{'collectionmeta'}->{$thisindex} = "\"$name\"";
+      } 
+  }
+  else {
+      print STDERR "Couldn't read $buildfile, could not add index data to configuration file\n";
+  }
+
+}
+
+sub get_index_name {
+    my ($thisindex) = @_;
+    return "paragraphs" if $thisindex =~ /paragraph/;
+    return "chapters" if $thisindex =~ /section.*text/;
+    return "titles" if $thisindex =~ /Title/;
+    return "other";
+}
+
+sub add_classify_cfg {
+
+    my ($classify, $classifier_name, $file) = @_;
+    $collect_cfg->{'classify'} = [] unless defined $collect_cfg->{'classify'};
+
+    my ($title) = $classifier_name =~ /^CL(.*)$/;
+    my ($filename) = $file =~ /\/([^\/]*)$/;
+    my $entry = "Hierarchy -hfile $filename -metadata $classifier_name -buttonname $title -sort Title";
+    push (@{$collect_cfg->{'classify'}},[$entry]);
+   
+
+}
+
+sub add_classify_cfg_list {
+
+    my ($classifier) = @_;
+    $collect_cfg->{'classify'} = [] unless defined $collect_cfg->{'classify'};
+    my ($title) = $classifier =~ /^CL(.*)$/;
+    my $entry = "AZList -metadata $classifier -buttonname $title";
+    push (@{$collect_cfg->{'classify'}},[$entry]);
+}    
+
+sub output_cfg_file {
+
+    my ($collfile) = @_;
+    
+    if (-e $collfile) { #collect.cfg already exists
+	$collfile .= ".new";
+    }
+    &cfgread::write_cfg_file($collfile, $collect_cfg, 
+			     '^(creator|maintainer|public|beta|defaultindex)$',
+                            '^(indexes)$', '^(format|collectionmeta)$',
+                            '^(plugin|classify)$');
+
+
+
+}
+
+
Index: /main/tags/3.01/gsdl/bin/script/update.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/update.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/update.pl	(revision 21112)
@@ -0,0 +1,198 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# update.pl
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program updates any collections that are based on mirrors of
+# web sites after a certain interval.  It first checks that the
+# collection is a mirrored collection and whether it is time to 
+# update the collection.  If so, it updates the mirror with 
+# mirror.pl; then imports the collection with import.pl; then 
+# builds the collection with buildcol.pl; then replaces the 
+# existing index directory with the new building directory.
+# The etc/mirror.log file stores the STDOUT output.
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docprint;
+use util;
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n";
+    print STDERR "update.pl: Updates a collection if it has been set up to mirror\n";
+    print STDERR "           a website. Calls mirror.pl to do the mirroring.\n\n";
+    print STDERR "  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -importdir directory   Where to place the mirrored material\n";
+    print STDERR "   -archivedir directory  Where the converted material ends up\n";
+}
+
+
+&main ();
+
+sub main {
+    my ($verbosity, $importdir, $archivedir, $builddir, $indexdir, $etcdir, 
+	$mirror, $interval, $logfile,
+        $collection, $configfilename, $collectcfg);
+
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'importdir/.*/', \$importdir,
+			 'archivedir/.*/', \$archivedir )) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+
+    # check the configuration file for options
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc/collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+	    $importdir = $collectcfg->{'importdir'};
+	}
+	if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	    $archivedir = $collectcfg->{'archivedir'};
+	}
+	if (defined $collectcfg->{'mirror'}) {
+	    $mirror = $collectcfg->{'mirror'};
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+    
+    # fill in the default import and archives directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = "$ENV{'GSDLCOLLECTDIR'}/import" if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+    $archivedir = "$ENV{'GSDLCOLLECTDIR'}/archives" if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+
+    $indexdir = "$ENV{'GSDLCOLLECTDIR'}/index";
+    $builddir = "$ENV{'GSDLCOLLECTDIR'}/building";
+    $etcdir = "$ENV{'GSDLCOLLECTDIR'}/etc";
+    $logfile = "$etcdir/mirror.log";
+
+    # if there is no mirror information, we're all done
+    if (!defined($mirror)) {
+	print "No mirror command in $configfilename\n";
+	exit;
+    }
+
+    # read the mirror interval
+    if (($#$mirror == 1) && ($$mirror[0] =~ /interval/)){
+	$interval = $$mirror[1];
+    } else {
+	die "Malformed mirror information: use \"mirror interval N\"\n" .
+	    "where N is the number of days between mirrors.\n";
+    }
+
+    # make sure there is an import directory
+    if (! -e "$importdir") {
+	&util::mk_dir($importdir);
+    }
+
+    print "archives directory: $archivedir\n";
+    print "  import directory: $importdir\n";
+    print "     etc directory: $etcdir\n";
+    print "          interval: $interval days\n";
+
+    # how many days is it since the last mirror
+    my $seconds = 0;
+    if (-e "$logfile") {
+	my $now = time;
+	my @stats = stat("$logfile");
+	my $then = $stats[9];
+	# calculate the number of days since the last mirror
+	$seconds = $now - $then;
+    }
+    my $days = (($seconds / 3600) / 24);
+
+    # Is it too soon to start mirroring?
+    if (($seconds > 0) && ($interval > $days)) {
+	printf "Mirror not started: only %.1f days have passed\n", $days;
+	exit;
+    }
+    
+    # Mirror the remote site
+    open(WLOG, ">$logfile");
+    print WLOG "Starting mirror at " . time . "\n\n";
+    
+    my $command = "mirror.pl $collection";
+    print WLOG "Executing: $command\n";
+    print WLOG `$command`;
+    
+    # Import the collection
+    print WLOG "\n\nStarting import at " . time . "\n\n";
+    $command = "import.pl -removeold $collection";
+    print WLOG "Executing: $command\n";
+    print WLOG `$command`;
+
+    # Build the collection
+    print WLOG "\n\nStarting buildcol.pl at " . time . "\n\n";
+    $command = "buildcol.pl $collection";
+    print WLOG "Executing: $command\n";
+    print WLOG `$command`;
+
+    # Renaming the building directory to index
+    print WLOG "\n\nRenaming building directory at " . time . "\n\n";
+    if (-e $indexdir) {
+	&util::mv($indexdir, "$indexdir.old");
+	&util::mv($builddir, $indexdir);
+	&util::rm_r ("$indexdir.old");
+    } else {
+	&util::mv($builddir, $indexdir);
+    }
+
+
+    close WLOG;
+    
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl/bin/script/xlstohtml.pl
===================================================================
--- /main/tags/3.01/gsdl/bin/script/xlstohtml.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/bin/script/xlstohtml.pl	(revision 21112)
@@ -0,0 +1,69 @@
+eval 'exec perl -x -- $0 $@'
+    if 0;
+#! perl
+# line 6
+
+sub usage() {
+    print "$0 <input.xls> <output.html>\n";
+}
+
+
+if (@ARGV != 2) {
+    usage();
+    exit(1);
+}
+my $input_xls=shift;
+my $output_html=shift;
+
+# try to find xlhtml binary in GSDLHOME
+my $xlhtml_binary="xlhtml";
+my $GSDLOS=$ENV{'GSDLOS'};
+my $GSDLHOME=$ENV{'GSDLHOME'};
+if ($GSDLOS =~ /^windows/i) {
+    $xlhtml_binary.=".exe";
+}
+
+# Trouble happens if there are spaces in GSDLHOME. Assume that on windows
+# the program is found in the path.
+if ($GSDLOS !~ /windows/i && -x "$GSDLHOME/bin/$GSDLOS/$xlhtml_binary") {
+    $xlhtml_binary="$GSDLHOME/bin/$GSDLOS/$xlhtml_binary";
+}
+
+if (! -r $input_xls) {
+    print STDERR "Unable to read file `$input_xls'\n";
+    exit (1);
+}
+
+my $return_value=
+    system("$xlhtml_binary \"$input_xls\" > \"$output_html\"");
+
+if ($return_value != 0) {
+    exit (1);
+}
+
+# Ok, we made an html file. Check to see if it has any content, and remove
+# the little nag link at the the bottom.
+my $html="";
+open (HTML, "$output_html") || die "Can't read file:$!";
+$html=join('', <HTML>);
+close HTML;
+
+$html =~ s@<hr><FONT SIZE=-1>Created with.*\n</BODY></HTML>$@</BODY></HTML>@s;
+
+# xlHtml uses the filename as the title.
+# HTMLPlug will use the first 100 chars instead if there's no title.
+# Don't know if that's a good idea with a spreadsheet, though...
+$html =~ s@<title>.*?\.xls</title>@<title></title>@i;
+
+my $tmp=$html;
+$tmp =~ s/^.*?<BODY>//ms;
+$tmp =~ s/(&nbsp;)|\s//gims;
+if ($tmp !~ m/(>[^<]+<)/) {
+    print STDERR "No text found in extracted html file!\n";
+    exit(1);
+}
+open (NEWHTML, ">$output_html") || die "Can't create file:$!";
+print NEWHTML $html;
+close NEWHTML;
+exit (0);
+
Index: /main/tags/3.01/gsdl/collect/modelcol/etc/collect.cfg
===================================================================
--- /main/tags/3.01/gsdl/collect/modelcol/etc/collect.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl/collect/modelcol/etc/collect.cfg	(revision 21112)
@@ -0,0 +1,18 @@
+creator       **creator**
+maintainer    **maintainer**
+public        **public**
+
+indexes        document:text document:Title document:Source
+defaultindex   document:text
+
+**plugins**
+
+classify       AZList -metadata Title
+classify       AZList -metadata Source
+
+collectionmeta collectionname    "**title**"
+collectionmeta iconcollection    ""
+collectionmeta collectionextra   "**about**"
+collectionmeta .document:text    "text"
+collectionmeta .document:Title  "titles"
+collectionmeta .document:Source  "filenames"
Index: /main/tags/3.01/gsdl/config.guess
===================================================================
--- /main/tags/3.01/gsdl/config.guess	(revision 21112)
+++ /main/tags/3.01/gsdl/config.guess	(revision 21112)
@@ -0,0 +1,1298 @@
+#! /bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
+#   Free Software Foundation, Inc.
+
+timestamp='2001-07-12'
+
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# Please send patches to <config-patches@gnu.org>.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit build system type.
+
+me=`echo "$0" | sed -e 's,.*/,,'`
+
+usage="\
+Usage: $0 [OPTION]
+
+Output the configuration name of the system \`$me' is run on.
+
+Operation modes:
+  -h, --help         print this help, then exit
+  -t, --time-stamp   print date of last modification, then exit
+  -v, --version      print version number, then exit
+
+Report bugs and patches to <config-patches@gnu.org>."
+
+version="\
+GNU config.guess ($timestamp)
+
+Originally written by Per Bothner.
+Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
+Free Software Foundation, Inc.
+
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+
+help="
+Try \`$me --help' for more information."
+
+# Parse command line
+while test $# -gt 0 ; do
+  case $1 in
+    --time-stamp | --time* | -t )
+       echo "$timestamp" ; exit 0 ;;
+    --version | -v )
+       echo "$version" ; exit 0 ;;
+    --help | --h* | -h )
+       echo "$usage"; exit 0 ;;
+    -- )     # Stop option processing
+       shift; break ;;
+    - )	# Use stdin as input.
+       break ;;
+    -* )
+       echo "$me: invalid option $1$help" >&2
+       exit 1 ;;
+    * )
+       break ;;
+  esac
+done
+
+if test $# != 0; then
+  echo "$me: too many arguments$help" >&2
+  exit 1
+fi
+
+
+dummy=dummy-$$
+trap 'rm -f $dummy.c $dummy.o $dummy.rel $dummy; exit 1' 1 2 15
+
+# CC_FOR_BUILD -- compiler used by this script.
+# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still
+# use `HOST_CC' if defined, but it is deprecated.
+
+set_cc_for_build='case $CC_FOR_BUILD,$HOST_CC,$CC in
+ ,,)    echo "int dummy(){}" > $dummy.c ;
+	for c in cc gcc c89 ; do
+	  ($c $dummy.c -c -o $dummy.o) >/dev/null 2>&1 ;
+	  if test $? = 0 ; then
+	     CC_FOR_BUILD="$c"; break ;
+	  fi ;
+	done ;
+	rm -f $dummy.c $dummy.o $dummy.rel ;
+	if test x"$CC_FOR_BUILD" = x ; then
+	  CC_FOR_BUILD=no_compiler_found ;
+	fi
+	;;
+ ,,*)   CC_FOR_BUILD=$CC ;;
+ ,*,*)  CC_FOR_BUILD=$HOST_CC ;;
+esac'
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 1994-08-24)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null`  || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    *:NetBSD:*:*)
+	# Netbsd (nbsd) targets should (where applicable) match one or
+	# more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*,
+	# *-*-netbsdecoff* and *-*-netbsd*.  For targets that recently
+	# switched to ELF, *-*-netbsd* would select the old
+	# object file format.  This provides both forward
+	# compatibility and a consistent mechanism for selecting the
+	# object file format.
+	# Determine the machine/vendor (is the vendor relevant).
+	case "${UNAME_MACHINE}" in
+	    amiga) machine=m68k-unknown ;;
+	    arm32) machine=arm-unknown ;;
+	    atari*) machine=m68k-atari ;;
+	    sun3*) machine=m68k-sun ;;
+	    mac68k) machine=m68k-apple ;;
+	    macppc) machine=powerpc-apple ;;
+	    hp3[0-9][05]) machine=m68k-hp ;;
+	    ibmrt|romp-ibm) machine=romp-ibm ;;
+	    *) machine=${UNAME_MACHINE}-unknown ;;
+	esac
+	# The Operating System including object format, if it has switched
+	# to ELF recently, or will in the future.
+	case "${UNAME_MACHINE}" in
+	    i386|sparc|amiga|arm*|hp300|mvme68k|vax|atari|luna68k|mac68k|news68k|next68k|pc532|sun3*|x68k)
+		eval $set_cc_for_build
+		if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
+			| grep __ELF__ >/dev/null
+		then
+		    # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout).
+		    # Return netbsd for either.  FIX?
+		    os=netbsd
+		else
+		    os=netbsdelf
+		fi
+		;;
+	    *)
+	        os=netbsd
+		;;
+	esac
+	# The OS release
+	release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
+	# contains redundant information, the shorter form:
+	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
+	echo "${machine}-${os}${release}"
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	if test $UNAME_RELEASE = "V4.0"; then
+		UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
+	fi
+	# A Vn.n version is a released version.
+	# A Tn.n version is a released field test version.
+	# A Xn.n version is an unreleased experimental baselevel.
+	# 1.2 uses "1.2" for uname -r.
+	cat <<EOF >$dummy.s
+	.data
+\$Lformat:
+	.byte 37,100,45,37,120,10,0	# "%d-%x\n"
+
+	.text
+	.globl main
+	.align 4
+	.ent main
+main:
+	.frame \$30,16,\$26,0
+	ldgp \$29,0(\$27)
+	.prologue 1
+	.long 0x47e03d80 # implver \$0
+	lda \$2,-1
+	.long 0x47e20c21 # amask \$2,\$1
+	lda \$16,\$Lformat
+	mov \$0,\$17
+	not \$1,\$18
+	jsr \$26,printf
+	ldgp \$29,0(\$26)
+	mov 0,\$16
+	jsr \$26,exit
+	.end main
+EOF
+	eval $set_cc_for_build
+	$CC_FOR_BUILD $dummy.s -o $dummy 2>/dev/null
+	if test "$?" = 0 ; then
+		case `./$dummy` in
+			0-0)
+				UNAME_MACHINE="alpha"
+				;;
+			1-0)
+				UNAME_MACHINE="alphaev5"
+				;;
+			1-1)
+				UNAME_MACHINE="alphaev56"
+				;;
+			1-101)
+				UNAME_MACHINE="alphapca56"
+				;;
+			2-303)
+				UNAME_MACHINE="alphaev6"
+				;;
+			2-307)
+				UNAME_MACHINE="alphaev67"
+				;;
+		esac
+	fi
+	rm -f $dummy.s $dummy
+	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[VTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+	exit 0 ;;
+    Alpha\ *:Windows_NT*:*)
+	# How do we know it's Interix rather than the generic POSIX subsystem?
+	# Should we change UNAME_MACHINE based on the output of uname instead
+	# of the specific Alpha model?
+	echo alpha-pc-interix
+	exit 0 ;;
+    21064:Windows_NT:50:3)
+	echo alpha-dec-winnt3.5
+	exit 0 ;;
+    Amiga*:UNIX_System_V:4.0:*)
+	echo m68k-unknown-sysv4
+	exit 0;;
+    amiga:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    *:[Aa]miga[Oo][Ss]:*:*)
+	echo ${UNAME_MACHINE}-unknown-amigaos
+	exit 0 ;;
+    arc64:OpenBSD:*:*)
+	echo mips64el-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    arc:OpenBSD:*:*)
+	echo mipsel-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    hkmips:OpenBSD:*:*)
+	echo mips-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    pmax:OpenBSD:*:*)
+	echo mipsel-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    sgi:OpenBSD:*:*)
+	echo mips-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    wgrisc:OpenBSD:*:*)
+	echo mipsel-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    *:OS/390:*:*)
+	echo i370-ibm-openedition
+	exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
+	echo hppa1.1-hitachi-hiuxmpp
+	exit 0;;
+    Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*)
+	# akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE.
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    NILE*:*:*:dcosx)
+	echo pyramid-pyramid-svr4
+	exit 0 ;;
+    sun4H:SunOS:5.*:*)
+	echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    i86pc:SunOS:5.*:*)
+	echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	case "`/usr/bin/arch -k`" in
+	    Series*|S4*)
+		UNAME_RELEASE=`uname -v`
+		;;
+	esac
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    sun*:*:4.2BSD:*)
+	UNAME_RELEASE=`(head -1 /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
+	test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
+	case "`/bin/arch`" in
+	    sun3)
+		echo m68k-sun-sunos${UNAME_RELEASE}
+		;;
+	    sun4)
+		echo sparc-sun-sunos${UNAME_RELEASE}
+		;;
+	esac
+	exit 0 ;;
+    aushp:SunOS:*:*)
+	echo sparc-auspex-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    atari*:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    # The situation for MiNT is a little confusing.  The machine name
+    # can be virtually everything (everything which is not
+    # "atarist" or "atariste" at least should have a processor
+    # > m68000).  The system name ranges from "MiNT" over "FreeMiNT"
+    # to the lowercase version "mint" (or "freemint").  Finally
+    # the system name "TOS" denotes a system which is actually not
+    # MiNT.  But MiNT is downward compatible to TOS, so this should
+    # be no problem.
+    atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
+        echo m68k-atari-mint${UNAME_RELEASE}
+	exit 0 ;;
+    atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
+	echo m68k-atari-mint${UNAME_RELEASE}
+        exit 0 ;;
+    *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
+        echo m68k-atari-mint${UNAME_RELEASE}
+	exit 0 ;;
+    milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
+        echo m68k-milan-mint${UNAME_RELEASE}
+        exit 0 ;;
+    hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
+        echo m68k-hades-mint${UNAME_RELEASE}
+        exit 0 ;;
+    *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
+        echo m68k-unknown-mint${UNAME_RELEASE}
+        exit 0 ;;
+    sun3*:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    mac68k:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    mvme68k:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    mvme88k:OpenBSD:*:*)
+	echo m88k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    powerpc:machten:*:*)
+	echo powerpc-apple-machten${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:Mach:*:*)
+	echo mips-dec-mach_bsd4.3
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    2020:CLIX:*:* | 2430:CLIX:*:*)
+	echo clipper-intergraph-clix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:*:UMIPS | mips:*:*:RISCos)
+	sed 's/^	//' << EOF >$dummy.c
+#ifdef __cplusplus
+#include <stdio.h>  /* for printf() prototype */
+	int main (int argc, char *argv[]) {
+#else
+	int main (argc, argv) int argc; char *argv[]; {
+#endif
+	#if defined (host_mips) && defined (MIPSEB)
+	#if defined (SYSTYPE_SYSV)
+	  printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0);
+	#endif
+	#if defined (SYSTYPE_SVR4)
+	  printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0);
+	#endif
+	#if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD)
+	  printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0);
+	#endif
+	#endif
+	  exit (-1);
+	}
+EOF
+	eval $set_cc_for_build
+	$CC_FOR_BUILD $dummy.c -o $dummy \
+	  && ./$dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \
+	  && rm -f $dummy.c $dummy && exit 0
+	rm -f $dummy.c $dummy
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    Motorola:PowerMAX_OS:*:*)
+	echo powerpc-motorola-powermax
+	exit 0 ;;
+    Night_Hawk:Power_UNIX:*:*)
+	echo powerpc-harris-powerunix
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+        # DG/UX returns AViiON for all architectures
+        UNAME_PROCESSOR=`/usr/bin/uname -p`
+	if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
+	then
+	    if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
+	       [ ${TARGET_BINARY_INTERFACE}x = x ]
+	    then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	    else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	    fi
+	else
+	    echo i586-dg-dgux${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX*:*:*)
+	echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'`
+	exit 0 ;;
+    ????????:AIX?:[12].1:2)   # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX.
+	echo romp-ibm-aix      # uname -m gives an 8 hex-code CPU id
+	exit 0 ;;              # Note that: echo "'`uname -s`'" gives 'AIX '
+    i*86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    ia64:AIX:*:*)
+	if [ -x /usr/bin/oslevel ] ; then
+		IBM_REV=`/usr/bin/oslevel`
+	else
+		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+	fi
+	echo ${UNAME_MACHINE}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >$dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		eval $set_cc_for_build
+		$CC_FOR_BUILD $dummy.c -o $dummy && ./$dummy && rm -f $dummy.c $dummy && exit 0
+		rm -f $dummy.c $dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:[45])
+	IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | head -1 | awk '{ print $1 }'`
+	if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if [ -x /usr/bin/oslevel ] ; then
+		IBM_REV=`/usr/bin/oslevel`
+	else
+		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    ibmrt:4.4BSD:*|romp-ibm:BSD:*)
+	echo romp-ibm-bsd4.4
+	exit 0 ;;
+    ibmrt:*BSD:*|romp-ibm:BSD:*)            # covers RT/PC BSD and
+	echo romp-ibm-bsd${UNAME_RELEASE}   # 4.3 with uname added to
+	exit 0 ;;                           # report: romp-ibm BSD 4.3
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[34678]??:HP-UX:*:*)
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/[678][0-9][0-9])
+              case "${HPUX_REV}" in
+                11.[0-9][0-9])
+                  if [ -x /usr/bin/getconf ]; then
+                    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
+                    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+                    case "${sc_cpu_version}" in
+                      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
+                      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+                      532)                      # CPU_PA_RISC2_0
+                        case "${sc_kernel_bits}" in
+                          32) HP_ARCH="hppa2.0n" ;;
+                          64) HP_ARCH="hppa2.0w" ;;
+                        esac ;;
+                    esac
+                  fi ;;
+              esac
+              if [ "${HP_ARCH}" = "" ]; then
+              sed 's/^              //' << EOF >$dummy.c
+
+              #define _HPUX_SOURCE
+              #include <stdlib.h>
+              #include <unistd.h>
+
+              int main ()
+              {
+              #if defined(_SC_KERNEL_BITS)
+                  long bits = sysconf(_SC_KERNEL_BITS);
+              #endif
+                  long cpu  = sysconf (_SC_CPU_VERSION);
+
+                  switch (cpu)
+              	{
+              	case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
+              	case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
+              	case CPU_PA_RISC2_0:
+              #if defined(_SC_KERNEL_BITS)
+              	    switch (bits)
+              		{
+              		case 64: puts ("hppa2.0w"); break;
+              		case 32: puts ("hppa2.0n"); break;
+              		default: puts ("hppa2.0"); break;
+              		} break;
+              #else  /* !defined(_SC_KERNEL_BITS) */
+              	    puts ("hppa2.0"); break;
+              #endif
+              	default: puts ("hppa1.0"); break;
+              	}
+                  exit (0);
+              }
+EOF
+	eval $set_cc_for_build
+	(CCOPTS= $CC_FOR_BUILD $dummy.c -o $dummy 2>/dev/null ) && HP_ARCH=`./$dummy`
+	if test -z "$HP_ARCH"; then HP_ARCH=hppa; fi
+	rm -f $dummy.c $dummy
+	fi ;;
+	esac
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    ia64:HP-UX:*:*)
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ia64-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >$dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	eval $set_cc_for_build
+	$CC_FOR_BUILD $dummy.c -o $dummy && ./$dummy && rm -f $dummy.c $dummy && exit 0
+	rm -f $dummy.c $dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*)
+	echo hppa1.0-hp-mpeix
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    i*86:OSF1:*:*)
+	if [ -x /usr/sbin/sysversion ] ; then
+	    echo ${UNAME_MACHINE}-unknown-osf1mk
+	else
+	    echo ${UNAME_MACHINE}-unknown-osf1
+	fi
+	exit 0 ;;
+    parisc*:Lites*:*:*)
+	echo hppa1.1-hp-lites
+	exit 0 ;;
+    hppa*:OpenBSD:*:*)
+	echo hppa-unknown-openbsd
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:*:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:*:*:*)
+	echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY*[A-Z]90:*:*:*)
+	echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \
+	| sed -e 's/CRAY.*\([A-Z]90\)/\1/' \
+	      -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \
+	      -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY*TS:*:*:*)
+	echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY*T3D:*:*:*)
+	echo alpha-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY*T3E:*:*:*)
+	echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY*SV1:*:*:*)
+	echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
+	exit 0 ;;
+    CRAY-2:*:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
+	FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+        FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+        FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+        echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+        exit 0 ;;
+    hp300:OpenBSD:*:*)
+	echo m68k-unknown-openbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
+	echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    sparc*:BSD/OS:*:*)
+	echo sparc-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:BSD/OS:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:OpenBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-openbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    i*:CYGWIN*:*)
+	echo ${UNAME_MACHINE}-pc-cygwin
+	exit 0 ;;
+    i*:MINGW*:*)
+	echo ${UNAME_MACHINE}-pc-mingw32
+	exit 0 ;;
+    i*:PW*:*)
+	echo ${UNAME_MACHINE}-pc-pw32
+	exit 0 ;;
+    i*:Windows_NT*:* | Pentium*:Windows_NT*:*)
+	# How do we know it's Interix rather than the generic POSIX subsystem?
+	# It also conflicts with pre-2.0 versions of AT&T UWIN. Should we
+	# UNAME_MACHINE based on the output of uname instead of i386?
+	echo i386-pc-interix
+	exit 0 ;;
+    i*:UWIN*:*)
+	echo ${UNAME_MACHINE}-pc-uwin
+	exit 0 ;;
+    p*:CYGWIN*:*)
+	echo powerpcle-unknown-cygwin
+	exit 0 ;;
+    prep*:SunOS:5.*:*)
+	echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    i*86:Minix:*:*)
+	echo ${UNAME_MACHINE}-pc-minix
+	exit 0 ;;
+    arm*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	exit 0 ;;
+    ia64:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+    m68*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	exit 0 ;;
+    mips:Linux:*:*)
+	case `sed -n '/^byte/s/^.*: \(.*\) endian/\1/p' < /proc/cpuinfo` in
+	  big)    echo mips-unknown-linux-gnu && exit 0 ;;
+	  little) echo mipsel-unknown-linux-gnu && exit 0 ;;
+	esac
+	;;
+    ppc:Linux:*:*)
+	echo powerpc-unknown-linux-gnu
+	exit 0 ;;
+    alpha:Linux:*:*)
+	case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
+	  EV5)   UNAME_MACHINE=alphaev5 ;;
+	  EV56)  UNAME_MACHINE=alphaev56 ;;
+	  PCA56) UNAME_MACHINE=alphapca56 ;;
+	  PCA57) UNAME_MACHINE=alphapca56 ;;
+	  EV6)   UNAME_MACHINE=alphaev6 ;;
+	  EV67)  UNAME_MACHINE=alphaev67 ;;
+	  EV68*) UNAME_MACHINE=alphaev67 ;;
+        esac
+	objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null
+	if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
+	echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
+	exit 0 ;;
+    parisc:Linux:*:* | hppa:Linux:*:*)
+	# Look for CPU level
+	case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
+	  PA7*) echo hppa1.1-unknown-linux-gnu ;;
+	  PA8*) echo hppa2.0-unknown-linux-gnu ;;
+	  *)    echo hppa-unknown-linux-gnu ;;
+	esac
+	exit 0 ;;
+    parisc64:Linux:*:* | hppa64:Linux:*:*)
+	echo hppa64-unknown-linux-gnu
+	exit 0 ;;
+    s390:Linux:*:* | s390x:Linux:*:*)
+	echo ${UNAME_MACHINE}-ibm-linux
+	exit 0 ;;
+    sh*:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	exit 0 ;;
+    sparc:Linux:*:* | sparc64:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-gnu
+	exit 0 ;;
+    x86_64:Linux:*:*)
+	echo x86_64-unknown-linux-gnu
+	exit 0 ;;
+    i*86:Linux:*:*)
+	# The BFD linker knows what the default object file format is, so
+	# first see if it will tell us. cd to the root directory to prevent
+	# problems with other programs or directories called `ld' in the path.
+	ld_supported_targets=`cd /; ld --help 2>&1 \
+			 | sed -ne '/supported targets:/!d
+				    s/[ 	][ 	]*/ /g
+				    s/.*supported targets: *//
+				    s/ .*//
+				    p'`
+        case "$ld_supported_targets" in
+	  elf32-i386)
+		TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu"
+		;;
+	  a.out-i386-linux)
+		echo "${UNAME_MACHINE}-pc-linux-gnuaout"
+		exit 0 ;;		
+	  coff-i386)
+		echo "${UNAME_MACHINE}-pc-linux-gnucoff"
+		exit 0 ;;
+	  "")
+		# Either a pre-BFD a.out linker (linux-gnuoldld) or
+		# one that does not give us useful --help.
+		echo "${UNAME_MACHINE}-pc-linux-gnuoldld"
+		exit 0 ;;
+	esac
+	# Determine whether the default compiler is a.out or elf
+	cat >$dummy.c <<EOF
+#include <features.h>
+#ifdef __cplusplus
+#include <stdio.h>  /* for printf() prototype */
+	int main (int argc, char *argv[]) {
+#else
+	int main (argc, argv) int argc; char *argv[]; {
+#endif
+#ifdef __ELF__
+# ifdef __GLIBC__
+#  if __GLIBC__ >= 2
+    printf ("%s-pc-linux-gnu\n", argv[1]);
+#  else
+    printf ("%s-pc-linux-gnulibc1\n", argv[1]);
+#  endif
+# else
+   printf ("%s-pc-linux-gnulibc1\n", argv[1]);
+# endif
+#else
+  printf ("%s-pc-linux-gnuaout\n", argv[1]);
+#endif
+  return 0;
+}
+EOF
+	eval $set_cc_for_build
+	$CC_FOR_BUILD $dummy.c -o $dummy 2>/dev/null && ./$dummy "${UNAME_MACHINE}" && rm -f $dummy.c $dummy && exit 0
+	rm -f $dummy.c $dummy
+	test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0
+	;;
+    i*86:DYNIX/ptx:4*:*)
+	# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
+	# earlier versions are messed up and put the nodename in both
+	# sysname and nodename.
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i*86:UNIX_SV:4.2MP:2.*)
+        # Unixware is an offshoot of SVR4, but it has its own version
+        # number series starting with 2...
+        # I am not positive that other SVR4 systems won't match this,
+	# I just have to hope.  -- rms.
+        # Use sysv4.2uw... so that sysv4* matches it.
+	echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
+	exit 0 ;;
+    i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*)
+	UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'`
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL}
+	else
+		echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL}
+	fi
+	exit 0 ;;
+    i*86:*:5:[78]*)
+	case `/bin/uname -X | grep "^Machine"` in
+	    *486*)	     UNAME_MACHINE=i486 ;;
+	    *Pentium)	     UNAME_MACHINE=i586 ;;
+	    *Pent*|*Celeron) UNAME_MACHINE=i686 ;;
+	esac
+	echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}
+	exit 0 ;;
+    i*86:*:3.2:*)
+	if test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-pc-isc$UNAME_REL
+	elif /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		(/bin/uname -X|egrep '^Machine.*Pentium' >/dev/null) \
+			&& UNAME_MACHINE=i586
+		(/bin/uname -X|egrep '^Machine.*Pent ?II' >/dev/null) \
+			&& UNAME_MACHINE=i686
+		(/bin/uname -X|egrep '^Machine.*Pentium Pro' >/dev/null) \
+			&& UNAME_MACHINE=i686
+		echo ${UNAME_MACHINE}-pc-sco$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-pc-sysv32
+	fi
+	exit 0 ;;
+    i*86:*DOS:*:*)
+	echo ${UNAME_MACHINE}-pc-msdosdjgpp
+	exit 0 ;;
+    pc:*:*:*)
+	# Left here for compatibility:
+        # uname -m prints for DJGPP always 'pc', but it prints nothing about
+        # the processor, so we play safe by assuming i386.
+	echo i386-pc-msdosdjgpp
+        exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-pc-mach3
+	exit 0 ;;
+    paragon:*:*:*)
+	echo i860-intel-osf1
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M68*:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 4850:*:4.0:3.0)
+	OS_REL=''
+	test -r /etc/.relid \
+	&& OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid`
+	/bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+	  && echo i486-ncr-sysv4.3${OS_REL} && exit 0
+	/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
+	  && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*)
+	echo m68k-unknown-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*)
+	echo i386-unknown-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.*:*)
+	echo sparc-unknown-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.*:*)
+	echo rs6000-unknown-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*)
+	echo powerpc-unknown-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    SM[BE]S:UNIX_SV:*:*)
+	echo mips-dde-sysv${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:ReliantUNIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+    PENTIUM:CPunix:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
+                           # says <Richard.M.Bartel@ccMail.Census.GOV>
+        echo i586-unisys-sysv4
+        exit 0 ;;
+    *:UNIX_System_V:4*:FTX*)
+	# From Gerald Hewes <hewes@openmarket.com>.
+	# How about differentiating between stratus architectures? -djm
+	echo hppa1.1-stratus-sysv4
+	exit 0 ;;
+    *:*:*:FTX*)
+	# From seanf@swdc.stratus.com.
+	echo i860-stratus-sysv4
+	exit 0 ;;
+    mc68*:A/UX:*:*)
+	echo m68k-apple-aux${UNAME_RELEASE}
+	exit 0 ;;
+    news*:NEWS-OS:6*:*)
+	echo mips-sony-newsos6
+	exit 0 ;;
+    R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
+	if [ -d /usr/nec ]; then
+	        echo mips-nec-sysv${UNAME_RELEASE}
+	else
+	        echo mips-unknown-sysv${UNAME_RELEASE}
+	fi
+        exit 0 ;;
+    BeBox:BeOS:*:*)	# BeOS running on hardware made by Be, PPC only.
+	echo powerpc-be-beos
+	exit 0 ;;
+    BeMac:BeOS:*:*)	# BeOS running on Mac or Mac clone, PPC only.
+	echo powerpc-apple-beos
+	exit 0 ;;
+    BePC:BeOS:*:*)	# BeOS running on Intel PC compatible.
+	echo i586-pc-beos
+	exit 0 ;;
+    SX-4:SUPER-UX:*:*)
+	echo sx4-nec-superux${UNAME_RELEASE}
+	exit 0 ;;
+    SX-5:SUPER-UX:*:*)
+	echo sx5-nec-superux${UNAME_RELEASE}
+	exit 0 ;;
+    Power*:Rhapsody:*:*)
+	echo powerpc-apple-rhapsody${UNAME_RELEASE}
+	exit 0 ;;
+    *:Rhapsody:*:*)
+	echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE}
+	exit 0 ;;
+    *:Darwin:*:*)
+	echo `uname -p`-apple-darwin${UNAME_RELEASE}
+	exit 0 ;;
+    *:procnto*:*:* | *:QNX:[0123456789]*:*)
+	if test "${UNAME_MACHINE}" = "x86pc"; then
+		UNAME_MACHINE=pc
+	fi
+	echo `uname -p`-${UNAME_MACHINE}-nto-qnx
+	exit 0 ;;
+    *:QNX:*:4*)
+	echo i386-pc-qnx
+	exit 0 ;;
+    NSR-[KW]:NONSTOP_KERNEL:*:*)
+	echo nsr-tandem-nsk${UNAME_RELEASE}
+	exit 0 ;;
+    *:NonStop-UX:*:*)
+	echo mips-compaq-nonstopux
+	exit 0 ;;
+    BS2000:POSIX*:*:*)
+	echo bs2000-siemens-sysv
+	exit 0 ;;
+    DS/*:UNIX_System_V:*:*)
+	echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE}
+	exit 0 ;;
+    *:Plan9:*:*)
+	# "uname -m" is not consistent, so use $cputype instead. 386
+	# is converted to i386 for consistency with other x86
+	# operating systems.
+	if test "$cputype" = "386"; then
+	    UNAME_MACHINE=i386
+	else
+	    UNAME_MACHINE="$cputype"
+	fi
+	echo ${UNAME_MACHINE}-unknown-plan9
+	exit 0 ;;
+    i*86:OS/2:*:*)
+	# If we were able to find `uname', then EMX Unix compatibility
+	# is probably installed.
+	echo ${UNAME_MACHINE}-pc-os2-emx
+	exit 0 ;;
+    *:TOPS-10:*:*)
+	echo pdp10-unknown-tops10
+	exit 0 ;;
+    *:TENEX:*:*)
+	echo pdp10-unknown-tenex
+	exit 0 ;;
+    KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*)
+	echo pdp10-dec-tops20
+	exit 0 ;;
+    XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*)
+	echo pdp10-xkl-tops20
+	exit 0 ;;
+    *:TOPS-20:*:*)
+	echo pdp10-unknown-tops20
+	exit 0 ;;
+    *:ITS:*:*)
+	echo pdp10-unknown-its
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >$dummy.c <<EOF
+#ifdef _SEQUENT_
+# include <sys/types.h>
+# include <sys/utsname.h>
+#endif
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+#include <sys/param.h>
+  printf ("m68k-sony-newsos%s\n",
+#ifdef NEWSOS4
+          "4"
+#else
+	  ""
+#endif
+         ); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  if (version < 4)
+    printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
+  else
+    printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-pc-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+    struct utsname un;
+
+    uname(&un);
+
+    if (strncmp(un.version, "V2", 2) == 0) {
+	printf ("i386-sequent-ptx2\n"); exit (0);
+    }
+    if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
+	printf ("i386-sequent-ptx1\n"); exit (0);
+    }
+    printf ("i386-sequent-ptx\n"); exit (0);
+
+#endif
+
+#if defined (vax)
+# if !defined (ultrix)
+#  include <sys/param.h>
+#  if defined (BSD)
+#   if BSD == 43
+      printf ("vax-dec-bsd4.3\n"); exit (0);
+#   else
+#    if BSD == 199006
+      printf ("vax-dec-bsd4.3reno\n"); exit (0);
+#    else
+      printf ("vax-dec-bsd\n"); exit (0);
+#    endif
+#   endif
+#  else
+    printf ("vax-dec-bsd\n"); exit (0);
+#  endif
+# else
+    printf ("vax-dec-ultrix\n"); exit (0);
+# endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+eval $set_cc_for_build
+$CC_FOR_BUILD $dummy.c -o $dummy 2>/dev/null && ./$dummy && rm -f $dummy.c $dummy && exit 0
+rm -f $dummy.c $dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+cat >&2 <<EOF
+$0: unable to guess system type
+
+This script, last modified $timestamp, has failed to recognize
+the operating system you are using. It is advised that you
+download the most up to date version of the config scripts from
+
+    ftp://ftp.gnu.org/pub/gnu/config/
+
+If the version you run ($0) is already up to date, please
+send the following data and any information you think might be
+pertinent to <config-patches@gnu.org> in order to provide the needed
+information to handle your system.
+
+config.guess timestamp = $timestamp
+
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null`
+
+hostinfo               = `(hostinfo) 2>/dev/null`
+/bin/universe          = `(/bin/universe) 2>/dev/null`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null`
+/bin/arch              = `(/bin/arch) 2>/dev/null`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null`
+
+UNAME_MACHINE = ${UNAME_MACHINE}
+UNAME_RELEASE = ${UNAME_RELEASE}
+UNAME_SYSTEM  = ${UNAME_SYSTEM}
+UNAME_VERSION = ${UNAME_VERSION}
+EOF
+
+exit 1
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "timestamp='"
+# time-stamp-format: "%:y-%02m-%02d"
+# time-stamp-end: "'"
+# End:
Index: /main/tags/3.01/gsdl/config.h.in
===================================================================
--- /main/tags/3.01/gsdl/config.h.in	(revision 21112)
+++ /main/tags/3.01/gsdl/config.h.in	(revision 21112)
@@ -0,0 +1,217 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+#ifndef _GSDL_CONFIG_H
+#define _GSDL_CONFIG_H
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+/* Define if you don't want to use long long's with GCC */
+#undef DISABLE_LONG_LONG
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define FASTCGI if needsbe */
+#undef USE_FASTCGI
+
+/* Define USE_CORBA if needsbe */
+#undef USE_CORBA
+
+/* Define MICO_DIR if needsbe */
+#undef MICO_DIR
+
+/* Define MICO_VERSION if required */
+#undef MICO_VERSION
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#undef HAVE_STRERROR
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <crypt.h> header file.  */
+#undef HAVE_CRYPT_H
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the crypt library (-lcrypt).  */
+#undef HAVE_LIBCRYPT
+
+/* Define if you have the g library (-lg).  */
+#undef HAVE_LIBG
+
+/* Define if you have the gdbm library (-lgdbm).  */
+#undef HAVE_LIBGDBM
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if you have the stdc++ library (-lstdc++).  */
+#undef HAVE_LIBSTDC__
+
+/* Define if you have gettext (eg glibc) */
+#undef ENABLE_NLS
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
+
+#endif /* _GSDL_CONFIG_H */
Index: /main/tags/3.01/gsdl/config.sub
===================================================================
--- /main/tags/3.01/gsdl/config.sub	(revision 21112)
+++ /main/tags/3.01/gsdl/config.sub	(revision 21112)
@@ -0,0 +1,1375 @@
+#! /bin/sh
+# Configuration validation subroutine script.
+#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
+#   Free Software Foundation, Inc.
+
+timestamp='2001-06-08'
+
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Please send patches to <config-patches@gnu.org>.
+#
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# or in some cases, the newer four-part form:
+#	CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+me=`echo "$0" | sed -e 's,.*/,,'`
+
+usage="\
+Usage: $0 [OPTION] CPU-MFR-OPSYS
+       $0 [OPTION] ALIAS
+
+Canonicalize a configuration name.
+
+Operation modes:
+  -h, --help         print this help, then exit
+  -t, --time-stamp   print date of last modification, then exit
+  -v, --version      print version number, then exit
+
+Report bugs and patches to <config-patches@gnu.org>."
+
+version="\
+GNU config.sub ($timestamp)
+
+Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001
+Free Software Foundation, Inc.
+
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+
+help="
+Try \`$me --help' for more information."
+
+# Parse command line
+while test $# -gt 0 ; do
+  case $1 in
+    --time-stamp | --time* | -t )
+       echo "$timestamp" ; exit 0 ;;
+    --version | -v )
+       echo "$version" ; exit 0 ;;
+    --help | --h* | -h )
+       echo "$usage"; exit 0 ;;
+    -- )     # Stop option processing
+       shift; break ;;
+    - )	# Use stdin as input.
+       break ;;
+    -* )
+       echo "$me: invalid option $1$help"
+       exit 1 ;;
+
+    *local*)
+       # First pass through any local machine types.
+       echo $1
+       exit 0;;
+
+    * )
+       break ;;
+  esac
+done
+
+case $# in
+ 0) echo "$me: missing argument$help" >&2
+    exit 1;;
+ 1) ;;
+ *) echo "$me: too many arguments$help" >&2
+    exit 1;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any).
+# Here we must recognize all the valid KERNEL-OS combinations.
+maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
+case $maybe_os in
+  nto-qnx* | linux-gnu* | storm-chaos* | os2-emx* | windows32-*)
+    os=-$maybe_os
+    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+    ;;
+  *)
+    basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+    if [ $basic_machine != $1 ]
+    then os=`echo $1 | sed 's/.*-/-/'`
+    else os=; fi
+    ;;
+esac
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
+	-apple | -axis)
+		os=
+		basic_machine=$1
+		;;
+	-sim | -cisco | -oki | -wec | -winbond)
+		os=
+		basic_machine=$1
+		;;
+	-scout)
+		;;
+	-wrs)
+		os=-vxworks
+		basic_machine=$1
+		;;
+	-chorusos*)
+		os=-chorusos
+		basic_machine=$1
+		;;
+ 	-chorusrdb)
+ 		os=-chorusrdb
+		basic_machine=$1
+ 		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco5)
+		os=-sco3.2v5
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-udk*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-lynx*)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+	-psos*)
+		os=-psos
+		;;
+	-mint | -mint[0-9]*)
+		basic_machine=m68k-atari
+		os=-mint
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i860 | ia64 | m32r | m68k | m68000 | m88k | ns32k | arc \
+	        | arm | arme[lb] | arm[bl]e | armv[2345] | armv[345][lb] | strongarm | xscale \
+		| pyramid | mn10200 | mn10300 | tron | a29k \
+		| 580 | i960 | h8300 \
+		| x86 | ppcbe | mipsbe | mipsle | shbe | shle \
+		| hppa | hppa1.0 | hppa1.1 | hppa2.0 | hppa2.0w | hppa2.0n \
+		| hppa64 \
+		| alpha | alphaev[4-8] | alphaev56 | alphapca5[67] \
+		| alphaev6[78] \
+		| we32k | ns16k | clipper | i370 | sh | sh[34] \
+		| powerpc | powerpcle \
+		| 1750a | dsp16xx | pdp10 | pdp11 \
+		| mips16 | mips64 | mipsel | mips64el \
+		| mips64orion | mips64orionel | mipstx39 | mipstx39el \
+		| mips64vr4300 | mips64vr4300el | mips64vr4100 | mips64vr4100el \
+		| mips64vr5000 | mips64vr5000el | mcore | s390 | s390x \
+		| sparc | sparclet | sparclite | sparc64 | sparcv9 | sparcv9b \
+		| v850 | c4x \
+		| thumb | d10v | d30v | fr30 | avr | openrisc | tic80 \
+		| pj | pjl | h8500 | z8k)
+		basic_machine=$basic_machine-unknown
+		;;
+	m6811 | m68hc11 | m6812 | m68hc12)
+		# Motorola 68HC11/12.
+		basic_machine=$basic_machine-unknown
+		os=-none
+		;;
+	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
+		;;
+
+	# We use `pc' rather than `unknown'
+	# because (1) that's what they normally are, and
+	# (2) the word "unknown" tends to confuse beginning users.
+	i*86 | x86_64)
+	  basic_machine=$basic_machine-pc
+	  ;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	# FIXME: clean up the formatting here.
+	vax-* | tahoe-* | i*86-* | i860-* | ia64-* | m32r-* | m68k-* | m68000-* \
+	      | m88k-* | sparc-* | ns32k-* | fx80-* | arc-* | c[123]* \
+	      | arm-*  | armbe-* | armle-* | armv*-* | strongarm-* | xscale-* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | power-* | none-* | 580-* | cray2-* | h8300-* | h8500-* | i960-* \
+	      | xmp-* | ymp-* \
+	      | x86-* | ppcbe-* | mipsbe-* | mipsle-* | shbe-* | shle-* \
+	      | hppa-* | hppa1.0-* | hppa1.1-* | hppa2.0-* | hppa2.0w-* \
+	      | hppa2.0n-* | hppa64-* \
+	      | alpha-* | alphaev[4-8]-* | alphaev56-* | alphapca5[67]-* \
+	      | alphaev6[78]-* \
+	      | we32k-* | cydra-* | ns16k-* | pn-* | np1-* | xps100-* \
+	      | clipper-* | orion-* \
+	      | sparclite-* | pdp10-* | pdp11-* | sh-* | sh[34]-* | sh[34]eb-* \
+	      | powerpc-* | powerpcle-* | sparc64-* | sparcv9-* | sparcv9b-* | sparc86x-* \
+	      | mips16-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* \
+	      | mips64vr4100-* | mips64vr4100el-* | mips64vr4300-* | mips64vr4300el-* \
+	      | mipstx39-* | mipstx39el-* | mcore-* \
+	      | f30[01]-* | f700-* | s390-* | s390x-* | sv1-* | t3e-* \
+	      | [cjt]90-* \
+	      | m88110-* | m680[01234]0-* | m683?2-* | m68360-* | z8k-* | d10v-* \
+	      | thumb-* | v850-* | d30v-* | tic30-* | tic80-* | c30-* | fr30-* \
+	      | bs2000-* | tic54x-* | c54x-* | x86_64-* | pj-* | pjl-*)
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	386bsd)
+		basic_machine=i386-unknown
+		os=-bsd
+		;;
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	a29khif)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+	adobe68k)
+		basic_machine=m68010-adobe
+		os=-scout
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-unknown
+		;;
+	amigaos | amigados)
+		basic_machine=m68k-unknown
+		os=-amigaos
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-unknown
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	apollo68bsd)
+		basic_machine=m68k-apollo
+		os=-bsd
+		;;
+	aux)
+		basic_machine=m68k-apple
+		os=-aux
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	[cjt]90)
+		basic_machine=${basic_machine}-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	cris | cris-* | etrax*)
+		basic_machine=cris-axis
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	es1800 | OSE68k | ose68k | ose | OSE)
+		basic_machine=m68k-ericsson
+		os=-ose
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	go32)
+		basic_machine=i386-pc
+		os=-go32
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	h8300xray)
+		basic_machine=h8300-hitachi
+		os=-xray
+		;;
+	h8500hms)
+		basic_machine=h8500-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp3k9[0-9][0-9] | hp9[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k6[0-9][0-9] | hp6[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hp9k7[0-79][0-9] | hp7[0-79][0-9])
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k78[0-9] | hp78[0-9])
+		# FIXME: really hppa2.0-hp
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893)
+		# FIXME: really hppa2.0-hp
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][13679] | hp8[0-9][13679])
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hppa-next)
+		os=-nextstep3
+		;;
+	hppaosf)
+		basic_machine=hppa1.1-hp
+		os=-osf
+		;;
+	hppro)
+		basic_machine=hppa1.1-hp
+		os=-proelf
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i*86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv32
+		;;
+	i*86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv4
+		;;
+	i*86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv
+		;;
+	i*86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-solaris2
+		;;
+	i386mach)
+		basic_machine=i386-mach
+		os=-mach
+		;;
+	i386-vsta | vsta)
+		basic_machine=i386-unknown
+		os=-vsta
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	mingw32)
+		basic_machine=i386-pc
+		os=-mingw32
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	*mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*)
+		basic_machine=m68k-atari
+		os=-mint
+		;;
+	mipsel*-linux*)
+		basic_machine=mipsel-unknown
+		os=-linux-gnu
+		;;
+	mips*-linux*)
+		basic_machine=mips-unknown
+		os=-linux-gnu
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	mmix*)
+		basic_machine=mmix-knuth
+		os=-mmixware
+		;;
+	monitor)
+		basic_machine=m68k-rom68k
+		os=-coff
+		;;
+	msdos)
+		basic_machine=i386-pc
+		os=-msdos
+		;;
+	mvs)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	netbsd386)
+		basic_machine=i386-unknown
+		os=-netbsd
+		;;
+	netwinder)
+		basic_machine=armv4l-rebel
+		os=-linux
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	necv70)
+		basic_machine=v70-nec
+		os=-sysv
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	mon960)
+		basic_machine=i960-intel
+		os=-mon960
+		;;
+	nonstopux)
+		basic_machine=mips-compaq
+		os=-nonstopux
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	nsr-tandem)
+		basic_machine=nsr-tandem
+		;;
+	op50n-* | op60c-*)
+		basic_machine=hppa1.1-oki
+		os=-proelf
+		;;
+	OSE68000 | ose68000)
+		basic_machine=m68000-ericsson
+		os=-ose
+		;;
+	os68k)
+		basic_machine=m68k-none
+		os=-os68k
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium | p5 | k5 | k6 | nexgen)
+		basic_machine=i586-pc
+		;;
+	pentiumpro | p6 | 6x86 | athlon)
+		basic_machine=i686-pc
+		;;
+	pentiumii | pentium2)
+		basic_machine=i686-pc
+		;;
+	pentium-* | p5-* | k5-* | k6-* | nexgen-*)
+		basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pentiumpro-* | p6-* | 6x86-* | athlon-*)
+		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pentiumii-* | pentium2-*)
+		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	power)	basic_machine=power-ibm
+		;;
+	ppc)	basic_machine=powerpc-unknown
+	        ;;
+	ppc-*)	basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ppcle | powerpclittle | ppc-le | powerpc-little)
+		basic_machine=powerpcle-unknown
+	        ;;
+	ppcle-* | powerpclittle-*)
+		basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	pw32)
+		basic_machine=i586-unknown
+		os=-pw32
+		;;
+	rom68k)
+		basic_machine=m68k-rom68k
+		os=-coff
+		;;
+	rm[46]00)
+		basic_machine=mips-siemens
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sa29200)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sparclite-wrs)
+		basic_machine=sparclite-wrs
+		os=-vxworks
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	st2000)
+		basic_machine=m68k-tandem
+		;;
+	stratus)
+		basic_machine=i860-stratus
+		os=-sysv4
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun4sol2)
+		basic_machine=sparc-sun
+		os=-solaris2
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	sv1)
+		basic_machine=sv1-cray
+		os=-unicos
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	t3e)
+		basic_machine=t3e-cray
+		os=-unicos
+		;;
+	tic54x | c54x*)
+		basic_machine=tic54x-unknown
+		os=-coff
+		;;
+	tx39)
+		basic_machine=mipstx39-unknown
+		;;
+	tx39el)
+		basic_machine=mipstx39el-unknown
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	udi29k)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	v810 | necv810)
+		basic_machine=v810-nec
+		os=-none
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vpp*|vx|vx-*)
+               basic_machine=f301-fujitsu
+               ;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	vxworks29k)
+		basic_machine=a29k-wrs
+		os=-vxworks
+		;;
+	w65*)
+		basic_machine=w65-wdc
+		os=-none
+		;;
+	w89k-*)
+		basic_machine=hppa1.1-winbond
+		os=-proelf
+		;;
+	windows32)
+		basic_machine=i386-pc
+		os=-windows32-msvcrt
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	z8k-*-coff)
+		basic_machine=z8k-unknown
+		os=-sim
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	w89k)
+		basic_machine=hppa1.1-winbond
+		;;
+	op50n)
+		basic_machine=hppa1.1-oki
+		;;
+	op60c)
+		basic_machine=hppa1.1-oki
+		;;
+	mips)
+		if [ x$os = x-linux-gnu ]; then
+			basic_machine=mips-unknown
+		else
+			basic_machine=mips-mips
+		fi
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp10)
+		# there are many clones, so DEC is not a safe bet
+		basic_machine=pdp10-unknown
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sh3 | sh4)
+		basic_machine=sh-unknown
+		;;
+	sparc | sparcv9 | sparcv9b)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	mac | mpw | mac-mpw)
+		basic_machine=m68k-apple
+		;;
+	pmac | pmac-mpw)
+		basic_machine=powerpc-apple
+		;;
+	c4x*)
+		basic_machine=c4x-none
+		os=-coff
+		;;
+	*-unknown)
+		# Make sure to match an already-canonicalized machine name.
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+        # First match some system type aliases
+        # that might get confused with valid system types.
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-svr4*)
+		os=-sysv4
+		;;
+	-unixware*)
+		os=-sysv4.2uw
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative MUST END IN A *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
+	      | -aos* \
+	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
+	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -openbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
+	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
+	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
+	      | -chorusos* | -chorusrdb* \
+	      | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+	      | -mingw32* | -linux-gnu* | -uxpv* | -beos* | -mpeix* | -udk* \
+	      | -interix* | -uwin* | -rhapsody* | -darwin* | -opened* \
+	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
+	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* | -os2*)
+	# Remember, each alternative MUST END IN *, to match a version number.
+		;;
+	-qnx*)
+		case $basic_machine in
+		    x86-* | i*86-*)
+			;;
+		    *)
+			os=-nto$os
+			;;
+		esac
+		;;
+	-nto*)
+		os=-nto-qnx
+		;;
+	-sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
+	      | -windows* | -osx | -abug | -netware* | -os9* | -beos* \
+	      | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
+		;;
+	-mac*)
+		os=`echo $os | sed -e 's|mac|macos|'`
+		;;
+	-linux*)
+		os=`echo $os | sed -e 's|linux|linux-gnu|'`
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-opened*)
+		os=-openedition
+		;;
+	-wince*)
+		os=-wince
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-386bsd)
+		os=-bsd
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-ns2 )
+	        os=-nextstep2
+		;;
+	-nsk*)
+		os=-nsk
+		;;
+	# Preserve the version number of sinix5.
+	-sinix5.*)
+		os=`echo $os | sed -e 's|sinix|sysv|'`
+		;;
+	-sinix*)
+		os=-sysv4
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-ose*)
+		os=-ose
+		;;
+	-es1800*)
+		os=-ose
+		;;
+	-xenix)
+		os=-xenix
+		;;
+        -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+	        os=-mint
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+	arm*-rebel)
+		os=-linux
+		;;
+	arm*-semi)
+		os=-aout
+		;;
+	pdp10-*)
+		os=-tops20
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	m68*-apollo)
+		os=-domain
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	m68*-cisco)
+		os=-aout
+		;;
+	mips*-cisco)
+		os=-elf
+		;;
+	mips*-*)
+		os=-elf
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-be)
+		os=-beos
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-wec)
+		os=-proelf
+		;;
+	*-winbond)
+		os=-proelf
+		;;
+	*-oki)
+		os=-proelf
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigaos
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-next )
+		os=-nextstep
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+        *-siemens)
+		os=-sysv4
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	f30[01]-fujitsu | f700-fujitsu)
+		os=-uxpv
+		;;
+	*-rom68k)
+		os=-coff
+		;;
+	*-*bug)
+		os=-coff
+		;;
+	*-apple)
+		os=-macos
+		;;
+	*-atari*)
+		os=-mint
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-beos*)
+				vendor=be
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-mpeix*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs* | -opened*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+			-vxsim* | -vxworks*)
+				vendor=wrs
+				;;
+			-aux*)
+				vendor=apple
+				;;
+			-hms*)
+				vendor=hitachi
+				;;
+			-mpw* | -macos*)
+				vendor=apple
+				;;
+			-*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+				vendor=atari
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
+exit 0
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "timestamp='"
+# time-stamp-format: "%:y-%02m-%02d"
+# time-stamp-end: "'"
+# End:
Index: /main/tags/3.01/gsdl/configtest.pl
===================================================================
--- /main/tags/3.01/gsdl/configtest.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/configtest.pl	(revision 21112)
@@ -0,0 +1,1 @@
+print $]
Index: /main/tags/3.01/gsdl/etc/marctodc.txt
===================================================================
--- /main/tags/3.01/gsdl/etc/marctodc.txt	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/marctodc.txt	(revision 21112)
@@ -0,0 +1,86 @@
+#
+# This file is used by MARCPlug for mapping MARC field numbers to
+# Greenstone-style metadata.
+#
+# The field numbers listed in this file are for USMARC / MARC21.
+#
+# Lines beginning with "#" are comments.
+# The format for these files are:
+#  <number> -> GreenstoneMetadataName
+#
+# This is loosely based on the MARC -> Dublin Core mapping as found at
+#   http://lcweb.loc.gov/marc/dccross.html
+#
+# The problem with that is that some mappings are conditional on
+# subfield values...
+
+### Contributor	###
+# 720 = "Uncontrolled Name", with $e=collaborator
+
+### Coverage	###
+#500 -> Coverage
+
+### Creator	###
+# 720 = "Uncontrolled Name" with $e=author
+720 -> Creator
+# 100 = "Personal Name"
+100	-> Creator
+# 110 = "Corporate Name" 
+110	-> Creator
+# 111 = "Meeting Name"
+111	-> Creator
+
+### Date	###
+# Don't do this yet, as greenstone expects dates in YYYYMMDD format...
+# 260, $c (but also see Publisher below)
+# 260 -> Date
+
+### Description	###
+# 520 = "Summary, Note"
+520 -> Description
+
+### Identifier	###
+# 024 = "Standard Identifier/number"
+# greenstone uses its own Identifier metadata
+024 -> MarcIdentifier
+
+# 856 = "Electronic Location"
+856 -> URL
+
+### Language	###
+# Greenstone has its own Language metadata...
+546 -> MarcLanguage
+
+### Publisher	###
+# 260 = "Publication, etc"
+# 260 $b if $e=publisher... (but see Date)
+260 -> Publisher
+
+### Relation	###
+# 787 = "Nonspecific Relationship Note"
+787 -> Relation
+
+### Rights	###
+# 540 = "Reproduction Note"
+540 -> Rights
+
+### Source	###
+# greenstone has its own Source metadata...
+# 786 = "Data Source Entry"
+786 -> MarcSource
+
+### Subject	###
+# 650 = "Subject: Topical Term"
+650 -> Subject
+# 653 = "Index Term: Uncontrolled"
+653 -> Subject
+
+### Title	###
+# 245 = "Title Statement"
+245	-> Title
+# 246 = "Varying Form of Title"
+
+### Type	###
+# 655 = "Index Term - Genre/Form"
+655 -> Type
+
Index: /main/tags/3.01/gsdl/etc/packages/example.dbi
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/example.dbi	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/example.dbi	(revision 21112)
@@ -0,0 +1,92 @@
+# -*- perl -*-
+# This is an example configuration file for greenstone's DBPlug
+# (DataBase plugin).
+#
+# This config file should be a fragment of perl, specifying how to
+# get records out of a database. This uses perl's DBI module, which
+# includes back-ends for mysql, postgresql, comma separated values (CSV),
+# MS Excel, ODBC, sybase, etc...
+#
+# You need to install the separate backends - For example, on debian you
+# need the libdbi-perl base package and then the libdbd-mysql-perl,
+# libdbd-csv-perl, libdbd-pg-perl and/or libdbd-sybase-perl (etc) packages.
+
+
+# debugging. Uncomment this line if you want extra information printed
+# out during greenstone's import process.
+#
+$dbplug_debug=1;
+
+
+# If these are commented out, then we will use the plugin's defaults.
+# However, you might want to override them here if you know what they
+# are for this particular database
+# run "$ pluginfo.pl DBPlug | less" to see a list of valid encodings
+# Language metadata isn't really necessary unless you want indexes by language
+# However, Encoding is important, for converting data to utf8
+#
+$language = "en";
+$encoding = "iso_8859_1";
+
+
+# Data Source.
+# This is going to be dependent on which back-end you use.
+# Eg, for postgres, it is "DBI:Pg:dbname=<database>".
+# For mysql, it might look like "DBI:mysql:database=<database>;host=localhost"
+# For the CSV (comma-separated values text-file) backend, try something like
+#   'DBI:CSV:f_dir=/path/to/file;csv_quote_char=\";csv_sep_char=	'
+# with an embedded tab character for tab-separated instead of the
+# default comma-separated.
+# Case may be important!
+#
+#$db='';
+
+# If you need a username and password to connect to this database,
+# uncomment these lines and fill in the details
+#
+#$username='';
+#$password='';
+
+# sql query required to get your rows out of the database. Each row will
+# become a separate greenstone document.
+#
+#$sql_query = 'SELECT id, name,  data FROM table1 JOIN table2 WHERE something';
+# For CSV, the table name is the file name. Eg:
+#$sql_query = 'SELECT * FROM values.txt'
+# CSV may also require DOS '\r\n'-style line endings instead of unix '\n'.
+
+
+# a map of database field names to Greenstone metadata field names.
+# You should have a "Title" field and a "text" field, at a minimum.
+# If you want to give records a unique identifier from a database field
+# rather than have greenstone perform a hash on the document, also provide
+# an "Identifier" metadata field.
+# If you do map something to Identifier, make sure that the field is unique
+# for each record, and also note that because greenstone does special things
+# if this field is only numeric, DBPlug will add some letters to the front
+# if this is the case.
+#
+#%db_to_greenstone_fields=(
+#			  "name" => "Title",
+#			  "data" => "text",
+#			  "id" => "Identifier"
+#			  );
+
+
+# finally, you can declare subroutines here to post-process the fields
+# in case you want to do any mark up or processing of it. 
+# For example, you might want to do HTML formatting.
+# The subroutine's name should be the greenstone metadata field name,
+# followed by "_callback", and must end with "}" in the first column.
+#
+#sub text_callback {
+#    my $text=shift;
+#    # do markup stuff...
+#    return $text;
+#}
+#
+#sub Title_callback {
+#    my $title=shift;
+#    return ucfirst($title); # uppercase first letter
+#}
+
Index: /main/tags/3.01/gsdl/etc/packages/phind/stopword/en/brown.sw
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/phind/stopword/en/brown.sw	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/phind/stopword/en/brown.sw	(revision 21112)
@@ -0,0 +1,2009 @@
+a
+aah
+aback
+abed
+abjectly
+ablaze
+ably
+abnormally
+aboard
+about
+abreast
+abroad
+abroade
+abruptly
+absently
+absentmindedly
+absolutely
+abstractly
+absurdly
+abundantly
+academically
+accidentally
+according
+accordingly
+accurately
+accusingly
+acourse
+acoustically
+across
+actively
+actually
+actuarially
+acutely
+adamantly
+additionally
+adequately
+administratively
+admirably
+admiringly
+admittedly
+adrift
+advantageously
+adversely
+advisedly
+afar
+affectingly
+affectionately
+affirmatively
+afield
+afloat
+afoot
+aforethought
+afresh
+after
+afterward
+afterwards
+aftuh
+again
+against
+agayne
+aggressively
+agilely
+agin
+ago
+agreeably
+agriculturally
+ah
+ahah
+ahead
+ahem
+ahn
+ai
+aimlessly
+airily
+ajar
+alarmingly
+alas
+albeit
+alertly
+algebraically
+alike
+all
+allegedly
+allso
+almost
+aloft
+alone
+along
+alongside
+aloud
+alreadeh
+already
+also
+alternately
+alternatively
+altho
+although
+altogether
+altruistically
+always
+am
+amazingly
+ambitiously
+ame
+amen
+amicably
+amid
+amidst
+among
+amongst
+amorphously
+amply
+amusedly
+amusingly
+an
+analogously
+analytically
+anatomically
+anciently
+and
+anesthetically
+anew
+angrily
+anhydrously
+ani
+annually
+another
+anti
+anxiously
+any
+anybody
+anye
+anyhow
+anymore
+anyone
+anyplace
+anythin
+anything
+anyway
+anyways
+anywhere
+apart
+apiece
+apologetically
+appallingly
+apparently
+appraisingly
+appreciably
+appreciatively
+apprehensively
+appropriately
+approvingly
+approximately
+aptly
+arbitrarily
+are
+aristocratically
+around
+arrogantly
+artfully
+artificially
+artistically
+as
+asham
+ashore
+aside
+askance
+askew
+asleep
+associatively
+assuredly
+astonishingly
+astray
+astride
+astronomically
+asunder
+asymmetrically
+asymptotically
+at
+atonally
+atop
+atrociously
+att
+atta
+attentively
+attractively
+audibly
+aurally
+auspiciously
+austerely
+authentically
+authoritatively
+automatically
+avidly
+aw
+away
+awaye
+awfully
+awhile
+awkwardly
+axially
+aye
+backstage
+backstairs
+backward
+backwards
+badly
+bah
+bannnnnng
+barely
+basically
+be
+beautifully
+because
+beefore
+been
+befoh
+before
+beforehand
+behahn
+behaviorally
+behind
+bein
+being
+beinge
+belatedly
+believably
+belligerently
+below
+beneath
+beside
+besides
+between
+bewilderedly
+bewilderingly
+beyond
+bi
+billion
+biologically
+bitterly
+blandly
+bleakly
+blindly
+blissfully
+blithely
+bluntly
+boastfully
+bodily
+boldly
+bong
+both
+bravely
+brazenly
+breathlessly
+briefly
+brightly
+brilliantly
+briskly
+broadly
+brokenly
+brusquely
+brutally
+busily
+but
+by
+callously
+calmly
+can
+candidly
+cannot
+capably
+carefully
+carelessly
+casually
+catastrophically
+categorically
+causally
+cautiously
+ceaselessly
+centrally
+ceremonially
+ceremoniously
+certainly
+characteristically
+charmingly
+cheaply
+cheerfully
+chemically
+chiefly
+childishly
+chronically
+chronologically
+circa
+circumspectly
+clairaudiently
+classically
+cleanly
+clearly
+cleverly
+clinically
+clockwise
+closely
+clumsily
+coarsely
+cogently
+cohesively
+colde
+coldly
+collectively
+comfortably
+comically
+commercially
+commonly
+compactly
+comparatively
+compassionately
+competently
+competitively
+completely
+comprehensively
+compulsively
+concededly
+conceivably
+conceptually
+concerning
+conclusively
+concretely
+concurrently
+confidentially
+confidently
+consanguineously
+consciously
+consequently
+considerably
+considerately
+consisently
+consistently
+conspicuously
+constantly
+constructively
+consummately
+contemptuously
+contentedly
+continentally
+continually
+continuously
+contradictorily
+contrarily
+conveniently
+conventionally
+conversely
+convincingly
+convulsively
+coolly
+copiously
+correctly
+correlatively
+correspondingly
+cortically
+cosily
+coud
+coudn
+could
+courageously
+courteously
+covertly
+coyly
+crazily
+creatively
+credibly
+crisply
+critically
+crossways
+crosswise
+crucially
+crudely
+cruelly
+culturally
+cunningly
+curiously
+currently
+curtly
+curvaceously
+customarily
+cynically
+d
+da
+daintily
+dammit
+damnit
+dandily
+dangerously
+darkly
+dearly
+deathly
+decently
+deceptively
+decidedly
+decisively
+deeply
+defiantly
+definitely
+dejectedly
+deliberately
+delicately
+deliciously
+delightfully
+demandingly
+demonstrably
+denominationally
+depending
+deplorably
+depressingly
+dere
+derisively
+despairingly
+desperately
+despite
+determinedly
+devastatingly
+devotedly
+devoutly
+dey
+diagonally
+dialectically
+diametrically
+did
+diddle
+didn
+differently
+diffusely
+diligently
+dimensionally
+dimly
+directionally
+directly
+disapprovingly
+disconcertingly
+discordantly
+discreetly
+dismally
+dispassionately
+disproportionately
+distally
+distantly
+distastefully
+distinctively
+distinctly
+distractedly
+disturbingly
+divinely
+dizzily
+do
+doan
+docilely
+doctrinally
+does
+doggedly
+doggone
+dogmatically
+domestically
+dominantly
+doo
+dost
+doubly
+doubtfully
+doubtingly
+doubtless
+dourly
+down
+downright
+downstairs
+downwind
+dramatically
+drastically
+dreadfully
+dreamlessly
+drowsily
+drunkenly
+druther
+dryly
+dully
+duly
+during
+dutifully
+dynamically
+each
+eagerly
+earnestly
+easily
+eastward
+eclectically
+economically
+edgewise
+editorially
+eerily
+effectively
+efficaciously
+efficiently
+effortlessly
+egregiously
+eh
+eight
+eighteen
+eighteenth
+eighth
+eighty
+either
+elaborately
+electrically
+elegantly
+eleven
+eleventh
+eloquently
+else
+elsewhere
+em
+embarrassingly
+eminently
+emotionally
+emphatically
+empirically
+enchantingly
+encouragingly
+endlessly
+enduringly
+energetically
+engagingly
+enny
+enormously
+enough
+enterprisingly
+enthusiastically
+entirely
+enviably
+enviously
+epicyclically
+equally
+equidistantly
+equitably
+ere
+erotically
+erratically
+erroneously
+especially
+essentially
+etcetera
+ether
+ethically
+evenly
+eventfully
+eventshahleh
+eventually
+evenutally
+ever
+everlastingly
+every
+everybody
+everyone
+everything
+everywhere
+evidently
+exactly
+exasperatingly
+exceedingly
+excellently
+except
+excepting
+exceptionally
+excessively
+excitedly
+excluding
+exclusively
+exhaustingly
+exhaustively
+expansively
+expectantly
+expectedly
+expeditiously
+experientially
+experimentally
+expertly
+explicitly
+explosively
+expressly
+exquisitely
+extempore
+extensively
+externally
+extraordinarily
+extremely
+exuberantly
+exultantly
+facetiously
+faintly
+fairly
+faithfully
+familiarly
+fantastically
+farther
+fascinatingly
+fatally
+favorably
+fearfully
+fearlessly
+feebly
+fer
+ferociously
+fervently
+feverishly
+few
+fewer
+fiercely
+fifteen
+fifteenth
+fifth
+fiftieth
+fifty
+finally
+financially
+finely
+firmly
+first
+fitfully
+five
+flagrantly
+flamboyantly
+flatly
+flatteringly
+fluently
+focally
+foh
+fondly
+foolishly
+for
+forcibly
+foremost
+forever
+formally
+former
+formerly
+formidably
+forth
+forthrightly
+fortunately
+forty
+foully
+four
+fourteen
+fourteenth
+fourth
+fragmentarily
+frankly
+frantically
+freely
+frenziedly
+frequently
+freshly
+friendlily
+frighteningly
+frightfully
+from
+frowningly
+frugally
+fruitfully
+fruitlessly
+fully
+functionally
+fundamentally
+furiously
+furthermore
+furtively
+fussily
+futhermore
+gaily
+gaspingly
+gawdamighty
+generally
+generously
+gently
+genuinely
+geographically
+geometrically
+girlishly
+gladly
+glaringly
+gleefully
+glibly
+globally
+gloomily
+gloriously
+glumly
+godamit
+goddamit
+goddammit
+golly
+goodbye
+goodnight
+goody
+goolick
+goooolick
+gorgeously
+gosh
+governmentally
+gracefully
+graciously
+gradually
+grammatically
+grandly
+graphically
+gratefully
+gratifyingly
+gratingly
+gratis
+gratuitously
+gravely
+greatly
+greedily
+greenly
+grevouselye
+grimly
+grossly
+grotesquely
+grudgingly
+ha
+habitually
+had
+hadd
+haf
+haggardly
+haint
+half
+halfways
+haltingly
+handsomely
+haphazardly
+happily
+harder
+hardly
+harmlessly
+harmoniously
+harro
+harshly
+has
+hast
+hastily
+hath
+haughtily
+have
+having
+he
+healthily
+heartily
+heatedly
+heavily
+hello
+helpfully
+helplessly
+hence
+henceforth
+henh
+her
+here
+hereabouts
+hereafter
+hereby
+herein
+hereinafter
+heretofore
+hereunto
+herewith
+heroically
+hers
+herself
+hesitantly
+hesitatingly
+hev
+hevin
+hey
+hi
+hideously
+highly
+hilariously
+him
+himself
+himselfe
+hir
+his
+hisself
+historically
+hither
+hitherto
+hmpf
+hoarsely
+homeward
+homewards
+homogeneously
+honestly
+honorably
+hopefully
+hopelessly
+horizontally
+horribly
+horrifyingly
+hotly
+how
+howe
+however
+howsomever
+hubba
+huh
+humanely
+humanly
+humbly
+humiliatingly
+hundred
+hundredth
+hurriedly
+huskily
+hydraulically
+hym
+hyperbolically
+hypnotically
+hypothalamically
+hys
+hytt
+ideally
+identically
+idiotically
+idly
+if
+illegally
+imaginatively
+immeasurably
+immediately
+immensely
+impassively
+impatiently
+impeccably
+imperceptibly
+imperfectly
+imperiously
+impersonally
+implausibly
+implicitly
+importantly
+importunately
+impossibly
+imprecisely
+improbably
+improperly
+imprudently
+impudently
+in
+inadequately
+inadvertently
+inasmuch
+incessantly
+incidentally
+including
+incoherently
+incomparably
+incompletely
+inconspicuously
+inconveniently
+increasingly
+incredibly
+incredulously
+incurably
+indecisively
+indeed
+indefinitely
+indelibly
+independently
+indignantly
+indirectly
+indiscriminantly
+indisputably
+individually
+indolently
+indoors
+industrially
+industriously
+ineffectively
+ineptly
+inescapably
+inevitably
+inexorably
+inexplicably
+inexpressibly
+infernally
+infinitely
+infinitesimally
+informally
+infra
+infrequently
+ingeniously
+inherently
+initially
+inland
+innocently
+inordinately
+insanely
+insidiously
+insofar
+insolently
+insomma
+instantaneously
+instantly
+instead
+instinctively
+instrumentally
+insufficiently
+insuperably
+intellectually
+intelligently
+intensely
+intensively
+intentionally
+intently
+interestingly
+intermittently
+internally
+internationally
+intimately
+into
+intramuscularly
+intricately
+intriguingly
+intrinsically
+intuitively
+invariably
+inversely
+invisibly
+involuntarily
+inwardly
+ironically
+irrationally
+irredeemably
+irregularly
+irreparably
+irresistibly
+irreversibly
+irrevocably
+irritably
+is
+isothermally
+it
+its
+itself
+jaggedly
+jealously
+jist
+jocularly
+jointly
+joyfully
+joyously
+jubilantly
+judiciously
+july
+jurisprudentially
+justifiably
+justly
+kaboom
+keenly
+kin
+kinda
+kinesthetically
+knowingly
+laboriously
+lahk
+lak
+largely
+last
+lastly
+lately
+latter
+laudably
+laughingly
+lavishly
+lazily
+least
+legally
+legitimately
+lengthily
+less
+lest
+lewdly
+liberally
+lightly
+likewise
+limply
+linearly
+linguistically
+listlessly
+literally
+litle
+lo
+locally
+logically
+loosely
+lopsidedly
+loudly
+lovingly
+luckily
+lustily
+m
+macroscopically
+madly
+magically
+magnetically
+magnificently
+mah
+maht
+mai
+mainly
+majestically
+maliciously
+manifestly
+manually
+many
+manye
+marginally
+markedly
+marketwise
+marvelously
+masterfully
+materially
+mathematically
+may
+maybe
+maye
+mayst
+me
+meaningfully
+meanwhile
+measurably
+mechanically
+mechanochemically
+medically
+mee
+meekly
+melodically
+mentally
+mercifully
+mercilessly
+merely
+merrily
+methodically
+meticulously
+metrically
+microscopically
+midway
+might
+mightily
+mildly
+militantly
+militarily
+million
+mimetically
+mine
+minimally
+minus
+minutely
+miraculously
+miserably
+mistakenly
+mmm
+mmmm
+mockingly
+moderately
+modestly
+momentarily
+monolithically
+monumentally
+moodily
+mor
+morally
+more
+moreover
+morosely
+mortally
+most
+mostly
+mountainously
+mournfully
+movingly
+much
+municipally
+musically
+must
+mutely
+mutually
+my
+myn
+myne
+myself
+mysteriously
+n
+na
+nae
+naively
+nakedly
+namely
+narrowly
+nary
+natch
+nationally
+naturally
+naught
+naw
+nawt
+nay
+nearly
+nearsightedly
+neatly
+necessarily
+needlessly
+negatively
+neither
+nervously
+nether
+never
+nevertheless
+next
+nicely
+nigh
+nightly
+nilly
+nimbly
+nine
+nineteen
+nineteenth
+ninetieth
+ninety
+ninth
+no
+nobody
+noisily
+nominally
+noncommittally
+nondescriptly
+none
+nonetheless
+nonspecifically
+nope
+nor
+normally
+northward
+nostalgically
+not
+notably
+noth
+nothin
+nothing
+noticeably
+notoriously
+nott
+notwithstanding
+now
+nowadays
+nowhere
+nuf
+numbingly
+numerically
+o
+objectively
+obligingly
+obliquely
+obscurely
+obviously
+occasionally
+octillion
+oddly
+of
+off
+offensively
+officially
+offstage
+oft
+often
+oftener
+oftentimes
+oh
+okay
+ominously
+on
+once
+onct
+one
+oneself
+onleh
+only
+onto
+ontologically
+onward
+onwards
+oooo
+oops
+openly
+operationally
+optically
+or
+orally
+ordinarily
+organically
+organizationally
+originally
+ornately
+ostensibly
+other
+otherwise
+ought
+our
+ours
+ourselves
+out
+outdoors
+outstandingly
+outwardly
+over
+overboard
+overland
+overly
+overtly
+overwhelmingly
+ovuh
+owly
+ownself
+painfully
+painlessly
+painstakingly
+palely
+palpably
+paradoxically
+parenthetically
+partially
+particularly
+partly
+passionately
+passively
+paternally
+patiently
+peacefully
+peculiarly
+per
+perchance
+perennially
+perfectly
+perforce
+perfunctorily
+perhaps
+perilously
+periodically
+peripherally
+permanently
+perpendicularly
+perpetually
+persistently
+personally
+persuasively
+pervasively
+perversely
+philosophically
+photographically
+physically
+physiologically
+pictorially
+piecewise
+pinkly
+piously
+pip
+pitifully
+pitilessly
+pityingly
+plainly
+plastically
+pleasantly
+pleasingly
+plumb
+plus
+poetically
+poignantly
+pointedly
+politely
+politically
+pompously
+poorly
+popularly
+positively
+possibly
+potentially
+powerfully
+practically
+prayerfully
+precariously
+precisely
+precociously
+predictably
+predominantly
+predominately
+preferably
+preferentially
+prematurely
+preponderantly
+presente
+presently
+presentlye
+presumably
+prettily
+pretty
+previously
+primarily
+primly
+principally
+privately
+probably
+probly
+prodigally
+professedly
+professionally
+profitably
+profoundly
+profusely
+progressively
+prominently
+promptly
+pronto
+properly
+prophetically
+proportionally
+proportionately
+protectively
+proudly
+provocatively
+prudentially
+prudently
+psychically
+psychologically
+publically
+publicly
+pugh
+punctually
+pungently
+purely
+purportedly
+purposefully
+purposely
+purposively
+pursuant
+quadrillion
+qualitatively
+quantitatively
+querulously
+questioningly
+quickly
+quietly
+quintillion
+quite
+quitely
+racially
+radically
+rakishly
+randomly
+rapidly
+rarely
+rather
+rationally
+raucously
+readily
+realistically
+really
+reasonably
+reassuringly
+rebelliously
+recently
+recklessly
+rectlinearly
+recurrently
+reflexly
+refreshingly
+regarding
+regardless
+regionally
+regretfully
+regrettably
+regularly
+relatively
+relentlessly
+reliably
+religiously
+reluctantly
+remarkably
+remotely
+repeatedly
+reportedly
+reproducibly
+reprovingly
+reputedly
+residentially
+resignedly
+resolutely
+resourcefully
+respectfully
+respectively
+responsibly
+responsively
+restively
+restlessly
+reverently
+rhythmically
+richly
+ridiculously
+rightfully
+rightly
+rigidly
+rigorously
+rollickingly
+romantically
+rotationally
+roughly
+roughtly
+roundly
+routinely
+rudely
+ruefully
+ruggedly
+ruthlessly
+sadly
+safely
+same
+sarcastically
+satirically
+satisfactorily
+savagely
+scarcely
+scathingly
+schematically
+scholastically
+scientifically
+scornfully
+scrupulously
+searchingly
+seasonally
+second
+secondarily
+secondly
+secretly
+securely
+sedately
+sedulously
+seekingly
+seemingly
+seldom
+selectively
+semantically
+senselessly
+sensibly
+sensitively
+separately
+septillion
+serenely
+seriously
+seven
+seventeen
+seventeenth
+seventh
+seventy
+several
+severally
+severely
+severly
+sextillion
+sexually
+sha
+shabbily
+shakily
+shall
+shamefacedly
+sharply
+shatteringly
+she
+shh
+shiningly
+shockingly
+shortly
+should
+shrewdly
+shrilly
+shucks
+shuld
+shulde
+shyly
+sidewise
+signally
+significantly
+silently
+similarly
+simply
+simultaneously
+since
+sincerely
+single
+singlehandedly
+singly
+singularly
+sinuously
+six
+sixteen
+sixteenth
+sixth
+sixty
+skeptically
+skilfully
+skillfully
+sleepily
+sleeplessly
+slightly
+slimly
+sloppily
+slowly
+sluggishly
+slyly
+smartly
+smilingly
+smolderingly
+smoothly
+snobbishly
+snugly
+so
+sobbingly
+soberly
+socially
+sociologically
+soddenly
+soe
+sofar
+softly
+solely
+solemnly
+solidly
+some
+somebody
+someday
+somehow
+someone
+someplace
+somethin
+something
+sometime
+sometimes
+somewhat
+somewhere
+somewheres
+sompin
+sonuvabitch
+soon
+sooner
+soothingly
+sorely
+soulfully
+soundly
+sourly
+sparsely
+spatially
+specially
+specifically
+spectacularly
+spectrally
+speculatively
+speedily
+spiritually
+splendidly
+spontaneously
+squarely
+sssshoo
+staggeringly
+starkly
+startlingly
+statistically
+steadfastly
+steadily
+stealthily
+steamily
+steeply
+sternly
+stiffly
+stirringly
+stolidly
+stonily
+stoutly
+straightway
+strangely
+strategically
+strenuously
+strictly
+strikingly
+stringently
+strongly
+structurally
+stubbornly
+studiously
+stunningly
+stupidly
+subconsciously
+subjectively
+subsequently
+substantially
+substantively
+subtly
+successfully
+successively
+succinctly
+suddenly
+sufficiently
+suitably
+sulkily
+sullenly
+superbly
+superficially
+supinely
+supposedly
+supposing
+supra
+supremely
+surely
+surprisingly
+surreptitiously
+suspiciously
+sweepingly
+sweetly
+swiftly
+sycophantically
+symbolically
+symmetrically
+sympathetically
+synce
+syntactically
+systematically
+ta
+tacitly
+tactically
+tactually
+tangibly
+tantalizingly
+tarantara
+tardily
+tartly
+tauntingly
+tearfully
+technically
+technologically
+tediously
+telepathically
+temperately
+temporally
+temporarily
+temptingly
+ten
+tenaciously
+tenderly
+tenfold
+tensely
+tentatively
+tenth
+tenuously
+terribly
+tersely
+testily
+than
+thar
+that
+thatt
+thay
+the
+theare
+thease
+theatrically
+thee
+thees
+thei
+their
+theirs
+them
+themselves
+thence
+thenceforth
+theoretically
+ther
+there
+thereabouts
+thereafter
+thereby
+therefor
+therefore
+therefrom
+therein
+thereof
+thereon
+thereto
+theretofore
+thereunder
+thereupon
+therewith
+thermally
+thermodynamically
+these
+thet
+they
+thickly
+thine
+thinly
+third
+thirdly
+thirteen
+thirteenth
+thirtieth
+thirty
+this
+thither
+thoroughly
+those
+thou
+though
+thoughtfully
+thoughtlessly
+thousand
+thousandth
+threateningly
+three
+thrice
+through
+throughout
+thru
+thum
+thus
+thy
+tighter
+tightly
+till
+timidly
+tiredly
+tirelessly
+to
+together
+tonally
+too
+totally
+toward
+towardes
+towards
+traditionally
+tragically
+transversally
+transversely
+tremendously
+tremulously
+trillion
+triumphantly
+truly
+trustfully
+trustingly
+truthfully
+tunelessly
+twelfth
+twelve
+twentieth
+twenty
+twice
+twirlingly
+twise
+two
+typically
+u
+ugh
+uh
+uhhu
+ultimately
+ultracentrifugally
+ultrasonically
+um
+umm
+unaccountably
+unambiguously
+unanimously
+unappeasably
+unashamedly
+unavoidably
+unbearably
+unbelievably
+unblinkingly
+unceasingly
+uncertainly
+uncomfortably
+uncommonly
+uncomplainingly
+unconcernedly
+unconditionally
+unconsciously
+uncritically
+undeniably
+underfoot
+underneath
+understandably
+understandingly
+underway
+undoubtedly
+unduly
+uneasily
+unequally
+unequivocally
+unerringly
+unexpectedly
+unfailingly
+unfairly
+unfalteringly
+unfortunately
+unfunnily
+unhappily
+unhesitatingly
+unhurriedly
+uniformly
+unilaterally
+unimpeachably
+unintentionally
+uninterruptedly
+uniquely
+universally
+unknowingly
+unless
+unlike
+unluckily
+unmanageably
+unmistakably
+unnaturally
+unnecessarily
+unnnt
+unobtrusively
+unofficially
+unpleasantly
+unpredictably
+unqualifiedly
+unquestionably
+unquestioningly
+unrealistically
+unreasonably
+unreassuringly
+unreservedly
+unrestrictedly
+unselfishly
+unsmilingly
+unsteadily
+unsuccessfully
+unsuitably
+until
+unto
+unusually
+unutterably
+unwaveringly
+unwillingly
+unwisely
+unwittingly
+up
+upon
+uppermost
+uppon
+uproariously
+upside
+upstate
+upwards
+urgently
+us
+usefully
+uselessly
+usually
+utterly
+vaguely
+vainly
+valiantly
+validly
+variously
+vastly
+vehemently
+verbally
+verbatim
+verie
+versa
+versus
+vertically
+very
+via
+victoriously
+vigorously
+violently
+virtually
+visibly
+visually
+vitally
+vividly
+vocally
+vocationally
+vociferously
+volumetrically
+voluntarily
+voraciously
+warily
+warmly
+warningly
+was
+waspishly
+we
+weakly
+wearily
+wee
+weirdly
+were
+westward
+westwards
+wetly
+whad
+what
+whatever
+whatsoever
+wheare
+when
+whence
+whenever
+wher
+where
+whereas
+whereby
+whereever
+wherefore
+wherein
+whereon
+whereupon
+wherever
+wherewith
+whether
+which
+whichever
+whitely
+who
+whoe
+whoever
+wholeheartedly
+wholly
+whom
+whoosh
+whose
+whosoever
+why
+wickedly
+widely
+wil
+wildly
+wilfully
+will
+willfully
+willingly
+willy
+wilt
+wisely
+wistfully
+with
+withal
+within
+without
+wittingly
+wod
+woefully
+wold
+wolde
+wolfishly
+wonderfully
+wonderingly
+wondrously
+wordlessly
+worriedly
+would
+wow
+wrongly
+wryly
+wuh
+ya
+ye
+yea
+yeah
+yearningly
+yehhh
+yes
+yesiree
+yet
+yff
+yon
+yonder
+yooee
+you
+your
+yours
+yourself
+yourselves
+yow
+ys
+yuh
+zealously
+zero
+zoooop
+zounds
Index: /main/tags/3.01/gsdl/etc/packages/phind/stopword/fr/JASIS.sw
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/phind/stopword/fr/JASIS.sw	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/phind/stopword/fr/JASIS.sw	(revision 21112)
@@ -0,0 +1,215 @@
+a
+afin
+ai
+ainsi
+aprÃšs
+attendu
+au
+aujourd
+auquel
+aussi
+autre
+autres
+aux
+auxquelles
+auxquels
+avait
+avant
+avec
+avoir
+c
+car
+ce
+ceci
+cela
+celle
+celles
+celui
+cependant
+certain
+certaine
+certaines
+certains
+ces
+cet
+cette
+ceux
+chez
+ci
+combien
+comme
+comment
+concernant
+contre
+d
+dans
+de
+debout
+dedans
+dehors
+delÃ 
+depuis
+derriÃšre
+des
+desquelles
+desquels
+dessous
+dessus
+devant
+devers
+devra
+divers
+diverse
+diverses
+doit
+donc
+dont
+du
+duquel
+durant
+dÃšs
+elle
+elles
+en
+entre
+environ
+est
+et
+etc
+etre
+eux
+exceptÃ©
+hormis
+hors
+hui
+hÃ©las
+il
+ils
+j
+je
+jusqu
+jusque
+l
+la
+laquelle
+le
+lequel
+les
+lesquelles
+lesquels
+leur
+leurs
+lorsque
+lui
+lÃ 
+ma
+mais
+malgrÃ©
+me
+merci
+mes
+mien
+mienne
+miennes
+miens
+moi
+moins
+mon
+moyennant
+mÃªme
+mÃªmes
+n
+ne
+ni
+non
+nos
+notre
+nous
+nÃ©anmoins
+nÃŽtre
+nÃŽtres
+on
+ou
+out
+outre
+oÃ¹
+par
+parmi
+partant
+pas
+passÃ©
+pendant
+plein
+plus
+plusieurs
+pour
+pourquoi
+proche
+prÃšs
+puisque
+qu
+quand
+que
+quel
+quelle
+quelles
+quels
+qui
+quoi
+quoique
+revoici
+revoilÃ 
+s
+sa
+sans
+sauf
+se
+selon
+seront
+ses
+si
+sien
+sienne
+siennes
+siens
+sinon
+soi
+soit
+son
+sont
+sous
+suivant
+sur
+ta
+te
+tes
+tien
+tienne
+tiennes
+tiens
+toi
+ton
+tous
+tout
+toute
+toutes
+tu
+un
+une
+va
+vers
+voici
+voilÃ 
+vos
+votre
+vous
+vu
+vÃŽtre
+vÃŽtres
+y
+Ã 
+Ã§a
+Ãšs
+Ã©tÃ©
+Ãªtre
+ÃŽ
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.gen
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.gen	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.gen	(revision 21112)
@@ -0,0 +1,215 @@
+# RTF ANSI character set (\ansi) general map
+# These are taken from the ISO-Latin-1 (ISO-8859-1) encodings, with
+# a few additions
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+# jrm21 - ie, given the input in field 2, what name do we use for it?
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+equal		=
+greater		>
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII characters (128-255)
+null		0x80
+
+nobrkspace	0xa0
+exclamdown	0xa1
+cent		0xa2
+sterling	0xa3
+currency	0xa4
+yen		0xa5
+brokenbar	0xa6
+section		0xa7
+dieresis	0xa8
+copyright	0xa9
+ordfeminine	0xaa
+guillemotleft	0xab
+logicalnot	0xac
+opthyphen	0xad
+registered	0xae
+macron		0xaf
+degree		0xb0
+plusminus	0xb1
+twosuperior	0xb2
+threesuperior	0xb3
+acute		0xb4
+mu		0xb5
+paragraph	0xb6
+periodcentered	0xb7
+cedilla		0xb8
+onesuperior	0xb9
+ordmasculine	0xba
+guillemotright	0xbb
+onequarter	0xbc
+onehalf		0xbd
+threequarters	0xbe
+questiondown	0xbf
+Agrave		0xc0
+Aacute		0xc1
+Acircumflex	0xc2
+Atilde		0xc3
+Adieresis	0xc4
+Aring		0xc5
+AE		0xc6
+Ccedilla	0xc7
+Egrave		0xc8
+Eacute		0xc9
+Ecircumflex	0xca
+Edieresis	0xcb
+Igrave		0xcc
+Iacute		0xcd
+Icircumflex	0xce
+Idieresis	0xcf
+Eth		0xd0
+Ntilde		0xd1
+Ograve		0xd2
+Oacute		0xd3
+Ocircumflex	0xd4
+Otilde		0xd5
+Odieresis	0xd6
+multiply	0xd7
+Oslash		0xd8
+Ugrave		0xd9
+Uacute		0xda
+Ucircumflex	0xdb
+Udieresis	0xdc
+Yacute		0xdd
+Thorn		0xde
+germandbls	0xdf
+agrave		0xe0
+aacute		0xe1
+acircumflex	0xe2
+atilde		0xe3
+adieresis	0xe4
+aring		0xe5
+ae		0xe6
+ccedilla	0xe7
+egrave		0xe8
+eacute		0xe9
+ecircumflex	0xea
+edieresis	0xeb
+igrave		0xec
+iacute		0xed
+icircumflex	0xee
+idieresis	0xef
+eth		0xf0
+ntilde		0xf1
+ograve		0xf2
+oacute		0xf3
+ocircumflex	0xf4
+otilde		0xf5
+odieresis	0xf6
+divide		0xf7
+oslash		0xf8
+ugrave		0xf9
+uacute		0xfa
+ucircumflex	0xfb
+udieresis	0xfc
+yacute		0xfd
+thorn		0xfe
+ydieresis	0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.sym
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.sym	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/ansi.sym	(revision 21112)
@@ -0,0 +1,114 @@
+# RTF ANSI character set (\ansi) Symbol font map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+universal	'"'
+mathnumbersign	#
+existential	$
+percent		%
+ampersand	&
+suchthat	"'"
+parenleft	(
+parenright	)
+mathasterisk	*
+mathplus	+
+comma		,
+mathminus	-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+mathequal	=
+greater		>
+question	?
+congruent	@
+Alpha		A
+Beta		B
+Chi		C
+Delta		D
+Epsilon		E
+Phi		F
+Gamma		G
+Eta		H
+Iota		I
+# J		J
+Kappa		K
+Lambda		L
+Mu		M
+Nu		N
+Omicron		O
+Pi		P
+Theta		Q
+Rho		R
+Sigma		S
+Tau		T
+Upsilon		U
+varsigma	V
+Omega		W
+Xi		X
+Psi		Y
+Zeta		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+alpha		a
+beta		b
+chi		c
+delta		d
+epsilon		e
+phi		f
+gamma		g
+eta		h
+iota		i
+# j		j
+kappa		k
+lambda		l
+mu		m
+nu		n
+omicron		o
+pi		p
+theta		q
+rho		r
+sigma		s
+tau		t
+upsilon		u
+# xxx		v
+omega		w
+xi		x
+psi		y
+zeta		z
+braceleft	{
+bar		|
+braceright	}
+mathtilde	~
+
+# non-ASCII characters (128-255)
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.gen
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.gen	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.gen	(revision 21112)
@@ -0,0 +1,244 @@
+# RTF Windows Codepage-1252 character set (\ansi) general map
+# Note (jrm21) - this is basically a modified version of the ANSI map.
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# printable characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "	# 0x20
+exclam		!	# 0x21
+quotedbl	'"'	# 0x22
+numbersign	#	# 0x23
+dollar		$	# 0x24
+percent		%	# 0x25
+ampersand	&	# 0x26
+quoteright	"'"	# 0x27
+parenleft	(	# 0x28
+parenright	)	# 0x29
+asterisk	*	# 0x2a
+plus		+	# 0x2b
+comma		,	# 0x2c
+hyphen		-	# 0x2d
+period		.	# 0x2e
+slash		/	# 0x2f
+zero		0	# 0x30
+one		1	# 0x31
+two		2	# 0x32
+three		3	# 0x33
+four		4	# 0x34
+five		5	# 0x35
+six		6	# 0x36
+seven		7	# 0x37
+eight		8	# 0x38
+nine		9	# 0x39
+colon		:	# 0x3a
+semicolon	;	# 0x3b
+less		<	# 0x3c
+equal		=	# 0x3d
+greater		>	# 0x3e
+question	?	# 0x3f
+at		@	# 0x40
+A		A	# 0x41
+B		B	# 0x42
+C		C	# 0x43
+D		D	# 0x44
+E		E	# 0x45
+F		F	# 0x46
+G		G	# 0x47
+H		H	# 0x48
+I		I	# 0x49
+J		J	# 0x4a
+K		K	# 0x4b
+L		L	# 0x4c
+M		M	# 0x4d
+N		N	# 0x4e
+O		O	# 0x4f
+P		P	# 0x50
+Q		Q	# 0x51
+R		R	# 0x52
+S		S	# 0x53
+T		T	# 0x54
+U		U	# 0x55
+V		V	# 0x56
+W		W	# 0x57
+X		X	# 0x58
+Y		Y	# 0x59
+Z		Z	# 0x5a
+bracketleft	[	# 0x5b
+backslash	\	# 0x5c
+bracketright	]	# 0x5d
+asciicircum	^	# 0x5e
+underscore	_	# 0x5f
+quoteleft	"`"	# 0x60
+a		a	# 0x61
+b		b	# 0x62
+c		c	# 0x63
+d		d	# 0x64
+e		e	# 0x65
+f		f	# 0x66
+g		g	# 0x67
+h		h	# 0x68
+i		i	# 0x69
+j		j	# 0x6a
+k		k	# 0x6b
+l		l	# 0x6c
+m		m	# 0x6d
+n		n	# 0x6e
+o		o	# 0x6f
+p		p	# 0x70
+q		q	# 0x71
+r		r	# 0x72
+s		s	# 0x73
+t		t	# 0x74
+u		u	# 0x75
+v		v	# 0x76
+w		w	# 0x77
+x		x	# 0x78
+y		y	# 0x79
+z		z	# 0x7a
+braceleft	{	# 0x7b
+bar		|	# 0x7c
+braceright	}	# 0x7d
+asciitilde	~	# 0x7e
+# undefined (delete)	# 0x7f
+
+# non-ASCII characters (128-255)
+euro		0x80
+# undefined	0x81
+quotesinglbase	0x82 # single low-9 quotation mark
+florin		0x83 # f with hook (fnof) ## not sure if florin is right...
+quotedblbase	0x84 # double low-9 quotation mark
+ellipsis	0x85
+dagger		0x86
+daggerdbl	0x87
+circumflex	0x88 # modifier
+perthousand	0x89
+Scaron		0x8a
+guilsinglleft	0x8b
+OE		0x8c # OE ligature
+# undefined	0x8d
+Zcaron		0x8e
+# undefined	0x8f
+# undefined	0x90
+quoteleft	0x91 # single quote left - use ascii for now
+quoteright	0x92 # single quote right - use ascii for now
+quotedblleft	0x93
+quotedblright	0x94
+bullet		0x95
+endash		0x96
+emdash		0x97
+asciitilde	0x98 # use ascii tilde
+trademark	0x99
+scaron		0x9a
+guilsinglright	0x9b # single rightpointing quotemark
+oe		0x9c
+# undefined	0x9d
+zcaron		0x9e
+Ydieresis	0x9f # Y with diaeresis
+nobrkspace	0xa0
+exclamdown	0xa1
+cent		0xa2
+sterling	0xa3
+currency	0xa4
+yen		0xa5
+brokenbar	0xa6
+section		0xa7
+dieresis	0xa8
+copyright	0xa9
+ordfeminine	0xaa
+guillemotleft	0xab
+logicalnot	0xac
+opthyphen	0xad
+registered	0xae
+macron		0xaf
+degree		0xb0
+plusminus	0xb1
+twosuperior	0xb2
+threesuperior	0xb3
+acute		0xb4
+mu		0xb5
+paragraph	0xb6
+periodcentered	0xb7
+cedilla		0xb8
+onesuperior	0xb9
+ordmasculine	0xba
+guillemotright	0xbb
+onequarter	0xbc
+onehalf		0xbd
+threequarters	0xbe
+questiondown	0xbf
+Agrave		0xc0
+Aacute		0xc1
+Acircumflex	0xc2
+Atilde		0xc3
+Adieresis	0xc4
+Aring		0xc5
+AE		0xc6
+Ccedilla	0xc7
+Egrave		0xc8
+Eacute		0xc9
+Ecircumflex	0xca
+Edieresis	0xcb
+Igrave		0xcc
+Iacute		0xcd
+Icircumflex	0xce
+Idieresis	0xcf
+Eth		0xd0
+Ntilde		0xd1
+Ograve		0xd2
+Oacute		0xd3
+Ocircumflex	0xd4
+Otilde		0xd5
+Odieresis	0xd6
+multiply	0xd7
+Oslash		0xd8
+Ugrave		0xd9
+Uacute		0xda
+Ucircumflex	0xdb
+Udieresis	0xdc
+Yacute		0xdd
+Thorn		0xde
+germandbls	0xdf
+agrave		0xe0
+aacute		0xe1
+acircumflex	0xe2
+atilde		0xe3
+adieresis	0xe4
+aring		0xe5
+ae		0xe6
+ccedilla	0xe7
+egrave		0xe8
+eacute		0xe9
+ecircumflex	0xea
+edieresis	0xeb
+igrave		0xec
+iacute		0xed
+icircumflex	0xee
+idieresis	0xef
+eth		0xf0
+ntilde		0xf1
+ograve		0xf2
+oacute		0xf3
+ocircumflex	0xf4
+otilde		0xf5
+odieresis	0xf6
+divide		0xf7
+oslash		0xf8
+ugrave		0xf9
+uacute		0xfa
+ucircumflex	0xfb
+udieresis	0xfc
+yacute		0xfd
+thorn		0xfe
+ydieresis	0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.sym
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.sym	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/cpg1252.sym	(revision 21112)
@@ -0,0 +1,221 @@
+# RTF ANSI character set (\ansi) general map
+# These are taken from the ISO-Latin-1 (ISO-8859-1) encodings, with
+# a few additions
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# jrm21 - the character name is used when outputting a particular character
+# code. This name is then looked up in the *-gen file to see what to output
+# for that symbol name. It's a bit round-about...
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+equal		=
+greater		>
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII characters (128-255)
+#ellipsis		0x85
+#dagger		0x86
+#daggerdbl	0x87
+
+nobrkspace	0xa0
+exclamdown	0xa1
+cent		0xa2
+sterling	0xa3
+currency	0xa4
+yen		0xa5
+brokenbar	0xa6
+section		0xa7
+dieresis	0xa8
+copyright	0xa9
+ordfeminine	0xaa
+guillemotleft	0xab
+logicalnot	0xac
+opthyphen	0xad
+registered	0xae
+macron		0xaf
+degree		0xb0
+plusminus	0xb1
+twosuperior	0xb2
+threesuperior	0xb3
+acute		0xb4
+mu		0xb5
+paragraph	0xb6
+periodcentered	0xb7
+#asterisk	0xb7
+cedilla		0xb8
+onesuperior	0xb9
+ordmasculine	0xba
+guillemotright	0xbb
+onequarter	0xbc
+onehalf		0xbd
+threequarters	0xbe
+questiondown	0xbf
+Agrave		0xc0
+Aacute		0xc1
+Acircumflex	0xc2
+Atilde		0xc3
+Adieresis	0xc4
+Aring		0xc5
+AE		0xc6
+Ccedilla	0xc7
+Egrave		0xc8
+Eacute		0xc9
+Ecircumflex	0xca
+Edieresis	0xcb
+Igrave		0xcc
+Iacute		0xcd
+Icircumflex	0xce
+Idieresis	0xcf
+Eth		0xd0
+Ntilde		0xd1
+Ograve		0xd2
+Oacute		0xd3
+Ocircumflex	0xd4
+Otilde		0xd5
+Odieresis	0xd6
+multiply	0xd7
+Oslash		0xd8
+Ugrave		0xd9
+Uacute		0xda
+Ucircumflex	0xdb
+Udieresis	0xdc
+Yacute		0xdd
+Thorn		0xde
+germandbls	0xdf
+agrave		0xe0
+aacute		0xe1
+acircumflex	0xe2
+atilde		0xe3
+adieresis	0xe4
+aring		0xe5
+ae		0xe6
+ccedilla	0xe7
+egrave		0xe8
+eacute		0xe9
+ecircumflex	0xea
+edieresis	0xeb
+igrave		0xec
+iacute		0xed
+icircumflex	0xee
+idieresis	0xef
+eth		0xf0
+ntilde		0xf1
+ograve		0xf2
+oacute		0xf3
+ocircumflex	0xf4
+otilde		0xf5
+odieresis	0xf6
+divide		0xf7
+oslash		0xf8
+ugrave		0xf9
+uacute		0xfa
+ucircumflex	0xfb
+udieresis	0xfc
+yacute		0xfd
+thorn		0xfe
+ydieresis	0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-map
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-map	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-map	(revision 21112)
@@ -0,0 +1,265 @@
+# Output sequence map for rtftohtml
+
+# Field 1 is the standard character name.  Field 2 is the output sequence
+# to produce for that character.
+
+# The output sequence is simply a string of characters.  If it contains
+# whitespace, it may be quoted.  If it contains quotes, it may be quoted
+# with a different quote character.
+
+null		''
+# characters in ASCII range (32-127)
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&amp;
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+onesuperior	&sup1;
+two		2
+twosuperior	&sup2;
+three		3
+threesuperior	&sup3;
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		&lt;
+equal		=
+greater		&gt;
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII
+# use HTML 4.0 entity names where practical
+# starting at decimal code 160
+nobrkspace	&nbsp;
+# inverted exclamation mark 161
+cent		&cent;
+sterling	&pound;
+currency	&curren;
+yen		&yen;
+#netscape will only do this in utf-8: brokenbar	&brvbar;
+brokenbar	|
+section		&sect;
+# uml / diaeresis = 168
+copyright	&copy;
+copyrightsans	&copy;
+
+
+# Latin Extended-A
+OE		&OElig;
+oe		&oelig;
+Scaron		&Scaron;
+scaron		&scaron;
+Ydieresis	&Yuml;
+
+acute		"'"
+angleleft	[
+angleright	&gt;
+approxequal	~
+arrowboth	&lt;-&gt;
+arrowdblboth	&lt;=&gt;
+arrowdblleft	&lt;=
+arrowdblright	=&gt;
+arrowleft	&lt;-
+arrowright	-&gt;
+#bullet		*
+bullet		&middot;
+circumflex	^
+# next 2 have named entities, which netscape 4 doesn't use...
+# unicode &#8224; and &#8225;
+dagger		&#134;
+daggerdbl	&#135;
+
+degree		&deg;
+delta		d
+divide		&divide;
+dotlessi	i
+ellipsis	...
+# next 2 have named entities, which netscape 4 doesn't use...
+endash		&#150;
+emdash		&#151;
+euro		&euro;
+fi		fi
+fl		fl
+fraction	/
+grave		"`"
+greaterequal	&gt;=
+# &#171; # was &lt;&lt;
+guillemotleft	&laquo;
+# &#187; # was &gt;&gt;
+guillemotright	&raquo;
+# netscape 4.7 doesn't do this? - unicode: &#2039; or &lsaquo;
+guilsinglleft	&lt;
+# ditto? unicode: &#203A; or &rsaquo;
+guilsinglright	&gt;
+lessequal	&lt;=
+logicalnot	&not;
+mathasterisk	*
+mathequal	=
+mathminus	-
+mathnumbersign	'#'
+mathplus	+
+mathtilde	~
+minus		-
+mu		&micro;
+multiply	&times;
+nobrkhyphen	-
+
+notequal	!=
+onehalf		&frac12;
+onequarter	&frac14;
+periodcentered	&middot;
+plusminus	&plusmn;
+quotedblbase	,,
+quotedblleft	'"'
+quotedblright	'"'
+quotesinglbase	,
+registered	&reg;
+registersans	&reg;
+threequarters	&frac34;
+tilde		~
+trademark	&trade;
+AE		&AElig;
+Aacute      &Aacute;
+Acircumflex &Acirc;
+Agrave      &Agrave;
+Aring       &Aring;
+Atilde		&Atilde;
+Adieresis	&Auml;
+Ccedilla	&Ccedil;
+#ETH		&ETH;
+Eacute	&Eacute;
+Ecircumflex	&Ecirc;
+Egrave	&Egrave;
+Edieresis	&Euml;
+Iacute	&Iacute;
+Icircumflex	&Icirc;
+Igrave	&Igrave;
+Idieresis	&Iuml;
+Ntilde	&Ntilde;
+Oacute	&Oacute;
+Ocircumflex	&Ocirc;
+Ograve	&Ograve;
+Oslash	&Oslash;
+#Otilde	&Otilde;
+Odieresis	&Ouml;
+#THORN	&THORN;
+Uacute	&Uacute;
+Ucircumflex	&Ucirc;
+Ugrave	&Ugrave;
+Udieresis	&Uuml;
+#Yacute	&Yacute;
+ae		&aelig;
+aacute	&aacute;
+acircumflex	&acirc;
+agrave	&agrave;
+aring	&aring;
+atilde	&atilde;
+adieresis	&auml;
+ccedilla	&ccedil;
+eacute	&eacute;
+ecircumflex	&ecirc;
+egrave	&egrave;
+#eth	&eth;
+edieresis	&euml;
+iacute	&iacute;
+icircumflex	&icirc;
+igrave	&igrave;
+idieresis	&iuml;
+ntilde	&ntilde;
+oacute	&oacute;
+ocircumflex	&ocirc;
+ograve	&ograve;
+oslash	&oslash;
+otilde	&otilde;
+odieresis	&ouml;
+germandbls	&szlig;
+#thorn	&thorn;
+uacute	&uacute;
+ucircumflex	&ucirc;
+ugrave	&ugrave;
+udieresis	&uuml;
+yacute	&yacute;
+ydieresis	&yuml;
+newline	<br>
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-trn
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-trn	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/html-trn	(revision 21112)
@@ -0,0 +1,214 @@
+# Starting with rtftohtml 2.7, all html-trans files have a version number
+# The version number must appear before any other tables.
+.Version,3
+
+# Paragraph Style Tags table
+# Format:
+#"name","starttag","endtag","col2mark","tabmark","parmark",allowtext,cannest,DeleteCol1,fold,TocStyl
+.PTag
+"Normal","","\n","\t","\t","<p>\n",1,0,0,1,0
+"h1","<h1>\n","</h1>\n","\t","\t","<br>\n",0,0,0,1,1
+"h2","<h2>\n","</h2>\n","\t","\t","<br>\n",0,0,0,1,2
+"h3","<h3>\n","</h3>\n","\t","\t","<br>\n",0,0,0,1,3
+"h4","<h4>\n","</h4>\n","\t","\t","<br>\n",0,0,0,1,4
+"h5","<h5>\n","</h5>\n","\t","\t","<br>\n",0,0,0,1,5
+"h6","<h6>\n","</h6>\n","\t","\t","<br>\n",0,0,0,1,6
+"ul","<ul>\n<li>","</ul>","\t","\t","\n<li>",1,1,0,1,0
+"ul-d","<ul>\n<li>","</ul>","\t","\t","\n<li>",1,1,1,1,0
+"ol","<ol>\n<li>","</ol>","\t","\t","\n<li>",1,1,0,1,0
+"ol-d","<ol>\n<li>","</ol>","\t","\t","\n<li>",1,1,1,1,0
+"dl","<dl>\n<dt>","</dl>","\n<dd>","\t","\n<dt>",1,1,0,1,0
+"TERM","<dl>\n<dt>","</dl>","\t","\t","\n<dt>",1,1,0,1,0
+"DEF","<dd>","","\t","\t","\n<br>",1,1,0,1,0
+"dir","<dir>\n<li>","</dir>","\n<li>","\n<li>","\n<li>",0,1,0,1,0
+"menu","<menu>\n<li>","</menu>\n","\n<li>","\n<li>","\n<li>",0,1,0,1,0
+"address","<address>","</address>\n","\t","\t","<br>\n",1,0,0,1,0
+"pre","\n<pre>\n","\n</pre>\n","\t","\t","\n",0,0,0,0,0
+"blockquote","<blockquote>","</blockquote>","\t","\t","\n",1,0,0,1,0
+"hr","<hr>","","","","",1,0,0,1,0
+# This is a required entry; tables will be formatted with this entry
+"_Table","\n<pre>\n","\n</pre>\n","\t","\t","\n",0,0,0,0,0
+
+# Text STyle Tags table
+# Format
+# "name","Starttag","Endtag"
+.TTag
+"b","<b>","</b>"
+"i","<i>","</i>"
+"u","<u>","</u>"
+"em","<em>","</em>"
+"cite","<cite>","</cite>"
+"tt","<tt>","</tt>"
+"strong","<strong>","</strong>"
+"code","<code>","</code>"
+"samp","<samp>","</samp>"
+"kbd","<kbd>","</kbd>"
+"var","<var>","</var>"
+"dfn","<dfn>","</dfn>"
+"sup","<sup>","</sup>"
+
+# Text Match Table
+# Format:
+# "Font",FontSize,Match,Mask,"TextStyleName"
+# "" is don't care for fonts
+# 0 is don't care for font size
+# Special names for TextStyleName
+# _Discard - text will not be output
+# _Name    - text will be used for an named anchor
+# _HRef    - text is an href
+# _Hot     - end of this text marks the end of hot text
+# _Literal - text is sent thorough without any translation
+# The order of bits for match and mask is:
+#    v^bDWUHACSOTIB - Bold
+#    v^bDWUHACSOTI - Italic
+#    v^bDWUHACSOT - StrikeThrough
+#    v^bDWUHACSO - Outline
+#    v^bDWUHACS - Shadow
+#    v^bDWUHAC - SmallCaps
+#    v^bDWUHA - AllCaps
+#    v^bDWUH - Hidden
+#    v^bDWU - Underline
+#    v^bDW - Word Underline
+#    v^bD - Dotted Underline
+#    v^b - Double Underline
+#    v^ - SuperScript
+#    v - SubScript
+#    v^bDWUIACSOTIB
+.TMatch
+#
+# First do specials, because they are higher priority than other matches
+# 
+# double-underline/not hidden -> hot text
+# double-underline/hidden -> href
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00100000000000,00100010000000,"_Hot"
+"",0,00100010000000,00100010000000,"_HRef"
+
+# hidden/shadow -> literal
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00000010010000,00000010010000,"_Literal"
+
+# hidden/outline -> Name
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00000010001000,00000010001000,"_Name"
+
+# hidden/all other hiddens are discarded
+"",0,00000010000000,00000010000000,"_Discard"
+
+# All other Superscripts just superscripts
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,01000000000000,01000000000000,"sup"
+
+# Regular matches - You can have multiple of these active
+
+# monospace fonts -> tt
+"Courier",0,00000000000000,00000000000000,"tt"
+"Monoco",0,00000000000000,00000000000000,"tt"
+
+# bold -> bold
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00000000000001,00000000000001,"b"
+
+# italic/underline -> cite, italic/nounderline -> italic
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00000000000010,00000100000010,"i"
+"",0,00000100000010,00000100000010,"cite"
+
+# any underline without italic -> underline
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00010000000000,00111100000010,"u"
+"",0,00001000000000,00111100000010,"u"
+"",0,00000100000000,00111100000010,"u"
+"",0,00100000000000,00111100000010,"u"
+
+# outline,shadow -> em
+#    v^bDWUHACSOTIB,v^bDWUHACSOTIB
+"",0,00000000001000,00000000001000,"em"
+"",0,00000000010000,00000000010000,"em"
+
+
+# Format:
+# "Paragraph Style",NestLevel,"ParagraphStyleName"
+# First Entry is the default if no style match. Should be level 0 
+# point to a Paragraph style with no markup
+.PMatch
+"Normal",0,"Normal"
+# This is a required entry; tables will be formatted with this entry
+"_Table",0,"_Table"
+"heading 1",0,"h1"
+"heading 2",0,"h2"
+"heading 3",0,"h3"
+"heading 4",0,"h4"
+"heading 5",0,"h5"
+"heading 6",0,"h6"
+"ol",0,"ol"
+"ol 1",1,"ol"
+"ol 2",2,"ol"
+"ol 3",3,"ol"
+"ol 4",4,"ol"
+"ol 5",5,"ol"
+"ul",0,"ul"
+"ul 1",1,"ul"
+"ul 2",2,"ul"
+"ul 3",3,"ul"
+"ul 4",4,"ul"
+"ul 5",5,"ul"
+"numbered list",0,"ol-d"
+"numbered list 1",1,"ol-d"
+"numbered list 2",2,"ol-d"
+"numbered list 3",3,"ol-d"
+"numbered list 4",4,"ol-d"
+"numbered list 5",5,"ol-d"
+"bullet list",0,"ul-d"
+"bullet list 1",1,"ul-d"
+"bullet list 2",2,"ul-d"
+"bullet list 3",3,"ul-d"
+"bullet list 4",4,"ul-d"
+"bullet list 5",5,"ul-d"
+"dir",0,"dir"
+"dir 1",1,"dir"
+"dir 2",2,"dir"
+"dir 3",3,"dir"
+"dir 4",4,"dir"
+"dir 5",5,"dir"
+"small gloss",0,"dl"
+"small gloss 1",1,"dl"
+"small gloss 2",2,"dl"
+"small gloss 3",3,"dl"
+"small gloss 4",4,"dl"
+"small gloss 5",5,"dl"
+"glossary",0,"dl"
+"glossary 1",1,"dl"
+"glossary 2",2,"dl"
+"glossary 3",3,"dl"
+"glossary 4",4,"dl"
+"glossary 5",5,"dl"
+"menu",0,"menu"
+"menu 1",1,"menu"
+"menu 2",2,"menu"
+"menu 3",3,"menu"
+"menu 4",4,"menu"
+"menu 5",5,"menu"
+"pre",0,"pre"
+"blockquote",0,"blockquote"
+"address",0,"address"
+"hr",0,"hr"
+# the following paragraph styles will be discarded
+"toc 1",0,"_Discard"
+"toc 2",0,"_Discard"
+"toc 3",0,"_Discard"
+"toc 4",0,"_Discard"
+"toc 5",0,"_Discard"
+"index 1",0,"_Discard"
+"index 2",0,"_Discard"
+"index 3",0,"_Discard"
+"index 4",0,"_Discard"
+"index 5",0,"_Discard"
+"TERM",0,"TERM"
+"DEF",1,"DEF"
+"TERM 1",1,"TERM"
+"DEF 1",2,"DEF"
+"TERM 2",2,"TERM"
+"DEF 2",3,"DEF"
+"HTML",0,"_Literal"
+"footnote text",0,"Normal"
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.gen
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.gen	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.gen	(revision 21112)
@@ -0,0 +1,243 @@
+# RTF Macintosh character set (\mac) general map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+equal		=
+greater		>
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII characters (128-255)
+
+# ?? Ygrave		0x07
+# ?? yacute		0x08
+Adieresis	0x80
+Aring		0x81
+Ccedilla	0x82
+Eacute		0x83
+Ntilde		0x84
+Odieresis	0x85
+Udieresis	0x86
+aacute		0x87
+agrave		0x88
+acircumflex	0x89
+adieresis	0x8a
+atilde		0x8b
+aring		0x8c
+ccedilla	0x8d
+eacute		0x8e
+egrave		0x8f
+ecircumflex	0x90
+edieresis	0x91
+iacute		0x92
+igrave		0x93
+icircumflex	0x94
+idieresis	0x95
+ntilde		0x96
+oacute		0x97
+ograve		0x98
+ocircumflex	0x99
+odieresis	0x9a
+otilde		0x9b
+uacute		0x9c
+ugrave		0x9d
+ucircumflex	0x9e
+udieresis	0x9f
+dagger		0xa0
+degree		0xa1
+cent		0xa2
+sterling	0xa3
+section		0xa4
+bullet		0xa5
+paragraph	0xa6
+germandbls	0xa7
+registered	0xa8
+copyright	0xa9
+trademark	0xaa
+acute		0xab
+dieresis	0xac
+notequal	0xad
+AE		0xae
+Oslash		0xaf
+infinity	0xb0
+plusminus	0xb1
+lessequal	0xb2
+greaterequal	0xb3
+yen		0xb4
+mu		0xb5
+partialdiff	0xb6
+Sigma		0xb7
+Pi		0xb8
+pi		0xb9
+integral	0xba
+ordfeminine	0xbb
+ordmasculine	0xbc
+Omega		0xbd
+ae		0xbe
+oslash		0xbf
+questiondown	0xc0
+exclamdown	0xc1
+logicalnot	0xc2
+radical		0xc3
+florin		0xc4
+approxequal	0xc5
+Delta		0xc6
+guillemotleft	0xc7
+guillemotright	0xc8
+ellipsis	0xc9
+nobrkspace	0xca
+Agrave		0xcb
+OE		0xce
+oe		0xcf
+endash		0xd0
+emdash		0xd1
+quotedblleft	0xd2
+quotedblright	0xd3
+quoteleft	0xd4
+quoteright	0xd5
+divide		0xd6
+lozenge		0xd7
+ydieresis	0xd8
+Ydieresis	0xd9
+fraction	0xda
+currency	0xdb
+guilsinglleft	0xdc
+guilsinglright	0xdd
+fi		0xde
+fl		0xdf
+daggerdbl	0xe0
+periodcentered	0xe1
+quotesinglbase	0xe2
+quotedblbase	0xe3
+perthousand	0xe4
+Acircumflex	0xe5
+Ecircumflex	0xe6
+Aacute		0xe7
+Edieresis	0xe8
+Egrave		0xe9
+Iacute		0xea
+Icircumflex	0xeb
+Idieresis	0xec
+Igrave		0xed
+Oacute		0xee
+Ocircumflex	0xef
+apple		0xf0
+Ograve		0xf1
+Uacute		0xf2
+Ucircumflex	0xf3
+Ugrave		0xf4
+dotlessi	0xf5
+circumflex	0xf6
+tilde		0xf7
+macron		0xf8
+breve		0xf9
+dotaccent	0xfa
+ring		0xfb
+cedilla		0xfc
+hungarumlaut	0xfd
+ogonek		0xfe
+caron		0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.sym
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.sym	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/mac.sym	(revision 21112)
@@ -0,0 +1,241 @@
+# RTF Macintosh character set (\mac) Symbol font map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+universal	'"'
+mathnumbersign	#
+existential	$
+percent		%
+ampersand	&
+suchthat	"'"
+parenleft	(
+parenright	)
+mathasterisk	*
+mathplus	+
+comma		,
+mathminus	-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+mathequal	=
+greater		>
+question	?
+congruent	@
+Alpha		A
+Beta		B
+Chi		C
+Delta		D
+Epsilon		E
+Phi		F
+Gamma		G
+Eta		H
+Iota		I
+# J		J
+Kappa		K
+Lambda		L
+Mu		M
+Nu		N
+Omicron		O
+Pi		P
+Theta		Q
+Rho		R
+Sigma		S
+Tau		T
+Upsilon		U
+varsigma	V
+Omega		W
+Xi		X
+Psi		Y
+Zeta		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+alpha		a
+beta		b
+chi		c
+delta		d
+epsilon		e
+phi		f
+gamma		g
+eta		h
+iota		i
+# j		j
+kappa		k
+lambda		l
+mu		m
+nu		n
+omicron		o
+pi		p
+theta		q
+rho		r
+sigma		s
+tau		t
+upsilon		u
+# xxx		v
+omega		w
+xi		x
+psi		y
+zeta		z
+braceleft	{
+bar		|
+braceright	}
+mathtilde	~
+
+# non-ASCII characters (128-255)
+
+# Adieresis	0x80
+# Aring		0x81
+# Ccedilla	0x82
+# Eacute	0x83
+# Ntilde	0x84
+# Odieresis	0x85
+# Udieresis	0x86
+# aacute	0x87
+# agrave	0x88
+# acircumflex	0x89
+# adieresis	0x8a
+# atilde	0x8b
+# aring		0x8c
+# ccedilla	0x8d
+# eacute	0x8e
+# egrave	0x8f
+# ecircumflex	0x90
+# edieresis	0x91
+# iacute	0x92
+# igrave	0x93
+# icircumflex	0x94
+# idieresis	0x95
+# ntilde	0x96
+# oacute	0x97
+# ograve	0x98
+# ocircumflex	0x99
+# odieresis	0x9a
+# otilde	0x9b
+# uacute	0x9c
+# ugrave	0x9d
+# ucircumflex	0x9e
+# udieresis	0x9f
+# dagger	0xa0
+# degree	0xa1
+minute		0xa2
+lessequal	0xa3
+fraction	0xa4
+infinity	0xa5
+florin		0xa6
+club		0xa7
+diamond		0xa8
+heart		0xa9
+spade		0xaa
+arrowboth	0xab
+arrowleft	0xac
+arrowup		0xad
+arrowright	0xae
+arrowdown	0xaf
+degree		0xb0
+plusminus	0xb1
+second		0xb2
+greaterequal	0xb3
+multiply	0xb4
+proportional	0xb5
+partialdiff	0xb6
+bullet		0xb7
+divide		0xb8
+notequal	0xb9
+equivalence	0xba
+approxequal	0xbb
+ellipsis	0xbc
+arrowvertex	0xbd
+arrowhorizex	0xbe
+carriagereturn	0xbf
+aleph		0xc0
+Ifraktur	0xc1
+Rfraktur	0xc2
+weierstrass	0xc3
+circlemultiply	0xc4
+circleplus	0xc5
+emptyset	0xc6
+intersection	0xc7
+union		0xc8
+propersuperset	0xc9
+reflexsuperset	0xca
+notsubset	0xcb
+propersubset	0xce
+reflexsubset	0xcf
+angle		0xd0
+gradient	0xd1
+registered	0xd2
+copyright	0xd3
+trademark	0xd4
+product		0xd5
+radical		0xd6
+periodcentered	0xd7
+logicalnot	0xd8
+logicaland	0xd9
+logicalor	0xda
+arrowdblboth	0xdb
+arrowdblleft	0xdc
+arrowdblup	0xdd
+arrowdblright	0xde
+arrowdbldown	0xdf
+lozenge		0xe0
+angleleft	0xe1
+registersans	0xe2
+copyrightsans	0xe3
+trademarksans	0xe4
+Sigma		0xe5
+# Ecircumflex	0xe6
+# Aacute	0xe7
+# Edieresis	0xe8
+# Egrave	0xe9
+# Iacute	0xea
+# Icircumflex	0xeb
+# Idieresis	0xec
+# Igrave	0xed
+# Oacute	0xee
+# Ocircumflex	0xef
+apple		0xf0
+angleright	0xf1
+# Uacute	0xf2
+# Ucircumflex	0xf3
+# Ugrave	0xf4
+# dotlessi	0xf5
+# circumflex	0xf6
+# tilde		0xf7
+# macron	0xf8
+# breve		0xf9
+# dotaccent	0xfa
+# ring		0xfb
+# cedilla	0xfc
+# hungarumlaut	0xfd
+# ogonek	0xfe
+# caron		0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.gen
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.gen	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.gen	(revision 21112)
@@ -0,0 +1,243 @@
+# RTF PC character set (\pc) general map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (32-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+equal		=
+greater		>
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII characters (128-255)
+
+Ccedilla	0x80
+udieresis	0x81
+eacute		0x82
+acircumflex	0x83
+adieresis	0x84
+agrave		0x85
+aring		0x86
+ccedilla	0x87
+ecircumflex	0x88
+edieresis	0x89
+egrave		0x8a
+idieresis	0x8b
+icircumflex	0x8c
+igrave		0x8d
+Adieresis	0x8e
+Aring		0x8f
+Eacute		0x90
+ae		0x91
+AE		0x92
+ocircumflex	0x93
+odieresis	0x94
+ograve		0x95
+ucircumflex	0x96
+ugrave		0x97
+ydieresis	0x98
+Odieresis	0x99
+Udieresis	0x9a
+cent		0x9b
+sterling	0x9c
+yen		0x9d
+ucircumflex	0x9e
+florin		0x9f
+aacute		0xa0
+iacute		0xa1
+oacute		0xa2
+uacute		0xa3
+ntilde		0xa4
+Ntilde		0xa5
+ordfeminine	0xa6
+ordmasculine	0xa7
+questiondown	0xa8
+copyright	0xa9
+trademark	0xaa
+acute		0xab
+dieresis	0xac
+exclamdown	0xad
+guillemotleft	0xae
+guillemotright	0xaf
+space		0xb0
+plusminus	0xb1
+lessequal	0xb2
+bar		0xb3
+yen		0xb4
+mu		0xb5
+partialdiff	0xb6
+Sigma		0xb7
+Pi		0xb8
+pi		0xb9
+integral	0xba
+ordfeminine	0xbb
+ordmasculine	0xbc
+Omega		0xbd
+ae		0xbe
+logicalnot	0xbf
+questiondown	0xc0
+exclamdown	0xc1
+logicalnot	0xc2
+radical		0xc3
+emdash		0xc4
+approxequal	0xc5
+delta		0xc6
+guillemotleft	0xc7
+guillemotright	0xc8
+ellipsis	0xc9
+nobrkspace	0xca
+Agrave		0xcb
+Atilde		0xcc
+Otilde		0xcd
+OE		0xce
+oe		0xcf
+endash		0xd0
+emdash		0xd1
+quotedblleft	0xd2
+quotedblright	0xd3
+quoteleft	0xd4
+quoteright	0xd5
+divide		0xd6
+lozenge		0xd7
+ydieresis	0xd8
+Ydieresis	0xd9
+fraction	0xda
+Ydieresis	0xdb
+guilsinglleft	0xdc
+greater		0xdd
+less		0xde
+fl		0xdf
+a		0xe0
+germandbls	0xe1
+quotesinglbase	0xe2
+pi		0xe3
+Sigma		0xe4
+Acircumflex	0xe5
+mu		0xe6
+Aacute		0xe7
+Edieresis	0xe8
+Egrave		0xe9
+Omega		0xea
+partialdiff	0xeb
+infinity	0xec
+oslash		0xed
+e		0xee
+Ocircumflex	0xef
+equal		0xf0
+plusminus	0xf1
+greaterequal	0xf2
+lessequal	0xf3
+integral	0xf4
+integral	0xf5
+divide		0xf6
+approxequal	0xf7
+degree		0xf8
+bullet		0xf9
+bullet		0xfa
+radical		0xfb
+cedilla		0xfc
+hungarumlaut	0xfd
+ogonek		0xfe
+nobrkspace	0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.sym
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.sym	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/pc.sym	(revision 21112)
@@ -0,0 +1,115 @@
+# RTF PC character set (\pc) Symbol font map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+universal	'"'
+mathnumbersign	#
+existential	$
+percent		%
+ampersand	&
+suchthat	"'"
+parenleft	(
+parenright	)
+mathasterisk	*
+mathplus	+
+comma		,
+mathminus	-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+mathequal	=
+greater		>
+question	?
+congruent	@
+Alpha		A
+Beta		B
+Chi		C
+Delta		D
+Epsilon		E
+Phi		F
+Gamma		G
+Eta		H
+Iota		I
+# J		J
+Kappa		K
+Lambda		L
+Mu		M
+Nu		N
+Omicron		O
+Pi		P
+Theta		Q
+Rho		R
+Sigma		S
+Tau		T
+Upsilon		U
+varsigma	V
+Omega		W
+Xi		X
+Psi		Y
+Zeta		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+alpha		a
+beta		b
+chi		c
+delta		d
+epsilon		e
+phi		f
+gamma		g
+eta		h
+iota		i
+# j		j
+kappa		k
+lambda		l
+mu		m
+nu		n
+omicron		o
+pi		p
+theta		q
+rho		r
+sigma		s
+tau		t
+upsilon		u
+# xxx		v
+omega		w
+xi		x
+psi		y
+zeta		z
+braceleft	{
+bar		|
+braceright	}
+mathtilde	~
+
+# non-ASCII characters (128-255)
+
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.gen
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.gen	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.gen	(revision 21112)
@@ -0,0 +1,244 @@
+# RTF PC-A character set (\pca) general map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (32-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+quotedbl	'"'
+numbersign	#
+dollar		$
+percent		%
+ampersand	&
+quoteright	"'"
+parenleft	(
+parenright	)
+asterisk	*
+plus		+
+comma		,
+hyphen		-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+equal		=
+greater		>
+question	?
+at		@
+A		A
+B		B
+C		C
+D		D
+E		E
+F		F
+G		G
+H		H
+I		I
+J		J
+K		K
+L		L
+M		M
+N		N
+O		O
+P		P
+Q		Q
+R		R
+S		S
+T		T
+U		U
+V		V
+W		W
+X		X
+Y		Y
+Z		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+a		a
+b		b
+c		c
+d		d
+e		e
+f		f
+g		g
+h		h
+i		i
+j		j
+k		k
+l		l
+m		m
+n		n
+o		o
+p		p
+q		q
+r		r
+s		s
+t		t
+u		u
+v		v
+w		w
+x		x
+y		y
+z		z
+braceleft	{
+bar		|
+braceright	}
+asciitilde	~
+
+# non-ASCII characters (128-255)
+
+
+Ccedilla	0x80
+udieresis	0x81
+eacute		0x82
+acircumflex	0x83
+adieresis	0x84
+agrave		0x85
+aring		0x86
+ccedilla	0x87
+ecircumflex	0x88
+edieresis	0x89
+egrave		0x8a
+idieresis	0x8b
+icircumflex	0x8c
+igrave		0x8d
+Adieresis	0x8e
+Aring		0x8f
+Eacute		0x90
+ae		0x91
+AE		0x92
+ocircumflex	0x93
+odieresis	0x94
+ograve		0x95
+ucircumflex	0x96
+ugrave		0x97
+ydieresis	0x98
+Odieresis	0x99
+Udieresis	0x9a
+oslash		0x9b
+sterling	0x9c
+Oslash		0x9d
+# ???		0x9e
+florin		0x9f
+aacute		0xa0
+iacute		0xa1
+oacute		0xa2
+uacute		0xa3
+ntilde		0xa4
+Ntilde		0xa5
+ordfeminine	0xa6
+ordmasculine	0xa7
+questiondown	0xa8
+registered	0xa9
+logicalnot	0xaa
+# ???		0xab
+# ???		0xac
+exclamdown	0xad
+guillemotleft	0xae
+guillemotright	0xaf
+# ???		0xb0
+# ???		0xb1
+# ???		0xb2
+# ???		0xb3
+# ???		0xb4
+# ???		0xb5
+# ???		0xb6
+Agrave		0xb7
+copyright	0xb8
+# ???		0xb9
+# ???		0xba
+# ???		0xbb
+# ???		0xbc
+cent		0xbd
+yen		0xbe
+# ???		0xbf
+# ???		0xc0
+# ???		0xc1
+# ???		0xc2
+# ???		0xc3
+# ???		0xc4
+# ???		0xc5
+atilde		0xc6
+Atilde		0xc7
+# ???		0xc8
+# ???		0xc9
+# ???		0xca
+# ???		0xcb
+# ???		0xcc
+# ???		0xcd
+# ???		0xce
+# ???		0xcf
+partialdiff	0xd0
+# ???		0xd1
+# ???		0xd2
+# ???		0xd3
+# ???		0xd4
+# ???		0xd5
+# ???		0xd6
+# ???		0xd7
+# ???		0xd8
+# ???		0xd9
+# ???		0xda
+# ???		0xdb
+# ???		0xdc
+# ???		0xdd
+# ???		0xde
+# ???		0xdf
+# ???		0xe0
+germandbls	0xe1
+# ???		0xe2
+# ???		0xe3
+otilde		0xe4
+Otilde		0xe5
+mu		0xe6
+# ???		0xe7
+# ???		0xe8
+# ???		0xe9
+# ???		0xea
+# ???		0xeb
+# ???		0xec
+# ???		0xed
+# ???		0xee
+acute		0xef
+endash		0xf0
+plusminus	0xf1
+equal		0xf2
+# ???		0xf3
+paragraph	0xf4
+section		0xf5
+divide		0xf6
+# ???		0xf7
+degree		0xf8
+dieresis	0xf9
+bullet		0xfa
+# ???		0xfb
+# ???		0xfc
+# ???		0xfd
+# ???		0xfe
+nobrkspace	0xff
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.sym
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.sym	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/pca.sym	(revision 21112)
@@ -0,0 +1,115 @@
+# RTF PC-A character set (\pca) Symbol font map
+
+# Field 1 is the standard character name which the character value in
+# field 2 maps onto.  (It doesn't mean "to produce the character in field 1,
+# use the value in field 2.)
+
+# The character value may be given either as a single character (which will be
+# converted to the ASCII value of the character), or in numeric format, either
+# in decimal or 0xyy as hex yy.  Single or double quotes may be used to quote
+# characters.
+
+# characters in ASCII range (00-127)
+
+formula		0x06
+nobrkhyphen	0x1e
+opthyphen	0x1f
+
+space		" "
+exclam		!
+universal	'"'
+mathnumbersign	#
+existential	$
+percent		%
+ampersand	&
+suchthat	"'"
+parenleft	(
+parenright	)
+mathasterisk	*
+mathplus	+
+comma		,
+mathminus	-
+period		.
+slash		/
+zero		0
+one		1
+two		2
+three		3
+four		4
+five		5
+six		6
+seven		7
+eight		8
+nine		9
+colon		:
+semicolon	;
+less		<
+mathequal	=
+greater		>
+question	?
+congruent	@
+Alpha		A
+Beta		B
+Chi		C
+Delta		D
+Epsilon		E
+Phi		F
+Gamma		G
+Eta		H
+Iota		I
+# J		J
+Kappa		K
+Lambda		L
+Mu		M
+Nu		N
+Omicron		O
+Pi		P
+Theta		Q
+Rho		R
+Sigma		S
+Tau		T
+Upsilon		U
+varsigma	V
+Omega		W
+Xi		X
+Psi		Y
+Zeta		Z
+bracketleft	[
+backslash	\
+bracketright	]
+asciicircum	^
+underscore	_
+quoteleft	"`"
+alpha		a
+beta		b
+chi		c
+delta		d
+epsilon		e
+phi		f
+gamma		g
+eta		h
+iota		i
+# j		j
+kappa		k
+lambda		l
+mu		m
+nu		n
+omicron		o
+pi		p
+theta		q
+rho		r
+sigma		s
+tau		t
+upsilon		u
+# xxx		v
+omega		w
+xi		x
+psi		y
+zeta		z
+braceleft	{
+bar		|
+braceright	}
+mathtilde	~
+
+# non-ASCII characters (128-255)
+
Index: /main/tags/3.01/gsdl/etc/packages/rtftohtm/rtf-ctrl
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/rtftohtm/rtf-ctrl	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/rtftohtm/rtf-ctrl	(revision 21112)
@@ -0,0 +1,769 @@
+# number of control words
+766
+# control word table
+6 0 "vern"
+6 5 "edmins"
+6 6 "yr"
+6 7 "mo"
+6 8 "dy"
+6 9 "hr"
+6 10 "min"
+6 11 "sec"
+6 12 "nofpages"
+6 13 "nofwords"
+6 14 "nofchars"
+6 15 "id"
+6 16 "chdate"
+6 17 "chdpl"
+6 18 "chdpa"
+6 19 "chtime"
+6 20 "chpgn"
+6 21 "sectnum"
+6 22 "chftn"
+6 23 "chatn"
+6 24 "chftnsep"
+6 25 "chftnsepc"
+6 26 "cell"
+6 27 "row"
+6 28 "par"
+6 28 "\n"
+6 28 "\r"
+6 29 "sect"
+6 30 "page"
+6 31 "column"
+6 32 "line"
+6 33 "softpage"
+6 34 "softcol"
+6 35 "softline"
+6 36 "softlheight"
+6 37 "tab"
+6 38 "emdash"
+6 39 "endash"
+6 40 "emspace"
+6 41 "enspace"
+6 42 "bullet"
+6 43 "lquote"
+6 44 "rquote"
+6 45 "ldblquote"
+6 46 "rdblquote"
+6 47 "|"
+6 49 "~"
+6 50 "-"
+6 51 "_"
+6 52 "*"
+6 53 "ltrmark"
+6 54 "rtlmark"
+6 55 "zwj"
+6 56 "zwnj"
+6 57 "chpict"
+6 59 "version"
+12 0 "plain"
+12 1 "b"
+12 2 "caps"
+12 37 "cgrid"
+12 3 "deleted"
+12 4 "dn"
+12 5 "sub"
+12 6 "nosupersub"
+12 7 "expnd"
+12 8 "expndtw"
+12 9 "kerning"
+12 38 "langfe"
+12 39 "langfenp"
+12 35 "lang"
+12 40 "langnp"
+12 10 "f"
+12 11 "fs"
+12 12 "i"
+12 13 "outl"
+12 14 "revised"
+12 15 "revauth"
+12 16 "revdttm"
+12 17 "scaps"
+12 18 "shad"
+12 19 "strike"
+12 20 "ul"
+12 21 "uld"
+12 22 "uldb"
+12 23 "ulnone"
+12 24 "ulw"
+12 25 "up"
+12 26 "super"
+12 27 "v"
+12 28 "cf"
+12 29 "cb"
+12 30 "rtlch"
+12 31 "ltrch"
+12 32 "noproof"
+12 33 "cs"
+12 34 "cchs"
+12 36 "gray"
+11 0 "pard"
+11 1 "s"
+11 2 "hyphpar"
+11 3 "intbl"
+11 104 "itap"
+11 4 "keep"
+11 6 "keepn"
+11 7 "level"
+11 8 "noline"
+11 5 "nowidctlpar"
+11 105 "widctlpar"
+11 7 "outlinelevel"
+11 9 "pagebb"
+11 10 "sbys"
+11 11 "ql"
+11 12 "qr"
+11 13 "qj"
+11 14 "qc"
+11 15 "fi"
+11 16 "li"
+11 16 "lin"
+11 17 "ri"
+11 17 "rin"
+11 17 "curi"
+11 106 "adjustright"
+11 18 "sb"
+11 19 "sa"
+11 20 "sl"
+11 21 "slmult"
+11 22 "subdocument"
+11 23 "rtlpar"
+11 24 "ltrpar"
+11 25 "tx"
+11 26 "tql"
+11 27 "tqr"
+11 28 "tqc"
+11 29 "tqdec"
+11 30 "tb"
+11 31 "tldot"
+11 32 "tlhyph"
+11 33 "tlul"
+11 34 "tlth"
+11 35 "tleq"
+11 36 "pnlvl"
+11 37 "pnlvlblt"
+11 38 "pnlvlbody"
+11 39 "pnlvlcont"
+11 40 "pnnumonce"
+11 41 "pnacross"
+11 42 "pnhang"
+11 43 "pnrestart"
+11 44 "pncard"
+11 45 "pndec"
+11 46 "pnucltr"
+11 47 "pnucrm"
+11 48 "pnlcltr"
+11 49 "pnlcrm"
+11 50 "pnord"
+11 51 "pnordt"
+11 52 "pnb"
+11 53 "pni"
+11 54 "pncaps"
+11 55 "pnscaps"
+11 56 "pnul"
+11 57 "pnuld"
+11 58 "pnuldb"
+11 59 "pnulnone"
+11 60 "pnulw"
+11 61 "pnstrike"
+11 62 "pncf"
+11 63 "pnf"
+11 64 "pnfs"
+11 65 "pnindent"
+11 66 "pnsp"
+11 67 "pnprev"
+11 68 "pnqc"
+11 69 "pnql"
+11 70 "pnqr"
+11 71 "pnstart"
+11 72 "brdrt"
+11 73 "brdrb"
+11 74 "brdrl"
+11 75 "brdrr"
+11 76 "brdrbtw"
+11 77 "brdrbar"
+11 78 "box"
+11 79 "brdrs"
+11 80 "brdrth"
+11 81 "brdrsh"
+11 82 "brdrdb"
+11 83 "brdrdot"
+11 83 "brdrdash"
+11 85 "brdrhair"
+11 86 "brdrw"
+11 87 "brdrcf"
+11 88 "brsp"
+11 89 "shading"
+11 90 "bghoriz"
+11 91 "bgvert"
+11 92 "bgfdiag"
+11 93 "bgbdiag"
+11 94 "bgcross"
+11 95 "bgdcross"
+11 96 "bgdkhoriz"
+11 97 "bgdkvert"
+11 98 "bgdkfdiag"
+11 99 "bgdkbdiag"
+11 100 "bgdkcross"
+11 101 "bgdkdcross"
+11 102 "cfpat"
+11 103 "cbpat"
+11 39 "ls"
+11 40 "ilvl"
+9 0 "sectd"
+9 1 "endnhere"
+9 2 "binfsxn"
+9 3 "binsxn"
+9 4 "ds"
+9 5 "sbknone"
+9 6 "sbkcol"
+9 7 "sbkpage"
+9 8 "sbkeven"
+9 9 "sbkodd"
+9 10 "cols"
+9 11 "colsx"
+9 12 "colno"
+9 13 "colsr"
+9 14 "colw"
+9 15 "linebetcol"
+9 16 "linemod"
+9 17 "linex"
+9 18 "linestarts"
+9 19 "linerestart"
+9 20 "lineppage"
+9 21 "linecont"
+9 22 "pgwsxn"
+9 23 "pghsxn"
+9 24 "marglsxn"
+9 25 "margrsxn"
+9 26 "margtsxn"
+9 27 "margbsxn"
+9 28 "guttersxn"
+9 29 "lndscpsxn"
+9 30 "titlepg"
+9 31 "headery"
+9 32 "footery"
+9 33 "pgnstarts"
+9 34 "pgncont"
+9 35 "pgnrestart"
+9 36 "pgnx"
+9 37 "pgny"
+9 38 "pgndec"
+9 39 "pgnucrm"
+9 40 "pgnlcrm"
+9 41 "pgnucltr"
+9 42 "pgnlcltr"
+9 43 "pgnhnsh"
+9 44 "pgnhnsp"
+9 45 "pgnhnsc"
+9 46 "pgnhnsm"
+9 47 "pgnhnsn"
+9 48 "vertalt"
+9 49 "vertalb"
+9 50 "vertalc"
+9 51 "vertalj"
+9 52 "rtlsect"
+9 53 "ltrsect"
+9 54 "horzsect"
+9 55 "vertsect"
+9 100 "sectexpand"
+9 100 "sectlinegrid"
+9 100 "sectdefaultcl"
+9 100 "sectspecifycl"
+9 100 "sectspecifyl"
+9 100 "sectspecifygen"
+9 56 "pgnhn"
+9 57 "sectunlocked"
+8 160 "uc"
+8 0 "deftab"
+8 1 "hyphhotz"
+8 2 "hyphconsec"
+8 3 "hyphcaps"
+8 4 "hyphauto"
+8 5 "linestart"
+8 6 "fracwidth"
+8 7 "makeback"
+8 7 "makebackup"
+8 8 "defformat"
+8 9 "psover"
+8 10 "doctemp"
+8 11 "deflang"
+8 150 "deflangfe"
+8 151 "windowcaption"
+8 152 "doctype"
+8 153 "fromtext"
+8 154 "fromhtml"
+8 155 "horzdoc"
+8 156 "vertdoc"
+8 157 "jcompress"
+8 158 "jexpand"
+8 159 "lnongrid"
+8 78 "viewkind"
+8 79 "viewscale"
+8 79 "viewzk"
+8 12 "fet"
+8 13 "endnotes"
+8 14 "enddoc"
+8 15 "ftntj"
+8 16 "ftnbj"
+8 17 "aendnotes"
+8 18 "aenddoc"
+8 19 "aftntj"
+8 20 "aftnbj"
+8 21 "ftnstart"
+8 22 "aftnstart"
+8 23 "ftnrstpg"
+8 24 "ftnrestart"
+8 25 "ftnrstcont"
+8 26 "aftnrestart"
+8 27 "aftnrstcont"
+8 28 "ftnnar"
+8 29 "ftnnalc"
+8 30 "ftnnauc"
+8 31 "ftnnrlc"
+8 32 "ftnnruc"
+8 33 "ftnnchi"
+8 34 "aftnnar"
+8 35 "aftnnalc"
+8 36 "aftnnauc"
+8 37 "aftnnrlc"
+8 38 "aftnnruc"
+8 39 "aftnnchi"
+8 40 "paperw"
+8 41 "paperh"
+8 42 "psz"
+8 43 "margl"
+8 44 "margr"
+8 45 "margt"
+8 46 "margb"
+8 47 "facingp"
+8 48 "gutter"
+8 49 "margmirror"
+8 50 "landscape"
+8 51 "pgnstart"
+8 52 "widowctrl"
+8 53 "linkstyles"
+8 54 "notabind"
+8 55 "wraptrsp"
+8 56 "prcolbl"
+8 57 "noextrasprl"
+8 58 "nocolbal"
+8 59 "cvmme"
+8 60 "sprstsp"
+8 61 "sprsspbf"
+8 62 "otblrul"
+8 63 "transmf"
+8 64 "swpbdr"
+8 65 "brkfrm"
+8 82 "formprot"
+8 83 "allprot"
+8 84 "formshade"
+8 85 "formdisp"
+8 86 "printdata"
+8 87 "revprot"
+8 88 "revisions"
+8 89 "revprop"
+8 90 "revbar"
+8 91 "annotprot"
+8 92 "rtldoc"
+8 93 "ltrdoc"
+8 94 "cts"
+8 95 "dghspace"
+8 96 "dgvspace"
+8 97 "dghorigin"
+8 98 "dgvorigin"
+8 99 "dghshow"
+8 100 "dgvshow"
+8 101 "dgsnap"
+8 102 "dgmargin"
+8 103 "pgbrdrhead"
+8 104 "pgbrdrfoot"
+8 105 "aspalpha"
+8 106 "aspnum"
+8 107 "faauto"
+8 66 "truncex"
+8 67 "bdbfhdr"
+8 68 "dntblnsbdb"
+8 69 "expshrtn"
+8 70 "lytexcttp"
+8 71 "lytprtmet"
+8 72 "msmcap"
+8 73 "nolead"
+8 74 "nospaceforul"
+8 75 "noultrlspc"
+8 76 "noxlattoyen"
+8 77 "oldas"
+8 78 "lnbrkrule"
+8 79 "bdrrlswsix"
+8 81 "sprstsm"
+8 80 "nolnhtadjtbl"
+7 0 "additive"
+7 1 "sbasedon"
+7 2 "snext"
+7 3 "sautoupd"
+13 32 "emfblip"
+13 33 "pngblip"
+13 34 "jpegblip"
+13 36 "nonshppict"
+13 0 "macpict"
+13 1 "pmmetafile"
+13 2 "wmetafile"
+13 3 "dibitmap"
+13 4 "wbitmap"
+13 5 "wbmbitspixel"
+13 6 "wbmplanes"
+13 7 "wbmwidthbytes"
+13 8 "picw"
+13 9 "pich"
+13 10 "picwgoal"
+13 11 "pichgoal"
+13 10 "picwGoal"
+13 11 "pichGoal"
+13 12 "picscalex"
+13 13 "picscaley"
+13 14 "picscaled"
+13 15 "piccropt"
+13 16 "piccropb"
+13 17 "piccropl"
+13 18 "piccropr"
+13 19 "picbmp"
+13 20 "picbpp"
+13 21 "bin"
+13 22 "blipupi"
+13 24 "bliptag"
+15 0 "width"
+15 1 "height"
+3 0 "fonttbl"
+3 1 "falt"
+3 2 "fonteb"
+3 3 "fontfile"
+3 4 "filetbl"
+3 5 "file"
+3 6 "colortbl"
+3 7 "stylesheet"
+3 8 "keycode"
+3 9 "revtbl"
+3 10 "info"
+3 11 "title"
+3 12 "subject"
+3 13 "author"
+3 14 "operator"
+3 15 "keywords"
+3 16 "comment"
+3 18 "doccomm"
+3 19 "verscomm"
+3 20 "nextfile"
+3 21 "template"
+3 22 "ftnsep"
+3 23 "ftnsepc"
+3 24 "ftncn"
+3 25 "aftnsep"
+3 26 "aftnsepc"
+3 27 "aftncn"
+3 29 "pnseclvl"
+3 30 "header"
+3 31 "footer"
+3 32 "headerl"
+3 33 "headerr"
+3 34 "headerf"
+3 35 "footerl"
+3 36 "footerr"
+3 37 "footerf"
+3 38 "pntext"
+3 39 "pn"
+3 40 "pntxta"
+3 41 "pntxtb"
+3 42 "bkmkstart"
+3 43 "bkmkend"
+3 44 "pict"
+3 45 "object"
+3 46 "objclass"
+3 47 "objname"
+3 48 "objtime"
+3 49 "objdata"
+3 50 "objalias"
+3 51 "objsect"
+3 53 "objitem"
+3 54 "objtopic"
+3 52 "result"
+3 55 "do"
+3 56 "footnote"
+3 57 "atrfstart"
+3 58 "atrfend"
+3 59 "atnid"
+3 60 "atnauthor"
+3 61 "annotation"
+3 62 "atnref"
+3 63 "atntime"
+3 64 "atnicn"
+3 65 "field"
+3 66 "fldinst"
+3 67 "fldrslt"
+3 68 "datafield"
+3 69 "xe"
+3 70 "txe"
+3 71 "rxe"
+3 72 "tc"
+3 73 "NeXTGraphic"
+3 74 "NeXTHelpMarker"
+3 28 "NeXTHelpLink"
+3 75 "buptim"
+3 76 "creatim"
+3 77 "printim"
+3 78 "revtim"
+3 79 "dptxbxtext"
+3 80 "fontemb"
+3 81 "tcn"
+3 35 "shppict"
+3 41 "listtext"
+3 79 "private"
+3 82 "shpgrp"
+4 0 "fnil"
+4 1 "froman"
+4 2 "fswiss"
+4 3 "fmodern"
+4 4 "fscript"
+4 5 "fdecor"
+4 6 "ftech"
+4 7 "fbidi"
+23 0 "fcharset"
+23 1 "fprq"
+23 2 "cpg"
+23 3 "ftnil"
+23 4 "fttruetype"
+24 0 "fid"
+24 1 "frelative"
+24 2 "fosnum"
+25 0 "fvalidmac"
+25 1 "fvaliddos"
+25 2 "fvalidntfs"
+25 3 "fvalidhpfs"
+25 4 "fnetwork"
+5 0 "red"
+5 1 "green"
+5 2 "blue"
+2 2 "mac"
+2 0 "ansi"
+2 3 "pc"
+2 4 "pca"
+2 1 "ansicpg"
+10 0 "trowd"
+10 1 "trgaph"
+10 2 "cellx"
+10 3 "clmgf"
+10 4 "clmrg"
+10 5 "trql"
+10 6 "trqr"
+10 7 "trqc"
+10 8 "trleft"
+10 9 "trrh"
+10 10 "trhdr"
+10 11 "trkeep"
+10 12 "rtlrow"
+10 13 "ltrrow"
+10 14 "trbrdrt"
+10 15 "trbrdrl"
+10 16 "trbrdrb"
+10 17 "trbrdrr"
+10 18 "trbrdrh"
+10 19 "trbrdrv"
+10 20 "clbrdrb"
+10 21 "clbrdrt"
+10 22 "clbrdrl"
+10 23 "clbrdrr"
+10 42 "clvertalt"
+10 43 "clvertalc"
+10 44 "clvertalb"
+10 45 "cltxlrtb"
+10 46 "cltxlbrl"
+10 24 "clshdng"
+10 25 "clbghoriz"
+10 26 "clbgvert"
+10 27 "clbgfdiag"
+10 28 "clbgbdiag"
+10 29 "clbgcross"
+10 30 "clbgdcross"
+10 31 "clbgdkhoriz"
+10 31 "clbgdkhor"
+10 32 "clbgdkvert"
+10 33 "clbgdkfdiag"
+10 34 "clbgdkbdiag"
+10 35 "clbgdkcross"
+10 36 "clbgdkdcross"
+10 37 "clcfpat"
+10 38 "clcbpat"
+16 0 "flddirty"
+16 1 "fldedit"
+16 2 "fldlock"
+16 3 "fldpriv"
+16 4 "fldalt"
+18 0 "absw"
+18 1 "absh"
+18 2 "phmrg"
+18 3 "phpg"
+18 4 "phcol"
+18 5 "posx"
+18 6 "posnegx"
+18 7 "posxc"
+18 8 "posxi"
+18 9 "posxo"
+18 10 "posxr"
+18 11 "posxl"
+18 12 "pvmrg"
+18 13 "pvpg"
+18 14 "pvpara"
+18 15 "posy"
+18 16 "posnegy"
+18 17 "posyil"
+18 18 "posyt"
+18 19 "posyc"
+18 20 "posyb"
+18 21 "nowrap"
+18 22 "dxfrtext"
+18 23 "dfrmtxtx"
+18 24 "dfrmtxty"
+18 25 "dyfrtext"
+18 26 "dropcapli"
+18 27 "dropcapt"
+19 0 "objemb"
+19 1 "objlink"
+19 2 "objautlink"
+19 3 "objsub"
+19 4 "objpub"
+19 5 "objicemb"
+19 6 "linkself"
+19 7 "objupdate"
+19 8 "objlock"
+19 9 "objh"
+19 10 "objw"
+19 11 "objsetsize"
+19 12 "objalign"
+19 13 "objtransy"
+19 14 "objcropt"
+19 15 "objcropb"
+19 16 "objcropl"
+19 17 "objcropr"
+19 18 "objscalex"
+19 19 "objscaley"
+19 20 "rsltrtf"
+19 21 "rsltpict"
+19 22 "rsltbmp"
+19 23 "rslttxt"
+19 24 "rsltmerge"
+19 25 "bkmkpub"
+19 26 "pubauto"
+22 0 "ab"
+22 1 "acaps"
+22 2 "acf"
+22 3 "adn"
+22 4 "aexpnd"
+22 5 "af"
+22 6 "afs"
+22 7 "ai"
+22 8 "alang"
+22 9 "aoutl"
+22 10 "ascaps"
+22 11 "ashad"
+22 12 "astrike"
+22 13 "aul"
+22 14 "auld"
+22 15 "auldb"
+22 16 "aulnone"
+22 17 "aulw"
+22 18 "aup"
+20 0 "ftnalt"
+21 0 "alt"
+21 1 "shift"
+21 2 "ctrl"
+21 3 "fn"
+14 0 "bkmkcolf"
+14 1 "bkmkcoll"
+27 0 "xef"
+27 1 "bxe"
+27 2 "ixe"
+17 0 "tcf"
+17 1 "tcl"
+26 0 "dolock"
+26 1 "dobxpage"
+26 2 "dobxcolumn"
+26 3 "dobxmargin"
+26 4 "dobypage"
+26 6 "dobymargin"
+26 7 "dodhgt"
+26 8 "dpgroup"
+26 9 "dpcount"
+26 10 "dpendgroup"
+26 11 "dparc"
+26 12 "dpcallout"
+26 13 "dpellipse"
+26 14 "dpline"
+26 15 "dppolygon"
+26 16 "dppolyline"
+26 17 "dprect"
+26 18 "dptxbx"
+26 19 "dpx"
+26 20 "dpxsize"
+26 21 "dpy"
+26 22 "dpysize"
+26 23 "dpcoa"
+26 24 "dpcoaccent"
+26 25 "dpcobestfit"
+26 26 "dpcoborder"
+26 27 "dpcodabs"
+26 28 "dpcodbottom"
+26 29 "dpcodcenter"
+26 30 "dpcodtop"
+26 31 "dpcolength"
+26 32 "dpcominusx"
+26 33 "dpcominusy"
+26 34 "dpcooffset"
+26 35 "dpcosmarta"
+26 36 "dpcotdouble"
+26 37 "dpcotright"
+26 38 "dpcotsingle"
+26 39 "dpcottriple"
+26 40 "dptxbxmar"
+26 42 "dproundr"
+26 43 "dpptx"
+26 44 "dppty"
+26 45 "dppolycount"
+26 46 "dparcflipx"
+26 47 "dparcflipy"
+26 48 "dplinecob"
+26 49 "dplinecog"
+26 50 "dplinecor"
+26 51 "dplinepal"
+26 52 "dplinedado"
+26 53 "dplinedadodo"
+26 54 "dplinedash"
+26 55 "dplinedot"
+26 56 "dplinegray"
+26 57 "dplinehollow"
+26 58 "dplinesolid"
+26 59 "dplinew"
+26 60 "dpaendhol"
+26 61 "dpaendl"
+26 62 "dpaendsol"
+26 63 "dpaendw"
+26 64 "dpastarthol"
+26 65 "dpastartl"
+26 66 "dpastartsol"
+26 67 "dpastartw"
+26 68 "dpfillbgcb"
+26 69 "dpfillbgcg"
+26 70 "dpfillbgcr"
+26 71 "dpfillbgpal"
+26 72 "dpfillbggray"
+26 73 "dpfillfgcb"
+26 74 "dpfillfgcg"
+26 75 "dpfillfgcr"
+26 76 "dpfillfgpal"
+26 77 "dpfillfggray"
+26 78 "dpfillpat"
+26 79 "dpshadow"
+26 80 "dpshadx"
+26 81 "dpshady"
+26 82 "dobypara"
+0 -1 "rtf"
+1 -1 "deff"
Index: /main/tags/3.01/gsdl/etc/packages/wv/wvHtml.xml
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/wv/wvHtml.xml	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/wv/wvHtml.xml	(revision 21112)
@@ -0,0 +1,397 @@
+<main>
+<charentity>
+<begin>HTML</begin>
+</charentity>
+
+<document>
+<begin>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0 Transitional//EN&quot; &quot;http://www.w3.org/TR/REC-html40/loose.dtd&quot;&gt;
+&lt;html&gt; 
+&lt;head&gt; 
+&lt;META HTTP-EQUIV=&quot;Content-Type&quot; CONTENT=&quot;text/html; charset=<charset/>&quot;&gt;
+&lt;META NAME=&quot;GENERATOR&quot; CONTENT=&quot;wvWare/<version/>&quot;&gt;
+&lt;title&gt; 
+<title/>
+&lt;/title&gt; 
+&lt;/head&gt; 
+&lt;body bgcolor=&quot;#FFFFFF&quot; text=&quot;#000000&quot; link=&quot;#0000ee&quot; vlink=&quot;#551a8b&quot;&gt;
+</begin>
+<end>
+&lt;/body&gt;
+&lt;/html&gt;
+</end>
+</document>
+
+<section>
+<begin>
+&lt;!--Section Begins--&gt;&lt;br&gt;
+</begin>
+<end>
+&lt;!--Section Ends--&gt;
+</end>
+</section>
+
+<justification>
+<left>left</left>
+<right>right</right>
+<center>center</center>
+<block>justify</block>
+<asian>&lt;!--Could Someone who sees this tag tell me what was is this type of justification, asian languages only i thing--&gt;</asian>
+</justification>
+
+<numbering>
+<Arabic>type=&quot;1&quot;</Arabic>
+<UpperRoman>type=&quot;I&quot;</UpperRoman>
+<LowerRoman>type=&quot;i&quot;</LowerRoman>
+<UpperCaseN>type=&quot;A&quot;</UpperCaseN>
+<LowerCaseN>type=&quot;a&quot;</LowerCaseN>
+</numbering>
+
+<border>
+<noned>none</noned>
+<singled>solid</singled>
+<thickd>solid</thickd>
+<doubled>double</doubled>
+<number4d>double</number4d>
+<hairlined>solid</hairlined>
+<dotd>dotted</dotd>
+<dashlargegapd>dashed</dashlargegapd>
+<dotdashd>dotted</dotdashd>
+<dotdotdashd>dotted</dotdotdashd>
+<tripled>double</tripled>
+<thin-thicksmallgapd>dashed</thin-thicksmallgapd>
+<thick-thinsmallgapd>dashed</thick-thinsmallgapd>
+<thin-thick-thinsmallgapd>dashed</thin-thick-thinsmallgapd>
+<thin-thickmediumgapd>dashed</thin-thickmediumgapd>
+<thick-thinmediumgapd>dashed</thick-thinmediumgapd>
+<thin-thick-thinmediumgapd>dashed</thin-thick-thinmediumgapd>
+<thin-thicklargegapd>dashed</thin-thicklargegapd>
+<thick-thinlargegapd>dashed</thick-thinlargegapd>
+<thin-thick-thinlargegapd>dashed</thin-thick-thinlargegapd>
+<waved>solid</waved>
+<doublewaved>double</doublewaved>
+<dashsmallgapd>sashed</dashsmallgapd>
+<dashdotstrokedd>dashed</dashdotstrokedd>
+<emboss3Dd>ridge</emboss3Dd>
+<engrave3Dd>groove</engrave3Dd>
+<defaultd>ridge</defaultd>
+</border>
+
+<olist>
+<begin>&lt;ol <nfc/> start=&quot;<start/>&quot;&gt;
+</begin>
+<end>&lt;/ol&gt;</end>
+</olist>
+
+<ulist>
+<begin>&lt;ul&gt;
+</begin>
+<end>&lt;/ul&gt;</end>
+</ulist>
+
+<entry>
+<begin>&lt;li&gt;</begin>
+<end></end>
+</entry>
+
+
+<!-- 
+this tableoverride option can be used to turn off handling of
+these tags in tables, which I find is necessary for at least netscape
+-->
+<tableoverrides>
+<ParaBefore>0</ParaBefore>
+<ParaRight>0</ParaRight>
+<ParaAfter>0</ParaAfter>
+<ParaLeft>0</ParaLeft>
+<ParaLeft1>0</ParaLeft1>
+<VertMergedCells>0</VertMergedCells>
+</tableoverrides>
+
+<table>
+<begin>&lt;table width=&quot;<tablerelwidth/>%&quot; border=&quot;1&quot; cols=&quot;<no_cols/>&quot; rows=&quot;<no_rows/>&quot;&gt;</begin>
+<end>&lt;/table&gt;</end>
+</table>
+
+<row>
+<begin>&lt;tr&gt;</begin>
+<end>&lt;/tr&gt;</end>
+</row>
+
+<cell>
+<begin>&lt;td bgcolor=&quot;<cellbgcolor/>&quot; width=&quot;<cellrelwidth/>%&quot; rowspan=&quot;<rowspan/>&quot; colspan=&quot;<colspan/>&quot;&gt;</begin>
+<end>&lt;/td&gt;</end>
+</cell>
+
+<paragraph>
+<begin>
+<table.end/>
+<table.begin/>
+<row.begin/>
+<cell.begin/><olist.end/><olist.begin/><ulist.end/><ulist.begin/><entry.begin/>
+<text.begin/>
+</begin>
+<end>
+<text.end/>
+<entry.end/>
+<olist.end/>
+<ulist.end/>
+<cell.end/>
+<row.end/>
+</end>
+</paragraph>
+
+<!-- these are all the character properties that can show up in word -->
+<bold><begin>&lt;b&gt;</begin><end>&lt;/b&gt;</end></bold>
+<italic><begin>&lt;i&gt;</begin><end>&lt;/i&gt;</end></italic>
+
+<!--
+text that has been deleted and will be displayed with strikethrough when
+revision marked text is to be displayed
+
+use either this line...
+--> 
+<RMarkDel><begin>&lt;s&gt;</begin><end>&lt;/s&gt;&lt;a href=&quot;#author<ibstRMarkDel/>&quot;&gt;[Author ID<ibstRMarkDel/>: at <dttmRMarkDel/> ]&lt;/a&gt;</end></RMarkDel>
+
+<!--
+or uncomment below to make deleted text dissappear (well, become commented out)
+-->
+<!--
+<RMarkDel><begin>&lt;!-&#45;</begin><end>-&#45;&gt;</end></RMarkDel>
+-->
+
+<!-- I don't even know what outline means -->
+<outline><begin></begin><end></end></outline>
+<smallcaps><begin></begin><end></end></smallcaps>
+<caps><begin></begin><end></end></caps>
+<vanish><begin></begin><end></end></vanish>
+
+<!--If you uncomment this then the annotation text links will become commented out by html tags-->
+<!--
+<vanish><begin>&lt;!-&#45;</begin><end>-&#45;&gt;</end></vanish>
+-->
+
+<!--
+text that has been newly typed since the last time revision marks have been accepted
+and will be displayed with underline when revision marked text is to be displayed
+
+use either this line...
+-->
+<RMark><begin>&lt;u&gt;</begin><end>&lt;/u&gt;&lt;a href=&quot;#author<ibstRMark/>&quot;&gt;[Author ID<ibstRMark/>: at <dttmRMark/>]&lt;/a&gt;</end></RMark>
+
+<!--
+or uncomment below to make the underline dissappear
+-->
+<!--
+<RMark><begin></begin><end></end></RMark>
+-->
+
+
+<strike><begin>&lt;s&gt;</begin><end>&lt;/s&gt;</end></strike>
+<shadow><begin></begin><end></end></shadow>
+<lowercase><begin></begin><end></end></lowercase>
+<emboss><begin></begin><end></end></emboss>
+<imprint><begin></begin><end></end></imprint>
+<!--double strike-->
+<dstrike><begin>&lt;s&gt;</begin><end>&lt;/s&gt;</end></dstrike>
+
+<!--
+ftc's
+&
+hps
+
+keep them for font face and do that later.
+-->
+
+<super><begin>&lt;sup&gt;</begin><end>&lt;/sup&gt;</end></super>
+<sub><begin>&lt;sub&gt;</begin><end>&lt;/sub&gt;</end></sub>
+
+<singleu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></singleu>
+<wordu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></wordu>
+<doubleu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></doubleu>
+<dottedu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></dottedu>
+<hiddenu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></hiddenu>
+<thicku><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></thicku>
+<dashu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></dashu>
+<dotu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></dotu>
+<dotdashu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></dotdashu>
+<dotdotdashu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></dotdotdashu>
+<waveu><begin>&lt;u&gt;</begin><end>&lt;/u&gt;</end></waveu>
+
+<!--
+text whose properties have been changed since the last time revision marks have been accepted
+and will be displayed with a note showing the change points.
+
+use either this line (which admit it a bit scary looking, but harmless)...
+--> 
+<PropRMark><begin>&lt;a href=&quot;#author<ibstPropRMark/>&quot;&gt;-&#45;&amp;gt;&lt;/a&gt;</begin><end>&lt;a href=&quot;#author<ibstPropRMark/>&quot;&gt;[Author ID<ibstPropRMark/>: at <dttmPropRMark/>]&lt;/a&gt;</end></PropRMark>
+
+<!--
+or uncomment below to make the notes dissappear
+-->
+<!--
+<PropRMark><begin></begin><end></end></PropRMark>
+-->
+
+<!--
+<color>
+-->
+<Black><begin>Black</begin><end></end></Black>
+<Blue><begin>Blue</begin><end></end></Blue>
+<Cyan><begin>Cyan</begin><end></end></Cyan>
+<Green><begin>Green</begin><end></end></Green>
+<Magenta><begin>Magenta</begin><end></end></Magenta>
+<Red><begin>Red</begin><end></end></Red>
+<Yellow><begin>Yellow</begin><end></end></Yellow>
+<White><begin>White</begin><end></end></White>
+<DkBlue><begin>DarkBlue</begin><end></end></DkBlue>
+<DkCyan><begin>DarkCyan</begin><end></end></DkCyan>
+<DkGreen><begin>DarkGreen</begin><end></end></DkGreen>
+<DkMagenta><begin>DarkMagenta</begin><end></end></DkMagenta>
+<DkRed><begin>DarkRed</begin><end></end></DkRed>
+<DkYellow><begin>#8b8b00</begin><end></end></DkYellow>
+<DkGray><begin>DarkGray</begin><end></end></DkGray>
+<LtGray><begin>LightGrey</begin><end></end></LtGray>
+<!--
+</color>
+-->
+
+<!--
+<animation>
+-->
+<LasVegas><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></LasVegas>
+<BackgroundBlink><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></BackgroundBlink>
+<SparkleText><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></SparkleText>
+<MarchingAnts><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></MarchingAnts>
+<MarchingRedAnts><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></MarchingRedAnts>
+<Shimmer><begin>&lt;blink&gt;</begin><end>&lt;/blink&gt;</end></Shimmer>
+<!--
+</animation>
+-->
+
+<!--
+I dont understand what this one is, and ive never come across it
+
+use this sample line (which admit it a bit scary looking, but harmless)...
+-->
+<DispFldRMark><begin>&lt;a href=&quot;#author<ibstDispFldRMark/>&quot;&gt;--&amp;gt;&lt;/a&gt;</begin><end>&lt;a href=&quot;#author<ibstDispFldRMark/>&quot;&gt;[Author ID<ibstDispFldRMark/>: at <dttmDispFldRMark/> (<xstDispFldRMark/>)]&lt;/a&gt;</end></DispFldRMark>
+
+<!--
+or uncomment below to ignore it, the previous might even crash wv ?
+-->
+<!--
+<DispFldRMark><begin></begin><end></end></DispFldRMark>
+-->
+
+<animation>
+<begin><LasVegas.begin/><BackgroundBlink.begin/><SparkleText.begin/><MarchingAnts.begin/><MarchingRedAnts.begin/><Shimmer.begin/></begin>
+<end><Shimmer.end/><MarchingRedAnts.end/><MarchingAnts.end/><SparkleText.end/><BackgroundBlink.end/><LasVegas.end/></end>
+</animation>
+
+<fontstr>
+<begin>&lt;font color=&quot;<black.begin/><blue.begin/><cyan.begin/><green.begin/><magenta.begin/><red.begin/><yellow.begin/><white.begin/><dkblue.begin/><dkcyan.begin/><dkgreen.begin/><dkmagenta.begin/><dkred.begin/><dkyellow.begin/><dkgray.begin/><ltgray.begin/>&quot;&gt;</begin>
+<end><ltgray.end/><dkgray.end/><dkyellow.end/><dkred.end/><dkmagenta.end/><dkgreen.end/><dkcyan.end/><dkblue.end/><white.end/><yellow.end/><red.end/><magenta.end/><green.end/><cyan.end/><blue.end/><black.end/>&lt;/font&gt;</end>
+</fontstr>
+
+<comment>
+<begin>
+&lt;a href=&quot;#comment<ibstAnno/> &quot;&gt;-&#45;&amp;gt;&lt;/a&gt;
+</begin>
+<end>&lt;a href=&quot;#comment<ibstAnno/> &quot;&gt;[Author:<xstUsrInitl/>]&lt;/a&gt;
+</end>
+</comment>
+
+<style name="Normal">
+<character>
+<begin><PropRMark.begin/><DispFldRMark.begin/><animation.begin/><fontstr.begin/><bold.begin/><italic.begin/><strike.begin/><RMarkDel.begin/><outline.begin/><smallcaps.begin/><caps.begin/><vanish.begin/><RMark.begin/><shadow.begin/><lowercase.begin/><emboss.begin/><imprint.begin/><dstrike.begin/><super.begin/><sub.begin/><singleu.begin/><wordu.begin/><doubleu.begin/><dottedu.begin/><hiddenu.begin/><thicku.begin/><dashu.begin/><dotu.begin/><dotdashu.begin/><dotdotdashu.begin/><waveu.begin/></begin>
+<end><waveu.end/><dotdotdashu.end/><dotdashu.end/><dotu.end/><dashu.end/><thicku.end/><hiddenu.end/><dottedu.end/><doubleu.end/><wordu.end/><singleu.end/><sub.end/><super.end/><dstrike.end/><imprint.end/><emboss.end/><lowercase.end/><shadow.end/><RMark.end/><vanish.end/><caps.end/><smallcaps.end/><outline.end/><RMarkDel.end/><strike.end/><italic.end/><bold.end/><fontstr.end/><animation.end/><DispFldRMark.end/><PropRMark.end/></end>
+</character>
+
+<!-- Netscape does handle this correctly yet, here is how each different side of the border should work.
+border-top: thin <bordertopstyle/> <bordertopcolor/>;
+border-left: thin <borderleftstyle/> <borderleftcolor/>;
+border-right: thin <borderrightstyle/> <borderrightcolor/>;
+border-bottom: thin <borderbottomstyle/> <borderbottomcolor/>
+-->
+
+
+<pmargin>
+<begin>margin: <mmParaBefore/> <mmParaRight/> <mmParaAfter/> <mmParaLeft/>;</begin>
+</pmargin>
+
+<pborder>
+<begin>
+border: thin <borderleftstyle/> <borderleftcolor/>;
+<!--
+border-top: thin <bordertopstyle/> <bordertopcolor/>;
+border-left: thin <borderleftstyle/> <borderleftcolor/>;
+border-right: thin <borderrightstyle/> <borderrightcolor/>;
+border-bottom: thin <borderbottomstyle/> <borderbottomcolor/>
+-->
+</begin>
+</pborder>
+
+<text>	
+<begin>
+&lt;div align=&quot;<just/>&quot; style=&quot;<paramargin/> <paraborder/>
+padding: <mmPadTop/> <mmPadRight/> <mmPadBottom/> <mmPadLeft/>; &quot;&gt;
+&lt;p style=&quot;text-indent: <mmParaLeft1/>; text-align: <just/>; color: <parafgcolor/>; background-color: <parabgcolor/>; &quot;&gt;
+</begin>
+<end>
+&lt;/p&gt;
+&lt;/div&gt;
+</end>
+</text>
+
+<picture>
+<begin>
+&lt;img <htmlAlignGuess/> width=&quot;<pixPicWidth/>&quot; height=&quot;<pixPicHeight/>&quot; alt=&quot;0x01 graphic&quot; src=&quot;placeholder.png&quot;&gt;<htmlNextLineGuess/>
+</begin>
+</picture>
+
+</style>
+
+<!--we need to be override the character properties-->
+<!--
+<style name="Normal">
+<character>
+<begin><PropRMark.begin/><DispFldRMark.begin/><animation.begin/><fontstr.begin/><bold.begin/><italic.begin/><strike.begin/><RMarkDel.begin/><outline.begin/><smallcaps.begin/><caps.begin/><vanish.begin/><RMark.begin/><shadow.begin/><lowercase.begin/><emboss.begin/><imprint.begin/><dstrike.begin/><super.begin/><sub.begin/><singleu.begin/><wordu.begin/><doubleu.begin/><dottedu.begin/><hiddenu.begin/><thicku.begin/><dashu.begin/><dotu.begin/><dotdashu.begin/><dotdotdashu.begin/><waveu.begin/></begin>
+<end><waveu.end/><dotdotdashu.end/><dotdashu.end/><dotu.end/><dashu.end/><thicku.end/><hiddenu.end/><dottedu.end/><doubleu.end/><wordu.end/><singleu.end/><sub.end/><super.end/><dstrike.end/><imprint.end/><emboss.end/><lowercase.end/><shadow.end/><RMark.end/><vanish.end/><caps.end/><smallcaps.end/><outline.end/><RMarkDel.end/><strike.end/><italic.end/><bold.end/><fontstr.end/><animation.end/><DispFldRMark.end/><PropRMark.end/></end>
+</character>
+
+<text>	
+<begin>
+&lt;div align=&quot;<just/>&quot;&gt;
+&lt;p&gt;
+</begin>
+<end>
+&lt;/p&gt;
+&lt;/div&gt;
+</end>
+</text>
+
+</style>
+
+<style name="Heading 1">
+
+<character>
+<begin></begin>
+<end></end>
+</character>
+
+<text>	
+<begin>
+&lt;div align=&quot;<just/>&quot;&gt;
+&lt;H1&gt;
+</begin>
+<end>
+&lt;/H1&gt;
+&lt;/div&gt;
+</end>
+</text>
+
+
+
+</style>
+-->
+
+</main>
Index: /main/tags/3.01/gsdl/etc/packages/z3950.cfg
===================================================================
--- /main/tags/3.01/gsdl/etc/packages/z3950.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl/etc/packages/z3950.cfg	(revision 21112)
@@ -0,0 +1,43 @@
+[GSDL2.1] # config file version
+#
+# This file specifies a list of z39.50 servers to use.
+# Each field is separated by white-space.
+# If you want a " character inside a string, escape it - ie \" or ""
+
+#
+# Each server configuration consists of a stanza. The items are described here:
+#	Item				Description
+#	###############			########################
+#	collectname			<GSDL internal name (alphanumeric) >
+#	servername:port			<Internet address (def. port is 210) >
+#	dbname				<z39.50 server database name>
+#	"humanname"			<Brief description>
+#	icon "url to icon"		<optional GSDL icon (eg collection) >
+#	smallicon "url to small icon"	<optional GSDL icon (eg homepage) >
+#	About "about text"		<optional>
+
+
+# For an exhaustive list of servers to try, have a look at
+#  http://www.indexdata.dk/targettest/
+
+
+#LOCtest
+#       ilssun2.loc.gov:7090
+#       Voyager
+#       "Library of Congress TEST z39.50 server"
+#       About "This database/server is provided by the US LoC for the testing
+#           of z39.50 clients. For server capabilities, look at
+#           <a href=\"http://lcweb.loc.gov/z3950/lcserver.html\">their web 
+#           pages</a>. Note that <b>no records will ever be returned</b> -
+#           this server is for testing the protocols only."
+
+
+#LOC
+#        z3950.loc.gov:7090
+#        Voyager
+#        "Library of Congress z39.50 server"
+#        About   "This is the z39.50 interface to the US LoC catalogue
+#           system. It contains approximately 12 million bibliographic records.
+#           For server capabilities, look at
+#           <a href=\"http://lcweb.loc.gov/z3950/lcserver.html\">their web 
+#           pages</a>."
Index: /main/tags/3.01/gsdl/install-sh
===================================================================
--- /main/tags/3.01/gsdl/install-sh	(revision 21112)
+++ /main/tags/3.01/gsdl/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/gsdl/lib/gsdlconf.h
===================================================================
--- /main/tags/3.01/gsdl/lib/gsdlconf.h	(revision 21112)
+++ /main/tags/3.01/gsdl/lib/gsdlconf.h	(revision 21112)
@@ -0,0 +1,161 @@
+/**********************************************************************
+ *
+ * gsdlconf.h -- system specific includes
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifndef GSDLCONF_H
+#define GSDLCONF_H
+
+#define GSDL_VERSION "x.xx"
+
+#if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_) || defined(WIN32)
+#ifndef __WIN32__
+#define __WIN32__ 1
+#endif
+#endif
+
+#if defined(__WIN32__) && !defined(__CYGWIN__)
+#include "../win32cfg.h"
+#else
+#include "../config.h"
+#endif
+
+// Note: all macros used in the gsdl software (except the _H macros) should
+// now start with GSDL_
+
+// GSDL_NEED_STRINGS_H should be defined if some string functions
+// needed (like strcasecmp) are in strings.h
+#if defined(__IBMCPP__)
+#  define GSDL_NEED_STRINGS_H
+#endif
+
+// GSDL_USE_IOS_H should be defined if the .h extension needs to
+// be used for the io stream headers
+#if (defined(__GNUG__)  && __GNUG__ + 0 < 3) || defined(__IBMCPP__)
+#  define GSDL_USE_IOS_H 1
+#endif
+
+// GSDL_USE_STL_H should be defined if the .h extension needs to
+// be used for the stl header files
+#if defined(__IBMCPP__)
+#  define GSDL_USE_STL_H 1
+#endif
+
+// GSDL_USE_ALGO_H should be defined if algo.h should be used 
+// instead of algorithm.h
+#if defined(__IBMCPP__)
+#  define GSDL_USE_ALGO_H 1
+#endif
+
+// GSDL_NAMESPACE_BROKEN should be defined if namespaces should
+// not be used
+#if (defined(__GNUG__)  && __GNUG__ + 0 < 3) || defined(__IBMCPP__)
+#  define GSDL_NAMESPACE_BROKEN 1
+#endif
+
+// GSDL_USE_OBJECTSPACE should now be used instead of USE_OBJECTSPACE
+#if defined(USE_OBJECTSPACE)
+#  define GSDL_USE_OBJECTSPACE
+#endif
+
+// GSDL_NEED_DESTROY_USHORT should be defined if the compiler needs
+// a definition for void destroy(short unsigned int *)
+#if defined(__IBMCPP__)
+# define GSDL_NEED_DESTROY_USHORT
+#endif
+
+// mktemp
+
+#if !defined(__WIN32__) || defined(__GNUC__)
+#include <unistd.h>
+#define GSDL_MKTEMP(str) mktemp(str)
+#else
+#include <io.h>
+#define GSDL_MKTEMP(str) _mktemp(str)
+#endif
+
+
+
+// file locking
+
+#if defined(__WIN32__)
+
+#if !defined(LK_UNLOCK) && defined(LK_UNLCK)
+#define LK_UNLOCK LK_UNLCK
+#endif
+
+#include <io.h>
+#include <sys/locking.h>
+
+#define GSDL_UNLOCK_FILE(fd) _locking(fd, LK_UNLCK, 200)
+#define GSDL_LOCK_FILE(fd) lock_val = _locking(fd, LK_NBLCK, 200)
+
+#else     /* not WIN32 */
+
+#if GSDL_USE_FLOCK
+#include <sys/lock.h>
+
+#ifndef LOCK_SH
+#define LOCK_SH	1
+#endif
+
+#ifndef LOCK_EX
+#define LOCK_EX	2
+#endif
+
+#ifndef LOCK_NB
+#define LOCK_NB 4
+#endif
+
+#ifndef LOCK_UN
+#define LOCK_UN 8
+#endif
+
+#define GSDL_UNLOCK_FILE(fd) flock (fd, LOCK_UN)
+#define GSDL_LOCK_FILE(fd) lock_val = flock (fd, LOCK_EX + LOCK_NB)
+
+#else  /* ! GSDL_USE_FLOCK */
+// use fcntl
+#include <fcntl.h>
+#define GSDL_UNLOCK_FILE(fd) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_UNLCK;		\
+	  flock.l_whence = SEEK_SET;		\
+	  flock.l_start = flock.l_len = 0L;	\
+	  fcntl (fd, F_SETLK, &flock);	\
+	}
+#define GSDL_LOCK_FILE(fd) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_WRLCK;		\
+	  flock.l_whence = SEEK_SET;			\
+	  flock.l_start = flock.l_len = 0L;	\
+	  lock_val = fcntl (fd, F_SETLK, &flock);	\
+	}
+
+#endif
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/lib/text_t.cpp
===================================================================
--- /main/tags/3.01/gsdl/lib/text_t.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/lib/text_t.cpp	(revision 21112)
@@ -0,0 +1,939 @@
+/**********************************************************************
+ *
+ * text_t.cpp -- a simple 16-bit character string class
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.25  2005/06/16 22:49:44  kjdon
+   added starts_with and ends_with functions
+
+   Revision 1.24  2005/04/07 04:27:18  kjdon
+   added some x++ -> ++x changes submitted by Emanuel Dejanu
+
+   Revision 1.23  2004/12/02 22:21:34  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.22  2004/05/24 03:43:22  mdewsnip
+   (Human Info) Added const text_t g_EmptyText("") to use instead of plain ol' "".
+
+   Revision 1.21  2001/06/01 02:51:28  sjboddie
+   Changes to get phind working under windows
+
+   Revision 1.20  2001/01/25 18:26:44  cs025
+   Included CORBA branch for first time
+
+   Revision 1.15.2.2  2000/04/05 10:19:38  syeates
+   added automatic conversion to allow text_t's to be <<'ed to ostreams
+
+   Revision 1.15.2.1  2000/04/04 15:02:29  cs025
+   Corba first commit
+
+   Revision 1.15  1999/10/14 22:52:39  sjboddie
+   joinchar can join using text_t string now too
+
+   Revision 1.14  1999/09/24 02:30:03  rjmcnab
+   added function has_unicode_letdig
+
+   Revision 1.13  1999/09/07 04:57:43  sjboddie
+   added gpl notice
+
+   Revision 1.12  1999/08/31 08:04:41  rjmcnab
+   Fixed a small but hard to find bug in getcarr
+
+   Revision 1.11  1999/07/01 04:05:09  rjmcnab
+   Optimised append functions slightly and added a reserve function.
+
+   Revision 1.10  1999/04/26 03:58:03  sjboddie
+   added is_number function
+
+   Revision 1.9  1999/04/06 22:17:24  rjmcnab
+   Added splits and joins using text_tset.
+
+   Revision 1.8  1999/02/28 23:14:41  rjmcnab
+
+   Added uc and lc to convert to uppercase and lowercase.
+
+   Revision 1.7  1999/02/21 22:26:39  rjmcnab
+
+   Made getint() a constant function.
+
+   Revision 1.6  1999/02/03 01:13:26  sjboddie
+
+   Got interface to handle subcollections and language subcollections -
+   committed changes made to some of the collections
+
+   Revision 1.5  1999/01/19 01:38:14  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.4  1999/01/12 01:51:00  rjmcnab
+
+   Standard header.
+
+   Revision 1.3  1999/01/08 02:33:16  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+#include "text_t.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\algorithm>
+#elif defined(GSDL_USE_STL_H)
+#  if defined(GSDL_USE_ALGO_H)
+#    include <algo.h>
+#  else
+#    include <algorithm.h>
+#  endif
+#else
+#  include <algorithm>
+#endif
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include "WIN32cfg.h"
+# else
+#  include "config.h"
+# endif
+#endif
+
+
+#include "unitool.h"
+
+const text_t g_EmptyText("");
+
+////////////////////////////////////
+// text_t methods
+////////////////////////////////////
+
+// new stream converter ...
+ostream& operator<< (ostream &o, const text_t &text)
+{
+  text_t::const_iterator ithere = text.begin();
+  text_t::const_iterator itend = text.end();
+
+  while (ithere != itend)
+    {
+      if (*ithere < 256)
+	{
+	  o << (unsigned char)(*ithere);
+	}
+      else 
+	{
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*ithere)) 
+	  o << ' ';
+	else 
+	  o << '?';
+      }
+      ++ithere;
+    }
+
+  return o;
+}
+
+text_t::text_t () 
+{
+  setencoding(0);
+  clear ();
+}
+
+text_t::text_t (int i) 
+{
+  setencoding(0);
+  clear ();
+  appendint (i);
+}
+
+text_t::text_t (const char *s) 
+{ 
+  setencoding(0);
+  clear ();
+  appendcstr (s);
+}
+
+text_t::text_t (const char *s, size_type nLength)
+{
+  setencoding(0);
+  clear ();
+  appendcarr(s, nLength);
+}
+
+
+void text_t::append (const text_t &t) 
+{
+  text.insert(text.end(), t.begin(), t.end());
+}
+
+void text_t::appendrange (iterator first, iterator last) 
+{
+  text.insert(text.end(), first, last);
+}
+
+void text_t::appendrange (const_iterator first, const_iterator last) 
+{
+  text.insert(text.end(), first, last);
+}
+
+void text_t::appendint (int i)
+{
+  // deal with zeros and negatives
+  if (i == 0) 
+    {
+      text.push_back('0');
+      return;
+    }
+  else if (i < 0)
+    {
+      text.push_back('-');
+      i *= -1;
+    }
+
+  // get a buffer for the conversion
+  int maxbuflen = sizeof(int)*3;
+  char *buf = new char[maxbuflen];
+  int len = 0;
+  
+  // get the number in reverse
+  while (i > 0)
+    {
+      buf[len++] = '0'+ (i%10);
+      i = i/10;
+    }
+
+  // reverse the number
+  while (len > 0)
+    {
+      text.push_back(buf[--len]);
+    }
+
+  delete []buf;
+}
+
+int text_t::getint () const
+{
+  int i = 0;
+  int mult = 1; // become -1 for negative numbers
+
+  const_iterator here = text.begin();
+  const_iterator end = text.end();
+  
+  // do plus and minus signs
+  if (here != end)
+    {
+      if (*here == '-')
+	{
+	  mult = -1;
+	  ++here;
+	}
+      else if (*here == '+')
+	{
+	  mult = 1;
+	  ++here;
+	}
+    }
+
+  // deal with the number
+  while ((here != end) && (*here >= '0') && (*here <= '9'))
+    {
+      i = 10*i + (*here - '0');
+      ++here;
+    }
+
+  i *= mult;
+  return i;
+}
+
+unsigned long text_t::getulong () const
+{
+  unsigned long i = 0;
+
+  const_iterator here = text.begin();
+  const_iterator end = text.end();
+
+  while ((here != end) && (*here >= '0') && (*here <= '9'))
+    {
+      i = 10*i + (*here - '0');
+      ++here;
+    }
+
+  return i;
+}
+
+void text_t::appendcarr (const char *s, size_type len)
+{
+  unsigned char *us = (unsigned char *)s;
+  if (text.capacity() < (text.size() + len + 2)) {
+    text.reserve(text.size() + len + 2);
+  }
+
+  while (len > 0) 
+    {
+      text.push_back (*us); // append this character
+      ++us;
+      --len;
+    }
+}
+
+void text_t::appendcstr (const char *s) 
+{
+  size_t len = strlen(s);
+  if (text.capacity() < (text.size() + len + 2)) {
+    text.reserve(text.size() + len + 2);
+  }
+  
+  unsigned char *us = (unsigned char *)s;
+  while (*us != '\0') 
+    {
+      text.push_back (*us); // append this character
+      ++us;
+    }
+}
+
+
+// strings returned from getcarr and getcstr become the callers 
+// responsibility and should be deallocated with "delete []"
+
+char *text_t::getcarr(size_type &len) const
+{
+  unsigned char *cstr = new unsigned char[size()];
+  len = 0;
+
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+  while (ithere != itend)
+    {
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*ithere)) cstr[len] = ' ';
+	else cstr[len] = '?';
+      }
+      ++len;
+      ++ithere;
+    }
+
+  return (char *)cstr;
+}
+
+char *text_t::getcstr() const
+{
+  unsigned char *cstr = new unsigned char[size() + 1];
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+  int len = 0;
+
+  while (ithere != itend)
+    {
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*ithere)) cstr[len] = ' ';
+	else cstr[len] = '?';
+      }
+      ++len;
+      ++ithere;
+    }
+
+  cstr[len] = '\0';
+
+  return (char *)cstr;
+}
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      ++first;
+    }
+  return first;
+}
+
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      ++first;
+    }
+  return first;
+}
+
+text_t::iterator findword (text_t::iterator first, 
+			   text_t::iterator last, 
+			   const text_t& word)
+{
+  text_t::const_iterator word_begin = word.begin();
+  text_t::const_iterator word_end = word.end();
+
+  while (first != last)
+    {
+      text_t::iterator char_match = first;
+      text_t::const_iterator word_here = word_begin;
+      while (word_here!=word_end)
+	{
+	  if (*char_match != *word_here)
+	    {
+	      break;
+	    }
+	  ++char_match;
+	  ++word_here;
+	}
+      if (word_here==word_end)
+	{
+	  return first;
+	}
+      ++first;
+    }
+  return last; // get to here only if there is no match
+}
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr)
+{
+  text_t::const_iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) ++here; // skip c
+  return here;
+}
+
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr)
+{
+  text_t::iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) ++here; // skip c
+  return here;
+}
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tset &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.insert (t);
+    }
+}
+
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+// join a string using a character
+void joinchar (const text_tset &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tset::const_iterator here = inlist.begin ();
+  text_tset::const_iterator end = inlist.end ();
+
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext.push_back (c);
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tlist::const_iterator here = inlist.begin ();
+  text_tlist::const_iterator end = inlist.end ();
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext.push_back (c);
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tarray::const_iterator here = inlist.begin ();
+  text_tarray::const_iterator end = inlist.end ();
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext.push_back (c);
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+void joinchar (const text_tlist &inlist, const text_t &c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tlist::const_iterator here = inlist.begin ();
+  text_tlist::const_iterator end = inlist.end ();
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext += c;
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+void joinchar (const text_tset &inlist, const text_t &c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tset::const_iterator here = inlist.begin ();
+  text_tset::const_iterator end = inlist.end ();
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext += c;
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+void joinchar (const text_tarray &inlist, const text_t &c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tarray::const_iterator here = inlist.begin ();
+  text_tarray::const_iterator end = inlist.end ();
+  if (here != end) {
+    outtext += *here; ++here;
+    while (here != end) {
+      outtext += c;
+      outtext += *here;
+      ++here;
+    }
+  }
+}
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c)
+{
+  int count = 0;
+  while (first != last) {
+    if (*first == c) ++count;
+    ++first;
+  }
+  return count;
+}
+
+// return a substring of string from first up to but not including last
+text_t substr (text_t::const_iterator first, text_t::const_iterator last) {
+
+  text_t substr; substr.reserve(last - first + 2);
+  while (first != last) {
+    substr.push_back(*first);
+    ++first;
+  }
+  return substr;
+}
+
+
+// convert to lowercase
+void lc (text_t::iterator first, text_t::iterator last) {
+  while (first != last) {
+    *first = unicode_tolower(*first);
+    ++first;
+  }
+}
+
+// convert to uppercase
+void uc (text_t::iterator first, text_t::iterator last) {
+  while (first != last) {
+    *first = unicode_toupper(*first);
+    ++first;
+  }
+}
+
+
+// checks to see if it is a number (i.e. contains only 0-9)
+bool is_number (const text_t &text) {
+
+  text_t::const_iterator here = text.begin();
+  text_t::const_iterator end = text.end();
+
+  while (here != end) {
+    if ((*here!='0') && (*here!='1') && (*here!='2') &&
+	(*here!='3') && (*here!='4') && (*here!='5') &&
+	(*here!='6') && (*here!='7') && (*here!='8') &&
+	(*here!='9')) return false;
+    ++here;
+  }
+  return true;
+}
+
+
+// checks to see if the text has any letters or digits
+bool has_unicode_letdig (const text_t &text) {
+  if (text.empty()) return false;
+  
+  text_t::const_iterator here = text.begin();
+  text_t::const_iterator end = text.end();
+  while (here != end) {
+    if (is_unicode_letdig (*here)) return true;
+    ++here;
+  }
+
+  return false;
+}
+
+// checks to see if a text_t starts with the specified prefix
+bool starts_with(const text_t& text, const text_t& prefix) {
+  if (prefix.empty()) return true;
+  if (text.empty() || text.size()<prefix.size()) return false;
+  text_t substring = substr(text.begin(), text.begin()+prefix.size());
+  return substring == prefix;
+}
+// checks to see if a text_t ends with the specified suffix
+bool ends_with(const text_t& text, const text_t& suffix) {
+  if (suffix.empty()) return true;
+  if (text.empty() || text.size() < suffix.size()) return false;
+  text_t substring = substr(text.end()-suffix.size(),text.end());
+  return substring == suffix;
+
+}
+
+
+////////////////////////////////////
+// convertclass methods
+////////////////////////////////////
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+convertclass::convertclass () 
+{
+  // nothing to do
+}
+
+void convertclass::reset ()
+{
+  // nothing to do
+}
+
+
+////////////////////////////////////
+// inconvertclass methods
+////////////////////////////////////
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+
+inconvertclass::inconvertclass () 
+{
+  start = NULL;
+  len = 0;
+}
+
+
+void inconvertclass::reset ()
+{
+  start = NULL;
+  len = 0;
+}
+
+void inconvertclass::setinput (char *thestart, size_t thelen)
+{
+  start = thestart;
+  len = thelen;
+}
+
+void inconvertclass::convert (text_t &output, status_t &status)
+{
+  output.clear();
+
+  if (start == NULL || len == 0)
+    {
+      status = finished;
+      return;
+    }
+
+  if (output.capacity() < len + 2)
+    output.reserve(len + 2);
+  
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+  while (len > 0) 
+    {
+      output.push_back (*here); // append this character
+      ++here;
+      --len;
+    }
+
+  start = (char *)here; // save current position
+  status = finished;
+}
+
+// will treat the text_t as a 8-bit string and convert
+// it to a 16-bit string using the about convert method.
+text_t inconvertclass::convert (const text_t &t) {
+  text_t out;
+  text_t tmpout;
+  status_t status;
+  text_t::const_iterator here = t.begin();
+  text_t::const_iterator end = t.end();
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+ 
+  out.clear();
+  if (out.capacity() < t.size() + 2)
+    out.reserve(t.size() + 2);
+  while (here != end) {
+    while (here != end && cbuflen < 256) {
+      cbuf[cbuflen++] = (unsigned char)(*here & 0xff);
+      ++here;
+    }
+
+    if (cbuflen > 0) {
+      setinput ((char *)cbuf, cbuflen);
+      status = unfinished;
+      while (status == unfinished) {
+	convert (tmpout, status);
+	out += tmpout;
+      }
+      cbuflen = 0;
+    }
+  }
+
+  out.setencoding (0); // unicode
+
+  return out;
+}
+
+// an instance of the default inconvertclass to do simple
+// conversions. Note that any functions that use this are
+// not reentrant. If a function needs to be reentrant it
+// should declare its own instance.
+inconvertclass ascii2text_t;
+
+
+////////////////////////////////////
+// outconvertclass methods
+////////////////////////////////////
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << text_tstr << anothertext_tstr;
+//
+outconvertclass::outconvertclass ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::reset ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::setinput (text_t *theinput)
+{
+  input = theinput;
+  if (input != NULL) texthere = input->begin();
+}
+
+void outconvertclass::setdata(text_t *theinput, text_t::iterator thetexthere)
+{
+  input = theinput;     
+  texthere = thetexthere;
+}
+ 
+void outconvertclass::convert (char *output, size_t maxlen, 
+		      size_t &len, status_t &status)
+{
+  if (input == NULL || output == NULL)
+    {
+      status = finished;
+      return;
+    }
+
+  // don't want any funny sign conversions happening
+  unsigned char *uoutput = (unsigned char *)output;
+  text_t::iterator textend = input->end();
+  len = 0;
+  while ((len < maxlen) && (texthere != textend)) 
+    {
+      if (*texthere < 256) *uoutput = (unsigned char)(*texthere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*texthere)) *uoutput = ' ';
+	else *uoutput = '?';
+      }
+      ++uoutput;
+      ++len;
+      ++texthere;
+    }
+  
+  if (texthere == textend) status = finished;
+  else status = unfinished;
+}
+
+// will convert the 16-bit string to a 8-bit stream
+// and place the result in a text_t. This method uses
+// the above convert function.
+text_t outconvertclass::convert (const text_t &t) {
+  text_t out;
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+  status_t status = unfinished;
+  
+  out.clear(); 
+  if (out.capacity() < t.size() + 2)
+    out.reserve(t.size() + 2);
+  setinput ((text_t *)&t); // discard constant
+  while (status == unfinished) {
+    convert ((char *)cbuf, 256, cbuflen, status);
+    out.appendcarr ((char *)cbuf, cbuflen);
+  }
+
+  out.setencoding (1); // other encoding
+  
+  return out;
+}
+
+
+void outconvertclass::setostream (ostream *theouts)
+{
+  outs = theouts;
+}
+
+ostream *outconvertclass::getostream ()
+{
+  return outs;
+}
+
+
+
+
+// an instance of the default outconvertclass to do simple
+// conversions
+outconvertclass text_t2ascii;
+
+
+
+// stream operators for the output class
+
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter)
+{
+  outconverter.setostream(&theouts);
+  return outconverter;
+}
+
+
+#define STREAMBUFSIZE 256
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t)
+{
+  ostream *outstream = outconverter.getostream();
+
+  if (outstream == NULL) return outconverter;
+
+  char outbuf[STREAMBUFSIZE];
+  size_t len;
+  outconvertclass::status_t status = outconvertclass::unfinished;
+
+  // assume that there is no data needing converting
+  // left in the converter
+  outconverter.setinput ((text_t *)(&t)); // note the const -> nonconst conversion
+
+  while (status == outconvertclass::unfinished)
+    {
+      outconverter.convert (outbuf, STREAMBUFSIZE, len, status);
+      if (len > 0) outstream->write(outbuf, len);
+    }
+
+  return outconverter;
+}
Index: /main/tags/3.01/gsdl/lib/text_t.h
===================================================================
--- /main/tags/3.01/gsdl/lib/text_t.h	(revision 21112)
+++ /main/tags/3.01/gsdl/lib/text_t.h	(revision 21112)
@@ -0,0 +1,375 @@
+/**********************************************************************
+ *
+ * text_t.h -- a simple 16-bit character string class
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef TEXT_T_H
+#define TEXT_T_H
+
+#include "gsdlconf.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\set>
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <list.h>
+#  include <set.h>
+#  include <map.h>
+#else
+#  include <vector>
+#  include <list>
+#  include <set>
+#  include <map>
+#endif
+
+// use the standard namespace
+#if !defined (GSDL_NAMESPACE_BROKEN)
+#if defined(GSDL_USE_OBJECTSPACE)
+using namespace ospace::std;
+#else
+using namespace std;
+#endif
+#endif
+
+// class prototypes
+class text_t;
+class inconvertclass;
+class outconvertclass;
+
+
+// for those stupid compilers which need it
+#if defined(GSDL_NEED_DESTROY_USHORT)
+inline void destroy(unsigned short *) {};
+inline void destroy(int *) {};
+#endif
+
+typedef vector<unsigned short> usvector;
+extern const text_t g_EmptyText;
+
+// The class text_t can handle long strings which may contain
+// null characters. It uses unsigned shorts to represent up to
+// 64K character values.
+class text_t {
+  public:
+  //type support for ucvector
+  typedef usvector::iterator iterator;
+  typedef usvector::const_iterator const_iterator;
+  typedef usvector::reference reference;
+  typedef usvector::const_reference const_reference;
+  typedef usvector::size_type size_type;
+  typedef usvector::difference_type difference_type;
+  typedef usvector::const_reverse_iterator const_reverse_iterator;
+  typedef usvector::reverse_iterator reverse_iterator;
+  
+protected:
+  usvector text;
+  unsigned short encoding; // 0 = unicode, 1 = other
+  
+public:
+  // constructors
+  text_t ();
+  text_t (int i);
+  text_t (const char *s); // assumed to be a normal c string
+  text_t (const char *s, size_type nLength); // support for arrays of chars
+  void setencoding (unsigned short theencoding) {encoding=theencoding;};
+  unsigned short getencoding () {return encoding;};
+
+  usvector& text_as_usvector() { return text ; };
+  const usvector& text_as_usvector() const { return text ; };
+
+  // basic container support
+  iterator begin () {return text.begin();}
+  const_iterator begin () const {return text.begin();}
+  iterator end () {return text.end();}
+  const_iterator end () const {return text.end();}
+
+  void erase(iterator pos) {text.erase(pos);}
+  void erase(iterator first, iterator last) {text.erase(first, last);}
+  void push_back(unsigned short c) {text.push_back(c);}
+  void pop_back() {text.pop_back();}
+  text_t &operator=(const text_t &x) {text=x.text; encoding=x.encoding; return *this;}
+  reference operator[](size_type n) {return text[n];};
+  const_reference operator[](size_type n) const {return text[n];};
+
+  void reserve (size_type n) {text.reserve(n);}
+  size_type capacity() const { return text.capacity(); }
+  
+  bool empty () const {return text.empty();}
+  size_type size() const {return text.size();}
+  friend inline bool operator!=(const text_t& x, const text_t& y)
+	{return (x.text != y.text);}
+  friend inline bool operator==(const text_t& x, const text_t& y) 
+	{return (x.text == y.text);}
+  friend inline bool operator<(const text_t& x, const text_t& y) 
+	{return (x.text < y.text);}
+  friend inline bool operator>(const text_t& x, const text_t& y) 
+	{return (x.text > y.text);}
+  friend inline bool operator>=(const text_t& x, const text_t& y) 
+	{return (x.text >= y.text);}
+  friend inline bool operator<=(const text_t& x, const text_t& y) 
+	{return (x.text <= y.text);}
+  
+  // added functionality
+  void clear () {text.erase(text.begin(),text.end());}
+  void append (const text_t &t);
+  void appendrange (iterator first, iterator last);
+  void appendrange (const_iterator first, const_iterator last);
+  text_t &operator+= (const text_t &t) {append(t);return *this;}
+
+  // support for integers
+  void appendint (int i);
+  void setint (int i) {clear();appendint(i);}
+  text_t &operator=(int i) {setint (i);return *this;}
+  text_t &operator+= (int i) {appendint(i);return *this;}
+  int getint () const;
+
+  // same as getint but returns an unsigned long
+  unsigned long getulong () const;
+
+  // support for arrays of chars
+  void appendcarr (const char *s, size_type len);
+  void setcarr (char *s, size_type len) {clear();appendcarr(s,len);}
+
+  // support for const null-terminated C strings
+  void appendcstr (const char *s);
+  void setcstr (const char *s) {clear();appendcstr(s);}
+  text_t &operator= (const char *s) {setcstr(s);return *this;} // c string
+  text_t &operator+= (const char *s) {appendcstr(s);return *this;} // c string
+
+  // strings returned from getcarr and getcstr become the callers 
+  // responsibility and should be deallocated with "delete []"
+  char *getcarr(size_type &len) const;
+  char *getcstr() const;
+
+};
+
+// new stream converter ...
+ostream& operator<< (ostream &o, const text_t &text);
+
+inline text_t operator+(const text_t &t1, const text_t &t2)
+{
+  text_t tnew = t1;
+  tnew.append(t2);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, int i1)
+{
+  text_t tnew = t1;
+  tnew.appendint(i1);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, char *s1)
+{
+  text_t tnew = t1;
+  tnew.appendcstr(s1);
+  return tnew;
+}
+
+
+
+
+struct eqtext_t 
+{
+  bool operator()(const text_t &t1, const text_t &t2) const
+  { return t1 == t2; }
+};
+
+struct lttext_t 
+{
+  bool operator()(const text_t &t1, const text_t &t2) const
+  { return t1 < t2; }
+};
+
+
+// frequently used derived types
+typedef set<text_t,lttext_t> text_tset;
+typedef list<text_t> text_tlist;  // more efficient for insertions/deletions
+typedef vector<text_t> text_tarray; // more space efficient than text_tlist
+typedef map<text_t, text_t, lttext_t> text_tmap;
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c);
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c);
+
+text_t::iterator findword (text_t::iterator first, text_t::iterator last,
+			   const text_t &word);
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr);
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr);
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tset &outlist);
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist);
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist);
+
+// join a string using a character
+void joinchar (const text_tset &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tset &inlist, const text_t &c, text_t &outtext);
+void joinchar (const text_tlist &inlist, const text_t &c, text_t &outtext);
+void joinchar (const text_tarray &inlist, const text_t &c, text_t &outtext);
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c);
+
+// return a substring of string from first up to but not including last
+text_t substr (text_t::const_iterator first, text_t::const_iterator last);
+
+// convert to lowercase
+void lc (text_t::iterator first, text_t::iterator last);
+inline void lc (text_t &t) {lc (t.begin(), t.end());}
+
+// convert to uppercase
+void uc (text_t::iterator first, text_t::iterator last);
+inline void uc (text_t &t) {uc (t.begin(), t.end());}
+
+// checks to see if it is a number (i.e. contains only 0-9)
+bool is_number (const text_t &text);
+
+// checks to see if the text has any letters or digits
+bool has_unicode_letdig (const text_t &text);
+
+// checks to see if a text_t starts with the specified prefix
+bool starts_with(const text_t& text, const text_t& prefix);
+// checks to see if a text_t ends with the specified suffix
+bool ends_with(const text_t& text, const text_t& suffix);
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+class convertclass 
+{
+public:
+  enum status_t {finished, stopped, unfinished};
+  
+  convertclass ();
+  virtual void reset ();
+};
+
+
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+class inconvertclass : public convertclass
+{
+public:
+  inconvertclass ();
+  void reset ();
+  void setinput (char *thestart, size_t thelen);
+
+  // output will be cleared before the conversion
+  virtual void convert (text_t &output, status_t &status);
+
+  // will treat the text_t as a 8-bit string and convert
+  // it to a 16-bit string using the about convert method.
+  text_t convert (const text_t &t);
+
+protected:
+  char *start;
+  size_t len;
+};
+
+// to get something which will do the conversion
+// to ascii declare a (non global!) instance like
+// this
+// inconvertclass ascii2text_t;
+
+#if defined(GSDL_USE_IOS_H)
+#include <iostream.h> // darwin doesn't have ostream.h...
+#else
+#include <ostream>
+#endif
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << textstr << anothertextstr;
+//
+// this class assumes that the input text doesn't change
+// while the conversion takes place
+class outconvertclass : public convertclass
+{
+public:
+  outconvertclass ();
+  virtual void reset ();
+  virtual void setinput (text_t *theinput);
+  virtual void setdata(text_t *input, text_t::iterator texthere);
+  // note that convert does not null-terminate the
+  // output array of characters
+  virtual void convert (char *output, size_t maxlen, 
+			size_t &len, status_t &status);
+
+  // will convert the 16-bit string to a 8-bit stream
+  // and place the result in a text_t. This method uses
+  // the above convert function.
+  text_t convert (const text_t &t);
+
+  virtual void setostream (ostream *theouts);
+  ostream *getostream ();
+
+protected:
+  text_t *input;
+  text_t::iterator texthere; // only valid if input is valid
+
+  ostream *outs;
+};
+
+// to get something which will do the conversion
+// to text_t declare a (non global!) instance like
+// this
+// outconvertclass text_t2ascii;
+
+
+// stream operators for the output class
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter);
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t);
+
+
+#endif
Index: /main/tags/3.01/gsdl/mappings/README
===================================================================
--- /main/tags/3.01/gsdl/mappings/README	(revision 21112)
+++ /main/tags/3.01/gsdl/mappings/README	(revision 21112)
@@ -0,0 +1,22 @@
+This directory and its subdirectories contain .ump mapping files for
+converting various character encodings to and from unicode.
+
+To generate .ump files use a command like "makemapfile.pl -encoding
+encodingname -mapfile textmapfile" where encodingname becomes the filename
+of the two new .ump files and textmapfile is a plain text file containing a
+tab separated list of the form:
+0x8167	      0x201C
+where the first column is the hexadecimal value of the encoded character
+and the second is the hexadecimal value of it's unicode equivalent.
+
+
+
+The following .ump files were generated from their corresponding Microsoft
+codepages. These codepages do, in some cases, differ very slightly from the
+standards they were based on but we've used them anyway as they're so
+extensively used on the web.
+
+* gbk.ump: Simplified Chinese - generated from Microsoft's codepage 936
+* shiftjis.ump: Japanese - generated from Microsoft's codepage 932
+* uhc.ump: UHC Korean - generated from Microsoft's codepage 949
+* big5.ump: Traditional Chinese - generated from Microsoft's codepage 950
Index: /main/tags/3.01/gsdl/packages/cpan/README
===================================================================
--- /main/tags/3.01/gsdl/packages/cpan/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/cpan/README	(revision 21112)
@@ -0,0 +1,42 @@
+This directory contains perl modules from cpan for use with the Greenstone
+collection building code. After a "make install" these modules will be
+installed to the gsdl/perllib/cpan directory.
+
+Note that pure perl modules (i.e. those not containing any C code to
+compile) don't need building so they don't live in the packages directory
+tree (they live in perllib/cpan).
+
+
+The following modules are currently included:
+
+XML::Parser: Copyright (c) 1998-2000 Larry Wall and Clark Cooper
+ 
+ - Version 2.34
+
+  Originally, we used Version 2.27 cut down a bit for size, plus
+  a few modifications to Makefile.PL.  See below.
+
+
+HISTORY
+-------
+
+Mar 2005
+
+XML::Parser
+
+ - Version 2.34 and version 2.27
+added MAKEFILE_OLD => 'Makefile' to the Makefile.PL WriteMakeFile directive.
+previously, doing a make clean renamed the Makefile to Makefile.old, so then
+a make wouldn't work. Now Makefile gets renamed to Makefile. Hopefully this
+won't stuff anything else up.
+ 
+< Aug 2004
+
+XML::Parser: Copyright (c) 1998-2000 Larry Wall and Clark Cooper
+
+- Version 2.27 - as downloaded from cpan.org but with a few files removed
+  (to minimize the size of the tarball we distribute) and minor Greenstone
+  specific modifications to Makefile.PL. Originally used version 2.29 but
+  it appeared to be incompatible with perl 5.6.1.
+
+
Index: /main/tags/3.01/gsdl/packages/expat/Makefile
===================================================================
--- /main/tags/3.01/gsdl/packages/expat/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/expat/Makefile	(revision 21112)
@@ -0,0 +1,12 @@
+
+all:
+	(cd expat-1.95.8 ; make all installlib)
+
+clean:
+	(cd expat-1.95.8 ; make clean)
+
+distclean:
+	(cd expat-1.95.8 ; make distclean)
+
+install:
+	@echo Expat installed
Index: /main/tags/3.01/gsdl/packages/isis-gdl/AbstractIsisDb.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/AbstractIsisDb.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/AbstractIsisDb.h	(revision 21112)
@@ -0,0 +1,171 @@
+/**********************************************************************
+ *
+ * AbstractIsisDb.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+///////////////////////////////////////////////////////////////////////////////////
+// AbstractIsisDb.h -- Abstract Base Class Definitions for an ISIS db
+//
+//
+// The objective is to provide an interface for creating families of Isis
+// db without specifying their concrete classes. 
+// ABSTRACT FACTORY PATTERN is used.
+//
+// Author: J-C Dauphin, UNESCO, August 2000
+///////////////////////////////////////////////////////////////////////////////////
+ 
+#ifndef __ABSTRACT_ISISDB_H__
+#define __ABSTRACT_ISISDB_H__
+
+#include <vector>
+#include "IsisDef.h"
+#include "mytchar.h"
+
+
+/////////////////////////////////////////////////////////////////////////////////// 
+//   Abstract Base Class, set the DB interface. All db concrete classes must be
+//   derived from this class.
+//
+// Note that this abstract base class can be extended by specifying abstract base
+// classes for class class MfFile, class XrfFile, class CFdtCWorksheet, 
+// class CDisplayFormat, class CFieldSelectTable, class CSortTable, 
+// class MfRecord and struct SMfHeader. Pure virtual methods would then be added
+// in the class AbstractIsisDbFactory:
+//
+// 	virtual AbstractMfFile* makeMfFile() = 0;
+//	virtual AbstractXrfFile* makeXrfFile() = 0;
+//  ................
+ 
+
+class MfFile;
+class XrfFile;
+class CFdt; // Forward 
+
+class CWorksheet;
+class CDisplayFormat;
+class CFieldSelectTable;
+class CSortTable;
+class MfRecord;
+struct SMfHeader;
+ 
+class AbstractIsisDb 
+{
+protected:
+	ustring    	name;
+
+    MfFile*   mf_;   // Has-a master file
+    XrfFile*  xf_;   // Has-a cross ref file
+    CFdt*     fdt_;  // Has-a FDT
+
+    std::vector<CWorksheet*>            apWS_;   // Worksheets
+    std::vector<CDisplayFormat*>        apDFMT_; // Display formats
+    std::vector<CFieldSelectTable*>     apFST_;  // Field Select Tables
+    std::vector<CSortTable*>            apST_;   // Sort Tables   
+    
+public:
+	AbstractIsisDb() { };                    // Trivial default C++ constructor
+	virtual ~AbstractIsisDb() { };           // Trivial C++ destructor
+
+// Opening the database
+   virtual int OpenDb(const TCHAR *fname, FileSystem::AccessMode mode =
+	                                          FileSystem::FILE_READWRITE) = 0; 
+// Creating the database
+   virtual int CreateDb(const TCHAR* fname) = 0;
+
+// Closing the database
+   virtual void CloseDb()   = 0;    // Close the db
+   virtual void ClearDb()   = 0;    // Clear the db
+   virtual void DestroyDb() = 0;    // Delete the db
+
+// Recovery
+   virtual bool IsDamaged() = 0;   // Test if the database is damaged
+   virtual bool Recover()   = 0;   // Recover the db
+
+// Transactions
+   virtual void BeginTransaction()     = 0;   // Begin a transaction
+   virtual void CommitTRansaction()    = 0;   // Commit a transaction
+   virtual void RollbackTransaction()  = 0;   // Rollback a transaction
+   virtual bool InTransaction()        = 0;   // Test if transaction is pending
+
+// (FDT)
+   virtual void SetFdt(CFdt *pFdt) { fdt_ = pFdt; }
+   virtual CFdt* GetFdt() { return fdt_; }
+
+//  worksheets (WORKSHEETS). 
+// (used to create/update the master records of the db)
+   virtual void SetWorksheetAt(int i, CWorksheet* pWS) { apWS_.insert(apWS_.begin()+i, pWS); }
+   virtual CWorksheet* GetWorksheetAt(int i)           { return apWS_[i]; }
+   virtual void RemoveWorksheetAt(int i)               { apWS_.erase(apWS_.begin()+i); }
+
+// display formats (DISPLAY FORMATS).
+// (used for specifying the formatting requirements for either on-line
+// display of records during searching or for the generation of printed
+// output documents.
+   virtual void SetDisplayFormatAt(int i, CDisplayFormat* pDFMT) { apDFMT_.insert(apDFMT_.begin()+i, pDFMT); }
+   virtual CDisplayFormat* GetDisplayFormatAt(int i)             { return apDFMT_[i]; }
+   virtual void RemoveDisplayFormatAt(int i)                     { apDFMT_.erase(apDFMT_.begin()+i); }
+
+// Field Select Table (FST).
+// One FST defines the fields of the database to be made searchable
+// through the inverted dile.
+   virtual void SetFieldSelectTableAt(int i, CFieldSelectTable* pFST) { apFST_.insert(apFST_.begin()+i, pFST); }
+   virtual CFieldSelectTable* GetFieldSelectTableAt(int i)            { return apFST_[i]; }
+   virtual void RemoveFieldSelectTableAt(int i)                       { apFST_.erase(apFST_.begin()+i); }
+
+   virtual void CreateIndex(int i) = 0;
+   virtual void DropIndex(int i)   = 0;
+
+// Sort Select Table (SST).
+// One FST defines the fields of the database to be made searchable
+// through the inverted dile.
+
+   virtual void SetSortTableAt(int i, CSortTable* pST) { apST_.insert(apST_.begin()+i, pST); }
+   virtual CSortTable* GetSortTableAt(int i)           { return apST_[i]; }
+   virtual void RemoveSortTableAt(int i)               { apST_.erase(apST_.begin()+i); }
+
+   virtual void  CreateDbRecord(MfRecord &m) = 0;
+   virtual void  UpdateDbRecord(MfRecord &m) = 0;
+   
+   virtual int   ReadDbRecord(long mfn, MfRecord &m) = 0;
+   virtual int   ReadNextDbRecord(MfRecord &m)       = 0;
+   
+   virtual bool  DeleteRecord(long mfn)              = 0;
+
+   virtual void  PrintDbRecord(MfRecord &m)          = 0;
+   virtual void  PrintDbHdr()                        = 0;
+
+   virtual long GetNextMfn() = 0;
+   virtual long GetNextActiveMfn(long mfn=-1) = 0;
+
+   virtual SMfHeader& GetSMfHeader() = 0;
+   virtual _TCHAR *GetIdxFileName()  = 0;
+ 
+
+};
+ 
+class AbstractIsisDbFactory
+{
+public:
+	virtual AbstractIsisDb* MakeIsisDb() = 0;
+};
+#endif   // __ABSTRACT_ISISDB_H__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/BlkFile.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/BlkFile.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/BlkFile.h	(revision 21112)
@@ -0,0 +1,155 @@
+/**********************************************************************
+ *
+ * BlkFile.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+//////////////////////////////////////////////////////////////////////
+//  BLKFILE.H
+//
+//  Class for accessing a file per blocks of blksize bytes, blocks
+//  are sequential from 1 to nblock.
+//                                                                       
+//////////////////////////////////////////////////////////////////////
+ 
+#ifndef _BLKFILE_H_
+#define _BLKFILE_H_
+#include "CacheMan.h"
+#include "File.h"
+ 
+//////////////////////////
+// Block I/O file class
+//
+ 
+ 
+ 
+class BlkFile : public CFileBase
+{
+public:
+   enum { blk_size = 512 };
+protected:
+   unsigned     blkSize_;      // Block Size
+   CacheManager cache_;
+public:
+   BlkFile(int bs=BlkFile::blk_size);
+   BlkFile(const _TCHAR *fname, int bs=BlkFile::blk_size);
+   virtual FileError Close();
+   virtual ~BlkFile();
+   virtual void ReadBlk(void* d, long b);
+   virtual void WriteBlk(void* d, long b);
+   virtual void ReadBlk(void* d);
+   virtual void WriteBlk(void* d);
+
+   void SetBlockSize(int blockSize) { blkSize_ = blockSize; }
+   int  GetBlockSize() { return blkSize_; }
+};
+ 
+////////////////////////////////////////
+ 
+inline BlkFile::BlkFile(int bs) :
+// Makes a Blocked File object that isn't connected to a file
+// Block size is bs
+CFileBase(), blkSize_(bs)
+{
+	cache_.Create(bs, 40);
+	cache_.Connect(this);
+   // Nothing more to do
+}
+ 
+ 
+////////////////////////////////////////
+ 
+inline BlkFile::BlkFile(const _TCHAR *fname, int bs) :
+// Makes a Blocked File object connected to a file
+// Block size is bs
+CFileBase(), blkSize_(bs), cache_(bs, 20)
+{
+	CFileBase::Open(fname, FileSystem::FILE_READWRITE);
+	cache_.Create(bs, 40);
+	cache_.Connect(this);
+   // Nothing more to do
+}
+ 
+////////////////////////////////////////
+ 
+inline BlkFile::~BlkFile()
+// Destructor  for BlkFile object
+{
+	cache_.Disconnect();
+   // Nothing more to do
+}
+ 
+////////////////////////////////////////////
+
+
+inline FileError BlkFile::Close()
+{
+	if (IsOpen())
+	    cache_.Invalidate();
+	return CFileBase::Close();
+}
+////////////////////////////////////////
+ 
+inline void BlkFile::ReadBlk(void* d, long b)
+// Reads block number b into d, b is in range [1:NBLOCK]
+{
+	ASSERT(d!=NULL && b>0);
+	//TRACE("\nBlkFile::ReadBlk -- %s Read Block %ld", file_name, b);
+    //CFileBase::Fetch(d, blkSize_, (b-1)*blkSize_);
+	cache_.Read((b-1)*blkSize_, d);
+}
+ 
+////////////////////////////////////////
+ 
+inline void BlkFile::WriteBlk(void* d, long b)
+// Writes blksize bytes from d at block number b
+{
+	ASSERT(d!=NULL && b>0);
+	//TRACE("\nBlkFile::WriteBlk -- %s Write Block %ld", file_name, b);
+    //CFileBase::Store(d, blkSize_, (b-1)*blkSize_);
+	cache_.Write((b-1)*blkSize_, d);
+}
+ 
+ 
+////////////////////////////////////////
+ 
+inline void BlkFile::ReadBlk(void* d)
+// Reads block at current address into d
+{
+	ASSERT(d!=NULL);
+    //CFileBase::Fetch(d, blkSize_);
+	long addr = CFileBase::FilePosition();
+	cache_.Read(addr, d);
+}
+ 
+////////////////////////////////////////
+ 
+inline void BlkFile::WriteBlk(void* d)
+// Writes blkSize_ bytes from d at current address
+{
+	ASSERT(d!=NULL);
+    //CFileBase::Store(d, blkSize_);
+	long addr = CFileBase::FilePosition();
+	cache_.Write(addr, d);
+}
+ 
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.cpp	(revision 21112)
@@ -0,0 +1,170 @@
+/**********************************************************************
+ *
+ * CRC32.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+// CRC32.cpp --
+//  
+// The CRC32 functions (Cyclic Redundancy Check) are used to
+// calculate a sophisticated checksum based on the algebra of
+// polynomials. The Cyclic Redundancy Check, is a way to detect
+// bit errors that occur during data storage or transmission.
+// The CRC-32 algorithm operates on a block of data as a single
+// large numerical value. The algorithm divides this large value
+// by the CRC-32 polynomial or generator polynomial, leaving the
+// remainder 32-bit, which is the checksum. 
+//
+// #include "stdafx.h" 
+#include <iomanip>
+#include "CRC32.h"
+#include "CRCTab.h"
+
+unsigned long CalcCRC32(const char *buf, unsigned len)
+// Calculate a 32-bit CRC for a raw pattern of bytes.
+// Returns a 32-bit checksum.
+{
+ unsigned long CRC=0xffffffffL;
+ unsigned int n = len;
+
+ while(n--) 
+   CRC=crc32tab[(CRC ^ (*buf++)) & 0xFF] ^ ((CRC>>8) & 0x00ffffffL);
+
+ return CRC ^ 0xffffffffL;
+}
+
+unsigned long CalcCRC32(char *buf, unsigned len)
+// Calculate a 32-bit CRC for a raw pattern of bytes.
+// Returns a 32-bit checksum.
+{
+ unsigned long CRC=0xffffffffL;
+ unsigned int n = len;
+ char *p = (char *)buf;
+ while(n--) 
+   CRC=crc32tab[(CRC ^ (*p++)) & 0xFF] ^ ((CRC>>8) & 0x00ffffffL);
+
+ return CRC ^ 0xffffffffL;
+}
+
+unsigned long CalcCRC32(unsigned char c, unsigned long CRC)
+// Calculate a 32-bit CRC table value for a single byte.
+// NOTE: The initial CRC value must be set to 0xffffffffL
+// and the final 32-bit value that must be XOR'ed with
+// 0xffffffffL to obtain the checksum.
+{
+  unsigned int i;
+  i = (unsigned int)c;
+  i &= 0xFF; // Reset all the bits
+  CRC=crc32tab[(CRC ^ i) & 0xFF] ^ ((CRC>>8) & 0x00ffffffL);
+  return CRC;
+}
+
+unsigned long CalcCRC32(std::fstream &infile)
+// Calculate a 32-bit CRC for a file. Assumes the stream
+// is already open. Returns a 32-bit checksum.
+{
+ unsigned long CRC=0xffffffffL;
+#ifdef _WIN32
+ char c;
+#else
+ unsigned char c;
+#endif
+ unsigned int i;
+
+ // Rewind to the start of the stream
+ infile.clear(); 
+ infile.seekg(0, std::ios::beg);
+ infile.seekp(0, std::ios::beg);
+
+ while(!infile.eof()) {
+   c=infile.get();
+   i = (unsigned int)c;
+   i &= 0xFF; // Reset all the bits
+   if(infile.eof()) break;
+   CRC=crc32tab[(CRC ^ i) & 0xFF] ^ ((CRC>>8) & 0x00ffffffL);
+ }
+ return CRC ^ 0xffffffffL;
+}
+
+#ifdef __USE_CRC32_TABLE_FUNCTIONS__
+int MakeCRC32(ostream &stream)
+// Write a CRC 32 table for a byte-wise 32-bit CRC calculation
+// based on the Autodin/Ethernet/ADCCP polynomial of 0x4C11DB7:
+// 0000 0100 1100 0001 0001 1101 1011 0111 (binary) or a poly of
+// x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7 +x^5+x^4+x^2+x^1+x^0
+// In this representation the coefficient of x^0 is stored in the 
+// MSB of the 32-bit word and the coefficient of x^31 is stored in 
+// the LSB. Thus 0x4C11DB7 becomes 0xEDB88320:
+// 1110 1101 1011 1000 1000 0011 0010 0000 (binary)
+
+// Adding the polynomials is performed using an exclusive-or, and
+// multiplying a polynomial by x is a right shift by one. If the 
+// polynomial is called "p", and each byte is represented as polynomial
+// "q", with the lowest power in the most significant bit (so the byte
+// 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) 
+// mod "p", where "a" mod "b" means the remainder after dividing "a" by
+// "b". This calculation is done using the shift-register method of
+// multiplying and taking the remainder. The register is initialized
+// to zero, and for each incoming bit, x^32 is added mod "p" to the
+// register if the bit is a one (where x^32 mod p is p+x^32 = x^26+...+1),
+// and the register is multiplied mod "p" by "x" (which is shifting right
+// by one and adding x^32 mod "p" if the bit shifted out is a one.) This
+// algorithm starts with the highest power (least significant bit) of "q"
+// and repeats for all eight bits of "q".
+{
+  unsigned long c; // CRC shift register 
+  unsigned long e; // Polynomial exclusive-or pattern 
+  int i;           // Counter for all possible eight bit values 
+  int k;           // Byte being shifted into crc apparatus 
+
+  // Terms of polynomial defining this crc (except x^32): 
+  static const int p[14] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26};
+
+  // Make exclusive-or pattern from polynomial 
+  e = 0;
+  for (i = 0; i < sizeof(p)/sizeof(int); i++)
+    e |= 1L << (31 - p[i]);
+
+  // Compute and print table of CRC's, five per line 
+  stream << endl;
+  stream << "  0x00000000U";
+  for (i = 1; i < 256; i++)
+  {
+    c = 0;
+    for (k = i | 256; k != 1; k >>= 1)
+    {
+      c = c & 1 ? (c >> 1) ^ e : c >> 1;
+      if (k & 1)
+        c ^= e;
+    }
+
+    if(i % 5)
+      stream << ", 0x" << setfill('0') << setw(8) << hex << c << "U";
+    else
+      stream << "," << endl << "  0x" << setfill('0') << setw(8) << hex
+	     << c << "U";
+  }
+  stream << endl;
+  return 1;
+}
+#endif // __USE_CRC32_TABLE_FUNCTIONS__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/CRC32.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**********************************************************************
+ *
+ * CRC32.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+// CRC32.H
+//
+// The CRC32 functions (Cyclic Redundancy Check) are used to
+// calculate a sophisticated checksum based on the algebra of
+// polynomials. The Cyclic Redundancy Check, is a way to detect
+// bit errors that occur during data storage or transmission.
+// The CRC-32 algorithm operates on a block of data as a single
+// large numerical value. The algorithm divides this large value
+// by the CRC-32 polynomial or generator polynomial, leaving the
+// remainder 32-bit, which is the checksum. 
+#ifndef __CRC32_H__
+#define __CRC32_H__
+
+#include <fstream>
+
+unsigned long CalcCRC32(char *buf, unsigned len);
+unsigned long CalcCRC32(const char *buf, unsigned len);
+unsigned long CalcCRC32(unsigned char c, unsigned long CRC);
+unsigned long CalcCRC32(std::fstream &infile);
+
+#ifdef __USE_CRC32_TABLE_FUNCTIONS__
+#include <iostream>
+int makeCRC32(std::ostream &stream);
+#endif
+
+#endif  // __CRC32_H__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/CRCTab.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/CRCTab.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/CRCTab.h	(revision 21112)
@@ -0,0 +1,100 @@
+/**********************************************************************
+ *
+ * CRCTab.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+// CRCTAB.H --
+
+// CRC 32 table of all possible single-byte values. The table is
+// the CRC of all possible eight bit values. This is all the info
+// needed to generate 32-bit CRC's on data one a byte at a time 
+// for all combinations of CRC register values and incoming bytes.   
+// This table was generated with the makeCRC32() function. It is
+// based on the Autodin/Ethernet/ADCCP polynomial of 0x4C11DB7:
+// 0000 0100 1100 0001 0001 1101 1011 0111 (binary) or a poly of
+// x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7 +x^5+x^4+x^2+x^1+x^0
+// In this representation the coefficient of x^0 is stored in the 
+// MSB of the 32-bit word and the coefficient of x^31 is stored in 
+// the LSB. Thus 0x4C11DB7 becomes 0xEDB88320:
+// 1110 1101 1011 1000 1000 0011 0010 0000 (binary)
+
+#ifndef __CRC32_TABLE_HPP__
+#define __CRC32_TABLE_HPP__
+
+const unsigned long crc32tab[256] = {
+  0x00000000U, 0x77073096U, 0xee0e612cU, 0x990951baU, 0x076dc419U,
+  0x706af48fU, 0xe963a535U, 0x9e6495a3U, 0x0edb8832U, 0x79dcb8a4U,
+  0xe0d5e91eU, 0x97d2d988U, 0x09b64c2bU, 0x7eb17cbdU, 0xe7b82d07U,
+  0x90bf1d91U, 0x1db71064U, 0x6ab020f2U, 0xf3b97148U, 0x84be41deU,
+  0x1adad47dU, 0x6ddde4ebU, 0xf4d4b551U, 0x83d385c7U, 0x136c9856U,
+  0x646ba8c0U, 0xfd62f97aU, 0x8a65c9ecU, 0x14015c4fU, 0x63066cd9U,
+  0xfa0f3d63U, 0x8d080df5U, 0x3b6e20c8U, 0x4c69105eU, 0xd56041e4U,
+  0xa2677172U, 0x3c03e4d1U, 0x4b04d447U, 0xd20d85fdU, 0xa50ab56bU,
+  0x35b5a8faU, 0x42b2986cU, 0xdbbbc9d6U, 0xacbcf940U, 0x32d86ce3U,
+  0x45df5c75U, 0xdcd60dcfU, 0xabd13d59U, 0x26d930acU, 0x51de003aU,
+  0xc8d75180U, 0xbfd06116U, 0x21b4f4b5U, 0x56b3c423U, 0xcfba9599U,
+  0xb8bda50fU, 0x2802b89eU, 0x5f058808U, 0xc60cd9b2U, 0xb10be924U,
+  0x2f6f7c87U, 0x58684c11U, 0xc1611dabU, 0xb6662d3dU, 0x76dc4190U,
+  0x01db7106U, 0x98d220bcU, 0xefd5102aU, 0x71b18589U, 0x06b6b51fU,
+  0x9fbfe4a5U, 0xe8b8d433U, 0x7807c9a2U, 0x0f00f934U, 0x9609a88eU,
+  0xe10e9818U, 0x7f6a0dbbU, 0x086d3d2dU, 0x91646c97U, 0xe6635c01U,
+  0x6b6b51f4U, 0x1c6c6162U, 0x856530d8U, 0xf262004eU, 0x6c0695edU,
+  0x1b01a57bU, 0x8208f4c1U, 0xf50fc457U, 0x65b0d9c6U, 0x12b7e950U,
+  0x8bbeb8eaU, 0xfcb9887cU, 0x62dd1ddfU, 0x15da2d49U, 0x8cd37cf3U,
+  0xfbd44c65U, 0x4db26158U, 0x3ab551ceU, 0xa3bc0074U, 0xd4bb30e2U,
+  0x4adfa541U, 0x3dd895d7U, 0xa4d1c46dU, 0xd3d6f4fbU, 0x4369e96aU,
+  0x346ed9fcU, 0xad678846U, 0xda60b8d0U, 0x44042d73U, 0x33031de5U,
+  0xaa0a4c5fU, 0xdd0d7cc9U, 0x5005713cU, 0x270241aaU, 0xbe0b1010U,
+  0xc90c2086U, 0x5768b525U, 0x206f85b3U, 0xb966d409U, 0xce61e49fU,
+  0x5edef90eU, 0x29d9c998U, 0xb0d09822U, 0xc7d7a8b4U, 0x59b33d17U,
+  0x2eb40d81U, 0xb7bd5c3bU, 0xc0ba6cadU, 0xedb88320U, 0x9abfb3b6U,
+  0x03b6e20cU, 0x74b1d29aU, 0xead54739U, 0x9dd277afU, 0x04db2615U,
+  0x73dc1683U, 0xe3630b12U, 0x94643b84U, 0x0d6d6a3eU, 0x7a6a5aa8U,
+  0xe40ecf0bU, 0x9309ff9dU, 0x0a00ae27U, 0x7d079eb1U, 0xf00f9344U,
+  0x8708a3d2U, 0x1e01f268U, 0x6906c2feU, 0xf762575dU, 0x806567cbU,
+  0x196c3671U, 0x6e6b06e7U, 0xfed41b76U, 0x89d32be0U, 0x10da7a5aU,
+  0x67dd4accU, 0xf9b9df6fU, 0x8ebeeff9U, 0x17b7be43U, 0x60b08ed5U,
+  0xd6d6a3e8U, 0xa1d1937eU, 0x38d8c2c4U, 0x4fdff252U, 0xd1bb67f1U,
+  0xa6bc5767U, 0x3fb506ddU, 0x48b2364bU, 0xd80d2bdaU, 0xaf0a1b4cU,
+  0x36034af6U, 0x41047a60U, 0xdf60efc3U, 0xa867df55U, 0x316e8eefU,
+  0x4669be79U, 0xcb61b38cU, 0xbc66831aU, 0x256fd2a0U, 0x5268e236U,
+  0xcc0c7795U, 0xbb0b4703U, 0x220216b9U, 0x5505262fU, 0xc5ba3bbeU,
+  0xb2bd0b28U, 0x2bb45a92U, 0x5cb36a04U, 0xc2d7ffa7U, 0xb5d0cf31U,
+  0x2cd99e8bU, 0x5bdeae1dU, 0x9b64c2b0U, 0xec63f226U, 0x756aa39cU,
+  0x026d930aU, 0x9c0906a9U, 0xeb0e363fU, 0x72076785U, 0x05005713U,
+  0x95bf4a82U, 0xe2b87a14U, 0x7bb12baeU, 0x0cb61b38U, 0x92d28e9bU,
+  0xe5d5be0dU, 0x7cdcefb7U, 0x0bdbdf21U, 0x86d3d2d4U, 0xf1d4e242U,
+  0x68ddb3f8U, 0x1fda836eU, 0x81be16cdU, 0xf6b9265bU, 0x6fb077e1U,
+  0x18b74777U, 0x88085ae6U, 0xff0f6a70U, 0x66063bcaU, 0x11010b5cU,
+  0x8f659effU, 0xf862ae69U, 0x616bffd3U, 0x166ccf45U, 0xa00ae278U,
+  0xd70dd2eeU, 0x4e048354U, 0x3903b3c2U, 0xa7672661U, 0xd06016f7U,
+  0x4969474dU, 0x3e6e77dbU, 0xaed16a4aU, 0xd9d65adcU, 0x40df0b66U,
+  0x37d83bf0U, 0xa9bcae53U, 0xdebb9ec5U, 0x47b2cf7fU, 0x30b5ffe9U,
+  0xbdbdf21cU, 0xcabac28aU, 0x53b39330U, 0x24b4a3a6U, 0xbad03605U,
+  0xcdd70693U, 0x54de5729U, 0x23d967bfU, 0xb3667a2eU, 0xc4614ab8U,
+  0x5d681b02U, 0x2a6f2b94U, 0xb40bbe37U, 0xc30c8ea1U, 0x5a05df1bU,
+  0x2d02ef8dU
+};
+
+#endif // __CRC32_TABLE_HPP__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.cpp	(revision 21112)
@@ -0,0 +1,213 @@
+/**********************************************************************
+ *
+ * CacheMan.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#include "stdafx.h"
+#include "File.h"
+
+#include "CacheMan.h"
+
+
+CacheManager::CacheManager()
+{
+	maxblocks_ = 0;
+}
+CacheManager::CacheManager(unsigned blocksz, unsigned mxblks)
+{
+	PRECONDITION(maxblocks_ == 0);
+	maxblocks_ = mxblks;
+	nused_     = 0;
+	blocksize_ = blocksz;
+	buff_      = new char[blocksize_*maxblocks_];
+	diskAddrs_ = new FileSystem::FAU_t[maxblocks_];
+	useCounts_ = new unsigned[maxblocks_];
+	theFile_   = NULL;
+}
+/*
+ * Construct a cache manager for blocks of size blocksz and mxblks buffers
+ */
+void CacheManager::Create(unsigned blocksz, unsigned mxblks)
+{
+	PRECONDITION(maxblocks_ == 0);
+	maxblocks_ = mxblks;
+	nused_     = 0;
+	blocksize_ = blocksz;
+	buff_      = new char[blocksize_*maxblocks_];
+	diskAddrs_ = new FileSystem::FAU_t[maxblocks_];
+	useCounts_ = new unsigned[maxblocks_];
+	theFile_   = NULL;
+}
+
+//
+// Connect to a CFileBase
+//
+void CacheManager::Connect(CFileBase* file)
+{
+	PRECONDITION(nused_ == 0);
+	theFile_   = file;
+}
+//
+// Disconnect the cache from the CFileBase
+//
+void CacheManager::Disconnect()
+{
+	if (theFile_)
+	{
+		if (theFile_->IsOpen())
+		    Flush();
+	}
+	theFile_   = NULL;
+}
+
+void CacheManager::Erase()
+{
+	Disconnect();
+	delete[] useCounts_;
+	delete[] diskAddrs_;
+	delete[] buff_;
+	maxblocks_ = 0;
+}
+
+
+CacheManager::~CacheManager()
+{
+	Disconnect();
+	delete[] useCounts_;
+	delete[] diskAddrs_;
+	delete[] buff_;
+}
+
+bool CacheManager::Flush()
+{
+	PRECONDITION(theFile_ != NULL && theFile_->IsOpen());
+	// Because we use "write through", this is all that's necessary:
+	if (theFile_->Flush() == FILE_NO_ERROR)
+		return true;
+	return false;
+}
+
+void CacheManager::Invalidate()
+{
+	nused_ = 0;
+}
+
+bool CacheManager::Read(FileSystem::FAU_t locn, void* dat)
+{
+	PRECONDITION(dat != NULL);
+	size_t islot = AgeAndFindSlot(locn);
+	
+	if (islot == INVALID_NPOS)
+	{
+        // Not in buffer;  we'll have to read it in from disk.
+	    // Get a free slot.
+		if ((islot = GetFreeSlot()) == INVALID_NPOS)
+			return false;
+		diskAddrs_[islot] = locn;
+		if (!(theFile_->SeekTo(locn) == locn) || 
+			!(theFile_->Fetch(buff_ + islot*blocksize_, blocksize_) == FILE_NO_ERROR))
+			return false;
+	}
+	useCounts_[islot] = 0;
+	memcpy(dat, buff_ + islot*blocksize_, blocksize_);
+	return true;
+}
+
+bool CacheManager::Write(FileSystem::FAU_t locn, void* dat)
+{
+	PRECONDITION(dat != NULL);
+	PRECONDITION(theFile_->ReadyForWriting());
+	size_t islot = AgeAndFindSlot(locn);
+	
+	if (islot == INVALID_NPOS)
+	{
+        // Not in buffer; find a free slot.
+		if ((islot = GetFreeSlot()) == INVALID_NPOS)
+			return false;
+		diskAddrs_[islot] = locn;
+	}
+	
+	useCounts_[islot] = 0;
+	memcpy(buff_ + islot*blocksize_, dat, blocksize_);
+	if ((theFile_->SeekTo(locn) == locn) &&
+		(theFile_->Store(buff_ + islot*blocksize_, blocksize_) == FILE_NO_ERROR))
+		return true;
+	return false;
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+// Private helper functions	
+
+size_t CacheManager::AgeAndFindSlot(FileSystem::FAU_t locn)
+{
+	size_t islot = INVALID_NPOS;
+	for (register unsigned i = 0; i < nused_; i++)
+	{
+		if (diskAddrs_[i] == locn)
+			islot = i;
+		useCounts_[i]++;		// Age the blocks
+	}
+	return islot;
+}
+
+bool CacheManager::Flush(unsigned)
+{
+	if (theFile_->Flush() == FILE_NO_ERROR)
+		return true;
+	return false;
+}
+
+size_t CacheManager::GetFreeSlot()
+{
+	size_t islot;
+	
+	if (nused_ < maxblocks_)
+	{
+		islot = nused_++;		// Found an unused slot.
+	}
+	else 
+	{
+		// No free slots; get the Least Recently Used block
+		islot = LRU();
+
+		if (theFile_->ReadyForWriting())
+			if (!Flush(islot))
+			    islot = INVALID_NPOS;
+	}
+	return islot;
+}
+
+size_t CacheManager::LRU() const
+{
+	size_t islot = 0;
+	unsigned maxCount = useCounts_[0];
+	for (register unsigned i = 1; i < nused_; i++)
+	{
+		if (useCounts_[i] > maxCount)
+		{
+			maxCount = useCounts_[islot = i]; 
+		}
+	}
+	return islot;
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/CacheMan.h	(revision 21112)
@@ -0,0 +1,71 @@
+/**********************************************************************
+ *
+ * CacheMan.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef __CACHEMAN_H__
+#define __CACHEMAN_H__
+
+
+#include "File.h"
+
+class CacheManager
+{
+private:
+	
+	CFileBase*	       theFile_;
+	unsigned		   maxblocks_;  // Max # blocks in buff_er
+	unsigned		   nused_;	    // Number being used.
+	unsigned		   blocksize_;  // Size of a block
+	FileSystem::FAU_t* diskAddrs_;  // Its disk address
+	unsigned*		   useCounts_;  // Use count for the block
+	char*			   buff_;	    // The set of blocks.
+	
+public:
+	
+	CacheManager();
+    CacheManager::CacheManager(unsigned blocksz, unsigned mxblks);
+
+	~CacheManager();
+	
+	void        Create(unsigned blocksz, unsigned mxblks = 10);
+	void        Erase();
+	
+	void        Connect(CFileBase* file);
+	void        Disconnect();
+	bool  		Flush();	        // Perform any pending writes.
+	void		Invalidate();	    // Invalidate the entire cache
+	bool  		Read (FileSystem::FAU_t locn, void* dat);
+	bool  		Write(FileSystem::FAU_t locn, void* dat);
+	
+private:
+	
+	CacheManager(const CacheManager&);          // Private to insure no copies
+	void		operator=(const CacheManager&); // Ditto
+	size_t		AgeAndFindSlot(FileSystem::FAU_t);
+	bool  		Flush(unsigned);
+	size_t		GetFreeSlot();
+	size_t		LRU() const; // Find least recently used block
+};
+
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Checking.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Checking.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Checking.cpp	(revision 21112)
@@ -0,0 +1,65 @@
+/**********************************************************************
+ *
+ * Checking.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////////////////
+// Checking -- 
+//               
+
+
+#include "stdafx.h"
+#include "File.h"
+#include "Master.h"
+#include "MFFile.h"
+#include "Checking.h"
+
+using namespace Checking;
+//---------------------------------------------------------------------
+// Checking::CheckMfHeader(const char *mfname, SMfHeader *mfh)
+//
+// Invariant to be respected by the mf header
+//---------------------------------------------------------------------
+void Checking::CheckMfHeader(const _TCHAR *mfname, SMfHeader &mfh)
+{
+	long size = FileSystem::FileSize(mfname);
+	TRACE(_T("\nsize=%ld "), size);
+	long mfSize = mfh.nxtmfb_ * MFBLKSIZE;
+	//ASSERT(size == mfSize);
+	ASSERT(mfh.nxtmfp_>0 && mfh.nxtmfp_<=MFBLKSIZE);
+	ASSERT(mfh.mftype_ == USERMST);
+	ASSERT(mfh.nxtmfn_>0);
+	ASSERT(mfh.ctlmfn_ == 0);
+	//assert(mfh.reccnt == 0);
+	//assert(mfh.mfcxx1 == 0);
+	//assert(mfh.mfcxx2 == 0);
+	//assert(mfh.mfcxx3 == 0);
+}
+
+void Checking::CheckMfRecordHeader(long addr)
+{
+
+}
+
+using namespace std;
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Checking.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Checking.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Checking.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**********************************************************************
+ *
+ * Checking.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/////////////////////////////////////////////////////////////////////////////////
+// Checking.h --
+
+
+namespace Checking {
+   void CheckMfHeader(const _TCHAR *mfname, SMfHeader &mfh);
+   void CheckMfRecordHeader(long addr);
+
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Compat.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Compat.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Compat.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**********************************************************************
+ *
+ * Compat.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef _WIN32
+
+#include <string>
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#define TCHAR  char
+#define BOOL   bool
+#define WORD   unsigned short
+#define DWORD  unsigned long
+#define INT32  int
+#define UINT32 unsigned int
+#define LONG   long int
+#define TRUE   true
+#define FALSE  false
+
+#define _MAX_PATH 512
+// #define SHRT_MAX  32732
+
+/* Formatted i/o */
+
+#define _tprintf    printf
+#define _ftprintf   fprintf
+#define _stprintf   sprintf
+#define _sntprintf  _snprintf
+#define _vtprintf   vprintf
+#define _vftprintf  vfprintf
+#define _vstprintf  vsprintf
+#define _vsntprintf _vsnprintf
+#define _tscanf     scanf
+#define _ftscanf    fscanf
+#define _stscanf    sscanf
+/* String conversion functions */
+
+#define _tcstod     strtod
+#define _tcstol     strtol
+#define _tcstoul    strtoul
+
+#define _itot       _itoa
+#define _ltot       _ltoa
+#define _ultot      _ultoa
+#define _ttoi       atoi
+#define _ttol       atol
+
+#define _ttoi64     _atoi64
+#define _i64tot     _i64toa
+#define _ui64tot    _ui64toa
+/* String functions */
+
+/* Note that _mbscat, _mbscpy and _mbsdup are functionally equivalent to 
+   strcat, strcpy and strdup, respectively. */
+
+#define _tcscat     strcat
+#define _tcscpy     strcpy
+#define _tcsdup     _strdup
+
+#define _tcslen     strlen
+#define _tcsxfrm    strxfrm
+
+
+// #define assert
+#define ASSERT
+#define AssertFailed
+
+// #define _T(par) par
+#define LPCTSTR const char *
+#define LPTSTR char *
+#define FAR
+#define TRACE
+#define PRECONDITION
+#define SIZE_T size_t
+
+
+
+#endif
+#define MoveMemory memmove
+#define ZeroMemory(p,size) memset(p,0,size)
Index: /main/tags/3.01/gsdl/packages/isis-gdl/DLLCode.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/DLLCode.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/DLLCode.h	(revision 21112)
@@ -0,0 +1,74 @@
+/**********************************************************************
+ *
+ * DLLCode.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/////////////////////////////////////////////////////////////////////////////
+// DLLCODE.H - include file used by all platforms to define macros required 
+//             to build static and dynamic libraries.
+//             The DLLCODE.H file is included in every include file extending
+//             the visibility of the macros defined here to the every source
+//             code file. 
+
+
+#ifndef __DLL_CODE_H__
+#define __DLL_CODE_H__
+
+#if defined (__MSVC_DLL__) // Defines the entry point for the DLL application
+//--------------------------------------------------------------------------
+// The following ifdef block is the standard way of creating macros which 
+// make exporting from a DLL simpler. All files within this DLL are compiled 
+// with the DLL_CODE_EXPORTS  symbol defined on the command line. This 
+// symbol should not be defined on any project that uses this DLL. This way 
+// any other project whose source files include this file see DLL_CODE_API 
+// functions as being imported from a DLL, whereas this DLL sees symbols
+// defined with this macro as being exported.
+//---------------------------------------------------------------------------
+#ifdef DLL_CODE_EXPORTS
+#define DLL_CODE_API __declspec(dllexport)
+#else
+#define DLL_CODE_API __declspec(dllimport)
+#endif
+#else // Macro used for non DLL code
+#define DLL_CODE_API
+#endif // __MSVC_DLL__
+
+// Force ANSI compatibility for Visual C++ .NET
+#if defined (__MSDEVNET__) && !defined (__USE_ANSI_CPP__)
+#define __USE_ANSI_CPP__
+#endif // __MSDEVNET__
+
+// Force ANSI compatibility for BCC32
+#if defined (__BCC32__) && !defined (__USE_ANSI_CPP__)
+#define __USE_ANSI_CPP__
+#endif // __BCC32__
+
+#if defined (__USE_ANSI_CPP__)
+// Macro used to define the std name space used by the ANSI 
+// Standard C++ library.
+#define GXSTD std
+#else // Use the old Standard C++ library by default  
+#define GXSTD
+#endif // __USE_ANSI_CPP__
+
+#endif // __DLL_CODE_H__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Debug.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Debug.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Debug.cpp	(revision 21112)
@@ -0,0 +1,213 @@
+#include "stdafx.h"
+#include "Debug.h"
+
+//---------------------------------------------------------------
+//
+// AssertFailed
+//
+//---------------------------------------------------------------
+#if DEBUG || ASSERTS_THROW
+void AssertFailed(const char* expr, const char* file, int line)
+{
+	/*
+	_TCHAR mesg[512];							
+
+	const char* fileName = file + strlen(file);		// strip off path
+	while (fileName > file && fileName[-1] != '\\')
+		--fileName;
+
+	_stprintf(mesg, _T("ASSERT(%s) in '%s' at line %d failed."), expr, fileName, line);
+	
+	MSG msg;							// remove pending quit messages so the message box displays
+	bool quitting = (bool) ::PeekMessage(&msg, NULL, WM_QUIT, WM_QUIT, PM_REMOVE);
+
+	UINT32 flags = MB_OK +				// just an OK button
+				   MB_ICONERROR +		// display the icon for errors
+				   MB_DEFBUTTON1 +		// select the OK button
+				   MB_TASKMODAL +		// don't let the user do anything else in the app
+				   MB_SETFOREGROUND;	// bring the app to the front
+
+	(void) MessageBox(NULL, mesg, NULL, flags);
+//#if DEBUG
+//	if (gBreakOnAssert)
+//		BreakStrToDebugger(mesg);
+//#endif
+
+//#if ASSERTS_THROW
+	CString text = L"Internal (Assert) Error: ";
+	AfxThrowUserException( ); 
+//#endif
+*/
+}
+#endif	// DEBUG || ASSERTS_THROW
+
+
+//---------------------------------------------------------------
+//
+// RequireFailed
+//
+//---------------------------------------------------------------
+#if !DEBUG
+void RequireFailed(const char* expr, const char* file, int line)
+{
+	/*
+	char mesg[512];							
+
+	const char* fileName = file + strlen(file);		// strip off path
+	while (fileName > file && fileName[-1] != '\\')
+		--fileName;
+
+	sprintf(mesg, "There has been a fatal error in '%s' at line %d.", fileName, line);
+
+	MSG msg;							// remove pending quit messages so the message box displays
+	bool quitting = (bool) ::PeekMessage(&msg, NULL, WM_QUIT, WM_QUIT, PM_REMOVE);
+
+	UINT32 flags = MB_OK +				// just an OK button
+				   MB_ICONERROR +		// display the icon for errors
+				   MB_DEFBUTTON1 +		// select the OK button
+				   MB_TASKMODAL +		// don't let the user do anything else in the app
+				   MB_SETFOREGROUND;	// bring the app to the front
+
+	(void) MessageBox(NULL, mesg, NULL, flags);
+
+	abort();
+	*/
+}
+#endif // !DEBUG
+
+//---------------------------------------------------------------
+//
+// ToStr (bool)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(bool value)
+{
+	return value ? L"true" : L"false";
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (char)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(char value)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%c", value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (int16, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(int16_t value, int32_t fieldWidth)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%*d", fieldWidth, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (uint16, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(uint16_t value, int32_t fieldWidth)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%*d", fieldWidth, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (int32, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(int32_t value, int32_t fieldWidth)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%*ld", fieldWidth, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (uint32, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(uint32_t value, int32_t fieldWidth)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%*lu", fieldWidth, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (float, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(float value, int32_t precision)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%.*f", precision, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (double, int32)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(double_t value, int32_t precision)
+{
+	wchar_t str[32];							
+	::swprintf(str, L"%.*f", precision, value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (const char*)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(const char* value)
+{
+	wchar_t str[2048];							
+	::swprintf(str, L"%s", value);
+	
+	return str;
+}
+
+
+//---------------------------------------------------------------
+//
+// ToStr (std::string)
+//
+//---------------------------------------------------------------
+std::wstring ToStr(const std::string& value)
+{
+	wchar_t str[2048];							
+	::swprintf(str, L"%s", value.c_str());
+	
+	return str;
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Debug.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Debug.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Debug.h	(revision 21112)
@@ -0,0 +1,161 @@
+#ifndef __DEBUG_H__
+#define __DEBUG_H__
+
+//	Abstract:	This file provides the following macros:
+//
+//				ASSERT(p) - If p is false and DEBUG is true drops into the 
+//				debugger, if ASSERTS_THROW is true throws a XAssertException exception.
+//
+//				VERIFY(p) - Like ASSERT except that p is evaluated in release builds.
+//
+//				REQUIRE(p) - Like VERIFY except that it exits the app if the test fails
+//				(this is needed because the boot strap code can't safely throw).
+//
+//				COMPILE_CHECK(p) - An ASSERT that fires at compile time. This is useful
+//				for doing things like verifying the layout of items within a struct.
+//				Note that this check is also done in release builds.
+//
+//				DEBUGSTR(formatStr, n1, n2, ...) - Drops into the debugger and
+//				displays a string.
+//
+//				TRACE(formatStr, n1, n2, ...) - Writes a string to a debug window
+//				without suspending the program.
+//
+//				TRACEFLOW(category, formatStr, n1, n2, ...) - Like TRACE, but with 
+//				the addition of a category string. The category string allows users 
+//				to turn off entire categories of messages. Note that DEBUGSTR, TRACE, 
+//				and TRACEFLOW may be used with boolean expressions by tacking an _IF 
+//				on the end (eg DEBUGSTR_IF(x < 0, "Sqrt was passed a negative argument.")).
+//
+//				PRECONDITION(p)  - Used in conjunction with POSTCONDITION to verify 
+//				the integrity of an object: each public method should include one call to 
+//				PRECONDITION and another to POSTCONDITION. The idea here is to check to 
+//				see if the object stays in a valid state given valid arguments. Note that 
+//				these macros call a method named Invariant. This method should use ASSERT's 
+//				to check to see if the object is in a sane state. See ZInvariant.h for
+//				more details.
+//
+//				CHECK_INVARIANT() - Calls the Invariant. This is useful in ctors because
+//				PRECONDITION doesn't call the dtor (PRECONDITION constructs an object on
+//				the stack which calls the Invariant when the function exits).
+
+#pragma once      // Open only once per build
+
+#define RUNTIME_EXPORT 
+
+RUNTIME_EXPORT std::wstring ToStr(bool value);				// I originally used iostreams, but that caused a ridiculous amount of bloat in object files (but much less so in the exe)
+RUNTIME_EXPORT std::wstring ToStr(char value);
+inline 		   std::wstring ToStr(wchar_t value)						{return std::wstring(1, value);}
+RUNTIME_EXPORT std::wstring ToStr(int16_t value, int32_t fieldWidth = 1);
+RUNTIME_EXPORT std::wstring ToStr(uint16_t value, int32_t fieldWidth = 1);
+RUNTIME_EXPORT std::wstring ToStr(int32_t value, int32_t fieldWidth = 1);
+RUNTIME_EXPORT std::wstring ToStr(uint32_t value, int32_t fieldWidth = 1);
+RUNTIME_EXPORT std::wstring ToStr(float value, int32_t precision = 6);
+RUNTIME_EXPORT std::wstring ToStr(double_t value, int32_t precision = 6);
+RUNTIME_EXPORT std::wstring ToStr(const char* value);
+inline 		   std::wstring ToStr(const wchar_t* value)					{return value;}
+RUNTIME_EXPORT std::wstring ToStr(const std::string& value);
+inline 		   std::wstring ToStr(const std::wstring& value)			{return value;}
+
+
+#ifdef _DEBUG
+	#define	DEBUG					1
+#else		
+	#define	DEBUG					0
+	
+	#if	!defined(NDEBUG)
+		#define	NDEBUG							// used by <assert.h>
+	#endif
+#endif
+
+#ifndef ASSERTS_THROW
+#define ASSERTS_THROW				0			
+#endif
+// #define ASSERTS_THROW               1
+// Synch with ANSI definitions.
+#if DEBUG && defined(NDEBUG)
+	#error DEBUG and NDEBUG are out of sync!
+#endif
+
+#if !DEBUG && !defined(NDEBUG)
+	#error DEBUG and NDEBUG are out of sync!
+#endif
+
+// Synch with MSVC.
+#if _MSC_VER && DEBUG != defined(_DEBUG)
+	#error DEBUG and _DEBUG are out of sync!
+#endif
+
+#ifndef _lint
+	#define COMPILE_CHECK(p)	{struct _CC {char a[(p) ? 1 : -1];};} (void) 0
+#else
+	#define COMPILE_CHECK(p)					
+#endif
+
+#if DEBUG
+	
+	void AssertFailed(const char* expr, const char* file, int line);
+	
+	#undef assert
+
+	#define ASSERT(p)					do {if (!(p)) AssertFailed(#p, __FILE__, __LINE__);} while (false)
+
+	#define ASSERT_IF(p, x)				do {if ((p) && !(x)) AssertFailed(#x, __FILE__, __LINE__);} while (false)
+	#define assert(p) 					ASSERT(p)
+	#define VERIFY(p)					ASSERT(p)
+	#define REQUIRE(p)					ASSERT(p)
+
+	#define	PRECONDITION(p)				ASSERT(p)
+	#define	POSTCONDITION(p)			ASSERT(p)
+
+    #ifdef NO_NESTED_QUOTES
+          #define PRECONDITION2(a,b)	ASSERT(a)
+          #define POSTCONDITION2(a,b)	ASSERT(a)
+    #else
+          #define PRECONDITION2(a,b)	ASSERT((b, (a) !=0))
+          #define POSTCONDITION2(a,b)	ASSERT((b, (a) !=0))
+    #endif
+
+#else	// #if DEBUG
+
+
+	#if ASSERTS_THROW
+		void AssertFailed(const char*, const char*, int);
+
+		#define ASSERT(p)				do {if (!(p)) AssertFailed(#p, __FILE__, __LINE__);} while (false)
+
+		#define ASSERT_IF(p, x)			do {if ((p) && !(x)) AssertFailed(#x, __FILE__, __LINE__);} while (false)
+		#define VERIFY(p)				ASSERT(p)
+
+		#define	PRECONDITION(p)			ASSERT(p)
+		#define	POSTCONDITION(p)		ASSERT(p)
+
+        #define PRECONDITION2(a,b)	    ASSERT(a)
+        #define POSTCONDITION2(a,b)	    ASSERT(a)
+	#else
+		#define	ASSERT(p)				((void) 0)
+
+		#define	ASSERT_IF(p, x)			((void) 0)
+		#define VERIFY(p)				do {if (p) 0;} while (false)
+
+		#define	PRECONDITION(p)			((void) 0)
+		#define	POSTCONDITION(p)		((void) 0)
+
+        #define PRECONDITION2(a,b)
+        #define POSTCONDITION2(a,b)
+	#endif
+
+	void RequireFailed(const char*, const char*, int);
+	
+	#define REQUIRE(p)					do {if (!(p)) RequireFailed(#p, __FILE__, __LINE__);} while (false)
+
+
+#endif	// DEBUG
+
+#ifdef _CONSOLE
+#define TRACE
+#endif
+	
+
+
+#endif /* __DEBUG_H__ */
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.cpp	(revision 21112)
@@ -0,0 +1,536 @@
+/**********************************************************************
+ *
+ * Fdt.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+///////////////////////////////////////////////////////////////////////////
+//  fdt.cpp
+
+#include "stdafx.h"
+#include "Fdt.h"
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <sstream>
+#include <stdio.h>
+
+//////////////////////////////
+ 
+CFdt::CFdt()   
+// Constructor
+{
+   InitData();
+   InitFields();
+}
+
+//////////////////////////////
+
+CFdt::~CFdt()
+{
+	for (int i = 0; i < m_nFieldCount; i++)
+	{
+#ifndef _WIN32
+		CFieldDef* p = (CFieldDef*) m_fieldArray[i];
+#else
+		CFieldDef* p = (CFieldDef*) m_fieldArray.at(i);
+#endif
+    	delete p;
+	}
+	m_fieldArray.clear();
+
+	Close();
+}
+
+////////////////////////////
+
+bool CFdt::Open(const _TCHAR *szFileName, bool readOnly)
+{
+	if (!m_bIsEmpty)
+		Close();
+	m_sFileName = szFileName;
+	if (!Load(szFileName))
+	{
+		Close();
+		return false;
+	}
+
+	m_bReadOnly = readOnly;
+	return Seek(0);   // 
+}
+
+
+
+
+//////////////////////////////
+
+void CFdt::InitData()
+{
+   m_iCurrentRecord = -1;
+   m_sFileName.erase();
+   
+   m_bReadOnly = false;
+   m_bIsEmpty  = true;
+   m_fdtEntry.Clear();
+   m_bDirty    = false;
+   m_aFdt.clear();
+   m_asPrelude.clear();
+   // Suppose the fdt is empty and we are 
+   // going to append 
+   m_bAppend = true;
+}
+
+//////////////////////////////
+
+void CFdt::Close()
+{
+	if (!m_bIsEmpty)
+		DeleteContents();
+
+    InitData();        // Reset data members   
+	m_bIsEmpty = true;
+}
+
+////////////////////////////////////
+ 
+void CFdt::DeleteContents()
+// Delete the fdt data structures
+{
+    m_aFdt.clear();
+  
+}
+
+
+
+bool CFdt::Load(const _TCHAR *dbaseName)
+{
+	std::ifstream in((const char*)dbaseName);
+	if (!in)
+    {
+//#ifdef _WIN32
+//	   CString strMsg;
+//		strMsg.Format(_T("Error opening %s for reading. "), dbaseName);
+//		AfxMessageBox(strMsg, MB_OK | MB_ICONEXCLAMATION);
+//#else
+		std::cout << _T("Error opening") << dbaseName << _T("for reading") << std::endl;
+//#endif
+		return false;
+    }
+
+
+	std::string s;
+
+    while (std::getline(in,s) && s.substr(0,3) != "***")
+    {
+		m_asPrelude.push_back(s);
+    }
+
+    while (std::getline(in,s))
+    {
+        CFdtEntry e;
+        strncpy(e.name, s.c_str(), FDT_NAME_LENG);
+	    e.name[FDT_NAME_LENG] = _T('\0');
+	    for (int i=FDT_NAME_LENG-1; e.name[i]==_T(' '); e.name[i--]=_T('\0'))
+		   ;
+        if (s[FDT_NAME_LENG] == _T(' '))        //no subfield delimiters spec.
+		 	e.subfields[0] = _T('\0');
+	    else
+		{
+		 	strncpy(e.subfields, s.c_str()+FDT_NAME_LENG, FDT_SFLD_LENG);
+			e.subfields[FDT_SFLD_LENG] = _T('\0');
+			for (int i= FDT_SFLD_LENG-1; e.subfields[i]==_T(' '); e.subfields[i--]='\0')
+				 ;
+		}
+	    sscanf(s.c_str() + FDT_NAME_LENG + FDT_SFLD_LENG, "%d %d %d %d",
+										  &e.tag, &e.len, &e.type, &e.rep);
+	    m_aFdt.push_back(e);
+    }
+    in.close();
+    return true;
+}
+
+		
+		
+		
+
+
+// Define the fields used for the dictionary
+
+static CFieldDef fdtFields[7] =  { 
+   {"Name",       'C', 30, 30, 0, 0, 30},
+   {"Subfields",  'C', 20, 20, 0, 30, 20},
+   {"Tag",        'N', 4, 4, 0,50, 4},
+   {"Len",        'N', 4, 4, 0,54, 4},
+   {"Type",       'N', 1, 1, 0,58, 4},
+   {"Rep",        'C', 1, 1, 0,62, 4},
+   {0,             0, 0, 0, 0, 0, 0}
+}; 
+
+
+
+
+
+
+//+----------------------------------------------------------------
+//
+// Member:     Store(const char* fileName)
+//
+// Synopsis:   Store dictionary tables on a file
+//
+// Parameters: 
+//
+//     fileName -- Full path of dictionary file
+// 
+//-----------------------------------------------------------------   
+
+void FormatFdtEntry(CFdtEntry &e, ustring &s)
+{
+#ifdef _WIN32
+	ustringstream ost;
+
+	ost.setf(std::ios_base::left, std::ios_base::adjustfield);
+	ost << std::setw(30) << e.name << std::setw(20) << e.subfields;
+	ost.setf(std::ios_base::internal,std::ios_base::adjustfield);
+	ost << " " << e.tag << " " << e.len << " " << e.type << " " << e.rep;
+	s = ost.str();
+#endif
+}
+
+void CFdt::Store(const char* fileName)
+{
+	if (m_bDirty)
+	{
+		Flush();
+		m_bDirty = false;
+	}
+
+	std::ofstream out((const char *) fileName);
+	TRACE(_T("\n<CFdt::Store> fileName=%s"),fileName);
+	for (int i=0; i<m_asPrelude.size(); i++)
+		out << m_asPrelude[i] << std::endl;
+	
+	out << "***" << std::endl;
+
+	ustring s;
+#ifdef _WIN32
+	for (i=0; i< m_aFdt.size(); i++)
+#else
+	for (int i=0; i< m_aFdt.size(); i++)
+#endif
+	{
+		FormatFdtEntry(m_aFdt[i], s);
+		out << s << std::endl;
+	}
+    out.close();
+
+}
+
+//////////////////////////////
+
+int CFdt::InitFields ()
+{
+	m_nFieldCount = 0;
+
+	for (int i = 0; fdtFields[i].width != 0; i++)
+	{
+		CFieldDef* fld = (CFieldDef*) new CFieldDef();
+		//TRACE("\n<CFdt::InitFields>i=%d after new",i);
+		//TRACE("<CFdt::InitFields>After SetAtGrow");
+		strcpy(fld->name, fdtFields[i].name);
+		fld->type          = fdtFields[i].type;
+		fld->len           = fdtFields[i].len;
+		fld->width         = fdtFields[i].width;
+		fld->decimals      = fdtFields[i].decimals;
+		fld->offset        = fdtFields[i].offset;
+		fld->display_width = fdtFields[i].display_width;
+
+		int l = strlen(fld->name);
+		if (fld->width < l)
+			fld->width = static_cast<short>(l);
+
+		m_fieldArray.push_back(fld);
+		m_nFieldCount++;
+	}
+	//TRACE("\n<CFdt::InitFields>InitField After loop");
+	return 1;
+}
+
+////////////////////////////////////////////////////////////////////
+
+CFieldDef* CFdt::GetField(int n) const
+// Returns field info
+{
+	ASSERT(n >= 0 && n <= m_nFieldCount);
+#ifndef _WIN32
+	return (CFieldDef*) m_fieldArray[n];
+#else
+	return (CFieldDef*) m_fieldArray.at(n);
+#endif
+}
+
+/////////////////////////////////////////////////////////////////////
+// Methods for accessing and maintaining the list of CDictTRecords 
+// pointers
+
+////////////////////////////
+// First the List Accessors
+////////////////////////////
+
+CFdtEntry CFdt::GetAt(int i)
+{
+#ifndef _WIN32
+	return m_aFdt[i];
+#else
+	return m_aFdt.at(i);
+#endif
+
+}
+
+
+//////////////////////////
+
+const CFdtEntry CFdt::GetAt(int i) const
+{
+#ifndef _WIN32
+	return m_aFdt[i];
+#else
+	return m_aFdt.at(i);
+#endif
+}
+
+//////////////////////////
+// And now the modifiers
+//////////////////////////
+
+int CFdt::Append(CFdtEntry& e)
+{
+	int n = m_aFdt.size();
+	m_aFdt.push_back(e);
+    m_iCurrentRecord = -1;
+    m_bIsEmpty = false;
+
+    return m_aFdt.size();
+}
+
+
+/**-------------------------------------------------------------------
+ * RemoveAt(int i)
+ *
+ *--------------------------------------------------------------------
+ */
+void CFdt::RemoveAt(int i)
+{
+
+	m_aFdt.erase(m_aFdt.begin()+i);
+    m_iCurrentRecord = -1;
+}
+
+/**-------------------------------------------------------------------
+ * SetAt(int i, CFdtEntry& e)
+ *
+ *--------------------------------------------------------------------
+ */
+void CFdt::SetAt(int i, CFdtEntry& e)
+{
+
+	m_aFdt[i] = e;
+	// Force reloading
+    m_iCurrentRecord = -1;
+}
+
+/////////////////////////////////////////////////////////////////
+// 
+// We use a buffer "m_fdtEntry" to hold the content of the current
+// fdt entry. This buffer is filled by the "Seek" member function
+// as follow:
+//
+
+//////////////////////////////
+
+bool CFdt::Seek(long iRecord)
+{
+
+	// move to new record and flush changes of previous record
+	if (iRecord < 0 || iRecord > GetRecordCount())
+		return false;
+	else if (iRecord == m_iCurrentRecord)
+		return true;
+
+	if (m_iCurrentRecord != -1 &&  m_bDirty)
+		Flush();
+	if (m_bDirty)
+	   /*AfxMessageBox(_T("Seek Alert Modification not saved"))*/;
+	m_iCurrentRecord = iRecord;
+	
+	if (iRecord == GetRecordCount())
+	{
+		m_fdtEntry.Clear();
+        m_bAppend = true;
+	}
+	else
+	{
+		TRACE(_T("\n<CFdt::Seek> Load iRecord=%d GetRecordCount()=%d"), iRecord, GetRecordCount());
+		m_fdtEntry = GetAt(iRecord);
+		m_bAppend = false;
+	}
+    m_bDirty = false;
+	return true;
+}
+
+
+
+//+------------------------------------------------------------------
+//
+// Member:   Flush()
+//
+// Synopsis: Moves the changes made to the current record from
+//           the buffer to the dictionary data structure so that
+//           they are effective.
+//-------------------------------------------------------------------    
+
+void CFdt::Flush()
+{
+	if (m_iCurrentRecord == GetRecordCount())
+	{
+		Append(m_fdtEntry); 
+	}
+	else
+	{
+		SetAt(m_iCurrentRecord, m_fdtEntry);
+	}
+	m_bDirty = false;
+}
+
+//////////////////////////////////////////////////////////////
+// Get the value of field n for the current record
+//
+
+bool CFdt::GetValue(int n, std::string& result) 
+// Read value from buffer
+{
+	ASSERT(n >= 0 && n <= m_nFieldCount);
+    ASSERT(m_iCurrentRecord >= 0 && m_iCurrentRecord <= GetRecordCount());
+	
+	if (!(n >= 0 && n <= m_nFieldCount))
+		return false;
+
+	result.erase();
+
+	std::ostringstream os;
+    switch (n)
+	{
+	    case 0:       // Field Name
+			os << m_fdtEntry.name;
+			result = os.str();
+		    break;
+	    case 1:       // Subfields
+			os << m_fdtEntry.subfields;
+			result = os.str();
+		    break;
+	    case 2:       // Tag
+		  	if (m_fdtEntry.tag == 0)
+				result = "";
+			else
+			{
+			    os << m_fdtEntry.tag;
+			    result = os.str();
+			}
+		    break;
+	    case 3:       // length
+		  	if (m_fdtEntry.len == 0)
+				result = "";
+			else
+			{
+			    os << m_fdtEntry.len;
+			    result = os.str();
+			}
+		    break;
+	    case 4:       // type
+		  	if (m_fdtEntry.type == 0)
+				result = "";
+			else
+			{
+			    os << m_fdtEntry.type;
+			    result = os.str();
+			}
+		    break;
+	    case 5:       // Rep
+		  	if (m_fdtEntry.rep == 0)
+				result = "";
+			else
+			{
+			    os << m_fdtEntry.rep;
+			    result = os.str();
+			}
+		    break;
+	}
+	return true;
+}
+
+
+//////////////////////////////
+
+bool CFdt::SetValue(int n, const char* pszValue)
+// Write value to buffer
+{
+	ASSERT(n >= 0 && n <= m_nFieldCount);
+    ASSERT(m_iCurrentRecord >= 0 && m_iCurrentRecord <= GetRecordCount());
+
+	if (m_bReadOnly)
+		return false;
+
+	if (!(n >= 0 && n <= m_nFieldCount))
+		return false;
+
+    bool bEmpty = true;
+
+
+	switch (n) 
+	{
+	case 0:       // Name
+		strcpy(m_fdtEntry.name,(const char*)pszValue);
+	//	TRACE("\nvalue=%s num=%d",pszValue,m_tRecord.num);
+		break;
+	case 1:       // Subfields
+		strcpy(m_fdtEntry.subfields,(const char*)pszValue);
+	//	TRACE("\nvalue=%s name=%d",pszValue,m_tRecord.name);
+		break;
+	case 2:       // Tag
+		m_fdtEntry.tag = _ttoi(pszValue);
+	//	TRACE("\nvalue=%s loc=%d",pszValue,m_tRecord.loc);
+		break;
+	case 3:       // Len
+		m_fdtEntry.len = _ttoi(pszValue);
+	//	TRACE("\nvalue=%s wid=%d",pszValue,m_tRecord.wid);
+		break;
+	case 4:       // Type
+		m_fdtEntry.type = _ttoi(pszValue);
+		break;
+	case 5:       // Rep
+		m_fdtEntry.type = _ttoi(pszValue);
+		break;
+	}
+	m_bDirty = true;
+	
+	return true;
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Fdt.h	(revision 21112)
@@ -0,0 +1,172 @@
+/**********************************************************************
+ *
+ * Fdt.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+// Fdt.h
+
+#ifndef __FDT_H__
+#define __FDT_H__
+
+#include <vector>
+#include "mytchar.h"
+
+#define FDT_NAME_LENG    30     // maximum name length in FDT_FILE
+#define FDT_SFLD_LENG    20    // maximum subfield length in FDT_FILE
+
+#ifndef __CFIELDDEF__
+#define __CFIELDDEF__
+
+class CFieldDef
+{
+public:
+	enum FieldType {
+		charField       = 'C',
+		dateField       = 'D',
+		numericField    = 'N',
+		logicalField    = 'L',
+		memoField       = 'M'
+	};
+
+	char  name[31];   // dbf field names are 10 chars long max.
+		// If this application is compiled with _UNICODE switch,
+		// the field name will be converted to wide chars
+		// when the column header is displayed in the grid.
+	char  type;
+	short len;
+	short width;
+	short decimals;
+	int   offset;
+	short display_width;
+};
+#endif // __CFIELDDEF__
+
+
+
+class CFdtEntry
+{
+public:
+	CFdtEntry() :  tag(0), len(0), type(0), rep(0) 
+	{ name[0] = '\0'; subfields[0] = '\0'; }
+    void Clear()
+	{
+		tag = len = type = rep = 0;
+        name[0] = '\0'; subfields[0] = '\0';
+	}
+    char    name[FDT_NAME_LENG+1];
+    char    subfields[FDT_SFLD_LENG+1];
+    int      tag;
+    int      len;
+    int      type;     //X=0
+    int      rep;      //repeatable 1, non rep. 0
+   CFdtEntry& operator=(const CFdtEntry& rhs)
+   {
+	   if (&name[0] != &rhs.name[0])
+	   {
+		   Copy(rhs);
+	   }
+	   return *this;
+   }
+
+   void Copy(const CFdtEntry &rhs)
+   {
+	   strcpy(name, rhs.name);
+	   strcpy(subfields, rhs.subfields);
+	   tag  = rhs.tag;
+	   len  = rhs.len;
+	   type = rhs.type;
+	   rep  = rhs.rep;
+   }
+};
+
+
+void FormatFdtEntry(CFdtEntry &e, ustring &s);
+
+
+
+class CFdt
+{
+protected:
+	std::vector<CFdtEntry>  m_aFdt;
+    int                     m_nFieldCount;       // Num. of fields
+	std::vector<CFieldDef*> m_fieldArray;        // Array with fields
+
+	long        m_iCurrentRecord;    // Current read record stored in
+	CFdtEntry   m_fdtEntry;
+	bool        m_bReadOnly;
+	bool        m_bIsEmpty;
+	bool        m_bDirty;            // True if m_fdtEntry modified 
+	bool        m_bAppend;
+public:
+	ustring  m_sFileName;              // Fdt full pathname
+	std::vector<std::string> m_asPrelude;  // What is bef ***
+public:
+	CFdt();          
+	virtual ~CFdt();
+
+	bool      Load(const _TCHAR *fname);
+	int       GetEntryCount() { return m_aFdt.size(); }
+
+	CFdtEntry GetEntryAt(int i) 
+	{ 
+      #ifdef _WIN32
+	     return m_aFdt.at(i); 
+      #else
+	     return m_aFdt[i];
+      #endif
+	}
+
+	int       GetFieldCount() { return  m_nFieldCount; }
+
+    bool      Open(const char* szFileName, bool readOnly=true);
+
+    void InitData();
+
+    void Close();
+ 
+    void DeleteContents();
+
+    void Store(const char* fileName);
+    int InitFields ();
+
+    CFieldDef* GetField(int n) const;
+
+    CFdtEntry GetAt(int i);
+
+    const CFdtEntry GetAt(int i) const;
+
+    int Append(CFdtEntry& e);
+
+    void RemoveAt(int i);
+    void SetAt(int i, CFdtEntry& e);
+
+    bool Seek(long iRecord);
+    void Flush();
+    bool GetValue(int n, std::string& result);
+    bool SetValue(int n, const char* pszValue);
+
+	int GetRecordCount() { return m_aFdt.size(); }
+};
+
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/File.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/File.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/File.cpp	(revision 21112)
@@ -0,0 +1,618 @@
+/**********************************************************************
+ *
+ * File.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// File.cpp   File object methods
+////////////////////////////////////////////////////////////
+#include "stdafx.h"
+
+ 
+#include <assert.h>
+#include "CRC32.h"
+#include "File.h"
+
+
+
+
+static TCHAR closed_file_name[] = _T("closed-file");
+
+static const int MaxExceptionMessages = 34;
+static const TCHAR *fileExceptionMsg[MaxExceptionMessages] =
+{
+  _T("File exception: No exception reported"),               // FILE_NO_ERROR
+  _T("File exception: Checksum error"),                      // FILE_CHECKSUM_ERROR
+  _T("File exception: Error closing file"),                  // FILE_CLOSE_ERROR
+  _T("File exception: File is corrupted"),
+  _T("File exception: Error creating file"),                 // FILE_CREATION_ERROR
+  _T("File exception: File already exists"),                 // FILE_EXISTS
+  _T("File exception: Trying to use a closed file"),         // FILE_NOT_OPEN_ERROR
+  _T("File exception: File not ready for reading/writing"),  // FILE_NOT_READY
+  _T("File exception: Could not write to file"),             // FILE_NOT_WRITEABLE
+  _T("File exception: Error opening file"),                  // FILE_OPEN_ERROR
+  _T("File exception: Cannot obtain a file position"),       // FILE_POSITION_ERROR
+  _T("File exception: Error reading from file"),             // FILE_READ_ERROR
+  _T("File exception: Error seeking in file"),               // FILE_SEEK_ERROR
+  _T("File exception: Error writing to file"),               // FILE_WRITE_ERROR
+
+  _T("Fmgr exception: No database open"),                    // FMGR_NO_DATABASE_OPEN
+  _T("Fmgr exception: No such file exists"),                 // FMGR_NO_FILE_EXISTS
+  _T("Fmgr exception: No objects exist"),                    // FMGR_NO_OBJECTS_EXIST
+  _T("Fmgr exception: Accessing a null pointer"),            // FMGR_NULL_PTR
+  _T("Fmgr exception: Object already exists"),               // FMGR_OBJECT_EXISTS
+  _T("Fmgr exception: Another object is referencing this file"), // FMGR_OBJECT_EXISTS
+  _T("Fmgr exception: Math overflow error"),                 // FMGR_OVERFLOW
+  _T("Fmgr exception: Parse error"),                         // FMGR_PARSE_ERROR
+  _T("Fmgr exception: Invalid path"),                        // FMGR_PATH_ERROR
+  _T("Fmgr exception: Trying to write to a read-only file"), // FMGR_READ_ONLY_FILE
+  _T("Fmgr exception: Stack empty"),                         // FMGR_STACK_EMPTY
+  _T("Fmgr exception: Stack full"),                          // FMGR_STACK_FULL
+  _T("Fmgr exception: Synchronization Error"),               // FMGR_SYNC_ERROR    
+  _T("Fmgr exception: Math under-flow error"),               // FMGR_UNDERFLOW
+  _T("Fmgr exception: Wrong file type"),                     // FMGR_WRONG_FILE_TYPE
+  
+  // Persistent lock error messages
+  _T("Fmgr exception: Invalid lock type specified"),          // FMGR_INVALID_LOCK_TYPE
+  _T("Fmgr exception: Cannot access the file lock header"),   // FMGR_FILELOCK_ACCESS_ERROR
+  _T("Fmgr exception: Cannot lock the file"),                 // FMGR_FILELOCK_ERROR
+  _T("Fmgr exception: Cannot access the record lock header"), // FMGR_RECORDLOCK_ACCESS_ERROR
+  _T("Fmgr exception: Cannot lock the specified record")      // FMGR_RECORDLOCK_ERROR
+};
+ 
+//---------------------------------------------------------------------------
+// CFileBase::CFileBase()
+//
+// Default constructor, makes a File object that isn't attached to a file
+//--------------------------------------------------------------------------- 
+CFileBase::CFileBase()
+{
+    _tcscpy(file_name, closed_file_name); // Set the initial file name
+    fp = 0;
+
+    // Reset the last reported error and the file status members
+    file_error        = FILE_NO_ERROR;
+    is_open           = 0;
+    is_ok             = 0;
+    ready_for_reading = 0;
+    ready_for_writing = 0;
+}
+  
+//---------------------------------------------------------------------------
+// CFileBase::~CFileBase()
+//
+// Destructor for CFileBase file object
+//--------------------------------------------------------------------------- 
+CFileBase::~CFileBase()
+{
+	Close();
+}
+
+//---------------------------------------------------------------------------
+// const char* CFileBase::FileExceptionMessage()
+//
+// Returns a null terminated string that can / be used to log or print a
+// file exception.
+//---------------------------------------------------------------------------
+const TCHAR *CFileBase::FileExceptionMessage()
+{
+	assert(file_error<MaxExceptionMessages);
+    return fileExceptionMsg[file_error];
+}
+ 
+//--------------------------------------------------------------------------------
+// int Create(const char* fname)
+//
+// Creates a new file, truncate the file if it already exists. The file open for
+// Read/Write access. 
+// Returns a non-zero value to indicate an error condition or zero if successful.
+//--------------------------------------------------------------------------------
+FileError CFileBase::Create(const TCHAR *fname)
+{
+    // Close any open files 
+    if (Close() != FILE_NO_ERROR) 
+		return file_error;
+	
+    // Create and truncate existing files
+    fp = FileSystem::Create(fname);
+	
+    if (fp == 0)
+	{
+		is_ok             = 0;
+		is_open           = 0;
+		ready_for_writing = 0;
+		ready_for_reading = 0;
+		file_error        = FILE_CREATION_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(file_error);
+#else
+		return file_error;
+#endif
+	}
+	else
+	{
+		is_open           = 1;
+		is_ok             = 1;
+		ready_for_writing = 1;
+		ready_for_reading = 1;
+		last_operation    = IO_READ;
+		_tcscpy(file_name, fname);
+	}
+	
+	// Returns 0 if the file was successfully created and opened.
+	return file_error = FILE_NO_ERROR; 
+}
+ 
+//----------------------------------------------------------------------------------------
+// FileError CFileBase::Open(const TCHAR *fname, FileSystem::AccessMode mode
+//                                               /* = FileSystem::FILE_READWRITE */ )
+//
+// Open an existing file. The "mode" variable determines if the file is opened for
+// read only or read/write access.  Returns a non-zero value to indicate an error
+// condition or zero if successful.
+// NOTE: This version of the open functions will only accept:
+// FILE_READONLY and FILE_READWRITE access modes.
+//---------------------------------------------------------------------------------------
+FileError CFileBase::Open(const TCHAR *fname, FileSystem::AccessMode mode /* = FileSystem::FILE_READWRITE */)
+{
+	// Close any open files
+	if (Close() != FILE_NO_ERROR) 
+		return file_error;
+	
+	if (mode == FileSystem::FILE_READONLY)
+	{   // Open with read only access
+		ready_for_reading = 1;
+		ready_for_writing = 0;
+		fp = FileSystem::Open(fname, FileSystem::FILE_READONLY);
+	}
+	else
+	{   // Open with read/write access
+		ready_for_reading = 1;
+		ready_for_writing = 1;
+		fp = FileSystem::Open(fname, FileSystem::FILE_READWRITE);
+	}
+    
+	if (fp == 0)
+	{
+		ready_for_reading = 0;
+		ready_for_writing = 0;
+		file_error        = FILE_OPEN_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(file_error);
+#else
+		return file_error;
+#endif
+	}
+	else
+	{
+		is_open         = 1;
+		is_ok           = 1;
+		last_operation  = IO_WRITE;
+		_tcscpy(file_name, fname);
+	}
+	return file_error = FILE_NO_ERROR;
+}
+ 
+//--------------------------------------------------------------------------------
+// void CFileBase::Close()
+//
+// Close the open database file. Returns a non-zero value
+// to indicate an error condition or zero if successful.
+//--------------------------------------------------------------------------------
+FileError CFileBase::Close()
+{
+   if (IsOpen()) 
+   {
+       if (FileSystem::Close(fp) != 0) 
+	   {
+		   file_error = FILE_CLOSE_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		   throw CFileBaseException(file_error);
+#else
+		   return file_error;
+#endif
+	   }
+       _tcscpy(file_name, closed_file_name); // Set the initial file name
+   }
+
+   is_open           = 0;
+   is_ok             = 0;
+   ready_for_reading = 0;
+   ready_for_writing = 0;
+   fp                = 0; // Reset the file pointer to zero after the file is closed
+
+   return file_error = FILE_NO_ERROR; 
+}
+
+//----------------------------------------------------------------------------------
+// FileError CFileBase::Seek(FAU offset, FileSystem::SeekMode mode)
+//
+// Seek to the specified offset starting at the beginning (SEEK_SET), end (SEEK_END)
+// or current offset (SEEK_CUR). Returns a non-zero value to indicate an error
+// condition or zero if successful.
+//----------------------------------------------------------------------------------
+FileError CFileBase::Seek(FileSystem::FAU_t offset, FileSystem::SeekMode mode)
+{
+	if (IsOK()) 
+	{
+		if (FileSystem::Seek(fp, offset, mode) == (isis::int32_t) - 1)
+		{
+			file_error = FILE_SEEK_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(file_error);
+#else
+			return file_error;
+#endif
+		}
+		last_operation = IO_SEEK;
+	}
+	else 
+	{
+		file_error = FILE_NOT_READY;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(file_error);
+#else
+		return file_error;
+#endif
+	}
+	return file_error = FILE_NO_ERROR;
+}
+
+//----------------------------------------------------------------------------------
+// FAU CFileBase::SeekTo(FileSystem::FAU_t file_address)
+//
+// Seek to the specified address, optimizing the seek operation by moving the 
+// file position indicator based on the current stream position. Returns the
+// current file position after performing the seek operation.
+//----------------------------------------------------------------------------------
+FileSystem::FAU_t CFileBase::SeekTo(FileSystem::FAU_t file_address)
+{
+	// Get the current stream position
+	FileSystem::StreamPos pos = FilePosition();
+	
+	if (file_address == CURRADDR) 
+	{   // Do not perform a seek operation
+		return pos;
+	}
+	else if (file_address > pos)
+	{   // Seek forward to the specified address
+		FileSystem::StreamPos offset = file_address - pos;
+		Seek(offset, FileSystem::FILE_SEEK_CUR);
+	}
+	else if (file_address < pos) 
+	{   // Seek backward to the specified address
+		Seek(file_address, FileSystem::FILE_SEEK_BEG);
+	}
+	else 
+	{  // Current file position equals the specified address
+		// Find current the position
+		Seek((FileSystem::FAU_t)0, FileSystem::FILE_SEEK_CUR);
+	}
+	
+	return FilePosition(); // Return current file position after seeking
+}
+
+//-------------------------------------------------------------------------------
+// FileError Store(const void* buf, isis::uint32_t nBytes,
+//                             FAU file_addres = CFileBase::CURRADDR,
+//			                   int flush = 1, int bit_test = 1);
+//
+// Write a specific number of bytes from a memory buffer to a specified file
+// offset. If the "flush" variable is true, the file buffers will be flushed to
+// disk with each write operation. If the bit_test variable if true, the CRC of 
+// the buffer will be compared to the CRC of the actual bytes written to disk.
+// Returns a non-zero value to indicate an error condition or zero if successful.
+//
+// void* buf         - Pointer to the user-supplied buffer that is to receive
+//                     the data read from the file.
+// isis::uint32_t nBytes - The maximum number of bytes to be read from the 
+//                     file.  Note that an CFileBase is always a binary file,
+//                     so a CR/LF pair will always be read or written as two bytes.
+// FAU file_address  - The file byte address. The address is always interpreted to 
+//                     be from the beginning of the file, unless dwAddr is equal 
+//                     to CURRADDR, which means from the current position.
+//------------------------------------------------------------------------------- 
+FileError CFileBase::Store(const void* buf, isis::uint32_t nBytes,
+						   FileSystem::FAU_t file_address /*= CURRADDR */, 
+						   int flush_flag /* = 1 */,
+						   int bit_test /* = 1 */)
+{
+    FileSystem::FAU_t buf_address;
+	
+    if (ReadyForWriting())
+	{
+        if (file_address == CURRADDR)
+		{
+			if (last_operation == IO_READ)
+			{
+				if (Seek(0, FileSystem::FILE_SEEK_CUR) != FILE_NO_ERROR)
+					return file_error;
+			}
+		}
+        else
+		{
+			if (Seek(file_address, FileSystem::FILE_SEEK_BEG) != FILE_NO_ERROR)
+				return file_error;
+		}
+		
+        buf_address = FilePosition(); 
+		
+        if (file_error != FILE_NO_ERROR)
+			return file_error;
+		
+        if (FileSystem::Write(fp, buf, nBytes) != nBytes)
+		{
+			file_error = FILE_WRITE_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(file_error);
+#else
+			return file_error;
+#endif
+		}
+        last_operation = IO_WRITE;
+	}
+    else
+	{
+        file_error = FILE_NOT_WRITEABLE;
+#ifdef __CPP_EXCEPTIONS__
+        throw CFileBaseException(file_error);
+#else
+        return file_error;
+#endif
+	} 
+	
+    // Allow application to flush disk buffers after each write
+    // operation to ensure the file data stays in sync during multiple
+    // file access.
+    if (flush_flag)
+	{  
+		if (FileSystem::Flush(fp) != FILE_NO_ERROR)
+		{
+			file_error = FILE_WRITE_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(file_error);
+#else
+			return file_error;
+#endif
+		}
+        if (Seek(0, FileSystem::FILE_SEEK_CUR) != FILE_NO_ERROR)
+			return file_error;
+	}
+	
+    if (bit_test) 
+	{
+        isis::uint32_t w_csum = CalcCRC32((char* ) buf, nBytes);
+        isis::uint32_t r_csum = CalcChecksum(nBytes, buf_address);
+		
+		// Check for file errors
+		
+		if (file_error != FILE_NO_ERROR)
+			return file_error;
+		
+		if (w_csum ^ r_csum)
+		{
+			file_error = FILE_CHECKSUM_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(file_error);
+#else
+			return file_error;
+#endif
+		}
+	}
+	return file_error = FILE_NO_ERROR;	
+}
+ 
+//-------------------------------------------------------------------------------
+// FileError CFileBase::Fetch(void* buf, isis::uint32_t nBytes, FAU file_addres
+//                                                  /* = CFileBase::CURRADDR */)
+//
+// Read a specified number of bytes from the specified file offset into a memory
+// buffer. Returns a non-zero value to indicate an error condition or zero if
+// successful.
+// void* buf         - Pointer to the user-supplied buffer that is to receive
+//                     the data read from the file.
+// isis::uint32_t nBytes - The maximum number of bytes to be read from the file. 
+//                     Note that an CFileBase is always a binary file, so a 
+//                     CR/LF pair will always be read or written as two bytes.
+// FAU file_addres   - The file byte address. The address is always interpreted 
+//                     to be from the beginning of the file, unless dwAddr is 
+//                     equal to CURRADDR, which means from the current position.
+//------------------------------------------------------------------------------- 
+FileError CFileBase::Fetch(void* buf, isis::uint32_t nBytes, 
+						   FileSystem::FAU_t file_address /* = CFileBase::CURRADDR */)
+{
+    if (IsOK())
+	{
+        if (file_address == CURRADDR)
+		{
+            if (last_operation == IO_WRITE)
+			{
+	           if (Seek(0, FileSystem::FILE_SEEK_CUR) != FILE_NO_ERROR) 
+				   return file_error;
+			}
+		}
+        else
+		{
+            if (Seek(file_address, FileSystem::FILE_SEEK_BEG) != FILE_NO_ERROR)
+              	return file_error;
+		}
+
+		//TRACE("\nFetchfile_address=%ld nBytes=%u",file_address, nBytes);
+        if (FileSystem::Read(fp, buf, nBytes) != nBytes)
+		{
+            file_error = FILE_READ_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+            throw CFileBaseException(file_error);
+#else
+            return file_error;
+#endif
+		}
+        last_operation = IO_READ;
+	}
+    else 
+	{
+        file_error = FILE_NOT_READY;
+#ifdef __CPP_EXCEPTIONS__
+        throw CFileBaseException(file_error);
+#else
+        return file_error;
+#endif
+	}
+
+    return file_error = FILE_NO_ERROR;
+}
+//------------------------------------------------------------------------------- 
+// FileError Flush()
+//
+// Flush any open disk buffers. Returns a non-zero value to indicate an error
+// condition or zero if successful.
+//------------------------------------------------------------------------------- 
+FileError CFileBase::Flush()
+{
+	PRECONDITION(ReadyForWriting());
+	if (FileSystem::Flush(fp) != FILE_NO_ERROR)
+	{
+		file_error = FILE_WRITE_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(file_error);
+#else
+		return file_error;
+#endif
+	}
+	Seek((FileSystem::FAU_t)0, FileSystem::FILE_SEEK_CUR);
+	return file_error = FILE_NO_ERROR;
+}
+
+
+//------------------------------------------------------------------------------- 
+// StreamPos CFileBase::FilePosition()
+//
+// Returns the current file position or -1 to indicate an error condition.
+//------------------------------------------------------------------------------- 
+FileSystem::StreamPos CFileBase::FilePosition()
+{
+   if (!IsOK())
+   {
+      file_error = FILE_NOT_READY;
+#ifdef __CPP_EXCEPTIONS__
+      throw CFileBase();
+#else
+      return (StreamPos) - 1;
+#endif
+   }
+
+   FileSystem::StreamPos pos = FileSystem::Tell(fp);
+   if (pos < 0)
+   {
+      file_error = FILE_POSITION_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+      throw CFileBaseException(file_error);
+#else
+      return (StreamPos) - 1;
+#endif
+   }
+   return pos;
+}
+
+//------------------------------------------------------------------------------- 
+// isis::uint32_t CFileBase::CalcChecksum(isis::uint32_t bytes, FAU file_address, 
+//                                                         int mem_alloc)
+//
+// Calculate a 32-bit CRC checksum for a given number of bytes starting at the
+// specified address. Returns a 32-bit CRC value. If the mem_alloc variable is
+// true, a buffer equal to the specified number of bytes will be created in
+// memory. If the mem_alloc variable is false or memory allocation fails, the CRC
+// will be calculated byte by byte starting at the specified address. 
+// NOTE: the calling function must check for disk file errors.
+// Returns the CRC value.
+//------------------------------------------------------------------------------- 
+
+isis::uint32_t CFileBase::CalcChecksum(isis::uint32_t bytes, FileSystem::FAU_t file_address, int mem_alloc)
+{
+   isis::uint32_t CRC;
+   isis::uint32_t len = bytes;
+   unsigned char data;
+   char *buf = 0;  
+
+   // Create a buffer equal to the object length
+   if (mem_alloc) 
+	   buf = new char[bytes]; 
+
+   if (buf)
+   {
+      if (IsOK())
+	  {
+         if (Fetch(buf, bytes, file_address) != FILE_NO_ERROR) return 0;
+         CRC = CalcCRC32(buf, bytes);
+         delete[] buf;
+	  }
+   }
+   else
+   {
+      if (IsOK())
+	  {
+          // Seek to the specified file address
+          SeekTo(file_address);
+          if (file_error != FILE_NO_ERROR) return 0; 
+          CRC = 0xffffffffL;
+          while (len--)
+		  {
+	         if (Fetch(&data, sizeof(data)) != FILE_NO_ERROR) return 0;
+	         CRC = CalcCRC32(data, CRC);
+		  }
+          CRC ^= 0xffffffffL;
+	  }
+   }
+
+   return CRC; 
+}
+///////////////////////////////////////////////////////////////////////////
+// General purpose file utilites 
+//
+int CFileBase::Exists(const TCHAR *fname)
+// Returns true if the file exists
+{
+	return FileSystem::Exists(fname);
+}
+
+FileSystem::FAU_t CFileBase::FileSize(const TCHAR *fname)
+// Returns the file size. Use after file has been closed
+// and re-opened to ensure that all the buffers are flushed
+// to disk. Returns -1 to indicate an error condition.
+{
+	return (FileSystem::FAU_t) FileSystem::FileSize(fname);
+}
+
+int CFileBase::CanOpenForWriting(const TCHAR *fname)
+// Returns true if the file is opened
+{
+	return 1;
+	//return FileSystem::CanOpenForWriting(fname);
+}
+
+int CFileBase::CanOpenReadOnly(const TCHAR *fname)
+// Returns true if the file is opened
+{
+	return 1;
+	//return FileSystem::CanOpenReadOnly(fname);
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/File.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/File.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/File.h	(revision 21112)
@@ -0,0 +1,188 @@
+/**********************************************************************
+ *
+ * File.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+//////////////////////////////////////////////////////////////////////////
+// File.h -- File Base class
+// 
+//////////////////////////////////////////////////////////////////////////
+ 
+
+ 
+#ifndef __FILE_BASE__
+#define __FILE_BASE__
+
+#include "FileSystem.h"
+# define __CPP_EXCEPTIONS__ 1
+
+// CRC-32 checksum used to detect bit errors
+typedef unsigned long Checksum;
+
+
+// Constants for file operations and stream position 
+const int MaxNameLength             = 512;        // Max length of names buffers
+const FileSystem::FAU_t StartOfFile = (FileSystem::FAU_t)0; // First byte in the file
+const FileSystem::FAU_t CURRADDR    = (FileSystem::FAU_t)-1;// Indicates current location
+const isis::uint32_t CheckWord      = 0xfefefefe;           // Default synchronization word
+const isis::int32_t  FSListCorrupt  = (FileSystem::FAU_t)-1;// Free space list is corrupt
+
+
+//------------------
+// File  error codes
+//------------------
+enum FileError { 
+  FILE_NO_ERROR = 0,   // No errors reported
+  FILE_CHECKSUM_ERROR, // Checksum Error
+  FILE_CLOSE_ERROR,    // Error closing file
+  FILE_CORRUPT,        // File corrupted
+  FILE_CREATION_ERROR, // Error creating file
+  FILE_EXISTS,         // File already exists
+  FILE_NOT_OPEN_ERROR, // Trying to use a closed file
+  FILE_NOT_READY,      // File not ready (failed or closed file)
+  FILE_NOT_WRITEABLE,  // Could not write to file 
+  FILE_OPEN_ERROR,     // Error opening file
+  FILE_POSITION_ERROR, // Cannot obtain the current file position
+  FILE_READ_ERROR,     // Error reading file  
+  FILE_SEEK_ERROR,     // Error seeking in file
+  FILE_WRITE_ERROR,    // Error writing to file
+
+  FMGR_NO_DATABASE_OPEN,    // No database open
+  FMGR_NO_FILE_EXISTS,      // No such file exists
+  FMGR_NO_OBJECTS_EXIST,    // No objects exist
+  FMGR_NULL_PTR,            // Accessing a null pointer 
+  FMGR_OBJECT_EXISTS,       // Object already exists
+  FMGR_OPEN_FILE_REFERENCE, // Another object is referencing this file
+  FMGR_OVERFLOW,            // Math overflow
+  FMGR_PARSE_ERROR,         // Parse error
+  FMGR_PATH_ERROR,          // Invalid path
+  FMGR_READ_ONLY_FILE,      // Trying to write to read-only file
+  FMGR_STACK_EMPTY,         // Stack empty
+  FMGR_STACK_FULL,          // Stack full
+  FMGR_SYNC_ERROR,          // Synchronization Error
+  FMGR_UNDERFLOW,           // Math under-flow
+  FMGR_WRONG_FILE_TYPE,     // Wrong file type
+
+  // Persistent lock error codes
+  FMGR_INVALID_LOCK_TYPE,        // Invalid lock type specified
+  FMGR_FILELOCK_ACCESS_ERROR,    // The file lock cannot be accessed
+  FMGR_FILELOCK_ERROR,           // Error locking the file
+  FMGR_RECORDLOCK_ACCESS_ERROR,  // The record lock cannot be accessed
+  FMGR_RECORDLOCK_ERROR          // Error locking a record};
+
+
+};
+
+
+
+
+class DLL_CODE_API CFileBase 
+{
+protected:
+    // File I/O operation codes
+  enum IO_Op { 
+     IO_READ,         // A read was performed
+     IO_WRITE,        // A write operation was performed
+     IO_REWIND,       // A rewind operation was performed
+     IO_NO_OPERATION, // No operation was performed
+     IO_SEEK          // A seek operation was preformed 
+  };
+
+  _TCHAR            file_name[MaxNameLength];  // Open file name
+  FileSystem::FPTR* fp;                        // Stream file handle
+  IO_Op             last_operation;            // Last I/O operation preformed
+  FileError         file_error;                // Last reported file error
+
+  // File status members
+  int is_ok;             // Used to signal a fatal error condition
+  int is_open;           // True if the file is open
+  int ready_for_reading; // True if the file is ready for reading
+  int ready_for_writing; // True if the file is ready for writing
+
+
+public:
+   CFileBase();
+   virtual ~CFileBase();
+
+public: // File functions
+   virtual FileError Create(const TCHAR *fname);
+   virtual FileError Open(const TCHAR *fname, FileSystem::AccessMode mode = FileSystem::FILE_READWRITE); 
+   virtual FileError Close();
+   
+   FileError         Seek(FileSystem::FAU_t offset, FileSystem::SeekMode mode = FileSystem::FILE_SEEK_BEG);
+ 
+   FileError         Fetch(void* buf, isis::uint32_t nBytes, FileSystem::FAU_t file_addres = CURRADDR);
+   FileError         Store(const void* buf, isis::uint32_t nBytes, FileSystem::FAU_t file_address = CURRADDR,
+			                int flush_flag = 1, int bit_test = 1);
+
+   FileSystem::FAU_t     SeekTo(FileSystem::FAU_t file_address);
+   FileSystem::StreamPos FilePosition();
+
+   FileError Flush();
+
+   _TCHAR *FileName() { return file_name; }
+
+public: // 32-bit CRC checksum routines 
+  isis::uint32_t CalcChecksum(isis::uint32_t bytes, FileSystem::FAU_t file_address, int mem_alloc = 1);
+
+public:  // File Status functions
+  int IsOK()                  { return ((is_ok == 1) && (is_open == 1)); }
+  int IsOK()            const { return ((is_ok == 1) && (is_open == 1)); }
+  int IsOpen()                { return ((is_ok == 1) && (is_open == 1)); }
+  int IsOpen()          const { return ((is_ok == 1) && (is_open == 1)); }
+  int ReadyForReading()       { return ready_for_reading == 1; }
+  int ReadyForReading() const { return ready_for_reading == 1; }
+  int ReadyForWriting()       { return ready_for_writing == 1; }
+  int ReadyForWriting() const { return ready_for_writing == 1; }
+
+public:  // Exception handling functions
+  FileError   GetFileError()              { return file_error; }
+  FileError   GetFileError() const        { return file_error; }
+  void        SetFileError(FileError err) { file_error = err; }
+  void        ResetFileError()            { file_error = FILE_NO_ERROR; }
+  const TCHAR *FileExceptionMessage();
+
+public: // General purpose file utilities
+  static int Exists(const TCHAR *fname);
+  static FileSystem::FAU_t FileSize(const TCHAR *fname);
+  static int CanOpenForWriting(const TCHAR *fname);
+  static int CanOpenReadOnly(const TCHAR *fname);
+  
+};
+
+
+#ifdef __CPP_EXCEPTIONS__
+// Class declarations for exceptions representing program errors. 
+// This implementation is provided for use C++'s built-in exception
+// handling routines.
+
+class CFileBaseException 
+{
+	int error_;
+public:
+	CFileBaseException(int error) { error_ = error; }
+};  
+
+#endif // __CPP_EXCEPTIONS__
+ 
+#endif   // __FILE_BASE__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.cpp	(revision 21112)
@@ -0,0 +1,482 @@
+/**********************************************************************
+ *
+ * FileSystem.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FileSystem -- The generic file pointer routines are a collection of member
+//               functions used to wrap  the underlying file system.
+
+
+#include "errno.h"
+#include "stdafx.h"
+#include "FileSystem.h"
+
+
+using namespace FileSystem;
+//---------------------------------------------------------------------
+// FileSystem::FPTR *Create(const _TCHAR *fname)
+//
+// Create a new file and truncate existing files.
+// Returns a null value if the file cannot be
+// created.
+//---------------------------------------------------------------------
+DLL_CODE_API FileSystem::FPTR *FileSystem::Create(const char *fname)
+{
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Create(fname);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	FPTR *stream = new FPTR;
+	if (!stream)
+		return 0;
+	stream->fptr = CreateFile(fname, 
+		GENERIC_READ | GENERIC_WRITE, 
+		FILE_SHARE_READ | FILE_SHARE_WRITE, 
+		NULL, // No security
+		CREATE_ALWAYS, 
+		FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS,
+		NULL // No attr. template 
+		);
+	
+	if (stream->fptr == INVALID_HANDLE_VALUE)
+	{
+		delete stream;
+		return 0;
+	}
+	return stream;
+	
+#else // Use the 32-bit stdio version by default
+	FPTR *stream = new FPTR;
+	if (!stream)
+		return 0;
+	stream->fptr = fopen(fname, "w+b");
+	if (stream->fptr == 0)
+	{
+		delete stream; // 01/30/2002: Prevent memory leak
+		return 0;
+	}
+	return stream;
+#endif
+}
+
+//---------------------------------------------------------------------
+// FileSystem::FPTR *Open(const _TCHAR *fname, FileSystemAccessMode mode)
+//
+// Open an existing file. The "mode" variable determines if the file
+// is opened for read only or read/write access. Returns a null value
+// if the specified file cannot be opened. NOTE: This version of the
+// open functions will only accept: FILE_READONLY and FILE_READWRITE
+// access modes.
+//---------------------------------------------------------------------
+DLL_CODE_API FileSystem::FPTR *FileSystem::Open(const char *fname, FileSystem::AccessMode mode)
+{
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FSOpen(fname, mode);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	FPTR *stream = new FPTR;
+	if (!stream)
+		return 0;
+	if (mode == FILE_READONLY)
+	{
+		// Open for read only access
+		stream->fptr = CreateFile(fname, 
+			GENERIC_READ,
+			FILE_SHARE_READ | FILE_SHARE_WRITE, 
+			NULL, // No security
+			OPEN_EXISTING, 
+			FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, 
+			NULL // No attr. template 
+			);
+		if (stream->fptr == INVALID_HANDLE_VALUE)
+		{
+			delete stream;
+			return 0;
+		}
+	}
+	else 
+	{ // Open for read/write access
+		stream->fptr = CreateFile(fname, 
+			GENERIC_READ | GENERIC_WRITE, 
+			FILE_SHARE_READ | FILE_SHARE_WRITE, 
+			NULL, // No security 
+			OPEN_EXISTING, 
+			FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, 
+			NULL // No attr. template 
+			);
+		
+		if (stream->fptr == INVALID_HANDLE_VALUE)
+		{
+			delete stream;
+			return 0;
+		}
+	}
+	return stream;
+	
+#else // Use the 32-bit stdio version by default
+	FPTR *stream = new FPTR;
+	if (!stream)
+		return 0;
+	
+	if (mode == FILE_READONLY)
+	{
+		// Open for read only access
+		stream->fptr = fopen(fname, "rb");
+		if (stream->fptr == 0)
+		{
+			delete stream; // 01/30/2002: Prevent memory leak
+			return 0;
+		}
+	}
+	else 
+	{ // Open for read/write access
+		stream->fptr = fopen(fname, "r+b");
+		if (stream->fptr == 0)
+		{
+			delete stream; // 01/30/2002: Prevent memory leak
+			return 0;
+		}
+	}
+	return stream;
+#endif
+}
+
+//---------------------------------------------------------------------
+// int Close(FileSystem::FPTR *stream)
+//
+// Close an open file. Returns a non-zero value to  indicate an error 
+// condition or zero if successful.
+//---------------------------------------------------------------------
+DLL_CODE_API int FileSystem::Close(FPTR *stream)
+{
+	if (!stream)
+		return 0; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Close(stream);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	if (!CloseHandle(stream->fptr))
+		return 1;
+	delete stream;
+	stream = 0;
+	return 0;
+	
+#else // Use the 32-bit stdio version by default
+	if (fclose(stream->fptr) != 0)
+		return 1;
+	delete stream;
+	stream = 0; // 02/02/2002: Set pointer to zero after releasing it
+	return 0;
+#endif
+}
+
+//-----------------------------------------------------------------------------
+// int Flush(FileSystem::FPTR *stream)
+//
+// Flush the any open disk buffers. Returns a non-zero value to indicate
+// an error // condition or zero if successful. 
+//
+// ATTENTION: ****MS VC++*** YOU SHOULD LINK YOUR APPLICATION WITH COMMODE.OBJ 
+// file. COMMODE.OBJ changes the global commit flag such that 
+// calling fflush() or _flushall() commits the file buffer to disk.
+// The Windows NT operating system provides the WIN32 API FlushFileBuffers().
+// The _commit() function included with the Visual C++ 32-bit Edition 
+// CRT calls FlushFileBuffers to write buffered data to disk. 
+//-----------------------------------------------------------------------------
+DLL_CODE_API int FileSystem::Flush(FPTR *stream)
+{
+	if (!stream)
+		return 0; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Flush(stream);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	if (!FlushFileBuffers(stream->fptr))
+		return 1;
+	return 0;
+	
+#else // Use the 32-bit stdio version by default
+	int ret = fflush(stream->fptr);
+	if (ret != 0)
+	{
+		TRACE("\nerrno=%d\n\n\n", errno);
+		return 1;       // Error
+	}
+	else
+		return 0;
+#endif
+}
+
+//---------------------------------------------------------------------
+// int Read(FileSystem::FPTR *stream, void *buf, isis::uint32_t bytes)
+//
+// Read a specified number of bytes from the current file position
+// into a memory buffer. Returns the total number if bytes read from the
+// file.
+//---------------------------------------------------------------------
+DLL_CODE_API isis::ulong32_t FileSystem::Read(FPTR *stream, void *buf, 
+											  isis::ulong32_t bytes)
+{
+	if (!stream)
+		return (isis::ulong32_t)0; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Read(stream, buf, bytes);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	DWORD bytes_read; 
+	if (!ReadFile(stream->fptr, (LPVOID)buf, (DWORD)bytes, &bytes_read, NULL))
+	{
+		return (isis::ulong32_t)0;
+	}
+	return (isis::ulong32_t)bytes_read;
+	
+#else // Use the 32-bit stdio version by default
+	return (isis::ulong32_t)fread((unsigned char *)buf, 1, bytes, stream->fptr);
+#endif
+}
+//---------------------------------------------------------------------
+// int Write(FileSystem::FPTR *stream, const void *buf, isis::uint32_t bytes) 
+//
+// Write a specific number of bytes from a memory buffer to the
+// current file position. Returns the total number if bytes written
+// to the file.
+//---------------------------------------------------------------------
+DLL_CODE_API isis::ulong32_t FileSystem::Write(FPTR *stream, const void *buf, 
+				                   isis::ulong32_t bytes) 
+{
+	if (!stream)
+		return (isis::ulong32_t)0; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Write(stream, buf, bytes);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	DWORD bytes_moved; 
+	if (!WriteFile(stream->fptr, (LPVOID)buf, (DWORD)bytes, &bytes_moved, NULL))
+	{
+		return (isis::ulong32_t)0;
+	}
+	return (isis::ulong32_t)bytes_moved;
+	
+#else // Use the 32-bit stdio version by default
+	return (isis::ulong32_t)fwrite((const unsigned char *)buf, 1, bytes, stream->fptr);
+#endif
+}
+//---------------------------------------------------------------------------
+// isis::int32_t Seek(FileSystem::FPTR *stream, isis::int32_t offset,  FileSystemSeekMode mode)
+//
+// Seek to the specified offset starting at the beginning (SEEK_SET),
+// end (SEEK_END) or current offset (SEEK_CUR). Returns a negative 1
+// to indicate an error condition or the current stream position if
+// successful.
+//----------------------------------------------------------------------------
+DLL_CODE_API FAU_t FileSystem::Seek(FPTR *stream, FAU_t offset, FileSystem::SeekMode mode)
+{
+	if (!stream)
+		return (FAU_t) - 1; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Seek(stream, offset, mode);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	if (mode == FILE_SEEK_BEG)
+	{
+		return (FAU_t)SetFilePointer(stream->fptr, offset, NULL, FILE_BEGIN);
+	}
+	else if (mode == FILE_SEEK_CUR)
+	{
+		return (FAU_t)SetFilePointer(stream->fptr, offset, NULL, FILE_CURRENT);
+	}
+	else if (mode == FILE_SEEK_END)
+	{
+		return (FAU_t)SetFilePointer(stream->fptr, offset, NULL, FILE_END);
+	}
+	else 
+	{ // An invalid seek mode was specified
+		return (FAU_t) - 1;
+	}
+	
+#else // Use the 32-bit stdio version by default
+	if (mode == FILE_SEEK_BEG)
+	{
+		return fseek(stream->fptr, offset, SEEK_SET);
+	}
+	else if (mode == FILE_SEEK_CUR)
+	{
+		return fseek(stream->fptr, offset, SEEK_CUR);
+	}
+	else if (mode == FILE_SEEK_END)
+	{
+		return fseek(stream->fptr, offset, SEEK_CUR);
+	}
+	else 
+	{ // An invalid seek mode was specified
+		return (FAU_t) - 1;
+	}
+#endif
+}
+//----------------------------------------------------------------------------
+// isis::int32_t Tell(FileSystem::FPTR *stream)
+//
+//----------------------------------------------------------------------------
+DLL_CODE_API FAU_t FileSystem::Tell(FPTR *stream)
+{
+	if (!stream)
+		return (FAU_t) - 1; // The file stream is not open
+	
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Tell(stream);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	return (__LWORD__)SetFilePointer(stream->fptr, 0, NULL, FILE_CURRENT);
+	
+#else // Use the 32-bit stdio version by default
+	return ftell(stream->fptr);
+#endif
+}
+
+//----------------------------------------------------------------------------
+// int Exists(const char *fname)
+//
+// Returns true if the specified file exists.
+//----------------------------------------------------------------------------
+DLL_CODE_API int FileSystem::Exists(const char *fname)
+// Returns true if the specified file exists.
+{
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64Exists(fname);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	HANDLE hFile = CreateFile(fname, 
+		GENERIC_READ,
+		FILE_SHARE_READ | FILE_SHARE_WRITE, 
+		NULL, // No security
+		OPEN_EXISTING, 
+		FILE_ATTRIBUTE_NORMAL, 
+		NULL // No attr. template
+		);
+	
+	if (hFile == INVALID_HANDLE_VALUE)
+	{
+		return 0;
+	}
+	
+	CloseHandle(hFile);
+	return 1;
+	
+#else // Use the 32-bit stdio version by default
+	struct stat buf;
+	return stat(fname, &buf) ==(FAU_t)0;
+#endif
+}
+
+
+
+//----------------------------------------------------------------------------
+// FAU_t FileSize(const _TCHAR *fname)
+//
+// Returns the length of the specified file or negative 1 to indicate an
+// error condition.
+//----------------------------------------------------------------------------
+DLL_CODE_API FAU_t FileSystem::FileSize(const char *fname)
+{
+#if defined(__64_BIT_FILE_SYSTEM__)
+	return FS64FileSize(fname);
+	
+#elif defined(__WIN32__) && defined(__NTFS__)
+	HANDLE hFile = CreateFile(fname, 
+		GENERIC_READ,
+		FILE_SHARE_READ | FILE_SHARE_WRITE, 
+		NULL, // No security
+		OPEN_EXISTING, 
+		FILE_ATTRIBUTE_NORMAL, 
+		NULL // No attr. template 
+		);
+	
+	// Could not open the specified file
+	if (hFile == INVALID_HANDLE_VALUE)
+	{
+		return (FAU_t) - 1;
+	}
+	
+	BY_HANDLE_FILE_INFORMATION info;
+	
+	if (GetFileInformationByHandle(hFile, &info) == 0)
+	{
+		return (FAU_t) - 1;
+	}
+	CloseHandle(hFile);
+	return (FAU_t)info.nFileSizeLow;;
+	
+#else // Use the 32-bit stdio version by default
+	struct stat buf;
+	int result = stat(fname, &buf);
+	if (result != 0)
+		return -1;
+	return buf.st_size;
+#endif
+}
+
+//  #include <io.h>
+//  #include <share.h>
+
+
+//  int FileSystem::CanOpenForWriting(const _TCHAR *fname)
+//  {
+//  	// Try to open for writing
+//  	int fd = _tsopen(fname, _O_WRONLY | O_BINARY, _SH_DENYWR);
+//  	if (fd == -1)
+//  		return 0;
+//  	_close(fd);
+//  	return 1;
+//  }
+
+//  int FileSystem::CanOpenReadOnly(const _TCHAR *fname)
+//  {
+//  	// Try to open for read only
+//  	int fd = _tsopen(fname, _O_RDONLY | O_BINARY, _SH_DENYNO);
+//  	if (fd == -1)
+//  		return 0;
+//  	_close(fd);
+//  	return 1;
+//  }
+
+//----------------------------------------------------------------------------
+// int IsReadOnly(const _TCHAR* fname)
+//
+// Returns true if the specified file is read only.
+//----------------------------------------------------------------------------
+//  int FileSystem::IsReadOnly(const _TCHAR *fname)
+//  {
+//  	struct _stat buf;
+//  	_tstat(fname, &buf);
+//  	if (buf.st_mode & S_IWRITE)
+//  		return 0;
+//  	return 1;
+//  }
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/FileSystem.h	(revision 21112)
@@ -0,0 +1,121 @@
+/**********************************************************************
+ *
+ * FileSystem.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////
+// FileSystem.h -- The generic file pointer routines are a collection of member
+//                 functions used to wrap  the underlying file system.
+
+//
+
+  
+#ifndef __FILE_SYSTEM__
+#define __FILE_SYSTEM__
+
+// Non-platform specific include files
+#include <string.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdio.h>
+#include "DLLCode.h"
+#include "IsisTypes.h"
+
+namespace FileSystem {
+	
+// ----------------
+// Type definitions 
+// ----------------
+// typedefs for 64-bit and 32-bit file operations and stream position
+#if defined (__64_BIT_FILE_SYSTEM__)
+    typedef isis::int64_t FAU;       // 64-bit File Address Unit, physical file address type
+    typedef isis::int64_t FAU_t;     // Native 64-bit FAU type
+    typedef isis::int64_t StreamPos; // 64-bit stream position
+#else // Use the 32-bit version by default
+	typedef isis::int32_t FAU;       // 32-bit File Address Unit, physical file address type
+	typedef isis::int32_t FAU_t;     // Native 32-bit FAU type
+	typedef isis::int32_t StreamPos; // 32-bit stream position
+#endif
+
+
+// Select the underlying file system 
+#if defined (__64_BIT_FILE_SYSTEM__)
+	#include "FS64.H"              // WIN32 will always use NTFS for large files
+	typedef  FPTR64 FPTR;
+
+
+#elif defined (__WIN32__) && defined (__NTFS__)
+	// WIN32 file I/O API
+	#include <windows.h>
+	#include <io.h>
+	#include <string.h>
+	#include <stdlib.h>
+	#include <fcntl.h>
+	#include <sys/types.h>
+	#include <sys/stat.h>
+	#include <stdio.h>
+	#include "gxdtypes.h"
+	#include "gxheader.h"
+
+	struct DLL_CODE_API FPTR { // Platform specific file pointer type
+		HANDLE fptr;
+	};
+
+#else // Use the stdio version by default. Common to all platforms.
+	// Non-platform specific stdio include files
+	#include <string.h>
+	#include <stdlib.h>
+	#include <fcntl.h>
+	#include <sys/types.h>
+	#include <sys/stat.h>
+	#include <stdio.h>
+	#include "IsisTypes.h"
+	//#include "gxheader.h"
+
+	struct DLL_CODE_API FPTR { // File pointer type
+		 FILE *fptr;
+	};
+#endif // File system type
+
+
+
+    // NOTE: Any underlying file system used must provide the basic
+    // functionallity defined here.
+    DLL_CODE_API FPTR            *Create(const char  *fname);
+    DLL_CODE_API FPTR            *Open(const char *fname, AccessMode mode);
+    DLL_CODE_API int             Close(FPTR *stream);
+    DLL_CODE_API int             Flush(FPTR *stream);
+    DLL_CODE_API isis::ulong32_t Read(FPTR *stream, void *buf, isis::ulong32_t bytes);
+    DLL_CODE_API isis::ulong32_t Write(FPTR *stream, const void *buf, isis::ulong32_t bytes);
+    DLL_CODE_API FAU_t           Seek(FPTR *stream, FAU_t, SeekMode mode);
+    DLL_CODE_API FAU_t           Tell(FPTR *stream);
+    DLL_CODE_API int             Exists(const _TCHAR *fname);
+    DLL_CODE_API FAU_t           FileSize(const _TCHAR *fname);
+    // DLL_CODE_API int             CanOpenForWriting(const _TCHAR *fname);
+    // DLL_CODE_API int             CanOpenReadOnly(const _TCHAR *fname);
+    // DLL_CODE_API int             IsReadOnly(const _TCHAR *fname);
+}
+
+#endif // __FILE_SYSTEM__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.cpp	(revision 21112)
@@ -0,0 +1,113 @@
+/**********************************************************************
+ *
+ * IErrors.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////////////////////////////
+// Error.cpp
+
+// #include "stdafx.h"
+// The Error functions and classes are used to catch and/or
+// record database exceptions that occur during run-time. This
+// implementation can be used with or without C++ built-in exception
+// handling. If C++ exception handling is not enabled with the
+// __CPP_EXCEPTIONS__ macro, then the vbDatabaseError enumerated
+// constants can be used to record database errors.
+#include "IErrors.h"
+
+// NOTE: This array must contain the same number of exceptions as the
+// IsisError enumeration. 
+const int MaxExceptionMessages = 57;
+const char* isisExceptionMessages[MaxExceptionMessages] = {
+  "Database exception: No exception reported",               // None
+  "Database exception: Invalid error code reported",         // Invalid
+  "Database exception: Memory exhausted",
+  "Database exception: Error opening xrf file",
+  "Database exception: Error creating xrf file",
+  "Database exception: Error reading xrf file",
+  "Database exception: Error writing xrf file",
+  "Database exception: xrf file is corrupted",
+  "Database exception: xrf file, invalid block",
+  "Database exception: xrf file, invalid status",
+  "Database exception: Error creating an existing mf file",
+  "Database exception: Error opening mf file",
+  "Database exception: Error opening mf file",
+  "Database exception: Error creating mf file",
+  "Database exception: Error reading mf file",
+  "Database exception: Error writing mf file",
+  "Database exception: Access Violation",                    // AccessViolation
+  "Database exception: Assertion failed",                    // AssertError
+  "Database exception: Wrong object type",                   // BadClassID
+  "Database exception: Bad object address",                  // BadObjectAddress
+  "Database exception: Bad reference",                       // BadReference
+  "Database exception: Cache full",                          // CacheFull
+  "Database exception: Checksum error",                      // ChecksumError
+  "Database exception: Divide by zero error",                // DivideByZero
+  "Database exception: Unexpected end of file was reached",  // EOFError
+  "Database exception: Error closing file",                  // FileCloseError
+  "Database exception: File is corrupt",                     // FileCorrupt
+  "Database exception: Error creating file",                 // CreationError
+  "Database exception: File already exists",                 // FileExists
+  "Database exception: Trying to use a closed file",         // FileNotOpenError
+  "Database exception: File not ready for reading/writing",  // FileNotReady
+  "Database exception: Could not write to file",             // NotWriteable
+  "Database exception: Error opening file",                  // FileOpenError
+  "Database exception: Cannot obtain a file position",       // FilePostionError
+  "Database exception: Error reading from file",             // FileReadError
+  "Database exception: Error seeking in file",               // FileSeekError
+  "Database exception: Error writing to file",               // FileWriteError
+  "Database exception: No database open",                    // NoDatabaseOpen
+  "Database exception: No such file exists",                 // NoFileExists
+  "Database exception: No objects exist",                    // NoObjectsExist
+  "Database exception: Accessing a null pointer",            // NullPtr
+  "Database exception: Object already exists",               // ObjectExists
+  "Database exception: Another object is referencing this file", // Referenced
+  "Database exception: Math overflow error",                 // OverFlow
+  "Database exception: Parse error",                         // ParseError
+  "Database exception: Invalid path",                        // PathError
+  "Database exception: Trying to write to a read-only file", // ReadOnlyFile
+  "Database exception: Stack empty",                         // StackEmpty
+  "Database exception: Stack full",                          // StackFull
+  "Database exception: Synchronization Error",               // SyncError    
+  "Database exception: Math under-flow error",               // UnderFlow
+  "Database exception: Wrong file type",                     // WrongFileType
+
+  // Persistent lock error messages
+  "Database exception: Invalid lock type specified",          // LOCK_TYPE
+  "Database exception: Cannot access the file lock header",   // FLK_ACCESS
+  "Database exception: Cannot lock the file",                 // FLK_ERROR
+  "Database exception: Cannot access the record lock header", // RLK_ACCESS
+  "Database exception: Cannot lock the specified record"      // RLK_ERROR
+};
+
+const char *isisExceptionMessage(IsisError err)
+// Standalone function that returns a null terminated string that can
+// be use to log or print a database exception message.
+{
+  int error = (int)err;
+  if(error > MaxExceptionMessages) error = ISIS_INVALID_CODE;
+
+  // Find the corresponding message in the exception array
+  return isisExceptionMessages[error];
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IErrors.h	(revision 21112)
@@ -0,0 +1,104 @@
+/**********************************************************************
+ *
+ * IErrors.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifndef __ISIS_ERROR_H__
+#define __ISIS_ERROR_H__
+
+enum IsisError { // Database error codes
+  ISIS_NO_ERROR = 0,        // No errors reported
+  ISIS_INVALID_CODE,
+  ISIS_BAD_ALLOC,
+  ISIS_XRF_OPEN_ERROR,      // Xrf open error
+  ISIS_XRF_CREATE_ERROR,    // Xrf creation error
+  ISIS_XRF_READ_ERROR,      // Xrf read error
+  ISIS_XRF_WRITE_ERROR,     // Xrf write error
+  ISIS_XRF_CORRUPT,         // Xrf corrupted
+  ISIS_XRF_INVALID_BLOCK,   // Xrf invalid block
+  ISIS_XRF_INVALID_STATUS,  // Xrf invalid status
+  ISIS_MF_ALREADY_EXIST_ERROR, // Attempt to create an existing mf
+  ISIS_MF_OPEN_ERROR,       // Mf open error
+  ISIS_MF_CREATE_ERROR,     // Mf creation error
+  ISIS_MF_READ_ERROR,       // Mf read error
+  ISIS_MF_WRITE_ERROR,      // Mf write error 
+  ISIS_ASSERT_ERROR,        // Assertion failed
+  ISIS_BAD_CLASS_ID,        // Wrong object type
+  ISIS_BAD_OBJECT_ADDRESS,  // Bad object address  ISIS_BAD_REFERENCE,       // Bad Reference
+  ISIS_CACHE_FULL,          // Cache full
+  ISIS_CHECKSUM_ERROR,      // Checksum Error
+  ISIS_DIVIDEBY_ZERO,       // Divide By Zero Error
+  ISIS_EOF_ERROR,           // Unexpected end of file
+  ISIS_FILE_CLOSE_ERROR,    // Error closing file
+  ISIS_FILE_CORRUPT,        // File corrupted
+  ISIS_FILE_CREATION_ERROR, // Error creating file
+  ISIS_FILE_EXISTS,         // File already exists
+  ISIS_FILE_NOT_OPEN_ERROR, // Trying to use a closed file
+  ISIS_FILE_NOT_READY,      // File not ready (failed or closed file)
+  ISIS_FILE_NOT_WRITEABLE,  // Could not write to file 
+  ISIS_FILE_OPEN_ERROR,     // Error opening file
+  ISIS_FILE_POSITION_ERROR, // Cannot obtain the current file position
+  ISIS_FILE_READ_ERROR,     // Error reading file  
+  ISIS_FILE_SEEK_ERROR,     // Error seeking in file
+  ISIS_FILE_WRITE_ERROR,    // Error writing to file
+  ISIS_NO_DATABASE_OPEN,    // No database open
+  ISIS_NO_FILE_EXISTS,      // No such file exists
+  ISIS_NO_OBJECTS_EXIST,    // No objects exist
+  ISIS_NULL_PTR,            // Accessing a null pointer 
+  ISIS_OBJECT_EXISTS,       // Object already exists
+  ISIS_OPEN_FILE_REFERENCE, // Another object is referencing this file
+  ISIS_OVERFLOW,            // Math overflow
+  ISIS_PARSE_ERROR,         // Parse error
+  ISIS_PATH_ERROR,          // Invalid path
+  ISIS_READ_ONLY_FILE,      // Trying to write to read-only file
+  ISIS_STACK_EMPTY,         // Stack empty
+  ISIS_STACK_FULL,          // Stack full
+  ISIS_SYNC_ERROR,          // Synchronization Error
+  ISIS_UNDERFLOW,           // Math under-flow
+  ISIS_WRONG_FILE_TYPE,     // Wrong file type
+
+  // Persistent lock error codes
+  ISIS_INVALID_LOCK_TYPE,        // Invalid lock type specified
+  ISIS_MFLOCK_ACCESS_ERROR,    // The file lock cannot be accessed
+  ISIS_MFLOCK_ERROR,           // Error locking the file
+  ISIS_RECORDLOCK_ACCESS_ERROR,  // The record lock cannot be accessed
+  ISIS_RECORDLOCK_ERROR          // Error locking a record
+};
+
+// Standalone functions use to report database errors
+const char *isisExceptionMessage(IsisError err);
+
+#ifdef __CPP_EXCEPTIONS__
+// Class declarations for exceptions representing program errors. 
+// This implementation is provided for use C++'s built-in exception
+// handling routines.
+
+// A fatal file error condition has been reached. All database engine
+// calls must catch this exception and evaluate it by testing the
+// vbDatabase or vbDatabase64 internal error variable.
+class CIsisException { };  
+
+#endif // __CPP_EXCEPTIONS__
+
+#endif // __ISIS_ERROR_H__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.cpp	(revision 21112)
@@ -0,0 +1,252 @@
+/**********************************************************************
+ *
+ * IsisDb.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// ISISDB: Isis Master file methods
+////////////////////////////////////////////////////////////
+#include "stdafx.h"
+#include "IsisDb.h"
+#include "XRFFile.h"
+
+
+
+//-------------------------------------------------------------------------------- 
+// IsisDb::IsisDb()
+//
+// Constructor, Makes an ISIS Data Base object that isn't connected to a Master 
+// file, Cross Reference file and Fdt.
+//-------------------------------------------------------------------------------
+IsisDb::IsisDb()
+{
+	bReadOnly_ = false;
+	try
+	{
+        mf_  = new MfFile;
+	    xf_  = new XrfFile; 
+	    fdt_ = new CFdt;
+	}
+	catch(std::bad_alloc)
+	{
+		throw CIsisDbException(ISIS_BAD_ALLOC, __FILE__, __LINE__);
+	}
+}
+ 
+//////////////////////////////
+ 
+IsisDb::~IsisDb( )
+{
+   delete mf_; delete xf_; delete fdt_;
+}
+ 
+ 
+//////////////////////////////
+ 
+int IsisDb::CreateDb(const TCHAR *fname)
+{
+   name = fname;
+   MkNames();
+   int rc = ISIS_NO_ERROR;
+
+   // We should not try to create a mf_ which already exist!
+   if (CFileBase::Exists(mfname_))
+   {
+	   throw CIsisDbException(ISIS_MF_ALREADY_EXIST_ERROR, __FILE__, __LINE__);
+   }
+   else
+   {
+       try
+	   {
+           rc = mf_->CreateMf(mfname_, USERMST);   
+           rc = xf_->CreateXrf(xrfname_);
+           rc = fdt_->Load(fdtname_);
+	   }
+       catch(CIsisDbException)
+	   {
+    	   throw CIsisDbException(ISIS_MF_OPEN_ERROR, __FILE__, __LINE__);
+	   }
+   }
+   
+   return rc;
+}
+ 
+ 
+//////////////////////////////
+ 
+void IsisDb::CloseDb()
+{
+   mf_->CloseMf();
+   xf_->Close();
+}
+ 
+//////////////////////////////
+ 
+void IsisDb::MkNames()
+// Make file names
+{
+    const _TCHAR *s = name.c_str();
+	_TCHAR *ps = new _TCHAR[_tcslen(s)+1];
+	_tcscpy(ps, s);
+	_TCHAR * tmp = _tcsrchr(ps, _T('.'));
+	assert(tmp != NULL);
+    // Terminate the string at the last dot 
+    *tmp = _T('\0');
+
+   _tcscpy(mfname_,ps);
+   _tcscat(mfname_,_T(".mst"));
+   _tcscpy(xrfname_,ps);
+   _tcscat(xrfname_,_T(".xrf"));
+   _tcscpy(fdtname_,ps);
+   _tcscat(fdtname_,_T(".fdt"));
+   _tcscpy(idxname_,ps);
+   _tcscat(idxname_,_T(".idx"));
+   delete[] ps;
+}
+ 
+//////////////////////////////
+ 
+int IsisDb::OpenDb(const TCHAR *fname,  FileSystem::AccessMode mode
+				                              /* = FileSystem::FILE_READWRITE */)
+{
+   name = fname;
+   MkNames();
+   int rc;
+   try
+   {
+     // if (FileSystem::IsReadOnly(mfname_))
+     //	   mode = FileSystem::FILE_READONLY;
+       rc = mf_->OpenMf(mfname_, mode);
+       rc = xf_->OpenXrf(xrfname_, mode);
+       rc = fdt_->Load(fdtname_);
+   }
+   catch(CIsisDbException)
+   {
+	   throw CIsisDbException(ISIS_MF_OPEN_ERROR, __FILE__, __LINE__);
+   }
+
+   return rc;
+}
+ 
+//----------------------------------------------------------------------------------
+// int  IsisDb::ReadDbRecord(long mfn, MfRecord &m)
+//
+// This function reads Master File record with MFN "mfn" into m
+//
+// Returns the status of the record: 
+// Active=0, LogicalyDeleted=1, PhysicalyDeleted=2, EndOfFile=-1, NewRecord = 3
+//----------------------------------------------------------------------------------
+int  IsisDb::ReadDbRecord(long mfn, MfRecord &m)
+{
+   long mfb;
+   int mfp;
+   int status;
+   bool rc;
+
+   try
+   {
+       status = xf_->GetMfp(mfn, mfb, mfp);
+       if (status == Active || status == LogicalyDeleted)
+	       if (!mf_->ReadMfRecord(mfb, mfp, m))
+			   status = -1;    // Error in reading the record!
+   }
+   catch(CIsisDbException)
+   {
+	   throw CIsisDbException(ISIS_MF_OPEN_ERROR, __FILE__, __LINE__);
+   }
+   return status;
+}
+ 
+//--------------------------------------------------------------------------------
+// void CreateDbRecord(MfRecord &m)
+//
+// Create a new record at the end of the master file. The Master file address is
+// set into the pair mfb/mfp and the XRF file is updated.
+//---------------------------------------------------------------------------------
+ void IsisDb::CreateDbRecord(MfRecord &m)
+{
+ 
+}
+
+//--------------------------------------------------------------------------------
+// void CreateDbRecordWithMfn(MfRecord &m)
+//
+// Create a new record at the end of the master file. The Master file address is
+// set into the pair mfb/mfp and the XRF file is updated.
+//---------------------------------------------------------------------------------
+ void IsisDb::CreateDbRecordWithMfn(MfRecord &m)
+{
+ 
+}
+
+ 
+//----------------------------------------------------------------------------------
+// void  UpdateDbRecord(MfRecord &m)
+//
+// Update of existing records. Whenever a record is updated, the system writes
+// the record back to the master file. Where it is written depends on the status
+// of the record when it was initially read.
+//
+//----------------------------------------------------------------------------------
+ void  IsisDb::UpdateDbRecord(MfRecord &m)
+{
+}
+//----------------------------------------------------------------------------------
+// int  IsisDb::DeleteRecord(long mfn)
+//
+// This function logically deletes a Master File record with MFN "mfn".
+// On XRF, xrfmfb is negative and on MST, status is set to 1
+//
+// Returns true if successful 
+//----------------------------------------------------------------------------------
+bool  IsisDb::DeleteRecord(long mfn)
+{
+
+   return false;
+}
+
+long  IsisDb::GetNextActiveMfn(long mfn /* = -1 */)
+{
+	if (mfn == -1)
+		mfn = 1;
+	else
+		mfn++;
+	long  mfb;
+	int  mfp;
+	int iRet = xf_->GetMfp(mfn, mfb, mfp);
+	while (iRet != Active && iRet != EndOfFile)
+	{
+		mfn++;
+        iRet = xf_->GetMfp(mfn, mfb, mfp);
+	}
+	if (iRet == EndOfFile)
+		return 0;
+	else
+		return mfn;
+}
+	
+_TCHAR *IsisDb::GetIdxFileName()
+{
+	return idxname_;
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisDb.h	(revision 21112)
@@ -0,0 +1,130 @@
+/**********************************************************************
+ *
+ * IsisDb.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// Isisdb.h : Class Definition for an ISIS db
+////////////////////////////////////////////////////////////
+ 
+#ifndef __ISISDB_H__
+#define __ISISDB_H__
+
+#include "AbstractIsisDb.h"
+#include "IsisDef.h"
+#ifndef _MFFILE_H_
+#include "MFFile.h"
+#endif
+
+ 
+class IsisDb : public AbstractIsisDb
+{
+	friend class CDBaseFile;
+
+	bool  bReadOnly_;
+    TCHAR xrfname_[_MAX_PATH];
+    TCHAR mfname_[_MAX_PATH];
+    TCHAR fdtname_[_MAX_PATH];
+    TCHAR idxname_[_MAX_PATH];
+protected:
+   void MkNames();
+public:
+   IsisDb();
+   virtual ~IsisDb( );
+   int OpenDb(const TCHAR *fname, FileSystem::AccessMode mode =
+	                                          FileSystem::FILE_READWRITE);
+   int CreateDb(const TCHAR *fname);
+
+
+// Closing the database
+   virtual void CloseDb();       // Close the db
+
+   virtual void ClearDb()   { }    // Clear the db
+   virtual void DestroyDb() { }  // Delete the db
+
+// Recovery
+   virtual bool IsDamaged() { return false;}   // Test if the database is damaged
+   virtual bool Recover()   { return true; }   // Recover the db
+
+// Transactions
+   virtual void BeginTransaction()    { }  // Begin a transaction
+   virtual void CommitTRansaction()   { }  // Commit a transaction
+   virtual void RollbackTransaction() { }  // Rollback a transaction
+   virtual bool InTransaction()       { return false; }  // Test if transaction is pending
+
+   virtual void CreateIndex(int i) { }
+   virtual void DropIndex(int i)   { }
+
+   void  CreateDbRecord(MfRecord &m);
+   void  CreateDbRecordWithMfn(MfRecord &m);
+   void  UpdateDbRecord(MfRecord &m);
+   bool  DeleteRecord(long mfn);
+   int   ReadDbRecord(long mfn, MfRecord &m);
+   int   ReadNextDbRecord(MfRecord &m);
+   void  PrintDbRecord(MfRecord &m);
+   void  PrintDbHdr();
+
+   void SetNextMfn(long mfn) { mf_->setNextMfn(mfn); }
+   long GetNextMfn()        { return mf_->GetNextMfn(); }
+   long GetNextActiveMfn(long mfn=-1);
+
+   SMfHeader& GetSMfHeader() { return mf_->GetSMfHeader(); }
+   _TCHAR *GetIdxFileName();
+
+   bool IsReadOnly() { return bReadOnly_; }
+};
+ 
+//////////////////////////////
+ 
+inline int IsisDb::ReadNextDbRecord(MfRecord &m)
+{
+   return 1;
+}
+ 
+//////////////////////////////
+ 
+inline void  IsisDb::PrintDbRecord(MfRecord &m)
+{
+   mf_->PrintMfRecord(m);
+}
+ 
+//////////////////////////////
+ 
+inline void  IsisDb::PrintDbHdr()
+{
+   mf_->PrintMfHdr();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class IsisDbFactory : public AbstractIsisDbFactory
+{
+public:
+	virtual AbstractIsisDb* MakeIsisDb() { return new IsisDb(); }
+};
+
+/////////////////////////////////////////////////////////////////////////////////
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisDef.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisDef.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisDef.h	(revision 21112)
@@ -0,0 +1,138 @@
+/**********************************************************************
+ *
+ * IsisDef.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifndef __ISIS_DEFINITION_H__
+#define __ISIS_DEFINITION_H__
+
+// disable warning C4786: symbol greater than 255 character,
+// okay to ignore
+#pragma warning(disable: 4786)
+
+#ifndef __ISIS_TYPE_H__
+#include "IsisTypes.h"
+#endif
+
+#include <sstream>
+
+typedef std::string ustring;
+typedef std::string cstring;
+// typedef std::wstring awstring;
+typedef std::stringstream ustringstream;
+typedef std::stringstream cstringstream;
+typedef wchar_t WCHAR;
+
+
+const size_t INVALID_NPOS = ~(size_t)0;
+
+namespace FileSystem {
+    // Generic File access/open mode enumeration
+    enum AccessMode { 
+       FILE_READONLY,   // Open file with read access only
+       FILE_WRITEONLY,  // Open file with write access only
+       FILE_READWRITE,  // Open file with read and write access
+       FILE_CREATE,     // Create the file if it does not exist
+       FILE_NO_CREATE,  // Do not create the file if it does not exist
+       FILE_TRUNCATE,   // Truncate the file  
+       FILE_APPEND,     // Append to the file
+       FILE_SHARE,      // Enable file sharing  (Platform specific)
+       FILE_EXCLUSIVE   // Disable file sharing (Platform specific)
+	};
+
+
+    // Generic File seek mode enumeration
+    enum SeekMode { 
+      FILE_SEEK_BEG, // Seek starting from the beginning of the file
+      FILE_SEEK_CUR, // Seek starting from the current location
+      FILE_SEEK_END  // Seek starting from the end of the file
+    };
+};
+
+#ifndef __FILE_SYSTEM__
+#include "FileSystem.h"
+#endif
+
+#ifndef __FILE_BASE__
+#include "File.h"
+#endif
+
+#ifndef __ISIS_ERROR_H__
+#include "IErrors.h"
+#endif
+
+#ifdef _WIN32
+#ifndef __DEBUG_H__
+#define _DEBUG
+#define _CONSOLE
+#include "debug.h"
+#endif
+#endif
+
+#ifndef __ABSTRACT_ISISDB_H__
+#include "AbstractIsisDb.h"
+#endif
+
+#ifndef __ISISDB_H__
+#include "IsisDb.h"
+#endif
+
+
+#ifndef __MASTER_RECORD_H__
+#include "Master.h"
+#endif
+
+#ifndef __FDT_H__
+#include "Fdt.h"
+#endif
+
+
+enum { RecordDeleted=0, RecordUpdated=1, RecordCreated=2};
+enum { Active=0, LogicalyDeleted=1, PhysicalyDeleted=2,
+       EndOfFile=-1, NewRecord = 3 };
+
+
+
+
+// Class declarations for exceptions representing program errors. 
+// This implementation is provided for use C++'s built-in exception
+// handling routines.
+
+class CIsisDbException { 
+public:
+	int m_iError;
+	CIsisDbException(const int iError, const char* file_name, const long line_number) 
+	{
+		m_iError = iError;
+#ifdef _WIN32
+		AssertFailed(isisExceptionMessage((IsisError) m_iError), file_name, line_number);
+#else
+		std::cout << isisExceptionMessage((IsisError) m_iError) << file_name
+			      << " Line=" << line_number;
+#endif
+	}
+};  
+
+
+#endif // __ISIS_DEFINITION_H__
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisGdl.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisGdl.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisGdl.cpp	(revision 21112)
@@ -0,0 +1,135 @@
+/**********************************************************************
+ *
+ * IsisGdl.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+// IsisGdl.cpp : Defines the entry point for the console application.
+//
+
+#include "stdafx.h"
+#include "IsisDb.h"
+#include <iostream>
+
+void PausePrg()
+{
+	std::cout << "\n";
+	std::cout << "Press enter to continue..." << "\n";
+	std::cin.get();
+}
+
+void ClearInputStream(std::istream &s)
+// Used to clear istream
+{
+	char c;
+	s.clear();
+	while(s.get(c) && c != '\n') { ; }
+}
+
+// Should be called with the master file name as parameter
+// IsisGdl "full_path\master_file_name" 
+// IsisGdl C:\IsisGdl\cds.mst for ex
+
+
+int main(int argc, _TCHAR* argv[])
+{
+
+	if (argc < 2)
+	{
+		std::cout << "usage: IsisGdl master_file_full_name" << std::endl;
+		return 1;
+	}
+	// Instantiation of the IsisDb Object
+	IsisDb db;
+	try
+	{
+		// The file name should be prefixed with the full path
+		// and should be the master file name with extension ".mst"
+		// "C:\IsisGdl\cds.mst" for ex
+		db.OpenDb(argv[1]);
+	}
+	catch(CIsisDbException e)
+	{
+		std::string msg = "Cannot open isis db: ";
+		msg += argv[1];
+		msg += isisExceptionMessage((IsisError) e.m_iError);
+		std::cerr<< msg << std::endl;
+		return false;
+	}
+
+	// Create a Master File object without the Field Definition
+	// Information
+	MfRecord mfr(0); 
+
+	// Extract the number of the next master file number to
+	// assign
+	const long nextMfn = db.GetNextMfn();
+
+
+
+	// Browse the master file in increasing order of Master
+	// file numbers (MFN is the record ID)
+	for (long i=1; i<nextMfn; i++)
+	{
+
+		int status = db.ReadDbRecord(i, mfr);
+
+		if (status == Active || status == LogicalyDeleted)
+		{
+			// If the record is physically on the master file
+			// then get the vector of field tags.
+			
+			// 1) Repetitive fields have the same tag and are stored in
+			//    a single field but are separated by the "%" char
+			// 2) Subfields are separated by "^"letter (^a ^b ^c ...)
+			//
+			// 3) Terms or Phrases between "<>" are terms/Phrases to 
+			//    index
+			std::vector<int> vf = mfr.GetFieldTags();
+			std::cout << "----------" << std::endl;
+			  //				      << "mfn=" << i  
+				// 	  << " nvf=" << vf.size() << std::endl;
+
+			for (int j=0; j<vf.size(); j++)
+			{
+				std::string stemp;
+				// Special case when tag is <=0
+				if (vf[j]<=0 || vf[j]>SHRT_MAX)
+					continue;
+				// Extract the field with tag contained into vf[j]
+				bool rc = mfr.GetField(vf[j], stemp);
+				if (rc)
+				{
+					std::cout << "tag=" << vf[j]
+					          << " data=" << stemp << std::endl;
+				}
+			}
+
+		}
+		else
+			continue;
+		// PausePrg();
+
+	}
+	return 0;
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisTypes.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisTypes.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisTypes.h	(revision 21112)
@@ -0,0 +1,98 @@
+/**********************************************************************
+ *
+ * IsisTypes.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+///////////////////////////////////////////////////////////////////////////////
+//  isisType.h 
+
+
+#ifndef __ISIS_TYPE_H__
+#define __ISIS_TYPE_H__
+
+#include "DLLCode.h"
+
+#ifdef __USE_ISOC99 // get double_t type from math.h
+#include <math.h>
+#endif
+namespace isis
+{
+
+
+// ----------------------------------------
+// Type definitions common to all platforms
+// ----------------------------------------
+typedef char            int8_t;    // 8-bit signed value
+typedef unsigned char   uint8_t;   // 8-bit unsigned value
+typedef int             int32_t;   // Word size (4 bytes)
+typedef unsigned int    uint32_t;  // Unsigned word size (4 bytes)
+typedef long            long32_t;  // Long word (4 bytes)
+typedef unsigned long   ulong32_t; // Unsigned long word (4 bytes)
+typedef short           int16_t;   // short word (2 bytes)
+typedef unsigned short  uint16_t;  // Unsigned short word (2 bytes)
+#ifndef __USE_ISOC99 // gnu lic math.h has a double_t
+typedef double          double_t;  // Double precision floating point (8 bytes)
+#endif
+
+
+// --------------------------------------------------------------
+// Platform/compiler specific INT64 types
+// --------------------------------------------------------------
+#if defined (__64_BIT_FILE_SYSTEM__)
+#include "TYPES64.H" 
+#endif
+} // namespace isis
+
+namespace isis
+{
+    typedef  isis::int16_t  db_t;
+    typedef  isis::int16_t  server_t;
+    typedef  isis::int16_t  IsisClientId;
+    typedef  isis::uint32_t mfn_t;
+}
+// typedef char            int8_t;    // 8-bit signed value
+typedef unsigned char   uint8_t;   // 8-bit unsigned value
+typedef int             int32_t;   // Word size (4 bytes)
+typedef unsigned int    uint32_t;  // Unsigned word size (4 bytes)
+typedef long            long32_t;  // Long word (4 bytes)
+typedef unsigned long   ulong32_t; // Unsigned long word (4 bytes)
+typedef short           int16_t;   // short word (2 bytes)
+typedef unsigned short  uint16_t;  // Unsigned short word (2 bytes)
+#ifndef __USE_ISOC99 // gnu lic math.h has a double_t
+typedef double     double_t;  // Double precision floating point (8 bytes) // ****
+#endif
+
+typedef uint32_t        xrf_page_no_t;
+
+typedef uint32_t        mst_page_no_t;
+typedef uint16_t        mst_offset_t;
+typedef uint32_t        mfn_t;
+typedef uint32_t        mfrl_t;
+
+typedef isis::uint32_t file_size_t;
+
+
+#endif // __ISIS_TYPE_H__
+
+
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.cpp	(revision 21112)
@@ -0,0 +1,183 @@
+/**********************************************************************
+ *
+ * IsisUtil.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include "stdafx.h"
+#include "IsisUtil.h" 
+#include <algorithm>
+
+
+//---------------------------------------------------------------------------------
+// bool RepFieldSep(TCHAR c)
+//
+// This function is a predicate that takes a TCHAR and returns a value that indicates
+// if that TCHAR is a repeteable field separator.
+//----------------------------------------------------------------------------------
+bool RepFieldSep(char c)
+{
+	return (c == SYSRSEP);
+}
+
+//--------------------------------------------------------------------------------
+// vector<string> SplitOccurences(const cstring& s)
+//
+// This function takes as input a string which contains a field and return a 
+// vector<string> which will contain an entry for each field occurence.
+//--------------------------------------------------------------------------------
+std::vector<cstring> SplitOccurences(const cstring& s)
+{
+	typedef cstring::const_iterator iter;
+	std::vector<cstring> ret;
+
+	iter i = s.begin();
+	//TRACE("\ni=%s",i);
+	while (i != s.end())
+	{
+		// find end of occurence
+		iter j = std::find_if(i,s.end(),RepFieldSep);
+
+		// copy the characters in [i, j]
+		ret.push_back(cstring(i, j));
+		//TRACE("\ni=%s j=%s string(i,j)=%s", i, j, string(i,j).c_str());
+
+		i = j;
+		if (j != s.end())
+		    i++;
+	}
+	return ret;
+}
+
+
+bool WSubFieldSep(WCHAR c)
+{
+	return (c == SUBFIELD_SEP);
+}
+
+//--------------------------------------------------------------------------------
+// vector<cstring> SplitSubfields(const cstring& s, TCHAR sc)
+//
+// This function takes as input a string which contains a field and return a 
+// vector<string> which will contain an entry for each subfield occurrence.
+//--------------------------------------------------------------------------------
+std::vector<cstring> SplitSubfields(const cstring& s, TCHAR sc/*=0*/)
+{
+	typedef cstring::const_iterator iter;
+	std::vector<cstring> ret;
+
+	iter i= s.begin();
+	iter j;
+	while (i < s.end())
+	{
+		// Look for subfield separator (^)
+		iter k =  std::find_if(i ,s.end(),WSubFieldSep);
+		if (k == s.end())
+		{
+			j = s.end(); // separator not found
+		}
+		else
+		{
+			// Special case when field is terminated by "^x" or "^"
+			if (k==s.end()-1 || k==s.end()-2)
+				break;
+			i = k + 2;  // separator found, skip next char
+			assert(i<s.end());
+			j = std::find_if(i ,s.end() ,WSubFieldSep);
+		}
+		ret.push_back(cstring(i, j));
+		i = j;
+	}
+	return ret;
+}
+
+
+//--------------------------------------------------------------------------------
+// vector<string> SplitOccurences(const awstring& s)
+//
+// This function takes as input a string which contains a field and return a 
+// vector<string> which will contain an entry for each field occurence.
+//--------------------------------------------------------------------------------
+//  std::vector<awstring> SplitOccurences(const awstring& s)
+//  {
+//  	typedef awstring::const_iterator iter;
+//  	std::vector<awstring> ret;
+
+//  	iter i = s.begin();
+//  	//TRACE("\ni=%s",i);
+//  	while (i != s.end())
+//  	{
+//  		// find end of occurence
+//  		iter j = std::find_if(i,s.end(),RepFieldSep);
+
+//  		// copy the characters in [i, j]
+//  		ret.push_back(awstring(i, j));
+//  		//TRACE("\ni=%s j=%s string(i,j)=%s", i, j, string(i,j).c_str());
+
+//  		i = j;
+//  		if (j != s.end())
+//  		    i++;
+//  	}
+//  	return ret;
+//  }
+
+
+bool SubFieldSep(TCHAR c)
+{
+	return (c == SUBFIELD_SEP);
+}
+
+//--------------------------------------------------------------------------------
+// vector<awstring> SplitSubfields(const awstring& s, TCHAR sc)
+//
+// This function takes as input a string which contains a field and return a 
+// vector<string> which will contain an entry for each subfield occurrence.
+//--------------------------------------------------------------------------------
+//  std::vector<awstring> SplitSubfields(const awstring& s, TCHAR sc/*=0*/)
+//  {
+//  	typedef awstring::const_iterator iter;
+//  	std::vector<awstring> ret;
+
+//  	iter i = s.begin();
+//  	//TRACE("\ni=%s",i);
+//  	bool found=false;
+//  	while (i != s.end())
+//  	{
+//  		// find index of subfield separator (^)
+//  		iter j = std::find_if(i,s.end(),SubFieldSep);
+//  		if (found && j<s.end()-1)
+//  		{
+//  		    // copy the characters in [i, j]
+//  			ret.push_back(awstring(i+1, j));
+//  		}
+//  		if (sc==0 || s.at(j-s.begin()+1)==sc)
+//  		    found = true;
+//  		else
+//  		    found = false;
+
+
+//  		i = j+1;
+//  		if (j != s.end())
+//  		    i++;
+//  	}
+//  	return ret;
+//  }
Index: /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/IsisUtil.h	(revision 21112)
@@ -0,0 +1,35 @@
+/**********************************************************************
+ *
+ * IsisUtil.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef __ISIS_UTIL__
+#define __ISIS_UTIL__
+#include <vector>
+#include <string>
+std::vector<cstring> SplitOccurences(const cstring &s);
+// std::vector<awstring> SplitOccurences(const awstring &s);
+
+std::vector<cstring> SplitSubfields(const cstring &s, TCHAR sc=0);
+// std::vector<awstring> SplitSubfields(const awstring &s, TCHAR sc=0);
+#endif // __ISIS_UTIL_
Index: /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.cpp	(revision 21112)
@@ -0,0 +1,470 @@
+/**********************************************************************
+ *
+ * MFFile.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// MfFile.cpp : Master file object methods using blocked I/O
+//
+////////////////////////////////////////////////////////////
+#include "stdafx.h"
+#include "IsisDb.h"
+#include "Checking.h"
+
+
+
+//---------------------------------------------------------------------------------- 
+// MfFile::MfFile() :
+//
+// Constructor, makes a Masterfile object that isn't connected to a file
+//---------------------------------------------------------------------------------- 
+MfFile::MfFile() : BlkFile(), hdr_created_(false), nextaddr_(MFCTLLEN)
+{
+    lastWrittenBlock_ = -1;
+}
+ 
+//----------------------------------------------------------------------------------
+// int MfFile::OpenMf(const TCHAR *fname, 
+//                    FileSystem::AccessMode mode /* = FileSystem::FILE_READWRITE */)
+//
+// Opens a Masterfile object
+//---------------------------------------------------------------------------------- 
+int MfFile::OpenMf(const TCHAR *fname, FileSystem::AccessMode mode
+				                              /* = FileSystem::FILE_READWRITE */)
+{
+	int rc;
+	try
+	{
+	    rc = CFileBase::Open(fname, mode);
+		hdr_created_ = true;
+	    ReadMfHdr();
+		PrintMfHdr();
+		Checking::CheckMfHeader(fname, mfh_);
+	}
+	catch (CFileBaseException e)
+	{
+        throw CIsisDbException(ISIS_MF_OPEN_ERROR, __FILE__, __LINE__);
+	}
+    return rc;
+}
+ 
+//----------------------------------------------------------------------------------
+// int MfFile::createMf(const TCHAR *fname, int  type)
+//
+// Creates a Masterfile object
+//---------------------------------------------------------------------------------- 
+int MfFile::CreateMf(const TCHAR *fname, int  type)
+{
+    return 0;
+}
+ 
+//////////////////////////////
+MfFile::~MfFile()
+{
+}
+ 
+//----------------------------------------------------------------------------------
+// void  MfFile::ReadMfRecord(long mfb, int mfp, MfRecord &m)
+//
+// Reads the master file record beginning at address (mfb,mfp) into m
+//----------------------------------------------------------------------------------
+bool  MfFile::ReadMfRecord(long mfb, int mfp, MfRecord &m)
+{
+	return m.Read(*this, mfb, mfp);
+}
+
+//-------------------------------------------------------------------------------
+//  void WriteMfRecord(long mfb, int mfp, MfRecord &m)
+//
+//  Helper protected method, which writes the Master file record m at address 
+//  (mfb, mfp)   mfb (1:MAXMFB),   mfp (0: MFBLKSIZE)
+//-------------------------------------------------------------------------------
+void  MfFile::WriteMfRecord(long mfb, int mfp, MfRecord &m)
+{
+	m.Write(*this, mfb, mfp);
+} 
+
+
+//--------------------------------------------------------------------------------
+// void CreateMfRecord(MfRecord &m, long &mfb, int &mfp)
+//
+//  Create a new record. The MFN to be assigned is obtained from the field nxtmfn_
+//  in the Master file header. Then addMfRecord is called for adding the record
+//  at the end of the master file, at the position indicated by the fields
+//  nxtmfb_/nxtmfb_. The next MFN to be assigned is updated.
+//
+//--------------------------------------------------------------------------------
+void  MfFile::CreateMfRecord(MfRecord &m, long &mfb, int &mfp)
+{
+}
+
+void  MfFile::AppendNewBlock()
+{
+}
+
+//--------------------------------------------------------------------------------
+// void AddMfRecord(MfRecord &m, long &mfb, int &mfp)
+//
+// Add the record at the end of the master file, at the position indicated by
+// the fields nxtmfb_/nxtmfp_. (nxtmfp_ is in range [1:MFBLKSIZE])
+// Return the address where the record was stored into mfb & mfp
+//--------------------------------------------------------------------------------
+void  MfFile::AddMfRecord(MfRecord &m, long &mfb, int &mfp)
+{
+}
+ 
+ 
+//--------------------------------------------------------------------------------
+// void UpdateMfRecord(MfRecord &m, long &mfb, int &mfp)
+//
+// Update a Master file record.If possible, i.e. if the record length was not
+// increased, the record is written back at its original location otherwise it is
+// written at the end of the file. In both cases, MFBWB/MFBWP are not changed.
+// 
+//--------------------------------------------------------------------------------
+void  MfFile::UpdateMfRecord(MfRecord &m, long &mfb, int &mfp)
+{
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////
+// methods for dumping file and record headers
+
+ 
+void MfFile::PrintMfHdr(SMfHeader &h)
+// Prints header record
+{
+}
+ 
+//////////////////////////////
+ 
+void MfFile::PrintMfRecord(MfRecord &m)
+// Print a master file record structure
+{
+	m.Dump();
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// Methods for accessing records directly
+ 
+//----------------------------------------------------------------------------------
+// void MfFile::ReadMfRecord(long daddr, MfRecord &m)
+//
+// Reads a master file record into the master record object m.
+// Non blocked version
+//----------------------------------------------------------------------------------
+void MfFile::ReadMfRecord(long daddr, MfRecord &m)
+{
+	m.Clear();
+    ReadLeader(daddr, m);       // Read record leader (Leader is added to record)
+	m.ExtractSubLeader();
+    
+	//TRACE("\nReadMfRecord -- mfn=%ld mfrl=%d", m.GetMfn(), m.GetMfrl());
+	//assert(m.CheckIntegrity());
+	int mfrl = m.GetMfrl();
+	if (mfrl < 0) mfrl = -mfrl; 	// mfrl can be negative!!!!
+	char* p = new char[mfrl];
+    CFileBase::Fetch(p, mfrl, daddr);
+	m.AddToRecord(p+GetLeaderSize(), mfrl-GetLeaderSize());
+	delete[] p;
+    nextaddr_ += mfrl;
+ 	int mfb = nextaddr_ / MFBLKSIZE + 1;
+    int mfp = nextaddr_ % MFBLKSIZE;
+    if (mfp > MFBLKSIZE - GetMinLeaderSize())
+	{
+		mfb++;
+	    mfp = 0;
+	}
+	nextaddr_ = (mfb-1)*MFBLKSIZE + mfp;
+	m.DecomposeRecord();
+}
+
+//---------------------------------------------------------------------------------
+// void  MfFile::WriteMfRecord(long daddr, MfRecord &m)
+//
+// Stores Master record object m to disk address daddr
+// Non blocked version
+//---------------------------------------------------------------------------------
+void  MfFile::WriteMfRecord(long daddr, MfRecord &m)
+{
+}
+/////////////////////////////////////////////////////////////////////////////////////
+// Master file locking methods
+
+
+ 
+//-----------------------------------------------------------------------------------
+// void MfFile::InitFileLockHdr(SMfLockHeader &hdr)
+//
+// Initialize a the lock header for a newly constructed file lock header.
+//-----------------------------------------------------------------------------------
+void MfFile::InitFileLockHdr(SMfLockHeader &hdr)
+{
+	hdr.lockProtect_ = (uint32_t)0;
+	hdr.readLock_    = (uint32_t)0;
+	hdr.writeLock_   = (uint32_t)0;
+}
+
+//-----------------------------------------------------------------------------------
+// FileError MfFile::WriteFileLockHdr(const SMfLockHeader &hdr)
+//
+// Write the lock header to the file. 
+// Returns a non-zero value to indicate an error condition or zero if successful.
+//-----------------------------------------------------------------------------------
+FileError MfFile::WriteFileLockHdr(const SMfLockHeader &hdr)
+{
+	
+	return CFileBase::Store(&hdr, sizeof(SMfLockHeader), sizeof(SMfHeader));
+}
+
+//-----------------------------------------------------------------------------------
+// FileError MfFile::ReadFileLockHdr(SMfLockHeader &hdr)
+//
+// Read the lock header from the file.
+// Returns a non-zero value to indicate an error condition or zero if successful.
+//-----------------------------------------------------------------------------------
+FileError MfFile::ReadFileLockHdr(SMfLockHeader &hdr)
+{
+	
+	return CFileBase::Fetch(&hdr, sizeof(SMfLockHeader), sizeof(SMfHeader));
+}
+
+//-----------------------------------------------------------------------------------
+// int MfFile::LockFile(CFileMgrLockType l_type) 
+//
+// Lock the file and set the lock access mode to shared or exclusive.
+// Returns a non-zero value if the file cannot be locked or the lock variable cannot
+// be changed because it is exclusive or another thread is currently updating it. 
+//-----------------------------------------------------------------------------------
+int MfFile::LockFile(MfLockType l_type) 
+{
+	// Read the file lock to ensure that this file has a lock header
+	// and to keep the header in sync during multiple file access.
+	SMfLockHeader lock_header;
+	if (ReadFileLockHdr(lock_header) != FILE_NO_ERROR)
+		return fileError_;
+	
+	// Cannot modifiy this lock until the thread holding it releases the
+	// lock header.
+	if (lock_header.lockProtect_ != 0)
+	{
+		fileError_ = ISIS_MFLOCK_ACCESS_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(fileError_);
+#else
+		return fileError_;
+#endif
+	}
+	
+	switch (l_type)
+	{
+		case MF_WRITE_LOCK :
+			// Cannot obtain an exclusive lock until all the threads in the
+			// read lock queue have finished
+			if ((lock_header.readLock_ != 0) || 
+				(lock_header.writeLock_ != 0))
+			{
+				fileError_ = ISIS_MFLOCK_ERROR;
+	#ifdef __CPP_EXCEPTIONS__
+				throw CFileBaseException(fileError_);
+	#else
+				return fileError_;
+	#endif
+			}
+			else 
+			{
+				lock_header.writeLock_ = 1;
+			}
+			
+			lock_header.lockProtect_ = 1;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			
+			// Skip over the read lock member
+			SeekTo(sizeof(SMfHeader) + (sizeof(uint32_t) * 2));
+			if (CFileBase::Store(&lock_header.writeLock_, sizeof(uint32_t)) !=
+				FILE_NO_ERROR)
+				return fileError_;
+			
+			lock_header.lockProtect_ = (uint32_t)0;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			break;
+			
+		case MF_READ_LOCK :
+			// This lock is exclusively owned and cannot be read locked
+			// until the thread holding this lock release the exclusive lock.
+			if (lock_header.writeLock_ == 1)
+			{
+				fileError_ = ISIS_MFLOCK_ERROR;
+	#ifdef __CPP_EXCEPTIONS__
+				throw CFileBaseException(fileError_);
+	#else
+				return fileError_;
+	#endif
+			}
+			
+			if (lock_header.readLock_ < 0xFFFFFFFF)
+			{
+				lock_header.readLock_++;
+			}
+			else 
+			{
+				fileError_ = ISIS_MFLOCK_ERROR;
+	#ifdef __CPP_EXCEPTIONS__
+				throw CFileBaseException(fileError_);
+	#else
+				return fileError_;
+	#endif
+			}
+			
+			lock_header.lockProtect_ = 1;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			
+			if (CFileBase::Store(&lock_header.readLock_, sizeof(uint32_t)) !=
+				FILE_NO_ERROR)
+				return fileError_; 
+			
+			lock_header.lockProtect_ = (uint32_t)0;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			break;
+			
+		default: // Invalid lock type specified
+			fileError_ =  ISIS_INVALID_LOCK_TYPE;
+	#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(fileError_);
+	#else
+			return fileError_;
+	#endif
+	}
+	
+	return fileError_ = fileError_ = FILE_NO_ERROR;
+}
+
+//-----------------------------------------------------------------------------------
+// int MfFile::UnlockFile(CFileMgrLockType l_type)
+//
+// Unlock the file. Returns a non-zero value if the file cannot be unlocked or the
+// lock variable cannot be changed because it is exclusive or another thread is
+// currently updating it. 
+//-----------------------------------------------------------------------------------
+int MfFile::UnlockFile(MfLockType l_type)
+{
+	
+	// Read the file lock to ensure that this file has a lock header
+	// and to keep the header in sync during multiple file access.
+	SMfLockHeader lock_header;
+	if (ReadFileLockHdr(lock_header) != FILE_NO_ERROR)
+		return fileError_;
+	
+	// Cannot modifiy this lock until the thread holding it releases the
+	// lock header.
+	if (lock_header.lockProtect_ != 0)
+	{
+		fileError_ = ISIS_MFLOCK_ACCESS_ERROR;
+#ifdef __CPP_EXCEPTIONS__
+		throw CFileBaseException(fileError_);
+#else
+		return fileError_;
+#endif
+	}
+	
+	switch (l_type)
+	{
+		case MF_WRITE_LOCK :
+			if (lock_header.writeLock_ == 0)
+				return FILE_NO_ERROR;
+			else
+				lock_header.writeLock_ = 0;
+			
+			lock_header.lockProtect_ = 1;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			
+			// Skip over the read lock member
+			SeekTo(sizeof(SMfHeader) + (sizeof(uint32_t) * 2));
+			if (CFileBase::Store(&lock_header.writeLock_, sizeof(uint32_t)) !=
+				FILE_NO_ERROR)
+				return fileError_;
+			
+			lock_header.lockProtect_ = (uint32_t)0;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			break;
+			
+		case MF_READ_LOCK :
+			if (lock_header.readLock_ == 0)      // This record is not locked
+				return fileError_ = FILE_NO_ERROR;
+			else if (lock_header.readLock_  > 0) // Prevent read lock rollover
+				lock_header.readLock_--;
+			else
+				lock_header.readLock_ = 0;
+			
+			lock_header.lockProtect_ = 1;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			
+			if (CFileBase::Store(&lock_header.readLock_, sizeof(uint32_t)) !=
+				FILE_NO_ERROR)
+				return fileError_; 
+			
+			lock_header.lockProtect_ = (uint32_t)0;
+			if (CFileBase::Store(&lock_header.lockProtect_, sizeof(uint32_t),
+				sizeof(SMfHeader)) != FILE_NO_ERROR)
+				return fileError_;
+			break;
+			
+		default: // Invalid lock type specified
+			fileError_ =  FMGR_INVALID_LOCK_TYPE;
+	#ifdef __CPP_EXCEPTIONS__
+			throw CFileBaseException(fileError_);
+	#else
+			return fileError_;
+	#endif
+	}
+	
+	return fileError_ = fileError_ = FILE_NO_ERROR;
+}
+
+//-----------------------------------------------------------------------------------
+// FileError MfFile::ResetFileLock()
+//
+// Reset the file's lock header. This function will clear all the file lock fields
+// without testing the lock or the lock protect. Returns a non-zero value to indicate
+// an error condition or zero if successful.
+//-----------------------------------------------------------------------------------
+FileError MfFile::ResetFileLock()
+{
+	SMfLockHeader lock_header;
+	InitFileLockHdr(lock_header);
+	return WriteFileLockHdr(lock_header);
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/MFFile.h	(revision 21112)
@@ -0,0 +1,318 @@
+/**********************************************************************
+ *
+ * MFFile.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// MFFILE.H: Class definition for blocked I/O on the master
+//           file
+////////////////////////////////////////////////////////////
+ 
+#ifndef _MFFILE_H_
+#define _MFFILE_H_
+#ifndef _MASTER_H_
+#include "Master.h"
+#endif
+#ifndef _BLKFILE_H_
+#include "BlkFile.h"
+#endif
+#include <string>
+
+enum MfLockType
+{
+	MF_READ_LOCK,
+	MF_WRITE_LOCK
+};
+
+struct SMfLockHeader
+{
+	long lockProtect_; // Serialize access to the file lock members
+	long readLock_;    // Shared (or read_only) file lock
+	long writeLock_;   // Exclusive (or write-only) file lock
+
+	SMfLockHeader() : lockProtect_(0), readLock_(0), writeLock_(0)
+	{}
+};
+
+enum  { USERMST=0, MSGMST=1 };
+//  binary  hex
+//  ------  ---
+//  0000    0
+//  0001    1
+//  0010    2
+//  0011    3
+
+class MfAddress
+{
+private:
+	long  m_mfb;
+	short m_mfp;
+	char  m_status;
+public:
+	MfAddress() { m_status = 0x00; }
+	MfAddress(long mfb, short mfp)
+	{ ASSERT(mfb>0); ASSERT(mfp>=0); m_mfb = mfb; m_mfp = mfp; m_status = 0x03; }
+
+	long getMfb()          { ASSERT(m_status & 0x01); return m_mfb; }
+	short getMfp()         { ASSERT(m_status & 0x02); return m_mfp; } 
+
+	void setMfb(long mfb)  { ASSERT(mfb>0);  m_mfb = mfb; m_status |= 0x01; }
+	void setMfp(short mfp) { ASSERT(mfp>=0); m_mfp = mfp; m_status |= 0x02; }
+	bool isValid()         { return (m_status == 0x03); }
+};
+ 
+/////////////////////////////////////////////////////////////////////////
+// Master File header record aggregate structure
+//
+struct  SMfHeader 
+{
+   ISIS_LONG   ctlmfn_;    // Always 0
+   ISIS_LONG   nxtmfn_;    // MFN to be assigned to next record created
+   ISIS_LONG   nxtmfb_;    // Last block number allocated (1st is 1)
+   ISIS_INT    nxtmfp_;    // Offset to next available position in last block (1:MFBLKSIZE)
+   ISIS_INT    mftype_;    // Type of master file (USERMST or MSGMST)
+   ISIS_LONG   reccnt_;
+   ISIS_LONG   mfcxx1_;
+   ISIS_LONG   mfcxx2_;
+   ISIS_LONG   mfcxx3_;
+public:
+   SMfHeader(const char *dbname="");
+   ~SMfHeader()                 { }
+};
+ 
+//--------------------------------------------------------------------------
+// SMfHeader::SMfHeader(const char *dbname /* ="" */)
+//
+// Default constructor, initializes the Master file header to default values
+//--------------------------------------------------------------------------
+inline SMfHeader::SMfHeader(const char *dbname /* ="" */)
+{
+   ctlmfn_ = 0;
+   nxtmfn_ = 1L;
+   nxtmfb_ = 1L;
+   nxtmfp_ = MFCTLLEN+1;
+   mftype_ = USERMST;
+   reccnt_ = mfcxx1_=mfcxx2_=mfcxx3_=0;
+}
+ 
+//////////////////////
+// Master file class
+//
+class MfFile : public BlkFile  
+{
+   friend class IsisDb;
+   friend class MfRecord;
+
+protected:
+   SMfHeader mfh_;            // Header structure
+   long nextaddr_;            // Address of next record
+   char buf_[MFBLKSIZE];      // Buffer to hold 1 Master File block
+   bool hdr_created_;         // True if 1st block exists
+   int  fileError_;
+   long lastWrittenBlock_;
+
+protected:  
+   void WriteMfRecord(long mfb, int mfp, MfRecord &r);
+   void AppendNewBlock();
+
+public:
+   void setNextMfn(long mfn)       // Set MFN to be assigned to next record created
+   { mfh_.nxtmfn_ = mfn; }
+
+protected:
+   void setNextMfb(long nxtmfb_)  // Set last block number allocated (1st is 1)
+   { mfh_.nxtmfb_ = nxtmfb_; }
+   void setNextMfp(int nxtmfp_)   // Set offset to next available position in last block
+   { mfh_.nxtmfp_ = nxtmfp_; }
+   int setm_mftype(int mftype_)     // Set type of master file (USERMST or MSGMST)
+   { mfh_.mftype_ = mftype_; }     
+   virtual void ReadBlk(void* d, long b)  { BlkFile::ReadBlk(d, b);  }
+   virtual void WriteBlk(void* d, long b) { BlkFile::WriteBlk(d, b); }
+   void ReadMfHdr(SMfHeader &h);
+   void ReadMfHdr();
+   void WriteMfHdr(SMfHeader &h);
+   void WriteMfHdr();
+   void SetLastWrittenBlock(long b) { lastWrittenBlock_ = b; }
+public:
+   MfFile();
+   virtual ~MfFile();
+
+   int  OpenMf(const TCHAR* name, FileSystem::AccessMode mode = FileSystem::FILE_READWRITE);
+   int  CreateMf(const TCHAR* name, int MSTtype = USERMST);
+   void CloseMf() { if (ReadyForWriting()) WriteMfHdr(); Close(); }
+
+   // Accessors for the Master File Header
+   long GetNextMfn()        // Get MFN to be assigned to next record created
+   { return mfh_.nxtmfn_; }
+   long GetNextMfb()        // Get last block number allocated (1st is 1)
+   { return mfh_.nxtmfb_; }
+   int GetNextMfp()         // Get offset to next available position in last block
+   { return mfh_.nxtmfp_; }
+   int GetMftype()          // Get type of master file (USERMST or MSGMST)
+   { return mfh_.mftype_; }   
+   SMfHeader& GetSMfHeader() { return mfh_; }
+
+
+   void CreateMfRecord(MfRecord &m, long &mfb, int &mfp);
+   void AddMfRecord(MfRecord &m, long &mfb, int &mfp);
+   void UpdateMfRecord(MfRecord &m, long &mfb, int &mfp);
+   bool ReadMfRecord(long mfb, int mfp, MfRecord &r);
+
+   void ReadMfRecord(long addr, MfRecord &r);
+   void WriteMfRecord(long addr, MfRecord &r);
+   void RewindMf();
+   bool ReadMfNext(MfRecord &r);
+   void WriteMfNext(MfRecord &r);
+   void ReadLeader(long daddr, MfRecord& m)
+   {
+	   m.ReadLeader(*this, daddr);
+   }
+
+
+   void LockMfHdr();
+   void UnlockMfHdr();
+   void LockMfr(long mfn);
+   void UnlockMfr(long mfn);
+
+public: // File lock functions
+	void InitFileLockHdr(SMfLockHeader &hdr);
+	FileError ResetFileLock();
+	FileError WriteFileLockHdr(const SMfLockHeader &hdr);
+	FileError ReadFileLockHdr(SMfLockHeader &hdr);
+	int LockFile(MfLockType l_type = MF_WRITE_LOCK);
+	int UnlockFile(MfLockType l_type = MF_WRITE_LOCK);
+
+
+   void PrintMfRecord(MfRecord &r);
+   void PrintMfHdr(SMfHeader &h);
+   void PrintMfHdr();
+};
+ 
+
+class CMfIterator
+{
+	const MfFile& m_mf;
+	long cur;
+public:
+	CMfIterator(const MfFile& mf) : m_mf(mf) { }
+
+	bool More() const;
+	void Advance();
+
+	MfRecord Current() 
+	{ 
+		MfRecord mfr(NULL);
+		
+		return mfr;
+	}
+};
+
+ 
+//////////////////////////////
+ 
+inline void  MfFile::LockMfHdr()
+//  Locks the master file header record
+{
+	double_t dwPos   = 0;
+	double_t dwCount = sizeof(SMfHeader);
+	//CFile::LockRange(dwPos, dwCount);
+}
+ 
+//////////////////////////////
+ 
+inline void MfFile::UnlockMfHdr()
+//  Unlocks the master file header record
+{
+	double_t dwPos   = 0;
+	double_t dwCount = sizeof(SMfHeader);
+	//CFile::UnlockRange(dwPos, dwCount);
+}
+ 
+//////////////////////////////
+ 
+inline void MfFile::ReadMfHdr()
+//  Reads the master file header record
+{
+   ReadBlk(buf_, 1L);
+   TRACE(_T("\nsizeof(SMfHeader)=%d"), sizeof(SMfHeader));
+   ::MoveMemory(&mfh_, buf_, sizeof(SMfHeader));
+}
+ 
+//////////////////////////////
+ 
+inline void MfFile::WriteMfHdr()
+//  Updates the master file header record
+{
+    if (hdr_created_) 
+	   ReadBlk(buf_, 1L);
+    TRACE(_T("\nsizeof(SMfHeader)=%d"), sizeof(SMfHeader));
+    ::MoveMemory(buf_, &mfh_, sizeof(SMfHeader));
+    WriteBlk(buf_, 1L);
+}
+ 
+ 
+//////////////////////////////
+ 
+inline void MfFile::ReadMfHdr(SMfHeader &h)
+//  Reads the master file header record
+{
+   ReadBlk(buf_, 1L);
+   ::MoveMemory(&h, buf_, sizeof(SMfHeader));
+}
+ 
+//////////////////////////////
+ 
+inline void MfFile::WriteMfHdr(SMfHeader &h)
+//  Updates the master file header record
+{
+   if (hdr_created_)
+	   ReadBlk(buf_, 1L);
+   ::MoveMemory(buf_, &h, sizeof(SMfHeader));
+   WriteBlk(buf_, 1L);
+}
+ 
+ 
+inline void MfFile::PrintMfHdr()
+{
+   PrintMfHdr(mfh_);
+}
+ 
+//////////////////////////////
+ 
+inline bool MfFile::ReadMfNext(MfRecord &r)
+// Reads the next physically record
+{
+	if (nextaddr_ >= CFileBase::FileSize(file_name))
+	    return false;
+   ReadMfRecord(nextaddr_, r);
+   return true;
+}
+ 
+//////////////////////////////
+ 
+inline void MfFile::WriteMfNext(MfRecord &r)
+// Writes physically at next position
+{
+   WriteMfRecord(nextaddr_, r);
+}
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Makefile.in	(revision 21112)
@@ -0,0 +1,33 @@
+prefix	    = @prefix@
+exec_prefix = @exec_prefix@
+bindir	    = @bindir@
+
+CXX	    = @CXX@
+CXXFLAGS    = @CXXFLAGS@
+EXEEXT	    = @EXEEXT@
+
+INSTALL     = @INSTALL@
+
+LDFLAGS     = @LDFLAGS@
+
+ISIS_OBJS   = IsisGdl.o CacheMan.o Checking.o CRC32.o Fdt.o File.o \
+              FileSystem.o IErrors.o IsisDb.o IsisUtil.o Master.o MFFile.o \
+              stdafx.o Unimarc.o XRFFile.o
+
+
+all: IsisGdl$(EXEEXT)
+
+IsisGdl$(EXEEXT): $(ISIS_OBJS)
+	$(CXX) -o IsisGdl$(EXEEXT) $(ISIS_OBJS) $(LDFLAGS)
+
+$(ISIS_OBJS) :%.o :%.cpp
+	$(CXX) -c $(CXXFLAGS) -o $@ $<
+
+install: all
+	$(INSTALL) IsisGdl$(EXEEXT) $(bindir)
+
+clean:
+	rm -f IsisGdl$(EXEEXT) $(ISIS_OBJS)
+
+distclean: clean
+	rm -f Makefile config.status config.cache config.log
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Master.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Master.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Master.cpp	(revision 21112)
@@ -0,0 +1,1071 @@
+/**********************************************************************
+ *
+ * Master.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+//  MASTER.CPP: Master file record object methods
+//                                                                        
+////////////////////////////////////////////////////////////
+#include "stdafx.h"
+#include <iomanip>
+#include "XRFFile.h"
+#include "IsisUtil.h" 
+#include "Master.h"
+#include "MFFile.h"
+#include "IsisDb.h"
+#include "Unimarc.h"
+#include <algorithm>
+
+
+using namespace std;
+
+   
+
+char MfRecord::m_buf[MFBLKSIZE];      // Buffer to hold 1 File block
+
+
+
+/////////////////////////////////////////////////////////////////////
+// LeaderAndDirectory methods
+
+int LeaderAndDirectory::GetFieldSize()
+{
+	int field_len = 0;
+	for (int i=0; i!=nvf_; ++i)
+		field_len  += dir_[i].len_;
+	return field_len;
+}
+
+bool LeaderAndDirectory::operator==(const LeaderAndDirectory& rhs) const
+{
+    if (mfn_ != rhs.mfn_)         // Master file number
+		return false;
+    if (mfrl_ != rhs.mfrl_)       // Record length
+		return false;
+	if (mfbwb_ != rhs.mfbwb_)     // Backward pointer block number
+		return false;
+    if (mfbwp_ != rhs.mfbwp_)     // Backward pointer offset
+		return false;
+    if (base_ != rhs.base_)       // Offset to begining of variable fields
+		return false;
+    if (nvf_ != rhs.nvf_)         // Number of directory entries
+		return false;
+    if (status_ != rhs.status_)   // 1 = DELETED
+		return false;
+	for(std::vector<DirEntry>::size_type i=0; i!=nvf_; i++) // Directory
+	{
+		if (dir_[i]!=rhs.dir_[i])
+		    return false;
+	}
+    return true;
+}
+
+bool LeaderAndDirectory::operator!=(const LeaderAndDirectory& rhs) const
+{
+    if (mfn_ != rhs.mfn_)         // Master file number
+		return true;
+    if (mfrl_ != rhs.mfrl_)       // Record length
+		return true;
+	if (mfbwb_ != rhs.mfbwb_)     // Backward pointer block number
+		return true;
+    if (mfbwp_ != rhs.mfbwp_)     // Backward pointer offset
+		return true;
+    if (base_ != rhs.base_)       // Offset to begining of variable fields
+		return true;
+    if (nvf_ != rhs.nvf_)         // Number of directory entries
+		return true;
+    if (status_ != rhs.status_)   // 1 = DELETED
+		return true;
+	for(std::vector<DirEntry>::size_type i=0; i!=nvf_; i++) // Directory
+	{
+		if (dir_[i]!=rhs.dir_[i])
+		    return true;
+	}
+    return false;
+}
+//-------------------------------------------------------------------------------
+// ostream& operator<<(ostream& s, const LeaderAndDirectory& leader_directory)
+//
+// Overloaded instance of the output operator for LeaderAndDirectory objects
+//---------------------------------------------------------------------------------
+cstringstream& operator<<(cstringstream& s, const LeaderAndDirectory& leader_directory)
+{
+	s.write((char *) &leader_directory.mfn_,    sizeof(ISIS_LONG));
+	s.write((char *) &leader_directory.mfrl_,   sizeof(ISIS_INT));
+	s.write((char *) &leader_directory.mfbwb_,  sizeof(ISIS_LONG));
+	s.write((char *) &leader_directory.mfbwp_,  sizeof(ISIS_INT));
+#ifdef UNIX_BIREME
+	ISIS_INT dummy = 0;
+	s.write((char *) &dummy,   sizeof(ISIS_INT));
+#endif
+
+	s.write((char *) &leader_directory.base_,   sizeof(ISIS_INT));
+	s.write((char *) &leader_directory.nvf_,    sizeof(ISIS_INT));
+	s.write((char *) &leader_directory.status_, sizeof(ISIS_INT));
+
+	int n = leader_directory.dir_.size();
+    assert(n==leader_directory.nvf_);	
+	for(vector<DirEntry>::size_type i=0; i!=leader_directory.dir_.size(); i++)
+	{
+	     assert(leader_directory.dir_[i].tag_>0 && leader_directory.dir_[i].tag_<SHRT_MAX);
+		 DirEntry de = leader_directory.dir_[i];
+         s.write((char *) &de, GetDirEntrySize());
+	}
+	
+	return s;
+}
+
+//----------------------------------------------------------------------------------
+// istream& operator>>(istream& s, LeaderAndDirectory& leader_directory)
+//
+// Overloaded instance of the input operator for LeaderAndDirectory objects
+//----------------------------------------------------------------------------------
+cstringstream& operator>>(cstringstream& s, LeaderAndDirectory& leader_directory)
+{
+	if (!s.good()) return s;
+
+	ios::iostate err = ios::goodbit;
+	try
+	{
+		s.read((char *) &leader_directory.mfn_,    sizeof(ISIS_LONG));
+	    s.read((char *) &leader_directory.mfrl_,   sizeof(ISIS_INT));
+	    s.read((char *) &leader_directory.mfbwb_,  sizeof(ISIS_LONG));
+	    s.read((char *) &leader_directory.mfbwp_,  sizeof(ISIS_INT));
+#ifdef UNIX_BIREME
+	    ISIS_INT dummy;
+	    s.read((char *) &dummy,   sizeof(ISIS_INT));
+#endif
+	    s.read((char *) &leader_directory.base_,   sizeof(ISIS_INT));
+	    s.read((char *) &leader_directory.nvf_,    sizeof(ISIS_INT));
+	    s.read((char *) &leader_directory.status_, sizeof(ISIS_INT));
+
+		if (!leader_directory.dir_.empty())
+			leader_directory.dir_.clear();
+
+		// Try to recover if nvf_ = 0
+		if (leader_directory.nvf_==0)
+			leader_directory.nvf_ = (leader_directory.base_ - GetLeaderSize()) / GetDirEntrySize();
+
+		if (leader_directory.nvf_>0)
+		{
+			for(vector<DirEntry>::size_type i=0; i!=leader_directory.nvf_; i++)
+			{
+				DirEntry de;
+		        s.read((char *) &de, GetDirEntrySize());
+			    // Don't check integrity of directory entries here
+			    leader_directory.dir_.push_back(de);
+			}
+
+		}
+	}
+	catch(bad_alloc&)
+	{
+		err |= ios::badbit;
+		ios::iostate exception_mask = s.exceptions();
+		if (  (exception_mask & ios::failbit) &&
+			!(exception_mask & ios::badbit))
+		{
+			s.setstate(err);
+		}
+		else if (exception_mask & ios::badbit)
+		{
+		  s.setstate(err);
+		  // try { s.setstate(err); }
+		  // catch( ios::failure& ) { }
+			throw;
+		
+		}
+	}
+    catch(...)
+	{
+		err |= ios::failbit;
+		ios::iostate exception_mask = s.exceptions();
+		if ( (exception_mask & ios::badbit) &&
+			(err & ios::badbit))
+		{
+			s.setstate(err);
+		}
+		else if (exception_mask & ios::failbit)
+		{
+		  s.setstate(err);
+		  // try { s.setstate(err); }
+		  // catch( ios::failure& ) { }
+			throw;
+
+		}
+	}
+
+	if (err) s.setstate(err);
+	return s;
+}
+
+
+//////////////////////////////////////////////////////////////////////////////////
+//
+
+//  void MfRecord::Copy(UMfRecord& rhs)           
+//  {
+//  	Clear();
+//  	pFdt_      = rhs.pFdt_;
+//      state_     = Record_Created;
+	
+//  	// 1 - Prepare the LeaderAndDirectory, directory, end data part
+//  	int n   = 0;
+//  	int pos = 0;
+
+//  	cstring all_fields_mb;
+//      leader_directory_.dir_.clear();	
+//  	all_fields_mb.erase();
+//  	for (vector<int>::size_type i=0; i!=rhs.tags_.size(); ++i)
+//  	{
+//  		bool bRep = false;
+//  		if (rhs.pFdt_)
+//  		{
+//  			if (rhs.pFdt_->GetEntryAt(i).rep)
+//  			   bRep = true;
+//  		}
+//  		// Get the field, all occurences combined
+//  		int tag = rhs.tags_[i];
+//  	    awstring ws;
+//  		bool rc = rhs.GetField(tag, ws);
+//  		vector<awstring> occ;
+//  		if (bRep)
+//  			vector<awstring> occ = SplitOccurences(ws);
+//  		else
+//  			occ.push_back(ws);
+//          // one entry per occurence
+//  		int nocc = occ.size();
+//  		for (vector<awstring>::size_type j=0; j!= nocc; ++j)
+//  		{
+//  			DirEntry de;
+//  			de.tag_ = tag;
+//  			de.pos_ = pos;
+//  			cstring cs; // For storing the unicode string
+//  			de.len_ = UnicodeToCodePage(occ[j], cs);
+//              leader_directory_.dir_.push_back(de);
+//  			all_fields_mb += cs;
+//  			n++;
+//  			pos += occ[j].length();
+			
+//  		}
+//  	}
+
+//  	leader_directory_.nvf_  = n;
+//  	leader_directory_.base_ = GetLeaderSize() + leader_directory_.nvf_*GetDirEntrySize();
+//  	leader_directory_.mfrl_ = ComputeMfrl();
+
+//  	// 2   - Make record
+
+//  	record_.erase(); // Empty the record
+
+//  	// Prepare the binary part, note that this part must be  independent
+//  	// of the character set.
+
+//  	cstringstream ost(std::ios::out | std::ios::binary);   
+//  	ost << leader_directory_;   // Output the LeaderAndDirectory to a string
+
+	
+//  	AddToRecord(ost.str().c_str(), ost.str().length());
+
+//  	// Here comes the part which is dependent from the character set,
+//  	// the data itself.
+//  	AddToRecord((char*)all_fields_mb.c_str(), all_fields_mb.length());
+
+//  	DecomposeRecord();
+//  }
+
+//  MfRecord::MfRecord(const UMfRecord& rhs)     // Copy constructor
+//  { 
+//     Copy(rhs);
+//  }
+
+//  MfRecord& MfRecord::operator=(const UMfRecord& rhs) // Assignment
+//  {
+//     if (&leader_directory_ != &rhs.leader_directory_)
+//     {
+//     	   Copy(rhs);
+//     }
+//     return *this;
+//  }
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// Accessors
+ 
+ 
+
+////////////////////////////////////////////////////////////////////////////////////////
+// Methods working on the in memory image of the record
+ 
+//-------------------------------------------------------------------------------
+// bool  MfRecord::GetField(int tag, string& s)
+//
+// This function takes as input a tag and will place the field into the string
+// s. It will return true if the field exists and false otherwise.
+//-------------------------------------------------------------------------------
+bool  MfRecord::GetField(int tag, ustring& s)
+{
+	PRECONDITION(tag>=0 && tag < SHRT_MAX);
+
+	map<int, ustring>::iterator it = fields_.find(tag);
+
+	bool ret = false;
+	if (it != fields_.end())
+	{
+		s   = it->second;
+		ret = true;
+	}
+	return ret;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Modifiers, They synchronize the record structure with the internal structures
+
+
+//---------------------------------------------------------------------------------
+// AddField --- Add a field to the master record structure.
+//            
+//--------------------------------------------------------------------------------- 
+bool MfRecord::AddField(int tag, ustring& s)
+{
+	PRECONDITION(tag>=0 && tag < SHRT_MAX);
+
+	map<int, ustring>::iterator it = fields_.find(tag);
+
+	bool ret = false;
+	if (it == fields_.end())
+	{
+		fields_.insert(make_pair(tag, s));
+		tags_.push_back(tag);
+		sort(tags_.begin(), tags_.end());
+		ret = true;
+		ComposeRecord();
+	}
+	return ret;
+}
+//---------------------------------------------------------------------------------
+// AddOccurence --- Add an occurence to an existing field.
+//--------------------------------------------------------------------------------- 
+bool MfRecord::AddOccurence(int tag, ustring& o)
+{
+	PRECONDITION(tag>=0 && tag < SHRT_MAX);
+
+	map<int, ustring>::iterator it = fields_.find(tag);
+
+	bool ret = false;
+	if (it != fields_.end())
+	{
+	    it->second += SYSRSEP;
+		it->second += o;
+		ret = true;
+		ComposeRecord();
+	}
+	return ret;
+}
+
+//-----------------------------------------------------------------------------
+// bool  MfRecord::DeleteField(int tag)
+//
+//-----------------------------------------------------------------------------
+bool  MfRecord::DeleteField(int tag)
+{
+	PRECONDITION(tag>=0 && tag < SHRT_MAX);
+
+	map<int, ustring>::iterator it = fields_.find(tag);
+
+	bool ret = false;
+	if (it != fields_.end())
+	{
+		fields_.erase(it);
+		ret = true;
+		vector<int>::iterator j = find(tags_.begin(), tags_.end(), tag);
+		if (j != tags_.end())
+			tags_.erase(j);
+		ComposeRecord();
+	}
+	return ret;
+}
+
+//-------------------------------------------------------------------------------
+// bool ReplaceField(int tag, string& s)
+//
+//-------------------------------------------------------------------------------
+bool MfRecord::ReplaceField(int tag, ustring& s)
+{
+    
+	PRECONDITION(tag>=0 && tag < SHRT_MAX);
+
+	map<int, ustring>::iterator it = fields_.find(tag);
+
+	bool ret = false;
+	if (it != fields_.end())
+	{
+		it->second  = s;
+		ret         = true;
+	}
+	else
+		AddField(tag, s);
+	ComposeRecord();
+
+	return ret;
+}
+
+ustring MfRecord::GetAllFields()
+{
+	cstring all_fields; // As a string of ANSI characters
+	all_fields.erase();
+	all_fields = record_.substr(GetBase(), GetFieldSize());
+	return ustring(all_fields);
+}
+
+void MfRecord::ConvertToAnsi()
+{
+	/*
+	int mfrl = GetMfrl();
+	int size1 = record_.length();
+	if (GetFieldSize()==0)
+		return;
+	cstring s = record_.substr(GetBase(), GetFieldSize());
+	CString cs(s.c_str());
+	cs.OemToAnsi();
+	TRACE("\nbase=%d GetFieldSize=%d",GetBase(),GetFieldSize());
+	record_.replace(GetBase(),GetBase()+GetFieldSize(),
+		                                 cs.GetBuffer(GetFieldSize()));
+	int size2 = record_.length();
+	cs.ReleaseBuffer();
+	int padding = size1-size2;
+	for (int i=0; i<padding; i++)
+		record_ += " ";
+*/
+}
+
+long MfRecord::GetDirectorySize()
+{
+	return (GetNvf()*GetDirEntrySize());
+}
+
+long MfRecord::ComputeMfrl()
+{
+	return (GetLeaderSize() + GetDirectorySize() + GetFieldSize());
+}
+
+//--------------------------------------------------------------------------------------
+// void MfRecord::ComposeRecord()
+//
+// Build the record into continuous bytes for output
+//--------------------------------------------------------------------------------------
+void MfRecord::ComposeRecord()
+{
+	PRECONDITION(tags_.size() == fields_.size());
+	
+	// 1 - Prepare the LeaderAndDirectory, directory, end data part
+	int n   = 0;
+	int pos = 0;
+
+	ustring all_fields;
+    leader_directory_.dir_.clear();	
+	all_fields.erase();
+	for (vector<int>::size_type i=0; i!=tags_.size(); ++i)
+	{
+		int tag = tags_[i];
+		bool bRep = false;
+		if (pFdt_)
+		{
+			if (pFdt_->GetEntryAt(i).rep)
+			   bRep = true;
+		}
+		vector<ustring> occ;
+		if (bRep)
+			vector<ustring> occ = SplitOccurences(fields_[tags_[i]]);
+		else
+			occ.push_back(fields_[tags_[i]]);
+        // one entry per occurence
+		int nocc = occ.size();
+		for (vector<ustring>::size_type j=0; j!= nocc; ++j)
+		{
+			DirEntry de;
+			de.tag_ = tag;
+			de.pos_ = pos;
+			de.len_ = occ[j].length();
+            leader_directory_.dir_.push_back(de);
+			all_fields += occ[j];
+			n++;
+			pos += occ[j].length();
+			
+		}
+	}
+
+	leader_directory_.nvf_  = n;
+	leader_directory_.base_ = GetLeaderSize() + leader_directory_.nvf_*GetDirEntrySize();
+	leader_directory_.mfrl_ = ComputeMfrl();
+
+	// 2   - Make record
+
+	record_.erase(); // Empty the record
+
+	// Prepare the binary part, note that this part must be  independent
+	// of the character set.
+
+	cstringstream ost(std::ios::out | std::ios::binary);   
+	ost << leader_directory_;   // Output the LeaderAndDirectory to a string
+
+	
+	AddToRecord(ost.str().c_str(), ost.str().length());
+
+	// Here comes the part which is dependent from the character set,
+	// the data itself.
+	cstring all_fields_ansi(all_fields); 
+	AddToRecord(all_fields_ansi.c_str(), all_fields_ansi.length());
+	
+}
+
+//-------------------------------
+// Helper functions for decompose
+//-------------------------------
+
+bool MfRecord::CheckLeaderAndDirectory()
+{
+	if (GetMfrl() < 0)                  // Negative mfrl
+	{
+		//VLOG("DecomposeRecord -  Warning! Negative mfrl - mfn=%d", GetMfn());
+		SetMfrl(-GetMfrl());
+	}
+
+	if (GetMfrl()<=GetBase())
+	{
+		//VLOG("DecomposeRecord - Error! mfn=%d GetMfrl()=%d > GetBase()=%d", 
+		//	  GetMfn(),  GetMfrl(), GetBase());
+		return false;
+	}
+
+	
+
+	if (GetLeaderSize()>GetBase())
+	{
+		//VLOG("DecomposeRecord - Error! mfn=%d GetLeaderSize()=%d > GetBase()=%d", 
+		//	  GetMfn(),  GetLeaderSize(), GetBase());
+		return false;
+	}
+	if (GetNvf()==0) 
+	{
+	    // Null nvf      
+		//VLOG("DecomposeRecord -  Warning! mfn=%d nvf=0", GetMfn());
+		//SetNvf((GetBase() - GetLeaderSize()) / GetDirEntrySize());
+		return false;
+	}
+
+	if (GetMfn()<=0 || GetMfrl()<=0 || GetBase()<0 || GetNvf()<0 || GetStatus()< 0) 
+	{
+		//VLOG("DecomposeRecord - Error! mfn=%d GetMfrl()=%d GetLeaderSize()=%d GetBase()=%d GetNvf()=%d GetStatus()=%d", 
+		//      GetMfn(), GetMfrl(), GetLeaderSize(), GetBase(), GetNvf(), GetStatus());
+		return false;
+	}
+
+
+	int base = GetLeaderSize() + leader_directory_.nvf_*GetDirEntrySize();
+
+	if (record_.length() <= base)
+	{
+        //VLOG("DecomposeRecord - Error! mfn=%d record_.length()=%d < base=%d",
+		//	 GetMfn(), record_.length(), base);
+		return false;
+	}
+
+	int field_len = 0;
+	for (int i=0; i!=GetNvf(); ++i)
+	{
+		if (GetFieldTag(i)<0 || GetFieldTag(i)>SHRT_MAX ||
+			GetFieldPos(i)<0 || GetFieldPos(i)>SHRT_MAX ||
+			GetFieldLen(i)<0 || GetFieldLen(i)>SHRT_MAX)
+		{
+            //VLOG("DecomposeRecord - Error! mfn=%d Bad dir entry i=%d tag=%d pos=%d len=%d",
+			//    	GetMfn(), GetFieldTag(i), GetFieldPos(i), GetFieldLen(i));
+			return false;
+		}
+		field_len  += GetFieldLen(i);
+	}
+	
+    for (int j=0; j!=GetNvf(); ++j)
+	{
+		if (GetFieldPos(j)>=field_len || GetFieldPos(j)+GetFieldLen(j)>field_len)
+		{
+            //VLOG("DecomposeRecord - Error! mfn=%d Bad dir entry j=%d tag=%d pos=%d len=%d",
+			//    	GetMfn(), GetFieldTag(j), GetFieldPos(j), GetFieldLen(j));
+			return false;
+		}
+	}
+	// Check for bad values
+    if (GetBase() != base)	
+		SetBase(base);
+	
+	if (record_.length() < base + field_len)
+	{
+        //VLOG("DecomposeRecord - Error! mfn=%d record_.length()=%d < base=%d + field_len=%d",
+        //GetMfn(), record_.length(), base, field_len);  
+		return false;
+	}
+	return true;
+}
+
+void MfRecord::MakeTagToFieldMap()
+{
+	cstring all_fields_ansi;
+	all_fields_ansi.erase();
+	all_fields_ansi = record_.substr(leader_directory_.base_, GetFieldSize());
+	PRECONDITION(fields_.empty() && tags_.empty());
+
+
+	// Build the map from tags to fields
+	for (int i=0; i!=GetNvf(); ++i)
+	{
+		//int    all_len = all_fields.length();
+		//if (pos>all_len || pos+len>all_len)
+		//	return false;
+		cstring field_ansi = 
+			   all_fields_ansi.substr(GetFieldPos(i), GetFieldLen(i));
+		ustring field(field_ansi);
+	    map<int, ustring>::iterator it = fields_.find(GetFieldTag(i));
+	    if (it != fields_.end())
+		{
+			// Repeatable field, several occ
+		    it->second += SYSRSEP;
+			it->second += field;
+		}
+		else
+		{
+		    fields_.insert(make_pair(GetFieldTag(i), field));
+			
+		}
+		
+	}
+}
+void MfRecord::MakeTagVector()
+{
+	// Build a vector of tags
+	map<int, ustring>::const_iterator it;
+    for ( it = fields_.begin(); it != fields_.end(); ++it )
+		tags_.push_back(it->first);
+
+	// Sort in ascending order
+	sort(tags_.begin(), tags_.end());
+}
+
+
+//--------------------------------------------------------------------------------------
+// bool MfRecord::DecomposeRecord()
+//
+// Make a memory version of the record to facilitate manipulations.
+//--------------------------------------------------------------------------------------
+bool MfRecord::DecomposeRecord()
+{
+	PRECONDITION(!record_.empty());
+
+	if (record_.length() <= GetLeaderSize())
+	{
+		//VLOG("DecomposeRecord - Error! record_.length()=%d < GetLeaderSize()=%d",record_.length(), GetLeaderSize());
+		return false;
+	}
+
+	// Input the LeaderAndDirectory from a string
+    cstringstream ist(record_,cstringstream::in|cstringstream::binary);
+	if (!(ist >> leader_directory_))
+		return false;
+
+	if (!CheckLeaderAndDirectory())
+	{
+		//VLOG("Record rejected mfn=%ld", GetMfn());
+		return false;
+	}
+
+	//VLOG("Record accepted mfn=%ld", GetMfn());
+
+    MakeTagToFieldMap();
+    MakeTagVector();
+
+	return true;
+}
+
+
+
+void MfRecord::ExtractSubLeader()
+{
+	//PRECONDITION(!record_.empty() && record_.length()>=sizeof(leader_directory_.mfn_)+sizeof(leader_directory_.mfrl_));
+
+	leader_directory_.mfn_  = *(reinterpret_cast<ISIS_LONG*>((char *) record_.c_str()));
+	leader_directory_.mfrl_ = *(reinterpret_cast<ISIS_INT*> ((char *)(record_.c_str()+sizeof(ISIS_LONG))));
+	if (leader_directory_.mfrl_ < 0) leader_directory_.mfrl_ = -leader_directory_.mfrl_; 	// mfrl can be negative!!!!
+
+	//POSTCONDITION(leader_directory_.mfn_>0 && leader_directory_.mfrl_>=0);
+}
+
+bool MfRecord::CheckLeader()
+{
+	int leader_size = GetLeaderSize();
+	if (leader_directory_.mfn_<=0)
+		return false;
+	if (abs(leader_directory_.mfrl_)<leader_size)
+		return false;
+	return true;
+}
+
+
+bool MfRecord::CheckIntegrity()
+{
+	int leader_size = GetLeaderSize();
+	if (leader_directory_.mfn_<=0)
+		return false;
+	if (abs(leader_directory_.mfrl_)<leader_size)
+		return false;
+	if (leader_directory_.nvf_<0)
+		return false;
+	//if (leader_directory_.base_ != leader_directory_size + leader_directory_.nvf_*sizeof(MAXDIRL))
+	//	return false;
+	if (leader_directory_.status_ == 1)
+		return true;
+	else if (leader_directory_.status_ == 0)
+		return true;
+	else
+		return false;
+
+	return true;
+}
+ 
+
+void  MfRecord::Dump()
+{
+
+	ustringstream os;
+    os.str("");
+
+
+    os <<  "\n================= entry in MfRecord::Dump==============";
+    os <<  "\n   MFN      " << GetMfn();
+    os <<  "\n   MFRL     " << GetMfrl();
+    os <<  "\n   MFBWB    " << GetMfbwb();
+    os <<  "\n   MFBWP    " << GetMfbwp();
+    os <<  "\n   BASE     " << GetBase();
+    os <<  "\n   NVF      " << GetNvf();
+    os <<  "\n   STATUS   " << GetStatus();
+ 	//TRACE(os.str().c_str());
+
+
+	for (int i=0; i<GetNvf(); i++)
+	{
+		  ustring stemp;
+		  bool rc = GetField(GetDirEntry(i).tag_, stemp);
+		  int len;
+		  if (rc)
+		  {
+		       len = 50;
+		       if (stemp.length()<len)
+			      len = stemp.length();
+		  }
+		  ustring fld = (rc==false) ? _T("Empty") : stemp.substr(0,len);
+	      os.str("");
+		  os << _T(" tag=") << ISIS_INT(GetDirEntry(i).tag_)
+             << _T(" pos=") << ISIS_INT(GetDirEntry(i).pos_)
+			 << _T(" len=") << ISIS_INT(GetDirEntry(i).len_)
+			 << _T(" ")     << fld.c_str() << std::endl; 
+          TRACE(_T("\n%s"),os.str().c_str());   
+	  }
+
+      TRACE("\n---------------- exit in MfRecord::Dump-------------");
+}
+
+
+
+
+bool MfRecord::CheckRecord()
+{
+	PRECONDITION(!record_.empty());
+
+	if (record_.length() < GetLeaderSize())
+	{
+		TRACE(_T("\nWrong LeaderAndDirectory in record!"));
+		return false;
+	}
+
+    cstringstream ist(record_,cstringstream::in|cstringstream::binary);
+
+	LeaderAndDirectory leader_directory;
+	ist >> leader_directory;
+
+	if (leader_directory.mfrl_ < 0)                  // Negative mfrl
+	{
+		//VLOG("CheckRecord -  Warning! Negative mfrl - mfn=%d", GetMfn());
+	}
+	
+	if (leader_directory.nvf_<=0) 
+	{
+		//VLOG("CheckRecord -  Warning! mfn=%d nvf=0", leader_directory.mfn_);
+	}
+
+	if (leader_directory.mfn_<=0 || leader_directory.mfrl_<=0 ||
+		leader_directory.base_<0 || leader_directory.nvf_ < 0 ||
+		leader_directory.status_ < 0) 
+	{
+		//VLOG("CheckRecord - Corrupted record! leader_directory.mfn_=%d leader_directory.mfrl_=%d leader_directory.base_=%d leader_directory.nvf_=%d leader_directory.status_=%d",
+        //      leader_directory.mfn_, leader_directory.mfrl_,
+		//	  leader_directory.base_, leader_directory.nvf_,
+		//	  leader_directory.status_);
+		return false;
+	}
+
+
+	int base = GetLeaderSize() + GetDirectorySize();
+
+	if (record_.length() < base)
+	{
+		//VLOG("CheckRecord - Corrupted record! mfn=%d record_.length()=%d base=%d",
+		//	  leader_directory.mfn_, record_.length(), base);
+		return false;
+	}
+
+	int field_len = 0;
+	for (int i=0; i!=leader_directory.nvf_; ++i)
+	{
+		DirEntry de = leader_directory.dir_[i];
+//  		if (de.tag_<0 || de.tag_>SHRT_MAX || de.pos_<0 || de.pos_>SHRT_MAX || de.len_<0 || de.len_>SHRT_MAX)
+//  		{
+//  		    //VLOG("CheckRecord - Corrupted record! mfn=%d tag=%d pos=%d len=%d",
+//  			//	  leader_directory.mfn_, de.tag_, de.pos_, de.len_);
+//  			return false;
+//  		}
+		if (i>0)
+		{
+			if(de.pos_ != leader_directory.dir_[i-1].pos_+leader_directory.dir_[i-1].len_)
+			{
+		    //VLOG("CheckRecord - Position not consistent - mfn=%d",
+			//	  leader_directory.mfn_);
+			return false;
+			}
+		}
+		field_len  += de.len_;
+	}
+	
+
+	// Check for bad values
+    if (leader_directory.base_ != base)	
+		leader_directory.base_ = base;
+	
+	if (record_.length() < base + field_len)
+	{
+		    //VLOG("CheckRecord - Corrupted record, record length < base+field_len  - mfn=%ld",
+			//	  leader_directory.mfn_);
+		return false;
+	}
+
+	return true;
+}
+
+void MfRecord::ReadLeader(MfFile& f, long daddr)
+{
+    int leader_size = GetLeaderSize();
+    char* buffer = new char[leader_size];
+    f.Fetch(buffer, leader_size, daddr);  // Read record leader
+    Clear();
+    AddToRecord(buffer, leader_size);
+    delete[] buffer;
+}
+
+bool  MfRecord::Read(MfFile& f, long mfb, int mfp)
+{
+    //TRACE("\nMfFile::ReadMfRecord -- mfb=%ld mfp=%d", mfb, mfp);
+	PRECONDITION(mfb>0 && mfb<=MAXMFB);
+    long  blk;
+    int   mfrr, offs, l;
+
+	try
+	{
+		Clear();
+	    // Save original mfb, mfp pair into m
+	    SetMfbRead(mfb);
+	    SetMfpRead(mfp);
+	    SetState(MfRecord::Record_Read);
+
+	    // Be sure that mfp is in range [0:MFBLKSIZE-1]
+	    mfp = mfp%MFBLKSIZE;
+		assert(mfp>=0 && mfp<MFBLKSIZE);
+        //TRACE("\nMfFile::ReadMfRecord - mfb=%ld mfp=%d", mfb, mfp);
+
+		// 1 - Read block where record starts 
+        f.ReadBlk(m_buf,mfb);                    
+
+        blk = mfb;
+		// 2 - Extract the record leader (Mfn+Mfrl) from the block buffer, the 
+		//     record leader cannot be split on two  consecutive blocks.
+        int len = GetSubLeaderSize(); // Mfn + Mfrl
+		ASSERT(mfp+len<MFBLKSIZE);
+
+        AddToRecord(m_buf+mfp, len); // Move 1st 6 bytes  (Mfn+Mfrl)
+		ExtractSubLeader();
+
+		// 3 - Continue to transfer data for building the MfRecord object
+        int mflen = GetMfrl();                // Get record length
+	    SetMflenRead(mflen);                  // Save it 
+        //TRACE("\nMfFile::readMfRecord - After reading leader mfn=%ld mflen=%d",GetMfn(), mflen);
+        mfrr = mflen - len;                     // Not yet moved
+        l = (mfrr > (f.GetBlockSize()-(mfp+len)))      // What remains to move
+            ? (f.GetBlockSize()-(mfp+len)) : mfrr;         // in this block
+        offs = len;
+        AddToRecord(m_buf+mfp+len, l); // Move-it
+        mfrr -= l;
+        while (mfrr>0)                        // Record continues on next block ?
+		{                
+            f.ReadBlk(m_buf,++blk);             // Yes read next block
+            offs += l;
+            l = (mfrr > f.GetBlockSize()) ? f.GetBlockSize() : mfrr;
+            AddToRecord(m_buf, l); // Move data
+            mfrr -= l;
+		}
+	    //TRACE("\nMfFile::ReadMfRecord -- mfb=%ld mfp=%d nvf=%d", mfb, mfp, GetNvf());
+	}
+	catch (CFileBaseException e)
+	{
+        throw CIsisDbException(ISIS_MF_READ_ERROR, __FILE__, __LINE__);
+	}
+
+	return DecomposeRecord();
+}
+
+
+void  MfRecord::Write(MfFile& f, long mfb, int mfp)
+{
+	TRACE(_T("\nMfFile::writeMfRecord - mfb=%ld mfp=%d"), mfb, mfp);
+	PRECONDITION(mfb>0 && mfb <= MAXMFB);
+	// Be sure that mfp is in range [0:MFBLKSIZE-1]
+	mfp = mfp%MFBLKSIZE;
+	
+    long hiblk = f.GetNextMfb(); // Highest block number in master file
+	
+    int s  = 0;              // Index in master file record [0:mfrl-1]
+    int d  = mfp;            // Index in block buffer [0:bsize-1]
+    long b = mfb;            // Starting block number
+	
+    // Adjust record length
+	int padding = GetMfrl() % MFALIGN;
+	if (padding>0)
+	{
+		SetMfrl(GetMfrl() + padding);
+		for (int i=0; i<padding;i++)
+			AddToRecord(" ", 1);
+	}
+	
+    int mfrl = GetMfrl();
+    TRACE(_T("\nMfFile::writeMfRecord - b=%ld d=%d mfrl=%ld hiblk=%ld"), b, d, mfrl, hiblk);
+    TRACE(_T("\nBefore writing->"));
+	do 
+    {
+		if (b <= hiblk)        
+			f.ReadBlk(m_buf, b);
+		else
+			::memset(m_buf, ' ', MFBLKSIZE);
+		
+		while (d < MFBLKSIZE && s < mfrl)
+			m_buf[d++] = record_.at(s++);
+		if (d >= MFBLKSIZE) 
+		{
+			TRACE(_T("MfFile::WriteMfRecord - Before WriteBlk b=%ld"), b);
+			f.SetLastWrittenBlock(b);
+			f.WriteBlk(m_buf, b++);
+			// Reset m_buf pos
+			d = 0;
+		}
+	} while (s < mfrl);
+	
+    // Write last block
+	if (d == 0)
+		::memset(m_buf, ' ', MFBLKSIZE); // Should be an empty block
+	
+	f.WriteBlk(m_buf, b);       
+	f.SetLastWrittenBlock(b);
+} 
+
+void IntToChar(int value, int width, std::string& s)
+{
+	std::ostringstream ost;
+
+#ifdef _WIN32
+	ost.setf(std::ios_base::right, std::ios_base::adjustfield);
+#endif
+	ost << std::setw(width) 
+		<< std::setfill('0') 
+		<< value;
+	s = ost.str();
+}
+//--------------------------------------------------------------------------------------
+// void MfRecord::MakeISORecord(std::string& s)
+//
+// Build the record into continuous bytes for output
+//--------------------------------------------------------------------------------------
+void MfRecord::MakeISORecord(std::string& s)
+{
+	std::string    tmp;
+
+	MarcLeader l;
+
+	// 1 - Complete the leader (base and record length), output the directory 
+	//     to a string and prepare a string with the data fields
+	int base = MARC_LEADER_LEN + GetNvf()*MARC_DIRENT_LEN + 1;
+	IntToChar(base, 5, tmp);
+	strncpy(l.base_ ,tmp.c_str(),5);
+
+	DirectoryEntry mde;
+	ustring    all_fields;
+	all_fields.erase();
+	ostringstream  osd;     // Output the directory to a string
+	int pos = 0; 
+	for (int i=0; i!=GetNvf(); ++i)
+	{
+		IntToChar(GetFieldTag(i), 3, tmp); 
+		strncpy(mde.tag_,tmp.c_str(),3);
+		int len = GetFieldLen(i)+1;  // Include space for separator
+		IntToChar(len, 4, tmp);
+		strncpy(mde.len_, tmp.c_str(), 4);
+		IntToChar(pos, 5, tmp);
+		strncpy(mde.scp_, tmp.c_str(), 5);
+		osd << mde; 
+		pos += len;
+
+		cstring field = record_.substr(GetBase()+GetFieldPos(i), GetFieldLen(i));
+		all_fields += ustring(field);
+		all_fields += FIELDTERM;
+	}
+	// (+2) ->Field sep at end of directory and record sep
+    int reclen =  MARC_LEADER_LEN + GetNvf()*MARC_DIRENT_LEN + pos +2;
+	IntToChar(reclen, 5, tmp);
+	strncpy(l.lreclen_,tmp.c_str(),5);
+
+
+	ostringstream osl;     // Output the Leader to a string
+	osl << l;
+
+	// 2   - Make record
+	s.erase();       // Empty the record
+    s += osl.str();  // Add the leader
+	s += osd.str();  // Add the directory
+	s += FIELDTERM;  // Separator
+	s += cstring(all_fields); // Data part
+	s += RECTERM;
+		
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Master.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Master.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Master.h	(revision 21112)
@@ -0,0 +1,609 @@
+/**********************************************************************
+ *
+ * Master.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+///////////////////////////////////////                                   */
+//  Master.h: Master File Record object
+///////////////////////////////////////                                   */
+ 
+#ifndef __MASTER_RECORD_H__
+#define __MASTER_RECORD_H__
+
+#include "BlkFile.h"
+#include <vector>
+#include <map>
+#include <sstream>
+
+
+ 
+// Length in bytes of master file header 
+#define MFCTLLEN sizeof(SMfHeader)+sizeof(SMfHeader) // Length in bytes of master file header
+
+const int MFALIGN   =    2;   // Master file record alignment
+const int MFBLKSIZE =  512;   // Master file block size
+ 
+typedef short int ISIS_INT;
+typedef long  int ISIS_LONG;
+
+#ifdef UNIX_BIREME
+    //Length of Leader in master record
+	inline 	int GetLeaderSize()
+	{
+		return (2*sizeof(ISIS_LONG)+6*sizeof(ISIS_INT));
+	}
+    // Mini Leader length guaranteed to be in same block
+	inline 	int GetMinLeaderSize()
+	{ 
+		return (2*sizeof(ISIS_LONG)+4*sizeof(ISIS_INT));
+	}
+#else
+    //Length of Leader in master record
+	inline 	int GetLeaderSize() 
+	{
+		return (2*sizeof(ISIS_LONG)+5*sizeof(ISIS_INT)); 
+	}
+    // Mini Leader length guaranteed to be in same block
+	inline 	int GetMinLeaderSize()
+	{ 
+		return (2*sizeof(ISIS_LONG)+3*sizeof(ISIS_INT));
+	}
+#endif
+
+// Size of Mfn and Mfrl in the leader part
+inline int GetSubLeaderSize() { return sizeof(ISIS_LONG)+sizeof(ISIS_INT); }
+ 
+inline int GetDirEntrySize() { return (3*sizeof(ISIS_INT)); }
+ 
+ 
+const TCHAR SYSRSEP      =  TCHAR('%'); // Repeatable field separator
+const TCHAR SUBFIELD_SEP =  TCHAR('^'); // Subfield separator
+
+////////////////////////////////////////
+// Directory entry aggregate structure
+//
+struct DirEntry  
+{
+   ISIS_INT  tag_;
+   ISIS_INT  pos_;
+   ISIS_INT  len_;
+   DirEntry() 
+	   : tag_(0), pos_(0), len_ (0) {  }
+
+   DirEntry(ISIS_INT tag, ISIS_INT pos, ISIS_INT len)
+	   : tag_(tag), pos_(pos), len_ (len) {  }
+   bool operator==(const DirEntry& rhs) const
+   {
+	   if (tag_!=rhs.tag_ || pos_!=rhs.pos_ || len_!=rhs.len_)
+		   return false;
+	   return true;
+   }
+   bool operator!=(const DirEntry& rhs) const
+   {
+	   if (tag_!=rhs.tag_ || pos_!=rhs.pos_ || len_!=rhs.len_)
+		   return true;
+	   return false;
+   }
+
+};
+ 
+
+//////////////////////////////////////////////////////////////////////////
+// Data Master record aggregate structure
+//
+struct LeaderAndDirectory 
+{
+   ISIS_LONG        mfn_;            // Master file number
+   ISIS_INT         mfrl_;           // Record length
+   ISIS_LONG        mfbwb_;          // Backward pointer block number
+   ISIS_INT         mfbwp_;          // Backward pointer offset
+   ISIS_INT         base_;           // Offset to begining of variable fields
+   ISIS_INT         nvf_;            // Number of directory entries
+   ISIS_INT         status_;         // 1 = DELETED
+   std::vector<DirEntry> dir_;            // Directory
+
+   LeaderAndDirectory()
+   {
+	   Clear();
+   }
+   LeaderAndDirectory(const LeaderAndDirectory& src)
+   {
+	   Copy(src);
+   }
+
+   LeaderAndDirectory& operator=(const LeaderAndDirectory& rhs)
+   {
+	   if (&mfn_ != &rhs.mfn_)
+	   {
+		   Copy(rhs);
+	   }
+	   return *this;
+   }
+
+   void Copy(const LeaderAndDirectory& rhs)
+   {
+	   mfn_    = rhs.mfn_;
+	   mfrl_   = rhs.mfrl_;
+	   mfbwb_  = rhs.mfbwb_;
+	   mfbwp_  = rhs.mfbwp_;
+       base_   = rhs.base_;
+       nvf_    = rhs.nvf_;
+	   status_ = rhs.status_;
+	   dir_    = rhs.dir_;
+   }
+   void Clear()
+   {
+	   mfn_ = mfrl_ = mfbwb_ = mfbwp_ = base_ = nvf_ = status_ = 0;
+	   dir_.clear();
+   }
+
+   bool operator==(const LeaderAndDirectory& rhs) const;
+   bool operator!=(const LeaderAndDirectory& rhs) const;
+   int GetFieldSize();
+};
+
+cstringstream& operator<<(cstringstream& s, const LeaderAndDirectory& leader);
+cstringstream& operator>>(cstringstream& s, LeaderAndDirectory& leader);
+
+////////////////////////////////////////////////////////////
+//
+ 
+/////////////////////////////////
+// Data master file record class
+//
+
+typedef std::map<int, ustring> TAG2FIELD;
+class MfFile;
+class MstFile;
+class IsisDb;
+class ACE_InputCDR;
+class ACE_OutputCDR;
+class UMfRecord;
+
+
+class  MfRecord
+{
+	friend class MfFile;
+	friend class MstFile;
+	friend class IsisDb;
+	friend class UMfRecord;
+    friend int operator>>(ACE_InputCDR& cdr, MfRecord& mfr);
+    friend int operator<<(ACE_OutputCDR& cdr, MfRecord& mfr);
+protected:
+
+	CFdt*                pFdt_;        // Use a copy of the FDT
+	cstring              record_;      // Record in continuous storage
+	                                   // always as ANSI!
+	
+	LeaderAndDirectory           leader_directory_;
+	
+	TAG2FIELD        fields_;
+	std::vector<int> tags_;
+    static char      m_buf[MFBLKSIZE];      // Buffer to hold 1 File block
+
+protected: // Additional data
+	short int state_;         // Record state in memory
+	long      mfb_read_;      // Record was read at Block number [1...]
+	int       mfp_read_;      // Record was read at position in block
+	int       mflen_read_;    // Record length was when record read 
+
+protected:
+   void         Copy(const MfRecord& rhs); 
+   
+   void         AddToRecord(const char* p, int len);
+   int          AddDirEntry(const DirEntry& d);
+   int          AddDirEntry(int tag, int pos, int len);
+   int          DelDirEntry(int i);
+   void         SetMfrl(int mfrl);
+   void         SetMfbwb(long mfbwb);
+   void         SetMfbwp(int mfbwp);
+   void         SetBase(int base);
+   void         SetNvf(int nvf);
+   void         SetStatus(int s);
+   int          SetDirEntry(int i, int tag, int pos, int len);
+   int          SetDirEntry(int i, const DirEntry& d);
+   void         SetMfbRead(long b);
+   void         SetMfpRead(int pos);
+   void         SetMflenRead(int len);
+
+ 
+   void         SetSubLeaderAndDirectory(const char* p, int len);
+
+
+   void         MakeTagToFieldMap();
+   void         MakeTagVector();
+
+   long         ComputeMfrl();
+
+   void         Copy(UMfRecord& rhs);           
+public:
+	enum { Record_Empty, Record_Read, Record_Changed, Record_Created };
+
+public:
+   //-----------------
+   // Public Interface
+   //-----------------
+   MfRecord(CFdt* pFdt=NULL);                // Default constructor   
+   virtual ~MfRecord();                      // Destructor
+
+   MfRecord(const MfRecord& rhs);            // Copy constructor
+   MfRecord& operator=(const MfRecord& rhs); // Assignment
+ 
+   MfRecord(const UMfRecord& umfr);
+   MfRecord& operator=(const UMfRecord& rhs);
+
+   void Clear();
+   void ClearData();
+
+
+   void ReadLeader(MfFile& f, long daddr);
+   bool Read (MfFile& f, long mfb, int mfp);
+   void Write(MfFile& f, long mfb, int mfp);
+
+   void MakeISORecord(std::string& s);
+ 
+   long         GetMfn()           const; // Get MFN
+   int          GetMfrl()          const; // Get Master File Record Length
+   long         GetMfbwb()         const; // Get Master File Backward Block number
+   int          GetMfbwp()         const; // Get Master File Backward block position
+   int          GetBase()          const; // Get Variable fields starting byte
+   int          GetNvf()           const; // Get Number of entries in the directory
+   int          GetStatus()        const; // Get Record Status
+   DirEntry     GetDirEntry(int i) const; // Get a reference to the ieme directory entry
+   int          GetState()         const; // Get record state in memory
+   long         GetMfbRead()       const;
+   int          GetMfpRead()       const;
+   int          GetMflenRead()     const;
+
+   ustring     GetAllFields(); // ANSI or UNICODE
+   cstring     GetRecordAsString() { return record_; }
+   const char* GetRecord()         { return record_.c_str(); }
+   int         GetRecordLength()   { return record_.length(); }
+
+   void        ConvertToAnsi();
+
+
+   bool         GetField(int tag, std::string& s);  // Get field content for field with tag "tag" 
+   bool         AddField(int tag, std::string& s);
+   bool         DeleteField(int tag);
+   bool         ReplaceField(int tag, std::string& s);
+   bool         AddOccurence(int tag, std::string& o);
+
+   int          GetFieldTag(int i);
+   int          GetFieldPos(int i);
+   int          GetFieldLen(int i);
+ 
+   void         SetMfn(long mfn);
+   void         SetState(int s);
+
+   long         GetDirectorySize();
+   int          GetFieldSize()     { return leader_directory_.GetFieldSize(); }
+
+   std::vector<int>  GetFieldTags(); // Get tags of fields in increasing order,
+                                     // Occurences made only 1 field
+   void         ExtractSubLeader();
+   void         ExtractLeader();
+   bool         CheckIntegrity();
+   bool         CheckLeader();
+   bool         CheckLeaderAndDirectory(); 
+
+   bool operator==(const MfRecord& rhs) const
+   {
+	   return (record_ == rhs.record_); 
+   }
+   bool operator!=(const MfRecord& rhs) const
+   {
+	   return (record_ != rhs.record_); 
+   }
+
+   void         SetRecord(const char *s) { record_ = s;  DecomposeRecord();}
+   void         SetRecord(const cstring &s) 
+				{ record_ = s;  DecomposeRecord();}
+   void         ComposeRecord();
+   bool         DecomposeRecord();
+
+   bool         CheckRecord();
+   void         Dump();
+} ;
+ 
+//////////////////////////////
+ 
+
+
+
+inline void MfRecord::AddToRecord(const char* p, int len)
+{
+	record_ += cstring(p,len);
+} 
+
+/////////////////////////////
+
+inline MfRecord::MfRecord(CFdt* pFdt) 
+// Makes a Master file Record object
+{
+	Clear();
+	pFdt_ = pFdt;
+   // Nothing more to do
+}
+ 
+inline void MfRecord::Copy(const MfRecord& rhs)           
+{
+   record_     = rhs.record_;
+   leader_directory_     = rhs.leader_directory_;
+   fields_     = rhs.fields_;
+   state_      = rhs.state_;
+   mfb_read_   = rhs.mfb_read_;
+   mfp_read_   = rhs.mfp_read_;
+   mflen_read_ = rhs.mflen_read_;
+   tags_       = rhs.tags_;         
+	pFdt_      = rhs.pFdt_;
+}
+
+inline MfRecord::MfRecord(const MfRecord& rhs)     // Copy constructor
+{ 
+   Copy(rhs);
+}
+
+inline MfRecord& MfRecord::operator=(const MfRecord& rhs) // Assignment
+{
+   if (&leader_directory_ != &rhs.leader_directory_)
+   {
+   	   Copy(rhs);
+   }
+   return *this;
+}
+ 
+inline void MfRecord::Clear()
+{
+   tags_.clear();
+   record_.erase();
+   leader_directory_.Clear();
+   fields_.clear();
+   SetBase(GetLeaderSize());
+   SetMfrl(GetLeaderSize());
+   state_ = Record_Empty;
+   mfb_read_ = mfp_read_ = mflen_read_ = 0;
+   pFdt_ = NULL;
+}
+
+// Clear the data part but keep other info intact so that
+// it appears as an empty record
+inline void MfRecord::ClearData()
+{
+   tags_.clear();
+   record_.erase();
+   leader_directory_.Clear();
+   fields_.clear();
+   SetBase(GetLeaderSize());
+   SetMfrl(GetLeaderSize());
+}
+
+
+//////////////////////////////
+ 
+inline MfRecord::~MfRecord()
+{
+}
+ 
+//////////////////////////////
+ 
+inline long MfRecord::GetMfn() const
+// Returns Master file number
+{
+   return leader_directory_.mfn_;
+}
+ 
+//////////////////////////////
+ 
+inline int MfRecord::GetMfrl() const
+// Returns Master file record length
+{
+   return leader_directory_.mfrl_;
+}
+ 
+//////////////////////////////
+ 
+inline long MfRecord::GetMfbwb() const
+// Returns backward pointer block number
+{
+   return leader_directory_.mfbwb_;
+}
+ 
+//////////////////////////////
+ 
+inline int MfRecord::GetMfbwp() const
+// Returns backward pointer offset
+{
+   return leader_directory_.mfbwp_;
+}
+ 
+//////////////////////////////
+ 
+inline int MfRecord::GetBase() const
+// Returns base value to field data
+{
+    return leader_directory_.base_;
+}
+ 
+//////////////////////////////
+ 
+inline int MfRecord::GetNvf() const
+// Returns the number of directory entries
+// in the master file record
+{
+   return leader_directory_.nvf_;
+}
+
+//////////////////////////////
+ 
+inline int MfRecord::GetStatus() const
+// Returns the status of the record
+{
+   return leader_directory_.status_;
+}
+
+//////////////////////////////
+ 
+ 
+//////////////////////////////
+ 
+inline DirEntry MfRecord::GetDirEntry(int i) const
+// Returns the ith dir entry
+{
+   return leader_directory_.dir_[i];
+} 
+
+//////////////////////////////
+ 
+inline int MfRecord::GetState() const
+{
+   return state_;
+} 
+
+//////////////////////////////
+ 
+inline long MfRecord::GetMfbRead() const
+{
+   return mfb_read_;
+} 
+//////////////////////////////
+ 
+inline int MfRecord::GetMfpRead() const
+{
+   return mfp_read_;
+} 
+    
+//////////////////////////////
+ 
+// Get tags of fields in increasing order,
+inline std::vector<int> MfRecord::GetFieldTags()
+{
+	PRECONDITION(tags_.size() == fields_.size());
+	return tags_;
+}
+//////////////////////////////
+ 
+inline int MfRecord::GetMflenRead() const
+{
+   return mflen_read_;
+} 
+    
+///////////////////////////////////////////////////////////////////////////////
+ 
+inline void MfRecord::SetMfn(long mfn)
+// Set Master file number
+{
+	assert(mfn>0);
+    leader_directory_.mfn_ = mfn;
+}
+//////////////////////////////
+ 
+inline void MfRecord::SetMfrl(int mfrl)
+// Set Master file record length
+{
+	assert(mfrl>0);
+    leader_directory_.mfrl_ = mfrl;
+}
+//////////////////////////////
+ 
+inline void MfRecord::SetMfbwb(long mfbwb)
+// Sets backward pointer block number to mfbwb
+{
+	assert(mfbwb>=0);
+    leader_directory_.mfbwb_ = mfbwb;
+}
+
+//////////////////////////////
+ 
+inline void MfRecord::SetMfbwp(int mfbwp)
+// Sets backward pointer block offset to mfbwp
+{
+	assert(mfbwp>=0);
+    leader_directory_.mfbwp_ = mfbwp;
+}
+//////////////////////////////
+ 
+inline void  MfRecord::SetBase(int base)
+// Sets the base value in the record aggregate
+{
+	assert(base>=0);
+    leader_directory_.base_ = base;
+}
+
+//////////////////////////////
+ 
+inline void MfRecord::SetNvf(int nvf)
+// Sets the number of entries in the directory part
+{
+	assert(nvf>=0);
+    leader_directory_.nvf_ = nvf;
+}
+ 
+//////////////////////////////
+ 
+inline void MfRecord::SetStatus(int s)
+// Sets the status of the record
+{
+   leader_directory_.status_ = s;
+}
+ 
+//////////////////////////////
+ 
+inline void MfRecord::SetState(int s)
+// Sets the status of the record
+{
+	assert(s==MfRecord::Record_Empty || s==MfRecord::Record_Read 
+		|| s==MfRecord::Record_Changed || s==MfRecord::Record_Created);
+   state_ = s;
+}
+
+inline void MfRecord::SetMfbRead(long b)
+{
+	mfb_read_ = b;
+}
+
+inline void MfRecord::SetMfpRead(int pos)
+{
+	mfp_read_ = pos;
+}
+
+inline void MfRecord::SetMflenRead(int len)
+{
+	mflen_read_ = len;
+}
+
+inline int MfRecord::GetFieldTag(int i)
+{
+	return leader_directory_.dir_[i].tag_;
+}
+inline int MfRecord::GetFieldPos(int i)
+{
+	return leader_directory_.dir_[i].pos_;
+}
+
+inline int MfRecord::GetFieldLen(int i)
+{
+	return leader_directory_.dir_[i].len_;
+}
+
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.cpp	(revision 21112)
@@ -0,0 +1,620 @@
+/**********************************************************************
+ *
+ * Unimarc.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include "stdafx.h"
+
+///////////////////////////////////////////////////////////////////////////////////
+// MarcRecord - This class can instantiate objects that represent a MARC
+//              bibilographic record and includes methods for manipulating the 
+//              record.
+//
+//             example:
+//
+//	#include "Unimarc.h"
+//
+//     // Create a MarcRecord object with a raw MARC record
+//     MarcRecord mr(buf,1)
+//     mr.PrettyFormat("The title is %245, which is the field with tag 245");
+#include <vector>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "Unimarc.h"
+
+//-------------------------------------------------------------------------------
+// ostream& operator<<(ostream& s, const MarcLeader& leader)
+//
+// Overloaded instance of the output operator for MarcLeader objects
+//---------------------------------------------------------------------------------
+std::ostream& operator<<(std::ostream& s, const MarcLeader& leader)
+{
+	s.write(leader.lreclen_, 5);	// Logical record length
+	s.write(&leader.status_, 1);	// Record status  
+	s.write(&leader.type_, 1);		// Type of record
+	s.write(&leader.bibLevel_, 1);	// Bibliographic level
+	s.write(&leader.ctrlType_, 1);	// Type of control
+	s.write(&leader.undPos_, 1);	// Undefined character position
+	s.write(&leader.indCount_, 1);	// Indicator count
+	s.write(&leader.scd_, 1);		// Subfield code count
+	s.write(leader.base_, 5);   	// Base address of data
+	s.write(&leader.encLevel_, 1);	// Encoding level
+	s.write(&leader.dcf_, 1);		// Descriptive cataloging form
+	s.write(&leader.lrr_, 1);		// Linked record requirement
+	  // Directory  entry map
+	s.write(&leader.llof_, 1);		// Length of the length-of-field portion
+	s.write(&leader.lscp_, 1);		// Length of starting-character-position portion
+	s.write(&leader.lidp_, 1);		// Length of implementation-defined portion
+	s.write(&leader.uep_, 1);		// Undefined entry map character position
+	return s;
+}
+
+//----------------------------------------------------------------------------------
+// istream& operator>>(istream& s, const MarcLeader& leader)
+//
+// Overloaded instance of the input operator for MarcLeader objects
+//----------------------------------------------------------------------------------
+std::istream& operator>>(std::istream& s, MarcLeader& leader)
+{
+	s.read(leader.lreclen_, 5);	    // Logical record length
+	s.read(&leader.status_, 1);	    // Record status  
+	s.read(&leader.type_, 1);		// Type of record
+	s.read(&leader.bibLevel_, 1);	// Bibliographic level
+	s.read(&leader.ctrlType_, 1);	// Type of control
+	s.read(&leader.undPos_, 1);	    // Undefined character position
+	s.read(&leader.indCount_, 1);	// Indicator count
+	s.read(&leader.scd_, 1);		// Subfield code count
+	s.read(leader.base_, 5);   	    // Base address of data
+	s.read(&leader.encLevel_, 1);	// Encoding level
+	s.read(&leader.dcf_, 1);		// Descriptive cataloging form
+	s.read(&leader.lrr_, 1);		// Linked record requirement
+	  // Directory  entry map
+	s.read(&leader.llof_, 1);		// Length of the length-of-field portion
+	s.read(&leader.lscp_, 1);		// Length of starting-character-position portion
+	s.read(&leader.lidp_, 1);		// Length of implementation-defined portion
+	s.read(&leader.uep_, 1);		// Undefined entry map character position
+	return s;
+}
+
+/************************************************************************
+Local prototypes
+ ***********************************************************************/
+static void Printx(FILE *fp, char *s, int x);
+
+//----------------------------------------------------------------------
+// MarcRecord::MarcRecord(char *data, int length, int copyData)
+// 
+// Constructor that creates a new MarcRecord object. 
+// data is raw marc record data for a single record.  If copyData is 1, 
+// data is encapsulated within this object.  If 0, does not copy data and 
+// responsibility falls to caller to maintain data pointer.
+//----------------------------------------------------------------------
+MarcRecord::MarcRecord(char* data, int length, int copyData)
+{
+	encapsulated_ = copyData;
+	
+	/* encapsulate or set pointer to data */
+	if (encapsulated_)
+	{
+		data_ = new char[length + 1]; 
+		memcpy(data_, data, length);
+		data_[length] = '\0';
+	}
+	else
+		data = data;
+	
+	// Lay leader overlay onto data
+	leader_ = *(MarcLeader *) (data_);
+	
+	// Parse the record.  sets dirCount_, directory_ 
+	if (!ParseMarcRecord()) 
+	{
+		// marcRecord_destroy(m);
+		// return NULL;
+	}
+}
+
+//----------------------------------------------------------------------
+// void MarcRecord::~MarcRecord()
+//
+// Destructor, destroys a MarcRecord object and all used resources.
+//----------------------------------------------------------------------
+MarcRecord::~MarcRecord()
+{
+	if ((encapsulated_) && (data_))
+		delete[] data_;
+
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+		directory_[i].field_.erase(directory_[i].field_.begin(),directory_[i].field_.end());
+	directory_.erase(directory_.begin(),directory_.end());
+}
+
+#define MARC_STATE_IDLE 0
+#define MARC_STATE_TAG 1
+#define MARC_STATE_WIDTH 2
+//----------------------------------------------------------------------
+// int MarcRecord::PrettyFormat(char* format, char* buf, int maxlen)
+//
+//  Formats the contents of the marc record according to a format string.
+//  Useful for generating a buffer to pretty-print the record.
+//  format contains a format-specifier string.  The format-specifier string
+//  is kindof like printf's, except the only control character is '%'.
+//  Following the '%' should be the integer MARC field code as defined in 
+//  the (huge) list of available MARC fields from the library of congress
+//  (http://lcweb.loc.gov/marc/).  buf is a previously allocated 
+// buffer into which the pretty version will be copied.  maxlen is the max 
+// length to write into buf.  After call, buf contains pretty formatted text.
+//----------------------------------------------------------------------
+int MarcRecord::PrettyFormat(char* format, char* buf, int maxlen)
+{
+	char b[1024];
+	int bufpos = 0, fmtpos = 0;
+	
+	b[0] = '\0';
+	buf[0] = '\0';
+	
+	for (; format[fmtpos] != '\0'; )
+	{
+		switch (format[fmtpos])
+		{
+			case '%': 
+				{
+					char tag[4];
+					strncpy(tag, (char *)(format + fmtpos + 1), 3);
+					tag[3] = '\0';
+					GetSubField(tag, format[fmtpos + 4], b,	sizeof(b) - 1);
+					if (*b)
+					{
+						int l = strlen(b);
+						strncat(buf, b, maxlen - bufpos - 1);
+						bufpos += l;
+						if (bufpos >= maxlen)
+						{
+							buf[maxlen - 1] = '\0';
+							return maxlen;
+						}
+					}
+					fmtpos += 5;
+					break;
+				}
+			default:
+				buf[bufpos] = format[fmtpos];
+				bufpos++;
+				fmtpos++;
+				buf[bufpos] = '\0';
+				break;
+		}
+	}
+	return bufpos;
+}
+
+//----------------------------------------------------------------------
+// int MarcRecord::GetField(char* tag, char* buf, int maxlen)
+//
+// append all subfields in a field together. 
+//----------------------------------------------------------------------
+int MarcRecord::GetField(char* tag, char* buf, int maxlen)
+{
+	int remain = maxlen - 1;
+	
+	buf[0] = '\0';
+	
+	if ((tag == NULL) || (*tag == '\0'))
+		return 0;
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+	{
+		if (!strncmp(tag, directory_[i].tag_, strlen(tag)))
+		{
+			for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+			{
+				int flen = directory_[i].field_[j].len_;
+				if (flen > remain)
+				{
+					strncat(buf, directory_[i].field_[j].data_ + 1, remain);
+					buf[maxlen] = '\0';
+					return 1;
+				}
+				else 
+				{
+					strncat(buf, directory_[i].field_[j].data_ + 1, flen);
+					remain -= flen;
+				}
+			}
+			return 1;
+		}
+	}
+	return 0;
+}
+
+//----------------------------------------------------------------------
+// int MarcRecord::GetSubFields(char* tag, char subtag, char* buf, int maxlen)
+//
+// append all fields with a certain subfield together.
+//----------------------------------------------------------------------
+int MarcRecord::GetSubFields(char* tag, char subtag, char* buf, int maxlen)
+{
+	int remain = maxlen - 1;
+	
+	buf[0] = '\0';
+	
+	if ((tag == NULL) || (*tag == '\0'))
+		return 0;
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+	{
+		if (!strncmp(tag, directory_[i].tag_, strlen(tag)))
+		{
+			for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+			{
+				if (*(directory_[i].field_[j].data_) == subtag)
+				{
+					int flen = directory_[i].field_[j].len_;
+					if (flen > remain)
+					{
+						strncpy(buf, directory_[i].field_[j].data_ + 1, remain);
+						buf[maxlen] = '\0';
+						return 1;
+					}
+					else 
+					{
+						strcpy(buf, directory_[i].field_[j].data_ + 1);
+						remain -= flen;
+					}
+				}
+			}
+		}
+	}
+	return 0;
+}
+
+int MarcRecord::HasField(char* tag)
+{
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+		if (!strncmp(tag, directory_[i].tag_, strlen(tag)))
+			return 1;
+		
+		return 0;
+}
+
+int MarcRecord::HasSubField(char* tag, char subtag)
+{
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+		if (!strncmp(tag, directory_[i].tag_, strlen(tag)))
+			for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+				if (*directory_[i].field_[j].data_ == subtag)
+					return 1;
+				return 0;
+}
+
+int MarcRecord::GetSubField(char *tag, char subtag, char *buf, 	int maxlen)
+{
+	
+	buf[0] = '\0';
+	
+	if ((tag == NULL) || (*tag == '\0'))
+		return 0;
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+	{
+		if (!strncmp(tag, directory_[i].tag_, strlen(tag)))
+		{
+			for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+			{
+				if (*(directory_[i].field_[j].data_) == subtag)
+				{
+					int l = directory_[i].field_[j].len_ - 1 > maxlen ? maxlen : directory_[i].field_[j].len_ - 1;
+					strncpy(buf, directory_[i].field_[j].data_ + 1, l);
+					buf[l] = '\0';
+					return 1;
+				}
+			}
+		}
+	}
+	return 0;
+}
+
+
+int GetNum(char *s, int n)
+{
+	char nbuf[10]; /* no more than 9 digits */
+	
+	strncpy(&nbuf[0], s, n);
+	nbuf[n] = '\0';
+	return (atoi(nbuf));
+}
+
+void MarcRecord::Dump()
+{
+	int dc = 0, fc = 0;
+	
+	printf("tags: ");
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+	{
+		Printx(stdout, directory_[i].tag_, 3);
+		dc++;
+		for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+		{
+			if (directory_[i].field_[j].data_)
+			{
+				putchar((int)directory_[i].field_[j].data_);
+			}
+			fc++;
+			/*
+			printx(stdout, (char*)f->data + 1, f->len);
+			printf("\n"); */
+		}
+		printf(",");
+	}
+	putchar('\n');
+}
+
+
+
+
+
+void MarcRecord::PrintDetailed(FILE *fp)
+{
+	int count, i;
+	
+	fprintf(fp, "RecordLength:\t\t%i\n", GetNum(leader_.lreclen_, 5));
+	fprintf(fp, "RecordStatus:\t\t%c\n", leader_.status_);
+	fprintf(fp, "RecordType:\t\t%c\n", leader_.type_);
+	fprintf(fp, "BibLevel:\t\t%c\n", leader_.bibLevel_);
+	fprintf(fp, "ControlType:\t\t%c\n", leader_.ctrlType_);
+	fprintf(fp, "UndefCharPos:\t\t%c\n", leader_.undPos_);
+	fprintf(fp, "IndCount:\t\t%i\n", GetNum(&leader_.indCount_, 1));
+	fprintf(fp, "SubCodeCount:\t\t%i\n", GetNum(&leader_.scd_, 1));
+	fprintf(fp, "DataBaseAddr:\t\t%i\n", GetNum(leader_.base_, 5));
+	fprintf(fp, "EncodingLevel:\t\t%c\n", leader_.encLevel_);
+	fprintf(fp, "DescCatForm:\t\t%c\n", leader_.dcf_);
+	fprintf(fp, "LinkedRecReq:\t\t%c\n", leader_.lrr_);
+	fprintf(fp, "LengthOf LengthOfField:\t%i\n", GetNum(&leader_.llof_, 1));
+	fprintf(fp, "LengthOf StartCharPos:\t%i\n", GetNum(&leader_.lscp_, 1));
+	fprintf(fp, "LengthOf ImpDefined:\t%i\n", GetNum(&leader_.lidp_, 1));
+	fprintf(fp, "UndefinedEntry:\t\t%i\n", GetNum(&leader_.uep_, 1));
+	
+	count = (GetNum(leader_.base_, 5) - MARC_LEADER_LEN) / MARC_DIRENT_LEN;
+	for (i = 0; i < count; i++)
+	{
+		DirectoryEntry dir = *(DirectoryEntry*)(data_+MARC_LEADER_LEN+(i*MARC_DIRENT_LEN));
+		fprintf(fp, "\n");
+		fprintf(fp, "Tag:\t\t");
+		Printx(fp, dir.tag_, 3);
+		fprintf(fp, "\nLength:\t\t%i\n", GetNum(dir.len_, 4));
+		fprintf(fp, "StartPos:\t%i\n", GetNum(dir.scp_, 5));
+		fprintf(fp, "Data:\t\t");
+		Printx(fp, (char *)(data_ + GetNum(leader_.base_, 5) + GetNum(dir.scp_, 5)), GetNum(dir.len_, 4));
+		fprintf(fp, "\n");
+	}
+}
+
+/*
+desc:		parses the marc record and builds appropriate data structures in m
+
+params:
+
+	m		newly created marcRecord
+	
+returns:
+
+	1 on success
+	0 on parse or memory failure
+*/
+int MarcRecord::ParseMarcRecord()
+{
+	dirCount_ = (GetNum(leader_.base_, 5) - MARC_LEADER_LEN) / MARC_DIRENT_LEN;
+	
+	// For each MarcDirectory entry, create list of subfields
+	MarcDirectoryEntry dir_entry;
+	for (int i = 0; i < dirCount_; i++)
+	{
+		char*  data;
+		// Extract the directory component
+		DirectoryEntry dir_entry = *(DirectoryEntry*)(data_+MARC_LEADER_LEN+(i*MARC_DIRENT_LEN));
+
+		MarcDirectoryEntry marc_dir_entry;
+		
+		// Get a pointer to the data for this MarcDirectory
+		data = (char *)(data_ + GetNum(leader_.base_, 5) + GetNum(dir_entry.scp_, 5));
+		
+		// set the tag for this MarcDirectory 
+		marc_dir_entry.tag_ = dir_entry.tag_;
+		
+		/* add the list of subfields for this MarcDirectory */
+		if (!AddSubFields(marc_dir_entry, data))
+			return 0;
+		directory_.push_back(marc_dir_entry);
+		
+	}
+	
+	return 1;
+}
+
+void Printx(FILE* fp, char* s, int x)
+{
+	int i;
+	for (i = 0; i < x; i++)
+		fprintf(fp, "%c", s[i]);
+}
+
+//----------------------------------------------------------------------
+// int MarcRecord::AddSubFields(MarcDirectory& d, char* b)
+//
+// Parses and adds subfields to MarcDirectory
+//
+// params:
+//
+//	d		MarcDirectory to which subfields are added
+//	b		buffer of data for d
+//
+// returns:
+//
+//	1 on success
+//	0 on parse or memory error
+//----------------------------------------------------------------------
+int MarcRecord::AddSubFields(MarcDirectoryEntry& d, char* b)
+{
+	char *p, *start;
+	int cont = 1, length = 0;
+	
+	/* create new field */
+	MarcField field;
+	
+	
+	start = b;
+	for (p = b; ((*p != '\0') && (cont == 1)); p = (char *)p + 1)
+	{
+		switch (*p)
+		{
+			case SUBFDELIM:
+				/* end of current subfield, but more exist */
+				
+				/* set length and pointer to data for this field */
+				field.len_  = length;
+				field.data_ = start;
+				d.field_.push_back(field);
+				/* by definition, there should be more subfields, but we'll
+				check for NULL just in case
+				*/
+				if (*p + 1)
+				{
+				}
+				else 
+				{
+					/* uh oh */
+					return 0;
+				}
+				length = 0;
+				start = (char *)p + 1;
+				break;
+			case FIELDTERM:
+				/* end of subfield list */
+				
+				/* set length and pointer to data for this field */
+				field.len_  = length;
+				field.data_ = start;
+				d.field_.push_back(field);
+				
+				/* break out of loop */
+				cont = 0;
+				break;
+			case RECTERM:
+				/* end of record */
+				cont = 0;
+				break;
+			default:
+				/* regular character */			
+				length++;
+				break;
+		}
+	}
+	return 1;
+}
+
+void MarcRecord::Format(char *buf, int maxlen, int format)
+{
+	char b[1024];
+	char date[24];
+	char title[128];
+	
+	b[0] = '\0';
+	buf[0] = '\0';
+	
+	/* FIXME: get rid of this debugging crap sometime */
+	Dump();
+	
+	switch (format)
+	{
+		case MARC_FMT_BRIEF: 
+			{
+				GetSubField("245", 'a', title, 	sizeof(title) - 1);
+				GetSubField("260", 'c', date, sizeof(date) - 1);
+				if ((strlen(date) + strlen(title) + 2) <=(size_t)maxlen)
+				{
+					sprintf(buf, "%s %s", date, title);
+				}
+				else 
+				{
+					/* FIXME: we're overflowing the buffer! */
+				}
+				break;
+			}
+		case MARC_FMT_FULL: 
+			{
+				int dc = 0, fc = 0;
+				int remain = maxlen;
+				
+	            for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+				{
+					dc++;
+		            for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+					{
+						fc++;
+						if ((directory_[i].field_[j].len_ + 2) > remain)
+						{
+							strncat(buf, (char *)directory_[i].field_[j].data_ + 1, remain - 2);
+							* (buf + remain - 2) = '\n';
+							* (buf + remain - 1) = '\0';
+							/* we're out of room in the buffer. */
+							return;
+						}
+						if (directory_[i].field_[j].len_ > 0)
+						{
+							strncat(buf, (char*)directory_[i].field_[j].data_ + 1, directory_[i].field_[j].len_);
+							strncat(buf, "\n\0", 2);
+							remain -=(directory_[i].field_[j].len_ + 2);
+						}
+					} 
+				}
+				/*
+				marcRecord_getSubField(m, "245", 'a', title, 
+				sizeof(title) - 1);
+				marcRecord_getSubField(m, "245", 'a', title, 
+				sizeof(title) - 1);
+				marcRecord_getSubField(m, "260", 'c', date, 
+				sizeof(date) - 1);
+				sprintf(buf, "%s %s", date, title);
+				*/
+				break;
+			}
+		default:
+			break;
+	}
+}
+
+int MarcRecord::RecordLength()
+{
+	register int l = 0;
+	
+	
+	for (std::vector<MarcDirectoryEntry>::size_type i=0; i!=directory_.size(); ++i)
+		for (std::vector<MarcField>::size_type j=0; j!=directory_[i].field_.size(); ++j)
+			l += directory_[i].field_[i].len_;
+		
+        return l;
+}
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/Unimarc.h	(revision 21112)
@@ -0,0 +1,175 @@
+/**********************************************************************
+ *
+ * Unimarc.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifndef MARC_RECORD
+#define MARC_RECORD
+
+#define SUBFDELIM '\037'   // %
+#define FIELDTERM '\036'   // $
+#define RECTERM   '\035'   // # sharp
+
+#define MARC_LEADER_LEN 24
+#define MARC_DIRENT_LEN 12
+
+#define MARC_FMT_BRIEF 0
+#define MARC_FMT_FULL 1
+
+//---------------------
+// UNIMARC Record Label
+//---------------------
+struct  MarcLeader
+{
+	char	lreclen_[5];	/* logical record length */
+	char	status_;		/* record status_ */
+	char	type_;			/* type_ of record */
+	char	bibLevel_;	    /* bibliographic level */
+	char	ctrlType_;	    /* type_ of control */
+	char	undPos_;		/* undefined character position */
+	char	indCount_;	    /* indicator count */
+	char	scd_;			/* subfield code count */
+	char	base_[5];		/* base_ address of data */
+	char	encLevel_;	    /* encoding level */
+	char	dcf_;			/* descriptive cataloging form */
+	char	lrr_;			/* linked record requirement */
+	char	llof_;			/* length of the length-of-field portion */
+	char	lscp_;			/* length of starting-character-position portion */
+	char	lidp_;			/* length of implementation-defined portion */
+	char	uep_;			/* undefined entry map character position */
+
+	MarcLeader()
+	{
+		lreclen_[0]=' ';
+		lreclen_[1]=' ';
+		lreclen_[2]=' ';
+		lreclen_[3]=' ';
+		lreclen_[4]=' ';
+		status_   = '0';     // New record
+        type_     = '0';     // Language materials, printed
+	    bibLevel_ = '0';	     // Monographic
+	    ctrlType_ = '0';     // Hierarchical relationship undefined
+	    undPos_   = '0';     // Undefined, contains a blank
+	    indCount_ = '0';     // Invariably 2 in UNIMARC
+	    scd_      = '0';     // Invariably 2 in UNIMARC
+	    base_[0]  = ' ';     // Base_ address of data
+		base_[1]  = ' ';
+		base_[2]  = ' ';
+		base_[3]  = ' ';
+		base_[4]  = ' ';
+	    encLevel_ = '0';     // (Blank) full level
+	    dcf_      = '0';     // (Blank) record is in full ISBD form
+	    lrr_      = '0';     // Undefined, contains a blank
+	    llof_     = '4';     // Length of the length-of-field portion
+	    lscp_     = '5';     // Length of starting-character-position portion 
+	    lidp_     = '0';     // Length of implementation-defined portion
+	    uep_      = '0';     // Undefined, contains a blank
+	}
+
+   friend std::ostream& operator<<(std::ostream& s, const MarcLeader& leader);
+   friend std::istream& operator>>(std::istream& s, MarcLeader& leader);
+};
+
+//----------------
+// Directory entry
+//----------------
+struct DirectoryEntry
+{
+	char	tag_[3];		/* tag */
+	char	len_[4];		/* field length */
+	char	scp_[5];		/* starting character position */
+   friend std::ostream& operator<<(std::ostream& s, const DirectoryEntry& de);
+   friend std::istream& operator>>(std::istream& s, DirectoryEntry& leader);
+};
+inline std::istream& operator>>(std::istream& s, DirectoryEntry& de)
+{
+	s.read(de.tag_, 3);
+	s.read(de.len_, 4);
+	s.read(de.scp_, 5);
+	return s;
+}
+inline std::ostream& operator<<(std::ostream& s, const DirectoryEntry& de)
+{
+	s.write(de.tag_, 3);
+	s.write(de.len_, 4);
+	s.write(de.scp_, 5);
+	return s;
+}
+
+
+
+//------------------
+// Subfield item 
+//------------------
+struct MarcField
+{
+	int			len_;
+	char*       data_;
+};
+
+//-------------------
+// Directory entry
+//-------------------
+struct MarcDirectoryEntry
+{
+	char*                  tag_;
+	std::vector<MarcField> field_;
+};
+
+
+//---------------------------
+// UNIMARC record structure 
+//---------------------------
+
+class MarcRecord
+{
+protected:
+	MarcLeader      leader_;
+	char*           data_;
+	int			    length_;
+	int				dirCount_;
+	int				encapsulated_;
+	std::vector<MarcDirectoryEntry>   directory_;
+	
+    int ParseMarcRecord();
+    int AddSubFields(MarcDirectoryEntry& d, char* b);
+public:
+    MarcRecord(char* data, int length, int copyData);
+    ~MarcRecord();
+    void PrintDetailed(FILE* fp);
+    void Get(char* buf,int maxlen,int format);
+    int  GetSubField(char* tag, char subtag, char* buf, int maxlen);
+    int  GetField(char* tag, char* buf, int maxlen);
+    int  GetSubFields(char* tag, char subtag, char* buf, int maxlen);
+
+    int  HasField(char* tag);
+    int  HasSubField(char* tag, char subtag);
+
+    void Format(char* buf, int maxlen, int format);
+    int  RecordLength();
+    int  PrettyFormat(char* format, char* buf, int maxlen);
+
+    void Dump ();
+};
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.cpp	(revision 21112)
@@ -0,0 +1,293 @@
+/**********************************************************************
+ *
+ * XRFFile.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// XrfFile.cpp: XrfFile object methods
+//
+////////////////////////////////////////////////////////////
+#include "stdafx.h"
+#include "XRFFile.h"
+
+
+
+//-----------------------------------------------------------------------------------
+// IsisError  XrfFile::OpenXrf(const _TCHAR *fname, FileSystem::AccessMode mode
+//				                              /* = FileSystem::FILE_READWRITE */)
+//
+// Open xrf_ file with file name fname 
+//-----------------------------------------------------------------------------------
+IsisError  XrfFile::OpenXrf(const _TCHAR *fname, FileSystem::AccessMode mode
+				                              /* = FileSystem::FILE_READWRITE */)
+{
+    IsisError isis_error;
+    xrfa_->xrfpos_ = -1;   // No block into buffer
+	try {
+        FileError rc = CFileBase::Open(fname, mode);
+	}
+	catch(CFileBaseException)
+	{
+		isis_error = ISIS_XRF_OPEN_ERROR;
+        throw CIsisDbException(isis_error, __FILE__, __LINE__); 
+	}
+  	long file_size   = CFileBase::FileSize(file_name);
+	
+	last_block_  = file_size/XRFBLKSIZE;
+
+	if (last_block_<1 || last_block_*XRFBLKSIZE != file_size)
+	{
+		isis_error =  ISIS_XRF_CORRUPT;
+        throw CIsisDbException(isis_error, __FILE__, __LINE__); 
+	}
+
+	return ISIS_NO_ERROR;
+}
+ 
+//////////////////////////////
+ 
+ 
+IsisError  XrfFile::CreateXrf(const _TCHAR *fname)
+// Creates an xrf_ file
+{
+    IsisError isis_error;
+	try 
+	{
+        FileError rc = CFileBase::Create(fname);
+    	// Initialize an empty block and write-it
+		xrfa_->Init();
+        xrfa_->xrfpos_ = 1; // Block number
+		xrfa_->xrfeof_ = 1; // Last xrf_ block
+        last_block_    = 1;		
+        WriteXrf(1L);
+	}
+	catch(CFileBaseException)
+	{
+		isis_error = ISIS_XRF_CREATE_ERROR;
+        throw CIsisDbException(isis_error, __FILE__, __LINE__);
+	}
+	
+    return ISIS_NO_ERROR;
+}
+//-----------------------------------------------------------------------------------
+// IsisError  XrfFile::CreateXrf(const _TCHAR *fname, long nextMfn)
+//
+// Special xrf_ file creation for building xrf_ blocks of an existing mst
+//-----------------------------------------------------------------------------------
+IsisError  XrfFile::CreateXrf(const _TCHAR *fname, long nextMfn)
+// Creates an xrf_ file
+{
+    IsisError isis_error;
+    long nBlocks = (nextMfn-1)/MAXTIV + 1;       // Number of blocks to allocate
+	try 
+	{
+        FileError rc = CFileBase::Create(fname);
+		xrfa_->Init();
+    	// Initialize an empty block and write-it
+		for (int iBlock=1; iBlock<= nBlocks; iBlock++)
+		{
+			TRACE("\niBlock=%d", iBlock);
+			xrfa_->xrfpos_ = iBlock;
+			xrfa_->xrfeof_ = 0;
+			if (iBlock == nBlocks)
+			    xrfa_->xrfeof_ = 1;
+
+			WriteXrf(iBlock);
+		}
+	}
+	catch(CFileBaseException)
+	{
+		isis_error = ISIS_XRF_CREATE_ERROR;
+        throw CIsisDbException(isis_error, __FILE__, __LINE__);
+	}
+
+    last_block_ = nBlocks;	
+    return ISIS_NO_ERROR;
+}
+
+//------------------------------------------------------------------------------
+// void XrfFile::UnPack(void)
+//
+// Unpack addresses of master file records. The current packed XRF block 
+// contained in xrf_ structure will be unpacked in xrfa_ structure.
+//
+//  A master file address is packed as follow: addr = mfb*2048 + mfp
+//
+//  2048 = 2**11 e.g. mfp->[0:2047] and block number -> [1: 2**20]
+//  2**20 = 1 048 576
+//
+// As we can put 127 addresses per block, then the limit on number of records
+// is 127 * 1 048 576
+//------------------------------------------------------------------------------
+void XrfFile::UnPack(void)
+{
+   xrfa_->xrfpos_ = xrf_->xrfpos_;
+   if (xrfa_->xrfpos_ < 0)
+   {
+      xrfa_->xrfpos_ = - xrf_->xrfpos_;
+      xrfa_->xrfeof_ = 1;
+   }
+   else
+      xrfa_->xrfeof_ = 0;
+ 
+   for (int i=0; i<MAXTIV; i++)
+   {
+      long j = xrf_->xrftiv_[i];             // Packed value
+      bool b = (j < 0);
+      if (b)
+         j = - j;
+      xrfa_->xrftiv_[i].xrfmfb_ = int(j / 2048L);   // Unpack block number
+      if (b)
+         xrfa_->xrftiv_[i].xrfmfb_ = -xrfa_->xrftiv_[i].xrfmfb_;
+      xrfa_->xrftiv_[i].xrfmfp_ = int(j % 2048L);   // Unpack offset
+   }
+}
+ 
+//------------------------------------------------------------------------------
+// void XrfFile::Pack(void)
+//
+// Pack addresses of master file records. The current unpacked XRF block 
+// contained in xrfa_ will be packed in xrf_ structure.
+//------------------------------------------------------------------------------
+void XrfFile::Pack(void)
+{
+   long j = xrfa_->xrfpos_;
+   if (xrfa_->xrfeof_ != 0)
+      j = -j;
+   xrf_->xrfpos_ = j;
+
+ 
+   for (int i=0; i<MAXTIV; i++)
+   {
+      j = xrfa_->xrftiv_[i].xrfmfb_;    // Unpacked block number
+      int b = (j < 0);
+      if (b)
+         j = - j;
+      //-------------------------------------------
+      // pack block number and offset in a long int
+      //-------------------------------------------
+      j = j * 2048 + xrfa_->xrftiv_[i].xrfmfp_;
+      if (b) j = -j;
+      xrf_->xrftiv_[i] = j;     // Put value in xrf_
+   }
+}
+ 
+//------------------------------------------------------------------------------
+// int XrfFile::GetMfp(long mfn, long& mfb, int& mfp)
+//
+// Gets the address of master file data record with MFN "mfn" by accessing the 
+// xrf_ file). mfb is the master file block number [1:NBLK] and mfp is the offset
+// in the master file block where the record starts [0:BLKSIZE-1]
+//
+//  Returns   Active              = Record active
+//            LogicalyDeleted     = Record logically deleted
+//            PhysicalyDeleted    = Record physically deleted
+//            EndOfFile
+//  Note: mfp can be >=      than BLKSIZE if the inverted file
+//  wasn't updated, then true mfp is mfp%BLKSIZE
+//------------------------------------------------------------------------------
+int XrfFile::GetMfp(long mfn, long& mfb, int& mfp)
+{
+	PRECONDITION(mfn>0);
+	PRECONDITION(last_block_>0);
+    int  comp;
+    long comb;
+    int  rc;
+	
+    long psxrf=(mfn-1)/MAXTIV+1;       // Block number
+	if (psxrf > last_block_)
+	{
+        rc = EndOfFile;                // Inexistent record (eof)
+		return rc;
+		throw CIsisDbException(ISIS_XRF_INVALID_BLOCK, __FILE__, __LINE__);
+	}
+    if (psxrf != xrfa_->xrfpos_) 
+	{                                  // If current block in xrf_ different
+		try
+		{
+            ReadXrf(psxrf);            // then read the block
+		}
+		catch (CFileBaseException e)
+		{
+            throw CIsisDbException(ISIS_XRF_READ_ERROR, __FILE__, __LINE__);
+		}
+
+    }
+    int i = int((mfn-1) % MAXTIV);     // Index
+    comb = xrfa_->xrftiv_[i].xrfmfb_;     // Master file block #
+    comp = xrfa_->xrftiv_[i].xrfmfp_;     // Offset in block
+
+	if (comb > 0)
+        rc = Active;                   // Record active 
+    else if (comb < 0 && comp > 0)
+        rc = LogicalyDeleted;          // Record logicaly deleted 
+	else if (comb == -1 && comp == 0)
+        rc = PhysicalyDeleted;         // Record physically deleted 
+    else if (comb == 0 && comp == 0)
+        rc = EndOfFile;                // Inexistent record (eof)
+	else
+	{
+        throw CIsisDbException(ISIS_XRF_INVALID_STATUS, __FILE__, __LINE__); 
+	}
+    mfb = labs(comb);
+    mfp = comp;
+	if (rc == Active)
+	{
+	    ASSERT(mfb>0 && mfb <= MAXMFB);
+		ASSERT(mfp>=0 && mfp < 2048);
+	    int mfpTemp = mfp%XRFBLKSIZE;
+	    ASSERT(mfpTemp>=0 && mfpTemp<XRFBLKSIZE);
+	}
+    return(rc);
+}
+ 
+void XrfFile::Grow(long from_lastb, long to_lastb)
+{
+	for (long i=from_lastb+1; i<=to_lastb; i++)
+	{
+        //--------------------
+        // append empty blocks
+        //--------------------
+ 		xrfa_->Init();
+		xrfa_->xrfpos_ = i;
+		xrfa_->xrfeof_ = (i==to_lastb) ? 1 : 0; // Eof flag
+	    try
+		{
+           WriteXrf(i);           // Write xrf_ at block address xrfb
+		}
+		catch (CFileBaseException)
+		{
+			throw(ISIS_XRF_WRITE_ERROR);
+		}
+	}
+}
+ 
+//------------------------------------------------------------------------------
+// void XrfFile::PutMfp(long mfn, long mfb, int mfp, int status)
+//
+// Puts the address of a master file record (mfb,mfp) as the address for MFN 
+// "mfn" in xrf_ file
+//------------------------------------------------------------------------------
+void XrfFile::PutMfp(long mfn, long mfb, int mfp, int status)
+{
+}
Index: /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/XRFFile.h	(revision 21112)
@@ -0,0 +1,151 @@
+/**********************************************************************
+ *
+ * XRFFile.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+////////////////////////////////////////////////////////////
+// XRFFILE.H : Cross Ref File object
+//
+////////////////////////////////////////////////////////////
+ 
+#ifndef _XRFFILE_H_
+#define _XRFFILE_H_
+#ifndef _BLKFILE_H_
+#include "blkfile.h"
+#endif
+#include <assert.h>
+#include "IErrors.h"
+ 
+const int MAXTIV     =     127; //  # of XRF entries per block
+const int XRFBLKSIZE =     512; //
+const int MAXMFB     = 1048576; 
+
+ 
+ 
+/////////////////////////////////////////
+// Cross reference file data structures
+//
+ 
+ 
+struct MSTA_ENTRY                 // Unpacked MST address
+{             
+   long       xrfmfb_;             // Master file block
+   short int  xrfmfp_;             // Offset
+} ;
+ 
+struct XRFA                       // To strore unpacked addresses
+{                
+   long          xrfpos_;          // Block number on xrf_ file or -1 if empty
+   short int     xrfeof_;          // 1 if last block of xrf_ file
+   MSTA_ENTRY xrftiv_[MAXTIV];     // MAXTIV unpacked MST addresses
+
+   XRFA() { Init(); }
+
+   void Init()
+   {
+	   xrfeof_ = 0;              // Not eof
+	   xrfpos_ = -1;             // empty   
+	   for (int i=0; i<MAXTIV; i++)
+	   {
+		   xrftiv_[i].xrfmfb_  = -1;
+	       xrftiv_[i].xrfmfp_  = 0;
+	   }
+   }
+
+
+} ;
+ 
+struct XRF                       // To store packed addresses
+{                                // for reading and wrting
+   long xrfpos_;                  // block number on xrf_ file
+   long xrftiv_[MAXTIV];          // MAXTIV packed MST addresses 
+} ;
+ 
+///////////////////
+// XrfFile class
+//
+
+ 
+class XrfFile : public BlkFile 
+{
+private:
+   XRF*  xrf_;             // Buffer to store packed addresses
+   XRFA* xrfa_;            // Buffer to store unpacked addresses
+   long  last_block_;      // Should always contain the last allocated block
+   
+   void ReadXrf(long b);
+   void WriteXrf(long b);
+   void UnPack();
+   void Pack();
+   void Grow(long from_lastb, long to_lastb);
+public:
+   XrfFile();
+   IsisError OpenXrf(const _TCHAR *fname,
+	                 FileSystem::AccessMode mode = FileSystem::FILE_READWRITE);
+   IsisError CreateXrf(const _TCHAR *fname);
+   IsisError CreateXrf(const _TCHAR *fname, long nextMfn);
+  ~XrfFile();
+   int  GetMfp(long mfn, long& mfb, int& mfp);
+   void PutMfp(long mfn, long mfb, int mfp, int status);
+};
+ 
+//////////////////////////////
+ 
+inline XrfFile::XrfFile()  :
+// Makes a XrfFile object that isn't connected to a file
+   xrf_(new XRF),
+   xrfa_(new XRFA)
+{
+   last_block_ = -1;   // Not yet initialize
+   // Nothing more to do
+}
+ 
+//////////////////////////////
+ 
+ 
+inline XrfFile::~XrfFile()
+{
+   if (xrf_)  delete xrf_;
+   if (xrfa_) delete xrfa_;
+}
+ 
+//////////////////////////////
+ 
+inline void XrfFile::ReadXrf(long b)
+// Reads block # b into xrf_ buffer
+{
+	assert(b>0);
+    ReadBlk(xrf_, b);
+    UnPack();
+}
+ 
+//////////////////////////////
+ 
+inline void XrfFile::WriteXrf(long b)
+// Writes the content of xrf_ buffer as block number b
+{
+	assert(b>0);
+    Pack();
+    WriteBlk(xrf_, b);
+}
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/configure
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/configure	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/configure	(revision 21112)
@@ -0,0 +1,2029 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by Autoconf 2.50.
+#
+# Copyright 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001
+# Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+
+# Name of the executable.
+as_me=`echo "$0" |sed 's,.*[\\/],,'`
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+as_executable_p="test -f"
+
+# Support unset when possible.
+if (FOO=FOO; unset FOO) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+# NLS nuisances.
+$as_unset LANG || test "${LANG+set}" != set || { LANG=C; export LANG; }
+$as_unset LC_ALL || test "${LC_ALL+set}" != set || { LC_ALL=C; export LC_ALL; }
+$as_unset LC_TIME || test "${LC_TIME+set}" != set || { LC_TIME=C; export LC_TIME; }
+$as_unset LC_CTYPE || test "${LC_CTYPE+set}" != set || { LC_CTYPE=C; export LC_CTYPE; }
+$as_unset LANGUAGE || test "${LANGUAGE+set}" != set || { LANGUAGE=C; export LANGUAGE; }
+$as_unset LC_COLLATE || test "${LC_COLLATE+set}" != set || { LC_COLLATE=C; export LC_COLLATE; }
+$as_unset LC_NUMERIC || test "${LC_NUMERIC+set}" != set || { LC_NUMERIC=C; export LC_NUMERIC; }
+$as_unset LC_MESSAGES || test "${LC_MESSAGES+set}" != set || { LC_MESSAGES=C; export LC_MESSAGES; }
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH || test "${CDPATH+set}" != set || { CDPATH=:; export CDPATH; }
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+cross_compiling=no
+subdirs=
+MFLAGS= MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Avoid depending upon Character Ranges.
+ac_cr_az='abcdefghijklmnopqrstuvwxyz'
+ac_cr_AZ='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ac_cr_09='0123456789'
+ac_cr_alnum=$ac_cr_az$ac_cr_AZ$ac_cr_09
+
+# Sed expression to map a string onto a valid sh and CPP variable names.
+ac_tr_sh="sed y%*+%pp%;s%[^_$ac_cr_alnum]%_%g"
+ac_tr_cpp="sed y%*$ac_cr_az%P$ac_cr_AZ%;s%[^_$ac_cr_alnum]%_%g"
+
+ac_default_prefix=../..
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$ac_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$ac_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$ac_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$ac_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$ac_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$ac_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+              localstatedir libdir includedir oldincludedir infodir mandir \
+              exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    NONE ) ;;
+    *)  { echo "$as_me: error: expected an absolute path for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: should be removed in autoconf 3.0.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_prog=$0
+  ac_confdir=`echo "$ac_prog" | sed 's%[\\/][^\\/][^\\/]*$%%'`
+  test "x$ac_confdir" = "x$ac_prog" && ac_confdir=.
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<EOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+EOF
+
+  cat <<EOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+                          [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+                          [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+EOF
+
+  cat <<\EOF
+EOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\EOF
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+EOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_subdir in : $ac_subdirs_all; do test "x$ac_subdir" = x: && continue
+    cd $ac_subdir
+    # A "../" for each directory in /$ac_subdir.
+    ac_dots=`echo $ac_subdir |
+             sed 's,^\./,,;s,[^/]$,&/,;s,[^/]*/,../,g'`
+
+    case $srcdir in
+    .) # No --srcdir option.  We are building in place.
+      ac_sub_srcdir=$srcdir ;;
+    [\\/]* | ?:[\\/]* ) # Absolute path.
+      ac_sub_srcdir=$srcdir/$ac_subdir ;;
+    *) # Relative path.
+      ac_sub_srcdir=$ac_dots$srcdir/$ac_subdir ;;
+    esac
+
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_sub_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_sub_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_sub_srcdir/configure; then
+      echo
+      $SHELL $ac_sub_srcdir/configure  --help=recursive
+    elif test -f $ac_sub_srcdir/configure.ac ||
+           test -f $ac_sub_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_subdir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\EOF
+
+Copyright 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001
+Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+EOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<EOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.50.  Invocation command line was
+
+  $ $0 $@
+
+EOF
+{
+cat <<_ASUNAME
+## ---------- ##
+## Platform.  ##
+## ---------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+PATH = $PATH
+
+_ASUNAME
+} >&5
+
+cat >&5 <<EOF
+## ------------ ##
+## Core tests.  ##
+## ------------ ##
+
+EOF
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Also quote any args containing shell meta-characters.
+ac_configure_args=
+ac_sep=
+for ac_arg
+do
+  case $ac_arg in
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c) ;;
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) ;;
+  *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+    ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"`
+    ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+    ac_sep=" " ;;
+  *) ac_configure_args="$ac_configure_args$ac_sep$ac_arg"
+     ac_sep=" " ;;
+  esac
+  # Get rid of the leading space.
+done
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  echo >&5
+  echo "## ----------------- ##" >&5
+  echo "## Cache variables.  ##" >&5
+  echo "## ----------------- ##" >&5
+  echo >&5
+  # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+        "s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+    	  s/^\\([_$ac_cr_alnum]*_cv_[_$ac_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+        "s/^\\([_$ac_cr_alnum]*_cv_[_$ac_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} >&5
+  sed "/^$/d" confdefs.h >conftest.log
+  if test -s conftest.log; then
+    echo >&5
+    echo "## ------------ ##" >&5
+    echo "## confdefs.h.  ##" >&5
+    echo "## ------------ ##" >&5
+    echo >&5
+    cat conftest.log >&5
+  fi
+  (echo; echo) >&5
+  test "$ac_signal" != 0 &&
+    echo "$as_me: caught signal $ac_signal" >&5
+  echo "$as_me: exit $exit_status" >&5
+  rm -rf conftest* confdefs* core core.* *.core conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_status=$?; ac_signal='$ac_signal'; { (exit $ac_status); exit $ac_status; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:802: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    cat "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:813: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:821: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_suggest_removing_cache=false
+for ac_var in `(set) 2>&1 |
+               sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:837: WARNING: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: WARNING: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_suggest_removing_cache=: ;;
+    ,set)
+      { echo "$as_me:841: WARNING: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: WARNING: \`$ac_var' was not set in the previous run" >&2;}
+      ac_suggest_removing_cache=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+        { echo "$as_me:847: WARNING: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: WARNING: \`$ac_var' has changed since the previous run:" >&2;}
+        { echo "$as_me:849: WARNING:   former value:  $ac_old_val" >&5
+echo "$as_me: WARNING:   former value:  $ac_old_val" >&2;}
+        { echo "$as_me:851: WARNING:   current value: $ac_new_val" >&5
+echo "$as_me: WARNING:   current value: $ac_new_val" >&2;}
+        ac_suggest_removing_cache=:
+      fi;;
+  esac
+done
+if $ac_suggest_removing_cache; then
+  { echo "$as_me:858: WARNING: changes in the environment can compromise the build" >&5
+echo "$as_me: WARNING: changes in the environment can compromise the build" >&2;}
+  { echo "$as_me:860: WARNING: consider removing $cache_file and starting over" >&5
+echo "$as_me: WARNING: consider removing $cache_file and starting over" >&2;}
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)      ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+echo "#! $SHELL" >conftest.sh
+echo  "exit 0"   >>conftest.sh
+chmod +x conftest.sh
+if { (echo "$as_me:879: PATH=\".;.\"; conftest.sh") >&5
+  (PATH=".;."; conftest.sh) 2>&5
+  ac_status=$?
+  echo "$as_me:882: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  ac_path_separator=';'
+else
+  ac_path_separator=:
+fi
+PATH_SEPARATOR="$ac_path_separator"
+rm -f conftest.sh
+
+ac_aux_dir=
+for ac_dir in ../.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:908: error: cannot find install-sh or install.sh in ../.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in ../.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:926: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+  ac_save_IFS=$IFS; IFS=$ac_path_separator
+ac_dummy="$PATH"
+for ac_dir in $ac_dummy; do
+  IFS=$ac_save_IFS
+  test -z "$ac_dir" && ac_dir=.
+  $as_executable_p "$ac_dir/$ac_word" || continue
+ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+echo "$as_me:941: found $ac_dir/$ac_word" >&5
+break
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:949: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:952: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:965: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+  ac_save_IFS=$IFS; IFS=$ac_path_separator
+ac_dummy="$PATH"
+for ac_dir in $ac_dummy; do
+  IFS=$ac_save_IFS
+  test -z "$ac_dir" && ac_dir=.
+  $as_executable_p "$ac_dir/$ac_word" || continue
+ac_cv_prog_ac_ct_CXX="$ac_prog"
+echo "$as_me:980: found $ac_dir/$ac_word" >&5
+break
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:988: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:991: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+cat >conftest.$ac_ext <<_ACEOF
+#line 1003 "configure"
+#include "confdefs.h"
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compiler, and finding out an intuition
+# of exeext.
+echo "$as_me:1019: checking for C++ compiler default output" >&5
+echo $ECHO_N "checking for C++ compiler default output... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:1022: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:1025: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `ls a.exe conftest.exe a.* conftest conftest.* 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.o | *.obj | *.xcoff | *.tds | *.d | *.pdb ) ;;
+    a.out ) # We found the default executable, but exeext='' is most
+            # certainly right.
+            break;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+          export ac_cv_exeext
+          break;;
+    * ) break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+{ { echo "$as_me:1042: error: C++ compiler cannot create executables" >&5
+echo "$as_me: error: C++ compiler cannot create executables" >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:1048: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:1053: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:1059: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:1062: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:1069: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:1077: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:1084: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:1086: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:1089: checking for executable suffix" >&5
+echo $ECHO_N "checking for executable suffix... $ECHO_C" >&6
+if { (eval echo "$as_me:1091: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:1094: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in `(ls conftest.exe; ls conftest; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.o | *.obj | *.xcoff | *.tds | *.d | *.pdb ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+          export ac_cv_exeext
+          break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:1110: error: cannot compute EXEEXT: cannot compile and link" >&5
+echo "$as_me: error: cannot compute EXEEXT: cannot compile and link" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:1116: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:1122: checking for object suffix" >&5
+echo $ECHO_N "checking for object suffix... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+#line 1128 "configure"
+#include "confdefs.h"
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:1140: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:1143: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+{ { echo "$as_me:1155: error: cannot compute OBJEXT: cannot compile" >&5
+echo "$as_me: error: cannot compute OBJEXT: cannot compile" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:1162: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:1166: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+#line 1172 "configure"
+#include "confdefs.h"
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:1187: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:1190: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+         { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:1193: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:1196: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+ac_compiler_gnu=no
+fi
+rm -f conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:1208: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:1214: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+#line 1220 "configure"
+#include "confdefs.h"
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:1232: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:1235: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+         { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:1238: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:1241: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:1251: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   ''\
+   '#include <stdlib.h>' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+#line 1278 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:1291: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:1294: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+         { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:1297: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:1300: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+continue
+fi
+rm -f conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+#line 1310 "configure"
+#include "confdefs.h"
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:1322: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:1325: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+         { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:1328: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:1331: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+cat conftest.$ac_ext >&5
+fi
+rm -f conftest.$ac_objext conftest.$ac_ext
+done
+echo '#ifdef __cplusplus' >>confdefs.h
+echo $ac_declaration      >>confdefs.h
+echo '#endif'             >>confdefs.h
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:1362: checking for a BSD compatible install" >&5
+echo $ECHO_N "checking for a BSD compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_save_IFS=$IFS; IFS=$ac_path_separator
+  for ac_dir in $PATH; do
+    IFS=$ac_save_IFS
+    # Account for people who put trailing slashes in PATH elements.
+    case $ac_dir/ in
+    / | ./ | .// | /cC/* \
+    | /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* \
+    | /usr/ucb/* ) ;;
+    *)
+      # OSF1 and SCO ODT 3.0 have their own names for install.
+      # Don't use installbsd from OSF since it installs stuff as root
+      # by default.
+      for ac_prog in ginstall scoinst install; do
+        if $as_executable_p "$ac_dir/$ac_prog"; then
+	  if test $ac_prog = install &&
+            grep dspmsg "$ac_dir/$ac_prog" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$ac_dir/$ac_prog" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$ac_dir/$ac_prog -c"
+	    break 2
+	  fi
+	fi
+      done
+      ;;
+    esac
+  done
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:1411: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+ac_config_files="$ac_config_files Makefile"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overriden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+        "s/'/'\\\\''/g;
+    	  s/^\\([_$ac_cr_alnum]*_cv_[_$ac_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+        "s/^\\([_$ac_cr_alnum]*_cv_[_$ac_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if cmp -s $cache_file confcache; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[ 	]*VPATH[ 	]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[ 	]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[ 	]*$//;
+}'
+fi
+
+# Transform confdefs.h into DEFS.
+# Protect against shell expansion while executing Makefile rules.
+# Protect against Makefile macro expansion.
+#
+# If the first sed substitution is executed (which looks for macros that
+# take arguments), then we branch to the quote section.  Otherwise,
+# look for a macro that doesn't take arguments.
+cat >confdef2opt.sed <<\EOF
+t clear
+: clear
+s,^[ 	]*#[ 	]*define[ 	][ 	]*\([^ 	(][^ 	(]*([^)]*)\)[ 	]*\(.*\),-D\1=\2,g
+t quote
+s,^[ 	]*#[ 	]*define[ 	][ 	]*\([^ 	][^ 	]*\)[ 	]*\(.*\),-D\1=\2,g
+t quote
+d
+: quote
+s,[ 	`~#$^&*(){}\\|;'"<>?],\\&,g
+s,\[,\\&,g
+s,\],\\&,g
+s,\$,$$,g
+p
+EOF
+# We use echo to avoid assuming a particular line-breaking character.
+# The extra dot is to prevent the shell from consuming trailing
+# line-breaks from the sub-command output.  A line-break within
+# single-quotes doesn't work because, if this script is created in a
+# platform that uses two characters for line-breaks (e.g., DOS), tr
+# would break.
+ac_LF_and_DOT=`echo; echo .`
+DEFS=`sed -n -f confdef2opt.sed confdefs.h | tr "$ac_LF_and_DOT" ' .'`
+rm -f confdef2opt.sed
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:1532: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated automatically by configure.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+ac_cs_invocation="\$0 \$@"
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+
+# Name of the executable.
+as_me=`echo "$0" |sed 's,.*[\\/],,'`
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+as_executable_p="test -f"
+
+# Support unset when possible.
+if (FOO=FOO; unset FOO) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+# NLS nuisances.
+$as_unset LANG || test "${LANG+set}" != set || { LANG=C; export LANG; }
+$as_unset LC_ALL || test "${LC_ALL+set}" != set || { LC_ALL=C; export LC_ALL; }
+$as_unset LC_TIME || test "${LC_TIME+set}" != set || { LC_TIME=C; export LC_TIME; }
+$as_unset LC_CTYPE || test "${LC_CTYPE+set}" != set || { LC_CTYPE=C; export LC_CTYPE; }
+$as_unset LANGUAGE || test "${LANGUAGE+set}" != set || { LANGUAGE=C; export LANGUAGE; }
+$as_unset LC_COLLATE || test "${LC_COLLATE+set}" != set || { LC_COLLATE=C; export LC_COLLATE; }
+$as_unset LC_NUMERIC || test "${LC_NUMERIC+set}" != set || { LC_NUMERIC=C; export LC_NUMERIC; }
+$as_unset LC_MESSAGES || test "${LC_MESSAGES+set}" != set || { LC_MESSAGES=C; export LC_MESSAGES; }
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH || test "${CDPATH+set}" != set || { CDPATH=:; export CDPATH; }
+
+exec 6>&1
+
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\EOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+                   instantiate the configuration file FILE
+
+Configuration files:
+$config_files
+
+Report bugs to <bug-autoconf@gnu.org>."
+EOF
+
+cat >>$CONFIG_STATUS <<EOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.50,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001
+Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+EOF
+
+cat >>$CONFIG_STATUS <<\EOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    shift
+    set dummy "$ac_option" "$ac_optarg" ${1+"$@"}
+    shift
+    ;;
+  -*);;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_need_defaults=false;;
+  esac
+
+  case $1 in
+  # Handling of the options.
+EOF
+cat >>$CONFIG_STATUS <<EOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    echo "running $SHELL $0 " $ac_configure_args " --no-create --no-recursion"
+    exec $SHELL $0 $ac_configure_args --no-create --no-recursion ;;
+EOF
+cat >>$CONFIG_STATUS <<\EOF
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:1700: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    shift
+    CONFIG_FILES="$CONFIG_FILES $1"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $1"
+    ac_need_defaults=false;;
+
+  # Handling of arguments.
+  'Makefile' ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:1722: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+  *) { { echo "$as_me:1727: error: invalid argument: $1" >&5
+echo "$as_me: error: invalid argument: $1" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+  shift
+done
+
+exec 5>>config.log
+cat >&5 << _ACEOF
+
+## ----------------------- ##
+## Running config.status.  ##
+## ----------------------- ##
+
+This file was extended by $as_me 2.50, executed with
+  > $ac_cs_invocation
+on `(hostname || uname -n) 2>/dev/null | sed 1q`
+
+_ACEOF
+EOF
+
+cat >>$CONFIG_STATUS <<\EOF
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+fi
+
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit $?); exit $?; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+: ${TMPDIR=/tmp}
+{
+  tmp=`(umask 077 && mktemp -d -q "$TMPDIR/csXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=$TMPDIR/cs$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in $TMPDIR" >&2
+   { (exit 1); exit 1; }
+}
+
+EOF
+
+cat >>$CONFIG_STATUS <<EOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@DEFS@,$DEFS,;t t
+s,@LIBS@,$LIBS,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+CEOF
+
+EOF
+
+  cat >>$CONFIG_STATUS <<\EOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+  	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+  	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+EOF
+cat >>$CONFIG_STATUS <<\EOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+        cat >$tmp/stdin
+        ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+        ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+        ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+         X"$ac_file" : 'X\(//\)[^/]' \| \
+         X"$ac_file" : 'X\(//\)$' \| \
+         X"$ac_file" : 'X\(/\)' \| \
+         .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  if test "$ac_dir" != "$ac_file" && test "$ac_dir" != .; then
+    { case "$ac_dir" in
+  [\\/]* | ?:[\\/]* ) as_incr_dir=;;
+  *)                      as_incr_dir=.;;
+esac
+as_dummy="$ac_dir"
+for as_mkdir_dir in `IFS='/\\'; set X $as_dummy; shift; echo "$@"`; do
+  case $as_mkdir_dir in
+    # Skip DOS drivespec
+    ?:) as_incr_dir=$as_mkdir_dir ;;
+    *)
+      as_incr_dir=$as_incr_dir/$as_mkdir_dir
+      test -d "$as_incr_dir" || mkdir "$as_incr_dir"
+    ;;
+  esac
+done; }
+
+    ac_dir_suffix="/`echo $ac_dir|sed 's,^\./,,'`"
+    # A "../" for each directory in $ac_dir_suffix.
+    ac_dots=`echo "$ac_dir_suffix" | sed 's,/[^/]*,../,g'`
+  else
+    ac_dir_suffix= ac_dots=
+  fi
+
+  case $srcdir in
+  .)  ac_srcdir=.
+      if test -z "$ac_dots"; then
+         ac_top_srcdir=.
+      else
+         ac_top_srcdir=`echo $ac_dots | sed 's,/$,,'`
+      fi ;;
+  [\\/]* | ?:[\\/]* )
+      ac_srcdir=$srcdir$ac_dir_suffix;
+      ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_dots$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_dots$srcdir ;;
+  esac
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_dots$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:1944: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated automatically by config.status.  */
+  configure_input="Generated automatically from `echo $ac_file_in |
+                                                 sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+         # Absolute (can't be DOS-style, as IFS=:)
+         test -f "$f" || { { echo "$as_me:1962: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+         echo $f;;
+      *) # Relative
+         if test -f "$f"; then
+           # Build tree
+           echo $f
+         elif test -f "$srcdir/$f"; then
+           # Source tree
+           echo $srcdir/$f
+         else
+           # /dev/null tree
+           { { echo "$as_me:1975: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+         fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+EOF
+cat >>$CONFIG_STATUS <<EOF
+  sed "$ac_vpsub
+$extrasub
+EOF
+cat >>$CONFIG_STATUS <<\EOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+EOF
+
+cat >>$CONFIG_STATUS <<\EOF
+
+{ (exit 0); exit 0; }
+EOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/configure.in
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/configure.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/configure.in	(revision 21112)
@@ -0,0 +1,19 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT()
+
+dnl  default installation
+dnl will be overridden by command-line args from packages/configure
+AC_PREFIX_DEFAULT(../..)
+
+dnl this uses the config.guess, config.sub and install-sh from gsdl.
+AC_CONFIG_AUX_DIR(../..)
+
+dnl compiler - need c++
+AC_PROG_CXX
+
+dnl Windows extension
+AC_EXEEXT
+
+AC_PROG_INSTALL
+
+AC_OUTPUT(Makefile)
Index: /main/tags/3.01/gsdl/packages/isis-gdl/mytchar.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/mytchar.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/mytchar.h	(revision 21112)
@@ -0,0 +1,35 @@
+/**********************************************************************
+ *
+ * mytchar.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef __MY_TCHAR__
+#define __MY_TCHAR__
+
+#include "tchar.h"
+
+#ifndef _WIN32
+typedef wchar_t TCHAR;
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.cpp
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.cpp	(revision 21112)
@@ -0,0 +1,33 @@
+/**********************************************************************
+ *
+ * stdafx.cpp
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+// stdafx.cpp : source file that includes just the standard includes
+// IsisGdl.pch will be the pre-compiled header
+// stdafx.obj will contain the pre-compiled type information
+
+#include "stdafx.h"
+
+// TODO: reference any additional headers you need in STDAFX.H
+// and not in this file
Index: /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/stdafx.h	(revision 21112)
@@ -0,0 +1,40 @@
+/**********************************************************************
+ *
+ * stdafx.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+// stdafx.h : include file for standard system include files,
+// or project specific include files that are used frequently, but
+// are changed infrequently
+//
+
+// #pragma once
+
+
+#include <iostream>
+#include <assert.h>
+#include "mytchar.h"
+#include "Compat.h"
+// TODO: reference additional headers your program requires here
+#include "IsisDef.h"
+
Index: /main/tags/3.01/gsdl/packages/isis-gdl/tchar.h
===================================================================
--- /main/tags/3.01/gsdl/packages/isis-gdl/tchar.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/isis-gdl/tchar.h	(revision 21112)
@@ -0,0 +1,1046 @@
+/**********************************************************************
+ *
+ * tchar.h
+ * Copyright (C) 2003  UNESCO
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/***
+*tchar.h - definitions for generic international text functions
+*
+*       Copyright (c) 1991-1997, Microsoft Corporation. All rights reserved.
+*
+*Purpose:
+*       Definitions for generic international functions, mostly defines
+*       which map string/formatted-io/ctype functions to char, wchar_t, or
+*       MBCS versions.  To be used for compatibility between single-byte,
+*       multi-byte and Unicode text models.
+*
+*       [Public]
+*
+****/
+
+#if     _MSC_VER > 1000 /*IFSTRIP=IGN*/
+#pragma once
+#endif
+
+#ifndef _INC_TCHAR
+#define _INC_TCHAR
+
+#ifdef  _MSC_VER
+#pragma warning(disable:4514)       /* disable unwanted C++ /W4 warning */
+/* #pragma warning(default:4514) */ /* use this to reenable, if necessary */
+#endif  /* _MSC_VER */
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+
+/* Define __cdecl for non-Microsoft compilers */
+
+#if     ( !defined(_MSC_VER) && !defined(__cdecl) )
+#define __cdecl
+#endif
+
+
+/* Define _CRTIMP */
+
+#ifndef _CRTIMP
+#ifdef  CRTDLL
+#define _CRTIMP __declspec(dllexport)
+#else   /* ndef CRTDLL */
+#ifdef  _DLL
+#define _CRTIMP __declspec(dllimport)
+#else   /* ndef _DLL */
+#define _CRTIMP
+#endif  /* _DLL */
+#endif  /* CRTDLL */
+#endif  /* _CRTIMP */
+
+
+#define _ftcscat    _tcscat
+#define _ftcschr    _tcschr
+#define _ftcscpy    _tcscpy
+#define _ftcscspn   _tcscspn
+#define _ftcslen    _tcslen
+#define _ftcsncat   _tcsncat
+#define _ftcsncpy   _tcsncpy
+#define _ftcspbrk   _tcspbrk
+#define _ftcsrchr   _tcsrchr
+#define _ftcsspn    _tcsspn
+#define _ftcsstr    _tcsstr
+#define _ftcstok    _tcstok
+
+#define _ftcsdup    _tcsdup
+#define _ftcsnset   _tcsnset
+#define _ftcsrev    _tcsrev
+#define _ftcsset    _tcsset
+
+#define _ftcscmp      _tcscmp
+#define _ftcsicmp     _tcsicmp
+#define _ftcsnccmp    _tcsnccmp
+#define _ftcsncmp     _tcsncmp
+#define _ftcsncicmp   _tcsncicmp
+#define _ftcsnicmp    _tcsnicmp
+
+#define _ftcscoll     _tcscoll
+#define _ftcsicoll    _tcsicoll
+#define _ftcsnccoll   _tcsnccoll
+#define _ftcsncoll    _tcsncoll
+#define _ftcsncicoll  _tcsncicoll
+#define _ftcsnicoll   _tcsnicoll
+
+/* Redundant "logical-character" mappings */
+
+#define _ftcsclen   _tcsclen
+#define _ftcsnccat  _tcsnccat
+#define _ftcsnccpy  _tcsnccpy
+#define _ftcsncset  _tcsncset
+
+#define _ftcsdec    _tcsdec
+#define _ftcsinc    _tcsinc
+#define _ftcsnbcnt  _tcsnbcnt
+#define _ftcsnccnt  _tcsnccnt
+#define _ftcsnextc  _tcsnextc
+#define _ftcsninc   _tcsninc
+#define _ftcsspnp   _tcsspnp
+
+#define _ftcslwr    _tcslwr
+#define _ftcsupr    _tcsupr
+
+#define _ftclen     _tclen
+#define _ftccpy     _tccpy
+#define _ftccmp     _tccmp
+
+
+#ifdef  _UNICODE
+
+#ifdef  __cplusplus
+}   /* ... extern "C" */
+#endif
+
+/* ++++++++++++++++++++ UNICODE ++++++++++++++++++++ */
+
+#include <wchar.h>
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#ifndef _WCTYPE_T_DEFINED
+typedef wchar_t wint_t;
+typedef wchar_t wctype_t;
+#define _WCTYPE_T_DEFINED
+#endif
+
+#ifndef __TCHAR_DEFINED
+typedef wchar_t     _TCHAR;
+typedef wchar_t     _TSCHAR;
+typedef wchar_t     _TUCHAR;
+typedef wchar_t     _TXCHAR;
+typedef wint_t      _TINT;
+#define __TCHAR_DEFINED
+#endif
+
+#ifndef _TCHAR_DEFINED
+#if     !__STDC__
+typedef wchar_t     TCHAR;
+#endif
+#define _TCHAR_DEFINED
+#endif
+
+#define _TEOF       WEOF
+
+#define __T(x)      L ## x
+
+
+/* Program */
+
+#define _tmain      wmain
+#define _tWinMain   wWinMain
+#define _tenviron   _wenviron
+#define __targv     __wargv
+
+/* Formatted i/o */
+
+#define _tprintf    wprintf
+#define _ftprintf   fwprintf
+#define _stprintf   swprintf
+#define _sntprintf  _snwprintf
+#define _vtprintf   vwprintf
+#define _vftprintf  vfwprintf
+#define _vstprintf  vswprintf
+#define _vsntprintf _vsnwprintf
+#define _tscanf     wscanf
+#define _ftscanf    fwscanf
+#define _stscanf    swscanf
+
+
+/* Unformatted i/o */
+
+#define _fgettc     fgetwc
+#define _fgettchar  _fgetwchar
+#define _fgetts     fgetws
+#define _fputtc     fputwc
+#define _fputtchar  _fputwchar
+#define _fputts     fputws
+#define _gettc      getwc
+#define _gettchar   getwchar
+#define _getts      _getws
+#define _puttc      putwc
+#define _puttchar   putwchar
+#define _putts      _putws
+#define _ungettc    ungetwc
+
+
+/* String conversion functions */
+
+#define _tcstod     wcstod
+#define _tcstol     wcstol
+#define _tcstoul    wcstoul
+
+#define _itot       _itow
+#define _ltot       _ltow
+#define _ultot      _ultow
+#define _ttoi       _wtoi
+#define _ttol       _wtol
+
+#define _ttoi64     _wtoi64
+#define _i64tot     _i64tow
+#define _ui64tot    _ui64tow
+
+/* String functions */
+
+#define _tcscat     wcscat
+#define _tcschr     wcschr
+#define _tcscpy     wcscpy
+#define _tcscspn    wcscspn
+#define _tcslen     wcslen
+#define _tcsncat    wcsncat
+#define _tcsncpy    wcsncpy
+#define _tcspbrk    wcspbrk
+#define _tcsrchr    wcsrchr
+#define _tcsspn     wcsspn
+#define _tcsstr     wcsstr
+#define _tcstok     wcstok
+
+#define _tcsdup     _wcsdup
+#define _tcsnset    _wcsnset
+#define _tcsrev     _wcsrev
+#define _tcsset     _wcsset
+
+#define _tcscmp     wcscmp
+#define _tcsicmp    _wcsicmp
+#define _tcsnccmp   wcsncmp
+#define _tcsncmp    wcsncmp
+#define _tcsncicmp  _wcsnicmp
+#define _tcsnicmp   _wcsnicmp
+
+#define _tcscoll    wcscoll
+#define _tcsicoll   _wcsicoll
+#define _tcsnccoll  _wcsncoll
+#define _tcsncoll   _wcsncoll
+#define _tcsncicoll _wcsnicoll
+#define _tcsnicoll  _wcsnicoll
+
+
+/* Execute functions */
+
+#define _texecl     _wexecl
+#define _texecle    _wexecle
+#define _texeclp    _wexeclp
+#define _texeclpe   _wexeclpe
+#define _texecv     _wexecv
+#define _texecve    _wexecve
+#define _texecvp    _wexecvp
+#define _texecvpe   _wexecvpe
+
+#define _tspawnl    _wspawnl
+#define _tspawnle   _wspawnle
+#define _tspawnlp   _wspawnlp
+#define _tspawnlpe  _wspawnlpe
+#define _tspawnv    _wspawnv
+#define _tspawnve   _wspawnve
+#define _tspawnvp   _wspawnvp
+#define _tspawnvp   _wspawnvp
+#define _tspawnvpe  _wspawnvpe
+
+#define _tsystem    _wsystem
+
+
+/* Time functions */
+
+#define _tasctime   _wasctime
+#define _tctime     _wctime
+#define _tstrdate   _wstrdate
+#define _tstrtime   _wstrtime
+#define _tutime     _wutime
+#define _tcsftime   wcsftime
+
+
+/* Directory functions */
+
+#define _tchdir     _wchdir
+#define _tgetcwd    _wgetcwd
+#define _tgetdcwd   _wgetdcwd
+#define _tmkdir     _wmkdir
+#define _trmdir     _wrmdir
+
+
+/* Environment/Path functions */
+
+#define _tfullpath  _wfullpath
+#define _tgetenv    _wgetenv
+#define _tmakepath  _wmakepath
+#define _tputenv    _wputenv
+#define _tsearchenv _wsearchenv
+#define _tsplitpath _wsplitpath
+
+
+/* Stdio functions */
+
+#define _tfdopen    _wfdopen
+#define _tfsopen    _wfsopen
+#define _tfopen     _wfopen
+#define _tfreopen   _wfreopen
+#define _tperror    _wperror
+#define _tpopen     _wpopen
+#define _ttempnam   _wtempnam
+#define _ttmpnam    _wtmpnam
+
+
+/* Io functions */
+
+#define _taccess    _waccess
+#define _tchmod     _wchmod
+#define _tcreat     _wcreat
+#define _tfindfirst _wfindfirst
+#define _tfindfirsti64  _wfindfirsti64
+#define _tfindnext  _wfindnext
+#define _tfindnexti64   _wfindnexti64
+#define _tmktemp    _wmktemp
+#define _topen      _wopen
+#define _tremove    _wremove
+#define _trename    _wrename
+#define _tsopen     _wsopen
+#define _tunlink    _wunlink
+
+#define _tfinddata_t    _wfinddata_t
+#define _tfinddatai64_t _wfinddatai64_t
+
+
+/* Stat functions */
+
+#define _tstat      _wstat
+#define _tstati64   _wstati64
+
+
+/* Setlocale functions */
+
+#define _tsetlocale _wsetlocale
+
+
+/* Redundant "logical-character" mappings */
+
+#define _tcsclen    wcslen
+#define _tcsnccat   wcsncat
+#define _tcsnccpy   wcsncpy
+#define _tcsncset   _wcsnset
+
+#define _tcsdec     _wcsdec
+#define _tcsinc     _wcsinc
+#define _tcsnbcnt   _wcsncnt
+#define _tcsnccnt   _wcsncnt
+#define _tcsnextc   _wcsnextc
+#define _tcsninc    _wcsninc
+#define _tcsspnp    _wcsspnp
+
+#define _tcslwr     _wcslwr
+#define _tcsupr     _wcsupr
+#define _tcsxfrm    wcsxfrm
+
+
+#if     (__STDC__ || defined(_NO_INLINING)) && !defined(_M_M68K)
+#define _tclen(_pc) (1)
+#define _tccpy(_pc1,_cpc2) ((*(_pc1) = *(_cpc2)))
+#define _tccmp(_cpc1,_cpc2) ((*(_cpc1))-(*(_cpc2)))
+#else   /* __STDC__ */
+__inline size_t __cdecl _tclen(const wchar_t *_cpc) { return (_cpc,1); }
+__inline void __cdecl _tccpy(wchar_t *_pc1, const wchar_t *_cpc2) { *_pc1 = (wchar_t)*_cpc2; }
+__inline int __cdecl _tccmp(const wchar_t *_cpc1, const wchar_t *_cpc2) { return (int) ((*_cpc1)-(*_cpc2)); }
+#endif  /* __STDC__ */
+
+
+/* ctype functions */
+
+#define _istalnum   iswalnum
+#define _istalpha   iswalpha
+#define _istascii   iswascii
+#define _istcntrl   iswcntrl
+#define _istdigit   iswdigit
+#define _istgraph   iswgraph
+#define _istlower   iswlower
+#define _istprint   iswprint
+#define _istpunct   iswpunct
+#define _istspace   iswspace
+#define _istupper   iswupper
+#define _istxdigit  iswxdigit
+
+#define _totupper   towupper
+#define _totlower   towlower
+
+#define _istlegal(_c)   (1)
+#define _istlead(_c)    (0)
+#define _istleadbyte(_c)    (0)
+
+
+#if     (__STDC__ || defined(_NO_INLINING)) && !defined(_M_M68K)
+#define _wcsdec(_cpc1, _cpc2) ((_cpc1)>=(_cpc2) ? NULL : (_cpc2)-1)
+#define _wcsinc(_pc)    ((_pc)+1)
+#define _wcsnextc(_cpc) ((unsigned int) *(_cpc))
+#define _wcsninc(_pc, _sz) (((_pc)+(_sz)))
+#define _wcsncnt(_cpc, _sz) ((wcslen(_cpc)>_sz) ? _sz : wcslen(_cpc))
+#define _wcsspnp(_cpc1, _cpc2) ((*((_cpc1)+wcsspn(_cpc1,_cpc2))) ? ((_cpc1)+wcsspn(_cpc1,_cpc2)) : NULL)
+#else   /* __STDC__ */
+__inline wchar_t * __cdecl _wcsdec(const wchar_t * _cpc1, const wchar_t * _cpc2) { return (wchar_t *)((_cpc1)>=(_cpc2) ? NULL : ((_cpc2)-1)); }
+__inline wchar_t * __cdecl _wcsinc(const wchar_t * _pc) { return (wchar_t *)(_pc+1); }
+__inline unsigned int __cdecl _wcsnextc(const wchar_t * _cpc) { return (unsigned int)*_cpc; }
+__inline wchar_t * __cdecl _wcsninc(const wchar_t * _pc, size_t _sz) { return (wchar_t *)(_pc+_sz); }
+__inline size_t __cdecl _wcsncnt( const wchar_t * _cpc, size_t _sz) { size_t len; len = wcslen(_cpc); return (len>_sz) ? _sz : len; }
+__inline wchar_t * __cdecl _wcsspnp( const wchar_t * _cpc1, const wchar_t * _cpc2) { return (*(_cpc1 += wcsspn(_cpc1,_cpc2))!='\0') ? (wchar_t*)_cpc1 : NULL; }
+#endif  /* __STDC__ */
+
+
+#else   /* ndef _UNICODE */
+
+/* ++++++++++++++++++++ SBCS and MBCS ++++++++++++++++++++ */
+
+#ifdef  __cplusplus
+}   /* ... extern "C" */
+#endif
+
+#include <string.h>
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+
+#define _TEOF       EOF
+
+#define __T(x)      x
+
+
+/* Program */
+
+#define _tmain      main
+#define _tWinMain   WinMain
+#ifdef  _POSIX_
+#define _tenviron   environ
+#else
+#define _tenviron  _environ
+#endif
+#define __targv     __argv
+
+
+/* Formatted i/o */
+
+#define _tprintf    printf
+#define _ftprintf   fprintf
+#define _stprintf   sprintf
+#define _sntprintf  _snprintf
+#define _vtprintf   vprintf
+#define _vftprintf  vfprintf
+#define _vstprintf  vsprintf
+#define _vsntprintf _vsnprintf
+#define _tscanf     scanf
+#define _ftscanf    fscanf
+#define _stscanf    sscanf
+
+
+/* Unformatted i/o */
+
+#define _fgettc     fgetc
+#define _fgettchar  _fgetchar
+#define _fgetts     fgets
+#define _fputtc     fputc
+#define _fputtchar  _fputchar
+#define _fputts     fputs
+#define _gettc      getc
+#define _gettchar   getchar
+#define _getts      gets
+#define _puttc      putc
+#define _puttchar   putchar
+#define _putts      puts
+#define _ungettc    ungetc
+
+
+/* String conversion functions */
+
+#define _tcstod     strtod
+#define _tcstol     strtol
+#define _tcstoul    strtoul
+
+#define _itot       _itoa
+#define _ltot       _ltoa
+#define _ultot      _ultoa
+#define _ttoi       atoi
+#define _ttol       atol
+
+#define _ttoi64     _atoi64
+#define _i64tot     _i64toa
+#define _ui64tot    _ui64toa
+
+/* String functions */
+
+/* Note that _mbscat, _mbscpy and _mbsdup are functionally equivalent to 
+   strcat, strcpy and strdup, respectively. */
+
+#define _tcscat     strcat
+#define _tcscpy     strcpy
+#define _tcsdup     _strdup
+
+#define _tcslen     strlen
+#define _tcsxfrm    strxfrm
+
+
+/* Execute functions */
+
+#define _texecl     _execl
+#define _texecle    _execle
+#define _texeclp    _execlp
+#define _texeclpe   _execlpe
+#define _texecv     _execv
+#define _texecve    _execve
+#define _texecvp    _execvp
+#define _texecvpe   _execvpe
+
+#define _tspawnl    _spawnl
+#define _tspawnle   _spawnle
+#define _tspawnlp   _spawnlp
+#define _tspawnlpe  _spawnlpe
+#define _tspawnv    _spawnv
+#define _tspawnve   _spawnve
+#define _tspawnvp   _spawnvp
+#define _tspawnvpe  _spawnvpe
+
+#define _tsystem    system
+
+
+/* Time functions */
+
+#define _tasctime   asctime
+#define _tctime     ctime
+#define _tstrdate   _strdate
+#define _tstrtime   _strtime
+#define _tutime     _utime
+#define _tcsftime   strftime
+
+
+/* Directory functions */
+
+#define _tchdir     _chdir
+#define _tgetcwd    _getcwd
+#define _tgetdcwd   _getdcwd
+#define _tmkdir     _mkdir
+#define _trmdir     _rmdir
+
+
+/* Environment/Path functions */
+
+#define _tfullpath  _fullpath
+#define _tgetenv    getenv
+#define _tmakepath  _makepath
+#define _tputenv    _putenv
+#define _tsearchenv _searchenv
+#define _tsplitpath _splitpath
+
+
+/* Stdio functions */
+
+#ifdef  _POSIX_
+#define _tfdopen    fdopen
+#else
+#define _tfdopen    _fdopen
+#endif
+#define _tfsopen    _fsopen
+#define _tfopen     fopen
+#define _tfreopen   freopen
+#define _tperror    perror
+#define _tpopen     _popen
+#define _ttempnam   _tempnam
+#define _ttmpnam    tmpnam
+
+
+/* Io functions */
+
+#define _tchmod     _chmod
+#define _tcreat     _creat
+#define _tfindfirst _findfirst
+#define _tfindfirsti64  _findfirsti64
+#define _tfindnext  _findnext
+#define _tfindnexti64   _findnexti64
+#define _tmktemp    _mktemp
+
+#ifdef  _POSIX_
+#define _topen      open
+#define _taccess    access
+#else
+#define _topen      _open
+#define _taccess    _access
+#endif
+
+#define _tremove    remove
+#define _trename    rename
+#define _tsopen     _sopen
+#define _tunlink    _unlink
+
+#define _tfinddata_t    _finddata_t
+#define _tfinddatai64_t _finddatai64_t
+
+
+/* ctype functions */
+
+#define _istascii   isascii
+#define _istcntrl   iscntrl
+#define _istxdigit  isxdigit
+
+
+/* Stat functions */
+
+#define _tstat      _stat
+#define _tstati64   _stati64
+
+
+/* Setlocale functions */
+
+#define _tsetlocale setlocale
+
+
+#ifdef  _MBCS
+
+/* ++++++++++++++++++++ MBCS ++++++++++++++++++++ */
+
+#ifdef  __cplusplus
+}   /* ... extern "C" */
+#endif
+
+#include <mbstring.h>
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef __TCHAR_DEFINED
+typedef char            _TCHAR;
+typedef signed char     _TSCHAR;
+typedef unsigned char   _TUCHAR;
+typedef unsigned char   _TXCHAR;
+typedef unsigned int    _TINT;
+#define __TCHAR_DEFINED
+#endif
+
+#ifndef _TCHAR_DEFINED
+#if     !__STDC__
+typedef char            TCHAR;
+#endif
+#define _TCHAR_DEFINED
+#endif
+
+
+#ifdef  _MB_MAP_DIRECT
+
+/* use mb functions directly - types must match */
+
+/* String functions */
+
+#define _tcschr     _mbschr
+#define _tcscspn    _mbscspn
+#define _tcsncat    _mbsnbcat
+#define _tcsncpy    _mbsnbcpy
+#define _tcspbrk    _mbspbrk
+#define _tcsrchr    _mbsrchr
+#define _tcsspn     _mbsspn
+#define _tcsstr     _mbsstr
+#define _tcstok     _mbstok
+
+#define _tcsnset    _mbsnbset
+#define _tcsrev     _mbsrev
+#define _tcsset     _mbsset
+
+#define _tcscmp     _mbscmp
+#define _tcsicmp    _mbsicmp
+#define _tcsnccmp   _mbsncmp
+#define _tcsncmp    _mbsnbcmp
+#define _tcsncicmp  _mbsnicmp
+#define _tcsnicmp   _mbsnbicmp
+
+#define _tcscoll    _mbscoll
+#define _tcsicoll   _mbsicoll
+#define _tcsnccoll  _mbsncoll
+#define _tcsncoll   _mbsnbcoll
+#define _tcsncicoll _mbsnicoll
+#define _tcsnicoll  _mbsnbicoll
+
+
+/* "logical-character" mappings */
+
+#define _tcsclen    _mbslen
+#define _tcsnccat   _mbsncat
+#define _tcsnccpy   _mbsncpy
+#define _tcsncset   _mbsnset
+
+
+/* MBCS-specific mappings */
+
+#define _tcsdec     _mbsdec
+#define _tcsinc     _mbsinc
+#define _tcsnbcnt   _mbsnbcnt
+#define _tcsnccnt   _mbsnccnt
+#define _tcsnextc   _mbsnextc
+#define _tcsninc    _mbsninc
+#define _tcsspnp    _mbsspnp
+
+#define _tcslwr     _mbslwr
+#define _tcsupr     _mbsupr
+
+#define _tclen      _mbclen
+#define _tccpy      _mbccpy
+
+#define _tccmp(_cpuc1,_cpuc2)   _tcsnccmp(_cpuc1,_cpuc2,1)
+
+
+#else /* _MB_MAP_DIRECT */
+
+#if     (__STDC__ || defined(_NO_INLINING)) && !defined(_M_M68K)
+
+/* use type-safe linked-in function thunks */
+
+/* String functions */
+
+_CRTIMP char * __cdecl _tcschr(const char *, unsigned int);
+_CRTIMP size_t __cdecl _tcscspn(const char *, const char *);
+_CRTIMP char * __cdecl _tcsncat(char *, const char *, size_t);
+_CRTIMP char * __cdecl _tcsncpy(char *, const char *, size_t);
+_CRTIMP char * __cdecl _tcspbrk(const char *, const char *);
+_CRTIMP char * __cdecl _tcsrchr(const char *, int);
+_CRTIMP size_t __cdecl _tcsspn(const char *, const char *);
+_CRTIMP char * __cdecl _tcsstr(const char *, const char *);
+_CRTIMP char * __cdecl _tcstok(char *, const char *);
+
+_CRTIMP char * __cdecl _tcsnset(char *, unsigned int, size_t);
+_CRTIMP char * __cdecl _tcsrev(char *);
+_CRTIMP char * __cdecl _tcsset(char *, unsigned int);
+
+_CRTIMP int __cdecl _tcscmp(const char *, const char *);
+_CRTIMP int __cdecl _tcsicmp(const char *, const char *);
+_CRTIMP int __cdecl _tcsnccmp(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsncmp(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsncicmp(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsnicmp(const char *, const char *, size_t);
+
+_CRTIMP int __cdecl _tcscoll(const char *, const char *);
+_CRTIMP int __cdecl _tcsicoll(const char *, const char *);
+_CRTIMP int __cdecl _tcsnccoll(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsncoll(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsncicoll(const char *, const char *, size_t);
+_CRTIMP int __cdecl _tcsnicoll(const char *, const char *, size_t);
+
+
+/* "logical-character" mappings */
+
+_CRTIMP size_t __cdecl _tcsclen(const char *);
+_CRTIMP char * __cdecl _tcsnccat(char *, const char *, size_t);
+_CRTIMP char * __cdecl _tcsnccpy(char *, const char *, size_t);
+_CRTIMP char * __cdecl _tcsncset(char *, unsigned int, size_t);
+
+
+/* MBCS-specific mappings */
+
+_CRTIMP char * __cdecl _tcsdec(const char *, const char *);
+_CRTIMP char * __cdecl _tcsinc(const char *);
+_CRTIMP size_t __cdecl _tcsnbcnt(const char *, size_t);
+_CRTIMP size_t __cdecl _tcsnccnt(const char *, size_t);
+_CRTIMP unsigned int __cdecl _tcsnextc (const char *);
+_CRTIMP char * __cdecl _tcsninc(const char *, size_t);
+_CRTIMP char * __cdecl _tcsspnp(const char *, const char *);
+
+_CRTIMP char * __cdecl _tcslwr(char *);
+_CRTIMP char * __cdecl _tcsupr(char *);
+
+_CRTIMP size_t __cdecl _tclen(const char *);
+_CRTIMP void __cdecl _tccpy(char *, const char *);
+
+
+#else   /* __STDC__ */
+
+/* the default: use type-safe inline function thunks */
+
+#define _PUC    unsigned char *
+#define _CPUC   const unsigned char *
+#define _PC     char *
+#define _CPC    const char *
+#define _UI     unsigned int
+
+
+/* String functions */
+
+__inline _PC _tcschr(_CPC _s1,_UI _c) {return (_PC)_mbschr((_CPUC)_s1,_c);}
+__inline size_t _tcscspn(_CPC _s1,_CPC _s2) {return _mbscspn((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcsncat(_PC _s1,_CPC _s2,size_t _n) {return (_PC)_mbsnbcat((_PUC)_s1,(_CPUC)_s2,_n);}
+__inline _PC _tcsncpy(_PC _s1,_CPC _s2,size_t _n) {return (_PC)_mbsnbcpy((_PUC)_s1,(_CPUC)_s2,_n);}
+__inline _PC _tcspbrk(_CPC _s1,_CPC _s2) {return (_PC)_mbspbrk((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcsrchr(_CPC _s1,_UI _c) {return (_PC)_mbsrchr((_CPUC)_s1,_c);}
+__inline size_t _tcsspn(_CPC _s1,_CPC _s2) {return _mbsspn((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcsstr(_CPC _s1,_CPC _s2) {return (_PC)_mbsstr((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcstok(_PC _s1,_CPC _s2) {return (_PC)_mbstok((_PUC)_s1,(_CPUC)_s2);}
+
+__inline _PC _tcsnset(_PC _s1,_UI _c,size_t _n) {return (_PC)_mbsnbset((_PUC)_s1,_c,_n);}
+__inline _PC _tcsrev(_PC _s1) {return (_PC)_mbsrev((_PUC)_s1);}
+__inline _PC _tcsset(_PC _s1,_UI _c) {return (_PC)_mbsset((_PUC)_s1,_c);}
+
+__inline int _tcscmp(_CPC _s1,_CPC _s2) {return _mbscmp((_CPUC)_s1,(_CPUC)_s2);}
+__inline int _tcsicmp(_CPC _s1,_CPC _s2) {return _mbsicmp((_CPUC)_s1,(_CPUC)_s2);}
+__inline int _tcsnccmp(_CPC _s1,_CPC _s2,size_t _n) {return _mbsncmp((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsncmp(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnbcmp((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsncicmp(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnicmp((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsnicmp(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnbicmp((_CPUC)_s1,(_CPUC)_s2,_n);}
+
+__inline int _tcscoll(_CPC _s1,_CPC _s2) {return _mbscoll((_CPUC)_s1,(_CPUC)_s2);}
+__inline int _tcsicoll(_CPC _s1,_CPC _s2) {return _mbsicoll((_CPUC)_s1,(_CPUC)_s2);}
+__inline int _tcsnccoll(_CPC _s1,_CPC _s2,size_t _n) {return _mbsncoll((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsncoll(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnbcoll((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsncicoll(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnicoll((_CPUC)_s1,(_CPUC)_s2,_n);}
+__inline int _tcsnicoll(_CPC _s1,_CPC _s2,size_t _n) {return _mbsnbicoll((_CPUC)_s1,(_CPUC)_s2,_n);}
+
+
+/* "logical-character" mappings */
+
+__inline size_t _tcsclen(_CPC _s1) {return _mbslen((_CPUC)_s1);}
+__inline _PC _tcsnccat(_PC _s1,_CPC _s2,size_t _n) {return (_PC)_mbsncat((_PUC)_s1,(_CPUC)_s2,_n);}
+__inline _PC _tcsnccpy(_PC _s1,_CPC _s2,size_t _n) {return (_PC)_mbsncpy((_PUC)_s1,(_CPUC)_s2,_n);}
+__inline _PC _tcsncset(_PC _s1,_UI _c,size_t _n) {return (_PC)_mbsnset((_PUC)_s1,_c,_n);}
+
+
+/* MBCS-specific mappings */
+
+__inline _PC _tcsdec(_CPC _s1,_CPC _s2) {return (_PC)_mbsdec((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcsinc(_CPC _s1) {return (_PC)_mbsinc((_CPUC)_s1);}
+__inline size_t _tcsnbcnt(_CPC _s1,size_t _n) {return _mbsnbcnt((_CPUC)_s1,_n);}
+__inline size_t _tcsnccnt(_CPC _s1,size_t _n) {return _mbsnccnt((_CPUC)_s1,_n);}
+__inline _PC _tcsninc(_CPC _s1,size_t _n) {return (_PC)_mbsninc((_CPUC)_s1,_n);}
+__inline _PC _tcsspnp(_CPC _s1,_CPC _s2) {return (_PC)_mbsspnp((_CPUC)_s1,(_CPUC)_s2);}
+__inline _PC _tcslwr(_PC _s1) {return (_PC)_mbslwr((_PUC)_s1);}
+__inline _PC _tcsupr(_PC _s1) {return (_PC)_mbsupr((_PUC)_s1);}
+
+__inline size_t _tclen(_CPC _s1) {return _mbclen((_CPUC)_s1);}
+__inline void _tccpy(_PC _s1,_CPC _s2) {_mbccpy((_PUC)_s1,(_CPUC)_s2); return;}
+
+
+/* inline helper */
+__inline _UI _tcsnextc(_CPC _s1) {_UI _n=0; if (_ismbblead((_UI)*(_PUC)_s1)) _n=((_UI)*_s1++)<<8; _n+=(_UI)*_s1; return(_n);}
+
+
+#endif  /* __STDC__ */
+
+#endif  /* _MB_MAP_DIRECT */
+
+
+/* MBCS-specific mappings */
+
+#define _tccmp(_cp1,_cp2)   _tcsnccmp(_cp1,_cp2,1)
+
+
+/* ctype functions */
+
+#define _istalnum   _ismbcalnum
+#define _istalpha   _ismbcalpha
+#define _istdigit   _ismbcdigit
+#define _istgraph   _ismbcgraph
+#define _istlegal   _ismbclegal
+#define _istlower   _ismbclower
+#define _istprint   _ismbcprint
+#define _istpunct   _ismbcpunct
+#define _istspace   _ismbcspace
+#define _istupper   _ismbcupper
+
+#define _totupper   _mbctoupper
+#define _totlower   _mbctolower
+
+#define _istlead    _ismbblead
+#define _istleadbyte    isleadbyte
+
+#else   /* !_MBCS */
+
+/* ++++++++++++++++++++ SBCS ++++++++++++++++++++ */
+
+
+#ifndef __TCHAR_DEFINED
+typedef char            _TCHAR;
+typedef signed char     _TSCHAR;
+typedef unsigned char   _TUCHAR;
+typedef char            _TXCHAR;
+typedef int             _TINT;
+#define __TCHAR_DEFINED
+#endif
+
+#ifndef _TCHAR_DEFINED
+#if     !__STDC__
+typedef char            TCHAR;
+#endif
+#define _TCHAR_DEFINED
+#endif
+
+
+/* String functions */
+
+#define _tcschr     strchr
+#define _tcscspn    strcspn
+#define _tcsncat    strncat
+#define _tcsncpy    strncpy
+#define _tcspbrk    strpbrk
+#define _tcsrchr    strrchr
+#define _tcsspn     strspn
+#define _tcsstr     strstr
+#define _tcstok     strtok
+
+#define _tcsnset    _strnset
+#define _tcsrev     _strrev
+#define _tcsset     _strset
+
+#define _tcscmp     strcmp
+#define _tcsicmp    _stricmp
+#define _tcsnccmp   strncmp
+#define _tcsncmp    strncmp
+#define _tcsncicmp  _strnicmp
+#define _tcsnicmp   _strnicmp
+
+#define _tcscoll    strcoll
+#define _tcsicoll   _stricoll
+#define _tcsnccoll  _strncoll
+#define _tcsncoll   _strncoll
+#define _tcsncicoll _strnicoll
+#define _tcsnicoll  _strnicoll
+
+
+/* "logical-character" mappings */
+
+#define _tcsclen    strlen
+#define _tcsnccat   strncat
+#define _tcsnccpy   strncpy
+#define _tcsncset   _strnset
+
+
+/* MBCS-specific functions */
+
+#define _tcsdec     _strdec
+#define _tcsinc     _strinc
+#define _tcsnbcnt   _strncnt
+#define _tcsnccnt   _strncnt
+#define _tcsnextc   _strnextc
+#define _tcsninc    _strninc
+#define _tcsspnp    _strspnp
+
+#define _tcslwr     _strlwr
+#define _tcsupr     _strupr
+#define _tcsxfrm    strxfrm
+
+#define _istlead(_c)    (0)
+#define _istleadbyte(_c)    (0)
+
+#if     (__STDC__ || defined(_NO_INLINING)) && !defined(_M_M68K)
+#define _tclen(_pc) (1)
+#define _tccpy(_pc1,_cpc2) (*(_pc1) = *(_cpc2))
+#define _tccmp(_cpc1,_cpc2) (((unsigned char)*(_cpc1))-((unsigned char)*(_cpc2)))
+#else   /* __STDC__ */
+__inline size_t __cdecl _tclen(const char *_cpc) { return (_cpc,1); }
+__inline void __cdecl _tccpy(char *_pc1, const char *_cpc2) { *_pc1 = *_cpc2; }
+__inline int __cdecl _tccmp(const char *_cpc1, const char *_cpc2) { return (int) (((unsigned char)*_cpc1)-((unsigned char)*_cpc2)); }
+#endif  /* __STDC__ */
+
+
+/* ctype-functions */
+
+#define _istalnum   isalnum
+#define _istalpha   isalpha
+#define _istdigit   isdigit
+#define _istgraph   isgraph
+#define _istlower   islower
+#define _istprint   isprint
+#define _istpunct   ispunct
+#define _istspace   isspace
+#define _istupper   isupper
+
+#define _totupper   toupper
+#define _totlower   tolower
+
+#define _istlegal(_c)   (1)
+
+
+/* the following is optional if functional versions are available */
+
+/* define NULL pointer value */
+
+#ifndef NULL
+#ifdef __cplusplus
+#define NULL    0
+#else
+#define NULL    ((void *)0)
+#endif
+#endif
+
+
+#if     (__STDC__ || defined(_NO_INLINING)) && !defined(_M_M68K)
+#define _strdec(_cpc1, _cpc2) ((_cpc1)>=(_cpc2) ? NULL : (_cpc2)-1)
+#define _strinc(_pc)    ((_pc)+1)
+#define _strnextc(_cpc) ((unsigned int) *(_cpc))
+#define _strninc(_pc, _sz) (((_pc)+(_sz)))
+#define _strncnt(_cpc, _sz) ((strlen(_cpc)>_sz) ? _sz : strlen(_cpc))
+#define _strspnp(_cpc1, _cpc2) ((*((_cpc1)+strspn(_cpc1,_cpc2))) ? ((_cpc1)+strspn(_cpc1,_cpc2)) : NULL)
+#else   /* __STDC__ */
+__inline char * __cdecl _strdec(const char * _cpc1, const char * _cpc2) { return (char *)((_cpc1)>=(_cpc2) ? NULL : (_cpc2-1)); }
+__inline char * __cdecl _strinc(const char * _pc) { return (char *)(_pc+1); }
+__inline unsigned int __cdecl _strnextc(const char * _cpc) { return (unsigned int)*_cpc; }
+__inline char * __cdecl _strninc(const char * _pc, size_t _sz) { return (char *)(_pc+_sz); }
+__inline size_t __cdecl _strncnt( const char * _cpc, size_t _sz) { size_t len; len = strlen(_cpc); return (len>_sz) ? _sz : len; }
+__inline char * __cdecl _strspnp( const char * _cpc1, const char * _cpc2) { return (*(_cpc1 += strspn(_cpc1,_cpc2))!='\0') ? (char*)_cpc1 : NULL; }
+#endif  /* __STDC__ */
+
+
+#endif  /* _MBCS */
+
+#endif  /* _UNICODE */
+
+
+/* Generic text macros to be used with string literals and character constants.
+   Will also allow symbolic constants that resolve to same. */
+
+#define _T(x)       __T(x)
+#define _TEXT(x)    __T(x)
+
+
+#ifdef  __cplusplus
+}   /* ... extern "C" */
+#endif
+
+#endif  /* _INC_TCHAR */
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/ACKNOWLEDGMENTS
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/ACKNOWLEDGMENTS	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/ACKNOWLEDGMENTS	(revision 21112)
@@ -0,0 +1,2 @@
+This product includes software developed by the
+Apache Software Foundation (http://www.apache.org/).
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/COPYING
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/COPYING	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/COPYING	(revision 21112)
@@ -0,0 +1,345 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+ 		59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	Appendix: How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/Counter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/Counter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/Counter.java	(revision 21112)
@@ -0,0 +1,73 @@
+/*
+ *    Counter.java
+ *    Copyright (C) 2000 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.io.*;
+
+/**
+ * Class that implements a simple counter.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class Counter implements Serializable {
+  
+  /** Integer value stored */
+    private int m_val = 1;
+  
+  /**
+   * Initializes the counter to 1
+   */
+  public Counter() {
+    
+    m_val = 1;
+  }
+  
+  /**
+   * Initializes the counter to the given value
+   */
+  public Counter(int val) {
+    
+    m_val = val;
+  }
+  
+  /**
+   * Increments the counter.
+   */
+  public void increment() {
+    
+    m_val++;
+  }
+  
+  /**
+   * Gets the value.
+   * @return the value
+   */
+  public int value() {
+    
+    return m_val;
+  }
+  
+  /**
+   * Returns string containing value.
+   */
+  public String toString() {
+    
+    return String.valueOf(m_val);
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/GermanStemmer.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/GermanStemmer.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/GermanStemmer.java	(revision 21112)
@@ -0,0 +1,309 @@
+
+/* ====================================================================
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2001 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution,
+ *    if any, must include the following acknowledgment:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowledgment may appear in the software itself,
+ *    if and wherever such third-party acknowledgments normally appear.
+ *
+ * 4. The names "Apache" and "Apache Software Foundation" and
+ *    "Apache Lucene" must not be used to endorse or promote products
+ *    derived from this software without prior written permission. For
+ *    written permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache",
+ *    "Apache Lucene", nor may "Apache" appear in their name, without
+ *    prior written permission of the Apache Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ */
+
+/**
+ * A stemmer for German words. The algorithm is based on the report
+ * "A Fast and Simple Stemming Algorithm for German Words" by Jörg
+ * Caumanns (joerg.caumanns@isst.fhg.de).
+ *
+ * Changed stem() from protected to public.
+ * Changed coding for umlaute to unicode.
+ *
+ * @author    Gerhard Schwarz
+ * @version   $Id$
+ */
+public class GermanStemmer extends Stemmer
+{
+    /**
+     * Buffer for the terms while stemming them.
+     */
+    private StringBuffer sb = new StringBuffer();
+
+    /**
+     * Indicates if a term is handled as a noun.
+     */
+    private boolean uppercase = false;
+
+    /**
+     * Amount of characters that are removed with <tt>substitute()</tt> while stemming.
+     */
+    private int substCount = 0;
+
+    /**
+     * Stemms the given term to an unique <tt>discriminator</tt>.
+     *
+     * @param term  The term that should be stemmed.
+     * @return      Discriminator for <tt>term</tt>
+     */
+    public String stem( String term )
+    {
+	// Mark a possible noun.
+	uppercase = Character.isUpperCase( term.charAt( 0 ) );
+	// Use lowercase for medium stemming.
+	term = term.toLowerCase();
+	if ( !isStemmable( term ) )
+	    return term;
+	// Reset the StringBuffer.
+	sb.delete( 0, sb.length() );
+	sb.insert( 0, term );
+	// Stemming starts here...
+	substitute( sb );
+	strip( sb );
+	optimize( sb );
+	resubstitute( sb );
+	removeParticleDenotion( sb );
+	return sb.toString();
+    }
+
+    /**
+     * Checks if a term could be stemmed.
+     *
+     * @return  true if, and only if, the given term consists in letters.
+     */
+    private boolean isStemmable( String term )
+    {
+	for ( int c = 0; c < term.length(); c++ ) {
+	    if ( !Character.isLetter( term.charAt( c ) ) ) return false;
+	}
+	return true;
+    }
+
+    /**
+     * suffix stripping (stemming) on the current term. The stripping is reduced
+     * to the seven "base" suffixes "e", "s", "n", "t", "em", "er" and * "nd",
+     * from which all regular suffixes are build of. The simplification causes
+     * some overstemming, and way more irregular stems, but still provides unique.
+     * discriminators in the most of those cases.
+     * The algorithm is context free, except of the length restrictions.
+     */
+    private void strip( StringBuffer buffer )
+    {
+	boolean doMore = true;
+	while ( doMore && buffer.length() > 3 ) {
+	    if ( ( buffer.length() + substCount > 5 ) &&
+		buffer.substring( buffer.length() - 2, buffer.length() ).equals( "nd" ) )
+	    {
+		buffer.delete( buffer.length() - 2, buffer.length() );
+	    }
+	    else if ( ( buffer.length() + substCount > 4 ) &&
+		buffer.substring( buffer.length() - 2, buffer.length() ).equals( "em" ) ) {
+		buffer.delete( buffer.length() - 2, buffer.length() );
+	    }
+	    else if ( ( buffer.length() + substCount > 4 ) &&
+		buffer.substring( buffer.length() - 2, buffer.length() ).equals( "er" ) ) {
+		buffer.delete( buffer.length() - 2, buffer.length() );
+	    }
+	    else if ( buffer.charAt( buffer.length() - 1 ) == 'e' ) {
+		buffer.deleteCharAt( buffer.length() - 1 );
+	    }
+	    else if ( buffer.charAt( buffer.length() - 1 ) == 's' ) {
+		buffer.deleteCharAt( buffer.length() - 1 );
+	    }
+	    else if ( buffer.charAt( buffer.length() - 1 ) == 'n' ) {
+		buffer.deleteCharAt( buffer.length() - 1 );
+	    }
+	    // "t" occurs only as suffix of verbs.
+	    else if ( buffer.charAt( buffer.length() - 1 ) == 't' && !uppercase ) {
+		buffer.deleteCharAt( buffer.length() - 1 );
+	    }
+	    else {
+		doMore = false;
+	    }
+	}
+    }
+
+    /**
+     * Does some optimizations on the term. This optimisations are
+     * contextual.
+     */
+    private void optimize( StringBuffer buffer )
+    {
+	// Additional step for female plurals of professions and inhabitants.
+	if ( buffer.length() > 5 && buffer.substring( buffer.length() - 5, buffer.length() ).equals( "erin*" ) ) {
+	    buffer.deleteCharAt( buffer.length() -1 );
+	    strip( buffer );
+	}
+	// Additional step for irregular plural nouns like "Matrizen -> Matrix".
+	if ( buffer.charAt( buffer.length() - 1 ) == ( 'z' ) ) {
+	    buffer.setCharAt( buffer.length() - 1, 'x' );
+	}
+    }
+
+    /**
+     * Removes a particle denotion ("ge") from a term.
+     */
+    private void removeParticleDenotion( StringBuffer buffer )
+    {
+	if ( buffer.length() > 4 ) {
+	    for ( int c = 0; c < buffer.length() - 3; c++ ) {
+		if ( buffer.substring( c, c + 4 ).equals( "gege" ) ) {
+		    buffer.delete( c, c + 2 );
+		    return;
+		}
+	    }
+	}
+    }
+
+    /**
+     * Do some substitutions for the term to reduce overstemming:
+     *
+     * - Substitute Umlauts with their corresponding vowel: äöü -> aou,
+     *   "ß" is substituted by "ss"
+     * - Substitute a second char of a pair of equal characters with
+     *   an asterisk: ?? -> ?*
+     * - Substitute some common character combinations with a token:
+     *   sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
+     */
+    private void substitute( StringBuffer buffer )
+    {
+	substCount = 0;
+	for ( int c = 0; c < buffer.length(); c++ ) {
+	    // Replace the second char of a pair of the equal characters with an asterisk
+	    if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 )  ) {
+		buffer.setCharAt( c, '*' );
+	    }
+	    // Substitute Umlauts.
+	    else if ( buffer.charAt( c ) == '\u00E4' ) {
+		buffer.setCharAt( c, 'a' );
+	    }
+	    else if ( buffer.charAt( c ) == '\u00F6' ) {
+		buffer.setCharAt( c, 'o' );
+	    }
+	    else if ( buffer.charAt( c ) == '\u00FC' ) {
+		buffer.setCharAt( c, 'u' );
+	    }
+	    // Take care that at least one character is left left side from the current one
+	    if ( c < buffer.length() - 1 ) {
+		if ( buffer.charAt( c ) == '\u00DF' ) {
+		    buffer.setCharAt( c, 's' );
+		    buffer.insert( c + 1, 's' );
+		    substCount++;
+		}
+		// Masking several common character combinations with an token
+		else if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' &&
+		    buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' )
+		{
+		    buffer.setCharAt( c, '$' );
+		    buffer.delete( c + 1, c + 3 );
+		    substCount =+ 2;
+		}
+		else if ( buffer.charAt( c ) == 'c' && buffer.charAt( c + 1 ) == 'h' ) {
+		    buffer.setCharAt( c, '§' );
+		    buffer.deleteCharAt( c + 1 );
+		    substCount++;
+		}
+		else if ( buffer.charAt( c ) == 'e' && buffer.charAt( c + 1 ) == 'i' ) {
+		    buffer.setCharAt( c, '%' );
+		    buffer.deleteCharAt( c + 1 );
+		    substCount++;
+		}
+		else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'e' ) {
+		    buffer.setCharAt( c, '&' );
+		    buffer.deleteCharAt( c + 1 );
+		    substCount++;
+		}
+		else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'g' ) {
+		    buffer.setCharAt( c, '#' );
+		    buffer.deleteCharAt( c + 1 );
+		    substCount++;
+		}
+		else if ( buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 't' ) {
+		    buffer.setCharAt( c, '!' );
+		    buffer.deleteCharAt( c + 1 );
+		    substCount++;
+		}
+	    }
+	}
+    }
+
+    /**
+     * Undoes the changes made by substitute(). That are character pairs and
+     * character combinations. Umlauts will remain as their corresponding vowel,
+     * as "ß" remains as "ss".
+     */
+    private void resubstitute( StringBuffer buffer )
+    {
+	for ( int c = 0; c < buffer.length(); c++ ) {
+	    if ( buffer.charAt( c ) == '*' ) {
+		char x = buffer.charAt( c - 1 );
+		buffer.setCharAt( c, x );
+	    }
+	    else if ( buffer.charAt( c ) == '$' ) {
+		buffer.setCharAt( c, 's' );
+		buffer.insert( c + 1, new char[]{'c', 'h'}, 0, 2 );
+	    }
+	    else if ( buffer.charAt( c ) == '§' ) {
+		buffer.setCharAt( c, 'c' );
+		buffer.insert( c + 1, 'h' );
+	    }
+	    else if ( buffer.charAt( c ) == '%' ) {
+		buffer.setCharAt( c, 'e' );
+		buffer.insert( c + 1, 'i' );
+	    }
+	    else if ( buffer.charAt( c ) == '&' ) {
+		buffer.setCharAt( c, 'i' );
+		buffer.insert( c + 1, 'e' );
+	    }
+	    else if ( buffer.charAt( c ) == '#' ) {
+		buffer.setCharAt( c, 'i' );
+		buffer.insert( c + 1, 'g' );
+	    }
+	    else if ( buffer.charAt( c ) == '!' ) {
+		buffer.setCharAt( c, 's' );
+		buffer.insert( c + 1, 't' );
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/IteratedLovinsStemmer.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/IteratedLovinsStemmer.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/IteratedLovinsStemmer.java	(revision 21112)
@@ -0,0 +1,76 @@
+/*
+ *    IteratedLovinsStemmer.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+
+/**
+ * Implements the iterated version of the Lovins stemmer.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class IteratedLovinsStemmer extends LovinsStemmer {
+
+  /**
+   * Iterated stemming of the given word.
+   */
+  public String stem(String str) {
+
+    if (str.length() <= 2) {
+      return str;
+    }
+    String stemmed = super.stem(str);
+    while (!stemmed.equals(str)) {
+      str = stemmed;
+      stemmed = super.stem(stemmed);
+    }
+    return stemmed;
+  }
+
+  /**
+   * Stems text coming into stdin and writes it to stdout.
+   */
+  public static void main(String[] ops) {
+
+    IteratedLovinsStemmer ls = new IteratedLovinsStemmer();
+
+    try {
+      int num;
+      StringBuffer wordBuffer = new StringBuffer();
+      while ((num = System.in.read()) != -1) {
+	char c = (char)num;
+	if (((num >= (int)'A') && (num <= (int)'Z')) ||
+	    ((num >= (int)'a') && (num <= (int)'z'))) {
+	  wordBuffer.append(c);
+	} else {
+	  if (wordBuffer.length() > 0) {
+	    System.out.print(ls.stem(wordBuffer.toString().
+				     toLowerCase()));
+	    wordBuffer = new StringBuffer();
+	  }
+	  System.out.print(c);
+	}
+      }
+    } catch (Exception e) {
+      System.err.println(e.getMessage());
+    }
+  }
+}
+    
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAFilter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAFilter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAFilter.java	(revision 21112)
@@ -0,0 +1,1496 @@
+/*
+ *    KEAFilter.java
+ *    Copyright (C) 2000, 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+import java.io.*;
+
+import weka.core.*;
+import weka.filters.*;
+import weka.classifiers.*;
+
+/**
+ * This filter converts the incoming data into data appropriate for
+ * keyphrase classification. It assumes that the dataset contains two
+ * string attributes. The first attribute should contain the text of a
+ * document. The second attribute should contain the keyphrases
+ * associated with that document (if present). 
+ *
+ * The filter converts every instance (i.e. document) into a set of
+ * instances, one for each word-based n-gram in the document. The
+ * string attribute representing the document is replaced by some
+ * numeric features, the estimated probability of each n-gram being a
+ * keyphrase, and the rank of this phrase in the document according to
+ * the probability.  Each new instances also has a class value
+ * associated with it. The class is "true" if the n-gram is a true
+ * keyphrase, and "false" otherwise. Of course, if the input document
+ * doesn't come with author-assigned keyphrases, the class values for
+ * that document will be missing.  
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+  */
+public class KEAFilter extends Filter implements OptionHandler {
+
+  /** Index of attribute containing the documents */
+  private int m_DocumentAtt = 0;
+
+  /** Index of attribute containing the keyphrases */
+  private int m_KeyphrasesAtt = 1;
+
+  /** The maximum length of phrases */
+  private int m_MaxPhraseLength = 3;
+
+  /** The minimum length of phrases */
+  private int m_MinPhraseLength = 1;
+
+  /** Is keyphrase frequency attribute being used? */
+  private boolean m_KFused = false;
+
+  /** Flag for debugging mode */
+  private boolean m_Debug = false;
+
+  /** Determines whether internal periods are allowed */
+  private boolean m_DisallowInternalPeriods = false;
+
+  /** The minimum number of occurences of a phrase */
+  private int m_MinNumOccur = 2;
+
+  /** The number of features describing a phrase */
+  private int m_NumFeatures = 2;
+
+  /* Indices of attributes in m_ClassifierData */
+  private int m_TfidfIndex = 0;
+  private int m_FirstOccurIndex = 1;
+  private int m_KeyFreqIndex = 2;
+
+  /** The punctuation filter used by this filter */
+  private KEAPhraseFilter m_PunctFilter = null;
+
+  /** The numbers filter used by this filter */
+  private NumbersFilter m_NumbersFilter = null;
+ 
+  /** The actual classifier used to compute probabilities */
+  private DistributionClassifier m_Classifier = null;
+
+  /** The dictionary containing the document frequencies */
+  private HashMap m_Dictionary = null;
+
+  /** The dictionary containing the keyphrases */
+  private HashMap m_KeyphraseDictionary = null;
+
+  /** The number of documents in the global frequencies corpus */
+  private int m_NumDocs = 0;
+
+  /** Template for the classifier data */
+  private Instances m_ClassifierData = null;
+
+  /** The stemmer to be used */
+  private Stemmer m_Stemmer = new IteratedLovinsStemmer();
+
+  /** The list of stop words to be used */
+  private Stopwords m_Stopwords = new StopwordsEnglish();
+
+  /** Determines whether check for proper nouns is performed */
+  private boolean m_CheckForProperNouns = true;
+
+  /**
+   * Get the M_CheckProperNouns value.
+   * @return the M_CheckProperNouns value.
+   */
+  public boolean getCheckForProperNouns() {
+    return m_CheckForProperNouns;
+  }
+
+  /**
+   * Set the M_CheckProperNouns value.
+   * @param newM_CheckProperNouns The new M_CheckProperNouns value.
+   */
+  public void setCheckForProperNouns(boolean newM_CheckProperNouns) {
+    this.m_CheckForProperNouns = newM_CheckProperNouns;
+  }
+  
+  /**
+   * Get the M_Stopwords value.
+   * @return the M_Stopwords value.
+   */
+  public Stopwords getStopwords() {
+    return m_Stopwords;
+  }
+
+  /**
+   * Set the M_Stopwords value.
+   * @param newM_Stopwords The new M_Stopwords value.
+   */
+  public void setStopwords(Stopwords newM_Stopwords) {
+    this.m_Stopwords = newM_Stopwords;
+  }
+
+  
+  /**
+   * Get the Stemmer value.
+   * @return the Stemmer value.
+   */
+  public Stemmer getStemmer() {
+
+    return m_Stemmer;
+  }
+
+  /**
+   * Set the Stemmer value.
+   * @param newStemmer The new Stemmer value.
+   */
+  public void setStemmer(Stemmer newStemmer) {
+
+    this.m_Stemmer = newStemmer;
+  }
+
+  /**
+   * Get the value of MinNumOccur.
+   *
+   * @return Value of MinNumOccur.
+   */
+  public int getMinNumOccur() {
+    
+    return m_MinNumOccur;
+  }
+  
+  /**
+   * Set the value of MinNumOccur.
+   *
+   * @param newMinNumOccur Value to assign to MinNumOccur.
+   */
+  public void setMinNumOccur(int newMinNumOccur) {
+    
+    m_MinNumOccur = newMinNumOccur;
+  }
+  
+  /**
+   * Get the value of MaxPhraseLength.
+   *
+   * @return Value of MaxPhraseLength.
+   */
+  public int getMaxPhraseLength() {
+    
+    return m_MaxPhraseLength;
+  }
+  
+  /**
+   * Set the value of MaxPhraseLength.
+   *
+   * @param newMaxPhraseLength Value to assign to MaxPhraseLength.
+   */
+  public void setMaxPhraseLength(int newMaxPhraseLength) {
+    
+    m_MaxPhraseLength = newMaxPhraseLength;
+  }
+  
+  /**
+   * Get the value of MinPhraseLength.
+   *
+   * @return Value of MinPhraseLength.
+   */
+  public int getMinPhraseLength() {
+    
+    return m_MinPhraseLength;
+  }
+  
+  /**
+   * Set the value of MinPhraseLength.
+   *
+   * @param newMinPhraseLength Value to assign to MinPhraseLength.
+   */
+  public void setMinPhraseLength(int newMinPhraseLength) {
+    
+    m_MinPhraseLength = newMinPhraseLength;
+  }
+
+  /**
+   * Returns the index of the stemmed phrases in the output ARFF file.
+   */
+  public int getStemmedPhraseIndex() {
+
+    return m_DocumentAtt;
+  }
+
+  /**
+   * Returns the index of the unstemmed phrases in the output ARFF file.
+   */
+  public int getUnstemmedPhraseIndex() {
+
+    return m_DocumentAtt + 1;
+  }
+
+  /**
+   * Returns the index of the phrases' probabilities in the output ARFF file.
+   */
+  public int getProbabilityIndex() {
+
+    int index = m_DocumentAtt + 4;
+
+    if (m_Debug) {
+      if (m_KFused) {
+	index++;
+      }
+    }
+    return index;
+  }
+
+  /**
+   * Returns the index of the phrases' ranks in the output ARFF file.
+   */
+  public int getRankIndex() {
+
+    return getProbabilityIndex() + 1;
+  }
+
+  /**
+   * Get the value of DocumentAtt.
+   *
+   * @return Value of DocumentAtt.
+   */
+  public int getDocumentAtt() {
+    
+    return m_DocumentAtt;
+  }
+  
+  /**
+   * Set the value of DocumentAtt.
+   *
+   * @param newDocumentAtt Value to assign to DocumentAtt.
+   */
+  public void setDocumentAtt(int newDocumentAtt) {
+    
+    m_DocumentAtt = newDocumentAtt;
+  }
+  
+  /**
+   * Get the value of KeyphraseAtt.
+   *
+   * @return Value of KeyphraseAtt.
+   */
+  public int getKeyphrasesAtt() {
+    
+    return m_KeyphrasesAtt;
+  }
+  
+  /**
+   * Set the value of KeyphrasesAtt.
+   *
+   * @param newKeyphrasesAtt Value to assign to KeyphrasesAtt.
+   */
+  public void setKeyphrasesAtt(int newKeyphrasesAtt) {
+    
+    m_KeyphrasesAtt = newKeyphrasesAtt;
+  }
+  
+  
+  /**
+   * Get the value of Debug.
+   *
+   * @return Value of Debug.
+   */
+  public boolean getDebug() {
+    
+    return m_Debug;
+  }
+  
+  /**
+   * Set the value of Debug.
+   *
+   * @param newDebug Value to assign to Debug.
+   */
+  public void setDebug(boolean newDebug) {
+    
+    m_Debug = newDebug;
+  }
+
+  /**
+   * Sets whether keyphrase frequency attribute is used.
+   */
+  public void setKFused(boolean flag) {
+
+    m_KFused = flag;
+    if (flag) {
+      m_NumFeatures = 3;
+    } else {
+      m_NumFeatures = 2;
+    }
+  }
+
+  /**
+   * Gets whether keyphrase frequency attribute is used.
+   */
+  public boolean getKFused() {
+
+    return m_KFused;
+  }
+
+  /**
+   * Get whether the supplied columns are to be processed
+   *
+   * @return true if the supplied columns won't be processed
+   */
+  public boolean getDisallowInternalPeriods() {
+
+    return m_DisallowInternalPeriods;
+  }
+
+  /**
+   * Set whether selected columns should be processed. If true the 
+   * selected columns won't be processed.
+   *
+   * @param invert the new invert setting
+   */
+  public void setDisallowInternalPeriods(boolean disallow) {
+
+    m_DisallowInternalPeriods = disallow;
+  }
+
+  /**
+   * Parses a given list of options controlling the behaviour of this object.
+   * Valid options are:<p>
+   *
+   * -K<br>
+   * Specifies whether keyphrase frequency statistic is used.<p>
+   *
+   * -M length<br>
+   * Sets the maximum phrase length (default: 3).<p>
+   *
+   * -L length<br>
+   * Sets the minimum phrase length (default: 1).<p>
+   *
+   * -D<br>
+   * Turns debugging mode on.<p>
+   *
+   * -I index<br>
+   * Sets the index of the attribute containing the documents (default: 0).<p>
+   *
+   * -J index<br>
+   * Sets the index of the attribute containing the keyphrases (default: 1).<p>
+   *
+   * -P<br>
+   * Disallow internal periods <p>
+   *
+   * -O number<br>
+   * The minimum number of times a phrase needs to occur (default: 2). <p>
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    setKFused(Utils.getFlag('K', options));
+    setDebug(Utils.getFlag('D', options));
+    String docAttIndexString = Utils.getOption('I', options);
+    if (docAttIndexString.length() > 0) {
+      setDocumentAtt(Integer.parseInt(docAttIndexString) - 1);
+    } else {
+      setDocumentAtt(0);
+    }
+    String keyphraseAttIndexString = Utils.getOption('J', options);
+    if (keyphraseAttIndexString.length() > 0) {
+      setKeyphrasesAtt(Integer.parseInt(keyphraseAttIndexString) - 1);
+    } else {
+      setKeyphrasesAtt(1);
+    }
+    String maxPhraseLengthString = Utils.getOption('M', options);
+    if (maxPhraseLengthString.length() > 0) {
+      setMaxPhraseLength(Integer.parseInt(maxPhraseLengthString));
+    } else {
+      setMaxPhraseLength(3);
+    }
+    String minPhraseLengthString = Utils.getOption('M', options);
+    if (minPhraseLengthString.length() > 0) {
+      setMinPhraseLength(Integer.parseInt(minPhraseLengthString));
+    } else {
+      setMinPhraseLength(1);
+    }
+    String minNumOccurString = Utils.getOption('O', options);
+    if (minNumOccurString.length() > 0) {
+      setMinNumOccur(Integer.parseInt(minNumOccurString));
+    } else {
+      setMinNumOccur(2);
+    }
+    setDisallowInternalPeriods(Utils.getFlag('P', options));
+   }
+
+  /**
+   * Gets the current settings of the filter.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [13];
+    int current = 0;
+
+    if (getKFused()) {
+      options[current++] = "-K";
+    }
+    if (getDebug()) {
+      options[current++] = "-D";
+    }
+    options[current++] = "-I"; 
+    options[current++] = "" + (getDocumentAtt() + 1);
+    options[current++] = "-J"; 
+    options[current++] = "" + (getKeyphrasesAtt() + 1);
+    options[current++] = "-M"; 
+    options[current++] = "" + (getMaxPhraseLength());
+    options[current++] = "-L"; 
+    options[current++] = "" + (getMinPhraseLength());
+    options[current++] = "-O"; 
+    options[current++] = "" + (getMinNumOccur());
+
+    if (getDisallowInternalPeriods()) {
+      options[current++] = "-P";
+    }
+
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+  /**
+   * Returns an enumeration describing the available options
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(7);
+
+    newVector.addElement(new Option(
+	      "\tSpecifies whether keyphrase frequency statistic is used.",
+              "K", 0, "-K"));
+    newVector.addElement(new Option(
+	      "\tSets the maximum phrase length (default: 3).",
+              "M", 1, "-M <length>"));
+    newVector.addElement(new Option(
+	      "\tSets the minimum phrase length (default: 1).",
+              "L", 1, "-L <length>"));
+    newVector.addElement(new Option(
+	      "\tTurns debugging mode on.",
+              "D", 0, "-D"));
+    newVector.addElement(new Option(
+	      "\tSets the index of the document attribute (default: 0).",
+              "I", 1, "-I"));
+    newVector.addElement(new Option(
+	      "\tSets the index of the keyphrase attribute (default: 1).",
+              "J", 1, "-J"));
+    newVector.addElement(new Option(
+	      "\tDisallow internal periods.",
+              "P", 0, "-P"));
+    newVector.addElement(new Option(
+	      "\tSet the minimum number of occurences (default: 2).",
+              "O", 1, "-O"));
+
+    return newVector.elements();
+  }
+
+  /**
+   * Returns a string describing this filter
+   *
+   * @return a description of the filter suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String globalInfo() {
+    return "Converts incoming data into data appropriate for " +
+      "keyphrase classification.";
+  }
+
+  /**
+   * Sets the format of the input instances.
+   *
+   * @param instanceInfo an Instances object containing the input
+   * instance structure (any instances contained in the object are
+   * ignored - only the structure is required).
+   * @return true if the outputFormat may be collected immediately 
+   */
+  public boolean setInputFormat(Instances instanceInfo) throws Exception {
+
+    if (instanceInfo.classIndex() >= 0) {
+      throw new Exception("Don't know what do to if class index set!");
+    }
+    if (!instanceInfo.attribute(m_KeyphrasesAtt).isString() ||
+	!instanceInfo.attribute(m_DocumentAtt).isString()) {
+      throw new Exception("Keyphrase attribute and document attribute " +
+			  "need to be string attributes.");
+    }
+    m_PunctFilter = new KEAPhraseFilter();
+    int[] arr = new int[1];
+    arr[0] = m_DocumentAtt;
+    m_PunctFilter.setAttributeIndicesArray(arr);
+    m_PunctFilter.setInputFormat(instanceInfo);
+    m_PunctFilter.setDisallowInternalPeriods(getDisallowInternalPeriods());
+    m_NumbersFilter = new NumbersFilter();
+    m_NumbersFilter.setInputFormat(m_PunctFilter.getOutputFormat());
+    super.setInputFormat(m_NumbersFilter.getOutputFormat());
+    return false;
+  }
+
+  /**
+   * Input an instance for filtering. Ordinarily the instance is processed
+   * and made available for output immediately. Some filters require all
+   * instances be read before producing output.
+   *
+   * @param instance the input instance
+   * @return true if the filtered instance may now be
+   * collected with output().
+   * @exception Exception if the input instance was not of the correct 
+   * format or if there was a problem with the filtering.
+   */
+  public boolean input(Instance instance) throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new Exception("No input instance format defined");
+    }
+    if (m_NewBatch) {
+      resetQueue();
+      m_NewBatch = false;
+    }
+
+    if (m_Debug) {
+      System.err.println("-- Reading instance");
+    }
+
+    m_PunctFilter.input(instance);
+    m_PunctFilter.batchFinished();
+    instance = m_PunctFilter.output();
+
+    m_NumbersFilter.input(instance);
+    m_NumbersFilter.batchFinished();
+    instance = m_NumbersFilter.output();
+    
+    if (m_Dictionary == null) {
+      bufferInput(instance);
+      return false;
+    } else {
+      FastVector vector = convertInstance(instance, false);
+      Enumeration enum = vector.elements();
+      while (enum.hasMoreElements()) {
+	Instance inst = (Instance)enum.nextElement();
+	push(inst);
+      }
+      return true;
+    }
+  }
+
+  /**
+   * Signify that this batch of input to the filter is finished. 
+   * If the filter requires all instances prior to filtering,
+   * output() may now be called to retrieve the filtered instances.
+   *
+   * @return true if there are instances pending output
+   * @exception Exception if no input structure has been defined
+   */
+  public boolean batchFinished() throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new Exception("No input instance format defined");
+    }
+    if (m_Dictionary == null) {
+      buildGlobalDictionaries();
+      buildClassifier();
+      convertPendingInstances();
+    } 
+    flushInput();
+    m_NewBatch = true;
+    return (numPendingOutput() != 0);
+  }
+
+  /**
+   * Builds the global dictionaries.
+   */
+  private void buildGlobalDictionaries() throws Exception {
+
+    if (m_Debug) {
+      System.err.println("--- Building global dictionaries");
+    }
+    
+    // Build dictionary of n-grams with associated
+    // document frequencies
+    m_Dictionary = new HashMap();
+    for (int i = 0; i < getInputFormat().numInstances(); i++) {
+      String str = getInputFormat().instance(i).stringValue(m_DocumentAtt);
+      HashMap hash = getPhrasesForDictionary(str);
+      Iterator it = hash.keySet().iterator();
+      while (it.hasNext()) {
+	String phrase = (String)it.next();
+	Counter counter = (Counter)m_Dictionary.get(phrase);
+	if (counter == null) {
+	  m_Dictionary.put(phrase, new Counter());
+	} else {
+	  counter.increment();
+	}
+      }
+    }
+    
+    if (m_KFused) {
+      
+      // Build dictionary of n-grams that occur as keyphrases
+      // with associated keyphrase frequencies
+      m_KeyphraseDictionary = new HashMap();
+      for (int i = 0; i < getInputFormat().numInstances(); i++) {
+	String str = getInputFormat().instance(i).stringValue(m_KeyphrasesAtt);
+	HashMap hash = getGivenKeyphrases(str, false);
+	if (hash != null) {
+	  Iterator it = hash.keySet().iterator();
+	  while (it.hasNext()) {
+	    String phrase = (String)it.next();
+	    Counter counter = (Counter)m_KeyphraseDictionary.get(phrase);
+	    if (counter == null) {
+	      m_KeyphraseDictionary.put(phrase, new Counter());
+	    } else {
+	      counter.increment();
+	    }
+	  }
+	}
+      }
+    } else {
+      m_KeyphraseDictionary = null;
+    }
+    
+    // Set the number of documents in the global corpus
+    m_NumDocs = getInputFormat().numInstances();
+  }
+ 
+  /**
+   * Builds the classifier.
+   */
+  private void buildClassifier() throws Exception {
+    
+    // Generate input format for classifier
+    FastVector atts = new FastVector();
+    for (int i = 0; i < getInputFormat().numAttributes(); i++) {
+      if (i == m_DocumentAtt) {
+	atts.addElement(new Attribute("TFxIDF"));
+	atts.addElement(new Attribute("First_occurrence"));
+	if (m_KFused) {
+	  atts.addElement(new Attribute("Keyphrase_frequency"));
+	}
+      } else if (i == m_KeyphrasesAtt) {
+	FastVector vals = new FastVector(2);
+	vals.addElement("False");
+	vals.addElement("True");
+	atts.addElement(new Attribute("Keyphrase?", vals));
+      } 
+    }
+    m_ClassifierData = new Instances("ClassifierData", atts, 0);
+    m_ClassifierData.setClassIndex(m_NumFeatures);
+    
+    if (m_Debug) {
+      System.err.println("--- Converting instances for classifier");
+    }
+    
+    // Convert pending input instances into data for classifier
+    for(int i = 0; i < getInputFormat().numInstances(); i++) {
+      Instance current = getInputFormat().instance(i);
+
+      // Get the key phrases for the document
+      String keyphrases = current.stringValue(m_KeyphrasesAtt);
+      HashMap hashKeyphrases = getGivenKeyphrases(keyphrases, false);
+      HashMap hashKeysEval = getGivenKeyphrases(keyphrases, true);
+
+      // Get the phrases for the document
+      HashMap hash = new HashMap();
+      int length = getPhrases(hash, current.stringValue(m_DocumentAtt));
+
+      // Compute the feature values for each phrase and
+      // add the instance to the data for the classifier
+      Iterator it = hash.keySet().iterator();
+      while (it.hasNext()) {
+	String phrase = (String)it.next();
+	FastVector phraseInfo = (FastVector)hash.get(phrase);
+	double[] vals =  featVals(phrase, phraseInfo, true,
+				  hashKeysEval, hashKeyphrases, length);
+	Instance inst = new Instance(current.weight(), vals);
+	m_ClassifierData.add(inst);
+      }
+    }
+    
+    if (m_Debug) {
+      System.err.println("--- Building classifier");
+    }
+    
+    // Build classifier
+    FilteredClassifier fclass = new FilteredClassifier();
+    fclass.setClassifier(new weka.classifiers.NaiveBayesSimple());
+    fclass.setFilter(new DiscretizeFilter());
+    m_Classifier = fclass;
+    m_Classifier.buildClassifier(m_ClassifierData);
+
+    if (m_Debug) {
+      System.err.println(m_Classifier);
+    }
+
+    // Save space
+    m_ClassifierData = new Instances(m_ClassifierData, 0);
+  }
+
+  /** 
+   * Conmputes the feature values for a given phrase.
+   */
+  private double[] featVals(String phrase, FastVector phraseInfo, 
+			    boolean training, HashMap hashKeysEval,
+			    HashMap hashKeyphrases, int length) {
+
+    // Compute feature values
+    Counter counterLocal = (Counter)phraseInfo.elementAt(1);
+    double[] newInst = new double[m_NumFeatures + 1];
+	  
+    // Compute TFxIDF
+    Counter counterGlobal = (Counter)m_Dictionary.get(phrase);
+    double localVal = counterLocal.value(), globalVal = 0;
+    if (counterGlobal != null) {
+      globalVal = counterGlobal.value();
+      if (training) {
+	globalVal = globalVal - 1;
+      }
+    }
+    
+    // Just devide by length to get approximation of probability
+    // that phrase in document is our phrase
+    newInst[m_TfidfIndex] = (localVal / ((double)length)) *
+      (-Math.log((globalVal + 1)/ ((double)m_NumDocs + 1)));
+	  
+    // Compute first occurrence
+    Counter counterFirst = (Counter)phraseInfo.elementAt(0);
+    newInst[m_FirstOccurIndex] = (double)counterFirst.value() /
+      (double)length;
+    
+    // Is keyphrase frequency attribute being used?
+    if (m_KFused) {
+      Counter keyphraseC = (Counter)m_KeyphraseDictionary.get(phrase);
+      if ((training) && (hashKeyphrases != null) &&
+	  (hashKeyphrases.containsKey(phrase))) {
+	newInst[m_KeyFreqIndex] = keyphraseC.value() - 1;
+      } else {
+	if (keyphraseC != null) {
+	  newInst[m_KeyFreqIndex] = keyphraseC.value();
+	} else {
+	  newInst[m_KeyFreqIndex] = 0;
+	}
+      }
+    }
+
+    // Compute class value
+    String phraseInEvalFormat = evalFormat((String)phraseInfo.elementAt(2));
+    if (hashKeysEval == null) { // no author-assigned keyphrases
+      newInst[m_NumFeatures] = Instance.missingValue();
+    } else if (!hashKeysEval.containsKey(phraseInEvalFormat)) {
+      newInst[m_NumFeatures] = 0; // No keyphrase
+    } else {
+      hashKeysEval.remove(phraseInEvalFormat);
+      newInst[m_NumFeatures] = 1; // Keyphrase
+    }
+    return newInst;
+  }
+
+  /**
+   * Sets output format and converts pending input instances.
+   */
+  private void convertPendingInstances() throws Exception {
+
+    if (m_Debug) {
+      System.err.println("--- Converting pending instances");
+    }
+    
+    // Create output format for filter
+    FastVector atts = new FastVector();
+    for (int i = 0; i < getInputFormat().numAttributes(); i++) {
+      if (i == m_DocumentAtt) {
+	atts.addElement(new Attribute("N-gram", null));
+	atts.addElement(new Attribute("N-gram-original", null));
+	atts.addElement(new Attribute("TFxIDF"));
+	atts.addElement(new Attribute("First_occurrence"));
+	if (m_Debug) {
+	  if (m_KFused) {
+	    atts.addElement(new Attribute("Keyphrase_frequency"));
+	  }
+	}
+	atts.addElement(new Attribute("Probability"));
+	atts.addElement(new Attribute("Rank"));
+      } else if (i == m_KeyphrasesAtt) {
+	FastVector vals = new FastVector(2);
+	vals.addElement("False");
+	vals.addElement("True");
+	atts.addElement(new Attribute("Keyphrase?", vals));
+      } else {
+	atts.addElement(getInputFormat().attribute(i));
+      }
+    }
+    Instances outFormat = new Instances("KEAdata", atts, 0);
+    setOutputFormat(outFormat);
+    
+    // Convert pending input instances into output data
+    for(int i = 0; i < getInputFormat().numInstances(); i++) {
+      Instance current = getInputFormat().instance(i);
+      FastVector vector = convertInstance(current, true);
+      Enumeration enum = vector.elements();
+      while (enum.hasMoreElements()) {
+	Instance inst = (Instance)enum.nextElement();
+	push(inst);
+      }
+    }
+  } 
+
+  /**
+   * Converts an instance.
+   */
+  private FastVector convertInstance(Instance instance, boolean training) 
+    throws Exception {
+
+    FastVector vector = new FastVector();
+
+    if (m_Debug) {
+      System.err.println("-- Converting instance");
+    }
+
+    // Get the key phrases for the document
+    HashMap hashKeyphrases = null;
+    HashMap hashKeysEval = null;
+    if (!instance.isMissing(m_KeyphrasesAtt)) {
+      String keyphrases = instance.stringValue(m_KeyphrasesAtt);
+      hashKeyphrases = getGivenKeyphrases(keyphrases, false);
+      hashKeysEval = getGivenKeyphrases(keyphrases, true);
+    }
+
+    // Get the phrases for the document
+    HashMap hash = new HashMap();
+    int length = getPhrases(hash, instance.stringValue(m_DocumentAtt));
+
+    // Compute number of extra attributes
+    int numFeatures = 5;
+    if (m_Debug) {
+      if (m_KFused) {
+	numFeatures = numFeatures + 1;
+      }
+    } 
+
+    // Set indices of key attributes
+    int phraseAttIndex = m_DocumentAtt;
+    int tfidfAttIndex = m_DocumentAtt + 2;
+    int distAttIndex = m_DocumentAtt + 3;
+    int probsAttIndex = m_DocumentAtt + numFeatures - 1;
+
+    // Go through the phrases and convert them into instances
+    Iterator it = hash.keySet().iterator();
+    while (it.hasNext()) {
+      String phrase = (String)it.next();
+      FastVector phraseInfo = (FastVector)hash.get(phrase);
+      double[] vals =  featVals(phrase, phraseInfo, training,
+				hashKeysEval, hashKeyphrases, length);
+      Instance inst = new Instance(instance.weight(), vals);
+      inst.setDataset(m_ClassifierData);
+
+      // Get probability of phrase being key phrase
+      double[] probs = m_Classifier.distributionForInstance(inst);
+      double prob = probs[1];
+
+      // Compute attribute values for final instance
+      double[] newInst = 
+	new double[instance.numAttributes() + numFeatures];
+      int pos = 0;
+      for (int i = 0; i < instance.numAttributes(); i++) {
+	if (i == m_DocumentAtt) {
+	    
+	  // Add phrase
+	  int index = outputFormatPeek().attribute(pos).
+	    addStringValue(phrase);
+	  newInst[pos++] = index;
+	  
+	  // Add original version
+	  index = outputFormatPeek().attribute(pos).
+	    addStringValue((String)phraseInfo.elementAt(2));
+	  newInst[pos++] = index;
+
+	  // Add TFxIDF
+	  newInst[pos++] = inst.value(m_TfidfIndex);
+	    
+	  // Add distance
+	  newInst[pos++] = inst.value(m_FirstOccurIndex);
+
+	    // Add other features
+	  if (m_Debug) {
+	    if (m_KFused) {
+	      newInst[pos++] = inst.value(m_KeyFreqIndex);
+	    }
+	  }
+
+	  // Add probability 
+	  probsAttIndex = pos;
+	  newInst[pos++] = prob;
+
+	  // Set rank to missing (computed below)
+	  newInst[pos++] = Instance.missingValue();
+	} else if (i == m_KeyphrasesAtt) {
+	  newInst[pos++] = inst.classValue();
+	} else {
+	  newInst[pos++] = instance.value(i);
+	}
+      }
+      Instance ins = new Instance(instance.weight(), newInst);
+      ins.setDataset(outputFormatPeek());
+      vector.addElement(ins);
+    }
+
+    // Add dummy instances for keyphrases that don't occur
+    // in the document
+    if (hashKeysEval != null) {
+      Iterator phrases = hashKeysEval.keySet().iterator();
+      while (phrases.hasNext()) {
+	String phrase = (String)phrases.next();
+	double[] newInst = 
+	  new double[instance.numAttributes() + numFeatures];
+	int pos = 0;
+	for (int i = 0; i < instance.numAttributes(); i++) {
+	  if (i == m_DocumentAtt) {
+	    
+	    // Add phrase
+	    int index = outputFormatPeek().attribute(pos).
+	      addStringValue(phrase);
+	    newInst[pos++] = (double)index;
+	    
+	    // Add original version
+	    index = outputFormatPeek().attribute(pos).
+	      addStringValue((String)hashKeysEval.get(phrase));
+	    newInst[pos++] = (double)index;
+	    
+	    // Add TFxIDF
+	    newInst[pos++] = Instance.missingValue();
+	    
+	    // Add distance
+	    newInst[pos++] = Instance.missingValue();
+	      
+	    // Add other features
+	    if (m_Debug) {
+	      if (m_KFused) {
+		newInst[pos++] = Instance.missingValue();
+	      }
+	    }
+
+	    // Add probability and rank
+	    newInst[pos++] = -Double.MAX_VALUE;
+	    newInst[pos++] = Instance.missingValue();
+	  } else if (i == m_KeyphrasesAtt) {
+	    newInst[pos++] = 1; // Keyphrase
+	  } else {
+	    newInst[pos++] = instance.value(i);
+	  } 
+	}
+	Instance inst = new Instance(instance.weight(), newInst);
+	inst.setDataset(outputFormatPeek());
+	vector.addElement(inst);
+      }
+    }
+
+    // Sort phrases according to their distance (stable sort)
+    double[] vals = new double[vector.size()];
+    for (int i = 0; i < vals.length; i++) {
+      vals[i] = ((Instance)vector.elementAt(i)).value(distAttIndex);
+    }
+    FastVector newVector = new FastVector(vector.size());
+    int[] sortedIndices = Utils.stableSort(vals);
+    for (int i = 0; i < vals.length; i++) {
+      newVector.addElement(vector.elementAt(sortedIndices[i]));
+    }
+    vector = newVector;
+
+    // Sort phrases according to their tfxidf value (stable sort)
+    for (int i = 0; i < vals.length; i++) {
+      vals[i] = -((Instance)vector.elementAt(i)).value(tfidfAttIndex);
+    }
+    newVector = new FastVector(vector.size());
+    sortedIndices = Utils.stableSort(vals);
+    for (int i = 0; i < vals.length; i++) {
+      newVector.addElement(vector.elementAt(sortedIndices[i]));
+    }
+    vector = newVector;
+
+    // Sort phrases according to their probability (stable sort)
+    for (int i = 0; i < vals.length; i++) {
+      vals[i] = 1 - ((Instance)vector.elementAt(i)).value(probsAttIndex);
+    }
+    newVector = new FastVector(vector.size());
+    sortedIndices = Utils.stableSort(vals);
+    for (int i = 0; i < vals.length; i++) {
+      newVector.addElement(vector.elementAt(sortedIndices[i]));
+    }
+    vector = newVector;
+
+    // Compute rank of phrases. Check for subphrases that are ranked
+    // lower than superphrases and assign probability -1 and set the
+    // rank to Integer.MAX_VALUE
+    int rank = 1;
+    for (int i = 0; i < vals.length; i++) {
+      Instance currentInstance = (Instance)vector.elementAt(i);
+
+      // Short cut: if phrase very unlikely make rank very low and continue
+      if (Utils.grOrEq(vals[i], 1.0)) {
+	currentInstance.setValue(probsAttIndex + 1, Integer.MAX_VALUE);
+	continue;
+      }
+
+      // Otherwise look for super phrase starting with first phrase
+      // in list that has same probability, TFxIDF value, and distance as
+      // current phrase. We do this to catch all superphrases
+      // that have same probability, TFxIDF value and distance as current phrase.
+      int startInd = i;
+      while (startInd < vals.length) {
+	Instance inst = (Instance)vector.elementAt(startInd);
+	if ((inst.value(tfidfAttIndex) != 
+	     currentInstance.value(tfidfAttIndex)) ||
+	    (inst.value(probsAttIndex) != 
+	     currentInstance.value(probsAttIndex)) ||
+	    (inst.value(distAttIndex) !=
+	     currentInstance.value(distAttIndex))) {
+	  break;
+	}
+	startInd++;
+      }
+      String val = currentInstance.stringValue(phraseAttIndex);
+      boolean foundSuperphrase = false;
+      for (int j = startInd - 1; j >= 0; j--) {
+	if (j != i) {
+	  Instance candidate = (Instance)vector.elementAt(j);
+	  String potSuperphrase = candidate.stringValue(phraseAttIndex);
+	  if (val.length() <= potSuperphrase.length()) {
+	    if (KEAFilter.contains(val, potSuperphrase)) {
+	      foundSuperphrase = true;
+	      break;
+	    }
+	  }
+	}
+      }
+      if (foundSuperphrase) {
+	currentInstance.setValue(probsAttIndex + 1, Integer.MAX_VALUE);
+      } else {
+	currentInstance.setValue(probsAttIndex + 1, rank++);
+      }
+    }
+    return vector;
+  }
+    
+  /** 
+   * Checks whether one phrase is a subphrase of another phrase.
+   */
+  private static boolean contains(String sub, String sup) {
+
+    int i = 0;
+    while (i + sub.length() - 1 < sup.length()) {
+      int j;
+      for (j = 0; j < sub.length(); j++) {
+	if (sub.charAt(j) != sup.charAt(i + j)) {
+	  break;
+	}
+      }
+      if (j == sub.length()) {
+	if ((i + j) < sup.length()) {
+	  if (sup.charAt(i + j) == ' ') {
+	    return true;
+	  } else {
+	    return false;
+	  }
+	} else {
+	  return true;
+	}
+      }
+
+      // Skip forward to next space
+      do {
+	i++;
+      } while ((i < sup.length()) && (sup.charAt(i) != ' '));
+      i++;
+    }
+    return false;
+  }
+
+  /**
+   * Returns a hashtable. Fills the hashtable
+   * with the stemmed n-grams occuring in the given string
+   * (as keys) and the number of times it occurs.
+   */
+  private HashMap getPhrasesForDictionary(String str) {
+
+    String[] buffer = new String[m_MaxPhraseLength];
+    HashMap hash = new HashMap();
+
+    StringTokenizer tok = new StringTokenizer(str, "\n");
+    while (tok.hasMoreTokens()) {
+      String phrase = tok.nextToken();
+      int numSeen = 0;
+      StringTokenizer wordTok = new StringTokenizer(phrase, " ");
+      while (wordTok.hasMoreTokens()) {
+	String word = wordTok.nextToken();
+	
+	// Store word in buffer
+	for (int i = 0; i < m_MaxPhraseLength - 1; i++) {
+	  buffer[i] = buffer[i + 1];
+	}
+	buffer[m_MaxPhraseLength - 1] = word;
+	
+	// How many are buffered?
+	numSeen++;
+	if (numSeen > m_MaxPhraseLength) {
+	  numSeen = m_MaxPhraseLength;
+	}
+	
+	// Don't consider phrases that end with a stop word
+	if (m_Stopwords.isStopword(buffer[m_MaxPhraseLength - 1])) {
+	  continue;
+	}
+	
+	// Loop through buffer and add phrases to hashtable
+	StringBuffer phraseBuffer = new StringBuffer();
+	for (int i = 1; i <= numSeen; i++) {
+	  if (i > 1) {
+	    phraseBuffer.insert(0, ' ');
+	  }
+	  phraseBuffer.insert(0, buffer[m_MaxPhraseLength - i]);
+	  
+	  // Don't consider phrases that begin with a stop word
+	  if ((i > 1) && 
+	      (m_Stopwords.isStopword(buffer[m_MaxPhraseLength - i]))) {
+	    continue;
+	  }
+	  
+	  // Only consider phrases with minimum length
+	  if (i >= m_MinPhraseLength) {
+	    
+	    // Stem string
+	    String orig = phraseBuffer.toString();
+	    String internal = internalFormat(orig);
+	    Counter count = (Counter)hash.get(internal);
+	    if (count == null) {
+	      hash.put(internal, new Counter());
+	    } else {
+	      count.increment();
+	    }
+	    
+	    // Add components if phrase is single word before
+	    // conversion into internal format (i.e. error-correcting)
+	    /*if ((orig.indexOf(' ') == -1) &&
+	      (internal.indexOf(' ') != -1)) {
+	      StringTokenizer tokW = new StringTokenizer(internal, " ");
+	      while (tokW.hasMoreTokens()) {
+	      String comp = (String)tokW.nextToken();
+	      Counter countW = (Counter)hash.get(comp);
+	      if (countW == null) {
+	      hash.put(comp, new Counter());
+	      } else {
+	      countW.increment();
+	      }
+	      }
+	      }*/
+	  }
+	}
+      }
+    }
+    return hash;
+  }
+
+  /**
+   * Expects an empty hashtable. Fills the hashtable
+   * with the stemmed n-grams occuring in the given string
+   * (as keys). Stores the position, the number of occurences,
+   * and the most commonly occurring orgininal version of
+   * each n-gram.
+   *
+   * N-grams that occur less than m_MinNumOccur are not used.
+   *
+   * Returns the total number of words (!) in the string.
+   */
+  private int getPhrases(HashMap hash, String str) {
+
+    String[] buffer = new String[m_MaxPhraseLength];
+    
+    StringTokenizer tok = new StringTokenizer(str, "\n");
+    int pos = 1; 
+    while (tok.hasMoreTokens()) {
+      String phrase = tok.nextToken();
+      int numSeen = 0;
+      StringTokenizer wordTok = new StringTokenizer(phrase, " ");
+      while (wordTok.hasMoreTokens()) {
+	String word = wordTok.nextToken();
+	
+	// Store word in buffer
+	for (int i = 0; i < m_MaxPhraseLength - 1; i++) {
+	  buffer[i] = buffer[i + 1];
+	}
+	buffer[m_MaxPhraseLength - 1] = word;
+	
+	// How many are buffered?
+	numSeen++;
+	if (numSeen > m_MaxPhraseLength) {
+	  numSeen = m_MaxPhraseLength;
+	}
+	
+	// Don't consider phrases that end with a stop word
+	if (m_Stopwords.isStopword(buffer[m_MaxPhraseLength - 1])) {
+	  pos++;
+	  continue;
+	}
+	
+	// Loop through buffer and add phrases to hashtable
+	StringBuffer phraseBuffer = new StringBuffer();
+	for (int i = 1; i <= numSeen; i++) {
+	  if (i > 1) {
+	    phraseBuffer.insert(0, ' ');
+	  }
+	  phraseBuffer.insert(0, buffer[m_MaxPhraseLength - i]);
+	  
+	  // Don't consider phrases that begin with a stop word
+	  if ((i > 1) && 
+	      (m_Stopwords.isStopword(buffer[m_MaxPhraseLength - i]))) {
+	    continue;
+	  }
+	  
+	  // Only consider phrases with minimum length
+	  if (i >= m_MinPhraseLength) {
+	    
+	    // Stem string
+	    String phrStr = phraseBuffer.toString();
+	    String internal = internalFormat(phrStr);
+	    FastVector vec = (FastVector)hash.get(internal);
+	    if (vec == null) {
+	      vec = new FastVector(3);
+	      
+	      // HashMap for storing all versions
+	      HashMap secHash = new HashMap();
+	      secHash.put(phrStr, new Counter());
+	      
+	      // Update hashtable with all the info
+	      vec.addElement(new Counter(pos + 1 - i));
+	      vec.addElement(new Counter());
+	      vec.addElement(secHash);
+	      hash.put(internal, vec);
+	    } else {
+	      
+	      // Update number of occurrences
+	      ((Counter)((FastVector)vec).elementAt(1)).increment();
+	      
+	      // Update hashtable storing different versions
+	      HashMap secHash = (HashMap)vec.elementAt(2);
+	      Counter count = (Counter)secHash.get(phrStr);
+	      if (count == null) {
+		secHash.put(phrStr, new Counter());
+	      } else {
+		count.increment();
+	      }
+	    }
+	  }
+	}
+	pos++;
+      }
+    }
+    
+    // Replace secondary hashtables with most commonly occurring
+    // version of each phrase (canonical) form. Delete all words
+    // that are proper nouns.
+    Iterator phrases = hash.keySet().iterator();
+    while (phrases.hasNext()) {
+      String phrase = (String)phrases.next();
+      FastVector info = (FastVector)hash.get(phrase);
+
+      // Occurring less than m_MinNumOccur?
+      if (((Counter)((FastVector)info).elementAt(1)).value() < m_MinNumOccur) {
+	phrases.remove();
+	continue;
+      }
+
+      // Get canonical form
+      String canForm = canonicalForm((HashMap)info.elementAt(2));
+      if (canForm == null) {
+	phrases.remove();
+      } else {
+	info.setElementAt(canForm, 2);
+      }
+    }
+    return pos;
+  }
+
+  /**
+   * Create canonical form of phrase.
+   */
+  private String canonicalForm(HashMap secHash) {
+
+    int max = 0; String bestVersion = null;
+    boolean allFullyHyphenated = true;
+    int num = 0;
+    Iterator versions = secHash.keySet().iterator();
+    while (versions.hasNext()) {
+      num++;
+      String version = (String)versions.next();
+
+      // Are all the words joined up?
+      if (!isFullyHyphenated(version)) {
+	allFullyHyphenated = false;
+      }
+
+      // Check for how often this version occurs
+      Counter count = (Counter)secHash.get(version);
+      if (count.value() > max) {
+	max = count.value();
+	bestVersion = version;
+      }
+    }
+    if ((getCheckForProperNouns()) && (num == 1) && properNoun(bestVersion)) {
+    //if (allProperNouns) {
+      return null;
+    } else {
+      if (isFullyHyphenated(bestVersion) &&
+	  !allFullyHyphenated) {
+	bestVersion = bestVersion.replace('-', ' ');
+      }
+      return bestVersion;
+    }
+  }
+
+  /**
+   * Checks whether the given phrase is
+   * fully hyphenated.
+   */
+  private boolean isFullyHyphenated(String str) {
+    
+    return (str.indexOf(' ') == -1);
+  }
+
+  /**
+   * Checks whether the given string is a 
+   * proper noun.
+   *
+   * @return true if it is a potential proper noun
+   */
+  private static boolean properNoun(String str) {
+
+    // Is it more than one word?
+    if (str.indexOf(' ') != -1) {
+      return false;
+    }
+
+    // Does it start with an upper-case character?
+    if (Character.isLowerCase(str.charAt(0))) {
+      return false;
+    } 
+
+    // Is there at least one character that's
+    // not upper-case?
+    for (int i = 1; i < str.length(); i++) {
+      /*if (Character.isUpperCase(str.charAt(i))) {
+	return false;
+	}*/
+      if (!Character.isUpperCase(str.charAt(i))) {
+	return true;
+      }
+    }
+    //return true;
+    return false;
+  }
+
+  /**
+   * Generates the evaluation format of a phrase.
+   */
+  private String evalFormat(String str) {
+    
+    return m_Stemmer.stemString(str);
+  }
+
+  /** 
+   * Generates the internal format of a phrase.
+   */
+  private String internalFormat(String str) {
+
+    // Remove some non-alphanumeric characters
+    str = str.replace('-', ' ');
+    str = str.replace('/', ' ');
+    str = str.replace('&', ' ');
+
+    // Stem string
+    return m_Stemmer.stemString(str);
+  }
+
+  /**
+   * Gets all the phrases in the given string and puts them into the
+   * hashtable.  Also stores the original version of the stemmed
+   * phrase in the hash table.  
+   */
+  private HashMap getGivenKeyphrases(String str,
+				       boolean forEval) {
+
+    FastVector vector = new FastVector();
+    HashMap hash = new HashMap();
+    
+    StringTokenizer tok = new StringTokenizer(str, "\n");
+    while (tok.hasMoreTokens()) {
+      String orig = tok.nextToken();
+      orig = orig.trim();
+      if (orig.length() > 0) {
+	String modified;
+	if (!forEval) {
+	  modified = internalFormat(orig);
+	} else {
+	  modified = evalFormat(orig);
+	}
+	if (!hash.containsKey(modified)) {
+	  hash.put(modified, orig);
+	} else {
+	  if (forEval) {
+	    System.err.println("WARNING: stem of author-assigned keyphrase " +
+			       orig + " matches existing stem (skipping it)!");
+	  }
+	}
+      }
+    }
+    if (hash.size() == 0) {
+      return null;
+    } else {
+      return hash;
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain arguments to the filter: use -h for help
+   */
+  public static void main(String [] argv) {
+    
+    try {
+      if (Utils.getFlag('b', argv)) {
+	Filter.batchFilterFile(new KEAFilter(), argv);
+      } else {
+	Filter.filterFile(new KEAFilter(), argv);
+      }
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAKeyphraseExtractor.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAKeyphraseExtractor.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAKeyphraseExtractor.java	(revision 21112)
@@ -0,0 +1,532 @@
+/*
+ *    KEAKeyphraseExtractor.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+import java.io.*;
+import java.util.*;
+import weka.core.*;
+import weka.filters.*;
+
+/**
+ * Extracts keyphrases from the documents in a given directory.
+ * Assumes that the file names for the documents end with ".txt".
+ * Puts extracted keyphrases into corresponding files ending with
+ * ".key" (if those are not already present). Optionally an encoding
+ * for the documents/keyphrases can be defined (e.g. for Chinese
+ * text). Documents for which ".key" exists, are used for evaluation.
+ *
+ * Valid options are:<p>
+ *
+ * -l "directory name"<br>
+ * Specifies name of directory.<p>
+ *
+ * -m "model name"<br>
+ * Specifies name of model.<p>
+ *
+ * -e "encoding"<br>
+ * Specifies encoding.<p>
+ *
+ * -n <br>
+ * Specifies number of phrases to be output (default: 5).<p>
+ *
+ * -d<br>
+ * Turns debugging mode on.<p>
+ *
+ * -a<br>
+ * Also write stemmed phrase and score into ".key" file.<p>
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class KEAKeyphraseExtractor implements OptionHandler {
+  
+  /** Name of directory */
+  String m_dirName = null;
+  
+  /** Name of model */
+  String m_modelName = null;
+
+  /** Encoding */
+  String m_encoding = "default";
+
+  /** Debugging mode? */
+  boolean m_debug = false;
+
+  /** The KEA filter object */
+  KEAFilter m_KEAFilter = null;
+  
+  /** The number of phrases to extract. */
+  int m_numPhrases = 5;
+
+  /** Also write stemmed phrase and score into .key file. */
+  boolean m_AdditionalInfo = false;
+  
+  /**
+   * Get the value of AdditionalInfo.
+   *
+   * @return Value of AdditionalInfo.
+   */
+  public boolean getAdditionalInfo() {
+    
+    return m_AdditionalInfo;
+  }
+  
+  /**
+   * Set the value of AdditionalInfo.
+   *
+   * @param newAdditionalInfo Value to assign to AdditionalInfo.
+   */
+  public void setAdditionalInfo(boolean newAdditionalInfo) {
+    
+    m_AdditionalInfo = newAdditionalInfo;
+  }
+  
+  /**
+   * Get the value of numPhrases.
+   *
+   * @return Value of numPhrases.
+   */
+  public int getNumPhrases() {
+    
+    return m_numPhrases;
+  }
+  
+  /**
+   * Set the value of numPhrases.
+   *
+   * @param newnumPhrases Value to assign to numPhrases.
+   */
+  public void setNumPhrases(int newnumPhrases) {
+    
+    m_numPhrases = newnumPhrases;
+  }
+  
+  /**
+   * Get the value of debug.
+   *
+   * @return Value of debug.
+   */
+  public boolean getDebug() {
+    
+    return m_debug;
+  }
+  
+  /**
+   * Set the value of debug.
+   *
+   * @param newdebug Value to assign to debug.
+   */
+  public void setDebug(boolean newdebug) {
+    
+    m_debug = newdebug;
+  }
+  
+  /**
+   * Get the value of encoding.
+   *
+   * @return Value of encoding.
+   */
+  public String getEncoding() {
+    
+    return m_encoding;
+  }
+  
+  /**
+   * Set the value of encoding.
+   *
+   * @param newencoding Value to assign to encoding.
+   */
+  public void setEncoding(String newencoding) {
+    
+    m_encoding = newencoding;
+  }
+  
+  /**
+   * Get the value of modelName.
+   *
+   * @return Value of modelName.
+   */
+  public String getModelName() {
+    
+    return m_modelName;
+  }
+  
+  /**
+   * Set the value of modelName.
+   *
+   * @param newmodelName Value to assign to modelName.
+   */
+  public void setModelName(String newmodelName) {
+    
+    m_modelName = newmodelName;
+  }
+  
+  /**
+   * Get the value of dirName.
+   *
+   * @return Value of dirName.
+   */
+  public String getDirName() {
+    
+    return m_dirName;
+  }
+  
+  /**
+   * Set the value of dirName.
+   *
+   * @param newdirName Value to assign to dirName.
+   */
+  public void setDirName(String newdirName) {
+    
+    m_dirName = newdirName;
+  }
+  
+  /**
+   * Parses a given list of options controlling the behaviour of this object.
+   * Valid options are:<p>
+   *
+   * -l "directory name"<br>
+   * Specifies name of directory.<p>
+   *
+   * -m "model name"<br>
+   * Specifies name of model.<p>
+   *
+   * -e "encoding"<br>
+   * Specifies encoding.<p>
+   *
+   * -n<br>
+   * Specifies number of phrases to be output (default: 5).<p>
+   *
+   * -d<br>
+   * Turns debugging mode on.<p>
+   *
+   * -a<br>
+   * Also write stemmed phrase and score into ".key" file.<p>
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    String dirName = Utils.getOption('l', options);
+    if (dirName.length() > 0) {
+      setDirName(dirName);
+    } else {
+      setDirName(null);
+      throw new Exception("Name of directory required argument.");
+    }
+    String modelName = Utils.getOption('m', options);
+    if (modelName.length() > 0) {
+      setModelName(modelName);
+    } else {
+      setModelName(null);
+      throw new Exception("Name of model required argument.");
+    }
+    String encoding = Utils.getOption('e', options);
+    if (encoding.length() > 0) {
+      setEncoding(encoding);
+    } else {
+      setEncoding("default");
+    }
+    String numPhrases = Utils.getOption('n', options);
+    if (numPhrases.length() > 0) {
+      setNumPhrases(Integer.parseInt(numPhrases));
+    } else {
+      setNumPhrases(5);
+    }
+    setDebug(Utils.getFlag('d', options));
+    setAdditionalInfo(Utils.getFlag('a', options));
+    Utils.checkForRemainingOptions(options);
+   }
+
+  /**
+   * Gets the current option settings.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [10];
+    int current = 0;
+
+    options[current++] = "-l"; 
+    options[current++] = "" + (getDirName());
+    options[current++] = "-m"; 
+    options[current++] = "" + (getModelName());
+    options[current++] = "-e"; 
+    options[current++] = "" + (getEncoding());
+    options[current++] = "-n"; 
+    options[current++] = "" + (getNumPhrases());
+    if (getDebug()) {
+      options[current++] = "-d";
+    }
+    if (getAdditionalInfo()) {
+      options[current++] = "-a";
+    }
+
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+  /**
+   * Returns an enumeration describing the available options.
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(6);
+
+    newVector.addElement(new Option(
+	      "\tSpecifies name of directory.",
+              "l", 1, "-l <directory name>"));
+    newVector.addElement(new Option(
+	      "\tSpecifies name of model.",
+              "m", 1, "-m <model name>"));
+    newVector.addElement(new Option(
+	      "\tSpecifies encoding.",
+              "e", 1, "-e <encoding>"));
+    newVector.addElement(new Option(
+	      "\tSpecifies number of phrases to be output (default: 5).",
+              "n", 1, "-n"));
+    newVector.addElement(new Option(
+	      "\tTurns debugging mode on.",
+              "d", 0, "-d"));
+    newVector.addElement(new Option(
+	      "\tAlso write stemmed phrase and score into \".key\" file.",
+              "a", 0, "-a"));
+
+    return newVector.elements();
+  }
+
+  /**
+   * Collects the stems of the file names.
+   */
+  public Hashtable collectStems() throws Exception {
+
+    Hashtable stems = new Hashtable();
+
+    try {
+      File dir = new File(m_dirName);
+      String[] files = dir.list();
+      for (int i = 0; i < files.length; i++) {
+	if (files[i].endsWith(".txt")) {
+	  String stem = files[i].substring(0, files[i].length() - 4);
+	  if (!stems.containsKey(stem)) {
+	    stems.put(stem, new Double(0));
+	  }
+	}
+      }
+    } catch (Exception e) {
+      throw new Exception("Problem opening directory " + m_dirName);
+    }
+    return stems;
+  }
+
+  /**
+   * Builds the model from the files
+   */
+  public void extractKeyphrases(Hashtable stems) throws Exception {
+    
+    Vector stats = new Vector();
+
+    // Check whether there is actually any data
+    if (stems.size() == 0) {
+      throw new Exception("Couldn't find any data!");
+    }
+
+    FastVector atts = new FastVector(2);
+    atts.addElement(new Attribute("doc", null));
+    atts.addElement(new Attribute("keyphrases", null));
+    Instances data = new Instances("keyphrase_training_data", atts, 0);
+
+    // Extract keyphrases
+    Enumeration elem = stems.keys();
+    while (elem.hasMoreElements()) {
+      String str = (String)elem.nextElement();
+      double[] newInst = new double[2];
+      try {
+	File txt = new File(m_dirName + "/" + str + ".txt");
+	InputStreamReader is;
+	if (!m_encoding.equals("default")) {
+	  is = new InputStreamReader(new FileInputStream(txt), m_encoding);
+	} else {
+	  is = new InputStreamReader(new FileInputStream(txt));
+	}
+	StringBuffer txtStr = new StringBuffer();
+	int c;
+	while ((c = is.read()) != -1) {
+	  txtStr.append((char)c);
+	}
+	newInst[0] = (double)data.attribute(0).addStringValue(txtStr.toString());
+      } catch (Exception e) {
+	if (m_debug) {
+	  System.err.println("Can't read document " + str + ".txt");
+	}
+	newInst[0] = Instance.missingValue();
+      }
+      try {
+	File key = new File(m_dirName + "/" + str + ".key");
+	InputStreamReader is; 
+	if (!m_encoding.equals("default")) {
+	  is = new InputStreamReader(new FileInputStream(key), m_encoding);
+	} else {
+	  is = new InputStreamReader(new FileInputStream(key));
+	}
+	StringBuffer keyStr = new StringBuffer();
+	int c;
+	while ((c = is.read()) != -1) {
+	  keyStr.append((char)c);
+	}      
+	newInst[1] = (double)data.attribute(1).addStringValue(keyStr.toString());
+      } catch (Exception e) {
+	if (m_debug) {
+	  System.err.println("No keyphrases for stem " + str + ".");
+	}
+	newInst[1] = Instance.missingValue();
+      }
+      data.add(new Instance(1.0, newInst));
+      m_KEAFilter.input(data.instance(0));
+      data = data.stringFreeStructure();
+      if (m_debug) {
+	System.err.println("-- Document: " + str);
+      }
+      Instance[] topRankedInstances = new Instance[m_numPhrases];
+      Instance inst;
+      while ((inst = m_KEAFilter.output()) != null) {
+	int index = (int)inst.value(m_KEAFilter.getRankIndex()) - 1;
+	if (index < m_numPhrases) {
+	  topRankedInstances[index] = inst;
+	}
+      }
+      if (m_debug) {
+	System.err.println("-- Keyphrases and feature values:");
+      }
+      FileOutputStream out = null;
+      PrintWriter printer = null; 
+      File key = new File(m_dirName + "/" + str + ".key");
+      if (!key.exists()) {
+	out = new FileOutputStream(m_dirName + "/" + str + ".key");
+	if (!m_encoding.equals("default")) {
+	  printer = new PrintWriter(new OutputStreamWriter(out, m_encoding));
+	} else {
+	  printer = new PrintWriter(out);
+	}
+      }
+      double numExtracted = 0, numCorrect = 0;
+      for (int i = 0; i < m_numPhrases; i++) {
+	if (topRankedInstances[i] != null) {
+	  if (!topRankedInstances[i].
+	      isMissing(topRankedInstances[i].numAttributes() - 1)) {
+	    numExtracted += 1.0;
+	  }
+	  if ((int)topRankedInstances[i].
+	      value(topRankedInstances[i].numAttributes() - 1) == 
+	      topRankedInstances[i].
+	      attribute(topRankedInstances[i].numAttributes() - 1).
+	      indexOfValue("True")) {
+	    numCorrect += 1.0;
+	  }
+	  if (printer != null) {
+	    printer.print(topRankedInstances[i].
+			  stringValue(m_KEAFilter.getUnstemmedPhraseIndex()));
+	    if (m_AdditionalInfo) {
+	      printer.print("\t");
+	      printer.print(topRankedInstances[i].
+			  stringValue(m_KEAFilter.getStemmedPhraseIndex()));
+	      printer.print("\t");
+	      printer.print(Utils.
+			    doubleToString(topRankedInstances[i].
+					   value(m_KEAFilter.
+						 getProbabilityIndex()), 4));
+	    }
+	    printer.println();
+	  }
+	  if (m_debug) {
+	    System.err.println(topRankedInstances[i]);
+	  }
+	}
+      }
+      if (numExtracted > 0) {
+	if (m_debug) {
+	  System.err.println("-- " + numCorrect + " correct");
+	}
+	stats.addElement(new Double(numCorrect));
+      }
+      if (printer != null) {
+	printer.flush();
+	printer.close();
+	out.close();
+      }
+    }
+    double[] st = new double[stats.size()];
+    for (int i = 0; i < stats.size(); i++) {
+      st[i] = ((Double)stats.elementAt(i)).doubleValue();
+    }
+    double avg = Utils.mean(st);
+    double stdDev = Math.sqrt(Utils.variance(st));
+    System.err.println("Avg. number of correct keyphrases: " +
+		       Utils.doubleToString(avg, 2) + " +/- " + 
+		       Utils.doubleToString(stdDev, 2));
+    System.err.println("Based on " + stats.size() + " documents");
+    m_KEAFilter.batchFinished();
+  }
+  
+  /** 
+   * Loads the extraction model from the file.
+   */
+  public void loadModel() throws Exception {
+    
+    BufferedInputStream inStream =
+      new BufferedInputStream(new FileInputStream(m_modelName));
+    ObjectInputStream in = new ObjectInputStream(inStream);
+    m_KEAFilter = (KEAFilter)in.readObject();
+    in.close();
+  }
+  
+  /**
+   * The main method.  
+   */
+  public static void main(String[] ops) {
+    
+    KEAKeyphraseExtractor kmb = new KEAKeyphraseExtractor();
+    try {
+      kmb.setOptions(ops);
+      System.err.print("Extracting keyphrases with options: ");
+      String[] optionSettings = kmb.getOptions();
+      for (int i = 0; i < optionSettings.length; i++) {
+	System.err.print(optionSettings[i] + " ");
+      }
+      System.err.println();
+      kmb.loadModel();
+      kmb.extractKeyphrases(kmb.collectStems());
+    } catch (Exception e) {
+      e.printStackTrace();
+      System.err.println(e.getMessage());
+      System.err.println("\nOptions:\n");
+      Enumeration enum = kmb.listOptions();
+      while (enum.hasMoreElements()) {
+	Option option = (Option) enum.nextElement();
+	System.err.println(option.synopsis());
+	System.err.println(option.description());
+      }
+    }
+  }
+}
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAModelBuilder.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAModelBuilder.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAModelBuilder.java	(revision 21112)
@@ -0,0 +1,689 @@
+/*
+ *    KEAModelBuilder.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.io.*;
+import java.util.*;
+import weka.core.*;
+import weka.filters.*;
+
+/**
+ * Builds a keyphrase extraction model from the documents in a given
+ * directory.  Assumes that the file names for the documents end with
+ * ".txt".  Assumes that files containing corresponding
+ * author-assigned keyphrases end with ".key". Optionally an encoding
+ * for the documents/keyphrases can be defined (e.g. for Chinese
+ * text).
+ *
+ * Valid options are:<p>
+ *
+ * -l "directory name"<br>
+ * Specifies name of directory.<p>
+ *
+ * -m "model name"<br>
+ * Specifies name of model.<p>
+ *
+ * -e "encoding"<br>
+ * Specifies encoding.<p>
+ *
+ * -d<br>
+ * Turns debugging mode on.<p>
+ *
+ * -k<br>
+ * Use keyphrase frequency statistic.<p>
+ *
+ * -p<br>
+ * Disallow internal periods.<p>
+ *
+ * -x "length"<br>
+ * Sets maximum phrase length (default: 3).<p>
+ *
+ * -y "length"<br>
+ * Sets minimum phrase length (default: 1).<p>
+ *
+ * -o "number"<br>
+ * The minimum number of times a phrase needs to occur (default: 2). <p>
+ *
+ * -s "name of class implementing list of stop words"<br>
+ * Sets list of stop words to used (default: StopwordsEnglish).<p>
+ *
+ * -t "name of class implementing stemmer"<br>
+ * Sets stemmer to use (default: IteratedLovinsStemmer). <p>
+ *
+ * -n<br>
+ * Do not check for proper nouns. <p>
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class KEAModelBuilder implements OptionHandler {
+  
+  /** Name of directory */
+  String m_dirName = null;
+  
+  /** Name of model */
+  String m_modelName = null;
+
+  /** Encoding */
+  String m_encoding = "default";
+
+  /** Debugging mode? */
+  boolean m_debug = false;
+
+  /** Use keyphrase frequency attribute? */
+  boolean m_useKFrequency = false;
+
+  /** Disallow internal periods? */
+  boolean m_disallowIPeriods = false;
+
+  /** The maximum length of phrases */
+  private int m_MaxPhraseLength = 3;
+
+  /** The minimum length of phrases */
+  private int m_MinPhraseLength = 1;
+
+  /** The minimum number of occurences of a phrase */
+  private int m_MinNumOccur = 2;
+
+  /** The KEA filter object */
+  KEAFilter m_KEAFilter = null;
+
+  /** The stemmer to be used */
+  private Stemmer m_Stemmer = new IteratedLovinsStemmer();
+
+  /** The list of stop words to be used */
+  private Stopwords m_Stopwords = new StopwordsEnglish();
+
+  /** Determines whether check for proper nouns is performed */
+  private boolean m_CheckForProperNouns = true;
+
+  /**
+   * Get the M_CheckProperNouns value.
+   * @return the M_CheckProperNouns value.
+   */
+  public boolean getCheckForProperNouns() {
+    return m_CheckForProperNouns;
+  }
+
+  /**
+   * Set the M_CheckProperNouns value.
+   * @param newM_CheckProperNouns The new M_CheckProperNouns value.
+   */
+  public void setCheckForProperNouns(boolean newM_CheckProperNouns) {
+    this.m_CheckForProperNouns = newM_CheckProperNouns;
+  }
+
+  /**
+   * Get the M_Stopwords value.
+   * @return the M_Stopwords value.
+   */
+  public Stopwords getStopwords() {
+
+    return m_Stopwords;
+  }
+
+  /**
+   * Set the M_Stopwords value.
+   * @param newM_Stopwords The new M_Stopwords value.
+   */
+  public void setStopwords(Stopwords newM_Stopwords) {
+
+    this.m_Stopwords = newM_Stopwords;
+  }
+
+  
+  /**
+   * Get the Stemmer value.
+   * @return the Stemmer value.
+   */
+  public Stemmer getStemmer() {
+
+    return m_Stemmer;
+  }
+
+  /**
+   * Set the Stemmer value.
+   * @param newStemmer The new Stemmer value.
+   */
+  public void setStemmer(Stemmer newStemmer) {
+
+    this.m_Stemmer = newStemmer;
+  }
+  
+  /**
+   * Get the value of MinNumOccur.
+   *
+   * @return Value of MinNumOccur.
+   */
+  public int getMinNumOccur() {
+    
+    return m_MinNumOccur;
+  }
+  
+  /**
+   * Set the value of MinNumOccur.
+   *
+   * @param newMinNumOccur Value to assign to MinNumOccur.
+   */
+  public void setMinNumOccur(int newMinNumOccur) {
+    
+    m_MinNumOccur = newMinNumOccur;
+  }
+  
+  /**
+   * Get the value of MaxPhraseLength.
+   *
+   * @return Value of MaxPhraseLength.
+   */
+  public int getMaxPhraseLength() {
+    
+    return m_MaxPhraseLength;
+  }
+  
+  /**
+   * Set the value of MaxPhraseLength.
+   *
+   * @param newMaxPhraseLength Value to assign to MaxPhraseLength.
+   */
+  public void setMaxPhraseLength(int newMaxPhraseLength) {
+    
+    m_MaxPhraseLength = newMaxPhraseLength;
+  }
+  
+  /**
+   * Get the value of MinPhraseLength.
+   *
+   * @return Value of MinPhraseLength.
+   */
+  public int getMinPhraseLength() {
+    
+    return m_MinPhraseLength;
+  }
+  
+  /**
+   * Set the value of MinPhraseLength.
+   *
+   * @param newMinPhraseLength Value to assign to MinPhraseLength.
+   */
+  public void setMinPhraseLength(int newMinPhraseLength) {
+    
+    m_MinPhraseLength = newMinPhraseLength;
+  }
+  
+  /**
+   * Get the value of disallowIPeriods.
+   *
+   * @return Value of disallowIPeriods.
+   */
+  public boolean getDisallowIPeriods() {
+    
+    return m_disallowIPeriods;
+  }
+  
+  /**
+   * Set the value of disallowIPeriods.
+   *
+   * @param newdisallowIPeriods Value to assign to disallowIPeriods.
+   */
+  public void setDisallowIPeriods(boolean newdisallowIPeriods) {
+    
+    m_disallowIPeriods = newdisallowIPeriods;
+  }
+  
+  /**
+   * Get the value of useKFrequency.
+   *
+   * @return Value of useKFrequency.
+   */
+  public boolean getUseKFrequency() {
+    
+    return m_useKFrequency;
+  }
+  
+  /**
+   * Set the value of useKFrequency.
+   *
+   * @param newuseKFrequency Value to assign to useKFrequency.
+   */
+  public void setUseKFrequency(boolean newuseKFrequency) {
+    
+    m_useKFrequency = newuseKFrequency;
+  }
+  
+  /**
+   * Get the value of debug.
+   *
+   * @return Value of debug.
+   */
+  public boolean getDebug() {
+    
+    return m_debug;
+  }
+  
+  /**
+   * Set the value of debug.
+   *
+   * @param newdebug Value to assign to debug.
+   */
+  public void setDebug(boolean newdebug) {
+    
+    m_debug = newdebug;
+  }
+  
+  /**
+   * Get the value of encoding.
+   *
+   * @return Value of encoding.
+   */
+  public String getEncoding() {
+    
+    return m_encoding;
+  }
+  
+  /**
+   * Set the value of encoding.
+   *
+   * @param newencoding Value to assign to encoding.
+   */
+  public void setEncoding(String newencoding) {
+    
+    m_encoding = newencoding;
+  }
+  
+  /**
+   * Get the value of modelName.
+   *
+   * @return Value of modelName.
+   */
+  public String getModelName() {
+    
+    return m_modelName;
+  }
+  
+  /**
+   * Set the value of modelName.
+   *
+   * @param newmodelName Value to assign to modelName.
+   */
+  public void setModelName(String newmodelName) {
+    
+    m_modelName = newmodelName;
+  }
+  
+  /**
+   * Get the value of dirName.
+   *
+   * @return Value of dirName.
+   */
+  public String getDirName() {
+    
+    return m_dirName;
+  }
+  
+  /**
+   * Set the value of dirName.
+   *
+   * @param newdirName Value to assign to dirName.
+   */
+  public void setDirName(String newdirName) {
+    
+    m_dirName = newdirName;
+  }
+  
+  /**
+   * Parses a given list of options controlling the behaviour of this object.
+   * Valid options are:<p>
+   *
+   * -l "directory name" <br>
+   * Specifies name of directory.<p>
+   *
+   * -m "model name" <br>
+   * Specifies name of model.<p>
+   *
+   * -e "encoding" <br>
+   * Specifies encoding.<p>
+   *
+   * -d<br>
+   * Turns debugging mode on.<p>
+   *
+   * -k<br>
+   * Use keyphrase frequency statistic.<p>
+   *
+   * -p<br>
+   * Disallow internal periods. <p>
+   *
+   * -x "length"<br>
+   * Sets maximum phrase length (default: 3).<p>
+   *
+   * -y "length"<br>
+   * Sets minimum phrase length (default: 3).<p>
+   *
+   * -o "number"<br>
+   * The minimum number of times a phrase needs to occur (default: 2). <p>
+   *
+   * -s "name of class implementing list of stop words"<br>
+   * Sets list of stop words to used (default: StopwordsEnglish).<p>
+   *
+   * -t "name of class implementing stemmer"<br>
+   * Sets stemmer to use (default: IteratedLovinsStemmer). <p>
+   *
+   * -n<br>
+   * Do not check for proper nouns. <p>
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    String dirName = Utils.getOption('l', options);
+    if (dirName.length() > 0) {
+      setDirName(dirName);
+    } else {
+      setDirName(null);
+      throw new Exception("Name of directory required argument.");
+    }
+    String modelName = Utils.getOption('m', options);
+    if (modelName.length() > 0) {
+      setModelName(modelName);
+    } else {
+      setModelName(null);
+      throw new Exception("Name of model required argument.");
+    }
+    String encoding = Utils.getOption('e', options);
+    if (encoding.length() > 0) {
+      setEncoding(encoding);
+    } else {
+      setEncoding("default");
+    }
+    String maxPhraseLengthString = Utils.getOption('x', options);
+    if (maxPhraseLengthString.length() > 0) {
+      setMaxPhraseLength(Integer.parseInt(maxPhraseLengthString));
+    } else {
+      setMaxPhraseLength(3);
+    }
+    String minPhraseLengthString = Utils.getOption('y', options);
+    if (minPhraseLengthString.length() > 0) {
+      setMinPhraseLength(Integer.parseInt(minPhraseLengthString));
+    } else {
+      setMinPhraseLength(1);
+    }
+    String minNumOccurString = Utils.getOption('o', options);
+    if (minNumOccurString.length() > 0) {
+      setMinNumOccur(Integer.parseInt(minNumOccurString));
+    } else {
+      setMinNumOccur(2);
+    }
+    String stopwordsString = Utils.getOption('s', options);
+    if (stopwordsString.length() > 0) {
+      setStopwords((Stopwords)Class.forName(stopwordsString).newInstance());
+    }
+    String stemmerString = Utils.getOption('t', options);
+    if (stemmerString.length() > 0) {
+      setStemmer((Stemmer)Class.forName(stemmerString).newInstance());
+    }
+    setDebug(Utils.getFlag('d', options));
+    setUseKFrequency(Utils.getFlag('k', options));
+    setDisallowIPeriods(Utils.getFlag('p', options));
+    setCheckForProperNouns(!Utils.getFlag('n', options));
+    Utils.checkForRemainingOptions(options);
+   }
+
+  /**
+   * Gets the current option settings.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [20];
+    int current = 0;
+
+    options[current++] = "-l"; 
+    options[current++] = "" + (getDirName());
+    options[current++] = "-m"; 
+    options[current++] = "" + (getModelName());
+    options[current++] = "-e"; 
+    options[current++] = "" + (getEncoding());
+    if (getUseKFrequency()) {
+      options[current++] = "-k";
+    }
+    if (getDebug()) {
+      options[current++] = "-d";
+    }
+    if (getDisallowIPeriods()) {
+      options[current++] = "-p";
+    }
+    options[current++] = "-x"; 
+    options[current++] = "" + (getMaxPhraseLength());
+    options[current++] = "-y"; 
+    options[current++] = "" + (getMinPhraseLength());
+    options[current++] = "-o"; 
+    options[current++] = "" + (getMinNumOccur());
+    options[current++] = "-s"; 
+    options[current++] = "" + (getStopwords().getClass().getName());
+    options[current++] = "-t"; 
+    options[current++] = "" + (getStemmer().getClass().getName());
+    if (getCheckForProperNouns()) {
+      options[current++] = "-n";
+    }
+
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+  /**
+   * Returns an enumeration describing the available options.
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(12);
+
+    newVector.addElement(new Option(
+	      "\tSpecifies name of directory.",
+              "l", 1, "-l <directory name>"));
+    newVector.addElement(new Option(
+	      "\tSpecifies name of model.",
+              "m", 1, "-m <model name>"));
+    newVector.addElement(new Option(
+	      "\tSpecifies encoding.",
+              "e", 1, "-e <encoding>"));
+    newVector.addElement(new Option(
+	      "\tTurns debugging mode on.",
+              "d", 0, "-d"));
+    newVector.addElement(new Option(
+	      "\tUse keyphrase frequency statistic.",
+              "k", 0, "-k"));
+    newVector.addElement(new Option(
+	      "\tDisallow internal periods.",
+              "p", 0, "-p"));
+    newVector.addElement(new Option(
+	      "\tSets the maximum phrase length (default: 3).",
+              "x", 1, "-x <length>"));
+    newVector.addElement(new Option(
+	      "\tSets the minimum phrase length (default: 1).",
+              "y", 1, "-y <length>"));
+    newVector.addElement(new Option(
+	      "\tSet the minimum number of occurences (default: 2).",
+              "o", 1, "-o"));
+    newVector.addElement(new Option(
+	      "\tSets the list of stopwords to use (default: StopwordsEnglish).",
+              "s", 1, "-s <name of stopwords class>"));
+    newVector.addElement(new Option(
+	      "\tSet the stemmer to use (default: IteratedLovinsStemmer).",
+              "t", 1, "-t <name of stemmer class>"));
+    newVector.addElement(new Option(
+	      "\tDo not check for proper nouns.",
+              "n", 0, "-n"));
+ 
+    return newVector.elements();
+  }
+
+  /**
+   * Collects the stems of the file names.
+   */
+  public Hashtable collectStems() throws Exception {
+
+    Hashtable stems = new Hashtable();
+
+    try {
+      File dir = new File(m_dirName);
+      String[] files = dir.list();
+      for (int i = 0; i < files.length; i++) {
+	if (files[i].endsWith(".key") ||
+	    files[i].endsWith(".txt")) {
+	  String stem = files[i].substring(0, files[i].length() - 4);
+	  if (!stems.containsKey(stem)) {
+	    stems.put(stem, new Double(0));
+	  }
+	}
+      }
+    } catch (Exception e) {
+      throw new Exception("Problem opening directory " + m_dirName);
+    }
+    return stems;
+  }
+
+  /**
+   * Builds the model from the files
+   */
+  public void buildModel(Hashtable stems) throws Exception {
+    
+    // Check whether there is actually any data
+    if (stems.size() == 0) {
+      throw new Exception("Couldn't find any data!");
+    }
+
+    FastVector atts = new FastVector(2);
+    atts.addElement(new Attribute("doc", null));
+    atts.addElement(new Attribute("keyphrases", null));
+    Instances data = new Instances("keyphrase_training_data", atts, 0);
+
+    // Build model
+    m_KEAFilter = new KEAFilter();
+    m_KEAFilter.setDebug(m_debug);
+    m_KEAFilter.setDisallowInternalPeriods(getDisallowIPeriods());
+    m_KEAFilter.setKFused(getUseKFrequency());
+    m_KEAFilter.setMaxPhraseLength(getMaxPhraseLength());
+    m_KEAFilter.setMinPhraseLength(getMinPhraseLength());
+    m_KEAFilter.setMinNumOccur(getMinNumOccur());
+    m_KEAFilter.setInputFormat(data);
+    m_KEAFilter.setStemmer(getStemmer());
+    m_KEAFilter.setStopwords(getStopwords());
+    m_KEAFilter.setCheckForProperNouns(getCheckForProperNouns());
+    Enumeration elem = stems.keys();
+    while (elem.hasMoreElements()) {
+      String str = (String)elem.nextElement();
+      double[] newInst = new double[2];
+      try {
+	File txt = new File(m_dirName + "/" + str + ".txt");
+	InputStreamReader is;
+	if (!m_encoding.equals("default")) {
+	  is = new InputStreamReader(new FileInputStream(txt), m_encoding);
+	} else {
+	  is = new InputStreamReader(new FileInputStream(txt));
+	}
+	StringBuffer txtStr = new StringBuffer();
+	int c;
+	while ((c = is.read()) != -1) {
+	  txtStr.append((char)c);
+	}
+	newInst[0] = (double)data.attribute(0).addStringValue(txtStr.toString());
+      } catch (Exception e) {
+	if (m_debug) {
+	  System.err.println("Can't find document for stem " + str + ".");
+	}
+	newInst[0] = Instance.missingValue();
+      }
+      try {
+	File key = new File(m_dirName + "/" + str + ".key");
+	InputStreamReader is; 
+	if (!m_encoding.equals("default")) {
+	  is = new InputStreamReader(new FileInputStream(key), m_encoding);
+	} else {
+	  is = new InputStreamReader(new FileInputStream(key));
+	}
+	StringBuffer keyStr = new StringBuffer();
+	int c;
+	while ((c = is.read()) != -1) {
+	  keyStr.append((char)c);
+	}      
+	newInst[1] = (double)data.attribute(1).addStringValue(keyStr.toString());
+      } catch (Exception e) {
+	if (m_debug) {
+	  System.err.println("Can't find keyphrases for stem " + str + ".");
+	}
+	newInst[1] = Instance.missingValue();
+      }
+      data.add(new Instance(1.0, newInst));
+      m_KEAFilter.input(data.instance(0));
+      data = data.stringFreeStructure();
+    }
+    m_KEAFilter.batchFinished();
+    
+    // Get rid of instances in filter
+    Instance dummy;
+    while ((dummy = m_KEAFilter.output()) != null) {};
+  }
+  
+  /** 
+   * Saves the extraction model to the file.
+   */
+  public void saveModel() throws Exception {
+    
+    BufferedOutputStream bufferedOut = 
+      new BufferedOutputStream(new FileOutputStream(m_modelName));
+    ObjectOutputStream out = new ObjectOutputStream(bufferedOut);
+    out.writeObject(m_KEAFilter);
+    out.flush();
+    out.close();
+  }
+  
+  /**
+   * The main method.  
+   */
+  public static void main(String[] ops) {
+    
+    KEAModelBuilder kmb = new KEAModelBuilder();
+    try {
+      kmb.setOptions(ops);
+      System.err.print("Building model with options: ");
+      String[] optionSettings = kmb.getOptions();
+      for (int i = 0; i < optionSettings.length; i++) {
+	System.err.print(optionSettings[i] + " ");
+      }
+      System.err.println();
+      kmb.buildModel(kmb.collectStems());
+      kmb.saveModel();
+    } catch (Exception e) {
+      e.printStackTrace();
+      System.err.println(e.getMessage());
+      System.err.println("\nOptions:\n");
+      Enumeration enum = kmb.listOptions();
+      while (enum.hasMoreElements()) {
+	Option option = (Option) enum.nextElement();
+	System.err.println(option.synopsis());
+	System.err.println(option.description());
+      }
+    }
+  }
+}
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAPhraseFilter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAPhraseFilter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/KEAPhraseFilter.java	(revision 21112)
@@ -0,0 +1,487 @@
+/*
+ *    KEAPhraseFilter.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+
+import weka.core.*;
+import weka.filters.*;
+
+/**
+ * This filter splits the text in selected string
+ * attributes into phrases. The resulting
+ * string attributes contain these phrases
+ * separated by '\n' characters.
+ *
+ * Phrases are identified according to the
+ * following definitions:
+ * 
+ * A phrase is a sequence of words interrupted
+ * only by sequences of whitespace characters,
+ * where each sequence of whitespace characters
+ * contains at most one '\n'.
+ *
+ * A word is a sequence of letters or digits
+ * that contains at least one letter, with
+ * the following exceptions:
+ *
+ * a) '.', '@', '_', '&', '/', '-' are allowed
+ * if surrounded by letters or digits,
+ *
+ * b) '\'' is allowed if preceeded by a letter
+ * or digit,
+ * 
+ * c) '-', '/' are also allowed if succeeded by
+ * whitespace characters followed by another
+ * word. In that case the whitespace characters
+ * will be deleted.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class KEAPhraseFilter extends Filter implements OptionHandler {
+
+  /** Stores which columns to select as a funky range */
+  protected Range m_SelectCols = new Range();
+
+  /** Determines whether internal periods are allowed */
+  protected boolean m_DisallowInternalPeriods = false;
+
+  /**
+   * Returns a string describing this filter
+   *
+   * @return a description of the filter suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String globalInfo() {
+    return "This filter splits the text contained " +
+      "by the selected string attributes into phrases.";
+  }
+
+  /**
+   * Returns an enumeration describing the available options
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(3);
+
+    newVector.addElement(new Option(
+              "\tSpecify list of attributes to process. First and last are valid\n"
+	      +"\tindexes. (default none)",
+              "R", 1, "-R <index1,index2-index4,...>"));
+    newVector.addElement(new Option(
+	      "\tInvert matching sense",
+              "V", 0, "-V"));
+    newVector.addElement(new Option(
+	      "\tDisallow internal periods",
+              "P", 0, "-P"));
+
+    return newVector.elements();
+  }
+
+  /**
+   * Parses a given list of options controlling the behaviour of this object.
+   * Valid options are:<p>
+   *
+   * -R index1,index2-index4,...<br>
+   * Specify list of attributes to process. First and last are valid indexes.
+   * (default none)<p>
+   *
+   * -V<br>
+   * Invert matching sense <p>
+   *
+   * -P<br>
+   * Disallow internal periods <p>
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    String list = Utils.getOption('R', options);
+    if (list.length() != 0) {
+      setAttributeIndices(list);
+    }
+    setInvertSelection(Utils.getFlag('V', options));
+
+    setDisallowInternalPeriods(Utils.getFlag('P', options));
+    
+    if (getInputFormat() != null) {
+      setInputFormat(getInputFormat());
+    }
+  }
+
+  /**
+   * Gets the current settings of the filter.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [4];
+    int current = 0;
+
+    if (getInvertSelection()) {
+      options[current++] = "-V";
+    }
+    if (getDisallowInternalPeriods()) {
+      options[current++] = "-P";
+    }
+    if (!getAttributeIndices().equals("")) {
+      options[current++] = "-R"; options[current++] = getAttributeIndices();
+    }
+
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+  /**
+   * Sets the format of the input instances.
+   *
+   * @param instanceInfo an Instances object containing the input
+   * instance structure (any instances contained in the object are
+   * ignored - only the structure is required).
+   * @return true if the outputFormat may be collected immediately 
+   */
+  public boolean setInputFormat(Instances instanceInfo) throws Exception {
+
+    super.setInputFormat(instanceInfo);
+    setOutputFormat(instanceInfo);
+    m_SelectCols.setUpper(instanceInfo.numAttributes() - 1);
+
+    return true;
+  }
+
+  /**
+   * Input an instance for filtering. Ordinarily the instance is processed
+   * and made available for output immediately. Some filters require all
+   * instances be read before producing output.
+   *
+   * @param instance the input instance
+   * @return true if the filtered instance may now be
+   * collected with output().
+   * @exception Exception if the input instance was not of the correct 
+   * format or if there was a problem with the filtering.
+   */
+  public boolean input(Instance instance) throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new Exception("No input instance format defined");
+    }
+    if (m_NewBatch) {
+      resetQueue();
+      m_NewBatch = false;
+    }
+    convertInstance(instance);
+    return true;
+  }
+
+  /**
+   * Signify that this batch of input to the filter is finished. If
+   * the filter requires all instances prior to filtering, output()
+   * may now be called to retrieve the filtered instances. Any
+   * subsequent instances filtered should be filtered based on setting
+   * obtained from the first batch (unless the inputFormat has been
+   * re-assigned or new options have been set). This default
+   * implementation assumes all instance processing occurs during
+   * inputFormat() and input().
+   *
+   * @return true if there are instances pending output
+   * @exception NullPointerException if no input structure has been defined,
+   * @exception Exception if there was a problem finishing the batch.
+   */
+  public boolean batchFinished() throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new NullPointerException("No input instance format defined");
+    }
+    m_NewBatch = true;
+    return (numPendingOutput() != 0);
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain arguments to the filter: use -h for help
+   */
+  public static void main(String [] argv) {
+    
+    try {
+      if (Utils.getFlag('b', argv)) {
+	Filter.batchFilterFile(new KEAPhraseFilter(), argv);
+      } else {
+	Filter.filterFile(new KEAPhraseFilter(), argv);
+      }
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+ 
+  /** 
+   * Converts an instance by removing all non-alphanumeric characters
+   * from its string attribute values.
+   */
+  private void convertInstance(Instance instance) throws Exception {
+  
+    double[] instVals = new double[instance.numAttributes()];
+
+    for (int i = 0; i < instance.numAttributes(); i++) {
+      if (!instance.attribute(i).isString() || 
+	  instance.isMissing(i)) {
+	instVals[i] = instance.value(i);
+      } else {
+	if (!m_SelectCols.isInRange(i)) {
+	  int index = getOutputFormat().attribute(i).
+	    addStringValue(instance.stringValue(i));
+	  instVals[i] = (double)index;
+	  continue;
+	}
+	String str = instance.stringValue(i);
+	StringBuffer resultStr = new StringBuffer();
+	int j = 0;
+	boolean phraseStart = true;
+	boolean seenNewLine = false;
+	boolean haveSeenHyphen = false;
+	boolean haveSeenSlash = false;
+	while (j < str.length()) {
+	  boolean isWord = false;
+	  boolean potNumber = false;
+	  int startj = j;
+	  while (j < str.length()) {
+	    char ch = str.charAt(j);
+	    if (Character.isLetterOrDigit(ch)) {
+	      potNumber = true;
+	      if (Character.isLetter(ch)) {
+		isWord = true;
+	      }
+	      j++;
+	    } else if ((!m_DisallowInternalPeriods && (ch == '.')) ||
+		       (ch == '@') ||
+		       (ch == '_') ||
+		       (ch == '&') ||
+		       (ch == '/') ||
+		       (ch == '-')) {
+	      if ((j > 0) && (j  + 1 < str.length()) &&
+		  Character.isLetterOrDigit(str.charAt(j - 1)) &&
+		  Character.isLetterOrDigit(str.charAt(j + 1))) {
+		j++;
+	      } else {
+		break;
+	      }
+	    } else if (ch == '\'') {
+	      if ((j > 0) &&
+		  Character.isLetterOrDigit(str.charAt(j - 1))) {
+		j++;
+	      } else {
+		break;
+	      }
+	    } else {
+	      break;
+	    }
+	  }
+	  if (isWord == true) {
+	    if (!phraseStart) {
+	      if (haveSeenHyphen) {
+		resultStr.append('-');
+	      } else if (haveSeenSlash) {
+		resultStr.append('/');
+	      } else {
+		resultStr.append(' ');
+	      }
+	    }
+	    resultStr.append(str.substring(startj, j));
+	    if (j == str.length()) {
+	      break;
+	    }
+	    phraseStart = false;
+	    seenNewLine = false;
+	    haveSeenHyphen = false;
+	    haveSeenSlash = false;
+	    if (Character.isWhitespace(str.charAt(j))) {
+	      if (str.charAt(j) == '\n') {
+		seenNewLine = true;
+	      } 
+	    } else if (str.charAt(j) == '-') {
+	      haveSeenHyphen = true;
+	    } else if (str.charAt(j) == '/') {
+	      haveSeenSlash = true;
+	    } else {
+	      phraseStart = true;
+	      resultStr.append('\n');
+	    }
+	    j++;
+	  } else if (j == str.length()) {
+	    break;
+	  } else if (str.charAt(j) == '\n') {
+	    if (seenNewLine) {
+	      if (phraseStart == false) {
+		resultStr.append('\n');
+		phraseStart = true;
+	      }
+	    } else if (potNumber) {
+	      if (phraseStart == false) {
+		phraseStart = true;
+		resultStr.append('\n');
+	      }
+	    }
+	    seenNewLine = true;
+	    j++;
+	  } else if (Character.isWhitespace(str.charAt(j))) {
+	    if (potNumber) {
+	      if (phraseStart == false) {
+		phraseStart = true;
+		resultStr.append('\n');
+	      }
+	    }
+	    j++;
+	  } else {
+	    if (phraseStart == false) {
+	      resultStr.append('\n');
+	      phraseStart = true;
+	    }
+	    j++;
+	  }
+	}
+	int index = getOutputFormat().attribute(i).
+	  addStringValue(resultStr.toString());
+	instVals[i] = (double)index;
+      }
+    }
+    Instance inst = new Instance(instance.weight(), instVals);
+    inst.setDataset(getOutputFormat());
+    push(inst);
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String invertSelectionTipText() {
+
+    return "If set to false, the specified attributes will be processed;"
+      + " If set to true, specified attributes won't be processed.";
+  }
+
+  /**
+   * Get whether the supplied columns are to be processed
+   *
+   * @return true if the supplied columns won't be processed
+   */
+  public boolean getInvertSelection() {
+
+    return m_SelectCols.getInvert();
+  }
+
+  /**
+   * Set whether selected columns should be processed. If true the 
+   * selected columns won't be processed.
+   *
+   * @param invert the new invert setting
+   */
+  public void setInvertSelection(boolean invert) {
+
+    m_SelectCols.setInvert(invert);
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String disallowInternalPeriodsTipText() {
+
+    return "If set to false, internal periods are allowed.";
+  }
+
+  /**
+   * Get whether the supplied columns are to be processed
+   *
+   * @return true if the supplied columns won't be processed
+   */
+  public boolean getDisallowInternalPeriods() {
+
+    return m_DisallowInternalPeriods;
+  }
+
+  /**
+   * Set whether selected columns should be processed. If true the 
+   * selected columns won't be processed.
+   *
+   * @param invert the new invert setting
+   */
+  public void setDisallowInternalPeriods(boolean disallow) {
+
+    m_DisallowInternalPeriods = disallow;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String attributeIndicesTipText() {
+
+    return "Specify range of attributes to act on."
+      + " This is a comma separated list of attribute indices, with"
+      + " \"first\" and \"last\" valid values. Specify an inclusive"
+      + " range with \"-\". E.g: \"first-3,5,6-10,last\".";
+  }
+
+  /**
+   * Get the current range selection.
+   *
+   * @return a string containing a comma separated list of ranges
+   */
+  public String getAttributeIndices() {
+
+    return m_SelectCols.getRanges();
+  }
+
+  /**
+   * Set which attributes are to be processed
+   *
+   * @param rangeList a string representing the list of attributes.  Since
+   * the string will typically come from a user, attributes are indexed from
+   * 1. <br>
+   * eg: first-3,5,6-last
+   */
+  public void setAttributeIndices(String rangeList) {
+
+    m_SelectCols.setRanges(rangeList);
+  }
+
+  /**
+   * Set which attributes are to be processed
+   *
+   * @param attributes an array containing indexes of attributes to select.
+   * Since the array will typically come from a program, attributes are indexed
+   * from 0.
+   */
+  public void setAttributeIndicesArray(int [] attributes) {
+    
+    setAttributeIndices(Range.indicesToRangeList(attributes));
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/LovinsStemmer.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/LovinsStemmer.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/LovinsStemmer.java	(revision 21112)
@@ -0,0 +1,890 @@
+/*
+ *    LovinsStemmer.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+import java.io.*;
+
+/** 
+ * Implements the Lovins stemmer.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class LovinsStemmer extends Stemmer implements Serializable {
+
+  /** C version compatibility mode (emulates bugs
+      in original C implementation) */
+  private static boolean m_CompMode = true;
+
+  /** The hash tables containing the list of endings. */
+  private static HashMap m_l11 = null;
+  private static HashMap m_l10 = null;
+  private static HashMap m_l9 = null;
+  private static HashMap m_l8 = null;
+  private static HashMap m_l7 = null;
+  private static HashMap m_l6 = null;
+  private static HashMap m_l5 = null;
+  private static HashMap m_l4 = null;
+  private static HashMap m_l3 = null;
+  private static HashMap m_l2 = null;
+  private static HashMap m_l1 = null;
+
+  static {
+
+    m_l11 = new HashMap();
+    m_l11.put("alistically", "B");
+    m_l11.put("arizability", "A");
+    m_l11.put("izationally", "B");
+    m_l10 = new HashMap();
+    m_l10.put("antialness", "A");
+    m_l10.put("arisations", "A");
+    m_l10.put("arizations", "A");
+    m_l10.put("entialness", "A");
+    m_l9 = new HashMap();
+    m_l9.put("allically", "C");
+    m_l9.put("antaneous", "A");
+    m_l9.put("antiality", "A");
+    m_l9.put("arisation", "A");
+    m_l9.put("arization", "A");
+    m_l9.put("ationally", "B");
+    m_l9.put("ativeness", "A");
+    m_l9.put("eableness", "E");
+    m_l9.put("entations", "A");
+    m_l9.put("entiality", "A");
+    m_l9.put("entialize", "A");
+    m_l9.put("entiation", "A");
+    m_l9.put("ionalness", "A");
+    m_l9.put("istically", "A");
+    m_l9.put("itousness", "A");
+    m_l9.put("izability", "A");
+    m_l9.put("izational", "A");
+    m_l8 = new HashMap();
+    m_l8.put("ableness", "A");
+    m_l8.put("arizable", "A");
+    m_l8.put("entation", "A");
+    m_l8.put("entially", "A");
+    m_l8.put("eousness", "A");
+    m_l8.put("ibleness", "A");
+    m_l8.put("icalness", "A");
+    m_l8.put("ionalism", "A");
+    m_l8.put("ionality", "A");
+    m_l8.put("ionalize", "A");
+    m_l8.put("iousness", "A");
+    m_l8.put("izations", "A");
+    m_l8.put("lessness", "A");
+    m_l7 = new HashMap();
+    m_l7.put("ability", "A");
+    m_l7.put("aically", "A");
+    m_l7.put("alistic", "B");
+    m_l7.put("alities", "A");
+    m_l7.put("ariness", "E");
+    m_l7.put("aristic", "A");
+    m_l7.put("arizing", "A");
+    m_l7.put("ateness", "A");
+    m_l7.put("atingly", "A");
+    m_l7.put("ational", "B");
+    m_l7.put("atively", "A");
+    m_l7.put("ativism", "A");
+    m_l7.put("elihood", "E");
+    m_l7.put("encible", "A");
+    m_l7.put("entally", "A");
+    m_l7.put("entials", "A");
+    m_l7.put("entiate", "A");
+    m_l7.put("entness", "A");
+    m_l7.put("fulness", "A");
+    m_l7.put("ibility", "A");
+    m_l7.put("icalism", "A");
+    m_l7.put("icalist", "A");
+    m_l7.put("icality", "A");
+    m_l7.put("icalize", "A");
+    m_l7.put("ication", "G");
+    m_l7.put("icianry", "A");
+    m_l7.put("ination", "A");
+    m_l7.put("ingness", "A");
+    m_l7.put("ionally", "A");
+    m_l7.put("isation", "A");
+    m_l7.put("ishness", "A");
+    m_l7.put("istical", "A");
+    m_l7.put("iteness", "A");
+    m_l7.put("iveness", "A");
+    m_l7.put("ivistic", "A");
+    m_l7.put("ivities", "A");
+    m_l7.put("ization", "F");
+    m_l7.put("izement", "A");
+    m_l7.put("oidally", "A");
+    m_l7.put("ousness", "A");
+    m_l6 = new HashMap();
+    m_l6.put("aceous", "A");
+    m_l6.put("acious", "B");
+    m_l6.put("action", "G");
+    m_l6.put("alness", "A");
+    m_l6.put("ancial", "A");
+    m_l6.put("ancies", "A");
+    m_l6.put("ancing", "B");
+    m_l6.put("ariser", "A");
+    m_l6.put("arized", "A");
+    m_l6.put("arizer", "A");
+    m_l6.put("atable", "A");
+    m_l6.put("ations", "B");
+    m_l6.put("atives", "A");
+    m_l6.put("eature", "Z");
+    m_l6.put("efully", "A");
+    m_l6.put("encies", "A");
+    m_l6.put("encing", "A");
+    m_l6.put("ential", "A");
+    m_l6.put("enting", "C");
+    m_l6.put("entist", "A");
+    m_l6.put("eously", "A");
+    m_l6.put("ialist", "A");
+    m_l6.put("iality", "A");
+    m_l6.put("ialize", "A");
+    m_l6.put("ically", "A");
+    m_l6.put("icance", "A");
+    m_l6.put("icians", "A");
+    m_l6.put("icists", "A");
+    m_l6.put("ifully", "A");
+    m_l6.put("ionals", "A");
+    m_l6.put("ionate", "D");
+    m_l6.put("ioning", "A");
+    m_l6.put("ionist", "A");
+    m_l6.put("iously", "A");
+    m_l6.put("istics", "A");
+    m_l6.put("izable", "E");
+    m_l6.put("lessly", "A");
+    m_l6.put("nesses", "A");
+    m_l6.put("oidism", "A");
+    m_l5 = new HashMap();
+    m_l5.put("acies", "A");
+    m_l5.put("acity", "A");
+    m_l5.put("aging", "B");
+    m_l5.put("aical", "A");
+    if (!m_CompMode) {
+      m_l5.put("alist", "A");
+    }
+    m_l5.put("alism", "B");
+    m_l5.put("ality", "A");
+    m_l5.put("alize", "A");
+    m_l5.put("allic", "b");
+    m_l5.put("anced", "B");
+    m_l5.put("ances", "B");
+    m_l5.put("antic", "C");
+    m_l5.put("arial", "A");
+    m_l5.put("aries", "A");
+    m_l5.put("arily", "A");
+    m_l5.put("arity", "B");
+    m_l5.put("arize", "A");
+    m_l5.put("aroid", "A");
+    m_l5.put("ately", "A");
+    m_l5.put("ating", "I");
+    m_l5.put("ation", "B");
+    m_l5.put("ative", "A");
+    m_l5.put("ators", "A");
+    m_l5.put("atory", "A");
+    m_l5.put("ature", "E");
+    m_l5.put("early", "Y");
+    m_l5.put("ehood", "A");
+    m_l5.put("eless", "A");
+    if (!m_CompMode) {
+      m_l5.put("elily", "A");
+    } else {
+      m_l5.put("elity", "A");
+    }
+    m_l5.put("ement", "A");
+    m_l5.put("enced", "A");
+    m_l5.put("ences", "A");
+    m_l5.put("eness", "E");
+    m_l5.put("ening", "E");
+    m_l5.put("ental", "A");
+    m_l5.put("ented", "C");
+    m_l5.put("ently", "A");
+    m_l5.put("fully", "A");
+    m_l5.put("ially", "A");
+    m_l5.put("icant", "A");
+    m_l5.put("ician", "A");
+    m_l5.put("icide", "A");
+    m_l5.put("icism", "A");
+    m_l5.put("icist", "A");
+    m_l5.put("icity", "A");
+    m_l5.put("idine", "I");
+    m_l5.put("iedly", "A");
+    m_l5.put("ihood", "A");
+    m_l5.put("inate", "A");
+    m_l5.put("iness", "A");
+    m_l5.put("ingly", "B");
+    m_l5.put("inism", "J");
+    m_l5.put("inity", "c");
+    m_l5.put("ional", "A");
+    m_l5.put("ioned", "A");
+    m_l5.put("ished", "A");
+    m_l5.put("istic", "A");
+    m_l5.put("ities", "A");
+    m_l5.put("itous", "A");
+    m_l5.put("ively", "A");
+    m_l5.put("ivity", "A");
+    m_l5.put("izers", "F");
+    m_l5.put("izing", "F");
+    m_l5.put("oidal", "A");
+    m_l5.put("oides", "A");
+    m_l5.put("otide", "A");
+    m_l5.put("ously", "A");
+    m_l4 = new HashMap();
+    m_l4.put("able", "A");
+    m_l4.put("ably", "A");
+    m_l4.put("ages", "B");
+    m_l4.put("ally", "B");
+    m_l4.put("ance", "B");
+    m_l4.put("ancy", "B");
+    m_l4.put("ants", "B");
+    m_l4.put("aric", "A");
+    m_l4.put("arly", "K");
+    m_l4.put("ated", "I");
+    m_l4.put("ates", "A");
+    m_l4.put("atic", "B");
+    m_l4.put("ator", "A");
+    m_l4.put("ealy", "Y");
+    m_l4.put("edly", "E");
+    m_l4.put("eful", "A");
+    m_l4.put("eity", "A");
+    m_l4.put("ence", "A");
+    m_l4.put("ency", "A");
+    m_l4.put("ened", "E");
+    m_l4.put("enly", "E");
+    m_l4.put("eous", "A");
+    m_l4.put("hood", "A");
+    m_l4.put("ials", "A");
+    m_l4.put("ians", "A");
+    m_l4.put("ible", "A");
+    m_l4.put("ibly", "A");
+    m_l4.put("ical", "A");
+    m_l4.put("ides", "L");
+    m_l4.put("iers", "A");
+    m_l4.put("iful", "A");
+    m_l4.put("ines", "M");
+    m_l4.put("ings", "N");
+    m_l4.put("ions", "B");
+    m_l4.put("ious", "A");
+    m_l4.put("isms", "B");
+    m_l4.put("ists", "A");
+    m_l4.put("itic", "H");
+    m_l4.put("ized", "F");
+    m_l4.put("izer", "F");
+    m_l4.put("less", "A");
+    m_l4.put("lily", "A");
+    m_l4.put("ness", "A");
+    m_l4.put("ogen", "A");
+    m_l4.put("ward", "A");
+    m_l4.put("wise", "A");
+    m_l4.put("ying", "B");
+    m_l4.put("yish", "A");
+    m_l3 = new HashMap();
+    m_l3.put("acy", "A");
+    m_l3.put("age", "B");
+    m_l3.put("aic", "A");
+    m_l3.put("als", "b");
+    m_l3.put("ant", "B");
+    m_l3.put("ars", "O");
+    m_l3.put("ary", "F");
+    m_l3.put("ata", "A");
+    m_l3.put("ate", "A");
+    m_l3.put("eal", "Y");
+    m_l3.put("ear", "Y");
+    m_l3.put("ely", "E");
+    m_l3.put("ene", "E");
+    m_l3.put("ent", "C");
+    m_l3.put("ery", "E");
+    m_l3.put("ese", "A");
+    m_l3.put("ful", "A");
+    m_l3.put("ial", "A");
+    m_l3.put("ian", "A");
+    m_l3.put("ics", "A");
+    m_l3.put("ide", "L");
+    m_l3.put("ied", "A");
+    m_l3.put("ier", "A");
+    m_l3.put("ies", "P");
+    m_l3.put("ily", "A");
+    m_l3.put("ine", "M");
+    m_l3.put("ing", "N");
+    m_l3.put("ion", "Q");
+    m_l3.put("ish", "C");
+    m_l3.put("ism", "B");
+    m_l3.put("ist", "A");
+    m_l3.put("ite", "a");
+    m_l3.put("ity", "A");
+    m_l3.put("ium", "A");
+    m_l3.put("ive", "A");
+    m_l3.put("ize", "F");
+    m_l3.put("oid", "A");
+    m_l3.put("one", "R");
+    m_l3.put("ous", "A");
+    m_l2 = new HashMap();
+    m_l2.put("ae", "A"); 
+    m_l2.put("al", "b");
+    m_l2.put("ar", "X");
+    m_l2.put("as", "B");
+    m_l2.put("ed", "E");
+    m_l2.put("en", "F");
+    m_l2.put("es", "E");
+    m_l2.put("ia", "A");
+    m_l2.put("ic", "A");
+    m_l2.put("is", "A");
+    m_l2.put("ly", "B");
+    m_l2.put("on", "S");
+    m_l2.put("or", "T");
+    m_l2.put("um", "U");
+    m_l2.put("us", "V");
+    m_l2.put("yl", "R");
+    m_l2.put("s\'", "A");
+    m_l2.put("\'s", "A");
+    m_l1 = new HashMap();
+    m_l1.put("a", "A");
+    m_l1.put("e", "A");
+    m_l1.put("i", "A");
+    m_l1.put("o", "A");
+    m_l1.put("s", "W");
+    m_l1.put("y", "B");	
+  }
+
+  /**
+   * Finds and removes ending from given word.
+   */
+  private String removeEnding(String word) {
+
+    int length = word.length();
+    int el = 11;
+
+    while (el > 0) {
+      if (length - el > 1) {
+	String ending = word.substring(length - el);
+	String conditionCode = null;
+	switch (el) {
+	case 11: conditionCode = (String)m_l11.get(ending);
+	  break;
+	case 10: conditionCode = (String)m_l10.get(ending);
+	  break; 
+	case 9: conditionCode = (String)m_l9.get(ending);
+	  break;
+	case 8: conditionCode = (String)m_l8.get(ending);
+	  break;   
+	case 7: conditionCode = (String)m_l7.get(ending);
+	  break;   
+	case 6: conditionCode = (String)m_l6.get(ending);
+	  break;   
+	case 5: conditionCode = (String)m_l5.get(ending);
+	  break;   
+	case 4: conditionCode = (String)m_l4.get(ending);
+	  break;   
+	case 3: conditionCode = (String)m_l3.get(ending);
+	  break;   
+	case 2: conditionCode = (String)m_l2.get(ending);
+	  break;   
+	case 1: conditionCode = (String)m_l1.get(ending);
+	  break;   
+	default:
+	}
+	if (conditionCode != null) {
+	  switch (conditionCode.charAt(0)) {
+	  case 'A':
+	    return word.substring(0, length - el);
+	  case 'B':
+	    if (length - el > 2) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'C':
+	    if (length - el > 3) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'D':
+	    if (length - el > 4) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'E':
+	    if (word.charAt(length - el - 1) != 'e') {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'F':
+	    if ((length - el > 2) &&
+		(word.charAt(length - el - 1) != 'e')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'G':
+	    if ((length - el > 2) &&
+		(word.charAt(length - el - 1) == 'f')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'H':
+	    if ((word.charAt(length - el - 1) == 't') ||
+		((word.charAt(length - el - 1) == 'l') &&
+		 (word.charAt(length - el - 2) == 'l'))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'I':
+	    if ((word.charAt(length - el - 1) != 'o') &&
+		(word.charAt(length - el - 1) != 'e')) { 
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'J':
+	    if ((word.charAt(length - el - 1) != 'a') &&
+		(word.charAt(length - el - 1) != 'e')) { 
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'K':
+	    if ((length - el > 2) &&
+		((word.charAt(length - el - 1) == 'l') ||
+		 (word.charAt(length - el - 1) == 'i') ||
+		 ((word.charAt(length - el - 1) == 'e') &&
+		  (word.charAt(length - el - 3) == 'u')))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'L':
+	    if ((word.charAt(length - el - 1) != 'u') &&
+		(word.charAt(length - el - 1) != 'x') &&
+		((word.charAt(length - el - 1) != 's') ||
+		 (word.charAt(length - el - 2) == 'o'))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'M':
+	    if ((word.charAt(length - el - 1) != 'a') &&
+		(word.charAt(length - el - 1) != 'c') &&
+		(word.charAt(length - el - 1) != 'e') &&
+		(word.charAt(length - el - 1) != 'm')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'N':
+	    if ((length - el > 3) || 
+		((length - el == 3) &&
+		 ((word.charAt(length - el - 3) != 's')))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'O':
+	    if ((word.charAt(length - el - 1) == 'l') ||
+		(word.charAt(length - el - 1) == 'i')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'P':
+	    if (word.charAt(length - el - 1) != 'c') {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'Q':
+	    if ((length - el > 2) &&
+		(word.charAt(length - el - 1) != 'l') &&
+		(word.charAt(length - el - 1) != 'n')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'R':
+	    if ((word.charAt(length - el - 1) == 'n') ||
+		(word.charAt(length - el - 1) == 'r')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'S':
+	    if (((word.charAt(length - el - 1) == 'r') &&
+		 (word.charAt(length - el - 2) == 'd')) ||
+		((word.charAt(length - el - 1) == 't') &&
+		 (word.charAt(length - el - 2) != 't'))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'T':
+	    if ((word.charAt(length - el - 1) == 's') ||
+		((word.charAt(length - el - 1) == 't') &&
+		 (word.charAt(length - el - 2) != 'o'))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'U':
+	    if ((word.charAt(length - el - 1) == 'l') ||
+		(word.charAt(length - el - 1) == 'm') ||
+		(word.charAt(length - el - 1) == 'n') ||
+		(word.charAt(length - el - 1) == 'r')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'V':
+	    if (word.charAt(length - el - 1) == 'c') {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'W':
+	    if ((word.charAt(length - el - 1) != 's') &&
+		(word.charAt(length - el - 1) != 'u')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'X':
+	    if ((word.charAt(length - el - 1) == 'l') ||
+		(word.charAt(length - el - 1) == 'i') ||
+		((length - el > 2) &&
+		 (word.charAt(length - el - 1) == 'e') &&
+		 (word.charAt(length - el - 3) == 'u'))) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'Y':
+	    if ((word.charAt(length - el - 1) == 'n') &&
+		(word.charAt(length - el - 2) == 'i')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'Z':
+	    if (word.charAt(length - el - 1) != 'f') {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'a':
+	    if ((word.charAt(length - el - 1) == 'd') ||
+		(word.charAt(length - el - 1) == 'f') ||
+		(((word.charAt(length - el - 1) == 'h') &&
+		  (word.charAt(length - el - 2) == 'p'))) ||
+		(((word.charAt(length - el - 1) == 'h') &&
+		  (word.charAt(length - el - 2) == 't'))) ||
+		(word.charAt(length - el - 1) == 'l') ||
+		(((word.charAt(length - el - 1) == 'r') &&
+		  (word.charAt(length - el - 2) == 'e'))) ||
+		(((word.charAt(length - el - 1) == 'r') &&
+		  (word.charAt(length - el - 2) == 'o'))) ||
+		(((word.charAt(length - el - 1) == 's') &&
+		  (word.charAt(length - el - 2) == 'e'))) ||
+		(word.charAt(length - el - 1) == 't')) {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  case 'b':
+	    if (m_CompMode) {
+	      if (((length - el == 3 ) &&
+		   (!((word.charAt(length - el - 1) == 't') &&
+		      (word.charAt(length - el - 2) == 'e') &&
+		      (word.charAt(length - el - 3) == 'm')))) ||
+		  ((length - el > 3) &&
+		   (!((word.charAt(length - el - 1) == 't') &&
+		      (word.charAt(length - el - 2) == 's') &&
+		      (word.charAt(length - el - 3) == 'y') &&
+		      (word.charAt(length - el - 4) == 'r'))))) {
+		return word.substring(0, length - el);
+	      }
+	    } else {
+	      if ((length - el > 2) &&
+		  (!((word.charAt(length - el - 1) == 't') &&
+		     (word.charAt(length - el - 2) == 'e') &&
+		     (word.charAt(length - el - 3) == 'm'))) &&
+		  ((length - el < 4) ||
+		   (!((word.charAt(length - el - 1) == 't') &&
+		      (word.charAt(length - el - 2) == 's') &&
+		      (word.charAt(length - el - 3) == 'y') &&
+		      (word.charAt(length - el - 4) == 'r'))))) {
+		return word.substring(0, length - el);
+	      }
+	    } 
+	    break;
+	  case 'c':
+	    if (word.charAt(length - el - 1) == 'l') {
+	      return word.substring(0, length - el);
+	    }
+	    break;
+	  default:
+	    throw new IllegalArgumentException("Fatal error.");
+	  }
+	}
+      }
+      el--;
+    }
+    return word;
+  }
+
+  /**
+   * Recodes ending of given word.
+   */
+  private String recodeEnding(String word) {
+
+    int lastPos = word.length() - 1;
+
+    // Rule 1
+    if (word.endsWith("bb") ||
+	word.endsWith("dd") ||
+	word.endsWith("gg") ||
+	word.endsWith("ll") ||
+	word.endsWith("mm") ||
+	word.endsWith("nn") ||
+	word.endsWith("pp") ||
+	word.endsWith("rr") ||
+	word.endsWith("ss") ||
+	word.endsWith("tt")) {
+      word = word.substring(0, lastPos);
+      lastPos--;
+    }
+
+    // Rule 2
+    if (word.endsWith("iev")) {
+      word = word.substring(0, lastPos - 2).concat("ief");
+    }
+
+    // Rule 3
+    if (word.endsWith("uct")) {
+      word = word.substring(0, lastPos - 2).concat("uc");
+      lastPos--;
+    }
+
+    // Rule 4
+    if (word.endsWith("umpt")) {
+      word = word.substring(0, lastPos - 3).concat("um");
+      lastPos -= 2;
+    }
+
+    // Rule 5
+    if (word.endsWith("rpt")) {
+      word = word.substring(0, lastPos - 2).concat("rb");
+      lastPos--;
+    }
+
+    // Rule 6
+    if (word.endsWith("urs")) {
+      word = word.substring(0, lastPos - 2).concat("ur");
+      lastPos--;
+    }
+
+    // Rule 7
+    if (word.endsWith("istr")) {
+      word = word.substring(0, lastPos - 3).concat("ister");
+      lastPos++;
+    }
+ 
+    // Rule 7a
+    if (word.endsWith("metr")) {
+      word = word.substring(0, lastPos - 3).concat("meter");
+      lastPos++;
+    }
+
+    // Rule 8
+    if (word.endsWith("olv")) {
+      word = word.substring(0, lastPos - 2).concat("olut");
+      lastPos++;
+    }
+ 
+    // Rule 9
+    if (word.endsWith("ul")) {
+      if ((lastPos - 2 < 0) ||
+	  ((word.charAt(lastPos - 2) != 'a') &&
+	   (word.charAt(lastPos - 2) != 'i') &&
+	   (word.charAt(lastPos - 2) != 'o'))) {
+	word = word.substring(0, lastPos - 1).concat("l");
+	lastPos--;
+      }
+    }
+
+    // Rule 10
+    if (word.endsWith("bex")) {
+      word = word.substring(0, lastPos - 2).concat("bic");
+    }
+
+    // Rule 11
+    if (word.endsWith("dex")) {
+      word = word.substring(0, lastPos - 2).concat("dic");
+    }
+
+    // Rule 12
+    if (word.endsWith("pex")) {
+      word = word.substring(0, lastPos - 2).concat("pic");
+    }
+
+    // Rule 13
+    if (word.endsWith("tex")) {
+      word = word.substring(0, lastPos - 2).concat("tic");
+    }
+
+    // Rule 14
+    if (word.endsWith("ax")) {
+      word = word.substring(0, lastPos - 1).concat("ac");
+    }
+
+    // Rule 15
+    if (word.endsWith("ex")) {
+      word = word.substring(0, lastPos - 1).concat("ec");
+    }
+
+    // Rule 16
+    if (word.endsWith("ix")) {
+      word = word.substring(0, lastPos - 1).concat("ic");
+    }
+
+    // Rule 17
+    if (word.endsWith("lux")) {
+      word = word.substring(0, lastPos - 2).concat("luc");
+    }
+
+    // Rule 18
+    if (word.endsWith("uad")) {
+      word = word.substring(0, lastPos - 2).concat("uas");
+    }
+
+    // Rule 19
+    if (word.endsWith("vad")) {
+      word = word.substring(0, lastPos - 2).concat("vas");
+    }
+
+    // Rule 20
+    if (word.endsWith("cid")) {
+      word = word.substring(0, lastPos - 2).concat("cis");
+    }
+
+    // Rule 21
+    if (word.endsWith("lid")) {
+      word = word.substring(0, lastPos - 2).concat("lis");
+    }
+
+    // Rule 22
+    if (word.endsWith("erid")) {
+      word = word.substring(0, lastPos - 3).concat("eris");
+    }
+
+    // Rule 23
+    if (word.endsWith("pand")) {
+      word = word.substring(0, lastPos - 3).concat("pans");
+    }
+ 
+    // Rule 24
+    if (word.endsWith("end")) {
+      if ((lastPos - 3 < 0) ||
+	   (word.charAt(lastPos - 3) != 's')) {
+	word = word.substring(0, lastPos - 2).concat("ens");
+      }
+    }
+
+    // Rule 25
+    if (word.endsWith("ond")) {
+      word = word.substring(0, lastPos - 2).concat("ons");
+    }
+
+    // Rule 26
+    if (word.endsWith("lud")) {
+      word = word.substring(0, lastPos - 2).concat("lus");
+    }
+
+    // Rule 27
+    if (word.endsWith("rud")) {
+      word = word.substring(0, lastPos - 2).concat("rus");
+    }
+
+    // Rule 28
+    if (word.endsWith("her")) {
+      if ((lastPos - 3 < 0) ||
+	  ((word.charAt(lastPos - 3) != 'p') &&
+	   (word.charAt(lastPos - 3) != 't'))) {
+	word = word.substring(0, lastPos - 2).concat("hes");
+      }
+    }
+
+    // Rule 29
+    if (word.endsWith("mit")) {
+      word = word.substring(0, lastPos - 2).concat("mis");
+    }
+
+    // Rule 30
+    if (word.endsWith("end")) {
+      if ((lastPos - 3 < 0) ||
+	  (word.charAt(lastPos - 3) != 'm')) {
+	word = word.substring(0, lastPos - 2).concat("ens");
+      }
+    }
+
+    // Rule 31
+    if (word.endsWith("ert")) {
+      word = word.substring(0, lastPos - 2).concat("ers");
+    }
+
+    // Rule 32
+    if (word.endsWith("et")) {
+      if ((lastPos - 2 < 0) ||
+	  (word.charAt(lastPos - 2) != 'n')) {
+	word = word.substring(0, lastPos - 1).concat("es");
+      }
+    }
+
+    // Rule 33
+    if (word.endsWith("yt")) {
+      word = word.substring(0, lastPos - 1).concat("ys");
+    }
+
+    // Rule 34
+    if (word.endsWith("yz")) {
+      word = word.substring(0, lastPos - 1).concat("ys");
+    }
+
+    return word;
+  }
+
+  /**
+   * Returns the stemmed version of the given word.
+   * 
+   * @param word a string consisting of a single word
+   */
+  public String stem(String word) {
+
+    if (word.length() > 2) {
+      return recodeEnding(removeEnding(word.toLowerCase()));
+    } else {
+      return word.toLowerCase();
+    }
+  }
+
+  /**
+   * Stems text coming into stdin and writes it to stdout.
+   */
+  public static void main(String[] ops) {
+
+    LovinsStemmer ls = new LovinsStemmer();
+
+    try {
+      int num;
+      StringBuffer wordBuffer = new StringBuffer();
+      while ((num = System.in.read()) != -1) {
+	char c = (char)num;
+	if (((num >= (int)'A') && (num <= (int)'Z')) ||
+	    ((num >= (int)'a') && (num <= (int)'z'))) {
+	  wordBuffer.append(c);
+	} else {
+	  if (wordBuffer.length() > 0) {
+	    System.out.print(ls.stem(wordBuffer.toString().
+				     toLowerCase()));
+	    wordBuffer = new StringBuffer();
+	  }
+	  System.out.print(c);
+	}
+      }
+    } catch (Exception e) {
+      System.err.println(e.getMessage());
+    }
+  }
+}
+    
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/NumbersFilter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/NumbersFilter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/NumbersFilter.java	(revision 21112)
@@ -0,0 +1,182 @@
+/*
+ *    NumbersFilter.java
+ *    Copyright (C) 2000 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import weka.core.*;
+import weka.filters.*;
+
+import java.util.*;
+
+/**
+ * Removes all numbers from all the string attributes in the given
+ * dataset. Assumes that words are separated by whitespace.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class NumbersFilter extends Filter {
+
+  /**
+   * Returns a string describing this filter
+   *
+   * @return a description of the filter suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String globalInfo() {
+      return "Removes all numbers from all the string attributes in " +
+      "the given dataset. Assumes that words are separated by whitespace.";
+  }
+
+  /**
+   * Signify that this batch of input to the filter is finished. If
+   * the filter requires all instances prior to filtering, output()
+   * may now be called to retrieve the filtered instances. Any
+   * subsequent instances filtered should be filtered based on setting
+   * obtained from the first batch (unless the inputFormat has been
+   * re-assigned or new options have been set). This default
+   * implementation assumes all instance processing occurs during
+   * inputFormat() and input().
+   *
+   * @return true if there are instances pending output
+   * @exception NullPointerException if no input structure has been defined,
+   * @exception Exception if there was a problem finishing the batch.
+   */
+  public boolean batchFinished() throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new NullPointerException("No input instance format defined");
+    }
+    m_NewBatch = true;
+    return (numPendingOutput() != 0);
+  }
+
+  /**
+   * Sets the format of the input instances.
+   *
+   * @param instanceInfo an Instances object containing the input
+   * instance structure (any instances contained in the object are
+   * ignored - only the structure is required).
+   * @return true if the outputFormat may be collected immediately 
+   */
+  public boolean setInputFormat(Instances instanceInfo) throws Exception {
+
+    super.setInputFormat(instanceInfo);
+    setOutputFormat(instanceInfo);
+    return true;
+  }
+
+  /**
+   * Input an instance for filtering. Ordinarily the instance is processed
+   * and made available for output immediately. Some filters require all
+   * instances be read before producing output.
+   *
+   * @param instance the input instance
+   * @return true if the filtered instance may now be
+   * collected with output().
+   * @exception Exception if the input instance was not of the correct 
+   * format or if there was a problem with the filtering.
+   */
+  public boolean input(Instance instance) throws Exception {
+
+    if (getInputFormat() == null) {
+      throw new Exception("No input instance format defined");
+    }
+    if (m_NewBatch) {
+      resetQueue();
+      m_NewBatch = false;
+    }
+    convertInstance(instance);
+    return true;
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain arguments to the filter: use -h for help
+   */
+  public static void main(String [] argv) {
+    
+    try {
+      if (Utils.getFlag('b', argv)) {
+	Filter.batchFilterFile(new NumbersFilter(), argv);
+      } else {
+	Filter.filterFile(new NumbersFilter(), argv);
+      }
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+ 
+  /** 
+   * Converts an instance. A phrase boundary is inserted where
+   * a number is found.
+   */
+  private void convertInstance(Instance instance) throws Exception {
+  
+    double[] instVals = new double[instance.numAttributes()];
+
+    for (int i = 0; i < instance.numAttributes(); i++) {
+      if ((!instance.attribute(i).isString()) || instance.isMissing(i)) {
+	instVals[i] = instance.value(i);
+      } else {
+	String str = instance.stringValue(i);
+	StringBuffer resultStr = new StringBuffer();
+	StringTokenizer tok = new StringTokenizer(str, " \t\n", true);
+	while (tok.hasMoreTokens()) {
+	  String token = tok.nextToken();
+
+	  // Everything that doesn't contain at least
+	  // one letter is considered to be a number
+	  boolean isNumber = true;
+	  for (int j = 0; j < token.length(); j++) {
+	    if (Character.isLetter(token.charAt(j))) {
+	      isNumber = false;
+	      break;
+	    }
+	  }
+	  if (!isNumber) {
+	    resultStr.append(token);
+	  } else {
+	    if (token.equals(" ") || token.equals("\t") ||
+		token.equals("\n")) {
+	      resultStr.append(token);
+	    } else {
+	      resultStr.append(" \n ");
+	    }
+	  }
+	}
+	int index = getOutputFormat().attribute(i).addStringValue(resultStr.toString());
+	instVals[i] = (double)index;
+      }
+    }
+    Instance inst = new Instance(instance.weight(), instVals);
+    inst.setDataset(getOutputFormat());
+    push(inst);
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/README
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/README	(revision 21112)
@@ -0,0 +1,284 @@
+=====================================================================
+                                                                       
+                              ======                                   
+                              README                                   
+                              ======                                   
+                                                                       
+                             KEA 3.0
+                          18 March 2004
+                                                                       
+           Java Programs for Automatic Keyphrase Extraction
+                                                                       
+               Copyright (C) 2000, 2001, 2004  Eibe Frank
+
+                   email: eibe@cs.waikato.ac.nz            
+                                                                       
+=====================================================================
+
+Contents:
+---------
+
+1. Installation
+
+2. Getting started
+
+   - Building a keyphrase extraction model
+   - Extracting keyphrases
+   - Important comment
+
+3. Examples
+
+4. Other documentation
+
+5. Copyright
+
+----------------------------------------------------------------------
+
+NOTE:
+-----
+
+This distribution includes a cut-down version of WEKA, the GPL'ed
+machine learning workbench available from 
+
+  http://www.cs.waikato.ac.nz/ml/weka.
+
+----------------------------------------------------------------------
+
+1. Installation:
+----------------
+
+KEA is implemented as a set of Java classes (located in the same
+directory as this README file). To run KEA you need to tell the Java
+Virtual Machine where to look for KEA classes.  One possible way of
+doing this is to add the directory that contains this README file to
+the CLASSPATH environment variable that is used by the Java Virtual
+Machine.
+
+Under Linux you would do the following:
+
+a) Set KEAHOME to be the directory which contains this README.
+
+b) Add $KEAHOME to your CLASSPATH environment variable.
+
+The on-line documentation (generated from the source code) is located
+in the doc directory. You might want to do the following to have the
+documentation handy in you web browser:
+
+c) Bookmark $KEAHOME/doc/packages.html in your web browser.
+
+----------------------------------------------------------------------
+
+2. Getting started:
+-------------------
+
+Building a keyphrase extraction model
+=====================================
+
+To extract keyphrases for new documents, you first need to build a KEA
+keyphrase extraction model from a set of documents (preferably from
+the same domain) for which you have author- assigned keyphrases. To
+this end you have to go through the following steps:
+
+a) Create a directory, called, for example, "training_documents",
+   containing the documents that you want to use for training the
+   keyphrase extractor.
+
+b) Rename the document files in that directory so that they end with
+   the suffix ".txt".
+
+c) Delete the author-assigned keyphrases from those documents
+   and put them into separate ".key" files. For example, if
+   your document file is called doc1.txt, move the keyphrases
+   into a new file called "doc1.key". It is important that
+   you put each keyphrase on a separate line in the .key file!
+
+d) Build the keyphrase extraction model by running the
+   KEAModelBuilder:
+
+   java KEAModelBuilder -l <name_of_directory> -m <name_of_model>
+
+   This will use the documents in <name_of_directory> to build a
+   keyphrase extraction model and save it in <name_of_model>.
+
+KEAModelBuilder has a few other options that you can view if you run
+KEAModelBuilder without any arguments. Here is a list of all the
+options:
+
+-l <directory name>
+	Specifies name of directory.
+-m <model name>
+	Specifies name of model.
+-e <encoding>
+	Specifies encoding.
+-d
+	Turns debugging mode on.
+-k
+	Use keyphrase frequency statistic.
+-p
+	Disallow internal periods.
+-x <length>
+	Sets the maximum phrase length (default: 3).
+-y <length>
+	Sets the minimum phrase length (default: 1).
+-o <number>
+	The minimum number of times a phrase needs to occur 
+	(default: 2).
+-s <name of stopwords class>
+        Sets the list of stopwords to use (default: StopwordsEnglish).
+-t <name of stemmer class>
+        Set the stemmer to use (default: IteratedLovinsStemmer).
+-n
+        Do not check for proper nouns.
+   
+The -e option allows you to specify a different character encoding
+supported by Java. For example, to extract keyphrases from Chinese
+documents encoded using GBK, you would use specify "-e GBK" as an
+argument.
+
+The -d option generates some output that shows the progress of the
+model builder.
+
+If -k is set, the keyphrase frequency attribute is used in the
+model. For more info on this, have a look at the paper on
+"Domain-specific keyphrase extraction" listed below. Using this option
+improves accuracy if the domain of the documents for which you want to
+extract keyphrases is the same as the domain of the training
+documents. In other words, if you want to extract keyphrases from
+papers on radiology, and your training documents are about radiology,
+you should use this option.
+
+If -p is set, KEA does not consider phrases with internal periods as
+candidate keyphrases. It is important to use this if a full stop is
+not always followed by white space in the documents.
+
+Using -s and -t you can set different classes for stopword detection
+and stemming respectively (for languages other than English).
+
+Using -d you turn KEA's heuristic for detecting proper nouns off. This
+is important for languages like German, where all nouns start with an
+uppercase letter, not just proper nouns.
+
+Extracting keyphrases
+=====================
+
+To extract keyphrases for some documents, put them into an empty
+directory. Then rename them so that they end with the suffix ".txt".
+
+If you've previously built a keyphrase extraction model you can now
+apply keyphrases for these documents using:
+
+java KEAKeyphraseExtractor -l <name_of_directory> -m <name_of_model>
+
+This will create a ".key" file for each document in the
+directory. Each file will contain five extracted keyphrases for the
+corresponding document.
+
+If a ".key" file is already present it won't be overwritten.  Instead,
+the keyphrases present in that file will be used to evaluate the
+extraction model. The stemmed extracted phrases are compared to the
+stemmed versions of the phrases in the ".key"
+file. KEAKeyphraseExtractor reports the number of hits among the total
+number of extracted phrases for those documents that have associated
+".key" files in the directory.
+
+KEAKeyphraseExtractor has a few options. Here they are:
+
+-l <directory name>
+	Specifies name of directory.
+-m <model name>
+	Specifies name of model.
+-e <encoding>
+	Specifies encoding.
+-n
+	Specifies number of phrases to be output (default: 5).
+-d
+	Turns debugging mode on.
+-a
+	Also write stemmed phrase and score into ".key" file.
+
+Important comment
+-----------------
+
+To get good results, it is important that the input text for KEA is as
+"clean" as possible. That means html tags etc. in the input documents
+need to be deleted before the model is built and before keyphrases are
+extracted from new documents.
+
+----------------------------------------------------------------------
+
+3. Examples:
+------------
+
+The directory contains two example collections, each split up into a
+train and test directory. Note that these collections are only
+included to show how the system can be applied to actual documents.
+Due to the lack of data, the accuracy isn't very good on either
+example collection.
+
+Collection A
+------------
+
+A collection of abstracts taken from computer science technical
+reports:
+
+   CSTR_abstracts_train
+   CSTR_abstracts_test
+
+To build a model from the training data, try:
+
+   java KEAModelBuilder -l CSTR_abstracts_train -m CSTR_abstracts_model
+
+To evaluate that model on the test data, try:
+
+   java KEAKeyphraseExtractor -l CSTR_abstracts_test -m CSTR_abstracts_model
+
+Collection B
+------------
+
+A small collection of Chinese documents (in GBK encoding):
+
+   Journals_train
+   Journals_test
+
+To build a model from the training data, try:
+
+   java KEAModelBuilder -l Chinese_train -m Chinese_model -e GBK
+
+To evaluate that model on the test data, try:
+
+   java KEAKeyphraseExtractor -l Chinese_test -m Chinese_model -e GBK 
+
+----------------------------------------------------------------------
+
+4. Other documentation:
+-----------------------
+
+There are several papers on the KEA algorithm, listed below. Note that
+this implementation differs slightly from the version described in the
+papers, mainly in the pre-processing step (i.e. in the way candidate
+keyphrases are generated). For more info on the new method please
+consult the online documentation.
+
+Witten I.H., Paynter G.W., Frank E., Gutwin C. and Nevill-Manning
+C.G. (2000) "KEA: Practical automatic keyphrase extraction." Working
+Paper 00/5, Department of Computer Science, The University of Waikato.
+
+Witten I.H., Paynter G.W., Frank E., Gutwin C. and Nevill-Manning
+C.G. (1999) "KEA: Practical automatic keyphrase extraction." Proc. DL
+'99, pp. 254-256. (Poster presentation.)
+
+Frank E., Paynter G.W., Witten I.H., Gutwin C. and Nevill-Manning
+C.G. (1999) "Domain-specific keyphrase extraction" Proc. Sixteenth
+International Joint Conference on Artificial Intelligence, Morgan
+Kaufmann Publishers, San Francisco, CA, pp. 668-673.
+
+-----------------------------------------------------------------------
+
+5. Copyright:
+-------------
+
+KEA is distributed under the GNU public license. Please read the file
+COPYING.
+
+-----------------------------------------------------------------------
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stemmer.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stemmer.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stemmer.java	(revision 21112)
@@ -0,0 +1,68 @@
+/*
+ *    Stemmer.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Abstract class for stemmers.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public abstract class Stemmer implements Serializable {
+
+  /**
+   * Iterated stemming of the given word.
+   */
+  public abstract String stem(String str);
+
+  /**
+   * Stems everything in the given string.
+   */
+  public String stemString(String str) {
+
+    StringBuffer result = new StringBuffer();
+    int start = -1;
+    for (int j = 0; j < str.length(); j++) {
+      char c = str.charAt(j);
+      if (Character.isLetterOrDigit(c)) {
+	if (start == -1) {
+	  start = j;
+	}
+      } else if (c == '\'') {
+	if (start == -1) {
+	  result.append(c);
+	}
+      } else {
+	if (start != -1) {
+	  result.append(stem(str.substring(start, j)));
+	  start = -1;
+	}
+	result.append(c);
+      }
+    }
+    if (start != -1) {
+      result.append(stem(str.substring(start, str.length())));
+    }
+    return result.toString();  
+  }
+}
+    
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stopwords.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stopwords.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/Stopwords.java	(revision 21112)
@@ -0,0 +1,38 @@
+/*
+ *    Stopwords.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Class that can test whether a given string is a stop word.
+ * Lowercases all words before the test.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public abstract class Stopwords implements Serializable {
+  
+  /** 
+   * Returns true if the given string is a stop word.
+   */
+  public abstract boolean isStopword(String str);
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsEnglish.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsEnglish.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsEnglish.java	(revision 21112)
@@ -0,0 +1,504 @@
+/*
+ *    StopwordsEnglish.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+
+/**
+ * Class that can test whether a given string is a stop word.
+ * Lowercases all words before the test.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class StopwordsEnglish extends Stopwords {
+  
+  /** The hashtable containing the list of stopwords */
+  private static Hashtable m_Stopwords = null;
+
+  static {
+   
+    if (m_Stopwords == null) {
+      m_Stopwords = new Hashtable();
+      Double dummy = new Double(0);
+      
+      m_Stopwords.put("a", dummy);
+      m_Stopwords.put("abaft", dummy);
+      m_Stopwords.put("aboard", dummy);
+      m_Stopwords.put("about", dummy);
+      m_Stopwords.put("above", dummy);
+      m_Stopwords.put("across", dummy);
+      m_Stopwords.put("afore", dummy);
+      m_Stopwords.put("aforesaid", dummy);
+      m_Stopwords.put("after", dummy);
+      m_Stopwords.put("again", dummy);
+      m_Stopwords.put("against", dummy);
+      m_Stopwords.put("agin", dummy);
+      m_Stopwords.put("ago", dummy);
+      m_Stopwords.put("aint", dummy);
+      m_Stopwords.put("albeit", dummy);
+      m_Stopwords.put("all", dummy);
+      m_Stopwords.put("almost", dummy);
+      m_Stopwords.put("alone", dummy);
+      m_Stopwords.put("along", dummy);
+      m_Stopwords.put("alongside", dummy);
+      m_Stopwords.put("already", dummy);
+      m_Stopwords.put("also", dummy);
+      m_Stopwords.put("although", dummy);
+      m_Stopwords.put("always", dummy);
+      m_Stopwords.put("am", dummy);
+      m_Stopwords.put("american", dummy);
+      m_Stopwords.put("amid", dummy);
+      m_Stopwords.put("amidst", dummy);
+      m_Stopwords.put("among", dummy);
+      m_Stopwords.put("amongst", dummy);
+      m_Stopwords.put("an", dummy);
+      m_Stopwords.put("and", dummy);
+      m_Stopwords.put("anent", dummy);
+      m_Stopwords.put("another", dummy);
+      m_Stopwords.put("any", dummy);
+      m_Stopwords.put("anybody", dummy);
+      m_Stopwords.put("anyone", dummy);
+      m_Stopwords.put("anything", dummy);
+      m_Stopwords.put("are", dummy);
+      m_Stopwords.put("aren't", dummy);
+      m_Stopwords.put("around", dummy);
+      m_Stopwords.put("as", dummy);
+      m_Stopwords.put("aslant", dummy);
+      m_Stopwords.put("astride", dummy);
+      m_Stopwords.put("at", dummy);
+      m_Stopwords.put("athwart", dummy);
+      m_Stopwords.put("away", dummy);
+      m_Stopwords.put("b", dummy);
+      m_Stopwords.put("back", dummy);
+      m_Stopwords.put("bar", dummy);
+      m_Stopwords.put("barring", dummy);
+      m_Stopwords.put("be", dummy);
+      m_Stopwords.put("because", dummy);
+      m_Stopwords.put("been", dummy);
+      m_Stopwords.put("before", dummy);
+      m_Stopwords.put("behind", dummy);
+      m_Stopwords.put("being", dummy);
+      m_Stopwords.put("below", dummy);
+      m_Stopwords.put("beneath", dummy);
+      m_Stopwords.put("beside", dummy);
+      m_Stopwords.put("besides", dummy);
+      m_Stopwords.put("best", dummy);
+      m_Stopwords.put("better", dummy);
+      m_Stopwords.put("between", dummy);
+      m_Stopwords.put("betwixt", dummy);
+      m_Stopwords.put("beyond", dummy);
+      m_Stopwords.put("both", dummy);
+      m_Stopwords.put("but", dummy);
+      m_Stopwords.put("by", dummy);
+      m_Stopwords.put("c", dummy);
+      m_Stopwords.put("can", dummy);
+      m_Stopwords.put("cannot", dummy);
+      m_Stopwords.put("can't", dummy);
+      m_Stopwords.put("certain", dummy);
+      m_Stopwords.put("circa", dummy);
+      m_Stopwords.put("close", dummy);
+      m_Stopwords.put("concerning", dummy);
+      m_Stopwords.put("considering", dummy);
+      m_Stopwords.put("cos", dummy);
+      m_Stopwords.put("could", dummy);
+      m_Stopwords.put("couldn't", dummy);
+      m_Stopwords.put("couldst", dummy);
+      m_Stopwords.put("d", dummy);
+      m_Stopwords.put("dare", dummy);
+      m_Stopwords.put("dared", dummy);
+      m_Stopwords.put("daren't", dummy);
+      m_Stopwords.put("dares", dummy);
+      m_Stopwords.put("daring", dummy);
+      m_Stopwords.put("despite", dummy);
+      m_Stopwords.put("did", dummy);
+      m_Stopwords.put("didn't", dummy);
+      m_Stopwords.put("different", dummy);
+      m_Stopwords.put("directly", dummy);
+      m_Stopwords.put("do", dummy);
+      m_Stopwords.put("does", dummy);
+      m_Stopwords.put("doesn't", dummy);
+      m_Stopwords.put("doing", dummy);
+      m_Stopwords.put("done", dummy);
+      m_Stopwords.put("don't", dummy);
+      m_Stopwords.put("dost", dummy);
+      m_Stopwords.put("doth", dummy);
+      m_Stopwords.put("down", dummy);
+      m_Stopwords.put("during", dummy);
+      m_Stopwords.put("durst", dummy);
+      m_Stopwords.put("e", dummy);
+      m_Stopwords.put("each", dummy);
+      m_Stopwords.put("early", dummy);
+      m_Stopwords.put("either", dummy);
+      m_Stopwords.put("em", dummy);
+      m_Stopwords.put("english", dummy);
+      m_Stopwords.put("enough", dummy);
+      m_Stopwords.put("ere", dummy);
+      m_Stopwords.put("even", dummy);
+      m_Stopwords.put("ever", dummy);
+      m_Stopwords.put("every", dummy);
+      m_Stopwords.put("everybody", dummy);
+      m_Stopwords.put("everyone", dummy);
+      m_Stopwords.put("everything", dummy);
+      m_Stopwords.put("except", dummy);
+      m_Stopwords.put("excepting", dummy);
+      m_Stopwords.put("f", dummy);
+      m_Stopwords.put("failing", dummy);
+      m_Stopwords.put("far", dummy);
+      m_Stopwords.put("few", dummy);
+      m_Stopwords.put("first", dummy);
+      m_Stopwords.put("five", dummy);
+      m_Stopwords.put("following", dummy);
+      m_Stopwords.put("for", dummy);
+      m_Stopwords.put("four", dummy);
+      m_Stopwords.put("from", dummy);
+      m_Stopwords.put("g", dummy);
+      m_Stopwords.put("gonna", dummy);
+      m_Stopwords.put("gotta", dummy);
+      m_Stopwords.put("h", dummy);
+      m_Stopwords.put("had", dummy);
+      m_Stopwords.put("hadn't", dummy);
+      m_Stopwords.put("hard", dummy);
+      m_Stopwords.put("has", dummy);
+      m_Stopwords.put("hasn't", dummy);
+      m_Stopwords.put("hast", dummy);
+      m_Stopwords.put("hath", dummy);
+      m_Stopwords.put("have", dummy);
+      m_Stopwords.put("haven't", dummy);
+      m_Stopwords.put("having", dummy);
+      m_Stopwords.put("he", dummy);
+      m_Stopwords.put("he'd", dummy);
+      m_Stopwords.put("he'll", dummy);
+      m_Stopwords.put("her", dummy);
+      m_Stopwords.put("here", dummy);
+      m_Stopwords.put("here's", dummy);
+      m_Stopwords.put("hers", dummy);
+      m_Stopwords.put("herself", dummy);
+      m_Stopwords.put("he's", dummy);
+      m_Stopwords.put("high", dummy);
+      m_Stopwords.put("him", dummy);
+      m_Stopwords.put("himself", dummy);
+      m_Stopwords.put("his", dummy);
+      m_Stopwords.put("home", dummy);
+      m_Stopwords.put("how", dummy);
+      m_Stopwords.put("howbeit", dummy);
+      m_Stopwords.put("however", dummy);
+      m_Stopwords.put("how's", dummy);
+      m_Stopwords.put("i", dummy);
+      m_Stopwords.put("id", dummy);
+      m_Stopwords.put("if", dummy);
+      m_Stopwords.put("ill", dummy);
+      m_Stopwords.put("i'm", dummy);
+      m_Stopwords.put("immediately", dummy);
+      m_Stopwords.put("important", dummy);
+      m_Stopwords.put("in", dummy);
+      m_Stopwords.put("inside", dummy);
+      m_Stopwords.put("instantly", dummy);
+      m_Stopwords.put("into", dummy);
+      m_Stopwords.put("is", dummy);
+      m_Stopwords.put("isn't", dummy);
+      m_Stopwords.put("it", dummy);
+      m_Stopwords.put("it'll", dummy);
+      m_Stopwords.put("it's", dummy);
+      m_Stopwords.put("its", dummy);
+      m_Stopwords.put("itself", dummy);
+      m_Stopwords.put("i've", dummy);
+      m_Stopwords.put("j", dummy);
+      m_Stopwords.put("just", dummy);
+      m_Stopwords.put("k", dummy);
+      m_Stopwords.put("l", dummy);
+      m_Stopwords.put("large", dummy);
+      m_Stopwords.put("last", dummy);
+      m_Stopwords.put("later", dummy);
+      m_Stopwords.put("least", dummy);
+      m_Stopwords.put("left", dummy);
+      m_Stopwords.put("less", dummy);
+      m_Stopwords.put("lest", dummy);
+      m_Stopwords.put("let's", dummy);
+      m_Stopwords.put("like", dummy);
+      m_Stopwords.put("likewise", dummy);
+      m_Stopwords.put("little", dummy);
+      m_Stopwords.put("living", dummy);
+      m_Stopwords.put("long", dummy);
+      m_Stopwords.put("m", dummy);
+      m_Stopwords.put("many", dummy);
+      m_Stopwords.put("may", dummy);
+      m_Stopwords.put("mayn't", dummy);
+      m_Stopwords.put("me", dummy);
+      m_Stopwords.put("mid", dummy);
+      m_Stopwords.put("midst", dummy);
+      m_Stopwords.put("might", dummy);
+      m_Stopwords.put("mightn't", dummy);
+      m_Stopwords.put("mine", dummy);
+      m_Stopwords.put("minus", dummy);
+      m_Stopwords.put("more", dummy);
+      m_Stopwords.put("most", dummy);
+      m_Stopwords.put("much", dummy);
+      m_Stopwords.put("must", dummy);
+      m_Stopwords.put("mustn't", dummy);
+      m_Stopwords.put("my", dummy);
+      m_Stopwords.put("myself", dummy);
+      m_Stopwords.put("n", dummy);
+      m_Stopwords.put("near", dummy);
+      m_Stopwords.put("'neath", dummy);
+      m_Stopwords.put("need", dummy);
+      m_Stopwords.put("needed", dummy);
+      m_Stopwords.put("needing", dummy);
+      m_Stopwords.put("needn't", dummy);
+      m_Stopwords.put("needs", dummy);
+      m_Stopwords.put("neither", dummy);
+      m_Stopwords.put("never", dummy);
+      m_Stopwords.put("nevertheless", dummy);
+      m_Stopwords.put("new", dummy);
+      m_Stopwords.put("next", dummy);
+      m_Stopwords.put("nigh", dummy);
+      m_Stopwords.put("nigher", dummy);
+      m_Stopwords.put("nighest", dummy);
+      m_Stopwords.put("nisi", dummy);
+      m_Stopwords.put("no", dummy);
+      m_Stopwords.put("no-one", dummy);
+      m_Stopwords.put("nobody", dummy);
+      m_Stopwords.put("none", dummy);
+      m_Stopwords.put("nor", dummy);
+      m_Stopwords.put("not", dummy);
+      m_Stopwords.put("nothing", dummy);
+      m_Stopwords.put("notwithstanding", dummy);
+      m_Stopwords.put("now", dummy);
+      m_Stopwords.put("o", dummy);
+      m_Stopwords.put("o'er", dummy);
+      m_Stopwords.put("of", dummy);
+      m_Stopwords.put("off", dummy);
+      m_Stopwords.put("often", dummy);
+      m_Stopwords.put("on", dummy);
+      m_Stopwords.put("once", dummy);
+      m_Stopwords.put("one", dummy);
+      m_Stopwords.put("oneself", dummy);
+      m_Stopwords.put("only", dummy);
+      m_Stopwords.put("onto", dummy);
+      m_Stopwords.put("open", dummy);
+      m_Stopwords.put("or", dummy);
+      m_Stopwords.put("other", dummy);
+      m_Stopwords.put("otherwise", dummy);
+      m_Stopwords.put("ought", dummy);
+      m_Stopwords.put("oughtn't", dummy);
+      m_Stopwords.put("our", dummy);
+      m_Stopwords.put("ours", dummy);
+      m_Stopwords.put("ourselves", dummy);
+      m_Stopwords.put("out", dummy);
+      m_Stopwords.put("outside", dummy);
+      m_Stopwords.put("over", dummy);
+      m_Stopwords.put("own", dummy);
+      m_Stopwords.put("p", dummy);
+      m_Stopwords.put("past", dummy);
+      m_Stopwords.put("pending", dummy);
+      m_Stopwords.put("per", dummy);
+      m_Stopwords.put("perhaps", dummy);
+      m_Stopwords.put("plus", dummy);
+      m_Stopwords.put("possible", dummy);
+      m_Stopwords.put("present", dummy);
+      m_Stopwords.put("probably", dummy);
+      m_Stopwords.put("provided", dummy);
+      m_Stopwords.put("providing", dummy);
+      m_Stopwords.put("public", dummy);
+      m_Stopwords.put("q", dummy);
+      m_Stopwords.put("qua", dummy);
+      m_Stopwords.put("quite", dummy);
+      m_Stopwords.put("r", dummy);
+      m_Stopwords.put("rather", dummy);
+      m_Stopwords.put("re", dummy);
+      m_Stopwords.put("real", dummy);
+      m_Stopwords.put("really", dummy);
+      m_Stopwords.put("respecting", dummy);
+      m_Stopwords.put("right", dummy);
+      m_Stopwords.put("round", dummy);
+      m_Stopwords.put("s", dummy);
+      m_Stopwords.put("same", dummy);
+      m_Stopwords.put("sans", dummy);
+      m_Stopwords.put("save", dummy);
+      m_Stopwords.put("saving", dummy);
+      m_Stopwords.put("second", dummy);
+      m_Stopwords.put("several", dummy);
+      m_Stopwords.put("shall", dummy);
+      m_Stopwords.put("shalt", dummy);
+      m_Stopwords.put("shan't", dummy);
+      m_Stopwords.put("she", dummy);
+      m_Stopwords.put("shed", dummy);
+      m_Stopwords.put("shell", dummy);
+      m_Stopwords.put("she's", dummy);
+      m_Stopwords.put("short", dummy);
+      m_Stopwords.put("should", dummy);
+      m_Stopwords.put("shouldn't", dummy);
+      m_Stopwords.put("since", dummy);
+      m_Stopwords.put("six", dummy);
+      m_Stopwords.put("small", dummy);
+      m_Stopwords.put("so", dummy);
+      m_Stopwords.put("some", dummy);
+      m_Stopwords.put("somebody", dummy);
+      m_Stopwords.put("someone", dummy);
+      m_Stopwords.put("something", dummy);
+      m_Stopwords.put("sometimes", dummy);
+      m_Stopwords.put("soon", dummy);
+      m_Stopwords.put("special", dummy);
+      m_Stopwords.put("still", dummy);
+      m_Stopwords.put("such", dummy);
+      m_Stopwords.put("summat", dummy);
+      m_Stopwords.put("supposing", dummy);
+      m_Stopwords.put("sure", dummy);
+      m_Stopwords.put("t", dummy);
+      m_Stopwords.put("than", dummy);
+      m_Stopwords.put("that", dummy);
+      m_Stopwords.put("that'd", dummy);
+      m_Stopwords.put("that'll", dummy);
+      m_Stopwords.put("that's", dummy);
+      m_Stopwords.put("the", dummy);
+      m_Stopwords.put("thee", dummy);
+      m_Stopwords.put("their", dummy);
+      m_Stopwords.put("theirs", dummy);
+      m_Stopwords.put("their's", dummy);
+      m_Stopwords.put("them", dummy);
+      m_Stopwords.put("themselves", dummy);
+      m_Stopwords.put("then", dummy);
+      m_Stopwords.put("there", dummy);
+      m_Stopwords.put("there's", dummy);
+      m_Stopwords.put("these", dummy);
+      m_Stopwords.put("they", dummy);
+      m_Stopwords.put("they'd", dummy);
+      m_Stopwords.put("they'll", dummy);
+      m_Stopwords.put("they're", dummy);
+      m_Stopwords.put("they've", dummy);
+      m_Stopwords.put("thine", dummy);
+      m_Stopwords.put("this", dummy);
+      m_Stopwords.put("tho", dummy);
+      m_Stopwords.put("those", dummy);
+      m_Stopwords.put("thou", dummy);
+      m_Stopwords.put("though", dummy);
+      m_Stopwords.put("three", dummy);
+      m_Stopwords.put("thro'", dummy);
+      m_Stopwords.put("through", dummy);
+      m_Stopwords.put("throughout", dummy);
+      m_Stopwords.put("thru", dummy);
+      m_Stopwords.put("thyself", dummy);
+      m_Stopwords.put("till", dummy);
+      m_Stopwords.put("to", dummy);
+      m_Stopwords.put("today", dummy);
+      m_Stopwords.put("together", dummy);
+      m_Stopwords.put("too", dummy);
+      m_Stopwords.put("touching", dummy);
+      m_Stopwords.put("toward", dummy);
+      m_Stopwords.put("towards", dummy);
+      m_Stopwords.put("true", dummy);
+      m_Stopwords.put("'twas", dummy);
+      m_Stopwords.put("'tween", dummy);
+      m_Stopwords.put("'twere", dummy);
+      m_Stopwords.put("'twill", dummy);
+      m_Stopwords.put("'twixt", dummy);
+      m_Stopwords.put("two", dummy);
+      m_Stopwords.put("'twould", dummy);
+      m_Stopwords.put("u", dummy);
+      m_Stopwords.put("under", dummy);
+      m_Stopwords.put("underneath", dummy);
+      m_Stopwords.put("unless", dummy);
+      m_Stopwords.put("unlike", dummy);
+      m_Stopwords.put("until", dummy);
+      m_Stopwords.put("unto", dummy);
+      m_Stopwords.put("up", dummy);
+      m_Stopwords.put("upon", dummy);
+      m_Stopwords.put("us", dummy);
+      m_Stopwords.put("used", dummy);
+      m_Stopwords.put("usually", dummy);
+      m_Stopwords.put("v", dummy);
+      m_Stopwords.put("versus", dummy);
+      m_Stopwords.put("very", dummy);
+      m_Stopwords.put("via", dummy);
+      m_Stopwords.put("vice", dummy);
+      m_Stopwords.put("vis-a-vis", dummy);
+      m_Stopwords.put("w", dummy);
+      m_Stopwords.put("wanna", dummy);
+      m_Stopwords.put("wanting", dummy);
+      m_Stopwords.put("was", dummy);
+      m_Stopwords.put("wasn't", dummy);
+      m_Stopwords.put("way", dummy);
+      m_Stopwords.put("we", dummy);
+      m_Stopwords.put("we'd", dummy);
+      m_Stopwords.put("well", dummy);
+      m_Stopwords.put("were", dummy);
+      m_Stopwords.put("weren't", dummy);
+      m_Stopwords.put("wert", dummy);
+      m_Stopwords.put("we've", dummy);
+      m_Stopwords.put("what", dummy);
+      m_Stopwords.put("whatever", dummy);
+      m_Stopwords.put("what'll", dummy);
+      m_Stopwords.put("what's", dummy);
+      m_Stopwords.put("when", dummy);
+      m_Stopwords.put("whencesoever", dummy);
+      m_Stopwords.put("whenever", dummy);
+      m_Stopwords.put("when's", dummy);
+      m_Stopwords.put("whereas", dummy);
+      m_Stopwords.put("where's", dummy);
+      m_Stopwords.put("whether", dummy);
+      m_Stopwords.put("which", dummy);
+      m_Stopwords.put("whichever", dummy);
+      m_Stopwords.put("whichsoever", dummy);
+      m_Stopwords.put("while", dummy);
+      m_Stopwords.put("whilst", dummy);
+      m_Stopwords.put("who", dummy);
+      m_Stopwords.put("who'd", dummy);
+      m_Stopwords.put("whoever", dummy);
+      m_Stopwords.put("whole", dummy);
+      m_Stopwords.put("who'll", dummy);
+      m_Stopwords.put("whom", dummy);
+      m_Stopwords.put("whore", dummy);
+      m_Stopwords.put("who's", dummy);
+      m_Stopwords.put("whose", dummy);
+      m_Stopwords.put("whoso", dummy);
+      m_Stopwords.put("whosoever", dummy);
+      m_Stopwords.put("will", dummy);
+      m_Stopwords.put("with", dummy);
+      m_Stopwords.put("within", dummy);
+      m_Stopwords.put("without", dummy);
+      m_Stopwords.put("wont", dummy);
+      m_Stopwords.put("would", dummy);
+      m_Stopwords.put("wouldn't", dummy);
+      m_Stopwords.put("wouldst", dummy);
+      m_Stopwords.put("x", dummy);
+      m_Stopwords.put("y", dummy);
+      m_Stopwords.put("ye", dummy);
+      m_Stopwords.put("yet", dummy);
+      m_Stopwords.put("you", dummy);
+      m_Stopwords.put("you'd", dummy);
+      m_Stopwords.put("you'll", dummy);
+      m_Stopwords.put("your", dummy);
+      m_Stopwords.put("you're", dummy);
+      m_Stopwords.put("yours", dummy);
+      m_Stopwords.put("yourself", dummy);
+      m_Stopwords.put("yourselves", dummy);
+      m_Stopwords.put("you've", dummy);
+      m_Stopwords.put("z", dummy);
+    }
+  }
+  
+  /** 
+   * Returns true if the given string is a stop word.
+   */
+  public boolean isStopword(String str) {
+
+    return m_Stopwords.containsKey(str.toLowerCase());
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsGerman.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsGerman.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/StopwordsGerman.java	(revision 21112)
@@ -0,0 +1,330 @@
+/*
+ *    StopwordsGerman.java
+ *    Copyright (C) 2001 Eibe Frank
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+import java.util.*;
+
+/**
+ * Class that can test whether a given string is a stop word.
+ * Lowercases all words before the test.
+ *
+ * This list of German stop words has been obtained from
+ * http://snowball.tartarus.org/german/stop.txt
+ *
+ * But I have deleted/changed some words that I haven't seen before.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version 1.0
+ */
+public class StopwordsGerman extends Stopwords {
+  
+  /** The hashtable containing the list of stopwords */
+  private static Hashtable m_Stopwords = null;
+
+  static {
+   
+    if (m_Stopwords == null) {
+      m_Stopwords = new Hashtable();
+      Double dummy = new Double(0);
+
+      m_Stopwords.put("aber", dummy);
+
+      m_Stopwords.put("alle", dummy);
+      m_Stopwords.put("allem", dummy);
+      m_Stopwords.put("allen", dummy);
+      m_Stopwords.put("aller", dummy);
+      m_Stopwords.put("alles", dummy);
+
+      m_Stopwords.put("als", dummy);
+      m_Stopwords.put("also", dummy);
+      m_Stopwords.put("am", dummy);
+      m_Stopwords.put("an", dummy);
+
+      m_Stopwords.put("ander", dummy);
+      m_Stopwords.put("andere", dummy);
+      m_Stopwords.put("anderem", dummy);
+      m_Stopwords.put("anderen", dummy);
+      m_Stopwords.put("anderer", dummy);
+      m_Stopwords.put("anderes", dummy);
+      m_Stopwords.put("anderm", dummy);
+      m_Stopwords.put("andern", dummy);
+      m_Stopwords.put("anders", dummy);
+
+      m_Stopwords.put("auch", dummy);
+      m_Stopwords.put("auf", dummy);
+      m_Stopwords.put("aus", dummy);
+      m_Stopwords.put("bei", dummy);
+      m_Stopwords.put("bin", dummy);
+      m_Stopwords.put("bis", dummy);
+      m_Stopwords.put("bist", dummy);
+      m_Stopwords.put("da", dummy);
+      m_Stopwords.put("damit", dummy);
+      m_Stopwords.put("dann", dummy);
+      
+      m_Stopwords.put("der", dummy);
+      m_Stopwords.put("den", dummy);
+      m_Stopwords.put("des", dummy);
+      m_Stopwords.put("dem", dummy);
+      m_Stopwords.put("die", dummy);
+      m_Stopwords.put("das", dummy);
+
+      m_Stopwords.put("da\u00df", dummy);
+
+      m_Stopwords.put("derselbe", dummy);
+      m_Stopwords.put("derselben", dummy);
+      m_Stopwords.put("denselben", dummy);
+      m_Stopwords.put("desselben", dummy);
+      m_Stopwords.put("demselben", dummy);
+      m_Stopwords.put("dieselbe", dummy);
+      m_Stopwords.put("dieselben", dummy);
+      m_Stopwords.put("dasselbe", dummy);
+
+      m_Stopwords.put("dazu", dummy);
+
+      m_Stopwords.put("dein", dummy);
+      m_Stopwords.put("deine", dummy);
+      m_Stopwords.put("deinem", dummy);
+      m_Stopwords.put("deinen", dummy);
+      m_Stopwords.put("deiner", dummy);
+      m_Stopwords.put("deines", dummy);
+
+      m_Stopwords.put("denn", dummy);
+      
+      m_Stopwords.put("derer", dummy);
+      m_Stopwords.put("dessen", dummy);
+      
+      m_Stopwords.put("dich", dummy);
+      m_Stopwords.put("dir", dummy);
+      m_Stopwords.put("du", dummy);
+
+      m_Stopwords.put("dies", dummy);
+      m_Stopwords.put("diese", dummy);
+      m_Stopwords.put("diesem", dummy);
+      m_Stopwords.put("diesen", dummy);
+      m_Stopwords.put("dieser", dummy);
+      m_Stopwords.put("dieses", dummy);
+
+      m_Stopwords.put("doch", dummy);
+      m_Stopwords.put("dort", dummy);
+
+      m_Stopwords.put("durch", dummy);
+
+      m_Stopwords.put("ein", dummy);
+      m_Stopwords.put("eine", dummy);
+      m_Stopwords.put("einem", dummy);
+      m_Stopwords.put("einen", dummy);
+      m_Stopwords.put("einer", dummy);
+      m_Stopwords.put("eines", dummy);
+
+      m_Stopwords.put("einig", dummy);
+      m_Stopwords.put("einige", dummy);
+      m_Stopwords.put("einigem", dummy);
+      m_Stopwords.put("einigen", dummy);
+      m_Stopwords.put("einiger", dummy);
+      m_Stopwords.put("einiges", dummy);
+
+      m_Stopwords.put("einmal", dummy);
+
+      m_Stopwords.put("er", dummy);
+      m_Stopwords.put("ihn", dummy);
+      m_Stopwords.put("ihm", dummy);
+
+      m_Stopwords.put("es", dummy);
+      m_Stopwords.put("etwas", dummy);
+
+      m_Stopwords.put("euer", dummy);
+      m_Stopwords.put("eure", dummy);
+      m_Stopwords.put("eurem", dummy);
+      m_Stopwords.put("euren", dummy);
+      m_Stopwords.put("eurer", dummy);
+      m_Stopwords.put("eures", dummy);
+
+      m_Stopwords.put("f\u00fcr", dummy);
+      m_Stopwords.put("gegen", dummy);
+      m_Stopwords.put("gewesen", dummy);
+      m_Stopwords.put("hab", dummy);
+      m_Stopwords.put("habe", dummy);
+      m_Stopwords.put("haben", dummy);
+      m_Stopwords.put("hat", dummy);
+      m_Stopwords.put("hatte", dummy);
+      m_Stopwords.put("hatten", dummy);
+      m_Stopwords.put("hier", dummy);
+      m_Stopwords.put("hin", dummy);
+      m_Stopwords.put("hinter", dummy);
+      
+      m_Stopwords.put("ich", dummy);
+      m_Stopwords.put("mich", dummy);
+      m_Stopwords.put("mir", dummy);
+
+      m_Stopwords.put("ihr", dummy);
+      m_Stopwords.put("ihre", dummy);
+      m_Stopwords.put("ihrem", dummy);
+      m_Stopwords.put("ihren", dummy);
+      m_Stopwords.put("ihrer", dummy);
+      m_Stopwords.put("ihres", dummy);
+      m_Stopwords.put("euch", dummy);
+
+      m_Stopwords.put("im", dummy);
+      m_Stopwords.put("in", dummy);
+      m_Stopwords.put("indem", dummy);
+      m_Stopwords.put("ins", dummy);
+      m_Stopwords.put("ist", dummy);
+
+      m_Stopwords.put("jede", dummy);
+      m_Stopwords.put("jedem", dummy);
+      m_Stopwords.put("jeden", dummy);
+      m_Stopwords.put("jeder", dummy);
+      m_Stopwords.put("jedes", dummy);
+
+      m_Stopwords.put("jene", dummy);
+      m_Stopwords.put("jenem", dummy);
+      m_Stopwords.put("jenen", dummy);
+      m_Stopwords.put("jener", dummy);
+      m_Stopwords.put("jenes", dummy);
+
+      m_Stopwords.put("jetzt", dummy);
+      m_Stopwords.put("kann", dummy);
+
+      m_Stopwords.put("kein", dummy);
+      m_Stopwords.put("keine", dummy);
+      m_Stopwords.put("keinem", dummy);
+      m_Stopwords.put("keinen", dummy);
+      m_Stopwords.put("keiner", dummy);
+      m_Stopwords.put("keines", dummy);
+
+      m_Stopwords.put("k\u00f6nnen", dummy);
+      m_Stopwords.put("k\u00f6nnte", dummy);
+      m_Stopwords.put("machen", dummy);
+      m_Stopwords.put("man", dummy);
+
+      m_Stopwords.put("manche", dummy);
+      m_Stopwords.put("manchem", dummy);
+      m_Stopwords.put("manchen", dummy);
+      m_Stopwords.put("mancher", dummy);
+      m_Stopwords.put("manches", dummy);
+
+      m_Stopwords.put("mein", dummy);
+      m_Stopwords.put("meine", dummy);
+      m_Stopwords.put("meinem", dummy);
+      m_Stopwords.put("meinen", dummy);
+      m_Stopwords.put("meiner", dummy);
+      m_Stopwords.put("meines", dummy);
+
+      m_Stopwords.put("mit", dummy);
+      m_Stopwords.put("muss", dummy);
+      m_Stopwords.put("musste", dummy);
+      m_Stopwords.put("nach", dummy);
+      m_Stopwords.put("nicht", dummy);
+      m_Stopwords.put("nichts", dummy);
+      m_Stopwords.put("noch", dummy);
+      m_Stopwords.put("nun", dummy);
+      m_Stopwords.put("nur", dummy);
+      m_Stopwords.put("ob", dummy);
+      m_Stopwords.put("oder", dummy);
+      m_Stopwords.put("ohne", dummy);
+      m_Stopwords.put("sehr", dummy);
+
+      m_Stopwords.put("sein", dummy);
+      m_Stopwords.put("seine", dummy);
+      m_Stopwords.put("seinem", dummy);
+      m_Stopwords.put("seinen", dummy);
+      m_Stopwords.put("seiner", dummy);
+      m_Stopwords.put("seines", dummy);
+
+      m_Stopwords.put("selbst", dummy);
+      m_Stopwords.put("sich", dummy);
+      
+      m_Stopwords.put("sie", dummy);
+      m_Stopwords.put("ihnen", dummy);
+
+      m_Stopwords.put("sind", dummy);
+      m_Stopwords.put("so", dummy);
+
+      m_Stopwords.put("solche", dummy);
+      m_Stopwords.put("solchem", dummy);
+      m_Stopwords.put("solchen", dummy);
+      m_Stopwords.put("solcher", dummy);
+      m_Stopwords.put("solches", dummy);
+
+      m_Stopwords.put("soll", dummy);
+      m_Stopwords.put("sollte", dummy);
+      m_Stopwords.put("sondern", dummy);
+      m_Stopwords.put("sonst", dummy);
+      m_Stopwords.put("\00fcber", dummy);
+      m_Stopwords.put("um", dummy);
+      m_Stopwords.put("und", dummy);
+
+      m_Stopwords.put("uns", dummy);
+      m_Stopwords.put("unser", dummy);
+      m_Stopwords.put("unserem", dummy);
+      m_Stopwords.put("unseren", dummy);
+      m_Stopwords.put("unsere", dummy);
+      m_Stopwords.put("unseres", dummy);
+
+      m_Stopwords.put("unter", dummy);
+      m_Stopwords.put("viel", dummy);
+      m_Stopwords.put("vom", dummy);
+      m_Stopwords.put("von", dummy);
+      m_Stopwords.put("vor", dummy);
+      m_Stopwords.put("w\u00e4hrend", dummy);
+      m_Stopwords.put("war", dummy);
+      m_Stopwords.put("waren", dummy);
+      m_Stopwords.put("warst", dummy);
+      m_Stopwords.put("was", dummy);
+      m_Stopwords.put("weg", dummy);
+      m_Stopwords.put("weil", dummy);
+      m_Stopwords.put("weiter", dummy);
+
+      m_Stopwords.put("welche", dummy);
+      m_Stopwords.put("welchem", dummy);
+      m_Stopwords.put("welchen", dummy);
+      m_Stopwords.put("welcher", dummy);
+      m_Stopwords.put("welches", dummy);
+
+      m_Stopwords.put("wenn", dummy);
+      m_Stopwords.put("werde", dummy);
+      m_Stopwords.put("werden", dummy);
+      m_Stopwords.put("wie", dummy);
+      m_Stopwords.put("wieder", dummy);
+      m_Stopwords.put("will", dummy);
+      m_Stopwords.put("wir", dummy);
+      m_Stopwords.put("wird", dummy);
+      m_Stopwords.put("wirst", dummy);
+      m_Stopwords.put("wo", dummy);
+      m_Stopwords.put("wollen", dummy);
+      m_Stopwords.put("wollte", dummy);
+      m_Stopwords.put("w\u00fcrde", dummy);
+      m_Stopwords.put("w\u00fcrden", dummy);
+      m_Stopwords.put("zu", dummy);
+      m_Stopwords.put("zum", dummy);
+      m_Stopwords.put("zur", dummy);
+      m_Stopwords.put("zwar", dummy);
+      m_Stopwords.put("zwischen", dummy);
+    }
+  }
+  
+  /** 
+   * Returns true if the given string is a stop word.
+   */
+  public boolean isStopword(String str) {
+
+    return m_Stopwords.containsKey(str.toLowerCase());
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Counter.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Counter.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Counter.html	(revision 21112)
@@ -0,0 +1,318 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+Counter
+</TITLE>
+<META NAME="keywords" CONTENT="Counter,Counter class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Counter";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV CLASS&nbsp;
+&nbsp;<A HREF="GermanStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Counter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class Counter</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<B>Counter</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>Counter</B><DT>extends java.lang.Object<DT>implements java.io.Serializable</DL>
+
+<P>
+Class that implements a simple counter.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="Counter">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="Counter.html#Counter()">Counter</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Initializes the counter to 1</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="Counter.html#Counter(int)">Counter</A></B>(int&nbsp;val)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Initializes the counter to the given value</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Counter.html#increment()">increment</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Increments the counter.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Counter.html#toString()">toString</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns string containing value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Counter.html#value()">value</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets the value.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="Counter()"><!-- --></A><H3>
+Counter</H3>
+<PRE>
+public <B>Counter</B>()</PRE>
+<DL>
+<DD>Initializes the counter to 1
+<P>
+</DL>
+<HR>
+
+<A NAME="Counter(int)"><!-- --></A><H3>
+Counter</H3>
+<PRE>
+public <B>Counter</B>(int&nbsp;val)</PRE>
+<DL>
+<DD>Initializes the counter to the given value
+<P>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="increment()"><!-- --></A><H3>
+increment</H3>
+<PRE>
+public void <B>increment</B>()</PRE>
+<DL>
+<DD>Increments the counter.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="value()"><!-- --></A><H3>
+value</H3>
+<PRE>
+public int <B>value</B>()</PRE>
+<DL>
+<DD>Gets the value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the value</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="toString()"><!-- --></A><H3>
+toString</H3>
+<PRE>
+public java.lang.String <B>toString</B>()</PRE>
+<DL>
+<DD>Returns string containing value.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>toString</CODE> in class <CODE>java.lang.Object</CODE></DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV CLASS&nbsp;
+&nbsp;<A HREF="GermanStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Counter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/GermanStemmer.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/GermanStemmer.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/GermanStemmer.html	(revision 21112)
@@ -0,0 +1,268 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+GermanStemmer
+</TITLE>
+<META NAME="keywords" CONTENT="GermanStemmer,GermanStemmer class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="GermanStemmer";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Counter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="IteratedLovinsStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="GermanStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class GermanStemmer</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<A HREF="Stemmer.html">Stemmer</A>
+        |
+        +--<B>GermanStemmer</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>GermanStemmer</B><DT>extends <A HREF="Stemmer.html">Stemmer</A></DL>
+
+<P>
+A stemmer for German words. The algorithm is based on the report
+ "A Fast and Simple Stemming Algorithm for German Words" by Jörg
+ Caumanns (joerg.caumanns@isst.fhg.de).
+
+ Changed stem() from protected to public.
+ Changed coding for umlaute to unicode.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="GermanStemmer">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="GermanStemmer.html#GermanStemmer()">GermanStemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="GermanStemmer.html#stem(java.lang.String)">stem</A></B>(java.lang.String&nbsp;term)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Stemms the given term to an unique <tt>discriminator</tt>.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_Stemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class <A HREF="Stemmer.html">Stemmer</A></B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><A HREF="Stemmer.html#stemString(java.lang.String)">stemString</A></CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="GermanStemmer()"><!-- --></A><H3>
+GermanStemmer</H3>
+<PRE>
+public <B>GermanStemmer</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="stem(java.lang.String)"><!-- --></A><H3>
+stem</H3>
+<PRE>
+public java.lang.String <B>stem</B>(java.lang.String&nbsp;term)</PRE>
+<DL>
+<DD>Stemms the given term to an unique <tt>discriminator</tt>.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE><A HREF="Stemmer.html#stem(java.lang.String)">stem</A></CODE> in class <CODE><A HREF="Stemmer.html">Stemmer</A></CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>term</CODE> - The term that should be stemmed.
+<DT><B>Returns:</B><DD>Discriminator for <tt>term</tt></DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Counter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="IteratedLovinsStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="GermanStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/IteratedLovinsStemmer.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/IteratedLovinsStemmer.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/IteratedLovinsStemmer.html	(revision 21112)
@@ -0,0 +1,285 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+IteratedLovinsStemmer
+</TITLE>
+<META NAME="keywords" CONTENT="IteratedLovinsStemmer,IteratedLovinsStemmer class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="IteratedLovinsStemmer";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="GermanStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="IteratedLovinsStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class IteratedLovinsStemmer</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<A HREF="Stemmer.html">Stemmer</A>
+        |
+        +--<A HREF="LovinsStemmer.html">LovinsStemmer</A>
+              |
+              +--<B>IteratedLovinsStemmer</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>IteratedLovinsStemmer</B><DT>extends <A HREF="LovinsStemmer.html">LovinsStemmer</A></DL>
+
+<P>
+Implements the iterated version of the Lovins stemmer.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="IteratedLovinsStemmer">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="IteratedLovinsStemmer.html#IteratedLovinsStemmer()">IteratedLovinsStemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="IteratedLovinsStemmer.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;ops)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Stems text coming into stdin and writes it to stdout.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="IteratedLovinsStemmer.html#stem(java.lang.String)">stem</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Iterated stemming of the given word.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_Stemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class <A HREF="Stemmer.html">Stemmer</A></B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><A HREF="Stemmer.html#stemString(java.lang.String)">stemString</A></CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="IteratedLovinsStemmer()"><!-- --></A><H3>
+IteratedLovinsStemmer</H3>
+<PRE>
+public <B>IteratedLovinsStemmer</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="stem(java.lang.String)"><!-- --></A><H3>
+stem</H3>
+<PRE>
+public java.lang.String <B>stem</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Iterated stemming of the given word.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE><A HREF="LovinsStemmer.html#stem(java.lang.String)">stem</A></CODE> in class <CODE><A HREF="LovinsStemmer.html">LovinsStemmer</A></CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>str</CODE> - a string consisting of a single word</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;ops)</PRE>
+<DL>
+<DD>Stems text coming into stdin and writes it to stdout.
+<P>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="GermanStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="IteratedLovinsStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAFilter.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAFilter.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAFilter.html	(revision 21112)
@@ -0,0 +1,1147 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+KEAFilter
+</TITLE>
+<META NAME="keywords" CONTENT="KEAFilter,KEAFilter class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="KEAFilter";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="IteratedLovinsStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAKeyphraseExtractor.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#fields_inherited_from_class_weka.filters.Filter">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class KEAFilter</H2>
+<PRE>
+java.lang.Object
+  |
+  +--weka.filters.Filter
+        |
+        +--<B>KEAFilter</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>weka.core.OptionHandler, java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>KEAFilter</B><DT>extends weka.filters.Filter<DT>implements weka.core.OptionHandler</DL>
+
+<P>
+This filter converts the incoming data into data appropriate for
+ keyphrase classification. It assumes that the dataset contains two
+ string attributes. The first attribute should contain the text of a
+ document. The second attribute should contain the keyphrases
+ associated with that document (if present). 
+
+ The filter converts every instance (i.e. document) into a set of
+ instances, one for each word-based n-gram in the document. The
+ string attribute representing the document is replaced by some
+ numeric features, the estimated probability of each n-gram being a
+ keyphrase, and the rank of this phrase in the document according to
+ the probability.  Each new instances also has a class value
+ associated with it. The class is "true" if the n-gram is a true
+ keyphrase, and "false" otherwise. Of course, if the input document
+ doesn't come with author-assigned keyphrases, the class values for
+ that document will be missing.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="KEAFilter">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+<A NAME="field_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Field Summary</B></FONT></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="fields_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Fields inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>m_NewBatch</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="KEAFilter.html#KEAFilter()">KEAFilter</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#batchFinished()">batchFinished</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Signify that this batch of input to the filter is finished.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getCheckForProperNouns()">getCheckForProperNouns</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the M_CheckProperNouns value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getDebug()">getDebug</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of Debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getDisallowInternalPeriods()">getDisallowInternalPeriods</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get whether the supplied columns are to be processed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getDocumentAtt()">getDocumentAtt</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of DocumentAtt.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getKeyphrasesAtt()">getKeyphrasesAtt</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of KeyphraseAtt.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getKFused()">getKFused</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets whether keyphrase frequency attribute is used.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getMaxPhraseLength()">getMaxPhraseLength</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MaxPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getMinNumOccur()">getMinNumOccur</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MinNumOccur.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getMinPhraseLength()">getMinPhraseLength</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MinPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String[]</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getOptions()">getOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets the current settings of the filter.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getProbabilityIndex()">getProbabilityIndex</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the index of the phrases' probabilities in the output ARFF file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getRankIndex()">getRankIndex</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the index of the phrases' ranks in the output ARFF file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getStemmedPhraseIndex()">getStemmedPhraseIndex</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the index of the stemmed phrases in the output ARFF file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;Stemmer</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getStemmer()">getStemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the Stemmer value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;Stopwords</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getStopwords()">getStopwords</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the M_Stopwords value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#getUnstemmedPhraseIndex()">getUnstemmedPhraseIndex</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the index of the unstemmed phrases in the output ARFF file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#globalInfo()">globalInfo</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns a string describing this filter</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#input(weka.core.Instance)">input</A></B>(weka.core.Instance&nbsp;instance)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Input an instance for filtering.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Enumeration</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#listOptions()">listOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns an enumeration describing the available options</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;argv)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Main method for testing this class.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setCheckForProperNouns(boolean)">setCheckForProperNouns</A></B>(boolean&nbsp;newM_CheckProperNouns)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the M_CheckProperNouns value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setDebug(boolean)">setDebug</A></B>(boolean&nbsp;newDebug)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of Debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setDisallowInternalPeriods(boolean)">setDisallowInternalPeriods</A></B>(boolean&nbsp;disallow)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set whether selected columns should be processed.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setDocumentAtt(int)">setDocumentAtt</A></B>(int&nbsp;newDocumentAtt)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of DocumentAtt.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setInputFormat(weka.core.Instances)">setInputFormat</A></B>(weka.core.Instances&nbsp;instanceInfo)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Sets the format of the input instances.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setKeyphrasesAtt(int)">setKeyphrasesAtt</A></B>(int&nbsp;newKeyphrasesAtt)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of KeyphrasesAtt.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setKFused(boolean)">setKFused</A></B>(boolean&nbsp;flag)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Sets whether keyphrase frequency attribute is used.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setMaxPhraseLength(int)">setMaxPhraseLength</A></B>(int&nbsp;newMaxPhraseLength)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MaxPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setMinNumOccur(int)">setMinNumOccur</A></B>(int&nbsp;newMinNumOccur)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MinNumOccur.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setMinPhraseLength(int)">setMinPhraseLength</A></B>(int&nbsp;newMinPhraseLength)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MinPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setOptions(java.lang.String[])">setOptions</A></B>(java.lang.String[]&nbsp;options)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Parses a given list of options controlling the behaviour of this object.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setStemmer(Stemmer)">setStemmer</A></B>(Stemmer&nbsp;newStemmer)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the Stemmer value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAFilter.html#setStopwords(Stopwords)">setStopwords</A></B>(Stopwords&nbsp;newM_Stopwords)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the M_Stopwords value.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>batchFilterFile, bufferInput, copyStringValues, copyStringValues, filterFile, flushInput, getInputFormat, getInputStringIndex, getOutputFormat, getOutputStringIndex, getStringIndices, inputFormat, isOutputFormatDefined, numPendingOutput, output, outputFormat, outputFormatPeek, outputPeek, push, resetQueue, setOutputFormat, useFilter</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="KEAFilter()"><!-- --></A><H3>
+KEAFilter</H3>
+<PRE>
+public <B>KEAFilter</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="getCheckForProperNouns()"><!-- --></A><H3>
+getCheckForProperNouns</H3>
+<PRE>
+public boolean <B>getCheckForProperNouns</B>()</PRE>
+<DL>
+<DD>Get the M_CheckProperNouns value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the M_CheckProperNouns value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setCheckForProperNouns(boolean)"><!-- --></A><H3>
+setCheckForProperNouns</H3>
+<PRE>
+public void <B>setCheckForProperNouns</B>(boolean&nbsp;newM_CheckProperNouns)</PRE>
+<DL>
+<DD>Set the M_CheckProperNouns value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newM_CheckProperNouns</CODE> - The new M_CheckProperNouns value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getStopwords()"><!-- --></A><H3>
+getStopwords</H3>
+<PRE>
+public Stopwords <B>getStopwords</B>()</PRE>
+<DL>
+<DD>Get the M_Stopwords value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the M_Stopwords value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setStopwords(Stopwords)"><!-- --></A><H3>
+setStopwords</H3>
+<PRE>
+public void <B>setStopwords</B>(Stopwords&nbsp;newM_Stopwords)</PRE>
+<DL>
+<DD>Set the M_Stopwords value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newM_Stopwords</CODE> - The new M_Stopwords value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getStemmer()"><!-- --></A><H3>
+getStemmer</H3>
+<PRE>
+public Stemmer <B>getStemmer</B>()</PRE>
+<DL>
+<DD>Get the Stemmer value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the Stemmer value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setStemmer(Stemmer)"><!-- --></A><H3>
+setStemmer</H3>
+<PRE>
+public void <B>setStemmer</B>(Stemmer&nbsp;newStemmer)</PRE>
+<DL>
+<DD>Set the Stemmer value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newStemmer</CODE> - The new Stemmer value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMinNumOccur()"><!-- --></A><H3>
+getMinNumOccur</H3>
+<PRE>
+public int <B>getMinNumOccur</B>()</PRE>
+<DL>
+<DD>Get the value of MinNumOccur.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MinNumOccur.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMinNumOccur(int)"><!-- --></A><H3>
+setMinNumOccur</H3>
+<PRE>
+public void <B>setMinNumOccur</B>(int&nbsp;newMinNumOccur)</PRE>
+<DL>
+<DD>Set the value of MinNumOccur.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMinNumOccur</CODE> - Value to assign to MinNumOccur.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMaxPhraseLength()"><!-- --></A><H3>
+getMaxPhraseLength</H3>
+<PRE>
+public int <B>getMaxPhraseLength</B>()</PRE>
+<DL>
+<DD>Get the value of MaxPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MaxPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMaxPhraseLength(int)"><!-- --></A><H3>
+setMaxPhraseLength</H3>
+<PRE>
+public void <B>setMaxPhraseLength</B>(int&nbsp;newMaxPhraseLength)</PRE>
+<DL>
+<DD>Set the value of MaxPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMaxPhraseLength</CODE> - Value to assign to MaxPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMinPhraseLength()"><!-- --></A><H3>
+getMinPhraseLength</H3>
+<PRE>
+public int <B>getMinPhraseLength</B>()</PRE>
+<DL>
+<DD>Get the value of MinPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MinPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMinPhraseLength(int)"><!-- --></A><H3>
+setMinPhraseLength</H3>
+<PRE>
+public void <B>setMinPhraseLength</B>(int&nbsp;newMinPhraseLength)</PRE>
+<DL>
+<DD>Set the value of MinPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMinPhraseLength</CODE> - Value to assign to MinPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getStemmedPhraseIndex()"><!-- --></A><H3>
+getStemmedPhraseIndex</H3>
+<PRE>
+public int <B>getStemmedPhraseIndex</B>()</PRE>
+<DL>
+<DD>Returns the index of the stemmed phrases in the output ARFF file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getUnstemmedPhraseIndex()"><!-- --></A><H3>
+getUnstemmedPhraseIndex</H3>
+<PRE>
+public int <B>getUnstemmedPhraseIndex</B>()</PRE>
+<DL>
+<DD>Returns the index of the unstemmed phrases in the output ARFF file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getProbabilityIndex()"><!-- --></A><H3>
+getProbabilityIndex</H3>
+<PRE>
+public int <B>getProbabilityIndex</B>()</PRE>
+<DL>
+<DD>Returns the index of the phrases' probabilities in the output ARFF file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getRankIndex()"><!-- --></A><H3>
+getRankIndex</H3>
+<PRE>
+public int <B>getRankIndex</B>()</PRE>
+<DL>
+<DD>Returns the index of the phrases' ranks in the output ARFF file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDocumentAtt()"><!-- --></A><H3>
+getDocumentAtt</H3>
+<PRE>
+public int <B>getDocumentAtt</B>()</PRE>
+<DL>
+<DD>Get the value of DocumentAtt.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of DocumentAtt.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDocumentAtt(int)"><!-- --></A><H3>
+setDocumentAtt</H3>
+<PRE>
+public void <B>setDocumentAtt</B>(int&nbsp;newDocumentAtt)</PRE>
+<DL>
+<DD>Set the value of DocumentAtt.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newDocumentAtt</CODE> - Value to assign to DocumentAtt.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getKeyphrasesAtt()"><!-- --></A><H3>
+getKeyphrasesAtt</H3>
+<PRE>
+public int <B>getKeyphrasesAtt</B>()</PRE>
+<DL>
+<DD>Get the value of KeyphraseAtt.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of KeyphraseAtt.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setKeyphrasesAtt(int)"><!-- --></A><H3>
+setKeyphrasesAtt</H3>
+<PRE>
+public void <B>setKeyphrasesAtt</B>(int&nbsp;newKeyphrasesAtt)</PRE>
+<DL>
+<DD>Set the value of KeyphrasesAtt.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newKeyphrasesAtt</CODE> - Value to assign to KeyphrasesAtt.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDebug()"><!-- --></A><H3>
+getDebug</H3>
+<PRE>
+public boolean <B>getDebug</B>()</PRE>
+<DL>
+<DD>Get the value of Debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of Debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDebug(boolean)"><!-- --></A><H3>
+setDebug</H3>
+<PRE>
+public void <B>setDebug</B>(boolean&nbsp;newDebug)</PRE>
+<DL>
+<DD>Set the value of Debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newDebug</CODE> - Value to assign to Debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setKFused(boolean)"><!-- --></A><H3>
+setKFused</H3>
+<PRE>
+public void <B>setKFused</B>(boolean&nbsp;flag)</PRE>
+<DL>
+<DD>Sets whether keyphrase frequency attribute is used.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getKFused()"><!-- --></A><H3>
+getKFused</H3>
+<PRE>
+public boolean <B>getKFused</B>()</PRE>
+<DL>
+<DD>Gets whether keyphrase frequency attribute is used.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDisallowInternalPeriods()"><!-- --></A><H3>
+getDisallowInternalPeriods</H3>
+<PRE>
+public boolean <B>getDisallowInternalPeriods</B>()</PRE>
+<DL>
+<DD>Get whether the supplied columns are to be processed
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if the supplied columns won't be processed</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDisallowInternalPeriods(boolean)"><!-- --></A><H3>
+setDisallowInternalPeriods</H3>
+<PRE>
+public void <B>setDisallowInternalPeriods</B>(boolean&nbsp;disallow)</PRE>
+<DL>
+<DD>Set whether selected columns should be processed. If true the 
+ selected columns won't be processed.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setOptions(java.lang.String[])"><!-- --></A><H3>
+setOptions</H3>
+<PRE>
+public void <B>setOptions</B>(java.lang.String[]&nbsp;options)
+                throws java.lang.Exception</PRE>
+<DL>
+<DD>Parses a given list of options controlling the behaviour of this object.
+ Valid options are:<p>
+
+ -K<br>
+ Specifies whether keyphrase frequency statistic is used.<p>
+
+ -M length<br>
+ Sets the maximum phrase length (default: 3).<p>
+
+ -L length<br>
+ Sets the minimum phrase length (default: 1).<p>
+
+ -D<br>
+ Turns debugging mode on.<p>
+
+ -I index<br>
+ Sets the index of the attribute containing the documents (default: 0).<p>
+
+ -J index<br>
+ Sets the index of the attribute containing the keyphrases (default: 1).<p>
+
+ -P<br>
+ Disallow internal periods <p>
+
+ -O number<br>
+ The minimum number of times a phrase needs to occur (default: 2). <p>
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>setOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>options</CODE> - the list of options as an array of strings
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if an option is not supported</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getOptions()"><!-- --></A><H3>
+getOptions</H3>
+<PRE>
+public java.lang.String[] <B>getOptions</B>()</PRE>
+<DL>
+<DD>Gets the current settings of the filter.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>getOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an array of strings suitable for passing to setOptions</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="listOptions()"><!-- --></A><H3>
+listOptions</H3>
+<PRE>
+public java.util.Enumeration <B>listOptions</B>()</PRE>
+<DL>
+<DD>Returns an enumeration describing the available options
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>listOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an enumeration of all the available options</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="globalInfo()"><!-- --></A><H3>
+globalInfo</H3>
+<PRE>
+public java.lang.String <B>globalInfo</B>()</PRE>
+<DL>
+<DD>Returns a string describing this filter
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>a description of the filter suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setInputFormat(weka.core.Instances)"><!-- --></A><H3>
+setInputFormat</H3>
+<PRE>
+public boolean <B>setInputFormat</B>(weka.core.Instances&nbsp;instanceInfo)
+                       throws java.lang.Exception</PRE>
+<DL>
+<DD>Sets the format of the input instances.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>setInputFormat</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instanceInfo</CODE> - an Instances object containing the input
+ instance structure (any instances contained in the object are
+ ignored - only the structure is required).
+<DT><B>Returns:</B><DD>true if the outputFormat may be collected immediately
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="input(weka.core.Instance)"><!-- --></A><H3>
+input</H3>
+<PRE>
+public boolean <B>input</B>(weka.core.Instance&nbsp;instance)
+              throws java.lang.Exception</PRE>
+<DL>
+<DD>Input an instance for filtering. Ordinarily the instance is processed
+ and made available for output immediately. Some filters require all
+ instances be read before producing output.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>input</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instance</CODE> - the input instance
+<DT><B>Returns:</B><DD>true if the filtered instance may now be
+ collected with output().
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if the input instance was not of the correct 
+ format or if there was a problem with the filtering.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="batchFinished()"><!-- --></A><H3>
+batchFinished</H3>
+<PRE>
+public boolean <B>batchFinished</B>()
+                      throws java.lang.Exception</PRE>
+<DL>
+<DD>Signify that this batch of input to the filter is finished. 
+ If the filter requires all instances prior to filtering,
+ output() may now be called to retrieve the filtered instances.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>batchFinished</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if there are instances pending output
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if no input structure has been defined</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;argv)</PRE>
+<DL>
+<DD>Main method for testing this class.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>argv</CODE> - should contain arguments to the filter: use -h for help</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="IteratedLovinsStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAKeyphraseExtractor.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#fields_inherited_from_class_weka.filters.Filter">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAKeyphraseExtractor.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAKeyphraseExtractor.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAKeyphraseExtractor.html	(revision 21112)
@@ -0,0 +1,742 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+KEAKeyphraseExtractor
+</TITLE>
+<META NAME="keywords" CONTENT="KEAKeyphraseExtractor,KEAKeyphraseExtractor class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="KEAKeyphraseExtractor";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAModelBuilder.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAKeyphraseExtractor.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class KEAKeyphraseExtractor</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<B>KEAKeyphraseExtractor</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>weka.core.OptionHandler</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>KEAKeyphraseExtractor</B><DT>extends java.lang.Object<DT>implements weka.core.OptionHandler</DL>
+
+<P>
+Extracts keyphrases from the documents in a given directory.
+ Assumes that the file names for the documents end with ".txt".
+ Puts extracted keyphrases into corresponding files ending with
+ ".key" (if those are not already present). Optionally an encoding
+ for the documents/keyphrases can be defined (e.g. for Chinese
+ text). Documents for which ".key" exists, are used for evaluation.
+
+ Valid options are:<p>
+
+ -l "directory name"<br>
+ Specifies name of directory.<p>
+
+ -m "model name"<br>
+ Specifies name of model.<p>
+
+ -e "encoding"<br>
+ Specifies encoding.<p>
+
+ -n <br>
+ Specifies number of phrases to be output (default: 5).<p>
+
+ -d<br>
+ Turns debugging mode on.<p>
+
+ -a<br>
+ Also write stemmed phrase and score into ".key" file.<p>
+<P>
+
+<P>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#KEAKeyphraseExtractor()">KEAKeyphraseExtractor</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Hashtable</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#collectStems()">collectStems</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Collects the stems of the file names.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#extractKeyphrases(java.util.Hashtable)">extractKeyphrases</A></B>(java.util.Hashtable&nbsp;stems)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Builds the model from the files</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getAdditionalInfo()">getAdditionalInfo</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of AdditionalInfo.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getDebug()">getDebug</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getDirName()">getDirName</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of dirName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getEncoding()">getEncoding</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of encoding.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getModelName()">getModelName</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of modelName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getNumPhrases()">getNumPhrases</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of numPhrases.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String[]</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#getOptions()">getOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets the current option settings.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Enumeration</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#listOptions()">listOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns an enumeration describing the available options.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#loadModel()">loadModel</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Loads the extraction model from the file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;ops)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;The main method.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setAdditionalInfo(boolean)">setAdditionalInfo</A></B>(boolean&nbsp;newAdditionalInfo)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of AdditionalInfo.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setDebug(boolean)">setDebug</A></B>(boolean&nbsp;newdebug)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setDirName(java.lang.String)">setDirName</A></B>(java.lang.String&nbsp;newdirName)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of dirName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setEncoding(java.lang.String)">setEncoding</A></B>(java.lang.String&nbsp;newencoding)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of encoding.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setModelName(java.lang.String)">setModelName</A></B>(java.lang.String&nbsp;newmodelName)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of modelName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setNumPhrases(int)">setNumPhrases</A></B>(int&nbsp;newnumPhrases)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of numPhrases.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAKeyphraseExtractor.html#setOptions(java.lang.String[])">setOptions</A></B>(java.lang.String[]&nbsp;options)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Parses a given list of options controlling the behaviour of this object.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="KEAKeyphraseExtractor()"><!-- --></A><H3>
+KEAKeyphraseExtractor</H3>
+<PRE>
+public <B>KEAKeyphraseExtractor</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="getAdditionalInfo()"><!-- --></A><H3>
+getAdditionalInfo</H3>
+<PRE>
+public boolean <B>getAdditionalInfo</B>()</PRE>
+<DL>
+<DD>Get the value of AdditionalInfo.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of AdditionalInfo.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setAdditionalInfo(boolean)"><!-- --></A><H3>
+setAdditionalInfo</H3>
+<PRE>
+public void <B>setAdditionalInfo</B>(boolean&nbsp;newAdditionalInfo)</PRE>
+<DL>
+<DD>Set the value of AdditionalInfo.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newAdditionalInfo</CODE> - Value to assign to AdditionalInfo.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getNumPhrases()"><!-- --></A><H3>
+getNumPhrases</H3>
+<PRE>
+public int <B>getNumPhrases</B>()</PRE>
+<DL>
+<DD>Get the value of numPhrases.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of numPhrases.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setNumPhrases(int)"><!-- --></A><H3>
+setNumPhrases</H3>
+<PRE>
+public void <B>setNumPhrases</B>(int&nbsp;newnumPhrases)</PRE>
+<DL>
+<DD>Set the value of numPhrases.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newnumPhrases</CODE> - Value to assign to numPhrases.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDebug()"><!-- --></A><H3>
+getDebug</H3>
+<PRE>
+public boolean <B>getDebug</B>()</PRE>
+<DL>
+<DD>Get the value of debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDebug(boolean)"><!-- --></A><H3>
+setDebug</H3>
+<PRE>
+public void <B>setDebug</B>(boolean&nbsp;newdebug)</PRE>
+<DL>
+<DD>Set the value of debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newdebug</CODE> - Value to assign to debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getEncoding()"><!-- --></A><H3>
+getEncoding</H3>
+<PRE>
+public java.lang.String <B>getEncoding</B>()</PRE>
+<DL>
+<DD>Get the value of encoding.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of encoding.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setEncoding(java.lang.String)"><!-- --></A><H3>
+setEncoding</H3>
+<PRE>
+public void <B>setEncoding</B>(java.lang.String&nbsp;newencoding)</PRE>
+<DL>
+<DD>Set the value of encoding.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newencoding</CODE> - Value to assign to encoding.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getModelName()"><!-- --></A><H3>
+getModelName</H3>
+<PRE>
+public java.lang.String <B>getModelName</B>()</PRE>
+<DL>
+<DD>Get the value of modelName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of modelName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setModelName(java.lang.String)"><!-- --></A><H3>
+setModelName</H3>
+<PRE>
+public void <B>setModelName</B>(java.lang.String&nbsp;newmodelName)</PRE>
+<DL>
+<DD>Set the value of modelName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newmodelName</CODE> - Value to assign to modelName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDirName()"><!-- --></A><H3>
+getDirName</H3>
+<PRE>
+public java.lang.String <B>getDirName</B>()</PRE>
+<DL>
+<DD>Get the value of dirName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of dirName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDirName(java.lang.String)"><!-- --></A><H3>
+setDirName</H3>
+<PRE>
+public void <B>setDirName</B>(java.lang.String&nbsp;newdirName)</PRE>
+<DL>
+<DD>Set the value of dirName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newdirName</CODE> - Value to assign to dirName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setOptions(java.lang.String[])"><!-- --></A><H3>
+setOptions</H3>
+<PRE>
+public void <B>setOptions</B>(java.lang.String[]&nbsp;options)
+                throws java.lang.Exception</PRE>
+<DL>
+<DD>Parses a given list of options controlling the behaviour of this object.
+ Valid options are:<p>
+
+ -l "directory name"<br>
+ Specifies name of directory.<p>
+
+ -m "model name"<br>
+ Specifies name of model.<p>
+
+ -e "encoding"<br>
+ Specifies encoding.<p>
+
+ -n<br>
+ Specifies number of phrases to be output (default: 5).<p>
+
+ -d<br>
+ Turns debugging mode on.<p>
+
+ -a<br>
+ Also write stemmed phrase and score into ".key" file.<p>
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>setOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>options</CODE> - the list of options as an array of strings
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if an option is not supported</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getOptions()"><!-- --></A><H3>
+getOptions</H3>
+<PRE>
+public java.lang.String[] <B>getOptions</B>()</PRE>
+<DL>
+<DD>Gets the current option settings.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>getOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an array of strings suitable for passing to setOptions</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="listOptions()"><!-- --></A><H3>
+listOptions</H3>
+<PRE>
+public java.util.Enumeration <B>listOptions</B>()</PRE>
+<DL>
+<DD>Returns an enumeration describing the available options.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>listOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an enumeration of all the available options</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="collectStems()"><!-- --></A><H3>
+collectStems</H3>
+<PRE>
+public java.util.Hashtable <B>collectStems</B>()
+                                 throws java.lang.Exception</PRE>
+<DL>
+<DD>Collects the stems of the file names.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="extractKeyphrases(java.util.Hashtable)"><!-- --></A><H3>
+extractKeyphrases</H3>
+<PRE>
+public void <B>extractKeyphrases</B>(java.util.Hashtable&nbsp;stems)
+                       throws java.lang.Exception</PRE>
+<DL>
+<DD>Builds the model from the files
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="loadModel()"><!-- --></A><H3>
+loadModel</H3>
+<PRE>
+public void <B>loadModel</B>()
+               throws java.lang.Exception</PRE>
+<DL>
+<DD>Loads the extraction model from the file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;ops)</PRE>
+<DL>
+<DD>The main method.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAModelBuilder.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAKeyphraseExtractor.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAModelBuilder.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAModelBuilder.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAModelBuilder.html	(revision 21112)
@@ -0,0 +1,1072 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+KEAModelBuilder
+</TITLE>
+<META NAME="keywords" CONTENT="KEAModelBuilder,KEAModelBuilder class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="KEAModelBuilder";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAKeyphraseExtractor.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAPhraseFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAModelBuilder.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class KEAModelBuilder</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<B>KEAModelBuilder</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>weka.core.OptionHandler</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>KEAModelBuilder</B><DT>extends java.lang.Object<DT>implements weka.core.OptionHandler</DL>
+
+<P>
+Builds a keyphrase extraction model from the documents in a given
+ directory.  Assumes that the file names for the documents end with
+ ".txt".  Assumes that files containing corresponding
+ author-assigned keyphrases end with ".key". Optionally an encoding
+ for the documents/keyphrases can be defined (e.g. for Chinese
+ text).
+
+ Valid options are:<p>
+
+ -l "directory name"<br>
+ Specifies name of directory.<p>
+
+ -m "model name"<br>
+ Specifies name of model.<p>
+
+ -e "encoding"<br>
+ Specifies encoding.<p>
+
+ -d<br>
+ Turns debugging mode on.<p>
+
+ -k<br>
+ Use keyphrase frequency statistic.<p>
+
+ -p<br>
+ Disallow internal periods.<p>
+
+ -x "length"<br>
+ Sets maximum phrase length (default: 3).<p>
+
+ -y "length"<br>
+ Sets minimum phrase length (default: 1).<p>
+
+ -o "number"<br>
+ The minimum number of times a phrase needs to occur (default: 2). <p>
+
+ -s "name of class implementing list of stop words"<br>
+ Sets list of stop words to used (default: StopwordsEnglish).<p>
+
+ -t "name of class implementing stemmer"<br>
+ Sets stemmer to use (default: IteratedLovinsStemmer). <p>
+
+ -n<br>
+ Do not check for proper nouns. <p>
+<P>
+
+<P>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="KEAModelBuilder.html#KEAModelBuilder()">KEAModelBuilder</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#buildModel(java.util.Hashtable)">buildModel</A></B>(java.util.Hashtable&nbsp;stems)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Builds the model from the files</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Hashtable</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#collectStems()">collectStems</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Collects the stems of the file names.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getCheckForProperNouns()">getCheckForProperNouns</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the M_CheckProperNouns value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getDebug()">getDebug</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getDirName()">getDirName</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of dirName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getDisallowIPeriods()">getDisallowIPeriods</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of disallowIPeriods.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getEncoding()">getEncoding</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of encoding.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getMaxPhraseLength()">getMaxPhraseLength</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MaxPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getMinNumOccur()">getMinNumOccur</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MinNumOccur.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;int</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getMinPhraseLength()">getMinPhraseLength</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of MinPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getModelName()">getModelName</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of modelName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String[]</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getOptions()">getOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets the current option settings.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;Stemmer</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getStemmer()">getStemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the Stemmer value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;Stopwords</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getStopwords()">getStopwords</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the M_Stopwords value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#getUseKFrequency()">getUseKFrequency</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the value of useKFrequency.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Enumeration</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#listOptions()">listOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns an enumeration describing the available options.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;ops)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;The main method.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#saveModel()">saveModel</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Saves the extraction model to the file.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setCheckForProperNouns(boolean)">setCheckForProperNouns</A></B>(boolean&nbsp;newM_CheckProperNouns)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the M_CheckProperNouns value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setDebug(boolean)">setDebug</A></B>(boolean&nbsp;newdebug)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of debug.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setDirName(java.lang.String)">setDirName</A></B>(java.lang.String&nbsp;newdirName)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of dirName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setDisallowIPeriods(boolean)">setDisallowIPeriods</A></B>(boolean&nbsp;newdisallowIPeriods)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of disallowIPeriods.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setEncoding(java.lang.String)">setEncoding</A></B>(java.lang.String&nbsp;newencoding)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of encoding.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setMaxPhraseLength(int)">setMaxPhraseLength</A></B>(int&nbsp;newMaxPhraseLength)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MaxPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setMinNumOccur(int)">setMinNumOccur</A></B>(int&nbsp;newMinNumOccur)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MinNumOccur.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setMinPhraseLength(int)">setMinPhraseLength</A></B>(int&nbsp;newMinPhraseLength)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of MinPhraseLength.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setModelName(java.lang.String)">setModelName</A></B>(java.lang.String&nbsp;newmodelName)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of modelName.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setOptions(java.lang.String[])">setOptions</A></B>(java.lang.String[]&nbsp;options)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Parses a given list of options controlling the behaviour of this object.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setStemmer(Stemmer)">setStemmer</A></B>(Stemmer&nbsp;newStemmer)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the Stemmer value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setStopwords(Stopwords)">setStopwords</A></B>(Stopwords&nbsp;newM_Stopwords)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the M_Stopwords value.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAModelBuilder.html#setUseKFrequency(boolean)">setUseKFrequency</A></B>(boolean&nbsp;newuseKFrequency)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set the value of useKFrequency.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="KEAModelBuilder()"><!-- --></A><H3>
+KEAModelBuilder</H3>
+<PRE>
+public <B>KEAModelBuilder</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="getCheckForProperNouns()"><!-- --></A><H3>
+getCheckForProperNouns</H3>
+<PRE>
+public boolean <B>getCheckForProperNouns</B>()</PRE>
+<DL>
+<DD>Get the M_CheckProperNouns value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the M_CheckProperNouns value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setCheckForProperNouns(boolean)"><!-- --></A><H3>
+setCheckForProperNouns</H3>
+<PRE>
+public void <B>setCheckForProperNouns</B>(boolean&nbsp;newM_CheckProperNouns)</PRE>
+<DL>
+<DD>Set the M_CheckProperNouns value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newM_CheckProperNouns</CODE> - The new M_CheckProperNouns value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getStopwords()"><!-- --></A><H3>
+getStopwords</H3>
+<PRE>
+public Stopwords <B>getStopwords</B>()</PRE>
+<DL>
+<DD>Get the M_Stopwords value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the M_Stopwords value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setStopwords(Stopwords)"><!-- --></A><H3>
+setStopwords</H3>
+<PRE>
+public void <B>setStopwords</B>(Stopwords&nbsp;newM_Stopwords)</PRE>
+<DL>
+<DD>Set the M_Stopwords value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newM_Stopwords</CODE> - The new M_Stopwords value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getStemmer()"><!-- --></A><H3>
+getStemmer</H3>
+<PRE>
+public Stemmer <B>getStemmer</B>()</PRE>
+<DL>
+<DD>Get the Stemmer value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>the Stemmer value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setStemmer(Stemmer)"><!-- --></A><H3>
+setStemmer</H3>
+<PRE>
+public void <B>setStemmer</B>(Stemmer&nbsp;newStemmer)</PRE>
+<DL>
+<DD>Set the Stemmer value.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newStemmer</CODE> - The new Stemmer value.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMinNumOccur()"><!-- --></A><H3>
+getMinNumOccur</H3>
+<PRE>
+public int <B>getMinNumOccur</B>()</PRE>
+<DL>
+<DD>Get the value of MinNumOccur.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MinNumOccur.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMinNumOccur(int)"><!-- --></A><H3>
+setMinNumOccur</H3>
+<PRE>
+public void <B>setMinNumOccur</B>(int&nbsp;newMinNumOccur)</PRE>
+<DL>
+<DD>Set the value of MinNumOccur.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMinNumOccur</CODE> - Value to assign to MinNumOccur.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMaxPhraseLength()"><!-- --></A><H3>
+getMaxPhraseLength</H3>
+<PRE>
+public int <B>getMaxPhraseLength</B>()</PRE>
+<DL>
+<DD>Get the value of MaxPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MaxPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMaxPhraseLength(int)"><!-- --></A><H3>
+setMaxPhraseLength</H3>
+<PRE>
+public void <B>setMaxPhraseLength</B>(int&nbsp;newMaxPhraseLength)</PRE>
+<DL>
+<DD>Set the value of MaxPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMaxPhraseLength</CODE> - Value to assign to MaxPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getMinPhraseLength()"><!-- --></A><H3>
+getMinPhraseLength</H3>
+<PRE>
+public int <B>getMinPhraseLength</B>()</PRE>
+<DL>
+<DD>Get the value of MinPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of MinPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setMinPhraseLength(int)"><!-- --></A><H3>
+setMinPhraseLength</H3>
+<PRE>
+public void <B>setMinPhraseLength</B>(int&nbsp;newMinPhraseLength)</PRE>
+<DL>
+<DD>Set the value of MinPhraseLength.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newMinPhraseLength</CODE> - Value to assign to MinPhraseLength.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDisallowIPeriods()"><!-- --></A><H3>
+getDisallowIPeriods</H3>
+<PRE>
+public boolean <B>getDisallowIPeriods</B>()</PRE>
+<DL>
+<DD>Get the value of disallowIPeriods.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of disallowIPeriods.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDisallowIPeriods(boolean)"><!-- --></A><H3>
+setDisallowIPeriods</H3>
+<PRE>
+public void <B>setDisallowIPeriods</B>(boolean&nbsp;newdisallowIPeriods)</PRE>
+<DL>
+<DD>Set the value of disallowIPeriods.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newdisallowIPeriods</CODE> - Value to assign to disallowIPeriods.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getUseKFrequency()"><!-- --></A><H3>
+getUseKFrequency</H3>
+<PRE>
+public boolean <B>getUseKFrequency</B>()</PRE>
+<DL>
+<DD>Get the value of useKFrequency.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of useKFrequency.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setUseKFrequency(boolean)"><!-- --></A><H3>
+setUseKFrequency</H3>
+<PRE>
+public void <B>setUseKFrequency</B>(boolean&nbsp;newuseKFrequency)</PRE>
+<DL>
+<DD>Set the value of useKFrequency.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newuseKFrequency</CODE> - Value to assign to useKFrequency.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDebug()"><!-- --></A><H3>
+getDebug</H3>
+<PRE>
+public boolean <B>getDebug</B>()</PRE>
+<DL>
+<DD>Get the value of debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDebug(boolean)"><!-- --></A><H3>
+setDebug</H3>
+<PRE>
+public void <B>setDebug</B>(boolean&nbsp;newdebug)</PRE>
+<DL>
+<DD>Set the value of debug.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newdebug</CODE> - Value to assign to debug.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getEncoding()"><!-- --></A><H3>
+getEncoding</H3>
+<PRE>
+public java.lang.String <B>getEncoding</B>()</PRE>
+<DL>
+<DD>Get the value of encoding.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of encoding.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setEncoding(java.lang.String)"><!-- --></A><H3>
+setEncoding</H3>
+<PRE>
+public void <B>setEncoding</B>(java.lang.String&nbsp;newencoding)</PRE>
+<DL>
+<DD>Set the value of encoding.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newencoding</CODE> - Value to assign to encoding.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getModelName()"><!-- --></A><H3>
+getModelName</H3>
+<PRE>
+public java.lang.String <B>getModelName</B>()</PRE>
+<DL>
+<DD>Get the value of modelName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of modelName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setModelName(java.lang.String)"><!-- --></A><H3>
+setModelName</H3>
+<PRE>
+public void <B>setModelName</B>(java.lang.String&nbsp;newmodelName)</PRE>
+<DL>
+<DD>Set the value of modelName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newmodelName</CODE> - Value to assign to modelName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDirName()"><!-- --></A><H3>
+getDirName</H3>
+<PRE>
+public java.lang.String <B>getDirName</B>()</PRE>
+<DL>
+<DD>Get the value of dirName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>Value of dirName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDirName(java.lang.String)"><!-- --></A><H3>
+setDirName</H3>
+<PRE>
+public void <B>setDirName</B>(java.lang.String&nbsp;newdirName)</PRE>
+<DL>
+<DD>Set the value of dirName.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>newdirName</CODE> - Value to assign to dirName.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setOptions(java.lang.String[])"><!-- --></A><H3>
+setOptions</H3>
+<PRE>
+public void <B>setOptions</B>(java.lang.String[]&nbsp;options)
+                throws java.lang.Exception</PRE>
+<DL>
+<DD>Parses a given list of options controlling the behaviour of this object.
+ Valid options are:<p>
+
+ -l "directory name" <br>
+ Specifies name of directory.<p>
+
+ -m "model name" <br>
+ Specifies name of model.<p>
+
+ -e "encoding" <br>
+ Specifies encoding.<p>
+
+ -d<br>
+ Turns debugging mode on.<p>
+
+ -k<br>
+ Use keyphrase frequency statistic.<p>
+
+ -p<br>
+ Disallow internal periods. <p>
+
+ -x "length"<br>
+ Sets maximum phrase length (default: 3).<p>
+
+ -y "length"<br>
+ Sets minimum phrase length (default: 3).<p>
+
+ -o "number"<br>
+ The minimum number of times a phrase needs to occur (default: 2). <p>
+
+ -s "name of class implementing list of stop words"<br>
+ Sets list of stop words to used (default: StopwordsEnglish).<p>
+
+ -t "name of class implementing stemmer"<br>
+ Sets stemmer to use (default: IteratedLovinsStemmer). <p>
+
+ -n<br>
+ Do not check for proper nouns. <p>
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>setOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>options</CODE> - the list of options as an array of strings
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if an option is not supported</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getOptions()"><!-- --></A><H3>
+getOptions</H3>
+<PRE>
+public java.lang.String[] <B>getOptions</B>()</PRE>
+<DL>
+<DD>Gets the current option settings.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>getOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an array of strings suitable for passing to setOptions</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="listOptions()"><!-- --></A><H3>
+listOptions</H3>
+<PRE>
+public java.util.Enumeration <B>listOptions</B>()</PRE>
+<DL>
+<DD>Returns an enumeration describing the available options.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>listOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an enumeration of all the available options</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="collectStems()"><!-- --></A><H3>
+collectStems</H3>
+<PRE>
+public java.util.Hashtable <B>collectStems</B>()
+                                 throws java.lang.Exception</PRE>
+<DL>
+<DD>Collects the stems of the file names.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="buildModel(java.util.Hashtable)"><!-- --></A><H3>
+buildModel</H3>
+<PRE>
+public void <B>buildModel</B>(java.util.Hashtable&nbsp;stems)
+                throws java.lang.Exception</PRE>
+<DL>
+<DD>Builds the model from the files
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="saveModel()"><!-- --></A><H3>
+saveModel</H3>
+<PRE>
+public void <B>saveModel</B>()
+               throws java.lang.Exception</PRE>
+<DL>
+<DD>Saves the extraction model to the file.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;ops)</PRE>
+<DL>
+<DD>The main method.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAKeyphraseExtractor.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="KEAPhraseFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAModelBuilder.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAPhraseFilter.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAPhraseFilter.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/KEAPhraseFilter.html	(revision 21112)
@@ -0,0 +1,817 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:49 NZDT 2004 -->
+<TITLE>
+KEAPhraseFilter
+</TITLE>
+<META NAME="keywords" CONTENT="KEAPhraseFilter,KEAPhraseFilter class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="KEAPhraseFilter";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAModelBuilder.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="LovinsStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAPhraseFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#field_summary">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;<A HREF="#field_detail">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class KEAPhraseFilter</H2>
+<PRE>
+java.lang.Object
+  |
+  +--weka.filters.Filter
+        |
+        +--<B>KEAPhraseFilter</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>weka.core.OptionHandler, java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>KEAPhraseFilter</B><DT>extends weka.filters.Filter<DT>implements weka.core.OptionHandler</DL>
+
+<P>
+This filter splits the text in selected string
+ attributes into phrases. The resulting
+ string attributes contain these phrases
+ separated by '\n' characters.
+
+ Phrases are identified according to the
+ following definitions:
+ 
+ A phrase is a sequence of words interrupted
+ only by sequences of whitespace characters,
+ where each sequence of whitespace characters
+ contains at most one '\n'.
+
+ A word is a sequence of letters or digits
+ that contains at least one letter, with
+ the following exceptions:
+
+ a) '.', '@', '_', '&', '/', '-' are allowed
+ if surrounded by letters or digits,
+
+ b) '\'' is allowed if preceeded by a letter
+ or digit,
+ 
+ c) '-', '/' are also allowed if succeeded by
+ whitespace characters followed by another
+ word. In that case the whitespace characters
+ will be deleted.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="KEAPhraseFilter">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+<A NAME="field_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Field Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>protected &nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#m_DisallowInternalPeriods">m_DisallowInternalPeriods</A></B></CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Determines whether internal periods are allowed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>protected &nbsp;weka.core.Range</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#m_SelectCols">m_SelectCols</A></B></CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Stores which columns to select as a funky range</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="fields_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Fields inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>m_NewBatch</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#KEAPhraseFilter()">KEAPhraseFilter</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#attributeIndicesTipText()">attributeIndicesTipText</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the tip text for this property</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#batchFinished()">batchFinished</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Signify that this batch of input to the filter is finished.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#disallowInternalPeriodsTipText()">disallowInternalPeriodsTipText</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the tip text for this property</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#getAttributeIndices()">getAttributeIndices</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get the current range selection.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#getDisallowInternalPeriods()">getDisallowInternalPeriods</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get whether the supplied columns are to be processed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#getInvertSelection()">getInvertSelection</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Get whether the supplied columns are to be processed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String[]</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#getOptions()">getOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Gets the current settings of the filter.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#globalInfo()">globalInfo</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns a string describing this filter</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#input(weka.core.Instance)">input</A></B>(weka.core.Instance&nbsp;instance)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Input an instance for filtering.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#invertSelectionTipText()">invertSelectionTipText</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the tip text for this property</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.util.Enumeration</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#listOptions()">listOptions</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns an enumeration describing the available options</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;argv)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Main method for testing this class.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setAttributeIndices(java.lang.String)">setAttributeIndices</A></B>(java.lang.String&nbsp;rangeList)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set which attributes are to be processed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setAttributeIndicesArray(int[])">setAttributeIndicesArray</A></B>(int[]&nbsp;attributes)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set which attributes are to be processed</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setDisallowInternalPeriods(boolean)">setDisallowInternalPeriods</A></B>(boolean&nbsp;disallow)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set whether selected columns should be processed.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setInputFormat(weka.core.Instances)">setInputFormat</A></B>(weka.core.Instances&nbsp;instanceInfo)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Sets the format of the input instances.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setInvertSelection(boolean)">setInvertSelection</A></B>(boolean&nbsp;invert)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Set whether selected columns should be processed.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="KEAPhraseFilter.html#setOptions(java.lang.String[])">setOptions</A></B>(java.lang.String[]&nbsp;options)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Parses a given list of options controlling the behaviour of this object.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>batchFilterFile, bufferInput, copyStringValues, copyStringValues, filterFile, flushInput, getInputFormat, getInputStringIndex, getOutputFormat, getOutputStringIndex, getStringIndices, inputFormat, isOutputFormatDefined, numPendingOutput, output, outputFormat, outputFormatPeek, outputPeek, push, resetQueue, setOutputFormat, useFilter</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+<A NAME="field_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Field Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="m_SelectCols"><!-- --></A><H3>
+m_SelectCols</H3>
+<PRE>
+protected weka.core.Range <B>m_SelectCols</B></PRE>
+<DL>
+<DD>Stores which columns to select as a funky range
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<A NAME="m_DisallowInternalPeriods"><!-- --></A><H3>
+m_DisallowInternalPeriods</H3>
+<PRE>
+protected boolean <B>m_DisallowInternalPeriods</B></PRE>
+<DL>
+<DD>Determines whether internal periods are allowed
+<P>
+<DL>
+</DL>
+</DL>
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="KEAPhraseFilter()"><!-- --></A><H3>
+KEAPhraseFilter</H3>
+<PRE>
+public <B>KEAPhraseFilter</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="globalInfo()"><!-- --></A><H3>
+globalInfo</H3>
+<PRE>
+public java.lang.String <B>globalInfo</B>()</PRE>
+<DL>
+<DD>Returns a string describing this filter
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>a description of the filter suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="listOptions()"><!-- --></A><H3>
+listOptions</H3>
+<PRE>
+public java.util.Enumeration <B>listOptions</B>()</PRE>
+<DL>
+<DD>Returns an enumeration describing the available options
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>listOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an enumeration of all the available options</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setOptions(java.lang.String[])"><!-- --></A><H3>
+setOptions</H3>
+<PRE>
+public void <B>setOptions</B>(java.lang.String[]&nbsp;options)
+                throws java.lang.Exception</PRE>
+<DL>
+<DD>Parses a given list of options controlling the behaviour of this object.
+ Valid options are:<p>
+
+ -R index1,index2-index4,...<br>
+ Specify list of attributes to process. First and last are valid indexes.
+ (default none)<p>
+
+ -V<br>
+ Invert matching sense <p>
+
+ -P<br>
+ Disallow internal periods <p>
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>setOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>options</CODE> - the list of options as an array of strings
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if an option is not supported</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getOptions()"><!-- --></A><H3>
+getOptions</H3>
+<PRE>
+public java.lang.String[] <B>getOptions</B>()</PRE>
+<DL>
+<DD>Gets the current settings of the filter.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE>getOptions</CODE> in interface <CODE>weka.core.OptionHandler</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>an array of strings suitable for passing to setOptions</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setInputFormat(weka.core.Instances)"><!-- --></A><H3>
+setInputFormat</H3>
+<PRE>
+public boolean <B>setInputFormat</B>(weka.core.Instances&nbsp;instanceInfo)
+                       throws java.lang.Exception</PRE>
+<DL>
+<DD>Sets the format of the input instances.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>setInputFormat</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instanceInfo</CODE> - an Instances object containing the input
+ instance structure (any instances contained in the object are
+ ignored - only the structure is required).
+<DT><B>Returns:</B><DD>true if the outputFormat may be collected immediately
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="input(weka.core.Instance)"><!-- --></A><H3>
+input</H3>
+<PRE>
+public boolean <B>input</B>(weka.core.Instance&nbsp;instance)
+              throws java.lang.Exception</PRE>
+<DL>
+<DD>Input an instance for filtering. Ordinarily the instance is processed
+ and made available for output immediately. Some filters require all
+ instances be read before producing output.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>input</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instance</CODE> - the input instance
+<DT><B>Returns:</B><DD>true if the filtered instance may now be
+ collected with output().
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if the input instance was not of the correct 
+ format or if there was a problem with the filtering.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="batchFinished()"><!-- --></A><H3>
+batchFinished</H3>
+<PRE>
+public boolean <B>batchFinished</B>()
+                      throws java.lang.Exception</PRE>
+<DL>
+<DD>Signify that this batch of input to the filter is finished. If
+ the filter requires all instances prior to filtering, output()
+ may now be called to retrieve the filtered instances. Any
+ subsequent instances filtered should be filtered based on setting
+ obtained from the first batch (unless the inputFormat has been
+ re-assigned or new options have been set). This default
+ implementation assumes all instance processing occurs during
+ inputFormat() and input().
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>batchFinished</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if there are instances pending output
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.NullPointerException</CODE> - if no input structure has been defined,
+<DD><CODE>java.lang.Exception</CODE> - if there was a problem finishing the batch.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;argv)</PRE>
+<DL>
+<DD>Main method for testing this class.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>argv</CODE> - should contain arguments to the filter: use -h for help</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="invertSelectionTipText()"><!-- --></A><H3>
+invertSelectionTipText</H3>
+<PRE>
+public java.lang.String <B>invertSelectionTipText</B>()</PRE>
+<DL>
+<DD>Returns the tip text for this property
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>tip text for this property suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getInvertSelection()"><!-- --></A><H3>
+getInvertSelection</H3>
+<PRE>
+public boolean <B>getInvertSelection</B>()</PRE>
+<DL>
+<DD>Get whether the supplied columns are to be processed
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if the supplied columns won't be processed</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setInvertSelection(boolean)"><!-- --></A><H3>
+setInvertSelection</H3>
+<PRE>
+public void <B>setInvertSelection</B>(boolean&nbsp;invert)</PRE>
+<DL>
+<DD>Set whether selected columns should be processed. If true the 
+ selected columns won't be processed.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>invert</CODE> - the new invert setting</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="disallowInternalPeriodsTipText()"><!-- --></A><H3>
+disallowInternalPeriodsTipText</H3>
+<PRE>
+public java.lang.String <B>disallowInternalPeriodsTipText</B>()</PRE>
+<DL>
+<DD>Returns the tip text for this property
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>tip text for this property suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getDisallowInternalPeriods()"><!-- --></A><H3>
+getDisallowInternalPeriods</H3>
+<PRE>
+public boolean <B>getDisallowInternalPeriods</B>()</PRE>
+<DL>
+<DD>Get whether the supplied columns are to be processed
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if the supplied columns won't be processed</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setDisallowInternalPeriods(boolean)"><!-- --></A><H3>
+setDisallowInternalPeriods</H3>
+<PRE>
+public void <B>setDisallowInternalPeriods</B>(boolean&nbsp;disallow)</PRE>
+<DL>
+<DD>Set whether selected columns should be processed. If true the 
+ selected columns won't be processed.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="attributeIndicesTipText()"><!-- --></A><H3>
+attributeIndicesTipText</H3>
+<PRE>
+public java.lang.String <B>attributeIndicesTipText</B>()</PRE>
+<DL>
+<DD>Returns the tip text for this property
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>tip text for this property suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="getAttributeIndices()"><!-- --></A><H3>
+getAttributeIndices</H3>
+<PRE>
+public java.lang.String <B>getAttributeIndices</B>()</PRE>
+<DL>
+<DD>Get the current range selection.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>a string containing a comma separated list of ranges</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setAttributeIndices(java.lang.String)"><!-- --></A><H3>
+setAttributeIndices</H3>
+<PRE>
+public void <B>setAttributeIndices</B>(java.lang.String&nbsp;rangeList)</PRE>
+<DL>
+<DD>Set which attributes are to be processed
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>rangeList</CODE> - a string representing the list of attributes.  Since
+ the string will typically come from a user, attributes are indexed from
+ 1. <br>
+ eg: first-3,5,6-last</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setAttributeIndicesArray(int[])"><!-- --></A><H3>
+setAttributeIndicesArray</H3>
+<PRE>
+public void <B>setAttributeIndicesArray</B>(int[]&nbsp;attributes)</PRE>
+<DL>
+<DD>Set which attributes are to be processed
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>attributes</CODE> - an array containing indexes of attributes to select.
+ Since the array will typically come from a program, attributes are indexed
+ from 0.</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAModelBuilder.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="LovinsStemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="KEAPhraseFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#field_summary">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;<A HREF="#field_detail">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/LovinsStemmer.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/LovinsStemmer.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/LovinsStemmer.html	(revision 21112)
@@ -0,0 +1,289 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+LovinsStemmer
+</TITLE>
+<META NAME="keywords" CONTENT="LovinsStemmer,LovinsStemmer class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="LovinsStemmer";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAPhraseFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="NumbersFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="LovinsStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class LovinsStemmer</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<A HREF="Stemmer.html">Stemmer</A>
+        |
+        +--<B>LovinsStemmer</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<DL>
+<DT><B>Direct Known Subclasses:</B> <DD><A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A></DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>LovinsStemmer</B><DT>extends <A HREF="Stemmer.html">Stemmer</A><DT>implements java.io.Serializable</DL>
+
+<P>
+Implements the Lovins stemmer.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="LovinsStemmer">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="LovinsStemmer.html#LovinsStemmer()">LovinsStemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="LovinsStemmer.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;ops)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Stems text coming into stdin and writes it to stdout.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="LovinsStemmer.html#stem(java.lang.String)">stem</A></B>(java.lang.String&nbsp;word)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns the stemmed version of the given word.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_Stemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class <A HREF="Stemmer.html">Stemmer</A></B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><A HREF="Stemmer.html#stemString(java.lang.String)">stemString</A></CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="LovinsStemmer()"><!-- --></A><H3>
+LovinsStemmer</H3>
+<PRE>
+public <B>LovinsStemmer</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="stem(java.lang.String)"><!-- --></A><H3>
+stem</H3>
+<PRE>
+public java.lang.String <B>stem</B>(java.lang.String&nbsp;word)</PRE>
+<DL>
+<DD>Returns the stemmed version of the given word.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE><A HREF="Stemmer.html#stem(java.lang.String)">stem</A></CODE> in class <CODE><A HREF="Stemmer.html">Stemmer</A></CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>word</CODE> - a string consisting of a single word</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;ops)</PRE>
+<DL>
+<DD>Stems text coming into stdin and writes it to stdout.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="KEAPhraseFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="NumbersFilter.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="LovinsStemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/NumbersFilter.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/NumbersFilter.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/NumbersFilter.html	(revision 21112)
@@ -0,0 +1,396 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+NumbersFilter
+</TITLE>
+<META NAME="keywords" CONTENT="NumbersFilter,NumbersFilter class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="NumbersFilter";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="LovinsStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="Stemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="NumbersFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#fields_inherited_from_class_weka.filters.Filter">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class NumbersFilter</H2>
+<PRE>
+java.lang.Object
+  |
+  +--weka.filters.Filter
+        |
+        +--<B>NumbersFilter</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>NumbersFilter</B><DT>extends weka.filters.Filter</DL>
+
+<P>
+Removes all numbers from all the string attributes in the given
+ dataset. Assumes that words are separated by whitespace.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="NumbersFilter">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+<A NAME="field_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Field Summary</B></FONT></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="fields_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Fields inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>m_NewBatch</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="NumbersFilter.html#NumbersFilter()">NumbersFilter</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="NumbersFilter.html#batchFinished()">batchFinished</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Signify that this batch of input to the filter is finished.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="NumbersFilter.html#globalInfo()">globalInfo</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns a string describing this filter</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="NumbersFilter.html#input(weka.core.Instance)">input</A></B>(weka.core.Instance&nbsp;instance)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Input an instance for filtering.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>static&nbsp;void</CODE></FONT></TD>
+<TD><CODE><B><A HREF="NumbersFilter.html#main(java.lang.String[])">main</A></B>(java.lang.String[]&nbsp;argv)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Main method for testing this class.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="NumbersFilter.html#setInputFormat(weka.core.Instances)">setInputFormat</A></B>(weka.core.Instances&nbsp;instanceInfo)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Sets the format of the input instances.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_weka.filters.Filter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class weka.filters.Filter</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>batchFilterFile, bufferInput, copyStringValues, copyStringValues, filterFile, flushInput, getInputFormat, getInputStringIndex, getOutputFormat, getOutputStringIndex, getStringIndices, inputFormat, isOutputFormatDefined, numPendingOutput, output, outputFormat, outputFormatPeek, outputPeek, push, resetQueue, setOutputFormat, useFilter</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="NumbersFilter()"><!-- --></A><H3>
+NumbersFilter</H3>
+<PRE>
+public <B>NumbersFilter</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="globalInfo()"><!-- --></A><H3>
+globalInfo</H3>
+<PRE>
+public java.lang.String <B>globalInfo</B>()</PRE>
+<DL>
+<DD>Returns a string describing this filter
+<P>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>a description of the filter suitable for
+ displaying in the explorer/experimenter gui</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="batchFinished()"><!-- --></A><H3>
+batchFinished</H3>
+<PRE>
+public boolean <B>batchFinished</B>()
+                      throws java.lang.Exception</PRE>
+<DL>
+<DD>Signify that this batch of input to the filter is finished. If
+ the filter requires all instances prior to filtering, output()
+ may now be called to retrieve the filtered instances. Any
+ subsequent instances filtered should be filtered based on setting
+ obtained from the first batch (unless the inputFormat has been
+ re-assigned or new options have been set). This default
+ implementation assumes all instance processing occurs during
+ inputFormat() and input().
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>batchFinished</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+
+<DT><B>Returns:</B><DD>true if there are instances pending output
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.NullPointerException</CODE> - if no input structure has been defined,
+<DD><CODE>java.lang.Exception</CODE> - if there was a problem finishing the batch.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="setInputFormat(weka.core.Instances)"><!-- --></A><H3>
+setInputFormat</H3>
+<PRE>
+public boolean <B>setInputFormat</B>(weka.core.Instances&nbsp;instanceInfo)
+                       throws java.lang.Exception</PRE>
+<DL>
+<DD>Sets the format of the input instances.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>setInputFormat</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instanceInfo</CODE> - an Instances object containing the input
+ instance structure (any instances contained in the object are
+ ignored - only the structure is required).
+<DT><B>Returns:</B><DD>true if the outputFormat may be collected immediately
+<DD><CODE>java.lang.Exception</CODE></DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="input(weka.core.Instance)"><!-- --></A><H3>
+input</H3>
+<PRE>
+public boolean <B>input</B>(weka.core.Instance&nbsp;instance)
+              throws java.lang.Exception</PRE>
+<DL>
+<DD>Input an instance for filtering. Ordinarily the instance is processed
+ and made available for output immediately. Some filters require all
+ instances be read before producing output.
+<P>
+<DD><DL>
+<DT><B>Overrides:</B><DD><CODE>input</CODE> in class <CODE>weka.filters.Filter</CODE></DL>
+</DD>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>instance</CODE> - the input instance
+<DT><B>Returns:</B><DD>true if the filtered instance may now be
+ collected with output().
+<DT><B>Throws:</B>
+<DD><CODE>java.lang.Exception</CODE> - if the input instance was not of the correct 
+ format or if there was a problem with the filtering.</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="main(java.lang.String[])"><!-- --></A><H3>
+main</H3>
+<PRE>
+public static void <B>main</B>(java.lang.String[]&nbsp;argv)</PRE>
+<DL>
+<DD>Main method for testing this class.
+<P>
+<DD><DL>
+<DT><B>Parameters:</B><DD><CODE>argv</CODE> - should contain arguments to the filter: use -h for help</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="LovinsStemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="Stemmer.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="NumbersFilter.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;<A HREF="#fields_inherited_from_class_weka.filters.Filter">FIELD</A>&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stemmer.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stemmer.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stemmer.html	(revision 21112)
@@ -0,0 +1,278 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+Stemmer
+</TITLE>
+<META NAME="keywords" CONTENT="Stemmer,Stemmer class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Stemmer";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="NumbersFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="Stopwords.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Stemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class Stemmer</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<B>Stemmer</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<DL>
+<DT><B>Direct Known Subclasses:</B> <DD><A HREF="GermanStemmer.html">GermanStemmer</A>, <A HREF="LovinsStemmer.html">LovinsStemmer</A></DD>
+</DL>
+<HR>
+<DL>
+<DT>public abstract class <B>Stemmer</B><DT>extends java.lang.Object<DT>implements java.io.Serializable</DL>
+
+<P>
+Abstract class for stemmers.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="Stemmer">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="Stemmer.html#Stemmer()">Stemmer</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>abstract &nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Stemmer.html#stem(java.lang.String)">stem</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Iterated stemming of the given word.</TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;java.lang.String</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Stemmer.html#stemString(java.lang.String)">stemString</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Stems everything in the given string.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="Stemmer()"><!-- --></A><H3>
+Stemmer</H3>
+<PRE>
+public <B>Stemmer</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="stem(java.lang.String)"><!-- --></A><H3>
+stem</H3>
+<PRE>
+public abstract java.lang.String <B>stem</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Iterated stemming of the given word.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<HR>
+
+<A NAME="stemString(java.lang.String)"><!-- --></A><H3>
+stemString</H3>
+<PRE>
+public java.lang.String <B>stemString</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Stems everything in the given string.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="NumbersFilter.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="Stopwords.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Stemmer.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stopwords.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stopwords.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/Stopwords.html	(revision 21112)
@@ -0,0 +1,255 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+Stopwords
+</TITLE>
+<META NAME="keywords" CONTENT="Stopwords,Stopwords class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Stopwords";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Stemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="StopwordsEnglish.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Stopwords.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class Stopwords</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<B>Stopwords</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<DL>
+<DT><B>Direct Known Subclasses:</B> <DD><A HREF="StopwordsEnglish.html">StopwordsEnglish</A>, <A HREF="StopwordsGerman.html">StopwordsGerman</A></DD>
+</DL>
+<HR>
+<DL>
+<DT>public abstract class <B>Stopwords</B><DT>extends java.lang.Object<DT>implements java.io.Serializable</DL>
+
+<P>
+Class that can test whether a given string is a stop word.
+ Lowercases all words before the test.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="Stopwords">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="Stopwords.html#Stopwords()">Stopwords</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>abstract &nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="Stopwords.html#isStopword(java.lang.String)">isStopword</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns true if the given string is a stop word.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="Stopwords()"><!-- --></A><H3>
+Stopwords</H3>
+<PRE>
+public <B>Stopwords</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="isStopword(java.lang.String)"><!-- --></A><H3>
+isStopword</H3>
+<PRE>
+public abstract boolean <B>isStopword</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Returns true if the given string is a stop word.
+<P>
+<DD><DL>
+</DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Stemmer.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="StopwordsEnglish.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="Stopwords.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsEnglish.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsEnglish.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsEnglish.html	(revision 21112)
@@ -0,0 +1,254 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+StopwordsEnglish
+</TITLE>
+<META NAME="keywords" CONTENT="StopwordsEnglish,StopwordsEnglish class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="StopwordsEnglish";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Stopwords.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="StopwordsGerman.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="StopwordsEnglish.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class StopwordsEnglish</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<A HREF="Stopwords.html">Stopwords</A>
+        |
+        +--<B>StopwordsEnglish</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>StopwordsEnglish</B><DT>extends <A HREF="Stopwords.html">Stopwords</A></DL>
+
+<P>
+Class that can test whether a given string is a stop word.
+ Lowercases all words before the test.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="StopwordsEnglish">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="StopwordsEnglish.html#StopwordsEnglish()">StopwordsEnglish</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="StopwordsEnglish.html#isStopword(java.lang.String)">isStopword</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns true if the given string is a stop word.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="StopwordsEnglish()"><!-- --></A><H3>
+StopwordsEnglish</H3>
+<PRE>
+public <B>StopwordsEnglish</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="isStopword(java.lang.String)"><!-- --></A><H3>
+isStopword</H3>
+<PRE>
+public boolean <B>isStopword</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Returns true if the given string is a stop word.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE><A HREF="Stopwords.html#isStopword(java.lang.String)">isStopword</A></CODE> in class <CODE><A HREF="Stopwords.html">Stopwords</A></CODE></DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="Stopwords.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;<A HREF="StopwordsGerman.html"><B>NEXT CLASS</B></A></FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="StopwordsEnglish.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsGerman.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsGerman.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/StopwordsGerman.html	(revision 21112)
@@ -0,0 +1,259 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+StopwordsGerman
+</TITLE>
+<META NAME="keywords" CONTENT="StopwordsGerman,StopwordsGerman class">
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="StopwordsGerman";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="StopwordsEnglish.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;NEXT CLASS</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="StopwordsGerman.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<!-- ======== START OF CLASS DATA ======== -->
+<H2>
+Class StopwordsGerman</H2>
+<PRE>
+java.lang.Object
+  |
+  +--<A HREF="Stopwords.html">Stopwords</A>
+        |
+        +--<B>StopwordsGerman</B>
+</PRE>
+<DL>
+<DT><B>All Implemented Interfaces:</B> <DD>java.io.Serializable</DD>
+</DL>
+<HR>
+<DL>
+<DT>public class <B>StopwordsGerman</B><DT>extends <A HREF="Stopwords.html">Stopwords</A></DL>
+
+<P>
+Class that can test whether a given string is a stop word.
+ Lowercases all words before the test.
+
+ This list of German stop words has been obtained from
+ http://snowball.tartarus.org/german/stop.txt
+
+ But I have deleted/changed some words that I haven't seen before.
+<P>
+
+<P>
+<DL>
+<DT><B>See Also:</B><DD><A HREF="serialized-form.html" TARGET="StopwordsGerman">Serialized Form</A></DL>
+<HR>
+
+<P>
+<!-- ======== NESTED CLASS SUMMARY ======== -->
+
+
+<!-- =========== FIELD SUMMARY =========== -->
+
+
+<!-- ======== CONSTRUCTOR SUMMARY ======== -->
+
+<A NAME="constructor_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Constructor Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE><B><A HREF="StopwordsGerman.html#StopwordsGerman()">StopwordsGerman</A></B>()</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</TD>
+</TR>
+</TABLE>
+&nbsp;
+<!-- ========== METHOD SUMMARY =========== -->
+
+<A NAME="method_summary"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Method Summary</B></FONT></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD ALIGN="right" VALIGN="top" WIDTH="1%"><FONT SIZE="-1">
+<CODE>&nbsp;boolean</CODE></FONT></TD>
+<TD><CODE><B><A HREF="StopwordsGerman.html#isStopword(java.lang.String)">isStopword</A></B>(java.lang.String&nbsp;str)</CODE>
+
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Returns true if the given string is a stop word.</TD>
+</TR>
+</TABLE>
+&nbsp;<A NAME="methods_inherited_from_class_java.lang.Object"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#EEEEFF" CLASS="TableSubHeadingColor">
+<TD><B>Methods inherited from class java.lang.Object</B></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
+<TD><CODE>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</CODE></TD>
+</TR>
+</TABLE>
+&nbsp;
+<P>
+
+<!-- ============ FIELD DETAIL =========== -->
+
+
+<!-- ========= CONSTRUCTOR DETAIL ======== -->
+
+<A NAME="constructor_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Constructor Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="StopwordsGerman()"><!-- --></A><H3>
+StopwordsGerman</H3>
+<PRE>
+public <B>StopwordsGerman</B>()</PRE>
+<DL>
+</DL>
+
+<!-- ============ METHOD DETAIL ========== -->
+
+<A NAME="method_detail"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Method Detail</B></FONT></TD>
+</TR>
+</TABLE>
+
+<A NAME="isStopword(java.lang.String)"><!-- --></A><H3>
+isStopword</H3>
+<PRE>
+public boolean <B>isStopword</B>(java.lang.String&nbsp;str)</PRE>
+<DL>
+<DD>Returns true if the given string is a stop word.
+<P>
+<DD><DL>
+<DT><B>Specified by:</B><DD><CODE><A HREF="Stopwords.html#isStopword(java.lang.String)">isStopword</A></CODE> in class <CODE><A HREF="Stopwords.html">Stopwords</A></CODE></DL>
+</DD>
+<DD><DL>
+</DL>
+</DD>
+</DL>
+<!-- ========= END OF CLASS DATA ========= -->
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">      <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Class</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;<A HREF="StopwordsEnglish.html"><B>PREV CLASS</B></A>&nbsp;
+&nbsp;NEXT CLASS</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="StopwordsGerman.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+<TR>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+  SUMMARY:&nbsp;NESTED&nbsp;|&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_summary">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_summary">METHOD</A></FONT></TD>
+<TD VALIGN="top" CLASS="NavBarCell3"><FONT SIZE="-2">
+DETAIL:&nbsp;FIELD&nbsp;|&nbsp;<A HREF="#constructor_detail">CONSTR</A>&nbsp;|&nbsp;<A HREF="#method_detail">METHOD</A></FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-frame.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-frame.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-frame.html	(revision 21112)
@@ -0,0 +1,55 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+All Classes
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="All Classes";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+<FONT size="+1" CLASS="FrameHeadingFont">
+<B>All Classes</B></FONT>
+<BR>
+
+<TABLE BORDER="0" WIDTH="100%">
+<TR>
+<TD NOWRAP><FONT CLASS="FrameItemFont"><A HREF="Counter.html" TARGET="classFrame">Counter</A>
+<BR>
+<A HREF="GermanStemmer.html" TARGET="classFrame">GermanStemmer</A>
+<BR>
+<A HREF="IteratedLovinsStemmer.html" TARGET="classFrame">IteratedLovinsStemmer</A>
+<BR>
+<A HREF="KEAFilter.html" TARGET="classFrame">KEAFilter</A>
+<BR>
+<A HREF="KEAKeyphraseExtractor.html" TARGET="classFrame">KEAKeyphraseExtractor</A>
+<BR>
+<A HREF="KEAModelBuilder.html" TARGET="classFrame">KEAModelBuilder</A>
+<BR>
+<A HREF="KEAPhraseFilter.html" TARGET="classFrame">KEAPhraseFilter</A>
+<BR>
+<A HREF="LovinsStemmer.html" TARGET="classFrame">LovinsStemmer</A>
+<BR>
+<A HREF="NumbersFilter.html" TARGET="classFrame">NumbersFilter</A>
+<BR>
+<A HREF="Stemmer.html" TARGET="classFrame">Stemmer</A>
+<BR>
+<A HREF="Stopwords.html" TARGET="classFrame">Stopwords</A>
+<BR>
+<A HREF="StopwordsEnglish.html" TARGET="classFrame">StopwordsEnglish</A>
+<BR>
+<A HREF="StopwordsGerman.html" TARGET="classFrame">StopwordsGerman</A>
+<BR>
+</FONT></TD>
+</TR>
+</TABLE>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-noframe.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-noframe.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/allclasses-noframe.html	(revision 21112)
@@ -0,0 +1,55 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+All Classes
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="All Classes";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+<FONT size="+1" CLASS="FrameHeadingFont">
+<B>All Classes</B></FONT>
+<BR>
+
+<TABLE BORDER="0" WIDTH="100%">
+<TR>
+<TD NOWRAP><FONT CLASS="FrameItemFont"><A HREF="Counter.html" TARGET="">Counter</A>
+<BR>
+<A HREF="GermanStemmer.html" TARGET="">GermanStemmer</A>
+<BR>
+<A HREF="IteratedLovinsStemmer.html" TARGET="">IteratedLovinsStemmer</A>
+<BR>
+<A HREF="KEAFilter.html" TARGET="">KEAFilter</A>
+<BR>
+<A HREF="KEAKeyphraseExtractor.html" TARGET="">KEAKeyphraseExtractor</A>
+<BR>
+<A HREF="KEAModelBuilder.html" TARGET="">KEAModelBuilder</A>
+<BR>
+<A HREF="KEAPhraseFilter.html" TARGET="">KEAPhraseFilter</A>
+<BR>
+<A HREF="LovinsStemmer.html" TARGET="">LovinsStemmer</A>
+<BR>
+<A HREF="NumbersFilter.html" TARGET="">NumbersFilter</A>
+<BR>
+<A HREF="Stemmer.html" TARGET="">Stemmer</A>
+<BR>
+<A HREF="Stopwords.html" TARGET="">Stopwords</A>
+<BR>
+<A HREF="StopwordsEnglish.html" TARGET="">StopwordsEnglish</A>
+<BR>
+<A HREF="StopwordsGerman.html" TARGET="">StopwordsGerman</A>
+<BR>
+</FONT></TD>
+</TR>
+</TABLE>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/constant-values.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/constant-values.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/constant-values.html	(revision 21112)
@@ -0,0 +1,123 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:47 NZDT 2004 -->
+<TITLE>
+Constant Field Values
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Constant Field Values";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="constant-values.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<CENTER>
+<H1>
+Constant Field Values</H1>
+</CENTER>
+<HR SIZE="4" NOSHADE>
+<B>Contents</B><UL>
+</UL>
+
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="constant-values.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/deprecated-list.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/deprecated-list.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/deprecated-list.html	(revision 21112)
@@ -0,0 +1,119 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+Deprecated List
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Deprecated List";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Deprecated</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="deprecated-list.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<CENTER>
+<H2>
+<B>Deprecated API</B></H2>
+</CENTER>
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Deprecated</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="deprecated-list.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/help-doc.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/help-doc.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/help-doc.html	(revision 21112)
@@ -0,0 +1,168 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+API Help
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="API Help";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Help</B></FONT>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="help-doc.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<CENTER>
+<H1>
+How This API Document Is Organized</H1>
+</CENTER>
+This API (Application Programming Interface) document has pages corresponding to the items in the navigation bar, described as follows.<H3>
+Package</H3>
+<BLOCKQUOTE>
+
+<P>
+Each package has a page that contains a list of its classes and interfaces, with a summary for each. This page can contain four categories:<UL>
+<LI>Interfaces (italic)<LI>Classes<LI>Exceptions<LI>Errors</UL>
+</BLOCKQUOTE>
+<H3>
+Class/Interface</H3>
+<BLOCKQUOTE>
+
+<P>
+Each class, interface, nested class and nested interface has its own separate page. Each of these pages has three sections consisting of a class/interface description, summary tables, and detailed member descriptions:<UL>
+<LI>Class inheritance diagram<LI>Direct Subclasses<LI>All Known Subinterfaces<LI>All Known Implementing Classes<LI>Class/interface declaration<LI>Class/interface description
+<P>
+<LI>Nested Class Summary<LI>Field Summary<LI>Constructor Summary<LI>Method Summary
+<P>
+<LI>Field Detail<LI>Constructor Detail<LI>Method Detail</UL>
+Each summary entry contains the first sentence from the detailed description for that item. The summary entries are alphabetical, while the detailed descriptions are in the order they appear in the source code. This preserves the logical groupings established by the programmer.</BLOCKQUOTE>
+<H3>
+Tree (Class Hierarchy)</H3>
+<BLOCKQUOTE>
+There is a <A HREF="overview-tree.html">Class Hierarchy</A> page for all packages, plus a hierarchy for each package. Each hierarchy page contains a list of classes and a list of interfaces. The classes are organized by inheritance structure starting with <code>java.lang.Object</code>. The interfaces do not inherit from <code>java.lang.Object</code>.<UL>
+<LI>When viewing the Overview page, clicking on "Tree" displays the hierarchy for all packages.<LI>When viewing a particular package, class or interface page, clicking "Tree" displays the hierarchy for only that package.</UL>
+</BLOCKQUOTE>
+<H3>
+Deprecated API</H3>
+<BLOCKQUOTE>
+The <A HREF="deprecated-list.html">Deprecated API</A> page lists all of the API that have been deprecated. A deprecated API is not recommended for use, generally due to improvements, and a replacement API is usually given. Deprecated APIs may be removed in future implementations.</BLOCKQUOTE>
+<H3>
+Index</H3>
+<BLOCKQUOTE>
+The <A HREF="index-all.html">Index</A> contains an alphabetic list of all classes, interfaces, constructors, methods, and fields.</BLOCKQUOTE>
+<H3>
+Prev/Next</H3>
+These links take you to the next or previous class, interface, package, or related page.<H3>
+Frames/No Frames</H3>
+These links show and hide the HTML frames.  All pages are available with or without frames.
+<P>
+<H3>
+Serialized Form</H3>
+Each serializable or externalizable class has a description of its serialization fields and methods. This information is of interest to re-implementors, not to developers using the API. While there is no link in the navigation bar, you can get to this information by going to any serialized class and clicking "Serialized Form" in the "See also" section of the class description.
+<P>
+<FONT SIZE="-1">
+<EM>
+This help file applies to API documentation generated using the standard doclet.</EM>
+</FONT>
+<BR>
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Help</B></FONT>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="help-doc.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index-all.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index-all.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index-all.html	(revision 21112)
@@ -0,0 +1,596 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+Index
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Index";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Index</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="index-all.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<A HREF="#_A_">A</A> <A HREF="#_B_">B</A> <A HREF="#_C_">C</A> <A HREF="#_D_">D</A> <A HREF="#_E_">E</A> <A HREF="#_G_">G</A> <A HREF="#_I_">I</A> <A HREF="#_K_">K</A> <A HREF="#_L_">L</A> <A HREF="#_M_">M</A> <A HREF="#_N_">N</A> <A HREF="#_S_">S</A> <A HREF="#_T_">T</A> <A HREF="#_V_">V</A> <HR>
+<A NAME="_A_"><!-- --></A><H2>
+<B>A</B></H2>
+<DL>
+<DT><A HREF="KEAPhraseFilter.html#attributeIndicesTipText()"><B>attributeIndicesTipText()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Returns the tip text for this property
+</DL>
+<HR>
+<A NAME="_B_"><!-- --></A><H2>
+<B>B</B></H2>
+<DL>
+<DT><A HREF="KEAFilter.html#batchFinished()"><B>batchFinished()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Signify that this batch of input to the filter is finished.
+<DT><A HREF="KEAPhraseFilter.html#batchFinished()"><B>batchFinished()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Signify that this batch of input to the filter is finished.
+<DT><A HREF="NumbersFilter.html#batchFinished()"><B>batchFinished()</B></A> - 
+Method in class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>Signify that this batch of input to the filter is finished.
+<DT><A HREF="KEAModelBuilder.html#buildModel(java.util.Hashtable)"><B>buildModel(Hashtable)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Builds the model from the files
+</DL>
+<HR>
+<A NAME="_C_"><!-- --></A><H2>
+<B>C</B></H2>
+<DL>
+<DT><A HREF="KEAKeyphraseExtractor.html#collectStems()"><B>collectStems()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Collects the stems of the file names.
+<DT><A HREF="KEAModelBuilder.html#collectStems()"><B>collectStems()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Collects the stems of the file names.
+<DT><A HREF="Counter.html"><B>Counter</B></A> - class <A HREF="Counter.html">Counter</A>.<DD>Class that implements a simple counter.<DT><A HREF="Counter.html#Counter()"><B>Counter()</B></A> - 
+Constructor for class <A HREF="Counter.html">Counter</A>
+<DD>Initializes the counter to 1
+<DT><A HREF="Counter.html#Counter(int)"><B>Counter(int)</B></A> - 
+Constructor for class <A HREF="Counter.html">Counter</A>
+<DD>Initializes the counter to the given value
+</DL>
+<HR>
+<A NAME="_D_"><!-- --></A><H2>
+<B>D</B></H2>
+<DL>
+<DT><A HREF="KEAPhraseFilter.html#disallowInternalPeriodsTipText()"><B>disallowInternalPeriodsTipText()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Returns the tip text for this property
+</DL>
+<HR>
+<A NAME="_E_"><!-- --></A><H2>
+<B>E</B></H2>
+<DL>
+<DT><A HREF="KEAKeyphraseExtractor.html#extractKeyphrases(java.util.Hashtable)"><B>extractKeyphrases(Hashtable)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Builds the model from the files
+</DL>
+<HR>
+<A NAME="_G_"><!-- --></A><H2>
+<B>G</B></H2>
+<DL>
+<DT><A HREF="GermanStemmer.html"><B>GermanStemmer</B></A> - class <A HREF="GermanStemmer.html">GermanStemmer</A>.<DD>A stemmer for German words.<DT><A HREF="GermanStemmer.html#GermanStemmer()"><B>GermanStemmer()</B></A> - 
+Constructor for class <A HREF="GermanStemmer.html">GermanStemmer</A>
+<DD>&nbsp;
+<DT><A HREF="KEAKeyphraseExtractor.html#getAdditionalInfo()"><B>getAdditionalInfo()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of AdditionalInfo.
+<DT><A HREF="KEAPhraseFilter.html#getAttributeIndices()"><B>getAttributeIndices()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Get the current range selection.
+<DT><A HREF="KEAFilter.html#getCheckForProperNouns()"><B>getCheckForProperNouns()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the M_CheckProperNouns value.
+<DT><A HREF="KEAModelBuilder.html#getCheckForProperNouns()"><B>getCheckForProperNouns()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the M_CheckProperNouns value.
+<DT><A HREF="KEAFilter.html#getDebug()"><B>getDebug()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of Debug.
+<DT><A HREF="KEAKeyphraseExtractor.html#getDebug()"><B>getDebug()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of debug.
+<DT><A HREF="KEAModelBuilder.html#getDebug()"><B>getDebug()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of debug.
+<DT><A HREF="KEAKeyphraseExtractor.html#getDirName()"><B>getDirName()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of dirName.
+<DT><A HREF="KEAModelBuilder.html#getDirName()"><B>getDirName()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of dirName.
+<DT><A HREF="KEAFilter.html#getDisallowInternalPeriods()"><B>getDisallowInternalPeriods()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get whether the supplied columns are to be processed
+<DT><A HREF="KEAPhraseFilter.html#getDisallowInternalPeriods()"><B>getDisallowInternalPeriods()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Get whether the supplied columns are to be processed
+<DT><A HREF="KEAModelBuilder.html#getDisallowIPeriods()"><B>getDisallowIPeriods()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of disallowIPeriods.
+<DT><A HREF="KEAFilter.html#getDocumentAtt()"><B>getDocumentAtt()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of DocumentAtt.
+<DT><A HREF="KEAKeyphraseExtractor.html#getEncoding()"><B>getEncoding()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of encoding.
+<DT><A HREF="KEAModelBuilder.html#getEncoding()"><B>getEncoding()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of encoding.
+<DT><A HREF="KEAPhraseFilter.html#getInvertSelection()"><B>getInvertSelection()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Get whether the supplied columns are to be processed
+<DT><A HREF="KEAFilter.html#getKeyphrasesAtt()"><B>getKeyphrasesAtt()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of KeyphraseAtt.
+<DT><A HREF="KEAFilter.html#getKFused()"><B>getKFused()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Gets whether keyphrase frequency attribute is used.
+<DT><A HREF="KEAFilter.html#getMaxPhraseLength()"><B>getMaxPhraseLength()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of MaxPhraseLength.
+<DT><A HREF="KEAModelBuilder.html#getMaxPhraseLength()"><B>getMaxPhraseLength()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of MaxPhraseLength.
+<DT><A HREF="KEAFilter.html#getMinNumOccur()"><B>getMinNumOccur()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of MinNumOccur.
+<DT><A HREF="KEAModelBuilder.html#getMinNumOccur()"><B>getMinNumOccur()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of MinNumOccur.
+<DT><A HREF="KEAFilter.html#getMinPhraseLength()"><B>getMinPhraseLength()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the value of MinPhraseLength.
+<DT><A HREF="KEAModelBuilder.html#getMinPhraseLength()"><B>getMinPhraseLength()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of MinPhraseLength.
+<DT><A HREF="KEAKeyphraseExtractor.html#getModelName()"><B>getModelName()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of modelName.
+<DT><A HREF="KEAModelBuilder.html#getModelName()"><B>getModelName()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of modelName.
+<DT><A HREF="KEAKeyphraseExtractor.html#getNumPhrases()"><B>getNumPhrases()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Get the value of numPhrases.
+<DT><A HREF="KEAFilter.html#getOptions()"><B>getOptions()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Gets the current settings of the filter.
+<DT><A HREF="KEAKeyphraseExtractor.html#getOptions()"><B>getOptions()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Gets the current option settings.
+<DT><A HREF="KEAModelBuilder.html#getOptions()"><B>getOptions()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Gets the current option settings.
+<DT><A HREF="KEAPhraseFilter.html#getOptions()"><B>getOptions()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Gets the current settings of the filter.
+<DT><A HREF="KEAFilter.html#getProbabilityIndex()"><B>getProbabilityIndex()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns the index of the phrases' probabilities in the output ARFF file.
+<DT><A HREF="KEAFilter.html#getRankIndex()"><B>getRankIndex()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns the index of the phrases' ranks in the output ARFF file.
+<DT><A HREF="KEAFilter.html#getStemmedPhraseIndex()"><B>getStemmedPhraseIndex()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns the index of the stemmed phrases in the output ARFF file.
+<DT><A HREF="KEAFilter.html#getStemmer()"><B>getStemmer()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the Stemmer value.
+<DT><A HREF="KEAModelBuilder.html#getStemmer()"><B>getStemmer()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the Stemmer value.
+<DT><A HREF="KEAFilter.html#getStopwords()"><B>getStopwords()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Get the M_Stopwords value.
+<DT><A HREF="KEAModelBuilder.html#getStopwords()"><B>getStopwords()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the M_Stopwords value.
+<DT><A HREF="KEAFilter.html#getUnstemmedPhraseIndex()"><B>getUnstemmedPhraseIndex()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns the index of the unstemmed phrases in the output ARFF file.
+<DT><A HREF="KEAModelBuilder.html#getUseKFrequency()"><B>getUseKFrequency()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Get the value of useKFrequency.
+<DT><A HREF="KEAFilter.html#globalInfo()"><B>globalInfo()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns a string describing this filter
+<DT><A HREF="KEAPhraseFilter.html#globalInfo()"><B>globalInfo()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Returns a string describing this filter
+<DT><A HREF="NumbersFilter.html#globalInfo()"><B>globalInfo()</B></A> - 
+Method in class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>Returns a string describing this filter
+</DL>
+<HR>
+<A NAME="_I_"><!-- --></A><H2>
+<B>I</B></H2>
+<DL>
+<DT><A HREF="Counter.html#increment()"><B>increment()</B></A> - 
+Method in class <A HREF="Counter.html">Counter</A>
+<DD>Increments the counter.
+<DT><A HREF="KEAFilter.html#input(weka.core.Instance)"><B>input(Instance)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Input an instance for filtering.
+<DT><A HREF="KEAPhraseFilter.html#input(weka.core.Instance)"><B>input(Instance)</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Input an instance for filtering.
+<DT><A HREF="NumbersFilter.html#input(weka.core.Instance)"><B>input(Instance)</B></A> - 
+Method in class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>Input an instance for filtering.
+<DT><A HREF="KEAPhraseFilter.html#invertSelectionTipText()"><B>invertSelectionTipText()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Returns the tip text for this property
+<DT><A HREF="Stopwords.html#isStopword(java.lang.String)"><B>isStopword(String)</B></A> - 
+Method in class <A HREF="Stopwords.html">Stopwords</A>
+<DD>Returns true if the given string is a stop word.
+<DT><A HREF="StopwordsEnglish.html#isStopword(java.lang.String)"><B>isStopword(String)</B></A> - 
+Method in class <A HREF="StopwordsEnglish.html">StopwordsEnglish</A>
+<DD>Returns true if the given string is a stop word.
+<DT><A HREF="StopwordsGerman.html#isStopword(java.lang.String)"><B>isStopword(String)</B></A> - 
+Method in class <A HREF="StopwordsGerman.html">StopwordsGerman</A>
+<DD>Returns true if the given string is a stop word.
+<DT><A HREF="IteratedLovinsStemmer.html"><B>IteratedLovinsStemmer</B></A> - class <A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A>.<DD>Implements the iterated version of the Lovins stemmer.<DT><A HREF="IteratedLovinsStemmer.html#IteratedLovinsStemmer()"><B>IteratedLovinsStemmer()</B></A> - 
+Constructor for class <A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A>
+<DD>&nbsp;
+</DL>
+<HR>
+<A NAME="_K_"><!-- --></A><H2>
+<B>K</B></H2>
+<DL>
+<DT><A HREF="KEAFilter.html"><B>KEAFilter</B></A> - class <A HREF="KEAFilter.html">KEAFilter</A>.<DD>This filter converts the incoming data into data appropriate for
+ keyphrase classification.<DT><A HREF="KEAFilter.html#KEAFilter()"><B>KEAFilter()</B></A> - 
+Constructor for class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>&nbsp;
+<DT><A HREF="KEAKeyphraseExtractor.html"><B>KEAKeyphraseExtractor</B></A> - class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>.<DD>Extracts keyphrases from the documents in a given directory.<DT><A HREF="KEAKeyphraseExtractor.html#KEAKeyphraseExtractor()"><B>KEAKeyphraseExtractor()</B></A> - 
+Constructor for class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>&nbsp;
+<DT><A HREF="KEAModelBuilder.html"><B>KEAModelBuilder</B></A> - class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>.<DD>Builds a keyphrase extraction model from the documents in a given
+ directory.<DT><A HREF="KEAModelBuilder.html#KEAModelBuilder()"><B>KEAModelBuilder()</B></A> - 
+Constructor for class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>&nbsp;
+<DT><A HREF="KEAPhraseFilter.html"><B>KEAPhraseFilter</B></A> - class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>.<DD>This filter splits the text in selected string
+ attributes into phrases.<DT><A HREF="KEAPhraseFilter.html#KEAPhraseFilter()"><B>KEAPhraseFilter()</B></A> - 
+Constructor for class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>&nbsp;
+</DL>
+<HR>
+<A NAME="_L_"><!-- --></A><H2>
+<B>L</B></H2>
+<DL>
+<DT><A HREF="KEAFilter.html#listOptions()"><B>listOptions()</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Returns an enumeration describing the available options
+<DT><A HREF="KEAKeyphraseExtractor.html#listOptions()"><B>listOptions()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Returns an enumeration describing the available options.
+<DT><A HREF="KEAModelBuilder.html#listOptions()"><B>listOptions()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Returns an enumeration describing the available options.
+<DT><A HREF="KEAPhraseFilter.html#listOptions()"><B>listOptions()</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Returns an enumeration describing the available options
+<DT><A HREF="KEAKeyphraseExtractor.html#loadModel()"><B>loadModel()</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Loads the extraction model from the file.
+<DT><A HREF="LovinsStemmer.html"><B>LovinsStemmer</B></A> - class <A HREF="LovinsStemmer.html">LovinsStemmer</A>.<DD>Implements the Lovins stemmer.<DT><A HREF="LovinsStemmer.html#LovinsStemmer()"><B>LovinsStemmer()</B></A> - 
+Constructor for class <A HREF="LovinsStemmer.html">LovinsStemmer</A>
+<DD>&nbsp;
+</DL>
+<HR>
+<A NAME="_M_"><!-- --></A><H2>
+<B>M</B></H2>
+<DL>
+<DT><A HREF="KEAPhraseFilter.html#m_DisallowInternalPeriods"><B>m_DisallowInternalPeriods</B></A> - 
+Variable in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Determines whether internal periods are allowed
+<DT><A HREF="KEAPhraseFilter.html#m_SelectCols"><B>m_SelectCols</B></A> - 
+Variable in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Stores which columns to select as a funky range
+<DT><A HREF="IteratedLovinsStemmer.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A>
+<DD>Stems text coming into stdin and writes it to stdout.
+<DT><A HREF="KEAFilter.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Main method for testing this class.
+<DT><A HREF="KEAKeyphraseExtractor.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>The main method.
+<DT><A HREF="KEAModelBuilder.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>The main method.
+<DT><A HREF="KEAPhraseFilter.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Main method for testing this class.
+<DT><A HREF="LovinsStemmer.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="LovinsStemmer.html">LovinsStemmer</A>
+<DD>Stems text coming into stdin and writes it to stdout.
+<DT><A HREF="NumbersFilter.html#main(java.lang.String[])"><B>main(String[])</B></A> - 
+Static method in class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>Main method for testing this class.
+</DL>
+<HR>
+<A NAME="_N_"><!-- --></A><H2>
+<B>N</B></H2>
+<DL>
+<DT><A HREF="NumbersFilter.html"><B>NumbersFilter</B></A> - class <A HREF="NumbersFilter.html">NumbersFilter</A>.<DD>Removes all numbers from all the string attributes in the given
+ dataset.<DT><A HREF="NumbersFilter.html#NumbersFilter()"><B>NumbersFilter()</B></A> - 
+Constructor for class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>&nbsp;
+</DL>
+<HR>
+<A NAME="_S_"><!-- --></A><H2>
+<B>S</B></H2>
+<DL>
+<DT><A HREF="KEAModelBuilder.html#saveModel()"><B>saveModel()</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Saves the extraction model to the file.
+<DT><A HREF="KEAKeyphraseExtractor.html#setAdditionalInfo(boolean)"><B>setAdditionalInfo(boolean)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of AdditionalInfo.
+<DT><A HREF="KEAPhraseFilter.html#setAttributeIndices(java.lang.String)"><B>setAttributeIndices(String)</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Set which attributes are to be processed
+<DT><A HREF="KEAPhraseFilter.html#setAttributeIndicesArray(int[])"><B>setAttributeIndicesArray(int[])</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Set which attributes are to be processed
+<DT><A HREF="KEAFilter.html#setCheckForProperNouns(boolean)"><B>setCheckForProperNouns(boolean)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the M_CheckProperNouns value.
+<DT><A HREF="KEAModelBuilder.html#setCheckForProperNouns(boolean)"><B>setCheckForProperNouns(boolean)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the M_CheckProperNouns value.
+<DT><A HREF="KEAFilter.html#setDebug(boolean)"><B>setDebug(boolean)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of Debug.
+<DT><A HREF="KEAKeyphraseExtractor.html#setDebug(boolean)"><B>setDebug(boolean)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of debug.
+<DT><A HREF="KEAModelBuilder.html#setDebug(boolean)"><B>setDebug(boolean)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of debug.
+<DT><A HREF="KEAKeyphraseExtractor.html#setDirName(java.lang.String)"><B>setDirName(String)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of dirName.
+<DT><A HREF="KEAModelBuilder.html#setDirName(java.lang.String)"><B>setDirName(String)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of dirName.
+<DT><A HREF="KEAFilter.html#setDisallowInternalPeriods(boolean)"><B>setDisallowInternalPeriods(boolean)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set whether selected columns should be processed.
+<DT><A HREF="KEAPhraseFilter.html#setDisallowInternalPeriods(boolean)"><B>setDisallowInternalPeriods(boolean)</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Set whether selected columns should be processed.
+<DT><A HREF="KEAModelBuilder.html#setDisallowIPeriods(boolean)"><B>setDisallowIPeriods(boolean)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of disallowIPeriods.
+<DT><A HREF="KEAFilter.html#setDocumentAtt(int)"><B>setDocumentAtt(int)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of DocumentAtt.
+<DT><A HREF="KEAKeyphraseExtractor.html#setEncoding(java.lang.String)"><B>setEncoding(String)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of encoding.
+<DT><A HREF="KEAModelBuilder.html#setEncoding(java.lang.String)"><B>setEncoding(String)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of encoding.
+<DT><A HREF="KEAFilter.html#setInputFormat(weka.core.Instances)"><B>setInputFormat(Instances)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Sets the format of the input instances.
+<DT><A HREF="KEAPhraseFilter.html#setInputFormat(weka.core.Instances)"><B>setInputFormat(Instances)</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Sets the format of the input instances.
+<DT><A HREF="NumbersFilter.html#setInputFormat(weka.core.Instances)"><B>setInputFormat(Instances)</B></A> - 
+Method in class <A HREF="NumbersFilter.html">NumbersFilter</A>
+<DD>Sets the format of the input instances.
+<DT><A HREF="KEAPhraseFilter.html#setInvertSelection(boolean)"><B>setInvertSelection(boolean)</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Set whether selected columns should be processed.
+<DT><A HREF="KEAFilter.html#setKeyphrasesAtt(int)"><B>setKeyphrasesAtt(int)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of KeyphrasesAtt.
+<DT><A HREF="KEAFilter.html#setKFused(boolean)"><B>setKFused(boolean)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Sets whether keyphrase frequency attribute is used.
+<DT><A HREF="KEAFilter.html#setMaxPhraseLength(int)"><B>setMaxPhraseLength(int)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of MaxPhraseLength.
+<DT><A HREF="KEAModelBuilder.html#setMaxPhraseLength(int)"><B>setMaxPhraseLength(int)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of MaxPhraseLength.
+<DT><A HREF="KEAFilter.html#setMinNumOccur(int)"><B>setMinNumOccur(int)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of MinNumOccur.
+<DT><A HREF="KEAModelBuilder.html#setMinNumOccur(int)"><B>setMinNumOccur(int)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of MinNumOccur.
+<DT><A HREF="KEAFilter.html#setMinPhraseLength(int)"><B>setMinPhraseLength(int)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the value of MinPhraseLength.
+<DT><A HREF="KEAModelBuilder.html#setMinPhraseLength(int)"><B>setMinPhraseLength(int)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of MinPhraseLength.
+<DT><A HREF="KEAKeyphraseExtractor.html#setModelName(java.lang.String)"><B>setModelName(String)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of modelName.
+<DT><A HREF="KEAModelBuilder.html#setModelName(java.lang.String)"><B>setModelName(String)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of modelName.
+<DT><A HREF="KEAKeyphraseExtractor.html#setNumPhrases(int)"><B>setNumPhrases(int)</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Set the value of numPhrases.
+<DT><A HREF="KEAFilter.html#setOptions(java.lang.String[])"><B>setOptions(String[])</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Parses a given list of options controlling the behaviour of this object.
+<DT><A HREF="KEAKeyphraseExtractor.html#setOptions(java.lang.String[])"><B>setOptions(String[])</B></A> - 
+Method in class <A HREF="KEAKeyphraseExtractor.html">KEAKeyphraseExtractor</A>
+<DD>Parses a given list of options controlling the behaviour of this object.
+<DT><A HREF="KEAModelBuilder.html#setOptions(java.lang.String[])"><B>setOptions(String[])</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Parses a given list of options controlling the behaviour of this object.
+<DT><A HREF="KEAPhraseFilter.html#setOptions(java.lang.String[])"><B>setOptions(String[])</B></A> - 
+Method in class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A>
+<DD>Parses a given list of options controlling the behaviour of this object.
+<DT><A HREF="KEAFilter.html#setStemmer(Stemmer)"><B>setStemmer(Stemmer)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the Stemmer value.
+<DT><A HREF="KEAModelBuilder.html#setStemmer(Stemmer)"><B>setStemmer(Stemmer)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the Stemmer value.
+<DT><A HREF="KEAFilter.html#setStopwords(Stopwords)"><B>setStopwords(Stopwords)</B></A> - 
+Method in class <A HREF="KEAFilter.html">KEAFilter</A>
+<DD>Set the M_Stopwords value.
+<DT><A HREF="KEAModelBuilder.html#setStopwords(Stopwords)"><B>setStopwords(Stopwords)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the M_Stopwords value.
+<DT><A HREF="KEAModelBuilder.html#setUseKFrequency(boolean)"><B>setUseKFrequency(boolean)</B></A> - 
+Method in class <A HREF="KEAModelBuilder.html">KEAModelBuilder</A>
+<DD>Set the value of useKFrequency.
+<DT><A HREF="GermanStemmer.html#stem(java.lang.String)"><B>stem(String)</B></A> - 
+Method in class <A HREF="GermanStemmer.html">GermanStemmer</A>
+<DD>Stemms the given term to an unique <tt>discriminator</tt>.
+<DT><A HREF="IteratedLovinsStemmer.html#stem(java.lang.String)"><B>stem(String)</B></A> - 
+Method in class <A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A>
+<DD>Iterated stemming of the given word.
+<DT><A HREF="LovinsStemmer.html#stem(java.lang.String)"><B>stem(String)</B></A> - 
+Method in class <A HREF="LovinsStemmer.html">LovinsStemmer</A>
+<DD>Returns the stemmed version of the given word.
+<DT><A HREF="Stemmer.html#stem(java.lang.String)"><B>stem(String)</B></A> - 
+Method in class <A HREF="Stemmer.html">Stemmer</A>
+<DD>Iterated stemming of the given word.
+<DT><A HREF="Stemmer.html"><B>Stemmer</B></A> - class <A HREF="Stemmer.html">Stemmer</A>.<DD>Abstract class for stemmers.<DT><A HREF="Stemmer.html#Stemmer()"><B>Stemmer()</B></A> - 
+Constructor for class <A HREF="Stemmer.html">Stemmer</A>
+<DD>&nbsp;
+<DT><A HREF="Stemmer.html#stemString(java.lang.String)"><B>stemString(String)</B></A> - 
+Method in class <A HREF="Stemmer.html">Stemmer</A>
+<DD>Stems everything in the given string.
+<DT><A HREF="Stopwords.html"><B>Stopwords</B></A> - class <A HREF="Stopwords.html">Stopwords</A>.<DD>Class that can test whether a given string is a stop word.<DT><A HREF="Stopwords.html#Stopwords()"><B>Stopwords()</B></A> - 
+Constructor for class <A HREF="Stopwords.html">Stopwords</A>
+<DD>&nbsp;
+<DT><A HREF="StopwordsEnglish.html"><B>StopwordsEnglish</B></A> - class <A HREF="StopwordsEnglish.html">StopwordsEnglish</A>.<DD>Class that can test whether a given string is a stop word.<DT><A HREF="StopwordsEnglish.html#StopwordsEnglish()"><B>StopwordsEnglish()</B></A> - 
+Constructor for class <A HREF="StopwordsEnglish.html">StopwordsEnglish</A>
+<DD>&nbsp;
+<DT><A HREF="StopwordsGerman.html"><B>StopwordsGerman</B></A> - class <A HREF="StopwordsGerman.html">StopwordsGerman</A>.<DD>Class that can test whether a given string is a stop word.<DT><A HREF="StopwordsGerman.html#StopwordsGerman()"><B>StopwordsGerman()</B></A> - 
+Constructor for class <A HREF="StopwordsGerman.html">StopwordsGerman</A>
+<DD>&nbsp;
+</DL>
+<HR>
+<A NAME="_T_"><!-- --></A><H2>
+<B>T</B></H2>
+<DL>
+<DT><A HREF="Counter.html#toString()"><B>toString()</B></A> - 
+Method in class <A HREF="Counter.html">Counter</A>
+<DD>Returns string containing value.
+</DL>
+<HR>
+<A NAME="_V_"><!-- --></A><H2>
+<B>V</B></H2>
+<DL>
+<DT><A HREF="Counter.html#value()"><B>value()</B></A> - 
+Method in class <A HREF="Counter.html">Counter</A>
+<DD>Gets the value.
+</DL>
+<HR>
+<A HREF="#_A_">A</A> <A HREF="#_B_">B</A> <A HREF="#_C_">C</A> <A HREF="#_D_">D</A> <A HREF="#_E_">E</A> <A HREF="#_G_">G</A> <A HREF="#_I_">I</A> <A HREF="#_K_">K</A> <A HREF="#_L_">L</A> <A HREF="#_M_">M</A> <A HREF="#_N_">N</A> <A HREF="#_S_">S</A> <A HREF="#_T_">T</A> <A HREF="#_V_">V</A> 
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Index</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="index-all.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/index.html	(revision 21112)
@@ -0,0 +1,22 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004-->
+<TITLE>
+Generated Documentation (Untitled)
+</TITLE>
+</HEAD>
+<FRAMESET cols="20%,80%">
+<FRAME src="allclasses-frame.html" name="packageFrame">
+<FRAME src="Counter.html" name="classFrame">
+</FRAMESET>
+<NOFRAMES>
+<H2>
+Frame Alert</H2>
+
+<P>
+This document is designed to be viewed using the frames feature. If you see this message, you are using a non-frame-capable web client.
+<BR>
+Link to<A HREF="Counter.html">Non-frame version.</A></NOFRAMES>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/overview-tree.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/overview-tree.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/overview-tree.html	(revision 21112)
@@ -0,0 +1,143 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+Class Hierarchy
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Class Hierarchy";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Tree</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="overview-tree.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<CENTER>
+<H2>
+Hierarchy For All Packages</H2>
+</CENTER>
+<H2>
+Class Hierarchy
+</H2>
+<UL>
+<LI TYPE="circle">class java.lang.Object<UL>
+<LI TYPE="circle">class <A HREF="Counter.html"><B>Counter</B></A> (implements java.io.Serializable)
+<LI TYPE="circle">class weka.filters.Filter (implements java.io.Serializable)
+<UL>
+<LI TYPE="circle">class <A HREF="KEAFilter.html"><B>KEAFilter</B></A> (implements weka.core.OptionHandler)
+<LI TYPE="circle">class <A HREF="KEAPhraseFilter.html"><B>KEAPhraseFilter</B></A> (implements weka.core.OptionHandler)
+<LI TYPE="circle">class <A HREF="NumbersFilter.html"><B>NumbersFilter</B></A></UL>
+<LI TYPE="circle">class <A HREF="KEAKeyphraseExtractor.html"><B>KEAKeyphraseExtractor</B></A> (implements weka.core.OptionHandler)
+<LI TYPE="circle">class <A HREF="KEAModelBuilder.html"><B>KEAModelBuilder</B></A> (implements weka.core.OptionHandler)
+<LI TYPE="circle">class <A HREF="Stemmer.html"><B>Stemmer</B></A> (implements java.io.Serializable)
+<UL>
+<LI TYPE="circle">class <A HREF="GermanStemmer.html"><B>GermanStemmer</B></A><LI TYPE="circle">class <A HREF="LovinsStemmer.html"><B>LovinsStemmer</B></A> (implements java.io.Serializable)
+<UL>
+<LI TYPE="circle">class <A HREF="IteratedLovinsStemmer.html"><B>IteratedLovinsStemmer</B></A></UL>
+</UL>
+<LI TYPE="circle">class <A HREF="Stopwords.html"><B>Stopwords</B></A> (implements java.io.Serializable)
+<UL>
+<LI TYPE="circle">class <A HREF="StopwordsEnglish.html"><B>StopwordsEnglish</B></A><LI TYPE="circle">class <A HREF="StopwordsGerman.html"><B>StopwordsGerman</B></A></UL>
+</UL>
+</UL>
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> &nbsp;<FONT CLASS="NavBarFont1Rev"><B>Tree</B></FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="overview-tree.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/package-list
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/package-list	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/package-list	(revision 21112)
@@ -0,0 +1,1 @@
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/packages.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/packages.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/packages.html	(revision 21112)
@@ -0,0 +1,26 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:48 NZDT 2004 -->
+<TITLE>
+
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<BODY BGCOLOR="white">
+
+<BR>
+
+<BR>
+
+<BR>
+<CENTER>
+The front page has been relocated.Please see:
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<A HREF="index.html">Frame version</A>
+<BR>
+&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<A HREF="Counter.html">Non-frame version.</A></CENTER>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/serialized-form.html
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/serialized-form.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/serialized-form.html	(revision 21112)
@@ -0,0 +1,604 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN""http://www.w3.org/TR/REC-html40/loose.dtd">
+<!--NewPage-->
+<HTML>
+<HEAD>
+<!-- Generated by javadoc on Fri Mar 19 17:50:50 NZDT 2004 -->
+<TITLE>
+Serialized Form
+</TITLE>
+<LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
+</HEAD>
+<SCRIPT>
+function asd()
+{
+parent.document.title="Serialized Form";
+}
+</SCRIPT>
+<BODY BGCOLOR="white" onload="asd();">
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_top"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_top_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="serialized-form.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+<CENTER>
+<H1>
+Serialized Form</H1>
+</CENTER>
+<A NAME="Counter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="Counter.html">Counter</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+<A NAME="serializedForm"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Serialized Fields</B></FONT></TD>
+</TR>
+</TABLE>
+
+<H3>
+m_val</H3>
+<PRE>
+int <B>m_val</B></PRE>
+<DL>
+<DD>Integer value stored
+<P>
+<DL>
+</DL>
+</DL>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="GermanStemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="GermanStemmer.html">GermanStemmer</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+<A NAME="serializedForm"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Serialized Fields</B></FONT></TD>
+</TR>
+</TABLE>
+
+<H3>
+sb</H3>
+<PRE>
+java.lang.StringBuffer <B>sb</B></PRE>
+<DL>
+<DD>Buffer for the terms while stemming them.
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+uppercase</H3>
+<PRE>
+boolean <B>uppercase</B></PRE>
+<DL>
+<DD>Indicates if a term is handled as a noun.
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+substCount</H3>
+<PRE>
+int <B>substCount</B></PRE>
+<DL>
+<DD>Amount of characters that are removed with <tt>substitute()</tt> while stemming.
+<P>
+<DL>
+</DL>
+</DL>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="IteratedLovinsStemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="IteratedLovinsStemmer.html">IteratedLovinsStemmer</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="KEAFilter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="KEAFilter.html">KEAFilter</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+<A NAME="serializedForm"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Serialized Fields</B></FONT></TD>
+</TR>
+</TABLE>
+
+<H3>
+m_DocumentAtt</H3>
+<PRE>
+int <B>m_DocumentAtt</B></PRE>
+<DL>
+<DD>Index of attribute containing the documents
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_KeyphrasesAtt</H3>
+<PRE>
+int <B>m_KeyphrasesAtt</B></PRE>
+<DL>
+<DD>Index of attribute containing the keyphrases
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_MaxPhraseLength</H3>
+<PRE>
+int <B>m_MaxPhraseLength</B></PRE>
+<DL>
+<DD>The maximum length of phrases
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_MinPhraseLength</H3>
+<PRE>
+int <B>m_MinPhraseLength</B></PRE>
+<DL>
+<DD>The minimum length of phrases
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_KFused</H3>
+<PRE>
+boolean <B>m_KFused</B></PRE>
+<DL>
+<DD>Is keyphrase frequency attribute being used?
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_Debug</H3>
+<PRE>
+boolean <B>m_Debug</B></PRE>
+<DL>
+<DD>Flag for debugging mode
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_DisallowInternalPeriods</H3>
+<PRE>
+boolean <B>m_DisallowInternalPeriods</B></PRE>
+<DL>
+<DD>Determines whether internal periods are allowed
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_MinNumOccur</H3>
+<PRE>
+int <B>m_MinNumOccur</B></PRE>
+<DL>
+<DD>The minimum number of occurences of a phrase
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_NumFeatures</H3>
+<PRE>
+int <B>m_NumFeatures</B></PRE>
+<DL>
+<DD>The number of features describing a phrase
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_TfidfIndex</H3>
+<PRE>
+int <B>m_TfidfIndex</B></PRE>
+<DL>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_FirstOccurIndex</H3>
+<PRE>
+int <B>m_FirstOccurIndex</B></PRE>
+<DL>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_KeyFreqIndex</H3>
+<PRE>
+int <B>m_KeyFreqIndex</B></PRE>
+<DL>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_PunctFilter</H3>
+<PRE>
+KEAPhraseFilter <B>m_PunctFilter</B></PRE>
+<DL>
+<DD>The punctuation filter used by this filter
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_NumbersFilter</H3>
+<PRE>
+NumbersFilter <B>m_NumbersFilter</B></PRE>
+<DL>
+<DD>The numbers filter used by this filter
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_Classifier</H3>
+<PRE>
+weka.classifiers.DistributionClassifier <B>m_Classifier</B></PRE>
+<DL>
+<DD>The actual classifier used to compute probabilities
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_Dictionary</H3>
+<PRE>
+java.util.HashMap <B>m_Dictionary</B></PRE>
+<DL>
+<DD>The dictionary containing the document frequencies
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_KeyphraseDictionary</H3>
+<PRE>
+java.util.HashMap <B>m_KeyphraseDictionary</B></PRE>
+<DL>
+<DD>The dictionary containing the keyphrases
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_NumDocs</H3>
+<PRE>
+int <B>m_NumDocs</B></PRE>
+<DL>
+<DD>The number of documents in the global frequencies corpus
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_ClassifierData</H3>
+<PRE>
+weka.core.Instances <B>m_ClassifierData</B></PRE>
+<DL>
+<DD>Template for the classifier data
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_Stemmer</H3>
+<PRE>
+Stemmer <B>m_Stemmer</B></PRE>
+<DL>
+<DD>The stemmer to be used
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_Stopwords</H3>
+<PRE>
+Stopwords <B>m_Stopwords</B></PRE>
+<DL>
+<DD>The list of stop words to be used
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_CheckForProperNouns</H3>
+<PRE>
+boolean <B>m_CheckForProperNouns</B></PRE>
+<DL>
+<DD>Determines whether check for proper nouns is performed
+<P>
+<DL>
+</DL>
+</DL>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="KEAPhraseFilter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="KEAPhraseFilter.html">KEAPhraseFilter</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+<A NAME="serializedForm"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor">
+<TD COLSPAN=1><FONT SIZE="+2">
+<B>Serialized Fields</B></FONT></TD>
+</TR>
+</TABLE>
+
+<H3>
+m_SelectCols</H3>
+<PRE>
+weka.core.Range <B>m_SelectCols</B></PRE>
+<DL>
+<DD>Stores which columns to select as a funky range
+<P>
+<DL>
+</DL>
+</DL>
+<HR>
+
+<H3>
+m_DisallowInternalPeriods</H3>
+<PRE>
+boolean <B>m_DisallowInternalPeriods</B></PRE>
+<DL>
+<DD>Determines whether internal periods are allowed
+<P>
+<DL>
+</DL>
+</DL>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="LovinsStemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="LovinsStemmer.html">LovinsStemmer</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="NumbersFilter"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="NumbersFilter.html">NumbersFilter</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="Stemmer"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="Stemmer.html">Stemmer</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="Stopwords"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="Stopwords.html">Stopwords</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="StopwordsEnglish"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="StopwordsEnglish.html">StopwordsEnglish</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR SIZE="4" NOSHADE>
+<A NAME="StopwordsGerman"><!-- --></A>
+<TABLE BORDER="1" CELLPADDING="3" CELLSPACING="0" WIDTH="100%">
+<TR BGCOLOR="#CCCCFF" CLASS="TableSubHeadingColor">
+<TD COLSPAN=2><FONT SIZE="+2">
+<B>Class <A HREF="StopwordsGerman.html">StopwordsGerman</A> implements Serializable</B></FONT></TD>
+</TR>
+</TABLE>
+
+<P>
+
+<P>
+<HR>
+
+<!-- ========== START OF NAVBAR ========== -->
+<A NAME="navbar_bottom"><!-- --></A>
+<TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0">
+<TR>
+<TD COLSPAN=3 BGCOLOR="#EEEEFF" CLASS="NavBarCell1">
+<A NAME="navbar_bottom_firstrow"><!-- --></A>
+<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3">
+  <TR ALIGN="center" VALIGN="top">
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Package</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <FONT CLASS="NavBarFont1">Class</FONT>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="overview-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A>&nbsp;</TD>
+  <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1">    <A HREF="help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A>&nbsp;</TD>
+  </TR>
+</TABLE>
+</TD>
+<TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM>
+</EM>
+</TD>
+</TR>
+
+<TR>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+&nbsp;PREV&nbsp;
+&nbsp;NEXT</FONT></TD>
+<TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2">
+  <A HREF="index.html" TARGET="_top"><B>FRAMES</B></A>  &nbsp;
+&nbsp;<A HREF="serialized-form.html" TARGET="_top"><B>NO FRAMES</B></A>  &nbsp;
+&nbsp;
+<SCRIPT>
+  <!--
+  if(window==top) {
+    document.writeln('<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>');
+  }
+  //-->
+</SCRIPT>
+<NOSCRIPT>
+<A HREF="allclasses-noframe.html" TARGET=""><B>All Classes</B></A>
+</NOSCRIPT>
+</FONT></TD>
+</TR>
+</TABLE>
+<!-- =========== END OF NAVBAR =========== -->
+
+<HR>
+
+</BODY>
+</HTML>
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/stylesheet.css
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/stylesheet.css	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/doc/stylesheet.css	(revision 21112)
@@ -0,0 +1,29 @@
+/* Javadoc style sheet */
+
+/* Define colors, fonts and other style attributes here to override the defaults */
+
+/* Page background color */
+body { background-color: #FFFFFF }
+
+/* Table colors */
+.TableHeadingColor     { background: #CCCCFF } /* Dark mauve */
+.TableSubHeadingColor  { background: #EEEEFF } /* Light mauve */
+.TableRowColor         { background: #FFFFFF } /* White */
+
+/* Font used in left-hand frame lists */
+.FrameTitleFont   { font-size: 10pts; font-family: Helvetica, Arial, san-serif }
+.FrameHeadingFont { font-size: 10pts; font-family: Helvetica, Arial, san-serif }
+.FrameItemFont    { font-size: 10pts; font-family: Helvetica, Arial, san-serif }
+
+/* Example of smaller, sans-serif font in frames */
+/* .FrameItemFont  { font-size: 10pt; font-family: Helvetica, Arial, sans-serif } */
+
+/* Navigation bar fonts and colors */
+.NavBarCell1    { background-color:#EEEEFF;}/* Light mauve */
+.NavBarCell1Rev { background-color:#00008B;}/* Dark Blue */
+.NavBarFont1    { font-family: Arial, Helvetica, sans-serif; color:#000000;}
+.NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;}
+
+.NavBarCell2    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}
+.NavBarCell3    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/Classifier.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/Classifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/Classifier.java	(revision 21112)
@@ -0,0 +1,115 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Classifier.java
+ *    Copyright (C) 1999 Eibe Frank, Len Trigg
+ *
+ */
+
+package weka.classifiers;
+
+import java.io.Serializable;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.SerializedObject;
+import weka.core.Utils;
+
+
+/** 
+ * Abstract classifier. All schemes for numeric or nominal prediction in
+ * Weka extend this class.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public abstract class Classifier implements Cloneable, Serializable {
+ 
+  /**
+   * Generates a classifier. Must initialize all fields of the classifier
+   * that are not being set via options (ie. multiple calls of buildClassifier
+   * must always lead to the same result). Must not change the dataset
+   * in any way.
+   *
+   * @param data set of instances serving as training data 
+   * @exception Exception if the classifier has not been 
+   * generated successfully
+   */
+  public abstract void buildClassifier(Instances data) throws Exception;
+
+  /**
+   * Classifies a given instance.
+   *
+   * @param instance the instance to be classified
+   * @return index of the predicted class as a double
+   * if the class is nominal, otherwise the predicted value
+   * @exception Exception if instance could not be classified
+   * successfully
+   */
+  public abstract double classifyInstance(Instance instance) throws Exception; 
+
+
+  
+  /**
+   * Creates a new instance of a classifier given it's class name and
+   * (optional) arguments to pass to it's setOptions method. If the
+   * classifier implements OptionHandler and the options parameter is
+   * non-null, the classifier will have it's options set.
+   *
+   * @param classifierName the fully qualified class name of the classifier
+   * @param options an array of options suitable for passing to setOptions. May
+   * be null.
+   * @return the newly created classifier, ready for use.
+   * @exception Exception if the classifier name is invalid, or the options
+   * supplied are not acceptable to the classifier
+   */
+  public static Classifier forName(String classifierName,
+				   String [] options) throws Exception {
+
+    return (Classifier)Utils.forName(Classifier.class,
+				     classifierName,
+				     options);
+  }
+
+  /**
+   * Creates copies of the current classifier, which can then
+   * be used for boosting etc. Note that this method now uses
+   * Serialization to perform a deep copy, so the Classifier
+   * object must be fully Serializable. Any currently built model
+   * will now be copied as well.
+   *
+   * @param model an example classifier to copy
+   * @param num the number of classifiers copies to create.
+   * @return an array of classifiers.
+   * @exception Exception if an error occurs
+   */
+  public static Classifier [] makeCopies(Classifier model,
+					 int num) throws Exception {
+
+    if (model == null) {
+      throw new Exception("No model classifier set");
+    }
+    Classifier [] classifiers = new Classifier [num];
+    SerializedObject so = new SerializedObject(model);
+    for(int i = 0; i < classifiers.length; i++) {
+      classifiers[i] = (Classifier) so.getObject();
+    }
+    return classifiers;
+  }
+
+}
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/DistributionClassifier.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/DistributionClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/DistributionClassifier.java	(revision 21112)
@@ -0,0 +1,92 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    DistributionClassifier.java
+ *    Copyright (C) 1999 Eibe Frank, Len Trigg
+ *
+ */
+
+package weka.classifiers;
+
+import weka.core.*;
+
+/** 
+ * Abstract classification model that produces (for each test instance)
+ * an estimate of the membership in each class 
+ * (ie. a probability distribution).
+ *
+ * @author   Eibe Frank (trigg@cs.waikato.ac.nz)
+ * @author   Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public abstract class DistributionClassifier extends Classifier {
+
+  /**
+   * Predicts the class memberships for a given instance. If
+   * an instance is unclassified, the returned array elements
+   * must be all zero. If the class is numeric, the array
+   * must consist of only one element, which contains the
+   * predicted value.
+   *
+   * @param instance the instance to be classified
+   * @return an array containing the estimated membership 
+   * probabilities of the test instance in each class (this 
+   * should sum to at most 1)
+   * @exception Exception if distribution could not be 
+   * computed successfully
+   */
+  public abstract double[] distributionForInstance(Instance instance) 
+       throws Exception;
+
+  /**
+   * Classifies the given test instance. The instance has to belong to a
+   * dataset when it's being classified.
+   *
+   * @param instance the instance to be classified
+   * @return the predicted most likely class for the instance or 
+   * Instance.missingValue() if no prediction is made
+   * @exception Exception if an error occurred during the prediction
+   */
+  public double classifyInstance(Instance instance) throws Exception {
+
+    double [] dist = distributionForInstance(instance);
+    if (dist == null) {
+      throw new Exception("Null distribution predicted");
+    }
+    switch (instance.classAttribute().type()) {
+    case Attribute.NOMINAL:
+      double max = 0;
+      int maxIndex = 0;
+      
+      for (int i = 0; i < dist.length; i++) {
+	if (dist[i] > max) {
+	  maxIndex = i;
+	  max = dist[i];
+	}
+      }
+      if (max > 0) {
+	return maxIndex;
+      } else {
+	return Instance.missingValue();
+      }
+    case Attribute.NUMERIC:
+      return dist[0];
+    default:
+      return Instance.missingValue();
+    }
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/FilteredClassifier.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/FilteredClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/FilteredClassifier.java	(revision 21112)
@@ -0,0 +1,374 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    FilteredClassifier.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+package weka.classifiers;
+
+import java.util.Enumeration;
+import java.util.Vector;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.Option;
+import weka.core.OptionHandler;
+import weka.core.Utils;
+import weka.filters.Filter;
+import weka.core.Attribute;
+
+
+/**
+ * Class for running an arbitrary classifier on data that has been passed
+ * through an arbitrary filter.<p>
+ *
+ * Valid options from the command line are:<p>
+ *
+ * -B classifierstring <br>
+ * Classifierstring should contain the full class name of a classifier
+ * followed by options to the classifier.
+ * (required).<p>
+ *
+ * -F filterstring <br>
+ * Filterstring should contain the full class name of a filter
+ * followed by options to the filter.
+ * (required).<p>
+ *
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class FilteredClassifier extends DistributionClassifier
+  implements OptionHandler {
+
+  /** The classifier */
+  protected Classifier m_Classifier = new weka.classifiers.NaiveBayesSimple();
+
+  /** The filter */
+  protected Filter m_Filter = new weka.filters.DiscretizeFilter();
+
+  /** The instance structure of the filtered instances */
+  protected Instances m_FilteredInstances;
+  
+  /**
+   * Default constructor specifying NaiveBayesSimple as the classifier and
+   * DiscretizeFilter as the filter. Both of these are just placeholders
+   * for more useful selections.
+   */
+  public FilteredClassifier() {
+
+    this(new weka.classifiers.NaiveBayesSimple(), 
+	 new weka.filters.DiscretizeFilter());
+  }
+
+  /**
+   * Constructor that specifies the subclassifier and filter to use.
+   *
+   * @param classifier the Classifier to receive filtered instances.
+   * @param filter the Filter that will process instances before
+   * passing to the Classifier.
+   */
+  public FilteredClassifier(Classifier classifier, Filter filter) {
+
+    m_Classifier = classifier;
+    m_Filter = filter;
+  }
+
+  /**
+   * Returns an enumeration describing the available options
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(2);
+
+    newVector.addElement(new Option(
+	      "\tFull class name of classifier to use, followed\n"
+	      + "\tby scheme options. (required)\n"
+	      + "\teg: \"weka.classifiers.NaiveBayes -D\"",
+	      "B", 1, "-B <classifier specification>"));
+    newVector.addElement(new Option(
+	      "\tFull class name of filter to use, followed\n"
+	      + "\tby filter options. (required)\n"
+	      + "\teg: \"weka.filters.AttributeFilter -V -R 1,2\"",
+	      "F", 1, "-F <filter specification>"));
+    return newVector.elements();
+  }
+
+  /**
+   * Parses a given list of options. Valid options are:<p>
+   *
+   * -B classifierstring <br>
+   * Classifierstring should contain the full class name of a classifier
+   * followed by options to the classifier.
+   * (required).<p>
+   *
+   * -F filterstring <br>
+   * Filterstring should contain the full class name of a filter
+   * followed by options to the filter.
+   * (required).<p>
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    String classifierString = Utils.getOption('B', options);
+    if (classifierString.length() == 0) {
+      throw new Exception("A classifier must be specified"
+			  + " with the -B option.");
+    }
+    String [] classifierSpec = Utils.splitOptions(classifierString);
+    if (classifierSpec.length == 0) {
+      throw new Exception("Invalid classifier specification string");
+    }
+    String classifierName = classifierSpec[0];
+    classifierSpec[0] = "";
+    setClassifier(Classifier.forName(classifierName, classifierSpec));
+    
+    // Same for filter
+    String filterString = Utils.getOption('F', options);
+    if (filterString.length() == 0) {
+      throw new Exception("A filter must be specified"
+			  + " with the -F option.");
+    }
+    String [] filterSpec = Utils.splitOptions(filterString);
+    if (filterSpec.length == 0) {
+      throw new Exception("Invalid filter specification string");
+    }
+    String filterName = filterSpec[0];
+    filterSpec[0] = "";
+    setFilter((Filter) Utils.forName(Filter.class, filterName, filterSpec));
+  }
+
+  /**
+   * Gets the current settings of the Classifier.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [4];
+    int current = 0;
+
+    options[current++] = "-B";
+    options[current++] = "" + getClassifierSpec();
+
+    // Same for filter
+    options[current++] = "-F";
+    options[current++] = "" + getFilterSpec();
+    
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+  /**
+   * Sets the classifier
+   *
+   * @param classifier the classifier with all options set.
+   */
+  public void setClassifier(Classifier classifier) {
+
+    m_Classifier = classifier;
+  }
+
+  /**
+   * Gets the classifier used.
+   *
+   * @return the classifier
+   */
+  public Classifier getClassifier() {
+
+    return m_Classifier;
+  }
+  
+  /**
+   * Gets the classifier specification string, which contains the class name of
+   * the classifier and any options to the classifier
+   *
+   * @return the classifier string.
+   */
+  protected String getClassifierSpec() {
+    
+    Classifier c = getClassifier();
+    if (c instanceof OptionHandler) {
+      return c.getClass().getName() + " "
+	+ Utils.joinOptions(((OptionHandler)c).getOptions());
+    }
+    return c.getClass().getName();
+  }
+
+  /**
+   * Sets the filter
+   *
+   * @param filter the filter with all options set.
+   */
+  public void setFilter(Filter filter) {
+
+    m_Filter = filter;
+  }
+
+  /**
+   * Gets the filter used.
+   *
+   * @return the filter
+   */
+  public Filter getFilter() {
+
+    return m_Filter;
+  }
+  
+  /**
+   * Gets the filter specification string, which contains the class name of
+   * the filter and any options to the filter
+   *
+   * @return the filter string.
+   */
+  protected String getFilterSpec() {
+    
+    Filter c = getFilter();
+    if (c instanceof OptionHandler) {
+      return c.getClass().getName() + " "
+	+ Utils.joinOptions(((OptionHandler)c).getOptions());
+    }
+    return c.getClass().getName();
+  }
+
+  /**
+   * Build the classifier on the filtered data.
+   *
+   * @param data the training data
+   * @exception Exception if the classifier could not be built successfully
+   */
+  public void buildClassifier(Instances data) throws Exception {
+
+    if (m_Classifier == null) {
+      throw new Exception("No base classifiers have been set!");
+    }
+    /*
+    String fname = m_Filter.getClass().getName();
+    fname = fname.substring(fname.lastIndexOf('.') + 1);
+    util.Timer t = util.Timer.getTimer("FilteredClassifier::" + fname);
+    t.start();
+    */
+    m_Filter.setInputFormat(data);
+    data = Filter.useFilter(data, m_Filter);
+    //t.stop();
+    m_FilteredInstances = data.stringFreeStructure();
+    m_Classifier.buildClassifier(data);
+  }
+
+  /**
+   * Classifies a given instance after filtering.
+   *
+   * @param instance the instance to be classified
+   * @exception Exception if instance could not be classified
+   * successfully
+   */
+  public double [] distributionForInstance(Instance instance)
+    throws Exception {
+
+    /*
+      System.err.println("FilteredClassifier:: " 
+                         + m_Filter.getClass().getName()
+                         + " in: " + instance);
+    */
+    if (m_Filter.numPendingOutput() > 0) {
+      throw new Exception("Filter output queue not empty!");
+    }
+    /*
+    String fname = m_Filter.getClass().getName();
+    fname = fname.substring(fname.lastIndexOf('.') + 1);
+    util.Timer t = util.Timer.getTimer("FilteredClassifier::" + fname);
+    t.start();
+    */
+    if (!m_Filter.input(instance)) {
+      throw new Exception("Filter didn't make the test instance"
+			  + " immediately available!");
+    }
+    m_Filter.batchFinished();
+    Instance newInstance = m_Filter.output();
+    //t.stop();
+    /*
+    System.err.println("FilteredClassifier:: " 
+                       + m_Filter.getClass().getName()
+                       + " out: " + newInstance);
+    */
+    if (m_Classifier instanceof DistributionClassifier) {
+      return ((DistributionClassifier)m_Classifier)
+	.distributionForInstance(newInstance);
+    }
+    double pred = m_Classifier.classifyInstance(newInstance);
+    double [] result = new double [m_FilteredInstances.numClasses()];
+    if (Instance.isMissingValue(pred)) {
+      return result;
+    }
+    switch (instance.classAttribute().type()) {
+    case Attribute.NOMINAL:
+      result[(int) pred] = 1.0;
+      break;
+    case Attribute.NUMERIC:
+      result[0] = pred;
+      break;
+    default:
+      throw new Exception("Unknown class type");
+    }
+    return result;
+  }
+
+  /**
+   * Output a representation of this classifier
+   */
+  public String toString() {
+
+    if (m_FilteredInstances == null) {
+      return "FilteredClassifier: No model built yet.";
+    }
+
+    String result = "FilteredClassifier using "
+      + getClassifierSpec()
+      + " on data filtered through "
+      + getFilterSpec()
+      + "\n\nFiltered Header\n"
+      + m_FilteredInstances.toString()
+      + "\n\nClassifier Model\n"
+      + m_Classifier.toString();
+    return result;
+  }
+
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain the following arguments:
+   * -t training file [-T test file] [-c class index]
+   */
+  public static void main(String [] argv) {
+
+    try {
+      System.out.println("Evaluation disabled!");
+      //System.out.println(Evaluation.evaluateModel(new FilteredClassifier(),
+      //					  argv));
+    } catch (Exception e) {
+      System.err.println(e.getMessage());
+    }
+  }
+
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/NaiveBayesSimple.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/NaiveBayesSimple.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/classifiers/NaiveBayesSimple.java	(revision 21112)
@@ -0,0 +1,331 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    NaiveBayesSimple.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.classifiers;
+
+import java.io.*;
+import java.util.*;
+import weka.core.*;
+
+/**
+ * Class for building and using a simple Naive Bayes classifier.
+ * Numeric attributes are modelled by a normal distribution. For more
+ * information, see<p>
+ *
+ * Richard Duda and Peter Hart (1973).<i>Pattern
+ * Classification and Scene Analysis</i>. Wiley, New York.
+
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$ 
+*/
+public class NaiveBayesSimple extends DistributionClassifier {
+
+  /** All the counts for nominal attributes. */
+  private double [][][] m_Counts;
+  
+  /** The means for numeric attributes. */
+  private double [][] m_Means;
+
+  /** The standard deviations for numeric attributes. */
+  private double [][] m_Devs;
+
+  /** The prior probabilities of the classes. */
+  private double [] m_Priors;
+
+  /** The instances used for training. */
+  private Instances m_Instances;
+
+  /** Constant for normal distribution. */
+  private static double NORM_CONST = Math.sqrt(2 * Math.PI);
+
+  /**
+   * Generates the classifier.
+   *
+   * @param instances set of instances serving as training data 
+   * @exception Exception if the classifier has not been generated successfully
+   */
+  public void buildClassifier(Instances instances) throws Exception {
+
+    int attIndex = 0;
+    double sum;
+    
+    if (instances.checkForStringAttributes()) {
+      throw new Exception("Can't handle string attributes!");
+    }
+    if (instances.classAttribute().isNumeric()) {
+      throw new Exception("Naive Bayes: Class is numeric!");
+    }
+    
+    m_Instances = new Instances(instances, 0);
+    
+    // Reserve space
+    m_Counts = new double[instances.numClasses()]
+      [instances.numAttributes() - 1][0];
+    m_Means = new double[instances.numClasses()]
+      [instances.numAttributes() - 1];
+    m_Devs = new double[instances.numClasses()]
+      [instances.numAttributes() - 1];
+    m_Priors = new double[instances.numClasses()];
+    Enumeration enum = instances.enumerateAttributes();
+    while (enum.hasMoreElements()) {
+      Attribute attribute = (Attribute) enum.nextElement();
+      if (attribute.isNominal()) {
+	for (int j = 0; j < instances.numClasses(); j++) {
+	  m_Counts[j][attIndex] = new double[attribute.numValues()];
+	}
+      } else {
+	for (int j = 0; j < instances.numClasses(); j++) {
+	  m_Counts[j][attIndex] = new double[1];
+	}
+      }
+      attIndex++;
+    }
+    
+    // Compute counts and sums
+    Enumeration enumInsts = instances.enumerateInstances();
+    while (enumInsts.hasMoreElements()) {
+      Instance instance = (Instance) enumInsts.nextElement();
+      if (!instance.classIsMissing()) {
+	Enumeration enumAtts = instances.enumerateAttributes();
+	attIndex = 0;
+	while (enumAtts.hasMoreElements()) {
+	  Attribute attribute = (Attribute) enumAtts.nextElement();
+	  if (!instance.isMissing(attribute)) {
+	    if (attribute.isNominal()) {
+	      m_Counts[(int)instance.classValue()][attIndex]
+		[(int)instance.value(attribute)]++;
+	    } else {
+	      m_Means[(int)instance.classValue()][attIndex] +=
+		instance.value(attribute);
+	      m_Counts[(int)instance.classValue()][attIndex][0]++;
+	    }
+	  }
+	  attIndex++;
+	}
+	m_Priors[(int)instance.classValue()]++;
+      }
+    }
+    
+    // Compute means
+    Enumeration enumAtts = instances.enumerateAttributes();
+    attIndex = 0;
+    while (enumAtts.hasMoreElements()) {
+      Attribute attribute = (Attribute) enumAtts.nextElement();
+      if (attribute.isNumeric()) {
+	for (int j = 0; j < instances.numClasses(); j++) {
+	  if (m_Counts[j][attIndex][0] < 2) {
+	    throw new Exception("attribute " + attribute.name() +
+				": less than two values for class " +
+				instances.classAttribute().value(j));
+	  }
+	  m_Means[j][attIndex] /= m_Counts[j][attIndex][0];
+	}
+      }
+      attIndex++;
+    }    
+    
+    // Compute standard deviations
+    enumInsts = instances.enumerateInstances();
+    while (enumInsts.hasMoreElements()) {
+      Instance instance = 
+	(Instance) enumInsts.nextElement();
+      if (!instance.classIsMissing()) {
+	enumAtts = instances.enumerateAttributes();
+	attIndex = 0;
+	while (enumAtts.hasMoreElements()) {
+	  Attribute attribute = (Attribute) enumAtts.nextElement();
+	  if (!instance.isMissing(attribute)) {
+	    if (attribute.isNumeric()) {
+	      m_Devs[(int)instance.classValue()][attIndex] +=
+		(m_Means[(int)instance.classValue()][attIndex]-
+		 instance.value(attribute))*
+		(m_Means[(int)instance.classValue()][attIndex]-
+		 instance.value(attribute));
+	    }
+	  }
+	  attIndex++;
+	}
+      }
+    }
+    enumAtts = instances.enumerateAttributes();
+    attIndex = 0;
+    while (enumAtts.hasMoreElements()) {
+      Attribute attribute = (Attribute) enumAtts.nextElement();
+      if (attribute.isNumeric()) {
+	for (int j = 0; j < instances.numClasses(); j++) {
+	  if (m_Devs[j][attIndex] <= 0) {
+	    throw new Exception("attribute " + attribute.name() +
+				": standard deviation is 0 for class " +
+				instances.classAttribute().value(j));
+	  }
+	  else {
+	    m_Devs[j][attIndex] /= m_Counts[j][attIndex][0] - 1;
+	    m_Devs[j][attIndex] = Math.sqrt(m_Devs[j][attIndex]);
+	  }
+	}
+      }
+      attIndex++;
+    } 
+    
+    // Normalize counts
+    enumAtts = instances.enumerateAttributes();
+    attIndex = 0;
+    while (enumAtts.hasMoreElements()) {
+      Attribute attribute = (Attribute) enumAtts.nextElement();
+      if (attribute.isNominal()) {
+	for (int j = 0; j < instances.numClasses(); j++) {
+	  sum = Utils.sum(m_Counts[j][attIndex]);
+	  for (int i = 0; i < attribute.numValues(); i++) {
+	    m_Counts[j][attIndex][i] =
+	      (m_Counts[j][attIndex][i] + 1) 
+	      / (sum + (double)attribute.numValues());
+	  }
+	}
+      }
+      attIndex++;
+    }
+    
+    // Normalize priors
+    sum = Utils.sum(m_Priors);
+    for (int j = 0; j < instances.numClasses(); j++)
+      m_Priors[j] = (m_Priors[j] + 1) 
+	/ (sum + (double)instances.numClasses());
+  }
+
+  /**
+   * Calculates the class membership probabilities for the given test instance.
+   *
+   * @param instance the instance to be classified
+   * @return predicted class probability distribution
+   * @exception Exception if distribution can't be computed
+   */
+  public double[] distributionForInstance(Instance instance) throws Exception {
+    
+    double [] probs = new double[instance.numClasses()];
+    int attIndex;
+    
+    for (int j = 0; j < instance.numClasses(); j++) {
+      probs[j] = 1;
+      Enumeration enumAtts = instance.enumerateAttributes();
+      attIndex = 0;
+      while (enumAtts.hasMoreElements()) {
+	Attribute attribute = (Attribute) enumAtts.nextElement();
+	if (!instance.isMissing(attribute)) {
+	  if (attribute.isNominal()) {
+	    probs[j] *= m_Counts[j][attIndex][(int)instance.value(attribute)];
+	  } else {
+	    probs[j] *= normalDens(instance.value(attribute),
+				   m_Means[j][attIndex],
+				   m_Devs[j][attIndex]);}
+	}
+	attIndex++;
+      }
+      probs[j] *= m_Priors[j];
+    }
+    
+    // Normalize probabilities
+    Utils.normalize(probs);
+
+    return probs;
+  }
+
+  /**
+   * Returns a description of the classifier.
+   *
+   * @return a description of the classifier as a string.
+   */
+  public String toString() {
+
+    if (m_Instances == null) {
+      return "Naive Bayes (simple): No model built yet.";
+    }
+    try {
+      StringBuffer text = new StringBuffer("Naive Bayes (simple)");
+      int attIndex;
+      
+      for (int i = 0; i < m_Instances.numClasses(); i++) {
+	text.append("\n\nClass " + m_Instances.classAttribute().value(i) 
+		    + ": P(C) = " 
+		    + Utils.doubleToString(m_Priors[i], 10, 8)
+		    + "\n\n");
+	Enumeration enumAtts = m_Instances.enumerateAttributes();
+	attIndex = 0;
+	while (enumAtts.hasMoreElements()) {
+	  Attribute attribute = (Attribute) enumAtts.nextElement();
+	  text.append("Attribute " + attribute.name() + "\n");
+	  if (attribute.isNominal()) {
+	    for (int j = 0; j < attribute.numValues(); j++) {
+	      text.append(attribute.value(j) + "\t");
+	    }
+	    text.append("\n");
+	    for (int j = 0; j < attribute.numValues(); j++)
+	      text.append(Utils.
+			  doubleToString(m_Counts[i][attIndex][j], 10, 8)
+			  + "\t");
+	  } else {
+	    text.append("Mean: " + Utils.
+			doubleToString(m_Means[i][attIndex], 10, 8) + "\t");
+	    text.append("Standard Deviation: " 
+			+ Utils.doubleToString(m_Devs[i][attIndex], 10, 8));
+	  }
+	  text.append("\n\n");
+	  attIndex++;
+	}
+      }
+      
+      return text.toString();
+    } catch (Exception e) {
+      return "Can't print Naive Bayes classifier!";
+    }
+  }
+
+  /**
+   * Density function of normal distribution.
+   */
+  private double normalDens(double x, double mean, double stdDev) {
+    
+    double diff = x - mean;
+    
+    return (1 / (NORM_CONST * stdDev)) 
+      * Math.exp(-(diff * diff / (2 * stdDev * stdDev)));
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv the options
+   */
+  public static void main(String [] argv) {
+
+    Classifier scheme;
+
+    try {
+      scheme = new NaiveBayesSimple();
+      System.out.println("Evaluation disabled!");
+      //System.out.println(Evaluation.evaluateModel(scheme, argv));
+    } catch (Exception e) {
+      System.err.println(e.getMessage());
+    }
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Attribute.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Attribute.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Attribute.java	(revision 21112)
@@ -0,0 +1,682 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Attribute.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.io.*;
+import java.util.*;
+
+/** 
+ * Class for handling an attribute. Once an attribute has been created,
+ * it can't be changed. <p>
+ *
+ * Three attribute types are supported:
+ * <ul>
+ *    <li> numeric: <ul>
+ *         This type of attribute represents a floating-point number.
+ *    </ul>
+ *    <li> nominal: <ul>
+ *         This type of attribute represents a fixed set of nominal values.
+ *    </ul>
+ *    <li> string: <ul>
+ *         This type of attribute represents a dynamically expanding set of
+ *         nominal values. String attributes are not used by the learning
+ *         schemes in Weka. They can be used, for example,  to store an 
+ *         identifier with each instance in a dataset.
+ *    </ul>
+ * </ul>
+ * Typical usage (code from the main() method of this class): <p>
+ *
+ * <code>
+ * ... <br>
+ *
+ * // Create numeric attributes "length" and "weight" <br>
+ * Attribute length = new Attribute("length"); <br>
+ * Attribute weight = new Attribute("weight"); <br><br>
+ * 
+ * // Create vector to hold nominal values "first", "second", "third" <br>
+ * FastVector my_nominal_values = new FastVector(3); <br>
+ * my_nominal_values.addElement("first"); <br>
+ * my_nominal_values.addElement("second"); <br>
+ * my_nominal_values.addElement("third"); <br><br>
+ *
+ * // Create nominal attribute "position" <br>
+ * Attribute position = new Attribute("position", my_nominal_values);<br>
+ *
+ * ... <br>
+ * </code><p>
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Attribute implements Copyable, Serializable {
+
+  /** Constant set for numeric attributes. */
+  public final static int NUMERIC = 0;
+
+  /** Constant set for nominal attributes. */
+  public final static int NOMINAL = 1;
+
+  /** Constant set for attributes with string values. */
+  public final static int STRING = 2;
+
+  /** Strings longer than this will be stored compressed. */
+  private final static int STRING_COMPRESS_THRESHOLD = 200;
+  //private final static int STRING_COMPRESS_THRESHOLD = Integer.MAX_VALUE;
+
+  /** The attribute's name. */
+  private String m_Name;
+
+  /** The attribute's type. */
+  private int m_Type;
+
+  /** The attribute's values (if nominal or string). */
+  private FastVector m_Values;
+
+  /** Mapping of values to indices (if nominal or string). */
+  private Hashtable m_Hashtable;
+
+  /** The attribute's index. */
+  private int m_Index;
+
+  /**
+   * Constructor for a numeric attribute.
+   *
+   * @param attributeName the name for the attribute
+   */
+  public Attribute(String attributeName) {
+
+    m_Name = attributeName;
+    m_Index = -1;
+    m_Values = null;
+    m_Hashtable = null;
+    m_Type = NUMERIC;
+  }
+
+  /**
+   * Constructor for nominal attributes and string attributes.
+   * If a null vector of attribute values is passed to the method,
+   * the attribute is assumed to be a string.
+   *
+   * @param attributeName the name for the attribute
+   * @param attributeValues a vector of strings denoting the 
+   * attribute values. Null if the attribute is a string attribute.
+   */
+  public Attribute(String attributeName, 
+		   FastVector attributeValues) {
+
+    m_Name = attributeName;
+    m_Index = -1;
+    if (attributeValues == null) {
+      m_Values = new FastVector();
+      m_Hashtable = new Hashtable();
+      m_Type = STRING;
+    } else {
+      m_Values = (FastVector) attributeValues.copy();
+      m_Hashtable = new Hashtable(m_Values.size());
+      for (int i = 0; i < m_Values.size(); i++) {
+	m_Hashtable.put(m_Values.elementAt(i), new Integer(i));
+      }
+      m_Type = NOMINAL;
+    }
+  }
+
+  /**
+   * Produces a shallow copy of this attribute.
+   *
+   * @return a copy of this attribute with the same index
+   */
+  public Object copy() {
+
+    Attribute copy = new Attribute(m_Name);
+
+    copy.m_Index = m_Index;
+    if (!isNominal() && !isString())
+      return copy;
+    copy.m_Type = m_Type;
+    copy.m_Values = m_Values;
+    copy.m_Hashtable = m_Hashtable;
+ 
+    return copy;
+  }
+
+  /**
+   * Returns an enumeration of all the attribute's values if
+   * the attribute is nominal or a string, null otherwise. 
+   *
+   * @return enumeration of all the attribute's values
+   */
+  public final Enumeration enumerateValues() {
+
+    if (isNominal() || isString()) {
+      final Enumeration ee = m_Values.elements();
+      return new Enumeration () {
+          public boolean hasMoreElements() {
+            return ee.hasMoreElements();
+          }
+          public Object nextElement() {
+            Object oo = ee.nextElement();
+            if (oo instanceof SerializedObject) {
+              return ((SerializedObject)oo).getObject();
+            } else {
+              return oo;
+            }
+          }
+        };
+    }
+    return null;
+  }
+
+  /**
+   * Tests if given attribute is equal to this attribute.
+   *
+   * @param other the Object to be compared to this attribute
+   * @return true if the given attribute is equal to this attribute
+   */
+  public final boolean equals(Object other) {
+
+    if ((other == null) || !(other.getClass().equals(this.getClass()))) {
+      return false;
+    }
+    Attribute att = (Attribute) other;
+    if (!m_Name.equals(att.m_Name)) {
+      return false;
+    }
+    if (isNumeric() && att.isNumeric()) {
+      return true;
+    }
+    if (isNumeric() || att.isNumeric()) {
+      return false;
+    }
+    if (m_Values.size() != att.m_Values.size()) {
+      return false;
+    }
+    for (int i = 0; i < m_Values.size(); i++) {
+      if (!m_Values.elementAt(i).equals(att.m_Values.elementAt(i))) {
+	return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns the index of this attribute.
+   *
+   * @return the index of this attribute
+   */
+  public final int index() {
+
+    return m_Index;
+  }
+
+  /**
+   * Returns the index of a given attribute value. (The index of
+   * the first occurence of this value.)
+   *
+   * @param value the value for which the index is to be returned
+   * @return the index of the given attribute value if attribute
+   * is nominal or a string, -1 if it is numeric or the value 
+   * can't be found
+   */
+  public final int indexOfValue(String value) {
+
+    if (!isNominal() && !isString())
+      return -1;
+    Object store = value;
+    if (value.length() > STRING_COMPRESS_THRESHOLD) {
+      try {
+        store = new SerializedObject(value, true);
+      } catch (Exception ex) {
+        System.err.println("Couldn't compress string attribute value -"
+                           + " searching uncompressed.");
+      }
+    }
+    Integer val = (Integer)m_Hashtable.get(store);
+    if (val == null) {
+      throw new IllegalArgumentException("Value \"" + value + 
+					 "\" not found in attribute " +
+					 "declaration!");
+    } else {
+      return val.intValue();
+    }
+  }
+
+  /**
+   * Test if the attribute is nominal.
+   *
+   * @return true if the attribute is nominal
+   */
+  public final boolean isNominal() {
+
+    return (m_Type == NOMINAL);
+  }
+
+  /**
+   * Tests if the attribute is numeric.
+   *
+   * @return true if the attribute is numeric
+   */
+  public final boolean isNumeric() {
+
+    return (m_Type == NUMERIC);
+  }
+
+  /**
+   * Tests if the attribute is a string.
+   *
+   * @return true if the attribute is a string
+   */
+  public final boolean isString() {
+
+    return (m_Type == STRING);
+  }
+
+  /**
+   * Returns the attribute's name.
+   *
+   * @return the attribute's name as a string
+   */
+  public final String name() {
+
+    return m_Name;
+  }
+  
+  /**
+   * Returns the number of attribute values. Returns 0 for numeric attributes.
+   *
+   * @return the number of attribute values
+   */
+  public final int numValues() {
+
+    if (!isNominal() && !isString()) {
+      return 0;
+    } else {
+      return m_Values.size();
+    }
+  }
+
+  /**
+   * Returns a description of this attribute in ARFF format. Quotes
+   * strings if they contain whitespace characters, or if they
+   * are a question mark.
+   *
+   * @return a description of this attribute as a string
+   */
+  public final String toString() {
+    
+    StringBuffer text = new StringBuffer();
+    
+    text.append("@attribute " + Utils.quote(m_Name) + " ");
+    if (isNominal()) {
+      text.append('{');
+      Enumeration enum = enumerateValues();
+      while (enum.hasMoreElements()) {
+	text.append(Utils.quote((String) enum.nextElement()));
+	if (enum.hasMoreElements())
+	  text.append(',');
+      }
+      text.append('}');
+    } else {
+      if (isNumeric()) {
+	text.append("numeric");
+      } else {
+	text.append("string");
+      }
+    }
+    return text.toString();
+  }
+
+  /**
+   * Returns the attribute's type as an integer.
+   *
+   * @returns the attribute's type.
+   */
+  public final int type() {
+
+    return m_Type;
+  }
+
+  /**
+   * Returns a value of a nominal or string attribute. 
+   * Returns an empty string if the attribute is neither
+   * nominal nor a string attribute.
+   *
+   * @param valIndex the value's index
+   * @return the attribute's value as a string
+   */
+  public final String value(int valIndex) {
+    
+    if (!isNominal() && !isString()) {
+      return "";
+    } else {
+      Object val = m_Values.elementAt(valIndex);
+      
+      // If we're storing strings compressed, uncompress it.
+      if (val instanceof SerializedObject) {
+        val = ((SerializedObject)val).getObject();
+      }
+      return (String) val;
+    }
+  }
+
+  /**
+   * Constructor for a numeric attribute with a particular index.
+   *
+   * @param attributeName the name for the attribute
+   * @param index the attribute's index
+   */
+  Attribute(String attributeName, int index) {
+
+    this(attributeName);
+
+    m_Index = index;
+  }
+
+  /**
+   * Constructor for nominal attributes and string attributes with
+   * a particular index.
+   * If a null vector of attribute values is passed to the method,
+   * the attribute is assumed to be a string.
+   *
+   * @param attributeName the name for the attribute
+   * @param attributeValues a vector of strings denoting the attribute values.
+   * Null if the attribute is a string attribute.
+   * @param index the attribute's index
+   */
+  Attribute(String attributeName, FastVector attributeValues, 
+	    int index) {
+
+    this(attributeName, attributeValues);
+
+    m_Index = index;
+  }
+
+  /**
+   * Adds a string value to the list of valid strings for attributes
+   * of type STRING and returns the index of the string.
+   *
+   * @param value The string value to add
+   * @return the index assigned to the string, or -1 if the attribute is not
+   * of type Attribute.STRING 
+   */
+  public int addStringValue(String value) {
+
+    if (!isString()) {
+      return -1;
+    }
+    Object store = value;
+
+    if (value.length() > STRING_COMPRESS_THRESHOLD) {
+      try {
+        store = new SerializedObject(value, true);
+      } catch (Exception ex) {
+        System.err.println("Couldn't compress string attribute value -"
+                           + " storing uncompressed.");
+      }
+    }
+    Integer index = (Integer)m_Hashtable.get(store);
+    if (index != null) {
+      return index.intValue();
+    } else {
+      int intIndex = m_Values.size();
+      m_Values.addElement(store);
+      m_Hashtable.put(store, new Integer(intIndex));
+      return intIndex;
+    }
+  }
+
+  /**
+   * Adds a string value to the list of valid strings for attributes
+   * of type STRING and returns the index of the string. This method is
+   * more efficient than addStringValue(String) for long strings.
+   *
+   * @param src The Attribute containing the string value to add.
+   * @param int index the index of the string value in the source attribute.
+   * @return the index assigned to the string, or -1 if the attribute is not
+   * of type Attribute.STRING 
+   */
+  public int addStringValue(Attribute src, int index) {
+
+    if (!isString()) {
+      return -1;
+    }
+    Object store = src.m_Values.elementAt(index);
+    Integer oldIndex = (Integer)m_Hashtable.get(store);
+    if (oldIndex != null) {
+      return oldIndex.intValue();
+    } else {
+      int intIndex = m_Values.size();
+      m_Values.addElement(store);
+      m_Hashtable.put(store, new Integer(intIndex));
+      return intIndex;
+    }
+  }
+
+  /**
+   * Adds an attribute value. Creates a fresh list of attribute
+   * values before adding it.
+   *
+   * @param value the attribute value
+   */
+  final void addValue(String value) {
+
+    m_Values = (FastVector)m_Values.copy();
+    m_Hashtable = (Hashtable)m_Hashtable.clone();
+    forceAddValue(value);
+  }
+
+  /**
+   * Produces a shallow copy of this attribute with a new name.
+   *
+   * @param newName the name of the new attribute
+   * @return a copy of this attribute with the same index
+   */
+  final Attribute copy(String newName) {
+
+    Attribute copy = new Attribute(newName);
+
+    copy.m_Index = m_Index;
+    if (!isNominal() && !isString())
+      return copy;
+    copy.m_Type = m_Type;
+    copy.m_Values = m_Values;
+    copy.m_Hashtable = m_Hashtable;
+ 
+    return copy;
+  }
+
+  /**
+   * Removes a value of a nominal or string attribute. Creates a 
+   * fresh list of attribute values before removing it.
+   *
+   * @param index the value's index
+   * @exception IllegalArgumentException if the attribute is not nominal
+   */
+  final void delete(int index) {
+    
+    if (!isNominal() && !isString()) 
+      throw new IllegalArgumentException("Can only remove value of" +
+                                         "nominal or string attribute!");
+    else {
+      m_Values = (FastVector)m_Values.copy();
+      m_Values.removeElementAt(index);
+      Hashtable hash = new Hashtable(m_Hashtable.size());
+      Enumeration enum = hash.keys();
+      while (enum.hasMoreElements()) {
+	String string = (String)enum.nextElement();
+	Integer valIndexObject = (Integer)m_Hashtable.get(string);
+	int valIndex = valIndexObject.intValue();
+	if (valIndex > index) {
+	  hash.put(string, new Integer(valIndex - 1));
+	} else if (valIndex < index) {
+	  hash.put(string, valIndexObject);
+	}
+      }
+      m_Hashtable = hash;
+    }
+  }
+
+  /**
+   * Adds an attribute value.
+   *
+   * @param value the attribute value
+   */
+  final void forceAddValue(String value) {
+
+    Object store = value;
+    if (value.length() > STRING_COMPRESS_THRESHOLD) {
+      try {
+        store = new SerializedObject(value, true);
+      } catch (Exception ex) {
+        System.err.println("Couldn't compress string attribute value -"
+                           + " storing uncompressed.");
+      }
+    }
+    m_Values.addElement(store);
+    m_Hashtable.put(store, new Integer(m_Values.size() - 1));
+  }
+
+  /**
+   * Sets the index of this attribute.
+   *
+   * @param the index of this attribute
+   */
+  final void setIndex(int index) {
+
+    m_Index = index;
+  }
+
+  /**
+   * Sets a value of a nominal attribute or string attribute.
+   * Creates a fresh list of attribute values before it is set.
+   *
+   * @param index the value's index
+   * @param string the value
+   * @exception IllegalArgumentException if the attribute is not nominal or 
+   * string.
+   */
+  final void setValue(int index, String string) {
+
+    if (!isNominal() && !isString()) {
+      throw new IllegalArgumentException("Can only set value of nominal"+
+                                         "or string attribute!");
+    } else {
+      m_Values = (FastVector)m_Values.copy();
+      m_Hashtable = (Hashtable)m_Hashtable.clone();
+      Object store = string;
+      if (string.length() > STRING_COMPRESS_THRESHOLD) {
+        try {
+          store = new SerializedObject(string, true);
+        } catch (Exception ex) {
+          System.err.println("Couldn't compress string attribute value -"
+                             + " storing uncompressed.");
+        }
+      }
+      m_Hashtable.remove(m_Values.elementAt(index));
+      m_Values.setElementAt(store, index);
+      m_Hashtable.put(store, new Integer(index));
+    }
+  }
+
+  /**
+   * Simple main method for testing this class.
+   */
+  public static void main(String[] ops) {
+
+    try {
+      
+      // Create numeric attributes "length" and "weight"
+      Attribute length = new Attribute("length");
+      Attribute weight = new Attribute("weight");
+      
+      // Create vector to hold nominal values "first", "second", "third" 
+      FastVector my_nominal_values = new FastVector(3); 
+      my_nominal_values.addElement("first"); 
+      my_nominal_values.addElement("second"); 
+      my_nominal_values.addElement("third"); 
+      
+      // Create nominal attribute "position" 
+      Attribute position = new Attribute("position", my_nominal_values);
+
+      // Print the name of "position"
+      System.out.println("Name of \"position\": " + position.name());
+
+      // Print the values of "position"
+      Enumeration attValues = position.enumerateValues();
+      while (attValues.hasMoreElements()) {
+	String string = (String)attValues.nextElement();
+	System.out.println("Value of \"position\": " + string);
+      }
+
+      // Shallow copy attribute "position"
+      Attribute copy = (Attribute) position.copy();
+
+      // Test if attributes are the same
+      System.out.println("Copy is the same as original: " + copy.equals(position));
+
+      // Print index of attribute "weight" (should be unset: -1)
+      System.out.println("Index of attribute \"weight\" (should be -1): " + 
+			 weight.index());
+
+      // Print index of value "first" of attribute "position"
+      System.out.println("Index of value \"first\" of \"position\" (should be 0): " +
+			 position.indexOfValue("first"));
+
+      // Tests type of attribute "position"
+      System.out.println("\"position\" is numeric: " + position.isNumeric());
+      System.out.println("\"position\" is nominal: " + position.isNominal());
+      System.out.println("\"position\" is string: " + position.isString());
+
+      // Prints name of attribute "position"
+      System.out.println("Name of \"position\": " + position.name());
+    
+      // Prints number of values of attribute "position"
+      System.out.println("Number of values for \"position\": " + position.numValues());
+
+      // Prints the values (againg)
+      for (int i = 0; i < position.numValues(); i++) {
+	System.out.println("Value " + i + ": " + position.value(i));
+      }
+
+      // Prints the attribute "position" in ARFF format
+      System.out.println(position);
+
+      // Checks type of attribute "position" using constants
+      switch (position.type()) {
+      case Attribute.NUMERIC:
+	System.out.println("\"position\" is numeric");
+	break;
+      case Attribute.NOMINAL:
+	System.out.println("\"position\" is nominal");
+	break;
+      case Attribute.STRING:
+	System.out.println("\"position\" is string");
+	break;
+      default:
+	System.out.println("\"position\" has unknown type");
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}
+  
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/AttributeStats.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/AttributeStats.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/AttributeStats.java	(revision 21112)
@@ -0,0 +1,136 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    AttributeStats.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+package weka.core;
+
+/**
+ * A Utility class that contains summary information on an
+ * the values that appear in a dataset for a particular attribute.
+ *
+ * @author <a href="mailto:len@webmind.com">Len Trigg</a>
+ * @version $Revision$
+ */
+public class AttributeStats {    
+  
+  /** The number of int-like values */
+  public int intCount = 0;
+  
+  /** The number of real-like values (i.e. have a fractional part) */
+  public int realCount = 0;
+  
+  /** The number of missing values */
+  public int missingCount = 0;
+  
+  /** The number of distinct values */
+  public int distinctCount = 0;
+  
+  /** The number of values that only appear once */
+  public int uniqueCount = 0;
+  
+  /** The total number of values (i.e. number of instances) */
+  public int totalCount = 0;
+  
+  /** Stats on numeric value distributions */
+  // perhaps Stats should be moved from weka.experiment to weka.core
+  public weka.experiment.Stats numericStats;
+  
+  /** Counts of each nominal value */
+  public int [] nominalCounts;
+    
+  /**
+   * Updates the counters for one more observed distinct value.
+   *
+   * @param value the value that has just been seen
+   * @param count the number of times the value appeared
+   */
+  protected void addDistinct(double value, int count) {
+    
+    if (count > 0) {
+      if (count == 1) {
+	uniqueCount++;
+	}
+      if (Utils.eq(value, (double)((int)value))) {
+	intCount += count;
+      } else {
+	realCount += count;
+      }
+      if (nominalCounts != null) {
+	nominalCounts[(int)value] = count;
+      }
+      if (numericStats != null) {
+	  numericStats.add(value, count);
+	  numericStats.calculateDerived();
+      }
+    }
+    distinctCount++;
+  }
+
+  /**
+   * Returns a human readable representation of this AttributeStats instance.
+   *
+   * @return a String represtinging these AttributeStats.
+   */
+  public String toString() {
+
+    StringBuffer sb = new StringBuffer();
+    sb.append(Utils.padLeft("Type", 4)).append(Utils.padLeft("Nom", 5));
+    sb.append(Utils.padLeft("Int", 5)).append(Utils.padLeft("Real", 5));
+    sb.append(Utils.padLeft("Missing", 12));
+    sb.append(Utils.padLeft("Unique", 12));
+    sb.append(Utils.padLeft("Dist", 6));
+    if (nominalCounts != null) {
+      sb.append(' ');
+      for (int i = 0; i < nominalCounts.length; i++) {
+        sb.append(Utils.padLeft("C[" + i + "]", 5));
+      }
+    }
+    sb.append('\n');
+
+    long percent;
+    percent = Math.round(100.0 * intCount / totalCount);
+    if (nominalCounts != null) {
+      sb.append(Utils.padLeft("Nom", 4)).append(' ');
+      sb.append(Utils.padLeft("" + percent, 3)).append("% ");
+      sb.append(Utils.padLeft("" + 0, 3)).append("% ");
+    } else {
+      sb.append(Utils.padLeft("Num", 4)).append(' ');
+      sb.append(Utils.padLeft("" + 0, 3)).append("% ");
+      sb.append(Utils.padLeft("" + percent, 3)).append("% ");
+    }
+    percent = Math.round(100.0 * realCount / totalCount);
+    sb.append(Utils.padLeft("" + percent, 3)).append("% ");
+    sb.append(Utils.padLeft("" + missingCount, 5)).append(" /");
+    percent = Math.round(100.0 * missingCount / totalCount);
+    sb.append(Utils.padLeft("" + percent, 3)).append("% ");
+    sb.append(Utils.padLeft("" + uniqueCount, 5)).append(" /");
+    percent = Math.round(100.0 * uniqueCount / totalCount);
+    sb.append(Utils.padLeft("" + percent, 3)).append("% ");
+    sb.append(Utils.padLeft("" + distinctCount, 5)).append(' ');
+    if (nominalCounts != null) {
+      for (int i = 0; i < nominalCounts.length; i++) {
+        sb.append(Utils.padLeft("" + nominalCounts[i], 5));
+      }
+    }
+    sb.append('\n');
+    return sb.toString();
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/ContingencyTables.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/ContingencyTables.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/ContingencyTables.java	(revision 21112)
@@ -0,0 +1,635 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    ContingencyTables.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+/**
+ * Class implementing some statistical routines for contingency tables.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class ContingencyTables {
+
+  /** The natural logarithm of 2 */
+  private static double log2 = Math.log(2);
+
+  /**
+   * Returns chi-squared probability for a given matrix.
+   *
+   * @param matrix the contigency table
+   * @param yates is Yates' correction to be used?
+   * @return the chi-squared probability
+   */
+
+  public static double chiSquared(double [][] matrix, boolean yates) {
+
+    int df = (matrix.length - 1) * (matrix[0].length - 1);
+
+    return Statistics.chiSquaredProbability(chiVal(matrix, yates), df);
+  }
+
+  /**
+   * Computes chi-squared statistic for a contingency table.
+   *
+   * @param matrix the contigency table
+   * @param yates is Yates' correction to be used?
+   * @return the value of the chi-squared statistic
+   */
+  public static double chiVal(double [][] matrix, boolean useYates) {
+    
+    int df, nrows, ncols, row, col;
+    double[] rtotal, ctotal;
+    double expect = 0, chival = 0, n = 0;
+    boolean yates = true;
+    
+    nrows = matrix.length;
+    ncols = matrix[0].length;
+    rtotal = new double [nrows];
+    ctotal = new double [ncols];
+    for (row = 0; row < nrows; row++) {
+      for (col = 0; col < ncols; col++) {
+	rtotal[row] += matrix[row][col];
+	ctotal[col] += matrix[row][col];
+	n += matrix[row][col];
+      }
+    }
+    df = (nrows - 1)*(ncols - 1);
+    if ((df > 1) || (!useYates)) {
+      yates = false;
+    } else if (df <= 0) {
+      return 0;
+    }
+    chival = 0.0;
+    for (row = 0; row < nrows; row++) {
+      if (Utils.gr(rtotal[row], 0)) {
+	for (col = 0; col < ncols; col++) {
+	  if (Utils.gr(ctotal[col], 0)) {
+	    expect = (ctotal[col] * rtotal[row]) / n;
+	    chival += chiCell (matrix[row][col], expect, yates);
+	  }
+	}
+      }
+    }
+    return chival;
+  }
+
+  /**
+   * Tests if Cochran's criterion is fullfilled for the given
+   * contingency table. Rows and columns with all zeros are not considered
+   * relevant.
+   *
+   * @param matrix the contigency table to be tested
+   * @return true if contingency table is ok, false if not
+   */
+  public static boolean cochransCriterion(double[][] matrix) {
+
+    double[] rtotal, ctotal;
+    double n = 0, expect, smallfreq = 5;
+    int smallcount = 0, nonZeroRows = 0, nonZeroColumns = 0, nrows, ncols, 
+      row, col;
+
+    nrows = matrix.length;
+    ncols = matrix[0].length;
+
+    rtotal = new double [nrows];
+    ctotal = new double [ncols];
+    for (row = 0; row < nrows; row++) {
+      for (col = 0; col < ncols; col++) {
+	rtotal[row] += matrix[row][col];
+	ctotal[col] += matrix[row][col];
+	n += matrix[row][col];
+      }
+    }
+    for (row = 0; row < nrows; row++) {
+      if (Utils.gr(rtotal[row], 0)) {
+	nonZeroRows++;
+      }
+    }
+    for (col = 0; col < ncols; col++) {
+      if (Utils.gr(ctotal[col], 0)) {
+	nonZeroColumns++;
+      }
+    }
+    for (row = 0; row < nrows; row++) {
+      if (Utils.gr(rtotal[row], 0)) {
+	for (col = 0; col < ncols; col++) {
+	  if (Utils.gr(ctotal[col], 0)) {
+	    expect = (ctotal[col] * rtotal[row]) / n;
+	    if (Utils.sm(expect, smallfreq)) {
+	      if (Utils.sm(expect, 1)) {
+		return false;
+	      } else {
+		smallcount++;
+		if (smallcount > (nonZeroRows * nonZeroColumns) / smallfreq) {
+		  return false;
+		}
+	      }
+	    }
+	  }
+	}
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Computes Cramer's V for a contingency table.
+   *
+   * @param matrix the contingency table
+   * @return Cramer's V
+   */
+  public static double CramersV(double [][] matrix) {
+
+    int row, col, nrows,ncols, min;
+    double n = 0;
+    
+    nrows = matrix.length;
+    ncols = matrix[0].length;
+    for (row = 0; row < nrows; row++) {
+      for (col = 0; col < ncols; col++) {
+	n += matrix[row][col];
+      }
+    }
+    min = nrows < ncols ? nrows-1 : ncols-1;
+    if ((min == 0) || Utils.eq(n, 0))
+      return 0;
+    return Math.sqrt(chiVal(matrix, false) / (n * (double)min)); 
+  } 
+
+  /**
+   * Computes the entropy of the given array.
+   *
+   * @param array the array
+   * @return the entropy
+   */
+  public static double entropy(double[] array) {
+
+    double returnValue = 0, sum = 0;
+
+    for (int i = 0; i < array.length; i++) {
+      returnValue -= lnFunc(array[i]);
+      sum += array[i];
+    }
+    if (Utils.eq(sum, 0)) {
+      return 0;
+    } else {
+      return (returnValue + lnFunc(sum)) / (sum * log2);
+    }
+  }
+
+  /**
+   * Computes conditional entropy of the rows given
+   * the columns.
+   *
+   * @param matrix the contingency table
+   * @return the conditional entropy of the rows given the columns
+   */
+  public static double entropyConditionedOnColumns(double[][] matrix) {
+    
+    double returnValue = 0, sumForColumn, total = 0;
+
+    for (int j = 0; j < matrix[0].length; j++) {
+      sumForColumn = 0;
+      for (int i = 0; i < matrix.length; i++) {
+	returnValue = returnValue + lnFunc(matrix[i][j]);
+	sumForColumn += matrix[i][j];
+      }
+      returnValue = returnValue - lnFunc(sumForColumn);
+      total += sumForColumn;
+    }
+    if (Utils.eq(total, 0)) {
+      return 0;
+    }
+    return -returnValue / (total * log2);
+  }
+
+  /**
+   * Computes conditional entropy of the columns given
+   * the rows.
+   *
+   * @param matrix the contingency table
+   * @return the conditional entropy of the columns given the rows
+   */
+  public static double entropyConditionedOnRows(double[][] matrix) {
+    
+    double returnValue = 0, sumForRow, total = 0;
+
+    for (int i = 0; i < matrix.length; i++) {
+      sumForRow = 0;
+      for (int j = 0; j < matrix[0].length; j++) {
+	returnValue = returnValue + lnFunc(matrix[i][j]);
+	sumForRow += matrix[i][j];
+      }
+      returnValue = returnValue - lnFunc(sumForRow);
+      total += sumForRow;
+    }
+    if (Utils.eq(total, 0)) {
+      return 0;
+    }
+    return -returnValue / (total * log2);
+  }
+
+  /**
+   * Computes conditional entropy of the columns given the rows
+   * of the test matrix with respect to the train matrix. Uses a
+   * Laplace prior. Does NOT normalize the entropy.
+   *
+   * @param train the train matrix 
+   * @param test the test matrix
+   * @param the number of symbols for Laplace
+   * @return the entropy
+   */
+  public static double entropyConditionedOnRows(double[][] train, 
+						double[][] test,
+						double numClasses) {
+    
+    double returnValue = 0, trainSumForRow, testSumForRow, testSum = 0;
+
+    for (int i = 0; i < test.length; i++) {
+      trainSumForRow = 0;
+      testSumForRow = 0;
+      for (int j = 0; j < test[0].length; j++) {
+	returnValue -= test[i][j] * Math.log(train[i][j] + 1);
+	trainSumForRow += train[i][j];
+	testSumForRow += test[i][j];
+      }
+      testSum = testSumForRow;
+      returnValue += testSumForRow * Math.log(trainSumForRow + 
+					     numClasses);
+    }
+    return returnValue / (testSum * log2);
+  }
+
+  /**
+   * Computes the rows' entropy for the given contingency table.
+   *
+   * @param matrix the contingency table
+   * @return the rows' entropy
+   */
+  public static double entropyOverRows(double[][] matrix) {
+    
+    double returnValue = 0, sumForRow, total = 0;
+
+    for (int i = 0; i < matrix.length; i++) {
+      sumForRow = 0;
+      for (int j = 0; j < matrix[0].length; j++) {
+	sumForRow += matrix[i][j];
+      }
+      returnValue = returnValue - lnFunc(sumForRow);
+      total += sumForRow;
+    }
+    if (Utils.eq(total, 0)) {
+      return 0;
+    }
+    return (returnValue + lnFunc(total)) / (total * log2);
+  }
+
+  /**
+   * Computes the columns' entropy for the given contingency table.
+   *
+   * @param matrix the contingency table
+   * @return the columns' entropy
+   */
+  public static double entropyOverColumns(double[][] matrix){
+    
+    double returnValue = 0, sumForColumn, total = 0;
+
+    for (int j = 0; j < matrix[0].length; j++){
+      sumForColumn = 0;
+      for (int i = 0; i < matrix.length; i++) {
+	sumForColumn += matrix[i][j];
+      }
+      returnValue = returnValue - lnFunc(sumForColumn);
+      total += sumForColumn;
+    }
+    if (Utils.eq(total, 0)) {
+      return 0;
+    }
+    return (returnValue + lnFunc(total)) / (total * log2);
+  }
+
+  /**
+   * Computes gain ratio for contingency table (split on rows).
+   * Returns Double.MAX_VALUE if the split entropy is 0.
+   *
+   * @param matrix the contingency table
+   * @return the gain ratio
+   */
+  public static double gainRatio(double[][] matrix){
+    
+    double preSplit = 0, postSplit = 0, splitEnt = 0,
+      sumForRow, sumForColumn, total = 0, infoGain;
+
+    // Compute entropy before split
+    for (int i = 0; i < matrix[0].length; i++) {
+      sumForColumn = 0;
+      for (int j = 0; j < matrix.length; j++) 
+	sumForColumn += matrix[j][i];
+      preSplit += lnFunc(sumForColumn);
+      total += sumForColumn;
+    }
+    preSplit -= lnFunc(total);
+
+    // Compute entropy after split and split entropy
+    for (int i = 0; i < matrix.length; i++) {
+      sumForRow = 0;
+      for (int j = 0; j < matrix[0].length; j++) {
+	postSplit += lnFunc(matrix[i][j]);
+	sumForRow += matrix[i][j];
+      }
+      splitEnt += lnFunc(sumForRow);
+    }
+    postSplit -= splitEnt;
+    splitEnt -= lnFunc(total);
+
+    infoGain = preSplit - postSplit;
+    if (Utils.eq(splitEnt, 0))
+      return 0;
+    return infoGain / splitEnt;
+  }
+
+  /**
+   * Returns negative base 2 logarithm of multiple hypergeometric
+   * probability for a contingency table.
+   *
+   * @param matrix the contingency table
+   * @return the log of the hypergeometric probability of the contingency table 
+   */
+  public static double log2MultipleHypergeometric(double[][] matrix) {
+
+    double sum = 0, sumForRow, sumForColumn, total = 0;
+
+    for (int i = 0; i < matrix.length; i++) {
+      sumForRow = 0;
+      for (int j = 0; j < matrix[i].length; j++) {
+	sumForRow += matrix[i][j];
+      }
+      sum += SpecialFunctions.lnFactorial(sumForRow);
+      total += sumForRow;
+    }
+    for (int j = 0; j < matrix[0].length; j++) {
+      sumForColumn = 0;
+      for (int i = 0; i < matrix.length; i++) {
+	sumForColumn += matrix [i][j];
+      }
+      sum += SpecialFunctions.lnFactorial(sumForColumn);
+    }
+    for (int i = 0; i < matrix.length; i++) {
+      for (int j = 0; j < matrix[i].length; j++) {
+	sum -= SpecialFunctions.lnFactorial(matrix[i][j]);
+      }
+    }
+    sum -= SpecialFunctions.lnFactorial(total);
+    return -sum / log2;
+  }
+
+  /**
+   * Reduces a matrix by deleting all zero rows and columns.
+   *
+   * @param matrix the matrix to be reduced
+   * @param the matrix with all zero rows and columns deleted
+   */
+  public static double[][] reduceMatrix(double[][] matrix) {
+
+    int row, col, currCol, currRow, nrows, ncols, 
+      nonZeroRows = 0, nonZeroColumns = 0;
+    double[] rtotal, ctotal;
+    double[][] newMatrix;
+
+    nrows = matrix.length;
+    ncols = matrix[0].length;
+    rtotal = new double [nrows];
+    ctotal = new double [ncols];
+    for (row = 0; row < nrows; row++) {
+      for (col = 0; col < ncols; col++) {
+	rtotal[row] += matrix[row][col];
+	ctotal[col] += matrix[row][col];
+      }
+    }
+    for (row = 0; row < nrows; row++) {
+      if (Utils.gr(rtotal[row],0)) {
+	nonZeroRows++;
+      }
+    }
+    for (col = 0; col < ncols; col++) {
+      if (Utils.gr(ctotal[col],0)) {
+	nonZeroColumns++;
+      }
+    }
+    newMatrix = new double[nonZeroRows][nonZeroColumns];
+    currRow = 0;
+    for (row = 0; row < nrows; row++) {
+      if (Utils.gr(rtotal[row],0)) {
+	currCol = 0;
+	for (col = 0; col < ncols; col++) {
+	  if (Utils.gr(ctotal[col],0)) {
+	    newMatrix[currRow][currCol] = matrix[row][col];
+	    currCol++;
+	  }
+	}
+	currRow++;
+      }
+    }
+    return newMatrix;
+  }
+    
+   /**
+    * Calculates the symmetrical uncertainty for base 2.
+    *
+    * @param matrix the contingency table
+    * @return the calculated symmetrical uncertainty
+    *
+    */
+  public static double symmetricalUncertainty(double matrix[][]) {
+
+    double sumForColumn, sumForRow, total = 0, columnEntropy = 0, 
+      rowEntropy = 0, entropyConditionedOnRows = 0, infoGain = 0;
+
+    // Compute entropy for columns
+    for (int i = 0; i < matrix[0].length; i++) {
+      sumForColumn = 0;
+      for (int j = 0; j < matrix.length; j++) {
+	sumForColumn += matrix[j][i];
+      }
+      columnEntropy += lnFunc(sumForColumn);
+      total += sumForColumn;
+    }
+    columnEntropy -= lnFunc(total);
+
+    // Compute entropy for rows and conditional entropy
+    for (int i = 0; i < matrix.length; i++) {
+      sumForRow = 0;
+      for (int j = 0; j < matrix[0].length; j++) { 
+	sumForRow += matrix[i][j];
+	entropyConditionedOnRows += lnFunc(matrix[i][j]);
+      }
+      rowEntropy += lnFunc(sumForRow);
+    }
+    entropyConditionedOnRows -= rowEntropy;
+    rowEntropy -= lnFunc(total);
+    infoGain = columnEntropy - entropyConditionedOnRows;
+    if (Utils.eq(columnEntropy, 0) || Utils.eq(rowEntropy, 0))
+      return 0;
+    return 2.0 * (infoGain / (columnEntropy + rowEntropy));
+  }
+
+  /**
+   * Computes Goodman and Kruskal's tau-value for a contingency table.
+   *
+   * @param matrix the contingency table
+   * @param Goodman and Kruskal's tau-value
+   */
+  public static double tauVal(double[][] matrix) {
+    
+    int nrows, ncols, row, col;
+    double [] ctotal;
+    double maxcol = 0, max, maxtotal = 0, n = 0;
+    
+    nrows = matrix.length;
+    ncols = matrix[0].length;
+    ctotal = new double [ncols];
+    for (row = 0; row < nrows; row++) {
+      max = 0;
+      for (col = 0; col < ncols; col++) {
+	if (Utils.gr(matrix[row][col], max)) 
+	  max = matrix[row][col];
+	ctotal[col] += matrix[row][col];
+	n += matrix[row][col];
+      }
+      maxtotal += max;
+    }
+    if (Utils.eq(n, 0)) {
+      return 0;
+    }
+    maxcol = ctotal[Utils.maxIndex(ctotal)];
+    return (maxtotal - maxcol)/(n - maxcol);
+  }
+
+  /**
+   * Help method for computing entropy.
+   */
+  private static double lnFunc(double num){
+    
+    // Constant hard coded for efficiency reasons
+    if (num < 1e-6) {
+      return 0;
+    } else {
+      return num * Math.log(num);
+    }
+  }
+  
+  /**
+   * Computes chi-value for one cell in matrix.
+   * From Gary Perlman's unixstat.
+   */
+  private static double chiCell(double freq, double expect, boolean yates){
+
+    double  diff = freq - expect;
+       
+    if (yates) {
+      diff = Math.abs (diff) - 0.5;
+      if (diff < 0.0) { // over-correction
+	diff = 0.0;
+      }
+    }
+    if (Math.abs(expect) < 10e-10) {
+      return (0.0);
+    } else {
+      return (diff * diff / expect);
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   */
+  public static void main(String[] ops) {
+
+    double[] firstRow = {10, 5, 20};
+    double[] secondRow = {2, 10, 6};
+    double[] thirdRow = {5, 10, 10};
+    double[][] matrix = new double[3][0];
+
+    matrix[0] = firstRow; matrix[1] = secondRow; matrix[2] = thirdRow;
+    for (int i = 0; i < matrix.length; i++) {
+      for (int j = 0; j < matrix[i].length; j++) {
+	System.out.print(matrix[i][j] + " ");
+      }
+      System.out.println();
+    }
+    System.out.println("Chi-squared probability: " +
+		       ContingencyTables.chiSquared(matrix, false));
+    System.out.println("Chi-squared value: " +
+		       ContingencyTables.chiVal(matrix, false));
+    System.out.println("Cochran's criterion fullfilled: " +
+		       ContingencyTables.cochransCriterion(matrix));
+    System.out.println("Cramer's V: " +
+		       ContingencyTables.CramersV(matrix));
+    System.out.println("Entropy of first row: " +
+		       ContingencyTables.entropy(firstRow));
+    System.out.println("Entropy conditioned on columns: " +
+		       ContingencyTables.entropyConditionedOnColumns(matrix));
+    System.out.println("Entropy conditioned on rows: " +
+		       ContingencyTables.entropyConditionedOnRows(matrix));
+    System.out.println("Entropy conditioned on rows (with Laplace): " +
+		       ContingencyTables.entropyConditionedOnRows(matrix, matrix, 3));
+    System.out.println("Entropy of rows: " +
+		       ContingencyTables.entropyOverRows(matrix));
+    System.out.println("Entropy of columns: " +
+		       ContingencyTables.entropyOverColumns(matrix));
+    System.out.println("Gain ratio: " +
+		       ContingencyTables.gainRatio(matrix));
+    System.out.println("Negative log2 of multiple hypergeometric probability: " +
+		       ContingencyTables.log2MultipleHypergeometric(matrix));
+    System.out.println("Symmetrical uncertainty: " +
+		       ContingencyTables.symmetricalUncertainty(matrix));
+    System.out.println("Tau value: " +
+		       ContingencyTables.tauVal(matrix));
+    double[][] newMatrix = new double[3][3];
+    newMatrix[0][0] = 1; newMatrix[0][1] = 0; newMatrix[0][2] = 1;
+    newMatrix[1][0] = 0; newMatrix[1][1] = 0; newMatrix[1][2] = 0;
+    newMatrix[2][0] = 1; newMatrix[2][1] = 0; newMatrix[2][2] = 1;
+    System.out.println("Matrix with empty row and column: ");
+    for (int i = 0; i < newMatrix.length; i++) {
+      for (int j = 0; j < newMatrix[i].length; j++) {
+	System.out.print(newMatrix[i][j] + " ");
+      }
+      System.out.println();
+    }
+    System.out.println("Reduced matrix: ");
+    newMatrix = ContingencyTables.reduceMatrix(newMatrix);
+    for (int i = 0; i < newMatrix.length; i++) {
+      for (int j = 0; j < newMatrix[i].length; j++) {
+	System.out.print(newMatrix[i][j] + " ");
+      }
+      System.out.println();
+    }
+  }
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Copyable.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Copyable.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Copyable.java	(revision 21112)
@@ -0,0 +1,41 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Copyable.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+/**
+ * Interface implemented by classes that can produce "shallow" copies
+ * of their objects. (As opposed to clone(), which is supposed to
+ * produce a "deep" copy.)
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public interface Copyable {
+
+  /**
+   * This method produces a shallow copy of an object.
+   * It does the same as the clone() method in Object, which also produces
+   * a shallow copy.
+   */
+  public Object copy();
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/FastVector.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/FastVector.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/FastVector.java	(revision 21112)
@@ -0,0 +1,432 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    FastVector.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Implements a fast vector class without synchronized
+ * methods. Replaces java.util.Vector. (Synchronized methods tend to
+ * be slow.)
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$ */
+public class FastVector implements Copyable, Serializable {
+
+  /**
+   * Class for enumerating the vector's elements.
+   */
+  public class FastVectorEnumeration implements Enumeration {
+
+    /** The counter. */
+    private int m_Counter;
+
+    /** The vector. */
+    private FastVector m_Vector;
+
+    /** Special element. Skipped during enumeration. */
+    private int m_SpecialElement;
+
+    /**
+     * Constructs an enumeration.
+     *
+     * @param vector the vector which is to be enumerated
+     */
+    public FastVectorEnumeration(FastVector vector) {
+
+      m_Counter = 0;
+      m_Vector = vector;
+      m_SpecialElement = -1;
+    }
+
+    /**
+     * Constructs an enumeration with a special element.
+     * The special element is skipped during the enumeration.
+     *
+     * @param vector the vector which is to be enumerated
+     * @param special the index of the special element
+     */
+    public FastVectorEnumeration(FastVector vector, int special) {
+
+      m_Vector = vector;
+      m_SpecialElement = special;
+      if (special == 0) {
+	m_Counter = 1;
+      } else {
+	m_Counter = 0;
+      }
+    }
+
+
+    /**
+     * Tests if there are any more elements to enumerate.
+     *
+     * @return true if there are some elements left
+     */
+    public final boolean hasMoreElements() {
+
+      if (m_Counter < m_Vector.size()) {
+	return true;
+      }
+      return false;
+    }
+
+    /**
+     * Returns the next element.
+     *
+     * @return the next element to be enumerated
+     */
+    public final Object nextElement() {
+  
+      Object result = m_Vector.elementAt(m_Counter);
+
+      m_Counter++;
+      if (m_Counter == m_SpecialElement) {
+	m_Counter++;
+      }
+      return result;
+    }
+  }
+
+  /** The array of objects. */
+  private Object[] m_Objects;
+
+  /** The current size; */
+  private int m_Size;
+
+  /** The capacity increment */
+  private int m_CapacityIncrement;
+  
+  /** The capacity multiplier. */
+  private double m_CapacityMultiplier;
+
+  /**
+   * Constructs an empty vector with initial
+   * capacity zero.
+   */
+  public FastVector() {
+  
+    m_Objects = new Object[0];
+    m_Size = 0;
+    m_CapacityIncrement = 1;
+    m_CapacityMultiplier = 2;
+  }
+
+  /**
+   * Constructs a vector with the given capacity.
+   *
+   * @param capacity the vector's initial capacity
+   */
+  public FastVector(int capacity) {
+
+    m_Objects = new Object[capacity];
+    m_Size = 0;
+    m_CapacityIncrement = 1;
+    m_CapacityMultiplier = 2;
+  }
+
+  /**
+   * Constructs a vector with the given capacity, capacity 
+   * increment and capacity mulitplier.
+   *
+   * @param capacity the vector's initial capacity
+   */
+  public FastVector(int capacity, int capIncrement, 
+		    double capMultiplier) {
+
+    m_Objects = new Object[capacity];
+    m_Size = 0;
+    m_CapacityIncrement = capIncrement;
+    m_CapacityMultiplier = capMultiplier;
+  }
+
+  /**
+   * Adds an element to this vector. Increases its
+   * capacity if its not large enough.
+   *
+   * @param element the element to add
+   */
+  public final void addElement(Object element) {
+
+    Object[] newObjects;
+
+    if (m_Size == m_Objects.length) {
+      newObjects = new Object[(int)m_CapacityMultiplier *
+			     (m_Objects.length +
+			      m_CapacityIncrement)];
+      System.arraycopy(m_Objects, 0, newObjects, 0, m_Size);
+      m_Objects = newObjects;
+    }
+    m_Objects[m_Size] = element;
+    m_Size++;
+  }
+
+  /**
+   * Returns the capacity of the vector.
+   *
+   * @return the capacity of the vector
+   */
+  public final int capacity() {
+  
+    return m_Objects.length;
+  }
+
+  /**
+   * Produces a shallow copy of this vector.
+   *
+   * @return the new vector
+   */
+  public final Object copy() {
+
+    FastVector copy = new FastVector(m_Objects.length, 
+				     m_CapacityIncrement,
+				     m_CapacityMultiplier);
+    copy.m_Size = m_Size;
+    System.arraycopy(m_Objects, 0, copy.m_Objects, 0, m_Size);
+    return copy;
+  }
+
+  /**
+   * Clones the vector and shallow copies all its elements.
+   * The elements have to implement the Copyable interface.
+   * 
+   * @return the new vector
+   */
+  public final Object copyElements() {
+
+    FastVector copy = new FastVector(m_Objects.length, 
+				     m_CapacityIncrement,
+				     m_CapacityMultiplier);
+    copy.m_Size = m_Size;
+    for (int i = 0; i < m_Size; i++) {
+      copy.m_Objects[i] = ((Copyable)m_Objects[i]).copy();
+    }
+    return copy;
+  }
+
+  /**
+   * Returns the element at the given position.
+   *
+   * @param index the element's index
+   * @return the element with the given index
+   */
+  public final Object elementAt(int index) {
+
+    return m_Objects[index];
+  }
+
+  /**
+   * Returns an enumeration of this vector.
+   *
+   * @return an enumeration of this vector
+   */
+  public final Enumeration elements() {
+  
+    return new FastVectorEnumeration(this);
+  }
+
+  /**
+   * Returns an enumeration of this vector, skipping the
+   * element with the given index.
+   *
+   * @param index the element to skip
+   * @return an enumeration of this vector
+   */
+  public final Enumeration elements(int index) {
+  
+    return new FastVectorEnumeration(this, index);
+  }
+
+  /**
+   * Returns the first element of the vector.
+   *
+   * @return the first element of the vector
+   */
+  public final Object firstElement() {
+
+    return m_Objects[0];
+  }
+
+  /**
+   * Searches for the first occurence of the given argument, 
+   * testing for equality using the equals method. 
+   *
+   * @param element the element to be found
+   * @return the index of the first occurrence of the argument 
+   * in this vector; returns -1 if the object is not found
+   */
+  public final int indexOf(Object element) {
+
+    for (int i = 0; i < m_Size; i++) {
+      if (element.equals(m_Objects[i])) {
+	return i;
+      }
+    }
+    return -1;
+  }
+
+  /**
+   * Inserts an element at the given position.
+   *
+   * @param element the element to be inserted
+   * @param index the element's index
+   */
+  public final void insertElementAt(Object element, int index) {
+
+    Object[] newObjects;
+
+    if (m_Size < m_Objects.length) {
+      System.arraycopy(m_Objects, index, m_Objects, index + 1, 
+                       m_Size - index);
+      m_Objects[index] = element;
+    } else {
+      newObjects = new Object[(int)m_CapacityMultiplier *
+			     (m_Objects.length +
+			      m_CapacityIncrement)];
+      System.arraycopy(m_Objects, 0, newObjects, 0, index);
+      newObjects[index] = element;
+      System.arraycopy(m_Objects, index, newObjects, index + 1,
+		       m_Size - index);
+      m_Objects = newObjects;
+    }
+    m_Size++;
+  }
+
+  /**
+   * Returns the last element of the vector.
+   *
+   * @return the last element of the vector
+   */
+  public final Object lastElement() {
+
+    return m_Objects[m_Size - 1];
+  }
+
+  /**
+   * Deletes an element from this vector.
+   *
+   * @param index the index of the element to be deleted
+   */
+  public final void removeElementAt(int index) {
+
+    System.arraycopy(m_Objects, index + 1, m_Objects, index, 
+                     m_Size - index - 1);
+    m_Size--;
+  }
+
+  /**
+   * Removes all components from this vector and sets its 
+   * size to zero. 
+   */
+  public final void removeAllElements() {
+
+    m_Objects = new Object[m_Objects.length];
+    m_Size = 0;
+  }
+
+  /**
+   * Appends all elements of the supplied vector to this vector.
+   *
+   * @param toAppend the FastVector containing elements to append.
+   */
+  public final void appendElements(FastVector toAppend) {
+
+    setCapacity(size() + toAppend.size());
+    System.arraycopy(toAppend.m_Objects, 0, m_Objects, size(), toAppend.size());
+    m_Size = m_Objects.length;
+  }
+
+  /** 
+   * Returns all the elements of this vector as an array
+   *
+   * @param an array containing all the elements of this vector
+   */
+  public final Object [] toArray() {
+
+    Object [] newObjects = new Object[size()];
+    System.arraycopy(m_Objects, 0, newObjects, 0, size());
+    return newObjects;
+  }
+
+  /**
+   * Sets the vector's capacity to the given value.
+   *
+   * @param capacity the new capacity
+   */
+  public final void setCapacity(int capacity) {
+
+    Object[] newObjects = new Object[capacity];
+   
+    System.arraycopy(m_Objects, 0, newObjects, 0, Math.min(capacity, m_Size));
+    m_Objects = newObjects;
+    if (m_Objects.length < m_Size)
+      m_Size = m_Objects.length;
+  }
+
+  /**
+   * Sets the element at the given index.
+   *
+   * @param element the element to be put into the vector
+   * @param index the index at which the element is to be placed
+   */
+  public final void setElementAt(Object element, int index) {
+
+    m_Objects[index] = element;
+  }
+
+  /**
+   * Returns the vector's current size.
+   *
+   * @return the vector's current size
+   */
+  public final int size() {
+
+    return m_Size;
+  }
+
+  /**
+   * Swaps two elements in the vector.
+   *
+   * @param first index of the first element
+   * @param second index of the second element
+   */
+  public final void swap(int first, int second) {
+
+    Object help = m_Objects[first];
+
+    m_Objects[first] = m_Objects[second];
+    m_Objects[second] = help;
+  }
+
+  /**
+   * Sets the vector's capacity to its size.
+   */
+  public final void trimToSize() {
+
+    Object[] newObjects = new Object[m_Size];
+    
+    System.arraycopy(m_Objects, 0, newObjects, 0, m_Size);
+    m_Objects = newObjects;
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instance.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instance.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instance.java	(revision 21112)
@@ -0,0 +1,1064 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Instance.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Class for handling an instance. All values (numeric, nominal, or
+ * string) are internally stored as floating-point numbers. If an
+ * attribute is nominal (or a string), the stored value is the index
+ * of the corresponding nominal (or string) value in the attribute's
+ * definition. We have chosen this approach in favor of a more elegant
+ * object-oriented approach because it is much faster. <p>
+ *
+ * Typical usage (code from the main() method of this class): <p>
+ *
+ * <code>
+ * ... <br>
+ *      
+ * // Create empty instance with three attribute values <br>
+ * Instance inst = new Instance(3); <br><br>
+ *     
+ * // Set instance's values for the attributes "length", "weight", and "position"<br>
+ * inst.setValue(length, 5.3); <br>
+ * inst.setValue(weight, 300); <br>
+ * inst.setValue(position, "first"); <br><br>
+ *   
+ * // Set instance's dataset to be the dataset "race" <br>
+ * inst.setDataset(race); <br><br>
+ *   
+ * // Print the instance <br>
+ * System.out.println("The instance: " + inst); <br>
+ *
+ * ... <br>
+ * </code><p>
+ *
+ * All methods that change an instance are safe, ie. a change of an
+ * instance does not affect any other instances. All methods that
+ * change an instance's attribute values clone the attribute value
+ * vector before it is changed. If your application heavily modifies
+ * instance values, it may be faster to create a new instance from scratch.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$ 
+ */
+public class Instance implements Copyable, Serializable {
+  
+  /** Constant representing a missing value. */
+  protected final static double MISSING_VALUE = Double.NaN;
+
+  /** 
+   * The dataset the instance has access to.  Null if the instance
+   * doesn't have access to any dataset.  Only if an instance has
+   * access to a dataset, it knows about the actual attribute types.  
+   */
+  protected Instances m_Dataset;
+
+  /** The instance's attribute values. */
+  protected double[] m_AttValues;
+
+  /** The instance's weight. */
+  protected double m_Weight;
+
+  /**
+   * Constructor that copies the attribute values and the weight from
+   * the given instance. Reference to the dataset is set to null.
+   * (ie. the instance doesn't have access to information about the
+   * attribute types)
+   *
+   * @param instance the instance from which the attribute
+   * values and the weight are to be copied 
+   */
+  public Instance(Instance instance) {
+    
+    m_AttValues = instance.m_AttValues;
+    m_Weight = instance.m_Weight;
+    m_Dataset = null;
+  }
+
+  /**
+   * Constructor that inititalizes instance variable with given
+   * values. Reference to the dataset is set to null. (ie. the instance
+   * doesn't have access to information about the attribute types)
+   *
+   * @param weight the instance's weight
+   * @param attValues a vector of attribute values 
+   */
+  public Instance(double weight, double[] attValues){
+    
+    m_AttValues = attValues;
+    m_Weight = weight;
+    m_Dataset = null;
+  }
+
+  /**
+   * Constructor of an instance that sets weight to one, all values to
+   * be missing, and the reference to the dataset to null. (ie. the instance
+   * doesn't have access to information about the attribute types)
+   *
+   * @param numAttributes the size of the instance 
+   */
+  public Instance(int numAttributes) {
+    
+    m_AttValues = new double[numAttributes];
+    for (int i = 0; i < m_AttValues.length; i++) {
+      m_AttValues[i] = MISSING_VALUE;
+    }
+    m_Weight = 1;
+    m_Dataset = null;
+  }
+
+  /**
+   * Returns the attribute with the given index.
+   *
+   * @param index the attribute's index
+   * @return the attribute at the given position
+   * @exception UnassignedDatasetException if instance doesn't have access to a
+   * dataset
+   */ 
+  public Attribute attribute(int index) {
+   
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.attribute(index);
+  }
+
+  /**
+   * Returns the attribute with the given index. Does the same
+   * thing as attribute().
+   *
+   * @param indexOfIndex the index of the attribute's index 
+   * @return the attribute at the given position
+   * @exception UnassignedDatasetException if instance doesn't have access to a
+   * dataset
+   */ 
+  public Attribute attributeSparse(int indexOfIndex) {
+   
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.attribute(indexOfIndex);
+  }
+
+  /**
+   * Returns class attribute.
+   *
+   * @return the class attribute
+   * @exception UnassignedDatasetException if the class is not set or the
+   * instance doesn't have access to a dataset
+   */
+  public Attribute classAttribute() {
+
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.classAttribute();
+  }
+
+  /**
+   * Returns the class attribute's index.
+   *
+   * @return the class index as an integer 
+   * @exception UnassignedDatasetException if instance doesn't have access to a dataset 
+   */
+  public int classIndex() {
+    
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.classIndex();
+  }
+
+  /**
+   * Tests if an instance's class is missing.
+   *
+   * @return true if the instance's class is missing
+   * @exception UnassignedClassException if the class is not set or the instance doesn't
+   * have access to a dataset
+   */
+  public boolean classIsMissing() {
+
+    if (classIndex() < 0) {
+      throw new UnassignedClassException("Class is not set!");
+    }
+    return isMissing(classIndex());
+  }
+
+  /**
+   * Returns an instance's class value in internal format. (ie. as a
+   * floating-point number)
+   *
+   * @return the corresponding value as a double (If the 
+   * corresponding attribute is nominal (or a string) then it returns the 
+   * value's index as a double).
+   * @exception UnassignedClassException if the class is not set or the instance doesn't
+   * have access to a dataset 
+   */
+  public double classValue() {
+    
+    if (classIndex() < 0) {
+      throw new UnassignedClassException("Class is not set!");
+    }
+    return value(classIndex());
+  }
+
+  /**
+   * Produces a shallow copy of this instance. The copy has
+   * access to the same dataset. (if you want to make a copy
+   * that doesn't have access to the dataset, use 
+   * <code>new Instance(instance)</code>
+   *
+   * @return the shallow copy
+   */
+  public Object copy() {
+
+    Instance result = new Instance(this);
+    result.m_Dataset = m_Dataset;
+    return result;
+  }
+
+  /**
+   * Returns the dataset this instance has access to. (ie. obtains
+   * information about attribute types from) Null if the instance
+   * doesn't have access to a dataset.
+   *
+   * @return the dataset the instance has accesss to
+   */
+  public Instances dataset() {
+
+    return m_Dataset;
+  }
+
+  /**
+   * Deletes an attribute at the given position (0 to 
+   * numAttributes() - 1). Only succeeds if the instance does not
+   * have access to any dataset because otherwise inconsistencies
+   * could be introduced.
+   *
+   * @param pos the attribute's position
+   * @exception RuntimeException if the instance has access to a
+   * dataset 
+   */
+  public void deleteAttributeAt(int position) {
+
+    if (m_Dataset != null) {
+      throw new RuntimeException("Instance has access to a dataset!");
+    }
+    forceDeleteAttributeAt(position);
+  }
+
+  /**
+   * Returns an enumeration of all the attributes.
+   *
+   * @return enumeration of all the attributes
+   * @exception UnassignedDatasetException if the instance doesn't
+   * have access to a dataset 
+   */
+  public Enumeration enumerateAttributes() {
+
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.enumerateAttributes();
+  }
+
+  /**
+   * Tests if the headers of two instances are equivalent.
+   *
+   * @param instance another instance
+   * @return true if the header of the given instance is 
+   * equivalent to this instance's header
+   * @exception UnassignedDatasetException if instance doesn't have access to any
+   * dataset
+   */
+  public boolean equalHeaders(Instance inst) {
+
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.equalHeaders(inst.m_Dataset);
+  }
+
+  /**
+   * Returns the index of the attribute stored at the given position.
+   * Just returns the given value.
+   *
+   * @param position the position 
+   * @return the index of the attribute stored at the given position
+   */
+  public int index(int position) {
+
+    return position;
+  }
+
+  /**
+   * Inserts an attribute at the given position (0 to 
+   * numAttributes()). Only succeeds if the instance does not
+   * have access to any dataset because otherwise inconsistencies
+   * could be introduced.
+   *
+   * @param pos the attribute's position
+   * @exception RuntimeException if the instance has accesss to a
+   * dataset
+   * @exception IllegalArgumentException if the position is out of range
+   */
+  public void insertAttributeAt(int position) {
+
+    if (m_Dataset != null) {
+      throw new RuntimeException("Instance has accesss to a dataset!");
+    }
+    if ((position < 0) ||
+	(position > numAttributes())) {
+      throw new IllegalArgumentException("Can't insert attribute: index out "+
+                                         "of range");
+    }
+    forceInsertAttributeAt(position);
+  }
+
+  /**
+   * Tests if a specific value is "missing".
+   *
+   * @param attIndex the attribute's index
+   */
+  public boolean isMissing(int attIndex) {
+
+    if (Double.isNaN(m_AttValues[attIndex])) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Tests if a specific value is "missing". Does
+   * the same thing as isMissing() if applied to an Instance.
+   *
+   * @param indexOfIndex the index of the attribute's index 
+   */
+  public boolean isMissingSparse(int indexOfIndex) {
+
+    if (Double.isNaN(m_AttValues[indexOfIndex])) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Tests if a specific value is "missing".
+   * The given attribute has to belong to a dataset.
+   *
+   * @param att the attribute
+   */
+  public boolean isMissing(Attribute att) {
+
+    return isMissing(att.index());
+  }
+
+  /**
+   * Tests if the given value codes "missing".
+   *
+   * @param val the value to be tested
+   * @return true if val codes "missing"
+   */
+  public static boolean isMissingValue(double val) {
+
+    return Double.isNaN(val);
+  }
+
+  /**
+   * Merges this instance with the given instance and returns
+   * the result. Dataset is set to null.
+   *
+   * @param inst the instance to be merged with this one
+   * @return the merged instances
+   */
+  public Instance mergeInstance(Instance inst) {
+
+    int m = 0;
+    double [] newVals = new double[numAttributes() + inst.numAttributes()];
+    for (int j = 0; j < numAttributes(); j++, m++) {
+      newVals[m] = value(j);
+    }
+    for (int j = 0; j < inst.numAttributes(); j++, m++) {
+      newVals[m] = inst.value(j);
+    }
+    return new Instance(1.0, newVals);
+  }
+
+  /**
+   * Returns the double that codes "missing".
+   *
+   * @return the double that codes "missing"
+   */
+  public static double missingValue() {
+
+    return MISSING_VALUE;
+  }
+
+  /**
+   * Returns the number of attributes.
+   *
+   * @return the number of attributes as an integer
+   */
+  public int numAttributes() {
+
+    return m_AttValues.length;
+  }
+
+  /**
+   * Returns the number of class labels.
+   *
+   * @return the number of class labels as an integer if the 
+   * class attribute is nominal, 1 otherwise.
+   * @exception UnassignedDatasetException if instance doesn't have access to any
+   * dataset
+   */
+  public int numClasses() {
+    
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.numClasses();
+  }
+
+  /**
+   * Returns the number of values present. Always the same as numAttributes().
+   *
+   * @return the number of values
+   */
+  public int numValues() {
+
+    return m_AttValues.length;
+  }
+
+  /** 
+   * Replaces all missing values in the instance with the
+   * values contained in the given array. A deep copy of
+   * the vector of attribute values is performed before the
+   * values are replaced.
+   *
+   * @param array containing the means and modes
+   * @exception IllegalArgumentException if numbers of attributes are unequal
+   */
+  public void replaceMissingValues(double[] array) {
+	 
+    if ((array == null) || 
+	(array.length != m_AttValues.length)) {
+      throw new IllegalArgumentException("Unequal number of attributes!");
+    }
+    freshAttributeVector();
+    for (int i = 0; i < m_AttValues.length; i++) {
+      if (isMissing(i)) {
+	m_AttValues[i] = array[i];
+      }
+    }
+  }
+
+  /**
+   * Sets the class value of an instance to be "missing". A deep copy of
+   * the vector of attribute values is performed before the
+   * value is set to be missing.
+   *
+   * @exception UnassignedClassException if the class is not set
+   * @exception UnassignedDatasetException if the instance doesn't
+   * have access to a dataset
+   */
+  public void setClassMissing() {
+
+    if (classIndex() < 0) {
+      throw new UnassignedClassException("Class is not set!");
+    }
+    setMissing(classIndex());
+  }
+
+  /**
+   * Sets the class value of an instance to the given value (internal
+   * floating-point format).  A deep copy of the vector of attribute
+   * values is performed before the value is set.
+   *
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   * @exception UnassignedClassException if the class is not set
+   * @exception UnaddignedDatasetException if the instance doesn't
+   * have access to a dataset 
+   */
+  public void setClassValue(double value) {
+
+    if (classIndex() < 0) {
+      throw new UnassignedClassException("Class is not set!");
+    }
+    setValue(classIndex(), value);
+  }
+
+  /**
+   * Sets the class value of an instance to the given value. A deep
+   * copy of the vector of attribute values is performed before the
+   * value is set.
+   *
+   * @param value the new class value (If the class
+   * is a string attribute and the value can't be found,
+   * the value is added to the attribute).
+   * @exception UnassignedClassException if the class is not set
+   * @exception UnassignedDatasetException if the dataset is not set
+   * @exception IllegalArgumentException if the attribute is not
+   * nominal or a string, or the value couldn't be found for a nominal
+   * attribute 
+   */
+  public final void setClassValue(String value) {
+
+    if (classIndex() < 0) {
+      throw new UnassignedClassException("Class is not set!");
+    }
+    setValue(classIndex(), value);
+  }
+
+  /**
+   * Sets the reference to the dataset. Does not check if the instance
+   * is compatible with the dataset. Note: the dataset does not know
+   * about this instance. If the structure of the dataset's header
+   * gets changed, this instance will not be adjusted automatically.
+   *
+   * @param instances the reference to the dataset 
+   */
+  public final void setDataset(Instances instances) {
+    
+    m_Dataset = instances;
+  }
+
+  /**
+   * Sets a specific value to be "missing". Performs a deep copy
+   * of the vector of attribute values before the value is set to
+   * be missing.
+   *
+   * @param attIndex the attribute's index
+   */
+  public final void setMissing(int attIndex) {
+
+    setValue(attIndex, MISSING_VALUE);
+  }
+
+  /**
+   * Sets a specific value to be "missing". Performs a deep copy
+   * of the vector of attribute values before the value is set to
+   * be missing. The given attribute has to belong to a dataset.
+   *
+   * @param att the attribute
+   */
+  public final void setMissing(Attribute att) {
+
+    setMissing(att.index());
+  }
+
+  /**
+   * Sets a specific value in the instance to the given value 
+   * (internal floating-point format). Performs a deep copy
+   * of the vector of attribute values before the value is set.
+   *
+   * @param attIndex the attribute's index 
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   */
+  public void setValue(int attIndex, double value) {
+    
+    freshAttributeVector();
+    m_AttValues[attIndex] = value;
+  }
+
+  /**
+   * Sets a specific value in the instance to the given value 
+   * (internal floating-point format). Performs a deep copy
+   * of the vector of attribute values before the value is set.
+   * Does exactly the same thing as setValue().
+   *
+   * @param indexOfIndex the index of the attribute's index 
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   */
+  public void setValueSparse(int indexOfIndex, double value) {
+    
+    freshAttributeVector();
+    m_AttValues[indexOfIndex] = value;
+  }
+
+  /**
+   * Sets a value of a nominal or string attribute to the given
+   * value. Performs a deep copy of the vector of attribute values
+   * before the value is set.
+   *
+   * @param attIndex the attribute's index
+   * @param value the new attribute value (If the attribute
+   * is a string attribute and the value can't be found,
+   * the value is added to the attribute).
+   * @exception UnassignedDatasetException if the dataset is not set
+   * @exception IllegalArgumentException if the selected
+   * attribute is not nominal or a string, or the supplied value couldn't 
+   * be found for a nominal attribute 
+   */
+  public final void setValue(int attIndex, String value) {
+    
+    int valIndex;
+
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    if (!attribute(attIndex).isNominal() &&
+	!attribute(attIndex).isString()) {
+      throw new IllegalArgumentException("Attribute neither nominal nor string!");
+    }
+    valIndex = attribute(attIndex).indexOfValue(value);
+    if (valIndex == -1) {
+      if (attribute(attIndex).isNominal()) {
+	throw new IllegalArgumentException("Value not defined for given nominal attribute!");
+      } else {
+	attribute(attIndex).forceAddValue(value);
+	valIndex = attribute(attIndex).indexOfValue(value);
+      }
+    }
+    setValue(attIndex, (double)valIndex); 
+  }
+
+  /**
+   * Sets a specific value in the instance to the given value
+   * (internal floating-point format). Performs a deep copy of the
+   * vector of attribute values before the value is set, so if you are
+   * planning on calling setValue many times it may be faster to
+   * create a new instance using toDoubleArray.  The given attribute
+   * has to belong to a dataset.
+   *
+   * @param att the attribute 
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   */
+  public final void setValue(Attribute att, double value) {
+
+    setValue(att.index(), value);
+  }
+
+  /**
+   * Sets a value of an nominal or string attribute to the given
+   * value. Performs a deep copy of the vector of attribute values
+   * before the value is set, so if you are planning on calling setValue many
+   * times it may be faster to create a new instance using toDoubleArray.
+   * The given attribute has to belong to a dataset.
+   *
+   * @param att the attribute
+   * @param value the new attribute value (If the attribute
+   * is a string attribute and the value can't be found,
+   * the value is added to the attribute).
+   * @exception IllegalArgumentException if the the attribute is not
+   * nominal or a string, or the value couldn't be found for a nominal
+   * attribute 
+   */
+  public final void setValue(Attribute att, String value) {
+
+    if (!att.isNominal() &&
+	!att.isString()) {
+      throw new IllegalArgumentException("Attribute neither nominal nor string!");
+    }
+    int valIndex = att.indexOfValue(value);
+    if (valIndex == -1) {
+      if (att.isNominal()) {
+	throw new IllegalArgumentException("Value not defined for given nominal attribute!");
+      } else {
+	att.forceAddValue(value);
+	valIndex = att.indexOfValue(value);
+      }
+    }
+    setValue(att.index(), (double)valIndex);
+  }
+
+  /**
+   * Sets the weight of an instance.
+   *
+   * @param weight the weight
+   */
+  public final void setWeight(double weight) {
+
+    m_Weight = weight;
+  }
+
+  /** 
+   * Returns the value of a nominal (or string) attribute
+   * for the instance.
+   *
+   * @param attIndex the attribute's index
+   * @return the value as a string
+   * @exception IllegalArgumentException if the attribute is not a nominal
+   * (or string) attribute.
+   * @exception UnassignedDatasetException if the instance doesn't belong
+   * to a dataset.
+   */
+  public final String stringValue(int attIndex) {
+
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    } 
+    if (!m_Dataset.attribute(attIndex).isNominal() &&
+	!m_Dataset.attribute(attIndex).isString()) {
+      throw new IllegalArgumentException("Attribute neither nominal nor string!");
+    }
+    return m_Dataset.attribute(attIndex).
+      value((int) value(attIndex));
+  }
+
+  /** 
+   * Returns the value of a nominal (or string) attribute
+   * for the instance.
+   *
+   * @param att the attribute
+   * @return the value as a string
+   * @exception IllegalArgumentException if the attribute is not a nominal
+   * (or string) attribute.
+   * @exception UnassignedDatasetException if the instance doesn't belong
+   * to a dataset.
+   */
+  public final String stringValue(Attribute att) {
+
+    return stringValue(att.index());
+  }
+
+  /**
+   * Returns the values of each attribute as an array of doubles.
+   *
+   * @return an array containing all the instance attribute values
+   */
+  public double[] toDoubleArray() {
+
+    double[] newValues = new double[m_AttValues.length];
+    System.arraycopy(m_AttValues, 0, newValues, 0, 
+		     m_AttValues.length);
+    return newValues;
+  }
+
+  /**
+   * Returns the description of one instance. If the instance
+   * doesn't have access to a dataset, it returns the internal
+   * floating-point values. Quotes string
+   * values that contain whitespace characters.
+   *
+   * @return the instance's description as a string
+   */
+  public String toString() {
+
+    StringBuffer text = new StringBuffer();
+    
+    for (int i = 0; i < m_AttValues.length; i++) {
+      if (i > 0) text.append(",");
+      text.append(toString(i));
+    }
+
+    return text.toString();
+  }
+
+  /**
+   * Returns the description of one value of the instance as a 
+   * string. If the instance doesn't have access to a dataset, it 
+   * returns the internal floating-point value. Quotes string
+   * values that contain whitespace characters, or if they
+   * are a question mark.
+   *
+   * @param attIndex the attribute's index
+   * @return the value's description as a string
+   */
+  public final String toString(int attIndex) {
+
+   StringBuffer text = new StringBuffer();
+   
+   if (isMissing(attIndex)) {
+     text.append("?");
+   } else {
+     if (m_Dataset == null) {
+       text.append(Utils.doubleToString(m_AttValues[attIndex],6));
+     } else {
+       if (m_Dataset.attribute(attIndex).isNominal() || 
+	   m_Dataset.attribute(attIndex).isString()) {
+         text.append(Utils.quote(stringValue(attIndex)));
+       } else {
+	 text.append(Utils.doubleToString(value(attIndex),6));
+       }
+     }
+   }
+   return text.toString();
+  }
+
+  /**
+   * Returns the description of one value of the instance as a 
+   * string. If the instance doesn't have access to a dataset it 
+   * returns the internal floating-point value. Quotes string
+   * values that contain whitespace characters, or if they
+   * are a question mark.
+   * The given attribute has to belong to a dataset.
+   *
+   * @param att the attribute
+   * @return the value's description as a string
+   */
+  public final String toString(Attribute att) {
+   
+   return toString(att.index());
+  }
+
+  /**
+   * Returns an instance's attribute value in internal format.
+   *
+   * @param attIndex the attribute's index
+   * @return the specified value as a double (If the corresponding
+   * attribute is nominal (or a string) then it returns the value's index as a 
+   * double).
+   */
+  public double value(int attIndex) {
+
+    return m_AttValues[attIndex];
+  }
+
+  /**
+   * Returns an instance's attribute value in internal format.
+   * Does exactly the same thing as value() if applied to an Instance.
+   *
+   * @param indexOfIndex the index of the attribute's index
+   * @return the specified value as a double (If the corresponding
+   * attribute is nominal (or a string) then it returns the value's index as a 
+   * double).
+   */
+  public double valueSparse(int indexOfIndex) {
+
+    return m_AttValues[indexOfIndex];
+  }  
+
+  /**
+   * Returns an instance's attribute value in internal format.
+   * The given attribute has to belong to a dataset.
+   *
+   * @param att the attribute
+   * @return the specified value as a double (If the corresponding
+   * attribute is nominal (or a string) then it returns the value's index as a
+   * double).
+   */
+  public double value(Attribute att) {
+
+    return value(att.index());
+  }
+
+  /**
+   * Returns the instance's weight.
+   *
+   * @return the instance's weight as a double
+   */
+  public final double weight() {
+
+    return m_Weight;
+  }
+
+  /**
+   * Deletes an attribute at the given position (0 to 
+   * numAttributes() - 1).
+   *
+   * @param pos the attribute's position
+   */
+
+  void forceDeleteAttributeAt(int position) {
+
+    double[] newValues = new double[m_AttValues.length - 1];
+
+    System.arraycopy(m_AttValues, 0, newValues, 0, position);
+    if (position < m_AttValues.length - 1) {
+      System.arraycopy(m_AttValues, position + 1, 
+		       newValues, position, 
+		       m_AttValues.length - (position + 1));
+    }
+    m_AttValues = newValues;
+  }
+
+  /**
+   * Inserts an attribute at the given position
+   * (0 to numAttributes()) and sets its value to be missing. 
+   *
+   * @param pos the attribute's position
+   */
+  void forceInsertAttributeAt(int position)  {
+
+    double[] newValues = new double[m_AttValues.length + 1];
+
+    System.arraycopy(m_AttValues, 0, newValues, 0, position);
+    newValues[position] = MISSING_VALUE;
+    System.arraycopy(m_AttValues, position, newValues, 
+		     position + 1, m_AttValues.length - position);
+    m_AttValues = newValues;
+  }
+
+  /**
+   * Private constructor for subclasses. Does nothing.
+   */
+  protected Instance() {
+  }
+
+  /**
+   * Clones the attribute vector of the instance and
+   * overwrites it with the clone.
+   */
+  private void freshAttributeVector() {
+
+    m_AttValues = toDoubleArray();
+  }
+
+  /**
+   * Main method for testing this class.
+   */
+  public static void main(String[] options) {
+
+    try {
+
+      // Create numeric attributes "length" and "weight"
+      Attribute length = new Attribute("length");
+      Attribute weight = new Attribute("weight");
+      
+      // Create vector to hold nominal values "first", "second", "third" 
+      FastVector my_nominal_values = new FastVector(3); 
+      my_nominal_values.addElement("first"); 
+      my_nominal_values.addElement("second"); 
+      my_nominal_values.addElement("third"); 
+      
+      // Create nominal attribute "position" 
+      Attribute position = new Attribute("position", my_nominal_values);
+      
+      // Create vector of the above attributes 
+      FastVector attributes = new FastVector(3);
+      attributes.addElement(length);
+      attributes.addElement(weight);
+      attributes.addElement(position);
+      
+      // Create the empty dataset "race" with above attributes
+      Instances race = new Instances("race", attributes, 0);
+      
+      // Make position the class attribute
+      race.setClassIndex(position.index());
+      
+      // Create empty instance with three attribute values
+      Instance inst = new Instance(3);
+      
+      // Set instance's values for the attributes "length", "weight", and "position"
+      inst.setValue(length, 5.3);
+      inst.setValue(weight, 300);
+      inst.setValue(position, "first");
+      
+      // Set instance's dataset to be the dataset "race"
+      inst.setDataset(race);
+      
+      // Print the instance
+      System.out.println("The instance: " + inst);
+      
+      // Print the first attribute
+      System.out.println("First attribute: " + inst.attribute(0));
+      
+      // Print the class attribute
+      System.out.println("Class attribute: " + inst.classAttribute());
+      
+      // Print the class index
+      System.out.println("Class index: " + inst.classIndex());
+      
+      // Say if class is missing
+      System.out.println("Class is missing: " + inst.classIsMissing());
+      
+      // Print the instance's class value in internal format
+      System.out.println("Class value (internal format): " + inst.classValue());
+      
+      // Print a shallow copy of this instance
+      Instance copy = (Instance) inst.copy();
+      System.out.println("Shallow copy: " + copy);
+      
+      // Set dataset for shallow copy
+      copy.setDataset(inst.dataset());
+      System.out.println("Shallow copy with dataset set: " + copy);
+      
+      // Unset dataset for copy, delete first attribute, and insert it again
+      copy.setDataset(null);
+      copy.deleteAttributeAt(0);
+      copy.insertAttributeAt(0);
+      copy.setDataset(inst.dataset());
+      System.out.println("Copy with first attribute deleted and inserted: " + copy); 
+      
+      // Enumerate attributes (leaving out the class attribute)
+      System.out.println("Enumerating attributes (leaving out class):");
+      Enumeration enum = inst.enumerateAttributes();
+      while (enum.hasMoreElements()) {
+	Attribute att = (Attribute) enum.nextElement();
+	System.out.println(att);
+      }
+      
+      // Headers are equivalent?
+      System.out.println("Header of original and copy equivalent: " +
+			 inst.equalHeaders(copy));
+
+      // Test for missing values
+      System.out.println("Length of copy missing: " + copy.isMissing(length));
+      System.out.println("Weight of copy missing: " + copy.isMissing(weight.index()));
+      System.out.println("Length of copy missing: " + 
+			 Instance.isMissingValue(copy.value(length)));
+      System.out.println("Missing value coded as: " + Instance.missingValue());
+
+      // Prints number of attributes and classes
+      System.out.println("Number of attributes: " + copy.numAttributes());
+      System.out.println("Number of classes: " + copy.numClasses());
+
+      // Replace missing values
+      double[] meansAndModes = {2, 3, 0};
+      copy.replaceMissingValues(meansAndModes);
+      System.out.println("Copy with missing value replaced: " + copy);
+
+      // Setting and getting values and weights
+      copy.setClassMissing();
+      System.out.println("Copy with missing class: " + copy);
+      copy.setClassValue(0);
+      System.out.println("Copy with class value set to first value: " + copy);
+      copy.setClassValue("third");
+      System.out.println("Copy with class value set to \"third\": " + copy);
+      copy.setMissing(1);
+      System.out.println("Copy with second attribute set to be missing: " + copy);
+      copy.setMissing(length);
+      System.out.println("Copy with length set to be missing: " + copy);
+      copy.setValue(0, 0);
+      System.out.println("Copy with first attribute set to 0: " + copy);
+      copy.setValue(weight, 1);
+      System.out.println("Copy with weight attribute set to 1: " + copy);
+      copy.setValue(position, "second");
+      System.out.println("Copy with position set to \"second\": " + copy);
+      copy.setValue(2, "first");
+      System.out.println("Copy with last attribute set to \"first\": " + copy);
+      System.out.println("Current weight of instance copy: " + copy.weight());
+      copy.setWeight(2);
+      System.out.println("Current weight of instance copy (set to 2): " + copy.weight());
+      System.out.println("Last value of copy: " + copy.toString(2));
+      System.out.println("Value of position for copy: " + copy.toString(position));
+      System.out.println("Last value of copy (internal format): " + copy.value(2));
+      System.out.println("Value of position for copy (internal format): " + 
+			 copy.value(position));
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instances.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instances.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Instances.java	(revision 21112)
@@ -0,0 +1,2150 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Instances.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Class for handling an ordered set of weighted instances. <p>
+ *
+ * Typical usage (code from the main() method of this class): <p>
+ *
+ * <code>
+ * ... <br>
+ * 
+ * // Read all the instances in the file <br>
+ * reader = new FileReader(filename); <br>
+ * instances = new Instances(reader); <br><br>
+ *
+ * // Make the last attribute be the class <br>
+ * instances.setClassIndex(instances.numAttributes() - 1); <br><br>
+ * 
+ * // Print header and instances. <br>
+ * System.out.println("\nDataset:\n"); <br> 
+ * System.out.println(instances); <br><br>
+ *
+ * ... <br>
+ * </code><p>
+ *
+ * All methods that change a set of instances are safe, ie. a change
+ * of a set of instances does not affect any other sets of
+ * instances. All methods that change a datasets's attribute
+ * information clone the dataset before it is changed.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$ 
+ */
+public class Instances implements Serializable {
+ 
+  /** The filename extension that should be used for arff files */
+  public static String FILE_EXTENSION = ".arff";
+
+  /** The dataset's name. */
+  protected String m_RelationName;         
+
+  /** The attribute information. */
+  protected FastVector m_Attributes;
+
+  /** The instances. */
+  protected FastVector m_Instances;
+
+  /** The class attribute's index */
+  protected int m_ClassIndex;
+
+  /** Buffer of values for sparse instance */
+  protected double[] m_ValueBuffer;
+
+  /** Buffer of indices for sparse instance */
+  protected int[] m_IndicesBuffer;
+
+  /**
+   * Reads an ARFF file from a reader, and assigns a weight of
+   * one to each instance. Lets the index of the class 
+   * attribute be undefined (negative).
+   *
+   * @param reader the reader
+   * @exception IOException if the ARFF file is not read 
+   * successfully
+   */
+  public Instances(Reader reader) throws IOException {
+
+    StreamTokenizer tokenizer;
+
+    tokenizer = new StreamTokenizer(reader);
+    initTokenizer(tokenizer);
+    readHeader(tokenizer);
+    m_ClassIndex = -1;
+    m_Instances = new FastVector(1000);
+    while (getInstance(tokenizer, true)) {};
+    compactify();
+  }
+ 
+  /**
+   * Reads the header of an ARFF file from a reader and 
+   * reserves space for the given number of instances. Lets
+   * the class index be undefined (negative).
+   *
+   * @param reader the reader
+   * @param capacity the capacity
+   * @exception IllegalArgumentException if the header is not read successfully
+   * or the capacity is negative.
+   * @exception IOException if there is a problem with the reader.
+   */
+   public Instances(Reader reader, int capacity) throws IOException {
+
+    StreamTokenizer tokenizer;
+
+    if (capacity < 0) {
+      throw new IllegalArgumentException("Capacity has to be positive!");
+    }
+    tokenizer = new StreamTokenizer(reader); 
+    initTokenizer(tokenizer);
+    readHeader(tokenizer);
+    m_ClassIndex = -1;
+    m_Instances = new FastVector(capacity);
+  }
+
+  /**
+   * Constructor copying all instances and references to
+   * the header information from the given set of instances.
+   *
+   * @param instances the set to be copied
+   */
+  public Instances(Instances dataset) {
+
+    this(dataset, dataset.numInstances());
+
+    dataset.copyInstances(0, this, dataset.numInstances());
+  }
+
+  /**
+   * Constructor creating an empty set of instances. Copies references
+   * to the header information from the given set of instances. Sets
+   * the capacity of the set of instances to 0 if its negative.
+   *
+   * @param instances the instances from which the header 
+   * information is to be taken
+   * @param capacity the capacity of the new dataset 
+   */
+  public Instances(Instances dataset, int capacity) {
+    
+    if (capacity < 0) {
+      capacity = 0;
+    }
+    
+    // Strings only have to be "shallow" copied because
+    // they can't be modified.
+    m_ClassIndex = dataset.m_ClassIndex;
+    m_RelationName = dataset.m_RelationName;
+    m_Attributes = dataset.m_Attributes;
+    m_Instances = new FastVector(capacity);
+  }
+
+  /**
+   * Creates a new set of instances by copying a 
+   * subset of another set.
+   *
+   * @param source the set of instances from which a subset 
+   * is to be created
+   * @param first the index of the first instance to be copied
+   * @param toCopy the number of instances to be copied
+   * @exception IllegalArgumentException if first and toCopy are out of range
+   */
+  public Instances(Instances source, int first, int toCopy) {
+    
+    this(source, toCopy);
+
+    if ((first < 0) || ((first + toCopy) > source.numInstances())) {
+      throw new IllegalArgumentException("Parameters first and/or toCopy out "+
+                                         "of range");
+    }
+    source.copyInstances(first, this, toCopy);
+  }
+
+  /**
+   * Creates an empty set of instances. Uses the given
+   * attribute information. Sets the capacity of the set of 
+   * instances to 0 if its negative. Given attribute information
+   * must not be changed after this constructor has been used.
+   *
+   * @param name the name of the relation
+   * @param attInfo the attribute information
+   * @param capacity the capacity of the set
+   */
+  public Instances(String name, FastVector attInfo, int capacity) {
+
+    m_RelationName = name;
+    m_ClassIndex = -1;
+    m_Attributes = attInfo;
+    for (int i = 0; i < numAttributes(); i++) {
+      attribute(i).setIndex(i);
+    }
+    m_Instances = new FastVector(capacity);
+  }
+ 
+  /**
+   * Create a copy of the structure, but "cleanse" string types (i.e.
+   * doesn't contain references to the strings seen in the past).
+   *
+   * @return a copy of the instance structure.
+   */
+  public Instances stringFreeStructure() {
+
+    FastVector atts = (FastVector)m_Attributes.copy();
+    for (int i = 0 ; i < atts.size(); i++) {
+      Attribute att = (Attribute)atts.elementAt(i);
+      if (att.type() == Attribute.STRING) {
+        atts.setElementAt(new Attribute(att.name(), null), i);
+      }
+    }
+    Instances result = new Instances(relationName(), atts, 0);
+    result.m_ClassIndex = m_ClassIndex;
+    return result;
+  }
+
+  /**
+   * Adds one instance to the end of the set. 
+   * Shallow copies instance before it is added. Increases the
+   * size of the dataset if it is not large enough. Does not
+   * check if the instance is compatible with the dataset.
+   *
+   * @param instance the instance to be added
+   */
+  public final void add(Instance instance) {
+
+    Instance newInstance = (Instance)instance.copy();
+
+    newInstance.setDataset(this);
+    m_Instances.addElement(newInstance);
+  }
+
+  /**
+   * Returns an attribute.
+   *
+   * @param index the attribute's index
+   * @return the attribute at the given position
+   */ 
+  public final Attribute attribute(int index) {
+    
+    return (Attribute) m_Attributes.elementAt(index);
+  }
+
+  /**
+   * Returns an attribute given its name. If there is more than
+   * one attribute with the same name, it returns the first one.
+   * Returns null if the attribute can't be found.
+   *
+   * @param name the attribute's name
+   * @return the attribute with the given name, null if the
+   * attribute can't be found
+   */ 
+  public final Attribute attribute(String name) {
+    
+    for (int i = 0; i < numAttributes(); i++) {
+      if (attribute(i).name().equals(name)) {
+	return attribute(i);
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Checks for string attributes in the dataset
+   *
+   * @return true if string attributes are present, false otherwise
+   */
+  public boolean checkForStringAttributes() {
+
+    int i = 0;
+   
+    while (i < m_Attributes.size()) {
+      if (attribute(i++).isString()) {
+	return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Checks if the given instance is compatible
+   * with this dataset. Only looks at the size of
+   * the instance and the ranges of the values for 
+   * nominal and string attributes.
+   *
+   * @return true if the instance is compatible with the dataset 
+   */
+  public final boolean checkInstance(Instance instance) {
+
+    if (instance.numAttributes() != numAttributes()) {
+      return false;
+    }
+    for (int i = 0; i < numAttributes(); i++) {
+      if (instance.isMissing(i)) {
+	continue;
+      } else if (attribute(i).isNominal() ||
+		 attribute(i).isString()) {
+	if (!(Utils.eq(instance.value(i),
+		       (double)(int)instance.value(i)))) {
+	  return false;
+	} else if (Utils.sm(instance.value(i), 0) ||
+		   Utils.gr(instance.value(i),
+			    attribute(i).numValues())) {
+	  return false;
+	}
+      }
+    }
+    return true;
+  }
+	
+  /**
+   * Returns the class attribute.
+   *
+   * @return the class attribute
+   * @exception UnassignedClassException if the class is not set
+   */
+  public final Attribute classAttribute() {
+
+    if (m_ClassIndex < 0) {
+      throw new UnassignedClassException("Class index is negative (not set)!");
+    }
+    return attribute(m_ClassIndex);
+  }
+
+  /**
+   * Returns the class attribute's index. Returns negative number
+   * if it's undefined.
+   *
+   * @return the class index as an integer
+   */
+  public final int classIndex() {
+    
+    return m_ClassIndex;
+  }
+ 
+  /**
+   * Compactifies the set of instances. Decreases the capacity of
+   * the set so that it matches the number of instances in the set.
+   */
+  public final void compactify() {
+
+    m_Instances.trimToSize();
+  }
+
+  /**
+   * Removes all instances from the set.
+   */
+  public final void delete() {
+    
+    m_Instances = new FastVector();
+  }
+
+  /**
+   * Removes an instance at the given position from the set.
+   *
+   * @param index the instance's position
+   */
+  public final void delete(int index) {
+    
+    m_Instances.removeElementAt(index);
+  }
+
+  /**
+   * Deletes an attribute at the given position 
+   * (0 to numAttributes() - 1). A deep copy of the attribute
+   * information is performed before the attribute is deleted.
+   *
+   * @param pos the attribute's position
+   * @exception IllegalArgumentException if the given index is out of range or the
+   * class attribute is being deleted
+   */
+  public void deleteAttributeAt(int position) {
+	 
+    if ((position < 0) || (position >= m_Attributes.size())) {
+      throw new IllegalArgumentException("Index out of range");
+    }
+    if (position == m_ClassIndex) {
+      throw new IllegalArgumentException("Can't delete class attribute");
+    }
+    freshAttributeInfo();
+    if (m_ClassIndex > position) {
+      m_ClassIndex--;
+    }
+    m_Attributes.removeElementAt(position);
+    for (int i = position; i < m_Attributes.size(); i++) {
+      Attribute current = (Attribute)m_Attributes.elementAt(i);
+      current.setIndex(current.index() - 1);
+    }
+    for (int i = 0; i < numInstances(); i++) {
+      instance(i).forceDeleteAttributeAt(position); 
+    }
+  }
+
+  /**
+   * Deletes all string attributes in the dataset. A deep copy of the attribute
+   * information is performed before an attribute is deleted.
+   *
+   * @exception IllegalArgumentException if string attribute couldn't be 
+   * successfully deleted (probably because it is the class attribute).
+   */
+  public void deleteStringAttributes() {
+
+    int i = 0;
+    while (i < m_Attributes.size()) {
+      if (attribute(i).isString()) {
+	deleteAttributeAt(i);
+      } else {
+	i++;
+      }
+    }
+  }
+
+  /**
+   * Removes all instances with missing values for a particular
+   * attribute from the dataset.
+   *
+   * @param attIndex the attribute's index
+   */
+  public final void deleteWithMissing(int attIndex) {
+
+    FastVector newInstances = new FastVector(numInstances());
+
+    for (int i = 0; i < numInstances(); i++) {
+      if (!instance(i).isMissing(attIndex)) {
+	newInstances.addElement(instance(i));
+      }
+    }
+    m_Instances = newInstances;
+  }
+
+  /**
+   * Removes all instances with missing values for a particular
+   * attribute from the dataset.
+   *
+   * @param att the attribute
+   */
+  public final void deleteWithMissing(Attribute att) {
+
+    deleteWithMissing(att.index());
+  }
+
+  /**
+   * Removes all instances with a missing class value
+   * from the dataset.
+   *
+   * @exception UnassignedClassException if class is not set
+   */
+  public final void deleteWithMissingClass() {
+
+    if (m_ClassIndex < 0) {
+      throw new UnassignedClassException("Class index is negative (not set)!");
+    }
+    deleteWithMissing(m_ClassIndex);
+  }
+
+  /**
+   * Returns an enumeration of all the attributes.
+   *
+   * @return enumeration of all the attributes.
+   */
+  public Enumeration enumerateAttributes() {
+
+    return m_Attributes.elements(m_ClassIndex);
+  }
+
+  /**
+   * Returns an enumeration of all instances in the dataset.
+   *
+   * @return enumeration of all instances in the dataset
+   */
+  public final Enumeration enumerateInstances() {
+
+    return m_Instances.elements();
+  }
+
+  /**
+   * Checks if two headers are equivalent.
+   *
+   * @param dataset another dataset
+   * @return true if the header of the given dataset is equivalent 
+   * to this header
+   */
+  public final boolean equalHeaders(Instances dataset){
+
+    // Check class and all attributes
+    if (m_ClassIndex != dataset.m_ClassIndex) {
+      return false;
+    }
+    if (m_Attributes.size() != dataset.m_Attributes.size()) {
+      return false;
+    }
+    for (int i = 0; i < m_Attributes.size(); i++) {
+      if (!(attribute(i).equals(dataset.attribute(i)))) {
+	return false;
+      }
+    }
+    return true;
+  }
+ 
+  /**
+   * Returns the first instance in the set.
+   *
+   * @return the first instance in the set
+   */
+  public final Instance firstInstance() {
+    
+    return (Instance)m_Instances.firstElement();
+  }
+ 
+  /**
+   * Inserts an attribute at the given position (0 to 
+   * numAttributes()) and sets all values to be missing.
+   * Shallow copies the attribute before it is inserted, and performs
+   * a deep copy of the existing attribute information.
+   *
+   * @param att the attribute to be inserted
+   * @param pos the attribute's position
+   * @exception IllegalArgumentException if the given index is out of range
+   */
+  public void insertAttributeAt(Attribute att, int position) {
+	 
+    if ((position < 0) ||
+	(position > m_Attributes.size())) {
+      throw new IllegalArgumentException("Index out of range");
+    }
+    att = (Attribute)att.copy();
+    freshAttributeInfo();
+    att.setIndex(position);
+    m_Attributes.insertElementAt(att, position);
+    for (int i = position + 1; i < m_Attributes.size(); i++) {
+      Attribute current = (Attribute)m_Attributes.elementAt(i);
+      current.setIndex(current.index() + 1);
+    }
+    for (int i = 0; i < numInstances(); i++) {
+      instance(i).forceInsertAttributeAt(position);
+    }
+    if (m_ClassIndex >= position) {
+      m_ClassIndex++;
+    }
+  }
+
+  /**
+   * Returns the instance at the given position.
+   *
+   * @param index the instance's index
+   * @return the instance at the given position
+   */
+  public final Instance instance(int index) {
+
+    return (Instance)m_Instances.elementAt(index);
+  }
+
+  /**
+   * Returns the last instance in the set.
+   *
+   * @return the last instance in the set
+   */
+  public final Instance lastInstance() {
+    
+    return (Instance)m_Instances.lastElement();
+  }
+
+  /**
+   * Returns the mean (mode) for a numeric (nominal) attribute as
+   * a floating-point value. Returns 0 if the attribute is neither nominal nor 
+   * numeric. If all values are missing it returns zero.
+   *
+   * @param attIndex the attribute's index
+   * @return the mean or the mode
+   */
+  public final double meanOrMode(int attIndex) {
+
+    double result, found;
+    int [] counts;
+
+    if (attribute(attIndex).isNumeric()) {
+      result = found = 0;
+      for (int j = 0; j < numInstances(); j++) {
+	if (!instance(j).isMissing(attIndex)) {
+	  found += instance(j).weight();
+	  result += instance(j).weight()*instance(j).value(attIndex);
+	}
+      }
+      if (Utils.eq(found, 0)) {
+	return 0;
+      } else {
+	return result / found;
+      }
+    } else if (attribute(attIndex).isNominal()) {
+      counts = new int[attribute(attIndex).numValues()];
+      for (int j = 0; j < numInstances(); j++) {
+	if (!instance(j).isMissing(attIndex)) {
+	  counts[(int) instance(j).value(attIndex)] += instance(j).weight();
+	}
+      }
+      return (double)Utils.maxIndex(counts);
+    } else {
+      return 0;
+    }
+  }
+
+  /**
+   * Returns the mean (mode) for a numeric (nominal) attribute as a
+   * floating-point value.  Returns 0 if the attribute is neither
+   * nominal nor numeric.  If all values are missing it returns zero.
+   *
+   * @param att the attribute
+   * @return the mean or the mode 
+   */
+  public final double meanOrMode(Attribute att) {
+
+    return meanOrMode(att.index());
+  }
+
+  /**
+   * Returns the number of attributes.
+   *
+   * @return the number of attributes as an integer
+   */
+  public final int numAttributes() {
+
+    return m_Attributes.size();
+  }
+
+  /**
+   * Returns the number of class labels.
+   *
+   * @return the number of class labels as an integer if the class 
+   * attribute is nominal, 1 otherwise.
+   * @exception UnassignedClassException if the class is not set
+   */
+  public final int numClasses() {
+    
+    if (m_ClassIndex < 0) {
+      throw new UnassignedClassException("Class index is negative (not set)!");
+    }
+    if (!classAttribute().isNominal()) {
+      return 1;
+    } else {
+      return classAttribute().numValues();
+    }
+  }
+
+  /**
+   * Returns the number of distinct values of a given attribute.
+   * Returns the number of instances if the attribute is a
+   * string attribute. The value 'missing' is not counted.
+   *
+   * @param attIndex the attribute
+   * @return the number of distinct values of a given attribute
+   */
+  public final int numDistinctValues(int attIndex) {
+
+    if (attribute(attIndex).isNumeric()) {
+      double [] attVals = attributeToDoubleArray(attIndex);
+      int [] sorted = Utils.sort(attVals);
+      double prev = 0;
+      int counter = 0;
+      for (int i = 0; i < sorted.length; i++) {
+	Instance current = instance(sorted[i]);
+	if (current.isMissing(attIndex)) {
+	  break;
+	}
+	if ((i == 0) || 
+	    Utils.gr(current.value(attIndex), prev)) {
+	  prev = current.value(attIndex);
+	  counter++;
+	}
+      }
+      return counter;
+    } else {
+      return attribute(attIndex).numValues();
+    }
+  }
+
+  /**
+   * Returns the number of distinct values of a given attribute.
+   * Returns the number of instances if the attribute is a
+   * string attribute. The value 'missing' is not counted.
+   *
+   * @param att the attribute
+   * @return the number of distinct values of a given attribute
+   */
+  public final int numDistinctValues(Attribute att) {
+
+    return numDistinctValues(att.index());
+  }
+  
+  /**
+   * Returns the number of instances in the dataset.
+   *
+   * @return the number of instances in the dataset as an integer
+   */
+  public final int numInstances() {
+
+    return m_Instances.size();
+  }
+
+  /**
+   * Shuffles the instances in the set so that they are ordered 
+   * randomly.
+   *
+   * @param random a random number generator
+   */
+  public final void randomize(Random random) {
+
+    for (int j = numInstances() - 1; j > 0; j--)
+      swap(j,(int)(random.nextDouble()*(double)j));
+  }
+
+  /**
+   * Reads a single instance from the reader and appends it
+   * to the dataset.  Automatically expands the dataset if it
+   * is not large enough to hold the instance. This method does
+   * not check for carriage return at the end of the line.
+   *
+   * @param reader the reader 
+   * @return false if end of file has been reached
+   * @exception IOException if the information is not read 
+   * successfully
+   */ 
+  public final boolean readInstance(Reader reader) 
+       throws IOException {
+
+    StreamTokenizer tokenizer = new StreamTokenizer(reader);
+    
+    initTokenizer(tokenizer);
+    return getInstance(tokenizer, false);
+  }    
+
+  /**
+   * Returns the relation's name.
+   *
+   * @return the relation's name as a string
+   */
+  public final String relationName() {
+
+    return m_RelationName;
+  }
+
+  /**
+   * Renames an attribute. This change only affects this
+   * dataset.
+   *
+   * @param att the attribute's index
+   * @param name the new name
+   */
+  public final void renameAttribute(int att, String name) {
+
+    Attribute newAtt = attribute(att).copy(name);
+    FastVector newVec = new FastVector(numAttributes());
+
+    for (int i = 0; i < numAttributes(); i++) {
+      if (i == att) {
+	newVec.addElement(newAtt);
+      } else {
+	newVec.addElement(attribute(i));
+      }
+    }
+    m_Attributes = newVec;
+  }
+
+  /**
+   * Renames an attribute. This change only affects this
+   * dataset.
+   *
+   * @param att the attribute
+   * @param name the new name
+   */
+  public final void renameAttribute(Attribute att, String name) {
+
+    renameAttribute(att.index(), name);
+  }
+
+  /**
+   * Renames the value of a nominal (or string) attribute value. This
+   * change only affects this dataset.
+   *
+   * @param att the attribute's index
+   * @param val the value's index
+   * @param name the new name 
+   */
+  public final void renameAttributeValue(int att, int val, String name) {
+
+    Attribute newAtt = (Attribute)attribute(att).copy();
+    FastVector newVec = new FastVector(numAttributes());
+
+    newAtt.setValue(val, name);
+    for (int i = 0; i < numAttributes(); i++) {
+      if (i == att) {
+	newVec.addElement(newAtt);
+      } else {
+	newVec.addElement(attribute(i));
+      }
+    }
+    m_Attributes = newVec;
+  }
+
+  /**
+   * Renames the value of a nominal (or string) attribute value. This
+   * change only affects this dataset.
+   *
+   * @param att the attribute
+   * @param val the value
+   * @param name the new name
+   */
+  public final void renameAttributeValue(Attribute att, String val, 
+                                         String name) {
+
+    renameAttributeValue(att.index(), att.indexOfValue(val), name);
+  }
+
+  /**
+   * Creates a new dataset of the same size using random sampling
+   * with replacement.
+   *
+   * @param random a random number generator
+   * @return the new dataset
+   */
+  public final Instances resample(Random random) {
+
+    Instances newData = new Instances(this, numInstances());
+    while (newData.numInstances() < numInstances()) {
+      int i = (int) (random.nextDouble() * (double) numInstances());
+      newData.add(instance(i));
+    }
+    return newData;
+  }
+
+  /**
+   * Creates a new dataset of the same size using random sampling
+   * with replacement according to the current instance weights. The
+   * weights of the instances in the new dataset are set to one.
+   *
+   * @param random a random number generator
+   * @return the new dataset
+   */
+  public final Instances resampleWithWeights(Random random) {
+
+    double [] weights = new double[numInstances()];
+    boolean foundOne = false;
+    for (int i = 0; i < weights.length; i++) {
+      weights[i] = instance(i).weight();
+      if (!Utils.eq(weights[i], weights[0])) {
+        foundOne = true;
+      }
+    }
+    if (foundOne) {
+      return resampleWithWeights(random, weights);
+    } else {
+      return new Instances(this);
+    }
+  }
+
+
+  /**
+   * Creates a new dataset of the same size using random sampling
+   * with replacement according to the given weight vector. The
+   * weights of the instances in the new dataset are set to one.
+   * The length of the weight vector has to be the same as the
+   * number of instances in the dataset, and all weights have to
+   * be positive.
+   *
+   * @param random a random number generator
+   * @param weights the weight vector
+   * @return the new dataset
+   * @exception IllegalArgumentException if the weights array is of the wrong
+   * length or contains negative weights.
+   */
+  public final Instances resampleWithWeights(Random random, 
+					     double[] weights) {
+
+    if (weights.length != numInstances()) {
+      throw new IllegalArgumentException("weights.length != numInstances.");
+    }
+    Instances newData = new Instances(this, numInstances());
+    double[] probabilities = new double[numInstances()];
+    double sumProbs = 0, sumOfWeights = Utils.sum(weights);
+    for (int i = 0; i < numInstances(); i++) {
+      sumProbs += random.nextDouble();
+      probabilities[i] = sumProbs;
+    }
+    Utils.normalize(probabilities, sumProbs / sumOfWeights);
+
+    // Make sure that rounding errors don't mess things up
+    probabilities[numInstances() - 1] = sumOfWeights;
+    int k = 0; int l = 0;
+    sumProbs = 0;
+    while ((k < numInstances() && (l < numInstances()))) {
+      if (weights[l] < 0) {
+	throw new IllegalArgumentException("Weights have to be positive.");
+      }
+      sumProbs += weights[l];
+      while ((k < numInstances()) &&
+	     (probabilities[k] <= sumProbs)) { 
+	newData.add(instance(l));
+	newData.instance(k).setWeight(1);
+	k++;
+      }
+      l++;
+    }
+    return newData;
+  }
+
+  /** 
+   * Sets the class attribute.
+   *
+   * @param att attribute to be the class
+   */
+  public final void setClass(Attribute att) {
+
+    m_ClassIndex = att.index();
+  }
+
+  /** 
+   * Sets the class index of the set.
+   * If the class index is negative there is assumed to be no class.
+   * (ie. it is undefined)
+   *
+   * @param classIndex the new class index
+   * @exception IllegalArgumentException if the class index is too big or < 0
+   */
+  public final void setClassIndex(int classIndex) {
+
+    if (classIndex >= numAttributes()) {
+      throw new IllegalArgumentException("Invalid class index: " + classIndex);
+    }
+    m_ClassIndex = classIndex;
+  }
+
+  /**
+   * Sets the relation's name.
+   *
+   * @param newName the new relation name.
+   */
+  public final void setRelationName(String newName) {
+    
+    m_RelationName = newName;
+  }
+
+  /**
+   * Sorts the instances based on an attribute. For numeric attributes, 
+   * instances are sorted in ascending order. For nominal attributes, 
+   * instances are sorted based on the attribute label ordering 
+   * specified in the header. Instances with missing values for the 
+   * attribute are placed at the end of the dataset.
+   *
+   * @param attIndex the attribute's index
+   */
+  public final void sort(int attIndex) {
+
+    int i,j;
+
+    // move all instances with missing values to end
+    j = numInstances() - 1;
+    i = 0;
+    while (i <= j) {
+      if (instance(j).isMissing(attIndex)) {
+	j--;
+      } else {
+	if (instance(i).isMissing(attIndex)) {
+	  swap(i,j);
+	  j--;
+	}
+	i++;
+      }
+    }
+    quickSort(attIndex, 0, j);
+  }
+
+  /**
+   * Sorts the instances based on an attribute. For numeric attributes, 
+   * instances are sorted into ascending order. For nominal attributes, 
+   * instances are sorted based on the attribute label ordering 
+   * specified in the header. Instances with missing values for the 
+   * attribute are placed at the end of the dataset.
+   *
+   * @param att the attribute
+   */
+  public final void sort(Attribute att) {
+
+    sort(att.index());
+  }
+
+  /**
+   * Stratifies a set of instances according to its class values 
+   * if the class attribute is nominal (so that afterwards a 
+   * stratified cross-validation can be performed).
+   *
+   * @param numFolds the number of folds in the cross-validation
+   * @exception UnassignedClassException if the class is not set
+   */
+  public final void stratify(int numFolds) {
+    
+    if (numFolds <= 0) {
+      throw new IllegalArgumentException("Number of folds must be greater than 1");
+    }
+    if (m_ClassIndex < 0) {
+      throw new UnassignedClassException("Class index is negative (not set)!");
+    }
+    if (classAttribute().isNominal()) {
+
+      // sort by class
+      int index = 1;
+      while (index < numInstances()) {
+	Instance instance1 = instance(index - 1);
+	for (int j = index; j < numInstances(); j++) {
+	  Instance instance2 = instance(j);
+	  if ((instance1.classValue() == instance2.classValue()) ||
+	      (instance1.classIsMissing() && 
+	       instance2.classIsMissing())) {
+	    swap(index,j);
+	    index++;
+	  }
+	}
+	index++;
+      }
+      stratStep(numFolds);
+    }
+  }
+ 
+  /**
+   * Computes the sum of all the instances' weights.
+   *
+   * @return the sum of all the instances' weights as a double
+   */
+  public final double sumOfWeights() {
+    
+    double sum = 0;
+
+    for (int i = 0; i < numInstances(); i++) {
+      sum += instance(i).weight();
+    }
+    return sum;
+  }
+
+  /**
+   * Creates the test set for one fold of a cross-validation on 
+   * the dataset.
+   *
+   * @param numFolds the number of folds in the cross-validation. Must
+   * be greater than 1.
+   * @param numFold 0 for the first fold, 1 for the second, ...
+   * @return the test set as a set of weighted instances
+   * @exception IllegalArgumentException if the number of folds is less than 2
+   * or greater than the number of instances.
+   */
+  public Instances testCV(int numFolds, int numFold) {
+
+    int numInstForFold, first, offset;
+    Instances test;
+    
+    if (numFolds < 2) {
+      throw new IllegalArgumentException("Number of folds must be at least 2!");
+    }
+    if (numFolds > numInstances()) {
+      throw new IllegalArgumentException("Can't have more folds than instances!");
+    }
+    numInstForFold = numInstances() / numFolds;
+    if (numFold < numInstances() % numFolds){
+      numInstForFold++;
+      offset = numFold;
+    }else
+      offset = numInstances() % numFolds;
+    test = new Instances(this, numInstForFold);
+    first = numFold * (numInstances() / numFolds) + offset;
+    copyInstances(first, test, numInstForFold);
+    return test;
+  }
+ 
+  /**
+   * Returns the dataset as a string in ARFF format. Strings
+   * are quoted if they contain whitespace characters, or if they
+   * are a question mark.
+   *
+   * @return the dataset in ARFF format as a string
+   */
+  public final String toString() {
+    
+    StringBuffer text = new StringBuffer();
+    
+    text.append("@relation " + Utils.quote(m_RelationName) + "\n\n");
+    for (int i = 0; i < numAttributes(); i++) {
+      text.append(attribute(i) + "\n");
+    }
+    text.append("\n@data\n");
+    for (int i = 0; i < numInstances(); i++) {
+      text.append(instance(i));
+      if (i < numInstances() - 1) {
+	text.append('\n');
+      }
+    }
+    return text.toString();
+  }
+
+  /**
+   * Creates the training set for one fold of a cross-validation 
+   * on the dataset.
+   *
+   * @param numFolds the number of folds in the cross-validation. Must
+   * be greater than 1.
+   * @param numFold 0 for the first fold, 1 for the second, ...
+   * @return the training set as a set of weighted 
+   * instances
+   * @exception IllegalArgumentException if the number of folds is less than 2
+   * or greater than the number of instances.
+   */
+  public Instances trainCV(int numFolds, int numFold) {
+
+    int numInstForFold, first, offset;
+    Instances train;
+ 
+    if (numFolds < 2) {
+      throw new IllegalArgumentException("Number of folds must be at least 2!");
+    }
+    if (numFolds > numInstances()) {
+      throw new IllegalArgumentException("Can't have more folds than instances!");
+    }
+    numInstForFold = numInstances() / numFolds;
+    if (numFold < numInstances() % numFolds) {
+      numInstForFold++;
+      offset = numFold;
+    }else
+      offset = numInstances() % numFolds;
+    train = new Instances(this, numInstances() - numInstForFold);
+    first = numFold * (numInstances() / numFolds) + offset;
+    copyInstances(0, train, first);
+    copyInstances(first + numInstForFold, train,
+		  numInstances() - first - numInstForFold);
+
+    return train;
+  }
+
+  /**
+   * Computes the variance for a numeric attribute.
+   *
+   * @param attIndex the numeric attribute
+   * @return the variance if the attribute is numeric
+   * @exception IllegalArgumentException if the attribute is not numeric
+   */
+  public final double variance(int attIndex) {
+  
+    double sum = 0, sumSquared = 0, sumOfWeights = 0;
+
+    if (!attribute(attIndex).isNumeric()) {
+      throw new IllegalArgumentException("Can't compute variance because attribute is " +
+			  "not numeric!");
+    }
+    for (int i = 0; i < numInstances(); i++) {
+      if (!instance(i).isMissing(attIndex)) {
+	sum += instance(i).weight() * 
+	  instance(i).value(attIndex);
+	sumSquared += instance(i).weight() * 
+	  instance(i).value(attIndex) *
+	  instance(i).value(attIndex);
+	sumOfWeights += instance(i).weight();
+      }
+    }
+    if (Utils.smOrEq(sumOfWeights, 1)) {
+      return 0;
+    }
+    return (sumSquared - (sum * sum / sumOfWeights)) / 
+      (sumOfWeights - 1);
+  }
+
+  /**
+   * Computes the variance for a numeric attribute.
+   *
+   * @param att the numeric attribute
+   * @return the variance if the attribute is numeric
+   * @exception IllegalArgumentException if the attribute is not numeric
+   */
+  public final double variance(Attribute att) {
+    
+    return variance(att.index());
+  }
+  
+  /**
+   * Calculates summary statistics on the values that appear in this
+   * set of instances for a specified attribute.
+   *
+   * @param index the index of the attribute to summarize.
+   * @return an AttributeStats object with it's fields calculated.
+   */
+  public AttributeStats attributeStats(int index) {
+
+    AttributeStats result = new AttributeStats();
+    if (attribute(index).isNominal()) {
+      result.nominalCounts = new int [attribute(index).numValues()];
+    }
+    if (attribute(index).isNumeric()) {
+      result.numericStats = new weka.experiment.Stats();
+    }
+    result.totalCount = numInstances();
+
+    double [] attVals = attributeToDoubleArray(index);
+    int [] sorted = Utils.sort(attVals);
+    int currentCount = 0;
+    double prev = Instance.missingValue();
+    for (int j = 0; j < numInstances(); j++) {
+      Instance current = instance(sorted[j]);
+      if (current.isMissing(index)) {
+	result.missingCount = numInstances() - j;
+	break;
+      }
+      if (Utils.eq(current.value(index), prev)) {
+	currentCount++;
+      } else {
+	result.addDistinct(prev, currentCount);
+	currentCount = 1;
+	prev = current.value(index);
+      }
+    }
+    result.addDistinct(prev, currentCount);
+    result.distinctCount--; // So we don't count "missing" as a value 
+    return result;
+  }
+  
+  /**
+   * Gets the value of all instances in this dataset for a particular
+   * attribute. Useful in conjunction with Utils.sort to allow iterating
+   * through the dataset in sorted order for some attribute.
+   *
+   * @param index the index of the attribute.
+   * @return an array containing the value of the desired attribute for
+   * each instance in the dataset. 
+   */
+  public double [] attributeToDoubleArray(int index) {
+
+    double [] result = new double[numInstances()];
+    for (int i = 0; i < result.length; i++) {
+      result[i] = instance(i).value(index);
+    }
+    return result;
+  }
+
+  /**
+   * Generates a string summarizing the set of instances. Gives a breakdown
+   * for each attribute indicating the number of missing/discrete/unique
+   * values and other information.
+   *
+   * @return a string summarizing the dataset
+   */
+  public String toSummaryString() {
+
+    StringBuffer result = new StringBuffer();
+    result.append("Relation Name:  ").append(relationName()).append('\n');
+    result.append("Num Instances:  ").append(numInstances()).append('\n');
+    result.append("Num Attributes: ").append(numAttributes()).append('\n');
+    result.append('\n');
+
+    result.append(Utils.padLeft("", 5)).append(Utils.padRight("Name", 25));
+    result.append(Utils.padLeft("Type", 5)).append(Utils.padLeft("Nom", 5));
+    result.append(Utils.padLeft("Int", 5)).append(Utils.padLeft("Real", 5));
+    result.append(Utils.padLeft("Missing", 12));
+    result.append(Utils.padLeft("Unique", 12));
+    result.append(Utils.padLeft("Dist", 6)).append('\n');
+    for (int i = 0; i < numAttributes(); i++) {
+      Attribute a = attribute(i);
+      AttributeStats as = attributeStats(i);
+      result.append(Utils.padLeft("" + (i + 1), 4)).append(' ');
+      result.append(Utils.padRight(a.name(), 25)).append(' ');
+      long percent;
+      switch (a.type()) {
+      case Attribute.NOMINAL:
+	result.append(Utils.padLeft("Nom", 4)).append(' ');
+	percent = Math.round(100.0 * as.intCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	result.append(Utils.padLeft("" + 0, 3)).append("% ");
+	percent = Math.round(100.0 * as.realCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	break;
+      case Attribute.NUMERIC:
+	result.append(Utils.padLeft("Num", 4)).append(' ');
+	result.append(Utils.padLeft("" + 0, 3)).append("% ");
+	percent = Math.round(100.0 * as.intCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	percent = Math.round(100.0 * as.realCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	break;
+      case Attribute.STRING:
+	result.append(Utils.padLeft("Str", 4)).append(' ');
+	percent = Math.round(100.0 * as.intCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	result.append(Utils.padLeft("" + 0, 3)).append("% ");
+	percent = Math.round(100.0 * as.realCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	break;
+      default:
+	result.append(Utils.padLeft("???", 4)).append(' ');
+	result.append(Utils.padLeft("" + 0, 3)).append("% ");
+	percent = Math.round(100.0 * as.intCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	percent = Math.round(100.0 * as.realCount / as.totalCount);
+	result.append(Utils.padLeft("" + percent, 3)).append("% ");
+	break;
+      }
+      result.append(Utils.padLeft("" + as.missingCount, 5)).append(" /");
+      percent = Math.round(100.0 * as.missingCount / as.totalCount);
+      result.append(Utils.padLeft("" + percent, 3)).append("% ");
+      result.append(Utils.padLeft("" + as.uniqueCount, 5)).append(" /");
+      percent = Math.round(100.0 * as.uniqueCount / as.totalCount);
+      result.append(Utils.padLeft("" + percent, 3)).append("% ");
+      result.append(Utils.padLeft("" + as.distinctCount, 5)).append(' ');
+      result.append('\n');
+    }
+    return result.toString();
+  }
+  
+  /**
+   * Reads a single instance using the tokenizer and appends it
+   * to the dataset. Automatically expands the dataset if it
+   * is not large enough to hold the instance.
+   *
+   * @param tokenizer the tokenizer to be used
+   * @param flag if method should test for carriage return after 
+   * each instance
+   * @return false if end of file has been reached
+   * @exception IOException if the information is not read 
+   * successfully
+   */ 
+  protected boolean getInstance(StreamTokenizer tokenizer, 
+				boolean flag) 
+       throws IOException {
+    
+    // Check if any attributes have been declared.
+    if (m_Attributes.size() == 0) {
+      errms(tokenizer,"no header information available");
+    }
+
+    // Check if end of file reached.
+    getFirstToken(tokenizer);
+    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
+      return false;
+    }
+    
+    // Parse instance
+    if (tokenizer.ttype == '{') {
+      return getInstanceSparse(tokenizer, flag);
+    } else {
+      return getInstanceFull(tokenizer, flag);
+    }
+  }
+
+  /**
+   * Reads a single instance using the tokenizer and appends it
+   * to the dataset. Automatically expands the dataset if it
+   * is not large enough to hold the instance.
+   *
+   * @param tokenizer the tokenizer to be used
+   * @param flag if method should test for carriage return after 
+   * each instance
+   * @return false if end of file has been reached
+   * @exception IOException if the information is not read 
+   * successfully
+   */ 
+  protected boolean getInstanceSparse(StreamTokenizer tokenizer, 
+				      boolean flag) 
+       throws IOException {
+
+    int valIndex, numValues = 0, maxIndex = -1;
+    
+    // Get values
+    do {
+      
+      // Get index
+      getIndex(tokenizer);
+      if (tokenizer.ttype == '}') {
+	break;
+      }
+       
+      // Is index valid?
+      try{
+	m_IndicesBuffer[numValues] = Integer.valueOf(tokenizer.sval).intValue();
+      } catch (NumberFormatException e) {
+	errms(tokenizer,"index number expected");
+      }
+      if (m_IndicesBuffer[numValues] <= maxIndex) {
+	errms(tokenizer,"indices have to be ordered");
+      }
+      if ((m_IndicesBuffer[numValues] < 0) || 
+	  (m_IndicesBuffer[numValues] >= numAttributes())) {
+	errms(tokenizer,"index out of bounds");
+      }
+      maxIndex = m_IndicesBuffer[numValues];
+
+      // Get value;
+      getNextToken(tokenizer);
+
+      // Check if value is missing.
+      if  (tokenizer.ttype == '?') {
+	m_ValueBuffer[numValues] = Instance.missingValue();
+      } else {
+
+	// Check if token is valid.
+	if (tokenizer.ttype != StreamTokenizer.TT_WORD) {
+	  errms(tokenizer,"not a valid value");
+	}
+	if (attribute(m_IndicesBuffer[numValues]).isNominal()) {
+	  
+	  // Check if value appears in header.
+	  valIndex = 
+	    attribute(m_IndicesBuffer[numValues]).indexOfValue(tokenizer.sval);
+	  if (valIndex == -1) {
+	    errms(tokenizer,"nominal value not declared in header");
+	  }
+	  m_ValueBuffer[numValues] = (double)valIndex;
+	} else if (attribute(m_IndicesBuffer[numValues]).isNumeric()) {
+	  
+	  // Check if value is really a number.
+	  try{
+	    m_ValueBuffer[numValues] = Double.valueOf(tokenizer.sval).
+	      doubleValue();
+	  } catch (NumberFormatException e) {
+	    errms(tokenizer,"number expected");
+	  }
+	} else { 
+	  m_ValueBuffer[numValues] = 
+	    attribute(m_IndicesBuffer[numValues]).addStringValue(tokenizer.sval);
+	}
+      }
+      numValues++;
+    } while (true);
+    if (flag) {
+      getLastToken(tokenizer,true);
+    }
+      
+    // Add instance to dataset
+    double[] tempValues = new double[numValues];
+    int[] tempIndices = new int[numValues];
+    System.arraycopy(m_ValueBuffer, 0, tempValues, 0, numValues);
+    System.arraycopy(m_IndicesBuffer, 0, tempIndices, 0, numValues);
+    add(new SparseInstance(1, tempValues, tempIndices, numAttributes()));
+    return true;
+  }
+
+  /**
+   * Reads a single instance using the tokenizer and appends it
+   * to the dataset. Automatically expands the dataset if it
+   * is not large enough to hold the instance.
+   *
+   * @param tokenizer the tokenizer to be used
+   * @param flag if method should test for carriage return after 
+   * each instance
+   * @return false if end of file has been reached
+   * @exception IOException if the information is not read 
+   * successfully
+   */ 
+  protected boolean getInstanceFull(StreamTokenizer tokenizer, 
+				    boolean flag) 
+       throws IOException {
+
+    double[] instance = new double[numAttributes()];
+    int index;
+    
+    // Get values for all attributes.
+    for (int i = 0; i < numAttributes(); i++){
+      
+      // Get next token
+      if (i > 0) {
+	getNextToken(tokenizer);
+      }
+            
+      // Check if value is missing.
+      if  (tokenizer.ttype == '?') {
+	instance[i] = Instance.missingValue();
+      } else {
+
+	// Check if token is valid.
+	if (tokenizer.ttype != StreamTokenizer.TT_WORD) {
+	  errms(tokenizer,"not a valid value");
+	}
+	if (attribute(i).isNominal()) {
+	  
+	  // Check if value appears in header.
+	  index = attribute(i).indexOfValue(tokenizer.sval);
+	  if (index == -1) {
+	    errms(tokenizer,"nominal value not declared in header");
+	  }
+	  instance[i] = (double)index;
+	} else if (attribute(i).isNumeric()) {
+	  
+	  // Check if value is really a number.
+	  try{
+	    instance[i] = Double.valueOf(tokenizer.sval).
+	      doubleValue();
+	  } catch (NumberFormatException e) {
+	    errms(tokenizer,"number expected");
+	  }
+	} else { 
+	  instance[i] = attribute(i).addStringValue(tokenizer.sval);
+	}
+      }
+    }
+    if (flag) {
+      getLastToken(tokenizer,true);
+    }
+      
+    // Add instance to dataset
+    add(new Instance(1, instance));
+    return true;
+  }
+
+  /**
+   * Reads and stores header of an ARFF file.
+   *
+   * @param tokenizer the stream tokenizer
+   * @exception IOException if the information is not read 
+   * successfully
+   */ 
+  protected void readHeader(StreamTokenizer tokenizer) 
+     throws IOException{
+    
+    String attributeName;
+    FastVector attributeValues;
+    int i;
+
+    // Get name of relation.
+    getFirstToken(tokenizer);
+    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
+      errms(tokenizer,"premature end of file");
+    }
+    if (tokenizer.sval.equalsIgnoreCase("@relation")){
+      getNextToken(tokenizer);
+      m_RelationName = tokenizer.sval;
+      getLastToken(tokenizer,false);
+    } else {
+      errms(tokenizer,"keyword @relation expected");
+    }
+
+    // Create vectors to hold information temporarily.
+    m_Attributes = new FastVector();
+ 
+    // Get attribute declarations.
+    getFirstToken(tokenizer);
+    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
+      errms(tokenizer,"premature end of file");
+    }
+    while (tokenizer.sval.equalsIgnoreCase("@attribute")) {
+
+      // Get attribute name.
+      getNextToken(tokenizer);
+      attributeName = tokenizer.sval;
+      getNextToken(tokenizer);
+
+      // Check if attribute is nominal.
+      if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
+
+	// Attribute is real, integer, or string.
+	if (tokenizer.sval.equalsIgnoreCase("real") ||
+	    tokenizer.sval.equalsIgnoreCase("integer") ||
+	    tokenizer.sval.equalsIgnoreCase("numeric")) {
+	  m_Attributes.addElement(new Attribute(attributeName,
+						 numAttributes()));
+	  readTillEOL(tokenizer);
+	} else if (tokenizer.sval.equalsIgnoreCase("string")) {
+	  m_Attributes.
+	    addElement(new Attribute(attributeName, null,
+				     numAttributes()));
+	  readTillEOL(tokenizer);
+	} else {
+	  errms(tokenizer,"no valid attribute type or invalid "+
+		"enumeration");
+	}
+      } else {
+
+	// Attribute is nominal.
+	attributeValues = new FastVector();
+	tokenizer.pushBack();
+	
+	// Get values for nominal attribute.
+	if (tokenizer.nextToken() != '{') {
+	  errms(tokenizer,"{ expected at beginning of enumeration");
+	}
+	while (tokenizer.nextToken() != '}') {
+	  if (tokenizer.ttype == StreamTokenizer.TT_EOL) {
+	    errms(tokenizer,"} expected at end of enumeration");
+	  } else {
+	    attributeValues.addElement(tokenizer.sval);
+	  }
+	}
+	if (attributeValues.size() == 0) {
+	  errms(tokenizer,"no nominal values found");
+	}
+	m_Attributes.
+	  addElement(new Attribute(attributeName, attributeValues,
+				   numAttributes()));
+      }
+      getLastToken(tokenizer,false);
+      getFirstToken(tokenizer);
+      if (tokenizer.ttype == StreamTokenizer.TT_EOF)
+	errms(tokenizer,"premature end of file");
+    }
+
+    // Check if data part follows. We can't easily check for EOL.
+    if (!tokenizer.sval.equalsIgnoreCase("@data")) {
+      errms(tokenizer,"keyword @data expected");
+    }
+    
+    // Check if any attributes have been declared.
+    if (m_Attributes.size() == 0) {
+      errms(tokenizer,"no attributes declared");
+    }
+
+    // Allocate buffers in case sparse instances have to be read
+    m_ValueBuffer = new double[numAttributes()];
+    m_IndicesBuffer = new int[numAttributes()];
+  }
+
+  /**
+   * Copies instances from one set to the end of another 
+   * one.
+   *
+   * @param source the source of the instances
+   * @param from the position of the first instance to be copied
+   * @param dest the destination for the instances
+   * @param num the number of instances to be copied
+   */
+  private void copyInstances(int from, Instances dest, int num) {
+    
+    for (int i = 0; i < num; i++) {
+      dest.add(instance(from + i));
+    }
+  }
+  
+  /**
+   * Throws error message with line number and last token read.
+   *
+   * @param theMsg the error message to be thrown
+   * @param tokenizer the stream tokenizer
+   * @throws IOExcpetion containing the error message
+   */
+  private void errms(StreamTokenizer tokenizer, String theMsg) 
+       throws IOException {
+    
+    throw new IOException(theMsg + ", read " + tokenizer.toString());
+  }
+  
+  /**
+   * Replaces the attribute information by a clone of
+   * itself.
+   */
+  private void freshAttributeInfo() {
+
+    m_Attributes = (FastVector) m_Attributes.copyElements();
+  }
+
+  /**
+   * Gets next token, skipping empty lines.
+   *
+   * @param tokenizer the stream tokenizer
+   * @exception IOException if reading the next token fails
+   */
+  private void getFirstToken(StreamTokenizer tokenizer) 
+    throws IOException{
+    
+    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL){};
+    if ((tokenizer.ttype == '\'') ||
+	(tokenizer.ttype == '"')) {
+      tokenizer.ttype = StreamTokenizer.TT_WORD;
+    } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) &&
+	       (tokenizer.sval.equals("?"))){
+      tokenizer.ttype = '?';
+    }
+  }
+
+  /**
+   * Gets index, checking for a premature and of line.
+   *
+   * @param tokenizer the stream tokenizer
+   * @exception IOException if it finds a premature end of line
+   */
+  private void getIndex(StreamTokenizer tokenizer) throws IOException{
+    
+    if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
+      errms(tokenizer,"premature end of line");
+    }
+    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
+      errms(tokenizer,"premature end of file");
+    }
+  }
+  
+  /**
+   * Gets token and checks if its end of line.
+   *
+   * @param tokenizer the stream tokenizer
+   * @exception IOException if it doesn't find an end of line
+   */
+  private void getLastToken(StreamTokenizer tokenizer, boolean endOfFileOk) 
+       throws IOException{
+
+    if ((tokenizer.nextToken() != StreamTokenizer.TT_EOL) &&
+	((tokenizer.nextToken() != StreamTokenizer.TT_EOF) || !endOfFileOk)) {
+      errms(tokenizer,"end of line expected");
+    }
+  }
+
+  /**
+   * Gets next token, checking for a premature and of line.
+   *
+   * @param tokenizer the stream tokenizer
+   * @exception IOException if it finds a premature end of line
+   */
+  private void getNextToken(StreamTokenizer tokenizer) 
+       throws IOException{
+    
+    if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
+      errms(tokenizer,"premature end of line");
+    }
+    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
+      errms(tokenizer,"premature end of file");
+    } else if ((tokenizer.ttype == '\'') ||
+	       (tokenizer.ttype == '"')) {
+      tokenizer.ttype = StreamTokenizer.TT_WORD;
+    } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) &&
+	       (tokenizer.sval.equals("?"))){
+      tokenizer.ttype = '?';
+    }
+  }
+	
+  /**
+   * Initializes the StreamTokenizer used for reading the ARFF file.
+   *
+   * @param tokenizer the stream tokenizer
+   */
+  private void initTokenizer(StreamTokenizer tokenizer){
+
+    tokenizer.resetSyntax();         
+    tokenizer.whitespaceChars(0, ' ');    
+    tokenizer.wordChars(' '+1,'\u00FF');
+    tokenizer.whitespaceChars(',',',');
+    tokenizer.commentChar('%');
+    tokenizer.quoteChar('"');
+    tokenizer.quoteChar('\'');
+    tokenizer.ordinaryChar('{');
+    tokenizer.ordinaryChar('}');
+    tokenizer.eolIsSignificant(true);
+  }
+ 
+  /**
+   * Returns string including all instances, their weights and
+   * their indices in the original dataset.
+   *
+   * @return description of instance and its weight as a string
+   */
+  private String instancesAndWeights(){
+
+    StringBuffer text = new StringBuffer();
+
+    for (int i = 0; i < numInstances(); i++) {
+      text.append(instance(i) + " " + instance(i).weight());
+      if (i < numInstances() - 1) {
+	text.append("\n");
+      }
+    }
+    return text.toString();
+  }
+  
+  /**
+   * Implements quicksort.
+   *
+   * @param attIndex the attribute's index
+   * @param lo0 the first index of the subset to be sorted
+   * @param hi0 the last index of the subset to be sorted
+   */
+  private void quickSort(int attIndex, int lo0, int hi0) {
+    
+    int lo = lo0, hi = hi0;
+    double mid, midPlus, midMinus;
+    
+    if (hi0 > lo0) {
+      
+      // Arbitrarily establishing partition element as the 
+      // midpoint of the array.
+      mid = instance((lo0 + hi0) / 2).value(attIndex);
+      midPlus = mid + 1e-6;
+      midMinus = mid - 1e-6;
+
+      // loop through the array until indices cross
+      while(lo <= hi) {
+	
+	// find the first element that is greater than or equal to 
+	// the partition element starting from the left Index.
+	while ((instance(lo).value(attIndex) < 
+		midMinus) && (lo < hi0)) {
+	  ++lo;
+	}
+	
+	// find an element that is smaller than or equal to
+	// the partition element starting from the right Index.
+	while ((instance(hi).value(attIndex)  > 
+		midPlus) && (hi > lo0)) {
+	  --hi;
+	}
+	
+	// if the indexes have not crossed, swap
+	if(lo <= hi) {
+	  swap(lo,hi);
+	  ++lo;
+	  --hi;
+	}
+      }
+      
+      // If the right index has not reached the left side of array
+      // must now sort the left partition.
+      if(lo0 < hi) {
+	quickSort(attIndex,lo0,hi);
+      }
+      
+      // If the left index has not reached the right side of array
+      // must now sort the right partition.
+      if(lo < hi0) {
+	quickSort(attIndex,lo,hi0);
+      }
+    }
+  }
+
+  /**
+   * Reads and skips all tokens before next end of line token.
+   *
+   * @param tokenizer the stream tokenizer
+   */
+  private void readTillEOL(StreamTokenizer tokenizer) 
+       throws IOException{
+    
+    while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {};
+    tokenizer.pushBack();
+  }
+
+  /**
+   * Help function needed for stratification of set.
+   *
+   * @param numFolds the number of folds for the stratification
+   */
+  private void stratStep (int numFolds){
+    
+    FastVector newVec = new FastVector(m_Instances.capacity());
+    int start = 0, j;
+
+    // create stratified batch
+    while (newVec.size() < numInstances()) {
+      j = start;
+      while (j < numInstances()) {
+	newVec.addElement(instance(j));
+	j = j + numFolds;
+      }
+      start++;
+    }
+    m_Instances = newVec;
+  }
+  
+  /**
+   * Swaps two instances in the set.
+   *
+   * @param i the first instance's index
+   * @param j the second instance's index
+   */
+  private void swap(int i, int j){
+    
+    m_Instances.swap(i, j);
+  }
+
+  /**
+   * Merges two sets of Instances together. The resulting set will have
+   * all the attributes of the first set plus all the attributes of the 
+   * second set. The number of instances in both sets must be the same.
+   *
+   * @param first the first set of Instances
+   * @param second the second set of Instances
+   * @return the merged set of Instances
+   * @exception IllegalArgumentException if the datasets are not the same size
+   */
+  public static Instances mergeInstances(Instances first, Instances second) {
+
+    if (first.numInstances() != second.numInstances()) {
+      throw new IllegalArgumentException("Instance sets must be of the same size");
+    }
+
+    // Create the vector of merged attributes
+    FastVector newAttributes = new FastVector();
+    for (int i = 0; i < first.numAttributes(); i++) {
+      newAttributes.addElement(first.attribute(i));
+    }
+    for (int i = 0; i < second.numAttributes(); i++) {
+      newAttributes.addElement(second.attribute(i));
+    }
+    
+    // Create the set of Instances
+    Instances merged = new Instances(first.relationName() + '_'
+				     + second.relationName(), 
+				     newAttributes, 
+				     first.numInstances());
+    // Merge each instance
+    for (int i = 0; i < first.numInstances(); i++) {
+      merged.add(first.instance(i).mergeInstance(second.instance(i)));
+    }
+    return merged;
+  }
+
+  /**
+   * Method for testing this class.
+   *
+   * @param argv should contain one element: the name of an ARFF file
+   */
+  public static void test(String [] argv) {
+
+    Instances instances, secondInstances, train, test, transformed, empty;
+    Instance instance;
+    Random random = new Random(2);
+    Reader reader;
+    int start, num;
+    double newWeight;
+    FastVector testAtts, testVals;
+    int i,j;
+    
+    try{
+      if (argv.length > 1) {
+	throw (new Exception("Usage: Instances [<filename>]"));
+      }
+      
+      // Creating set of instances from scratch
+      testVals = new FastVector(2);
+      testVals.addElement("first_value");
+      testVals.addElement("second_value");
+      testAtts = new FastVector(2);
+      testAtts.addElement(new Attribute("nominal_attribute", testVals));
+      testAtts.addElement(new Attribute("numeric_attribute"));
+      instances = new Instances("test_set", testAtts, 10);
+      instances.add(new Instance(instances.numAttributes()));
+      instances.add(new Instance(instances.numAttributes()));
+      instances.add(new Instance(instances.numAttributes()));
+      instances.setClassIndex(0);
+      System.out.println("\nSet of instances created from scratch:\n");
+      System.out.println(instances);
+      
+      if (argv.length == 1) {
+	String filename = argv[0];
+	reader = new FileReader(filename);
+	
+	// Read first five instances and print them
+	System.out.println("\nFirst five instances from file:\n");
+	instances = new Instances(reader, 1);
+	instances.setClassIndex(instances.numAttributes() - 1);
+	i = 0;
+	while ((i < 5) && (instances.readInstance(reader))) {
+	  i++;
+	}
+	System.out.println(instances);
+
+	// Read all the instances in the file
+	reader = new FileReader(filename);
+	instances = new Instances(reader);
+
+	// Make the last attribute be the class 
+	instances.setClassIndex(instances.numAttributes() - 1);
+	
+	// Print header and instances.
+	System.out.println("\nDataset:\n");
+	System.out.println(instances);
+	System.out.println("\nClass index: "+instances.classIndex());
+      }
+      
+      // Test basic methods based on class index.
+      System.out.println("\nClass name: "+instances.classAttribute().name());
+      System.out.println("\nClass index: "+instances.classIndex());
+      System.out.println("\nClass is nominal: " +
+			 instances.classAttribute().isNominal());
+      System.out.println("\nClass is numeric: " +
+			 instances.classAttribute().isNumeric());
+      System.out.println("\nClasses:\n");
+      for (i = 0; i < instances.numClasses(); i++) {
+	System.out.println(instances.classAttribute().value(i));
+      }
+      System.out.println("\nClass values and labels of instances:\n");
+      for (i = 0; i < instances.numInstances(); i++) {
+	Instance inst = instances.instance(i);
+	System.out.print(inst.classValue() + "\t");
+	System.out.print(inst.toString(inst.classIndex()));
+	if (instances.instance(i).classIsMissing()) {
+	  System.out.println("\tis missing");
+	} else {
+	  System.out.println();
+	}
+      }
+      
+      // Create random weights.
+      System.out.println("\nCreating random weights for instances.");
+      for (i = 0; i < instances.numInstances(); i++) {
+	instances.instance(i).setWeight(random.nextDouble()); 
+      }
+      
+      // Print all instances and their weights (and the sum of weights).
+      System.out.println("\nInstances and their weights:\n");
+      System.out.println(instances.instancesAndWeights());
+      System.out.print("\nSum of weights: ");
+      System.out.println(instances.sumOfWeights());
+      
+      // Insert an attribute
+      secondInstances = new Instances(instances);
+      Attribute testAtt = new Attribute("Inserted");
+      secondInstances.insertAttributeAt(testAtt, 0);
+      System.out.println("\nSet with inserted attribute:\n");
+      System.out.println(secondInstances);
+      System.out.println("\nClass name: "
+			 + secondInstances.classAttribute().name());
+      
+      // Delete the attribute
+      secondInstances.deleteAttributeAt(0);
+      System.out.println("\nSet with attribute deleted:\n");
+      System.out.println(secondInstances);
+      System.out.println("\nClass name: "
+			 + secondInstances.classAttribute().name());
+      
+      // Test if headers are equal
+      System.out.println("\nHeaders equal: "+
+			 instances.equalHeaders(secondInstances) + "\n");
+      
+      // Print data in internal format.
+      System.out.println("\nData (internal values):\n");
+      for (i = 0; i < instances.numInstances(); i++) {
+	for (j = 0; j < instances.numAttributes(); j++) {
+	  if (instances.instance(i).isMissing(j)) {
+	    System.out.print("? ");
+	  } else {
+	    System.out.print(instances.instance(i).value(j) + " ");
+	  }
+	}
+	System.out.println();
+      }
+      
+      // Just print header
+      System.out.println("\nEmpty dataset:\n");
+      empty = new Instances(instances, 0);
+      System.out.println(empty);
+      System.out.println("\nClass name: "+empty.classAttribute().name());
+
+      // Create copy and rename an attribute and a value (if possible)
+      if (empty.classAttribute().isNominal()) {
+	Instances copy = new Instances(empty, 0);
+	copy.renameAttribute(copy.classAttribute(), "new_name");
+	copy.renameAttributeValue(copy.classAttribute(), 
+				  copy.classAttribute().value(0), 
+				  "new_val_name");
+	System.out.println("\nDataset with names changed:\n" + copy);
+	System.out.println("\nOriginal dataset:\n" + empty);
+      }
+
+      // Create and prints subset of instances.
+      start = instances.numInstances() / 4;
+      num = instances.numInstances() / 2;
+      System.out.print("\nSubset of dataset: ");
+      System.out.println(num + " instances from " + (start + 1) 
+			 + ". instance");
+      secondInstances = new Instances(instances, start, num);
+      System.out.println("\nClass name: "
+			 + secondInstances.classAttribute().name());
+
+      // Print all instances and their weights (and the sum of weights).
+      System.out.println("\nInstances and their weights:\n");
+      System.out.println(secondInstances.instancesAndWeights());
+      System.out.print("\nSum of weights: ");
+      System.out.println(secondInstances.sumOfWeights());
+      
+      // Create and print training and test sets for 3-fold
+      // cross-validation.
+      System.out.println("\nTrain and test folds for 3-fold CV:");
+      if (instances.classAttribute().isNominal()) {
+	instances.stratify(3);
+      }
+      for (j = 0; j < 3; j++) {
+        train = instances.trainCV(3,j);
+	test = instances.testCV(3,j);
+                      
+	// Print all instances and their weights (and the sum of weights).
+	System.out.println("\nTrain: ");
+	System.out.println("\nInstances and their weights:\n");
+	System.out.println(train.instancesAndWeights());
+	System.out.print("\nSum of weights: ");
+	System.out.println(train.sumOfWeights());
+	System.out.println("\nClass name: "+train.classAttribute().name());
+	System.out.println("\nTest: ");
+	System.out.println("\nInstances and their weights:\n");
+	System.out.println(test.instancesAndWeights());
+	System.out.print("\nSum of weights: ");
+	System.out.println(test.sumOfWeights());
+	System.out.println("\nClass name: "+test.classAttribute().name());
+      }
+
+      // Randomize instances and print them.
+      System.out.println("\nRandomized dataset:");
+      instances.randomize(random);
+      
+      // Print all instances and their weights (and the sum of weights).
+      System.out.println("\nInstances and their weights:\n");
+      System.out.println(instances.instancesAndWeights());
+      System.out.print("\nSum of weights: ");
+      System.out.println(instances.sumOfWeights());
+
+      // Sort instances according to first attribute and
+      // print them.
+      System.out.print("\nInstances sorted according to first attribute:\n ");
+      instances.sort(0);
+        
+      // Print all instances and their weights (and the sum of weights).
+      System.out.println("\nInstances and their weights:\n");
+      System.out.println(instances.instancesAndWeights());
+      System.out.print("\nSum of weights: ");
+      System.out.println(instances.sumOfWeights());
+    } catch (Exception e) {
+      e.printStackTrace(); 
+    }
+  }
+
+  /**
+   * Main method for this class -- just prints a summary of a set
+   * of instances.
+   *
+   * @param argv should contain one element: the name of an ARFF file
+   */
+  public static void main(String [] args) {
+
+    try {
+      Reader r = null;
+      if (args.length > 1) {
+	throw (new Exception("Usage: Instances <filename>"));
+      } else if (args.length == 0) {
+        r = new BufferedReader(new InputStreamReader(System.in));
+      } else {
+        r = new BufferedReader(new FileReader(args[0]));
+      }
+      Instances i = new Instances(r);
+      System.out.println(i.toSummaryString());
+    } catch (Exception ex) {
+      System.err.println(ex.getMessage());
+    }
+  }
+}
+
+     
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Option.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Option.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Option.java	(revision 21112)
@@ -0,0 +1,105 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Option.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+/** 
+ * Class to store information about an option. <p>
+ *
+ * Typical usage: <p>
+ *
+ * <code>Option myOption = new Option("Uses extended mode.", "E", 0, "-E")); </code><p>
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Option {
+
+  /** What does this option do? */
+  private String m_Description;
+
+  /** The synopsis. */
+  private String m_Synopsis;
+
+  /** What's the option's name? */
+  private String m_Name;
+
+  /** How many arguments does it take? */
+  private int m_NumArguments;
+
+  /**
+   * Creates new option with the given parameters.
+   *
+   * @String description the option's description
+   * @String name the option's name
+   * @String numArguments the number of arguments
+   */
+  public Option(String description, String name, 
+		int numArguments, String synopsis) {
+  
+    m_Description = description;
+    m_Name = name;
+    m_NumArguments = numArguments;
+    m_Synopsis = synopsis;
+  }
+
+  /**
+   * Returns the option's description.
+   *
+   * @return the option's description
+   */
+  public String description() {
+  
+    return m_Description;
+  }
+
+  /**
+   * Returns the option's name.
+   *
+   * @return the option's name
+   */
+  public String name() {
+
+    return m_Name;
+  }
+
+  /**
+   * Returns the option's number of arguments.
+   *
+   * @return the option's number of arguments
+   */
+  public int numArguments() {
+  
+    return m_NumArguments;
+  }
+
+  /**
+   * Returns the option's synopsis.
+   *
+   * @return the option's synopsis
+   */
+  public String synopsis() {
+  
+    return m_Synopsis;
+  }
+}
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/OptionHandler.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/OptionHandler.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/OptionHandler.java	(revision 21112)
@@ -0,0 +1,67 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    OptionHandler.java
+ *    Copyright (C) 1999 Eibe Frank,Len Trigg
+ *
+ */
+
+package weka.core;
+
+import java.util.*;
+
+/** 
+ * Interface to something that understands options.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public interface OptionHandler {
+
+  /**
+   * Returns an enumeration of all the available options.
+   *
+   * @return an enumeration of all available options
+   */
+  public Enumeration listOptions();
+
+  /**
+   * Sets the OptionHandler's options using the given list. All options
+   * will be set (or reset) during this call (i.e. incremental setting
+   * of options is not possible).
+   *
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception;
+
+  /**
+   * Gets the current option settings for the OptionHandler.
+   *
+   * @return the list of current option settings as an array of strings
+   */
+  public String[] getOptions();
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Queue.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Queue.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Queue.java	(revision 21112)
@@ -0,0 +1,229 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Queue.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+package weka.core;
+
+import java.io.*;
+
+/** 
+ * Class representing a FIFO queue.
+ *
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Queue extends Object implements Serializable {
+
+  /**
+   * Represents one node in the queue.
+   */
+  protected class QueueNode implements Serializable {
+
+    /** The next node in the queue */
+    protected QueueNode m_Next;
+
+    /** The nodes contents */
+    protected Object m_Contents;
+
+    /** 
+     * Creates a queue node with the given contents 
+     */
+    public QueueNode(Object contents) {
+
+      m_Contents = contents;
+      next(null);
+    }
+
+    /**
+     * Sets the next node in the queue, and returns it.
+     */
+    public QueueNode next(QueueNode next) {
+
+      return m_Next = next;
+    }
+
+    /**
+     * Gets the next node in the queue. 
+     */
+    public QueueNode next() {
+
+      return m_Next;
+    }
+
+    /**
+     * Sets the contents of the node.
+     */
+    public Object contents(Object contents) {
+
+      return m_Contents = contents;
+    }
+
+    /**
+     * Returns the contents in the node.
+     */
+    public Object contents() {
+
+      return m_Contents;
+    }
+  }
+
+  /** Store a reference to the head of the queue */
+  protected QueueNode m_Head = null;
+
+  /** Store a reference to the tail of the queue */
+  protected QueueNode m_Tail = null;
+
+  /** Store the current number of elements in the queue */
+  protected int m_Size = 0;
+
+  /**
+   * Removes all objects from the queue.
+   */
+  public final synchronized void removeAllElements() {
+
+    m_Size = 0;
+    m_Head = null;
+    m_Tail = null;
+  }
+
+  /**
+   * Appends an object to the back of the queue.
+   *
+   * @param item the object to be appended
+   * @return the object appended
+   */
+  public synchronized Object push(Object item) {
+
+    QueueNode newNode = new QueueNode(item);
+    if (m_Head == null) {
+      m_Head = m_Tail = newNode;
+    } else {
+      m_Tail = m_Tail.next(newNode);
+    }
+    m_Size++;
+    return item;
+  }
+
+  /**
+   * Pops an object from the front of the queue.
+   *
+   * @return the object at the front of the queue
+   * @exception RuntimeException if the queue is empty
+   */
+  public synchronized Object pop() {
+
+    if (m_Head == null) {
+      throw new RuntimeException("Queue is empty");
+    }
+    Object retval = m_Head.contents();
+    m_Size--;
+    m_Head = m_Head.next();
+    if (m_Head == null) {
+      m_Tail = null;
+    }
+    return retval;
+  }
+
+  /**
+   * Gets object from the front of the queue.
+   *
+   * @return the object at the front of the queue
+   * @exception RuntimeException if the queue is empty
+   */
+  public synchronized Object peek() {
+    
+    if (m_Head == null) {
+      throw new RuntimeException("Queue is empty");
+    }
+    return m_Head.contents();
+  }
+
+  /**
+   * Checks if queue is empty.
+   * 
+   * @return true if queue is empty
+   */
+  public boolean empty() {
+
+    return (m_Head == null);
+  }
+
+  /** 
+   * Gets queue's size.
+   *
+   * @return size of queue
+   */
+  public int size() {
+
+    return m_Size;
+  }
+
+  /**
+   * Produces textual description of queue.
+   *
+   * @return textual description of queue
+   */
+  public String toString() {
+
+    String retval = "Queue Contents "+m_Size+" elements\n";
+    QueueNode current = m_Head;
+    if (current == null) {
+      return retval + "Empty\n";
+    } else {
+      while (current != null) {
+	retval += current.contents().toString()+"\n";
+	current = current.next();
+      }
+    }
+    return retval;
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv a set of strings that are pushed on a test queue
+   */
+  public static void main(String [] argv) {
+
+    try {
+      Queue queue = new Queue();
+      for(int i = 0; i < argv.length; i++) {
+	queue.push(argv[i]);
+      }
+      System.out.println("After Pushing");
+      System.out.println(queue.toString());
+      System.out.println("Popping...");
+      while (!queue.empty()) {
+	System.out.println(queue.pop().toString());
+      }
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Range.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Range.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Range.java	(revision 21112)
@@ -0,0 +1,439 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Range.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+package weka.core;
+
+import java.io.*;
+import java.util.*;
+
+/** 
+ * Class representing a range of cardinal numbers. The range is set by a 
+ * string representation such as: <P>
+ *
+ * <code>
+ *   all
+ *   first-last
+ *   1,2,3,4
+ * </code> <P>
+ * or combinations thereof. The range is internally converted from
+ * 1-based to 0-based (so methods that set or get numbers not in string
+ * format should use 0-based numbers).
+ *
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Range implements Serializable {
+
+  /** Record the string representations of the columns to delete */
+  Vector m_RangeStrings = new Vector();
+
+  /** Whether matching should be inverted */
+  boolean m_Invert;
+
+  /** The array of flags for whether an column is selected */
+  boolean [] m_SelectFlags;
+
+  /** Store the maximum value permitted in the range. -1 indicates that
+      no upper value has been set */
+  int m_Upper = -1;
+
+  /** Default constructor. */
+  public Range() {
+  }
+
+  /**
+   * Constructor to set initial range.
+   *
+   * @param rangeList the initial range
+   * @exception IllegalArgumentException if the range list is invalid
+   */
+  public Range(String rangeList) {
+
+    setRanges(rangeList);
+  }
+
+  /**
+   * Sets the value of "last".
+   *
+   * @param newUpper the value of "last"
+   */
+  public void setUpper(int newUpper) {
+
+    if (newUpper >= 0) {
+      m_Upper = newUpper;
+      setFlags();
+    }
+  }
+  
+  /**
+   * Gets whether the range sense is inverted, i.e. all <i>except</i>
+   * the values included by the range string are selected.
+   * 
+   * @return whether the matching sense is inverted
+   */
+  public boolean getInvert() {
+
+    return m_Invert;
+  }
+
+  /**
+   * Sets whether the range sense is inverted, i.e. all <i>except</i>
+   * the values included by the range string are selected.
+   * 
+   * @param newSetting true if the matching sense is inverted
+   */
+  public void setInvert(boolean newSetting) {
+
+    m_Invert = newSetting;
+  }
+
+  /**
+   * Gets the string representing the selected range of values
+   *
+   * @return the range selection string
+   */
+  public String getRanges() {
+
+    String result = null;
+    Enumeration enum = m_RangeStrings.elements();
+    while (enum.hasMoreElements()) {
+      if (result == null) {
+	result = (String)enum.nextElement();
+      } else {
+	result += ',' + (String)enum.nextElement();
+      }
+    }
+    return (result == null) ? "" : result;
+  }
+
+  /**
+   * Sets the ranges from a string representation.
+   *
+   * @param rangeList the comma separated list of ranges. The empty
+   * string sets the range to empty.
+   * @exception IllegalArgumentException if the rangeList was not well formed
+   */
+  public void setRanges(String rangeList) {
+
+    Vector ranges = new Vector (10);
+
+    // Split the rangeList up into the vector
+    while (!rangeList.equals("")) {
+      String range = rangeList.trim();
+      int commaLoc = rangeList.indexOf(',');
+      if (commaLoc != -1) {
+	range = rangeList.substring(0, commaLoc).trim();
+	rangeList = rangeList.substring(commaLoc + 1).trim();
+      } else {
+	rangeList = "";
+      }
+      if (!range.equals("")) {
+	if (isValidRange(range)) {
+	  ranges.addElement(range);
+	} else {
+	  throw new IllegalArgumentException("Invalid range list at " + range
+                                             + rangeList);
+	}
+      }
+    }
+    m_RangeStrings = ranges;
+    
+    if (m_Upper >= 0) {
+      setFlags();
+    }
+  }
+
+  /**
+   * Gets whether the supplied cardinal number is included in the current
+   * range.
+   *
+   * @param index the number of interest
+   * @return true if index is in the current range
+   * @exception RuntimeException if the upper limit of the range hasn't been defined
+   */
+  public boolean isInRange(int index) {
+
+    if (m_Upper == -1) {
+      throw new RuntimeException("No upper limit has been specified for range");
+    }
+    if (m_Invert) {
+      return !m_SelectFlags[index];
+    } else {
+      return m_SelectFlags[index];
+    }
+  }
+
+  /**
+   * Constructs a representation of the current range. Being a string
+   * representation, the numbers are based from 1.
+   * 
+   * @return the string representation of the current range
+   */
+  public String toString() {
+
+    if (m_RangeStrings.size() == 0) {
+      return "Empty";
+    }
+    String result ="Strings: ";
+    Enumeration enum = m_RangeStrings.elements();
+    while (enum.hasMoreElements()) {
+      result += (String)enum.nextElement() + " ";
+    }
+    result += "\n";
+
+    result += "Invert: " + m_Invert + "\n";
+
+    try {
+      if (m_Upper == -1) {
+	throw new RuntimeException("Upper limit has not been specified");
+      }
+      String cols = null;
+      for (int i = 0; i < m_SelectFlags.length; i++) {
+	if (isInRange(i)) {
+	  if (cols == null) {
+	    cols = "Cols: " + (i + 1);
+	  } else {
+	    cols += "," + (i + 1);
+	  }
+	}
+      }
+      if (cols != null) {
+	result += cols + "\n";
+      }
+    } catch (Exception ex) {
+      result += ex.getMessage();
+    }
+    return result;
+  }
+
+  /**
+   * Gets an array containing all the selected values, in the order
+   * that they were selected (or ascending order if range inversion is on)
+   *
+   * @return the array of selected values
+   * @exception RuntimeException if the upper limit of the range hasn't been defined
+   */
+  public int [] getSelection() {
+
+    if (m_Upper == -1) {
+      throw new RuntimeException("No upper limit has been specified for range");
+    }
+    int [] selectIndices = new int [m_Upper + 1];
+    int numSelected = 0;
+    if (m_Invert)
+    {
+      for (int i = 0; i <= m_Upper; i++) {
+	if (!m_SelectFlags[i]) {
+	  selectIndices[numSelected++] = i;
+	}
+      }
+    }
+    else
+    {
+      Enumeration enum = m_RangeStrings.elements();
+      while (enum.hasMoreElements()) {
+	String currentRange = (String)enum.nextElement();
+	int start = rangeLower(currentRange);
+	int end = rangeUpper(currentRange);
+	for (int i = start; (i <= m_Upper) && (i <= end); i++) {
+	  if (m_SelectFlags[i]) {
+	    selectIndices[numSelected++] = i;
+	  }
+	}
+      }
+    }
+    int [] result = new int [numSelected];
+    System.arraycopy(selectIndices, 0, result, 0, numSelected);
+    return result;
+  }
+
+  /**
+   * Creates a string representation of the indices in the supplied array.
+   *
+   * @param indices an array containing indices to select.
+   * Since the array will typically come from a program, indices are assumed
+   * from 0, and thus will have 1 added in the String representation.
+   */
+  public static String indicesToRangeList(int []indices) {
+
+    StringBuffer rl = new StringBuffer();
+    int last = -2;
+    boolean range = false;
+    for(int i = 0; i < indices.length; i++) {
+      if (i == 0) {
+	rl.append(indices[i] + 1);
+      } else if (indices[i] == last) {
+        range = true;
+      } else {
+        if (range) {
+          rl.append('-').append(last);
+          range = false;
+        }
+	rl.append(',').append(indices[i] + 1);
+      }
+      last = indices[i] + 1;
+    }
+    if (range) {
+      rl.append('-').append(last);
+    }
+    return rl.toString();
+  }
+
+  /** Sets the flags array. */
+  protected void setFlags() {
+
+    m_SelectFlags = new boolean [m_Upper + 1];
+    Enumeration enum = m_RangeStrings.elements();
+    while (enum.hasMoreElements()) {
+      String currentRange = (String)enum.nextElement();
+      int start = rangeLower(currentRange);
+      int end = rangeUpper(currentRange);
+      for (int i = start; (i <= m_Upper) && (i <= end); i++) {
+	m_SelectFlags[i] = true;
+      }
+    }
+  }
+
+
+  /**
+   * Translates a single string selection into it's internal 0-based equivalent
+   *
+   * @param single the string representing the selection (eg: 1 first last)
+   * @return the number corresponding to the selected value
+   */
+  protected int rangeSingle(String single) {
+
+    if (single.toLowerCase().equals("first")) {
+      return 0;
+    }
+    if (single.toLowerCase().equals("last")) {
+      return m_Upper;
+    }
+    int index = Integer.parseInt(single) - 1;
+    if (index < 0) {
+      index = 0;
+    }
+    if (index > m_Upper) {
+      index = m_Upper;
+    }
+    return index;
+  }
+
+  /**
+   * Translates a range into it's lower index.
+   *
+   * @param range the string representation of the range
+   * @return the lower index of the range
+   */
+  protected int rangeLower(String range) {
+
+    int hyphenIndex;
+    if ((hyphenIndex = range.indexOf('-')) >= 0) {
+      return Math.min(rangeLower(range.substring(0, hyphenIndex)),
+		       rangeLower(range.substring(hyphenIndex + 1)));
+    }
+    return rangeSingle(range);
+  }
+
+  /**
+   * Translates a range into it's upper index. Must only be called once
+   * setUpper has been called.
+   *
+   * @param range the string representation of the range
+   * @return the upper index of the range
+   */
+  protected int rangeUpper(String range) {
+
+    int hyphenIndex;
+    if ((hyphenIndex = range.indexOf('-')) >= 0) {
+      return Math.max(rangeUpper(range.substring(0, hyphenIndex)),
+		       rangeUpper(range.substring(hyphenIndex + 1)));
+    }
+    return rangeSingle(range);
+  }
+
+  /**
+   * Determines if a string represents a valid index or simple range.
+   * Examples: <code>first  last   2   first-last  first-4  4-last</code>
+   * Doesn't check that a < b for a-b
+   *
+   * @param range
+   * @return true if the range is valid
+   */
+  protected boolean isValidRange(String range) {
+
+    if (range == null) {
+      return false;
+    }
+    int hyphenIndex;
+    if ((hyphenIndex = range.indexOf('-')) >= 0) {
+      if (isValidRange(range.substring(0, hyphenIndex)) &&
+	  isValidRange(range.substring(hyphenIndex + 1))) {
+	return true;
+      }
+      return false;
+    }
+    if (range.toLowerCase().equals("first")) {
+      return true;
+    }
+    if (range.toLowerCase().equals("last")) {
+      return true;
+    }
+    try {
+      int index = Integer.parseInt(range);
+      if (index > 0) {
+	return true;
+      }
+      return false;
+    } catch (NumberFormatException ex) {
+      return false;
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv one parameter: a test range specification
+   */
+  public static void main(String [] argv) {
+
+    try {
+      if (argv.length == 0) {
+	throw new Exception("Usage: Range <rangespec>");
+      }
+      Range range = new Range();
+      range.setRanges(argv[0]);
+      range.setUpper(9);
+      range.setInvert(false);
+      System.out.println("Input: " + argv[0] + "\n"
+			 + range.toString());
+      int [] rangeIndices = range.getSelection();
+      for (int i = 0; i < rangeIndices.length; i++)
+	System.out.print(" " + (rangeIndices[i] + 1));
+      System.out.println("");
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SerializedObject.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SerializedObject.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SerializedObject.java	(revision 21112)
@@ -0,0 +1,249 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    SerializedObject.java
+ *    Copyright (C) 2000 Webmind Inc.
+ *
+ */
+
+package weka.core;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * This class stores an object serialized in memory. It allows compression,
+ * to be used to conserve memory (for example, when storing large strings
+ * in memory), or can be used as a mechanism for deep copying objects.
+ *
+ * @author Len Trigg (len@intelligenesis.net)
+ * @version $Revision$
+ */
+public class SerializedObject implements Serializable {
+
+  /** Stores the serialized object */
+  private byte [] m_Serialized;
+
+  /** True if the object has been compressed during storage */
+  private boolean m_Compressed;
+
+  /**
+   * Serializes the supplied object into a byte array without compression.
+   *
+   * @param obj the Object to serialize.
+   * @exception Exception if the object is not Serializable.
+   */
+  public SerializedObject(Object obj) throws Exception {
+
+    this(obj, false);
+  }
+
+  /**
+   * Serializes the supplied object into a byte array.
+   *
+   * @param obj the Object to serialize.
+   * @param compress true if the object should be stored compressed.
+   * @exception Exception if the object is not Serializable.
+   */
+  public SerializedObject(Object obj, boolean compress) throws Exception {
+
+    //System.err.print("."); System.err.flush();
+    m_Compressed = compress;
+    m_Serialized = toByteArray(obj, m_Compressed);
+  }
+
+  /**
+   * Serializes the supplied object to a byte array.
+   *
+   * @param obj the Object to serialize
+   * @param compress true if the object should be compressed.
+   * @return the byte array containing the serialized object.
+   * @exception Exception if the object is not Serializable.
+   */
+  protected static byte [] toByteArray(Object obj, boolean compress) 
+    throws Exception {
+
+    ByteArrayOutputStream bo = new ByteArrayOutputStream();
+    OutputStream os = bo;
+    if (compress) {
+      os = new GZIPOutputStream(os);
+    }
+    os = new BufferedOutputStream(os);
+    ObjectOutputStream oo = new ObjectOutputStream(os);
+    oo.writeObject(obj);
+    oo.close();
+    return bo.toByteArray();
+  }
+
+  /**
+   * Gets the object stored in this SerializedObject. The object returned
+   * will be a deep copy of the original stored object.
+   *
+   * @return the deserialized Object.
+   */
+  public Object getObject() {
+    try {
+      InputStream is = new ByteArrayInputStream(m_Serialized);
+      if (m_Compressed) {
+        is = new GZIPInputStream(is);
+      }
+      is = new BufferedInputStream(is);
+      ObjectInputStream oi = new ObjectInputStream(is);
+      Object result = oi.readObject();
+      oi.close();
+      return result;
+    } catch (Exception ex) {
+      ex.printStackTrace();
+    }
+    return null;
+  }
+
+  /**
+   * Compares this object with another for equality.
+   *
+   * @param other the other Object.
+   * @return true if the objects are equal.
+   */
+  public final boolean equals(Object other) {
+
+    // Check class type
+    if ((other == null) || !(other.getClass().equals(this.getClass()))) {
+      return false;
+    }
+    // Check serialized length
+    byte [] os = ((SerializedObject)other).m_Serialized;
+    if (os.length != m_Serialized.length) {
+      return false;
+    }
+    // Check serialized contents
+    for (int i = 0; i < m_Serialized.length; i++) {
+      if (m_Serialized[i] != os[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns a hashcode for this object.
+   *
+   * @return the hashcode for this object.
+   */
+  public final int hashCode() {
+    
+    return m_Serialized.length;
+  }
+
+  /**
+   * Returns a text representation of the state of this object.
+   *
+   * @return a String representing this object.
+   */
+  public String toString() {
+    
+    return (m_Compressed ? "Compressed object: " : "Uncompressed object: ")
+      + m_Serialized.length + " bytes";
+  }
+
+  /**
+   * Test routine, reads text from stdin and measures memory usage
+   */
+  public static void main2(String []args) {
+
+    try {
+      Runtime r = Runtime.getRuntime();
+      r.gc();
+      java.io.LineNumberReader lnr = new java.io.LineNumberReader(new java.io.InputStreamReader(System.in));
+      StringBuffer sb = new StringBuffer();
+      String line;
+      while ((line = lnr.readLine()) != null) {
+        sb.append(line).append('\n');
+      }
+      String text = sb.toString();
+      //System.err.println("TEXT:");
+      //System.err.println(text);
+      r.gc();
+
+      System.err.print("TOTAL: " + r.totalMemory());
+      System.err.println("\tFREE: " + r.freeMemory());
+      long used = r.totalMemory() - r.freeMemory();
+      
+      // convert to a compressedobject
+      SerializedObject co = new SerializedObject(text, true);
+
+      System.err.print("TOTAL: " + r.totalMemory());
+      System.err.println("\tFREE: " + r.freeMemory());
+      r.gc();
+      long used1 = r.totalMemory() - r.freeMemory();
+      long csize = used1 - used;
+      System.err.println("CompressedSize = " + csize);
+
+      String newstr = (String)co.getObject();
+      r.gc();
+      System.err.print("TOTAL: " + r.totalMemory());
+      System.err.println("\tFREE: " + r.freeMemory());
+      long used2 = r.totalMemory() - r.freeMemory();
+      long usize = used2 - used1;
+      System.err.println("UncompressedSize = " + usize);
+
+      // A couple of references to the original objects
+      // so they don't get freed prematurely and muck up the
+      // measurements.
+      newstr = newstr.toLowerCase();
+      text = text.toLowerCase();
+      System.err.println("Byte array: " + co.toString());
+      System.err.println("Length of text: " + text.length());
+    } catch (Exception ex) {
+      ex.printStackTrace();
+    }
+  }
+
+
+  /**
+   * Test routine, reads an arff file from stdin and measures memory usage
+   * (the arff file should have long string attribute values)
+   */
+  public static void main(String []args) {
+
+    try {
+      Runtime r = Runtime.getRuntime();
+      r.gc();
+      System.err.print("TOTAL: " + r.totalMemory());
+      System.err.println("\tFREE: " + r.freeMemory());
+      long used = r.totalMemory() - r.freeMemory();
+      Instances inst = new Instances(new java.io.BufferedReader(new java.io.InputStreamReader(System.in)));
+      r.gc();
+      long used1 = r.totalMemory() - r.freeMemory();
+      long csize = used1 - used;
+      System.err.print("\nTOTAL: " + r.totalMemory());
+      System.err.println("\tFREE: " + r.freeMemory());
+      System.err.println("size = " + csize);
+      int blah = inst.numAttributes();
+    } catch (Exception ex) {
+      ex.printStackTrace();
+    }
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SparseInstance.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SparseInstance.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SparseInstance.java	(revision 21112)
@@ -0,0 +1,773 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    SparseInstance.java
+ *    Copyright (C) 2000 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Class for storing an instance as a sparse vector. A sparse instance
+ * only requires storage for those attribute values that are non-zero.
+ * Since the objective is to reduce storage requirements for datasets
+ * with large numbers of default values, this also includes nominal
+ * attributes -- the first nominal value (i.e. that which has index 0)
+ * will not require explicit storage, so rearrange your nominal attribute
+ * value orderings if necessary. Missing values will be stored
+ * explicitly.
+ */
+public class SparseInstance extends Instance {
+
+  /** The index of the attribute associated with each stored value. */
+  protected int[] m_Indices;
+
+  /** The maximum number of values that can be stored. */
+  protected int m_NumAttributes;
+
+  protected SparseInstance() {
+
+  }
+
+  /**
+   * Constructor that generates a sparse instance from the given
+   * instance. Reference to the dataset is set to null.
+   * (ie. the instance doesn't have access to information about the
+   * attribute types)
+   *
+   * @param instance the instance from which the attribute values
+   * and the weight are to be copied
+   */
+  public SparseInstance(Instance instance) {
+    
+    m_Weight = instance.m_Weight;
+    m_Dataset = null;
+    m_NumAttributes = instance.numAttributes();
+    if (instance instanceof SparseInstance) {
+      m_AttValues = ((SparseInstance)instance).m_AttValues;
+      m_Indices = ((SparseInstance)instance).m_Indices;
+    } else {
+      double[] tempValues = new double[instance.numAttributes()];
+      int[] tempIndices = new int[instance.numAttributes()];
+      int vals = 0;
+      for (int i = 0; i < instance.numAttributes(); i++) {
+	if (instance.value(i) != 0) {
+	  tempValues[vals] = instance.value(i);
+	  tempIndices[vals] = i;
+	  vals++;
+	}
+      }
+      m_AttValues = new double[vals];
+      m_Indices = new int[vals];
+      System.arraycopy(tempValues, 0, m_AttValues, 0, vals);
+      System.arraycopy(tempIndices, 0, m_Indices, 0, vals);
+    }
+  }
+
+  /**
+   * Constructor that copies the info from the given instance. 
+   * Reference to the dataset is set to null.
+   * (ie. the instance doesn't have access to information about the
+   * attribute types)
+   *
+   * @param instance the instance from which the attribute
+   * info is to be copied 
+   */
+  public SparseInstance(SparseInstance instance) {
+    
+    m_AttValues = instance.m_AttValues;
+    m_Indices = instance.m_Indices;
+    m_Weight = instance.m_Weight;
+    m_NumAttributes = instance.m_NumAttributes;
+    m_Dataset = null;
+  }
+
+  /**
+   * Constructor that generates a sparse instance from the given
+   * parameters. Reference to the dataset is set to null.
+   * (ie. the instance doesn't have access to information about the
+   * attribute types)
+   *
+   * @param weight the instance's weight
+   * @param attValues a vector of attribute values 
+   */
+  public SparseInstance(double weight, double[] attValues) {
+    
+    m_Weight = weight;
+    m_Dataset = null;
+    m_NumAttributes = attValues.length;
+    double[] tempValues = new double[m_NumAttributes];
+    int[] tempIndices = new int[m_NumAttributes];
+    int vals = 0;
+    for (int i = 0; i < m_NumAttributes; i++) {
+      if (attValues[i] != 0) {
+	tempValues[vals] = attValues[i];
+	tempIndices[vals] = i;
+	vals++;
+      }
+    }
+    m_AttValues = new double[vals];
+    m_Indices = new int[vals];
+    System.arraycopy(tempValues, 0, m_AttValues, 0, vals);
+    System.arraycopy(tempIndices, 0, m_Indices, 0, vals);
+  }
+  
+  /**
+   * Constructor that inititalizes instance variable with given
+   * values. Reference to the dataset is set to null. (ie. the instance
+   * doesn't have access to information about the attribute types)
+   *
+   * @param weight the instance's weight
+   * @param attValues a vector of attribute values (just the ones to be stored)
+   * @param indices the indices of the given values in the full vector
+   * @param maxNumValues the maximium number of values that can be stored
+   */
+  public SparseInstance(double weight, double[] attValues,
+			int[] indices, int maxNumValues){
+    
+    int vals = 0; 
+    m_AttValues = new double [attValues.length];
+    m_Indices = new int [indices.length];
+    for (int i = 0; i < attValues.length; i++) {
+      if (attValues[i] != 0) {
+        m_AttValues[vals] = attValues[i];
+        m_Indices[vals] = indices[i];
+        vals++;
+      }
+    }
+    if (vals != attValues.length) {
+      // Need to truncate.
+      double [] newVals = new double[vals];
+      System.arraycopy(m_AttValues, 0, newVals, 0, vals);
+      m_AttValues = newVals;
+      int [] newIndices = new int[vals];
+      System.arraycopy(m_Indices, 0, newIndices, 0, vals);
+      m_Indices = newIndices;
+    }
+    m_Weight = weight;
+    m_NumAttributes = maxNumValues;
+    m_Dataset = null;
+  }
+
+  /**
+   * Constructor of an instance that sets weight to one, all values to
+   * be missing, and the reference to the dataset to null. (ie. the instance
+   * doesn't have access to information about the attribute types)
+   *
+   * @param numAttributes the size of the instance 
+   */
+  public SparseInstance(int numAttributes) {
+    
+    m_AttValues = new double[numAttributes];
+    m_NumAttributes = numAttributes;
+    m_Indices = new int[numAttributes];
+    for (int i = 0; i < m_AttValues.length; i++) {
+      m_AttValues[i] = MISSING_VALUE;
+      m_Indices[i] = i;
+    }
+    m_Weight = 1;
+    m_Dataset = null;
+  }
+
+  /**
+   * Returns the attribute associated with the internal index. 
+   *
+   * @param indexOfIndex the index of the attribute's index 
+   * @return the attribute at the given position
+   * @exception UnassignedDatasetException if instance doesn't have access to a
+   * dataset
+   */ 
+  public Attribute attributeSparse(int indexOfIndex) {
+   
+    if (m_Dataset == null) {
+      throw new UnassignedDatasetException("Instance doesn't have access to a dataset!");
+    }
+    return m_Dataset.attribute(m_Indices[indexOfIndex]);
+  }
+
+  /**
+   * Produces a shallow copy of this instance. The copy has
+   * access to the same dataset. (if you want to make a copy
+   * that doesn't have access to the dataset, use 
+   * <code>new SparseInstance(instance)</code>
+   *
+   * @return the shallow copy
+   */
+  public Object copy() {
+
+    Instance result = new SparseInstance(this);
+    result.m_Dataset = m_Dataset;
+    return result;
+  }
+
+  /**
+   * Returns the index of the attribute stored at the given position.
+   *
+   * @param position the position 
+   * @return the index of the attribute stored at the given position
+   */
+  public int index(int position) {
+
+    return m_Indices[position];
+  }
+
+  /**
+   * Tests if a specific value is "missing".
+   *
+   * @param attIndex the attribute's index
+   */
+  public boolean isMissing(int attIndex) {
+
+    if (Double.isNaN(value(attIndex))) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Locates the greatest index that is not greater than the
+   * given index.
+   *
+   * @return the internal index of the attribute index. Returns
+   * -1 if no index with this property couldn't be found
+   */
+  public int locateIndex(int index) {
+
+    int min = 0, max = m_Indices.length - 1;
+
+    // Binary search
+    while (max >= min) {
+      int current = (max + min) / 2;
+      if (m_Indices[current] > index) {
+	max = current - 1;
+      } else if (m_Indices[current] < index) {
+	min = current + 1;
+      } else {
+	return current;
+      }
+    }
+    return max;
+  }
+
+  /**
+   * Merges this instance with the given instance and returns
+   * the result. Dataset is set to null.
+   *
+   * @param inst the instance to be merged with this one
+   * @return the merged instances
+   */
+  public Instance mergeInstance(Instance inst) {
+
+    double[] values = new double[numValues() + inst.numValues()];
+    int[] indices = new int[numValues() + inst.numValues()];
+
+    int m = 0;
+    for (int j = 0; j < numValues(); j++, m++) {
+      values[m] = valueSparse(j);
+      indices[m] = index(j);
+    }
+    for (int j = 0; j < inst.numValues(); j++, m++) {
+      values[m] = inst.valueSparse(j);
+      indices[m] = inst.index(j) + inst.numAttributes();
+    }
+    
+    return new SparseInstance(1.0, values, indices, numAttributes() +
+			      inst.numAttributes());
+  }
+
+  /**
+   * Returns the number of attributes.
+   *
+   * @return the number of attributes as an integer
+   */
+  public int numAttributes() {
+
+    return m_NumAttributes;
+  }
+
+  /**
+   * Returns the number of values in the sparse vector.
+   *
+   * @return the number of values
+   */
+  public int numValues() {
+
+    return m_Indices.length;
+  }
+
+  /** 
+   * Replaces all missing values in the instance with the 
+   * values contained in the given array. A deep copy of
+   * the vector of attribute values is performed before the
+   * values are replaced.
+   *
+   * @param array containing the means and modes
+   * @exception IllegalArgumentException if numbers of attributes are unequal
+   */
+  public void replaceMissingValues(double[] array) {
+	 
+    if ((array == null) || (array.length != m_NumAttributes)) {
+      throw new IllegalArgumentException("Unequal number of attributes!");
+    }
+    double[] tempValues = new double[m_AttValues.length];
+    int[] tempIndices = new int[m_AttValues.length];
+    int vals = 0;
+    for (int i = 0; i < m_AttValues.length; i++) {
+      if (isMissingValue(m_AttValues[i])) {
+	if (array[m_Indices[i]] != 0) {
+	  tempValues[vals] = array[m_Indices[i]];
+	  tempIndices[vals] = m_Indices[i];
+	  vals++;
+	} 
+      } else {
+	tempValues[vals] = m_AttValues[i];
+	tempIndices[vals] = m_Indices[i];
+	vals++;
+      }
+    }
+    m_AttValues = new double[vals];
+    m_Indices = new int[vals];
+    System.arraycopy(tempValues, 0, m_AttValues, 0, vals);
+    System.arraycopy(tempIndices, 0, m_Indices, 0, vals);
+  }
+
+  /**
+   * Sets a specific value in the instance to the given value 
+   * (internal floating-point format). Performs a deep copy
+   * of the vector of attribute values before the value is set.
+   *
+   * @param attIndex the attribute's index 
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   */
+  public void setValue(int attIndex, double value) {
+
+    int index = locateIndex(attIndex);
+    
+    if ((index >= 0) && (m_Indices[index] == attIndex)) {
+      if (value != 0) {
+	double[] tempValues = new double[m_AttValues.length];
+	System.arraycopy(m_AttValues, 0, tempValues, 0, m_AttValues.length);
+	tempValues[index] = value;
+	m_AttValues = tempValues;
+      } else {
+	double[] tempValues = new double[m_AttValues.length - 1];
+	int[] tempIndices = new int[m_Indices.length - 1];
+	System.arraycopy(m_AttValues, 0, tempValues, 0, index);
+	System.arraycopy(m_Indices, 0, tempIndices, 0, index);
+	System.arraycopy(m_AttValues, index + 1, tempValues, index, 
+			 m_AttValues.length - index - 1);
+	System.arraycopy(m_Indices, index + 1, tempIndices, index, 
+			 m_Indices.length - index - 1);
+	m_AttValues = tempValues;
+	m_Indices = tempIndices;
+      }
+    } else {
+      if (value != 0) {
+	double[] tempValues = new double[m_AttValues.length + 1];
+	int[] tempIndices = new int[m_Indices.length + 1];
+	System.arraycopy(m_AttValues, 0, tempValues, 0, index + 1);
+	System.arraycopy(m_Indices, 0, tempIndices, 0, index + 1);
+	tempIndices[index + 1] = attIndex;
+	tempValues[index + 1] = value;
+	System.arraycopy(m_AttValues, index + 1, tempValues, index + 2, 
+			 m_AttValues.length - index - 1);
+	System.arraycopy(m_Indices, index + 1, tempIndices, index + 2, 
+			 m_Indices.length - index - 1);
+	m_AttValues = tempValues;
+	m_Indices = tempIndices;
+      }
+    }
+  }
+
+  /**
+   * Sets a specific value in the instance to the given value 
+   * (internal floating-point format). Performs a deep copy
+   * of the vector of attribute values before the value is set.
+   *
+   * @param indexOfIndex the index of the attribute's index 
+   * @param value the new attribute value (If the corresponding
+   * attribute is nominal (or a string) then this is the new value's
+   * index as a double).  
+   */
+  public void setValueSparse(int indexOfIndex, double value) {
+
+    if (value != 0) {
+      double[] tempValues = new double[m_AttValues.length];
+      System.arraycopy(m_AttValues, 0, tempValues, 0, m_AttValues.length);
+      m_AttValues = tempValues;
+      m_AttValues[indexOfIndex] = value;
+    } else {
+      double[] tempValues = new double[m_AttValues.length - 1];
+      int[] tempIndices = new int[m_Indices.length - 1];
+      System.arraycopy(m_AttValues, 0, tempValues, 0, indexOfIndex);
+      System.arraycopy(m_Indices, 0, tempIndices, 0, indexOfIndex);
+      System.arraycopy(m_AttValues, indexOfIndex + 1, tempValues, indexOfIndex, 
+		       m_AttValues.length - indexOfIndex - 1);
+      System.arraycopy(m_Indices, indexOfIndex + 1, tempIndices, indexOfIndex, 
+		       m_Indices.length - indexOfIndex - 1);
+      m_AttValues = tempValues;
+      m_Indices = tempIndices;
+    }
+  }
+
+  /**
+   * Returns the values of each attribute as an array of doubles.
+   *
+   * @return an array containing all the instance attribute values
+   */
+  public double[] toDoubleArray() {
+
+    double[] newValues = new double[m_NumAttributes];
+    for (int i = 0; i < m_AttValues.length; i++) {
+      newValues[m_Indices[i]] = m_AttValues[i];
+    }
+    return newValues;
+  }
+
+  /**
+   * Returns the description of one instance in sparse format. 
+   * If the instance doesn't have access to a dataset, it returns the 
+   * internal floating-point values. Quotes string values that contain 
+   * whitespace characters.
+   *
+   * @return the instance's description as a string
+   */
+  public String toString() {
+
+    StringBuffer text = new StringBuffer();
+    
+    text.append('{');
+    for (int i = 0; i < m_Indices.length; i++) {
+      if (i > 0) text.append(",");
+      if (isMissingValue(m_AttValues[i])) {
+	text.append(m_Indices[i] + " ?");
+      } else {
+	if (m_Dataset == null) {
+	  text.append(m_Indices[i] + " " + 
+		      Utils.doubleToString(m_AttValues[i],6));
+	} else {
+	  if (m_Dataset.attribute(m_Indices[i]).isNominal() || 
+	      m_Dataset.attribute(m_Indices[i]).isString()) {
+	    try {
+	      text.append(m_Indices[i] + " " +
+			  Utils.quote(m_Dataset.attribute(m_Indices[i]).
+				      value((int)valueSparse(i))));
+	    } catch (Exception e) {
+              e.printStackTrace();
+              System.err.println(new Instances(m_Dataset, 0));
+              System.err.println("Att:" + m_Indices[i] + " Val:" + valueSparse(i));
+	      throw new Error("This should never happen!");
+	    }
+	  } else {
+	    text.append(m_Indices[i] + " " +
+			Utils.doubleToString(m_AttValues[i],6));
+	  }
+	}
+      }
+    }
+    text.append('}');
+    return text.toString();
+  }
+
+  /**
+   * Returns an instance's attribute value in internal format.
+   *
+   * @param attIndex the attribute's index
+   * @return the specified value as a double (If the corresponding
+   * attribute is nominal (or a string) then it returns the value's index as a 
+   * double).
+   */
+  public double value(int attIndex) {
+
+    int index = locateIndex(attIndex);
+    if ((index >= 0) && (m_Indices[index] == attIndex)) {
+      return m_AttValues[index];
+    } else {
+      return 0.0;
+    }
+  }  
+
+  /**
+   * Deletes an attribute at the given position (0 to 
+   * numAttributes() - 1).
+   *
+   * @param pos the attribute's position
+   */
+  void forceDeleteAttributeAt(int position) {
+
+    int index = locateIndex(position);
+
+    m_NumAttributes--;
+    if ((index >= 0) && (m_Indices[index] == position)) {
+      int[] tempIndices = new int[m_Indices.length - 1];
+      double[] tempValues = new double[m_AttValues.length - 1];
+      System.arraycopy(m_Indices, 0, tempIndices, 0, index);
+      System.arraycopy(m_AttValues, 0, tempValues, 0, index);
+      for (int i = index; i < m_Indices.length - 1; i++) {
+	tempIndices[i] = m_Indices[i + 1] - 1;
+	tempValues[i] = m_AttValues[i + 1];
+      }
+      m_Indices = tempIndices;
+      m_AttValues = tempValues;
+    } else {
+      int[] tempIndices = new int[m_Indices.length];
+      double[] tempValues = new double[m_AttValues.length];
+      System.arraycopy(m_Indices, 0, tempIndices, 0, index + 1);
+      System.arraycopy(m_AttValues, 0, tempValues, 0, index + 1);
+      for (int i = index + 1; i < m_Indices.length - 1; i++) {
+	tempIndices[i] = m_Indices[i] - 1;
+	tempValues[i] = m_AttValues[i];
+      }
+      m_Indices = tempIndices;
+      m_AttValues = tempValues;
+    }
+  }
+
+  /**
+   * Inserts an attribute at the given position
+   * (0 to numAttributes()) and sets its value to be missing. 
+   *
+   * @param pos the attribute's position
+   */
+  void forceInsertAttributeAt(int position)  {
+
+    int index = locateIndex(position);
+
+    m_NumAttributes++;
+    if ((index >= 0) && (m_Indices[index] == position)) {
+      int[] tempIndices = new int[m_Indices.length + 1];
+      double[] tempValues = new double[m_AttValues.length + 1];
+      System.arraycopy(m_Indices, 0, tempIndices, 0, index);
+      System.arraycopy(m_AttValues, 0, tempValues, 0, index);
+      tempIndices[index] = position;
+      tempValues[index] = MISSING_VALUE;
+      for (int i = index; i < m_Indices.length; i++) {
+	tempIndices[i + 1] = m_Indices[i] + 1;
+	tempValues[i + 1] = m_AttValues[i];
+      }
+      m_Indices = tempIndices;
+      m_AttValues = tempValues;
+    } else {
+      int[] tempIndices = new int[m_Indices.length + 1];
+      double[] tempValues = new double[m_AttValues.length + 1];
+      System.arraycopy(m_Indices, 0, tempIndices, 0, index + 1);
+      System.arraycopy(m_AttValues, 0, tempValues, 0, index + 1);
+      tempIndices[index + 1] = position;
+      tempValues[index + 1] = MISSING_VALUE;
+      for (int i = index + 1; i < m_Indices.length; i++) {
+	tempIndices[i + 1] = m_Indices[i] + 1;
+	tempValues[i + 1] = m_AttValues[i];
+      }
+      m_Indices = tempIndices;
+      m_AttValues = tempValues;
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   */
+  public static void main(String[] options) {
+
+    try {
+
+      // Create numeric attributes "length" and "weight"
+      Attribute length = new Attribute("length");
+      Attribute weight = new Attribute("weight");
+      
+      // Create vector to hold nominal values "first", "second", "third" 
+      FastVector my_nominal_values = new FastVector(3); 
+      my_nominal_values.addElement("first"); 
+      my_nominal_values.addElement("second"); 
+      my_nominal_values.addElement("third"); 
+      
+      // Create nominal attribute "position" 
+      Attribute position = new Attribute("position", my_nominal_values);
+      
+      // Create vector of the above attributes 
+      FastVector attributes = new FastVector(3);
+      attributes.addElement(length);
+      attributes.addElement(weight);
+      attributes.addElement(position);
+      
+      // Create the empty dataset "race" with above attributes
+      Instances race = new Instances("race", attributes, 0);
+      
+      // Make position the class attribute
+      race.setClassIndex(position.index());
+      
+      // Create empty instance with three attribute values
+      SparseInstance inst = new SparseInstance(3);
+      
+      // Set instance's values for the attributes "length", "weight", and "position"
+      inst.setValue(length, 5.3);
+      inst.setValue(weight, 300);
+      inst.setValue(position, "first");
+      
+      // Set instance's dataset to be the dataset "race"
+      inst.setDataset(race);
+      
+      // Print the instance
+      System.out.println("The instance: " + inst);
+      
+      // Print the first attribute
+      System.out.println("First attribute: " + inst.attribute(0));
+      
+      // Print the class attribute
+      System.out.println("Class attribute: " + inst.classAttribute());
+      
+      // Print the class index
+      System.out.println("Class index: " + inst.classIndex());
+      
+      // Say if class is missing
+      System.out.println("Class is missing: " + inst.classIsMissing());
+      
+      // Print the instance's class value in internal format
+      System.out.println("Class value (internal format): " + inst.classValue());
+      
+      // Print a shallow copy of this instance
+      SparseInstance copy = (SparseInstance) inst.copy();
+      System.out.println("Shallow copy: " + copy);
+      
+      // Set dataset for shallow copy
+      copy.setDataset(inst.dataset());
+      System.out.println("Shallow copy with dataset set: " + copy);
+
+      // Print out all values in internal format
+      System.out.print("All stored values in internal format: ");
+      for (int i = 0; i < inst.numValues(); i++) {
+	if (i > 0) {
+	  System.out.print(",");
+	}
+	System.out.print(inst.valueSparse(i));
+      }
+      System.out.println();
+
+      // Set all values to zero
+      System.out.print("All values set to zero: ");
+      while (inst.numValues() > 0) {
+	inst.setValueSparse(0, 0);
+      }
+      for (int i = 0; i < inst.numValues(); i++) {
+	if (i > 0) {
+	  System.out.print(",");
+	}
+	System.out.print(inst.valueSparse(i));
+      }
+      System.out.println();
+
+      // Set all values to one
+      System.out.print("All values set to one: ");
+      for (int i = 0; i < inst.numAttributes(); i++) {
+	inst.setValue(i, 1);
+      }
+      for (int i = 0; i < inst.numValues(); i++) {
+	if (i > 0) {
+	  System.out.print(",");
+	}
+	System.out.print(inst.valueSparse(i));
+      }
+      System.out.println();
+
+      // Unset dataset for copy, delete first attribute, and insert it again
+      copy.setDataset(null);
+      copy.deleteAttributeAt(0);
+      copy.insertAttributeAt(0);
+      copy.setDataset(inst.dataset());
+      System.out.println("Copy with first attribute deleted and inserted: " + copy); 
+
+      // Same for second attribute
+      copy.setDataset(null);
+      copy.deleteAttributeAt(1);
+      copy.insertAttributeAt(1);
+      copy.setDataset(inst.dataset());
+      System.out.println("Copy with second attribute deleted and inserted: " + copy); 
+
+      // Same for last attribute
+      copy.setDataset(null);
+      copy.deleteAttributeAt(2);
+      copy.insertAttributeAt(2);
+      copy.setDataset(inst.dataset());
+      System.out.println("Copy with third attribute deleted and inserted: " + copy); 
+      
+      // Enumerate attributes (leaving out the class attribute)
+      System.out.println("Enumerating attributes (leaving out class):");
+      Enumeration enum = inst.enumerateAttributes();
+      while (enum.hasMoreElements()) {
+	Attribute att = (Attribute) enum.nextElement();
+	System.out.println(att);
+      }
+      
+      // Headers are equivalent?
+      System.out.println("Header of original and copy equivalent: " +
+			 inst.equalHeaders(copy));
+
+      // Test for missing values
+      System.out.println("Length of copy missing: " + copy.isMissing(length));
+      System.out.println("Weight of copy missing: " + copy.isMissing(weight.index()));
+      System.out.println("Length of copy missing: " + 
+			 Instance.isMissingValue(copy.value(length)));
+      System.out.println("Missing value coded as: " + Instance.missingValue());
+
+      // Prints number of attributes and classes
+      System.out.println("Number of attributes: " + copy.numAttributes());
+      System.out.println("Number of classes: " + copy.numClasses());
+
+      // Replace missing values
+      double[] meansAndModes = {2, 3, 0};
+      copy.replaceMissingValues(meansAndModes);
+      System.out.println("Copy with missing value replaced: " + copy);
+
+      // Setting and getting values and weights
+      copy.setClassMissing();
+      System.out.println("Copy with missing class: " + copy);
+      copy.setClassValue(0);
+      System.out.println("Copy with class value set to first value: " + copy);
+      copy.setClassValue("third");
+      System.out.println("Copy with class value set to \"third\": " + copy);
+      copy.setMissing(1);
+      System.out.println("Copy with second attribute set to be missing: " + copy);
+      copy.setMissing(length);
+      System.out.println("Copy with length set to be missing: " + copy);
+      copy.setValue(0, 0);
+      System.out.println("Copy with first attribute set to 0: " + copy);
+      copy.setValue(weight, 1);
+      System.out.println("Copy with weight attribute set to 1: " + copy);
+      copy.setValue(position, "second");
+      System.out.println("Copy with position set to \"second\": " + copy);
+      copy.setValue(2, "first");
+      System.out.println("Copy with last attribute set to \"first\": " + copy);
+      System.out.println("Current weight of instance copy: " + copy.weight());
+      copy.setWeight(2);
+      System.out.println("Current weight of instance copy (set to 2): " + copy.weight());
+      System.out.println("Last value of copy: " + copy.toString(2));
+      System.out.println("Value of position for copy: " + copy.toString(position));
+      System.out.println("Last value of copy (internal format): " + copy.value(2));
+      System.out.println("Value of position for copy (internal format): " + 
+			 copy.value(position));
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SpecialFunctions.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SpecialFunctions.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/SpecialFunctions.java	(revision 21112)
@@ -0,0 +1,127 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    SpecialFunctions.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+import java.lang.Math;
+
+/**
+ * Class implementing some mathematical functions.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public final class SpecialFunctions {
+
+  /** Some constants */
+  private static double log2 = Math.log(2);
+  private static double[] cofLogGamma = {76.18009172947146,-86.50532032941677,
+					 24.01409824083091,-1.231739572450155,
+					 0.1208650973866179e-2,-0.5395239384953e-5};
+
+  /**
+   * Returns natural logarithm of factorial using gamma function.
+   *
+   * @param x the value
+   * @return natural logarithm of factorial
+   */
+  public static double lnFactorial(double x){
+
+    return lnGamma(x+1);
+  }
+
+  /**
+   * Returns natural logarithm of gamma function. Converted to java
+   * from Numerical Recipes in C.
+   *
+   * @param x the value
+   * @return natural logarithm of gamma function
+   */
+  public static double lnGamma(double x){
+    
+    double y,tmp,ser;
+    int j;
+    
+    y=x;
+    tmp=x+5.5;
+    tmp -= (x+0.5)*Math.log(tmp);
+    ser=1.000000000190015;
+    for (j=0;j<=5;j++) {
+      ser += cofLogGamma[j]/++y;
+    }
+    return -tmp+Math.log(2.5066282746310005*ser/x);
+  }
+
+  /**
+   * Returns base 2 logarithm of binomial coefficient using gamma function.
+   *
+   * @param a upper part of binomial coefficient
+   * @param b lower part
+   * @return the base 2 logarithm of the binominal coefficient a over b
+   */
+  public static double log2Binomial(double a, double b) throws ArithmeticException{
+    
+    if (Utils.gr(b,a)) {
+      throw new ArithmeticException("Can't compute binomial coefficient.");
+    }
+    return (lnFactorial(a)-lnFactorial(b)-lnFactorial(a-b))/log2;
+  }
+
+  /**
+   * Returns base 2 logarithm of multinomial using gamma function.
+   *
+   * @param a upper part of multinomial coefficient
+   * @param bs lower part
+   * @return multinomial coefficient of a over the bs
+   */
+  public static double log2Multinomial(double a, double[] bs)
+       throws ArithmeticException{
+    
+    double sum = 0;
+    int i;
+    
+    for (i=0;i<bs.length;i++) {
+      if (Utils.gr(bs[i],a)) {
+	throw 
+	  new ArithmeticException("Can't compute multinomial coefficient.");
+      } else {
+	sum = sum+lnFactorial(bs[i]);
+      }
+    }
+    return (lnFactorial(a)-sum)/log2;
+  }
+
+  /**
+   * Main method for testing this class.
+   */
+  public static void main(String[] ops) {
+
+    double[] doubles = {1, 2, 3};
+
+    System.out.println("6!: " + Math.exp(SpecialFunctions.lnFactorial(6)));
+    System.out.println("lnGamma(6): "+ SpecialFunctions.lnGamma(6));
+    System.out.println("Binomial 6 over 2: " +
+		       Math.pow(2, SpecialFunctions.log2Binomial(6, 2)));
+    System.out.println("Multinomial 6 over 1, 2, 3: " +
+		       Math.pow(2, SpecialFunctions.log2Multinomial(6, doubles)));
+  }    
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Statistics.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Statistics.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Statistics.java	(revision 21112)
@@ -0,0 +1,280 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Statistics.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+/**
+ * Class implementing some distributions, tests, etc. Most of the
+ * code is adapted from Gary Perlman's unixstat.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Statistics {
+
+  /** Some constants */
+  private static double logSqrtPi = Math.log(Math.sqrt(Math.PI));
+  private static double rezSqrtPi = 1/Math.sqrt(Math.PI);
+  private static double bigx = 20.0;
+
+  /**
+   * Computes standard error for observed values of a binomial
+   * random variable.
+   *
+   * @param p the probability of success
+   * @param n the size of the sample
+   * @return the standard error
+   */
+  public static double binomialStandardError(double p, int n) {
+
+    if (n == 0) {
+      return 0; 
+    }
+    return Math.sqrt((p*(1-p))/(double) n);
+  }
+
+  /**
+   * Returns chi-squared probability for given value and degrees
+   * of freedom. (The probability that the chi-squared variate
+   * will be greater than x for the given degrees of freedom.)
+   * Adapted from unixstat by Gary Perlman.
+   *
+   * @param x the value
+   * @param df the number of degrees of freedom
+   */
+  public static double chiSquaredProbability(double x, int df) {
+
+    double a, y = 0, s, e, c, z, val;
+    boolean even;
+   
+    if (x <= 0 || df < 1)
+      return (1);
+    a = 0.5 * x;
+    even = (((int)(2*(df/2))) == df);
+    if (df > 1)
+      y = Math.exp(-a); //((-a < -bigx) ? 0.0 : Math.exp (-a));
+    s = (even ? y : (2.0 * normalProbability(-Math.sqrt (x))));
+    if (df > 2){
+      x = 0.5 * (df - 1.0);
+      z = (even ? 1.0 : 0.5);
+      if (a > bigx){
+	e = (even ? 0.0 : logSqrtPi);
+	c = Math.log (a);
+	while (z <= x){
+	  e = Math.log (z) + e;
+	  val = c*z-a-e;
+	  s += Math.exp (val); //((val < -bigx) ? 0.0 : Math.exp (val));
+	  z += 1.0;
+	}
+	return (s);
+      }else{
+	e = (even ? 1.0 : (rezSqrtPi / Math.sqrt (a)));
+	c = 0.0;
+	while (z <= x){
+	  e = e * (a / z);
+	  c = c + e;
+	  z += 1.0;
+	}
+	return (c * y + s);
+      }
+    }else{
+      return (s);
+    }
+  }
+
+  /**
+   * Critical value for given probability of F-distribution.
+   * Adapted from unixstat by Gary Perlman.
+   *
+   * @param p the probability
+   * @param df1 the first number of degrees of freedom
+   * @param df2 the second number of degrees of freedom
+   * @return the critical value for the given probability
+   */
+  public static double FCriticalValue(double p, int df1, int df2) {
+
+    double  fval;
+    double  maxf = 99999.0;     /* maximum possible F ratio */
+    double  minf = .000001;     /* minimum possible F ratio */
+    
+    if (p <= 0.0 || p >= 1.0)
+      return (0.0);
+    
+    fval = 1.0 / p; /* the smaller the p, the larger the F */
+    
+    while (Math.abs (maxf - minf) > .000001) {
+      if (FProbability(fval, df1, df2) < p) /* F too large */
+	maxf = fval;
+      else /* F too small */
+	minf = fval;
+      fval = (maxf + minf) * 0.5;
+    }
+    
+    return (fval);
+  }
+
+  /**
+   * Computes probability of F-ratio.
+   * Adapted from unixstat by Gary Perlman.
+   * Collected Algorithms of the CACM 
+   * Algorithm 322
+   * Egon Dorrer
+   *
+   * @param F the F-ratio
+   * @param df1 the first number of degrees of freedom
+   * @param df2 the second number of degrees of freedom
+   * @return the probability of the F-ratio.
+   */
+  public static double FProbability(double F, int df1, int df2) {
+
+    int     i, j;
+    int     a, b;
+    double  w, y, z, d, p;
+
+    if ((Math.abs(F) < 10e-10) || df1 <= 0 || df2 <= 0)
+      return (1.0);
+    a = (df1%2 == 1) ? 1 : 2;
+    b = (df2%2 == 1) ? 1 : 2;
+    w = (F * df1) / df2;
+    z = 1.0 / (1.0 + w);
+    if (a == 1)
+      if (b == 1) {
+	p = Math.sqrt (w);
+	y = 1/Math.PI; /* 1 / 3.14159 */
+	d = y * z / p;
+	p = 2.0 * y * Math.atan (p);
+      } else {
+	p = Math.sqrt (w * z);
+	d = 0.5 * p * z / w;
+      } else if (b == 1) {
+	p = Math.sqrt (z);
+	d = 0.5 * z * p;
+	p = 1.0 - p;
+      } else {
+	d = z * z;
+	p = w * z;
+      }
+    y = 2.0 * w / z;
+    for (j = b + 2; j <= df2; j += 2) {
+      d *= (1.0 + a / (j - 2.0)) * z;
+      p = (a == 1 ? p + d * y / (j - 1.0) : (p + w) * z);
+    }
+    y = w * z;
+    z = 2.0 / z;
+    b = df2 - 2;
+    for (i = a + 2; i <= df1; i += 2) {
+      j = i + b;
+      d *= y * j / (i - 2.0);
+      p -= z * d / j;
+    }
+    
+    // correction for approximation errors suggested in certification
+    if (p < 0.0)
+      p = 0.0;
+    else if (p > 1.0)
+      p = 1.0;
+    return (1.0-p);
+  }
+  
+  /**
+   * Returns probability that the standardized normal variate Z (mean = 0, standard
+   * deviation = 1) is less than z.
+   * Adapted from unixstat by Gary Perlman.
+   *
+   * @param the z-value
+   * @return the probability of the z value according to the normal pdf
+   */
+  public static double normalProbability(double z) {
+
+    double  y, x, w;
+    
+    if (z == 0.0)
+      x = 0.0;
+    else{
+      y = 0.5 * Math.abs (z);
+      if (y >= 3.0)
+	x = 1.0;
+      else if (y < 1.0){
+	w = y*y;
+	x = ((((((((0.000124818987 * w
+                   -0.001075204047) * w +0.005198775019) * w
+                   -0.019198292004) * w +0.059054035642) * w
+                   -0.151968751364) * w +0.319152932694) * w
+                   -0.531923007300) * w +0.797884560593) * y * 2.0;
+      }else{
+	y -= 2.0;
+	x = (((((((((((((-0.000045255659 * y
+                         +0.000152529290) * y -0.000019538132) * y
+                         -0.000676904986) * y +0.001390604284) * y
+                         -0.000794620820) * y -0.002034254874) * y
+                         +0.006549791214) * y -0.010557625006) * y
+                         +0.011630447319) * y -0.009279453341) * y
+                         +0.005353579108) * y -0.002141268741) * y
+                         +0.000535310849) * y +0.999936657524;
+      }
+    }
+  
+    return (z > 0.0 ? ((x + 1.0) / 2.0) : ((1.0 - x) / 2.0));
+  }
+
+  /**
+   * Computes absolute size of half of a student-t confidence interval 
+   * for given degrees of freedom, probability, and observed value.
+   *
+   * @param df the number of degrees of freedom
+   * @param p the probability
+   * @param se the observed value
+   * @return absolute size of half of a student-t confidence interval
+   */
+  public static double studentTConfidenceInterval(int df, double p,
+						  double se) {
+
+    return Math.sqrt(FCriticalValue(p, 1, df))*se;
+  }
+
+  /**
+   * Main method for testing this class.
+   */
+  public static void main(String[] ops) {
+
+    System.out.println("Binomial standard error (0.5, 100): " + 
+		       Statistics.binomialStandardError(0.5, 100));
+    System.out.println("Chi-squared probability (2.558, 10): " +
+		       Statistics.chiSquaredProbability(2.558, 10));
+    System.out.println("Normal probability (0.2): " +
+		       Statistics.normalProbability(0.2));
+    System.out.println("F critical value (0.05, 4, 5): " +
+		       Statistics.FCriticalValue(0.05, 4, 5));
+    System.out.println("F probability (5.1922, 4, 5): " +
+		       Statistics.FProbability(5.1922, 4, 5));
+    System.out.println("Student-t confidence interval (9, 0.01, 2): " +
+		       Statistics.studentTConfidenceInterval(9, 0.01, 2));
+  }
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedClassException.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedClassException.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedClassException.java	(revision 21112)
@@ -0,0 +1,52 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    UnassignedClassException.java
+ *    Copyright (C) 2001 Webmind Corp.
+ */
+
+package weka.core;
+
+/**
+ * <code>UnassignedClassException</code> is used when
+ * a method requires access to the Attribute designated as 
+ * the class attribute in a set of Instances, but the Instances does not
+ * have any class attribute assigned (such as by setClassIndex()).
+ *
+ * @author <a href="mailto:len@webmind.com">Len Trigg</a>
+ * @version $Revision$
+ */
+public class UnassignedClassException extends RuntimeException {
+
+  /**
+   * Creates a new <code>UnassignedClassException</code> instance
+   * with no detail message.
+   */
+  public UnassignedClassException() { 
+    super(); 
+  }
+
+  /**
+   * Creates a new <code>UnassignedClassException</code> instance
+   * with a specified message.
+   *
+   * @param messagae a <code>String</code> containing the message.
+   */
+  public UnassignedClassException(String message) { 
+    super(message); 
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedDatasetException.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedDatasetException.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnassignedDatasetException.java	(revision 21112)
@@ -0,0 +1,53 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    UnassignedDatasetException.java
+ *    Copyright (C) 2001 Webmind Corp.
+ */
+
+package weka.core;
+
+/**
+ * <code>UnassignedDatasetException</code> is used when
+ * a method of an Instance is called that requires access to
+ * the Instance structure, but that the Instance does not contain
+ * a reference to any Instances (as set by Instance.setDataset(), or when
+ * an Instance is added to a set of Instances)).
+ *
+ * @author <a href="mailto:len@webmind.com">Len Trigg</a>
+ * @version $Revision$
+ */
+public class UnassignedDatasetException extends RuntimeException {
+
+  /**
+   * Creates a new <code>UnassignedDatasetException</code> instance
+   * with no detail message.
+   */
+  public UnassignedDatasetException() { 
+    super(); 
+  }
+
+  /**
+   * Creates a new <code>UnassignedDatasetException</code> instance
+   * with a specified message.
+   *
+   * @param messagae a <code>String</code> containing the message.
+   */
+  public UnassignedDatasetException(String message) { 
+    super(message); 
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnsupportedClassTypeException.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnsupportedClassTypeException.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/UnsupportedClassTypeException.java	(revision 21112)
@@ -0,0 +1,51 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    UnsupportedClassTypeException.java
+ *    Copyright (C) 2001 Webmind Corp.
+ */
+
+package weka.core;
+
+/**
+ * <code>UnsupportedClassTypeException</code> is used in situations
+ * where the throwing object is not able to accept Instances with the
+ * supplied structure, because the class Attribute is of the wrong type.
+ *
+ * @author <a href="mailto:len@webmind.com">Len Trigg</a>
+ * @version $Revision$
+ */
+public class UnsupportedClassTypeException extends WekaException {
+
+  /**
+   * Creates a new <code>UnsupportedClassTypeException</code> instance
+   * with no detail message.
+   */
+  public UnsupportedClassTypeException() { 
+    super(); 
+  }
+
+  /**
+   * Creates a new <code>UnsupportedClassTypeException</code> instance
+   * with a specified message.
+   *
+   * @param messagae a <code>String</code> containing the message.
+   */
+  public UnsupportedClassTypeException(String message) { 
+    super(message); 
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Utils.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Utils.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/Utils.java	(revision 21112)
@@ -0,0 +1,1384 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Utils.java
+ *    Copyright (C) 1999 Eibe Frank,Len Trigg,Yong Wang
+ *
+ */
+
+package weka.core;
+
+import java.lang.Math;
+import java.util.StringTokenizer;
+import java.util.Properties;
+import java.io.File;
+import java.io.FileInputStream;
+
+/**
+ * Class implementing some simple utility methods.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @author Yong Wang (yongwang@cs.waikato.ac.nz)
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public final class Utils {
+
+  /** The natural logarithm of 2. */
+  public static double log2 = Math.log(2);
+
+  /** The small deviation allowed in double comparisons */
+  public static double SMALL = 1e-6;
+
+  
+  /**
+   * Reads properties that inherit from three locations. Properties
+   * are first defined in the system resource location (i.e. in the
+   * CLASSPATH).  These default properties must exist. Properties
+   * defined in the users home directory (optional) override default
+   * settings. Properties defined in the current directory (optional)
+   * override all these settings.
+   *
+   * @param resourceName the location of the resource that should be
+   * loaded.  e.g.: "weka/core/Utils.props". (The use of hardcoded
+   * forward slashes here is OK - see
+   * jdk1.1/docs/guide/misc/resources.html) This routine will also
+   * look for the file (in this case) "Utils.props" in the users home
+   * directory and the current directory.
+   * @return the Properties
+   * @exception Exception if no default properties are defined, or if
+   * an error occurs reading the properties files.  
+   */
+  public static Properties readProperties(String resourceName)
+    throws Exception{
+
+    Properties defaultProps = new Properties();
+    try {
+      // Apparently hardcoded slashes are OK here
+      // jdk1.1/docs/guide/misc/resources.html
+      defaultProps.load(ClassLoader.getSystemResourceAsStream(resourceName));
+    } catch (Exception ex) {
+/*      throw new Exception("Problem reading default properties: "
+	+ ex.getMessage()); */
+      System.err.println("Warning, unable to load properties file from "
+			 +"system resource (Utils.java)");
+    }
+
+    // Hardcoded slash is OK here
+    // eg: see jdk1.1/docs/guide/misc/resources.html
+    int slInd = resourceName.lastIndexOf('/');
+    if (slInd != -1) {
+      resourceName = resourceName.substring(slInd + 1);
+    }
+
+    // Allow a properties file in the home directory to override
+    Properties userProps = new Properties(defaultProps);    
+    File propFile = new File(System.getProperties().getProperty("user.home")
+                             + File.separatorChar
+                             + resourceName);
+    if (propFile.exists()) {
+      try {
+        userProps.load(new FileInputStream(propFile));
+      } catch (Exception ex) {
+        throw new Exception("Problem reading user properties: " + propFile);
+      }
+    }
+
+    // Allow a properties file in the current directory to override
+    Properties localProps = new Properties(userProps);
+    propFile = new File(resourceName);
+    if (propFile.exists()) {
+      try {
+        localProps.load(new FileInputStream(propFile));
+      } catch (Exception ex) {
+        throw new Exception("Problem reading local properties: " + propFile);
+      }
+    }
+    
+    return localProps;
+  }
+
+  /**
+   * Returns the correlation coefficient of two double vectors.
+   *
+   * @param y1 double vector 1
+   * @param y2 double vector 2
+   * @param n the length of two double vectors
+   * @return the correlation coefficient
+   */
+  public final static double correlation(double y1[],double y2[],int n) {
+
+    int i;
+    double av1 = 0.0, av2 = 0.0, y11 = 0.0, y22 = 0.0, y12 = 0.0, c;
+    
+    if (n <= 1) {
+      return 1.0;
+    }
+    for (i = 0; i < n; i++) {
+      av1 += y1[i];
+      av2 += y2[i];
+    }
+    av1 /= (double) n;
+    av2 /= (double) n;
+    for (i = 0; i < n; i++) {
+      y11 += (y1[i] - av1) * (y1[i] - av1);
+      y22 += (y2[i] - av2) * (y2[i] - av2);
+      y12 += (y1[i] - av1) * (y2[i] - av2);
+    }
+    if (y11 * y22 == 0.0) {
+      c=1.0;
+    } else {
+      c = y12 / Math.sqrt(Math.abs(y11 * y22));
+    }
+    
+    return c;
+  }
+
+  /**
+   * Removes all occurrences of a string from another string.
+   *
+   * @param inString the string to remove substrings from.
+   * @param substring the substring to remove.
+   * @return the input string with occurrences of substring removed.
+   */
+  public static String removeSubstring(String inString, String substring) {
+
+    StringBuffer result = new StringBuffer();
+    int oldLoc = 0, loc = 0;
+    while ((loc = inString.indexOf(substring, oldLoc))!= -1) {
+      result.append(inString.substring(oldLoc, loc));
+      oldLoc = loc + substring.length();
+    }
+    result.append(inString.substring(oldLoc));
+    return result.toString();
+  }
+
+  /**
+   * Replaces with a new string, all occurrences of a string from 
+   * another string.
+   *
+   * @param inString the string to replace substrings in.
+   * @param substring the substring to replace.
+   * @param replaceString the replacement substring
+   * @return the input string with occurrences of substring replaced.
+   */
+  public static String replaceSubstring(String inString, String subString,
+					String replaceString) {
+
+    StringBuffer result = new StringBuffer();
+    int oldLoc = 0, loc = 0;
+    while ((loc = inString.indexOf(subString, oldLoc))!= -1) {
+      result.append(inString.substring(oldLoc, loc));
+      result.append(replaceString);
+      oldLoc = loc + subString.length();
+    }
+    result.append(inString.substring(oldLoc));
+    return result.toString();
+  }
+
+
+  /**
+   * Pads a string to a specified length, inserting spaces on the left
+   * as required. If the string is too long, characters are removed (from
+   * the right).
+   *
+   * @param inString the input string
+   * @param length the desired length of the output string
+   * @return the output string
+   */
+  public static String padLeft(String inString, int length) {
+
+    return fixStringLength(inString, length, false);
+  }
+  
+  /**
+   * Pads a string to a specified length, inserting spaces on the right
+   * as required. If the string is too long, characters are removed (from
+   * the right).
+   *
+   * @param inString the input string
+   * @param length the desired length of the output string
+   * @return the output string
+   */
+  public static String padRight(String inString, int length) {
+
+    return fixStringLength(inString, length, true);
+  }
+  
+  /**
+   * Pads a string to a specified length, inserting spaces as
+   * required. If the string is too long, characters are removed (from
+   * the right).
+   *
+   * @param inString the input string
+   * @param length the desired length of the output string
+   * @param right true if inserted spaces should be added to the right
+   * @return the output string
+   */
+  private static String fixStringLength(String inString, int length,
+					boolean right) {
+
+    if (inString.length() < length) {
+      while (inString.length() < length) {
+	inString = (right ? inString.concat(" ") : " ".concat(inString));
+      }
+    } else if (inString.length() > length) {
+      inString = inString.substring(0, length);
+    }
+    return inString;
+  }
+ 
+  /**
+   * Rounds a double and converts it into String.
+   *
+   * @param value the double value
+   * @param afterDecimalPoint the (maximum) number of digits permitted
+   * after the decimal point
+   * @return the double as a formatted string
+   */
+  public static String doubleToString(double value, int afterDecimalPoint) {
+    
+    StringBuffer stringBuffer;
+    double temp;
+    int i,dotPosition;
+    long precisionValue;
+    
+    temp = value * Math.pow(10.0, afterDecimalPoint);
+    if (Math.abs(temp) < Long.MAX_VALUE) {
+      precisionValue = 	(temp > 0) ? (long)(temp + 0.5) 
+                                   : -(long)(Math.abs(temp) + 0.5);
+      if (precisionValue == 0) {
+	stringBuffer = new StringBuffer(String.valueOf(0));
+      } else {
+	stringBuffer = new StringBuffer(String.valueOf(precisionValue));
+      }
+      if (afterDecimalPoint == 0) {
+	return stringBuffer.toString();
+      }
+      dotPosition = stringBuffer.length() - afterDecimalPoint;
+      while (((precisionValue < 0) && (dotPosition < 1)) ||
+	     (dotPosition < 0)) {
+	if (precisionValue < 0) {
+	  stringBuffer.insert(1, 0);
+	} else {
+	  stringBuffer.insert(0, 0);
+	}
+	dotPosition++;
+      }
+      stringBuffer.insert(dotPosition, '.');
+      if ((precisionValue < 0) && (stringBuffer.charAt(1) == '.')) {
+	stringBuffer.insert(1, 0);
+      } else if (stringBuffer.charAt(0) == '.') {
+	stringBuffer.insert(0, 0);
+      }
+      int currentPos = stringBuffer.length() - 1;
+      while ((currentPos > dotPosition) &&
+	     (stringBuffer.charAt(currentPos) == '0')) {
+	stringBuffer.setCharAt(currentPos--, ' ');
+      }
+      if (stringBuffer.charAt(currentPos) == '.') {
+	stringBuffer.setCharAt(currentPos, ' ');
+      }
+      
+      return stringBuffer.toString().trim();
+    }
+    return new String("" + value);
+  }
+
+  /**
+   * Rounds a double and converts it into a formatted decimal-justified String.
+   * Trailing 0's are replaced with spaces.
+   *
+   * @param value the double value
+   * @param width the width of the string
+   * @param afterDecimalPoint the number of digits after the decimal point
+   * @return the double as a formatted string
+   */
+  public static String doubleToString(double value, int width,
+				      int afterDecimalPoint) {
+    
+    String tempString = doubleToString(value, afterDecimalPoint);
+    char[] result;
+    int dotPosition;
+
+    if ((afterDecimalPoint >= width) 
+        || (tempString.indexOf('E') != -1)) { // Protects sci notation
+      return tempString;
+    }
+
+    // Initialize result
+    result = new char[width];
+    for (int i = 0; i < result.length; i++) {
+      result[i] = ' ';
+    }
+
+    if (afterDecimalPoint > 0) {
+      // Get position of decimal point and insert decimal point
+      dotPosition = tempString.indexOf('.');
+      if (dotPosition == -1) {
+	dotPosition = tempString.length();
+      } else {
+	result[width - afterDecimalPoint - 1] = '.';
+      }
+    } else {
+      dotPosition = tempString.length();
+    }
+    
+
+    int offset = width - afterDecimalPoint - dotPosition;
+    if (afterDecimalPoint > 0) {
+      offset--;
+    }
+
+    // Not enough room to decimal align within the supplied width
+    if (offset < 0) {
+      return tempString;
+    }
+
+    // Copy characters before decimal point
+    for (int i = 0; i < dotPosition; i++) {
+      result[offset + i] = tempString.charAt(i);
+    }
+
+    // Copy characters after decimal point
+    for (int i = dotPosition + 1; i < tempString.length(); i++) {
+      result[offset + i] = tempString.charAt(i);
+    }
+
+    return new String(result);
+  }
+
+  /**
+   * Tests if a is equal to b.
+   *
+   * @param a a double
+   * @param b a double
+   */
+  public static boolean eq(double a, double b){
+    
+    return (a - b < SMALL) && (b - a < SMALL); 
+  }
+
+  /**
+   * Checks if the given array contains any non-empty options.
+   *
+   * @param strings an array of strings
+   * @exception Exception if there are any non-empty options
+   */
+  public static void checkForRemainingOptions(String [] options) 
+    throws Exception {
+    
+    int illegalOptionsFound = 0;
+    StringBuffer text = new StringBuffer();
+
+    if (options == null) {
+      return;
+    }
+    for (int i = 0; i < options.length; i++) {
+      if (options[i].length() > 0) {
+	illegalOptionsFound++;
+	text.append(options[i] + ' ');
+      }
+    }
+    if (illegalOptionsFound > 0) {
+      throw new Exception("Illegal options: " + text);
+    }
+  }
+  
+  /**
+   * Checks if the given array contains the flag "-Char". Stops
+   * searching at the first marker "--". If the flag is found,
+   * it is replaced with the empty string.
+   *
+   * @param flag the character indicating the flag.
+   * @param strings the array of strings containing all the options.
+   * @return true if the flag was found
+   * @exception Exception if an illegal option was found
+   */
+
+  public static boolean getFlag(char flag, String [] options) 
+    throws Exception {
+
+    if (options == null) {
+      return false;
+    }
+    for (int i = 0; i < options.length; i++) {
+      if ((options[i].length() > 1) && (options[i].charAt(0) == '-')) {
+	try {
+	  Double dummy = Double.valueOf(options[i]);
+	} catch (NumberFormatException e) {
+	  if (options[i].length() > 2) {
+	    throw new Exception("Illegal option: " + options[i]);
+	  }
+	  if (options[i].charAt(1) == flag) {
+	    options[i] = "";
+	    return true;
+	  }
+	  if (options[i].charAt(1) == '-') {
+	    return false;
+	  }
+	}
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Gets an option indicated by a flag "-Char" from the given array
+   * of strings. Stops searching at the first marker "--". Replaces 
+   * flag and option with empty strings.
+   *
+   * @param flag the character indicating the option.
+   * @param options the array of strings containing all the options.
+   * @return the indicated option or an empty string
+   * @exception Exception if the option indicated by the flag can't be found
+   */
+  public static String getOption(char flag, String [] options) 
+    throws Exception {
+
+    String newString;
+
+    if (options == null)
+      return "";
+    for (int i = 0; i < options.length; i++) {
+      if ((options[i].length() > 0) && (options[i].charAt(0) == '-')) {
+	
+	// Check if it is a negative number
+	try {
+	  Double dummy = Double.valueOf(options[i]);
+	} catch (NumberFormatException e) {
+	  if (options[i].length() > 2) {
+	    throw new Exception("Illegal option: " + options[i]);
+	  }
+	  if (options[i].charAt(1) == flag) {
+	    if (i + 1 == options.length) {
+	      throw new Exception("No value given for -" + flag + " option.");
+	    }
+	    options[i] = "";
+	    newString = new String(options[i + 1]);
+	    options[i + 1] = "";
+	    return newString;
+	  }
+	  if (options[i].charAt(1) == '-') {
+	    return "";
+	  }
+	}
+      }
+    }
+    return "";
+  }
+
+  /**
+   * Quotes a string if it contains special characters.
+   * 
+   * The following rules are applied:
+   *
+   * A character is backquoted version of it is one 
+   * of <tt>" ' % \ \n \r \t</tt>.
+   *
+   * A string is enclosed within single quotes if a character has been
+   * backquoted using the previous rule above or contains 
+   * <tt>{ }</tt> or is exactly equal to the strings 
+   * <tt>, ? space or ""</tt> (empty string).
+   *
+   * A quoted question mark distinguishes it from the missing value which
+   * is represented as an unquoted question mark in arff files.
+   *
+   * @param string the string to be quoted
+   * @return the string (possibly quoted)
+   */
+  public static String quote(String string) {
+      boolean quote = false;
+
+      // backquote the following characters 
+      if ((string.indexOf('\n') != -1) || (string.indexOf('\r') != -1) || 
+	  (string.indexOf('\'') != -1) || (string.indexOf('"') != -1) || 
+	  (string.indexOf('\\') != -1) || 
+	  (string.indexOf('\t') != -1) || (string.indexOf('%') != -1)) {
+	  string = backQuoteChars(string);
+	  quote = true;
+      }
+
+      // Enclose the string in 's if the string contains a recently added
+      // backquote or contains one of the following characters.
+      if((quote == true) || 
+	 (string.indexOf('{') != -1) || (string.indexOf('}') != -1) ||
+	 (string.indexOf(',') != -1) || (string.equals("?")) ||
+	 (string.indexOf(' ') != -1) || (string.equals(""))) {
+	  string = ("'".concat(string)).concat("'");
+      }
+
+      return string;
+  }
+
+  /**
+   * Converts carriage returns and new lines in a string into \r and \n.
+   * Backquotes the following characters: ` " \ \t and %
+   * @param string the string
+   * @return the converted string
+   */
+  public static String backQuoteChars(String string) {
+
+    int index;
+    StringBuffer newStringBuffer;
+
+    // replace each of the following characters with the backquoted version
+    char   charsFind[] =    {'\\',   '\'',  '\t',  '"',    '%'};
+    String charsReplace[] = {"\\\\", "\\'", "\\t", "\\\"", "\\%"};
+    for(int i = 0; i < charsFind.length; i++) {
+	if (string.indexOf(charsFind[i]) != -1 ) {
+	    newStringBuffer = new StringBuffer();
+	    while ((index = string.indexOf(charsFind[i])) != -1) {
+		if (index > 0) {
+		    newStringBuffer.append(string.substring(0, index));
+		}
+		newStringBuffer.append(charsReplace[i]);
+		if ((index + 1) < string.length()) {
+		    string = string.substring(index + 1);
+		} else {
+		    string = "";
+		}
+	    }
+	    newStringBuffer.append(string);
+	    string = newStringBuffer.toString();
+	}
+    }
+
+    return Utils.convertNewLines(string);
+  }
+
+  /**
+   * Converts carriage returns and new lines in a string into \r and \n.
+   * @param string the string
+   * @return the converted string
+   */
+  public static String convertNewLines(String string) {
+    
+    int index;
+
+    // Replace with \n
+    StringBuffer newStringBuffer = new StringBuffer();
+    while ((index = string.indexOf('\n')) != -1) {
+      if (index > 0) {
+	newStringBuffer.append(string.substring(0, index));
+      }
+      newStringBuffer.append('\\');
+      newStringBuffer.append('n');
+      if ((index + 1) < string.length()) {
+	string = string.substring(index + 1);
+      } else {
+	string = "";
+      }
+    }
+    newStringBuffer.append(string);
+    string = newStringBuffer.toString();
+
+    // Replace with \r
+    newStringBuffer = new StringBuffer();
+    while ((index = string.indexOf('\r')) != -1) {
+      if (index > 0) {
+	newStringBuffer.append(string.substring(0, index));
+      }
+      newStringBuffer.append('\\');
+      newStringBuffer.append('r');
+      if ((index + 1) < string.length()){
+	string = string.substring(index + 1);
+      } else {
+	string = "";
+      }
+    }
+    newStringBuffer.append(string);
+    return newStringBuffer.toString();
+  }
+    
+
+  /**
+   * Returns the secondary set of options (if any) contained in
+   * the supplied options array. The secondary set is defined to
+   * be any options after the first "--". These options are removed from
+   * the original options array.
+   *
+   * @param options the input array of options
+   * @return the array of secondary options
+   */
+  public static String [] partitionOptions(String [] options) {
+
+    for (int i = 0; i < options.length; i++) {
+      if (options[i].equals("--")) {
+	options[i++] = "";
+	String [] result = new String [options.length - i];
+	for (int j = i; j < options.length; j++) {
+	  result[j - i] = options[j];
+	  options[j] = "";
+	}
+	return result;
+      }
+    }
+    return new String [0];
+  }
+
+  /**
+   * Split up a string containing options into an array of strings,
+   * one for each option.
+   *
+   * @param optionString the string containing the options
+   * @return the array of options
+   */
+  public static String [] splitOptions(String optionString) {
+
+    FastVector optionsVec = new FastVector();
+    StringTokenizer st = new StringTokenizer(optionString);
+    while (st.hasMoreTokens())
+      optionsVec.addElement(st.nextToken());
+
+    String [] options = new String[optionsVec.size()];
+    for (int i = 0; i < optionsVec.size(); i++) {
+      options[i] = (String)optionsVec.elementAt(i);
+    }
+    return options;
+  }
+
+  /**
+   * Joins all the options in an option array into a single string,
+   * as might be used on the command line.
+   *
+   * @param optionArray the array of options
+   * @return the string containing all options.
+   */
+  public static String joinOptions(String [] optionArray) {
+
+    String optionString = "";
+    for (int i = 0; i < optionArray.length; i++) {
+      if (optionArray[i].equals("")) {
+	continue;
+      }
+      if (optionArray[i].indexOf(' ') != -1) {
+	optionString += '"' + optionArray[i] + '"';
+      } else {
+	optionString += optionArray[i];
+      }
+      optionString += " ";
+    }
+    return optionString.trim();
+  }
+  
+  /**
+   * Creates a new instance of an object given it's class name and
+   * (optional) arguments to pass to it's setOptions method. If the
+   * object implements OptionHandler and the options parameter is
+   * non-null, the object will have it's options set. Example use:<p>
+   *
+   * <code> <pre>
+   * String classifierName = Utils.getOption('W', options);
+   * Classifier c = (Classifier)Utils.forName(Classifier.class,
+   *                                          classifierName,
+   *                                          options);
+   * setClassifier(c);
+   * </pre></code>
+   *
+   * @param classType the class that the instantiated object should
+   * be assignable to -- an exception is thrown if this is not the case
+   * @param className the fully qualified class name of the object
+   * @param options an array of options suitable for passing to setOptions. May
+   * be null. Any options accepted by the object will be removed from the
+   * array.
+   * @return the newly created object, ready for use.
+   * @exception Exception if the class name is invalid, or if the
+   * class is not assignable to the desired class type, or the options
+   * supplied are not acceptable to the object
+   */
+  public static Object forName(Class classType,
+			       String className,
+			       String [] options) throws Exception {
+
+    Class c = null;
+    try {
+      c = Class.forName(className);
+    } catch (Exception ex) {
+      throw new Exception("Can't find class called: " + className);
+    }
+    if (!classType.isAssignableFrom(c)) {
+      throw new Exception(classType.getName() + " is not assignable from "
+			  + className);
+    }
+    Object o = c.newInstance();
+    if ((o instanceof OptionHandler)
+	&& (options != null)) {
+      ((OptionHandler)o).setOptions(options);
+      Utils.checkForRemainingOptions(options);
+    }
+    return o;
+  }
+
+  /**
+   * Computes entropy for an array of integers.
+   *
+   * @param counts array of counts
+   * @returns - a log2 a - b log2 b - c log2 c + (a+b+c) log2 (a+b+c)
+   * when given array [a b c]
+   */
+  public static double info(int counts[]) {
+    
+    int total = 0; int c;
+    double x = 0;
+    for (int j = 0; j < counts.length; j++) {
+      x -= xlogx(counts[j]);
+      total += counts[j];
+    }
+    return x + xlogx(total);
+  }
+
+  /**
+   * Tests if a is smaller or equal to b.
+   *
+   * @param a a double
+   * @param b a double
+   */
+  public static boolean smOrEq(double a,double b) {
+    
+    return (a-b < SMALL);
+  }
+
+  /**
+   * Tests if a is greater or equal to b.
+   *
+   * @param a a double
+   * @param b a double
+   */
+  public static boolean grOrEq(double a,double b) {
+    
+    return (b-a < SMALL);
+  }
+  
+  /**
+   * Tests if a is smaller than b.
+   *
+   * @param a a double
+   * @param b a double
+   */
+  public static boolean sm(double a,double b) {
+    
+    return (b-a > SMALL);
+  }
+
+  /**
+   * Tests if a is smaller than b.
+   *
+   * @param a a double
+   * @param b a double 
+   */
+  public static boolean gr(double a,double b) {
+    
+    return (a-b > SMALL);
+  }
+
+  /**
+   * Returns the logarithm of a for base 2.
+   *
+   * @param a a double
+   */
+  public static double log2(double a) {
+    
+    return Math.log(a) / log2;
+  }
+
+  /**
+   * Returns index of maximum element in a given
+   * array of doubles. First maximum is returned.
+   *
+   * @param doubles the array of doubles
+   * @return the index of the maximum element
+   */
+  public static int maxIndex(double [] doubles) {
+
+    double maximum = 0;
+    int maxIndex = 0;
+
+    for (int i = 0; i < doubles.length; i++) {
+      if ((i == 0) || (doubles[i] > maximum)) {
+	maxIndex = i;
+	maximum = doubles[i];
+      }
+    }
+
+    return maxIndex;
+  }
+
+  /**
+   * Returns index of maximum element in a given
+   * array of integers. First maximum is returned.
+   *
+   * @param ints the array of integers
+   * @return the index of the maximum element
+   */
+  public static int maxIndex(int [] ints) {
+
+    int maximum = 0;
+    int maxIndex = 0;
+
+    for (int i = 0; i < ints.length; i++) {
+      if ((i == 0) || (ints[i] > maximum)) {
+	maxIndex = i;
+	maximum = ints[i];
+      }
+    }
+
+    return maxIndex;
+  }
+
+  /**
+   * Computes the mean for an array of doubles.
+   *
+   * @param vector the array
+   * @return the mean
+   */
+  public static double mean(double[] vector) {
+  
+    double sum = 0;
+
+    if (vector.length == 0) {
+      return 0;
+    }
+    for (int i = 0; i < vector.length; i++) {
+      sum += vector[i];
+    }
+    return sum / (double) vector.length;
+  }
+
+  /**
+   * Returns index of minimum element in a given
+   * array of integers. First minimum is returned.
+   *
+   * @param ints the array of integers
+   * @return the index of the minimum element
+   */
+  public static int minIndex(int [] ints) {
+
+    int minimum = 0;
+    int minIndex = 0;
+
+    for (int i = 0; i < ints.length; i++) {
+      if ((i == 0) || (ints[i] < minimum)) {
+	minIndex = i;
+	minimum = ints[i];
+      }
+    }
+
+    return minIndex;
+  }
+
+  /**
+   * Returns index of minimum element in a given
+   * array of doubles. First minimum is returned.
+   *
+   * @param doubles the array of doubles
+   * @return the index of the minimum element
+   */
+  public static int minIndex(double [] doubles) {
+
+    double minimum = 0;
+    int minIndex = 0;
+
+    for (int i = 0; i < doubles.length; i++) {
+      if ((i == 0) || (doubles[i] < minimum)) {
+	minIndex = i;
+	minimum = doubles[i];
+      }
+    }
+
+    return minIndex;
+  }
+
+  /**
+   * Normalizes the doubles in the array by their sum.
+   *
+   * @param doubles the array of double
+   * @exception IllegalArgumentException if sum is Zero or NaN
+   */
+  public static void normalize(double[] doubles) {
+
+    double sum = 0;
+    for (int i = 0; i < doubles.length; i++) {
+      sum += doubles[i];
+    }
+    normalize(doubles, sum);
+  }
+
+  /**
+   * Normalizes the doubles in the array using the given value.
+   *
+   * @param doubles the array of double
+   * @param sum the value by which the doubles are to be normalized
+   * @exception IllegalArgumentException if sum is zero or NaN
+   */
+  public static void normalize(double[] doubles, double sum) {
+
+    if (Double.isNaN(sum)) {
+      throw new IllegalArgumentException("Can't normalize array. Sum is NaN.");
+    }
+    if (sum == 0) {
+      // Maybe this should just be a return.
+      throw new IllegalArgumentException("Can't normalize array. Sum is zero.");
+    }
+    for (int i = 0; i < doubles.length; i++) {
+      doubles[i] /= sum;
+    }
+  }
+
+  /**
+   * Rounds a double to the next nearest integer value. The JDK version
+   * of it doesn't work properly.
+   *
+   * @param value the double value
+   * @return the resulting integer value
+   */
+  public static int round(double value) {
+
+    int roundedValue = value > 0
+      ? (int)(value + 0.5)
+      : -(int)(Math.abs(value) + 0.5);
+    
+    return roundedValue;
+  }
+
+  /**
+   * Rounds a double to the given number of decimal places.
+   *
+   * @param value the double value
+   * @param afterDecimalPoint the number of digits after the decimal point
+   * @return the double rounded to the given precision
+   */
+  public static double roundDouble(double value,int afterDecimalPoint) {
+
+    double mask = Math.pow(10.0, (double)afterDecimalPoint);
+
+    return (double)(Math.round(value * mask)) / mask;
+  }
+
+  /**
+   * Sorts a given array of integers in ascending order and returns an 
+   * array of integers with the positions of the elements of the original 
+   * array in the sorted array. The sort is stable. (Equal elements remain
+   * in their original order.)
+   *
+   * @param array this array is not changed by the method!
+   * @return an array of integers with the positions in the sorted
+   * array.
+   */
+  public static int[] sort(int [] array) {
+
+    int [] index = new int[array.length];
+    int [] newIndex = new int[array.length];
+    int [] helpIndex;
+    int numEqual;
+    
+    for (int i = 0; i < index.length; i++) {
+      index[i] = i;
+    }
+    quickSort(array, index, 0, array.length - 1);
+
+    // Make sort stable
+    int i = 0;
+    while (i < index.length) {
+      numEqual = 1;
+      for (int j = i + 1; ((j < index.length)
+			   && (array[index[i]] == array[index[j]]));
+	   j++) {
+	numEqual++;
+      }
+      if (numEqual > 1) {
+	helpIndex = new int[numEqual];
+	for (int j = 0; j < numEqual; j++) {
+	  helpIndex[j] = i + j;
+	}
+	quickSort(index, helpIndex, 0, numEqual - 1);
+	for (int j = 0; j < numEqual; j++) {
+	  newIndex[i + j] = index[helpIndex[j]];
+	}
+	i += numEqual;
+      } else {
+	newIndex[i] = index[i];
+	i++;
+      }
+    }
+    return newIndex;
+  }
+
+  /**
+   * Sorts a given array of doubles in ascending order and returns an
+   * array of integers with the positions of the elements of the
+   * original array in the sorted array. NOTE THESE CHANGES: the sort
+   * is no longer stable and it doesn't use safe floating-point
+   * comparisons anymore. Occurrences of Double.NaN are treated as 
+   * Double.MAX_VALUE
+   *
+   * @param array this array is not changed by the method!
+   * @return an array of integers with the positions in the sorted
+   * array.  
+   */
+  public static int[] sort(double [] array) {
+
+    int [] index = new int[array.length];
+    array = (double [])array.clone();
+    for (int i = 0; i < index.length; i++) {
+      index[i] = i;
+      if (Double.isNaN(array[i])) {
+        array[i] = Double.MAX_VALUE;
+      }
+    }
+    quickSort(array, index, 0, array.length - 1);
+    return index;
+  }
+
+  /**
+   * Sorts a given array of doubles in ascending order and returns an 
+   * array of integers with the positions of the elements of the original 
+   * array in the sorted array. The sort is stable (Equal elements remain
+   * in their original order.) Occurrences of Double.NaN are treated as 
+   * Double.MAX_VALUE
+   *
+   * @param array this array is not changed by the method!
+   * @return an array of integers with the positions in the sorted
+   * array.
+   */
+  public static int[] stableSort(double [] array){
+
+    int [] index = new int[array.length];
+    int [] newIndex = new int[array.length];
+    int [] helpIndex;
+    int numEqual;
+    
+    array = (double [])array.clone();
+    for (int i = 0; i < index.length; i++) {
+      index[i] = i;
+      if (Double.isNaN(array[i])) {
+        array[i] = Double.MAX_VALUE;
+      }
+    }
+    quickSort(array,index,0,array.length-1);
+
+    // Make sort stable
+
+    int i = 0;
+    while (i < index.length) {
+      numEqual = 1;
+      for (int j = i+1; ((j < index.length) && Utils.eq(array[index[i]],
+							array[index[j]])); j++)
+	numEqual++;
+      if (numEqual > 1) {
+	helpIndex = new int[numEqual];
+	for (int j = 0; j < numEqual; j++)
+	  helpIndex[j] = i+j;
+	quickSort(index, helpIndex, 0, numEqual-1);
+	for (int j = 0; j < numEqual; j++) 
+	  newIndex[i+j] = index[helpIndex[j]];
+	i += numEqual;
+      } else {
+	newIndex[i] = index[i];
+	i++;
+      }
+    }
+
+    return newIndex;
+  }
+
+  /**
+   * Computes the variance for an array of doubles.
+   *
+   * @param vector the array
+   * @return the variance
+   */
+  public static double variance(double[] vector) {
+  
+    double sum = 0, sumSquared = 0;
+
+    if (vector.length <= 1) {
+      return 0;
+    }
+    for (int i = 0; i < vector.length; i++) {
+      sum += vector[i];
+      sumSquared += (vector[i] * vector[i]);
+    }
+    return (sumSquared - (sum * sum / (double) vector.length)) / 
+      (double) (vector.length - 1);
+  }
+
+  /**
+   * Computes the sum of the elements of an array of doubles.
+   *
+   * @param doubles the array of double
+   * @returns the sum of the elements
+   */
+  public static double sum(double[] doubles) {
+
+    double sum = 0;
+
+    for (int i = 0; i < doubles.length; i++) {
+      sum += doubles[i];
+    }
+    return sum;
+  }
+
+  /**
+   * Computes the sum of the elements of an array of integers.
+   *
+   * @param ints the array of integers
+   * @returns the sum of the elements
+   */
+  public static int sum(int[] ints) {
+
+    int sum = 0;
+
+    for (int i = 0; i < ints.length; i++) {
+      sum += ints[i];
+    }
+    return sum;
+  }
+
+  /**
+   * Returns c*log2(c) for a given integer value c.
+   *
+   * @param c an integer value
+   * @returns c*log2(c) (but is careful to return 0 if c is 0)
+   */
+  public static double xlogx(int c) {
+    
+    if (c == 0) {
+      return 0.0;
+    }
+    return c * Utils.log2((double) c);
+  }
+
+  /**
+   * Implements quicksort for an array of indices.
+   *
+   * @param array the array of integers to be sorted
+   * @param index the index which should contain the positions in the
+   * sorted array
+   * @param lo0 the first index of the subset to be sorted
+   * @param hi0 the last index of the subset to be sorted
+   */
+  private static void quickSort(int [] array, int [] index,
+				int lo0, int hi0) {
+
+    int lo = lo0;
+    int hi = hi0;
+    int mid;
+    int help;
+    
+    if (hi0 > lo0) {
+      
+      // Arbitrarily establishing partition element as the midpoint of
+      // the array.
+      mid = array[index[(lo0 + hi0) / 2]];
+
+      // loop through the array until indices cross
+      while (lo <= hi) {
+	
+	// find the first element that is greater than or equal to  
+	// the partition element starting from the left Index.
+	while ((array[index[lo]] < mid) && (lo < hi0)) {
+	  ++lo;
+	}
+	
+	// find an element that is smaller than or equal to 
+	// the partition element starting from the right Index.
+	while ((array[index[hi]] > mid) && (hi > lo0)) {
+	  --hi;
+	}
+	
+	// if the indexes have not crossed, swap
+	if (lo <= hi) {
+	  help = index[lo];
+	  index[lo] = index[hi];
+	  index[hi] = help;
+	  ++lo;
+	  --hi;
+	}
+      }
+      
+      // If the right index has not reached the left side of array
+      // must now sort the left partition.
+      if (lo0 < hi) {
+	quickSort(array, index, lo0, hi);
+      }
+      
+      // If the left index has not reached the right side of array
+      // must now sort the right partition.
+      if (lo < hi0) {
+	quickSort(array, index, lo, hi0);
+      }
+    }
+  }
+
+  /**
+   * Implements unsafe quicksort for an array of indices.
+   *
+   * @param array the array of doubles to be sorted
+   * @param index the index which should contain the positions in the
+   * sorted array
+   * @param lo0 the first index of the subset to be sorted
+   * @param hi0 the last index of the subset to be sorted
+   */
+  private static void quickSort(double [] array, int [] index, int lo0, int hi0) {
+
+    int lo = lo0;
+    int hi = hi0;
+    double mid;
+    int help;
+    
+    if (hi0 > lo0) {
+      
+      // Arbitrarily establishing partition element as the midpoint of
+      // the array.
+      mid = array[index[(lo0 + hi0) / 2]];
+
+      // loop through the array until indices cross
+      while (lo <= hi) {
+	
+	// find the first element that is greater than or equal to  
+	// the partition element starting from the left Index.
+	while ((array[index[lo]] < mid) && (lo < hi0)) {
+	  ++lo;
+	}
+	
+	// find an element that is smaller than or equal to 
+	// the partition element starting from the right Index.
+	while ((array[index[hi]] > mid) && (hi > lo0)) {
+	  --hi;
+	}
+	
+	// if the indexes have not crossed, swap
+	if (lo <= hi) {
+	  help = index[lo];
+	  index[lo] = index[hi];
+	  index[hi] = help;
+	  ++lo;
+	  --hi;
+	}
+      }
+      
+      // If the right index has not reached the left side of array
+      // must now sort the left partition.
+      if (lo0 < hi) {
+	quickSort(array, index, lo0, hi);
+      }
+      
+      // If the left index has not reached the right side of array
+      // must now sort the right partition.
+      if (lo < hi0) {
+	quickSort(array, index, lo, hi0);
+      }
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param ops some dummy options
+   */
+  public static void main(String[] ops) {
+
+    double[] doubles = {4.5, 6.7, Double.NaN, 3.4, 4.8, 1.2, 3.4};
+    int[] ints = {12, 6, 2, 18, 16, 6, 7, 5};
+
+    try {
+
+      // Option handling
+      System.out.println("First option split up:");
+      if (ops.length > 0) {
+	String[] firstOptionSplitUp = Utils.splitOptions(ops[0]);
+	for (int i = 0; i < firstOptionSplitUp.length; i ++) {
+	  System.out.println(firstOptionSplitUp[i]);
+	}
+      }					       
+      System.out.println("Partitioned options: ");
+      String[] partitionedOptions = Utils.partitionOptions(ops);
+      for (int i  = 0; i < partitionedOptions.length; i++) {
+	System.out.println(partitionedOptions[i]);
+      }
+      System.out.println("Get flag -f: " + Utils.getFlag('f', ops));
+      System.out.println("Get option -o: " + Utils.getOption('o', ops));
+      System.out.println("Checking for remaining options... ");
+      Utils.checkForRemainingOptions(ops);
+      
+      // Statistics
+      System.out.println("Original array (doubles): ");
+      for (int i = 0; i < doubles.length; i++) {
+	System.out.print(doubles[i] + " ");
+      }
+      System.out.println();
+      System.out.println("Original array (ints): ");
+      for (int i = 0; i < ints.length; i++) {
+	System.out.print(ints[i] + " ");
+      }
+      System.out.println();
+      System.out.println("Correlation: " + Utils.correlation(doubles, doubles, 
+							     doubles.length));
+      System.out.println("Mean: " + Utils.mean(doubles));
+      System.out.println("Variance: " + Utils.variance(doubles));
+      System.out.println("Sum (doubles): " + Utils.sum(doubles));
+      System.out.println("Sum (ints): " + Utils.sum(ints));
+      System.out.println("Max index (doubles): " + Utils.maxIndex(doubles));
+      System.out.println("Max index (ints): " + Utils.maxIndex(ints));
+      System.out.println("Min index (doubles): " + Utils.minIndex(doubles));
+      System.out.println("Min index (ints): " + Utils.minIndex(ints));
+      
+      // Sorting and normalizing
+      System.out.println("Sorted array (doubles): ");
+      int[] sorted = Utils.sort(doubles);
+      for (int i = 0; i < doubles.length; i++) {
+	System.out.print(doubles[sorted[i]] + " ");
+      }
+      System.out.println();
+      System.out.println("Normalized array (doubles): ");
+      Utils.normalize(doubles);
+      for (int i = 0; i < doubles.length; i++) {
+	System.out.print(doubles[i] + " ");
+      }
+      System.out.println();
+      System.out.println("Normalized again (doubles): ");
+      Utils.normalize(doubles, Utils.sum(doubles));
+      for (int i = 0; i < doubles.length; i++) {
+	System.out.print(doubles[i] + " ");
+      }
+      System.out.println();
+      
+      // Pretty-printing
+      System.out.println("-4.58: " + Utils.doubleToString(-4.57826535, 2));
+      System.out.println("-6.78: " + Utils.doubleToString(-6.78214234, 6,2));
+      
+      // Comparisons
+      System.out.println("5.70001 == 5.7 ? " + Utils.eq(5.70001, 5.7));
+      System.out.println("5.70001 > 5.7 ? " + Utils.gr(5.70001, 5.7));
+      System.out.println("5.70001 >= 5.7 ? " + Utils.grOrEq(5.70001, 5.7));
+      System.out.println("5.7 < 5.70001 ? " + Utils.sm(5.7, 5.70001));
+      System.out.println("5.7 <= 5.70001 ? " + Utils.smOrEq(5.7, 5.70001));
+      
+      // Math
+      System.out.println("Info (ints): " + Utils.info(ints));
+      System.out.println("log2(4.6): " + Utils.log2(4.6));
+      System.out.println("5 * log(5): " + Utils.xlogx(5));
+      System.out.println("5.5 rounded: " + Utils.round(5.5));
+      System.out.println("5.55555 rounded to 2 decimal places: " + 
+			 Utils.roundDouble(5.55555, 2));
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}
+  
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WeightedInstancesHandler.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WeightedInstancesHandler.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WeightedInstancesHandler.java	(revision 21112)
@@ -0,0 +1,43 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    WeightedInstancesHandler.java
+ *    Copyright (C) 1999 Eibe Frank
+ *
+ */
+
+package weka.core;
+
+/** 
+ * Interface to something that makes use of the information provided
+ * by instance weights.
+ *
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$ 
+*/
+public interface WeightedInstancesHandler {
+
+  // Nothing in here, because the class is just used as an indicator
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WekaException.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WekaException.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/core/WekaException.java	(revision 21112)
@@ -0,0 +1,50 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    WekaException.java
+ *    Copyright (C) 2001 Webmind Corp.
+ */
+
+package weka.core;
+
+/**
+ * <code>WekaException</code> is used when some Weka-specific
+ * checked exception must be raised.
+ *
+ * @author <a href="mailto:len@webmind.com">Len Trigg</a>
+ * @version $Revision$
+ */
+public class WekaException extends Exception {
+
+  /**
+   * Creates a new <code>WekaException</code> instance
+   * with no detail message.
+   */
+  public WekaException() { 
+    super(); 
+  }
+
+  /**
+   * Creates a new <code>WekaException</code> instance
+   * with a specified message.
+   *
+   * @param messagae a <code>String</code> containing the message.
+   */
+  public WekaException(String message) { 
+    super(message); 
+  }
+}
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/experiment/Stats.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/experiment/Stats.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/experiment/Stats.java	(revision 21112)
@@ -0,0 +1,188 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    Stats.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+
+package weka.experiment;
+import weka.core.Utils;
+
+/**
+ * A class to store simple statistics
+ *
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class Stats {
+  
+  /** The number of values seen */
+  public double count = 0;
+
+  /** The sum of values seen */
+  public double sum = 0;
+
+  /** The sum of values squared seen */
+  public double sumSq = 0;
+
+  /** The std deviation of values at the last calculateDerived() call */    
+  public double stdDev = Double.NaN;
+
+  /** The mean of values at the last calculateDerived() call */    
+  public double mean = Double.NaN;
+
+  /** The minimum value seen, or Double.NaN if no values seen */
+  public double min = Double.NaN;
+
+  /** The maximum value seen, or Double.NaN if no values seen */
+  public double max = Double.NaN;
+    
+  /**
+   * Adds a value to the observed values
+   *
+   * @param value the observed value
+   */
+  public void add(double value) {
+
+    add(value, 1);
+  }
+
+  /**
+   * Adds a value that has been seen n times to the observed values
+   *
+   * @param value the observed value
+   * @param n the number of times to add value
+   */
+  public void add(double value, double n) {
+
+    sum += value * n;
+    sumSq += value * value * n;
+    count += n;
+    if (Double.isNaN(min)) {
+      min = max = value;
+    } else if (value < min) {
+      min = value;
+    } else if (value > max) {
+      max = value;
+    }
+  }
+
+  /**
+   * Removes a value to the observed values (no checking is done
+   * that the value being removed was actually added). 
+   *
+   * @param value the observed value
+   */
+  public void subtract(double value) {
+
+    sum -= value;
+    sumSq -= value * value;
+    count --;
+  }
+
+  /**
+   * Tells the object to calculate any statistics that don't have their
+   * values automatically updated during add. Currently updates the mean
+   * and standard deviation.
+   */
+  public void calculateDerived() {
+
+    mean = Double.NaN;
+    stdDev = Double.NaN;
+    if (count > 0) {
+      mean = sum / count;
+      stdDev = Double.POSITIVE_INFINITY;
+      if (count > 1) {
+	stdDev = sumSq - (sum * sum) / count;
+	stdDev /= (count - 1);
+        if (stdDev < 0) {
+          System.err.println("Warning: stdDev value = " + stdDev 
+                             + " -- rounded to zero.");
+          stdDev = 0;
+        }
+	stdDev = Math.sqrt(stdDev);
+      }
+    }
+  }
+    
+  /**
+   * Returns a string summarising the stats so far.
+   *
+   * @return the summary string
+   */
+  public String toString() {
+
+    calculateDerived();
+    return
+      "Count   " + Utils.doubleToString(count, 8) + '\n'
+      + "Min     " + Utils.doubleToString(min, 8) + '\n'
+      + "Max     " + Utils.doubleToString(max, 8) + '\n'
+      + "Sum     " + Utils.doubleToString(sum, 8) + '\n'
+      + "SumSq   " + Utils.doubleToString(sumSq, 8) + '\n'
+      + "Mean    " + Utils.doubleToString(mean, 8) + '\n'
+      + "StdDev  " + Utils.doubleToString(stdDev, 8) + '\n';
+  }
+
+  /**
+   * Tests the paired stats object from the command line.
+   * reads line from stdin, expecting two values per line.
+   *
+   * @param args ignored.
+   */
+  public static void main(String [] args) {
+
+    try {
+      Stats ps = new Stats();
+      java.io.LineNumberReader r = new java.io.LineNumberReader(
+				   new java.io.InputStreamReader(System.in));
+      String line;
+      while ((line = r.readLine()) != null) {
+        line = line.trim();
+        if (line.equals("") || line.startsWith("@") || line.startsWith("%")) {
+          continue;
+        }
+	java.util.StringTokenizer s 
+          = new java.util.StringTokenizer(line, " ,\t\n\r\f");
+	int count = 0;
+	double v1 = 0;
+	while (s.hasMoreTokens()) {
+	  double val = (new Double(s.nextToken())).doubleValue();
+	  if (count == 0) {
+	    v1 = val;
+	  } else {
+            System.err.println("MSG: Too many values in line \"" 
+                               + line + "\", skipped.");
+	    break;
+	  }
+	  count++;
+	}
+        if (count == 1) {
+          ps.add(v1);
+        }
+      }
+      ps.calculateDerived();
+      System.err.println(ps);
+    } catch (Exception ex) {
+      ex.printStackTrace();
+      System.err.println(ex.getMessage());
+    }
+  }
+
+} // Stats
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/DiscretizeFilter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/DiscretizeFilter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/DiscretizeFilter.java	(revision 21112)
@@ -0,0 +1,1292 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ *    DiscretizeFilter.java
+ *    Copyright (C) 1999 Eibe Frank,Len Trigg
+ *
+ */
+
+
+package weka.filters;
+
+import java.io.*;
+import java.util.*;
+import weka.core.*;
+
+/** 
+ * An instance filter that discretizes a range of numeric attributes in 
+ * the dataset into nominal attributes. Discretization can be either by 
+ * simple binning, or by Fayyad & Irani's MDL method (the default).<p>
+ *
+ * Valid filter-specific options are: <p>
+ *
+ * -B num <br>
+ * Specifies the (maximum) number of bins to divide numeric attributes into.
+ * (default: class-based discretisation).<p>
+ *
+ * -F <br>
+ * Use equal-frequency instead of equal-width discretization if 
+ * class-based discretisation is turned off.<p>
+ *
+ * -O <br>
+ * Optimize the number of bins using a leave-one-out estimate of the 
+ * entropy (for equal-width binning).<p>
+ *
+ * -R col1,col2-col4,... <br>
+ * Specifies list of columns to Discretize. First
+ * and last are valid indexes. (default: none) <p>
+ *
+ * -V <br>
+ * Invert matching sense.<p>
+ *
+ * -D <br>
+ * Make binary nominal attributes. <p>
+ *
+ * -E <br>
+ * Use better encoding of split point for MDL. <p>
+ *   
+ * -K <br>
+ * Use Kononeko's MDL criterion. <p>
+ * 
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @author Eibe Frank (eibe@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public class DiscretizeFilter extends Filter 
+  implements OptionHandler, WeightedInstancesHandler {
+
+  /** Stores which columns to Discretize */
+  protected Range m_DiscretizeCols = new Range();
+
+  /** The number of bins to divide the attribute into */
+  protected int m_NumBins = 10;
+
+  /** Store the current cutpoints */
+  protected double [][] m_CutPoints = null;
+
+  /** True if discretisation will be done by MDL rather than binning */
+  protected boolean m_UseMDL = true;
+
+  /** Output binary attributes for discretized attributes. */
+  protected boolean m_MakeBinary = false;
+
+  /** Use better encoding of split point for MDL. */
+  protected boolean m_UseBetterEncoding = false;
+
+  /** Use Kononenko's MDL criterion instead of Fayyad et al.'s */
+  protected boolean m_UseKononenko = false;
+
+  /** Find the number of bins using cross-validated entropy. */
+  protected boolean m_FindNumBins = false;
+
+  /** Use equal-frequency binning if unsupervised discretization turned on */
+  protected boolean m_UseEqualFrequency = false;
+
+  /** Constructor - initialises the filter */
+  public DiscretizeFilter() {
+
+    setAttributeIndices("first-last");
+  }
+
+
+  /**
+   * Gets an enumeration describing the available options
+   *
+   * @return an enumeration of all the available options
+   */
+  public Enumeration listOptions() {
+
+    Vector newVector = new Vector(7);
+
+    newVector.addElement(new Option(
+              "\tSpecifies the (maximum) number of bins to divide numeric"
+	      + " attributes into.\n"
+	      + "\t(default class-based discretization)",
+              "B", 1, "-B <num>"));
+
+    newVector.addElement(new Option(
+              "\tUse equal-frequency instead of equal-width with\n"+
+	      "\tunsupervised discretization.",
+              "F", 0, "-F"));
+
+    newVector.addElement(new Option(
+              "\tOptimize number of bins using leave-one-out estimate\n"+
+	      "\tof estimated entropy (for equal-width discretization).",
+              "O", 0, "-O"));
+
+    /* If we decide to implement loading and saving cutfiles like 
+     * the C Discretizer (which is probably not necessary)
+    newVector.addElement(new Option(
+              "\tSpecify that the cutpoints should be loaded from a file.",
+              "L", 1, "-L <file>"));
+    newVector.addElement(new Option(
+              "\tSpecify that the chosen cutpoints should be saved to a file.",
+              "S", 1, "-S <file>"));
+    */
+
+    newVector.addElement(new Option(
+              "\tSpecifies list of columns to Discretize. First"
+	      + " and last are valid indexes.\n"
+	      + "\t(default none)",
+              "R", 1, "-R <col1,col2-col4,...>"));
+
+    newVector.addElement(new Option(
+              "\tInvert matching sense of column indexes.",
+              "V", 0, "-V"));
+
+    newVector.addElement(new Option(
+              "\tOutput binary attributes for discretized attributes.",
+              "D", 0, "-D"));
+
+    newVector.addElement(new Option(
+              "\tUse better encoding of split point for MDL.",
+              "E", 0, "-E"));
+
+    newVector.addElement(new Option(
+              "\tUse Kononenko's MDL criterion.",
+              "K", 0, "-K"));
+
+    return newVector.elements();
+  }
+
+
+  /**
+   * Parses the options for this object. Valid options are: <p>
+   *
+   * -B num <br>
+   * Specifies the (maximum) number of bins to divide numeric attributes into.
+   * (default class-based discretisation).<p>
+   *
+   * -F <br>
+   * Use equal-frequency instead of equal-width discretization if 
+   * class-based discretisation is turned off.<p>
+   *
+   * -O <br>
+   * Optimize the number of bins using a leave-one-out estimate of the 
+   * entropy (for equal-width binning).<p>
+   *
+   * -R col1,col2-col4,... <br>
+   * Specifies list of columns to Discretize. First
+   * and last are valid indexes. (default none) <p>
+   *
+   * -V <br>
+   * Invert matching sense.<p>
+   *
+   * -D <br>
+   * Make binary nominal attributes. <p>
+   *
+   * -E <br>
+   * Use better encoding of split point for MDL. <p>
+   *   
+   * -K <br>
+   * Use Kononeko's MDL criterion. <p>
+   * 
+   * @param options the list of options as an array of strings
+   * @exception Exception if an option is not supported
+   */
+  public void setOptions(String[] options) throws Exception {
+
+    setMakeBinary(Utils.getFlag('D', options));
+    setUseEqualFrequency(Utils.getFlag('F', options));
+    setUseBetterEncoding(Utils.getFlag('E', options));
+    setUseKononenko(Utils.getFlag('K', options));
+    setFindNumBins(Utils.getFlag('O', options));
+    setInvertSelection(Utils.getFlag('V', options));
+    setUseMDL(true);
+
+    String numBins = Utils.getOption('B', options);
+    if (numBins.length() != 0) {
+      setBins(Integer.parseInt(numBins));
+      setUseMDL(false);
+    } else {
+      setBins(10);
+    }
+    
+    String convertList = Utils.getOption('R', options);
+    if (convertList.length() != 0) {
+      setAttributeIndices(convertList);
+    } else {
+      setAttributeIndices("first-last");
+    }
+
+    if (getInputFormat() != null) {
+      setInputFormat(getInputFormat());
+    }
+  }
+  /**
+   * Gets the current settings of the filter.
+   *
+   * @return an array of strings suitable for passing to setOptions
+   */
+  public String [] getOptions() {
+
+    String [] options = new String [12];
+    int current = 0;
+
+    if (getMakeBinary()) {
+      options[current++] = "-D";
+    }
+    if (getUseEqualFrequency()) {
+      options[current++] = "-F";
+    }
+    if (getUseBetterEncoding()) {
+      options[current++] = "-E";
+    }
+    if (getUseKononenko()) {
+      options[current++] = "-K";
+    }
+    if (getFindNumBins()) {
+      options[current++] = "-O";
+    }
+    if (getInvertSelection()) {
+      options[current++] = "-V";
+    }
+    if (!getUseMDL()) {
+      options[current++] = "-B"; options[current++] = "" + getBins();
+    }
+    if (!getAttributeIndices().equals("")) {
+      options[current++] = "-R"; options[current++] = getAttributeIndices();
+    }
+    while (current < options.length) {
+      options[current++] = "";
+    }
+    return options;
+  }
+
+
+  /**
+   * Sets the format of the input instances.
+   *
+   * @param instanceInfo an Instances object containing the input instance
+   * structure (any instances contained in the object are ignored - only the
+   * structure is required).
+   * @return true if the outputFormat may be collected immediately
+   * @exception Exception if the input format can't be set successfully
+   */
+  public boolean setInputFormat(Instances instanceInfo) throws Exception {
+
+    super.setInputFormat(instanceInfo);
+
+    m_DiscretizeCols.setUpper(instanceInfo.numAttributes() - 1);
+    m_CutPoints = null;
+    if (m_UseMDL) {
+      if (instanceInfo.classIndex() < 0) {
+	throw new UnassignedClassException("Cannot use class-based discretization: "
+                                           + "no class assigned to the dataset");
+      }
+      if (!instanceInfo.classAttribute().isNominal()) {
+	throw new UnsupportedClassTypeException("Supervised discretization not possible:"
+                                                + " class is not nominal!");
+      }
+    } else {
+      if (getFindNumBins() && getUseEqualFrequency()) {
+	throw new IllegalArgumentException("Bin number optimization in conjunction "+
+					   "with equal-frequency binning not implemented.");
+      }
+    }
+
+    // If we implement loading cutfiles, then load 
+    //them here and set the output format
+    return false;
+  }
+
+  
+
+  /**
+   * Input an instance for filtering. Ordinarily the instance is processed
+   * and made available for output immediately. Some filters require all
+   * instances be read before producing output.
+   *
+   * @param instance the input instance
+   * @return true if the filtered instance may now be
+   * collected with output().
+   * @exception IllegalStateException if no input format has been defined.
+   */
+  public boolean input(Instance instance) {
+
+    if (getInputFormat() == null) {
+      throw new IllegalStateException("No input instance format defined");
+    }
+    if (m_NewBatch) {
+      resetQueue();
+      m_NewBatch = false;
+    }
+    
+    if (m_CutPoints != null) {
+      convertInstance(instance);
+      return true;
+    }
+
+    bufferInput(instance);
+    return false;
+  }
+
+
+  /**
+   * Signifies that this batch of input to the filter is finished. If the 
+   * filter requires all instances prior to filtering, output() may now 
+   * be called to retrieve the filtered instances.
+   *
+   * @return true if there are instances pending output
+   * @exception IllegalStateException if no input structure has been defined
+   */
+  public boolean batchFinished() {
+
+    if (getInputFormat() == null) {
+      throw new IllegalStateException("No input instance format defined");
+    }
+    if (m_CutPoints == null) {
+      calculateCutPoints();
+
+      setOutputFormat();
+
+      // If we implement saving cutfiles, save the cuts here
+
+      // Convert pending input instances
+      for(int i = 0; i < getInputFormat().numInstances(); i++) {
+	convertInstance(getInputFormat().instance(i));
+      }
+    } 
+    flushInput();
+
+    m_NewBatch = true;
+    return (numPendingOutput() != 0);
+  }
+
+  /**
+   * Returns a string describing this filter
+   *
+   * @return a description of the filter suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String globalInfo() {
+
+    return "An instance filter that discretizes a range of numeric"
+      + " attributes in the dataset into nominal attributes."
+      + " Discretization can be either by simple binning, or by"
+      + " Fayyad & Irani's MDL method (the default).";
+  }
+  
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String findNumBinsTipText() {
+
+    return "Optimize number of equal-width bins using leave-one-out.";
+  }
+
+  /**
+   * Get the value of FindNumBins.
+   *
+   * @return Value of FindNumBins.
+   */
+  public boolean getFindNumBins() {
+    
+    return m_FindNumBins;
+  }
+  
+  /**
+   * Set the value of FindNumBins.
+   *
+   * @param newFindNumBins Value to assign to FindNumBins.
+   */
+  public void setFindNumBins(boolean newFindNumBins) {
+    
+    m_FindNumBins = newFindNumBins;
+  }
+  
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String makeBinaryTipText() {
+
+    return "Make resulting attributes binary.";
+  }
+
+  /**
+   * Gets whether binary attributes should be made for discretized ones.
+   *
+   * @return true if attributes will be binarized
+   */
+  public boolean getMakeBinary() {
+
+    return m_MakeBinary;
+  }
+
+  /** 
+   * Sets whether binary attributes should be made for discretized ones.
+   *
+   * @param makeBinary if binary attributes are to be made
+   */
+  public void setMakeBinary(boolean makeBinary) {
+
+    m_MakeBinary = makeBinary;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String useMDLTipText() {
+
+    return "Use class-based discretization. If set to false, does"
+      + " not require a class attribute, and uses a fixed number"
+      + " of bins (according to bins setting).";
+  }
+
+  /**
+   * Gets whether MDL will be used as the discretisation method.
+   *
+   * @return true if so, false if fixed bins should be used.
+   */
+  public boolean getUseMDL() {
+
+    return m_UseMDL;
+  }
+
+  /** 
+   * Sets whether MDL will be used as the discretisation method.
+   *
+   * @param useMDL true if MDL should be used, false if fixed bins should
+   * be used.
+   */
+  public void setUseMDL(boolean useMDL) {
+
+    m_UseMDL = useMDL;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String useKononenkoTipText() {
+
+    return "Use Kononenko's MDL criterion. If set to false"
+      + " uses the Fayyad & Irani criterion.";
+  }
+  
+  /**
+   * Get the value of UseEqualFrequency.
+   *
+   * @return Value of UseEqualFrequency.
+   */
+  public boolean getUseEqualFrequency() {
+    
+    return m_UseEqualFrequency;
+  }
+  
+  /**
+   * Set the value of UseEqualFrequency.
+   *
+   * @param newUseEqualFrequency Value to assign to UseEqualFrequency.
+   */
+  public void setUseEqualFrequency(boolean newUseEqualFrequency) {
+    
+    m_UseEqualFrequency = newUseEqualFrequency;
+  }
+  
+  /**
+   * Gets whether Kononenko's MDL criterion is to be used.
+   *
+   * @return true if Kononenko's criterion will be used.
+   */
+  public boolean getUseKononenko() {
+
+    return m_UseKononenko;
+  }
+
+  /** 
+   * Sets whether Kononenko's MDL criterion is to be used.
+   *
+   * @param useKon true if Kononenko's one is to be used
+   */
+  public void setUseKononenko(boolean useKon) {
+
+    m_UseKononenko = useKon;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String useBetterEncodingTipText() {
+
+    return "Uses a more efficient split point encoding.";
+  }
+
+  /**
+   * Gets whether better encoding is to be used for MDL.
+   *
+   * @return true if the better MDL encoding will be used
+   */
+  public boolean getUseBetterEncoding() {
+
+    return m_UseBetterEncoding;
+  }
+
+  /** 
+   * Sets whether better encoding is to be used for MDL.
+   *
+   * @param useBetterEncoding true if better encoding to be used.
+   */
+  public void setUseBetterEncoding(boolean useBetterEncoding) {
+
+    m_UseBetterEncoding = useBetterEncoding;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String binsTipText() {
+
+    return "Number of bins for class-blind discretisation. This"
+      + " setting is ignored if MDL-based discretisation is used.";
+  }
+
+  /**
+   * Gets the number of bins numeric attributes will be divided into
+   *
+   * @return the number of bins.
+   */
+  public int getBins() {
+
+    return m_NumBins;
+  }
+
+  /**
+   * Sets the number of bins to divide each selected numeric attribute into
+   *
+   * @param numBins the number of bins
+   */
+  public void setBins(int numBins) {
+
+    m_NumBins = numBins;
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String invertSelectionTipText() {
+
+    return "Set attribute selection mode. If false, only selected"
+      + " (numeric) attributes in the range will be discretized; if"
+      + " true, only non-selected attributes will be discretized.";
+  }
+
+  /**
+   * Gets whether the supplied columns are to be removed or kept
+   *
+   * @return true if the supplied columns will be kept
+   */
+  public boolean getInvertSelection() {
+
+    return m_DiscretizeCols.getInvert();
+  }
+
+  /**
+   * Sets whether selected columns should be removed or kept. If true the 
+   * selected columns are kept and unselected columns are deleted. If false
+   * selected columns are deleted and unselected columns are kept.
+   *
+   * @param invert the new invert setting
+   */
+  public void setInvertSelection(boolean invert) {
+
+    m_DiscretizeCols.setInvert(invert);
+  }
+
+  /**
+   * Returns the tip text for this property
+   *
+   * @return tip text for this property suitable for
+   * displaying in the explorer/experimenter gui
+   */
+  public String attributeIndicesTipText() {
+    return "Specify range of attributes to act on."
+      + " This is a comma separated list of attribute indices, with"
+      + " \"first\" and \"last\" valid values. Specify an inclusive"
+      + " range with \"-\". E.g: \"first-3,5,6-10,last\".";
+  }
+
+  /**
+   * Gets the current range selection
+   *
+   * @return a string containing a comma separated list of ranges
+   */
+  public String getAttributeIndices() {
+
+    return m_DiscretizeCols.getRanges();
+  }
+
+  /**
+   * Sets which attributes are to be Discretized (only numeric
+   * attributes among the selection will be Discretized).
+   *
+   * @param rangeList a string representing the list of attributes. Since
+   * the string will typically come from a user, attributes are indexed from
+   * 1. <br>
+   * eg: first-3,5,6-last
+   * @exception IllegalArgumentException if an invalid range list is supplied 
+   */
+  public void setAttributeIndices(String rangeList) {
+
+    m_DiscretizeCols.setRanges(rangeList);
+  }
+
+  /**
+   * Sets which attributes are to be Discretized (only numeric
+   * attributes among the selection will be Discretized).
+   *
+   * @param attributes an array containing indexes of attributes to Discretize.
+   * Since the array will typically come from a program, attributes are indexed
+   * from 0.
+   * @exception IllegalArgumentException if an invalid set of ranges
+   * is supplied 
+   */
+  public void setAttributeIndicesArray(int [] attributes) {
+
+    setAttributeIndices(Range.indicesToRangeList(attributes));
+  }
+
+  /**
+   * Gets the cut points for an attribute
+   *
+   * @param the index (from 0) of the attribute to get the cut points of
+   * @return an array containing the cutpoints (or null if the
+   * attribute requested isn't being Discretized
+   */
+  public double [] getCutPoints(int attributeIndex) {
+
+    if (m_CutPoints == null) {
+      return null;
+    }
+    return m_CutPoints[attributeIndex];
+  }
+
+  /** Generate the cutpoints for each attribute */
+  protected void calculateCutPoints() {
+
+    Instances copy = null;
+
+    m_CutPoints = new double [getInputFormat().numAttributes()] [];
+    for(int i = getInputFormat().numAttributes() - 1; i >= 0; i--) {
+      if ((m_DiscretizeCols.isInRange(i)) && 
+	  (getInputFormat().attribute(i).isNumeric())) {
+	if (m_UseMDL) {
+
+	  // Use copy to preserve order
+	  if (copy == null) {
+	    copy = new Instances(getInputFormat());
+	  }
+	  calculateCutPointsByMDL(i, copy);
+	} else {
+	  if (m_FindNumBins) {
+	    findNumBins(i);
+	  } else if (!m_UseEqualFrequency) {
+	    calculateCutPointsByEqualWidthBinning(i);
+	  } else {
+	    calculateCutPointsByEqualFrequencyBinning(i);
+	  }
+	}
+      }
+    }
+  }
+
+  /**
+   * Set cutpoints for a single attribute using MDL.
+   *
+   * @param index the index of the attribute to set cutpoints for
+   */
+  protected void calculateCutPointsByMDL(int index,
+					 Instances data) {
+
+    // Sort instances
+    data.sort(data.attribute(index));
+
+    // Find first instances that's missing
+    int firstMissing = data.numInstances();
+    for (int i = 0; i < data.numInstances(); i++) {
+      if (data.instance(i).isMissing(index)) {
+        firstMissing = i;
+        break;
+      }
+    }
+    m_CutPoints[index] = cutPointsForSubset(data, index, 0, firstMissing);
+  }
+
+  /** Test using Kononenko's MDL criterion. */
+  private boolean KononenkosMDL(double[] priorCounts,
+				double[][] bestCounts,
+				double numInstances,
+				int numCutPoints) {
+
+    double distPrior, instPrior, distAfter = 0, sum, instAfter = 0;
+    double before, after;
+    int numClassesTotal;
+
+    // Number of classes occuring in the set
+    numClassesTotal = 0;
+    for (int i = 0; i < priorCounts.length; i++) {
+      if (priorCounts[i] > 0) {
+	numClassesTotal++;
+      }
+    }
+
+    // Encode distribution prior to split
+    distPrior = SpecialFunctions.log2Binomial(numInstances 
+					      + numClassesTotal - 1,
+					      numClassesTotal - 1);
+
+    // Encode instances prior to split.
+    instPrior = SpecialFunctions.log2Multinomial(numInstances,
+						 priorCounts);
+
+    before = instPrior + distPrior;
+
+    // Encode distributions and instances after split.
+    for (int i = 0; i < bestCounts.length; i++) {
+      sum = Utils.sum(bestCounts[i]);
+      distAfter += SpecialFunctions.log2Binomial(sum + numClassesTotal - 1,
+						 numClassesTotal - 1);
+      instAfter += SpecialFunctions.log2Multinomial(sum,
+						    bestCounts[i]);
+    }
+
+    // Coding cost after split
+    after = Utils.log2(numCutPoints) + distAfter + instAfter;
+
+    // Check if split is to be accepted
+    return (Utils.gr(before, after));
+  }
+
+
+  /** Test using Fayyad and Irani's MDL criterion. */
+  private boolean FayyadAndIranisMDL(double[] priorCounts,
+				     double[][] bestCounts,
+				     double numInstances,
+				     int numCutPoints) {
+
+    double priorEntropy, entropy, gain; 
+    double entropyLeft, entropyRight, delta;
+    int numClassesTotal, numClassesRight, numClassesLeft;
+
+    // Compute entropy before split.
+    priorEntropy = ContingencyTables.entropy(priorCounts);
+
+    // Compute entropy after split.
+    entropy = ContingencyTables.entropyConditionedOnRows(bestCounts);
+
+    // Compute information gain.
+    gain = priorEntropy - entropy;
+
+    // Number of classes occuring in the set
+    numClassesTotal = 0;
+    for (int i = 0; i < priorCounts.length; i++) {
+      if (priorCounts[i] > 0) {
+	numClassesTotal++;
+      }
+    }
+
+    // Number of classes occuring in the left subset
+    numClassesLeft = 0;
+    for (int i = 0; i < bestCounts[0].length; i++) {
+      if (bestCounts[0][i] > 0) {
+	numClassesLeft++;
+      }
+    }
+
+    // Number of classes occuring in the right subset
+    numClassesRight = 0;
+    for (int i = 0; i < bestCounts[1].length; i++) {
+      if (bestCounts[1][i] > 0) {
+	numClassesRight++;
+      }
+    }
+
+    // Entropy of the left and the right subsets
+    entropyLeft = ContingencyTables.entropy(bestCounts[0]);
+    entropyRight = ContingencyTables.entropy(bestCounts[1]);
+
+    // Compute terms for MDL formula
+    delta = Utils.log2(Math.pow(3, numClassesTotal) - 2) - 
+      (((double) numClassesTotal * priorEntropy) - 
+       (numClassesRight * entropyRight) - 
+       (numClassesLeft * entropyLeft));
+
+    // Check if split is to be accepted
+    return (Utils.gr(gain, (Utils.log2(numCutPoints) + delta) /
+		     (double)numInstances));
+  }
+    
+
+  /** Selects cutpoints for sorted subset. */
+  private double[] cutPointsForSubset(Instances instances, int attIndex, 
+				      int first, int lastPlusOne) { 
+
+    double[][] counts, bestCounts;
+    double[] priorCounts, left, right, cutPoints;
+    double currentCutPoint = -Double.MAX_VALUE, bestCutPoint = -1, 
+      currentEntropy, bestEntropy, priorEntropy, gain;
+    int bestIndex = -1, numInstances = 0, numCutPoints = 0;
+
+    // Compute number of instances in set
+    if ((lastPlusOne - first) < 2) {
+      return null;
+    }
+
+    // Compute class counts.
+    counts = new double[2][instances.numClasses()];
+    for (int i = first; i < lastPlusOne; i++) {
+      numInstances += instances.instance(i).weight();
+      counts[1][(int)instances.instance(i).classValue()] +=
+	instances.instance(i).weight();
+    }
+
+    // Save prior counts
+    priorCounts = new double[instances.numClasses()];
+    System.arraycopy(counts[1], 0, priorCounts, 0, 
+		     instances.numClasses());
+
+    // Entropy of the full set
+    priorEntropy = ContingencyTables.entropy(priorCounts);
+    bestEntropy = priorEntropy;
+    
+    // Find best entropy.
+    bestCounts = new double[2][instances.numClasses()];
+    for (int i = first; i < (lastPlusOne - 1); i++) {
+      counts[0][(int)instances.instance(i).classValue()] +=
+	instances.instance(i).weight();
+      counts[1][(int)instances.instance(i).classValue()] -=
+	instances.instance(i).weight();
+      if (Utils.sm(instances.instance(i).value(attIndex), 
+		   instances.instance(i + 1).value(attIndex))) {
+	currentCutPoint = instances.instance(i).value(attIndex); //+ 
+	//instances.instance(i + 1).value(attIndex)) / 2.0;
+	currentEntropy = ContingencyTables.entropyConditionedOnRows(counts);
+	if (Utils.sm(currentEntropy, bestEntropy)) {
+	  bestCutPoint = currentCutPoint;
+	  bestEntropy = currentEntropy;
+	  bestIndex = i;
+	  System.arraycopy(counts[0], 0, 
+			   bestCounts[0], 0, instances.numClasses());
+	  System.arraycopy(counts[1], 0, 
+			   bestCounts[1], 0, instances.numClasses()); 
+	}
+	numCutPoints++;
+      }
+    }
+
+    // Use worse encoding?
+    if (!m_UseBetterEncoding) {
+      numCutPoints = (lastPlusOne - first) - 1;
+    }
+
+    // Checks if gain is zero
+    gain = priorEntropy - bestEntropy;
+    if (Utils.eq(gain, 0)) {
+      return null;
+    }
+
+    // Check if split is to be accepted
+    if ((m_UseKononenko && KononenkosMDL(priorCounts, bestCounts,
+					 numInstances, numCutPoints)) ||
+	(!m_UseKononenko && FayyadAndIranisMDL(priorCounts, bestCounts,
+					       numInstances, numCutPoints))) {
+      
+      // Select split points for the left and right subsets
+      left = cutPointsForSubset(instances, attIndex, first, bestIndex + 1);
+      right = cutPointsForSubset(instances, attIndex, 
+				 bestIndex + 1, lastPlusOne);
+      
+      // Merge cutpoints and return them
+      if ((left == null) && (right) == null) {
+	cutPoints = new double[1];
+	cutPoints[0] = bestCutPoint;
+      } else if (right == null) {
+	cutPoints = new double[left.length + 1];
+	System.arraycopy(left, 0, cutPoints, 0, left.length);
+	cutPoints[left.length] = bestCutPoint;
+      } else if (left == null) {
+	cutPoints = new double[1 + right.length];
+	cutPoints[0] = bestCutPoint;
+	System.arraycopy(right, 0, cutPoints, 1, right.length);
+      } else {
+	cutPoints =  new double[left.length + right.length + 1];
+	cutPoints = new double[left.length + right.length + 1];
+	System.arraycopy(left, 0, cutPoints, 0, left.length);
+	cutPoints[left.length] = bestCutPoint;
+	System.arraycopy(right, 0, cutPoints, left.length + 1, right.length);
+      }
+      
+      return cutPoints;
+    } else
+      return null;
+  }
+ 
+  /**
+   * Set cutpoints for a single attribute.
+   *
+   * @param index the index of the attribute to set cutpoints for
+   */
+  protected void calculateCutPointsByEqualWidthBinning(int index) {
+
+    // Scan for max and min values
+    double max = 0, min = 1, currentVal;
+    Instance currentInstance;
+    for(int i = 0; i < getInputFormat().numInstances(); i++) {
+      currentInstance = getInputFormat().instance(i);
+      if (!currentInstance.isMissing(index)) {
+	currentVal = currentInstance.value(index);
+	if (max < min) {
+	  max = min = currentVal;
+	}
+	if (currentVal > max) {
+	  max = currentVal;
+	}
+	if (currentVal < min) {
+	  min = currentVal;
+	}
+      }
+    }
+    double binWidth = (max - min) / m_NumBins;
+    double [] cutPoints = null;
+    if ((m_NumBins > 1) && (binWidth > 0)) {
+      cutPoints = new double [m_NumBins - 1];
+      for(int i = 1; i < m_NumBins; i++) {
+	cutPoints[i - 1] = min + binWidth * i;
+      }
+    }
+    m_CutPoints[index] = cutPoints;
+  }
+ 
+  /**
+   * Set cutpoints for a single attribute.
+   *
+   * @param index the index of the attribute to set cutpoints for
+   */
+  protected void calculateCutPointsByEqualFrequencyBinning(int index) {
+
+    // Copy data so that it can be sorted
+    Instances data = new Instances(getInputFormat());
+
+    // Sort input data
+    data.sort(index);
+
+    // Compute weight of instances without missing values
+    double sumOfWeights = 0;
+    for (int i = 0; i < data.numInstances(); i++) {
+      if (data.instance(i).isMissing(index)) {
+	break;
+      } else {
+	sumOfWeights += data.instance(i).weight();
+      }
+    }
+    double freq = sumOfWeights / m_NumBins;
+
+    // Compute break points
+    double[] cutPoints = new double[m_NumBins - 1];
+    double counter = 0;
+    int cpindex = 0;
+    for (int i = 0; i < data.numInstances() - 1; i++) {
+
+      // Stop if value missing
+      if (data.instance(i).isMissing(index)) {
+	break;
+      }
+      counter += data.instance(i).weight();
+
+      // Do we have a potential breakpoint?
+      if (data.instance(i).value(index) < 
+	  data.instance(i + 1).value(index)) {
+	if (counter >= freq) {
+	  cutPoints[cpindex] = (data.instance(i).value(index) +
+				data.instance(i + 1).value(index)) / 2;
+	  cpindex++;
+	  counter = counter - freq;
+	}
+      }
+    }
+
+    // Did we find any cutpoints?
+    if (cpindex == 0) {
+      m_CutPoints[index] = null;
+    } else {
+      double[] cp = new double[cpindex];
+      for (int i = 0; i < cpindex; i++) {
+	cp[i] = cutPoints[i];
+      }
+      m_CutPoints[index] = cp;
+    }
+  }
+
+  /**
+   * Optimizes the number of bins using leave-one-out cross-validation.
+   *
+   * @param index the attribute index
+   */
+  protected void findNumBins(int index) {
+
+    double min = Double.MAX_VALUE, max = -Double.MIN_VALUE, binWidth = 0, 
+      entropy, bestEntropy = Double.MAX_VALUE, currentVal;
+    double[] distribution;
+    int bestNumBins  = 1;
+    Instance currentInstance;
+
+    // Find minimum and maximum
+    for (int i = 0; i < getInputFormat().numInstances(); i++) {
+      currentInstance = getInputFormat().instance(i);
+      if (!currentInstance.isMissing(index)) {
+	currentVal = currentInstance.value(index);
+	if (currentVal > max) {
+	  max = currentVal;
+	}
+	if (currentVal < min) {
+	  min = currentVal;
+	}
+      }
+    }
+
+    // Find best number of bins
+    for (int i = 0; i < m_NumBins; i++) {
+      distribution = new double[i + 1];
+      binWidth = (max - min) / (i + 1);
+
+      // Compute distribution
+      for (int j = 0; j < getInputFormat().numInstances(); j++) {
+	currentInstance = getInputFormat().instance(j);
+	if (!currentInstance.isMissing(index)) {
+	  for (int k = 0; k < i + 1; k++) {
+	    if (currentInstance.value(index) <= 
+		(min + (((double)k + 1) * binWidth))) {
+	      distribution[k] += currentInstance.weight();
+	      break;
+	    }
+	  }
+	}
+      }
+
+      // Compute cross-validated entropy
+      entropy = 0;
+      for (int k = 0; k < i + 1; k++) {
+	if (distribution[k] < 2) {
+	  entropy = Double.MAX_VALUE;
+	  break;
+	}
+	entropy -= distribution[k] * Math.log((distribution[k] - 1) / 
+					      binWidth);
+      }
+
+      // Best entropy so far?
+      if (entropy < bestEntropy) {
+	bestEntropy = entropy;
+	bestNumBins = i + 1;
+      }
+    }
+
+    // Compute cut points
+    double [] cutPoints = null;
+    if ((bestNumBins > 1) && (binWidth > 0)) {
+      cutPoints = new double [bestNumBins - 1];
+      for(int i = 1; i < bestNumBins; i++) {
+	cutPoints[i - 1] = min + binWidth * i;
+      }
+    }
+    m_CutPoints[index] = cutPoints;
+   }
+
+  /**
+   * Set the output format. Takes the currently defined cutpoints and 
+   * m_InputFormat and calls setOutputFormat(Instances) appropriately.
+   */
+  protected void setOutputFormat() {
+
+    if (m_CutPoints == null) {
+      setOutputFormat(null);
+      return;
+    }
+    FastVector attributes = new FastVector(getInputFormat().numAttributes());
+    int classIndex = getInputFormat().classIndex();
+    for(int i = 0; i < getInputFormat().numAttributes(); i++) {
+      if ((m_DiscretizeCols.isInRange(i)) 
+	  && (getInputFormat().attribute(i).isNumeric())) {
+	if (!m_MakeBinary) {
+	  FastVector attribValues = new FastVector(1);
+	  if (m_CutPoints[i] == null) {
+	    attribValues.addElement("'All'");
+	  } else {
+	    for(int j = 0; j <= m_CutPoints[i].length; j++) {
+	      if (j == 0) {
+		attribValues.addElement("'(-inf-"
+			+ Utils.doubleToString(m_CutPoints[i][j], 6) + "]'");
+	      } else if (j == m_CutPoints[i].length) {
+		attribValues.addElement("'("
+			+ Utils.doubleToString(m_CutPoints[i][j - 1], 6) 
+					+ "-inf)'");
+	      } else {
+		attribValues.addElement("'("
+			+ Utils.doubleToString(m_CutPoints[i][j - 1], 6) + "-"
+			+ Utils.doubleToString(m_CutPoints[i][j], 6) + "]'");
+	      }
+	    }
+	  }
+	  attributes.addElement(new Attribute(getInputFormat().
+					      attribute(i).name(),
+					      attribValues));
+	} else {
+	  if (m_CutPoints[i] == null) {
+	    FastVector attribValues = new FastVector(1);
+	    attribValues.addElement("'All'");
+	    attributes.addElement(new Attribute(getInputFormat().
+						attribute(i).name(),
+						attribValues));
+	  } else {
+	    if (i < getInputFormat().classIndex()) {
+	      classIndex += m_CutPoints[i].length - 1;
+	    }
+	    for(int j = 0; j < m_CutPoints[i].length; j++) {
+	      FastVector attribValues = new FastVector(2);
+	      attribValues.addElement("'(-inf-"
+		      + Utils.doubleToString(m_CutPoints[i][j], 6) + "]'");
+	      attribValues.addElement("'("
+		      + Utils.doubleToString(m_CutPoints[i][j], 6) + "-inf)'");
+	      attributes.addElement(new Attribute(getInputFormat().
+						  attribute(i).name(),
+						  attribValues));
+	    }
+	  }
+	}
+      } else {
+	attributes.addElement(getInputFormat().attribute(i).copy());
+      }
+    }
+    Instances outputFormat = 
+      new Instances(getInputFormat().relationName(), attributes, 0);
+    outputFormat.setClassIndex(classIndex);
+    setOutputFormat(outputFormat);
+  }
+
+  /**
+   * Convert a single instance over. The converted instance is added to 
+   * the end of the output queue.
+   *
+   * @param instance the instance to convert
+   */
+  protected void convertInstance(Instance instance) {
+
+    int index = 0;
+    double [] vals = new double [outputFormatPeek().numAttributes()];
+    // Copy and convert the values
+    for(int i = 0; i < getInputFormat().numAttributes(); i++) {
+      if (m_DiscretizeCols.isInRange(i) && 
+	  getInputFormat().attribute(i).isNumeric()) {
+	int j;
+	double currentVal = instance.value(i);
+	if (m_CutPoints[i] == null) {
+	  if (instance.isMissing(i)) {
+	    vals[index] = Instance.missingValue();
+	  } else {
+	    vals[index] = 0;
+	  }
+	  index++;
+	} else {
+	  if (!m_MakeBinary) {
+	    if (instance.isMissing(i)) {
+	      vals[index] = Instance.missingValue();
+	    } else {
+	      for (j = 0; j < m_CutPoints[i].length; j++) {
+		if (currentVal <= m_CutPoints[i][j]) {
+		  break;
+		}
+	      }
+              vals[index] = j;
+	    }
+	    index++;
+	  } else {
+	    for (j = 0; j < m_CutPoints[i].length; j++) {
+	      if (instance.isMissing(i)) {
+                vals[index] = Instance.missingValue();
+	      } else if (currentVal <= m_CutPoints[i][j]) {
+                vals[index] = 0;
+	      } else {
+                vals[index] = 1;
+	      }
+	      index++;
+	    }
+	  }   
+	}
+      } else {
+        vals[index] = instance.value(i);
+	index++;
+      }
+    }
+    
+    Instance inst = null;
+    if (instance instanceof SparseInstance) {
+      inst = new SparseInstance(instance.weight(), vals);
+    } else {
+      inst = new Instance(instance.weight(), vals);
+    }
+    copyStringValues(inst, false, instance.dataset(), getInputStringIndex(),
+                     getOutputFormat(), getOutputStringIndex());
+    inst.setDataset(getOutputFormat());
+    push(inst);
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain arguments to the filter: use -h for help
+   */
+  public static void main(String [] argv) {
+
+    try {
+      if (Utils.getFlag('b', argv)) {
+ 	Filter.batchFilterFile(new DiscretizeFilter(), argv);
+      } else {
+	Filter.filterFile(new DiscretizeFilter(), argv);
+      }
+    } catch (Exception ex) {
+      System.out.println(ex.getMessage());
+    }
+  }
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/Filter.java
===================================================================
--- /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/Filter.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/kea/kea-3.0/weka/filters/Filter.java	(revision 21112)
@@ -0,0 +1,934 @@
+/*
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+/*
+ *    Filter.java
+ *    Copyright (C) 1999 Len Trigg
+ *
+ */
+
+
+package weka.filters;
+
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.io.Serializable;
+import java.util.Enumeration;
+import weka.core.Attribute;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.Option;
+import weka.core.OptionHandler;
+import weka.core.Queue;
+import weka.core.Utils;
+
+/** 
+ * An abstract class for instance filters: objects that take instances
+ * as input, carry out some transformation on the instance and then
+ * output the instance. The method implementations in this class
+ * assume that most of the work will be done in the methods overridden
+ * by subclasses.<p>
+ *
+ * A simple example of filter use. This example doesn't remove
+ * instances from the output queue until all instances have been
+ * input, so has higher memory consumption than an approach that
+ * uses output instances as they are made available:<p>
+ *
+ * <code> <pre>
+ *  Filter filter = ..some type of filter..
+ *  Instances instances = ..some instances..
+ *  for (int i = 0; i < data.numInstances(); i++) {
+ *    filter.input(data.instance(i));
+ *  }
+ *  filter.batchFinished();
+ *  Instances newData = filter.outputFormat();
+ *  Instance processed;
+ *  while ((processed = filter.output()) != null) {
+ *    newData.add(processed);
+ *  }
+ *  ..do something with newData..
+ * </pre> </code>
+ *
+ * @author Len Trigg (trigg@cs.waikato.ac.nz)
+ * @version $Revision$
+ */
+public abstract class Filter implements Serializable {
+
+  /*
+   * Filter refactoring TODO:
+   *
+   * - Update all filters to use getOutputFormat and setInputFormat
+   * instead of outputFormat, outputFormatPeek and inputFormat.
+   * - Update users of filters to use getOutputFormat and setInputFormat
+   * - remove outputFormat, outputFormatPeek and inputFormat
+   *
+   */
+
+  /** Debugging mode */
+  private boolean m_Debug = false;
+
+  /** The output format for instances */
+  private Instances m_OutputFormat = null;
+
+  /** The output instance queue */
+  private Queue m_OutputQueue = null;
+
+  /** Indices of string attributes in the output format */
+  private int [] m_OutputStringAtts = null;
+
+  /** Indices of string attributes in the input format */
+  private int [] m_InputStringAtts = null;
+
+  /** The input format for instances */
+  private Instances m_InputFormat = null;
+
+  /** Record whether the filter is at the start of a batch */
+  protected boolean m_NewBatch = true;
+
+  /**
+   * Sets the format of output instances. The derived class should use this
+   * method once it has determined the outputformat. The 
+   * output queue is cleared.
+   *
+   * @param outputFormat the new output format
+   */
+  protected void setOutputFormat(Instances outputFormat) {
+
+    if (outputFormat != null) {
+      m_OutputFormat = outputFormat.stringFreeStructure();
+      m_OutputStringAtts = getStringIndices(m_OutputFormat);
+
+      // Rename the attribute
+      String relationName = outputFormat.relationName() 
+        + "-" + this.getClass().getName();
+      if (this instanceof OptionHandler) {
+        String [] options = ((OptionHandler)this).getOptions();
+        for (int i = 0; i < options.length; i++) {
+          relationName += options[i].trim();
+        }
+      }
+      m_OutputFormat.setRelationName(relationName);
+    } else {
+      m_OutputFormat = null;
+    }
+    m_OutputQueue = new Queue();
+  }
+
+  /**
+   * Gets the currently set inputformat instances. This dataset may contain
+   * buffered instances.
+   *
+   * @return the input Instances.
+   */
+  protected Instances getInputFormat() {
+
+    return m_InputFormat;
+  }
+
+  /**
+   * Returns a reference to the current output format without
+   * copying it.
+   *
+   * @return a reference to the current output format
+   */
+  protected Instances outputFormatPeek() {
+
+    return m_OutputFormat;
+  }
+
+  /**
+   * Adds an output instance to the queue. The derived class should use this
+   * method for each output instance it makes available. 
+   *
+   * @param instance the instance to be added to the queue
+   */
+  protected void push(Instance instance) {
+
+    if (instance != null) {
+      copyStringValues(instance, m_OutputFormat, m_OutputStringAtts);
+      instance.setDataset(m_OutputFormat);
+      m_OutputQueue.push(instance);
+    }
+  }
+
+  /**
+   * Clears the output queue.
+   */
+  protected void resetQueue() {
+
+    m_OutputQueue = new Queue();
+  }
+
+  /**
+   * Adds the supplied input instance to the inputformat dataset for
+   * later processing.  Use this method rather than
+   * getInputFormat().add(instance). Or else.
+   *
+   * @param instance the <code>Instance</code> to buffer.  
+   */
+  protected void bufferInput(Instance instance) {
+
+    if (instance != null) {
+      copyStringValues(instance, m_InputFormat, m_InputStringAtts);
+      instance.setDataset(m_InputFormat);
+      m_InputFormat.add(instance);
+    }
+  }
+
+  /**
+   * Returns an array containing the indices of all string attributes in the
+   * input format. This index is created during setInputFormat()
+   *
+   * @return an array containing the indices of string attributes in the 
+   * input dataset.
+   */
+  protected int [] getInputStringIndex() {
+
+    return m_InputStringAtts;
+  }
+
+  /**
+   * Returns an array containing the indices of all string attributes in the
+   * output format. This index is created during setOutputFormat()
+   *
+   * @return an array containing the indices of string attributes in the 
+   * output dataset.
+   */
+  protected int [] getOutputStringIndex() {
+
+    return m_OutputStringAtts;
+  }
+
+  /**
+   * Copies string values contained in the instance copied to a new
+   * dataset. The Instance must already be assigned to a dataset. This
+   * dataset and the destination dataset must have the same structure.
+   *
+   * @param instance the Instance containing the string values to copy.
+   * @param destDataset the destination set of Instances
+   * @param strAtts an array containing the indices of any string attributes
+   * in the dataset.  
+   */
+  private void copyStringValues(Instance inst, Instances destDataset, 
+                                int []strAtts) {
+
+    if (strAtts.length == 0) {
+      return;
+    }
+    if (inst.dataset() == null) {
+      throw new IllegalArgumentException("Instance has no dataset assigned!!");
+    } else if (inst.dataset().numAttributes() != destDataset.numAttributes()) {
+      throw new IllegalArgumentException("Src and Dest differ in # of attributes!!");
+    } 
+    copyStringValues(inst, true, inst.dataset(), strAtts,
+                     destDataset, strAtts);
+  }
+
+  /**
+   * Takes string values referenced by an Instance and copies them from a
+   * source dataset to a destination dataset. The instance references are
+   * updated to be valid for the destination dataset. The instance may have the 
+   * structure (i.e. number and attribute position) of either dataset (this
+   * affects where references are obtained from). The source dataset must
+   * have the same structure as the filter input format and the destination
+   * must have the same structure as the filter output format.
+   *
+   * @param instance the instance containing references to strings in the source
+   * dataset that will have references updated to be valid for the destination
+   * dataset.
+   * @param instSrcCompat true if the instance structure is the same as the
+   * source, or false if it is the same as the destination
+   * @param srcDataset the dataset for which the current instance string
+   * references are valid (after any position mapping if needed)
+   * @param destDataset the dataset for which the current instance string
+   * references need to be inserted (after any position mapping if needed)
+   */
+  protected void copyStringValues(Instance instance, boolean instSrcCompat,
+                                  Instances srcDataset, Instances destDataset) {
+
+    copyStringValues(instance, instSrcCompat, srcDataset, m_InputStringAtts,
+                     destDataset, m_OutputStringAtts);
+  }
+
+  /**
+   * Takes string values referenced by an Instance and copies them from a
+   * source dataset to a destination dataset. The instance references are
+   * updated to be valid for the destination dataset. The instance may have the 
+   * structure (i.e. number and attribute position) of either dataset (this
+   * affects where references are obtained from). Only works if the number
+   * of string attributes is the same in both indices (implicitly these string
+   * attributes should be semantically same but just with shifted positions).
+   *
+   * @param instance the instance containing references to strings in the source
+   * dataset that will have references updated to be valid for the destination
+   * dataset.
+   * @param instSrcCompat true if the instance structure is the same as the
+   * source, or false if it is the same as the destination (i.e. which of the
+   * string attribute indices contains the correct locations for this instance).
+   * @param srcDataset the dataset for which the current instance string
+   * references are valid (after any position mapping if needed)
+   * @param srcStrAtts an array containing the indices of string attributes
+   * in the source datset.
+   * @param destDataset the dataset for which the current instance string
+   * references need to be inserted (after any position mapping if needed)
+   * @param destStrAtts an array containing the indices of string attributes
+   * in the destination datset.
+   */
+  protected void copyStringValues(Instance instance, boolean instSrcCompat,
+                                  Instances srcDataset, int []srcStrAtts,
+                                  Instances destDataset, int []destStrAtts) {
+    if (srcDataset == destDataset) {
+      return;
+    }
+    if (srcStrAtts.length != destStrAtts.length) {
+      throw new IllegalArgumentException("Src and Dest string indices differ in length!!");
+    }
+    for (int i = 0; i < srcStrAtts.length; i++) {
+      int instIndex = instSrcCompat ? srcStrAtts[i] : destStrAtts[i];
+      Attribute src = srcDataset.attribute(srcStrAtts[i]);
+      Attribute dest = destDataset.attribute(destStrAtts[i]);
+      if (!instance.isMissing(instIndex)) {
+        //System.err.println(instance.value(srcIndex) 
+        //                   + " " + src.numValues()
+        //                   + " " + dest.numValues());
+        int valIndex = dest.addStringValue(src, (int)instance.value(instIndex));
+        // setValue here shouldn't be too slow here unless your dataset has
+        // squillions of string attributes
+        instance.setValue(instIndex, (double)valIndex);
+      }
+    }
+  }
+
+  /**
+   * This will remove all buffered instances from the inputformat dataset.
+   * Use this method rather than getInputFormat().delete();
+   */
+  protected void flushInput() {
+
+    if (m_InputStringAtts.length > 0) {
+      m_InputFormat = m_InputFormat.stringFreeStructure();
+    } else {
+      // This more efficient than new Instances(m_InputFormat, 0);
+      m_InputFormat.delete();
+    }
+  }
+
+  /**
+   * @deprecated use <code>setInputFormat(Instances)</code> instead.
+   */
+  public boolean inputFormat(Instances instanceInfo) throws Exception {
+
+    return setInputFormat(instanceInfo);
+  }
+
+  /**
+   * Sets the format of the input instances. If the filter is able to
+   * determine the output format before seeing any input instances, it
+   * does so here. This default implementation clears the output format
+   * and output queue, and the new batch flag is set. Overriders should
+   * call <code>super.setInputFormat(Instances)</code>
+   *
+   * @param instanceInfo an Instances object containing the input instance
+   * structure (any instances contained in the object are ignored - only the
+   * structure is required).
+   * @return true if the outputFormat may be collected immediately
+   * @exception Exception if the inputFormat can't be set successfully 
+   */
+  public boolean setInputFormat(Instances instanceInfo) throws Exception {
+
+    m_InputFormat = instanceInfo.stringFreeStructure();
+    m_InputStringAtts = getStringIndices(instanceInfo);
+    m_OutputFormat = null;
+    m_OutputQueue = new Queue();
+    m_NewBatch = true;
+    return false;
+  }
+
+  /**
+   * @deprecated use <code>getOutputFormat()</code> instead.
+   */
+  public final Instances outputFormat() {
+
+    return getOutputFormat();
+  }
+
+  /**
+   * Gets the format of the output instances. This should only be called
+   * after input() or batchFinished() has returned true. The relation
+   * name of the output instances should be changed to reflect the
+   * action of the filter (eg: add the filter name and options).
+   *
+   * @return an Instances object containing the output instance
+   * structure only.
+   * @exception NullPointerException if no input structure has been
+   * defined (or the output format hasn't been determined yet) 
+   */
+  public final Instances getOutputFormat() {
+
+    if (m_OutputFormat == null) {
+      throw new NullPointerException("No output format defined.");
+    }
+    return new Instances(m_OutputFormat, 0);
+  }
+
+  /**
+   * Input an instance for filtering. Ordinarily the instance is
+   * processed and made available for output immediately. Some filters
+   * require all instances be read before producing output, in which
+   * case output instances should be collected after calling
+   * batchFinished(). If the input marks the start of a new batch, the
+   * output queue is cleared. This default implementation assumes all
+   * instance conversion will occur when batchFinished() is called.
+   *
+   * @param instance the input instance
+   * @return true if the filtered instance may now be
+   * collected with output().
+   * @exception NullPointerException if the input format has not been
+   * defined.
+   * @exception Exception if the input instance was not of the correct 
+   * format or if there was a problem with the filtering.  
+   */
+  public boolean input(Instance instance) throws Exception {
+
+    if (m_InputFormat == null) {
+      throw new NullPointerException("No input instance format defined");
+    }
+    if (m_NewBatch) {
+      m_OutputQueue = new Queue();
+      m_NewBatch = false;
+    }
+    bufferInput(instance);
+    return false;
+  }
+
+  /**
+   * Signify that this batch of input to the filter is finished. If
+   * the filter requires all instances prior to filtering, output()
+   * may now be called to retrieve the filtered instances. Any
+   * subsequent instances filtered should be filtered based on setting
+   * obtained from the first batch (unless the inputFormat has been
+   * re-assigned or new options have been set). This default
+   * implementation assumes all instance processing occurs during
+   * inputFormat() and input().
+   *
+   * @return true if there are instances pending output
+   * @exception NullPointerException if no input structure has been defined,
+   * @exception Exception if there was a problem finishing the batch.
+   */
+  public boolean batchFinished() throws Exception {
+
+    if (m_InputFormat == null) {
+      throw new NullPointerException("No input instance format defined");
+    }
+    flushInput();
+    m_NewBatch = true;
+    return (numPendingOutput() != 0);
+  }
+
+
+  /**
+   * Output an instance after filtering and remove from the output queue.
+   *
+   * @return the instance that has most recently been filtered (or null if
+   * the queue is empty).
+   * @exception NullPointerException if no output structure has been defined
+   */
+  public Instance output() {
+
+    if (m_OutputFormat == null) {
+      throw new NullPointerException("No output instance format defined");
+    }
+    if (m_OutputQueue.empty()) {
+      return null;
+    }
+    Instance result = (Instance)m_OutputQueue.pop();
+    // Clear out references to old strings occasionally
+    if (m_OutputQueue.empty() && m_NewBatch) {
+      if (m_OutputStringAtts.length > 0) {
+        m_OutputFormat = m_OutputFormat.stringFreeStructure();
+      }
+    }
+    return result;
+  }
+  
+  /**
+   * Output an instance after filtering but do not remove from the
+   * output queue.
+   *
+   * @return the instance that has most recently been filtered (or null if
+   * the queue is empty).
+   * @exception NullPointerException if no input structure has been defined 
+   */
+  public Instance outputPeek() {
+
+    if (m_OutputFormat == null) {
+      throw new NullPointerException("No output instance format defined");
+    }
+    if (m_OutputQueue.empty()) {
+      return null;
+    }
+    Instance result = (Instance)m_OutputQueue.peek();
+    return result;
+  }
+
+  /**
+   * Returns the number of instances pending output
+   *
+   * @return the number of instances  pending output
+   * @exception NullPointerException if no input structure has been defined
+   */
+  public int numPendingOutput() {
+
+    if (m_OutputFormat == null) {
+      throw new NullPointerException("No output instance format defined");
+    }
+    return m_OutputQueue.size();
+  }
+
+  /**
+   * Returns whether the output format is ready to be collected
+   *
+   * @return true if the output format is set
+   */
+  public boolean isOutputFormatDefined() {
+
+    return (m_OutputFormat != null);
+  }
+
+  /**
+   * Gets an array containing the indices of all string attributes.
+   *
+   * @param insts the Instances to scan for string attributes. 
+   * @return an array containing the indices of string attributes in
+   * the input structure. Will be zero-length if there are no
+   * string attributes
+   */
+  protected int [] getStringIndices(Instances insts) {
+    
+    // Scan through getting the indices of String attributes
+    int [] index = new int [insts.numAttributes()];
+    int indexSize = 0;
+    for (int i = 0; i < insts.numAttributes(); i++) {
+      if (insts.attribute(i).type() == Attribute.STRING) {
+        index[indexSize++] = i;
+      }
+    }
+    int [] result = new int [indexSize];
+    System.arraycopy(index, 0, result, 0, indexSize);
+    return result;
+  }
+  
+  /**
+   * Filters an entire set of instances through a filter and returns
+   * the new set. 
+   *
+   * @param data the data to be filtered
+   * @param filter the filter to be used
+   * @return the filtered set of data
+   * @exception Exception if the filter can't be used successfully
+   */
+  public static Instances useFilter(Instances data,
+				    Filter filter) throws Exception {
+    /*
+    System.err.println(filter.getClass().getName() 
+                       + " in:" + data.numInstances());
+    */
+    for (int i = 0; i < data.numInstances(); i++) {
+      filter.input(data.instance(i));
+    }
+    filter.batchFinished();
+    Instances newData = filter.getOutputFormat();
+    Instance processed;
+    while ((processed = filter.output()) != null) {
+      newData.add(processed);
+    }
+
+    /*
+    System.err.println(filter.getClass().getName() 
+                       + " out:" + newData.numInstances());
+    */
+    return newData;
+  }
+
+  /**
+   * Method for testing filters.
+   *
+   * @param argv should contain the following arguments: <br>
+   * -i input_file <br>
+   * -o output_file <br>
+   * -c class_index <br>
+   * or -h for help on options
+   * @exception Exception if something goes wrong or the user requests help on
+   * command options
+   */
+  public static void filterFile(Filter filter, String [] options) 
+    throws Exception {
+
+    boolean debug = false;
+    Instances data = null;
+    Reader input = null;
+    PrintWriter output = null;
+    boolean helpRequest;
+
+    try {
+       helpRequest = Utils.getFlag('h', options);
+
+      if (Utils.getFlag('d', options)) {
+	debug = true;
+      }
+      String infileName = Utils.getOption('i', options);
+      String outfileName = Utils.getOption('o', options); 
+      String classIndex = Utils.getOption('c', options);
+      
+      if (filter instanceof OptionHandler) {
+	((OptionHandler)filter).setOptions(options);
+      }
+
+      Utils.checkForRemainingOptions(options);
+      if (helpRequest) {
+	throw new Exception("Help requested.\n");
+      }
+      if (infileName.length() != 0) {
+	input = new BufferedReader(new FileReader(infileName));
+      } else {
+	input = new BufferedReader(new InputStreamReader(System.in));
+      }
+      if (outfileName.length() != 0) {
+	output = new PrintWriter(new FileOutputStream(outfileName));
+      } else { 
+	output = new PrintWriter(System.out);
+      }
+
+      data = new Instances(input, 1);
+      if (classIndex.length() != 0) {
+	if (classIndex.equals("first")) {
+	  data.setClassIndex(0);
+	} else if (classIndex.equals("last")) {
+	  data.setClassIndex(data.numAttributes() - 1);
+	} else {
+	  data.setClassIndex(Integer.parseInt(classIndex) - 1);
+	}
+      }
+    } catch (Exception ex) {
+      String filterOptions = "";
+      // Output the error and also the valid options
+      if (filter instanceof OptionHandler) {
+	filterOptions += "\nFilter options:\n\n";
+	Enumeration enum = ((OptionHandler)filter).listOptions();
+	while (enum.hasMoreElements()) {
+	  Option option = (Option) enum.nextElement();
+	  filterOptions += option.synopsis() + '\n'
+	    + option.description() + "\n";
+	}
+      }
+
+      String genericOptions = "\nGeneral options:\n\n"
+	+ "-h\n"
+	+ "\tGet help on available options.\n"
+	+ "\t(use -b -h for help on batch mode.)\n"
+	+ "-i <file>\n"
+	+ "\tThe name of the file containing input instances.\n"
+	+ "\tIf not supplied then instances will be read from stdin.\n"
+	+ "-o <file>\n"
+	+ "\tThe name of the file output instances will be written to.\n"
+	+ "\tIf not supplied then instances will be written to stdout.\n"
+	+ "-c <class index>\n"
+	+ "\tThe number of the attribute to use as the class.\n"
+	+ "\t\"first\" and \"last\" are also valid entries.\n"
+	+ "\tIf not supplied then no class is assigned.\n";
+
+      throw new Exception('\n' + ex.getMessage()
+			  + filterOptions+genericOptions);
+    }
+    
+    if (debug) {
+      System.err.println("Setting input format");
+    }
+    boolean printedHeader = false;
+    if (filter.setInputFormat(data)) {
+      if (debug) {
+	System.err.println("Getting output format");
+      }
+      output.println(filter.getOutputFormat().toString());
+      printedHeader = true;
+    }
+    
+    // Pass all the instances to the filter
+    while (data.readInstance(input)) {
+      if (debug) {
+	System.err.println("Input instance to filter");
+      }
+      if (filter.input(data.instance(0))) {
+	if (debug) {
+	  System.err.println("Filter said collect immediately");
+	}
+	if (!printedHeader) {
+	  throw new Error("Filter didn't return true from setInputFormat() "
+			  + "earlier!");
+	}
+	if (debug) {
+	  System.err.println("Getting output instance");
+	}
+	output.println(filter.output().toString());
+      }
+      data.delete(0);
+    }
+    
+    // Say that input has finished, and print any pending output instances
+    if (debug) {
+      System.err.println("Setting end of batch");
+    }
+    if (filter.batchFinished()) {
+      if (debug) {
+	System.err.println("Filter said collect output");
+      }
+      if (!printedHeader) {
+	if (debug) {
+	  System.err.println("Getting output format");
+	}
+	output.println(filter.getOutputFormat().toString());
+      }
+      if (debug) {
+	System.err.println("Getting output instance");
+      }
+      while (filter.numPendingOutput() > 0) {
+	output.println(filter.output().toString());
+	if (debug){
+	  System.err.println("Getting output instance");
+	}
+      }
+    }
+    if (debug) {
+      System.err.println("Done");
+    }
+    
+    if (output != null) {
+      output.close();
+    }
+  }
+
+  /**
+   * Method for testing filters ability to process multiple batches.
+   *
+   * @param argv should contain the following arguments:<br>
+   * -i (first) input file <br>
+   * -o (first) output file <br>
+   * -r (second) input file <br>
+   * -s (second) output file <br>
+   * -c class_index <br>
+   * or -h for help on options
+   * @exception Exception if something goes wrong or the user requests help on
+   * command options
+   */
+  public static void batchFilterFile(Filter filter, String [] options) 
+    throws Exception {
+
+    Instances firstData = null;
+    Instances secondData = null;
+    Reader firstInput = null;
+    Reader secondInput = null;
+    PrintWriter firstOutput = null;
+    PrintWriter secondOutput = null;
+    boolean helpRequest;
+    try {
+      helpRequest = Utils.getFlag('h', options);
+
+      String fileName = Utils.getOption('i', options); 
+      if (fileName.length() != 0) {
+	firstInput = new BufferedReader(new FileReader(fileName));
+      } else {
+	throw new Exception("No first input file given.\n");
+      }
+
+      fileName = Utils.getOption('r', options); 
+      if (fileName.length() != 0) {
+	secondInput = new BufferedReader(new FileReader(fileName));
+      } else {
+	throw new Exception("No second input file given.\n");
+      }
+
+      fileName = Utils.getOption('o', options); 
+      if (fileName.length() != 0) {
+	firstOutput = new PrintWriter(new FileOutputStream(fileName));
+      } else {
+	firstOutput = new PrintWriter(System.out);
+      }
+      
+      fileName = Utils.getOption('s', options); 
+      if (fileName.length() != 0) {
+	secondOutput = new PrintWriter(new FileOutputStream(fileName));
+      } else {
+	secondOutput = new PrintWriter(System.out);
+      }
+      String classIndex = Utils.getOption('c', options);
+
+      if (filter instanceof OptionHandler) {
+	((OptionHandler)filter).setOptions(options);
+      }
+      Utils.checkForRemainingOptions(options);
+      
+      if (helpRequest) {
+	throw new Exception("Help requested.\n");
+      }
+      firstData = new Instances(firstInput, 1);
+      secondData = new Instances(secondInput, 1);
+      if (!secondData.equalHeaders(firstData)) {
+	throw new Exception("Input file formats differ.\n");
+      }
+      if (classIndex.length() != 0) {
+	if (classIndex.equals("first")) {
+	  firstData.setClassIndex(0);
+	  secondData.setClassIndex(0);
+	} else if (classIndex.equals("last")) {
+	  firstData.setClassIndex(firstData.numAttributes() - 1);
+	  secondData.setClassIndex(secondData.numAttributes() - 1);
+	} else {
+	  firstData.setClassIndex(Integer.parseInt(classIndex) - 1);
+	  secondData.setClassIndex(Integer.parseInt(classIndex) - 1);
+	}
+      }
+    } catch (Exception ex) {
+      String filterOptions = "";
+      // Output the error and also the valid options
+      if (filter instanceof OptionHandler) {
+	filterOptions += "\nFilter options:\n\n";
+	Enumeration enum = ((OptionHandler)filter).listOptions();
+	while (enum.hasMoreElements()) {
+	  Option option = (Option) enum.nextElement();
+	  filterOptions += option.synopsis() + '\n'
+	    + option.description() + "\n";
+	}
+      }
+
+      String genericOptions = "\nGeneral options:\n\n"
+	+ "-h\n"
+	+ "\tGet help on available options.\n"
+	+ "-i <filename>\n"
+	+ "\tThe file containing first input instances.\n"
+	+ "-o <filename>\n"
+	+ "\tThe file first output instances will be written to.\n"
+	+ "-r <filename>\n"
+	+ "\tThe file containing second input instances.\n"
+	+ "-s <filename>\n"
+	+ "\tThe file second output instances will be written to.\n"
+	+ "-c <class index>\n"
+	+ "\tThe number of the attribute to use as the class.\n"
+	+ "\t\"first\" and \"last\" are also valid entries.\n"
+	+ "\tIf not supplied then no class is assigned.\n";
+
+      throw new Exception('\n' + ex.getMessage()
+			  + filterOptions+genericOptions);
+    }
+    boolean printedHeader = false;
+    if (filter.setInputFormat(firstData)) {
+      firstOutput.println(filter.getOutputFormat().toString());
+      printedHeader = true;
+    }
+    
+    // Pass all the instances to the filter
+    while (firstData.readInstance(firstInput)) {
+      if (filter.input(firstData.instance(0))) {
+	if (!printedHeader) {
+	  throw new Error("Filter didn't return true from setInputFormat() "
+			  + "earlier!");
+	}
+	firstOutput.println(filter.output().toString());
+      }
+      firstData.delete(0);
+    }
+    
+    // Say that input has finished, and print any pending output instances
+    if (filter.batchFinished()) {
+      if (!printedHeader) {
+	firstOutput.println(filter.getOutputFormat().toString());
+      }
+      while (filter.numPendingOutput() > 0) {
+	firstOutput.println(filter.output().toString());
+      }
+    }
+    
+    if (firstOutput != null) {
+      firstOutput.close();
+    }    
+    printedHeader = false;
+    if (filter.isOutputFormatDefined()) {
+      secondOutput.println(filter.getOutputFormat().toString());
+      printedHeader = true;
+    }
+    // Pass all the second instances to the filter
+    while (secondData.readInstance(secondInput)) {
+      if (filter.input(secondData.instance(0))) {
+	if (!printedHeader) {
+	  throw new Error("Filter didn't return true from"
+			  + " isOutputFormatDefined() earlier!");
+	}
+	secondOutput.println(filter.output().toString());
+      }
+      secondData.delete(0);
+    }
+    
+    // Say that input has finished, and print any pending output instances
+    if (filter.batchFinished()) {
+      if (!printedHeader) {
+	secondOutput.println(filter.getOutputFormat().toString());
+      }
+      while (filter.numPendingOutput() > 0) {
+	secondOutput.println(filter.output().toString());
+      }
+    }
+    if (secondOutput != null) {
+      secondOutput.close();
+    }
+  }
+
+  /**
+   * Main method for testing this class.
+   *
+   * @param argv should contain arguments to the filter: use -h for help
+   */
+  public static void main(String [] args) {
+    
+    try {
+      if (args.length == 0) {
+        throw new Exception("First argument must be the class name of a Filter");
+      }
+      String fname = args[0];
+      Filter f = (Filter)Class.forName(fname).newInstance();
+      args[0] = "";
+      if (Utils.getFlag('b', args)) {
+	Filter.batchFilterFile(f, args);
+      } else {
+	Filter.filterFile(f, args);
+      }
+    } catch (Exception ex) {
+      ex.printStackTrace();
+      System.out.println(ex.getMessage());
+    }
+  }
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/mg/COPYING
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/COPYING	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl/packages/mg/INSTALL
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/INSTALL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/INSTALL	(revision 21112)
@@ -0,0 +1,176 @@
+Basic Installation
+==================
+
+   These are generic installation instructions.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, a file
+`config.cache' that saves the results of its tests to speed up
+reconfiguring, and a file `config.log' containing compiler output
+(useful mainly for debugging `configure').
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If at some point `config.cache'
+contains results you don't want to keep, you may remove or edit it.
+
+   The file `configure.in' is used to create `configure' by a program
+called `autoconf'.  You only need `configure.in' if you want to change
+it or regenerate `configure' using a newer version of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.  If you're
+     using `csh' on an old version of System V, you might need to type
+     `sh ./configure' instead to prevent `csh' from trying to execute
+     `configure' itself.
+
+     Running `configure' takes awhile.  While running, it prints some
+     messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+   Some systems require unusual options for compilation or linking that
+the `configure' script does not know about.  You can give `configure'
+initial values for variables by setting them in the environment.  Using
+a Bourne-compatible shell, you can do that on the command line like
+this:
+     CC=c89 CFLAGS=-O2 LIBS=-lposix ./configure
+
+Or on systems that have the `env' program, you can do it like this:
+     env CPPFLAGS=-I/usr/local/include LDFLAGS=-s ./configure
+
+Compiling For Multiple Architectures
+====================================
+
+   You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you must use a version of `make' that
+supports the `VPATH' variable, such as GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   If you have to use a `make' that does not supports the `VPATH'
+variable, you have to compile the package for one architecture at a time
+in the source code directory.  After you have installed the package for
+one architecture, use `make distclean' before reconfiguring for another
+architecture.
+
+Installation Names
+==================
+
+   By default, `make install' will install the package's files in
+`/usr/local/bin', `/usr/local/man', etc.  You can specify an
+installation prefix other than `/usr/local' by giving `configure' the
+option `--prefix=PATH'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+give `configure' the option `--exec-prefix=PATH', the package will use
+PATH as the prefix for installing programs and libraries.
+Documentation and other data files will still use the regular prefix.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+   Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+   There may be some features `configure' can not figure out
+automatically, but needs to determine by the type of host the package
+will run on.  Usually `configure' can figure that out, but if it prints
+a message saying it can not guess the host type, give it the
+`--host=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name with three fields:
+     CPU-COMPANY-SYSTEM
+
+See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the host type.
+
+   If you are building compiler tools for cross-compiling, you can also
+use the `--target=TYPE' option to select the type of system they will
+produce code for and the `--build=TYPE' option to select the type of
+system on which you are compiling the package.
+
+Sharing Defaults
+================
+
+   If you want to set default values for `configure' scripts to share,
+you can create a site shell script called `config.site' that gives
+default values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Operation Controls
+==================
+
+   `configure' recognizes the following options to control how it
+operates.
+
+`--cache-file=FILE'
+     Use and save the results of the tests in FILE instead of
+     `./config.cache'.  Set FILE to `/dev/null' to disable caching, for
+     debugging `configure'.
+
+`--help'
+     Print a summary of the options to `configure', and exit.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`--version'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`configure' also accepts some other, not widely useful, options.
+
Index: /main/tags/3.01/gsdl/packages/mg/INSTALL.mg
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/INSTALL.mg	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/INSTALL.mg	(revision 21112)
@@ -0,0 +1,38 @@
+                     BUILDING AND INSTALLATION OF MG
+                     ===============================
+ 
+This file explains how to build the mg system.
+ 
+(1) Follow the general GNU install instructions in the INSTALL file.
+    NOTE: this procedure has changed from version mg-1.1 which did
+    not use a GNU configure script.
+    e.g.
+      cd ~/mg-1.2
+      ./configure --prefix=`pwd`
+      make
+      make check
+      make install
+    Report portability problems to tes@kbs.citri.edu.au .
+ 
+(2) You will need to set the environment variable MGDATA which 
+    indicates where the mg data files will be created. You will also
+    need to create the directory you specify.
+ 
+    e.g.
+        setenv MGDATA ~/mgdata
+	mkdir ~/mgdata
+
+(4) You may also need to set the environment variable MGSAMPLE which
+    is used by mg_get and indicates where the sample data included
+    with this package is located.
+    If this is not specified it defaults to ``./SampleData''
+
+(4) Included with mg is an X11 interface to mgquery called xmg. 
+    Xmg is a ``wish'' script which uses the Tcl/Tk packages available
+    by anonymous ftp from allspice.berkeley.edu [128.32.150.27].
+ 
+(5) Another environment variable that you may wish to set is
+    MGIMAGEVIEWER, this variable sets the image viewer to be used
+    to display images. The image viewer must take the image from stdin.
+    If this is not specified it defaults to ``xv -''.
+ 
Index: /main/tags/3.01/gsdl/packages/mg/MODIFICATIONS
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,1023 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Similarity variants
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  Can only use one type of similarity measure - the
+  standard cosine measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+  The standard measure can be broken up into 7 components.
+  The 7 components are  
+  Each one of these components has a number of alternatives.
+  The overall measure, S_qd, can also be altered.
+  Thus the particular similarity measure used can be specified
+  by an 8 dimensional vector.
+  What is desired is to be able to specify to mgquery an option
+  and a 8-digit string representing this vector (assuming that
+  any one alternative can have at most 9 (not using zero) variants).
+SOLUTION
+  The programs which had to be modified were:
+  (i) mgquery,
+  (ii) mg_weights_build.
+  The other mg programs in existence store the text, indexing info,
+  and the basic statistics such as N, n, ft, fdt.  
+  Other programs which had to be created were:
+  (i) mg_fmd_build,
+  (ii) mg_wt_build.
+  Mg_fmd_build will create the file to store the f_md statistic, 
+  where f_md is the largest (maximum) f_dt of any term in document, d.
+  Mg_wt_build will create the file to store the w_t primitive.
+  It only creates this for the w_t variants 6-9 which would require
+  extra passes of invf at query time if they were not stored here.
+  For details on similarity changes for mgquery and mg_weights_build,
+  please see the other modification entries.  
+FILES
+ *   mg_fmd_build.c
+ *   mg_wt_build.c
+ *   build_lib.c, build_lib.h
+*************************************************************
+TITLE
+  Similarity variants for mgquery
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mgquery" needs to be altered to allow modification of
+  the similarity measure. 
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+
+  Most of the similarity measures, Sqd, are of the 
+  form:   Aqd
+         -----
+          Bqd
+  where Bqd is an expression involving Wd and possibly Wq,
+  where Aqd is a sum over the common document/query terms
+  of w_qt and w_dt.
+  Building of Aqd
+  ===============
+  The calculation of Aqd is done in the file build_Aqd.c .
+  The functions for doing this used to be in the file invf_get.c .
+  Build_Aqd.c contains 4 different functions for building Aqd, each
+  of them building a different data structure:
+    (i) Array, (ii) Splay Tree, (iii) Hash Table, (iv) List Table.
+  Each of these routines seems to have been construction by modifications
+  to duplicated code. This is often the easiest way to construct variants
+  but is quite difficult to maintain consistency. 
+  As the aim of the exercise was to try out different sim. measures for
+  retrieval effectiveness, I only modified the code that constructed
+  an array. This routine was called "CosineDecode"; I changed it to
+  "build_Aqd_Array." This change reflects the fact that we are only 
+  calculating Aqd and this need not be used for the Cosine measure.
+  The other routines: "CosineDecodeSplay," "CosineDecodeHash," and
+  "CosineDecodeList" have been left unaltered - they need to be updated
+  in the future which would be best be done by abstracting out common code.
+  By the stage of building Aqd, the query terms have been looked up in
+  the inverted file dictionary and put into a list.
+  This list of common terms is traversed to lookup the corresponding
+  invf entries. Before the invf entry is processed, all query and term
+  relevant statistics and primitive quantities are calculated.
+  For example, fqt, ft, wt, rqt, wqt, Wq-partial-sum.
+  To save unnecessary calculations, there is a test for each value
+  to see whether it is needed e.g. "if (sim->variant_needs & NEEDS_wt) ... ".  
+  Aside: Variant Needs
+  --------------------
+  The idea behind the "variant_needs" field is to be able to have all
+  the code in the one place for each possible variant and this code would
+  get the information at the correct time/place only when it is needed.
+  The overhead is a "bit-and" and "test" for each component.
+  The important concern though, is that the "variant_needs" must be
+  accurate i.e. it should be carefully maintained.
+  Each possible need is stored as a bit position in a constant/macro
+  of the form "NEEDS_component" e.g. NEEDS_wt, NEEDS_nt 
+  More recently (Jan/95), I have found that the _need_ing of a component
+  may be relevant to a particular purpose, that is, it may be needed in
+  one situation and not another. This was the case for Wd. 
+  For rdt#6, Wd was needed, however, it might not be needed for Bqd, the
+  denominator of Sqd. So I changed from NEEDS_Wd to NEEDS_Wd_for_Sqd and
+  NEEDS_Wd_for_rdt. 
+  --------------------
+  To handle the different calculations of the variants, I wrote macros
+  based around a switch statement. All these macros are stored in the file
+  "similarity.h" . The point of doing this is to centralise most of the changes
+  and cut down on the number of files which have to be altered if a new
+  way of calculating a primitive is to be added.  
+  This is achieved by having a data record called, "Similarity_variant" whose
+  fields includes all the statistics and the similarity primitives.
+  So the standard procedure is to see if something is needed and if so, then
+  extract it from an mg file or calculate it using a macro - most of the
+  input and output to the macro is done via the "Similarity_variant" structure.
+  As well as calculating Aqd, it is also a convenient place to calculate Wq.
+  Previously, for Cosine measure, Wq was not needed because each Aqd was directly
+  divided by Wq and thus would not change the ordering. However, there are
+  some Sqd's which divide by a sum involving Wq and thus need Wq. 
+  Calculating Sqd and Ordering Documents
+  ====================================== 
+  The file query.ranked.c contains the code for calculating Sqd using the
+  approximate Wd in order to do the ranking of the documents.
+  The mg-1 file was cleaned up and modified slightly.
+  All the heap data structures and routines were taken out and placed in 
+  their own file, heap_weights.c/.h .
+  The major components/steps in the ranking process were abstracted into
+  macros and functions:
+    calc_MaxParas
+    insert_heap_entry
+    insert_greater_heap_entry
+    approx_guided_insert
+    fill_initial_heap
+    add_heap_remainders
+    change_heap_to_exact
+    add_remainder_exact_weights
+    Make_Exact_Root
+    build_doc_list
+  Aside: Zeroing of an Aqd Element
+  -------------------------
+  One interesting change concerned the use of zeroing out an Aqd element
+  so as to mark it as being used in the heap.
+  In the heap, the approx. Sqd is stored based on Aqd/Wd-approx.
+  Later when Aqd is required, it can be extracted from the approx. Sqd
+  by muliplication of Wd-approx i.e. Aqd = Sqd-approx * Wd-approx.
+  This, however, is not always possible for the various Sqd variants.
+  So, instead of zeroing out Aqd, I decided just to make it negative.
+  -------------------------
+  In some Sqd cases, Wd and Wd-approx. are not required. In which case,
+  step 3 which calls on "change_heap_to_exact", "Heap_Build" and
+  "add_remainder_exact_weights" is not required.
+ 
+FILES
+ *    build_Aqd.c
+ *    query.ranked.c
+ *    similarity.c/.h (in libmg)
+ *    heap_weights.c/.h
+ *    backend.c/.h
+*************************************************************
+TITLE
+  Similarity variants for mg_weights_build
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mg_weights_build" needs to be altered to allow modification of
+  the similarity measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+  The weight files which are generated for a particular similarity
+  measure have their names extended by a suffix.
+  In the case of Wd#1 no weights are generated.
+  In the case of the standard cosine weights, Wd#2, a 3 letter suffix
+  is used to represent .
+  In the case of the other Wd variants, a one letter suffix is used
+  to represent which Wd variant it is.
+  In each case, the variant input (e.g. -q 22222222) should be the whole
+  similarity variant string and the relevant fields will be extracted out.
+  This is done for consistency in code and interface. 
+  The code is fairly similar to the original.
+  A dependency check has been added so that the dates of files and the
+  type of needed files is verified before building.
+  The dependencies include, invf dictionary, invf index, invf, fmd and wt files.
+  The major change here, is the possible use of fmd and/or wt files.
+  Later when I was having to write mg_fmd_build and mg_wt_build,
+  I decided to abstract out some macros, namely:
+    Get_ft, Get_ft_Ft, loop_invf_entry,
+  which were put into "build_lib.c" .
+  "loop_invf_entry" takes a function/macro name as a parameter and applies
+  it to the sim record (with field fdt set), current doc number and modifies
+  the return value. 
+FILES
+ *   mg_weights_build.c
+ *   build_lib.c
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  Boolean tree optimisations
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  (i) tes@kbs.citri.edu.au - 28/Sep/94
+  (ii) tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  (i) tes@kbs.citri.edu.au - 18/Oct/94
+  (ii) tes@kbs.citri.edu.au - 21/Dec/94
+CLAIM
+  The initial prompt for investigating the optimisation of
+  boolean queries is noted in the mg-1 mod14.txt. 
+  The code for optimising seemed to have a number of faults.
+PROBLEM
+  Boolean optimisation was unreliable.
+SOLUTION
+  Initially (in case (i) above, see mg-1/mod14.txt), I rewrote 
+  all the boolean tree and optimising code. I converted the boolean
+  expression into DNF. I did this after reading some notes about 
+  the steps involved in optimisation and they suggested standardising
+  in some normal form. I thought that DNF would be appropriate so that
+  all the terms are converted to be part of "and" expressions and be
+  evaluated quickly using skipping.
+  This, however, can suffer quite badly if the distributive law is
+  applied to often and the query expands in size. If there was
+  some sort of cacheing of invf entries, then it might not be so
+  bad otherwise there is quite an overhead on reading the same
+  invf entry more than once.
+
+  As it happens, CNF queries are reasonably common, where the user
+  queries with a conjunction of disjunctions of similes:
+    e.g. (car | automobile | vehicle) & (fast | quick | speedy)
+  This sort of CNF query expands a hell of a lot !
+  So after speaking with Justin who wanted to benchmark Atlas with Mg on
+  these sort of queries, I looked up the MG book for other ideas.
+
+  The method that I implemented was the following:
+
+  -----------------------------------------------------------------
+  Steps of tree modifications:
+    Gets literals by pushing the nots down, detecting T/F at leaves
+    and collapsing the tree by detecting  'and' of 'and's and 'or' of
+    'or's.
+    Next it looks at the or nodes and if all the children are terms
+    then it marks the or-node as such.
+    Finally, the or-term-nodes are sorted by using the sum of their
+    ft's for comparison.
+  Steps at query evaluation:
+    If it comes across an 'and' of 'or-terms' then the evaluation is
+    done noting the distributive law.
+    I.e. a & (b | c | d) = (a & b) | (a & c) | (a & d)
+    Assuming 'a' is the c-set of documents.
+    All of 'a' is tested against 'b' and matching ones are marked.
+    Next, all the unmarked members of 'a' are tested against 'c'.
+    Likewise for 'd'.
+    Now all the marked members of our c-set are kept.
+    When we do the testing, we can use the skipping in the invf entries.
+  -----------------------------------------------------------------
+
+
+  After doing this, I added the choice of which type of optimisation
+  the user wanted by adding query-environment-variable, "optimise_type".
+  Type 0 = no parse tree modification. 
+  Type 1 = Or-term recognition and CNF query evaluation optimisation. 
+  Type 2 = Put into DNF form. [generally not recommended]
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+ *   query_env.c
+ *   invf_get.c [GetDocsOp]
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  TREC DocNo file   
+APPLICATION
+  mg-2
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 26/May/95
+FIX
+  tes@kbs.citri.edu.au - 26/May/95
+CLAIM
+  MG has problems dealing with trec docnos for trec disk 3.
+PROBLEM
+  Trec DocNos file didn't have a wide enough field
+  to handle disk 3. 
+SOLUTION
+  Allow different width fields for file.
+  It is still fixed width but a number in the header
+  says how wide the field is.
+FILES
+ *   passes/mg.special.c
+ *   query/mgquery.c
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+   Autoconfiguring mg-1  
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  many people - 94/95
+FIX
+  tes@kbs.citri.edu.au - Aug/95
+CLAIM
+  Portability is limited by setting up c-macros just for particular
+  machines and operating systems.
+  People had to make changes for HP, Next, Linux, Dec Alpha, ...
+PROBLEM
+  Porting was only targetting at the machines that the author had
+  access to.
+SOLUTION
+  Use GNU's autoconfigure program.
+  This allows checking of the systems features/characteristics.
+  It also allows some checking for specific machines/OS - although
+  I have not utilised this option.
+  I used GNU's tar-1.11.8 as an example to base my changes on.
+  I also used autoscan to generate the initial "configure.in".
+  The "Makefile.in"s were done very similarly to GNU tar's.
+  The "config.h" and "sysfuncs.h" files were scrapped and 
+  rewritten. The new "config.h" file is generated by the configure
+  script - it contains all the #define's for the system features.
+  The "sysfuncs.h" file wraps up a number of system headers. 
+  For example, some systems use , while some use ;
+  which one is included is decided in "sysfuncs.h".
+  I have also used GNU tar's use of ansi2knr in its Makefiles.
+  This should hopefully allow the package to work on a system with
+  only a K&R C compiler.
+  However, there are probably problems with what I have done.
+  I am concerned about <stdarg.h> for example.
+  I also noticed that "ansi2knr" require function definitions as
+  the GNU coding style recommends ie. with function name the first
+  string on the line. This prompted me to run all the package's code
+  thru GNU's indent.
+  Setting up the configure changes is difficult. It really seems
+  necessary to try the package out on many target machines so one
+  can know what is necessary.
+  A simple check target for the main Makefile has been written.
+  It is used to see if the installation is working - it does
+  not test much of the functionality of mg.
+  It does cmp's on data files and diff's on query/result files.
+FILES
+  Most of the files in the distribution.
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/gsdl/packages/mg/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,163 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+#SUBDIRS = lib src/text src/images src/scripts
+SUBDIRS = lib src/text
+
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall: ../../config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+check: all
+	echo making $@ in ./test directory
+	(cd ./test && $(MAKE) all) || exit 1
+	./test/testmg `pwd`
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+# the toplevel Makefile already does our subdirs.
+#distclean: distclean-recursive distclean-local
+#	rm -f config.status
+distclean:
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm -f config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive \
+maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile stamp-h
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in sysfuncs.h
+
+dist: $(DISTFILES)
+#-----------------------------------------------------------------
+	#create distribution directory
+	-rm -rf $(distdir)
+	mkdir $(distdir)
+#
+	#link in local distribution files
+	ln $(DISTFILES) $(distdir)
+#
+	#link in SampleData
+	mkdir $(distdir)/SampleData
+	(cd SampleData; $(MAKE) "distdir=../$(distdir)/SampleData" dist)
+	# link in davinci stuff
+	mkdir $(distdir)/SampleData/davinci
+	(cd SampleData/davinci; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/davinci" dist)
+	# link in bible stuff
+	mkdir $(distdir)/SampleData/bible
+	(cd SampleData/bible; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/bible" dist)
+#
+	#link in tests
+	mkdir $(distdir)/test
+	(cd test; $(MAKE) "distdir=../$(distdir)/test" dist)
+#
+	#link in docs directory
+	mkdir $(distdir)/docs
+	(cd docs; $(MAKE) "distdir=../$(distdir)/docs" dist)
+#
+	#link in lib directory
+	mkdir $(distdir)/lib
+	(cd lib; $(MAKE) "distdir=../$(distdir)/lib" dist)
+#
+	#link in src distribution files
+	mkdir $(distdir)/src
+	for dir in src/text src/images src/scripts; do \
+	  echo makeing $@ in $$dir ; \
+	  mkdir $(distdir)/$$dir; \
+	  (cd $$dir; $(MAKE) "distdir=../../$(distdir)/$$dir" $@) ; \
+	done
+#
+	# archive and compress
+	tar -chf - $(distdir) | gzip -9 > $(distname).tar.gz
+	tar -chf - $(distdir) | compress > $(distname).tar.Z
+#
+	# cleanup
+	rm -rf $(distdir)
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl/packages/mg/README
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/README	(revision 21112)
@@ -0,0 +1,156 @@
+                 MG INFORMATION RETRIEVAL SYSTEM
+                 ===============================
+
+The MG system is a suite of programs for compressing and indexing text
+and images. Most of the functionality implemented in the suite is as
+described in the book ``Managing Gigabytes:  Compressing and Indexing
+Documents and Images'', I.H. Witten, A.  Moffat, and T.C. Bell; Van
+Nostrand Reinhold, New York, 1994, ISBN 0-442-01863-0; US $54.95; call
+1 (800) 544-0550 to order.
+
+These features include:
+
+-- text compression using a Huffman-coded semi-static word-based scheme
+-- two-level context-based compression of bi-level images
+-- FELICS lossless compression of gray-scale images
+-- combined lossy/lossless compression for textual images
+-- indexing algorithms for large volumes of text in limited main memory
+-- index compression
+-- a retrieval system that processes Boolean and ranked queries
+-- an X windows interface to the retrieval system
+
+As one example, a collection of 2 Gb of text (1,700,000 documents) can
+be indexed (on a SPARC 10 Model 512) in about four hours and compressed
+in a further four hours to make a database that in total occupies less
+than 800 Mb, or 40% of the original size. This includes a full index to
+every word and number in the original text. Boolean queries such as
+``managing AND gigabytes'' run in a few seconds, and ranked queries of
+30--50 terms are evaluated in 10--30 seconds.
+
+Details of these methods and further performance results appear in the
+MG book.
+
+The MG system comes with ABSOLUTELY NO WARRANTY; for details see the
+file COPYING.
+
+Instructions on how to build and install mg are in the file INSTALL.
+
+
+** CHANGES FROM BOOK
+
+For copyright reasons the stemmer used in this distribution of MG is
+not the same as the one illustrated in Figure 3.8 on page 108 of the MG
+book.  This means that the numbers generated by the command ``mgstat
+alice'' will not match those numbers in Figure A.1 on page 394.
+Another stemmer was initially written as a simple stopgap for version
+1.0.  That stemmer has been replaced by a stemmer based on the Lovin's
+stemming algorithm for mg-1.1.
+
+The output format of ``mgstat'' has changed since Figure A.1 (page 394)
+was prepared. The same information is displayed but formatted
+differently.
+
+** MG VERSIONS
+
+The current version is mg-1.2, September 1995. The changes from earlier 
+versions are listed in the file MODIFICATIONS. This can be accessed
+with mg by building a database using ``mgbuild mods'' and can also be
+accessed from the mg web page (see below). 
+
+The mg-1.2 extensions include:
+
+-- Source modifications for use of GNU's autoconf.
+
+The mg-1.1 extensions include:
+
+-- A new highlighting mode.
+   The output mode ``hilite'' will highlight the query terms in the
+   retrieved text documents. The variable ``hilite_style'' can be set
+   to ``bold'' or ``underline''. It works best with the pager
+   ``less''.  A .mgrc to use would include:
+	.set pager less
+	.set mode hilite
+	.set hilite_style bold
+
+-- A web site containing manual pages, documentation, and a
+   mgquery demo page (utilising cgi scripts). 
+   See: http://www.kbs.citri.edu.au/mg
+   One of these pages ``about_mg.html'' is included in this
+   distribution.
+
+-- A revised mg_get script which uses a .mg_getrc file to map
+   specific collection names to filter types.  (Modifications by Bruce
+   McKenzie). See mg_get.1 for more details.
+
+-- Code to perform merging of existing databases. This code
+   was created by Shane Hudson and is documented in the mgmerge.README
+   file found in the docs subdirectory.  This code is maintained by
+   Shane Hudson (shane@cosc.canterbury.ac.nz).
+
+-- Revised man pages, including some new entries (thanks to Nelson
+   Beebe). See mg.1, mgintro.1, mgintro++.1.
+
+-- A real (rather than toy) stemmer.
+
+** PORTABILITY
+
+Please refer to "README.port".
+
+** CREDITS
+ 
+The MG development is largely the result of research collaboration
+between:
+
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Ian Witten             <ihw@cs.waikato.ac.nz>
+        Alistair Moffat        <alistair@cs.mu.oz.au>
+        Justin Zobel           <jz@cs.rmit.oz.au>
+
+The bulk of the programming work has been carried out by:
+
+        Stuart Inglis          <singlis@cs.waikato.ac.nz>
+        Craig Nevill-Manning   <cgn@cs.waikato.ac.nz>
+        Neil Sharman           <neil@kbs.citri.edu.au>
+        Tim Shimmin            <tes@kbs.citri.edu.au>
+
+In addition to these, the following people have contributed to the
+development of the MG software:
+
+        Lachlan Andrew         <lha@kbs.citri.edu.au>
+        Gary Eddy              <gary@cs.mu.oz.au>
+        Hugh Emberson          <hugh@cosc.canterbury.ac.nz>
+        Kerry Guise            <kerry@cs.waikato.ac.nz>
+	Shane Hudson           <shane@cosc.canterbury.ac.nz>
+        Linh Huynh             <linh@kbs.citri.edu.au>
+        Bohdan S. Majewski     <bohdan@cs.uq.oz.au>
+        Bruce McKenzie         <bruce@cosc.canterbury.ac.nz>
+
+In addition to these, the following people have submitted bug reports
+and suggestions/fixes:
+
+        Rex Barzee             <rex@python.cs.byu.edu>
+        Tim A.H. Bell          <bhat@cs.mu.oz.au>
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Nelson Beebe           <beebe@math.utah.edu>
+	Rodney Brown           <RBrown@cmutual.com.au>
+        Rok Sosic              <R.Sosic@cit.gu.edu.au>
+        Carl Staelin           <staelin@cello.hpl.hp.com>
+
+Development of the MG system was supported by the Australian Research
+Council; the Universities of Melbourne, Waikato, Canterbury, and
+Calgary; RMIT; and the Collaborative Information Technology Research
+Institute (Melbourne).
+
+** BUG REPORTS
+
+Send bug reports to <alistair@cs.mu.oz.au> and <tes@kbs.citri.edu.au>.
+Back-traces from gdb are always welcome but not mandatory :-)
+
+** FURTHER READING
+
+A bibliography of MG related research work appears in the files
+MG.Bibliography.ps and MG.Bibliography.bib on the ftp site and
+is accessible through the Web page.
+
+
+
Index: /main/tags/3.01/gsdl/packages/mg/README.port
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/README.port	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/README.port	(revision 21112)
@@ -0,0 +1,125 @@
+P O R T A B I L I T Y
+=====================
+
+The main development of MG is done under Solaris on a Sun sparc.
+However, we would also like MG to be reasonably portable.
+Therefore, MG-1.2 now uses a configure stript generated by GNU's autoconf.
+Below, is a list of the machines that the s/ware has been ported to. 
+Most only need the configure script and some require manual modifications 
+mentioned in the "Problems" field.
+
+------------------------------------------------------------------------- 
+** Systems which have successfully been built.
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2 8 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2_DB 14 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.3_U1 1 sun4c
+Compiler:	gcc version 2.5.7
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.3 Generic_Patch sun4m sparc
+Compiler:	gcc version 2.4.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Silicon Graphics
+System:		IRIX 5.2 02282014 IP7
+Compiler:	gcc version 2.6.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.4 Generic_101945-27 sun4c sparc
+Compiler:	cc: SC3.0.1 13 Jul 1994
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:
+System:		SCO 3.2v4.1
+Compiler:	gcc 2.6.1
+Problems:	requires RANLIB=true, use of tcsh
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	HP
+System:		HP-UX A.09.04 D 9000/837 415303291 64-user license
+Compiler:	HP92453-01 A.09.73 HP C Compiler
+Problems:
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Data General
+System:		dgux 5.4R3.00 generic AViiON mc88110 
+Compiler:	gcc 2.4.5 (dg supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Pyramid MIPS R3000 box
+System:		
+Compiler:	cc (vendor supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	IBM RS6000
+System:		AIX 3.2
+Compiler:	AIX XL C Compiler/6000
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Linux 1.2.9
+System:		
+Compiler:	gcc 2.6.3
+Problems:	none
+
+-----------------------------------------------------------------------
+** Systems which have NOT successfully been built.
+
+Tester:		RBrown@cmutual.com.au, 
+		Lloyd.Parkes@vuw.ac.nz
+Date:		Oct, Nov, 1995
+Machine:	Dec Alpha OSF/1
+System:
+Compiler:	cc
+Problems: 	Many - see below.
+
+* Problems with Dec Alpha
+  - 64 bit length longs as opposed to 32 bit longs.
+  - Lloyd.Parkes@vuw.ac.nz says that there is some bit encoding problems 
+    but the decoding seems to work.
+  - RBrown@cmutual.com.au says that it will compile but will core dump
+    when  building the test database.
+  - tes.kbs.citri.edu.au says that he needs to look into the repercussions
+    of 64 bit longs, however, he does not have access to such a machine.
+		
+-----------------------------------------------------------------------
+** Other systems. 
+
+Systems other than the above may have installation problems.
+If this is the case, please send me (tes@kbs.citri.edu.au) a log of the output
+and I will modify the configuration files in an effort to fix the problems.
+
+-----------------------------------------------------------------------
Index: /main/tags/3.01/gsdl/packages/mg/SampleData/MODIFICATIONS
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/SampleData/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/SampleData/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,664 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/gsdl/packages/mg/SampleData/Makefile
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/SampleData/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/SampleData/Makefile	(revision 21112)
@@ -0,0 +1,4 @@
+DISTFILES = Makefile alice13a.txt.Z MODIFICATIONS 
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/gsdl/packages/mg/about_mg.html
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/about_mg.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/about_mg.html	(revision 21112)
@@ -0,0 +1,75 @@
+<HTML>
+<HEAD>
+<TITLE>About MG</TITLE>
+</HEAD>
+<BODY>
+<H1>About MG</H1>
+ 
+<HR SIZE=4>
+
+MG is a public domain
+indexing and retrieval system for text, images, and textual images.
+MG is covered by a 
+<a href="http://deimos.kbs.citri.edu.au/mg/intro/copying.html">GNU 
+public licence</a>.
+<P>
+MG is available for ftp <a href="ftp://munnari.oz.au/pub/mg">here</a>
+
+<H2>Developers</H2>
+
+The MG system is an embodiment of ideas developed primarily by:
+<ul>
+<li> <a href="http://www.cosc.canterbury.ac.nz/~tim">Tim C. Bell</a>,
+	University of Canterbury;
+<li> <a href="http://www.cs.mu.oz.au/~alistair">Alistair Moffat</a>,
+	University of Melbourne;
+<li> <a href="http://www.cs.waikato.ac.nz/~ihw">Ian Witten</a>,
+	University of Waikato; and
+<li> <a href="http://www.cs.rmit.edu.au/~jz">Justin Zobel</a>,
+	RMIT.
+</ul>
+
+<H2>Programmers</H2>
+
+The bulk of the programming work has been undertaken by:
+<ul>
+<li><A HREF="http://www.cs.waikato.ac.nz/~singlis/">Stuart Inglis</A>,
+       University of Waikato;
+<li><A HREF="http://www.cs.waikato.ac.nz/~cgn/">Craig Nevill-Manning</A>,
+       University of Waikato;
+<li> Neil Sharman, University of Melbourne; and
+<li> Tim Shimmin, Collaborative Information Technology Research Institute (CITRI).
+</ul>
+A full list of contributors appears
+<A HREF="http://deimos.kbs.citri.edu.au/mg/intro/mgintro.html#xtocid2063010">
+here</a>.
+
+<H2>Institutional Support</H2>
+
+The MG project has been supported by the following organisations:
+<ul>
+<li>	The Australian Research Council;
+<li>	<a HREF="http://www.citri.edu.au/">CITRI</a>, Melbourne;
+<li>	The <a HREF="http://www.cpsc.ucalgary.ca/">University
+		of Calgary</a>, Canada;
+<li>	The <a HREF="http://www.cosc.canterbury.ac.nz/">University
+		of Canterbury</a>, New Zealand;
+<li>	The <a HREF="http://www.cs.mu.oz.au/">University
+		of Melbourne</a>, Australia;
+<li>	<a HREF="http://www.cs.rmit.edu.au/">RMIT</a>, Australia; and
+<li>	The <a HREF="http://www.cs.waikato.ac.nz/">University
+		of Waikato</a>, New Zealand.
+</ul>
+
+<H2>Backpointer</H2>
+
+If you didn't just come from there, the MG home page is accessible
+<a href="http://www.kbs.citri.edu.au/mg/">here</a>.
+
+<HR SIZE=4>
+<ADDRESS>alistair@cs.mu.oz.au</ADDRESS><BR>
+ 
+</BODY>
+</HTML>
+
+
Index: /main/tags/3.01/gsdl/packages/mg/acconfig.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/acconfig.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/acconfig.h	(revision 21112)
@@ -0,0 +1,35 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
Index: /main/tags/3.01/gsdl/packages/mg/aclocal.m4
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/aclocal.m4	(revision 21112)
@@ -0,0 +1,140 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ(regex)
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , [AC_LIBOBJ(rx)])
+fi
+])
Index: /main/tags/3.01/gsdl/packages/mg/build_options.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/build_options.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/build_options.in	(revision 21112)
@@ -0,0 +1,2 @@
+set stem_method = 3
+set do_indexes = 1
Index: /main/tags/3.01/gsdl/packages/mg/config.guess
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/config.guess	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/gsdl/packages/mg/config.h.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/config.h.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/config.h.in	(revision 21112)
@@ -0,0 +1,174 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* define if big endian */
+#undef WORDS_BIGENDIAN
Index: /main/tags/3.01/gsdl/packages/mg/config.sub
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/config.sub	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/gsdl/packages/mg/configure
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/configure	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/configure	(revision 21112)
@@ -0,0 +1,7724 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="src/text/mgquery.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mg
+VERSION=1.2
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+fi
+
+fi
+
+
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+
+                                                  ac_config_files="$ac_config_files Makefile src/text/Makefile src/images/Makefile src/scripts/Makefile lib/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "src/text/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/text/Makefile" ;;
+  "src/images/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/images/Makefile" ;;
+  "src/scripts/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/scripts/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/gsdl/packages/mg/configure.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/configure.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/configure.in	(revision 21112)
@@ -0,0 +1,101 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(src/text/mgquery.c)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mg
+VERSION=1.2
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile ], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/gsdl/packages/mg/docs/Makefile
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/Makefile	(revision 21112)
@@ -0,0 +1,5 @@
+DISTFILES = Makefile mgmerge.README mg_invf_dump.changes \
+  README.man mg.1 mgintro.1 mgintro++.1
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/gsdl/packages/mg/docs/README.man
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/README.man	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/README.man	(revision 21112)
@@ -0,0 +1,1 @@
+The rest of the man files are in the respective src directories.
Index: /main/tags/3.01/gsdl/packages/mg/docs/mg.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/mg.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/mg.1	(revision 21112)
@@ -0,0 +1,462 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg 1 \*(Dt CITRI
+.\"=====================================================================
+.\" Author:
+.\"     Nelson H. F. Beebe
+.\"     Center for Scientific Computing
+.\"     Department of Mathematics
+.\"     University of Utah
+.\"     Salt Lake City, UT 84112
+.\"     USA
+.\"     Email: beebe@math.utah.edu (Internet)
+.\"=====================================================================
+.if t .ds Bi B\s-2IB\s+2T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Bi BibTeX
+.if t .ds Te T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Te TeX
+.\"=====================================================================
+.SH NAME
+mg \- full-text inverted index support
+.\"=====================================================================
+.SH DESCRIPTION
+.B mg
+is a suite of programs that can be used to create
+and query full-text inverted indexes for a
+collection of documents.
+.PP
+An inverted index is essentially a list of
+pointers to occurrences of every word in a
+collection of documents, so that, for example, in
+a collection of Shakespeare's works, one can pose
+questions like
+.I "How often is a fool mentioned in the plays?"
+and
+.I "Are Romeo and Juliet ever mentioned in the same sentence?"
+While search utilities like the UNIX
+.BR grep (1)
+family can be used for simple queries, they suffer
+from several limitations:
+.TP \w'\(bu'u+2n
+\(bu
+Each invocation requires a separate pass over the
+documents, which becomes impossibly slow if the
+document collection is very large.
+.TP
+\(bu
+It is difficult to compose Boolean queries like
+.IR "Caesar AND Brutus AND NOT Antony" .
+.TP
+\(bu
+They provide no mechanism for ranking matches
+according to importance, which is extremely
+important when queries are complex and numerous
+matches are found.
+.TP
+\(bu
+They provide no mechanism for displaying
+surrounding context (although the Free Software
+Foundation
+implementations for the GNU Project remedy this
+with their
+.BI \-A " num"
+(after)
+and
+.BI \-B " num"
+(before)
+switches).
+.TP
+\(bu
+They provide no mechanism for searching for
+occurrences in the same paragraph, unless
+preprocessing is done on the files to wrap
+paragraphs into long lines.  Even that will often
+fail, because input buffer sizes may be limited to
+a few hundred characters.
+.TP
+\(bu
+They provide no easy way to deal with grammatical
+word-ending variants, except by explicit
+enumeration, such as searching for
+.IR compress ,
+.IR compressed ,
+.IR compresses ,
+.IR compressing ,
+and
+.IR compression .
+.PP
+Most computer users have been faced with the
+problem of finding information from a large
+collection of files, such as electronic mail,
+on-line documentation, or source code.  Inverted
+indices provide a highly-effective solution to
+this problem.
+.PP
+The
+.B mg
+software is described in Appendix A of the book
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994
+.fi
+.RE
+.PP
+Many of the algorithms implemented in
+.B mg
+represent significant advances over previous work,
+both in speed, and in storage requirements.  On a
+fast workstation, in tens of minutes, or a few
+hours,
+.BR mgbuild (1)
+can create an index to
+.I all
+of the words in hundreds of thousands of documents
+occupying hundreds of megabytes, or even more than
+a gigabyte, of disk space.
+.BR mgquery (1)
+can then be used to answer complex queries, with
+responses often returned in a second or less.
+.B mg
+also contains algorithms to deal with images, so
+that with a small amount of descriptive text for
+each image, it is possible to do searches in
+collections of images, and to have retrievals
+display the images using a viewer like
+.BR xv (1).
+.PP
+.B mg
+can deal with compressed text and image files and
+surprisingly, it usually runs faster than it would
+if the files were not compressed!  Thus, the
+considerable disk space savings possible from
+file compression are not lost because of the need
+for fast document search and retrieval.
+.PP
+The Free Software Foundation GNU Project
+compression utilities
+.BR gzip (1)
+and
+.BR gunzip (1)
+are recommended for general use over older
+alternatives, like
+.BR compress (1),
+because of their speed and high compression
+ratios.
+.\"=====================================================================
+.SH AVAILABILITY
+The
+.B mg
+software can be obtained via anonymous ftp to the
+Australian archive host
+.I "munnari.oz.au [128.250.1.21]"
+from the directory
+.IR "/pub/mg" .
+.\"=====================================================================
+.SH "TYPICAL USE OF mg"
+Although
+.B mg
+consists of more than 20 separate programs, many
+of which have complicated command-line options,
+take heart: most users require only two or three
+of these programs, and nothing more than a
+document name on the command line.
+.PP
+A
+.I document
+for
+.B mg
+is a fragment of text suitable for retrieval as a
+unit when it is found to contain a requested word,
+or words.  In a collection of poetry, a document
+might be a stanza, while in a novel, it could be a
+paragraph.  In an index of first lines of poems, a
+document would likely be just a single line.
+.PP
+Just what constitutes a document is decided by a
+user-modifiable UNIX shell script,
+.BR mg_get (1).
+The default script provided with the
+.B mg
+source distribution knows about these named
+document collections:
+.TP \w'mailfiles'u+2n
+.I alice
+Lewis Carroll's
+.I "Alice in Wonderland"
+book.
+.TP
+.I allfiles
+all mail files in the directory tree
+.IR $HOME/Mail ,
+including all of its nested subdirectories.
+.TP
+.I mailfiles
+individual mail messages in
+.IR $HOME/mbox
+and
+.IR $HOME/.sentmail.
+.TP
+.I davinci
+A small collection of text and images from the
+work of Leonardo da Vinci.
+.PP
+A document collection name is used by
+.B mg
+as a
+.BR csh (1)
+.I case
+statement selector, and as a subdirectory name in the
+.B $MGDATA
+directory, or the current directory, if
+.B MGDATA
+is not defined.
+.\"=====================================================================
+.SH "EXTENDING mg_get"
+This section describes how to extend
+.BR mg_get (1)
+to handle a new document collection.
+.PP
+Let us take two examples: all \*(Bi
+.I .bib
+files, and all \*(Te files, contained in
+subdirectories under the login directory.
+.PP
+For \*(Bi, each bibliographic entry will be
+considered a separate document.  In order to
+facilitate easy identification of entries, we
+shall require them to begin at the start of a
+line; the
+.BR bibclean (1)
+utility can be used to standardize the format of
+.I .bib
+files, and to validate their string values, so
+that this requirement is met.
+.PP
+For \*(Te, each paragraph will be a separate
+document, and we assume that paragraphs are
+separated by blank lines.  We assume that files
+with extensions
+.IR .atx ,
+.IR .ltx ,
+.IR .stx ,
+and
+.I .tex
+contain input to common \*(Te macro package
+variants.
+.PP
+Make a personal copy of the
+.I mg_get
+script, using the one in the
+.B mg
+source distribution
+.RI ( mg-1.0/mg/mg_get.sh ),
+or the one in the local binary program directory,
+at many sites called
+.IR /usr/local/bin/mg_get .
+.PP
+Examination of the
+.I mg_get
+script shows that each document collection name is
+used in a
+.BR csh (1)
+.I case
+statement selector, and that most of work is done
+by very simple
+.BR awk (1)
+programs that extract documents from files.  In
+your private copy of the
+.I mg_get
+file, after the line
+.PP
+.nf
+  breaksw #davinci
+.fi
+.PP
+and before the line
+.PP
+.nf
+  default:
+.fi
+.PP
+insert this new code:
+.PP
+.nf
+  case bibfiles:
+# Takes a list of files that contain BibTeX entries, and splits them up
+#   by putting ^B after each entry. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*.bib' -print | \e
+        sort | \e
+        xargs -l100 awk \e
+          '/^@/&&NR!=1{print "^B"} {print $0} END{print "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bibfiles
+
+  case texfiles:
+# Takes a list of TeX files and split them up
+#   by putting ^B after each paragraph. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*x' -print | \e
+        egrep '[.]tex$|[.]ltx$|[.]atx$|[.]stx$' | \e
+        sort | \e
+        xargs -l100 nawk '   /^ *$/ {if (b!=1) printf "^B";b=1} \e
+            \e!/^ *$/ {print;b=0} \e
+            END {printf "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #texfiles
+.fi
+.PP
+The ^B characters here are Control-B characters,
+.I not
+caret-B pairs.
+.PP
+If you have a large number of \*(Bi or \*(Te
+files, it is likely that a list of them would be
+too long for the UNIX shell to hold in a single
+variable, or on a single command line.  Thus,
+instead of storing the output of
+.BR find (1)
+in a variable, we proceed more cautiously, and
+employ it to produce a list of the required files,
+then pipe them to
+.BR xargs (1),
+which in turn passes up to 100 filenames at a time
+to
+.BR nawk (1)
+for document selection.
+.PP
+Install this modified
+.I mg_get
+script in your private directory for executable
+programs (e.g.
+.IR $HOME/bin ),
+create a directory
+.I $HOME/mgdata
+to hold the index, issue a
+.B rehash
+command if you are using
+.BR csh (1)
+or
+.BR tcsh (1),
+ensure that
+.I mg_get
+occurs in your search path
+.I before
+any system-wide one (the command
+.BI which " mg_get"
+will tell you which version will be selected),
+then create the inverted indexes by
+.BI mgbuild " bibfiles"
+and
+.B mgbuild
+.IR texfiles .
+These commands may take several minutes to run if
+you have a lot of \*(Bi or \*(Te files, or a large
+home directory tree.  Once they are complete, you
+can then query the index with the commands
+.BI mgquery " bibfiles"
+and
+.B mgquery
+.IR texfiles .
+These should respond very rapidly.
+.PP
+In order to keep your index up-to-date, you should
+arrange for it to be recreated automatically and
+regularly, probably every night.  You can do this
+with
+.BR cron (1).
+Use the command
+.B "crontab \-e"
+to edit your
+.I crontab
+file and add two lines like this:
+.PP
+.nf
+00 04 * * * mgbuild bibfiles >$HOME/mgdata/bibfiles.log 2>&1
+15 04 * * * mgbuild texfiles >$HOME/mgdata/texfiles.log 2>&1
+.fi
+.PP
+Save the file and exit the editor.  Now, every
+night at 4am and 4:15am,
+.BR mgbuild (1)
+will reconstruct your inverted indexes, and the
+results of the builds will be saved in log files
+in your
+.I $HOME/mgdata
+directory.
+.\"=====================================================================
+.SH "SEE ALSO"
+.na
+.BR awk (1),
+.BR bibclean (1),
+.BR bibtex (1),
+.BR compress (1),
+.BR csh (1),
+.BR grep (1),
+.BR gunzip (1),
+.BR gzip (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR nawk (1),
+.BR tcsh (1),
+.BR tex (1),
+.BR xargs (1),
+.BR xmg (1),
+.BR xv (1).
+.\"=====================================================================
+.\" This is for GNU Emacs file-specific customization:
+.\" Local Variables:
+.\" fill-column: 50
+.\" End:
Index: /main/tags/3.01/gsdl/packages/mg/docs/mgintro++.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/mgintro++.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/mgintro++.1	(revision 21112)
@@ -0,0 +1,212 @@
+.\"------------------------------------------------------------  
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4 
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------  
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO++ 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro++ \- extended introduction to the MG system
+.\"-------------------------------------------------------------
+.SH DESCRIPTION
+This manual assumes the reader has already read
+.BR mgintro (1).
+.\"-------------------------------------------------------------
+.SS Creating Different Databases
+If a user wants to build databases other than for some
+predefined ones, such as "alice", "davinci", "mailfiles", "allfiles",
+then the user has a couple of choices.
+Ultimately (s)he must produce a text file with control-Bs 
+terminating the documents. 
+To do this one can produce one or more such files, or write a
+"get" command (typically in the form of a script or c program).
+.\"-------------------------------------------------------------
+.SS Using Input Files for mgbuild
+If you don't want to write a "get" script and just want to use
+one or more text files as input, then you must first generate
+the file with control-Bs. For a simple example, you could take
+any text file(s) such as "test1.txt" and "test2.txt", and use 
+.BR vi (1)
+to insert control-Bs by typing "control-V b".
+Next you should create a file with "set" statements 
+in the following form:
+.PP
+.IP
+\fBset pipe = 0 # do not use pipe - use file instead 
+.br
+\fBset input_files = 'test1.txt test2.txt'
+.LP
+Let's call this file, "build_options".
+Now issue the command:
+.IP
+.B mgbuild -s build_options test
+.LP
+This should build a database called "test" in the $MGDATA directory,
+based on the source data of "test1.txt" and "test2.txt".
+The build_options file is simply sourced by 
+.BR mgbuild (1)
+after it has set up its variables. 
+Therefore, any settings one makes in the
+build_options file will override the standard settings.
+See 
+.BR mgbuild (1) 
+for more information.
+.\"-------------------------------------------------------------
+.SS Writing A Get Program
+Instead of using files as input, it is often more convenient to 
+write a "get" program. This program is called by 
+.BR mgbuild (1)
+to get the text data with control-Bs as document terminators.
+It should take three options:
+.br
+(i) -init; (ii) -text; (iii) -cleanup.
+.br
+Get will be called with "init" first and with "cleanup" at the end.
+It will call get with "text" when it wants the text and it should
+write the text to stdout.
+.br
+See 
+.BR mg_get (1)
+for an example.
+.\"-------------------------------------------------------------
+.SS Regular Builds
+The MG system provides a static database; 
+there are no update commands.
+So if one wants to keep one's database reasonably up-to-date
+then one can have this done automatically on a regular basis by 
+.BR cron (1).
+A crontab file can be created using:
+crontab -e
+A crontab file contains lines of the form:
+.nf
+.IP
+\fBminute hour day-of-month month day-of-week shell-command.
+.LP
+.fi
+See
+.BR crontab (1)
+for more information.
+.nf
+An example crontab entry is:
+.IP
+\fB15 02 * * * mgbuild allfiles >$MGDATA/allfiles/allfiles.log 2>&1
+.LP
+.fi
+This will build up the mg database for "allfiles", your mail in
+the folders, every morning at 2:15am. 
+.\"
+.\"-------------------------------------------------------------
+.SS Command Structure
+There are 22 commands that make up the mg system. However,
+a user may only need to be aware of a few: 
+.BR mgbuild (1),
+.BR mgquery (1), 
+and perhaps
+.BR mg_get (1). 
+Many of the commands are called by 
+.BR mgbuild(1).
+The commands can be broken up into a hierarchy.
+.PP
+--------------------------------------
+.br
+MG--+--image compression
+    |  |
+    |  +--mgbilevel
+    |  |
+    |  +--mgfelics
+    |  |
+    |  +--mgtic
+    |  |
+    |  +--mgticbuild
+    |  |
+    |  +--mgticdump
+    |  |
+    |  +--mgticprune
+    |  |
+    |  +--mgticstat
+    |
+    +--text 
+       |
+       +--compression
+       |  |
+       |  +--mg_passes -T1
+       |  |
+       |  +--mg_passes -T2
+       |  |
+       |  +--mg_compression_dict
+       |  |
+       |  +--mg_fast_comp_dict
+       |
+       +--indexing
+       |  |
+       |  +--mg_passes -N1
+       |  |
+       |  +--mg_passes -N2
+       |  |
+       |  +--mg_perf_hash_build
+       |  |
+       |  +--mg_invf_dict
+       |  |
+       |  +--mg_invf_rebuild
+       |  
+       +--weights
+       |  |
+       |  +--mg_weights_build
+       |
+       +--query
+       |  |
+       |  +--mgquery
+       |
+       +--tools
+          |
+          +--mg_invf_dump
+          |
+          +--mg_text_estimate
+          |
+          +--mgdictlist
+          |
+          +--mgstat
+.br
+--------------------------------------
+.PP
+.nf
+.BR mgbuild (1)
+calls the following commands:
+.RS
+.BR mg_passes (1), mg_compression_dict (1)
+.BR mg_perf_hash_build (1), mg_invf_dict (1), mg_invf_rebuild (1)
+.BR mg_weights_build (1)
+.RE
+.fi
+.\"--------------------------------------------
+.SH SEE ALSO
+.BR mgintro (1),
+.BR mgbuild (1),
+.BR mg_get (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95 
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
+
Index: /main/tags/3.01/gsdl/packages/mg/docs/mgintro.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/mgintro.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/mgintro.1	(revision 21112)
@@ -0,0 +1,155 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro \- introduction to the MG system
+.\"--------------------------------------------------------------
+.SH DESCRIPTION
+The MG (Managing Gigabytes) system is a collection of
+programs which comprise a full-text retrieval system.
+A full-text retrieval system allows one to create a
+database out of some given documents and then do queries
+upon it to retrieve any relevant documents.
+It is "full-text" in the sense that every word in the
+text is indexed and the query operates only on this index 
+to do the searching.
+.PP
+For example, one could have a database on the book,
+"Alice in Wonderland." A document could be represented by
+each paragraph in the book.
+Having built up the "Alice" database, one could do queries
+such as "cat alice grin" and retrieve any paragraphs which
+match the query. The matching could either be boolean, that
+is the retrieved paragraphs contain a boolean expression of
+the query terms e.g. "cat alice grin"; or the matching
+could be ranked i.e. the most relevant documents to the query
+in relevance order, using some standard heuristic measure.
+.\"--------------------------------------------------------------
+.SS Motivation
+If one wants to find some particular information which
+is stored in a computer text file then one has a few alternative
+courses of action. One can operate directly on the text files
+with utilities such as grep or can process the text files into
+some form of database. Grep is generally limited to identifying
+lines by matching on regular expressions. If the collection 
+of files which grep operates on becomes large, then continual
+passes over the entire text on each query becomes expensive.
+However, its usage is simple as no auxiliary files must be created.
+.PP
+A database consists of some data and indexes into that data. By having
+indexes one can query a large database quickly. Standard databases
+divide the data up into records of fields. This means that the granularity
+of search is a field. In a full-text system, such as \*r,
+there are no fields 
+(or there is an arbitrary sized list of word fields per document) 
+and instead every word is indexed. 
+Using this method, we can except free-form information and yet be fast on searches.
+The next question is what is the overhead of this database.
+In \*r most files which are produced are in a compressed form. The
+two notable compressed files being the given data and the index, called
+an "inverted file". By compressing the files it is possible to have the
+size of the database smaller than the size of the source data.
+.\"--------------------------------------------------------------
+.SS Typical Usage
+The most common use for \*r
+has been as a search database on unix mail files.
+However, any set of text data can be used, one just needs to determine
+what constitutes a document (see 
+.BR mgintro++ (1)
+).
+\*r has also been used on large collections such
+as Comact (Commonwealth Acts of Australia) which is around 132 megabytes 
+and also on sizes up to around 2 gigabytes for TREC 
+(a mixture of collections such as the Wall Street Journal 
+and Associated Press).
+.\"--------------------------------------------------------------
+.SS Getting Started with \*r
+The first thing to do is install the package; 
+please follow the INSTALL instructions.
+Having done this, it is necessary to set a couple of environment variables.
+MGDATA should be set to a directory which is to hold subdirectories for
+each database that you build. For example: 
+.IP
+.B mkdir ~/mgdata; setenv MGDATA ~/mgdata.
+.LP
+If you want to try out building some sample databases then there is
+some sample data such as the "Alice In Wonderland" book. To make sure
+this is accessible you should set the environment variable MGSAMPLE.
+For example: 
+.IP
+.B setenv MGSAMPLE ~/mg/SampleData
+.LP
+Here, "~/mg/SampleData" should contain alice.z .
+.PP
+To build the Alice database (to be contained in $MGDATA/alice subdirectory), 
+type the command 
+.IP
+.B mgbuild alice 
+.LP
+Assuming all went well and some status messages 
+were printed indicating the build was completed, then type 
+.IP
+.B mgquery alice 
+.LP
+to query the database. 
+You can type a few words at the prompt, hit return and
+some relevant documents, Alice paragraphs, should be retrieved.
+Type ".set query ranked" to do ranking queries. Please refer to the 
+.BR mgquery (1)
+man-page for more information on the commands and options of 
+.BR mgquery (1).
+.PP
+The next thing to do is to use \*r
+on a more personal database. If you have
+your mail stored in subdirectories of ~/Mail, such as is done if you use the
+typical set up of 
+.BR elm (1), 
+then type 
+.IP
+.B mgbuild allfiles 
+.LP
+If, however, you keep all your mail in ~/mbox or ~/sentmail, then type 
+.IP
+.B mgbuild mailfiles
+.LP
+.\"--------------------------------------------------------------
+.SH AVAILABILITY
+The \*r software for SunOS 4, Solaris, HPUX, and MIPS,
+can be ftped from: munnari.oz.au [128.250.1.21] in the directory
+/pub/mg. 
+.\"--------------------------------------------------------------
+.SH SEE ALSO
+.na
+.BR mgintro++ (1),
+.BR mgbuild (1),
+.BR mgquery (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
Index: /main/tags/3.01/gsdl/packages/mg/docs/mgmerge.README
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/docs/mgmerge.README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/docs/mgmerge.README	(revision 21112)
@@ -0,0 +1,230 @@
+A CRASH COURSE GUIDE TO MGMERGE
+===============================
+
+Shane Hudson
+15 November 1994
+
+
+This document is intended as a note to the maintainers of mgmerge
+outlining the mgmerge utility, the changes
+that had to be made to the existing mg code to make mgmerge
+work, and the new source code for mgmerge.
+
+
+****  NOTE:    ****
+All the NEW/CHANGED files are in the file "new_mg.tar" which
+you should have received with this; files that were
+not changed are NOT in the tar file  so it will have to be
+extracted "on top of" the existing code. You'll figure it out :)
+
+
+****  DESCRIPTION:   ****
+mgmerge adds documents to an existing mg database
+without the need to rebuild the database from scratch.
+It works by building a temporary new database from the
+new documents, then merging the "old" and "new" databases
+to give one merged database.
+
+
+The source consists of:
+    mgmerge.sh		The main script, a lot like mgbuild.sh
+    mg_get_merge.sh	Script to retrieve new documents, a lot like
+			mg_get.sh
+    mg_merge.h		header file used by mg_invf_merge.c
+			and mg_text_merge.c
+    mg_text_merge.c	Program to append one compressed text file
+			to another
+    mg_invf_merge.c	Program to merge two inverted files and
+			stemmed dictionaries
+
+See the man pages for mgmerge, mg_get_merge, mg_text_merge and
+mg_invf_merge for a brief overview.
+
+mg_get_merge.sh should really be put into mg_get.sh, with a "-merge"
+option if it is being called by the mgmerge utility, but I was
+too lazy to change it to that.
+
+
+The files updated by mgmerge are:
+
+*.text			with mg_text_merge
+*.text.idx		with mg_text_merge
+*.text.idx.wgt		with mg_weights_build (after merging)
+*.invf			with mg_invf_merge
+*.invf.idx		with      "
+*.invf.dict		with      "
+*.weight		with mg_invf_merge or mg_weights_build
+*.invf.dict.blocked	with mg_invf_dict (after merging)
+
+The *.text.stats file and *.text.dict file remain the same.
+Any other database files (ie, *.invf.dict.hash)
+will be out of date after a merge and will need to be recomputed.
+
+--------------------------------------------------------------------------
+
+MG_TEXT_MERGE
+=============
+
+The two "merging" utilities are mg_text_merge and mg_invf_merge
+
+mg_text_merge simply appends the compressed text for the new
+documents to the old compressed text file.
+For this to succeed, the two files should have been created
+with the same parameters to mg_passes and the same compression
+model must be used. So *.text.dict from the old (static) database
+is used as the model for compressing the new documents.
+The default "-C" option to mg_compression_dict in mgbuild was
+changed to "-S" so novel words in the new documents can be coded.
+The option must be the same in mgmerge -- it is "-S" there too.
+
+---------------
+
+EFFECT on FILE COMPRESSION PERFORMANCE
+
+Since the compression model is not being updated,compression
+performance for the text will slowly degrade. The only solution
+is a periodic rebuild from scratch.
+
+---------------------------------------------------------------------------
+
+MG_INVF_MERGE
+=============
+
+mg_invf_merge updates the stemmed dictionary and
+inverted file.
+For reasons of simplicity, the stemmed dictionaries merged
+are in the ".invf.dict" format, NOT the ".invf.dict.blocked"
+format. 
+So if a database is to be merged the .invf.dict file should
+not be deleted after the .invf.dict.blocked file used by mgquery
+has been created.
+Also for simplicity, level 3 inverted files are not supported, nor
+are skipped format files.
+
+
+
+Merging the inverted files requires every entry in each file
+to be decoded and merged if an entry for the same term appears
+in the other inverted file.
+This can be a slow process if the old inverted file is very large.
+Especially since each entry is decoded bit-by-bit with
+the bit-level I/O rouutines.
+As was noted on page 94 of the book "Managing Gigabytes" (hereafter 
+called "MG"), switching from bernoulli coding (called "Bblock" in
+the mg source code) to gamma or delta coding for the
+inter-document gaps would be an advantage since they require no
+parameters.
+With Bblock coding, N (the number of documents) is a parameter and
+as N changes whenever documents are added, every entry must
+be decoded and recoded.
+My approach was to keep Bblock coding, but keep the N parameter
+constant by recording the artificial N  used to code document
+gaps (called "Nstatic") as well as the real number of documents.
+
+Then, any entry in the old inverted file (called "IFold") that
+is not merged with an entry in the new inverted file (called "IFnew")
+can be copied directly to the merged inverted file (called "IFmerge").
+This can give a significant increase in speed, since usually the
+size of IFnew is very small if only a few documents are being added.
+
+To store Nstatic, a field "static_num_of_docs" was added to
+the invf_dict_header and stem_dict_header structs defined
+in invf.h
+Any program that decodes inverted file entries also had to be changed.
+(The existing mg source code that I changed has been returned with
+this document.)
+Most changes were simply altering a line that
+called BIO_Bblock_Init() so it used the static_num_of_docs
+value rather than num_of_docs.
+
+So since the file format has changed, any existing collection will
+have to be rebuilt first even using mgmerge with it is not
+intended.
+
+-------------
+
+EFFECT on INVERTED FILE COMPRESSION PERFORMANCE
+
+As more merges are accumulatively performed on a database, the
+compression performance will decline.
+The solution is an option in mg_invf_merge to do a "slow"
+merge where every inverted file entry is decoded and recoded using
+the real value of N.
+This is not a lot slower than a fast merge when compared to the
+cost of completely rebuilding the collection from scratch, and 
+the option can be used periodically (when, say, 40% of the
+inverted file has been added since the previous slow merge).
+
+-----------------
+
+THE EXACT WEIGHTS FILE
+
+Recomputing exact document weights requires a complete scan over the
+merged inverted file. This is a waste since the weights for old
+documents will (hopefully) not change much anyway, and the weights
+for new documents can be computed exactly when merging the inverted files,
+at no extra time cost.
+So mg_invf_merge computes the weights for new documents and
+updates the .weight file.
+The weights for old documents are left untouched.
+mgmerge has an option to recompute the weights file.
+This is not much more expensive in terms of time, and is only
+periodically needed.
+
+The effect of leaving the old weights unchanged is difficult to
+asess since the "correct" ranking of documents for a query is
+subjective.
+The frequency with which a rebuild of the weights file with the
+"-w" option should be done is hard to guess.
+But if most merges involves only a few documents, it makes sense
+to not bother recomputing it since the change in old weights values
+is very small and the weights only stray from their "true"
+values slowly over accumulated merging.
+
+-------------------------------------------------------------------------
+
+EXPERIMENTAL RESULTS
+====================
+
+With some small test collections (only a few Mb of text)
+and a slow merge, and a weights file rebuild,
+mgmerge typically took under 20% of the time to completely
+rebuild. Using the default options (fast merge, dont rebuild weights)
+took around 10% or more.
+For larger collections, the savings were greater, especially if the
+added text is very small.
+For example, when the last 6Kb of text of the GNUBib collection was
+added to the rest of it (14Mb) using mgmerge, the total mgmerge time
+was around 20 seconds whereas a complete mgbuild took 280 seconds.
+Only one larger collection was tested (resource contraints prevented
+testing on any really large collections): two short (one-line)
+documents were added to the Gutenberg collection, which comprised
+of nearly 74Mb of source text and had an inverted file size of 9Mb.
+mgmerge took 42 seconds (or 142 seconds with a slow merge and rebuilding the
+weights file), and I'd estimate that an mgbuild on the same machine would 
+have taken at least 20 minutes.
+
+To summarise, mgmerge is not as fast as some sophisticated method
+such as the fixed-length blocks described in section 5.7 of "MG",
+but its huge advantage is that it required almost no changes to
+the existing mg code and is still far better than using mgbuild
+every time a document needs to be added to a collection.
+
+----------------------------------------------------------------------------
+
+OTHER NOTES
+===========
+
+Since it works like mgbuild, mgmerge needs mg_get_merge
+to return the SAME text each time "mg_get_merge -text .." is called.
+
+----
+
+One other thing discovered (but not fixed) was a bug in "invf.pass2.c":
+It crashes on parsed words longer than 128 characters.
+So if any text with words >128 chars is piped to mg_passes,
+the "-N1" and "-N2" options (the memory-efficient inversion method)
+had better be used and not the "-I1"/"-I2" option (which would only
+work on small collections anyway, being the memory-inefficient
+method).
+
Index: /main/tags/3.01/gsdl/packages/mg/install-sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/install-sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGDocInfo.java
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
@@ -0,0 +1,42 @@
+/*
+ *    MGDocInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** DocInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @See MGQueryResult
+ */
+public class MGDocInfo
+{
+    public long num_ = 0;
+    public float rank_ = 0;
+
+    public MGDocInfo(long doc, float rank)
+    {
+	num_ = doc;
+	rank_ = rank;
+    }
+
+    public String toString()
+    {
+	return "" + num_ + " (" + rank_ + ")";
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGQueryResult.java
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
@@ -0,0 +1,140 @@
+/*
+ *    MGQueryResult.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** a java version of QueryResult for mg
+ *
+ * contains get methods for the java side, and simple
+ * set methods to be called from c side
+ *
+ */
+public class MGQueryResult
+{
+    /** the list of MGDocInfo */
+    protected Vector docs_ = null;
+    /** the list of MGTermInfo */
+    protected Vector terms_ = null;
+    /** the total number of docs found - not likely to be the size of docs_ */
+    protected long total_num_docs_ = 0;
+
+
+    MGQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+    }
+
+
+    /** clear the info from the last query
+	should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_ = 0;
+	docs_.clear();
+	terms_.clear();
+    }
+
+
+    /** returns the result as a String - useful for printing out results */
+    public String toString()
+    {
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i = 0; i < docs_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGDocInfo) docs_.elementAt(i)).toString();
+	}
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = " + total_num_docs_;
+	return result;
+    }
+
+
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+
+
+    // set methods used by c++ code
+
+    public void setTotalDocs(long num) {
+	total_num_docs_=num;
+    }
+
+
+    public void addDoc(long doc, float rank) {
+	MGDocInfo doc_info = new MGDocInfo(doc, rank);
+	System.out.println("(Java) Added doc " + doc_info);
+
+	docs_.add(doc_info);
+    }
+
+
+    // public void addTerm(String term, int stem, long match, long freq)
+    public void addTerm(String term, String tag, int stem,
+			long match, long freq, String[] equivs)
+    {
+	MGTermInfo ti = new MGTermInfo();
+	ti.term_ = term;
+	ti.tag_ = tag;
+	ti.stem_method_ = stem;
+	ti.match_docs_ = match;
+	ti.term_freq_ = freq;
+	if (equivs != null) {
+	    for (int i = 0; i < equivs.length; i++) {
+		ti.equiv_terms_.add(equivs[i]);
+	    }
+	}
+	System.out.println("(Java) Added term " + ti);
+
+	terms_.add(ti);
+    }
+
+
+    // Get methods for the java side - GS2MGSearch.java, GS2MGRetrieve.java
+    public Vector getDocs() {
+	return docs_;
+    }
+
+
+    public Vector getTerms() {
+	return terms_;
+    }
+
+
+    public long getTotalDocs() {
+	return total_num_docs_;
+    }
+}
+
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGTermInfo.java
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
@@ -0,0 +1,81 @@
+/*
+ *    MGTermInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** TermInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @see MGQueryResult
+ */
+public class MGTermInfo
+{
+    /** the term itself */
+    public String term_ = null;
+
+    /** the tag - level or metadata - for which the query was done */
+    public String tag_ = null;
+
+    /** the stem and casefold method used
+	0 = none
+	1 = casefold only
+	2 = stem only
+	3 = casefold and stem */
+    public int stem_method_ = 0;
+
+    /** the number of documents containing this term
+	(where document is defined by the tag data) */
+    public long match_docs_ = 0;
+
+    /** overall term freq - word level */
+    public long term_freq_ = 0;
+
+    /** list of stemmed and casefolded equivalent terms - if stem_method_ is non-zero 
+      - Vector of Strings */
+    public Vector equiv_terms_ = null;
+
+
+    public MGTermInfo()
+    {
+	equiv_terms_ = new Vector();
+    }
+
+
+    /** output the class as a string */
+    public String toString()
+    {
+	String result = "\"" + term_ + "\"";
+	result += " tag(" + tag_ + ")";
+	result += " stem(" + stem_method_ + ")";
+	result += " docs(" + match_docs_ + ")";
+	result += " freq(" + term_freq_ + ")";
+
+	result += " equiv terms(";
+	for (int i = 0; i < equiv_terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += (String) equiv_terms_.elementAt(i);
+	}
+	result += ")";
+
+	return result;
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGWrapper.java
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
@@ -0,0 +1,102 @@
+/*
+ *    MGWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** java wrapper class for access to mg in C
+ *
+ * the native side implemented in MGWrapperImpl.c
+ * uses MGQueryResult to hold the result of a query.
+ * the result of a getDocument is a String
+ * uses the jni
+ *
+ *@see MGQueryResult
+ */
+public class MGWrapper
+{
+    /** the query result, filled in by runQuery */
+    protected MGQueryResult mg_query_result_ = null;
+
+    /** pointer to c MGWrapperData class - cached indexData and queryInfo */
+    protected long mg_data_ptr_ = 0;
+
+    static {
+	System.loadLibrary("mgjni");
+	initIDs();
+    }
+
+    public MGWrapper() {
+	mg_query_result_ = new MGQueryResult();
+	initCSide();
+    }
+
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises the mg_data_ptr_ */
+    private native boolean initCSide();
+
+    /** returns a document: number docnum at level level 
+     * the base_dir and text_path paths should join together to provide
+     * the absolute location of the mg text files eg ..../index/text/demo
+     * returns the doc in utf-8
+     */
+    public native String getDocument(String base_dir, String text_dir, long docnum);
+
+    /** unloads the data */
+    public native boolean unloadIndexData();
+
+    // query param methods
+
+    /** if on=true, sets default casefolding on - it's off by default */
+    public native void setCase(boolean on);
+    /** if on=true, sets default stemming on - it's off by default */
+    public native void setStem(boolean on);
+    /** default is 50 */
+    public native void setMaxDocs(int num);
+    /** if on=true, sorts by rank, otherwise returns in build order - 
+	default is on */
+    public native void setSortByRank(boolean on);
+    /** if on=true, a query returns term freq info - default is on */
+    public native void setReturnTerms(boolean on);
+    /** sets the index to search - default is 'dtx' */
+    public native void setIndex(String index);
+    /** sets the default boolean operator - AND(=1)/OR(=0) */
+    public native void setMatchMode(int mode);
+
+    /** returns a string with all the current query param settings */
+    public native String getQueryParams();
+
+    /** actually carry out the query.
+	Use the set methods to set query results.
+	Writes the result to query_result.
+     * - maintains state between requests as can be slow  
+     * base_dir and index_path should join together to provide
+     * the absolute location of the mg index files eg ..../index/dtx/demo 
+     * base_dir must end with a file separator (OS dependant)
+     */
+    public native void runQuery(String base_dir, String text_dir, String query_string);
+
+
+    /** get the result out of the wrapper */
+    public MGQueryResult getQueryResult()
+    {
+	return mg_query_result_;
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,64 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = java/org/greenstone/mg
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+JAVAC = javac
+JAVAH = javah
+JAR = jar
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS =  -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+MGHOME=../../../..
+JAVALIBDIR=$(libdir)/java
+JAVACLASSDIR=./classes
+JAVASRCDIR=../../..
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) *.java
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGHOME)/jni org.greenstone.mg.MGWrapper
+	$(JAR) cf mg.jar -C $(JAVACLASSDIR) org
+
+clean:
+	rm -rf $(JAVACLASSDIR) mg.jar $(MGHOME)/jni/org_greenstone_mg_MGWrapper.h
+
+install: compile
+	$(MGHOME)/mkinstalldirs $(JAVALIBDIR); \
+	cp mg.jar $(JAVALIBDIR)/
+
+distclean: clean
+	rm -f Makefile
+
+
Index: /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Queryer.java
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
@@ -0,0 +1,185 @@
+/*
+ *    Queryer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+/** Queryer - java port of c++ Queryer 
+ * - uses MGWrapper to access mg
+ * only has get document and search - full text browse not implemented
+ *
+ * run as:
+ * java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>
+ *@see MGWrapper
+ */
+public class Queryer {
+  
+    /** outputs to std out a help message describing the commands used by 
+     * Queryer 
+     */
+    public void printHelp()
+    {
+	System.out.println( "commands available are:\n"+
+			    "\t.q\t\tquit\n"+
+			    "\t.h\t\tprint this help message\n"+
+			    "\t.d\t\tprint the current query parameter settings\n" +
+			    "\t.i<index>\tsearch index <index>\n"+
+			    "\t.r0/.r1\t\tranking off/on\n"+
+			    "\t.t0/.t1\t\tquery type some/all\n"+
+			    "\t.c0/.c1\t\tcasefolding off/on\n"+
+			    "\t.s0/.s1\t\tstemming off/on\n"+
+			    "\t.o0/.o1\t\tshort output off/on\n"+
+			    "\t.m<num>\t\tset max docs to return to num\n\n"+
+			    "\t.p<docnum>\tprint document docnum\n"+
+			    "\t<query string>\tdo a query\n");
+    }
+
+   
+    public static void main(String[] args)
+    {
+	if (args.length != 3) {
+	    System.out.println("Usage: java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>");
+	    return;
+	}
+
+	Queryer self = new Queryer();
+
+	String base_dir = args[0];
+	String index_path = args[1];
+	String text_path = args[2];
+
+	// the jni class to access mg stuff
+	MGWrapper wrapper = new MGWrapper();
+	wrapper.setIndex(index_path);
+
+	System.out.println("Welcome to Java Queryer :-)");
+	self.printHelp();
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	boolean shortOutput = false;
+
+	String command; // user input
+	char x; // the command letter
+	String data = null; // any auxiliary input to a command
+	while (true) {
+	    System.out.print(">"); // the prompt
+            try {
+                command = br.readLine();
+                command = command.trim();
+		if (command.startsWith(".")) {
+		    // a system command
+		    x = command.charAt(1);
+		    if (command.length() > 2) {
+			data = command.substring(2);
+			data = data.trim();
+		    }
+
+		    switch (x) {
+		    case 'q': // clean up and exit
+			wrapper.unloadIndexData();
+			return;
+		    case 'h': // print help message
+			self.printHelp();
+			break;
+		    case 'd': // print query param settings
+			String info = wrapper.getQueryParams();
+			System.out.println(info);
+			break;
+		    case 'p': // print doc
+			int docnum = Integer.parseInt(data);
+			String doc = wrapper.getDocument(base_dir, text_path, docnum);
+			System.out.println(doc);
+			break;
+		    case 'm': //match docs
+			int match = Integer.parseInt(data);
+			wrapper.setMaxDocs(match);
+			break;
+		    case 's': // set stem on/off
+			int stem = Integer.parseInt(data);
+			if (stem==0 ){
+			    wrapper.setStem(false);
+			} else if(stem==1) {
+			    wrapper.setStem(true);
+			} else {
+			    System.err.println("Error: stem should be 0 or 1");
+			}
+			break;
+		    case 'c': // set case on/off
+			int casef = Integer.parseInt(data);
+			if (casef==0) {
+			    wrapper.setCase(false);
+			} else if (casef==1) {
+			    wrapper.setCase(true);
+			} else {
+			    System.err.println("Error: case should be 0 or 1");
+			}
+			break;
+		    case 'i': // set index
+			wrapper.setIndex(data);
+			break;
+		    case 'r': // set rank on/off
+			int rank = Integer.parseInt(data);
+			if (rank==0) {
+			    wrapper.setSortByRank(false);
+			} else if (rank==1) {
+			    wrapper.setSortByRank(true);
+			} else {
+			    System.err.println("Error: rank should be 0 or 1");
+			}
+			break;
+		    case 't': // set query type some/all
+			int type = Integer.parseInt(data);
+			if (type==0 || type==1) {
+			    wrapper.setMatchMode(type);
+			} else {
+			    System.err.println("Error: type should be 0 (some) or 1 (all)");
+			}
+			break;
+		    case 'o': // set output short/long
+			int output = Integer.parseInt(data);
+			if (output==0) {
+			    shortOutput = false;
+			} else if (output==1) {
+			    shortOutput = true;
+			} else {
+			    System.err.println("Error: output should be 0 or 1");
+			}
+			break;
+		    }
+		}
+		else {
+		    // a query
+		    wrapper.runQuery(base_dir, text_path, command);
+		    MGQueryResult res = wrapper.getQueryResult();
+		    System.out.println("(Java) Matching documents: " + res.getTotalDocs());
+		    if (shortOutput) {
+			System.out.println(res.toShortString());
+		    } else {
+			System.out.println(res.toString());
+		    }
+		}
+
+	    } catch (Exception e) {
+		System.out.println("Queryer error: "+e.getClass() + " "+e.getMessage());
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.c	(revision 21112)
@@ -0,0 +1,662 @@
+/*
+ *    MGWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "MGWrapperImpl.h"
+#include <jni.h>
+#include "org_greenstone_mg_MGWrapper.h"
+
+#include "backend.h"
+#include "environment.h"
+
+
+/*************************************************************************
+   NOTES
+
+ - Features supported by MGPP but not by MG (AFAIK)
+
+   - Sorting results by rank.
+     Done here as a post-processing operation. Could be more efficient in
+     some cases: the current solution is not very good when the number of
+     matching documents is large and the number of desired matches is
+     small. In this case it would be better to iterate through the array
+     picking out the best documents rather than sorting them all.
+
+   - Asking for query term frequencies to be returned.
+     This cannot be turned off in MG. If the term frequencies are not
+     required, they are simply not passed bassed to the Java side.
+
+   - Choosing the index for queries.
+     It is possible for MG to build Section and Document indexes (for
+     example), but these are two separate indexes, and must be loaded
+     separately. This module can load more than one index at a time, thus
+     processing queries to different indexes more quickly.
+     NOTE: This replaces TWO options in the MGPP version: returnLevel and
+           queryLevel.
+
+   - "Some" (boolean OR) queries.
+     In boolean querying mode, MG assumes query terms are joined by ANDs.
+     There is no way to tell MG to treat the query terms as if they were
+     joined by ORs. Here, "some" queries are handled by explicitly
+     inserting OR ("|") characters between the query terms.
+
+ *************************************************************************
+   TO DO
+
+   - Find out about necessity of other query term information
+     (eg. equivalent terms).
+
+ *************************************************************************/
+
+
+#define MAX_INDEXES_CACHED 2
+
+
+/*********************************************
+   initialisation stuff
+ *********************************************/
+
+/* cached ids for java stuff */
+jfieldID FID_mg_data = NULL; /* MGWrapperData */
+jfieldID FID_query_result = NULL; /* MGQueryResult */
+jmethodID MID_addDoc = NULL; /* MGQueryResult.addDoc() */
+jmethodID MID_addTerm = NULL; /* MGQueryResult.addTerm() */
+jmethodID MID_setTotalDocs = NULL; /* MGQueryResult.setTotalDocs() */
+jmethodID MID_clearResult = NULL; /* MGQueryResult.clear() */
+
+
+/* to access objects and methods on java side, need their field/method ids - 
+ this initialises them at the start to avoid recalculating them each time they
+ are needed
+Note: the descriptors need to be exactly right, otherwise you get an error 
+saying "no such field" but no reference to the fact that it has the right 
+name but the wrong type.
+Note: apparently the jclass is a local ref and should only work 
+in the method that created it. It seems to work ok, but I'll make it
+global cos the book said I should, and it may avoid future hassles.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  jclass JC_MGQueryResult;
+
+  /* a long-"J" */
+  FID_mg_data = (*j_env)->GetFieldID(j_env, j_cls, "mg_data_ptr_", "J");
+  assert(FID_mg_data != NULL);
+
+  /* an object -"L<class name>;" */
+  FID_query_result = (*j_env)->GetFieldID(j_env, j_cls, "mg_query_result_",
+					  "Lorg/greenstone/mg/MGQueryResult;");
+  assert(FID_query_result != NULL);
+
+  /* the methods we want to use */
+  JC_MGQueryResult = (*j_env)->FindClass(j_env, "org/greenstone/mg/MGQueryResult");
+
+  /* addDoc(long doc, float rank) */
+  MID_addDoc = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addDoc", "(JF)V");
+  assert(MID_addDoc != NULL);
+
+  /* addTerm(String term, String tag, int stem, long match, long freq, String[] equivs) */
+  MID_addTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addTerm", "(Ljava/lang/String;Ljava/lang/String;IJJ[Ljava/lang/String;)V");
+  assert(MID_addTerm != NULL);
+
+  /* setTotalDocs(long) */
+  MID_setTotalDocs = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "setTotalDocs", "(J)V");
+  assert(MID_setTotalDocs != NULL);
+
+  /* clear(void) */
+  MID_clearResult = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "clear", "()V");
+  assert(MID_clearResult != NULL);
+}
+
+
+/* the java side MGWrapper has a pointer to a C object - MGWrapperData
+   initialise this and set the pointer 
+*/
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  /* Allocate a MGWrapperData object to store query parameters */
+  MGWrapperData* data = (MGWrapperData*) malloc(sizeof(MGWrapperData));
+  assert(data != NULL);
+
+  /* Set default values - no stemming, no case-folding, boolean OR queries */
+  data->defaultStemMethod = 0;
+  data->defaultBoolCombine = 0;
+
+  /* Allocate a QueryInfo object to store more query parameters */
+  data->queryInfo = (QueryInfo*) malloc(sizeof(QueryInfo));
+  assert(data->queryInfo != NULL);
+
+  /* Set default values - 50 documents max, return term freqs, sort by rank */
+  data->queryInfo->index = NULL;
+  data->queryInfo->maxDocs = 50;
+  data->queryInfo->needTermFreqs = 1;
+  data->queryInfo->sortByRank = 1;
+
+  /* Save the object on the Java side */
+  (*j_env)->SetIntField(j_env, j_obj, FID_mg_data, (long) data);
+
+  /* Initialise MG environment variables */
+  InitEnv();
+  SetEnv("expert", "true", NULL);
+  SetEnv("mode", "docnums", NULL);
+  return 1;  /* true - no errors */
+}
+
+
+/*******************************************
+   Index caching
+ *******************************************/
+
+query_data* cached_indexes[MAX_INDEXES_CACHED] = { NULL };
+
+
+/* Get the index data necessary to perform a query or document retrieval */
+query_data*
+loadIndexData(char* base_dir, char* index_path, char* text_path)
+{
+  char* index_path_name;
+  char* text_path_name;
+  query_data* qd;
+  int i = 0;
+
+  /* Form the path name of the desired indexes */
+  index_path_name = (char*) malloc(strlen(base_dir) + strlen(index_path) + 1);
+  assert(index_path_name != NULL);
+  strcpy(index_path_name, base_dir);
+  strcat(index_path_name, index_path);
+  printf("Index pathname: %s\n", index_path_name);
+
+  text_path_name = (char*) malloc(strlen(base_dir) + strlen(text_path) + 1);
+  assert(text_path_name != NULL);
+  strcpy(text_path_name, base_dir);
+  strcat(text_path_name, text_path);
+  printf("Text pathname: %s\n", text_path_name);
+
+  /* Search through the cached indexes for the desired one */
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    printf("(Cached) Pathname: %s\n", cached_indexes[i]->pathname);
+    printf("(Cached) Textpathname: %s\n", cached_indexes[i]->textpathname);
+
+    /* Check if the index has already been loaded */
+    if ((strcmp(index_path_name, cached_indexes[i]->pathname) == 0) &&
+	(strcmp(text_path_name, cached_indexes[i]->textpathname) == 0)) {
+      /* Index has already been loaded and cached, so return it */
+      printf("Found index!\n");
+      free(index_path_name);
+      free(text_path_name);
+      return cached_indexes[i];
+    }
+
+    i++;
+  }
+
+  /* Text strings no longer needed */
+  free(index_path_name);
+  free(text_path_name);
+
+  /* The index is not cached, so load it now */
+  qd = InitQuerySystem(base_dir, index_path, text_path, NULL);
+  if (!qd) {
+    printf("Error: Could not InitQuerySystem()...\n");
+    return NULL;
+  }
+
+  /* The index loaded OK, so cache it */
+  /* This could be a little more sophisticated, eg. replace least frequently used index */
+  if (i >= MAX_INDEXES_CACHED)
+    i = MAX_INDEXES_CACHED - 1;
+
+  /* Free the index being replaced */
+  if (cached_indexes[i] != NULL)
+    FinishQuerySystem(cached_indexes[i]);
+
+  /* Cache the loaded index, and return it */
+  cached_indexes[i] = qd;
+  return cached_indexes[i];
+}
+
+
+/* Clean up by unloading all cached indexes */
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_unloadIndexData(JNIEnv* j_env, jobject j_obj)
+{
+  /* Free all the loaded indexes */
+  int i = 0;
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    FinishQuerySystem(cached_indexes[i]);
+    cached_indexes[i] = NULL;
+    i++;
+  }
+
+  return 1;  /* true - no errors */
+}
+
+
+/****************************************************
+   retrieve a document
+ ****************************************************/
+
+/* Returns a document from mg as a string */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getDocument(JNIEnv *j_env, jobject j_obj,
+					     jstring j_base_dir, jstring j_text_path,
+					     jlong j_docnum)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  u_long pos, len;
+  u_char* c_buffer = NULL;
+  u_char* uc_buffer = NULL;
+  int ULen;
+
+  jstring result;
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return NULL;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return NULL;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  assert(qd != NULL);
+
+  /* Get the document position and length in the text file */
+  printf("Fetching document number %ld...\n", (unsigned long) j_docnum);
+  FetchDocStart(qd, (unsigned long) j_docnum, &pos, &len);
+  printf("Fetched document start. Pos: %ld, Len: %ld\n", pos, len);
+
+  /* Allocate memory for the document text (from mg/src/text/mgquery.c:RawDocOutput()) */
+  c_buffer = (u_char*) malloc(len);
+  assert(c_buffer != NULL);
+  uc_buffer = (u_char*) malloc((int) (qd->td->cth.ratio * 1.01 * len) + 100);
+  assert(uc_buffer != NULL);
+
+  /* Seek to the correct position in the file and read the document text */
+  Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+
+  /* Decode (?) the document text into another buffer, and terminate it */
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  uc_buffer[ULen] = '\0';
+
+  /* Load the document text into a Java string */
+  result = (*j_env)->NewStringUTF(j_env, uc_buffer);
+  assert(result != NULL);
+
+  /* Free C buffers */
+  free(c_buffer);
+  free(uc_buffer);
+
+  /* Return the document text */
+  return result;
+}
+
+
+/*******************************************
+   do a query
+ *******************************************/
+
+/* Comparison function for sorting documents by their weight (decreasing order) */
+int
+document_weight_comp(const void *A, const void *B)
+{
+  const DocEntry *a = A;
+  const DocEntry *b = B;
+
+  /* Compare on weight, highest wins */
+  if (a->Weight < b->Weight)
+    return  1;
+  if (a->Weight > b->Weight)
+    return -1;
+  return 0;
+}
+
+
+/* do the actual query - the results are written to query_result held on the Java side */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_runQuery(JNIEnv *j_env, jobject j_obj,
+					  jstring j_base_dir, jstring j_text_path,
+					  jstring j_query)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  jobject result_ptr;
+  char* query;
+  BooleanQueryInfo bqi;
+  int i, j;
+
+  jthrowable exc;
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  assert(qd != NULL);
+
+  /* Remove anything hanging around from last time */
+  FreeQueryDocs(qd);
+
+  /* The result to write to */
+  result_ptr = (*j_env)->GetObjectField(j_env, j_obj, FID_query_result);
+  assert(result_ptr != NULL);
+
+  /* Clear any previous query results */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_clearResult);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Obtain a C version of the query string */
+  query = (char*) (*j_env)->GetStringUTFChars(j_env, j_query, NULL);
+  assert(query != NULL);
+  printf("Searching for query \"%s\"...\n", query);
+
+  /* Make sure the query isn't empty */
+  if (strlen(query) == 0) {
+    printf("Warning: Empty query.\n");
+    return;
+  }
+
+  /* Boolean OR ("some") queries: must manually insert OR ("|") tokens */
+  if (data->defaultBoolCombine == 0) {
+    int in_space = 0;
+    for (i = 0; i < strlen(query); i++) {
+      if (in_space) {
+	if (query[i] == '|') /* OR character already inserted, so remove others */
+	  query[i] = ' ';
+	else if (!isspace(query[i])) /* moving out of a space region */
+	  in_space = 0;
+      }
+      else if (!in_space && isspace(query[i])) { /* moving into a space region */
+	in_space = 1;
+	query[i] = '|'; /* insert an OR character */
+      }
+    }
+    printf("Boolean OR query string: %s\n", query);
+  }
+
+  /* If the documents need to be sorted by rank, all of them must be retrieved */
+  bqi.MaxDocsToRetrieve = ((data->queryInfo->sortByRank) ? -1 : data->queryInfo->maxDocs);
+
+  /* Perform query */
+  /* Had to add "words$o" to LIB_OBJS in mg/src/text/Makefile and recompile mg for this */
+  BooleanQuery(qd, (char*) query, &bqi, data->defaultStemMethod);
+
+  /* Finished with the C query string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_query, query);
+
+  /* Check the query was processed successfully */
+  assert(qd->DL != NULL);
+  assert(qd->QTL != NULL);
+  assert(qd->TL != NULL);
+
+  /* Record the total number of matching documents */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_setTotalDocs, (jlong) qd->DL->num);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Sort the documents by rank, if desired */
+  if (data->queryInfo->sortByRank) {
+    qsort(qd->DL->DE, qd->DL->num, sizeof(DocEntry), document_weight_comp);
+  }
+
+  /* Record the matching documents, but only the number requested */
+  printf("Number of matching documents: %d\n", qd->DL->num);
+  for (i = 0; (i < qd->DL->num && i < data->queryInfo->maxDocs); i++) {
+    int doc_num = qd->DL->DE[i].DocNum;
+    float doc_weight = qd->DL->DE[i].Weight;
+
+    /* Call the addDoc function (Java side) to record a matching document */
+    (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addDoc,
+			     (jlong) doc_num, (jfloat) doc_weight);
+    exc = (*j_env)->ExceptionOccurred(j_env);
+    if (exc) {
+      (*j_env)->ExceptionDescribe(j_env);
+      return;
+    }
+  }
+
+  /* Record the term information, if desired */
+  if (data->queryInfo->needTermFreqs) {
+    printf("Number of terms: %d\n", qd->TL->num);
+    printf("Number of query terms: %d\n", qd->QTL->num);
+
+    /* Find each query term in the term list, and grab its frequency */
+    for (i = 0; i < qd->QTL->num; i++) {
+      unsigned int query_term_length = (unsigned int) qd->QTL->QTE[i].Term[0];
+      unsigned char* query_term;
+      jstring j_query_term;
+      jint stem = qd->QTL->QTE[i].stem_method;
+      jlong match;
+      jlong freq;
+
+      /* Allocate memory to store this query term, and fill it */
+      query_term = (unsigned char*) malloc(query_term_length + 1);
+      assert(query_term != NULL);
+      strncpy(query_term, &(qd->QTL->QTE[i].Term[1]), query_term_length);
+      query_term[query_term_length] = '\0';
+      printf("Query term: %s\n", query_term);
+
+      /* Allocate a new jstring for the query term */
+      j_query_term = (*j_env)->NewStringUTF(j_env, query_term);
+      assert(j_query_term != NULL);
+
+      /* Find the matching term in the term list */
+      for (j = 0; j < qd->TL->num; j++) {
+	unsigned int term_length = (unsigned int) qd->TL->TE[j].Word[0];
+
+	/* Stemming and case-folding mean both comparisons are necessary */
+	if ((strncmp(query_term, &(qd->TL->TE[j].Word[1]), term_length) == 0) &&
+	    (strncmp(query_term, &(qd->TL->TE[j].Word[1]), query_term_length) == 0)) {
+	  /* Get the document count and total frequency of the term */
+	  match = qd->TL->TE[j].WE.doc_count;
+	  freq = qd->TL->TE[j].WE.count;
+
+	  /* Call the addTerm function (Java side) to record term information */
+	  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addTerm,
+				   j_query_term, NULL, stem, match, freq, NULL);
+	  exc = (*j_env)->ExceptionOccurred(j_env);
+	  if (exc) {
+	    (*j_env)->ExceptionDescribe(j_env);
+	    return;
+	  }
+
+	  /* There can only be one match */
+	  break;
+	}
+      }
+
+      /* Finished with this query term */
+      free(query_term);
+    }
+  }
+}
+
+
+/*******************************************
+   set query options
+ *******************************************/
+
+/* Turn casefolding on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setCase(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 1;
+  } else {
+    data->defaultStemMethod &= 0xe;
+  }
+}
+
+
+/* Turn stemming on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setStem(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 2;
+  } else {
+    data->defaultStemMethod &= 0xd;
+  }
+}
+
+
+/* Set the maximum number of documents to return from a query */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setMaxDocs(JNIEnv *j_env, jobject j_obj, 
+					    jint j_max)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->maxDocs = j_max;
+}
+
+
+/* Turn sorting by rank on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setSortByRank(JNIEnv *j_env, jobject j_obj, 
+                                               jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);  
+  data->queryInfo->sortByRank = j_on;
+}
+
+
+/* Turn term frequency recording on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setReturnTerms(JNIEnv *j_env, jobject j_obj, 
+						jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->needTermFreqs = j_on;
+}
+
+
+/* Choose MG index to search */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setIndex(JNIEnv *j_env, jobject j_obj,
+					  jstring j_index)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  /* Get the index name as a C string */
+  const char* index = (*j_env)->GetStringUTFChars(j_env, j_index, NULL);
+  assert(index != NULL);
+  printf("Choosing index %s...\n", index);
+
+  /* Free the previous index name */
+  if (data->queryInfo->index)
+    free(data->queryInfo->index);
+
+  /* Allocate memory for the index name, and fill it */
+  data->queryInfo->index = (char*) malloc(strlen(index) + 1);
+  assert(data->queryInfo->index != NULL);
+  strcpy(data->queryInfo->index, index);
+
+  /* Release the index string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_index, index);
+}
+
+
+/* Choose boolean AND or boolean OR queries */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mg_MGWrapper_setMatchMode(JNIEnv *j_env, jobject j_obj,
+					      jint j_mode)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->defaultBoolCombine = j_mode;
+}
+
+
+/* Get a text representation of the current parameter values */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getQueryParams(JNIEnv *j_env, jobject j_obj)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  char result[512];  /* Assume this is big enough */
+
+  /* Print the data to a character array */
+  sprintf(result, "Query params:\nindex\t\t%s\ncasefold\t%d\nstem\t\t%d\norder by rank\t%d\nquery type\t%s\nmax docs\t%d\n",
+	  (data->queryInfo->index == NULL ? "<none loaded>" : data->queryInfo->index),
+	  (data->defaultStemMethod & 1),
+	  (data->defaultStemMethod & 2),
+	  (data->queryInfo->sortByRank),
+	  (data->defaultBoolCombine == 1 ? "all" : "some"),
+	  (data->queryInfo->maxDocs));
+
+  /* Convert to a jstring, and return */
+  return (*j_env)->NewStringUTF(j_env, result);
+}
Index: /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/jni/MGWrapperImpl.h	(revision 21112)
@@ -0,0 +1,55 @@
+/*
+ *    MGWrapperImpl.h
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "backend.h"
+
+
+/*************************************************************************
+   NOTES
+
+ - These data structures are based on the MGPP ones but are modified
+   slightly to reflect the different capabilities of the MG system.
+
+ *************************************************************************/
+
+ 
+typedef struct QueryInfo
+{
+  /* Name of index to use */
+  char* index;
+  /* Maximum number of documents to retrieve */
+  unsigned long maxDocs;
+  /* Whether to sort the matching documents by weight (boolean value) */
+  int sortByRank;
+
+  /* Whether term frequency information is desired (boolean value) */
+  int needTermFreqs;
+} QueryInfo;
+
+
+typedef struct MGWrapperData
+{
+  /* Information about a query, see above */
+  QueryInfo* queryInfo;
+  /* Whether to perform stemming and case-folding */
+  int defaultStemMethod;
+  /* Whether to perform boolean AND (1) or boolean OR (0) queries */
+  int defaultBoolCombine;
+} MGWrapperData;
Index: /main/tags/3.01/gsdl/packages/mg/jni/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,62 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = jni
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX -DNZDL -DQUIET -DPARADOCNUM
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#NOTE: libtextin.a needs to come before libmg.a !!!
+LIBS = ../src/text/libtextin.a ../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+CCOMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CFLAGS)
+
+.SUFFIXES:
+
+.SUFFIXES: .c .o
+.c.o:
+	$(CCOMPILE) $<
+
+
+INCLUDES = -I.. -I../lib -I../src/text -I/usr/local/share/j2sdk1.4.0/include -I/usr/local/share/j2sdk1.4.0/include/linux
+
+all: compile link
+
+compile:
+	$(CCOMPILE) MGWrapperImpl.c
+
+link:
+	$(CC) -shared -o libmgjni.so MGWrapperImpl.o $(LIBS)
+
+clean:
+	rm -rf *.o libmgjni.so 
+
+distclean: clean
+	rm -f Makefile
+
+install:
+	../mkinstalldirs $(libdir); \
+	cp libmgjni.so  $(libdir)/;
+
+
+
Index: /main/tags/3.01/gsdl/packages/mg/jni/org_greenstone_mg_MGWrapper.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
@@ -0,0 +1,117 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mg_MGWrapper */
+
+#ifndef _Included_org_greenstone_mg_MGWrapper
+#define _Included_org_greenstone_mg_MGWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getDocument
+ * Signature: (Ljava/lang/String;Ljava/lang/String;J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getDocument
+  (JNIEnv *, jobject, jstring, jstring, jlong);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    unloadIndexData
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_unloadIndexData
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setCase
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setCase
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setStem
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setStem
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMaxDocs
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMaxDocs
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setSortByRank
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setSortByRank
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setReturnTerms
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setReturnTerms
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setIndex
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setIndex
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMatchMode
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMatchMode
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getQueryParams
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getQueryParams
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    runQuery
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_runQuery
+  (JNIEnv *, jobject, jstring, jstring, jstring);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/GCC.mak
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/GCC.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/GCC.mak	(revision 21112)
@@ -0,0 +1,84 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+
+CPPFLAGS = -g
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I".." -I"."
+
+o = .o
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+
+
+HEADERS = \
+    bitio_m_random.h filestats.h local_strings.h lovinstem.h \
+    bitio_m_stdio.h getopt.h memlib.h bitio_gen.h bitio_mem.h \
+    mgheap.h messages.h bitio_m.h bitio_mems.h huffman.h \
+    timing.h bitio_m_mem.h bitio_random.h huffman_mem.h \
+    perf_hash.h bitio_m_mems.h bitio_stdio.h huffman_stdio.h \
+    sptree.h rx.h pathmax.h getpagesize.h random.h \
+    win32in.h simplefrenchstem.h unitool.h
+
+SOURCES = \
+    bitio_random.c mgheap.c memlib.c bitio_stdio.c huffman.c \
+    messages.c bitio_gen.c filestats.c huffman_mem.c \
+    perf_hash.c bitio_mem.c getopt.c huffman_stdio.c \
+    sptree.c bitio_mems.c getopt1.c local_strings.c \
+    lovinstem.c timing.c rx.c alloca.c error.c xmalloc.c \
+    strstr.c ftruncate.c strcasecmp.c random.c win32in.c \
+    simplefrenchstem.c unitool.h
+
+OBJECTS = \
+    rx$o bitio_random$o mgheap$o memlib$o bitio_stdio$o \
+    huffman$o messages$o bitio_gen$o filestats$o huffman_mem$o \
+    perf_hash$o bitio_mem$o getopt$o huffman_stdio$o sptree$o \
+    bitio_mems$o getopt1$o local_strings$o lovinstem$o \
+    timing$o error$o xmalloc$o random$o win32in$o \
+    simplefrenchstem$o unitool$o
+ 
+all : libmg.a
+
+libmg.a : $(OBJECTS)
+	$(DELETE) libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+install: 
+ 
+clean: 
+	$(DELETE) *$o
+	$(DELETE) libmg.a
Index: /main/tags/3.01/gsdl/packages/mg/lib/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,174 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DQUIET
+##########################################
+
+ 
+prefix = @prefix@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+ 
+INCLUDES = -I../../.. -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+    pathmax.h          getpagesize.h \
+    random.h           simplefrenchstem.h unitool.h  longlong.h
+# regex.h            rx.h               
+
+
+SOURCES = \
+    bitio_random.c    mgheap.c            memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c  \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    gmalloc.c         ftruncate.c       strcasecmp.c      random.c \
+    simplefrenchstem.c                  unitool.c
+# regex.c           rx.c              
+
+OBJECTS = @ALLOCA@ @LIBOBJS@ \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+#   gmalloc$o regex$o		rx$o 
+
+LIB_OBJECTS =   \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o          \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.a gsdllibmg.a
+
+libmg.a : $(OBJECTS)
+	rm -f libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+$(OBJECTS): ../../../config.h
+
+gsdllibmg.a: $(LIB_OBJECTS)
+	rm -f gsdllibmg.a
+	$(AR) cru gsdllibmg.a $(LIB_OBJECTS)
+	$(RANLIB) gsdllibmg.a
+ 
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmg.a gsdllibmg.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl/packages/mg/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,104 @@
+###########################################################################
+#
+# win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+GSDLHOME = ..\..\..
+DLL = 0
+DLLDEBUG = 0
+
+AR = lib
+CC = cl
+
+CPPFLAGS = -GX
+!IF $(DLL)
+CPPFLAGS = $(CPPFLAGS) -MD
+!ELSE
+!IF ($(DLLDEBUG))
+CPPFLAGS = $(CPPFLAGS) -MDd
+!ENDIF
+!ENDIF
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN -DPARADOCNUM \
+       -DSHORT_SUFFIX
+INCLUDES = -I"$(GSDLHOME)\packages\mg\lib" -I"$(GSDLHOME)\packages\mg" \
+           -I"$(GSDLHOME)"
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+                       rx.h               pathmax.h          getpagesize.h \
+    random.h           win32in.h          simplefrenchstem.h unitool.h \
+    longlong.h	
+
+SOURCES = \
+    bitio_random.c    mgheap.c          memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c                            rx.c              \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    ftruncate.c       strcasecmp.c      random.c          win32in.c \
+    simplefrenchstem.c                  unitool.h
+
+OBJECTS =   rx$o \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o           \
+    error$o           xmalloc$o \
+    random$o          win32in$o         simplefrenchstem$o \
+    unitool$o
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.lib
+
+libmg.lib : $(OBJECTS)
+	if EXIST libmg.lib del libmg.lib
+	$(AR) /out:libmg.lib $(OBJECTS)
+
+$(OBJECTS): "$(GSDLHOME)\win32cfg.h"
+
+install: 
+clean:
+	del *$o 
+	del libmg.lib
Index: /main/tags/3.01/gsdl/packages/mg/lib/alloca.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+
+
+
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BINARY_LENGTH (val, b, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Gamma_Length (unsigned long val)
+{
+  register unsigned long num;
+  GAMMA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  DELTA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  ELIAS_LENGTH (val, b, s, num);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BBLOCK_LENGTH (val, b, num);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  FLOORLOG_2 ((N - p) / p, logb);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  CEILLOG_2 (b, clogb);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+/* copied from mg-1.3f, 17-09-01 kjm18 */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  /* return ((int) (p * (2 * log2 ((double) N / p) + 1))); */
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  FLOORLOG_2 (b, logb);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  CEILLOG_2 (b, logb);
+  return logb;
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,63 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_m.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_m.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_m.h	(revision 21112)
@@ -0,0 +1,543 @@
+/**************************************************************************
+ *
+ * bitio_m.h -- Macros for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_M
+#define H_BITIO_M
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#ifndef BIO_ENCODE_PROLOGUE
+#define BIO_ENCODE_PROLOGUE
+#endif
+
+#ifndef BIO_DECODE_PROLOGUE
+#define BIO_DECODE_PROLOGUE
+#endif
+
+#ifndef BIO_ENCODE_EPILOGUE
+#define BIO_ENCODE_EPILOGUE
+#endif
+
+#ifndef BIO_DECODE_EPILOGUE
+#define BIO_DECODE_EPILOGUE
+#endif
+
+#ifndef DECODE_ADD
+#define DECODE_ADD(b) (b) += (b) + DECODE_BIT
+#endif
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#define POSITIVE(f, x)							\
+  if ((x)<=0) 								\
+    fprintf(stderr,"Error: Cannot "#f" encode %lu\n",(unsigned long)x),exit(1);
+
+
+#define CEILLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x) - 1;						\
+  (v) = 0;								\
+  for (; _B_x ; _B_x>>=1, ++(v));					\
+} while(0)
+
+
+#define FLOORLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x);						\
+  (v) = -1;								\
+  for (; _B_x ; _B_x>>=1, ++(v));					\
+} while(0)
+
+
+/****************************************************************************/
+
+
+
+#define UNARY_ENCODE(x)							\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  (count) += _B_x;							\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE(x)							\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) ++(x);						\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE_L(x, count)					\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) ++(x);						\
+  (count) += (x);							\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+
+#define UNARY_LENGTH(x, count)						\
+do {									\
+  POSITIVE(unary, x);							\
+  (count) = (x);							\
+} while(0)
+
+
+/****************************************************************************/
+
+
+#define BINARY_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  (count) += _B_nbits;							\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+
+#define BINARY_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      --_B_logofb;							\
+      for (_B_i=0; _B_i < _B_logofb; ++_B_i)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      --_B_logofb;							\
+      (count) += _B_logofb;					       	\
+      for (_B_i=0; _B_i < _B_logofb; ++_B_i)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+          ++(count);							\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    _B_nbits = _B_logofb;						\
+  (count) = _B_nbits;							\
+} while(0)
+
+/****************************************************************************/
+
+
+
+
+#define GAMMA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  (count) += _B_logofb*2+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) ++_B_nbits;					\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) ++_B_nbits;					\
+  (count) += _B_nbits*2+1;						\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  (count) = _B_logofb*2+1;						\
+} while (0)
+
+
+
+/****************************************************************************/
+
+
+#define DELTA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE(_B_logx+1);						\
+  BINARY_ENCODE(_B_xxx+1, 1<<_B_logx);					\
+} while (0)
+
+#define DELTA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE_L(_B_logx+1, count);					\
+  BINARY_ENCODE_L(_B_xxx+1, 1<<_B_logx, count);				\
+} while (0)
+
+
+#define DELTA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE(_B_logx); --_B_logx;					\
+  BINARY_DECODE(_B_xxx, 1<<_B_logx); --_B_xxx;				\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE_L(_B_logx, count); --_B_logx;				\
+  BINARY_DECODE_L(_B_xxx, 1<<_B_logx, count); --_B_xxx;			\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx, _B_dcount;					\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_LENGTH(_B_logx+1, count);					\
+  BINARY_LENGTH(_B_xxx+1, 1<<_B_logx, _B_dcount);			\
+  (count) += _B_dcount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+
+
+#define ELIAS_ENCODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      ++_B_k;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE(_B_k+1);							\
+  BINARY_ENCODE(_B_xx-_B_lower+1, _B_upper-_B_lower+1);			\
+} while (0)
+
+#define ELIAS_ENCODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      ++_B_k;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE_L(_B_k+1, count);					\
+  BINARY_ENCODE_L(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);	\
+} while (0)
+
+#define ELIAS_DECODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE(_B_k); --_B_k;						\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE(_B_xx, _B_upper-_B_lower+1); --_B_xx;			\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_DECODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE_L(_B_k, count); --_B_k;					\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE_L(_B_xx, _B_upper-_B_lower+1, count); --_B_xx;		\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_LENGTH(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0, _B_ecount;					\
+  register double _B_pow=1.0;						\
+  POSITIVE(gamma, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      ++_B_k;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_LENGTH(_B_k+1, count);						\
+  BINARY_LENGTH(_B_xx-_B_lower+1, _B_upper-_B_lower+1, _B_ecount);	\
+  (count) += _B_ecount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+#define BBLOCK_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  --_B_xx;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      ++_B_xdivb;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE(_B_xdivb+1);						\
+  BINARY_ENCODE(_B_xx+1, _B_bb);					\
+} while (0)
+
+#define BBLOCK_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  --_B_xx;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      ++_B_xdivb;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE_L(_B_xdivb+1, count);					\
+  BINARY_ENCODE_L(_B_xx+1, _B_bb, count);				\
+} while (0)
+
+#define BBLOCK_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE(_B_xdivb); --_B_xdivb;					\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE(_B_x1, _B_bb);						\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE_L(_B_xdivb, count); --_B_xdivb;				\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE_L(_B_x1, _B_bb, count);					\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_bcount, _B_xx = (x);			\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  --_B_xx;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      ++_B_xdivb;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_LENGTH(_B_xdivb+1, count);					\
+  BINARY_LENGTH(_B_xx+1, _B_bb, _B_bcount);				\
+  (count) += _B_bcount;  						\
+} while (0)
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,191 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    --__btg;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+	    *__pos++ = __buff;						\
+	    --__remaining;						\
+	  }  								\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_FLUSH							\
+  do {									\
+    if (__btg != sizeof(__buff)*8)					\
+      if (__remaining)							\
+	{								\
+          *__pos++ = __buff;						\
+          --__remaining;						\
+	}  								\
+    __btg = sizeof(__buff)*8;						\
+  } while (0)
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    --__remaining;						\
+	  }								\
+        else								\
+          __buff = 0xff;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    --__remaining;						\
+	  }								\
+        else								\
+          __buff = 0x00;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (!__remaining ? 							\
+    (DECODE_ERROR, 0) :							\
+    (__buff = *__pos++, __remaining--, __btg = sizeof(__buff)*8, 	\
+     (__buff >> --__btg) & 1)))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    __pos = __base + (__new_pos >> 3);					\
+    __buff = *__pos++;							\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+#define ENCODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __base[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __base[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    ++__pos;								\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define ENCODE_FLUSH
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    (b) += (b) + (__base[__pos>>3] & (0x80 >> (__pos&7)) != 0);		\
+    ++__pos;								\
+  } while(0)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_BIT 							\
+    (__pos++, ((__base[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define DECODE_SEEK(pos) __pos = (pos)
+
+#define ENCODE_SEEK(pos) __pos = (pos)
+
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,168 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+#define ENCODE_START(f,l)						\
+  {									\
+    register FILE *__file = f;						\
+    register unsigned char *__buf;					\
+    register unsigned long __pos = 0;					\
+    register unsigned long __base = 0;					\
+    register unsigned long __used = 0;					\
+    register unsigned long __len = (l)-1;				\
+    register unsigned long __sft = 0;					\
+    while (__len) { ++__sft; __len >>=1; }				\
+    __len = 1<<__sft;							\
+    __buf = Xmalloc(__len);						\
+    fseek(__file, 0, 0);						\
+    fread(__buf, 1, __len, __file);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register FILE *__file = (b).File;					\
+    register unsigned char *__buf = (b).Buf;				\
+    register unsigned long __pos = (b).pos;				\
+    register unsigned long __base = (b).Base;				\
+    register unsigned long __used = (b).Used;				\
+    register unsigned long __len = (b).len;				\
+    register unsigned long __sft = (b).sft;
+
+#define SEEK fprintf(stderr, "Seek to %d\n",__base)
+#define READ fprintf(stderr, "Read of %d\n",__len)
+#define WRITE fprintf(stderr, "Write of %d\n",__used)
+
+#define WRITE_READ 							\
+	(__used ? (fseek(__file, __base, 0),				\
+                   fwrite(__buf, 1, __len, __file)) : 0,		\
+        __base += __len,						\
+	fseek(__file, __base, 0),					\
+        fread(__buf, 1, __len, __file),					\
+	__pos = 0, __used = 0)
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __buf[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __buf[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __used = 1;								\
+    ++__pos;								\
+    if ((__pos>>3) >= __len)						\
+      (void)WRITE_READ;							\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __file;							\
+    (b).Buf = __buf;							\
+    (b).pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Used = __used;							\
+    (b).len = __len;							\
+    (b).sft = __sft;							\
+  }
+
+#define ENCODE_FLUSH							\
+  if (__used)								\
+    {									\
+      fseek(__file, __base, 0);						\
+      fwrite(__buf, 1, __len, __file);					\
+      __used = 0;							\
+    }
+
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+    free(__buf);							\
+  }
+
+
+#define DECODE_START(f,l) ENCODE_START(f,l)
+
+#define DECODE_CONTINUE(b) ENCODE_CONTINUE(b)
+
+#define DECODE_BIT 							\
+    (__pos++, (((__pos-1)>>3) >= __len ? WRITE_READ, __pos=1 : 0),	\
+     ((__buf[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_ADD_FF(b) ((b) += (b) + DECODE_BIT)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_DONE ENCODE_DONE
+
+#define DECODE_PAUSE(b)	ENCODE_PAUSE(b)
+
+#define ENCODE_SEEK(pos) 						\
+  do {									\
+    if ((((pos)>>3) >= __base) && ((((pos)+7)>>3) < (__base+__len)))   	\
+      __pos = (pos)-(__base << 3); 					\
+    else								\
+      {									\
+        ENCODE_FLUSH;							\
+        __base = (pos) >> 3;						\
+        __base = ((pos)  >> (__sft+3)) << __sft;			\
+        fseek(__file, __base, 0);					\
+        fread(__buf, 1, __len, __file);					\
+        __pos = (pos)&7; 						\
+        __pos = (pos)&((8 << __sft)-1);					\
+      }									\
+  } while(0)
+
+
+#define DECODE_SEEK(pos) ENCODE_SEEK(pos)
+
+#define ENCODE_TELL ((__base << 3) + __pos)
+
+#define DECODE_TELL ENCODE_TELL
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,147 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(f)							\
+  {									\
+    FILE *__outfile = f;						\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    FILE *__outfile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    --__btg;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	putc(__buff, __outfile);					\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __outfile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_DONE							\
+    if (__btg != sizeof(__buff)*8)					\
+      putc(__buff,__outfile);						\
+  }
+
+
+#define DECODE_START(f)							\
+  {									\
+    FILE *__infile = f;							\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    FILE *__infile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (__buff = getc(__infile), (feof(__infile) ?				\
+    (DECODE_ERROR, 0) :							\
+    (__btg = sizeof(__buff)*8, (__buff >> --__btg) & 1))))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).File = __infile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    fseek(__infile, __new_pos >> 3, 0);					\
+    __buff = getc(__infile);						\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.c	(revision 21112)
@@ -0,0 +1,256 @@
+/**************************************************************************
+ *
+ * bitio_mem.c -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  ENCODE_START (buf, rem)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mem_Encode_Done (mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  DECODE_START (buf, rem)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Elias_Decode (unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_mem.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * bitio_mem.h -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are faster than the ones 
+ *  in "mems" files. But with these routines you cannot mix reads and writes 
+ *  on the array of chars, or multiple write, at the same time and guarantee
+ *  them to work, also you cannot seek to a point and do a write. The decode
+ *  routine can detect when you run off the end of the array and will produce
+ *  an approate error message, and the encode routine will stop when it gets
+ *  to the end of the character array. 
+ *
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_MEM
+#define H_BITIO_MEM
+
+
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs);
+void BIO_Mem_Encode_Done (mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+				    unsigned long *bits);
+
+
+
+void BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+			   mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Elias_Decode (unsigned long b, double s,
+				 mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+void BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.c	(revision 21112)
@@ -0,0 +1,265 @@
+/**************************************************************************
+ *
+ * bitio_mems.c -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mems.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mems_Encode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  ENCODE_START (buf, pos)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Done (mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mems_Decode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  DECODE_START (buf, pos)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Elias_Decode (unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_mems.h	(revision 21112)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * bitio_mems.h -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_MEMS
+#define H_BITIO_MEMS
+
+
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mems_Encode_Start (char *buf, unsigned long pos, mems_bitio_state * bs);
+void BIO_Mems_Encode_Done (mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Decode_Start (void *buf, unsigned long pos, mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+			    mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Elias_Decode (unsigned long b, double s,
+				mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.c	(revision 21112)
@@ -0,0 +1,328 @@
+/*************************************************************************
+ *
+ * bitio_random.c -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2001/09/21 12:45:03  kjm18
+   updated mg to be in line with version mg-1.3f. Now uses long long for
+   some variables to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:16:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:50  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:40  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "longlong.h"
+
+#include "bitio_m_random.h"
+#include "bitio_m.h"
+
+
+void 
+BIO_Random_Start (FILE * f, unsigned long len,
+		  random_bitio_state * bs)
+{
+  ENCODE_START (f, len)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Random_Done (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+void 
+BIO_Random_Encode_Bit (int bit, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+int 
+BIO_Random_Decode_Bit (random_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+void 
+BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Unary_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Binary_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Gamma_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Delta_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			 random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Elias_Decode (unsigned long b, double s,
+			 random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Bblock_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Seek (unsigned long pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
+void 
+BIO_Random_Flush (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_FLUSH;
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Random_Tell (random_bitio_state * bs)
+{
+  register unsigned long t;
+  ENCODE_CONTINUE (*bs)
+    t = ENCODE_TELL;
+  ENCODE_PAUSE (*bs)
+    return (t);
+}
+
+#ifdef USE_LONG_LONG
+
+void 
+BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE(*bs)
+
+    if ((((pos) >> 3) >= (mg_ullong)__base) &&
+	((((pos)+7) >> 3) < (mg_ullong)(__base + __len)))
+      {
+	__pos = (long)((pos) - (mg_ullong)(__base << 3));
+      }
+    else
+      {
+        ENCODE_FLUSH;
+	__base = (long)(((pos) >> (__sft+3)) << __sft);
+	
+	fseek(__file,__base,0);
+	fread(__buf,1,__len,__file);
+	__pos = (long)((pos) & ((8 << __sft)-1));
+      }
+  
+  ENCODE_PAUSE(*bs)
+}
+
+mg_ullong 
+BIO_Random_Tell_LL (random_bitio_state * bs)
+{
+  mg_ullong t;
+  ENCODE_CONTINUE(*bs)
+    t = (((mg_ullong)__base) << 3ull) + __pos;
+  ENCODE_PAUSE(*bs)
+  return(t);  
+}
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_random.h -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ *  Modified:
+ *   - long long seek and tell ops
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ **************************************************************************/
+
+#ifndef H_BITIO_RANDOM
+#define H_BITIO_RANDOM
+
+#include "longlong.h"
+
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Random_Start (FILE * f, unsigned long len,
+		       random_bitio_state * bs);
+void BIO_Random_Done (random_bitio_state * bs);
+
+
+
+void BIO_Random_Decode_Start (void *buf, unsigned long pos,
+			      random_bitio_state * bs);
+
+void BIO_Random_Encode_Bit (int bit, random_bitio_state * bs);
+
+int BIO_Random_Decode_Bit (random_bitio_state * bs);
+
+
+void BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Unary_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Binary_Decode (unsigned long b, random_bitio_state * bs,
+					unsigned long *bits);
+
+
+
+void BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Gamma_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Delta_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			      random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Elias_Decode (unsigned long b, double s,
+				       random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Bblock_Decode (unsigned long b,
+					random_bitio_state * bs,
+					unsigned long *bits);
+
+
+void BIO_Random_Seek (unsigned long pos, random_bitio_state * bs);
+
+void BIO_Random_Flush (random_bitio_state * bs);
+
+unsigned long BIO_Random_Tell (random_bitio_state * bs);
+
+#ifdef USE_LONG_LONG
+
+void BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs);
+mg_ullong BIO_Random_Tell_LL (random_bitio_state * bs);
+
+#endif /* USE_LONG_LONG */
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.c	(revision 21112)
@@ -0,0 +1,281 @@
+/**************************************************************************
+ *
+ * bitio_stdio.c -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:41  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_stdio.h"
+#include "bitio_m.h"
+
+
+
+void 
+BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  ENCODE_START (f)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Done (stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  DECODE_START (f)
+    DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+
+int 
+BIO_Stdio_Decode_Bit (stdio_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Elias_Decode (unsigned long b, double s,
+			stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+void 
+BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/bitio_stdio.h	(revision 21112)
@@ -0,0 +1,106 @@
+/**************************************************************************
+ *
+ * bitio_stdio.h -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of number on a FILE*. With these function you cannot mix reads and writes
+ *  on the FILE, or multiple writes, at the same time and guarantee them to 
+ *  work, also you  cannot seek to a point and do a write. The decode function
+ *  can detect when you run off the end of the file and will produce an 
+ *  approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_STDIO
+#define H_BITIO_STDIO
+
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs);
+void BIO_Stdio_Encode_Done (stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs);
+int BIO_Stdio_Decode_Bit (stdio_bitio_state * bs);
+
+
+void BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			     stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Elias_Decode (unsigned long b, double s,
+				      stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/error.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/error.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/error.c	(revision 21112)
@@ -0,0 +1,127 @@
+/* error.c -- error handler for noninteractive utilities
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by David MacKenzie <djm@gnu.ai.mit.edu>.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <stdio.h>
+
+#if HAVE_VPRINTF || HAVE_DOPRNT
+# if __STDC__
+#  include <stdarg.h>
+#  define VA_START(args, lastarg) va_start(args, lastarg)
+# else
+#  include <varargs.h>
+#  define VA_START(args, lastarg) va_start(args)
+# endif
+#else
+# define va_alist a1, a2, a3, a4, a5, a6, a7, a8
+# define va_dcl char *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8;
+#endif
+
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <string.h>
+#else
+void exit ();
+#endif
+
+/* If NULL, error will flush stdout, then print on stderr the program
+   name, a colon and a space.  Otherwise, error will call this
+   function without parameters instead.  */
+void (*error_print_progname) () = NULL;
+
+/* The calling program should define program_name and set it to the
+   name of the executing program.  */
+#if 0 /* [RB/TS:10/95] do not use program_name */
+extern char *program_name;
+#endif
+
+#if HAVE_STRERROR
+char *strerror (int errnum);
+#else
+static char *
+private_strerror (errnum)
+     int errnum;
+{
+  extern char *sys_errlist[];
+  extern int sys_nerr;
+
+  if (errnum > 0 && errnum <= sys_nerr)
+    return sys_errlist[errnum];
+  return "Unknown system error";
+}
+#define strerror private_strerror
+#endif
+
+/* Print the program name and error message MESSAGE, which is a printf-style
+   format string with optional args.
+   If ERRNUM is nonzero, print its corresponding system error message.
+   Exit with status STATUS if it is nonzero.  */
+/* VARARGS */
+
+void
+#if defined(VA_START) && __STDC__
+error (int status, int errnum, const char *message, ...)
+#else
+error (status, errnum, message, va_alist)
+     int status;
+     int errnum;
+     char *message;
+     va_dcl
+#endif
+{
+#ifdef VA_START
+  va_list args;
+#endif
+
+  if (error_print_progname)
+    (*error_print_progname) ();
+  else
+    {
+      fflush (stdout);
+#if 0 /* [RB/TS:10/95] do not use program_name */
+      fprintf (stderr, "%s: ", program_name);
+#endif
+    }
+
+#ifdef VA_START
+  VA_START (args, message);
+# if HAVE_VPRINTF
+  vfprintf (stderr, message, args);
+# else
+  _doprnt (message, args, stderr);
+# endif
+  va_end (args);
+#else
+  fprintf (stderr, message, a1, a2, a3, a4, a5, a6, a7, a8);
+#endif
+
+  if (errnum)
+    fprintf (stderr, ": %s", strerror (errnum));
+  putc ('\n', stderr);
+  fflush (stderr);
+  if (status)
+    exit (status);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/filestats.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/filestats.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/filestats.c	(revision 21112)
@@ -0,0 +1,133 @@
+/**************************************************************************
+ *
+ * filestats.c -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:54  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:42  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+File *
+Fopen (char *name, char *mode, unsigned long magic)
+{
+  FILE *f;
+  File *F;
+  unsigned long m;
+  if (!(f = fopen (name, mode)))
+    return (NULL);
+  if (magic)
+    switch (*mode)
+      {
+      case 'r':
+	fread ((char *) &m, sizeof (m), 1, f);
+	NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (m != magic)
+	  {
+	    fclose (f);
+	    return (NULL);
+	  }
+	break;
+      case 'w':
+	HTONUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+	fwrite ((char *) &magic, sizeof (magic), 1, f);
+      }
+  if (!(F = Xmalloc (sizeof (File))))
+    {
+      fclose (f);
+      return (NULL);
+    }
+  F->pathname = Xstrdup (name);
+  F->name = strrchr (F->pathname, '/');
+  F->name = F->name ? F->name + 1 : F->pathname;
+  F->f = f;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+  F->Cumulative = F->Current;
+  return (F);
+}
+
+
+int 
+Fclose (File * F)
+{
+  int num;
+  if (!F)
+    return (0);
+  num = fclose (F->f);
+  if (F->pathname)
+    Xfree (F->pathname);
+  Xfree (F);
+  return (num);
+}
+
+size_t 
+Fread (void *ptr, size_t size, size_t nitems, File * F)
+{
+  int num;
+  num = fread ((char *) ptr, size, nitems, F->f);
+  F->Current.NumReads++;
+  F->Current.NumBytes += num * size;
+  return (num);
+}
+
+int 
+Fseek (File * F, long offset, int ptrname)
+{
+  int num;
+  num = fseek (F->f, offset, ptrname);
+  F->Current.NumSeeks++;
+  return (num);
+}
+
+void 
+Rewind (File * F)
+{
+  rewind (F->f);
+  F->Current.NumSeeks++;
+}
+
+
+void 
+ZeroFileStats (File * F)
+{
+  F->Cumulative.NumSeeks += F->Current.NumSeeks;
+  F->Cumulative.NumReads += F->Current.NumReads;
+  F->Cumulative.NumBytes += F->Current.NumBytes;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/filestats.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/filestats.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/filestats.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filestats.h -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_FILESTATS
+#define H_FILESTATS
+
+#include "sysfuncs.h"
+
+typedef struct FileStats
+  {
+    unsigned long NumSeeks;
+    unsigned long NumReads;
+    unsigned long NumBytes;
+  }
+FileStats;
+
+
+
+typedef struct
+  {
+    FILE *f;
+    char *pathname;
+    char *name;
+    FileStats Current;
+    FileStats Cumulative;
+  }
+File;
+
+/* if magic is 0 no magic number is read or written */
+File *Fopen (char *name, char *mode, unsigned long magic);
+
+size_t Fread (void *ptr, size_t size, size_t nitems, File * F);
+
+int Fseek (File * F, long offset, int ptrname);
+
+void Rewind (File * F);
+
+int Fclose (File * F);
+
+#define Getc(F) (F->Current.NumReads++, F->Current.NumBytes++, getc(F->f))
+
+/* This adds the Current file stats to the Cumulative stats and then zeros
+   the Current stats */
+void ZeroFileStats (File * F);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/ftruncate.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/ftruncate.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/ftruncate.c	(revision 21112)
@@ -0,0 +1,107 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/gsdl/packages/mg/lib/getopt.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/getopt.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/getopt.c	(revision 21112)
@@ -0,0 +1,750 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "getopt.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/gsdl/packages/mg/lib/getopt.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/getopt.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/getopt.h	(revision 21112)
@@ -0,0 +1,130 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  extern int getopt ();
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
Index: /main/tags/3.01/gsdl/packages/mg/lib/getopt1.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/getopt1.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/getopt1.c	(revision 21112)
@@ -0,0 +1,178 @@
+/* getopt_long and getopt_long_only entry points for GNU getopt.
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#include "getopt.h"
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef __GNU_LIBRARY__
+#include <stdlib.h>
+#else
+char *getenv ();
+#endif
+
+#ifndef	NULL
+#define NULL 0
+#endif
+
+int
+getopt_long (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 0);
+}
+
+/* Like getopt_long, but '-' as well as '--' can indicate a long option.
+   If an option that starts with '-' (not '--') doesn't match a long option,
+   but does match a short option, it is parsed as a short option
+   instead.  */
+
+int
+getopt_long_only (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 1);
+}
+
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+#include <stdio.h>
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+      int option_index = 0;
+      static struct option long_options[] =
+      {
+	{"add", 1, 0, 0},
+	{"append", 0, 0, 0},
+	{"delete", 1, 0, 0},
+	{"verbose", 0, 0, 0},
+	{"create", 0, 0, 0},
+	{"file", 1, 0, 0},
+	{0, 0, 0, 0}
+      };
+
+      c = getopt_long (argc, argv, "abc:d:0123456789",
+		       long_options, &option_index);
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case 0:
+	  printf ("option %s", long_options[option_index].name);
+	  if (optarg)
+	    printf (" with arg %s", optarg);
+	  printf ("\n");
+	  break;
+
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case 'd':
+	  printf ("option d with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/gsdl/packages/mg/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/gsdl/packages/mg/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman.c	(revision 21112)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:44  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = Xmalloc (sizeof (huff_data))))
+	goto error0;
+    }
+  if (!(heap = Xmalloc (num * 2 * sizeof (*heap))))
+    goto error1;
+  if (!(hd->clens = Xmalloc (num * sizeof (char))))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  bcopy ((char *) freqs, (char *) &heap[num], num * sizeof (*heap));
+  bzero ((char *) heap, num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; i++)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; i--)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    child++;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; i++)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		child++;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	HNum++;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; i++)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  bzero ((char *) hd->lencount, sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; i++)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      hd->lencount[codelen]++;
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; i--)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  free (heap);
+  return (hd);
+
+error2:
+  free (heap);
+error1:
+  if (!data)
+    free (hd);
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = Xmalloc (data->num_codes * sizeof (*codes))))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  bcopy ((char *) data->min_code, (char *) mc, sizeof (mc));
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = Xmalloc (data->num_codes * sizeof (*vals))))
+    return (NULL);
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *))))
+    {
+      free (vals);
+      return (NULL);
+    }
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; i++)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; i++)
+    size += counts[i] * hd.clens[i];
+  Xfree (hd.clens);
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+
+static int 
+General_Read_Huffman_Data (size_t (*rd) (), void *f,
+			   huff_data * hd, u_long * mem, u_long * disk)
+{
+  if (rd (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      bzero ((char *) (hd->lencount), sizeof (hd->lencount));
+      if (rd (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      bzero ((char *) (hd->min_code), sizeof (hd->min_code));
+
+      if (rd (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = Xmalloc (sizeof (*hd->clens) * hd->num_codes)))
+	return -1;
+
+
+      if (rd (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
+
+
+
+int 
+Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (fread, f, hd, mem, disk);
+}
+
+
+
+
+int 
+F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (Fread, f, hd, mem, disk);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+
+
+#define MAX_HUFFCODE_LEN 31
+
+typedef struct huff_data
+  {
+    int num_codes;
+    int mincodelen;
+    int maxcodelen;
+    int lencount[MAX_HUFFCODE_LEN + 1];
+    unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+    char *clens;
+  }
+huff_data;
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+int F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+
+#define HUFF_ENCODE(x, codes, lens)					\
+  do {									\
+    register int __i;							\
+    register int __clen = (lens)[x];					\
+    register unsigned long __code = (codes)[x];				\
+    for (__i=__clen-1; __i>=0; --__i)					\
+      ENCODE_BIT((__code >> __i) & 1);					\
+  } while(0)
+
+#define HUFF_ENCODE_L(x, codes, lens, count)				\
+  do {									\
+    HUFF_ENCODE(x, codes, lens);		   			\
+    (count) += (lens)[x];						\
+  } while(0)
+
+
+#define HUFF_DECODE(x, mcodes, values)					\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+#define HUFF_DECODE_L(x, mcodes, values, count)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+	(count)++;							\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.c	(revision 21112)
@@ -0,0 +1,66 @@
+/**************************************************************************
+ *
+ * huffman_mem.c -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:45  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+#include "bitio_m_mem.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+#endif
+
+void 
+BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+		     char *clens, mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Mem_Huff_Decode (unsigned long *mincodes,
+		     unsigned long **values, mem_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman_mem.h	(revision 21112)
@@ -0,0 +1,36 @@
+/**************************************************************************
+ *
+ * huffman_mem.h -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_MEM
+#define H_HUFFMAN_MEM
+
+
+
+void BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+			  char *clens, mem_bitio_state * bs);
+
+unsigned long BIO_Mem_Huff_Decode (unsigned long *mincodes,
+			      unsigned long **values, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.c	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * huffman_stdio.c -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:52  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:07  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:46  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+int _filbuf (FILE *);
+int _flsbuf (unsigned, FILE *);
+#endif
+
+void 
+BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+		       char *clens, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+		       unsigned long **values,
+		       stdio_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/huffman_stdio.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * huffman_stdio.h -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_STDIO
+#define H_HUFFMAN_STDIO
+
+
+
+void BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+			    char *clens, stdio_bitio_state * bs);
+
+
+unsigned long BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+				     unsigned long **values,
+				     stdio_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/local_strings.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/local_strings.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/local_strings.c	(revision 21112)
@@ -0,0 +1,415 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = Xmalloc (src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  bcopy ((char *) src_str, (char *) (copy), src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    i++;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  bzero (buf, sizeof (buf));
+  for (i = 0; i < len; i++)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  s++;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      s--;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; i++)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/local_strings.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/longlong.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/longlong.h	(revision 21112)
@@ -0,0 +1,81 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifndef __WIN32__
+#include "config.h"
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+	mgstat.c
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.c	(revision 21112)
@@ -0,0 +1,1187 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();
+
+static int 
+aio (ch)			/* Rule number 9     */
+     char ch;
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (ch)				/* Rule Number 24    */
+     char ch;
+{
+  return (ch != 's');
+}
+
+static int 
+pt (ch)				/* Rule number 28    */
+     char ch;
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (ch)				/* Rule number 30    */
+     char ch;
+{
+  return (ch != 'm');
+}
+
+static int 
+n (ch)				/* Rule number 32    */
+     char ch;
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+
+static int 
+B (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'e');
+}
+
+static int 
+F (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'c');
+}
+
+static int 
+W (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  bcopy ((char *) word, (char *) copy, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  bcopy ((char *) copy, (char *) word, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) ();		/* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  rule++;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) ();		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      suffix_start++;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:36  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:32:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:20:10  tes
+   * Changes made for 1.1
+   *
+   *
+ */
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/memlib.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/memlib.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/memlib.c	(revision 21112)
@@ -0,0 +1,63 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/11/29 02:56:09  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:47  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+/* change the name so we do not have problems with other libs */
+char *
+my_mg_strdup(const char *str)
+{
+  char *ret_str = malloc(strlen(str)+1);
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_mg_strdup;
Index: /main/tags/3.01/gsdl/packages/mg/lib/memlib.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/memlib.h	(revision 21112)
@@ -0,0 +1,51 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+#ifdef STD_MEM
+
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+#define Xstrdup strdup
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/messages.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/messages.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/messages.c	(revision 21112)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/23 03:38:06  rjmcnab
+
+   combined unix and windows versions of mg
+
+   Revision 1.1  1998/11/19 03:00:32  sjboddie
+   Added windows gdbm and mg versions
+
+   * Revision 1.1  1994/08/22  00:24:48  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef QUIET
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/messages.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/messages.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+
+void Message (const char *fmt,...);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/mgheap.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/mgheap.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/mgheap.c	(revision 21112)
@@ -0,0 +1,170 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1999/01/15 03:05:50  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.1  1998/11/17 09:32:01  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:43  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; __i--)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; __i--)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	child++;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; i--)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; i--)
+    {
+      SWAP (size, heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)--;
+  SWAP (size, heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)++;
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/mgheap.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,84 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/gsdl/packages/mg/lib/netorder.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/netorder.h	(revision 21112)
@@ -0,0 +1,177 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &hd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &hd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &nd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &nd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &nd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &hd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &hd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define NTOHF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define HTONF2(hf, nf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(hf), (char *) &tmp, sizeof (float));                \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(nf), sizeof (float));                \
+	}while(0)
+#define NTOHF2(nf, hf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(nf), (char *) &tmp, sizeof (float));                \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(hf), sizeof (float));                \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/gsdl/packages/mg/lib/pathmax.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.c	(revision 21112)
@@ -0,0 +1,1108 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2001/02/22 01:35:03  jrm21
+   Fixed bug that caused perf_hash to fail for lexicon of < 4 words.
+   Added a #define to increase size of n (for vertices in the graph) as we
+   need to allocate more memory somewhere else before calling tree_builder()
+
+   Revision 1.1  1999/08/10 21:16:57  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:17  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/08/22  00:26:30  tes
+   * Made perf_hash take the ceiling of 1.23 * num
+   * in lieu of the floor.
+   * This was done so that perf_hash works ok on
+   * a small number of keys.
+   *
+   * Revision 1.1  1994/08/22  00:24:49  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define STRUCT
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+/* An arbitrary amount to add to both MAX_N and n for building tree/acyclic
+   graph -- John McPherson - jrm21 (22-02-2001) */
+#define MAX_N_HEADROOM  100
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  StackPos++;
+  while (StackPos)
+    {
+      int m, b;
+      StackPos--;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; e++)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; v++)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      /* Added variable - need to increase MAX_N on allocate as well */
+      *n = (int) (num * c + MAX_N_HEADROOM); /* john mcpherson 22-02-2001 */
+
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; i++)
+	for (j = 0; j < MAX_L; j++)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; e++)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      w++;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; e++)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; i++)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  bzero ((char *) translate, sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; i++)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, s++)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; i++)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; i++)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = Xmalloc (sizeof (int) * MAX_M);
+  mk = Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    if (tb)
+      if (!(tb[i] = Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = Xmalloc (sizeof (u_char) * 256);
+  bzero ((char *) translate, sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  /* For extremely small lexicons (<4 words), this isn't good enough! 
+   -- John McPherson jrm21 22-02-2001 */
+  /* MAX_N = MAX_M + MAX_M / 4; */
+  MAX_N = (int)(ceil (1.25 * MAX_M)); 
+  /* Add some space for "experimentation" of n in tree_builder() */
+  MAX_N += MAX_N_HEADROOM;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  bcopy ((char *) &g[1], (char *) g, n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = Xmalloc (sizeof (u_char) * 256);
+  phd->g = Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb)
+      if (!(phd->tb[i] = Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      l--;
+      s++;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+
+
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/random.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/random.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/random.c	(revision 21112)
@@ -0,0 +1,225 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:16:57  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:20  rjmcnab
+*** empty log message ***
+
+*/
+
+static char *RCSID = "$Id$";
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/random.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/random.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/random.h	(revision 21112)
@@ -0,0 +1,54 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:16:58  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:21  rjmcnab
+*** empty log message ***
+
+*/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/regex.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/regex.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/regex.c	(revision 21112)
@@ -0,0 +1,5426 @@
+/* Extended regular expression matching and search library,
+   version 0.12.
+   (Implements POSIX draft P10003.2/D11.2, except for
+   internationalization features.)
+
+   Copyright (C) 1993, 1994, 1995 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* AIX requires this to be the first thing in the file. */
+#if defined (_AIX) && !defined (REGEX_MALLOC)
+  #pragma alloca
+#endif
+
+#define _GNU_SOURCE
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+/* We need this for `regex.h', and perhaps for the Emacs include files.  */
+#include <sys/types.h>
+
+/* This is for other GNU distributions with internationalized messages.  */
+#if HAVE_LIBINTL_H || defined (_LIBC)
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+/* The `emacs' switch turns on certain matching commands
+   that make sense only in Emacs. */
+#ifdef emacs
+
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+
+#else  /* not emacs */
+
+/* If we are not linking with Emacs proper,
+   we can't use the relocating allocator
+   even if sysfuncs.h says that we can.  */
+#undef REL_ALLOC
+
+#if defined (STDC_HEADERS) || defined (_LIBC)
+#include <stdlib.h>
+#else
+char *malloc ();
+char *realloc ();
+#endif
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+   `BSTRING', as far as I know, and neither of them use this code.  */
+#ifndef INHIBIT_STRING_HEADER
+#if HAVE_STRING_H || STDC_HEADERS || defined (_LIBC)
+#include <string.h>
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+#else
+#include <strings.h>
+#endif
+#endif
+
+/* Define the syntax stuff for \<, \>, etc.  */
+
+/* This must be nonzero for the wordchar and notwordchar pattern
+   commands in re_match_2.  */
+#ifndef Sword 
+#define Sword 1
+#endif
+
+#ifdef SWITCH_ENUM_BUG
+#define SWITCH_ENUM_CAST(x) ((int)(x))
+#else
+#define SWITCH_ENUM_CAST(x) (x)
+#endif
+
+#ifdef SYNTAX_TABLE
+
+extern char *re_syntax_table;
+
+#else /* not SYNTAX_TABLE */
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE 256
+
+static char re_syntax_table[CHAR_SET_SIZE];
+
+static void
+init_syntax_once ()
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+
+#endif /* not SYNTAX_TABLE */
+
+#define SYNTAX(c) re_syntax_table[c]
+
+#endif /* not emacs */
+
+
+/* Get the interface, including the syntax bits.  */
+#include "regex.h"
+
+/* isalpha etc. are used for the character classes.  */
+#include <ctype.h>
+
+/* Jim Meyering writes:
+
+   "... Some ctype macros are valid only for character codes that
+   isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when
+   using /bin/cc or gcc but without giving an ansi option).  So, all
+   ctype uses should be through macros like ISPRINT...  If
+   STDC_HEADERS is defined, then autoconf has verified that the ctype
+   macros don't need to be guarded with references to isascii. ...
+   Defining isascii to 1 should let any compiler worth its salt
+   eliminate the && through constant folding."  */
+
+#ifndef ISASCII
+#  if defined (STDC_HEADERS) || (!defined (isascii) && !defined (HAVE_ISASCII))
+#    define ISASCII(c) 1
+#  else
+#    define ISASCII(c) isascii(c)
+#  endif
+#endif
+
+#ifdef isblank
+#define ISBLANK(c) (ISASCII (c) && isblank (c))
+#else
+#define ISBLANK(c) ((c) == ' ' || (c) == '\t')
+#endif
+#ifdef isgraph
+#define ISGRAPH(c) (ISASCII (c) && isgraph (c))
+#else
+#define ISGRAPH(c) (ISASCII (c) && isprint (c) && !isspace (c))
+#endif
+
+#define ISPRINT(c) (ISASCII (c) && isprint (c))
+#define ISDIGIT(c) (ISASCII (c) && isdigit (c))
+#define ISALNUM(c) (ISASCII (c) && isalnum (c))
+#define ISALPHA(c) (ISASCII (c) && isalpha (c))
+#define ISCNTRL(c) (ISASCII (c) && iscntrl (c))
+#define ISLOWER(c) (ISASCII (c) && islower (c))
+#define ISPUNCT(c) (ISASCII (c) && ispunct (c))
+#define ISSPACE(c) (ISASCII (c) && isspace (c))
+#define ISUPPER(c) (ISASCII (c) && isupper (c))
+#define ISXDIGIT(c) (ISASCII (c) && isxdigit (c))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* We remove any previous definition of `SIGN_EXTEND_CHAR',
+   since ours (we hope) works properly with all combinations of
+   machines, compilers, `char' and `unsigned char' argument types.
+   (Per Bothner suggested the basic approach.)  */
+#undef SIGN_EXTEND_CHAR
+#if __STDC__
+#define SIGN_EXTEND_CHAR(c) ((signed char) (c))
+#else  /* not __STDC__ */
+/* As in Harbison and Steele.  */
+#define SIGN_EXTEND_CHAR(c) ((((unsigned char) (c)) ^ 128) - 128)
+#endif
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+   use `alloca' instead of `malloc'.  This is because using malloc in
+   re_search* or re_match* could cause memory leaks when C-g is used in
+   Emacs; also, malloc is slower and causes storage fragmentation.  On
+   the other hand, malloc is more portable, and easier to debug.  
+   
+   Because we sometimes use alloca, some routines have to be macros,
+   not functions -- `alloca'-allocated space disappears at the end of the
+   function it is called in.  */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE malloc
+#define REGEX_REALLOCATE(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE free
+
+#else /* not REGEX_MALLOC  */
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+#define REGEX_ALLOCATE alloca
+
+/* Assumes a `char *destination' variable.  */
+#define REGEX_REALLOCATE(source, osize, nsize)				\
+  (destination = (char *) alloca (nsize),				\
+   bcopy (source, destination, osize),					\
+   destination)
+
+/* No need to do anything to free, after alloca.  */
+#define REGEX_FREE(arg) ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+
+#endif /* not REGEX_MALLOC */
+
+/* Define how to allocate the failure stack.  */
+
+#ifdef REL_ALLOC
+#define REGEX_ALLOCATE_STACK(size)				\
+  r_alloc (&failure_stack_ptr, (size))
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)		\
+  r_re_alloc (&failure_stack_ptr, (nsize))
+#define REGEX_FREE_STACK(ptr)					\
+  r_alloc_free (&failure_stack_ptr)
+
+#else /* not REL_ALLOC */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE_STACK malloc
+#define REGEX_REALLOCATE_STACK(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE_STACK free
+
+#else /* not REGEX_MALLOC */
+
+#define REGEX_ALLOCATE_STACK alloca
+
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)			\
+   REGEX_REALLOCATE (source, osize, nsize)
+/* No need to explicitly free anything.  */
+#define REGEX_FREE_STACK(arg)
+
+#endif /* not REGEX_MALLOC */
+#endif /* not REL_ALLOC */
+
+
+/* True if `size1' is non-NULL and PTR is pointing anywhere inside
+   `string1' or just past its end.  This works if PTR is NULL, which is
+   a good thing.  */
+#define FIRST_STRING_P(ptr) 					\
+  (size1 && string1 <= (ptr) && (ptr) <= string1 + size1)
+
+/* (Re)Allocate N items of type T using malloc, or fail.  */
+#define TALLOC(n, t) ((t *) malloc ((n) * sizeof (t)))
+#define RETALLOC(addr, n, t) ((addr) = (t *) realloc (addr, (n) * sizeof (t)))
+#define RETALLOC_IF(addr, n, t) \
+  if (addr) RETALLOC((addr), (n), t); else (addr) = TALLOC ((n), t)
+#define REGEX_TALLOC(n, t) ((t *) REGEX_ALLOCATE ((n) * sizeof (t)))
+
+#define BYTEWIDTH 8 /* In bits.  */
+
+#define STREQ(s1, s2) ((strcmp (s1, s2) == 0))
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+static int re_match_2_internal ();
+
+
+/* These are the command codes that appear in compiled regular
+   expressions.  Some opcodes are followed by argument bytes.  A
+   command code can specify any interpretation whatsoever for its
+   arguments.  Zero bytes may appear in the compiled regular expression.  */
+
+typedef enum
+{
+  no_op = 0,
+
+  /* Succeed right away--no more backtracking.  */
+  succeed,
+
+        /* Followed by one byte giving n, then by n literal bytes.  */
+  exactn,
+
+        /* Matches any (more or less) character.  */
+  anychar,
+
+        /* Matches any one char belonging to specified set.  First
+           following byte is number of bitmap bytes.  Then come bytes
+           for a bitmap saying which chars are in.  Bits in each byte
+           are ordered low-bit-first.  A character is in the set if its
+           bit is 1.  A character too large to have a bit in the map is
+           automatically not in the set.  */
+  charset,
+
+        /* Same parameters as charset, but match any character that is
+           not one of those specified.  */
+  charset_not,
+
+        /* Start remembering the text that is matched, for storing in a
+           register.  Followed by one byte with the register number, in
+           the range 0 to one less than the pattern buffer's re_nsub
+           field.  Then followed by one byte with the number of groups
+           inner to this one.  (This last has to be part of the
+           start_memory only because we need it in the on_failure_jump
+           of re_match_2.)  */
+  start_memory,
+
+        /* Stop remembering the text that is matched and store it in a
+           memory register.  Followed by one byte with the register
+           number, in the range 0 to one less than `re_nsub' in the
+           pattern buffer, and one byte with the number of inner groups,
+           just like `start_memory'.  (We need the number of inner
+           groups here because we don't have any easy way of finding the
+           corresponding start_memory when we're at a stop_memory.)  */
+  stop_memory,
+
+        /* Match a duplicate of something remembered. Followed by one
+           byte containing the register number.  */
+  duplicate,
+
+        /* Fail unless at beginning of line.  */
+  begline,
+
+        /* Fail unless at end of line.  */
+  endline,
+
+        /* Succeeds if at beginning of buffer (if emacs) or at beginning
+           of string to be matched (if not).  */
+  begbuf,
+
+        /* Analogously, for end of buffer/string.  */
+  endbuf,
+ 
+        /* Followed by two byte relative address to which to jump.  */
+  jump, 
+
+	/* Same as jump, but marks the end of an alternative.  */
+  jump_past_alt,
+
+        /* Followed by two-byte relative address of place to resume at
+           in case of failure.  */
+  on_failure_jump,
+	
+        /* Like on_failure_jump, but pushes a placeholder instead of the
+           current string position when executed.  */
+  on_failure_keep_string_jump,
+  
+        /* Throw away latest failure point and then jump to following
+           two-byte relative address.  */
+  pop_failure_jump,
+
+        /* Change to pop_failure_jump if know won't have to backtrack to
+           match; otherwise change to jump.  This is used to jump
+           back to the beginning of a repeat.  If what follows this jump
+           clearly won't match what the repeat does, such that we can be
+           sure that there is no use backtracking out of repetitions
+           already matched, then we change it to a pop_failure_jump.
+           Followed by two-byte address.  */
+  maybe_pop_jump,
+
+        /* Jump to following two-byte address, and push a dummy failure
+           point. This failure point will be thrown away if an attempt
+           is made to use it for a failure.  A `+' construct makes this
+           before the first repeat.  Also used as an intermediary kind
+           of jump when compiling an alternative.  */
+  dummy_failure_jump,
+
+	/* Push a dummy failure point and continue.  Used at the end of
+	   alternatives.  */
+  push_dummy_failure,
+
+        /* Followed by two-byte relative address and two-byte number n.
+           After matching N times, jump to the address upon failure.  */
+  succeed_n,
+
+        /* Followed by two-byte relative address, and two-byte number n.
+           Jump to the address N times, then fail.  */
+  jump_n,
+
+        /* Set the following two-byte relative address to the
+           subsequent two-byte number.  The address *includes* the two
+           bytes of number.  */
+  set_number_at,
+
+  wordchar,	/* Matches any word-constituent character.  */
+  notwordchar,	/* Matches any char that is not a word-constituent.  */
+
+  wordbeg,	/* Succeeds if at word beginning.  */
+  wordend,	/* Succeeds if at word end.  */
+
+  wordbound,	/* Succeeds if at a word boundary.  */
+  notwordbound	/* Succeeds if not at a word boundary.  */
+
+#ifdef emacs
+  ,before_dot,	/* Succeeds if before point.  */
+  at_dot,	/* Succeeds if at point.  */
+  after_dot,	/* Succeeds if after point.  */
+
+	/* Matches any character whose syntax is specified.  Followed by
+           a byte which contains a syntax code, e.g., Sword.  */
+  syntaxspec,
+
+	/* Matches any character whose syntax is not that specified.  */
+  notsyntaxspec
+#endif /* emacs */
+} re_opcode_t;
+
+
+/* Common operations on the compiled pattern.  */
+
+/* Store NUMBER in two contiguous bytes starting at DESTINATION.  */
+
+#define STORE_NUMBER(destination, number)				\
+  do {									\
+    (destination)[0] = (number) & 0377;					\
+    (destination)[1] = (number) >> 8;					\
+  } while (0)
+
+/* Same as STORE_NUMBER, except increment DESTINATION to
+   the byte after where the number is stored.  Therefore, DESTINATION
+   must be an lvalue.  */
+
+#define STORE_NUMBER_AND_INCR(destination, number)			\
+  do {									\
+    STORE_NUMBER (destination, number);					\
+    (destination) += 2;							\
+  } while (0)
+
+/* Put into DESTINATION a number stored in two contiguous bytes starting
+   at SOURCE.  */
+
+#define EXTRACT_NUMBER(destination, source)				\
+  do {									\
+    (destination) = *(source) & 0377;					\
+    (destination) += SIGN_EXTEND_CHAR (*((source) + 1)) << 8;		\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number (dest, source)
+    int *dest;
+    unsigned char *source;
+{
+  int temp = SIGN_EXTEND_CHAR (*(source + 1)); 
+  *dest = *source & 0377;
+  *dest += temp << 8;
+}
+
+#ifndef EXTRACT_MACROS /* To debug the macros.  */
+#undef EXTRACT_NUMBER
+#define EXTRACT_NUMBER(dest, src) extract_number (&dest, src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+/* Same as EXTRACT_NUMBER, except increment SOURCE to after the number.
+   SOURCE must be an lvalue.  */
+
+#define EXTRACT_NUMBER_AND_INCR(destination, source)			\
+  do {									\
+    EXTRACT_NUMBER (destination, source);				\
+    (source) += 2; 							\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number_and_incr (destination, source)
+    int *destination;
+    unsigned char **source;
+{ 
+  extract_number (destination, *source);
+  *source += 2;
+}
+
+#ifndef EXTRACT_MACROS
+#undef EXTRACT_NUMBER_AND_INCR
+#define EXTRACT_NUMBER_AND_INCR(dest, src) \
+  extract_number_and_incr (&dest, &src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+
+/* If DEBUG is defined, Regex prints many voluminous messages about what
+   it is doing (if the variable `debug' is nonzero).  If linked with the
+   main program in `iregex.c', you can enter patterns and strings
+   interactively.  And if linked with the main program in `main.c' and
+   the other test files, you can run the already-written tests.  */
+
+#ifdef DEBUG
+
+/* We use standard I/O for debugging.  */
+#include <stdio.h>
+
+/* It is useful to test things that ``must'' be true when debugging.  */
+#include <assert.h>
+
+static int debug = 0;
+
+#define DEBUG_STATEMENT(e) e
+#define DEBUG_PRINT1(x) if (debug) printf (x)
+#define DEBUG_PRINT2(x1, x2) if (debug) printf (x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3) if (debug) printf (x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4) if (debug) printf (x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e) 				\
+  if (debug) print_partial_compiled_pattern (s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)			\
+  if (debug) print_double_string (w, s1, sz1, s2, sz2)
+
+
+/* Print the fastmap in human-readable form.  */
+
+void
+print_fastmap (fastmap)
+    char *fastmap;
+{
+  unsigned was_a_range = 0;
+  unsigned i = 0;  
+  
+  while (i < (1 << BYTEWIDTH))
+    {
+      if (fastmap[i++])
+	{
+	  was_a_range = 0;
+          putchar (i - 1);
+          while (i < (1 << BYTEWIDTH)  &&  fastmap[i])
+            {
+              was_a_range = 1;
+              i++;
+            }
+	  if (was_a_range)
+            {
+              printf ("-");
+              putchar (i - 1);
+            }
+        }
+    }
+  putchar ('\n'); 
+}
+
+
+/* Print a compiled pattern string in human-readable form, starting at
+   the START pointer into it and ending just before the pointer END.  */
+
+void
+print_partial_compiled_pattern (start, end)
+    unsigned char *start;
+    unsigned char *end;
+{
+  int mcnt, mcnt2;
+  unsigned char *p = start;
+  unsigned char *pend = end;
+
+  if (start == NULL)
+    {
+      printf ("(null)\n");
+      return;
+    }
+    
+  /* Loop over pattern commands.  */
+  while (p < pend)
+    {
+      printf ("%d:\t", p - start);
+
+      switch ((re_opcode_t) *p++)
+	{
+        case no_op:
+          printf ("/no_op");
+          break;
+
+	case exactn:
+	  mcnt = *p++;
+          printf ("/exactn/%d", mcnt);
+          do
+	    {
+              putchar ('/');
+	      putchar (*p++);
+            }
+          while (--mcnt);
+          break;
+
+	case start_memory:
+          mcnt = *p++;
+          printf ("/start_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case stop_memory:
+          mcnt = *p++;
+	  printf ("/stop_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case duplicate:
+	  printf ("/duplicate/%d", *p++);
+	  break;
+
+	case anychar:
+	  printf ("/anychar");
+	  break;
+
+	case charset:
+        case charset_not:
+          {
+            register int c, last = -100;
+	    register int in_range = 0;
+
+	    printf ("/charset [%s",
+	            (re_opcode_t) *(p - 1) == charset_not ? "^" : "");
+            
+            assert (p + *p < pend);
+
+            for (c = 0; c < 256; c++)
+	      if (c / 8 < *p
+		  && (p[1 + (c/8)] & (1 << (c % 8))))
+		{
+		  /* Are we starting a range?  */
+		  if (last + 1 == c && ! in_range)
+		    {
+		      putchar ('-');
+		      in_range = 1;
+		    }
+		  /* Have we broken a range?  */
+		  else if (last + 1 != c && in_range)
+              {
+		      putchar (last);
+		      in_range = 0;
+		    }
+                
+		  if (! in_range)
+		    putchar (c);
+
+		  last = c;
+              }
+
+	    if (in_range)
+	      putchar (last);
+
+	    putchar (']');
+
+	    p += 1 + *p;
+	  }
+	  break;
+
+	case begline:
+	  printf ("/begline");
+          break;
+
+	case endline:
+          printf ("/endline");
+          break;
+
+	case on_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case on_failure_keep_string_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_keep_string_jump to %d", p + mcnt - start);
+          break;
+
+	case dummy_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/dummy_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case push_dummy_failure:
+          printf ("/push_dummy_failure");
+          break;
+          
+        case maybe_pop_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/maybe_pop_jump to %d", p + mcnt - start);
+	  break;
+
+        case pop_failure_jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/pop_failure_jump to %d", p + mcnt - start);
+	  break;          
+          
+        case jump_past_alt:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump_past_alt to %d", p + mcnt - start);
+	  break;          
+          
+        case jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump to %d", p + mcnt - start);
+	  break;
+
+        case succeed_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/succeed_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case jump_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/jump_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case set_number_at: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/set_number_at location %d to %d", p + mcnt - start, mcnt2);
+          break;
+        
+        case wordbound:
+	  printf ("/wordbound");
+	  break;
+
+	case notwordbound:
+	  printf ("/notwordbound");
+          break;
+
+	case wordbeg:
+	  printf ("/wordbeg");
+	  break;
+          
+	case wordend:
+	  printf ("/wordend");
+          
+#ifdef emacs
+	case before_dot:
+	  printf ("/before_dot");
+          break;
+
+	case at_dot:
+	  printf ("/at_dot");
+          break;
+
+	case after_dot:
+	  printf ("/after_dot");
+          break;
+
+	case syntaxspec:
+          printf ("/syntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+          break;
+	  
+	case notsyntaxspec:
+          printf ("/notsyntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+	  break;
+#endif /* emacs */
+
+	case wordchar:
+	  printf ("/wordchar");
+          break;
+	  
+	case notwordchar:
+	  printf ("/notwordchar");
+          break;
+
+	case begbuf:
+	  printf ("/begbuf");
+          break;
+
+	case endbuf:
+	  printf ("/endbuf");
+          break;
+
+        default:
+          printf ("?%d", *(p-1));
+	}
+
+      putchar ('\n');
+    }
+
+  printf ("%d:\tend of pattern.\n", p - start);
+}
+
+
+void
+print_compiled_pattern (bufp)
+    struct re_pattern_buffer *bufp;
+{
+  unsigned char *buffer = bufp->buffer;
+
+  print_partial_compiled_pattern (buffer, buffer + bufp->used);
+  printf ("%d bytes used/%d bytes allocated.\n", bufp->used, bufp->allocated);
+
+  if (bufp->fastmap_accurate && bufp->fastmap)
+    {
+      printf ("fastmap: ");
+      print_fastmap (bufp->fastmap);
+    }
+
+  printf ("re_nsub: %d\t", bufp->re_nsub);
+  printf ("regs_alloc: %d\t", bufp->regs_allocated);
+  printf ("can_be_null: %d\t", bufp->can_be_null);
+  printf ("newline_anchor: %d\n", bufp->newline_anchor);
+  printf ("no_sub: %d\t", bufp->no_sub);
+  printf ("not_bol: %d\t", bufp->not_bol);
+  printf ("not_eol: %d\t", bufp->not_eol);
+  printf ("syntax: %d\n", bufp->syntax);
+  /* Perhaps we should print the translate table?  */
+}
+
+
+void
+print_double_string (where, string1, size1, string2, size2)
+    const char *where;
+    const char *string1;
+    const char *string2;
+    int size1;
+    int size2;
+{
+  unsigned this_char;
+  
+  if (where == NULL)
+    printf ("(null)");
+  else
+    {
+      if (FIRST_STRING_P (where))
+        {
+          for (this_char = where - string1; this_char < size1; this_char++)
+            putchar (string1[this_char]);
+
+          where = string2;    
+        }
+
+      for (this_char = where - string2; this_char < size2; this_char++)
+        putchar (string2[this_char]);
+    }
+}
+
+#else /* not DEBUG */
+
+#undef assert
+#define assert(e)
+
+#define DEBUG_STATEMENT(e)
+#define DEBUG_PRINT1(x)
+#define DEBUG_PRINT2(x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)
+
+#endif /* not DEBUG */
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+/* This has no initializer because initialized variables in Emacs
+   become read-only after dumping.  */
+reg_syntax_t re_syntax_options;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+{
+  reg_syntax_t ret = re_syntax_options;
+  
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.
+   POSIX doesn't require that we do anything for REG_NOERROR,
+   but why not be nice?  */
+
+static const char *re_error_msgid[] =
+  { "Success",					/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+  };
+
+
+/* Avoiding alloca during matching, to placate r_alloc.  */
+
+/* Define MATCH_MAY_ALLOCATE unless we need to make sure that the
+   searching and matching functions should not call alloca.  On some
+   systems, alloca is implemented in terms of malloc, and if we're
+   using the relocating allocator routines, then malloc could cause a
+   relocation, which might (if the strings being searched are in the
+   ralloc heap) shift the data out from underneath the regexp
+   routines.
+
+   Here's another reason to avoid allocation: Emacs 
+   processes input from X in a signal handler; processing X input may
+   call malloc; if input arrives while a matching routine is calling
+   malloc, then we're scrod.  But Emacs can't just block input while
+   calling matching routines; then we don't notice interrupts when
+   they come in.  So, Emacs blocks input around all regexp calls
+   except the matching calls, which it leaves unprotected, in the
+   faith that they will not malloc.  */
+
+/* Normally, this is fine.  */
+#define MATCH_MAY_ALLOCATE
+
+/* When using GNU C, we are not REALLY using the C alloca, no matter
+   what sysfuncs.h may say.  So don't take precautions for it.  */
+#ifdef __GNUC__
+#undef C_ALLOCA
+#endif
+
+/* The match routines may not allocate if (1) they would do it with malloc
+   and (2) it's not safe for them to use malloc.
+   Note that if REL_ALLOC is defined, matching would not use malloc for the
+   failure stack, but we would still use it for the register vectors;
+   so REL_ALLOC should not affect this.  */
+#if (defined (C_ALLOCA) || defined (REGEX_MALLOC)) && defined (emacs)
+#undef MATCH_MAY_ALLOCATE
+#endif
+
+
+
+/* Failure stack declarations and macros; both re_compile_fastmap and
+   re_match_2 use a failure stack.  These have to be macros because of
+   REGEX_ALLOCATE_STACK.  */
+   
+
+/* Number of failure points for which to initially allocate space
+   when matching.  If this number is exceeded, we allocate more
+   space, so it is not a hard limit.  */
+#ifndef INIT_FAILURE_ALLOC
+#define INIT_FAILURE_ALLOC 5
+#endif
+
+/* Roughly the maximum number of failure points on the stack.  Would be
+   exactly that if always used MAX_FAILURE_SPACE each time we failed.
+   This is a variable only so users of regex can assign to it; we never
+   change it ourselves.  */
+#if defined (MATCH_MAY_ALLOCATE)
+int re_max_failures = 200000;
+#else
+int re_max_failures = 2000;
+#endif
+
+union fail_stack_elt
+{
+  unsigned char *pointer;
+  int integer;
+};
+
+typedef union fail_stack_elt fail_stack_elt_t;
+
+typedef struct
+{
+  fail_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} fail_stack_type;
+
+#define FAIL_STACK_EMPTY()     (fail_stack.avail == 0)
+#define FAIL_STACK_PTR_EMPTY() (fail_stack_ptr->avail == 0)
+#define FAIL_STACK_FULL()      (fail_stack.avail == fail_stack.size)
+
+
+/* Define macros to initialize and free the failure stack.
+   Do `return -2' if the alloc fails.  */
+
+#ifdef MATCH_MAY_ALLOCATE
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.stack = (fail_stack_elt_t *)				\
+      REGEX_ALLOCATE_STACK (INIT_FAILURE_ALLOC * sizeof (fail_stack_elt_t));	\
+									\
+    if (fail_stack.stack == NULL)					\
+      return -2;							\
+									\
+    fail_stack.size = INIT_FAILURE_ALLOC;				\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()  REGEX_FREE_STACK (fail_stack.stack)
+#else
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()
+#endif
+
+
+/* Double the size of FAIL_STACK, up to approximately `re_max_failures' items.
+
+   Return 1 if succeeds, and 0 if either ran out of memory
+   allocating space for it or it was already too large.  
+   
+   REGEX_REALLOCATE_STACK requires `destination' be declared.   */
+
+#define DOUBLE_FAIL_STACK(fail_stack)					\
+  ((fail_stack).size > re_max_failures * MAX_FAILURE_ITEMS		\
+   ? 0									\
+   : ((fail_stack).stack = (fail_stack_elt_t *)				\
+        REGEX_REALLOCATE_STACK ((fail_stack).stack, 			\
+          (fail_stack).size * sizeof (fail_stack_elt_t),		\
+          ((fail_stack).size << 1) * sizeof (fail_stack_elt_t)),	\
+									\
+      (fail_stack).stack == NULL					\
+      ? 0								\
+      : ((fail_stack).size <<= 1, 					\
+         1)))
+
+
+/* Push pointer POINTER on FAIL_STACK. 
+   Return 1 if was able to do so and 0 if ran out of memory allocating
+   space to do so.  */
+#define PUSH_PATTERN_OP(POINTER, FAIL_STACK)				\
+  ((FAIL_STACK_FULL ()							\
+    && !DOUBLE_FAIL_STACK (FAIL_STACK))					\
+   ? 0									\
+   : ((FAIL_STACK).stack[(FAIL_STACK).avail++].pointer = POINTER,	\
+      1))
+
+/* Push a pointer value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_POINTER(item)					\
+  fail_stack.stack[fail_stack.avail++].pointer = (unsigned char *) (item)
+
+/* This pushes an integer-valued item onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_INT(item)					\
+  fail_stack.stack[fail_stack.avail++].integer = (item)
+
+/* Push a fail_stack_elt_t value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_ELT(item)					\
+  fail_stack.stack[fail_stack.avail++] =  (item)
+
+/* These three POP... operations complement the three PUSH... operations.
+   All assume that `fail_stack' is nonempty.  */
+#define POP_FAILURE_POINTER() fail_stack.stack[--fail_stack.avail].pointer
+#define POP_FAILURE_INT() fail_stack.stack[--fail_stack.avail].integer
+#define POP_FAILURE_ELT() fail_stack.stack[--fail_stack.avail]
+
+/* Used to omit pushing failure point id's when we're not debugging.  */
+#ifdef DEBUG
+#define DEBUG_PUSH PUSH_FAILURE_INT
+#define DEBUG_POP(item_addr) *(item_addr) = POP_FAILURE_INT ()
+#else
+#define DEBUG_PUSH(item)
+#define DEBUG_POP(item_addr)
+#endif
+
+
+/* Push the information about the state we will need
+   if we ever fail back to it.  
+   
+   Requires variables fail_stack, regstart, regend, reg_info, and
+   num_regs be declared.  DOUBLE_FAIL_STACK requires `destination' be
+   declared.
+   
+   Does `return FAILURE_CODE' if runs out of memory.  */
+
+#define PUSH_FAILURE_POINT(pattern_place, string_place, failure_code)	\
+  do {									\
+    char *destination;							\
+    /* Must be int, so when we don't save any registers, the arithmetic	\
+       of 0 + -1 isn't done as unsigned.  */				\
+    int this_reg;							\
+    									\
+    DEBUG_STATEMENT (failure_id++);					\
+    DEBUG_STATEMENT (nfailure_points_pushed++);				\
+    DEBUG_PRINT2 ("\nPUSH_FAILURE_POINT #%u:\n", failure_id);		\
+    DEBUG_PRINT2 ("  Before push, next avail: %d\n", (fail_stack).avail);\
+    DEBUG_PRINT2 ("                     size: %d\n", (fail_stack).size);\
+									\
+    DEBUG_PRINT2 ("  slots needed: %d\n", NUM_FAILURE_ITEMS);		\
+    DEBUG_PRINT2 ("     available: %d\n", REMAINING_AVAIL_SLOTS);	\
+									\
+    /* Ensure we have enough space allocated for what we will push.  */	\
+    while (REMAINING_AVAIL_SLOTS < NUM_FAILURE_ITEMS)			\
+      {									\
+        if (!DOUBLE_FAIL_STACK (fail_stack))				\
+          return failure_code;						\
+									\
+        DEBUG_PRINT2 ("\n  Doubled stack; size now: %d\n",		\
+		       (fail_stack).size);				\
+        DEBUG_PRINT2 ("  slots available: %d\n", REMAINING_AVAIL_SLOTS);\
+      }									\
+									\
+    /* Push the info, starting with the registers.  */			\
+    DEBUG_PRINT1 ("\n");						\
+									\
+    for (this_reg = lowest_active_reg; this_reg <= highest_active_reg;	\
+         this_reg++)							\
+      {									\
+	DEBUG_PRINT2 ("  Pushing reg: %d\n", this_reg);			\
+        DEBUG_STATEMENT (num_regs_pushed++);				\
+									\
+	DEBUG_PRINT2 ("    start: 0x%x\n", regstart[this_reg]);		\
+        PUSH_FAILURE_POINTER (regstart[this_reg]);			\
+                                                                        \
+	DEBUG_PRINT2 ("    end: 0x%x\n", regend[this_reg]);		\
+        PUSH_FAILURE_POINTER (regend[this_reg]);			\
+									\
+	DEBUG_PRINT2 ("    info: 0x%x\n      ", reg_info[this_reg]);	\
+        DEBUG_PRINT2 (" match_null=%d",					\
+                      REG_MATCH_NULL_STRING_P (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" active=%d", IS_ACTIVE (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" matched_something=%d",				\
+                      MATCHED_SOMETHING (reg_info[this_reg]));		\
+        DEBUG_PRINT2 (" ever_matched=%d",				\
+                      EVER_MATCHED_SOMETHING (reg_info[this_reg]));	\
+	DEBUG_PRINT1 ("\n");						\
+        PUSH_FAILURE_ELT (reg_info[this_reg].word);			\
+      }									\
+									\
+    DEBUG_PRINT2 ("  Pushing  low active reg: %d\n", lowest_active_reg);\
+    PUSH_FAILURE_INT (lowest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing high active reg: %d\n", highest_active_reg);\
+    PUSH_FAILURE_INT (highest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing pattern 0x%x: ", pattern_place);		\
+    DEBUG_PRINT_COMPILED_PATTERN (bufp, pattern_place, pend);		\
+    PUSH_FAILURE_POINTER (pattern_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing string 0x%x: `", string_place);		\
+    DEBUG_PRINT_DOUBLE_STRING (string_place, string1, size1, string2,   \
+				 size2);				\
+    DEBUG_PRINT1 ("'\n");						\
+    PUSH_FAILURE_POINTER (string_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing failure id: %u\n", failure_id);		\
+    DEBUG_PUSH (failure_id);						\
+  } while (0)
+
+/* This is the number of items that are pushed and popped on the stack
+   for each register.  */
+#define NUM_REG_ITEMS  3
+
+/* Individual items aside from the registers.  */
+#ifdef DEBUG
+#define NUM_NONREG_ITEMS 5 /* Includes failure point id.  */
+#else
+#define NUM_NONREG_ITEMS 4
+#endif
+
+/* We push at most this many items on the stack.  */
+#define MAX_FAILURE_ITEMS ((num_regs - 1) * NUM_REG_ITEMS + NUM_NONREG_ITEMS)
+
+/* We actually push this many items.  */
+#define NUM_FAILURE_ITEMS						\
+  ((highest_active_reg - lowest_active_reg + 1) * NUM_REG_ITEMS 	\
+    + NUM_NONREG_ITEMS)
+
+/* How many items can still be added to the stack without overflowing it.  */
+#define REMAINING_AVAIL_SLOTS ((fail_stack).size - (fail_stack).avail)
+
+
+/* Pops what PUSH_FAIL_STACK pushes.
+
+   We restore into the parameters, all of which should be lvalues:
+     STR -- the saved data position.
+     PAT -- the saved pattern position.
+     LOW_REG, HIGH_REG -- the highest and lowest active registers.
+     REGSTART, REGEND -- arrays of string positions.
+     REG_INFO -- array of information about each subexpression.
+   
+   Also assumes the variables `fail_stack' and (if debugging), `bufp',
+   `pend', `string1', `size1', `string2', and `size2'.  */
+
+#define POP_FAILURE_POINT(str, pat, low_reg, high_reg, regstart, regend, reg_info)\
+{									\
+  DEBUG_STATEMENT (fail_stack_elt_t failure_id;)			\
+  int this_reg;								\
+  const unsigned char *string_temp;					\
+									\
+  assert (!FAIL_STACK_EMPTY ());					\
+									\
+  /* Remove failure points and point to how many regs pushed.  */	\
+  DEBUG_PRINT1 ("POP_FAILURE_POINT:\n");				\
+  DEBUG_PRINT2 ("  Before pop, next avail: %d\n", fail_stack.avail);	\
+  DEBUG_PRINT2 ("                    size: %d\n", fail_stack.size);	\
+									\
+  assert (fail_stack.avail >= NUM_NONREG_ITEMS);			\
+									\
+  DEBUG_POP (&failure_id);						\
+  DEBUG_PRINT2 ("  Popping failure id: %u\n", failure_id);		\
+									\
+  /* If the saved string location is NULL, it came from an		\
+     on_failure_keep_string_jump opcode, and we want to throw away the	\
+     saved NULL, thus retaining our current position in the string.  */	\
+  string_temp = POP_FAILURE_POINTER ();					\
+  if (string_temp != NULL)						\
+    str = (const char *) string_temp;					\
+									\
+  DEBUG_PRINT2 ("  Popping string 0x%x: `", str);			\
+  DEBUG_PRINT_DOUBLE_STRING (str, string1, size1, string2, size2);	\
+  DEBUG_PRINT1 ("'\n");							\
+									\
+  pat = (unsigned char *) POP_FAILURE_POINTER ();			\
+  DEBUG_PRINT2 ("  Popping pattern 0x%x: ", pat);			\
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, pat, pend);			\
+									\
+  /* Restore register info.  */						\
+  high_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping high active reg: %d\n", high_reg);		\
+									\
+  low_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping  low active reg: %d\n", low_reg);		\
+									\
+  for (this_reg = high_reg; this_reg >= low_reg; this_reg--)		\
+    {									\
+      DEBUG_PRINT2 ("    Popping reg: %d\n", this_reg);			\
+									\
+      reg_info[this_reg].word = POP_FAILURE_ELT ();			\
+      DEBUG_PRINT2 ("      info: 0x%x\n", reg_info[this_reg]);		\
+									\
+      regend[this_reg] = (const char *) POP_FAILURE_POINTER ();		\
+      DEBUG_PRINT2 ("      end: 0x%x\n", regend[this_reg]);		\
+									\
+      regstart[this_reg] = (const char *) POP_FAILURE_POINTER ();	\
+      DEBUG_PRINT2 ("      start: 0x%x\n", regstart[this_reg]);		\
+    }									\
+									\
+  set_regs_matched_done = 0;						\
+  DEBUG_STATEMENT (nfailure_points_popped++);				\
+} /* POP_FAILURE_POINT */
+
+
+
+
+/* Structure for per-register (a.k.a. per-group) information.
+   Other register information, such as the
+   starting and ending positions (which are addresses), and the list of
+   inner groups (which is a bits list) are maintained in separate
+   variables.  
+   
+   We are making a (strictly speaking) nonportable assumption here: that
+   the compiler will pack our bit fields into something that fits into
+   the type of `word', i.e., is something that fits into one item on the
+   failure stack.  */
+
+typedef union
+{
+  fail_stack_elt_t word;
+  struct
+  {
+      /* This field is one if this group can match the empty string,
+         zero if not.  If not yet determined,  `MATCH_NULL_UNSET_VALUE'.  */
+#define MATCH_NULL_UNSET_VALUE 3
+    unsigned match_null_string_p : 2;
+    unsigned is_active : 1;
+    unsigned matched_something : 1;
+    unsigned ever_matched_something : 1;
+  } bits;
+} register_info_type;
+
+#define REG_MATCH_NULL_STRING_P(R)  ((R).bits.match_null_string_p)
+#define IS_ACTIVE(R)  ((R).bits.is_active)
+#define MATCHED_SOMETHING(R)  ((R).bits.matched_something)
+#define EVER_MATCHED_SOMETHING(R)  ((R).bits.ever_matched_something)
+
+
+/* Call this when have matched a real character; it sets `matched' flags
+   for the subexpressions which we are currently inside.  Also records
+   that those subexprs have matched.  */
+#define SET_REGS_MATCHED()						\
+  do									\
+    {									\
+      if (!set_regs_matched_done)					\
+	{								\
+	  unsigned r;							\
+	  set_regs_matched_done = 1;					\
+	  for (r = lowest_active_reg; r <= highest_active_reg; r++)	\
+	    {								\
+	      MATCHED_SOMETHING (reg_info[r])				\
+		= EVER_MATCHED_SOMETHING (reg_info[r])			\
+		= 1;							\
+	    }								\
+	}								\
+    }									\
+  while (0)
+
+/* Registers are set to a sentinel when they haven't yet matched.  */
+static char reg_unset_dummy;
+#define REG_UNSET_VALUE (&reg_unset_dummy)
+#define REG_UNSET(e) ((e) == REG_UNSET_VALUE)
+
+
+/* Subroutine declarations and macros for regex_compile.  */
+
+static void store_op1 (), store_op2 ();
+static void insert_op1 (), insert_op2 ();
+static boolean at_begline_loc_p (), at_endline_loc_p ();
+static boolean group_in_compile_stack ();
+static reg_errcode_t compile_range ();
+
+/* Fetch the next character in the uncompiled pattern---translating it 
+   if necessary.  Also cast from a signed character in the constant
+   string passed to us by the user to an unsigned char that we can use
+   as an array index (in, e.g., `translate').  */
+#define PATFETCH(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    if (translate) c = translate[c]; 					\
+  } while (0)
+
+/* Fetch the next character in the uncompiled pattern, with no
+   translation.  */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+/* If `translate' is non-null, return translate[D], else just D.  We
+   cast the subscript to translate because some data is declared as
+   `char *', to avoid warnings when a string constant is passed.  But
+   when we use a character as a subscript we must make it unsigned.  */
+#define TRANSLATE(d) (translate ? translate[(unsigned char) (d)] : (d))
+
+
+/* Macros for outputting the compiled pattern into `buffer'.  */
+
+/* If the buffer isn't allocated when it comes in, use this.  */
+#define INIT_BUF_SIZE  32
+
+/* Make sure we have at least N more bytes of space in buffer.  */
+#define GET_BUFFER_SPACE(n)						\
+    while (b - bufp->buffer + (n) > bufp->allocated)			\
+      EXTEND_BUFFER ()
+
+/* Make sure we have one more byte of buffer space and then add C to it.  */
+#define BUF_PUSH(c)							\
+  do {									\
+    GET_BUFFER_SPACE (1);						\
+    *b++ = (unsigned char) (c);						\
+  } while (0)
+
+
+/* Ensure we have two more bytes of buffer space and then append C1 and C2.  */
+#define BUF_PUSH_2(c1, c2)						\
+  do {									\
+    GET_BUFFER_SPACE (2);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+  } while (0)
+
+
+/* As with BUF_PUSH_2, except for three bytes.  */
+#define BUF_PUSH_3(c1, c2, c3)						\
+  do {									\
+    GET_BUFFER_SPACE (3);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+    *b++ = (unsigned char) (c3);					\
+  } while (0)
+
+
+/* Store a jump with opcode OP at LOC to location TO.  We store a
+   relative address offset by the three bytes the jump itself occupies.  */
+#define STORE_JUMP(op, loc, to) \
+  store_op1 (op, loc, (to) - (loc) - 3)
+
+/* Likewise, for a two-argument jump.  */
+#define STORE_JUMP2(op, loc, to, arg) \
+  store_op2 (op, loc, (to) - (loc) - 3, arg)
+
+/* Like `STORE_JUMP', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP(op, loc, to) \
+  insert_op1 (op, loc, (to) - (loc) - 3, b)
+
+/* Like `STORE_JUMP2', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP2(op, loc, to, arg) \
+  insert_op2 (op, loc, (to) - (loc) - 3, arg, b)
+
+
+/* This is not an arbitrary limit: the arguments which represent offsets
+   into the pattern are two bytes long.  So if 2^16 bytes turns out to
+   be too small, many things would have to change.  */
+#define MAX_BUF_SIZE (1L << 16)
+
+
+/* Extend the buffer by twice its current size via realloc and
+   reset the pointers that pointed into the old block to point to the
+   correct places in the new one.  If extending the buffer results in it
+   being larger than MAX_BUF_SIZE, then flag memory exhausted.  */
+#define EXTEND_BUFFER()							\
+  do { 									\
+    unsigned char *old_buffer = bufp->buffer;				\
+    if (bufp->allocated == MAX_BUF_SIZE) 				\
+      return REG_ESIZE;							\
+    bufp->allocated <<= 1;						\
+    if (bufp->allocated > MAX_BUF_SIZE)					\
+      bufp->allocated = MAX_BUF_SIZE; 					\
+    bufp->buffer = (unsigned char *) realloc (bufp->buffer, bufp->allocated);\
+    if (bufp->buffer == NULL)						\
+      return REG_ESPACE;						\
+    /* If the buffer moved, move all the pointers into it.  */		\
+    if (old_buffer != bufp->buffer)					\
+      {									\
+        b = (b - old_buffer) + bufp->buffer;				\
+        begalt = (begalt - old_buffer) + bufp->buffer;			\
+        if (fixup_alt_jump)						\
+          fixup_alt_jump = (fixup_alt_jump - old_buffer) + bufp->buffer;\
+        if (laststart)							\
+          laststart = (laststart - old_buffer) + bufp->buffer;		\
+        if (pending_exact)						\
+          pending_exact = (pending_exact - old_buffer) + bufp->buffer;	\
+      }									\
+  } while (0)
+
+
+/* Since we have one byte reserved for the register number argument to
+   {start,stop}_memory, the maximum number of groups we can report
+   things about is what fits in that byte.  */
+#define MAX_REGNUM 255
+
+/* But patterns can have more than `MAX_REGNUM' registers.  We just
+   ignore the excess.  */
+typedef unsigned regnum_t;
+
+
+/* Macros for the compile stack.  */
+
+/* Since offsets can go either forwards or backwards, this type needs to
+   be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.  */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  pattern_offset_t begalt_offset;
+  pattern_offset_t fixup_alt_jump;
+  pattern_offset_t inner_group_offset;
+  pattern_offset_t laststart_offset;  
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / BYTEWIDTH]               \
+   |= 1 << (((unsigned char) c) % BYTEWIDTH))
+
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (ISDIGIT (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (STREQ (string, "alpha") || STREQ (string, "upper")			\
+    || STREQ (string, "lower") || STREQ (string, "digit")		\
+    || STREQ (string, "alnum") || STREQ (string, "xdigit")		\
+    || STREQ (string, "space") || STREQ (string, "print")		\
+    || STREQ (string, "punct") || STREQ (string, "graph")		\
+    || STREQ (string, "cntrl") || STREQ (string, "blank"))
+
+
+#ifndef MATCH_MAY_ALLOCATE
+
+/* If we cannot allocate large objects within re_match_2_internal,
+   we make the fail stack and register vectors global.
+   The fail stack, we grow to the maximum size when a regexp
+   is compiled.
+   The register vectors, we adjust in size each time we
+   compile a regexp, according to the number of registers it needs.  */
+
+static fail_stack_type fail_stack;
+
+/* Size with which the following vectors are currently allocated.
+   That is so we can make them bigger as needed,
+   but never make them smaller.  */
+static int regs_allocated_size;
+
+static const char **     regstart, **     regend;
+static const char ** old_regstart, ** old_regend;
+static const char **best_regstart, **best_regend;
+static register_info_type *reg_info; 
+static const char **reg_dummy;
+static register_info_type *reg_info_dummy;
+
+/* Make the register vectors big enough for NUM_REGS registers,
+   but don't make them smaller.  */
+
+static
+regex_grow_registers (num_regs)
+     int num_regs;
+{
+  if (num_regs > regs_allocated_size)
+    {
+      RETALLOC_IF (regstart,	 num_regs, const char *);
+      RETALLOC_IF (regend,	 num_regs, const char *);
+      RETALLOC_IF (old_regstart, num_regs, const char *);
+      RETALLOC_IF (old_regend,	 num_regs, const char *);
+      RETALLOC_IF (best_regstart, num_regs, const char *);
+      RETALLOC_IF (best_regend,	 num_regs, const char *);
+      RETALLOC_IF (reg_info,	 num_regs, register_info_type);
+      RETALLOC_IF (reg_dummy,	 num_regs, const char *);
+      RETALLOC_IF (reg_info_dummy, num_regs, register_info_type);
+
+      regs_allocated_size = num_regs;
+    }
+}
+
+#endif /* not MATCH_MAY_ALLOCATE */
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is zero;
+     `re_nsub' is the number of subexpressions in PATTERN;
+     `not_bol' and `not_eol' are zero;
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+/* Return, freeing storage we allocated.  */
+#define FREE_STACK_RETURN(value)		\
+  return (free (compile_stack.stack), value)
+
+static reg_errcode_t
+regex_compile (pattern, size, syntax, bufp)
+     const char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer *bufp;
+{
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random temporary spot in PATTERN.  */
+  const char *p1;
+
+  /* Points to the end of the buffer, where we should append.  */
+  register unsigned char *b;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  const char *p = pattern;
+  const char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  char *translate = bufp->translate;
+
+  /* Address of the count-byte of the most recently inserted `exactn'
+     command.  This makes it possible to tell if a new exact-match
+     character can be added to that command or if the character requires
+     a new `exactn' command.  */
+  unsigned char *pending_exact = 0;
+
+  /* Address of start of the most recently finished expression.
+     This tells, e.g., postfix * where to find the start of its
+     operand.  Reset at the beginning of groups and alternatives.  */
+  unsigned char *laststart = 0;
+
+  /* Address of beginning of regexp, or inside of last group.  */
+  unsigned char *begalt;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+     which to go back if the interval is invalid.  */
+  const char *beg_interval;
+                
+  /* Address of the place where a forward jump should go to the end of
+     the containing expression.  Each alternative of an `or' -- except the
+     last -- ends with a forward jump of this sort.  */
+  unsigned char *fixup_alt_jump = 0;
+
+  /* Counts open-groups as they are encountered.  Remembered for the
+     matching close-group on the compile stack, so the same register
+     number is put in the stop_memory as the start_memory.  */
+  regnum_t regnum = 0;
+
+#ifdef DEBUG
+  DEBUG_PRINT1 ("\nCompiling pattern: ");
+  if (debug)
+    {
+      unsigned debug_count;
+      
+      for (debug_count = 0; debug_count < size; debug_count++)
+        putchar (pattern[debug_count]);
+      putchar ('\n');
+    }
+#endif /* DEBUG */
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack = TALLOC (INIT_COMPILE_STACK_SIZE, compile_stack_elt_t);
+  if (compile_stack.stack == NULL)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  bufp->syntax = syntax;
+  bufp->fastmap_accurate = 0;
+  bufp->not_bol = bufp->not_eol = 0;
+
+  /* Set `used' to zero, so that if we return an error, the pattern
+     printer (for debugging) will think there's no pattern.  We reset it
+     at the end.  */
+  bufp->used = 0;
+  
+  /* Always count groups, whether or not bufp->no_sub is set.  */
+  bufp->re_nsub = 0;				
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  if (bufp->allocated == 0)
+    {
+      if (bufp->buffer)
+	{ /* If zero allocated, but buffer is non-null, try to realloc
+             enough space.  This loses if buffer's address is bogus, but
+             that is the user's responsibility.  */
+          RETALLOC (bufp->buffer, INIT_BUF_SIZE, unsigned char);
+        }
+      else
+        { /* Caller did not allocate a buffer.  Do it for them.  */
+          bufp->buffer = TALLOC (INIT_BUF_SIZE, unsigned char);
+        }
+      if (!bufp->buffer) FREE_STACK_RETURN (REG_ESPACE);
+
+      bufp->allocated = INIT_BUF_SIZE;
+    }
+
+  begalt = b = bufp->buffer;
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+              BUF_PUSH (begline);
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+               BUF_PUSH (endline);
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (!laststart)
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                FREE_STACK_RETURN (REG_BADRPT);
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* Are we optimizing this jump?  */
+            boolean keep_string_p = false;
+            
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!laststart)  
+              break;
+
+            /* Now we know whether or not zero matches is allowed
+               and also whether or not two or more matches is allowed.  */
+            if (many_times_ok)
+              { /* More than one repetition is allowed, so put in at the
+                   end a backward relative jump from `b' to before the next
+                   jump we're going to put in below (which jumps from
+                   laststart to after this jump).  
+
+                   But if we are at the `*' in the exact sequence `.*\n',
+                   insert an unconditional jump backwards to the .,
+                   instead of the beginning of the loop.  This way we only
+                   push a failure point once, instead of every time
+                   through the loop.  */
+                assert (p - 1 > pattern);
+
+                /* Allocate the space for the jump.  */
+                GET_BUFFER_SPACE (3);
+
+                /* We know we are not at the first character of the pattern,
+                   because laststart was nonzero.  And we've already
+                   incremented `p', by the way, to be the character after
+                   the `*'.  Do we have to do something analogous here
+                   for null bytes, because of RE_DOT_NOT_NULL?  */
+                if (TRANSLATE (*(p - 2)) == TRANSLATE ('.')
+		    && zero_times_ok
+                    && p < pend && TRANSLATE (*p) == TRANSLATE ('\n')
+                    && !(syntax & RE_DOT_NEWLINE))
+                  { /* We have .*\n.  */
+                    STORE_JUMP (jump, b, laststart);
+                    keep_string_p = true;
+                  }
+                else
+                  /* Anything else.  */
+                  STORE_JUMP (maybe_pop_jump, b, laststart - 3);
+
+                /* We've added more stuff to the buffer.  */
+                b += 3;
+              }
+
+            /* On failure, jump from laststart to b + 3, which will be the
+               end of the buffer after this jump is inserted.  */
+            GET_BUFFER_SPACE (3);
+            INSERT_JUMP (keep_string_p ? on_failure_keep_string_jump
+                                       : on_failure_jump,
+                         laststart, b + 3);
+            pending_exact = 0;
+            b += 3;
+
+            if (!zero_times_ok)
+              {
+                /* At least one repetition is required, so insert a
+                   `dummy_failure_jump' before the initial
+                   `on_failure_jump' instruction of the loop. This
+                   effects a skip over that instruction the first time
+                   we hit that loop.  */
+                GET_BUFFER_SPACE (3);
+                INSERT_JUMP (dummy_failure_jump, laststart, laststart + 6);
+                b += 3;
+              }
+            }
+	  break;
+
+
+	case '.':
+          laststart = b;
+          BUF_PUSH (anychar);
+          break;
+
+
+        case '[':
+          {
+            boolean had_char_class = false;
+
+            if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+            /* Ensure that we have enough space to push a charset: the
+               opcode, the length count, and the bitset; 34 bytes in all.  */
+	    GET_BUFFER_SPACE (34);
+
+            laststart = b;
+
+            /* We test `*p == '^' twice, instead of using an if
+               statement, so we only need one BUF_PUSH.  */
+            BUF_PUSH (*p == '^' ? charset_not : charset); 
+            if (*p == '^')
+              p++;
+
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+
+            /* Push the number of bytes in the bitmap.  */
+            BUF_PUSH ((1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* Clear the whole map.  */
+            bzero (b, (1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* charset_not matches newline according to a syntax bit.  */
+            if ((re_opcode_t) b[-2] == charset_not
+                && (syntax & RE_HAT_LISTS_NOT_NEWLINE))
+              SET_LIST_BIT ('\n');
+
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                PATFETCH (c);
+
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    SET_LIST_BIT (c1);
+                    continue;
+                  }
+
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  break;
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  FREE_STACK_RETURN (REG_ERANGE);
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                /* See if we're at the beginning of a possible character
+                   class.  */
+
+                else if (syntax & RE_CHAR_CLASSES && c == '[' && *p == ':')
+                  { /* Leave room for the null.  */
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+
+                    PATFETCH (c);
+                    c1 = 0;
+
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+                          break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = STREQ (str, "alnum");
+                        boolean is_alpha = STREQ (str, "alpha");
+                        boolean is_blank = STREQ (str, "blank");
+                        boolean is_cntrl = STREQ (str, "cntrl");
+                        boolean is_digit = STREQ (str, "digit");
+                        boolean is_graph = STREQ (str, "graph");
+                        boolean is_lower = STREQ (str, "lower");
+                        boolean is_print = STREQ (str, "print");
+                        boolean is_punct = STREQ (str, "punct");
+                        boolean is_space = STREQ (str, "space");
+                        boolean is_upper = STREQ (str, "upper");
+                        boolean is_xdigit = STREQ (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str))
+			  FREE_STACK_RETURN (REG_ECTYPE);
+
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+
+                        if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                        for (ch = 0; ch < 1 << BYTEWIDTH; ch++)
+                          {
+			    /* This was split into 3 if's to
+			       avoid an arbitrary limit in some compiler.  */
+                            if (   (is_alnum  && ISALNUM (ch))
+                                || (is_alpha  && ISALPHA (ch))
+                                || (is_blank  && ISBLANK (ch))
+                                || (is_cntrl  && ISCNTRL (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_digit  && ISDIGIT (ch))
+                                || (is_graph  && ISGRAPH (ch))
+                                || (is_lower  && ISLOWER (ch))
+                                || (is_print  && ISPRINT (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_punct  && ISPUNCT (ch))
+                                || (is_space  && ISSPACE (ch))
+                                || (is_upper  && ISUPPER (ch))
+                                || (is_xdigit && ISXDIGIT (ch)))
+			      SET_LIST_BIT (ch);
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+                        SET_LIST_BIT ('[');
+                        SET_LIST_BIT (':');
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+                    SET_LIST_BIT (c);
+                  }
+              }
+
+            /* Discard any (non)matching list bytes that are all 0 at the
+               end of the map.  Decrease the map-length byte too.  */
+            while ((int) b[-1] > 0 && b[b[-1] - 1] == 0) 
+              b[-1]--; 
+            b += b[-1];
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+           if (syntax & RE_INTERVALS && syntax & RE_NO_BK_BRACES)
+             goto handle_interval;
+           else
+             goto normal_char;
+
+
+        case '\\':
+          if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              bufp->re_nsub++;
+              regnum++;
+
+              if (COMPILE_STACK_FULL)
+                { 
+                  RETALLOC (compile_stack.stack, compile_stack.size << 1,
+                            compile_stack_elt_t);
+                  if (compile_stack.stack == NULL) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+              /* These are the values to restore when we hit end of this
+                 group.  They are all relative offsets, so that if the
+                 whole pattern moves because of realloc, they will still
+                 be valid.  */
+              COMPILE_STACK_TOP.begalt_offset = begalt - bufp->buffer;
+              COMPILE_STACK_TOP.fixup_alt_jump 
+                = fixup_alt_jump ? fixup_alt_jump - bufp->buffer + 1 : 0;
+              COMPILE_STACK_TOP.laststart_offset = b - bufp->buffer;
+              COMPILE_STACK_TOP.regnum = regnum;
+
+              /* We will eventually replace the 0 with the number of
+                 groups inner to this one.  But do not push a
+                 start_memory for groups beyond the last one we can
+                 represent in the compiled pattern.  */
+              if (regnum <= MAX_REGNUM)
+                {
+                  COMPILE_STACK_TOP.inner_group_offset = b - bufp->buffer + 2;
+                  BUF_PUSH_3 (start_memory, regnum, 0);
+                }
+                
+              compile_stack.avail++;
+
+              fixup_alt_jump = 0;
+              laststart = 0;
+              begalt = b;
+	      /* If we've reached MAX_REGNUM groups, then this open
+		 won't actually generate any code, so we'll have to
+		 clear pending_exact explicitly.  */
+	      pending_exact = 0;
+              break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_backslash;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+            handle_close:
+              if (fixup_alt_jump)
+                { /* Push a dummy failure point at the end of the
+                     alternative for a possible future
+                     `pop_failure_jump' to pop.  See comments at
+                     `push_dummy_failure' in `re_match_2'.  */
+                  BUF_PUSH (push_dummy_failure);
+                  
+                  /* We allocated space for this jump when we assigned
+                     to `fixup_alt_jump', in the `handle_alt' case below.  */
+                  STORE_JUMP (jump_past_alt, fixup_alt_jump, b - 1);
+                }
+
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+              assert (compile_stack.avail != 0);
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+
+                compile_stack.avail--;		
+                begalt = bufp->buffer + COMPILE_STACK_TOP.begalt_offset;
+                fixup_alt_jump
+                  = COMPILE_STACK_TOP.fixup_alt_jump
+                    ? bufp->buffer + COMPILE_STACK_TOP.fixup_alt_jump - 1 
+                    : 0;
+                laststart = bufp->buffer + COMPILE_STACK_TOP.laststart_offset;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		/* If we've reached MAX_REGNUM groups, then this open
+		   won't actually generate any code, so we'll have to
+		   clear pending_exact explicitly.  */
+		pending_exact = 0;
+
+                /* We're at the end of the group, so now we know how many
+                   groups were inside this one.  */
+                if (this_group_regnum <= MAX_REGNUM)
+                  {
+                    unsigned char *inner_group_loc
+                      = bufp->buffer + COMPILE_STACK_TOP.inner_group_offset;
+                    
+                    *inner_group_loc = regnum - this_group_regnum;
+                    BUF_PUSH_3 (stop_memory, this_group_regnum,
+                                regnum - this_group_regnum);
+                  }
+              }
+              break;
+
+
+            case '|':					/* `\|'.  */
+              if (syntax & RE_LIMITED_OPS || syntax & RE_NO_BK_VBAR)
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+              /* Insert before the previous alternative a jump which
+                 jumps to this alternative if the former fails.  */
+              GET_BUFFER_SPACE (3);
+              INSERT_JUMP (on_failure_jump, begalt, b + 6);
+              pending_exact = 0;
+              b += 3;
+
+              /* The alternative before this one has a jump after it
+                 which gets executed if it gets matched.  Adjust that
+                 jump so it will jump to this alternative's analogous
+                 jump (put in below, which in turn will jump to the next
+                 (if any) alternative's such jump, etc.).  The last such
+                 jump jumps to the correct final destination.  A picture:
+                          _____ _____ 
+                          |   | |   |   
+                          |   v |   v 
+                         a | b   | c   
+
+                 If we are at `b', then fixup_alt_jump right now points to a
+                 three-byte space after `a'.  We'll put in the jump, set
+                 fixup_alt_jump to right after `b', and leave behind three
+                 bytes which we'll fill in when we get to after `c'.  */
+
+              if (fixup_alt_jump)
+                STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+              /* Mark and leave space for a jump after this alternative,
+                 to be filled in later either by next alternative or
+                 when know we're at the end of a series of alternatives.  */
+              fixup_alt_jump = b;
+              GET_BUFFER_SPACE (3);
+              b += 3;
+
+              laststart = 0;
+              begalt = b;
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      FREE_STACK_RETURN (REG_EBRACE);
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') FREE_STACK_RETURN (REG_EBRACE);
+
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (!laststart)
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      FREE_STACK_RETURN (REG_BADRPT);
+                    else if (syntax & RE_CONTEXT_INDEP_OPS)
+                      laststart = b;
+                    else
+                      goto unfetch_interval;
+                  }
+
+                /* If the upper bound is zero, don't want to succeed at
+                   all; jump from `laststart' to `b + 3', which will be
+                   the end of the buffer after we insert the jump.  */
+                 if (upper_bound == 0)
+                   {
+                     GET_BUFFER_SPACE (3);
+                     INSERT_JUMP (jump, laststart, b + 3);
+                     b += 3;
+                   }
+
+                 /* Otherwise, we have a nontrivial interval.  When
+                    we're all done, the pattern will look like:
+                      set_number_at <jump count> <upper bound>
+                      set_number_at <succeed_n count> <lower bound>
+                      succeed_n <after jump addr> <succeed_n count>
+                      <body of loop>
+                      jump_n <succeed_n addr> <jump count>
+                    (The upper bound and `jump_n' are omitted if
+                    `upper_bound' is 1, though.)  */
+                 else 
+                   { /* If the upper bound is > 1, we need to insert
+                        more at the end of the loop.  */
+                     unsigned nbytes = 10 + (upper_bound > 1) * 10;
+
+                     GET_BUFFER_SPACE (nbytes);
+
+                     /* Initialize lower bound of the `succeed_n', even
+                        though it will be set during matching by its
+                        attendant `set_number_at' (inserted next),
+                        because `re_compile_fastmap' needs to know.
+                        Jump to the `jump_n' we might insert below.  */
+                     INSERT_JUMP2 (succeed_n, laststart,
+                                   b + 5 + (upper_bound > 1) * 5,
+                                   lower_bound);
+                     b += 5;
+
+                     /* Code to initialize the lower bound.  Insert 
+                        before the `succeed_n'.  The `5' is the last two
+                        bytes of this `set_number_at', plus 3 bytes of
+                        the following `succeed_n'.  */
+                     insert_op2 (set_number_at, laststart, 5, lower_bound, b);
+                     b += 5;
+
+                     if (upper_bound > 1)
+                       { /* More than one repetition is allowed, so
+                            append a backward jump to the `succeed_n'
+                            that starts this interval.
+                            
+                            When we've reached this during matching,
+                            we'll have matched the interval once, so
+                            jump back only `upper_bound - 1' times.  */
+                         STORE_JUMP2 (jump_n, b, laststart + 5,
+                                      upper_bound - 1);
+                         b += 5;
+
+                         /* The location we want to set is the second
+                            parameter of the `jump_n'; that is `b-2' as
+                            an absolute address.  `laststart' will be
+                            the `set_number_at' we're about to insert;
+                            `laststart+3' the number to set, the source
+                            for the relative address.  But we are
+                            inserting into the middle of the pattern --
+                            so everything is getting moved up by 5.
+                            Conclusion: (b - 2) - (laststart + 3) + 5,
+                            i.e., b - laststart.
+                            
+                            We insert this at the beginning of the loop
+                            so that if we fail during matching, we'll
+                            reinitialize the bounds.  */
+                         insert_op2 (set_number_at, laststart, b - laststart,
+                                     upper_bound - 1, b);
+                         b += 5;
+                       }
+                   }
+                pending_exact = 0;
+                beg_interval = NULL;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               assert (beg_interval);
+               p = beg_interval;
+               beg_interval = NULL;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+              BUF_PUSH (at_dot);
+              break;
+
+            case 's':	
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (syntaxspec, syntax_spec_code[c]);
+              break;
+
+            case 'S':
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (notsyntaxspec, syntax_spec_code[c]);
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+              laststart = b;
+              BUF_PUSH (wordchar);
+              break;
+
+
+            case 'W':
+              laststart = b;
+              BUF_PUSH (notwordchar);
+              break;
+
+
+            case '<':
+              BUF_PUSH (wordbeg);
+              break;
+
+            case '>':
+              BUF_PUSH (wordend);
+              break;
+
+            case 'b':
+              BUF_PUSH (wordbound);
+              break;
+
+            case 'B':
+              BUF_PUSH (notwordbound);
+              break;
+
+            case '`':
+              BUF_PUSH (begbuf);
+              break;
+
+            case '\'':
+              BUF_PUSH (endbuf);
+              break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                FREE_STACK_RETURN (REG_ESUBREG);
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+                goto normal_char;
+
+              laststart = b;
+              BUF_PUSH_2 (duplicate, c1);
+              break;
+
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	      /* If no exactn currently being built.  */
+          if (!pending_exact 
+
+              /* If last exactn not at current position.  */
+              || pending_exact + *pending_exact + 1 != b
+              
+              /* We have only one byte following the exactn for the count.  */
+	      || *pending_exact == (1 << BYTEWIDTH) - 1
+
+              /* If followed by a repetition operator.  */
+              || *p == '*' || *p == '^'
+	      || ((syntax & RE_BK_PLUS_QM)
+		  ? *p == '\\' && (p[1] == '+' || p[1] == '?')
+		  : (*p == '+' || *p == '?'))
+	      || ((syntax & RE_INTERVALS)
+                  && ((syntax & RE_NO_BK_BRACES)
+		      ? *p == '{'
+                      : (p[0] == '\\' && p[1] == '{'))))
+	    {
+	      /* Start building a new exactn.  */
+              
+              laststart = b;
+
+	      BUF_PUSH_2 (exactn, 0);
+	      pending_exact = b - 1;
+            }
+            
+	  BUF_PUSH (c);
+          (*pending_exact)++;
+	  break;
+        } /* switch (c) */
+    } /* while p != pend */
+
+  
+  /* Through the pattern now.  */
+  
+  if (fixup_alt_jump)
+    STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+  if (!COMPILE_STACK_EMPTY) 
+    FREE_STACK_RETURN (REG_EPAREN);
+
+  /* If we don't want backtracking, force success
+     the first time we reach the end of the compiled pattern.  */
+  if (syntax & RE_NO_POSIX_BACKTRACKING)
+    BUF_PUSH (succeed);
+
+  free (compile_stack.stack);
+
+  /* We have succeeded; set the length of the buffer.  */
+  bufp->used = b - bufp->buffer;
+
+#ifdef DEBUG
+  if (debug)
+    {
+      DEBUG_PRINT1 ("\nCompiled pattern: \n");
+      print_compiled_pattern (bufp);
+    }
+#endif /* DEBUG */
+
+#ifndef MATCH_MAY_ALLOCATE
+  /* Initialize the failure stack to the largest possible stack.  This
+     isn't necessary unless we're trying to avoid calling alloca in
+     the search and match routines.  */
+  {
+    int num_regs = bufp->re_nsub + 1;
+
+    /* Since DOUBLE_FAIL_STACK refuses to double only if the current size
+       is strictly greater than re_max_failures, the largest possible stack
+       is 2 * re_max_failures failure points.  */
+    if (fail_stack.size < (2 * re_max_failures * MAX_FAILURE_ITEMS))
+      {
+	fail_stack.size = (2 * re_max_failures * MAX_FAILURE_ITEMS);
+
+#ifdef emacs
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xmalloc (fail_stack.size 
+					    * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xrealloc (fail_stack.stack,
+					     (fail_stack.size
+					      * sizeof (fail_stack_elt_t)));
+#else /* not emacs */
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) malloc (fail_stack.size 
+					   * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) realloc (fail_stack.stack,
+					    (fail_stack.size
+					     * sizeof (fail_stack_elt_t)));
+#endif /* not emacs */
+      }
+
+    regex_grow_registers (num_regs);
+  }
+#endif /* not MATCH_MAY_ALLOCATE */
+
+  return REG_NOERROR;
+} /* regex_compile */
+
+
+/* Subroutines for `regex_compile'.  */
+
+/* Store OP at LOC followed by two-byte integer parameter ARG.  */
+
+static void
+store_op1 (op, loc, arg)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg);
+}
+
+
+/* Like `store_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+store_op2 (op, loc, arg1, arg2)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg1);
+  STORE_NUMBER (loc + 3, arg2);
+}
+
+
+/* Copy the bytes from LOC to END to open up three bytes of space at LOC
+   for OP followed by two-byte integer parameter ARG.  */
+
+static void
+insert_op1 (op, loc, arg, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 3;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op1 (op, loc, arg);
+}
+
+
+/* Like `insert_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+insert_op2 (op, loc, arg1, arg2, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 5;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op2 (op, loc, arg1, arg2);
+}
+
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+   after an alternative or a begin-subexpression.  We assume there is at
+   least one character before the ^.  */
+
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+    const char *pattern, *p;
+    reg_syntax_t syntax;
+{
+  const char *prev = p - 2;
+  boolean prev_prev_backslash = prev > pattern && prev[-1] == '\\';
+  
+  return
+       /* After a subexpression?  */
+       (*prev == '(' && (syntax & RE_NO_BK_PARENS || prev_prev_backslash))
+       /* After an alternative?  */
+    || (*prev == '|' && (syntax & RE_NO_BK_VBAR || prev_prev_backslash));
+}
+
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+   at least one character after the $, i.e., `P < PEND'.  */
+
+static boolean
+at_endline_loc_p (p, pend, syntax)
+    const char *p, *pend;
+    int syntax;
+{
+  const char *next = p;
+  boolean next_backslash = *next == '\\';
+  const char *next_next = p + 1 < pend ? p + 1 : NULL;
+  
+  return
+       /* Before a subexpression?  */
+       (syntax & RE_NO_BK_PARENS ? *next == ')'
+        : next_backslash && next_next && *next_next == ')')
+       /* Before an alternative?  */
+    || (syntax & RE_NO_BK_VBAR ? *next == '|'
+        : next_backslash && next_next && *next_next == '|');
+}
+
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/* Read the ending character of a range (in a bracket expression) from the
+   uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+   starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+   Then we set the translation of all bits between the starting and
+   ending characters (inclusive) in the compiled pattern B.
+   
+   Return an error code.
+   
+   We use these short variable names so we can use the same macros as
+   `regex_compile' itself.  */
+
+static reg_errcode_t
+compile_range (p_ptr, pend, translate, syntax, b)
+    const char **p_ptr, *pend;
+    char *translate;
+    reg_syntax_t syntax;
+    unsigned char *b;
+{
+  unsigned this_char;
+
+  const char *p = *p_ptr;
+  int range_start, range_end;
+  
+  if (p == pend)
+    return REG_ERANGE;
+
+  /* Even though the pattern is a signed `char *', we need to fetch
+     with unsigned char *'s; if the high bit of the pattern character
+     is set, the range endpoints will be negative if we fetch using a
+     signed char *.
+
+     We also want to fetch the endpoints without translating them; the 
+     appropriate translation is done in the bit-setting loop below.  */
+  /* The SVR4 compiler on the 3B2 had trouble with unsigned const char *.  */
+  range_start = ((const unsigned char *) p)[-2];
+  range_end   = ((const unsigned char *) p)[0];
+
+  /* Have to increment the pointer into the pattern string, so the
+     caller isn't still at the ending character.  */
+  (*p_ptr)++;
+
+  /* If the start is after the end, the range is empty.  */
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  /* Here we see why `this_char' has to be larger than an `unsigned
+     char' -- the range is inclusive, so if `range_end' == 0xff
+     (assuming 8-bit characters), we would otherwise go into an infinite
+     loop, since all characters <= 0xff.  */
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      SET_LIST_BIT (TRANSLATE (this_char));
+    }
+  
+  return REG_NOERROR;
+}
+
+
+/* re_compile_fastmap computes a ``fastmap'' for the compiled pattern in
+   BUFP.  A fastmap records which of the (1 << BYTEWIDTH) possible
+   characters can start a string that matches the pattern.  This fastmap
+   is used by re_search to skip quickly over impossible starting points.
+
+   The caller must supply the address of a (1 << BYTEWIDTH)-byte data
+   area as BUFP->fastmap.
+   
+   We set the `fastmap', `fastmap_accurate', and `can_be_null' fields in
+   the pattern buffer.
+
+   Returns 0 if we succeed, -2 if an internal error.   */
+
+int
+re_compile_fastmap (bufp)
+     struct re_pattern_buffer *bufp;
+{
+  int j, k;
+#ifdef MATCH_MAY_ALLOCATE
+  fail_stack_type fail_stack;
+#endif
+#ifndef REGEX_MALLOC
+  char *destination;
+#endif
+  /* We don't push any register information onto the failure stack.  */
+  unsigned num_regs = 0;
+  
+  register char *fastmap = bufp->fastmap;
+  unsigned char *pattern = bufp->buffer;
+  unsigned long size = bufp->used;
+  unsigned char *p = pattern;
+  register unsigned char *pend = pattern + size;
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* Assume that each path through the pattern can be null until
+     proven otherwise.  We set this false at the bottom of switch
+     statement, to which we get only if a particular path doesn't
+     match the empty string.  */
+  boolean path_can_be_null = true;
+
+  /* We aren't doing a `succeed_n' to begin with.  */
+  boolean succeed_n_p = false;
+
+  assert (fastmap != NULL && p != NULL);
+  
+  INIT_FAIL_STACK ();
+  bzero (fastmap, 1 << BYTEWIDTH);  /* Assume nothing's valid.  */
+  bufp->fastmap_accurate = 1;	    /* It will be when we're done.  */
+  bufp->can_be_null = 0;
+      
+  while (1)
+    {
+      if (p == pend || *p == succeed)
+	{
+	  /* We have reached the (effective) end of pattern.  */
+	  if (!FAIL_STACK_EMPTY ())
+	    {
+	      bufp->can_be_null |= path_can_be_null;
+
+	      /* Reset for next path.  */
+	      path_can_be_null = true;
+
+	      p = fail_stack.stack[--fail_stack.avail].pointer;
+
+	      continue;
+	    }
+	  else
+	    break;
+	}
+
+      /* We should never be about to go beyond the end of the pattern.  */
+      assert (p < pend);
+      
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+
+        /* I guess the idea here is to simply not bother with a fastmap
+           if a backreference is used, since it's too hard to figure out
+           the fastmap for the corresponding group.  Setting
+           `can_be_null' stops `re_search_2' from using the fastmap, so
+           that is all we do.  */
+	case duplicate:
+	  bufp->can_be_null = 1;
+          goto done;
+
+
+      /* Following are the cases which match a character.  These end
+         with `break'.  */
+
+	case exactn:
+          fastmap[p[1]] = 1;
+	  break;
+
+
+        case charset:
+          for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH)))
+              fastmap[j] = 1;
+	  break;
+
+
+	case charset_not:
+	  /* Chars beyond end of map must be allowed.  */
+	  for (j = *p * BYTEWIDTH; j < (1 << BYTEWIDTH); j++)
+            fastmap[j] = 1;
+
+	  for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (!(p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH))))
+              fastmap[j] = 1;
+          break;
+
+
+	case wordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notwordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+        case anychar:
+	  {
+	    int fastmap_newline = fastmap['\n'];
+
+	    /* `.' matches anything ...  */
+	    for (j = 0; j < (1 << BYTEWIDTH); j++)
+	      fastmap[j] = 1;
+
+	    /* ... except perhaps newline.  */
+	    if (!(bufp->syntax & RE_DOT_NEWLINE))
+	      fastmap['\n'] = fastmap_newline;
+
+	    /* Return if we have already set `can_be_null'; if we have,
+	       then the fastmap is irrelevant.  Something's wrong here.  */
+	    else if (bufp->can_be_null)
+	      goto done;
+
+	    /* Otherwise, have to check alternative paths.  */
+	    break;
+	  }
+
+#ifdef emacs
+        case syntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notsyntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+      /* All cases after this match the empty string.  These end with
+         `continue'.  */
+
+
+	case before_dot:
+	case at_dot:
+	case after_dot:
+          continue;
+#endif /* not emacs */
+
+
+        case no_op:
+        case begline:
+        case endline:
+	case begbuf:
+	case endbuf:
+	case wordbound:
+	case notwordbound:
+	case wordbeg:
+	case wordend:
+        case push_dummy_failure:
+          continue;
+
+
+	case jump_n:
+        case pop_failure_jump:
+	case maybe_pop_jump:
+	case jump:
+        case jump_past_alt:
+	case dummy_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+	  p += j;	
+	  if (j > 0)
+	    continue;
+            
+          /* Jump backward implies we just went through the body of a
+             loop and matched nothing.  Opcode jumped to should be
+             `on_failure_jump' or `succeed_n'.  Just treat it like an
+             ordinary jump.  For a * loop, it has pushed its failure
+             point already; if so, discard that as redundant.  */
+          if ((re_opcode_t) *p != on_failure_jump
+	      && (re_opcode_t) *p != succeed_n)
+	    continue;
+
+          p++;
+          EXTRACT_NUMBER_AND_INCR (j, p);
+          p += j;		
+	  
+          /* If what's on the stack is where we are now, pop it.  */
+          if (!FAIL_STACK_EMPTY () 
+	      && fail_stack.stack[fail_stack.avail - 1].pointer == p)
+            fail_stack.avail--;
+
+          continue;
+
+
+        case on_failure_jump:
+        case on_failure_keep_string_jump:
+	handle_on_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+
+          /* For some patterns, e.g., `(a?)?', `p+j' here points to the
+             end of the pattern.  We don't want to push such a point,
+             since when we restore it above, entering the switch will
+             increment `p' past the end of the pattern.  We don't need
+             to push such a point since we obviously won't find any more
+             fastmap entries beyond `pend'.  Such a pattern can match
+             the null string, though.  */
+          if (p + j < pend)
+            {
+              if (!PUSH_PATTERN_OP (p + j, fail_stack))
+		{
+		  RESET_FAIL_STACK ();
+		  return -2;
+		}
+            }
+          else
+            bufp->can_be_null = 1;
+
+          if (succeed_n_p)
+            {
+              EXTRACT_NUMBER_AND_INCR (k, p);	/* Skip the n.  */
+              succeed_n_p = false;
+	    }
+
+          continue;
+
+
+	case succeed_n:
+          /* Get to the number of times to succeed.  */
+          p += 2;		
+
+          /* Increment p past the n for when k != 0.  */
+          EXTRACT_NUMBER_AND_INCR (k, p);
+          if (k == 0)
+	    {
+              p -= 4;
+  	      succeed_n_p = true;  /* Spaghetti code alert.  */
+              goto handle_on_failure_jump;
+            }
+          continue;
+
+
+	case set_number_at:
+          p += 4;
+          continue;
+
+
+	case start_memory:
+        case stop_memory:
+	  p += 2;
+	  continue;
+
+
+	default:
+          abort (); /* We have listed all the cases.  */
+        } /* switch *p++ */
+
+      /* Getting here means we have found the possible starting
+         characters for one path of the pattern -- and that the empty
+         string does not match.  We need not follow this path further.
+         Instead, look at the next alternative (remembered on the
+         stack), or quit if no more.  The test at the top of the loop
+         does these things.  */
+      path_can_be_null = false;
+      p = pend;
+    } /* while p */
+
+  /* Set `can_be_null' for the last path (also the first path, if the
+     pattern is empty).  */
+  bufp->can_be_null |= path_can_be_null;
+
+ done:
+  RESET_FAIL_STACK ();
+  return 0;
+} /* re_compile_fastmap */
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+    struct re_pattern_buffer *bufp;
+    struct re_registers *regs;
+    unsigned num_regs;
+    regoff_t *starts, *ends;
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t *) 0;
+    }
+}
+
+
+/* Searching routines.  */
+
+/* Like re_search_2, below, but only one string is specified, and
+   doesn't let you say where to stop matching. */
+
+int
+re_search (bufp, string, size, startpos, range, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, startpos, range;
+     struct re_registers *regs;
+{
+  return re_search_2 (bufp, NULL, 0, string, size, startpos, range, 
+		      regs, size);
+}
+
+
+/* Using the compiled pattern in BUFP->buffer, first tries to match the
+   virtual concatenation of STRING1 and STRING2, starting first at index
+   STARTPOS, then at STARTPOS + 1, and so on.
+   
+   STRING1 and STRING2 have length SIZE1 and SIZE2, respectively.
+   
+   RANGE is how far to scan while trying to match.  RANGE = 0 means try
+   only at STARTPOS; in general, the last start tried is STARTPOS +
+   RANGE.
+   
+   In REGS, return the indices of the virtual concatenation of STRING1
+   and STRING2 that matched the entire BUFP->buffer and its contained
+   subexpressions.
+   
+   Do not consider matching one past the index STOP in the virtual
+   concatenation of STRING1 and STRING2.
+
+   We return either the position in the strings at which the match was
+   found, -1 if no match, or -2 if error (such as failure
+   stack overflow).  */
+
+int
+re_search_2 (bufp, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+{
+  int val;
+  register char *fastmap = bufp->fastmap;
+  register char *translate = bufp->translate;
+  int total_size = size1 + size2;
+  int endpos = startpos + range;
+
+  /* Check for out-of-range STARTPOS.  */
+  if (startpos < 0 || startpos > total_size)
+    return -1;
+    
+  /* Fix up RANGE if it might eventually take us outside
+     the virtual concatenation of STRING1 and STRING2.  */
+  if (endpos < -1)
+    range = -1 - startpos;
+  else if (endpos > total_size)
+    range = total_size - startpos;
+
+  /* If the search isn't to be a backwards one, don't waste time in a
+     search for a pattern that must be anchored.  */
+  if (bufp->used > 0 && (re_opcode_t) bufp->buffer[0] == begbuf && range > 0)
+    {
+      if (startpos > 0)
+	return -1;
+      else
+	range = 1;
+    }
+
+  /* Update the fastmap now if not correct already.  */
+  if (fastmap && !bufp->fastmap_accurate)
+    if (re_compile_fastmap (bufp) == -2)
+      return -2;
+  
+  /* Loop through the string, looking for a place to start matching.  */
+  for (;;)
+    { 
+      /* If a fastmap is supplied, skip quickly over characters that
+         cannot be the start of a match.  If the pattern can match the
+         null string, however, we don't need to skip characters; we want
+         the first null string.  */
+      if (fastmap && startpos < total_size && !bufp->can_be_null)
+	{
+	  if (range > 0)	/* Searching forwards.  */
+	    {
+	      register const char *d;
+	      register int lim = 0;
+	      int irange = range;
+
+              if (startpos < size1 && startpos + range >= size1)
+                lim = range - (size1 - startpos);
+
+	      d = (startpos >= size1 ? string2 - size1 : string1) + startpos;
+   
+              /* Written out as an if-else to avoid testing `translate'
+                 inside the loop.  */
+	      if (translate)
+                while (range > lim
+                       && !fastmap[(unsigned char)
+				   translate[(unsigned char) *d++]])
+                  range--;
+	      else
+                while (range > lim && !fastmap[(unsigned char) *d++])
+                  range--;
+
+	      startpos += irange - range;
+	    }
+	  else				/* Searching backwards.  */
+	    {
+	      register char c = (size1 == 0 || startpos >= size1
+                                 ? string2[startpos - size1] 
+                                 : string1[startpos]);
+
+	      if (!fastmap[(unsigned char) TRANSLATE (c)])
+		goto advance;
+	    }
+	}
+
+      /* If can't match the null string, and that's all we have left, fail.  */
+      if (range >= 0 && startpos == total_size && fastmap
+          && !bufp->can_be_null)
+	return -1;
+
+      val = re_match_2_internal (bufp, string1, size1, string2, size2,
+				 startpos, regs, stop);
+#ifndef REGEX_MALLOC
+#ifdef C_ALLOCA
+      alloca (0);
+#endif
+#endif
+
+      if (val >= 0)
+	return startpos;
+        
+      if (val == -2)
+	return -2;
+
+    advance:
+      if (!range) 
+        break;
+      else if (range > 0) 
+        {
+          range--; 
+          startpos++;
+        }
+      else
+        {
+          range++; 
+          startpos--;
+        }
+    }
+  return -1;
+} /* re_search_2 */
+
+
+/* Declarations and macros for re_match_2.  */
+
+static int bcmp_translate ();
+static boolean alt_match_null_string_p (),
+               common_op_match_null_string_p (),
+               group_match_null_string_p ();
+
+/* This converts PTR, a pointer into one of the search strings `string1'
+   and `string2' into an offset from the beginning of that string.  */
+#define POINTER_TO_OFFSET(ptr)			\
+  (FIRST_STRING_P (ptr)				\
+   ? ((regoff_t) ((ptr) - string1))		\
+   : ((regoff_t) ((ptr) - string2 + size1)))
+
+/* Macros for dealing with the split strings in re_match_2.  */
+
+#define MATCHING_IN_FIRST_STRING  (dend == end_match_1)
+
+/* Call before fetching a character with *d.  This switches over to
+   string2 if necessary.  */
+#define PREFETCH()							\
+  while (d == dend)						    	\
+    {									\
+      /* End of string2 => fail.  */					\
+      if (dend == end_match_2) 						\
+        goto fail;							\
+      /* End of string1 => advance to string2.  */ 			\
+      d = string2;						        \
+      dend = end_match_2;						\
+    }
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+   of `string1' and `string2'.  If only one string, it's `string2'.  */
+#define AT_STRINGS_BEG(d) ((d) == (size1 ? string1 : string2) || !size2)
+#define AT_STRINGS_END(d) ((d) == end2)	
+
+
+/* Test if D points to a character which is word-constituent.  We have
+   two special cases to check for: if past the end of string1, look at
+   the first character in string2; and if before the beginning of
+   string2, look at the last character in string1.  */
+#define WORDCHAR_P(d)							\
+  (SYNTAX ((d) == end1 ? *string2					\
+           : (d) == string2 - 1 ? *(end1 - 1) : *(d))			\
+   == Sword)
+
+/* Test if the character before D and the one at D differ with respect
+   to being word-constituent.  */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG (d) || AT_STRINGS_END (d)				\
+   || WORDCHAR_P (d - 1) != WORDCHAR_P (d))
+
+
+/* Free everything we malloc.  */
+#ifdef MATCH_MAY_ALLOCATE
+#define FREE_VAR(var) if (var) REGEX_FREE (var); var = NULL
+#define FREE_VARIABLES()						\
+  do {									\
+    REGEX_FREE_STACK (fail_stack.stack);				\
+    FREE_VAR (regstart);						\
+    FREE_VAR (regend);							\
+    FREE_VAR (old_regstart);						\
+    FREE_VAR (old_regend);						\
+    FREE_VAR (best_regstart);						\
+    FREE_VAR (best_regend);						\
+    FREE_VAR (reg_info);						\
+    FREE_VAR (reg_dummy);						\
+    FREE_VAR (reg_info_dummy);						\
+  } while (0)
+#else
+#define FREE_VARIABLES() ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+#endif /* not MATCH_MAY_ALLOCATE */
+
+/* These values must meet several constraints.  They must not be valid
+   register values; since we have a limit of 255 registers (because
+   we use only one byte in the pattern for the register number), we can
+   use numbers larger than 255.  They must differ by 1, because of
+   NUM_FAILURE_ITEMS above.  And the value for the lowest register must
+   be larger than the value for the highest register, so we do not try
+   to actually save any registers when none are active.  */
+#define NO_HIGHEST_ACTIVE_REG (1 << BYTEWIDTH)
+#define NO_LOWEST_ACTIVE_REG (NO_HIGHEST_ACTIVE_REG + 1)
+
+
+/* Matching routines.  */
+
+#ifndef emacs   /* Emacs never uses this.  */
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+int
+re_match (bufp, string, size, pos, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, pos;
+     struct re_registers *regs;
+{
+  int result = re_match_2_internal (bufp, NULL, 0, string, size,
+				    pos, regs, size);
+  alloca (0);
+  return result;
+}
+#endif /* not emacs */
+
+
+/* re_match_2 matches the compiled pattern in BUFP against the
+   the (virtual) concatenation of STRING1 and STRING2 (of length SIZE1
+   and SIZE2, respectively).  We start matching at POS, and stop
+   matching at STOP.
+   
+   If REGS is non-null and the `no_sub' field of BUFP is nonzero, we
+   store offsets for the substring each group matched in REGS.  See the
+   documentation for exactly how many groups we fill.
+
+   We return -1 if no match, -2 if an internal error (such as the
+   failure stack overflowing).  Otherwise, we return the length of the
+   matched substring.  */
+
+int
+re_match_2 (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  int result = re_match_2_internal (bufp, string1, size1, string2, size2,
+				    pos, regs, stop);
+  alloca (0);
+  return result;
+}
+
+/* This is a separate function so that we can force an alloca cleanup
+   afterwards.  */
+static int
+re_match_2_internal (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  /* General temporaries.  */
+  int mcnt;
+  unsigned char *p1;
+
+  /* Just past the end of the corresponding string.  */
+  const char *end1, *end2;
+
+  /* Pointers into string1 and string2, just past the last characters in
+     each to consider matching.  */
+  const char *end_match_1, *end_match_2;
+
+  /* Where we are in the data, and the end of the current string.  */
+  const char *d, *dend;
+  
+  /* Where we are in the pattern, and the end of the pattern.  */
+  unsigned char *p = bufp->buffer;
+  register unsigned char *pend = p + bufp->used;
+
+  /* Mark the opcode just after a start_memory, so we can test for an
+     empty subpattern when we get to the stop_memory.  */
+  unsigned char *just_past_start_mem = 0;
+
+  /* We use this to map every character in the string.  */
+  char *translate = bufp->translate;
+
+  /* Failure point stack.  Each place that can handle a failure further
+     down the line pushes a failure point on this stack.  It consists of
+     restart, regend, and reg_info for all registers corresponding to
+     the subexpressions we're currently inside, plus the number of such
+     registers, and, finally, two char *'s.  The first char * is where
+     to resume scanning the pattern; the second one is where to resume
+     scanning the strings.  If the latter is zero, the failure point is
+     a ``dummy''; if a failure happens and the failure point is a dummy,
+     it gets discarded and the next next one is tried.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  fail_stack_type fail_stack;
+#endif
+#ifdef DEBUG
+  static unsigned failure_id = 0;
+  unsigned nfailure_points_pushed = 0, nfailure_points_popped = 0;
+#endif
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* We fill all the registers internally, independent of what we
+     return, for use in backreferences.  The number here includes
+     an element for register zero.  */
+  unsigned num_regs = bufp->re_nsub + 1;
+  
+  /* The currently active registers.  */
+  unsigned lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+  unsigned highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+
+  /* Information on the contents of registers. These are pointers into
+     the input strings; they record just what was matched (on this
+     attempt) by a subexpression part of the pattern, that is, the
+     regnum-th regstart pointer points to where in the pattern we began
+     matching and the regnum-th regend points to right after where we
+     stopped matching the regnum-th subexpression.  (The zeroth register
+     keeps track of what the whole pattern matches.)  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **regstart, **regend;
+#endif
+
+  /* If a group that's operated upon by a repetition operator fails to
+     match anything, then the register for its start will need to be
+     restored because it will have been set to wherever in the string we
+     are when we last see its open-group operator.  Similarly for a
+     register's end.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **old_regstart, **old_regend;
+#endif
+
+  /* The is_active field of reg_info helps us keep track of which (possibly
+     nested) subexpressions we are currently in. The matched_something
+     field of reg_info[reg_num] helps us tell whether or not we have
+     matched any of the pattern so far this time through the reg_num-th
+     subexpression.  These two fields get reset each time through any
+     loop their register is in.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  register_info_type *reg_info; 
+#endif
+
+  /* The following record the register info as found in the above
+     variables when we find a match better than any we've seen before. 
+     This happens as we backtrack through the failure points, which in
+     turn happens only if we have not yet matched the entire string. */
+  unsigned best_regs_set = false;
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **best_regstart, **best_regend;
+#endif
+  
+  /* Logically, this is `best_regend[0]'.  But we don't want to have to
+     allocate space for that if we're not allocating space for anything
+     else (see below).  Also, we never need info about register 0 for
+     any of the other register vectors, and it seems rather a kludge to
+     treat `best_regend' differently than the rest.  So we keep track of
+     the end of the best match so far in a separate variable.  We
+     initialize this to NULL so that when we backtrack the first time
+     and need to test it, it's not garbage.  */
+  const char *match_end = NULL;
+
+  /* This helps SET_REGS_MATCHED avoid doing redundant work.  */
+  int set_regs_matched_done = 0;
+
+  /* Used when we pop values we don't care about.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **reg_dummy;
+  register_info_type *reg_info_dummy;
+#endif
+
+#ifdef DEBUG
+  /* Counts the total number of registers pushed.  */
+  unsigned num_regs_pushed = 0; 	
+#endif
+
+  DEBUG_PRINT1 ("\n\nEntering re_match_2.\n");
+  
+  INIT_FAIL_STACK ();
+  
+#ifdef MATCH_MAY_ALLOCATE
+  /* Do not bother to initialize all the register variables if there are
+     no groups in the pattern, as it takes a fair amount of time.  If
+     there are groups, we include space for register 0 (the whole
+     pattern), even though we never use it, since it simplifies the
+     array indexing.  We should fix this.  */
+  if (bufp->re_nsub)
+    {
+      regstart = REGEX_TALLOC (num_regs, const char *);
+      regend = REGEX_TALLOC (num_regs, const char *);
+      old_regstart = REGEX_TALLOC (num_regs, const char *);
+      old_regend = REGEX_TALLOC (num_regs, const char *);
+      best_regstart = REGEX_TALLOC (num_regs, const char *);
+      best_regend = REGEX_TALLOC (num_regs, const char *);
+      reg_info = REGEX_TALLOC (num_regs, register_info_type);
+      reg_dummy = REGEX_TALLOC (num_regs, const char *);
+      reg_info_dummy = REGEX_TALLOC (num_regs, register_info_type);
+
+      if (!(regstart && regend && old_regstart && old_regend && reg_info 
+            && best_regstart && best_regend && reg_dummy && reg_info_dummy)) 
+        {
+          FREE_VARIABLES ();
+          return -2;
+        }
+    }
+  else
+    {
+      /* We must initialize all our variables to NULL, so that
+         `FREE_VARIABLES' doesn't try to free them.  */
+      regstart = regend = old_regstart = old_regend = best_regstart
+        = best_regend = reg_dummy = NULL;
+      reg_info = reg_info_dummy = (register_info_type *) NULL;
+    }
+#endif /* MATCH_MAY_ALLOCATE */
+
+  /* The starting position is bogus.  */
+  if (pos < 0 || pos > size1 + size2)
+    {
+      FREE_VARIABLES ();
+      return -1;
+    }
+    
+  /* Initialize subexpression text positions to -1 to mark ones that no
+     start_memory/stop_memory has been seen for. Also initialize the
+     register information struct.  */
+  for (mcnt = 1; mcnt < num_regs; mcnt++)
+    {
+      regstart[mcnt] = regend[mcnt] 
+        = old_regstart[mcnt] = old_regend[mcnt] = REG_UNSET_VALUE;
+        
+      REG_MATCH_NULL_STRING_P (reg_info[mcnt]) = MATCH_NULL_UNSET_VALUE;
+      IS_ACTIVE (reg_info[mcnt]) = 0;
+      MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+      EVER_MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+    }
+  
+  /* We move `string1' into `string2' if the latter's empty -- but not if
+     `string1' is null.  */
+  if (size2 == 0 && string1 != NULL)
+    {
+      string2 = string1;
+      size2 = size1;
+      string1 = 0;
+      size1 = 0;
+    }
+  end1 = string1 + size1;
+  end2 = string2 + size2;
+
+  /* Compute where to stop matching, within the two strings.  */
+  if (stop <= size1)
+    {
+      end_match_1 = string1 + stop;
+      end_match_2 = string2;
+    }
+  else
+    {
+      end_match_1 = end1;
+      end_match_2 = string2 + stop - size1;
+    }
+
+  /* `p' scans through the pattern as `d' scans through the data. 
+     `dend' is the end of the input string that `d' points within.  `d'
+     is advanced into the following input string whenever necessary, but
+     this happens before fetching; therefore, at the beginning of the
+     loop, `d' can be pointing at the end of a string, but it cannot
+     equal `string2'.  */
+  if (size1 > 0 && pos <= size1)
+    {
+      d = string1 + pos;
+      dend = end_match_1;
+    }
+  else
+    {
+      d = string2 + pos - size1;
+      dend = end_match_2;
+    }
+
+  DEBUG_PRINT1 ("The compiled pattern is: ");
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, p, pend);
+  DEBUG_PRINT1 ("The string to match is: `");
+  DEBUG_PRINT_DOUBLE_STRING (d, string1, size1, string2, size2);
+  DEBUG_PRINT1 ("'\n");
+  
+  /* This loops over pattern commands.  It exits by returning from the
+     function if the match is complete, or it drops through if the match
+     fails at this starting point in the input data.  */
+  for (;;)
+    {
+      DEBUG_PRINT2 ("\n0x%x: ", p);
+
+      if (p == pend)
+	{ /* End of pattern means we might have succeeded.  */
+          DEBUG_PRINT1 ("end of pattern ... ");
+          
+	  /* If we haven't matched the entire string, and we want the
+             longest match, try backtracking.  */
+          if (d != end_match_2)
+	    {
+	      /* 1 if this match ends in the same string (string1 or string2)
+		 as the best previous match.  */
+	      boolean same_str_p = (FIRST_STRING_P (match_end) 
+				    == MATCHING_IN_FIRST_STRING);
+	      /* 1 if this match is the best seen so far.  */
+	      boolean best_match_p;
+
+	      /* AIX compiler got confused when this was combined
+		 with the previous declaration.  */
+	      if (same_str_p)
+		best_match_p = d > match_end;
+	      else
+		best_match_p = !MATCHING_IN_FIRST_STRING;
+
+              DEBUG_PRINT1 ("backtracking.\n");
+              
+              if (!FAIL_STACK_EMPTY ())
+                { /* More failure points to try.  */
+
+                  /* If exceeds best match so far, save it.  */
+                  if (!best_regs_set || best_match_p)
+                    {
+                      best_regs_set = true;
+                      match_end = d;
+                      
+                      DEBUG_PRINT1 ("\nSAVING match as best so far.\n");
+                      
+                      for (mcnt = 1; mcnt < num_regs; mcnt++)
+                        {
+                          best_regstart[mcnt] = regstart[mcnt];
+                          best_regend[mcnt] = regend[mcnt];
+                        }
+                    }
+                  goto fail;	       
+                }
+
+              /* If no failure points, don't restore garbage.  And if
+                 last match is real best match, don't restore second
+                 best one. */
+              else if (best_regs_set && !best_match_p)
+                {
+  	        restore_best_regs:
+                  /* Restore best match.  It may happen that `dend ==
+                     end_match_1' while the restored d is in string2.
+                     For example, the pattern `x.*y.*z' against the
+                     strings `x-' and `y-z-', if the two strings are
+                     not consecutive in memory.  */
+                  DEBUG_PRINT1 ("Restoring best registers.\n");
+                  
+                  d = match_end;
+                  dend = ((d >= string1 && d <= end1)
+		           ? end_match_1 : end_match_2);
+
+		  for (mcnt = 1; mcnt < num_regs; mcnt++)
+		    {
+		      regstart[mcnt] = best_regstart[mcnt];
+		      regend[mcnt] = best_regend[mcnt];
+		    }
+                }
+            } /* d != end_match_2 */
+
+	succeed_label:
+          DEBUG_PRINT1 ("Accepting match.\n");
+
+          /* If caller wants register contents data back, do it.  */
+          if (regs && !bufp->no_sub)
+	    {
+              /* Have the register data arrays been allocated?  */
+              if (bufp->regs_allocated == REGS_UNALLOCATED)
+                { /* No.  So allocate them with malloc.  We need one
+                     extra element beyond `num_regs' for the `-1' marker
+                     GNU code uses.  */
+                  regs->num_regs = MAX (RE_NREGS, num_regs + 1);
+                  regs->start = TALLOC (regs->num_regs, regoff_t);
+                  regs->end = TALLOC (regs->num_regs, regoff_t);
+                  if (regs->start == NULL || regs->end == NULL)
+		    {
+		      FREE_VARIABLES ();
+		      return -2;
+		    }
+                  bufp->regs_allocated = REGS_REALLOCATE;
+                }
+              else if (bufp->regs_allocated == REGS_REALLOCATE)
+                { /* Yes.  If we need more elements than were already
+                     allocated, reallocate them.  If we need fewer, just
+                     leave it alone.  */
+                  if (regs->num_regs < num_regs + 1)
+                    {
+                      regs->num_regs = num_regs + 1;
+                      RETALLOC (regs->start, regs->num_regs, regoff_t);
+                      RETALLOC (regs->end, regs->num_regs, regoff_t);
+                      if (regs->start == NULL || regs->end == NULL)
+			{
+			  FREE_VARIABLES ();
+			  return -2;
+			}
+                    }
+                }
+              else
+		{
+		  /* These braces fend off a "empty body in an else-statement"
+		     warning under GCC when assert expands to nothing.  */
+		  assert (bufp->regs_allocated == REGS_FIXED);
+		}
+
+              /* Convert the pointer data in `regstart' and `regend' to
+                 indices.  Register zero has to be set differently,
+                 since we haven't kept track of any info for it.  */
+              if (regs->num_regs > 0)
+                {
+                  regs->start[0] = pos;
+                  regs->end[0] = (MATCHING_IN_FIRST_STRING
+				  ? ((regoff_t) (d - string1))
+			          : ((regoff_t) (d - string2 + size1)));
+                }
+              
+              /* Go through the first `min (num_regs, regs->num_regs)'
+                 registers, since that is all we initialized.  */
+	      for (mcnt = 1; mcnt < MIN (num_regs, regs->num_regs); mcnt++)
+		{
+                  if (REG_UNSET (regstart[mcnt]) || REG_UNSET (regend[mcnt]))
+                    regs->start[mcnt] = regs->end[mcnt] = -1;
+                  else
+                    {
+		      regs->start[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regstart[mcnt]);
+                      regs->end[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regend[mcnt]);
+                    }
+		}
+              
+              /* If the regs structure we return has more elements than
+                 were in the pattern, set the extra elements to -1.  If
+                 we (re)allocated the registers, this is the case,
+                 because we always allocate enough to have at least one
+                 -1 at the end.  */
+              for (mcnt = num_regs; mcnt < regs->num_regs; mcnt++)
+                regs->start[mcnt] = regs->end[mcnt] = -1;
+	    } /* regs && !bufp->no_sub */
+
+          DEBUG_PRINT4 ("%u failure points pushed, %u popped (%u remain).\n",
+                        nfailure_points_pushed, nfailure_points_popped,
+                        nfailure_points_pushed - nfailure_points_popped);
+          DEBUG_PRINT2 ("%u registers pushed.\n", num_regs_pushed);
+
+          mcnt = d - pos - (MATCHING_IN_FIRST_STRING 
+			    ? string1 
+			    : string2 - size1);
+
+          DEBUG_PRINT2 ("Returning %d from re_match_2.\n", mcnt);
+
+          FREE_VARIABLES ();
+          return mcnt;
+        }
+
+      /* Otherwise match next pattern command.  */
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+        /* Ignore these.  Used to ignore the n of succeed_n's which
+           currently have n == 0.  */
+        case no_op:
+          DEBUG_PRINT1 ("EXECUTING no_op.\n");
+          break;
+
+	case succeed:
+          DEBUG_PRINT1 ("EXECUTING succeed.\n");
+	  goto succeed_label;
+
+        /* Match the next n pattern characters exactly.  The following
+           byte in the pattern defines n, and the n bytes after that
+           are the characters to match.  */
+	case exactn:
+	  mcnt = *p++;
+          DEBUG_PRINT2 ("EXECUTING exactn %d.\n", mcnt);
+
+          /* This is written out as an if-else so we don't waste time
+             testing `translate' inside the loop.  */
+          if (translate)
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (translate[(unsigned char) *d++] != (char) *p++)
+                    goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  else
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (*d++ != (char) *p++) goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  SET_REGS_MATCHED ();
+          break;
+
+
+        /* Match any character except possibly a newline or a null.  */
+	case anychar:
+          DEBUG_PRINT1 ("EXECUTING anychar.\n");
+
+          PREFETCH ();
+
+          if ((!(bufp->syntax & RE_DOT_NEWLINE) && TRANSLATE (*d) == '\n')
+              || (bufp->syntax & RE_DOT_NOT_NULL && TRANSLATE (*d) == '\000'))
+	    goto fail;
+
+          SET_REGS_MATCHED ();
+          DEBUG_PRINT2 ("  Matched `%d'.\n", *d);
+          d++;
+	  break;
+
+
+	case charset:
+	case charset_not:
+	  {
+	    register unsigned char c;
+	    boolean not = (re_opcode_t) *(p - 1) == charset_not;
+
+            DEBUG_PRINT2 ("EXECUTING charset%s.\n", not ? "_not" : "");
+
+	    PREFETCH ();
+	    c = TRANSLATE (*d); /* The character to match.  */
+
+            /* Cast to `unsigned' instead of `unsigned char' in case the
+               bit list is a full 32 bytes long.  */
+	    if (c < (unsigned) (*p * BYTEWIDTH)
+		&& p[1 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+	      not = !not;
+
+	    p += 1 + *p;
+
+	    if (!not) goto fail;
+            
+	    SET_REGS_MATCHED ();
+            d++;
+	    break;
+	  }
+
+
+        /* The beginning of a group is represented by start_memory.
+           The arguments are the register number in the next byte, and the
+           number of groups inner to this one in the next.  The text
+           matched within the group is recorded (in the internal
+           registers data structure) under the register number.  */
+        case start_memory:
+	  DEBUG_PRINT3 ("EXECUTING start_memory %d (%d):\n", *p, p[1]);
+
+          /* Find out if this group can match the empty string.  */
+	  p1 = p;		/* To send to group_match_null_string_p.  */
+          
+          if (REG_MATCH_NULL_STRING_P (reg_info[*p]) == MATCH_NULL_UNSET_VALUE)
+            REG_MATCH_NULL_STRING_P (reg_info[*p]) 
+              = group_match_null_string_p (&p1, pend, reg_info);
+
+          /* Save the position in the string where we were the last time
+             we were at this open-group operator in case the group is
+             operated upon by a repetition operator, e.g., with `(a*)*b'
+             against `ab'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regstart[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                             ? REG_UNSET (regstart[*p]) ? d : regstart[*p]
+                             : regstart[*p];
+	  DEBUG_PRINT2 ("  old_regstart: %d\n", 
+			 POINTER_TO_OFFSET (old_regstart[*p]));
+
+          regstart[*p] = d;
+	  DEBUG_PRINT2 ("  regstart: %d\n", POINTER_TO_OFFSET (regstart[*p]));
+
+          IS_ACTIVE (reg_info[*p]) = 1;
+          MATCHED_SOMETHING (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+          
+          /* This is the new highest active register.  */
+          highest_active_reg = *p;
+          
+          /* If nothing was active before, this is the new lowest active
+             register.  */
+          if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+            lowest_active_reg = *p;
+
+          /* Move past the register number and inner group count.  */
+          p += 2;
+	  just_past_start_mem = p;
+
+          break;
+
+
+        /* The stop_memory opcode represents the end of a group.  Its
+           arguments are the same as start_memory's: the register
+           number, and the number of inner groups.  */
+	case stop_memory:
+	  DEBUG_PRINT3 ("EXECUTING stop_memory %d (%d):\n", *p, p[1]);
+             
+          /* We need to save the string position the last time we were at
+             this close-group operator in case the group is operated
+             upon by a repetition operator, e.g., with `((a*)*(b*)*)*'
+             against `aba'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regend[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                           ? REG_UNSET (regend[*p]) ? d : regend[*p]
+			   : regend[*p];
+	  DEBUG_PRINT2 ("      old_regend: %d\n", 
+			 POINTER_TO_OFFSET (old_regend[*p]));
+
+          regend[*p] = d;
+	  DEBUG_PRINT2 ("      regend: %d\n", POINTER_TO_OFFSET (regend[*p]));
+
+          /* This register isn't active anymore.  */
+          IS_ACTIVE (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+
+          /* If this was the only register active, nothing is active
+             anymore.  */
+          if (lowest_active_reg == highest_active_reg)
+            {
+              lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+              highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+            }
+          else
+            { /* We must scan for the new highest active register, since
+                 it isn't necessarily one less than now: consider
+                 (a(b)c(d(e)f)g).  When group 3 ends, after the f), the
+                 new highest active register is 1.  */
+              unsigned char r = *p - 1;
+              while (r > 0 && !IS_ACTIVE (reg_info[r]))
+                r--;
+              
+              /* If we end up at register zero, that means that we saved
+                 the registers as the result of an `on_failure_jump', not
+                 a `start_memory', and we jumped to past the innermost
+                 `stop_memory'.  For example, in ((.)*) we save
+                 registers 1 and 2 as a result of the *, but when we pop
+                 back to the second ), we are at the stop_memory 1.
+                 Thus, nothing is active.  */
+	      if (r == 0)
+                {
+                  lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+                  highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+                }
+              else
+                highest_active_reg = r;
+            }
+          
+          /* If just failed to match something this time around with a
+             group that's operated on by a repetition operator, try to
+             force exit from the ``loop'', and restore the register
+             information for this group that we had before trying this
+             last match.  */
+          if ((!MATCHED_SOMETHING (reg_info[*p])
+               || just_past_start_mem == p - 1)
+	      && (p + 2) < pend)              
+            {
+              boolean is_a_jump_n = false;
+              
+              p1 = p + 2;
+              mcnt = 0;
+              switch ((re_opcode_t) *p1++)
+                {
+                  case jump_n:
+		    is_a_jump_n = true;
+                  case pop_failure_jump:
+		  case maybe_pop_jump:
+		  case jump:
+		  case dummy_failure_jump:
+                    EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+		    if (is_a_jump_n)
+		      p1 += 2;
+                    break;
+                  
+                  default:
+                    /* do nothing */ ;
+                }
+	      p1 += mcnt;
+        
+              /* If the next operation is a jump backwards in the pattern
+	         to an on_failure_jump right before the start_memory
+                 corresponding to this stop_memory, exit from the loop
+                 by forcing a failure after pushing on the stack the
+                 on_failure_jump's jump in the pattern, and d.  */
+              if (mcnt < 0 && (re_opcode_t) *p1 == on_failure_jump
+                  && (re_opcode_t) p1[3] == start_memory && p1[4] == *p)
+		{
+                  /* If this group ever matched anything, then restore
+                     what its registers were before trying this last
+                     failed match, e.g., with `(a*)*b' against `ab' for
+                     regstart[1], and, e.g., with `((a*)*(b*)*)*'
+                     against `aba' for regend[3].
+                     
+                     Also restore the registers for inner groups for,
+                     e.g., `((a*)(b*))*' against `aba' (register 3 would
+                     otherwise get trashed).  */
+                     
+                  if (EVER_MATCHED_SOMETHING (reg_info[*p]))
+		    {
+		      unsigned r; 
+        
+                      EVER_MATCHED_SOMETHING (reg_info[*p]) = 0;
+                      
+		      /* Restore this and inner groups' (if any) registers.  */
+                      for (r = *p; r < *p + *(p + 1); r++)
+                        {
+                          regstart[r] = old_regstart[r];
+
+                          /* xx why this test?  */
+                          if (old_regend[r] >= regstart[r])
+                            regend[r] = old_regend[r];
+                        }     
+                    }
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  PUSH_FAILURE_POINT (p1 + mcnt, d, -2);
+
+                  goto fail;
+                }
+            }
+          
+          /* Move past the register number and the inner group count.  */
+          p += 2;
+          break;
+
+
+	/* \<digit> has been turned into a `duplicate' command which is
+           followed by the numeric value of <digit> as the register number.  */
+        case duplicate:
+	  {
+	    register const char *d2, *dend2;
+	    int regno = *p++;   /* Get which register to match against.  */
+	    DEBUG_PRINT2 ("EXECUTING duplicate %d.\n", regno);
+
+	    /* Can't back reference a group which we've never matched.  */
+            if (REG_UNSET (regstart[regno]) || REG_UNSET (regend[regno]))
+              goto fail;
+              
+            /* Where in input to try to start matching.  */
+            d2 = regstart[regno];
+            
+            /* Where to stop matching; if both the place to start and
+               the place to stop matching are in the same string, then
+               set to the place to stop, otherwise, for now have to use
+               the end of the first string.  */
+
+            dend2 = ((FIRST_STRING_P (regstart[regno]) 
+		      == FIRST_STRING_P (regend[regno]))
+		     ? regend[regno] : end_match_1);
+	    for (;;)
+	      {
+		/* If necessary, advance to next segment in register
+                   contents.  */
+		while (d2 == dend2)
+		  {
+		    if (dend2 == end_match_2) break;
+		    if (dend2 == regend[regno]) break;
+
+                    /* End of string1 => advance to string2. */
+                    d2 = string2;
+                    dend2 = regend[regno];
+		  }
+		/* At end of register contents => success */
+		if (d2 == dend2) break;
+
+		/* If necessary, advance to next segment in data.  */
+		PREFETCH ();
+
+		/* How many characters left in this segment to match.  */
+		mcnt = dend - d;
+                
+		/* Want how many consecutive characters we can match in
+                   one shot, so, if necessary, adjust the count.  */
+                if (mcnt > dend2 - d2)
+		  mcnt = dend2 - d2;
+                  
+		/* Compare that many; failure if mismatch, else move
+                   past them.  */
+		if (translate 
+                    ? bcmp_translate (d, d2, mcnt, translate) 
+                    : bcmp (d, d2, mcnt))
+		  goto fail;
+		d += mcnt, d2 += mcnt;
+
+		/* Do this because we've match some characters.  */
+		SET_REGS_MATCHED ();
+	      }
+	  }
+	  break;
+
+
+        /* begline matches the empty string at the beginning of the string
+           (unless `not_bol' is set in `bufp'), and, if
+           `newline_anchor' is set, after newlines.  */
+	case begline:
+          DEBUG_PRINT1 ("EXECUTING begline.\n");
+          
+          if (AT_STRINGS_BEG (d))
+            {
+              if (!bufp->not_bol) break;
+            }
+          else if (d[-1] == '\n' && bufp->newline_anchor)
+            {
+              break;
+            }
+          /* In all other cases, we fail.  */
+          goto fail;
+
+
+        /* endline is the dual of begline.  */
+	case endline:
+          DEBUG_PRINT1 ("EXECUTING endline.\n");
+
+          if (AT_STRINGS_END (d))
+            {
+              if (!bufp->not_eol) break;
+            }
+          
+          /* We have to ``prefetch'' the next character.  */
+          else if ((d == end1 ? *string2 : *d) == '\n'
+                   && bufp->newline_anchor)
+            {
+              break;
+            }
+          goto fail;
+
+
+	/* Match at the very beginning of the data.  */
+        case begbuf:
+          DEBUG_PRINT1 ("EXECUTING begbuf.\n");
+          if (AT_STRINGS_BEG (d))
+            break;
+          goto fail;
+
+
+	/* Match at the very end of the data.  */
+        case endbuf:
+          DEBUG_PRINT1 ("EXECUTING endbuf.\n");
+	  if (AT_STRINGS_END (d))
+	    break;
+          goto fail;
+
+
+        /* on_failure_keep_string_jump is used to optimize `.*\n'.  It
+           pushes NULL as the value for the string on the stack.  Then
+           `pop_failure_point' will keep the current value for the
+           string, instead of restoring it.  To see why, consider
+           matching `foo\nbar' against `.*\n'.  The .* matches the foo;
+           then the . fails against the \n.  But the next thing we want
+           to do is match the \n against the \n; if we restored the
+           string value, we would be back at the foo.
+           
+           Because this is used only in specific cases, we don't need to
+           check all the things that `on_failure_jump' does, to make
+           sure the right things get saved on the stack.  Hence we don't
+           share its code.  The only reason to push anything on the
+           stack at all is that otherwise we would have to change
+           `anychar's code to do something besides goto fail in this
+           case; that seems worse than this.  */
+        case on_failure_keep_string_jump:
+          DEBUG_PRINT1 ("EXECUTING on_failure_keep_string_jump");
+          
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x):\n", mcnt, p + mcnt);
+
+          PUSH_FAILURE_POINT (p + mcnt, NULL, -2);
+          break;
+
+
+	/* Uses of on_failure_jump:
+        
+           Each alternative starts with an on_failure_jump that points
+           to the beginning of the next alternative.  Each alternative
+           except the last ends with a jump that in effect jumps past
+           the rest of the alternatives.  (They really jump to the
+           ending jump of the following alternative, because tensioning
+           these jumps is a hassle.)
+
+           Repeats start with an on_failure_jump that points past both
+           the repetition text and either the following jump or
+           pop_failure_jump back to this on_failure_jump.  */
+	case on_failure_jump:
+        on_failure:
+          DEBUG_PRINT1 ("EXECUTING on_failure_jump");
+
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x)", mcnt, p + mcnt);
+
+          /* If this on_failure_jump comes right before a group (i.e.,
+             the original * applied to a group), save the information
+             for that group and all inner ones, so that if we fail back
+             to this point, the group's information will be correct.
+             For example, in \(a*\)*\1, we need the preceding group,
+             and in \(\(a*\)b*\)\2, we need the inner group.  */
+
+          /* We can't use `p' to check ahead because we push
+             a failure point to `p + mcnt' after we do this.  */
+          p1 = p;
+
+          /* We need to skip no_op's before we look for the
+             start_memory in case this on_failure_jump is happening as
+             the result of a completed succeed_n, as in \(a\)\{1,3\}b\1
+             against aba.  */
+          while (p1 < pend && (re_opcode_t) *p1 == no_op)
+            p1++;
+
+          if (p1 < pend && (re_opcode_t) *p1 == start_memory)
+            {
+              /* We have a new highest active register now.  This will
+                 get reset at the start_memory we are about to get to,
+                 but we will have saved all the registers relevant to
+                 this repetition op, as described above.  */
+              highest_active_reg = *(p1 + 1) + *(p1 + 2);
+              if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+                lowest_active_reg = *(p1 + 1);
+            }
+
+          DEBUG_PRINT1 (":\n");
+          PUSH_FAILURE_POINT (p + mcnt, d, -2);
+          break;
+
+
+        /* A smart repeat ends with `maybe_pop_jump'.
+	   We change it to either `pop_failure_jump' or `jump'.  */
+        case maybe_pop_jump:
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT2 ("EXECUTING maybe_pop_jump %d.\n", mcnt);
+          {
+	    register unsigned char *p2 = p;
+
+            /* Compare the beginning of the repeat with what in the
+               pattern follows its end. If we can establish that there
+               is nothing that they would both match, i.e., that we
+               would have to backtrack because of (as in, e.g., `a*a')
+               then we can change to pop_failure_jump, because we'll
+               never have to backtrack.
+               
+               This is not true in the case of alternatives: in
+               `(a|ab)*' we do need to backtrack to the `ab' alternative
+               (e.g., if the string was `ab').  But instead of trying to
+               detect that here, the alternative has put on a dummy
+               failure point which is what we will end up popping.  */
+
+	    /* Skip over open/close-group commands.
+	       If what follows this loop is a ...+ construct,
+	       look at what begins its body, since we will have to
+	       match at least one of that.  */
+	    while (1)
+	      {
+		if (p2 + 2 < pend
+		    && ((re_opcode_t) *p2 == stop_memory
+			|| (re_opcode_t) *p2 == start_memory))
+		  p2 += 3;
+		else if (p2 + 6 < pend
+			 && (re_opcode_t) *p2 == dummy_failure_jump)
+		  p2 += 6;
+		else
+		  break;
+	      }
+
+	    p1 = p + mcnt;
+	    /* p1[0] ... p1[2] are the `on_failure_jump' corresponding
+	       to the `maybe_finalize_jump' of this case.  Examine what 
+	       follows.  */
+
+            /* If we're at the end of the pattern, we can change.  */
+            if (p2 == pend)
+	      {
+		/* Consider what happens when matching ":\(.*\)"
+		   against ":/".  I don't really understand this code
+		   yet.  */
+  	        p[-3] = (unsigned char) pop_failure_jump;
+                DEBUG_PRINT1
+                  ("  End of pattern: change to `pop_failure_jump'.\n");
+              }
+
+            else if ((re_opcode_t) *p2 == exactn
+		     || (bufp->newline_anchor && (re_opcode_t) *p2 == endline))
+	      {
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+
+                if ((re_opcode_t) p1[3] == exactn && p1[5] != c)
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset
+			 || (re_opcode_t) p1[3] == charset_not)
+		  {
+		    int not = (re_opcode_t) p1[3] == charset_not;
+                    
+		    if (c < (unsigned char) (p1[4] * BYTEWIDTH)
+			&& p1[5 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+		      not = !not;
+
+                    /* `not' is equal to 1 if c would match, which means
+                        that we can't change to pop_failure_jump.  */
+		    if (!not)
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+            else if ((re_opcode_t) *p2 == charset)
+	      {
+#ifdef DEBUG
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+#endif
+
+                if ((re_opcode_t) p1[3] == exactn
+		    && ! ((int) p2[1] * BYTEWIDTH > (int) p1[4]
+			  && (p2[1 + p1[4] / BYTEWIDTH]
+			      & (1 << (p1[4] % BYTEWIDTH)))))
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset_not)
+		  {
+		    int idx;
+		    /* We win if the charset_not inside the loop
+		       lists every character listed in the charset after.  */
+		    for (idx = 0; idx < (int) p2[1]; idx++)
+		      if (! (p2[2 + idx] == 0
+			     || (idx < (int) p1[4]
+				 && ((p2[2 + idx] & ~ p1[5 + idx]) == 0))))
+			break;
+
+		    if (idx == p2[1])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+		else if ((re_opcode_t) p1[3] == charset)
+		  {
+		    int idx;
+		    /* We win if the charset inside the loop
+		       has no overlap with the one after the loop.  */
+		    for (idx = 0;
+			 idx < (int) p2[1] && idx < (int) p1[4];
+			 idx++)
+		      if ((p2[2 + idx] & p1[5 + idx]) != 0)
+			break;
+
+		    if (idx == p2[1] || idx == p1[4])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+	  }
+	  p -= 2;		/* Point at relative address again.  */
+	  if ((re_opcode_t) p[-1] != pop_failure_jump)
+	    {
+	      p[-1] = (unsigned char) jump;
+              DEBUG_PRINT1 ("  Match => jump.\n");
+	      goto unconditional_jump;
+	    }
+        /* Note fall through.  */
+
+
+	/* The end of a simple repeat has a pop_failure_jump back to
+           its matching on_failure_jump, where the latter will push a
+           failure point.  The pop_failure_jump takes off failure
+           points put on by this pop_failure_jump's matching
+           on_failure_jump; we got through the pattern to here from the
+           matching on_failure_jump, so didn't fail.  */
+        case pop_failure_jump:
+          {
+            /* We need to pass separate storage for the lowest and
+               highest registers, even though we don't care about the
+               actual values.  Otherwise, we will restore only one
+               register from the stack, since lowest will == highest in
+               `pop_failure_point'.  */
+            unsigned dummy_low_reg, dummy_high_reg;
+            unsigned char *pdummy;
+            const char *sdummy;
+
+            DEBUG_PRINT1 ("EXECUTING pop_failure_jump.\n");
+            POP_FAILURE_POINT (sdummy, pdummy,
+                               dummy_low_reg, dummy_high_reg,
+                               reg_dummy, reg_dummy, reg_info_dummy);
+          }
+          /* Note fall through.  */
+
+          
+        /* Unconditionally jump (without popping any failure points).  */
+        case jump:
+	unconditional_jump:
+	  EXTRACT_NUMBER_AND_INCR (mcnt, p);	/* Get the amount to jump.  */
+          DEBUG_PRINT2 ("EXECUTING jump %d ", mcnt);
+	  p += mcnt;				/* Do the jump.  */
+          DEBUG_PRINT2 ("(to 0x%x).\n", p);
+	  break;
+
+	
+        /* We need this opcode so we can detect where alternatives end
+           in `group_match_null_string_p' et al.  */
+        case jump_past_alt:
+          DEBUG_PRINT1 ("EXECUTING jump_past_alt.\n");
+          goto unconditional_jump;
+
+
+        /* Normally, the on_failure_jump pushes a failure point, which
+           then gets popped at pop_failure_jump.  We will end up at
+           pop_failure_jump, also, and with a pattern of, say, `a+', we
+           are skipping over the on_failure_jump, so we have to push
+           something meaningless for pop_failure_jump to pop.  */
+        case dummy_failure_jump:
+          DEBUG_PRINT1 ("EXECUTING dummy_failure_jump.\n");
+          /* It doesn't matter what we push for the string here.  What
+             the code at `fail' tests is the value for the pattern.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          goto unconditional_jump;
+
+
+        /* At the end of an alternative, we need to push a dummy failure
+           point in case we are followed by a `pop_failure_jump', because
+           we don't want the failure point for the alternative to be
+           popped.  For example, matching `(a|ab)*' against `aab'
+           requires that we match the `ab' alternative.  */
+        case push_dummy_failure:
+          DEBUG_PRINT1 ("EXECUTING push_dummy_failure.\n");
+          /* See comments just above at `dummy_failure_jump' about the
+             two zeroes.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          break;
+
+        /* Have to succeed matching what follows at least n times.
+           After that, handle like `on_failure_jump'.  */
+        case succeed_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING succeed_n %d.\n", mcnt);
+
+          assert (mcnt >= 0);
+          /* Originally, this is how many times we HAVE to succeed.  */
+          if (mcnt > 0)
+            {
+               mcnt--;
+	       p += 2;
+               STORE_NUMBER_AND_INCR (p, mcnt);
+               DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p, mcnt);
+            }
+	  else if (mcnt == 0)
+            {
+              DEBUG_PRINT2 ("  Setting two bytes from 0x%x to no_op.\n", p+2);
+	      p[2] = (unsigned char) no_op;
+              p[3] = (unsigned char) no_op;
+              goto on_failure;
+            }
+          break;
+        
+        case jump_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING jump_n %d.\n", mcnt);
+
+          /* Originally, this is how many times we CAN jump.  */
+          if (mcnt)
+            {
+               mcnt--;
+               STORE_NUMBER (p + 2, mcnt);
+	       goto unconditional_jump;	     
+            }
+          /* If don't have to jump any more, skip over the rest of command.  */
+	  else      
+	    p += 4;		     
+          break;
+        
+	case set_number_at:
+	  {
+            DEBUG_PRINT1 ("EXECUTING set_number_at.\n");
+
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            p1 = p + mcnt;
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p1, mcnt);
+	    STORE_NUMBER (p1, mcnt);
+            break;
+          }
+
+        case wordbound:
+          DEBUG_PRINT1 ("EXECUTING wordbound.\n");
+          if (AT_WORD_BOUNDARY (d))
+	    break;
+          goto fail;
+
+	case notwordbound:
+          DEBUG_PRINT1 ("EXECUTING notwordbound.\n");
+	  if (AT_WORD_BOUNDARY (d))
+	    goto fail;
+          break;
+
+	case wordbeg:
+          DEBUG_PRINT1 ("EXECUTING wordbeg.\n");
+	  if (WORDCHAR_P (d) && (AT_STRINGS_BEG (d) || !WORDCHAR_P (d - 1)))
+	    break;
+          goto fail;
+
+	case wordend:
+          DEBUG_PRINT1 ("EXECUTING wordend.\n");
+	  if (!AT_STRINGS_BEG (d) && WORDCHAR_P (d - 1)
+              && (!WORDCHAR_P (d) || AT_STRINGS_END (d)))
+	    break;
+          goto fail;
+
+#ifdef emacs
+  	case before_dot:
+          DEBUG_PRINT1 ("EXECUTING before_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) >= point)
+  	    goto fail;
+  	  break;
+  
+  	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) != point)
+  	    goto fail;
+  	  break;
+  
+  	case after_dot:
+          DEBUG_PRINT1 ("EXECUTING after_dot.\n");
+          if (PTR_CHAR_POS ((unsigned char *) d) <= point)
+  	    goto fail;
+  	  break;
+#if 0 /* not emacs19 */
+	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+	  if (PTR_CHAR_POS ((unsigned char *) d) + 1 != point)
+	    goto fail;
+	  break;
+#endif /* not emacs19 */
+
+	case syntaxspec:
+          DEBUG_PRINT2 ("EXECUTING syntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchsyntax;
+
+        case wordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs wordchar.\n");
+	  mcnt = (int) Sword;
+        matchsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) != (enum syntaxcode) mcnt)
+	    goto fail;
+          SET_REGS_MATCHED ();
+	  break;
+
+	case notsyntaxspec:
+          DEBUG_PRINT2 ("EXECUTING notsyntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchnotsyntax;
+
+        case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs notwordchar.\n");
+	  mcnt = (int) Sword;
+        matchnotsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) == (enum syntaxcode) mcnt)
+	    goto fail;
+	  SET_REGS_MATCHED ();
+          break;
+
+#else /* not emacs */
+	case wordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs wordchar.\n");
+	  PREFETCH ();
+          if (!WORDCHAR_P (d))
+            goto fail;
+	  SET_REGS_MATCHED ();
+          d++;
+	  break;
+	  
+	case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs notwordchar.\n");
+	  PREFETCH ();
+	  if (WORDCHAR_P (d))
+            goto fail;
+          SET_REGS_MATCHED ();
+          d++;
+	  break;
+#endif /* not emacs */
+          
+        default:
+          abort ();
+	}
+      continue;  /* Successfully executed one pattern command; keep going.  */
+
+
+    /* We goto here if a matching operation fails. */
+    fail:
+      if (!FAIL_STACK_EMPTY ())
+	{ /* A restart point is known.  Restore to that state.  */
+          DEBUG_PRINT1 ("\nFAIL:\n");
+          POP_FAILURE_POINT (d, p,
+                             lowest_active_reg, highest_active_reg,
+                             regstart, regend, reg_info);
+
+          /* If this failure point is a dummy, try the next one.  */
+          if (!p)
+	    goto fail;
+
+          /* If we failed to the end of the pattern, don't examine *p.  */
+	  assert (p <= pend);
+          if (p < pend)
+            {
+              boolean is_a_jump_n = false;
+              
+              /* If failed to a backwards jump that's part of a repetition
+                 loop, need to pop this failure point and use the next one.  */
+              switch ((re_opcode_t) *p)
+                {
+                case jump_n:
+                  is_a_jump_n = true;
+                case maybe_pop_jump:
+                case pop_failure_jump:
+                case jump:
+                  p1 = p + 1;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  p1 += mcnt;	
+
+                  if ((is_a_jump_n && (re_opcode_t) *p1 == succeed_n)
+                      || (!is_a_jump_n
+                          && (re_opcode_t) *p1 == on_failure_jump))
+                    goto fail;
+                  break;
+                default:
+                  /* do nothing */ ;
+                }
+            }
+
+          if (d >= string1 && d <= end1)
+	    dend = end_match_1;
+        }
+      else
+        break;   /* Matching at this starting point really fails.  */
+    } /* for (;;) */
+
+  if (best_regs_set)
+    goto restore_best_regs;
+
+  FREE_VARIABLES ();
+
+  return -1;         			/* Failure to match.  */
+} /* re_match_2 */
+
+
+/* Subroutine definitions for re_match_2.  */
+
+
+/* We are passed P pointing to a register number after a start_memory.
+   
+   Return true if the pattern up to the corresponding stop_memory can
+   match the empty string, and false otherwise.
+   
+   If we find the matching stop_memory, sets P to point to one past its number.
+   Otherwise, sets P to an undefined byte less than or equal to END.
+
+   We don't handle duplicates properly (yet).  */
+
+static boolean
+group_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  /* Point to after the args to the start_memory.  */
+  unsigned char *p1 = *p + 2;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and return true or
+	 false, as appropriate, when we get to one that can't, or to the
+         matching stop_memory.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+        /* Could be either a loop or a series of alternatives.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          
+          /* If the next operation is not a jump backwards in the
+	     pattern.  */
+
+	  if (mcnt >= 0)
+	    {
+              /* Go through the on_failure_jumps of the alternatives,
+                 seeing if any of the alternatives cannot match nothing.
+                 The last alternative starts with only a jump,
+                 whereas the rest start with on_failure_jump and end
+                 with a jump, e.g., here is the pattern for `a|b|c':
+
+                 /on_failure_jump/0/6/exactn/1/a/jump_past_alt/0/6
+                 /on_failure_jump/0/6/exactn/1/b/jump_past_alt/0/3
+                 /exactn/1/c						
+
+                 So, we have to first go through the first (n-1)
+                 alternatives and then deal with the last one separately.  */
+
+
+              /* Deal with the first (n-1) alternatives, which start
+                 with an on_failure_jump (see above) that jumps to right
+                 past a jump_past_alt.  */
+
+              while ((re_opcode_t) p1[mcnt-3] == jump_past_alt)
+                {
+                  /* `mcnt' holds how many bytes long the alternative
+                     is, including the ending `jump_past_alt' and
+                     its number.  */
+
+                  if (!alt_match_null_string_p (p1, p1 + mcnt - 3, 
+				                      reg_info))
+                    return false;
+
+                  /* Move to right after this alternative, including the
+		     jump_past_alt.  */
+                  p1 += mcnt;	
+
+                  /* Break if it's the beginning of an n-th alternative
+                     that doesn't begin with an on_failure_jump.  */
+                  if ((re_opcode_t) *p1 != on_failure_jump)
+                    break;
+		
+		  /* Still have to check that it's not an n-th
+		     alternative that starts with an on_failure_jump.  */
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  if ((re_opcode_t) p1[mcnt-3] != jump_past_alt)
+                    {
+		      /* Get to the beginning of the n-th alternative.  */
+                      p1 -= 3;
+                      break;
+                    }
+                }
+
+              /* Deal with the last alternative: go back and get number
+                 of the `jump_past_alt' just before it.  `mcnt' contains
+                 the length of the alternative.  */
+              EXTRACT_NUMBER (mcnt, p1 - 2);
+
+              if (!alt_match_null_string_p (p1, p1 + mcnt, reg_info))
+                return false;
+
+              p1 += mcnt;	/* Get past the n-th alternative.  */
+            } /* if mcnt > 0 */
+          break;
+
+          
+        case stop_memory:
+	  assert (p1[1] == **p);
+          *p = p1 + 2;
+          return true;
+
+        
+        default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    } /* while p1 < end */
+
+  return false;
+} /* group_match_null_string_p */
+
+
+/* Similar to group_match_null_string_p, but doesn't deal with alternatives:
+   It expects P to be the first byte of a single alternative and END one
+   byte past the last. The alternative can contain groups.  */
+   
+static boolean
+alt_match_null_string_p (p, end, reg_info)
+    unsigned char *p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  unsigned char *p1 = p;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and break when we get 
+         to one that can't.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+	/* It's a loop.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+          break;
+          
+	default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    }  /* while p1 < end */
+
+  return true;
+} /* alt_match_null_string_p */
+
+
+/* Deals with the ops common to group_match_null_string_p and
+   alt_match_null_string_p.  
+   
+   Sets P to one after the op and its arguments, if any.  */
+
+static boolean
+common_op_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  boolean ret;
+  int reg_no;
+  unsigned char *p1 = *p;
+
+  switch ((re_opcode_t) *p1++)
+    {
+    case no_op:
+    case begline:
+    case endline:
+    case begbuf:
+    case endbuf:
+    case wordbeg:
+    case wordend:
+    case wordbound:
+    case notwordbound:
+#ifdef emacs
+    case before_dot:
+    case at_dot:
+    case after_dot:
+#endif
+      break;
+
+    case start_memory:
+      reg_no = *p1;
+      assert (reg_no > 0 && reg_no <= MAX_REGNUM);
+      ret = group_match_null_string_p (&p1, end, reg_info);
+      
+      /* Have to set this here in case we're checking a group which
+         contains a group and a back reference to it.  */
+
+      if (REG_MATCH_NULL_STRING_P (reg_info[reg_no]) == MATCH_NULL_UNSET_VALUE)
+        REG_MATCH_NULL_STRING_P (reg_info[reg_no]) = ret;
+
+      if (!ret)
+        return false;
+      break;
+          
+    /* If this is an optimized succeed_n for zero times, make the jump.  */
+    case jump:
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+      if (mcnt >= 0)
+        p1 += mcnt;
+      else
+        return false;
+      break;
+
+    case succeed_n:
+      /* Get to the number of times to succeed.  */
+      p1 += 2;		
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+
+      if (mcnt == 0)
+        {
+          p1 -= 4;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+        }
+      else
+        return false;
+      break;
+
+    case duplicate: 
+      if (!REG_MATCH_NULL_STRING_P (reg_info[*p1]))
+        return false;
+      break;
+
+    case set_number_at:
+      p1 += 4;
+
+    default:
+      /* All other opcodes mean we cannot match the empty string.  */
+      return false;
+  }
+
+  *p = p1;
+  return true;
+} /* common_op_match_null_string_p */
+
+
+/* Return zero if TRANSLATE[S1] and TRANSLATE[S2] are identical for LEN
+   bytes; nonzero otherwise.  */
+   
+static int
+bcmp_translate (s1, s2, len, translate)
+     unsigned char *s1, *s2;
+     register int len;
+     char *translate;
+{
+  register unsigned char *p1 = s1, *p2 = s2;
+  while (len)
+    {
+      if (translate[*p1++] != translate[*p2++]) return 1;
+      len--;
+    }
+  return 0;
+}
+
+
+/* Entry points for GNU code.  */
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in BUFP.
+   Returns 0 if the pattern was valid, otherwise an error string.
+   
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in BUFP on entry.
+   
+   We call regex_compile to do the actual compilation.  */
+
+const char *
+re_compile_pattern (pattern, length, bufp)
+     const char *pattern;
+     int length;
+     struct re_pattern_buffer *bufp;
+{
+  reg_errcode_t ret;
+  
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  bufp->regs_allocated = REGS_UNALLOCATED;
+  
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  bufp->no_sub = 0;
+  
+  /* Match anchors at newline.  */
+  bufp->newline_anchor = 1;
+  
+  ret = regex_compile (pattern, length, re_syntax_options, bufp);
+
+  if (!ret)
+    return NULL;
+  return gettext (re_error_msgid[(int) ret]);
+}     
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them unless specifically requested.  */
+
+#ifdef _REGEX_RE_COMP
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer re_comp_buf;
+
+char *
+re_comp (s)
+    const char *s;
+{
+  reg_errcode_t ret;
+  
+  if (!s)
+    {
+      if (!re_comp_buf.buffer)
+	return gettext ("No previous regular expression");
+      return 0;
+    }
+
+  if (!re_comp_buf.buffer)
+    {
+      re_comp_buf.buffer = (unsigned char *) malloc (200);
+      if (re_comp_buf.buffer == NULL)
+        return gettext (re_error_msgid[(int) REG_ESPACE]);
+      re_comp_buf.allocated = 200;
+
+      re_comp_buf.fastmap = (char *) malloc (1 << BYTEWIDTH);
+      if (re_comp_buf.fastmap == NULL)
+	return gettext (re_error_msgid[(int) REG_ESPACE]);
+    }
+
+  /* Since `re_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  re_comp_buf.newline_anchor = 1;
+
+  ret = regex_compile (s, strlen (s), re_syntax_options, &re_comp_buf);
+  
+  if (!ret)
+    return NULL;
+
+  /* Yes, we're discarding `const' here if !HAVE_LIBINTL.  */
+  return (char *) gettext (re_error_msgid[(int) ret]);
+}
+
+
+int
+re_exec (s)
+    const char *s;
+{
+  const int len = strlen (s);
+  return
+    0 <= re_search (&re_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* _REGEX_RE_COMP */
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#ifndef emacs
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+int
+regcomp (preg, pattern, cflags)
+    regex_t *preg;
+    const char *pattern; 
+    int cflags;
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = (cflags & REG_EXTENDED) ?
+      RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->used = 0;
+  
+  /* Don't bother to use a fastmap when searching.  This simplifies the
+     REG_NEWLINE case: if we used a fastmap, we'd have to put all the
+     characters after newlines into the fastmap.  This way, we just try
+     every character.  */
+  preg->fastmap = 0;
+  
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+      
+      preg->translate = (char *) malloc (CHAR_SET_SIZE);
+      if (preg->translate == NULL)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = ISUPPER (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = NULL;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we 
+     can use strlen here in compiling the pattern.  */
+  ret = regex_compile (pattern, strlen (pattern), syntax, preg);
+  
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+  
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+   
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+   
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+   
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    const regex_t *preg;
+    const char *string; 
+    size_t nmatch; 
+    regmatch_t pmatch[]; 
+    int eflags;
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+  
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+  
+  /* The user has told us exactly how many registers to return
+     information about, via `nmatch'.  We have to pass that on to the
+     matching routines.  */
+  private_preg.regs_allocated = REGS_FIXED;
+  
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start = TALLOC (nmatch, regoff_t);
+      regs.end = TALLOC (nmatch, regoff_t);
+      if (regs.start == NULL || regs.end == NULL)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg, string, len,
+                   /* start: */ 0, /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+  
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   We don't use PREG here.  */
+
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    const regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+{
+  const char *msg;
+  size_t msg_size;
+
+  if (errcode < 0
+      || errcode >= (sizeof (re_error_msgid) / sizeof (re_error_msgid[0])))
+    /* Only error codes returned by the rest of the code should be passed 
+       to this routine.  If we are given anything else, or if other regex
+       code generates an invalid error code, then the program has a bug.
+       Dump core so we can fix it.  */
+    abort ();
+
+  msg = gettext (re_error_msgid[errcode]);
+
+  msg_size = strlen (msg) + 1; /* Includes the null.  */
+  
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+void
+regfree (preg)
+    regex_t *preg;
+{
+  if (preg->buffer != NULL)
+    free (preg->buffer);
+  preg->buffer = NULL;
+  
+  preg->allocated = 0;
+  preg->used = 0;
+
+  if (preg->fastmap != NULL)
+    free (preg->fastmap);
+  preg->fastmap = NULL;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != NULL)
+    free (preg->translate);
+  preg->translate = NULL;
+}
+
+#endif /* not emacs  */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/gsdl/packages/mg/lib/regex.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/regex.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/regex.h	(revision 21112)
@@ -0,0 +1,497 @@
+/* Definitions for data structures and routines for the regular
+   expression library, version 0.12.
+
+   Copyright (C) 1985, 89, 90, 91, 92, 93, 95 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef __REGEXP_LIBRARY_H__
+#define __REGEXP_LIBRARY_H__
+
+/* POSIX says that <sys/types.h> must be included (by the caller) before
+   <regex.h>.  */
+
+#if !defined (_POSIX_C_SOURCE) && !defined (_POSIX_SOURCE) && defined (VMS)
+/* VMS doesn't have `size_t' in <sys/types.h>, even though POSIX says it
+   should be there.  */
+#include <stddef.h>
+#endif
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+typedef unsigned reg_syntax_t;
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* If this bit is set, succeed as soon as we match the whole pattern,
+   without further backtracking.  */
+#define RE_NO_POSIX_BACKTRACKING (RE_UNMATCHED_RIGHT_PAREN_ORD << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+/* P1003.2/D11.2, section 4.20.7.1, lines 5078ff.  */
+#define RE_SYNTAX_ED RE_SYNTAX_POSIX_BASIC
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+
+/* This data structure represents a compiled pattern.  Before calling
+   the pattern compiler, the fields `buffer', `allocated', `fastmap',
+   `translate', and `no_sub' can be set.  After the pattern has been
+   compiled, the `re_nsub' field is available.  All other fields are
+   private to the regex routines.  */
+
+struct re_pattern_buffer
+{
+/* [[[begin pattern_buffer]]] */
+	/* Space that holds the compiled pattern.  It is declared as
+          `unsigned char *' because its elements are
+           sometimes used as array indexes.  */
+  unsigned char *buffer;
+
+	/* Number of bytes to which `buffer' points.  */
+  unsigned long allocated;
+
+	/* Number of bytes actually used in `buffer'.  */
+  unsigned long used;	
+
+        /* Syntax setting with which the pattern was compiled.  */
+  reg_syntax_t syntax;
+
+        /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+           the fastmap, if there is one, to skip over impossible
+           starting points for matches.  */
+  char *fastmap;
+
+        /* Either a translate table to apply to all characters before
+           comparing them, or zero for no translation.  The translation
+           is applied to a pattern when it is compiled and to a string
+           when it is matched.  */
+  char *translate;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+        /* Zero if this pattern cannot match the empty string, one else.
+           Well, in truth it's used only in `re_search_2', to see
+           whether or not we should use the fastmap, so we don't set
+           this absolutely perfectly; see `re_compile_fastmap' (the
+           `duplicate' case).  */
+  unsigned can_be_null : 1;
+
+        /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+             for `max (RE_NREGS, re_nsub + 1)' groups.
+           If REGS_REALLOCATE, reallocate space if necessary.
+           If REGS_FIXED, use what's there.  */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned regs_allocated : 2;
+
+        /* Set to zero when `regex_compile' compiles a pattern; set to one
+           by `re_compile_fastmap' if it updates the fastmap.  */
+  unsigned fastmap_accurate : 1;
+
+        /* If set, `re_match_2' does not return information about
+           subexpressions.  */
+  unsigned no_sub : 1;
+
+        /* If set, a beginning-of-line anchor doesn't match at the
+           beginning of the string.  */ 
+  unsigned not_bol : 1;
+
+        /* Similarly for an end-of-line anchor.  */
+  unsigned not_eol : 1;
+
+        /* If true, an anchor at a newline matches.  */
+  unsigned newline_anchor : 1;
+
+/* [[[end pattern_buffer]]] */
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+   `re_match_2' returns information about at least this many registers
+   the first time a `regs' structure is passed.  */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+/* Declarations for routines.  */
+
+/* To avoid duplicating every routine declaration -- once with a
+   prototype (if we are ANSI), and once without (if we aren't) -- we
+   use the following macro to declare argument types.  This
+   unfortunately clutters up the declarations a bit, but I think it's
+   worth it.  */
+
+#if __STDC__
+
+#define _RE_ARGS(args) args
+
+#else /* not __STDC__ */
+
+#define _RE_ARGS(args) ()
+
+#endif /* not __STDC__ */
+
+/* Sets the current default syntax to SYNTAX, and return the old syntax.
+   You can also simply assign to the `re_syntax_options' variable.  */
+extern reg_syntax_t re_set_syntax _RE_ARGS ((reg_syntax_t syntax));
+
+/* Compile the regular expression PATTERN, with length LENGTH
+   and syntax given by the global `re_syntax_options', into the buffer
+   BUFFER.  Return NULL if successful, and an error string if not.  */
+extern const char *re_compile_pattern
+  _RE_ARGS ((const char *pattern, int length,
+             struct re_pattern_buffer *buffer));
+
+
+/* Compile a fastmap for the compiled pattern in BUFFER; used to
+   accelerate searches.  Return 0 if successful and -2 if was an
+   internal error.  */
+extern int re_compile_fastmap _RE_ARGS ((struct re_pattern_buffer *buffer));
+
+
+/* Search in the string STRING (with length LENGTH) for the pattern
+   compiled into BUFFER.  Start searching at position START, for RANGE
+   characters.  Return the starting position of the match, -1 for no
+   match, or -2 for an internal error.  Also return register
+   information in REGS (if REGS and BUFFER->no_sub are nonzero).  */
+extern int re_search
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+            int length, int start, int range, struct re_registers *regs));
+
+
+/* Like `re_search', but search in the concatenation of STRING1 and
+   STRING2.  Also, stop searching at index START + STOP.  */
+extern int re_search_2
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, int range, struct re_registers *regs, int stop));
+
+
+/* Like `re_search', but return how many characters in STRING the regexp
+   in BUFFER matched, starting at position START.  */
+extern int re_match
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+             int length, int start, struct re_registers *regs));
+
+
+/* Relates to `re_match' as `re_search_2' relates to `re_search'.  */
+extern int re_match_2 
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, struct re_registers *regs, int stop));
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using BUFFER and REGS will use this memory
+   for recording register information.  STARTS and ENDS must be
+   allocated with malloc, and must each be at least `NUM_REGS * sizeof
+   (regoff_t)' bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+extern void re_set_registers
+  _RE_ARGS ((struct re_pattern_buffer *buffer, struct re_registers *regs,
+             unsigned num_regs, regoff_t *starts, regoff_t *ends));
+
+#ifdef _REGEX_RE_COMP
+/* 4.2 bsd compatibility.  */
+extern char *re_comp _RE_ARGS ((const char *));
+extern int re_exec _RE_ARGS ((const char *));
+#endif
+
+/* POSIX compatibility.  */
+extern int regcomp _RE_ARGS ((regex_t *preg, const char *pattern, int cflags));
+extern int regexec
+  _RE_ARGS ((const regex_t *preg, const char *string, size_t nmatch,
+             regmatch_t pmatch[], int eflags));
+extern size_t regerror
+  _RE_ARGS ((int errcode, const regex_t *preg, char *errbuf,
+             size_t errbuf_size));
+extern void regfree _RE_ARGS ((regex_t *preg));
+
+#endif /* not __REGEXP_LIBRARY_H__ */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/gsdl/packages/mg/lib/rx.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/rx.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/rx.c	(revision 21112)
@@ -0,0 +1,7271 @@
+/*	Copyright (C) 1992, 1993, 1994, 1995 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+
+/* NOTE!!!  AIX is so losing it requires this to be the first thing in the 
+ * file. 
+ * Do not put ANYTHING before it!  
+ */
+#if !defined (__GNUC__) && defined (_AIX)
+ #pragma alloca
+#endif
+
+/* To make linux happy? */
+#ifndef	_GNU_SOURCE
+#define	_GNU_SOURCE
+#endif
+
+#if HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+#  include <malloc.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+
+
+char rx_version_string[] = "GNU Rx version 0.07.1";
+
+			/* ``Too hard!''
+			 *	    -- anon.
+			 */
+
+
+
+#include <stdio.h>
+#include <ctype.h>
+#ifndef isgraph
+#define isgraph(c) (isprint (c) && !isspace (c))
+#endif
+#ifndef isblank
+#define isblank(c) ((c) == ' ' || (c) == '\t')
+#endif
+
+#include <sys/types.h>
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+#ifndef __GCC__
+#undef __inline__
+#define __inline__
+#endif
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+/* Memory management and stuff for emacs. */
+
+#define CHARBITS 8
+#define remalloc(M, S) (M ? realloc (M, S) : malloc (S))
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+ * use `alloca' instead of `malloc' for the backtracking stack.
+ *
+ * Emacs will die miserably if we don't do this.
+ */
+
+#ifdef REGEX_MALLOC
+#define REGEX_ALLOCATE malloc
+#else /* not REGEX_MALLOC  */
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#define REGEX_ALLOCATE _alloca
+#else
+#define REGEX_ALLOCATE alloca
+#endif /* __WIN32__ */
+#endif /* not REGEX_MALLOC */
+
+
+#ifdef RX_WANT_RX_DEFS
+#define RX_DECL extern
+#define RX_DEF_QUAL 
+#else
+#define RX_WANT_RX_DEFS
+#define RX_DECL static
+#define RX_DEF_QUAL static
+#endif
+#include "rx.h"
+#undef RX_DECL
+#define RX_DECL RX_DEF_QUAL
+
+
+#ifndef emacs
+
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else /* not SYNTAX_TABLE */
+
+#ifdef __STDC__
+static void
+init_syntax_once (void)
+#else
+static void
+init_syntax_once ()
+#endif
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+#endif /* not SYNTAX_TABLE */
+#endif /* not emacs */
+
+
+/* Compile with `-DRX_DEBUG' and use the following flags.
+ *
+ * Debugging flags:
+ *   	rx_debug - print information as a regexp is compiled
+ * 	rx_debug_trace - print information as a regexp is executed
+ */
+
+#ifdef RX_DEBUG
+
+int rx_debug_compile = 0;
+int rx_debug_trace = 0;
+static struct re_pattern_buffer * dbug_rxb = 0;
+
+#ifdef __STDC__
+typedef void (*side_effect_printer) (struct rx *, void *, FILE *);
+#else
+typedef void (*side_effect_printer) ();
+#endif
+
+#ifdef __STDC__
+static void print_cset (struct rx *rx, rx_Bitset cset, FILE * fp);
+#else
+static void print_cset ();
+#endif
+
+#ifdef __STDC__
+static void
+print_rexp (struct rx *rx,
+	    struct rexp_node *node, int depth,
+	    side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_rexp (rx, node, depth, seprint, fp)
+     struct rx *rx;
+     struct rexp_node *node;
+     int depth;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  if (!node)
+    return;
+  else
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  {
+	    fprintf (fp, "%*s", depth, "");
+	    print_cset (rx, node->params.cset, fp);
+	    fputc ('\n', fp);
+	    break;
+	  }
+
+ 	case r_opt:
+	case r_star:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_opt ? "opt" : "star");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+	case r_2phase_star:
+	  fprintf (fp, "%*s2phase star\n", depth, "");
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+
+	case r_alternate:
+	case r_concat:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_alternate ? "alt" : "concat");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  break;
+	case r_side_effect:
+	  fprintf (fp, "%*sSide effect: ", depth, "");
+	  seprint (rx, node->params.side_effect, fp);
+	  fputc ('\n', fp);
+	}
+    }
+}
+
+#ifdef __STDC__
+static void
+print_nfa (struct rx * rx,
+	   struct rx_nfa_state * n,
+	   side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_nfa (rx, n, seprint, fp)
+     struct rx * rx;
+     struct rx_nfa_state * n;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      fprintf (fp, "node %d %s\n", n->id,
+	       n->is_final ? "final" : (n->is_start ? "start" : ""));
+      while (e)
+	{
+	  fprintf (fp, "   edge to %d, ", e->dest->id);
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	      fprintf (fp, "epsilon\n");
+	      break;
+	    case ne_side_effect:
+	      fprintf (fp, "side effect ");
+	      seprint (rx, e->params.side_effect, fp);
+	      fputc ('\n', fp);
+	      break;
+	    case ne_cset:
+	      fprintf (fp, "cset ");
+	      print_cset (rx, e->params.cset, fp);
+	      fputc ('\n', fp);
+	      break;
+	    }
+	  e = e->next;
+	}
+
+      while (ec)
+	{
+	  int x;
+	  struct rx_nfa_state_set * s;
+	  struct rx_se_list * l;
+	  fprintf (fp, "   eclosure to {");
+	  for (s = ec->destset; s; s = s->cdr)
+	    fprintf (fp, "%d ", s->car->id);
+	  fprintf (fp, "} (");
+	  for (l = ec->effects; l; l = l->cdr)
+	    {
+	      seprint (rx, l->car, fp);
+	      fputc (' ', fp);
+	    }
+	  fprintf (fp, ")\n");
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+}
+
+static char * efnames [] =
+{
+  "bogon",
+  "re_se_try",
+  "re_se_pushback",
+  "re_se_push0",
+  "re_se_pushpos",
+  "re_se_chkpos",
+  "re_se_poppos",
+  "re_se_at_dot",
+  "re_se_syntax",
+  "re_se_not_syntax",
+  "re_se_begbuf",
+  "re_se_hat",
+  "re_se_wordbeg",
+  "re_se_wordbound",
+  "re_se_notwordbound",
+  "re_se_wordend",
+  "re_se_endbuf",
+  "re_se_dollar",
+  "re_se_fail",
+};
+
+static char * efnames2[] =
+{
+  "re_se_win",
+  "re_se_lparen",
+  "re_se_rparen",
+  "re_se_backref",
+  "re_se_iter",
+  "re_se_end_iter",
+  "re_se_tv"
+};
+
+static char * inx_names[] = 
+{
+  "rx_backtrack_point",
+  "rx_do_side_effects",
+  "rx_cache_miss",
+  "rx_next_char",
+  "rx_backtrack",
+  "rx_error_inx",
+  "rx_num_instructions"
+};
+
+
+#ifdef __STDC__
+static void
+re_seprint (struct rx * rx, void * effect, FILE * fp)
+#else
+static void
+re_seprint (rx, effect, fp)
+     struct rx * rx;
+     void * effect;
+     FILE * fp;
+#endif
+{
+  if ((int)effect < 0)
+    fputs (efnames[-(int)effect], fp);
+  else if (dbug_rxb)
+    {
+      struct re_se_params * p = &dbug_rxb->se_params[(int)effect];
+      fprintf (fp, "%s(%d,%d)", efnames2[p->se], p->op1, p->op2);
+    }
+  else
+    fprintf (fp, "[complex op # %d]", (int)effect);
+}
+
+
+/* These are so the regex.c regression tests will compile. */
+void
+print_compiled_pattern (rxb)
+     struct re_pattern_buffer * rxb;
+{
+}
+
+void
+print_fastmap (fm)
+     char * fm;
+{
+}
+
+#endif /* RX_DEBUG */
+
+
+
+
+/* This page: Bitsets.  Completely unintersting. */
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_equal (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  RX_subset s = b[0];
+  b[0] = ~a[0];
+
+  for (x = rx_bitset_numb_subsets(size) - 1; a[x] == b[x]; --x)
+    ;
+
+  b[0] = s;
+  return !x && s == a[0];
+}
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_subset (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets(size) - 1;
+  while (x-- && (a[x] & b[x]) == a[x]);
+  return x == -1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_empty (int size, rx_Bitset set)
+#else
+RX_DECL int
+rx_bitset_empty (size, set)
+     int size;
+     rx_Bitset set;
+#endif
+{
+  int x;
+  RX_subset s = set[0];
+  set[0] = 1;
+  for (x = rx_bitset_numb_subsets(size) - 1; !set[x]; --x)
+    ;
+  set[0] = s;
+  return !s;
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_null (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_null (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  bzero (b, rx_sizeof_bitset(size));
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_universe (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_universe (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    *b++ = ~(RX_subset)0;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_complement (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_complement (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    {
+      *b = ~*b;
+      ++b;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_assign (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_union (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_union (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] |= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_intersection (int size,
+			rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_intersection (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_difference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &=  ~ b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_revdifference (int size,
+			 rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_revdifference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = ~a[x] & b[x];
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_xor (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] ^= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL unsigned long
+rx_bitset_hash (int size, rx_Bitset b)
+#else
+RX_DECL unsigned long
+rx_bitset_hash (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  unsigned long hash = (unsigned long)rx_bitset_hash;
+
+  for (x = rx_bitset_numb_subsets(size) - 1; x >= 0; --x)
+    hash ^= rx_bitset_subset_val(b, x);
+
+  return hash;
+}
+
+
+RX_DECL RX_subset rx_subset_singletons [RX_subset_bits] = 
+{
+  0x1,
+  0x2,
+  0x4,
+  0x8,
+  0x10,
+  0x20,
+  0x40,
+  0x80,
+  0x100,
+  0x200,
+  0x400,
+  0x800,
+  0x1000,
+  0x2000,
+  0x4000,
+  0x8000,
+  0x10000,
+  0x20000,
+  0x40000,
+  0x80000,
+  0x100000,
+  0x200000,
+  0x400000,
+  0x800000,
+  0x1000000,
+  0x2000000,
+  0x4000000,
+  0x8000000,
+  0x10000000,
+  0x20000000,
+  0x40000000,
+  0x80000000
+};
+
+#ifdef RX_DEBUG
+
+#ifdef __STDC__
+static void
+print_cset (struct rx *rx, rx_Bitset cset, FILE * fp)
+#else
+static void
+print_cset (rx, cset, fp)
+     struct rx *rx;
+     rx_Bitset cset;
+     FILE * fp;
+#endif
+{
+  int x;
+  fputc ('[', fp);
+  for (x = 0; x < rx->local_cset_size; ++x)
+    if (RX_bitset_member (cset, x))
+      {
+	if (isprint(x))
+	  fputc (x, fp);
+	else
+	  fprintf (fp, "\\0%o ", x);
+      }
+  fputc (']', fp);
+}
+
+#endif /*  RX_DEBUG */
+
+
+
+
+static unsigned long rx_hash_masks[4] =
+{
+  0x12488421,
+  0x96699669,
+  0xbe7dd7eb,
+  0xffffffff
+};
+
+
+/* Hash tables */
+#ifdef __STDC__
+RX_DECL struct rx_hash_item * 
+rx_hash_find (struct rx_hash * table,
+	      unsigned long hash,
+	      void * value,
+	      struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item * 
+rx_hash_find (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks [0];
+  int bucket = (hash & mask) % 13;
+
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+    }
+
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+
+  return 0;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_hash_item *
+rx_hash_store (struct rx_hash * table,
+	       unsigned long hash,
+	       void * value,
+	       struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item *
+rx_hash_store (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks[0];
+  int bucket = (hash & mask) % 13;
+  int depth = 0;
+  
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+      ++depth;
+    }
+  
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+  
+  {
+    if (   (depth < 3)
+	&& (table->bucket_size [bucket] >= 4))
+      {
+	struct rx_hash * newtab = ((struct rx_hash *)
+				   rules->hash_alloc (rules));
+	if (!newtab)
+	  goto add_to_bucket;
+	bzero (newtab, sizeof (*newtab));
+	newtab->parent = table;
+	{
+	  struct rx_hash_item * them = table->buckets[bucket];
+	  unsigned long newmask = rx_hash_masks[maskc + 1];
+	  while (them)
+	    {
+	      struct rx_hash_item * save = them->next_same_hash;
+	      int new_buck = (them->hash & newmask) % 13;
+	      them->next_same_hash = newtab->buckets[new_buck];
+	      newtab->buckets[new_buck] = them;
+	      them->table = newtab;
+	      them = save;
+	      ++newtab->bucket_size[new_buck];
+	      ++newtab->refs;
+	    }
+	  table->refs = (table->refs - table->bucket_size[bucket] + 1);
+	  table->bucket_size[bucket] = 0;
+	  table->buckets[bucket] = 0;
+	  table->children[bucket] = newtab;
+	  table = newtab;
+	  bucket = (hash & newmask) % 13;
+	}
+      }
+  }
+ add_to_bucket:
+  {
+    struct rx_hash_item  * it = ((struct rx_hash_item *)
+				 rules->hash_item_alloc (rules, value));
+    if (!it)
+      return 0;
+    it->hash = hash;
+    it->table = table;
+    /* DATA and BINDING are to be set in hash_item_alloc */
+    it->next_same_hash = table->buckets [bucket];
+    table->buckets[bucket] = it;
+    ++table->bucket_size [bucket];
+    ++table->refs;
+    return it;
+  }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_hash_free (it, rules)
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+  if (it)
+    {
+      struct rx_hash * table = it->table;
+      unsigned long hash = it->hash;
+      int depth = (table->parent
+		   ? (table->parent->parent
+		      ? (table->parent->parent->parent
+			 ? 3
+			 : 2)
+		      : 1)
+		   : 0);
+      int bucket = (hash & rx_hash_masks [depth]) % 13;
+      struct rx_hash_item ** pos = &table->buckets [bucket];
+      
+      while (*pos != it)
+	pos = &(*pos)->next_same_hash;
+      *pos = it->next_same_hash;
+      rules->free_hash_item (it, rules);
+      --table->bucket_size[bucket];
+      --table->refs;
+      while (!table->refs && depth)
+	{
+	  struct rx_hash * save = table;
+	  table = table->parent;
+	  --depth;
+	  bucket = (hash & rx_hash_masks [depth]) % 13;
+	  --table->refs;
+	  table->children[bucket] = 0;
+	  rules->free_hash (save, rules);
+	}
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+		    struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_free_hash_table (tab, freefn, rules)
+     struct rx_hash * tab;
+     rx_hash_freefn freefn;
+     struct rx_hash_rules * rules;
+#endif
+{
+  int x;
+
+  for (x = 0; x < 13; ++x)
+    if (tab->children[x])
+      {
+	rx_free_hash_table (tab->children[x], freefn, rules);
+	rules->free_hash (tab->children[x], rules);
+      }
+    else
+      {
+	struct rx_hash_item * them = tab->buckets[x];
+	while (them)
+	  {
+	    struct rx_hash_item * that = them;
+	    them = that->next_same_hash;
+	    freefn (that);
+	    rules->free_hash_item (that, rules);
+	  }
+      }
+}
+
+
+
+
+/* Utilities for manipulating bitset represntations of characters sets. */
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_cset (struct rx *rx)
+#else
+RX_DECL rx_Bitset
+rx_cset (rx)
+     struct rx *rx;
+#endif
+{
+  rx_Bitset b = (rx_Bitset) malloc (rx_sizeof_bitset (rx->local_cset_size));
+  if (b)
+    rx_bitset_null (rx->local_cset_size, b);
+  return b;
+}
+
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_copy_cset (struct rx *rx, rx_Bitset a)
+#else
+RX_DECL rx_Bitset
+rx_copy_cset (rx, a)
+     struct rx *rx;
+     rx_Bitset a;
+#endif
+{
+  rx_Bitset cs = rx_cset (rx);
+
+  if (cs)
+    rx_bitset_union (rx->local_cset_size, cs, a);
+
+  return cs;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_cset (struct rx * rx, rx_Bitset c)
+#else
+RX_DECL void
+rx_free_cset (rx, c)
+     struct rx * rx;
+     rx_Bitset c;
+#endif
+{
+  if (c)
+    free ((char *)c);
+}
+
+
+
+/* Hash table memory allocation policy for the regexp compiler */
+
+#ifdef __STDC__
+static struct rx_hash *
+compiler_hash_alloc (struct rx_hash_rules * rules)
+#else
+static struct rx_hash *
+compiler_hash_alloc (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  return (struct rx_hash *)malloc (sizeof (struct rx_hash));
+}
+
+
+#ifdef __STDC__
+static struct rx_hash_item *
+compiler_hash_item_alloc (struct rx_hash_rules * rules, void * value)
+#else
+static struct rx_hash_item *
+compiler_hash_item_alloc (rules, value)
+     struct rx_hash_rules * rules;
+     void * value;
+#endif
+{
+  struct rx_hash_item * it;
+  it = (struct rx_hash_item *)malloc (sizeof (*it));
+  if (it)
+    {
+      it->data = value;
+      it->binding = 0;
+    }
+  return it;
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash (struct rx_hash * tab,
+		    struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash (tab, rules)
+     struct rx_hash * tab;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)tab);
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash_item (struct rx_hash_item * item,
+			 struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash_item (item, rules)
+     struct rx_hash_item * item;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)item);
+}
+
+
+
+/* This page: REXP_NODE (expression tree) structures. */
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rexp_node (struct rx *rx,
+	   enum rexp_node_type type)
+#else
+RX_DECL struct rexp_node *
+rexp_node (rx, type)
+     struct rx *rx;
+     enum rexp_node_type type;
+#endif
+{
+  struct rexp_node *n;
+
+  n = (struct rexp_node *)malloc (sizeof (*n));
+  bzero (n, sizeof (*n));
+  if (n)
+    n->type = type;
+  return n;
+}
+
+
+/* free_rexp_node assumes that the bitset passed to rx_mk_r_cset
+ * can be freed using rx_free_cset.
+ */
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_cset (struct rx * rx,
+	      rx_Bitset b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_cset (rx, b)
+     struct rx * rx;
+     rx_Bitset b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_cset);
+  if (n)
+    n->params.cset = b;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_concat (struct rx * rx,
+		struct rexp_node * a,
+		struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_concat (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_concat);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (struct rx * rx,
+		   struct rexp_node * a,
+		   struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_alternate);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_opt (struct rx * rx,
+	     struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_opt (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_opt);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_star (struct rx * rx,
+	      struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_star (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (struct rx * rx,
+		     struct rexp_node * a,
+		     struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_2phase_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (struct rx * rx,
+		     rx_side_effect a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (rx, a)
+     struct rx * rx;
+     rx_side_effect a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_side_effect);
+  if (n)
+    {
+      n->params.side_effect = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_data  (struct rx * rx,
+	       void * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_data  (rx, a)
+     struct rx * rx;
+     void * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_data);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_rexp (struct rx * rx, struct rexp_node * node)
+#else
+RX_DECL void
+rx_free_rexp (rx, node)
+     struct rx * rx;
+     struct rexp_node * node;
+#endif
+{
+  if (node)
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  if (node->params.cset)
+	    rx_free_cset (rx, node->params.cset);
+
+	case r_side_effect:
+	  break;
+	  
+	case r_concat:
+	case r_alternate:
+	case r_2phase_star:
+	case r_opt:
+	case r_star:
+	  rx_free_rexp (rx, node->params.pair.left);
+	  rx_free_rexp (rx, node->params.pair.right);
+	  break;
+
+	case r_data:
+	  /* This shouldn't occur. */
+	  break;
+	}
+      free ((char *)node);
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node * 
+rx_copy_rexp (struct rx *rx,
+	   struct rexp_node *node)
+#else
+RX_DECL struct rexp_node * 
+rx_copy_rexp (rx, node)
+     struct rx *rx;
+     struct rexp_node *node;
+#endif
+{
+  if (!node)
+    return 0;
+  else
+    {
+      struct rexp_node *n = rexp_node (rx, node->type);
+      if (!n)
+	return 0;
+      switch (node->type)
+	{
+	case r_cset:
+	  n->params.cset = rx_copy_cset (rx, node->params.cset);
+	  if (!n->params.cset)
+	    {
+	      rx_free_rexp (rx, n);
+	      return 0;
+	    }
+	  break;
+
+	case r_side_effect:
+	  n->params.side_effect = node->params.side_effect;
+	  break;
+
+	case r_concat:
+	case r_alternate:
+	case r_opt:
+	case r_2phase_star:
+	case r_star:
+	  n->params.pair.left =
+	    rx_copy_rexp (rx, node->params.pair.left);
+	  n->params.pair.right =
+	    rx_copy_rexp (rx, node->params.pair.right);
+	  if (   (node->params.pair.left && !n->params.pair.left)
+	      || (node->params.pair.right && !n->params.pair.right))
+	    {
+	      rx_free_rexp  (rx, n);
+	      return 0;
+	    }
+	  break;
+	case r_data:
+	  /* shouldn't happen */
+	  break;
+	}
+      return n;
+    }
+}
+
+
+
+
+/* This page: functions to build and destroy graphs that describe nfa's */
+
+/* Constructs a new nfa node. */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (struct rx *rx)
+#else
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state * n = (struct rx_nfa_state *)malloc (sizeof (*n));
+  if (!n)
+    return 0;
+  bzero (n, sizeof (*n));
+  n->next = rx->nfa_states;
+  rx->nfa_states = n;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_state (struct rx_nfa_state * n)
+#else
+RX_DECL void
+rx_free_nfa_state (n)
+  struct rx_nfa_state * n;
+#endif
+{
+  free ((char *)n);
+}
+
+
+/* This looks up an nfa node, given a numeric id.  Numeric id's are
+ * assigned after the nfa has been built.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (struct rx * rx,
+		    int id)
+#else
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (rx, id)
+     struct rx * rx;
+     int id;
+#endif
+{
+  struct rx_nfa_state * n;
+  for (n = rx->nfa_states; n; n = n->next)
+    if (n->id == id)
+      return n;
+  return 0;
+}
+
+
+/* This adds an edge between two nodes, but doesn't initialize the 
+ * edge label.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (struct rx *rx,
+	     enum rx_nfa_etype type,
+	     struct rx_nfa_state *start,
+	     struct rx_nfa_state *dest)
+#else
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (rx, type, start, dest)
+     struct rx *rx;
+     enum rx_nfa_etype type;
+     struct rx_nfa_state *start;
+     struct rx_nfa_state *dest;
+#endif
+{
+  struct rx_nfa_edge *e;
+  e = (struct rx_nfa_edge *)malloc (sizeof (*e));
+  if (!e)
+    return 0;
+  e->next = start->edges;
+  start->edges = e;
+  e->type = type;
+  e->dest = dest;
+  return e;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_edge (struct rx_nfa_edge * e)
+#else
+RX_DECL void
+rx_free_nfa_edge (e)
+     struct rx_nfa_edge * e;
+#endif
+{
+  free ((char *)e);
+}
+
+
+/* This constructs a POSSIBLE_FUTURE, which is a kind epsilon-closure
+ * of an NFA.  These are added to an nfa automaticly by eclose_nfa.
+ */  
+
+#ifdef __STDC__
+static struct rx_possible_future * 
+rx_possible_future (struct rx * rx,
+		 struct rx_se_list * effects)
+#else
+static struct rx_possible_future * 
+rx_possible_future (rx, effects)
+     struct rx * rx;
+     struct rx_se_list * effects;
+#endif
+{
+  struct rx_possible_future *ec;
+  ec = (struct rx_possible_future *) malloc (sizeof (*ec));
+  if (!ec)
+    return 0;
+  ec->destset = 0;
+  ec->next = 0;
+  ec->effects = effects;
+  return ec;
+}
+
+
+#ifdef __STDC__
+static void
+rx_free_possible_future (struct rx_possible_future * pf)
+#else
+static void
+rx_free_possible_future (pf)
+     struct rx_possible_future * pf;
+#endif
+{
+  free ((char *)pf);
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa (struct rx *rx)
+#else
+RX_DECL void
+rx_free_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  while (rx->nfa_states)
+    {
+      while (rx->nfa_states->edges)
+	{
+	  switch (rx->nfa_states->edges->type)
+	    {
+	    case ne_cset:
+	      rx_free_cset (rx, rx->nfa_states->edges->params.cset);
+	      break;
+	    default:
+	      break;
+	    }
+	  {
+	    struct rx_nfa_edge * e;
+	    e = rx->nfa_states->edges;
+	    rx->nfa_states->edges = rx->nfa_states->edges->next;
+	    rx_free_nfa_edge (e);
+	  }
+	} /* while (rx->nfa_states->edges) */
+      {
+	/* Iterate over the partial epsilon closures of rx->nfa_states */
+	struct rx_possible_future * pf = rx->nfa_states->futures;
+	while (pf)
+	  {
+	    struct rx_possible_future * pft = pf;
+	    pf = pf->next;
+	    rx_free_possible_future (pft);
+	  }
+      }
+      {
+	struct rx_nfa_state *n;
+	n = rx->nfa_states;
+	rx->nfa_states = rx->nfa_states->next;
+	rx_free_nfa_state (n);
+      }
+    }
+}
+
+
+
+
+/* This page: translating a pattern expression into an nfa and doing the 
+ * static part of the nfa->super-nfa translation.
+ */
+
+/* This is the thompson regexp->nfa algorithm. 
+ * It is modified to allow for `side-effect epsilons.'  Those are
+ * edges that are taken whenever a similar epsilon edge would be,
+ * but which imply that some side effect occurs when the edge 
+ * is taken.
+ *
+ * Side effects are used to model parts of the pattern langauge 
+ * that are not regular (in the formal sense).
+ */
+
+#ifdef __STDC__
+RX_DECL int
+rx_build_nfa (struct rx *rx,
+	      struct rexp_node *rexp,
+	      struct rx_nfa_state **start,
+	      struct rx_nfa_state **end)
+#else
+RX_DECL int
+rx_build_nfa (rx, rexp, start, end)
+     struct rx *rx;
+     struct rexp_node *rexp;
+     struct rx_nfa_state **start;
+     struct rx_nfa_state **end;
+#endif
+{
+  struct rx_nfa_edge *edge;
+
+  /* Start & end nodes may have been allocated by the caller. */
+  *start = *start ? *start : rx_nfa_state (rx);
+
+  if (!*start)
+    return 0;
+
+  if (!rexp)
+    {
+      *end = *start;
+      return 1;
+    }
+
+  *end = *end ? *end : rx_nfa_state (rx);
+
+  if (!*end)
+    {
+      rx_free_nfa_state (*start);
+      return 0;
+    }
+
+  switch (rexp->type)
+    {
+    case r_data:
+      return 0;
+
+    case r_cset:
+      edge = rx_nfa_edge (rx, ne_cset, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.cset = rx_copy_cset (rx, rexp->params.cset);
+      if (!edge->params.cset)
+	{
+	  rx_free_nfa_edge (edge);
+	  return 0;
+	}
+      return 1;
+ 
+    case r_opt:
+      return (rx_build_nfa (rx, rexp->params.pair.left, start, end)
+	      && rx_nfa_edge (rx, ne_epsilon, *start, *end));
+
+    case r_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& star_start
+		&& star_end
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, star_end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, star_start));
+      }
+
+    case r_2phase_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	struct rx_nfa_state * loop_exp_start = 0;
+	struct rx_nfa_state * loop_exp_end = 0;
+
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& rx_build_nfa (rx, rexp->params.pair.right,
+				 &loop_exp_start, &loop_exp_end)
+		&& star_start
+		&& star_end
+		&& loop_exp_end
+		&& loop_exp_start
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, loop_exp_start)
+		&& rx_nfa_edge (rx, ne_epsilon, loop_exp_end, star_start));
+      }
+
+
+    case r_concat:
+      {
+	struct rx_nfa_state *shared = 0;
+	return
+	  (rx_build_nfa (rx, rexp->params.pair.left, start, &shared)
+	   && rx_build_nfa (rx, rexp->params.pair.right, &shared, end));
+      }
+
+    case r_alternate:
+      {
+	struct rx_nfa_state *ls = 0;
+	struct rx_nfa_state *le = 0;
+	struct rx_nfa_state *rs = 0;
+	struct rx_nfa_state *re = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left, &ls, &le)
+		&& rx_build_nfa (rx, rexp->params.pair.right, &rs, &re)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, ls)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, rs)
+		&& rx_nfa_edge (rx, ne_epsilon, le, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, re, *end));
+      }
+
+    case r_side_effect:
+      edge = rx_nfa_edge (rx, ne_side_effect, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.side_effect = rexp->params.side_effect;
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+/* RX_NAME_NFA_STATES identifies all nodes with outgoing non-epsilon
+ * transitions.  Only these nodes can occur in super-states.  
+ * All nodes are given an integer id. 
+ * The id is non-negative if the node has non-epsilon out-transitions, negative
+ * otherwise (this is because we want the non-negative ids to be used as 
+ * array indexes in a few places).
+ */
+
+#ifdef __STDC__
+RX_DECL void
+rx_name_nfa_states (struct rx *rx)
+#else
+RX_DECL void
+rx_name_nfa_states (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+
+  rx->nodec = 0;
+  rx->epsnodec = -1;
+
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+
+      if (n->is_start)
+	n->eclosure_needed = 1;
+
+      while (e)
+	{
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      break;
+
+	    case ne_cset:
+	      n->id = rx->nodec++;
+	      {
+		struct rx_nfa_edge *from_n = n->edges;
+		while (from_n)
+		  {
+		    from_n->dest->eclosure_needed = 1;
+		    from_n = from_n->next;
+		  }
+	      }
+	      goto cont;
+	    }
+	  e = e->next;
+	}
+      n->id = rx->epsnodec--;
+    cont:
+      n = n->next;
+    }
+  rx->epsnodec = -rx->epsnodec;
+}
+
+
+
+/* This page: data structures for the static part of the nfa->supernfa
+ * translation.
+ *
+ * There are side effect lists -- lists of side effects occuring
+ * along an uninterrupted, acyclic path of side-effect epsilon edges.
+ * Such paths are collapsed to single edges in the course of computing
+ * epsilon closures.  Such single edges are labled with a list of all
+ * the side effects entailed in crossing them.  Like lists of side
+ * effects are made == by the constructors below.
+ *
+ * There are also nfa state sets.  These are used to hold a list of all
+ * states reachable from a starting state for a given type of transition
+ * and side effect list.   These are also hash-consed.
+ */
+
+/* The next several functions compare, construct, etc. lists of side
+ * effects.  See ECLOSE_NFA (below) for details.
+ */
+
+/* Ordering of rx_se_list
+ * (-1, 0, 1 return value convention).
+ */
+
+#ifdef __STDC__
+static int 
+se_list_cmp (void * va, void * vb)
+#else
+static int 
+se_list_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_se_list * a = (struct rx_se_list *)va;
+  struct rx_se_list * b = (struct rx_se_list *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: ((long)a->car < (long)b->car
+		   ? 1
+		   : ((long)a->car > (long)b->car
+		      ? -1
+		      : se_list_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+
+#ifdef __STDC__
+static int 
+se_list_equal (void * va, void * vb)
+#else
+static int 
+se_list_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !(se_list_cmp (va, vb));
+}
+
+static struct rx_hash_rules se_list_hash_rules =
+{
+  se_list_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_se_list * 
+side_effect_cons (struct rx * rx,
+		  void * se, struct rx_se_list * list)
+#else
+static struct rx_se_list * 
+side_effect_cons (rx, se, list)
+     struct rx * rx;
+     void * se;
+     struct rx_se_list * list;
+#endif
+{
+  struct rx_se_list * l;
+  l = ((struct rx_se_list *) malloc (sizeof (*l)));
+  if (!l)
+    return 0;
+  l->car = se;
+  l->cdr = list;
+  return l;
+}
+
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_cons_se_prog (struct rx * rx,
+		   struct rx_hash * memo,
+		   void * car, struct rx_se_list * cdr)
+#else
+static struct rx_se_list *
+hash_cons_se_prog (rx, memo, car, cdr)
+     struct rx * rx;
+     struct rx_hash * memo;
+     void * car;
+     struct rx_se_list * cdr;
+#endif
+{
+  long hash = (long)car ^ (long)cdr;
+  struct rx_se_list template;
+
+  template.car = car;
+  template.cdr = cdr;
+  {
+    struct rx_hash_item * it = rx_hash_store (memo, hash,
+					      (void *)&template,
+					      &se_list_hash_rules);
+    if (!it)
+      return 0;
+    if (it->data == (void *)&template)
+      {
+	struct rx_se_list * consed;
+	consed = (struct rx_se_list *) malloc (sizeof (*consed));
+	*consed = template;
+	it->data = (void *)consed;
+      }
+    return (struct rx_se_list *)it->data;
+  }
+}
+     
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_se_prog (struct rx * rx, struct rx_hash * memo, struct rx_se_list * prog)
+#else
+static struct rx_se_list *
+hash_se_prog (rx, memo, prog)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_se_list * prog;
+#endif
+{
+  struct rx_se_list * answer = 0;
+  while (prog)
+    {
+      answer = hash_cons_se_prog (rx, memo, prog->car, answer);
+      if (!answer)
+	return 0;
+      prog = prog->cdr;
+    }
+  return answer;
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_cmp (void * va, void * vb)
+#else
+static int 
+nfa_set_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state_set * a = (struct rx_nfa_state_set *)va;
+  struct rx_nfa_state_set * b = (struct rx_nfa_state_set *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: (a->car->id < b->car->id
+		   ? 1
+		   : (a->car->id > b->car->id
+		      ? -1
+		      : nfa_set_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_equal (void * va, void * vb)
+#else
+static int 
+nfa_set_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !nfa_set_cmp (va, vb);
+}
+
+static struct rx_hash_rules nfa_set_hash_rules =
+{
+  nfa_set_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_cons (struct rx * rx,
+	      struct rx_hash * memo, struct rx_nfa_state * state,
+	      struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_cons (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  struct rx_nfa_state_set template;
+  struct rx_hash_item * node;
+  template.car = state;
+  template.cdr = set;
+  node = rx_hash_store (memo,
+			(((long)state) >> 8) ^ (long)set,
+			&template, &nfa_set_hash_rules);
+  if (!node)
+    return 0;
+  if (node->data == &template)
+    {
+      struct rx_nfa_state_set * l;
+      l = (struct rx_nfa_state_set *) malloc (sizeof (*l));
+      node->data = (void *) l;
+      if (!l)
+	return 0;
+      *l = template;
+    }
+  return (struct rx_nfa_state_set *)node->data;
+}
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (struct rx * rx,
+		struct rx_hash * memo, struct rx_nfa_state * state,
+		struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  if (!set || state->id < set->car->id)
+    return nfa_set_cons (rx, memo, state, set);
+  if (state->id == set->car->id)
+    return set;
+  else
+    {
+      struct rx_nfa_state_set * newcdr
+	= nfa_set_enjoin (rx, memo, state, set->cdr);
+      if (newcdr != set->cdr)
+	set = nfa_set_cons (rx, memo, set->car, newcdr);
+      return set;
+    }
+}
+
+
+
+
+/* This page: computing epsilon closures.  The closures aren't total.
+ * Each node's closures are partitioned according to the side effects entailed
+ * along the epsilon edges.  Return true on success.
+ */ 
+
+struct eclose_frame
+{
+  struct rx_se_list *prog_backwards;
+};
+
+
+#ifdef __STDC__
+static int 
+eclose_node (struct rx *rx, struct rx_nfa_state *outnode,
+	     struct rx_nfa_state *node, struct eclose_frame *frame)
+#else
+static int 
+eclose_node (rx, outnode, node, frame)
+     struct rx *rx;
+     struct rx_nfa_state *outnode;
+     struct rx_nfa_state *node;
+     struct eclose_frame *frame;
+#endif
+{
+  struct rx_nfa_edge *e = node->edges;
+
+  /* For each node, we follow all epsilon paths to build the closure.
+   * The closure omits nodes that have only epsilon edges.
+   * The closure is split into partial closures -- all the states in
+   * a partial closure are reached by crossing the same list of
+   * of side effects (though not necessarily the same path).
+   */
+  if (node->mark)
+    return 1;
+  node->mark = 1;
+
+  if (node->id >= 0 || node->is_final)
+    {
+      struct rx_possible_future **ec;
+      struct rx_se_list * prog_in_order
+	= ((struct rx_se_list *)hash_se_prog (rx,
+					      &rx->se_list_memo,
+					      frame->prog_backwards));
+      int cmp;
+
+      ec = &outnode->futures;
+
+      while (*ec)
+	{
+	  cmp = se_list_cmp ((void *)(*ec)->effects, (void *)prog_in_order);
+	  if (cmp <= 0)
+	    break;
+	  ec = &(*ec)->next;
+	}
+      if (!*ec || (cmp < 0))
+	{
+	  struct rx_possible_future * saved = *ec;
+	  *ec = rx_possible_future (rx, prog_in_order);
+	  (*ec)->next = saved;
+	  if (!*ec)
+	    return 0;
+	}
+      if (node->id >= 0)
+	{
+	  (*ec)->destset = nfa_set_enjoin (rx, &rx->set_list_memo,
+					   node, (*ec)->destset);
+	  if (!(*ec)->destset)
+	    return 0;
+	}
+    }
+
+  while (e)
+    {
+      switch (e->type)
+	{
+	case ne_epsilon:
+	  if (!eclose_node (rx, outnode, e->dest, frame))
+	    return 0;
+	  break;
+	case ne_side_effect:
+	  {
+	    frame->prog_backwards = side_effect_cons (rx, 
+						      e->params.side_effect,
+						      frame->prog_backwards);
+	    if (!frame->prog_backwards)
+	      return 0;
+	    if (!eclose_node (rx, outnode, e->dest, frame))
+	      return 0;
+	    {
+	      struct rx_se_list * dying = frame->prog_backwards;
+	      frame->prog_backwards = frame->prog_backwards->cdr;
+	      free ((char *)dying);
+	    }
+	    break;
+	  }
+	default:
+	  break;
+	}
+      e = e->next;
+    }
+  node->mark = 0;
+  return 1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int 
+rx_eclose_nfa (struct rx *rx)
+#else
+RX_DECL int 
+rx_eclose_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct eclose_frame frame;
+  static int rx_id = 0;
+  
+  frame.prog_backwards = 0;
+  rx->rx_id = rx_id++;
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+  while (n)
+    {
+      n->futures = 0;
+      if (n->eclosure_needed && !eclose_node (rx, n, n, &frame))
+	return 0;
+      /* clear_marks (rx); */
+      n = n->next;
+    }
+  return 1;
+}
+
+
+/* This deletes epsilon edges from an NFA.  After running eclose_node,
+ * we have no more need for these edges.  They are removed to simplify
+ * further operations on the NFA.
+ */
+
+#ifdef __STDC__
+RX_DECL void 
+rx_delete_epsilon_transitions (struct rx *rx)
+#else
+RX_DECL void 
+rx_delete_epsilon_transitions (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct rx_nfa_edge **e;
+
+  while (n)
+    {
+      e = &n->edges;
+      while (*e)
+	{
+	  struct rx_nfa_edge *t;
+	  switch ((*e)->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      t = *e;
+	      *e = t->next;
+	      rx_free_nfa_edge (t);
+	      break;
+
+	    default:
+	      e = &(*e)->next;
+	      break;
+	    }
+	}
+      n = n->next;
+    }
+}
+
+
+
+/* This page: storing the nfa in a contiguous region of memory for
+ * subsequent conversion to a super-nfa.
+ */
+
+/* This is for qsort on an array of nfa_states. The order
+ * is based on state ids and goes 
+ *		[0...MAX][MIN..-1] where (MAX>=0) and (MIN<0)
+ * This way, positive ids double as array indices.
+ */
+
+#ifdef __STDC__
+static int 
+nfacmp (void * va, void * vb)
+#else
+static int 
+nfacmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state **a = (struct rx_nfa_state **)va;
+  struct rx_nfa_state **b = (struct rx_nfa_state **)vb;
+  return (*a == *b		/* &&&& 3.18 */
+	  ? 0
+	  : (((*a)->id < 0) == ((*b)->id < 0)
+	     ? (((*a)->id  < (*b)->id) ? -1 : 1)
+	     : (((*a)->id < 0)
+		? 1 : -1)));
+}
+
+#ifdef __STDC__
+static int 
+count_hash_nodes (struct rx_hash * st)
+#else
+static int 
+count_hash_nodes (st)
+     struct rx_hash * st;
+#endif
+{
+  int x;
+  int count = 0;
+  for (x = 0; x < 13; ++x)
+    count += ((st->children[x])
+	      ? count_hash_nodes (st->children[x])
+	      : st->bucket_size[x]);
+  
+  return count;
+}
+
+
+#ifdef __STDC__
+static void 
+se_memo_freer (struct rx_hash_item * node)
+#else
+static void 
+se_memo_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+#ifdef __STDC__
+static void 
+nfa_set_freer (struct rx_hash_item * node)
+#else
+static void 
+nfa_set_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+/* This copies an entire NFA into a single malloced block of memory.
+ * Mostly this is for compatability with regex.c, though it is convenient
+ * to have the nfa nodes in an array.
+ */
+
+#ifdef __STDC__
+RX_DECL int 
+rx_compactify_nfa (struct rx *rx,
+		   void **mem, unsigned long *size)
+#else
+RX_DECL int 
+rx_compactify_nfa (rx, mem, size)
+     struct rx *rx;
+     void **mem;
+     unsigned long *size;
+#endif
+{
+  int total_nodec;
+  struct rx_nfa_state *n;
+  int edgec = 0;
+  int eclosec = 0;
+  int se_list_consc = count_hash_nodes (&rx->se_list_memo);
+  int nfa_setc = count_hash_nodes (&rx->set_list_memo);
+  unsigned long total_size;
+
+  /* This takes place in two stages.   First, the total size of the
+   * nfa is computed, then structures are copied.  
+   */   
+  n = rx->nfa_states;
+  total_nodec = 0;
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      ++total_nodec;
+      while (e)
+	{
+	  ++edgec;
+	  e = e->next;
+	}
+      while (ec)
+	{
+	  ++eclosec;
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+
+  total_size = (total_nodec * sizeof (struct rx_nfa_state)
+		+ edgec * rx_sizeof_bitset (rx->local_cset_size)
+		+ edgec * sizeof (struct rx_nfa_edge)
+		+ nfa_setc * sizeof (struct rx_nfa_state_set)
+		+ eclosec * sizeof (struct rx_possible_future)
+		+ se_list_consc * sizeof (struct rx_se_list)
+		+ rx->reserved);
+
+  if (total_size > *size)
+    {
+      *mem = remalloc (*mem, total_size);
+      if (*mem)
+	*size = total_size;
+      else
+	return 0;
+    }
+  /* Now we've allocated the memory; this copies the NFA. */
+  {
+    static struct rx_nfa_state **scratch = 0;
+    static int scratch_alloc = 0;
+    struct rx_nfa_state *state_base = (struct rx_nfa_state *) * mem;
+    struct rx_nfa_state *new_state = state_base;
+    struct rx_nfa_edge *new_edge =
+      (struct rx_nfa_edge *)
+	((char *) state_base + total_nodec * sizeof (struct rx_nfa_state));
+    struct rx_se_list * new_se_list =
+      (struct rx_se_list *)
+	((char *)new_edge + edgec * sizeof (struct rx_nfa_edge));
+    struct rx_possible_future *new_close =
+      ((struct rx_possible_future *)
+       ((char *) new_se_list
+	+ se_list_consc * sizeof (struct rx_se_list)));
+    struct rx_nfa_state_set * new_nfa_set =
+      ((struct rx_nfa_state_set *)
+       ((char *)new_close + eclosec * sizeof (struct rx_possible_future)));
+    char *new_bitset =
+      ((char *) new_nfa_set + nfa_setc * sizeof (struct rx_nfa_state_set));
+    int x;
+    struct rx_nfa_state *n;
+
+    if (scratch_alloc < total_nodec)
+      {
+	scratch = ((struct rx_nfa_state **)
+		   remalloc (scratch, total_nodec * sizeof (*scratch)));
+	if (scratch)
+	  scratch_alloc = total_nodec;
+	else
+	  {
+	    scratch_alloc = 0;
+	    return 0;
+	  }
+      }
+
+    for (x = 0, n = rx->nfa_states; n; n = n->next)
+      scratch[x++] = n;
+
+    qsort (scratch, total_nodec,
+	   sizeof (struct rx_nfa_state *), (int (*)())nfacmp);
+
+    for (x = 0; x < total_nodec; ++x)
+      {
+	struct rx_possible_future *eclose = scratch[x]->futures;
+	struct rx_nfa_edge *edge = scratch[x]->edges;
+	struct rx_nfa_state *cn = new_state++;
+	cn->futures = 0;
+	cn->edges = 0;
+	cn->next = (x == total_nodec - 1) ? 0 : (cn + 1);
+	cn->id = scratch[x]->id;
+	cn->is_final = scratch[x]->is_final;
+	cn->is_start = scratch[x]->is_start;
+	cn->mark = 0;
+	while (edge)
+	  {
+	    int indx = (edge->dest->id < 0
+			 ? (total_nodec + edge->dest->id)
+			 : edge->dest->id);
+	    struct rx_nfa_edge *e = new_edge++;
+	    rx_Bitset cset = (rx_Bitset) new_bitset;
+	    new_bitset += rx_sizeof_bitset (rx->local_cset_size);
+	    rx_bitset_null (rx->local_cset_size, cset);
+	    rx_bitset_union (rx->local_cset_size, cset, edge->params.cset);
+	    e->next = cn->edges;
+	    cn->edges = e;
+	    e->type = edge->type;
+	    e->dest = state_base + indx;
+	    e->params.cset = cset;
+	    edge = edge->next;
+	  }
+	while (eclose)
+	  {
+	    struct rx_possible_future *ec = new_close++;
+	    struct rx_hash_item * sp;
+	    struct rx_se_list ** sepos;
+	    struct rx_se_list * sesrc;
+	    struct rx_nfa_state_set * destlst;
+	    struct rx_nfa_state_set ** destpos;
+	    ec->next = cn->futures;
+	    cn->futures = ec;
+	    for (sepos = &ec->effects, sesrc = eclose->effects;
+		 sesrc;
+		 sesrc = sesrc->cdr, sepos = &(*sepos)->cdr)
+	      {
+		sp = rx_hash_find (&rx->se_list_memo,
+				   (long)sesrc->car ^ (long)sesrc->cdr,
+				   sesrc, &se_list_hash_rules);
+		if (sp->binding)
+		  {
+		    sesrc = (struct rx_se_list *)sp->binding;
+		    break;
+		  }
+		*new_se_list = *sesrc;
+		sp->binding = (void *)new_se_list;
+		*sepos = new_se_list;
+		++new_se_list;
+	      }
+	    *sepos = sesrc;
+	    for (destpos = &ec->destset, destlst = eclose->destset;
+		 destlst;
+		 destpos = &(*destpos)->cdr, destlst = destlst->cdr)
+	      {
+		sp = rx_hash_find (&rx->set_list_memo,
+				   ((((long)destlst->car) >> 8)
+				    ^ (long)destlst->cdr),
+				   destlst, &nfa_set_hash_rules);
+		if (sp->binding)
+		  {
+		    destlst = (struct rx_nfa_state_set *)sp->binding;
+		    break;
+		  }
+		*new_nfa_set = *destlst;
+		new_nfa_set->car = state_base + destlst->car->id;
+		sp->binding = (void *)new_nfa_set;
+		*destpos = new_nfa_set;
+		++new_nfa_set;
+	      }
+	    *destpos = destlst;
+	    eclose = eclose->next;
+	  }
+      }
+  }
+  rx_free_hash_table (&rx->se_list_memo, se_memo_freer, &se_list_hash_rules);
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  rx_free_hash_table (&rx->set_list_memo, nfa_set_freer, &nfa_set_hash_rules);
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+
+  rx_free_nfa (rx);
+  rx->nfa_states = (struct rx_nfa_state *)*mem;
+  return 1;
+}
+
+
+
+/* The functions in the next several pages define the lazy-NFA-conversion used
+ * by matchers.  The input to this construction is an NFA such as 
+ * is built by compactify_nfa (rx.c).  The output is the superNFA.
+ */
+
+/* Match engines can use arbitrary values for opcodes.  So, the parse tree 
+ * is built using instructions names (enum rx_opcode), but the superstate
+ * nfa is populated with mystery opcodes (void *).
+ *
+ * For convenience, here is an id table.  The opcodes are == to their inxs
+ *
+ * The lables in re_search_2 would make good values for instructions.
+ */
+
+void * rx_id_instruction_table[rx_num_instructions] =
+{
+  (void *) rx_backtrack_point,
+  (void *) rx_do_side_effects,
+  (void *) rx_cache_miss,
+  (void *) rx_next_char,
+  (void *) rx_backtrack,
+  (void *) rx_error_inx
+};
+
+
+
+
+/* Memory mgt. for superstate graphs. */
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc (struct rx_cache * cache, int bytes)
+#else
+static char *
+rx_cache_malloc (cache, bytes)
+     struct rx_cache * cache;
+     int bytes;
+#endif
+{
+  while (cache->bytes_left < bytes)
+    {
+      if (cache->memory_pos)
+	cache->memory_pos = cache->memory_pos->next;
+      if (!cache->memory_pos)
+	{
+	  cache->morecore (cache);
+	  if (!cache->memory_pos)
+	    return 0;
+	}
+      cache->bytes_left = cache->memory_pos->bytes;
+      cache->memory_addr = ((char *)cache->memory_pos
+			    + sizeof (struct rx_blocklist));
+    }
+  cache->bytes_left -= bytes;
+  {
+    char * addr = cache->memory_addr;
+    cache->memory_addr += bytes;
+    return addr;
+  }
+}
+
+#ifdef __STDC__
+static void
+rx_cache_free (struct rx_cache * cache,
+	       struct rx_freelist ** freelist, char * mem)
+#else
+static void
+rx_cache_free (cache, freelist, mem)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     char * mem;
+#endif
+{
+  struct rx_freelist * it = (struct rx_freelist *)mem;
+  it->next = *freelist;
+  *freelist = it;
+}
+
+
+/* The partially instantiated superstate graph has a transition 
+ * table at every node.  There is one entry for every character.
+ * This fills in the transition for a set.
+ */
+#ifdef __STDC__
+static void 
+install_transition (struct rx_superstate *super,
+		    struct rx_inx *answer, rx_Bitset trcset) 
+#else
+static void 
+install_transition (super, answer, trcset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     rx_Bitset trcset;
+#endif
+{
+  struct rx_inx * transitions = super->transitions;
+  int chr;
+  for (chr = 0; chr < 256; )
+    if (!*trcset)
+      {
+	++trcset;
+	chr += 32;
+      }
+    else
+      {
+	RX_subset sub = *trcset;
+	RX_subset mask = 1;
+	int bound = chr + 32;
+	while (chr < bound)
+	  {
+	    if (sub & mask)
+	      transitions [chr] = *answer;
+	    ++chr;
+	    mask <<= 1;
+	  }
+	++trcset;
+      }
+}
+
+
+#ifdef __STDC__
+static int
+qlen (struct rx_superstate * q)
+#else
+static int
+qlen (q)
+     struct rx_superstate * q;
+#endif
+{
+  int count = 1;
+  struct rx_superstate * it;
+  if (!q)
+    return 0;
+  for (it = q->next_recyclable; it != q; it = it->next_recyclable)
+    ++count;
+  return count;
+}
+
+#ifdef __STDC__
+static void
+check_cache (struct rx_cache * cache)
+#else
+static void
+check_cache (cache)
+     struct rx_cache * cache;
+#endif
+{
+  struct rx_cache * you_fucked_up = 0;
+  int total = cache->superstates;
+  int semi = cache->semifree_superstates;
+  if (semi != qlen (cache->semifree_superstate))
+    check_cache (you_fucked_up);
+  if ((total - semi) != qlen (cache->lru_superstate))
+    check_cache (you_fucked_up);
+}
+
+/* When a superstate is old and neglected, it can enter a 
+ * semi-free state.  A semi-free state is slated to die.
+ * Incoming transitions to a semi-free state are re-written
+ * to cause an (interpreted) fault when they are taken.
+ * The fault handler revives the semi-free state, patches
+ * incoming transitions back to normal, and continues.
+ *
+ * The idea is basicly to free in two stages, aborting 
+ * between the two if the state turns out to be useful again.
+ * When a free is aborted, the rescued superstate is placed
+ * in the most-favored slot to maximize the time until it
+ * is next semi-freed.
+ */
+
+#ifdef __STDC__
+static void
+semifree_superstate (struct rx_cache * cache)
+#else
+static void
+semifree_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int disqualified = cache->semifree_superstates;
+  if (disqualified == cache->superstates)
+    return;
+  while (cache->lru_superstate->locks)
+    {
+      cache->lru_superstate = cache->lru_superstate->next_recyclable;
+      ++disqualified;
+      if (disqualified == cache->superstates)
+	return;
+    }
+  {
+    struct rx_superstate * it = cache->lru_superstate;
+    it->next_recyclable->prev_recyclable = it->prev_recyclable;
+    it->prev_recyclable->next_recyclable = it->next_recyclable;
+    cache->lru_superstate = (it == it->next_recyclable
+			     ? 0
+			     : it->next_recyclable);
+    if (!cache->semifree_superstate)
+      {
+	cache->semifree_superstate = it;
+	it->next_recyclable = it;
+	it->prev_recyclable = it;
+      }
+    else
+      {
+	it->prev_recyclable = cache->semifree_superstate->prev_recyclable;
+	it->next_recyclable = cache->semifree_superstate;
+	it->prev_recyclable->next_recyclable = it;
+	it->next_recyclable->prev_recyclable = it;
+      }
+    {
+      struct rx_distinct_future *df;
+      it->is_semifree = 1;
+      ++cache->semifree_superstates;
+      df = it->transition_refs;
+      if (df)
+	{
+	  df->prev_same_dest->next_same_dest = 0;
+	  for (df = it->transition_refs; df; df = df->next_same_dest)
+	    {
+	      df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	      df->future_frame.data = 0;
+	      df->future_frame.data_2 = (void *) df;
+	      /* If there are any NEXT-CHAR instruction frames that
+	       * refer to this state, we convert them to CACHE-MISS frames.
+	       */
+	      if (!df->effects
+		  && (df->edge->options->next_same_super_edge[0]
+		      == df->edge->options))
+		install_transition (df->present, &df->future_frame,
+				    df->edge->cset);
+	    }
+	  df = it->transition_refs;
+	  df->prev_same_dest->next_same_dest = df;
+	}
+    }
+  }
+}
+
+
+#ifdef __STDC__
+static void 
+refresh_semifree_superstate (struct rx_cache * cache,
+			     struct rx_superstate * super)
+#else
+static void 
+refresh_semifree_superstate (cache, super)
+     struct rx_cache * cache;
+     struct rx_superstate * super;
+#endif
+{
+  struct rx_distinct_future *df;
+
+  if (super->transition_refs)
+    {
+      super->transition_refs->prev_same_dest->next_same_dest = 0; 
+      for (df = super->transition_refs; df; df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_next_char];
+	  df->future_frame.data = (void *) super->transitions;
+	  /* CACHE-MISS instruction frames that refer to this state,
+	   * must be converted to NEXT-CHAR frames.
+	   */
+	  if (!df->effects
+	      && (df->edge->options->next_same_super_edge[0]
+		  == df->edge->options))
+	    install_transition (df->present, &df->future_frame,
+				df->edge->cset);
+	}
+      super->transition_refs->prev_same_dest->next_same_dest
+	= super->transition_refs;
+    }
+  if (cache->semifree_superstate == super)
+    cache->semifree_superstate = (super->prev_recyclable == super
+				  ? 0
+				  : super->prev_recyclable);
+  super->next_recyclable->prev_recyclable = super->prev_recyclable;
+  super->prev_recyclable->next_recyclable = super->next_recyclable;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = super->next_recyclable
+     = super->prev_recyclable
+     = super);
+  else
+    {
+      super->next_recyclable = cache->lru_superstate;
+      super->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      super->next_recyclable->prev_recyclable = super;
+      super->prev_recyclable->next_recyclable = super;
+    }
+  super->is_semifree = 0;
+  --cache->semifree_superstates;
+}
+
+#ifdef __STDC__
+static void
+rx_refresh_this_superstate (struct rx_cache * cache, struct rx_superstate * superstate)
+#else
+static void
+rx_refresh_this_superstate (cache, superstate)
+     struct rx_cache * cache;
+     struct rx_superstate * superstate;
+#endif
+{
+  if (superstate->is_semifree)
+    refresh_semifree_superstate (cache, superstate);
+  else if (cache->lru_superstate == superstate)
+    cache->lru_superstate = superstate->next_recyclable;
+  else if (superstate != cache->lru_superstate->prev_recyclable)
+    {
+      superstate->next_recyclable->prev_recyclable
+	= superstate->prev_recyclable;
+      superstate->prev_recyclable->next_recyclable
+	= superstate->next_recyclable;
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      superstate->next_recyclable->prev_recyclable = superstate;
+      superstate->prev_recyclable->next_recyclable = superstate;
+    }
+}
+
+#ifdef __STDC__
+static void 
+release_superset_low (struct rx_cache * cache,
+		     struct rx_superset *set)
+#else
+static void 
+release_superset_low (cache, set)
+     struct rx_cache * cache;
+     struct rx_superset *set;
+#endif
+{
+  if (!--set->refs)
+    {
+      if (set->cdr)
+	release_superset_low (cache, set->cdr);
+
+      set->starts_for = 0;
+
+      rx_hash_free
+	(rx_hash_find
+	 (&cache->superset_table,
+	  (unsigned long)set->car ^ set->id ^ (unsigned long)set->cdr,
+	  (void *)set,
+	  &cache->superset_hash_rules),
+	 &cache->superset_hash_rules);
+      rx_cache_free (cache, &cache->free_supersets, (char *)set);
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void 
+rx_release_superset (struct rx *rx,
+		     struct rx_superset *set)
+#else
+RX_DECL void 
+rx_release_superset (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  release_superset_low (rx->cache, set);
+}
+
+/* This tries to add a new superstate to the superstate freelist.
+ * It might, as a result, free some edge pieces or hash tables.
+ * If nothing can be freed because too many locks are being held, fail.
+ */
+
+#ifdef __STDC__
+static int
+rx_really_free_superstate (struct rx_cache * cache)
+#else
+static int
+rx_really_free_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int locked_superstates = 0;
+  struct rx_superstate * it;
+
+  if (!cache->superstates)
+    return 0;
+
+  {
+    /* This is a total guess.  The idea is that we should expect as
+     * many misses as we've recently experienced.  I.e., cache->misses
+     * should be the same as cache->semifree_superstates.
+     */
+    while ((cache->hits + cache->misses) > cache->superstates_allowed)
+      {
+	cache->hits >>= 1;
+	cache->misses >>= 1;
+      }
+    if (  ((cache->hits + cache->misses) * cache->semifree_superstates)
+	< (cache->superstates		 * cache->misses))
+      {
+	semifree_superstate (cache);
+	semifree_superstate (cache);
+      }
+  }
+
+  while (cache->semifree_superstate && cache->semifree_superstate->locks)
+    {
+      refresh_semifree_superstate (cache, cache->semifree_superstate);
+      ++locked_superstates;
+      if (locked_superstates == cache->superstates)
+	return 0;
+    }
+
+  if (cache->semifree_superstate)
+    {
+      it = cache->semifree_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->semifree_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+      --cache->semifree_superstates;
+    }
+  else
+    {
+      while (cache->lru_superstate->locks)
+	{
+	  cache->lru_superstate = cache->lru_superstate->next_recyclable;
+	  ++locked_superstates;
+	  if (locked_superstates == cache->superstates)
+	    return 0;
+	}
+      it = cache->lru_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->lru_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+    }
+
+  if (it->transition_refs)
+    {
+      struct rx_distinct_future *df;
+      for (df = it->transition_refs,
+	   df->prev_same_dest->next_same_dest = 0;
+	   df;
+	   df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	  df->future_frame.data = 0;
+	  df->future_frame.data_2 = (void *) df;
+	  df->future = 0;
+	}
+      it->transition_refs->prev_same_dest->next_same_dest =
+	it->transition_refs;
+    }
+  {
+    struct rx_super_edge *tc = it->edges;
+    while (tc)
+      {
+	struct rx_distinct_future * df;
+	struct rx_super_edge *tct = tc->next;
+	df = tc->options;
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+	while (df)
+	  {
+	    struct rx_distinct_future *dft = df;
+	    df = df->next_same_super_edge[0];
+	    
+	    
+	    if (dft->future && dft->future->transition_refs == dft)
+	      {
+		dft->future->transition_refs = dft->next_same_dest;
+		if (dft->future->transition_refs == dft)
+		  dft->future->transition_refs = 0;
+	      }
+	    dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+	    dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+	    rx_cache_free (cache, &cache->free_discernable_futures,
+			   (char *)dft);
+	  }
+	rx_cache_free (cache, &cache->free_transition_classes, (char *)tc);
+	tc = tct;
+      }
+  }
+  
+  if (it->contents->superstate == it)
+    it->contents->superstate = 0;
+  release_superset_low (cache, it->contents);
+  rx_cache_free (cache, &cache->free_superstates, (char *)it);
+  --cache->superstates;
+  return 1;
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get (struct rx_cache * cache,
+	      struct rx_freelist ** freelist)
+#else
+static char *
+rx_cache_get (cache, freelist)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+#endif
+{
+  while (!*freelist && rx_really_free_superstate (cache))
+    ;
+  if (!*freelist)
+    return 0;
+  {
+    struct rx_freelist * it = *freelist;
+    *freelist = it->next;
+    return (char *)it;
+  }
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc_or_get (struct rx_cache * cache,
+			struct rx_freelist ** freelist, int bytes)
+#else
+static char *
+rx_cache_malloc_or_get (cache, freelist, bytes)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     int bytes;
+#endif
+{
+  if (!*freelist)
+    {
+      char * answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	return answer;
+    }
+
+  return rx_cache_get (cache, freelist);
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get_superstate (struct rx_cache * cache)
+#else
+static char *
+rx_cache_get_superstate (cache)
+	  struct rx_cache * cache;
+#endif
+{
+  char * answer;
+  int bytes = (   sizeof (struct rx_superstate)
+	       +  cache->local_cset_size * sizeof (struct rx_inx));
+  if (!cache->free_superstates
+      && (cache->superstates < cache->superstates_allowed))
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	{
+	  ++cache->superstates;
+	  return answer;
+	}
+    }
+  answer = rx_cache_get (cache, &cache->free_superstates);
+  if (!answer)
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	++cache->superstates_allowed;
+    }
+  ++cache->superstates;
+  return answer;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+supersetcmp (void * va, void * vb)
+#else
+static int
+supersetcmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_superset * a = (struct rx_superset *)va;
+  struct rx_superset * b = (struct rx_superset *)vb;
+  return (   (a == b)
+	  || (a && b && (a->car == b->car) && (a->cdr == b->cdr)));
+}
+
+#ifdef __STDC__
+static struct rx_hash_item *
+superset_allocator (struct rx_hash_rules * rules, void * val)
+#else
+static struct rx_hash_item *
+superset_allocator (rules, val)
+     struct rx_hash_rules * rules;
+     void * val;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  struct rx_superset * template = (struct rx_superset *)val;
+  struct rx_superset * newset
+    = ((struct rx_superset *)
+       rx_cache_malloc_or_get (cache,
+			       &cache->free_supersets,
+			       sizeof (*template)));
+  if (!newset)
+    return 0;
+  newset->refs = 0;
+  newset->car = template->car;
+  newset->id = template->car->id;
+  newset->cdr = template->cdr;
+  newset->superstate = 0;
+  rx_protect_superset (rx, template->cdr);
+  newset->hash_item.data = (void *)newset;
+  newset->hash_item.binding = 0;
+  return &newset->hash_item;
+}
+
+#ifdef __STDC__
+static struct rx_hash * 
+super_hash_allocator (struct rx_hash_rules * rules)
+#else
+static struct rx_hash * 
+super_hash_allocator (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  return ((struct rx_hash *)
+	  rx_cache_malloc_or_get (cache,
+				  &cache->free_hash, sizeof (struct rx_hash)));
+}
+
+
+#ifdef __STDC__
+static void
+super_hash_liberator (struct rx_hash * hash, struct rx_hash_rules * rules)
+#else
+static void
+super_hash_liberator (hash, rules)
+     struct rx_hash * hash;
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       (char *)rules - (long)(&((struct rx_cache *)0)->superset_hash_rules));
+  rx_cache_free (cache, &cache->free_hash, (char *)hash);
+}
+
+#ifdef __STDC__
+static void
+superset_hash_item_liberator (struct rx_hash_item * it,
+			      struct rx_hash_rules * rules)
+#else
+static void
+superset_hash_item_liberator (it, rules) /* Well, it does ya know. */
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+}
+
+int rx_cache_bound = 128;
+static int rx_default_cache_got = 0;
+
+#ifdef __STDC__
+static int
+bytes_for_cache_size (int supers, int cset_size)
+#else
+static int
+bytes_for_cache_size (supers, cset_size)
+     int supers;
+     int cset_size;
+#endif
+{
+  /* What the hell is this? !!!*/
+  return (int)
+    ((float)supers *
+     (  (1.03 * (float) (  rx_sizeof_bitset (cset_size)
+			 + sizeof (struct rx_super_edge)))
+      + (1.80 * (float) sizeof (struct rx_possible_future))
+      + (float) (  sizeof (struct rx_superstate)
+		 + cset_size * sizeof (struct rx_inx))));
+}
+
+#ifdef __STDC__
+static void
+rx_morecore (struct rx_cache * cache)
+#else
+static void
+rx_morecore (cache)
+     struct rx_cache * cache;
+#endif
+{
+  if (rx_default_cache_got >= rx_cache_bound)
+    return;
+
+  rx_default_cache_got += 16;
+  cache->superstates_allowed = rx_cache_bound;
+  {
+    struct rx_blocklist ** pos = &cache->memory;
+    int size = bytes_for_cache_size (16, cache->local_cset_size);
+    while (*pos)
+      pos = &(*pos)->next;
+    *pos = ((struct rx_blocklist *)
+	    malloc (size + sizeof (struct rx_blocklist))); 
+    if (!*pos)
+      return;
+
+    (*pos)->next = 0;
+    (*pos)->bytes = size;
+    cache->memory_pos = *pos;
+    cache->memory_addr = (char *)*pos + sizeof (**pos);
+    cache->bytes_left = size;
+  }
+}
+
+static struct rx_cache default_cache = 
+{
+  {
+    supersetcmp,
+    super_hash_allocator,
+    super_hash_liberator,
+    superset_allocator,
+    superset_hash_item_liberator,
+  },
+  0,
+  0,
+  0,
+  0,
+  rx_morecore,
+
+  0,
+  0,
+  0,
+  0,
+  0,
+
+  0,
+  0,
+
+  0,
+
+  0,
+  0,
+  0,
+  0,
+  128,
+
+  256,
+  rx_id_instruction_table,
+
+  {
+    0,
+    0,
+    {0},
+    {0},
+    {0}
+  }
+};
+
+/* This adds an element to a superstate set.  These sets are lists, such
+ * that lists with == elements are ==.  The empty set is returned by
+ * superset_cons (rx, 0, 0) and is NOT equivelent to 
+ * (struct rx_superset)0.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superset_cons (struct rx * rx,
+		  struct rx_nfa_state *car, struct rx_superset *cdr)
+#else
+RX_DECL struct rx_superset *
+rx_superset_cons (rx, car, cdr)
+     struct rx * rx;
+     struct rx_nfa_state *car;
+     struct rx_superset *cdr;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  if (!car && !cdr)
+    {
+      if (!cache->empty_superset)
+	{
+	  cache->empty_superset
+	    = ((struct rx_superset *)
+	       rx_cache_malloc_or_get (cache, &cache->free_supersets,
+				       sizeof (struct rx_superset)));
+	  if (!cache->empty_superset)
+	    return 0;
+	  bzero (cache->empty_superset, sizeof (struct rx_superset));
+	  cache->empty_superset->refs = 1000;
+	}
+      return cache->empty_superset;
+    }
+  {
+    struct rx_superset template;
+    struct rx_hash_item * hit;
+    template.car = car;
+    template.cdr = cdr;
+    template.id = car->id;
+    hit = rx_hash_store (&cache->superset_table,
+			 (unsigned long)car ^ car->id ^ (unsigned long)cdr,
+			 (void *)&template,
+			 &cache->superset_hash_rules);
+    return (hit
+	    ?  (struct rx_superset *)hit->data
+	    : 0);
+  }
+}
+
+/* This computes a union of two NFA state sets.  The sets do not have the
+ * same representation though.  One is a RX_SUPERSET structure (part
+ * of the superstate NFA) and the other is an NFA_STATE_SET (part of the NFA).
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl) 
+#else
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union (rx, set, ecl)
+     struct rx * rx;
+     struct rx_superset *set;
+     struct rx_nfa_state_set *ecl;
+#endif
+{
+  if (!ecl)
+    return set;
+
+  if (!set->car)
+    return rx_superset_cons (rx, ecl->car,
+			     rx_superstate_eclosure_union (rx, set, ecl->cdr));
+  if (set->car == ecl->car)
+    return rx_superstate_eclosure_union (rx, set, ecl->cdr);
+
+  {
+    struct rx_superset * tail;
+    struct rx_nfa_state * first;
+
+    if (set->car > ecl->car)
+      {
+	tail = rx_superstate_eclosure_union (rx, set->cdr, ecl);
+	first = set->car;
+      }
+    else
+      {
+	tail = rx_superstate_eclosure_union (rx, set, ecl->cdr);
+	first = ecl->car;
+      }
+    if (!tail)
+      return 0;
+    else
+      {
+	struct rx_superset * answer;
+	answer = rx_superset_cons (rx, first, tail);
+	if (!answer)
+	  {
+	    rx_protect_superset (rx, tail);
+	    rx_release_superset (rx, tail);
+	    return 0;
+	  }
+	else
+	  return answer;
+      }
+  }
+}
+
+
+
+
+
+/*
+ * This makes sure that a list of rx_distinct_futures contains
+ * a future for each possible set of side effects in the eclosure
+ * of a given state.  This is some of the work of filling in a
+ * superstate transition. 
+ */
+
+#ifdef __STDC__
+static struct rx_distinct_future *
+include_futures (struct rx *rx,
+		 struct rx_distinct_future *df, struct rx_nfa_state
+		 *state, struct rx_superstate *superstate) 
+#else
+static struct rx_distinct_future *
+include_futures (rx, df, state, superstate)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+     struct rx_nfa_state *state;
+     struct rx_superstate *superstate;
+#endif
+{
+  struct rx_possible_future *future;
+  struct rx_cache * cache = rx->cache;
+  for (future = state->futures; future; future = future->next)
+    {
+      struct rx_distinct_future *dfp;
+      struct rx_distinct_future *insert_before = 0;
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      for (dfp = df; dfp; dfp = dfp->next_same_super_edge[0])
+	if (dfp->effects == future->effects)
+	  break;
+	else
+	  {
+	    int order = rx->se_list_cmp (rx, dfp->effects, future->effects);
+	    if (order > 0)
+	      {
+		insert_before = dfp;
+		dfp = 0;
+		break;
+	      }
+	  }
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+      if (!dfp)
+	{
+	  dfp
+	    = ((struct rx_distinct_future *)
+	       rx_cache_malloc_or_get (cache, &cache->free_discernable_futures,
+				       sizeof (struct rx_distinct_future)));
+	  if (!dfp)
+	    return 0;
+	  if (!df)
+	    {
+	      df = insert_before = dfp;
+	      df->next_same_super_edge[0] = df->next_same_super_edge[1] = df;
+	    }
+	  else if (!insert_before)
+	    insert_before = df;
+	  else if (insert_before == df)
+	    df = dfp;
+
+	  dfp->next_same_super_edge[0] = insert_before;
+	  dfp->next_same_super_edge[1]
+	    = insert_before->next_same_super_edge[1];
+	  dfp->next_same_super_edge[1]->next_same_super_edge[0] = dfp;
+	  dfp->next_same_super_edge[0]->next_same_super_edge[1] = dfp;
+	  dfp->next_same_dest = dfp->prev_same_dest = dfp;
+	  dfp->future = 0;
+	  dfp->present = superstate;
+	  dfp->future_frame.inx = rx->instruction_table[rx_cache_miss];
+	  dfp->future_frame.data = 0;
+	  dfp->future_frame.data_2 = (void *) dfp;
+	  dfp->side_effects_frame.inx
+	    = rx->instruction_table[rx_do_side_effects];
+	  dfp->side_effects_frame.data = 0;
+	  dfp->side_effects_frame.data_2 = (void *) dfp;
+	  dfp->effects = future->effects;
+	}
+    }
+  return df;
+}
+
+
+
+
+/* This constructs a new superstate from its state set.  The only 
+ * complexity here is memory management.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_superstate *
+rx_superstate (struct rx *rx,
+	       struct rx_superset *set)
+#else
+RX_DECL struct rx_superstate *
+rx_superstate (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  struct rx_superstate * superstate = 0;
+
+  /* Does the superstate already exist in the cache? */
+  if (set->superstate)
+    {
+      if (set->superstate->rx_id != rx->rx_id)
+	{
+	  /* Aha.  It is in the cache, but belongs to a superstate
+	   * that refers to an NFA that no longer exists.
+	   * (We know it no longer exists because it was evidently
+	   *  stored in the same region of memory as the current nfa
+	   *  yet it has a different id.)
+	   */
+	  superstate = set->superstate;
+	  if (!superstate->is_semifree)
+	    {
+	      if (cache->lru_superstate == superstate)
+		{
+		  cache->lru_superstate = superstate->next_recyclable;
+		  if (cache->lru_superstate == superstate)
+		    cache->lru_superstate = 0;
+		}
+	      {
+		superstate->next_recyclable->prev_recyclable
+		  = superstate->prev_recyclable;
+		superstate->prev_recyclable->next_recyclable
+		  = superstate->next_recyclable;
+		if (!cache->semifree_superstate)
+		  {
+		    (cache->semifree_superstate
+		     = superstate->next_recyclable
+		     = superstate->prev_recyclable
+		     = superstate);
+		  }
+		else
+		  {
+		    superstate->next_recyclable = cache->semifree_superstate;
+		    superstate->prev_recyclable
+		      = cache->semifree_superstate->prev_recyclable;
+		    superstate->next_recyclable->prev_recyclable
+		      = superstate;
+		    superstate->prev_recyclable->next_recyclable
+		      = superstate;
+		    cache->semifree_superstate = superstate;
+		  }
+		++cache->semifree_superstates;
+	      }
+	    }
+	  set->superstate = 0;
+	  goto handle_cache_miss;
+	}
+      ++cache->hits;
+      superstate = set->superstate;
+
+      rx_refresh_this_superstate (cache, superstate);
+      return superstate;
+    }
+
+ handle_cache_miss:
+
+  /* This point reached only for cache misses. */
+  ++cache->misses;
+#if RX_DEBUG
+  if (rx_debug_trace > 1)
+    {
+      struct rx_superset * setp = set;
+      fprintf (stderr, "Building a superstet %d(%d): ", rx->rx_id, set);
+      while (setp)
+	{
+	  fprintf (stderr, "%d ", setp->id);
+	  setp = setp->cdr;
+	}
+      fprintf (stderr, "(%d)\n", set);
+    }
+#endif
+  superstate = (struct rx_superstate *)rx_cache_get_superstate (cache);
+  if (!superstate)
+    return 0;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = superstate->next_recyclable
+     = superstate->prev_recyclable
+     = superstate);
+  else
+    {
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      (  superstate->prev_recyclable->next_recyclable
+       = superstate->next_recyclable->prev_recyclable
+       = superstate);
+    }
+  superstate->rx_id = rx->rx_id;
+  superstate->transition_refs = 0;
+  superstate->locks = 0;
+  superstate->is_semifree = 0;
+  set->superstate = superstate;
+  superstate->contents = set;
+  rx_protect_superset (rx, set);
+  superstate->edges = 0;
+  {
+    int x;
+    /* None of the transitions from this superstate are known yet. */
+    for (x = 0; x < rx->local_cset_size; ++x) /* &&&&& 3.8 % */
+      {
+	struct rx_inx * ifr = &superstate->transitions[x];
+	ifr->inx = rx->instruction_table [rx_cache_miss];
+	ifr->data = ifr->data_2 = 0;
+      }
+  }
+  return superstate;
+}
+
+
+
+/* This computes the destination set of one edge of the superstate NFA.
+ * Note that a RX_DISTINCT_FUTURE is a superstate edge.
+ * Returns 0 on an allocation failure.
+ */
+
+#ifdef __STDC__
+static int 
+solve_destination (struct rx *rx, struct rx_distinct_future *df)
+#else
+static int 
+solve_destination (rx, df)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc = df->edge;
+  struct rx_superset *nfa_state;
+  struct rx_superset *nil_set = rx_superset_cons (rx, 0, 0);
+  struct rx_superset *solution = nil_set;
+  struct rx_superstate *dest;
+
+  rx_protect_superset (rx, solution);
+  /* Iterate over all NFA states in the state set of this superstate. */
+  for (nfa_state = df->present->contents;
+       nfa_state->car;
+       nfa_state = nfa_state->cdr)
+    {
+      struct rx_nfa_edge *e;
+      /* Iterate over all edges of each NFA state. */
+      for (e = nfa_state->car->edges; e; e = e->next)
+        /* If we find an edge that is labeled with 
+	 * the characters we are solving for.....
+	 */
+	if (rx_bitset_is_subset (rx->local_cset_size,
+				 tc->cset, e->params.cset))
+	  {
+	    struct rx_nfa_state *n = e->dest;
+	    struct rx_possible_future *pf;
+	    /* ....search the partial epsilon closures of the destination
+	     * of that edge for a path that involves the same set of
+	     * side effects we are solving for.
+	     * If we find such a RX_POSSIBLE_FUTURE, we add members to the
+	     * stateset we are computing.
+	     */
+	    for (pf = n->futures; pf; pf = pf->next)
+	      if (pf->effects == df->effects)
+		{
+		  struct rx_superset * old_sol;
+		  old_sol = solution;
+		  solution = rx_superstate_eclosure_union (rx, solution,
+							   pf->destset);
+		  if (!solution)
+		    return 0;
+		  rx_protect_superset (rx, solution);
+		  rx_release_superset (rx, old_sol);
+		}
+	  }
+    }
+  /* It is possible that the RX_DISTINCT_FUTURE we are working on has 
+   * the empty set of NFA states as its definition.  In that case, this
+   * is a failure point.
+   */
+  if (solution == nil_set)
+    {
+      df->future_frame.inx = (void *) rx_backtrack;
+      df->future_frame.data = 0;
+      df->future_frame.data_2 = 0;
+      return 1;
+    }
+  dest = rx_superstate (rx, solution);
+  rx_release_superset (rx, solution);
+  if (!dest)
+    return 0;
+
+  {
+    struct rx_distinct_future *dft;
+    dft = df;
+    df->prev_same_dest->next_same_dest = 0;
+    while (dft)
+      {
+	dft->future = dest;
+	dft->future_frame.inx = rx->instruction_table[rx_next_char];
+	dft->future_frame.data = (void *) dest->transitions;
+	dft = dft->next_same_dest;
+      }
+    df->prev_same_dest->next_same_dest = df;
+  }
+  if (!dest->transition_refs)
+    dest->transition_refs = df;
+  else
+    {
+      struct rx_distinct_future *dft = dest->transition_refs->next_same_dest;
+      dest->transition_refs->next_same_dest = df->next_same_dest;
+      df->next_same_dest->prev_same_dest = dest->transition_refs;
+      df->next_same_dest = dft;
+      dft->prev_same_dest = df;
+    }
+  return 1;
+}
+
+
+/* This takes a superstate and a character, and computes some edges
+ * from the superstate NFA.  In particular, this computes all edges
+ * that lead from SUPERSTATE given CHR.   This function also 
+ * computes the set of characters that share this edge set.
+ * This returns 0 on allocation error.
+ * The character set and list of edges are returned through 
+ * the paramters CSETOUT and DFOUT.
+} */
+
+#ifdef __STDC__
+static int 
+compute_super_edge (struct rx *rx, struct rx_distinct_future **dfout,
+			  rx_Bitset csetout, struct rx_superstate *superstate,
+			  unsigned char chr)  
+#else
+static int 
+compute_super_edge (rx, dfout, csetout, superstate, chr)
+     struct rx *rx;
+     struct rx_distinct_future **dfout;
+     rx_Bitset csetout;
+     struct rx_superstate *superstate;
+     unsigned char chr;
+#endif
+{
+  struct rx_superset *stateset = superstate->contents;
+
+  /* To compute the set of characters that share edges with CHR, 
+   * we start with the full character set, and subtract.
+   */
+  rx_bitset_universe (rx->local_cset_size, csetout);
+  *dfout = 0;
+
+  /* Iterate over the NFA states in the superstate state-set. */
+  while (stateset->car)
+    {
+      struct rx_nfa_edge *e;
+      for (e = stateset->car->edges; e; e = e->next)
+	if (RX_bitset_member (e->params.cset, chr))
+	  {
+	    /* If we find an NFA edge that applies, we make sure there
+	     * are corresponding edges in the superstate NFA.
+	     */
+	    {
+	      struct rx_distinct_future * saved;
+	      saved = *dfout;
+	      *dfout = include_futures (rx, *dfout, e->dest, superstate);
+	      if (!*dfout)
+		{
+		  struct rx_distinct_future * df;
+		  df = saved;
+		  df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+		  while (df)
+		    {
+		      struct rx_distinct_future *dft;
+		      dft = df;
+		      df = df->next_same_super_edge[0];
+
+		      if (dft->future && dft->future->transition_refs == dft)
+			{
+			  dft->future->transition_refs = dft->next_same_dest;
+			  if (dft->future->transition_refs == dft)
+			    dft->future->transition_refs = 0;
+			}
+		      dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+		      dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+		      rx_cache_free (rx->cache,
+				     &rx->cache->free_discernable_futures,
+				     (char *)dft);
+		    }
+		  return 0;
+		}
+	    }
+	    /* We also trim the character set a bit. */
+	    rx_bitset_intersection (rx->local_cset_size,
+				    csetout, e->params.cset);
+	  }
+	else
+	  /* An edge that doesn't apply at least tells us some characters
+	   * that don't share the same edge set as CHR.
+	   */
+	  rx_bitset_difference (rx->local_cset_size, csetout, e->params.cset);
+      stateset = stateset->cdr;
+    }
+  return 1;
+}
+
+
+
+/* This is a constructor for RX_SUPER_EDGE structures.  These are
+ * wrappers for lists of superstate NFA edges that share character sets labels.
+ * If a transition class contains more than one rx_distinct_future (superstate
+ * edge), then it represents a non-determinism in the superstate NFA.
+ */
+
+#ifdef __STDC__
+static struct rx_super_edge *
+rx_super_edge (struct rx *rx,
+	       struct rx_superstate *super, rx_Bitset cset,
+	       struct rx_distinct_future *df) 
+#else
+static struct rx_super_edge *
+rx_super_edge (rx, super, cset, df)
+     struct rx *rx;
+     struct rx_superstate *super;
+     rx_Bitset cset;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc =
+    (struct rx_super_edge *)rx_cache_malloc_or_get
+      (rx->cache, &rx->cache->free_transition_classes,
+       sizeof (struct rx_super_edge) + rx_sizeof_bitset (rx->local_cset_size));
+
+  if (!tc)
+    return 0;
+  tc->next = super->edges;
+  super->edges = tc;
+  tc->rx_backtrack_frame.inx = rx->instruction_table[rx_backtrack_point];
+  tc->rx_backtrack_frame.data = 0;
+  tc->rx_backtrack_frame.data_2 = (void *) tc;
+  tc->options = df;
+  tc->cset = (rx_Bitset) ((char *) tc + sizeof (*tc));
+  rx_bitset_assign (rx->local_cset_size, tc->cset, cset);
+  if (df)
+    {
+      struct rx_distinct_future * dfp = df;
+      df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      while (dfp)
+	{
+	  dfp->edge = tc;
+	  dfp = dfp->next_same_super_edge[0];
+	}
+      df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+    }
+  return tc;
+}
+
+
+/* There are three kinds of cache miss.  The first occurs when a
+ * transition is taken that has never been computed during the
+ * lifetime of the source superstate.  That cache miss is handled by
+ * calling COMPUTE_SUPER_EDGE.  The second kind of cache miss
+ * occurs when the destination superstate of a transition doesn't
+ * exist.  SOLVE_DESTINATION is used to construct the destination superstate.
+ * Finally, the third kind of cache miss occurs when the destination
+ * superstate of a transition is in a `semi-free state'.  That case is
+ * handled by UNFREE_SUPERSTATE.
+ *
+ * The function of HANDLE_CACHE_MISS is to figure out which of these
+ * cases applies.
+ */
+
+#ifdef __STDC__
+static void
+install_partial_transition  (struct rx_superstate *super,
+			     struct rx_inx *answer,
+			     RX_subset set, int offset)
+#else
+static void
+install_partial_transition  (super, answer, set, offset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     RX_subset set;
+     int offset;
+#endif
+{
+  int start = offset;
+  int end = start + 32;
+  RX_subset pos = 1;
+  struct rx_inx * transitions = super->transitions;
+  
+  while (start < end)
+    {
+      if (set & pos)
+	transitions[start] = *answer;
+      pos <<= 1;
+      ++start;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_inx *
+rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data) 
+#else
+RX_DECL struct rx_inx *
+rx_handle_cache_miss (rx, super, chr, data)
+     struct rx *rx;
+     struct rx_superstate *super;
+     unsigned char chr;
+     void *data;
+#endif
+{
+  int offset = chr / RX_subset_bits;
+  struct rx_distinct_future *df = data;
+
+  if (!df)			/* must be the shared_cache_miss_frame */
+    {
+      /* Perhaps this is just a transition waiting to be filled. */
+      struct rx_super_edge *tc;
+      RX_subset mask = rx_subset_singletons [chr % RX_subset_bits];
+
+      for (tc = super->edges; tc; tc = tc->next)
+	if (tc->cset[offset] & mask)
+	  {
+	    struct rx_inx * answer;
+	    df = tc->options;
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	    install_partial_transition (super, answer,
+					tc->cset [offset], offset * 32);
+	    return answer;
+	  }
+      /* Otherwise, it's a flushed or  newly encountered edge. */
+      {
+	char cset_space[1024];	/* this limit is far from unreasonable */
+	rx_Bitset trcset;
+	struct rx_inx *answer;
+
+	if (rx_sizeof_bitset (rx->local_cset_size) > sizeof (cset_space))
+	  return 0;		/* If the arbitrary limit is hit, always fail */
+				/* cleanly. */
+	trcset = (rx_Bitset)cset_space;
+	rx_lock_superstate (rx, super);
+	if (!compute_super_edge (rx, &df, trcset, super, chr))
+	  {
+	    rx_unlock_superstate (rx, super);
+	    return 0;
+	  }
+	if (!df)		/* We just computed the fail transition. */
+	  {
+	    static struct rx_inx
+	      shared_fail_frame = { 0, 0, (void *)rx_backtrack, 0 };
+	    answer = &shared_fail_frame;
+	  }
+	else
+	  {
+	    tc = rx_super_edge (rx, super, trcset, df);
+	    if (!tc)
+	      {
+		rx_unlock_superstate (rx, super);
+		return 0;
+	      }
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	  }
+	install_partial_transition (super, answer,
+				    trcset[offset], offset * 32);
+	rx_unlock_superstate (rx, super);
+	return answer;
+      }
+    }
+  else if (df->future) /* A cache miss on an edge with a future? Must be
+			* a semi-free destination. */
+    {				
+      if (df->future->is_semifree)
+	refresh_semifree_superstate (rx->cache, df->future);
+      return &df->future_frame;
+    }
+  else
+    /* no future superstate on an existing edge */
+    {
+      rx_lock_superstate (rx, super);
+      if (!solve_destination (rx, df))
+	{
+	  rx_unlock_superstate (rx, super);
+	  return 0;
+	}
+      if (!df->effects
+	  && (df->edge->options->next_same_super_edge[0] == df->edge->options))
+	install_partial_transition (super, &df->future_frame,
+				    df->edge->cset[offset], offset * 32);
+      rx_unlock_superstate (rx, super);
+      return &df->future_frame;
+    }
+}
+
+
+
+
+
+/* The rest of the code provides a regex.c compatable interface. */
+
+
+__const__ char *re_error_msg[] =
+{
+  0,						/* REG_NOUT */
+  "No match",					/* REG_NOMATCH */
+  "Invalid regular expression",			/* REG_BADPAT */
+  "Invalid collation character",		/* REG_ECOLLATE */
+  "Invalid character class name",		/* REG_ECTYPE */
+  "Trailing backslash",				/* REG_EESCAPE */
+  "Invalid back reference",			/* REG_ESUBREG */
+  "Unmatched [ or [^",				/* REG_EBRACK */
+  "Unmatched ( or \\(",				/* REG_EPAREN */
+  "Unmatched \\{",				/* REG_EBRACE */
+  "Invalid content of \\{\\}",			/* REG_BADBR */
+  "Invalid range end",				/* REG_ERANGE */
+  "Memory exhausted",				/* REG_ESPACE */
+  "Invalid preceding regular expression",	/* REG_BADRPT */
+  "Premature end of regular expression",	/* REG_EEND */
+  "Regular expression too big",			/* REG_ESIZE */
+  "Unmatched ) or \\)",				/* REG_ERPAREN */
+};
+
+
+
+
+/* 
+ * Macros used while compiling patterns.
+ *
+ * By convention, PEND points just past the end of the uncompiled pattern,
+ * P points to the read position in the pattern.  `translate' is the name
+ * of the translation table (`TRANSLATE' is the name of a macro that looks
+ * things up in `translate').
+ */
+
+
+/*
+ * Fetch the next character in the uncompiled pattern---translating it 
+ * if necessary. *Also cast from a signed character in the constant
+ * string passed to us by the user to an unsigned char that we can use
+ * as an array index (in, e.g., `translate').
+ */
+#define PATFETCH(c)							\
+ do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    c = translate[c];		 					\
+ } while (0)
+
+/* 
+ * Fetch the next character in the uncompiled pattern, with no
+ * translation.
+ */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+#define TRANSLATE(d) translate[(unsigned char) (d)]
+
+typedef unsigned regnum_t;
+
+/* Since offsets can go either forwards or backwards, this type needs to
+ * be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.
+ */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  struct rexp_node ** top_expression; /* was begalt */
+  struct rexp_node ** last_expression; /* was laststart */
+  pattern_offset_t inner_group_offset;
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / CHARBITS]               \
+   |= 1 << (((unsigned char) c) % CHARBITS))
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (isdigit (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (!strcmp (string, "alpha") || !strcmp (string, "upper")		\
+    || !strcmp (string, "lower") || !strcmp (string, "digit")		\
+    || !strcmp (string, "alnum") || !strcmp (string, "xdigit")		\
+    || !strcmp (string, "space") || !strcmp (string, "print")		\
+    || !strcmp (string, "punct") || !strcmp (string, "graph")		\
+    || !strcmp (string, "cntrl") || !strcmp (string, "blank"))
+
+
+
+/* These predicates are used in regex_compile. */
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+ * after an alternative or a begin-subexpression.  We assume there is at
+ * least one character before the ^.  
+ */
+
+#ifdef __STDC__
+static boolean
+at_begline_loc_p (__const__ char *pattern, __const__ char * p, reg_syntax_t syntax)
+#else
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+     __const__ char *pattern;
+     __const__ char * p;
+     reg_syntax_t syntax;
+#endif
+{
+  __const__ char *prev = p - 2;
+  boolean prev_prev_backslash = ((prev > pattern) && (prev[-1] == '\\'));
+  
+    return
+      
+      (/* After a subexpression?  */
+       ((*prev == '(') && ((syntax & RE_NO_BK_PARENS) || prev_prev_backslash))
+       ||
+       /* After an alternative?  */
+       ((*prev == '|') && ((syntax & RE_NO_BK_VBAR) || prev_prev_backslash))
+       );
+}
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+ * at least one character after the $, i.e., `P < PEND'.
+ */
+
+#ifdef __STDC__
+static boolean
+at_endline_loc_p (__const__ char *p, __const__ char *pend, int syntax)
+#else
+static boolean
+at_endline_loc_p (p, pend, syntax)
+     __const__ char *p;
+     __const__ char *pend;
+     int syntax;
+#endif
+{
+  __const__ char *next = p;
+  boolean next_backslash = (*next == '\\');
+  __const__ char *next_next = (p + 1 < pend) ? (p + 1) : 0;
+  
+  return
+    (
+     /* Before a subexpression?  */
+     ((syntax & RE_NO_BK_PARENS)
+      ? (*next == ')')
+      : (next_backslash && next_next && (*next_next == ')')))
+    ||
+     /* Before an alternative?  */
+     ((syntax & RE_NO_BK_VBAR)
+      ? (*next == '|')
+      : (next_backslash && next_next && (*next_next == '|')))
+     );
+}
+
+
+
+unsigned char rx_id_translation[256] =
+{
+  0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+ 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+ 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+ 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+ 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+ 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
+ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
+ 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
+
+ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
+ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249,
+ 250, 251, 252, 253, 254, 255
+};
+
+/* The compiler keeps an inverted translation table.
+ * This looks up/inititalize elements.
+ * VALID is an array of booleans that validate CACHE.
+ */
+
+#ifdef __STDC__
+static rx_Bitset
+inverse_translation (struct re_pattern_buffer * rxb,
+		     char * valid, rx_Bitset cache,
+		     unsigned char * translate, int c)
+#else
+static rx_Bitset
+inverse_translation (rxb, valid, cache, translate, c)
+     struct re_pattern_buffer * rxb;
+     char * valid;
+     rx_Bitset cache;
+     unsigned char * translate;
+     int c;
+#endif
+{
+  rx_Bitset cs
+    = cache + c * rx_bitset_numb_subsets (rxb->rx.local_cset_size); 
+
+  if (!valid[c])
+    {
+      int x;
+      int c_tr = TRANSLATE(c);
+      rx_bitset_null (rxb->rx.local_cset_size, cs);
+      for (x = 0; x < 256; ++x)	/* &&&& 13.37 */
+	if (TRANSLATE(x) == c_tr)
+	  RX_bitset_enjoin (cs, x);
+      valid[c] = 1;
+    }
+  return cs;
+}
+
+
+
+
+
+/* More subroutine declarations and macros for regex_compile.  */
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+#ifdef __STDC__
+static boolean
+group_in_compile_stack (compile_stack_type compile_stack, regnum_t regnum)
+#else
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+#endif
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/*
+ * Read the ending character of a range (in a bracket expression) from the
+ * uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+ * starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+ * Then we set the translation of all bits between the starting and
+ * ending characters (inclusive) in the compiled pattern B.
+ * 
+ * Return an error code.
+ * 
+ * We use these short variable names so we can use the same macros as
+ * `regex_compile' itself.  
+ */
+
+#ifdef __STDC__
+static reg_errcode_t
+compile_range (struct re_pattern_buffer * rxb, rx_Bitset cs,
+	       __const__ char ** p_ptr, __const__ char * pend,
+	       unsigned char * translate, reg_syntax_t syntax,
+	       rx_Bitset inv_tr,  char * valid_inv_tr)
+#else
+static reg_errcode_t
+compile_range (rxb, cs, p_ptr, pend, translate, syntax, inv_tr, valid_inv_tr)
+     struct re_pattern_buffer * rxb;
+     rx_Bitset cs;
+     __const__ char ** p_ptr;
+     __const__ char * pend;
+     unsigned char * translate;
+     reg_syntax_t syntax;
+     rx_Bitset inv_tr;
+     char * valid_inv_tr;
+#endif
+{
+  unsigned this_char;
+
+  __const__ char *p = *p_ptr;
+
+  unsigned char range_end;
+  unsigned char range_start = TRANSLATE(p[-2]);
+
+  if (p == pend)
+    return REG_ERANGE;
+
+  PATFETCH (range_end);
+
+  (*p_ptr)++;
+
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      rx_Bitset it =
+	inverse_translation (rxb, valid_inv_tr, inv_tr, translate, this_char);
+      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+    }
+  
+  return REG_NOERROR;
+}
+
+
+
+/* This searches a regexp for backreference side effects.
+ * It fills in the array OUT with 1 at the index of every register pair
+ * referenced by a backreference.
+ *
+ * This is used to help optimize patterns for searching.  The information is
+ * useful because, if the caller doesn't want register values, backreferenced
+ * registers are the only registers for which we need rx_backtrack.
+ */
+
+#ifdef __STDC__
+static void
+find_backrefs (char * out, struct rexp_node * rexp,
+	       struct re_se_params * params)
+#else
+static void
+find_backrefs (out, rexp, params)
+     char * out;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (rexp)
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return;
+      case r_alternate:
+      case r_concat:
+      case r_opt:
+      case r_star:
+      case r_2phase_star:
+	find_backrefs (out, rexp->params.pair.left, params);
+	find_backrefs (out, rexp->params.pair.right, params);
+	return;
+      case r_side_effect:
+	if (   ((long)rexp->params.side_effect >= 0)
+	    && (params [(long)rexp->params.side_effect].se == re_se_backref))
+	  out[ params [(long)rexp->params.side_effect].op1] = 1;
+	return;
+      }
+}
+
+
+
+
+/* Returns 0 unless the pattern can match the empty string. */
+
+#ifdef __STDC__
+static int
+compute_fastset (struct re_pattern_buffer * rxb, struct rexp_node * rexp)
+#else
+static int
+compute_fastset (rxb, rexp)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 1;
+  switch (rexp->type)
+    {
+    case r_data:
+      return 1;
+    case r_cset:
+      {
+	rx_bitset_union (rxb->rx.local_cset_size,
+			 rxb->fastset, rexp->params.cset);
+      }
+      return 0;
+    case r_concat:
+      return (compute_fastset (rxb, rexp->params.pair.left)
+	      && compute_fastset (rxb, rexp->params.pair.right));
+    case r_2phase_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      /* compute_fastset (rxb, rexp->params.pair.right);  nope... */
+      return 1;
+    case r_alternate:
+      return !!(compute_fastset (rxb, rexp->params.pair.left)
+		+ compute_fastset (rxb, rexp->params.pair.right));
+    case r_opt:
+    case r_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      return 1;
+    case r_side_effect:
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* returns
+ *  1 -- yes, definately anchored by the given side effect.
+ *  2 -- maybe anchored, maybe the empty string.
+ *  0 -- definately not anchored
+ *  There is simply no other possibility.
+ */
+
+#ifdef __STDC__
+static int
+is_anchored (struct rexp_node * rexp, rx_side_effect se)
+#else
+static int
+is_anchored (rexp, se)
+     struct rexp_node * rexp;
+     rx_side_effect se;
+#endif
+{
+  if (!rexp)
+    return 2;
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+    case r_concat:
+    case r_2phase_star:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	return (l == 2 ? is_anchored (rexp->params.pair.right, se) : l);
+      }
+    case r_alternate:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	int r = l ? is_anchored (rexp->params.pair.right, se) : 0;
+
+	if (l == r)
+	  return l;
+	else if ((l == 0) || (r == 0))
+	  return 0;
+	else
+	  return 2;
+      }
+    case r_opt:
+    case r_star:
+      return is_anchored (rexp->params.pair.left, se) ? 2 : 0;
+      
+    case r_side_effect:
+      return ((rexp->params.side_effect == se)
+	      ? 1 : 2);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This removes register assignments that aren't required by backreferencing.
+ * This can speed up explore_future, especially if it eliminates
+ * non-determinism in the superstate NFA.
+ * 
+ * NEEDED is an array of characters, presumably filled in by FIND_BACKREFS.
+ * The non-zero elements of the array indicate which register assignments
+ * can NOT be removed from the expression.
+ */
+
+#ifdef __STDC__
+static struct rexp_node *
+remove_unecessary_side_effects (struct rx * rx, char * needed,
+				struct rexp_node * rexp,
+				struct re_se_params * params)
+#else
+static struct rexp_node *
+remove_unecessary_side_effects (rx, needed, rexp, params)
+     struct rx * rx;
+     char * needed;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  struct rexp_node * l;
+  struct rexp_node * r;
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return rexp;
+      case r_alternate:
+      case r_concat:
+      case r_2phase_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	r = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.right, params);
+	if ((l && r) || (rexp->type != r_concat))
+	  {
+	    rexp->params.pair.left = l;
+	    rexp->params.pair.right = r;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = rexp->params.pair.right = 0;
+	    rx_free_rexp (rx, rexp);
+	    return l ? l : r;
+	  }
+      case r_opt:
+      case r_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	if (l)
+	  {
+	    rexp->params.pair.left = l;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = 0;
+	    rx_free_rexp (rx, rexp);
+	    return 0;
+	  }
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   (se >= 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen))
+	      && (params [se].op1 > 0)
+	      && (!needed [params [se].op1]))
+	    {
+	      rx_free_rexp (rx, rexp);
+	      return 0;
+	    }
+	  else
+	    return rexp;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+pointless_if_repeated (struct rexp_node * node, struct re_se_params * params)
+#else
+static int
+pointless_if_repeated (node, params)
+     struct rexp_node * node;
+     struct re_se_params * params;
+#endif
+{
+  if (!node)
+    return 1;
+  switch (node->type)
+    {
+    case r_cset:
+      return 0;
+    case r_alternate:
+    case r_concat:
+    case r_2phase_star:
+      return (pointless_if_repeated (node->params.pair.left, params)
+	      && pointless_if_repeated (node->params.pair.right, params));
+    case r_opt:
+    case r_star:
+      return pointless_if_repeated (node->params.pair.left, params);
+    case r_side_effect:
+      switch ((enum re_side_effects) (((long)node->params.side_effect < 0)
+	      ? (enum re_side_effects) (long) node->params.side_effect
+	      : (enum re_side_effects)params[(long)node->params.side_effect].se))
+	{
+	case re_se_try:
+	case re_se_at_dot:
+	case re_se_begbuf:
+	case re_se_hat:
+	case re_se_wordbeg:
+	case re_se_wordbound:
+	case re_se_notwordbound:
+	case re_se_wordend:
+	case re_se_endbuf:
+	case re_se_dollar:
+	case re_se_fail:
+	case re_se_win:
+	  return 1;
+	case re_se_lparen:
+	case re_se_rparen:
+	case re_se_iter:
+	case re_se_end_iter:
+	case re_se_syntax:
+	case re_se_not_syntax:
+	case re_se_backref:
+	  return 0;
+	}
+    case r_data:
+    default:
+      return 0;
+    }
+}
+
+
+
+
+#ifdef __STDC__
+static int
+registers_on_stack (struct re_pattern_buffer * rxb,
+		    struct rexp_node * rexp, int in_danger,
+		    struct re_se_params * params)
+#else
+static int
+registers_on_stack (rxb, rexp, in_danger, params)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+     int in_danger;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return 0;
+      case r_alternate:
+      case r_concat:
+	return (   registers_on_stack (rxb, rexp->params.pair.left,
+				       in_danger, params)
+		|| (registers_on_stack
+		    (rxb, rexp->params.pair.right,
+		     in_danger, params)));
+      case r_opt:
+	return registers_on_stack (rxb, rexp->params.pair.left, 0, params);
+      case r_star:
+	return registers_on_stack (rxb, rexp->params.pair.left, 1, params);
+      case r_2phase_star:
+	return
+	  (   registers_on_stack (rxb, rexp->params.pair.left, 1, params)
+	   || registers_on_stack (rxb, rexp->params.pair.right, 1, params));
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   in_danger
+	      && (se >= 0)
+	      && (params [se].op1 > 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen)))
+	    return 1;
+	  else
+	    return 0;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+static char idempotent_complex_se[] =
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     IDEM,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  23
+};
+
+static char idempotent_se[] =
+{
+  13,
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      IDEM,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  42
+};
+
+
+
+
+
+#ifdef __STDC__
+static int
+has_any_se (struct rx * rx,
+	    struct rexp_node * rexp)
+#else
+static int
+has_any_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return 1;
+      
+    case r_2phase_star:
+    case r_concat:
+    case r_alternate:
+      return
+	(   has_any_se (rx, rexp->params.pair.left)
+	 || has_any_se (rx, rexp->params.pair.right));
+
+    case r_opt:
+    case r_star:
+      return has_any_se (rx, rexp->params.pair.left);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This must be called AFTER `convert_hard_loops' for a given REXP. */
+#ifdef __STDC__
+static int
+has_non_idempotent_epsilon_path (struct rx * rx,
+				 struct rexp_node * rexp,
+				 struct re_se_params * params)
+#else
+static int
+has_non_idempotent_epsilon_path (rx, rexp, params)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_star:
+      return 0;
+
+    case r_side_effect:
+      return
+	!((long)rexp->params.side_effect > 0
+	  ? idempotent_complex_se [ params [(long)rexp->params.side_effect].se ]
+	  : idempotent_se [-(long)rexp->params.side_effect]);
+      
+    case r_alternate:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 || has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_2phase_star:
+    case r_concat:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 && has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_opt:
+      return has_non_idempotent_epsilon_path (rx,
+					      rexp->params.pair.left, params);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This computes rougly what it's name suggests.   It can (and does) go wrong 
+ * in the direction of returning spurious 0 without causing disasters.
+ */
+#ifdef __STDC__
+static int
+begins_with_complex_se (struct rx * rx, struct rexp_node * rexp)
+#else
+static int
+begins_with_complex_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return ((long)rexp->params.side_effect >= 0);
+      
+    case r_alternate:
+      return
+	(   begins_with_complex_se (rx, rexp->params.pair.left)
+	 && begins_with_complex_se (rx, rexp->params.pair.right));
+
+
+    case r_concat:
+      return has_any_se (rx, rexp->params.pair.left);
+    case r_opt:
+    case r_star:
+    case r_2phase_star:
+      return 0;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This destructively removes some of the re_se_tv side effects from 
+ * a rexp tree.  In particular, during parsing re_se_tv was inserted on the
+ * right half of every | to guarantee that posix path preference could be 
+ * honored.  This function removes some which it can be determined aren't 
+ * needed.  
+ */
+
+#ifdef __STDC__
+static void
+speed_up_alt (struct rx * rx,
+	      struct rexp_node * rexp,
+	      int unposix)
+#else
+static void
+speed_up_alt (rx, rexp, unposix)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     int unposix;
+#endif
+{
+  if (!rexp)
+    return;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_side_effect:
+      return;
+
+    case r_opt:
+    case r_star:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      return;
+
+    case r_2phase_star:
+    case r_concat:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right, unposix);
+      return;
+
+    case r_alternate:
+      /* the right child is guaranteed to be (concat re_se_tv <subexp>) */
+
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right->params.pair.right, unposix);
+      
+      if (   unposix
+	  || (begins_with_complex_se
+	      (rx, rexp->params.pair.right->params.pair.right))
+	  || !(   has_any_se (rx, rexp->params.pair.right->params.pair.right)
+	       || has_any_se (rx, rexp->params.pair.left)))
+	{
+	  struct rexp_node * conc = rexp->params.pair.right;
+	  rexp->params.pair.right = conc->params.pair.right;
+	  conc->params.pair.right = 0;
+	  rx_free_rexp (rx, conc);
+	}
+    }
+}
+
+
+
+
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is set to zero;
+     `re_nsub' is set to the number of groups in PATTERN;
+     `not_bol' and `not_eol' are set to zero.
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+
+
+#ifdef __STDC__
+RX_DECL reg_errcode_t
+rx_compile (__const__ char *pattern, int size,
+	    reg_syntax_t syntax,
+	    struct re_pattern_buffer * rxb) 
+#else
+RX_DECL reg_errcode_t
+rx_compile (pattern, size, syntax, rxb)
+     __const__ char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  RX_subset
+    inverse_translate [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+  char
+    validate_inv_tr [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random tempory spot in PATTERN.  */
+  __const__ char *p1;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  __const__ char *p = pattern;
+  __const__ char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  unsigned char *translate = (rxb->translate
+			      ? rxb->translate
+			      : rx_id_translation);
+
+  /* When parsing is done, this will hold the expression tree. */
+  struct rexp_node * rexp = 0;
+
+  /* In the midst of compilation, this holds onto the regexp 
+   * first parst while rexp goes on to aquire additional constructs.
+   */
+  struct rexp_node * orig_rexp = 0;
+  struct rexp_node * fewer_side_effects = 0;
+
+  /* This and top_expression are saved on the compile stack. */
+  struct rexp_node ** top_expression = &rexp;
+  struct rexp_node ** last_expression = top_expression;
+  
+  /* Parameter to `goto append_node' */
+  struct rexp_node * append;
+
+  /* Counts open-groups as they are encountered.  This is the index of the
+   * innermost group being compiled.
+   */
+  regnum_t regnum = 0;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+   * which to go back if the interval is invalid.  
+   */
+  __const__ char *beg_interval;
+
+  struct re_se_params * params = 0;
+  int paramc = 0;		/* How many complex side effects so far? */
+
+  rx_side_effect side;		/* param to `goto add_side_effect' */
+
+  bzero (validate_inv_tr, sizeof (validate_inv_tr));
+
+  rxb->rx.instruction_table = rx_id_instruction_table;
+
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack =  (( compile_stack_elt_t *) malloc ((INIT_COMPILE_STACK_SIZE) * sizeof ( compile_stack_elt_t)));
+  if (compile_stack.stack == 0)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  rxb->rx.cache = &default_cache;
+  rxb->syntax = syntax;
+  rxb->fastmap_accurate = 0;
+  rxb->not_bol = rxb->not_eol = 0;
+  rxb->least_subs = 0;
+  
+  /* Always count groups, whether or not rxb->no_sub is set.  
+   * The whole pattern is implicitly group 0, so counting begins
+   * with 1.
+   */
+  rxb->re_nsub = 0;
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_hat);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_dollar);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (pointless_if_repeated (*last_expression, params))
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                return REG_BADRPT;
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!last_expression)
+              break;
+
+	    /* Now we know whether or not zero matches is allowed
+	     * and also whether or not two or more matches is allowed.
+	     */
+
+	    {
+	      struct rexp_node * inner_exp = *last_expression;
+	      int need_sync = 0;
+
+	      if (many_times_ok
+		  && has_non_idempotent_epsilon_path (&rxb->rx,
+						      inner_exp, params))
+		{
+		  struct rexp_node * pusher
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushpos);
+		  struct rexp_node * checker
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_chkpos);
+		  struct rexp_node * pushback
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushback);
+		  rx_Bitset cs = rx_cset (&rxb->rx);
+		  struct rexp_node * lit_t = rx_mk_r_cset (&rxb->rx, cs);
+		  struct rexp_node * fake_state
+		    = rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		  struct rexp_node * phase2
+		    = rx_mk_r_concat (&rxb->rx, checker, fake_state);
+		  struct rexp_node * popper
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_poppos);
+		  struct rexp_node * star
+		    = rx_mk_r_2phase_star (&rxb->rx, inner_exp, phase2);
+		  struct rexp_node * a
+		    = rx_mk_r_concat (&rxb->rx, pusher, star);
+		  struct rexp_node * whole_thing
+		    = rx_mk_r_concat (&rxb->rx, a, popper);
+		  if (!(pusher && star && pushback && lit_t && fake_state
+			&& lit_t && phase2 && checker && popper
+			&& a && whole_thing))
+		    return REG_ESPACE;
+		  RX_bitset_enjoin (cs, 't');
+		  *last_expression = whole_thing;
+		}
+	      else
+		{
+		  struct rexp_node * star =
+		    (many_times_ok ? rx_mk_r_star : rx_mk_r_opt)
+		      (&rxb->rx, *last_expression);
+		  if (!star)
+		    return REG_ESPACE;
+		  *last_expression = star;
+		  need_sync = has_any_se (&rxb->rx, *last_expression);
+		}
+	      if (!zero_times_ok)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, inner_exp,
+				      rx_copy_rexp (&rxb->rx,
+						    *last_expression));
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		}
+	      if (need_sync)
+		{
+		  int sync_se = paramc;
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					sizeof (*params) * (1 + paramc)))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (*params))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+		  params [sync_se].se = re_se_tv;
+		  side = (rx_side_effect)sync_se;
+		  goto add_side_effect;
+		}
+	    }
+	    /* The old regex.c used to optimize `.*\n'.  
+	     * Maybe rx should too?
+	     */
+	  }
+	  break;
+
+
+	case '.':
+	  {
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * n = rx_mk_r_cset (&rxb->rx, cs);
+	    if (!(cs && n))
+	      return REG_ESPACE;
+
+	    rx_bitset_universe (rxb->rx.local_cset_size, cs);
+	    if (!(rxb->syntax & RE_DOT_NEWLINE))
+	      RX_bitset_remove (cs, '\n');
+	    if (!(rxb->syntax & RE_DOT_NOT_NULL))
+	      RX_bitset_remove (cs, 0);
+
+	    append = n;
+	    goto append_node;
+	    break;
+	  }
+
+
+        case '[':
+	  if (p == pend) return REG_EBRACK;
+          {
+            boolean had_char_class = false;
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * node = rx_mk_r_cset (&rxb->rx, cs);
+	    int is_inverted = *p == '^';
+	    
+	    if (!(node && cs))
+	      return REG_ESPACE;
+	    
+	    /* This branch of the switch is normally exited with
+	     *`goto append_node'
+	     */
+	    append = node;
+	    
+            if (is_inverted)
+	      p++;
+	    
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+	    
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) return REG_EBRACK;
+		
+                PATFETCH (c);
+		
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+		    
+                    PATFETCH (c1);
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c1);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                    continue;
+                  }
+		
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  goto finalize_class_and_append;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  return REG_ERANGE;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (rxb, cs, &p, pend, translate, syntax,
+				       inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+		    
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (rxb, cs, &p, pend, translate, syntax,
+					 inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                /* See if we're at the beginning of a possible character
+                   class.  */
+		
+		else if ((syntax & RE_CHAR_CLASSES)
+			 && (c == '[') && (*p == ':'))
+                  {
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+		    
+                    PATFETCH (c);
+                    c1 = 0;
+		    
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) return REG_EBRACK;
+		    
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+			  break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+		    
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = !strcmp (str, "alnum");
+                        boolean is_alpha = !strcmp (str, "alpha");
+                        boolean is_blank = !strcmp (str, "blank");
+                        boolean is_cntrl = !strcmp (str, "cntrl");
+                        boolean is_digit = !strcmp (str, "digit");
+                        boolean is_graph = !strcmp (str, "graph");
+                        boolean is_lower = !strcmp (str, "lower");
+                        boolean is_print = !strcmp (str, "print");
+                        boolean is_punct = !strcmp (str, "punct");
+                        boolean is_space = !strcmp (str, "space");
+                        boolean is_upper = !strcmp (str, "upper");
+                        boolean is_xdigit = !strcmp (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str)) return REG_ECTYPE;
+			
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+			
+                        if (p == pend) return REG_EBRACK;
+			
+                        for (ch = 0; ch < 1 << CHARBITS; ch++)
+                          {
+                            if (   (is_alnum  && isalnum (ch))
+                                || (is_alpha  && isalpha (ch))
+                                || (is_blank  && isblank (ch))
+                                || (is_cntrl  && iscntrl (ch))
+                                || (is_digit  && isdigit (ch))
+                                || (is_graph  && isgraph (ch))
+                                || (is_lower  && islower (ch))
+                                || (is_print  && isprint (ch))
+                                || (is_punct  && ispunct (ch))
+                                || (is_space  && isspace (ch))
+                                || (is_upper  && isupper (ch))
+                                || (is_xdigit && isxdigit (ch)))
+			      {
+				rx_Bitset it =
+				  inverse_translation (rxb, 
+						       validate_inv_tr,
+						       inverse_translate,
+						       translate,
+						       ch);
+				rx_bitset_union (rxb->rx.local_cset_size,
+						 cs, it);
+			      }
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 '[');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 ':');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                  }
+              }
+
+	  finalize_class_and_append:
+	    if (is_inverted)
+	      {
+		rx_bitset_complement (rxb->rx.local_cset_size, cs);
+		if (syntax & RE_HAT_LISTS_NOT_NEWLINE)
+		  RX_bitset_remove (cs, '\n');
+	      }
+	    goto append_node;
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+	  if ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+	    goto handle_interval;
+	  else
+	    goto normal_char;
+
+
+        case '\\':
+          if (p == pend) return REG_EESCAPE;
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              rxb->re_nsub++;
+              regnum++;
+              if (COMPILE_STACK_FULL)
+                { 
+                  ((compile_stack.stack) =
+		   (compile_stack_elt_t *) realloc (compile_stack.stack, ( compile_stack.size << 1) * sizeof (
+													      compile_stack_elt_t)));
+                  if (compile_stack.stack == 0) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, 0);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+
+              /*
+	       * These are the values to restore when we hit end of this
+               * group.  
+	       */
+	      COMPILE_STACK_TOP.top_expression = top_expression;
+	      COMPILE_STACK_TOP.last_expression = last_expression;
+              COMPILE_STACK_TOP.regnum = regnum;
+	      
+              compile_stack.avail++;
+	      
+	      top_expression = last_expression;
+	      break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+            handle_close:
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  return REG_ERPAREN;
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+		struct rexp_node ** inner = top_expression;
+
+                compile_stack.avail--;
+		top_expression = COMPILE_STACK_TOP.top_expression;
+		last_expression = COMPILE_STACK_TOP.last_expression;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		{
+		  int left_se = paramc;
+		  int right_se = paramc + 1;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 2) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (2 * sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  paramc += 2;
+
+		  params[left_se].se = re_se_lparen;
+		  params[left_se].op1 = this_group_regnum;
+		  params[right_se].se = re_se_rparen;
+		  params[right_se].op1 = this_group_regnum;
+		  {
+		    struct rexp_node * left
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)left_se);
+		    struct rexp_node * right
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)right_se);
+		    struct rexp_node * c1
+		      = (*inner
+			 ? rx_mk_r_concat (&rxb->rx, left, *inner) : left);
+		    struct rexp_node * c2
+		      = rx_mk_r_concat (&rxb->rx, c1, right);
+		    if (!(left && right && c1 && c2))
+		      return REG_ESPACE;
+		    *inner = c2;
+		  }
+		}
+		break;
+	      }
+
+            case '|':					/* `\|'.  */
+              if ((syntax & RE_LIMITED_OPS) || (syntax & RE_NO_BK_VBAR))
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+	      {
+		struct rexp_node * alt
+		  = rx_mk_r_alternate (&rxb->rx, *top_expression, 0);
+		if (!alt)
+		  return REG_ESPACE;
+		*top_expression = alt;
+		last_expression = &alt->params.pair.right;
+		{
+		  int sync_se = paramc;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 1) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+
+		  params[sync_se].se = re_se_tv;
+		  {
+		    struct rexp_node * sync
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)sync_se);
+		    struct rexp_node * conc
+		      = rx_mk_r_concat (&rxb->rx, sync, 0);
+
+		    if (!sync || !conc)
+		      return REG_ESPACE;
+
+		    *last_expression = conc;
+		    last_expression = &conc->params.pair.right;
+		  }
+		}
+	      }
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      return REG_EBRACE;
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') return REG_EBRACE;
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (pointless_if_repeated (*last_expression, params))
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      return REG_BADRPT;
+                    else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                      goto unfetch_interval;
+		    /* was: else laststart = b; */
+                  }
+
+                /* If the upper bound is zero, don't want to iterate
+                 * at all.
+		 */
+                 if (upper_bound == 0)
+		   {
+		     if (*last_expression)
+		       {
+			 rx_free_rexp (&rxb->rx, *last_expression);
+			 *last_expression = 0;
+		       }
+		   }
+		else
+		  /* Otherwise, we have a nontrivial interval. */
+		  {
+		    int iter_se = paramc;
+		    int end_se = paramc + 1;
+		    params = (params
+			      ? ((struct re_se_params *)
+				 realloc (params,
+					  sizeof (*params) * (2 + paramc)))
+			      : ((struct re_se_params *)
+				 malloc (2 * sizeof (*params))));
+		    if (!params)
+		      return REG_ESPACE;
+		    paramc += 2;
+		    params [iter_se].se = re_se_iter;
+		    params [iter_se].op1 = lower_bound;
+		    params[iter_se].op2 = upper_bound;
+
+		    params[end_se].se = re_se_end_iter;
+		    params[end_se].op1 = lower_bound;
+		    params[end_se].op2 = upper_bound;
+		    {
+		      struct rexp_node * push0
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_push0);
+		      struct rexp_node * start_one_iter
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)iter_se);
+		      struct rexp_node * phase1
+			= rx_mk_r_concat (&rxb->rx, start_one_iter,
+					  *last_expression);
+		      struct rexp_node * pushback
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_pushback);
+		      rx_Bitset cs = rx_cset (&rxb->rx);
+		      struct rexp_node * lit_t
+			= rx_mk_r_cset (&rxb->rx, cs);
+		      struct rexp_node * phase2
+			= rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		      struct rexp_node * loop
+			= rx_mk_r_2phase_star (&rxb->rx, phase1, phase2);
+		      struct rexp_node * push_n_loop
+			= rx_mk_r_concat (&rxb->rx, push0, loop);
+		      struct rexp_node * final_test
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)end_se);
+		      struct rexp_node * full_exp
+			= rx_mk_r_concat (&rxb->rx, push_n_loop, final_test);
+
+		      if (!(push0 && start_one_iter && phase1
+			    && pushback && lit_t && phase2
+			    && loop && push_n_loop && final_test && full_exp))
+			return REG_ESPACE;
+
+		      RX_bitset_enjoin(cs, 't');
+
+		      *last_expression = full_exp;
+		    }
+		  }
+                beg_interval = 0;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               p = beg_interval;
+               beg_interval = 0;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+	      side = (rx_side_effect)rx_se_at_dot;
+	      goto add_side_effect;
+              break;
+
+            case 's':
+	    case 'S':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * set = rx_mk_r_cset (&rxb->rx, cs);
+		if (!(cs && set))
+		  return REG_ESPACE;
+		if (c == 'S')
+		  rx_bitset_universe (rxb->rx.local_cset_size, cs);
+
+		PATFETCH (c);
+		{
+		  int x;
+		  enum syntaxcode code = syntax_spec_code [c];
+		  for (x = 0; x < 256; ++x)
+		    {
+		      
+		      if (SYNTAX (x) == code)
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, validate_inv_tr,
+						 inverse_translate,
+						 translate, x);
+			  rx_bitset_xor (rxb->rx.local_cset_size, cs, it);
+			}
+		    }
+		}
+		append = set;
+		goto append_node;
+	      }
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+            case 'W':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * n = (cs ? rx_mk_r_cset (&rxb->rx, cs) : 0);
+		if (!(cs && n))
+		  return REG_ESPACE;
+		if (c == 'W')
+		  rx_bitset_universe (rxb->rx.local_cset_size ,cs);
+		{
+		  int x;
+		  for (x = rxb->rx.local_cset_size - 1; x > 0; --x)
+		    if (SYNTAX(x) & Sword)
+		      RX_bitset_toggle (cs, x);
+		}
+		append = n;
+		goto append_node;
+	      }
+              break;
+
+/* With a little extra work, some of these side effects could be optimized
+ * away (basicly by looking at what we already know about the surrounding
+ * chars).  
+ */
+            case '<':
+	      side = (rx_side_effect)re_se_wordbeg;
+	      goto add_side_effect;
+              break;
+
+            case '>':
+              side = (rx_side_effect)re_se_wordend;
+	      goto add_side_effect;
+              break;
+
+            case 'b':
+              side = (rx_side_effect)re_se_wordbound;
+	      goto add_side_effect;
+              break;
+
+            case 'B':
+              side = (rx_side_effect)re_se_notwordbound;
+	      goto add_side_effect;
+              break;
+
+            case '`':
+	      side = (rx_side_effect)re_se_begbuf;
+	      goto add_side_effect;
+	      break;
+	      
+            case '\'':
+	      side = (rx_side_effect)re_se_endbuf;
+	      goto add_side_effect;
+              break;
+
+	    add_side_effect:
+	      {
+		struct rexp_node * se
+		  = rx_mk_r_side_effect (&rxb->rx, side);
+		if (!se)
+		  return REG_ESPACE;
+		append = se;
+		goto append_node;
+	      }
+	      break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                return REG_ESUBREG;
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+		return REG_ESUBREG;
+
+	      {
+		int backref_se = paramc;
+		params = (params
+			  ? ((struct re_se_params *)
+			     realloc (params,
+				      sizeof (*params) * (1 + paramc)))
+			  : ((struct re_se_params *)
+			     malloc (sizeof (*params))));
+		if (!params)
+		  return REG_ESPACE;
+		++paramc;
+		params[backref_se].se = re_se_backref;
+		params[backref_se].op1 = c1;
+		side = (rx_side_effect)backref_se;
+		goto add_side_effect;
+	      }
+              break;
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	    {
+	      rx_Bitset cs = rx_cset(&rxb->rx);
+	      struct rexp_node * match = rx_mk_r_cset (&rxb->rx, cs);
+	      rx_Bitset it;
+	      if (!(cs && match))
+		return REG_ESPACE;
+	      it = inverse_translation (rxb, validate_inv_tr,
+					inverse_translate, translate, c);
+	      rx_bitset_union (CHAR_SET_SIZE, cs, it);
+	      append = match;
+
+	    append_node:
+	      /* This genericly appends the rexp APPEND to *LAST_EXPRESSION
+	       * and then parses the next character normally.
+	       */
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, append);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+	      else
+		*last_expression = append;
+	    }
+	} /* switch (c) */
+    } /* while p != pend */
+
+  
+  {
+    int win_se = paramc;
+    params = (params
+	      ? ((struct re_se_params *)
+		 realloc (params,
+			  sizeof (*params) * (1 + paramc)))
+	      : ((struct re_se_params *)
+		 malloc (sizeof (*params))));
+    if (!params)
+      return REG_ESPACE;
+    ++paramc;
+    params[win_se].se = re_se_win;
+    {
+      struct rexp_node * se
+	= rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)win_se);
+      struct rexp_node * concat
+	= rx_mk_r_concat (&rxb->rx, rexp, se);
+      if (!(se && concat))
+	return REG_ESPACE;
+      rexp = concat;
+    }
+  }
+
+
+  /* Through the pattern now.  */
+
+  if (!COMPILE_STACK_EMPTY) 
+    return REG_EPAREN;
+
+      free (compile_stack.stack);
+
+  orig_rexp = rexp;
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      dbug_rxb = rxb;
+      fputs ("\n\nCompiling ", stdout);
+      fwrite (pattern, 1, size, stdout);
+      fputs (":\n", stdout);
+      rxb->se_params = params;
+      print_rexp (&rxb->rx, orig_rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    rx_Bitset cs = rx_cset(&rxb->rx);
+    rx_Bitset cs2 = rx_cset(&rxb->rx);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+    char * se_map = (char *) _alloca (paramc);
+#else
+    char * se_map = (char *) alloca (paramc);
+#endif
+    struct rexp_node * new_rexp = 0;
+
+
+    bzero (se_map, paramc);
+    find_backrefs (se_map, rexp, params);
+    fewer_side_effects =
+      remove_unecessary_side_effects (&rxb->rx, se_map,
+				      rx_copy_rexp (&rxb->rx, rexp), params);
+
+    speed_up_alt (&rxb->rx, rexp, 0);
+    speed_up_alt (&rxb->rx, fewer_side_effects, 1);
+
+    {
+      char * syntax_parens = rxb->syntax_parens;
+      if (syntax_parens == (char *)0x1)
+	rexp = remove_unecessary_side_effects
+	  (&rxb->rx, se_map, rexp, params);
+      else if (syntax_parens)
+	{
+	  int x;
+	  for (x = 0; x < paramc; ++x)
+	    if ((   (params[x].se == re_se_lparen)
+		 || (params[x].se == re_se_rparen))
+		&& (!syntax_parens [params[x].op1]))
+	      se_map [x] = 1;
+	  rexp = remove_unecessary_side_effects
+	    (&rxb->rx, se_map, rexp, params);
+	}
+    }
+
+    /* At least one more optimization would be nice to have here but i ran out 
+     * of time.  The idea would be to delay side effects.  
+     * For examle, `(abc)' is the same thing as `abc()' except that the
+     * left paren is offset by 3 (which we know at compile time).
+     * (In this comment, write that second pattern `abc(:3:)' 
+     * where `(:3:' is a syntactic unit.)
+     *
+     * Trickier:  `(abc|defg)'  is the same as `(abc(:3:|defg(:4:))'
+     * (The paren nesting may be hard to follow -- that's an alternation
+     *	of `abc(:3:' and `defg(:4:' inside (purely syntactic) parens
+     *  followed by the closing paren from the original expression.)
+     *
+     * Neither the expression tree representation nor the the nfa make
+     * this very easy to write. :(
+     */
+
+  /* What we compile is different than what the parser returns.
+   * Suppose the parser returns expression R.
+   * Let R' be R with unnecessary register assignments removed 
+   * (see REMOVE_UNECESSARY_SIDE_EFFECTS, above).
+   *
+   * What we will compile is the expression:
+   *
+   *    m{try}R{win}\|s{try}R'{win}
+   *
+   * {try} and {win} denote side effect epsilons (see EXPLORE_FUTURE).
+   * 
+   * When trying a match, we insert an `m' at the beginning of the 
+   * string if the user wants registers to be filled, `s' if not.
+   */
+    new_rexp =
+      rx_mk_r_alternate
+	(&rxb->rx,
+	 rx_mk_r_concat (&rxb->rx, rx_mk_r_cset (&rxb->rx, cs2), rexp),
+	 rx_mk_r_concat (&rxb->rx,
+			 rx_mk_r_cset (&rxb->rx, cs), fewer_side_effects));
+
+    if (!(new_rexp && cs && cs2))
+      return REG_ESPACE;
+    RX_bitset_enjoin (cs2, '\0'); /* prefixed to the rexp used for matching. */
+    RX_bitset_enjoin (cs, '\1'); /* prefixed to the rexp used for searching. */
+    rexp = new_rexp;
+  }
+
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      fputs ("\n...which is compiled as:\n", stdout);
+      print_rexp (&rxb->rx, rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    struct rx_nfa_state *start = 0;
+    struct rx_nfa_state *end = 0;
+
+    if (!rx_build_nfa (&rxb->rx, rexp, &start, &end))
+      return REG_ESPACE;	/*  */
+    else
+      {
+	void * mem = (void *)rxb->buffer;
+	unsigned long size = rxb->allocated;
+	int start_id;
+	char * perm_mem;
+	int iterator_size = paramc * sizeof (params[0]);
+
+	end->is_final = 1;
+	start->is_start = 1;
+	rx_name_nfa_states (&rxb->rx);
+	start_id = start->id;
+#ifdef RX_DEBUG
+	if (rx_debug_compile)
+	  {
+	    fputs ("...giving the NFA: \n", stdout);
+	    dbug_rxb = rxb;
+	    print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	  }
+#endif
+	if (!rx_eclose_nfa (&rxb->rx))
+	  return REG_ESPACE;
+	else
+	  {
+	    rx_delete_epsilon_transitions (&rxb->rx);
+	    
+	    /* For compatability reasons, we need to shove the
+	     * compiled nfa into one chunk of malloced memory.
+	     */
+	    rxb->rx.reserved = (   sizeof (params[0]) * paramc
+				+  rx_sizeof_bitset (rxb->rx.local_cset_size));
+#ifdef RX_DEBUG
+	    if (rx_debug_compile)
+	      {
+		dbug_rxb = rxb;
+		fputs ("...which cooks down (uncompactified) to: \n", stdout);
+		print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	      }
+#endif
+	    if (!rx_compactify_nfa (&rxb->rx, &mem, &size))
+	      return REG_ESPACE;
+	    rxb->buffer = mem;
+	    rxb->allocated = size;
+	    rxb->rx.buffer = mem;
+	    rxb->rx.allocated = size;
+	    perm_mem = ((char *)rxb->rx.buffer
+			+ rxb->rx.allocated - rxb->rx.reserved);
+	    rxb->se_params = ((struct re_se_params *)perm_mem);
+	    bcopy (params, rxb->se_params, iterator_size);
+	    perm_mem += iterator_size;
+	    rxb->fastset = (rx_Bitset) perm_mem;
+	    rxb->start = rx_id_to_nfa_state (&rxb->rx, start_id);
+	  }
+	rx_bitset_null (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->can_match_empty = compute_fastset (rxb, orig_rexp);
+	rxb->match_regs_on_stack =
+	  registers_on_stack (rxb, orig_rexp, 0, params); 
+	rxb->search_regs_on_stack =
+	  registers_on_stack (rxb, fewer_side_effects, 0, params);
+	if (rxb->can_match_empty)
+	  rx_bitset_universe (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->is_anchored = is_anchored (orig_rexp, (rx_side_effect) re_se_hat);
+	rxb->begbuf_only = is_anchored (orig_rexp,
+					(rx_side_effect) re_se_begbuf);
+      }
+    rx_free_rexp (&rxb->rx, rexp);
+    if (params)
+      free (params);
+#ifdef RX_DEBUG
+    if (rx_debug_compile)
+      {
+	dbug_rxb = rxb;
+	fputs ("...which cooks down to: \n", stdout);
+	print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+      }
+#endif
+  }
+  return REG_NOERROR;
+}
+
+
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.  */
+
+__const__ char * rx_error_msg[] =
+{ 0,						/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+};
+
+
+
+
+
+char rx_slowmap [256] =
+{
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+};
+
+#ifdef __STDC__
+RX_DECL void
+rx_blow_up_fastmap (struct re_pattern_buffer * rxb)
+#else
+RX_DECL void
+rx_blow_up_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  int x;
+  for (x = 0; x < 256; ++x)	/* &&&& 3.6 % */
+    rxb->fastmap [x] = !!RX_bitset_member (rxb->fastset, x);
+  rxb->fastmap_accurate = 1;
+}
+
+
+
+
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#define RE_SEARCH_2_FN	inner_re_search_2
+#define RE_S2_QUAL static
+#else
+#define RE_SEARCH_2_FN	re_search_2
+#define RE_S2_QUAL 
+#endif
+
+struct re_search_2_closure
+{
+  __const__ char * string1;
+  int size1;
+  __const__ char * string2;
+  int size2;
+};
+
+
+static __inline__ enum rx_get_burst_return
+re_search_2_get_burst (pos, vclosure, stop)
+     struct rx_string_position * pos;
+     void * vclosure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)vclosure;
+  if (!closure->string2)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      if ((inset < -1) || (inset > closure->size1))
+	return rx_get_burst_no_more;
+      else
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+  else if (!closure->string1)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      pos->pos = (__const__ unsigned char *) closure->string2 + inset;
+      pos->string = (__const__ unsigned char *) closure->string2;
+      pos->size = closure->size2;
+      pos->end = ((__const__ unsigned char *)
+		  MIN(closure->string2 + closure->size2,
+		      closure->string2 + stop));
+      pos->offset = 0;
+      return ((pos->pos < pos->end)
+	      ? rx_get_burst_ok
+	      :  rx_get_burst_no_more);
+    }
+  else
+    {
+      int inset;
+
+      inset = pos->pos - pos->string + pos->offset;
+      if (inset < closure->size1)
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return rx_get_burst_ok;
+	}
+      else
+	{
+	  pos->pos = ((__const__ unsigned char *)
+		      closure->string2 + inset - closure->size1);
+	  pos->string = (__const__ unsigned char *) closure->string2;
+	  pos->size = closure->size2;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string2 + closure->size2,
+			  closure->string2 + stop - closure->size1));
+	  pos->offset = closure->size1;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+}
+
+
+static __inline__ enum rx_back_check_return
+re_search_2_back_check (pos, lparen, rparen, translate, vclosure, stop)
+     struct rx_string_position * pos;
+     int lparen;
+     int rparen;
+     unsigned char * translate;
+     void * vclosure;
+     int stop;
+{
+  struct rx_string_position there;
+  struct rx_string_position past;
+
+  there = *pos;
+  there.pos = there.string + lparen - there.offset;
+  re_search_2_get_burst (&there, vclosure, stop);
+
+  past = *pos;
+  past.pos = past.string + rparen - there.offset;
+  re_search_2_get_burst (&past, vclosure, stop);
+
+  ++pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+
+  while (   (there.pos != past.pos)
+	 && (pos->pos != pos->end))
+    if (TRANSLATE(*there.pos) != TRANSLATE(*pos->pos))
+      return rx_back_check_fail;
+    else
+      {
+	++there.pos;
+	++pos->pos;
+	if (there.pos == there.end)
+	  re_search_2_get_burst (&there, vclosure, stop);
+	if (pos->pos == pos->end)
+	  re_search_2_get_burst (pos, vclosure, stop);
+      }
+
+  if (there.pos != past.pos)
+    return rx_back_check_fail;
+  --pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+  return rx_back_check_pass;
+}
+
+static __inline__ int
+re_search_2_fetch_char (pos, offset, app_closure, stop)
+     struct rx_string_position * pos;
+     int offset;
+     void * app_closure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)app_closure;
+  if (offset == 0)
+    {
+      if (pos->pos >= pos->string)
+	return *pos->pos;
+      else
+	{
+	  if (   (pos->string == (__const__ unsigned char *) closure->string2)
+	      && (closure->string1)
+	      && (closure->size1))
+	    return closure->string1[closure->size1 - 1];
+	  else
+	    return 0;		/* sure, why not. */
+	}
+    }
+  if (pos->pos == pos->end)
+    return *closure->string2;
+  else
+    return pos->pos[1];
+}
+     
+
+#ifdef __STDC__
+RE_S2_QUAL int
+RE_SEARCH_2_FN (struct re_pattern_buffer *rxb,
+		__const__ char * string1, int size1,
+		__const__ char * string2, int size2,
+		int startpos, int range,
+		struct re_registers *regs,
+		int stop)
+#else
+RE_S2_QUAL int
+RE_SEARCH_2_FN (rxb,
+		string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int answer;
+  struct re_search_2_closure closure;
+  closure.string1 = string1;
+  closure.size1 = size1;
+  closure.string2 = string2;
+  closure.size2 = size2;
+  answer = rx_search (rxb, startpos, range, stop, size1 + size2,
+		      re_search_2_get_burst,
+		      re_search_2_back_check,
+		      re_search_2_fetch_char,
+		      (void *)&closure,
+		      regs,
+		      0,
+		      0);
+  switch (answer)
+    {
+    case rx_search_continuation:
+      abort ();
+    case rx_search_error:
+      return -2;
+    case rx_search_soft_fail:
+    case rx_search_fail:
+      return -1;
+    default:
+      return answer;
+    }
+}
+
+/* Export rx_search to callers outside this file.  */
+
+int
+re_rx_search (rxb, startpos, range, stop, total_size,
+	      get_burst, back_check, fetch_char,
+	      app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+{
+  return rx_search (rxb, startpos, range, stop, total_size,
+		    get_burst, back_check, fetch_char, app_closure,
+		    regs, resume_state, save_state);
+}
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#ifdef __STDC__
+int
+re_search_2 (struct re_pattern_buffer *rxb,
+	     __const__ char * string1, int size1,
+	     __const__ char * string2, int size2,
+	     int startpos, int range,
+	     struct re_registers *regs,
+	     int stop)
+#else
+int
+re_search_2 (rxb, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int ret;
+  ret = inner_re_search_2 (rxb, string1, size1, string2, size2, startpos,
+			   range, regs, stop);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return ret;
+}
+#endif
+
+
+/* Like re_search_2, above, but only one string is specified, and
+ * doesn't let you say where to stop matching.
+ */
+
+#ifdef __STDC__
+int
+re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+	   int size, int startpos, int range,
+	   struct re_registers *regs)
+#else
+int
+re_search (rxb, string, size, startpos, range, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string;
+     int size;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+#endif
+{
+  return re_search_2 (rxb, 0, 0, string, size, startpos, range, regs, size);
+}
+
+#ifdef __STDC__
+int
+re_match_2 (struct re_pattern_buffer * rxb,
+	    __const__ char * string1, int size1,
+	    __const__ char * string2, int size2,
+	    int pos, struct re_registers *regs, int stop)
+#else
+int
+re_match_2 (rxb, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  struct re_registers some_regs;
+  regoff_t start;
+  regoff_t end;
+  int srch;
+  int save = rxb->regs_allocated;
+  struct re_registers * regs_to_pass = regs;
+
+  if (!regs)
+    {
+      some_regs.start = &start;
+      some_regs.end = &end;
+      some_regs.num_regs = 1;
+      regs_to_pass = &some_regs;
+      rxb->regs_allocated = REGS_FIXED;
+    }
+
+  srch = re_search_2 (rxb, string1, size1, string2, size2,
+		      pos, 1, regs_to_pass, stop);
+  if (regs_to_pass != regs)
+    rxb->regs_allocated = save;
+  if (srch < 0)
+    return srch;
+  return regs_to_pass->end[0] - regs_to_pass->start[0];
+}
+
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+#ifdef __STDC__
+int
+re_match (struct re_pattern_buffer * rxb,
+	  __const__ char * string,
+	  int size, int pos,
+	  struct re_registers *regs)
+#else
+int
+re_match (rxb, string, size, pos, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char *string;
+     int size;
+     int pos;
+     struct re_registers *regs;
+#endif
+{
+  return re_match_2 (rxb, string, size, 0, 0, pos, regs, size);
+}
+
+
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+reg_syntax_t re_syntax_options = RE_SYNTAX_EMACS;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+#ifdef __STDC__
+reg_syntax_t
+re_set_syntax (reg_syntax_t syntax)
+#else
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+#endif
+{
+  reg_syntax_t ret = re_syntax_options;
+
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+#ifdef __STDC__
+void
+re_set_registers (struct re_pattern_buffer *bufp,
+		  struct re_registers *regs,
+		  unsigned num_regs,
+		  regoff_t * starts, regoff_t * ends)
+#else
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+     struct re_pattern_buffer *bufp;
+     struct re_registers *regs;
+     unsigned num_regs;
+     regoff_t * starts;
+     regoff_t * ends;
+#endif
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t) 0;
+    }
+}
+
+
+
+
+
+#ifdef __STDC__
+static int 
+cplx_se_sublist_len (struct rx_se_list * list)
+#else
+static int 
+cplx_se_sublist_len (list)
+     struct rx_se_list * list;
+#endif
+{
+  int x = 0;
+  while (list)
+    {
+      if ((long)list->car >= 0)
+	++x;
+      list = list->cdr;
+    }
+  return x;
+}
+
+
+/* For rx->se_list_cmp */
+
+#ifdef __STDC__
+static int 
+posix_se_list_order (struct rx * rx,
+		     struct rx_se_list * a, struct rx_se_list * b)
+#else
+static int 
+posix_se_list_order (rx, a, b)
+     struct rx * rx;
+     struct rx_se_list * a;
+     struct rx_se_list * b;
+#endif
+{
+  int al = cplx_se_sublist_len (a);
+  int bl = cplx_se_sublist_len (b);
+
+  if (!al && !bl)
+    return ((a == b)
+	    ? 0
+	    : ((a < b) ? -1 : 1));
+  
+  else if (!al)
+    return -1;
+
+  else if (!bl)
+    return 1;
+
+  else
+    {
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+      rx_side_effect * av = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (bl + 1)));
+#else
+      rx_side_effect * av = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (bl + 1)));
+#endif
+      struct rx_se_list * ap = a;
+      struct rx_se_list * bp = b;
+      int ai, bi;
+      
+      for (ai = al - 1; ai >= 0; --ai)
+	{
+	  while ((long)ap->car < 0)
+	    ap = ap->cdr;
+	  av[ai] = ap->car;
+	  ap = ap->cdr;
+	}
+      av[al] = (rx_side_effect)-2;
+      for (bi = bl - 1; bi >= 0; --bi)
+	{
+	  while ((long)bp->car < 0)
+	    bp = bp->cdr;
+	  bv[bi] = bp->car;
+	  bp = bp->cdr;
+	}
+      bv[bl] = (rx_side_effect)-1;
+
+      {
+	int ret;
+	int x = 0;
+	while (av[x] == bv[x])
+	  ++x;
+ 	ret = (((unsigned *)(av[x]) < (unsigned *)(bv[x])) ? -1 : 1);
+	return ret;
+      }
+    }
+}
+
+
+
+
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in RXB.
+   Returns 0 if the pattern was valid, otherwise an error string.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in RXB on entry.
+
+   We call rx_compile to do the actual compilation.  */
+
+#ifdef __STDC__
+__const__ char *
+re_compile_pattern (__const__ char *pattern,
+		    int length,
+		    struct re_pattern_buffer * rxb)
+#else
+__const__ char *
+re_compile_pattern (pattern, length, rxb)
+     __const__ char *pattern;
+     int length;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  reg_errcode_t ret;
+
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  rxb->regs_allocated = REGS_UNALLOCATED;
+
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  rxb->no_sub = 0;
+
+  rxb->rx.local_cset_size = 256;
+
+  /* Match anchors at newline.  */
+  rxb->newline_anchor = 1;
+ 
+  rxb->re_nsub = 0;
+  rxb->start = 0;
+  rxb->se_params = 0;
+  rxb->rx.nodec = 0;
+  rxb->rx.epsnodec = 0;
+  rxb->rx.instruction_table = 0;
+  rxb->rx.nfa_states = 0;
+  rxb->rx.se_list_cmp = posix_se_list_order;
+  rxb->rx.start_set = 0;
+
+  ret = rx_compile (pattern, length, re_syntax_options, rxb);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return rx_error_msg[(int) ret];
+}
+
+
+
+#ifdef __STDC__
+int
+re_compile_fastmap (struct re_pattern_buffer * rxb)
+#else
+int
+re_compile_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  rx_blow_up_fastmap (rxb);
+  return 0;
+}
+
+
+
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them if this is an Emacs or POSIX compilation.  */
+
+#if (!defined (emacs) && !defined (_POSIX_SOURCE)) || defined(USE_BSD_REGEX)
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer rx_comp_buf;
+
+#ifdef __STDC__
+char *
+re_comp (__const__ char *s)
+#else
+char *
+re_comp (s)
+    __const__ char *s;
+#endif
+{
+  reg_errcode_t ret;
+
+  if (!s || (*s == '\0'))
+    {
+      if (!rx_comp_buf.buffer)
+	return "No previous regular expression";
+      return 0;
+    }
+
+  if (!rx_comp_buf.fastmap)
+    {
+      rx_comp_buf.fastmap = (char *) malloc (1 << CHARBITS);
+      if (!rx_comp_buf.fastmap)
+	return "Memory exhausted";
+    }
+
+  /* Since `rx_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  rx_comp_buf.newline_anchor = 1;
+
+  rx_comp_buf.re_nsub = 0;
+  rx_comp_buf.start = 0;
+  rx_comp_buf.se_params = 0;
+  rx_comp_buf.rx.nodec = 0;
+  rx_comp_buf.rx.epsnodec = 0;
+  rx_comp_buf.rx.instruction_table = 0;
+  rx_comp_buf.rx.nfa_states = 0;
+  rx_comp_buf.rx.start = 0;
+  rx_comp_buf.rx.se_list_cmp = posix_se_list_order;
+  rx_comp_buf.rx.local_cset_size = 256;
+
+  ret = rx_compile (s, strlen (s), re_syntax_options, &rx_comp_buf);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* Yes, we're discarding `__const__' here.  */
+  return (char *) rx_error_msg[(int) ret];
+}
+
+
+#ifdef __STDC__
+int
+re_exec (__const__ char *s)
+#else
+int
+re_exec (s)
+    __const__ char *s;
+#endif
+{
+  __const__ int len = strlen (s);
+  return
+    0 <= re_search (&rx_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* not emacs and not _POSIX_SOURCE */
+
+
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#if !defined(emacs)
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+
+#ifdef __STDC__
+int
+regcomp (regex_t * preg, __const__ char * pattern, int cflags)
+#else
+int
+regcomp (preg, pattern, cflags)
+    regex_t * preg;
+    __const__ char * pattern;
+    int cflags;
+#endif
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = cflags & REG_EXTENDED ? RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->fastmap = malloc (256);
+  if (!preg->fastmap)
+    return REG_ESPACE;
+  preg->fastmap_accurate = 0;
+
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+
+      preg->translate = (unsigned char *) malloc (256);
+      if (!preg->translate)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = isupper (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = 0;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we
+     can use strlen here in compiling the pattern.  */
+  preg->re_nsub = 0;
+  preg->start = 0;
+  preg->se_params = 0;
+  preg->syntax_parens = 0;
+  preg->rx.nodec = 0;
+  preg->rx.epsnodec = 0;
+  preg->rx.instruction_table = 0;
+  preg->rx.nfa_states = 0;
+  preg->rx.local_cset_size = 256;
+  preg->rx.start = 0;
+  preg->rx.se_list_cmp = posix_se_list_order;
+  preg->rx.start_set = 0;
+  ret = rx_compile (pattern, strlen (pattern), syntax, preg);
+
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+#ifdef __STDC__
+int
+regexec (__const__ regex_t *preg, __const__ char *string,
+	 size_t nmatch, regmatch_t pmatch[],
+	 int eflags)
+#else
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    __const__ regex_t *preg;
+    __const__ char *string;
+    size_t nmatch;
+    regmatch_t pmatch[];
+    int eflags;
+#endif
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+
+  /* The user has told us exactly how many registers to return
+   * information about, via `nmatch'.  We have to pass that on to the
+   * matching routines.
+   */
+  private_preg.regs_allocated = REGS_FIXED;
+
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      regs.end =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      if (regs.start == 0 || regs.end == 0)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg,
+		   string, len,
+                   /* start: */ 0,
+		   /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   */
+
+#ifdef __STDC__
+size_t
+regerror (int errcode, __const__ regex_t *preg,
+	  char *errbuf, size_t errbuf_size)
+#else
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    __const__ regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+#endif
+{
+  __const__ char *msg
+    = rx_error_msg[errcode] == 0 ? "Success" : rx_error_msg[errcode];
+  size_t msg_size = strlen (msg) + 1; /* Includes the 0.  */
+
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+#ifdef __STDC__
+void
+regfree (regex_t *preg)
+#else
+void
+regfree (preg)
+    regex_t *preg;
+#endif
+{
+  if (preg->buffer != 0)
+    free (preg->buffer);
+  preg->buffer = 0;
+  preg->allocated = 0;
+
+  if (preg->fastmap != 0)
+    free (preg->fastmap);
+  preg->fastmap = 0;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != 0)
+    free (preg->translate);
+  preg->translate = 0;
+}
+
+#endif /* not emacs  */
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/mg/lib/rx.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/rx.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/rx.h	(revision 21112)
@@ -0,0 +1,3755 @@
+#if !defined(RXH) || defined(RX_WANT_SE_DEFS)
+#define RXH
+
+/*	Copyright (C) 1992, 1993 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+/*  t. lord	Wed Sep 23 18:20:57 1992	*/
+
+
+
+
+
+
+
+
+
+#ifndef RX_WANT_SE_DEFS
+
+/* This page: Bitsets */
+
+#ifndef RX_subset
+typedef unsigned int RX_subset;
+#define RX_subset_bits	(32)
+#define RX_subset_mask	(RX_subset_bits - 1)
+#endif
+
+typedef RX_subset * rx_Bitset;
+
+#ifdef __STDC__
+typedef void (*rx_bitset_iterator) (void *, int member_index);
+#else
+typedef void (*rx_bitset_iterator) ();
+#endif
+
+#define rx_bitset_subset(N)  ((N) / RX_subset_bits)
+#define rx_bitset_subset_val(B,N)  ((B)[rx_bitset_subset(N)])
+#define RX_bitset_access(B,N,OP) \
+  ((B)[rx_bitset_subset(N)] OP rx_subset_singletons[(N) & RX_subset_mask])
+#define RX_bitset_member(B,N)   RX_bitset_access(B, N, &)
+#define RX_bitset_enjoin(B,N)   RX_bitset_access(B, N, |=)
+#define RX_bitset_remove(B,N)   RX_bitset_access(B, N, &= ~)
+#define RX_bitset_toggle(B,N)   RX_bitset_access(B, N, ^= )
+#define rx_bitset_numb_subsets(N) (((N) + RX_subset_bits - 1) / RX_subset_bits)
+#define rx_sizeof_bitset(N)	(rx_bitset_numb_subsets(N) * sizeof(RX_subset))
+
+
+
+
+/* This page: Splay trees. */
+
+#ifdef __STDC__
+typedef int (*rx_sp_comparer) (void * a, void * b);
+#else
+typedef int (*rx_sp_comparer) ();
+#endif
+
+struct rx_sp_node 
+{
+  void * key;
+  void * data;
+  struct rx_sp_node * kids[2];
+};
+
+#ifdef __STDC__
+typedef void (*rx_sp_key_data_freer) (struct rx_sp_node *);
+#else
+typedef void (*rx_sp_key_data_freer) ();
+#endif
+
+
+
+/* giant inflatable hash trees */
+
+struct rx_hash_item
+{
+  struct rx_hash_item * next_same_hash;
+  struct rx_hash * table;
+  unsigned long hash;
+  void * data;
+  void * binding;
+};
+
+struct rx_hash
+{
+  struct rx_hash * parent;
+  int refs;
+  struct rx_hash * children[13];
+  struct rx_hash_item * buckets [13];
+  int bucket_size [13];
+};
+
+struct rx_hash_rules;
+
+#ifdef __STDC__
+/* should return like == */
+typedef int (*rx_hash_eq)(void *, void *);
+typedef struct rx_hash * (*rx_alloc_hash)(struct rx_hash_rules *);
+typedef void (*rx_free_hash)(struct rx_hash *,
+			    struct rx_hash_rules *);
+typedef struct rx_hash_item * (*rx_alloc_hash_item)(struct rx_hash_rules *,
+						    void *);
+typedef void (*rx_free_hash_item)(struct rx_hash_item *,
+				 struct rx_hash_rules *);
+#else
+typedef int (*rx_hash_eq)();
+typedef struct rx_hash * (*rx_alloc_hash)();
+typedef void (*rx_free_hash)();
+typedef struct rx_hash_item * (*rx_alloc_hash_item)();
+typedef void (*rx_free_hash_item)();
+#endif
+
+struct rx_hash_rules
+{
+  rx_hash_eq eq;
+  rx_alloc_hash hash_alloc;
+  rx_free_hash free_hash;
+  rx_alloc_hash_item hash_item_alloc;
+  rx_free_hash_item free_hash_item;
+};
+
+
+
+/* Forward declarations */
+
+struct rx_cache;
+struct rx_superset;
+struct rx;
+struct rx_se_list;
+
+
+
+
+/* 
+ * GLOSSARY
+ *
+ * regexp
+ * regular expression
+ * expression
+ * pattern - a `regular' expression.  The expression
+ *       need not be formally regular -- it can contain
+ *       constructs that don't correspond to purely regular
+ *       expressions.
+ *
+ * buffer
+ * string - the string (or strings) being searched or matched.
+ *
+ * pattern buffer - a structure of type `struct re_pattern_buffer'
+ *       This in turn contains a `struct rx', which holds the
+ *       NFA compiled from a pattern, as well as some of the state
+ *       of a matcher using the pattern.
+ *
+ * NFA - nondeterministic finite automata.  Some people
+ *       use this term to a member of the class of 
+ *       regular automata (those corresponding to a regular
+ *       language).  However, in this code, the meaning is
+ *       more general.  The automata used by Rx are comperable
+ *       in power to what are usually called `push down automata'.
+ *
+ *       Two NFA are built by rx for every pattern.  One is built
+ *       by the compiler.  The other is built from the first, on
+ *       the fly, by the matcher.  The latter is called the `superstate
+ *       NFA' because its states correspond to sets of states from
+ *       the first NFA.  (Joe Keane gets credit for the name
+ *       `superstate NFA').
+ *
+ * NFA edges
+ * epsilon edges
+ * side-effect edges - The NFA compiled from a pattern can have three
+ *       kinds of edges.  Epsilon edges can be taken freely anytime
+ *       their source state is reached.  Character set edges can be
+ *       taken when their source state is reached and when the next 
+ *       character in the buffer is a member of the set.  Side effect
+ *       edges imply a transition that can only be taken after the
+ *       indicated side effect has been successfully accomplished.
+ *       Some examples of side effects are:
+ *
+ *		Storing the current match position to record the
+ *              location of a parentesized subexpression.
+ *
+ *              Advancing the matcher over N characters if they
+ *              match the N characters previously matched by a 
+ *              parentesized subexpression.
+ *
+ *       Both of those kinds of edges occur in the NFA generated
+ *       by the pattern:  \(.\)\1
+ *
+ *       Epsilon and side effect edges are similar.  Unfortunately,
+ *       some of the code uses the name `epsilon edge' to mean
+ *       both epsilon and side effect edges.  For example,  the
+ *       function has_non_idempotent_epsilon_path computes the existance
+ *       of a non-trivial path containing only a mix of epsilon and
+ *       side effect edges.  In that case `nonidempotent epsilon' is being
+ *       used to mean `side effect'.
+ */
+
+
+
+
+
+
+/* LOW LEVEL PATTERN BUFFERS */
+
+/* Suppose that from some NFA state, more than one path through
+ * side-effect edges is possible.  In what order should the paths
+ * be tried?  A function of type rx_se_list_order answers that
+ * question.  It compares two lists of side effects, and says
+ * which list comes first.
+ */
+ 
+#ifdef __STDC__
+typedef int (*rx_se_list_order) (struct rx *,
+				 struct rx_se_list *, 
+				 struct rx_se_list *);
+#else
+typedef int (*rx_se_list_order) ();
+#endif
+
+
+
+/* Struct RX holds a compiled regular expression - that is, an nfa
+ * ready to be converted on demand to a more efficient superstate nfa.
+ * This is for the low level interface.  The high-level interfaces enclose
+ * this in a `struct re_pattern_buffer'.  
+ */
+struct rx
+{
+  /* The compiler assigns a unique id to every pattern.
+   * Like sequence numbers in X, there is a subtle bug here
+   * if you use Rx in a system that runs for a long time.
+   * But, because of the way the caches work out, it is almost
+   * impossible to trigger the Rx version of this bug.
+   *
+   * The id is used to validate superstates found in a cache
+   * of superstates.  It isn't sufficient to let a superstate
+   * point back to the rx for which it was compiled -- the caller
+   * may be re-using a `struct rx' in which case the superstate
+   * is not really valid.  So instead, superstates are validated
+   * by checking the sequence number of the pattern for which
+   * they were built.
+   */
+  int rx_id;
+
+  /* This is memory mgt. state for superstates.  This may be 
+   * shared by more than one struct rx.
+   */
+  struct rx_cache * cache;
+
+  /* Every regex defines the size of its own character set. 
+   * A superstate has an array of this size, with each element
+   * a `struct rx_inx'.  So, don't make this number too large.
+   * In particular, don't make it 2^16.
+   */
+  int local_cset_size;
+
+  /* After the NFA is built, it is copied into a contiguous region
+   * of memory (mostly for compatability with GNU regex).
+   * Here is that region, and it's size:
+   */
+  void * buffer;
+  unsigned long allocated;
+
+  /* Clients of RX can ask for some extra storage in the space pointed
+   * to by BUFFER.  The field RESERVED is an input parameter to the
+   * compiler.  After compilation, this much space will be available 
+   * at (buffer + allocated - reserved)
+   */
+  unsigned long reserved;
+
+  /* --------- The remaining fields are for internal use only. --------- */
+  /* --------- But! they must be initialized to 0.	       --------- */
+
+  /* NODEC is the number of nodes in the NFA with non-epsilon
+   * transitions. 
+   */
+  int nodec;
+
+  /* EPSNODEC is the number of nodes with only epsilon transitions. */
+  int epsnodec;
+
+  /* The sum (NODEC + EPSNODEC) is the total number of states in the
+   * compiled NFA.
+   */
+
+  /* Lists of side effects as stored in the NFA are `hash consed'..meaning
+   * that lists with the same elements are ==.  During compilation, 
+   * this table facilitates hash-consing.
+   */
+  struct rx_hash se_list_memo;
+
+  /* Lists of NFA states are also hashed. 
+   */
+  struct rx_hash set_list_memo;
+
+
+
+
+  /* The compiler and matcher must build a number of instruction frames.
+   * The format of these frames is fixed (c.f. struct rx_inx).  The values
+   * of the instructions is not fixed.
+   *
+   * An enumerated type (enum rx_opcode) defines the set of instructions
+   * that the compiler or matcher might generate.  When filling an instruction
+   * frame, the INX field is found by indexing this instruction table
+   * with an opcode:
+   */
+  void ** instruction_table;
+
+  /* The list of all states in an NFA.
+   * During compilation, the NEXT field of NFA states links this list.
+   * After compilation, all the states are compacted into an array,
+   * ordered by state id numbers.  At that time, this points to the base 
+   * of that array.
+   */
+  struct rx_nfa_state *nfa_states;
+
+  /* Every nfa begins with one distinguished starting state:
+   */
+  struct rx_nfa_state *start;
+
+  /* This orders the search through super-nfa paths.
+   * See the comment near the typedef of rx_se_list_order.
+   */
+  rx_se_list_order se_list_cmp;
+
+  struct rx_superset * start_set;
+};
+
+
+
+
+
+/* SYNTAX TREES */
+
+/* Compilation is in stages.  
+ *
+ * In the first stage, a pattern specified by a string is 
+ * translated into a syntax tree.  Later stages will convert
+ * the syntax tree into an NFA optimized for conversion to a
+ * superstate-NFA.
+ *
+ * This page is about syntax trees.
+ */
+
+enum rexp_node_type
+{
+  r_cset,			/* Match from a character set. `a' or `[a-z]'*/
+  r_concat,			/* Concat two subexpressions.   `ab' */
+  r_alternate,			/* Choose one of two subexpressions. `a\|b' */
+  r_opt,			/* Optional subexpression. `a?' */
+  r_star,			/* Repeated subexpression. `a*' */
+
+
+  /* A 2phase-star is a variation on a repeated subexpression.
+   * In this case, there are two subexpressions.  The first, if matched,
+   * begins a repitition (otherwise, the whole expression is matches the
+   * empth string).  
+   * 
+   * After matching the first subexpression, a 2phase star either finishes,
+   * or matches the second subexpression.  If the second subexpression is
+   * matched, then the whole construct repeats.
+   *
+   * 2phase stars are used in two circumstances.  First, they
+   * are used as part of the implementation of POSIX intervals (counted
+   * repititions).  Second, they are used to implement proper star
+   * semantics when the repeated subexpression contains paths of
+   * only side effects.  See rx_compile for more information.
+   */
+  r_2phase_star,
+
+
+  /* c.f. "typedef void * rx_side_effect" */
+  r_side_effect,
+
+  /* This is an extension type:  It is for transient use in source->source
+   * transformations (implemented over syntax trees).
+   */
+  r_data
+};
+
+/* A side effect is a matcher-specific action associated with
+ * transitions in the NFA.  The details of side effects are up
+ * to the matcher.  To the compiler and superstate constructors
+ * side effects are opaque:
+ */
+
+typedef void * rx_side_effect;
+
+/* Nodes in a syntax tree are of this type:
+ */
+struct rexp_node
+{
+  enum rexp_node_type type;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+    struct
+      {
+	struct rexp_node *left;
+	struct rexp_node *right;
+      } pair;
+    void * data;
+  } params;
+};
+
+
+
+
+/* NFA
+ *
+ * A syntax tree is compiled into an NFA.  This page defines the structure
+ * of that NFA.
+ */
+
+struct rx_nfa_state
+{
+  /* These are kept in a list as the NFA is being built. */
+  struct rx_nfa_state *next;
+
+  /* After the NFA is built, states are given integer id's.
+   * States whose outgoing transitions are all either epsilon or 
+   * side effect edges are given ids less than 0.  Other states
+   * are given successive non-negative ids starting from 0.
+   */
+  int id;
+
+  /* The list of NFA edges that go from this state to some other. */
+  struct rx_nfa_edge *edges;
+
+  /* If you land in this state, then you implicitly land
+   * in all other states reachable by only epsilon translations.
+   * Call the set of maximal paths to such states the epsilon closure
+   * of this state.
+   *
+   * There may be other states that are reachable by a mixture of
+   * epsilon and side effect edges.  Consider the set of maximal paths
+   * of that sort from this state.  Call it the epsilon-side-effect
+   * closure of the state.
+   * 
+   * The epsilon closure of the state is a subset of the epsilon-side-
+   * effect closure.  It consists of all the paths that contain 
+   * no side effects -- only epsilon edges.
+   * 
+   * The paths in the epsilon-side-effect closure  can be partitioned
+   * into equivalance sets. Two paths are equivalant if they have the
+   * same set of side effects, in the same order.  The epsilon-closure
+   * is one of these equivalance sets.  Let's call these equivalance
+   * sets: observably equivalant path sets.  That name is chosen
+   * because equivalance of two paths means they cause the same side
+   * effects -- so they lead to the same subsequent observations other
+   * than that they may wind up in different target states.
+   *
+   * The superstate nfa, which is derived from this nfa, is based on
+   * the observation that all of the paths in an observably equivalant
+   * path set can be explored at the same time, provided that the
+   * matcher keeps track not of a single nfa state, but of a set of
+   * states.   In particular, after following all the paths in an
+   * observably equivalant set, you wind up at a set of target states.
+   * That set of target states corresponds to one state in the
+   * superstate NFA.
+   *
+   * Staticly, before matching begins, it is convenient to analyze the
+   * nfa.  Each state is labeled with a list of the observably
+   * equivalant path sets who's union covers all the
+   * epsilon-side-effect paths beginning in this state.  This list is
+   * called the possible futures of the state.
+   *
+   * A trivial example is this NFA:
+   *             s1
+   *         A  --->  B
+   *
+   *             s2  
+   *            --->  C
+   *
+   *             epsilon           s1
+   *            --------->  D   ------> E
+   * 
+   * 
+   * In this example, A has two possible futures.
+   * One invokes the side effect `s1' and contains two paths,
+   * one ending in state B, the other in state E.
+   * The other invokes the side effect `s2' and contains only
+   * one path, landing in state C.
+   */
+  struct rx_possible_future *futures;
+
+
+  /* There are exactly two distinguished states in every NFA: */
+  unsigned int is_final:1;
+  unsigned int is_start:1;
+
+  /* These are used during NFA construction... */
+  unsigned int eclosure_needed:1;
+  unsigned int mark:1;
+};
+
+
+/* An edge in an NFA is typed: */
+enum rx_nfa_etype
+{
+  /* A cset edge is labled with a set of characters one of which
+   * must be matched for the edge to be taken.
+   */
+  ne_cset,
+
+  /* An epsilon edge is taken whenever its starting state is
+   * reached. 
+   */
+  ne_epsilon,
+
+  /* A side effect edge is taken whenever its starting state is
+   * reached.  Side effects may cause the match to fail or the
+   * position of the matcher to advance.
+   */
+  ne_side_effect		/* A special kind of epsilon. */
+};
+
+struct rx_nfa_edge
+{
+  struct rx_nfa_edge *next;
+  enum rx_nfa_etype type;
+  struct rx_nfa_state *dest;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+  } params;
+};
+
+
+
+/* A possible future consists of a list of side effects
+ * and a set of destination states.  Below are their
+ * representations.  These structures are hash-consed which
+ * means that lists with the same elements share a representation
+ * (their addresses are ==).
+ */
+
+struct rx_nfa_state_set
+{
+  struct rx_nfa_state * car;
+  struct rx_nfa_state_set * cdr;
+};
+
+struct rx_se_list
+{
+  rx_side_effect car;
+  struct rx_se_list * cdr;
+};
+
+struct rx_possible_future
+{
+  struct rx_possible_future *next;
+  struct rx_se_list * effects;
+  struct rx_nfa_state_set * destset;
+};
+
+
+
+
+/* This begins the description of the superstate NFA.
+ *
+ * The superstate NFA corresponds to the NFA in these ways:
+ *
+ * Every superstate NFA states SUPER correspond to sets of NFA states,
+ * nfa_states(SUPER).
+ *
+ * Superstate edges correspond to NFA paths.
+ *
+ * The superstate has no epsilon transitions;
+ * every edge has a character label, and a (possibly empty) side
+ * effect label.   The side effect label corresponds to a list of
+ * side effects that occur in the NFA.  These parts are referred
+ * to as:   superedge_character(EDGE) and superedge_sides(EDGE).
+ *
+ * For a superstate edge EDGE starting in some superstate SUPER,
+ * the following is true (in pseudo-notation :-):
+ *
+ *       exists DEST in nfa_states s.t. 
+ *         exists nfaEDGE in nfa_edges s.t.
+ *                 origin (nfaEDGE) == DEST
+ *              && origin (nfaEDGE) is a member of nfa_states(SUPER)
+ *              && exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *
+ * also:
+ *
+ *      let SUPER2 := superedge_destination(EDGE)
+ *          nfa_states(SUPER2)
+ *           == union of all nfa state sets S s.t.
+ *                          exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                       sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *                          && S == dests_of_possible_future (PF) }
+ *
+ * Or in english, every superstate is a set of nfa states.  A given
+ * character and a superstate implies many transitions in the NFA --
+ * those that begin with an edge labeled with that character from a
+ * state in the set corresponding to the superstate.
+ * 
+ * The destinations of those transitions each have a set of possible
+ * futures.  A possible future is a list of side effects and a set of
+ * destination NFA states.  Two sets of possible futures can be
+ * `merged' by combining all pairs of possible futures that have the
+ * same side effects.  A pair is combined by creating a new future
+ * with the same side effect but the union of the two destination sets.
+ * In this way, all the possible futures suggested by a superstate
+ * and a character can be merged into a set of possible futures where
+ * no two elements of the set have the same set of side effects.
+ *
+ * The destination of a possible future, being a set of NFA states, 
+ * corresponds to a supernfa state.  So, the merged set of possible
+ * futures we just created can serve as a set of edges in the
+ * supernfa.
+ *
+ * The representation of the superstate nfa and the nfa is critical.
+ * The nfa has to be compact, but has to facilitate the rapid
+ * computation of missing superstates.  The superstate nfa has to 
+ * be fast to interpret, lazilly constructed, and bounded in space.
+ *
+ * To facilitate interpretation, the superstate data structures are 
+ * peppered with `instruction frames'.  There is an instruction set
+ * defined below which matchers using the supernfa must be able to
+ * interpret.
+ *
+ * We'd like to make it possible but not mandatory to use code
+ * addresses to represent instructions (c.f. gcc's computed goto).
+ * Therefore, we define an enumerated type of opcodes, and when
+ * writing one of these instructions into a data structure, use
+ * the opcode as an index into a table of instruction values.
+ * 
+ * Here are the opcodes that occur in the superstate nfa:
+ */
+ 
+
+/* Every superstate contains a table of instruction frames indexed 
+ * by characters.  A normal `move' in a matcher is to fetch the next
+ * character and use it as an index into a superstates transition
+ * table.
+ *
+ * In the fasted case, only one edge follows from that character.
+ * In other cases there is more work to do.
+ * 
+ * The descriptions of the opcodes refer to data structures that are
+ * described further below. 
+ */
+
+enum rx_opcode
+{
+  /* 
+   * BACKTRACK_POINT is invoked when a character transition in 
+   * a superstate leads to more than one edge.  In that case,
+   * the edges have to be explored independently using a backtracking
+   * strategy.
+   *
+   * A BACKTRACK_POINT instruction is stored in a superstate's 
+   * transition table for some character when it is known that that
+   * character crosses more than one edge.  On encountering this
+   * instruction, the matcher saves enough state to backtrack to this
+   * point in the match later.
+   */
+  rx_backtrack_point = 0,	/* data is (struct transition_class *) */
+
+  /* 
+   * RX_DO_SIDE_EFFECTS evaluates the side effects of an epsilon path.
+   * There is one occurence of this instruction per rx_distinct_future.
+   * This instruction is skipped if a rx_distinct_future has no side effects.
+   */
+  rx_do_side_effects = rx_backtrack_point + 1,
+
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_CACHE_MISS instructions are stored in rx_distinct_futures whose
+   * destination superstate has been reclaimed (or was never built).
+   * It recomputes the destination superstate.
+   * RX_CACHE_MISS is also stored in a superstate transition table before
+   * any of its edges have been built.
+   */
+  rx_cache_miss = rx_do_side_effects + 1,
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_NEXT_CHAR is called to consume the next character and take the
+   * corresponding transition.  This is the only instruction that uses 
+   * the DATA field of the instruction frame instead of DATA_2.
+   * (see EXPLORE_FUTURE in regex.c).
+   */
+  rx_next_char = rx_cache_miss + 1, /* data is (struct superstate *) */
+
+  /* RX_BACKTRACK indicates that a transition fails.
+   */
+  rx_backtrack = rx_next_char + 1, /* no data */
+
+  /* 
+   * RX_ERROR_INX is stored only in places that should never be executed.
+   */
+  rx_error_inx = rx_backtrack + 1, /* Not supposed to occur. */
+
+  rx_num_instructions = rx_error_inx + 1
+};
+
+/* An id_instruction_table holds the values stored in instruction
+ * frames.  The table is indexed by the enums declared above.
+ */
+extern void * rx_id_instruction_table[rx_num_instructions];
+
+/* The heart of the matcher is a `word-code-interpreter' 
+ * (like a byte-code interpreter, except that instructions
+ * are a full word wide).
+ *
+ * Instructions are not stored in a vector of code, instead,
+ * they are scattered throughout the data structures built
+ * by the regexp compiler and the matcher.  One word-code instruction,
+ * together with the arguments to that instruction, constitute
+ * an instruction frame (struct rx_inx).
+ *
+ * This structure type is padded by hand to a power of 2 because
+ * in one of the dominant cases, we dispatch by indexing a table
+ * of instruction frames.  If that indexing can be accomplished
+ * by just a shift of the index, we're happy.
+ *
+ * Instructions take at most one argument, but there are two
+ * slots in an instruction frame that might hold that argument.
+ * These are called data and data_2.  The data slot is only
+ * used for one instruction (RX_NEXT_CHAR).  For all other 
+ * instructions, data should be set to 0.
+ *
+ * RX_NEXT_CHAR is the most important instruction by far.
+ * By reserving the data field for its exclusive use, 
+ * instruction dispatch is sped up in that case.  There is
+ * no need to fetch both the instruction and the data,
+ * only the data is needed.  In other words, a `cycle' begins
+ * by fetching the field data.  If that is non-0, then it must
+ * be the destination state of a next_char transition, so
+ * make that value the current state, advance the match position
+ * by one character, and start a new cycle.  On the other hand,
+ * if data is 0, fetch the instruction and do a more complicated
+ * dispatch on that.
+ */
+
+struct rx_inx 
+{
+  void * data;
+  void * data_2;
+  void * inx;
+  void * fnord;
+};
+
+#ifndef RX_TAIL_ARRAY
+#define RX_TAIL_ARRAY  1
+#endif
+
+/* A superstate corresponds to a set of nfa states.  Those sets are
+ * represented by STRUCT RX_SUPERSET.  The constructors
+ * guarantee that only one (shared) structure is created for a given set.
+ */
+struct rx_superset
+{
+  int refs;			/* This is a reference counted structure. */
+
+  /* We keep these sets in a cache because (in an unpredictable way),
+   * the same set is often created again and again.  But that is also
+   * problematic -- compatibility with POSIX and GNU regex requires
+   * that we not be able to tell when a program discards a particular
+   * NFA (thus invalidating the supersets created from it).
+   *
+   * But when a cache hit appears to occur, we will have in hand the
+   * nfa for which it may have happened.  That is why every nfa is given
+   * its own sequence number.  On a cache hit, the cache is validated
+   * by comparing the nfa sequence number to this field:
+   */
+  int id;
+
+  struct rx_nfa_state * car;	/* May or may not be a valid addr. */
+  struct rx_superset * cdr;
+
+  /* If the corresponding superstate exists: */
+  struct rx_superstate * superstate;
+
+
+  /* There is another bookkeeping problem.  It is expensive to 
+   * compute the starting nfa state set for an nfa.  So, once computed,
+   * it is cached in the `struct rx'.
+   *
+   * But, the state set can be flushed from the superstate cache.
+   * When that happens, we can't know if the corresponding `struct rx'
+   * is still alive or if it has been freed or re-used by the program.
+   * So, the cached pointer to this set in a struct rx might be invalid
+   * and we need a way to validate it.
+   *
+   * Fortunately, even if this set is flushed from the cache, it is
+   * not freed.  It just goes on the free-list of supersets.
+   * So we can still examine it.  
+   *
+   * So to validate a starting set memo, check to see if the
+   * starts_for field still points back to the struct rx in question,
+   * and if the ID matches the rx sequence number.
+   */
+  struct rx * starts_for;
+
+  /* This is used to link into a hash bucket so these objects can
+   * be `hash-consed'.
+   */
+  struct rx_hash_item hash_item;
+};
+
+#define rx_protect_superset(RX,CON) (++(CON)->refs)
+
+/* The terminology may be confusing (rename this structure?).
+ * Every character occurs in at most one rx_super_edge per super-state.
+ * But, that structure might have more than one option, indicating a point
+ * of non-determinism. 
+ *
+ * In other words, this structure holds a list of superstate edges
+ * sharing a common starting state and character label.  The edges
+ * are in the field OPTIONS.  All superstate edges sharing the same
+ * starting state and character are in this list.
+ */
+struct rx_super_edge
+{
+  struct rx_super_edge *next;
+  struct rx_inx rx_backtrack_frame;
+  int cset_size;
+  rx_Bitset cset;
+  struct rx_distinct_future *options;
+};
+
+/* A superstate is a set of nfa states (RX_SUPERSET) along
+ * with a transition table.  Superstates are built on demand and reclaimed
+ * without warning.  To protect a superstate from this ghastly fate,
+ * use LOCK_SUPERSTATE. 
+ */
+struct rx_superstate
+{
+  int rx_id;			/* c.f. the id field of rx_superset */
+  int locks;			/* protection from reclamation */
+
+  /* Within a superstate cache, all the superstates are kept in a big
+   * queue.  The tail of the queue is the state most likely to be
+   * reclaimed.  The *recyclable fields hold the queue position of 
+   * this state.
+   */
+  struct rx_superstate * next_recyclable;
+  struct rx_superstate * prev_recyclable;
+
+  /* The supernfa edges that exist in the cache and that have
+   * this state as their destination are kept in this list:
+   */
+  struct rx_distinct_future * transition_refs;
+
+  /* The list of nfa states corresponding to this superstate: */
+  struct rx_superset * contents;
+
+  /* The list of edges in the cache beginning from this state. */
+  struct rx_super_edge * edges;
+
+  /* A tail of the recyclable queue is marked as semifree.  A semifree
+   * state has no incoming next_char transitions -- any transition
+   * into a semifree state causes a complex dispatch with the side
+   * effect of rescuing the state from its semifree state.
+   *
+   * An alternative to this might be to make next_char more expensive,
+   * and to move a state to the head of the recyclable queue whenever
+   * it is entered.  That way, popular states would never be recycled.
+   *
+   * But unilaterally making next_char more expensive actually loses.
+   * So, incoming transitions are only made expensive for states near
+   * the tail of the recyclable queue.  The more cache contention
+   * there is, the more frequently a state will have to prove itself
+   * and be moved back to the front of the queue.  If there is less 
+   * contention, then popular states just aggregate in the front of 
+   * the queue and stay there.
+   */
+  int is_semifree;
+
+
+  /* This keeps track of the size of the transition table for this
+   * state.  There is a half-hearted attempt to support variable sized
+   * superstates.
+   */
+  int trans_size;
+
+  /* Indexed by characters... */
+  struct rx_inx transitions[RX_TAIL_ARRAY];
+};
+
+
+/* A list of distinct futures define the edges that leave from a 
+ * given superstate on a given character.  c.f. rx_super_edge.
+ */
+
+struct rx_distinct_future
+{
+  struct rx_distinct_future * next_same_super_edge[2];
+  struct rx_distinct_future * next_same_dest;
+  struct rx_distinct_future * prev_same_dest;
+  struct rx_superstate * present;	/* source state */
+  struct rx_superstate * future;	/* destination state */
+  struct rx_super_edge * edge;
+
+
+  /* The future_frame holds the instruction that should be executed
+   * after all the side effects are done, when it is time to complete
+   * the transition to the next state.
+   *
+   * Normally this is a next_char instruction, but it may be a
+   * cache_miss instruction as well, depending on whether or not
+   * the superstate is in the cache and semifree.
+   * 
+   * If this is the only future for a given superstate/char, and
+   * if there are no side effects to be performed, this frame is
+   * not used (directly) at all.  Instead, its contents are copied
+   * into the transition table of the starting state of this dist. future.
+   */
+  struct rx_inx future_frame;
+
+  struct rx_inx side_effects_frame;
+  struct rx_se_list * effects;
+};
+
+#define rx_lock_superstate(R,S)  ((S)->locks++)
+#define rx_unlock_superstate(R,S) (--(S)->locks)
+
+
+
+/* This page destined for rx.h */
+
+struct rx_blocklist
+{
+  struct rx_blocklist * next;
+  int bytes;
+};
+
+struct rx_freelist
+{
+  struct rx_freelist * next;
+};
+
+struct rx_cache;
+
+#ifdef __STDC__
+typedef void (*rx_morecore_fn)(struct rx_cache *);
+#else
+typedef void (*rx_morecore_fn)();
+#endif
+
+/* You use this to control the allocation of superstate data 
+ * during matching.  Most of it should be initialized to 0.
+ *
+ * A MORECORE function is necessary.  It should allocate
+ * a new block of memory or return 0.
+ * A default that uses malloc is called `rx_morecore'.
+ *
+ * The number of SUPERSTATES_ALLOWED indirectly limits how much memory
+ * the system will try to allocate.  The default is 128.  Batch style
+ * applications that are very regexp intensive should use as high a number
+ * as possible without thrashing.
+ * 
+ * The LOCAL_CSET_SIZE is the number of characters in a character set.
+ * It is therefore the number of entries in a superstate transition table.
+ * Generally, it should be 256.  If your character set has 16 bits, 
+ * it is better to translate your regexps into equivalent 8 bit patterns.
+ */
+
+struct rx_cache
+{
+  struct rx_hash_rules superset_hash_rules;
+
+  /* Objects are allocated by incrementing a pointer that 
+   * scans across rx_blocklists.
+   */
+  struct rx_blocklist * memory;
+  struct rx_blocklist * memory_pos;
+  int bytes_left;
+  char * memory_addr;
+  rx_morecore_fn morecore;
+
+  /* Freelists. */
+  struct rx_freelist * free_superstates;
+  struct rx_freelist * free_transition_classes;
+  struct rx_freelist * free_discernable_futures;
+  struct rx_freelist * free_supersets;
+  struct rx_freelist * free_hash;
+
+  /* Two sets of superstates -- those that are semifreed, and those
+   * that are being used.
+   */
+  struct rx_superstate * lru_superstate;
+  struct rx_superstate * semifree_superstate;
+
+  struct rx_superset * empty_superset;
+
+  int superstates;
+  int semifree_superstates;
+  int hits;
+  int misses;
+  int superstates_allowed;
+
+  int local_cset_size;
+  void ** instruction_table;
+
+  struct rx_hash superset_table;
+};
+
+
+
+
+/* The lowest-level search function supports arbitrarily fragmented
+ * strings and (optionally) suspendable/resumable searches.
+ *
+ * Callers have to provide a few hooks.
+ */
+
+#ifndef __GNUC__
+#ifdef __STDC__
+#define __const__ const
+#else
+#define __const__
+#endif
+#endif
+
+/* This holds a matcher position */
+struct rx_string_position
+{
+  __const__ unsigned char * pos;	/* The current pos. */
+  __const__ unsigned char * string; /* The current string burst. */
+  __const__ unsigned char * end;	/* First invalid position >= POS. */
+  int offset;			/* Integer address of the current burst. */
+  int size;			/* Current string's size. */
+  int search_direction;		/* 1 or -1 */
+  int search_end;		/* First position to not try. */
+};
+
+
+enum rx_get_burst_return
+{
+  rx_get_burst_continuation,
+  rx_get_burst_error,
+  rx_get_burst_ok,
+  rx_get_burst_no_more
+};
+
+
+/* A call to get burst should make POS valid.  It might be invalid
+ * if the STRING field doesn't point to a burst that actually
+ * contains POS.
+ *
+ * GET_BURST should take a clue from SEARCH_DIRECTION (1 or -1) as to
+ * whether or not to pad to the left.  Padding to the right is always
+ * appropriate, but need not go past the point indicated by STOP.
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the get_burst.
+ */
+
+#ifdef __STDC__
+typedef enum rx_get_burst_return
+  (*rx_get_burst_fn) (struct rx_string_position * pos,
+		      void * app_closure,
+		      int stop);
+					       
+#else
+typedef enum rx_get_burst_return (*rx_get_burst_fn) ();
+#endif
+
+
+enum rx_back_check_return
+{
+  rx_back_check_continuation,
+  rx_back_check_error,
+  rx_back_check_pass,
+  rx_back_check_fail
+};
+
+/* Back_check should advance the position it is passed 
+ * over rparen - lparen characters and return pass iff
+ * the characters starting at POS match those indexed
+ * by [LPAREN..RPAREN].
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the back_check.
+ */
+
+#ifdef __STDC__
+typedef enum rx_back_check_return
+  (*rx_back_check_fn) (struct rx_string_position * pos,
+		       int lparen,
+		       int rparen,
+		       unsigned char * translate,
+		       void * app_closure,
+		       int stop);
+					       
+#else
+typedef enum rx_back_check_return (*rx_back_check_fn) ();
+#endif
+
+
+
+
+/* A call to fetch_char should return the character at POS or POS + 1.
+ * Returning continuations here isn't supported.  OFFSET is either 0 or 1
+ * and indicates which characters is desired.
+ */
+
+#ifdef __STDC__
+typedef int (*rx_fetch_char_fn) (struct rx_string_position * pos,
+				 int offset,
+				 void * app_closure,
+				 int stop);
+#else
+typedef int (*rx_fetch_char_fn) ();
+#endif
+
+
+enum rx_search_return
+{
+  rx_search_continuation = -4,
+  rx_search_error = -3,
+  rx_search_soft_fail = -2,	/* failed by running out of string */
+  rx_search_fail = -1		/* failed only by reaching failure states */
+  /* return values >= 0 indicate the position of a successful match */
+};
+
+
+
+
+
+
+
+/* regex.h
+ * 
+ * The remaining declarations replace regex.h.
+ */
+
+/* This is an array of error messages corresponding to the error codes.
+ */
+extern __const__ char *re_error_msg[];
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+/* The regex.c support, as a client of rx, defines a set of possible
+ * side effects that can be added to the edge lables of nfa edges.
+ * Here is the list of sidef effects in use.
+ */
+
+enum re_side_effects
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      NAME VALUE,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     NAME VALUE,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+   re_floogle_flap = 65533
+};
+
+/* These hold paramaters for the kinds of side effects that are possible
+ * in the supported pattern languages.  These include things like the 
+ * numeric bounds of {} operators and the index of paren registers for 
+ * subexpression measurement or backreferencing.
+ */
+struct re_se_params
+{
+  enum re_side_effects se;
+  int op1;
+  int op2;
+};
+
+typedef unsigned reg_syntax_t;
+
+struct re_pattern_buffer
+{
+  struct rx rx;
+  reg_syntax_t syntax;		/* See below for syntax bit definitions. */
+
+  unsigned int no_sub:1;	/* If set, don't  return register offsets. */
+  unsigned int not_bol:1;	/* If set, the anchors ('^' and '$') don't */
+  unsigned int not_eol:1;	/*     match at the ends of the string.  */  
+  unsigned int newline_anchor:1;/* If true, an anchor at a newline matches.*/
+  unsigned int least_subs:1;	/* If set, and returning registers, return
+				 * as few values as possible.  Only 
+				 * backreferenced groups and group 0 (the whole
+				 * match) will be returned.
+				 */
+
+  /* If true, this says that the matcher should keep registers on its
+   * backtracking stack.  For many patterns, we can easily determine that
+   * this isn't necessary.
+   */
+  unsigned int match_regs_on_stack:1;
+  unsigned int search_regs_on_stack:1;
+
+  /* is_anchored and begbuf_only are filled in by rx_compile. */
+  unsigned int is_anchored:1;	/* Anchorded by ^? */
+  unsigned int begbuf_only:1;	/* Anchored to char position 0? */
+
+  
+  /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+   * for `max (RE_NREGS, re_nsub + 1)' groups.
+   * If REGS_REALLOCATE, reallocate space if necessary.
+   * If REGS_FIXED, use what's there.  
+   */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned int regs_allocated:2;
+
+  
+  /* Either a translate table to apply to all characters before
+   * comparing them, or zero for no translation.  The translation
+   * is applied to a pattern when it is compiled and to a string
+   * when it is matched.
+   */
+  unsigned char * translate;
+
+  /* If this is a valid pointer, it tells rx not to store the extents of 
+   * certain subexpressions (those corresponding to non-zero entries).
+   * Passing 0x1 is the same as passing an array of all ones.  Passing 0x0
+   * is the same as passing an array of all zeros.
+   * The array should contain as many entries as their are subexps in the 
+   * regexp.
+   *
+   * For POSIX compatability, when using regcomp and regexec this field
+   * is zeroed and ignored.
+   */
+  char * syntax_parens;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+  void * buffer;		/* Malloced memory for the nfa. */
+  unsigned long allocated;	/* Size of that memory. */
+
+  /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+   * the fastmap, if there is one, to skip over impossible
+   * starting points for matches.  */
+  char *fastmap;
+
+  unsigned int fastmap_accurate:1; /* These three are internal. */
+  unsigned int can_match_empty:1;  
+  struct rx_nfa_state * start;	/* The nfa starting state. */
+
+  /* This is the list of iterator bounds for {lo,hi} constructs.
+   * The memory pointed to is part of the rx->buffer.
+   */
+  struct re_se_params *se_params;
+
+  /* This is a bitset representation of the fastmap.
+   * This is a true fastmap that already takes the translate
+   * table into account.
+   */
+  rx_Bitset fastset;
+};
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+ * `re_match_2' returns information about at least this many registers
+ * the first time a `regs' structure is passed. 
+ *
+ * Also, this is the greatest number of backreferenced subexpressions
+ * allowed in a pattern being matched without caller-supplied registers.
+ */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+extern int rx_cache_bound;
+extern char rx_version_string[];
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+/* This is decls to the interesting subsystems and lower layers
+ * of rx.  Everything which doesn't have a public counterpart in 
+ * regex.c is declared here.
+ */
+
+
+#ifdef __STDC__
+typedef void (*rx_hash_freefn) (struct rx_hash_item * it);
+#else /* ndef __STDC__ */
+typedef void (*rx_hash_freefn) ();
+#endif /* ndef __STDC__ */
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL int rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_empty (int size, rx_Bitset set);
+RX_DECL void rx_bitset_null (int size, rx_Bitset b);
+RX_DECL void rx_bitset_universe (int size, rx_Bitset b);
+RX_DECL void rx_bitset_complement (int size, rx_Bitset b);
+RX_DECL void rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_union (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_intersection (int size,
+				     rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_revdifference (int size,
+				      rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL unsigned long rx_bitset_hash (int size, rx_Bitset b);
+RX_DECL struct rx_hash_item * rx_hash_find (struct rx_hash * table,
+					    unsigned long hash,
+					    void * value,
+					    struct rx_hash_rules * rules);
+RX_DECL struct rx_hash_item * rx_hash_store (struct rx_hash * table,
+					     unsigned long hash,
+					     void * value,
+					     struct rx_hash_rules * rules);
+RX_DECL void rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules);
+RX_DECL void rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+				 struct rx_hash_rules * rules);
+RX_DECL rx_Bitset rx_cset (struct rx *rx);
+RX_DECL rx_Bitset rx_copy_cset (struct rx *rx, rx_Bitset a);
+RX_DECL void rx_free_cset (struct rx * rx, rx_Bitset c);
+RX_DECL struct rexp_node * rexp_node (struct rx *rx,
+				      enum rexp_node_type type);
+RX_DECL struct rexp_node * rx_mk_r_cset (struct rx * rx,
+					 rx_Bitset b);
+RX_DECL struct rexp_node * rx_mk_r_concat (struct rx * rx,
+					   struct rexp_node * a,
+					   struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_alternate (struct rx * rx,
+					      struct rexp_node * a,
+					      struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_opt (struct rx * rx,
+					struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_star (struct rx * rx,
+					 struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_2phase_star (struct rx * rx,
+						struct rexp_node * a,
+						struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_side_effect (struct rx * rx,
+						rx_side_effect a);
+RX_DECL struct rexp_node * rx_mk_r_data  (struct rx * rx,
+					  void * a);
+RX_DECL void rx_free_rexp (struct rx * rx, struct rexp_node * node);
+RX_DECL struct rexp_node * rx_copy_rexp (struct rx *rx,
+					 struct rexp_node *node);
+RX_DECL struct rx_nfa_state * rx_nfa_state (struct rx *rx);
+RX_DECL void rx_free_nfa_state (struct rx_nfa_state * n);
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state (struct rx * rx,
+						  int id);
+RX_DECL struct rx_nfa_edge * rx_nfa_edge (struct rx *rx,
+					  enum rx_nfa_etype type,
+					  struct rx_nfa_state *start,
+					  struct rx_nfa_state *dest);
+RX_DECL void rx_free_nfa_edge (struct rx_nfa_edge * e);
+RX_DECL void rx_free_nfa (struct rx *rx);
+RX_DECL int rx_build_nfa (struct rx *rx,
+			  struct rexp_node *rexp,
+			  struct rx_nfa_state **start,
+			  struct rx_nfa_state **end);
+RX_DECL void rx_name_nfa_states (struct rx *rx);
+RX_DECL int rx_eclose_nfa (struct rx *rx);
+RX_DECL void rx_delete_epsilon_transitions (struct rx *rx);
+RX_DECL int rx_compactify_nfa (struct rx *rx,
+			       void **mem, unsigned long *size);
+RX_DECL void rx_release_superset (struct rx *rx,
+				  struct rx_superset *set);
+RX_DECL struct rx_superset * rx_superset_cons (struct rx * rx,
+					       struct rx_nfa_state *car, struct rx_superset *cdr);
+RX_DECL struct rx_superset * rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl);
+RX_DECL struct rx_superstate * rx_superstate (struct rx *rx,
+					      struct rx_superset *set);
+RX_DECL struct rx_inx * rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data);
+RX_DECL reg_errcode_t rx_compile (__const__ char *pattern, int size,
+				  reg_syntax_t syntax,
+				  struct re_pattern_buffer * rxb);
+RX_DECL void rx_blow_up_fastmap (struct re_pattern_buffer * rxb);
+#else /* STDC */
+RX_DECL int rx_bitset_is_equal ();
+RX_DECL int rx_bitset_is_subset ();
+RX_DECL int rx_bitset_empty ();
+RX_DECL void rx_bitset_null ();
+RX_DECL void rx_bitset_universe ();
+RX_DECL void rx_bitset_complement ();
+RX_DECL void rx_bitset_assign ();
+RX_DECL void rx_bitset_union ();
+RX_DECL void rx_bitset_intersection ();
+RX_DECL void rx_bitset_difference ();
+RX_DECL void rx_bitset_revdifference ();
+RX_DECL void rx_bitset_xor ();
+RX_DECL unsigned long rx_bitset_hash ();
+RX_DECL struct rx_hash_item * rx_hash_find ();
+RX_DECL struct rx_hash_item * rx_hash_store ();
+RX_DECL void rx_hash_free ();
+RX_DECL void rx_free_hash_table ();
+RX_DECL rx_Bitset rx_cset ();
+RX_DECL rx_Bitset rx_copy_cset ();
+RX_DECL void rx_free_cset ();
+RX_DECL struct rexp_node * rexp_node ();
+RX_DECL struct rexp_node * rx_mk_r_cset ();
+RX_DECL struct rexp_node * rx_mk_r_concat ();
+RX_DECL struct rexp_node * rx_mk_r_alternate ();
+RX_DECL struct rexp_node * rx_mk_r_opt ();
+RX_DECL struct rexp_node * rx_mk_r_star ();
+RX_DECL struct rexp_node * rx_mk_r_2phase_star ();
+RX_DECL struct rexp_node * rx_mk_r_side_effect ();
+RX_DECL struct rexp_node * rx_mk_r_data  ();
+RX_DECL void rx_free_rexp ();
+RX_DECL struct rexp_node * rx_copy_rexp ();
+RX_DECL struct rx_nfa_state * rx_nfa_state ();
+RX_DECL void rx_free_nfa_state ();
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state ();
+RX_DECL struct rx_nfa_edge * rx_nfa_edge ();
+RX_DECL void rx_free_nfa_edge ();
+RX_DECL void rx_free_nfa ();
+RX_DECL int rx_build_nfa ();
+RX_DECL void rx_name_nfa_states ();
+RX_DECL int rx_eclose_nfa ();
+RX_DECL void rx_delete_epsilon_transitions ();
+RX_DECL int rx_compactify_nfa ();
+RX_DECL void rx_release_superset ();
+RX_DECL struct rx_superset * rx_superset_cons ();
+RX_DECL struct rx_superset * rx_superstate_eclosure_union ();
+RX_DECL struct rx_superstate * rx_superstate ();
+RX_DECL struct rx_inx * rx_handle_cache_miss ();
+RX_DECL reg_errcode_t rx_compile ();
+RX_DECL void rx_blow_up_fastmap ();
+#endif /* STDC */
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+
+#ifdef __STDC__
+extern int re_search_2 (struct re_pattern_buffer *rxb,
+			__const__ char * string1, int size1,
+			__const__ char * string2, int size2,
+			int startpos, int range,
+			struct re_registers *regs,
+			int stop);
+extern int re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+		      int size, int startpos, int range,
+		      struct re_registers *regs);
+extern int re_match_2 (struct re_pattern_buffer * rxb,
+		       __const__ char * string1, int size1,
+		       __const__ char * string2, int size2,
+		       int pos, struct re_registers *regs, int stop);
+extern int re_match (struct re_pattern_buffer * rxb,
+		     __const__ char * string,
+		     int size, int pos,
+		     struct re_registers *regs);
+extern reg_syntax_t re_set_syntax (reg_syntax_t syntax);
+extern void re_set_registers (struct re_pattern_buffer *bufp,
+			      struct re_registers *regs,
+			      unsigned num_regs,
+			      regoff_t * starts, regoff_t * ends);
+extern __const__ char * re_compile_pattern (__const__ char *pattern,
+					int length,
+					struct re_pattern_buffer * rxb);
+extern int re_compile_fastmap (struct re_pattern_buffer * rxb);
+extern char * re_comp (__const__ char *s);
+extern int re_exec (__const__ char *s);
+extern int regcomp (regex_t * preg, __const__ char * pattern, int cflags);
+extern int regexec (__const__ regex_t *preg, __const__ char *string,
+		    size_t nmatch, regmatch_t pmatch[],
+		    int eflags);
+extern size_t regerror (int errcode, __const__ regex_t *preg,
+			char *errbuf, size_t errbuf_size);
+extern void regfree (regex_t *preg);
+
+#else /* STDC */
+extern int re_search_2 ();
+extern int re_search ();
+extern int re_match_2 ();
+extern int re_match ();
+extern reg_syntax_t re_set_syntax ();
+extern void re_set_registers ();
+extern __const__ char * re_compile_pattern ();
+extern int re_compile_fastmap ();
+extern char * re_comp ();
+extern int re_exec ();
+extern int regcomp ();
+extern int regexec ();
+extern size_t regerror ();
+extern void regfree ();
+
+#endif /* STDC */
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+struct rx_counter_frame
+{
+  int tag;
+  int val;
+  struct rx_counter_frame * inherited_from; /* If this is a copy. */
+  struct rx_counter_frame * cdr;
+};
+
+struct rx_backtrack_frame
+{
+  char * counter_stack_sp;
+
+  /* A frame is used to save the matchers state when it crosses a 
+   * backtracking point.  The `stk_' fields correspond to variables
+   * in re_search_2 (just strip off thes `stk_').  They are documented
+   * tere.
+   */
+  struct rx_superstate * stk_super;
+  unsigned int stk_c;
+  struct rx_string_position stk_test_pos;
+  int stk_last_l;
+  int stk_last_r;
+  int stk_test_ret;
+
+  /* This is the list of options left to explore at the backtrack
+   * point for which this frame was created. 
+   */
+  struct rx_distinct_future * df;
+  struct rx_distinct_future * first_df;
+
+#ifdef RX_DEBUG
+   int stk_line_no;
+#endif
+};
+
+struct rx_stack_chunk
+{
+  struct rx_stack_chunk * next_chunk;
+  int bytes_left;
+  char * sp;
+};
+
+enum rx_outer_entry
+{
+  rx_outer_start,
+  rx_outer_fastmap,
+  rx_outer_test,
+  rx_outer_restore_pos
+};
+
+enum rx_fastmap_return
+{
+  rx_fastmap_continuation,
+  rx_fastmap_error,
+  rx_fastmap_ok,
+  rx_fastmap_fail
+};
+
+enum rx_fastmap_entry
+{
+  rx_fastmap_start,
+  rx_fastmap_string_break
+};
+
+enum rx_test_return
+{
+  rx_test_continuation,
+  rx_test_error,
+  rx_test_fail,
+  rx_test_ok
+};
+
+enum rx_test_internal_return
+{
+  rx_test_internal_error,
+  rx_test_found_first,
+  rx_test_line_finished
+};
+
+enum rx_test_match_entry
+{
+  rx_test_start,
+  rx_test_cache_hit_loop,
+  rx_test_backreference_check,
+  rx_test_backtrack_return
+};
+
+struct rx_search_state
+{
+  /* Two groups of registers are kept.  The group with the register state
+   * of the current test match, and the group that holds the state at the end
+   * of the best known match, if any.
+   *
+   * For some patterns, there may also be registers saved on the stack.
+   */
+  unsigned num_regs;		/* Includes an element for register zero. */
+  regoff_t * lparen;		/* scratch space for register returns */
+  regoff_t * rparen;
+  regoff_t * best_lpspace;	/* in case the user doesn't want these */
+  regoff_t * best_rpspace;	/* values, we still need space to store
+				 * them.  Normally, this memoryis unused
+				 * and the space pointed to by REGS is 
+				 * used instead.
+				 */
+  
+  int last_l;			/* Highest index of a valid lparen. */
+  int last_r;			/* It's dual. */
+  
+  int * best_lparen;		/* This contains the best known register */
+  int * best_rparen;		/* assignments. 
+				 * This may point to the same mem as
+				 * best_lpspace, or it might point to memory
+				 * passed by the caller.
+				 */
+  int best_last_l;		/* best_last_l:best_lparen::last_l:lparen */
+  int best_last_r;
+
+
+  unsigned char * translate;  
+
+  struct rx_string_position outer_pos;
+
+  struct rx_superstate * start_super;
+  int nfa_choice;
+  int first_found;		/* If true, return after finding any match. */
+  int ret_val;
+
+  /* For continuations... */
+  enum rx_outer_entry outer_search_resume_pt;
+  struct re_pattern_buffer * saved_rxb;
+  int saved_startpos;
+  int saved_range;
+  int saved_stop;
+  int saved_total_size;
+  rx_get_burst_fn saved_get_burst;
+  rx_back_check_fn saved_back_check;
+  struct re_registers * saved_regs;
+  
+  /**
+   ** state for fastmap
+   **/
+  char * fastmap;
+  int fastmap_chr;
+  int fastmap_val;
+
+  /* for continuations in the fastmap procedure: */
+  enum rx_fastmap_entry fastmap_resume_pt;
+
+  /**
+   ** state for test_match 
+   **/
+
+  /* The current superNFA position of the matcher. */
+  struct rx_superstate * super;
+  
+  /* The matcher interprets a series of instruction frames.
+   * This is the `instruction counter' for the interpretation.
+   */
+  struct rx_inx * ifr;
+  
+  /* We insert a ghost character in the string to prime
+   * the nfa.  test_pos.pos, test_pos.str_half, and test_pos.end_half
+   * keep track of the test-match position and string-half.
+   */
+  unsigned char c;
+  
+  /* Position within the string. */
+  struct rx_string_position test_pos;
+
+  struct rx_stack_chunk * counter_stack;
+  struct rx_stack_chunk * backtrack_stack;
+  int backtrack_frame_bytes;
+  int chunk_bytes;
+  struct rx_stack_chunk * free_chunks;
+
+  /* To return from this function, set test_ret and 
+   * `goto test_do_return'.
+   *
+   * Possible return values are:
+   *     1   --- end of string while the superNFA is still going
+   *     0   --- internal error (out of memory)
+   *	-1   --- search completed by reaching the superNFA fail state
+   *    -2   --- a match was found, maybe not the longest.
+   *
+   * When the search is complete (-1), best_last_r indicates whether
+   * a match was found.
+   *
+   * -2 is return only if search_state.first_found is non-zero.
+   *
+   * if search_state.first_found is non-zero, a return of -1 indicates no match,
+   * otherwise, best_last_r has to be checked.
+   */
+  int test_ret;
+
+  int could_have_continued;
+  
+#ifdef RX_DEBUG
+  int backtrack_depth;
+  /* There is a search tree with every node as set of deterministic
+   * transitions in the super nfa.  For every branch of a 
+   * backtrack point is an edge in the tree.
+   * This counts up a pre-order of nodes in that tree.
+   * It's saved on the search stack and printed when debugging. 
+   */
+  int line_no;
+  int lines_found;
+#endif
+
+
+  /* For continuations within the match tester */
+  enum rx_test_match_entry test_match_resume_pt;
+  struct rx_inx * saved_next_tr_table;
+  struct rx_inx * saved_this_tr_table;
+  int saved_reg;
+  struct rx_backtrack_frame * saved_bf;
+  
+};
+
+
+
+extern char rx_slowmap[];
+extern unsigned char rx_id_translation[];
+
+static __inline__ void
+init_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  search_state->fastmap = (rxb->fastmap
+			   ? (char *)rxb->fastmap
+			   : (char *)rx_slowmap);
+  /* Update the fastmap now if not correct already. 
+   * When the regexp was compiled, the fastmap was computed
+   * and stored in a bitset.  This expands the bitset into a
+   * character array containing 1s and 0s.
+   */
+  if ((search_state->fastmap == rxb->fastmap) && !rxb->fastmap_accurate)
+    rx_blow_up_fastmap (rxb);
+  search_state->fastmap_chr = -1;
+  search_state->fastmap_val = 0;
+  search_state->fastmap_resume_pt = rx_fastmap_start;
+}
+
+static __inline__ void
+uninit_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  /* Unset the fastmap sentinel */
+  if (search_state->fastmap_chr >= 0)
+    search_state->fastmap[search_state->fastmap_chr]
+      = search_state->fastmap_val;
+}
+
+static __inline__ int
+fastmap_search (rxb, stop, get_burst, app_closure, search_state)
+     struct re_pattern_buffer * rxb;
+     int stop;
+     rx_get_burst_fn get_burst;
+     void * app_closure;
+     struct rx_search_state * search_state;
+{
+  enum rx_fastmap_entry pc;
+
+  if (0)
+    {
+    return_continuation:
+      search_state->fastmap_resume_pt = pc;
+      return rx_fastmap_continuation;
+    }
+
+  pc = search_state->fastmap_resume_pt;
+
+  switch (pc)
+    {
+    default:
+      return rx_fastmap_error;
+    case rx_fastmap_start:
+    init_fastmap_sentinal:
+      /* For the sake of fast fastmapping, set a sentinal in the fastmap.
+       * This sentinal will trap the fastmap loop when it reaches the last
+       * valid character in a string half.
+       *
+       * This must be reset when the fastmap/search loop crosses a string 
+       * boundry, and before returning to the caller.  So sometimes,
+       * the fastmap loop is restarted with `continue', othertimes by
+       * `goto init_fastmap_sentinal'.
+       */
+      if (search_state->outer_pos.size)
+	{
+	  search_state->fastmap_chr = ((search_state->outer_pos.search_direction == 1)
+				       ? *(search_state->outer_pos.end - 1)
+				       : *search_state->outer_pos.string);
+	  search_state->fastmap_val
+	    = search_state->fastmap[search_state->fastmap_chr];
+	  search_state->fastmap[search_state->fastmap_chr] = 1;
+	}
+      else
+	{
+	  search_state->fastmap_chr = -1;
+	  search_state->fastmap_val = 0;
+	}
+      
+      if (search_state->outer_pos.pos >= search_state->outer_pos.end)
+	goto fastmap_hit_bound;
+      else
+	{
+	  if (search_state->outer_pos.search_direction == 1)
+	    {
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      if (*search_state->outer_pos.pos != search_state->fastmap_chr)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  ++search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == search_state->outer_pos.end)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	  else
+	    {
+	      __const__ unsigned char * bound;
+	      bound = search_state->outer_pos.string - 1;
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      if ((*search_state->outer_pos.pos != search_state->fastmap_chr) || search_state->fastmap_val)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  --search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == bound)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	}
+      
+    case rx_fastmap_string_break:
+    fastmap_hit_bound:
+      {
+	/* If we hit a bound, it may be time to fetch another burst
+	 * of string, or it may be time to return a continuation to 
+ 	 * the caller, or it might be time to fail.
+	 */
+
+	int burst_state;
+	burst_state = get_burst (&search_state->outer_pos, app_closure, stop);
+	switch (burst_state)
+	  {
+	  default:
+	  case rx_get_burst_error:
+	    return rx_fastmap_error;
+	  case rx_get_burst_continuation:
+	    {
+	      pc = rx_fastmap_string_break;
+	      goto return_continuation;
+	    }
+	  case rx_get_burst_ok:
+	    goto init_fastmap_sentinal;
+	  case rx_get_burst_no_more:
+	    /* ...not a string split, simply no more string. 
+	     *
+	     * When searching backward, running out of string
+	     * is reason to quit.
+	     *
+	     * When searching forward, we allow the possibility
+	     * of an (empty) match after the last character in the
+	     * virtual string.  So, fall through to the matcher
+	     */
+	    return (  (search_state->outer_pos.search_direction == 1)
+		    ? rx_fastmap_ok
+		    : rx_fastmap_fail);
+	  }
+      }
+    }
+
+}
+
+
+
+
+#ifdef emacs
+/* The `emacs' switch turns on certain matching commands
+ * that make sense only in Emacs. 
+ */
+#include "sysfuncs.h"
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+#endif /* emacs */
+
+/* Setting RX_MEMDBUG is useful if you have dbmalloc.  Maybe with similar
+ * packages too.
+ */
+#ifdef RX_MEMDBUG
+#include <malloc.h>
+#endif /* RX_RX_MEMDBUG */
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+ * `BSTRING', as far as I know, and neither of them use this code.  
+ */
+#if HAVE_STRING_H || STDC_HEADERS
+#include <string.h>
+
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+
+#else /*  HAVE_STRING_H || STDC_HEADERS */
+#include <strings.h>
+#endif   /* not (HAVE_STRING_H || STDC_HEADERS) */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else /* not STDC_HEADERS */
+char *malloc ();
+char *realloc ();
+#endif /* not STDC_HEADERS */
+
+
+
+
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE (1 << CHARBITS)
+
+#ifndef emacs
+/* Define the syntax basics for \<, \>, etc.
+ * This must be nonzero for the wordchar and notwordchar pattern
+ * commands in re_match_2.
+ */
+#ifndef Sword 
+#define Sword 1
+#endif
+#define SYNTAX(c) re_syntax_table[c]
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else
+RX_DECL char re_syntax_table[CHAR_SET_SIZE];
+#endif
+#endif /* not emacs */
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+ *  of `string1' and `string2'.  If only one string, it's `string2'.  
+ */
+
+#define AT_STRINGS_BEG() \
+  (   -1		 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+#define AT_STRINGS_END() \
+  (   (total_size - 1)	 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+
+/* Test if POS + 1 points to a character which is word-constituent.  We have
+ * two special cases to check for: if past the end of string1, look at
+ * the first character in string2; and if before the beginning of
+ * string2, look at the last character in string1.
+ *
+ * Assumes `string1' exists, so use in conjunction with AT_STRINGS_BEG ().  
+ */
+#define LETTER_P(POS,OFF)						\
+  (   SYNTAX (fetch_char(POS, OFF, app_closure, stop))			\
+   == Sword)
+
+/* Test if the character at D and the one after D differ with respect
+ * to being word-constituent.  
+ */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG () || AT_STRINGS_END () || LETTER_P (d,0) != LETTER_P (d, 1))
+
+
+#ifdef RX_SUPPORT_CONTINUATIONS
+#define RX_STACK_ALLOC(BYTES) malloc(BYTES)
+#define RX_STACK_FREE(MEM) free(MEM)
+#else
+#define RX_STACK_ALLOC(BYTES) alloca(BYTES)
+#define RX_STACK_FREE(MEM) \
+      ((struct rx_stack_chunk *)MEM)->next_chunk = search_state.free_chunks; \
+      search_state.free_chunks = ((struct rx_stack_chunk *)MEM);
+
+#endif
+
+#define PUSH(CHUNK_VAR,BYTES)   \
+  if (!CHUNK_VAR || (CHUNK_VAR->bytes_left < (BYTES)))  \
+    {					\
+      struct rx_stack_chunk * new_chunk;	\
+      if (search_state.free_chunks)			\
+	{				\
+	  new_chunk = search_state.free_chunks;	\
+	  search_state.free_chunks = search_state.free_chunks->next_chunk; \
+	}				\
+      else				\
+	{				\
+	  new_chunk = (struct rx_stack_chunk *)RX_STACK_ALLOC(search_state.chunk_bytes); \
+	  if (!new_chunk)		\
+	    {				\
+	      search_state.ret_val = 0;		\
+	      goto test_do_return;	\
+	    }				\
+	}				\
+      new_chunk->sp = (char *)new_chunk + sizeof (struct rx_stack_chunk); \
+      new_chunk->bytes_left = (search_state.chunk_bytes \
+			       - (BYTES) \
+			       - sizeof (struct rx_stack_chunk)); \
+      new_chunk->next_chunk = CHUNK_VAR; \
+      CHUNK_VAR = new_chunk;		\
+    } \
+  else \
+    (CHUNK_VAR->sp += (BYTES)), (CHUNK_VAR->bytes_left -= (BYTES))
+
+#define POP(CHUNK_VAR,BYTES) \
+  if (CHUNK_VAR->sp == ((char *)CHUNK_VAR + sizeof(*CHUNK_VAR))) \
+    { \
+      struct rx_stack_chunk * new_chunk = CHUNK_VAR->next_chunk; \
+      RX_STACK_FREE(CHUNK_VAR); \
+      CHUNK_VAR = new_chunk; \
+    } \
+  else \
+    (CHUNK_VAR->sp -= BYTES), (CHUNK_VAR->bytes_left += BYTES)
+
+
+
+#define SRCH_TRANSLATE(C)  search_state.translate[(unsigned char) (C)]
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL __inline__ int
+rx_search  (struct re_pattern_buffer * rxb,
+	    int startpos,
+	    int range,
+	    int stop,
+	    int total_size,
+	    rx_get_burst_fn get_burst,
+	    rx_back_check_fn back_check,
+	    rx_fetch_char_fn fetch_char,
+	    void * app_closure,
+	    struct re_registers * regs,
+	    struct rx_search_state * resume_state,
+	    struct rx_search_state * save_state)
+#else
+RX_DECL __inline__ int
+rx_search  (rxb, startpos, range, stop, total_size,
+	    get_burst, back_check, fetch_char,
+	    app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+#endif
+{
+  int pc;
+  int test_state;
+  struct rx_search_state search_state;
+
+  search_state.free_chunks = 0;
+  if (!resume_state)
+    pc = rx_outer_start;
+  else
+    {
+      search_state = *resume_state;
+      regs = search_state.saved_regs;
+      rxb = search_state.saved_rxb;
+      startpos = search_state.saved_startpos;
+      range = search_state.saved_range;
+      stop = search_state.saved_stop;
+      total_size = search_state.saved_total_size;
+      get_burst = search_state.saved_get_burst;
+      back_check = search_state.saved_back_check;
+      pc = search_state.outer_search_resume_pt;
+      if (0)
+	{
+	return_continuation:
+	  if (save_state)
+	    {
+	      *save_state = search_state;
+	      save_state->saved_regs = regs;
+	      save_state->saved_rxb = rxb;
+	      save_state->saved_startpos = startpos;
+	      save_state->saved_range = range;
+	      save_state->saved_stop = stop;
+	      save_state->saved_total_size = total_size;
+	      save_state->saved_get_burst = get_burst;
+	      save_state->saved_back_check = back_check;
+	      save_state->outer_search_resume_pt = pc;
+	    }
+	  return rx_search_continuation;
+	}
+    }
+
+  switch (pc)
+    {
+    case rx_outer_start:
+      search_state.ret_val = rx_search_fail;
+      (  search_state.lparen
+       = search_state.rparen
+       = search_state.best_lpspace
+       = search_state.best_rpspace
+       = 0);
+      
+      /* figure the number of registers we may need for use in backreferences.
+       * the number here includes an element for register zero.  
+       */
+      search_state.num_regs = rxb->re_nsub + 1;
+      
+      
+      /* check for out-of-range startpos.  */
+      if ((startpos < 0) || (startpos > total_size))
+	return rx_search_fail;
+      
+      /* fix up range if it might eventually take us outside the string. */
+      {
+	int endpos;
+	endpos = startpos + range;
+	if (endpos < -1)
+	  range = (-1 - startpos);
+	else if (endpos > (total_size + 1))
+	  range = total_size - startpos;
+      }
+      
+      /* if the search isn't to be a backwards one, don't waste time in a
+       * long search for a pattern that says it is anchored.
+       */
+      if (rxb->begbuf_only && (range > 0))
+	{
+	  if (startpos > 0)
+	    return rx_search_fail;
+	  else
+	    range = 1;
+	}
+      
+      /* decide whether to use internal or user-provided reg buffers. */
+      if (!regs || rxb->no_sub)
+	{
+	  search_state.best_lpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_rpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_lparen = search_state.best_lpspace;
+	  search_state.best_rparen = search_state.best_rpspace;
+	}
+      else
+	{	
+	  /* have the register data arrays been allocated?  */
+	  if (rxb->regs_allocated == REGS_UNALLOCATED)
+	    { /* no.  so allocate them with malloc.  we need one
+		 extra element beyond `search_state.num_regs' for the `-1' marker
+		 gnu code uses.  */
+	      regs->num_regs = MAX (RE_NREGS, rxb->re_nsub + 1);
+	      regs->start = ((regoff_t *)
+			     malloc (regs->num_regs * sizeof ( regoff_t)));
+	      regs->end = ((regoff_t *)
+			   malloc (regs->num_regs * sizeof ( regoff_t)));
+	      if (regs->start == 0 || regs->end == 0)
+		return rx_search_error;
+	      rxb->regs_allocated = REGS_REALLOCATE;
+	    }
+	  else if (rxb->regs_allocated == REGS_REALLOCATE)
+	    { /* yes.  if we need more elements than were already
+		 allocated, reallocate them.  if we need fewer, just
+		 leave it alone.  */
+	      if (regs->num_regs < search_state.num_regs + 1)
+		{
+		  regs->num_regs = search_state.num_regs + 1;
+		  regs->start = ((regoff_t *)
+				 realloc (regs->start,
+					  regs->num_regs * sizeof (regoff_t)));
+		  regs->end = ((regoff_t *)
+			       realloc (regs->end,
+					regs->num_regs * sizeof ( regoff_t)));
+		  if (regs->start == 0 || regs->end == 0)
+		    return rx_search_error;
+		}
+	    }
+	  else if (rxb->regs_allocated != REGS_FIXED)
+	    return rx_search_error;
+	  
+	  if (regs->num_regs < search_state.num_regs + 1)
+	    {
+	      search_state.best_lpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_rpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_lparen = search_state.best_lpspace;
+	      search_state.best_rparen = search_state.best_rpspace;
+	    }
+	  else
+	    {
+	      search_state.best_lparen = regs->start;
+	      search_state.best_rparen = regs->end;
+	    }
+	}
+      
+      search_state.lparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+      search_state.rparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)); 
+      
+      if (! (   search_state.best_rparen
+	     && search_state.best_lparen
+	     && search_state.lparen && search_state.rparen))
+	return rx_search_error;
+      
+      search_state.best_last_l = search_state.best_last_r = -1;
+      
+      search_state.translate = (rxb->translate
+				? rxb->translate
+				: rx_id_translation);
+      
+      
+      
+      /*
+       * two nfa's were compiled.  
+       * `0' is complete.
+       * `1' faster but gets registers wrong and ends too soon.
+       */
+      search_state.nfa_choice = (regs && !rxb->least_subs) ? '\0' : '\1';
+      
+      /* we have the option to look for the best match or the first
+       * one we can find.  if the user isn't asking for register information,
+       * we don't need to find the best match.
+       */
+      search_state.first_found = !regs;
+      
+      if (range >= 0)
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = 1;
+	}
+      else
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = -1;
+	}
+      
+      /* the vacuous search always turns up nothing. */
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos > search_state.outer_pos.search_end)
+	  : (startpos < search_state.outer_pos.search_end))
+	return rx_search_fail;
+      
+      /* now we build the starting state of the supernfa. */
+      {
+	struct rx_superset * start_contents;
+	struct rx_nfa_state_set * start_nfa_set;
+	
+	/* we presume here that the nfa start state has only one
+	 * possible future with no side effects.  
+	 */
+	start_nfa_set = rxb->start->futures->destset;
+	if (   rxb->rx.start_set
+	    && (rxb->rx.start_set->starts_for == &rxb->rx))
+	  start_contents = rxb->rx.start_set;
+	else
+	  {
+	    start_contents =
+	      rx_superstate_eclosure_union (&rxb->rx,
+					    rx_superset_cons (&rxb->rx, 0, 0),
+					    start_nfa_set);
+	    
+	    if (!start_contents)
+	      return rx_search_fail;
+	    
+	    start_contents->starts_for = &rxb->rx;
+	    rxb->rx.start_set = start_contents;
+	  }
+	if (   start_contents->superstate
+	    && (start_contents->superstate->rx_id == rxb->rx.rx_id))
+	  {
+	    search_state.start_super = start_contents->superstate;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	  }
+	else
+	  {
+	    rx_protect_superset (&rxb->rx, start_contents);
+	    
+	    search_state.start_super = rx_superstate (&rxb->rx, start_contents);
+	    if (!search_state.start_super)
+	      return rx_search_fail;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	    rx_release_superset (&rxb->rx, start_contents);
+	  }
+      }
+      
+      
+      /* The outer_pos tracks the position within the strings
+       * as seen by loop that calls fastmap_search.
+       *
+       * The caller supplied get_burst function actually 
+       * gives us pointers to chars.
+       * 
+       * Communication with the get_burst function is through an
+       * rx_string_position structure.  Here, the structure for
+       * outer_pos is initialized.   It is set to point to the
+       * NULL string, at an offset of STARTPOS.  STARTPOS is out
+       * of range of the NULL string, so the first call to 
+       * getburst will patch up the rx_string_position to point
+       * to valid characters.
+       */
+
+      (  search_state.outer_pos.string
+       = search_state.outer_pos.end
+       = 0);
+
+      search_state.outer_pos.offset = 0;
+      search_state.outer_pos.size = 0;
+      search_state.outer_pos.pos = (unsigned char *)startpos;
+      init_fastmap (rxb, &search_state);
+
+      search_state.fastmap_resume_pt = rx_fastmap_start;
+    case rx_outer_fastmap:
+      /* do { */
+    pseudo_do:
+      {
+	{
+	  int fastmap_state;
+	  fastmap_state = fastmap_search (rxb, stop, get_burst, app_closure,
+					  &search_state);
+	  switch (fastmap_state)
+	    {
+	    case rx_fastmap_continuation:
+	      pc = rx_outer_fastmap;
+	      goto return_continuation;
+	    case rx_fastmap_fail:
+	      goto finish;
+	    case rx_fastmap_ok:
+	      break;
+	    }
+	}
+	
+	/* now the fastmap loop has brought us to a plausible 
+	 * starting point for a match.  so, it's time to run the
+	 * nfa and see if a match occured.
+	 */
+	startpos = (  search_state.outer_pos.pos
+		    - search_state.outer_pos.string
+		    + search_state.outer_pos.offset);
+/*|*/	if ((range > 0) && (startpos == search_state.outer_pos.search_end))
+/*|*/	  goto finish;
+      }
+
+      search_state.test_match_resume_pt = rx_test_start;
+      /* do interrupted for entry point... */
+    case rx_outer_test:
+      /* ...do continued */
+      {
+	goto test_match;
+      test_returns_to_search:
+	switch (test_state)
+	  {
+	  case rx_test_continuation:
+	    pc = rx_outer_test;
+	    goto return_continuation;
+	  case rx_test_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_test_fail:
+	    break;
+	  case rx_test_ok:
+	    goto finish;
+	  }
+	search_state.outer_pos.pos += search_state.outer_pos.search_direction;
+	startpos += search_state.outer_pos.search_direction;
+#if 0
+/*|*/	if (search_state.test_pos.pos < search_state.test_pos.end)
+/*|*/	  break;
+#endif
+      }
+      /* do interrupted for entry point... */
+    case rx_outer_restore_pos:
+      {
+	int x;
+	x = get_burst (&search_state.outer_pos, app_closure, stop);
+	switch (x)
+	  {
+	  case rx_get_burst_continuation:
+	    pc = rx_outer_restore_pos;
+	    goto return_continuation;
+	  case rx_get_burst_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_get_burst_no_more:
+	    if (rxb->can_match_empty)
+	      break;
+	    goto finish;
+	  case rx_get_burst_ok:
+	    break;
+	  }
+      } /* } while (...see below...) */
+
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos < search_state.outer_pos.search_end)
+	  : (startpos > search_state.outer_pos.search_end))
+	goto pseudo_do;
+
+	
+    finish:
+      uninit_fastmap (rxb, &search_state);
+      if (search_state.start_super)
+	rx_unlock_superstate (&rxb->rx, search_state.start_super);
+      
+#ifdef regex_malloc
+      if (search_state.lparen) free (search_state.lparen);
+      if (search_state.rparen) free (search_state.rparen);
+      if (search_state.best_lpspace) free (search_state.best_lpspace);
+      if (search_state.best_rpspace) free (search_state.best_rpspace);
+#endif
+      return search_state.ret_val;
+    }
+
+
+ test_match:
+  {
+    enum rx_test_match_entry test_pc;
+    int inx;
+    test_pc = search_state.test_match_resume_pt;
+    if (test_pc == rx_test_start)
+      {
+#ifdef RX_DEBUG
+	search_state.backtrack_depth = 0;
+#endif
+	search_state.last_l = search_state.last_r = 0;
+	search_state.lparen[0] = startpos;
+	search_state.super = search_state.start_super;
+	search_state.c = search_state.nfa_choice;
+	search_state.test_pos.pos = search_state.outer_pos.pos - 1;    
+	search_state.test_pos.string = search_state.outer_pos.string;
+	search_state.test_pos.end = search_state.outer_pos.end;
+	search_state.test_pos.offset = search_state.outer_pos.offset;
+	search_state.test_pos.size = search_state.outer_pos.size;
+	search_state.test_pos.search_direction = 1;
+	search_state.counter_stack = 0;
+	search_state.backtrack_stack = 0;
+	search_state.backtrack_frame_bytes =
+	  (sizeof (struct rx_backtrack_frame)
+	   + (rxb->match_regs_on_stack
+	      ? sizeof (regoff_t) * (search_state.num_regs + 1) * 2
+	      : 0));
+	search_state.chunk_bytes = search_state.backtrack_frame_bytes * 64;
+	search_state.test_ret = rx_test_line_finished;
+	search_state.could_have_continued = 0;
+      }  
+    /* This is while (1)...except that the body of the loop is interrupted 
+     * by some alternative entry points.
+     */
+  pseudo_while_1:
+    switch (test_pc)
+      {
+      case rx_test_cache_hit_loop:
+	goto resume_continuation_1;
+      case rx_test_backreference_check:
+	goto resume_continuation_2;
+      case rx_test_backtrack_return:
+	goto resume_continuation_3;
+      case rx_test_start:
+#ifdef RX_DEBUG
+	/* There is a search tree with every node as set of deterministic
+	 * transitions in the super nfa.  For every branch of a 
+	 * backtrack point is an edge in the tree.
+	 * This counts up a pre-order of nodes in that tree.
+	 * It's saved on the search stack and printed when debugging. 
+	 */
+	search_state.line_no = 0;
+	search_state.lines_found = 0;
+#endif
+	
+      top_of_cycle:
+	/* A superstate is basicly a transition table, indexed by 
+	 * characters from the string being tested, and containing 
+	 * RX_INX (`instruction frame') structures.
+	 */
+	search_state.ifr = &search_state.super->transitions [search_state.c];
+	
+      recurse_test_match:
+	/* This is the point to which control is sent when the
+	 * test matcher `recurses'.  Before jumping here, some variables
+	 * need to be saved on the stack and the next instruction frame
+	 * has to be computed.
+	 */
+	
+      restart:
+	/* Some instructions don't advance the matcher, but just
+	 * carry out some side effects and fetch a new instruction.
+	 * To dispatch that new instruction, `goto restart'.
+	 */
+	
+	{
+	  struct rx_inx * next_tr_table;
+	  struct rx_inx * this_tr_table;
+	  /* The fastest route through the loop is when the instruction 
+	   * is RX_NEXT_CHAR.  This case is detected when SEARCH_STATE.IFR->DATA
+	   * is non-zero.  In that case, it points to the next
+	   * superstate. 
+	   *
+	   * This allows us to not bother fetching the bytecode.
+	   */
+	  next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	  this_tr_table = search_state.super->transitions;
+	  while (next_tr_table)
+	    {
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		{
+		  struct rx_superset * setp;
+		  
+		  fprintf (stderr, "%d %d>> re_next_char @ %d (%d)",
+			   search_state.line_no,
+			   search_state.backtrack_depth,
+			   (search_state.test_pos.pos - search_state.test_pos.string
+			    + search_state.test_pos.offset), search_state.c);
+		  
+		  search_state.super =
+		    ((struct rx_superstate *)
+		     ((char *)this_tr_table
+		      - ((unsigned long)
+			 ((struct rx_superstate *)0)->transitions)));
+		  
+		  setp = search_state.super->contents;
+		  fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+			   rxb->rx.rx_id, setp);
+		  while (setp)
+		    {
+		      fprintf (stderr, "%d ", setp->id);
+		      setp = setp->cdr;
+		    }
+		  fprintf (stderr, "\n");
+		}
+#endif
+	      this_tr_table = next_tr_table;
+	      ++search_state.test_pos.pos;
+	      if (search_state.test_pos.pos == search_state.test_pos.end)
+		{
+		  int burst_state;
+		try_burst_1:
+		  burst_state = get_burst (&search_state.test_pos,
+					   app_closure, stop);
+		  switch (burst_state)
+		    {
+		    case rx_get_burst_continuation:
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      test_pc = rx_test_cache_hit_loop;
+		      goto test_return_continuation;
+		      
+		    resume_continuation_1:
+		      /* Continuation one jumps here to do its work: */
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      goto try_burst_1;
+		      
+		    case rx_get_burst_ok:
+		      /* get_burst succeeded...keep going */
+		      break;
+		      
+		    case rx_get_burst_no_more:
+		      search_state.test_ret = rx_test_line_finished;
+		      search_state.could_have_continued = 1;
+		      goto test_do_return;
+		      
+		    case rx_get_burst_error:
+		      /* An error... */
+		      search_state.test_ret = rx_test_internal_error;
+		      goto test_do_return;
+		    }
+		}
+	      search_state.c = *search_state.test_pos.pos;
+	      search_state.ifr = this_tr_table + search_state.c;
+	      next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	    } /* Fast loop through cached transition tables */
+	  
+	  /* Here when we ran out of cached next-char transitions. 
+	   * So, it will be necessary to do a more expensive
+	   * dispatch on the current instruction.  The superstate
+	   * pointer is allowed to become invalid during next-char
+	   * transitions -- now we must bring it up to date.
+	   */
+	  search_state.super =
+	    ((struct rx_superstate *)
+	     ((char *)this_tr_table
+	      - ((unsigned long)
+		 ((struct rx_superstate *)0)->transitions)));
+	}
+	
+	/* We've encountered an instruction other than next-char.
+	 * Dispatch that instruction:
+	 */
+	inx = (int)search_state.ifr->inx;
+#ifdef RX_DEBUG_0
+	if (rx_debug_trace)
+	  {
+	    struct rx_superset * setp = search_state.super->contents;
+	    
+	    fprintf (stderr, "%d %d>> %s @ %d (%d)", search_state.line_no,
+		     search_state.backtrack_depth,
+		     inx_names[inx],
+		     (search_state.test_pos.pos - search_state.test_pos.string
+		      + (test_pos.half == 0 ? 0 : size1)), search_state.c);
+	    
+	    fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+		     rxb->rx.rx_id, setp);
+	    while (setp)
+	      {
+		fprintf (stderr, "%d ", setp->id);
+		setp = setp->cdr;
+	      }
+	    fprintf (stderr, "\n");
+	  }
+#endif
+	switch ((enum rx_opcode)inx)
+	  {
+	  case rx_do_side_effects:
+	    
+	    /*  RX_DO_SIDE_EFFECTS occurs when we cross epsilon 
+	     *  edges associated with parentheses, backreferencing, etc.
+	     */
+	    {
+	      struct rx_distinct_future * df =
+		(struct rx_distinct_future *)search_state.ifr->data_2;
+	      struct rx_se_list * el = df->effects;
+	      /* Side effects come in lists.  This walks down
+	       * a list, dispatching.
+	       */
+	      while (el)
+		{
+		  long effect;
+		  effect = (long)el->car;
+		  if (effect < 0)
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			{
+			  struct rx_superset * setp = search_state.super->contents;
+			  
+			  fprintf (stderr, "....%d %d>> %s\n", search_state.line_no,
+				   search_state.backtrack_depth,
+				   efnames[-effect]);
+			}
+#endif
+		      switch ((enum re_side_effects) effect)
+
+			{
+			case re_se_pushback:
+			  search_state.ifr = &df->future_frame;
+			  if (!search_state.ifr->data)
+			    {
+			      struct rx_superstate * sup;
+			      sup = search_state.super;
+			      rx_lock_superstate (rx, sup);
+			      if (!rx_handle_cache_miss (&rxb->rx,
+							 search_state.super,
+							 search_state.c,
+							 (search_state.ifr
+							  ->data_2)))
+				{
+				  rx_unlock_superstate (rx, sup);
+				  search_state.test_ret = rx_test_internal_error;
+				  goto test_do_return;
+				}
+			      rx_unlock_superstate (rx, sup);
+			    }
+			  /* --search_state.test_pos.pos; */
+			  search_state.c = 't';
+			  search_state.super
+			    = ((struct rx_superstate *)
+			       ((char *)search_state.ifr->data
+				- (long)(((struct rx_superstate *)0)
+					 ->transitions)));
+			  goto top_of_cycle;
+			  break;
+			case re_se_push0:
+			  {
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH (search_state.counter_stack,
+				  sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_iter;
+			    cf->val = 0;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			case re_se_fail:
+			  goto test_do_return;
+			case re_se_begbuf:
+			  if (!AT_STRINGS_BEG ())
+			    goto test_do_return;
+			  break;
+			case re_se_endbuf:
+			  if (!AT_STRINGS_END ())
+			    goto test_do_return;
+			  break;
+			case re_se_wordbeg:
+			  if (   LETTER_P (&search_state.test_pos, 1)
+			      && (   AT_STRINGS_BEG()
+				  || !LETTER_P (&search_state.test_pos, 0)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordend:
+			  if (   !AT_STRINGS_BEG ()
+			      && LETTER_P (&search_state.test_pos, 0)
+			      && (AT_STRINGS_END ()
+				  || !LETTER_P (&search_state.test_pos, 1)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordbound:
+			  if (AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_notwordbound:
+			  if (!AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_hat:
+			  if (AT_STRINGS_BEG ())
+			    {
+			      if (rxb->not_bol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      char pos_c = *search_state.test_pos.pos;
+			      if (   (SRCH_TRANSLATE (pos_c)
+				      == SRCH_TRANSLATE('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			case re_se_dollar:
+			  if (AT_STRINGS_END ())
+			    {
+			      if (rxb->not_eol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      if (   (   SRCH_TRANSLATE (fetch_char
+						    (&search_state.test_pos, 1,
+						     app_closure, stop))
+				      == SRCH_TRANSLATE ('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			  
+			case re_se_try:
+			  /* This is the first side effect in every
+			   * expression.
+			   *
+			   *  FOR NO GOOD REASON...get rid of it...
+			   */
+			  break;
+			  
+			case re_se_pushpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH(search_state.counter_stack,
+				 sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_pushpos;
+			    cf->val = urhere;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			  
+			case re_se_chkpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * cf
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (cf->val == urhere)
+			      goto test_do_return;
+			    cf->val = urhere;
+			    break;
+			  }
+			  break;
+			  
+			case re_se_poppos:
+			  POP(search_state.counter_stack,
+			      sizeof (struct rx_counter_frame));
+			  break;
+			  
+			  
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+#ifdef emacs
+			  /* 
+			   * this release lacks emacs support
+			   */
+#endif
+			  break;
+			case re_se_win:
+			case re_se_lparen:
+			case re_se_rparen:
+			case re_se_backref:
+			case re_se_iter:
+			case re_se_end_iter:
+			case re_se_tv:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  else
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			fprintf (stderr, "....%d %d>> %s %d %d\n", search_state.line_no,
+				 search_state.backtrack_depth,
+				 efnames2[rxb->se_params [effect].se],
+				 rxb->se_params [effect].op1,
+				 rxb->se_params [effect].op2);
+#endif
+		      switch (rxb->se_params [effect].se)
+			{
+			case re_se_win:
+			  /* This side effect indicates that we've 
+			   * found a match, though not necessarily the 
+			   * best match.  This is a fancy assignment to 
+			   * register 0 unless the caller didn't 
+			   * care about registers.  In which case,
+			   * this stops the match.
+			   */
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    if (   (search_state.best_last_r < 0)
+				|| (urhere + 1 > search_state.best_rparen[0]))
+			      {
+				/* Record the best known and keep
+				 * looking.
+				 */
+				int x;
+				for (x = 0; x <= search_state.last_l; ++x)
+				  search_state.best_lparen[x] = search_state.lparen[x];
+				search_state.best_last_l = search_state.last_l;
+				for (x = 0; x <= search_state.last_r; ++x)
+				  search_state.best_rparen[x] = search_state.rparen[x];
+				search_state.best_rparen[0] = urhere + 1;
+				search_state.best_last_r = search_state.last_r;
+			      }
+			    /* If we're not reporting the match-length 
+			     * or other register info, we need look no
+			     * further.
+			     */
+			    if (search_state.first_found)
+			      {
+				search_state.test_ret = rx_test_found_first;
+				goto test_do_return;
+			      }
+			  }
+			  break;
+			case re_se_lparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    int reg = rxb->se_params [effect].op1;
+#if 0
+			    if (reg > search_state.last_l)
+#endif
+			      {
+				search_state.lparen[reg] = urhere + 1;
+				/* In addition to making this assignment,
+				 * we now know that lower numbered regs
+				 * that haven't already been assigned,
+				 * won't be.  We make sure they're
+				 * filled with -1, so they can be
+				 * recognized as unassigned.
+				 */
+				if (search_state.last_l < reg)
+				  while (++search_state.last_l < reg)
+				    search_state.lparen[search_state.last_l] = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_rparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    int reg = rxb->se_params [effect].op1;
+			    search_state.rparen[reg] = urhere + 1;
+			    if (search_state.last_r < reg)
+			      {
+				while (++search_state.last_r < reg)
+				  search_state.rparen[search_state.last_r]
+				    = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_backref:
+			  {
+			    int reg = rxb->se_params [effect].op1;
+			    if (   reg > search_state.last_r
+				|| search_state.rparen[reg] < 0)
+			      goto test_do_return;
+			    
+			    {
+			      int backref_status;
+			    check_backreference:
+			      backref_status
+				= back_check (&search_state.test_pos,
+					      search_state.lparen[reg],
+					      search_state.rparen[reg],
+					      search_state.translate,
+					      app_closure,
+					      stop);
+			      switch (backref_status)
+				{
+				case rx_back_check_continuation:
+				  search_state.saved_reg = reg;
+				  test_pc = rx_test_backreference_check;
+				  goto test_return_continuation;
+				resume_continuation_2:
+				  reg = search_state.saved_reg;
+				  goto check_backreference;
+				case rx_back_check_fail:
+				  /* Fail */
+				  goto test_do_return;
+				case rx_back_check_pass:
+				  /* pass --
+				   * test_pos now advanced to last
+				   * char matched by backref
+				   */
+				  break;
+				}
+			    }
+			    break;
+			  }
+			case re_se_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val == rxb->se_params[effect].op2)
+			      goto test_do_return;
+			    else
+			      ++csp->val;
+			    break;
+			  }
+			case re_se_end_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val < rxb->se_params[effect].op1)
+			      goto test_do_return;
+			    else
+			      {
+				struct rx_counter_frame * source = csp;
+				while (source->inherited_from)
+				  source = source->inherited_from;
+				if (!source || !source->cdr)
+				  {
+				    POP(search_state.counter_stack,
+					sizeof(struct rx_counter_frame));
+				  }
+				else
+				  {
+				    source = source->cdr;
+				    csp->val = source->val;
+				    csp->tag = source->tag;
+				    csp->cdr = 0;
+				    csp->inherited_from = source;
+				  }
+			      }
+			    break;
+			  }
+			case re_se_tv:
+			  /* is a noop */
+			  break;
+			case re_se_try:
+			case re_se_pushback:
+			case re_se_push0:
+			case re_se_pushpos:
+			case re_se_chkpos:
+			case re_se_poppos:
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+			case re_se_begbuf:
+			case re_se_hat:
+			case re_se_wordbeg:
+			case re_se_wordbound:
+			case re_se_notwordbound:
+			case re_se_wordend:
+			case re_se_endbuf:
+			case re_se_dollar:
+			case re_se_fail:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  el = el->cdr;
+		}
+	      /* Now the side effects are done,
+	       * so get the next instruction.
+	       * and move on.
+	       */
+	      search_state.ifr = &df->future_frame;
+	      goto restart;
+	    }
+	    
+	  case rx_backtrack_point:
+	    {
+	      /* A backtrack point indicates that we've reached a
+	       * non-determinism in the superstate NFA.  This is a
+	       * loop that exhaustively searches the possibilities.
+	       *
+	       * A backtracking strategy is used.  We keep track of what
+	       * registers are valid so we can erase side effects.
+	       *
+	       * First, make sure there is some stack space to hold 
+	       * our state.
+	       */
+	      
+	      struct rx_backtrack_frame * bf;
+	      
+	      PUSH(search_state.backtrack_stack,
+		   search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG_0
+	      ++search_state.backtrack_depth;
+#endif
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+	      {
+		bf->stk_super = search_state.super;
+		/* We prevent the current superstate from being
+		 * deleted from the superstate cache.
+		 */
+		rx_lock_superstate (&rxb->rx, search_state.super);
+#ifdef RX_DEBUG_0
+		bf->stk_search_state.line_no = search_state.line_no;
+#endif
+		bf->stk_c = search_state.c;
+		bf->stk_test_pos = search_state.test_pos;
+		bf->stk_last_l = search_state.last_l;
+		bf->stk_last_r = search_state.last_r;
+		bf->df = ((struct rx_super_edge *)
+			  search_state.ifr->data_2)->options;
+		bf->first_df = bf->df;
+		bf->counter_stack_sp = (search_state.counter_stack
+					? search_state.counter_stack->sp
+					: 0);
+		bf->stk_test_ret = search_state.test_ret;
+		if (rxb->match_regs_on_stack)
+		  {
+		    int x;
+		    regoff_t * stk =
+		      (regoff_t *)((char *)bf + sizeof (*bf));
+		    for (x = 0; x <= search_state.last_l; ++x)
+		      stk[x] = search_state.lparen[x];
+		    stk += x;
+		    for (x = 0; x <= search_state.last_r; ++x)
+		      stk[x] = search_state.rparen[x];
+		  }
+	      }
+	      
+	      /* Here is a while loop whose body is mainly a function
+	       * call and some code to handle a return from that
+	       * function.
+	       *
+	       * From here on for the rest of `case backtrack_point' it
+	       * is unsafe to assume that the search_state copies of 
+	       * variables saved on the backtracking stack are valid
+	       * -- so read their values from the backtracking stack.
+	       *
+	       * This lets us use one generation fewer stack saves in
+	       * the call-graph of a search.
+	       */
+	      
+	    while_non_det_options:
+#ifdef RX_DEBUG_0
+	      ++search_state.lines_found;
+	      if (rx_debug_trace)
+		fprintf (stderr, "@@@ %d calls %d @@@\n",
+			 search_state.line_no, search_state.lines_found);
+	      
+	      search_state.line_no = search_state.lines_found;
+#endif
+	      
+	      if (bf->df->next_same_super_edge[0] == bf->first_df)
+		{
+		  /* This is a tail-call optimization -- we don't recurse
+		   * for the last of the possible futures.
+		   */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  
+		  rx_unlock_superstate (&rxb->rx, search_state.super);
+		  POP(search_state.backtrack_stack,
+		      search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG
+		  --search_state.backtrack_depth;
+#endif
+		  goto restart;
+		}
+	      else
+		{
+		  if (search_state.counter_stack)
+		    {
+		      struct rx_counter_frame * old_cf
+			= ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      struct rx_counter_frame * cf;
+		      PUSH(search_state.counter_stack, sizeof (struct rx_counter_frame));
+		      cf = ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      cf->tag = old_cf->tag;
+		      cf->val = old_cf->val;
+		      cf->inherited_from = old_cf;
+		      cf->cdr = 0;
+		    }			
+		  /* `Call' this test-match block */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  goto recurse_test_match;
+		}
+	      
+	      /* Returns in this block are accomplished by
+	       * goto test_do_return.  There are two cases.
+	       * If there is some search-stack left,
+	       * then it is a return from a `recursive' call.
+	       * If there is no search-stack left, then
+	       * we should return to the fastmap/search loop.
+	       */
+	      
+	    test_do_return:
+	      
+	      if (!search_state.backtrack_stack)
+		{
+#ifdef RX_DEBUG_0
+		  if (rx_debug_trace)
+		    fprintf (stderr, "!!! %d bails returning %d !!!\n",
+			     search_state.line_no, search_state.test_ret);
+#endif
+		  
+		  /* No more search-stack -- this test is done. */
+		  if (search_state.test_ret)
+		    goto return_from_test_match;
+		  else
+		    goto error_in_testing_match;
+		}
+	      
+	      /* Returning from a recursive call to 
+	       * the test match block:
+	       */
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		fprintf (stderr, "+++ %d returns %d (to %d)+++\n",
+			 search_state.line_no,
+			 search_state.test_ret,
+			 bf->stk_search_state.line_no);
+#endif
+	      
+	      while (search_state.counter_stack
+		     && (!bf->counter_stack_sp
+			 || (bf->counter_stack_sp
+			     != search_state.counter_stack->sp)))
+		{
+		  POP(search_state.counter_stack,
+		      sizeof (struct rx_counter_frame));
+		}
+	      
+	      if (search_state.test_ret == rx_test_error)
+		{
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      /* If a non-longest match was found and that is good 
+	       * enough, return immediately.
+	       */
+	      if (   (search_state.test_ret == rx_test_found_first)
+		  && search_state.first_found)
+		{
+		  rx_unlock_superstate (&rxb->rx, bf->stk_super);
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      search_state.test_ret = bf->stk_test_ret;
+	      search_state.last_l = bf->stk_last_l;
+	      search_state.last_r = bf->stk_last_r;
+	      bf->df = bf->df->next_same_super_edge[0];
+	      search_state.super = bf->stk_super;
+	      search_state.c = bf->stk_c;
+#ifdef RX_DEBUG_0
+	      search_state.line_no = bf->stk_search_state.line_no;
+#endif
+	      
+	      if (rxb->match_regs_on_stack)
+		{
+		  int x;
+		  regoff_t * stk =
+		    (regoff_t *)((char *)bf + sizeof (*bf));
+		  for (x = 0; x <= search_state.last_l; ++x)
+		    search_state.lparen[x] = stk[x];
+		  stk += x;
+		  for (x = 0; x <= search_state.last_r; ++x)
+		    search_state.rparen[x] = stk[x];
+		}
+	      
+	      {
+		int x;
+	      try_burst_2:
+		x = get_burst (&bf->stk_test_pos, app_closure, stop);
+		switch (x)
+		  {
+		  case rx_get_burst_continuation:
+		    search_state.saved_bf = bf;
+		    test_pc = rx_test_backtrack_return;
+		    goto test_return_continuation;
+		  resume_continuation_3:
+		    bf = search_state.saved_bf;
+		    goto try_burst_2;
+		  case rx_get_burst_no_more:
+		    /* Since we've been here before, it is some kind of
+		     * error that we can't return.
+		     */
+		  case rx_get_burst_error:
+		    search_state.test_ret = rx_test_internal_error;
+		    goto test_do_return;
+		  case rx_get_burst_ok:
+		    break;
+		  }
+	      }
+	      search_state.test_pos = bf->stk_test_pos;
+	      goto while_non_det_options;
+	    }
+	    
+	    
+	  case rx_cache_miss:
+	    /* Because the superstate NFA is lazily constructed,
+	     * and in fact may erode from underneath us, we sometimes
+	     * have to construct the next instruction from the hard way.
+	     * This invokes one step in the lazy-conversion.
+	     */
+	    search_state.ifr = rx_handle_cache_miss (&rxb->rx,
+						     search_state.super,
+						     search_state.c,
+						     search_state.ifr->data_2);
+	    if (!search_state.ifr)
+	      {
+		search_state.test_ret = rx_test_internal_error;
+		goto test_do_return;
+	      }
+	    goto restart;
+	    
+	  case rx_backtrack:
+	    /* RX_BACKTRACK means that we've reached the empty
+	     * superstate, indicating that match can't succeed
+	     * from this point.
+	     */
+	    goto test_do_return;
+	    
+	  case rx_next_char:
+	  case rx_error_inx:
+	  case rx_num_instructions:
+	    search_state.ret_val = 0;
+	    goto test_do_return;
+	  }
+	goto pseudo_while_1;
+      }
+    
+    /* Healthy exits from the test-match loop do a 
+     * `goto return_from_test_match'   On the other hand, 
+     * we might end up here.
+     */
+  error_in_testing_match:
+    test_state = rx_test_error;
+    goto test_returns_to_search;
+    
+    /***** fastmap/search loop body
+     *	      considering the results testing for a match
+     */
+    
+  return_from_test_match:
+    
+    if (search_state.best_last_l >= 0)
+      {
+	if (regs && (regs->start != search_state.best_lparen))
+	  {
+	    bcopy (search_state.best_lparen, regs->start,
+		   regs->num_regs * sizeof (int));
+	    bcopy (search_state.best_rparen, regs->end,
+		   regs->num_regs * sizeof (int));
+	  }
+	if (regs && !rxb->no_sub)
+	  {
+	    int q;
+	    int bound = (regs->num_regs > search_state.num_regs
+			 ? regs->num_regs
+			 : search_state.num_regs);
+	    regoff_t * s = regs->start;
+	    regoff_t * e = regs->end;
+	    for (q = search_state.best_last_l + 1;  q < bound; ++q)
+	      s[q] = e[q] = -1;
+	  }
+	search_state.ret_val = search_state.best_lparen[0];
+	test_state = rx_test_ok;
+	goto test_returns_to_search;
+      }
+    else
+      {
+	test_state = rx_test_fail;
+	goto test_returns_to_search;
+      }
+    
+  test_return_continuation:
+    search_state.test_match_resume_pt = test_pc;
+    test_state = rx_test_continuation;
+    goto test_returns_to_search;
+  }
+}
+
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+#else /* RX_WANT_SE_DEFS */
+  /* Integers are used to represent side effects.
+   *
+   * Simple side effects are given negative integer names by these enums.
+   * 
+   * Non-negative names are reserved for complex effects.
+   *
+   * Complex effects are those that take arguments.  For example, 
+   * a register assignment associated with a group is complex because
+   * it requires an argument to tell which group is being matched.
+   * 
+   * The integer name of a complex effect is an index into rxb->se_params.
+   */
+ 
+  RX_DEF_SE(1, re_se_try, = -1)		/* Epsilon from start state */
+
+  RX_DEF_SE(0, re_se_pushback, = re_se_try - 1)
+  RX_DEF_SE(0, re_se_push0, = re_se_pushback -1)
+  RX_DEF_SE(0, re_se_pushpos, = re_se_push0 - 1)
+  RX_DEF_SE(0, re_se_chkpos, = re_se_pushpos -1)
+  RX_DEF_SE(0, re_se_poppos, = re_se_chkpos - 1)
+
+  RX_DEF_SE(1, re_se_at_dot, = re_se_poppos - 1)	/* Emacs only */
+  RX_DEF_SE(0, re_se_syntax, = re_se_at_dot - 1) /* Emacs only */
+  RX_DEF_SE(0, re_se_not_syntax, = re_se_syntax - 1) /* Emacs only */
+
+  RX_DEF_SE(1, re_se_begbuf, = re_se_not_syntax - 1) /* match beginning of buffer */
+  RX_DEF_SE(1, re_se_hat, = re_se_begbuf - 1) /* match beginning of line */
+
+  RX_DEF_SE(1, re_se_wordbeg, = re_se_hat - 1) 
+  RX_DEF_SE(1, re_se_wordbound, = re_se_wordbeg - 1)
+  RX_DEF_SE(1, re_se_notwordbound, = re_se_wordbound - 1)
+
+  RX_DEF_SE(1, re_se_wordend, = re_se_notwordbound - 1)
+  RX_DEF_SE(1, re_se_endbuf, = re_se_wordend - 1)
+
+  /* This fails except at the end of a line. 
+   * It deserves to go here since it is typicly one of the last steps 
+   * in a match.
+   */
+  RX_DEF_SE(1, re_se_dollar, = re_se_endbuf - 1)
+
+  /* Simple effects: */
+  RX_DEF_SE(1, re_se_fail, = re_se_dollar - 1)
+
+  /* Complex effects.  These are used in the 'se' field of 
+   * a struct re_se_params.  Indexes into the se array
+   * are stored as instructions on nfa edges.
+   */
+  RX_DEF_CPLX_SE(1, re_se_win, = 0)
+  RX_DEF_CPLX_SE(1, re_se_lparen, = re_se_win + 1)
+  RX_DEF_CPLX_SE(1, re_se_rparen, = re_se_lparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_backref, = re_se_rparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_iter, = re_se_backref + 1) 
+  RX_DEF_CPLX_SE(0, re_se_end_iter, = re_se_iter + 1)
+  RX_DEF_CPLX_SE(0, re_se_tv, = re_se_end_iter + 1)
+
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.c	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      last--;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') last--;
+      if (last>=0 && wordstart[last]=='r') last--;
+      if (last>=0 && wordstart[last]=='e') last--;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	last--;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) last--;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,50 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:37  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/24 06:37:53  rjmcnab
+
+   Incorporated the french stemmer better.
+
+
+ */
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/sptree.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/sptree.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/sptree.c	(revision 21112)
@@ -0,0 +1,391 @@
+/**************************************************************************
+ *
+ * sptree.c -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include	"sysfuncs.h"
+#include	"sptree.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:50  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define TRACE(w)
+
+#define POOL_SIZE 1024
+
+
+typedef struct node_rec node;
+
+struct node_rec
+  {
+    node *left, *rght;
+    node *par;
+    char *datsun;
+  };
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    node nodes[POOL_SIZE];
+  };
+
+static node *SP_splay ();
+
+#define	CNULL	((char *) NULL)
+#define	NNULL	((node *) NULL)
+#define	SNULL	((Splay_Tree *) NULL)
+
+
+/*
+ *    createset():
+ *              allocates memory for and initialises a set.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+Splay_Tree *
+SP_createset (int (*cmp) (void *, void *))
+{
+  Splay_Tree *S;
+
+  if (!(S = (Splay_Tree *) malloc (sizeof (Splay_Tree))))
+    return (SNULL);
+  S->cmp = cmp;
+  S->root = CNULL;
+  S->no_of_items = 0;
+  S->mem_in_use = sizeof (Splay_Tree);
+  S->pool = NULL;
+  S->pool_chunks = 1024;
+  return S;
+}				/* createset() */
+
+
+
+static node *
+SP_get_node (Splay_Tree * S)
+{
+  struct pool *p = S->pool;
+  if (!p || p->pos == p->length)
+    {
+      p = malloc (sizeof (struct pool));
+      if (!p)
+	return NULL;
+      S->mem_in_use += sizeof (struct pool);
+      p->length = POOL_SIZE;
+      p->pos = 0;
+      p->next = S->pool;
+      S->pool = p;
+    }
+  return &(p->nodes[p->pos++]);
+}
+
+
+
+/*
+ *    SP_freeset():
+ *              free the memory allocated to a splay tree. Note this Does not
+ *              free the data in the tree you must do this yourself.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+void 
+SP_freeset (Splay_Tree * S)
+{
+  while (S->pool)
+    {
+      struct pool *p = S->pool;
+      S->pool = p->next;
+      free (p);
+    }
+  free (S);
+}
+
+
+/*
+ *    insert():
+ *              inserts the item given into the set given in the order
+ *              specified by the comparison function in the set 
+ *              definition.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+
+void *
+SP_insert (void *item, Splay_Tree * S)
+{
+  int dir = 0;
+  register node *curr, *prev;
+  node *tmp;
+
+  prev = NNULL;
+  curr = (node *) S->root;
+  for (; curr;)
+    {
+      prev = curr;
+      dir = S->cmp (item, curr->datsun);
+      if (dir < 0)
+	curr = curr->left;
+      else
+	curr = curr->rght;
+    }
+  if (!(tmp = SP_get_node (S)))
+    return NULL;
+  tmp->datsun = item;
+
+  if (prev == NNULL)
+    S->root = (char *) tmp;
+  else if (dir < 0)
+    prev->left = tmp;
+  else
+    prev->rght = tmp;
+
+  tmp->par = prev;
+  tmp->left = NNULL;
+  tmp->rght = NNULL;
+  S->no_of_items++;
+  S->root = (char *) SP_splay (tmp);
+  return (tmp->datsun);
+}				/* insert() */
+
+/*
+ *    get_first():
+ *              returns a pointer to the first item in the set. If the set 
+ *              is empty a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_first (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  for (curr = (node *) S->root, prev = NNULL; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_first() */
+
+/*
+ *    get_next():
+ *              returns the in-order successor of the currently fingered
+ *              item of the set. If no item is fingered or if there is 
+ *              no in-order successor a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_next (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  curr = (node *) S->root;
+  if (!curr || !curr->rght)
+    return (CNULL);
+
+  for (prev = curr, curr = curr->rght; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_next() */
+
+/*
+ *    deletemin():
+ *              removes the first element in the set and returns a
+ *              pointer to it. If the set is empty a NULL pointer 
+ *              is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_deletemin (Splay_Tree * S)
+{
+  node *tmp;
+  char *dat;
+
+  dat = SP_get_first (S);
+  if (dat)
+    {
+      tmp = (node *) S->root;
+      S->root = (char *) tmp->rght;
+      if (S->root)
+	((node *) S->root)->par = NULL;
+    }
+  S->no_of_items--;
+  return dat;
+}				/* deletemin() */
+
+
+/*
+ *    member():
+ *              searches the given set for an item which matches (has
+ *              the same key) and returns a pointer to a copy of that item.
+ *              If no matching item is found a NULL pointer is 
+ *              returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_member (void *dat, Splay_Tree * S)
+{
+  register node *curr;
+  register int dir;
+  static int calls = 0;
+
+  TRACE ("in member");
+  for (curr = (node *) S->root; curr;)
+    {
+      TRACE ("call to cmp");
+      dir = S->cmp (dat, curr->datsun);
+      TRACE ("out of cmp");
+      if (dir < 0)
+	curr = curr->left;
+      else if (dir > 0)
+	curr = curr->rght;
+      else
+	{
+	  /* splay occasionaly to speed up execution */
+	  if ((calls++ % 4) == 0 && curr)
+	    S->root = (char *) SP_splay (curr);
+	  TRACE ("out of member");
+	  return (curr->datsun);
+	}
+    }
+  TRACE ("out of member");
+  return (CNULL);
+}				/* member() */
+
+#define	ROTATEL(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->rght = p->left;			\
+		if(q->rght)				\
+			q->rght->par = q;		\
+		p->left = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+#define	ROTATER(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->left = p->rght;			\
+		if(q->left)				\
+			q->left->par = q;		\
+		p->rght = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+
+/*
+ *    splay():
+ *              i
+ *
+ *      Accesses outside variables:     i
+ *
+ *      Return value...:                i
+ */
+
+static node *
+SP_splay (p)
+     register node *p;
+{
+  register node *q, *g;
+
+  q = p->par;
+  /* splay the tree about p */
+  while (q != NNULL)
+    {
+      g = q->par;
+      if (p == q->left)
+	{
+	  if (g == NNULL)
+	    ROTATER (p, q);
+	  else if (q == g->left)
+	    {
+	      ROTATER (q, g);
+	      ROTATER (p, q);
+	    }
+	  else
+	    {
+	      ROTATER (p, q);
+	      ROTATEL (p, g);
+	    }
+	}
+      else
+	{
+	  if (g == NNULL)
+	    ROTATEL (p, q);
+	  else if (q == g->rght)
+	    {
+	      ROTATEL (q, g);
+	      ROTATEL (p, q);
+	    }
+	  else
+	    {
+	      ROTATEL (p, q);
+	      ROTATER (p, g);
+	    }
+	}
+      q = p->par;
+    }
+  return (p);
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/sptree.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/sptree.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/sptree.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * sptree.h -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef SPTREE_H
+#define SPTREE_H
+
+typedef struct set_rec Splay_Tree;
+
+struct set_rec
+  {
+    int no_of_items;
+    unsigned long mem_in_use;
+    int (*cmp) (void *, void *);
+    void *root;
+    void *pool;
+    int pool_chunks;
+  };
+
+Splay_Tree *SP_createset (int (*cmp) (void *, void *));
+void *SP_insert (void *item, Splay_Tree * S);
+void *SP_get_first (Splay_Tree * S);
+void *SP_get_next (Splay_Tree * S);
+void *SP_deletemin (Splay_Tree * S);
+void *SP_member (void *dat, Splay_Tree * S);
+void SP_freeset (Splay_Tree * S);
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/strcasecmp.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/strcasecmp.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/strcasecmp.c	(revision 21112)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(s1, s2)
+    char *s1, *s2;
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/gsdl/packages/mg/lib/strstr.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/strstr.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/strstr.c	(revision 21112)
@@ -0,0 +1,44 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (s1, s2)
+     char *s1;
+     char *s2;
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, i++, s++)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; p1++, p2++)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/lib/timing.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/timing.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/timing.c	(revision 21112)
@@ -0,0 +1,234 @@
+/**************************************************************************
+ *
+ * timing.c -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:05  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:53  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "timing.h"
+
+
+/*
+ * Return the time in seconds since 00:00:00 GMT, Jan. 1, 1970 to the
+ * best precision possible
+ *
+ */
+double 
+RealTime (void)
+{
+  return ((double) time (NULL));
+}
+
+
+
+/*
+ * Return the amount of system and user CPU used by the process to date to 
+ * the best precision possible. If user is non-null then it is initialised to 
+ * the user time. If sys is non-null then it is initialised to the system time.
+ *
+ */
+double 
+CPUTime (double *user, double *sys)
+{
+
+#if defined(HAVE_TIMES)
+
+  struct tms buffer;
+  static double clk_tck = 0;
+  double u, s;
+
+  times (&buffer);
+
+  if (clk_tck == 0)
+    clk_tck = CLK_TCK;
+
+  u = (double) buffer.tms_utime / clk_tck;
+  s = (double) buffer.tms_stime / clk_tck;
+  if (user)
+    *user = u;
+  if (sys)
+    *sys = s;
+  return u + s;
+
+#elif defined(HAVE_GETRUSAGE)
+
+  struct rusage ruse;
+  getrusage (RUSAGE_SELF, &ruse);
+  if (user)
+    *user = (double) ruse.ru_utime.tv_sec +
+      (double) ruse.ru_utime.tv_usec / 1000000;
+  if (sys)
+    *sys = (double) ruse.ru_stime.tv_sec +
+      (double) ruse.ru_stime.tv_usec / 1000000;
+  return ((double) ruse.ru_utime.tv_sec + ruse.ru_stime.tv_sec +
+	((double) ruse.ru_utime.tv_usec + ruse.ru_stime.tv_usec) / 1000000);
+
+#else
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (user) *user = 0.0;
+  if (sys) *sys = 0.0;
+  return 0.0;
+#else
+  -->Can NOT find a system time routine to use ! !!
+#endif  /* __WIN32__ */
+
+#endif
+
+}
+
+
+
+
+/*
+ * Get the Real and CPU time and store them in a ProgTime structure
+ */
+void 
+GetTime (ProgTime * StartTime)
+{
+  StartTime->RealTime = RealTime ();
+  StartTime->CPUTime = CPUTime (NULL, NULL);
+}
+
+
+
+
+/*
+ * Display the Real and CPU time elapsed since the StartTime anf FinishTime 
+ * structures were initialised. If FinishTime is NULL then FinishTime is
+ * Now.
+ */
+char *
+ElapsedTime (ProgTime * StartTime,
+	     ProgTime * FinishTime)
+{
+  static char buf[50];
+  double Real, CPU, hour, min, sec;
+  if (!FinishTime)
+    {
+      Real = RealTime () - StartTime->RealTime;
+      CPU = CPUTime (NULL, NULL) - StartTime->CPUTime;
+    }
+  else
+    {
+      Real = FinishTime->RealTime - StartTime->RealTime;
+      CPU = FinishTime->CPUTime - StartTime->CPUTime;
+    }
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f cpu, %02d:%02d:%02d elapsed.",
+	   hour, min, sec,
+	   ((int) ceil (Real)) / 3600,
+	   (((int) ceil (Real)) % 3600) / 60, ((int) ceil (Real)) % 60);
+  return (buf);
+}
+
+
+#define MILLION 1000000
+
+/* ===============================================================================
+ * Function: cputime_string
+ * Description:
+ *      Prints out given cpu time into a string buffer.
+ * Input: cpu-time in clock_t or timeval format
+ * Output: pointer to internal buffer 
+ * =============================================================================== */
+
+#ifdef HAVE_TIMES
+char *
+cputime_string (clock_t clk)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) clk / (double) CLK_TCK;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#else
+char *
+cputime_string (struct timeval *t)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) t->tv_sec + (double) t->tv_usec / MILLION;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#endif
+
+
+
+/* ===============================================================================
+ * Function: time_normalise
+ * Description:
+ *      If the micro-second component is negative or over a million,
+ *      then must take the over/under flow and add it to the second component.
+ * Usage:
+ *      Use this routine when you have just been doing straight addition and
+ *      subtraction on the micro-second component. For this could lead to a
+ *      negative number or over a million.
+ *
+ * =============================================================================== */
+
+#ifndef HAVE_TIMES
+
+#ifndef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+static 
+#endif
+void
+time_normalise (struct timeval *t)
+{
+  while (t->tv_usec < 0)
+    {
+      t->tv_usec += MILLION;
+      t->tv_sec--;
+    }
+  while (t->tv_usec > MILLION)
+    {
+      t->tv_usec -= MILLION;
+      t->tv_sec++;
+    }
+}
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/timing.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/timing.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/timing.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * timing.h -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TIMING
+#define H_TIMING
+
+
+#ifdef HAVE_TIMES
+# if TIME_WITH_SYS_TIME
+#  include <sys/times.h>
+#  include <time.h>
+#  ifndef CLK_TCK
+#   define CLK_TCK sysconf(_SC_CLK_TCK)
+#  endif
+# else
+#  if HAVE_SYS_TIME_H
+#   include <sys/time.h>
+#  else
+#   include <time.h>
+#  endif
+# endif
+#else
+/* [RPAP - Feb 97: WIN32 Port] */
+# ifdef __WIN32__
+   struct timeval { long tv_sec, tv_usec; };
+# else
+#  include <sys/time.h>
+#  include <sys/resource.h>
+# endif
+#endif
+
+
+typedef struct
+  {
+    double RealTime, CPUTime;
+  }
+ProgTime;
+
+double RealTime (void);
+
+double CPUTime (double *user, double *sys);
+
+void GetTime (ProgTime * StartTime);
+
+/* FinishTime may be NULL */
+char *ElapsedTime (ProgTime * StartTime, ProgTime * FinishTime);
+
+#ifdef HAVE_TIMES
+char *cputime_string (clock_t clk);
+#else
+void time_normalise (struct timeval *t);
+char *cputime_string (struct timeval *t);
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/unitool.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/unitool.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/unitool.c	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; i--) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; i++) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      curpos++;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data;
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) letter_here++;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) digit_here++;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) space_here++;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	i++;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      outhere++;
+      outlen++;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    in_i++;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/gsdl/packages/mg/lib/unitool.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/lib/win32in.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/win32in.c	(revision 21112)
@@ -0,0 +1,86 @@
+#include "longlong.h"
+
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/gsdl/packages/mg/lib/win32in.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/win32in.h	(revision 21112)
@@ -0,0 +1,10 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+#include "longlong.h"
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
Index: /main/tags/3.01/gsdl/packages/mg/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif;
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_passes.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mg_weights_build.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgbilevel.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild_4.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgdictlist.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgfelics.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgmerge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgquery.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgquery.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgstat.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgtic.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgtic.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgticbuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgticdump.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgticprune.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/mgticstat.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/gsdl/packages/mg/man/man1/xmg.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/man/man1/xmg.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/man/man1/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/mkinstalldirs
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/gsdl/packages/mg/src/images/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/Makefile.in	(revision 21112)
@@ -0,0 +1,217 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/images
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+GSDLOS = @gsdlos@
+TOUCH = echo timestamp >
+ 
+bindir = ../../../../bin/$(GSDLOS)
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics mgbilevel mgtic mgticbuild mgticprune mgticstat \
+                mgticdump
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl/packages/mg/src/images/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/WIN32.MAK	(revision 21112)
@@ -0,0 +1,201 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+#PACKAGE = mg
+#VERSION = 1.2
+ 
+#SHELL = /bin/sh
+#exec_prefix = ${prefix}
+#prefix = /home/rpeeters/mg/mg-1.3.2
+#srcdir = .
+#subdir = src/images
+#top_srcdir = ../..
+#transform = s,x,x,
+ 
+CC = cl
+#CFLAGS = -g -O -Wall 
+CPPFLAGS = 
+DEFS = -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+#INSTALL = /usr/local/gnu/bin/install -c
+#INSTALL_DATA = ${INSTALL} -m 644
+#INSTALL_PROGRAM = ${INSTALL}
+LDFLAGS = 
+LIBS = ..\..\lib\libmg.lib 
+#TOUCH = echo timestamp >
+ 
+#bindir = $(exec_prefix)/bin
+#datadir = $(prefix)/share
+#libexecdir = $(exec_prefix)/libexec
+#localedir = $(datadir)/locale
+#mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) -I. -I..\.. -I..\..\lib
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+ 
+
+o = .obj
+e = .exe
+ 
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics$e mgbilevel$e mgtic$e mgticbuild$e mgticprune$e mgticstat$e \
+                mgticdump$e
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics$e: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel$e: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic$e: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild$e: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune$e: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat$e: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump$e: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#ansi2knr: ansi2knr.o
+#	$(LINK) ansi2knr.o $(LIBS)
+#
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+#execinstall: $(EXEC)
+#	$(top_srcdir)/mkinstalldirs $(bindir); \
+#	for name in $(EXEC); do \
+#	  $(INSTALL_PROGRAM) $$name \
+#	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+#execuninstall:
+#	for name in $(EXEC); do \
+#	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#	for file in $(DISTFILES); do \
+#	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#	  || cp -p $(srcdir)/$$file $(distdir); \
+#	done
+ 
+#Makefile: Makefile.in ../../config.status
+#	cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.c	(revision 21112)
@@ -0,0 +1,471 @@
+/**************************************************************************
+ *
+ * arithcode.c -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *	Arithmetic encoding. Taken largely from Witten, Neal, Cleary,
+ *		CACM, 1987, p520.
+ *	Includes bitfiddling routines.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "arithcode.h"
+
+
+FILE *arith_in, *arith_out;
+
+/*
+   $log$
+ */
+
+static char *RCSID = "$Id$";
+
+#undef BECAREFUL
+#define INTEGRATED
+
+
+codevalue S_low = 0, S_high = 0, S_value = 0;
+long S_bitstofollow = 0;
+int S_buffer = 0, S_bitstogo = 0;
+
+long cmpbytes = 0, rawbytes = 0;
+
+#ifdef INTEGRATED
+long CountOfBitsOut;
+#endif
+
+
+/*==================================*/
+
+#define BITPLUSFOLLOW(b)		\
+{	OUTPUTBIT((b));			\
+	while (bitstofollow)		\
+	{	OUTPUTBIT(!(b));	\
+		bitstofollow -= 1;	\
+	}				\
+}
+
+#ifdef INTEGRATED
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+	CountOfBitsOut++;		\
+}
+#else
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+}
+#endif
+
+#define ADDNEXTINPUTBIT(v)		\
+{	if (bitstogo==0)		\
+	{	buffer = getc(arith_in);	\
+		bitstogo = 8;		\
+	}				\
+	v = (v<<1)+(buffer&1);		\
+	buffer >>= 1;			\
+	bitstogo -=1;			\
+}
+
+/*==================================*/
+
+void
+arithmetic_encode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue H;
+    register long bitstofollow;
+    register int buffer, bitstogo;
+
+    bitstofollow = S_bitstofollow;
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+
+    for (;;)
+      {
+	if (high < H)
+	  {
+	    BITPLUSFOLLOW (0);
+	  }
+	else if (low >= H)
+	  {
+	    BITPLUSFOLLOW (1);
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    bitstofollow++;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high++;
+      }
+    S_bitstofollow = bitstofollow;
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+  }
+}
+
+/*==================================*/
+
+/* Made into a macro 
+
+   codevalue
+   arithmetic_decode_target(totl)
+   register int totl;
+   {    return (((S_value-S_low+1)*totl-1) / (S_high -S_low+1));
+   } 
+ */
+
+/*==================================*/
+
+void
+arithmetic_decode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue value, H;
+    register int buffer, bitstogo;
+
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+    value = S_value;
+
+    for (;;)
+      {
+	if (high < H)
+	  {			/* nothing */
+	  }
+	else if (low >= H)
+	  {
+	    value -= H;
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    value -= firstqtr;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high += 1;
+	ADDNEXTINPUTBIT (value);
+      }
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+    S_value = value;
+  }
+}
+
+/*==================================*/
+
+
+static void
+startoutputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 8;
+}
+
+static void
+startinputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 0;
+}
+
+static void
+doneoutputingbits ()
+{				/* write another byte if necessary */
+  if (S_bitstogo < 8)
+    {
+      putc (S_buffer >> S_bitstogo, arith_out);
+      cmpbytes += 1;
+    }
+}
+
+/*==================================*/
+
+static void
+startencoding ()
+{
+  S_low = 0;
+  S_high = topvalue;
+  S_bitstofollow = 0;
+}
+
+static void
+startdecoding ()
+{
+  register int i;
+  register int buffer, bitstogo;
+  S_low = 0;
+  S_high = topvalue;
+  S_value = 0;
+  bitstogo = S_bitstogo;
+  buffer = S_buffer;
+  for (i = 0; i < codevaluebits; i++)
+    ADDNEXTINPUTBIT (S_value);
+  S_bitstogo = bitstogo;
+  S_buffer = buffer;
+}
+
+static void
+doneencoding ()
+{
+  register long bitstofollow;
+  register int buffer, bitstogo;
+  bitstofollow = S_bitstofollow;
+  buffer = S_buffer;
+  bitstogo = S_bitstogo;
+  bitstofollow += 1;
+  if (S_low < firstqtr)
+    {
+      BITPLUSFOLLOW (0);
+    }
+  else
+    {
+      BITPLUSFOLLOW (1);
+    }
+  S_bitstofollow = bitstofollow;
+  S_buffer = buffer;
+  S_bitstogo = bitstogo;
+}
+
+
+
+void 
+EncodeGammaDist (int Off)
+/* Encode Off with probability distribution derived from the Elias gamma distribution. */
+{
+  int Len, i;
+
+  Off++;
+  for (Len = 31; Len >= 0 && !(Off & (1 << Len)); Len--);
+
+  for (i = Len; i >= 0; i--)
+    arithmetic_encode (0, 1, 2);
+  arithmetic_encode (1, 2, 2);
+
+  for (i = Len - 1; i >= 0; i--)
+    if (Off & (1 << i))
+      arithmetic_encode (1, 2, 2);
+    else
+      arithmetic_encode (0, 1, 2);
+}
+
+
+
+int 
+DecodeGammaDist ()
+/* Decode next Off using probability distribution derived from the Elias gamma distribution. Inverse of EncodeGammaDist. */
+{
+  int i = 1, Len = 0, bit;
+
+  do
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	{
+	  arithmetic_decode (0, 1, 2);
+	  Len++;
+	}
+    }
+  while (!bit);
+
+  Len--;
+  while (Len-- > 0)
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	arithmetic_decode (0, 1, 2);
+      i = (i << 1) | bit;
+    }
+  return (i - 1);
+}
+
+
+
+
+
+void 
+EncodeGammaSigned (int snum, int *pos, int *neg)
+{
+  if (snum >= 0)
+    arithmetic_encode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_encode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++;
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+  EncodeGammaDist (abs (snum));
+}
+
+
+int 
+DecodeGammaSigned (int *pos, int *neg)
+{
+  int signedpos = 1;
+  if (arithmetic_decode_target (*pos + (*neg)) < *pos)
+    arithmetic_decode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_decode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++, signedpos = 0;
+
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+
+  if (signedpos)
+    return DecodeGammaDist ();
+  else
+    return (-DecodeGammaDist ());
+}
+
+
+void 
+InitArithEncoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startoutputingbits ();
+  startencoding ();
+}
+
+void 
+InitArithDecoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startinputingbits ();
+  startdecoding ();
+}
+
+void 
+CloseDownArithEncoding (void)
+{
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void 
+CloseDownArithDecoding (void)
+{
+}
+
+
+
+
+
+/* encode that famous number, 42 uses 12 bits */
+void 
+EncodeChecksum ()
+{
+  EncodeGammaDist (42);
+}
+
+void 
+DecodeChecksum (char str[])
+{
+  if (DecodeGammaDist () != 42)
+    {
+      fprintf (stderr, "%s: checksum error, file corrupt.\n", str);
+      exit (1);
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/arithcode.h	(revision 21112)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * arithcode.h -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef _arithcode_h
+#define _arithcode_h
+
+extern FILE *arith_in, *arith_out;
+
+#define codevaluebits 16
+#if (codevaluebits<16)
+typedef unsigned short codevalue;
+#else
+typedef unsigned long codevalue;
+#endif
+
+#define topvalue ((codevalue)((1<<codevaluebits)-1))
+#define maxfrequency ((topvalue+1)/4 + 1)
+
+#define firstqtr (topvalue/4+1)
+#define half     (2*firstqtr)
+#define thirdqtr (3*firstqtr)
+
+#define	escape_event	U->totalcnt-U->notfound, U->totalcnt, U->totalcnt
+
+void arithmetic_encode ();
+codevalue arithmetic_decode_target ();
+
+#define	arithmetic_decode_target(totl)	\
+	(  ((S_value-S_low+1)*(totl)-1) / (S_high-S_low+1)  )
+
+void arithmetic_decode ();
+
+
+extern codevalue S_low, S_high, S_value;
+extern long S_bitstofollow;
+extern int S_buffer, S_bitstogo;
+
+extern long cmpbytes, rawbytes;
+
+extern long CountOfBitsOut;
+
+void EncodeGammaDist (int Off);
+int DecodeGammaDist ();
+
+void EncodeGammaSigned (int snum, int *pos, int *neg);
+int DecodeGammaSigned (int *pos, int *neg);
+
+
+void InitArithEncoding (void);
+void InitArithDecoding (void);
+
+void CloseDownArithEncoding (void);
+void CloseDownArithDecoding (void);
+
+void EncodeChecksum ();
+void DecodeChecksum (char str[]);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/basictypes.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/basictypes.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/basictypes.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * basictypes.h -- base data types
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file contains information relating to the base data types
+ *
+ **************************************************************************/
+
+#ifndef __basictypes_h
+#define __basictypes_h
+
+typedef unsigned long Pixel;
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.c	(revision 21112)
@@ -0,0 +1,747 @@
+/**************************************************************************
+ *
+ * bilevel.c -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  
+ *  
+ *  This is the code used to compress a bitmap.  The template is given in
+ *  the following format. 
+ *   '.'  = pixel not used
+ *   'p'  = pixel used in 1st order context (the superset)
+ *   '2'  = pixel used in 2nd order context (the subset)
+ *   ';'  = marker used to denote the end of a row in the template
+ *   '*'  = the current context pixel.
+ *  for instance, the common 22/10 template would be shown as:
+ *     ".ppppp.;"
+ *     "pp222pp;"
+ *     "p22222p;"
+ *     "p22*...;"
+ * 
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "bilevel.h"
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+
+
+#define INITVALUE 2
+#define INCREMENT 4
+
+#define MAX_T_W 32
+
+/* note memory usage is 2*TEMPLATESIZE*sizeof(prob), which is currently 4.9 Mb */
+#define TEMPLATESIZE 612317
+
+
+#define HASH(x) ((x)%TEMPLATESIZE)
+
+
+typedef struct
+  {
+    unsigned short s, n;
+  }
+prob;				/* set and notset */
+typedef struct
+  {
+    short int x, y;
+  }
+coordinate;
+
+coordinate MM[MAX_T_W][MAX_T_W];	/* the actual (x,y) positions */
+unsigned int MMcount[MAX_T_W];	/* number of bits in this row of the template */
+unsigned int MMmask[MAX_T_W];	/* the "current" mask value */
+unsigned int MMbits[MAX_T_W];	/* the bitmask to mask out higher bits */
+unsigned int MMrows;		/* total number of rows */
+unsigned int MM2level[MAX_T_W];	/* bitmask to calculate the 2nd level mask */
+int twolevel, curr_row, surr;
+
+prob *P = NULL;			/* probability array, up to N bits */
+prob *P2 = NULL;
+
+void 
+bl_clearmodel ()
+{
+  int i;
+
+  P = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  P2 = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P2)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  for (i = 0; i < TEMPLATESIZE; i++)
+    {
+      P[i].s = P[i].n = INITVALUE;
+      P2[i].s = P2[i].n = INITVALUE;
+    }
+}
+
+
+void 
+bl_freemodel ()
+{
+  free (P);
+  free (P2);
+}
+
+
+
+
+#define update_counts(p, pixel)                        \
+    pixel ? (p->s+=INCREMENT) : (p->n+=INCREMENT);     \
+                                                       \
+    if(p->s+p->n >= (unsigned)(maxfrequency)) {        \
+       p->s=(p->s+1)/(unsigned)2;                      \
+       p->n=(p->n+1)/(unsigned)2;                      \
+    }
+
+
+
+
+/* mask is the bigger one, mask2l is smaller one */
+static void 
+bl_encode_in_context (unsigned long mask, unsigned long mask2l, unsigned long pixel)
+{
+  register int twol, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+  twol = twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT));
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_encode (0, p->s, t);
+      else
+	arithmetic_encode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_encode (0, p2->s, t);
+      else
+	arithmetic_encode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+}
+
+
+static void 
+bl_decode_in_context (unsigned long mask, unsigned long mask2l, unsigned long *val)
+{
+  register int twol, pixel, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+
+  if (twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT)))
+    {
+      pixel = arithmetic_decode_target (p2->s + p2->n) < p2->s, twol = 1;
+    }
+  else
+    {
+      pixel = arithmetic_decode_target (p->s + p->n) < p->s, twol = 0;
+    }
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_decode (0, p->s, t);
+      else
+	arithmetic_decode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_decode (0, p2->s, t);
+      else
+	arithmetic_decode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+
+  *val = pixel;
+}
+
+static void 
+bl_string_to_array (char inputstr[], char parsestr[], int *w, int *h)
+{
+  int r, c;
+  int i, j, lastc, p, l;
+  int xc, yc;
+  int foundstatus;
+
+  surr = twolevel = 0;
+  for (i = 0; i < MAX_T_W; i++)
+    MMcount[i] = MM2level[i] = 0;
+  MMrows = 0;
+
+  for (i = 0; i < (int) strlen (inputstr); i++)
+    {
+      p = inputstr[i];
+      if ((p != '.') && (p != '2') && (p != '*') && (p != 'p') && (p != ';'))
+	{
+	  fprintf (stderr, "Template string can only contain [.p2*;], not \"%c\".\n", inputstr[i]);
+	  exit (1);
+	}
+    }
+
+  /* get the width and height, test rectangularity... */
+  parsestr[l = 0] = 0;
+  for (c = 0, r = 0, lastc = -1, p = 0; p < (int) strlen (inputstr); p++)
+    {
+      if ((inputstr[p] == ';'))
+	{
+	  r++;
+	  if (lastc != -1)
+	    {			/* this is the 2nd row at least */
+	      if (lastc != c)
+		{
+		  fprintf (stderr, "The rows have to be the same width. "
+			   "Row %d has width %d (previous row has width %d)\n", r, c, lastc);
+		  exit (1);
+		}
+	    }
+	  lastc = c;
+	  c = 0;
+	}
+      else
+	{
+	  c++;
+	  parsestr[l] = inputstr[p];
+	  parsestr[++l] = 0;
+	}
+    }
+
+  if (c != 0)
+    lastc = c;
+  *w = lastc;
+  *h = r;
+  if ((*w) <= 0)
+    {
+      fprintf (stderr, "Must have at least one column!\n");
+      exit (1);
+    }
+  if ((*h) <= 0)
+    {
+      fprintf (stderr, "Must have at least one row! Terminate each row with a ';'\n");
+      exit (1);
+    }
+  if ((*w > MAX_T_W) || (*h > MAX_T_W))
+    {
+      fprintf (stderr, "template size bigger than internal threshold of %d\n", MAX_T_W);
+      exit (1);
+    }
+
+  /* get the (x,y) position of the current (*) pixel */
+  for (xc = -1, yc = 0, i = 0; i < *w; i++)
+    for (j = 0; j < *h; j++)
+      if (parsestr[j * (*w) + i] == '*')
+	{
+	  if (xc >= 0)
+	    {
+	      fprintf (stderr, "There can only be ONE current ('*') pixel.\n");
+	      exit (1);
+	    }
+	  xc = i;
+	  yc = j;
+	  curr_row = j + 1;
+	}
+  if (xc < 0)
+    error_msg ("bl_string_to_array", "Error in mask, no current ('*') pixel.", "");
+  /* found the position (xc,yc) */
+
+
+  /* convert parsestr[] into a MM[][] structure */
+  for (j = 0; j < *h; j++)
+    {
+      foundstatus = 0;
+      for (i = 0; i < *w; i++)
+	{
+	  p = j * (*w) + i;
+	  if ((parsestr[p] == 'p') || (parsestr[p] == '2'))
+	    {
+	      if (!foundstatus)
+		foundstatus = 1;
+	      MM2level[j] = MM2level[j] * 2 | (parsestr[p] == '2');
+	      if (parsestr[p] == '2')
+		twolevel = 1;
+
+	      MM[j][MMcount[j]].x = i - xc;
+	      MM[j][MMcount[j]++].y = j - yc;
+	      if (((i - xc) >= 0) && ((j - yc) >= 0))
+		surr = 1;
+	    }
+	  else
+	    {
+	      if (foundstatus == 1)
+		foundstatus = 2;
+	    }
+
+	}
+      if (foundstatus)
+	MMrows++;
+      MMbits[j] = (1 << MMcount[j]) - 1;
+    }
+
+  /* reverse matrix, so [][0] is the closest to the right */
+  {
+    coordinate temp;
+    for (i = 0; i < MMrows; i++)
+      for (j = 0; j < MMcount[i] / (unsigned) 2; j++)
+	{
+	  temp = MM[i][MMcount[i] - 1 - j];
+	  MM[i][MMcount[i] - 1 - j] = MM[i][j];
+	  MM[i][j] = temp;
+	}
+  }
+}
+
+
+
+
+
+
+
+
+void 
+bl_writetemplate (char inputstr[])
+{
+  register int i, j;
+  int w, h;
+  char ind[255];
+  char *str;
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+
+  bl_string_to_array (inputstr, str, &w, &h);
+
+  EncodeGammaDist (surr);
+  EncodeGammaDist (w);
+  EncodeGammaDist (h);
+  ind['.'] = 0;
+  ind['p'] = 1;
+  ind['2'] = 2;
+  ind['*'] = 3;
+  for (j = 0; j < h; j++)
+    for (i = 0; i < w; i++)
+      EncodeGammaDist (ind[(int) str[j * w + i]]);
+
+  free (str);
+}
+
+
+void 
+bl_readtemplate ()
+{
+  register int i, j;
+  int w, h, p;
+  char ind[4];
+  char *inputstr, *str;
+
+  surr = DecodeGammaDist (surr);
+  if (surr)
+    {
+      fprintf (stderr, "It's fine compressing an image using a 'surrounding' template, but I can't "
+	       "decompress it!\n");
+      exit (1);
+    }
+  w = DecodeGammaDist ();
+  h = DecodeGammaDist ();
+
+  inputstr = (char *) malloc ((w + 2) * h * sizeof (char) + 2);
+  ind[0] = '.';
+  ind[1] = 'p';
+  ind[2] = '2';
+  ind[3] = '*';
+  for (p = 0, j = 0; j < h; j++)
+    {
+      for (i = 0; i < w; i++)
+	{
+	  inputstr[p++] = ind[DecodeGammaDist ()];
+	}
+      inputstr[p++] = ';';
+      inputstr[p] = 0;
+    }
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+  bl_string_to_array (inputstr, str, &w, &h);
+}
+
+
+
+
+
+/* Currently if there are more than 32 bits, the topmost 32 will be discarded, if the 
+   topbits are to be used in compression, define _MORETHAN32 */
+#undef _MORETHAN32
+
+void 
+bl_compress_mark (marktype d)
+{
+  register int r, c;
+  register int i, j;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d.w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  EncodeGammaDist (d.w);
+  EncodeGammaDist (d.h);
+
+  for (r = 0; r < min (curr_row, d.h); r++)
+    for (c = 0; c < d.w; c++)
+      array[r % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r);
+
+  for (r = 0; r < d.h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+
+      if (r + curr_row < d.h)
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r + curr_row);
+      else
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = 0;
+
+      for (c = 0; c < d.w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (d.bitmap, c, r));
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_decompress_mark (marktype * d)
+{
+  register int r, c;
+  register int i, j;
+  unsigned long pix;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  d->w = DecodeGammaDist ();
+  d->h = DecodeGammaDist ();
+
+  d->bitmap = pbm_allocarray (d->w, d->h);
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d->w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  for (r = 0; r < d->h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+      for (c = 0; c < d->w; c++)
+	array[r % MAX_T_W][c] = pbm_getpixel (d->bitmap, c, r);
+      for (c = 0; c < d->w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)	/* get all pixels */
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_decode_in_context (mask, mask2level, &pix);
+	  if (pix)
+	    {
+	      pbm_putpixel (d->bitmap, c, r, pix);
+	      array[r % MAX_T_W][c] = pix;
+	    }
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_compress (marktype d, char inputstr[])
+{
+  bl_clearmodel ();
+  bl_writetemplate (inputstr);
+  bl_compress_mark (d);
+  bl_freemodel ();
+}
+
+
+void 
+bl_decompress (marktype * d)
+{
+  bl_clearmodel ();
+  bl_readtemplate ();
+  bl_decompress_mark (d);
+  bl_freemodel ();
+}
+
+
+
+
+
+
+/* 
+   The bitmap in d2, can be examined clairvoyantly.  The ordering of the
+   params is therefore non-clairvoyant, and clairvoyant in that order.
+
+ */
+/*
+   coordinate M1[6]={        { 0,-2},
+   {-1,-1},{ 0,-1}, {1,-1},
+   {-2,0},  {-1, 0}};
+   };
+   coordinate M2[13]={       { 0,-2},
+   {-1,-1}, { 0,-1},   {1, -1},
+   {-2,0}, {-1, 0}, { 0, 0},   { 1, 0},  {2,0},
+   {-1, 1}, { 0, 1},   { 1, 1},
+   { 0, 2}
+ */
+
+coordinate M1[6] =
+{
+  {0, -2},
+  {-2, 0},
+  {1, -1},
+  {-1, -1},
+  {0, -1},
+  {-1, 0}};
+coordinate M2[13] =
+{
+  {0, -2},
+  {1, -1},
+  {-2, 0},
+  {2, 0},
+  {-1, 1},
+  {1, 1},
+  {0, 2},
+  {-1, -1},
+  {0, -1},
+  {1, 0},
+  {-1, 0},
+  {0, 1},
+  {0, 0}
+};
+#define M1S 6
+#define M1S_2level 3
+#define M2S 13
+#define M2S_2level 6
+
+
+void 
+bl_clair_compress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  EncodeGammaDist (cols);
+  EncodeGammaDist (rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (b1, c, r));
+
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
+
+
+void 
+bl_clair_decompress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  unsigned long p;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  i = DecodeGammaDist ();
+  j = DecodeGammaDist ();
+  if ((i != cols) || (j != rows))
+    error_msg ("bl_clair_decompress", "incorrect size of residue image", "");
+
+  if (V)
+    fprintf (stderr, "cols=%d rows=%d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+
+	  bl_decode_in_context (mask, mask2level, &p);
+	  if (p)
+	    pbm_putpixel (b1, c, r, 1);
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/bilevel.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * bilevel.h -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __BILEVEL_H
+#define __BILEVEL_H
+
+#include "marklist.h"
+
+/* must be called before the start of de/compression */
+void bl_clearmodel ();
+void bl_freemodel ();
+
+/* writes the template to the stream, and sets up the de/compression params */
+void bl_writetemplate (char inputstr[]);
+void bl_readtemplate ();
+
+/* actually performs the de/compression */
+void bl_compress (marktype d, char str[]);
+void bl_compress_mark (marktype d);
+void bl_decompress (marktype * d);
+void bl_decompress_mark (marktype * d);
+
+/* clairvoyantly de/compress d1 with respect to clairvoyantly viewable d2 */
+void bl_clair_compress (marktype d1, marktype d2);
+void bl_clair_decompress (marktype d1, marktype d2);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.c	(revision 21112)
@@ -0,0 +1,257 @@
+/**************************************************************************
+ *
+ * codeoffsets.c -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+#define TOP_INC 2
+#define MID_INC 1
+#define BOT_INC 1
+
+
+void 
+EncodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int found;
+  int dox = 0, leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;	/* all previous offsets */
+  int p, f, t, xoff, yoff, symnum, count;
+  elemlist *xcond, *ycond;	/* previous offsets conditioned on symbol num */
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (t = 0; t < nsyms; t++)
+    {
+      xcond[t].head = ycond[t].head = NULL;
+      xcond[t].tot = ycond[t].tot = 0;
+    }
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      xoff = step->data.xoffset;
+      yoff = step->data.yoffset;
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* encode the offsets, x first then y */
+
+	  if (dox)
+	    found = elem_find (&xcond[symnum], xoff, &p, &f, &t);
+	  else
+	    found = elem_find (&ycond[symnum], yoff, &p, &f, &t);
+
+	  if (found)
+	    {
+	      arithmetic_encode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+	      arithmetic_encode (p, f, t);
+	    }
+	  else
+	    {
+	      if (dox)
+		found = elem_find (&xlist, xoff, &p, &f, &t);
+	      else
+		found = elem_find (&ylist, yoff, &p, &f, &t);
+
+	      if (found)
+		{
+		  arithmetic_encode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+		  levelmid += MID_INC;
+		  arithmetic_encode (p, f, t);
+		}
+	      else
+		{
+		  arithmetic_encode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+		  levelbot += BOT_INC;
+		  if (dox)
+		    EncodeGammaSigned (xoff, &xpos, &xneg);
+		  else
+		    EncodeGammaSigned (yoff, &ypos, &yneg);
+		}
+	    }
+	  if (dox)
+	    {
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+  if (V)
+    fprintf (stderr, "offset models: top %d, mid %d, bot %d\n", (leveltop - 1) / TOP_INC, (levelmid - 1) / MID_INC, (levelbot - 1) / BOT_INC);
+}
+
+
+
+
+
+void 
+DecodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int i, target, dox;
+  int leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;
+  int p, f, t, xoff = 0, yoff = 0, symnum, count = 0;
+  elemlist *xcond, *ycond;
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (i = 0; i < nsyms; i++)
+    {
+      xcond[i].head = ycond[i].head = NULL;
+      xcond[i].tot = ycond[i].tot = 0;
+    }
+
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* decode x offset first, then y */
+	  target = arithmetic_decode_target (leveltop + levelmid + levelbot);
+	  if (target < leveltop)
+	    {
+	      arithmetic_decode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xcond[symnum].tot);
+	      else
+		i = arithmetic_decode_target (ycond[symnum].tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xcond[symnum], i, &xoff);
+		  elem_find (&xcond[symnum], xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ycond[symnum], i, &yoff);
+		  elem_find (&ycond[symnum], yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else if (target < leveltop + levelmid)
+	    {
+	      arithmetic_decode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+	      levelmid += MID_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xlist.tot);
+	      else
+		i = arithmetic_decode_target (ylist.tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xlist, i, &xoff);
+		  elem_find (&xlist, xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ylist, i, &yoff);
+		  elem_find (&ylist, yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else
+	    {
+	      arithmetic_decode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+	      levelbot += BOT_INC;
+	      if (dox)
+		xoff = DecodeGammaSigned (&xpos, &xneg);
+	      else
+		yoff = DecodeGammaSigned (&ypos, &yneg);
+	    }
+
+	  if (dox)
+	    {
+	      step->data.xoffset = xoff;
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      step->data.yoffset = yoff;
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/codeoffsets.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * codeoffsets.h -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __CODEOFFSETS_H
+#define __CODEOFFSETS_H
+
+#include "marklist.h"
+
+void EncodeOffsets (marklistptr MarkList, int num_symbols);
+void DecodeOffsets (marklistptr MarkList, int num_symbols);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.c	(revision 21112)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * codesyms.c -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Hugh Emberson
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+#include "codesyms.h"
+
+int charsetsize = 256;
+
+void 
+EncodeSymbols (marklistptr MarkList, int NSyms)
+{
+  marklistptr step;
+  unsigned short Events[PPM_CHUNK];
+  int j = 0;
+
+  charsetsize = max (NSyms, 256);
+
+  step = MarkList;
+  while (step)
+    {
+      Events[j] = step->data.symnum;
+
+      assert (Events[j] > 0 || Events[j] < (unsigned) NSyms);
+
+      j++;
+
+      if (j == PPM_CHUNK)
+	{
+	  encodestring (Events, PPM_CHUNK);
+	  j = 0;
+	}
+      step = step->next;
+    };
+
+  if (j != 0)
+    encodestring (Events, j);
+
+  encodestring (Events, 0);
+}
+
+
+marklistptr 
+DecodeSymbols (int NSyms)
+{
+  marklistptr MarkList = NULL;
+  unsigned short Events[PPM_CHUNK];
+  int j, n;
+  marktype d;
+
+  charsetsize = max (NSyms, 256);
+
+  do
+    {
+      decodestring (Events, &n);
+
+      for (j = 0; j < n; j++)
+	{
+	  d.symnum = Events[j];
+	  marklist_add (&MarkList, d);
+	}
+    }
+  while (n != 0);
+
+  return MarkList;
+}
+
+void 
+InitPPM (void)
+{
+  kbytes = 896;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/codesyms.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * codesyms.h -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef _CodeSymbols_h_
+#define _CodeSymbols_h_
+
+#include "marklist.h"
+#include "arithcode.h"
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+#define PPM_CHUNK	1024
+
+extern int kbytes;
+extern void decodestring ();
+extern void encodestring ();
+
+void EncodeSymbols (marklistptr MarkList, int NSyms);
+marklistptr DecodeSymbols (int NSyms);
+
+void InitPPM (void);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/extractor.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/extractor.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/extractor.c	(revision 21112)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * extractor.c -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "utils.h"
+
+
+/* 
+   'splitjoined' defines whether the extractor modules tries to break
+   "seemingly" merged/joined marks.
+ */
+int splitjoined = 1;
+
+#define MAX_FILL_STACK_SIZE 10000
+
+/* RunRegionFill uses line conherence to implement a better fill
+   routine.  The stack has currently got 10000 elements, although the
+   biggest I have encountered used a maximum stack size of 1294.  usage:
+   RunRegionFill(bitmap,mark,i,j,xpos,ypos); xpos and ypos are the top
+   left corners of the mark.  */
+
+int maxst = 0, st = 0;
+struct
+  {
+    int x, y;
+  }
+a[MAX_FILL_STACK_SIZE];
+
+#define push(i,j) a[st].x=i;a[st].y=j;st++;if(st>MAX_FILL_STACK_SIZE){error_msg ("extractor/flood_fill","woooo! Fill stack overflow!","");}
+#define pop(i,j) st--;i=a[st].x;j=a[st].y
+
+void 
+RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows)
+{
+  int l, r, t, start_up, start_down;
+
+  push (i, j);
+  while (st > 0)
+    {
+      if (st > maxst)
+	maxst = st;
+
+      pop (l, j);
+      r = l;
+
+      while (pbm_getpixel (bitmap, l - 1, j))
+	l--;
+      while (pbm_getpixel (bitmap, r + 1, j))
+	r++;
+      start_up = start_down = 1;
+      for (t = min (r + 1, cols - 1); t >= max (l - 1, 0); t--)
+	{
+	  if ((t >= l) && (t <= r))
+	    {
+	      pbm_putpixel (bitmap, t, j, 0);
+
+	      if (mark)
+		pbm_putpixel (mark->bitmap, t - xl, j - yt, 1);
+	    }
+
+	  if ((j - 1 >= 0) && (pbm_getpixel (bitmap, t, j - 1)))
+	    {
+	      if (start_up)
+		{
+		  push (t, j - 1);
+		  start_up = 0;
+		}
+	    }
+	  else
+	    start_up = 1;
+	  if ((j + 1 < rows) && (pbm_getpixel (bitmap, t, j + 1)))
+	    {
+	      if (start_down)
+		{
+		  push (t, j + 1);
+		  start_down = 0;
+		}
+	    }
+	  else
+	    start_down = 1;
+	}
+    }
+}
+
+
+
+
+/*
+
+   ExtractNextMark starts at (tx,ty) and looks for a set (==1) pixel.
+   The mark is then boundary traced to give an outline {The process is a
+   4-pixel exterior check -- giving 8 way connectivity}, and then
+   RunRegionFill is used to remove the mark and set the pixels in the
+   mark.
+
+   usage: ExtractNextMark(bitmap,&marklist,&startx,&starty,int cols,int rows);
+   Defines:
+   gp - gets a pixel, but returns 0 for outside regions.
+   boundary_{x,y}comp - returns the {x,y} component of the direction.
+   dir 0=right, 1=down, 2=left, 3=up.
+ */
+
+#define gp(i,j) ( (((i)<0)||((j)<0)||((i)>=cols)||((j)>=rows))? 0 : pbm_getpixel(bitmap,(i),(j)))
+
+#define boundary_xcomp(x) (x==0?1:(x==2?-1 : 0))
+#define boundary_ycomp(x) (x==1?1:(x==3?-1 : 0))
+
+int 
+ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows)
+{
+  int i, j, xl, xr, yt, yb;
+  register int dir, tempdir;
+  marktype d;
+
+  xl = INT_MAX;
+  xr = (-1);
+  yt = INT_MAX;
+  yb = (-1);
+
+  i = *tx;
+  j = *ty;
+  if ((i < 0) || (j < 0))
+    error_msg ("ExtractNextMark()", "Stupid values for x,y in findnextmark()", "");
+  while (!pbm_getpixel (bitmap, i, j))
+    {
+      i++;
+      if (i >= cols)
+	{
+	  i = 0;
+	  j++;
+	  if (j >= rows)
+	    return 0;
+	}
+    }
+
+  *tx = i;
+  *ty = j;
+  dir = 0;
+  j = j - 1;			/* start above */
+  do
+    {
+      if (i > xr)
+	xr = i - 1;
+      if (i < xl)
+	xl = i + 1;
+      if (j < yt)
+	yt = j + 1;
+      if (j > yb)
+	yb = j - 1;
+      tempdir = (dir + 1) % 4;
+      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	dir = tempdir;
+      else
+	{
+	  tempdir = dir;
+	  if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	    dir = tempdir;
+	  else
+	    {
+	      tempdir = (dir + 3) % 4;
+	      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+		dir = tempdir;
+	      else
+		dir = (dir + 2) % 4;
+	    }
+	}
+      i += boundary_xcomp (dir);
+      j += boundary_ycomp (dir);
+    }
+  while ((i != (*tx)) || (j != (*ty - 1)));
+
+  d.w = (xr - xl + 1);
+  d.h = (yb - yt + 1);
+  d.xpos = xl;
+  d.ypos = yt;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+  RunRegionFill (bitmap, &d, *tx, *ty, xl, yt, cols, rows);
+
+  marktype_calc_features (&d, "?");
+  marklist_add (marklist, d);
+
+  *tx = *tx + 1;
+  return 1;
+}
+
+
+
+
+static void 
+normalise (float *p, int w)
+{
+  int x;
+  float mx;
+
+  mx = p[0];
+  for (x = 0; x < w; x++)
+    if (p[x] > mx)
+      mx = p[x];
+  if (mx != 0)
+    {
+      for (x = 0; x < w; x++)
+	p[x] = p[x] / mx;
+    }
+}
+
+
+
+#define MAXPROFILE 1000
+float profile[MAXPROFILE], p2[MAXPROFILE];
+
+
+static int 
+possibly_joined (marktype * d)
+{
+  float aspect;
+  int i, n, x, y;
+  int leftable, l2;
+
+
+  aspect = d->w / (float) d->h;
+
+  if ((aspect >= 1.1) || (d->w < MAXPROFILE))
+    {
+      /* examine it more closely */
+
+
+      /* set up profile[] with a vertical profile */
+      for (x = 0; x < d->w; x++)
+	{
+	  profile[x] = 0;
+	  for (y = 0; y < d->h; y++)
+	    if (pbm_getpixel (d->bitmap, x, y))
+	      profile[x]++;
+	}
+
+      /* average the profile from (up to) 3 either way */
+      for (x = 0; x < d->w; x++)
+	{
+	  n = 0;
+	  p2[x] = 0;
+	  for (i = x - 3; i <= x + 3; i++)
+	    if ((i >= 0) && (i < d->w))
+	      {
+		n++;
+		p2[x] += profile[x];
+	      }
+	  p2[x] = p2[x] / (float) n;
+	}
+      for (x = 0; x < d->w; x++)
+	profile[x] = p2[x];
+
+      /* normalise profile[] */
+      normalise (profile, d->w);
+
+      /* find a little local minimum, with profile <0.25, make p2[] either 0 or 1 */
+      for (x = 0; x < d->w; x++)
+	p2[x] = 0;
+      for (x = 3; x < d->w - 3; x++)
+	{
+	  if (((profile[x - 2] - profile[x]) > 0.00) &&
+	      ((profile[x + 2] - profile[x]) > 0.00) &&
+	      (profile[x] < .25))
+	    p2[x] = 1;
+	}
+
+      /* travel rightwards, checking aspect ratio, looking at p2 now */
+      leftable = l2 = 0;
+      for (x = 0; x < d->w; x++)
+	if (p2[x] == 1)
+	  {
+	    if ((fabs ((abs (x - leftable) / (float) d->h) - 0.7) > 0.3) &&
+		(fabs ((abs (x - l2) / (float) d->h) - 0.7) > 0.3))
+	      p2[x] = 0;
+	    if (p2[x] == 1)
+	      {
+		leftable = l2;
+		l2 = x;
+	      }
+	  }
+
+      /* travel inwards, checking widths of chars, looking at p2 */
+      /* if a char has an aspect ratio of <= ? kill it */
+      l2 = 0;
+      leftable = d->w - 1;
+      x = 0;
+      y = d->w - 1;
+      while (x <= y)
+	{
+	  if (p2[x])
+	    {
+	      if (abs (x - l2) < max (d->h * 0.6, 5))
+		p2[x] = 0;
+	      if (p2[x])
+		l2 = x;
+	    }
+	  if (p2[y])
+	    {
+	      if (abs (y - leftable) < max (d->h * 0.6, 5))
+		p2[y] = 0;
+	      if (p2[y])
+		leftable = y;
+	    }
+	  x++;
+	  y--;
+	}
+
+
+      for (x = 0; x < d->w; x++)
+	if (p2[x])
+	  return 1;
+    }
+  return 0;
+}
+
+
+static void 
+split_mark (marklistptr * list, marktype * joinedmark)
+{
+  marktype new, old;
+  int x, found = 0, left = 0;
+  int i, j;
+  int oldw, oldh;
+  int overwrite = 1;
+
+  old = marktype_copy (*joinedmark);
+  oldw = joinedmark->w;
+  oldh = joinedmark->h;
+  for (x = 0; x < old.w; x++)
+    if (p2[x])
+      {
+	found = x;
+	break;
+      }
+  do
+    {
+      /* split at (found) position */
+      new.w = found + 1 - left;
+      new.h = old.h;
+      new.xpos = old.xpos + left;
+      new.ypos = old.ypos;
+      new.bitmap = pbm_allocarray (new.w, new.h);
+
+      for (i = 0; i < new.w; i++)
+	for (j = 0; j < new.h; j++)
+	  pbm_putpixel (new.bitmap, i, j, pbm_getpixel (old.bitmap, left + i, j));
+
+      marktype_adj_bound (&new);
+
+      if (overwrite)
+	{
+	  overwrite = 0;
+	  pbm_freearray (&(joinedmark->bitmap), oldh);
+	  *joinedmark = new;
+	}
+      else
+	marklist_add (list, new);
+
+
+      p2[found] = 0;
+      left = found;
+      found = 0;
+      for (x = left; x < old.w; x++)
+	if (p2[x])
+	  {
+	    found = x;
+	    break;
+	  }
+      if (left == old.w - 1)
+	break;			/* if we've got to the right */
+      if (found == 0)
+	found = old.w - 1;
+    }
+  while (found);
+  pbm_freearray (&(old.bitmap), oldh);
+}
+
+
+
+
+
+void 
+ExtractAllMarks (Pixel ** bitmap, marklistptr * list, int cols, int rows)
+{
+  int startx, starty, count;
+  marklistptr step;
+
+  startx = starty = 0;
+  while (ExtractNextMark (bitmap, list, &startx, &starty, cols, rows))
+    ;
+
+  if (splitjoined)
+    for (step = (*list), count = 0; step; step = step->next, count++)
+      if (possibly_joined (&(step->data)))
+	split_mark (list, &(step->data));
+
+  for (step = (*list), count = 0; step; step = step->next, count++)
+    step->data.symnum = count;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/extractor.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/extractor.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/extractor.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * extractor.h -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _EXTRACTOR_H_
+#define _EXTRACTOR_H_
+
+#include "marklist.h"
+#include "pbmtools.h"
+
+extern int splitjoined;
+
+void RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows);
+
+int ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows);
+void ExtractAllMarks (Pixel ** bitmap, marklistptr * marklist, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/felics.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/felics.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/felics.c	(revision 21112)
@@ -0,0 +1,340 @@
+/**************************************************************************
+ *
+ * felics.c -- The main guts of the felics algorithm
+ * Copyright (C) 1994  Neil Sharman and Kerry Guise
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+
+
+#undef putw
+#define putw(x)	{					\
+	short z;					\
+	byteptr = (unsigned char *)(x);			\
+	for (z = 0; z < sizeof(int); z++)		\
+		(void) putc((int) byteptr[z], fp_out);	\
+}
+#define INTSIZE sizeof(int) * 8
+#define BOOLEAN short
+#define TRUE 1
+#define FALSE 0
+
+#define IN_RANGE 0
+#define OUT_OF_RANGE 1
+#define BELOW_RANGE 0
+#define ABOVE_RANGE 1
+
+
+void 
+encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out, int (*get) (FILE *))
+{
+  int x, y;
+  unsigned long P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  ENCODE_START (fp_out)
+
+    for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		line[x] = get (fp_in);	/* +------- */
+		BINARY_ENCODE (line[x] + 1, maxval + 1);	/* |P****** */
+		continue;	/* |******* */
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	line[x] = P = get (fp_in);
+
+	if ((L <= P) && (P <= H))
+	  {
+	    /* IN-RANGE */
+	    unsigned long range, logofrange, thresh, numlong;
+	    int nbits;
+
+	    ENCODE_BIT (IN_RANGE);
+
+	    P -= L;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P -= (numlong >> 1);
+		if ((long) P < 0)
+		  P += range;
+
+		if (P < thresh)
+		  nbits = logofrange - 1;
+		else
+		  {
+		    nbits = logofrange;
+		    P += thresh;
+		  }
+		while (--nbits >= 0)
+		  ENCODE_BIT ((P >> nbits) & 1);
+	      }
+	  }
+	else
+	  {
+	    /* OUT_OF_RANGE */
+	    unsigned long diff, parm, i, min;
+
+	    ENCODE_BIT (OUT_OF_RANGE);
+
+	    if (P < L)
+	      {
+		ENCODE_BIT (BELOW_RANGE);
+		diff = L - P - 1;
+	      }
+	    else
+	      {
+		ENCODE_BIT (ABOVE_RANGE);
+		diff = P - H - 1;
+	      }
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_ENCODE ((diff >> parm) + 1);
+	    BINARY_ENCODE ((diff & ((1 << parm) - 1)) + 1, 1 << parm);
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+	  }
+
+      }
+
+  ENCODE_DONE
+
+}
+
+
+
+
+
+
+void 
+decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out)
+{
+  int x, y;
+  unsigned int P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  DECODE_START (fp_in);
+
+  for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		BINARY_DECODE (line[x], maxval + 1);
+		line[x]--;
+		if (maxval < 256)
+		  putc (line[x], fp_out);	/* +------- */
+		else		/* |P****** */
+		  fprintf (fp_out, "%u ", line[x]);	/* |******* */
+		continue;
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	if (DECODE_BIT == IN_RANGE)
+	  {
+	    unsigned long i, range, logofrange, thresh, numlong;
+	    unsigned xx, l;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+		for (P = i = 0; i < logofrange - 1; i++)
+		  DECODE_ADD (P);
+		xx = P;
+		l = logofrange - 1;
+		if (P >= thresh)
+		  {
+		    DECODE_ADD (P);
+		    xx = P;
+		    l++;
+		    P -= thresh;
+		  }
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P += (numlong >> 1);
+		if ((long) P >= range)
+		  P -= range;
+
+		P += L;
+	      }
+	    else
+	      P = L;
+	  }
+	else
+	  {
+	    int out_of_range = DECODE_BIT;
+	    unsigned long diff, unary, binary, i, min, parm;
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_DECODE (unary);
+	    diff = (unary - 1) << parm;
+	    BINARY_DECODE (binary, 1 << parm);
+	    diff |= binary - 1;
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+
+	    if (out_of_range == BELOW_RANGE)
+	      {
+		P = L - diff - 1;
+	      }
+	    else
+	      {
+		P = diff + H + 1;
+	      }
+	  }
+
+	line[x] = P;
+
+	if (maxval < 256)
+	  (void) putc (P, fp_out);
+	else
+	  fprintf (fp_out, "%u ", P);
+      }
+
+  DECODE_DONE
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/lstevent.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/lstevent.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/lstevent.c	(revision 21112)
@@ -0,0 +1,421 @@
+/**************************************************************************
+ *
+ * lstevent.c -- Routines to implement "events" using MTF lists.
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:40  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+void
+write_method ()
+{
+  fprintf (stderr, "C");
+}
+
+
+/*===========================================*/
+
+/*
+ * [Rodney Brown - Sep/95]
+ * On IBM RS/6000 AIX version 3.2 the compiler defines NULL as (void *)0
+ * when in ANSI mode. Hence the (event) and (point) casts
+ */
+
+
+#define GETNEWNODE(e,node)		\
+do 					\
+{	node = p32(numnodes) ;		\
+	node->eventnum = (event)e ;	\
+	node->count = increment ;	\
+	node->foll.totalcnt = 0 ;	\
+	node->foll.notfound = 0 ;	\
+	node->foll.list = (point)NULL ;	\
+	numnodes += 1 ;			\
+	nfreenodes -= 1 ;		\
+}					\
+while (false)
+
+/*===========================================*/
+
+#define INSERTNODE(U,node)		\
+do					\
+{	node->next = U->list ;		\
+	U->list = p16(node) ;		\
+	U->totalcnt += increment;	\
+}					\
+while (false)
+
+/*===========================================*/
+
+int
+halvelist (p)
+     register eventnode *p;
+{
+  register int sum = 0;
+  while (p != ENULL)
+    {
+      sum += p->count = (p->count + 1) / 2;
+      p = p32 (p->next);
+    }
+  return (sum);
+}
+
+/*===========================================*/
+
+void
+halveallcounts (U)
+     register eventset *U;
+{
+  U->notfound = (U->notfound + 1) / 2;
+  U->totalcnt = U->notfound;
+  U->totalcnt += halvelist (p32 (U->list));
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event_noex (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (e == node->eventnum)
+	break;
+      lbnd += node->count;
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  totl = U->totalcnt;
+
+  if (node == ENULL)
+    {				/* event not found in list */
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event_noex (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = U->totalcnt;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (tget < (lbnd += node->count))
+	    break;
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node, *p;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	{
+	  if (e == node->eventnum)
+	    break;
+	  lbnd += node->count;
+	}
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  if (node == ENULL)
+    {				/* event not found in list, and whole list has been scanned */
+      totl = lbnd + U->notfound;
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      /* first, total up rest of list */
+      totl = lbnd + node->count;
+      p = p32 (node->next);
+      while (p != ENULL)
+	{
+	  if (!excluded[p->eventnum])
+	    totl += p->count;
+	  p = p32 (p->next);
+	}
+      totl += U->notfound;
+
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = 0;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	totl += node->count;
+      node = p32 (node->next);
+    }
+  totl += U->notfound;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (!excluded[node->eventnum])
+	    {
+	      lbnd += node->count;
+	      if (tget < lbnd)
+		break;
+	    }
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+newnode (e)
+     event e;
+{
+  register eventnode *node;
+  GETNEWNODE (e, node);
+  node->next = (point)NULL;
+  node->prev = (point)NULL;
+  return (node);
+}
+
+/*===========================================*/
+
+/* not required in this version */
+
+#if 0
+eventnode *
+addevent (U, e)
+     register eventset *U;
+     event e;
+{
+  register eventnode *node;
+  /* get an event node */
+  GETNEWNODE (e, node);
+  INSERTNODE (U, node);
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/marklist.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/marklist.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/marklist.c	(revision 21112)
@@ -0,0 +1,827 @@
+/**************************************************************************
+ *
+ * marklist.c -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+void 
+marktype_dump (marktype d)
+{
+  fprintf (stderr, "symnum: %d\n", d.symnum);
+  fprintf (stderr, "xpos: %d, ypos: %d\n", d.xpos, d.ypos);
+  fprintf (stderr, "w: %d, h: %d\n", d.w, d.h);
+  fprintf (stderr, "xcen: %d, ycen: %d\n", d.xcen, d.ycen);
+  write_library_mark (stderr, d);
+}
+
+
+/* copy, including make a copy of the bitmap */
+marktype 
+marktype_copy (marktype d)
+{
+  marktype d2;
+  int i, j;
+
+  d2 = d;
+  d2.bitmap = pbm_allocarray (d2.w, d2.h);
+  for (i = 0; i < d2.w; i++)
+    for (j = 0; j < d2.h; j++)
+      pbm_putpixel (d2.bitmap, i, j, pbm_getpixel (d.bitmap, i, j));
+  return d2;
+}
+
+
+float 
+marktype_density (marktype d)
+{
+  return d.set / (float) (d.w * d.h);
+}
+
+
+
+void 
+marktype_calc_centroid (marktype * b)
+{
+  int r, c;
+  int count = 0;
+  int xtot = 0, ytot = 0;
+
+  for (r = 0; r < b->h; r++)
+    for (c = 0; c < b->w; c++)
+      if (pbm_getpixel (b->bitmap, c, r))
+	{
+	  count++;
+	  xtot += c;
+	  ytot += r;
+	}
+  if (count)
+    {
+      xtot /= count;
+      ytot /= count;
+    }
+  b->xcen = xtot;
+  b->ycen = ytot;
+}
+
+
+
+static void 
+marktype_area (marktype * d)
+{
+  int x, y;
+  int set = 0;
+
+  for (y = 0; y < d->h; y++)
+    for (x = 0; x < d->w; x++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	set++;
+  d->set = set;
+}
+
+
+static void 
+marktype_h_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+  for (y = 0; y < d->h; y++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (x = 0; x < d->w; x++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->h_runs = num;
+}
+
+static void 
+marktype_v_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+
+  for (x = 0; x < d->w; x++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (y = 0; y < d->h; y++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->v_runs = num;
+}
+
+
+/* assume we already have the centroid! */
+static void 
+marktype_prog_cent (marktype * d)
+{
+  int x, y;
+  int xtot = 0, ytot = 0;
+  int num = 0;
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen1 = -1;
+      d->ycen1 = -1;
+    }
+  else
+    {
+      d->xcen1 = xtot / num;
+      d->ycen1 = ytot / num;
+    }
+
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen2 = -1;
+      d->ycen2 = -1;
+    }
+  else
+    {
+      d->xcen2 = xtot / num;
+      d->ycen2 = ytot / num;
+    }
+
+
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+
+
+  if (num == 0)
+    {
+      d->xcen3 = -1;
+      d->ycen3 = -1;
+    }
+  else
+    {
+      d->xcen3 = xtot / num;
+      d->ycen3 = ytot / num;
+    }
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen4 = -1;
+      d->ycen4 = -1;
+    }
+  else
+    {
+      d->xcen4 = xtot / num;
+      d->ycen4 = ytot / num;
+    }
+}
+
+
+
+/* call this to get all features */
+void 
+marktype_calc_features (marktype * d, char str[])
+{
+  if (strlen (str))
+    strcpy (d->name, str);
+  marktype_calc_centroid (d);
+  marktype_area (d);
+  marktype_prog_cent (d);
+  marktype_h_runs (d);
+  marktype_v_runs (d);
+}
+
+
+
+
+
+void 
+marktype_adj_bound (marktype * m)
+{
+  int x, y, w, h;
+  int xpos, ypos;
+  int top = 0, left = 0, right = 0, bot = 0;
+  marktype temp;
+
+  w = m->w;
+  h = m->h;
+  xpos = m->xpos;
+  ypos = m->ypos;
+
+  for (y = 0; y < h; y++)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  top = y;
+	  x = w;
+	  y = h;
+	}
+
+  for (y = h - 1; y >= 0; y--)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  bot = y;
+	  x = w;
+	  y = -1;
+	}
+
+  for (x = 0; x < w; x++)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  left = x;
+	  x = w;
+	  y = h;
+	}
+
+  for (x = w - 1; x >= 0; x--)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  right = x;
+	  x = -1;
+	  y = h;
+	}
+
+  temp.w = right - left + 1;
+  temp.h = bot - top + 1;
+  temp.xpos = xpos + left;
+  temp.ypos = ypos + top;
+  temp.bitmap = pbm_allocarray (temp.w, temp.h);
+
+  for (x = 0; x < temp.w; x++)
+    for (y = 0; y < temp.h; y++)
+      pbm_putpixel (temp.bitmap, x, y, pbm_getpixel (m->bitmap, x + left, y + top));
+
+  pbm_freearray (&(m->bitmap), h);
+
+  marktype_calc_features (&temp, "?");
+  *m = temp;
+}
+
+
+
+
+
+
+/* returns a newly created mark reprenting the average of the marklist */
+void 
+marklist_average (marklistptr list, marktype * avgmark)
+{
+  int w = 0, h = 0;
+  int t = INT_MAX, b = 0, l = INT_MAX, r = 0, matched = 0;
+  marktype d, temp;
+  marklistptr step;
+  static int a[100][100];
+  int x, y;
+
+  if (list == NULL)
+    error_msg ("marklist_average", "Can't average 'nothing'!", "");
+
+  for (step = list; step; step = step->next)
+    {
+      matched++;
+      temp = step->data;
+      w = max (w, temp.w);
+      h = max (h, temp.h);
+      r = max (r, temp.xcen);
+      l = min (l, temp.xcen);
+      b = max (b, temp.ycen);
+      t = min (t, temp.ycen);
+    }
+  w = w + (r - l);
+  h = h + (b - t);
+  if ((w >= 100) || (h >= 100))
+    matched = 0;
+
+  if (matched > 1)
+    {
+      int x_off, y_off;
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  a[x][y] = 0;
+
+      for (step = list; step; step = step->next)
+	{
+	  x_off = r - step->data.xcen;
+	  y_off = b - step->data.ycen;
+	  for (x = 0; x < step->data.w; x++)
+	    for (y = 0; y < step->data.h; y++)
+	      if (pbm_getpixel (step->data.bitmap, x, y))
+		a[x + x_off][y + y_off]++;
+	}
+
+      d.w = w;
+      d.h = h;
+      d.bitmap = pbm_allocarray (w, h);
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  if (a[x][y] > max (1, (matched / 2)))		/* > gives better CR than >= */
+	    pbm_putpixel (d.bitmap, x, y, 1);
+      marktype_adj_bound (&d);
+      strcpy (d.name, "?");
+      marktype_calc_centroid (&d);
+    }
+  else
+    d = marktype_copy (list->data);
+
+  *avgmark = d;
+}
+
+
+
+
+
+
+void 
+marklist_setnull (marklistptr * list)
+{
+  *list = NULL;
+}
+
+
+marklistptr 
+marklist_add (marklistptr * list, marktype m)
+{
+  marklistptr p;
+
+  if (*list == NULL)
+    {
+      *list = (marklistptr) malloc (sizeof (marklisttype));
+      if ((*list) == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = *list;
+    }
+  else
+    {
+      p = *list;
+      while (p->next != NULL)
+	p = p->next;
+      p->next = (marklistptr) malloc (sizeof (marklisttype));
+      if (p->next == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = p->next;
+    }
+  p->data = m;
+  p->next = NULL;
+  return p;
+
+}
+
+
+
+marklistptr 
+marklist_addcopy (marklistptr * list, marktype m)
+{
+  marktype d;
+
+  d = marktype_copy (m);
+  return marklist_add (list, d);
+}
+
+
+
+int 
+marklist_length (marklistptr list)
+{
+  int count = 0;
+
+  while (list)
+    {
+      count++;
+      list = list->next;
+    }
+  return count;
+}
+
+
+
+
+
+void 
+marklist_dump (marklistptr list)
+{
+  int count = 0;
+  marktype d;
+
+  while (list)
+    {
+      d = list->data;
+      marktype_dump (d);
+
+      count++;
+      list = list->next;
+    }
+}
+
+
+
+
+/* pos starts from 0, to up marklist_length-1 */
+marklistptr 
+marklist_getat (marklistptr list, int pos, marktype * d)
+{
+  int count = 0;
+
+  while (list)
+    {
+      if (count == pos)
+	{
+	  *d = list->data;
+	  return list;
+	}
+      count++;
+      list = list->next;
+    }
+  fprintf (stderr, "marklist_getat(): access off ends of list: pos %d\n", pos);
+  return NULL;
+}
+
+
+
+void 
+marklist_free (marklistptr * list)
+{
+  marklistptr n;
+
+  while ((*list) != NULL)
+    {
+      n = (*list)->next;
+      pbm_freearray (&((*list)->data.bitmap), (*list)->data.h);
+      free ((*list));
+      (*list) = n;
+    }
+}
+
+
+
+int 
+marklist_removeat (marklistptr * list, int pos)
+{
+  marklistptr c, n, p = NULL;
+  int count = 0;
+
+  if ((pos == 0) && (*list))
+    {				/* if removing the 1st element */
+      *list = (*list)->next;
+      return 1;
+    }
+  else
+    {
+      c = *list;
+      while (c)
+	{
+	  if (count == pos)
+	    {			/* two cases, no previous, >=1 previous */
+	      if (p == NULL)	/* no previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  c = n;
+		  return 1;
+		}
+	      else
+		/* >=1 previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  p->next = n;
+		  return 1;
+		}
+	    }
+	  count++;
+	  p = c;
+	  c = c->next;
+	}
+    }
+  fprintf (stderr, "marklist_removeat(): access off ends of list: pos %d\n", pos);
+  return 0;			/* nothing was removed */
+}
+
+
+
+marklistptr 
+marklist_next (marklistptr list)
+{
+  return list->next;
+}
+
+
+/* only writes out ascii form */
+void 
+write_library_mark (FILE * fp, marktype d)
+{
+  int r, c, rows, cols;
+  int xtot = d.xcen, ytot = d.ycen;
+
+  rows = d.h;
+  cols = d.w;
+
+  fprintf (fp, "Mark: %d\n", d.symnum);
+  fprintf (fp, "Char: %s\n", d.name);
+  fprintf (fp, "Cols: %d Rows: %d\n", d.w, d.h);
+  for (r = 0; r < d.h; r++)
+    {
+      for (c = 0; c < d.w; c++)
+	if ((r == ytot) && (c == xtot))
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'C' : '!', fp);
+	else
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'X' : '.', fp);
+      putc ('\n', fp);
+    }
+}
+
+
+
+/* only reads in ascii library */
+int 
+read_library_mark (FILE * fp, marktype * d)
+{
+  int r, c, rows, cols, numread, ch1 = 'C';
+  char str1[100], str2[100];
+
+  numread = fscanf (fp, "%s %d\n", str1, &(d->symnum));
+  if ((numread != 2) || (strcmp (str1, "Mark:") != 0))
+    return 0;
+  fscanf (fp, "%s %s\n", str1, (d->name));
+  fscanf (fp, "%s %d %s %d\n", str1, &(d->w), str2, &(d->h));
+  cols = d->w;
+  rows = d->h;
+  d->bitmap = pbm_allocarray (cols, rows);
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c++)
+	{
+	  ch1 = getc (fp);
+	  if ((ch1 == 'X') || (ch1 == 'C'))
+	    pbm_putpixel (d->bitmap, c, r, 1);
+	  else
+	    pbm_putpixel (d->bitmap, c, r, 0);
+	}
+      ch1 = getc (fp);
+    }
+  marktype_calc_features (d, "?");
+
+  return 1;
+}
+
+
+
+
+
+
+
+int 
+read_library (char libraryname[], marklistptr * library)
+{
+  int err, count;
+  FILE *lib;
+  marktype d;
+  marklistptr step;
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg ("read_library()", "Trouble opening library file", "");
+  *library = NULL;
+  count = 0;
+  while (!isEOF (lib))
+    {
+      err = read_library_mark (lib, &d);
+      if (!err)
+	error_msg ("read_library()", "unknown format of the library file.", "");
+      d.symnum = count;
+      count++;
+      if ((*library) == NULL)
+	step = marklist_add (library, d);
+      else
+	step = marklist_add (&step, d);
+    }
+  count = marklist_length (*library);
+  fclose (lib);
+
+  return count;
+}
+
+
+
+
+
+
+/**********************************************************************
+
+'elem' routines for coding offsets
+
+the elem datatype contains the 3 params for arithmetic coding,
+stored as a no-duplicates order linked list.
+
+************************************************************************/
+
+
+
+
+void 
+elem_add (elemlist * list, int num)
+{
+  elemptr temp, nn;
+  int p, f;
+
+  list->tot++;
+  if (list->head == NULL)
+    {
+      list->head = (elemptr) malloc (sizeof (elem));
+      if ((list->head) == NULL)
+	error_msg ("", "ekk", "");
+      temp = list->head;
+      temp->next = NULL;
+      temp->num = num;
+      temp->pos = 0;
+      temp->freq = 1;
+    }
+  else
+    {				/* add to list */
+      temp = list->head;
+      p = 0;
+      f = 0;
+      while ((num > temp->num) && (temp->next != NULL))
+	{
+	  p = temp->pos;
+	  f = temp->freq;
+	  temp = temp->next;
+	}
+
+      if (temp->num == num)
+	temp->freq++;
+      else
+	{			/* insert it */
+	  nn = (elemptr) malloc (sizeof (elem));
+	  if (nn == NULL)
+	    error_msg ("", "ekk", "");
+
+	  if (num < temp->num)
+	    {
+	      nn->num = temp->num;
+	      nn->pos = temp->pos;
+	      nn->freq = temp->freq;
+	      nn->next = temp->next;
+	      temp->num = num;
+	      temp->pos = p + f;
+	      temp->freq = 1;
+	      temp->next = nn;
+	    }
+	  else
+	    {
+	      nn->next = temp->next;
+	      temp->next = nn;
+	      nn->pos = temp->pos + temp->freq;
+	      nn->num = num;
+	      nn->freq = 1;
+	      temp = nn;
+	    }
+	}
+    }
+  /* update frequencies */
+  temp = temp->next;
+  if (temp)
+    do
+      {
+	temp->pos++;
+	temp = temp->next;
+      }
+    while (temp);
+}
+
+
+void 
+elem_free (elemlist * list)
+{
+  elemptr n, p;
+
+  p = list->head;
+  while (p != NULL)
+    {
+      n = p->next;
+      free (p);
+      p = n;
+    }
+  list->head = NULL;
+}
+
+int 
+elem_find (elemlist * list, int n, int *p, int *f, int *t)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if (temp->num == n)
+	{
+	  *p = temp->pos;
+	  *f = temp->freq + temp->pos;
+	  *t = list->tot;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
+
+
+int 
+elem_getrange (elemlist * list, int range, int *num)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if ((range >= temp->pos) && (range < (temp->pos + temp->freq)))
+	{
+	  *num = temp->num;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/marklist.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/marklist.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/marklist.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * marklist.h -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MARKLIST_H
+#define __MARKLIST_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+typedef struct
+  {
+    int symnum;
+    int xpos, ypos;
+    int w, h;
+    int xcen, ycen;
+    int xoffset, yoffset;
+    char name[10];
+    Pixel **bitmap;
+
+    int xcen1, ycen1;
+    int xcen2, ycen2;
+    int xcen3, ycen3;
+    int xcen4, ycen4;
+    int set;
+    int h_runs;
+    int v_runs;
+  }
+marktype;
+
+
+typedef struct marklisttype *marklistptr;
+typedef struct marklisttype
+  {
+    marktype data;
+    marklistptr next;
+  }
+marklisttype;
+
+
+void marktype_dump (marktype d);
+marktype marktype_copy (marktype d);
+
+float marktype_density (marktype d);
+void marktype_calc_centroid (marktype * b);
+void marktype_calc_features (marktype * d, char str[]);
+void marktype_adj_bound (marktype * m);
+
+void marklist_average (marklistptr list, marktype * avgmark);
+void marklist_setnull (marklistptr * list);
+marklistptr marklist_add (marklistptr * list, marktype m);
+marklistptr marklist_addcopy (marklistptr * list, marktype m);
+int marklist_length (marklistptr list);
+void marklist_dump (marklistptr list);
+marklistptr marklist_getat (marklistptr list, int pos, marktype * d);
+void marklist_free (marklistptr * list);
+int marklist_removeat (marklistptr * list, int pos);
+marklistptr marklist_next (marklistptr list);
+
+void write_library_mark (FILE * fp, marktype d);
+int read_library_mark (FILE * fp, marktype * d);
+int read_library (char libraryname[], marklistptr * library);
+
+
+
+
+
+
+/* 
+   The 'elem' structure is an ordered linked list, with fields corresponding
+   directly to the arithmetic coding routines.  
+ */
+typedef struct elem *elemptr;
+typedef struct elem
+  {
+    int num;
+    int pos;
+    int freq;
+    elemptr next;
+  }
+elem;
+
+typedef struct elemlist
+  {
+    elemptr head;
+    int tot;
+  }
+elemlist;
+
+void elem_add (elemlist * list, int num);
+void elem_free (elemlist * list);
+int elem_find (elemlist * list, int n, int *p, int *f, int *t);
+int elem_getrange (elemlist * list, int range, int *num);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/match.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/match.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/match.c	(revision 21112)
@@ -0,0 +1,417 @@
+/**************************************************************************
+ *
+ * match.c -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/* error mask goes from 0..8, single pixel errors are worth nothing */
+int 
+WXOR_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+  marktype d;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  d.w = maxc - minc + 1;
+  d.h = maxr - minr + 1;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 = back ground colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;		/* background colour */
+	pbm_putpixel (d.bitmap, c - minc, r - minr, p1 != p2);
+      }
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (pbm_getpixel (d.bitmap, c - minc, r - minr))
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= 0) && (c + i >= 0) && (r + j < d.h) && (c + i < d.w) && ((i != 0) || (j != 0)) && (pbm_getpixel (d.bitmap, c + i - minc, r + j - minr)))
+	      count++;
+  pbm_freearray (&d.bitmap, d.h);
+  return count;
+}
+
+
+
+/* define an array for most matches, if it is a massive mark, then WXOR is called */
+#define S_SIZE 500
+
+static unsigned char scratch[S_SIZE][S_SIZE], sc2[S_SIZE][S_SIZE];
+
+
+
+
+/* Weighted AND-NOT */
+
+int 
+WAN_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (b1, b2);
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	scratch[r - minr][c - minc] = sc2[r - minr][c - minc] = 0;
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 == background colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;
+
+	if ((p1 == 0) && (p2 == 1))	/* p1 not set, p2 = set */
+	  scratch[r - minr][c - minc] = 1;
+	if ((p1 == 1) && (p2 == 0))	/* p1 set, p2 not set */
+	  sc2[r - minr][c - minc] = 1;
+      }
+
+  count = 0;
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && scratch[r - minr + j][c - minc + i])
+	      count++;
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (sc2[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && sc2[r - minr + j][c - minc + i])
+	      count++;
+
+  return count;
+}
+
+/* oc 8 must be in a clock wise order, with oc[0] a corner */
+struct
+{
+  int x, y;
+}
+oc[8] =
+{
+  {
+    -1, -1
+  }
+  ,
+  {
+    0, -1
+  }
+  ,
+  {
+    1, -1
+  }
+  ,
+  {
+    1, 0
+  }
+  ,
+  {
+    1, 1
+  }
+  ,
+  {
+    0, 1
+  }
+  ,
+  {
+    -1, 1
+  }
+  ,
+  {
+    -1, 0
+  }
+};
+
+
+
+static void 
+xormatchscratch (marktype * b1, marktype * b2, int minr, int maxr, int minc, int maxc, int offx, int offy)
+{
+  int r, c, rmm, roo, coo;
+  int p1, p2;
+  Pixel **bm1 = b1->bitmap, **bm2 = b2->bitmap;
+  int w1 = b1->w, h1 = b1->h, w2 = b2->w, h2 = b2->h;
+
+  for (r = minr; r < maxr; r++)
+    {
+      rmm = r - minr;
+      roo = r - offy;
+      p1 = 0;
+      for (c = minc; c < 0; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+      for (c = 0; c < maxc; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((r >= 0) && (c < w1) && (r < h1))
+	    p1 = pbm_getpixel (bm1, c, r);
+	  else
+	    p1 = 0;		/* background */
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+    }
+}
+
+
+
+int 
+CSIS_match (marktype * b1, marktype * b2)
+{
+  int r, c, i;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy, fepcount;
+  int totscratch = 0;
+
+
+  offx = b1->xcen - b2->xcen;
+  offy = b1->ycen - b2->ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1->w, b2->w + offx);
+  maxr = max (b1->h, b2->h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (*b1, *b2);
+
+  /* set the scratch array to 1 if there is an XOR match */
+
+  xormatchscratch (b1, b2, minr, maxr, minc, maxc, offx, offy);
+  /* We how have the XOR map. Scratch contains the number of neighbours, 1=1 neighbour set...etc */
+
+  /* CSIS rule 1, if FEP (four error pels) then exit */
+  fepcount = 0;
+  for (r = minr; r < maxr - 1; r++)
+    for (c = minc; c < maxc - 1; c++)
+      if ((scratch[r - minr][c - minc]) && (scratch[r - minr + 1][c - minc]) &&
+	  (scratch[r - minr][c - minc + 1]) && (scratch[r - minr + 1][c - minc + 1]))
+	{
+	  fepcount++;
+	  if (fepcount == 1)
+	    return 10001;
+	}
+
+  /* foreach set pixel, update scratch with the number of pels around in five dir's. 
+     FPN (five-pel neighbour hood) */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	{
+	  sc2[r - minr][c - minc] = 1;	/* itself */
+	  for (i = 1; i < 8; i += 3)
+	    if (((r + oc[i].y) >= minr) && ((c + oc[i].x) >= minc) && ((r + oc[i].y) < maxr) && ((c + oc[i].x) < maxc))
+	      if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])
+		sc2[r - minr][c - minc]++;
+	}
+  /* now scratch has the value of the FPN - 1=single error, 5=4 neighbouring errors etc... */
+
+  /* set scratch and sc2 to be the same */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      totscratch += scratch[r - minr][c - minc] = sc2[r - minr][c - minc];
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	/* CSIS rule 2i: an error pel has 2 or more neighbours---this num is the same as PMS */
+	if (scratch[r - minr][c - minc] >= 3)
+	  if ((r - minr >= 1) && (c - minc >= 1) && (r - minr < (maxr - 1)) && (c - minc < (maxc - 1)))
+	    {
+	      int i, numsep = 0;
+	      /* find how many neighbours a set pixel has, only check the 4 edges-FPN */
+	      for (i = 1; i < 8; i += 2)
+		if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])	/* if a scratch pixel is set */
+		  {
+		    if ((scratch[r + oc[(i + 2) % 8].y - minr][c + oc[(i + 2) % 8].x - minc] == 0) &&
+			(scratch[r + oc[(i + 6) % 8].y - minr][c + oc[(i + 6) % 8].x - minc] == 0)
+		      )
+		      numsep++;
+		  }
+
+	      if ((b1->w >= 12) || (b2->w >= 12) || (b1->h >= 12) || (b2->h >= 12))
+		{
+
+		  /* PMS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, j, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i >= 0) && (r + j >= 0) && (c + i < b1->w) && (r + j < b1->h))
+				  p2 = pbm_getpixel (b1->bitmap, c + i, r + j);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10002;
+			    }
+			}	/* (*b1) in range */
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i - offx >= 0) && (r + j - offy >= 0) && (c + i - offx < b2->w) && (r + j - offy < b2->h))
+				  p2 = pbm_getpixel (b2->bitmap, c + i - offx, r + j - offy);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	      else
+		{		/* do the CSIS rule */
+		  /* CSIS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x >= 0) && (r + oc[i].y >= 0) && (c + oc[i].x < b1->w) && (r + oc[i].y < b1->h))
+			      p2 = pbm_getpixel (b1->bitmap, c + oc[i].x, r + oc[i].y);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10002;
+			    }
+			}
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x - offx >= 0) && (r + oc[i].y - offy >= 0) && (c + oc[i].x - offx < b2->w) && (r + oc[i].y - offy < b2->h))
+			      p2 = pbm_getpixel (b2->bitmap, c + oc[i].x - offx, r + oc[i].y - offy);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	    }
+      }
+
+  return totscratch;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/match.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/match.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/match.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * match.h -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MATCH_H
+#define __MATCH_H
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/*
+   #define NOT_SCREENED(b1,b2) ((abs(b1.w-b2.w)<=4) && (abs(b1.h-b2.h)<=4))
+ */
+
+#define NOT_SCREENED(b1,b2) ((abs(b1.xcen-b2.xcen)+abs(b1.ycen-b2.ycen))<=3)
+
+#define MATCH_VAL 1000
+#define MATCH(b1,b2) (CSIS_match(b1,b2))
+
+int XOR_match (marktype b1, marktype b2);
+int WXOR_match (marktype b1, marktype b2);
+int WAN_match (marktype b1, marktype b2);
+int PMS_match (marktype b1, marktype b2);
+int CSIS_match (marktype * b1, marktype * b2);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgbilevel.c	(revision 21112)
@@ -0,0 +1,149 @@
+/**************************************************************************
+ *
+ * mgbilevel.c -- Program to compress/decompress bilevel images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "utils.h"
+#include "pbmtools.h"
+#include "arithcode.h"
+#include "bilevel.h"
+
+
+
+static char default_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+Encode_bitmap (marktype b, char template[])
+{
+  InitArithEncoding ();
+
+  bl_compress (b, template);
+
+  CloseDownArithEncoding ();
+}
+
+void 
+Decode_bitmap (marktype * b)
+{
+  InitArithDecoding ();
+
+  bl_decompress (b);
+
+  CloseDownArithDecoding ();
+}
+
+
+void 
+usage ()
+{
+  fprintf (stderr,
+	   "usage:\n"
+	   "       mgbilevel -e  [-t \"string\"]  infile      >compressed\n"
+	   "or\n"
+	   "       mgbilevel -d                 compressed  >outfile\n");
+  exit (1);
+}
+
+#define BUF_SIZE 65536
+
+void 
+main (int argc, char *args[])
+{
+  marktype d;
+  char *filename = NULL, *template = NULL;
+  Pixel **bitmap;
+  int encode = 0, decode = 0, i;
+  char buffer[BUF_SIZE];
+
+  if (argc < 2)
+    usage ();
+  template = default_template;
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp (args[i], "-e") == 0)
+	encode = 1;
+      else if (strcmp (args[i], "-d") == 0)
+	decode = 1;
+      else if (strcmp (args[i], "-h") == 0)
+	usage ();
+      else if (strcmp (args[i], "-t") == 0)
+	{
+	  template = args[i + 1];
+	  i++;
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!filename)
+	filename = args[i];
+      else
+	error_msg (args[0], "only specify 1 filename", "");
+    }
+  if (encode == decode)
+    {
+      fprintf (stderr, "You must specify either encode XOR decode.\n");
+      exit (1);
+    }
+
+  if (!filename)
+    {
+      fprintf (stderr, "You must specify a filename.\n");
+      exit (1);
+    }
+
+  if (encode)
+    {
+      bitmap = pbm_readnamedfile (filename, &d.w, &d.h);
+      if (bitmap == NULL)
+	error_msg (args[0], "problem reading bitmap", "");
+
+      arith_out = stdout;
+      setbuffer (stdout, buffer, BUF_SIZE);
+      magic_write (stdout, MAGIC_BILEVEL);
+      if (bitmap)
+	{
+	  d.bitmap = bitmap;
+	  Encode_bitmap (d, template);
+	}
+    }
+  else if (decode)
+    {
+      FILE *tpin;
+      tpin = fopen (filename, "rb");
+      if (!tpin)
+	error_msg ("mgbilevel", "Can't open file:", filename);
+      arith_in = tpin;
+      setbuffer (tpin, buffer, BUF_SIZE);
+      magic_check (tpin, MAGIC_BILEVEL);
+      Decode_bitmap (&d);
+      fclose (tpin);
+
+      setbuffer (stdout, buffer, BUF_SIZE);
+      pbm_writefile (stdout, d.bitmap, d.w, d.h);
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgfelics.c	(revision 21112)
@@ -0,0 +1,155 @@
+/**************************************************************************
+ *
+ * mgfelics.c -- Program to compress/decompress grey scale images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "utils.h"
+
+
+#define MAXKVALUE 32
+
+
+
+void encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out, int (*get) (FILE *));
+
+void decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out);
+
+
+void 
+usage (char *prog)
+{
+  fprintf (stderr, "usage:\n");
+  fprintf (stderr, "\t%s -e [-k n]   infile    [>compressed]\n"
+	   "or\n"
+	   "\t%s -d  compressed-infile   [>outfile]\n", prog, prog);
+  exit (1);
+}
+
+
+
+
+int 
+main (int argc, char **argv)
+{
+  FILE *fp_in = (stdin), *fp_out = (stdout);
+  int width, height, maxval;
+  u_long magic;
+  char *filename = NULL;
+  unsigned int *line;
+  int maxk = 0;
+  int i;
+  int encode_flag = 0, decode_flag = 0;
+
+  if (argc < 2)
+    usage (argv[0]);
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (argv[i], "-h"))
+	usage (argv[0]);
+      else if (!strcmp (argv[i], "-e"))
+	encode_flag = 1;
+      else if (!strcmp (argv[i], "-d"))
+	decode_flag = 1;
+      else if (!strcmp (argv[i], "-k"))
+	{
+	  maxk = atoi (argv[++i]);
+	}
+      else if (argv[i][0] == '-')
+	error_msg (argv[0], "unknown switch:", argv[i]);
+      else if (!filename)
+	filename = argv[i];
+      else
+	error_msg (argv[0], "too many filenames specified.", "");
+    }
+
+  if (encode_flag == decode_flag)
+    error_msg (argv[0], "please specify either encode XOR decode.", "");
+
+  if (!filename)
+    error_msg (argv[0], "please specify a filename.", "");
+
+  if ((encode_flag) && (maxk >= MAXKVALUE))
+    {				/* to fit into 32 bit integers */
+      fprintf (stderr, "%s: k value of %d is too large; it must be less than %d\n",
+	       argv[0], maxk, MAXKVALUE);
+      exit (1);
+    }
+
+
+  if ((fp_in = fopen (filename, "rb")) == NULL)
+    error_msg (argv[0], "unable to open file:", filename);
+
+  /* read file header to determine the type and dimensions */
+
+
+  magic = magic_read (fp_in);
+  if (magic != MAGIC_FELICS)
+    {
+      rewind (fp_in);
+      magic = getmagicno_short_pop (fp_in);
+    }
+
+  if ((decode_flag && (magic != MAGIC_FELICS)) ||
+      (encode_flag && ((magic != MAGIC_P2) && (magic != MAGIC_P5))))
+    {
+      fclose (fp_in);
+      error_msg (argv[0], encode_flag ? "only PGM files can be used." : "only felics compressed files can be used.", "");
+    }
+
+  width = gethint (fp_in);
+  height = gethint (fp_in);
+  maxval = gethint (fp_in);
+
+  if ((line = (unsigned int *) malloc (width * sizeof (int *))) == NULL)
+    {
+      fclose (fp_in);
+      error_msg (argv[0], "unable to allocate memory,", "");
+    }
+
+
+  if (encode_flag)
+    {
+      magic_write (fp_out, MAGIC_FELICS);
+      fprintf (fp_out, "%d\n%d\n%d\n%d\n", width, height, maxval, maxk);
+      encode (line, width, height, maxval, maxk, fp_in, fp_out,
+	      magic == MAGIC_P2 ? getint : fgetc);
+    }
+  else if (decode_flag)
+    {
+      if (maxk)
+	warn (argv[0], "k value not required for decompression, ignored.", "");
+
+      maxk = gethint (fp_in);
+      fprintf (fp_out, "P%d\n%d %d\n%d\n", maxval < 256 ? 5 : 2,
+	       width, height, maxval);
+      decode (line, width, height, maxval, maxk, fp_in, fp_out);
+    }
+
+
+  fclose (fp_in);
+  fclose (fp_out);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgtic.c	(revision 21112)
@@ -0,0 +1,596 @@
+/**************************************************************************
+ *
+ * mgtic.c -- Program to compress/decompress textual images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file is the basis of mgtic, which compresses the components of an
+ * image. 
+ * The compressed file contains the following sections:
+ *         mgtic magic_no
+ *         mgtic version
+ *         lossy/lossless flag 
+ *         internal/external library flag 
+ *         cols, rows, number of marks, library size
+ *         [library sequence]
+ *         symbol sequence
+ *         offset sequence
+ *         [residue bitmap]
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+#include "sortmarks.h"
+#include "codesyms.h"
+#include "arithcode.h"
+#include "codeoffsets.h"
+#include "bilevel.h"
+
+static char library_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgtic -e [-L] [-Q] [-X]  libraryfile   infile    [>compressed]\n"
+	   "or\n"
+	   "\tmgtic -d [-L] [-X libraryfile]  compressed  [>outfile]\n");
+  exit (1);
+}
+
+
+/* encode the list of marks 'list', w.r.t. the library 'library' */
+void 
+match_sequence (marklistptr list, marklistptr library, marklistptr * symbol_list, Pixel ** recon, int quick)
+{
+  int carryx, carryy, gap;
+  int lastx = 0, lasty = 0;
+  int i, j;
+  marktype d, d2;
+  marklistptr liststep, librarystep;
+  int score, found;
+  int r, c;
+  int p;
+
+  carryx = carryy = 0;
+  gap = max (1, marklist_length (list) / 10);
+
+  for (liststep = list, i = 0; liststep; liststep = liststep->next, i++)
+    {
+      if (V)
+	if (!(i % gap))
+	  fprintf (stderr, ".");
+      d = liststep->data;
+
+      found = -1;
+      score = INT_MAX;
+      for (librarystep = library, j = 0; librarystep; librarystep = librarystep->next, j++)
+	{
+	  d2 = librarystep->data;
+
+	  if (NOT_SCREENED (d2, d))
+	    if ((p = MATCH (&d, &d2)) < MATCH_VAL)
+	      {
+		if (p < score)
+		  {
+		    score = p;
+		    found = j;
+		    if (quick)
+		      break;
+		  }
+	      }
+	}
+      /*
+         if(found<0){
+         for(librarystep=library,j=0; librarystep; librarystep=librarystep->next,j++){
+         d2=librarystep->data;
+
+         if((p=MATCH(&d,&d2))<MATCH_VAL){
+         if(p<score){
+         score=p;found=j; if(quick) break;
+         }
+         }
+         }
+         }
+       */
+      if (found >= 0)
+	{
+	  liststep->data.symnum = found;
+	  marklist_getat (library, found, &d2);
+	  d2.symnum = found;
+
+	  d2.xoffset = d.xpos - lastx + carryx;		/* propagate changes to next mark */
+	  d2.yoffset = d.ypos - lasty + carryy;
+	  carryx = (d2.xcen - d.xcen);	/* offet centroids because thats where we match them! */
+	  carryy = (d2.ycen - d.ycen);
+	  d2.xoffset -= carryx;	/* make the changes to this mark */
+	  d2.yoffset -= carryy;
+	  lastx = d.xpos + d2.w;
+	  lasty = d.ypos;
+
+	  marklist_add (symbol_list, d2);
+
+	  /* add to the reconstructed image--this is the lossy version */
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, d.xpos + c - carryx, d.ypos + r - carryy, 1, d2.w, d2.h);
+	}
+      else
+	{
+	  /* didn't match at all..hopefully noise */
+	  if ((liststep->data.w >= 10) || (liststep->data.w >= 10))
+	    {
+	      fprintf (stderr, "mgtic: warning, unable to match a char.\n");
+	      write_library_mark (stderr, liststep->data);
+	    }
+	  if (V)
+	    fprintf (stderr, "x");
+	  liststep->data.symnum = -1;
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+}
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i, n, r, c, ch;
+  FILE *inf;
+  marklistptr library = NULL, list = NULL, step = NULL;
+  marklistptr symbol_list = NULL;
+  marktype d, d2;
+  Pixel **bitmap, **recon, **bitmapcopy;
+  char *splitfilename = NULL;
+  int cols, rows, count, librarysize, lossy = 1;
+  int quick = 0;
+  int ticencode = 0, ticdecode = 0, external = 0;
+  char *libraryname = NULL, *infile = NULL;
+  char *s1 = NULL, *s2 = NULL;
+  char bufferin[BUFSIZ], bufferout[BUFSIZ];
+
+  if (argc < 2)
+    usage ();
+
+  while ((ch = getopt (argc, args, "edlLQhvXR:")) != -1)
+    switch (ch)
+      {
+      case 'e':
+	ticencode = 1;
+	break;
+      case 'd':
+	ticdecode = 1;
+	break;
+      case 'l':
+	lossy = 1;
+	break;
+      case 'L':
+	lossy = 0;
+	break;
+      case 'Q':
+	quick = 1;
+	break;
+      case 'v':
+	V = 1;
+	break;
+      case 'X':
+	external = 1;
+	break;
+      case 'R':
+	splitfilename = optarg;
+	break;
+      case 'h':
+      case '?':
+	usage ();
+      }
+
+  for (i = 1; i < argc; i++)
+    if (args[i][0] != '-')
+      if (strcmp (args[i - 1], "-R"))
+	{			/* ignore arg following -R, ie. -R <filename> */
+	  if (!s1)
+	    s1 = args[i];
+	  else if (!s2)
+	    s2 = args[i];
+	  else
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+  if (ticencode == ticdecode)
+    error_msg (args[0], "please specify either encode XOR decode", "");
+
+  if (ticencode)
+    {
+	  /*****************
+	    ENCODING STAGE
+	  *****************/
+      long PrevBits = 0;
+      long BITS_header = 0, BITS_symbols = 0, BITS_offsets = 0, BITS_residue = 0,
+        BITS_footer = 0, BITS_library = 0;
+
+      libraryname = s1;
+      if (!libraryname)
+	error_msg (args[0], "please specify a library file", "");
+      infile = s2;
+      if (!infile)
+	error_msg (args[0], "please specify a file name", "");
+
+      inf = fopen (infile, "rb");
+      if (inf == NULL)
+	error_msg (args[0], "Trouble opening file:", infile);
+      setbuf (inf, bufferin);
+
+      count = librarysize = read_library (libraryname, &library);
+
+      if (V)
+	fprintf (stderr, "%s: processing...\n", args[0]);
+
+      if (pbm_isapbmfile (inf))
+	{
+	  if (V)
+	    fprintf (stderr, "reading file %s...\n", infile);
+	  bitmap = pbm_readfile (inf, &cols, &rows);
+	  fclose (inf);
+
+	  bitmapcopy = pbm_copy (bitmap, cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "extracting...");
+	  ExtractAllMarks (bitmap, &list, cols, rows);
+	  if (V)
+	    fprintf (stderr, "(%d marks)\n", marklist_length (list));
+
+	  /* sort into reading order */
+	  if (V)
+	    fprintf (stderr, "sorting...\n");
+	  list = sortmarks (list);
+
+	  pbm_freearray (&bitmap, rows);	/* clear old version */
+	  bitmap = bitmapcopy;	/* point to the copy */
+	  recon = pbm_allocarray (cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "matching...");
+
+	  match_sequence (list, library, &symbol_list, recon, quick);
+
+
+	  PrevBits = 0;
+	  setbuf (stdout, bufferout);
+	  /* start output */
+	  magic_write (stdout, MAGIC_TIC);
+	  InitArithEncoding ();
+
+	  EncodeGammaDist (1);	/* version 1 of the program */
+	  EncodeGammaDist (lossy);	/* lossy=1 = no residue */
+	  EncodeGammaDist (external);	/* 1== external file */
+
+	  count = marklist_length (symbol_list);
+	  if (V)
+	    fprintf (stderr, "encoding cols, rows, and number of symbols=%d\n", count);
+
+	  EncodeGammaDist (cols);
+	  EncodeGammaDist (rows);
+	  EncodeGammaDist (count);
+	  EncodeGammaDist (librarysize);
+
+	  BITS_header = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the library sequence */
+	  if (external == 0)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding library\n");
+
+	      bl_clearmodel ();
+	      bl_writetemplate (library_template);
+	      for (step = library; step; step = step->next)
+		bl_compress_mark (step->data);
+	      bl_freemodel ();
+	      BITS_library = CountOfBitsOut - PrevBits;
+	      PrevBits = CountOfBitsOut;
+	    }
+
+	  /* output the symbol sequence */
+	  if (V)
+	    fprintf (stderr, "encoding symbol sequence\n");
+	  InitPPM ();
+	  EncodeSymbols (symbol_list, count);
+	  BITS_symbols = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the offset sequence */
+	  if (V)
+	    fprintf (stderr, "encoding offset sequence\n");
+	  EncodeOffsets (symbol_list, count);
+	  BITS_offsets = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+	  EncodeChecksum ();	/* code lossy checksum */
+
+	  /* calculate the residue...and compress it---if need be! */
+	  if (!lossy)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding residue...\n");
+
+	      d.bitmap = bitmap;
+	      d.h = rows;
+	      d.w = cols;
+	      d2.bitmap = recon;
+	      d2.h = rows;
+	      d2.w = cols;
+
+	      if (splitfilename)
+		{
+		  FILE *temp;
+
+		  CloseDownArithEncoding ();
+		  fclose (stdout);
+		  /* no residue result */
+		  BITS_footer = CountOfBitsOut - PrevBits;
+
+		  temp = fopen (splitfilename, "wb");
+		  if (temp == NULL)
+		    error_msg (args[0], "Trouble creating file:", splitfilename);
+
+		  arith_out = temp;
+		  InitArithEncoding ();
+		  bl_clair_compress (d, d2);
+		  CloseDownArithEncoding ();
+		  BITS_residue = CountOfBitsOut;
+		  fclose (temp);
+		}
+	      else
+		{
+		  bl_clair_compress (d, d2);
+		  BITS_residue = CountOfBitsOut - PrevBits;
+		  PrevBits = CountOfBitsOut;
+		  EncodeChecksum ();	/* code lossless checksum */
+		  CloseDownArithEncoding ();
+		  BITS_footer = CountOfBitsOut - PrevBits;
+		}
+	    }
+	  else
+	    {
+	      if (V)
+		fprintf (stderr, "not encoding residue..lossy mode\n");
+	      CloseDownArithEncoding ();
+	      BITS_footer = CountOfBitsOut - PrevBits;
+	    }
+
+	  /* because we edit the values above */
+	  CountOfBitsOut = BITS_header + BITS_library + BITS_symbols + BITS_offsets + BITS_residue + BITS_footer;
+
+	  fprintf (stderr, "bits: header=%ld, library=%ld, "
+		   "symbols=%ld, offsets=%ld, residue=%ld, footer=%ld\n",
+		   BITS_header, BITS_library, BITS_symbols,
+		   BITS_offsets, BITS_residue, BITS_footer);
+	  fprintf (stderr, "total bits: %ld, ", CountOfBitsOut);
+	  fprintf (stderr, "Lossy CR: %4.2f", (cols * rows) / (float) (CountOfBitsOut - BITS_residue));
+	  if (external)
+	    fprintf (stderr, " (excluding external lib)");
+	  fprintf (stderr, ", Lossless CR: %4.2f\n", (!lossy) * (cols * rows) / (float) (CountOfBitsOut));
+	}
+      else
+	error_msg (args[0], "unknown format of bitmap--expecting PBM.", "");
+    }
+  else
+    {
+	  /*****************
+	    DECODING STAGE
+	  *****************/
+      int lastx, lasty;
+      librarysize = 0;
+
+      if (external)
+	{
+	  libraryname = s1;
+	  infile = s2;
+	  count = librarysize = read_library (libraryname, &library);
+	}
+      else
+	{
+	  infile = s1;
+	  if (infile && s2)
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+      if (!freopen (infile, "rb", stdin))
+	error_msg (args[0], "Trouble opening file:", infile);
+
+
+      if (V)
+	fprintf (stderr, "decompressing...\n");
+
+      setbuf (stdin, bufferin);
+      magic_check (stdin, MAGIC_TIC);
+
+      InitArithDecoding ();
+
+      {
+	int version = DecodeGammaDist ();
+	if (version != 1)
+	  error_msg (args[0], "Need later version of decompressor.", "");
+      }
+
+      {
+	int templossy = DecodeGammaDist ();
+	if (!lossy)
+	  lossy = templossy;	/* can only choose if encoded file is lossless */
+	if (V)
+	  {
+	    if (lossy)
+	      fprintf (stderr, "lossy mode\n");
+	    else
+	      fprintf (stderr, "lossless mode\n");
+	  }
+      }
+
+      if (DecodeGammaDist ())
+	{			/* if compressed file doesn't contain library */
+	  if (!external)
+	    error_msg (args[0], "compressed file doesn't contain library, specify externally", "");
+	  external = 1;
+	}
+      else
+	{			/* if compressed file contains library */
+	  if (external)
+	    fprintf (stderr, "ignoring external library file\n");
+	  external = 0;
+	}
+
+      cols = DecodeGammaDist ();
+      rows = DecodeGammaDist ();
+      count = DecodeGammaDist ();
+
+      i = DecodeGammaDist ();	/* librarysize */
+      if (external)
+	{
+	  if (i > librarysize)
+	    error_msg (args[0], "external library file is too small!", "");
+	  else if (i < librarysize)
+	    fprintf (stderr, "%s: warning, expecting a different (smaller) library.\n", args[0]);
+	}
+      librarysize = i;
+
+      if (V)
+	fprintf (stderr, "cols %d, rows %d, num syms %d, library size %d\n", cols, rows, count, librarysize);
+
+      /* decode library */
+      if (external == 0)
+	{
+	  if (V)
+	    fprintf (stderr, "reading library\n");
+	  bl_clearmodel ();
+	  bl_readtemplate ();
+	  for (n = 0; n < librarysize; n++)
+	    {
+	      bl_decompress_mark (&d);
+	      d.symnum = n;
+	      if (library == NULL)
+		step = marklist_add (&library, d);
+	      else
+		step = marklist_add (&step, d);
+	    }
+	  bl_freemodel ();
+	  if (V)
+	    fprintf (stderr, "read %d marks from library\n", marklist_length (library));
+	}
+
+      recon = pbm_allocarray (cols, rows);
+
+      /* decode symbols */
+      InitPPM ();
+      if (V)
+	fprintf (stderr, "decompressing %d symbols\n", count);
+      symbol_list = DecodeSymbols (count);
+
+      /* decode offsets */
+      if (V)
+	fprintf (stderr, "reading offsets...\n");
+      DecodeOffsets (symbol_list, count);
+      lastx = lasty = 0;
+      for (step = symbol_list; step; step = step->next)
+	{
+	  lastx = lastx + step->data.xoffset;
+	  lasty = lasty + step->data.yoffset;
+
+	  marklist_getat (library, step->data.symnum, &d2);
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, lastx + c, lasty + r, 1, cols, rows);	/* we don't care, already warned them! */
+	  lastx += d2.w;
+	}
+
+      DecodeChecksum (args[0]);
+
+      /* decode the residue */
+      if (!lossy)
+	{
+	  if (V)
+	    fprintf (stderr, "decoding residue...\n");
+
+	  bitmap = pbm_allocarray (cols, rows);
+	  d.bitmap = bitmap;
+	  d.w = d2.w = cols;
+	  d.h = d2.h = rows;
+
+	  d2.bitmap = recon;
+
+	  /* NOTE: the 2nd argument is clairvoyantly compressed */
+	  if (splitfilename)
+	    {
+	      FILE *temp;
+
+	      CloseDownArithDecoding ();
+	      fclose (stdin);
+
+	      temp = fopen (splitfilename, "rb");
+	      if (temp == NULL)
+		error_msg (args[0], "Trouble opening file:", splitfilename);
+
+	      arith_in = temp;
+	      InitArithDecoding ();
+	      bl_clair_decompress (d, d2);
+	      CloseDownArithDecoding ();
+	      fclose (temp);
+	    }
+	  else
+	    {
+	      bl_clair_decompress (d, d2);
+	      DecodeChecksum (args[0]);
+	      CloseDownArithDecoding ();
+	    }
+	  pbm_freearray (&recon, rows);
+	  recon = bitmap;	/* point to the bitmap */
+	}
+      else
+	{
+	  CloseDownArithDecoding ();
+	}
+
+      if (V)
+	fprintf (stderr, "writing pbm file...\n");
+      setbuf (stdout, bufferout);
+      pbm_writefile (stdout, recon, cols, rows);
+      pbm_freearray (&recon, rows);
+    }				/* end decoding */
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticbuild.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * mgticbuild.c -- Program to build the textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgticbuild   libraryfile   infile.pbm  [infile2 infile3 ...]\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib = NULL, *inf = NULL;
+  marklistptr list = NULL, step;
+  Pixel **bitmap;
+  int cols, rows;
+  int count, pos = 0;
+  char *libraryname = NULL;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	pos = i;
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+  if (pos < 2)
+    usage ();
+
+  count = 0;
+  for (i = pos; i < argc; i++)
+    if (args[i][0] != '-')
+      {
+	inf = fopen (args[i], "rb");
+	count = 0;
+	if (inf == NULL)
+	  fprintf (stderr, "Trouble opening file: %s\n", args[i]);
+	else
+	  {
+	    if (pbm_isapbmfile (inf))
+	      {
+		if (V)
+		  fprintf (stderr, "%s: processing...\n", args[0]);
+		if (V)
+		  fprintf (stderr, "reading file %s...", args[i]);
+		bitmap = pbm_readfile (inf, &cols, &rows);
+		if (V && (count == 0))
+		  fprintf (stderr, "building...");
+
+		list = NULL;
+
+		ExtractAllMarks (bitmap, &list, cols, rows);
+
+		if (V)
+		  fprintf (stderr, "writing...");
+
+		if ((!lib) && (marklist_length (list)))
+		  {
+		    lib = fopen (libraryname, "r");
+		    if (lib)
+		      {
+			fclose (lib);
+			lib = fopen (libraryname, "ab");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble appending library file", "");
+			fprintf (stderr, "\nwarning: appending library file: %s\n", libraryname);
+		      }
+		    else
+		      {
+			lib = fopen (libraryname, "wb");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble creating library file", "");
+		      }
+		  }		/* if !lib */
+
+		step = list;
+		while (step)
+		  {
+		    step->data.symnum = count;
+		    write_library_mark (lib, step->data);
+		    count++;
+		    step = step->next;
+		  }
+		pbm_freearray (&bitmap, rows);
+	      }
+	    else
+	      fprintf (stderr, "warning: %s is not a PBM file.\n", args[i]);
+	    fclose (inf);
+	  }
+	if (count)
+	  {
+	    if (V)
+	      fprintf (stderr, "%d marks written.\n", count);
+	  }
+	else
+	  fprintf (stderr, "No marks written!\n");
+      }
+
+  if (lib)
+    fclose (lib);
+  marklist_free (&list);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticdump.c	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mgticdump.c -- Program to dump out a library in a human readable form
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticdump   libraryfile\n");
+  exit (1);
+}
+
+
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib;
+  int count, i;
+  marktype d;
+  char *libraryname = NULL;
+  FILE *outf = (stdout);
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble opening library file:", libraryname);
+
+  count = 0;
+  while (!isEOF (lib))
+    {
+      read_library_mark (lib, &d);
+
+      write_library_mark (outf, d);
+      count++;
+
+      pbm_freearray (&d.bitmap, d.h);
+    }
+  fprintf (stderr, "count=%d", count);
+  fclose (lib);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticprune.c	(revision 21112)
@@ -0,0 +1,393 @@
+/**************************************************************************
+ *
+ * mgticprune.c -- Program to prune a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+
+marklistptr *avglib = NULL;
+int avglibsize;
+int libsize = 0;
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+    "\tmgticbuild [-a n]  [-s n]  [-Q]   libraryfile   [pruned-library]\n");
+  exit (1);
+}
+
+
+
+
+/* foreach list in the avglib[] array, an average mark is returned */
+marklistptr 
+average_library ()
+{
+  marklistptr librarylist;
+  int i;
+  marktype avgmark;
+
+  librarylist = NULL;
+  for (i = 0; i < libsize; i++)
+    {
+      if (avglib[i])
+	{
+	  marklist_average (avglib[i], &avgmark);
+	  avgmark.symnum = i;
+	  marklist_addcopy (&librarylist, avgmark);
+	}
+    }
+  return librarylist;
+}
+
+
+void 
+average_space_enlarge ()
+{
+  int j;
+
+  if (libsize >= avglibsize)
+    {
+      avglib = (marklistptr *) realloc (avglib, (avglibsize + 100) * sizeof (marklistptr));
+      if (!avglib)
+	error_msg ("mgticprune", "out of memory in reallocating..", "");
+      for (j = avglibsize; j < avglibsize + 100; j++)
+	avglib[j] = NULL;
+      avglibsize += 100;
+    }
+}
+
+
+/* for each mark in 'origlist', if it can't be matched with one in 
+   'librarylist', then add to the librarylist, and update the relevant
+   avglib[] position */
+void 
+greedy_add (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, step_i;
+  int i, j, notfound;
+  marktype d_i;
+
+  for (i = 0, step_i = origlist; step_i; i++, step_i = step_i->next)
+    {
+      d_i = step_i->data;
+
+      notfound = 1;
+      for (j = 0, step = *librarylist; step; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d_i, step->data))
+	    if (MATCH (&d_i, &step->data) < MATCH_VAL)
+	      {
+		d_i.symnum = j;
+		marklist_addcopy (&avglib[j], d_i);
+		notfound = 0;
+		break;
+	      }
+	}
+
+      if (notfound)
+	{
+	  d_i.symnum = libsize;
+	  marklist_addcopy (librarylist, d_i);
+	  marklist_addcopy (&avglib[libsize], d_i);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+/* for each mark in 'library', if another matches with it, remove
+   one copy from the library, and erase the relevant avglib[] */
+void 
+greedy_prune (marklistptr * library)
+{
+  int count;
+  int i, j;
+  marklistptr step;
+  marktype d;
+
+  count = marklist_length (*library);
+  /* an invariant, all marks <i have been kept in the pruned library */
+  for (i = 0; i < count; i++)
+    {
+      marklist_getat (*library, i, &d);
+
+      for (j = 0, step = *library; j < i; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d, step->data))
+	    if (MATCH (&d, &step->data) < MATCH_VAL)
+	      {
+		/* [i,d] has matched with library element [j,d2]. */
+		marklist_removeat (library, i);		/* because we kill it here */
+		marklist_free (&avglib[d.symnum]);
+		avglib[d.symnum] = NULL;
+		i--, count--;
+		break;
+	      }
+	}
+    }
+}
+
+
+void 
+prune_on_size (marklistptr * list, int sizethres)
+{
+  int count, i;
+  marktype d;
+
+  if (sizethres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if ((d.w <= sizethres) && (d.h <= sizethres))
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+
+void 
+prune_on_area (marklistptr * list, int areathres)
+{
+  int count, i;
+  marktype d;
+
+  if (areathres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if (d.set <= areathres)
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+void 
+averagedout_addagain (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, stepi;
+  int notfound;
+  int i, j;
+
+  for (i = 0, step = origlist; step; i++, step = step->next)
+    {
+      notfound = 1;
+
+      for (j = 0, stepi = *librarylist; stepi; j++, stepi = stepi->next)
+	if (NOT_SCREENED (step->data, stepi->data))
+	  if (MATCH (&step->data, &stepi->data) < MATCH_VAL)
+	    {
+	      notfound = 0;
+	      break;
+	    }
+
+      if (notfound)
+	{
+	  marklist_addcopy (librarylist, step->data);
+	  marklist_addcopy (&avglib[libsize], step->data);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib;
+
+  marklistptr origlist = NULL, librarylist = NULL;
+  marklistptr step;
+  char tempn[1000], sysstr[1000];
+  int count;
+  char *libraryname = NULL, *prunename = NULL;
+  int sizethres = 3, areathres = 0;
+  int pass;
+  int num_passes = 3;
+  int overwrite = 0;
+
+  V = 0;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (!strcmp (args[i], "-Q"))
+	{
+	  num_passes = 0;
+	}
+      else if (!strcmp (args[i], "-s"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    sizethres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -s option with a numeric arg >=0", "");
+	}
+      else if (!strcmp (args[i], "-a"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    areathres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -a option with a numeric arg >=0", "");
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else if (!prunename)
+	prunename = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+  if (V)
+    fprintf (stderr, "%s: processing...\n", args[0]);
+
+  overwrite = 0;
+  if (!prunename)
+    {
+      prunename = libraryname;
+      if (V)
+	fprintf (stderr, "overwriting previous library file.\n");
+      overwrite = 1;
+    }
+
+  if (overwrite)
+    {
+      tmpnam (tempn);
+    }
+  else
+    strcpy (tempn, prunename);
+
+
+  avglibsize = count = read_library (libraryname, &origlist);
+  if (V)
+    fprintf (stderr, "%d marks...", count);
+
+  avglib = (marklistptr *) malloc (count * sizeof (marklistptr));
+  if (!avglib)
+    error_msg (args[0], "out of memory", "");
+  for (i = 0; i < avglibsize; i++)
+    avglib[i] = NULL;
+
+
+  prune_on_size (&origlist, sizethres);
+
+  prune_on_area (&origlist, areathres);
+  if (V)
+    fprintf (stderr, "after threshold pruning, %d marks\n", count = marklist_length (origlist));
+
+
+  if (V)
+    fprintf (stderr, "starting greedy library creation...\n");
+
+  greedy_add (origlist, &librarylist);
+
+
+  for (pass = 0; pass < num_passes; pass++)
+    {
+      marklist_free (&librarylist);
+      if (V)
+	fprintf (stderr, "averaging...\n");
+      librarylist = average_library ();
+      averagedout_addagain (origlist, &librarylist);
+      greedy_prune (&librarylist);
+      if (V)
+	fprintf (stderr, "after pass %d greedy prune %d marks\n", 1 + pass, marklist_length (librarylist));
+    }
+
+
+  lib = fopen (tempn, "wb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble creating prune-library file.", "");
+
+
+  if (V)
+    fprintf (stderr, "writing to file: %s...", prunename);
+  for (step = librarylist, count = 0; step; step = step->next, count++)
+    {
+      step->data.symnum = count;
+      write_library_mark (lib, step->data);
+    }
+
+  if (V)
+    fprintf (stderr, "%d marks.\n", count);
+  fclose (lib);
+
+  if (overwrite)
+    {
+      sprintf (sysstr, "mv %s %s", tempn, prunename);
+      system (sysstr);		/* move temp to proper name */
+    }
+
+  for (i = 0; i < avglibsize; i++)
+    {
+      marklist_free (&avglib[i]);
+      free (avglib[i]);
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/mgticstat.c	(revision 21112)
@@ -0,0 +1,130 @@
+/**************************************************************************
+ *
+ * mgticstat.c -- Program to print out stats about a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticstat   libraryfile\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib, *outf = (stdout);
+  int count;
+  int small1, small2, small4, small9, small16, small25, small36, small49,
+    small64, small81, small100;
+  marktype d;
+  int avx, avy;
+  int hhh, www, i;
+  int avruns_h = 0, avruns_v = 0;
+  int av_area = 0;
+  char *libraryname = NULL;
+
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "r");
+  if (lib == NULL)
+    error_msg (args[0], "Trouble opening library file", "");
+
+  count = 0;
+  small1 = small2 = small4 = small9 = small16 = small25 = small36 = small49 = small64 = small81 = small100 = 0;
+  avx = avy = 0;
+  www = hhh = 0;
+  while (!isEOF (lib))
+    {
+      if (!read_library_mark (lib, &d))
+	error_msg (args[0], "error in library file", "");
+
+      av_area += d.set;
+      if (d.set <= 1)
+	small1++;
+      if (d.set <= 2)
+	small2++;
+      if (d.set <= 4)
+	small4++;
+      if (d.set <= 9)
+	small9++;
+      if (d.set <= 16)
+	small16++;
+      if (d.set <= 25)
+	small25++;
+      if (d.set <= 36)
+	small36++;
+      if (d.set <= 49)
+	small49++;
+      if (d.set <= 64)
+	small64++;
+      if (d.set <= 81)
+	small81++;
+      if (d.set <= 100)
+	small100++;
+      avx += d.xcen;
+      avy += d.ycen;
+      www += d.w;
+      hhh += d.h;
+      avruns_h += d.h_runs;
+      avruns_v += d.v_runs;
+      pbm_freearray (&d.bitmap, d.h);
+      count++;
+      if (count % 5000 == 0)
+	fprintf (outf, "%dk ", count / 1000);
+    }
+  fprintf (outf, "number of marks=%d\n", count);
+  fprintf (outf, "average area=%d\n", av_area / count);
+  fprintf (outf, "area <=1:%d, <=2:%d, <=4:%d, <=9:%d, <=16:%d, <=25:%d, <=36:%d, <=49:%d, <=64:%d, <=81:%d, <=100:%d\n", small1, small2, small4, small9, small16, small25, small36, small49, small64, small81, small100);
+
+  fprintf (outf, "average num runs, horizontal=%d, vertical=%d\n", avruns_h / count, avruns_v / count);
+  fprintf (outf, "average centroid position=(%d,%d)\n", avx / count, avy / count);
+  fprintf (outf, "average width=%d, average height=%d\n", www / count, hhh / count);
+  fclose (lib);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/model.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/model.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/model.h	(revision 21112)
@@ -0,0 +1,92 @@
+/**************************************************************************
+ *
+ * model.h -- Part pf PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#define defaultkbytes	896
+#define minkbytes	56
+#define maxkbytes	((65536/1024)*sizeof(eventnode))
+
+#ifndef increment
+#define	increment	8
+#endif
+#ifndef halveat
+#define halveat		64
+#endif
+#define maxtotalcnt 	(increment*halveat)
+
+typedef int boolean;
+#define false           0
+#define true            1
+
+typedef short event;
+typedef unsigned short point;
+
+typedef struct
+  {
+    short totalcnt;
+    short notfound;
+    point list;			/* list storing the event records */
+  }
+eventset;
+
+typedef struct
+  {
+    event eventnum;
+    short count;
+    point next, prev;
+    eventset foll;
+  }
+eventnode, *eventptr;
+
+#define memsize		(kbytes*1024)
+#define maxnodes	(memsize/sizeof(eventnode))
+/* #define charsetsize     256 -- MAHE */
+#define nchars          (charsetsize+1)
+
+extern int charsetsize;		/* -- MAHE */
+
+extern eventptr E;
+extern long kbytes;
+extern point numnodes, nfreenodes;
+extern boolean excluded[];	/* -- MAHE */
+
+#define ENULL		(E)
+
+typedef unsigned u;
+#define p32(p)	(E+p)
+#define p16(p)	(p-E)
+
+/* in ppm.c */
+void write_model ();
+void encodestring ();
+void decodestring ();
+void encodefile ();
+void decodefile ();
+
+/* in lstevent.c */
+void write_method ();
+eventnode *encode_event_noex ();
+eventnode *decode_event_noex ();
+eventnode *encode_event ();
+eventnode *decode_event ();
+/* eventnode *addevent(); */
+eventnode *newnode ();
Index: /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.c	(revision 21112)
@@ -0,0 +1,288 @@
+/**************************************************************************
+ *
+ * pbmtools.c -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+
+void 
+pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    pbm_putpixel (bitmap, c, r, val);
+  else
+    fprintf (stderr, "putpitxel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+}
+
+int 
+pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    return pbm_getpixel (bitmap, c, r);
+  else
+    {
+      fprintf (stderr, "getpixel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+      return 0;
+    }
+}
+
+
+
+
+
+/*
+   Returns 1 if *fp points to a PBM (magic is P1 or P4) file.
+   leaves the fp in it's original position.
+   usage: if(pbm_isapbmfile(fp)) process(fp);
+ */
+int 
+pbm_isapbmfile (FILE * fp)
+{
+  u_long magic;
+
+  magic = getmagicno_short (fp);
+  if ((magic == MAGIC_P4) || (magic == MAGIC_P1))
+    return 1;
+  else
+    return 0;
+}
+
+
+/*
+   Frees the memory used in the bitmap structure.
+   usage: pbm_freearray(&bitmap);
+   Sets bitmap to be NULL on return.
+ */
+void 
+pbm_freearray (Pixel *** bitmap, int rows)
+{
+  int r;
+  if (rows == 0)
+    rows++;
+  for (r = 0; r < rows; r++)
+    free ((*bitmap)[r]);
+  free (*bitmap);
+  (*bitmap) = NULL;
+}
+
+
+/*
+   This function allocates the bitmap.
+   usage: bitmap=pbm_allocarray(cols,rows);
+ */
+Pixel **
+pbm_allocarray (int cols, int rows)
+{
+  Pixel **p;
+  int i, newc = (cols + PBITS) / PBITS;
+
+  if (cols == 0)
+    cols++;
+  if (rows == 0)
+    rows++;
+  p = (Pixel **) malloc (rows * sizeof (Pixel *));
+  if (p == NULL)
+    error_msg ("pbm_allocarray", "out of memory", "");
+  for (i = 0; i < rows; i++)
+    {
+      p[i] = (Pixel *) calloc (newc, sizeof (Pixel));
+      if (p[i] == NULL)
+	error_msg ("pbm_allocarray", "out of memory", "");
+    }
+  return p;
+}
+
+
+
+Pixel **
+pbm_copy (Pixel ** bitmap, int cols, int rows)
+{
+  Pixel **p;
+  int newc = (cols + PBITS) / PBITS;
+  int x, y;
+
+  p = pbm_allocarray (cols, rows);
+  for (x = 0; x < newc; x++)
+    for (y = 0; y < rows; y++)
+      p[y][x] = bitmap[y][x];
+  return p;
+}
+
+
+
+
+/*
+   Reads in a pbm file and returns a pointer to the allocated bitmap, 
+   in the P4 format.
+   Assumes fp was opened and keeps it that way.
+   usage: bitmap=pbm_readfile(fp,&cols,&rows);
+ */
+Pixel **
+pbm_readfile (FILE * fp, int *cols, int *rows)
+{
+  Pixel **p = NULL;
+  char s[100];
+  unsigned char *buf;
+  int r, c, ii = 0;
+  int p1 = 0;
+  int arg;
+
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  if (strcmp (s, "P1") == 0)
+    p1 = 1;
+  else if (strcmp (s, "P4") == 0)
+    p1 = 0;
+  else
+    error_msg ("pbm_readfile", "only pbm files are supported", "");
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  arg = sscanf (s, "%d %d", cols, rows);
+  if (arg == 1)
+    {
+      readline (s, fp);
+      arg = sscanf (s, "%d", rows);
+      if (arg != 1)
+	error_msg ("pbm_readfile", "bizzare pbm format", "");
+    }
+  p = pbm_allocarray (*cols, *rows);
+  buf = (unsigned char *) malloc ((*cols + 2) * sizeof (unsigned char));
+  if (!buf)
+    error_msg ("pbm_readfile", "out of memory", "");
+
+  if (p1)
+    {
+      for (r = 0; r < *rows; r++)
+	for (c = 0; c < *cols; c++)
+	  {
+	    fscanf (fp, "%d", &ii);
+	    pbm_putpixel (p, c, r, ii);
+	  }
+    }
+  else
+    {
+      for (r = 0; r < *rows; r++)
+	{
+	  c = 0;
+	  fread (buf, 1, (*cols + 7) / 8, fp);
+	  for (c = 0; c < *cols; c++)
+	    {
+	      if (!(c % 8))
+		ii = buf[c / 8];
+	      if (ii < 0)
+		error_msg ("pbm_readfile", "file reading error", "");
+	      pbm_putpixel (p, c, r, (ii & 128) ? 1 : 0);
+	      ii <<= 1;
+	    }
+	}
+    }				/* p4 type */
+  free (buf);
+  return p;
+}
+
+
+/*
+   Writes a pbm file given a file pointer and bitmap. Assumes that
+   fp is already open, and keeps in that way.
+   usage: pbm_writefile(fp,bitmap,cols,rows);
+ */
+void 
+pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows)
+{
+  int r, c, ii = 0, j;
+  unsigned char *buf;
+
+  buf = (unsigned char *) malloc (((cols + 14) / 8) * sizeof (unsigned char));
+
+  fprintf (fp, "%s\n", "P4");
+  fprintf (fp, "%d %d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c += 8)
+	{
+	  ii = 0;
+	  for (j = 0; j < 8; j++)
+	    if (c + j < cols)
+	      {
+		if (pbm_getpixel (bitmap, c + j, r))
+		  ii |= (128 >> j);
+	      }
+	  buf[c / 8] = ii;
+	}
+      fwrite (buf, 1, (cols + 7) / 8, fp);
+    }
+  free (buf);
+/*      putc('\n',fp);   if this line is included programs don't say things like 'short image' &c, but then again,
+   in most cases a 'cmp' between the original will fail because of the last byte...sigh, "standards". */
+}
+
+#define BUF_SIZE 131072
+char buffer[BUF_SIZE];
+
+void 
+pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows)
+{
+  FILE *tempfp;
+
+  tempfp = fopen (fn, "wb");
+  if (tempfp == NULL)
+    error_msg ("pbm_writenamedfile", "can't open file:", fn);
+  setbuffer (tempfp, buffer, BUF_SIZE);
+  pbm_writefile (tempfp, bitmap, cols, rows);
+  fclose (tempfp);
+}
+
+
+
+Pixel **
+pbm_readnamedfile (char fn[], int *cols, int *rows)
+{
+  FILE *tempfp;
+  Pixel **temp;
+
+  tempfp = fopen (fn, "rb");
+  if (tempfp == NULL)
+    return NULL;
+  setbuffer (tempfp, buffer, BUF_SIZE);
+
+  temp = pbm_readfile (tempfp, cols, rows);
+
+  fclose (tempfp);
+  return temp;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/pbmtools.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * pbmtools.h -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __PBMTOOLS_H
+#define __PBMTOOLS_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+#include "utils.h"
+#include "marklist.h"
+
+#define PBITS 32
+#define PSH 5
+
+
+/* The next two are ordered pixels col0=left, cols31=right */
+
+/*
+   #define pbm_getpixel(bitmap,c,r)  \
+   ((bitmap[r][(c)>>PSH] >> ((PBITS-1)-((c)&(PBITS-1))))&1)
+   #define pbm_putpixel(bitmap,c,r,val) \
+   if((val))  \
+   bitmap[r][(c)>>PSH] |=  (((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))); \
+   else \
+   bitmap[r][(c)>>PSH] &= ~(((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))) 
+ */
+
+
+
+
+/* The next two defines are ordered pixel cols0=right, cols31=left */
+
+#define pbm_getpixel(bitmap,c,r)                            \
+         ((bitmap[r][(c)>>PSH] >> ((c)&(PBITS-1)))&1)
+#define pbm_putpixel(bitmap,c,r,val)                        \
+         if((val))                                          \
+              bitmap[r][(c)>>PSH] |= (1<<((c)&(PBITS-1)));  \
+         else                                               \
+              bitmap[r][(c)>>PSH] &= ~(1<<((c)&(PBITS-1)))
+
+
+#define pbm_putpixel_trunc(bitmap,c,r,val,cols,rows) \
+       if(((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) \
+             pbm_putpixel(bitmap,c,r,val)
+
+#define pbm_getpixel_trunc(bitmap,c,r,cols,rows)       \
+       ((((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) ?  \
+             pbm_getpixel(bitmap,c,r) : 0)
+
+
+
+
+int pbm_isapbmfile (FILE * fp);
+void pbm_freearray (Pixel *** bitmap, int rows);
+Pixel **pbm_copy (Pixel ** bitmap, int cols, int rows);
+Pixel **pbm_allocarray (int cols, int rows);
+Pixel **pbm_readfile (FILE * fp, int *cols, int *rows);
+Pixel **pbm_readnamedfile (char fn[], int *cols, int *rows);
+void pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows);
+void pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows);
+
+void pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows);
+int pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/ppm.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/ppm.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/ppm.c	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * ppm.c -- PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:33  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:55  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:02  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* FIXEDORDER sets the maximum context length that will be used.
+   Making FIXEDORDER large increases the compression if there is enough
+   space to store the model, but when space limit is reached increasing
+   FIXEDORDER will decrease compression because of model thrashing.
+   Third order (FIXEDORDER=3) is suitable for a few hundred Kb of model
+   space, FIXEDORDER=1 is suitable for a few tens of kB.
+   This version requires that FIXEDORDER be a preprocessor rather
+   than runtime variable to allow loops to be unrolled and make for faster
+   execution. Maximum in this implementation is 3 */
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+/* global variables */
+eventptr E = NULL;		/* array of eventnodes */
+point numnodes = 0, nfreenodes = 0;
+long kbytes = defaultkbytes;
+/*boolean               excluded[nchars]={false};  - MAHE */
+boolean excluded[16384] =
+{false};			/* excluded characters */
+
+/* local variables */
+#define defaultchunksize        4096	/* compression chunks */
+
+#define EOSchar         (charsetsize)
+
+unsigned chunksize = defaultchunksize;
+eventptr s[FIXEDORDER + 1] =
+{NULL},				/* current contexts */
+  t[FIXEDORDER + 1] =
+{NULL};				/* new contexts */
+
+#if (FIXEDORDER==3)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1] ; s[3] = t[2]
+#else
+#if (FIXEDORDER==2)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1]
+#else
+#if (FIXEDORDER==1)
+#define SHIFT_ST	s[1] = t[0]
+#else
+#define SHIFT_ST
+#endif
+#endif
+#endif
+
+/* follow a character list and set the exclusions */
+#define SET_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = true;	\
+}						\
+while (false)
+
+/* clear the exclusions */
+#define CLEAR_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = false;	\
+}						\
+while (false)
+
+/*==================================*/
+
+void
+write_model ()
+{
+  fprintf (stderr, "ppm");
+  write_method ();
+#ifdef NOEXCLUSIONS
+  fprintf (stderr, "nx");
+#endif
+  fprintf (stderr, "-o%1d", FIXEDORDER);
+  fprintf (stderr, "-k%ld", kbytes);
+}
+
+/*==================================*/
+
+void
+encode_in_order (n, e)
+     int n;
+     event e;
+/* encode character e in order n */
+{
+  boolean encoded;
+  int i, c;
+  t[n] = encode_event_noex (&s[n]->foll, e, &encoded);
+  if (encoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = encode_event_noex (&s[c]->foll, e, &encoded);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = encode_event (&s[c]->foll, e, &encoded);
+#endif
+	if (encoded)
+	  break;
+      }
+  if (!encoded)
+    arithmetic_encode (e, e + 1, nchars);
+  for (i = c + 1; i <= n; i++)
+    if (i > 0)
+      t[i]->prev = p16 (t[i - 1]);
+    else
+      t[0]->prev = p16 (s[0]);
+  for (i = c - 1; i >= 0; i--)
+    t[i] = p32 (t[i + 1]->prev);
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+/*==================================*/
+
+/* not required in this version */
+
+#if 0
+
+void
+decode_in_order (n, e)
+     int n;
+     event *e;
+{
+  boolean decoded;
+  int i, c;
+  t[n] = decode_event_noex (&s[n]->foll, &decoded, e);
+  if (decoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = decode_event_noex (&s[c]->foll, &decoded, e);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = decode_event (&s[c]->foll, &decoded, e);
+#endif
+	if (decoded)
+	  break;
+      }
+  if (!decoded)
+    {
+      *e = arithmetic_decode_target (nchars);
+      arithmetic_decode (*e, *e + 1, nchars);
+    }
+  for (i = c + 1; i <= n; i++)
+    {
+      t[i]->eventnum = *e;
+      if (i > 0)
+	t[i]->prev = p16 (t[i - 1]);
+      else
+	t[0]->prev = p16 (s[0]);
+    }
+  {
+    register int i;
+    for (i = c - 1; i >= 0; i--)
+      t[i] = p32 (t[i + 1]->prev);
+  }
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+#endif
+
+/*==================================*/
+
+void
+start_model ()
+{
+  int i;
+  /* is this the first call? */
+  if (E == NULL)
+    E = (eventptr) malloc ((unsigned) kbytes * 1024);
+  /* start off with no information */
+  numnodes = 0;
+  nfreenodes = maxnodes;
+  for (i = 0; i <= FIXEDORDER; i++)
+    {
+      s[i] = newnode (NULL);
+      if (i > 0)
+	s[i]->prev = p16 (s[i - 1]);
+    }
+}
+
+/*==================================*/
+
+void
+encodestring (str, n)
+     unsigned short *str;
+     int n;
+/* encode all n characters of the supplied string str */
+{
+  event e;
+  boolean encoded;
+  register eventptr S, T;
+  int i;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  if (E == NULL)
+    start_model ();
+  /* set the initial context node */
+  S = s[FIXEDORDER];
+
+  for (i = 0; i < n; i++)
+    {
+      /* encode one character */
+      e = *str++;
+      /* this next monstrosity is logically equivalent to 
+         encode_in_order(FIXEDORDER,e)
+         the loop has been unrolled to make it all go faster */
+      for (;;)
+	{
+	  T = encode_event_noex (&S->foll, e, &encoded);
+	  if (encoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = encode_event_noex (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = encode_event_noex (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = encode_event_noex (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    break;
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = encode_event (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = encode_event (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = encode_event (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  arithmetic_encode (e, e + 1, nchars);
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      /* phew! Update the current context to the latest node */
+      S = p32 (T->prev);
+
+      /* is there a danger of running out of space? */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+    }
+
+  /* save the current contexts for the various order submodels */
+  s[FIXEDORDER] = S;
+  for (i = FIXEDORDER; i > 0; i--)
+    s[i - 1] = p32 (s[i]->prev);
+  /* and send the terminator character */
+  encode_in_order (FIXEDORDER, EOSchar);
+  /* and check the space again */
+  if (nfreenodes <= FIXEDORDER)
+    start_model ();
+}
+
+/*==================================*/
+
+void
+decodestring (str, n)
+     unsigned short *str;
+     int *n;
+{
+  event e;
+  boolean decoded;
+  register eventptr S, T;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  /* just mimic the corresponding encoding actions */
+  if (E == NULL)
+    start_model ();
+  S = s[FIXEDORDER];
+
+  *n = 0;
+  for (;;)
+    {				/* logically equivalent to "decode_in_order(FIXEDORDER,&e)" */
+
+      for (;;)
+	{
+	  T = decode_event_noex (&S->foll, &decoded, &e);
+	  if (decoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      break;
+	    }
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = decode_event (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = decode_event (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = decode_event (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  e = arithmetic_decode_target (nchars);
+	  arithmetic_decode (e, e + 1, nchars);
+#if (FIXEDORDER>2)
+	  t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+	    t[2]->eventnum =
+#endif
+#if (FIXEDORDER>0)
+	    t[1]->eventnum =
+#endif
+	    t[0]->eventnum = e;
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      S = p32 (T->prev);
+
+      /* check the space requirements */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+      /* are we done?  or do we save the char */
+      if (e == EOSchar)
+	break;
+      *str++ = e;
+      (*n)++;
+    }
+  s[FIXEDORDER] = S;
+}
+
+
+#if 0				/* Writing my own version of this - MAHE */
+void
+encodefile ()
+/* encodes a file by breaking into a sequence of fixed length chunks */
+{
+  unsigned char *str;
+  int c, n = 0;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startoutputingbits ();
+  startencoding ();
+
+  /* to encode the file, break it into chunks and encode the
+     chunks one by one */
+  while ((c = getchar ()) != EOF)
+    {
+      str[n++] = c;
+      if (n == chunksize)
+	{
+	  encodestring (str, n);
+	  rawbytes += n;
+	  n = 0;
+	}
+    }
+  /* send final partial chunk */
+  if (n > 0)
+    {
+      encodestring (str, n);
+      rawbytes += n;
+    }
+  /* send EOF as an empty string */
+  encodestring (str, 0);
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void
+decodefile ()
+{
+  unsigned char *str;
+  int n, i;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startinputingbits ();
+  startdecoding ();
+  for (;;)
+    {
+      decodestring (str, &n);
+      if (n == 0)
+	break;
+      for (i = 0; i < n; i++)
+	putchar ((int) str[i]);
+      rawbytes += n;
+    }
+}
+
+#endif /* #if 0 */
Index: /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.c	(revision 21112)
@@ -0,0 +1,93 @@
+/**************************************************************************
+ *
+ * sortmarks.c -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "sortmarks.h"
+
+#define GAP 4
+
+
+
+static int 
+CmpOnY (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->ycen * 0 + ((marktype *) e1)->ypos) -
+    (((marktype *) e2)->ycen * 0 + ((marktype *) e2)->ypos);
+}
+
+static int 
+CmpOnX (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->xcen * 0 + ((marktype *) e1)->xpos) -
+    (((marktype *) e2)->xcen * 0 + ((marktype *) e2)->xpos);
+}
+
+
+
+
+
+
+
+
+marklistptr 
+sortmarks (marklistptr listofmarks)
+{
+  marktype *table;
+  int count = 0;
+  int i, s, e;
+
+
+  table = calloc ((unsigned int) marklist_length (listofmarks), sizeof (marklisttype));
+
+  while (listofmarks)
+    {
+      table[count] = listofmarks->data;
+      marklist_removeat (&listofmarks, 0);
+      count++;
+    }
+
+  /* sort first on the y position of their centroids */
+  qsort ((void *) table, (unsigned int) (count), sizeof (*table), CmpOnY);
+
+  for (s = 0; s < count;)
+    {
+      int c;
+
+      for (e = s + 1, c = 0; (e < count) &&
+	   ((table[e].ypos + table[e].ycen * 0) - (table[e - 1].ypos + table[e - 1].ycen * 0) <= GAP); c++, e++);
+
+      if (e < count && c >= 1)
+	qsort (&table[s], (unsigned int) (e - s), sizeof (*table), CmpOnX);
+      s = e;
+    }
+
+  for (i = 0; i < count; i++)
+    {
+      table[i].symnum = i;
+      marklist_add (&listofmarks, table[i]);
+    }
+
+  return listofmarks;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/sortmarks.h	(revision 21112)
@@ -0,0 +1,30 @@
+/**************************************************************************
+ *
+ * sortmarks.h -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _SortMarks_h_
+#define _SortMarks_h_
+
+marklistptr sortmarks (marklistptr listofmarks);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/images/utils.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/utils.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/utils.c	(revision 21112)
@@ -0,0 +1,270 @@
+/**************************************************************************
+ *
+ * utils.c -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "utils.h"
+
+
+int V = 0;
+
+
+int 
+isEOF (FILE * fp)
+{
+  int ch;
+
+  ch = getc (fp);
+  ungetc (ch, fp);
+  if (ch == EOF)
+    return 1;
+  return 0;
+}
+
+
+int 
+getmagicno_short (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 8) | ch2);
+}
+
+int 
+getmagicno_short_pop (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    return 0;
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    return 0;
+
+  return ((ch1 << 8) | ch2);  
+}
+
+int 
+getmagicno_byte (FILE * fp)
+{
+  int ch1;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ungetc (ch1, fp);
+  return (ch1);
+}
+
+
+int 
+getmagicno_long (FILE * fp)
+{
+  int ch1, ch2, ch3, ch4;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ch3 = getc (fp);
+  if (ch3 == EOF)
+    {
+      ungetc (ch3, fp);
+      return 0;
+    }
+  ch4 = getc (fp);
+  if (ch4 == EOF)
+    {
+      ungetc (ch4, fp);
+      return 0;
+    }
+  ungetc (ch4, fp);
+  ungetc (ch3, fp);
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 24) | (ch2 << 16) | (ch3 << 8) | (ch4));
+}
+
+
+
+
+void 
+error_msg (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+  exit (1);
+}
+
+void 
+warn (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+}
+
+
+
+void 
+readline (char str[], FILE * fp)
+{
+  int i = 0, ch;
+
+  while (((ch = fgetc (fp)) != '\n') && (!feof (fp)))
+    str[i++] = ch;
+  str[i] = '\0';
+}
+
+
+
+
+
+
+
+int 
+getint (FILE * fp)
+{
+  register char ch;
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+/* get_header_int i.e. get an integer from a PNM header ; basically  
+   the same as
+   above except for comment checking ; above kept because of lower  
+   overhead */
+
+unsigned int 
+gethint (FILE * fp)
+{
+  register char ch, prev = '\0';
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if ((ch == '#') && (prev == '\n'))
+	while (((ch = getc (fp)) != '\n') && !(feof (fp)));
+      prev = ch;
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+
+
+
+void 
+magic_write (FILE * fp, u_long magic_num)
+{
+  HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fwrite (&magic_num, sizeof (magic_num), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't write magic number.", "");
+}
+
+
+void 
+magic_check (FILE * fp, u_long magic_num)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1 || NTOHUL(magic) != magic_num)  /* [RPAP - Jan 97: Endian Ordering] */
+    error_msg ("magic num", "Incorrect magic number.", "");
+}
+
+
+u_long 
+magic_read (FILE * fp)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't read magic number.", "");
+  return NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+isinteger (char s[])
+{
+  int i = 0;
+
+  for (i = 0; s[i] != '\0'; i++)
+    if (!isdigit (s[i]))
+      return 0;
+  return 1;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/images/utils.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/images/utils.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/images/utils.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * utils.h -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __UTILS_H
+#define __UTILS_H
+
+#include "sysfuncs.h"
+
+extern int V;
+
+#define DEBUG
+
+#define min(a,b) ((a)<(b) ? (a) : (b))
+#define max(a,b) ((a)>(b) ? (a) : (b))
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+                                            ((c)<<8) + (d)))
+ 
+/* --- MG magic file codes [TS:Aug/95] --- */
+#define MAGIC_FELICS            GEN_MAGIC('M','G','F','X')
+#define MAGIC_BILEVEL           GEN_MAGIC('M','G','B','I')
+#define MAGIC_TIC               GEN_MAGIC('M','G','T','C')
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+/* --------------------------------------- */
+
+#define MAGIC_P1      0x5031	/* P1   - pbm */
+#define MAGIC_P2      0x5032	/* P2   - pgm */
+#define MAGIC_P3      0x5033	/* P3   - ppm */
+#define MAGIC_P4      0x5034	/* P4   - rawbits pbm */
+#define MAGIC_P5      0x5035	/* P5   - rawbits pgm */
+#define MAGIC_P6      0x5036	/* P6   - rawbits ppm */
+
+int isEOF (FILE * fp);
+int getmagicno_byte (FILE * fp);
+int getmagicno_short (FILE * fp);
+int getmagicno_short_pop (FILE * fp);
+int getmagicno_long (FILE * fp);
+
+
+
+void error_msg (char *prog, char *message, char *extra);
+void warn (char *prog, char *message, char *extra);
+
+void readline (char str[], FILE * fp);
+
+void magic_write (FILE * fp, u_long magic_num);
+void magic_check (FILE * fp, u_long magic_num);
+u_long magic_read (FILE * fp);
+/* kerry's code */
+
+int getint (FILE * fp);
+unsigned int gethint (FILE * fp);
+int isinteger (char s[]);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/Makefile.in	(revision 21112)
@@ -0,0 +1,152 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/scripts
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+GSDLOS = @gsdlos@
+ 
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+TOUCH = echo timestamp >
+ 
+bindir = ../../../../bin/$(GSDLOS)
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+SHFILES       =	mgbuild.sh mgbuild_4.sh mg_get.sh xmg.sh  mg_get_merge.sh mgmerge.sh
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SHFILES) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgbuild mg_get xmg mg_get_merge mgmerge mgbuild_4
+
+MAN           = mgbuild.1 mgbuild_4.1 mg_get.1 xmg.1 mg_get_merge.1 mgmerge.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+mgbuild : mgbuild.sh
+	@rm -f $@
+	cat mgbuild.sh >$@
+	chmod +x $@
+
+mgbuild_4 : mgbuild_4.sh
+	@rm -f $@
+	cat mgbuild_4.sh >$@
+	chmod +x $@
+
+mg_get : mg_get.sh
+	@rm -f $@
+	cat mg_get.sh >$@
+	chmod +x $@
+
+mgmerge : mgmerge.sh
+	@rm -f $@
+	cat mgmerge.sh >$@
+	chmod +x $@
+
+mg_get_merge : mg_get_merge.sh
+	@rm -f $@
+	cat mg_get_merge.sh >$@
+	chmod +x $@
+
+xmg : xmg.sh
+	@rm -f $@
+	cat xmg.sh >$@
+	chmod +x $@
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+mostlyclean:
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get.sh	(revision 21112)
@@ -0,0 +1,447 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell
+# Changed to allow it to be driven from a file ~/.mg_getrc
+# by Bruce McKenzie, Oct 1994
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mg_get.sh	1.11 25 Mar 1994
+#
+###########################################################################
+#
+# "mg_get" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+# The file ~/.mg_get contains lines of the form:
+#     name	TYPE	files and directories
+# where name is the name supplied to mg_get, TYPE is one of
+# PARA, MAIL, DIR, DIR2, BIB or TXTIMG and says how mg_get should process the
+# named files and directories
+#
+###########################################################################
+
+if ($?MG_GETRC) then
+  set mg_getrc = $MG_GETRC
+else
+  set mg_getrc = ~/.mg_getrc
+endif
+
+# Added by [TS:May/95]
+# if we don't have an mg_getrc file in existence then create a default one
+if !(-e $mg_getrc) then 
+  echo "Can not find a .mg_getrc, so creating a default one: $mg_getrc"
+
+  if ($?MGSAMPLE) then
+    set SampleData = $MGSAMPLE
+  else
+    set SampleData = ./SampleData
+  endif
+
+  cat > $mg_getrc << END
+alice	PARA	$SampleData/alice13a.txt.Z
+davinci	TXTIMG	$SampleData/davinci
+mailfiles	MAIL	~/mbox ~/.sentmail
+allfiles	DIR	~/Mail
+mods	LINESEP	$SampleData/MODIFICATIONS
+bible	PLAIN	$SampleData/bible/genesis.txt.Z $SampleData/bible/revelation.txt.Z
+END
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: mg_get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set COLLECTION = $1
+set TYPE  = `grep \^$1 $mg_getrc | cut -f2`
+set FILES = `grep \^$1 $mg_getrc | cut -f3`
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($TYPE)
+  case PLAIN:
+# Just outputs the given text, assumes ^Bs already inserted 
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case LINESEP:
+# Assumes that the documents are separated by a line of =====
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f |  sed -e '/\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/s/.*//' < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case PARA:
+# Splits the file at blank lines, which corresponds to paragraphs.
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '   /^ *$/ {if (b!=1) printf "";b=1} \
+		\!/^ *$/ {print;b=0} \
+		 END {printf ""}'
+       end #foreach
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case BIB:
+# Takes a list of files that contain bibliographies, and splits them up 
+#   by putting ^B after each reference. Assumes that each reference
+#   begins with a line '^@'.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '/^@/&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #BIB
+
+  case MAIL:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f| awk '/xbtoa Begin/,/xbtoa End/ {next} /^From /&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #MAIL
+
+  case DIR:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz' -exec echo '<' {} '>' \;		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' -exec echo '<' {} '>' \;		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)' -exec echo '<' {} \
+#			'>' \; -exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec echo '<' {} '>' \; \
+			-exec cat {} \; -exec echo -n '' \;
+
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR
+
+
+
+  case DIR2:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory. Does not include filename
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz'  		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' 	 		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)'	\
+#			-exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR2
+
+
+
+
+  case TXTIMG:
+# compress and index the collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# Files that are related have the same prefix. For example,
+#  monaLisa.pgm might be a gray-level image, and
+#  monaLisa.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt	ascii text
+#    .ptm	scanned text stored as a bilevel image
+#    .pbm	a black and white image (typically a line drawing)
+#    .pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $FILES
+  set targetDir = $MGDATA/$COLLECTION
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   # create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null     # in case it already existed
+    
+      # take care if no match to foreach statements
+      set nonomatch
+
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+        if ($f == "$sourceDir/*.ptm") then
+                break
+        endif
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+        if ($f == "$sourceDir/*.pgm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+        if ($f == "$sourceDir/*.pbm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+      unset nonomatch
+    breaksw #-init
+
+    case '-text'
+      # take care if no match to foreach statements
+      set nonomatch
+
+      #output each text file and tmp.txt, with names of other associated files
+
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+        if ($f == "$sourceDir/*.txt") then
+           continue
+        endif
+        if ($f == "$sourceDir/*.tmp") then
+           continue
+        endif
+	set r = $f:r	#root name of file
+	set base = $r:t
+        set first_image = 1 # set so that I print out image header once
+                            # also signifies if header printed out at all
+
+	foreach d ($targetDir/$base.*)
+          if ($d == "$targetDir/$base.*") then
+                break
+          endif
+
+          if ($first_image) then
+            set first_image = 0
+            echo '::::::::::'  #separate the image information from the text
+            echo 'Image(s) available:'
+          endif
+
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+
+	end #foreach d (associated file)
+
+        if (! $first_image) then
+	  echo '::::::::::'  #separate the image information from the text
+        endif
+
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+      unset nonomatch
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #IMGTXT
+
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mg_get_merge.sh	(revision 21112)
@@ -0,0 +1,313 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get_merge.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# "mg_get_merge" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+###########################################################################
+
+
+
+# The directory containing the data files is stored in the environment
+#   variable MGSAMPLE
+
+if ($?MGSAMPLE) then
+  set SampleData = $MGSAMPLE
+else
+  set SampleData = ./SampleData
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($1)
+  case alice:
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+	echo "This is a test document 1 2 3 4 5 6 7 8 9 "
+	echo ""
+	echo "A B C"
+     breaksw
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case bible:
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+       # use uncompressed file if it exists
+       # revelation.txt already contains '^B' characters between verses
+       if (-e $SampleData/bible/revelation.txt) then
+          cat $SampleData/bible/revelation.txt
+       else
+          uncompress <$SampleData/bible/revelation.txt.Z
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bible
+
+
+
+  case mailfiles:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+# If messages are already in individual files, use "get allfiles"
+#   rather than this one.
+# The EXTRA documents added are in files in the directory mailextra
+#### The variable 'mailfiles' contains a list of files to be split up
+  set mailfiles = '~/mbox ~/.sentmail'
+  set mailextra = '~/extra/*'
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      foreach f ($mailextra)
+	awk '/^From /&&NR!=1{print ""} {print $0} END{print ""}' $f
+      end # foreach
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #mailfiles
+
+
+
+  case allfiles:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+# The variable 'directory' on the next line contains the directory to be used
+  set directory = ~/Mail
+
+# The date of the file ".mgbuildtime" in 'directory' is the last
+# build or merge. At the start of the merge ("-init" argument) the
+# file ".mgmergetime" is touched. Only files newer than ".mgbuildtime"
+# but older than ".mgmergetime" are output by mg_get_merge
+
+  switch ($flag)
+    case '-init':
+        touch $directory/.mgmergetime
+    breaksw
+
+    case '-text'
+      find $directory -type f -newer $directory/.mgbuildtime \! -newer $directory/.mgmergetime \! -name '.*' -exec echo '<' {} '>' \; -exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+        touch $directory/.mgmergetime
+        touch $directory/.mgbuildtime
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #allfiles
+
+
+
+
+  case davinci:
+# compress and index the DaVinci collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# It assumes that all files are in one directory ($sourceDir).
+# Files that are related have the same prefix. For example,
+#  monaLisa.img.pgm might be a gray-level image, and
+#  monaLisa.txt.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt.txt	ascii text
+#    .txt.pbm	scanned text stored as a bilevel image
+#    .img.pbm	a black and white image (typically a line drawing)
+#    .img.pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $SampleData/davinci
+  set targetDir = $MGDATA/davinci
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   #create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null #in case it already existed
+    
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+    breaksw #-init
+
+    case '-text'
+      #output each text file and tmp.txt, with names of other associated files
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+	set r = $f:r	#root name of file
+	set base = $r:t
+	set n = `/bin/ls $sourceDir/$base.* | wc -l`
+	if ($n != 1) then # more files than just the text one
+	 echo '::::::::::'  #separate the image information from the text
+	 echo 'Image(s) available:'
+	 foreach d ($targetDir/$base.*)
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+	 end #foreach d (associated file)
+	echo '::::::::::'  #separate the image information from the text
+	endif # if more than one file
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #davinci
+
+
+#---------------------------
+  case mods:
+# Splits the modifications file at the mod separator "--------"
+  switch ($flag)
+     case '-init':
+     breaksw
+ 
+     case '-text':
+       sed -e 's/^---.*//g' < ${SampleData}/MODIFICATIONS
+     breaksw
+ 
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw #mods
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild.sh	(revision 21112)
@@ -0,0 +1,339 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+###########################################################################
+#  mg.build
+# Creates a Full Text Retrieval database
+#
+#  @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+
+set complex = ""
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-d") then
+    shift
+    if ($#argv >= 1) then
+      set mgdata = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-d mgdata dir] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get
+endif
+
+if ($?mgdata) then
+  setenv MGDATA $mgdata
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# If do_text == 1 then do the text component of building a mg database. 
+# Otherwise don't do the text component.
+set do_text = 1
+
+
+# If do_invf == 1 then do the inversion component of building a mg database. 
+# Otherwise don't do the inversion component.
+set do_invf = 1
+
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+set do_pass2 = 1
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+#### Note: Set -S so novel words can be encoded, as this will happen if
+#### mgmerge is used.
+set mcd = -S
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+
+#build up the command lines for pass 1 and 2
+set pass1 = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${bname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+if ($do_text) then
+  set pass1 = (${pass1} -T1)
+  set pass2 = (${pass2} -T2)
+endif
+
+if ($do_invf) then
+  set pass1 = (${pass1} -I1)
+  set pass2 = (${pass2} -I2)
+endif
+  
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1}"
+      $get $text $complex| $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1} ${input_files}
+    $bindir/mg_passes ${pass1} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2}"
+      $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2}"
+      $get $text $complex | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2} ${input_files}
+    $bindir/mg_passes ${pass2} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+echo "mg_weights_build -f ${bname} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+echo "mg_invf_dict -f ${bname} -b 4096"
+$bindir/mg_invf_dict -f ${bname} -b 4096
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+# [RPAP - Jan 97: Stem Index Change]
+if ("$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+echo "-- The fast-loading compression dictionary has not been built."
+echo "-- If you wish to build it, execute the following command:"
+echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+echo "-----------------------------------"
+
+echo ""
+
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgbuild_4.sh	(revision 21112)
@@ -0,0 +1,435 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild_4.sh -- Script used to build mg text collection in 4 passes.
+# Copyright (C) 1994  Neil Sharman; Modified by tes@kbs 1995
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# Does an mgbuild in 4 passes instead of 2 passes.
+# This is useful if one does not have as much RAM and a large collection.
+#
+
+set complex = ""
+
+# ************ parse the command line arguments ************
+
+while ($#argv >= 1)
+  if ("$1" == "-s" || "$1" == "--source") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g" || "$1" == "--get") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c" || "$1" == "--complex") then
+    set complex = "-text"
+    shift
+  else
+    if (! $?text ) then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if (! $?text ) then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = `which $0` # find which one we are executing
+set bindir = $bindir:h # use it as the bindir path
+
+# ************ set up shell variables ************
+# Override these by providing a script to source using the -s option
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+set get_options = ""
+
+
+if (! $?get ) then
+  set complex = "-text"
+  set get = $bindir/mg_get $get_options
+endif
+
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# whether to build weights or not
+set do_weights = 1
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+# Text 1st pass
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+# Text 2nd pass
+set do_pass2 = 1
+
+# If do_pass3 == 1 then do pass3 of building a mg database. 
+# Otherwise don't do pass3.
+# Invf 1st pass
+set do_pass3 = 1
+
+# If do_pass4 == 1 then do pass4 of building a mg database. 
+# Otherwise don't do pass4.
+# Invf 2nd pass
+set do_pass4 = 1
+
+# Buffer size used by mg_passes
+# Should be big enough to hold largest document
+# In Kilobytes
+set buf_size = 3072
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 5
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -C
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+# ************ set up pass parameters ************
+
+if ($do_pass1) then
+  set pass1_text = (-f ${bname} -T1)
+endif
+
+if ($do_pass2) then
+  set pass2_text = (-f ${bname} -T2)
+endif
+
+if ($do_pass3) then
+  set pass3_invf = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+  set pass3_invf = (${pass3_invf} -N1)
+endif
+
+if ($do_pass4) then
+  set pass4_invf = (-f ${bname} -${invf_level} -N2)
+endif
+
+if ($strip_sgml) then
+  set pass1_text = (${pass1_text} -G)
+  set pass2_text = (${pass2_text} -G)
+  set pass3_invf = (${pass3_invf} -G)
+  set pass4_invf = (${pass4_invf} -G)
+endif
+
+if ($?trace) then
+  set pass1_text = (${pass1_text} -t ${trace})
+  set pass2_text = (${pass2_text} -t ${trace})
+  set pass3_invf = (${pass3_invf} -t ${trace})
+  set pass4_invf = (${pass4_invf} -t ${trace})
+endif
+
+if ($?trace_name) then
+  set pass1_text = (${pass1_text} -n ${trace_name})
+  set pass2_text = (${pass2_text} -n ${trace_name})
+  set pass3_invf = (${pass3_invf} -n ${trace_name})
+  set pass4_invf = (${pass4_invf} -n ${trace_name})
+endif
+
+if ($buf_size) then
+  set pass1_text = (${pass1_text} -b ${buf_size})
+  set pass2_text = (${pass2_text} -b ${buf_size})
+  set pass3_invf = (${pass3_invf} -b ${buf_size})
+  set pass4_invf = (${pass4_invf} -b ${buf_size})
+endif
+  
+if ($?comp_stats) then
+  set pass2_text = (${pass2_text} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+
+# ************ init get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+# ************************************* T E X T **********************************
+
+# ************ pass 1_text ************
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1_text}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1_text}"
+      $get $text $complex| $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1_text} ${input_files}
+    $bindir/mg_passes ${pass1_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ compression dict  ************
+if ($do_pass2) then 
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ************ pass 2_text ************
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2_text}"
+      $get $text $complex| $pass2filter | $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2_text}"
+      $get $text $complex| $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2_text} ${input_files}
+    $bindir/mg_passes ${pass2_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************************************* I N V F **********************************
+
+# ************ pass 3_invf ************
+if (${do_pass3}) then
+  if ($pipe) then
+    if ($?pass3filter) then
+      echo "$get $text $complex | $pass3filter | mg_passes ${pass3_invf}"
+      $get $text $complex| $pass3filter | $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass3_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass3_invf} ${input_files}
+    $bindir/mg_passes ${pass3_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ perfect hash  ************
+if ($do_pass4) then
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ pass 4_invf ************
+if (${do_pass4}) then
+  if ($pipe) then
+    if ($?pass4filter) then
+      echo "$get $text $complex | $pass4filter | mg_passes ${pass4_invf}"
+      $get $text $complex| $pass4filter | $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass4_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass4_invf} ${input_files}
+    $bindir/mg_passes ${pass4_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+
+# ************ build invf dictionary ************
+
+if ($do_pass4) then
+  echo "mg_invf_dict -f ${bname} -b 4096"
+  $bindir/mg_invf_dict -f ${bname} -b 4096
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ******** build indexes to stem dict *********
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_pass4 && "$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ build weights ************
+
+if ($do_pass4 && $do_weights) then
+  echo "mg_weights_build -f ${bname} -b ${weight_bits} "
+  $bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# *******************************************************************************
+
+
+# ************ print out statistics ************
+if ($do_pass1 && $do_pass2 && $do_pass3 && $do_pass4) then
+  echo "mgstat -f ${bname} -E"
+  $bindir/mgstat -f ${bname} -E
+  if ("$status" != "0") exit 1 
+endif
+
+
+# ************ cleanup get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+if ($do_pass2) then
+  echo "-- The fast-loading compression dictionary has not been built."
+  echo "-- If you wish to build it, execute the following command:"
+  echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+  echo "-----------------------------------"
+endif
+
+echo ""
+
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/mgmerge.sh	(revision 21112)
@@ -0,0 +1,417 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgmerge.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+
+set complex = ""
+set slow_merge = ""
+set guess_weights = "-w "
+# [RPAP - Feb 97: Level 3 Merge]
+set text_pass = 1
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else if ("$1" == "-S") then
+# set slow merge option in mg_invf_merge
+    set slow_merge = "-s "
+    shift
+  else if ("$1" == "-w") then
+# turn weights guessing off (rebuild weights file from scratch)
+    set guess_weights = ""
+    shift
+
+# [RPAP - Feb 97: Level 3 Merge]
+  else if ("$1" == "-T") then
+# turn text pass off
+    set text_pass = 0
+    shift
+
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get_merge
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the inversion method this may be either "I" or "N"
+# N is memory-efficient, I is faster for small collections
+set invf_method = N
+
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+# This only has an effect if $invf_method is "N".
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+# This only has an effect if $invf_method is "N".
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+# Note: The value *MUST* be the same as was used to build the collection
+# that is being merged.
+# In the current mg_invf_merge implementation the maximum level is 2. 
+# Paragraph-level inversion (invf_level = 3) is NOT supported.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -S
+
+# $merge_name is the name of the subdirectory under ${MGDATA} where
+# the mergeing steps will be performed. There should NOT be an mg
+# collection with this name! The default is "MERGE"
+set merge = "MERGE"
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ($do_indexes) then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+
+# Generate the directory where merging will take place
+if (-e $MGDATA/${merge}) then
+else
+  mkdir $MGDATA/${merge}
+endif
+
+# Generate the base name for the collection.
+# [RPAP - Feb 97: Level 3 Merge]
+set bname = ${coll_name}
+
+# generate the old, new and merge names in the $merge directory
+set oldname = ${merge}/${coll_name}.old
+set newname = ${merge}/${coll_name}.new
+set mergename = ${merge}/${coll_name}
+
+# build up the command lines for pass 1 and 2 to build a
+# dummy collection for the new documents
+set pass1 = (-f ${newname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${newname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+# Note that a -T1 pass isnt done since the old .text.stats file is used
+# as the compression model for compressing the new documents.
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Do not perform -T2 pass if text_pass == 0
+if ($text_pass) then
+  set pass2 = (${pass2} -T2)
+endif
+
+set pass1 = (${pass1} -${invf_method}1)
+set pass2 = (${pass2} -${invf_method}2)
+
+
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+########################################################################
+# Here is where mgmerge goes to work.
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "MGMERGE collection: ${bname}"
+echo "--------------------------------------------------------------"
+echo "FIRST PHASE: build new collection from new documents"
+echo "--------------------------------------------------------------"
+
+# move and rename the text dictionary file
+# [RPAP - Feb 97: Level 3 Merge] - only if performing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text.dict $MGDATA/${newname}.text.dict
+endif
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo ">> $get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if ($pipe) then
+  if ($?pass1filter) then
+    echo ">> $get $text $complex | $pass1filter | mg_passes ${pass1}"
+    $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass1}"
+    $get $text $complex| $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass1} ${input_files}"
+  $bindir/mg_passes ${pass1} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+echo "mg_perf_hash_build -f ${newname}"
+$bindir/mg_perf_hash_build -f ${newname}
+if ("$status" != "0") exit 1 
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ($?pass2filter) then
+    echo ">> $get $text $complex | $pass2filter | mg_passes ${pass2}"
+    $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass2}"
+    $get $text $complex | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass2} ${input_files}"
+  $bindir/mg_passes ${pass2} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo ">> $get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "---------------------------------------"
+date
+echo "--------------------------------------------------------------"
+echo "SECOND PHASE: merge the two collections"
+echo "--------------------------------------------------------------"
+
+# move the appropriate files from the base directory to the merge dir
+echo "(moving files....)"
+# [RPAP - Feb 97: Level 3 Merge]
+# Move files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text       $MGDATA/${oldname}.text
+  mv $MGDATA/${bname}.text.idx   $MGDATA/${oldname}.text.idx
+else
+# Need this file if building weights
+  cp -d $MGDATA/${bname}.text.idx   $MGDATA/${mergename}.text.idx
+endif
+mv $MGDATA/${bname}.invf       $MGDATA/${oldname}.invf
+mv $MGDATA/${bname}.invf.idx   $MGDATA/${oldname}.invf.idx
+mv $MGDATA/${bname}.invf.dict  $MGDATA/${oldname}.invf.dict
+# the old weight file may be used for the new file
+mv $MGDATA/${bname}.weight     $MGDATA/${mergename}.weight
+# EOC: the old invf.paragraph file may be used for the new file for Level 3 Invf
+if (-e $MGDATA/${bname}.invf.paragraph) then
+  mv $MGDATA/${bname}.invf.paragraph  $MGDATA/${mergename}.invf.paragraph
+endif
+echo "---------------------------------------"
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Only merge text if doing text pass
+if ($text_pass) then
+  echo ">> mg_text_merge -f ${mergename}"
+  $bindir/mg_text_merge -f $mergename
+  echo "---------------------------------------"
+endif
+
+echo ">> mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}"
+$bindir/mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}
+echo "---------------------------------------"
+
+#remove .weight file, so a new one will be created by mg_weights_build
+if ("$guess_weights" == "") then
+  echo "rm ${mergename}.weight"
+  rm $MGDATA/${mergename}.weight
+endif
+
+# mv $MGDATA/${mergename}.weight $MGDATA/${mergename}.w2
+
+echo ">> mg_weights_build -f ${mergename} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${mergename} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "---------------------------------------"
+
+echo ">> mg_invf_dict -f ${mergename} -b 4096"
+$bindir/mg_invf_dict -f ${mergename} -b 4096
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  echo "-----------------------------------"
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+endif
+
+echo "--------------------------------------------------------------"
+
+# move files back to base dir
+# [RPAP - Feb 97: Level 3 Merge]
+# mv the text files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${oldname}.text         $MGDATA/${bname}.text
+  mv $MGDATA/${mergename}.text.idx   $MGDATA/${bname}.text.idx
+  mv $MGDATA/${newname}.text.dict    $MGDATA/${bname}.text.dict
+endif
+mv $MGDATA/${mergename}.invf       $MGDATA/${bname}.invf
+mv $MGDATA/${mergename}.invf.idx   $MGDATA/${bname}.invf.idx
+mv $MGDATA/${mergename}.invf.dict  $MGDATA/${bname}.invf.dict
+mv $MGDATA/${mergename}.invf.dict.blocked  $MGDATA/${bname}.invf.dict.blocked
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  mv $MGDATA/${mergename}.invf.dict.blocked.1 $MGDATA/${bname}.invf.dict.blocked.1
+  mv $MGDATA/${mergename}.invf.dict.blocked.2 $MGDATA/${bname}.invf.dict.blocked.2
+  mv $MGDATA/${mergename}.invf.dict.blocked.3 $MGDATA/${bname}.invf.dict.blocked.3
+endif
+
+mv $MGDATA/${mergename}.weight     $MGDATA/${bname}.weight
+mv $MGDATA/${mergename}.weight.approx $MGDATA/${bname}.weight.approx
+mv $MGDATA/${mergename}.text.idx.wgt $MGDATA/${bname}.text.idx.wgt
+
+# EOC: mv .invf.paragraph file if exists
+if (-e $MGDATA/${mergename}.invf.paragraph) then
+  mv $MGDATA/${mergename}.invf.paragraph   $MGDATA/${bname}.invf.paragraph
+endif
+
+# remove old files in the MERGE directory
+rm -f $MGDATA/${merge}/*
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "--------------------------------------------------------------"
+
+echo ""
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/scripts/xmg.sh	(revision 21112)
@@ -0,0 +1,1376 @@
+#!/usr/local/bin/wish -f
+# Edit the line above to point to your version of wish
+#
+###############################################################################
+#
+# xmg -- X interface to the mg information retrieval system
+# Copyright (C) 1994  Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)xmg.sh	1.4 25 Mar 1994
+#
+###############################################################################
+#
+# "xmg" is an X interface to the mg information retrieval system.
+#
+# Type a query in the "Query" box.
+# The first line or so of each matching document is listed.
+# Click on any line to show the complete document.
+# Shift-click to show the document in a new window.
+# Press Next and Previous to cycle through the list of documents (Shift works
+#   here too). 
+# If you change the query, existing windows continue to refer to the original
+#   query. 
+# 
+# You can select either Boolean or Ranked queries.
+# Boolean queries: use "&" for AND, "|" for OR, "!" for NOT, and parentheses
+# for grouping 
+# Ranked queries:  - documents are listed in order of relevance; a weight is
+# 		     shown alongside each 
+#                  - weights are normalized to 100 for the most relevant
+#		     document 
+#                  - choose Approximate Ranking for greater retrieval speed. 
+# 
+# In document windows (other than the main window): 
+#		Typing 'n' = next document
+#		Typing 'N' = next document in new window
+#		Typing 'p' = previous document
+#		Typing 'P' = previous document in new window
+#		Typing 'q' = close window
+#
+###############################################################################
+
+# Date: 13 March 1994
+# Authors: Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+
+###############################################################################
+# Customizing xmg
+###############################################################################
+
+# To override the default font options, use RESOURCE_MANAGER properties like
+# the following in your .Xdefaults file
+#
+#	xmg.textfont: fixed
+#	xmg.buttonfont: fixed
+#
+# To override the default query type of ranked, use
+#
+#	xmg.querytype: boolean
+#
+# To override the default collection name (the first one listed by /bin/ls from
+# your MGDATA environment variable), use
+#
+#	xmg.collection: mail93
+
+###############################################################################
+# Global variables
+###############################################################################
+
+#   genuine constants:
+#
+#	env		the environment variables
+#	Terminator	terminator for mgquery
+#	MaxWindowHeight	the maximum height permitted for a document window
+#	ListBoxLength	the number of items in the document list
+#	HeadLength	the length document head displayed in the list box
+#	ImageString	the string at the start of a document associated
+#			with an image
+#
+#   things that rarely get changed:
+#
+#	querytype	the current query type (boolean, ranked, approx-ranked)
+#	booleanquery	says whether the query is boolean or ranked
+#	approxquery	says whether ranking is to be approximate or not
+#	collection	the name of the document collection being used
+#	fileId		file identifer for a pipe to the mgquery process
+#       errorOccurred	flag to say that there has been an I/O error on the
+#			current collection
+#
+#   things that reflect items in the main panel
+#
+#	query		the current query
+#	hits		number of hits for the current query
+#	display		shell command to display the image associated with
+#			the last document retrieved
+#	scrolledto	document prefixes have been obtained for all documents
+#			up to this position in the document list
+#
+#   static variable needed in just one procedure
+#
+#	top_rank	weight associated with the first document retrieved
+#	windowname	unique id for each document window
+
+###############################################################################
+# High-level procedures
+###############################################################################
+
+proc SaveDocument {w} {
+  set file [FSBox "Save document to file:"]
+
+  if {$file != ""} {
+    if [file exists $file] {
+      toplevel .r
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text "The file already exists"
+      button .r.replace -text "Replace" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"w\" $w" 
+      button .r.append -text "Append" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"a\" $w"
+      button .r.cancel -text "Cancel" -command {destroy .r} -width 10 -height 2
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.replace .r.append .r.cancel -side left -pady 10 -padx 20
+
+    } else {
+       WriteDocument $file "w" $w
+    }
+  }
+}
+
+proc Dialog {text command} {
+      if {![winfo exists .r] } { toplevel .r }
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text $text
+      button .r.ok -text "OK" -width 10 -height 2 \
+          -command "destroy .r $command"
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.ok -side bottom -pady 10 -padx 20
+}
+
+proc WriteDocument {file mode w} {
+    if {[catch {set f [open $file $mode]}] == 0} {
+        if [winfo exists $w] {
+            puts $f [$w get 0.0 end]
+        } else {
+            Dialog "Unable to save text" ""
+        }
+        flush $f
+        close $f
+    } else {
+        Dialog "Unable to open file for writing" "; SaveDocument $w"
+    }
+}
+
+# Send a query to mg and enter the response in the document list
+#
+proc sendQuery {window} {
+    global query hits querytype errorOccurred scrolledto
+
+    .m.t.q.retrieved.label configure -foreground Gray
+    .m.t.q.retrieved.hits configure -foreground Gray
+
+    sendto ".set query \"$querytype\""
+    sendto ".set mode \"docnums\""
+    foreach f "sdocnums sdocs srank" { $window.$f delete 0 end }
+    disableButton .m.d.buttons.previous
+    disableButton .m.d.buttons.next
+    if {[string index $query 0] == "." || [string length $query] > 255} {
+        set doclist ""
+    } else { set doclist [sendandget $query] }
+
+    if {$errorOccurred} {
+        enterText .m.d.text [mgqueryFailure]
+        .m.t.q.retrieved.label configure -foreground Black
+        .m.t.q.retrieved.hits configure -foreground Black 
+        update
+        return
+    }
+    sendto ".set query \"DocNums\""
+    sendto ".set mode  \"text\""
+    .m.t.q.retrieved.label configure -foreground Black
+    .m.t.q.retrieved.hits configure -foreground Black 
+    set hits [llength $doclist]
+    set i 0
+    global top_rank
+
+    scan [lindex $doclist 0] "%d %f %d" d rank size
+    set top_rank $rank
+    if {$top_rank == 0} {set top_rank 1}
+
+    while {$i < $hits} {
+	scan [lindex $doclist $i] "%d %f %d" d rank size
+        if {$rank == "Inf"} {set rank 1}
+	if {$querytype == "boolean"} {
+	    $window.srank insert end ""
+	} else {
+	    $window.srank insert end \
+		[format "%3.0f" [expr $rank / $top_rank * 99 + 1]]
+	}
+	$window.sdocnums insert end [format " %d" $d]
+	incr i
+    }
+
+    set scrolledto 0
+    scroll $window 0
+    bind $window.sdocs <ButtonRelease-1> \
+	"showDocument $window.sdocs %y .m.d \"$doclist\" \"$query\""
+    bind $window.sdocs <Shift-ButtonRelease-1> \
+	"showDocument $window.sdocs %y New \"$doclist\" \"$query\""
+    enterText .m.d.text ""
+}
+
+# Scrolling routine for the document list
+#
+proc scroll {window n} {
+    global hits scrolledto ListBoxLength Terminator fileId ImageString
+
+    $window.sdocnums yview $n
+    $window.sdocs yview $n
+    $window.srank yview $n
+    set last $n
+    if {$last <= 0} {set last 0}
+    incr last $ListBoxLength
+    if {$last > $hits} {set last $hits}
+
+#    .m config -cursor {watch}
+#    update
+    sendto ".set mode \"heads\""
+    sendto ".set terminator \"\""
+    while {$scrolledto < $last} {
+	if {[scan [$window.sdocnums get $scrolledto] "%d" d] == 1} {
+            sendto $d
+            gets $fileId line
+            set i [string first " " $line]
+            if {$i != -1} {set line [string range $line [expr $i + 1] end]}
+            if {[string compare $ImageString [string range $line 0 9]] == 0} {
+                set line [getImageDocumentHead $d]
+            }
+	    $window.sdocs insert $scrolledto $line
+	}
+	incr scrolledto
+    }
+    sendto ".set terminator \"$Terminator\\n\""
+    sendto ".set mode text"
+#   .m config -cursor {}
+}
+
+proc getImageDocumentHead {d} {
+    global HeadLength ImageString fileId
+
+    sendto ".set heads_length 1000"
+    sendto $d
+    gets $fileId line
+
+    set line [string range $line 11 end]
+
+    set s [string first $ImageString $line]
+    if {$s != -1} {
+        incr s 11
+        set line [string range $line $s [expr $s + $HeadLength]]
+    }
+  
+    sendto ".set heads_length $HeadLength"
+
+    return "* $line"
+}
+
+# Show document from the document list
+#
+proc showDocument {window y newWindow doclist query} {
+    showThisDocument [$window nearest $y] $doclist $newWindow $query
+}
+
+proc adjacentDocument {index doclist window q button} {
+    global ListBoxLength
+
+    showThisDocument $index $doclist $window $q
+
+    if {$window == ".m.d"} {
+      set first [.m.t.q.r.sdocs nearest 0]
+      if {$index < $first || $index >= $first + $ListBoxLength} {
+          scroll .m.t.q.r [expr $index - $ListBoxLength / 2]
+      }
+      .m.t.q.r.sdocs select from $index
+    } 
+}
+
+
+# Show document with this index in the document list
+#
+proc showThisDocument {index doclist window q} {
+    global collection MaxWindowHeight display windowname
+
+    set document ""
+    scan [lindex $doclist $index] "%d %f" document rank
+    if {$document == ""} { return }
+    set textlist [sendandget $document]
+	
+    if {[string length $display] != 0} { exec csh -c $display >& /dev/null & }
+	
+# throw away the first line
+    set text [join [lrange $textlist 1 end] "\n"]
+
+# if necessary, create a new window for this document
+    if {$window == "New"} {
+        .m.t.q.r.sdocs select clear
+	set window .m.document$windowname
+        incr windowname
+        set existing ""
+        set raised 0
+        foreach w [lrange [winfo children .m] 2 end] {
+           if {[lindex [$w.docnum configure -text] 4] == $index && \
+               [lindex [$w.query configure -text] 4] == $q} {
+              if {$raised} {
+                 closeDocumentWindow $w
+              } else {
+                  raise $w
+		  getFocus $w
+                  set window $w
+                  set raised 1
+              }
+           }
+        }
+  
+        if {!$raised} {
+   	    toplevel $window
+            $window configure -borderwidth 2 -relief raised
+            wm protocol $window WM_TAKE_FOCUS "getFocus $window"
+	    getFocus $window
+	    makeDocumentPanel $window
+            bind $window q "$window.buttons.close invoke"
+            label $window.docnum -text $index
+            label $window.query -text $q
+       }
+    }
+
+# configure the window for this document
+    set rows [expr [llength $textlist] - 1]
+    if {$window != ".m.d"} {
+	wm title $window "$collection \"$q\""
+	wm iconname $window "$document"
+	$window.text configure -height $rows
+	if {$rows > $MaxWindowHeight} {
+	    $window.text configure -height $MaxWindowHeight
+	}
+    }
+
+# show the document text in the window
+    enterText $window.text $text
+
+# configure the window's Previous and Next buttons
+    if {$index == 0} {
+        disableButton $window.buttons.previous
+    } else {
+	set newindex [expr $index-1]
+	bindBoth $window previous $newindex $doclist $q
+        bind $window p [bind $window.buttons.previous <1>]
+        bind $window P [bind $window.buttons.previous <Shift-1>]
+    }
+    if {$index == [expr [llength $doclist]-1]} {
+	disableButton $window.buttons.next 
+    } else {
+	set newindex [expr $index+1]
+	bindBoth $window next $newindex $doclist $q
+        bind $window n [bind $window.buttons.next <1>]
+        bind $window N [bind $window.buttons.next <Shift-1>]
+    }
+
+    if {$window != ".m.d"} { $window.docnum configure -text $index }
+
+    if {[string length $display] != 0} { 
+        set on "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground Black
+                 update\}
+        }"
+        set off "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground [lindex [.m.d configure -background] 4]
+                 update\}
+        }"
+        set t 0
+        while {$t < 5} {
+            after [expr $t * 1000] eval $on
+            after [expr $t * 1000 + 500] eval $off
+            incr t
+        }   
+    }
+}
+
+# Bind both Button-1 and Shift-Button-1 commands
+#
+proc bindBoth {window button index doclist query} {
+    $window.buttons.$button configure -state normal
+    bind $window.buttons.$button <Button-1> \
+	"adjacentDocument $index \"$doclist\" $window \"$query\" $button"
+    bind $window.buttons.$button <Shift-Button-1> \
+	"adjacentDocument $index \"$doclist\" New \"$query\" $button"
+}
+
+# Change the type of query
+#
+proc changeQuerytype {name element op} {
+    global querytype booleanquery approxquery
+
+    if {$booleanquery} {
+	.m.t.b.type.approx configure -state disabled
+	.m.t.q.retrieved.weight configure \
+             -foreground [lindex [.m configure -background] 4]
+	set querytype boolean
+    } else {
+	.m.t.b.type.approx configure -state normal
+	.m.t.q.retrieved.weight configure -foreground Black
+	if {$approxquery} {
+	    set querytype "approx-ranked"
+	} else {
+	    set querytype "ranked"
+	}
+    }
+}
+
+proc enterText {window text} {
+    $window configure -state normal
+    $window delete 0.0 end
+    $window insert 0.0 $text
+    $window configure -state disabled
+}
+
+###############################################################################
+# IO and initialization
+###############################################################################
+
+# Send a message to mg and return the reply
+#
+proc sendandget {message} {
+    global fileId Terminator display errorOccurred ImageString
+    set display ""
+
+    .m config -cursor {watch}
+    update idletasks
+    set reply ""
+    sendto "$message\n"
+    if {$message != "" && !$errorOccurred} {
+        gets $fileId line
+    	while {[string compare $Terminator $line] != 0} {
+            lappend reply $line;
+            gets $fileId line
+        }
+
+        if {[string compare $ImageString [lindex $reply 1]] == 0 } {
+          if {[string compare "MGDATA" [string range [lindex $reply 3] 0 5]] == 0} {
+             set display [string range [lindex $reply 3] 7 end];
+          }
+          set reply [lrange $reply 4 end]
+        }
+    }
+
+    .m config -cursor {}
+    return $reply
+}
+    
+# Send a message to mg
+#
+proc sendto {message} {
+    global fileId errorOccurred
+
+    puts $fileId $message
+    set errorOccurred [catch {flush $fileId}]
+}
+
+# Reinitialise mgquery to work with a new document collection
+#
+proc reInitMG {c} {
+    global fileId
+    catch {close $fileId}
+    initMG $c
+}
+
+# Initialize mgquery to work with a specified text collection
+#
+proc initMG {c} {
+    global query hits fileId collection Terminator env errorOccurred
+    global HeadLength
+
+    set collection $c
+    set query ""
+    set hits 0
+    set errorOccurred 0
+
+    foreach f "sdocs srank" { .m.t.q.r.$f delete 0 end }
+    enterText .m.d.text ""
+    .m.t.b.collect.m configure -text $collection
+    foreach w [winfo children .m] {
+	if {[string first ".m.document" $w] == 0} { closeDocumentWindow $w }
+    }
+
+    disableButton .m.d.buttons.previous 
+    disableButton .m.d.buttons.next
+
+    if {$c != ""} {
+        set f $env(MGDATA)/$c
+
+        if {[file exists $f] && [file readable $f]} {
+    	    set fileId [open "| mgquery $collection" RDWR]
+	    sendto ".set expert \"true\""
+	    sendto ".set terminator \"$Terminator\\n\""
+	    sendto ".set pager \"cat\""
+            sendto ".set heads_length $HeadLength"
+	    return
+        }
+    }
+    bind .m.t.q.e.entry <Return> {null}
+    enterText .m.d.text [initializationFailure]
+    update
+}
+
+# Routine to obtain values from RESOURCE_MANAGER
+#
+proc query_resource {option class default} {
+	set val [option get . $option $class]
+	if {[string length $val] == 0} { return $default }
+	return $val
+}
+
+
+proc disableButton {b} {
+    bind $b <Button-1> {null}
+    bind $b <Shift-Button-1> {null}
+    $b configure -state disabled
+}
+
+proc null {} {}
+
+###############################################################################
+# Setting up windows
+###############################################################################
+
+proc removeFromWindowList {window} {
+    global stacking
+
+    set i 0
+    while {$i < [llength $stacking]} {
+      if {$window == [lindex $stacking $i]} {
+         set stacking [lreplace $stacking $i $i]
+      }
+      incr i
+    }
+}
+
+proc getFocus {window} {
+    global stacking
+
+    focus $window
+    removeFromWindowList $window
+    set stacking [linsert $stacking 0 $window]
+}
+
+proc closeDocumentWindow {w} {
+   global stacking
+   removeFromWindowList $w
+   set t [lindex $stacking 0]
+
+   if {$t != ""} {
+     focus $t
+     raise $t
+  }
+  destroy $w
+}
+
+# Make a "document" panel, including Previous, Next, and Close buttons
+#
+proc makeDocumentPanel {window} {
+    frame $window.buttons
+    button $window.buttons.previous -text "Previous" -width 10 -state disabled
+    button $window.buttons.next -text "Next" -width 10 -state disabled
+    button $window.buttons.close -text "Close" -width 10 \
+	-command "closeDocumentWindow $window"
+    button $window.buttons.save -text "Save" -width 10 \
+        -command "SaveDocument $window.text"
+
+    label $window.buttons.image -text "Fetching image..." \
+            -foreground [lindex [$window configure -background] 4]
+    pack $window.buttons.previous $window.buttons.next -side left
+    pack $window.buttons.image -side left -padx 10
+    pack $window.buttons.close $window.buttons.save -side right 
+    pack $window.buttons -side top -fill x 
+
+    text $window.text -borderwidth 2 -relief raised \
+             -setgrid true -width 80 -state disabled
+
+    $window.text configure -yscrollcommand "$window.sy set" 
+    scrollbar $window.sy -relief flat -command "$window.text yview"
+    pack $window.sy -side right -fill y 
+    pack $window.text -side top -expand yes -fill both
+}
+
+# Make the "buttons" panel, with the query type and collection name
+#
+proc mkButtons {window} {
+    global collection booleanquery approxquery env
+    frame $window.type
+    label $window.type.label -text "Query type"
+
+    radiobutton $window.type.boolean -text "Boolean" -variable booleanquery \
+	  -value 1 -relief flat
+
+    radiobutton $window.type.ranked -text "Ranked" -variable booleanquery \
+	  -value 0 -relief flat
+
+    checkbutton $window.type.approx -text "Approximate" -variable approxquery \
+	  -relief flat
+
+    trace variable booleanquery w changeQuerytype
+    trace variable approxquery w changeQuerytype
+
+    pack $window.type.label   -side top -anchor w
+    pack $window.type.boolean -side top -anchor w -padx 12
+    pack $window.type.ranked  -side top -anchor w -padx 12
+    pack $window.type.approx  -side top -anchor w -padx 36
+
+    pack $window.type -side top -anchor w -pady 12
+
+    frame $window.collect
+    label $window.collect.label -text "Collection: "
+    pack $window.collect.label -side left -anchor w
+
+    getViewer
+
+    set dirs [getMGDATA]
+    set collection ""
+    if {$dirs != ""} { set collection [lindex $dirs 0] }
+   
+    menubutton $window.collect.m -menu $window.collect.m.menu \
+	-relief sunken
+    menu $window.collect.m.menu
+    foreach c $dirs {
+        if {[file readable $env(MGDATA)/$c] && 
+            [file isdirectory $env(MGDATA)/$c]} {
+    	    $window.collect.m.menu add command -label $c \
+	        -command "reInitMG $c"
+        }
+    }
+    pack $window.collect.m -side left
+    pack $window.collect -side bottom -anchor w
+
+
+}
+
+# Check that the MGDATA environment variable is set, and is non-null
+#
+proc getMGDATA {} {
+    global env
+
+    set MGDATA 0
+    foreach i [array names env] {
+	if {$i == "MGDATA"} {
+	    set dirs [exec /bin/ls $env(MGDATA)]
+	    if {[info exists dirs]} { return $dirs }
+	}
+    }
+
+    return ""
+}
+
+proc getViewer {} {
+    global env
+
+    set viewer 0
+    foreach i [array names env] {
+	if {$i == "MGIMAGEVIEWER"} {set viewer $env(MGIMAGEVIEWER)}
+    }
+
+    if {$viewer == 0} {set env(MGIMAGEVIEWER) "xloadimage -quiet -fit stdin"}
+}
+
+# Make the "query" panel, with the query and list of retrieved documents
+#
+proc mkQuery {window} {
+    global hits ListBoxLength
+
+# at the top of the "query" panel is the "entry" frame
+    frame $window.e
+    label $window.e.label -text "Query"
+    entry $window.e.entry -relief sunk -width 47 -textvariable query
+    pack $window.e.label -side left
+    pack $window.e.entry -side right -expand yes -fill x
+    focus default $window.e.entry
+    focus $window.e.entry
+
+    bind $window.e.entry <Control-Key> {null}
+    bind $window.e.entry <Return> "sendQuery $window.r"
+    bind $window.e.entry <Escape> { set query "" }
+    bind $window.e.entry <2> {
+	%W insert insert [selection get STRING]
+	tk_entrySeeCaret %W
+    }
+
+# next is the "retrieved" frame
+    frame $window.retrieved
+    label $window.retrieved.weight -text "Weight"
+    label $window.retrieved.label -text "   Documents retrieved: "
+    label $window.retrieved.hits -width 4 -relief sunk -textvariable hits
+    set documentsRetrieved 0
+    pack $window.retrieved.weight $window.retrieved.label \
+        $window.retrieved.hits -side left
+
+# next is the "results" frame (note that the docnums listbox doesn't appear on
+# the screen)
+    frame $window.r
+    scrollbar $window.r.sscroll -relief sunken -command "scroll $window.r"
+#    bind $window.r.sscroll <ButtonRelease-1> "updateSdocs $window.r.sdocs $window.r.sdocnums"
+    listbox $window.r.sdocnums
+    listbox $window.r.sdocs -relief sunken \
+            -setgrid 1 -geometry 50x$ListBoxLength
+    listbox $window.r.srank -yscroll "$window.r.sscroll set" \
+            -setgrid 1 -geometry 4x$ListBoxLength
+  
+    bind $window.r.srank <Any-1> {null}
+    bind $window.r.srank <Any-2> {null}
+    bind $window.r.srank <Any-3> {null}
+
+    bind $window.r.srank <Any-B1-Motion> {null}
+    bind $window.r.srank <Any-B2-Motion> {null}
+    bind $window.r.srank <Any-B3-Motion> {null}
+
+    bind $window.r.sdocs <Any-2> {null}
+    bind $window.r.sdocs <Any-B2-Motion> {null}
+
+    pack $window.r.sscroll -side right -fill y
+    pack $window.r.srank -side left 
+    pack $window.r.sdocs -side left -fill x -expand yes
+
+# pack them up
+    pack $window.e -side top -anchor e -pady 10 -fill x -expand yes
+    pack $window.retrieved -anchor w
+    pack $window.r -side bottom -anchor e -pady 10 -fill x -expand yes
+}
+
+# Make a "help" window
+#
+proc mkHelp {} {
+    if [winfo exists .help] { destroy .help }
+    toplevel .help
+    wm title .help "X interface to the mg information retrieval system"
+    wm iconname .help "About xmg"
+    frame .help.buttons
+    button .help.buttons.close -text Close -command "destroy .help" -width 10
+    button .help.buttons.help -text Help -width 10
+    button .help.buttons.conditions -text Conditions -width 10
+    button .help.buttons.warranty -text Warranty -width 10
+    button .help.buttons.bugs -text Bugs -width 10
+
+    frame .help.t
+    text .help.t.text -relief raised -bd 2 -setgrid true -height 24 -width 95
+    .help.t.text configure -yscrollcommand ".help.t.sy set" 
+    scrollbar .help.t.sy -relief flat -command ".help.t.text yview"
+    
+    pack .help.buttons.close .help.buttons.help -side right -fill x
+    pack .help.buttons.conditions .help.buttons.warranty \
+         .help.buttons.bugs -side left
+    pack .help.buttons -side bottom -fill x
+
+    pack .help.t.text -side left -expand yes -fill both
+    pack .help.t.sy -side right -fill y 
+    pack .help.t -side top -fill both -expand yes
+    enterText .help.t.text [HelpText]
+    grayButton help
+}
+
+proc grayButton {button} {
+    bind .help.buttons.help <1> "enterText .help.t.text \[HelpText\]
+         grayButton help" 
+    .help.buttons.help configure -state normal
+
+    bind .help.buttons.bugs <1> "enterText .help.t.text \[Bugs\]
+         grayButton bugs" 
+    .help.buttons.bugs configure -state normal
+
+    bind .help.buttons.warranty <1> "enterText .help.t.text \[Warranty\]
+         grayButton warranty"
+    .help.buttons.warranty configure -state normal
+
+    bind .help.buttons.conditions <1> "enterText .help.t.text \[Conditions\]
+         grayButton conditions" 
+    .help.buttons.conditions configure -state normal
+    disableButton .help.buttons.$button
+}
+
+# The text for errors in the MG environment variable
+#
+
+
+proc bookDetails {} {
+  return { \
+
+ This X interface was written by Bruce McKenzie, Ian Witten and Craig Nevill-Manning.  
+ For information on mg, see
+             Witten, I.H.,  Moffat, A. and Bell, T.C. (1994) "Managing gigabytes: compressing
+             and indexing documents and images." Van Nostrand Reinhold, New York.
+             ISBN 0-442-01863-0, US $54.95; call 1 (800) 544-0550 to order.
+}
+}
+
+proc initializationFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ Your MGDATA variable is unset or does not specify a directory with files in; therefore XMG
+ cannot proceed.
+} [bookDetails]]
+}
+
+proc mgqueryFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ There is a problem with the collection that you have chosen. Please re-index the collection before
+ you use it again.
+} [bookDetails]]
+
+}
+
+# The text for the help message
+#
+proc HelpText {} {
+    return [format "%s\n%s" {\
+This is an X interface to the mg information retrieval system.
+
+ Type a query in the "Query" box.
+ The first line or so of each matching document is listed. 
+ Click on any line to show the complete document.
+ Shift-click to show the document in its own window
+ Press Next and Previous to cycle through the list of documents (Shift works here too).
+ In document windows (other than the main window), typing n or p shows the next and previous 
+ documents; N or P creates a new window for them. q closes the most recently created window.
+ If you change the query, existing windows continue to refer to the original query.
+ 
+ You can select either Boolean or Ranked queries.
+ Boolean queries: use & for AND, | for OR, ! for NOT, and parentheses for grouping
+ Ranked queries:  documents are listed in order of relevance; a weight is shown alongside each
+                                  weights are normalized to 100 for the most relevant document
+                                  choose Approximate Ranking for greater retrieval speed. 
+ } [bookDetails]]
+}
+
+proc reformat {text} {
+    regsub -all "\n *\n *" $text "BLEQUEWFI" text
+    regsub -all "\[ \t\n\r\]+" $text " " text
+    regsub -all "BLEQUEWFI" $text "\n\n" text
+    return $text
+}
+
+proc Conditions {} {
+    return [reformat [join [sendandget ".conditions\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Warranty {} {
+    return [reformat [join [sendandget ".warranty\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Bugs {} {
+    return [reformat {
+Known shortcomings of xmg include:
+
+    Slow picture viewing.  Pictures are decompressed in their entirety before a window is created 
+        for the picture.  This often results in a long delay with no feedback. Moreover, when 
+        an image's document is displayed repeatedly, multiple identical images are shown.
+
+    Sequential viewing.  There is no way of going to the next document in the collection to see the
+        context of documents that have been retrieved.  This would be trivial to implement but
+        conflicts with the interpretation of the "previous" and "next" buttons.
+    }]
+}
+
+###############################################################################
+# The main program
+###############################################################################
+
+scan [info tclversion] "%d.%d" major minor
+
+if {$major < 7} {
+    puts "xmg requires version 7.0 of the tcl interpreter. You are using version $major.$minor"
+    exit
+}
+
+# Default font settings
+set defaulttextfont   -adobe-times-medium-r-normal--14-140-75-75-p-74-iso8859-1
+set defaultbuttonfont -adobe-times-bold-r-normal--14-140-75-75-p-77-iso8859-1
+set textfont   [query_resource textfont Textfont $defaulttextfont]
+set buttonfont [query_resource buttonfont Buttonfont $defaultbuttonfont]
+
+set MaxWindowHeight 10
+set HeadLength 100
+set scrolledto 0
+set ListBoxLength 12
+set windowname 0
+set Terminator "RECORD_END"
+set fileId ""
+set query ""
+set ImageString "::::::::::"
+set stacking ".m.t"
+
+tk_listboxSingleSelect Listbox
+option add *Text.wrap word
+
+option add *Text.font $textfont
+option add *Listbox.font $textfont
+option add *Entry.font $textfont
+
+option add *Button.font $buttonfont
+option add *Menubutton.font $buttonfont
+option add *Radiobutton.font $buttonfont
+option add *Checkbutton.font $buttonfont
+option add *Label.font $buttonfont
+option add *Menu.font $buttonfont
+
+wm title . "Managing Gigabytes"
+wm iconname . "xmg"
+frame .m -borderwidth 5 -relief raised
+
+# top part of frame
+frame .m.t
+
+# on the left side of the top part is the "buttons" panel
+frame .m.t.b
+mkButtons .m.t.b
+
+# on the right side is the "query" panel
+frame .m.t.q
+mkQuery .m.t.q
+
+set q [query_resource querytype Querytype approx-ranked]
+set booleanquery [expr \"$q\" == \"boolean\"]
+set approxquery [expr \"$q\" == \"approx-ranked\"]
+# the above lines will trigger changeQuerytype
+
+pack .m.t.b -side left -anchor nw
+pack .m.t.q -side right -anchor ne -fill both -expand yes
+
+# at the bottom is the "document" panel
+frame .m.d
+makeDocumentPanel .m.d
+
+.m.d.buttons.close configure -text "Exit" \
+    -command "destroy ."
+button .m.d.buttons.help -text "Help" -width 10 \
+	-command "mkHelp"
+
+pack .m.d.buttons.help -side right
+pack .m.t -side top -fill x -expand yes
+pack .m.d -side bottom -fill both -expand yes
+
+pack .m -fill both -expand yes
+wm geometry . +300+85
+
+set collection [query_resource collection Collection $collection]
+if {$argc>0} { set collection $argv }
+
+enterText .m.d.text [HelpText]
+
+initMG $collection
+
+###############################################################################
+###############################################################################
+# The following file selection box procedure was written by Andrew Donkin
+# of the University of Waikato. 
+###############################################################################
+
+proc FSBox { {fsBoxMessage "Select file:"}} {
+# xf ignore me 5
+  global fsBox
+
+  set fsBox(name) ""
+  set fsBox(internalPath) [pwd]
+  set fsBox(pattern) "*"
+  set fsBox(all) 0
+
+  # build widget structure
+
+  # start build of toplevel
+  if {"[info commands XFDestroy]" != ""} {
+    catch {XFDestroy .fsBox}
+  } {
+    catch {destroy .fsBox}
+  }
+  toplevel .fsBox -borderwidth 0
+  wm geometry .fsBox 350x300 
+  wm title .fsBox {File select box}
+  wm maxsize .fsBox 1000 1000
+  wm minsize .fsBox 100 100
+  # end build of toplevel
+
+  label .fsBox.message1 -anchor c -text "$fsBoxMessage"
+  frame .fsBox.frame1  -borderwidth 0  -relief raised
+
+  button .fsBox.frame1.ok  -text "OK"  -command "
+      global fsBox
+      set fsBox(name) \[.fsBox.file.file get\]
+      set fsBox(path) \[.fsBox.path.path get\]
+
+      set fsBox(internalPath) \[.fsBox.path.path get\]
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  button .fsBox.frame1.rescan -text "Rescan" -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+  }
+
+  button .fsBox.frame1.cancel -text "Cancel" -command "
+      global fsBox
+      set fsBox(name) {}
+      set fsBox(path) {}
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  frame .fsBox.path -borderwidth 0 -relief raised
+  frame .fsBox.path.paths 
+  label .fsBox.path.paths.paths -borderwidth 0 -text "Pathname:"
+
+  entry .fsBox.path.path -relief sunken
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+  frame .fsBox.pattern -borderwidth 0 -relief raised
+  frame .fsBox.pattern.patterns 
+
+  label .fsBox.pattern.patterns.patterns -borderwidth 0 -text "Selection pattern:"
+  entry .fsBox.pattern.pattern -relief sunken
+  .fsBox.pattern.pattern insert 0 $fsBox(pattern)
+  
+  frame .fsBox.files -borderwidth 0 -relief raised
+  scrollbar .fsBox.files.vscroll -relief raised -command ".fsBox.files.files yview"
+  scrollbar .fsBox.files.hscroll -orient horiz -relief raised -command ".fsBox.files.files xview" 
+  listbox .fsBox.files.files -exportselection false -relief raised -xscrollcommand \
+        ".fsBox.files.hscroll set" -yscrollcommand ".fsBox.files.vscroll set"
+  frame .fsBox.file -borderwidth 0 -relief raised
+  label .fsBox.file.labelfile -text "Filename:"
+  entry .fsBox.file.file -relief sunken
+  focus .fsBox.file.file
+
+  .fsBox.file.file delete 0 end
+  .fsBox.file.file insert 0 $fsBox(name)
+  
+  checkbutton .fsBox.pattern.all -offvalue 0 -onvalue 1 -text "Show all files" -variable fsBox(all) -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    }
+
+  FSBoxFSShow $fsBox(internalPath) $fsBox(pattern) $fsBox(all)
+
+  # bindings
+  bind .fsBox.files.files <Double-Button-1> "FSBoxFSFileSelectDouble %W %y"
+  bind .fsBox.files.files <ButtonPress-1> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-ButtonPress-1> "FSBoxFSFileSelect %W %y"
+
+  bind .fsBox.path.path <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.path.path <Up> {}"
+  bind .fsBox.path.path <Down> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+
+  bind .fsBox.file.file <Return> "
+    global fsBox
+    set fsBox(name) \[.fsBox.file.file get\]
+    set fsBox(path) \[.fsBox.path.path get\]
+    set fsBox(internalPath) \[.fsBox.path.path get\]
+
+    if {\"\[info commands XFDestroy\]\" != \"\"} {
+      catch {XFDestroy .fsBox}
+    } {
+      catch {destroy .fsBox}
+    }"
+  bind .fsBox.file.file <Up> {.fsBox.path.path icursor end
+    focus .fsBox.path.path}
+  bind .fsBox.file.file <Down> {.fsBox.pattern.pattern icursor end
+    focus .fsBox.pattern.pattern}
+
+  bind .fsBox.pattern.pattern <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)}
+  bind .fsBox.pattern.pattern <Up> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.pattern.pattern <Down> {}"
+
+  # packing
+  pack .fsBox.files.vscroll -side left -fill y 
+  pack .fsBox.files.hscroll -side bottom -fill x
+  pack .fsBox.files.files -side left -fill both -expand yes
+
+  pack .fsBox.file.labelfile -side left
+  pack .fsBox.file.file -side left -fill  both -expand yes
+
+  pack .fsBox.frame1.ok -side left -fill both -expand yes
+  pack .fsBox.frame1.rescan -side left -fill both -expand yes
+  pack .fsBox.frame1.cancel -side left -fill both -expand yes
+
+  pack .fsBox.path.paths.paths -side left
+
+  pack .fsBox.pattern.patterns.patterns -side left
+
+  pack .fsBox.path.paths -side left
+  pack .fsBox.path.path -side left -fill both -expand yes
+
+  pack .fsBox.pattern.patterns -side left
+  pack .fsBox.pattern.all -side right -fill both
+  pack .fsBox.pattern.pattern -side left -fill both -expand yes
+
+  pack .fsBox.message1 -side top -fill both
+  pack .fsBox.frame1 -side bottom -fill both
+  pack .fsBox.pattern -side bottom -fill both
+  pack .fsBox.file -side bottom -fill both
+  pack .fsBox.path -side bottom -fill both
+  pack .fsBox.files -side left -fill both -expand yes
+
+  update idletask
+  grab .fsBox
+  tkwait window .fsBox
+
+  if {"[string trim $fsBox(path)]" != "" ||
+      "[string trim $fsBox(name)]" != ""} {
+    if {"[string trimleft [string trim $fsBox(name)] /]" == ""} {
+      return [string trimright [string trim $fsBox(path)] /]
+    } {
+      return [string trimright [string trim $fsBox(path)] /]/[string trimleft [string trim $fsBox(name)] /]
+    }
+  }
+}
+
+proc FSBoxBindSelectOne { fsBoxW fsBoxY} {
+# xf ignore me 6
+
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    $fsBoxW select from $fsBoxNearest
+    $fsBoxW select to $fsBoxNearest
+  }
+}
+
+proc FSBoxFSFileSelect { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+        "[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+      set fsBoxFileName [string range $fsBoxTmpEntry 0             [expr [string length $fsBoxTmpEntry]-2]]
+      if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+          ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    } {
+      if {"[string index $fsBoxTmpEntry             [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0           [expr [string length $fsBoxTmpEntry]-2]]
+        if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    }
+    if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+      set fsBox(name) $fsBoxFileName
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 $fsBox(name)
+    }
+  }
+}
+
+proc FSBoxFSFileSelectDouble { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"$fsBoxTmpEntry" == "../"} {
+      set fsBoxTmpEntry [string trimright [string trim $fsBox(internalPath)] "@/"]
+      if {"$fsBoxTmpEntry" == ""} {
+        return
+      }
+      FSBoxFSShow [file dirname $fsBoxTmpEntry]         [.fsBox.pattern.pattern get] $fsBox(all)
+      .fsBox.path.path delete 0 end
+      .fsBox.path.path insert 0 $fsBox(internalPath)
+    } {
+      if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+          "[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0               [expr [string length $fsBoxTmpEntry]-2]]
+        if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+            ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+          set fsBoxFileName [string range $fsBoxTmpEntry 0                 [expr [string length $fsBoxTmpEntry]-2]]
+          if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+            set fsBoxFileName $fsBoxTmpEntry
+          }
+        } {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      }
+      if {[IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBox(internalPath) "[string trimright $fsBox(internalPath) {/@}]/$fsBoxFileName"
+        FSBoxFSShow $fsBox(internalPath) [.fsBox.pattern.pattern get] $fsBox(all)
+        .fsBox.path.path delete 0 end
+        .fsBox.path.path insert 0 $fsBox(internalPath)
+      } {
+        set fsBox(name) $fsBoxFileName
+        set fsBox(path) $fsBox(internalPath)
+        if {"[info commands XFDestroy]" != ""} {
+          catch {XFDestroy .fsBox}
+        } {
+          catch {destroy .fsBox}
+        }
+      }
+    }
+  }
+}
+
+proc FSBoxFSShow { fsBoxPath fsBoxPattern fsBoxAll} {
+# xf ignore me 6
+  global fsBox
+
+  set fsBox(pattern) $fsBoxPattern
+
+  if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+      [IsADir $fsBoxPath]} {
+    set fsBox(internalPath) $fsBoxPath
+  } {
+    if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+        [IsAFile $fsBoxPath]} {
+      set fsBox(internalPath) [file dirname $fsBoxPath]
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 [file tail $fsBoxPath]
+      set fsBoxPath $fsBox(internalPath)
+    } {
+      while {"$fsBoxPath" != "" && "$fsBoxPath" != "/" &&
+             ![file isdirectory $fsBoxPath]} {
+        set fsBox(internalPath) [file dirname $fsBoxPath]
+         set fsBoxPath $fsBox(internalPath)
+      }
+    }
+  }
+  if {"$fsBoxPath" == ""} {
+    set fsBoxPath "/"
+    set fsBox(internalPath) "/"
+  }
+  .fsBox.path.path delete 0 end
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+
+  if {[.fsBox.files.files size] > 0} {
+    .fsBox.files.files delete 0 end
+  }
+  if {$fsBoxAll} {
+    if {[catch "exec ls -F -a $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  } {
+    if {[catch "exec ls -F $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  }
+  set fsBoxElementList [lsort $fsBoxResult]
+
+  foreach fsBoxCounter [winfo children .fsBox.pattern.patterns.patterns] {
+    if {[string length [info commands XFDestroy]] > 0} {
+      catch {XFDestroy $fsBoxCounter}
+    } {
+      catch {destroy $fsBoxCounter}
+    }
+  }
+  menu .fsBox.pattern.patterns.patterns.menu
+
+  if {"$fsBoxPath" != "/"} {
+    .fsBox.files.files insert end "../"
+  }
+  foreach fsBoxCounter $fsBoxElementList {
+    if {[string match $fsBoxPattern $fsBoxCounter] ||
+        [IsADir [string trimright $fsBoxPath/$fsBoxCounter "/@"]]} {
+      if {"$fsBoxCounter" != "../" &&
+          "$fsBoxCounter" != "./"} {
+        .fsBox.files.files insert end $fsBoxCounter
+      }
+    }
+  }
+}
+
+proc IsADir { pathName} {
+# xf ignore me 5
+  if {[file isdirectory $pathName]} {
+    return 1
+  } {
+    catch "file type $pathName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $pathName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isdirectory $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsAFile { fileName} {
+# xf ignore me 5
+  if {[file isfile $fileName]} {
+    return 1
+  } {
+    catch "file type $fileName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $fileName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isfile $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsASymlink { fileName} {
+# xf ignore me 5
+  catch "file type $fileName" fileType
+  if {"$fileType" == "link"} {
+    return 1
+  }
+  return 0
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/GCC.mak
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/GCC.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/GCC.mak	(revision 21112)
@@ -0,0 +1,205 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+INSTALL = copy
+
+o = .o
+e = .exe
+/ = \\
+
+CPPFLAGS = -g
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D_LITTLE_ENDIAN
+LIBS = ..$/..$/lib$/libmg.a
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+ 
+INCLUDES = -I"..$/.." -I"..$/..$/lib"
+
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+EXEC = \
+  mgquery$e mg_weights_build$e mgstat$e mg_invf_dump$e \
+  mg_invf_dict$e mg_invf_rebuild$e mgdictlist$e \
+  mg_passes$e mg_perf_hash_build$e mg_compression_dict$e \
+  mg_text_estimate$e mg_fast_comp_dict$e mg_hilite_words$e \
+  mg_invf_merge$e mg_text_merge$e bool_tester$e \
+  mg_stem_idx$e  mgstemidxlist$e mg_decompress_text$e
+
+all:    $(EXEC) libtextin.a
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(QUERY_OBJS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+install: 
+	$(INSTALL) *$e ..$/..$/..$/..$/bin$/windows
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	$(DELETE) libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+clean:
+	$(DELETE) *$o 
+	$(DELETE) *$e
Index: /main/tags/3.01/gsdl/packages/mg/src/text/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/Makefile.in	(revision 21112)
@@ -0,0 +1,386 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#LIBS = ../../lib/libmg.a -lm
+LIBS = ../../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+YACC = @YACC@
+GSDLOS = @gsdlos@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+ 
+bindir = ../../../../bin/$(GSDLOS)
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+
+MISC          = Makefile.in help.mg.src bool_parser.y warranty.src conditions.src
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgquery mg_weights_build mgstat mg_invf_dump mg_invf_dict \
+                mg_invf_rebuild mgdictlist mg_passes mg_perf_hash_build \
+                mg_compression_dict mg_text_estimate mg_fast_comp_dict \
+                mg_hilite_words mg_invf_merge mg_text_merge bool_tester \
+                mg_stem_idx mgstemidxlist mg_decompress_text
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+                mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+                mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+                mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+                mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a 
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+help.mg.h : help.mg.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+warranty.h : warranty.src
+	sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+conditions.h : conditions.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+
+mg_hilite_words: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+bool_parser.c: bool_parser.y
+	$(YACC) $?
+	mv y.tab.c $@
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+            term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o environment$o
+
+bool_tester: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+
+mgquery: $(QUERY_OBJS)	
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat:	$(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+
+mg_invf_rebuild: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+
+mgdictlist: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = \
+                mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+
+mgstemidxlist: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text.o query_term_list.o mg_files.o mg_errors.o \
+                text_get.o locallib.o backend.o stem_search.o \
+                term_lists.o stemmer.o weights.o invf_get.o lists.o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+
+mg_invf_merge: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+# words$o,  removed to avoid conflict with mgpp libtextin.a
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o 
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC) libtextin.a
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+ 
+# This screws up distclean, and isn't needed as configure makes the
+# Makefile. (jrm21)
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/WIN32.MAK	(revision 21112)
@@ -0,0 +1,227 @@
+###########################################################################
+#
+# win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..\..\..
+DLL = 0
+DLLDEBUG = 0
+
+CC = cl
+
+CPPFLAGS = -GX
+!IF $(DLL)
+CPPFLAGS = $(CPPFLAGS) -MD
+!ELSE
+!IF ($(DLLDEBUG))
+CPPFLAGS = $(CPPFLAGS) -MDd
+!ENDIF
+!ENDIF
+
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I"$(GSDLHOME)\packages\mg\src\text" -I"$(GSDLHOME)\packages\mg" \
+	   -I"$(GSDLHOME)\packages\mg\lib" -I"$(GSDLHOME)"
+LDFLAGS =
+
+LIBS = "$(GSDLHOME)\packages\mg\lib\libmg.lib"
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj .cpp
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h          query_term_list.h
+
+EXEC          = mgquery$e mg_weights_build$e mgstat$e \
+		mg_invf_dump$e     mg_invf_dict$e \
+		mg_invf_rebuild$e     mgdictlist$e \
+		mg_passes$e     mg_perf_hash_build$e \
+		mg_compression_dict$e     mg_text_estimate$e \
+		mg_fast_comp_dict$e \
+		mg_hilite_words$e     mg_invf_merge$e \
+		mg_text_merge$e     bool_tester$e \
+		mg_stem_idx$e  mgstemidxlist$e \
+                mg_decompress_text$e
+
+all:    $(EXEC) libtextin.lib
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o environment$o
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
+ advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
+ odbccp32.lib
+
+BUILDER_OBJS = \
+	builder$o
+builder$e: $(BUILDER_OBJS)
+	$(LINK) $(BUILDER_OBJS) $(LIBS) $(LINK32_FLAGS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+install: 
+	if NOT EXIST "$(GSDLHOME)\bin\windows" mkdir "$(GSDLHOME)\bin\windows"
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDLHOME)\bin\windows"
+
+# words$o,  removed to avoid conflict with mgpp libtextin.a
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o
+
+libtextin.lib: $(LIB_OBJS)
+	if EXIST libtextin.lib del libtextin.lib
+	lib /out:libtextin.lib $(LIB_OBJS)
+
+clean:
+	del *$o 
+	del $(EXEC)
Index: /main/tags/3.01/gsdl/packages/mg/src/text/backend.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/backend.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/backend.c	(revision 21112)
@@ -0,0 +1,838 @@
+/**************************************************************************
+ *
+ * backend.c -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "filestats.h"
+#include "sptree.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+
+static File *
+OpenFile (char *base, char *suffix, unsigned long magic, int *ok)
+{
+  char FileName[512];
+  File *F;
+  sprintf (FileName, "%s%s", base, suffix);
+  if (!(F = Fopen (FileName, "rb", 0)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      mg_errno = MG_NOFILE;
+      MgErrorData (FileName);
+      if (ok)
+	*ok = 0;
+      return (NULL);
+    }
+  if (magic)
+    {
+      unsigned long m;
+      if (fread ((char *) &m, sizeof (m), 1, F->f) == 0)
+	{
+	  mg_errno = MG_READERR;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+      NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (m != magic)
+	{
+	  mg_errno = MG_BADMAGIC;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+    }
+  return (F);
+}
+
+
+static int 
+open_all_files (query_data * qd)
+{
+  int ok = 1;
+
+  qd->File_text = OpenFile (qd->textpathname, TEXT_SUFFIX, /* [RJM 06/97: text filename] */
+			    MAGIC_TEXT, &ok);
+  qd->File_fast_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				      TEXT_DICT_FAST_SUFFIX, MAGIC_FAST_DICT, NULL);
+  if (!qd->File_fast_comp_dict)
+    {
+      qd->File_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				     TEXT_DICT_SUFFIX, MAGIC_DICT, &ok);
+      qd->File_aux_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				    TEXT_DICT_AUX_SUFFIX, MAGIC_AUX_DICT, NULL);
+    }
+  else
+    qd->File_comp_dict = qd->File_aux_dict = NULL;
+
+  qd->File_stem = OpenFile (qd->pathname, INVF_DICT_BLOCKED_SUFFIX,
+			    MAGIC_STEM, &ok);
+
+  /* [RPAP - Jan 97: Stem Index Change]
+     These will fail if collection not built with stem indexes */
+  qd->File_stem1 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_1_SUFFIX,
+			     MAGIC_STEM_1, NULL);
+  qd->File_stem2 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_2_SUFFIX,
+			     MAGIC_STEM_2, NULL);
+  qd->File_stem3 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_3_SUFFIX,
+			     MAGIC_STEM_3, NULL);
+
+  qd->File_invf = OpenFile (qd->pathname, INVF_SUFFIX,
+			    MAGIC_INVF, &ok);
+
+  /* These will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_text_idx_wgt = OpenFile (qd->pathname, TEXT_IDX_WGT_SUFFIX,
+				    MAGIC_TEXI_WGT, NULL);
+  qd->File_weight_approx = OpenFile (qd->pathname, APPROX_WEIGHTS_SUFFIX,
+				     MAGIC_WGHT_APPROX, NULL);
+  if (qd->File_text_idx_wgt == NULL && qd->File_weight_approx == NULL)
+    qd->File_text_idx = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				  TEXT_IDX_SUFFIX, MAGIC_TEXI, NULL);
+  else
+    qd->File_text_idx = NULL;
+
+
+  if (!ok)
+    {
+      Fclose (qd->File_text);
+      if (qd->File_fast_comp_dict)
+	Fclose (qd->File_fast_comp_dict);
+      if (qd->File_comp_dict)
+	Fclose (qd->File_comp_dict);
+      Fclose (qd->File_stem);
+
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+
+      Fclose (qd->File_invf);
+      if (qd->File_text_idx_wgt)
+	Fclose (qd->File_text_idx_wgt);
+      if (qd->File_weight_approx)
+	Fclose (qd->File_weight_approx);
+      if (qd->File_text_idx)
+	Fclose (qd->File_text_idx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+static void 
+close_all_files (query_data * qd)
+{
+  Fclose (qd->File_text);
+  if (qd->File_fast_comp_dict)
+    Fclose (qd->File_fast_comp_dict);
+  if (qd->File_aux_dict)
+    Fclose (qd->File_aux_dict);
+  if (qd->File_comp_dict)
+    Fclose (qd->File_comp_dict);
+  Fclose (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    Fclose (qd->File_stem1);
+  if (qd->File_stem2)
+    Fclose (qd->File_stem2);
+  if (qd->File_stem3)
+    Fclose (qd->File_stem3);
+
+  Fclose (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    Fclose (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    Fclose (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    Fclose (qd->File_text_idx);
+}
+
+/* If textname is equal to null then name will be used instead */
+/* [RJM 06/97: text filename] */
+query_data *
+InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{
+  query_data *qd;
+  char *s;
+
+  if (textname == NULL) textname = name; /* [RJM 06/97: text filename] */
+
+  if (!(qd = Xmalloc (sizeof (query_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) qd, sizeof (*qd));
+
+  qd->mem_in_use = qd->max_mem_in_use = 0;
+
+  qd->doc_pos = qd->buf_in_use = 0;
+  qd->TextBufferLen = 0;
+  qd->DL = NULL;
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  qd->TL = NULL;
+  qd->QTL = NULL;
+
+  qd->TextBuffer = NULL;
+
+  qd->tot_hops_taken = 0;
+  qd->tot_num_of_ptrs = 0;
+  qd->tot_num_of_accum = 0;
+  qd->tot_num_of_terms = 0;
+  qd->tot_num_of_ans = 0;
+  qd->tot_text_idx_lookups = 0;
+
+  qd->hops_taken = 0;
+  qd->num_of_ptrs = 0;
+  qd->num_of_accum = 0;
+  qd->num_of_terms = 0;
+  qd->num_of_ans = 0;
+  qd->text_idx_lookups = 0;
+
+  qd->pathname = NULL; /* RJM 06/97: text filename] */
+  qd->textpathname = NULL; /* RJM 06/97: text filename] */
+
+  s = strrchr (dir, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 1)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 1)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      sprintf (qd->pathname, "%s%s", dir, name);
+      sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+    }
+
+  else
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 2)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 2)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      if (dir == NULL || dir[0] == '\0') {
+	sprintf (qd->pathname, "%s", name);
+	sprintf (qd->textpathname, "%s", textname); /* [RJM 06/97: text filename] */
+      } else {
+	sprintf (qd->pathname, "%s%s", dir, name);
+	sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+      }
+#else
+      sprintf (qd->pathname, "%s/%s", dir, name);
+      sprintf (qd->textpathname, "%s/%s", dir, textname); /* [RJM 06/97: text filename] */
+#endif
+    }
+
+  if (open_all_files (qd) == -1)
+    {
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->Start);
+
+  /* Initialise the stemmed dictionary system */
+  if (!(qd->sd = ReadStemDictBlk (qd->File_stem)))
+    {
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if ((qd->sd->sdh.indexed & 7) && qd->File_stem1 && qd->File_stem2 && qd->File_stem3)
+    {
+      if (!(qd->sd->stem1 = ReadStemIdxBlk (qd->File_stem1)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem2 = ReadStemIdxBlk (qd->File_stem2)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem3 = ReadStemIdxBlk (qd->File_stem3)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      }
+  else if (qd->sd->sdh.indexed != 0)
+    {
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  else
+    {
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+      qd->File_stem1 = NULL;
+      qd->File_stem2 = NULL;
+      qd->File_stem3 = NULL;
+      qd->sd->stem1 = NULL;
+      qd->sd->stem2 = NULL;
+      qd->sd->stem3 = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->StemDict);
+  if (qd->File_weight_approx)
+    {
+      if (!(qd->awd = LoadDocWeights (qd->File_weight_approx,
+				      qd->sd->sdh.num_of_docs)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+    }
+  else
+    qd->awd = NULL;
+
+
+  if (iqt)
+    GetTime (&iqt->ApproxWeights);
+
+  if (!(qd->cd = LoadCompDict (qd->File_comp_dict, qd->File_aux_dict,
+			       qd->File_fast_comp_dict)))
+    {
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->CompDict);
+
+  if (!(qd->id = InitInvfFile (qd->File_invf, qd->sd)))
+    {
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  if ((qd->File_text_idx_wgt == NULL || qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      mg_errno = MG_INVERSION;
+      return (NULL);
+    }
+  if (iqt)
+    GetTime (&iqt->Invf);
+
+  if (!(qd->td = LoadTextData (qd->File_text, qd->File_text_idx_wgt,
+			       qd->File_text_idx)))
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+
+/*
+
+This code is based on the TREC_MODE code below to read the .paragraph
+file to determine what document numbers correspond to what paragraphs.
+This code is more space efficient, reading in the .paragraph file
+into memory as an accumulate docnum array.  Eg.  the .paragraph may contain
+
+	[5 3 6 4 7 9 4]
+
+indicating the first document has 5 paragraphs, the next 3, etc.
+This will be stored in memory as
+
+	[0 5 8 14 18 25 34 38]
+
+so a binary search can be performed.  The first 0 is for convenience;
+it prevents testing boundary conditions.
+
+
+The TREC_MODE code does this differently; it stores the array
+
+	[1 1 1 1 1 2 2 2 3 3 3 3 3 3 ....]
+
+allowing directy paragraph to docnum conversion, at the expense
+of memory.
+
+*/ 
+  qd->paragraph = NULL;
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      unsigned long magic;
+      FILE *paragraph;
+      int i;
+      char paraFile[512];
+
+      sprintf(paraFile, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+      paragraph = fopen(paraFile, "rb");
+      if (!paragraph)
+	FatalError(1, "Unable to open 'paraFile'.", paraFile);
+
+      fread((void *)&magic, sizeof(magic), 1, paragraph);
+      qd->paragraph = Xmalloc((qd->td->cth.num_of_docs+1)*sizeof(int));
+      qd->paragraph[0] = 0;
+      for (i = 1; i <= qd->td->cth.num_of_docs; i++)
+	{
+	  int count;
+
+	  if (fread((void *)&count, sizeof(count), 1, paragraph) != 1)
+	    FatalError(1, "Unexpected EOF while reading '%s'.", paraFile);
+	  NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  qd->paragraph[i] = qd->paragraph[i-1]+count;
+	}
+
+      fclose (paragraph); /* [RJM 07/98: Memory Leak] */
+    }
+  
+  
+#endif
+
+#ifdef TREC_MODE
+  {
+    extern char *trec_ids;
+    extern long *trec_paras;
+    int size;
+    char FileName[512];
+    FILE *f;
+    if (!strstr (qd->pathname, "trec"))
+      goto error;
+    sprintf (FileName, "%s%s", qd->pathname, ".DOCIDS");
+    if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+      {
+	Message ("Unable to open \"%s\"", FileName);
+	goto error;
+      }
+    fseek (f, 0, 2);
+    size = ftell (f);
+    fseek (f, 0, 0);
+    trec_ids = Xmalloc (size);
+    if (!trec_ids)
+      {
+	fclose (f);
+	goto error;
+      }
+    fread (trec_ids, 1, size, f);
+    fclose (f);
+    if (qd->id->ifh.InvfLevel == 3)
+      {
+	int i, d;
+	unsigned long magic;
+	trec_paras = Xmalloc (qd->sd->sdh.num_of_docs * sizeof (long));
+	if (!trec_paras)
+	  {
+	    Xfree (trec_ids);
+	    trec_ids = NULL;
+	    goto error;
+	  }
+	sprintf (FileName, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+	if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	  {
+	    Message ("Unable to open \"%s\"", FileName);
+	    goto error;
+	  }
+	if (fread ((char *) &magic, sizeof (magic), 1, f) != 1 ||
+	    NTOHUL(magic) != MAGIC_PARAGRAPH)  /* [RPAP - Jan 97: Endian Ordering] */
+	  {
+	    fclose (f);
+	    Message ("Bad magic number in \"%s\"", FileName);
+	    goto error;
+	  }
+
+	for (d = i = 0; i < qd->td->cth.num_of_docs; i++)
+	  {
+	    int count;
+	    if (fread ((char *) &count, sizeof (count), 1, f) != 1)
+	      {
+		fclose (f);
+		goto error;
+	      }
+	    NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	    while (count--)
+	      trec_paras[d++] = i;
+	  }
+	fclose (f);
+      }
+    goto ok;
+  error:
+    if (trec_ids)
+      Xfree (trec_ids);
+    if (trec_paras)
+      Xfree (trec_paras);
+    trec_ids = NULL;
+    trec_paras = NULL;
+  ok:
+    ;
+  }
+#endif
+
+  if (iqt)
+    GetTime (&iqt->Text);
+
+  return (qd);
+}
+
+
+
+
+
+
+/*
+ * Change the amount of memory currently in use
+ *
+ */
+void 
+ChangeMemInUse (query_data * qd, long delta)
+{
+  qd->mem_in_use += delta;
+  if (qd->mem_in_use > qd->max_mem_in_use)
+    qd->max_mem_in_use = qd->mem_in_use;
+}
+
+
+void 
+FinishQuerySystem (query_data * qd)
+{
+/* [RJM 07/98: Memory Leak] */
+#if defined(PARADOCNUM) || defined(NZDL)
+  if (qd->paragraph != NULL) {
+    Xfree (qd->paragraph);
+    qd->paragraph = NULL;
+  }
+#endif
+
+  FreeTextData (qd->td);
+  FreeInvfData (qd->id);
+  FreeCompDict (qd->cd);
+  if (qd->awd)
+    FreeWeights (qd->awd);
+  FreeStemDict (qd->sd);
+  close_all_files (qd);
+  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+  Xfree (qd->pathname);
+  FreeQueryDocs (qd);
+  if (qd->TL != NULL) FreeTermList(&qd->TL);         /* [RJM 07/98: Memory Leak] */
+  if (qd->QTL != NULL) FreeQueryTermList(&qd->QTL);  /* [RJM 07/98: Memory Leak] */
+  Xfree (qd);
+
+  /* other global stuff hanging around */
+  MgErrorDeinit ();
+}
+
+
+void 
+ResetFileStats (query_data * qd)
+{
+  ZeroFileStats (qd->File_text);
+  if (qd->File_comp_dict)
+    ZeroFileStats (qd->File_comp_dict);
+  if (qd->File_fast_comp_dict)
+    ZeroFileStats (qd->File_fast_comp_dict);
+  ZeroFileStats (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    ZeroFileStats (qd->File_stem1);
+  if (qd->File_stem2)
+    ZeroFileStats (qd->File_stem2);
+  if (qd->File_stem3)
+    ZeroFileStats (qd->File_stem3);
+
+  ZeroFileStats (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    ZeroFileStats (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    ZeroFileStats (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    ZeroFileStats (qd->File_text_idx);
+}
+
+
+void 
+TransFileStats (query_data * qd)
+{
+  qd->File_text->Current = qd->File_text->Cumulative;
+  if (qd->File_comp_dict)
+    qd->File_comp_dict->Current = qd->File_comp_dict->Cumulative;
+  if (qd->File_fast_comp_dict)
+    qd->File_fast_comp_dict->Current = qd->File_fast_comp_dict->Cumulative;
+  qd->File_stem->Current = qd->File_stem->Cumulative;
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    qd->File_stem1->Current = qd->File_stem1->Cumulative;
+  if (qd->File_stem2)
+    qd->File_stem2->Current = qd->File_stem2->Cumulative;
+  if (qd->File_stem3)
+    qd->File_stem3->Current = qd->File_stem3->Cumulative;
+
+  qd->File_invf->Current = qd->File_invf->Cumulative;
+  if (qd->File_text_idx_wgt)
+    qd->File_text_idx_wgt->Current = qd->File_text_idx_wgt->Cumulative;
+  if (qd->File_weight_approx)
+    qd->File_weight_approx->Current = qd->File_weight_approx->Cumulative;
+  if (qd->File_text_idx)
+    qd->File_text_idx->Current = qd->File_text_idx->Cumulative;
+}
+
+
+void 
+FreeTextBuffer (query_data * qd)
+{
+  if (qd->TextBuffer)
+    {
+      Xfree (qd->TextBuffer);
+      ChangeMemInUse (qd, -qd->TextBufferLen);
+    }
+  qd->TextBuffer = NULL;
+  qd->TextBufferLen = 0;
+}
+
+void 
+FreeQueryDocs (query_data * qd)
+{
+  qd->doc_pos = 0;
+  qd->buf_in_use = 0;
+  if (qd->DL)
+    {
+      int i;
+      for (i = 0; i < qd->DL->num; i++)
+	if (qd->DL->DE[i].CompTextBuffer)
+	  {
+	    Xfree (qd->DL->DE[i].CompTextBuffer);
+	    qd->DL->DE[i].CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -qd->DL->DE[i].Len);
+	  }
+      Xfree (qd->DL);
+    }
+  qd->DL = NULL;
+  FreeTextBuffer (qd);
+}
+
+int 
+LoadCompressedText (query_data * qd, int max_mem)
+{
+  DocEntry *DE;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+  if (!DE->CompTextBuffer)
+    {
+      int i;
+      DocEntry *de;
+      for (i = 0, de = qd->DL->DE; i < qd->DL->num; i++, de++)
+	if (de->CompTextBuffer)
+	  {
+	    Xfree (de->CompTextBuffer);
+	    de->CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -de->Len);
+	  }
+      if (LoadBuffers (qd, &qd->DL->DE[qd->doc_pos], max_mem,
+		       qd->DL->num - qd->doc_pos) == -1)
+	return -1;
+    }
+  return 0;
+}
+
+int 
+GetDocNum (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].DocNum;
+}
+
+DocEntry *
+GetDocChain (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+  return &(qd->DL->DE[qd->doc_pos]);
+}
+
+float 
+GetDocWeight (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Weight;
+}
+
+long 
+GetDocCompLength (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Len;
+}
+
+
+u_char *
+GetDocText (query_data * qd, unsigned long *len)
+{
+  DocEntry *DE;
+  int ULen;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+
+  if (!DE->CompTextBuffer)
+    {
+      fprintf (stderr, "The compressed text buffer is NULL\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  FreeTextBuffer (qd);
+
+  qd->TextBufferLen = (int) (qd->td->cth.ratio * 1.01 *
+			     DE->Len) + 100;
+  if (!(qd->TextBuffer = Xmalloc (qd->TextBufferLen)))
+    {
+      fprintf (stderr, "No memory for TextBuffer\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  DecodeText (qd->cd, (u_char *) (DE->CompTextBuffer), DE->Len,
+	      (u_char *) (qd->TextBuffer), &ULen);
+  qd->TextBuffer[ULen] = '\0';
+
+  if (ULen >= qd->TextBufferLen)
+    {
+      fprintf (stderr, "%d >= %d\n", ULen, qd->TextBufferLen);
+      mg_errno = MG_BUFTOOSMALL;
+      return (NULL);
+    }
+
+  if (len)
+    *len = ULen;
+
+  return qd->TextBuffer;
+}
+
+int 
+NextDoc (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return 0;
+  qd->doc_pos++;
+  return qd->doc_pos < qd->DL->num;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/backend.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/backend.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/backend.h	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * backend.h -- Underlying routines and datastructures for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef BACKEND_H
+#define BACKEND_H
+
+#include "sysfuncs.h"
+
+#include "timing.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	/* N parameter for decoding inverted file entries */
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+     *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    u_char *word_data[2];
+    u_char **words[2];
+    int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+						   for the hybrid methods */
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    u_char ***values[2];
+    u_char *escape[2];
+    huff_data *chars_huff[2];
+    u_long **chars_vals[2];
+    huff_data *lens_huff[2];
+    u_long **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx   /* [RPAP - Jan 97: Stem Index Change] */
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemDict;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+
+typedef struct RankedQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;           /* use exact weights for ranking or not */
+    long MaxDocsToRetrieve;	/* may be -1 for all */
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		/* 'A' = array,  'S' = splay tree,  'H' = hash_table */
+    long MaxAccums;		/* may be -1 for all */
+    long MaxTerms;		/* may be -1 for all */
+    int StopAtMaxAccum; /* Stop at maximum accumulator or not */
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo
+  {
+    long MaxDocsToRetrieve;
+  }
+BooleanQueryInfo;
+
+
+/* [TS:24/Aug/94] - maximum number of characters in term string */
+#define MAXTERMSTRLEN 1023
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+#if defined(PARADOCNUM) || defined(NZDL)
+    int *paragraph;
+#endif
+    char *pathname;
+    char *textpathname; /* [RJM 06/97: text filename] */
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		/* [TS:Oct/94] - so term list for query can easily be accessed */
+    u_char *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;    /* [RPAP - Feb 97: Term Frequency] */
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+/* [RJM 06/97: text filename] */
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void FreeTextBuffer (query_data * qd);
+
+void FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+u_char *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.c	(revision 21112)
@@ -0,0 +1,1108 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994,95  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:20  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1995/07/27  04:54:55  tes
+   * Committing optimiser before I add the code
+   * to find diff-nodes in any kind of tree.
+   *
+   * Revision 1.3  1994/10/20  00:02:55  tes
+   * Fixed bug with bool_optimiser:
+   *   - it couldn't handle "false | false"
+   *   - it couldn't handle "true & true"
+   * These would collapse into an empty list and cause an error.
+   *
+   * Revision 1.2  1994/10/18  06:11:02  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:28  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bool_tree.h"
+#include "term_lists.h"
+#include "messages.h"
+
+typedef int (*CompFunc) (const void *, const void *);
+typedef void (*TreeFunc) (bool_tree_node *);
+
+/* --- routines --- */
+static void PermeateNots (bool_tree_node * tree);
+static void DoubleNeg (bool_tree_node * not_tree);
+static void AndDeMorgan (bool_tree_node * not_tree);
+static void OrDeMorgan (bool_tree_node * not_tree);
+static int AndDistribute (bool_tree_node * tree);
+static void DNF_Distribute (bool_tree_node * tree);
+static void ReplaceWithChildren (bool_tree_node * tree);
+static void CollapseTree (bool_tree_node * tree);
+static void AndTrueFalse (bool_tree_node * and_tree);
+static void OrTrueFalse (bool_tree_node * or_tree);
+static void TF_Idempotent (bool_tree_node * tree);
+static void AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func);
+static void DiffDNF (bool_tree_node * or_node);
+static void DiffTree (bool_tree_node * tree);
+static void AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp);
+static void AndOrSort (bool_tree_node * tree, TermList * tl);
+static void AndSortDNF (bool_tree_node * or_node, TermList * tl);
+static void MarkOrTerms (bool_tree_node * tree, TermList * tl);
+static int calc_sum_ft (bool_tree_node * term_tree, TermList * tl);
+static int AndOrSort_comp (const void *A, const void *B);
+static int AndSort_comp (const void *A, const void *B);
+
+/* =========================================================================
+ * Function: OptimiseBoolTree
+ * Description: 
+ *      For case 2:
+ *        Do three major steps:
+ *        (i) put into standard form
+ *            -> put into DNF (disjunctive normal form - or of ands)
+ *            Use DeMorgan's, Double-negative, Distributive rules
+ *        (ii) simplify
+ *             apply idempotency rules
+ *        (iii) ameliorate
+ *              convert &! to diff nodes, order terms by frequency,...
+ *     Could also do the matching idempotency laws i.e. ...
+ *     (A | A), (A | !A), (A & !A), (A & A), (A & (A | B)), (A | (A & B)) 
+ *     Job for future.... ;-) 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+OptimiseBoolTree (bool_tree_node * parse_tree, TermList * tl, int type)
+{
+  switch (type)
+    {
+    case 1:
+      /* --- sorts for and of or-terms --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      CollapseTree (parse_tree);
+      DiffTree (parse_tree);
+      MarkOrTerms (parse_tree, tl);
+      AndOrSort (parse_tree, tl);
+      break;
+    case 2:
+      /* --- put into DNF --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      DNF_Distribute (parse_tree);
+      CollapseTree (parse_tree);
+      AndSortDNF (parse_tree, tl);
+      DiffDNF (parse_tree);
+      break;
+    default:
+      /* --- do nothing --- */
+      break;
+    }				/*switch */
+}				/*OptimiseBoolTree */
+
+/* =========================================================================
+ * Function: DoubleNeg
+ * Description: 
+ *      !(!(a) = a
+ *      Assumes binary tree.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+DoubleNeg (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *sibling = BOOL_SIBLING (not_tree);
+
+  /* copy a into not tree */
+  bcopy (a, not_tree, sizeof (bool_tree_node));
+
+  BOOL_SIBLING (not_tree) = sibling;
+
+  Xfree (child);
+  Xfree (a);
+
+}
+
+/* =========================================================================
+ * Function: AndDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'and'  i.e. !(a & b) <=> (!a) | (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+AndDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'or' */
+  BOOL_TAG (not_tree) = N_or;
+
+  /* change 'and' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: OrDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'or' i.e. !(a | b) <=> (!a) & (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+OrDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'and' */
+  BOOL_TAG (not_tree) = N_and;
+
+  /* change 'or' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: PermeateNots
+ * Description: 
+ *      Use DeMorgan's and Double-negative
+ *      Assumes tree in binary form (i.e. No ands/ors collapsed)
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PermeateNots (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if (BOOL_TAG (tree) == N_not)
+    {
+      switch (BOOL_TAG (BOOL_CHILD (tree)))
+	{
+	case N_not:
+	  DoubleNeg (tree);
+/** NOTE: I do not go on to do the children **/
+	  /* I permeate the nots on the tree here & leave it at that */
+	  /* In this case I do not know what type 'tree' will end up */
+	  PermeateNots (tree);
+	  return;
+	case N_and:
+	  AndDeMorgan (tree);
+	  break;
+	case N_or:
+	  OrDeMorgan (tree);
+	  break;
+	default:
+	  /* leave alone */
+	  break;
+	}
+    }
+
+  /* Permeate Nots for children */
+  if (BOOL_HAS_CHILD (tree))
+    {
+      PermeateNots (BOOL_CHILD (tree));
+      PermeateNots (BOOL_SIBLING (BOOL_CHILD (tree)));
+    }
+
+}
+
+/* =========================================================================
+ * Function: DNF_Distribute
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+#define ITER_LIMIT 500		/* if reach limit then probably in trouble */
+
+static void
+DNF_Distribute (bool_tree_node * tree)
+{
+  int i = 0;
+
+  /* multiple passes until no change */
+  /* not very smart :-( */
+  while (1)
+    {
+      if (AndDistribute (tree) == 0)
+	return;
+      if (i == ITER_LIMIT)
+	FatalError (1, "DNF infinite loop");
+      i++;
+    }				/*while */
+
+}
+
+/* =========================================================================
+ * Function: AndDistribute
+ * Description: 
+ *      (a | b) & A <=> (a & A) | (b & A)
+ * Input: 
+ *      binary tree of "AND" , "OR"s.
+ * Output: 
+ *      return 1 if changed the tree
+ *      return 0 if there was NO change (no distributive rule to apply)
+ * ========================================================================= */
+
+static int
+AndDistribute (bool_tree_node * tree)
+{
+  if (!tree)
+    return 0;
+
+  if (BOOL_TAG (tree) != N_and)
+    {
+      /* try the children if got some */
+      if (BOOL_TAG (tree) == N_or)
+	{
+	  bool_tree_node *left_child = BOOL_CHILD (tree);
+	  int left = AndDistribute (left_child);
+	  int right = AndDistribute (BOOL_SIBLING (left_child));
+	  return (left || right);
+	}
+
+      /* no children, then no change */
+      return 0;
+    }
+
+  /* AND case */
+  {
+    /* we have to check the children to see if can distribute */
+    bool_tree_node *left_child = BOOL_CHILD (tree);
+    bool_tree_node *right_child = BOOL_SIBLING (left_child);
+    int swap = 0;
+
+    /* If "or" on the right side then do a swap */
+    /* This is done for code simplicity */
+    if (BOOL_TAG (right_child) == N_or)
+      {
+	/* do a swap */
+	bool_tree_node *temp;
+	temp = right_child;
+	right_child = left_child;
+	left_child = temp;
+
+	BOOL_SIBLING (left_child) = right_child;
+	BOOL_SIBLING (right_child) = NULL;
+	BOOL_CHILD (tree) = left_child;
+
+	swap = 1;
+      }
+
+    /* do the actual distribution */
+    if (swap || BOOL_TAG (left_child) == N_or)
+      {
+	bool_tree_node *A = right_child;
+	bool_tree_node *A_copy = CopyBoolTree (A);
+	bool_tree_node *a = BOOL_CHILD (left_child);
+	bool_tree_node *b = BOOL_SIBLING (a);
+
+	/* then can distribute away ... */
+	BOOL_TAG (tree) = N_or;
+	BOOL_TAG (left_child) = N_and;
+	BOOL_SIBLING (left_child) = CreateBoolTreeNode (N_and, b, A);
+	BOOL_SIBLING (a) = A_copy;
+
+	AndDistribute (tree);
+	return 1;
+      }
+
+    /* if "and" of something other than "or" s then try the children */
+    {
+      int left = AndDistribute (left_child);
+      int right = AndDistribute (right_child);
+      return (left || right);
+    }
+  }				/* AND case */
+
+}				/*AndDistribute */
+
+/* =========================================================================
+ * Function: CollapseTree
+ * Description: 
+ *      Collapse "and" and "or"s.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+CollapseTree (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if ((BOOL_TAG (tree) == N_and) || (BOOL_TAG (tree) == N_or))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  while (BOOL_TAG (child) == BOOL_TAG (tree))
+	    {
+	      ReplaceWithChildren (child);
+	    }
+	  CollapseTree (child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+  /* assume "not"s permeated */
+  return;
+}
+
+/* =========================================================================
+ * Function: ReplaceWithChildren
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+ReplaceWithChildren (bool_tree_node * tree)
+{
+  bool_tree_node *next = BOOL_SIBLING (tree);
+  bool_tree_node *child = BOOL_CHILD (tree);
+  bool_tree_node *last = LastBoolSibling (child);
+
+  bcopy ((char *) child, (char *) tree, sizeof (bool_tree_node));
+  Xfree (child);
+  BOOL_SIBLING (last) = next;
+
+}
+
+/* =========================================================================
+ * Function: TF_Idempotent
+ * Description: 
+ *      Applies Idempotency laws which refer to True or False (hence "TF")
+ *      uses the true or false idempotent rules
+ * Input: 
+ *      operates on n-ary trees (not restricted to binary)
+ * Output: 
+ * ========================================================================= */
+
+static void
+TF_Idempotent (bool_tree_node * tree)
+{
+  bool_tree_node *child = NULL;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	{
+	  child = BOOL_CHILD (tree);
+	  TF_Idempotent (child);
+	}
+
+      /* --- process node --- */
+      switch (BOOL_TAG (tree))
+	{
+	case N_not:
+	  child = BOOL_CHILD (tree);
+
+	  /* if true then falsify */
+	  if (BOOL_TAG (child) == N_all)
+	    {
+	      BOOL_TAG (tree) = N_none;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  /* if false then trueify */
+	  if (BOOL_TAG (child) == N_none)
+	    {
+	      BOOL_TAG (tree) = N_all;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  break;
+	case N_and:
+	  AndTrueFalse (tree);
+	  break;
+	case N_or:
+	  OrTrueFalse (tree);
+	  break;
+
+	default:
+	  /* do nothing */
+	  break;
+	}			/*switch */
+
+      tree = BOOL_SIBLING (tree);
+    }				/*while */
+}				/*TF_Idempotent */
+
+/* =========================================================================
+ * Function: AndTrueFalse
+ * Description: 
+ *      If any "false" then falsify whole tree 
+ *      If any "true" then delete them
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndTrueFalse (bool_tree_node * and_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_none)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (and_tree)));
+	  BOOL_TAG (and_tree) = N_none;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_all)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+
+	  /* --- link over child --- */
+	  if (!prev)
+	    {
+	      BOOL_CHILD (and_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+
+	  Xfree (child);
+	  child = sibling;
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (and_tree);
+
+  /* if no children ... */
+  /* => all have been true */
+  if (!child)
+    {
+      BOOL_TAG (and_tree) = N_all;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (and_tree);
+      bcopy ((char *) child, (char *) and_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (and_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: OrTrueFalse
+ * Description: 
+ *      If any "true" then trueify whole tree 
+ *      If any "false" then delete them 
+ *      ***** NOTE: This is a reflection of AndTrueFalse *****
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+OrTrueFalse (bool_tree_node * or_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (or_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_all)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (or_tree)));
+	  BOOL_TAG (or_tree) = N_all;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_none)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+	  if (!prev)
+	    {
+	      BOOL_CHILD (or_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+	  Xfree (child);
+	  child = sibling;
+
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (or_tree);
+
+  /* if no children ... */
+  /* => all have been false */
+  if (!child)
+    {
+      BOOL_TAG (or_tree) = N_none;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (or_tree);
+      bcopy ((char *) child, (char *) or_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (or_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: AndNotToDiff
+ * Description: 
+ *      Convert an "And" tree with possibly some negated terms into
+ *      a "And - neg-terms". 
+ * Input: 
+ *      Expects "And" of only literals - as tree should be in DNF form.
+ *      This means that no recursion is necessary.
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func)
+{
+
+  /* -------------------------------------------------- */
+  /* --- Divide the children up into positive and negative literals --- */
+  /* Preserves ordering of children */
+  bool_tree_node *pos = NULL;
+  bool_tree_node *neg = NULL;
+  bool_tree_node *last_pos = NULL;
+  bool_tree_node *last_neg = NULL;
+
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+  while (child)
+    {
+      bool_tree_node *next = BOOL_SIBLING (child);
+      if (BOOL_TAG (child) == N_not)
+	{
+	  /* add to end of neg list */
+	  if (!neg)
+	    neg = child;
+	  else
+	    BOOL_SIBLING (last_neg) = child;
+	  last_neg = child;
+	}
+      else
+	{
+	  /* add to end of pos list */
+	  if (!pos)
+	    pos = child;
+	  else
+	    BOOL_SIBLING (last_pos) = child;
+	  last_pos = child;
+	}
+      child = next;
+    }				/*while */
+  if (last_pos)
+    BOOL_SIBLING (last_pos) = NULL;
+  if (last_neg)
+    BOOL_SIBLING (last_neg) = NULL;
+  /* -------------------------------------------------- */
+
+  if (!neg)			/* no negtives => no subtraction */
+    {
+      /* fix up parent's child link */
+      BOOL_CHILD (and_tree) = pos;
+      return;
+    }
+
+  if (!pos)			/* no positives = all negatives */
+    {
+      /* Choose one of the negs for the left-hand-side of the subtraction */
+      /* In particular, choose the first */
+      pos = neg;
+      neg = BOOL_SIBLING (neg);
+      BOOL_SIBLING (pos) = NULL;
+    }
+
+  /* Get rid of all '!' from not nodes */
+  /* Make a list out of the children */
+  {
+    bool_tree_node *n = neg;
+    bool_tree_node *next = NULL;
+
+    neg = BOOL_CHILD (neg);
+    while (n)
+      {
+	bool_tree_node *c = BOOL_CHILD (n);
+	next = BOOL_SIBLING (n);
+	Xfree (n);
+	if (next)
+	  BOOL_SIBLING (c) = BOOL_CHILD (next);
+	n = next;
+      }
+  }
+
+  /* --- restructure the tree with "and" and "diff" --- */
+  BOOL_TAG (and_tree) = N_diff;
+  if (!BOOL_SIBLING (pos))	/* i.e. only one pos */
+    {
+      /* (A & !B & !C & !D & ...) ==> A - (B,C,D,...) */
+
+      BOOL_CHILD (and_tree) = pos;
+      BOOL_SIBLING (pos) = neg;
+    }
+  else
+    {
+      /* (A & !C & B & !D & ...) ==> (A & B & ....) - (C, D, ...) */
+
+      bool_tree_node *n = CreateBoolNode (N_and);
+      BOOL_CHILD (and_tree) = n;
+      BOOL_CHILD (n) = pos;
+      BOOL_SIBLING (n) = neg;
+    }				/*if */
+
+  if (recurs_func)
+    {
+      recurs_func (BOOL_CHILD (and_tree));
+    }
+
+}				/*AndNotToDiff */
+
+/* =========================================================================
+ * Function: DiffTree
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ *      Unlike DiffDNF, this routine will search throughout the tree for
+ *      "and"-nodes. It will call AndNotToNeg with this as its recursive 
+ *      routine.
+ * Date: 27/July/95
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+DiffTree (bool_tree_node * tree)
+{
+  bool_tree_node *t = tree;
+
+  while (t)
+    {
+      if (BOOL_TAG (t) == N_and)
+	{
+	  AndNotToDiff (t, DiffTree);
+	}
+      t = BOOL_SIBLING (t);
+    }				/*while */
+
+}				/*DiffTree */
+
+/* =========================================================================
+ * Function: DiffDNF
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ * Input: 
+ *      A DNF tree
+ * Output: 
+ *      A tree possibly with some diffs in it.
+ * ========================================================================= */
+
+static void
+DiffDNF (bool_tree_node * DNF_tree)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndNotToDiff (child, NULL);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndNotToDiff (DNF_tree, NULL);
+    }
+}
+
+/* =========================================================================
+ * Function: AndSort_comp 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+/* Module variable, "term_list",  used as parameter for comparison function */
+/* The comparison function only allows certain fixed parameters */
+static TermList *term_list;
+
+static int
+AndSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) != N_term)
+    return -1;
+
+  if (BOOL_TAG (*a) != N_term && BOOL_TAG (*b) == N_term)
+    return 1;
+
+  return 0;
+}				/*AndSort_comp */
+
+/* =========================================================================
+ * Function: AndSort
+ * Description: 
+ *      Sort the list of nodes by increasing doc_count 
+ *      Using some Neil Sharman code - pretty straight forward.
+ *      Note: not-terms are sent to the end of the list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp)
+{
+  bool_tree_node **list = NULL;
+  bool_tree_node *child = NULL;
+  int count = 0;
+  int i = 0;
+
+  if (BOOL_TAG (and_tree) != N_and)
+    return;
+
+  /* --- Convert linked list into an array for sorting --- */
+  child = BOOL_CHILD (and_tree);
+  count = 0;
+  while (child)
+    {
+      count++;
+      child = BOOL_SIBLING (child);
+    }
+
+  if (!(list = Xmalloc (sizeof (bool_tree_node *) * count)))
+    return;
+
+  child = BOOL_CHILD (and_tree);
+  for (i = 0; i < count; i++)
+    {
+      list[i] = child;
+      child = BOOL_SIBLING (child);
+    }
+  /* ----------------------------------------------------- */
+
+  term_list = tl;
+  qsort (list, count, sizeof (bool_tree_node *), comp);
+
+  /* --- Convert array back into linked list --- */
+  for (i = 0; i < count - 1; i++)
+    BOOL_SIBLING (list[i]) = list[i + 1];
+  BOOL_SIBLING (list[count - 1]) = NULL;
+  BOOL_CHILD (and_tree) = list[0];
+  Xfree (list);
+  /* ----------------------------------------------------- */
+
+}				/*AndSort */
+
+/* =========================================================================
+ * Function: AndSortDNF
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+static void
+AndSortDNF (bool_tree_node * DNF_tree, TermList * tl)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndSort (child, tl, AndSort_comp);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndSort (DNF_tree, tl, AndSort_comp);
+    }
+}
+
+/* =========================================================================
+ * Function: MarkOrTerms
+ * Description: 
+ *      Mark or-nodes whose children are all terms.
+ *      Also store the sum of the ft's of these nodes.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+MarkOrTerms (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_TAG (tree) == N_or)
+    {
+      int found_non_term = 0;
+      bool_tree_node *child = BOOL_CHILD (tree);
+
+      /* --- check to see if got all terms --- */
+      while (child)
+	{
+	  if (BOOL_TAG (child) != N_term)
+	    {
+	      found_non_term = 1;
+	    }
+	  if (BOOL_HAS_CHILD (tree))
+	    {
+	      MarkOrTerms (child, tl);
+	    }
+	  child = BOOL_SIBLING (child);
+	}			/*while */
+
+      if (!found_non_term)
+	{
+	  /* Must have all terms for the OR node */
+	  /* So change its tag and calculate its sum */
+	  BOOL_TAG (tree) = N_or_terms;
+	  BOOL_SUM_FT (tree) = calc_sum_ft (tree, tl);
+	}			/*if */
+      /* ------------------------------------- */
+
+    }				/*if OR node */
+
+  else if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  MarkOrTerms (child, tl);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+}				/*MarkOrTerms */
+
+/* =========================================================================
+ * Function: calc_sum_ft
+ * Description: 
+ *      Sums up the ft's of all the terms 
+ * Input: 
+ *      Assumes that the tree only contains terms in it 
+ * Output: 
+ *      the ft sum
+ * ========================================================================= */
+static int
+calc_sum_ft (bool_tree_node * term_tree, TermList * tl)
+{
+  int sum_ft = 0;
+  bool_tree_node *child = BOOL_CHILD (term_tree);
+  while (child)
+    {
+      sum_ft += GetNthWE (tl, BOOL_TERM (child))->doc_count;
+      child = BOOL_SIBLING (child);
+    }
+  return sum_ft;
+}
+
+/* =========================================================================
+ * Function: AndOrSort
+ * Description: 
+ *      This is like an AndSort of just terms. 
+ *      The difference is that we are now also 
+ *      considering sorting with <or of terms> by using the sum of f_t's.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+AndOrSort (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+
+      if (BOOL_TAG (tree) == N_and)
+	{
+	  AndSort (tree, tl, AndOrSort_comp);
+	}
+
+      /* Check out any children down the tree */
+      /* Probably a waste of time */
+      {
+	bool_tree_node *child = BOOL_CHILD (tree);
+	while (child)
+	  {
+	    AndOrSort (child, tl);
+	    child = BOOL_SIBLING (child);
+	  }
+      }				/*do the tree */
+
+    }				/*if has children */
+}				/*AndOrSort */
+
+/* =========================================================================
+ * Function: AndOrSort_comp
+ * Description: 
+ *      Comparison function used to order "term"s and "or_terms"s.
+ *      It's purpose is to be used by a sort routine.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int
+AndOrSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if ((BOOL_TAG (*a) == N_term || BOOL_TAG (*a) == N_or_terms) &&
+      (BOOL_TAG (*b) != N_term && BOOL_TAG (*b) != N_or_terms))
+    return -1;
+
+  if ((BOOL_TAG (*a) != N_term && BOOL_TAG (*a) != N_or_terms) &&
+      (BOOL_TAG (*b) == N_term || BOOL_TAG (*b) == N_or_terms))
+    return 1;
+
+  return 0;
+}				/*AndOrSort_comp */
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_optimiser.h	(revision 21112)
@@ -0,0 +1,62 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:23  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:21  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.1  1994/10/20  03:56:32  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/10/18  06:11:03  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:29  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_OPTIMISER_H
+#define BOOL_OPTIMISER_H
+
+#include "bool_tree.h"
+#include "term_lists.h"
+
+void OptimiseBoolTree (bool_tree_node * parse_tree, TermList * term_list, int type);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.c	(revision 21112)
@@ -0,0 +1,1196 @@
+
+/*  A Bison parser, made from bool_parser.y
+ by  GNU Bison version 1.27
+  */
+
+#define YYBISON 1  /* Identify Bison output.  */
+
+#define	TERM	257
+
+#line 24 "bool_parser.y"
+
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+
+#line 66 "bool_parser.y"
+typedef union {
+  char *text;
+  bool_tree_node *node;
+} YYSTYPE;
+#include <stdio.h>
+
+#ifndef __cplusplus
+#ifndef __STDC__
+#define const
+#endif
+#endif
+
+
+
+#define	YYFINAL		20
+#define	YYFLAG		-32768
+#define	YYNTBASE	11
+
+#define YYTRANSLATE(x) ((unsigned)(x) <= 257 ? yytranslate[x] : 16)
+
+static const char yytranslate[] = {     0,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     8,     2,     2,     2,     2,     9,     2,     4,
+     5,     6,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     7,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,    10,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     1,     3
+};
+
+#if YYDEBUG != 0
+static const short yyprhs[] = {     0,
+     0,     2,     4,     8,    10,    12,    14,    17,    21,    24,
+    26,    30
+};
+
+static const short yyrhs[] = {    15,
+     0,     3,     0,     4,    15,     5,     0,     6,     0,     7,
+     0,    12,     0,     8,    13,     0,    14,     9,    13,     0,
+    14,    13,     0,    13,     0,    15,    10,    14,     0,    14,
+     0
+};
+
+#endif
+
+#if YYDEBUG != 0
+static const short yyrline[] = { 0,
+    76,    80,    81,    82,    83,    86,    87,    90,    91,    92,
+    95,    96
+};
+#endif
+
+
+#if YYDEBUG != 0 || defined (YYERROR_VERBOSE)
+
+static const char * const yytname[] = {   "$","error","$undefined.","TERM","'('",
+"')'","'*'","'_'","'!'","'&'","'|'","query","term","not","and","or", NULL
+};
+#endif
+
+static const short yyr1[] = {     0,
+    11,    12,    12,    12,    12,    13,    13,    14,    14,    14,
+    15,    15
+};
+
+static const short yyr2[] = {     0,
+     1,     1,     3,     1,     1,     1,     2,     3,     2,     1,
+     3,     1
+};
+
+static const short yydefact[] = {     0,
+     2,     0,     4,     5,     0,     6,    10,    12,     1,     0,
+     7,     0,     9,     0,     3,     8,    11,     0,     0,     0
+};
+
+static const short yydefgoto[] = {    18,
+     6,     7,     8,     9
+};
+
+static const short yypact[] = {    10,
+-32768,    10,-32768,-32768,    10,-32768,-32768,     2,    -9,    14,
+-32768,    10,-32768,    10,-32768,-32768,     2,     4,    15,-32768
+};
+
+static const short yypgoto[] = {-32768,
+-32768,    -5,   -12,    18
+};
+
+
+#define	YYLAST		24
+
+
+static const short yytable[] = {    11,
+    14,    17,    13,    19,     1,     2,    16,     3,     4,     5,
+    12,    13,     1,     2,    20,     3,     4,     5,    15,    10,
+     0,     0,     0,    14
+};
+
+static const short yycheck[] = {     5,
+    10,    14,     8,     0,     3,     4,    12,     6,     7,     8,
+     9,    17,     3,     4,     0,     6,     7,     8,     5,     2,
+    -1,    -1,    -1,    10
+};
+/* -*-C-*-  Note some compilers choke on comments on `#line' lines.  */
+#line 3 "/usr/lib/bison.simple"
+/* This file comes from bison-1.27.  */
+
+/* Skeleton output parser for bison,
+   Copyright (C) 1984, 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place - Suite 330,
+   Boston, MA 02111-1307, USA.  */
+
+/* As a special exception, when this file is copied by Bison into a
+   Bison output file, you may use that output file without restriction.
+   This special exception was added by the Free Software Foundation
+   in version 1.24 of Bison.  */
+
+/* This is the parser code that is written into each bison parser
+  when the %semantic_parser declaration is not specified in the grammar.
+  It was written by Richard Stallman by simplifying the hairy parser
+  used when %semantic_parser is specified.  */
+
+#ifndef YYSTACK_USE_ALLOCA
+#ifdef alloca
+#define YYSTACK_USE_ALLOCA
+#else /* alloca not defined */
+#ifdef __GNUC__
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#else /* not GNU C.  */
+#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386))
+#define YYSTACK_USE_ALLOCA
+#include <alloca.h>
+#else /* not sparc */
+/* We think this test detects Watcom and Microsoft C.  */
+/* This used to test MSDOS, but that is a bad idea
+   since that symbol is in the user namespace.  */
+#if (defined (_MSDOS) || defined (_MSDOS_)) && !defined (__TURBOC__)
+#if 0 /* No need for malloc.h, which pollutes the namespace;
+	 instead, just don't use alloca.  */
+#include <malloc.h>
+#endif
+#else /* not MSDOS, or __TURBOC__ */
+#if defined(_AIX)
+/* I don't know what this was needed for, but it pollutes the namespace.
+   So I turned it off.   rms, 2 May 1997.  */
+/* #include <malloc.h>  */
+ #pragma alloca
+#define YYSTACK_USE_ALLOCA
+#else /* not MSDOS, or __TURBOC__, or _AIX */
+#if 0
+#ifdef __hpux /* haible@ilog.fr says this works for HPUX 9.05 and up,
+		 and on HPUX 10.  Eventually we can turn this on.  */
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#endif /* __hpux */
+#endif
+#endif /* not _AIX */
+#endif /* not MSDOS, or __TURBOC__ */
+#endif /* not sparc */
+#endif /* not GNU C */
+#endif /* alloca not defined */
+#endif /* YYSTACK_USE_ALLOCA not defined */
+
+#ifdef YYSTACK_USE_ALLOCA
+#define YYSTACK_ALLOC alloca
+#else
+#define YYSTACK_ALLOC malloc
+#endif
+
+/* Note: there must be only one dollar sign in this file.
+   It is replaced by the list of actions, each action
+   as one case of the switch.  */
+
+#define yyerrok		(yyerrstatus = 0)
+#define yyclearin	(yychar = YYEMPTY)
+#define YYEMPTY		-2
+#define YYEOF		0
+#define YYACCEPT	goto yyacceptlab
+#define YYABORT 	goto yyabortlab
+#define YYERROR		goto yyerrlab1
+/* Like YYERROR except do call yyerror.
+   This remains here temporarily to ease the
+   transition to the new meaning of YYERROR, for GCC.
+   Once GCC version 2 has supplanted version 1, this can go.  */
+#define YYFAIL		goto yyerrlab
+#define YYRECOVERING()  (!!yyerrstatus)
+#define YYBACKUP(token, value) \
+do								\
+  if (yychar == YYEMPTY && yylen == 1)				\
+    { yychar = (token), yylval = (value);			\
+      yychar1 = YYTRANSLATE (yychar);				\
+      YYPOPSTACK;						\
+      goto yybackup;						\
+    }								\
+  else								\
+    { yyerror ("syntax error: cannot back up"); YYERROR; }	\
+while (0)
+
+#define YYTERROR	1
+#define YYERRCODE	256
+
+#ifndef YYPURE
+#define YYLEX		yylex()
+#endif
+
+#ifdef YYPURE
+#ifdef YYLSP_NEEDED
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, &yylloc, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval, &yylloc)
+#endif
+#else /* not YYLSP_NEEDED */
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval)
+#endif
+#endif /* not YYLSP_NEEDED */
+#endif
+
+/* If nonreentrant, generate the variables here */
+
+#ifndef YYPURE
+
+int	yychar;			/*  the lookahead symbol		*/
+YYSTYPE	yylval;			/*  the semantic value of the		*/
+				/*  lookahead symbol			*/
+
+#ifdef YYLSP_NEEDED
+YYLTYPE yylloc;			/*  location data for the lookahead	*/
+				/*  symbol				*/
+#endif
+
+int yynerrs;			/*  number of parse errors so far       */
+#endif  /* not YYPURE */
+
+#if YYDEBUG != 0
+int yydebug;			/*  nonzero means print parse trace	*/
+/* Since this is uninitialized, it does not stop multiple parsers
+   from coexisting.  */
+#endif
+
+/*  YYINITDEPTH indicates the initial size of the parser's stacks	*/
+
+#ifndef	YYINITDEPTH
+#define YYINITDEPTH 200
+#endif
+
+/*  YYMAXDEPTH is the maximum size the stacks can grow to
+    (effective only if the built-in stack extension method is used).  */
+
+#if YYMAXDEPTH == 0
+#undef YYMAXDEPTH
+#endif
+
+#ifndef YYMAXDEPTH
+#define YYMAXDEPTH 10000
+#endif
+
+
+/* Define __yy_memcpy.  Note that the size argument
+   should be passed with type unsigned int, because that is what the non-GCC
+   definitions require.  With GCC, __builtin_memcpy takes an arg
+   of type size_t, but it can handle unsigned int.  */
+
+#if __GNUC__ > 1		/* GNU C and GNU C++ define this.  */
+#define __yy_memcpy(TO,FROM,COUNT)	__builtin_memcpy(TO,FROM,COUNT)
+#else				/* not GNU C or C++ */
+#ifndef __cplusplus
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (to, from, count)
+     char *to;
+     char *from;
+     unsigned int count;
+{
+  register char *f = from;
+  register char *t = to;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#else /* __cplusplus */
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (char *to, char *from, unsigned int count)
+{
+  register char *t = to;
+  register char *f = from;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#endif
+#endif
+
+
+#line 216 "/usr/lib/bison.simple"
+
+/* The user can define YYPARSE_PARAM as the name of an argument to be passed
+   into yyparse.  The argument should have type void *.
+   It should actually point to an object.
+   Grammar actions can access the variable by casting it
+   to the proper pointer type.  */
+
+#ifdef YYPARSE_PARAM
+#ifdef __cplusplus
+#define YYPARSE_PARAM_ARG void *YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL
+#else /* not __cplusplus */
+#define YYPARSE_PARAM_ARG YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL void *YYPARSE_PARAM;
+#endif /* not __cplusplus */
+#else /* not YYPARSE_PARAM */
+#define YYPARSE_PARAM_ARG
+#define YYPARSE_PARAM_DECL
+#endif /* not YYPARSE_PARAM */
+
+/* Prevent warning if -Wstrict-prototypes.  */
+#ifdef __GNUC__
+#ifdef YYPARSE_PARAM
+int yyparse (void *);
+#else
+int yyparse (void);
+#endif
+#endif
+
+int
+yyparse(YYPARSE_PARAM_ARG)
+     YYPARSE_PARAM_DECL
+{
+  register int yystate;
+  register int yyn;
+  register short *yyssp;
+  register YYSTYPE *yyvsp;
+  int yyerrstatus;	/*  number of tokens to shift before error messages enabled */
+  int yychar1 = 0;		/*  lookahead token as an internal (translated) token number */
+
+  short	yyssa[YYINITDEPTH];	/*  the state stack			*/
+  YYSTYPE yyvsa[YYINITDEPTH];	/*  the semantic value stack		*/
+
+  short *yyss = yyssa;		/*  refer to the stacks thru separate pointers */
+  YYSTYPE *yyvs = yyvsa;	/*  to allow yyoverflow to reallocate them elsewhere */
+
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylsa[YYINITDEPTH];	/*  the location stack			*/
+  YYLTYPE *yyls = yylsa;
+  YYLTYPE *yylsp;
+
+#define YYPOPSTACK   (yyvsp--, yyssp--, yylsp--)
+#else
+#define YYPOPSTACK   (yyvsp--, yyssp--)
+#endif
+
+  int yystacksize = YYINITDEPTH;
+  int yyfree_stacks = 0;
+
+#ifdef YYPURE
+  int yychar;
+  YYSTYPE yylval;
+  int yynerrs;
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylloc;
+#endif
+#endif
+
+  YYSTYPE yyval;		/*  the variable used to return		*/
+				/*  semantic values from the action	*/
+				/*  routines				*/
+
+  int yylen;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Starting parse\n");
+#endif
+
+  yystate = 0;
+  yyerrstatus = 0;
+  yynerrs = 0;
+  yychar = YYEMPTY;		/* Cause a token to be read.  */
+
+  /* Initialize stack pointers.
+     Waste one element of value and location stack
+     so that they stay on the same level as the state stack.
+     The wasted elements are never initialized.  */
+
+  yyssp = yyss - 1;
+  yyvsp = yyvs;
+#ifdef YYLSP_NEEDED
+  yylsp = yyls;
+#endif
+
+/* Push a new state, which is found in  yystate  .  */
+/* In all cases, when you get here, the value and location stacks
+   have just been pushed. so pushing a state here evens the stacks.  */
+yynewstate:
+
+  *++yyssp = yystate;
+
+  if (yyssp >= yyss + yystacksize - 1)
+    {
+      /* Give user a chance to reallocate the stack */
+      /* Use copies of these so that the &'s don't force the real ones into memory. */
+      YYSTYPE *yyvs1 = yyvs;
+      short *yyss1 = yyss;
+#ifdef YYLSP_NEEDED
+      YYLTYPE *yyls1 = yyls;
+#endif
+
+      /* Get the current used size of the three stacks, in elements.  */
+      int size = yyssp - yyss + 1;
+
+#ifdef yyoverflow
+      /* Each stack pointer address is followed by the size of
+	 the data in use in that stack, in bytes.  */
+#ifdef YYLSP_NEEDED
+      /* This used to be a conditional around just the two extra args,
+	 but that might be undefined if yyoverflow is a macro.  */
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yyls1, size * sizeof (*yylsp),
+		 &yystacksize);
+#else
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yystacksize);
+#endif
+
+      yyss = yyss1; yyvs = yyvs1;
+#ifdef YYLSP_NEEDED
+      yyls = yyls1;
+#endif
+#else /* no yyoverflow */
+      /* Extend the stack our own way.  */
+      if (yystacksize >= YYMAXDEPTH)
+	{
+	  yyerror("parser stack overflow");
+	  if (yyfree_stacks)
+	    {
+	      free (yyss);
+	      free (yyvs);
+#ifdef YYLSP_NEEDED
+	      free (yyls);
+#endif
+	    }
+	  return 2;
+	}
+      yystacksize *= 2;
+      if (yystacksize > YYMAXDEPTH)
+	yystacksize = YYMAXDEPTH;
+#ifndef YYSTACK_USE_ALLOCA
+      yyfree_stacks = 1;
+#endif
+      yyss = (short *) YYSTACK_ALLOC (yystacksize * sizeof (*yyssp));
+      __yy_memcpy ((char *)yyss, (char *)yyss1,
+		   size * (unsigned int) sizeof (*yyssp));
+      yyvs = (YYSTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yyvsp));
+      __yy_memcpy ((char *)yyvs, (char *)yyvs1,
+		   size * (unsigned int) sizeof (*yyvsp));
+#ifdef YYLSP_NEEDED
+      yyls = (YYLTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yylsp));
+      __yy_memcpy ((char *)yyls, (char *)yyls1,
+		   size * (unsigned int) sizeof (*yylsp));
+#endif
+#endif /* no yyoverflow */
+
+      yyssp = yyss + size - 1;
+      yyvsp = yyvs + size - 1;
+#ifdef YYLSP_NEEDED
+      yylsp = yyls + size - 1;
+#endif
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Stack size increased to %d\n", yystacksize);
+#endif
+
+      if (yyssp >= yyss + yystacksize - 1)
+	YYABORT;
+    }
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Entering state %d\n", yystate);
+#endif
+
+  goto yybackup;
+ yybackup:
+
+/* Do appropriate processing given the current state.  */
+/* Read a lookahead token if we need one and don't already have one.  */
+/* yyresume: */
+
+  /* First try to decide what to do without reference to lookahead token.  */
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yydefault;
+
+  /* Not known => get a lookahead token if don't already have one.  */
+
+  /* yychar is either YYEMPTY or YYEOF
+     or a valid token in external form.  */
+
+  if (yychar == YYEMPTY)
+    {
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Reading a token: ");
+#endif
+      yychar = YYLEX;
+    }
+
+  /* Convert token to internal form (in yychar1) for indexing tables with */
+
+  if (yychar <= 0)		/* This means end of input. */
+    {
+      yychar1 = 0;
+      yychar = YYEOF;		/* Don't call YYLEX any more */
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Now at end of input.\n");
+#endif
+    }
+  else
+    {
+      yychar1 = YYTRANSLATE(yychar);
+
+#if YYDEBUG != 0
+      if (yydebug)
+	{
+	  fprintf (stderr, "Next token is %d (%s", yychar, yytname[yychar1]);
+	  /* Give the individual parser a way to print the precise meaning
+	     of a token, for further debugging info.  */
+#ifdef YYPRINT
+	  YYPRINT (stderr, yychar, yylval);
+#endif
+	  fprintf (stderr, ")\n");
+	}
+#endif
+    }
+
+  yyn += yychar1;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != yychar1)
+    goto yydefault;
+
+  yyn = yytable[yyn];
+
+  /* yyn is what to do for this token type in this state.
+     Negative => reduce, -yyn is rule number.
+     Positive => shift, yyn is new state.
+       New state is final state => don't bother to shift,
+       just return success.
+     0, or most negative number => error.  */
+
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrlab;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrlab;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+  /* Shift the lookahead token.  */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting token %d (%s), ", yychar, yytname[yychar1]);
+#endif
+
+  /* Discard the token being shifted unless it is eof.  */
+  if (yychar != YYEOF)
+    yychar = YYEMPTY;
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  /* count tokens shifted since error; after three, turn off error status.  */
+  if (yyerrstatus) yyerrstatus--;
+
+  yystate = yyn;
+  goto yynewstate;
+
+/* Do the default action for the current state.  */
+yydefault:
+
+  yyn = yydefact[yystate];
+  if (yyn == 0)
+    goto yyerrlab;
+
+/* Do a reduction.  yyn is the number of a rule to reduce with.  */
+yyreduce:
+  yylen = yyr2[yyn];
+  if (yylen > 0)
+    yyval = yyvsp[1-yylen]; /* implement default value of the action */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      int i;
+
+      fprintf (stderr, "Reducing via rule %d (line %d), ",
+	       yyn, yyrline[yyn]);
+
+      /* Print the symbols being reduced, and their result.  */
+      for (i = yyprhs[yyn]; yyrhs[i] > 0; i++)
+	fprintf (stderr, "%s ", yytname[yyrhs[i]]);
+      fprintf (stderr, " -> %s\n", yytname[yyr1[yyn]]);
+    }
+#endif
+
+
+  switch (yyn) {
+
+case 1:
+#line 76 "bool_parser.y"
+{ tree_base = yyvsp[0].node;;
+    break;}
+case 2:
+#line 80 "bool_parser.y"
+{ yyval.node = CreateBoolTermNode(term_list, yyvsp[0].text, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); ;
+    break;}
+case 3:
+#line 81 "bool_parser.y"
+{ yyval.node = yyvsp[-1].node; ;
+    break;}
+case 4:
+#line 82 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_all, NULL, NULL); ;
+    break;}
+case 5:
+#line 83 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_none, NULL, NULL); ;
+    break;}
+case 7:
+#line 87 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_not, yyvsp[0].node, NULL); ;
+    break;}
+case 8:
+#line 90 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+case 9:
+#line 91 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-1].node, yyvsp[0].node); ;
+    break;}
+case 11:
+#line 95 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_or, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+}
+   /* the action file gets copied in in place of this dollarsign */
+#line 542 "/usr/lib/bison.simple"
+
+
+  yyvsp -= yylen;
+  yyssp -= yylen;
+#ifdef YYLSP_NEEDED
+  yylsp -= yylen;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+  *++yyvsp = yyval;
+
+#ifdef YYLSP_NEEDED
+  yylsp++;
+  if (yylen == 0)
+    {
+      yylsp->first_line = yylloc.first_line;
+      yylsp->first_column = yylloc.first_column;
+      yylsp->last_line = (yylsp-1)->last_line;
+      yylsp->last_column = (yylsp-1)->last_column;
+      yylsp->text = 0;
+    }
+  else
+    {
+      yylsp->last_line = (yylsp+yylen-1)->last_line;
+      yylsp->last_column = (yylsp+yylen-1)->last_column;
+    }
+#endif
+
+  /* Now "shift" the result of the reduction.
+     Determine what state that goes to,
+     based on the state we popped back to
+     and the rule number reduced by.  */
+
+  yyn = yyr1[yyn];
+
+  yystate = yypgoto[yyn - YYNTBASE] + *yyssp;
+  if (yystate >= 0 && yystate <= YYLAST && yycheck[yystate] == *yyssp)
+    yystate = yytable[yystate];
+  else
+    yystate = yydefgoto[yyn - YYNTBASE];
+
+  goto yynewstate;
+
+yyerrlab:   /* here on detecting error */
+
+  if (! yyerrstatus)
+    /* If not already recovering from an error, report this error.  */
+    {
+      ++yynerrs;
+
+#ifdef YYERROR_VERBOSE
+      yyn = yypact[yystate];
+
+      if (yyn > YYFLAG && yyn < YYLAST)
+	{
+	  int size = 0;
+	  char *msg;
+	  int x, count;
+
+	  count = 0;
+	  /* Start X at -yyn if nec to avoid negative indexes in yycheck.  */
+	  for (x = (yyn < 0 ? -yyn : 0);
+	       x < (sizeof(yytname) / sizeof(char *)); x++)
+	    if (yycheck[x + yyn] == x)
+	      size += strlen(yytname[x]) + 15, count++;
+	  msg = (char *) malloc(size + 15);
+	  if (msg != 0)
+	    {
+	      strcpy(msg, "parse error");
+
+	      if (count < 5)
+		{
+		  count = 0;
+		  for (x = (yyn < 0 ? -yyn : 0);
+		       x < (sizeof(yytname) / sizeof(char *)); x++)
+		    if (yycheck[x + yyn] == x)
+		      {
+			strcat(msg, count == 0 ? ", expecting `" : " or `");
+			strcat(msg, yytname[x]);
+			strcat(msg, "'");
+			count++;
+		      }
+		}
+	      yyerror(msg);
+	      free(msg);
+	    }
+	  else
+	    yyerror ("parse error; also virtual memory exceeded");
+	}
+      else
+#endif /* YYERROR_VERBOSE */
+	yyerror("parse error");
+    }
+
+  goto yyerrlab1;
+yyerrlab1:   /* here on error raised explicitly by an action */
+
+  if (yyerrstatus == 3)
+    {
+      /* if just tried and failed to reuse lookahead token after an error, discard it.  */
+
+      /* return failure if at end of input */
+      if (yychar == YYEOF)
+	YYABORT;
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Discarding token %d (%s).\n", yychar, yytname[yychar1]);
+#endif
+
+      yychar = YYEMPTY;
+    }
+
+  /* Else will try to reuse lookahead token
+     after shifting the error token.  */
+
+  yyerrstatus = 3;		/* Each real token shifted decrements this */
+
+  goto yyerrhandle;
+
+yyerrdefault:  /* current state does not do anything special for the error token. */
+
+#if 0
+  /* This is wrong; only states that explicitly want error tokens
+     should shift them.  */
+  yyn = yydefact[yystate];  /* If its default is to accept any token, ok.  Otherwise pop it.*/
+  if (yyn) goto yydefault;
+#endif
+
+yyerrpop:   /* pop the current state because it cannot handle the error token */
+
+  if (yyssp == yyss) YYABORT;
+  yyvsp--;
+  yystate = *--yyssp;
+#ifdef YYLSP_NEEDED
+  yylsp--;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "Error: state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+yyerrhandle:
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yyerrdefault;
+
+  yyn += YYTERROR;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != YYTERROR)
+    goto yyerrdefault;
+
+  yyn = yytable[yyn];
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrpop;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrpop;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting error token, ");
+#endif
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  yystate = yyn;
+  goto yynewstate;
+
+ yyacceptlab:
+  /* YYACCEPT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 0;
+
+ yyabortlab:
+  /* YYABORT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 1;
+}
+#line 99 "bool_parser.y"
+
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:43  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:40  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:26  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:33  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:31  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_PARSER_H
+#define BOOL_PARSER_H
+
+#include "backend.h"
+#include "term_lists.h"
+#include "invf.h"
+
+bool_tree_node *ParseBool (char *query_line, int query_len,
+			   TermList ** the_term_list, int the_stemmer_num, 
+			   int the_stem_method, int *res,
+			   stemmed_dict * the_sd, int is_indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			   QueryTermList ** the_query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.y
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.y	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_parser.y	(revision 21112)
@@ -0,0 +1,316 @@
+/**************************************************************************
+ *
+ * bool_parser - boolean query parser
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+ 
+/**************************************************************************/
+%{
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+%}
+ 
+ 
+%union {
+  char *text;
+  bool_tree_node *node;
+}
+ 
+%token <text> TERM
+%type <node> query term not and or
+ 
+%%
+ 
+query: or  { tree_base = $1;}
+;
+ 
+ 
+term:     TERM  { $$ = CreateBoolTermNode(term_list, $1, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+        | '(' or ')' { $$ = $2; }
+        | '*' { $$ = CreateBoolTreeNode(N_all, NULL, NULL); }
+        | '_' { $$ = CreateBoolTreeNode(N_none, NULL, NULL); }
+;
+ 
+not:      term
+        | '!' not { $$ = CreateBoolTreeNode(N_not, $2, NULL); }
+;
+ 
+and:      and '&' not { $$ = CreateBoolTreeNode(N_and, $1, $3); }
+        | and not { $$ = CreateBoolTreeNode(N_and, $1, $2); }
+        | not
+;
+ 
+or:       or '|' and { $$ = CreateBoolTreeNode(N_or, $1, $3); }
+        | and
+;
+ 
+%%
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+      
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.c	(revision 21112)
@@ -0,0 +1,353 @@
+/**************************************************************************
+ *
+ * bool_query.c -- Boolean query evaluation
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "sptree.h"
+
+
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "backend.h"
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "read_line.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "environment.h"
+
+
+/**********************
+  [RPAP - Feb 97: Term Frequency]
+
+  Shifted to bool_parser.y
+
+  *********************/
+
+/* =========================================================================
+ * Function: ReadInTermInfo
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+/**************************
+static void 
+ReadInTermInfo (query_data * qd)
+{
+  int i;
+  TermList *tl = qd->TL;
+
+  for (i = 0; i < tl->num; i++)
+    {
+      TermEntry *te = &(tl->TE[i]);
+      WordEntry *we = &(te->WE);
+
+       we->word_num = FindWord (qd->sd, te->Word,
+			       &we->count, &we->doc_count,
+			       &we->invf_ptr, &we->invf_len);
+      if (we->word_num == -1)
+	we->count = we->doc_count = 0;
+    }
+}
+
+***********************/
+
+
+
+/* =========================================================================
+ * Function: FindNoneTerms 
+ * Description: 
+ *      Set any terms which are not in dictionary to N_none (i.e. false)
+ *      Assumes a prior call to ReadInTermInfo
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FindNoneTerms (query_data * qd, bool_tree_node * tree)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  FindNoneTerms (qd, child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (tree) == N_term)
+    {
+      int doc_count = qd->TL->TE[BOOL_TERM (tree)].WE.doc_count;
+      if (!doc_count)
+	BOOL_TAG (tree) = N_none;
+    }
+
+}				/*FindNoneTerms */
+
+/* =========================================================================
+ * Function: ReduceList
+ * Description: 
+ *      Go thru doc-set and get rid of all docs which are unmarked
+ *      The motivation for this routine is in the case of CNF queries.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ReduceList (DocList * cset)
+{
+  DocEntry *src;
+  DocEntry *dest;
+
+  src = dest = cset->DE;
+  for (src = dest = cset->DE; src < cset->DE + cset->num; src++)
+    {
+      if (src->or_included)
+	{
+	  src->or_included = 0;
+	  *dest++ = *src;
+	}
+    }
+  cset->num = dest - cset->DE;
+
+}				/*ReduceList */
+
+/* =========================================================================
+ * Function: BooleanGet
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static DocList *
+BooleanGet (query_data * qd, bool_tree_node * n,
+	    BooleanQueryInfo * bqi)
+{
+  if (n)
+    {
+      DocList *res = NULL;
+      switch (BOOL_TAG (n))
+	{
+	case N_all:
+	  Message ("Unexpected \"all\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_not:
+	  Message ("Unexpected \"not\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_none:
+	  {
+	    res = MakeDocList (0);
+	    break;
+	  }
+	case N_diff:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		if (BOOL_TAG (n) == N_term)
+		  {
+		    WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		    res = GetDocsOp (qd, we, op_diff_terms, res);
+		  }
+		else
+		  res = DiffLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_and:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);		/* initial fill of c-set */
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		switch (BOOL_TAG (n))
+		  {
+		  case N_term:
+		    {
+		      WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		      res = GetDocsOp (qd, we, op_and_terms, res);
+		    }
+		    break;
+		  case N_or_terms:
+		    {
+		      bool_tree_node *child = BOOL_CHILD (n);
+		      /* note: all children are terms - predetermined */
+		      while (child)
+			{
+			  WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (child));
+			  res = GetDocsOp (qd, we, op_and_or_terms, res);
+			  child = BOOL_SIBLING (child);
+			}
+		      ReduceList (res);
+		    }
+		    break;
+		  default:
+		    res = IntersectLists (res, BooleanGet (qd, n, bqi));
+		  }		/*switch */
+		n = next;
+	      }
+	    break;
+	  }
+	case N_or:
+	case N_or_terms:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		res = MergeLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_term:
+	  {
+	    res = GetDocsOp (qd, GetNthWE (qd->TL, BOOL_TERM (n)), op_term, NULL);
+	    break;
+	  }
+	}
+      return (res ? res : MakeDocList (0));
+    }
+  else
+    return MakeDocList (0);
+}
+
+/* =========================================================================
+ * Function: AdjustParaDocs
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+static void
+AdjustParaDocs (query_data * qd, DocList * Docs)
+{
+  DocEntry *DE = Docs->DE;
+  int s, d;
+  Splay_Tree *SP;
+
+
+  /* Make the doc-list contain only references to unique
+   * real documents. So can get rid of cases where there
+   * are multiple paragraphs for a real document.
+   */
+
+  /* Note: [TS: 29/Sep/94] */
+  /* reads in seek-pos for docnums so can compare docs */
+  /* seek-pos used as real doc identifier */
+  /* whereas doc-num entries refer to paragraphs -- I think */
+  for (s = 0; s < Docs->num; s++)
+    FetchDocStart (qd, DE[s].DocNum, &DE[s].SeekPos, &DE[s].Len);
+
+  Message ("Original Number = %d\n", Docs->num);
+  SP = SP_createset (DE_comp);
+  d = 0;
+  for (s = 0; s < Docs->num; s++)
+    if (!SP_member (&DE[s], SP))
+      {
+	DE[d] = DE[s];
+	SP_insert (&DE[d], SP);
+	d++;
+      }
+  SP_freeset (SP);
+  Docs->num = d;
+  Message ("Final Number = %d\n", Docs->num);
+}
+
+/* =========================================================================
+ * Function: BooleanQuery 
+ * Description: 
+ *      The main control function for the boolean query.
+ * Input: 
+ * Output: 
+ *      qd->DL
+ * ========================================================================= */
+
+void 
+BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+	      int stem_method)
+{
+  bool_tree_node *tree;
+  DocList *Docs;
+  int res = 0;
+
+  tree = ParseBool (Query, MAXLINEBUFFERLEN, &(qd->TL), 
+		    qd->sd->sdh.stemmer_num, stem_method, &res,
+		    qd->sd, qd->sd->sdh.indexed,         /* [RPAP - Jan 97: Stem Index Change] */
+		    &(qd->QTL));                         /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [TS:Aug/94] no longer tries to continue after parse failure */
+  if (res != 0)			/* failed the parse */
+    {
+      qd->DL = NULL;		/* have empty doc-list */
+      return;
+    }
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  /*  ReadInTermInfo (qd); */
+
+  FindNoneTerms (qd, tree);
+
+  OptimiseBoolTree (tree, qd->TL, atoi (GetEnv ("optimise_type")));
+
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_terms = 0;
+
+  Docs = BooleanGet (qd, tree, bqi);
+
+/* [RPAP - Feb 97: NZDL Specifics] */
+#if !defined(PARABOOLEANMATCHES) && !defined(NZDL)
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      AdjustParaDocs (qd, Docs);
+    }
+#endif
+
+  if (bqi->MaxDocsToRetrieve != -1 && Docs->num > bqi->MaxDocsToRetrieve)
+    Docs->num = bqi->MaxDocsToRetrieve;
+
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_query.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * bool_query.h -- boolean query processing
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:28  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:36  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_QUERY_H
+#define BOOL_QUERY_H
+
+#include "backend.h"
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_tester.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_tester.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_tester.c	(revision 21112)
@@ -0,0 +1,182 @@
+/**************************************************************************
+ *
+ * bool_tester -- used to test bool_tree, bool_parser, bool_optimiser
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2000/03/05 22:58:24  sjboddie
+   compiler complained about using stdin and stdout to do global
+   initialization - variables were never changed so they were
+   replaced with #defines
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:41  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:23  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:05  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#define MAX_LINE_LEN  255
+#define STEMMER_NUM 0 /* Lovin's stemmer */
+#define STEM_METHOD 3
+
+/* Newer compilers complain when you use stdin and stdout to do global 
+initialization. Since they're static we'll just use #defines instead
+static FILE *file_in = stdin;
+static FILE *file_out = stdout;
+*/
+#define FILE_IN stdin
+#define FILE_OUT stdout
+
+static char line[MAX_LINE_LEN + 1];
+
+/* --- prototypes --- */
+static char *prompt (char *str);
+
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int main (int argc, char *argv[])
+{
+  bool_tree_node *tree = NULL;
+  TermList *term_list = NULL;
+  QueryTermList *query_term_list = NULL;  /* [RPAP - Feb 97: Term Frequency] */
+  int opt_type = 0;
+
+  while (1)
+    {
+      int res = 0;
+      int len = 0;
+
+      /* get a line */
+      if (!prompt ("Please enter boolean expression..."))
+	break;
+
+      len = strlen (line) - 1;	/* -1 => ignore the \n */
+
+      tree = ParseBool (line, len, &term_list, STEMMER_NUM, STEM_METHOD, &res,
+			NULL, 0,             /* [RPAP - Jan 97: Stem Index Change] */
+			&query_term_list);   /* [RPAP - Feb 97: Term Frequency] */
+
+      {
+	int done = 0;
+	while (!done)
+	  {
+	    if (!prompt ("Which type of optimisation 1 or 2 ?"))
+	      break;
+	    opt_type = atoi (line);
+	    done = (opt_type == 1 || opt_type == 2);
+	  }
+      }
+
+      if (!prompt ("Do you want to assign doc_counts to terms ?"))
+	break;
+
+      if (toupper (line[0]) == 'Y')
+	{
+	  /* cycle thru terms asking for doc_counts */
+	  int i;
+	  for (i = 0; i < term_list->num; i++)
+	    {
+	      char str[80];
+	      TermEntry *te = &(term_list->TE[i]);
+
+	      sprintf (str, "Please enter doc count for term '%s'",
+		       str255_to_string (te->Word, NULL));
+	      prompt (str);
+	      te->WE.doc_count = atoi (line);
+	      printf ("doc_count = %ld\n", te->WE.doc_count);
+	    }			/*for */
+	}			/*if */
+
+
+      if (res == 0)
+	{
+	  fprintf (FILE_OUT, "\n***Parsed Expression***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	  OptimiseBoolTree (tree, term_list, opt_type);
+	  fprintf (FILE_OUT, "\n***Optimised Expression ***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	}
+
+    }
+  return 0;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static char *
+prompt (char *str)
+{
+  fprintf (FILE_OUT, "\n%s\n", str);
+  return fgets (line, MAX_LINE_LEN, FILE_IN);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.c	(revision 21112)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * bool_tree.c -- Boolean parse tree ADT
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *       @(#)query.bool.y	1.9 16 Mar 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bool_tree.h"
+
+static char *RCSID = "$Id$";
+
+
+/* =========================================================================
+ * Function: CreateBoolNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolNode (N_Tag tag)
+{
+  bool_tree_node *n;
+  if (!(n = Xmalloc (sizeof (bool_tree_node))))
+    FatalError (1, "Can not create bool_tree_node for boolean query");
+  n->tag = tag;
+  BOOL_SIBLING (n) = NULL;
+  return n;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+		    u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len,  /* [RPAP - Feb 97: Term Frequency] */
+		    int stemmer_num) 
+{
+  bool_tree_node *n = NULL;
+
+  /* allocate bool_tree_node */
+  n = CreateBoolNode (N_term);
+
+  BOOL_TERM (n) = AddTerm (tl, (u_char *) text, Count, word_num, count, doc_count, 
+			   invf_ptr, invf_len, stemmer_num);  /* [RPAP - Feb 97: Term Frequency] */
+
+  return n;
+}
+
+
+/* =========================================================================
+ * Function: CreateTreeNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CreateBoolTreeNode (N_Tag tag, bool_tree_node * left, bool_tree_node * right)
+{
+  bool_tree_node *n = CreateBoolNode (tag);
+
+  BOOL_CHILD (n) = left;
+  if (left)
+    BOOL_SIBLING (left) = right;
+  if (right)
+    BOOL_SIBLING (right) = NULL;
+
+  return (n);
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+EqualBoolNodes (bool_tree_node * a, bool_tree_node * b)
+{
+  if (BOOL_TAG (a) == N_term && BOOL_TAG (a) == N_term)
+    return BOOL_TERM (a) == BOOL_TERM (b);
+  return 0;
+}
+
+/* =========================================================================
+ * Function: FreeBoolTree
+ * Description: 
+ *      Notes: does not fix up any parent or sibling links around given tree
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FreeTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	FreeTree (BOOL_CHILD (tree));
+      sibling = BOOL_SIBLING (tree);
+      Xfree (tree);
+      tree = sibling;
+    }
+}
+
+void 
+FreeBoolTree (bool_tree_node ** the_tree)
+{
+  bool_tree_node *tree = *the_tree;
+
+  if (!tree)
+    return;
+
+  FreeTree (tree);
+  *the_tree = NULL;
+/*********************************/
+  /* THIS IS POTENTIALLY DANGEROUS */
+  /* FIX UP LATER */
+  /* Need to correct the links for parent/siblings */
+/*********************************/
+
+}
+
+/* =========================================================================
+ * Function: PrintBoolTree
+ * Description: 
+ *      Print out a text version of the tree to the file
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PrintBinaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+  assert (BOOL_HAS_CHILD (tree));
+
+  child = BOOL_CHILD (tree);
+  while (child)
+    {
+      PrintBoolTree (child, file);
+      child = BOOL_SIBLING (child);
+      if (child)
+	fprintf (file, " %c ", op);
+    }
+}
+
+static void
+PrintUnaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+
+  fprintf (file, " %c ", op);
+  assert (BOOL_HAS_CHILD (tree));
+  child = BOOL_CHILD (tree);
+  PrintBoolTree (child, file);
+}
+
+void
+PrintBoolTree (bool_tree_node * tree, FILE * file)
+{
+  if (!tree)
+    return;
+
+  fprintf (file, "(");
+  switch (BOOL_TAG (tree))
+    {
+    case N_term:
+      fprintf (file, "term %d", BOOL_TERM (tree));
+      break;
+    case N_and:
+      PrintBinaryOp (tree, file, '&');
+      break;
+    case N_or:
+      PrintBinaryOp (tree, file, '|');
+      break;
+    case N_or_terms:
+      PrintBinaryOp (tree, file, '+');
+      break;
+    case N_diff:
+      PrintBinaryOp (tree, file, '-');
+      break;
+    case N_not:
+      PrintUnaryOp (tree, file, '!');
+      break;
+    case N_all:
+      fprintf (file, " TRUE ");
+      break;
+    case N_none:
+      fprintf (file, " FALSE ");
+      break;
+    default:
+      break;
+    }
+
+  fprintf (file, ")");
+}
+
+
+/* =========================================================================
+ * Function: CopyBoolTree
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CopyBoolTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling = NULL;
+  bool_tree_node *tree_copy = NULL;
+  bool_tree_node *prev = NULL;
+
+  if (!tree)
+    return NULL;
+
+  sibling = tree;
+  prev = NULL;
+  while (sibling)
+    {
+      /* copy sibling */
+      bool_tree_node *sibling_copy = (bool_tree_node *) malloc (sizeof (bool_tree_node));
+      if (!sibling_copy)
+	FatalError (1, "No memory for bool tree copy");
+      bcopy ((char *) sibling, (char *) sibling_copy, sizeof (bool_tree_node));
+
+      if (BOOL_HAS_CHILD (sibling))
+	{
+	  BOOL_CHILD (sibling_copy) = CopyBoolTree (BOOL_CHILD (sibling));
+	}
+
+      /* do the link up of copies */
+      if (prev)
+	BOOL_SIBLING (prev) = sibling_copy;
+      else
+	tree_copy = sibling_copy;
+      prev = sibling_copy;
+
+      sibling = BOOL_SIBLING (sibling);
+    }
+  return tree_copy;
+}
+
+
+/* =========================================================================
+ * Function: LastBoolSibling
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+LastBoolSibling (bool_tree_node * tree)
+{
+
+  if (!tree)
+    return NULL;
+
+  do
+    {
+      tree = BOOL_SIBLING (tree);
+    }
+  while (BOOL_SIBLING (tree));
+
+  return tree;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/bool_tree.h	(revision 21112)
@@ -0,0 +1,110 @@
+/**************************************************************************
+ *
+ * bool_tree.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:42  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:25  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:07  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:35  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_TREE_H
+#define BOOL_TREE_H
+
+#include "sysfuncs.h"
+#include "term_lists.h"
+
+typedef enum
+  {
+    N_term, N_and, N_or, N_diff, N_none, N_all,
+    N_not, N_or_terms
+  }
+N_Tag;
+
+typedef struct bool_tree_node
+  {
+    N_Tag tag;
+    struct bool_tree_node *sibling;
+    union
+      {
+	struct bool_tree_node *child;
+	int term_id;
+      }
+    fields;
+    int sum_ft;			/* for or_terms nodes */
+  }
+bool_tree_node;
+
+
+#define BOOL_HAS_CHILD(n) (\
+			(n)->tag==N_and || \
+			(n)->tag==N_or || (n)->tag==N_or_terms || \
+			(n)->tag==N_not || (n)->tag==N_diff)
+#define BOOL_TAG(n) ((n)->tag)
+#define BOOL_SIBLING(n) ((n)->sibling)
+#define BOOL_TERM(n) ((n)->fields.term_id)
+#define BOOL_CHILD(n) ((n)->fields.child)
+#define BOOL_SUM_FT(n) ((n)->sum_ft)
+
+bool_tree_node *CreateBoolNode (N_Tag tag);
+bool_tree_node *CopyBoolTree (bool_tree_node * tree);
+bool_tree_node *CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+				    u_long count, u_long doc_count, u_long invf_ptr, 
+				    u_long invf_len, int stemmer_num);
+bool_tree_node *CreateBoolTreeNode (N_Tag tag,
+			     bool_tree_node * left, bool_tree_node * right);
+void FreeBoolTree (bool_tree_node ** tree);
+void PrintBoolTree (bool_tree_node * tree, FILE * file);
+bool_tree_node *LastBoolSibling (bool_tree_node * tree);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/build.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/build.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/build.h	(revision 21112)
@@ -0,0 +1,171 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#define	TERMPARAGRAPH	'\003'
+
+
+int init_special (char *file_name);
+int init_text_1 (char *file_name);
+int init_text_2 (char *file_name);
+int init_invf_1 (char *file_name);
+int init_invf_2 (char *file_name);
+int init_ivf_1 (char *file_name);
+int init_ivf_2 (char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (u_char * s_in, int l_in);
+int process_text_1 (u_char * s_in, int l_in);
+int process_text_2 (u_char * s_in, int l_in);
+int process_invf_1 (u_char * s_in, int l_in);
+int process_invf_2 (u_char * s_in, int l_in);
+int process_ivf_1 (u_char * s_in, int l_in);
+int process_ivf_2 (u_char * s_in, int l_in);
+/*
+ * s_in               IN      The binary string to be compressed
+ * l_in         IN      The number of characters in s_in
+ *
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (char *filename);
+int done_text_1 (char *filename);
+int done_text_2 (char *filename);
+int done_invf_1 (char *filename);
+int done_invf_2 (char *filename);
+int done_ivf_1 (char *filename);
+int done_ivf_2 (char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char InvfLevel;
+/*
+ * This will determine the level of the inverted file it can take on the
+ * values 1, 2, or 3.
+ *
+ * Level 1: The inverted file contains only document numbers making it possible
+ *          to do only boolean queries.
+ * 
+ * Level 2: The inverted file also contains word counts per document making it
+ *          possible to do cosine ranked queries.
+ *
+ * Level 3: The inverted file contains word positions.
+ */
+
+
+extern unsigned long buf_size;
+/*
+ * The size of the document input buffer.
+ */
+
+extern unsigned long ChunkLimit;
+/*
+ * The maximum number of chunks that can be written to disk.
+ */
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+extern char SkipSGML;
+/*
+ * 1 if SGML tags are to be considered non-words when building the 
+ * inverted file.
+ */
+
+extern char MakeWeights;
+/*
+ * 1 if the weights file should be generated.
+ */
+
+extern FILE *Comp_Stats;
+/*
+ * Contains a file pointer to the file where compression stats should be sent
+ */
+
+extern int comp_stat_point;
+/*
+ * Generate a compression stat entry every comp_stat_point bytes 
+ */
+
+extern mg_ullong bytes_processed;
+/*
+ * The number of bytes processed. NOTE: This excludes document separators.
+ */
+
+extern mg_ullong bytes_received;
+/*
+ * The number of bytes processed. NOTE: This includes document separators.
+ */
+
+
+extern int stemmer_num;
+/*
+ * The stemmer to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+
+extern int stem_method;
+/*
+ * The method to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/commands.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/commands.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/commands.c	(revision 21112)
@@ -0,0 +1,582 @@
+/**************************************************************************
+ *
+ * commands.c -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:32  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:22  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "environment.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"		/* for qd */
+#include "term_lists.h"		/* for MAXTERMSTRLEN */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define OPENPIPE        _popen
+# define CLOSEPIPE       _pclose
+#else
+# define OPENPIPE        popen
+# define CLOSEPIPE       pclose
+#endif
+
+extern char *PathName;
+extern FILE *OutFile, *InFile;
+extern int OutPipe, InPipe;
+extern int Quitting;
+
+char *CommandsErrorStr = NULL;
+
+/* [TS: June/95] */
+/* current qd for use by any commands */
+static query_data *the_qd = NULL;
+
+
+#include "help.mg.h"
+#include "warranty.h"
+#include "conditions.h"
+
+static void 
+CmdQuit (void)
+{
+  Quitting = 1;
+}
+
+static void 
+CmdPush (void)
+{
+  PushEnv ();
+}
+
+
+static void 
+CmdPop (void)
+{
+  PopEnv ();
+  if (EnvStackHeight () == 1)
+    PushEnv ();
+}
+
+static void 
+CmdHelp (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (help_str) / sizeof (help_str[0]); i++)
+    fputs (help_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdWarranty (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (warranty_str) / sizeof (warranty_str[0]); i++)
+    fputs (warranty_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdConditions (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (cond_str) / sizeof (cond_str[0]); i++)
+    fputs (cond_str[i], more);
+
+  CLOSEPIPE (more);
+}
+
+static void 
+CmdReset (void)
+{
+  while (EnvStackHeight () > 1)
+    PopEnv ();
+  PushEnv ();
+}
+
+static void 
+CmdDisplay (void)
+{
+  int i;
+  size_t l = 0;
+  char *name;
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    if (strlen (name) > l)
+      l = strlen (name);
+
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+}
+
+static void 
+CmdQueryTerms (void)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+
+  if (the_qd && the_qd->TL)
+    {
+      ConvertTermsToString (the_qd->TL, terms_str);
+      printf ("%s\n", terms_str);
+      fflush (stdout);
+    }
+
+}
+
+
+static void 
+CmdSet (char *name, char *value)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  switch (SetEnv (name, value, NULL))
+    {
+    case -1:
+      CommandsErrorStr = "Out of memory\n";
+      break;
+    case -2:
+      CommandsErrorStr = ConstraintErrorStr;
+      break;
+    }
+}
+
+static void 
+CmdUnset (char *name)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  if (UnsetEnv (name, 0) == -1)
+    CommandsErrorStr = "Specified name does not exist";
+}
+
+
+
+
+
+
+/*
+ * Do the .input command 
+ */
+static void 
+CmdInput (char *param1, char *param2)
+{
+  enum
+  {
+    READ, APPEND, PIPEFROM
+  }
+  mode;
+  FILE *NewInput;
+  char *buf, *s;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '<')
+    {
+      s++;
+      mode = READ;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPEFROM;
+      s++;
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == READ)
+	NewInput = fopen (s, "r");
+      else
+	NewInput = OPENPIPE (s, "r");  /* [RPAP - Feb 97: WIN32 Port] */
+
+      if (NewInput)
+	{
+	  if (InPipe)
+	    CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (InFile != stdin)
+	    fclose (InFile);
+	  InPipe = (mode == PIPEFROM);
+	  InFile = NewInput;
+	}
+      else
+	CommandsErrorStr = "Bad input file/pipe";
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+    }
+  Xfree (buf);
+}
+
+
+
+static void 
+CmdOutput (char *param1, char *param2)
+{
+  enum
+  {
+    WRITE, APPEND, PIPETO
+  }
+  mode;
+  char *buf, *s;
+  FILE *NewOutput;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '>')
+    {
+      s++;
+      mode = APPEND;
+      if (*s == '>')
+	s++;
+      else
+	mode = WRITE;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPETO;
+      s++;
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == PIPETO)
+	NewOutput = OPENPIPE (s, "w");  /* [RPAP - Feb 97: WIN32 Port] */
+      else
+	NewOutput = fopen (s, mode == WRITE ? "w" : "a");
+      if (NewOutput)
+	{
+	  if (OutPipe)
+	    CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (OutFile != stdout)
+	    fclose (OutFile);
+	  OutPipe = (mode == PIPETO);
+	  OutFile = NewOutput;
+	}
+      else
+	CommandsErrorStr = "Bad output file/pipe";
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+    }
+  Xfree (buf);
+}
+
+
+
+static char *
+ParseArg (char *line, char *buffer)
+{
+  if (*line == '\"')
+    {
+      line++;
+      while (*line != '\"' && *line != '\0')
+	{
+	  if (*line == '\\')
+	    *buffer++ = *line++;
+	  if (*line != '\0')
+	    *buffer++ = *line++;
+	}
+      if (*line == '\"')
+	line++;
+    }
+  else
+    while (*line != ' ' && *line != '\t' && *line != '\0')
+      {
+	if (*line == '\\')
+	  line++;
+	if (*line != '\0')
+	  *buffer++ = *line++;
+      }
+  *buffer = '\0';
+  while (*line == ' ' || *line == '\t')
+    line++;
+  return (line);
+}
+
+
+
+char *
+ProcessCommands (char *line, query_data * qd)
+{
+  struct
+  {
+    char *Command;
+    void (*action) (void);
+    void (*action1) (char *);
+    void (*action2) (char *, char *);
+  }
+   *dc, dot_command[] =
+  {
+    {
+      "set", NULL, NULL, CmdSet
+    }
+    ,
+    {
+      "input", NULL, NULL, CmdInput
+    }
+    ,
+    {
+      "output", NULL, NULL, CmdOutput
+    }
+    ,
+    {
+      "unset", NULL, CmdUnset, NULL
+    }
+    ,
+    {
+      "help", CmdHelp, NULL, NULL
+    }
+    ,
+    {
+      "reset", CmdReset, NULL, NULL
+    }
+    ,
+    {
+      "display", CmdDisplay, NULL, NULL
+    }
+    ,
+    {
+      "push", CmdPush, NULL, NULL
+    }
+    ,
+    {
+      "quit", CmdQuit, NULL, NULL
+    }
+    ,
+    {
+      "pop", CmdPop, NULL, NULL
+    }
+    ,
+    {
+      "warranty", CmdWarranty, NULL, NULL
+    }
+    ,
+    {
+      "conditions", CmdConditions, NULL, NULL
+    }
+    ,
+    {
+      "queryterms", CmdQueryTerms, NULL, NULL
+    }
+  };
+
+
+  char command[512];
+  char param1[512], param2[512];
+  CommandsErrorStr = NULL;
+  the_qd = qd;			/* set globally to be accessed by command functions if needed */
+  while (*line == ' ' || *line == '\t')
+    line++;
+  while (*line == '.')
+    {
+      int i;
+      line++;
+      for (i = 0; isalpha (*line) && i < 512; i++, line++)
+	command[i] = tolower (*line);
+      while (*line == ' ' || *line == '\t')
+	line++;
+      if (i != 512)
+	{
+	  command[i] = '\0';
+	  for (i = 0, dc = dot_command; i < NUMOF (dot_command); i++, dc++)
+	    if (!strcmp (dc->Command, command))
+	      {
+		if (dc->action)
+		  dc->action ();
+		else if (dc->action1)
+		  {
+		    line = ParseArg (line, param1);
+		    dc->action1 (param1);
+		    break;
+		  }
+		else if (dc->action2)
+		  {
+		    line = ParseArg (line, param1);
+		    line = ParseArg (line, param2);
+		    dc->action2 (param1, param2);
+		    break;
+		  }
+		if (CommandsErrorStr)
+		  *line = '\0';
+		break;
+	      }
+	  if (i == NUMOF (dot_command))
+	    i = 100;
+	}
+      if (i == 100)
+	{
+	  CommandsErrorStr = "Illegal command";
+	  *line = '\0';
+	}
+      while (*line == ' ' || *line == '\t')
+	line++;
+    }
+  return (line);
+}
+
+
+
+
+
+void 
+read_mgrc_file (void)
+{
+  char line[1000];
+  char FileName[256];
+  int LineNum = 0;
+  FILE *mgrc;
+
+  strcpy (FileName, ".mgrc");
+  mgrc = fopen (FileName, "r");
+
+  if (!mgrc)
+    {
+      char *home_path = getenv ("HOME");
+      if (home_path)
+	{
+	  sprintf (FileName, "%s/.mgrc", home_path);
+	  mgrc = fopen (FileName, "r");
+	}
+    }
+
+  if (!mgrc)
+    return;
+
+  while (fgets (line, sizeof (line), mgrc))
+    {
+      char *s, linebuf[1000];
+      LineNum++;
+      if ((s = strchr (line, '\n')) != NULL)
+	*s = '\0';
+      strcpy (linebuf, line);
+      if ((s = strchr (line, '#')) != NULL)
+	*s = '\0';
+      if (!*line)
+	continue;
+      ProcessCommands (line, NULL);
+      if (CommandsErrorStr)
+	Message ("%s\n%s : ERROR in line %d of %s\n%s : %s\n",
+		 linebuf,
+		 msg_prefix, LineNum, FileName,
+		 msg_prefix, CommandsErrorStr);
+    }
+  fclose (mgrc);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/commands.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/commands.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/commands.h	(revision 21112)
@@ -0,0 +1,33 @@
+/**************************************************************************
+ *
+ * commands.h -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef COMMANDS_H
+#define COMMANDS_H
+
+#include "backend.h"
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.c	(revision 21112)
@@ -0,0 +1,255 @@
+/**************************************************************************
+ *
+ * comp_dict.c -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:41  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:24  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = Xmalloc (sizeof (dict_hash_table) +
+		      (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  bzero ((char *) ht->table, ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; i++, words++)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register int tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+	    goto error;
+
+	  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+	    goto error;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(char_codes[which] =
+		Generate_Huffman_Codes (&char_huff[which], NULL)))
+	    goto error;
+	  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(lens_codes[which] =
+		Generate_Huffman_Codes (&lens_huff[which], NULL)))
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(char_codes[which] =
+		    Generate_Huffman_Codes (&char_huff[which], NULL)))
+		goto error;
+	      if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(lens_codes[which] =
+		    Generate_Huffman_Codes (&lens_huff[which], NULL)))
+		goto error;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+
+#define HASH_RATIO 2
+
+
+
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+
+
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/gsdl/packages/mg/src/text/conditions.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/conditions.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/conditions.h	(revision 21112)
@@ -0,0 +1,226 @@
+static char *cond_str[] = {
+"		    GNU GENERAL PUBLIC LICENSE\n",
+"   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n",
+"\n",
+"  0. This License applies to any program or other work which contains\n",
+"a notice placed by the copyright holder saying it may be distributed\n",
+"under the terms of this General Public License.  The \"Program\", below,\n",
+"refers to any such program or work, and a \"work based on the Program\"\n",
+"means either the Program or any derivative work under copyright law:\n",
+"that is to say, a work containing the Program or a portion of it,\n",
+"either verbatim or with modifications and/or translated into another\n",
+"language.  (Hereinafter, translation is included without limitation in\n",
+"the term \"modification\".)  Each licensee is addressed as \"you\".\n",
+"\n",
+"Activities other than copying, distribution and modification are not\n",
+"covered by this License; they are outside its scope.  The act of\n",
+"running the Program is not restricted, and the output from the Program\n",
+"is covered only if its contents constitute a work based on the\n",
+"Program (independent of having been made by running the Program).\n",
+"Whether that is true depends on what the Program does.\n",
+"\n",
+"  1. You may copy and distribute verbatim copies of the Program's\n",
+"source code as you receive it, in any medium, provided that you\n",
+"conspicuously and appropriately publish on each copy an appropriate\n",
+"copyright notice and disclaimer of warranty; keep intact all the\n",
+"notices that refer to this License and to the absence of any warranty;\n",
+"and give any other recipients of the Program a copy of this License\n",
+"along with the Program.\n",
+"\n",
+"You may charge a fee for the physical act of transferring a copy, and\n",
+"you may at your option offer warranty protection in exchange for a fee.\n",
+"\n",
+"  2. You may modify your copy or copies of the Program or any portion\n",
+"of it, thus forming a work based on the Program, and copy and\n",
+"distribute such modifications or work under the terms of Section 1\n",
+"above, provided that you also meet all of these conditions:\n",
+"\n",
+"    a) You must cause the modified files to carry prominent notices\n",
+"    stating that you changed the files and the date of any change.\n",
+"\n",
+"    b) You must cause any work that you distribute or publish, that in\n",
+"    whole or in part contains or is derived from the Program or any\n",
+"    part thereof, to be licensed as a whole at no charge to all third\n",
+"    parties under the terms of this License.\n",
+"\n",
+"    c) If the modified program normally reads commands interactively\n",
+"    when run, you must cause it, when started running for such\n",
+"    interactive use in the most ordinary way, to print or display an\n",
+"    announcement including an appropriate copyright notice and a\n",
+"    notice that there is no warranty (or else, saying that you provide\n",
+"    a warranty) and that users may redistribute the program under\n",
+"    these conditions, and telling the user how to view a copy of this\n",
+"    License.  (Exception: if the Program itself is interactive but\n",
+"    does not normally print such an announcement, your work based on\n",
+"    the Program is not required to print an announcement.)\n",
+"
+\n",
+"These requirements apply to the modified work as a whole.  If\n",
+"identifiable sections of that work are not derived from the Program,\n",
+"and can be reasonably considered independent and separate works in\n",
+"themselves, then this License, and its terms, do not apply to those\n",
+"sections when you distribute them as separate works.  But when you\n",
+"distribute the same sections as part of a whole which is a work based\n",
+"on the Program, the distribution of the whole must be on the terms of\n",
+"this License, whose permissions for other licensees extend to the\n",
+"entire whole, and thus to each and every part regardless of who wrote it.\n",
+"\n",
+"Thus, it is not the intent of this section to claim rights or contest\n",
+"your rights to work written entirely by you; rather, the intent is to\n",
+"exercise the right to control the distribution of derivative or\n",
+"collective works based on the Program.\n",
+"\n",
+"In addition, mere aggregation of another work not based on the Program\n",
+"with the Program (or with a work based on the Program) on a volume of\n",
+"a storage or distribution medium does not bring the other work under\n",
+"the scope of this License.\n",
+"\n",
+"  3. You may copy and distribute the Program (or a work based on it,\n",
+"under Section 2) in object code or executable form under the terms of\n",
+"Sections 1 and 2 above provided that you also do one of the following:\n",
+"\n",
+"    a) Accompany it with the complete corresponding machine-readable\n",
+"    source code, which must be distributed under the terms of Sections\n",
+"    1 and 2 above on a medium customarily used for software interchange; or,\n",
+"\n",
+"    b) Accompany it with a written offer, valid for at least three\n",
+"    years, to give any third party, for a charge no more than your\n",
+"    cost of physically performing source distribution, a complete\n",
+"    machine-readable copy of the corresponding source code, to be\n",
+"    distributed under the terms of Sections 1 and 2 above on a medium\n",
+"    customarily used for software interchange; or,\n",
+"\n",
+"    c) Accompany it with the information you received as to the offer\n",
+"    to distribute corresponding source code.  (This alternative is\n",
+"    allowed only for noncommercial distribution and only if you\n",
+"    received the program in object code or executable form with such\n",
+"    an offer, in accord with Subsection b above.)\n",
+"\n",
+"The source code for a work means the preferred form of the work for\n",
+"making modifications to it.  For an executable work, complete source\n",
+"code means all the source code for all modules it contains, plus any\n",
+"associated interface definition files, plus the scripts used to\n",
+"control compilation and installation of the executable.  However, as a\n",
+"special exception, the source code distributed need not include\n",
+"anything that is normally distributed (in either source or binary\n",
+"form) with the major components (compiler, kernel, and so on) of the\n",
+"operating system on which the executable runs, unless that component\n",
+"itself accompanies the executable.\n",
+"\n",
+"If distribution of executable or object code is made by offering\n",
+"access to copy from a designated place, then offering equivalent\n",
+"access to copy the source code from the same place counts as\n",
+"distribution of the source code, even though third parties are not\n",
+"compelled to copy the source along with the object code.\n",
+"
+\n",
+"  4. You may not copy, modify, sublicense, or distribute the Program\n",
+"except as expressly provided under this License.  Any attempt\n",
+"otherwise to copy, modify, sublicense or distribute the Program is\n",
+"void, and will automatically terminate your rights under this License.\n",
+"However, parties who have received copies, or rights, from you under\n",
+"this License will not have their licenses terminated so long as such\n",
+"parties remain in full compliance.\n",
+"\n",
+"  5. You are not required to accept this License, since you have not\n",
+"signed it.  However, nothing else grants you permission to modify or\n",
+"distribute the Program or its derivative works.  These actions are\n",
+"prohibited by law if you do not accept this License.  Therefore, by\n",
+"modifying or distributing the Program (or any work based on the\n",
+"Program), you indicate your acceptance of this License to do so, and\n",
+"all its terms and conditions for copying, distributing or modifying\n",
+"the Program or works based on it.\n",
+"\n",
+"  6. Each time you redistribute the Program (or any work based on the\n",
+"Program), the recipient automatically receives a license from the\n",
+"original licensor to copy, distribute or modify the Program subject to\n",
+"these terms and conditions.  You may not impose any further\n",
+"restrictions on the recipients' exercise of the rights granted herein.\n",
+"You are not responsible for enforcing compliance by third parties to\n",
+"this License.\n",
+"\n",
+"  7. If, as a consequence of a court judgment or allegation of patent\n",
+"infringement or for any other reason (not limited to patent issues),\n",
+"conditions are imposed on you (whether by court order, agreement or\n",
+"otherwise) that contradict the conditions of this License, they do not\n",
+"excuse you from the conditions of this License.  If you cannot\n",
+"distribute so as to satisfy simultaneously your obligations under this\n",
+"License and any other pertinent obligations, then as a consequence you\n",
+"may not distribute the Program at all.  For example, if a patent\n",
+"license would not permit royalty-free redistribution of the Program by\n",
+"all those who receive copies directly or indirectly through you, then\n",
+"the only way you could satisfy both it and this License would be to\n",
+"refrain entirely from distribution of the Program.\n",
+"\n",
+"If any portion of this section is held invalid or unenforceable under\n",
+"any particular circumstance, the balance of the section is intended to\n",
+"apply and the section as a whole is intended to apply in other\n",
+"circumstances.\n",
+"\n",
+"It is not the purpose of this section to induce you to infringe any\n",
+"patents or other property right claims or to contest validity of any\n",
+"such claims; this section has the sole purpose of protecting the\n",
+"integrity of the free software distribution system, which is\n",
+"implemented by public license practices.  Many people have made\n",
+"generous contributions to the wide range of software distributed\n",
+"through that system in reliance on consistent application of that\n",
+"system; it is up to the author/donor to decide if he or she is willing\n",
+"to distribute software through any other system and a licensee cannot\n",
+"impose that choice.\n",
+"\n",
+"This section is intended to make thoroughly clear what is believed to\n",
+"be a consequence of the rest of this License.\n",
+"
+\n",
+"  8. If the distribution and/or use of the Program is restricted in\n",
+"certain countries either by patents or by copyrighted interfaces, the\n",
+"original copyright holder who places the Program under this License\n",
+"may add an explicit geographical distribution limitation excluding\n",
+"those countries, so that distribution is permitted only in or among\n",
+"countries not thus excluded.  In such case, this License incorporates\n",
+"the limitation as if written in the body of this License.\n",
+"\n",
+"  9. The Free Software Foundation may publish revised and/or new versions\n",
+"of the General Public License from time to time.  Such new versions will\n",
+"be similar in spirit to the present version, but may differ in detail to\n",
+"address new problems or concerns.\n",
+"\n",
+"Each version is given a distinguishing version number.  If the Program\n",
+"specifies a version number of this License which applies to it and \"any\n",
+"later version\", you have the option of following the terms and conditions\n",
+"either of that version or of any later version published by the Free\n",
+"Software Foundation.  If the Program does not specify a version number of\n",
+"this License, you may choose any version ever published by the Free Software\n",
+"Foundation.\n",
+"\n",
+"  10. If you wish to incorporate parts of the Program into other free\n",
+"programs whose distribution conditions are different, write to the author\n",
+"to ask for permission.  For software which is copyrighted by the Free\n",
+"Software Foundation, write to the Free Software Foundation; we sometimes\n",
+"make exceptions for this.  Our decision will be guided by the two goals\n",
+"of preserving the free status of all derivatives of our free software and\n",
+"of promoting the sharing and reuse of software generally.\n",
+"\n",
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/gsdl/packages/mg/src/text/conditions.src
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/conditions.src	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/conditions.src	(revision 21112)
@@ -0,0 +1,225 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/gsdl/packages/mg/src/text/environment.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/environment.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/environment.c	(revision 21112)
@@ -0,0 +1,688 @@
+/**************************************************************************
+ *
+ * environment.c -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2003/04/18 01:41:46  sjboddie
+   Added a new -M option to mg_passes, allowing "maxnumeric" to be altered.
+
+   Revision 1.1  1999/08/10 21:17:50  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.5  1995/03/14  05:15:26  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.4  1994/11/25  03:47:43  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:56:43  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:25  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+
+#include "environment.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define STRCASECMP      stricmp
+#else
+# define STRCASECMP      strcasecmp
+#endif
+
+typedef struct
+  {
+    char *Name, *Data;
+    char *(*Constraint) (char *Old, char *New);
+  }
+EEntry;
+
+typedef struct Env
+  {
+    EEntry *Environment;
+    int NumEnv;
+    struct Env *Next;
+  }
+Env;
+
+
+char *ConstraintErrorStr;
+
+static Env Base =
+{NULL, 0, NULL};
+
+
+/*
+ * Add name and data to the environment (data may be NULL)
+ * All names are converted to lower case
+ * returns 0 on success, -1 on memory failure or -2 on constraint failure
+ */
+int 
+SetEnv (char *name, char *data, char *(*Constraint) (char *, char *))
+{
+  int i;
+  EEntry *newenv, *env = Base.Environment;
+
+  if (data && *data == '\0')
+    data = NULL;
+
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* if not found the increase the environment size */
+  if (i >= Base.NumEnv)
+    {
+      if (Constraint)
+	{
+	  data = Constraint (NULL, data);
+	  if (!data)
+	    return (-2);
+	}
+      /* allocate memory for the new environment */
+      newenv = Base.Environment ?
+	Xrealloc (Base.Environment, sizeof (EEntry) * (Base.NumEnv + 1)) :
+	Xmalloc (sizeof (EEntry) * (Base.NumEnv + 1));
+      if (!newenv)
+	return (-1);
+      Base.Environment = newenv;
+      env = &newenv[Base.NumEnv];
+      env->Data = NULL;
+      env->Name = NULL;
+      env->Constraint = Constraint;
+      env->Name = Xstrdup (name);
+      if (!env->Name)
+	return (-1);
+      Base.NumEnv++;
+      for (i = 0; env->Name[i]; i++)
+	env->Name[i] = tolower (env->Name[i]);
+    }
+  if (env->Constraint)
+    {
+      data = env->Constraint (env->Data, data);
+      if (!data)
+	return (-2);
+    }
+  if (env->Data)
+    Xfree (env->Data);
+  env->Data = NULL;
+  if (data && *data != '\0')
+    {
+      env->Data = Xstrdup (data);
+      if (!env->Data)
+	return (-1);
+    }
+  return (0);
+}
+
+
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns NULL.
+ */
+char *
+GetEnv (char *name)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv)
+    return (NULL);
+
+  return (env->Data ? env->Data : "");
+}
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns default.
+ */
+char *
+GetDefEnv (char *name, char *def)
+{
+  char *data = GetEnv (name);
+  return (data ? data : def);
+}
+
+
+
+/*
+ * This function pushes the environment on to a stack and duplicated it in 
+ * the current stack.
+ *
+ */
+int 
+PushEnv (void)
+{
+  Env *env = Xmalloc (sizeof (Env));
+  int i;
+  *env = Base;
+  Base.Next = env;
+  Base.Environment = NULL;
+  Base.NumEnv = 0;
+  for (i = 0; i < env->NumEnv; i++)
+    if (SetEnv (env->Environment[i].Name, env->Environment[i].Data,
+		env->Environment[i].Constraint))
+      return (-1);
+  return (0);
+}
+
+int 
+PopEnv (void)
+{
+  Env *env;
+  int i;
+  if (!Base.Next)
+    return (-1);
+  for (i = 0; i < Base.NumEnv; i++)
+    {
+      Xfree (Base.Environment[i].Name);
+      if (Base.Environment[i].Data)
+	Xfree (Base.Environment[i].Data);
+    }
+  if (Base.Environment)
+    Xfree (Base.Environment);
+  env = Base.Next;
+  Base = *env;
+  Xfree (env);
+  return (0);
+}
+
+
+/*
+ * Count the number of environments on the stack, including the current one.
+ *
+ */
+int 
+EnvStackHeight (void)
+{
+  int i = 1;
+  Env *env = &Base;
+  while (env->Next)
+    {
+      i++;
+      env = env->Next;
+    }
+  return (i);
+}
+
+/*
+ * Delete environment variable name. If name does not exist it returns -1
+ * otherwise it returns 0
+ */
+int 
+UnsetEnv (char *name, int Force)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv ||
+      (Base.Environment[i].Constraint && !Force))
+    return (-1);
+  Xfree (Base.Environment[i].Name);
+  if (Base.Environment[i].Data)
+    Xfree (Base.Environment[i].Data);
+  for (i++; i < Base.NumEnv; i++)
+    Base.Environment[i - 1] = Base.Environment[i];
+  Base.NumEnv--;
+  if (Base.NumEnv == 0)
+    {
+      Xfree (Base.Environment);
+      Base.Environment = NULL;
+    }
+  return (0);
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns the name of environment variable number i or NULL if i is greater 
+ * than or equal to the number of environment variables. Names start from
+ * zero. 
+ */
+char *
+GetEnvName (int i)
+{
+  return (i < Base.NumEnv ? Base.Environment[i].Name : NULL);
+}
+
+static char *BooleanStrs[] =
+{"false", "true", "no", "yes", "off", "on"};
+
+static char *
+BooleanCons (char *Old, char *New)
+{
+  int i;
+  int old = -1, new = -1;
+  if (Old)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (Old, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  old = i;
+	  break;
+	}
+  if (New)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  if (!New)
+    return (BooleanStrs[Old ? old ^ 1 : 0]);
+  ConstraintErrorStr = "Invalid argument [true|false|yes|no|on|off] required";
+  return (NULL);
+}
+
+/*
+ * returns 0 or 1  for a boolean string
+ * or default on a error
+ */
+int 
+BooleanEnv (char *data, int def)
+{
+  int i;
+  if (!data)
+    return (def);
+  for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+    if (!STRCASECMP (data, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (i & 1);
+  return (def);
+}
+
+/*
+ * returns the value for a integer string
+ * or default on a error
+ */
+long 
+IntEnv (char *data, long def)
+{
+  long val;
+  char *ptr;
+  if (!data)
+    return (def);
+  val = strtol (data, &ptr, 10);
+  return *ptr ? def : val;
+}
+
+static char *
+NumberCmp (char *num, int min, int max)
+{
+  char *err;
+  int val;
+  static char Err[100];
+  ConstraintErrorStr = "Not a valid number";
+  if (!num)
+    return (NULL);
+  val = strtol (num, &err, 10);
+  if (*err)
+    return (NULL);
+  sprintf (Err, "Not in legal range [%d <= num <= %d]", min, max);
+  ConstraintErrorStr = Err;
+  if (val < min || val > max)
+    return (NULL);
+  return (num);
+}
+
+/*ARGSUSED */
+static char *
+MaxDocsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/*ARGSUSED */
+static char *
+MaxTermsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/* ARGSUSED */
+static char *
+BufferCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 16 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxNodesCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHashCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxParasCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHeadsCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 1000));
+}
+
+/* ARGSUSED */
+char *
+OptimiseCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 2));
+}
+
+/* ARGSUSED */
+char *
+MaxNumericCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 4, 512));
+}
+
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+static char *
+QueryCons (char *Old, char *New)
+{
+  static char *QueryStrs[] =
+  {
+    "boolean", "ranked",
+    "docnums", "approx-ranked"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (QueryStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, QueryStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [boolean|ranked|docnums|approx-ranked] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid accum type
+ *
+ */
+/* ARGSUSED */
+static char *
+AccumCons (char *Old, char *New)
+{
+  static char *AccumStrs[] =
+  {
+    "array", "splay_tree", "hash_table", "list"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (AccumStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, AccumStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument "
+    "[array|splay_tree|hash_table|list] required";
+  return (NULL);
+}
+
+
+
+/*
+ * Makes sure that New is a valid output type type
+ *
+ */
+/* ARGSUSED */
+static char *
+OutputTypeCons (char *Old, char *New)
+{
+  static char *OutputTypeStrs[] =
+  {"text", "silent", "docnums",
+   "count", "heads", "hilite"
+#ifdef TREC_MODE
+   ,"extras_for_trec"
+#endif
+  };
+
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (OutputTypeStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, OutputTypeStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [text|silent|docnums|heads|count|hilite"
+#ifdef TREC_MODE
+    "|extras_for_trec"
+#endif
+    "] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+char *
+HiliteStyleCons (char *Old, char *New)
+{
+  static char *StyleStrs[] =
+  {"bold", "underline"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (StyleStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, StyleStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [bold|underline] required";
+  return (NULL);
+}
+
+/* =========================================================================
+ * Function: get_output_type
+ * Description:
+ *      Map a mode string onto a mode char e.g. OUTPUT_????
+ * Input:
+ * Output:
+ *      The mode char which is a subrange value.
+ * ========================================================================= */
+
+char
+get_output_type (void)
+{
+  char *env_str = GetDefEnv ("mode", "text");
+  char ch1 = toupper (env_str[0]);
+  char ch2 = toupper (env_str[1]);
+
+  switch (ch1)
+    {
+    case 'T':
+      return OUTPUT_TEXT;
+    case 'S':
+      return OUTPUT_SILENT;
+    case 'E':
+      return OUTPUT_EXTRAS;
+    case 'C':
+      return OUTPUT_COUNT;
+    case 'D':
+      return OUTPUT_DOCNUMS;
+    case 'H':
+      if (ch2 == 'E')
+	return OUTPUT_HEADERS;
+      else
+	return OUTPUT_HILITE;
+    default:
+      FatalError (1, "Problem in output type switch");
+    }
+
+  return '\0';			/*shouldn't reach here -- keep compiler happy */
+}
+
+/* =========================================================================
+ * Function: get_query_type
+ * Description:
+ *      Map a query type string onto a char.
+ * Input:
+ * Output:
+ *      Query type char.
+ * ========================================================================= */
+
+char
+get_query_type (void)
+{
+  char *env_str = GetDefEnv ("query", "boolean");
+  char ch1 = toupper (env_str[0]);
+
+  switch (ch1)
+    {
+    case 'R':
+      return QUERY_RANKED;
+    case 'A':
+      return QUERY_APPROX;
+    case 'B':
+      return QUERY_BOOLEAN;
+    case 'D':
+      return QUERY_DOCNUMS;
+    default:
+      FatalError (1, "Problem in query type switch");
+    }
+
+  return '\0';			/*shouldn't reach here */
+}
+
+
+
+/*
+ * This initialises certain environment variables
+ *
+ */
+void 
+InitEnv (void)
+{
+  SetEnv ("hilite_style", "underline", HiliteStyleCons);	/*[TS:Sep/94] */
+  SetEnv ("briefstats", "off", BooleanCons);
+  SetEnv ("diskstats", "off", BooleanCons);
+  SetEnv ("expert", "false", BooleanCons);
+  SetEnv ("mgdir", getenv ("MGDATA") ? getenv ("MGDATA") : ".", NULL);
+  SetEnv ("mgname", "", NULL);
+  SetEnv ("maxdocs", "all", MaxDocsCons);
+  SetEnv ("memstats", "off", BooleanCons);
+  SetEnv ("mode", "text", OutputTypeCons);
+  SetEnv ("pager", getenv ("PAGER") ? getenv ("PAGER") : "more", NULL);
+  SetEnv ("qfreq", "true", BooleanCons);
+  SetEnv ("query", "boolean", QueryCons);
+  SetEnv ("sizestats", "off", BooleanCons);
+  SetEnv ("timestats", "off", BooleanCons);
+  SetEnv ("verbatim", "off", BooleanCons);
+  SetEnv ("sorted_terms", "on", BooleanCons);
+  SetEnv ("accumulator_method", "array", AccumCons);
+  SetEnv ("stop_at_max_accum", "off", BooleanCons);
+  SetEnv ("buffer", "1048576", BufferCons);
+  SetEnv ("max_accumulators", "50000", MaxNodesCons);
+  SetEnv ("max_terms", "all", MaxTermsCons);
+  SetEnv ("maxparas", "1000", MaxParasCons);
+  SetEnv ("hash_tbl_size", "1000", MaxHashCons);
+  SetEnv ("skip_dump", "skips.%d", NULL);
+  SetEnv ("ranked_doc_sepstr", "---------------------------------- %n %w\\n", NULL);
+  SetEnv ("doc_sepstr", "---------------------------------- %n\\n", NULL);
+  SetEnv ("para_sepstr", "\\n######## PARAGRAPH %n ########\\n", NULL);
+  SetEnv ("para_start", "***** Weight = %w *****\\n", NULL);
+  SetEnv ("terminator", "", NULL);
+  SetEnv ("heads_length", "50", MaxHeadsCons);
+  SetEnv ("optimise_type", "1", OptimiseCons);	/*[TS:Mar/95] */
+  SetEnv ("casefold", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("stem", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("term_freq", "off", BooleanCons);  /* [RPAP - Feb 97: Term Frequency] */
+  SetEnv ("maxnumeric", "4", MaxNumericCons);  /* [sjboddie - Jun 2002: Max Numeric word length] */
+}
+
+
+void 
+UninitEnv (void)
+{
+  char *name;
+  while (PopEnv () == 0);
+  while ((name = GetEnvName (0)) != 0)
+    UnsetEnv (name, 1);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/environment.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/environment.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/environment.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * environment.h -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef ENVIRONMENT_H
+#define ENVIRONMENT_H
+
+#define OUTPUT_TEXT     'T'
+#define OUTPUT_HEADERS  'H'
+#define OUTPUT_SILENT   'S'
+#define OUTPUT_EXTRAS   'E'
+#define OUTPUT_HILITE   'I'
+#define OUTPUT_COUNT    'C'
+#define OUTPUT_DOCNUMS  'D'
+
+#define QUERY_RANKED    'R'
+#define QUERY_APPROX    'A'
+#define QUERY_BOOLEAN   'B'
+#define QUERY_DOCNUMS   'D'
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/globals.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/globals.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/globals.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * globals.h -- Some global stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_GLOBALS
+#define H_GLOBALS
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/hash.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/hash.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_HASH
+#define H_HASH
+
+
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; --len)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.h	(revision 21112)
@@ -0,0 +1,320 @@
+static char *help_str[] = {
+"\n",
+"			  HELP for mgquery\n",
+"		   ================================\n",
+"\n",
+"The text is a summary of the information in the \"mgquery\" manual pages.\n",
+"\n",
+"The input to 'mgquery' consists of a series of input lines. The backslash\n",
+"character (\"\\\") is used at the end of lines to indicate that input \n",
+"continues on the next line.\n",
+"\n",
+"Input lines on which the first character is a dot (\".\") are commands to\n",
+"the mgquery program. Input lines that do not start with a dot are queries.\n",
+"\n",
+"A query consists of two parts. One part is a boolean or ranked query that\n",
+"identifies documents. The second part is a post-processing pattern matching\n",
+"operation. Any text between the first speech mark (\") and the last speech\n",
+"mark is considered to the a post-processing pattern. \n",
+"\n",
+"\n",
+"The following command are available :-\n",
+"\n",
+"    .help	     - displays this text.\n",
+"    .quit            - quits the program.\n",
+"    .set name value  - sets parameter \"name\" to \"value\" . If the parameter\n",
+"                       is a boolean parameter and value is omitted the\n",
+"	    	       parameter will be inverted (i.e. if it is true it\n",
+"		       will change to false, if it is false it will change\n",
+"		       to true).\n",
+"    .unset name      - deletes parameter \"name\"\n",
+"    .reset           - sets all the parameters to their initial state.\n",
+"    .display         - displays the values of all the current parameters.\n",
+"    .push            - pushes the current parameters on to a stack.\n",
+"    .pop             - destroys the current parameters and pops a new set\n",
+"                       of parameters off the stack.\n",
+"    .output arg      - This is used to specify where to send the documents.\n",
+"                       Arg may one be of the following:\n",
+"		       > filename  : Send output to the specified file.\n",
+"		       >> filename : Append output to the specified file.\n",
+"                       | command   : The output is piped into command, \n",
+"                                     which is executed by sh.\n",
+"    .input arg       - This is used to specify where input comes from.\n",
+"                       Arg may one be of the following:\n",
+"		       < filename  : Get the input from the specified\n",
+"                                     file.\n",
+"                       | command   : The input comes from the standard\n",
+"                                     output of command, which is executed\n",
+"                                     by sh.\n",
+"\n",
+"On startup the mgquery program reads from the file .mgrc a sequence \n",
+"of commands (NOTE: The .mgrc file may not contain any queries). mgquery \n",
+"first looks for .mgrc in the current directory and then in the users home \n",
+"directory. Lines starting with a '#' in the .mgrc file are considered to\n",
+"be comments and are ignored.\n",
+"\n",
+"The following parameters (used in the .set and .unset commands) are \n",
+"predefined and have special significance :-\n",
+"\n",
+"accumulator_method = `array'\n",
+"	This parameter is used during ranking, and specifies how the\n",
+"	weight for each document should be accumulated. The following\n",
+"	methods are available `array', `splay_tree', `hash_table', and\n",
+"	`list'.\n",
+"\n",
+"briefstats = `off'\n",
+"	This is a boolean parameter that determines whether the the\n",
+"	totals for disk, memory and time usage statistics will be \n",
+"	displayed. at the end of each query. \n",
+"	NOTE: this takes precedence over the parameters \"diskstats\",\n",
+"	\"memstats\" and \"timestats\". This parameter may take the values \n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"buffer = `1048576'\n",
+"	When the documents are being read in they are read into a \n",
+"	buffer of this size and then displayed from this buffer. If \n",
+"        the documents are larger than this buffer the buffer is\n",
+"	expanded automatically. Having a large buffer gives a very\n",
+"	slight performance improvement because it allows the order of \n",
+"	disk operations to be optimised. The buffer size is measured\n",
+"	in bytes.\n",
+"\n",
+"diskstats = `off'\n",
+"	This is a boolean parameter that determines whether the disk\n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"doc_sepstr = `---------------------------------- %n\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `boolean' or `docnums'\n",
+"	queries. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the MG document number use\n",
+"	the sequence `%n'.\n",
+"\n",
+"expert = `false'\n",
+"	If this is true then a lot of the waffle that the program \n",
+"	spits out is suppressed. This parameter may take the values\n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"hash_tbl_size = `1000'\n",
+"	One of the options during ranking queries is to use a hash \n",
+"	table to accumulate the weights for each document. The hash \n",
+"	table is a simple chained type. This parameter specifies the \n",
+"	size of the hash table and may take any value between 8 and\n",
+"	268435456. \n",
+"\n",
+"heads_length = `50'\n",
+"	When the mode is `heads' this specifies the number of \n",
+"	characters that	will be output for each document.\n",
+"\n",
+"maxdocs = `all'\n",
+"	The maximum number of documents to display in response to a\n",
+"	query. This parameter may take on a numeric value between 1 \n",
+"	and 429467295 or the word `all'.\n",
+"\n",
+"maxparas = `1000'\n",
+"	The maximum number of paragraphs to identify during a ranked\n",
+"	query with paragraph indexing. After the paragraphs have been\n",
+"	identified the paragraphs are converted into documents, and \n",
+"	because some of the paragraphs may refer to the same documents \n",
+"	the final number of answers may be less that maxparas. The \n",
+"	maxdocs parameter will then be applied. This parameter may \n",
+"	take on a numeric value between 1 and 429467295.\n",
+"\n",
+"max_accumulators = `50000'\n",
+"	This parameter limits the number of different paragraph/\n",
+"	document numbers to be accumulated during ranked queries when \n",
+"	the parameter `accumulator_method` is set to `splay_tree',\n",
+"	`hash_table', or `list'.  This parameter may take any value \n",
+"	between 8 and 268435456.\n",
+"\n",
+"max_terms = `all'\n",
+"	This parameter limits the number of terms that will actually\n",
+"	be used during a ranked query. If more terms than the number\n",
+"	specified by max_terms are entered, then the extra terms will\n",
+"	be discarded. If `sorted_terms' is on then the limiting will \n",
+"	be done after the terms have been sorted. This parameter may\n",
+"	take any value between 1 and 429467295 or the word `all'.\n",
+"\n",
+"memstats = `off'\n",
+"	This is a boolean parameter that determines whether the memory \n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes', \n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"mgdir = `.'\n",
+"	This specifies the directory where the MG files may be found.\n",
+"	If the environment variable `MGDATA' is set then `mgdir' is\n",
+"	initialised to the value in `MGDATA'.\n",
+"\n",
+"mgname = `'\n",
+"	This specifies the name of the MG database to process.\n",
+"\n",
+"mode = `text'\n",
+"	This specifies how documents should be displayed when they\n",
+"	are retrieved it may take four different values `text', \n",
+"	`docnums', `silent', `heads' or `count'. `text' displays \n",
+"	the contents of the document. `docnums' displays only the\n",
+"	document numbers. `Silent' retrieves all the documents but\n",
+"	displays nothing except how many documents were retrieved.\n",
+"	This mode is intended to be used in timing experiments. \n",
+"	`Heads` is used to print out the head of each document.\n",
+"	`Count' does the minimum amount of work required to determine\n",
+"	how many documents would be retrieved, but does not retrieve\n",
+"	them.\n",
+"\n",
+"pager = `more'\n",
+"	This is the name of the program that will be used to display\n",
+"	the help and the retrieved documents. If the environment \n",
+"	variable \"PAGER\" is defined then `pager' takes on that value.\n",
+"\n",
+"para_sepstr = `\\n######## PARAGRAPH %n ########\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	paragraphs. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the paragraph number within\n",
+"	the document use the sequence `%n'.\n",
+"\n",
+"para_start = `***** Weight = %w *****\\n'\n",
+"	This specifies the string that will be used at the head of \n",
+"	paragraphs for a paraghaph level index following a ranked query.\n",
+"	The standard C escape character sequences (see the man page)\n",
+"	may be used to place special characters in the string. For \n",
+"	example, a newline would the `\\n'. To include a `%' use the\n",
+"	sequence `%%'. To include the paragraph weight use the \n",
+"	sequence `%w'.\n",
+"\n",
+"qfreq = `true'\n",
+"	This determine whether the ranked queries will take into \n",
+"	account the number of times each query term is specified.\n",
+"	When this is `true' the number of times a term appears in\n",
+"	the query is used in the ranking. When this is `false' all \n",
+"	query term are assumed to occur only once. This parameter\n",
+"	may take the values `yes', `no', `true', `false', `on' or\n",
+"	`off'.\n",
+"\n",
+"query = `boolean'\n",
+"	This specifies the type of queries that are to be specified.\n",
+"	It can take four different values `boolean', `ranked', \n",
+"	`docnums' or `approx-ranked'. \n",
+"\n",
+"	`boolean' is for boolean queries. \n",
+"		The yacc grammar for boolean queries is as follows :-\n",
+"\n",
+"		query   : or;\n",
+" \n",
+"		or      : or '|' and\n",
+"		        | and ;	\n",
+" \n",
+"		and     : and '&' not\n",
+"        		| and not\n",
+"        		| not ;\n",
+"\n",
+"		not     : term\n",
+"		        | '!' not ;\n",
+" \n",
+"		term    : TERM\n",
+"		        | '(' or ')' ;\n",
+" \n",
+"\n",
+"\n",
+"\n",
+"	`ranked' and `approx-ranked' are for queries ranked by the\n",
+"		cosine measure. `approx-ranked' uses only the low\n",
+"		precision document lengths, and therefore only\n",
+"		produces an approximation to full cosine ranking.\n",
+"\n",
+"		query   : TERM\n",
+"      			| query TERM ;\n",
+"	\n",
+"	`docnums' allows the entry of document numbers. Multiple \n",
+"		numbers separated by spaces may be specified\n",
+"		or ranges separated by hyphens.\n",
+"\n",
+"\n",
+"		query   : range\n",
+"			| query range ;\n",
+"\n",
+"\n",
+"		range   : num\n",
+"			| num '-' num ;\n",
+"\n",
+"\n",
+"ranked_doc_sepstr = `---------------------------------- %n %w\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `ranked' or \n",
+"	`approx-ranked'	queries. The standard C escape character \n",
+"	sequences (see the man page) may be used to place special\n",
+"	characters in the string. For example, a newline would the\n",
+"	`\\n'. To include a `%' use the sequence `%%'. To include the\n",
+"	MG document number use the sequence `%n'. To include the\n",
+"	document weight use the sequence `%w'.\n",
+"\n",
+"sizestats = `false'\n",
+"	If this is true then various numbers are output at the end\n",
+"	of each query indicating what went on during the query. \n",
+"	This parameter may take the values `yes', `no', `true', \n",
+"	`false', `on' or `off'.\n",
+"\n",
+"skip_dump = `skips.%d'\n",
+"	If this parameter is set then during ranked queries on skipped\n",
+"	inverted files when `accumulator_method' is set to `splay_tree',\n",
+"	`hash_table', or `list` a file will be produced in the current\n",
+"	directory. The name of the file is the value of this parameter,\n",
+"	a `%d' in the file name will be replaced with the process id of\n",
+"	mgquery. This file will contain information about the usage of\n",
+"	skips during the query processing. This option is expensive; \n",
+"	use `.unset skip_dump' to obtain optimal performance.\n",
+"\n",
+"sorted_terms = `on'\n",
+"	This specifies whether of not the terms should be sorted into\n",
+"	decreasing occurrence in documents so that the least often\n",
+"	occurring terms are processed first when ranked queries are\n",
+"	being done. When this is true the terms are sorted. When this\n",
+"	is false the terms are not sorted and are instead processed in\n",
+"	order of occurrence. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"\n",
+"stop_at_max_accum = `on'\n",
+"	This specifies what should happen when the maximum number of\n",
+"	accumulators set by `max_accumulators' is reached. When this\n",
+"	is true the the processing of terms is stopped at the completion\n",
+"	of the current term. When this is false processing continues but\n",
+"	no new accumulators are created. This parameter may take the \n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"terminator = `'\n",
+"	This specifies the string that will be output after the	last\n",
+"	document from the previous query has been output. The standard\n",
+"	C escape character sequences (see the man page) may be used to\n",
+"	place special characters in the string. For example, a newline\n",
+"	would the `\\n'. To include a `%' use the sequence `%%'.\n",
+"	\n",
+"\n",
+"timestats = `false'\n",
+"	If this is true then the time to process a query is displayed\n",
+"	in both real time and CPU time. This parameter may take the\n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"verbatim = `off'\n",
+"	This is a boolean parameter that determines whether the program\n",
+"	should attempt to do a regular expression match on the retrieved\n",
+"	text. If verbatim is `on' and a post-processing strng is specified\n",
+"	with the query then the post-processing string will be searched for\n",
+"	in the documents just before they are displayed. If the string is\n",
+"	found the document will be displayed, if not the document will not\n",
+"	be displayed. If verbatim is `off' the post-processing string will\n",
+"	be considered a regular expression like in `vi' or `egrep'.\n",
+"	E.G. If verbatim is `on', \"and.*the\" will look for the 8 character\n",
+"	     sequence \"and.*the\". If verbatim is `off', \"and.*the\" will\n",
+"	     look for the sequence \"and\" followed somewhere later in the\n",
+"	     document by the sequence \"the\".\n",
+"	This parameter may take the values `yes', `no', `true', `false',\n",
+"	`on' or `off'.\n",
+"\n",
+"\n"};
Index: /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.src
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.src	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/help.mg.src	(revision 21112)
@@ -0,0 +1,347 @@
+###########################################################################
+#
+# help.mg.src -- Source for the help command
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)help.mg.src	1.8 21 Mar 1994
+#
+###########################################################################
+#
+# The help file for mgquery.  
+# Lines starting with '#' are treated as comments and are discarded when
+# the 'help.mg.h' file is produced
+#
+###########################################################################
+
+			  HELP for mgquery
+		   ================================
+
+The text is a summary of the information in the "mgquery" manual pages.
+
+The input to 'mgquery' consists of a series of input lines. The backslash
+character ("\") is used at the end of lines to indicate that input 
+continues on the next line.
+
+Input lines on which the first character is a dot (".") are commands to
+the mgquery program. Input lines that do not start with a dot are queries.
+
+A query consists of two parts. One part is a boolean or ranked query that
+identifies documents. The second part is a post-processing pattern matching
+operation. Any text between the first speech mark (") and the last speech
+mark is considered to the a post-processing pattern. 
+
+
+The following command are available :-
+
+    .help	     - displays this text.
+    .quit            - quits the program.
+    .set name value  - sets parameter "name" to "value" . If the parameter
+                       is a boolean parameter and value is omitted the
+	    	       parameter will be inverted (i.e. if it is true it
+		       will change to false, if it is false it will change
+		       to true).
+    .unset name      - deletes parameter "name"
+    .reset           - sets all the parameters to their initial state.
+    .display         - displays the values of all the current parameters.
+    .push            - pushes the current parameters on to a stack.
+    .pop             - destroys the current parameters and pops a new set
+                       of parameters off the stack.
+    .output arg      - This is used to specify where to send the documents.
+                       Arg may one be of the following:
+		       > filename  : Send output to the specified file.
+		       >> filename : Append output to the specified file.
+                       | command   : The output is piped into command, 
+                                     which is executed by sh.
+    .input arg       - This is used to specify where input comes from.
+                       Arg may one be of the following:
+		       < filename  : Get the input from the specified
+                                     file.
+                       | command   : The input comes from the standard
+                                     output of command, which is executed
+                                     by sh.
+
+On startup the mgquery program reads from the file .mgrc a sequence 
+of commands (NOTE: The .mgrc file may not contain any queries). mgquery 
+first looks for .mgrc in the current directory and then in the users home 
+directory. Lines starting with a '#' in the .mgrc file are considered to
+be comments and are ignored.
+
+The following parameters (used in the .set and .unset commands) are 
+predefined and have special significance :-
+
+accumulator_method = `array'
+	This parameter is used during ranking, and specifies how the
+	weight for each document should be accumulated. The following
+	methods are available `array', `splay_tree', `hash_table', and
+	`list'.
+
+briefstats = `off'
+	This is a boolean parameter that determines whether the the
+	totals for disk, memory and time usage statistics will be 
+	displayed. at the end of each query. 
+	NOTE: this takes precedence over the parameters "diskstats",
+	"memstats" and "timestats". This parameter may take the values 
+	`yes', `no', `true', `false', `on' or `off'.
+
+buffer = `1048576'
+	When the documents are being read in they are read into a 
+	buffer of this size and then displayed from this buffer. If 
+        the documents are larger than this buffer the buffer is
+	expanded automatically. Having a large buffer gives a very
+	slight performance improvement because it allows the order of 
+	disk operations to be optimised. The buffer size is measured
+	in bytes.
+
+diskstats = `off'
+	This is a boolean parameter that determines whether the disk
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+doc_sepstr = `---------------------------------- %n\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `boolean' or `docnums'
+	queries. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the MG document number use
+	the sequence `%n'.
+
+expert = `false'
+	If this is true then a lot of the waffle that the program 
+	spits out is suppressed. This parameter may take the values
+	`yes', `no', `true', `false', `on' or `off'.
+
+hash_tbl_size = `1000'
+	One of the options during ranking queries is to use a hash 
+	table to accumulate the weights for each document. The hash 
+	table is a simple chained type. This parameter specifies the 
+	size of the hash table and may take any value between 8 and
+	268435456. 
+
+heads_length = `50'
+	When the mode is `heads' this specifies the number of 
+	characters that	will be output for each document.
+
+maxdocs = `all'
+	The maximum number of documents to display in response to a
+	query. This parameter may take on a numeric value between 1 
+	and 429467295 or the word `all'.
+
+maxparas = `1000'
+	The maximum number of paragraphs to identify during a ranked
+	query with paragraph indexing. After the paragraphs have been
+	identified the paragraphs are converted into documents, and 
+	because some of the paragraphs may refer to the same documents 
+	the final number of answers may be less that maxparas. The 
+	maxdocs parameter will then be applied. This parameter may 
+	take on a numeric value between 1 and 429467295.
+
+max_accumulators = `50000'
+	This parameter limits the number of different paragraph/
+	document numbers to be accumulated during ranked queries when 
+	the parameter `accumulator_method` is set to `splay_tree',
+	`hash_table', or `list'.  This parameter may take any value 
+	between 8 and 268435456.
+
+max_terms = `all'
+	This parameter limits the number of terms that will actually
+	be used during a ranked query. If more terms than the number
+	specified by max_terms are entered, then the extra terms will
+	be discarded. If `sorted_terms' is on then the limiting will 
+	be done after the terms have been sorted. This parameter may
+	take any value between 1 and 429467295 or the word `all'.
+
+memstats = `off'
+	This is a boolean parameter that determines whether the memory 
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes', 
+	`no', `true', `false', `on' or `off'.
+
+mgdir = `.'
+	This specifies the directory where the MG files may be found.
+	If the environment variable `MGDATA' is set then `mgdir' is
+	initialised to the value in `MGDATA'.
+
+mgname = `'
+	This specifies the name of the MG database to process.
+
+mode = `text'
+	This specifies how documents should be displayed when they
+	are retrieved it may take four different values `text', 
+	`docnums', `silent', `heads' or `count'. `text' displays 
+	the contents of the document. `docnums' displays only the
+	document numbers. `Silent' retrieves all the documents but
+	displays nothing except how many documents were retrieved.
+	This mode is intended to be used in timing experiments. 
+	`Heads` is used to print out the head of each document.
+	`Count' does the minimum amount of work required to determine
+	how many documents would be retrieved, but does not retrieve
+	them.
+
+pager = `more'
+	This is the name of the program that will be used to display
+	the help and the retrieved documents. If the environment 
+	variable "PAGER" is defined then `pager' takes on that value.
+
+para_sepstr = `\n######## PARAGRAPH %n ########\n'
+	This specifies the string that will be used to separate 
+	paragraphs. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the paragraph number within
+	the document use the sequence `%n'.
+
+para_start = `***** Weight = %w *****\n'
+	This specifies the string that will be used at the head of 
+	paragraphs for a paraghaph level index following a ranked query.
+	The standard C escape character sequences (see the man page)
+	may be used to place special characters in the string. For 
+	example, a newline would the `\n'. To include a `%' use the
+	sequence `%%'. To include the paragraph weight use the 
+	sequence `%w'.
+
+qfreq = `true'
+	This determine whether the ranked queries will take into 
+	account the number of times each query term is specified.
+	When this is `true' the number of times a term appears in
+	the query is used in the ranking. When this is `false' all 
+	query term are assumed to occur only once. This parameter
+	may take the values `yes', `no', `true', `false', `on' or
+	`off'.
+
+query = `boolean'
+	This specifies the type of queries that are to be specified.
+	It can take four different values `boolean', `ranked', 
+	`docnums' or `approx-ranked'. 
+
+	`boolean' is for boolean queries. 
+		The yacc grammar for boolean queries is as follows :-
+
+		query   : or;
+ 
+		or      : or '|' and
+		        | and ;	
+ 
+		and     : and '&' not
+        		| and not
+        		| not ;
+
+		not     : term
+		        | '!' not ;
+ 
+		term    : TERM
+		        | '(' or ')' ;
+ 
+
+
+
+	`ranked' and `approx-ranked' are for queries ranked by the
+		cosine measure. `approx-ranked' uses only the low
+		precision document lengths, and therefore only
+		produces an approximation to full cosine ranking.
+
+		query   : TERM
+      			| query TERM ;
+	
+	`docnums' allows the entry of document numbers. Multiple 
+		numbers separated by spaces may be specified
+		or ranges separated by hyphens.
+
+
+		query   : range
+			| query range ;
+
+
+		range   : num
+			| num '-' num ;
+
+
+ranked_doc_sepstr = `---------------------------------- %n %w\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `ranked' or 
+	`approx-ranked'	queries. The standard C escape character 
+	sequences (see the man page) may be used to place special
+	characters in the string. For example, a newline would the
+	`\n'. To include a `%' use the sequence `%%'. To include the
+	MG document number use the sequence `%n'. To include the
+	document weight use the sequence `%w'.
+
+sizestats = `false'
+	If this is true then various numbers are output at the end
+	of each query indicating what went on during the query. 
+	This parameter may take the values `yes', `no', `true', 
+	`false', `on' or `off'.
+
+skip_dump = `skips.%d'
+	If this parameter is set then during ranked queries on skipped
+	inverted files when `accumulator_method' is set to `splay_tree',
+	`hash_table', or `list` a file will be produced in the current
+	directory. The name of the file is the value of this parameter,
+	a `%d' in the file name will be replaced with the process id of
+	mgquery. This file will contain information about the usage of
+	skips during the query processing. This option is expensive; 
+	use `.unset skip_dump' to obtain optimal performance.
+
+sorted_terms = `on'
+	This specifies whether of not the terms should be sorted into
+	decreasing occurrence in documents so that the least often
+	occurring terms are processed first when ranked queries are
+	being done. When this is true the terms are sorted. When this
+	is false the terms are not sorted and are instead processed in
+	order of occurrence. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+
+stop_at_max_accum = `on'
+	This specifies what should happen when the maximum number of
+	accumulators set by `max_accumulators' is reached. When this
+	is true the the processing of terms is stopped at the completion
+	of the current term. When this is false processing continues but
+	no new accumulators are created. This parameter may take the 
+	values `yes', `no', `true', `false', `on' or `off'.
+
+terminator = `'
+	This specifies the string that will be output after the	last
+	document from the previous query has been output. The standard
+	C escape character sequences (see the man page) may be used to
+	place special characters in the string. For example, a newline
+	would the `\n'. To include a `%' use the sequence `%%'.
+	
+
+timestats = `false'
+	If this is true then the time to process a query is displayed
+	in both real time and CPU time. This parameter may take the
+	values `yes', `no', `true', `false', `on' or `off'.
+
+verbatim = `off'
+	This is a boolean parameter that determines whether the program
+	should attempt to do a regular expression match on the retrieved
+	text. If verbatim is `on' and a post-processing strng is specified
+	with the query then the post-processing string will be searched for
+	in the documents just before they are displayed. If the string is
+	found the document will be displayed, if not the document will not
+	be displayed. If verbatim is `off' the post-processing string will
+	be considered a regular expression like in `vi' or `egrep'.
+	E.G. If verbatim is `on', "and.*the" will look for the 8 character
+	     sequence "and.*the". If verbatim is `off', "and.*the" will
+	     look for the sequence "and" followed somewhere later in the
+	     document by the sequence "the".
+	This parameter may take the values `yes', `no', `true', `false',
+	`on' or `off'.
+
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/invf.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/invf.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/invf.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_INVF
+#define H_INVF
+
+/* NOTE: This does not include the magic number */
+struct invf_dict_header
+  {
+    unsigned long lookback;
+    unsigned long dict_size;
+    unsigned long total_bytes;
+    unsigned long index_string_bytes;
+    double input_bytes; /* [RJM 07/97: 4G limit] */
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+  };
+
+struct stem_dict_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+    unsigned long indexed;       /* [RPAP - Jan 97: Stem Index Change] */
+  };
+
+struct invf_file_header
+  {
+    unsigned long no_of_words;
+    unsigned long no_of_ptrs;
+    unsigned long skip_mode;
+    unsigned long params[16];
+    unsigned long InvfLevel;
+  };
+
+/* [RPAP - Jan 97: Stem Index Change] */
+struct stem_idx_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_words;
+  };
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.c	(revision 21112)
@@ -0,0 +1,1666 @@
+/**************************************************************************
+ *
+ * invf_get.c -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "bitio_gen.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "sptree.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "invf_get.h"
+#include "mg_errors.h"
+
+#define SKIPPING
+#define SHORT_CUT
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    Invf_Doc_Entry ide[1];
+  };
+
+static int skip_header = 0;
+
+static char skip_header_text0[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of docnum/word_count entries decoded.\n"
+"# 6\tNumber of hits.\n"
+"# 7\tNumber of entries in the splay tree or hash table.\n"
+"# 8\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char skip_header_text1[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of skips taken in evaluating the inverted file entry.\n"
+"# 6\tNumber of docnum/word_count entries decoded.\n"
+"# 7\tNumber of hits.\n"
+"# 8\tNumber of entries in the splay tree or hash table.\n"
+"# 9\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+
+static char skip_header_text2[] =
+"#######################################################################\n"
+"#\n"
+"#  1\tWord number.\n"
+"#  2\tNumber of time the word occurs in the collection.\n"
+"#  3\tNumber of documents the word occurs in.\n"
+"#  4\tThe word.\n"
+"#  5\tSkip size.\n"
+"#  6\tNumber of skips taken in evaluating the inverted file entry.\n"
+"#  7\tNumber of docnum/word_count entries decoded.\n"
+"#  8\tNumber of hits.\n"
+"#  9\tNumber of entries in the splay tree or hash table.\n"
+"# 10\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char *skip_header_text[] =
+{skip_header_text0,
+ skip_header_text1,
+ skip_header_text2};
+
+
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static Hash_Table *
+HT_create (query_data * qd, unsigned max_entries,
+	   unsigned max_size)
+{
+  unsigned es, hts;
+  Hash_Table *HT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * (max_entries - 1);
+  hts = sizeof (Invf_Doc_EntryH *) * max_size;
+
+  HT = Xmalloc (es);
+  if (!HT)
+    return NULL;
+
+  HT->HashTable = Xmalloc (hts);
+  if (!(HT->HashTable))
+    {
+      Xfree (HT);
+      return NULL;
+    }
+
+  ChangeMemInUse (qd, es + hts);
+  bzero ((char *) (HT->HashTable), hts);
+  HT->max = max_entries;
+  HT->num = 0;
+  HT->Suplimentary_Entries = NULL;
+  HT->Suplimentary_Size = 0;
+  HT->Suplimentary_Num = 0;
+  HT->Head = NULL;
+  HT->size = max_size;
+  return HT;
+}
+
+
+static void 
+HT_Sup_create (query_data * qd, Hash_Table * HT, unsigned max_size)
+{
+  int es = sizeof (Invf_Doc_EntryH) * max_size;
+  HT->Suplimentary_Entries = Xmalloc (es);
+  if (!HT->Suplimentary_Entries)
+    return;
+  HT->Suplimentary_Size = max_size;
+  HT->Suplimentary_Num = 0;
+  ChangeMemInUse (qd, es);
+}
+
+static void 
+HT_Sup_Add (Hash_Table * HT, unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH *IDEH;
+  IDEH = &HT->Suplimentary_Entries[HT->Suplimentary_Num++];
+  IDEH->next = NULL;
+  IDEH->IDE.DocNum = DN;
+  IDEH->IDE.Sum = Sum;
+}
+
+Invf_Doc_Entry *
+HT_find (Hash_Table * HT, unsigned long DN)
+{
+  Invf_Doc_EntryH *IDEH;
+  if (!HT->HashTable)
+    FatalError (1, "Accessing the hash_table through HT_find after HT_sort");
+  IDEH = HT->HashTable[DN % HT->size];
+  while (IDEH)
+    {
+      if (IDEH->IDE.DocNum == DN)
+	return &(IDEH->IDE);
+      IDEH = IDEH->next;
+    }
+  return NULL;
+}
+
+
+static int 
+HT_insert (Hash_Table * HT,
+	   unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH **IDEH;
+  if (HT->num == HT->max)
+    return 0;
+  IDEH = &(HT->HashTable[DN % HT->size]);
+  while (*IDEH)
+    IDEH = &((*IDEH)->next);
+  *IDEH = &(HT->IDEH[HT->num++]);
+  (*IDEH)->next = NULL;
+  (*IDEH)->IDE.DocNum = DN;
+  (*IDEH)->IDE.Sum = Sum;
+  return 1;
+}
+
+
+void 
+HT_free (query_data * qd, Hash_Table * HT)
+{
+  unsigned es;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * HT->max +
+    sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  if (HT->Suplimentary_Entries)
+    {
+      Xfree (HT->Suplimentary_Entries);
+      es += sizeof (Invf_Doc_EntryH) * HT->Suplimentary_Size;
+    }
+  Xfree (HT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDEH_comp (const void *a, const void *b)
+{
+  const Invf_Doc_EntryH *A = a;
+  const Invf_Doc_EntryH *B = b;
+  return A->IDE.DocNum - B->IDE.DocNum;
+}
+
+static void 
+HT_sort (query_data * qd, Hash_Table * HT)
+{
+  int i;
+  Invf_Doc_EntryH *l, *r, **b;
+  unsigned es = sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  HT->size = 0;
+  HT->HashTable = NULL;
+  ChangeMemInUse (qd, -es);
+
+  qsort (HT->IDEH, HT->num, sizeof (Invf_Doc_EntryH), IDEH_comp);
+
+  for (i = 0; i < HT->num - 1; i++)
+    HT->IDEH[i].next = &HT->IDEH[i + 1];
+  HT->IDEH[i].next = NULL;
+  l = HT->IDEH;
+
+  if (HT->Suplimentary_Entries)
+    {
+      for (i = 0; i < HT->Suplimentary_Num - 1; i++)
+	HT->Suplimentary_Entries[i].next = &HT->Suplimentary_Entries[i + 1];
+      HT->Suplimentary_Entries[i].next = NULL;
+    }
+  r = HT->Suplimentary_Entries;
+
+  b = &HT->Head;
+
+  while (l || r)
+    {
+      if (r && (!l || (l->IDE.DocNum > r->IDE.DocNum)))
+	{
+	  *b = r;
+	  b = &r->next;
+	  r = *b;
+	}
+      else
+	{
+	  *b = l;
+	  b = &l->next;
+	  l = *b;
+	}
+    }
+  *b = NULL;
+}
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static List_Table *
+LT_create (query_data * qd, unsigned max_entries)
+{
+  unsigned es;
+  List_Table *LT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * (max_entries - 1);
+
+  LT = Xmalloc (es);
+  if (!LT)
+    return NULL;
+
+  ChangeMemInUse (qd, es);
+  LT->max = max_entries;
+  LT->num = 0;
+  return LT;
+}
+
+
+
+Invf_Doc_Entry *
+LT_find (List_Table * LT, unsigned long DN)
+{
+  int l = 0;
+  int r = LT->num - 1;
+  int m, c;
+  while (l <= r)
+    {
+      m = (l + r) / 2;
+      c = DN - LT->IDE[m].DocNum;
+      if (c < 0)
+	r = m - 1;
+      else if (c > 0)
+	l = m + 1;
+      else
+	return &LT->IDE[m];
+    }
+  return NULL;
+}
+
+static List_Table *
+LT_add (query_data * qd, List_Table * LT,
+	unsigned long DN, float Sum)
+{
+  Invf_Doc_Entry *ide;
+
+  /* increase the size of the list if there is not enough */
+  /* room for this entry */
+  if (LT->num == LT->max)
+    {
+      int old, new, max_entries;
+      old = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+      max_entries = LT->max + (LT->max >> 1);
+      new = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * max_entries;
+
+      LT = Xrealloc (LT, new);
+      if (!LT)
+	return NULL;
+
+      ChangeMemInUse (qd, new - old);
+      LT->max = max_entries;
+    }
+
+  /* add the entry to the end of the list */
+  ide = &LT->IDE[LT->num];
+  ide->DocNum = DN;
+  ide->Sum = Sum;
+  LT->num++;
+
+  return LT;
+}
+
+
+void 
+LT_free (query_data * qd, List_Table * LT)
+{
+  unsigned es;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+  Xfree (LT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDE_comp (const void *a, const void *b)
+{
+  const Invf_Doc_Entry *A = a;
+  const Invf_Doc_Entry *B = b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+LT_sort (List_Table * LT)
+{
+  qsort (LT->IDE, LT->num, sizeof (Invf_Doc_Entry), IDE_comp);
+}
+
+
+/* [RJM 07/97: Ranked Required Terms] */
+/* accumulators with a sum less than or equal to 0.0 are to be deleted */
+static void 
+LT_pack (List_Table * LT)
+{
+  int s, d = 0;
+  for (s = 0; s < LT->num; s++) {
+      if (LT->IDE[s].Sum <= 0.0) {
+	/* ignore this accumulator */
+
+      } else if (d > 0 && LT->IDE[s].DocNum == LT->IDE[d-1].DocNum) {
+	LT->IDE[d-1].Sum += LT->IDE[s].Sum;
+
+      } else {
+	if (d != s) LT->IDE[d] = LT->IDE[s];
+	d++;
+      }
+  }
+
+  if (LT->num) LT->num = d;
+}
+
+
+/**************************************************************************/
+
+
+
+invf_data *
+InitInvfFile (File * InvfFile, stemmed_dict * sd)
+{
+  invf_data *id;
+  if (!(id = Xmalloc (sizeof (invf_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return NULL;
+    }
+
+  id->InvfFile = InvfFile;
+
+  Fread (&id->ifh, sizeof (id->ifh), 1, InvfFile);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  {
+    int i;
+    
+    NTOHUL(id->ifh.no_of_words);
+    NTOHUL(id->ifh.no_of_ptrs);
+    NTOHUL(id->ifh.skip_mode);
+    for (i = 0; i < 16; i++)
+      NTOHUL(id->ifh.params[i]);
+    NTOHUL(id->ifh.InvfLevel);
+  }
+
+  id->N = sd->sdh.num_of_docs;
+  id->Nstatic = sd->sdh.static_num_of_docs;
+
+  return (id);
+}
+
+
+void 
+FreeInvfData (invf_data * id)
+{
+  Xfree (id);
+}
+
+/****************************************************************************/
+
+
+static void 
+CalcBlks (query_data * qd, WordEntry * we, int *blk,
+	  int *dn_blk, int *len_blk, int *k, int *p)
+{
+  *p = we->doc_count;
+
+  *blk = BIO_Bblock_Init (qd->id->Nstatic, *p);
+  switch (qd->id->ifh.skip_mode)
+    {
+    case 1:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[0])
+	  *k = 0;
+	else
+	  {
+	    *k = qd->id->ifh.params[0];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs, (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    case 2:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[1])
+	  *k = 0;
+	else
+	  {
+	    *k = (int) (2 * sqrt (((double) (*p)) / qd->id->ifh.params[0]));
+	    if (*k <= qd->id->ifh.params[1])
+	      *k = qd->id->ifh.params[1];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs,
+				       (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    default:
+      *k = 0;
+    }
+}
+
+
+/* =========================================================================
+ * Function: GetDocsOp
+ * Description: 
+ *      if op == op_term then 
+ *        look up term in invf and decode all its entries/documents
+ *      if op == op_and_terms then 
+ *        go thru L, testing if it exists in invf using skips  
+ *        if it does exist then overwrite the entry nearer the start of the list
+ *      if op == op_and_or_terms then
+ *        go thru L, testing if it exists in invf using skips  
+ *        BUT only mark ones which are found - keep list intact
+ *      if op == op_diff_terms then     >>>> NOT tested <<<<
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+DocList *
+GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L)
+{
+  u_char *buffer;
+  DocEntry *src = NULL, *dest = NULL;
+  int i;			/* index counter into inverted list */
+  unsigned long pos;
+  int next_start, next_doc_num, next_i;
+  int block_counter;		/* within counter in block */
+  int dn_blk = 0, len_blk = 0;
+  int skip_size, have_skipping;
+  int blk;			/* bblock control parameter */
+  int ft;			/* The number of documents the word occurs in */
+  unsigned long CurrDoc = 0;	/* NOTE: Document numbers start at 1 */
+  float Weight;
+
+  /* ... AND empty */
+  if (op == op_and_terms && (!L || !L->num))
+    {
+      if (L)
+	Xfree (L);
+      return MakeDocList (0);
+    }
+
+  if (op == op_term)
+    L = MakeDocList (we->doc_count);
+
+  CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &skip_size, &ft);
+  have_skipping = (skip_size != 0);
+  if (!(buffer = Xmalloc (we->invf_len)))
+    return MakeDocList (0);
+
+  ChangeMemInUse (qd, we->invf_len);
+
+  qd->num_of_terms++;
+
+  /* --- Read an inverted file entry for the given word --- */
+  Fseek (qd->File_invf, we->invf_ptr, 0);
+  Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+  /* Read from the buffer */
+  DECODE_START (buffer, we->invf_len);
+
+  Weight = qd->sd->sdh.num_of_words / (100 * log2 ((float) we->count + 1));
+
+  if (op != op_term)
+    dest = src = L->DE;
+  i = pos = 0;
+  next_i = next_start = next_doc_num = 0;
+  block_counter = skip_size - 1;
+
+  /* Go thru the elements of an inverted list for a term */
+  /* May not go sequentially if skipping is done */
+  while (i < ft && ((op == op_term) || src - L->DE < L->num))
+    {
+      unsigned long diff, Temp;
+
+      if (have_skipping)
+	{
+
+	  /* --- Skip to next block --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    {
+	    ShortCut:
+	      i = next_i;
+	      DECODE_SEEK (next_start);
+	      pos = next_start;
+	      CurrDoc = next_doc_num;
+	      qd->hops_taken++;
+	      block_counter = skip_size - 1;
+	    }
+
+	  /* --- Decode the skip block header --- */
+	  if (block_counter == skip_size - 1)
+	    {
+	      if (i + skip_size < ft)
+		{
+		  /* --- Decode the skip block header --- */
+		  int doc_diff, pos_diff;
+		  BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		  next_doc_num += doc_diff;
+		  next_i += skip_size;
+		  BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		  next_start = pos + pos_diff;
+		}
+	      else
+		next_doc_num = -1;
+	    }
+
+	  /* --- Do a skip if c-set doc is higher than block header doc --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    goto ShortCut;
+
+
+	}			/*if skipping */
+
+      /* --- increment CurrDoc --- */
+      if (have_skipping && block_counter == 0 && i != ft - 1)
+	CurrDoc = next_doc_num;
+      else
+	{
+	  BBLOCK_DECODE_L (diff, blk, pos);
+	  CurrDoc += diff;
+	}
+
+      /* --- If have f_dt's then eat them up --- */
+      if (qd->id->ifh.InvfLevel >= 2)
+	GAMMA_DECODE_L (Temp, pos);	/* discard word count */
+
+      /* --- Increment counters --- */
+      qd->num_of_ptrs++;
+      i++;
+      block_counter--;
+      if (block_counter < 0)	/* ??? Why would this happen ? */
+	block_counter = skip_size - 1;
+
+      switch (op)
+	{
+	case op_term:
+	  /* --- Add the doc from invf onto the c-set list --- */
+	  L->DE[L->num].DocNum = CurrDoc;
+	  L->DE[L->num].Weight = Weight;
+	  L->DE[L->num].SeekPos = 0;
+	  L->DE[L->num].Len = 0;
+	  L->DE[L->num].CompTextBuffer = NULL;
+	  L->DE[L->num].Next = NULL;
+	  L->DE[L->num].or_included = 0;
+	  L->num++;
+	  break;
+
+	case op_and_terms:
+	case op_and_or_terms:
+	  /* --- see if have a match and save it --- */
+
+	  while (src < (L->DE + L->num) && CurrDoc > src->DocNum)
+	    {
+	      src++;
+	    }
+	  /* Unless at end of list, assert(CurrDoc <= src->DocNum) */
+
+	  /* --- Accept the doc as a candidate --- */
+	  if (src < (L->DE + L->num) && CurrDoc == src->DocNum)
+	    {
+	      if (op == op_and_terms)
+		{
+		  *dest++ = *src;
+		}
+	      else
+		{
+		  src->or_included = 1;
+		}
+	      src++;		/* Move onto next one */
+	    }
+
+	  /* If the next one to try has already been tried then move on again */
+	  if (op == op_and_or_terms)
+	    {
+	      while (src < (L->DE + L->num) && src->or_included)
+		src++;
+	    }
+
+	  break;
+
+	case op_diff_terms:
+	  while (src < (L->DE + L->num) && src->DocNum < CurrDoc)
+	    *dest++ = *src++;
+	  if (src < (L->DE + L->num) && src->DocNum == CurrDoc)
+	    src++;
+	  break;
+
+	default:
+	  break;
+	}			/*switch */
+
+    }				/*while traversing invf */
+
+  if (op == op_and_terms || op == op_diff_terms)
+    {
+      if (op == op_diff_terms)
+	while (src - L->DE < L->num)
+	  *dest++ = *src++;
+
+      /* update size of c-set */
+      L->num = dest - L->DE;
+    }
+  DECODE_DONE
+
+    Xfree (buffer);
+  ChangeMemInUse (qd, -we->invf_len);
+
+  return (L);
+}
+
+
+
+
+
+
+
+float *
+CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi)
+{
+  float *AccumulatedWeights;
+  int j, num_docs, num_terms;
+  int seen_must_match = 0; /* [RJM 07/97: Ranked Required Terms] says whether to 'and' queries or not */
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  if (!(AccumulatedWeights = Xmalloc (num_docs * sizeof (float))))
+      return (NULL);
+
+  bzero ((char *) AccumulatedWeights, sizeof (float) * num_docs);
+  ChangeMemInUse (qd, sizeof (float) * num_docs);
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      u_char *buffer;
+      int i, kd;
+      unsigned long next_mjr_dn = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long TempDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Terms] */
+      WordEntry *we = &Terms->TE[j].WE;
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      kd = 0;
+      for (i = 0; i < p; i++, kd++)
+	{
+	  int Count;		/* The number of times the word occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  if (kd == k)
+	    {
+	      kd = 0;
+	    }
+	  if (k && kd == 0)
+	    if (i + k < p)
+	      {
+		int temp;
+		BBLOCK_DECODE (next_mjr_dn, dn_blk);
+		next_mjr_dn += CurrDocNum;
+		BBLOCK_DECODE (temp, len_blk);
+	      }
+	  if (k && kd == k - 1 && i != p - 1)
+	    CurrDocNum = next_mjr_dn;
+	  else
+	    {
+	      BBLOCK_DECODE (diff, blk);
+	      CurrDocNum += diff;
+	    }
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE (Count);
+	  else
+	    Count = (double) we->count / p;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* zero the weights of intermediate documents if this term is required to match */
+	  if (require_match == 1) {
+	    for (TempDocNum = LastDocNum+1; TempDocNum < CurrDocNum; TempDocNum++)
+	      AccumulatedWeights[TempDocNum - 1] = 0.0;
+	  }
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if a term has been required to match before, the weight we are */
+	  /* adding to must be non-zero */
+	  if (!seen_must_match || (AccumulatedWeights[CurrDocNum - 1] > 0.0)) {
+	      AccumulatedWeights[CurrDocNum - 1] += Wqt * Wdt;
+	  }
+
+	  qd->num_of_ptrs++;
+	  LastDocNum = CurrDocNum; /* [RJM 07/97: Ranked Required Terms] */
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* zero the remaining weights if this term is required to match */
+      if (require_match == 1) {
+	for (TempDocNum = LastDocNum+1; TempDocNum <= num_docs; TempDocNum++)
+	  AccumulatedWeights[TempDocNum - 1] = 0.0;
+	seen_must_match = 1;
+      }
+    }
+  return AccumulatedWeights;
+}
+
+
+
+static int 
+doc_comp (void *a, void *b)
+{
+  return ((Invf_Doc_Entry *) a)->DocNum - ((Invf_Doc_Entry *) b)->DocNum;
+}
+
+
+static Invf_Doc_Entry *
+get_ide (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  struct pool *p = pool->pool;
+  if (!p || p->pos == p->length)
+    {
+      register unsigned size = sizeof (struct pool) +
+      (pool->pool_chunks - 1) * sizeof (Invf_Doc_Entry);
+      p = malloc (size);
+      if (!p)
+	return NULL;
+      ChangeMemInUse (qd, size);
+      p->length = pool->pool_chunks;
+      p->pos = 0;
+      p->next = pool->pool;
+      pool->pool = p;
+    }
+  return &(p->ide[p->pos++]);
+}
+
+
+void 
+free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  while (pool->pool)
+    {
+      struct pool *p = pool->pool;
+      pool->pool = p->next;
+      ChangeMemInUse (qd, -(sizeof (struct pool) + (p->length - 1) *
+			    sizeof (Invf_Doc_Entry)));
+      free (p);
+    }
+  pool->pool = NULL;
+}
+
+
+FILE *
+open_skip (char *sd)
+{
+  char buf[256];
+  if (sd == NULL || *sd == '\0')
+    return NULL;
+  sprintf (buf, sd, getpid ());
+  return fopen (buf, "a");
+}
+
+
+
+Splay_Tree *
+CosineDecodeSplay (query_data * qd, TermList * Terms,
+		   RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool)
+{
+  Splay_Tree *Doc_Set;
+  int j, num_docs, num_terms, MoreAccum;
+  int Anding = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  Doc_Set = SP_createset (doc_comp);
+
+  pool->pool = NULL;
+  pool->pool_chunks = 1024;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos, i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && Doc_Set->no_of_items >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  next = SP_get_first (Doc_Set);
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry ent, *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->DocNum < CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      if (next && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->DocNum <= CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      goto NextDocNum;
+	    }
+	  ent.DocNum = CurrDocNum - 1;
+	  if ((mem = SP_member (&ent, Doc_Set)) == NULL)
+	    {
+	      hits++;
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      if ((mem = get_ide (qd, pool)) == NULL)
+		return NULL;
+	      ChangeMemInUse (qd, sizeof (*mem));
+	      mem->DocNum = CurrDocNum - 1;
+	      mem->Sum = 0;
+	      ChangeMemInUse (qd, -Doc_Set->mem_in_use);
+	      if (SP_insert (mem, Doc_Set) == NULL)
+		{
+		  Message ("Unable to allocate memory for a splay node");
+		  Abort = 1;
+		  goto FastAbort;
+		}
+	      ChangeMemInUse (qd, Doc_Set->mem_in_use);
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+
+	  mem->Sum += Wqt * Wdt;
+	  total_added += Wqt * Wdt;
+
+	NextDocNum:
+	  ;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, Doc_Set->no_of_items, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  return Doc_Set;
+}
+
+
+
+
+Hash_Table *
+CosineDecodeHash (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  Hash_Table *HT;
+  int Anding = 0;
+  int Sorted = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  HT = HT_create (qd, rqi->MaxAccums, rqi->HashTblSize);
+  if (!HT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_EntryH *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  if (!Sorted)
+	    {
+	      HT_sort (qd, HT);
+	      Sorted = 1;
+	    }
+	  next = HT->Head;
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+#ifdef SHORT_CUT
+		ShortCut:
+#endif
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		{
+		  if (i + k < p)
+		    {
+		      int doc_diff, pos_diff;
+		      BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		      next_doc_num += doc_diff;
+		      next_i += k;
+		      BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		      next_start = pos + pos_diff;
+		    }
+		  else
+		    next_doc_num = -1;
+		}
+#ifdef SHORT_CUT
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		goto ShortCut;
+#else
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+#endif
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    {
+	      CurrDocNum = next_doc_num;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->IDE.DocNum < CurrDocNum - 1)
+		next = next->next;
+	      if (next && next->IDE.DocNum == CurrDocNum - 1)
+		{
+		  Wdt = Count * WordLog;
+		  next->IDE.Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->IDE.DocNum <= CurrDocNum - 1)
+		next = next->next;
+	      goto NextDocNum;
+	    }
+	  if ((mem = HT_find (HT, CurrDocNum - 1)) == NULL)
+	    {
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      Wdt = Count * WordLog;
+	      if (!HT_insert (HT, CurrDocNum - 1, Wqt * Wdt))
+		{
+		  if (!HT->Suplimentary_Entries)
+		    HT_Sup_create (qd, HT, we->doc_count - i + 1);
+		  HT_Sup_Add (HT, CurrDocNum - 1, Wqt * Wdt);
+		}
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+	  else
+	    {
+	      Wdt = Count * WordLog;
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+	  ;
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+	       ptrs_dec, hits, HT->num + HT->Suplimentary_Num, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  if (!HT->Head)
+    HT_sort (qd, HT);
+  return HT;
+}
+
+
+
+List_Table *
+CosineDecodeList (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  List_Table *LT;
+  int Anding = 0;
+  int Sorted = 0;
+  int has_grown = 0; /* [RJM 07/97: Ranked Required Match] whether elements have been added lately */
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+
+  /* Create the list for the documents */
+  LT = LT_create (qd, rqi->MaxAccums);
+  if (!LT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i, n = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0;	/* [RJM 07/97: Ranked Required Match] */
+      int this_item = 0; /* [RJM 07/97: Ranked Required Match] points to the next possible LT to delete */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Match] */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums)
+	MoreAccum = 0;
+
+      if ((rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums) || Anding)
+	{
+	  Anding = 1;
+	  if (!Sorted || has_grown)
+	    {
+	      LT_sort (LT);
+	      LT_pack (LT);
+	      Sorted = 1;
+	      has_grown = 0;
+	    }
+	  next = LT->IDE;
+	  n = 0;
+	}
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* make sure that the document list is set up correctly for */
+      /* requiring a term to match */
+      if (require_match == 1) {
+	if (!Sorted || has_grown) {
+	  LT_sort (LT);
+	  LT_pack (LT);
+	  Sorted = 1;
+	  has_grown = 0;
+	}
+	/* this_item is already set to zero */
+      }
+
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || n < LT->num))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+
+	  /* deal with skipping */
+	  if (k)
+	    {
+	      /* see if we should take a skip */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+
+	      /* should we take another skip? */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if this is a required match we must remove all accumulators */
+	  /* of intermediate documents */
+	  if (require_match == 1) {
+	    while (this_item < LT->num && LT->IDE[this_item].DocNum < CurrDocNum - 1) {
+	      if (LastDocNum==0 ||(LT->IDE[this_item].DocNum > LastDocNum - 1))
+		LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	      this_item++;
+	    }
+	  }
+
+	  if (Anding)
+	    {
+	      while (n < LT->num && next->DocNum < CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      if (n < LT->num && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (n < LT->num && next->DocNum <= CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      goto NextDocNum;
+	    }
+	  if (!Anding)
+	    {
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      LT = LT_add (qd, LT, CurrDocNum - 1, Wqt * Wdt);
+	      if (!LT)
+		{
+		  Abort = 1;
+		  goto FastAbort;
+		}
+
+	      /* the list of accumulators just got bigger */
+	      has_grown = 1;
+
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+
+
+	    }
+	  else if ((mem = LT_find (LT, CurrDocNum - 1)))
+	    {
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+
+	  LastDocNum = CurrDocNum;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, LT->num, total_added);
+	}
+      if (Abort)
+	break;
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* remove the remaining accumulators if this term is required to match */
+      if (require_match == 1) {
+	while (this_item < LT->num) {
+	  if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	    LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	  this_item++;
+	}
+	Anding = 1; /* no more documents should be added */
+
+	/* Get rid of any documents which have been deleted */
+	LT_pack (LT);
+      }
+    }
+
+  if (sk)
+    fclose (sk);
+  return LT;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/invf_get.h	(revision 21112)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * invf_get.h -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF_GET
+#define H_INVF_GET
+
+typedef struct Invf_Doc_Entry
+  {
+    int DocNum;
+    float Sum;
+  }
+Invf_Doc_Entry;
+
+typedef struct Invf_Doc_Entry_Pool
+  {
+    void *pool;
+    unsigned pool_chunks;
+  }
+Invf_Doc_Entry_Pool;
+
+typedef struct Invf_Doc_EntryH
+  {
+    Invf_Doc_Entry IDE;
+    struct Invf_Doc_EntryH *next;
+  }
+Invf_Doc_EntryH;
+
+typedef struct Hash_Table
+  {
+    Invf_Doc_EntryH *Suplimentary_Entries;
+    int Suplimentary_Size;
+    int Suplimentary_Num;
+    unsigned max;
+    unsigned num;
+    unsigned size;
+    Invf_Doc_EntryH **HashTable;
+    Invf_Doc_EntryH *Head;
+    Invf_Doc_EntryH IDEH[1];
+  }
+Hash_Table;
+
+typedef struct List_Table
+  {
+    unsigned max;
+    unsigned num;
+    Invf_Doc_Entry IDE[1];
+  }
+List_Table;
+
+typedef enum
+  {
+    op_term, op_and_terms, op_and_or_terms, op_diff_terms
+  }
+Op_types;
+
+/*Invf_Doc_Entry *HT_find(Hash_Table *HT, unsigned long DN); */
+void HT_free (query_data * qd, Hash_Table * HT);
+void LT_free (query_data * qd, List_Table * LT);
+
+invf_data *InitInvfFile (File * InvfFile, stemmed_dict * sd);
+void FreeInvfData (invf_data * id);
+
+DocList *GetDocs (query_data * qd, WordEntry * we, BooleanQueryInfo * bqi);
+
+float *CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi);
+
+void free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool);
+
+Splay_Tree *CosineDecodeSplay (query_data * qd, TermList * Terms,
+			 RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool);
+
+Hash_Table *CosineDecodeHash (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+List_Table *CosineDecodeList (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+
+
+DocList *GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass1.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass1.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass1.c	(revision 21112)
@@ -0,0 +1,822 @@
+/**************************************************************************
+ *
+ * ivf.pass1.c -- Memory efficient pass 1 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:50  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.6  1995/01/16  03:57:05  tes
+   * Fixed bug for index_string_bytes which was calculated
+   * by adding the suffix lengths except for every lookback (block)
+   * number of words. This was incorrect as invf.dict is fully
+   * front coded - perhaps it was partially coded in the past.
+   * This would only affect a statistic - hence the reason why the
+   * bug could exist ;-)
+   *
+   * Revision 1.5  1994/11/29  00:31:59  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:48  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:34  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long bit counts for inverted file
+ *     (1999-08-05 Tim Bell <bhat@cs.mu.oz.au>)
+ */
+static char *RCSID = "$Id$";
+
+#define LOGLOOKBACK	2
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define PARA_DIV 1.5
+#define NORM_DIV 1.0
+
+
+typedef struct hash_rec
+  {
+    unsigned long fcnt;		/* fragment frequency */
+    unsigned long lfcnt;	/* local fragment frequency */
+    unsigned long fnum;		/* last fragment to use stem */
+    unsigned long wcnt;		/* stem frequency */
+    unsigned long lwcnt;	/* local stem frequency */
+    u_char *word;
+  }
+hash_rec;
+
+
+
+
+
+
+
+
+static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0;
+static unsigned long outputbytes = 0;
+static unsigned long inputbytes = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static unsigned long ChunksWritten = 0;
+
+
+static FILE *ic;		/* The invf block file */
+static stdio_bitio_state sbs;
+
+static hash_rec **HashTable;
+static unsigned long HashSize;
+static unsigned long HashUsed;
+static u_char *Pool;
+static int PoolLeft;
+
+static hash_rec *hr_pool;
+static int hr_PoolLeft;
+
+static hash_rec **first_occr;
+static int max_first_occr;
+
+static mg_ullong L1_bits = 0;
+static mg_ullong L2_bits = 0;
+static mg_ullong L3_bits = 0;
+static unsigned long L1_ohead = 0;
+static unsigned long L2_ohead = 0;
+static unsigned long L3_ohead = 0;
+static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0;
+static unsigned long ptrcnt = 0;
+static unsigned long checknum;
+static long max_mem = 0;
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+int 
+init_ivf_1 (char *file_name)
+{
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fwrite ("    ", sizeof (u_long), 1, ic);	/* Space for the maxmem */
+  ENCODE_START (ic)
+    ENCODE_PAUSE (sbs)
+
+    if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (hash_rec))))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+
+  HashSize = INITIAL_HASH_SIZE;
+  HashUsed = 0;
+  if (!(HashTable = Xmalloc (sizeof (hash_rec *) * HashSize)))
+    {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * HashSize);
+  bzero ((char *) HashTable, sizeof (hash_rec *) * HashSize);
+  max_first_occr = 8192;
+  if (!(first_occr = Xmalloc (sizeof (hash_rec *) * max_first_occr)))
+    {
+      Message ("Unable to allocate memory for first_occur");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * max_first_occr);
+
+  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+
+  return (COMPALLOK);
+}
+
+
+
+static unsigned long 
+mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? lwordnum : lcallnum; */
+  register unsigned long N = lcallnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->lwcnt : ent->lfcnt; */
+      register unsigned long p = ent->lfcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->lwcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+static unsigned long 
+max_mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? wordnum : callnum; */
+  register unsigned long N = callnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->wcnt : ent->fcnt; */
+      register unsigned long p = ent->fcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->wcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+static void 
+dump_dict (unsigned long mem)
+{
+  int i;
+
+  ChunksWritten++;
+
+  ENCODE_CONTINUE (sbs)
+
+    GAMMA_ENCODE (lcallnum + 1);
+  if (mem > max_mem)
+    max_mem = mem;
+  GAMMA_ENCODE (mem + 1);
+  lwordnum = lcallnum = 0;
+  GAMMA_ENCODE (HashUsed + 1);
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *ent = first_occr[i];
+      GAMMA_ENCODE (ent->lwcnt + 1);
+      if (ent->lwcnt >= 2)
+	GAMMA_ENCODE (ent->lfcnt);
+      ent->lwcnt = ent->lfcnt = 0;
+    }
+  ptrcnt = 0;
+  ENCODE_PAUSE (sbs)
+}
+
+
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  u_char *end = s_in + l_in - 1;
+
+  callnum++;
+  lcallnum++;
+  inputbytes += l_in;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      
+      stemmer (stem_method, stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      words_read++;
+      wordnum++;
+      lwordnum++;
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = HashTable[hashval];
+	    if (!ent)
+	      {
+		int len = *Word + 1;
+		if (!hr_PoolLeft)
+		  {
+		    if (!(hr_pool = Xmalloc (HASH_POOL_SIZE *
+					     sizeof (hash_rec))))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    hr_PoolLeft = HASH_POOL_SIZE;
+		    ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+		  }
+		ent = hr_pool++;
+		hr_PoolLeft--;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->wcnt = 1;
+		ent->fcnt = 1;
+		ent->lwcnt = 1;
+		ent->lfcnt = 1;
+		ent->fnum = callnum;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		if (HashUsed == max_first_occr - 1)
+		  {
+		    ChangeMem (-sizeof (hash_rec *) * max_first_occr);
+		    max_first_occr *= 2;
+		    if (!(first_occr = Xrealloc (first_occr, sizeof (hash_rec *) *
+						 max_first_occr)))
+		      {
+			Message ("Unable to allocate memory for first_occr");
+			return (COMPERROR);
+		      }
+		    ChangeMem (sizeof (hash_rec *) * max_first_occr);
+		  }
+		first_occr[HashUsed] = ent;
+		HashUsed++;
+		HashTable[hashval] = ent;
+		bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		ent->lwcnt++;
+		if (callnum > ent->fnum)
+		  {
+		    ptrcnt++;
+		    ent->fcnt++;
+		    ent->lfcnt++;
+		    ent->fnum = callnum;
+		  }
+		break;
+	      }
+	  }
+      }
+
+      if (HashUsed >= HashSize >> 1)
+	{
+	  hash_rec **ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec *) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  bzero ((char *) ht, sizeof (hash_rec *) * size);
+	  ChangeMem (sizeof (hash_rec *) * size);
+
+	  for (i = 0; i < HashSize; i++)
+	    if (HashTable[i])
+	      {
+		register u_char *wptr;
+		hash_rec *ent;
+		register unsigned long hashval, step;
+
+		wptr = HashTable[i]->word;
+		HASH (hashval, step, wptr, size);
+		ent = ht[hashval];
+		while (ent)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    ent = ht[hashval];
+		  }
+		ht[hashval] = HashTable[i];
+	      }
+	  Xfree (HashTable);
+	  ChangeMem (-sizeof (hash_rec *) * HashSize);
+	  HashTable = ht;
+	  HashSize = size;
+	}
+
+    }
+
+  if (ptrcnt >= checknum)
+    {
+      unsigned long mem;
+      /*fprintf(stderr, "Checking at %u . . . ", ptrcnt); */
+      mem = mem_reqd ();
+      if (mem >= invf_buffer_size * CHECK_CLOSE)
+	{
+	  /*fprintf(stderr, "Got a match\n"); */
+	  dump_dict (mem);
+	  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+	    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+	}
+      else
+	{
+	  checknum = checknum * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	    checknum;
+	  if (checknum <= ptrcnt)
+	    checknum = ptrcnt + 1;
+	  /*fprintf(stderr, "Next check at %u\n", checknum); */
+	}
+    }
+  return (COMPALLOK);
+
+}				/* encode */
+
+
+int 
+process_ivf_1 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	return process_doc (start, pos + s_in - start);
+    }
+  return COMPALLOK;
+}
+
+static int 
+PackHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < HashSize; s++)
+    if (HashTable[s])
+      HashTable[d++] = HashTable[s];
+  ChangeMem (-sizeof (hash_rec *) * HashSize);
+  ChangeMem (sizeof (hash_rec *) * HashUsed);
+  if (!(HashTable = Xrealloc (HashTable, sizeof (hash_rec *) * HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  HashSize = HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((hash_rec **) A))->word;
+  u_char *s2 = (*((hash_rec **) B))->word;
+
+  return (casecompare (s1, s2)); /* [RPAP - Jan 97: Stem Index Change] */
+}				/* stem_comp */
+
+
+
+
+
+/*
+ * void count_text()
+ *
+ * The maths used in this function is described in the paper "Coding for 
+ * Compression in Full-Text Retrieval Systems"
+ *
+ */
+static void 
+count_text ()
+{
+  int i;
+  mg_ullong oldL12_bits = 0;
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *wrd = HashTable[i];
+      /* estimate size of a level 1 inverted file */
+      L1_bits += (int) (0.99 + wrd->fcnt * (1.6 + log2 (1.0 * callnum / wrd->fcnt)));
+      L1_ohead += BIO_Gamma_Length (wrd->fcnt);
+
+      L2_bits += BIO_Unary_Length (wrd->wcnt);
+      L2_ohead += BIO_Gamma_Length (wrd->wcnt - wrd->fcnt + 1);
+
+      L3_bits += (int) (0.99 + wrd->wcnt *
+		  (1.6 + log2 (1.0 * words_read / (wrd->wcnt + callnum))));
+      L3_ohead += 0;
+
+      /* check for overflow */
+      if (L1_bits + L2_bits < oldL12_bits) {
+	fprintf(stderr, "ERROR: Inverted file size will probably overflow %d byte unsigned integer\n", sizeof (mg_ullong));
+	fprintf(stderr, "       counter in pass 2.\n");
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+      oldL12_bits = L1_bits + L2_bits;
+    }
+  L3_bits = (L3_bits + L2_bits + L1_bits + 7) / 8;
+  L3_ohead = (L3_ohead + L2_ohead + L1_ohead + 7) / 8;
+  L2_bits = (L2_bits + L1_bits + 7) / 8;
+  L2_ohead = (L2_ohead + L1_ohead + 7) / 8;
+  L1_bits = (L1_bits + 7) / 8;
+  L1_ohead = (L1_ohead + 7) / 8;
+}				/* count_text */
+
+
+
+/*
+ *    write_stem_file():
+ *              writes out the stemmed dictionary file
+ *              in the following format 
+ *                      lookback value (int)
+ *                      totalbytes value (int)
+ *                      indexstringbytes (int)
+ *                      for each word 
+ *                        wordlen (4 bits)
+ *                        prefix match (4 bits)
+ *                        word (wordlen bytes)
+ *                        word frequency (int)
+ *                        word count (int)
+ *
+ *      Accesses outside variables:     
+ *
+ *      Return value...:                
+ */
+
+static void 
+write_stem_file (char *file_name)
+{
+  long j;
+  struct invf_dict_header idh;
+  long lookback = (1 << LOGLOOKBACK);	/* ???? */
+  long totalbytes = 0;		/* The sum of the length of all words, including 
+				   the length byte */
+  long indexstringbytes = 0;	/* The amount of space required to store the 
+				   words in the diction, this takes into account 
+				   the prefixes */
+  u_char *lastword = NULL;	/* A pointer to the last word processed */
+  FILE *sp;
+
+
+  /* Calculate the size of certain things */
+  for (j = 0; j < HashSize; j++)
+    {
+      u_char *word = HashTable[j]->word;
+      indexstringbytes += word[0] + 2;
+      totalbytes += word[0] + 1;
+      if (lastword)
+	indexstringbytes -= prefixlen (lastword, word);
+      lastword = word;
+    }
+
+  lastword = NULL;
+
+  if (!(sp = create_file (file_name, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(lookback, idh.lookback);
+  HTONUL2(HashSize, idh.dict_size);
+  HTONUL2(totalbytes, idh.total_bytes);
+  HTONUL2(indexstringbytes, idh.index_string_bytes);
+  HTOND2(inputbytes, idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL2(callnum, idh.num_of_docs);
+  HTONUL2(callnum, idh.static_num_of_docs);
+  HTONUL2(words_read, idh.num_of_words);
+  HTONUL2(stemmer_num, idh.stemmer_num);
+  HTONUL2(stem_method, idh.stem_method);
+
+  fwrite ((char *) &idh, sizeof (idh), 1, sp);
+  outputbytes += sizeof (idh);
+
+  for (j = 0; j < HashSize; j++)
+    {
+      int i;
+      unsigned long wcnt, fcnt;  /* [RPAP - Jan 97: Endian Ordering] */
+      hash_rec *ent = HashTable[j];
+      if (lastword != NULL)
+	/* look for prefix match with prev string */
+	i = prefixlen (lastword, ent->word);
+      else
+	i = 0;
+      fputc (i, sp);
+      fputc (ent->word[0] - i, sp);
+      fwrite ((char *) ent->word + i + 1, sizeof (u_char), ent->word[0] - i, sp);
+      outputbytes += ent->word[0] - i + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->fcnt, fcnt);
+      fwrite ((char *) &(fcnt), sizeof (fcnt), 1, sp);
+
+      outputbytes += sizeof (ent->fcnt);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->wcnt, wcnt);
+      fwrite ((char *) &(wcnt), sizeof (wcnt), 1, sp);
+
+      outputbytes += sizeof (ent->wcnt);
+      lastword = ent->word;
+    }
+
+  fclose (sp);
+}				/* write_stem_file() */
+
+
+
+/*
+ *    write_codes():
+ *              calls functions to assign and write out codes and
+ *              then prints out stats about execution results.
+ *
+ *      Accesses outside variables:     z
+ *
+ *      Return value...:                z
+ */
+
+static void 
+write_codes (char *file_name)
+{
+  unsigned long dicts = 0;
+  outputbytes = 0;
+  write_stem_file (file_name);
+  dicts = outputbytes;
+
+#ifndef SILENT
+  Message ("Chunks written      : %d\n",
+	   ChunksWritten);
+  if (InvfLevel == 3)
+    Message ("Paragraphs          : %8d\n", callnum);
+  Message ("Peak memory usage   : %10.1f Mb\n",
+	   (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("Stems size          : %10.1f kB  %5.2f%%\n",
+	   (double) dicts / 1024, (double) 100 * dicts / inputbytes);
+  Message ("Lev 1 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L1_bits + L1_ohead) / 1024,
+	   (double) 100 * (L1_bits + L1_ohead) / inputbytes);
+  Message ("Lev 2 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L2_bits + L2_ohead) / 1024,
+	   (double) 100 * (L2_bits + L2_ohead) / inputbytes);
+  Message ("Lev 3 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L3_bits + L3_ohead) / 1024,
+	   (double) 100 * (L3_bits + L3_ohead) / inputbytes);
+  Message ("Record addresses    : %10.1f kB  %5.2f%%\n",
+	   (double) words_diff * 4 / 1024, (double) 100 * words_diff * 4 / inputbytes);
+#endif
+}				/* write_codes() */
+
+
+
+void 
+write_num_file (char *file_name)
+{
+
+  int i;
+  FILE *f;
+
+  for (i = 0; i < HashSize; i++)
+    HashTable[i]->fnum = i;
+
+  if (!(f = create_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+#if 1
+  ENCODE_START (f)
+
+    for (i = 0; i < HashSize; i++)
+    BINARY_ENCODE (first_occr[i]->fnum + 1, HashSize + 1);
+
+  ENCODE_DONE;
+#else
+  for (i = 0; i < HashSize; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      unsigned long fnum;
+      HTONUL2(first_occur[i]->fnum, fnum);
+      fwrite ((char *) &fnum, sizeof (unsigned long), 1, f);
+    }
+#endif
+
+  fclose (f);
+}
+
+
+
+
+int 
+done_ivf_1 (char *FileName)
+{
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+#ifndef SILENT
+  Message ("Mem reqd for 1 chunk: %8u bytes\n",
+	   max_mem_reqd ());
+#endif
+  if (lcallnum)
+    dump_dict (mem_reqd ());
+
+  if (PackHashTable () == COMPERROR)
+    return COMPERROR;
+
+  qsort (HashTable, HashUsed, sizeof (hash_rec *), ent_comp);
+
+  count_text ();
+  write_codes (FileName);
+
+  ENCODE_CONTINUE (sbs)
+    GAMMA_ENCODE (1);
+  ENCODE_DONE
+    fseek (ic, sizeof (long), 0);
+  
+  HTONSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&max_mem, sizeof (max_mem), 1, ic);
+  NTOHSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (ic);
+
+  write_num_file (FileName);
+
+  msg_prefix = temp_str;
+
+  return (COMPALLOK);
+}				/* done_encode */
Index: /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass2.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass2.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/ivf.pass2.c	(revision 21112)
@@ -0,0 +1,1145 @@
+/**************************************************************************
+ *
+ * ivf.pass2.c -- Memory efficient pass 2 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+ * Revision 1.2  1997/08/02  05:01:57  wew
+ * changed literal values of 32 for the bit size of magic numbers of
+ * files to sizeof (unsigned long) * 8, increased the gap at the start
+ * of the invf during processing to 200 bytes
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:51  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:49  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:35  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long disk pointers and bit counts for inverted file
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ *     Code provided by Owen de Kretser <oldk@cs.mu.oz.au>
+ */
+
+static char *RCSID = "$Id$";
+
+#include "local_strings.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "stemmer.h"
+#include "perf_hash.h"
+#include "bitio_m.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include "bitio_random.h"
+#include "bitio_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "mg.h"
+#include "build.h"
+#include "words.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+#ifdef USE_LONG_LONG
+#define BIO_Random_Seek_X BIO_Random_Seek_LL
+#define BIO_Random_Tell_X BIO_Random_Tell_LL
+#else
+#define BIO_Random_Seek_X BIO_Random_Seek
+#define BIO_Random_Tell_X BIO_Random_Tell
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include <io.h>
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+/*#define RND_BUF_SIZE 128 */
+#endif
+
+#define print_fsize(file)\
+do\
+  {\
+    struct stat file_state;\
+    fstat(fileno(invf_out), &file_state);\
+    Message("len(invf) = %ld", file_state.st_size);\
+  }while(0)
+
+typedef struct word_rec
+  {
+    unsigned long ptr;
+    unsigned long last;
+  }
+word_rec;
+
+typedef struct invf_state_rec
+  {
+    mg_ullong Disk_Ptr;
+    mg_ullong Disk_Last;
+    unsigned long Disk_B;
+  }
+invf_state_rec;
+
+typedef struct chunk
+  {
+    unsigned long start_doc;
+    unsigned long params_pos;
+    unsigned long disk_pos;
+    unsigned long N;
+  }
+chunk;
+
+
+static FILE *dict;		/* Stemmed dictionary file */
+static FILE *hash;		/* Stemmed dictionary hash file */
+static FILE *invf;		/* Inverted file */
+static FILE *invf_in;		/* Inverted file */
+static FILE *invf_out;		/* Inverted file */
+static FILE *invf_idx;		/* Inverted index file */
+static FILE *count;		/* Count file */
+static FILE *count_trans;	/* Count translation file */
+static FILE *invf_state;	/* Inverted file State */
+static FILE *chunk_state;	/* Chunk state */
+static FILE *chunks;		/* Chunk state */
+static FILE *invf_para = NULL;	/* Paragraph counts file */
+static FILE *weights = NULL;	/* Weights file */
+
+static stdio_bitio_state sbs;
+static random_bitio_state crbs;
+static chunk *chunk_data = NULL;
+static random_bitio_state rbs, rbsp;
+
+static int docs_left = 0, next_docs_left = 0;
+static unsigned long N;
+
+static word_rec *WordRecs;
+static u_char *lg_bs;
+static float *idf = NULL;
+
+static char *MemoryBuffer = NULL;
+static unsigned long MemBufSize;
+static unsigned long BufToUse;
+static struct invf_dict_header idh;
+
+static perf_hash_data *phd;
+
+static unsigned long *word_list = NULL;
+static unsigned long wl_size = 0;
+
+static unsigned long dict_size;
+static unsigned long no_of_ptrs = 0;
+static unsigned long chunks_read = 0;
+static unsigned long Disk_pos = 0;
+static unsigned long callnum = 0;
+static unsigned long wordnum = 0;
+
+static unsigned long totalIbytes = 0;
+static unsigned long totalDbytes = 0;
+static unsigned long totalHbytes = 0;
+
+static unsigned long MemInUse = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long max_buffer_len;
+
+void 
+ChangeMemInUse (int mem)
+{
+  MemInUse += mem;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+static int 
+open_files (char *file_name)
+{
+  char FName[200];
+
+  if (!(dict = open_file (file_name, INVF_DICT_SUFFIX, "rb",
+			  MAGIC_STEM_BUILD, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(hash = open_file (file_name, INVF_DICT_HASH_SUFFIX, "rb",
+			  MAGIC_HASH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(count = open_file (file_name, INVF_CHUNK_SUFFIX, "rb",
+			   MAGIC_CHUNK, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fread (&max_buffer_len, sizeof (max_buffer_len), 1, count);
+  NTOHUL(max_buffer_len);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  BIO_Stdio_Decode_Start (count, &sbs);
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  if (!(count_trans = open_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "rb",
+				 MAGIC_CHUNK_TRANS, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(invf = create_file (file_name, INVF_SUFFIX, "w+b",
+			    MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fflush (invf);
+  if (!(invf_in = open_file (file_name, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  if (!(invf_out = create_file (file_name, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbs);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbsp);
+  ChangeMemInUse (RND_BUF_SIZE * 2);
+
+  if (!(invf_idx = create_file (file_name, INVF_IDX_SUFFIX, "wb",
+				MAGIC_INVI, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (InvfLevel == 3)
+    if (!(invf_para = create_file (file_name, INVF_PARAGRAPH_SUFFIX, "wb",
+				   MAGIC_PARAGRAPH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (COMPERROR);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".invf.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(invf_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunk.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunk_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+  BIO_Random_Start (chunk_state, RND_BUF_SIZE, &crbs);
+  ChangeMemInUse (RND_BUF_SIZE);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunks", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunks = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  return (COMPALLOK);
+}
+
+
+
+
+
+#define ISR_CACHE 1024
+#define ISR_ENTRY_SIZE (sizeof(mg_ullong)*2 + sizeof(unsigned long))
+
+invf_state_rec *
+in_cache (int pos)
+{
+  static char isr_data[ISR_CACHE * ISR_ENTRY_SIZE];
+  static invf_state_rec isr;
+  static int isr_base = 0, isr_num = -1, isr_pos = -1;
+  if (isr_pos >= 0)
+    bcopy ((char *) &isr, &isr_data[isr_pos * ISR_ENTRY_SIZE], ISR_ENTRY_SIZE);
+  if (pos < isr_base || pos >= isr_base + isr_num)
+    {
+      if (isr_num >= 0)
+	{
+	  fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+	  fwrite (isr_data, 1, ISR_ENTRY_SIZE * isr_num, invf_state);
+	}
+      isr_base = pos;
+      fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+      fread (isr_data, 1, ISR_ENTRY_SIZE * ISR_CACHE, invf_state);
+      isr_num = ISR_CACHE;
+    }
+  isr_pos = pos - isr_base;
+  bcopy (&isr_data[isr_pos * ISR_ENTRY_SIZE], (char *) &isr, ISR_ENTRY_SIZE);
+  return &isr;
+}
+
+
+
+
+
+unsigned long 
+occur_to_lexical (long occ)
+{
+  static long pos = -1;
+  static random_bitio_state rbs;
+  static int val = 0;
+  if (pos == -1)
+    {
+      BIO_Random_Start (count_trans, RND_BUF_SIZE, &rbs);
+      pos = 0x7fffffff;
+    }
+  if (occ < pos)
+    {
+      if (occ == -1)
+	{
+	  BIO_Random_Done (&rbs);
+	  return 0;
+	}
+      BIO_Random_Seek_X (sizeof (unsigned long) * 8, &rbs);
+      pos = 0;
+    }
+  while (pos <= occ)
+    {
+      val = BIO_Random_Binary_Decode (dict_size + 1, &rbs, NULL) - 1;
+      pos++;
+    }
+  return (val);
+}
+
+
+void 
+add_chunk_state (unsigned long pos, unsigned long start_doc,
+		 unsigned long N)
+{
+  chunk_data[chunks_read].params_pos = pos;
+  chunk_data[chunks_read].start_doc = start_doc;
+  chunk_data[chunks_read].N = N;
+  chunks_read++;
+}
+
+
+int 
+init_ivf_2 (char *file_name)
+{
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+  mg_ullong totalIbits;
+  mg_ullong lasttotalIbits;
+  double logN = 0.0;
+
+  
+  if (open_files (file_name) == COMPERROR)
+    return COMPERROR;
+
+
+  /* Read in the stemmed dictionary file header */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  dict_size = idh.dict_size;
+
+  N = idh.num_of_docs;
+
+  if (!(phd = read_perf_hash_data (hash)))
+    {
+      Message ("Unable to read in hash data");
+      return COMPERROR;
+    }
+  totalHbytes = sizeof (perf_hash_data) + sizeof (u_char) * 256 +
+    sizeof (int) * (phd->MAX_N + 1) + sizeof (int *) * 3 * phd->MAX_CH +
+    sizeof (long) * phd->MAX_CH * phd->MAX_L;
+
+  if (!(WordRecs = Xmalloc (sizeof (word_rec) * idh.dict_size)))
+    {
+      Message ("No memory for word entries");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (word_rec) * idh.dict_size;
+
+  /* separate storage for the log(b) values, one byte each */
+  if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size)))
+    {
+      Message ("No memory for lg b's");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (u_char) * idh.dict_size;
+
+  if (MakeWeights)
+    {
+      /* separate storage for the idf values, one single each */
+      if (!(idf = Xmalloc (sizeof (float) * idh.dict_size)))
+	{
+	  Message ("No memory for idf's");
+	  return COMPERROR;
+	}
+      totalDbytes += sizeof (float) * idh.dict_size;
+
+      if (!(weights = create_file (file_name, WEIGHTS_SUFFIX, "wb",
+				 MAGIC_WGHT, MG_CONTINUE))) { /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Couldn't open weights file for writing");
+	return (COMPERROR);
+      }
+    }
+  else
+    {
+      unlink (make_name (file_name, WEIGHTS_SUFFIX, NULL));
+    }
+
+  chunk_data = Xmalloc (sizeof (chunk) * (ChunkLimit + 2));
+  totalDbytes += sizeof (chunk) * (ChunkLimit + 2);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  if (MakeWeights)
+    {
+      wl_size = 1024;
+      if (!(word_list = Xmalloc (sizeof (*word_list) * wl_size)))
+	{
+	  Message ("No memory for word_list");
+	  return COMPERROR;
+	}
+
+      logN = log ((double) N);
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      invf_state_rec *isr;
+      register unsigned long copy, suff, p;
+      unsigned long fcnt, wcnt;
+
+      lasttotalIbits = totalIbits;
+      
+      copy = fgetc (dict);
+      suff = fgetc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      WordRecs[i].last = 0;
+      WordRecs[i].ptr = 0;
+
+      p = fcnt;
+
+      if (MakeWeights)
+	idf[i] = logN - log ((double) fcnt);
+
+
+      isr = in_cache (i);
+
+      isr->Disk_Last = 0;
+      isr->Disk_Ptr = totalIbits;
+
+      isr->Disk_B = BIO_Bblock_Init (N, p);
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+      if (totalIbits < lasttotalIbits) {
+	fprintf(stderr, "ERROR: The totalIbits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+    }
+
+
+  /* now convert to bytes, and actually get the space */
+#ifdef USE_LONG_LONG
+  totalIbytes = (totalIbits + 7ull) >> 3ull;
+#else
+  totalIbytes = (totalIbits + 7ul) >> 3ul;
+#endif
+
+  return (COMPALLOK);
+
+}
+
+
+
+
+
+static void 
+LoadCounts (void)
+{
+  unsigned long numwords, i, last_total;
+  static unsigned long local_N = 0;
+  unsigned long totalIbits, crbs_pos;
+  word_rec *wr;
+  unsigned long *counts;
+
+  if (MemoryBuffer == NULL)
+    {
+      MemBufSize = sizeof (unsigned long) * dict_size;
+      if (max_buffer_len > MemBufSize)
+	MemBufSize = max_buffer_len;
+      if (!(MemoryBuffer = Xmalloc (MemBufSize)))
+	FatalError (1, "Unable to allocate memory for buffer");
+      ChangeMemInUse (MemBufSize);
+    }
+
+  counts = (unsigned long *) MemoryBuffer;
+  bzero ((char *) counts, sizeof (unsigned long) * dict_size);
+
+  docs_left = next_docs_left;
+  if (!docs_left)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  BufToUse = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  numwords = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  local_N = docs_left;
+
+
+
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    wr->ptr = 0;
+
+  bzero ((char *) lg_bs, dict_size);
+
+  for (i = 0; i < numwords; i++)
+    {
+      unsigned long word_num, wcnt, fcnt, p;
+      word_num = occur_to_lexical (i);
+
+      wr = &WordRecs[word_num];
+
+      wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+      if (wcnt >= 2)
+	fcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL);
+      else
+	fcnt = wcnt;
+
+      p = fcnt;
+
+      if (wcnt)
+	{
+	  register unsigned long length;
+	  counts[word_num] = p;
+	  length = BIO_Bblock_Bound (local_N, p);
+	  if (InvfLevel >= 2)
+	    length += wcnt;
+	  wr->ptr = length;
+	  lg_bs[word_num] = floorlog_2 (BIO_Bblock_Init_W (local_N, p));
+	}
+
+    }
+
+  crbs_pos = BIO_Random_Tell (&crbs);
+
+  totalIbits = 0;
+  last_total = 0;
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    {
+      register unsigned long length;
+      length = wr->ptr;
+      wr->last = callnum;
+      BIO_Random_Gamma_Encode (counts[i] + 1, &crbs, NULL);
+      if (counts[i])
+	{
+	  if (i)
+	    BIO_Random_Delta_Encode (totalIbits - last_total + 1, &crbs, NULL);
+	  else
+	    BIO_Random_Delta_Encode (1, &crbs, NULL);
+
+	  last_total = totalIbits;
+	}
+      wr->ptr = totalIbits;
+      totalIbits += length;
+    }
+  add_chunk_state (crbs_pos, callnum, local_N);
+
+  if ((totalIbits + 7ul) >> 3ul > BufToUse)
+    FatalError (1, "Pointers exceed buffer size");
+
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+}
+
+
+
+
+static void 
+DumpChunk (void)
+{
+  chunk_data[chunks_read - 1].disk_pos = Disk_pos << 3;
+  fseek (chunks, Disk_pos, 0);
+  fwrite (MemoryBuffer, sizeof (char), BufToUse, chunks);
+  Disk_pos += BufToUse;
+}
+
+
+
+
+static void 
+DiskMerge (void)
+{
+  random_bitio_state *rbsi;
+  random_bitio_state *chks = NULL;
+  unsigned long *chunk_ptrs;
+  int i;
+
+  BIO_Random_Flush (&crbs);
+
+  chunk_ptrs = Xmalloc (chunks_read * sizeof (unsigned long));
+  ChangeMemInUse (chunks_read * sizeof (unsigned long));
+  bzero ((char *) chunk_ptrs, chunks_read * sizeof (unsigned long));
+
+  rbsi = Xmalloc (chunks_read * sizeof (random_bitio_state));
+  ChangeMemInUse (chunks_read * sizeof (random_bitio_state));
+  for (i = 0; i < chunks_read; i++)
+    {
+      rbsi[i] = crbs;
+      rbsi[i].Buf = Xmalloc (rbsi[i].len);
+      ChangeMemInUse (rbsi[i].len);
+      bcopy ((char *) (crbs.Buf), (char *) (rbsi[i].Buf), rbsi[i].len);
+      BIO_Random_Seek (chunk_data[i].params_pos, &rbsi[i]);
+    }
+
+  if (chunks_read > 1)
+    {
+      int j;
+      chks = Xmalloc ((chunks_read - 1) * sizeof (random_bitio_state));
+      ChangeMemInUse ((chunks_read - 1) * sizeof (random_bitio_state));
+      BIO_Random_Start (chunks, RND_BUF_SIZE, &chks[0]);
+      ChangeMemInUse (RND_BUF_SIZE);
+      for (j = 1; j < chunks_read - 1; j++)
+	{
+	  chks[j] = chks[0];
+	  chks[j].Buf = Xmalloc (chks[0].len);
+	  ChangeMemInUse (chks[0].len);
+	  bcopy ((char *) (chks[0].Buf), (char *) (chks[j].Buf), chks[0].len);
+	}
+    }
+  for (i = 0; i < dict_size; i++)
+    {
+      int j;
+      invf_state_rec *isr = in_cache (i);
+      register int B;
+
+      BIO_Random_Seek_X (isr->Disk_Ptr, &rbs);	/* Position in invf file */
+
+      B = isr->Disk_B;
+
+      for (j = 0; j < chunks_read; j++)
+	{
+	  int p;
+	  p = BIO_Random_Gamma_Decode (&rbsi[j], NULL) - 1;
+
+	  if (p)
+	    {
+	      int ptr, b;
+	      chunk_ptrs[j] += BIO_Random_Delta_Decode (&rbsi[j], NULL) - 1;
+	      ptr = chunk_ptrs[j];
+	      b = 1 << floorlog_2 (BIO_Bblock_Init_W (chunk_data[j].N, p));
+
+	      if (j == chunks_read - 1)
+		{
+		  int k, CurrDoc;
+		  DECODE_START ((u_char *) MemoryBuffer, ptr)
+		    CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      BBLOCK_DECODE (x, b);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  UNARY_DECODE (tf);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  DECODE_DONE
+		    isr->Disk_Last = CurrDoc;
+		}
+	      else
+		{
+		  int k, CurrDoc;
+		  random_bitio_state *Chks = chks + j;
+		  BIO_Random_Seek (chunk_data[j].disk_pos + ptr, Chks);
+		  CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      x = BIO_Random_Bblock_Decode (b, Chks, NULL);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  tf = BIO_Random_Unary_Decode (Chks, NULL);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  isr->Disk_Last = CurrDoc;
+		}
+	    }
+	}
+
+      isr->Disk_Ptr = BIO_Random_Tell_X (&rbs);
+
+    }
+  if (chunks_read > 1)
+    {
+      int j;
+      for (j = 0; j < chunks_read - 1; j++)
+	{
+	  Xfree (chks[j].Buf);
+	  ChangeMemInUse (-chks[j].len);
+	}
+      Xfree (chks);
+      ChangeMemInUse (-(chunks_read - 1) * sizeof (random_bitio_state));
+    }
+
+  for (i = 0; i < chunks_read; i++)
+    {
+      Xfree (rbsi[i].Buf);
+      ChangeMemInUse (-rbsi[i].len);
+    }
+  Xfree (rbsi);
+  ChangeMemInUse (-chunks_read * sizeof (random_bitio_state));
+  chunks_read = 0;
+  Xfree (chunk_ptrs);
+  ChangeMemInUse (-chunks_read * sizeof (unsigned long));
+  Disk_pos = 0;
+  BIO_Random_Seek (0, &crbs);
+}
+
+static void 
+MergeIn (void)
+{
+  static int disk_chunks = 0;
+  static header = 0;
+  if (!header)
+    {
+      fprintf (stderr, "ivf.pass2 : ");
+      header = 1;
+    }
+  if (disk_chunks == ChunkLimit || next_docs_left == 0)
+    {
+      fprintf (stderr, "M");
+      DiskMerge ();
+      disk_chunks = 0;
+    }
+  else
+    {
+      fprintf (stderr, "-");
+      DumpChunk ();
+      disk_chunks++;
+    }
+  if (next_docs_left == 0)
+    fprintf (stderr, "\n");
+}
+
+
+static int 
+wl_comp (const void *a, const void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  int res;
+  u_char *end = s_in + l_in - 1;
+  unsigned long tocode;
+  unsigned long wl_pos = 0;
+
+  if (!docs_left)
+    LoadCounts ();
+
+  callnum++;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      stemmer (idh.stem_method, idh.stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      if (*Word == 0)
+	continue;
+
+      res = perf_hash (phd, Word);
+
+      {
+	word_rec *arr = &WordRecs[res];
+	int b = 1 << lg_bs[res];
+	wordnum++;
+
+	tocode = callnum;
+
+	ENCODE_START ((u_char *) MemoryBuffer, arr->ptr)
+
+	  if (tocode > arr->last)
+	  {
+	    register int x;
+	    x = tocode - arr->last - 1;
+	    BBLOCK_ENCODE (x + 1, b);
+	    if (InvfLevel >= 2)
+	      ENCODE_BIT (1);
+	    no_of_ptrs++;
+	    arr->last = tocode;
+	  }
+	else if (InvfLevel >= 2)
+	  {
+	    __pos--;
+	    ENCODE_BIT (0);
+	    ENCODE_BIT (1);
+	  }
+	arr->ptr = __pos;
+	ENCODE_DONE
+      }
+
+      if (MakeWeights)
+	{
+	  if (wl_pos >= wl_size)
+	    {
+	      wl_size += (wl_size >> 1);
+	      word_list = Xrealloc (word_list, sizeof (*word_list) * wl_size);
+	    }
+	  word_list[wl_pos++] = res;
+	}
+    }
+  if (MakeWeights)
+    {
+      float doc_weight = 0.0;
+      if (wl_pos)
+	{
+	  unsigned long *wl = word_list;
+	  unsigned long i, count, val;
+	  qsort (wl, wl_pos, sizeof (*wl), wl_comp);
+	  count = 1;
+	  val = *wl++;
+	  for (i = 1; i <= wl_pos; i++, wl++)
+	    if (i == wl_pos || val != *wl)
+	      {
+		double weight = count * idf[val];
+		doc_weight += weight * weight;
+		count = 1;
+		val = *wl;
+	      }
+	    else
+	      count++;
+	}
+      HTONF(doc_weight);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &doc_weight, sizeof (doc_weight), 1, weights);
+    }
+  docs_left--;
+  if (!docs_left)
+    MergeIn ();
+
+  return COMPALLOK;
+}
+
+int 
+process_ivf_2 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int count = 0;
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	      count++;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	{
+	  if (process_doc (start, pos + s_in - start) != COMPALLOK)
+	    return (COMPERROR);
+	  count++;
+	}
+      HTONSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &count, sizeof (count), 1, invf_para);
+    }
+  return COMPALLOK;
+}
+
+
+
+
+
+static void 
+stats (unsigned long len)
+{
+#ifndef SILENT
+  fseek (count, 0, 2);
+  fseek (count_trans, 0, 2);
+  fseek (invf_state, 0, 2);
+  fseek (invf, 0, 0);
+  fseek (invf, 0, 2);
+  fseek (chunks, 0, 2);
+  fseek (chunk_state, 0, 2);
+  Message ("File sizes\n");
+  Message ("  Chunk desc    : %10u bytes\n", ftell (count));
+  Message ("  Chunk trans   : %10u bytes\n", ftell (count_trans));
+  Message ("  Chunks        : %10u bytes\n", ftell (chunks));
+  Message ("  Chunk state   : %10u bytes\n", ftell (chunk_state));
+  Message ("  Invf state    : %10u bytes\n", ftell (invf_state));
+  Message ("  Peak invf     : %10u bytes\n", len);
+  Message ("  Final invf    : %10u bytes\n", ftell (invf));
+  Message ("Peak disk usage : %10.2f %%\n",
+	   (double) (ftell (count) + ftell (count_trans) +
+		     ftell (invf_state) + ftell (chunks) +
+		     ftell (chunk_state) + len) / ftell (invf) * 100.0);
+#endif
+}
+
+
+/* ARGSUSED */
+int 
+done_ivf_2 (char *FileName)
+{
+  long i;
+  mg_ullong totalIbits;
+  unsigned long invf_len;
+  unsigned long bytes_output;
+  struct invf_file_header ifh;
+
+  if (weights)
+    fclose (weights);
+  if (invf_para)
+    fclose (invf_para);
+
+  free_perf_hash (phd);
+
+  free (MemoryBuffer);
+  ChangeMemInUse (-MemBufSize);
+
+  BIO_Random_Done (&rbs);
+  BIO_Random_Done (&rbsp);
+  fflush (invf);
+
+  fseek (invf, 0, 2);
+  invf_len = ftell (invf);
+
+  fseek (invf_out, sizeof (long), 0);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(dict_size, ifh.no_of_words);
+  HTONUL2(no_of_ptrs, ifh.no_of_ptrs);
+  ifh.skip_mode = 0;
+  bzero ((char *) ifh.params, sizeof (ifh.params));
+  HTONUL2(InvfLevel, ifh.InvfLevel);
+  fwrite ((char *) &ifh, sizeof (ifh), 1, invf_out);
+
+  bytes_output = ftell (invf_out);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  /* find the right place in the file to start reading p values */
+  fseek (dict, sizeof (unsigned long) + sizeof (struct invf_dict_header), 0);
+  for (i = 0; i < dict_size; i++)
+    {
+      invf_state_rec *isr;
+      unsigned long fcnt, wcnt, s, e;
+      register unsigned long p;
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+
+      /* output location to the invf_idx  */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      isr = in_cache (i);
+
+      e = (isr->Disk_Ptr + 7ul) >> 3ul;
+      s = totalIbits >> 3;
+
+      fseek (invf_in, s, 0);
+      while (s < e)
+	{
+	  u_char c = getc (invf_in);
+	  if (s == e - 1)
+	    {
+	      u_char ands[8] =
+	      {0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	      c &= ands[isr->Disk_Ptr & 7ul];
+	    }
+	  putc (c, invf_out);
+	  bytes_output++;
+	  s++;
+	}
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+    }
+
+  fclose (invf_in);
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (!(_chsize (_fileno (invf_out), bytes_output)))
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (invf_out), bytes_output);
+#endif
+
+  fclose (invf_out);
+
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (invf_idx);
+
+#ifndef SILENT
+  {
+    char *temp_str = msg_prefix;
+    unsigned long total;
+    msg_prefix = "ivf.pass2";
+    stats (invf_len);
+    Message ("Pass two data structures      : %6.3f Mbyte\n",
+	     (double) totalDbytes / 1024 / 1024);
+    total = totalDbytes;
+    Message ("Pass two hash structure(s)    : %6.3f Mbyte\n",
+	     (double) totalHbytes / 1024 / 1024);
+    total += totalHbytes;
+    Message ("Peak extra memory in use      : %6.3f Mbyte\n",
+	     (double) MaxMemInUse / 1024 / 1024);
+    total += MaxMemInUse;
+    Message ("Peak total memory in use      : %6.3f Mbyte\n",
+	     (double) total / 1024 / 1024);
+    msg_prefix = temp_str;
+  }
+#endif
+
+  Xfree (WordRecs);
+  Xfree (lg_bs);
+
+  /* Free the memory allocated for the BIO_Random */
+  occur_to_lexical (-1);
+
+  BIO_Random_Done (&crbs);
+
+  fclose (invf);
+  fclose (dict);
+  fclose (hash);
+  fclose (count);
+  fclose (count_trans);
+  fclose (chunk_state);
+  fclose (chunks);
+  fclose (invf_state);
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/lists.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/lists.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/lists.c	(revision 21112)
@@ -0,0 +1,214 @@
+/**************************************************************************
+ *
+ * lists.c -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+
+#include "lists.h"
+#include "local_strings.h"
+#include "locallib.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:01  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1999/07/01 09:30:48  rjmcnab
+   Changes for better match reporting.
+
+   Revision 1.1  1998/11/17 09:34:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:50  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:37  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/*
+ * Make a DocList large enough to hold num entries 
+ */
+DocList *
+MakeDocList (int num)
+{
+  DocList *d;
+  if (!(d = Xmalloc (sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  d->num = 0;
+  d->total_retrieved = 0;
+  d->is_approx = 0;
+  return (d);
+}
+
+
+/*
+ * Change the size of a DocList so that it can hold num entries 
+ */
+DocList *
+ResizeDocList (DocList * d, int num)
+{
+  if (!(d = Xrealloc (d, sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  return (d);
+}
+
+
+/* IntersectLists () */
+/* Returns the intersection of List1 and List2 by deleting from List1 all */
+/* entries not in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the intersection of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+IntersectLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List2 || !List1)
+    {
+      if (List1)
+	Xfree (List1);
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num && s2 - List2->DE < List2->num)
+    {
+      if (s1->DocNum == s2->DocNum)
+	*d1++ = *s1++;
+      if (s1 - List1->DE < List1->num)
+	while (s2 - List2->DE < List2->num && s2->DocNum < s1->DocNum)
+	  s2++;
+      if (s2 - List2->DE < List2->num)
+	while (s1 - List1->DE < List1->num && s1->DocNum < s2->DocNum)
+	  s1++;
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+
+/* DiffLists () */
+/* Returns the difference of List1 and List2 by deleting from List1 all */
+/* entries in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the differrence of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DiffLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num)
+    {
+      if (s2 - List2->DE == List2->num)
+	*d1++ = *s1++;
+      else
+	{
+	  if (s1->DocNum == s2->DocNum)
+	    {
+	      s1++;
+	      s2++;
+	    }
+	  else if (s1->DocNum < s2->DocNum)
+	    *d1++ = *s1++;
+	  else
+	    s2++;
+	}
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+/* MergeLists () */
+/* Finds the union of two lists and returns the head of the new list formed. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+MergeLists (DocList * List1, DocList * List2)
+{
+  DocList *NewList;
+  DocEntry *d, *s1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	return (List2);
+      else
+	return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  if (!(NewList = MakeDocList (List1->num + List2->num)))
+    {
+      Xfree (List1);
+      Xfree (List2);
+      return (NULL);
+    }
+
+  s1 = List1->DE;
+  s2 = List2->DE;
+  d = NewList->DE;
+  while (s1 - List1->DE < List1->num || s2 - List2->DE < List2->num)
+    {
+      if (s1 - List1->DE >= List1->num)
+	*d++ = *s2++;
+      else if (s2 - List2->DE >= List2->num)
+	*d++ = *s1++;
+      else if (s1->DocNum == s2->DocNum)
+	{
+	  *d = *s1++;
+	  (d++)->Weight += (s2++)->Weight;
+	}
+      else if (s1->DocNum < s2->DocNum)
+	*d++ = *s1++;
+      else
+	*d++ = *s2++;
+    }
+  NewList->num = d - NewList->DE;
+  Xfree (List1);
+  Xfree (List2);
+  return (NewList);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/lists.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/lists.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/lists.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * lists.h -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_LISTS
+#define H_LISTS
+
+
+
+typedef struct DocEntry
+  {
+    float Weight;
+    int DocNum;
+    unsigned long SeekPos;	/* position in the text file in bytes */
+    unsigned long Len;		/* length of the document in bytes */
+    char *CompTextBuffer;
+    struct DocEntry *Next;
+    short or_included;		/*[TS:Mar/95] whether included in an AND of ORed-terms */
+  }
+DocEntry;
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved; /* valid if > 0 */
+    int is_approx;       /* whether total_retrieved is approximate */
+    DocEntry DE[1];
+  }
+DocList;
+
+DocList *MakeDocList (int num);
+DocList *ResizeDocList (DocList * d, int num);
+DocList *IntersectLists (DocList * List1, DocList * List2);
+DocList *DiffLists (DocList * List1, DocList * List2);
+DocList *MergeLists (DocList * List1, DocList * List2);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/locallib.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/locallib.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/locallib.c	(revision 21112)
@@ -0,0 +1,304 @@
+/**************************************************************************
+ *
+ * locallib.c -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:02  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:51  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:38  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; i++)
+    total += A[i];
+  for (i = 0; i < n; i++)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; i--)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      n--;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  N++;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	limit++;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int 
+Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem,
+	  u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+	    u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (Fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
+
+
+int 
+F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (F_Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (Fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (Fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/locallib.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/locallib.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+int F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+		u_long * disk);
+
+int F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+
+
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg.special.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg.special.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg.special.c	(revision 21112)
@@ -0,0 +1,115 @@
+/**************************************************************************
+ *
+ * mg.special.c -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg.h"
+#include "build.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:03  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:53  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:44  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+FILE *fp;
+
+
+int 
+init_special (char *FileName)
+{
+  char FName[200];
+  sprintf (FName, "%s%s", FileName, ".docnums");
+  if (!(fp = fopen (FName, "w")))
+    return (COMPERROR);
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_special (u_char * s_in, int l_in)
+{
+  static int un = 1;
+  char *s_done = (char *) (s_in + l_in);
+  for (; (char *) s_in != s_done; s_in++)
+    if (*s_in == '<')
+      {
+	if (strncmp ((char *) s_in, "<DOCNO>", 7) == 0)
+	  {
+	    char *s_end = (char *) (s_in + 1);
+	    for (; s_end != s_done; s_end++)
+	      if (*s_end == '<')
+		if (strncmp (s_end, "</DOCNO>", 8) == 0)
+		  break;
+	    if (s_end == s_done)
+	      {
+		fprintf (fp, "NO DOC ID %d\n", un++);
+		return COMPALLOK;
+	      }
+	    s_in += 7;
+	    while (s_end != (char *) s_in &&
+		   (*(s_end - 1) == ' ' || *(s_end - 1) == '\n'))
+	      s_end--;
+	    while ((char *) s_in != s_end &&
+		   (*s_in == ' ' || *s_in == '\n'))
+	      s_in++;
+	    for (; (char *) s_in != s_end; s_in++)
+	      putc (*s_in, fp);
+	    putc ('\n', fp);
+	    return (COMPALLOK);
+	  }
+      }
+  fprintf (fp, "NO DOC ID %d\n", un++);
+  return (COMPALLOK);
+}
+
+
+
+
+
+
+
+
+int 
+done_special (char *FileName)
+{
+  fclose (fp);
+  return COMPALLOK;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_compression_dict.c	(revision 21112)
@@ -0,0 +1,1084 @@
+/**************************************************************************
+ *
+ * mg_compression_dict.c -- Routines for reading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.2  1999/12/10 15:04:25  cs025
+   Incorrect use of NTOHUL on a double value (num_bytes).
+
+   Revision 1.1  1999/08/10 21:18:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:52  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:54  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:45  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* #define DEBUG1 */
+
+/* #define DEBUG */
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct WordData
+  {
+    u_long freq, occur_num;
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+WordData;
+
+static WordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    WordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HBDYM")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'B':
+	novel_method = MG_NOVEL_BINARY;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'M':
+	novel_method = MG_NOVEL_HYBRID_MTF;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; i++)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      WordData *wd;
+      int j;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = Xmalloc (sizeof (WordData *) * Num[i] + 1);
+
+      buf = Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = Xmalloc (sizeof (WordData) * Num[i]);
+      for (j = 0; j < Num[i]; j++, wd++)
+	{
+	  int len;
+	  fread (&wd->freq, sizeof (wd->freq), 1, f);
+	  NTOHUL(wd->freq);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fread (&wd->occur_num, sizeof (wd->occur_num), 1, f);
+	  NTOHUL(wd->occur_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  int i;
+  WordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (WordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; i++, wd++)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; i++)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; i++)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->freq > bb->freq)
+    return -1;
+  else if (aa->freq < bb->freq)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->occur_num > bb->occur_num)
+    return 1;
+  else if (aa->occur_num < bb->occur_num)
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  WordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = Xmalloc (num * sizeof (WordData *));
+  for (i = 0; i < Num[0]; i++)
+    wd[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      WordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->freq, word->occur_num, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; i++)
+    {
+      WordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (WordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  bzero ((char *) char_freqs, sizeof (char_freqs));
+  bzero ((char *) len_freqs, sizeof (len_freqs));
+  for (i = 0; i < num; i++, wd++)
+    {
+      u_long freq = (*wd)->freq;
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; len--, buf++)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+
+
+
+
+void 
+CalcBitCost (WordData ** discard_word, int discard_num,
+	     WordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  int j;
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  for (j = 0; j < discard_num; j++, discard_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*discard_word)->word;
+      int len = *buf++;
+      u_long freq = (*discard_word)->freq;
+      int idx = KIND (*discard_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*discard_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*discard_word)->saving = ((*discard_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*discard_word)->word[0]);
+
+      (*discard_word)->num_trans = num_trans;
+    }
+  for (j = 0; j < keep_num; j++, keep_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*keep_word)->word;
+      int len = *buf++;
+      u_long freq = (*keep_word)->freq;
+      int idx = KIND (*keep_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*keep_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*keep_word)->saving = ((*keep_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*keep_word)->word[0]);
+
+      (*keep_word)->num_trans = num_trans;
+    }
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  WordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+  keep_num = 0;
+  keep_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+
+  for (i = 0; i < Num[0]; i++)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      WordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->freq;
+      num_trans++;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (keep_heap, sizeof (word), keep_num, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "DISCARD \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (discard_heap, sizeof (word),
+				discard_num, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      discard_heap[discard_num] = word;
+	      heap_additem (discard_heap, sizeof (word), &discard_num,
+			    BigSaving);
+	      freqs_trans[idx] -= word->freq;
+	      mem -= sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      keep_to_discard++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (discard_heap, sizeof (word),
+			       &discard_num, BigSaving);
+	      keep_heap[keep_num] = word;
+	      heap_additem (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      freqs_trans[idx] += word->freq;
+	      mem += sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      discard_to_keep++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "DISCARD -> KEEP %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  recalcs++;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; i++)
+    {
+      WordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; i++)
+    {
+      WordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  int i, us;
+  for (us = i = 0; i < dd->num_wds; i++)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = Xmalloc ((dd->num_wds) * sizeof (*freqs))))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      freqs[i] = dd->wd[i]->freq;
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  Xfree (freqs), freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  bzero ((char *) &huff_words_size, sizeof (huff_words_size));
+  bzero ((char *) &lencounts, sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      lencounts[codelen]++;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  Xfree (hd->clens);
+  Xfree (hd);
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  int j;
+  long freqs[256];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; len--, buf++)
+	freqs[(u_long) (*buf)] += (*wd)->freq;
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; j++)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  int j;
+  long freqs[16];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    freqs[(*wd)->word[0]] += (*wd)->freq;
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; j++)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  bzero((char *) &cdh, sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; i++)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < discard[i].num_wds; j++)
+		esc.freq += discard[i].wd[j]->freq;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < all[i].num_wds; j++)
+		if (all[i].wd[j]->freq == 1)
+		  esc.freq++;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_decompress_text.c	(revision 21112)
@@ -0,0 +1,167 @@
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "netorder.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "backend.h"  /* compressed_dict struct */
+#include "text_get.h"
+#include "words.h"
+
+#define EOD         '\002'
+#define EOP         '\003'
+
+/* These are defined in mgquery, and they need to be here for backend.o */
+/* Messy I know but this was meant to be a quick hack :-) */
+#if defined(PARADOCNUM) || defined(NZDL)
+int *Paragraph = NULL;
+int Documents = 0;
+#endif
+
+int
+main (int argc, char **argv)
+{
+  FILE *text, *text_idx, *para;
+  File *text_comp_dict = NULL, *text_aux_dict = NULL, *text_fast_comp_dict = NULL;
+
+  int ch;
+  unsigned long start, finish, len;
+  u_long total_chars = 0;
+  u_char *c_buffer = 0;
+  int buf_len = -1;
+  u_char *uc_buffer = 0;
+  int ULen;
+  compression_dict *cd = NULL;
+  compressed_text_header cth;
+  u_long doc;
+  int count;
+  char *filename = "";
+  char path[512];
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-d directory] -f name\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  /* Open files */
+  text = open_file (filename, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text);
+
+  text_idx = open_file (filename, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text_idx);
+
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  para = open_file (filename, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH, MG_CONTINUE);
+
+  sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_FAST_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(text_fast_comp_dict = Fopen (path, "rb", MAGIC_FAST_DICT)))
+    {
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_comp_dict = Fopen (path, "rb", MAGIC_DICT);
+      if (!text_comp_dict)
+	FatalError (1, "Could not open compressed dictionary");
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_AUX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_aux_dict = Fopen (path, "rb", MAGIC_AUX_DICT);
+    }
+
+  cd = LoadCompDict (text_comp_dict, text_aux_dict, text_fast_comp_dict);
+
+
+  /* Uncompress text */
+
+  doc = 1;
+  fread ((char *)&start, sizeof (start), 1, text_idx);
+  if (para)
+    fread ((char *)&count, sizeof (count), 1, para);
+  else
+    count = 1;
+
+  for (;;)
+    {
+      fread ((char *) &finish, sizeof (finish), 1, text_idx);
+      len = finish - start;
+      
+      if ((int) len > buf_len)
+	{
+	  if (c_buffer)
+	    {
+	      Xfree (c_buffer);
+	      Xfree (uc_buffer);
+	    }
+	  if (!(c_buffer = Xmalloc (len)))
+	    FatalError (1, "Cannot allocate memory for compressed buffer");
+	  if (!(uc_buffer = Xmalloc ((int) (cth.ratio * 1.01 * len) + 100)))
+	    FatalError (1, "Cannot allocate memory for uncompressed buffer");
+	  buf_len = len;
+	}
+
+      fread (c_buffer, 1, len, text);
+      DecodeText (cd, c_buffer, len, uc_buffer, &ULen);
+      fwrite (uc_buffer, ULen, sizeof (u_char), stdout);
+      fflush (stdout);
+
+      total_chars += ULen;
+
+      if (++doc > cth.num_of_docs)
+	{
+	  if (para)
+	    fputc (EOP, stdout);
+	  fputc (EOD, stdout);
+	  fflush (stdout);
+	  break;
+	}
+
+      if (!--count)
+	{
+	  /* End of document */
+	  fputc (EOD, stdout);
+	  if (para)
+	    fread ((char *)&count, sizeof (count), 1, para);
+	  else
+	    count = 1;
+	}
+      else
+	fputc (EOP, stdout);
+      fflush (stdout);
+      start = finish;
+    }
+
+  /* Close files */
+
+  fclose (text);
+  fclose (text_idx);
+  if (para)
+    fclose (para);
+
+#if 0
+  printf ("\n#Total chars output = %lu\n", total_chars);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.c	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * mg_errors.c -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
@@ -0,0 +1,798 @@
+/**************************************************************************
+ *
+ * mg_fast_comp_dict.c -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:06  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:57  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:55  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:47  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "huffman.h"
+#include "timing.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "backend.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; i++)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  bzero (buffer, mem);
+  fixup = Xmalloc (fixup_mem);
+  bzero (fixup, fixup_mem);
+
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+
+  load_comp_dict (filename);
+
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; which++)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+	cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+      for (which = 0; which <= 1; which++)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; j++)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; i++)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = buffer; (u_long) p < (u_long) cur; p++)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * cdh, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  u_long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; val--)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem += sizeof (comp_frags_header);
+	  Read_cfh (dict, &cfh, NULL, NULL);
+
+	  /* Don't need to count the space for the clens of the huffman data */
+
+	  mem += mem_for_words (dict, &cdh, &cfh);
+	  if (cfh.hd.clens)
+	    Xfree (cfh.hd.clens);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY ||
+      cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID ||
+      cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((long) cur > (long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID ||
+	  cdh->novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; i++)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+	  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+	  FIXUP (&cd->cfh[which]);
+	  FIXUP (&cd->values[which]);
+	  cd->escape[which] = NULL;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->chars_huff[which] = hd;
+	  FIXUP (&cd->chars_huff[which]);
+	  cd->chars_vals[which] = vals;
+	  FIXUP (&cd->chars_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->lens_huff[which] = hd;
+	  FIXUP (&cd->lens_huff[which]);
+	  cd->lens_vals[which] = vals;
+	  FIXUP (&cd->lens_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->chars_huff[which] = hd;
+	      FIXUP (&cd->chars_huff[which]);
+	      cd->chars_vals[which] = vals;
+	      FIXUP (&cd->chars_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->lens_huff[which] = hd;
+	      FIXUP (&cd->lens_huff[which]);
+	      cd->lens_vals[which] = vals;
+	      FIXUP (&cd->lens_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = buffer; (u_long) p < (u_long) cur; p++)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.c	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * mg_files.c -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      Xfree (basepath);
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = Xmalloc (strlen (bp) + 2);
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_files.h	(revision 21112)
@@ -0,0 +1,329 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_hilite_words.c	(revision 21112)
@@ -0,0 +1,693 @@
+/**************************************************************************
+ *
+ * mg_hilite_words -- display text and highlight particular words which 
+ *                    it contains
+ * Copyright (C) 1994 Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "getopt.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "stemmer.h"
+#include "words.h"
+
+
+/*
+ * Description
+ * -----------
+ *   Hilite_words reads text from stdin and outputs it into a pager such as
+ *   less. It's command arguments include a list of stemmed words 
+ *   which if extracted from the text will be highlighted on the output.
+ *
+ * Implementation
+ * --------------
+ *   Extracting of words - using PARSE_STEM_WORD & stemmer
+ *   
+ *   Highlighting of words - for standard pager:
+ *                             using back-space character code
+ *                             e.g. bolded a = "a\ba", underlined a = "_\ba"
+ *                         - for pager==html:
+ *                             use some of the standard html character 
+ *                             formatting tags
+ *
+ *   Storage of words - using hashtable (set) of size equalling a constant
+ *                      times the number of words
+ * 
+ * Usage
+ * -----
+ *   mg_hilite_words --stem_method [0-3] 
+ *                   --style [bold|underline|italic|emphasis|strong]
+ *                   --pager [less|more|html|???]
+ *                   --terminator [terminator-string]
+ *                   list-of-words-to-highlight
+ */
+
+/*
+ * Modifications:
+ *
+ * 21/Apr/95: To handle outputting html tags to stdout
+ *
+ */
+
+/* --- constants --- */
+
+/* highlighting styles */
+#define HILITE_MAX      5	/* the number of styles */
+#define BOLD 		0
+#define UNDERLINE	1
+#define ITALIC     	2
+#define EMPHASIS     	3
+#define STRONG     	4
+
+/* maximum length of line buffer */
+#define MAX_LINE_BUFFER 200
+
+/* set pager to this to get html hilited text to stdout */
+#define HTML_OUT "html"
+
+/* output types */
+#define PAGER 0
+#define HTML  1
+
+/* --- types --- */
+
+typedef u_char *Word;
+
+/* --- globals --- */
+
+/* keep in synch with constants */
+static char *hilite_names[] =
+{"bold", "underline", "italic", "emphasis", "strong"};
+static char *hilite_tags[] =
+{"B", "", "I", "EM", "STRONG"};
+static short hilite_style = BOLD;
+static char *pager = "less";
+static int stemmer_num = 0;	/* Lovin's stemmer */
+static int stem_method = 3;	/* fold & stem */
+static char **word_list;
+static int num_of_words = 0;
+static int output_type = PAGER;
+static char *terminator = NULL;
+
+/* --- prototypes --- */
+
+static Word copy_c_word (char *w);
+static void process_words (char **words, int num_of_words);
+static int get_line (u_char * line, int n, FILE * stream);
+static void process_text (FILE * input_file, FILE * output_file);
+static void copy_word (u_char * w1, u_char * w2);
+static void process_buffer (u_char * s_in, int len, FILE * output_file);
+static void output_word (u_char * s_start, u_char * s_finish, FILE * output_file);
+static void output_hilite_word (u_char * s_start, u_char * s_finish,
+				FILE * output_file);
+static void process_args (int argc, char *argv[]);
+static void print_line (u_char * line, int n, FILE * stream);
+
+
+/******************************** word set ************************************/
+
+#include "hash.h"
+#include "locallib.h"
+
+#define SIZE_FACTOR 2
+
+typedef Word *WordSet;
+static WordSet set_of_words = NULL;
+static int hash_size = 0;
+
+/* prototypes - set routines */
+static void set_create (int num_of_words);
+static void set_add (Word word);
+static int set_member (Word word);
+static void set_print (void);
+
+/* =========================================================================
+ * Function: set_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_print (void)
+{
+  int i = 0;
+
+  for (i = 0; i < hash_size; i++)
+    {
+      Word word = set_of_words[i];
+
+      if (word)
+	{
+	  int len = *word++;
+
+	  fprintf (stderr, "[%d] = ", i);
+	  while (len--)
+	    fputc (*word++, stderr);
+	  fputc ('\n', stderr);
+	}
+    }
+
+}
+
+
+/* =========================================================================
+ * Function: set_create
+ * Description: 
+ *      Allocate memory for the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_create (int num_of_words)
+{
+  WordSet set;
+
+  hash_size = prime (num_of_words * SIZE_FACTOR);
+  set = (WordSet) malloc (hash_size * sizeof (Word));
+  if (!set)
+    FatalError (1, "Runout of memory for word hashtable");
+  bzero ((char *) set, hash_size * sizeof (Word));  /* [RPAP - Feb 97: WIN32 Port] */
+
+  set_of_words = set;
+
+}
+
+/* =========================================================================
+ * Function: set_add
+ * Description: 
+ *      Add a string element to the set.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_add (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	{
+	  set_of_words[hash_val] = word;
+	  break;
+	}
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	break;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+
+/* =========================================================================
+ * Function: set_member
+ * Description: 
+ *      Tests whether a string is a member of the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+set_member (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	return 0;
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	return 1;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+/******************************** end of word set ******************************/
+
+
+
+/* =========================================================================
+ * Function: copy_c_word
+ * Description: 
+ *      Allocate enough memory and copy word over
+ * Input: 
+ *      w = null terminated string (c_word)
+ * Output: 
+ *      word with length in 1st byte
+ * ========================================================================= */
+
+static Word
+copy_c_word (char *w)
+{
+  int len = strlen (w);
+  Word w_copy = (Word) malloc (len + 1);
+  Word w_ptr = NULL;
+  int j = 0;
+
+  if (!w_copy)
+    FatalError (1, "Not enough memory to copy a word");
+
+  w_copy[0] = len;
+  w_ptr = w_copy + 1;
+  for (j = 0; j < len; j++)
+    *w_ptr++ = *w++;
+
+  return w_copy;
+}
+
+/* =========================================================================
+ * Function: process_words
+ * Description: 
+ *      Go through the stemmed words and add to word set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_words (char **words, int num_of_words)
+{
+  int i = 0;
+
+  set_create (num_of_words);
+
+  for (i = 0; i < num_of_words; i++)
+    {
+      Word word = copy_c_word (words[i]);
+      set_add (word);
+    }
+
+}
+
+/* =========================================================================
+ * Function: get_line
+ * Description: 
+ *      Equivalent of fgets for u_char*.
+ *      But returns length of read-in line.
+ *      Expects to see a '\n' before an EOF
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+get_line (u_char * line, int n, FILE * stream)
+{
+  int i = 0;
+  int ch = '\0';
+
+  while (1)
+    {
+      if (i == n)
+	return i;
+
+      ch = fgetc (stream);
+
+      if (ch == EOF)
+	{
+	  if (!feof (stream))
+	    FatalError (1, "Error on reading a line from stdin");
+	  return EOF;
+	}
+
+      if (ch == '\n')
+	return i;
+
+      *line++ = ch;
+      i++;
+    }
+
+}
+
+/* =========================================================================
+ * Function: print_line
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+print_line (u_char * line, int n, FILE * stream)
+{
+
+  while (n--)
+    {
+      fputc (*line++, stream);
+    }
+  fputc ('\n', stream);
+
+}
+
+/* =========================================================================
+ * Function: process_text
+ * Description: 
+ *      Go through the text from input_file and highlight to output_file 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_text (FILE * input_file, FILE * output_file)
+{
+  static u_char line_buffer[MAX_LINE_BUFFER];
+
+
+  while (1)
+    {
+      int len = get_line (line_buffer, MAX_LINE_BUFFER, input_file);
+
+      if (len == EOF)
+	break;
+      process_buffer (line_buffer, len, output_file);
+
+    }
+}
+
+/* =========================================================================
+ * Function: copy_word
+ * Description: 
+ *      Copies w2 into w1. Assumes both have storage allocated.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+copy_word (u_char * w1, u_char * w2)
+{
+  int i;
+  int len = w2[0];
+
+  for (i = 0; i <= len; i++)
+    *w1++ = *w2++;
+
+}
+
+/* =========================================================================
+ * Function: process_buffer
+ * Description: 
+ *      Parse & stem words of line buffer
+ *      Based on the usage of PARSEing in other mg files.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_buffer (u_char * s_in, int len, FILE * output_file)
+{
+  u_char *end = s_in + len - 1;
+  u_char *s_start = NULL;
+
+  if (!inaword (s_in, end))
+    {
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+    }
+
+  while (s_in <= end)
+    {
+      u_char word[MAXSTEMLEN + 1];
+
+      s_start = s_in;
+      PARSE_STEM_WORD (word, s_in, end);
+
+      stemmer (stem_method, stemmer_num, word);
+
+      if (set_member (word))	/* output with highlighting */
+	{
+	  output_hilite_word (s_start, s_start + word[0] - 1, output_file);
+	  s_start += word[0];	/* step over hilited output */
+	}
+      output_word (s_start, s_in - 1, output_file);
+
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+
+    }				/*while */
+
+  fputc ('\n', output_file);
+  fflush (output_file);
+
+}				/*process_buffer */
+
+/* =========================================================================
+ * Function: output_word
+ * Description: 
+ *      Output a word which lies from s_start to s_finish in buffer
+ * Input: 
+ *      s_start = ptr to 1st char
+ *      s_finish = ptr to last char
+ * Output: 
+ * ========================================================================= */
+
+static void
+output_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+  while (s_start <= s_finish)
+    {
+      fputc (*s_start++, output_file);
+    }
+}
+
+
+/* =========================================================================
+ * Function: output_hilite_word
+ * Description: 
+ *      Highlight a word (with length in 1st byte)
+ *      Pager highlighting:
+ *        Highlighting is either by bolding or underlining using
+ *        the method used by UNIX utilities More(1) and Less(1)
+ *      HTML highlighting:
+ *        use the appropriate start and end tags around the word
+ * ========================================================================= */
+
+static void
+output_hilite_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+
+  if (output_type == HTML)
+    {
+      char *hilite_tag = hilite_tags[hilite_style];
+
+      /* print start tag */
+      fprintf (output_file, "<%s>", hilite_tag);
+
+      output_word (s_start, s_finish, output_file);
+
+      /* print end tag */
+      fprintf (output_file, "</%s>", hilite_tag);
+    }
+
+  else
+    /* PAGER */
+    {
+      /* use backspaces around each letter */
+      while (s_start <= s_finish)
+	{
+	  switch (hilite_style)
+	    {
+	    case BOLD:
+	      fputc (*s_start, output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    case UNDERLINE:
+	      fputc ('_', output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    default:
+	      fputc (*s_start, output_file);
+	    }
+	  s_start++;
+	}			/*while */
+    }
+}
+
+/* =========================================================================
+ * Function: process_args
+ * Description: 
+ *      sets the global variables:
+ *              hilite_style, pager, num_of_words, word_list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+struct option long_opts[] =
+{
+  {"style", required_argument, 0, 's'},
+  {"terminator", required_argument, 0, 't'},
+  {"pager", required_argument, 0, 'p'},
+  {"stem_method", required_argument, 0, 'm'},
+  {"stemmer", required_argument, 0, 'a'},
+  {0, 0, 0, 0}
+};
+
+static void
+process_args (int argc, char *argv[])
+{
+  int ch;
+
+
+  opterr = 0;
+  while ((ch = getopt_long (argc, argv, "s:p:t:m:a:", long_opts, (int *) 0)) != -1)
+    {
+      switch (ch)
+	{
+	case 's':
+	  {
+	    int i;
+	    for (i = 0; i < HILITE_MAX; i++)
+	      if (strcmp (optarg, hilite_names[i]) == 0)
+		break;
+
+	    if (i < HILITE_MAX)
+	      hilite_style = i;
+	  }
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 't':
+	  terminator = optarg;
+	  break;
+	case 'm':
+	  stem_method = atoi (optarg);
+	  break;
+	case 'p':
+	  pager = optarg;
+	  break;
+	default:
+	  FatalError (1, "Usage: \n"
+		      "mg_hilite_words --stem_method [0-3]\n"
+	  "                --stemmer [english|lovin|french|simplefrench]\n"
+	  "                --style [bold|underline|italic|emphasis|strong]\n"
+		      "                --pager [less|more|html|???]\n");
+	}
+    }
+
+  num_of_words = argc - optind;
+
+  word_list = &argv[optind];
+
+  /* fix up output type */
+  if (strcmp (pager, HTML_OUT) == 0)
+    output_type = HTML;
+  else
+    output_type = PAGER;
+
+}
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+main (int argc, char *argv[])
+{
+  FILE *output = NULL;
+
+  process_args (argc, argv);
+
+  /* set output file */
+  if (output_type == PAGER)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    output = _popen (pager, "w");
+#else
+    output = popen (pager, "w");
+#endif
+  else
+    output = stdout;
+
+  if (!output)
+    FatalError (1, "Unable to run \"%s\"\n", pager);
+
+  if (num_of_words < 1)
+    {
+      int ch;
+
+      /* just echo the input */
+      /* better not to call this program at all ;-) */
+      while ((ch = fgetc (stdin)) != EOF)
+	{
+	  fputc (ch, output);
+	}
+    }
+  else
+    {
+      /* set up hash table for words */
+      process_words (word_list, num_of_words);
+
+
+      /* Go thru lines of text from stdin and 
+       * output words with hilite info if
+       * words parse into existence in the hash table 
+       */
+      process_text (stdin, output);
+    }
+
+  if (terminator)
+    fprintf (output, "%s\n", terminator);
+
+  if (output != stdout)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (output);
+#else
+    pclose (output);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dict.c	(revision 21112)
@@ -0,0 +1,344 @@
+/**************************************************************************
+ *
+ * mg_invf_dict.c -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:08  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:45  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:00  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:49  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static void process_files (char *filename);
+
+int main (int argc, char **argv)
+{
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:b:Fh")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b num] [-F] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  process_files (file_name);
+  return 0;
+}
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *id, *idb, *tmp, *ii;
+  unsigned long i, pos, num, First_word, invf_ptr, invf_len;
+  unsigned long last_ptr = 0;
+  char *FName;
+  struct invf_dict_header idh;
+  struct stem_dict_header sdh;
+  u_char prev[MAXSTEMLEN + 1];
+  u_char *buffer;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+
+  id = open_file (filename, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ii = open_file (filename, INVF_IDX_SUFFIX, "rb", MAGIC_INVI, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  idb = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "w+b", MAGIC_STEM,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  fread (&idh, sizeof (idh), 1, id);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  sdh.lookback = idh.lookback;
+  sdh.block_size = block_size;
+  sdh.num_blocks = 0;
+  sdh.index_chars = 0;
+  sdh.blocks_start = 0;
+  sdh.num_of_docs = idh.num_of_docs;
+  sdh.static_num_of_docs = idh.static_num_of_docs;
+  sdh.num_of_words = idh.num_of_words;
+  sdh.stemmer_num = idh.stemmer_num;
+  sdh.stem_method = idh.stem_method;
+  sdh.indexed = 0;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long extra, copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (id);
+      suff = getc (id);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, id);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, id);
+      fread (&wcnt, sizeof (wcnt), 1, id);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      /* read in the inverted file position */
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (word_num % idh.lookback == 0)
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + suff + 1 + sizeof (fcnt) + sizeof (wcnt) +
+	  sizeof (First_word) + sizeof (invf_ptr) + sizeof (invf_len) > block_size)
+	{
+	  int chunk;
+	  invf_len = invf_ptr - last_ptr;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUL(First_word);
+	  HTONUL(invf_len);
+	  HTONUS(word_num);
+
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sdh.num_blocks++;
+	}
+
+      if (word_num % idh.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (prev + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUL(fcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &fcnt, (char *) (buffer + buf_in_use), sizeof (fcnt));
+      buf_in_use += sizeof (fcnt);
+      HTONUL(wcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &wcnt, (char *) (buffer + buf_in_use), sizeof (wcnt));
+      buf_in_use += sizeof (wcnt);
+      last_ptr = invf_ptr;
+      HTONUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &invf_ptr, (char *) (buffer + buf_in_use), sizeof (invf_ptr));
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (invf_ptr);
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 34876234\n");
+      if (word_num == 0)
+	{
+	  fwrite (prev, sizeof (u_char), *prev + 1, idb);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idb);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sdh.index_chars += *prev + 1;
+	  First_word = i;
+	}
+      word_num++;
+    }
+  if (buf_in_use)
+    {
+      int chunk;
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      invf_len = invf_ptr - last_ptr;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUL(invf_len);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sdh.num_blocks++;
+    }
+  fclose (id);
+  fclose (ii);
+
+  rewind (tmp);
+  sdh.blocks_start = sdh.index_chars + sizeof (u_long) + sizeof (sdh) +
+    sdh.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sdh.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idb);
+	  sdh.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idb);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idb, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sdh.lookback);
+  HTONUL(sdh.block_size);
+  HTONUL(sdh.num_blocks);
+  HTONUL(sdh.blocks_start);
+  HTONUL(sdh.index_chars);
+  HTONUL(sdh.num_of_docs);
+  HTONUL(sdh.static_num_of_docs);
+  HTONUL(sdh.num_of_words);
+  HTONUL(sdh.stemmer_num);
+  HTONUL(sdh.stem_method);
+  HTONUL(sdh.indexed);
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+
+
+  Message ("Block size = %d\n", block_size);
+  Message ("Number of blocks written = %d\n", NTOHUL(sdh.num_blocks));
+
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_dump.c	(revision 21112)
@@ -0,0 +1,226 @@
+/**************************************************************************
+ *
+ * mg_invf_dump.c -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+
+extern unsigned long S_btg;
+
+/*
+   $Log$
+   Revision 1.2  2001/06/12 01:41:39  kjm18
+   changed the help message
+
+   Revision 1.1  1999/08/10 21:18:09  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/11/29  00:32:01  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.2  1994/09/20  04:41:50  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+static void process_files (char *filename);
+
+int binary = 0;
+int word_counts = 0;
+int term_dump = 0;
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hbwtf:d:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+        set_basepath(optarg);
+	break;
+      case 'b':
+	binary = 1;
+	break;
+      case 'w':
+	word_counts = 1;
+	break;
+      case 't':
+	term_dump = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-b] [-w] [-t] [-f input_file]"
+		 "[-d data directory]\n"
+		 "(b - binary mode, w - wordcount, t - term dump)\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+static void 
+process_files (char *name)
+{
+  unsigned long N, Nstatic, k;
+  FILE *dict;
+  FILE *invf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (!(invf = open_file (name, INVF_SUFFIX ".ORG", "rb", MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    invf = open_file (name, INVF_SUFFIX, "rb", MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  if (ifh.skip_mode != 0)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  DECODE_START (invf)
+    N = idh.num_of_docs;
+  Nstatic = idh.static_num_of_docs;
+  if (binary)
+    {
+      fwrite ((char *) &N, sizeof (N), 1, stdout);
+      fwrite ((char *) &ifh.no_of_words, sizeof (ifh.no_of_words), 1, stdout);
+    }
+  else
+    printf ("%ld %ld\n", N, ifh.no_of_words);
+  for (k = 0; k < ifh.no_of_words; k++)
+    {
+      int i, blk, doc;
+      register unsigned long suff, prefix;
+      unsigned long fcnt, wcnt;
+      char term[MAXSTEMLEN + 1];
+
+      prefix = fgetc (dict);
+      suff = fgetc (dict);
+      fread (&term[prefix], sizeof (char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+      term[prefix + suff] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (binary)
+	fwrite ((char *) &fcnt, sizeof (fcnt), 1, stdout);
+      else
+	{
+	  if (term_dump)
+	    printf ("%ld    \"%s\"\n", fcnt, term);
+	  else
+	    printf ("%ld\n", fcnt);
+	}
+      blk = BIO_Bblock_Init (Nstatic, fcnt);
+      for (doc = i = 0; i < fcnt; i++)
+	{
+	  int num;
+	  BBLOCK_DECODE (num, blk);
+	  doc += num;
+	  if (binary)
+	    fwrite ((char *) &doc, sizeof (doc), 1, stdout);
+	  else
+	    printf (" %d", doc);
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      int count;
+	      GAMMA_DECODE (count);
+	      if (word_counts)
+		if (binary)
+		  fwrite ((char *) &count, sizeof (count), 1, stdout);
+		else
+		  printf (" %d", count);
+	    }
+	  if (!binary)
+	    putchar ('\n');
+	}
+      while (__btg)
+	DECODE_BIT;
+    }
+  DECODE_DONE
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_merge.c	(revision 21112)
@@ -0,0 +1,852 @@
+/**************************************************************************
+ *
+ * mg_invf_merge.c -- description
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Last edited:  31 March, 1995 
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "bitio_m.h"
+#include "bitio_gen.h"
+#include "bitio_stdio.h"
+
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+
+typedef char FileName[256];
+
+
+/*
+ *  The merge_info struct contains all the data for an inverted file and
+ *  lexicon. We declare 3 merge_info variables, m[OLD] m[NEW] and m[MERGE].
+ */
+typedef struct merge_info_type
+  {
+    FileName fname;
+    FILE *invf;
+    FILE *dict;
+    FILE *idx;
+    FILE *para;  /* [RPAP - Feb 97: Level 3 Merge] */
+    struct invf_dict_header idh;
+    struct invf_file_header ifh;
+    u_long nDocs;
+
+    /* Lexicon processing variables */
+    u_long fcnt, wcnt;
+    int suff, pref;
+    unsigned char term[MAXSTEMLEN + 1];
+    u_long term_count;
+    int done;
+
+    /* Variables used for merging */
+    struct stdio_bitio_state sbs;	/* for bit-based I/O on inverted files */
+    u_long nTerms;		/* = number of invf entries */
+    int *fcntlist;		/* store fcnt values read in from the lexicons */
+
+  }
+merge_info;
+
+
+/***************************************************
+ **** GLOBALS ****
+ ***************************************************/
+
+merge_info m[3];		/*  m[OLD], m[NEW] and m[MERGE] */
+u_long magicsize;		/* how big is magic number? currently always 4 */
+u_long Nstatic;			/* static # of documents in file */
+
+float *DocWeightBuffer;
+int weightOption = 0;		/* true if weights file is to be appended */
+
+/*** Globals for processing the lexicons -- see readTerm() and writeTerm() 
+ ***/
+ProgTime start;			/* for printing elapsed time */
+unsigned char prevTerm[MAXSTEMLEN + 1];
+
+/*** Globals for merging -- see processEntry() 
+ ***/
+char *mergedata;		/* stores value OLD/NEW/MERGE for each term */
+int fastMerge = 1;		/* True if fast merge selected. Default = true */
+
+u_long oldIdxValue;		/* stores last read value from m[OLD].idx
+				   Used to calculate entry lengths */
+u_long bytes_output;		/* for writing .invf.idx file */
+u_long oldOnlyBits = 0;		/* entries in old only */
+u_long oldMergeBits = 0;	/* entries in old that are merged with 
+				   a new entry */
+
+/**************************************************************************/
+
+
+/*=======================================================================
+ * init_merge_invf(): open files, initialise globals, etc
+ *=======================================================================*/
+int 
+init_merge_invf ()
+{
+  prevTerm[0] = prevTerm[1] = 0;
+
+    /***
+     *  open .dict files
+     ***/
+  m[OLD].dict = open_file (m[OLD].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (m[OLD].dict);
+  fread (&(m[OLD].idh), sizeof (m[OLD].idh), 1, m[OLD].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].idh.lookback);
+  NTOHUL(m[OLD].idh.dict_size);
+  NTOHUL(m[OLD].idh.total_bytes);
+  NTOHUL(m[OLD].idh.index_string_bytes);
+  NTOHD(m[OLD].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[OLD].idh.num_of_docs);
+  NTOHUL(m[OLD].idh.static_num_of_docs);
+  NTOHUL(m[OLD].idh.num_of_words);
+  NTOHUL(m[OLD].idh.stemmer_num);
+  NTOHUL(m[OLD].idh.stem_method);
+
+  m[NEW].dict = open_file (m[NEW].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[NEW].idh), sizeof (m[NEW].idh), 1, m[NEW].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].idh.lookback);
+  NTOHUL(m[NEW].idh.dict_size);
+  NTOHUL(m[NEW].idh.total_bytes);
+  NTOHUL(m[NEW].idh.index_string_bytes);
+  NTOHD(m[NEW].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[NEW].idh.num_of_docs);
+  NTOHUL(m[NEW].idh.static_num_of_docs);
+  NTOHUL(m[NEW].idh.num_of_words);
+  NTOHUL(m[NEW].idh.stemmer_num);
+  NTOHUL(m[NEW].idh.stem_method);
+
+  m[MERGE].dict = create_file (m[MERGE].fname, INVF_DICT_SUFFIX, "wb",
+			       MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header */
+  fwrite ((char *) &(m[OLD].idh), sizeof (m[OLD].idh), 1, m[MERGE].dict);
+
+  m[OLD].nDocs = m[OLD].idh.num_of_docs;
+  m[NEW].nDocs = m[NEW].idh.num_of_docs;
+  Nstatic = m[OLD].idh.static_num_of_docs;
+  m[OLD].nTerms = m[OLD].idh.dict_size;
+  m[NEW].nTerms = m[NEW].idh.dict_size;
+  m[MERGE].nDocs = m[OLD].nDocs + m[NEW].nDocs;
+  m[MERGE].nTerms = 0;
+
+  /* Set up weight buffer for weights of new documents */
+  if (!(DocWeightBuffer = Xmalloc (m[NEW].nDocs * sizeof (*DocWeightBuffer))))
+    {
+      Message ("Insufficient memory\n");
+      return COMPERROR;
+    }
+  bzero ((char *) DocWeightBuffer, m[NEW].nDocs * sizeof (*DocWeightBuffer));
+
+    /***
+     *  open .invf files
+     *  Try ".ORG" extension first for m[OLD].invf since an inverted file
+     *  with skips may have been created.
+     ****/
+  if (!(m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX ".ORG", "rb",
+				 MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[OLD].ifh), sizeof (m[OLD].ifh), 1, m[OLD].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].ifh.no_of_words);
+  NTOHUL(m[OLD].ifh.no_of_ptrs);
+  NTOHUL(m[OLD].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[OLD].ifh.params[i]);
+  }
+  NTOHUL(m[OLD].ifh.InvfLevel);
+
+  m[NEW].invf = open_file (m[NEW].fname, INVF_SUFFIX, "rb",
+			   MAGIC_INVF, MG_ABORT);
+  fread (&(m[NEW].ifh), sizeof (m[NEW].ifh), 1, m[NEW].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].ifh.no_of_words);
+  NTOHUL(m[NEW].ifh.no_of_ptrs);
+  NTOHUL(m[NEW].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[NEW].ifh.params[i]);
+  }
+  NTOHUL(m[NEW].ifh.InvfLevel);
+
+  if (m[OLD].ifh.skip_mode != 0)
+    FatalError (1, "The old invf file contains skips. Unable to merge.");
+
+  if (m[OLD].ifh.InvfLevel != m[NEW].ifh.InvfLevel)
+    FatalError (1, "The two invf files have different inversion levels!");
+
+  /********** [RPAP - Feb 97: Level 3 Merge]
+
+  if ((m[OLD].ifh.InvfLevel > 2) || (m[NEW].ifh.InvfLevel > 2))
+    FatalError (1, "mgmerge cannot handle level 3 inverted files!");
+
+    *************/
+
+  m[MERGE].invf = create_file (m[MERGE].fname, INVF_SUFFIX, "wb",
+			       MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header in inverted file */
+  m[MERGE].ifh.no_of_words = m[MERGE].ifh.no_of_ptrs = 0;
+  m[MERGE].ifh.skip_mode = 0;
+  m[MERGE].ifh.InvfLevel = 2;
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = 0;
+  }
+  fwrite ((char *) &(m[MERGE].ifh), sizeof (m[MERGE].ifh), 1, m[MERGE].invf);
+
+    /***
+     *  open .invf.idx files
+     ***/
+  m[OLD].idx = open_file (m[OLD].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+  fread (&oldIdxValue, sizeof (u_long), 1, m[OLD].idx);
+  NTOHUL(oldIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  m[NEW].idx = open_file (m[NEW].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+
+  m[MERGE].idx = create_file (m[MERGE].fname, INVF_IDX_SUFFIX, "wb",
+			      MAGIC_INVI, MG_ABORT);
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+
+    /***
+     *  open .invf.parargraph files
+     ***/
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = open_file (m[NEW].fname, INVF_PARAGRAPH_SUFFIX, "rb",
+			       MAGIC_PARAGRAPH, MG_ABORT);
+      m[MERGE].para = open_file (m[MERGE].fname, INVF_PARAGRAPH_SUFFIX, "rb+",
+				 MAGIC_PARAGRAPH, MG_ABORT);
+    }
+  else
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = NULL;
+      m[MERGE].para = NULL;
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * procEntry_slow(): merge an entry, slow method
+ *=======================================================================*/
+void 
+procEntry_slow (int tnum, int type, u_long oldN, u_long newN, u_long mergeN)
+/* 
+ *  Process an invf entry. 
+ *  Slow Merge. (Always decode/re-encode).
+ *
+ *  tnum is the entry number, type = OLD or NEW or MERGE.
+ *  oldN, newN and mergeN are the N parameters for Bblock coding.
+ *  They are passed so other procEntry_fast can call this one
+ *  when a decode/recode is needed to save repeating code.
+ */
+{
+  int doc = 0;
+  int fcntOLD = m[OLD].fcntlist[tnum];
+  int fcntNEW = m[NEW].fcntlist[tnum];
+  int inblk, outblk;
+  u_long inbits = 0, outbits = 0;
+
+  /* write .invf.idx pointer */
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  outblk = BIO_Bblock_Init (mergeN, (fcntOLD + fcntNEW));
+
+/*** OLD entry -> merge entry ***/
+  if (type != NEW)
+    {
+      int i;
+      inblk = BIO_Bblock_Init (oldN, fcntOLD);
+      for (i = 0; i < fcntOLD; i++)
+	{
+	  int num;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[OLD].sbs), &inbits);
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[OLD].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	    }
+	}
+      if (type == OLD)
+	oldOnlyBits += (inbits + m[OLD].sbs.Btg);
+      else
+	oldMergeBits += (inbits + m[OLD].sbs.Btg);
+    }
+
+/*** NEW entry -> merge entry ***/
+  if (type != OLD)
+    {
+      int i;
+      int offset;
+      double logN = log ((double) m[MERGE].nDocs);
+      double idf = logN - log ((double) (fcntOLD + fcntNEW));
+
+      inblk = BIO_Bblock_Init (newN, fcntNEW);
+      offset = (m[OLD].nDocs - doc);	/* amount to be added to 1ST pointer */
+      for (i = 0; i < fcntNEW; i++)
+	{
+	  register int num;
+	  register double weight;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[NEW].sbs), &inbits);
+	  if (i == 0)
+	    num += offset;
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[NEW].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	      weight = num * idf;
+	      DocWeightBuffer[doc - m[OLD].nDocs - 1] += weight * weight;
+	    }
+	}
+    }
+/*** Now the padding bits ***/
+  while (m[OLD].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[OLD].sbs));
+  while (m[NEW].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[NEW].sbs));
+  while (m[MERGE].sbs.Btg != 8)
+    {
+      BIO_Stdio_Encode_Bit (0, &(m[MERGE].sbs));
+      outbits++;
+    }
+  bytes_output += (outbits >> 3);
+  return;
+}
+
+
+/*=======================================================================
+ * procEntry_fast(): merge an entry, WITHOUT decoding if it's in the
+ *                   old inverted file only
+ *=======================================================================*/
+void 
+procEntry_fast (int tnum, int type)
+/*
+ *  Merge an invf entry. 
+ *  Faster Method : Copy (don't re-code) entries for terms only in IFold.
+ *  The N parameter is Nstatic, from m[OLD].idh.static_num_of_docs.
+ *  If entry type isnt OLD, just call procEntry_slow()
+ */
+{
+  u_long newIdxValue, len = 0;
+
+  /* Calculate IFold entry length in bytes if not NEW type */
+  if (type != NEW)		/* read in an index number from m[OLD].idx */
+    {
+      fread (&newIdxValue, sizeof (u_long), 1, m[OLD].idx);
+      NTOHUL(newIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+      len = newIdxValue - oldIdxValue;
+      oldIdxValue = newIdxValue;
+    }
+
+  if (type != OLD)		/* must decode/recode */
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, Nstatic);
+  else
+    {
+      /* write .invf.idx pointer */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* copy entry */
+      {
+	int i;
+	char c;
+	for (i = 0; i < len; i++)
+	  {
+	    c = fgetc (m[OLD].invf);
+	    fputc (c, m[MERGE].invf);
+	  }
+      }
+      bytes_output += len;
+      oldOnlyBits += (len * 8);
+    }
+  return;
+}
+
+
+/*=======================================================================
+ * processEntry(): merge an entry. Call either procEntry_slow() or
+ *                 procEntry_fast.
+ *=======================================================================*/
+void 
+processEntry (int tnum, int type)
+/* 
+ *  Read an entry from IFold and/or IFnew, and write to IFmerge.
+ *  tnum is the term number, type is OLD or NEW or MERGE
+ */
+{
+  if (fastMerge)
+    procEntry_fast (tnum, type);
+  else
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, m[MERGE].nDocs);
+  return;
+}
+
+
+/*=======================================================================
+ * readTerm()
+ *=======================================================================*/
+void 
+readTerm (int x)
+/*
+ *  Read the next term from the appropriate ".invf.dict" file.
+ *  x should be either OLD or NEW.
+ *  Sets m[x].done to true when that lexicon has been completely read.
+ */
+{
+  int i;
+
+  if (m[x].term_count >= m[x].nTerms)
+    {
+      m[x].term[0] = 0;
+      m[x].done = 1;
+      return;
+    }
+  m[x].pref = fgetc (m[x].dict);
+  m[x].suff = fgetc (m[x].dict);
+  m[x].term[0] = m[x].pref + m[x].suff;
+
+  for (i = 0; i < m[x].suff; i++)
+    {
+      m[x].term[m[x].pref + i + 1] = fgetc (m[x].dict);
+    }
+  m[x].term_count = m[x].term_count + 1;
+
+  fread ((char *) &(m[x].fcnt), sizeof (u_long), 1, m[x].dict);
+  fread ((char *) &(m[x].wcnt), sizeof (u_long), 1, m[x].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[x].fcnt);
+  NTOHUL(m[x].wcnt);
+
+  return;
+}
+
+
+/*=======================================================================
+ * writeTerm()
+ *=======================================================================*/
+void 
+writeTerm (int x)
+/*
+ *  Write the current term to m[MERGE].dict file.
+ *  x is OLD, NEW or MERGE.
+ */
+{
+  unsigned char i, prefix, suffix;
+  if (x == MERGE)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt + m[NEW].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt + m[NEW].wcnt;
+      m[MERGE].pref = m[OLD].pref;
+      m[MERGE].suff = m[OLD].suff;
+      memcpy (m[MERGE].term, m[OLD].term, m[OLD].term[0] + 1);
+    }
+  if (x == OLD)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt;
+    }
+  if (x == NEW)
+    {
+      m[MERGE].fcnt = m[NEW].fcnt;
+      m[MERGE].wcnt = m[NEW].wcnt;
+    }
+  prefix = prefixlen (prevTerm, m[x].term);
+  suffix = m[x].term[0] - prefix;
+  fputc (prefix, m[MERGE].dict);	/* prefix length */
+  fputc (suffix, m[MERGE].dict);	/* suffix length */
+  for (i = 0; i < suffix; i++)
+    fputc (m[x].term[prefix + i + 1], m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].fcnt);
+  HTONUL(m[MERGE].wcnt);
+
+  fwrite ((char *) &(m[MERGE].fcnt), sizeof (m[MERGE].fcnt), 1, m[MERGE].dict);
+  fwrite ((char *) &(m[MERGE].wcnt), sizeof (m[MERGE].wcnt), 1, m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[MERGE].fcnt);
+  NTOHUL(m[MERGE].wcnt);
+
+  memcpy (prevTerm, m[x].term, prefix + suffix + 1);
+  m[MERGE].idh.total_bytes += m[x].term[0] + 1;
+  m[MERGE].idh.index_string_bytes += m[x].term[0] + 2 - prefix;
+  fflush (m[MERGE].dict);
+}
+
+
+/*=======================================================================
+ * process_merge_invf(): The main loop
+ *=======================================================================*/
+int 
+process_merge_invf (void)
+/*
+ *  The main function to merge the lexicons and inverted files.
+ */
+{
+/*** Some extra Initialisation stuff first ***/
+
+  m[OLD].term[0] = m[NEW].term[0] = '\0';
+  m[OLD].term_count = m[NEW].term_count = 0;
+  m[OLD].done = m[NEW].done = 0;
+  m[MERGE].idh.total_bytes = m[MERGE].idh.index_string_bytes = 0;
+
+/*** malloc arrays: mergedata, m[OLD].fcntlist, m[NEW].fcntlist ***/
+  mergedata = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (char));
+  if (mergedata == 0)
+    {
+      fprintf (stderr, "MALLOC error!\n");
+      exit (1);
+    }
+  {
+    int i;
+    for (i = 0; i < 2; i++)
+      {
+	m[i].fcntlist = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (int));
+	if (m[i].fcntlist == 0)
+	  {
+	    fprintf (stderr, "MALLOC error!\n");
+	    exit (1);
+	  }
+      }
+  }
+
+/*========================*/
+/***    LEXICON PASS    ***/
+/*========================*/
+
+/*** read first terms ***/
+  readTerm (OLD);
+  readTerm (NEW);
+
+  m[MERGE].nTerms = 0;
+  while (m[OLD].done == 0 || m[NEW].done == 0)
+    {
+      int i;
+      if (m[OLD].done)
+	i = 1; /* NEW will always be greater */
+      else if (m[NEW].done)
+	i = -1; /* OLD will always be greater */
+      else
+	i = casecompare (m[OLD].term, m[NEW].term);
+
+      if (i < 0)
+	{			/* term in OLD only */
+	  mergedata[m[MERGE].nTerms] = (char) OLD;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (OLD);
+	  readTerm (OLD);
+	}
+      if (i == 0)
+	{			/* term in both lexions */
+	  mergedata[m[MERGE].nTerms] = (char) MERGE;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[MERGE].nTerms++;
+	  writeTerm (MERGE);
+	  readTerm (OLD);
+	  readTerm (NEW);
+	}
+      if (i > 0)
+	{			/* term in NEW only */
+	  mergedata[m[MERGE].nTerms] = (char) NEW;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (NEW);
+	  readTerm (NEW);
+	}
+    }/*while*/
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  /* print some results about terms */
+  {
+    fprintf (stderr, "   Terms: OLD = %ld,  NEW = %ld,  MERGE = %ld\n",
+	     m[OLD].nTerms, m[NEW].nTerms, m[MERGE].nTerms);
+    fprintf (stderr, "   In OLD only: %ld,  In NEW only: %ld,  BOTH: %ld\n",
+	     (m[MERGE].nTerms - m[NEW].nTerms),
+	     (m[MERGE].nTerms - m[OLD].nTerms),
+	     (m[OLD].nTerms + m[NEW].nTerms - m[MERGE].nTerms));
+  }
+
+
+/*========================*/
+/***      INVF PASS     ***/
+/*========================*/
+
+  bytes_output = ftell (m[MERGE].invf);
+  BIO_Stdio_Decode_Start (m[OLD].invf, &m[OLD].sbs);
+  BIO_Stdio_Decode_Start (m[NEW].invf, &m[NEW].sbs);
+  BIO_Stdio_Encode_Start (m[MERGE].invf, &m[MERGE].sbs);
+  {
+    int i;
+    for (i = 0; i < m[MERGE].nTerms; i++)
+      {
+	processEntry (i, mergedata[i]);
+      }
+    /* write final .invf.idx pointer */
+    HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+    fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+    NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+    fprintf (stderr, "Old invf: old only bits: %ld, merged bits: %ld\n",
+	     oldOnlyBits, oldMergeBits);
+  }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * done_merge_invf(): write headers, close files, append weights to
+ *                     weights file
+ *=======================================================================*/
+int 
+done_merge_invf (void)
+{
+/*** write new lexicon header ***/
+  m[MERGE].idh.dict_size = m[MERGE].nTerms;
+  m[MERGE].idh.lookback = m[OLD].idh.lookback;
+  m[MERGE].idh.input_bytes = m[OLD].idh.input_bytes + m[NEW].idh.input_bytes;
+  m[MERGE].idh.total_bytes = m[OLD].idh.total_bytes + m[NEW].idh.total_bytes;
+  m[MERGE].idh.num_of_docs = m[MERGE].nDocs;
+  m[MERGE].idh.num_of_words = m[OLD].idh.num_of_words + m[NEW].idh.num_of_words;
+  m[MERGE].idh.stemmer_num = m[OLD].idh.stemmer_num;
+  m[MERGE].idh.stem_method = m[OLD].idh.stem_method;
+
+  /* if fastMerge, static num of docs stays the same! */
+  if (fastMerge)
+    m[MERGE].idh.static_num_of_docs = m[OLD].idh.static_num_of_docs;
+  else
+    m[MERGE].idh.static_num_of_docs = m[MERGE].nDocs;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].idh.lookback);
+  HTONUL(m[MERGE].idh.dict_size);
+  HTONUL(m[MERGE].idh.total_bytes);
+  HTONUL(m[MERGE].idh.index_string_bytes);
+  HTOND(m[MERGE].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(m[MERGE].idh.num_of_docs);
+  HTONUL(m[MERGE].idh.static_num_of_docs);
+  HTONUL(m[MERGE].idh.num_of_words);
+  HTONUL(m[MERGE].idh.stemmer_num);
+  HTONUL(m[MERGE].idh.stem_method);
+
+  fseek (m[MERGE].dict, magicsize, 0);
+  fwrite (&(m[MERGE].idh), sizeof (struct invf_dict_header), 1, m[MERGE].dict);
+
+/*** write new inverted file header ***/
+  m[MERGE].ifh.no_of_words = m[MERGE].nTerms;
+  m[MERGE].ifh.no_of_ptrs = m[OLD].ifh.no_of_ptrs + m[NEW].ifh.no_of_ptrs;
+  m[MERGE].ifh.skip_mode = m[OLD].ifh.skip_mode;
+  m[MERGE].ifh.InvfLevel = m[OLD].ifh.InvfLevel;
+
+  /* ifh.params[16] -- I have NO IDEA what these do!!!!! */
+  /* only set if a inverted file is in skipped format, I think */
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = m[OLD].ifh.params[i];
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].ifh.no_of_words);
+  HTONUL(m[MERGE].ifh.no_of_ptrs);
+  HTONUL(m[MERGE].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      HTONUL(m[MERGE].ifh.params[i]);
+  }
+  HTONUL(m[MERGE].ifh.InvfLevel);
+
+  fseek (m[MERGE].invf, magicsize, 0);	/* go to start of header */
+  fwrite (&(m[MERGE].ifh), sizeof (struct invf_file_header), 1, m[MERGE].invf);
+
+/*** Write new document weights --- leave old weights alone ***/
+  if ((weightOption) && (m[OLD].ifh.InvfLevel >= 2))
+    {
+      FILE *weightfile = open_file (m[MERGE].fname, WEIGHTS_SUFFIX, "r+b",
+				    MAGIC_WGHT, MG_CONTINUE);  /* [RPAP - Feb 97: WIN32 Port] */
+      if (weightfile)
+	{
+	  u_long i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+	  fprintf (stderr, "writing new weights: Nnew = %ld\n", m[NEW].nDocs);
+	  fseek (weightfile, 0, 2);	/* end of file */
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (i = 0; i < m[NEW].nDocs; i++)
+	    HTONF(DocWeightBuffer[i]);
+
+	  fwrite ((char *) DocWeightBuffer, sizeof (float), m[NEW].nDocs,
+		  weightfile);
+	}
+      fclose (weightfile);
+    }
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      /* Update paragraph file */
+      int buffer_size = 1000;
+      u_char *buffer;
+      int num;
+
+      if (!(buffer = (u_char *) Xmalloc (buffer_size)))
+	FatalError (1, "Could not allocate memory for paragraph buffer");
+
+      fseek (m[MERGE].para, 0, SEEK_END);
+      while ((num = fread ((char *) buffer, sizeof (*buffer), buffer_size, m[NEW].para)) != 0)
+	fwrite ((char *) buffer, sizeof (*buffer), num, m[MERGE].para);
+
+      Xfree (buffer);
+    }
+
+/*** close files ***/
+  fclose (m[MERGE].invf);
+  fclose (m[OLD].invf);
+  fclose (m[NEW].invf);
+  fclose (m[MERGE].dict);
+  fclose (m[OLD].dict);
+  fclose (m[NEW].dict);
+  fclose (m[MERGE].idx);
+  fclose (m[OLD].idx);
+  fclose (m[NEW].idx);
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      fclose (m[NEW].para);
+      fclose (m[MERGE].para);
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s [-s] [-w] [-d directory] -f mg_collection\n",
+	   progname);
+  fprintf (stderr, "  example: %s -s -w -f MERGE/mailfiles\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int 
+main (int argc, char *argv[])
+{
+  int ch;
+  char *progname;
+  progname = argv[0];
+  m[MERGE].fname[0] = '\0';
+
+  /* message and timing information */
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:swh")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (m[MERGE].fname, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	fastMerge = 0;
+	break;
+      case 'w':
+	weightOption = 1;
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (m[MERGE].fname[0] == '\0')
+    usage (progname);
+  strcpy (m[OLD].fname, m[MERGE].fname);
+  strcat (m[OLD].fname, ".old");
+  strcpy (m[NEW].fname, m[MERGE].fname);
+  strcat (m[NEW].fname, ".new");
+
+  GetTime (&start);
+  init_merge_invf ();
+  process_merge_invf ();
+  done_merge_invf ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  exit (0);
+}
+
+/*************************************************************************/
+/*   EOF: mg_invf_merge_new.c                                            */
+/*************************************************************************/
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_invf_rebuild.c	(revision 21112)
@@ -0,0 +1,564 @@
+/**************************************************************************
+ *
+ * mg_invf_rebuild.c -- Program to rebuild an inverted file with skipping
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define LEN 40
+#define WRD 1
+
+typedef struct invf_info
+  {
+    unsigned long doc_num, count, bits_so_far, count_bits;
+  }
+invf_info;
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:11  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:03  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:51  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+static char pathname[256];
+
+static void process_files (char *filename);
+
+static int k = -1;
+static int mode = 0;
+static int max_nodes = -1;
+static int mins = -1;
+
+void 
+usage (char *pgname)
+{
+  fprintf (stderr, "usage: %s [-f input_file]"
+       "[-d data directory] -0|-1|-2 [-k num] [-m num] [-s num]\n", pgname);
+  printf ("Mode 0: (the default)\n");
+  printf ("\tk, m, and s have no meaning and if specified produce an error\n");
+  printf ("Mode 1:\n");
+  printf ("\tm, and s have no meaning and if specified produce an error\n");
+  printf ("\tk The size of skips\n");
+  printf ("Mode 2:\n");
+  printf ("\tk has no meaning and if specified will produce an error\n");
+  printf ("\tm is the the number of accumulators that will be used for the\n");
+  printf ("\t  ranking. The program builds an inverted file that is \n");
+  printf ("\t  \"optimal\" for that number of accumulators.\n");
+  printf ("\ts is the minimum size for skips.\n");
+  exit (1);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  strcpy (pathname, dir_name ? dir_name : ".");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "012hf:d:k:b:s:m:")) != -1)
+    switch (ch)
+      {
+      case '0':
+	mode = 0;
+	break;
+      case '1':
+	mode = 1;
+	break;
+      case '2':
+	mode = 2;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	strcpy (pathname, optarg);
+	break;
+      case 'k':
+	k = atoi (optarg);
+	break;
+      case 'm':
+	max_nodes = atoi (optarg);
+	break;
+      case 's':
+	mins = atoi (optarg);
+	if (mins < 1)
+	  FatalError (1, "The number for the -m option must be greater"
+		      " than or equal to 1");
+	break;
+      case 'h':
+      case '?':
+	usage (argv[0]);
+      }
+
+  if ((mode == 0 && (k != -1 || max_nodes != -1 || mins != -1)) ||
+      (mode == 1 && (max_nodes != -1 || mins != -1)) ||
+      (mode == 2 && (k != -1)))
+    {
+      Message ("Illegal parameters for mode");
+      usage (argv[0]);
+    }
+
+  if (mode == 1 && k == -1)
+    {
+      k = 8;
+      Message ("k is required for mode 1: defaulting k to %d", k);
+    }
+  if (mode == 2 && max_nodes == -1)
+    {
+      max_nodes = 1024;
+      Message ("m is required for mode 2: defaulting m to %d", max_nodes);
+    }
+  if (mode == 2 && mins == -1)
+    {
+      mins = 1;
+      Message ("s is required for mode 2: defaulting s to %d", mins);
+    }
+
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  Message ("**** Don\'t forget to rebuild the stemmed dictionary with mg_invf_dict. ****\n");
+  Message ("**** If the collection was built with stem indexes don\'t forget to     ****\n");
+  Message ("**** rebuild them with mg_stem_idx.                                     ****\n");
+
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *in, *out, *idx, *odx, *dict;
+  unsigned long magic, outmode, N, in_k, out_k;
+  unsigned long bits_out, bytes_out, i, j;
+  stdio_bitio_state out_buf, in_buf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh_in, ifh_out;
+  char FName[256];
+
+  outmode = mode;
+
+  /* open .invf.ORG, rename .invf if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      rename (fname, FName);
+      if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.ORG */
+  if (fread (&magic, sizeof (magic), 1, in) != 1 || NTOHUL(magic) != MAGIC_INVF)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+
+  /* open .invf.idx.ORG, rename .invf.idx if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      rename (fname, FName);
+      if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.idx.ORG */
+  if (fread (&magic, sizeof (magic), 1, idx) != 1 || NTOHUL(magic) != MAGIC_INVI)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(out = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(odx = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Opening \"%s\"\n", FName);
+  if (!(dict = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  if (fread (&magic, sizeof (magic), 1, dict) != 1 || NTOHUL(magic) != MAGIC_STEM_BUILD)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  HTONUL2(MAGIC_INVF, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, out);
+
+  fread ((char *) &ifh_in, sizeof (ifh_in), 1, in);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_in.no_of_words);
+  NTOHUL(ifh_in.no_of_ptrs);
+  NTOHUL(ifh_in.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_in.params[i]);
+  NTOHUL(ifh_in.InvfLevel);
+
+  ifh_out = ifh_in;
+  ifh_out.skip_mode = outmode;
+  bzero ((char *) ifh_out.params, sizeof (ifh_out.params));
+  switch (outmode)
+    {
+    case 0:
+      break;
+    case 1:
+      ifh_out.params[0] = k;
+      break;
+    case 2:
+      ifh_out.params[0] = max_nodes;
+      ifh_out.params[1] = mins;
+      break;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(ifh_out.no_of_words);
+  HTONUL(ifh_out.no_of_ptrs);
+  HTONUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    HTONUL(ifh_out.params[i]);
+  HTONUL(ifh_out.InvfLevel);
+
+  fwrite ((char *) &ifh_out, sizeof (ifh_out), 1, out);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_out.no_of_words);
+  NTOHUL(ifh_out.no_of_ptrs);
+  NTOHUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_out.params[i]);
+  NTOHUL(ifh_out.InvfLevel);
+
+  Message ("The file is a level %d inverted file.\n", ifh_in.InvfLevel);
+
+  bits_out = ftell (out) * 8;
+
+
+  HTONUL2(MAGIC_INVI, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, odx);
+
+  DECODE_START (in)
+    DECODE_PAUSE (in_buf)
+
+    ENCODE_START (out)
+    ENCODE_PAUSE (out_buf)
+
+    N = idh.num_of_docs;
+
+  for (i = 0; i < ifh_in.no_of_words; i++)
+    {
+      unsigned long blk, p;
+      unsigned long odn_blk = 0, olen_blk = 0;
+      unsigned long idn_blk = 0, ilen_blk = 0;
+      register unsigned long suff;
+      unsigned long fcnt, wcnt, doc_num, bits_so_far, last_major;
+      unsigned long next_mjr_dn, kd;
+      char dummy2[MAXSTEMLEN + 1];
+      invf_info *ii;
+
+      fgetc (dict);
+      suff = fgetc (dict);
+      fread (dummy2, sizeof (u_char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+      NTOHUL(bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      p = fcnt;
+      blk = BIO_Bblock_Init (idh.static_num_of_docs, p);
+      switch (outmode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_out.params[0])
+	      out_k = 0;
+	    else
+	      {
+		out_k = ifh_out.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= mins)
+	      out_k = 0;
+	    else
+	      {
+		out_k = (int) (2 * sqrt ((double) p / max_nodes));
+		if (out_k <= mins)
+		  out_k = mins;
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	default:
+	  out_k = 0;
+	}
+
+      switch (ifh_in.skip_mode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[0])
+	      in_k = 0;
+	    else
+	      {
+		in_k = ifh_in.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[1])
+	      {
+		in_k = 0;
+	      }
+	    else
+	      {
+		in_k = (int) (2 * sqrt ((double) p / ifh_in.params[0]));
+		if (in_k <= ifh_in.params[1])
+		  in_k = ifh_in.params[1];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	default:
+	  in_k = 0;
+	}
+
+      if (!(ii = Xmalloc (sizeof (invf_info) * p)))
+	FatalError (1, "Unable to allocate memory for \"ii\"\n");
+
+      doc_num = bits_so_far = 0;
+      next_mjr_dn = 0;
+      kd = 0;
+      DECODE_CONTINUE (in_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  unsigned long doc_diff, count = 0;
+	  if (kd == in_k)
+	    kd = 0;
+	  if (in_k && kd == 0 && j + in_k < p)
+	    {
+	      int temp;
+	      BBLOCK_DECODE (next_mjr_dn, idn_blk);
+	      next_mjr_dn += doc_num;
+	      BBLOCK_DECODE (temp, ilen_blk);
+	    }
+	  ii[j].bits_so_far = bits_so_far;
+	  if (in_k && kd == in_k - 1 && j != p - 1)
+	    {
+	      int count;
+	      BBLOCK_LENGTH (next_mjr_dn - doc_num, blk, count);
+	      bits_so_far += count;
+	      doc_num = next_mjr_dn;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (doc_diff, blk, bits_so_far);
+	      doc_num += doc_diff;
+	    }
+	  ii[j].doc_num = doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    {
+	      int count_bits = 0;
+	      GAMMA_DECODE_L (count, count_bits);
+	      ii[j].count_bits = count_bits;
+	      bits_so_far += count_bits;
+	      ii[j].count = count;
+	    }
+	}
+
+      /* read till a byte boundary */
+      while (__btg)
+	{
+	  DECODE_BIT;
+	  bits_so_far++;
+	}
+
+      DECODE_PAUSE (in_buf)
+
+	doc_num = bits_so_far = 0;
+      last_major = 0;
+      kd = 0;
+      ENCODE_CONTINUE (out_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  if (kd == out_k)
+	    kd = 0;
+	  if (out_k && kd == 0)
+	    {
+	      if (j + out_k < p)
+		{
+		  int num = ii[j + out_k - 1].doc_num - last_major;
+		  BBLOCK_ENCODE_L (num, odn_blk, bits_out);
+		  last_major = ii[j + out_k - 1].doc_num;
+
+		  num = ii[j + out_k - 1].bits_so_far + ii[j + out_k - 1].count_bits -
+		    bits_so_far;
+		  BBLOCK_ENCODE_L (num, olen_blk, bits_out);
+		  bits_so_far = ii[j + out_k].bits_so_far;
+		}
+	    }
+	  if (!(out_k && kd == out_k - 1 && j != p - 1))
+	    BBLOCK_ENCODE_L (ii[j].doc_num - doc_num, blk, bits_out);
+	  doc_num = ii[j].doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    GAMMA_ENCODE_L (ii[j].count, bits_out);
+	}
+
+      /* write till a byte boundary */
+      while (__btg != 8)
+	{
+	  ENCODE_BIT (0);
+	  bits_out++;
+	}
+      ENCODE_PAUSE (out_buf)
+
+	Xfree (ii);
+
+    }
+  ENCODE_CONTINUE (out_buf)
+    ENCODE_DONE
+
+    HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+  NTOHUL(bytes_out);
+
+  fclose (idx);
+  fclose (odx);
+  fclose (in);
+  fclose (out);
+  fclose (dict);
+
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_merge.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_merge.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_merge.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mgmerge.h --- mgmerge #defines, etc
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edtited: November 12 1994
+ *
+ **************************************************************************/
+
+#ifndef MGMERGE_H
+#define MGMERGE_H
+
+typedef unsigned char byte;
+
+#define OLD 0
+#define NEW 1
+#define MERGE 2
+
+#define OK 0
+#define ERROR_MGMERGE 1
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_passes.c	(revision 21112)
@@ -0,0 +1,639 @@
+/**************************************************************************
+ *
+ * mg_passes.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include <stdlib.h>
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.5  2004/11/29 02:56:09  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.4  2003/04/18 01:41:46  sjboddie
+   Added a new -M option to mg_passes, allowing "maxnumeric" to be altered.
+
+   Revision 1.3  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.2  2001/06/12 23:23:42  jrm21
+   fixed a bug where mg_passes segfaults when trying to print the usage message.
+
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
+"  %*s [-a stemmer] [-M max-numeric] -f doc-collection-name\n";
+
+
+static void 
+usage (char *err)
+{
+  if (err)
+    Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "",strlen (msg_prefix), "",
+	   strlen (msg_prefix),"");
+  exit (1);
+}
+
+
+
+
+#if 0
+static char *
+str_comma (unsigned long u)
+{
+  static char buf[20];
+  unsigned long a, b, c, d;
+  a = u / 1000000000;
+  u -= a * 1000000000;
+  b = u / 1000000;
+  u -= b * 1000000;
+  c = u / 1000;
+  u -= c * 1000;
+  d = u;
+
+  if (a)
+    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
+  else if (b)
+    sprintf (buf, "%u,%03u,%03u", b, c, d);
+  else if (c)
+    sprintf (buf, "%u,%03u", c, d);
+  else
+    sprintf (buf, "%u", d);
+  return (buf);
+}
+#endif
+
+
+
+
+int 
+open_next_file (int in_fd)
+{
+  if (in_fd > 0)
+    close (in_fd);
+  if (num_files == 0)
+    return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  files++;
+  num_files--;
+  return (in_fd);
+}
+
+
+static void 
+driver (int in_fd, FILE * Trace, char *file_name)
+{
+  int pass, num = 1;
+
+  char *buffer = Xmalloc (buf_size);
+  unsigned long num_docs = 0;
+  unsigned long block_bytes = 0;
+  register int buf_left = buf_size;
+  register char *look_pos = buffer;
+  register char *end_pos = buffer;
+
+  ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+  GetTime (&StartTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->init_time);
+#endif
+      }
+
+  GetTime (&InitTime);
+  for (;;)
+    {
+      int len = 0;
+      char *base = look_pos;
+
+      while (look_pos != end_pos && *look_pos != TERMRECORD)
+	look_pos++;
+
+      while (look_pos == end_pos)
+	{
+	  if (buf_left < MIN_BUF)
+	    {
+	      bcopy (base, buffer, end_pos - base);
+	      look_pos = buffer + (end_pos - base);
+	      buf_left = buf_size - (end_pos - base);
+	      end_pos = look_pos;
+	      base = buffer;
+	    }
+	  if (buf_left)
+	    {
+    	      num = read (in_fd, end_pos, buf_left);
+	      if (num < 0) num = 0; /* RJM - quick hack :-) */
+	      if (num == 0)
+		if ((in_fd = open_next_file (in_fd)) != -1)
+		  num = read (in_fd, end_pos, buf_left);
+	      bytes_received += num;
+	      buf_left -= num;
+	      end_pos += num;
+	    }
+	  while (look_pos < end_pos && *look_pos != TERMRECORD)
+	    look_pos++;
+	  if (buf_left == 0 && base == buffer && look_pos == end_pos)
+	    {
+	      Message ("Unable to find document terminator (i.e ^B)"
+		       " in the document");
+	      FatalError (1, "The document is in excess of %d chars long",
+			  look_pos - base);
+	    }
+	  if (!num)
+	    break;
+	}
+      len = look_pos++ - base;
+
+      if (!num && base == end_pos)
+	break;
+
+      bytes_processed += len;
+
+#ifndef QUIET
+      if (!len)
+	Message ("Warning : Processing zero length document");
+#endif
+
+      for (pass = 0; pass < MAX_PASSES; pass++)
+	if (Passes & (1 << pass))
+	  {
+	    register pass_data *pd = &PassData[pass];
+
+#ifdef HAVE_TIMES
+	    struct tms tims;
+	    times (&tims);
+	    pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    struct rusage ru;
+	    register struct timeval *tv = &pd->process_time;
+
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+
+	    if (pd->process ((u_char *) base, len) == COMPERROR)
+	      {
+		Message ("Error during processing of \"%s\"", pd->name);
+		if (Dump || Trace)
+		  {
+		    int i;
+		    FILE *f = Trace ? Trace : stderr;
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		    for (i = 0; i < len; i++)
+		      {
+			char ch = base[i];
+			if (ch == '\1' || ch == '\2')
+			  ch = '\n';
+			putc (ch, f);
+		      }
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		  }
+		if (Trace)
+		  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+			   bytes_processed, num_docs,
+			   ElapsedTime (&StartTime, NULL));
+		exit (1);
+	      }
+
+#ifdef HAVE_TIMES
+	    times (&tims);
+	    pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	  }
+      num_docs++;
+      if (Trace)
+	{
+	  block_bytes += (look_pos - base);
+	  if (block_bytes >= trace)
+	    {
+#ifdef HAVE_MALLINFO
+	      struct mallinfo mi;
+	      mi = mallinfo ();
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		       ElapsedTime (&StartTime, NULL));
+#else
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+#endif
+	    }
+	}
+      if (!num && look_pos - 1 == end_pos)
+	break;
+    }
+
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  free (buffer);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  int ch, in_fd;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:M:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'M':
+	  SetEnv ("maxnumeric", optarg, NULL);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc)
+    {
+      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+	FatalError (1, "Cannot open %s", argv[optind]);
+      files = &argv[optind + 1];
+      num_files = argc - (optind + 1);
+    }
+  else
+    in_fd = 0;			/* stdin */
+
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (in_fd, Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_perf_hash_build.c	(revision 21112)
@@ -0,0 +1,183 @@
+/**************************************************************************
+ *
+ * mg_perf_hash_build.c -- Program to build a perfect hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:13  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:15  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:58  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:53  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#define POOL_SIZE 1024*1024
+
+static void process_files (char *filename);
+
+int r = -1;
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'r':
+	r = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *dict, *hash;
+  unsigned long i;
+  u_char prev[MAXSTEMLEN + 1];
+  struct invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  int pool_left;
+  u_char **starts;
+
+
+  dict = open_file (filename, INVF_DICT_SUFFIX, "rb",
+		    MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  hash = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+		      MAGIC_HASH, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (!(pool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = Xmalloc (sizeof (u_char *) * idh.dict_size)))
+    FatalError (1, "Out of memory");
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff, l;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (dict);
+      suff = getc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, dict);
+      fread (&wcnt, sizeof (wcnt), 1, dict);
+
+      l = *prev + 1;
+      if (pool_left < l)
+	{
+	  pool = Xmalloc (POOL_SIZE);
+	  pool_left = POOL_SIZE;
+	}
+      starts[i] = pool;
+      bcopy ((char *) prev, (char *) pool, l);
+      pool += l;
+      pool_left -= l;
+    }
+  if (!(phd = gen_hash_func (idh.dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hash, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (dict);
+  fclose (hash);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_stem_idx.c	(revision 21112)
@@ -0,0 +1,892 @@
+/**************************************************************************
+ *
+ * mg_stem_idx.c -- Memory efficient stem index builder
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"     /* For struct stemmed_dict */
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+#include "local_strings.h"
+
+typedef struct PosEntry
+{
+  unsigned int num_cases;
+  unsigned int blk;
+  unsigned short blk_index;
+  unsigned short offset;
+}
+PosEntry;
+
+
+typedef struct PosList
+{
+  unsigned int list_size;
+  unsigned int num_entries;
+  PosEntry PE[1];
+}
+PosList;
+
+
+typedef struct idx_hash_rec
+{
+  u_char *word;
+  PosList *PL;
+}
+idx_hash_rec;
+
+
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+
+static idx_hash_rec **IdxHashTable;
+static unsigned long IdxHashSize;
+static unsigned long IdxHashUsed;
+static u_char *IdxPool;
+static int IdxPoolLeft;
+
+static idx_hash_rec *idx_hr_pool;
+static int idx_hr_PoolLeft;
+
+static idx_hash_rec **idx_first_occr;
+static int idx_max_first_occr;
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static long lookback = 4;
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+/* =========================================================================
+ * Function: MakePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+PosList *
+MakePosList (int n)
+{
+  PosList *pl;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  pl = Xmalloc (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+  if (!pl)
+    FatalError (1, "Unable to allocate term list");
+  ChangeMem (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+
+  pl->num_entries = n;
+  pl->list_size = list_size;
+
+  return pl;
+}
+
+/* =========================================================================
+ * Function: ResizePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizePosList (PosList ** pos_list)
+{
+  PosList *pl = *pos_list;
+
+  ChangeMem (-(sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)));
+  if (pl->num_entries > pl->list_size)
+    {
+      if (pl->list_size)
+	pl->list_size *= GROWTH_FACTOR;
+      else
+	pl->list_size = MIN_SIZE;
+    }
+  pl = Xrealloc (pl, sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  if (!pl)
+    FatalError (1, "Unable to resize pos list");
+  ChangeMem (sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  *pos_list = pl;
+}
+
+
+/* =========================================================================
+ * Function: FreePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreePosList (PosList ** the_pl)
+{
+  PosList *pl = *the_pl;
+
+  ChangeMem (-(sizeof(PosList) + sizeof (PosEntry) * (pl->list_size - 1)));
+  Xfree (pl);
+
+  *the_pl = NULL;
+}
+
+
+/* =========================================================================
+ * Function: ResetPosList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetPosList (PosList ** pl)
+{
+  if (*pl)
+    FreePosList (pl);
+  *pl = MakePosList (0);
+}
+
+/* =========================================================================
+ * Function: AddPosEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddPosEntry (PosList ** pos_list, PosEntry * pe)
+{
+  PosList *pl = *pos_list;
+
+  pl->num_entries++;
+  ResizePosList (pos_list);
+  pl = *pos_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) pe, (char *) &(pl->PE[pl->num_entries - 1]), sizeof (PosEntry));
+
+  return pl->num_entries - 1;
+}
+
+
+/* Modified from stem_search.c */
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    FatalError (1, "Could not allocate memory for stemmed dict");
+
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stemmer_num);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  return sd;
+}
+
+
+void
+mg_init_process ()
+{
+  /* Allocate memory for idx hash table */
+
+  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  IdxPoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec))))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  idx_hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+
+  IdxHashSize = INITIAL_HASH_SIZE;
+  IdxHashUsed = 0;
+  if (!(IdxHashTable = Xmalloc (sizeof (idx_hash_rec *) * IdxHashSize)))
+    FatalError (1, "Unable to allocate memory for idx table");
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashSize);
+  bzero ((char *) IdxHashTable, sizeof (idx_hash_rec *) * IdxHashSize);
+  idx_max_first_occr = 8192;
+  if (!(idx_first_occr = Xmalloc (sizeof (idx_hash_rec *) * idx_max_first_occr)))
+    FatalError (1, "Unable to allocate memory for idx_first_occur");
+  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+}
+
+
+void
+PackIdxHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < IdxHashSize; s++)
+    if (IdxHashTable[s])
+      IdxHashTable[d++] = IdxHashTable[s];
+  ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashUsed);
+  if (!(IdxHashTable = Xrealloc (IdxHashTable, sizeof (idx_hash_rec *) * IdxHashUsed)))
+    FatalError (1, "Out of memory");
+  IdxHashSize = IdxHashUsed;
+}
+
+
+void
+process_stem_dict (stemmed_dict * sd, int stem_method)
+{
+  int block;
+  short blk_index = -1;
+  int wordnum = -1;
+  PosEntry *prevPE = NULL;
+  idx_hash_rec *prevIdx = NULL;
+  u_char word[MAXSTEMLEN + 1];
+  u_char prev[MAXSTEMLEN + 1];
+
+  /* For each block in stem dict... */
+  for (block = 0; block < sd->sdh.num_blocks; block++)
+    {
+      register unsigned int res;
+      int num_indexes;
+      unsigned long *first_word, *last_invf_len;
+      unsigned short *num_words;
+      u_char *base;
+      unsigned short *index;
+
+      /* Read block into buffer */
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* Move through block header */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      {
+	/* [RPAP - Jan 97: Endian Ordering] */
+	int i;
+	for (i = 0; i < num_indexes; i++)
+	  NTOHUS(index[i]);
+      }
+
+      base = (u_char *) (index + num_indexes);
+      blk_index = -1;
+
+      /* For each word in block... */
+      for (res = 0; res < *num_words; res++)
+	{
+	  unsigned copy, suff;
+	  register idx_hash_rec *idx_ent = 0;
+
+	  /* Update blk_index */
+	  if (!(res % sd->sdh.lookback))
+	    blk_index++;
+
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+
+	  /* Skip irrelevant word info */
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+
+	  /* Stem word */
+	  bcopy ((char *) prev, (char *) word, *prev + 1);
+	  stemmer (stem_method, sd->sdh.stemmer_num, word);
+
+	  /* Check if word follows straight on from previous word */
+	  if (prevIdx && !compare (word, prevIdx->word))
+	    prevPE->num_cases++;
+
+	  else 
+	    {
+	      /* Search the idx hash table for word */
+	      register unsigned long hashval, step;
+	      register int hsize = IdxHashSize;
+	      HASH (hashval, step, word, hsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  idx_ent = IdxHashTable[hashval];
+		  if (!idx_ent)
+		    {
+		      /* Create a next entry in the hash table */
+		      int len = *word + 1;
+		      if (!idx_hr_PoolLeft)
+			{
+			  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE *
+						       sizeof (idx_hash_rec))))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  idx_hr_PoolLeft = HASH_POOL_SIZE;
+			  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+			}
+		      idx_ent = idx_hr_pool++;
+		      idx_hr_PoolLeft--;
+		      if (len > IdxPoolLeft)
+			{
+			  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  IdxPoolLeft = POOL_SIZE;
+			  ChangeMem (POOL_SIZE);
+			}
+		      wordnum++;
+
+		      idx_ent->word = IdxPool;
+		      idx_ent->PL = MakePosList (0);
+		      {
+			PosEntry PE;
+			PE.num_cases = 1;
+			PE.blk = block;
+			PE.blk_index = (unsigned short) blk_index;
+			PE.offset = res % sd->sdh.lookback;
+			AddPosEntry (&(idx_ent->PL), &PE);
+		      }
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+
+		      bcopy ((char *) word, (char *) IdxPool, len);
+		      IdxPool += len;
+		      IdxPoolLeft -= len;
+		      if (IdxHashUsed == idx_max_first_occr - 1)
+			{
+			  ChangeMem (-sizeof (idx_hash_rec *) * idx_max_first_occr);
+			  idx_max_first_occr *= 2;
+			  if (!(idx_first_occr = Xrealloc (idx_first_occr, sizeof (idx_hash_rec *) *
+							   idx_max_first_occr)))
+			    FatalError (1, "Unable to allocate memory for idx_first_occr");
+			  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+			}
+		      idx_first_occr[IdxHashUsed] = idx_ent;
+		      IdxHashUsed++;
+		      IdxHashTable[hashval] = idx_ent;
+		      break;
+		    } /* if (!idx_ent) */
+		  
+		  /* Compare the words */
+		  s1 = word;
+		  s2 = idx_ent->word;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+		  
+		  if (len)
+		    {
+		      /* Entry is not the right one - move to next hash index */
+		      hashval = (hashval + step);
+		      if (hashval >= hsize)
+			hashval -= hsize;
+		    }
+		  else
+		    {
+		      /* Entry is correct - added PosEntry to word */
+		      PosEntry PE;
+		      PE.num_cases = 1;
+		      PE.blk = block;
+		      PE.blk_index = (unsigned short) blk_index;
+		      PE.offset = res % sd->sdh.lookback;
+		      AddPosEntry (&(idx_ent->PL), &PE);
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+		      break;
+		    }
+		} /* for (;;) */
+	    } /* else (the word does not follow on from previous one) */
+
+	  if (IdxHashUsed >= IdxHashSize >> 1)
+	    {
+	      idx_hash_rec **ht;
+	      unsigned long size;
+	      unsigned long i;
+	      size = prime (IdxHashSize * 2);
+	      if (!(ht = Xmalloc (sizeof (idx_hash_rec *) * size)))
+		FatalError (1, "Unable to allocate memory for idx table");
+	      bzero ((char *) ht, sizeof (idx_hash_rec *) * size);
+	      ChangeMem (sizeof (idx_hash_rec *) * size);
+
+	      for (i = 0; i < IdxHashSize; i++)
+		if (IdxHashTable[i])
+		  {
+		    register u_char *wptr;
+		    idx_hash_rec *ent;
+		    register unsigned long hashval, step;
+		    
+		    wptr = IdxHashTable[i]->word;
+		    HASH (hashval, step, wptr, size);
+		    ent = ht[hashval];
+		    while (ent)
+		      {
+			hashval += step;
+			if (hashval >= size)
+			  hashval -= size;
+			ent = ht[hashval];
+		      }
+		    ht[hashval] = IdxHashTable[i];
+		  }
+	      Xfree (IdxHashTable);
+	      ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+	      IdxHashTable = ht;
+	      IdxHashSize = size;
+	    }
+	  
+	} /* end for each word */
+
+    } /* end for each block */
+
+}
+ 
+static int 
+idx_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((idx_hash_rec **) A))->word;
+  u_char *s2 = (*((idx_hash_rec **) B))->word;
+  return (casecompare (s1, s2));
+}
+
+
+void
+save_idx (char * filename, int stem_method)
+{
+  char *FName;
+  unsigned long i, j, pos, First_word, num;
+  struct stem_idx_header sih;
+  u_char *buffer, *last_word = NULL;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+  FILE *idbi = NULL, *tmp = NULL;
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  /* Create appropriate stem index file */
+  switch (stem_method)
+    {
+    case (1):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "wb", MAGIC_STEM_1,
+			    MG_ABORT);
+	break;
+      }
+
+    case (2):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "wb", MAGIC_STEM_2,
+			    MG_ABORT);
+	break;
+      }
+    case (3):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "wb", MAGIC_STEM_3,
+			    MG_ABORT);
+	break;
+      }
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT create .invf.blocked.%d file", stem_method);
+
+  PackIdxHashTable();
+  qsort (IdxHashTable, IdxHashUsed, sizeof (idx_hash_rec *), idx_comp);
+
+  sih.lookback = lookback;
+  sih.block_size = block_size;
+  sih.num_blocks = 0;
+  sih.blocks_start = 0;
+  sih.index_chars = 0;
+  sih.num_of_words = IdxHashUsed;
+
+  fwrite ((char *) &sih, sizeof (sih), 1, idbi);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+
+  /* For each word in the hashtable... */
+  for (i = 0; i < IdxHashUsed; i++)
+    {
+      register unsigned long extra, copy, suff;
+      register struct idx_hash_rec *ent = IdxHashTable[i];
+
+      /* build a new word on top of prev */
+      if (last_word != NULL)
+	copy = prefixlen (last_word, ent->word);
+      else
+	copy = 0;
+      suff = *(ent->word) - copy;
+      last_word = ent->word;
+
+      if (word_num % sih.lookback == 0)
+	/* Will need copy chars to add + a pointer in index */
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + 
+	  ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	{
+	  /* Dump buffer to tmp file */
+	  int chunk;
+	  HTONUL(First_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUS(word_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sih.num_blocks++;
+
+	  /* Check that entry will fit into new block */
+	  if (sizeof (*pointers) + sizeof (ptrs_in_use) + extra + sizeof (First_word) +
+	      suff + 1 + sizeof (ent->PL->num_entries) + 
+	      ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	    FatalError (1, "Block size to small");
+
+	} /* if size > block_size */
+
+      if (word_num % sih.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+
+      /* Output Word information */
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (ent->word + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &(ent->PL->num_entries), (char *) (buffer + buf_in_use), sizeof (ent->PL->num_entries));
+      NTOHUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (ent->PL->num_entries);
+
+      for (j = 0; j < ent->PL->num_entries; j++)
+	{
+	  register PosEntry *pe = &(ent->PL->PE[j]);
+	  HTONUI(pe->num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->num_cases), (char *) (buffer + buf_in_use), sizeof (pe->num_cases));
+	  buf_in_use += sizeof (pe->num_cases);
+	  HTONUI(pe->blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk), (char *) (buffer + buf_in_use), sizeof (pe->blk));
+	  buf_in_use += sizeof (pe->blk);
+	  HTONUS(pe->blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk_index), (char *) (buffer + buf_in_use), sizeof (pe->blk_index));
+	  buf_in_use += sizeof (pe->blk_index);
+	  HTONUS(pe->offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->offset), (char *) (buffer + buf_in_use), sizeof (pe->offset));
+	  buf_in_use += sizeof (pe->offset);
+	}
+
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 64209258\n");
+
+      if (word_num == 0) /* the first word in each block - word_num set to 0 when buffer written to tmp file */
+	{
+	  /* Write word to main index */
+	  fwrite (ent->word, sizeof (u_char), *(ent->word) + 1, idbi);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idbi);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sih.index_chars += *(ent->word) + 1;
+	  First_word = i;
+	}
+      word_num++;
+    } /* end for each word */ /* for each hash entry */
+
+  if (buf_in_use)
+    {
+      /* Write last buffer to tmp file */
+      int chunk;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sih.num_blocks++;
+    }
+
+  rewind (tmp);
+  sih.blocks_start = sih.index_chars + sizeof (u_long) + sizeof (sih) +
+    sih.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sih.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idbi);
+	  sih.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idbi);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idbi, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sih.lookback);
+  HTONUL(sih.block_size);
+  HTONUL(sih.num_blocks);
+  HTONUL(sih.blocks_start);
+  HTONUL(sih.index_chars);
+  HTONUL(sih.num_of_words);
+
+  fwrite (&sih, sizeof (sih), 1, idbi);
+  fclose (idbi);
+
+#ifndef SILENT
+  Message ("Stem %d:\n", stem_method);
+  Message ("     Block size   : %10d\n", block_size);
+  Message ("     Num_blocks   : %10d\n", NTOHUL(sih.num_blocks));  /* [RPAP - Jan 97: Endian Ordering] */
+  Message ("     Max mem used : %10.1f Mb\n", (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("     Num_of_words : %10d\n", NTOHUL(sih.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+#endif
+}
+
+
+void
+UpdateStemDict (char * filename, int stem_method)
+{
+  FILE *idb;
+  struct stem_dict_header sdh;
+
+  if (!(idb = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "r+b",
+			 MAGIC_STEM, MG_CONTINUE)))
+    FatalError (1, "Could not update stemmed dict");
+
+  fread ((char *) &sdh, sizeof (sdh), 1, idb);
+  NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+  sdh.indexed |= 1 << (stem_method - 1);
+  HTONUL(sdh.indexed);
+  fseek (idb, sizeof (u_long), 0);
+  fwrite ((char *) &sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+}
+
+
+
+/* Main */
+int main (int argc, char **argv)
+{
+  File *idb;    /* File to .invf.dict.blocked */
+  char *filename = "";
+  stemmed_dict *sd;   /* Stemmed dictionary */
+  int ch;
+  char path[512];
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:b:hFs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d directory] "
+		 "[-b num] [-F] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  if (stem_method < 1 || stem_method > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  /* Open required stem dict file */
+  sprintf (path, FILE_NAME_FORMAT, get_basepath (), filename, INVF_DICT_BLOCKED_SUFFIX);
+  if (!(idb = Fopen (path, "rb", MAGIC_STEM)))
+    FatalError (1, "Unable to open \"%s\"", path);
+
+  /* Read in idb header and index to blocks */
+  if (!(sd = ReadStemDictBlk (idb)))
+    FatalError (1, "Could not read stemmed dictionary");
+
+  /* Process stemmed dictionary */
+  mg_init_process ();
+  process_stem_dict (sd, stem_method);
+  save_idx (filename, stem_method);
+ 
+  /* Close stemmed dict */
+  Fclose (idb);
+
+  /* Update stemmed dict */
+  UpdateStemDict (filename, stem_method);
+
+  return 0;
+}
+
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_estimate.c	(revision 21112)
@@ -0,0 +1,581 @@
+/**************************************************************************
+ *
+ * mg_text_estimate.c -- Program to estimate the compressed text size
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "comp_dict.h"
+#include "hash.h"
+
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:14  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:48  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:19  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:59  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:54  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+typedef struct stats_t
+  {
+    u_long freq, occur;
+    u_long len;
+  }
+stats_t;
+
+static u_long huff_count = 0;
+static u_long escape_count = 0;
+
+
+char mode = 'H';
+
+/* [RJM 07/97: 4G limit] */
+static double ReadInWordsStandard (char *name, double *num_bytes);
+static double ReadInWordsSpecial (char *name, double *num_bytes,
+				  double *num_extra_bytes,
+				  u_long *aux_compressed);
+
+
+
+
+int main (int argc, char **argv)
+{
+  char *stats_dict = NULL, *comp_dict = NULL;
+  ProgTime StartTime;
+  double num_bytes, num_extra_bytes = 0; /* [RJM 07/97: 4G limit] */
+  u_long aux_compressed;
+  int ch;
+  double bytes;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "BMDYHh")) != -1)
+    switch (ch)
+      {
+      case 'B':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = ch;
+	break;
+      case 'M':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = 'Y';
+	break;
+      case 'D':
+      case 'Y':
+      case 'H':
+	mode = ch;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    stats_dict = argv[optind++];
+  else
+    stats_dict = NULL;
+  if (optind < argc)
+    comp_dict = argv[optind++];
+  else
+    {
+      fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+	       argv[0]);
+      exit (1);
+    }
+
+
+
+  GetTime (&StartTime);
+
+  if (LoadCompressionDictionary (comp_dict) == COMPERROR)
+    exit (1);
+
+  if (mode == 'H')
+    bytes = ReadInWordsStandard (stats_dict, &num_bytes);
+  else
+    bytes = ReadInWordsSpecial (stats_dict, &num_bytes, &num_extra_bytes,
+				&aux_compressed);
+
+
+  Message ("Compressed Text %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   bytes / 1024.0 / 1024, bytes, bytes * 100 / num_bytes);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / num_bytes);
+  if (mode == 'H')
+    Message ("Huffman info for chars in dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	     Chars_disk / 1024.0 / 1024, Chars_disk * 1.0,
+	     (Chars_disk * 100.0) / num_bytes);
+  else
+    {
+      Message ("Aux dictionary (Uncompressed) %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       num_extra_bytes / 1024.0 / 1024, num_extra_bytes * 1.0,
+	       (num_extra_bytes * 100.0) / num_bytes);
+      Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	       (aux_compressed * 100.0) / num_bytes);
+    }
+  Message ("Words and non-words in the text %u", huff_count + escape_count);
+  Message ("Words and non-words coded by escapes %u ( %0.3f %%)", escape_count,
+	   escape_count * 100.0 / huff_count);
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+
+static double 
+ReadInWordsStandard (char *name, double *num_bytes)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long which;
+  double bytes, bits = 0;
+
+  f = open_named_file (name, "rb", MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      frags_stats_header fsh;
+      int j;
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, freq, res, occur_num;
+	  u_char Word[16];
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(freq);
+	  NTOHSI(occur_num);
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      escape_count += freq;
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	      if (cdh.dict_type == MG_SEED_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * freq;
+	      huff_count += freq;
+	    }
+	}
+    }
+  fclose (f);
+
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
+
+
+static int 
+stats_comp (const void *a, const void *b)
+{
+  return ((stats_t *) a)->occur - ((stats_t *) b)->occur;
+}
+
+
+
+
+
+static double 
+ReadInWordsSpecial (char *name, double *num_bytes,
+		    double *num_extra_bytes,
+		    u_long *aux_compressed)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long magic, which;
+  double bytes, bits = 0;
+  double size[2];
+
+  if (!(f = fopen (name, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open file \"%s\"\n", name);
+
+  if (fread (&magic, sizeof (magic), 1, f) != 1)
+    FatalError (1, "Unable to read magic number");
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (magic != MAGIC_STATS_DICT)
+    FatalError (1, "Bad magic number");
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      stats_t *stats;
+      frags_stats_header fsh;
+      int j, num = 0;
+      long lens[16], chars[256];
+
+      bzero ((char *) lens, sizeof (lens));
+      bzero ((char *) chars, sizeof (chars));
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      stats = Xmalloc (fsh.num_frags * sizeof (*stats));
+      if (!stats)
+	FatalError (1, "Unable to allocate memory for stats");
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, res;
+	  u_char Word[16];
+	  fread (&stats[num].freq, sizeof (stats[num].freq), 1, f);
+	  fread (&stats[num].occur, sizeof (stats[num].occur), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(stats[num].freq);
+	  NTOHSI(stats[num].occur);
+
+	  stats[num].len = len + 1;
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  assert (stats[num].freq > 0);
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if ((cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		   cdh.dict_type == MG_SEED_DICTIONARY) &&
+		  ht[which])
+		{
+		  int k;
+		  res = ht[which]->hd->num_codes - 1;
+		  bits += ht[which]->hd->clens[res] * stats[num].freq;
+		  lens[Word[0]]++;
+		  for (k = 0; k < (int) Word[0]; k++)
+		    chars[Word[k + 1]]++;
+		}
+	      num++;
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * stats[num].freq;
+	    }
+	}
+
+      {
+	int j, num_buckets = 0;
+	int start[10], end[10];
+	long flens[16], fchars[256];
+
+	for (j = 0; j < 256; j++)
+	  if (!chars[j] && PESINAWORD (j) == which)
+	    fchars[j] = 1;
+	  else
+	    fchars[j] = chars[j];
+	for (j = 0; j < 16; j++)
+	  if (!lens[j])
+	    flens[j] = 1;
+	  else
+	    flens[j] = lens[j];
+
+	size[which] = Calculate_Huffman_Size (16, flens, lens) +
+	  Calculate_Huffman_Size (256, fchars, chars);
+
+
+	qsort (stats, num, sizeof (*stats), stats_comp);
+
+
+	if (mode == 'Y')
+	  {
+	    int k;
+	    num_buckets = 1;
+	    start[0] = 0;
+	    end[0] = cdh.num_words[which] - 1;
+	    Message ("There are %d novel %swords", num, which ? "" : "non-");
+	    while (end[num_buckets - 1] < num)
+	      {
+		start[num_buckets] = end[num_buckets - 1] + 1;
+		end[num_buckets] = start[num_buckets] +
+		  (end[num_buckets - 1] - start[num_buckets - 1]) * 2;
+		num_buckets++;
+	      }
+	    for (k = 0; k < num_buckets; k++)
+	      Message ("Bucket %d   %d <= x <= %d\n", k + 1, start[k], end[k]);
+	  }
+
+	for (j = 0; j < num; j++)
+	  {
+	    (*num_extra_bytes) += stats[j].len;
+	    switch (mode)
+	      {
+	      case 'D':
+		bits += BIO_Delta_Length (j + 1) * stats[j].freq;
+		break;
+	      case 'B':
+		bits += BIO_Binary_Length (j + 1, num) * stats[j].freq;
+		break;
+	      case 'Y':
+		{
+		  int k = 0;
+		  while (j > end[k])
+		    k++;
+		  if (!(j - start[k] + 1 >= 1 &&
+			j - start[k] + 1 <= end[k] - start[k] + 1))
+		    {
+		      Message ("k = %d, j = %d", k, j);
+		      assert (j - start[k] + 1 >= 1 &&
+			      j - start[k] + 1 <= end[k] - start[k] + 1);
+		    }
+		  else
+		    {
+		      bits += (BIO_Gamma_Length (k + 1) +
+			       BIO_Binary_Length (j - start[k] + 1,
+						  end[k] - start[k] + 1)) *
+			stats[j].freq;
+		    }
+		}
+	      }
+	  }
+	if (mode == 'B' && num)
+	  bits += BIO_Delta_Length (num) * csh.num_docs;
+      }
+      Xfree (stats);
+    }
+  fclose (f);
+
+  *aux_compressed = (size[0] + size[1]) / 8;
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_text_merge.c	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * mg_text_merge.c --- merge  *.text, *.text.idx files
+ *                     part of the mgmerge utility
+ * Copyright (C) 1995 Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edited:  November 11 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "text.h"
+
+
+/**** GLOBALS ****/
+FILE *text[3], *idx[3];
+
+typedef char FileName[256];
+FileName old_name, new_name, merge_name;
+
+long magicsize;			/* == where the header in a file begins */
+
+compressed_text_header cth[3];
+
+/*=======================================================================
+ * init_merge_text(): open files, set up global variables, etc
+ *=======================================================================*/
+int 
+init_merge_text ()
+{
+
+  /* open .text files */
+  text[OLD] = open_file (old_name, TEXT_SUFFIX, "r+b",
+			 MAGIC_TEXT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (text[OLD]);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, text[OLD]);
+
+  text[NEW] = open_file (new_name, TEXT_SUFFIX, "rb+",
+			 MAGIC_TEXT, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, text[NEW]);
+
+  /* open .text.idx files */
+  idx[OLD] = open_file (old_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, idx[OLD]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[OLD].num_of_docs);
+  NTOHD(cth[OLD].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[OLD].num_of_words);
+  NTOHUL(cth[OLD].length_of_longest_doc);
+  NTOHD(cth[OLD].ratio);
+
+  idx[NEW] = open_file (new_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, idx[NEW]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[NEW].num_of_docs);
+  NTOHD(cth[NEW].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[NEW].num_of_words);
+  NTOHUL(cth[NEW].length_of_longest_doc);
+  NTOHD(cth[NEW].ratio);
+
+  idx[MERGE] = create_file (merge_name, TEXT_IDX_SUFFIX, "wb",
+			    MAGIC_TEXI, MG_ABORT);
+  return OK;
+}
+
+
+/*=======================================================================
+ * process_merge_text(): merge the files
+ *=======================================================================*/
+int 
+process_merge_text (void)
+{
+  int i;
+  u_long data, offset;
+  byte c;
+
+  /* update and write merged header to .text and .text.idx files */
+  /* they have the exact same header */
+  cth[MERGE].num_of_docs = cth[OLD].num_of_docs
+    + cth[NEW].num_of_docs;
+  cth[MERGE].num_of_bytes = cth[OLD].num_of_bytes
+    + cth[NEW].num_of_bytes;
+  cth[MERGE].num_of_words = cth[OLD].num_of_words;
+  cth[MERGE].length_of_longest_doc =
+    (cth[OLD].length_of_longest_doc > cth[NEW].length_of_longest_doc
+     ? cth[OLD].length_of_longest_doc
+     : cth[NEW].length_of_longest_doc);
+  cth[MERGE].ratio = ((cth[OLD].num_of_bytes * cth[OLD].ratio) +
+		      (cth[NEW].num_of_bytes * cth[NEW].ratio))
+    / cth[MERGE].num_of_bytes;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth[MERGE].num_of_docs);
+  HTOND(cth[MERGE].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth[MERGE].num_of_words);
+  HTONUL(cth[MERGE].length_of_longest_doc);
+  HTOND(cth[MERGE].ratio);
+
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, idx[MERGE]);
+  fseek (text[OLD], magicsize, 0);
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, text[OLD]);
+
+  /*
+   *  Update *.text.idx:  need to know where each new doc starts
+   *  in the appended .text file
+   */
+  for (i = 0; i < cth[OLD].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[OLD]);
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+
+  /* offset is the amount to add to each entry from idx[NEW] */
+  fread (&offset, sizeof (u_long), 1, idx[OLD]);
+  NTOHUL(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  offset -= (4 + sizeof (cth[OLD]));	/* 4 for the magic number */
+
+  for (i = 0; i < cth[NEW].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[NEW]);
+      NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      data += offset;
+      HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+  /* write last u_long in idx[MERGE] (= length of file) */
+  fread (&data, sizeof (u_long), 1, idx[NEW]);
+  NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  data += offset;
+  HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+
+/******* update .text *******/
+  /* simply cat's the files together, except for the headers 
+   * and magic numbers, of course
+   */
+  fseek (text[OLD], 0L, 2);
+  while (!feof (text[NEW]))
+    {
+      fread (&c, sizeof (c), 1, text[NEW]);
+      if (!feof (text[NEW]))
+	fwrite (&c, sizeof (c), 1, text[OLD]);
+    }
+
+  return OK;
+}
+
+
+
+/*=======================================================================
+ * done_merge_text(): close files.
+ *=======================================================================*/
+int 
+done_merge_text (void)
+{
+  fclose (idx[MERGE]);
+  fclose (text[OLD]);
+  fclose (idx[OLD]);
+  fclose (text[NEW]);
+  fclose (idx[NEW]);
+
+  fprintf (stderr, "mg_text_merge: %ld documents added to %s\n",
+	   cth[NEW].num_of_docs, merge_name);
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s -f collection_name\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int main (int argc, char *argv[])
+{
+  char *progname;
+  ProgTime start;
+  int ch;			/* for command line processing */
+
+  progname = argv[0];
+  msg_prefix = argv[0];
+  merge_name[0] = '\0';
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (merge_name, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (merge_name[0] == '\0')
+    usage (progname);
+  strcpy (old_name, merge_name);
+  strcat (old_name, ".old");
+  strcpy (new_name, merge_name);
+  strcat (new_name, ".new");
+
+  GetTime (&start);
+  init_merge_text ();
+  process_merge_text ();
+  done_merge_text ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mg_weights_build.c	(revision 21112)
@@ -0,0 +1,508 @@
+/**************************************************************************
+ *
+ * mg_weights_build.c -- Program to build the document weights file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:16  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:22  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:05  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:00  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:55  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+unsigned char bits = 8;
+static char *file_name = "";
+static char *text_file_name = "";
+static unsigned long NumPara = 0;
+static unsigned long StaticNumOfDocs = 0;
+
+unsigned long get_NumPara (void);
+unsigned long get_StaticNumOfDocs (void);
+void GenerateWeights (void);
+void Make_weight_approx (void);
+void Make_text_idx_wgt (void);
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:t:d:b:sh")) != -1) /* [RJM 10/98 - Text Filename] */
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	if (strlen(text_file_name) == 0) text_file_name = optarg;
+	break;
+      /* [RJM 10/98 - Text Filename] */
+      case 't':		/* text input file */
+	text_file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	bits = atoi (optarg);
+	if (bits > 32)
+	  {
+	    fprintf (stderr, "b may only take values 0-32\n");
+	    exit (1);
+	  }
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b bits] [-s] [-h]\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  GenerateWeights ();
+
+  Make_weight_approx ();
+
+  Make_text_idx_wgt ();
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+
+  return 0;
+}
+
+
+
+
+unsigned long 
+get_NumPara (void)
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (NumPara)
+    return (NumPara);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.num_of_docs, NumPara);  /* [RPAP - Jan 97: Endian Ordering] */
+  return NumPara;
+}
+
+
+
+unsigned long 
+get_StaticNumOfDocs (void)
+/* the static number of documents is the N parameter used to
+ * decode document gaps in the inverted file encoded using
+ * the Bblock method.
+ */
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (StaticNumOfDocs)
+    return (StaticNumOfDocs);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.static_num_of_docs, StaticNumOfDocs);  /* [RPAP - Jan 97: Endian Ordering] */
+  return StaticNumOfDocs;
+}
+
+
+
+void GenerateWeights (void) {
+  FILE *dict, *invf, *f, *idx;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;
+  double logN;
+  float *DocWeights;
+
+  /* make sure the globals NumPara and StaticNumOfDocs are loaded */
+  get_NumPara ();
+  get_StaticNumOfDocs ();
+
+  /* check to see if the weights file has already been built */
+  if ((f = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		      MG_CONTINUE)) != NULL) {
+      fclose (f);
+      return;
+  }
+  Message ("The file \"%s.weight\" does not exist.", file_name);
+  Message ("Building the weight data from the file \"%s.invf\".", file_name);
+
+  logN = log ((double) NumPara);
+
+  /* allocate memory for the weights */
+  if (!(DocWeights = Xmalloc (sizeof (float) * (NumPara + 1))))
+      FatalError (1, "No memory for doc weights");
+  bzero ((char *) DocWeights, sizeof (float) * (NumPara + 1));
+
+  /* open the .invf.dict file and read in its header */
+  dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+		    MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  /* open .invf.idx */
+  idx = open_file (file_name, INVF_IDX_SUFFIX, "rb", MAGIC_INVI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* open .invf and read in its header */
+  invf = open_file (file_name, INVF_SUFFIX, "rb", MAGIC_INVF,
+		    MG_ABORT);
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i <= 15; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  /* make sure the inverted file does not contain skips and is not level 1 */
+  if (ifh.skip_mode != 0)
+    FatalError (0, "Can\'t make weights file from a skipped inverted file.");
+  if (ifh.InvfLevel == 1)
+    FatalError (0, "Can\'t make weights file from level 1 inverted file.");
+
+  DECODE_START (invf)
+
+    /* process each word adding its contributions to the document weights */
+    for (i = 0; i < ifh.no_of_words; i++)
+    {
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+      unsigned long fcnt, wcnt, blk, CurrDoc, p, j;
+      float idf;
+
+      /* give a little feedback every 4096 words */
+      if ((i & 0xfff) == 0)
+	fprintf (stderr, ".");
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      dummy2[dummy1] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      idf = logN - log ((double) fcnt);
+      blk = BIO_Bblock_Init (StaticNumOfDocs, p);
+      CurrDoc = 0;
+
+      /* check the inverted file index entry for this word */
+      {
+	unsigned long loc;
+	fread ((char *) &loc, sizeof (loc), 1, idx);
+	NTOHUL(loc);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (ftell (invf) != loc)
+	  {
+	    FatalError (1, "Word %d  %d != %d", i, ftell (invf), loc);
+	  }
+      }
+
+      for (j = 0; j < p; j++)
+	{
+	  unsigned long x, tf;
+	  BBLOCK_DECODE (x, blk);
+	  CurrDoc += x;
+
+	  if (CurrDoc > idh.num_of_docs) {
+	    FatalError (1, "CurrDoc = %d, number of documents = %d", 
+			CurrDoc, idh.num_of_docs);
+	  }	  
+
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      double weight;
+	      GAMMA_DECODE (tf);
+	      weight = tf * idf;
+	      DocWeights[CurrDoc - 1] += weight * weight;
+	    }
+	}
+      
+      while (__btg)
+	DECODE_BIT;
+    }
+
+  DECODE_DONE
+
+  fclose (dict);
+  fclose (invf);
+  fprintf (stderr, "\n");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < NumPara; i++)
+    HTONF(DocWeights[i]);
+
+  f = create_file (file_name, WEIGHTS_SUFFIX, "wb", MAGIC_WGHT,
+		   MG_ABORT);
+
+  fwrite ((char *) DocWeights, sizeof (float), NumPara, f);
+  fclose (f);
+  Xfree (DocWeights);
+}
+
+
+
+
+
+
+
+
+
+
+
+void 
+Make_weight_approx (void)
+{
+  int i, pos, max;
+  unsigned long buf;
+  double U, L, B;
+  FILE *approx, *exact;
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* calculate U and L */
+  L = 1e300;
+  U = 0;
+  for (i = 0; i < NumPara; i++)
+    {
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt > U)
+	U = wgt;
+      if (wgt > 0 && wgt < L)
+	L = wgt;
+
+    }
+  fseek (exact, sizeof (u_long), SEEK_SET);
+
+  B = pow (U / L, pow (2.0, -(double) bits));
+
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+
+
+
+  approx = create_file (file_name, APPROX_WEIGHTS_SUFFIX, "wb",
+			MAGIC_WGHT_APPROX, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fwrite ((char *) &bits, sizeof (bits), 1, approx);
+  HTOND(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  HTOND(B);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &L, sizeof (L), 1, approx);
+  fwrite ((char *) &B, sizeof (B), 1, approx);
+  NTOHD(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHD(B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  max = bits == 32 ? 0xffffffff : (1 << bits) - 1;
+  for (buf = pos = i = 0; i < NumPara; i++)
+    {
+      unsigned long fx;
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt == 0)
+	{
+	  wgt = L;
+#ifndef QUIET
+	  Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+	}
+      fx = (int) floor (log (wgt / L) / log (B));
+
+      if (fx > max)
+	fx = max;
+
+      buf |= (fx << pos);
+      pos += bits;
+
+      if (pos >= MAXBITS)
+	{
+	  HTONUL(buf);
+	  fwrite ((char *) &buf, sizeof (buf), 1, approx);
+	  buf = fx >> (bits - (pos - MAXBITS));
+	  pos = pos - MAXBITS;
+	}
+    }
+  if (pos > 0)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(buf);
+      fwrite ((char *) &buf, sizeof (buf), 1, approx);
+    }
+
+  fclose (approx);
+  fclose (exact);
+}
+
+
+
+
+
+void 
+Make_text_idx_wgt (void)
+{
+  compressed_text_header cth;
+  int i;
+  FILE *idx_wgt, *idx, *para, *exact;
+
+  idx_wgt = create_file (file_name, TEXT_IDX_WGT_SUFFIX, "wb", MAGIC_TEXI_WGT,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* [RJM 10/98 - Text Filename] */
+  idx = open_file (text_file_name, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (fread (&cth, sizeof (cth), 1, idx) != 1)
+    FatalError (1, "Unable to read header of index file");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  get_NumPara ();
+  if (cth.num_of_docs != NumPara)
+    {
+      Message ("The number of documents does not equal "
+	       "the number of paragraphs.");
+      Message ("Using the \"%s.invf.paragraph\" file\n", file_name);
+      para = open_file (file_name, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH,
+			MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+    }
+  else
+    para = NULL;
+
+  {
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+    data;
+    for (i = 0; i < cth.num_of_docs; i++)
+      {
+	int count;
+	fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+	if (para && i < cth.num_of_docs)
+	  {
+	    /* [RPAP - Jan 97: Endian Ordering] */
+	    fread ((char *) &count, sizeof (count), 1, para);
+	    NTOHSI(count);
+	  }
+	else
+	  count = 1;
+	while (count--)
+	  {
+	    fread ((char *) &data.Weight, sizeof (float), 1, exact);
+	    NTOHF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    data.Weight = sqrt (data.Weight);
+	    HTONF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    fwrite ((char *) &data, sizeof (data), 1, idx_wgt);
+	  }
+      }
+    /* Write out the extra entry for the idx file */
+    fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+    data.Weight = 0;
+    fwrite((char*)&data, sizeof(data), 1, idx_wgt);
+  }
+
+  fclose (idx_wgt);
+  fclose (idx);
+  fclose (exact);
+  if (para)
+    fclose (para);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgdictlist.c	(revision 21112)
@@ -0,0 +1,405 @@
+/**************************************************************************
+ *
+ * mgdictlist.c -- Program to list a dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "text.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:17  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:24  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:07  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:01  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:56  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int quick = 0;
+int no_of_words[2];
+u_long maxcodelen[2];
+
+char *dictname = "";
+
+
+
+
+void 
+DumpStemDict (FILE * f)
+{
+  struct invf_dict_header idh;
+  int i;
+  u_char prev[MAXSTEMLEN + 1];
+
+  fread (&idh, sizeof (idh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (quick)
+    printf ("%ld\n", idh.dict_size);
+  else
+    {
+      printf ("# lookback           = %lu\n", idh.lookback);
+      printf ("# dict size          = %lu\n", idh.dict_size);
+      printf ("# total bytes        = %lu\n", idh.total_bytes);
+      printf ("# index string bytes = %lu\n", idh.index_string_bytes);
+      printf ("# input bytes        = %.0f\n", idh.input_bytes); /* [RJM 07/97: 4G limit] */
+      printf ("# num of docs        = %lu\n", idh.num_of_docs);
+      printf ("# static num of docs = %lu\n", idh.static_num_of_docs);
+      printf ("# num of words       = %lu\n", idh.num_of_words);
+      printf ("#\n");
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (f);
+      suff = getc (f);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, f);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, f);
+      fread (&wcnt, sizeof (wcnt), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (!quick)
+	{
+	  printf ("%d: %8ld ", i, wcnt);
+	  printf ("/ %5ld ", fcnt);
+	  printf ("%2d %2ld\t\"", *prev, copy);
+	}
+      printf ("%s", word2str (prev));
+      if (quick)
+	printf (" %ld %ld\n", wcnt, fcnt);
+      else
+	{
+	  putchar ('"');
+	  putchar ('\n');
+	}
+    }
+}
+
+
+
+
+void 
+ReadInWords (FILE * f)
+{
+  comp_frags_header cfh;
+  u_long *codes;
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+
+  if (Read_cfh (f, &cfh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read in the dictionary");
+
+  printf ("#\n");
+  printf ("#   max code len       = %u\n", cfh.hd.maxcodelen);
+  printf ("#   total bytes        = %lu\n", cfh.uncompressed_size);
+  printf ("#\n");
+
+  if (!(codes = Generate_Huffman_Codes (&cfh.hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  for (i = 0; i < cfh.hd.num_codes; i++)
+    {
+      register int val, copy, j, k;
+      char code[33];
+      val = fgetc (f);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (prev + copy + 1, sizeof (u_char), val, f);
+      *prev = val + copy;
+
+      for (k = 0, j = cfh.hd.clens[i] - 1; j >= 0; j--, k++)
+	code[k] = '0' + ((codes[i] >> j) & 1);
+      code[k] = '\0';
+
+      printf ("%d: %2d : %*s : \"%s\"\n", i, cfh.hd.clens[i],
+	      cfh.hd.maxcodelen, code, word2str (prev));
+    }
+  Xfree (codes);
+  Xfree (cfh.hd.clens);
+}
+
+
+void 
+ReadCharHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : \"%s\"\n", hd.clens[i],
+		hd.maxcodelen, code, char2str (i));
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+void 
+ReadLenHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : %d\n", hd.clens[i],
+		hd.maxcodelen, code, i);
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+
+
+
+void 
+DumpTextDict (FILE * f)
+{
+  struct compression_dict_header cdh;
+  int which;
+
+  if (Read_cdh (f, &cdh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read dictionary header");
+  switch (cdh.dict_type)
+    {
+    case MG_COMPLETE_DICTIONARY:
+      printf ("# COMPLETE DICTIONARY\n");
+      break;
+    case MG_PARTIAL_DICTIONARY:
+      printf ("# PARTIAL DICTIONARY\n");
+      break;
+    case MG_SEED_DICTIONARY:
+      printf ("# SEED DICTIONARY\n");
+      break;
+    }
+  printf ("# num words          = %lu\n", cdh.num_words[1]);
+  printf ("# num word chars     = %lu\n", cdh.num_word_chars[1]);
+  printf ("# num non-words      = %lu\n", cdh.num_words[0]);
+  printf ("# num non-word chars = %lu\n", cdh.num_word_chars[0]);
+  printf ("# lookback           = %lu\n", cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  ReadInWords (f);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      }
+}
+
+
+
+
+void 
+DumpStatsDict (FILE * f)
+{
+  int i;
+  compression_stats_header csh;
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  for (i = 0; i < 2; i++)
+    {
+      int j;
+      frags_stats_header fsh;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      if (!quick)
+	printf ("#\n# num %9s      = %lu\n#\n", i ? "words" : "non-words",
+		fsh.num_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  u_char Word[16];
+	  u_long freq, occur_num;
+
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHUL(freq);
+	  NTOHUL(occur_num);
+
+	  Word[0] = fgetc (f);
+	  fread (Word + 1, Word[0], 1, f);
+	  printf ("%d: %7ld : %7ld : \"%s\"\n", j, freq,
+		  occur_num, word2str (Word));
+	}
+    }
+}
+
+
+int main (int argc, char **argv)
+{
+  FILE *fp;
+  unsigned long magic = 0;
+
+  if (argc < 2)
+    FatalError (1, "A file name must be specified");
+  dictname = argv[1];
+  if (strcmp (dictname, "-q") == 0)
+    {
+      quick = 1;
+      if (argc < 3)
+	FatalError (1, "A file name must be specified");
+      dictname = argv[2];
+    }
+  if (!(fp = fopen (dictname, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\"", dictname);
+
+  fread (&magic, sizeof (magic), 1, fp);
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  switch (magic)
+    {
+    case MAGIC_STEM_BUILD:
+      if (!quick)
+	printf ("# Contents of STEM file \"%s\"\n#\n", dictname);
+      DumpStemDict (fp);
+      break;
+    case MAGIC_DICT:
+      if (!quick)
+	printf ("# Contents of DICT file \"%s\"\n#\n", dictname);
+      DumpTextDict (fp);
+      break;
+    case MAGIC_STATS_DICT:
+      if (!quick)
+	printf ("# Contents of STATS file \"%s\"\n#\n", dictname);
+      DumpStatsDict (fp);
+      break;
+    default:
+      FatalError (1, "Bad magic number. \"%s\" cannot be dumped", dictname);
+    }
+  fclose (fp);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgquery.c	(revision 21112)
@@ -0,0 +1,1495 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+ 
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+
+
+#include <stdarg.h>
+#include <signal.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F)
+    if ((int) F != -1)
+      {
+	if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	  Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		     F->Current.NumReads);
+	NumBytes += F->Current.NumBytes;
+	NumSeeks += F->Current.NumSeeks;
+	NumReads += F->Current.NumReads;
+      }
+    else
+      {
+	Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) NumBytes / 1024, NumSeeks, NumReads);
+	NumSeeks = NumReads = NumBytes = 0;
+      }
+
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+      if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+	{
+	  char *s;
+	  s = re_comp (post_proc);
+	  if (!s)
+	    {
+	      Xfree (post_proc);
+	      post_proc = NULL;
+	    }
+	}
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  if (verbatim) {
+    return (strstr (UDoc, post_proc) != NULL);
+  }
+  return re_exec ((char *) UDoc);
+}
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{
+	  /* load the compressed text */
+	  if (LoadCompressedText (qd, max_buf))
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+
+	  /* uncompress the loaded text */
+	  UDoc = GetDocText (qd, &ULen);
+	  if (UDoc == NULL)
+	    FatalError (1, "UDoc is unexpectedly NULL");
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->Weight);
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%7d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->Weight);
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->Weight);
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->num;
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      ConvertTermsToString (qd->TL, terms_str);
+      pager = Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      ConvertTermsToString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->num > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->num;
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->num,
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->num == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+  start_up_stats (qd, iqt);
+
+
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					     (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	    else
+	      BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFREG) != 0)
+	{
+	  /* The name is a directory */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    {
+      query ();
+    }
+  else
+    {
+      int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgstat.c	(revision 21112)
@@ -0,0 +1,403 @@
+/**************************************************************************
+ *
+ * mgstat.c -- Program to generate statistics on a text collection
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "mg.h"
+#include "words.h"
+#include "invf.h"
+#include "text.h"
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:18:19  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:59  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+char *search_for_collection (char *name);
+int process_file (char *, char *, int);
+void ProcessDict (char *);
+void ProcessStem (char *);
+long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */
+
+static mg_ullong inputbytes = 0;
+static mg_ullong total = 0;
+
+
+
+
+
+int main (int argc, char **argv)
+{
+  mg_ullong sub_total;
+  int fast;
+  char *file_name = "";
+  int ch;
+  int exact = 0;
+  long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
+    switch (ch)
+      {
+      case 'E':
+	exact = 1;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    file_name = search_for_collection (argv[optind]);
+
+  ProcessDict (file_name);
+  ProcessStem (file_name);
+  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */
+
+  {
+    char Name[256];
+    char *s;
+    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
+    s = strrchr (Name, '/');
+    if (s)
+      *s = '\0';
+    printf ("\nThe collection is in \"%s\"\n", Name);
+  }
+
+
+  printf ("\n\t\tFiles required by mgquery\n");
+  process_file (file_name, TEXT_SUFFIX, exact);
+  process_file (file_name, INVF_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
+  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (indexed)
+    {
+      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
+    }
+
+  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
+  if (!fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+
+
+  process_file (NULL, "SUB TOTAL", exact);
+  sub_total = total;
+  total = 0;
+
+  printf ("\n\t\tFiles NOT required by mgquery\n");
+  if (fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+  process_file (file_name, INVF_DICT_SUFFIX, exact);
+  process_file (file_name, INVF_IDX_SUFFIX, exact);
+  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_SUFFIX, exact);
+  process_file (file_name, WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
+  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
+  process_file (NULL, "SUB TOTAL", exact);
+  total += sub_total;
+  printf ("\n");
+  process_file (NULL, "TOTAL", exact);
+  return 0;
+
+}
+
+
+
+char *
+search_for_collection (char *name)
+{
+  char *dir = get_basepath ();
+  static char buffer[512];
+  struct stat stat_buf;
+
+  /* Look in the current directory first */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  set_basepath (".");
+	  return buffer;
+	}
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  set_basepath (".");
+	  return name;
+	}
+    }
+  sprintf (buffer, "%s/%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  return buffer;
+	}
+    }
+  return name;
+}
+
+
+
+
+
+
+void 
+ProcessDict (char *name)
+{
+  FILE *f;
+  compression_dict_header cdh;
+  int have_cdh = 0;
+  compressed_text_header cth;
+  int have_cth = 0;
+
+  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Read_cdh (f, &cdh, NULL, NULL);
+      fclose (f);
+      have_cdh = 1;
+    }
+
+  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fread ((char *) &cth, sizeof (cth), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(cth.num_of_docs);
+      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+      NTOHUL(cth.num_of_words);
+      NTOHUL(cth.length_of_longest_doc);
+      NTOHD(cth.ratio);
+
+      fclose (f);
+      have_cth = 1;
+    }
+
+  if (have_cth)
+    {
+      inputbytes = cth.num_of_bytes;
+      printf ("Input bytes                        : %10" ULL_FS ", %8.2f Mbyte\n",
+	      cth.num_of_bytes, (double) cth.num_of_bytes / 1024 / 1024);
+      printf ("Documents                          : %10lu\n", cth.num_of_docs);
+      printf ("Words in collection [dict]         : %10lu\n", cth.num_of_words);
+      printf ("Longest doc in collection [dict]   : %10lu characters\n",
+	      cth.length_of_longest_doc);
+      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
+    }
+
+  if (have_cdh)
+    {
+      printf ("Words in dict                      : %10lu\n", cdh.num_words[1]);
+      printf ("Non-words in dict                  : %10lu\n", cdh.num_words[0]);
+      printf ("Total chars of distinct words      : %10lu\n", cdh.num_word_chars[1]);
+      printf ("Total chars of distinct non-words  : %10lu\n", cdh.num_word_chars[0]);
+    }
+
+}
+
+
+
+
+
+
+
+
+void 
+ProcessStem (char *name)
+{
+  FILE *f;
+  struct invf_dict_header idh;
+
+  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
+		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+  fread ((char *) &idh, sizeof (idh), 1, f);
+  printf ("Words in collection [stem]         : %10ld\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Words in stem                      : %10ld\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Indexed fragments                  : %10ld\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Total chars of stem words          : %10ld\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
+  fclose (f);
+}
+
+
+
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+long 
+ProcessStemBlk (char *name)
+{
+  FILE *f;
+  struct stem_dict_header sdh;
+
+  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
+		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return 0;
+  fread ((char *) &sdh, sizeof (sdh), 1, f);
+  fclose (f);
+  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+process_file (char *name, char *ext, int exact)
+{
+  static double scale = 0;
+  static char *units;
+  struct stat buf;
+  if (scale == 0)
+    {
+      /* So can get output in Mb or Kb, */
+      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
+      /* Note: if inputbytes==0, then use 1024 as default */
+      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
+      units = scale == 1024 ? "Kb" : "Mb";
+    }
+  if (name)
+    {
+      char Name[256];
+      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
+      if (!stat (Name, &buf))
+	{
+	  char fname[256];
+	  char *nam = strrchr (name, '/');
+	  nam = nam ? nam + 1 : name;
+	  sprintf (fname, "%s%s", nam, ext);
+
+	  if (inputbytes == 0.0)
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size,
+			100.0 * buf.st_size / inputbytes);
+	      else
+		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units,
+			100.0 * buf.st_size / inputbytes);
+	    }
+	  else
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size);
+	      else
+		printf ("%s%*s: %8.2f %s\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units);
+	    }
+	  total += buf.st_size;
+	  return 1;
+	}
+      else
+	return 0;
+    }
+  else
+    {
+      if (inputbytes == 0.0)
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes   %7.3f%%\n", ext,
+		    total,
+		    100.0 * total / inputbytes);
+	  else
+	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
+		    total / scale, units,
+		    100.0 * total / inputbytes);
+	}
+      else
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes\n", ext, total);
+	  else
+	    printf ("%-34s : %8.2f %s\n", ext,
+		    total / scale, units);
+	}
+    }
+  return 1;
+
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/mgstemidxlist.c	(revision 21112)
@@ -0,0 +1,245 @@
+/**************************************************************************
+ *
+ * mgstemidxlist.c -- Text dumper for the stem indexes
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+
+void
+read_3_in_4 (FILE * idbi)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+  int block = 0;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      return;
+    }
+
+  si->MemForStemIdx = 0;
+
+  fread (&(si->sih), sizeof (si->sih), 1, idbi);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = fgetc (idbi);
+      *buffer++ = len;
+      fread (buffer, sizeof (u_char), len, idbi);
+      buffer += len;
+      fread (&si->pos[i], sizeof (*si->pos), 1, idbi);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  printf ("# lookback     = %lu\n", si->sih.lookback);
+  printf ("# block_size   = %lu\n", si->sih.block_size);
+  printf ("# num_blocks   = %lu\n", si->sih.num_blocks);
+  printf ("# blocks_start = %lu\n", si->sih.blocks_start);
+  printf ("# index_chars  = %lu\n", si->sih.index_chars);
+  printf ("# num_of_words = %lu\n", si->sih.num_of_words);
+
+  block = 0;
+  while (block < si->sih.num_blocks)
+    {
+      unsigned long *first_word;
+      unsigned short *num_words;
+      unsigned short *index;
+      long res;
+      u_char *base;
+      int num_indexes;
+
+      /* Read in next block */
+      fseek (idbi, si->pos[block] + si->sih.blocks_start, 0);
+      fread (si->buffer, si->sih.block_size, sizeof (u_char), idbi);
+      si->active = si->pos[block];
+
+      first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (first_word + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+
+      base = (u_char *) (index + num_indexes);
+      base += index[0];
+
+      printf ("\n# block      = %d\n", block);
+      printf ("# first_word = %lu\n", *first_word);
+      printf ("# num_words  = %u\n", *num_words);
+
+      res = 0;
+      while (res < *num_words)
+	{
+	  unsigned copy, suff;
+	  u_char prev[MAXSTEMLEN + 1];
+	  unsigned int num_entries, num_cases, blk;
+	  unsigned short blk_index, offset;
+
+	  /* Read word entry */
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  *prev = copy + suff;
+	  base += suff;
+	  bcopy ((char *) base, (char *) &num_entries, sizeof (num_entries));
+	  base += sizeof (num_entries);
+	  NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+	  printf ("%u \"%s\"\n", num_entries, word2str (prev));
+
+	  /* For all the PosEntries for the word... */
+	  for (i = 0; i < num_entries; i++)
+	    {
+	      bcopy ((char *) base, (char *) &num_cases, sizeof (num_cases));
+	      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (num_cases);
+	      bcopy ((char *) base, (char *) &blk, sizeof (blk));
+	      NTOHUI(blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk);
+	      bcopy ((char *) base, (char *) &blk_index, sizeof (blk_index));
+	      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk_index);
+	      bcopy ((char *) base, (char *) &offset, sizeof (offset));
+	      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (offset);
+
+	      printf ("    -> %4u %4u %4u %4u\n", num_cases, blk, blk_index, offset);
+	    }
+	  res++;
+	}
+      block++;
+    }
+  fclose (idbi);
+}
+
+int
+main (int argc, char **argv)
+{
+  FILE *idbi;
+  char *filename = "";
+  int ch;
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:hs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d data directory] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  /* Open required files */
+  switch (stem_method)
+    {
+    case (1):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "rb", MAGIC_STEM_1,
+			MG_ABORT);
+      break;
+    case (2):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "rb", MAGIC_STEM_2,
+			MG_ABORT);
+      break;
+    case (3):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "rb", MAGIC_STEM_3,
+			MG_ABORT);
+      break;
+    default:
+      FatalError (1, "Stem method must be 1, 2 or 3\n");
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT open file");
+
+  read_3_in_4 (idbi);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/query.docnums.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/query.docnums.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/query.docnums.c	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * query.docnums.c -- Docnums query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "filestats.h"
+#include "timing.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+
+
+static DocList *
+AddToDocList (DocList * Docs, int res)
+{
+  int j;
+  /* Search the list for the word */
+  for (j = 0; j < Docs->num; j++)
+    if (Docs->DE[j].DocNum == res)
+      break;
+
+  /* Increment the weight if the word is in the list */
+  if (j < Docs->num)
+    Docs->DE[j].Weight++;
+  else
+    {
+      /* Create a new entry in the list for the new document */
+      Docs = ResizeDocList (Docs, Docs->num + 1);
+      Docs->DE[Docs->num].DocNum = res;
+      Docs->DE[Docs->num].Weight = 1;
+      Docs->DE[Docs->num].SeekPos = 0;
+      Docs->DE[Docs->num].Len = 0;
+      Docs->DE[Docs->num].CompTextBuffer = NULL;
+      Docs->DE[Docs->num].Next = NULL;
+      Docs->num++;
+    }
+  return (Docs);
+}
+
+void 
+DocnumsQuery (query_data * qd, char *QueryLine)
+{
+  DocList *Docs = MakeDocList (0);
+  unsigned long start = 0, finish;
+  int range = 0; /* is there a range or not ? */
+
+  /* find the start of the first word */
+  while (*QueryLine)
+    {
+      char *end;
+      finish = strtol (QueryLine, &end, 10);
+      if (end != (char *) QueryLine && finish >= 1 &&
+	  finish <= qd->sd->sdh.num_of_docs)
+	{
+	  unsigned long i;
+	  if (!range)
+	    start = finish;
+	  else
+	    range = 0;
+	  for (i = start; i <= finish; i++)
+	    Docs = AddToDocList (Docs, i);
+	  while (*end == ' ' || *end == '\t' || *end == '\n')
+	    end++;
+	  if (*end == '-')
+	    {
+	      start = finish;
+	      range = 1;
+	      end++;
+	    }
+	}
+      else
+	break;
+      QueryLine = end;
+    }
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/query.ranked.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/query.ranked.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/query.ranked.c	(revision 21112)
@@ -0,0 +1,991 @@
+/**************************************************************************
+ *
+ * query.ranked.c -- Ranked query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "messages.h"
+#include "timing.h"
+#include "sptree.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "weights.h"
+#include "text_get.h"
+#include "invf_get.h"
+#include "words.h"
+#include "stemmer.h"
+#include "locallib.h"
+#include "environment.h"
+#include "term_lists.h"
+#include "local_strings.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:20  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:46  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:03  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:04  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/*************************************************************************/
+
+typedef struct HeapEntry
+  {
+    float Weight;
+    float *OrgWeight;
+    int DocNum;
+    u_long SeekPos;		/* position in the text file in bytes */
+    u_long Len;			/* length of the document in bytes */
+  }
+HeapEntry;
+
+typedef int (*HeapComp) (HeapEntry *, HeapEntry *);
+
+typedef struct Heap
+  {
+    int NumItems;
+    int MaxSize;
+    HeapComp HC;
+    HeapEntry HE[1];
+  }
+Heap;
+
+
+Heap *
+Heap_Make (int size, HeapComp hc)
+{
+  Heap *H;
+  H = Xmalloc (sizeof (Heap) + (size - 1) * sizeof (HeapEntry));
+  if (!H)
+    return NULL;
+  H->NumItems = 0;
+  H->MaxSize = size;
+  H->HC = hc;
+  return H;
+}
+
+int 
+Heap_Size (Heap * H)
+{
+  return sizeof (Heap) + H->MaxSize * sizeof (HeapEntry);
+}
+
+HeapEntry *
+Heap_GetHead (Heap * H)
+{
+  if (H && H->NumItems)
+    return &H->HE[0];
+  else
+    return NULL;
+}
+
+
+
+void 
+Heap_Heapify (Heap * H, int i)
+{
+  register int curr, child;
+  curr = i;
+  child = curr * 2;
+  while (child <= H->NumItems)
+    {
+      if (child < H->NumItems && H->HC (&H->HE[child], &H->HE[child - 1]) > 0)
+	child++;
+      if (H->HC (&H->HE[curr - 1], &H->HE[child - 1]) < 0)
+	{
+	  HeapEntry temp = H->HE[child - 1];
+	  H->HE[child - 1] = H->HE[curr - 1];
+	  H->HE[curr - 1] = temp;
+	  curr = child;
+	  child = 2 * child;
+	}
+      else
+	break;
+    }
+}
+
+
+void 
+Heap_Build (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems / 2; i > 0; i--)
+    Heap_Heapify (H, i);
+}
+
+
+void 
+Heap_Sort (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems; i > 1; i--)
+    {
+      HeapEntry temp = H->HE[0];
+      H->HE[0] = H->HE[i - 1];
+      H->HE[i - 1] = temp;
+      H->NumItems--;
+      Heap_Heapify (H, 1);
+    }
+}
+
+
+void 
+Heap_DeleteHead (Heap * H)
+{
+  H->HE[0] = H->HE[--H->NumItems];
+  Heap_Heapify (H, 1);
+}
+
+int 
+Heap_Lesser (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? -1 :
+	  (a->Weight == b->Weight ? 0 : 1));
+}
+
+
+int 
+Heap_Greater (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? 1 :
+	  (a->Weight == b->Weight ? 0 : -1));
+}
+
+int 
+Make_Exact_Root (query_data * qd, Heap * H)
+{
+  int num = 0;
+  HeapEntry *he = H->HE;
+  while (he->SeekPos == 0)
+    {
+      he->Weight = he->Weight *
+	GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+      Heap_Heapify (H, 1);
+      num++;
+    }
+  return num;
+}
+
+void 
+Heap_Dump (Heap * H, int num)
+{
+  int i, l, lines, r;
+  if (num > H->NumItems)
+    num = H->NumItems;
+  lines = (num + 3) / 4;
+  for (l = 0; l < lines; l++)
+    for (r = 0; r < 4; r++)
+      {
+	i = lines * r + l;
+	if (i < num)
+	  fprintf (stderr, "[%2d] %7.4f", i, H->HE[i].Weight);
+	fprintf (stderr, r == 3 ? "\n" : "        ");
+      }
+  fprintf (stderr, "\n");
+}
+
+/*************************************************************************/
+
+
+
+
+
+
+int 
+doc_count_comp (const void *A, const void *B)
+{
+  const TermEntry *a = A;
+  const TermEntry *b = B;
+  return (a->WE.doc_count - b->WE.doc_count);
+}
+
+/* =========================================================================
+ * Function: ParseRankedQuery
+ * Description:
+ *      Takes a string query line and extracts the terms in the query
+ *      which exist in the stemmed dictionary.
+ *      Optionally sorts the terms into ascending order by doc. number.
+ * Input:
+ *      stemmed dictionary, query line, sort-flag
+ * Output:
+ *      A list of terms.
+ * ========================================================================= */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+static TermList *
+ParseRankedQuery (stemmed_dict * sd, char *QueryLine, int Sort, int indexed,
+		  QueryTermList **query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  u_char Word[MAXSTEMLEN + 1];
+  u_char sWord[MAXSTEMLEN + 1];
+  u_char *end, *s_in;
+  int default_stem_method = 0;
+  TermList *Terms = MakeTermList(0);
+
+  s_in = (u_char *) QueryLine;
+  end = s_in + strlen ((char *) s_in) - 1;
+  *query_term_list = MakeQueryTermList(0);  /* [RPAP - Feb 97: Term Frequency] */
+
+  if (indexed)
+    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+  else
+    default_stem_method = sd->sdh.stem_method;
+
+  while (s_in <= end)
+    {
+      int j;
+      long num_entries, word_num;
+      unsigned long count, doc_count, invf_ptr, invf_len;
+      int weight_to_apply, stem_to_apply;
+      int method_using = -1;
+
+      /* 0=optional, 1=mustmatch */
+      int require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+      /* Skip over the non word separator taking note of any parameters */
+      PARSE_RANKED_NON_STEM_WORD (require_match, s_in, end); /* [RJM 07/97: Ranked Required Terms] */
+      if (s_in > end) break;
+
+      /* Get a word and stem it */
+      PARSE_STEM_WORD (Word, s_in, end);
+
+      /* Extract any parameters */
+      weight_to_apply = 1;
+      stem_to_apply = default_stem_method;
+      while (s_in <= end)
+	{
+	  int stem_param, weight_param, param_type;
+	  char param[MAXPARAMLEN + 1];
+
+	  param_type = 0;
+	  PARSE_OPT_TERM_PARAM (param, param_type, s_in, end);
+	  if (!param_type)
+	    break;
+
+	  switch (param_type)
+	    {
+	    case (WEIGHTPARAM):
+	      weight_param = atoi (param);
+	      if (errno != ERANGE && weight_param > 0)
+		weight_to_apply = weight_param;
+	      break;
+
+	    case (STEMPARAM):
+	      stem_param = atoi (param);
+	      if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		method_using = stem_to_apply = stem_param;
+	      break;
+	    }
+	}
+
+      bcopy ((char *) Word, (char *) sWord, *Word + 1);
+      stemmer (stem_to_apply, sd->sdh.stemmer_num, sWord);
+
+      if (!indexed || stem_to_apply == 0)
+	{
+	  /* Look for the word in the already identified terms */
+	  for (j = 0; j < Terms->num; j++)
+	    if (compare (Terms->TE[j].Word, Word) == 0)
+	      break;
+
+	  /* Increment the weight if the word is in the list */
+	  /* Update the require match attribute */
+	  if (j < Terms->num)
+	    {
+	      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+				    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+	      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+	      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+	    }
+	  else
+	    {
+	      /* Look for it in the stemmed dictionary */
+	      if ((word_num = FindWord (sd, sWord, &count, &doc_count,
+					&invf_ptr, &invf_len)) != -1)
+		{
+		  /* Search the list for the word */
+		  for (j = 0; j < Terms->num; j++)
+		    if (Terms->TE[j].WE.word_num == word_num)
+		      break;
+
+		  /* Increment the weight if the word is in the list */
+		  if (j < Terms->num)
+		    {
+		      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+					    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+		      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+		      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+		    }
+		  else
+		    {
+		      /* Create a new entry in the list for the new word */
+		      TermEntry te;
+
+		      te.WE.word_num = word_num;
+		      te.WE.count = count;
+		      te.WE.doc_count = doc_count;
+		      te.WE.max_doc_count = doc_count;
+		      te.WE.invf_ptr = invf_ptr;
+		      te.WE.invf_len = invf_len;
+		      te.Count = weight_to_apply;
+		      te.Word = copy_string (Word);
+		      if (!te.Word)
+			FatalError (1, "Could NOT create memory to add term");
+		      te.Stem = NULL;
+		      te.require_match = require_match;
+
+		      AddTermEntry (&Terms, &te);
+
+		      /* [RPAP - Feb 97: Term Frequency] */
+		      AddQueryTerm (query_term_list, Word, count, method_using);
+		    }
+		}
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      else
+		AddQueryTerm (query_term_list, Word, 0, method_using);
+	    }
+	}
+      else
+	{
+	  int total_count = 0;  /* [RPAP - Feb 97: Term Frequency] */
+	  TermList *tempList = MakeTermList (0);
+	  if ((num_entries = FindWords (sd, sWord, stem_to_apply, &tempList)) > 0)
+	    {
+	      int i;
+	      u_long max_doc_count = 0;
+
+	      /* get the maximum doc count */
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  if (tempList->TE[i].WE.doc_count > max_doc_count)
+		    max_doc_count = tempList->TE[i].WE.doc_count;
+		  total_count += tempList->TE[i].WE.count;  /* [RPAP - Feb 97: Term Frequency] */
+		}
+
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  /* Look for the word(s) in the already identified terms */
+		  for (j = 0; j < Terms->num; j++) 
+		    {
+		      if (compare (Terms->TE[j].Word, tempList->TE[i].Word) == 0)
+			{
+			  /* found the word */
+			  /* Modify weight */
+			  Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+						INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+			  if (Terms->TE[j].WE.max_doc_count < max_doc_count)
+			    Terms->TE[j].WE.max_doc_count = max_doc_count;
+			  break;
+			}
+		    }
+
+		  if (j == Terms->num)
+		    {
+		      /* word was not found */
+		      tempList->TE[i].WE.max_doc_count = max_doc_count;
+		      tempList->TE[i].Count = weight_to_apply;
+
+		      /* We cannot require a term to match if it is expanded */
+		      /* into multiple terms :-( */
+		      tempList->TE[i].require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+		      AddTermEntry (&Terms, &tempList->TE[i]);
+		    }
+		}
+	    }
+	  /* [RPAP - Feb 97: Term Frequency] */
+	  AddQueryTerm (query_term_list, Word, total_count, method_using);
+
+	  if (tempList != NULL) Xfree(tempList); /* [RJM 07/98: Memory Leak] */
+	} /* end indexed */
+    } /* end while */
+
+  if (Sort)
+    /* Sort the terms in ascending order by doc_count */
+    qsort (Terms->TE, Terms->num, sizeof (TermEntry), doc_count_comp);
+  return (Terms);
+}
+
+
+
+
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+
+
+
+
+/*
+ * This function is given a list of term numbers and it returns a list
+ * of document numbers based on the cosine document weighting system.
+ * This puts the entries in an array.
+ * inverted file.
+ * If MaxDocs == -1 then it means all 
+ */
+static DocList *
+CosineGet (query_data * qd, TermList * Terms, RankedQueryInfo * rqi) {
+  DocList *Docs;
+  float *AccumulatedWeights = NULL;
+  Splay_Tree *ST = NULL;
+  Splay_Tree *Para_ST = NULL;
+  Hash_Table *HT = NULL;
+  List_Table *LT = NULL;
+  Heap *H;
+  HeapEntry *he;
+  register float *fptr = NULL;
+  register Invf_Doc_Entry *ide = NULL;
+  register Invf_Doc_EntryH *ideh = NULL;
+  int BackEnd, NumExact, MaxExact, NumParas;
+  int MaxDocs = 0, MaxParas = 0;
+  int i;
+  Invf_Doc_Entry_Pool ide_pool;
+  ide_pool.pool = NULL;
+
+  qd->hops_taken = qd->num_of_ptrs = qd->num_of_accum = 0;
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      ST = CosineDecodeSplay (qd, Terms, rqi, &ide_pool);
+      if (!ST)
+	return NULL;
+      break;
+    case 'A':
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      if (!AccumulatedWeights)
+	return NULL;
+      break;
+    case 'H':
+      HT = CosineDecodeHash (qd, Terms, rqi);
+      if (!HT)
+	return NULL;
+      break;
+    case 'L':
+      LT = CosineDecodeList (qd, Terms, rqi);
+      if (!LT)
+	return NULL;
+      break;
+    }
+
+#if 0
+  if (rqi->UseSplayTree)
+    {
+
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      fptr = AccumulatedWeights;
+      ide = SP_get_first (ST);
+      for (i = 0; i < qd->sd->sdh.num_of_docs; i++)
+	{
+	  if (AccumulatedWeights[i] != 0)
+	    {
+	      if (i != ide->DocNum)
+		fprintf (stderr, "Sum mismatch for %d %f %d %f\n", i + 1,
+			 AccumulatedWeights[i], ide->DocNum + 1, ide->Sum);
+	      ide = SP_get_next (ST);
+	    }
+	}
+    }
+#endif
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      MaxParas = ST->no_of_items;
+      break;
+    case 'A':
+      {				/* count the number of non-zero document weights */
+	register int i = qd->sd->sdh.num_of_docs;
+	register float *d;
+	MaxParas = 0;
+	for (d = AccumulatedWeights; i; i--, d++)
+	  if (*d)
+	    MaxParas++;
+      }
+      break;
+    case 'H':
+      MaxParas = HT->num + HT->Suplimentary_Num;
+      break;
+    case 'L':
+      MaxParas = LT->num;
+      break;
+    }
+
+  if (rqi->MaxParasToRetrieve != -1 && MaxParas > rqi->MaxParasToRetrieve)
+    MaxParas = rqi->MaxParasToRetrieve;
+  MaxDocs = MaxParas;
+
+  /* Allocate memory for the heap */
+  Docs = MakeDocList (MaxDocs);
+  ChangeMemInUse (qd, sizeof (DocEntry) * MaxDocs);
+
+  H = Heap_Make (MaxDocs, Heap_Lesser);
+
+
+  /* Get the sums from the array divide the sums by the 
+     document weights which we retrieve from the ".idx.wgt" file and put
+     the resulting data into a heap */
+
+
+  he = H->HE;
+  H->NumItems = MaxDocs;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	ide = SP_get_first (ST);
+	for (i = 0; i < H->NumItems; i++, ide = SP_get_next (ST), he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'A':
+      {
+	fptr = AccumulatedWeights;
+	for (i = 0; i < H->NumItems; i++, fptr++, he++)
+	  {
+	    he->DocNum = i + 1;
+	    he->OrgWeight = fptr;
+	    if (*fptr)
+	      qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'H':
+      {
+	ideh = HT->Head;
+	for (i = 0; i < H->NumItems; i++, ideh = ideh->next, he++)
+	  {
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'L':
+      {
+	ide = LT->IDE;
+	for (i = 0; i < H->NumItems; i++, ide++, he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    }
+
+  he = H->HE;
+  for (i = 0; i < H->NumItems; i++, he++)
+    {
+      *he->OrgWeight /= GetLowerApproxDocWeight (qd->awd, he->DocNum - 1);
+      he->Weight = *he->OrgWeight;
+      *he->OrgWeight = 0;
+      he->SeekPos = he->Len = 0;
+    }
+
+  Heap_Build (H);
+
+  he = H->HE;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	for (i = MaxDocs; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	  {
+	    ide->Sum /= GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'A':
+      {
+	for (i = MaxDocs; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	  {
+	    if (!*fptr)
+	      continue;
+	    qd->num_of_accum++;
+	    *fptr /= GetLowerApproxDocWeight (qd->awd, i);
+	    if (*fptr <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = i + 1;
+	    he->Weight = *fptr;
+	    he->OrgWeight = fptr;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'H':
+      {
+	for (; ideh; ideh = ideh->next)
+	  {
+	    qd->num_of_accum++;
+	    ideh->IDE.Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum);
+	    if (ideh->IDE.Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->Weight = ideh->IDE.Sum;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'L':
+      {
+	for (i = MaxDocs; i < LT->num; i++, ide++)
+	  {
+	    qd->num_of_accum++;
+	    ide->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    }
+
+
+  if (rqi->Exact && qd->id->ifh.InvfLevel != 3)
+    {
+      HeapEntry *he = H->HE;
+
+      for (i = 0; i < H->NumItems; i++, he++)
+	{
+	  he->Weight = he->Weight *
+	    GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	    FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+	}
+
+      Heap_Build (H);
+
+      he = H->HE;
+
+      switch (rqi->AccumMethod)
+	{
+	case 'S':
+	  {
+	    ide = SP_get_first (ST);
+	    for (i = 0; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+
+	  /* up to here */
+
+	case 'A':
+	  {
+	    fptr = AccumulatedWeights;
+	    for (i = 0; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!*fptr)
+		  continue;
+		if (*fptr <= he->Weight)
+		  continue;
+		Weight = *fptr *
+		  GetLowerApproxDocWeight (qd->awd, i) /
+		  FetchDocStart (qd, i + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = i + 1;
+		he->OrgWeight = fptr;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		*fptr = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'H':
+	  {
+	    ideh = HT->Head;
+	    for (ideh = HT->Head; ideh; ideh = ideh->next)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ideh->IDE.Sum)
+		  continue;
+		if (ideh->IDE.Sum <= he->Weight)
+		  continue;
+		Weight = ideh->IDE.Sum *
+		  GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum) /
+		  FetchDocStart (qd, ideh->IDE.DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ideh->IDE.DocNum + 1;
+		he->OrgWeight = &ideh->IDE.Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ideh->IDE.Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'L':
+	  {
+	    ide = LT->IDE;
+	    for (i = 0; i < LT->num; i++, ide++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	}
+    }
+
+
+
+  H->HC = Heap_Greater;
+  Heap_Build (H);
+
+
+  MaxDocs = H->NumItems;
+  if (rqi->MaxDocsToRetrieve != -1 && MaxDocs > rqi->MaxDocsToRetrieve)
+    MaxDocs = rqi->MaxDocsToRetrieve;
+
+  /* Alarm */
+
+  he = H->HE;
+  BackEnd = H->NumItems - 1;
+  NumExact = 0;
+  MaxExact = H->NumItems;
+  NumParas = 0;
+  Para_ST = SP_createset (DE_comp);
+  while (H->NumItems && Docs->num < MaxDocs)
+    {
+      DocEntry DocEnt;
+      DocEntry *mem;
+
+      if (rqi->Exact)
+	{
+	  if (H->HE[0].SeekPos == 0)
+	    NumExact += Make_Exact_Root (qd, H);
+	}
+      else
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+
+      NumParas++;
+
+      DocEnt.DocNum = he->DocNum;
+      DocEnt.Weight = he->Weight;
+      DocEnt.Len = he->Len;
+      DocEnt.SeekPos = he->SeekPos;
+      DocEnt.CompTextBuffer = NULL;
+      DocEnt.Next = NULL;
+
+      Heap_DeleteHead (H);
+
+      if (!(mem = SP_member (&DocEnt, Para_ST)))
+	{
+	  Docs->DE[Docs->num] = DocEnt;
+	  SP_insert (&Docs->DE[Docs->num], Para_ST);
+	  Docs->num++;
+	}
+      else
+	{
+	  DocEnt.Next = mem->Next;
+	  Docs->DE[BackEnd] = DocEnt;
+	  mem->Next = &Docs->DE[BackEnd--];
+	}
+    }
+  SP_freeset (Para_ST);
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      Message ("%d Paragraphs were required to get %d documents",
+	       NumParas, Docs->num);
+      if (NumExact == MaxExact)
+	{
+	  Message ("The exact weights of all %d paragraphs had to be calculated", MaxExact);
+	  Message ("to obtain %d paragraphs. This may mean that the the documents", NumParas);
+	  Message ("returned do not necessarly represent an exact cosine ranking.");
+	  Message ("This problem may be corrected by increasing \'maxparas\'.");
+	}
+    }
+#if 0
+  {
+    int i;
+    FILE *f = fopen ("top.paras", "w");
+    fprintf (f, "=========================\nTop Paragraphs\n");
+    for (i = 0; i < Docs->num; i++)
+      {
+	DocEntry *d;
+	fprintf (f, "<%d(%f)>  ", Heap[i].DocNum, Heap[i].Weight);
+	for (d = Heap[i].Next; d; d = d->Next)
+	  fprintf (f, "%d(%f)  ", d->DocNum, d->Weight);
+	fprintf (f, "\n");
+      }
+    fprintf (f, "=========================\n");
+    fclose (f);
+  }
+#endif
+
+  if (AccumulatedWeights)
+    {
+      Xfree (AccumulatedWeights);
+      ChangeMemInUse (qd, -sizeof (float) * qd->sd->sdh.num_of_docs);
+    }
+  if (ST)
+    {
+      int mem = ST->mem_in_use;
+      SP_freeset (ST);
+      ChangeMemInUse (qd, -mem);
+      free_ide_pool (qd, &ide_pool);
+    }
+  if (HT)
+    HT_free (qd, HT);
+
+  if (LT)
+    LT_free (qd, LT);
+
+  if (H) Xfree (H); /* [RJM 07/98: Memory Leak] */
+
+  return (Docs);
+}
+
+
+
+
+/* if MaxDocs == -1 it means all */
+void 
+RankedQuery (query_data *qd, char *Query, RankedQueryInfo *rqi)
+{
+  DocList *dl;
+
+  if (qd->TL)
+    FreeTermList (&(qd->TL));
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  if (qd->QTL)
+    FreeQueryTermList (&(qd->QTL));
+
+  qd->TL = ParseRankedQuery (qd->sd, Query, rqi->Sort, qd->sd->sdh.indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			     &(qd->QTL));  /* [RPAP - Feb 97: Term Frequency] */
+
+  /*  PrintTermList (qd->TL, stderr); */
+
+  dl = CosineGet (qd, qd->TL, rqi);
+
+  if (!dl)
+    FatalError (1, "Out of memory\n");
+
+  FreeQueryDocs (qd);
+
+  qd->DL = dl;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.c	(revision 21112)
@@ -0,0 +1,225 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:36  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "query_term_list.h"
+#include "term_lists.h"  /* for MAXTERMSTRLEN */
+#include "messages.h"
+
+
+/* =========================================================================
+ * Function: MakeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+QueryTermList *
+MakeQueryTermList (int n)
+{
+  QueryTermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (QueryTermList) + (list_size - 1) * sizeof (QueryTermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate query term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeQueryTermList (QueryTermList ** query_list)
+{
+  QueryTermList *qtl = *query_list;
+
+  if (qtl->num > qtl->list_size)
+    {
+      if (qtl->list_size)
+	qtl->list_size *= GROWTH_FACTOR;
+      else
+	qtl->list_size = MIN_SIZE;
+    }
+  qtl = Xrealloc (qtl, sizeof (QueryTermList) + (qtl->list_size - 1) * sizeof (QueryTermEntry));
+
+  if (!qtl)
+    FatalError (1, "Unable to resize query term list");
+
+  *query_list = qtl;
+}
+
+/* =========================================================================
+ * Function: ConvertQueryTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = query list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertQueryTermsToString (QueryTermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->QTE[i].Term;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (query_term_list->QTE[i].stem_method >= 0)
+	total_len += 2;         /* '#' and the stem number */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (query_term_list->QTE[i].stem_method >= 0)
+	{
+	  *str++ = '#';
+	  *str++ = '0' + query_term_list->QTE[i].stem_method;
+	}
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetQueryTermList (QueryTermList ** qtl)
+{
+  if (*qtl)
+    FreeQueryTermList (qtl);
+  *qtl = MakeQueryTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddQueryTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTermEntry (QueryTermList ** query_term_list, QueryTermEntry * qte)
+{
+  QueryTermList *qtl = *query_term_list;
+
+  qtl->num++;
+  ResizeQueryTermList (query_term_list);
+  qtl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) qte, (char *) &(qtl->QTE[qtl->num - 1]), sizeof (QueryTermEntry));
+
+  return qtl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddQueryTerm
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method)
+{
+  /* Create a new entry in the list for the new word */
+  QueryTermEntry qte;
+
+  qte.Count = Count;
+  qte.stem_method = stem_method;
+  qte.Term = copy_string (Term);
+  if (!qte.Term)
+    FatalError (1, "Could NOT create memory to add term");
+
+  return AddQueryTermEntry (query_term_list, &qte);
+}
+
+/* =========================================================================
+ * Function: FreeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeQueryTermList (QueryTermList ** the_qtl)
+{
+  int j;
+  QueryTermList *qtl = *the_qtl;
+
+  for (j = 0; j < qtl->num; j++)
+    if (qtl->QTE[j].Term)
+      Xfree (qtl->QTE[j].Term);
+  Xfree (qtl);
+
+  *the_qtl = NULL;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/query_term_list.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef QUERY_TERM_LIST_H
+#define QUERY_TERM_LIST_H
+
+#include "sysfuncs.h"
+
+typedef struct QueryTermEntry
+  {
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Term;		/* The query term */
+    int stem_method;            /* The steming method to apply to the term : -1 is default */
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+/* --- prototypes --- */
+void ConvertQueryTermsToString (QueryTermList * query_term_list, char *str);
+int AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method);
+void ResetQueryTermList (QueryTermList ** tl);
+void FreeQueryTermList (QueryTermList ** the_qtl);
+QueryTermList *MakeQueryTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/read_line.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/read_line.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/read_line.c	(revision 21112)
@@ -0,0 +1,214 @@
+/**************************************************************************
+ *
+ * read_line.c -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#ifdef GNU_READLINE
+#include "readline.h"
+#include "chardefs.h"
+#include "history.h"
+#endif
+
+
+#include "globals.h"
+#include "environment.h"
+#include "read_line.h"
+
+
+
+#ifdef GNU_READLINE
+int rl_bind_key (int, int (*)());
+
+
+void 
+Init_ReadLine (void)
+{
+  /* Make TAB just insert itself rather than do a file name completion */
+  rl_bind_key (TAB, rl_insert);
+}
+#else
+
+static FILE *rl_instream = NULL;
+static FILE *rl_outstream = NULL;
+
+void 
+Init_ReadLine (void)
+{
+  rl_instream = stdin;
+  rl_outstream = stdout;
+}
+
+static char *
+readline (char *pmt)
+{
+  static char buf[MAXLINEBUFFERLEN + 1];
+  char *s;
+
+  fprintf (rl_outstream, "%s", pmt);
+  s = fgets (buf, sizeof (buf), rl_instream);
+  if (s)
+    {
+      char *s1 = strrchr (s, '\n');
+      if (s1 && *(s1 + 1) == '\0')
+	*s1 = '\0';
+    }
+  return s ? Xstrdup (s) : NULL;
+}
+#endif
+
+/* WritePrompt() */
+/* Write out a prompt if user is a TTY */
+void 
+WritePrompt (void)
+{
+  if (isatty (fileno (InFile)))
+    {
+      if (!BooleanEnv (GetEnv ("expert"), 0))
+	fprintf (stderr, "Enter a command or query (.quit to terminate, .help for assistance).\n");
+    }
+}
+
+#ifdef GNU_READLINE
+static void memory_error_and_abort ();
+
+char *
+xmalloc (bytes)
+     int bytes;
+{
+  char *temp = (char *) Xmalloc (bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+  return (temp);
+}
+
+char *
+xrealloc (pointer, bytes)
+     char *pointer;
+     int bytes;
+{
+  char *temp;
+
+  if (!pointer)
+    temp = (char *) xmalloc (bytes);
+  else
+    temp = (char *) Xrealloc (pointer, bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+
+  return (temp);
+}
+
+static void
+memory_error_and_abort ()
+{
+  fprintf (stderr, "history: Out of virtual memory!\n");
+  abort ();
+}
+#endif
+
+
+/*
+ * This routine returns a pointer to the users entered line 
+ *
+ */
+static char *
+GetLine (char *pmt)
+{
+  static char *the_line = NULL;
+  if (the_line)
+    Xfree (the_line);
+  the_line = NULL;
+  rl_instream = InFile;
+  if (isatty (fileno (InFile)))
+    {
+      fputc ('\r', stderr);
+      if (!isatty (fileno (OutFile)))
+	rl_outstream = stderr;
+      else
+	rl_outstream = OutFile;
+      the_line = readline (pmt);
+    }
+  else
+    {
+      if (isatty (fileno (OutFile)))
+	{
+	  the_line = readline (pmt);
+	  fprintf (stderr, "%s\n", the_line ? the_line : "");
+	}
+      else
+	{
+	  the_line = readline ("");
+	  if (the_line)
+	    fprintf (stderr, "%s%s\n", pmt, the_line);
+	}
+    }
+#ifdef GNU_READLINE
+  if (the_line && *the_line)
+    add_history (the_line);
+#endif
+  return (the_line);
+}
+
+
+
+
+
+
+char *
+GetMultiLine (void)
+{
+  static char *line = NULL;
+  char *s;
+  if (line)
+    Xfree (line);
+  line = NULL;
+  if (!(s = GetLine ("> ")))
+    return (NULL);
+  if (!(line = Xstrdup (s)))
+    {
+      fprintf (stderr, "Unable to allocate memory for the line\n");
+      abort ();
+    }
+
+  while ((s = strrchr (line, '\\')) && *(s + 1) == '\0')
+    {
+      char *new;
+      *strrchr (line, '\\') = '\0';
+      if (!(s = GetLine ("? ")))
+	return (NULL);
+      if (!(new = Xmalloc (strlen (line) + strlen (s) + 2)))
+	{
+	  fprintf (stderr, "Unable to allocate memory for the line\n");
+	  abort ();
+	}
+      strcpy (new, line);
+      strcat (new, "\n");
+      strcat (new, s);
+      Xfree (line);
+      line = new;
+    }
+  return (line);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/read_line.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/read_line.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/read_line.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * read_line.h -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef READ_LINE_H
+#define READ_LINE_H
+
+#define MAXLINEBUFFERLEN  65535
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.c	(revision 21112)
@@ -0,0 +1,772 @@
+/**************************************************************************
+ *
+ * stem_search.c -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "words.h"
+#include "locallib.h"
+#include "stem_search.h"
+#include "mg_errors.h"
+#include "term_lists.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:22  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/07/02 00:18:55  rjmcnab
+   Changed so FindWords could be used in new ways.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:39  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:04  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:08  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem dict = %i\n", sd->MemForStemDict); */
+
+  return sd;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_idx *
+ReadStemIdxBlk (File * stem_idx_file)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  si->stem_idx_file = stem_idx_file;
+  si->MemForStemIdx = 0;
+
+  Fread (&si->sih, sizeof (si->sih), 1, stem_idx_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = Getc (stem_idx_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_idx_file);
+      buffer += len;
+      Fread (&si->pos[i], sizeof (*si->pos), 1, stem_idx_file);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem idx = %i\n", si->MemForStemIdx); */
+
+  return si;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* word should be appropriately stemed */
+static int
+GetIdxBlock (stemmed_idx * si, u_char * word)
+{
+  register int lo = 0, hi = si->sih.num_blocks - 1;
+  register int mid = 0, c = 0;
+
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, si->index[mid]);
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+static int 
+GetBlock (stemmed_dict * sd, u_char * Word)
+{
+  register int lo = 0, hi = sd->sdh.num_blocks - 1;
+  register int mid = 0, c = 0;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, sd->index[mid]);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+/*
+ * This function looks up a word in the stemmed dictionary, it returns -1
+ * if the word cound not be found, and 0 if it successfully finds the word.
+ * If count is non-null the ulong it is pointing to is set to the number of 
+ * occurances of the stemmed word in the collection. i.e wcnt.
+ * If doc_count is non-null the ulong it is pointing to is set to the number
+ * of documents that the word occurs in. i.e fcnt
+ * If invf_ptr is non-null the ulong it is pointing to is set to the position
+ * of the inverted file where the entry for this word start.
+ */
+int 
+FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	  unsigned long *doc_count, unsigned long *invf_ptr,
+	  unsigned long *invf_len)
+{
+  register int lo, hi, mid, c;
+  register unsigned int res;
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  block = GetBlock (sd, Word);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (sd->active != sd->pos[block])
+    {
+      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+    }
+  else
+    {
+      first_word = (unsigned long *) (sd->buffer);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+    }
+  base = (u_char *) (index + num_indexes);
+
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, base + index[mid] + 1);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+
+  res = hi * sd->sdh.lookback;
+  base += index[hi];
+
+  for (;;)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      if (res >= *num_words)
+	return (-1);
+      copy = *base++;
+      suff = *base++;
+      bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+      base += suff;
+      *prev = copy + suff;
+
+      c = casecompare (Word, prev);   /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	return (-1);
+
+      if (c == 0 && doc_count)
+	{
+	  bcopy ((char *) base, (char *) doc_count, sizeof (*doc_count));
+	  NTOHUL(*doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*doc_count);
+
+      if (c == 0 && count)
+	{
+	  bcopy ((char *) base, (char *) count, sizeof (*count));
+	  NTOHUL(*count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*count);
+
+      if (c == 0 && invf_ptr)
+	{
+	  bcopy ((char *) base, (char *) &invfp, sizeof (invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_ptr = invfp;
+	}
+      base += sizeof (*invf_ptr);
+
+      if (c == 0)
+	{
+	  /* Calculate invf_len is necessary */
+	  unsigned long next_invfp;
+	  if (!invf_len)
+	    return (*first_word + res);
+
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    {
+	      *invf_len = *last_invf_len;
+	      return (*first_word + res);
+	    }
+
+	  /* Skip over most of the next word to get to the invf_ptr */
+	  base++;
+	  suff = *base++;
+	  base += suff + sizeof (unsigned long) * 2;
+	  bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	  NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_len = next_invfp - invfp;
+	  return (*first_word + res);
+	}
+      res++;
+    }
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int 
+FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl)
+{
+  register unsigned int res;
+  unsigned int idx_res;
+  unsigned copy, suff;
+  int j, k;
+
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  int idx_block, idx_num_indexes;
+  unsigned long *idx_first_word;
+  unsigned short *idx_num_words;
+  u_char *idx_base;
+  unsigned short *idx_index;
+  u_char idx_prev[MAXSTEMLEN + 1];
+
+  unsigned int num_entries, num_cases;
+  unsigned short blk_index, offset;
+  stemmed_idx * si = NULL;
+
+  /* handle stem_method 0 seperately */
+  if (stem_method == 0) {
+    TermEntry te;
+    if ((te.WE.word_num = FindWord (sd, sWord, &te.WE.count, &te.WE.doc_count,
+				   &te.WE.invf_ptr, &te.WE.invf_len)) != -1) {
+      te.WE.max_doc_count = te.WE.doc_count;
+
+      te.Count = 1;
+      te.Word = copy_string (sWord);
+      if (!te.Word)
+	FatalError (1, "Could NOT create memory to add term");
+      te.Stem = copy_string (sWord);
+      if (!te.Stem)
+	FatalError (1, "Could NOT create memory to add term");
+      /*      te.query_mask = NULL;*/
+      
+      AddTermEntry (tl, &te);
+      return (*tl)->num;
+      
+    } else {
+      /* didn't match */
+      return 0;
+    }
+  }
+  
+  if (stem_method == 1)
+    si = sd->stem1;
+  else if (stem_method == 2)
+    si = sd->stem2;
+  else
+    si = sd->stem3;
+
+  /* Locate block */
+  idx_block = GetIdxBlock (si, sWord);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (si->active != si->pos[idx_block])
+    {
+      Fseek (si->stem_idx_file, si->pos[idx_block] + si->sih.blocks_start, 0);
+      Fread (si->buffer, si->sih.block_size, sizeof (u_char), si->stem_idx_file);
+      si->active = si->pos[idx_block];
+
+      idx_first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*idx_first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      NTOHUS(*idx_num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < idx_num_indexes; j++)
+	NTOHUS(idx_index[j]);
+    }
+  else
+    {
+      idx_first_word = (unsigned long *) (si->buffer);
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+    }
+  idx_base = (u_char *) (idx_index + idx_num_indexes);
+  
+  {
+    /* Locate 3-in-4 block */
+    register int lo, hi, mid, c;
+    lo = 0;
+    hi = idx_num_indexes - 1;
+    while (lo <= hi)
+      {
+	mid = (lo + hi) / 2;
+	c = casecompare (sWord, idx_base + idx_index[mid] + 1);
+	if (c < 0)
+	  hi = mid - 1;
+	else if (c > 0)
+	  lo = mid + 1;
+	else
+	  {
+	    hi = mid;
+	    break;
+	  }
+      }
+    if (hi < 0)
+      hi = 0;
+
+    idx_res = hi * si->sih.lookback;
+    idx_base += idx_index[hi];
+  }
+
+  /* Locate actual word entry */
+  for (;;)
+    {
+      int c;
+      if (idx_res >= *idx_num_words)
+	return (-1);
+      copy = *idx_base++;
+      suff = *idx_base++;
+      bcopy ((char *) idx_base, (char *) (idx_prev + copy + 1), suff);
+      idx_base += suff;
+      *idx_prev = copy + suff;
+
+      c = casecompare (sWord, idx_prev);
+      if (c < 0)
+	return (-1);
+     
+      bcopy ((char *) idx_base, (char *) &num_entries, sizeof (num_entries));
+      NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_entries);
+      
+      if (c > 0)
+	idx_base += num_entries * (sizeof (num_cases) + sizeof (block) + 
+				   sizeof (blk_index) + sizeof (offset));
+
+      else
+	break;
+
+      idx_res++;
+    }
+
+  for (k = 0; k < num_entries; k++)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      /* Read next stem index pos */
+      bcopy ((char *) idx_base, (char *) &num_cases, sizeof (num_cases));
+      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_cases);
+      bcopy ((char *) idx_base, (char *) &block, sizeof (block));
+      NTOHUI(block);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (block);
+      bcopy ((char *) idx_base, (char *) &blk_index, sizeof (blk_index));
+      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (blk_index);
+      bcopy ((char *) idx_base, (char *) &offset, sizeof (offset));
+      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (offset);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      if (sd->active != sd->pos[block])
+	{
+	  Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	  Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	  sd->active = sd->pos[block];
+
+	  first_word = (unsigned long *) (sd->buffer);
+	  NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (j = 0; j < num_indexes; j++)
+	    NTOHUS(index[j]);
+	}
+      else
+	{
+	  first_word = (unsigned long *) (sd->buffer);
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+	}
+      base = (u_char *) (index + num_indexes);
+      
+      res = blk_index * sd->sdh.lookback;
+      base += index[blk_index];
+      
+      for (j = 0; j < offset; j++)
+	{
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  base += sizeof (unsigned long);   /* skip doc_count */
+	  base += sizeof (unsigned long);   /* skip count */
+	  base += sizeof (unsigned long);   /* skip invf_ptr */
+	  res++;
+	}
+
+      for (j = 0; j < num_cases; j++)
+	{
+	  TermEntry te;
+	  
+	  if (res >= *num_words)
+	    return (-1);
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	      
+	  te.Word = copy_string (prev);
+	  if (!te.Word)
+	    FatalError (1, "Could NOT create memory to add term");
+	  te.Stem = copy_string (prev);
+	  if (!te.Stem)
+	    FatalError (1, "Could NOT create memory to add term");
+	  stemmer (2, sd->sdh.stemmer_num, te.Stem);
+
+	  te.Count = 1;
+	  te.WE.word_num = *first_word + res;
+	  bcopy ((char *) base, (char *) &te.WE.doc_count, sizeof (te.WE.doc_count));
+	  NTOHUL(te.WE.doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.max_doc_count = te.WE.doc_count;
+	  base += sizeof (te.WE.doc_count);
+	      
+	  bcopy ((char *) base, (char *) &te.WE.count, sizeof (te.WE.count));
+	  NTOHUL(te.WE.count);
+	  base += sizeof (te.WE.count);
+	      
+	  bcopy ((char *) base, (char *) &invfp, sizeof (te.WE.invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.invf_ptr = invfp;
+	  base += sizeof (te.WE.invf_ptr);
+		
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    te.WE.invf_len = *last_invf_len;
+	  else
+	    {
+	      unsigned long next_invfp;
+	      u_char *oldbase = base;
+		  
+	      /* Skip over most of the next word to get to the invf_ptr */
+	      base++;
+	      suff = *base++;
+	      base += suff + sizeof (unsigned long) * 2;
+	      bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	      NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	      te.WE.invf_len = next_invfp - invfp;
+	      base = oldbase;
+	    }
+	      
+	  /* Add term entry to term list */
+	  /*	  te.query_mask = NULL;*/
+	  AddTermEntry (tl, &te);
+	      
+	  if (res == *num_words - 1 && j + 1 < num_cases)
+	    {
+	      int ii;
+	      /* Read in next block */
+	      block++;
+	      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	      sd->active = sd->pos[block];
+
+	      first_word = (unsigned long *) (sd->buffer);
+	      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	      last_invf_len = (unsigned long *) (first_word + 1);
+	      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	      num_words = (unsigned short *) (last_invf_len + 1);
+	      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	      index = num_words + 1;
+	      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	      /* [RPAP - Jan 97: Endian Ordering] */
+	      for (ii = 0; ii < num_indexes; ii++)
+		NTOHUS(index[ii]);
+
+	      base = (u_char *) (index + num_indexes);
+	      base += index[0];
+	      res = 0;
+	      blk_index = 0;
+	    }
+	  else
+	    res++;
+	} /* end for num_cases */
+    } /* end for num_entries */
+  return (*tl)->num;
+}
+
+
+void 
+FreeStemDict (stemmed_dict * sd)
+{
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (sd->stem1)
+    FreeStemIdx (sd->stem1);
+  if (sd->stem2)
+    FreeStemIdx (sd->stem2);
+  if (sd->stem3)
+    FreeStemIdx (sd->stem3);
+
+  Xfree (sd->index[0]);
+  Xfree (sd->index);
+  Xfree (sd->buffer);
+  Xfree (sd->pos);
+  Xfree (sd);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void
+FreeStemIdx (stemmed_idx * si)
+{
+  Xfree (si->index[0]);
+  Xfree (si->index);
+  Xfree (si->buffer);
+  Xfree (si->pos);
+  Xfree (si);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/stem_search.h	(revision 21112)
@@ -0,0 +1,43 @@
+/**************************************************************************
+ *
+ * stem_search.h -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_STEM_SEARCH
+#define H_STEM_SEARCH
+
+stemmed_dict *ReadStemDictBlk (File * stem_file);
+
+stemmed_idx *ReadStemIdxBlk (File * stem_idx_file);
+
+int FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	      unsigned long *doc_count, unsigned long *invf_ptr,
+	      unsigned long *invf_len);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl);
+
+void FreeStemDict (stemmed_dict * sd);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void FreeStemIdx (stemmed_idx * si);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.c	(revision 21112)
@@ -0,0 +1,142 @@
+/**************************************************************************
+ *
+ * stemmer.c -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:23  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:42  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:05  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:10  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; i++) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); i++)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp (descript, "0") == 0) ||
+      (strcmp (descript, "english") == 0) ||
+      (strcmp (descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp (descript, "1") == 1) ||
+      (strcmp (descript, "french") == 0) ||
+      (strcmp (descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/stemmer.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.c	(revision 21112)
@@ -0,0 +1,331 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:43  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "term_lists.h"
+#include "messages.h"
+#include "stemmer.h"
+
+TermList *query_term_list = NULL;
+
+/* =========================================================================
+ * Function: MakeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+TermList *
+MakeTermList (int n)
+{
+  TermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (TermList) + (list_size - 1) * sizeof (TermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeTermList (TermList ** term_list)
+{
+  TermList *tl = *term_list;
+
+  if (tl->num > tl->list_size)
+    {
+      if (tl->list_size)
+	tl->list_size *= GROWTH_FACTOR;
+      else
+	tl->list_size = MIN_SIZE;
+    }
+  tl = Xrealloc (tl, sizeof (TermList) + (tl->list_size - 1) * sizeof (TermEntry));
+
+  if (!tl)
+    FatalError (1, "Unable to resize term list");
+
+  *term_list = tl;
+}
+
+/* =========================================================================
+ * Function: ConvertTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = term list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertTermsToString (TermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->TE[i].Word;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetTermList (TermList ** tl)
+{
+  if (*tl)
+    FreeTermList (tl);
+  *tl = MakeTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTermEntry (TermList ** query_term_list, TermEntry * te)
+{
+  TermList *tl = *query_term_list;
+
+  tl->num++;
+  ResizeTermList (query_term_list);
+  tl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) te, (char *) &(tl->TE[tl->num - 1]), sizeof (TermEntry));
+
+  return tl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddTerm
+ * Description: Used in boolean parser - see bool_tree   [RPAP - Feb 97: Term Frequency] 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	 u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, int stemmer_num)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  int j;
+  TermList *tl = *query_term_list;
+
+  /* Look for the word in the already identified terms */
+  for (j = 0; j < tl->num; j++)
+    {
+      TermEntry *te = &(tl->TE[j]);
+      if (compare (te->Word, Word) == 0)
+	{
+	  te->Count++;
+	  return j;
+	}
+    }
+
+
+  {
+    /* Create a new entry in the list for the new word */
+    TermEntry te;
+
+    /* [RPAP - Feb 97: Term Frequency] */
+    te.WE.word_num = word_num;
+    te.WE.count = count;
+    te.WE.doc_count = doc_count;
+    te.WE.max_doc_count = doc_count;   /* [RPAP - Jan 97: Stem Index Change] */
+    te.WE.invf_ptr = invf_ptr;
+    te.WE.invf_len = invf_len;
+    te.Count = Count;
+    te.Word = copy_string (Word);
+    if (!te.Word)
+      FatalError (1, "Could NOT create memory to add term");
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    te.Stem = copy_string (Word);
+    if (!te.Stem)
+      FatalError (1, "Could NOT create memory to add term");
+    stemmer (2, stemmer_num, te.Stem);
+
+    te.require_match = 0;  /* [RJM - 07/97: Ranked Required Terms] */
+
+    return AddTermEntry (query_term_list, &te);
+  }
+
+}
+
+/* =========================================================================
+ * Function: FreeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeTermList (TermList ** the_tl)
+{
+  int j;
+  TermList *tl = *the_tl;
+
+  for (j = 0; j < tl->num; j++)
+    {
+      if (tl->TE[j].Word)
+	Xfree (tl->TE[j].Word);
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (tl->TE[j].Stem)
+	Xfree (tl->TE[j].Stem);
+    }
+  Xfree (tl);
+
+  *the_tl = NULL;
+}
+
+/* =========================================================================
+ * Function: PrintWordEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintWordEntry (WordEntry * we, FILE * file)
+{
+  fprintf (file, "we->word_num = %d\n", we->word_num);
+  fprintf (file, "we->count = %ld\n", we->count);
+  fprintf (file, "we->doc_count = %ld\n", we->doc_count);
+  fprintf (file, "we->max_doc_count = %ld\n", we->max_doc_count);
+  fprintf (file, "we->invf_ptr = %ld\n", we->invf_ptr);
+  fprintf (file, "we->invf_len = %ld\n", we->invf_len);
+}
+
+/* =========================================================================
+ * Function: PrintTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermEntry (TermEntry * te, FILE * file)
+{
+
+  fprintf (file, "Term Entry\n");
+  fprintf (file, "te->Count = %d\n", te->Count);
+  fprintf (file, "te->Word = %s\n", str255_to_string (te->Word, NULL));
+  if (te->Stem != NULL)
+    fprintf (file, "te->Stem = %s\n", str255_to_string (te->Stem, NULL));  /* [RPAP - Jan 97: Stem Index Change] */
+  fprintf (file, "te->require_match = %i\n", te->require_match);  /* [RJM 07/97: Ranked Required Terms] */
+  PrintWordEntry (&(te->WE), file);
+
+}
+
+/* =========================================================================
+ * Function: PrintTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermList (TermList * tl, FILE * file)
+{
+  int i;
+
+  fprintf (file, "Term List\n");
+  fprintf (file, "tl->list_size = %d\n", tl->list_size);
+  fprintf (file, "tl->num = %d\n", tl->num);
+
+  for (i = 0; i < tl->num; i++)
+    {
+      fprintf (file, "[%d]\n", i);
+      PrintTermEntry (&(tl->TE[i]), file);
+    }
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/term_lists.h	(revision 21112)
@@ -0,0 +1,102 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef TERM_LISTS_H
+#define TERM_LISTS_H
+
+#include "sysfuncs.h"
+
+#define MAXTERMSTRLEN 1023	/* maximum number of characters in term string */
+
+typedef struct WordEntry
+  {
+    int word_num;		/* Unique number for each different word */
+    u_long count;		/* Number of times the word occurs in the text */
+    u_long doc_count;		/* Number of documents that contain the word */
+    u_long max_doc_count;       /* [RPAP - Jan 97: Stem Index Change] */
+    u_long invf_ptr;		/* This is a byte position of the  
+				   inverted file entry corresponding to the word */
+    u_long invf_len;		/* This is the length of the inverted 
+				   file entry in bytes */
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Word;		/* The word. */
+    u_char *Stem;               /* [RPAP - Jan 97: Stem Index Change] - The stemmed non-cf version of Word */
+    int require_match;          /* [RJM 07/97: Ranked Required Terms] 0=optional match, 1=must match */
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry TE[1];
+  }
+TermList;
+
+#define GetNthWE(term_list, n) (&((term_list)->TE[(n)].WE))
+
+/* --- prototypes --- */
+void ConvertTermsToString (TermList * query_term_list, char *str);
+int AddTermEntry (TermList ** query_term_list, TermEntry * te);
+int AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	     u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, /* [RPAP - Feb 97: Term Frequency] */
+	     int stemmer_num);  
+void ResetTermList (TermList ** tl);
+void FreeTermList (TermList ** the_tl);
+void PrintWordEntry (WordEntry * we, FILE * file);
+void PrintTermEntry (TermEntry * te, FILE * file);
+void PrintTermList (TermList * tl, FILE * file);
+TermList *MakeTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/text.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/text.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/text.h	(revision 21112)
@@ -0,0 +1,176 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_TEXT
+#define H_TEXT
+
+#include "huffman.h"
+
+
+
+
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using binary codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_BINARY 1
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+/* This method codes novel words using hybrid version of delta and a MTF
+   operation. The novel words are stored in a auxillary dictionary which
+   is built by pass two. */
+#define MG_NOVEL_HYBRID_MTF 4
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+typedef struct compression_dict_header
+  {
+    u_long dict_type;
+    u_long novel_method;
+    u_long params[TEXT_PARAMS];
+    u_long num_words[2];
+    u_long num_word_chars[2];
+    u_long lookback;
+  }
+compression_dict_header;
+
+
+typedef struct comp_frags_header
+  {
+    huff_data hd;
+    u_long uncompressed_size;
+    u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  }
+comp_frags_header;
+
+
+
+typedef struct compressed_text_header
+  {
+    u_long num_of_docs;
+    u_long num_of_words;	/* number of words in collection */
+    double num_of_bytes; /* [RJM 07/97: 4G limit] */
+    double ratio;
+    u_long length_of_longest_doc;	/* (characters) */
+    u_long dummy;    /* added to make the structure the same on different architectures */
+  }
+compressed_text_header;
+
+
+typedef struct compression_stats_header
+  {
+    u_long num_docs;
+    u_long dummy;   /* added to make the structure the same on different architectures */
+    double num_bytes; /* [RJM 07/97: 4G limit] */
+  }
+compression_stats_header;
+
+
+typedef struct frags_stats_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+frags_stats_header;
+
+
+typedef struct aux_frags_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+aux_frags_header;
+
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/text.pass1.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/text.pass1.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/text.pass1.c	(revision 21112)
@@ -0,0 +1,409 @@
+/**************************************************************************
+ *
+ * text.pass1.c -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long LongestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh =
+{0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+int 
+init_text_1 (char *FileName)
+{
+  int which;
+
+  if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  for (which = 1; which >= 0; which--)
+    {
+      u_char *word;
+      hash_rec *ent;
+      dict_data *dd = &DictData[which];
+
+      dd->wordnum = 0;
+      dd->words_read = 0;
+      dd->bytes_diff = 0;
+      dd->HashSize = INITIAL_HASH_SIZE;
+      dd->HashUsed = 0;
+
+      if (!(dd->HashTable = Xmalloc (sizeof (hash_rec) * dd->HashSize)))
+	{
+	  Message ("Unable to allocate memory for table");
+	  return (COMPERROR);
+	}
+      ChangeMem (sizeof (hash_rec) * dd->HashSize);
+      bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);
+
+      word = Pool;
+      *Pool++ = '\0';
+      PoolLeft--;
+      {
+	register u_char *wptr;
+	register int hsize = dd->HashSize;
+	register unsigned long hashval, step;
+
+	HASH (hashval, step, word, hsize);
+	wptr = (dd->HashTable + hashval)->word;
+	while (wptr)
+	  {
+	    hashval += step;
+	    if (hashval >= hsize)
+	      hashval -= hsize;
+	    wptr = (dd->HashTable + hashval)->word;
+	  }
+	ent = dd->HashTable + hashval;
+      }
+      ent->wcnt = 1;
+      ent->word = word;
+      dd->HashUsed = 1;
+    }
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_text_1 (u_char * s_in, int l_in)
+{
+  int which;
+  u_char *end = s_in + l_in - 1;
+
+  if (l_in > LongestDoc)
+    LongestDoc = l_in;
+
+  csh.num_docs++;
+  csh.num_bytes += l_in;
+
+  which = inaword (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      dict_data *dd = &DictData[which];
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+      dd->wordnum++;
+      inputbytes += *Word;
+      dd->words_read++;
+
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = dd->HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = dd->HashTable + hashval;
+	    if (!ent->word)
+	      {
+		int len = *Word + 1;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->occurance_num = occurance_num++;
+		ent->wcnt = 1;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		dd->HashUsed++;
+		dd->bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		break;
+	      }
+	  }
+      }
+
+
+      if (dd->HashUsed >= dd->HashSize >> 1)
+	{
+	  hash_rec *ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (dd->HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  ChangeMem (sizeof (hash_rec) * size);
+	  bzero ((char *) ht, sizeof (hash_rec) * size);
+
+	  for (i = 0; i < dd->HashSize; i++)
+	    if (dd->HashTable[i].word)
+	      {
+		register u_char *wptr;
+		register unsigned long hashval, step;
+
+		wptr = dd->HashTable[i].word;
+		HASH (hashval, step, wptr, size);
+		wptr = (ht + hashval)->word;
+		while (wptr)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    wptr = (ht + hashval)->word;
+		  }
+		ht[hashval] = dd->HashTable[i];
+	      }
+	  Xfree (dd->HashTable);
+	  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+	  dd->HashTable = ht;
+	  dd->HashSize = size;
+
+
+	}
+    }
+  return (COMPALLOK);
+}				/* encode */
+
+
+
+static int 
+PackHashTable (dict_data * dd)
+{
+  int s, d;
+  for (s = d = 0; s < dd->HashSize; s++)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  if (!(dd->HashTable = Xrealloc (dd->HashTable,
+				  sizeof (hash_rec) * dd->HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *s1, const void *s2)
+{
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+
+static void 
+WriteHashTable (FILE * fp, dict_data * dd)
+{
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR)
+    return;
+
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; j++)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+
+  for (j = 0; j < dd->HashSize; j++)
+    {
+      curr = dd->HashTable[j].word;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(dd->HashTable[j].wcnt);
+      HTONUL(dd->HashTable[j].occurance_num);
+
+      fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+      fwrite (&dd->HashTable[j].occurance_num,
+	      sizeof (dd->HashTable[j].occurance_num), 1, fp);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(dd->HashTable[j].wcnt);
+      NTOHUL(dd->HashTable[j].occurance_num);
+
+      fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+    }
+}
+
+
+int 
+done_text_1 (char *file_name)
+{
+  char *temp_str;
+  FILE *fp;
+
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+	      file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+	      strerror(errno)
+#else
+	      " "
+#endif
+	      );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  fclose(fp);
+  return COMPALLOK;
+}				/* done_encode */
Index: /main/tags/3.01/gsdl/packages/mg/src/text/text.pass2.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/text.pass2.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/text.pass2.c	(revision 21112)
@@ -0,0 +1,788 @@
+/**************************************************************************
+ *
+ * text.pass2.c -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "huffman.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+
+
+
+
+/*
+   $Log$
+   Revision 1.2  2002/01/22 02:41:14  jrm21
+   Try to print out an error if the aux file couldn't be written to.
+
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:10  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:14  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define POOL_SIZE 1024*256
+
+typedef struct char_pool
+  {
+    struct char_pool *next;
+    u_long left;
+    u_char *ptr;
+    u_char pool[POOL_SIZE];
+  }
+char_pool;
+
+typedef struct novel_hash_rec
+  {
+    u_long ordinal_num;
+    u_char *word;
+  }
+novel_hash_rec;
+
+
+#define INITIAL_HASH_SIZE 7927
+#define MAX_SWAPS 10000
+
+typedef struct novel_hash_table
+  {
+    novel_hash_rec *HashTable;
+    u_long HashSize, HashUsed;
+    char_pool *first_pool;
+    char_pool *pool;
+    u_long next_num, binary_start;
+    novel_hash_rec **code_to_nhr;
+  }
+novel_hash_table;
+
+
+static FILE *text, *text_idx;
+
+static u_char *comp_buffer;
+
+static u_long text_length;
+
+/* [RJM 07/97: 4G limit] */
+static double stats_in_tot_bytes = 0.0;
+static double stats_in_bytes = 0.0;
+static double stats_out_bytes = 0.0;
+
+
+static novel_hash_table nht[2];
+
+static u_long prefix_len = 0;
+
+int blk_start[2][33], blk_end[2][33];
+
+
+static char_pool *
+new_pool (char_pool * pool)
+{
+  char_pool *p = Xmalloc (sizeof (char_pool));
+  if (!p)
+    FatalError (1, "Unable to allocate memory for pool");
+  if (pool)
+    pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+
+int 
+init_text_2 (char *file_name)
+{
+  char path[512];
+  int i;
+
+  if (LoadCompressionDictionary (make_name (file_name, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  if (!(text = create_file (file_name, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  bzero ((char *) &cth, sizeof (cth));
+
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  text_length = sizeof (u_long) + sizeof (cth);
+
+  if (!(text_idx = create_file (file_name, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  if (fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+
+  if (!(comp_buffer = Xmalloc (sizeof (u_char) * buf_size)))
+    {
+      Message ("No memory for compression buffer");
+      return (COMPERROR);
+    }
+
+#if 0
+  MaxMemInUse += sizeof (u_char) * buf_size;
+#endif
+
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; i++)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	bzero ((char *) nht[i].HashTable,
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  nht[i].code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+					((nht[i].HashSize >> 1) + 2));
+	else
+	  nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID ||
+	    cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		num++;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+
+int 
+ic (void *a, void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+
+
+/* #define DOCDUMP 477 */
+
+int 
+process_text_2 (u_char * s_in, int l_in)
+{
+  int which, byte_length;
+  u_char *end = s_in + l_in - 1;
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+
+  which = inaword (s_in, end);
+
+  ENCODE_START (comp_buffer, buf_size)
+
+    ENCODE_BIT (which);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_ENCODE_L (nht[0].binary_start, prefix_len);
+      DELTA_ENCODE_L (nht[1].binary_start, prefix_len);
+    }
+
+  novels_used[0] = novels_used[1] = 0;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("---------------------------------------------------\n");
+      printf ("which = %d\n", which);
+    }
+#endif
+
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      int res;
+
+      if (which)
+	cth.num_of_words++;
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+#ifdef DOCDUMP
+      if (cth.num_of_docs == DOCDUMP)
+	{
+	  printf ("%sword : \"%.*s\"", which ? "    " : "non-", Word[0], Word + 1);
+	}
+#endif
+
+      /* Search the hash table for Word */
+      if (ht[which])
+	{
+	  register unsigned long hashval, step;
+	  register int tsize = ht[which]->size;
+	  register u_char **wptr;
+	  HASH (hashval, step, Word, tsize);
+	  for (;;)
+	    {
+	      register u_char *s1;
+	      register u_char *s2;
+	      register int len;
+	      wptr = ht[which]->table[hashval];
+	      if (wptr == NULL)
+		{
+		  res = COMPERROR;
+		  break;
+		}
+
+	      /* Compare the words */
+	      s1 = Word;
+	      s2 = *wptr;
+	      len = *s1 + 1;
+	      for (; len; len--)
+		if (*s1++ != *s2++)
+		  break;
+
+	      if (len)
+		{
+		  hashval += step;
+		  if (hashval >= tsize)
+		    hashval -= tsize;
+		}
+	      else
+		{
+		  res = ht[which]->table[hashval] - ht[which]->words;
+		  break;
+		}
+	    }
+	}
+      else
+	res = COMPERROR;
+      /* Check that the word was found in the dictionary */
+      if (res == COMPERROR)
+	{
+	  if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	    {
+	      Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      return (COMPERROR);
+	    }
+	  if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	    {
+	      u_long i;
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      HUFF_ENCODE (Word[0], lens_codes[which], lens_huff[which].clens);
+	      for (i = 0; i < Word[0]; i++)
+		HUFF_ENCODE (Word[i + 1], char_codes[which],
+			     char_huff[which].clens);
+	    }
+	  if (cdh.dict_type == MG_SEED_DICTIONARY)
+	    {
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      switch (cdh.novel_method)
+		{
+		case MG_NOVEL_HUFFMAN_CHARS:
+		  {
+		    u_long i;
+		    HUFF_ENCODE (Word[0], lens_codes[which],
+				 lens_huff[which].clens);
+		    for (i = 0; i < Word[0]; i++)
+		      HUFF_ENCODE (Word[i + 1], char_codes[which],
+				   char_huff[which].clens);
+		  }
+		  break;
+		case MG_NOVEL_BINARY:
+		case MG_NOVEL_DELTA:
+		case MG_NOVEL_HYBRID:
+		case MG_NOVEL_HYBRID_MTF:
+		  {
+		    register unsigned long hashval, step;
+		    register novel_hash_table *h = &nht[which];
+		    register int hsize = h->HashSize;
+		    register novel_hash_rec *ent;
+		    HASH (hashval, step, Word, hsize);
+		    for (;;)
+		      {
+			register u_char *s1, *s2;
+			register int len;
+			ent = h->HashTable + hashval;
+			if (!ent->word)
+			  {
+			    int len = *Word + 1;
+			    if (len > h->pool->left)
+			      h->pool = new_pool (h->pool);
+			    ent->word = h->pool->ptr;
+			    ent->ordinal_num = h->next_num++;
+			    if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			      h->code_to_nhr[ent->ordinal_num - 1] = ent;
+			    memcpy (h->pool->ptr, Word, len);
+			    h->pool->ptr += len;
+			    h->pool->left -= len;
+			    h->HashUsed++;
+			    break;
+			  }
+			/* Compare the words */
+			s1 = Word;
+			s2 = ent->word;
+			len = *s1 + 1;
+			for (; len; len--)
+			  if (*s1++ != *s2++)
+			    break;
+
+			if (!len)
+			  break;
+
+			hashval = (hashval + step);
+			if (hashval >= hsize)
+			  hashval -= hsize;
+		      }
+
+		    switch (cdh.novel_method)
+		      {
+		      case MG_NOVEL_BINARY:
+			{
+			  BINARY_ENCODE (ent->ordinal_num, h->binary_start);
+			  if (ent->ordinal_num == h->binary_start)
+			    h->binary_start++;
+			}
+			break;
+		      case MG_NOVEL_DELTA:
+			{
+			  DELTA_ENCODE (ent->ordinal_num);
+			}
+			break;
+		      case MG_NOVEL_HYBRID:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+			}
+			break;
+		      case MG_NOVEL_HYBRID_MTF:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+
+			  if (ent->ordinal_num - 1 >= novels_used[which])
+			    {
+			      int a = novels_used[which];
+			      int b = ent->ordinal_num - 1;
+			      novel_hash_rec *temp;
+
+
+/*                            fprintf(stderr, "a = %d , b = %d\n", a, b);
+ */
+			      temp = h->code_to_nhr[a];
+			      h->code_to_nhr[a] = h->code_to_nhr[b];
+			      h->code_to_nhr[b] = temp;
+			      h->code_to_nhr[a]->ordinal_num = a + 1;
+			      h->code_to_nhr[b]->ordinal_num = b + 1;
+			      if (novels_used[which] == MAX_SWAPS)
+				FatalError (1, "Not enough mem for swapping");
+			      swaps[which][novels_used[which]] = b;
+			      novels_used[which]++;
+			    }
+			}
+			break;
+		      }
+		    if (h->HashUsed >= h->HashSize >> 1)
+		      {
+			novel_hash_rec *ht;
+			unsigned long size;
+			unsigned long i;
+			size = prime (h->HashSize * 2);
+			if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			  {
+			    Xfree (h->code_to_nhr);
+			    h->code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+						      ((size >> 1) + 2));
+			  }
+			if (!(ht = Xmalloc (sizeof (novel_hash_rec) * size)))
+			  {
+			    Message ("Unable to allocate memory for table");
+			    return (COMPERROR);
+			  }
+			bzero ((char *) ht, sizeof (novel_hash_rec) * size);
+
+			for (i = 0; i < h->HashSize; i++)
+			  if (h->HashTable[i].word)
+			    {
+			      register u_char *wptr;
+			      register unsigned long hashval, step;
+
+			      wptr = h->HashTable[i].word;
+			      HASH (hashval, step, wptr, size);
+			      wptr = (ht + hashval)->word;
+			      while (wptr)
+				{
+				  hashval += step;
+				  if (hashval >= size)
+				    hashval -= size;
+				  wptr = (ht + hashval)->word;
+				}
+			      ht[hashval] = h->HashTable[i];
+			      if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+				h->code_to_nhr[ht[hashval].ordinal_num - 1] =
+				  &ht[hashval];
+			    }
+			Xfree (h->HashTable);
+			h->HashTable = ht;
+			h->HashSize = size;
+		      }
+		  }
+		  break;
+		}
+	    }
+	}
+      else
+	{
+	  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+#ifdef DOCDUMP
+	  if (cth.num_of_docs == DOCDUMP)
+	    {
+	      printf ("   %d %d\n", ht[which]->hd->clens[res],
+		      ht[which]->codes[res]);
+	    }
+#endif
+	}
+    }
+
+
+  /* Add a 1 bit onto the end of the buffer the remaining bits in the last
+     byte will all be zero */
+
+  ENCODE_BIT (1);
+
+  ENCODE_FLUSH;
+
+  byte_length = __pos - __base;
+  if (!__remaining)
+    {
+      Message ("The end of the buffer was probably overrun");
+      return COMPERROR;
+    }
+
+  ENCODE_DONE
+
+#ifdef DOCDUMP
+    if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("unused bits = %d\n", bits_unused);
+    }
+#endif
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  text_length += byte_length;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      int i;
+      for (i = 0; i < byte_length; i++)
+	printf ("%02x ", comp_buffer[i]);
+      printf ("\n");
+    }
+#endif
+
+  if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  novel_hash_rec *temp;
+	  temp = nht[which].code_to_nhr[a];
+	  nht[which].code_to_nhr[a] = nht[which].code_to_nhr[b];
+	  nht[which].code_to_nhr[b] = temp;
+	  nht[which].code_to_nhr[a]->ordinal_num = a + 1;
+	  nht[which].code_to_nhr[b]->ordinal_num = b + 1;
+	}
+
+
+  fwrite (comp_buffer, sizeof (*comp_buffer), byte_length, text);
+
+  if ((double) l_in / (double) byte_length > cth.ratio)
+    cth.ratio = (double) l_in / (double) byte_length;
+
+  cth.num_of_docs++;
+  if (l_in > cth.length_of_longest_doc)
+    cth.length_of_longest_doc = l_in;
+
+  cth.num_of_bytes += l_in;
+
+  if (Comp_Stats)
+    {
+      stats_in_tot_bytes += l_in;
+      stats_in_bytes += l_in;
+      stats_out_bytes += byte_length;
+      if (stats_in_bytes >= comp_stat_point)
+	{
+	  fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+		   stats_in_bytes, stats_out_bytes,
+		   (double) stats_out_bytes / (double) stats_in_bytes);
+	  stats_in_bytes = 0.0;
+	  stats_out_bytes = 0.0;
+	}
+    }
+
+  return COMPALLOK;
+}
+
+
+
+
+
+
+int 
+write_aux_dict (char *FileName)
+{
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (FileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              FileName, TEXT_DICT_AUX_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+              " "
+#endif
+              );
+      return COMPERROR;
+    }
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+
+      fwrite (&afh, sizeof (afh), 1, aux);
+
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+void 
+estimate_compressed_aux_dict (void)
+{
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      bzero ((char *) chars, sizeof (chars));
+      bzero ((char *) lens, sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      lens[len]++;
+	      total_uncomp += len + 4;
+	      for (; len; len--)
+		chars[*buf++]++;
+	    }
+	}
+      for (j = 0; j < 256; j++)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; j++)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+
+      aux_compressed += (Calculate_Huffman_Size (16, flens, lens) +
+			 Calculate_Huffman_Size (256, fchars, chars)) / 8;
+
+    }
+
+  Message ("Aux dictionary (Uncompressed) %.2f Mb ( %u bytes   %0.3f %%)",
+	   total_uncomp / 1024.0 / 1024, total_uncomp,
+	   (total_uncomp * 100.0) / bytes_processed);
+  Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	   (aux_compressed * 100.0) / bytes_processed);
+}
+
+
+
+
+
+
+int 
+done_text_2 (char *FileName)
+{
+  if (Comp_Stats)
+    fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+	     stats_in_bytes, stats_out_bytes,
+	     (double) stats_out_bytes / (double) stats_in_bytes);
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth.num_of_docs);
+  HTOND(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth.num_of_words);
+  HTONUL(cth.length_of_longest_doc);
+  HTOND(cth.ratio);
+
+  if (fseek (text_idx, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+  fclose (text_idx);
+
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+  fclose (text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+  
+
+  Message ("Compressed Text %.2f Mb ( %u bytes   %0.3f %%)",
+	   text_length / 1024.0 / 1024.0, text_length,
+	   (text_length * 100.0) / bytes_processed);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / bytes_processed);
+
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_BINARY ||
+       cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID ||
+       cdh.novel_method == MG_NOVEL_HYBRID_MTF))
+    {
+      if (write_aux_dict (FileName) == COMPERROR)
+	return COMPERROR;
+      estimate_compressed_aux_dict ();
+    }
+  else
+    {
+      if (cdh.dict_type != MG_COMPLETE_DICTIONARY)
+	Message ("Huffman info for chars in dictionary %.2f Mb"
+		 " ( %u bytes   %0.3f %%)",
+		 Chars_disk / 1024.0 / 1024, Chars_disk,
+		 (Chars_disk * 100.0) / bytes_processed);
+      unlink (make_name (FileName, TEXT_DICT_AUX_SUFFIX, NULL));
+    }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/text_get.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/text_get.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/text_get.c	(revision 21112)
@@ -0,0 +1,1087 @@
+/**************************************************************************
+ *
+ * text_get.c -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "text_get.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg_errors.h"
+#include "local_strings.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:11  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:15  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+
+
+
+
+static double 
+FetchDocStartLev1 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  unsigned long data[2];
+  /* [TS:Sep/94] Fixed up the seek call to give the correct offset */
+  Fseek (td->TextIdxFile,
+	 sizeof (unsigned long) * (DN - 1) +	/* the doc offsets */
+	 sizeof (unsigned long) +	/* the magic number */
+	 sizeof (compressed_text_header),	/* the header */
+	 0);
+  Fread ((char *) &data, sizeof (data), 1, td->TextIdxFile);
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(data[0]);
+  NTOHUL(data[1]);
+
+  *seek_pos = data[0];
+  *len = data[1] - data[0];
+  return (1.0);
+}
+
+#define MG_PAGE_SIZE 2048
+
+static int 
+LoadIdx (text_data * td, unsigned long DN)
+{
+  if (!td->idx_data)
+    {
+      td->idx_data = Xmalloc (sizeof (*(td->idx_data)) * MG_PAGE_SIZE);
+      if (!td->idx_data)
+	FatalError (1, "Out of memory in FDSL2");
+    }
+  if (td->current_pos == -1 || DN >= td->current_pos + MG_PAGE_SIZE - 1 ||
+      DN < td->current_pos)
+    {
+      int i, num;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      long rn = (long) DN - (MG_PAGE_SIZE >> 1);
+      if (rn < 1)
+	rn = 1;
+      Fseek (td->TextIdxWgtFile, (sizeof (unsigned long) + sizeof (float)) *
+	       (rn - 1) + sizeof (unsigned long), 0);
+      num = Fread ((char *) td->idx_data, sizeof (*(td->idx_data)), MG_PAGE_SIZE, /* [RPAP - Jan 97: Endian Ordering] */
+		   td->TextIdxWgtFile);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num; i++)
+	{
+	  NTOHUL(td->idx_data[i].Start);
+	  NTOHF(td->idx_data[i].Weight);
+	}
+
+      td->current_pos = rn;
+    }
+  return DN - td->current_pos;
+}
+
+static double 
+FDSL2 (text_data * td, unsigned long DN, unsigned long *Pos)
+{
+  unsigned long pos = LoadIdx (td, DN);
+  *Pos = td->idx_data[pos].Start;
+  return (td->idx_data[pos].Weight);
+}
+
+
+static double 
+FetchDocStartLev2 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  double Weight;
+  unsigned long s1, s2;
+  Weight = FDSL2 (td, DN, &s1);
+  do
+    {
+      DN++;
+      FDSL2 (td, DN, &s2);
+    }
+  while (s2 == s1);
+  *seek_pos = s1;
+  *len = s2 - s1;
+  return (Weight);
+}
+
+
+
+
+double 
+FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len)
+{
+  qd->text_idx_lookups++;
+  if (qd->td->TextIdxWgtFile)
+    return FetchDocStartLev2 (qd->td, DN, seek_pos, len);
+  else
+    return FetchDocStartLev1 (qd->td, DN, seek_pos, len);
+}
+
+unsigned long 
+FetchInitialParagraph (text_data * td, unsigned long ParaNum)
+{
+  if (td->TextIdxWgtFile)
+    {
+      unsigned long pos;
+      unsigned long start;
+      int PN = ParaNum - 1;
+      pos = LoadIdx (td, ParaNum);
+      start = td->idx_data[pos].Start;
+      while (PN > 0)
+	{
+	  pos = LoadIdx (td, PN);
+	  if (td->idx_data[pos].Start != start)
+	    return PN + 1;
+	  PN--;
+	}
+      return PN + 1;
+    }
+  else
+    return ParaNum;
+}
+
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int 
+FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt)
+{
+  if (!DocEnt->SeekPos)
+    FetchDocStart (qd, DocEnt->DocNum, &DocEnt->SeekPos, &DocEnt->Len);
+  if (!(*DocBuff = Xmalloc (DocEnt->Len)))
+    return (-1);
+
+  if (Fseek (qd->td->TextFile, DocEnt->SeekPos, 0) == -1)
+    FatalError (1, "Error when seeking into text file");
+#if 0
+  printf ("Loading compressed text %d %d\n", DocEnt->SeekPos, DocEnt->Len);
+#endif
+  if (Fread (*DocBuff, 1, DocEnt->Len, qd->td->TextFile) != DocEnt->Len)
+    FatalError (1, "Error when reading data");
+
+  return (DocEnt->Len);
+
+}
+
+
+text_data *
+LoadTextData (File * text, File * text_idx_wgt, File * text_idx)
+{
+  text_data *td;
+
+  if (!(td = Xmalloc (sizeof (text_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  td->TextFile = text;
+  td->TextIdxWgtFile = text_idx_wgt;
+  td->TextIdxFile = text_idx;
+  td->current_pos = -1;
+  td->idx_data = NULL;
+  Fread (&td->cth, sizeof (td->cth), 1, text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(td->cth.num_of_docs);
+  NTOHD(td->cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(td->cth.num_of_words);
+  NTOHUL(td->cth.length_of_longest_doc);
+  NTOHD(td->cth.ratio);
+
+  return (td);
+}
+
+
+void 
+FreeTextData (text_data * td)
+{
+  if (td)
+    {
+      if (td->idx_data)
+	Xfree (td->idx_data);
+      Xfree (td);
+    }
+}
+
+
+static int 
+pts_comp (const void *A, const void *B)
+{
+  const DocEntry *const *a = A;
+  const DocEntry *const *b = B;
+  return (*a)->DocNum - (*b)->DocNum;
+}
+
+
+
+
+int 
+GetPosLens (query_data * qd, DocEntry * Docs, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+  for (i = j = 0; i < num; i++, Docs++)
+    if (!Docs->SeekPos)
+      pts[j++] = Docs;
+
+  if (j)
+    {
+      qsort (pts, j, sizeof (DocEntry *), pts_comp);
+      for (i = 0; i < j; i++)
+	FetchDocStart (qd, pts[i]->DocNum, &pts[i]->SeekPos, &pts[i]->Len);
+    }
+
+  Xfree (pts);
+  return (0);
+}
+
+
+
+
+
+int 
+LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  int mem;
+
+  if (!num)
+    return (0);
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+
+  mem = i = 0;
+  do
+    {
+      pts[i] = Docs;
+      mem += Docs->Len;
+      i++;
+      Docs++;
+    }
+  while (i < num && mem < max_mem);
+  if (i > 1)
+    qsort (pts, i, sizeof (DocEntry *), pts_comp);
+  for (j = 0; j < i; j++)
+    {
+      if (FetchCompressed (qd, &pts[j]->CompTextBuffer, pts[j]) == -1)
+	return (-1);
+      ChangeMemInUse (qd, pts[j]->Len);
+    }
+
+  Xfree (pts);
+
+  return (i);
+}
+
+
+
+
+
+void 
+FreeBuffers (query_data * qd, DocEntry * Docs, int num)
+{
+  int i;
+  for (i = 0; i < num; i++, Docs++)
+    if (Docs->CompTextBuffer)
+      {
+	Xfree (Docs->CompTextBuffer);
+	Docs->CompTextBuffer = NULL;
+	ChangeMemInUse (qd, -Docs->Len);
+      }
+}
+
+
+
+/****************************************************************************/
+
+static void 
+FreeAuxDict (auxiliary_dict * ad)
+{
+  if (!ad)
+    return;
+  if (ad->word_data[0])
+    Xfree (ad->word_data[0]);
+  if (ad->word_data[1])
+    Xfree (ad->word_data[1]);
+  if (ad->words[0])
+    Xfree (ad->words[0]);
+  if (ad->words[1])
+    Xfree (ad->words[1]);
+  Xfree (ad);
+}
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict * cd, File * text_aux_dict)
+{
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = Xmalloc (sizeof (auxiliary_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) ad, sizeof (*ad));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      Fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = Xmalloc (ad->afh[i].mem_for_frags)))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+      if (!(ad->words[i] = Xmalloc (ad->afh[i].num_frags * sizeof (u_char *))))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+
+      Fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	  cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd->cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+
+
+
+
+static u_char ***
+ReadInWords (File * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  if (!(vals = Xmalloc (ptrs_reqd * sizeof (*vals))))
+    return (NULL);
+
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **))))
+    return (NULL);
+
+  if (!(next_word[0] = Xmalloc (mem_reqd))) return (NULL);
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = Getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      Fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static compression_dict *
+Load_Comp_Dict (File * dict, File * aux_dict)
+{
+  int which;
+  compression_dict *cd;
+
+  if (!(cd = Xmalloc (sizeof (compression_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) cd, sizeof (compression_dict));
+
+  cd->MemForCompDict = sizeof (compression_dict);
+
+  if (F_Read_cdh (dict, &cd->cdh, &cd->MemForCompDict, NULL) == -1)
+    return NULL;
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+
+	  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						 NULL)))
+	    return NULL;
+	  cd->escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  cd->chars_huff[which] = hd;
+	  cd->chars_vals[which] = vals;
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  cd->lens_huff[which] = hd;
+	  cd->lens_vals[which] = vals;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->chars_huff[which] = hd;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict,
+				       NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->chars_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      cd->lens_huff[which] = hd;
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict
+				       ,NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->lens_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+
+      if (!(cd->ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+    }
+
+
+  mg_errno = MG_NOERROR;
+
+  cd->fast_loaded = 0;
+  return (cd);
+}
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p, cd) & 7))) != 0)
+
+
+static compression_dict *
+Load_Fast_Comp_Dict (File * text_fast_comp_dict)
+{
+  compression_dict *cd;
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  Fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (!(cd = Xmalloc (mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  Fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = Xmalloc (fixup_mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; p++)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (i = 0; i <= 1; i++)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; j++)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  Xfree (fixup);
+  return (cd);
+}
+
+
+
+
+compression_dict *
+LoadCompDict (File * text_comp_dict,
+	      File * text_aux_dict,
+	      File * text_fast_comp_dict)
+{
+  return text_fast_comp_dict ?
+    Load_Fast_Comp_Dict (text_fast_comp_dict) :
+    Load_Comp_Dict (text_comp_dict, text_aux_dict);
+}
+
+
+
+
+void 
+FreeCompDict (compression_dict * cd)
+{
+  int which;
+  if (cd->fast_loaded)
+    {
+      Xfree (cd);
+      return;
+    }
+  for (which = 0; which < 2; which++)
+    {
+      if (cd->cfh[which])
+	Xfree (cd->cfh[which]);
+      if (cd->chars_huff[which])
+	Xfree (cd->chars_huff[which]);
+      if (cd->lens_huff[which])
+	Xfree (cd->lens_huff[which]);
+      if (cd->values[which])
+	{
+	  Xfree (cd->values[which][0][0]); /* [RJM 07/98: Memory Leak] */
+	  Xfree (cd->values[which][0]);
+	  Xfree (cd->values[which]);
+	}
+      if (cd->chars_vals[which])
+	{
+	  Xfree (cd->chars_vals[which][0]);
+	  Xfree (cd->chars_vals[which]);
+	}
+      if (cd->lens_vals[which])
+	{
+	  Xfree (cd->lens_vals[which][0]);
+	  Xfree (cd->lens_vals[which]);
+	}
+    }
+  if (cd->ad)
+    FreeAuxDict (cd->ad);
+  Xfree (cd);
+}
+
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+/*#define DUMPDOC */
+
+#define MAX_SWAPS 10000
+
+int 
+DecodeText (compression_dict * cd,
+	    u_char * s_in, int l_in, u_char * s_out, int *l_out)
+{
+  auxiliary_dict *ad = cd->ad;
+  int which;
+  u_long num_bits, bits;
+  u_char *ptr = s_out;
+  static int num = 0;
+  u_long binary_start[2];
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+  novels_used[0] = novels_used[1] = 0;
+
+  {
+    unsigned char bf = s_in[l_in - 1];
+    num_bits = 1;
+    while ((bf & 1) != 1)
+      {
+	num_bits++;
+	bf >>= 1;
+      }
+    num_bits = l_in * 8 - num_bits;
+  }
+
+  DECODE_START (s_in, l_in)
+
+    which = DECODE_BIT;
+  bits = 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_DECODE_L (binary_start[0], bits);
+      DELTA_DECODE_L (binary_start[1], bits);
+    }
+
+
+  while (bits < num_bits)
+    {
+      register unsigned code, len;
+      register int r;
+      register u_char *t, *b = NULL;
+      u_char word[MAXWORDLEN + 1];
+
+#ifdef DUMPDOC
+      printf ("\n%d %d  ", bits, num_bits);
+#endif
+      if (cd->cfh[which])
+	{
+	  MY_HUFF_DECODE (len, code, cd->cfh[which]->hd.min_code);
+	  bits += len;
+
+	  r = code & ((1 << cd->cdh.lookback) - 1);
+	  t = cd->values[which][len][code >> cd->cdh.lookback];
+
+	  /* step through from base pointer */
+	  b = word + 1;
+	  while (r--)
+	    {
+	      register int copy = *t >> 4;
+	      memcpy (word + copy + 1, t + 1, *t & 0xf);
+	      word[0] = copy + (*t & 0xf);
+	      t += ((*t) & 0xf) + 1;
+	    }
+	}
+      else
+	t = NULL;
+      if (t == cd->escape[which])
+	{
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      {
+		int len, i;
+		int c;
+		HUFF_DECODE_L (len, cd->lens_huff[which]->min_code,
+			       cd->lens_vals[which], bits);
+		for (i = 0; i < len; i++)
+		  {
+		    HUFF_DECODE_L (c, cd->chars_huff[which]->min_code,
+				   cd->chars_vals[which], bits);
+		    *ptr++ = c;
+		  }
+	      }
+	      break;
+	    case MG_NOVEL_BINARY:
+	    case MG_NOVEL_DELTA:
+	    case MG_NOVEL_HYBRID:
+	    case MG_NOVEL_HYBRID_MTF:
+	      {
+		int idx = 0, len;
+		u_char *base;
+		switch (cd->cdh.novel_method)
+		  {
+		  case MG_NOVEL_BINARY:
+		    {
+		      BINARY_DECODE_L (idx, binary_start[which], bits);
+		      if (idx == binary_start[which])
+			binary_start[which]++;
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_DELTA:
+		    {
+		      DELTA_DECODE_L (idx, bits);
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID_MTF:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		      if (idx >= novels_used[which])
+			{
+			  u_char *temp;
+			  temp = ad->words[which][idx];
+			  ad->words[which][idx] =
+			    ad->words[which][novels_used[which]];
+			  ad->words[which][novels_used[which]] = temp;
+			  swaps[which][novels_used[which]] = idx;
+			  idx = novels_used[which]++;
+			}
+		    }
+		    break;
+		  }
+		base = ad->words[which][idx];
+		len = *base++;
+#ifdef DUMPDOC
+		printf ("[[");
+#endif
+		for (; len; len--)
+		  {
+		    *ptr++ = *base++;
+#ifdef DUMPDOC
+		    putchar (*(base - 1));
+#endif
+		  }
+#ifdef DUMPDOC
+		printf ("]]");
+#endif
+	      }
+	      break;
+	    }
+	}
+      else
+	{
+	  /* copy over the matching prefix */
+	  r = (*t >> 4);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *b++;
+#else
+	    {
+	      *ptr = *b++;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+
+	  /* and the stored suffix */
+	  r = ((*t) & 0xf);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *++t;
+#else
+	    {
+	      *ptr = *++t;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+	}
+      which = !which;
+    }
+
+  DECODE_DONE
+
+    * l_out = ptr - s_out;
+  num += *l_out + 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  u_char *temp;
+	  temp = ad->words[which][a];
+	  ad->words[which][a] = ad->words[which][b];
+	  ad->words[which][b] = temp;
+	}
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/text_get.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/text_get.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/text_get.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * text_get.h -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT_GET
+#define H_TEXT_GET
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+double FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		u_char * s_in, int l_in, u_char * s_out, int *l_out);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/warranty.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/warranty.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/warranty.h	(revision 21112)
@@ -0,0 +1,24 @@
+static char *warranty_str[] = {
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/gsdl/packages/mg/src/text/warranty.src
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/warranty.src	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/warranty.src	(revision 21112)
@@ -0,0 +1,23 @@
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/gsdl/packages/mg/src/text/weights.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/weights.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/weights.c	(revision 21112)
@@ -0,0 +1,158 @@
+/**************************************************************************
+ *
+ * weights.c -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:27  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:18  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+approx_weights_data *
+LoadDocWeights (File * weight_file,
+		unsigned long num_of_docs)
+{
+  approx_weights_data *awd;
+  int num;
+
+  if (!(awd = Xmalloc (sizeof (*awd))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fseek (weight_file, sizeof (long), 0);
+  Fread (&awd->bits, sizeof (awd->bits), 1, weight_file);
+  Fread (&awd->L, sizeof (awd->L), 1, weight_file);
+  NTOHD(awd->L);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&awd->B, sizeof (awd->B), 1, weight_file);
+  NTOHD(awd->B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  awd->mask = awd->bits == 32 ? 0xffffffff : (1 << awd->bits) - 1;
+
+  num = (num_of_docs * awd->bits + 31) / 32;
+  if (!(awd->DocWeights = Xmalloc (sizeof (unsigned long) * num)))
+    {
+      Xfree (awd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (awd->DocWeights, sizeof (unsigned long), num, weight_file);
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    unsigned long i;
+    for (i = 0; i < num; i++)
+      NTOHUL(awd->DocWeights[i]);
+  }
+
+  awd->MemForWeights = num * sizeof (unsigned long);
+  awd->num_of_docs = num_of_docs;
+
+  mg_errno = MG_NOERROR;
+
+  if (awd->bits <= 12)
+    {
+      int i, size = (1 << awd->bits);
+      if (!(awd->table = (float *) Xmalloc (size * sizeof (float))))
+	  return (awd);
+      awd->table[0] = awd->L;
+      for (i = 1; i < size; i++)
+	awd->table[i] = awd->table[i - 1] * awd->B;
+    }
+  else
+    awd->table = NULL;
+  return awd;
+}
+
+
+
+
+
+
+/*
+ * the first document in the collection has DocNum = 0
+ */
+float 
+GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum)
+{
+  register unsigned long c, Pos;
+  register unsigned long *dw;
+  if (awd == NULL)
+    return 1.0;
+#if 0
+  if (DocNum < 0 || DocNum >= awd->num_of_docs)
+    FatalError (1, "Something is wrong in \"GetDocWeight\" DocNum = %d\n",
+		DocNum);
+#endif
+  Pos = DocNum * awd->bits;
+  dw = &(awd->DocWeights[Pos / MAXBITS]);
+  Pos &= (MAXBITS - 1);
+  c = *dw >> Pos;
+  if (Pos + awd->bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - Pos);
+  c &= awd->mask;
+  if (awd->table)
+    return (awd->table[c]);
+  else
+    return (awd->L * pow (awd->B, (double) c));
+
+}
+
+void 
+FreeWeights (approx_weights_data * awd)
+{
+  Xfree (awd->DocWeights);
+  if (awd->table)
+    Xfree (awd->table);
+  Xfree (awd);
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/weights.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/weights.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/weights.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * weights.h -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_WEIGHTS
+#define H_WEIGHTS
+
+approx_weights_data *LoadDocWeights (File * weight_file,
+				     unsigned long num_of_docs);
+
+float GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum);
+
+void FreeWeights (approx_weights_data * awd);
+
+#endif
Index: /main/tags/3.01/gsdl/packages/mg/src/text/words.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/words.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/words.c	(revision 21112)
@@ -0,0 +1,34 @@
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skipped. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+
+/* It determines whether a given place in a UTF-8 encoded 
+   Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/packages/mg/src/text/words.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/src/text/words.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/src/text/words.h	(revision 21112)
@@ -0,0 +1,395 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+#include "unitool.h"
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+/*#define MAXNUMERIC	4*/
+
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded 
+	   Unicode string is a unicode space. */
+ 
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with beginning 
+	   unicode spaces skipped. */
+ 
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+  /*  
+#define PARSE_WORD(Word, s_in, end)                                  \
+  do {                                                               \
+	  register u_char  *wptr = (Word)+1;                         \
+	  register int    length = 0;                                \
+	  register int    c = *(s_in);                               \
+	  register int	numeric = 0;                                 \
+                                                                     \
+	  while( length < MAXWORDLEN && INAWORD(c) && (s_in)<=(end)) \
+	    {                                                        \
+	      if ((numeric += INNUMBER(c)) > MAXNUMERIC)             \
+		break;                                               \
+	      *wptr++ = c;                                           \
+	      ++length;                                              \
+	      c = *++(s_in);                                         \
+	    }                                                        \
+	  *(Word) = length;                                          \
+  }while(0)
+  */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+	  register u_char  *wptr = (Word)+1;                       \
+	  register int    length = 0;                              \
+	  register int    c = *(s_in);                             \
+                                                                   \
+	  while( length < MAXWORDLEN && !INAWORD(c) && (s_in)<=(end) ) \
+	    {                                                      \
+	      *wptr++ = c;                                         \
+	      ++length;                                            \
+	      c = *++(s_in);                                       \
+	    }                                                      \
+	  *(Word) = length;                                        \
+  }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_STEM_WORD 
+ * Description: 
+ *      Extracts out Word.      
+ * Input: 
+ *      s_in points to 1st letter in buffer to test
+ *      end points to last letter in buffer
+ * Output: 
+ *      s_in is modified to move to next word
+ *      Returns Word filled in with length in 1st byte.
+ * ========================================================================= */
+#define PARSE_STEM_WORD(Word, s_in, end)                           \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXSTEMLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+    /*
+#define PARSE_STEM_WORD(Word, s_in, end)                      \
+  do                                                          \
+    {                                                         \
+      register u_char  *wptr = (Word)+1;                      \
+      register int    length = 0;                             \
+      register int    c = *(s_in);                            \
+      register int    numeric = 0;                            \
+                                                              \
+      while ( length < MAXSTEMLEN && INAWORD(c) && (s_in)<=(end)) \
+        {                                                     \
+ 	  if ((numeric += INNUMBER(c)) > MAXNUMERIC)          \
+	    break;                                            \
+	  *wptr++ = c;                                        \
+	  ++length;                                           \
+	  c = *++(s_in);                                      \
+	}                                                     \
+      *(Word) = length;                                       \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD 
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD(s_in, end)                             \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD(s_in, end)           \
+  do                                             \
+    {                                            \
+      while (!INAWORD(*(s_in)) && (s_in)<=(end)) \
+	(s_in)++;                                \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD_OR_SGML_TAG 
+ * Description: 
+ *      Like PARSE_NON_STEM_WORD but also eats up SGML tags
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end)                 \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == '<') {                                              \
+	while (charlength > 0 && c != '>') {                       \
+	  (s_in) += charlength;                                    \
+	  charlength = parse_utf8_char((s_in),(end),&c);           \
+	}                                                          \
+      }                                                            \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end) \
+  do                                               \
+    {                                              \
+      register int    c = *(s_in);                 \
+                                                   \
+      while (!INAWORD(c) && (s_in)<=(end))         \
+        {                                          \
+	  if (c == '<')                            \
+            {                                      \
+	      while (c != '>' && (s_in)<=(end))    \
+		c = *++(s_in);                     \
+            }                                      \
+	  if ((s_in)<=(end))                       \
+	    c = *++(s_in);                         \
+	}                                          \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_OPT_TERM_PARAM     [RPAP - Jan 97: Stem Index Change]
+ * Description: 
+ *      Extracts out optional paramater for query term.
+ *      Needed only in parsing the query line !
+ *
+ *      Note: that this function has not been converted to use UTF-8
+ *            as it should still work as it is (only uses ascii
+ *            characters)
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_OPT_TERM_PARAM(Param, type, s_in, end)                       \
+  do                                                                       \
+    {                                                                      \
+	  register u_char  *wptr = (Param);                                \
+	  register int    length = 0;                                      \
+	  register int    c = *(s_in);                                     \
+                                                                           \
+          if (c == WEIGHTPARAM || c == STEMPARAM)                          \
+	    {                                                              \
+	      type = c;                                                    \
+	      c = *++(s_in);                                               \
+	      while( length < MAXPARAMLEN && isdigit(c) && (s_in)<=(end))  \
+		{                                                          \
+	           *wptr++ = c;                                            \
+	           ++length;                                               \
+	           c = *++(s_in);                                          \
+	        }                                                          \
+	      *wptr = '\0';                                                \
+              for (; isdigit(c) && (s_in)<=(end); c = *++(s_in))           \
+                ;                                                          \
+            }							           \
+    }while(0)
+
+/* =========================================================================
+ * Macro: PARSE_RANKED_NON_STEM_WORD    [RJM 07/97: Ranked Required Terms]
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      If come across a match requirement store it in require_match
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above
+ * Output: the requirement mode for the next term. -1=must not match,
+ *      0=optional match, 1=must match
+ * ========================================================================= */
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)       \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+    (require_match) = 0;                                           \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == MUSTMATCHPARAM) (require_match) = 1;                \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)         \
+  do {                                            \
+    (require_match) = 0;                          \
+    while (!INAWORD(*(s_in)) && (s_in)<=(end)) {  \
+      if (*(s_in) == MUSTMATCHPARAM) {            \
+        (require_match) = 1;                      \
+      }                                           \
+      (s_in)++;                                   \
+    }                                             \
+  } while (0)
+    */
Index: /main/tags/3.01/gsdl/packages/mg/stamp-h.in
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/gsdl/packages/mg/stemmerinterface.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/stemmerinterface.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/stemmerinterface.c	(revision 21112)
@@ -0,0 +1,39 @@
+#include "lib/lovinstem.h"
+#include <stdio.h>
+#include <ctype.h>
+
+int main () {
+  char word[256];
+  char c;
+  int len;
+
+  c = fgetc (stdin);
+  while (!feof (stdin)) {
+    /* output non words */
+    while (!feof (stdin) && !isalnum (c)) {
+      fputc (c, stdout);
+      c = fgetc (stdin);
+    }
+
+    /* get the word */
+    len = 0;
+    while (!feof (stdin) && isalnum (c) && len < 255) {
+      len++;
+      word[len] = c;
+      c = fgetc (stdin);
+    }
+    word[0] = len;
+
+    /* if we found a word stem and output it */
+    if (len > 0) {
+      lovinstem (word);
+      len = 0;
+      while (len < word[0]) {
+        len++;
+        fputc(word[len], stdout);
+      }
+    }
+  }
+}
+
+
Index: /main/tags/3.01/gsdl/packages/mg/sysfuncs.h
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/sysfuncs.h	(revision 21112)
@@ -0,0 +1,362 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#include <sys/stat.h>
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#if defined(__MSDOS__)
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#if defined(__GNUC__) && defined(__WIN32__)
+#include <limits.h>
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#if defined(__WIN32__)
+#include <io.h>
+#define lseek _lseek
+#elif defined(__MSDOS__)
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#include <pathmax.h>
+
+/* jrm21 2003 - this is in unistd.h now */
+#ifndef HAVE_UNISTD_H
+#include <getopt.h>
+#endif
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#if (HAVE_VPRINTF || HAVE_DOPRNT) && __STDC__
+/*void error __P ((int, int, const char *,...));*/
+void error (int, int, const char *,...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/gsdl/packages/mg/test/Makefile
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/Makefile	(revision 21112)
@@ -0,0 +1,10 @@
+DISTFILES = Makefile q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum simple_sum.c alice.len
+
+all: simple_sum
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+simple_sum: simple_sum.c
+	cc -o $@ $?
Index: /main/tags/3.01/gsdl/packages/mg/test/alice.len
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/alice.len	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/alice.len	(revision 21112)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19321
Index: /main/tags/3.01/gsdl/packages/mg/test/alice.sum
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/alice.sum	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/alice.sum	(revision 21112)
@@ -0,0 +1,8 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict.blocked: 1030626 33214
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /main/tags/3.01/gsdl/packages/mg/test/mgrc1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/mgrc1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/mgrc1	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query boolean
Index: /main/tags/3.01/gsdl/packages/mg/test/mgrc2
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/mgrc2	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/mgrc2	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query ranked
Index: /main/tags/3.01/gsdl/packages/mg/test/mgrc3
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/mgrc3	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/mgrc3	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode docnums
+.set query boolean
Index: /main/tags/3.01/gsdl/packages/mg/test/q1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/q1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/q1	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/gsdl/packages/mg/test/q2
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/q2	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/q2	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/gsdl/packages/mg/test/q3
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/q3	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/q3	(revision 21112)
@@ -0,0 +1,2 @@
+Rabbit & !(Alice | she)
+laser
Index: /main/tags/3.01/gsdl/packages/mg/test/r1
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/r1	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/r1	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /main/tags/3.01/gsdl/packages/mg/test/r2
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/r2	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/r2	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /main/tags/3.01/gsdl/packages/mg/test/r3
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/r3	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/r3	(revision 21112)
@@ -0,0 +1,21 @@
+> Rabbit & !(Alice | she)
+Rabbit Alice she
+     14.14     
+    126.126    
+    143.143    
+    478.478    
+    701.701    
+    704.704    
+    705.705    
+    755.755    
+    774.774    
+    786.786    
+    788.788    
+    791.791    
+    793.793    
+    801.801    
+    803.803    
+15 documents retrieved.
+> laser
+laser
+No entries correspond to that query.
Index: /main/tags/3.01/gsdl/packages/mg/test/simple_sum.c
===================================================================
--- /main/tags/3.01/gsdl/packages/mg/test/simple_sum.c	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/mg/test/simple_sum.c	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * sum.c -- byte summing program
+ * Copyright (C) 1995  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:18:33  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:36:00  rjmcnab
+*** empty log message ***
+
+*/
+
+/*
+ * If given -l option then just prints the length.
+ * Normally prints the sum and the length for the file.
+ */
+
+static int length_only = 0;
+
+#include <stdio.h>
+
+void 
+output_sum(fname, input_file, output_file)
+char *fname;
+FILE *input_file;
+FILE *output_file;
+{
+  int ch = '\0';
+  unsigned long sum = 0;
+  unsigned long num_bytes = 0;
+
+  while((ch = getc(input_file))!=EOF){
+    sum += ch;
+    num_bytes++;
+  }
+
+  if (length_only)
+    fprintf(output_file, "%s: %ld\n", fname, num_bytes);
+  else
+    fprintf(output_file, "%s: %ld %ld\n", fname, sum, num_bytes);
+}
+
+void
+main(argc, argv)
+int argc;
+char *argv[];
+{
+  int i = 0;
+
+  /* use stdin */
+  if (argc == 1){
+    output_sum("-", stdin, stdout);
+    exit(0);
+  }
+
+  /* use args as file names for input */
+  for (i=1;i<argc; i++){
+    char *fname = argv[i];
+    FILE *input_file = NULL;
+    
+    if (!length_only && (argv[i][0] == '-') && (argv[i][1] =='l') ) 
+      { 
+        length_only = 1; 
+        continue;
+      }
+
+    input_file = fopen(fname, "r");
+    if (!input_file){
+      fprintf(stderr,"Could not open %s\n", fname);
+      continue;
+    }
+    else{
+      output_sum(fname, input_file, stdout);
+      fclose(input_file);
+    }
+  }/*for*/
+  exit(0);
+
+}
Index: /main/tags/3.01/gsdl/packages/pdftohtml/readme-gs.txt
===================================================================
--- /main/tags/3.01/gsdl/packages/pdftohtml/readme-gs.txt	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/pdftohtml/readme-gs.txt	(revision 21112)
@@ -0,0 +1,16 @@
+This is pdftohtml, which was based at:
+     http://www.ra.informatik.uni-stuttgart.de/~gosho/pdftohtml/
+
+It has recently been picked up again, and is currently based at:
+    http://pdftohtml.sourceforge.net/
+
+We're currently using version 0.34 as downloaded from
+http://pdftohtml.sourceforge.net. The only alterations made to the source
+package were to remove some "CVS" directories and tidy up the makefiles
+slightly (to use ./configure as with the rest of Greenstone).
+
+Also, the #pragma statements were removed to work around a compiler bug
+on Mac OS X.
+
+Stefan Boddie
+December 2002
Index: /main/tags/3.01/gsdl/packages/rtftohtml/rtftohtml-gs.txt
===================================================================
--- /main/tags/3.01/gsdl/packages/rtftohtml/rtftohtml-gs.txt	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/rtftohtml/rtftohtml-gs.txt	(revision 21112)
@@ -0,0 +1,56 @@
+RTF TO HTML
+-----------
+
+This is based on version 2.7.5 of rtftohtml, which was last modified in
+November, 1994.
+It contains the text:
+###
+Initial Implementation by Chris Hector (cjh@cray.com)
+(c) 1994 by Chris Hector
+May be freely distributed.
+This translator was based on rtfskel.
+###
+The last known email address I can find for him is <cjh@cray.com>.
+I think that <chris@sunpack.com> might reach him, but I'm not sure.
+
+
+Versions 3.0 of rtftohtml and up are released under a shareware license,
+and available from http://www.logictran.com (although it now has a new
+name). You can download a free binary version of the software for most
+platforms from this site.
+
+
+
+
+rtftohtml is based on RTF Tools, which is (c) April 1994 by Paul DuBois
+<dubois@primate.wisc.edu>
+
+RTF Tools has the notice:
+##
+Distribution Availability
+This software may be redistributed without restriction and used for any
+purpose whatsoever.
+##
+
+The supplied version understands RTF version 1.2, while the
+latest version of the specification is RTF 1.6 (May 1999).
+
+I have added some of the newer rtf codes that were preventing some files from
+being parsed properly, but they are not acted on. Most of them do not
+actually affect the content anyway. Depending on how the rtf was created,
+there might be tags that cause us to trip up and fail. If you have
+rtf files that Greenstone is not extracting text from, please send me
+some email (jrm21@cs.waikato.ac.nz).
+
+Note - we extract but do not convert Windows Metafile Format (.wmf) images.
+On unix systems, these can be converted with newer versions of ImageMagick
+(ie "convert" version 4.2.9 or newer).
+Also, libwmf (for unix and windows) is available for conversion at:
+http://wvware.sourceforge.net/projects/libwmf.html
+but we do not include it here due to its dependencies
+(eg libttf for truetype fonts).
+
+Embedded images such as jpegs and pngs are extracted fine, though.
+
+John McPherson (jrm21@cs.waikato.ac.nz)
+June 2001.
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.pm	(revision 21112)
@@ -0,0 +1,198 @@
+#
+# $Id$
+
+package MIME::Base64;
+
+=head1 NAME
+
+MIME::Base64 - Encoding and decoding of base64 strings
+
+=head1 SYNOPSIS
+
+ use MIME::Base64;
+
+ $encoded = encode_base64('Aladdin:open sesame');
+ $decoded = decode_base64($encoded);
+
+=head1 DESCRIPTION
+
+This module provides functions to encode and decode strings into the
+Base64 encoding specified in RFC 2045 - I<MIME (Multipurpose Internet
+Mail Extensions)>. The Base64 encoding is designed to represent
+arbitrary sequences of octets in a form that need not be humanly
+readable. A 65-character subset ([A-Za-z0-9+/=]) of US-ASCII is used,
+enabling 6 bits to be represented per printable character.
+
+The following functions are provided:
+
+=over 4
+
+=item encode_base64($str, [$eol])
+
+Encode data by calling the encode_base64() function.  The first
+argument is the string to encode.  The second argument is the line
+ending sequence to use (it is optional and defaults to C<"\n">).  The
+returned encoded string is broken into lines of no more than 76
+characters each and it will end with $eol unless it is empty.  Pass an
+empty string as second argument if you do not want the encoded string
+broken into lines.
+
+=item decode_base64($str)
+
+Decode a base64 string by calling the decode_base64() function.  This
+function takes a single argument which is the string to decode and
+returns the decoded data.
+
+Any character not part of the 65-character base64 subset set is
+silently ignored.  Characters occuring after a '=' padding character
+are never decoded.
+
+If the length of the string to decode (after ignoring
+non-base64 chars) is not a multiple of 4 or padding occurs too ealy,
+then a warning is generated if perl is running under C<-w>.
+
+=back
+
+If you prefer not to import these routines into your namespace you can
+call them as:
+
+    use MIME::Base64 ();
+    $encoded = MIME::Base64::encode($decoded);
+    $decoded = MIME::Base64::decode($encoded);
+
+=head1 DIAGNOSTICS
+
+The following warnings might be generated if perl is invoked with the
+C<-w> switch:
+
+=over 4
+
+=item Premature end of base64 data
+
+The number of characters to decode is not a multiple of 4.  Legal
+base64 data should be padded with one or two "=" characters to make
+its length a multiple of 4.  The decoded result will anyway be as if
+the padding was there.
+
+=item Premature padding of base64 data
+
+The '=' padding character occurs as the first or second character
+in a base64 quartet.
+
+=back
+
+=head1 EXAMPLES
+
+If you want to encode a large file, you should encode it in chunks
+that are a multiple of 57 bytes.  This ensures that the base64 lines
+line up and that you do not end up with padding in the middle. 57
+bytes of data fills one complete base64 line (76 == 57*4/3):
+
+   use MIME::Base64 qw(encode_base64);
+
+   open(FILE, "/var/log/wtmp") or die "$!";
+   while (read(FILE, $buf, 60*57)) {
+       print encode_base64($buf);
+   }
+
+or if you know you have enough memory
+
+   use MIME::Base64 qw(encode_base64);
+   local($/) = undef;  # slurp
+   print encode_base64(<STDIN>);
+
+=head1 COPYRIGHT
+
+Copyright 1995-1999 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+Distantly based on LWP::Base64 written by Martijn Koster
+<m.koster@nexor.co.uk> and Joerg Reichelt <j.reichelt@nexor.co.uk> and
+code posted to comp.lang.perl <3pd2lp$6gf@wsinti07.win.tue.nl> by Hans
+Mulder <hansm@wsinti07.win.tue.nl>
+
+The XS implementation use code from metamail.  Copyright 1991 Bell
+Communications Research, Inc. (Bellcore)
+
+=cut
+
+use strict;
+use vars qw(@ISA @EXPORT $VERSION $OLD_CODE);
+
+require Exporter;
+require DynaLoader;
+@ISA = qw(Exporter DynaLoader);
+@EXPORT = qw(encode_base64 decode_base64);
+
+$VERSION = '2.11';
+
+eval { bootstrap MIME::Base64 $VERSION; };
+if ($@) {
+    # can't bootstrap XS implementation, use perl implementation
+    *encode_base64 = \&old_encode_base64;
+    *decode_base64 = \&old_decode_base64;
+
+    $OLD_CODE = $@;
+    #warn $@ if $^W;
+}
+
+# Historically this module has been implemented as pure perl code.
+# The XS implementation runs about 20 times faster, but the Perl
+# code might be more portable, so it is still here.
+
+use integer;
+
+sub old_encode_base64 ($;$)
+{
+    my $res = "";
+    my $eol = $_[1];
+    $eol = "\n" unless defined $eol;
+    pos($_[0]) = 0;                          # ensure start at the beginning
+    while ($_[0] =~ /(.{1,45})/gs) {
+	$res .= substr(pack('u', $1), 1);
+	chop($res);
+    }
+    $res =~ tr|` -_|AA-Za-z0-9+/|;               # `# help emacs
+    # fix padding at the end
+    my $padding = (3 - length($_[0]) % 3) % 3;
+    $res =~ s/.{$padding}$/'=' x $padding/e if $padding;
+    # break encoded string into lines of no more than 76 characters each
+    if (length $eol) {
+	$res =~ s/(.{1,76})/$1$eol/g;
+    }
+    $res;
+}
+
+
+sub old_decode_base64 ($)
+{
+    local($^W) = 0; # unpack("u",...) gives bogus warning in 5.00[123]
+
+    my $str = shift;
+    my $res = "";
+
+    $str =~ tr|A-Za-z0-9+=/||cd;            # remove non-base64 chars
+    if (length($str) % 4) {
+	require Carp;
+	Carp::carp("Length of base64 data not a multiple of 4")
+    }
+    $str =~ s/=+$//;                        # remove padding
+    $str =~ tr|A-Za-z0-9+/| -_|;            # convert to uuencoded format
+    while ($str =~ /(.{1,60})/gs) {
+	my $len = chr(32 + length($1)*3/4); # compute length byte
+	$res .= unpack("u", $len . $1 );    # uudecode
+    }
+    $res;
+}
+
+# Set up aliases so that these functions also can be called as
+#
+#    MIME::Base64::encode();
+#    MIME::Base64::decode();
+
+*encode = \&encode_base64;
+*decode = \&decode_base64;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.xs
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.xs	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Base64.xs	(revision 21112)
@@ -0,0 +1,215 @@
+/* $Id$
+
+Copyright 1997-1998 Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+
+The tables and some of the code that used to be here was borrowed from
+metamail, which comes with this message:
+
+  Copyright (c) 1991 Bell Communications Research, Inc. (Bellcore)
+
+  Permission to use, copy, modify, and distribute this material 
+  for any purpose and without fee is hereby granted, provided 
+  that the above copyright notice and this permission notice 
+  appear in all copies, and that the name of Bellcore not be 
+  used in advertising or publicity pertaining to this 
+  material without the specific, prior written permission 
+  of an authorized representative of Bellcore.	BELLCORE 
+  MAKES NO REPRESENTATIONS ABOUT THE ACCURACY OR SUITABILITY 
+  OF THIS MATERIAL FOR ANY PURPOSE.  IT IS PROVIDED "AS IS", 
+  WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
+
+*/
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#ifdef __cplusplus
+}
+#endif
+
+#include "patchlevel.h"
+#if PATCHLEVEL <= 4 && !defined(PL_dowarn)
+   #define PL_dowarn dowarn
+#endif
+
+#define MAX_LINE  76 /* size of encoded lines */
+
+static char basis_64[] =
+   "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+#define XX      255	/* illegal base64 char */
+#define EQ      254	/* padding */
+#define INVALID XX
+
+static unsigned char index_64[256] = {
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,62, XX,XX,XX,63,
+    52,53,54,55, 56,57,58,59, 60,61,XX,XX, XX,EQ,XX,XX,
+    XX, 0, 1, 2,  3, 4, 5, 6,  7, 8, 9,10, 11,12,13,14,
+    15,16,17,18, 19,20,21,22, 23,24,25,XX, XX,XX,XX,XX,
+    XX,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
+    41,42,43,44, 45,46,47,48, 49,50,51,XX, XX,XX,XX,XX,
+
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+    XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
+};
+
+
+
+MODULE = MIME::Base64		PACKAGE = MIME::Base64
+
+SV*
+encode_base64(sv,...)
+	SV* sv
+	PROTOTYPE: $;$
+
+	PREINIT:
+	char *str;     /* string to encode */
+	SSize_t len;   /* length of the string */
+	char *eol;     /* the end-of-line sequence to use */
+	STRLEN eollen; /* length of the EOL sequence */
+	char *r;       /* result string */
+	STRLEN rlen;   /* length of result string */
+	unsigned char c1, c2, c3;
+	int chunk;
+
+	CODE:
+	str = SvPV(sv, rlen); /* SvPV(sv, len) gives warning for signed len */
+	len = (SSize_t)rlen;
+
+	/* set up EOL from the second argument if present, default to "\n" */
+	if (items > 1 && SvOK(ST(1))) {
+	    eol = SvPV(ST(1), eollen);
+	} else {
+	    eol = "\n";
+	    eollen = 1;
+	}
+
+	/* calculate the length of the result */
+	rlen = (len+2) / 3 * 4;	 /* encoded bytes */
+	if (rlen) {
+	    /* add space for EOL */
+	    rlen += ((rlen-1) / MAX_LINE + 1) * eollen;
+	}
+
+	/* allocate a result buffer */
+	RETVAL = newSV(rlen ? rlen : 1);
+	SvPOK_on(RETVAL);	
+	SvCUR_set(RETVAL, rlen);
+	r = SvPVX(RETVAL);
+
+	/* encode */
+	for (chunk=0; len > 0; len -= 3, chunk++) {
+	    if (chunk == (MAX_LINE/4)) {
+		char *c = eol;
+		char *e = eol + eollen;
+		while (c < e)
+		    *r++ = *c++;
+		chunk = 0;
+	    }
+	    c1 = *str++;
+	    c2 = *str++;
+	    *r++ = basis_64[c1>>2];
+	    *r++ = basis_64[((c1 & 0x3)<< 4) | ((c2 & 0xF0) >> 4)];
+	    if (len > 2) {
+		c3 = *str++;
+		*r++ = basis_64[((c2 & 0xF) << 2) | ((c3 & 0xC0) >>6)];
+		*r++ = basis_64[c3 & 0x3F];
+	    } else if (len == 2) {
+		*r++ = basis_64[(c2 & 0xF) << 2];
+		*r++ = '=';
+	    } else { /* len == 1 */
+		*r++ = '=';
+		*r++ = '=';
+	    }
+	}
+	if (rlen) {
+	    /* append eol to the result string */
+	    char *c = eol;
+	    char *e = eol + eollen;
+	    while (c < e)
+		*r++ = *c++;
+	}
+	*r = '\0';  /* every SV in perl should be NUL-terminated */
+
+	OUTPUT:
+	RETVAL
+
+SV*
+decode_base64(sv)
+	SV* sv
+	PROTOTYPE: $
+
+	PREINIT:
+	STRLEN len;
+	register unsigned char *str = (unsigned char*)SvPV(sv, len);
+	unsigned char const* end = str + len;
+	char *r;
+	unsigned char c[4];
+
+	CODE:
+	{
+	    /* always enough, but might be too much */
+	    STRLEN rlen = len * 3 / 4;
+	    RETVAL = newSV(rlen ? rlen : 1);
+	}
+        SvPOK_on(RETVAL);
+        r = SvPVX(RETVAL);
+
+	while (str < end) {
+	    int i = 0;
+            do {
+		unsigned char uc = index_64[*str++];
+		if (uc != INVALID)
+		    c[i++] = uc;
+
+		if (str == end) {
+		    if (i < 4) {
+			if (i && PL_dowarn)
+			    warn("Premature end of base64 data");
+			if (i < 2) goto thats_it;
+			if (i == 2) c[2] = EQ;
+			c[3] = EQ;
+		    }
+		    break;
+		}
+            } while (i < 4);
+	    
+	    if (c[0] == EQ || c[1] == EQ) {
+		if (PL_dowarn) warn("Premature padding of base64 data");
+		break;
+            }
+	    /* printf("c0=%d,c1=%d,c2=%d,c3=%d\n", c[0],c[1],c[2],c[3]);/**/
+
+	    *r++ = (c[0] << 2) | ((c[1] & 0x30) >> 4);
+
+	    if (c[2] == EQ)
+		break;
+	    *r++ = ((c[1] & 0x0F) << 4) | ((c[2] & 0x3C) >> 2);
+
+	    if (c[3] == EQ)
+		break;
+	    *r++ = ((c[2] & 0x03) << 6) | c[3];
+	}
+
+      thats_it:
+	SvCUR_set(RETVAL, r - SvPVX(RETVAL));
+	*r = '\0';
+
+	OUTPUT:
+	RETVAL
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Changes
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Changes	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Changes	(revision 21112)
@@ -0,0 +1,119 @@
+1998-02-27   Gisle Aas <gisle@aas.no>
+
+   Release 2.11
+
+   Fixed bogues "Premature end of base64 data" warning.  Bug spotted
+   by Dwayne Jacques Fontenot.
+
+   Workaround for Redhat shipping trial releases of perl.
+
+
+
+1998-12-18   Gisle Aas <aas@sn.no>
+
+   Release 2.10
+
+   A tweak that should make compilation with some old perl5.00[23]
+   perls better.
+
+   A cast that make some compilers more happy.
+
+
+
+1998-11-13   Gisle Aas <aas@sn.no>
+
+   Release 2.09
+
+   The 2.08 release did not compile with perl5.005_53, because
+   all simple globals now need to be prefixed with "PL_".
+
+
+
+1998-10-22   Gisle Aas <aas@sn.no>
+
+   Release 2.08
+
+   Found another tweak to speed up decode_base64() with another 3%.
+
+   Improved MIME::Base64 documentation a little.
+
+
+
+1998-10-21   Gisle Aas <aas@sn.no>
+
+   Release 2.07
+
+   Faster and smarter C implementation of the decode_base64()
+   function.  The new decode_base64() was 25% faster when tested
+   on Linux, i586, gcc -O2.
+
+
+
+1998-07-15   Gisle Aas <aas@sn.no>
+
+   Release 2.06
+
+   The decode_base64() implemented in pure perl will only carp
+   (not croak) if length of data to decode is not a multiple 4.  This
+   actually made 'make test' fail after 'rm Base64.xs'.
+
+
+
+1998-01-27   Gisle Aas <aas@sn.no>
+
+   Release 2.05
+
+   The decode_base64() would previously allocate a too short buffer for the
+   result string when the trailing "==" padding was missing in the string to
+   be decoded.
+
+   The encode_base64() now allocate one byte less space in the result
+   strings returned.
+
+
+
+1997-12-02   Gisle Aas <aas@sn.no>
+
+   Release 2.04
+
+   Documentation expanded a bit.
+
+
+
+1997-07-10   Gisle Aas <aas@sn.no>
+
+   Release 2.03
+
+   Decode_base64() doesn't croak on premature ended data any more.
+   A warning is generated instead if running under -w.
+   
+
+
+1997-06-27   Gisle Aas <aas@sn.no>
+
+   Release 2.02
+
+   QuotedPrint fix by Roderick Schertler <roderick@argon.org>:
+
+      - Long lines were not broken unless they're at the beginning
+        of the text
+
+      - Lines near but not over 76 chars were broken when they
+        shouldn't be
+
+
+
+1997-06-13   Gisle Aas <aas@sn.no>
+
+   Release 2.01
+
+   Base64.xs: Avoid type convertion warnings with some compilers
+
+   Minor documentation updates
+
+
+
+1997-04-24   Gisle Aas <aas@sn.no>
+
+   Release 2.00, based on libwww-perl-5.08.
+
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/INSTALL.w3mir
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/INSTALL.w3mir	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/INSTALL.w3mir	(revision 21112)
@@ -0,0 +1,27 @@
+- You also have to get the MIME-Base64 package
+  (http://www.perl.com/CPAN/modules/by-module/MIME/)
+
+- Unpack it:
+
+    gzip -dc MIME-Base64-*.tar.gz | tar xvBf -
+
+- Install it:
+
+    cd MIME-Base64-*
+
+  Then, as above, either
+
+    perl Makefile.PL
+
+  or
+
+    perl Makefile.PL PREFIX=/some/dir
+
+  Continue:
+
+    make
+    make test      (it should print 'All tests successful.')
+    make install
+
+  Now MIME-Base64 should be installed and working.
+
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/MANIFEST
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/MANIFEST	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/MANIFEST	(revision 21112)
@@ -0,0 +1,9 @@
+Base64.pm	Bootstrap and perl implementation of MIME::Base64
+Base64.xs	Encoder and decoder written in C
+Changes		What's new
+MANIFEST	This file
+Makefile.PL	Writes the Makefile
+QuotedPrint.pm  MIME::QuotedPrint module, encoder and decoder
+README		Please do!
+t/base64.t	Test that it works
+t/quoted-print.t
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Makefile.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Makefile.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/Makefile.PL	(revision 21112)
@@ -0,0 +1,8 @@
+require 5.002;
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+    NAME	 => 'MIME::Base64',
+    VERSION_FROM => 'Base64.pm',
+    dist         => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
+);
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/QuotedPrint.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/QuotedPrint.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/QuotedPrint.pm	(revision 21112)
@@ -0,0 +1,115 @@
+#
+# $Id$
+
+package MIME::QuotedPrint;
+
+=head1 NAME
+
+MIME::QuotedPrint - Encoding and decoding of quoted-printable strings
+
+=head1 SYNOPSIS
+
+ use MIME::QuotedPrint;
+
+ $encoded = encode_qp($decoded);
+ $decoded = decode_qp($encoded);
+
+=head1 DESCRIPTION
+
+This module provides functions to encode and decode strings into the
+Quoted-Printable encoding specified in RFC 2045 - I<MIME (Multipurpose
+Internet Mail Extensions)>.  The Quoted-Printable encoding is intended
+to represent data that largely consists of bytes that correspond to
+printable characters in the ASCII character set.  Non-printable
+characters (as defined by english americans) are represented by a
+triplet consisting of the character "=" followed by two hexadecimal
+digits.
+
+The following functions are provided:
+
+=over 4
+
+=item encode_qp($str)
+
+This function will return an encoded version of the string given as
+argument.
+
+Note that encode_qp() does not change newlines C<"\n"> to the CRLF
+sequence even though this might be considered the right thing to do
+(RFC 2045 (Q-P Rule #4)).
+
+=item decode_qp($str);
+
+This function will return the plain text version of the string given
+as argument.
+
+=back
+
+
+If you prefer not to import these routines into your namespace you can
+call them as:
+
+  use MIME::QuotedPrint ();
+  $encoded = MIME::QuotedPrint::encode($decoded);
+  $decoded = MIME::QuotedPrint::decode($encoded);
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
+use strict;
+use vars qw(@ISA @EXPORT $VERSION);
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(encode_qp decode_qp);
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+
+sub encode_qp ($)
+{
+    my $res = shift;
+    $res =~ s/([^ \t\n!-<>-~])/sprintf("=%02X", ord($1))/eg;  # rule #2,#3
+    $res =~ s/([ \t]+)$/
+      join('', map { sprintf("=%02X", ord($_)) }
+		   split('', $1)
+      )/egm;                        # rule #3 (encode whitespace at eol)
+
+    # rule #5 (lines must be shorter than 76 chars, but we are not allowed
+    # to break =XX escapes.  This makes things complicated :-( )
+    my $brokenlines = "";
+    $brokenlines .= "$1=\n"
+	while $res =~ s/(.*?^[^\n]{73} (?:
+		 [^=\n]{2} (?! [^=\n]{0,1} $) # 75 not followed by .?\n
+		|[^=\n]    (?! [^=\n]{0,2} $) # 74 not followed by .?.?\n
+		|          (?! [^=\n]{0,3} $) # 73 not followed by .?.?.?\n
+	    ))//xsm;
+
+    "$brokenlines$res";
+}
+
+
+sub decode_qp ($)
+{
+    my $res = shift;
+    $res =~ s/[ \t]+?(\r?\n)/$1/g;  # rule #3 (trailing space must be deleted)
+    $res =~ s/=\r?\n//g;            # rule #5 (soft line breaks)
+    $res =~ s/=([\da-fA-F]{2})/pack("C", hex($1))/ge;
+    $res;
+}
+
+# Set up aliases so that these functions also can be called as
+#
+# MIME::QuotedPrint::encode();
+# MIME::QuotedPrint::decode();
+
+*encode = \&encode_qp;
+*decode = \&decode_qp;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/README
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/README	(revision 21112)
@@ -0,0 +1,38 @@
+This package contains a base64 encoder/decoder and a quoted-printable
+encoder/decoder.  These encoding methods are specified in RFC 2045 -
+MIME (Multipurpose Internet Mail Extensions).
+
+The Base64 encoding is designed to represent arbitrary sequences of
+octets in a form that need not be humanly readable. A 65-character
+subset ([A-Za-z0-9+/=]) of US-ASCII is used, enabling 6 bits to be
+represented per printable character.
+
+The quoted-printable encoding is intended to represent data that
+largely consists of bytes that correspond to printable characters in
+the ASCII character set.  Non-printable characters are represented by
+a triplet consisting of the character "=" followed by two hexadecimal
+digits.
+
+The MIME::Base64 and MIME::QuotedPrint modules used to be part of
+libwww-perl package.  They are now distributed separately (as this
+package).  The main improvement is that the base64 encoder/decoder is
+implemented by XS functions.  This makes it about 20 times faster than
+the old implementation as pure Perl.
+
+In order to install and use this package you will need Perl version
+5.002 or better.  Installation as usual:
+
+   perl Makefile.PL
+   make
+   make test
+   make install
+
+The library can also be installed without XS support (for instance because
+you don't have a working C compiler installed).  Simply remove the Base64.xs
+file before running 'perl Makefile.PL' and everything should work the same,
+although encoding and decoding will be much slower.
+
+Copyright 1995-1999 Gisle Aas <gisle@aas.no>
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/install.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/install.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/install.sh	(revision 21112)
@@ -0,0 +1,10 @@
+#!/bin/sh -v
+
+export PERL5LIB=$GSDLHOME/perllib/cpan/lib:$PERL5LIB
+export PATH=$GSDLHOME/perllib/cpan/bin:$PATH
+
+perl Makefile.PL PREFIX=$GSDLHOME/perllib/cpan
+
+make
+make test
+make install
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/base64.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/base64.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/base64.t	(revision 21112)
@@ -0,0 +1,362 @@
+use MIME::Base64;
+
+print "1..282\n";
+
+print "Testing MIME::Base64-", $MIME::Base64::VERSION, "\n";
+
+$testno = 1;
+
+encodeTest();
+decodeTest();
+
+# This used to generate a warning
+print "not " unless decode_base64(encode_base64("foo")) eq "foo";
+print "ok ", $testno++, "\n";
+
+sub encodeTest
+{
+    print "encode test\n";
+
+    my @encode_tests = (
+        [''    => ''],
+	['a'   => 'YQ=='],
+	['aa'  => 'YWE='],
+	['aaa' => 'YWFh'],
+
+	['aaa' => 'YWFh'],
+	['aaa' => 'YWFh'],
+	['aaa' => 'YWFh'],
+
+	["\000\377" => "AP8="],
+	["\377\000" => "/wA="],
+	["\000\000\000" => "AAAA"],
+
+	# from HTTP spec
+	['Aladdin:open sesame' => 'QWxhZGRpbjpvcGVuIHNlc2FtZQ=='],
+
+	['a' x 100 => 'YWFh' x 33 . 'YQ=='],
+
+	['Multipurpose Internet Mail Extensions: The Base64 Content-Transfer-Encoding is designed to represent sequences of octets in a form that is not humanly readable. '
+	=> "TXVsdGlwdXJwb3NlIEludGVybmV0IE1haWwgRXh0ZW5zaW9uczogVGhlIEJhc2U2NCBDb250ZW50LVRyYW5zZmVyLUVuY29kaW5nIGlzIGRlc2lnbmVkIHRvIHJlcHJlc2VudCBzZXF1ZW5jZXMgb2Ygb2N0ZXRzIGluIGEgZm9ybSB0aGF0IGlzIG5vdCBodW1hbmx5IHJlYWRhYmxlLiA="],
+
+	# All values
+	["\000" => "AA=="],
+	["\001" => "AQ=="],
+	["\002" => "Ag=="],
+	["\003" => "Aw=="],
+	["\004" => "BA=="],
+	["\005" => "BQ=="],
+	["\006" => "Bg=="],
+	["\007" => "Bw=="],
+	["\010" => "CA=="],
+	["\011" => "CQ=="],
+	["\012" => "Cg=="],
+	["\013" => "Cw=="],
+	["\014" => "DA=="],
+	["\015" => "DQ=="],
+	["\016" => "Dg=="],
+	["\017" => "Dw=="],
+	["\020" => "EA=="],
+	["\021" => "EQ=="],
+	["\022" => "Eg=="],
+	["\023" => "Ew=="],
+	["\024" => "FA=="],
+	["\025" => "FQ=="],
+	["\026" => "Fg=="],
+	["\027" => "Fw=="],
+	["\030" => "GA=="],
+	["\031" => "GQ=="],
+	["\032" => "Gg=="],
+	["\033" => "Gw=="],
+	["\034" => "HA=="],
+	["\035" => "HQ=="],
+	["\036" => "Hg=="],
+	["\037" => "Hw=="],
+	["\040" => "IA=="],
+	["\041" => "IQ=="],
+	["\042" => "Ig=="],
+	["\043" => "Iw=="],
+	["\044" => "JA=="],
+	["\045" => "JQ=="],
+	["\046" => "Jg=="],
+	["\047" => "Jw=="],
+	["\050" => "KA=="],
+	["\051" => "KQ=="],
+	["\052" => "Kg=="],
+	["\053" => "Kw=="],
+	["\054" => "LA=="],
+	["\055" => "LQ=="],
+	["\056" => "Lg=="],
+	["\057" => "Lw=="],
+	["\060" => "MA=="],
+	["\061" => "MQ=="],
+	["\062" => "Mg=="],
+	["\063" => "Mw=="],
+	["\064" => "NA=="],
+	["\065" => "NQ=="],
+	["\066" => "Ng=="],
+	["\067" => "Nw=="],
+	["\070" => "OA=="],
+	["\071" => "OQ=="],
+	["\072" => "Og=="],
+	["\073" => "Ow=="],
+	["\074" => "PA=="],
+	["\075" => "PQ=="],
+	["\076" => "Pg=="],
+	["\077" => "Pw=="],
+	["\100" => "QA=="],
+	["\101" => "QQ=="],
+	["\102" => "Qg=="],
+	["\103" => "Qw=="],
+	["\104" => "RA=="],
+	["\105" => "RQ=="],
+	["\106" => "Rg=="],
+	["\107" => "Rw=="],
+	["\110" => "SA=="],
+	["\111" => "SQ=="],
+	["\112" => "Sg=="],
+	["\113" => "Sw=="],
+	["\114" => "TA=="],
+	["\115" => "TQ=="],
+	["\116" => "Tg=="],
+	["\117" => "Tw=="],
+	["\120" => "UA=="],
+	["\121" => "UQ=="],
+	["\122" => "Ug=="],
+	["\123" => "Uw=="],
+	["\124" => "VA=="],
+	["\125" => "VQ=="],
+	["\126" => "Vg=="],
+	["\127" => "Vw=="],
+	["\130" => "WA=="],
+	["\131" => "WQ=="],
+	["\132" => "Wg=="],
+	["\133" => "Ww=="],
+	["\134" => "XA=="],
+	["\135" => "XQ=="],
+	["\136" => "Xg=="],
+	["\137" => "Xw=="],
+	["\140" => "YA=="],
+	["\141" => "YQ=="],
+	["\142" => "Yg=="],
+	["\143" => "Yw=="],
+	["\144" => "ZA=="],
+	["\145" => "ZQ=="],
+	["\146" => "Zg=="],
+	["\147" => "Zw=="],
+	["\150" => "aA=="],
+	["\151" => "aQ=="],
+	["\152" => "ag=="],
+	["\153" => "aw=="],
+	["\154" => "bA=="],
+	["\155" => "bQ=="],
+	["\156" => "bg=="],
+	["\157" => "bw=="],
+	["\160" => "cA=="],
+	["\161" => "cQ=="],
+	["\162" => "cg=="],
+	["\163" => "cw=="],
+	["\164" => "dA=="],
+	["\165" => "dQ=="],
+	["\166" => "dg=="],
+	["\167" => "dw=="],
+	["\170" => "eA=="],
+	["\171" => "eQ=="],
+	["\172" => "eg=="],
+	["\173" => "ew=="],
+	["\174" => "fA=="],
+	["\175" => "fQ=="],
+	["\176" => "fg=="],
+	["\177" => "fw=="],
+	["\200" => "gA=="],
+	["\201" => "gQ=="],
+	["\202" => "gg=="],
+	["\203" => "gw=="],
+	["\204" => "hA=="],
+	["\205" => "hQ=="],
+	["\206" => "hg=="],
+	["\207" => "hw=="],
+	["\210" => "iA=="],
+	["\211" => "iQ=="],
+	["\212" => "ig=="],
+	["\213" => "iw=="],
+	["\214" => "jA=="],
+	["\215" => "jQ=="],
+	["\216" => "jg=="],
+	["\217" => "jw=="],
+	["\220" => "kA=="],
+	["\221" => "kQ=="],
+	["\222" => "kg=="],
+	["\223" => "kw=="],
+	["\224" => "lA=="],
+	["\225" => "lQ=="],
+	["\226" => "lg=="],
+	["\227" => "lw=="],
+	["\230" => "mA=="],
+	["\231" => "mQ=="],
+	["\232" => "mg=="],
+	["\233" => "mw=="],
+	["\234" => "nA=="],
+	["\235" => "nQ=="],
+	["\236" => "ng=="],
+	["\237" => "nw=="],
+	["\240" => "oA=="],
+	["\241" => "oQ=="],
+	["\242" => "og=="],
+	["\243" => "ow=="],
+	["\244" => "pA=="],
+	["\245" => "pQ=="],
+	["\246" => "pg=="],
+	["\247" => "pw=="],
+	["\250" => "qA=="],
+	["\251" => "qQ=="],
+	["\252" => "qg=="],
+	["\253" => "qw=="],
+	["\254" => "rA=="],
+	["\255" => "rQ=="],
+	["\256" => "rg=="],
+	["\257" => "rw=="],
+	["\260" => "sA=="],
+	["\261" => "sQ=="],
+	["\262" => "sg=="],
+	["\263" => "sw=="],
+	["\264" => "tA=="],
+	["\265" => "tQ=="],
+	["\266" => "tg=="],
+	["\267" => "tw=="],
+	["\270" => "uA=="],
+	["\271" => "uQ=="],
+	["\272" => "ug=="],
+	["\273" => "uw=="],
+	["\274" => "vA=="],
+	["\275" => "vQ=="],
+	["\276" => "vg=="],
+	["\277" => "vw=="],
+	["\300" => "wA=="],
+	["\301" => "wQ=="],
+	["\302" => "wg=="],
+	["\303" => "ww=="],
+	["\304" => "xA=="],
+	["\305" => "xQ=="],
+	["\306" => "xg=="],
+	["\307" => "xw=="],
+	["\310" => "yA=="],
+	["\311" => "yQ=="],
+	["\312" => "yg=="],
+	["\313" => "yw=="],
+	["\314" => "zA=="],
+	["\315" => "zQ=="],
+	["\316" => "zg=="],
+	["\317" => "zw=="],
+	["\320" => "0A=="],
+	["\321" => "0Q=="],
+	["\322" => "0g=="],
+	["\323" => "0w=="],
+	["\324" => "1A=="],
+	["\325" => "1Q=="],
+	["\326" => "1g=="],
+	["\327" => "1w=="],
+	["\330" => "2A=="],
+	["\331" => "2Q=="],
+	["\332" => "2g=="],
+	["\333" => "2w=="],
+	["\334" => "3A=="],
+	["\335" => "3Q=="],
+	["\336" => "3g=="],
+	["\337" => "3w=="],
+	["\340" => "4A=="],
+	["\341" => "4Q=="],
+	["\342" => "4g=="],
+	["\343" => "4w=="],
+	["\344" => "5A=="],
+	["\345" => "5Q=="],
+	["\346" => "5g=="],
+	["\347" => "5w=="],
+	["\350" => "6A=="],
+	["\351" => "6Q=="],
+	["\352" => "6g=="],
+	["\353" => "6w=="],
+	["\354" => "7A=="],
+	["\355" => "7Q=="],
+	["\356" => "7g=="],
+	["\357" => "7w=="],
+	["\360" => "8A=="],
+	["\361" => "8Q=="],
+	["\362" => "8g=="],
+	["\363" => "8w=="],
+	["\364" => "9A=="],
+	["\365" => "9Q=="],
+	["\366" => "9g=="],
+	["\367" => "9w=="],
+	["\370" => "+A=="],
+	["\371" => "+Q=="],
+	["\372" => "+g=="],
+	["\373" => "+w=="],
+	["\374" => "/A=="],
+	["\375" => "/Q=="],
+	["\376" => "/g=="],
+	["\377" => "/w=="],
+    );
+
+    for $test (@encode_tests) {
+	my($plain, $expected) = ($$test[0], $$test[1]);
+
+	my $encoded = encode_base64($plain, '');
+	if ($encoded ne $expected) {
+	    print "test $testno ($plain): expected $expected, got $encoded\n";
+            print "not ";
+	}
+	my $decoded = decode_base64($encoded);
+	if ($decoded ne $plain) {
+	    print "test $testno ($plain): expected $expected, got $encoded\n";
+            print "not ";
+	}
+
+	# Try the old C versions too
+	if ($encoded ne MIME::Base64::old_encode_base64($plain, '')) {
+	    print "old_encode_base64 give different result.\n";
+	    print "not ";
+        }
+	if ($plain ne MIME::Base64::old_decode_base64($encoded)) {
+	    print "old_decode_base64 give different result.\n";
+	    print "not ";
+        }
+		
+	print "ok $testno\n";
+	$testno++;
+    }
+}
+
+sub decodeTest
+{
+    print "decode test:\n";
+    local $SIG{__WARN__} = sub { print $_[0] };  # avoid warnings on stderr
+
+    my @decode_tests = (
+	['YWE='   => 'aa'],
+	[' YWE='  => 'aa'],
+	['Y WE='  => 'aa'],
+	['YWE= '  => 'aa'],
+	["Y\nW\r\nE=" => 'aa'],
+
+	# These will generate some warnings
+        ['YWE=====' => 'aa'],    # extra padding
+	['YWE'      => 'aa'],    # missing padding
+        ['YWFh====' => 'aaa'],
+        ['YQ'       => 'a'],
+        ['Y'        => ''],
+        [''         => ''],
+        [undef()    => ''],
+    );
+
+    for $test (@decode_tests) {
+	my($encoded, $expected) = ($$test[0], $$test[1]);
+
+	my $decoded = decode_base64($encoded);
+	if ($decoded ne $expected) {
+	    die "test $testno ($encoded): expected $expected, got $decoded\n";
+	}
+	print "ok $testno\n";
+	$testno++;
+    }
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/quoted-print.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/quoted-print.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/MIME-Base64-2.11/t/quoted-print.t	(revision 21112)
@@ -0,0 +1,106 @@
+#!/local/bin/perl -w
+
+use MIME::QuotedPrint;
+
+$x70 = "x" x 70;
+
+@tests =
+  (
+   # plain ascii should not be encoded
+   ["quoted printable"  =>
+    "quoted printable"],
+
+   # 8-bit chars should be encoded
+   ["våre kjære norske tegn bør æres" =>
+    "v=E5re kj=E6re norske tegn b=F8r =E6res"],
+
+   # trailing space should be encoded
+   ["  " => "=20=20"],
+   ["\tt\t" => "\tt=09"],
+   ["test  \ntest\n\t \t \n" => "test=20=20\ntest\n=09=20=09=20\n"],
+
+   # "=" is special an should be decoded
+   ["=\n" => "=3D\n"],
+   ["\0\xff" => "=00=FF"],
+
+   # Very long lines should be broken (not more than 76 chars
+   ["The Quoted-Printable encoding is intended to represent data that largly consists of octets that correspond to printable characters in the ASCII character set." =>
+    "The Quoted-Printable encoding is intended to represent data that largly con=
+sists of octets that correspond to printable characters in the ASCII charac=
+ter set."
+    ],
+
+   # Long lines after short lines were broken through 2.01.
+   ["short line
+In America, any boy may become president and I suppose that's just one of the risks he takes. -- Adlai Stevenson" =>
+    "short line
+In America, any boy may become president and I suppose that's just one of t=
+he risks he takes. -- Adlai Stevenson"],
+
+   # My (roderick@argon.org) first crack at fixing that bug failed for
+   # multiple long lines.
+   ["College football is a game which would be much more interesting if the faculty played instead of the students, and even more interesting if the
+trustees played.  There would be a great increase in broken arms, legs, and necks, and simultaneously an appreciable diminution in the loss to humanity. -- H. L. Mencken" =>
+    "College football is a game which would be much more interesting if the facu=
+lty played instead of the students, and even more interesting if the
+trustees played.  There would be a great increase in broken arms, legs, and=
+ necks, and simultaneously an appreciable diminution in the loss to humanit=
+y. -- H. L. Mencken"],
+
+   # Don't break a line that's near but not over 76 chars.
+   ["$x70!23"		=> "$x70!23"],
+   ["$x70!234"		=> "$x70!234"],
+   ["$x70!2345"		=> "$x70!2345"],
+   ["$x70!23456"	=> "$x70!23456"],
+   ["$x70!23\n"		=> "$x70!23\n"],
+   ["$x70!234\n"	=> "$x70!234\n"],
+   ["$x70!2345\n"	=> "$x70!2345\n"],
+   ["$x70!23456\n"	=> "$x70!23456\n"],
+
+   # Not allowed to break =XX escapes using soft line break
+   ["$x70===xxxx" => "$x70=3D=\n=3D=3Dxxxx"],
+   ["$x70!===xxx" => "$x70!=3D=\n=3D=3Dxxx"],
+   ["$x70!!===xx" => "$x70!!=3D=\n=3D=3Dxx"],
+   ["$x70!!!===x" => "$x70!!!=\n=3D=3D=3Dx"],
+   #                            ^
+   #                    70123456|
+   #                           max
+   #                        line width
+);
+
+$notests = @tests + 2;
+print "1..$notests\n";
+
+$testno = 0;
+for (@tests) {
+    $testno++;
+    ($plain, $encoded) = @$_;
+    $x = encode_qp($plain);
+    if ($x ne $encoded) {
+	print "Encode test failed\n";
+	print "Got:      '$x'\n";
+	print "Expected: '$encoded'\n";
+	print "not ok $testno\n";
+	next;
+    }
+    $x = decode_qp($encoded);
+    if ($x ne $plain) {
+	print "Decode test failed\n";
+	print "Got:      '$x'\n";
+	print "Expected: '$plain'\n";
+	print "not ok $testno\n";
+	next;
+    }
+    print "ok $testno\n";
+}
+
+# Some extra testing for a case that was wrong until libwww-perl-5.09
+print "not " unless decode_qp("foo  \n\nfoo =\n\nfoo=20\n\n") eq
+                                "foo\n\nfoo \nfoo \n\n";
+$testno++; print "ok $testno\n";
+
+# Same test but with "\r\n" terminated lines
+print "not " unless decode_qp("foo  \r\n\r\nfoo =\r\n\r\nfoo=20\r\n\r\n") eq
+                                "foo\r\n\r\nfoo \r\nfoo \r\n\r\n";
+$testno++; print "ok $testno\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/ChangeLog
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/ChangeLog	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/ChangeLog	(revision 21112)
@@ -0,0 +1,1535 @@
+Tue Aug  4 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.36
+
+ o  The lwp-download script will call $ua->env_proxy now.
+
+ o  The lwp-request script allows content types (specified with the -c
+    option) with optional parameters like: multipart/mixed; boundary="--".
+
+ o  LWP::UserAgent will lowercase all authentication parameter names
+    before passing it to the authentication module.  Previous releases
+    ignored parameters like; Realm="Foo" (bacause Realm contained
+    upper case letters).
+
+ o  LWP::Protocol::ftp test for If-Modified-Since was wrong.
+
+ o  How the $url->abs method works can now be configured with the global
+    variables $URI::URL::ABS_ALLOW_RELATIVE_SCHEME and
+    $URI::URL::ABS_REMOTE_LEADING_DOTS.
+
+ o  The anonymous password guesser for ftp URLs will now call the external
+    `whoami` program any more.  Patch by Charles C. Fu <ccwf@bacchus.com>.
+
+ o  LWP::Protocol::http now allow dynamic requests without any
+    Content-Length specified when Content-Type is multipart/*
+
+ o  HTTP::Request::Common can now upload infinite files.
+    (Controlled by the $DYNAMIC_FILE_UPLOAD configuration variable.)
+
+
+
+Fri Jul 10 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.35
+
+ o  More lwp-rget patches from David D. Kilzer <ddkilzer@madison.dseg.ti.com>.
+    Adds the following new options: --iis, --keepext, --tolower
+
+ o  LWP::MediaTypes patches from MacEachern <dougm@pobox.com>.  Adds new
+    functions: add_type(), add_encoding(), read_media_types()
+
+
+
+Tue Jul  7 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.34
+
+ o  LWP::Protocol::ftp now try to use the MDTM command to support
+    the Last-Modified response header as well as
+    If-Modified-Since in requests.  Original and final patch by
+    Charles C. Fu <ccwf@bacchus.com>
+
+ o  $url->path_components will not escape "." any more.
+
+ o  WWW::RobotRules will now work for Mac text files too (lines delimited
+    by CR only).  Patch by Olly Betts <olly@muscat.co.uk>
+
+ o  lwp-rget support <area ..> links too.
+
+
+
+Thu May  7 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.33
+
+ o  LWP::Simple::get() did try to handle too many of the 3xx
+    codes as redirect when it bypasses full LWP.
+
+ o  LWP::UserAgent->mirror will now use utime(2) to set the
+    file modification time corresponding to the Last-Modified
+    header.
+
+ o  LWP::Protocol::http will not modify the HTTP::Request that
+    it is processing.  This avoids sticky Host header for
+    redirects.
+
+ o  URI::Heuristic and lwp-download documentation update.
+
+
+
+Wed Apr 15 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.32
+
+ o  Much improved HTTP::Daemon class. We now support persistent
+    connections.  Changes include:
+       - $c->get_request can be told to return after reading and
+         parsing headers only.
+       - $c->reason (new method)
+       - $c->read_buffer (new method)
+       - $c->proto_ge (new method)
+       - $c->force_last_request (new method)
+       - $c->send_response now support CODE reference content
+         and will use chunked transfer encoding for HTTP/1.1 clients.
+       - expanded the documentation.
+
+
+
+Fri Apr 10 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.31
+
+ o  Makefile.PL now checks that HTML::HeadParser is present.
+
+ o  Updated HTTP::Cookies according to draft-ietf-http-state-man-mec-08.txt
+    It now supports the .local domain and value less 'port' attribute in
+    the Set-Cookie2 header.
+
+ o  HTTP::Headers update:
+       - $h->content_type now always return a defined value
+       - $h->header($field) will now concatenate multi-valued header
+         fields with "," as separator in scalar context.
+
+ o  HTTP::Request::Common update:
+       - used to destroy the content of the  hash/array arguments
+         passed to its constructor functions.
+       - allow a hash reference to specify form-data content.
+       - you can override Content-Disposition for form-data now.
+       - set content-encoding for files if applicable
+       - default boundary string is now always "--000"
+
+ o  LWP::UserAgent will not follow more than 13 redirects
+    automatically.
+
+
+
+Wed Apr  1 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.30
+
+ o  Unbundled the following modules:
+
+       * HTML-Parser  (HTML::Parser, HTML::Entites, HTML::LinkExtor,...)
+       * HTML-Tree    (HTML::Element, HTML::TreeBuilder,...)
+       * Font-AFM     (Font::AFM, Font::Metrics::*)
+       * File-CounterFile
+
+ o  Simplified internal structure of HTML::Headers.  Hopefully,
+    nobody will notice.
+
+ o  New modules HTTP::Headers::Auth,  HTTP::Headers::ETag that adds
+    additional convenience methods to the HTTP::Headers class.
+
+ o  Removed split_etag_list() from HTTP::Headers::Util, in the hope
+    that nobody had starting using it.
+
+
+
+Tue Mar 24 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.22
+
+ o  HTTP::Cookies made more compatible with Netscape cookies.  Allow
+    the domain to match host, allow dots in the part of the hostname
+    not covered by domain.  Don't quote the cookie value even when it
+    contains non-token chars.  Based on patch from Kartik Subbarao
+    <subbarao@computer.org>.
+
+ o  Updated HTTP::Status to reflect <draft-ietf-http-v11-spec-rev-03>.
+    RC_MOVED_TEMPORARILY renamed to RC_FOUND.  Added codes
+    RC_TEMPORARY_REDIRECT (307) and RC_EXPECTATION_FAILED (417).
+    Slightly more documentation too.
+
+ o  The split_header_words() function HTTP::Headers::Util could go
+    into infinite loop on some header values.  Implemented split_etag_list()
+    too.  Added more documentation and test script for this module.
+
+ o  LWP::Simple now switch to full LWP implementation even for systems
+    that force all environment keys to be upper case.  Modification
+    suggested by Dale Couch <dcouch@training.orl.lmco.com>.
+
+ o  LWP::UserAgent allows redirects to a relative URL with scheme to be
+    made.  Suggested by Chris W. Unger <cunger@cas.org>.
+
+ o  Applied dtd2pm.pl patches from <peterm@zeta.org.au>.  It can now
+    extract information from the HTML40.dtd
+
+
+
+Thu Mar 12 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.21
+
+ o  lwp-rget patches from David D. Kilzer <ddkilzer@madison.dseg.ti.com>
+    (modified by Gisle).  Support the --hier  and the --auth options
+    and <frame>s.
+
+ o  File::CounterFile protect against bad $/ and $\ as suggested
+    by Frank Hoehne.
+ 
+ o  File::Listing used "next" when return was more appropriate.
+    Patch by erik@mediator.uni-c.dk.
+
+ o  HTML::Element support for multiple boolean attributes for a single
+    element.  Patch by Philip Guenther.
+
+ o  Can set $HTTP::Headers::TRANSLATE_UNDERSCORE to FALSE value to
+    suppress tr/_/-/ of header keys.
+
+ o  LWP::Protocol::http will not initialize the Host header if it is
+    already set. Patch by Ed Jordan.
+
+ o  LWP::Protocol::http did not handle responses with no header lines
+    correctly.  Patch by Daniel Buenzli <buenzli@rzu.unizh.ch>
+
+ o  $url->rel() handles path segments without leading "/" better.
+
+
+
+Fri Feb 13 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.20
+
+ o  Fixed the "500 Offset outside string" bug that affected perl
+    5.004_03 and older version of Perl.
+
+ o  Fixed a documentation typo spotted by Michael Quaranta
+    <quaranta@vnet.IBM.COM>
+
+ o  HTTP::Date: Protect against croaking from timegm/timelocal.
+
+
+
+Mon Jan 26 1998   Gisle Aas <aas@sn.no>
+
+ o  Release 5.19
+
+ o  HTML::Parser does not call $self->text() callback for empty text
+    any more.
+
+ o  LWP::Protocol::https was noisy when connections failed and the
+    script was running with '-w' (noise inherited from IO::Socket::INET)
+
+ o  $ua->use_alarm(BOOL) now gives a warning if running with -w
+
+
+
+Tue Jan 20 1998   Gisle Aas <aas@sn.no>
+
+ o  Developer release 5.18_05
+
+ o  HTTPS support based on my Crypt-SSLeay module.  The Net-SSLeay module
+    is not supported any more.
+
+ o  lwp-request documentation typo spotted Martijn Koster.
+
+ o  Removed superfluous \\ in the URI::Escape regex. This was also
+    spotted by Martijn.
+
+ o  File::Listing now handles timezones correctly.
+
+ o  Added $VERSION to modules that was missing it.
+
+ o  Added 'use strict' to several modules that was missing it.
+
+ o  LWP::Protocol::http now adds the Client-Peer header to responses and
+    has hooks for more callbacks.
+
+ o  LWP::Protocol::https adds Client-SSL-Cipher, Client-SSL-Cert-Subject
+    and Client-SSL-Cert-Issuer headers to the response.  The requests can
+    also be made conditional based on the peer certificate using the
+    If-SSL-Cert-Subject header in requests.
+
+ o  HTML::Parse is back.  (It was even used in lwpcook.pod)
+
+
+
+Wed Dec 17 1997   Gisle Aas <aas@sn.no>
+
+ o  Developer release 5.18_04
+
+ o  Makefile.PL fix based on report from Pon Hwa Lin <koala@fragment.com>
+
+ o  lwp-request will now print the response code message with -s and -S
+    options.
+
+ o  Hide IO::Socket::INET noise when running under -w
+
+ o  Don't set 'Content-Length: 0' in HTTP requests.
+
+ o  LWP::Protocol::http now calls LWP::Debug::conns again
+
+
+
+Tue Dec 16 1997   Gisle Aas <aas@sn.no>
+
+ o  Developer release 5.18_03
+
+ o  Got rid of alarms() and replaced LWP::Socket with IO::Socket::INET.
+    New protocol implementations for http, https, gopher, nntp.
+    $ua->use_alarm() is now a noop.
+
+ o  LWP::Protocol::ftp patch from  Tony Finch <fanf@demon.net>.
+
+ o  Removed depreciated modules from the distibution; HTML::Parse,
+    LWP::Socket, LWP::SecureSocket, LWP::IO, LWP::TkIO.
+
+
+
+Fri Dec 12 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.18
+
+ o  HTTP authorization patches from Tony Finch <fanf@demon.net>.
+    Allows "user:pass@" in HTTP URLs.
+
+ o  HTML::Parser patch by Brian McCauley <B.A.McCauley@bham.ac.uk>.
+    Pass original text to end() method.
+
+ o  The HTML::Parser->netscape_buggy_comment method is depreciated.
+    Use HTML::Parser->strict_comment instead.  The default value
+    has changed with the name.
+
+ o  Some HTML::Parser optimization tweaks.
+
+ o  New module named HTML::Filter
+
+ o  Updated HTTP::Headers to the latest HTTP spec.  Added knowledge
+    about the "Trailer", "Expect", "TE", "Accept-Range" headers.
+    "Public" header is gone.
+
+ o  Added some more header convenience methods: if_unmodified_since,
+    content_language, and proxy_authorization methods.
+
+ o  HTTP::{Request,Response}->clone can handle subclasses now.
+
+ o  HTTP::Request->url() can now undefine the URL.
+
+ o  HTTP::{Request,Response}->as_string format looks more like
+    the HTTP protocol formatting now.  Dashed lines above and
+    below is gone.
+
+ o  Documented HTTP::Response->status_line method
+
+ o  Compressed HTML::Response->error_as_HTML output
+
+ o  HTTP::Status updated to latest HTTP spec.  Added 
+    RC_REQUEST_RANGE_NOT_SATISFIABLE (416)
+
+
+
+Tue Dec  2 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.17
+
+ o  All authentication handling moved out of LWP::UserAgent and into
+    LWP::Authen::Basic and LWP::Authen::Digest.  We now also support
+    PROXY_AUTHENTICATION_REQUIRED responses.
+
+ o  HTML::Formatter will always add a blank line for <br>.
+
+ o  Avoid use of uninitialized value in HTTP::Daemon.
+
+ o  HTTP::Date allows seconds when recognizing 'ls -l' dates.  This
+    allows us to parse syslog time stamps.
+
+ o  HTTP::Request::Common::POST allows a hash reference as second
+    argument (in addition to an array reference).
+
+ o  LWP::Simple will initialize the $ua if it is exported.
+
+ o  Various documentation updates.
+
+
+
+Fri Nov 21 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.16
+
+ o  LWP::Simple::head() would die in array context because str2time
+    was not imported any more.
+
+ o  HTTP::Daemon->accept now takes an optional package argument like
+    IO::Socket->accept does.
+
+ o  Made HTTP::Request and HTTP::Response subclassable.
+
+ o  Added Proxy-Authorization example to lwpcook.
+
+
+
+Thu Nov  6 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.15
+
+ o  New module URI::Heuristic
+
+ o  The lwp-request script now use URI::Heuristic for it's URL arguments.
+    It means that 'lwp-request perl' will not get a file called "./perl"
+    but will fetch the page "http://www.perl.com" or something similar.
+    If you want to get the file you have to prefix it with "./".  Full
+    URLs are never affected by this.
+
+ o  LWP::Simple::get() will bypass LWP for simple HTTP requests.  This
+    should make it somewhat faster.
+
+ o  LWP::RobotUA has a new method called $ua->use_sleep() that
+    controls how niceness towards the servers are enforced.
+    Previously $ua->use_alarm() used to control this, but this did
+    not work well on Win32 systems.
+
+ o  URI::URL::rel() will handle URLs to a fragment within the same
+    document better. Initial patch from Nicolai Langfeldt
+    <janl@math.uio.no>.
+
+ o  HTML::Element don't consider </th>, </tr> and </td> optional any
+    more.  I wonder how Netscape managed to not implement this
+    correctly all this time.
+
+ o  Added lots of modern tags to HTML::AsSubs.
+
+ o  HTTP::Request::Common will read uploaded files in binmode().
+    This should be better for Win32 systems.  Contributed by
+    <Steve_Kilbane@cegelecproj.co.uk>.
+
+
+
+Sun Oct 12 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.14
+
+ o  HTML::Formatter patches from Andreas Gustafsson <gson@araneus.fi>.
+    The formatters handling of whitespace is much better now.  Thanks!
+
+ o  HTML::FormatText: can specify margins in the constructor.
+
+ o  URI::URL: the base will be absolutized internally now.
+
+ o  URI::URL will take advantage of void context provided by perl5.004.
+    This means that using $u->path and $u->query should be safer now.
+
+ o  URI::URL->print_on defaults to STDERR now (used to be STDOUT).
+
+ o  URI::URL: removed version 3 compatibility stuff ($COMPAT_VER_3).
+
+ o  $ua->mirror should work better on dosish systems (can not
+    rename when target file exists).
+
+ o  Typo in lwp-download prevented it from compiling.
+
+ o  Some minor documentations typos corrected.
+
+
+
+Sat Sep 20 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.13
+
+ o  Brand new module called HTTP::Cookies.  It stores cookies
+    (Set-Cookie and Set-Cookie2 headers) from responses and can
+    create appropriate Cookie headers for requests.  It can also
+    share cookie files with Netscape.
+
+ o  LWP::UserAgent now support the cookie_jar() attribute.  When
+    set to an HTTP::Cookies object, it will automatically manage
+    the cookies sent to the servers.  Off by default.
+
+ o  New header utility functions in HTTP::Headers::Util.
+
+ o  Win32 and OS/2 patches for the lwp-xxx utilities.  Turn on
+    binary mode by default (option to turn it off), avoid modifying $0,
+    and don't be confused about suffixes in the script names.
+    Contributed by Ben Coleman <bcoleman@mindspring.com>
+
+ o  OpenVMS patch for Font:AFM by Brad Hughes <brad@tmc.naecker.com>
+
+
+
+Fri Sep  5 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.12
+
+ o  decode_entities() would sometimes introduce ";" after
+    things that looked like they were entities.
+
+ o  HTML::LinkExtor knows about <applet code="...">
+
+ o  Patch from Gary Shea <shea@gtsdesign.com> that makes the
+    tests work even if perl is not called "perl"
+
+ o  HTTP::Date handles 12:00PM correctly now. Patch from
+    William York <william@mathworks.com>
+
+ o  HTTP::Request::Common don't quote the boundary string for
+    multipart/form-data messages any more.
+
+ o  Font::AFM works for encodings where .notdef is defined to
+    have some size.  Documentation and efficiency update.
+
+
+
+Wed Aug  6 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.11
+
+ o  Perl version 5.004 is now required for libwww-perl.
+
+ o  Win32 patches from Gurusamy Sarathy <gsar@engin.umich.edu>.
+    Now passes all tests on that platform.
+
+ o  HTTPS support contributed by Josh Kronengold <mneme@mcny.com>
+
+ o  Support hex entities &#xFF;  HTML::Entities::(en|de)code only
+    modify their argument in void context.
+
+ o  Fixed formatter bug with <font> tags which did not specify size.
+
+ o  Better HTML::HeadParser documentation
+
+ o  Fixed HTML::LinkExtor documentation typo spotted by Martijn.
+
+ o  HTTP::Request::Common now use CRLF for multipart/form-data
+
+
+
+Fri Jun 20 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.10
+
+ o  Make '+' a reserved URL character.  Decode unescaped '+' as
+    space in $url->query_form().
+
+ o  Use $Config{d_alarm} to determine default for $ua->use_alarm()
+
+
+
+Tue Jun 10 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.09
+
+ o  Removed the MIME modules from the distribution.  They are distributed
+    separately now.
+
+ o  Added a new module called HTTP::Request::Common
+
+ o  Improved HTTP::Status documentation.  It is now also possible
+    to import the is_client_error/is_server_error functions.
+
+ o  LWP::MediaTypes::guess_media_type() can now take an optional
+    HTTP::Header parameter.
+
+ o  LWP::Protocol ensures that scheme is legal as module name.
+
+ o  LWP::Protocol::http is not as strict when trying to verify the
+    method name.  It now also accepts responses without a message
+    on the status line.
+
+ o  WWW::RobotRules::AnyDBM_File: Some DBMs fail to allow multiple
+    opens of the same file.  Patch from Mark James <jamesm@skate.org>
+
+ o  Created Bundle::LWP
+
+
+
+Sat Apr  5 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.08
+
+ o  Made libwww-perl warning compatible with upcoming perl5.004beta2
+    (aka 5.003_98)
+
+ o  encode_base64() did not work properly if pos() of the string to
+    encode was different from 0.
+
+ o  HTML::Parser was confused about "</" when it did not start an end tag.
+
+ o  HTML::FormatPS will provide ISOLatin1Encoding in its output.
+
+ o  Calling HTML::LinkExtor->links will clear out old links.
+
+ o  url()->rel($base) would ignore the $base argument.
+
+ o  Don't croak() when setting url()->path().
+
+
+
+Tue Feb 11 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.07
+
+ o  Can limit the size of the response content with $ua->max_size()
+
+ o  Added time2iso() functions to HTTP::Date.
+
+ o  Made LWP::Protocol::http more portable to the MacPerl. /./ match
+    different things on MacPerl.
+
+
+
+Mon Jan 27 1997   Gisle Aas <aas@sn.no>
+
+ o  Release 5.06
+
+ o  URI::URL is now compatible with perl5.004 overloading.
+
+ o  HTML::HeadParser makes X-Meta-Name headers for <meta> elements
+    that does not specify an 'http-equiv' attribute.
+
+ o  URI::URL::ftp does not die if Net::Domain is not installed and
+    you ask for an anonymous username or password.
+
+ o  WWW::RobotRules:  The robots.txt parser did not ignore comment lines
+    as it should.
+
+ o  LWP::Protocol::http is more forgiving towards servers that return
+    bad responses.
+
+ o  Allow "?" before search string in gopher URLs.
+
+ o  LWP::Protocol::file did not escape funny filenames when generating
+    HTML directory listings.
+
+ o  LWP::Protocol::ftp now gets the Content-Encoding correct. 'CODE'
+    content in PUT requests also work now.
+
+ o  Relative locations in redirects did not work with URI::URL::strict.
+
+ o  OS/2 portability patches from Ilya Zakharevich
+
+ o  LWP::Authen::* patch from Doug MacEachern
+
+ o  Support experimental data:- URLs
+
+ o  Some tests (those using HTTP::Daemon) now die more gracefully if
+    IO::* modules is not installed.
+
+
+
+Wed Dec  4 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.05
+
+ o  LWP::UserAgent::simple_request: local($SIG{__DIE__}) protects us
+    against user defined die handlers.
+ 	
+ o  Use Net::Domain (instead of Sys::Hostname) to determine FQDN.  It
+    is used by URI::URL when it determines anonymous ftp login address.
+	
+ o  lwp-download: New program in the bin directory
+
+ o  HTML::Parser: Allow '_' in attribute names.  This makes it possible
+    to parse Netscape's bookmarks.html file.
+
+ o  HTTP::Daemon: Fixed chunked transfer encoding and multipart content
+    in get_request().  Support HTTP/0.9 clients.
+
+ o  Don't clobber regex variables when HTTP::Message delegates methods
+    to the header.
+
+ o  Base64::decode_base64 now checks that the length input string to
+    decode is a multiple of 4.
+
+ o  t/robot/rules-dbm.t clean up better and will use AnyDBM for dumping
+
+ o  File::CounterFile: $/ strikes again by Andreas König
+
+ o  File::Listing updates from William York <william@mathworks.com>. We
+    can now parse the MS-Windows ftp server listings.
+
+ o  HTTP::Date now supports the MS-Windows 'dir' date format.  Patch by
+    William York.
+
+ o  LWP::MediaTypes::media_suffix will return first type in scalar context.
+
+
+
+Tue Oct 22 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.04
+
+ o  Added HTTP::Daemon.  This is a HTTP/1.1 server class.  This means
+    that libwww-perl no longer is a client library only.  The HTTP::Daemon
+    is also used in the new test suite.
+
+ o  HTTP::Message support the protocol() method.  Used by HTTP::Daemon.
+
+ o  HTTP::Response can be constructed with a header and content as
+    argument.
+
+ o  Typo corrections in the documentation.
+
+ o  File::Listing::parse_dir accepts "GMT" as timezone now.
+
+ o  HTML::Parser will call the start() method with two new parameters;
+    $attrseq, $origtext.
+
+ o  Integrated HTML::FormatPS patches from
+    Jim Stern <jstern@world.northgrum.com>
+
+ o  Class modules don't inherit from AutoLoader any more.  They just
+    import the AUTOLOAD method.
+
+ o  LWP::Protocol will untaints scheme before loading protocol module.
+
+ o  Digest does not send "opaque" if it was not present in the request.
+    The "Extension" header is not returned any more.
+
+ o  New method: $url->crack that will return a list of the various
+    elements in a URI::URL.
+
+ o  WWW::RobotRules did not use the agent() method when determining
+    who we are.  This affected WWW::RobotRules::AnyDBM_File parsing
+    for robots.txt.  Visit count did not increment for
+    WWW::RobotRules::InCore.
+
+
+
+Tue Oct  1 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.03
+
+ o  Hakan Ardo's persistent robot rules is now part of the standard
+    distribution.  This is still experimental and might change in the
+    future.  It includes the new WWW::RobotRules::AnyDBM_File class
+    and updates to LWP::RobotUA.
+
+ o  HTML::Parser now supports buggy Netscape comment parsing.  Enable
+    it by calling $p->netscape_buggy_comment(1).  The previous version
+    of the parser could also (under very unlucky and unlikely
+    circumstances) call the $self->comment() method several times for
+    the same comment text.
+
+ o  HTML::Parser: Use new $p->eof to signal end of document instead of
+    $p->parse(undef).
+
+ o  HTML::Element::starttag() is now smarter about which quotes it
+    use around attribute values.
+
+ o  New HTTP::Response methods: current_age(), freshness_lifetime(),
+    is_fresh(), fresh_until().
+
+ o  HTTP::Message:  New method ($mess->content_ref) which will return
+    a reference to the current content.
+
+ o  URI::URL:  New method ($url->rel) which does the opposite of abs().
+    Example: url("http://host/a/b/c", "http://host/c/d/e")->rel would
+    return url("../../a/b/c", "http://host/c/d/e").  This was
+    suggested by Nicolai Langfeldt <janl@ifi.uio.no>
+
+ o  URI::URL:  $url->query_form can now take array references as value
+    specification.  For instance: $url->query_form(foo => ['bar', 'baz']
+
+ o  Avoid '"my" variable $var masks earlier declaration in same scope'
+    warnings in perl5.003_05.
+
+
+
+Wed Sep 11 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.02
+
+ o  lwp-rget:  Initialize proxy settings from envirionment
+
+ o  HTML::Entities::encode_entities: Don't encode $ and %
+
+ o  HTML::LinkExtor::links: Now works when no links were found.
+
+ o  HTTP::Headers::as_string: Allow \n in header value
+
+
+
+Tue Aug  1 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.01.
+
+ o  Updated ftp protocol module to be compatible with Net::FTP 
+    version 2.00 (the version found in libnet-1.00)
+
+ o  New HTML parser module called HTML::LinkExtor
+
+ o  Various documentation typo corrections.  Most of these contributed
+    by Bob Dalgleish.
+
+ o  HTML::HeadParser updates 'Content-Base' instead of 'Base'.  It also
+    updates the 'Link' header based on <link ...>
+
+ o  HTTP::Headers and HTTP::Status updated according to
+    draft-ietf-http-v11-spec-06
+
+ o  HTTP::Headers can now use "_" as alternative to "-" in field names.
+
+ o  HTTP::Response::base now looks for 'Content-Base',
+    'Content-Location' and 'Base' headers.
+
+ o  Avoid warning in LWP::MediaTypes if $ENV{HOME} is not defined.
+
+ o  The new $ua->parse_head() method can be used to turn off
+    automatic initialization of response headers from the <HEAD>
+    section of HTML documents.
+
+ o  Added eq() method for URI::URL objects
+
+ o  The HTML::Formatter recovers even if a handle method is not defined
+    for all tags found during traversal
+
+
+
+Sun May 26 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5.00.
+
+ o  LWP::Simple::head() now return something useful in scalar context.
+
+ o  Rewritten the HTML::Parse stuff.  Introduced the HTML::Parser class
+    that will tokenize a HTML document.  The rest of the old
+    HTML::Parse functionality has moved to HTML::TreeBuilder class.
+    Note, that the HTML stuff is still alpha.
+
+ o  Implemented HTML::HeadParser.  This is a lightweight parser for
+    the <HEAD> section of a HTML document.
+
+ o  HTML::Element had problem with presenting things like <foo
+    bar="bar">.
+
+ o  HTML::Entities: Included additional ISO-8859/1 entities listed in
+    RFC1866.
+
+ o  HTML::AsSubs exported 'header' instead of 'head'
+
+ o  We know about a few more of the HTML 3.2 element.
+
+ o  HTTP::Date had problems with years before 1970, because Time::Local
+    went into an infinite loop.  Check for this.
+
+ o  Added $header->title method.
+
+ o  Made $header->authorization_basic return "uname:passwd" in scalar
+    context
+
+ o  LWP::Protocol::collect() will let the HTML::HeadParser look at the
+    document content as it arrives.  This will initialize headers from
+    elements like <base href="...">, <title>...</title> and <meta
+    http-equiv="..." ...>.
+
+ o  Simplified $response->base implementation, because we don't have
+    to look into the content any more.
+
+ o  Added -quiet option to lwp-rget
+
+ o  Typo fixes and some documentation additions.
+
+
+
+Thu May  9 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b13
+
+ o  Made URI::URL::strict(0) the default.  I got tired of all this
+    eval { } stuff just to recover.  The URI::URL::strict'ness also
+    determine if calling some standard method that happens to be
+    illegal for some protocol scheme will croak or just be ignored.
+
+ o  Ensure correct $INPUT_RECORD_SEPARATOR and $OUTPUT_RECORD_SEPARATOR
+    at places where we <> or print.
+
+ o  Always localize $_ before any 'while(<FILE>) {}'-loops
+
+ o  Implemented $protocol->collect_once() and simplified several
+    of the protocol implementations by using it.
+
+ o  The HTML parser used to get it wrong if you were unlucky about the
+    breakup of the text. An example of broken behaviour was this:
+
+        $html = parse_html "<!-- A comment -";
+        $html = parse_html "-> and some text.";
+
+ o  The HTML parser does not ignore whitespace-only text any more.
+
+ o  HTML::Parse warnings are now optional and turned off by default.
+
+ o  New start for $html->as_HTML().
+
+ o  Fixed some typos
+
+
+
+Wed Apr 24 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b12
+
+ o  New utility program called 'lwp-rget'.
+
+ o  $response->base was broken for HTML documents
+
+ o  New fancy LWP::Debug import() method.  Can now turn on debugging with
+    "use LWP::Debug '+';"
+
+ o  Trap exceptions (die) from the response callback routine
+
+ o  The RobotUA now requires an e-mail address of the person reponsible
+    for the robot.
+
+ o  New $ua->from() method.
+
+ o  Support for gopher Index-Search (gopher type '7' requests).
+    Contributed by Harry Bochner <bochner@das.harvard.edu>
+
+ o  Cleaned up white-space usage in the source.
+
+
+
+Wed Apr  3 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b11
+
+ o  Implemented a NNTP protocol module.  The library can now fetch and
+    post news articles.
+
+ o  More documentation
+
+ o  Don't look at the URI header for redirects
+
+ o  New $res->base() method for HTTP::Responses
+
+ o  Graham Barr's patch to File::Listing to make it more OO internally
+
+ o  content_type() return value is canonicalized
+
+ o  $ua->request() does not die on bad URLs any more
+
+ o  LWP::MediaTypes merge all media.types files that if finds
+
+ o  FTP request with content to file or callback did not work
+
+ o  The value of HTTP Host: header is now $url->netloc;
+
+ o  The URI::URL constructor now accept URLs wrapped up in "<>"
+
+ o  $url->abs() now has a second optional argument that makes it accept
+    that relative URLs can have scheme, i.e. treat "http:img.gif" as a
+    relative URL.
+
+ o  Added prototypes to the HTTP::Status::is_xxx() functions
+
+ o  Added prototypes to the MIME:: encoding/decoding functions
+
+ o  Always return scheme for mailto and news URLs (as_string)
+
+ o  RobotRules patches from Henry A Rowley.
+
+ o  More tests
+
+ o  <SMALL> and <BIG> again
+
+
+
+Thu Mar 14 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b10
+
+ o  GET ftp://host/doc was never successful for normal files.
+
+ o  LWP::Socket: read_until() did not notice EOF condidtion.  I.e. if
+    a HTTP server closed the connection before any output was generated,
+    the we continued to read 0 bytes in a busy loop until the alarm()
+    killed us.
+
+ o  Added support for Digest Access Authentication.  Contributed by
+    Doug MacEachern <dougm@osf.org>.
+
+ o  Makefile.PL: check for MD5 library
+
+ o  No longer print message content in HTTP::Response::error_as_HTML()
+
+ o  Access to "file:/path" gave warning when the envirionment variable
+    no_proxy was set.
+
+ o  The http-post test sends a Content-Type header.  Some servers hang
+    if this header is missing.
+
+ o  HTML::Parse:
+     -   allow <SMALL> and <BIG> tags
+     -   allow empty attribute values
+
+
+
+Tue Mar  5 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b9
+
+ o  Started to write on the libwww-perl cookbook (lwpcook.pod)
+
+ o  The URI::URL module now exports the function url().  This is an
+    alternative (easy to use) constructor function.
+
+ o  Expanding relative file URLs starting with "#" did not work.
+
+ o  Fixed autoloaded DESTROY problem by adding empty DESTROY routine
+    to URI::URL.
+
+ o  Does not try generate password for ftp-URLs unless the username is
+    "anonymous" or "ftp"
+
+ o  The LWP::Simple user agent proxy settings are intialized from
+    the proxy environment variables. 
+
+ o  LWP::Protocol::ftp: Use the Net::FTP library to access ftp servers.
+    Convert directories to HTML on request (Accept: text/html).
+
+ o  New module HTTP::Negotiate
+
+ o  New module File::Listing
+
+ o  HTTP::Date::str2time can parse a few more formats, like the 'ls -l'
+    format and ISO 8601.  The function now also takes an optional second
+    parameter which specify a default time zone.
+
+ o  Added prototypes to the HTTP::Date functions.
+
+ o  The library adds a timestamp to responses ("Client-Date")
+
+ o  HTTP::Status:  Updated to proposed HTTP/1.1
+
+ o  HTTP::Headers: Updated to proposed HTTP/1.1
+
+ o  LWP::Protocol::http:  Updated to HTTP/1.1 methods
+
+ o  Took out stringify overloading in HTML::Element.
+
+
+
+Mon Feb 26 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b8
+
+ o  Renamed functions using thisStyleOfNames to this_style_of_names.
+    Provided a script called 'update_from_5b7'
+
+ o  Renamed the 'request' and 'mirror' scripts to 'lwp-request' and
+    'lwp-mirror'.  The GET, POST, HEAD aliases for 'lwp-request' are
+    the same.
+
+ o  Implemented LWP::RobotUA
+
+ o  Class name for RobotRules did not match the file name
+
+ o  URI::URL
+      - perl5.002gamma is required (because use vars).
+      - The leading slash in now part of the path if it is present.
+      - More documentation
+      - Use AutoLoader to speed things up.
+      - New class URI::URL::_login and made telnet, rlogin, tn3270
+        subclasses from this one.
+      - URI::URL::wais is better supported.
+      - URI::URL::news is better supported.
+      - New URI::URL::mailto methods: user/host
+
+ o  HTTP::Date::time2str now works correctly with '0' as argument
+
+ o  HTTP::Message delegates unknown methods to the headers.
+
+ o  HTTP::Request::uri is an alias for HTTP::Request::url.  Can set
+    the URL to undef.
+
+ o  Added convenience methods to HTTP::Headers for easy access to
+    frequently used headers.
+
+ o  Simplified LWP::Debug
+
+ o  Use standard AutoLoader for LWP::IO functions.
+
+ o  Played with the profiler (Devel::DProf) and made some things
+    go quicker.
+
+ o  Included the File::CounterFile module.  Excluded Mail::Cap module
+    as it is also part of the MailTools package.
+
+
+
+Mon Feb  5 1996   Gisle Aas <aas@sn.no>
+
+ o  Release 5b7
+
+ o  Perl5.002 is required now
+
+ o  Rewrite of the URI::URL module (version 4.00)
+      - escaping/unsafe stuff redone (and removed)
+      - URI::URL::_generic moved out of URL.pm
+      - netloc, path, params, query is now stored internally in escaped form
+      - new methods for URI::URL::_generic are:
+           epath
+           eparams
+           equery
+	   path_components
+           absolute_path
+      - new methods for URI::URL::http are:
+           keywords
+           query_form
+      - new methods for URI::URL::file are:
+           newlocal
+           local_path
+           unix_path
+           dos_path
+           mac_path
+           vms_path
+
+ o  URI::Escape now semi-compile regular expressions (by evaling an
+    anonymous sub).  Same techinque is also used in HTML::Entities.
+
+ o  HTTP::Date parser rewritten using regular expressions.
+
+ o  HTTP::Headers->as_string() now croaks if any field values
+    contain newline.
+
+ o  HTTP::Status constants use empty prototype.
+
+ o  Font metrics moved to a new subdirectory (lib/Font/Metrics)
+
+ o  Don't use the VERSION script any more (even if it was very clever)
+
+ o  HTML::Entities will now export the names decode_entities() and
+    encode_entities().
+
+ o  Andreas Koenig's black patch to HTML::Element.
+
+ o  The HTML::Formatter now knows about <menu> and <dir> tags
+
+ o  The construct "defined ref($arg)" did not work on perl5.002
+    becase ref now always return something defined.
+
+ o  LWP::UserAgent sends by default an 'User-Agent' header.
+
+ o  LWP::Simple sends 'User-Agent' header to servers.
+
+ o  Updated the LWP::Socket module to use the new Socket.pm interface.
+
+ o  LWP::Protocol::http sends the new HTTP/1.1 'Host' header.
+
+ o  LWP::Protocol::file use $url->local_path to get a file to open.
+    It also inserts a <BASE> tag in directories instead of a redirect.
+
+ o  MIME::Base64 routines can be called as MIME::Base64::encode() and
+    MIME::Base64::decode().  Same kind of thing for QuotedPrint.
+
+
+
+Mon Nov  6 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Release 5b6
+
+ o  Installation should work better for those that still runs
+    perl4 as 'perl'.  The mirror script is not installed by
+    default.
+
+ o  LWP::Socket::_getaddress() Numerical IP adresses did not work.
+
+ o  LWP::Socket::pushback() did not work.  This also avoids the bad
+    pp_select() core dump from perl.
+
+ o  LWP::IO now also selects on socket exceptions.
+
+ o  HTML::Parse:  Ignore <!DOCTYPE ...> regardless for case.  Some
+    bad insertElement() calls made infinite loops.
+
+ o  The uri.t test works better for places where /tmp is a sym-link.
+
+
+
+Sat Sep 16 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Release 5b5
+
+ o  newlocal URI::URL does not put "//localhost" into the URLs any
+    longer.
+
+ o  The request program: New -o option to reformat the HTML code
+    New -C option to provide credentials on the command line.
+    The -b option never worked.
+
+ o  LWP::Protocol::file now returns redirect for access to directories
+    where the trailing slash is missing.
+
+
+
+Thu Sep 14 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Speedups and bug fixes in the HTML parser.  The parser now
+    understands some more depreciated tags (like <xmp> and <listing>).
+
+ o  HTML::Elements are now stingified using perl overloading.
+    The interface to the traverse() callback has changed.
+
+ o  Implemented HTML formatters for plain text and Postscript.
+
+ o  Added lib/Font stuff to support the Postscript formatter.
+
+ o  Inspired by Tim Bunce, I implemented the HTML::AsSubs module.
+    Don't know if it is really useful.
+
+ o  The local/get test does not use /etc/passwd any more.
+
+
+
+Thu Sep  7 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Changed package name to libwww-perl-5xx
+
+ o  Made LWP::Protocol::ftp actually transfer data
+
+ o  Implemented methods for LWP::Socket to act as a server:
+    bind(), listen(), accept(), getsockname(), host(), port()
+
+
+
+Wed Sep  6 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Release 0.04
+
+ o  Implemented modules to parse HTML.
+
+
+
+Mon Sep  4 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Implemented Mail::Cap which will become part of the MailTools
+    package.
+
+ o  Moved Base64 to MIME::Base64.  Reimplemented MIME::Base64 by using
+    [un]pack("u",...)   Implemented LWP::MIME::QuotedPrint for
+    completness sake.  Routine names has changed as suggested by Tim
+    Bunce.
+
+ o  LWP::MediaType reads default types from media.types file.
+    guessMediaType() now also returns encodings.  New function mediaSuffix()
+
+ o  Pass $url to $ua->getBasicCredentials().  This also fixes security
+    hole with the old implementation of getBasicCredentials().
+
+ o  LWP::Protocol::file now sets Content-Encoding headers
+
+ o  Allow request content to be provided by a callback routine.
+
+ o  Fix bug that prevented response callback to work.  The first parameter
+    (data) is no longer a reference, because $_[0] is already a reference.
+    Don't callback unless successful response.  Callbacks during redirects
+    was confusing.
+
+ o  URI::URL.  Remove port from netloc if it is the default port.
+    Don't use anything, just require.
+
+ o  HTTP::Message->addContent() does not need a reference parameter.
+
+ o  LWP::Socket->open() has been renamed top LWP::Socket->connect()
+    LWP::Socket->close has gone.  Implemented new method LWP::Socket->read()
+    that returns as fast as it has some data to offer.  Implemented
+    LWP::Socket->pushback().
+
+ o  Does not die in LWP::UserAgent->request()
+
+ o  LWP::Socket now use LWP::IO for low level IO
+
+ o  Implemented LWP::TkIO as a replacement module for LWP::IO when using Tk.
+
+
+
+Thu Aug 17 1995   Gisle Aas <aas@oslonett.no>
+
+ o  $ua->redirectOK() for checking redirects
+
+ o  reorganized tests in the "t" directory.
+
+
+
+Fri Aug 11 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Release 0.03
+
+ o  Included RobotRules.pm from Martijn Koster
+
+
+
+Thu Aug 10 1995   Gisle Aas <aas@oslonett.no>
+
+ o  New module URI::Escape (URI::URL use this module for default
+    escaping) that provide the uri_escape() and uri_unescape()
+    functions.
+
+ o  Setting $url->scheme now changes the class of the object.
+
+ o  Made $httpurl->user() and $httpurl->password() illegal.
+    Likewise for other URL schemes.
+
+
+
+
+Wed Aug  9 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Reorganisation as discussed on <libwww-perl@ics.uci.edu>
+       LWP::Date       --> HTTP::Date
+       LWP::MIMEheader --> HTTP::Headers
+       LWP::StatusCode --> HTTP::Status
+       LWP::Message    --> HTTP::Message
+       LWP::Request    --> HTTP::Request
+       LWP::Response   --> HTTP::Response
+       LWP::MIMEtypes  --> LWP::MediaTypes
+
+ o  HTTP::Date parses ctime format with missing timezone as suggested
+    by Roy Fielding <fielding@beach.w3.org>
+
+ o  HTTP::Status and LWP::MediaTypes exports their functions by default.
+
+ o  Splitted up the URI::URL module.  Schemes are implemented by separate
+    files that are autoloaded when used.  Self test moved to "t/uri.t".
+
+
+       
+Mon Aug  7 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Applied patch from Marc Hedlund <hedlund@best.com>
+       - Update the @header_order according to the August 3rd draft.
+       - Added Response Header fields: 'Location', 'Public', 'Retry-After',
+         'Server', and 'WWW-Authenticate'.
+       - Moved 'unknown header' handling from &scan to &header. The old
+         implementation was forcing all unknown header-words to begin with
+	 an uppercase (as it should be), but disallowed other uppercase
+	 letters.
+       - updates the status code messages under the August
+         3rd HTTP/1.0 draft.  '203' became 'Non-Authoritative Information',
+	 '303' became 'See Other', and a new code,
+	 '411 Authorization Refused', was added.
+
+ o  Can remove multiple headers with single removeHeader() call in MIMEheader.
+
+ o  Can assign multiple field/value pairs in header() method of MIMEheader.
+
+ o  A field with multiple values is printed as separate values in
+    MIMEheader::as_string().
+
+ o  LWP::Response contain new attributes: previous() and request().  These
+    attributes are updated by the UserAgent.
+
+ o  Appended \n to some die statements in Socket so that line numbers are
+    suppressed in error messages.
+
+ o  Made UserAgent::clone work for reference members
+
+ o  Check for redirect loops and multiple authorization failures by
+    examination of the response chain.
+
+ o  Use "\015\012" instead of "\r\n" in protocol modules.  Some systems
+    define \r and \n differently.
+
+ o  request program can now handle documents that needs autorization by
+    prompting the user for username/password. Added new -S option to print
+    request/response chain.
+
+
+  
+Tue Jul 25 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Release 0.02
+
+ o  Included URI::URL in the release
+
+
+
+Mon Jul 24 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Incorporated Makemake.PL and VERSION from Andreas Koenig <koenig@mind.de>
+    As a result of this the following things have changed:
+       - programs in "bin" are extracted from .PL-files
+       - reintroduced "lib"
+       - "test" has been renamed as "t"
+       - test programs in "t" has been made Test::Harness compatible
+       - we now have a MANIFEST file
+       - no more need fro update_version, make-dist, lwp-install
+
+ o  Renamed bin/get to bin/request.  Links to it are now all upper case.
+
+ o  Proxy support in bin/request (provided by Martijn Koster)
+
+ o  UserAgent can now load proxy settings from environment
+
+ o  LWP::Protocol::ftp is under way but not really finished
+
+
+
+Tue Jul 18 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Implemented LWP::Protocol::gopher
+
+ o  Implemented LWP::Protocol::mailto
+
+ o  Fixed proxy typo
+
+
+
+Mon Jul 17 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Made release 0.01
+
+
+
+Mon Jul 17 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Don't loose first line of HTTP/0.9 requests
+
+ o  LWP::Socket use syswrite() for writing
+
+ o  Added RC_* documentation to LWP::StatusCode
+
+ o  LWP::Date now use hash to look up month numbers
+
+ o  Added -f option to "get"
+
+ o  Untabify
+
+ o  Added a "TODO" list
+
+ o  Fixed various typos
+
+
+
+Fri Jul 14 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Reorganized directories.  Moved LWP.pm up.  Moved file.pm and http.pm
+    into the LWP::Protocol direcotory.  Moved LWP out of the lib directory
+    and removed lib.
+
+ o  Implemented the "get" and "mirror" scripts in the "bin" directory.
+
+ o  Implemented "install-lwp", "update_version" and "make-dist".  The library
+    version number is found in the VERSION file.
+
+ o  Always adds 1900 to the year in LWP::Date
+
+ o  In LWP::MIMEheader:  Implemented clone(), removeHeader() and scan() 
+    methods.  Reimplemented asString.  Removed asMIME().  Moved "Good
+    Practice" into this file, and reimplemented it.
+
+ o  Moved "header" and "content" into LWP::Message class.  This change made
+    LWP::Request and LWP::Response much simpler.  Made clone() method
+    actually work.
+
+ o  LWP::Protocol::implementor does not die if it cannot load package.
+
+ o  Moved UserAgent convenience methods into LWP::Simple.  Made LWP::Simple
+    export LWP::StatusCode symbols and functions.
+
+ o  Implemented $ua->isProtocolSupported($scheme) method.
+
+ o  Nicer directory listing in LWP::Protocol::file.pm
+
+ o  Rely on URI::URL 3.00 behaviour for $url->full_path
+
+ o  Library version number now in LWP.pm.  You should be able to say
+    "use LWP 1.0;" if you need at least this version.
+
+ o  Various cleanups and arranging layout as I like it.  Use fooBar-style
+    (as opposed to foo_bar stype) everywhere.  This means that as_string()
+    now is called asString().
+
+ o  Added more documentation.
+
+
+
+Wed Jun 14 1995   Gisle Aas <aas@oslonett.no>
+
+ o  Removed lot of rendundant & before function calls.
+
+ o  $this --> $self
+
+ o  &collector passes content as a reference, don't want to copy so much
+
+ o  parameterlist to collect callback has been rearranged
+
+ o  Response::addContent gets a reference to the data
+
+ o  Added some callback documentation to UserAgent.pm
+
+ o  Protocol::file now uses the collector
+
+ o  Introduced LWP::Simple
+
+
+
+Sun Jun 11 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Added basic authentication support
+
+ o  Added mirroring of single documents
+
+ o  Change Protocol construction from whacky URL.pm (constructor returns
+    subclass) to more normal C++'ish way.
+
+
+
+Wed June 7 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Minor cleanups from printed code inspection
+
+
+
+Wed May 24 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Added redirection resolution
+
+ o  Added optional autoloading of protocols
+
+
+
+Tue May 23 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Separated socket stuff into separate module
+
+ o  Added HTTP proxy support
+
+ o  Made alarm handling optional
+
+ o  Added a LWP::Message for completeness sake
+
+ o  Added LWP::MemberMixin to reduce code duplication
+
+ o  Cosmetic changes to LWP::Date
+
+ o  Renamed LWP::Error to LWP::StatusCode
+
+ o  Renamed LWP::MIME to LWP::MIMEtype
+
+ o  Changed the tests to cope with all this
+
+It's getting there...
+
+
+
+Mon May 22 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Changed the socket reading to use sysread.  This will have to go
+    into a module of its own.
+
+
+
+Thu 18 May 1995   Martijn Koster <m.koster@nexor.co.uk>
+
+ o  Mentioned on libwww-perl that I had changed the classes around lots.
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/INSTALL.w3mir
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/INSTALL.w3mir	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/INSTALL.w3mir	(revision 21112)
@@ -0,0 +1,48 @@
+
+- Get libwww-perl from CPAN 
+  (http://www.perl.com/CPAN/modules/by-module/WWW/).  W3mir does not work
+  with LWP 5.40 and later versions.  Please get an earlier version.
+  It also does not work with the new URI package.  Use the URI package
+  enclosed in the older LWP.
+
+- Unpack it:
+
+    gzip -dc libwww-perl-*.tar.gz | tar xvBf -
+
+- Install it.  W3mir does not require the whole libwww-perl package and
+  all the packages that goes with it to work so you may follow these
+  install instructions instead of the libwww-perl install instructions
+  to get an easier install which will work with w3mir.
+
+    cd libwww-perl-*
+
+  Are you going to install it in the standard perl place (you need to
+  be the system administrator to do this)? If so:
+
+    perl Makefile.PL
+
+  If not (if you are a user):
+
+    perl Makefile.PL PREFIX=/some/dir
+
+  /some/dir could be ~ (as in your home directory).
+
+  IF you are installing an old LWP just for w3mir you can install both
+  the old LWP, the Base64 module and w3mir in a special PREFIX
+  separate from your usual install PREFIX.  This ensures that w3mir will
+  access the older version of LWP and that other software will access
+  the newer version.  If you do this make sure you install all of LWP,
+  Base64 and w3mir with the same PREFIX.
+
+  This step might produce some complaints;  A standard perl
+  installation will typically get messages about missing these
+  modules: IO::Socket, NET::FTP and MD5.  None of these need be
+  installed to run w3mir and so any error messages about these may
+  safely be ignored.
+
+    make
+    make test      (robot/ua and local/http might fail, that's OK)
+    make install
+
+  Now libwww-perl should be installed and working.
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/MANIFEST
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/MANIFEST	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/MANIFEST	(revision 21112)
@@ -0,0 +1,106 @@
+ChangeLog			History of this package
+MANIFEST			This file
+Makefile.PL			Makefile generator
+README				Get you started with this package
+TODO				The To-Do list for the library
+bin/lwp-download.PL             Writes bin/lwp-download script
+bin/lwp-mirror.PL		Writes bin/lwp-mirror script
+bin/lwp-request.PL		Writes bin/lwp-request script
+bin/lwp-rget.PL			Writes bin/lwp-rget script
+lwpcook.pod			Libwww-perl examples
+lib/Bundle/LWP.pm               Defines prereq modules
+lib/File/Listing.pm		Parse directory listings
+lib/HTTP/Cookies.pm             Cookie storage and management
+lib/HTTP/Daemon.pm		A simple httpd
+lib/HTTP/Date.pm		Date conversion routines
+lib/HTTP/Headers.pm		Class encapsulating HTTP Message headers
+lib/HTTP/Headers/Auth.pm	Some methods that deal with authorization.
+lib/HTTP/Headers/ETag.pm	Some methods that deal with entity tags
+lib/HTTP/Headers/Util.pm        Some utility functions for header values.
+lib/HTTP/Message.pm		Class encapsulating HTTP messages
+lib/HTTP/Negotiate.pm		Evaluate HTTP content negotiation algoritm
+lib/HTTP/Request.pm		Class encapsulating HTTP Requests
+lib/HTTP/Request/Common.pm      Generate common requests
+lib/HTTP/Response.pm		Class encapsulating HTTP Responses
+lib/HTTP/Status.pm		HTTP Status code processing
+lib/LWP.pm			Includes what you need
+lib/LWP/Authen/Basic.pm		Basic authentication scheme
+lib/LWP/Authen/Digest.pm	Digest authentication scheme
+lib/LWP/Debug.pm		Debugging support
+lib/LWP/MediaTypes.pm		Library for guessing media types
+lib/LWP/MemberMixin.pm		Helps you access %$self
+lib/LWP/Protocol.pm		Virtual base class for LWP protocols
+lib/LWP/Protocol/data.pm        Access to data URLs
+lib/LWP/Protocol/file.pm	Access local files
+lib/LWP/Protocol/ftp.pm		Access with the FTP protocol
+lib/LWP/Protocol/gopher.pm	Access with the Gopher protocol
+lib/LWP/Protocol/http.pm	Access with HTTP protocol
+lib/LWP/Protocol/https.pm	Access with HTTP protocol over SSL
+lib/LWP/Protocol/mailto.pm	Allows you to POST mail using sendmail
+lib/LWP/Protocol/nntp.pm	Handles access to news: and nntp: URLs
+lib/LWP/RobotUA.pm		Easy creation of conforming robots
+lib/LWP/Simple.pm		Procedural LWP interface
+lib/LWP/UserAgent.pm		A WWW UserAgent class
+lib/LWP/media.types		Mapping from file extentions media types
+lib/URI/Escape.pm		General URI escaping functions
+lib/URI/Heuristic.pm            User friendly URLs
+lib/URI/URL.pm			Class encapsulating URLs
+lib/URI/URL/_generic.pm		Implements generic hierarchical RL (RFC 1808)
+lib/URI/URL/_login.pm		Base class for telnet, tn3270 and rlogin
+lib/URI/URL/data.pm             Implements data URLs
+lib/URI/URL/file.pm		Implements file URLs
+lib/URI/URL/finger.pm		Implements finger URLs
+lib/URI/URL/ftp.pm		Implements ftp URLs
+lib/URI/URL/gopher.pm		Implements gopher URLs
+lib/URI/URL/http.pm		Implements http URLs
+lib/URI/URL/https.pm		Implements https URLs
+lib/URI/URL/mailto.pm		Implements mailto URLs
+lib/URI/URL/news.pm		Implements news URLs
+lib/URI/URL/nntp.pm		Implements nntp URLs
+lib/URI/URL/prospero.pm		Implements prospero URLs
+lib/URI/URL/rlogin.pm		Implements rlogin URLs
+lib/URI/URL/telnet.pm		Implements telnet URLs
+lib/URI/URL/tn3270.pm		Implements tn3270 URLs
+lib/URI/URL/wais.pm		Implements wais URLs
+lib/URI/URL/webster.pm		Implements webster URLs
+lib/URI/URL/whois.pm		Implements whois URLs
+lib/WWW/RobotRules.pm		Parse robot.txt files
+lib/WWW/RobotRules/AnyDBM_File.pm	Persistent robot rules
+t/README			How to run and set up tests
+t/TEST				Run tests
+t/base/common-req.t		Test HTTP::Request::Common module
+t/base/cookies.t                Test HTTP::Cookies module
+t/base/date.t			Test HTTP::Date module
+t/base/headers.t		Test HTTP::Headers module
+t/base/headers-auth.t		Test HTTP::Headers::Auth module
+t/base/headers-etag.t		Test HTTP::Headers::ETag module
+t/base/headers-util.t		Test HTTP::Headers::Util module
+t/base/listing.t		Test File::Listing module
+t/base/mediatypes.t		Test LWP::MediaTypes module
+t/base/message.t		Test HTTP::Request/HTTP::Response
+t/base/negotiate.t		Test HTTP::Negotiation module
+t/base/response.t		Test additional HTTP::Response methods
+t/base/status.t			Test HTTP::Status module
+t/local/autoload.t		Test autoloading of LWP::Protocol modules
+t/local/get.t			Try to get a local file
+t/local/http.t			Test http to local server
+t/local/protosub.t              Test with other protocol module
+t/net/cgi-bin/moved
+t/net/cgi-bin/nph-slowdata
+t/net/cgi-bin/slowread
+t/net/cgi-bin/test
+t/net/cgi-bin/timeout
+t/net/config.pl.dist		Suggested configuration for net tests
+t/net/http-get.t
+t/net/http-post.t
+t/net/http-timeout.t
+t/net/mirror.t
+t/net/moved.t
+t/net/proxy.t
+t/robot/rules-dbm.t		Test WWW::RobotRules::AnyDBM_File
+t/robot/rules.t			Test WWW::RobotRules
+t/robot/ua.t			Test LWP::RobotUA
+t/uri/abs-config.t              Test URI::URL configuration variables
+t/uri/base.t			Test URI::URL module
+t/uri/rel-base.t		Test URI::URL module some more
+t/uri/heuristic.t               Test URI::Heuristic module
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/Makefile.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/Makefile.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/Makefile.PL	(revision 21112)
@@ -0,0 +1,184 @@
+# This -*- perl -*- script writes the Makefile for libwww-perl
+# $Id$
+
+require 5.004;
+use strict;
+
+#--- Configuration section ---
+
+my @programs_to_install = qw(lwp-request lwp-mirror lwp-rget lwp-download);
+my @request_aliases     = qw(GET HEAD POST);
+
+#--- End Configuration - You should not have to change anything below this line
+
+
+# Allow us to suppress all program installation with the -n (library only)
+# option.  This is for those that don't want to mess with the configuration
+# section of this file.
+use Getopt::Std;
+use vars qw($opt_n);
+unless (getopts("n")) {
+    die "Usage: $0 [-n]\n";
+}
+@programs_to_install = () if $opt_n;
+
+
+# Check for non-standard modules that are used by this library.
+$| = 1;
+my $missing_modules = 0;
+
+print "Checking for HTML::Parser..";
+eval {
+    require HTML::HeadParser;
+};
+if ($@) {
+    print " failed\n";
+    $missing_modules++;
+    print <<EOT;
+$@
+The HTML::Parser is needed to extract correct <BASE> information from
+HTML so that we can resolv relative links correctly.
+
+EOT
+    sleep(2);  # Don't hurry too much
+} else {
+    print " ok\n";
+}
+
+print "Checking for MIME::Base64..";
+eval {
+    require MIME::Base64;
+    #MIME::Base64->require_version('2.00');
+};
+if ($@) {
+    print " failed\n";
+    $missing_modules++;
+    print <<EOT;
+$@
+The Base64 encoding is used in authentication headers in HTTP.
+
+EOT
+    sleep(2);  # Don't hurry too much
+} else {
+    print " ok\n";
+}
+
+print "Checking for Net::FTP......";
+eval {
+    require Net::FTP;
+    Net::FTP->require_version('2.00');
+};
+if ($@) {
+    print " failed\n";
+    $missing_modules++;
+    print <<EOT;
+$@
+The libwww-perl library normally use the Net::FTP module when
+accessing ftp servers.  You would have to install this package or
+configure your application to use a proxy server for making ftp
+requests work.  Net::FTP is part of the 'libnet' distribution.
+
+EOT
+    sleep(2);  # Don't hurry too much
+} else {
+    print " ok\n";
+}
+
+print "Checking for MD5 ..........";
+eval {
+    require MD5;
+};
+if ($@) {
+    print " failed\n";
+    $missing_modules++;
+    print <<EOT;
+$@
+The MD5 library is needed if you want to be able use the experimental
+"Digest Access Authentication" scheme.  Since very few servers
+implement this authentication scheme, you should normally not worry
+too much about this.
+
+EOT
+} else {
+    print " ok\n";
+}
+
+print <<EOT if $missing_modules;
+The missing modules can be obtained from CPAN.  Visit
+<URL:http://www.perl.com/CPAN/> to find a CPAN site near you.
+
+EOT
+
+
+# Ok, now it is time to really generate the Makefile
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+   NAME          => 'libwww-perl',
+   VERSION_FROM  => 'lib/LWP.pm',
+   PL_FILES      => { map {("bin/$_.PL" => "bin/$_")} @programs_to_install },
+   EXE_FILES     => [ map {"bin/$_"} @programs_to_install ],
+   'clean'       => { FILES => '$(EXE_FILES)' },
+   'dist'        => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
+);
+
+
+
+package MY;
+
+sub postamble
+{
+    my @m;
+    if (@request_aliases && grep($_ eq 'lwp-request', @programs_to_install)) {
+	push @m, "all ::\n";
+	if ($^O eq 'MSWin32') {
+	    push @m, "\t\$(FULLPERL) -e \"use Config; chdir q[\$(INST_EXE)]; "
+		    ."foreach (qw(@request_aliases)) { \" \\\n";
+	    push @m, <<'EOT';
+	-e "unlink \"$$_\"; " \
+	-e "system(\"copy lwp-request $$_\") && die; }"
+EOT
+	} else {
+	    push @m, "\t\$(FULLPERL) -e 'use Config; chdir q{\$(INST_EXE)}; "
+		    ."foreach (qw(@request_aliases)) {' \\\n";
+	    push @m, <<'EOT';
+	-e 'unlink "$$_";' \
+	-e 'system("$$Config{\"lns\"} lwp-request $$_") && die; }'
+EOT
+	}
+    }
+    join "", @m;
+}
+
+# What happens when we say 'make test'
+sub test
+{
+    q(
+TEST_VERBOSE=0
+
+test: all
+	$(FULLPERL) t/TEST $(TEST_VERBOSE)
+
+);
+}
+
+
+# Determine things that should *not* be installed
+sub libscan
+{
+    my($self, $path) = @_;
+    return '' if $path =~ m/.(pl|dtd|sgml)$/;
+    return '' if $path =~ m:\bCVS/:;
+    return '' if $path =~ m/~$/;
+    $path;
+}
+
+# Pass libwww-perl version number to pod2man
+sub manifypods
+{
+    my $self = shift;
+    my $ver = $self->{VERSION} || "";
+    local($_) = $self->SUPER::manifypods(@_);
+    s/pod2man\s*$/pod2man --release libwww-perl-$ver/m;
+    $_;
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/README
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/README	(revision 21112)
@@ -0,0 +1,124 @@
+                     L I B W W W - P E R L - 5
+                   -----------------------------
+
+Libwww-perl is a collection of Perl modules which provides a simple
+and consistent programming interface (API) to the World-Wide Web.  The
+main focus of the library is to provide classes and functions that
+allow you to write WWW clients, thus libwww-perl said to be a WWW
+client library. The library also contain modules that are of more
+general use and even classes that helps you implement simple HTTP
+servers.
+
+There are actually two versions of libwww-perl: one for Perl4, and one
+for Perl5. Both have a similar architecture, loosely based on the CERN
+Library of Common Code (nowadays known as 'w3c-libwww').
+
+The Perl4 version is still(?) maintained by Roy Fielding, and was the
+basis for tools such as MOMSpider. The perl4 version of libwww-perl
+and much more information about its libraries can be found at:
+http://www.ics.uci.edu/pub/websoft/libwww-perl/
+
+The Perl5 version (this package) is a complete rewrite for Perl5: the
+code is organized in Modules, provides an Object Oriented API, is
+updated for the latest HTTP and URL specification, and offers some
+extended functionality.
+
+Full HTTP/1.1 support is on its way.  Alpha releases of the LWPng
+package is currently available from CPAN.
+
+
+PREREQUISITES
+
+In order to install and use this package you will need Perl version
+5.004 or better.  Some modules within this package depend on other
+packages that are distributed separately from Perl.  We recommend that
+you have the following packages installed before you install
+libwww-perl:
+
+  MIME-Base64
+  HTML-Parser
+  libnet
+  MD5
+
+These packages should be available on CPAN.
+
+If you want to access sites using the https protocol, then you need to
+install the Crypt-SSLeay modules first.  (The Net::SSLeay module is not
+supported any more.)
+
+
+INSTALLATION
+
+You install libwww-perl, as you would install any perl module library,
+by running these commands:
+
+   perl Makefile.PL
+   make
+   make test
+   make install
+
+You can edit the configuration section of Makefile.PL to select which
+programs to install in addition to the library itself.  If you don't
+want to install any programs (only the library files) and don't want
+to mess with the Makefile.PL then pass the '-n' option to Makefile.PL:
+
+   perl Makefile.PL -n
+
+If you want to install a private copy of libwww-perl in your home
+directory, then you should try to produce the initial Makefile with
+something like this command:
+
+  perl Makefile.PL LIB=~/perl
+
+The Makefile.PL program will start out by checking your perl
+installation for a few packages that are recommended to be installed
+together with libwww-perl.
+
+Optional network tests are _not_ enabled automatically when you unpack
+this package.  Consult the t/README file for instructions on how to
+enable these tests.
+
+
+DOCUMENTATION
+
+See lib/LWP.pm for an overview of the library. See ChangeLog for
+recent changes.  POD style documentation is included in all modules
+and scripts.  These are normally converted to manual pages and
+installed as part of the "make install" process.  You should also be
+able to use the 'perldoc' utility to extract and read documentation
+from the module files directly.
+
+
+SUPPORT
+
+Question about how to use this library should be directed to the
+comp.lang.perl.modules USENET Newsgroup.  Bug reports and suggestions
+for improvements can be sendt to the <libwww-perl@ics.uci.edu> mailing
+list.  This mailing list is also the place for general discussions and
+development of the libwww-perl package.
+
+You can join the mailing list by sending a message to
+<libwww-perl-request@ics.uci.edu> with "Subscribe" as subject.
+            ^^^^^^^^
+
+AVAILABILITY
+
+The latest version of libwww-perl should always be available from:
+
+     http://www.linpro.no/lwp/
+
+The library is also available from the Comprehensive Perl Archive
+Network (CPAN).  Visit <URL:http://www.perl.com/CPAN/> to find a CPAN
+site near you.
+
+
+COPYRIGHT
+
+  © 1995-1998 Gisle Aas. All rights reserved.
+  © 1995 Martijn Koster. All rights reserved.
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself.
+
+
+Enjoy!
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/TODO
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/TODO	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/TODO	(revision 21112)
@@ -0,0 +1,46 @@
+                                            $Date$
+
+This is a list over things that we plan to implement in libwww-perl-5.
+The order of items does not reflect priority.
+
+  o  HTTP/1.1 support is on its way.  This also includes support of
+     multiple parallel connections.  Take a look at LWPng (aka
+     libwww-perl-6).
+
+  o  Provide initial credentials so that we don't always have to get a
+     401 response first.  Both for Basic and Digest authentication.  This
+     is in fact implemented for LWPng.
+
+  o  HTTP::Request::Common::POST() should be able to create large
+     file upload requests by making a closure content.
+
+  o  Support PICS??
+
+  o  Cache previous ftp connections.  "Keep-Alive"?
+
+  o  Support MOVE, DELETE method for ftp.
+
+  o  Support If-Modified-Since for ftp.
+
+  o  Use Mail::Send to implement LWP::Protocol::mailto.  Today we just
+     invoke sendmail directly, which is not very portable.
+
+  o  The lwp-request -o processing should check for Content-Encoding as
+     well as Content-Type.
+
+  o  The lwp-request script should optionally consult Mail::Cap for
+     handling of media types.
+
+  o  The lwp-request script should use the callback interface so that it
+     prints data as it is received.
+
+  o  The lwp-mirror script should be able to work recursively.
+
+  o  Include more useful programs
+
+  o  More documentation
+
+  o  More tests
+
+  o  Fix bugs, if there are any...
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-download.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-download.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-download.PL	(revision 21112)
@@ -0,0 +1,248 @@
+use Config;
+use File::Basename qw(basename dirname);
+chdir(dirname($0));
+($file = basename($0)) =~ s/\.PL$//;
+$file =~ s/\.pl$//
+	if ($Config{'osname'} eq 'VMS' or
+	    $Config{'osname'} eq 'OS2');  # "case-forgiving"
+open OUT,">$file" or die "Can't create $file: $!";
+chmod(0755, $file);
+print "Extracting $file (with variable substitutions)\n";
+
+print OUT <<"!GROK!THIS!";
+$Config{'startperl'} -w
+
+!GROK!THIS!
+
+print OUT <<'!NO!SUBS!';
+# $Id$
+
+=head1 NAME
+
+lwp-download - fetch large files from the net
+
+=head1 SYNOPSIS
+
+ lwp-download [-a] <url> [<local file>]
+
+=head1 DESCRIPTION
+
+The I<lwp-download> program will down load the document specified by the URL
+given as the first command line argument to a local file.  The local
+filename used to save the document is guessed from the URL unless
+specified as the second command line argument.
+
+The I<lwp-download> program is implemented using the I<libwww-perl>
+library.  It is better suited to down load big files than the
+I<lwp-request> program because it does not store the file in memory.
+Another benefit is that it will keep you updated about its progress
+and that you don't have much options to worry about.
+
+Use the C<-a> option to save the file in text (ascii) mode.  Might make a
+difference on dosish systems.
+
+=head1 EXAMPLE
+
+Fetch the newest and greatest perl version:
+
+ $ lwp-download http://www.perl.com/CPAN/src/latest.tar.gz
+ Saving to 'latest.tar.gz'...
+ 1.47 MB received in 22 seconds (68.7 KB/sec)
+
+=head1 AUTHOR
+
+Gisle Aas <gisle@aas.no>
+
+=cut
+
+use LWP::UserAgent;
+use LWP::MediaTypes;
+use URI::URL;
+use strict;
+
+my $progname = $0;
+$progname =~ s,.*/,,;    # only basename left in progname
+$progname =~ s/\.\w*$//; # strip extension if any
+
+#parse option
+use Getopt::Std;
+my %opts;
+unless (getopts('a', \%opts)) {
+    usage();
+}
+my $opt_a = $opts{a}; # save in binary mode
+
+my $url = url(shift || usage());
+my $argfile = shift;
+
+my $ua = new LWP::UserAgent;
+
+$ua->agent("lwp-download/0.1 " . $ua->agent);
+$ua->env_proxy;
+
+my $req = new HTTP::Request GET => $url;
+
+my $file;      # name of file we download into
+my $length;    # total number of bytes to download
+my $flength;   # formatted length
+my $size = 0;  # number of bytes received
+my $start_t;   # start time of download
+my $last_dur;  # time of last callback
+
+my $shown = 0; # have we called the show() function yet
+
+$SIG{INT} = sub { die "Interrupted\n"; };
+
+$| = 1;  # autoflush
+
+my $res = $ua->request($req,
+  sub {
+      unless($file) {
+	  my $res = $_[1];
+	  unless ($argfile) {
+	      # must find a suitable name to use.  First thing
+	      # to do is to look for the "Content-Disposition"
+	      # header defined by RFC1806.  This is also supported
+	      # by Netscape
+	      my $cd = $res->header("Content-Disposition");
+	      if ($cd && $cd =~ /\bfilename\s*=\s*(\S+)/) {
+		  $file = $1;
+		  $file =~ s/;$//;
+		  $file =~ s/^([\"\'])(.*)\1$/$2/;
+	      }
+		
+	      # if this fails we try to make something from the URL
+	      unless ($file) {
+		  my $req = $res->request;  # now always there
+		  my $rurl = $req ? $req->url : $url;
+		  
+		  $file = ($rurl->path_components)[-1];
+		  unless (length $file) {
+		      $file = "index";
+		      my $suffix = media_suffix($res->content_type);
+		      $file .= ".$suffix" if $suffix;
+		  } elsif ($rurl->scheme eq 'ftp' ||
+			   $file =~ /\.tgz$/      ||
+			   $file =~ /\.tar(\.(Z|gz))?$/
+			  ) {
+		      # leave the filename as it was
+		  } else {
+		      my $ct = guess_media_type($file);
+		      unless ($ct eq $res->content_type) {
+			  # need a better suffix for this type
+			  my $suffix = media_suffix($res->content_type);
+			  $file .= ".$suffix" if $suffix;
+		      }
+		  }
+	      }
+
+	      # Check if the file is already present
+	      if (-f $file && -t) {
+		  print "Overwrite $file? [y] ";
+		  my $ans = <STDIN>;
+		  exit if !defined($ans) || !($ans =~ /^y?\n/);
+	      } else {
+		  print "Saving to '$file'...\n";
+	      }
+	  } else {
+	      $file = $argfile;
+	  }
+	  open(FILE, ">$file") || die "Can't open $file: $!";
+          binmode FILE unless $opt_a;
+	  $length = $res->content_length;
+	  $flength = fbytes($length) if defined $length;
+	  $start_t = time;
+	  $last_dur = 0;
+      }
+      $size += length($_[0]);
+      print FILE $_[0];
+      if (defined $length) {
+	  my $dur  = time - $start_t;
+	  if ($dur != $last_dur) {  # don't update too often
+	      $last_dur = $dur;
+	      my $perc = $size / $length;
+	      my $speed;
+	      $speed = fbytes($size/$dur) . "/sec" if $dur > 3;
+	      my $secs_left = fduration($dur/$perc - $dur);
+	      $perc = int($perc*100);
+	      my $show = "$perc% of $flength";
+	      $show .= " (at $speed, $secs_left remaining)" if $speed;
+	      show($show);
+	  }
+      } else {
+	  show( fbytes($size) . " received");
+      }
+  }
+);
+
+if ($res->is_success || $res->message =~ /^Interrupted/) {
+    show("");  # clear text
+    print "\r";
+    print fbytes($size);
+    print " of ", fbytes($length) if defined($length) && $length != $size;
+    print " received";
+    my $dur = time - $start_t;
+    if ($dur) {
+	my $speed = fbytes($size/$dur) . "/sec";
+	print " in ", fduration($dur), " ($speed)";
+    }
+    print "\n";
+    my $died = $res->header("X-Died");
+    if ($died || !$res->is_success) {
+	if (-t) {
+	    print "Transfer aborted.  Delete $file? [n] ";
+	    my $ans = <STDIN>;
+	    unlink($file) if defined($ans) && $ans =~ /^y\n/;
+	} else {
+	    print "Transfer aborted, $file kept\n";
+	}
+    }
+} else {
+    print "\n" if $shown;
+    print "$progname: Can't download: ", $res->code, " ", $res->message, "\n";
+    exit 1;
+}
+
+
+sub fbytes
+{
+    my $n = int(shift);
+    if ($n >= 1024 * 1024) {
+	return sprintf "%.3g MB", $n / (1024.0 * 1024);
+    } elsif ($n >= 1024) {
+	return sprintf "%.3g KB", $n / 1024.0;
+    } else {
+	return "$n bytes";
+    }
+}
+
+sub fduration
+{
+    use integer;
+    my $secs = int(shift);
+    my $hours = $secs / (60*60);
+    $secs -= $hours * 60*60;
+    my $mins = $secs / 60;
+    $secs %= 60;
+    if ($hours) {
+	return "$hours hours $mins minutes";
+    } elsif ($mins >= 2) {
+	return "$mins minutes";
+    } else {
+	$secs += $mins * 60;
+	return "$secs seconds";
+    }
+}
+
+sub show
+{
+    my $mess = shift;
+    print "\r$mess", (" " x (75 - length $mess));
+    $shown++;
+}
+
+sub usage
+{
+    die "Usage: $progname [-a] <url> [<lpath>]\n";
+}
+!NO!SUBS!
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-mirror.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-mirror.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-mirror.PL	(revision 21112)
@@ -0,0 +1,120 @@
+use Config;
+use File::Basename qw(basename dirname);
+chdir(dirname($0));
+($file = basename($0)) =~ s/\.PL$//;
+$file =~ s/\.pl$//
+	if ($Config{'osname'} eq 'VMS' or
+	    $Config{'osname'} eq 'OS2');  # "case-forgiving"
+open OUT,">$file" or die "Can't create $file: $!";
+chmod(0755, $file);
+print "Extracting $file (with variable substitutions)\n";
+
+print OUT <<"!GROK!THIS!";
+$Config{'startperl'} -w
+
+!GROK!THIS!
+
+print OUT <<'!NO!SUBS!';
+# $Id$
+#
+# Simple mirror utility using LWP
+
+=head1 NAME
+
+lwp-mirror - Simple mirror utility for WWW
+
+=head1 SYNOPSIS
+
+ lwp-mirror [-v] [-t timeout] <url> <local file>
+
+=head1 DESCRIPTION
+
+This program can be used to mirror a document from a WWW server.  The
+document is only transfered if the remote copy is newer than the local
+copy.  If the local copy is newer nothing happens.
+
+Use the C<-v> option to print the version number of this program.
+
+The timeout value specified with the C<-t> option.  The timeout value
+is the time that the program will wait for response from the remote
+server before it fails.  The default unit for the timeout value is
+seconds.  You might append "m" or "h" to the timeout value to make it
+minutes or hours, repectively.
+
+Because this program is implemented using the LWP library, it only
+supports the protocols that LWP supports.
+
+=head1 SEE ALSO
+
+L<lwp-request>, L<LWP>
+
+=head1 AUTHOR
+
+Gisle Aas <aas@a.sn.no>
+
+=cut
+
+
+use LWP::Simple;
+use Getopt::Std;
+
+$progname = $0;
+$progname =~ s,.*/,,;  # use basename only
+$progname =~ s/\.\w*$//; #strip extension if any
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+$opt_h = undef;  # print usage
+$opt_v = undef;  # print version
+$opt_t = undef;  # timeout
+
+unless (getopts("hvt:")) {
+    usage();
+}
+
+if ($opt_v) {
+    require LWP;
+    my $DISTNAME = 'libwww-perl-' . LWP::Version();
+    die <<"EOT";
+This is lwp-mirror version $VERSION ($DISTNAME)
+
+Copyright 1995-1996, Gisle Aas.
+
+This program is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+EOT
+}
+
+$url  = shift or usage();
+$file = shift or usage();
+usage() if $opt_h or @ARGV;
+
+if (defined $opt_t) {
+    $opt_t =~ /^(\d+)([smh])?/;
+    die "$progname: Illegal timeout value!\n" unless defined $1;
+    $timeout = $1;
+    $timeout *= 60   if ($2 eq "m");
+    $timeout *= 3600 if ($2 eq "h");
+    $LWP::Simple::ua->timeout($timeout);
+}
+
+$rc = mirror($url, $file);
+
+if ($rc == 304) {
+    print STDERR "$progname: $file is up to date\n"
+} elsif (!is_success($rc)) {
+    print STDERR "$progname: $rc ", status_message($rc), "   ($url)\n";
+    exit 1;
+}
+exit;
+
+
+sub usage
+{
+    die <<"EOT";
+Usage: $progname [-options] <url> <file>
+    -v           print version number of program
+    -t <timeout> Set timeout value
+EOT
+}
+!NO!SUBS!
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-request.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-request.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-request.PL	(revision 21112)
@@ -0,0 +1,521 @@
+use Config;
+use File::Basename qw(basename dirname);
+chdir(dirname($0));
+($file = basename($0)) =~ s/\.PL$//;
+$file =~ s/\.pl$//
+	if ($Config{'osname'} eq 'VMS' or
+	    $Config{'osname'} eq 'OS2');  # "case-forgiving"
+open OUT,">$file" or die "Can't create $file: $!";
+chmod(0755, $file);
+print "Extracting $file (with variable substitutions)\n";
+
+print OUT <<"!GROK!THIS!";
+$Config{'startperl'} -w
+
+!GROK!THIS!
+
+print OUT <<'!NO!SUBS!';
+# $Id$
+#
+# Simple user agent using LWP library.
+
+=head1 NAME
+
+lwp-request, GET, HEAD, POST - Simple WWW user agent
+
+=head1 SYNOPSIS
+
+ lwp-request [-aeEdvhx] [-m method] [-b <base URL>] [-t <timeout>]
+             [-i <if-modified-since>] [-c <content-type>] [-C <credentials>]
+             [-p <proxy-url>] [-o <format>] <url>...
+
+=head1 DESCRIPTION
+
+This program can be used to send requests to WWW servers and your
+local file system. The request content for POST, PUT and CHECKIN
+methods is read from stdin.  The content of the response is printed on
+stdout.  Error messages are printed on stderr.  The program returns a
+status value indicating the number of URLs that failed.
+
+The options are:
+
+=over 4
+
+=item -m <method>
+
+Set which method to use for the request.  If this option is not used,
+then the method is derived from the name of the program.
+
+=item -f
+
+Force request through, even if the program believes that the method is
+illegal.  The server will probably reject the request.
+
+=item -b <url>
+
+This URL will be used as the base URL for the URLs that the method is
+applied to.  The base URL only takes effect for relative URLs.  If you
+do not provide this option and the URLs are relative, then they will
+be treated as files in the local file system.
+
+=item -t <timeout>
+
+Set the timeout value for the requests.  The timeout is the amount of
+time that the program will wait for a response from the remote server
+before it fails.  The default unit for the timeout value is seconds.
+You might append "m" or "h" to the timeout value to make it minutes or
+hours, respectively.  The default timeout is '3m', i.e. 3 minutes.
+
+=item -i <time>
+
+Set the If-Modified-Since header in the request. If I<time> it the
+name of a file, use the modification timestamp for this file. If
+I<time> is not a file, it is parsed as a literal date. Take a look at
+L<HTTP::Date> for recogniced formats.
+
+=item -c <content-type>
+
+Set the Content-Type for the request.  This option is only allowed for
+requests that take a content, i.e. POST, PUT and CHECKIN.  You can
+force methods to take content by using the C<-f> option together with
+C<-c>.  The default Content-Type for POST is
+C<application/x-www-form-urlencoded>.  The default Content-type for
+the others is C<text/plain>.
+
+=item -p <proxy-url>
+
+Set the proxy to be used for the requests.  The program also loads
+proxy settings from the environment.  You can disable this with the
+C<-P> option.
+
+=item -C <username>:<password>
+
+Provide credentials for documents that are protected by Basic
+Authentication.  If the document is protected and you did not specify
+the username and password with this option, then you will be prompted
+to provide these values.
+
+=back
+
+The following options controls what is displayed by the program:
+
+=over 4
+
+=item -u
+
+Print request method and absolute URL as requests are made.
+
+=item -U
+
+Print request headers in addition to request method and absolute URL.
+
+=item -s
+
+Print response status code.  This option is always on for HEAD requests.
+
+=item -S
+
+Print response status chain. This shows redirect and autorization
+requests that are handled by the library.
+
+=item -e
+
+Print response headers.  This option is always on for HEAD requests.
+
+=item -d
+
+Do B<not> print the content of the response.
+
+=item -o <format>
+
+Process HTML content in various ways before printing it.  If the
+content type of the response is not HTML, then this option has no
+effect.  The legal format values are; I<text>, I<ps>, I<links>,
+I<html> and I<dump>.
+
+If you specify the I<text> format then the HTML will be formatted as
+plain latin1 text.  If you specify the I<ps> format then it will be
+formatted as Postscript.
+
+The I<links> format will output all links found in the HTML document.
+Relative links will be expanded to absolute ones.
+
+The I<html> format will reformat the HTML code and the I<dump> format
+will just dump the HTML syntax tree.
+
+=item -v
+
+Print the version number of the program and quit.
+
+=item -h
+
+Print usage message and quit.
+
+=item -x
+
+Extra debugging output.
+
+=item -a
+
+Set text(ascii) mode for content input and output.  If this option is not 
+used, content input and output is done in binary mode.
+
+=back
+
+Because this program is implemented using the LWP library, it will
+only support the protocols that LWP supports.
+
+=head1 SEE ALSO
+
+L<lwp-mirror>, L<LWP>
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 AUTHOR
+
+Gisle Aas <aas@sn.no>
+
+=cut
+
+$progname = $0;
+$progname =~ s,.*/,,;  # use basename only
+$progname =~ s/\.\w*$//; # strip extension, if any
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+
+require LWP;
+require LWP::Debug;
+require URI::URL;
+
+use URI::Heuristic qw(uf_url);
+
+use HTTP::Status qw(status_message);
+use HTTP::Date qw(time2str str2time);
+
+
+# This table lists the methods that are allowed.  It should really be
+# a superset for all methods supported for every scheme that may be
+# supported by the library.  Currently it might be a bit too HTTP
+# specific.  You might use the -f option to force a method through.
+#
+# "" = No content in request, "C" = Needs content in request
+#
+%allowed_methods = (
+    GET        => "",
+    HEAD       => "",
+    POST       => "C",
+    PUT        => "C",
+    DELETE     => "",
+    TRACE      => "",
+    OPTIONS    => "",
+);
+
+
+# We make our own specialization of LWP::UserAgent that asks for
+# user/password if document is protected.
+{
+    package RequestAgent;
+    @ISA = qw(LWP::UserAgent);
+
+    sub new
+    { 
+	my $self = LWP::UserAgent::new(@_);
+	$self->agent("lwp-request/$main::VERSION");
+	$self;
+    }
+
+    sub get_basic_credentials
+    {
+	my($self, $realm, $uri) = @_;
+	if ($main::opt_C) {
+	    return split(':', $main::opt_C, 2);
+	} elsif (-t) {
+	    my $netloc = $uri->netloc;
+	    print "Enter username for $realm at $netloc: ";
+	    my $user = <STDIN>;
+	    chomp($user);
+	    return (undef, undef) unless length $user;
+	    print "Password: ";
+	    system("stty -echo");
+	    my $password = <STDIN>;
+	    system("stty echo");
+	    print "\n";  # because we disabled echo
+	    chomp($password);
+	    return ($user, $password);
+	} else {
+	    return (undef, undef)
+	}
+    }
+}
+
+$method = uc(lc($progname) eq "lwp-request" ? "GET" : $progname);
+
+# Parse command line
+use Getopt::Std;
+
+$opt_a = undef;  # content i/o in text(ascii) mode
+$opt_m = undef;  # set method
+$opt_f = undef;  # make request even if method is not in %allowed_methods
+$opt_b = undef;  # base url
+$opt_t = undef;  # timeout
+$opt_i = undef;  # if-modified-since
+$opt_c = undef;  # content type for POST
+$opt_C = undef;  # credidentials for basic authorization
+
+$opt_u = undef;  # display method, URL and headers of request
+$opt_U = undef;  # display request headers also
+$opt_s = undef;  # display status code
+$opt_S = undef;  # display whole chain of status codes
+$opt_e = undef;  # display response headers (default for HEAD)
+$opt_d = undef;  # don't display content
+
+$opt_h = undef;  # print usage
+$opt_v = undef;  # print version
+
+$opt_x = undef;  # extra debugging info
+$opt_p = undef;  # proxy URL
+$opt_P = undef;  # don't load proxy setting from environment
+
+$opt_o = undef;  # output format
+
+unless (getopts("axhvuUsSedPp:b:t:i:c:C:m:fo:")) {
+    usage();
+}
+
+if ($opt_v) {
+    require LWP;
+    my $DISTNAME = 'libwww-perl-' . LWP::Version();
+    die <<"EOT";
+This is lwp-request version $VERSION ($DISTNAME)
+
+Copyright 1995-1997, Gisle Aas.
+
+This program is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+EOT
+}
+
+usage() if $opt_h || !@ARGV;
+
+LWP::Debug::level('+') if $opt_x;
+
+# Create the user agent object
+$ua = new RequestAgent;
+
+# Load proxy settings from *_proxy environment variables.
+$ua->env_proxy unless $opt_P;
+
+$method = uc($opt_m) if defined $opt_m;
+
+if ($opt_f) {
+    if ($opt_c) {
+        $allowed_methods{$method} = "C";  # force content
+    } else {
+        $allowed_methods{$method} = "";
+    }
+} elsif (!defined $allowed_methods{$method}) {
+    die "$progname: $method is not an allowed method\n";
+}
+
+if ($method eq "HEAD") {
+    $opt_s = 1;
+    $opt_e = 1 unless $opt_d;
+    $opt_d = 1;
+}
+
+if (defined $opt_t) {
+    $opt_t =~ /^(\d+)([smh])?/;
+    die "$progname: Illegal timeout value!\n" unless defined $1;
+    $timeout = $1;
+    $timeout *= 60   if ($2 eq "m");
+    $timeout *= 3600 if ($2 eq "h");
+    $ua->timeout($timeout);
+}
+
+if (defined $opt_i) {
+    if (-e $opt_i) {
+        $time = (stat _)[9];
+    } else {
+        $time = str2time($opt_i);
+        die "$progname: Illegal time syntax for -i option\n"
+            unless defined $time;
+    }
+    $opt_i = time2str($time);
+}
+
+$content = undef;
+if ($allowed_methods{$method} eq "C") {
+    # This request needs some content
+    unless (defined $opt_c) {
+        # set default content type
+        $opt_c = ($method eq "POST") ?
+              "application/x-www-form-urlencoded"
+            : "text/plain";
+    } else {
+        die "$progname: Illegal Content-type format\n"
+            unless $opt_c =~ m,^[\w\-]+/[\w\-]+(?:\s*;.*)?$,
+    }
+    print "Please enter content ($opt_c) to be ${method}ed:\n"
+        if -t;
+    binmode STDIN unless -t or $opt_a;
+    $content = join("", <STDIN>);
+} else {
+    die "$progname: Can't set Content-type for $method requests\n"
+        if defined $opt_c;
+}
+
+# Set up a request.  We will use the same request object for all URLs.
+$request = new HTTP::Request $method;
+$request->header('If-Modified-Since', $opt_i) if defined $opt_i;
+#$request->header('Accept', '*/*');
+if ($opt_c) { # will always be set for request that wants content
+    $request->header('Content-Type', $opt_c);
+    $request->header('Content-Length', length $content);  # Not really needed
+    $request->content($content);
+}
+
+
+$errors = 0;
+
+# Ok, now we perform the requests, one URL at a time
+while ($url = shift) {
+    # Create the URL object, but protect us against bad URLs
+    eval {
+	if ($url =~ /^\w+:/ || $opt_b) {  # is there any scheme specification
+	    $url = URI::URL->new($url, $opt_b);
+	} else {
+	    $url = uf_url($url);
+        }
+    };
+    if ($@) {
+	$@ =~ s/at\s+\S+\s+line\s\d+//;
+	print STDERR $@;
+	$errors++;
+	next;
+    }
+
+    $ua->proxy($url->scheme, $opt_p) if $opt_p;
+
+    # Send the request and get a response back from the server
+    $request->url($url);
+    $response = $ua->request($request);
+
+    if ($opt_u || $opt_U) {
+        my $url = $response->request->url->as_string;
+        print "$method $url\n";
+        print $response->request->headers_as_string, "\n" if $opt_U;
+    }
+
+    if ($opt_S) {
+	printResponseChain($response);
+    } elsif ($opt_s) {
+        print $response->status_line, "\n";
+    }
+
+    if ($opt_e) {
+        # Display headers
+        print $response->headers_as_string;
+        print "\n";  # separate headers and content
+    }
+
+    if ($response->is_success) {
+	unless ($opt_d) {
+	    if ($opt_o &&
+		$response->content_type eq 'text/html') {
+		require HTML::Parse;
+		my $html = HTML::Parse::parse_html($response->content);
+		{
+		    $opt_o eq 'ps' && do {
+			require HTML::FormatPS;
+			my $f = new HTML::FormatPS;
+			print $f->format($html);
+			last;
+		    };
+		    $opt_o eq 'text' && do {
+			require HTML::FormatText;
+			my $f = new HTML::FormatText;
+			print $f->format($html);
+			last;
+		    };
+		    $opt_o eq 'html' && do {
+			print $html->as_HTML;
+			last;
+		    };
+		    $opt_o eq 'links' && do {
+			my $base = $response->base;
+			for ( @{ $html->extract_links } ) {
+			    my($link, $elem) = @$_;
+			    my $tag = uc $elem->tag;
+			    $link = new URI::URL $link, $base;
+			    print "$tag\t", $link->abs->as_string, "\n";
+			}
+			last;
+		    };
+		    $opt_o eq 'dump' && do {
+			$html->dump;
+			last;
+		    };
+		    # It is bad to not notice this before now :-(
+		    die "Illegal -o option value ($opt_o)\n";
+		}
+	    } else {
+                binmode STDOUT unless $opt_a;
+		print $response->content;
+	    }
+	}
+    } else {
+        print STDERR $response->error_as_HTML unless $opt_d;
+        $errors++;
+    }
+}
+
+exit $errors;
+
+
+sub printResponseChain
+{
+    my($response) = @_;
+    return unless defined $response;
+    printResponseChain($response->previous);
+    my $method = $response->request->method;
+    my $url = $response->request->url->as_string;
+    my $code = $response->code;
+    print "$method $url --> ", $response->status_line, "\n";
+}
+
+
+sub usage
+{
+    die <<"EOT";
+Usage: $progname [-options] <url>...
+    -m <method>   use method for the request (default is '$method')
+    -f            make request even if $progname believes method is illegal
+    -b <base>     Use the specified URL as base
+    -t <timeout>  Set timeout value
+    -i <time>     Set the If-Modified-Since header on the request
+    -c <conttype> use this content-type for POST, PUT, CHECKIN
+    -a            Use text mode for content I/O
+    -p <proxyurl> use this as a proxy
+    -P            don't load proxy settings from environment
+
+    -u            Display method and URL before any response
+    -U            Display request headers (implies -u)
+    -s            Display response status code
+    -S            Display response status chain
+    -e            Display response headers
+    -d            Do not display content
+    -o <format>   Process HTML content in various ways
+
+    -v            Show program version
+    -h            Print this message
+
+    -x            Extra debugging output
+EOT
+}
+!NO!SUBS!
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-rget.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-rget.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/bin/lwp-rget.PL	(revision 21112)
@@ -0,0 +1,600 @@
+use Config;
+use File::Basename qw(basename dirname);
+chdir(dirname($0));
+($file = basename($0)) =~ s/\.PL$//;
+$file =~ s/\.pl$//
+	if ($Config{'osname'} eq 'VMS' or
+	    $Config{'osname'} eq 'OS2');  # "case-forgiving"
+open OUT,">$file" or die "Can't create $file: $!";
+chmod(0755, $file);
+print "Extracting $file (with variable substitutions)\n";
+
+print OUT <<"!GROK!THIS!";
+$Config{'startperl'} -w
+
+!GROK!THIS!
+
+print OUT <<'!NO!SUBS!';
+#line 18
+
+=head1 NAME
+
+lwp-rget - Retrieve WWW documents recursively
+
+=head1 SYNOPSIS
+
+ lwp-rget [--verbose] [--auth=USER:PASS] [--depth=N] [--hier] [--iis]
+	  [--keepext=mime/type[,mime/type]] [--limit=N] [--nospace]
+	  [--prefix=URL] [--sleep=N] [--tolower] <URL>
+ lwp-rget --version
+
+=head1 DESCRIPTION
+
+This program will retrieve a document and store it in a local file.  It
+will follow any links found in the document and store these documents
+as well, patching links so that they refer to these local copies.
+This process continues until there are no more unvisited links or the
+process is stopped by the one or more of the limits which can be
+controlled by the command line arguments.
+
+This program is useful if you want to make a local copy of a
+collection of documents or want to do web reading off-line.
+
+All documents are stored as plain files in the current directory. The
+file names chosen are derived from the last component of URL paths.
+
+The options are:
+
+=over 3
+
+=item --auth=USER:PASS<n>
+
+Set the authentication credentials to user "USER" and password "PASS" if
+any restricted parts of the web site are hit.  If there are restricted
+parts of the web site and authentication credentials are not available,
+those pages will not be downloaded.
+
+=item --depth=I<n>
+
+Limit the recursive level. Embedded images are always loaded, even if
+they fall outside the I<--depth>. This means that one can use
+I<--depth=0> in order to fetch a single document together with all
+inline graphics.
+
+The default depth is 5.
+
+=item --hier
+
+Download files into a hierarchy that mimics the web site structure.
+The default is to put all files in the current directory.
+
+=item --iis
+
+Sends an "Accept: */*" on all URL requests as a workaround for a bug in
+IIS 2.0.  If no Accept MIME header is present, IIS 2.0 returns with a
+"406 No acceptable objects were found" error.  Also converts any back
+slashes (\\) in URLs to forward slashes (/).
+
+=item --keepext=I<mime/type[,mime/type]>
+
+Keeps the current extension for the list MIME types.  Useful when
+downloading text/plain documents that shouldn't all be translated to
+*.txt files.
+
+=item --limit=I<n>
+
+Limit the number of documents to get.  The default limit is 50.
+
+=item --nospace
+
+Changes spaces in all URLs to underscore characters (_).  Useful when
+downloading files from sites serving URLs with spaces in them.	Does not
+remove spaces from fragments, e.g., "file.html#somewhere in here".
+
+=item --prefix=I<url_prefix>
+
+Limit the links to follow. Only URLs that start the prefix string are
+followed.
+
+The default prefix is set as the "directory" of the initial URL to
+follow.	 For instance if we start lwp-rget with the URL
+C<http://www.sn.no/foo/bar.html>, then prefix will be set to
+C<http://www.sn.no/foo/>.
+
+Use C<--prefix=''> if you don't want the fetching to be limited by any
+prefix.
+
+=item --sleep=I<n>
+
+Sleep I<n> seconds before retrieving each document. This options allows
+you to go slowly, not loading the server you visiting too much.
+
+=item --tolower
+
+Translates all links to lowercase.  Useful when downloading files from
+IIS since it does not serve files in a case sensitive manner.
+
+=item --verbose
+
+Make more noise while running.
+
+=item --quiet
+
+Don't make any noise.
+
+=item --version
+
+Print program version number and quit.
+
+=item --help
+
+Print the usage message and quit.
+
+=back
+
+Before the program exits the name of the file, where the initial URL
+is stored, is printed on stdout.  All used filenames are also printed
+on stderr as they are loaded.  This printing can be suppressed with
+the I<--quiet> option.
+
+=head1 SEE ALSO
+
+L<lwp-request>, L<LWP>
+
+=head1 AUTHOR
+
+Gisle Aas <aas@sn.no>
+
+=cut
+
+use strict;
+
+use Getopt::Long    qw(GetOptions);
+use URI::URL	    qw(url);
+use LWP::MediaTypes qw(media_suffix);
+
+use vars qw($VERSION);
+use vars qw($MAX_DEPTH $MAX_DOCS $PREFIX $VERBOSE $QUIET $SLEEP $HIER $AUTH $IIS $TOLOWER $NOSPACE %KEEPEXT);
+
+my $progname = $0;
+$progname =~ s|.*/||;  # only basename left
+$progname =~ s/\.\w*$//; #strip extension if any
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+#$Getopt::Long::debug = 1;
+#$Getopt::Long::ignorecase = 0;
+
+# Defaults
+$MAX_DEPTH = 5;
+$MAX_DOCS  = 50;
+
+GetOptions('version'  => \&print_version,
+	   'help'     => \&usage,
+	   'depth=i'  => \$MAX_DEPTH,
+	   'limit=i'  => \$MAX_DOCS,
+	   'verbose!' => \$VERBOSE,
+	   'quiet!'   => \$QUIET,
+	   'sleep=i'  => \$SLEEP,
+	   'prefix:s' => \$PREFIX,
+	   'hier'     => \$HIER,
+	   'auth=s'   => \$AUTH,
+	   'iis'      => \$IIS,
+	   'tolower'  => \$TOLOWER,
+	   'nospace'  => \$NOSPACE,
+	   'keepext=s' => \$KEEPEXT{'OPT'},
+	  ) || usage();
+
+sub print_version {
+    require LWP;
+    my $DISTNAME = 'libwww-perl-' . LWP::Version();
+    print <<"EOT";
+This is lwp-rget version $VERSION ($DISTNAME)
+
+Copyright 1996-1998, Gisle Aas.
+
+This program is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+EOT
+    exit 0;
+}
+
+my $start_url = shift || usage();
+usage() if @ARGV;
+
+require LWP::UserAgent;
+my $ua = new LWP::UserAgent;
+$ua->agent("$progname/$VERSION " . $ua->agent);
+$ua->env_proxy;
+
+unless (defined $PREFIX) {
+    $PREFIX = url($start_url);	 # limit to URLs below this one
+    eval {
+	$PREFIX->eparams(undef);
+	$PREFIX->equery(undef);
+    };
+
+    $_ = $PREFIX->epath;
+    s|[^/]+$||;
+    $PREFIX->epath($_);
+    $PREFIX = $PREFIX->as_string;
+}
+
+%KEEPEXT = map { lc($_) => 1 } split(/\s*,\s*/, $KEEPEXT{'OPT'});
+
+print <<"" if $VERBOSE;
+START	  = $start_url
+MAX_DEPTH = $MAX_DEPTH
+MAX_DOCS  = $MAX_DOCS
+PREFIX	  = $PREFIX
+
+
+my $no_docs = 0;
+my %seen = ();	   # mapping from URL => local_file
+
+my $filename = fetch($start_url);
+print "$filename\n" unless $QUIET;
+
+sub fetch
+{
+    my($url, $type, $depth) = @_;
+
+    # Fix http://sitename.com/../blah/blah.html to
+    #	  http://sitename.com/blah/blah.html
+    $url = $url->as_string if (ref($url));
+    while ($url =~ s#(https?://[^/]+/)\.\.\/#$1#) {}
+
+    # Fix backslashes (\) in URL if $IIS defined
+    $url = fix_backslashes($url) if (defined $IIS);
+
+    $url = url($url) unless ref($url);
+    $type  ||= 'a';
+    # Might be the background attribute
+    $type = 'img' if ($type eq 'body' || $type eq 'td');
+    $depth ||= 0;
+
+    # Print the URL before we start checking...
+    my $out = (" " x $depth) . $url . " ";
+    $out .= "." x (60 - length($out));
+    print STDERR $out . " " if $VERBOSE;
+
+    # Can't get mailto things
+    if ($url->scheme eq 'mailto') {
+	print STDERR "*skipping mailto*\n" if $VERBOSE;
+	return $url->as_string;
+    }
+
+    # The $plain_url is a URL without the fragment part
+    my $plain_url = $url->clone;
+    $plain_url->frag(undef);
+
+    # Check PREFIX, but not for <IMG ...> links
+    if ($type ne 'img' and  $url->as_string !~ /^\Q$PREFIX/o) {
+	print STDERR "*outsider*\n" if $VERBOSE;
+	return $url->as_string;
+    }
+
+    # Translate URL to lowercase if $TOLOWER defined
+    $plain_url = to_lower($plain_url) if (defined $TOLOWER);
+
+    # If we already have it, then there is nothing to be done
+    my $seen = $seen{$plain_url->as_string};
+    if ($seen) {
+	my $frag = $url->frag;
+	$seen .= "#$frag" if defined($frag);
+	$seen = protect_frag_spaces($seen);
+	print STDERR "$seen (again)\n" if $VERBOSE;
+	return $seen;
+    }
+
+    # Too much or too deep
+    if ($depth > $MAX_DEPTH and $type ne 'img') {
+	print STDERR "*too deep*\n" if $VERBOSE;
+	return $url;
+    }
+    if ($no_docs > $MAX_DOCS) {
+	print STDERR "*too many*\n" if $VERBOSE;
+	return $url;
+    }
+
+    # Fetch document 
+    $no_docs++;
+    sleep($SLEEP) if $SLEEP;
+    my $req = HTTP::Request->new(GET => $url);
+    # See: http://ftp.sunet.se/pub/NT/mirror-microsoft/kb/Q163/7/74.TXT
+    $req->header ('Accept', '*/*') if (defined $IIS);  # GIF/JPG from IIS 2.0
+    $req->authorization_basic(split (/:/, $AUTH)) if (defined $AUTH);
+    my $res = $ua->request($req);
+
+    # Check outcome
+    if ($res->is_success) {
+	my $doc = $res->content;
+	my $ct = $res->content_type;
+	my $name = find_name($res->request->url, $ct);
+	print STDERR "$name\n" unless $QUIET;
+	$seen{$plain_url->as_string} = $name;
+
+	# If the file is HTML, then we look for internal links
+	if ($ct eq "text/html") {
+	    # Save an unprosessed version of the HTML document.	 This
+	    # both reserves the name used, and it also ensures that we
+	    # don't loose everything if this program is killed before
+	    # we finish.
+	    save($name, $doc);
+	    my $base = $res->base;
+
+	    # Follow and substitute links...
+	    $doc =~
+s/
+  (
+    <(img|a|body|area|frame|td)\b   # some interesting tag
+    [^>]+			    # still inside tag (not strictly correct)
+    \b(?:src|href|background)	    # some link attribute
+    \s*=\s*			    # =
+  )
+    (?:				    # scope of OR-ing
+	 (")([^"]*)"	|	    # value in double quotes  OR
+	 (')([^']*)'	|	    # value in single quotes  OR
+	    ([^\s>]+)		    # quoteless value
+    )
+/
+  new_link($1, lc($2), $3||$5, $4||$6||$7, $base, $name, $depth+1)
+/giex;
+	   # XXX
+	   # The regular expression above is not strictly correct.
+	   # It is not really possible to parse HTML with a single
+	   # regular expression, but it is faster.  Tags that might
+	   # confuse us include:
+	   #	<a alt="href" href=link.html>
+	   #	<a alt=">" href="link.html">
+	   #
+	}
+	save($name, $doc);
+	return $name;
+    } else {
+	print STDERR $res->code . " " . $res->message . "\n" if $VERBOSE;
+	$seen{$plain_url->as_string} = $url->as_string;
+	return $url->as_string;
+    }
+}
+
+sub new_link
+{
+    my($pre, $type, $quote, $url, $base, $localbase, $depth) = @_;
+
+    $url = protect_frag_spaces($url);
+
+    $url = fetch(url($url, $base)->abs, $type, $depth);
+    $url = url("file:$url", "file:$localbase")->rel
+	unless $url =~ /^[.+\-\w]+:/;
+
+    $url = unprotect_frag_spaces($url);
+
+    return $pre . $quote . $url . $quote;
+}
+
+
+sub protect_frag_spaces
+{
+    my ($url) = @_;
+
+    $url = $url->as_string if (ref($url));
+
+    if ($url =~ m/^([^#]*#)(.+)$/)
+    {
+      my ($base, $frag) = ($1, $2);
+      $frag =~ s/ /%20/g;
+      $url = $base . $frag;
+    }
+
+    return $url;
+}
+
+
+sub unprotect_frag_spaces
+{
+    my ($url) = @_;
+
+    $url = $url->as_string if (ref($url));
+
+    if ($url =~ m/^([^#]*#)(.+)$/)
+    {
+      my ($base, $frag) = ($1, $2);
+      $frag =~ s/%20/ /g;
+      $url = $base . $frag;
+    }
+
+    return $url;
+}
+
+
+sub fix_backslashes
+{
+    my ($url) = @_;
+    my ($base, $frag);
+
+    $url = $url->as_string if (ref($url));
+
+    if ($url =~ m/([^#]+)(#.*)/)
+    {
+      ($base, $frag) = ($1, $2);
+    }
+    else
+    {
+      $base = $url;
+      $frag = "";
+    }
+
+    $base =~ tr/\\/\//;
+    $base =~ s/%5[cC]/\//g;	# URL-encoded back slash is %5C
+
+    return $base . $frag;
+}
+
+
+sub to_lower
+{
+    my ($url) = @_;
+    my $was_object = 0;
+
+    if (ref($url))
+    {
+      $url = $url->as_string;
+      $was_object = 1;
+    }
+
+    if ($url =~ m/([^#]+)(#.*)/)
+    {
+      $url = lc($1) . $2;
+    }
+    else
+    {
+      $url = lc($url);
+    }
+
+    if ($was_object == 1)
+    {
+      return url($url);
+    }
+    else
+    {
+      return $url;
+    }
+}
+
+
+sub translate_spaces
+{
+    my ($url) = @_;
+    my ($base, $frag);
+
+    $url = $url->as_string if (ref($url));
+
+    if ($url =~ m/([^#]+)(#.*)/)
+    {
+      ($base, $frag) = ($1, $2);
+    }
+    else
+    {
+      $base = $url;
+      $frag = "";
+    }
+
+    $base =~ s/^ *//;	# Remove initial spaces from base
+    $base =~ s/ *$//;	# Remove trailing spaces from base
+
+    $base =~ tr/ /_/;
+    $base =~ s/%20/_/g; # URL-encoded space is %20
+
+    return $base . $frag;
+}
+
+
+sub mkdirp
+{
+    my($directory, $mode) = @_;
+    my @dirs = split(/\//, $directory);
+    my $path = shift(@dirs);   # build it as we go
+    my $result = 1;   # assume it will work
+
+    unless (-d $path) {
+	$result &&= mkdir($path, $mode);
+    }
+
+    foreach (@dirs) {
+	$path .= "/$_";
+	if ( ! -d $path) {
+	    $result &&= mkdir($path, $mode);
+	}
+    }
+
+    return $result;
+}
+
+
+sub find_name
+{
+    my($url, $type) = @_;
+    #print "find_name($url, $type)\n";
+
+    # Translate spaces in URL to underscores (_) if $NOSPACE defined
+    $url = translate_spaces($url) if (defined $NOSPACE);
+
+    # Translate URL to lowercase if $TOLOWER defined
+    $url = to_lower($url) if (defined $TOLOWER);
+
+    $url = url($url) unless ref($url);
+
+    my $path = $url->path;
+
+    # trim path until only the basename is left
+    $path =~ s|(.*/)||;
+    my $dirname = ".$1";
+    if (!$HIER) {
+	$dirname = "";
+    } elsif (! -d $dirname) {
+	mkdirp($dirname, 0775);
+    }
+
+    my $extra = "";  # something to make the name unique
+    my $suffix;
+
+    if ($KEEPEXT{lc($type)}) {
+        $suffix = ($path =~ m/\.(.*)/) ? $1 : "";
+    } else {
+        $suffix = media_suffix($type);
+    }
+
+    $path =~ s|\..*||;	# trim suffix
+    $path = "index" unless length $path;
+
+    while (1) {
+	# Construct a new file name
+	my $file = $dirname . $path . $extra;
+	$file .= ".$suffix" if $suffix;
+	# Check if it is unique
+	return $file unless -f $file;
+
+	# Try something extra
+	unless ($extra) {
+	    $extra = "001";
+	    next;
+	}
+	$extra++;
+    }
+}
+
+
+sub save
+{
+    my $name = shift;
+    #print "save($name,...)\n";
+    open(FILE, ">$name") || die "Can't save $name: $!";
+    binmode FILE;
+    print FILE $_[0];
+    close(FILE);
+}
+
+
+sub usage
+{
+    die <<"";
+Usage: $progname [options] <URL>
+Allowed options are:
+  --auth=USER:PASS  Set authentication credentials for web site
+  --depth=N	    Maximum depth to traverse (default is $MAX_DEPTH)
+  --hier	    Download into hierarchy (not all files into cwd)
+  --iis		    Workaround IIS 2.0 bug by sending "Accept: */*" MIME
+		    header; translates backslashes (\\) to forward slashes (/)
+  --keepext=type    Keep file extension for MIME types (comma-separated list)
+  --limit=N	    A limit on the number documents to get (default is $MAX_DOCS)
+  --nospace	    Translate spaces URLs (not #fragments) to underscores (_)
+  --version	    Print version number and quit
+  --verbose	    More output
+  --quiet	    No output
+  --sleep=SECS	    Sleep between gets, ie. go slowly
+  --prefix=PREFIX   Limit URLs to follow to those which begin with PREFIX
+  --tolower	    Translate all URLs to lowercase (useful with IIS servers)
+
+}
+!NO!SUBS!
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/install.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/install.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/install.sh	(revision 21112)
@@ -0,0 +1,10 @@
+#!/bin/sh -v
+
+export PERL5LIB=$GSDLHOME/perllib/cpan/lib:$PERL5LIB
+export PATH=$GSDLHOME/perllib/cpan/bin:$PATH
+
+perl Makefile.PL PREFIX=$GSDLHOME/perllib/cpan
+
+make
+make test
+make install
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/Bundle/LWP.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/Bundle/LWP.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/Bundle/LWP.pm	(revision 21112)
@@ -0,0 +1,38 @@
+package Bundle::LWP;
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+1;
+
+__END__
+
+=head1 NAME
+
+Bundle::LWP - A bundle to install all libwww-perl related modules
+
+=head1 SYNOPSIS
+
+ perl -MCPAN -e 'install Bundle::LWP'
+
+=head1 CONTENTS
+
+Net::FTP 2.00      - If you want ftp://-support
+
+MIME::Base64       - Used in authentication headers
+
+MD5                - Needed to do Digest authentication
+
+HTML::HeadParser   - To get the correct $res->base
+
+LWP                - The reason why you need the modules above
+
+=head1 DESCRIPTION
+
+This bundle defines all reqreq modules for libwww-perl.
+
+
+=head1 AUTHOR
+
+Gisle Aas E<lt>aas@sn.no>
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/File/Listing.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/File/Listing.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/File/Listing.pm	(revision 21112)
@@ -0,0 +1,314 @@
+#
+# $Id$
+
+package File::Listing;
+
+sub Version { $VERSION; }
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+=head1 NAME
+
+parse_dir - parse directory listing
+
+=head1 SYNOPSIS
+
+ use File::Listing;
+ for (parse_dir(`ls -l`)) {
+     ($name, $type, $size, $mtime, $mode) = @$_;
+     next if $type ne 'f'; # plain file
+     #...
+ }
+
+ # directory listing can also be read from a file
+ open(LISTING, "zcat ls-lR.gz|");
+ $dir = parse_dir(\*LISTING, '+0000');
+
+=head1 DESCRIPTION
+
+The parse_dir() routine can be used to parse directory
+listings. Currently it only understand Unix C<'ls -l'> and C<'ls -lR'>
+format.  It should eventually be able to most things you might get
+back from a ftp server file listing (LIST command), i.e. VMS listings,
+NT listings, DOS listings,...
+
+The first parameter to parse_dir() is the directory listing to parse.
+It can be a scalar, a reference to an array of directory lines or a
+glob representing a filehandle to read the directory listing from.
+
+The second parameter is the time zone to use when parsing time stamps
+in the listing. If this value is undefined, then the local time zone is
+assumed.
+
+The third parameter is the type of listing to assume.  The values will
+be strings like 'unix', 'vms', 'dos'.  Currently only 'unix' is
+implemented and this is also the default value.  Ideally, the listing
+type should be determined automatically.
+
+The fourth parameter specify how unparseable lines should be treated.
+Values can be 'ignore', 'warn' or a code reference.  Warn means that
+the perl warn() function will be called.  If a code reference is
+passed, then this routine will be called and the return value from it
+will be incorporated in the listing.  The default is 'ignore'.
+
+Only the first parameter is mandatory.  The parse_dir() prototype is
+($;$$$).
+
+The return value from parse_dir() is a list of directory entries.  In
+scalar context the return value is a reference to the list.  The
+directory entries are represented by an array consisting of [
+$filename, $filetype, $filesize, $filetime, $filemode ].  The
+$filetype value is one of the letters 'f', 'd', 'l' or '?'.  The
+$filetime value is converted to seconds since Jan 1, 1970.  The
+$filemode is a bitmask like the mode returned by stat().
+
+=head1 CREDITS
+
+Based on lsparse.pl (from Lee McLoughlin's ftp mirror package) and
+Net::FTP's parse_dir (Graham Barr).
+
+=cut
+
+require Exporter;
+@ISA = qw(Exporter);
+
+@EXPORT = qw(parse_dir);
+
+use strict;
+
+use Carp ();
+use HTTP::Date qw(str2time);
+
+sub parse_dir ($;$$$)
+{
+   my($dir, $tz, $fstype, $error) = @_;
+
+   $fstype ||= 'unix';
+   $fstype = "File::Listing::" . lc $fstype;
+
+   my @args = $_[0];
+   push(@args, $tz) if(@_ >= 2);
+   push(@args, $error) if(@_ >= 4);
+
+   $fstype->parse(@args);
+}
+
+sub line { Carp::croak("Not implemented yet"); }
+sub init { } # Dummy sub
+
+sub file_mode ($)
+{
+    # This routine was originally borrowed from Graham Barr's
+    # Net::FTP package.
+
+    local $_ = shift;
+    my $mode = 0;
+    my($type,$ch);
+
+    s/^(.)// and $type = $1;
+
+    while (/(.)/g) {
+	$mode <<= 1;
+	$mode |= 1 if $1 ne "-" &&
+		      $1 ne 'S' &&
+		      $1 ne 't' &&
+		      $1 ne 'T';
+    }
+
+    $type eq "d" and $mode |= 0040000 or	# Directory
+      $type eq "l" and $mode |= 0120000 or	# Symbolic Link
+	$mode |= 0100000;			# Regular File
+
+    $mode |= 0004000 if /^...s....../i;
+    $mode |= 0002000 if /^......s.../i;
+    $mode |= 0001000 if /^.........t/i;
+
+    $mode;
+}
+
+sub parse
+{
+   my($pkg, $dir, $tz, $error) = @_;
+
+   # First let's try to determine what kind of dir parameter we have
+   # received.  We allow both listings, reference to arrays and
+   # file handles to read from.
+
+   if (ref($dir) eq 'ARRAY') {
+       # Already splitted up
+   } elsif (ref($dir) eq 'GLOB') {
+       # A file handle
+   } elsif (ref($dir)) {
+      Carp::croak("Illegal argument to parse_dir()");
+   } elsif ($dir =~ /^\*\w+(::\w+)+$/) {
+      # This scalar looks like a file handle, so we assume it is
+   } else {
+      # A normal scalar listing
+      $dir = [ split(/\n/, $dir) ];
+   }
+
+   $pkg->init();
+
+   my @files = ();
+   if (ref($dir) eq 'ARRAY') {
+       for (@$dir) {
+	   push(@files, $pkg->line($_, $tz, $error));
+       }
+   } else {
+       local($_);
+       while (<$dir>) {
+	   chomp;
+	   push(@files, $pkg->line($_, $tz, $error));
+       }
+   }
+   wantarray ? @files : \@files;
+}
+
+
+package File::Listing::unix;
+
+use HTTP::Date qw(str2time);
+
+# A place to remember current directory from last line parsed.
+use vars qw($curdir);
+no strict qw(vars);
+
+@ISA = qw(File::Listing);
+
+
+sub init
+{
+    $curdir = '';
+}
+
+sub line
+{
+    shift; # package name
+    local($_) = shift;
+    my($tz, $error) = @_;
+
+    s/\015//g;
+    #study;
+
+    my ($kind, $size, $date, $name);
+    if (($kind, $size, $date, $name) =
+	/^([\-FlrwxsStTdD]{10})                   # Type and permission bits
+	 .*                                       # Graps
+	 \D(\d+)                                  # File size
+	 \s+                                      # Some space
+	 (\w{3}\s+\d+\s+(?:\d{1,2}:\d{2}|\d{4}))  # Date
+	 \s+                                      # Some more space
+	 (.*)$                                    # File name
+	/x )
+
+    {
+	return if $name eq '.' || $name eq '..';
+	$name = "$curdir/$name" if length $curdir;
+	my $type = '?';
+	if ($kind =~ /^l/ && $name =~ /(.*) -> (.*)/ ) {
+	    $name = $1;
+	    $type = "l $2";
+	} elsif ($kind =~ /^[\-F]/) { # (hopefully) a regular file
+	    $type = 'f';
+	} elsif ($kind =~ /^[dD]/) {
+	    $type = 'd';
+	    $size = undef;  # Don't believe the reported size
+	}
+	return [$name, $type, $size, str2time($date, $tz), 
+              File::Listing::file_mode($kind)];
+
+    } elsif (/^(.+):$/ && !/^[dcbsp].*\s.*\s.*:$/ ) {
+	my $dir = $1;
+	return () if $dir eq '.';
+	$curdir = $dir;
+	return ();
+    } elsif (/^[Tt]otal\s+(\d+)$/ || /^\s*$/) {
+	return ();
+    } elsif (/not found/    || # OSF1, HPUX, and SunOS return
+             # "$file not found"
+             /No such file/ || # IRIX returns
+             # "UX:ls: ERROR: Cannot access $file: No such file or directory"
+                               # Solaris returns
+             # "$file: No such file or directory"
+             /cannot find/     # Windows NT returns
+             # "The system cannot find the path specified."
+             ) {
+	return () unless defined $error;
+	&$error($_) if ref($error) eq 'CODE';
+	warn "Error: $_\n" if $error eq 'warn';
+	return ();
+    } elsif ($_ eq '') {       # AIX, and Linux return nothing
+	return () unless defined $error;
+	&$error("No such file or directory") if ref($error) eq 'CODE';
+	warn "Warning: No such file or directory\n" if $error eq 'warn';
+	return ();
+    } else {
+        # parse failed, check if the dosftp parse understands it
+        return(File::Listing::dosftp->line($_,$tz,$error));
+    }
+
+}
+
+package File::Listing::dosftp;
+
+use HTTP::Date qw(str2time);
+
+# A place to remember current directory from last line parsed.
+use vars qw($curdir);
+no strict qw(vars);
+
+@ISA = qw(File::Listing);
+
+sub init
+{
+    $curdir = '';
+}
+
+sub line
+{
+    shift; # package name
+    local($_) = shift;
+    my($tz, $error) = @_;
+
+    s/\015//g;
+
+    my ($kind, $size, $date, $name);
+
+    # 02-05-96  10:48AM                 1415 src.slf
+    # 09-10-96  09:18AM       <DIR>          sl_util
+    if (($date,$size_or_dir,$name) =
+        /^(\d\d-\d\d-\d\d\s+\d\d:\d\d\wM)         # Date and time info
+         \s+                                      # Some space
+         (<\w{3}>|\d+)                            # Dir or Size
+         \s+                                      # Some more space
+         (.+)$                                    # File name
+        /x )
+    {
+	return if $name eq '.' || $name eq '..';
+	$name = "$curdir/$name" if length $curdir;
+	my $type = '?';
+	if ($size_or_dir eq '<DIR>') {
+	    $type = "d";
+            $size = ""; # directories have no size in the pc listing
+        } else {
+	    $type = 'f';
+            $size = $size_or_dir;
+	}
+	return [$name, $type, $size, str2time($date, $tz),
+              File::Listing::file_mode($kind)];
+
+    } else {
+	return () unless defined $error;
+	&$error($_) if ref($error) eq 'CODE';
+	warn "Can't parse: $_\n" if $error eq 'warn';
+	return ();
+    }
+
+}
+
+package File::Listing::vms;
+@File::Listing::unix::ISA = qw(File::Listing);
+
+package File::Listing::netware;
+@File::Listing::unix::ISA = qw(File::Listing);
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Cookies.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Cookies.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Cookies.pm	(revision 21112)
@@ -0,0 +1,725 @@
+package HTTP::Cookies;
+
+# Based on draft-ietf-http-state-man-mec-08.txt and
+# http://www.netscape.com/newsref/std/cookie_spec.html
+
+use strict;
+use HTTP::Date qw(str2time time2str);
+use HTTP::Headers::Util qw(split_header_words join_header_words);
+use LWP::Debug ();
+
+use vars qw($VERSION);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+=head1 NAME
+
+HTTP::Cookies - Cookie storage and management
+
+=head1 SYNOPSIS
+
+ use HTTP::Cookies;
+ $cookie_jar = HTTP::Cookies->new;
+
+ $cookie_jar->add_cookie_header($request);
+ $cookie_jar->extract_cookies($response);
+
+=head1 DESCRIPTION
+
+Cookies are a general mechanism which server side connections can use
+to both store and retrieve information on the client side of the
+connection.  For more information about cookies referrer to
+<URL:http://www.netscape.com/newsref/std/cookie_spec.html> and
+<URL:http://www.cookiecentral.com/>.  This module also implements the
+new style cookies as described in I<draft-ietf-http-state-man-mec-08.txt>.
+The two variants of cookies is supposed to be able to coexist happily.
+
+Instances of the class I<HTTP::Cookies> are able to store a collection
+of Set-Cookie2: and Set-Cookie:-headers and is able to use this
+information to initialize Cookie-headers in I<HTTP::Request> objects.
+The state of the I<HTTP::Cookies> can be saved and restored from
+files.
+
+=head1 METHODS
+
+The following methods are provided:
+
+=over 4
+
+=cut
+
+# A HTTP::Cookies object is a hash.  The main attribute is the
+# COOKIES 3 level hash:  $self->{COOKIES}{$domain}{$path}{$key}.
+
+
+=item $cookie_jar = HTTP::Cookies->new;
+
+The constructor.  Takes hash style parameters.  The following
+parameters are recognized:
+
+  file:            name of the file to restore and save cookies to
+  autosave:        should we save during destruction (bool)
+  ignore_discard:  save even cookies that are requested to be discarded (bool)
+
+Future parameters might include (not yet implemented):
+
+  max_cookies               300
+  max_cookies_per_domain    20
+  max_cookie_size           4096
+
+  no_cookies   list of domain names that we never return cookies to
+
+=cut
+
+sub new
+{
+    my $class = shift;
+    my $self = bless {
+	COOKIES => {},
+    }, $class;
+    my %cnf = @_;
+    for (keys %cnf) {
+	$self->{lc($_)} = $cnf{$_};
+    }
+    $self->load;
+    $self;
+}
+
+
+=item $cookie_jar->add_cookie_header($request);
+
+The add_cookie_header() method will set the appropriate Cookie:-header
+for the I<HTTP::Request> object given as argument.  The $request must
+have a valid url() attribute before this method is called.
+
+=cut
+
+sub add_cookie_header
+{
+    my $self = shift;
+    my $request = shift || return;
+    my $url = $request->url;
+    my $domain = $url->host;
+    $domain = "$domain.local" unless $domain =~ /\./;
+    my $secure_request = ($url->scheme eq "https");
+    my $req_path = $url->epath;
+    my $req_port = $url->port;
+    my $now = time();
+    $self->_normalize_path($req_path) if $req_path =~ /%/;
+
+    my @cval;    # cookie values for the "Cookie" header
+    my $set_ver;
+
+    while (($domain =~ tr/././) >= 2 || # must be at least 2 dots
+           $domain =~ /\.local$/)
+    {
+
+        LWP::Debug::debug("Checking $domain for cookies");
+	my $cookies = $self->{COOKIES}{$domain};
+	next unless $cookies;
+
+	# Want to add cookies corresponding to the most specific paths
+	# first (i.e. longest path first)
+	my $path;
+	for $path (sort {length($b) <=> length($a) } keys %$cookies) {
+            LWP::Debug::debug("- checking cookie path=$path");
+	    if (index($req_path, $path) != 0) {
+	        LWP::Debug::debug("  path $path:$req_path does not fit");
+		next;
+	    }
+
+	    my($key,$array);
+	    while (($key,$array) = each %{$cookies->{$path}}) {
+		my($version,$val,$port,$path_spec,$secure,$expires) = @$array;
+	        LWP::Debug::debug(" - checking cookie $key=$val");
+		if ($secure && !$secure_request) {
+		    LWP::Debug::debug("   not a secure requests");
+		    next;
+		}
+		if ($expires && $expires < $now) {
+		    LWP::Debug::debug("   expired");
+		    next;
+		}
+		if ($port) {
+		    my $found;
+		    if ($port =~ s/^_//) {
+			# The correponding Set-Cookie attribute was empty
+			$found++ if $port eq $req_port;
+			$port = "";
+		    } else {
+			my $p;
+			for $p (split(/,/, $port)) {
+			    $found++, last if $p eq $req_port;
+			}
+		    }
+		    unless ($found) {
+		        LWP::Debug::debug("   port $port:$req_port does not fit");
+			next;
+		    }
+		}
+	        LWP::Debug::debug("   it's a match");
+
+		# set version number of cookie header.
+	        # XXX: What should it be if multiple matching
+                #      Set-Cookie headers have different versions themselves
+		if (!$set_ver++) {
+		    if ($version >= 1) {
+			push(@cval, "\$Version=$version");
+		    } else {
+			$request->header(Cookie2 => "\$Version=1");
+		    }
+		}
+
+		# do we need to quote the value
+		if ($val =~ /\W/ && $version) {
+		    $val =~ s/([\\\"])/\\$1/g;
+		    $val = qq("$val");
+		}
+
+		# and finally remember this cookie
+		push(@cval, "$key=$val");
+		if ($version >= 1) {
+		    push(@cval, qq(\$Path="$path"))     if $path_spec;
+		    push(@cval, qq(\$Domain="$domain")) if $domain =~ /^\./;
+		    if (defined $port) {
+			my $p = '$Port';
+			$p .= qq(="$port") if length $port;
+			push(@cval, $p);
+		    }
+		}
+
+	    }
+        }
+
+    } continue {
+	# Try with a more general domain:  www.sol.no ==> .sol.no
+	$domain =~ s/^\.?[^.]*//;
+    }
+
+    $request->header(Cookie => join("; ", @cval)) if @cval;
+
+    $request;
+}
+
+
+=item $cookie_jar->extract_cookies($response);
+
+The extract_cookies() method will look for Set-Cookie: and
+Set-Cookie2:-headers in the I<HTTP::Response> object passed as
+argument.  If some of these headers are found they are used to update
+the state of the $cookie_jar.
+
+=cut
+
+sub extract_cookies
+{
+    my $self = shift;
+    my $response = shift || return;
+    my @set = split_header_words($response->_header("Set-Cookie2"));
+    my $netscape_cookies;
+    unless (@set) {
+	@set = $response->_header("Set-Cookie");
+	return $response unless @set;
+	$netscape_cookies++;
+    }
+
+    my $url = $response->request->url;
+    my $req_host = $url->host;
+    $req_host = "$req_host.local" unless $req_host =~ /\./;
+    my $req_port = $url->port;
+    my $req_path = $url->epath;
+    $self->_normalize_path($req_path) if $req_path =~ /%/;
+    
+    if ($netscape_cookies) {
+	# The old Netscape cookie format for Set-Cookie
+        # http://www.netscape.com/newsref/std/cookie_spec.html
+	# can for instance contain an unquoted "," in the expires
+	# field, so we have to use this ad-hoc parser.
+	my $now = time();
+	my @old = @set;
+	@set = ();
+	my $set;
+	for $set (@old) {
+	    my @cur;
+	    my $param;
+	    my $expires;
+	    for $param (split(/\s*;\s*/, $set)) {
+		my($k,$v) = split(/\s*=\s*/, $param, 2);
+		#print "$k => $v\n";
+		my $lc = lc($k);
+		if ($lc eq "expires") {
+		    push(@cur, "Max-Age" => str2time($v) - $now);
+		    $expires++;
+		} else {
+		    push(@cur, $k => $v);
+		}
+	    }
+	    push(@cur, "Port" => $req_port);
+	    push(@cur, "Discard" => undef) unless $expires;
+	    push(@cur, "Version" => 0);
+	    push(@set, \@cur);
+	}
+    }
+
+  SET_COOKIE:
+    for my $set (@set) {
+	next unless @$set >= 2;
+
+	my $key = shift @$set;
+	my $val = shift @$set;
+
+        LWP::Debug::debug("Set cookie $key => $val");
+
+	my %hash;
+	while (@$set) {
+	    my $k = shift @$set;
+	    my $v = shift @$set;
+	    my $lc = lc($k);
+	    # don't loose case distinction for unknown fields
+	    $k = $lc if $lc =~ /^(?:discard|domain|max-age|
+                                    path|port|secure|version)$/x;
+	    if ($k eq "discard" || $k eq "secure") {
+		$v = 1 unless defined $v;
+	    }
+	    next if exists $hash{$k};  # only first value is signigicant
+	    $hash{$k} = $v;
+	};
+
+	my %orig_hash = %hash;
+	my $version   = delete $hash{version};
+	$version = 1 unless defined($version);
+	my $discard   = delete $hash{discard};
+	my $secure    = delete $hash{secure};
+	my $maxage    = delete $hash{'max-age'};
+
+	# Check domain
+	my $domain  = delete $hash{domain};
+	if (defined($domain) && $domain ne $req_host) {
+	    if ($domain !~ /\./ && $domain ne "local") {
+	        LWP::Debug::debug("Domain $domain contains no dot");
+		next SET_COOKIE;
+	    }
+	    $domain = ".$domain" unless $domain =~ /^\./;
+	    if ($domain =~ /\.\d+$/) {
+	        LWP::Debug::debug("IP-address $domain illeagal as domain");
+		next SET_COOKIE;
+	    }
+	    my $len = length($domain);
+	    unless (substr($req_host, -$len) eq $domain) {
+	        LWP::Debug::debug("Domain $domain does not match host $req_host");
+		next SET_COOKIE;
+	    }
+	    my $hostpre = substr($req_host, 0, length($req_host) - $len);
+	    if ($hostpre =~ /\./ && !$netscape_cookies) {
+	        LWP::Debug::debug("Host prefix contain a dot: $hostpre => $domain");
+		next SET_COOKIE;
+	    }
+	} else {
+	    $domain = $req_host;
+	}
+
+	my $path = delete $hash{path};
+	my $path_spec;
+	if (defined $path) {
+	    $path_spec++;
+	    $self->_normalize_path($path) if $path =~ /%/;
+	    if (!$netscape_cookies &&
+                substr($req_path, 0, length($path)) ne $path) {
+	        LWP::Debug::debug("Path $path is not a prefix of $req_path");
+		next SET_COOKIE;
+	    }
+	} else {
+	    $path = $req_path;
+	    $path =~ s,/[^/]*$,,;
+	    $path = "/" unless length($path);
+	}
+
+	my $port;
+	if (exists $hash{port}) {
+	    $port = delete $hash{port};
+	    if (defined $port) {
+		$port =~ s/\s+//g;
+		my $found;
+		for my $p (split(/,/, $port)) {
+		    unless ($p =~ /^\d+$/) {
+		      LWP::Debug::debug("Bad port $port (not numeric)");
+			next SET_COOKIE;
+		    }
+		    $found++ if $p eq $req_port;
+		}
+		unless ($found) {
+		    LWP::Debug::debug("Request port ($req_port) not found in $port");
+		    next SET_COOKIE;
+		}
+	    } else {
+		$port = "_$req_port";
+	    }
+	}
+	$self->set_cookie($version,$key,$val,$path,$domain,$port,$path_spec,$secure,$maxage,$discard, \%hash)
+	    if $self->set_cookie_ok(\%orig_hash);
+    }
+
+    $response;
+}
+
+sub set_cookie_ok { 1 };
+
+=item $cookie_jar->set_cookie($version, $key, $val, $path, $domain, $port, $path_spec, $secure, $maxage, $discard, \%rest)
+
+The set_cookie() method updates the state of the $cookie_jar.  The
+$key, $val, $domain, $port and $path arguments are strings.  The
+$path_spec, $secure, $discard arguments are boolean values. The $maxage
+value is a number indicating number of seconds that this cookie will
+live.  A value <= 0 will delete this cookie.  The %rest are a place
+for various other attributes like "Comment" and "CommentURL".
+
+=cut
+
+sub set_cookie
+{
+    my $self = shift;
+    my($version,
+       $key, $val, $path, $domain, $port,
+       $path_spec, $secure, $maxage, $discard, $rest) = @_;
+
+    # there must always be at least 2 dots in a domain
+    return $self if ($domain =~ tr/././) < 2 &&
+                     $domain !~ /\.local$/;
+
+    # path and key can not be empty (key can't start with '$')
+    return $self if !defined($path) || $path !~ m,^/, ||
+	            !defined($key)  || $key  !~ m,[^\$],;
+
+    # ensure legal port
+    if (defined $port) {
+	return $self unless $port =~ /^_?\d+(?:,\d+)*$/;
+    }
+
+    my $expires;
+    if (defined $maxage) {
+	if ($maxage <= 0) {
+	    delete $self->{COOKIES}{$domain}{$path}{$key};
+	    return $self;
+	}
+	$expires = time() + $maxage;
+    }
+    $version = 0 unless defined $version;
+
+    my @array = ($version, $val,$port,
+		 $path_spec,
+		 $secure, $expires, $discard);
+    push(@array, {%$rest}) if defined($rest) && %$rest;
+    # trim off undefined values at end
+    pop(@array) while !defined $array[-1];
+
+    $self->{COOKIES}{$domain}{$path}{$key} = \@array;
+    $self;
+}
+
+=item $cookie_jar->save( [$file] );
+
+Calling this method file save the state of the $cookie_jar to a file.
+The state can then be restored later using the load() method.  If a
+filename is not specified we will use the name specified during
+construction.  If the attribute I<ignore_discared> is set, then we
+will even save cookies that are marked to be discarded.
+
+The default is to save a sequence of "Set-Cookie3" lines.  The
+"Set-Cookie3" is a proprietary LWP format, not known to be compatible
+with any other browser.  The I<HTTP::Cookies::Netscape> sub-class can
+be used to save in a format compatible with Netscape.
+
+=cut
+
+sub save
+{
+    my $self = shift;
+    my $file = shift || $self->{'file'} || return;
+    local(*FILE);
+    open(FILE, ">$file") or die "Can't open $file: $!";
+    print FILE "#LWP-Cookies-1.0\n";
+    print FILE $self->as_string(!$self->{ignore_discard});
+    close(FILE);
+    1;
+}
+
+=item $cookie_jar->load( [$file] );
+
+This method will read the cookies from the file and add them to the
+$cookie_jar.  The file must be in the format written by the save()
+method.
+
+=cut
+
+sub load
+{
+    my $self = shift;
+    my $file = shift || $self->{'file'} || return;
+    local(*FILE, $_);
+    open(FILE, $file) or return;
+    my $magic = <FILE>;
+    unless ($magic =~ /^\#LWP-Cookies-(\d+\.\d+)/) {
+	warn "$file does not seem to contain cookies";
+	return;
+    }
+    while (<FILE>) {
+	next unless s/^Set-Cookie3:\s*//;
+	chomp;
+	my $cookie;
+	for $cookie (split_header_words($_)) {
+	    my($key,$val) = splice(@$cookie, 0, 2);
+	    my %hash;
+	    while (@$cookie) {
+		my $k = shift @$cookie;
+		my $v = shift @$cookie;
+		$hash{$k} = $v;
+	    }
+	    my $version   = delete $hash{version};
+	    my $path      = delete $hash{path};
+	    my $domain    = delete $hash{domain};
+	    my $port      = delete $hash{port};
+	    my $expires   = str2time(delete $hash{expires});
+
+	    my $path_spec = exists $hash{path_spec}; delete $hash{path_spec};
+	    my $secure    = exists $hash{secure};    delete $hash{secure};
+	    my $discard   = exists $hash{discard};   delete $hash{discard};
+
+	    my @array =	($version,$val,$port,
+			 $path_spec,$secure,$expires,$discard);
+	    push(@array, \%hash) if %hash;
+	    $self->{COOKIES}{$domain}{$path}{$key} = \@array;
+	}
+    }
+    close(FILE);
+    1;
+}
+
+=item $cookie_jar->revert;
+
+Will revert to the state of last save.
+
+=cut
+
+sub revert
+{
+    my $self = shift;
+    $self->clear->load;
+    $self;
+}
+
+=item $cookie_jar->clear( [$domain, [$path, [$key] ] ]);
+
+Invoking this method without arguments will empty the whole
+$cookie_jar.  If given a single argument only cookies belonging to
+that domain will be removed.  If given two arguments, cookies
+belonging to the specified path within that domain is removed.  If
+given three arguments, then the cookie with the specified key, path
+and domain is removed.
+
+=cut
+
+sub clear
+{
+    my $self = shift;
+    if (@_ == 0) {
+	$self->{COOKIES} = {};
+    } elsif (@_ == 1) {
+	delete $self->{COOKIES}{$_[0]};
+    } elsif (@_ == 2) {
+	delete $self->{COOKIES}{$_[0]}{$_[1]};
+    } elsif (@_ == 3) {
+	delete $self->{COOKIES}{$_[0]}{$_[1]}{$_[2]};
+    } else {
+	require Carp;
+        Carp::carp('Usage: $c->clear([domain [,path [,key]]])');
+    }
+    $self;
+}
+
+sub DESTROY
+{
+    my $self = shift;
+    $self->save if $self->{'autosave'};
+}
+
+
+=item $cookie_jar->scan( \&callback );
+
+The argument is a subroutine that will be invoked for each cookie
+stored within the $cookie_jar.  The subroutine will be invoked with
+the following arguments:
+
+  0  version
+  1  key
+  2  val
+  3  path
+  4  domain
+  5  port
+  6  path_spec
+  7  secure
+  8  expires
+  9  discard
+ 10  hash
+
+=cut
+
+sub scan
+{
+    my($self, $cb) = @_;
+    my($domain,$path,$key);
+    for $domain (sort keys %{$self->{COOKIES}}) {
+	for $path (sort keys %{$self->{COOKIES}{$domain}}) {
+	    for $key (sort keys %{$self->{COOKIES}{$domain}{$path}}) {
+		my($version,$val,$port,$path_spec,
+		   $secure,$expires,$discard,$rest) =
+		       @{$self->{COOKIES}{$domain}{$path}{$key}};
+		$rest = {} unless defined($rest);
+		&$cb($version,$key,$val,$path,$domain,$port,
+		     $path_spec,$secure,$expires,$discard,$rest);
+	    }
+	}
+    }
+}
+
+=item $cookie_jar->as_string( [$skip_discard] );
+
+The as_string() method will return the state of the $cookie_jar
+represented as a sequence of "Set-Cookie3" header lines separated by
+"\n".  If given a argument that is TRUE, it will not return lines for
+cookies with the I<Discard> attribute.
+
+=cut
+
+sub as_string
+{
+    my($self, $skip_discard) = @_;
+    my @res;
+    $self->scan(sub {
+	my($version,$key,$val,$path,$domain,$port,
+	   $path_spec,$secure,$expires,$discard,$rest) = @_;
+	return if $discard && $skip_discard;
+	my @h = ($key, $val);
+	push(@h, "path", $path);
+	push(@h, "domain" => $domain);
+	push(@h, "port" => $port) if defined $port;
+	push(@h, "path_spec" => undef) if $path_spec;
+	push(@h, "secure" => undef) if $secure;
+	push(@h, "expires" => HTTP::Date::time2isoz($expires)) if $expires;
+	push(@h, "discard" => undef) if $discard;
+	my $k;
+	for $k (sort keys %$rest) {
+	    push(@h, $k, $rest->{$k});
+	}
+	push(@h, "version" => $version);
+	push(@res, "Set-Cookie3: " . join_header_words(\@h));
+    });
+    join("\n", @res, "");
+}
+
+
+sub _normalize_path  # so that plain string compare can be used
+{
+    shift;  # $self
+    my $x;
+    $_[0] =~ s/%([0-9a-fA-F][0-9a-fA-F])/
+	         $x = uc($1);
+                 $x eq "2F" || $x eq "25" ? "%$x" :
+                                            pack("c", hex($x));
+              /eg;
+    $_[0] =~ s/([\0-\x20\x7f-\xff])/sprintf("%%%02X",ord($1))/eg;
+}
+
+
+
+=back
+
+=head1 SUB CLASSES
+
+We also provide a subclass called I<HTTP::Cookies::Netscape> which make
+cookie loading and saving compatible with Netscape cookie files.  You
+should be able to have LWP share Netscape's cookies by constructing
+your $cookie_jar like this:
+
+ $cookie_jar = HTTP::Cookies::Netscape->new(
+                   File     => "$ENV{HOME}/.netscape/cookies",
+                   AutoSave => 1,
+               );
+
+Please note that the Netscape cookie file format is not able to store
+all the information available in the Set-Cookie2 headers, so you will
+probably loose some information if you save using this format.
+
+=cut
+
+package HTTP::Cookies::Netscape;
+
+use vars qw(@ISA);
+@ISA=qw(HTTP::Cookies);
+
+sub load
+{
+    my($self, $file) = @_;
+    $file ||= $self->{'file'} || return;
+    local(*FILE, $_);
+    my @cookies;
+    open(FILE, $file) || return;
+    my $magic = <FILE>;
+    unless ($magic =~ /^\# Netscape HTTP Cookie File/) {
+	warn "$file does not look like a netscape cookies file" if $^W;
+	close(FILE);
+	return;
+    }
+    my $now = time();
+    while (<FILE>) {
+	next if /^\s*\#/;
+	next if /^\s*$/;
+	chomp;
+	my($domain,$bool1,$path,$secure, $expires,$key,$val) = split(/\t/, $_);
+	$secure = ($secure eq "TRUE");
+	$self->set_cookie(undef,$key,$val,$path,$domain,undef,
+			  0,$secure,$expires-$now, 0);
+    }
+    close(FILE);
+    1;
+}
+
+sub save
+{
+    my($self, $file) = @_;
+    $file ||= $self->{'file'} || return;
+    local(*FILE, $_);
+    open(FILE, ">$file") || return;
+
+    print FILE <<EOT;
+# Netscape HTTP Cookie File
+# http://www.netscape.com/newsref/std/cookie_spec.html
+# This is a generated file!  Do not edit.
+
+EOT
+
+    my $now = time;
+    $self->scan(sub {
+	my($version,$key,$val,$path,$domain,$port,
+	   $path_spec,$secure,$expires,$discard,$rest) = @_;
+	return if $discard && !$self->{ignore_discard};
+	$expires ||= 0;
+	return if $now > $expires;
+	$secure = $secure ? "TRUE" : "FALSE";
+	my $bool = $domain =~ /^\./ ? "TRUE" : "FALSE";
+	print FILE join("\t", $domain, $bool, $path, $secure, $expires, $key, $val), "\n";
+    });
+    close(FILE);
+    1;
+}
+
+1;
+
+__END__
+
+=head1 COPYRIGHT
+
+Copyright 1997, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Daemon.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Daemon.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Daemon.pm	(revision 21112)
@@ -0,0 +1,805 @@
+# $Id$
+#
+
+use strict;
+
+package HTTP::Daemon;
+
+=head1 NAME
+
+HTTP::Daemon - a simple http server class
+
+=head1 SYNOPSIS
+
+  use HTTP::Daemon;
+  use HTTP::Status;
+
+  my $d = new HTTP::Daemon;
+  print "Please contact me at: <URL:", $d->url, ">\n";
+  while (my $c = $d->accept) {
+      while (my $r = $c->get_request) {
+	  if ($r->method eq 'GET' and $r->url->path eq "/xyzzy") {
+              # remember, this is *not* recommened practice :-)
+	      $c->send_file_response("/etc/passwd");
+	  } else {
+	      $c->send_error(RC_FORBIDDEN)
+	  }
+      }
+      $c->close;
+      undef($c);
+  }
+
+=head1 DESCRIPTION
+
+Instances of the I<HTTP::Daemon> class are HTTP/1.1 servers that
+listens on a socket for incoming requests. The I<HTTP::Daemon> is a
+sub-class of I<IO::Socket::INET>, so you can do socket operations
+directly on it too.
+
+The accept() method will return when a connection from a client is
+available. The returned value will be a reference to a object of the
+I<HTTP::Daemon::ClientConn> class which is another I<IO::Socket::INET>
+subclass. Calling the get_request() method on this object will read
+data from the client and return an I<HTTP::Request> object reference.
+
+This HTTP daemon does not fork(2) for you.  Your application, i.e. the
+user of the I<HTTP::Daemon> is reponsible for forking if that is
+desirable.  Also note that the user is responsible for generating
+responses that conforms to the HTTP/1.1 protocol.  The
+I<HTTP::Daemon::ClientConn> provide some methods that make this easier.
+
+=head1 METHODS
+
+The following is a list of methods that are new (or enhanced) relative
+to the I<IO::Socket::INET> base class.
+
+=over 4
+
+=cut
+
+
+use vars qw($VERSION @ISA $PROTO $DEBUG);
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use IO::Socket ();
+@ISA=qw(IO::Socket::INET);
+
+$PROTO = "HTTP/1.1";
+
+=item $d = new HTTP::Daemon
+
+The object constructor takes the same parameters as the
+I<IO::Socket::INET> constructor.  It can be called without specifying
+any parameters. The daemon will then set up a listen queue of 5
+connections and allocate some random port number.  A server that want
+to bind to some specific address on the standard HTTP port will be
+constructed like this:
+
+  $d = new HTTP::Daemon
+        LocalAddr => 'www.someplace.com',
+        LocalPort => 80;
+
+=cut
+
+sub new
+{
+    my($class, %args) = @_;
+    $args{Listen} ||= 5;
+    $args{Proto}  ||= 'tcp';
+    my $self = $class->SUPER::new(%args);
+    return undef unless $self;
+
+    my $host = $args{LocalAddr};
+    unless ($host) {
+	require Sys::Hostname;
+	$host = Sys::Hostname::hostname();
+    }
+    ${*$self}{'httpd_server_name'} = $host;
+    $self;
+}
+
+
+=item $c = $d->accept([$pkg])
+
+Same as I<IO::Socket::accept> but will return an
+I<HTTP::Daemon::ClientConn> reference by default.  It will return
+undef if you have specified a timeout and no connection is made within
+that time.
+
+=cut
+
+sub accept
+{
+    my $self = shift;
+    my $pkg = shift || "HTTP::Daemon::ClientConn";
+    my $sock = $self->SUPER::accept($pkg);
+    ${*$sock}{'httpd_daemon'} = $self if $sock;
+    $sock;
+}
+
+
+=item $d->url
+
+Returns a URL string that can be used to access the server root.
+
+=cut
+
+sub url
+{
+    my $self = shift;
+    my $url = "http://";
+    $url .= ${*$self}{'httpd_server_name'};
+    my $port = $self->sockport;
+    $url .= ":$port" if $port != 80;
+    $url .= "/";
+    $url;
+}
+
+
+=item $d->product_tokens
+
+Returns the name that this server will use to identify itself.  This
+is the string that is sent with the I<Server> response header.  The
+main reason to have this method is that subclasses can override it if
+they want to use another product name.
+
+=cut
+
+sub product_tokens
+{
+    "libwww-perl-daemon/$HTTP::Daemon::VERSION";
+}
+
+
+package HTTP::Daemon::ClientConn;
+
+use vars qw(@ISA $DEBUG);
+use IO::Socket ();
+@ISA=qw(IO::Socket::INET);
+*DEBUG = \$HTTP::Daemon::DEBUG;
+
+use HTTP::Request  ();
+use HTTP::Response ();
+use HTTP::Status;
+use HTTP::Date qw(time2str);
+use URI::URL qw(url);
+use LWP::MediaTypes qw(guess_media_type);
+use Carp ();
+
+my $CRLF = "\015\012";   # "\r\n" is not portable
+my $HTTP_1_0 = _http_version("HTTP/1.0");
+my $HTTP_1_1 = _http_version("HTTP/1.1");
+
+=back
+
+The I<HTTP::Daemon::ClientConn> is also a I<IO::Socket::INET>
+subclass. Instances of this class are returned by the accept() method
+of the I<HTTP::Daemon>.  The following additional methods are
+provided:
+
+=over 4
+
+=item $c->get_request([$headers_only])
+
+This method will read data from the client and turn it into a
+I<HTTP::Request> object which is then returned.  It returns C<undef>
+if reading of the request fails.  If it fails, then the
+I<HTTP::Daemon::ClientConn> object ($c) should be discarded, and you
+should not call this method again.  The $c->reason method might give
+you some information on why $c->get_request returned C<undef>.
+
+The $c->get_request method support HTTP/1.1 request content bodies,
+including I<chunked> transfer encoding with footer and self delimiting
+I<multipart/*> content types.
+
+The $c->get_request method will normally not return until the whole
+request has been received from the client.  This might not be what you
+want if the request is an upload of a multi-mega-byte file (and with
+chunked transfer encoding HTTP can even support infinite request
+messages - uploading live audio for instance).  If you pass a TRUE
+value as the $headers_only argument, then $c->get_request will return
+immediately after parsing the request headers and you are responsible
+for reading the rest of the request content (and if you are going to
+call $c->get_request again on the same connection you better read the
+correct number of bytes).
+
+=cut
+
+sub get_request
+{
+    my($self, $only_headers) = @_;
+    if (${*$self}{'httpd_nomore'}) {
+        $self->reason("No more requests from this connection");
+	return;
+    }
+
+    $self->reason("");
+    my $buf = ${*$self}{'httpd_rbuf'};
+    $buf = "" unless defined $buf;
+    
+    my $timeout = $ {*$self}{'io_socket_timeout'};
+    my $fdset = "";
+    vec($fdset, $self->fileno, 1) = 1;
+    local($_);
+
+  READ_HEADER:
+    while (1) {
+	# loop until we have the whole header in $buf
+	$buf =~ s/^(?:\015?\012)+//;  # ignore leading blank lines
+	if ($buf =~ /\012/) {  # potential, has at least one line
+	    if ($buf =~ /^\w+[^\012]+HTTP\/\d+\.\d+\015?\012/) {
+		if ($buf =~ /\015?\012\015?\012/) {
+		    last READ_HEADER;  # we have it
+		} elsif (length($buf) > 16*1024) {
+		    $self->send_error(413); # REQUEST_ENTITY_TOO_LARGE
+		    $self->reason("Very long header");
+		    return;
+		}
+	    } else {
+		last READ_HEADER;  # HTTP/0.9 client
+	    }
+	} elsif (length($buf) > 16*1024) {
+	    $self->send_error(414); # REQUEST_URI_TOO_LARGE
+	    $self->reason("Very long first line");
+	    return;
+	}
+	print STDERR "Need more data for complete header\n" if $DEBUG;
+	return unless $self->_need_more($buf, $timeout, $fdset);
+    }
+    if ($buf !~ s/^(\w+)[ \t]+(\S+)(?:[ \t]+(HTTP\/\d+\.\d+))?[^\012]*\012//) {
+	$self->send_error(400);  # BAD_REQUEST
+	$self->reason("Bad request line");
+	return;
+    }
+    my $proto = $3 || "HTTP/0.9";
+    my $r = HTTP::Request->new($1, url($2, $self->daemon->url));
+    $r->protocol($proto);
+    ${*$self}{'httpd_client_proto'} = $proto = _http_version($proto);
+
+    if ($proto >= $HTTP_1_0) {
+	# we expect to find some headers
+	my($key, $val);
+      HEADER:
+	while ($buf =~ s/^([^\012]*)\012//) {
+	    $_ = $1;
+	    s/\015$//;
+	    if (/^([\w\-]+)\s*:\s*(.*)/) {
+		$r->push_header($key, $val) if $key;
+		($key, $val) = ($1, $2);
+	    } elsif (/^\s+(.*)/) {
+		$val .= " $1";
+	    } else {
+		last HEADER;
+	    }
+	}
+	$r->push_header($key, $val) if $key;
+    }
+
+    my $conn = $r->header('Connection');
+    if ($proto >= $HTTP_1_1) {
+	${*$self}{'httpd_nomore'}++ if $conn && lc($conn) =~ /\bclose\b/;
+    } else {
+	${*$self}{'httpd_nomore'}++ unless $conn &&
+                                           lc($conn) =~ /\bkeep-alive\b/;
+    }
+
+    if ($only_headers) {
+	${*$self}{'httpd_rbuf'} = $buf;
+        return $r;
+    }
+
+    # Find out how much content to read
+    my $te  = $r->header('Transfer-Encoding');
+    my $ct  = $r->header('Content-Type');
+    my $len = $r->header('Content-Length');
+
+    if ($te && lc($te) eq 'chunked') {
+	# Handle chunked transfer encoding
+	my $body = "";
+      CHUNK:
+	while (1) {
+	    print STDERR "Chunked\n" if $DEBUG;
+	    if ($buf =~ s/^([^\012]*)\012//) {
+		my $chunk_head = $1;
+		unless ($chunk_head =~ /^([0-9A-Fa-f]+)/) {
+		    $self->send_error(400);
+		    $self->reason("Bad chunk header $chunk_head");
+		    return;
+		}
+		my $size = hex($1);
+		last CHUNK if $size == 0;
+
+		my $missing = $size - length($buf) + 2; # 2=CRLF at chunk end
+		# must read until we have a complete chunk
+		while ($missing > 0) {
+		    print STDERR "Need $missing more bytes\n" if $DEBUG;
+		    my $n = $self->_need_more($buf, $timeout, $fdset);
+		    return unless $n;
+		    $missing -= $n;
+		}
+		$body .= substr($buf, 0, $size);
+		substr($buf, 0, $size+2) = '';
+
+	    } else {
+		# need more data in order to have a complete chunk header
+		return unless $self->_need_more($buf, $timeout, $fdset);
+	    }
+	}
+	$r->content($body);
+
+	# pretend it was a normal entity body
+	$r->remove_header('Transfer-Encoding');
+	$r->header('Content-Length', length($body));
+
+	my($key, $val);
+      FOOTER:
+	while (1) {
+	    if ($buf !~ /\012/) {
+		# need at least one line to look at
+		return unless $self->_need_more($buf, $timeout, $fdset);
+	    } else {
+		$buf =~ s/^([^\012]*)\012//;
+		$_ = $1;
+		s/\015$//;
+		if (/^([\w\-]+)\s*:\s*(.*)/) {
+		    $r->push_header($key, $val) if $key;
+		    ($key, $val) = ($1, $2);
+		} elsif (/^\s+(.*)/) {
+		    $val .= " $1";
+		} elsif (!length) {
+		    last FOOTER;
+		} else {
+		    $self->reason("Bad footer syntax");
+		    return;
+		}
+	    }
+	}
+	$r->push_header($key, $val) if $key;
+
+    } elsif ($te) {
+	$self->send_error(501); 	# Unknown transfer encoding
+	$self->reason("Unknown transfer encoding '$te'");
+	return;
+
+    } elsif ($ct && lc($ct) =~ m/^multipart\/\w+\s*;.*boundary\s*=\s*(\w+)/) {
+	# Handle multipart content type
+	my $boundary = "$CRLF--$1--$CRLF";
+	my $index;
+	while (1) {
+	    $index = index($buf, $boundary);
+	    last if $index >= 0;
+	    # end marker not yet found
+	    return unless $self->_need_more($buf, $timeout, $fdset);
+	}
+	$index += length($boundary);
+	$r->content(substr($buf, 0, $index));
+	substr($buf, 0, $index) = '';
+
+    } elsif ($len) {
+	# Plain body specified by "Content-Length"
+	my $missing = $len - length($buf);
+	while ($missing > 0) {
+	    print "Need $missing more bytes of content\n" if $DEBUG;
+	    my $n = $self->_need_more($buf, $timeout, $fdset);
+	    return unless $n;
+	    $missing -= $n;
+	}
+	if (length($buf) > $len) {
+	    $r->content(substr($buf,0,$len));
+	    substr($buf, 0, $len) = '';
+	} else {
+	    $r->content($buf);
+	    $buf='';
+	}
+    }
+    ${*$self}{'httpd_rbuf'} = $buf;
+
+    $r;
+}
+
+sub _need_more
+{
+    my $self = shift;
+    #my($buf,$timeout,$fdset) = @_;
+    if ($_[1]) {
+	my($timeout, $fdset) = @_[1,2];
+	print STDERR "select(,,,$timeout)\n" if $DEBUG;
+	my $n = select($fdset,undef,undef,$timeout);
+	unless ($n) {
+	    $self->reason(defined($n) ? "Timeout" : "select: $!");
+	    return;
+	}
+    }
+    print STDERR "sysread()\n" if $DEBUG;
+    my $n = sysread($self, $_[0], 2048, length($_[0]));
+    $self->reason(defined($n) ? "Client closed" : "sysread: $!") unless $n;
+    $n;
+}
+
+=item $c->read_buffer([$new_value])
+
+Bytes read by $c->get_request, but not used are placed in the I<read
+buffer>.  The next time $c->get_request is called it will consume the
+bytes in this buffer before reading more data from the network
+connection itself.  The read buffer is invalid after $c->get_request
+has returned an undefined value.
+
+If you handle the reading of the request content yourself you need to
+empty this buffer before you read more and you need to place
+unconsumed bytes here.  You also need this buffer if you implement
+services like I<101 Switching Protocols>.
+
+This method always return the old buffer content and can optionally
+update the buffer content if you pass it an argument.
+
+=cut
+
+sub read_buffer
+{
+    my $self = shift;
+    my $old = ${*$self}{'httpd_rbuf'};
+    if (@_) {
+	${*$self}{'httpd_rbuf'} = shift;
+    }
+    $old;
+}
+
+
+=item $c->reason
+
+When $c->get_request returns C<undef> you can obtain a short string
+describing why it happened by calling $c->reason.
+
+=cut
+
+sub reason
+{
+    my $self = shift;
+    my $old = ${*$self}{'httpd_reason'};
+    if (@_) {
+        ${*$self}{'httpd_reason'} = shift;
+    }
+    $old;
+}
+
+
+=item $c->proto_ge($proto)
+
+Returns TRUE if the client announced a protocol with version number
+greater or equal to the given argument.  The $proto argument can be a
+string like "HTTP/1.1" or just "1.1".
+
+=cut
+
+sub proto_ge
+{
+    my $self = shift;
+    ${*$self}{'httpd_client_proto'} >= _http_version(shift);
+}
+
+sub _http_version
+{
+    local($_) = shift;
+    return 0 unless m,^(?:HTTP/)?(\d+)\.(\d+)$,i;
+    $1 * 1000 + $2;
+}
+
+=item $c->antique_client
+
+Returns TRUE if the client speaks the HTTP/0.9 protocol.  No status
+code and no headers should be returned to such a client.  This should
+be the same as !$c->proto_ge("HTTP/1.0").
+
+=cut
+
+sub antique_client
+{
+    my $self = shift;
+    ${*$self}{'httpd_client_proto'} < $HTTP_1_0;
+}
+
+
+=item $c->force_last_request
+
+Make sure that $c->get_request will not try to read more requests off
+this connection.  If you generate a response that is not self
+delimiting, then you should signal this fact by calling this method.
+
+This attribute is turned on automatically if the client announce
+protocol HTTP/1.0 or worse and does not include a "Connection:
+Keep-Alive" header.  It is also turned on automatically when HTTP/1.1
+or better clients send the "Connection: close" request header.
+
+=cut
+
+sub force_last_request
+{
+    my $self = shift;
+    ${*$self}{'httpd_nomore'}++;
+}
+
+
+=item $c->send_status_line( [$code, [$mess, [$proto]]] )
+
+Sends the status line back to the client.  If $code is omitted 200 is
+assumed.  If $mess is omitted, then a message corresponding to $code
+is inserted.  If $proto is missing the content of the
+$HTTP::Daemon::PROTO variable is used.
+
+=cut
+
+sub send_status_line
+{
+    my($self, $status, $message, $proto) = @_;
+    return if $self->antique_client;
+    $status  ||= RC_OK;
+    $message ||= status_message($status) || "";
+    $proto   ||= $HTTP::Daemon::PROTO || "HTTP/1.1";
+    print $self "$proto $status $message$CRLF";
+}
+
+=item $c->send_crlf
+
+Send the CRLF sequence to the client.
+
+=cut
+
+
+sub send_crlf
+{
+    my $self = shift;
+    print $self $CRLF;
+}
+
+
+=item $c->send_basic_header( [$code, [$mess, [$proto]]] )
+
+Sends the status line and the "Date:" and "Server:" headers back to
+the client.  This header is assumed to be continued and does not end
+with an empty CRLF line.
+
+=cut
+
+sub send_basic_header
+{
+    my $self = shift;
+    return if $self->antique_client;
+    $self->send_status_line(@_);
+    print $self "Date: ", time2str(time), $CRLF;
+    my $product = $self->daemon->product_tokens;
+    print $self "Server: $product$CRLF" if $product;
+}
+
+
+=item $c->send_response( [$res] )
+
+Takes a I<HTTP::Response> object as parameter and write it back to the
+client as the response.  We try hard to make sure that the response is
+self delimiting so that the connection can stay persistent for further
+request/response exchanges.
+
+The content attribute of the I<HTTP::Response> object can be a normal
+string or a subroutine reference.  If it is a subroutine, then
+whatever this callback routine returns will be written back to the
+client as the response content.  The routine will be called until it
+return an undefined or empty value.  If the client is HTTP/1.1 aware
+then we will use the chunked transfer encoding for the response.
+
+=cut
+
+sub send_response
+{
+    my $self = shift;
+    my $res = shift;
+    if (!ref $res) {
+	$res ||= RC_OK;
+	$res = HTTP::Response->new($res, @_);
+    }
+    my $content = $res->content;
+    my $chunked;
+    unless ($self->antique_client) {
+	my $code = $res->code;
+	$self->send_basic_header($code, $res->message, $res->protocol);
+	if ($code =~ /^(1\d\d|[23]04)$/) {
+	    # make sure content is empty
+	    $res->remove_header("Content-Length");
+	    $content = "";
+	} elsif ($res->request && $res->request->method eq "HEAD") {
+	    # probably OK
+	} elsif (ref($content) eq "CODE") {
+	    if ($self->proto_ge("HTTP/1.1")) {
+		$res->push_header("Transfer-Encoding" => "chunked");
+		$chunked++;
+	    } else {
+		$self->force_last_request;
+	    }
+	} elsif (length($content)) {
+	    $res->header("Content-Length" => length($content));
+	} else {
+	    $self->force_last_request;
+	}
+	print $self $res->headers_as_string($CRLF);
+	print $self $CRLF;  # separates headers and content
+    }
+    if (ref($content) eq "CODE") {
+	while (1) {
+	    my $chunk = &$content();
+	    last unless defined($chunk) && length($chunk);
+	    if ($chunked) {
+		printf $self "%x%s%s%s", length($chunk), $CRLF, $chunk, $CRLF;
+	    } else {
+		print $self $chunk;
+	    }
+	}
+	print $self "0$CRLF$CRLF" if $chunked;  # no trailers either
+    } elsif (length $content) {
+	print $self $content;
+    }
+}
+
+
+=item $c->send_redirect( $loc, [$code, [$entity_body]] )
+
+Sends a redirect response back to the client.  The location ($loc) can
+be an absolute or a relative URL. The $code must be one the redirect
+status codes, and it defaults to "301 Moved Permanently"
+
+=cut
+
+sub send_redirect
+{
+    my($self, $loc, $status, $content) = @_;
+    $status ||= RC_MOVED_PERMANENTLY;
+    Carp::croak("Status '$status' is not redirect") unless is_redirect($status);
+    $self->send_basic_header($status);
+    $loc = url($loc, $self->daemon->url) unless ref($loc);
+    $loc = $loc->abs;
+    print $self "Location: $loc$CRLF";
+    if ($content) {
+	my $ct = $content =~ /^\s*</ ? "text/html" : "text/plain";
+	print $self "Content-Type: $ct$CRLF";
+    }
+    print $self $CRLF;
+    print $self $content if $content;
+    $self->force_last_request;  # no use keeping the connection open
+}
+
+
+=item $c->send_error( [$code, [$error_message]] )
+
+Send an error response back to the client.  If the $code is missing a
+"Bad Request" error is reported.  The $error_message is a string that
+is incorporated in the body of the HTML entity body.
+
+=cut
+
+sub send_error
+{
+    my($self, $status, $error) = @_;
+    $status ||= RC_BAD_REQUEST;
+    Carp::croak("Status '$status' is not an error") unless is_error($status);
+    my $mess = status_message($status);
+    $error  ||= "";
+    $mess = <<EOT;
+<title>$status $mess</title>
+<h1>$status $mess</h1>
+$error
+EOT
+    unless ($self->antique_client) {
+        $self->send_basic_header($status);
+        print $self "Content-Type: text/html$CRLF";
+	print $self "Content-Length: " . length($mess) . $CRLF;
+        print $self $CRLF;
+    }
+    print $self $mess;
+    $status;
+}
+
+
+=item $c->send_file_response($filename)
+
+Send back a response with the specified $filename as content.  If the
+file happen to be a directory we will try to generate an HTML index
+of it.
+
+=cut
+
+sub send_file_response
+{
+    my($self, $file) = @_;
+    if (-d $file) {
+	$self->send_dir($file);
+    } elsif (-f _) {
+	# plain file
+	local(*F);
+	sysopen(F, $file, 0) or 
+	  return $self->send_error(RC_FORBIDDEN);
+	my($ct,$ce) = guess_media_type($file);
+	my($size,$mtime) = (stat _)[7,9];
+	unless ($self->antique_client) {
+	    $self->send_basic_header;
+	    print $self "Content-Type: $ct$CRLF";
+	    print $self "Content-Encoding: $ce$CRLF" if $ce;
+	    print $self "Content-Length: $size$CRLF" if $size;
+	    print $self "Last-Modified: ", time2str($mtime), "$CRLF" if $mtime;
+	    print $self $CRLF;
+	}
+	$self->send_file(\*F);
+	return RC_OK;
+    } else {
+	$self->send_error(RC_NOT_FOUND);
+    }
+}
+
+
+sub send_dir
+{
+    my($self, $dir) = @_;
+    $self->send_error(RC_NOT_FOUND) unless -d $dir;
+    $self->send_error(RC_NOT_IMPLEMENTED);
+}
+
+
+=item $c->send_file($fd);
+
+Copies the file back to the client.  The file can be a string (which
+will be interpreted as a filename) or a reference to an I<IO::Handle>
+or glob.
+
+=cut
+
+sub send_file
+{
+    my($self, $file) = @_;
+    my $opened = 0;
+    if (!ref($file)) {
+	local(*F);
+	open(F, $file) || return undef;
+	binmode(F);
+	$file = \*F;
+	$opened++;
+    }
+    my $cnt = 0;
+    my $buf = "";
+    my $n;
+    while ($n = sysread($file, $buf, 8*1024)) {
+	last if !$n;
+	$cnt += $n;
+	print $self $buf;
+    }
+    close($file) if $opened;
+    $cnt;
+}
+
+
+=item $c->daemon
+
+Return a reference to the corresponding I<HTTP::Daemon> object.
+
+=cut
+
+sub daemon
+{
+    my $self = shift;
+    ${*$self}{'httpd_daemon'};
+}
+
+=back
+
+=head1 SEE ALSO
+
+RFC 2068
+
+L<IO::Socket>, L<Apache>
+
+=head1 COPYRIGHT
+
+Copyright 1996-1998, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Date.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Date.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Date.pm	(revision 21112)
@@ -0,0 +1,330 @@
+# $Id$
+#
+package HTTP::Date;
+
+=head1 NAME
+
+HTTP::Date - date conversion routines
+
+=head1 SYNOPSIS
+
+ use HTTP::Date;
+
+ $string = time2str($time);    # Format as GMT ASCII time
+ $time = str2time($string);    # convert ASCII date to machine time
+
+=head1 DESCRIPTION
+
+This module provides two functions that deal with the HTTP date
+format.  The following functions are provided:
+
+=over 4
+
+=item time2str([$time])
+
+The time2str() function converts a machine time (seconds since epoch)
+to a string.  If the function is called without an argument, it will
+use the current time.
+
+The string returned is in the format defined by the HTTP/1.0
+specification.  This is a fixed length subset of the format defined by
+RFC 1123, represented in Universal Time (GMT).  An example of this
+format is:
+
+   Thu, 03 Feb 1994 17:09:00 GMT
+
+=item str2time($str [, $zone])
+
+The str2time() function converts a string to machine time.  It returns
+C<undef> if the format is unrecognized, or the year is not between 1970
+and 2038.  The function is able to parse the following formats:
+
+ "Wed, 09 Feb 1994 22:23:32 GMT"       -- HTTP format
+ "Thu Feb  3 17:03:55 GMT 1994"        -- ctime(3) format
+ "Thu Feb  3 00:00:00 1994",           -- ANSI C asctime() format
+ "Tuesday, 08-Feb-94 14:15:29 GMT"     -- old rfc850 HTTP format
+ "Tuesday, 08-Feb-1994 14:15:29 GMT"   -- broken rfc850 HTTP format
+
+ "03/Feb/1994:17:03:55 -0700"   -- common logfile format
+ "09 Feb 1994 22:23:32 GMT"     -- HTTP format (no weekday)
+ "08-Feb-94 14:15:29 GMT"       -- rfc850 format (no weekday)
+ "08-Feb-1994 14:15:29 GMT"     -- broken rfc850 format (no weekday)
+
+ "1994-02-03 14:15:29 -0100"    -- ISO 8601 format
+ "1994-02-03 14:15:29"          -- zone is optional
+ "1994-02-03"                   -- only date
+ "1994-02-03T14:15:29"          -- Use T as separator
+ "19940203T141529Z"             -- ISO 8601 compact format
+ "19940203"                     -- only date
+
+ "08-Feb-94"         -- old rfc850 HTTP format    (no weekday, no time)
+ "08-Feb-1994"       -- broken rfc850 HTTP format (no weekday, no time)
+ "09 Feb 1994"       -- proposed new HTTP format  (no weekday, no time)
+ "03/Feb/1994"       -- common logfile format     (no time, no offset)
+
+ "Feb  3  1994"      -- Unix 'ls -l' format
+ "Feb  3 17:03"      -- Unix 'ls -l' format
+
+ "11-15-96  03:52PM" -- Windows 'dir' format
+
+The parser ignores leading and trailing whitespace.  It also allow the
+seconds to be missing and the month to be numerical in most formats.
+
+The str2time() function takes an optional second argument that
+specifies the default time zone to use when converting the date.  This
+zone specification should be numerical (like "-0800" or "+0100") or
+"GMT".  This parameter is ignored if the zone is specified in the date
+string itself.  It this parameter is missing, and the date string
+format does not contain any zone specification then the local time
+zone is assumed.
+
+If the year is missing, then we assume that the date is the first
+matching date I<before> current time.
+
+=back
+
+=head1 BUGS
+
+Non-numerical time zones (like MET, PST) are all treated like GMT.
+Do not use them.  HTTP does not use them.
+
+The str2time() function has been told how to parse far too many
+formats.  This makes the module name misleading. To be sure it is
+really misleading you can also import the time2iso() and time2isoz()
+functions.  They work like time2str() but produce ISO-8601 formated
+strings (YYYY-MM-DD hh:mm:ss).
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+sub Version { $VERSION; }
+
+require 5.002;
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(time2str str2time);
+@EXPORT_OK = qw(time2iso time2isoz);
+
+use Time::Local ();
+
+use strict;
+use vars qw(@DoW @MoY %MoY);
+
+#@DoW = qw(Sunday Monday Tuesday Wednesday Thursday Friday Saturday);
+@DoW = qw(Sun Mon Tue Wed Thu Fri Sat);
+@MoY = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
+# Build %MoY hash
+my $i = 0;
+foreach(@MoY) {
+   $MoY{lc $_} = $i++;
+}
+
+my($current_month, $current_year) = (localtime)[4, 5];
+
+
+
+
+sub time2str (;$)
+{
+   my $time = shift;
+   $time = time unless defined $time;
+   my ($sec, $min, $hour, $mday, $mon, $year, $wday) = gmtime($time);
+   sprintf("%s, %02d %s %04d %02d:%02d:%02d GMT",
+	   $DoW[$wday],
+	   $mday, $MoY[$mon], $year+1900,
+	   $hour, $min, $sec);
+}
+
+
+
+sub str2time ($;$)
+{
+   local($_) = shift;
+   return undef unless defined;
+   my($default_zone) = @_;
+
+   # Remove useless weekday, if it exists
+   s/^\s*(?:sun|mon|tue|wed|thu|fri|sat)\w*,?\s*//i;
+
+   my($day, $mon, $yr, $hr, $min, $sec, $tz, $aorp);
+   my $offset = 0;  # used when compensating for timezone
+
+ PARSEDATE: {
+      # Then we are able to check for most of the formats with this regexp
+      ($day,$mon,$yr,$hr,$min,$sec,$tz) =
+	/^\s*
+	 (\d\d?)               # day
+	    (?:\s+|[-\/])
+	 (\w+)                 # month
+	    (?:\s+|[-\/])
+	 (\d+)                 # year
+	 (?:
+	       (?:\s+|:)       # separator before clock
+	    (\d\d?):(\d\d)     # hour:min
+	    (?::(\d\d))?       # optional seconds
+	 )?                    # optional clock
+	    \s*
+	 ([-+]?\d{2,4}|GMT|gmt)? # timezone
+	    \s*$
+	/x
+	  and last PARSEDATE;
+
+      # Try the ctime and asctime format
+      ($mon, $day, $hr, $min, $sec, $tz, $yr) =
+	/^\s*                  # allow intial whitespace
+	 (\w{1,3})             # month
+	    \s+
+	 (\d\d?)               # day
+	    \s+
+	 (\d\d?):(\d\d)        # hour:min
+	 (?::(\d\d))?          # optional seconds
+	    \s+
+	 (?:(GMT|gmt)\s+)?     # optional GMT timezone
+	 (\d+)                 # year
+	    \s*$               # allow trailing whitespace
+	/x
+	  and last PARSEDATE;
+
+      # Then the Unix 'ls -l' date format
+      ($mon, $day, $yr, $hr, $min, $sec) =
+	/^\s*
+	 (\w{3})               # month
+	    \s+
+	 (\d\d?)               # day
+	    \s+
+	 (?:
+	    (\d\d\d\d) |       # year
+	    (\d{1,2}):(\d{2})  # hour:min
+            (?::(\d\d))?       # optional seconds
+	 )
+	 \s*$
+       /x
+	 and last PARSEDATE;
+
+      # ISO 8601 format '1996-02-29 12:00:00 -0100' and variants
+      ($yr, $mon, $day, $hr, $min, $sec, $tz) =
+	/^\s*
+	  (\d{4})              # year
+	     [-\/]?
+	  (\d\d?)              # numerical month
+	     [-\/]?
+	  (\d\d?)              # day
+	 (?:
+	       (?:\s+|:|T|-)   # separator before clock
+	    (\d\d?):?(\d\d)    # hour:min
+	    (?::?(\d\d))?      # optional seconds
+	 )?                    # optional clock
+	    \s*
+	 ([-+]?\d\d?:?(:?\d\d)?
+	  |Z|z)?               # timezone  (Z is "zero meridian", i.e. GMT)
+	    \s*$
+	/x
+	  and last PARSEDATE;
+
+      # Windows 'dir' 11-12-96  03:52PM
+      ($mon, $day, $yr, $hr, $min, $aorp) =
+        /^\s*
+          (\d{2})                # numerical month
+             -
+          (\d{2})                # day
+             -
+          (\d{2})                # year
+             \s+
+          (\d\d?):(\d\d)([apAP][mM])  # hour:min AM or PM
+             \s*$
+        /x
+          and last PARSEDATE;
+
+      # If it is not recognized by now we give up
+      return undef;
+   }
+
+   # Translate month name to number
+   if ($mon =~ /^\d+$/) {
+     # numeric month
+     return undef if $mon < 1 || $mon > 12;
+     $mon--;
+   } else {
+     $mon = lc $mon;
+     return undef unless exists $MoY{$mon};
+     $mon = $MoY{$mon};
+   }
+
+   # If the year is missing, we assume some date before the current,
+   # because these date are mostly present on "ls -l" listings.
+   unless (defined $yr) {
+	$yr = $current_year;
+	$yr-- if $mon > $current_month;
+    }
+
+   # Then we check if the year is acceptable
+   return undef if $yr > 99 && $yr < 1900;  # We ignore these years
+   $yr += 100 if $yr < 50;  # a stupid thing to do???
+   $yr -= 1900 if $yr >= 1900;
+   # The $yr is now relative to 1900 (as expected by timelocal())
+
+   # timelocal() seems to go into an infinite loop if it is given out
+   # of range parameters.  Let's check the year at least.
+
+   # Epoch counter maxes out in year 2038, assuming "time_t" is 32 bit
+   return undef if $yr > 138;
+   return undef if $yr <  70;  # 1970 is Unix epoch
+
+   # Compensate for AM/PM
+   if ($aorp) {
+       $aorp = uc $aorp;
+       $hr = 0 if $hr == 12 && $aorp eq 'AM';
+       $hr += 12 if $aorp eq 'PM' && $hr != 12;
+   }
+
+   # Make sure things are defined
+   for ($sec, $min, $hr) {  $_ = 0 unless defined   }
+
+   # Should we compensate for the timezone?
+   $tz = $default_zone unless defined $tz;
+   return eval {Time::Local::timelocal($sec, $min, $hr, $day, $mon, $yr)}
+     unless defined $tz;
+
+   # We can calculate offset for numerical time zones
+   if ($tz =~ /^([-+])?(\d\d?):?(\d\d)?$/) {
+       $offset = 3600 * $2;
+       $offset += 60 * $3 if $3;
+       $offset *= -1 if $1 && $1 ne '-';
+   }
+   eval{Time::Local::timegm($sec, $min, $hr, $day, $mon, $yr) + $offset};
+}
+
+
+
+# And then some bloat because I happen to like the ISO 8601 time
+# format.
+
+sub time2iso (;$)
+{
+   my $time = shift;
+   $time = time unless defined $time;
+   my($sec,$min,$hour,$mday,$mon,$year) = localtime($time);
+   sprintf("%04d-%02d-%02d %02d:%02d:%02d",
+	   $year+1900, $mon+1, $mday, $hour, $min, $sec);
+}
+
+
+sub time2isoz (;$)
+{
+    my $time = shift;
+    $time = time unless defined $time;
+    my($sec,$min,$hour,$mday,$mon,$year) = gmtime($time);
+    sprintf("%04d-%02d-%02d %02d:%02d:%02dZ",
+            $year+1900, $mon+1, $mday, $hour, $min, $sec);
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers.pm	(revision 21112)
@@ -0,0 +1,540 @@
+#
+# $Id$
+
+package HTTP::Headers;
+
+=head1 NAME
+
+HTTP::Headers - Class encapsulating HTTP Message headers
+
+=head1 SYNOPSIS
+
+ require HTTP::Headers;
+ $h = new HTTP::Headers;
+
+=head1 DESCRIPTION
+
+The C<HTTP::Headers> class encapsulates HTTP-style message headers.
+The headers consist of attribute-value pairs, which may be repeated,
+and which are printed in a particular order.
+
+Instances of this class are usually created as member variables of the
+C<HTTP::Request> and C<HTTP::Response> classes, internal to the
+library.
+
+The following methods are available:
+
+=over 4
+
+=cut
+
+use strict;
+use vars qw($VERSION $TRANSLATE_UNDERSCORE);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use Carp ();
+
+# Could not use the AutoLoader becase several of the method names are
+# not unique in the first 8 characters.
+#use SelfLoader;
+
+
+# "Good Practice" order of HTTP message headers:
+#    - General-Headers
+#    - Request-Headers
+#    - Response-Headers
+#    - Entity-Headers
+# (From draft-ietf-http-v11-spec-rev-01, Nov 21, 1997)
+
+my @header_order = qw(
+   Cache-Control Connection Date Pragma Transfer-Encoding Upgrade Trailer Via
+
+   Accept Accept-Charset Accept-Encoding Accept-Language
+   Authorization Expect From Host
+   If-Modified-Since If-Match If-None-Match If-Range If-Unmodified-Since
+   Max-Forwards Proxy-Authorization Range Referer TE User-Agent
+
+   Accept-Ranges Age Location Proxy-Authenticate Retry-After Server Vary
+   Warning WWW-Authenticate
+
+   Allow Content-Base Content-Encoding Content-Language Content-Length
+   Content-Location Content-MD5 Content-Range Content-Type
+   ETag Expires Last-Modified
+);
+
+# Make alternative representations of @header_order.  This is used
+# for sorting and case matching.
+my $i = 0;
+my %header_order;
+my %standard_case;
+for (@header_order) {
+    my $lc = lc $_;
+    $header_order{$lc} = $i++;
+    $standard_case{$lc} = $_;
+}
+
+$TRANSLATE_UNDERSCORE = 1 unless defined $TRANSLATE_UNDERSCORE;
+
+
+
+=item $h = new HTTP::Headers
+
+Constructs a new C<HTTP::Headers> object.  You might pass some initial
+attribute-value pairs as parameters to the constructor.  I<E.g.>:
+
+ $h = new HTTP::Headers
+     Date         => 'Thu, 03 Feb 1994 00:00:00 GMT',
+     Content_Type => 'text/html; version=3.2',
+     Content_Base => 'http://www.sn.no/';
+
+=cut
+
+sub new
+{
+    my($class) = shift;
+    my $self = bless {}, $class;
+    $self->header(@_); # set up initial headers
+    $self;
+}
+
+
+=item $h->header($field [=> $value],...)
+
+Get or set the value of a header.  The header field name is not case
+sensitive.  To make the life easier for perl users who wants to avoid
+quoting before the => operator, you can use '_' as a synonym for '-'
+in header names (this behaviour can be suppressed by setting
+$HTTP::Headers::TRANSLATE_UNDERSCORE to a FALSE value).
+
+The header() method accepts multiple ($field => $value) pairs, so you
+can update several fields with a single invocation.
+
+The optional $value argument may be a scalar or a reference to a list
+of scalars. If the $value argument is undefined or not given, then the
+header is not modified.
+
+The old value of the last of the $field values is returned.
+Multi-valued fields will be concatenated with "," as separator in
+scalar context.
+
+ $header->header(MIME_Version => '1.0',
+		 User_Agent   => 'My-Web-Client/0.01');
+ $header->header(Accept => "text/html, text/plain, image/*");
+ $header->header(Accept => [qw(text/html text/plain image/*)]);
+ @accepts = $header->header('Accept');
+
+=cut
+
+sub header
+{
+    my $self = shift;
+    my($field, $val, @old);
+    while (($field, $val) = splice(@_, 0, 2)) {
+	@old = $self->_header($field, $val);
+    }
+    return @old if wantarray;
+    return $old[0] if @old <= 1;
+    join(", ", @old);
+}
+
+sub _header
+{
+    my($self, $field, $val, $push) = @_;
+    $field =~ tr/_/-/ if $TRANSLATE_UNDERSCORE;
+
+    # $push is only used interally sub push_header
+    Carp::croak('Need a field name') unless length($field);
+
+    my $lc_field = lc $field;
+    unless(defined $standard_case{$lc_field}) {
+	# generate a %stadard_case entry for this field
+	$field =~ s/\b(\w)/\u$1/g;
+	$standard_case{$lc_field} = $field;
+    }
+
+    my $h = $self->{$lc_field};
+    my @old = ref($h) ? @$h : (defined($h) ? ($h) : ());
+
+    if (defined $val) {
+	my @new = $push ? @old : ();
+	if (!ref($val)) {
+	    push(@new, $val);
+	} elsif (ref($val) eq 'ARRAY') {
+	    push(@new, @$val);
+	} else {
+	    Carp::croak("Unexpected field value $val");
+	}
+	$self->{$lc_field} = @new > 1 ? \@new : $new[0];
+    }
+    @old;
+}
+
+
+# Compare function which makes it easy to sort headers in the
+# recommended "Good Practice" order.
+sub _header_cmp
+{
+    # Unknown headers are assign a large value so that they are
+    # sorted last.  This also helps avoiding a warning from -w
+    # about comparing undefined values.
+    $header_order{$a} = 999 unless defined $header_order{$a};
+    $header_order{$b} = 999 unless defined $header_order{$b};
+
+    $header_order{$a} <=> $header_order{$b} || $a cmp $b;
+}
+
+
+=item $h->scan(\&doit)
+
+Apply a subroutine to each header in turn.  The callback routine is
+called with two parameters; the name of the field and a single value.
+If the header has more than one value, then the routine is called once
+for each value.  The field name passed to the callback routine has
+case as suggested by HTTP Spec, and the headers will be visited in the
+recommended "Good Practice" order.
+
+=cut
+
+sub scan
+{
+    my($self, $sub) = @_;
+    my $key;
+    foreach $key (sort _header_cmp keys %$self) {
+        next if $key =~ /^_/;
+	my $vals = $self->{$key};
+	if (ref($vals)) {
+	    my $val;
+	    for $val (@$vals) {
+		&$sub($standard_case{$key} || $key, $val);
+	    }
+	} else {
+	    &$sub($standard_case{$key} || $key, $vals);
+	}
+    }
+}
+
+
+=item $h->as_string([$endl])
+
+Return the header fields as a formatted MIME header.  Since it
+internally uses the C<scan()> method to build the string, the result
+will use case as suggested by HTTP Spec, and it will follow
+recommended "Good Practice" of ordering the header fieds.  Long header
+values are not folded. 
+
+The optional parameter specifies the line ending sequence to use.  The
+default is C<"\n">.  Embedded "\n" characters in the header will be
+substitued with this line ending sequence.
+
+=cut
+
+sub as_string
+{
+    my($self, $endl) = @_;
+    $endl = "\n" unless defined $endl;
+
+    my @result = ();
+    $self->scan(sub {
+	my($field, $val) = @_;
+	if ($val =~ /\n/) {
+	    # must handle header values with embedded newlines with care
+	    $val =~ s/\s+$//;          # trailing newlines and space must go
+	    $val =~ s/\n\n+/\n/g;      # no empty lines
+	    $val =~ s/\n([^\040\t])/\n $1/g;  # intial space for continuation
+	    $val =~ s/\n/$endl/g;      # substitute with requested line ending
+	}
+	push(@result, "$field: $val");
+    });
+
+    join($endl, @result, '');
+}
+
+
+# The remaining functions should autoloaded only when needed
+
+# A bug in 5.002gamma makes it risky to have POD text inside the
+# autoloaded section of the code, so we keep the documentation before
+# the __DATA__ token.
+
+=item $h->push_header($field, $val)
+
+Add a new field value of the specified header.  The header field name
+is not case sensitive.  The field need not already have a
+value. Previous values for the same field are retained.  The argument
+may be a scalar or a reference to a list of scalars.
+
+ $header->push_header(Accept => 'image/jpeg');
+
+=item $h->remove_header($field,...)
+
+This function removes the headers with the specified names.
+
+=item $h->clone
+
+Returns a copy of this HTTP::Headers object.
+
+=back
+
+=head1 CONVENIENCE METHODS
+
+The most frequently used headers can also be accessed through the
+following convenience methods.  These methods can both be used to read
+and to set the value of a header.  The header value is set if you pass
+an argument to the method.  The old header value is always returned.
+
+Methods that deal with dates/times always convert their value to system
+time (seconds since Jan 1, 1970) and they also expect this kind of
+value when the header value is set.
+
+=over 4
+
+=item $h->date
+
+This header represents the date and time at which the message was
+originated. I<E.g.>:
+
+  $h->date(time);  # set current date
+
+=item $h->expires
+
+This header gives the date and time after which the entity should be
+considered stale.
+
+=item $h->if_modified_since
+
+=item $h->if_unmodified_since
+
+This header is used to make a request conditional.  If the requested
+resource has (not) been modified since the time specified in this field,
+then the server will return a C<"304 Not Modified"> response instead of
+the document itself.
+
+=item $h->last_modified
+
+This header indicates the date and time at which the resource was last
+modified. I<E.g.>:
+
+  # check if document is more than 1 hour old
+  if ($h->last_modified < time - 60*60) {
+	...
+  }
+
+=item $h->content_type
+
+The Content-Type header field indicates the media type of the message
+content. I<E.g.>:
+
+  $h->content_type('text/html');
+
+The value returned will be converted to lower case, and potential
+parameters will be chopped off and returned as a separate value if in
+an array context.  This makes it safe to do the following:
+
+  if ($h->content_type eq 'text/html') {
+     # we enter this place even if the real header value happens to
+     # be 'TEXT/HTML; version=3.0'
+     ...
+  }
+
+=item $h->content_encoding
+
+The Content-Encoding header field is used as a modifier to the
+media type.  When present, its value indicates what additional
+encoding mechanism has been applied to the resource.
+
+=item $h->content_length
+
+A decimal number indicating the size in bytes of the message content.
+
+=item $h->content_language
+
+The natural language(s) of the intended audience for the message
+content.  The value is one or more language tags as defined by RFC
+1766.  Eg. "no" for Norwegian and "en-US" for US-English.
+
+=item $h->title
+
+The title of the document.  In libwww-perl this header will be
+initialized automatically from the E<lt>TITLE>...E<lt>/TITLE> element
+of HTML documents.  I<This header is no longer part of the HTTP
+standard.>
+
+=item $h->user_agent
+
+This header field is used in request messages and contains information
+about the user agent originating the request.  I<E.g.>:
+
+  $h->user_agent('Mozilla/1.2');
+
+=item $h->server
+
+The server header field contains information about the software being
+used by the originating server program handling the request.
+
+=item $h->from
+
+This header should contain an Internet e-mail address for the human
+user who controls the requesting user agent.  The address should be
+machine-usable, as defined by RFC822.  E.g.:
+
+  $h->from('Gisle Aas <aas@sn.no>');
+
+=item $h->referer
+
+Used to specify the address (URI) of the document from which the
+requested resouce address was obtained.
+
+=item $h->www_authenticate
+
+This header must be included as part of a "401 Unauthorized" response.
+The field value consist of a challenge that indicates the
+authentication scheme and parameters applicable to the requested URI.
+
+=item $h->proxy_authenticate
+
+This header must be included in a "407 Proxy Authentication Required"
+response.
+
+=item $h->authorization
+
+=item $h->proxy_authorization
+
+A user agent that wishes to authenticate itself with a server or a
+proxy, may do so by including these headers.
+
+=item $h->authorization_basic
+
+This method is used to get or set an authorization header that use the
+"Basic Authentication Scheme".  In array context it will return two
+values; the user name and the password.  In scalar context it will
+return I<"uname:password"> as a single string value.
+
+When used to set the header value, it expects two arguments.  I<E.g.>:
+
+  $h->authorization_basic($uname, $password);
+
+The method will croak if the $uname contains a colon ':'.
+
+=item $h->proxy_authorization_basic
+
+Same as authorization_basic() but will set the "Proxy-Authorization"
+header instead.
+
+=back
+
+=head1 COPYRIGHT
+
+Copyright 1995-1998 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
+1;
+
+#__DATA__
+
+sub clone
+{
+    my $self = shift;
+    my $clone = new HTTP::Headers;
+    $self->scan(sub { $clone->push_header(@_);} );
+    $clone;
+}
+
+sub push_header
+{
+    Carp::croak('Usage: $h->push_header($field, $val)') if @_ != 3;
+    shift->_header(@_, 'PUSH');
+}
+
+
+sub remove_header
+{
+    my($self, @fields) = @_;
+    my $field;
+    foreach $field (@fields) {
+	$field =~ tr/_/-/ if $TRANSLATE_UNDERSCORE;
+	delete $self->{lc $field};
+    }
+}
+
+# Convenience access functions
+
+sub _date_header
+{
+    require HTTP::Date;
+    my($self, $header, $time) = @_;
+    my($old) = $self->_header($header);
+    if (defined $time) {
+	$self->_header($header, HTTP::Date::time2str($time));
+    }
+    HTTP::Date::str2time($old);
+}
+
+sub date                { shift->_date_header('Date',                @_); }
+sub expires             { shift->_date_header('Expires',             @_); }
+sub if_modified_since   { shift->_date_header('If-Modified-Since',   @_); }
+sub if_unmodified_since { shift->_date_header('If-Unmodified-Since', @_); }
+sub last_modified       { shift->_date_header('Last-Modified',       @_); }
+
+# This is used as a private LWP extention.  The Client-Date header is
+# added as a timestamp to a response when it has been received.
+sub client_date         { shift->_date_header('Client-Date',         @_); }
+
+# The retry_after field is dual format (can also be a expressed as
+# number of seconds from now), so we don't provide an easy way to
+# access it until we have know how both these interfaces can be
+# addressed.  One possibility is to return a negative value for
+# relative seconds and a positive value for epoch based time values.
+#sub retry_after       { shift->_date_header('Retry-After',       @_); }
+
+sub content_type      {
+  my $ct = (shift->_header('Content-Type', @_))[0];
+  return '' unless defined($ct) && length($ct);
+  my @ct = split(/\s*;\s*/, lc($ct));
+  wantarray ? @ct : $ct[0];
+}
+
+sub title             { (shift->_header('Title',            @_))[0] }
+sub content_encoding  { (shift->_header('Content-Encoding', @_))[0] }
+sub content_language  { (shift->_header('Content-Language', @_))[0] }
+sub content_length    { (shift->_header('Content-Length',   @_))[0] }
+
+sub user_agent        { (shift->_header('User-Agent',       @_))[0] }
+sub server            { (shift->_header('Server',           @_))[0] }
+
+sub from              { (shift->_header('From',             @_))[0] }
+sub referer           { (shift->_header('Referer',          @_))[0] }
+sub warning           { (shift->_header('Warning',          @_))[0] }
+
+sub www_authenticate  { (shift->_header('WWW-Authenticate', @_))[0] }
+sub authorization     { (shift->_header('Authorization',    @_))[0] }
+
+sub proxy_authenticate  { (shift->_header('Proxy-Authenticate',  @_))[0] }
+sub proxy_authorization { (shift->_header('Proxy-Authorization', @_))[0] }
+
+sub authorization_basic       { shift->_basic_auth("Authorization",       @_) }
+sub proxy_authorization_basic { shift->_basic_auth("Proxy-Authorization", @_) }
+
+sub _basic_auth {
+    require MIME::Base64;
+    my($self, $h, $user, $passwd) = @_;
+    my($old) = $self->_header($h);
+    if (defined $user) {
+	Carp::croak("Basic authorization user name can't contain ':'")
+	  if $user =~ /:/;
+	$passwd = '' unless defined $passwd;
+	$self->_header($h => 'Basic ' .
+                             MIME::Base64::encode("$user:$passwd", ''));
+    }
+    if (defined $old && $old =~ s/^\s*Basic\s+//) {
+	my $val = MIME::Base64::decode($old);
+	return $val unless wantarray;
+	return split(/:/, $val, 2);
+    }
+    return;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Auth.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Auth.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Auth.pm	(revision 21112)
@@ -0,0 +1,94 @@
+package HTTP::Headers::Auth;
+
+use strict;
+use vars qw($VERSION);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+require HTTP::Headers;
+package HTTP::Headers;
+
+BEGIN {
+    # we provide a new (and better) implementations below
+    undef(&www_authenticate);
+    undef(&proxy_authenticate);
+}
+
+require HTTP::Headers::Util;
+
+sub _parse_authenticate
+{
+    my @ret;
+    for (HTTP::Headers::Util::split_header_words(@_)) {
+	if (!defined($_->[1])) {
+	    # this is a new auth scheme
+	    push(@ret, lc(shift @$_) => {});
+	    shift @$_;
+	}
+	if (@ret) {
+	    # this a new parameter pair for the last auth scheme
+	    while (@$_) {
+		my $k = lc(shift @$_);
+		my $v = shift @$_;
+	        $ret[-1]{$k} = $v;
+	    }
+	} else {
+	    # something wrong, parameter pair without any scheme seen
+	    # IGNORE
+	}
+    }
+    @ret;
+}
+
+sub _authenticate
+{
+    my $self = shift;
+    my $header = shift;
+    my @old = $self->_header($header);
+    if (@_) {
+	$self->remove_header($header);
+	my @new = @_;
+	while (@new) {
+	    my $a_scheme = shift(@new);
+	    if ($a_scheme =~ /\s/) {
+		# assume complete valid value, pass it through
+		$self->push_header($header, $a_scheme);
+	    } else {
+		my @param;
+		if (@new) {
+		    my $p = $new[0];
+		    if (ref($p) eq "ARRAY") {
+			@param = @$p;
+			shift(@new);
+		    } elsif (ref($p) eq "HASH") {
+			@param = %$p;
+			shift(@new);
+		    }
+		}
+		my $val = ucfirst(lc($a_scheme));
+		if (@param) {
+		    my $sep = " ";
+		    while (@param) {
+			my $k = shift @param;
+			my $v = shift @param;
+			if ($v =~ /[^0-9a-zA-Z]/ || lc($k) eq "realm") {
+			    # must quote the value
+			    $v =~ s,([\\\"]),\\$1,g;
+			    $v = qq("$v");
+			}
+			$val .= "$sep$k=$v";
+			$sep = ", ";
+		    }
+		}
+		$self->push_header($header, $val);
+	    }
+	}
+    }
+    return unless defined wantarray;
+    wantarray ? _parse_authenticate(@old) : join(", ", @old);
+}
+
+
+sub www_authenticate    { shift->_authenticate("WWW-Authenticate", @_)   }
+sub proxy_authenticate  { shift->_authenticate("Proxy-Authenticate", @_) }
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/ETag.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/ETag.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/ETag.pm	(revision 21112)
@@ -0,0 +1,88 @@
+package HTTP::Header::ETag;
+
+use strict;
+use vars qw($VERSION);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+require HTTP::Date;
+
+require HTTP::Headers;
+package HTTP::Headers;
+
+sub _etags
+{
+    my $self = shift;
+    my $header = shift;
+    my @old = _split_etag_list($self->_header($header));
+    if (@_) {
+	$self->_header($header => join(", ", _split_etag_list(@_)));
+    }
+    wantarray ? @old : join(", ", @old);
+}
+
+sub etag          { shift->_etags("ETag", @_); }
+sub if_match      { shift->_etags("If-Match", @_); }
+sub if_none_match { shift->_etags("If-None-Match", @_); }
+
+sub if_range {
+    # Either a date or an entity-tag
+    my $self = shift;
+    my @old = $self->_header("If-Range");
+    if (@_) {
+	my $new = shift;
+	if (!defined $new) {
+	    $self->remove_header("If-Range");
+	} elsif ($new =~ /^\d+$/) {
+	    $self->_date_header("If-Range", $new);
+	} else {
+	    $self->_etags("If-Range", $new);
+	}
+    }
+    return unless defined(wantarray);
+    for (@old) {
+	my $t = HTTP::Date::str2time($_);
+	$_ = $t if $t;
+    }
+    wantarray ? @old : join(", ", @old);
+}
+
+
+# Split a list of entity tag values.  The return value is a list
+# consisting of one element per entity tag.  Suitable for parsing
+# headers like C<If-Match>, C<If-None-Match>.  You might even want to
+# use it on C<ETag> and C<If-Range> entity tag values, because it will
+# normalize them to the common form.
+#
+#  entity-tag	  = [ weak ] opaque-tag
+#  weak		  = "W/"
+#  opaque-tag	  = quoted-string
+
+
+sub _split_etag_list
+{
+    my(@val) = @_;
+    my @res;
+    for (@val) {
+        while (length) {
+            my $weak = "";
+	    $weak = "W/" if s,^\s*[wW]/,,;
+            my $etag = "";
+	    if (s/^\s*(\"[^\"\\]*(?:\\.[^\"\\]*)*\")//) {
+		push(@res, "$weak$1");
+            } elsif (s/^\s*,//) {
+                push(@res, qq(W/"")) if $weak;
+            } elsif (s/^\s*([^,\s]+)//) {
+                $etag = $1;
+		$etag =~ s/([\"\\])/\\$1/g;
+	        push(@res, qq($weak"$etag"));
+            } elsif (s/^\s+// || !length) {
+                push(@res, qq(W/"")) if $weak;
+            } else {
+	 	die "This should not happen: '$_'";
+            }
+        }
+   }
+   @res;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Util.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Util.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Headers/Util.pm	(revision 21112)
@@ -0,0 +1,180 @@
+package HTTP::Headers::Util;
+
+use strict;
+use vars qw($VERSION @ISA @EXPORT_OK);
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+require Exporter;
+@ISA=qw(Exporter);
+
+@EXPORT_OK=qw(split_header_words join_header_words);
+
+=head1 NAME
+
+HTTP::Headers::Util - Header value parsing utility functions
+
+=head1 SYNOPSIS
+
+  use HTTP::Headers::Util qw(split_header_words);
+  @values = split_header_words($h->header("Content-Type"));
+
+=head1 DESCRIPTION
+
+This module provide a few functions that helps parsing and
+construction of valid HTTP header values.  None of the functions are
+exported by default.
+
+The following functions are provided:
+
+=over 4
+
+
+=item split_header_words( @header_values )
+
+This function will parse the header values given as argument into a
+list of anonymous arrays containing key/value pairs.  The function
+know how to deal with ",", ";" and "=" as well as quoted values after
+"=".  A list of space separated tokens are parsed as if they were
+separated by ";".
+
+If the @header_values passed as argument contains multiple values,
+then they are treated as if they were a single value separated by
+comma ",".
+
+This means that this function is useful to parse header fields that
+follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
+the requirement for tokens).
+
+  headers           = #header
+  header            = (token | parameter) *( [";"] (token | parameter))
+
+  token             = 1*<any CHAR except CTLs or separators>
+  separators        = "(" | ")" | "<" | ">" | "@"
+                    | "," | ";" | ":" | "\" | <">
+                    | "/" | "[" | "]" | "?" | "="
+                    | "{" | "}" | SP | HT
+
+  quoted-string     = ( <"> *(qdtext | quoted-pair ) <"> )
+  qdtext            = <any TEXT except <">>
+  quoted-pair       = "\" CHAR
+
+  parameter         = attribute "=" value
+  attribute         = token
+  value             = token | quoted-string
+
+Each I<header> is represented by an anonymous array of key/value
+pairs.  If a token is recognized then the value will be C<undef>.
+Syntactically incorrect headers will not necessary be parsed as you
+would want.
+
+This is easier to describe with some examples:
+
+   split_header_words('foo="bar"; port="80,81"; discard, bar=baz')
+   split_header_words('text/html; charset="iso-8859-1");
+   split_header_words('Basic realm="\"foo\\bar\""');
+
+will return
+
+   [foo=>'bar', port=>'80,81', discard=> undef], [bar=>'baz' ]
+   ['text/html' => undef, charset => 'iso-8859-1']
+   [Basic => undef, realm => '"foo\bar"']
+
+=cut
+
+
+sub split_header_words
+{
+    my(@val) = @_;
+    my @res;
+    for (@val) {
+	my @cur;
+	while (length) {
+	    if (s/^\s*(=*[^\s=;,]+)//) {  # 'token' or parameter 'attribute'
+		push(@cur, $1);
+		# a quoted value
+		if (s/^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"//) {
+		    my $val = $1;
+		    $val =~ s/\\(.)/$1/g;
+		    push(@cur, $val);
+		# some unquoted value
+		} elsif (s/^\s*=\s*([^;,\s]*)//) {
+		    my $val = $1;
+		    $val =~ s/\s+$//;
+		    push(@cur, $val);
+		# no value, a lone token
+		} else {
+		    push(@cur, undef);
+		}
+	    } elsif (s/^\s*,//) {
+		push(@res, [@cur]) if @cur;
+		@cur = ();
+	    } elsif (s/^\s*;// || s/^\s+//) {
+		# continue
+	    } else {
+		die "This should not happen: '$_'";
+	    }
+	}
+	push(@res, \@cur) if @cur;
+    }
+    @res;
+}
+
+
+=item join_header_words( @arrays )
+
+This will do the opposite convertion of what split_header_words()
+does.  It takes a list of anonymous arrays as argument (or a list of
+key/value pairs) and produce a single header value.  Attribute values
+are quoted if needed.
+
+Example:
+
+   join_header_words(["text/plain" => undef, charset => "iso-8859/1"]);
+   join_header_words(""text/plain" => undef, charset => "iso-8859/1");
+
+will both return the string:
+
+   text/plain; charset="iso-8859/1"
+
+=cut
+
+sub join_header_words
+{
+    @_ = ([@_]) if @_ && !ref($_[0]);
+    my @res;
+    for (@_) {
+	my @cur = @$_;
+	my @attr;
+	while (@cur) {
+	    my $k = shift @cur;
+	    my $v = shift @cur;
+	    if (defined $v) {
+		if ($v =~ /^\w+$/) {
+		    $k .= "=$v";
+		} else {
+		    $v =~ s/([\"\\])/\\$1/g;  # escape " and \
+		    $k .= qq(="$v");
+		}
+	    }
+	    push(@attr, $k);
+	}
+	push(@res, join("; ", @attr)) if @attr;
+    }
+    join(", ", @res);
+}
+
+1;
+
+__END__
+
+=back
+
+=head1 COPYRIGHT
+
+Copyright 1997-1998, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Message.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Message.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Message.pm	(revision 21112)
@@ -0,0 +1,208 @@
+#
+# $Id$
+
+package HTTP::Message;
+
+=head1 NAME
+
+HTTP::Message - Class encapsulating HTTP messages
+
+=head1 SYNOPSIS
+
+ package HTTP::Request;  # or HTTP::Response
+ require HTTP::Message;
+ @ISA=qw(HTTP::Message);
+
+=head1 DESCRIPTION
+
+A C<HTTP::Message> object contains some headers and a content (body).
+The class is abstract, i.e. it only used as a base class for
+C<HTTP::Request> and C<HTTP::Response> and should never instantiated
+as itself.
+
+The following methods are available:
+
+=over 4
+
+=cut
+
+#####################################################################
+
+require HTTP::Headers;
+require Carp;
+use strict;
+use vars qw($VERSION $AUTOLOAD);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+=item $mess = new HTTP::Message;
+
+This is the object constructor.  It should only be called internally
+by this library.  External code should construct C<HTTP::Request> or
+C<HTTP::Response> objects.
+
+=cut
+
+sub new
+{
+    my($class, $header, $content) = @_;
+    if (defined $header) {
+	Carp::croak("Bad header argument") unless ref $header;
+	$header = $header->clone;
+    } else {
+	$header = HTTP::Headers->new;
+    }
+    $content = '' unless defined $content;
+    bless {
+	'_headers' => $header,
+	'_content' => $content,
+    }, $class;
+}
+
+
+=item $mess->clone()
+
+Returns a copy of the object.
+
+=cut
+
+sub clone
+{
+    my $self  = shift;
+    my $clone = HTTP::Message->new($self->{'_headers'}, $self->{'_content'});
+    $clone;
+}
+
+=item $mess->protocol([$proto])
+
+Sets the HTTP protocol used for the message.  The protocol() is a string
+like "HTTP/1.0" or "HTTP/1.1".
+
+=cut
+
+sub protocol { shift->_elem('_protocol',  @_); }
+
+=item $mess->content([$content])
+
+The content() method sets the content if an argument is given.  If no
+argument is given the content is not touched.  In either case the
+previous content is returned.
+
+=item $mess->add_content($data)
+
+The add_content() methods appends more data to the end of the previous
+content.
+
+=cut
+
+sub content   { shift->_elem('_content',  @_); }
+
+sub add_content
+{
+    my $self = shift;
+    if (ref($_[0])) {
+	$self->{'_content'} .= ${$_[0]};  # for backwards compatability
+    } else {
+	$self->{'_content'} .= $_[0];
+    }
+}
+
+=item $mess->content_ref
+
+The content_ref() method will return a reference to content string.
+It can be more efficient to access the content this way if the content
+is huge, and it can be used for direct manipulation of the content,
+for instance:
+
+  ${$res->content_ref} =~ s/\bfoo\b/bar/g;
+
+=cut
+
+sub content_ref
+{
+    my $self = shift;
+    \$self->{'_content'};
+}
+
+sub as_string
+{
+    "";  # To be overridden in subclasses
+}
+
+=item $mess->headers;
+
+Return the embedded HTTP::Headers object.
+
+=item $mess->headers_as_string([$endl])
+
+Call the HTTP::Headers->as_string() method for the headers in the
+message.
+
+=cut
+
+sub headers            { shift->{'_headers'};                }
+sub headers_as_string  { shift->{'_headers'}->as_string(@_); }
+
+=back
+
+All unknown C<HTTP::Message> methods are delegated to the
+C<HTTP::Headers> object that is part of every message.  This allows
+convenient access to these methods.  Refer to L<HTTP::Headers> for
+details of these methods:
+
+  $mess->header($field => $val);
+  $mess->scan(\&doit);
+  $mess->push_header($field => $val);
+  $mess->remove_header($field);
+
+  $mess->date;
+  $mess->expires;
+  $mess->if_modified_since;
+  $mess->if_unmodified_since;
+  $mess->last_modified;
+  $mess->content_type;
+  $mess->content_encoding;
+  $mess->content_length;
+  $mess->content_language
+  $mess->title;
+  $mess->user_agent;
+  $mess->server;
+  $mess->from;
+  $mess->referer;
+  $mess->www_authenticate;
+  $mess->authorization;
+  $mess->proxy_authorization;
+  $mess->authorization_basic;
+  $mess->proxy_authorization_basic;
+
+=cut
+
+
+# delegate all other method calls the the _headers object.
+sub AUTOLOAD
+{
+    my $self = shift;
+    my $method = substr($AUTOLOAD, rindex($AUTOLOAD, '::')+2);
+    return if $method eq "DESTROY";
+    $self->{'_headers'}->$method(@_);
+}
+
+# Private method to access members in %$self
+sub _elem
+{
+    my $self = shift;
+    my $elem = shift;
+    my $old = $self->{$elem};
+    $self->{$elem} = $_[0] if @_;
+    return $old;
+}
+
+1;
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Negotiate.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Negotiate.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Negotiate.pm	(revision 21112)
@@ -0,0 +1,505 @@
+# $Id$
+#
+
+package HTTP::Negotiate;
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+sub Version { $VERSION; }
+
+require 5.002;
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(choose);
+
+require HTTP::Headers;
+
+$DEBUG = 0;
+
+sub choose ($;$)
+{
+    my($variants, $request) = @_;
+    my(%accept);
+
+    unless (defined $request) {
+	# Create a request object from the CGI envirionment variables
+	$request = new HTTP::Headers;
+	$request->header('Accept', $ENV{HTTP_ACCEPT})
+	  if $ENV{HTTP_ACCEPT};
+	$request->header('Accept-Charset', $ENV{HTTP_ACCEPT_CHARSET})
+	  if $ENV{HTTP_ACCEPT_CHARSET};
+	$request->header('Accept-Encoding', $ENV{HTTP_ACCEPT_ENCODING})
+	  if $ENV{HTTP_ACCEPT_ENCODING};
+	$request->header('Accept-Language', $ENV{HTTP_ACCEPT_LANGUAGE})
+	  if $ENV{HTTP_ACCEPT_LANGUAGE};
+    }
+
+    # Get all Accept values from the request.  Build a hash initialized
+    # like this:
+    #
+    #   %accept = ( type =>     { 'audio/*'     => { q => 0.2, mbx => 20000 },
+    #                             'audio/basic' => { q => 1 },
+    #                           },
+    #               language => { 'no'          => { q => 1 },
+    #                           }
+    #             );
+
+    $request->scan(sub {
+	my($key, $val) = @_;
+	return unless $key =~ s/^Accept-?//;
+	my $type = lc $key;
+	$type = "type" unless length $key;
+	$val =~ s/\s+//g;
+	my $name;
+	for $name (split(/,/, $val)) {
+	    my(%param, $param);
+	    if ($name =~ s/;(.*)//) {
+		for $param (split(/;/, $1)) {
+		    my ($pk, $pv) = split(/=/, $param, 2);
+		    $param{$pk} = $pv;
+		}
+	    }
+	    $name = lc $name;
+	    if (defined $param{'q'}) {
+		$param{'q'} = 1 if $param{'q'} > 1;
+		$param{'q'} = 0 if $param{'q'} < 0;
+	    } else {
+		$param{'q'} = 1;
+	    }
+
+	    $param{'q'} = 1 unless defined $param{'q'};
+	    $accept{$type}{$name} = \%param;
+	}
+    });
+
+    # Check if any of the variants specify a language.  We do this
+    # because it influence how we treat those without (they default to
+    # 0.5 instead of 1).
+    my $any_lang = 0;
+    for $var (@$variants) {
+	if ($var->[5]) {
+	    $any_lang = 1;
+	    last;
+	}
+    }
+
+    if ($DEBUG) {
+	print "Netgotiantion parameters in the request\n";
+	for $type (keys %accept) {
+	    print " $type:\n";
+	    for $name (keys %{$accept{$type}}) {
+		print "    $name\n";
+		for $pv (keys %{$accept{$type}{$name}}) {
+		    print "      $pv = $accept{$type}{$name}{$pv}\n";
+		}
+	    }
+	}
+    }
+
+    my @Q = ();  # This is where we collect the results of the
+		 # quality calcualtions
+
+    # Calculate quality for all the variant's that are available.
+    for (@$variants) {
+	my($id, $qs, $ct, $enc, $cs, $lang, $bs) = @$_;
+	$qs = 1 unless defined $qs;
+	$bs = 0 unless defined $bs;
+	if ($DEBUG) {
+	    print "\nEvaluating $id ($ct)\n";
+	    printf "  qs   = %.3f\n", $qs;
+	    print  "  enc  = $enc\n"  if $enc && !ref($enc);
+	    print  "  enc  = @$enc\n" if $enc && ref($enc);
+	    print  "  cs   = $cs\n"   if $cs;
+	    print  "  lang = $lang\n" if $lang;
+	    print  "  bs   = $bs\n"   if $bs;
+	}
+
+	# Calculate encoding quality
+	my $qe = 1;
+	# If the variant has no assignes Content-Encoding, or if no
+	# Accept-Encoding field is present, then the value assigned
+	# is "qe=1".  If *all* of the variant's content encoddings
+	# are listed in the Accept-Encoding field, then the value
+	# assigned is "qw=1".  If *any* of the variant's content
+	# encodings are not listed in the provided Accept-Encoding
+	# field, then the value assigned is "qe=0"
+	if (exists $accept{'encoding'} && $enc) {
+	    my @enc = ref($enc) ? @$enc : ($enc);
+	    for (@enc) {
+		print "Is encoding $_ accepted? " if $DEBUG;
+		unless(exists $accept{'encoding'}{$_}) {
+		    print "no\n" if $DEBUG;
+		    $qe = 0;
+		    last;
+		} else {
+		    print "yes\n" if $DEBUG;
+		}
+	    }
+	}
+
+	# Calculate charset quality
+	my $qc  = 1;
+	# If the variant's media-type has not charset parameter,
+	# or the variant's charset is US-ASCII, or if no Accept-Charset
+	# field is present, then the value assigned is "qc=1".  If the
+	# variantæs charset is listed in the Accept-Charset field,
+	# then the value assigned is "qc=1.  Otherwise, if the variant's
+	# charset is not listed in the provided Accept-Encoding field,
+	# then the value assigned is "qc=0".
+	if (exists $accept{'charset'} && $cs && $cs ne 'us-ascii' ) {
+	    $qc = 0 unless $accept{'charset'}{$cs};
+	}
+
+	# Calculate language quality
+	my $ql  = 1;
+	if ($lang && exists $accept{'language'}) {
+	    my @lang = ref($lang) ? @$lang : ($lang);
+	    # If any of the variant's content languages are listed
+	    # in the Accept-Language field, the the value assigned is
+	    # the maximus of the "q" paramet values for thos language
+	    # tags.
+	    my $q = undef;
+	    for (@lang) {
+		next unless exists $accept{'language'}{$_};
+		my $this_q = $accept{'language'}{$_}{'q'};
+		$q = $this_q unless defined $q;
+		$q = $this_q if $this_q > $q;
+	    }
+	    unless (defined $q) {
+		# If there was no exact match and at least one of
+		# the Accept-Language field values is a complete
+		# subtag prefix of the content language tag(s), then
+		# the "q" parameter value of the largest matching
+		# prefix is used.
+		my $selected = undef;
+		for $al (keys %{ $accept{'language'} }) {
+		    if (substr($lang, 0, length($al)) eq $al) {
+			$selected = $al unless defined $selected;
+			$selected = $al if length($al) > length($selected);
+		    }
+		}
+		$q = $accept{'language'}{$selected}{'q'} if $selected;
+
+		# If none of the variant's content language tags or
+		# tag prefixes are listed in the provided
+		# Accept-Language field, then the value assigned
+		# is "ql=0.001"
+		$q = 0.001 unless defined $q;
+	    }
+	    $ql = $q;
+	} else {
+	    $ql = 0.5 if $any_lang && exists $accept{'language'};
+	}
+
+	my $q   = 1;
+	my $mbx = undef;
+	# If no Accept field is given, then the value assigned is "q=1".
+	# If at least one listed media range matches the variant's media
+	# type, then the "q" parameter value assigned to the most specific
+	# of those matched is used (e.g. "text/html;version=3.0" is more
+	# specific than "text/html", which is more specific than "text/*",
+	# which in turn is more specific than "*/*"). If not media range
+	# in the provided Accept field matches the variant's media type,
+	# then the value assigned is "q=0".
+	if (exists $accept{'type'} && $ct) {
+	    # First we clean up our content-type
+	    $ct =~ s/\s+//g;
+	    my $params = "";
+	    $params = $1 if $ct =~ s/;(.*)//;
+	    my($type, $subtype) = split("/", $ct, 2);
+	    my %param = ();
+	    for $param (split(/;/, $params)) {
+		my($pk,$pv) = split(/=/, $param, 2);
+		$param{$pk} = $pv;
+	    }
+
+	    my $sel_q = undef;
+	    my $sel_mbx = undef;
+	    my $sel_specificness = 0;
+
+	    ACCEPT_TYPE:
+	    for $at (keys %{ $accept{'type'} }) {
+		print "Consider $at...\n" if $DEBUG;
+		my($at_type, $at_subtype) = split("/", $at, 2);
+		# Is it a match on the type
+		next if $at_type    ne '*' && $at_type    ne $type;
+		next if $at_subtype ne '*' && $at_subtype ne $subtype;
+		my $specificness = 0;
+		$specificness++ if $at_type ne '*';
+		$specificness++ if $at_subtype ne '*';
+		# Let's see if content-type parameters also match
+		while (($pk, $pv) = each %param) {
+		    print "Check if $pk = $pv is true\n" if $DEBUG;
+		    next unless exists $accept{'type'}{$at}{$pk};
+		    next ACCEPT_TYPE
+		      unless $accept{'type'}{$at}{$pk} eq $pv;
+		    print "yes it is!!\n" if $DEBUG;
+		    $specificness++;
+		}
+		print "Hurray, type match with specificness = $specificness\n"
+		  if $DEBUG;
+
+		if (!defined($sel_q) || $sel_specificness < $specificness) {
+		    $sel_q   = $accept{'type'}{$at}{'q'};
+		    $sel_mbx = $accept{'type'}{$at}{'mbx'};
+		    $sel_specificness = $specificness;
+		}
+	    }
+	    $q   = $sel_q || 0;
+	    $mbx = $sel_mbx;
+	}
+
+	my $Q;
+	if (!defined($mbx) || $mbx >= $bs) {
+	    $Q = $qs * $qe * $qc * $ql * $q;
+	} else {
+	    $Q = 0;
+	    print "Variant's size is too large ==> Q=0\n" if $DEBUG;
+	}
+
+	if ($DEBUG) {
+	    $mbx = "undef" unless defined $mbx;
+	    printf "Q=%.3f", $Q;
+	    print "  (q=$q, mbx=$mbx, qe=$qe, qc=$qc, ql=$ql, qs=$qs)\n";
+	}
+
+	push(@Q, [$id, $Q, $bs]);
+    }
+
+
+    @Q = sort { $b->[1] <=> $a->[1] || $a->[2] <=> $b->[2] } @Q;
+
+    return @Q if wantarray;
+    return undef unless @Q;
+    return undef if $Q[0][1] == 0;
+    $Q[0][0];
+}
+
+1;
+
+__END__
+
+
+=head1 NAME
+
+choose - choose a variant of a document to serve (HTTP content negotiation)
+
+=head1 SYNOPSIS
+
+ use HTTP::Negotiate;
+
+ #  ID       QS     Content-Type   Encoding Char-Set        Lang   Size
+ $variants =
+  [['var1',  1.000, 'text/html',   undef,   'iso-8859-1',   'en',   3000],
+   ['var2',  0.950, 'text/plain',  'gzip',  'us-ascii',     'no',    400],
+   ['var3',  0.3,   'image/gif',   undef,   undef,          undef, 43555],
+  ];
+
+ @prefered = choose($variants, $request_headers);
+ $the_one  = choose($variants);
+
+=head1 DESCRIPTION
+
+This module provide a complete implementation of the HTTP content
+negotiation algorithm specified in F<draft-ietf-http-v11-spec-00.ps>
+chapter 12.  Content negotiation allows for the selection of a
+preferred content representation based upon attributes of the
+negotiable variants and the value of the various Accept* header fields
+in the request.
+
+The variants are ordered by preference by calling the function
+choose().
+
+The first parameter is a description of the variants that we can
+choose among.  The variants are described by a reference to an array.
+Each element in this array is an array with the values [$id, $qs,
+$content_type, $content_encoding, $charset, $content_language,
+$content_length].  The meaning of these values are described
+below. The $content_encoding and $content_language can be either a
+single scalar value or an array reference if there are several values.
+
+The second optional parameter is a reference to the request headers.
+This is used to look for "Accept*" headers.  You can pass a reference
+to either a HTTP::Request or a HTTP::Headers object.  If this
+parameter is missing, then the accept specification is initialized
+from the CGI environment variables HTTP_ACCEPT, HTTP_ACCEPT_CHARSET,
+HTTP_ACCEPT_ENCODING and HTTP_ACCEPT_LANGUAGE.
+
+In array context, choose() returns a list of variant
+identifier/calculated quality pairs.  The values are sorted by
+quality, highest quality first.  If the calculated quality is the same
+for two variants, then they are sorted by size (smallest first). I<E.g.>:
+
+  (['var1' => 1], ['var2', 0.3], ['var3' => 0]);
+
+Note that also zero quality variants are included in the return list
+even if these should never be served to the client.
+
+In scalar context, it returns the identifier of the variant with the
+highest score or C<undef> if none have non-zero quality.
+
+If the $HTTP::Negotiate::DEBUG variable is set to TRUE, then a lot of
+noise is generated on STDOUT during evaluation of choose().
+
+=head1 VARIANTS
+
+A variant is described by a list of the following values.  If the
+attribute does not make sense or is unknown for a variant, then use
+C<undef> instead.
+
+=over 3
+
+=item identifier
+
+This is just some string that you use as a name for the variant.  The
+identifier of the preferred variant is returned by choose().
+
+=item qs
+
+This is a number between 0.000 and 1.000 that describes the "source
+quality".  This is what F<draft-ietf-http-v11-spec-00.ps> says about this
+value:
+
+Source quality is measured by the content provider as representing the
+amount of degradation from the original source.  For example, a
+picture in JPEG form would have a lower qs when translated to the XBM
+format, and much lower qs when translated to an ASCII-art
+representation.  Note, however, that this is a function of the source
+- an original piece of ASCII-art may degrade in quality if it is
+captured in JPEG form.  The qs values should be assigned to each
+variant by the content provider; if no qs value has been assigned, the
+default is generally "qs=1".
+
+=item content-type
+
+This is the media type of the variant.  The media type does not
+include a charset attribute, but might contain other parameters.
+Examples are:
+
+  text/html
+  text/html;version=2.0
+  text/plain
+  image/gif
+  image/jpg
+
+=item content-encoding
+
+This is one or more content encodings that has been applied to the
+variant.  The content encoding is generally used as a modifier to the
+content media type.  The most common content encodings are:
+
+  gzip
+  compress
+
+=item content-charset
+
+This is the character set used when the variant contains textual
+content.  The charset value should generally be C<undef> or one of these:
+
+  us-ascii
+  iso-8859-1 ... iso-8859-9
+  iso-2022-jp
+  iso-2022-jp-2
+  iso-2022-kr
+  unicode-1-1
+  unicode-1-1-utf-7
+  unicode-1-1-utf-8
+
+=item content-language
+
+This describes one or more languages that are used in the variant.
+Language is described like this in F<draft-ietf-http-v11-spec-00.ps>: A
+language is in this context a natural language spoken, written, or
+otherwise conveyed by human beings for communication of information to
+other human beings.  Computer languages are explicitly excluded.
+
+The language tags are the same as those defined by RFC-1766.  Examples
+are:
+
+  no               Norwegian
+  en               International English
+  en-US            US English
+  en-cockney
+
+=item content-length
+
+This is the number of bytes used to represent the content.
+
+=back
+
+=head1 ACCEPT HEADERS
+
+The following Accept* headers can be used for describing content
+preferences in a request (This description is an edited extract from
+F<draft-ietf-http-v11-spec-00.ps>):
+
+=over 3
+
+=item Accept
+
+This header can be used to indicate a list of media ranges which are
+acceptable as a reponse to the request.  The "*" character is used to
+group media types into ranges, with "*/*" indicating all media types
+and "type/*" indicating all subtypes of that type.
+
+The parameter q is used to indicate the quality factor, which
+represents the user's preference for that range of media types.  The
+parameter mbx gives the maximum acceptable size of the response
+content. The default values are: q=1 and mbx=infinity. If no Accept
+header is present, then the client accepts all media types with q=1.
+
+For example:
+
+  Accept: audio/*;q=0.2;mbx=200000, audio/basic
+
+would mean: "I prefer audio/basic (of any size), but send me any audio
+type if it is the best available after an 80% mark-down in quality and
+its size is less than 200000 bytes"
+
+
+=item Accept-Charset
+
+Used to indicate what character sets are acceptable for the response.
+The "us-ascii" character set is assumed to be acceptable for all user
+agents.  If no Accept-Charset field is given, the default is that any
+charset is acceptable.  Example:
+
+  Accept-Charset: iso-8859-1, unicode-1-1
+
+
+=item Accept-Encoding
+
+Restricts the Content-Encoding values which are acceptable in the
+response.  If no Accept-Encoding field is present, the server may
+assume that the client will accept any content encoding.  An empty
+Accept-Encoding means that no content encoding is acceptable.  Example:
+
+  Accept-Encoding: compress, gzip
+
+
+=item Accept-Language
+
+This field is similar to Accept, but restrict the set of natural
+languages that are preferred as a response.  Each language may be
+given an associated quality value which represents an estimate of the
+user's comprehension of that language.  For example:
+
+  Accept-Language: no, en-gb;q=0.8, de;q=0.55
+
+would mean: "I prefer Norwegian, but will accept British English (with
+80% comprehension) or German (with 55% comprehension).
+
+=back
+
+
+=head1 COPYRIGHT
+
+Copyright 1996, Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 AUTHOR
+
+Gisle Aas <aas@sn.no>
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request.pm	(revision 21112)
@@ -0,0 +1,160 @@
+#
+# $Id$
+
+package HTTP::Request;
+
+=head1 NAME
+
+HTTP::Request - Class encapsulating HTTP Requests
+
+=head1 SYNOPSIS
+
+ require HTTP::Request;
+ $request = HTTP::Request->new(GET => 'http://www.oslonett.no/');
+
+=head1 DESCRIPTION
+
+C<HTTP::Request> is a class encapsulating HTTP style requests,
+consisting of a request line, some headers, and some (potentially empty)
+content. Note that the LWP library also uses this HTTP style requests
+for non-HTTP protocols.
+
+Instances of this class are usually passed to the C<request()> method
+of an C<LWP::UserAgent> object:
+
+ $ua = LWP::UserAgent->new;
+ $request = HTTP::Request->new(GET => 'http://www.oslonett.no/');
+ $response = $ua->request($request);
+
+C<HTTP::Request> is a subclass of C<HTTP::Message> and therefore
+inherits its methods.  The inherited methods often used are header(),
+push_header(), remove_header(), headers_as_string() and content().
+See L<HTTP::Message> for details.
+
+The following additional methods are available:
+
+=over 4
+
+=cut
+
+require HTTP::Message;
+@ISA = qw(HTTP::Message);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use URI::URL ();
+use strict;
+
+=item $r = HTTP::Request->new($method, $url, [$header, [$content]])
+
+Constructs a new C<HTTP::Request> object describing a request on the
+object C<$url> using method C<$method>.  The C<$url> argument can be
+either a string, or a reference to a C<URI::URL> object.  The $header
+argument should be a reference to an C<HTTP::Headers> object.
+
+=cut
+
+sub new
+{
+    my($class, $method, $url, $header, $content) = @_;
+    my $self = $class->SUPER::new($header, $content);
+    $self->method($method);
+    $self->url($url);
+    $self;
+}
+
+
+sub clone
+{
+    my $self = shift;
+    my $clone = bless $self->SUPER::clone, ref($self);
+    $clone->method($self->method);
+    $clone->url($self->url);
+    $clone;
+}
+
+
+=item $r->method([$val])
+
+=item $r->url([$val])
+
+These methods provide public access to the member variables containing
+respectively the method of the request and the URL object of the
+request.
+
+If an argument is given the member variable is given that as its new
+value. If no argument is given the value is not touched. In either
+case the previous value is returned.
+
+The url() method accept both a reference to a URI::URL object and a
+string as its argument.  If a string is given, then it should be
+parseable as an absolute URL.
+
+=cut
+
+sub method  { shift->_elem('_method', @_); }
+
+sub url
+{
+    my $self = shift;
+    my $old = $self->{'_url'};
+    if (@_) {
+	my $url = shift;
+	if (!defined $url) {
+	    # that's ok
+	} elsif (ref $url) {
+	    $url = $url->abs;
+	} else {
+	    $url = eval { URI::URL->new($url) };
+	}
+	$self->{'_url'} = $url;
+    }
+    $old;
+}
+
+*uri = \&url;  # this is the same for now
+
+=item $r->as_string()
+
+Method returning a textual representation of the request.
+Mainly useful for debugging purposes. It takes no arguments.
+
+=cut
+
+sub as_string
+{
+    my $self = shift;
+    my @result;
+    #push(@result, "---- $self -----");
+    my $req_line = $self->method || "[NO METHOD]";
+    my $url = $self->url;
+    $url = (defined $url) ? $url->as_string : "[NO URL]";
+    $req_line .= " $url";
+    my $proto = $self->protocol;
+    $req_line .= " $proto" if $proto;
+
+    push(@result, $req_line);
+    push(@result, $self->headers_as_string);
+    my $content = $self->content;
+    if (defined $content) {
+	push(@result, $content);
+    }
+    #push(@result, ("-" x 40));
+    join("\n", @result, "");
+}
+
+1;
+
+=back
+
+=head1 SEE ALSO
+
+L<HTTP::Headers>, L<HTTP::Message>, L<HTTP::Request::Common>
+
+=head1 COPYRIGHT
+
+Copyright 1995-1998 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request/Common.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request/Common.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Request/Common.pm	(revision 21112)
@@ -0,0 +1,389 @@
+# $Id$
+#
+package HTTP::Request::Common;
+
+use strict;
+use vars qw(@EXPORT @EXPORT_OK $VERSION $DYNAMIC_FILE_UPLOAD);
+
+$DYNAMIC_FILE_UPLOAD ||= 0;  # make it defined (don't know why)
+
+require Exporter;
+*import = \&Exporter::import;
+@EXPORT =qw(GET HEAD PUT POST);
+@EXPORT_OK = qw($DYNAMIC_FILE_UPLOAD);
+
+require HTTP::Request;
+use Carp();
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+my $CRLF = "\015\012";   # "\r\n" is not portable
+
+sub GET  { _simple_req('GET',  @_); }
+sub HEAD { _simple_req('HEAD', @_); }
+sub PUT  { _simple_req('PUT' , @_); }
+
+sub POST
+{
+    my $url = shift;
+    my $req = HTTP::Request->new(POST => $url);
+    my $content;
+    $content = shift if @_ and ref $_[0];
+    my($k, $v);
+    while (($k,$v) = splice(@_, 0, 2)) {
+	if (lc($k) eq 'content') {
+	    $content = $v;
+	} else {
+	    $req->push_header($k, $v);
+	}
+    }
+    my $ct = $req->header('Content-Type');
+    unless ($ct) {
+	$ct = 'application/x-www-form-urlencoded';
+    } elsif ($ct eq 'form-data') {
+	$ct = 'multipart/form-data';
+    }
+
+    if (ref $content) {
+	if (lc($ct) eq 'multipart/form-data') {    #XXX: boundary="..."
+	    my $boundary;
+	    ($content, $boundary) = form_data($content, $boundary);
+	    $boundary = qq("$boundary") if $boundary =~ /\W/;
+	    $ct = qq(multipart/form-data; boundary=$boundary);
+	} else {
+	    # We use a temporary URI::URL object to format
+	    # the application/x-www-form-urlencoded content.
+	    require URI::URL;
+	    my $url = URI::URL->new('http:');
+	    $url->query_form(ref($content) eq "HASH" ? %$content : @$content);
+	    $content = $url->equery;
+	}
+    }
+
+    $req->header('Content-Type' => $ct);  # might be redundant
+    if (defined($content)) {
+	$req->header('Content-Length' =>
+		     length($content)) unless ref($content);
+	$req->content($content);
+    }
+    $req;
+}
+
+
+sub _simple_req
+{
+    my($method, $url) = splice(@_, 0, 2);
+    my $req = HTTP::Request->new($method => $url);
+    my($k, $v);
+    while (($k,$v) = splice(@_, 0, 2)) {
+	if (lc($k) eq 'content') {
+	    $req->add_content($v);
+	} else {
+	    $req->push_header($k, $v);
+	}
+    }
+    $req;
+}
+
+
+sub form_data   # RFC1867
+{
+    my($data, $boundary) = @_;
+    my @data = ref($data) eq "HASH" ? %$data : @$data;  # copy
+    my $fhparts;
+    my @parts;
+    my($k,$v);
+    while (($k,$v) = splice(@data, 0, 2)) {
+	if (!ref($v)) {
+	    $k =~ s/([\\\"])/\\$1/g;  # escape quotes and backslashes
+	    push(@parts,
+		 qq(Content-Disposition: form-data; name="$k"$CRLF$CRLF$v));
+	} else {
+	    my($file, $usename, @headers) = @$v;
+	    unless (defined $usename) {
+		$usename = $file;
+		$usename =~ s,.*/,, if defined($usename);
+	    }
+	    my $disp = qq(form-data; name="$k");
+	    $disp .= qq(; filename="$usename") if $usename;
+	    my $content = "";
+	    my $h = HTTP::Headers->new(@headers);
+	    my $ct = $h->header("Content-Type");
+	    if ($file) {
+		require Symbol;
+		my $fh = Symbol::gensym();
+		open($fh, $file) or Carp::croak("Can't open file $file: $!");
+		binmode($fh);
+		if ($DYNAMIC_FILE_UPLOAD) {
+		    # will read file later
+		    $content = $fh;
+		} else {
+		    local($/) = undef; # slurp files
+		    $content = <$fh>;
+		    close($fh);
+		    $h->header("Content-Length" => length($content));
+		}
+		unless ($ct) {
+		    require LWP::MediaTypes;
+		    $ct = LWP::MediaTypes::guess_media_type($file, $h);
+		}
+	    }
+	    if ($h->header("Content-Disposition")) {
+		# just to get it sorted first
+		$disp = $h->header("Content-Disposition");
+		$h->remove_header("Content-Disposition");
+	    }
+	    if ($h->header("Content")) {
+		$content = $h->header("Content");
+		$h->remove_header("Content");
+	    }
+	    my $head = join($CRLF, "Content-Disposition: $disp",
+			           $h->as_string($CRLF),
+			           "");
+	    if (ref $content) {
+		push(@parts, [$head, $content]);
+		$fhparts++;
+	    } else {
+		push(@parts, $head . $content);
+	    }
+	}
+    }
+    return "" unless @parts;
+
+    my $content;
+    if ($fhparts) {
+	$boundary = boundary(10) # hopefully enough randomness
+	    unless $boundary;
+
+	# add the boundaries to the @parts array
+	for (1..@parts-1) {
+	    splice(@parts, $_*2-1, 0, "$CRLF--$boundary$CRLF");
+	}
+	unshift(@parts, "--$boundary$CRLF");
+	push(@parts, "$CRLF--$boundary--$CRLF");
+
+	# set up a closure that will return content piecemeal
+	$content = sub {
+	    for (;;) {
+		return unless @parts;
+		my $p = shift @parts;
+		unless (ref $p) {
+		    $p .= shift @parts while @parts && !ref($parts[0]);
+		    return $p;
+		}
+		my($buf, $fh) = @$p;
+		my $n = read($fh, $buf, 2048, length($buf));
+		if ($n) {
+		    unshift(@parts, ["", $fh]);
+		} else {
+		    close($fh);
+		}
+		return $buf if length $buf;
+	    }
+	};
+
+    } else {
+	$boundary = boundary() unless $boundary;
+
+	my $bno = 0;
+      CHECK_BOUNDARY:
+	{
+	    for (@parts) {
+		if (index($_, "--$boundary") >= 0) {
+		    # must have a better boundary
+		    #warn "Need something better that '$boundary' as boundary\n";
+		    $boundary = boundary(++$bno);
+		    redo CHECK_BOUNDARY;
+		}
+	    }
+	    last;
+	}
+	$content = "--$boundary$CRLF" .
+	           join("$CRLF--$boundary$CRLF", @parts) .
+		   "$CRLF--$boundary--$CRLF";
+    }
+
+    wantarray ? ($content, $boundary) : $content;
+}
+
+
+sub boundary
+{
+    my $size = shift || return "000";
+    require MIME::Base64;
+    my $b = MIME::Base64::encode(join("", map chr(rand(256)), 1..$size*3), "");
+    $b =~ s/[\W]/X/g;  # ensure alnum only
+    $b;
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+HTTP::Request::Common - Construct common HTTP::Request objects
+
+=head1 SYNOPSIS
+
+  use HTTP::Request::Common;
+  $ua = LWP::UserAgent->new;
+  $ua->request(GET 'http://www.sn.no/');
+  $ua->request(POST 'http://somewhere/foo', [foo => bar, bar => foo]);
+
+=head1 DESCRIPTION
+
+This module provide functions that return newly created HTTP::Request
+objects.  These functions are usually more convenient to use than the
+standard HTTP::Request constructor for these common requests.  The
+following functions are provided.
+
+=over 4
+
+=item GET $url, Header => Value,...
+
+The GET() function returns a HTTP::Request object initialized with the
+GET method and the specified URL.  Without additional arguments it
+is exactly equivalent to the following call
+
+  HTTP::Request->new(GET => $url)
+
+but is less clutter.  It also reads better when used together with the
+LWP::UserAgent->request() method:
+
+  my $ua = new LWP::UserAgent;
+  my $res = $ua->request(GET 'http://www.sn.no')
+  if ($res->is_success) { ...
+
+You can also initialize the header values in the request by specifying
+some key/value pairs as optional arguments.  For instance:
+
+  $ua->request(GET 'http://www.sn.no',
+	           If_Match => 'foo',
+                   From     => 'gisle@aas.no',
+              );
+
+A header key called 'Content' is special and when seen the value will
+initialize the content part of the request instead of setting a header.
+
+=item HEAD $url, [Header => Value,...]
+
+Like GET() but the method in the request is HEAD.
+
+=item PUT $url, [Header => Value,...]
+
+Like GET() but the method in the request is PUT.
+
+=item POST $url, [$form_ref], [Header => Value,...]
+
+This works mostly like GET() with POST as method, but this function
+also takes a second optional array or hash reference parameter
+($form_ref).  This argument can be used to pass key/value pairs for
+the form content.  By default we will initialize a request using the
+C<application/x-www-form-urlencoded> content type.  This means that
+you can emulate a HTML E<lt>form> POSTing like this:
+
+  POST 'http://www.perl.org/survey.cgi',
+       [ name   => 'Gisle Aas',
+         email  => 'gisle@aas.no',
+         gender => 'M',
+         born   => '1964',
+         perc   => '3%',
+       ];
+
+This will create a HTTP::Request object that looks like this:
+
+  POST http://www.perl.org/survey.cgi
+  Content-Length: 66
+  Content-Type: application/x-www-form-urlencoded
+
+  name=Gisle%20Aas&email=gisle%40aas.no&gender=M&born=1964&perc=3%25
+
+The POST method also supports the C<multipart/form-data> content used
+for I<Form-based File Upload> as specified in RFC 1867.  You trigger
+this content format by specifying a content type of C<'form-data'> as
+one of the request headers.  If one of the values in the $form_ref is
+an array reference, then it is treated as a file part specification
+with the following interpretation:
+
+  [ $file, $filename, Header => Value... ]
+
+The first value in the array ($file) is the name of a file to open.
+This file will be read an its content placed in the request.  The
+routine will croak if the file can't be opened.  Use an C<undef> as $file
+value if you want to specify the content directly.  The $filename is
+the filename to report in the request.  If this value is undefined,
+then the basename of the $file will be used.  You can specify an empty
+string as $filename if you don't want any filename in the request.
+
+Sending my F<~/.profile> to the survey used as example above can be
+achieved by this:
+
+  POST 'http://www.perl.org/survey.cgi',
+       Content_Type => 'form-data',
+       Content      => [ name  => 'Gisle Aas',
+                         email => 'gisle@aas.no',
+                         gender => 'M',
+                         born   => '1964',
+                         init   => ["$ENV{HOME}/.profile"],
+                       ]
+
+This will create a HTTP::Request object that almost looks this (the
+boundary and the content of your F<~/.profile> is likely to be
+different):
+
+  POST http://www.perl.org/survey.cgi
+  Content-Length: 388
+  Content-Type: multipart/form-data; boundary="6G+f"
+
+  --6G+f
+  Content-Disposition: form-data; name="name"
+  
+  Gisle Aas
+  --6G+f
+  Content-Disposition: form-data; name="email"
+  
+  gisle@aas.no
+  --6G+f
+  Content-Disposition: form-data; name="gender"
+  
+  M
+  --6G+f
+  Content-Disposition: form-data; name="born"
+  
+  1964
+  --6G+f
+  Content-Disposition: form-data; name="init"; filename=".profile"
+  Content-Type: text/plain
+  
+  PATH=/local/perl/bin:$PATH
+  export PATH
+
+  --6G+f--
+
+If you set the $DYNAMIC_FILE_UPLOAD variable (exportable) to some TRUE
+value, then you get back a request object with a subroutine closure as
+the content attribute.  This subroutine will read the content of any
+files on demand and return it in suitable chunks.  This allow you to
+upload arbitrary big files without using lots of memory.  You can even
+upload infinite files like F</dev/audio> if you wish.  Another
+difference is that there will be no Content-Length header defined for
+the request if you use this feature.  Not all servers (or server
+applications) like this.
+
+=back
+
+=head1 SEE ALSO
+
+L<HTTP::Request>, L<LWP::UserAgent>
+
+
+=head1 COPYRIGHT
+
+Copyright 1997-1998, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Response.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Response.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Response.pm	(revision 21112)
@@ -0,0 +1,396 @@
+#
+# $Id$
+
+package HTTP::Response;
+
+
+=head1 NAME
+
+HTTP::Response - Class encapsulating HTTP Responses
+
+=head1 SYNOPSIS
+
+ require HTTP::Response;
+
+=head1 DESCRIPTION
+
+The C<HTTP::Response> class encapsulate HTTP style responses.  A
+response consist of a response line, some headers, and a (potential
+empty) content. Note that the LWP library will use HTTP style
+responses also for non-HTTP protocol schemes.
+
+Instances of this class are usually created and returned by the
+C<request()> method of an C<LWP::UserAgent> object:
+
+ #...
+ $response = $ua->request($request)
+ if ($response->is_success) {
+     print $response->content;
+ } else {
+     print $response->error_as_HTML;
+ }
+
+C<HTTP::Response> is a subclass of C<HTTP::Message> and therefore
+inherits its methods.  The inherited methods often used are header(),
+push_header(), remove_header(), headers_as_string(), and content().
+The header convenience methods are also available.  See
+L<HTTP::Message> for details.
+
+The following additional methods are available:
+
+=over 4
+
+=cut
+
+
+require HTTP::Message;
+@ISA = qw(HTTP::Message);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use HTTP::Status ();
+use URI::URL ();
+use strict;
+
+
+=item $r = HTTP::Response->new($rc, [$msg, [$header, [$content]]])
+
+Constructs a new C<HTTP::Response> object describing a response with
+response code C<$rc> and optional message C<$msg>.  The message is a
+short human readable single line string that explains the response
+code.
+
+=cut
+
+sub new
+{
+    my($class, $rc, $msg, $header, $content) = @_;
+    my $self = $class->SUPER::new($header, $content);
+    $self->code($rc);
+    $self->message($msg);
+    $self;
+}
+
+
+sub clone
+{
+    my $self = shift;
+    my $clone = bless $self->SUPER::clone, ref($self);
+    $clone->code($self->code);
+    $clone->message($self->message);
+    $clone->request($self->request->clone) if $self->request;
+    # we don't clone previous
+    $clone;
+}
+
+=item $r->code([$code])
+
+=item $r->message([$message])
+
+=item $r->request([$request])
+
+=item $r->previous([$previousResponse])
+
+These methods provide public access to the member variables.  The
+first two containing respectively the response code and the message
+of the response.
+
+The request attribute is a reference the request that gave this
+response.  It does not have to be the same request as passed to the
+$ua->request() method, because there might have been redirects and
+authorization retries in between.
+
+The previous attribute is used to link together chains of responses.
+You get chains of responses if the first response is redirect or
+unauthorized.
+
+=cut
+
+sub code      { shift->_elem('_rc',      @_); }
+sub message   { shift->_elem('_msg',     @_); }
+sub previous  { shift->_elem('_previous',@_); }
+sub request   { shift->_elem('_request', @_); }
+
+=item $r->status_line
+
+Returns the string "E<lt>code> E<lt>message>".  If the message attribute
+is not set then the official name of E<lt>code> (see L<HTTP::Status>)
+is substituted.
+
+=cut
+
+sub status_line
+{
+    my $self = shift;
+    my $code = $self->{'_rc'}  || "000";
+    my $mess = $self->{'_msg'} || HTTP::Status::status_message($code) || "?";
+    return "$code $mess";
+}
+
+=item $r->base
+
+Returns the base URL for this response.  The return value will be a
+reference to a URI::URL object.
+
+The base URL is obtained from one the following sources (in priority
+order):
+
+=over 4
+
+=item 1.
+
+Embedded in the document content, for instance <BASE HREF="...">
+in HTML documents.
+
+=item 2.
+
+A "Content-Base:" or a "Content-Location:" header in the response.
+
+For backwards compatability with older HTTP implementations we will
+also look for the "Base:" header.
+
+
+=item 3.
+
+The URL used to request this response. This might not be the original
+URL that was passed to $ua->request() method, because we might have
+received some redirect responses first.
+
+=back
+
+When the LWP protocol modules produce the HTTP::Response object, then
+any base URL embedded in the document (step 1) will already have
+initialized the "Content-Base:" header. This means that this method
+only perform the last 2 steps (the content is not always available
+either).
+
+=cut
+
+sub base
+{
+    my $self = shift;
+    my $base = $self->header('Content-Base')     ||  # HTTP/1.1
+               $self->header('Content-Location') ||  # HTTP/1.1
+               $self->header('Base')             ||  # backwards compatability HTTP/1.0
+               $self->request->url;
+    $base = URI::URL->new($base) unless ref $base;
+    $base;
+}
+
+
+=item $r->as_string
+
+Method returning a textual representation of the response.  Mainly
+useful for debugging purposes. It takes no arguments.
+
+=cut
+
+sub as_string
+{
+    require HTTP::Status;
+    my $self = shift;
+    my @result;
+    #push(@result, "---- $self ----");
+    my $code = $self->code;
+    my $status_message = HTTP::Status::status_message($code) || "Unknown code";
+    my $message = $self->message || "";
+
+    my $status_line = "$code";
+    my $proto = $self->protocol;
+    $status_line = "$proto $status_line" if $proto;
+    $status_line .= " ($status_message)" if $status_message ne $message;
+    $status_line .= " $message";
+    push(@result, $status_line);
+    push(@result, $self->headers_as_string);
+    my $content = $self->content;
+    if (defined $content) {
+	push(@result, $content);
+    }
+    #push(@result, ("-" x 40));
+    join("\n", @result, "");
+}
+
+=item $r->is_info
+
+=item $r->is_success
+
+=item $r->is_redirect
+
+=item $r->is_error
+
+These methods indicate if the response was informational, sucessful, a
+redirection, or an error.
+
+=cut
+
+sub is_info     { HTTP::Status::is_info     (shift->{'_rc'}); }
+sub is_success  { HTTP::Status::is_success  (shift->{'_rc'}); }
+sub is_redirect { HTTP::Status::is_redirect (shift->{'_rc'}); }
+sub is_error    { HTTP::Status::is_error    (shift->{'_rc'}); }
+
+
+=item $r->error_as_HTML()
+
+Return a string containing a complete HTML document indicating what
+error occurred.  This method should only be called when $r->is_error
+is TRUE.
+
+=cut
+
+sub error_as_HTML
+{
+    my $self = shift;
+    my $title = 'An Error Occurred';
+    my $body  = $self->status_line;
+    return <<EOM;
+<HTML>
+<HEAD><TITLE>$title</TITLE></HEAD>
+<BODY>
+<H1>$title</h1>
+$body
+</BODY>
+</HTML>
+EOM
+}
+
+
+=item $r->current_age
+
+This function will calculate the "current age" of the response as
+specified by E<lt>draft-ietf-http-v11-spec-07> section 13.2.3.  The
+age of a response is the time since it was sent by the origin server.
+The returned value is a number representing the age in seconds.
+
+=cut
+
+sub current_age
+{
+    my $self = shift;
+    # Implementation of <draft-ietf-http-v11-spec-07> section 13.2.3
+    # (age calculations)
+    my $response_time = $self->client_date;
+    my $date = $self->date;
+
+    my $age = 0;
+    if ($response_time && $date) {
+	$age = $response_time - $date;  # apparent_age
+	$age = 0 if $age < 0;
+    }
+
+    my $age_v = $self->header('Age');
+    if ($age_v && $age_v > $age) {
+	$age = $age_v;   # corrected_received_age
+    }
+
+    my $request = $self->request;
+    if ($request) {
+	my $request_time = $request->date;
+	if ($request_time) {
+	    # Add response_delay to age to get 'corrected_initial_age'
+	    $age += $response_time - $request_time;
+	}
+    }
+    if ($response_time) {
+	$age += time - $response_time;
+    }
+    return $age;
+}
+
+
+=item $r->freshness_lifetime
+
+This function will calculate the "freshness lifetime" of the response
+as specified by E<lt>draft-ietf-http-v11-spec-07> section 13.2.4.  The
+"freshness lifetime" is the length of time between the generation of a
+response and its expiration time.  The returned value is a number
+representing the freshness lifetime in seconds.
+
+If the response does not contain an "Expires" or a "Cache-Control"
+header, then this function will apply some simple heuristic based on
+'Last-Modified' to determine a suitable lifetime.
+
+=cut
+
+sub freshness_lifetime
+{
+    my $self = shift;
+
+    # First look for the Cache-Control: max-age=n header
+    my @cc = $self->header('Cache-Control');
+    if (@cc) {
+	my $cc;
+	for $cc (@cc) {
+	    my $cc_dir;
+	    for $cc_dir (split(/\s*,\s*/, $cc)) {
+		if ($cc_dir =~ /max-age\s*=\s*(\d+)/i) {
+		    return $1;
+		}
+	    }
+	}
+    }
+
+    # Next possibility is to look at the "Expires" header
+    my $date = $self->date || $self->client_date || time;      
+    my $expires = $self->expires;
+    unless ($expires) {
+	# Must apply heuristic expiration
+	my $last_modified = $self->last_modified;
+	if ($last_modified) {
+	    my $h_exp = ($date - $last_modified) * 0.10;  # 10% since last-mod
+	    if ($h_exp < 60) {
+		return 60;  # minimum
+	    } elsif ($h_exp > 24 * 3600) {
+		# Should give a warning if more than 24 hours according to
+		# <draft-ietf-http-v11-spec-07> section 13.2.4, but I don't
+		# know how to do it from this function interface, so I just
+		# make this the maximum value.
+		return 24 * 3600;
+	    }
+	    return $h_exp;
+	} else {
+	    return 3600;  # 1 hour is fallback when all else fails
+	}
+    }
+    return $expires - $date;
+}
+
+
+=item $r->is_fresh
+
+Returns TRUE if the response is fresh, based on the values of
+freshness_lifetime() and current_age().  If the response is not longer
+fresh, then it has to be refetched or revalidated by the origin
+server.
+
+=cut
+
+sub is_fresh
+{
+    my $self = shift;
+    $self->freshness_lifetime > $self->current_age;
+}
+
+
+=item $r->fresh_until
+
+Returns the time when this entiy is no longer fresh.
+
+=cut
+
+sub fresh_until
+{
+    my $self = shift;
+    return $self->freshness_lifetime - $self->current_age + time;
+}
+
+1;
+
+=back 
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Status.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Status.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/HTTP/Status.pm	(revision 21112)
@@ -0,0 +1,233 @@
+#
+# $Id$
+
+package HTTP::Status;
+
+use strict;
+require 5.002;   # becase we use prototypes
+
+=head1 NAME
+
+HTTP::Status - HTTP Status code processing
+
+=head1 SYNOPSIS
+
+ use HTTP::Status;
+
+ if ($rc != RC_OK) {
+     print status_message($rc), "\n";
+ }
+
+ if (is_success($rc)) { ... }
+ if (is_error($rc)) { ... }
+ if (is_redirect($rc)) { ... }
+
+=head1 DESCRIPTION
+
+I<HTTP::Status> is a library of routines for defining and
+classification of HTTP status codes for libwww-perl.  Status codes are
+used to encode the overall outcome of a HTTP response message.  Codes
+correspond to those defined in F<draft-ietf-http-v11-spec-rev-03> (an
+update to RFC 2068).
+
+=head1 CONSTANTS
+
+The following constant functions can be used as mnemonic status code
+names:
+
+   RC_CONTINUE				(100)
+   RC_SWITCHING_PROTOCOLS		(101)
+
+   RC_OK				(200)
+   RC_CREATED				(201)
+   RC_ACCEPTED				(202)
+   RC_NON_AUTHORITATIVE_INFORMATION	(203)
+   RC_NO_CONTENT			(204)
+   RC_RESET_CONTENT			(205)
+   RC_PARTIAL_CONTENT			(206)
+
+   RC_MULTIPLE_CHOICES			(300)
+   RC_MOVED_PERMANENTLY			(301)
+   RC_FOUND				(302)
+   RC_SEE_OTHER				(303)
+   RC_NOT_MODIFIED			(304)
+   RC_USE_PROXY				(305)
+   RC_TEMPORARY_REDIRECT		(307)
+
+   RC_BAD_REQUEST			(400)
+   RC_UNAUTHORIZED			(401)
+   RC_PAYMENT_REQUIRED			(402)
+   RC_FORBIDDEN				(403)
+   RC_NOT_FOUND				(404)
+   RC_METHOD_NOT_ALLOWED		(405)
+   RC_NOT_ACCEPTABLE			(406)
+   RC_PROXY_AUTHENTICATION_REQUIRED	(407)
+   RC_REQUEST_TIMEOUT			(408)
+   RC_CONFLICT				(409)
+   RC_GONE				(410)
+   RC_LENGTH_REQUIRED			(411)
+   RC_PRECONDITION_FAILED		(412)
+   RC_REQUEST_ENTITY_TOO_LARGE		(413)
+   RC_REQUEST_URI_TOO_LARGE		(414)
+   RC_UNSUPPORTED_MEDIA_TYPE		(415)
+   RC_REQUEST_RANGE_NOT_SATISFIABLE     (416)
+   RC_EXPECTATION_FAILED		(417)
+
+   RC_INTERNAL_SERVER_ERROR		(500)
+   RC_NOT_IMPLEMENTED			(501)
+   RC_BAD_GATEWAY			(502)
+   RC_SERVICE_UNAVAILABLE		(503)
+   RC_GATEWAY_TIMEOUT			(504)
+   RC_HTTP_VERSION_NOT_SUPPORTED	(505)
+
+=cut
+
+#####################################################################
+
+use vars qw(@ISA @EXPORT @EXPORT_OK $VERSION);
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(is_info is_success is_redirect is_error status_message);
+@EXPORT_OK = qw(is_client_error is_server_error);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+# Note also addition of mnemonics to @EXPORT below
+
+my %StatusCode = (
+    100 => 'Continue',
+    101 => 'Switching Protocols',
+    200 => 'OK',
+    201 => 'Created',
+    202 => 'Accepted',
+    203 => 'Non-Authoritative Information',
+    204 => 'No Content',
+    205 => 'Reset Content',
+    206 => 'Partial Content',
+    300 => 'Multiple Choices',
+    301 => 'Moved Permanently',
+    302 => 'Found',
+    303 => 'See Other',
+    304 => 'Not Modified',
+    305 => 'Use Proxy',
+    307 => 'Temporary Redirect',
+    400 => 'Bad Request',
+    401 => 'Unauthorized',
+    402 => 'Payment Required',
+    403 => 'Forbidden',
+    404 => 'Not Found',
+    405 => 'Method Not Allowed',
+    406 => 'Not Acceptable',
+    407 => 'Proxy Authentication Required',
+    408 => 'Request Timeout',
+    409 => 'Conflict',
+    410 => 'Gone',
+    411 => 'Length Required',
+    412 => 'Precondition Failed',
+    413 => 'Request Entity Too Large',
+    414 => 'Request-URI Too Large',
+    415 => 'Unsupported Media Type',
+    416 => 'Request Range Not Satisfiable',
+    417 => 'Expectation Failed',
+    500 => 'Internal Server Error',
+    501 => 'Not Implemented',
+    502 => 'Bad Gateway',
+    503 => 'Service Unavailable',
+    504 => 'Gateway Timeout',
+    505 => 'HTTP Version Not Supported',
+);
+
+my $mnemonicCode = '';
+my ($code, $message);
+while (($code, $message) = each %StatusCode) {
+    # create mnemonic subroutines
+    $message =~ tr/a-z \-/A-Z__/;
+    $mnemonicCode .= "sub RC_$message () { $code }\t";
+    # make them exportable
+    $mnemonicCode .= "push(\@EXPORT, 'RC_$message');\n";
+}
+# warn $mnemonicCode; # for development
+eval $mnemonicCode; # only one eval for speed
+die if $@;
+
+# backwards compatibility
+*RC_MOVED_TEMPORARILY = \&RC_FOUND;  # 302 was renamed in the standard
+push(@EXPORT, "RC_MOVED_TEMPORARILY");
+
+
+=head1 FUNCTIONS
+
+The following additional functions are provided.  Most of them are
+exported by default.
+
+=over 4
+
+=item status_message($code)
+
+The status_message() function will translate status codes to human
+readable strings. The string is the same as found in the constant
+names above.  If the $code is unknown, then C<undef> is returned.
+
+=cut
+
+sub status_message ($)
+{
+    $StatusCode{$_[0]};
+}
+
+=item is_info($code)
+
+Return TRUE if C<$code> is an I<Informational> status code.  This
+class of status code indicates a provisional response which can't have
+any content.
+
+=item is_success($code)
+
+Return TRUE if C<$code> is a I<Successful> status code.
+
+=item is_redirect($code)
+
+Return TRUE if C<$code> is a I<Redirection> status code. This class of
+status code indicates that further action needs to be taken by the
+user agent in order to fulfill the request.
+
+=item is_error($code)
+
+Return TRUE if C<$code> is an I<Error> status code.  The function
+return TRUE for both client error or a server error status codes.
+
+=item is_client_error($code)
+
+Return TRUE if C<$code> is an I<Client Error> status code. This class
+of status code is intended for cases in which the client seems to have
+erred.
+
+This function is B<not> exported by default.
+
+=item is_server_error($code)
+
+Return TRUE if C<$code> is an I<Server Error> status code. This class
+of status codes is intended for cases in which the server is aware
+that it has erred or is incapable of performing the request.
+
+This function is B<not> exported by default.
+
+=back
+
+=cut
+
+sub is_info         ($) { $_[0] >= 100 && $_[0] < 200; }
+sub is_success      ($) { $_[0] >= 200 && $_[0] < 300; }
+sub is_redirect     ($) { $_[0] >= 300 && $_[0] < 400; }
+sub is_error        ($) { $_[0] >= 400 && $_[0] < 600; }
+sub is_client_error ($) { $_[0] >= 400 && $_[0] < 500; }
+sub is_server_error ($) { $_[0] >= 500 && $_[0] < 600; }
+
+1;
+
+=head1 BUGS
+
+Wished @EXPORT_OK had been used instead of @EXPORT in the beginning.
+Now too much is exported by default.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP.pm	(revision 21112)
@@ -0,0 +1,593 @@
+#
+# $Id$
+
+package LWP;
+
+$VERSION = "5.36";
+sub Version { $VERSION; }
+
+require 5.004;
+require LWP::UserAgent;  # this should load everything you need
+
+1;
+
+__END__
+
+=head1 NAME
+
+LWP - Library for WWW access in Perl
+
+=head1 SYNOPSIS
+
+  use LWP;
+  print "This is libwww-perl-$LWP::VERSION\n";
+
+
+=head1 DESCRIPTION
+
+Libwww-perl is a collection of Perl modules which provides a simple
+and consistent programming interface (API) to the World-Wide Web.  The
+main focus of the library is to provide classes and functions that
+allow you to write WWW clients, thus libwww-perl said to be a WWW
+client library. The library also contain modules that are of more
+general use.
+
+Most modules in this library are object oriented.  The user
+agent, requests sent and responses received from the WWW server are
+all represented by objects.  This makes a simple and powerful
+interface to these services.  The interface should be easy to extend
+and customize for your needs.
+
+The main features of the library are:
+
+=over 3
+
+=item *
+
+Contains various reusable components (modules) that can be
+used separately or together.
+
+=item *
+
+Provides an object oriented model of HTTP-style communication.  Within
+this framework we currently support access to http, https, gopher, ftp, news,
+file, and mailto resources.
+
+=item *
+
+The library be used through the full object oriented interface or
+through a very simple procedural interface.
+
+=item *
+
+Support the basic and digest authorization schemes.
+
+=item *
+
+Transparent redirect handling.
+
+=item *
+
+Supports access through proxy servers.
+
+=item *
+
+URL handling (both absolute and relative URLs are supported).
+
+=item *
+
+A parser for F<robots.txt> files and a framework for constructing robots.
+
+=item *
+
+The library can cooperate with Tk.  A simple Tk-based GUI browser
+called 'tkweb' is distributed with the Tk extension for perl.
+
+=item *
+
+An implementation of the HTTP content negotiation algorithm that can
+be used both in protocol modules and in server scripts (like CGI
+scripts).
+
+=item *
+
+It can deal with HTTP cookies.
+
+=item *
+
+A simple command line client application called C<lwp-request>.
+
+=back
+
+
+=head1 HTTP STYLE COMMUNICATION
+
+
+The libwww-perl library is based on HTTP style communication. This
+section try to describe what that means.
+
+Let us start with this quote from the HTTP specification document
+<URL:http://www.w3.org/pub/WWW/Protocols/>:
+
+=over 3
+
+=item
+
+The HTTP protocol is based on a request/response paradigm. A client
+establishes a connection with a server and sends a request to the
+server in the form of a request method, URI, and protocol version,
+followed by a MIME-like message containing request modifiers, client
+information, and possible body content. The server responds with a
+status line, including the message's protocol version and a success or
+error code, followed by a MIME-like message containing server
+information, entity meta-information, and possible body content.
+
+=back
+
+What this means to libwww-perl is that communication always take place
+through these steps: First a I<request> object is created and
+configured. This object is then passed to a server and we get a
+I<response> object in return that we can examine. A request is always
+independent of any previous requests, i.e. the service is stateless.
+The same simple model is used for any kind of service we want to
+access.
+
+For example, if we want to fetch a document from a remote file server,
+then we send it a request that contains a name for that document and
+the response will contain the document itself.  If we access a search
+engine, then the content of the request will contain the query
+parameters and the response will contain the query result.  If we want
+to send a mail message to somebody then we send a request object which
+contains our message to the mail server and the response object will
+contain an acknowledgment that tells us that the message has been
+accepted and will be forwarded to the recipient(s).
+
+It is as simple as that!
+
+
+=head2 The Request Object
+
+The request object has the class name C<HTTP::Request> in
+libwww-perl. The fact that the class name use C<HTTP::> as a name
+prefix only implies that we use the HTTP model of communication. It
+does not limit the kind of services we can try to pass this I<request>
+to.  For instance, we will send C<HTTP::Request>s both to ftp and
+gopher servers, as well as to the local file system.
+
+The main attributes of the request objects are:
+
+=over 3
+
+=item *
+
+The B<method> is a short string that tells what kind of
+request this is.  The most used methods are B<GET>, B<PUT>,
+B<POST> and B<HEAD>.
+
+=item *
+
+The B<url> is a string denoting the protocol, server and
+the name of the "document" we want to access.  The B<url> might
+also encode various other parameters.
+
+=item *
+
+The B<headers> contain additional information about the
+request and can also used to describe the content.  The headers
+is a set of keyword/value pairs.
+
+=item *
+
+The B<content> is an arbitrary amount of data.
+
+=back
+
+=head2 The Response Object
+
+The response object has the class name C<HTTP::Response> in
+libwww-perl.  The main attributes of objects of this class are:
+
+=over 3
+
+=item *
+
+The B<code> is a numerical value that encode the overall
+outcome of the request.
+
+=item *
+
+The B<message> is a short (human readable) string that
+corresponds to the I<code>.
+
+=item *
+
+The B<headers> contain additional information about the
+response and they also describe the content.
+
+=item *
+
+The B<content> is an arbitrary amount of data.
+
+=back
+
+Since we don't want to handle all possible I<code> values directly in
+our programs, the libwww-perl response object have methods that can be
+used to query what kind of response this is.  The most commonly used
+response classification methods are:
+
+=over 3
+
+=item is_success()
+
+The request was was successfully received, understood or accepted.
+
+=item is_error()
+
+The request failed.  The server or the resource might not be
+available, access to the resource might be denied or other things might
+have failed for some reason.
+
+=back
+
+=head2 The User Agent
+
+Let us assume that we have created a I<request> object. What do we
+actually do with it in order to receive a I<response>?
+
+The answer is that you pass it on to a I<user agent> object and this
+object will take care of all the things that need to be done
+(low-level communication and error handling). The user agent will give
+you back a I<response> object. The user agent represents your
+application on the network and it provides you with an interface that
+can accept I<requests> and will return I<responses>.
+
+You should think about the user agent as an interface layer between
+your application code and the network.  Through this interface you are
+able to access the various servers on the network.
+
+The libwww-perl class name for the user agent is
+C<LWP::UserAgent>. Every libwww-perl application that wants to
+communicate should create at least one object of this kind. The main
+method provided by this object is request(). This method takes an
+C<HTTP::Request> object as argument and will (eventually) return a
+C<HTTP::Response> object.
+
+The user agent has many other attributes that lets you
+configure how it will interact with the network and with your
+application code.
+
+=over 3
+
+=item *
+
+The B<timeout> specify how much time we give remote servers in
+creating responses before the library disconnect and creates an
+internal I<timeout> response.
+
+=item *
+
+The B<agent> specify the name that your application should use when it
+presents itself on the network.
+
+=item *
+
+The B<from> attribute can be set to the e-mail address of the person
+responsible for running the application.  If this is set, then the
+address will be sent to the servers with every request.
+
+=item *
+
+The B<parse_head> specify whether we should initialize response
+headers from the E<lt>head> section of HTML documents.
+
+=item *
+
+The B<proxy> and B<no_proxy> specify if and when communication should
+go through a proxy server. <URL:http://www.w3.org/pub/WWW/Proxies/>
+
+=item *
+
+The B<credentials> provide a way to set up user names and
+passwords that is needed to access certain services.
+
+=back
+
+Many applications would want even more control over how they interact
+with the network and they get this by specializing the
+C<LWP::UserAgent> by sub-classing.  The library provide a
+specialization called C<LWP::RobotUA> that is used by robot
+applications.
+
+=head2 An Example
+
+This example shows how the user agent, a request and a response are
+represented in actual perl code:
+
+  # Create a user agent object
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+  $ua->agent("AgentName/0.1 " . $ua->agent);
+
+  # Create a request
+  my $req = new HTTP::Request POST => 'http://www.perl.com/cgi-bin/BugGlimpse';
+  $req->content_type('application/x-www-form-urlencoded');
+  $req->content('match=www&errors=0');
+
+  # Pass request to the user agent and get a response back
+  my $res = $ua->request($req);
+
+  # Check the outcome of the response
+  if ($res->is_success) {
+      print $res->content;
+  } else {
+      print "Bad luck this time\n";
+  }
+
+The $ua is created once when the application starts up.  New request
+objects are normally created for each request sent.
+
+
+=head1 NETWORK SUPPORT
+
+This section goes through the various protocol schemes and describe
+the HTTP style methods that are supported and the headers that might
+have any effect.
+
+For all requests, a "User-Agent" header is added and initialized from
+the $ua->agent value before the request is handed to the network
+layer.  In the same way, a "From" header is initialized from the
+$ua->from value.
+
+For all responses, the library will add a header called "Client-Date".
+This header will encode the time when the response was received by
+your application.  This format and semantics of the header is just
+like the server created "Date" header.  You can also encounter other
+"Client-XXX" headers.  They are all generated by the library
+internally and not something really passed on from the servers.
+
+=head2 HTTP Requests
+
+HTTP request are really just handed off to an HTTP server and it will
+decide what happens.  Few servers implement methods beside the usual
+"GET", "HEAD", "POST" and "PUT" but CGI-scripts can really implement
+any method they like.
+
+If the server is not available then the library will generate an
+internal error response.
+
+The library automatically adds a "Host" and a "Content-Length" header
+to the HTTP request before it is sent over the network.
+
+For GET request you might want to add the "If-Modified-Since" header
+to make the request conditional.
+
+For POST request you should add the "Content-Type" header.  When you
+try to emulate HTML E<lt>FORM> handling you should usually let the value
+of the "Content-Type" header be "application/x-www-form-urlencoded".
+See L<lwpcook> for examples of this.
+
+The libwww-perl HTTP implementation currently support the HTTP/1.0
+protocol.  HTTP/0.9 servers are also handled correctly.
+
+The library allows you to access proxy server through HTTP.  This
+means that you can set up the library to forward all types of request
+through the HTTP protocol module.  See L<LWP::UserAgent> for
+documentation of this.
+
+
+=head2 HTTPS Requests
+
+HTTPS requests are HTTP requests over an encrypted network connection
+using the SSL protocol developed by Netscape.  Everything about HTTP
+requests above also hold for HTTPS requests.  In addition the library
+will add the headers "Client-SSL-Cipher", "Client-SSL-Cert-Subject" and
+"Client-SSL-Cert-Issuer" to the response.  These headers denote the
+encryption method used and the name of the server owner.
+
+The request can contain the header "If-SSL-Cert-Subject" in order to
+make the request conditional on the content of the server certificate.
+If the certificate subject does not match, no request is sent to the
+server and an internally generated error response is returned.  The
+value of the "If-SSL-Cert-Subject" header is interpreted as a Perl
+regular expression.
+
+
+=head2 FTP Requests
+
+The library currently support GET, HEAD and PUT requests.  GET will
+retrieve a file or a directory listing from an FTP server.  PUT will
+store a file on a ftp server.
+
+You can specify a ftp account for servers that want this in addition
+user name and password.  This is specified by passing an "Account"
+header in the request.
+
+User name/password can be specified using basic authorization or be
+encoded in the URL.  Bad logins return an UNAUTHORIZED response with
+"WWW-Authenticate: Basic" and can be treated as basic authorization
+for HTTP.
+
+The library support ftp ASCII transfer mode by specifying the "type=a"
+parameter in the URL.
+
+Directory listings are by default returned unprocessed (as returned
+from the ftp server) with the content media type reported to be
+"text/ftp-dir-listing". The C<File::Listing> module provide functionality
+for parsing of these directory listing.
+
+The ftp module is also able to convert directory listings to HTML and
+this can be requested via the standard HTTP content negotiation
+mechanisms (add an "Accept: text/html" header in the request if you
+want this).
+
+The normal file retrievals, the "Content-Type" is guessed based on the
+file name suffix. See L<LWP::MediaTypes>.
+
+The "If-Modified-Since" request header works for servers that implement
+the MDTM command.  It will probably not work for directory listings though.
+
+Example:
+
+  $req = HTTP::Request->new(GET => 'ftp://me:passwd@ftp.some.where.com/');
+  $req->header(Accept => "text/html, */*;q=0.1");
+
+=head2 News Requests
+
+Access to the USENET News system is implemented through the NNTP
+protocol.  The name of the news server is obtained from the
+NNTP_SERVER environment variable and defaults to "news".  It is not
+possible to specify the hostname of the NNTP server in the news:-URLs.
+
+The library support GET and HEAD to retrieve news articles through the
+NNTP protocol.  You can also post articles to newsgroups by using
+(surprise!) the POST method.
+
+GET on newsgroups is not implemented yet.
+
+Examples:
+
+  $req = HTTP::Request->new(GET => 'news:abc1234@a.sn.no');
+
+  $req = HTTP::Request->new(POST => 'news:comp.lang.perl.test');
+  $req->header(Subject => 'This is a test',
+               From    => 'me@some.where.org');
+  $req->content(<<EOT);
+  This is the content of the message that we are sending to
+  the world.
+  EOT
+
+
+=head2 Gopher Request
+
+The library supports the GET and HEAD method for gopher request.  All
+request header values are ignored.  HEAD cheats and will return a
+response without even talking to server.
+
+Gopher menus are always converted to HTML.
+
+The response "Content-Type" is generated from the document type
+encoded (as the first letter) in the request URL path itself.
+
+Example:
+
+  $req = HTTP::Request->new(GET => 'gopher://gopher.sn.no/');
+
+
+
+=head2 File Request
+
+The library supports GET and HEAD methods for file requests.  The
+"If-Modified-Since" header is supported.  All other headers are
+ignored.  The I<host> component of the file URL must be empty or set
+to "localhost".  Any other I<host> value will be treated as an error.
+
+Directories are always converted to an HTML document.  For normal
+files, the "Content-Type" and "Content-Encoding" in the response are
+guessed based on the file suffix.
+
+Example:
+
+  $req = HTTP::Request->new(GET => 'file:/etc/passwd');
+
+
+=head2 Mailto Request
+
+You can send (aka "POST") mail messages using the library.  All
+headers specified for the request are passed on to the mail system.
+The "To" header is initialized from the mail address in the URL.
+
+Example:
+
+  $req = HTTP::Request->new(POST => 'mailto:libwww-perl-request@ics.uci.edu');
+  $req->header(Subject => "subscribe");
+  $req->content("Please subscribe me to the libwww-perl mailing list!\n");
+
+
+=head1 OVERVIEW OF CLASSES AND PACKAGES
+
+This table should give you a quick overview of the classes provided by the
+library. Indentation shows class inheritance.
+
+ LWP::MemberMixin   -- Access to member variables of Perl5 classes
+   LWP::UserAgent   -- WWW user agent class
+     LWP::RobotUA   -- When developing a robot applications
+   LWP::Protocol          -- Interface to various protocol schemes
+     LWP::Protocol::http  -- http:// access
+     LWP::Protocol::file  -- file:// access
+     LWP::Protocol::ftp   -- ftp:// access
+     ...
+
+ LWP::Authen::Basic -- Handle 401 and 407 responses
+ LWP::Authen::Digest
+
+ HTTP::Headers      -- MIME/RFC822 style header (used by HTTP::Message)
+ HTTP::Message      -- HTTP style message
+   HTTP::Request    -- HTTP request
+   HTTP::Response   -- HTTP response
+ HTTP::Daemon       -- A HTTP server class
+
+ URI::URL           -- Uniform Resource Locators
+
+ WWW::RobotRules    -- Parse robots.txt files
+   WWW::RobotRules::AnyDBM_File -- Persistent RobotRules
+
+The following modules provide various functions and definitions.
+
+ LWP                -- This file.  Library version number and documentation.
+ LWP::MediaTypes    -- MIME types configuration (text/html etc.)
+ LWP::Debug         -- Debug logging module
+ LWP::Simple        -- Simplified procedural interface for common functions
+ HTTP::Status       -- HTTP status code (200 OK etc)
+ HTTP::Date         -- Date parsing module for HTTP date formats
+ HTTP::Negotiate    -- HTTP content negotiation calculation
+ File::Listing      -- Parse directory listings
+
+
+=head1 MORE DOCUMENTATION
+
+All modules contain detailed information on the interfaces they
+provide.  The I<lwpcook> manpage is the libwww-perl cookbook that contain
+examples of typical usage of the library.  You might want to take a
+look at how the scripts C<lwp-request>, C<lwp-rget> and C<lwp-mirror>
+are implemented.
+
+=head1 BUGS
+
+The library can not handle multiple simultaneous requests yet.  Also,
+check out what's left in the TODO file.
+
+=head1 ACKNOWLEDGEMENTS
+
+This package owes a lot in motivation, design, and code, to the
+libwww-perl library for Perl 4, maintained by Roy Fielding
+E<lt>fielding@ics.uci.edu>.
+
+That package used work from Alberto Accomazzi, James Casey, Brooks
+Cutter, Martijn Koster, Oscar Nierstrasz, Mel Melchner, Gertjan van
+Oosten, Jared Rhine, Jack Shirazi, Gene Spafford, Marc VanHeyningen,
+Steven E. Brenner, Marion Hakanson, Waldemar Kebsch, Tony Sanders, and
+Larry Wall; see the libwww-perl-0.40 library for details.
+
+The primary architect for this Perl 5 library is Martijn Koster and
+Gisle Aas, with lots of help from Graham Barr, Tim Bunce, Andreas
+Koenig, Jared Rhine, and Jack Shirazi.
+
+
+=head1 COPYRIGHT
+
+  Copyright 1995-1998, Gisle Aas
+  Copyright 1995, Martijn Koster
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=head1 AVAILABILITY
+
+The latest version of this library is likely to be available from:
+
+ http://www.sn.no/libwww-perl/
+
+The best place to discuss this code is on the
+<libwww-perl@ics.uci.edu> mailing list.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Basic.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Basic.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Basic.pm	(revision 21112)
@@ -0,0 +1,36 @@
+package LWP::Authen::Basic;
+use strict;
+
+require MIME::Base64;
+
+sub authenticate
+{
+    my($class, $ua, $proxy, $auth_param, $response,
+       $request, $arg, $size) = @_;
+
+    my($user, $pass) = $ua->get_basic_credentials($auth_param->{realm},
+                                                  $request->url, $proxy);
+    return $response unless defined $user and defined $pass;
+
+    my $auth_header = $proxy ? "Proxy-Authorization" : "Authorization";
+    my $auth_value = "Basic " . MIME::Base64::encode("$user:$pass", "");
+
+    # Need to check this isn't a repeated fail!
+    my $r = $response;
+    while ($r) {
+	my $auth = $r->request->header($auth_header);
+	if ($auth && $auth eq $auth_value) {
+	    # here we know this failed before
+	    $response->header("Client-Warning" =>
+			      "Credentials for '$user' failed before");
+	    return $response;
+	}
+	$r = $r->previous;
+    }
+
+    my $referral = $request->clone;
+    $referral->header($auth_header => $auth_value);
+    return $ua->request($referral, $arg, $size, $response);
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Digest.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Digest.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Authen/Digest.pm	(revision 21112)
@@ -0,0 +1,73 @@
+package LWP::Authen::Digest;
+use strict;
+
+require MD5;
+
+sub authenticate
+{
+    my($class, $ua, $proxy, $auth_param, $response,
+       $request, $arg, $size) = @_;
+
+    my($user, $pass) = $ua->get_basic_credentials($auth_param->{realm},
+                                                  $request->url, $proxy);
+    return $response unless defined $user and defined $pass;
+
+    my $md5 = new MD5;
+
+    my(@digest);
+    $md5->add(join(":", $user, $auth_param->{realm}, $pass));
+    push(@digest, $md5->hexdigest);
+    $md5->reset;
+
+    push(@digest, $auth_param->{nonce});
+
+    $md5->add(join(":", $request->method, $request->url->path));
+    push(@digest, $md5->hexdigest);
+    $md5->reset;
+
+    $md5->add(join(":", @digest));
+    my($digest) = $md5->hexdigest;
+    $md5->reset;
+
+    my %resp = map { $_ => $auth_param->{$_} } qw(realm nonce opaque);
+    @resp{qw(username uri response)} = ($user, $request->url->path, $digest);
+
+    my(@order) = qw(username realm nonce uri response);
+    if($request->method =~ /^(?:POST|PUT)$/) {
+	$md5->add($request->content);
+	my $content = $md5->hexdigest;
+	$md5->reset;
+	$md5->add(join(":", @digest[0..1], $content));
+	$md5->reset;
+	$resp{"message-digest"} = $md5->hexdigest;
+	push(@order, "message-digest");
+    }
+    push(@order, "opaque");
+    my @pairs;
+    for (@order) {
+	next unless defined $resp{$_};
+	push(@pairs, "$_=" . qq("$resp{$_}"));
+    }
+
+    my $auth_header = $proxy ? "Proxy-Authorization" : "Authorization";
+    my $auth_value  = "Digest " . join(", ", @pairs);
+    
+    # Need to check this isn't a repeated fail!
+    my $r = $response;
+    while ($r) {
+	my $auth = $r->request->header($auth_header);
+	if ($auth && $auth eq $auth_value) {
+	    # here we know this failed before
+	    $response->header("Client-Warning" =>
+			      "Credentials for '$user' failed before");
+	    return $response;
+	}
+	$r = $r->previous;
+    }
+
+    my $referral = $request->clone;
+    $referral->header($auth_header => $auth_value);
+    return $ua->request($referral, $arg, $size, $response);
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Debug.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Debug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Debug.pm	(revision 21112)
@@ -0,0 +1,129 @@
+#!/usr/local/bin/perl -w
+#
+# $Id$
+#
+package LWP::Debug;
+
+=head1 NAME
+
+LWP::Debug - debug routines for the libwww-perl library
+
+=head1 SYNOPSIS
+
+ use LWP::Debug qw(+ -conns);
+
+ # Used internally in the library
+ LWP::Debug::trace('send()');
+ LWP::Debug::debug('url ok');
+ LWP::Debug::conns("read $n bytes: $data");
+
+=head1 DESCRIPTION
+
+LWP::Debug provides tracing facilities. The trace(), debug() and
+conns() function are called within the library and they log
+information at increasing levels of detail. Which level of detail is
+actually printed is controlled with the C<level()> function.
+
+The following functions are available:
+
+=over 4
+
+=item level(...)
+
+The C<level()> function controls the level of detail being
+logged. Passing '+' or '-' indicates full and no logging
+respectively. Inidividual levels can switched on and of by passing the
+name of the level with a '+' or '-' prepended.  The levels are:
+
+  trace   : trace function calls
+  debug   : print debug messages
+  conns   : show all data transfered over the connections
+
+The LWP::Debug module provide a special import() method that allows
+you to pass the level() arguments with initial use statement.  If a
+use argument start with '+' or '-' then it is passed to the level
+function, else the name is exported as usual.  The following two
+statements are thus equivalent (if you ignore that the second pollutes
+your namespace):
+
+  use LWP::Debug qw(+);
+  use LWP::Debug qw(level); level('+');
+
+=item trace($msg)
+
+The C<trace()> function is used for tracing function
+calls. The package and calling subroutine name is
+printed along with the passed argument. This should
+be called at the start of every major function.
+
+=item debug($msg)
+
+The C<debug()> function is used for high-granularity
+reporting of state in functions.
+
+=item conns($msg)
+
+The C<conns()> function is used to show data being
+transferred over the connections. This may generate
+considerable output.
+
+=back
+
+=cut
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT_OK = qw(level trace debug conns);
+
+use Carp ();
+
+my @levels = qw(trace debug conns);
+%current_level = ();
+
+sub import
+{
+    my $pack = shift;
+    my $callpkg = caller(0);
+    my @symbols = ();
+    my @levels = ();
+    for (@_) {
+	if (/^[-+]/) {
+	    push(@levels, $_);
+	} else {
+	    push(@symbols, $_);
+	}
+    }
+    Exporter::export($pack, $callpkg, @symbols);
+    level(@levels);
+}
+
+sub level
+{
+    for (@_) {
+	if ($_ eq '+') {              # all on
+	    # switch on all levels
+	    %current_level = map { $_ => 1 } @levels;
+	} elsif ($_ eq '-') {           # all off
+	    %current_level = ();
+	} elsif (/^([-+])(\w+)$/) {
+	    $current_level{$2} = $1 eq '+';
+	} else {
+	    Carp::croak("Illegal level format $_");
+	}
+    }
+}
+
+sub trace  { _log(@_) if $current_level{'trace'}; }
+sub debug  { _log(@_) if $current_level{'debug'}; }
+sub conns  { _log(@_) if $current_level{'conns'}; }
+
+sub _log
+{
+    my $msg = shift;
+    $msg .= "\n" unless $msg =~ /\n$/;  # ensure trailing "\n"
+
+    my($package,$filename,$line,$sub) = caller(2);
+    print STDERR "$sub: $msg";
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MediaTypes.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MediaTypes.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MediaTypes.pm	(revision 21112)
@@ -0,0 +1,298 @@
+#
+# $Id$
+
+package LWP::MediaTypes;
+
+=head1 NAME
+
+LWP::MediaTypes - guess media type for a file or a URL
+
+=head1 SYNOPSIS
+
+ use LWP::MediaTypes qw(guess_media_type);
+ $type = guess_media_type("/tmp/foo.gif");
+
+=head1 DESCRIPTION
+
+This module provides functions for handling of media (also known as
+MIME) types and encodings.  The mapping from file extentions to media
+types is defined by the F<media.types> file.  If the F<~/.media.types>
+file exist it is used as a replacement.
+For backwards compatability we will also look for F<~/.mime.types>.
+
+The following functions are exported by default:
+
+=over 4
+
+=cut
+
+####################################################################
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(guess_media_type media_suffix);
+@EXPORT_OK = qw(add_type add_encoding);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+require LWP::Debug;
+use strict;
+
+# note: These hashes will also be filled with the entries found in
+# the 'media.types' file.
+
+my %suffixType = (
+    'txt'   => 'text/plain',
+    'html'  => 'text/html',
+    'gif'   => 'image/gif',
+    'jpg'   => 'image/jpeg',
+);
+
+my %suffixExt = (
+    'text/plain' => 'txt',
+    'text/html'  => 'h',
+    'image/gif'  => 'gif',
+    'image/jpeg' => 'jpg',
+);
+
+#XXX: there should be some way to define this in the media.types files.
+my %suffixEncoding = (
+    'Z'   => 'compress',
+    'gz'  => 'gzip',
+    'hqx' => 'x-hqx',
+    'uu'  => 'x-uuencode',
+    'z'   => 'x-pack'
+);
+
+sub _dump {
+    require Data::Dumper;
+    Data::Dumper->new([\%suffixType, \%suffixExt, \%suffixEncoding],
+		      [qw(*suffixType *suffixExt *suffixEncoding)])->Dump;
+}
+
+read_media_types();
+
+
+
+=item guess_media_type($filename_or_url, [$header_to_modify])
+
+This function tries to guess media type and encoding for given file.
+It returns the content-type, which is a string like C<"text/html">.
+In array context it also returns any content-encodings applied (in the
+order used to encode the file).  You can pass a URI::URL object
+reference, instead of the file name, as the first parameter too.
+
+If the type can not be deduced from looking at the file name only,
+then guess_media_type() will let the C<-T> Perl operator take a look.
+If this works (and C<-T> returns a TRUE value) then we return
+I<text/plain> as the type, otherwise we return
+I<application/octet-stream> as the type.
+
+The optional second argument should be a reference to a HTTP::Headers
+object (or any object that implement the $obj->header method in a
+similar way).  When present we will set the values of the
+'Content-Type' and 'Content-Encoding' for this header.
+
+=cut
+
+sub guess_media_type
+{
+    my($file, $header) = @_;
+    return undef unless defined $file;
+
+    my $fullname;
+    if (ref($file)) {
+	# assume URI::URL object
+	$file = $file->path;
+	#XXX should handle non http:, file: or ftp: URLs differently
+    } else {
+	$fullname = $file;  # enable peek at actual file
+    }
+
+    my @encoding = ();
+    my $ct = undef;
+    for (file_exts($file)) {
+	# first check this dot part as encoding spec
+	if (exists $suffixEncoding{$_}) {
+	    unshift(@encoding, $suffixEncoding{$_});
+	    next;
+	}
+	if (exists $suffixEncoding{lc $_}) {
+	    unshift(@encoding, $suffixEncoding{lc $_});
+	    next;
+	}
+
+	# check content-type
+	if (exists $suffixType{$_}) {
+	    $ct = $suffixType{$_};
+	    last;
+	}
+	if (exists $suffixType{lc $_}) {
+	    $ct = $suffixType{lc $_};
+	    last;
+	}
+
+	# don't know nothing about this dot part, bail out
+	last;
+    }
+    unless (defined $ct) {
+	# Take a look at the file
+	if (defined $fullname) {
+	    $ct = (-T $fullname) ? "text/plain" : "application/octet-stream";
+	} else {
+	    $ct = "application/octet-stream";
+	}
+    }
+
+    if ($header) {
+	$header->header('Content-Type' => $ct);
+	$header->header('Content-Encoding' => \@encoding) if @encoding;
+    }
+
+    wantarray ? ($ct, @encoding) : $ct;
+}
+
+
+=item media_suffix($type,...)
+
+This function will return all suffixes that can be used to denote the
+specified media type(s).  Wildcard types can be used.  In scalar
+context it will return the first suffix found.
+
+Examples:
+
+  @suffixes = media_suffix('image/*', 'audio/basic');
+  $suffix = media_suffix('text/html');
+
+=cut
+
+sub media_suffix {
+    if (!wantarray && @_ == 1 && $_[0] !~ /\*/) {
+	return $suffixExt{$_[0]};
+    }
+    my(@type) = @_;
+    my(@suffix, $ext, $type);
+    foreach (@type) {
+	if (s/\*/.*/) {
+	    while(($ext,$type) = each(%suffixType)) {
+		push(@suffix, $ext) if $type =~ /^$_$/;
+	    }
+	} else {
+	    while(($ext,$type) = each(%suffixType)) {
+		push(@suffix, $ext) if $type eq $_;
+	    }
+	}
+    }
+    wantarray ? @suffix : $suffix[0];
+}
+
+
+sub file_exts 
+{
+    my($file) = @_;
+    $file =~ s,.*/,,;   # only basename left
+    my @parts = reverse split(/\./, $file);
+    pop(@parts);        # never concider first part
+    @parts;
+}
+
+
+=back
+
+The following functions are only exported by explict request:
+
+=over 4
+
+=item add_type($type, @exts)
+
+Associate a list of file extensions with the given media type.
+
+Example:
+
+    add_type("x-world/x-vrml" => qw(wrl vrml));
+
+=cut
+
+sub add_type 
+{
+    my($type, @exts) = @_;
+    for my $ext (@exts) {
+	$ext =~ s/^\.//;
+	$suffixType{$ext} = $type;
+    }
+    $suffixExt{$type} = $exts[0] if @exts;
+}
+
+
+=item add_encoding($type, @ext)
+
+Associate a list of file extensions with and encoding type.
+
+Example:
+
+ add_encoding("x-gzip" => "gz");
+
+=cut
+
+sub add_encoding
+{
+    my($type, @exts) = @_;
+    for my $ext (@exts) {
+	$ext =~ s/^\.//;
+	$suffixEncoding{$ext} = $type;
+    }
+}
+
+
+=item read_media_types(@files)
+
+Parse a media types file from disk and add the type mappings found there.
+
+Example:
+
+    read_media_types("conf/mime.types");
+
+=cut
+
+sub read_media_types 
+{
+    my(@files) = @_;
+
+    local($/, $_) = ("\n", undef);  # ensure correct $INPUT_RECORD_SEPARATOR
+
+    my @priv_files = ();
+    push(@priv_files, "$ENV{HOME}/.media.types", "$ENV{HOME}/.mime.types")
+	if defined $ENV{HOME};  # Some doesn't have a home (for instance Win32)
+
+    # Try to locate "media.types" file, and initialize %suffixType from it
+    my $typefile;
+    unless (@files) {
+	@files = map {"$_/LWP/media.types"} @INC;
+	push @files, @priv_files;
+    }
+    for $typefile (@files) {
+	local(*TYPE);
+	open(TYPE, $typefile) || next;
+        LWP::Debug::debug("Reading media types from $typefile");
+	while (<TYPE>) {
+	    next if /^\s*#/; # comment line
+	    next if /^\s*$/; # blank line
+	    s/#.*//;         # remove end-of-line comments
+	    my($type, @exts) = split(' ', $_);
+	    add_type($type, @exts);
+	}
+	close(TYPE);
+    }
+}
+
+1;
+
+=back 
+
+=head1 COPYRIGHT
+
+Copyright 1995-1998 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MemberMixin.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MemberMixin.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/MemberMixin.pm	(revision 21112)
@@ -0,0 +1,46 @@
+#
+# $Id$
+
+package LWP::MemberMixin;
+
+=head1 NAME
+
+LWP::MemberMixin - Member access mixin class
+
+=head1 SYNOPSIS
+
+ package Foo;
+ require LWP::MemberMixin;
+ @ISA=qw(LWP::MemberMixin);
+
+=head1 DESCRIPTION
+
+A mixin class to get methods that provide easy access to member
+variables in the %$self.
+Ideally there should be better Perl langauge support for this.
+
+There is only one method provided:
+
+=over 4
+
+=item _elem($elem [, $val])
+
+Internal method to get/set the value of member variable
+C<$elem>. If C<$val> is defined it is used as the new value
+for the member variable.  If it is undefined the current
+value is not touched. In both cases the previous value of
+the member variable is returned.
+
+=back
+
+=cut
+
+sub _elem
+{
+    my($self, $elem, $val) = @_;
+    my $old = $self->{$elem};
+    $self->{$elem} = $val if defined $val;
+    return $old;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol.pm	(revision 21112)
@@ -0,0 +1,303 @@
+# $Id$
+
+package LWP::Protocol;
+
+=head1 NAME
+
+LWP::Protocol - Base class for LWP protocols
+
+=head1 SYNOPSIS
+
+ package LWP::Protocol::foo;
+ require LWP::Protocol;
+ @ISA=qw(LWP::Protocol);
+
+=head1 DESCRIPTION
+
+This class is used a the base class for all protocol implementations
+supported by the LWP library.
+
+When creating an instance of this class using
+C<LWP::Protocol::create($url)>, and you get an initialised subclass
+appropriate for that access method. In other words, the
+LWP::Protocol::create() function calls the constructor for one of its
+subclasses.
+
+All derived LWP::Protocol classes need to override the request()
+method which is used to service a request. The overridden method can
+make use of the collect() function to collect together chunks of data
+as it is received.
+
+The following methods and functions are provided:
+
+=over 4
+
+=cut
+
+#####################################################################
+
+require LWP::MemberMixin;
+@ISA = qw(LWP::MemberMixin);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use strict;
+use Carp ();
+use HTTP::Status 'RC_INTERNAL_SERVER_ERROR';
+require HTML::HeadParser;
+
+my %ImplementedBy = (); # scheme => classname
+
+
+=item $prot = new HTTP::Protocol;
+
+The LWP::Protocol constructor is inherited by subclasses. As this is a
+virtual base class this method should B<not> be called directly.
+
+=cut
+
+sub new
+{
+    my($class) = @_;
+
+    my $self = bless {
+	'timeout' => 0,
+	'parse_head' => 1,
+    }, $class;
+    $self;
+}
+
+
+=item $prot = LWP::Protocol::create($url)
+
+Create an object of the class implementing the protocol to handle the
+given scheme. This is a function, not a method. It is more an object
+factory than a constructor. This is the function user agents should
+use to access protocols.
+
+=cut
+
+sub create
+{
+    my $scheme = shift;
+    my $impclass = LWP::Protocol::implementor($scheme) or
+	Carp::croak("Protocol scheme '$scheme' is not supported");
+
+    # hand-off to scheme specific implementation sub-class
+    return $impclass->new($scheme);
+}
+
+
+=item $class = LWP::Protocol::implementor($scheme, [$class])
+
+Get and/or set implementor class for a scheme.  Returns '' if the
+specified scheme is not supported.
+
+=cut
+
+sub implementor
+{
+    my($scheme, $impclass) = @_;
+
+    if ($impclass) {
+	$ImplementedBy{$scheme} = $impclass;
+    }
+    my $ic = $ImplementedBy{$scheme};
+    return $ic if $ic;
+
+    return '' unless $scheme =~ /^([.+\-\w]+)$/;  # check valid URL schemes
+    $scheme = $1; # untaint
+    $scheme =~ s/[.+\-]/_/g;  # make it a legal module name
+
+    # scheme not yet known, look for a 'use'd implementation
+    $ic = "LWP::Protocol::$scheme";  # default location
+    $ic = "LWP::Protocol::nntp" if $scheme eq 'news'; #XXX ugly hack
+    no strict 'refs';
+    # check we actually have one for the scheme:
+    unless (defined @{"${ic}::ISA"}) {
+	# try to autoload it
+	eval "require $ic";
+	if ($@) {
+	    if ($@ =~ /^Can't locate/) { #' #emacs get confused by '
+		$ic = '';
+	    } else {
+		die "$@\n";
+	    }
+	}
+    }
+    $ImplementedBy{$scheme} = $ic if $ic;
+    $ic;
+}
+
+
+=item $prot->request(...)
+
+ $response = $protocol->request($request, $proxy, undef);
+ $response = $protocol->request($request, $proxy, '/tmp/sss');
+ $response = $protocol->request($request, $proxy, \&callback, 1024);
+
+Dispactches a request over the protocol, and returns a response
+object. This method needs to be overridden in subclasses.  Referer to
+L<LWP::UserAgent> for description of the arguments.
+
+=cut
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size, $timeout) = @_;
+    Carp::croak('LWP::Protocol::request() needs to be overridden in subclasses');
+}
+
+
+=item $prot->timeout($seconds)
+
+Get and set the timeout value in seconds
+
+
+=item $prot->parse_head($yesno)
+
+Should we initialize response headers from the <head> section of HTML
+documents.
+
+=cut
+
+sub timeout    { shift->_elem('timeout',    @_); }
+sub parse_head { shift->_elem('parse_head', @_); }
+sub max_size   { shift->_elem('max_size',   @_); }
+
+
+=item $prot->collect($arg, $response, $collector)
+
+Called to collect the content of a request, and process it
+appropriately into a scalar, file, or by calling a callback.  If $arg
+is undefined, then the content is stored within the $response.  If
+$arg is a simple scalar, then $arg is interpreted as a file name and
+the content is written to this file.  If $arg is a reference to a
+routine, then content is passed to this routine.
+
+The $collector is a routine that will be called and which is
+reponsible for returning pieces (as ref to scalar) of the content to
+process.  The $collector signals EOF by returning a reference to an
+empty sting.
+
+The return value from collect() is the $response object reference.
+
+B<Note:> We will only use the callback or file argument if
+$response->is_success().  This avoids sendig content data for
+redirects and authentization responses to the callback which would be
+confusing.
+
+=cut
+
+sub collect
+{
+    my ($self, $arg, $response, $collector) = @_;
+    my $content;
+    my($parse_head, $timeout, $max_size) =
+      @{$self}{qw(parse_head timeout max_size)};
+
+    my $parser;
+    if ($parse_head && $response->content_type eq 'text/html') {
+	$parser = HTML::HeadParser->new($response->{'_headers'});
+    }
+    my $content_size = 0;
+
+    if (!defined($arg) || !$response->is_success) {
+	# scalar
+	while ($content = &$collector, length $$content) {
+	    if ($parser) {
+		$parser->parse($$content) or undef($parser);
+	    }
+	    LWP::Debug::debug("read " . length($$content) . " bytes");
+	    $response->add_content($$content);
+	    $content_size += length($$content);
+	    if ($max_size && $content_size > $max_size) {
+		LWP::Debug::debug("Aborting because size limit exceeded");
+		my $tot = $response->header("Content-Length") || 0;
+		$response->header("X-Content-Range", "bytes 0-$content_size/$tot");
+		last;
+	    }
+	}
+    }
+    elsif (!ref($arg)) {
+	# filename
+	open(OUT, ">$arg") or
+	    return new HTTP::Response RC_INTERNAL_SERVER_ERROR,
+			  "Cannot write to '$arg': $!";
+        binmode(OUT);
+        local($\) = ""; # ensure standard $OUTPUT_RECORD_SEPARATOR
+	while ($content = &$collector, length $$content) {
+	    if ($parser) {
+		$parser->parse($$content) or undef($parser);
+	    }
+	    LWP::Debug::debug("read " . length($$content) . " bytes");
+	    print OUT $$content;
+	    $content_size += length($$content);
+	    if ($max_size && $content_size > $max_size) {
+		LWP::Debug::debug("Aborting because size limit exceeded");
+		my $tot = $response->header("Content-Length") || 0;
+		$response->header("X-Content-Range", "bytes 0-$content_size/$tot");
+		last;
+	    }
+	}
+	close(OUT);
+    }
+    elsif (ref($arg) eq 'CODE') {
+	# read into callback
+	while ($content = &$collector, length $$content) {
+	    if ($parser) {
+		$parser->parse($$content) or undef($parser);
+	    }
+	    LWP::Debug::debug("read " . length($$content) . " bytes");
+            eval {
+		&$arg($$content, $response, $self);
+	    };
+	    if ($@) {
+	        chomp($@);
+		$response->header('X-Died' => $@);
+		last;
+	    }
+	}
+    }
+    else {
+	return new HTTP::Response RC_INTERNAL_SERVER_ERROR,
+				  "Unexpected collect argument  '$arg'";
+    }
+    $response;
+}
+
+
+=item $prot->collect_once($arg, $response, $content)
+
+Can be called when the whole response content is available as
+$content.  This will invoke collect() with a collector callback that
+returns a reference to $content the first time and an empty string the
+next.
+
+=cut
+
+sub collect_once
+{
+    my($self, $arg, $response) = @_;
+    my $content = \ $_[3];
+    my $first = 1;
+    $self->collect($arg, $response, sub {
+	return $content if $first--;
+	return \ "";
+    });
+}
+
+1;
+
+=head1 SEE ALSO
+
+Inspect the F<LWP/Protocol/file.pm> and F<LWP/Protocol/http.pm> files
+for examples of usage.
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/data.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/data.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/data.pm	(revision 21112)
@@ -0,0 +1,53 @@
+#
+# $Id$
+#
+# Implements access to data:-URLs as specified in
+# draft-masinter-url-data-02.txt
+
+package LWP::Protocol::data;
+
+require HTTP::Response;
+require HTTP::Status;
+
+require LWP::Protocol;
+@ISA = qw(LWP::Protocol);
+
+use HTTP::Date qw(time2str);
+require LWP;  # needs version number
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size) = @_;
+
+    # check proxy
+    if (defined $proxy)
+    {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'You can not proxy with data';
+    }
+
+    # check method
+    if ($request->method ne 'GET') {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'Library does not allow method ' .
+				  "$method for 'data:' URLs";
+    }
+
+    my $url = $request->url;
+    my $response = new HTTP::Response &HTTP::Status::RC_OK, "Document follows";
+
+    my($media_type, $params) = $url->media_type;
+    $media_type .= ";$params" if $params;
+
+    my $data = $url->data;
+    $response->header('Content-Type'   => $media_type,
+		      'Content-Length' => length($data),
+		      'Date'           => time2str(time),
+		      'Server'         => "libwww-perl-internal/$LWP::VERSION"
+		     );
+    $response->content($data);
+
+    return $response;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/file.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/file.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/file.pm	(revision 21112)
@@ -0,0 +1,154 @@
+#
+# $Id$
+
+package LWP::Protocol::file;
+
+require LWP::Protocol;
+require LWP::MediaTypes;
+require HTTP::Request;
+require HTTP::Response;
+require HTTP::Status;
+require HTTP::Date;
+
+require URI::Escape;
+require HTML::Entities;
+
+use Carp;
+
+@ISA = qw(LWP::Protocol);
+
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size) = @_;
+
+    LWP::Debug::trace('()');
+
+    $size = 4096 unless defined $size and $size > 0;
+
+    # check proxy
+    if (defined $proxy)
+    {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'You can not proxy through the filesystem';
+    }
+
+    # check method
+    $method = $request->method;
+
+    unless ($method eq 'GET' || $method eq 'HEAD') {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'Library does not allow method ' .
+				  "$method for 'file:' URLs";
+    }
+
+    # check url
+    my $url = $request->url;
+
+    my $scheme = $url->scheme;
+    if ($scheme ne 'file') {
+	return new HTTP::Response &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				  "LWP::file::request called for '$scheme'";
+    }
+
+    my $host = $url->host;
+    if ($host and $host !~ /^localhost$/i) {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST_CLIENT,
+				  'Only file://localhost/ allowed';
+    }
+
+    # URL OK, look at file
+    my $path  = $url->local_path;
+
+    # test file exists and is readable
+    unless (-e $path) {
+	return new HTTP::Response &HTTP::Status::RC_NOT_FOUND,
+				  "File `$path' does not exist";
+    }
+    unless (-r _) {
+	return new HTTP::Response &HTTP::Status::RC_FORBIDDEN,
+				  'User does not have read permission';
+    }
+
+    # looks like file exists
+    my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$filesize,
+       $atime,$mtime,$ctime,$blksize,$blocks)
+	    = stat(_);
+
+    # XXX should check Accept headers?
+
+    # check if-modified-since
+    my $ims = $request->header('If-Modified-Since');
+    if (defined $ims) {
+	my $time = HTTP::Date::str2time($ims);
+	if (defined $time and $time >= $mtime) {
+	    return new HTTP::Response &HTTP::Status::RC_NOT_MODIFIED,
+				      "$method $path";
+	}
+    }
+
+    # Ok, should be an OK response by now...
+    $response = new HTTP::Response &HTTP::Status::RC_OK;
+
+    # fill in response headers
+    $response->header('Last-Modified', HTTP::Date::time2str($mtime));
+
+    if (-d _) {         # If the path is a directory, process it
+	# generate the HTML for directory
+	opendir(D, $path) or
+	   return new HTTP::Response &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				     "Cannot read directory '$path': $!";
+	my(@files) = sort readdir(D);
+	closedir(D);
+
+	# Make directory listing
+	for (@files) {
+	    $_ .= "/" if -d "$path/$_";
+	    my $furl = URI::Escape::uri_escape($_);
+	    my $desc = HTML::Entities::encode($_);
+	    $_ = qq{<LI><A HREF="$furl">$desc</A>};
+	}
+	# Ensure that the base URL is "/" terminated
+	my $base = $url->clone;
+	unless ($base->epath =~ m|/$|) {
+	    $base->epath($base->epath . "/");
+	}
+	my $html = join("\n",
+			"<HTML>\n<HEAD>",
+			"<TITLE>Directory $path</TITLE>",
+			"<BASE HREF=\"$base\">",
+			"</HEAD>\n<BODY>",
+			"<H1>Directory listing of $path</H1>",
+			"<UL>", @files, "</UL>",
+			"</BODY>\n</HTML>\n");
+
+	$response->header('Content-Type',   'text/html');
+	$response->header('Content-Length', length $html);
+
+	return $self->collect_once($arg, $response, $html);
+
+    } else {            # path is a regular file
+	my($type, @enc) = LWP::MediaTypes::guess_media_type($path);
+	$response->header('Content-Type',   $type) if $type;
+	$response->header('Content-Length', $filesize);
+	for (@enc) {
+	    $response->push_header('Content-Encoding', $_);
+	}
+
+	# read the file
+	open(F, $path) or return new
+	   HTTP::Response(&HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+			  "Cannot read file '$path': $!");
+	$response =  $self->collect($arg, $response, sub {
+	    my $content = "";
+	    my $bytes = sysread(F, $content, $size);
+	    return \$content if $bytes > 0;
+	    return \ "";
+	});
+	close(F);
+    }
+
+    $response;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/ftp.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/ftp.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/ftp.pm	(revision 21112)
@@ -0,0 +1,396 @@
+#
+# $Id$
+
+# Implementation of the ftp protocol (RFC 959). We let the Net::FTP
+# package do all the dirty work.
+
+package LWP::Protocol::ftp;
+
+use Carp ();
+
+use HTTP::Status ();
+use HTTP::Negotiate ();
+use HTTP::Response ();
+use LWP::MediaTypes ();
+use File::Listing ();
+
+require LWP::Protocol;
+@ISA = qw(LWP::Protocol);
+
+use strict;
+eval {
+    require Net::FTP;
+    Net::FTP->require_version(2.00);
+};
+my $init_failed = $@;
+
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size, $timeout) = @_;
+
+    $size = 4096 unless $size;
+
+    LWP::Debug::trace('()');
+
+    # check proxy
+    if (defined $proxy)
+    {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'You can not proxy through the ftp');
+    }
+
+    my $url = $request->url;
+    if ($url->scheme ne 'ftp') {
+	my $scheme = $url->scheme;
+	return HTTP::Response->new(&HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+		       "LWP::Protocol::ftp::request called for '$scheme'");
+    }
+
+    # check method
+    my $method = $request->method;
+
+    unless ($method eq 'GET' || $method eq 'HEAD' || $method eq 'PUT') {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'Library does not allow method ' .
+				   "$method for 'ftp:' URLs");
+    }
+
+    if ($init_failed) {
+	return HTTP::Response->new(&HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				   $init_failed);
+    }
+
+    my $host     = $url->host;
+    my $port     = $url->port;
+    my $user     = $url->user;
+    my $password = $url->password;
+
+    # If a basic autorization header is present than we prefer these over
+    # the username/password specified in the URL.
+    {
+	my($u,$p) = $request->authorization_basic;
+	if (defined $u) {
+	    $user = $u;
+	    $password = $p;
+	}
+    }
+
+    # We allow the account to be specified in the "Account" header
+    my $acct     = $request->header('Account');
+
+    # try to make a connection
+    my $ftp = Net::FTP->new($host, Port => $port);
+    unless ($ftp) {
+       $@ =~ s/^Net::FTP: //;
+       return HTTP::Response->new(&HTTP::Status::RC_INTERNAL_SERVER_ERROR, $@);
+    }
+
+    # Create an initial response object
+    my $response = HTTP::Response->new(&HTTP::Status::RC_OK,
+				       "Document follows");
+    $response->request($request);
+
+    my $mess = $ftp->message;  # welcome message
+    LWP::Debug::debug($mess);
+    $mess =~ s|\n.*||s; # only first line left
+    $mess =~ s|\s*ready\.?$||;
+    # Make the version number more HTTP like
+    $mess =~ s|\s*\(Version\s*|/| and $mess =~ s|\)$||;
+    $response->header("Server", $mess);
+
+    $ftp->timeout($timeout) if $timeout;
+
+    LWP::Debug::debug("Logging in as $user (password $password)...");
+    unless ($ftp->login($user, $password, $acct)) {
+	# Unauthorized.  Let's fake a RC_UNAUTHORIZED response
+	my $res =  HTTP::Response->new(&HTTP::Status::RC_UNAUTHORIZED,
+				       scalar($ftp->message));
+	$res->header("WWW-Authenticate", qq(Basic Realm="FTP login"));
+	return $res;
+    }
+    LWP::Debug::debug($ftp->message);
+
+    # Get & fix the path
+    my @path =  $url->path_components;
+    shift(@path);  # There will always be an empty first component
+    pop(@path) while @path && $path[-1] eq '';  # remove empty tailing comps
+    my $remote_file = pop(@path);
+    $remote_file = '' unless defined $remote_file;
+
+    my $params = $url->params;
+    if (defined($params) && $params eq 'type=a') {
+	$ftp->ascii;
+    } else {
+	$ftp->binary;
+    }
+
+    for (@path) {
+	LWP::Debug::debug("CWD $_");
+	unless ($ftp->cwd($_)) {
+	    return HTTP::Response->new(&HTTP::Status::RC_NOT_FOUND,
+				       "Can't chdir to $_");
+	}
+    }
+
+    if ($method eq 'GET' || $method eq 'HEAD') {
+	LWP::Debug::debug("MDTM");
+	if (my $mod_time = $ftp->mdtm($remote_file)) {
+	    $response->last_modified($mod_time);
+	    if (my $ims = $request->if_modified_since) {
+		if ($mod_time <= $ims) {
+		    $response->code(&HTTP::Status::RC_NOT_MODIFIED);
+		    $response->message("Not modified");
+		    return $response;
+		}
+	    }
+	}
+
+	my $data;  # the data handle
+	LWP::Debug::debug("retrieve file?");
+	if (length($remote_file) and $data = $ftp->retr($remote_file)) {
+	    my($type, @enc) = LWP::MediaTypes::guess_media_type($remote_file);
+	    $response->header('Content-Type',   $type) if $type;
+	    for (@enc) {
+		$response->push_header('Content-Encoding', $_);
+	    }
+	    my $mess = $ftp->message;
+	    LWP::Debug::debug($mess);
+	    if ($mess =~ /\((\d+)\s+bytes\)/) {
+		$response->header('Content-Length', "$1");
+	    }
+
+	    if ($method ne 'HEAD') {
+		# Read data from server
+		$response = $self->collect($arg, $response, sub {
+		    my $content = '';
+		    my $result = $data->read($content, $size);
+		    return \$content;
+		} );
+	    }
+	    unless ($data->close) {
+		# Something did not work too well
+		if ($method ne 'HEAD') {
+		    $response->code(&HTTP::Status::RC_INTERNAL_SERVER_ERROR);
+		    $response->message("FTP close response: " . $ftp->code .
+				       " " . $ftp->message);
+		}
+	    }
+	} elsif (!length($remote_file) || $ftp->code == 550) {
+	    # 550 not a plain file, try to list instead
+	    if (length($remote_file) && !$ftp->cwd($remote_file)) {
+		LWP::Debug::debug("chdir before listing failed");
+		return HTTP::Response->new(&HTTP::Status::RC_NOT_FOUND,
+					   "File '$remote_file' not found");
+	    }
+
+	    # It should now be safe to try to list the directory
+	    LWP::Debug::debug("dir");
+	    my @lsl = $ftp->dir;
+
+	    # Try to figure out if the user want us to convert the
+	    # directory listing to HTML.
+	    my @variants =
+	      (
+	       ['html',  0.60, 'text/html'            ],
+	       ['dir',   1.00, 'text/ftp-dir-listing' ]
+	      );
+	    #$HTTP::Negotiate::DEBUG=1;
+	    my $prefer = HTTP::Negotiate::choose(\@variants, $request);
+
+	    my $content = '';
+
+	    if (!defined($prefer)) {
+		return HTTP::Response->new(&HTTP::Status::RC_NOT_ACCEPTABLE,
+			       "Neither HTML nor directory listing wanted");
+	    } elsif ($prefer eq 'html') {
+		$response->header('Content-Type' => 'text/html');
+		$content = "<HEAD><TITLE>File Listing</TITLE>\n";
+		my $base = $request->url->clone;
+		my $path = $base->epath;
+		$base->epath("$path/") unless $path =~ m|/$|;
+		$content .= qq(<BASE HREF="$base">\n</HEAD>\n);
+		$content .= "<BODY>\n<UL>\n";
+		for (File::Listing::parse_dir(\@lsl, 'GMT')) {
+		    my($name, $type, $size, $mtime, $mode) = @$_;
+		    $content .= qq(  <LI> <a href="$name">$name</a>);
+		    $content .= " $size bytes" if $type eq 'f';
+		    $content .= "\n";
+		}
+		$content .= "</UL></body>\n";
+	    } else {
+		$response->header('Content-Type', 'text/ftp-dir-listing');
+		$content = join("\n", @lsl, '');
+	    }
+
+	    $response->header('Content-Length', length($content));
+
+	    if ($method ne 'HEAD') {
+		$response = $self->collect_once($arg, $response, $content);
+	    }
+	} else {
+	    my $res = HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+			  "FTP return code " . $ftp->code);
+	    $res->content_type("text/plain");
+	    $res->content($ftp->message);
+	    return $res;
+	}
+    } elsif ($method eq 'PUT') {
+	# method must be PUT
+	unless (length($remote_file)) {
+	    return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				       "Must have a file name to PUT to");
+	}
+	my $data;
+	if ($data = $ftp->stor($remote_file)) {
+	    LWP::Debug::debug($ftp->message);
+	    LWP::Debug::debug("$data");
+	    my $content = $request->content;
+	    my $bytes = 0;
+	    if (defined $content) {
+		if (ref($content) eq 'SCALAR') {
+		    $bytes = $data->write($$content, length($$content));
+		} elsif (ref($content) eq 'CODE') {
+		    my($buf, $n);
+		    while (length($buf = &$content)) {
+			$n = $data->write($buf, length($buf));
+			last unless $n;
+			$bytes += $n;
+		    }
+		} elsif (!ref($content)) {
+		    if (defined $content && length($content)) {
+			$bytes = $data->write($content, length($content));
+		    }
+		} else {
+		    die "Bad content";
+		}
+	    }
+	    $data->close;
+	    LWP::Debug::debug($ftp->message);
+
+	    $response->code(&HTTP::Status::RC_CREATED);
+	    $response->header('Content-Type', 'text/plain');
+	    $response->content("$bytes bytes stored as $remote_file on $host\n")
+
+	} else {
+	    my $res = HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+					  "FTP return code " . $ftp->code);
+	    $res->content_type("text/plain");
+	    $res->content($ftp->message);
+	    return $res;
+	}
+    } else {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   "Illegal method $method");
+    }
+
+    $response;
+}
+
+1;
+
+__END__
+
+# This is what RFC 1738 has to say about FTP access:
+# --------------------------------------------------
+#
+# 3.2. FTP
+#
+#    The FTP URL scheme is used to designate files and directories on
+#    Internet hosts accessible using the FTP protocol (RFC959).
+#
+#    A FTP URL follow the syntax described in Section 3.1.  If :<port> is
+#    omitted, the port defaults to 21.
+#
+# 3.2.1. FTP Name and Password
+#
+#    A user name and password may be supplied; they are used in the ftp
+#    "USER" and "PASS" commands after first making the connection to the
+#    FTP server.  If no user name or password is supplied and one is
+#    requested by the FTP server, the conventions for "anonymous" FTP are
+#    to be used, as follows:
+#
+#         The user name "anonymous" is supplied.
+#
+#         The password is supplied as the Internet e-mail address
+#         of the end user accessing the resource.
+#
+#    If the URL supplies a user name but no password, and the remote
+#    server requests a password, the program interpreting the FTP URL
+#    should request one from the user.
+#
+# 3.2.2. FTP url-path
+#
+#    The url-path of a FTP URL has the following syntax:
+#
+#         <cwd1>/<cwd2>/.../<cwdN>/<name>;type=<typecode>
+#
+#    Where <cwd1> through <cwdN> and <name> are (possibly encoded) strings
+#    and <typecode> is one of the characters "a", "i", or "d".  The part
+#    ";type=<typecode>" may be omitted. The <cwdx> and <name> parts may be
+#    empty. The whole url-path may be omitted, including the "/"
+#    delimiting it from the prefix containing user, password, host, and
+#    port.
+#
+#    The url-path is interpreted as a series of FTP commands as follows:
+#
+#       Each of the <cwd> elements is to be supplied, sequentially, as the
+#       argument to a CWD (change working directory) command.
+#
+#       If the typecode is "d", perform a NLST (name list) command with
+#       <name> as the argument, and interpret the results as a file
+#       directory listing.
+#
+#       Otherwise, perform a TYPE command with <typecode> as the argument,
+#       and then access the file whose name is <name> (for example, using
+#       the RETR command.)
+#
+#    Within a name or CWD component, the characters "/" and ";" are
+#    reserved and must be encoded. The components are decoded prior to
+#    their use in the FTP protocol.  In particular, if the appropriate FTP
+#    sequence to access a particular file requires supplying a string
+#    containing a "/" as an argument to a CWD or RETR command, it is
+#    necessary to encode each "/".
+#
+#    For example, the URL <URL:ftp://myname@host.dom/%2Fetc/motd> is
+#    interpreted by FTP-ing to "host.dom", logging in as "myname"
+#    (prompting for a password if it is asked for), and then executing
+#    "CWD /etc" and then "RETR motd". This has a different meaning from
+#    <URL:ftp://myname@host.dom/etc/motd> which would "CWD etc" and then
+#    "RETR motd"; the initial "CWD" might be executed relative to the
+#    default directory for "myname". On the other hand,
+#    <URL:ftp://myname@host.dom//etc/motd>, would "CWD " with a null
+#    argument, then "CWD etc", and then "RETR motd".
+#
+#    FTP URLs may also be used for other operations; for example, it is
+#    possible to update a file on a remote file server, or infer
+#    information about it from the directory listings. The mechanism for
+#    doing so is not spelled out here.
+#
+# 3.2.3. FTP Typecode is Optional
+#
+#    The entire ;type=<typecode> part of a FTP URL is optional. If it is
+#    omitted, the client program interpreting the URL must guess the
+#    appropriate mode to use. In general, the data content type of a file
+#    can only be guessed from the name, e.g., from the suffix of the name;
+#    the appropriate type code to be used for transfer of the file can
+#    then be deduced from the data content of the file.
+#
+# 3.2.4 Hierarchy
+#
+#    For some file systems, the "/" used to denote the hierarchical
+#    structure of the URL corresponds to the delimiter used to construct a
+#    file name hierarchy, and thus, the filename will look similar to the
+#    URL path. This does NOT mean that the URL is a Unix filename.
+#
+# 3.2.5. Optimization
+#
+#    Clients accessing resources via FTP may employ additional heuristics
+#    to optimize the interaction. For some FTP servers, for example, it
+#    may be reasonable to keep the control connection open while accessing
+#    multiple URLs from the same server. However, there is no common
+#    hierarchical model to the FTP protocol, so if a directory change
+#    command has been given, it is impossible in general to deduce what
+#    sequence should be given to navigate to another directory for a
+#    second retrieval, if the paths are different.  The only reliable
+#    algorithm is to disconnect and reestablish the control connection.
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/gopher.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/gopher.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/gopher.pm	(revision 21112)
@@ -0,0 +1,215 @@
+#
+# $Id$
+
+# Implementation of the gopher protocol (RFC 1436)
+#
+# This code is based on 'wwwgopher.pl,v 0.10 1994/10/17 18:12:34 shelden'
+# which in turn is a vastly modified version of Oscar's http'get()
+# dated 28/3/94 in <ftp://cui.unige.ch/PUBLIC/oscar/scripts/http.pl>
+# including contributions from Marc van Heyningen and Martijn Koster.
+#
+
+package LWP::Protocol::gopher;
+
+use strict;
+use vars qw(@ISA);
+
+require HTTP::Response;
+require HTTP::Status;
+require IO::Socket;
+require IO::Select;
+
+require LWP::Protocol;
+@ISA = qw(LWP::Protocol);
+
+
+my %gopher2mimetype = (
+    '0' => 'text/plain',                # 0 file
+    '1' => 'text/html',                 # 1 menu
+					# 2 CSO phone-book server
+					# 3 Error
+    '4' => 'application/mac-binhex40',  # 4 BinHexed Macintosh file
+    '5' => 'application/zip',           # 5 DOS binary archive of some sort
+    '6' => 'application/octet-stream',  # 6 UNIX uuencoded file.
+    '7' => 'text/html',                 # 7 Index-Search server
+					# 8 telnet session
+    '9' => 'application/octet-stream',  # 9 binary file
+    'h' => 'text/html',                 # html
+    'g' => 'image/gif',                 # gif
+    'I' => 'image/*',                   # some kind of image
+);
+
+my %gopher2encoding = (
+    '6' => 'x_uuencode',                # 6 UNIX uuencoded file.
+);
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size, $timeout) = @_;
+
+    LWP::Debug::trace('()');
+
+    $size = 4096 unless $size;
+
+    # check proxy
+    if (defined $proxy) {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'You can not proxy through the gopher');
+    }
+
+    my $url = $request->url;
+    die "bad scheme" if $url->scheme ne 'gopher';
+
+
+    my $method = $request->method;
+    unless ($method eq 'GET' || $method eq 'HEAD') {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'Library does not allow method ' .
+				   "$method for 'gopher:' URLs");
+    }
+
+    my $gophertype = $url->gtype;
+    unless (exists $gopher2mimetype{$gophertype}) {
+	return HTTP::Response->new(&HTTP::Status::RC_NOT_IMPLEMENTED,
+				   'Library does not support gophertype ' .
+				   $gophertype);
+    }
+
+    my $response = HTTP::Response->new(&HTTP::Status::RC_OK, "OK");
+    $response->header('Content-type' => $gopher2mimetype{$gophertype}
+					|| 'text/plain');
+    $response->header('Content-Encoding' => $gopher2encoding{$gophertype})
+	if exists $gopher2encoding{$gophertype};
+
+    if ($method eq 'HEAD') {
+	# XXX: don't even try it so we set this header
+	$response->header('Client-Warning' => 'Client answer only');
+	return $response;
+    }
+    
+    if ($gophertype eq '7' && ! $url->search) {
+      # the url is the prompt for a gopher search; supply boiler-plate
+      return $self->collect_once($arg, $response, <<"EOT");
+<HEAD>
+<TITLE>Gopher Index</TITLE>
+<ISINDEX>
+</HEAD>
+<BODY>
+<H1>$url<BR>Gopher Search</H1>
+This is a searchable Gopher index.
+Use the search function of your browser to enter search terms.
+</BODY>
+EOT
+    }
+
+    my $host = $url->host;
+    my $port = $url->port;
+
+    my $requestLine = "";
+
+    my $selector = $url->selector;
+    if (defined $selector) {
+	$requestLine .= $selector;
+	my $search = $url->search;
+	if (defined $search) {
+	    $requestLine .= "\t$search";
+	    my $string = $url->string;
+	    if (defined $string) {
+		$requestLine .= "\t$string";
+	    }
+	}
+    }
+    $requestLine .= "\015\012";
+
+    # potential request headers are just ignored
+
+    # Ok, lets make the request
+    my $socket = IO::Socket::INET->new(PeerAddr => $host,
+				       PeerPort => $port,
+				       Timeout  => $timeout);
+    die "Can't connect to $host:$port" unless $socket;
+    my $sel = IO::Select->new($socket);
+
+    {
+	die "write timeout" if $timeout && !$sel->can_write($timeout);
+	my $n = syswrite($socket, $requestLine, length($requestLine));
+	die $! unless defined($n);
+	die "short write" if $n != length($requestLine);
+    }
+
+    my $user_arg = $arg;
+
+    # must handle menus in a special way since they are to be
+    # converted to HTML.  Undefing $arg ensures that the user does
+    # not see the data before we get a change to convert it.
+    $arg = undef if $gophertype eq '1' || $gophertype eq '7';
+
+    # collect response
+    my $buf = '';
+    $response = $self->collect($arg, $response, sub {
+	die "read timeout" if $timeout && !$sel->can_read($timeout);
+        my $n = sysread($socket, $buf, $size);
+	die $! unless defined($n);
+	return \$buf;
+      } );
+
+    # Convert menu to HTML and return data to user.
+    if ($gophertype eq '1' || $gophertype eq '7') {
+	my $content = menu2html($response->content);
+	if (defined $user_arg) {
+	    $response = $self->collect_once($user_arg, $response, $content);
+	} else {
+	    $response->content($content);
+	}
+    }
+
+    $response;
+}
+
+
+sub gopher2url
+{
+    my($gophertype, $path, $host, $port) = @_;
+
+    my $url;
+
+    if ($gophertype eq '8' || $gophertype eq 'T') {
+	# telnet session
+	$url = URI::URL->new($gophertype eq '8' ? 'telnet:' : 'tn3270:');
+	$url->user($path) if defined $path;
+    } else {
+	$path = URI::Escape::uri_escape($path);
+	$url = URI::URL->new("gopher:/$gophertype$path");
+    }
+    $url->host($host);
+    $url->port($port);
+    $url;
+}
+
+sub menu2html {
+    my($menu) = @_;
+
+    $menu =~ s/\015//g;  # remove carriage return
+    my $tmp = <<"EOT";
+<HTML>
+<HEAD>
+   <TITLE>Gopher menu</TITLE>
+</HEAD>
+<BODY>
+<H1>Gopher menu</H1>
+EOT
+    for (split("\n", $menu)) {
+	last if /^\./;
+	my($pretty, $path, $host, $port) = split("\t");
+
+	$pretty =~ s/^(.)//;
+	my $type = $1;
+
+	my $url = gopher2url($type, $path, $host, $port)->as_string;
+	$tmp .= qq{<A HREF="$url">$pretty</A><BR>\n};
+    }
+    $tmp .= "</BODY>\n</HTML>\n";
+    $tmp;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/http.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/http.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/http.pm	(revision 21112)
@@ -0,0 +1,252 @@
+#
+# $Id$
+
+package LWP::Protocol::http;
+
+require LWP::Debug;
+require HTTP::Response;
+require HTTP::Status;
+require IO::Socket;
+require IO::Select;
+
+require LWP::Protocol;
+@ISA = qw(LWP::Protocol);
+
+use strict;
+my $CRLF         = "\015\012";     # how lines should be terminated;
+				   # "\r\n" is not correct on all systems, for
+				   # instance MacPerl defines it to "\012\015"
+
+sub _new_socket
+{
+    my($self, $host, $port, $timeout) = @_;
+
+    local($^W) = 0;  # IO::Socket::INET can be noisy
+    my $sock = IO::Socket::INET->new(PeerAddr => $host,
+				     PeerPort => $port,
+				     Proto    => 'tcp',
+				     Timeout  => $timeout,
+				    );
+    unless ($sock) {
+	# IO::Socket::INET leaves additional error messages in $@
+	$@ =~ s/^.*?: //;
+	die "Can't connect to $host:$port ($@)";
+    }
+    $sock;
+}
+
+
+sub _check_sock
+{
+    #my($self, $req, $sock) = @_;
+}
+
+sub _get_sock_info
+{
+    my($self, $res, $sock) = @_;
+    $res->header("Client-Peer" =>
+		 $sock->peerhost . ":" . $sock->peerport);
+}
+
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size, $timeout) = @_;
+    LWP::Debug::trace('()');
+
+    $size ||= 4096;
+
+    # check method
+    my $method = $request->method;
+    unless ($method =~ /^[A-Za-z0-9_!\#\$%&\'*+\-.^\`|~]+$/) {  # HTTP token
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'Library does not allow method ' .
+				  "$method for 'http:' URLs";
+    }
+
+    my $url = $request->url;
+    my($host, $port, $fullpath);
+
+    # Check if we're proxy'ing
+    if (defined $proxy) {
+	# $proxy is an URL to an HTTP server which will proxy this request
+	$host = $proxy->host;
+	$port = $proxy->port;
+	$fullpath = $url->as_string;
+    }
+    else {
+	$host = $url->host;
+	$port = $url->port;
+	$fullpath = $url->full_path;
+    }
+
+    # connect to remote site
+    my $socket = $self->_new_socket($host, $port, $timeout);
+    $self->_check_sock($request, $socket);
+	    
+    my $sel = IO::Select->new($socket) if $timeout;
+
+    my $request_line = "$method $fullpath HTTP/1.0$CRLF";
+
+    my $h = $request->headers->clone;
+    my $cont_ref = $request->content_ref;
+    $cont_ref = $$cont_ref if ref($$cont_ref);
+    my $ctype = ref($cont_ref);
+
+    # If we're sending content we *have* to specify a content length
+    # otherwise the server won't know a messagebody is coming.
+    if ($ctype eq 'CODE') {
+	die 'No Content-Length header for request with dynamic content'
+	    unless defined($h->header('Content-Length')) ||
+		   $h->content_type =~ /^multipart\//;
+	# For HTTP/1.1 we could have used chunked transfer encoding...
+    } else {
+	$h->header('Content-Length' => length $$cont_ref)
+	        if defined($$cont_ref) && length($$cont_ref);
+    }
+    
+    # HTTP/1.1 will require us to send the 'Host' header, so we might
+    # as well start now.
+    my $hhost = $url->netloc;
+    $hhost =~ s/^([^\@]*)\@//;  # get rid of potential "user:pass@"
+    $h->header('Host' => $hhost) unless defined $h->header('Host');
+
+    # add authorization header if we need them.  HTTP URLs do
+    # not really support specification of user and password, but
+    # we allow it.
+    if (defined($1) && not $h->header('Authorization')) {
+	$h->authorization_basic($url->user, $url->password);
+    }
+
+    my $buf = $request_line . $h->as_string($CRLF) . $CRLF;
+    my $n;  # used for return value from syswrite/sysread
+
+    die "write timeout" if $timeout && !$sel->can_write($timeout);
+    $n = $socket->syswrite($buf, length($buf));
+    die $! unless defined($n);
+    die "short write" unless $n == length($buf);
+    LWP::Debug::conns($buf);
+
+    if ($ctype eq 'CODE') {
+	while ( ($buf = &$cont_ref()), defined($buf) && length($buf)) {
+	    die "write timeout" if $timeout && !$sel->can_write($timeout);
+	    $n = $socket->syswrite($buf, length($buf));
+	    die $! unless defined($n);
+	    die "short write" unless $n == length($buf);
+	    LWP::Debug::conns($buf);
+	}
+    } elsif (defined($$cont_ref) && length($$cont_ref)) {
+	die "write timeout" if $timeout && !$sel->can_write($timeout);
+	$n = $socket->syswrite($$cont_ref, length($$cont_ref));
+	die $! unless defined($n);
+	die "short write" unless $n == length($$cont_ref);
+	LWP::Debug::conns($buf);
+    }
+    
+    # read response line from server
+    LWP::Debug::debug('reading response');
+
+    my $response;
+    $buf = '';
+
+    # Inside this loop we will read the response line and all headers
+    # found in the response.
+    while (1) {
+	{
+	    die "read timeout" if $timeout && !$sel->can_read($timeout);
+	    $n = $socket->sysread($buf, $size, length($buf));
+	    die $! unless defined($n);
+	    die "unexpected EOF before status line seen" unless $n;
+	    LWP::Debug::conns($buf);
+	}
+	if ($buf =~ s/^(HTTP\/\d+\.\d+)[ \t]+(\d+)[ \t]*([^\012]*)\012//) {
+	    # HTTP/1.0 response or better
+	    my($ver,$code,$msg) = ($1, $2, $3);
+	    $msg =~ s/\015$//;
+	    LWP::Debug::debug("$ver $code $msg");
+	    $response = HTTP::Response->new($code, $msg);
+	    $response->protocol($ver);
+
+	    # ensure that we have read all headers.  The headers will be
+	    # terminated by two blank lines
+	    until ($buf =~ /^\015?\012/ || $buf =~ /\015?\012\015?\012/) {
+		# must read more if we can...
+		LWP::Debug::debug("need more header data");
+		die "read timeout" if $timeout && !$sel->can_read($timeout);
+		$n = $socket->sysread($buf, $size, length($buf));
+		die $! unless defined($n);
+		die "unexpected EOF before all headers seen" unless $n;
+		#LWP::Debug::conns($buf);
+	    }
+
+	    # now we start parsing the headers.  The strategy is to
+	    # remove one line at a time from the beginning of the header
+	    # buffer ($res).
+	    my($key, $val);
+	    while ($buf =~ s/([^\012]*)\012//) {
+		my $line = $1;
+
+		# if we need to restore as content when illegal headers
+		# are found.
+		my $save = "$line\012"; 
+
+		$line =~ s/\015$//;
+		last unless length $line;
+
+		if ($line =~ /^([a-zA-Z0-9_\-]+)\s*:\s*(.*)/) {
+		    $response->push_header($key, $val) if $key;
+		    ($key, $val) = ($1, $2);
+		} elsif ($line =~ /^\s+(.*)/) {
+		    unless ($key) {
+			$response->header("Client-Warning" =>
+					 => "Illegal continuation header");
+			$buf = "$save$buf";
+			last;
+		    }
+		    $val .= " $1";
+		} else {
+		    $response->header("Client-Warning" =>
+				      "Illegal header '$line'");
+		    $buf = "$save$buf";
+		    last;
+		}
+	    }
+	    $response->push_header($key, $val) if $key;
+	    last;
+
+	} elsif ((length($buf) >= 5 and $buf !~ /^HTTP\//) or
+		 $buf =~ /\012/ ) {
+	    # HTTP/0.9 or worse
+	    LWP::Debug::debug("HTTP/0.9 assume OK");
+	    $response = HTTP::Response->new(&HTTP::Status::RC_OK, "OK");
+	    $response->protocol('HTTP/0.9');
+	    last;
+
+	} else {
+	    # need more data
+	    LWP::Debug::debug("need more status line data");
+	}
+    };
+    $response->request($request);
+    $self->_get_sock_info($response, $socket);
+
+
+    my $usebuf = length($buf) > 0;
+    $response = $self->collect($arg, $response, sub {
+        if ($usebuf) {
+	    $usebuf = 0;
+	    return \$buf;
+	}
+	die "read timeout" if $timeout && !$sel->can_read($timeout);
+	my $n = $socket->sysread($buf, $size);
+	die $! unless defined($n);
+	#LWP::Debug::conns($buf);
+	return \$buf;
+	} );
+
+    $socket->close;
+
+    $response;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/https.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/https.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/https.pm	(revision 21112)
@@ -0,0 +1,59 @@
+#
+# $Id$
+
+use strict;
+
+package LWP::Protocol::https;
+require Net::SSL;  # from Crypt-SSLeay
+
+use vars qw(@ISA);
+
+require LWP::Protocol::http;
+@ISA=qw(LWP::Protocol::http);
+
+sub _new_socket
+{
+    my($self, $host, $port, $timeout) = @_;
+    local($^W) = 0;  # IO::Socket::INET can be noisy
+    my $sock = Net::SSL->new(PeerAddr => $host,
+			     PeerPort => $port,
+			     Proto    => 'tcp',
+			     Timeout  => $timeout,
+			    );
+    unless ($sock) {
+	# IO::Socket::INET leaves additional error messages in $@
+	$@ =~ s/^.*?: //;
+	die "Can't connect to $host:$port ($@)";
+    }
+    $sock;
+}
+
+sub _check_sock
+{
+    my($self, $req, $sock) = @_;
+    my $check = $req->header("If-SSL-Cert-Subject");
+    if (defined $check) {
+	my $cert = $sock->get_peer_certificate ||
+	    die "Missing SSL certificate";
+	my $subject = $cert->subject_name;
+	die "Bad SSL certificate subject: '$subject' !~ /$check/"
+	    unless $subject =~ /$check/;
+	$req->remove_header("If-SSL-Cert-Subject");  # don't pass it on
+    }
+}
+
+sub _get_sock_info
+{
+    my $self = shift;
+    $self->SUPER::_get_sock_info(@_);
+    my($res, $sock) = @_;
+    $res->header("Client-SSL-Cipher" => $sock->get_cipher);
+    my $cert = $sock->get_peer_certificate;
+    if ($cert) {
+	$res->header("Client-SSL-Cert-Subject" => $cert->subject_name);
+	$res->header("Client-SSL-Cert-Issuer" => $cert->issuer_name);
+    }
+    $res->header("Client-SSL-Warning" => "Peer certificate not verified");
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/mailto.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/mailto.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/mailto.pm	(revision 21112)
@@ -0,0 +1,87 @@
+#
+# $Id$
+#
+# This module implements the mailto protocol.  It is just a simple
+# frontend to the Unix sendmail program.  In the long run this module
+# will built using the Mail::Send module.
+
+package LWP::Protocol::mailto;
+
+require LWP::Protocol;
+require HTTP::Request;
+require HTTP::Response;
+require HTTP::Status;
+
+use Carp;
+
+@ISA = qw(LWP::Protocol);
+
+$SENDMAIL = "/usr/lib/sendmail";
+
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size) = @_;
+
+    # check proxy
+    if (defined $proxy)
+    {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'You can not proxy with mail';
+    }
+
+    # check method
+    $method = $request->method;
+
+    if ($method ne 'POST') {
+	return new HTTP::Response &HTTP::Status::RC_BAD_REQUEST,
+				  'Library does not allow method ' .
+				  "$method for 'mailto:' URLs";
+    }
+
+    # check url
+    my $url = $request->url;
+
+    my $scheme = $url->scheme;
+    if ($scheme ne 'mailto') {
+	return new HTTP::Response &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				  "LWP::file::request called for '$scheme'";
+    }
+    unless (-x $SENDMAIL) {
+	return new HTTP::Response &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				  "You don't have $SENDMAIL";
+    }
+
+    open(SENDMAIL, "| $SENDMAIL -oi -t") or
+	return new HTTP::Response &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				  "Can't run $SENDMAIL: $!";
+
+    my $addr = $url->encoded822addr;
+
+    $request->header('To', $addr);
+    print SENDMAIL $request->headers_as_string;
+    print SENDMAIL "\n";
+    my $content = $request->content;
+    if (defined $content) {
+	my $contRef = ref($content) ? $content : \$content;
+	if (ref($contRef) eq 'SCALAR') {
+	    print SENDMAIL $$contRef;
+	} elsif (ref($contRef) eq 'CODE') {
+	    # Callback provides data
+	    my $d;
+	    while (length($d = &$contRef)) {
+		print SENDMAIL $d;
+	    }
+	}
+    }
+    close(SENDMAIL);
+
+    my $response = new HTTP::Response &HTTP::Status::RC_ACCEPTED,
+				     'Mail accepted by sendmail';
+    $response->header('Content-Type', 'text/plain');
+    $response->content("Mail sent to <$addr>\n");
+
+    return $response;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/nntp.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/nntp.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Protocol/nntp.pm	(revision 21112)
@@ -0,0 +1,142 @@
+#
+# $Id$
+
+# Implementation of the Network News Transfer Protocol (RFC 977)
+#
+
+package LWP::Protocol::nntp;
+
+require LWP::Protocol;
+@ISA = qw(LWP::Protocol);
+
+require LWP::Debug;
+require HTTP::Response;
+require HTTP::Status;
+require Net::NNTP;
+
+use strict;
+
+
+sub request
+{
+    my($self, $request, $proxy, $arg, $size, $timeout) = @_;
+
+    LWP::Debug::trace('()');
+
+    $size = 4096 unless $size;
+
+    # Check for proxy
+    if (defined $proxy) {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'You can not proxy through NNTP');
+    }
+
+    # Check that the scheme is as expected
+    my $url = $request->url;
+    my $scheme = $url->scheme;
+    unless ($scheme eq 'news') {
+	return HTTP::Response->new(&HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				   "LWP::Protocol::nntp::request called for '$scheme'");
+    }
+
+    # check for a valid method
+    my $method = $request->method;
+    unless ($method eq 'GET' || $method eq 'HEAD' || $method eq 'POST') {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   'Library does not allow method ' .
+				   "$method for 'news:' URLs");
+    }
+
+    # extract the identifier and check against posting to an article
+    my $groupart = $url->groupart;
+    my $is_art = $groupart =~ /@/;
+
+    if ($is_art && $method eq 'POST') {
+	return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST,
+				   "Can't post to an article <$groupart>");
+    }
+
+    my $nntp = Net::NNTP->new(undef,
+			      #Port    => 18574,
+			      Timeout => $timeout,
+			      #Debug   => 1,
+			     );
+    die "Can't connect to nntp server" unless $nntp;
+
+    # Check the initial welcome message from the NNTP server
+    if ($nntp->status != 2) {
+	return HTTP::Response->new(&HTTP::Status::RC_SERVICE_UNAVAILABLE,
+				   $nntp->message);
+    }
+    my $response = HTTP::Response->new(&HTTP::Status::RC_OK, "OK");
+
+    my $mess = $nntp->message;
+    LWP::Debug::debug($mess);
+
+    # Try to extract server name from greating message.
+    # Don't know if this works well for a large class of servers, but
+    # this works for our server.
+    $mess =~ s/\s+ready\b.*//;
+    $mess =~ s/^\S+\s+//;
+    $response->header(Server => $mess);
+
+
+    # First we handle posting of articles
+    if ($method eq 'POST') {
+	return HTTP::Response->new(&HTTP::Status::RC_NOT_IMPLEMENTED,
+				   "POST not implemented yet");
+    }
+
+    # The method must be "GET" or "HEAD" by now
+    if (!$is_art) {
+	if (!$nntp->group($groupart)) {
+	    return HTTP::Response->new(&HTTP::Status::RC_NOT_FOUND,
+				       $nntp->message);
+	}
+	return HTTP::Response->new(&HTTP::Status::RC_NOT_IMPLEMENTED,
+				   "GET newsgroup not implemented yet");
+    }
+
+    # Send command to server to retrieve an article (or just the headers)
+    my $get = $method eq 'HEAD' ? "head" : "article";
+    my $art = $nntp->$get("<$groupart>");
+    unless ($art) {
+	return HTTP::Response->new(&HTTP::Status::RC_NOT_FOUND,
+				   $nntp->message);
+    }
+    LWP::Debug::debug($nntp->message);
+    
+    # Parse headers
+    my($key, $val);
+    while ($_ = shift @$art) {
+	if (/^\s+$/) {
+	    last;  # end of headers
+	} elsif (/^(\S+):\s*(.*)/) {
+	    $response->push_header($key, $val) if $key;
+	    ($key, $val) = ($1, $2);
+	} elsif (/^\s+(.*)/) {
+	    next unless $key;
+	    $val .= $1;
+	} else {
+	    unshift(@$art, $_);
+	    last;
+	}
+    }
+    $response->push_header($key, $val) if $key;
+
+    # Ensure that there is a Content-Type header
+    $response->header("Content-Type", "text/plain")
+	unless $response->header("Content-Type");
+
+    # Collect the body
+    $response = $self->collect_once($arg, $response, join("", @$art))
+      if @$art;
+
+    # Say godbye to the server
+    $nntp->quit;
+    $nntp = undef;
+
+    $response;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/RobotUA.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/RobotUA.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/RobotUA.pm	(revision 21112)
@@ -0,0 +1,285 @@
+# $Id$
+
+package LWP::RobotUA;
+
+require LWP::UserAgent;
+@ISA = qw(LWP::UserAgent);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+require WWW::RobotRules;
+require HTTP::Request;
+require HTTP::Response;
+
+use Carp ();
+use LWP::Debug ();
+use HTTP::Status ();
+use HTTP::Date qw(time2str);
+use strict;
+
+=head1 NAME
+
+LWP::RobotUA - A class for Web Robots
+
+=head1 SYNOPSIS
+
+  require LWP::RobotUA;
+  $ua = new LWP::RobotUA 'my-robot/0.1', 'me@foo.com';
+  $ua->delay(10);  # be very nice, go slowly
+  ...
+  # just use it just like a normal LWP::UserAgent
+  $res = $ua->request($req);
+
+=head1 DESCRIPTION
+
+This class implements a user agent that is suitable for robot
+applications.  Robots should be nice to the servers they visit.  They
+should consult the F<robots.txt> file to ensure that they are welcomed
+and they should not send too frequent requests.
+
+But, before you consider writing a robot take a look at
+<URL:http://info.webcrawler.com/mak/projects/robots/robots.html>.
+
+When you use a I<LWP::RobotUA> as your user agent, then you do not
+really have to think about these things yourself.  Just send requests
+as you do when you are using a normal I<LWP::UserAgent> and this
+special agent will make sure you are nice.
+
+=head1 METHODS
+
+The LWP::RobotUA is a sub-class of LWP::UserAgent and implements the
+same methods. In addition the following methods are provided:
+
+=over 4
+
+=cut
+
+
+#
+# Additional attributes in addition to those found in LWP::UserAgent:
+#
+# $self->{'delay'}    Required delay between request to the same
+#                     server in minutes.
+#
+# $self->{'rules'}     A WWW::RobotRules object
+#
+
+
+=item $ua = LWP::RobotUA->new($agent_name, $from, [$rules])
+
+Your robot's name and the mail address of the human responsible for
+the robot (i.e. you) is required by the constructor.
+
+Optionally it allows you to specify the I<WWW::RobotRules> object to
+use.
+
+=cut
+
+sub new
+{
+    my($class,$name,$from,$rules) = @_;
+
+    Carp::croak('LWP::RobotUA name required') unless $name;
+    Carp::croak('LWP::RobotUA from address required') unless $from;
+
+    my $self = new LWP::UserAgent;
+    $self = bless $self, $class;
+
+    $self->{'delay'} = 1;   # minutes
+    $self->{'agent'} = $name;
+    $self->{'from'}  = $from;
+    $self->{'use_sleep'} = 1;
+
+    if ($rules) {
+	$rules->agent($name);
+	$self->{'rules'} = $rules;
+    } else {
+	$self->{'rules'} = new WWW::RobotRules $name;
+    }
+
+    $self;
+}
+
+
+=item $ua->delay([$minutes])
+
+Set the minimum delay between requests to the same server.  The
+default is 1 minute.
+
+=item $ua->use_sleep([$boolean])
+
+Get/set a value indicating wether the UA should sleep() if request
+arrive to fast (before $ua->delay minutes has passed).  The default is
+TRUE.  If this value is FALSE then an internal SERVICE_UNAVAILABLE
+response will be generated.  It will have an Retry-After header that
+indicate when it is OK to send another request to this server.
+
+=cut
+
+sub delay     { shift->_elem('delay',     @_); }
+sub use_sleep { shift->_elem('use_sleep', @_); }
+
+sub agent
+{
+    my $self = shift;
+    my $old = $self->SUPER::agent(@_);
+    if (@_) {
+	# Changing our name means to start fresh
+	$self->{'rules'}->agent($self->{'agent'}); 
+    }
+    $old;
+}
+
+
+=item $ua->rules([$rules])
+
+Set/get which I<WWW::RobotRules> object to use. 
+
+=cut
+
+sub rules {
+    my $self = shift;
+    my $old = $self->_elem('rules', @_);
+    $self->{'rules'}->agent($self->{'agent'}) if @_;
+    $old;
+}
+
+
+=item $ua->no_visits($netloc)
+
+Returns the number of documents fetched from this server host. Yes I
+know, this method should probably have been named num_visits() or
+something like that :-(
+
+=cut
+
+sub no_visits
+{
+    my($self, $netloc) = @_;
+    $self->{'rules'}->no_visits($netloc);
+}
+
+*host_count = \&no_visits;  # backwards compatibility with LWP-5.02
+
+
+=item $ua->host_wait($netloc)
+
+Returns the number of seconds (from now) you must wait before you can
+make a new request to this host.
+
+=cut
+
+sub host_wait
+{
+    my($self, $netloc) = @_;
+    return undef unless defined $netloc;
+    my $last = $self->{'rules'}->last_visit($netloc);
+    if ($last) {
+	my $wait = int($self->{'delay'} * 60 - (time - $last));
+	$wait = 0 if $wait < 0;
+	return $wait;
+    }
+    return 0;
+}
+
+
+sub simple_request
+{
+    my($self, $request, $arg, $size) = @_;
+
+    LWP::Debug::trace('()');
+
+    # Do we try to access a new server?
+    my $allowed = $self->{'rules'}->allowed($request->url);
+
+    if ($allowed < 0) {
+	LWP::Debug::debug("Host is not visited before, or robots.txt expired.");
+	# fetch "robots.txt"
+	my $robot_url = $request->url->clone;
+	$robot_url->path("robots.txt");
+	$robot_url->params(undef);
+	$robot_url->query(undef);
+	LWP::Debug::debug("Requesting $robot_url");
+
+	# make access to robot.txt legal since this will be a recursive call
+	$self->{'rules'}->parse($robot_url, ""); 
+
+	my $robot_req = new HTTP::Request 'GET', $robot_url;
+	my $robot_res = $self->request($robot_req);
+	my $fresh_until = $robot_res->fresh_until;
+	if ($robot_res->is_success) {
+	    LWP::Debug::debug("Parsing robot rules");
+	    $self->{'rules'}->parse($robot_url, $robot_res->content, 
+				    $fresh_until);
+	} else {
+	    LWP::Debug::debug("No robots.txt file found");
+	    $self->{'rules'}->parse($robot_url, "", $fresh_until);
+	}
+
+	# recalculate allowed...
+	$allowed = $self->{'rules'}->allowed($request->url);
+    }
+
+    # Check rules
+    unless ($allowed) {
+	return new HTTP::Response
+	  &HTTP::Status::RC_FORBIDDEN, 'Forbidden by robots.txt';
+    }
+
+    my $netloc = $request->url->netloc;
+    my $wait = $self->host_wait($netloc);
+
+    if ($wait) {
+	LWP::Debug::debug("Must wait $wait seconds");
+	if ($self->{'use_sleep'}) {
+	    sleep($wait)
+	} else {
+	    my $res = new HTTP::Response
+	      &HTTP::Status::RC_SERVICE_UNAVAILABLE, 'Please, slow down';
+	    $res->header('Retry-After', time2str(time + $wait));
+	    return $res;
+	}
+    }
+
+    # Perform the request
+    my $res = $self->SUPER::simple_request($request, $arg, $size);
+
+    $self->{'rules'}->visit($netloc);
+
+    $res;
+}
+
+
+=item $ua->as_string
+
+Returns a text that describe the state of the UA.
+Mainly useful for debugging.
+
+=cut
+
+sub as_string
+{
+    my $self = shift;
+    my @s;
+    push(@s, "Robot: $self->{'agent'} operated by $self->{'from'}  [$self]");
+    push(@s, "    Minimum delay: " . int($self->{'delay'}*60) . "s");
+    push(@s, "    Will sleep if too early") if $self->{'use_sleep'};
+    push(@s, "    Rules = $self->{'rules'}");
+    join("\n", @s, '');
+}
+
+1;
+
+=back
+
+=head1 SEE ALSO
+
+L<LWP::UserAgent>, L<WWW::RobotRules>
+
+=head1 COPYRIGHT
+
+Copyright 1996-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Simple.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Simple.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/Simple.pm	(revision 21112)
@@ -0,0 +1,329 @@
+#
+# $Id$
+
+=head1 NAME
+
+get, head, getprint, getstore, mirror - Procedural LWP interface
+
+=head1 SYNOPSIS
+
+ perl -MLWP::Simple -e 'getprint "http://www.sn.no"'
+
+ use LWP::Simple;
+ $content = get("http://www.sn.no/")
+ if (mirror("http://www.sn.no/", "foo") == RC_NOT_MODIFIED) {
+     ...
+ }
+
+ if (is_success(getprint("http://www.sn.no/"))) {
+     ...
+ }
+
+=head1 DESCRIPTION
+
+This interface is intended for those who want a simplified view of the
+libwww-perl library.  It should also be suitable for one-liners.  If
+you need more control or access to the header fields in the requests
+sent and responses received you should use the full object oriented
+interface provided by the C<LWP::UserAgent> module.
+
+The following functions are provided (and exported) by this module:
+
+=over 3
+
+=item get($url)
+
+The get() function will fetch the document identified by the given URL
+and return it.  It returns C<undef> if it fails.  The $url argument can
+be either a simple string or a reference to a URI::URL object.
+
+You will not be able to examine the response code or response headers
+(like 'Content-Type') when you are accessing the web using this
+function.  If you need this information you should use the full OO
+interface (see L<LWP::UserAgent>).
+
+=item head($url)
+
+Get document headers. Returns the following 5 values if successful:
+($content_type, $document_length, $modified_time, $expires, $server)
+
+Returns an empty list if it fails.  In scalar context returns TRUE if
+successful.
+
+=item getprint($url)
+
+Get and print a document identified by a URL. The document is printed
+on STDOUT as data is received from the network.  If the request fails,
+then the status code and message is printed on STDERR.  The return
+value is the HTTP response code.
+
+=item getstore($url, $file)
+
+Gets a document identified by a URL and stores it in the file. The
+return value is the HTTP response code.
+
+=item mirror($url, $file)
+
+Get and store a document identified by a URL, using
+I<If-modified-since>, and checking of the I<Content-Length>.  Returns
+the HTTP response code.
+
+=back
+
+This module also exports the HTTP::Status constants and procedures.
+These can be used when you check the response code from getprint(),
+getstore() and mirror().  The constants are:
+
+   RC_CONTINUE
+   RC_SWITCHING_PROTOCOLS
+   RC_OK
+   RC_CREATED
+   RC_ACCEPTED
+   RC_NON_AUTHORITATIVE_INFORMATION
+   RC_NO_CONTENT
+   RC_RESET_CONTENT
+   RC_PARTIAL_CONTENT
+   RC_MULTIPLE_CHOICES
+   RC_MOVED_PERMANENTLY
+   RC_MOVED_TEMPORARILY
+   RC_SEE_OTHER
+   RC_NOT_MODIFIED
+   RC_USE_PROXY
+   RC_BAD_REQUEST
+   RC_UNAUTHORIZED
+   RC_PAYMENT_REQUIRED
+   RC_FORBIDDEN
+   RC_NOT_FOUND
+   RC_METHOD_NOT_ALLOWED
+   RC_NOT_ACCEPTABLE
+   RC_PROXY_AUTHENTICATION_REQUIRED
+   RC_REQUEST_TIMEOUT
+   RC_CONFLICT
+   RC_GONE
+   RC_LENGTH_REQUIRED
+   RC_PRECONDITION_FAILED
+   RC_REQUEST_ENTITY_TOO_LARGE
+   RC_REQUEST_URI_TOO_LARGE
+   RC_UNSUPPORTED_MEDIA_TYPE
+   RC_INTERNAL_SERVER_ERROR
+   RC_NOT_IMPLEMENTED
+   RC_BAD_GATEWAY
+   RC_SERVICE_UNAVAILABLE
+   RC_GATEWAY_TIMEOUT
+   RC_HTTP_VERSION_NOT_SUPPORTED
+
+The HTTP::Status classification functions are:
+
+=over 3
+
+=item is_success($rc)
+
+Check if response code indicated successfull request.
+
+=item is_error($rc)
+
+Check if response code indicated that an error occured.
+
+=back
+
+The module will also export the LWP::UserAgent object as C<$ua> if you
+ask for it explicitly.
+
+The user agent created by this module will identify itself as
+"LWP::Simple/#.##" (where "#.##" is the libwww-perl version number)
+and will initialize its proxy defaults from the environment (by
+calling $ua->env_proxy).
+
+=head1 SEE ALSO
+
+L<LWP>, L<LWP::UserAgent>, L<HTTP::Status>, L<lwp-request>,
+L<lwp-mirror>
+
+=cut
+
+
+package LWP::Simple;
+
+use strict;
+use vars qw($ua %loop_check $FULL_LWP @EXPORT @EXPORT_OK $VERSION);
+
+require Exporter;
+
+@EXPORT = qw(get head getprint getstore mirror);
+@EXPORT_OK = qw($ua);
+
+# I really hate this.  I was a bad idea to do it in the first place.
+# Wonder how to get rid of it???  (It even makes LWP::Simple 7% slower
+# for trivial tests)
+use HTTP::Status;
+push(@EXPORT, @HTTP::Status::EXPORT);
+
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+$FULL_LWP++ if grep {lc($_) eq "http_proxy"} keys %ENV;
+
+
+sub import
+{
+    my $pkg = shift;
+    my $callpkg = caller;
+    if (grep $_ eq '$ua', @_) {
+	$FULL_LWP++;
+	_init_ua();
+    }
+    Exporter::export($pkg, $callpkg, @_);
+}
+
+
+sub _init_ua
+{
+    require LWP;
+    require LWP::UserAgent;
+    require HTTP::Status;
+    require HTTP::Date;
+    $ua = new LWP::UserAgent;  # we create a global UserAgent object
+    my $ver = $LWP::VERSION = $LWP::VERSION;  # avoid warning
+    $ua->agent("LWP::Simple/$LWP::VERSION");
+    $ua->env_proxy;
+}
+
+
+sub get ($)
+{
+    %loop_check = ();
+    goto \&_get;
+}
+
+
+sub get_old ($)
+{
+    my($url) = @_;
+    _init_ua() unless $ua;
+
+    my $request = HTTP::Request->new(GET => $url);
+    my $response = $ua->request($request);
+
+    return $response->content if $response->is_success;
+    return undef;
+}
+
+
+sub head ($)
+{
+    my($url) = @_;
+    _init_ua() unless $ua;
+
+    my $request = HTTP::Request->new(HEAD => $url);
+    my $response = $ua->request($request);
+
+    if ($response->is_success) {
+	return $response unless wantarray;
+	return ($response->header('Content-Type'),
+		$response->header('Content-Length'),
+		HTTP::Date::str2time($response->header('Last-Modified')),
+		HTTP::Date::str2time($response->header('Expires')),
+		$response->header('Server'),
+	       );
+    }
+    return;
+}
+
+
+sub getprint ($)
+{
+    my($url) = @_;
+    _init_ua() unless $ua;
+
+    my $request = HTTP::Request->new(GET => $url);
+    local($\) = ""; # ensure standard $OUTPUT_RECORD_SEPARATOR
+    my $response = $ua->request($request, sub { print $_[0] });
+    unless ($response->is_success) {
+	print STDERR $response->status_line, " <URL:$url>\n";
+    }
+    $response->code;
+}
+
+
+sub getstore ($$)
+{
+    my($url, $file) = @_;
+    _init_ua() unless $ua;
+
+    my $request = HTTP::Request->new(GET => $url);
+    my $response = $ua->request($request, $file);
+
+    $response->code;
+}
+
+
+sub mirror ($$)
+{
+    my($url, $file) = @_;
+    _init_ua() unless $ua;
+    my $response = $ua->mirror($url, $file);
+    $response->code;
+}
+
+
+sub _get
+{
+    my $url = shift;
+    my $ret;
+    if (!$FULL_LWP && $url =~ m,^http://([^/:]+)(?::(\d+))?(/\S*)?$,) {
+	my $host = $1;
+	my $port = $2 || 80;
+	my $path = $3;
+	$path = "/" unless defined($path);
+	return _trivial_http_get($host, $port, $path);
+    } else {
+        _init_ua() unless $ua;
+	my $request = HTTP::Request->new(GET => $url);
+	my $response = $ua->request($request);
+	return $response->is_success ? $response->content : undef;
+    }
+}
+
+
+sub _trivial_http_get
+{
+   my($host, $port, $path) = @_;
+   #print "HOST=$host, PORT=$port, PATH=$path\n";
+
+   require IO::Socket;
+   local($^W) = 0;
+   my $sock = IO::Socket::INET->new(PeerAddr => $host,
+                                    PeerPort => $port,
+                                    Proto    => 'tcp',
+                                    Timeout  => 60) || return;
+   $sock->autoflush;
+   my $netloc = $host;
+   $netloc .= ":$port" if $port != 80;
+   print $sock join("\015\012" =>
+                    "GET $path HTTP/1.0",
+                    "Host: $netloc",
+                    "User-Agent: lwp-trivial/$VERSION",
+                    "", "");
+
+   my $buf = "";
+   my $n;
+   1 while $n = sysread($sock, $buf, 8*1024, length($buf));
+   return undef unless defined($n);
+
+   if ($buf =~ m,^HTTP/\d+\.\d+\s+(\d+)[^\012]*\012,) {
+       my $code = $1;
+       #print "CODE=$code\n$buf\n";
+       if ($code =~ /^30[1237]/ && $buf =~ /\012Location:\s*(\S+)/) {
+           # redirect
+           my $url = $1;
+           return undef if $loop_check{$url}++;
+           return _get($url);
+       }
+       return undef unless $code =~ /^2/;
+       $buf =~ s/.+?\015?\012\015?\012//s;  # zap header
+   }
+
+   return $buf;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/UserAgent.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/UserAgent.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/UserAgent.pm	(revision 21112)
@@ -0,0 +1,732 @@
+# $Id$
+
+package LWP::UserAgent;
+use strict;
+
+=head1 NAME
+
+LWP::UserAgent - A WWW UserAgent class
+
+=head1 SYNOPSIS
+
+ require LWP::UserAgent;
+ $ua = new LWP::UserAgent;
+
+ $request = new HTTP::Request('GET', 'file://localhost/etc/motd');
+
+ $response = $ua->request($request); # or
+ $response = $ua->request($request, '/tmp/sss'); # or
+ $response = $ua->request($request, \&callback, 4096);
+
+ sub callback { my($data, $response, $protocol) = @_; .... }
+
+=head1 DESCRIPTION
+
+The C<LWP::UserAgent> is a class implementing a simple World-Wide Web
+user agent in Perl. It brings together the HTTP::Request,
+HTTP::Response and the LWP::Protocol classes that form the rest of the
+core of libwww-perl library. For simple uses this class can be used
+directly to dispatch WWW requests, alternatively it can be subclassed
+for application-specific behaviour.
+
+In normal usage the application creates a UserAgent object, and then
+configures it with values for timeouts proxies, name, etc. The next
+step is to create an instance of C<HTTP::Request> for the request that
+needs to be performed. This request is then passed to the UserAgent
+request() method, which dispatches it using the relevant protocol,
+and returns a C<HTTP::Response> object.
+
+The basic approach of the library is to use HTTP style communication
+for all protocol schemes, i.e. you will receive an C<HTTP::Response>
+object also for gopher or ftp requests.  In order to achieve even more
+similarities with HTTP style communications, gopher menus and file
+directories will be converted to HTML documents.
+
+The request() method can process the content of the response in one of
+three ways: in core, into a file, or into repeated calls of a
+subroutine.  You choose which one by the kind of value passed as the
+second argument to request().
+
+The in core variant simply returns the content in a scalar attribute
+called content() of the response object, and is suitable for small
+HTML replies that might need further parsing.  This variant is used if
+the second argument is missing (or is undef).
+
+The filename variant requires a scalar containing a filename as the
+second argument to request(), and is suitable for large WWW objects
+which need to be written directly to the file, without requiring large
+amounts of memory. In this case the response object returned from
+request() will have empty content().  If the request fails, then the
+content() might not be empty, and the file will be untouched.
+
+The subroutine variant requires a reference to callback routine as the
+second argument to request() and it can also take an optional chuck
+size as third argument.  This variant can be used to construct
+"pipe-lined" processing, where processing of received chuncks can
+begin before the complete data has arrived.  The callback function is
+called with 3 arguments: the data received this time, a reference to
+the response object and a reference to the protocol object.  The
+response object returned from request() will have empty content().  If
+the request fails, then the the callback routine will not have been
+called, and the response->content() might not be empty.
+
+The request can be aborted by calling die() within the callback
+routine.  The die message will be available as the "X-Died" special
+response header field.
+
+The library also accepts that you put a subroutine reference as
+content in the request object.  This subroutine should return the
+content (possibly in pieces) when called.  It should return an empty
+string when there is no more content.
+
+=head1 METHODS
+
+The following methods are available:
+
+=over 4
+
+=cut
+
+
+use vars qw(@ISA $VERSION);
+
+require LWP::MemberMixin;
+@ISA = qw(LWP::MemberMixin);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+
+require URI::URL;
+require HTTP::Request;
+require HTTP::Response;
+
+use HTTP::Date ();
+
+use LWP ();
+use LWP::Debug ();
+use LWP::Protocol ();
+
+use Carp ();
+
+use AutoLoader ();
+*AUTOLOAD = \&AutoLoader::AUTOLOAD;  # import the AUTOLOAD method
+
+
+=item $ua = new LWP::UserAgent;
+
+Constructor for the UserAgent.  Returns a reference to a
+LWP::UserAgent object.
+
+=cut
+
+sub new
+{
+    my($class, $init) = @_;
+    LWP::Debug::trace('()');
+
+    my $self;
+    if (ref $init) {
+	$self = $init->clone;
+    } else {
+	$self = bless {
+		'agent'       => "libwww-perl/$LWP::VERSION",
+		'from'        => undef,
+		'timeout'     => 3*60,
+		'proxy'       => undef,
+		'cookie_jar'  => undef,
+		'use_eval'    => 1,
+                'parse_head'  => 1,
+                'max_size'    => undef,
+		'no_proxy'    => [],
+	}, $class;
+    }
+}
+
+
+=item $ua->simple_request($request, [$arg [, $size]])
+
+This method dispatches a single WWW request on behalf of a user, and
+returns the response received.  The C<$request> should be a reference
+to a C<HTTP::Request> object with values defined for at least the
+method() and url() attributes.
+
+If C<$arg> is a scalar it is taken as a filename where the content of
+the response is stored.
+
+If C<$arg> is a reference to a subroutine, then this routine is called
+as chunks of the content is received.  An optional C<$size> argument
+is taken as a hint for an appropriate chunk size.
+
+If C<$arg> is omitted, then the content is stored in the response
+object itself.
+
+=cut
+
+sub simple_request
+{
+    my($self, $request, $arg, $size) = @_;
+    local($SIG{__DIE__});  # protect agains user defined die handlers
+
+    my($method, $url) = ($request->method, $request->url);
+
+    # Check that we have a METHOD and a URL first
+    return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST, "Method missing")
+	unless $method;
+    return HTTP::Response->new(&HTTP::Status::RC_BAD_REQUEST, "URL missing")
+	unless $url;
+
+    LWP::Debug::trace("$method $url");
+
+    # Locate protocol to use
+    my $scheme = '';
+    my $proxy = $self->_need_proxy($url);
+    if (defined $proxy) {
+	$scheme = $proxy->scheme;
+    } else {
+	$scheme = $url->scheme;
+    }
+    my $protocol;
+    eval {
+	$protocol = LWP::Protocol::create($scheme);
+    };
+    if ($@) {
+	$@ =~ s/\s+at\s+\S+\s+line\s+\d+\.?\s*//;  # remove file/line number
+	return HTTP::Response->new(&HTTP::Status::RC_NOT_IMPLEMENTED, $@)
+    }
+
+    # Extract fields that will be used below
+    my ($agent, $from, $timeout, $cookie_jar,
+        $use_eval, $parse_head, $max_size) =
+      @{$self}{qw(agent from timeout cookie_jar
+                  use_eval parse_head max_size)};
+
+    # Set User-Agent and From headers if they are defined
+    $request->header('User-Agent' => $agent) if $agent;
+    $request->header('From' => $from) if $from;
+    $cookie_jar->add_cookie_header($request) if $cookie_jar;
+
+    # Transfer some attributes to the protocol object
+    $protocol->parse_head($parse_head);
+    $protocol->max_size($max_size);
+    
+    my $response;
+    if ($use_eval) {
+	# we eval, and turn dies into responses below
+	eval {
+	    $response = $protocol->request($request, $proxy,
+					   $arg, $size, $timeout);
+	};
+	if ($@) {
+	    $@ =~ s/\s+at\s+\S+\s+line\s+\d+\.?\s*//;
+	    $response =
+	      HTTP::Response->new(&HTTP::Status::RC_INTERNAL_SERVER_ERROR,
+				  $@);
+	}
+    } else {
+	$response = $protocol->request($request, $proxy,
+				       $arg, $size, $timeout);
+	# XXX: Should we die unless $response->is_success ???
+    }
+
+    $response->request($request);  # record request for reference
+    $cookie_jar->extract_cookies($response) if $cookie_jar;
+    $response->header("Client-Date" => HTTP::Date::time2str(time));
+    return $response;
+}
+
+
+=item $ua->request($request, $arg [, $size])
+
+Process a request, including redirects and security.  This method may
+actually send several different simple reqeusts.
+
+The arguments are the same as for C<simple_request()>.
+
+=cut
+
+sub request
+{
+    my($self, $request, $arg, $size, $previous) = @_;
+
+    LWP::Debug::trace('()');
+
+    my $response = $self->simple_request($request, $arg, $size);
+
+    my $code = $response->code;
+    $response->previous($previous) if defined $previous;
+
+    LWP::Debug::debug('Simple result: ' . HTTP::Status::status_message($code));
+
+    if ($code == &HTTP::Status::RC_MOVED_PERMANENTLY or
+	$code == &HTTP::Status::RC_MOVED_TEMPORARILY) {
+
+	# Make a copy of the request and initialize it with the new URI
+	my $referral = $request->clone;
+
+	# And then we update the URL based on the Location:-header.
+	# Some servers erroneously return a relative URL for redirects,
+	# so make it absolute if it not already is.
+	my $referral_uri = (URI::URL->new($response->header('Location'),
+					  $response->base))->abs(undef,1);
+
+	$referral->url($referral_uri);
+
+	return $response unless $self->redirect_ok($referral);
+
+	# Check for loop in the redirects
+	my $count = 0;
+	my $r = $response;
+	while ($r) {
+	    if (++$count > 13 ||
+                $r->request->url->as_string eq $referral_uri->as_string) {
+		$response->header("Client-Warning" =>
+				  "Redirect loop detected");
+		return $response;
+	    }
+	    $r = $r->previous;
+	}
+
+	return $self->request($referral, $arg, $size, $response);
+
+    } elsif ($code == &HTTP::Status::RC_UNAUTHORIZED ||
+	     $code == &HTTP::Status::RC_PROXY_AUTHENTICATION_REQUIRED
+	    )
+    {
+	my $proxy = ($code == &HTTP::Status::RC_PROXY_AUTHENTICATION_REQUIRED);
+	my $ch_header = $proxy ?  "Proxy-Authenticate" : "WWW-Authenticate";
+	my $challenge = $response->header($ch_header);
+	unless (defined $challenge) {
+	    $response->header("Client-Warning" => 
+			      "Missing Authenticate header");
+	    return $response;
+	}
+
+	require HTTP::Headers::Util;
+	$challenge =~ tr/,/;/;  # "," is used to separate auth-params!!
+	($challenge) = HTTP::Headers::Util::split_header_words($challenge);
+	my $scheme = lc(shift(@$challenge));
+	shift(@$challenge); # no value
+	$challenge = { @$challenge };  # make rest into a hash
+	for (keys %$challenge) {       # make sure all keys are lower case
+	    $challenge->{lc $_} = delete $challenge->{$_};
+	}
+
+	unless ($scheme =~ /^([a-z]+(?:-[a-z]+)*)$/) {
+	    $response->header("Client-Warning" => 
+			      "Bad authentication scheme '$scheme'");
+	    return $response;
+	}
+	$scheme = $1;  # untainted now
+	my $class = "LWP::Authen::\u$scheme";
+	$class =~ s/-/_/g;
+	
+	no strict 'refs';
+	unless (defined %{"$class\::"}) {
+	    # try to load it
+	    eval "require $class";
+	    if ($@) {
+		if ($@ =~ /^Can\'t locate/) {
+		    $response->header("Client-Warning" =>
+				      "Unsupport authentication scheme '$scheme'");
+		} else {
+		    $response->header("Client-Warning" => $@);
+		}
+		return $response;
+	    }
+	}
+	return $class->authenticate($self, $proxy, $challenge, $response,
+				    $request, $arg, $size);
+    }
+    return $response;
+}
+
+
+=item $ua->redirect_ok
+
+This method is called by request() before it tries to do any
+redirects.  It should return a true value if the redirect is allowed
+to be performed. Subclasses might want to override this.
+
+The default implementation will return FALSE for POST request and TRUE
+for all others.
+
+=cut
+
+sub redirect_ok
+{
+    # draft-ietf-http-v10-spec-02.ps from www.ics.uci.edu, specify:
+    #
+    # If the 30[12] status code is received in response to a request using
+    # the POST method, the user agent must not automatically redirect the
+    # request unless it can be confirmed by the user, since this might change
+    # the conditions under which the request was issued.
+
+    my($self, $request) = @_;
+    return 0 if $request->method eq "POST";
+    1;
+}
+
+
+=item $ua->credentials($netloc, $realm, $uname, $pass)
+
+Set the user name and password to be used for a realm.  It is often more
+useful to specialize the get_basic_credentials() method instead.
+
+=cut
+
+sub credentials
+{
+    my($self, $netloc, $realm, $uid, $pass) = @_;
+    @{ $self->{'basic_authentication'}{$netloc}{$realm} } = ($uid, $pass);
+}
+
+
+=item $ua->get_basic_credentials($realm, $uri, [$proxy])
+
+This is called by request() to retrieve credentials for a Realm
+protected by Basic Authentication or Digest Authentication.
+
+Should return username and password in a list.  Return undef to abort
+the authentication resolution atempts.
+
+This implementation simply checks a set of pre-stored member
+variables. Subclasses can override this method to e.g. ask the user
+for a username/password.  An example of this can be found in
+C<lwp-request> program distributed with this library.
+
+=cut
+
+sub get_basic_credentials
+{
+    my($self, $realm, $uri, $proxy) = @_;
+    return if $proxy;
+
+    my $netloc = $uri->netloc;
+    if (exists $self->{'basic_authentication'}{$netloc}{$realm}) {
+	return @{ $self->{'basic_authentication'}{$netloc}{$realm} };
+    }
+
+    return (undef, undef);
+}
+
+
+=item $ua->agent([$product_id])
+
+Get/set the product token that is used to identify the user agent on
+the network.  The agent value is sent as the "User-Agent" header in
+the requests. The default agent name is "libwww-perl/#.##", where
+"#.##" is substitued with the version numer of this library.
+
+The user agent string should be one or more simple product identifiers
+with an optional version number separated by the "/" character.
+Examples are:
+
+  $ua->agent('Checkbot/0.4 ' . $ua->agent);
+  $ua->agent('Mozilla/5.0');
+
+=item $ua->from([$email_address])
+
+Get/set the Internet e-mail address for the human user who controls
+the requesting user agent.  The address should be machine-usable, as
+defined in RFC 822.  The from value is send as the "From" header in
+the requests.  There is no default.  Example:
+
+  $ua->from('aas@sn.no');
+
+=item $ua->timeout([$secs])
+
+Get/set the timeout value in seconds. The default timeout() value is
+180 seconds, i.e. 3 minutes.
+
+=item $ua->cookie_jar([$cookies])
+
+Get/set the I<HTTP::Cookies> object to use.  The default is to have no
+cookie_jar, i.e. never automatically add "Cookie" headers to the
+requests.
+
+=item $ua->parse_head([$boolean])
+
+Get/set a value indicating wether we should initialize response
+headers from the E<lt>head> section of HTML documents. The default is
+TRUE.  Do not turn this off, unless you know what you are doing.
+
+=item $ua->max_size([$bytes])
+
+Get/set the size limit for response content.  The default is undef,
+which means that there is not limit.  If the returned response content
+is only partial, because the size limit was exceeded, then a
+"X-Content-Range" header will be added to the response.
+
+=cut
+
+sub timeout    { shift->_elem('timeout',   @_); }
+sub agent      { shift->_elem('agent',     @_); }
+sub from       { shift->_elem('from',      @_); }
+sub cookie_jar { shift->_elem('cookie_jar',@_); }
+sub parse_head { shift->_elem('parse_head',@_); }
+sub max_size   { shift->_elem('max_size',  @_); }
+
+# depreciated
+sub use_eval   { shift->_elem('use_eval',  @_); }
+sub use_alarm
+{
+    Carp::carp("LWP::UserAgent->use_alarm(BOOL) is a no-op")
+	if @_ > 1 && $^W;
+    "";
+}
+
+
+# Declarations of AutoLoaded methods
+sub clone;
+sub is_protocol_supported;
+sub mirror;
+sub proxy;
+sub env_proxy;
+sub no_proxy;
+sub _need_proxy;
+
+
+1;
+__END__
+
+
+=item $ua->clone;
+
+Returns a copy of the LWP::UserAgent object
+
+=cut
+
+
+sub clone
+{
+    my $self = shift;
+    my $copy = bless { %$self }, ref $self;  # copy most fields
+
+    # elements that are references must be handled in a special way
+    $copy->{'no_proxy'} = [ @{$self->{'no_proxy'}} ];  # copy array
+
+    $copy;
+}
+
+
+=item $ua->is_protocol_supported($scheme)
+
+You can use this method to query if the library currently support the
+specified C<scheme>.  The C<scheme> might be a string (like 'http' or
+'ftp') or it might be an URI::URL object reference.
+
+=cut
+
+sub is_protocol_supported
+{
+    my($self, $scheme) = @_;
+    if (ref $scheme) {
+	# assume we got a reference to an URI::URL object
+	$scheme = $scheme->abs->scheme;
+    } else {
+	Carp::croak("Illeal scheme '$scheme' passed to is_protocol_supported")
+	    if $scheme =~ /\W/;
+	$scheme = lc $scheme;
+    }
+    return LWP::Protocol::implementor($scheme);
+}
+
+
+=item $ua->mirror($url, $file)
+
+Get and store a document identified by a URL, using If-Modified-Since,
+and checking of the Content-Length.  Returns a reference to the
+response object.
+
+=cut
+
+sub mirror
+{
+    my($self, $url, $file) = @_;
+
+    LWP::Debug::trace('()');
+    my $request = new HTTP::Request('GET', $url);
+
+    if (-e $file) {
+	my($mtime) = (stat($file))[9];
+	if($mtime) {
+	    $request->header('If-Modified-Since' =>
+			     HTTP::Date::time2str($mtime));
+	}
+    }
+    my $tmpfile = "$file-$$";
+
+    my $response = $self->request($request, $tmpfile);
+    if ($response->is_success) {
+
+	my $file_length = (stat($tmpfile))[7];
+	my($content_length) = $response->header('Content-length');
+
+	if (defined $content_length and $file_length < $content_length) {
+	    unlink($tmpfile);
+	    die "Transfer truncated: " .
+		"only $file_length out of $content_length bytes received\n";
+	} elsif (defined $content_length and $file_length > $content_length) {
+	    unlink($tmpfile);
+	    die "Content-length mismatch: " .
+		"expected $content_length bytes, got $file_length\n";
+	} else {
+	    # OK
+	    if (-e $file) {
+		# Some dosish systems fail to rename if the target exists
+		chmod 0777, $file;
+		unlink $file;
+	    }
+	    rename($tmpfile, $file) or
+		die "Cannot rename '$tmpfile' to '$file': $!\n";
+
+	    if (my $lm = $response->last_modified) {
+		# make sure the file has the same last modification time
+		utime $lm, $lm, $file;
+	    }
+	}
+    } else {
+	unlink($tmpfile);
+    }
+    return $response;
+}
+
+=item $ua->proxy(...)
+
+Set/retrieve proxy URL for a scheme:
+
+ $ua->proxy(['http', 'ftp'], 'http://proxy.sn.no:8001/');
+ $ua->proxy('gopher', 'http://proxy.sn.no:8001/');
+
+The first form specifies that the URL is to be used for proxying of
+access methods listed in the list in the first method argument,
+i.e. 'http' and 'ftp'.
+
+The second form shows a shorthand form for specifying
+proxy URL for a single access scheme.
+
+=cut
+
+sub proxy
+{
+    my($self, $key, $proxy) = @_;
+
+    LWP::Debug::trace("$key, $proxy");
+
+    if (!ref($key)) {   # single scalar passed
+	my $old = $self->{'proxy'}{$key};
+	$self->{'proxy'}{$key} = $proxy;
+	return $old;
+    } elsif (ref($key) eq 'ARRAY') {
+	for(@$key) {    # array passed
+	    $self->{'proxy'}{$_} = $proxy;
+	}
+    }
+    return undef;
+}
+
+=item $ua->env_proxy()
+
+Load proxy settings from *_proxy environment variables.  You might
+specify proxies like this (sh-syntax):
+
+  gopher_proxy=http://proxy.my.place/
+  wais_proxy=http://proxy.my.place/
+  no_proxy="my.place"
+  export gopher_proxy wais_proxy no_proxy
+
+Csh or tcsh users should use the C<setenv> command to define these
+envirionment variables.
+
+=cut
+
+sub env_proxy {
+    my ($self) = @_;
+    my($k,$v);
+    while(($k, $v) = each %ENV) {
+	$k = lc($k);
+	next unless $k =~ /^(.*)_proxy$/;
+	$k = $1;
+	if ($k eq 'no') {
+	    $self->no_proxy(split(/\s*,\s*/, $v));
+	}
+	else {
+	    $self->proxy($k, $v);
+	}
+    }
+}
+
+=item $ua->no_proxy($domain,...)
+
+Do not proxy requests to the given domains.  Calling no_proxy without
+any domains clears the list of domains. Eg:
+
+ $ua->no_proxy('localhost', 'no', ...);
+
+=cut
+
+sub no_proxy {
+    my($self, @no) = @_;
+    if (@no) {
+	push(@{ $self->{'no_proxy'} }, @no);
+    }
+    else {
+	$self->{'no_proxy'} = [];
+    }
+}
+
+
+# Private method which returns the URL of the Proxy configured for this
+# URL, or undefined if none is configured.
+sub _need_proxy
+{
+    my($self, $url) = @_;
+
+    $url = new URI::URL($url) unless ref $url;
+
+    LWP::Debug::trace("($url)");
+
+    # check the list of noproxies
+
+    if (@{ $self->{'no_proxy'} }) {
+	my $host = $url->host;
+	return undef unless defined $host;
+	my $domain;
+	for $domain (@{ $self->{'no_proxy'} }) {
+	    if ($host =~ /$domain$/) {
+		LWP::Debug::trace("no_proxy configured");
+		return undef;
+	    }
+	}
+    }
+
+    # Currently configured per scheme.
+    # Eventually want finer granularity
+
+    my $scheme = $url->scheme;
+    if (exists $self->{'proxy'}{$scheme}) {
+
+	LWP::Debug::debug('Proxied');
+	return new URI::URL($self->{'proxy'}{$scheme});
+    }
+
+    LWP::Debug::debug('Not proxied');
+    undef;
+}
+
+1;
+
+=back
+
+=head1 SEE ALSO
+
+See L<LWP> for a complete overview of libwww-perl5.  See F<lwp-request> and
+F<lwp-mirror> for examples of usage.
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/media.types
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/media.types	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/LWP/media.types	(revision 21112)
@@ -0,0 +1,120 @@
+# $Id$
+#
+# This file defines the mapping from file name extentions to media types.
+# Media types where the subtype does not have the "x-" prefix should be
+# registered by IANA:
+#
+#   ftp://www.isi.edu/in-notes/iana/assignments/media-types/
+#
+# See also: RFC-1590
+
+# Text formats
+text/html			html htm
+text/plain			txt text pm el c h cc hh cxx hxx f90
+text/richtext			rtx
+text/tab-separated-values	tsv
+
+text/x-setext			etx
+text/x-pod			pod			# Perl documentation
+
+# Image formats
+image/png			png
+image/gif			gif			# Compuserve
+image/ief			ief
+image/jpeg			jpeg jpg jpe jfif
+image/tiff			tiff tif
+
+image/x-cmu-raster		ras
+image/x-portable-anymap		pnm
+image/x-portable-bitmap		pbm
+image/x-portable-graymap	pgm
+image/x-portable-pixmap		ppm
+image/x-rgb			rgb			# SGI IRIS
+image/x-xbitmap			xbm			# X Windows
+image/x-xpixmap			xpm			# X Windows
+image/x-xwindowdump		xwd			# X Windows
+image/x-bmp			bmp			# MS Windows
+image/x-targa			tga
+image/x-fits			fts
+
+
+# Audio formats
+audio/basic			au snd
+audio/x-aiff			aif aiff aifc
+audio/x-wav			wav			# MS Windows
+audio/x-mpeg-2			mp2			# MPEG layer 2 audio
+
+# Video formats
+video/mpeg			mpeg mpg mpe
+video/quicktime			qt
+video/x-msvideo			avi
+video/x-sgi-movie		mov
+
+# Message
+message/rfc822			mail
+message/partial
+message/external-body
+message/news			art
+
+# Applications
+application/octet-stream	bin
+application/postscript		ai eps ps
+application/oda			oda
+application/atomicmail
+application/andrew-insert
+application/slate
+application/wita
+application/dec-dx
+application/dca-rtf
+application/activemessage
+application/rtf			rft			# MS Word
+application/applefile
+application/mac-binhex40	hqx
+application/news-message-id
+application/news-transmission
+application/wordperfect5.1
+application/pdf			pdf
+application/zip			zip
+application/macwriteii
+application/msword
+application/remote-printing
+application/mathematica
+application/cybercash
+application/commonground
+application/iges
+application/riscos
+application/eshop
+
+application/x-mif		mif			# FrameMaker
+application/x-maker		fm
+application/x-csh		csh
+application/x-dvi		dvi
+application/x-hdf		hdf
+application/x-latex		latex
+application/x-netcdf		nc cdf
+application/x-sh		sh
+application/x-tcl		tcl
+application/x-perl		pl
+application/x-tex		tex
+application/x-texinfo		texinfo texi
+application/x-troff		t tr roff
+application/x-troff-man		man 1 2 3 4 5 6 7 8
+application/x-troff-me		me
+application/x-troff-ms		ms
+application/x-wais-source	src
+application/x-bcpio		bcpio
+application/x-cpio		cpio
+application/x-gtar		gtar			# GNU tar
+application/x-shar		shar
+application/x-sv4cpio		sv4cpio
+application/x-sv4crc		sv4crc
+application/x-tar		tar
+application/x-ustar		ustar
+
+# Multipart
+multipart/mixed
+multipart/alternative
+multipart/digest
+multipart/parallel
+multipart/appledouble
+multipart/header-set
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Escape.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Escape.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Escape.pm	(revision 21112)
@@ -0,0 +1,119 @@
+#
+# $Id$
+#
+
+package URI::Escape;
+use strict;
+
+=head1 NAME
+
+URI::Escape - Escape and unescape unsafe characters
+
+=head1 SYNOPSIS
+
+ use URI::Escape;
+ $safe = uri_escape("10% is enough\n");
+ $verysafe = uri_escape("foo", "\0-\377");
+ $str  = uri_unescape($safe);
+
+=head1 DESCRIPTION
+
+This module provide functions to escape and unescape URI strings.
+Some characters are regarded as "unsafe" and must be escaped in
+accordance with RFC 1738.  Escaped characters are represented by a
+triplet consisting of the character "%" followed by two hexadecimal
+digits.  The following functions are provided (and exported by
+default):
+
+=over 4
+
+=item uri_escape($string, [$unsafe])
+
+This function replaces all unsafe characters in the $string with their
+escape sequence and return the result.
+
+The uri_escape() function takes an optional second argument that
+overrides the set of characters that are to be escaped.  The set is
+specified as a string that can be used in a regular expression
+character class (between [ ]).  E.g.:
+
+  \x00-\x1f\x7f-\xff          # all control and hi-bit characters
+  a-z                         # all lower case characters
+  ^A-Za-z                     # everything not a letter
+
+The default set of characters to be escaped is:
+
+  \x00-\x20"#%;<>?{}|\\^~`\[\]\x7F-\xFF
+
+=item uri_unescape($string)
+
+Returns a string with all %XX sequences replaced with the actual
+character.
+
+=back
+
+The module can also export the %escapes hash which contains the
+mapping from all characters to the corresponding escape code.
+
+=head1 SEE ALSO
+
+L<URI::URL>
+
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+
+This program is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself.
+
+=cut
+
+use vars qw(@ISA @EXPORT @EXPORT_OK $VERSION);
+use vars qw(%escapes);
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(uri_escape uri_unescape);
+@EXPORT_OK = qw(%escapes);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use Carp ();
+
+# Build a char->hex map
+for (0..255) {
+    $escapes{chr($_)} = sprintf("%%%02X", $_);
+}
+
+my %subst;  # compiled patternes
+
+sub uri_escape
+{
+    my($text, $patn) = @_;
+    return undef unless defined $text;
+    if (defined $patn){
+	unless (exists  $subst{$patn}) {
+	    # Because we can't compile regex we fake it with a cached sub
+	    $subst{$patn} =
+	      eval "sub {\$_[0] =~ s/([$patn])/\$escapes{\$1}/g; }";
+	    Carp::croak("uri_escape: $@") if $@;
+	}
+	&{$subst{$patn}}($text);
+    } else {
+	# Default unsafe characters. (RFC1738 section 2.2)
+	$text =~ s/([\x00-\x20\"#%;<>?{}|\\^~`\[\]\x7F-\xFF])/$escapes{$1}/g;
+    }
+    $text;
+}
+
+sub uri_unescape
+{
+    # Note from RFC1630:  "Sequences which start with a percent sign
+    # but are not followed by two hexadecimal characters are reserved
+    # for future extension"
+    my @copy = @_;
+    for (@copy) { s/%([\dA-Fa-f]{2})/chr(hex($1))/eg; }
+    wantarray ? @copy : $copy[0];
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Heuristic.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Heuristic.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/Heuristic.pm	(revision 21112)
@@ -0,0 +1,191 @@
+package URI::Heuristic;
+
+# $Id$
+
+=head1 NAME
+
+uf_urlstr - Expand URL using heuristics
+
+=head1 SYNOPSIS
+
+ use URI::Heuristic qw(uf_urlstr);
+ $url = uf_urlstr("perl");             # http://www.perl.com
+ $url = uf_urlstr("www.sol.no/sol");   # http://www.sol.no/sol
+ $url = uf_urlstr("aas");              # http://www.aas.no
+ $url = uf_urlstr("ftp.funet.fi");     # ftp://ftp.funet.fi
+ $url = uf_urlstr("/etc/passwd");      # file:/etc/passwd
+
+=head1 DESCRIPTION
+
+This module provide functions that expand strings into real absolute
+URLs using some builtin heuristics.  Strings that already represent
+absolute URLs (i.e. start with a C<scheme:> part) are never modified
+and are returned unchanged.  The main use of these functions are to
+allow abbreviated URLs similar to what many web browsers allow for URLs
+typed in by the user.
+
+The following functions are provided:
+
+=over 4
+
+=item uf_urlstr($str)
+
+The uf_urlstr() function will try to make the string passed as
+argument into a proper absolute URL string.  The "uf_" prefix stands
+for "User Friendly".
+
+=item uf_url($str)
+
+This functions work the same way as uf_urlstr() but it will
+return a C<URI::URL> object.
+
+=back
+
+=head1 ENVIRONMENT
+
+If the hostname portion of a URL does not contain any dots, then
+certain qualified guesses will be made.  These guesses are governed be
+the following two environment variables.
+
+=over 10
+
+=item COUNTRY
+
+This is the two letter country code (ISO 3166) for your location.  If
+the domain name of your host ends with two letters, then it is taken
+to be the default country. See also L<Locale::Country>.
+
+=item URL_GUESS_PATTERN
+
+Contain a space separated list of URL patterns to try.  The string
+"ACME" is for some reason used as a placeholder for the host name in
+the URL provided.  Example:
+
+ URL_GUESS_PATTERN="www.ACME.no www.ACME.se www.ACME.com"
+ export URL_GUESS_PATTERN
+
+Specifying URL_GUESS_PATTERN disables any guessing rules based on
+country.  An empty URL_GUESS_PATTERN disables any guessing that
+involves host name lookups.
+
+=back
+
+=head1 COPYRIGHT
+
+Copyright 1997-1998, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+=cut
+
+use strict;
+
+use vars qw(@EXPORT_OK $VERSION $MY_COUNTRY %LOCAL_GUESSING $DEBUG);
+
+require Exporter;
+*import = \&Exporter::import;
+@EXPORT_OK = qw(uf_url uf_urlstr);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+eval {
+    require Net::Domain;
+    my $fqdn = Net::Domain::hostfqdn();
+    $MY_COUNTRY = lc($1) if $fqdn =~ /\.([a-zA-Z]{2})$/;
+
+    # Some other heuristics to guess country?  Perhaps looking
+    # at some environment variable (LANG, LC_ALL, ???)
+    $MY_COUNTRY = $ENV{COUNTRY} if exists $ENV{COUNTRY};
+};
+
+%LOCAL_GUESSING =
+(
+ 'us' => [qw(www.ACME.gov www.ACME.mil)],
+ 'uk' => [qw(www.ACME.co.uk www.ACME.org.uk www.ACME.ac.uk)],
+ 'au' => [qw(www.ACME.com.au www.ACME.org.au www.ACME.edu.au)],
+ 'il' => [qw(www.ACME.co.il www.ACME.org.il www.ACME.net.il)],
+ # send corrections and new entries to <aas@sn.no>
+);
+
+
+sub uf_url ($)
+{
+    require URI::URL;
+    URI::URL->new(uf_urlstr($_[0]));
+}
+
+
+sub uf_urlstr ($)
+{
+    local($_) = @_;
+    print STDERR "uf_urlstr: resolving $_\n" if $DEBUG;
+    return unless defined;
+
+    s/^\s+//;
+    s/\s+$//;
+
+    if (/^(www|web|home)\./) {
+	$_ = "http://$_";
+
+    } elsif (/^(ftp|gopher|news|wais|http|https)\./) {
+	$_ = "$1://$_";
+
+    } elsif (m,^/,      ||          # absolute file name
+	     m,^\.\.?/, ||          # relative file name
+	     m,^[a-zA-Z]:[/\\],)    # dosish file name
+    {
+	$_ = "file:$_";
+
+    } elsif (/^\w+([\.\-]\w+)*\@(\w+\.)+\w{2,3}$/) {
+	$_ = "mailto:$_";
+
+    } elsif (!/^[.+\-\w]+:/) {      # no scheme specified
+	if (s/^(\w+(?:\.\w+)*)([\/:\?\#]|$)/$2/) {
+	    my $host = $1;
+
+	    if ($host !~ /\./ && $host ne "localhost") {
+		my @guess;
+		if (exists $ENV{URL_GUESS_PATTERN}) {
+		    @guess = map { s/\bACME\b/$host/; $_ }
+		             split(' ', $ENV{URL_GUESS_PATTERN});
+		} else {
+		    if ($MY_COUNTRY) {
+			my $special = $LOCAL_GUESSING{$MY_COUNTRY};
+			if ($special) {
+			    my @special = @$special;
+			    push(@guess, map { s/\bACME\b/$host/; $_ }
+                                               @special);
+			} else {
+			    push(@guess, "www.$host.$MY_COUNTRY");
+			}
+		    }
+		    push(@guess, map "www.$host.$_",
+			             "com", "org", "net", "edu", "int");
+		}
+
+
+		my $guess;
+		for $guess (@guess) {
+		    print STDERR "uf_urlstr: gethostbyname('$guess')..."
+		      if $DEBUG;
+		    if (gethostbyname($guess)) {
+			print STDERR "yes\n" if $DEBUG;
+			$host = $guess;
+			last;
+		    }
+		    print STDERR "no\n" if $DEBUG;
+		}
+	    }
+	    $_ = "http://$host$_";
+
+	} else {
+	    # pure junk, just return it unchanged...
+
+	}
+    }
+    print STDERR "uf_urlstr: ==> $_\n" if $DEBUG;
+
+    $_;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL.pm	(revision 21112)
@@ -0,0 +1,949 @@
+package URI::URL;
+
+$VERSION = "4.15";   # $Date$
+sub Version { $VERSION; }
+
+require 5.004;
+
+require Exporter;
+@ISA = qw(Exporter);
+@EXPORT = qw(url);
+
+require AutoLoader;
+*AUTOLOAD = \&AutoLoader::AUTOLOAD;
+
+use Carp ();
+
+# Basic lexical elements, taken from RFC 1738:
+#
+#  safe         = "$" | "-" | "_" | "." | "+"
+#  extra        = "!" | "*" | "'" | "(" | ")" | ","
+#  national     = "{" | "}" | "|" | "\" | "^" | "~" | "[" | "]" | "`"
+#  punctuation  = "<" | ">" | "#" | "%" | <">
+#  reserved     = ";" | "/" | "?" | ":" | "@" | "&" | "="
+#  escape       = "%" hex hex
+#  unreserved   = alpha | digit | safe | extra
+#  uchar        = unreserved | escape
+#  xchar        = unreserved | reserved | escape
+
+# draft-fielding-url-syntax-05.txt adds '+' to the reserved chars and
+# takes '~' out
+
+use strict;
+use vars qw($reserved $reserved_no_slash $reserved_no_form $unsafe
+	    $Debug $Strict_URL
+	   );
+
+$reserved          = ";\\/?:\\@&=+#%"; # RFC 1738 reserved pluss '#' and '%'
+$reserved_no_slash = ";?:\\@&=+#%";    # used when escaping path
+$reserved_no_form  = ";\\/?:\\@#%";    # used when escaping params and query
+
+# This is the unsafe characters (excluding those reserved)
+$unsafe   = "\x00-\x20{}|\\\\^\\[\\]`<>\"\x7F-\xFF";
+#$unsafe .= "~";  # according to RFC1738 but not to common practice
+
+$Debug         = 0;     # set to 1 to print URLs on creation
+$Strict_URL    = 0;     # see new()
+
+use overload ( '""' => 'as_string', 'fallback' => 1 );
+
+my %Implementor = (); # mapping from scheme to implementation class
+
+
+# Easy to use constructor
+sub url ($;$)
+{
+    URI::URL->new(@_);
+}
+
+
+# URI::URL objects are implemented as blessed hashes:
+#
+# Each of the URL components (scheme, netloc, user, password, host,
+# port, path, params, query, fragment) are stored under their
+# name. The netloc, path, params and query is stored in quoted
+# (escaped) form.  The others is stored unquoted (unescaped).
+#
+# Netloc is special since it is rendundant (same as
+# "user:password@host:port") and must be kept in sync with those.
+#
+# The '_str' key stores a cached stringified version of the URL
+# (by definition in quoted form).
+# The '_base' key stores the optional base of a relative URL.
+#
+# The '_orig_url' is used while debugging is on.
+#
+# Subclasses may add their own keys but must take great care to
+# avoid names which might be used in later verions of this module.
+
+sub new
+{
+    my($class, $init, $base) = @_;
+
+    my $self;
+    if (ref $init) {
+	$self = $init->clone;
+	$self->base($base) if $base;
+    } else {
+	$init = "" unless defined $init;
+	# RFC 1738 appendix suggest that we just ignore extra whitespace
+	$init =~ s/\s+//g;
+	# Also get rid of any <URL: > wrapper
+	$init =~ s/^<(?:URL:)?(.*)>$/$1/;
+
+	# We need a scheme to determine which class to use
+	my($scheme) = $init =~ m/^([.+\-\w]+):/;
+	if (!$scheme and $base){ # get scheme from base
+	    if (ref $base){ # may be object or just a string
+		$scheme = $base->scheme;
+	    } else {
+		$scheme = $1 if $base =~ m/^([.+\-\w]+):/;
+	    }
+	}
+	unless($scheme){
+	    Carp::croak("Unable to determine scheme for '$init'")
+		if $Strict_URL;
+	    $scheme = 'http';
+	}
+	my $impclass = URI::URL::implementor($scheme);
+	unless ($impclass) {
+	    Carp::croak("URI::URL scheme '$scheme' is not supported")
+		if $Strict_URL;
+	    # use generic as fallback
+	    require URI::URL::_generic;
+	    URI::URL::implementor($scheme, 'URI::URL::_generic');
+	    $impclass = 'URI::URL::_generic';
+	}
+
+	# hand-off to scheme specific implementation sub-class
+	$self->{'_orig_url'} = $init if $Debug;
+	$self = $impclass->new($init, $base);
+    }
+    $self->print_on('STDERR') if $Debug;
+    return $self;
+}
+
+
+sub clone
+{
+    my $self = shift;
+    # this work as long as none of the components are references themselves
+    bless { %$self }, ref $self;
+}
+
+
+sub implementor
+{
+    my($scheme, $impclass) = @_;
+    unless (defined $scheme) {
+	require URI::URL::_generic;
+	return 'URI::URL::_generic';
+    }
+
+    $scheme = lc($scheme);
+    if ($impclass) {
+	$impclass->_init_implementor($scheme);
+	my $old = $Implementor{$scheme};
+	$Implementor{$scheme} = $impclass;
+	return $old;
+    }
+
+    my $ic = $Implementor{$scheme};
+    return $ic if $ic;
+
+    # scheme not yet known, look for internal or
+    # preloaded (with 'use') implementation
+    $ic = "URI::URL::$scheme";  # default location
+    no strict 'refs';
+    # check we actually have one for the scheme:
+    unless (defined @{"${ic}::ISA"}) {
+	# Try to load it
+	eval { require "URI/URL/$scheme.pm"; };
+	die $@ if $@ && $@ !~ /Can\'t locate/;
+	$ic = '' unless defined @{"${ic}::ISA"};
+    }
+    if ($ic) {
+	$ic->_init_implementor($scheme);
+	$Implementor{$scheme} = $ic;
+    }
+    $ic;
+}
+
+
+sub _init_implementor
+{
+    my($class, $scheme) = @_;
+    # Remember that one implementor class may actually
+    # serve to implement several URL schemes.
+
+    if ($] < 5.003_17) {
+	no strict qw(refs);
+	# Setup overloading inheritace - experimental
+	%{"${class}::OVERLOAD"} = %URI::URL::OVERLOAD
+	    unless defined %{"${class}::OVERLOAD"};
+    }
+}
+
+
+# This private method help us implement access to the elements in the
+# URI::URL object hash (%$self).  You can set up access to an element
+# with a routine similar to this one:
+#
+#  sub component { shift->_elem('component', @_); }
+
+sub _elem
+{
+    my $self = shift;
+    my $elem = shift;
+    my $old = $self->{$elem};
+    if (@_) {
+	$self->{$elem} = shift;
+	$self->{'_str'} = '';        # void cached string
+    }
+    $old;
+}
+
+
+# Make all standard methods available to all kinds of URLs.  This allow
+# us to call these without to much worry when URI::URL::strict(0);
+
+sub bad_method;
+
+*netloc       = \&bad_method;
+*user         = \&bad_method;
+*password     = \&bad_method;
+*host         = \&bad_method;
+*port         = \&bad_method;
+*default_port = \&bad_method;
+
+*full_path    = \&bad_method;
+*epath        = \&bad_method;
+*eparams      = \&bad_method;
+*equery       = \&bad_method;
+
+*path         = \&bad_method;
+*path_components = \&bad_method;
+*params       = \&bad_method;
+*query        = \&bad_method;
+*frag         = \&bad_method;
+
+
+#
+# A U T O  L O A D I N G
+#
+# The rest of the methods are autoloaded because they should be less
+# frequently used.  We define stubs here so that inheritance works as
+# it should.
+sub newlocal;
+sub strict;
+sub base;
+sub scheme;
+sub crack;
+sub abs;
+sub rel;
+sub as_string;
+sub eq;
+sub print_on;
+
+# Don't need DESTROY but avoid trying to AUTOLOAD it.
+sub DESTROY { }
+
+1;
+__END__
+
+sub newlocal
+{
+    require URI::URL::file;
+    my $class = shift;
+    URI::URL::file->newlocal(@_);  # pass it on the the file class
+}
+
+sub strict
+{
+    return $Strict_URL unless @_;
+    my $old = $Strict_URL;
+    $Strict_URL = $_[0];
+    $old;
+}
+
+# Access some attributes of a URL object:
+sub base {
+    my $self = shift;
+    my $base  = $self->{'_base'};
+
+    if (@_) { # set
+	my $new_base = shift;
+	$new_base = $new_base->abs if ref($new_base);  # unsure absoluteness
+	$self->{_base} = $new_base;
+    }
+    return unless defined wantarray;
+
+    # The base attribute supports 'lazy' conversion from URL strings
+    # to URL objects. Strings may be stored but when a string is
+    # fetched it will automatically be converted to a URL object.
+    # The main benefit is to make it much cheaper to say:
+    #   new URI::URL $random_url_string, 'http:'
+    if (defined($base) && !ref($base)) {
+	$base = new URI::URL $base;
+	$self->_elem('_base', $base); # set new object
+    }
+    $base;
+}
+
+sub scheme {
+    my $self = shift;
+    my $old = $self->{'scheme'};
+    if (@_) {
+	my $new_scheme = shift;
+	if (defined($new_scheme) && length($new_scheme)) {
+	    # reparse URL with new scheme
+	    my $str = $self->as_string;
+	    $str =~ s/^[\w+\-.]+://;
+	    my $newself = new URI::URL "$new_scheme:$str";
+	    %$self = %$newself;
+	    bless $self, ref($newself);
+	} else {
+	    $self->{'scheme'} = undef;
+	}
+    }
+    $old;
+}
+
+sub crack
+{
+    # should be overridden by subclasses
+    my $self = shift;
+    ($self->scheme,  # 0: scheme
+     undef,          # 1: user
+     undef,          # 2: passwd
+     undef,          # 3: host
+     undef,          # 4: port
+     undef,          # 5: path
+     undef,          # 6: params
+     undef,          # 7: query
+     undef           # 8: fragment
+    )
+}
+
+# These are overridden by _generic (this is just a noop for those schemes that
+# do not wish to be a subclass of URI::URL::_generic)
+sub abs { shift->clone; }
+sub rel { shift->clone; }
+
+# This method should always be overridden in subclasses
+sub as_string {
+    "<URL>";
+}
+
+# Compare two URLs, subclasses will provide a more correct implementation
+sub eq {
+    my($self, $other) = @_;
+    $other = URI::URL->new($other, $self) unless ref $other;
+    ref($self) eq ref($other) &&
+      $self->scheme eq $other->scheme &&
+      $self->as_string eq $other->as_string;  # Case-sensitive
+}
+
+# This is set up as an alias for various methods
+sub bad_method {
+    my $self = shift;
+    my $scheme = $self->scheme;
+    Carp::croak("Illegal method called for $scheme: URL")
+	if $Strict_URL;
+    # Carp::carp("Illegal method called for $scheme: URL")
+    #     if $^W;
+    undef;
+}
+
+sub print_on
+{
+    no strict qw(refs);  # because we use strings as filehandles
+    my $self = shift;
+    my $fh = shift || 'STDERR';
+    my($k, $v);
+    print $fh "Dump of URI::URL $self...\n";
+    foreach $k (sort keys %$self){
+	$v = $self->{$k};
+	$v = 'UNDEF' unless defined $v;
+	print $fh "  $k\t'$v'\n";
+    }
+}
+
+1;
+
+
+#########################################################################
+#### D O C U M E N T A T I O N
+#########################################################################
+
+=head1 NAME
+
+URI::URL - Uniform Resource Locators (absolute and relative)
+
+=head1 SYNOPSIS
+
+ use URI::URL;
+
+ # Constructors
+ $url1 = new URI::URL 'http://www.perl.com/%7Euser/gisle.gif';
+ $url2 = new URI::URL 'gisle.gif', 'http://www.com/%7Euser';
+ $url3 = url 'http://www.sn.no/'; # handy constructor
+ $url4 = $url2->abs;       # get absolute url using base
+ $url5 = $url2->abs('http:/other/path');
+ $url6 = newlocal URI::URL 'test';
+
+ # Stringify URL
+ $str1 = $url->as_string;  # complete escaped URL string
+ $str2 = $url->full_path;  # escaped path+params+query
+ $str3 = "$url";           # use operator overloading
+
+ # Retrieving Generic-RL components:
+ $scheme   = $url->scheme;
+ $netloc   = $url->netloc; # see user,password,host,port below
+ $path     = $url->path;
+ $params   = $url->params;
+ $query    = $url->query;
+ $frag     = $url->frag;
+
+ # Accessing elements in their escaped form
+ $path     = $url->epath;
+ $params   = $url->eparams;
+ $query    = $url->equery;
+
+ # Retrieving Network location (netloc) components:
+ $user     = $url->user;
+ $password = $url->password;
+ $host     = $url->host;
+ $port     = $url->port;   # returns default if not defined
+
+ # Retrieve escaped path components as an array
+ @path     = $url->path_components;
+
+ # HTTP query-string access methods
+ @keywords = $url->keywords;
+ @form     = $url->query_form;
+
+ # All methods above can set the field values, e.g:
+ $url->scheme('http');
+ $url->host('www.w3.org');
+ $url->port($url->default_port);
+ $url->base($url5);                      # use string or object
+ $url->keywords(qw(dog bones));
+
+ # File methods
+ $url = new URI::URL "file:/foo/bar";
+ open(F, $url->local_path) or die;
+
+ # Compare URLs
+ if ($url->eq("http://www.sn.no")) or die;
+
+=head1 DESCRIPTION
+
+This module implements the URI::URL class representing Uniform
+Resource Locators (URL). URLs provide a compact string representation
+for resources available via the Internet. Both absolute (RFC 1738) and
+relative (RFC 1808) URLs are supported.
+
+URI::URL objects are created by calling new(), which takes as argument
+a string representation of the URL or an existing URL object reference
+to be cloned. Specific individual elements can then be accessed via
+the scheme(), user(), password(), host(), port(), path(), params(),
+query() and frag() methods.  In addition escaped versions of the path,
+params and query can be accessed with the epath(), eparams() and
+equery() methods.  Note that some URL schemes will support all these
+methods.
+
+The object constructor new() must be able to determine the scheme for
+the URL.  If a scheme is not specified in the URL itself, it will use
+the scheme specified by the base URL. If no base URL scheme is defined
+then new() will croak if URI::URL::strict(1) has been invoked,
+otherwise I<http> is silently assumed.  Once the scheme has been
+determined new() then uses the implementor() function to determine
+which class implements that scheme.  If no implementor class is
+defined for the scheme then new() will croak if URI::URL::strict(1)
+has been invoked, otherwise the internal generic URL class is assumed.
+
+Internally defined schemes are implemented by the
+URI::URL::I<scheme_name> module.  The URI::URL::implementor() function
+can be used to explicitly set the class used to implement a scheme if
+you want to override this.
+
+
+=head1 HOW AND WHEN TO ESCAPE
+
+
+=over 3
+
+=item This is an edited extract from a URI specification:
+
+The printability requirement has been met by specifying a safe set of
+characters, and a general escaping scheme for encoding "unsafe"
+characters. This "safe" set is suitable, for example, for use in
+electronic mail.  This is the canonical form of a URI.
+
+There is a conflict between the need to be able to represent many
+characters including spaces within a URI directly, and the need to be
+able to use a URI in environments which have limited character sets
+or in which certain characters are prone to corruption. This conflict
+has been resolved by use of an hexadecimal escaping method which may
+be applied to any characters forbidden in a given context. When URLs
+are moved between contexts, the set of characters escaped may be
+enlarged or reduced unambiguously.  The canonical form for URIs has
+all white spaces encoded.
+
+=item Notes:
+
+A URL string I<must>, by definition, consist of escaped
+components. Complete URLs are always escaped.
+
+The components of a URL string must be I<individually> escaped.  Each
+component of a URL may have a separate requirements regarding what
+must be escaped, and those requirements are also dependent on the URL
+scheme.
+
+Never escape an already escaped component string.
+
+=back
+
+This implementation expects an escaped URL string to be passed to
+new() and will return a fully escaped URL string from as_string()
+and full_path().
+
+Individual components can be manipulated in unescaped or escaped
+form. The following methods return/accept unescaped strings:
+
+    scheme                  path
+    user                    params
+    password                query
+    host                    frag
+    port
+
+The following methods return/accept partial I<escaped> strings:
+
+    netloc                  eparams
+    epath                   equery
+
+I<Partial escaped> means that only reserved characters
+(i.e. ':', '@', '/', ';', '?', '=', '&' in addition to '%', '.' and '#')
+needs to be escaped when they are to be treated as normal characters.
+I<Fully escaped> means that all unsafe characters are escaped. Unsafe
+characters are all all control characters (%00-%1F and %7F), all 8-bit
+characters (%80-%FF) as well
+as '{', '}', '|', '\', '^', '[', ']' '`', '"', '<' and '>'.
+Note that the character '~' is B<not> considered
+unsafe by this library as it is common practice to use it to reference
+personal home pages, but it is still unsafe according to RFC 1738.
+
+=head1 ADDING NEW URL SCHEMES
+
+New URL schemes or alternative implementations for existing schemes
+can be added to your own code. To create a new scheme class use code
+like:
+
+   package MYURL::foo;
+   @ISA = (URI::URL::implementor());   # inherit from generic scheme
+
+The 'URI::URL::implementor()' function call with no parameters returns
+the name of the class which implements the generic URL scheme
+behaviour (typically C<URI::URL::_generic>). All hierarchical schemes
+should be derived from this class.
+
+Your class can then define overriding methods (e.g., new(), _parse()
+as required).
+
+To register your new class as the implementor for a specific scheme
+use code like:
+
+   URI::URL::implementor('x-foo', 'MYURL::foo');
+
+Any new URL created for scheme 'x-foo' will be implemented by your
+C<MYURL::foo> class. Existing URLs will not be affected.
+
+
+=head1 FUNCTIONS
+
+=over 3
+
+=item $url = URI::URL->new( $url_string [, $base_url] )
+
+This is the object constructor.  It will create a new URI::URL object,
+initialized from the URL string.
+
+=item $url = URI::URL->newlocal($path);
+
+Returns an URL object that denotes a path within the local filesystem.
+Paths not starting with '/' are interpreted relative to the current
+working directory.  This constructor always return an absolute 'file'
+URL.
+
+=item $url = url($url_string, [, $base_url])
+
+Alternative constructor function.  The url() function is exported by
+the URI::URL module and is easier both to type and read than calling
+C<URI::URL->new> directly.  Useful for constructs like this:
+
+   $h = url($str)->host;
+
+This function is just a wrapper for URI::URL->new.
+
+=item URI::URL::strict($bool)
+
+If strict is true then we croak on errors.  The function returns the
+previous value.
+
+=item URI::URL::implementor([$scheme, [$class]])
+
+Use this function to get or set implementor class for a scheme.
+Returns '' if specified scheme is not supported.  Returns generic URL
+class if no scheme specified.
+
+=back
+
+=head1 METHODS
+
+This section describes the methods available for an URI::URL object.
+Note that some URL schemes will disallow some of these methods and
+will croak if they are used.  Some URL schemes add additional methods
+that are described in the sections to follow.
+
+Attribute access methods marked with (*) can take an optional argument
+to set the value of the attribute, and they always return the old
+value.
+
+=over 3
+
+=item $url->abs([$base])
+
+The abs() method attempts to return a new absolute URI::URL object
+for a given URL.  In order to convert a relative URL into an absolute
+one, a I<base> URL is required. You can associate a default base with a
+URL either by passing a I<base> to the new() constructor when a
+URI::URL is created or using the base() method on the object later.
+Alternatively you can specify a one-off base as a parameter to the
+abs() method.
+
+The rel() method will do the opposite transformation.
+
+Some older parsers used to allow the scheme name to be present in the
+relative URL if it was the same as the base URL scheme.  RFC1808 says
+that this should be avoided, but you can enable this old behaviour by
+setting the $URI::URL::ABS_ALLOW_RELATIVE_SCHEME variable to a TRUE
+value.  The difference is demonstrated by the following examples:
+
+  url("http:foo")->abs("http://host/a/b")     ==>  "http:foo"
+
+  local $URI::URL::ABS_ALLOW_RELATIVE_SCHEME = 1;
+  url("http:foo")->abs("http://host/a/b")     ==>  "http:/host/a/foo"
+
+You can also have the abs() method ignore if there is too many ".."
+segments in the relative URL by setting
+$URI::URL::ABS_REMOTE_LEADING_DOTS to a TRUE value.  The difference is
+demonstrated by the following examples:
+
+  url("../../../foo")->abs("http://host/a/b")   ==> "http://host/../../foo"
+
+  local $URI::URL::ABS_REMOTE_LEADING_DOTS = 1;
+  url("../../../foo")->abs("http://host/a/b")   ==> "http://host/foo"
+
+
+=item $url->as_string
+
+Returns a string representing the URL in its canonical form.  All
+unsafe characters will be escaped.  This method is overloaded as the
+perl "stringify" operator, which means that URLs can be used as
+strings in many contexts.
+
+=item $url->base (*)
+
+Get/set the base URL associated with the current URI::URL object.  The
+base URL matters when you call the abs() method.
+
+=item $url->clone
+
+Returns a copy of the current URI::URL object.
+
+=item $url->crack
+
+Return a 9 element array with the following content:
+
+   0: $url->scheme *)
+   1: $url->user
+   2: $url->password
+   3: $url->host
+   4: $url->port
+   5: $url->epath
+   6: $url->eparams
+   7: $url->equery
+   8: $url->frag
+
+All elements except I<scheme> will be undefined if the corresponding
+URL part is not available.
+
+B<Note:> The scheme (first element) returned by crack will aways be
+defined.  This is different from what the $url->scheme returns, since
+it will return I<undef> for relative URLs.
+
+=item $url->default_port
+
+Returns the default port number for the URL scheme that the URI::URL
+belongs too.
+
+=item $url->eparams (*)
+
+Get/set the URL parameters in escaped form.
+
+=item $url->epath (*)
+
+Get/set the URL path in escaped form.
+
+=item $url->eq($other_url)
+
+Compare two URLs to decide if they match or not.  The rules for how
+comparison is made varies for different parts of the URLs; scheme and
+netloc comparison is case-insensitive, and escaped chars match their
+%XX encoding unless they are "reserved" or "unsafe".
+
+=item $url->equery (*)
+
+Get/set the URL query string in escaped form.
+
+=item $url->full_path
+
+Returns the string "/path;params?query".  This is the string that is
+passed to a remote server in order to access the document.
+
+=item $url->frag (*)
+
+Get/set the fragment (unescaped)
+
+=item $url->host (*)
+
+Get/set the host (unescaped)
+
+=item $url->netloc (*)
+
+Get/set the network location in escaped form.  Setting the network
+location will affect 'user', 'password', 'host' and 'port'.
+
+=item $url->params (*)
+
+Get/set the URL parameters (unescaped)
+
+=item $url->password (*)
+
+Get/set the password (unescaped)
+
+=item $url->path (*)
+
+Get/set the path (unescaped).  This method will croak if any of the
+path components in the return value contain the "/" character.  You
+should use the epath() method to be safe.
+
+=item $url->path_components (*)
+
+Get/set the path using a list of unescaped path components.  The
+return value will loose the distinction beween '.' and '%2E'.  When
+setting a value, a '.' is converted to be a literal '.' and is
+therefore encoded as '%2E'.
+
+=item $url->port (*)
+
+Get/set the network port (unescaped)
+
+=item $url->rel([$base])
+
+Return a relative URL if possible.  This is the opposite of what the
+abs() method does.  For instance:
+
+   url("http://www.math.uio.no/doc/mail/top.html",
+       "http://www.math.uio.no/doc/linux/")->rel
+
+will return a relative URL with path set to "../mail/top.html" and
+with the same base as the original URL.
+
+If the original URL already is relative or the scheme or netloc does
+not match the base, then a copy of the original URL is returned.
+
+
+=item $url->print_on(*FILEHANDLE);
+
+Prints a verbose presentation of the contents of the URL object to
+the specified file handle (default STDERR).  Mainly useful for
+debugging.
+
+=item $url->scheme (*)
+
+Get/set the scheme for the URL.
+
+=item $url->query (*)
+
+Get/set the query string (unescaped).  This method will croak if the
+string returned contains both '+' and '%2B' or '=' together with '%3D'
+or '%26'.  You should use the equery() method to be safe.
+
+=item $url->user (*)
+
+Get/set the URL user name (unescaped)
+
+=back
+
+=head1 HTTP METHODS
+
+For I<http> URLs you may also access the query string using the
+keywords() and the query_form() methods.  Both will croak if the query
+is not of the correct format.  The encodings look like this:
+
+  word1+word2+word3..        # keywords
+  key1=val1&key2=val2...     # query_form
+
+Note: These functions does not return the old value when they are used
+to set a value of the query string.
+
+=over 3
+
+=item $url->keywords (*)
+
+The keywords() method returns a list of unescaped strings.  The method
+can also be used to set the query string by passing in the keywords as
+individual arguments to the method.
+
+=item $url->query_form (*)
+
+The query_form() method return a list of unescaped key/value pairs.
+If you assign the return value to a hash you might loose some values
+if the key is repeated (which it is allowed to do).
+
+This method can also be used to set the query sting of the URL like this:
+
+  $url->query_form(foo => 'bar', foo => 'baz', equal => '=');
+
+If the value part of a key/value pair is a reference to an array, then
+it will be converted to separate key/value pairs for each value.  This
+means that these two calls are equal:
+
+  $url->query_form(foo => 'bar', foo => 'baz');
+  $url->query_form(foo => ['bar', 'baz']);
+
+=back
+
+=head1 FILE METHODS
+
+The I<file> URLs implement the local_path() method that returns a path
+suitable for access to files within the current filesystem.  These
+methods can B<not> be used to set the path of the URL.
+
+=over 3
+
+=item $url->local_path
+
+This method is really just an alias for one of the methods below
+depending on what system you run on.
+
+=item $url->unix_path
+
+Returns a path suitable for use on a Unix system.  This method will
+croak if any of the path segments contains a "/" or a NULL character.
+
+=item $url->dos_path
+
+Returns a path suitable for use on a MS-DOS or MS-Windows system.
+
+=item $url->mac_path
+
+Returns a path suitable for use on a Macintosh system.
+
+=item $url->vms_path
+
+Returns a path suitable for use on a VMS system.  VMS is a trademark
+of Digital.
+
+=back
+
+=head1 GOPHER METHODS
+
+The methods access the parts that are specific for the gopher URLs.
+These methods access different parts of the $url->path.
+
+=over 3
+
+=item $url->gtype (*)
+
+=item $url->selector (*)
+
+=item $url->search (*)
+
+=item $url->string (*)
+
+=back
+
+=head1 NEWS METHODS
+
+=over 3
+
+=item $url->group (*)
+
+=item $url->article (*)
+
+=back
+
+=head1 WAIS METHODS
+
+The methods access the parts that are specific for the wais URLs.
+These methods access different parts of the $url->path.
+
+=over 3
+
+=item $url->database (*)
+
+=item $url->wtype (*)
+
+=item $url->wpath (*)
+
+=back
+
+=head1 MAILTO METHODS
+
+=over 3
+
+=item $url->address (*)
+
+The mail address can also be accessed with the netloc() method.
+
+=back
+
+
+=head1 WHAT A URL IS NOT
+
+URL objects do not, and should not, know how to 'get' or 'put' the
+resources they specify locations for, anymore than a postal address
+'knows' anything about the postal system. The actual access/transfer
+should be achieved by some form of transport agent class (see
+L<LWP::UserAgent>). The agent class can use the URL class, but should
+not be a subclass of it.
+
+=head1 AUTHORS / ACKNOWLEDGMENTS
+
+This module is (distantly) based on the C<wwwurl.pl> code in the
+libwww-perl distribution developed by Roy Fielding
+<fielding@ics.uci.edu>, as part of the Arcadia project at the
+University of California, Irvine, with contributions from Brooks
+Cutter.
+
+Gisle Aas <aas@sn.no>, Tim Bunce <Tim.Bunce@ig.co.uk>, Roy Fielding
+<fielding@ics.uci.edu> and Martijn Koster <m.koster@webcrawler.com>
+(in English alphabetical order) have collaborated on the complete
+rewrite for Perl 5, with input from other people on the libwww-perl
+mailing list.
+
+If you have any suggestions, bug reports, fixes, or enhancements, send
+them to the libwww-perl mailing list at <libwww-perl@ics.uci.edu>.
+
+=head1 COPYRIGHT
+
+Copyright 1995-1997 Gisle Aas.
+Copyright 1995 Martijn Koster.
+
+This program is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself.
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_generic.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_generic.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_generic.pm	(revision 21112)
@@ -0,0 +1,514 @@
+#####################################################################
+#
+#       Internal pre-defined generic scheme support
+#
+# In this implementation all schemes are subclassed from
+# URI::URL::_generic. This turns out to have reasonable mileage.
+# See also draft-ietf-uri-relative-url-06.txt
+
+package URI::URL::_generic;           # base support for generic-RL's
+require URI::URL;
+@ISA = qw(URI::URL);
+
+use URI::Escape qw(uri_escape uri_unescape %escapes);
+
+
+sub new {                               # inherited by subclasses
+    my($class, $init, $base) = @_;
+    my $url = bless { }, $class;        # create empty object
+    $url->_parse($init);                # parse $init into components
+    $url->base($base) if $base;
+    $url;
+}
+
+
+# Generic-RL parser
+# See draft-ietf-uri-relative-url-06.txt Section 2
+
+sub _parse {
+    my($self, $u, @comps) = @_;
+    return unless defined $u;
+
+    # Deside which components to parse (scheme & path is manatory)
+    @comps = qw(netloc query params frag) unless (@comps);
+    my %parse = map {$_ => 1} @comps;
+
+    # This parsing code is based on
+    #   draft-ietf-uri-relative-url-06.txt Section 2.4
+
+    # 2.4.1
+    $self->{'frag'} = uri_unescape($1)
+      if $parse{'frag'} && $u =~ s/#(.*)$//;
+    # 2.4.2
+    $self->{'scheme'} = lc($1) if $u =~ s/^\s*([\w\+\.\-]+)://;
+    # 2.4.3
+    $self->netloc("$1")	# passing $1 directly fails if netloc is autoloaded
+      if $parse{'netloc'} && $u =~ s!^//([^/]*)!!;
+    # 2.4.4
+    $self->{'query'} = $1
+      if $parse{'query'} && $u =~ s/\?(.*)//;
+    # 2.4.5
+    $self->{'params'} = $1
+      if $parse{'params'} && $u =~ s/;(.*)//;
+
+    # 2.4.6
+    #
+    # RFC 1738 says:
+    #
+    #     Note that the "/" between the host (or port) and the
+    #     url-path is NOT part of the url-path.
+    #
+    # however, RFC 1808, 2.4.6. says:
+    #
+    #    Even though the initial slash is not part of the URL path,
+    #    the parser must remember whether or not it was present so
+    #    that later processes can differentiate between relative
+    #    and absolute paths.  Often this is done by simply storing
+    #    he preceding slash along with the path.
+    #
+    # In version < 4.01 of URI::URL we used to strip the leading
+    # "/" when asked for $self->path().  This created problems for
+    # the consitency of the interface, so now we just consider the
+    # slash to be part of the path and we also make an empty path
+    # default to "/".
+
+    # we don't test for $parse{path} becase it is mandatory
+    $self->{'path'} = $u;
+}
+
+
+# Generic-RL stringify
+#
+sub as_string
+{
+    my $self = shift;
+    return $self->{'_str'} if $self->{'_str'};
+
+    my($scheme, $netloc, $frag) = @{$self}{qw(scheme netloc frag)};
+
+    my $u = $self->full_path(1);  # path+params+query
+
+    # rfc 1808 says:
+    #    Note that the fragment identifier (and the "#" that precedes
+    #    it) is not considered part of the URL.  However, since it is
+    #    commonly used within the same string context as a URL, a parser
+    #    must be able to recognize the fragment when it is present and
+    #    set it aside as part of the parsing process.
+    $u .= "#" . uri_escape($frag, $URI::URL::unsafe) if defined $frag;
+
+    $u = "//$netloc$u" if defined $netloc;
+    $u = "$scheme:$u" if $scheme;
+    # Inline: uri_escape($u, $URI::URL::unsafe);
+    $u =~ s/([$URI::URL::unsafe])/$escapes{$1}/go;
+    $self->{'_str'} = $u;  # set cache and return
+}
+
+# Generic-RL stringify full path "path;params?query"
+#
+sub full_path
+{
+    my($self, $dont_escape)  = @_;
+    my($path, $params, $query) = @{$self}{'path', 'params', 'query'};
+    my $p = '';
+    $p .= $path if defined $path;
+    # see comment in _parse 2.4.6 about the next line
+    $p = "/$p" if defined($self->{netloc}) && $p !~ m:^/:;
+    $p .= ";$params" if defined $params;
+    $p .= "?$query"  if defined $query;
+    return $p if $dont_escape;
+    # Inline: URI::Escape::uri_escape($p, $URI::URL::unsafe);
+    $p =~ s/([$URI::URL::unsafe])/$escapes{$1}/go;
+    $p;
+}
+
+# default_port()
+#
+# subclasses will usually want to override this
+#
+sub default_port { undef; }
+
+
+#####################################################################
+#
+# Methods to handle URL's elements
+
+# These methods always return the current value,
+# so you can use $url->path to read the current value.
+# If a new value is passed, e.g. $url->path('foo'),
+# it also sets the new value, and returns the previous value.
+# Use $url->path(undef) to set the value to undefined.
+
+sub netloc {
+    my $self = shift;
+    my $old = $self->_elem('netloc', @_);
+    return $old unless @_;
+
+    # update fields derived from netloc
+    my $nl = $self->{'netloc'} || '';
+    if ($nl =~ s/^([^:@]*):?(.*?)@//){
+	$self->{'user'}     = uri_unescape($1);
+	$self->{'password'} = uri_unescape($2) if $2 ne '';
+    }
+    if ($nl =~ /^([^:]*):?(\d*)$/){
+	my $port = $2;
+	# Since this happes so frequently, we inline this call:
+	#    my $host = uri_unescape($1);
+	my $host = $1;
+	$host =~ s/%([\dA-Fa-f]{2})/chr(hex($1))/eg;
+	$self->{'host'} = $host;
+	if ($port ne '') {
+	    $self->{'port'} = $port;
+	    if ($self->default_port == $port) {
+		$self->{'netloc'} =~ s/:\d+//;
+	    }
+	} elsif (defined $self->{'netloc'}) {
+	    $self->{'netloc'} =~ s/:$//;  # handle empty port spec
+	}
+    }
+    $self->{'_str'} = '';
+    $old;
+}
+
+
+# A U T O  L O A D E R
+# Don't remove this comment, it keeps AutoSplit happy!!
+# @ISA = qw(AutoLoader)
+#
+# The rest of the methods are only loaded on demand.  Stubs are neccesary
+# for inheritance to work.
+
+#sub netloc;  # because netloc is used by the _parse()
+sub user;
+sub password;
+sub host;
+sub port;
+sub _netloc_elem;
+sub epath;
+sub path;
+sub path_components;
+sub eparams;
+sub params;
+sub equery;
+sub query;
+sub frag;
+sub crack;
+sub abs;
+sub rel;
+sub eq;
+
+1;
+__END__
+
+
+# Fields derived from generic netloc:
+sub user     { shift->_netloc_elem('user',    @_); }
+sub password { shift->_netloc_elem('password',@_); }
+sub host     { shift->_netloc_elem('host',    @_); }
+
+sub port {
+    my $self = shift;
+    my $old = $self->_netloc_elem('port', @_);
+    defined($old) ? $old : $self->default_port;
+}
+
+sub _netloc_elem {
+    my($self, $elem, @val) = @_;
+    my $old = $self->_elem($elem, @val);
+    return $old unless @val;
+
+    # update the 'netloc' element
+    my $nl = '';
+    my $host = $self->{'host'};
+    if (defined $host) {  # can't be any netloc without any host
+	my $user = $self->{'user'};
+	$nl .= uri_escape($user, $URI::URL::reserved) if defined $user;
+	$nl .= ":" . uri_escape($self->{'password'}, $URI::URL::reserved)
+	  if defined($user) and defined($self->{'password'});
+	$nl .= '@' if length $nl;
+	$nl .= uri_escape($host, $URI::URL::reserved);
+	my $port = $self->{'port'};
+	$nl .= ":$port" if defined($port) && $port != $self->default_port;
+    }
+    $self->{'netloc'} = $nl;
+    $self->{'_str'} = '';
+    $old;
+}
+
+sub epath {
+     my $self = shift;
+     my $old = $self->_elem('path', @_);
+     return '/' if !defined($old) || !length($old);
+     return "/$old" if $old !~ m|^/| && defined $self->{'netloc'};
+     $old;
+}
+
+sub path {
+    my $self = shift;
+    my $old = $self->_elem('path',
+		      map uri_escape($_, $URI::URL::reserved_no_slash), @_);
+    return unless defined wantarray;
+    return '/' if !defined($old) || !length($old);
+    Carp::croak("Path components contain '/' (you must call epath)")
+	if $old =~ /%2[fF]/ and !@_;
+    $old = "/$old" if $old !~ m|^/| && defined $self->{'netloc'};
+    return uri_unescape($old);
+}
+
+sub path_components {
+    my $self = shift;
+    my $old = $self->{'path'};
+    $old = '' unless defined $old;
+    $old = "/$old" if $old !~ m|^/| && defined $self->{'netloc'};
+    if (@_) {
+	$self->_elem('path',
+		     join("/", map {uri_escape($_, $URI::URL::reserved)} @_));
+    }
+    map { uri_unescape($_) } split("/", $old, -1);
+}
+
+sub eparams  { shift->_elem('params',  @_); }
+
+sub params {
+    my $self = shift;
+    my $old = $self->_elem('params', map {uri_escape($_,$URI::URL::reserved_no_form)} @_);
+    return uri_unescape($old) if defined $old;
+    undef;
+}
+
+sub equery   { shift->_elem('query',   @_); }
+
+sub query {
+    my $self = shift;
+    my $old = $self->_elem('query', map { uri_escape($_, $URI::URL::reserved_no_form) } @_);
+    if (defined(wantarray) && defined($old)) {
+	if ($old =~ /%(?:26|2[bB]|3[dD])/) {  # contains escaped '=' '&' or '+'
+	    my $mess;
+	    for ($old) {
+		$mess = "Query contains both '+' and '%2B'"
+		  if /\+/ && /%2[bB]/;
+		$mess = "Form query contains escaped '=' or '&'"
+		  if /=/  && /%(?:3[dD]|26)/;
+	    }
+	    if ($mess) {
+		Carp::croak("$mess (you must call equery)");
+	    }
+	}
+	# Now it should be safe to unescape the string without loosing
+	# information
+	return uri_unescape($old);
+    }
+    undef;
+
+}
+
+# No efrag method because the fragment is always stored unescaped
+sub frag     { shift->_elem('frag', @_); }
+
+sub crack
+{
+    my $self = shift;
+    return $self unless wantarray;
+    my @c = @{$self}{qw(scheme user password host port path params query frag)};
+    if (!$c[0]) {
+	# try to determine scheme
+	my $base = $self->base;
+	$c[0] = $base->scheme if $base;
+	$c[0] ||= 'http';  # last resort, default in URI::URL::new
+    }
+    $c[4] ||= $self->default_port;
+    @c;
+}
+
+# Generic-RL: Resolving Relative URL into an Absolute URL
+#
+# Based on RFC1808 section 4
+#
+sub abs
+{
+    my($self, $base, $allow_relative_scheme) = @_;
+    $allow_relative_scheme = $URI::URL::ABS_ALLOW_RELATIVE_SCHEME if @_ < 3;
+    my $embed = $self->clone;
+
+    $base = $self->base unless $base;      # default to default base
+    return $embed unless $base;            # we have no base (step1)
+
+    $base = new URI::URL $base unless ref $base; # make obj if needed
+
+    my($scheme, $host, $path, $params, $query, $frag) =
+	@{$embed}{qw(scheme host path params query frag)};
+
+    # just use base if we are empty             (2a)
+    return $base->clone
+      unless grep(defined($_) && $_ ne '',
+		  $scheme,$host,$port,$path,$params,$query,$frag);
+
+    # if we have a scheme we must already be absolute   (2b),
+    #
+    # but sec. 5.2 also says: Some older parsers allow the scheme name
+    # to be present in a relative URL if it is the same as the base
+    # URL scheme.  This is considered to be a loophole in prior
+    # specifications of the partial URLs and should be avoided by
+    # future parsers.
+    #
+    # The old behavoir can be enabled by passing a TRUE value to the
+    # $allow_relative_scheme parameter.
+    return $embed if $scheme &&
+      (!$allow_relative_scheme || $scheme ne $base->{'scheme'});
+
+    $embed->{'_str'} = '';                      # void cached string
+    $embed->{'scheme'} = $base->{'scheme'};     # (2c)
+
+    return $embed if $embed->{'netloc'};        # (3)
+    $embed->netloc($base->{'netloc'});          # (3)
+
+    return $embed if $path =~ m:^/:;            # (4)
+
+    if ($path eq '') {                          # (5)
+	$embed->{'path'} = $base->{'path'};     # (5)
+
+	return $embed if defined $embed->{'params'}; # (5a)
+	$embed->{'params'} = $base->{'params'};      # (5a)
+
+	return $embed if defined $embed->{'query'};  # (5b)
+	$embed->{'query'} = $base->{'query'};        # (5b)
+
+	return $embed;
+    }
+
+    # (Step 6)  # draft 6 suggests stack based approach
+
+    my $basepath = $base->{'path'};
+    my $relpath  = $embed->{'path'};
+
+    $basepath =~ s!^/!!;
+    $basepath =~ s!/$!/.!;                # prevent empty segment
+    my @path = split('/', $basepath);     # base path into segments
+    pop(@path);                           # remove last segment
+
+    $relpath =~ s!/$!/.!;                 # prevent empty segment
+
+    push(@path, split('/', $relpath));    # append relative segments
+
+    my @newpath = ();
+    my $isdir = 0;
+    my $segment;
+
+    foreach $segment (@path) {            # left to right
+	if ($segment eq '.') {            # ignore "same" directory
+	    $isdir = 1;
+	}
+	elsif ($segment eq '..') {
+	    $isdir = 1;
+	    my $last = pop(@newpath);
+	    if (!defined $last) {         # nothing to pop
+		push(@newpath, $segment); # so must append
+	    }
+	    elsif ($last eq '..') {       # '..' cannot match '..'
+		# so put back again, and append
+		push(@newpath, $last, $segment);
+	    }
+	    #else
+		# it was a component,
+		# keep popped
+	} else {
+	    $isdir = 0;
+	    push(@newpath, $segment);
+	}
+    }
+
+    if ($URI::URL::ABS_REMOTE_LEADING_DOTS) {
+	shift @newpath while @newpath && $newpath[0] =~ /^\.\.?$/;
+    }
+
+    $embed->{'path'} = '/' . join('/', @newpath) .
+	($isdir && @newpath ? '/' : '');
+
+    $embed;
+}
+
+# The oposite of $url->abs.  Return a URL as much relative as possible
+sub rel {
+    my($self, $base) = @_;
+    my $rel = $self->clone;
+    $base = $self->base unless $base;
+    return $rel unless $base;
+    $base = new URI::URL $base unless ref $base;
+    $rel->base($base);
+
+    my($scheme, $netloc, $path) = @{$rel}{qw(scheme netloc path)};
+    if (!defined($scheme) && !defined($netloc)) {
+	# it is already relative
+	return $rel;
+    }
+
+    my($bscheme, $bnetloc, $bpath) = @{$base}{qw(scheme netloc path)};
+    for ($bscheme, $bnetloc, $netloc) { $_ = '' unless defined }
+
+    unless ($scheme eq $bscheme && $netloc eq $bnetloc) {
+	# different location, can't make it relative
+	return $rel;
+    }
+
+    for ($path, $bpath) {  $_ = "/$_" unless m,^/,; }
+
+    # Make it relative by eliminating scheme and netloc
+    $rel->{'scheme'} = undef;
+    $rel->netloc(undef);
+
+    # This loop is based on code from Nicolai Langfeldt <janl@ifi.uio.no>.
+    # First we calculate common initial path components length ($li).
+    my $li = 1;
+    while (1) {
+	my $i = index($path, '/', $li);
+	last if $i < 0 ||
+                $i != index($bpath, '/', $li) ||
+	        substr($path,$li,$i-$li) ne substr($bpath,$li,$i-$li);
+	$li=$i+1;
+    }
+    # then we nuke it from both paths
+    substr($path, 0,$li) = '';
+    substr($bpath,0,$li) = '';
+
+    if ($path eq $bpath && defined($rel->frag) && !defined($rel->equery)) {
+        $rel->epath('');
+    } else {
+        # Add one "../" for each path component left in the base path
+        $path = ('../' x $bpath =~ tr|/|/|) . $path;
+	$path = "./" if $path eq "";
+        $rel->epath($path);
+    }
+
+    $rel;
+}
+
+
+# Compare two URLs
+sub eq {
+    my($self, $other) = @_;
+    local($^W) = 0; # avoid warnings if we compare undef values
+    $other = URI::URL->new($other, $self) unless ref $other;
+
+    # Compare scheme and netloc
+    return 0 if ref($self) ne ref($other);                # must be same class
+    return 0 if $self->scheme ne $other->scheme;          # Always lower case
+    return 0 if lc($self->netloc) ne lc($other->netloc);  # Case-insensitive
+
+    # Compare full_path:
+    # According to <draft-ietf-http-v11-spec-05>:
+    # Characters other than those in the "reserved" and "unsafe" sets
+    # are equivalent to their %XX encodings.
+    my $fp1 = $self->full_path;
+    my $fp2 = $other->full_path;
+    for ($fp1, $fp2) {
+	s,%([\dA-Fa-f]{2}),
+	  my $x = $1;
+	  my $c = chr(hex($x));
+	  $c =~ /^[;\/?:\@&=+\"\#%<>\0-\040\177]/ ? "%\L$x" : $c;
+	,eg;
+    }
+    return 0 if $fp1 ne $fp2;
+    return 0 if $self->frag ne $other->frag;
+    1;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_login.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_login.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/_login.pm	(revision 21112)
@@ -0,0 +1,25 @@
+package URI::URL::_login;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+
+# Generic terminal logins.  This is used as a base class for 'telnet',
+# 'tn3270', and 'rlogin' URL schemes.
+
+
+sub _parse {
+    my($self, $init) = @_;
+    # All we want from _generic is the 'netloc' handling.
+    $self->URI::URL::_generic::_parse($init, 'netloc');
+}
+
+
+*path      = \&URI::URL::bad_method;
+*epath     = \&URI::URL::bad_method;
+*query     = \&URI::URL::bad_method;
+*equery    = \&URI::URL::bad_method;
+*params    = \&URI::URL::bad_method;
+*eparams   = \&URI::URL::bad_method;
+*frag      = \&URI::URL::bad_method;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/data.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/data.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/data.pm	(revision 21112)
@@ -0,0 +1,160 @@
+package URI::URL::data;
+
+# Implements data:-URLs as specified in draft-masinter-url-data-02.txt
+#
+# Abstract
+#
+# A new URL scheme, "data", is defined. It allows inclusion of small
+# data items as "immediate" data, as if it had been included externally.
+#
+# Description
+#
+# Some applications that use URLs also have a need to embed (small)
+# media type data directly inline. This document defines a new URL
+# scheme that would work like 'immediate addressing'. The URLs are of
+# the form:
+#
+#        data:[<mediatype>][;base64],<data>
+#
+
+require URI::URL;
+@ISA = qw(URI::URL);
+
+use URI::Escape;
+use MIME::Base64 ();
+
+# as defined in rfc1522.txt
+my $tspecial  = qq(()<>@,;:/[]?.=);
+my $tokenchar = qq([^\\s\000-\037\177-\377\Q$tspecial\E]);
+
+
+sub new {
+    my($class, $init, $base) = @_;
+
+    my $self = bless { }, $class;
+
+    $self->{'scheme'} = lc($1) if $init =~ s/^\s*([\w\+\.\-]+)://;
+    $self->{'frag'} = $1 if $init =~ s/\#(.*)$//;
+
+    my $type = "";
+    if ($init =~ s/^($tokenchar+\/$tokenchar+)//o) {
+	$type = $1;
+    }
+    while ($init =~ s/^;($tokenchar+)=([^,;]*)//o) {
+	# XXX should we unescape the parst
+	$type .= ";$1=$2";
+    }
+
+    $self->{'media_type'} = $type;
+
+    my $base64;
+    if ($init =~ s/^;base64//i) {
+	$base64 = 1;
+    }
+    
+    $init =~ s/^,//;
+    if ($base64) {
+	$self->{'base64'} = uri_unescape($init);
+    } else {
+	$self->{'data'} = uri_unescape($init);
+    }
+    $self->base($base) if $base;
+    $self;
+}
+
+sub media_type
+{
+    my $self = shift;
+    my $old = $self->{'media_type'};
+    if (@_) {
+	$self->{'media_type'} = shift || "";
+	delete $self->{'_str'};
+    }
+    my($type, $param) = split(/;/, $old, 2);
+    if ($type) {
+	$type = lc $type;
+    } else {
+	$type = "text/plain";
+	$param = "charset=US-ASCII" unless $param;
+    }
+    if (wantarray) {
+	return ($type, $param);
+    } else {
+	return $type;
+    }
+}
+
+sub data
+{
+    my $self = shift;
+    my $old_data;
+    my $old_base64;
+    $old_data = $self->{'data'};
+    $old_base64 = $self->{'base64'};
+    if (@_) {
+	if ($_[1]) { # base64 flag
+	    $self->{'base64'} = $_[0];
+	    delete $self->{'data'};
+	} else {
+	    $self->{'data'} = $_[0];
+	    delete $self->{'base64'};
+	}
+	delete $self->{'_str'};
+    }
+    unless (defined $old_data) {
+	$old_data = MIME::Base64::decode($old_base64);
+	$self->{'data'} = $old_data unless @_;
+    }
+    $old_data;
+}
+
+
+sub crack
+{
+    my $self = shift;
+    ($self->{'scheme'}
+     || 'data',          # scheme
+     undef,              # user
+     undef,              # passwd
+     undef,              # host
+     undef,              # port
+     $self->data,        # path
+     $self->{'media_type'},  # params
+     undef,              # query
+     $self->{'frag'}     # fragment
+    )
+}
+
+sub as_string {
+    my $self = shift;
+    return $self->{'_str'} if $self->{'_str'};
+    my $str = ($self->{'scheme'} || 'data') . ":";
+    $str .= $self->{'media_type'};
+    if (defined $self->{'base64'}) {
+	$str .= ";base64,$self->{'base64'}";
+    } else {
+	my $urlenc = uri_escape($self->{'data'});
+	my $base64 = MIME::Base64::encode($self->{'data'});
+	if (length($base64) + 7 < length($urlenc)) {
+	    $str .= ";base64,$base64";
+	    $self->{'base64'} = $base64;
+	} else {
+	    $str .= ",$urlenc";
+	}
+    }
+    $self->{'_str'} = $str;
+}
+
+sub eq {
+    my($self, $other) = @_;
+    return 0 if ref($self) ne ref($other);
+    return 0 if $self->scheme ne $other->scheme;
+
+    my $mt1 = join(";", $self->media_type);
+    my $mt2 = join(";", $other->media_type);
+    return 0 if $mt1 ne $mt2;
+
+    $self->data eq $other->data;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/file.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/file.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/file.pm	(revision 21112)
@@ -0,0 +1,165 @@
+package URI::URL::file;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+require Carp;
+require Config;
+
+# First we try to determine what kind of system we run on
+my $os = $Config::Config{'osname'};
+OS: {
+    $ostype = 'vms', last if $os eq 'VMS';
+    $ostype = 'dos', last if $os =~ /^(?:os2|mswin32|msdos)$/i;
+    $ostype = 'mac', last if $os eq "Mac";
+    $ostype = 'unix';  # The default
+}
+# NOTE: If you add more types to this list, remember to add a xxx_path
+# method as well.
+
+# This is the BNF found in RFC 1738:
+#
+# fileurl        = "file://" [ host | "localhost" ] "/" fpath
+# fpath          = fsegment *[ "/" fsegment ]
+# fsegment       = *[ uchar | "?" | ":" | "@" | "&" | "=" ]
+# Note that fsegment can contain '?' (query) but not ';' (param)
+
+sub _parse {
+    my($self, $init) = @_;
+    # The file URL can't have query
+    $self->URI::URL::_generic::_parse($init, qw(netloc path params frag));
+}
+
+# sub local_path { ... }
+#
+# Returns a path suitable for use on the local system (we just
+# set up an alias (derived from $ostype) to one of the path methods
+# defined below)
+*local_path = \&{$ostype . "_path"};
+
+*query  = \&URI::URL::bad_method;
+*equery = \&URI::URL::bad_method;
+
+# A U T O  L O A D E R
+# Don't remove this comment, it keeps AutoSplit happy!!
+# @ISA = qw(AutoLoader)
+#
+# These methods are autoloaded as needed
+sub newlocal;
+sub unix_path;
+sub dos_path ;
+sub mac_path ;
+sub vms_path ;
+1;
+__END__
+
+sub newlocal {
+    my($class, $path) = @_;
+
+    Carp::croak("Only implemented for Unix and OS/2 file systems")
+      unless $ostype eq "unix" or $^O =~ /os2|mswin32/i;
+    # XXX: Should implement the same thing for other systems
+
+    my $url = new URI::URL "file:";
+    unless (defined $path and
+    	    ($path =~ m:^/: or 
+	     ($^O eq 'os2' and Cwd::sys_is_absolute($path)) or
+	     ($^O eq 'MSWin32' and $path =~ m<^[A-Za-z]:[\\/]|^[\\/]{2}>))) {
+	require Cwd;
+	my $cwd = Cwd::fastcwd();
+	$cwd =~ s:/?$:/:; # force trailing slash on dir
+	$path = (defined $path) ? $cwd . $path : $cwd;
+    }
+    $url->path($path);
+    $url;
+}
+
+sub unix_path
+{
+    my $self = shift;
+    my @p;
+    for ($self->path_components) {
+	Carp::croak("Path component contains '/' or '\0'") if m|[\0/]|;
+	if (@p) {
+	    next unless length $_;   # skip empty path segments
+	    next if $_ eq '.';       # skip these too
+	    if ($_ eq '..' && $p[-1] ne '..') {  # go up one level
+		pop(@p) if $p[-1] ne '';
+		next;
+	    }
+	}
+	push(@p, $_);
+    }
+    shift(@p) if @p > 1 && $p[0] eq '.';   # './' rendundant if there is more
+    return '/' if !@p || (@p == 1 && $p[0] eq '');
+    join('/', @p);
+}
+
+sub dos_path
+{
+    my $self = shift;
+    my @p;
+    for ($self->path_components) {
+	Carp::croak("Path component contains '/' or '\\'") if m|[/\\]|;
+	push(@p, uc $_);
+    }
+    my $p = join("\\", @p);
+    $p =~ s/^\\([A-Z]:)/$1/;  # Fix drive letter specification
+    $p;
+}
+
+sub mac_path
+{
+    my $self = shift;
+    my @p;
+    for ($self->path_components) {
+	Carp::croak("Path component contains ':'") if /:/;
+	# XXX: Should probably want to do something about ".." and "."
+	# path segments.  I don't know how these are represented in
+	# the Machintosh file system.  If these are valid file names
+	# then we should split the path ourself, as $u->path_components
+	# loose the distinction between '.' and '%2E'.
+	push(@p, $_);
+    }
+    if (@p && $p[0] eq '') {
+	shift @p;
+    } else {
+	unshift(@p, '');
+    }
+    join(':', @p);
+}
+
+sub vms_path
+{
+    # ????? Can some VMS people please redo this function ??????
+
+    # This is implemented based on what RFC1738 (sec 3.10) says in the
+    # VMS file example:
+    #
+    #  DISK$USER:[MY.NOTES]NOTE123456.TXT
+    #
+    #      that might become
+    #
+    #  file:/disk$user/my/notes/note12345.txt
+    #
+    # BEWARE: I don't have a VMS machine myself so this is pure guesswork!!!
+
+    my $self = shift;
+    my @p = $self->path_components;
+    my $abs = 0;
+    if (@p && $p[0] eq '') {
+	shift @p;
+	$abs = 1;
+    }
+    # First I assume there must be a dollar in a disk spesification
+    my $p = '';
+    $p = uc(shift(@p)) . ":"  if @p && $p[0] =~ /\$/;
+    my $file = pop(@p);
+    $p .= "[" . join(".", map{uc($_)} @p) . "]" if @p;
+    $p .= uc $file;
+    # XXX: How is an absolute path different from a relative one??
+    $p =~ s/\[/[./ unless $abs;
+    # XXX: How is a directory denoted??
+    $p;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/finger.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/finger.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/finger.pm	(revision 21112)
@@ -0,0 +1,6 @@
+package URI::URL::finger;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+sub default_port { 79 }
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/ftp.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/ftp.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/ftp.pm	(revision 21112)
@@ -0,0 +1,76 @@
+package URI::URL::ftp;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+sub default_port { 21 }
+
+sub _parse {
+    my($self, $init) = @_;
+    # The ftp URLs can't have any query string
+    $self->URI::URL::_generic::_parse($init, qw(netloc path params frag));
+    1;
+}
+
+
+sub user
+{
+    my($self, @val) = @_;
+    my $old = $self->SUPER::user(@val);
+    defined $old ? $old : "anonymous";
+}
+
+BEGIN {
+    $whoami = undef;
+    $fqdn   = undef;
+}
+
+sub password
+{
+    my($self, @val) = @_;
+    my $old = $self->SUPER::password(@val);
+    unless (defined $old) {
+	my $user = $self->user;
+	if ($user eq 'anonymous' || $user eq 'ftp') {
+	    # anonymous ftp login password
+	    unless (defined $fqdn) {
+		eval {
+		    require Net::Domain;
+		    $fqdn = Net::Domain::hostfqdn();
+		};
+		if ($@) {
+		    $fqdn = '';
+		}
+	    }
+	    unless (defined $whoami) {
+		$whoami = $ENV{USER} || $ENV{LOGNAME} || $ENV{USERNAME};
+		unless ($whoami) {
+		    if ($^O eq 'MSWin32') { $whoami = Win32::LoginName() }
+		    else {
+		        $whoami = getlogin || getpwuid($<) || 'unknown';
+		    }
+		}
+	    }
+	    $old = "$whoami\@$fqdn";
+	} else {
+	    $old = "";
+	}
+    }
+    $old;
+}
+
+sub crack
+{
+    my $self = shift;
+    my @c = $self->SUPER::crack;
+    if ($c[3] && !$c[1]) {
+	# hostname defined, but user is undefined
+	$c[1] = $self->user;
+	$c[2] = $self->password;
+    }
+    @c;
+}
+
+*query  = \&URI::URL::bad_method;
+*equery = \&URI::URL::bad_method;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/gopher.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/gopher.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/gopher.pm	(revision 21112)
@@ -0,0 +1,82 @@
+package URI::URL::gopher;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+use URI::Escape qw(uri_unescape);
+
+sub default_port { 70 }
+
+sub _parse {
+    my($self, $init)   = @_;
+    $self->URI::URL::_generic::_parse($init, qw(netloc path));
+    $self->_parse_gopherpath;
+}
+
+sub path {
+    my $self = shift;
+    my $old = $self->URI::URL::_generic::path(@_);
+    return $old unless @_;
+    $self->_parse_gopherpath;
+    $old;
+}
+
+sub epath {
+    my $self = shift;
+    my $old = $self->URI::URL::_generic::epath(@_);
+    return $old unless @_;
+    $self->_parse_gopherpath;
+    $old;
+}
+
+sub _parse_gopherpath {
+    my $self = shift;
+    my $p = $self->{'path'};
+    # not according to RFC1738, but many popular browsers accept
+    # gopher URLs with a '?' before the search string.
+    $p =~ s/\?/\t/;
+    $p = uri_unescape($p);
+
+    if (defined($p) && $p ne '/' && $p =~ s!^/?(.)!!) {
+	$self->{'gtype'} = $1;
+    } else {
+	$self->{'gtype'} = "1";
+	$p = "";
+    }
+
+    delete $self->{'selector'};
+    delete $self->{'search'};
+    delete $self->{'string'};
+
+    my @parts = split(/\t/, $p, 3);
+    $self->{'selector'} = shift @parts if @parts;
+    $self->{'search'}   = shift @parts if @parts;
+    $self->{'string'}   = shift @parts if @parts;
+}
+
+
+sub gtype    { shift->_path_elem('gtype',    @_); }
+sub selector { shift->_path_elem('selector', @_); }
+sub search   { shift->_path_elem('search',   @_); }
+sub string   { shift->_path_elem('string',   @_); }
+
+sub _path_elem {
+    my($self, $elem, @val) = @_;
+    my $old = $self->_elem($elem, @val);
+    return $old unless @val;
+
+    # construct new path based on elements
+    my $path = "/$self->{'gtype'}";
+    $path .= "$self->{'selector'}" if defined $self->{'selector'};
+    $path .= "\t$self->{'search'}" if defined $self->{'search'};
+    $path .= "\t$self->{'string'}" if defined $self->{'string'};
+    $self->{'path'} = $path;
+
+    $old;
+}
+
+*params  = \&URI::URL::bad_method;
+*qparams = \&URI::URL::bad_method;
+*query   = \&URI::URL::bad_method;
+*equery  = \&URI::URL::bad_method;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/http.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/http.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/http.pm	(revision 21112)
@@ -0,0 +1,57 @@
+package URI::URL::http;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+sub default_port { 80 }
+
+require Carp;
+
+# @ISA = qw(AutoLoader)      # This comment is needed by AutoSplit.
+sub keywords;
+sub query_form;
+1;
+__END__
+
+# Handle ...?dog+bones type of query
+sub keywords {
+    my $self = shift;
+    $old = $self->{'query'};
+    if (@_) {
+	# Try to set query string
+	$self->equery(join('+', map { URI::Escape::uri_escape($_, $URI::URL::reserved) } @_));
+    }
+    return if !defined($old) || !defined(wantarray);
+
+    Carp::croak("Query is not keywords") if $old =~ /=/;
+    map { URI::Escape::uri_unescape($_) } split(/\+/, $old);
+}
+
+# Handle ...?foo=bar&bar=foo type of query
+sub query_form {
+    my $self = shift;
+    $old = $self->{'query'};
+    if (@_) {
+	# Try to set query string
+	my @query;
+	my($key,$vals);
+        my $esc = $URI::URL::reserved . $URI::URL::unsafe;
+	while (($key,$vals) = splice(@_, 0, 2)) {
+	    $key = '' unless defined $key;
+	    $key =  URI::Escape::uri_escape($key, $esc);
+	    $vals = [$vals] unless ref($vals) eq 'ARRAY';
+	    my $val;
+	    for $val (@$vals) {
+		$val = '' unless defined $val;
+		$val = URI::Escape::uri_escape($val, $esc);
+		push(@query, "$key=$val");
+	    }
+	}
+	$self->equery(join('&', @query));
+    }
+    return if !defined($old) || length($old) == 0 || !defined(wantarray);
+    Carp::croak("Query is not a form") unless $old =~ /=/;
+    map { s/\+/ /g; URI::Escape::uri_unescape($_) }
+	 map { /=/ ? split(/=/, $_, 2) : ($_ => '')} split(/&/, $old);
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/https.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/https.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/https.pm	(revision 21112)
@@ -0,0 +1,6 @@
+package URI::URL::https;
+require URI::URL::http;
+@ISA = qw(URI::URL::http);
+
+sub default_port { 443 }
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/mailto.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/mailto.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/mailto.pm	(revision 21112)
@@ -0,0 +1,78 @@
+package URI::URL::mailto;
+require URI::URL;
+@ISA = qw(URI::URL);
+
+use URI::Escape;
+
+sub new {
+    my($class, $init, $base) = @_;
+
+    my $self = bless { }, $class;
+    $self->{'scheme'} = lc($1) if $init =~ s/^\s*([\w\+\.\-]+)://;
+    $self->{'address'} = uri_unescape($init);
+    $self->base($base) if $base;
+    $self;
+}
+
+sub address { shift->_elem('address', @_); }
+
+# can use these as aliases
+*encoded822addr = \&address;   # URI::URL v3 compatibility
+*netloc         = \&address;
+
+sub user {
+    my $self = shift;
+    $old = $self->{'address'};
+    if (@_) {
+	my $new = $old;
+	$new =~ s/.*\@?/$_[0]\@/;
+	$self->{'address'} = $new;
+    }
+    $old =~ s/\@.*//;
+    $old;
+}
+
+sub host {
+    my $self = shift;
+    $old = $self->{'address'};
+    if (@_) {
+	my $new = $old;
+	$new =~ s/\@.*/\@$_[0]/;
+	$self->{'address'} = $new;
+    }
+    $old =~ s/.*\@//;
+    $old;
+}
+
+sub crack
+{
+    my $self = shift;
+    ('mailto',           # scheme
+     $self->user,        # user
+     undef,              # passwd
+     $self->host,        # host
+     undef,              # port
+     $self->{'address'}, # path
+     undef,              # params
+     undef,              # query
+     undef               # fragment
+    )
+}
+
+sub as_string {
+    my $self = shift;
+    my $str = ($self->{'scheme'} || "mailto") . ":";
+    $str .= uri_escape($self->{'address'}) if defined $self->{'address'};
+    $str;
+}
+
+sub eq {
+    my($self, $other) = @_;
+    $other = URI::URL->new($other) unless ref $other;
+
+    # Mail adresses are case-insensitive
+    $self->scheme eq $other->scheme &&
+      lc($self->{'address'}) eq lc($other->{'address'});
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/news.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/news.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/news.pm	(revision 21112)
@@ -0,0 +1,73 @@
+package URI::URL::news;
+require URI::URL;
+@ISA = qw(URI::URL);
+
+use URI::Escape;
+
+sub new {
+    my($class, $init, $base) = @_;
+    my $self = bless { }, $class;
+    $self->{'scheme'}  = lc($1) if $init =~ s/^\s*([\w\+\.\-]+)://;
+    $self->groupart(uri_unescape($init));
+    $self->base($base) if $base;
+    $self;
+}
+
+sub groupart {
+    my $self = shift;
+    my $old = $self->{'path'};
+    if (@_) {
+	my $p = shift;
+	if (defined $p && $p =~ /\@/) {
+	    # it is an message id
+	    $p =~ s/^<(.*)>$/$1/;  # "<" and ">" should not be part of it
+	}
+	$self->{'path'} = $p;
+    }
+    $old;
+}
+
+*path = \&groupart;
+
+sub article   {
+    my $self = shift;
+    Carp::croak("Illegal article id name (does not contain '\@')")
+      if @_ && $_[0] !~ /\@/;
+    my $old = $self->groupart(@_);
+    return undef if $old !~ /\@/;
+    $old;
+}
+
+
+sub group {
+    my $self = shift;
+    Carp::croak("Illegal group name (contains '\@')")
+      if @_ && $_[0] =~ /\@/;
+    my $old = $self->groupart(@_);
+    return undef if $old =~ /\@/;
+    $old;
+}
+
+sub crack
+{
+    my $self = shift;
+    ('news',             # scheme
+     undef,              # user
+     undef,              # passwd
+     undef,              # host
+     undef,              # port
+     $self->{'path'},    # path
+     undef,              # params
+     undef,              # query
+     undef               # fragment
+    )
+}
+
+
+sub as_string {
+    my $self = shift;
+    my $scheme = $self->{'scheme'} || "news";
+    "$scheme:" . uri_escape($self->{'path'});
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/nntp.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/nntp.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/nntp.pm	(revision 21112)
@@ -0,0 +1,44 @@
+package URI::URL::nntp;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+use URI::Escape;
+
+sub default_port { 119 }
+
+sub _parse {
+    my($self, $init) = @_;
+    $self->URI::URL::_generic::_parse($init, qw(netloc path frag));
+
+    my @parts = $self->path_components;
+    shift @parts if @parts && $parts[0] eq '';
+
+    $self->{'group'} = uri_unescape($parts[0]);
+    $self->{'digits'}= uri_unescape($parts[1]);
+}
+
+sub group   { shift->_elem('group',  @_); }
+sub digits  { shift->_elem('digits', @_); }
+sub article { shift->_elem('digits', @_); }
+
+sub as_string {
+    my $self = shift;
+    my $str = "$self->{'scheme'}:";
+    $str .= "//$self->{'netloc'}" if defined $self->{'netloc'};
+    $str .= "/" . uri_escape($self->{'group'}) . "/" .
+		  uri_escape($self->{'digits'});
+    $str;
+}
+
+# Standard methods are not legal for nntp URLs
+*path      = \&URI::URL::bad_method;
+*epath     = \&URI::URL::bad_method;
+*query     = \&URI::URL::bad_method;
+*equery    = \&URI::URL::bad_method;
+*params    = \&URI::URL::bad_method;
+*eparams   = \&URI::URL::bad_method;
+*frag      = \&URI::URL::bad_method;
+*user      = \&URI::URL::bad_method;
+*password  = \&URI::URL::bad_method;
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/prospero.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/prospero.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/prospero.pm	(revision 21112)
@@ -0,0 +1,27 @@
+package URI::URL::prospero;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+# RFC 1738 says:
+#
+#   A prospero URLs takes the form:
+#
+#      prospero://<host>:<port>/<hsoname>;<field>=<value>
+#
+#   where <host> and <port> are as described in Section 3.1. If :<port>
+#   is omitted, the port defaults to 1525. No username or password is
+#   allowed.
+#
+#   The <hsoname> is the host-specific object name in the Prospero
+#   protocol, suitably encoded.  This name is opaque and interpreted by
+#   the Prospero server.  The semicolon ";" is reserved and may not
+#   appear without quoting in the <hsoname>.
+
+sub default_port { 1525 }       # says rfc1738, section 3.11
+
+sub _parse {
+    my($self, $init) = @_;
+    $self->URI::URL::_generic::_parse($init, qw(netloc path params frag));
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/rlogin.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/rlogin.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/rlogin.pm	(revision 21112)
@@ -0,0 +1,7 @@
+package URI::URL::rlogin;
+require URI::URL::_login;
+@ISA = qw(URI::URL::_login);
+
+sub default_port { 513 }
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/telnet.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/telnet.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/telnet.pm	(revision 21112)
@@ -0,0 +1,7 @@
+package URI::URL::telnet;
+require URI::URL::_login;
+@ISA = qw(URI::URL::_login);
+
+sub default_port { 23 }
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/tn3270.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/tn3270.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/tn3270.pm	(revision 21112)
@@ -0,0 +1,7 @@
+package URI::URL::tn3270;
+require URI::URL::_login;
+@ISA = qw(URI::URL::_login);
+
+# sub default_port { 0 }
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/wais.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/wais.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/wais.pm	(revision 21112)
@@ -0,0 +1,50 @@
+package URI::URL::wais;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+use URI::Escape;
+
+# RFC 1738 says:
+#
+#   A WAIS URL takes one of the following forms:
+#
+#     wais://<host>:<port>/<database>
+#     wais://<host>:<port>/<database>?<search>
+#     wais://<host>:<port>/<database>/<wtype>/<wpath>
+#
+#   where <host> and <port> are as described in Section 3.1. If :<port>
+#   is omitted, the port defaults to 210.  The first form designates a
+#   WAIS database that is available for searching. The second form
+#   designates a particular search.  <database> is the name of the WAIS
+#   database being queried.
+
+
+sub default_port { 210 }
+
+sub _parse {
+    my($self, $init) = @_;
+    $self->URI::URL::_generic::_parse($init, qw(netloc path query frag));
+}
+
+# Set the path component with the specified number
+sub _path_comp
+{
+    my $self = shift;
+    my $no   = shift;
+    my @p = $self->path_components;
+    shift(@p) if @p && $p[0] eq '';
+    my $old = $p[$no];
+    if (@_) {
+	$p[$no] = $_[0];
+	$self->path_components(@p);
+    }
+    $old;
+}
+
+sub database { shift->_path_comp(0, @_); }
+sub wtype    { shift->_path_comp(1, @_); }
+sub wpath    { shift->_path_comp(2, @_); }
+
+*params   = \&URI::URL::bad_method;
+*eparams  = \&URI::URL::bad_method;
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/webster.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/webster.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/webster.pm	(revision 21112)
@@ -0,0 +1,6 @@
+package URI::URL::webster;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+sub default_port { 765 }
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/whois.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/whois.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/URI/URL/whois.pm	(revision 21112)
@@ -0,0 +1,6 @@
+package URI::URL::whois;
+require URI::URL::_generic;
+@ISA = qw(URI::URL::_generic);
+
+sub default_port { 43 }
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules.pm	(revision 21112)
@@ -0,0 +1,364 @@
+# $Id$
+
+package WWW::RobotRules;
+
+=head1 NAME
+
+WWW::RobotsRules - Parse robots.txt files
+
+=head1 SYNOPSIS
+
+ require WWW::RobotRules;
+ my $robotsrules = new WWW::RobotRules 'MOMspider/1.0';
+
+ use LWP::Simple qw(get);
+
+ $url = "http://some.place/robots.txt";
+ my $robots_txt = get $url;
+ $robotsrules->parse($url, $robots_txt);
+
+ $url = "http://some.other.place/robots.txt";
+ my $robots_txt = get $url;
+ $robotsrules->parse($url, $robots_txt);
+
+ # Now we are able to check if a URL is valid for those servers that
+ # we have obtained and parsed "robots.txt" files for.
+ if($robotsrules->allowed($url)) {
+     $c = get $url;
+     ...
+ }
+
+=head1 DESCRIPTION
+
+This module parses a F<robots.txt> file as specified in
+"A Standard for Robot Exclusion", described in
+<URL:http://info.webcrawler.com/mak/projects/robots/norobots.html>
+Webmasters can use the F<robots.txt> file to disallow conforming
+robots access to parts of their WWW server.
+
+The parsed file is kept in the WWW::RobotRules object, and this object
+provide methods to check if access to a given URL is prohibited.  The
+same WWW::RobotRules object can parse multiple F<robots.txt> files.
+
+The following methods are provided:
+
+=over 4
+
+=cut
+
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+sub Version { $VERSION; }
+
+
+use URI::URL ();
+use strict;
+
+
+=item $rules = new WWW::RobotRules 'MOMspider/1.0'
+
+This is the constructor for WWW::RobotRules objects.  The first 
+argument given to new() is the name of the robot. 
+
+=cut
+
+sub new {
+    my($class, $ua) = @_;
+
+    # This ugly hack is needed to ensure backwards compatability.
+    # The "WWW::RobotRules" class is now really abstract.
+    $class = "WWW::RobotRules::InCore" if $class eq "WWW::RobotRules";
+
+    my $self = bless { }, $class;
+    $self->agent($ua);
+    $self;
+}
+
+
+=item $rules->parse($url, $content, $fresh_until)
+
+The parse() method takes as arguments the URL that was used to
+retrieve the F</robots.txt> file, and the contents of the file.
+
+=cut
+
+sub parse {
+    my($self, $url, $txt, $fresh_until) = @_;
+
+    $url = new URI::URL $url unless ref($url);	# make it URL
+    my $netloc = $url->netloc;
+
+    $self->clear_rules($netloc);
+    $self->fresh_until($netloc, $fresh_until || (time + 365*24*3600));
+
+    my $ua;
+    my $is_me = 0;		# 1 iff this record is for me
+    my $is_anon = 0;		# 1 iff this record is for *
+    my @me_disallowed = ();	# rules disallowed for me
+    my @anon_disallowed = ();	# rules disallowed for *
+
+    # blank lines are significant, so turn CRLF into LF to avoid generating
+    # false ones
+    $txt =~ s/\015\012/\012/g;
+
+    # split at \012 (LF) or \015 (CR) (Mac text files have just CR for EOL)
+    for(split(/[\012\015]/, $txt)) {
+
+	# Lines containing only a comment are discarded completely, and
+        # therefore do not indicate a record boundary.
+	next if /^\s*\#/;
+
+	s/\s*\#.*//;        # remove comments at end-of-line
+
+	if (/^\s*$/) {	    # blank line
+	    last if $is_me; # That was our record. No need to read the rest.
+	    $is_anon = 0;
+	}
+        elsif (/^User-Agent:\s*(.*)/i) {
+	    $ua = $1;
+	    $ua =~ s/\s+$//;
+	    if ($is_me) {
+		# This record already had a User-agent that
+		# we matched, so just continue.
+	    }
+	    elsif ($ua eq '*') {
+		$is_anon = 1;
+	    }
+	    elsif($self->is_me($ua)) {
+		$is_me = 1;
+	    }
+	}
+	elsif (/^Disallow:\s*(.*)/i) {
+	    unless (defined $ua) {
+		warn "RobotRules: Disallow without preceding User-agent\n";
+		$is_anon = 1;  # assume that User-agent: * was intended
+	    }
+	    my $disallow = $1;
+	    $disallow =~ s/\s+$//;
+	    if (length $disallow) {
+		$disallow = URI::URL->new($disallow, $url)->full_path;
+	    }
+
+	    if ($is_me) {
+		push(@me_disallowed, $disallow);
+	    }
+	    elsif ($is_anon) {
+		push(@anon_disallowed, $disallow);
+	    }
+	}
+	else {
+	    warn "RobotRules: Unexpected line: $_\n";
+	}
+    }
+
+    if ($is_me) {
+	$self->push_rules($netloc, @me_disallowed);
+    } else {
+	$self->push_rules($netloc, @anon_disallowed);
+    }
+}
+
+# is_me()
+#
+# Returns TRUE if the given name matches the
+# name of this robot
+#
+sub is_me {
+    my($self, $ua) = @_;
+    my $me = $self->agent;
+    return index(lc($ua), lc($me)) >= 0;
+}
+
+=item $rules->allowed($url)
+
+Returns TRUE if this robot is allowed to retrieve this URL.
+
+=cut
+
+sub allowed {
+    my($self, $url) = @_;
+    $url = URI::URL->new($url) unless ref $url;	# make it URL
+
+    my $netloc = $url->netloc;
+
+    my $fresh_until = $self->fresh_until($netloc);
+    return -1 if !defined($fresh_until) || $fresh_until < time;
+
+    my $str = $url->full_path;
+    my $rule;
+    for $rule ($self->rules($netloc)) {
+	return 1 unless length $rule;
+	return 0 if index($str, $rule) == 0;
+    }
+    return 1;
+}
+
+# The following methods must be provided by the subclass.
+sub agent;
+sub visit;
+sub no_visits;
+sub last_visits;
+sub fresh_until;
+sub push_rules;
+sub clear_rules;
+sub rules;
+sub dump;
+
+package WWW::RobotRules::InCore;
+
+use vars qw(@ISA);
+@ISA = qw(WWW::RobotRules);
+
+=item $rules->agent([$name])
+
+Get/set the agent name. NOTE: Changing the agent name will clear the robots.txt
+rules and expire times out of the cache.
+
+=cut
+
+sub agent {
+    my ($self, $name) = @_;
+    my $old = $self->{'ua'};
+    if ($name) {
+	delete $self->{'loc'};   # all old info is now stale
+	$name =~ s!/?\s*\d+.\d+\s*$!!;  # loose version
+	$self->{'ua'}=$name;
+    }
+    $old;
+}
+
+sub visit {
+    my($self, $netloc, $time) = @_;
+    $time ||= time;
+    $self->{'loc'}{$netloc}{'last'} = $time;
+    
+    my $count = \$self->{'loc'}{$netloc}{'count'};
+    if (!defined $$count) {
+	$$count = 1;
+    } else {
+	$$count++;
+    }
+}
+
+sub no_visits {
+    my ($self, $netloc) = @_;
+    $self->{'loc'}{$netloc}{'count'};
+}
+
+sub last_visit {
+    my ($self, $netloc) = @_;
+    $self->{'loc'}{$netloc}{'last'};
+}
+
+sub fresh_until {
+    my ($self, $netloc, $fresh_until) = @_;
+    my $old = $self->{'loc'}{$netloc}{'fresh'};
+    if (defined $fresh_until) {
+	$self->{'loc'}{$netloc}{'fresh'} = $fresh_until;
+    }
+    $old;
+}
+
+sub push_rules {
+    my($self, $netloc, @rules) = @_;
+    push (@{$self->{'loc'}{$netloc}{'rules'}}, @rules);
+}
+
+sub clear_rules {
+    my($self, $netloc) = @_;
+    delete $self->{'loc'}{$netloc}{'rules'};
+}
+
+sub rules {
+    my($self, $netloc) = @_;
+    if (defined $self->{'loc'}{$netloc}{'rules'}) {
+	return @{$self->{'loc'}{$netloc}{'rules'}};
+    } else {
+	return ();
+    }
+}
+
+sub dump
+{
+    my $self = shift;
+    for (keys %$self) {
+	next if $_ eq 'loc';
+	print "$_ = $self->{$_}\n";
+    }
+    for (keys %{$self->{'loc'}}) {
+	my @rules = $self->rules($_);
+	print "$_: ", join("; ", @rules), "\n";
+	
+    }
+}
+
+1;
+
+__END__
+
+=back
+
+=head1 ROBOTS.TXT
+
+The format and semantics of the "/robots.txt" file are as follows
+(this is an edited abstract of
+<URL:http://info.webcrawler.com/mak/projects/robots/norobots.html>):
+
+The file consists of one or more records separated by one or more
+blank lines. Each record contains lines of the form
+
+  <field-name>: <value>
+
+The field name is case insensitive.  Text after the '#' character on a
+line is ignored during parsing.  This is used for comments.  The
+following <field-names> can be used:
+
+=over 3
+
+=item User-Agent
+
+The value of this field is the name of the robot the record is
+describing access policy for.  If more than one I<User-Agent> field is
+present the record describes an identical access policy for more than
+one robot. At least one field needs to be present per record.  If the
+value is '*', the record describes the default access policy for any
+robot that has not not matched any of the other records.
+
+=item Disallow
+
+The value of this field specifies a partial URL that is not to be
+visited. This can be a full path, or a partial path; any URL that
+starts with this value will not be retrieved
+
+=back
+
+=head1 ROBOTS.TXT EXAMPLES
+
+The following example "/robots.txt" file specifies that no robots
+should visit any URL starting with "/cyberworld/map/" or "/tmp/":
+
+  User-agent: *
+  Disallow: /cyberworld/map/ # This is an infinite virtual URL space
+  Disallow: /tmp/ # these will soon disappear
+
+This example "/robots.txt" file specifies that no robots should visit
+any URL starting with "/cyberworld/map/", except the robot called
+"cybermapper":
+
+  User-agent: *
+  Disallow: /cyberworld/map/ # This is an infinite virtual URL space
+
+  # Cybermapper knows where to go.
+  User-agent: cybermapper
+  Disallow:
+
+This example indicates that no robots should visit this site further:
+
+  # go away
+  User-agent: *
+  Disallow: /
+
+=head1 SEE ALSO
+
+L<LWP::RobotUA>, L<WWW::RobotRules::AnyDBM_File>
+
+=cut
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules/AnyDBM_File.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules/AnyDBM_File.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lib/WWW/RobotRules/AnyDBM_File.pm	(revision 21112)
@@ -0,0 +1,170 @@
+# $Id$
+
+package WWW::RobotRules::AnyDBM_File;
+
+require  WWW::RobotRules;
+@ISA = qw(WWW::RobotRules);
+$VERSION = sprintf("%d.%02d", q$Revision$ =~ /(\d+)\.(\d+)/);
+
+use Carp ();
+use AnyDBM_File;
+use Fcntl;
+use strict;
+
+=head1 NAME
+
+WWW::RobotRules::AnyDBM_File - Persistent RobotRules
+
+=head1 SYNOPSIS
+
+ require WWW::RobotRules::AnyDBM_File;
+ require LWP::RobotUA;
+
+ # Create a robot useragent that uses a diskcaching RobotRules
+ my $rules = new WWW::RobotRules::AnyDBM_File 'my-robot/1.0', 'cachefile';
+ my $ua = new WWW::RobotUA 'my-robot/1.0', 'me@foo.com', $rules;
+
+ # Then just use $ua as usual
+ $res = $ua->request($req);
+
+=head1 DESCRIPTION
+
+This is a subclass of I<WWW::RobotRules> that uses the AnyDBM_File
+package to implement persistent diskcaching of F<robots.txt> and host
+visit information.
+
+The constructor (the new() method) takes an extra argument specifying
+the name of the DBM file to use.  If the DBM file already exists, then
+you can specify undef as agent name as the name can be obtained from
+the DBM database.
+
+=cut
+
+sub new 
+{ 
+  my ($class, $ua, $file) = @_;
+  Carp::croak('WWW::RobotRules::AnyDBM_File filename required') unless $file;
+
+  my $self = bless { }, $class;
+  $self->{'filename'} = $file;
+  tie %{$self->{'dbm'}}, 'AnyDBM_File', $file, O_CREAT|O_RDWR, 0640
+    or Carp::croak("Can't open $file: $!");
+  
+  if ($ua) {
+      $self->agent($ua);
+  } else {
+      # Try to obtain name from DBM file
+      $ua = $self->{'dbm'}{"|ua-name|"};
+      Carp::croak("No agent name specified") unless $ua;
+  }
+
+  $self;
+}
+
+sub agent {
+    my($self, $newname) = @_;
+    my $old = $self->{'dbm'}{"|ua-name|"};
+    if (defined $newname) {
+	$newname =~ s!/?\s*\d+.\d+\s*$!!;  # loose version
+	unless ($old && $old eq $newname) {
+	# Old info is now stale.
+	    my $file = $self->{'filename'};
+	    untie %{$self->{'dbm'}};
+	    tie %{$self->{'dbm'}}, 'AnyDBM_File', $file, O_TRUNC|O_RDWR, 0640;
+	    $self->{'dbm'}{"|ua-name|"} = $newname;
+	}
+    }
+    $old;
+}
+
+sub no_visits {
+    my ($self, $netloc) = @_;
+    my $t = $self->{'dbm'}{"$netloc|vis"};
+    return 0 unless $t;
+    (split(/;\s*/, $t))[0];
+}
+
+sub last_visit {
+    my ($self, $netloc) = @_;
+    my $t = $self->{'dbm'}{"$netloc|vis"};
+    return undef unless $t;
+    (split(/;\s*/, $t))[1];
+}
+
+sub fresh_until {
+    my ($self, $netloc, $fresh) = @_;
+    my $old = $self->{'dbm'}{"$netloc|exp"};
+    if ($old) {
+	$old =~ s/;.*//;  # remove cleartext
+    }
+    if (defined $fresh) {
+	$fresh .= "; " . localtime($fresh);
+	$self->{'dbm'}{"$netloc|exp"} = $fresh;
+    }
+    $old;
+}
+
+sub visit {
+    my($self, $netloc, $time) = @_;
+    $time ||= time;
+
+    my $count = 0;
+    my $old = $self->{'dbm'}{"$netloc|vis"};
+    if ($old) {
+	my $last;
+	($count,$last) = split(/;\s*/, $old);
+	$time = $last if $last > $time;
+    }
+    $count++;
+    $self->{'dbm'}{"$netloc|vis"} = "$count; $time; " . localtime($time);
+}
+
+sub push_rules {
+    my($self, $netloc, @rules) = @_;
+    my $cnt = 1;
+    $cnt++ while $self->{'dbm'}{"$netloc|r$cnt"};
+
+    foreach (@rules) {
+	$self->{'dbm'}{"$netloc|r$cnt"} = $_;
+	$cnt++;
+    }
+}
+
+sub clear_rules {
+    my($self, $netloc) = @_;
+    my $cnt = 1;
+    while ($self->{'dbm'}{"$netloc|r$cnt"}) {
+	delete $self->{'dbm'}{"$netloc|r$cnt"};
+	$cnt++;
+    }
+}
+
+sub rules {
+    my($self, $netloc) = @_;
+    my @rules = ();
+    my $cnt = 1;
+    while (1) {
+	my $rule = $self->{'dbm'}{"$netloc|r$cnt"};
+	last unless $rule;
+	push(@rules, $rule);
+	$cnt++;
+    }
+    @rules;
+}
+
+sub dump
+{
+}
+
+1;
+
+=head1 SEE ALSO
+
+L<WWW::RobotRules>, L<LWP::RobotUA>
+
+=head1 AUTHORS
+
+Hakan Ardo E<lt>hakan@munin.ub2.lu.se>, Gisle Aas E<lt>aas@sn.no>
+
+=cut
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lwpcook.pod
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lwpcook.pod	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/lwpcook.pod	(revision 21112)
@@ -0,0 +1,343 @@
+=head1 NAME
+
+lwpcook - libwww-perl cookbook
+
+=head1 DESCRIPTION
+
+This document contain some examples that show typical usage of the
+libwww-perl library.  You should consult the documentation for the
+individual modules for more detail.
+
+All examples should be runnable programs. You can, in most cases, test
+the code sections by piping the program text directly to perl.
+
+
+
+=head1 GET
+
+It is very easy to use this library to just fetch documents from the
+net.  The LWP::Simple module provides the get() function that return
+the document specified by its URL argument:
+
+  use LWP::Simple;
+  $doc = get 'http://www.sn.no/libwww-perl/';
+
+or, as a perl one-liner using the getprint() function:
+
+  perl -MLWP::Simple -e 'getprint "http://www.sn.no/libwww-perl/"'
+
+or, how about fetching the latest perl by running this command:
+
+  perl -MLWP::Simple -e '
+    getstore "ftp://ftp.sunet.se/pub/lang/perl/CPAN/src/latest.tar.gz",
+             "perl.tar.gz"'
+
+You will probably first want to find a CPAN site closer to you by
+running something like the following command:
+
+  perl -MLWP::Simple -e 'getprint "http://www.perl.com/perl/CPAN/CPAN.html"'
+
+Enough of this simple stuff!  The LWP object oriented interface gives
+you more control over the request sent to the server.  Using this
+interface you have full control over headers sent and how you want to
+handle the response returned.
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+  $ua->agent("$0/0.1 " . $ua->agent);
+  # $ua->agent("Mozilla/8.0") # pretend we are very capable browser
+
+  $req = new HTTP::Request 'GET' => 'http://www.sn.no/libwww-perl';
+  $req->header('Accept' => 'text/html');
+
+  # send request
+  $res = $ua->request($req);
+
+  # check the outcome
+  if ($res->is_success) {
+     print $res->content;
+  } else {
+     print "Error: " . $res->status_line . "\n";
+  }
+
+The lwp-request program (alias GET) that is distributed with the
+library can also be used to fetch documents from WWW servers.
+  
+
+
+=head1 HEAD
+
+If you just want to check if a document is present (i.e. the URL is
+valid) try to run code that looks like this:
+
+  use LWP::Simple;
+
+  if (head($url)) {
+     # ok document exists
+  }
+
+The head() function really returns a list of meta-information about
+the document.  The first three values of the list returned are the
+document type, the size of the document, and the age of the document.
+
+More control over the request or access to all header values returned
+require that you use the object oriented interface described for GET
+above.  Just s/GET/HEAD/g.
+
+
+=head1 POST
+
+There is no simple procedural interface for posting data to a WWW server.  You
+must use the object oriented interface for this. The most common POST
+operation is to access a WWW form application:
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+
+  my $req = new HTTP::Request 'POST','http://www.perl.com/cgi-bin/BugGlimpse';
+  $req->content_type('application/x-www-form-urlencoded');
+  $req->content('match=www&errors=0');
+
+  my $res = $ua->request($req);
+  print $res->as_string;
+
+Lazy people use the HTTP::Request::Common module to set up a suitable
+POST request message (it handles all the escaping issues) and has a
+suitable default for the content_type:
+
+  use HTTP::Request::Common qw(POST);
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+
+  my $req = POST 'http://www.perl.com/cgi-bin/BugGlimpse',
+                [ search => 'www', errors => 0 ];
+
+  print $ua->request($req)->as_string;
+
+The lwp-request program (alias POST) that is distributed with the
+library can also be used for posting data.
+
+
+
+=head1 PROXIES
+
+Some sites use proxies to go through fire wall machines, or just as
+cache in order to improve performance.  Proxies can also be used for
+accessing resources through protocols not supported directly (or
+supported badly :-) by the libwww-perl library.
+
+You should initialize your proxy setting before you start sending
+requests:
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+  $ua->env_proxy; # initialize from environment variables
+  # or
+  $ua->proxy(ftp  => 'http://proxy.myorg.com');
+  $ua->proxy(wais => 'http://proxy.myorg.com');
+  $ua->no_proxy(qw(no se fi));
+
+  my $req = new HTTP::Request 'wais://xxx.com/';
+  print $ua->request($req)->as_string;
+
+The LWP::Simple interface will call env_proxy() for you automatically.
+Applications that use the $ua->env_proxy() method will normally not
+use the $ua->proxy() and $ua->no_proxy() methods.
+
+Some proxies also require that you send it a username/password in
+order to let requests through.  You should be able to add the
+required header, with something like this:
+
+ use LWP::UserAgent;
+
+ $ua = new LWP::UserAgent;
+ $ua->proxy(['http', 'ftp'] => 'http://proxy.myorg.com');
+
+ $req = new HTTP::Request 'GET',"http://www.perl.com";
+ $req->proxy_authorization_basic("proxy_user", "proxy_password");
+
+ $res = $ua->request($req);
+ print $res->content if $res->is_success;
+
+Replace C<proxy.myorg.com>, C<proxy_user> and
+C<proxy_password> with something suitable for your site.
+
+
+=head1 ACCESS TO PROTECTED DOCUMENTS
+
+Documents protected by basic authorization can easily be accessed
+like this:
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+  $req = new HTTP::Request GET => 'http://www.sn.no/secret/';
+  $req->authorization_basic('aas', 'mypassword');
+  print $ua->request($req)->as_string;
+
+The other alternative is to provide a subclass of I<LWP::UserAgent> that
+overrides the get_basic_credentials() method. Study the I<lwp-request>
+program for an example of this.
+
+
+=head1 MIRRORING
+
+If you want to mirror documents from a WWW server, then try to run
+code similar to this at regular intervals:
+
+  use LWP::Simple;
+
+  %mirrors = (
+     'http://www.sn.no/'             => 'sn.html',
+     'http://www.perl.com/'          => 'perl.html',
+     'http://www.sn.no/libwww-perl/' => 'lwp.html',
+     'gopher://gopher.sn.no/'        => 'gopher.html',
+  );
+
+  while (($url, $localfile) = each(%mirrors)) {
+     mirror($url, $localfile);
+  }
+
+Or, as a perl one-liner:
+
+  perl -MLWP::Simple -e 'mirror("http://www.perl.com/", "perl.html")';
+
+The document will not be transfered unless it has been updated.
+
+
+
+=head1 LARGE DOCUMENTS
+
+If the document you want to fetch is too large to be kept in memory,
+then you have two alternatives.  You can instruct the library to write
+the document content to a file (second $ua->request() argument is a file
+name):
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+
+  my $req = new HTTP::Request 'GET',
+                'http://www.sn.no/~aas/perl/www/libwww-perl-5.00.tar.gz';
+  $res = $ua->request($req, "libwww-perl.tar.gz");
+  if ($res->is_success) {
+     print "ok\n";
+  }
+
+Or you can process the document as it arrives (second $ua->request()
+argument is a code reference):
+
+  use LWP::UserAgent;
+  $ua = new LWP::UserAgent;
+  $URL = 'ftp://ftp.unit.no/pub/rfc/rfc-index.txt';
+
+  my $expected_length;
+  my $bytes_received = 0;
+  $ua->request(HTTP::Request->new('GET', $URL),
+               sub {
+                   my($chunk, $res) = @_;
+                   $bytes_received += length($chunk);
+	           unless (defined $expected_length) {
+	              $expected_length = $res->content_length || 0;
+                   }
+		   if ($expected_length) {
+		        printf STDERR "%d%% - ",
+	                          100 * $bytes_received / $expected_length;
+                   }
+	           print STDERR "$bytes_received bytes received\n";
+
+                   # XXX Should really do something with the chunk itself
+	           # print $chunk;
+               });
+
+
+
+=head1 HTML FORMATTING
+
+It is easy to convert HTML code to "readable" text.
+
+  use LWP::Simple;
+  use HTML::Parse;
+  print parse_html(get 'http://www.sn.no/libwww-perl/')->format;
+
+
+
+=head1 PARSE URLS
+
+To access individual elements of a URL, try this:
+
+  use URI::URL;
+  $host = url("http://www.sn.no/")->host;
+
+or
+
+  use URI::URL;
+  $u = url("ftp://ftp.sn.no/test/aas;type=i");
+  print "Protocol scheme is ", $u->scheme, "\n";
+  print "Host is ", $u->host, " at port ", $u->port, "\n";
+
+or even
+
+  use URI::URL;
+  my($host,$port) = (url("ftp://ftp.sn.no/test/aas;type=i")->crack)[3,4];
+
+
+=head1 EXPAND RELATIVE URLS
+
+This code reads URLs and print expanded version.
+
+  use URI::URL;
+  $BASE = "http://www.sn.no/some/place?query";
+  while (<>) {
+     print url($_, $BASE)->abs->as_string, "\n";
+  }
+
+We can expand URLs in an HTML document by using the parser to build a
+tree that we then traverse:
+
+  %link_elements =
+  (
+   'a'    => 'href',
+   'img'  => 'src',
+   'form' => 'action',
+   'link' => 'href',
+  );
+
+  use HTML::Parse;
+  use URI::URL;
+
+  $BASE = "http://somewhere/root/";
+  $h = parse_htmlfile("xxx.html");
+  $h->traverse(\&expand_urls, 1);
+
+  print $h->as_HTML;
+
+  sub expand_urls
+  {
+     my($e, $start) = @_;
+     return 1 unless $start;
+     my $attr = $link_elements{$e->tag};
+     return 1 unless defined $attr;
+     my $url = $e->attr($attr);
+     return 1 unless defined $url;
+     $e->attr($attr, url($url, $BASE)->abs->as_string);
+  }
+
+
+
+=head1 BASE URL
+
+If you want to resolve relative links in a page you will have to
+determine which base URL to use.  The HTTP::Response objects now has a
+base() method.
+
+  $BASE = $res->base;
+
+
+
+=head1 COPYRIGHT
+
+Copyright 1996-1997, Gisle Aas
+
+This library is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/README
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/README	(revision 21112)
@@ -0,0 +1,24 @@
+Self test suite for the libwww-perl library
+-------------------------------------------
+
+Tests are invoked by running the ./TEST script, but usually you run
+the tests with "make test" at the top libwww-perl directory.  Use -v
+option for verbose tests.  You might run an individual test like this:
+
+  ./TEST -v base/date
+
+or all tests in a directory like this
+
+  ./TEST base
+
+You enable network tests by creating the "net/config.pl" file.  A good
+start is to make a copy of the "net/config.pl.dist" file.  For network
+tests you should also make a link from the cgi-bin directory of your
+http server to the "net/cgi-bin" directory.  You might also have to
+check that the interpreter line (#!.../perl) in the scripts is ok for
+your system.  The following setup works for my site:
+
+  ln -s `pwd`/net/cgi-bin ~www/cgi-bin/lwp
+  cp net/config.pl.dist net/config.pl
+  emacs net/config.pl                        # fix if necessary
+  ./TEST net
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/TEST
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/TEST	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/TEST	(revision 21112)
@@ -0,0 +1,39 @@
+#!/usr/local/bin/perl
+
+# This script is run Test::Harness on the tests found under the
+# "t" directory.
+
+# First we check if we already are within the "t" directory
+unless (-d "base") {
+    # try to move into test directory
+    chdir "t" or die "Can't chdir: $!";
+
+    # fix all relative library locations
+    foreach (@INC) {
+	$_ = "../$_" unless m,^/,;
+    }
+}
+# Pick up the library files from the ../blib directory
+unshift(@INC, "../blib/lib", "../blib/arch");
+#print "@INC\n";
+
+
+use Test::Harness;
+$Test::Harness::verbose = shift
+  if $ARGV[0] =~ /^\d+$/ || $ARGV[0] eq "-v";
+
+if (@ARGV) {
+    for (@ARGV) {
+	if (-d $_) {
+	    push(@tests, <$_/*.t>);
+	} else {
+            $_ .= ".t" unless /\.t$/;
+	    push(@tests, $_);
+	}
+    }
+} else {
+    @tests = (<base/*.t>, <uri/*.t>, <robot/*.t>, <local/*.t>);
+    push(@tests, <net/*.t>) if -f "net/config.pl";
+}
+
+runtests @tests;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/common-req.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/common-req.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/common-req.t	(revision 21112)
@@ -0,0 +1,193 @@
+print "1..19\n";
+
+use HTTP::Request::Common;
+
+$r = GET 'http://www.sn.no/';
+print $r->as_string;
+
+print "not " unless $r->method eq "GET" and $r->url eq "http://www.sn.no/";
+print "ok 1\n";
+
+$r = HEAD "http://www.sn.no/",
+     If_Match => 'abc',
+     From => 'aas@sn.no';
+print $r->as_string;
+
+print "not " unless $r->method eq "HEAD" and $r->url->eq("http://www.sn.no");
+print "ok 2\n";
+
+print "not " unless $r->header('If-Match') eq "abc" and $r->header("from") eq "aas\@sn.no";
+print "ok 3\n";
+
+$r = PUT "http://www.sn.no",
+     Content => 'foo';
+print $r->as_string;
+
+print "not " unless $r->method eq "PUT" and $r->url->netloc eq "www.sn.no";
+print "ok 4\n";
+
+print "not " if defined($r->header("Content"));
+print "ok 5\n";
+
+print "not " unless ${$r->content_ref} eq "foo" and
+                    $r->content eq "foo";
+print "ok 6\n";
+
+#--- Test POST requests ---
+
+$r = POST "http://www.sn.no", [foo => 'bar',
+                               baz => [qw(a b c)],
+                               foo => 'zoo=&',
+                               "space " => " + ",
+                              ],
+                              bar => 'foo';
+print $r->as_string;
+
+print "not " unless $r->method eq "POST" and
+                    $r->content_type eq "application/x-www-form-urlencoded" and
+                    $r->content_length == 58 and
+                    $r->header("bar") eq "foo";
+print "ok 7\n";
+
+print "not " unless $r->content eq "foo=bar&baz=a&baz=b&baz=c&foo=zoo%3D%26&space%20=%20%2B%20";
+print "ok 8\n";
+
+$r = POST "mailto:gisle\@aas.no",
+     Subject => "Heisan",
+     Content_Type => "text/plain",
+     Content => "Howdy\n";
+print $r->as_string;
+
+print "not " unless $r->method eq "POST" and
+                    $r->header("Subject") eq "Heisan" and
+                    $r->content eq "Howdy\n" and
+	            $r->content_type eq "text/plain";
+print "ok 9\n";
+
+#
+# POST for File upload
+#
+$file = "test-$$";
+open(FILE, ">$file") or die "Can't create $file: $!";
+print FILE "foo\nbar\nbaz\n";
+close(FILE);
+
+$r = POST 'http://www.perl.org/survey.cgi',
+       Content_Type => 'form-data',
+       Content      => [ name  => 'Gisle Aas',
+                         email => 'gisle@aas.no',
+                         gender => 'm',
+                         born   => '1964',
+                         file   => [$file],
+                       ];
+print $r->as_string;
+
+unlink($file) or warn "Can't unlink $file: $!";
+
+print "not " unless $r->method eq "POST" and
+	            $r->url->path eq "/survey.cgi" and
+                    $r->content_type eq "multipart/form-data" and
+	            $r->header(Content_type) =~ /boundary="?([^"]+)"?/;
+print "ok 10\n";
+$boundary = $1;
+
+$c = $r->content;
+$c =~ s/\r//g;
+@c = split(/--\Q$boundary/, $c);
+print "$c[5]\n";
+
+print "not " unless @c == 7 and $c[6] =~ /^--\n/;  # 5 parts + header & trailer
+print "ok 11\n";
+
+print "not " unless $c[2] =~ /^Content-Disposition:\s*form-data;\s*name="email"/m and
+                    $c[2] =~ /^gisle\@aas.no$/m;
+print "ok 12\n";
+
+print "not " unless $c[5] =~ /^Content-Disposition:\s*form-data;\s*name="file";\s*filename="$file"/m and
+	            $c[5] =~ /^Content-Type:\s*text\/plain$/m and
+	            $c[5] =~ /^foo\nbar\nbaz/m;
+print "ok 13\n";
+
+$r = POST 'http://www.perl.org/survey.cgi',
+      [ file => [ undef, "xxx", Content_type => "text/html", Content => "<h1>Hello, world!</h1>" ]],
+      Content_type => 'multipart/form-data';
+print $r->as_string;
+
+print "not " unless $r->content =~ /^--\S+\015\012Content-Disposition:\s*form-data;\s*name="file";\s*filename="xxx"/m and
+	            $r->content =~ /^Content-Type: text\/html/m and
+	            $r->content =~ /^<h1>Hello, world/m;
+print "ok 14\n";
+
+
+$r = POST 'http://www.perl.org/survey.cgi',
+      Content_type => 'multipart/form-data',
+      Content => [ file => [ undef, undef, Content => "foo"]];
+print $r->as_string;
+
+print "not " if $r->content =~ /filename=/;
+print "ok 15\n";
+
+
+# The POST routine can now also take a hash reference.
+my %hash = (foo => 42, bar => 24);
+$r = POST 'http://www.perl.org/survey.cgi', \%hash;
+print $r->as_string;
+print "not " unless $r->content =~ /foo=42/ &&
+                    $r->content =~ /bar=24/ &&
+                    $r->content_type eq "application/x-www-form-urlencoded" &&
+                    $r->content_length == 13;
+print "ok 16\n";
+
+ 
+#
+# POST for File upload
+#
+use HTTP::Request::Common qw($DYNAMIC_FILE_UPLOAD);
+
+$file = "test-$$";
+open(FILE, ">$file") or die "Can't create $file: $!";
+for (1..1000) {
+   print FILE "a" .. "z";
+}
+close(FILE);
+
+$DYNAMIC_FILE_UPLOAD++;
+$r = POST 'http://www.perl.org/survey.cgi',
+       Content_Type => 'form-data',
+       Content      => [ name  => 'Gisle Aas',
+                         email => 'gisle@aas.no',
+                         gender => 'm',
+                         born   => '1964',
+                         file   => [$file],
+                       ];
+print $r->as_string;
+
+print "not " unless $r->method eq "POST" and
+	            $r->url->path eq "/survey.cgi" and
+                    $r->content_type eq "multipart/form-data" and
+	            $r->header(Content_type) =~ /boundary="?([^"]+)"?/ and
+		    ref($r->content) eq "CODE";
+print "ok 17\n";
+$boundary = $1;
+
+print "not " unless length($boundary) > 10;
+print "ok 18\n";
+
+$code = $r->content;
+my $chunk;
+my @chunks;
+while (defined($chunk = &$code) && length $chunk) {
+   push(@chunks, $chunk);
+}
+
+unlink($file) or warn "Can't unlink $file: $!";
+
+$_ = join("", @chunks);
+
+print int(@chunks), " chunks, total size is ", length($_), " bytes\n";
+
+# should be close to expected size and number of chunks
+print "not " unless abs(@chunks - 15 < 3) and
+                    abs(length($_) - 26589) < 20;
+print "ok 19\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/cookies.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/cookies.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/cookies.t	(revision 21112)
@@ -0,0 +1,515 @@
+print "1..32\n";
+
+#use LWP::Debug '+';
+
+use HTTP::Cookies;
+use HTTP::Request;
+use HTTP::Response;
+
+#-------------------------------------------------------------------
+# First we check that it works for the original example at
+# http://www.netscape.com/newsref/std/cookie_spec.html
+
+# Client requests a document, and receives in the response:
+# 
+#       Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT
+# 
+# When client requests a URL in path "/" on this server, it sends:
+# 
+#       Cookie: CUSTOMER=WILE_E_COYOTE
+# 
+# Client requests a document, and receives in the response:
+# 
+#       Set-Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001; path=/
+# 
+# When client requests a URL in path "/" on this server, it sends:
+# 
+#       Cookie: CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001
+# 
+# Client receives:
+# 
+#       Set-Cookie: SHIPPING=FEDEX; path=/fo
+# 
+# When client requests a URL in path "/" on this server, it sends:
+# 
+#       Cookie: CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001
+# 
+# When client requests a URL in path "/foo" on this server, it sends:
+# 
+#       Cookie: CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001; SHIPPING=FEDEX
+# 
+# The last Cookie is buggy, because both specifications says that the
+# most specific cookie must be sent first.  SHIPPING=FEDEX is the
+# most specific and should thus be first.
+
+$c = HTTP::Cookies->new;
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/");
+
+$res = HTTP::Response->new(200, "OK");
+$res->request($req);
+$res->header("Set-Cookie" => "CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT");
+$c->extract_cookies($res);
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/");
+$c->add_cookie_header($req);
+
+print "not " unless $req->header("Cookie") eq "CUSTOMER=WILE_E_COYOTE" &&
+                    $req->header("Cookie2") eq "\$Version=1";
+print "ok 1\n";
+
+$res->request($req);
+$res->header("Set-Cookie" => "PART_NUMBER=ROCKET_LAUNCHER_0001; path=/");
+$c->extract_cookies($res);
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/foo/bar");
+$c->add_cookie_header($req);
+
+$h = $req->header("Cookie");
+print "not " unless $h =~ /PART_NUMBER=ROCKET_LAUNCHER_0001/ &&
+                    $h =~ /CUSTOMER=WILE_E_COYOTE/;
+print "ok 2\n";
+
+$res->request($req);
+$res->header("Set-Cookie", "SHIPPING=FEDEX; path=/foo");
+$c->extract_cookies($res);
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/");
+$c->add_cookie_header($req);
+
+$h = $req->header("Cookie");
+print "not " unless $h =~ /PART_NUMBER=ROCKET_LAUNCHER_0001/ &&
+	            $h =~ /CUSTOMER=WILE_E_COYOTE/ &&
+	            $h !~ /SHIPPING=FEDEX/;
+print "ok 3\n";
+
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/foo/");
+$c->add_cookie_header($req);
+
+$h = $req->header("Cookie");
+print "not " unless $h =~ /PART_NUMBER=ROCKET_LAUNCHER_0001/ &&
+	            $h =~ /CUSTOMER=WILE_E_COYOTE/ &&
+		    $h =~ /^SHIPPING=FEDEX;/;
+print "ok 4\n";
+
+print $c->as_string;
+
+
+# Second Example transaction sequence:
+# 
+# Assume all mappings from above have been cleared.
+# 
+# Client receives:
+# 
+#       Set-Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001; path=/
+# 
+# When client requests a URL in path "/" on this server, it sends:
+# 
+#       Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001
+# 
+# Client receives:
+# 
+#       Set-Cookie: PART_NUMBER=RIDING_ROCKET_0023; path=/ammo
+# 
+# When client requests a URL in path "/ammo" on this server, it sends:
+# 
+#       Cookie: PART_NUMBER=RIDING_ROCKET_0023; PART_NUMBER=ROCKET_LAUNCHER_0001
+# 
+#       NOTE: There are two name/value pairs named "PART_NUMBER" due to
+#       the inheritance of the "/" mapping in addition to the "/ammo" mapping. 
+
+$c = HTTP::Cookies->new;  # clear it
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/");
+$res = HTTP::Response->new(200, "OK");
+$res->request($req);
+$res->header("Set-Cookie", "PART_NUMBER=ROCKET_LAUNCHER_0001; path=/");
+
+$c->extract_cookies($res);
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/");
+$c->add_cookie_header($req);
+
+print "not " unless $req->header("Cookie") eq "PART_NUMBER=ROCKET_LAUNCHER_0001";
+print "ok 5\n";
+
+$res->request($req);
+$res->header("Set-Cookie", "PART_NUMBER=RIDING_ROCKET_0023; path=/ammo");
+$c->extract_cookies($res);
+
+$req = HTTP::Request->new(GET => "http://www.acme.com/ammo");
+$c->add_cookie_header($req);
+
+print "not " unless $req->header("Cookie") =~
+       /^PART_NUMBER=RIDING_ROCKET_0023;\s*PART_NUMBER=ROCKET_LAUNCHER_0001/;
+print "ok 6\n";
+
+print $c->as_string;
+undef($c);
+
+
+#-------------------------------------------------------------------
+# When there are no "Set-Cookie" header, then even responses
+# without any request URLs connected should be allowed.
+
+$c = HTTP::Cookies->new;
+$c->extract_cookies(HTTP::Response->new("200", "OK"));
+print "not " if count_cookies($c) != 0;
+print "ok 7\n";
+
+
+#-------------------------------------------------------------------
+# Then we test with the examples from draft-ietf-http-state-man-mec-03.txt
+#
+# 5.  EXAMPLES
+
+$c = HTTP::Cookies->new;
+
+# 
+# 5.1  Example 1
+# 
+# Most detail of request and response headers has been omitted.  Assume
+# the user agent has no stored cookies.
+# 
+#   1.  User Agent -> Server
+# 
+#       POST /acme/login HTTP/1.1
+#       [form data]
+# 
+#       User identifies self via a form.
+# 
+#   2.  Server -> User Agent
+# 
+#       HTTP/1.1 200 OK
+#       Set-Cookie2: Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"
+# 
+#       Cookie reflects user's identity.
+
+$cookie = interact($c, 'http://www.acme.com/acme/login',
+                       'Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"');
+print "not " if $cookie;
+print "ok 8\n";
+
+# 
+#   3.  User Agent -> Server
+# 
+#       POST /acme/pickitem HTTP/1.1
+#       Cookie: $Version="1"; Customer="WILE_E_COYOTE"; $Path="/acme"
+#       [form data]
+# 
+#       User selects an item for ``shopping basket.''
+# 
+#   4.  Server -> User Agent
+# 
+#       HTTP/1.1 200 OK
+#       Set-Cookie2: Part_Number="Rocket_Launcher_0001"; Version="1";
+#               Path="/acme"
+# 
+#       Shopping basket contains an item.
+
+$cookie = interact($c, 'http://www.acme.com/acme/pickitem',
+		       'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"');
+print "not " unless $cookie =~ m(^\$Version="?1"?; Customer="?WILE_E_COYOTE"?; \$Path="/acme"$);
+print "ok 9\n";
+
+# 
+#   5.  User Agent -> Server
+# 
+#       POST /acme/shipping HTTP/1.1
+#       Cookie: $Version="1";
+#               Customer="WILE_E_COYOTE"; $Path="/acme";
+#               Part_Number="Rocket_Launcher_0001"; $Path="/acme"
+#       [form data]
+# 
+#       User selects shipping method from form.
+# 
+#   6.  Server -> User Agent
+# 
+#       HTTP/1.1 200 OK
+#       Set-Cookie2: Shipping="FedEx"; Version="1"; Path="/acme"
+# 
+#       New cookie reflects shipping method.
+
+$cookie = interact($c, "http://www.acme.com/acme/shipping",
+		   'Shipping="FedEx"; Version="1"; Path="/acme"');
+
+print "not " unless $cookie =~ /^\$Version="?1"?;/ &&
+     $cookie =~ /Part_Number="?Rocket_Launcher_0001"?;\s*\$Path="\/acme"/ &&
+     $cookie =~ /Customer="?WILE_E_COYOTE"?;\s*\$Path="\/acme"/;
+print "ok 10\n";
+
+# 
+#   7.  User Agent -> Server
+# 
+#       POST /acme/process HTTP/1.1
+#       Cookie: $Version="1";
+#               Customer="WILE_E_COYOTE"; $Path="/acme";
+#               Part_Number="Rocket_Launcher_0001"; $Path="/acme";
+#               Shipping="FedEx"; $Path="/acme"
+#       [form data]
+# 
+#       User chooses to process order.
+# 
+#   8.  Server -> User Agent
+# 
+#       HTTP/1.1 200 OK
+# 
+#       Transaction is complete.
+
+$cookie = interact($c, "http://www.acme.com/acme/process");
+print "FINAL COOKIE: $cookie\n";
+print "not " unless $cookie =~ /Shipping="?FedEx"?;\s*\$Path="\/acme"/ &&
+                    $cookie =~ /WILE_E_COYOTE/;
+print "ok 11\n";
+
+# 
+# The user agent makes a series of requests on the origin server, after
+# each of which it receives a new cookie.  All the cookies have the same
+# Path attribute and (default) domain.  Because the request URLs all have
+# /acme as a prefix, and that matches the Path attribute, each request
+# contains all the cookies received so far.
+
+print $c->as_string;
+
+
+# 5.2  Example 2
+# 
+# This example illustrates the effect of the Path attribute.  All detail
+# of request and response headers has been omitted.  Assume the user agent
+# has no stored cookies.
+
+$c = HTTP::Cookies->new;
+
+# Imagine the user agent has received, in response to earlier requests,
+# the response headers
+# 
+# Set-Cookie2: Part_Number="Rocket_Launcher_0001"; Version="1";
+#         Path="/acme"
+# 
+# and
+# 
+# Set-Cookie2: Part_Number="Riding_Rocket_0023"; Version="1";
+#         Path="/acme/ammo"
+
+interact($c, "http://www.acme.com/acme/ammo/specific",
+             'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"',
+             'Part_Number="Riding_Rocket_0023"; Version="1"; Path="/acme/ammo"');
+
+# A subsequent request by the user agent to the (same) server for URLs of
+# the form /acme/ammo/...  would include the following request header:
+# 
+# Cookie: $Version="1";
+#         Part_Number="Riding_Rocket_0023"; $Path="/acme/ammo";
+#         Part_Number="Rocket_Launcher_0001"; $Path="/acme"
+# 
+# Note that the NAME=VALUE pair for the cookie with the more specific Path
+# attribute, /acme/ammo, comes before the one with the less specific Path
+# attribute, /acme.  Further note that the same cookie name appears more
+# than once.
+
+$cookie = interact($c, "http://www.acme.com/acme/ammo/...");
+print "not " unless $cookie =~ /Riding_Rocket_0023.*Rocket_Launcher_0001/;
+print "ok 12\n";
+
+# A subsequent request by the user agent to the (same) server for a URL of
+# the form /acme/parts/ would include the following request header:
+# 
+# Cookie: $Version="1"; Part_Number="Rocket_Launcher_0001"; $Path="/acme"
+# 
+# Here, the second cookie's Path attribute /acme/ammo is not a prefix of
+# the request URL, /acme/parts/, so the cookie does not get forwarded to
+# the server.
+
+$cookie = interact($c, "http://www.acme.com/acme/parts/");
+print "not " unless $cookie =~ /Rocket_Launcher_0001/ &&
+		    $cookie !~ /Riding_Rocket_0023/;
+print "ok 13\n";
+
+print $c->as_string;
+
+#-----------------------------------------------------------------------
+
+# Test rejection of Set-Cookie2 responses based on domain, path or port
+
+$c = HTTP::Cookies->new;
+
+# illegal domain (no embedded dots)
+$cookie = interact($c, "http://www.acme.com", 'foo=bar; domain=".com"');
+print "not " if count_cookies($c) > 0;
+print "ok 14\n";
+
+# legal domain
+$cookie = interact($c, "http://www.acme.com", 'foo=bar; domain="acme.com"');
+print "not " if count_cookies($c) != 1;
+print "ok 15\n";
+
+# illegal domain (host prefix "www.a" contains a dot)
+$cookie = interact($c, "http://www.a.acme.com", 'foo=bar; domain="acme.com"');
+print "not " if count_cookies($c) != 1;
+print "ok 16\n";
+
+# legal domain
+$cookie = interact($c, "http://www.a.acme.com", 'foo=bar; domain=".a.acme.com"');
+print "not " if count_cookies($c) != 2;
+print "ok 17\n";
+
+# can't use a IP-address as domain
+$cookie = interact($c, "http://125.125.125.125", 'foo=bar; domain="125.125.125"');
+print "not " if count_cookies($c) != 2;
+print "ok 18\n";
+
+# illegal path (must be prefix of request path)
+$cookie = interact($c, "http://www.sol.no", 'foo=bar; domain=".sol.no"; path="/foo"');
+print "not " if count_cookies($c) != 2;
+print "ok 19\n";
+
+# legal path
+$cookie = interact($c, "http://www.sol.no/foo/bar", 'foo=bar; domain=".sol.no"; path="/foo"');
+print "not " if count_cookies($c) != 3;
+print "ok 20\n";
+
+# illegal port (request-port not in list)
+$cookie = interact($c, "http://www.sol.no", 'foo=bar; domain=".sol.no"; port="90,100"');
+print "not " if count_cookies($c) != 3;
+print "ok 21\n";
+
+# legal port
+$cookie = interact($c, "http://www.sol.no", 'foo=bar; domain=".sol.no"; port="90,100, 80,8080"; max-age=100; Comment = "Just kidding! (\"|\\\\) "');
+print "not " if count_cookies($c) != 4;
+print "ok 22\n";
+
+# port attribute without any value (current port)
+$cookie = interact($c, "http://www.sol.no", 'foo9=bar; domain=".sol.no"; port; max-age=100;');
+print "not " if count_cookies($c) != 5;
+print "ok 23\n";
+
+# encoded path
+$cookie = interact($c, "http://www.sol.no/foo/", 'foo8=bar; path="/%66oo"');
+print "not " if count_cookies($c) != 6;
+print "ok 24\n";
+
+my $file = "lwp-cookies-$$.txt";
+$c->save($file);
+$old = $c->as_string;
+undef($c);
+
+$c = HTTP::Cookies->new;
+$c->load($file);
+unlink($file) || warn "Can't unlink $file: $!";
+
+print "not " unless $old eq $c->as_string;
+print "ok 25\n";
+
+undef($c);
+
+#
+# Try some URL encodings of the PATHs
+#
+$c = HTTP::Cookies->new;
+interact($c, "http://www.acme.com/foo%2f%25/%40%40%0Anew%E5/%E5", 'foo  =   bar; version    =   1');
+print $c->as_string;
+
+$cookie = interact($c, "http://www.acme.com/foo%2f%25/@@%0anewå/æøå", "bar=baz; path=\"/foo/\"; version=1");
+print "not " unless $cookie =~ /foo=bar/ && $cookie =~ /^\$version=\"?1\"?/i;
+print "ok 26\n";
+
+$cookie = interact($c, "http://www.acme.com/foo/%25/@@%0anewå/æøå");
+print "not " if $cookie;
+print "ok 27\n";
+
+undef($c);
+
+#
+# Try to use the Netscape cookie file format for saving
+#
+$file = "cookies-$$.txt";
+$c = HTTP::Cookies::Netscape->new(file => $file);
+interact($c, "http://www.acme.com/", "foo1=bar; max-age=100");
+interact($c, "http://www.acme.com/", "foo2=bar; port=\"80\"; max-age=100; Discard; Version=1");
+interact($c, "http://www.acme.com/", "foo3=bar; secure; Version=1");
+$c->save;
+undef($c);
+
+$c = HTTP::Cookies::Netscape->new(file => $file);
+print "not " unless count_cookies($c) == 1;     # 2 of them discarded on save
+print "ok 28\n";
+
+print "not " unless $c->as_string =~ /foo1=bar/;
+print "ok 29\n";
+undef($c);
+unlink($file);
+
+
+#
+# Some additional Netscape cookies test
+#
+$c = HTTP::Cookies->new;
+$req = HTTP::Request->new(POST => "http://foo.bar.acme.com/foo");
+
+# Netscape allows a host part that contains dots
+$res = HTTP::Response->new(200, "OK");
+$res->header(set_cookie => 'Customer=WILE_E_COYOTE; domain=.acme.com');
+$res->request($req);
+$c->extract_cookies($res);
+
+# and that the domain is the same as the host without adding a leading
+# dot to the domain.  Should not quote even if strange chars are used
+# in the cookie value.
+$res = HTTP::Response->new(200, "OK");
+$res->header(set_cookie => 'PART_NUMBER=3,4; domain=foo.bar.acme.com');
+$res->request($req);
+$c->extract_cookies($res);
+
+print $c->as_string;
+
+$req = HTTP::Request->new(POST => "http://foo.bar.acme.com/foo");
+$c->add_cookie_header($req);
+#print $req->as_string;
+print "not " unless $req->header("Cookie") =~ /PART_NUMBER=3,4/ &&
+	            $req->header("Cookie") =~ /Customer=WILE_E_COYOTE/;
+print "ok 30\n";
+
+
+
+# Test handling of local intranet hostnames without a dot
+$c->clear;
+print "---\n";
+#require LWP::Debug;
+#LWP::Debug::level('+');
+
+interact($c, "http://example/", "foo1=bar; PORT; Discard;");
+$_=interact($c, "http://example/", 'foo2=bar; domain=".local"');
+print "not " unless /foo1=bar/;
+print "ok 31\n";
+
+$_=interact($c, "http://example/", 'foo3=bar');
+$_=interact($c, "http://example/");
+print "Cookie: $_\n";
+print "not " unless /foo2=bar/ && count_cookies($c) == 3;
+print "ok 32\n";
+print $c->as_string;
+
+#-------------------------------------------------------------------
+
+sub interact
+{
+    my $c = shift;
+    my $url = shift;
+    my $req = HTTP::Request->new(POST => $url);
+    $c->add_cookie_header($req);
+    my $cookie = $req->header("Cookie");
+    my $res = HTTP::Response->new(200, "OK");
+    $res->request($req);
+    for (@_) { $res->push_header("Set-Cookie2" => $_) }
+    $c->extract_cookies($res);
+    return $cookie;
+}
+
+sub count_cookies
+{
+    my $c = shift;
+    my $no = 0;
+    $c->scan(sub { $no++ });
+    $no;
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/date.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/date.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/date.t	(revision 21112)
@@ -0,0 +1,165 @@
+use HTTP::Date;
+
+print "1..59\n";
+
+$no = 1;
+$| = 1;
+sub ok {
+   print "not " if $_[0];
+   print "ok $no\n";
+   $no++;
+}
+
+# test str2time for supported dates
+my(@tests) =
+(
+ 'Thu Feb  3 00:00:00 GMT 1994',        # ctime format
+ 'Thu Feb  3 00:00:00 1994',            # same as ctime, except no TZ
+
+ 'Thu, 03 Feb 1994 00:00:00 GMT',       # proposed new HTTP format
+ 'Thursday, 03-Feb-94 00:00:00 GMT',    # old rfc850 HTTP format
+ 'Thursday, 03-Feb-1994 00:00:00 GMT',  # broken rfc850 HTTP format
+
+ '03/Feb/1994:00:00:00 0000',   # common logfile format
+ '03/Feb/1994:01:00:00 +0100',  # common logfile format
+ '02/Feb/1994:23:00:00 -0100',  # common logfile format
+
+ '03 Feb 1994 00:00:00 GMT',    # HTTP format (no weekday)
+ '03-Feb-94 00:00:00 GMT',      # old rfc850 (no weekday)
+ '03-Feb-1994 00:00:00 GMT',    # broken rfc850 (no weekday)
+ '03-Feb-1994 00:00 GMT',       # broken rfc850 (no weekday, no seconds)
+ '03-Feb-1994 00:00',           # VMS dir listing format
+
+ '03-Feb-94',    # old rfc850 HTTP format    (no weekday, no time)
+ '03-Feb-1994',  # broken rfc850 HTTP format (no weekday, no time)
+ '03 Feb 1994',  # proposed new HTTP format  (no weekday, no time)
+ '03/Feb/1994',  # common logfile format     (no time, no offset)
+
+ #'Feb  3 00:00',     # Unix 'ls -l' format (can't really test it here)
+ 'Feb  3 1994',       # Unix 'ls -l' format
+
+ "02-03-94  12:00AM", # Windows 'dir' format
+
+ # ISO 8601 formats
+ '1994-02-03 00:00:00 +0000',
+ '1994-02-03',
+ '19940203',
+ '1994-02-03T00:00:00+0000',
+ '1994-02-02T23:00:00-0100',
+ '1994-02-02T23:00:00-01:00',
+ '1994-02-03T00:00:00 Z',
+ '19940203T000000Z',
+ '199402030000',
+
+ # A few tests with extra space at various places
+ '  03/Feb/1994      ',
+ '  03   Feb   1994  0:00  ',
+);
+
+my $time = 760233600;  # assume broken POSIX counting of seconds
+for (@tests) {
+    if (/GMT/i) {
+	$t = str2time($_);
+    } else {
+	$t = str2time($_, "GMT");
+    }
+    $t = "UNDEF" unless defined $t;
+    print "'$_'  =>  $t\n";
+    print $@ if $@;
+    print "not " if $t eq 'UNDEF' || $t != $time;
+    ok;
+}
+
+# test time2str
+die "time2str failed"
+    unless time2str($time) eq 'Thu, 03 Feb 1994 00:00:00 GMT';
+
+# test the 'ls -l' format with missing year$
+# round to nearest minute 3 days ago.
+$time = int((time - 3 * 24*60*60) /60)*60;
+($min, $hr, $mday, $mon) = (localtime $time)[1,2,3,4];
+$mon = (qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec))[$mon];
+$str = sprintf("$mon %02d %02d:%02d", $mday, $hr, $min);
+$t = str2time($str);
+$t = "UNDEF" unless defined $t;
+print "'$str'  =>  $t ($time)\n";
+print "not " if $t != $time;
+ok;
+
+# try some out of bounds date and some garbage.
+for ('03-Feb-1969', '03-Feb-2039',
+     undef, '', 'Garbage',
+     'Mandag 16. September 1996',
+     'Thu Feb  3 00:00:00 CET 1994',
+     'Thu, 03 Feb 1994 00:00:00 CET',
+     'Wednesday, 31-Dec-69 23:59:59 GMT',
+
+     '1980-00-01',
+     '1980-13-01',
+     '1980-01-00',
+     '1980-01-32',
+     '1980-01-01 25:00:00',
+     '1980-01-01 00:61:00',
+     '1980-01-01 00:00:61',
+    )
+{
+    my $bad = 0;
+    eval {
+	if (defined str2time $_) {
+	    print "str2time($_) is not undefined\n";
+	    $bad++;
+	}
+    };
+    print defined($_) ? "'$_'\n" : "undef\n";
+    print $@ if $@;
+    print "not " if $bad;
+    ok;
+}
+
+print "Testing AM/PM gruff...\n";
+
+$t = time2iso(str2time("11-12-96  0:00AM"));print "$t\n";
+ok($t ne "1996-11-12 00:00:00");
+
+$t = time2iso(str2time("11-12-96 12:00AM"));print "$t\n";
+ok($t ne "1996-11-12 00:00:00");
+
+$t = time2iso(str2time("11-12-96  0:00PM"));print "$t\n";
+ok($t ne "1996-11-12 12:00:00");
+
+$t = time2iso(str2time("11-12-96 12:00PM"));print "$t\n";
+ok($t ne "1996-11-12 12:00:00");
+
+
+$t = time2iso(str2time("11-12-96  1:05AM"));print "$t\n";
+ok($t ne "1996-11-12 01:05:00");
+
+$t = time2iso(str2time("11-12-96 12:05AM"));print "$t\n";
+ok($t ne "1996-11-12 00:05:00");
+
+$t = time2iso(str2time("11-12-96  1:05PM"));print "$t\n";
+ok($t ne "1996-11-12 13:05:00");
+
+$t = time2iso(str2time("11-12-96 12:05PM"));print "$t\n";
+ok($t ne "1996-11-12 12:05:00");
+
+
+
+# Test the str2iso routines
+use HTTP::Date qw(time2iso time2isoz);
+
+print "Testing time2iso functions\n";
+
+$a = time2iso;
+$b = time2iso(500000);
+print "LOCAL $a  $b\n";
+$az = time2isoz;
+$bz = time2isoz(500000);
+print "GMT   $az $bz\n";
+
+for ($a,  $b)  { ok if /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/;  }
+for ($az, $bz) { ok if /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\dZ$/; }
+
+
+
+print "HTTP::Date $HTTP::Date::VERSION tested ok\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-auth.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-auth.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-auth.t	(revision 21112)
@@ -0,0 +1,40 @@
+print "1..4\n";
+
+use HTTP::Response;
+use HTTP::Headers::Auth;
+
+$res = HTTP::Response->new(401);
+$res->push_header(WWW_Authenticate => qq(Foo realm="WallyWorld", foo=bar, Bar realm="WallyWorld2"));
+$res->push_header(WWW_Authenticate => qq(Basic Realm="WallyWorld", foo=bar, bar=baz));
+
+print $res->as_string;
+
+%auth = $res->www_authenticate;
+
+print "not " unless keys(%auth) == 3;
+print "ok 1\n";
+
+print "not " unless $auth{basic}{realm} eq "WallyWorld" &&
+                    $auth{bar}{realm} eq "WallyWorld2";
+print "ok 2\n";
+
+$a = $res->www_authenticate;
+print "not " unless $a eq 'Foo realm="WallyWorld", foo=bar, Bar realm="WallyWorld2", Basic Realm="WallyWorld", foo=bar, bar=baz';
+print "ok 3\n";
+
+$res->www_authenticate("Basic realm=foo1");
+print $res->as_string;
+
+$res->www_authenticate(Basic => {realm => foo2});
+print $res->as_string;
+
+$res->www_authenticate(Basic => [realm => foo3, foo=>33],
+Digest => {nonce=>"bar", foo=>'foo'});
+print $res->as_string;
+
+$_ = $res->as_string;
+
+print "not " unless /WWW-Authenticate: Basic realm="foo3", foo=33/ &&
+                    /WWW-Authenticate: Digest nonce=bar, foo=foo/;
+print "ok 4\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-etag.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-etag.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-etag.t	(revision 21112)
@@ -0,0 +1,28 @@
+print "1..4\n";
+
+require HTTP::Headers::ETag;
+
+$h = HTTP::Headers->new;
+$h->etag("tag1");
+print "not " unless $h->etag eq qq("tag1");
+print "ok 1\n";
+
+$h->etag("w/tag2");
+print "not " unless $h->etag eq qq(W/"tag2");
+print "ok 2\n";
+
+$h->if_match(qq(W/"foo", bar, baz), "bar");
+$h->if_none_match(333);
+
+$h->if_range("tag3");
+print "not " unless $h->if_range eq qq("tag3");
+print "ok 3\n";
+
+$t = time;
+$h->if_range($t);
+print "not " unless $h->if_range == $t;
+print "ok 4\n";
+
+
+print $h->as_string;
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-util.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-util.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers-util.t	(revision 21112)
@@ -0,0 +1,61 @@
+use strict;
+use HTTP::Headers::Util qw(split_header_words join_header_words);
+
+my $extra_tests = 2;
+
+my @s_tests = (
+
+   ["foo"                     => "foo"],
+   ["foo=bar"                 => "foo=bar"],
+   ["   foo   "               => "foo"],
+   ["foo="                    => 'foo=""'],
+   ["foo=bar bar=baz"         => "foo=bar; bar=baz"],
+   ["foo=bar;bar=baz"         => "foo=bar; bar=baz"],
+   ['foo bar baz'             => "foo; bar; baz"],
+   ['foo="\"" bar="\\\\"'     => 'foo="\""; bar="\\\\"'],
+   ['foo,,,bar'               => 'foo, bar'],
+   ['foo=bar,bar=baz'         => 'foo=bar, bar=baz'],
+
+   ['text/html; charset=iso-8859-1' =>
+    'text/html; charset="iso-8859-1"'],
+
+   ['foo="bar"; port="80,81"; discard, bar=baz' =>
+    'foo=bar; port="80,81"; discard, bar=baz'],
+
+   ['Basic realm="\"foo\\\\bar\""' =>
+    'Basic; realm="\"foo\\\\bar\""'],
+);
+
+print "1..", @s_tests + $extra_tests, "\n";
+
+my $testno = 1;
+
+print "split_header_words() tests\n";
+for (@s_tests) {
+   my($arg, $expect) = @$_;
+   my @arg = ref($arg) ? @$arg : $arg;
+
+   my $res = join_header_words(split_header_words(@arg));
+   if ($res ne $expect) {
+       print "\nUnexpected result: '$res'\n";
+       print "         Expected: '$expect'\n";
+       print "  when parsing '", join(", ", @arg), "'\n";
+       eval {
+	   require Data::Dumper;
+           my @p = split_header_words(@arg);
+           print Data::Dumper::Dumper(\@p);
+       };
+       print "not ";
+   }
+   print "ok ", $testno++, "\n";
+}
+
+
+print "Extra tests\n";
+# some extra tests
+print "not " unless join_header_words("foo" => undef, "bar" => "baz")
+                    eq "foo; bar=baz";
+print "ok ", $testno++, "\n";
+
+print "not " unless join_header_words() eq "";
+print "ok ", $testno++, "\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/headers.t	(revision 21112)
@@ -0,0 +1,128 @@
+require HTTP::Headers;
+
+print "1..11\n";
+
+$h = new HTTP::Headers
+	mime_version  => "1.0",
+	content_type  => "text/html";
+
+$h->header(URI => "http://www.oslonett.no/");
+
+if ($h->header("MIME-Version") eq "1.0") {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+if ($h->header('Uri') =~ /^http:/) {
+    print "ok 2\n";
+} else {
+    print "not ok 2\n";
+}
+
+
+$h->header("MY-header" => "foo",
+	   "Date" => "somedate",
+	   "Accept" => ["text/plain", "image/*"],
+	  );
+$h->push_header("accept" => "audio/basic");
+
+if ($h->header("date") eq "somedate") {
+     print "ok 3\n";
+}
+
+@accept = $h->header("accept");
+if (@accept == 3) {
+    print "ok 4\n";
+}
+
+$h->remove_header("uri", "date");
+
+
+$str = $h->as_string;
+print "\nHeader looks like this now:\n$str\n";
+
+$lines = ($str =~ tr/\n/\n/);
+
+if ($lines == 6) {
+    print "ok 5\n";
+} else {
+    print "Header has $lines lines\n";
+    print "not ok 5\n";
+}
+
+$h2 = $h->clone;
+
+$h->header("accept", "*/*");
+$h->remove_header("my-header");
+
+@accept = $h2->header("accept");
+if (@accept == 3) {
+    print "ok 6\n";
+}
+
+@accept = $h->header("accept");
+if (@accept == 1) {
+    print "ok 7\n";
+}
+
+# Check order of headers, but first remove this one
+$h2->remove_header('mime_version');
+
+# and add this general header
+$h2->header(Connection => 'close');
+
+@x = ();
+$h2->scan(sub {push(@x, shift);});
+
+$str = join(";", @x);
+$expected = "Connection;Accept;Accept;Accept;Content-Type;MY-Header";
+
+if ($str eq $expected) {
+    print "ok 8\n";
+} else {
+    print "Headers are '$str',\nexpected    '$expected'\n";
+    print "not ok 8\n";
+}
+
+# Check headers with embedded newlines:
+
+$h = new HTTP::Headers
+	a => "foo\n\n",
+	b => "foo\nbar",
+	c => "foo\n\nbar\n\n",
+	d => "foo\n\tbar";
+$str = $h->as_string("<<\n");
+print "-----\n$str------\n";
+
+print "not " unless $str =~ /^A:\s*foo<<\n
+                              B:\s*foo<<\n
+	                        \s+bar<<\n
+                              C:\s*foo<<\n
+                                \s+bar<<\n
+	                      D:\s*foo<<\n
+                                \t bar<<\n
+                             $/x;
+print "ok 9\n";
+
+
+# Check with FALSE $HTML::Headers::TRANSLATE_UNDERSCORE
+
+local($HTTP::Headers::TRANSLATE_UNDERSCORE);
+$HTTP::Headers::TRANSLATE_UNDERSCORE = undef;  # avoid -w warning
+
+$h = HTTP::Headers->new;
+
+$h->header(abc_abc   => "foo");
+$h->header("abc-abc" => "bar");
+
+#print $h->as_string;
+
+print "not " unless $h->header("ABC_ABC") eq "foo" &&
+                    $h->header("ABC-ABC") eq "bar";
+print "ok 10\n";
+
+$h->remove_header("Abc_Abc");
+print "not " unless !defined($h->header("abc_abc")) &&
+                    $h->header("ABC-ABC") eq "bar";
+print "ok 11\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/listing.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/listing.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/listing.t	(revision 21112)
@@ -0,0 +1,86 @@
+print "1..6\n";
+
+
+use File::Listing;
+
+$dir = <<'EOL';
+total 68
+drwxr-xr-x   4 aas      users        1024 Mar 16 15:47 .
+drwxr-xr-x  11 aas      users        1024 Mar 15 19:22 ..
+drwxr-xr-x   2 aas      users        1024 Mar 16 15:47 CVS
+-rw-r--r--   1 aas      users        2384 Feb 26 21:14 Debug.pm
+-rw-r--r--   1 aas      users        2145 Feb 26 20:09 IO.pm
+-rw-r--r--   1 aas      users        3960 Mar 15 18:05 MediaTypes.pm
+-rw-r--r--   1 aas      users         792 Feb 26 20:12 MemberMixin.pm
+drwxr-xr-x   3 aas      users        1024 Mar 15 18:05 Protocol
+-rw-r--r--   1 aas      users        5613 Feb 26 20:16 Protocol.pm
+-rw-r--r--   1 aas      users        5963 Feb 26 21:27 RobotUA.pm
+-rw-r--r--   1 aas      users        5071 Mar 16 12:25 Simple.pm
+-rw-r--r--   1 aas      users        8817 Mar 15 18:05 Socket.pm
+-rw-r--r--   1 aas      users        2121 Feb  5 14:22 TkIO.pm
+-rw-r--r--   1 aas      users       19628 Mar 15 18:05 UserAgent.pm
+-rw-r--r--   1 aas      users        2841 Feb  5 19:06 media.types
+
+CVS:
+total 5
+drwxr-xr-x   2 aas      users        1024 Mar 16 15:47 .
+drwxr-xr-x   4 aas      users        1024 Mar 16 15:47 ..
+-rw-r--r--   1 aas      users         545 Mar 16 15:47 Entries
+-rw-r--r--   1 aas      users          39 Mar 10 09:05 Repository
+-rw-r--r--   1 aas      users          19 Mar 10 09:05 Root
+
+Protocol:
+total 37
+drwxr-xr-x   3 aas      users        1024 Mar 15 18:05 .
+drwxr-xr-x   4 aas      users        1024 Mar 16 15:47 ..
+drwxr-xr-x   2 aas      users        1024 Mar 15 18:05 CVS
+-rw-r--r--   1 aas      users        4646 Feb 26 20:13 file.pm
+-rw-r--r--   1 aas      users       13006 Mar 15 18:05 ftp.pm
+-rw-r--r--   1 aas      users        5935 Mar  6 10:29 gopher.pm
+-rw-r--r--   1 aas      users        5453 Mar  6 10:29 http.pm
+-rw-r--r--   1 aas      users        2365 Feb 26 20:13 mailto.pm
+
+Protocol/CVS:
+total 5
+drwxr-xr-x   2 aas      users        1024 Mar 15 18:05 .
+drwxr-xr-x   3 aas      users        1024 Mar 15 18:05 ..
+-rw-r--r--   1 aas      users         238 Mar 15 18:05 Entries
+-rw-r--r--   1 aas      users          48 Mar 10 09:05 Repository
+-rw-r--r--   1 aas      users          19 Mar 10 09:05 Root
+EOL
+
+@dir = parse_dir($dir, undef, 'unix');
+
+print int(@dir) ." lines found\n";
+@dir != 25 && print "not ";
+print "ok 1\n";
+
+for (@dir) {
+   ($name, $type, $size, $mtime, $mode) = @$_;
+   $size ||= 0;  # ensure that it is defined
+   printf "%-25s $type %6d  ", $name, $size;
+   print scalar(localtime($mtime));
+   printf "  %06o", $mode;
+   print "\n";
+}
+
+# Pick out the Socket.pm line as the sample we check carefully
+($name, $type, $size, $mtime, $mode) = @{$dir[9]};
+
+$name eq "Socket.pm" || print "not ";
+print "ok 2\n";
+
+$type eq "f" || print "not ";
+print "ok 3\n";
+
+$size == 8817 || print "not ";
+print "ok 4\n";
+
+# Must be careful when checking the time stamps because we don't know
+# which year if this script lives for a long time.
+$timestring = scalar(localtime($mtime));
+$timestring =~ /Mar\s+15\s+18:05/ or print "not ";
+print "ok 5\n";
+
+$mode == 0100644 || print "not ";
+print "ok 6\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/mediatypes.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/mediatypes.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/mediatypes.t	(revision 21112)
@@ -0,0 +1,87 @@
+use LWP::MediaTypes;
+
+require URI::URL;
+
+$url1 = new URI::URL 'http://www/foo/test.gif?search+x#frag';
+$url2 = new URI::URL 'http:test';
+
+$file = "/etc/passwd";
+-r $file or $file = "./README";
+
+@tests =
+(
+ ["/this.dir/file.html" => "text/html",],
+ ["test.gif.htm"        => "text/html",],
+ ["test.txt.gz"         => "text/plain", "gzip"],
+ ["gif.foo"             => "application/octet-stream",],
+ ["lwp-0.03.tar.Z"      => "application/x-tar", "compress"],
+ [$file		        => "text/plain",],
+ ["/random/file"        => "application/octet-stream",],
+ ["/dev/null"	        => "text/plain",],
+ [$url1	        	=> "image/gif",],
+ [$url2	        	=> "application/octet-stream",],
+ ["x.ppm.Z.UU"		=> "image/x-portable-pixmap","compress","x-uuencode",],
+);
+
+$notests = @tests + 3;
+print "1..$notests\n";
+
+if (-f "$ENV{HOME}/.mime.types") {
+   warn "
+The MediaTypes test might fail because you have a private ~/.mime.types file
+If you get a failed test, try to move it away while testing.
+";
+}
+
+
+$testno = 1;
+for (@tests) {
+    ($file, $expectedtype, @expectedEnc) = @$_;
+    $type1 = guess_media_type($file);
+    ($type, @enc) = guess_media_type($file);
+    if ($type1 ne $type) {
+       print "guess_media_type does not return same content-type in scalar and array conext.\n";
+	next;
+    }
+    $type = "undef" unless defined $type;
+    if ($type eq $expectedtype and "@enc" eq "@expectedEnc") {
+	print "ok $testno\n";
+    } else {
+	print "expected '$expectedtype' for '$file', got '$type'\n";
+	print "encoding: expected: '@expectedEnc', got '@enc'\n"
+	  if @expectedEnc || @enc;
+	print "nok ok $testno\n";
+    }
+    $testno++;
+}
+
+@imgSuffix = media_suffix('image/*');
+print "Image suffixes: @imgSuffix\n";
+
+print "\n";
+require HTTP::Response;
+$r = new HTTP::Response 200, "Document follows";
+$r->title("file.tar.gz.uu");
+guess_media_type($r->title, $r);
+print $r->as_string;
+
+print "not " unless $r->content_type eq "application/x-tar";
+print "ok $testno\n"; $testno++;
+
+@enc = $r->header("Content-Encoding");
+print "not " unless "@enc" eq "gzip x-uuencode";
+print "ok $testno\n"; $testno++;
+
+#
+use LWP::MediaTypes qw(add_type add_encoding);
+add_type("x-world/x-vrml", qw(wrl vrml));
+add_encoding("x-gzip" => "gz");
+add_encoding(rot13 => "r13");
+
+@x = guess_media_type("foo.vrml.r13.gz");
+#print "@x\n";
+print "not " unless "@x" eq "x-world/x-vrml rot13 x-gzip";
+print "ok $testno\n"; $testno++;
+
+#print LWP::MediaTypes::_dump();
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/message.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/message.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/message.t	(revision 21112)
@@ -0,0 +1,104 @@
+print "1..15\n";
+
+require HTTP::Request;
+require HTTP::Response;
+
+$req = new HTTP::Request 'GET', "http://www.sn.no/";
+$req->header(
+	"if-modified-since" => "Thu, 03 Feb 1994 00:00:00 GMT",
+	"mime-version"      => "1.0");
+
+$str = $req->as_string;
+
+print $str;
+
+$str =~ /^GET/m || print "not ";
+print "ok 1\n";
+
+$req->header("MIME-Version") eq "1.0" || print "not ";
+print "ok 2\n";
+
+$req->content("gisle");
+$req->add_content(" aas");
+$req->add_content(\ " old interface is depreciated");
+
+${$req->content_ref} =~ s/\s+is\s+depreciated//;
+
+print "Content is: ", $req->content, "\n";
+
+$req->content eq "gisle aas old interface" || print "not ";
+print "ok 3\n";
+
+$req->if_modified_since == 760233600 || print "not ";
+print "ok 4\n";
+
+$time = time;
+
+$req->date($time);
+$timestr = gmtime($time);
+($month) = ($timestr =~ /^\S+\s+(\S+)/);  # extract month;
+
+print "These should represent the same time:\n\t", $req->header('Date'), "\n\t$timestr\n";
+
+$req->header('Date') =~ /\Q$month/ || print "not ";
+print "ok 5\n";
+
+$req->authorization_basic("gisle", "passwd");
+$auth = $req->header("Authorization");
+
+print "$auth\n";
+$auth =~ /Z2lzbGU6cGFzc3dk/ || print "not ";
+print "ok 6\n";
+
+($user, $pass) = $req->authorization_basic;
+($user eq "gisle" && $pass eq "passwd") || print "not ";
+print "ok 7\n";
+
+# Check the response
+$res = new HTTP::Response 200, "This message";
+
+$html = $res->error_as_HTML;
+print $html;
+
+($html =~ /<head>/i && $html =~ /This message/) || print "not ";
+print "ok 8\n";
+
+$res->is_success || print "not ";
+print "ok 9\n";
+
+$res->content_type("text/html;version=3.0");
+$res->content("<html>...</html>\n");
+
+$res2 = $res->clone;
+
+print $res2->as_string;
+
+$res2->header("cOntent-TYPE") eq "text/html;version=3.0" || print "not ";
+print "ok 10\n";
+
+$res2->code == 200 || print "not ";
+print "ok 11\n";
+
+$res2->content =~ />\.\.\.</ || print "not ";
+print "ok 12\n";
+
+# Check the base method:
+
+$res = new HTTP::Response 200, "This message";
+$res->request($req);
+$res->content_type("image/gif");
+
+$res->base eq "http://www.sn.no/" || print "not ";
+print "ok 13\n";
+
+$res->header('Base', 'http://www.sn.no/xxx/');
+
+$res->base eq "http://www.sn.no/xxx/" || print "not ";
+print "ok 14\n";
+
+# Check the AUTLOAD delegate method with regular expressions
+"This string contains text/html" =~ /(\w+\/\w+)/;
+$res->content_type($1);
+
+$res->content_type eq "text/html" || print "not ";
+print "ok 15\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/negotiate.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/negotiate.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/negotiate.t	(revision 21112)
@@ -0,0 +1,105 @@
+print "1..3\n";
+
+use HTTP::Request;
+use HTTP::Negotiate;
+
+
+$no = 1;
+sub ok
+{
+    print "ok " . $no++ . "\n";
+}
+
+sub not_ok
+{
+    print "not ";
+    ok;
+}
+
+
+ #  ID       QS     Content-Type             Encoding     Char-Set      Lang    Size
+ $variants =
+  [
+   ['var1',  0.950, 'text/plain',           ['uuencode',
+					     'compress'], 'iso-8859-2', 'se',    400],
+   ['var2',  1.000, 'text/html;version=2.0', 'gzip',      'iso-8859-1', 'en',   3000],
+   ['var3',  0.333, 'image/gif',            undef,        undef,        undef, 43555],
+ ];
+
+
+# First we try a request with not accept headers
+$request = new HTTP::Request 'GET', 'http://localhost/';
+@a = choose($variants, $request);
+show_res(@a);
+expect(\@a, [['var2' => 1],
+	     ['var1' => 0.95],
+	     ['var3' => 0.333]
+	    ]
+);
+
+
+$a = choose($variants, $request);
+print "The chosen one is '$a'\n";
+if ($a eq 'var2') {
+    ok;
+} else {
+    not_ok;
+}
+
+#------------------
+
+$request = new HTTP::Request 'GET', 'http://localhost/';
+$request->header('Accept', 'text/plain; q=0.55, image/gif; mbx=10000');
+$request->push_header('Accept', 'text/*; q=0.25');
+$request->header('Accept-Language', 'no, en');
+$request->header('Accept-Charset', 'iso-8859-1');
+$request->header('Accept-Encoding', 'gzip');
+
+@a = choose($variants, $request);
+show_res(@a);
+expect(\@a, [['var2' => 0.25],
+	     ['var1' => 0],
+	     ['var3' => 0]
+	    ]
+);
+
+
+
+
+#------------------
+
+sub expect
+{
+    my($res, $exp) = @_;
+    do {
+	$a = shift @$res;
+	$b = shift @$exp;
+	last if defined($a) ne defined($b);
+	if (defined($a)) {
+	    ($va, $qa) = @$a;
+	    ($vb, $qb) = @$b;
+	    if ($va ne $vb) {
+		print "$va == $vb ?\n";
+		not_ok;
+		return;
+	    }
+	    if (abs($qa - $qb) > 0.002) {
+		print "$qa ~= $qb ?\n";
+		not_ok;
+		return;
+	    }
+	}
+
+    } until (!defined($a) || !defined($b));
+    return not_ok if defined($a) ne defined($b);
+    ok;
+}
+
+sub show_res
+{
+    print "-------------\n";
+    for (@_) {
+	printf "%-6s %.3f\n", @$_;
+    }
+    print "-------------\n";
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/response.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/response.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/response.t	(revision 21112)
@@ -0,0 +1,86 @@
+
+# Test extra HTTP::Response methods.  Basic operation is tested in the
+# message.t test suite.
+
+
+print "1..7\n";
+
+
+use HTTP::Date;
+use HTTP::Request;
+use HTTP::Response;
+
+my $time = time;
+
+$req = HTTP::Request->new(GET => 'http://www.sn.no');
+$req->date($time - 30);
+
+$r = new HTTP::Response 200, "OK";
+$r->client_date($time - 20);
+$r->date($time - 25);
+$r->last_modified($time - 5000000);
+$r->request($req);
+
+print $r->as_string;
+
+$current_age = $r->current_age;
+
+if ($current_age < 35  || $current_age > 40) {
+    print "not ";
+}
+print "ok 1\n";
+
+$freshness_lifetime = $r->freshness_lifetime;
+if ($freshness_lifetime < 12 * 3600) {
+    print "not ";
+}
+print "ok 2\n";
+
+$is_fresh = $r->is_fresh;
+
+print "not " unless $is_fresh;
+print "ok 3\n";
+
+print "current_age        = $current_age\n";
+print "freshness_lifetime = $freshness_lifetime\n";
+print "response is ";
+print "not " unless $is_fresh;
+print "fresh\n";
+
+
+print "it will be fresh is ";
+print $freshness_lifetime - $current_age;
+print " more seconds\n";
+
+# OK, now we add an Expires header
+$r->expires($time);
+print $r->as_string;
+
+$freshness_lifetime = $r->freshness_lifetime;
+print "freshness_lifetime = $freshness_lifetime\n";
+print "not " unless $freshness_lifetime == 25;
+print "ok 4\n";
+$r->remove_header('expires');
+
+# Now we try the 'Age' header and the Cache-Contol:
+
+$r->header('Age', 300);
+$r->push_header('Cache-Control', 'junk');
+$r->push_header(Cache_Control => 'max-age = 10');
+
+print $r->as_string;
+
+$current_age = $r->current_age;
+$freshness_lifetime = $r->freshness_lifetime;
+
+print "current_age        = $current_age\n";
+print "freshness_lifetime = $freshness_lifetime\n";
+
+print "not " if $current_age < 300;
+print "ok 5\n";
+
+print "not " if $freshness_lifetime != 10;
+print "ok 6\n";
+
+print "not " unless $r->fresh_until;  # should return something
+print "ok 7\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/status.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/status.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/base/status.t	(revision 21112)
@@ -0,0 +1,32 @@
+use HTTP::Status;
+
+print "1..8\n";
+
+200 == RC_OK || print "not ";
+print "ok 1\n";
+
+is_success(RC_ACCEPTED) || print "not ";
+print "ok 2\n";
+
+is_error(RC_BAD_REQUEST) || print "not ";
+print "ok 3\n";
+
+is_redirect(RC_MOVED_PERMANENTLY) || print "not ";
+print "ok 4\n";
+
+is_success(RC_NOT_FOUND) && print "not ";
+print "ok 5\n";
+
+$mess = status_message(0);
+
+defined $mess && print "not ";
+print "ok 6\n";
+
+$mess = status_message(200);
+
+if ($mess =~ /ok/i) {
+    print "ok 7\n";
+}
+
+is_info(RC_CONTINUE) || print "not ";
+print "ok 8\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/autoload.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/autoload.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/autoload.t	(revision 21112)
@@ -0,0 +1,30 @@
+#
+# See if autoloading of protocol schemes work
+#
+
+print "1..1\n";
+
+require LWP::UserAgent;
+# note no LWP::Protocol::file;
+
+# localhost will not work on win32 (when networking is disabled)
+$url = $^O eq 'MSWin32' ? 'file:.' : 'file://localhost/';
+
+print "Trying to fetch " . (new URI::URL $url)->local_path . " ...\n";
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+$ua->timeout(30);               # timeout in seconds
+#$ua->use_alarm(0);               # don't use alarms
+#$ua->use_eval(0);                # don't eval, just die when thing go wrong
+				# (easier to read while debugging)
+
+my $request = new HTTP::Request('GET', $url);
+
+my $response = $ua->request($request, undef, undef);
+if ($response->is_success) {
+    print "ok 1\n";
+} else {
+    print $response->error_as_HTML;
+    print "not ok 1\n";
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/get.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/get.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/get.t	(revision 21112)
@@ -0,0 +1,85 @@
+#
+# Test retrieving a file with a 'file://' URL,
+#
+
+# First locate some suitable tmp-dir.  We need an absolute path.
+$TMPDIR = undef;
+for ("/tmp/", "/var/tmp", "/usr/tmp", "/local/tmp") {
+    if (open(TEST, ">$_/test-$$")) {
+        close(TEST);
+	unlink("$_/test-$$");
+	$TMPDIR = $_;
+	last;
+    }
+}
+$TMPDIR ||= $ENV{TEMP} if $^O eq 'MSWin32';
+unless ($TMPDIR) {
+   # Can't run any tests
+   print "1..0\n";
+   print "ok 1\n";
+   exit;
+}
+$TMPDIR =~ tr|\\|/|;
+print "1..2\n";
+
+use LWP::Simple;
+require LWP::Protocol::file;
+
+my $orig = "$TMPDIR/lwp-orig-$$";          # local file
+my $copy = "$TMPDIR/lwp-copy-$$"; 	    # downloaded copy
+
+# First we create the original
+open(OUT, ">$orig") or die "Cannot open $orig: $!";
+for (1..100) {
+    print OUT "This is line $_ of $orig\n";
+}
+close(OUT);
+
+
+# Then we make a test using getprint(), so we need to capture stdout
+open (OUT, ">$copy") or die "Cannot open $copy: $!";
+select(OUT);
+
+# do the retrieval
+getprint("file://localhost" . ($orig =~ m|^/| ? $orig : "/$orig"));
+
+close(OUT);
+select(STDOUT);
+
+# read and compare the files
+open(IN, $orig) or die "Cannot open '$orig': $!";
+undef($/);
+$origtext = <IN>;
+close(IN);
+open(IN, $copy) or die "Cannot open '$copy': $!";
+undef($/);
+$copytext = <IN>;
+close(IN);
+
+unlink($copy);
+
+if ($origtext eq $copytext) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+
+# Test getstore() function
+
+getstore("file:$orig", $copy);
+
+# Take a look at the new copy
+open(IN, $copy) or die "Cannot open '$copy': $!";
+undef($/);
+$copytext = <IN>;
+close(IN);
+
+unlink($orig);
+unlink($copy);
+
+if ($origtext eq $copytext) {
+    print "ok 2\n";
+} else {
+    print "not ok 2\n";
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/http.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/http.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/http.t	(revision 21112)
@@ -0,0 +1,319 @@
+$| = 1; # autoflush
+
+require IO::Socket;  # make sure this work before we try to make a HTTP::Daemon
+
+# First we make ourself a daemon in another process
+my $D = shift || '';
+if ($D eq 'daemon') {
+
+    require HTTP::Daemon;
+
+    my $d = new HTTP::Daemon Timeout => 10;
+
+    print "Please to meet you at: <URL:", $d->url, ">\n";
+    open(STDOUT, ">/dev/null");
+
+    while ($c = $d->accept) {
+	$r = $c->get_request;
+	if ($r) {
+	    my $p = ($r->url->path_components)[1];
+	    my $func = lc("httpd_" . $r->method . "_$p");
+	    if (defined &$func) {
+		&$func($c, $r);
+	    } else {
+		$c->send_error(404);
+	    }
+	}
+	$c = undef;  # close connection
+    }
+    print STDERR "HTTP Server terminated\n";
+    exit;
+}
+else {
+    use Config;
+    open(DAEMON, "$Config{'perlpath'} local/http.t daemon |") or die "Can't exec daemon: $!";
+}
+
+print "1..18\n";
+
+
+my $greeting = <DAEMON>;
+$greeting =~ /(<[^>]+>)/;
+
+require URI::URL;
+URI::URL->import;
+my $base = new URI::URL $1;
+
+print "Will access HTTP server at $base\n";
+
+require LWP::UserAgent;
+require HTTP::Request;
+$ua = new LWP::UserAgent;
+$ua->agent("Mozilla/0.01 " . $ua->agent);
+$ua->from('gisle@aas.no');
+
+#----------------------------------------------------------------
+print "Bad request...\n";
+$req = new HTTP::Request GET => url("/not_found", $base);
+$req->header(X_Foo => "Bar");
+$res = $ua->request($req);
+
+print "not " unless $res->is_error
+                and $res->code == 404
+                and $res->message =~ /not\s+found/i;
+print "ok 1\n";
+# we also expect a few headers
+print "not " if !$res->server and !$res->date;
+print "ok 2\n";
+
+#----------------------------------------------------------------
+print "Simple echo...\n";
+sub httpd_get_echo
+{
+    my($c, $req) = @_;
+    $c->send_basic_header(200);
+    print $c "Content-Type: text/plain\015\012";
+    $c->send_crlf;
+    print $c $req->as_string;
+}
+
+$req = new HTTP::Request GET => url("/echo/path_info?query", $base);
+$req->push_header(Accept => 'text/html');
+$req->push_header(Accept => 'text/plain; q=0.9');
+$req->push_header(Accept => 'image/*');
+$req->if_modified_since(time - 300);
+$req->header(Long_text => 'This is a very long header line
+which is broken between
+more than one line.');
+$req->header(X_Foo => "Bar");
+
+$res = $ua->request($req);
+#print $res->as_string;
+
+print "not " unless $res->is_success
+               and  $res->code == 200 && $res->message eq "OK";
+print "ok 3\n";
+
+$_ = $res->content;
+@accept = /^Accept:\s*(.*)/mg;
+
+print "not " unless /^From:\s*gisle\@aas\.no$/m
+                and /^Host:/m
+                and @accept == 3
+	        and /^Accept:\s*text\/html/m
+	        and /^Accept:\s*text\/plain/m
+	        and /^Accept:\s*image\/\*/m
+		and /^If-Modified-Since:\s*\w{3},\s+\d+/m
+                and /^Long-Text:\s*This.*broken between/m
+		and /^X-Foo:\s*Bar$/m
+		and /^User-Agent:\s*Mozilla\/0.01/m;
+print "ok 4\n";
+
+#----------------------------------------------------------------
+print "Send file...\n";
+
+my $file = "test-$$.html";
+open(FILE, ">$file") or die "Can't create $file: $!";
+binmode FILE or die "Can't binmode $file: $!";
+print FILE <<EOT;
+<html><title>En prøve</title>
+<h1>Dette er en testfil</h1>
+Jeg vet ikke hvor stor fila behøver å være heller, men dette
+er sikkert nok i massevis.
+EOT
+close(FILE);
+
+sub httpd_get_file
+{
+    my($c, $r) = @_;
+    my %form = $r->url->query_form;
+    my $file = $form{'name'};
+    $c->send_file_response($file);
+    unlink($file);
+}
+
+$req = new HTTP::Request GET => url("/file?name=$file", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+
+print "not " unless $res->is_success
+                and $res->content_type eq 'text/html'
+		and $res->content_length == 147
+		and $res->title eq 'En prøve'
+		and $res->content =~ /å være/;
+print "ok 5\n";		
+
+
+# A second try on the same file, should fail because we unlink it
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->is_error
+                and $res->code == 404;   # not found
+print "ok 6\n";
+		
+# Then try to list current directory
+$req = new HTTP::Request GET => url("/file?name=.", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->code == 501;   # NYI
+print "ok 7\n";
+
+
+#----------------------------------------------------------------
+print "Check redirect...\n";
+sub httpd_get_redirect
+{
+   my($c) = @_;
+   $c->send_redirect("/echo/redirect");
+}
+
+$req = new HTTP::Request GET => url("/redirect/foo", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+
+print "not " unless $res->is_success
+                and $res->content =~ m|/echo/redirect|;
+print "ok 8\n";
+print "not " unless $res->previous->is_redirect
+                and $res->previous->code == 301;
+print "ok 9\n";
+
+# Let's test a redirect loop too
+sub httpd_get_redirect2 { shift->send_redirect("/redirect3/") }
+sub httpd_get_redirect3 { shift->send_redirect("/redirect4/") }
+sub httpd_get_redirect4 { shift->send_redirect("/redirect5/") }
+sub httpd_get_redirect5 { shift->send_redirect("/redirect6/") }
+sub httpd_get_redirect6 { shift->send_redirect("/redirect2/") }
+
+$req->url(url("/redirect2", $base));
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->is_redirect
+                and $res->header("Client-Warning") =~ /loop detected/i;
+print "ok 10\n";
+$i = 1;
+while ($res->previous) {
+   $i++;
+   $res = $res->previous;
+}
+print "not " unless $i == 6;
+print "ok 11\n";
+
+#----------------------------------------------------------------
+print "Check basic authorization...\n";
+sub httpd_get_basic
+{
+    my($c, $r) = @_;
+    #print STDERR $r->as_string;
+    my($u,$p) = $r->authorization_basic;
+    if (defined($u) && $u eq 'ok 12' && $p eq 'xyzzy') {
+        $c->send_basic_header(200);
+	print $c "Content-Type: text/plain";
+	$c->send_crlf;
+	$c->send_crlf;
+	$c->print("$u\n");
+    } else {
+        $c->send_basic_header(401);
+	$c->print("WWW-Authenticate: Basic realm=\"libwww-perl\"\015\012");
+	$c->send_crlf;
+    }
+}
+
+{
+   package MyUA; @ISA=qw(LWP::UserAgent);
+   sub get_basic_credentials {
+      my($self, $realm, $uri, $proxy) = @_;
+      my $x =  $uri->rel($base);
+      if ($realm eq "libwww-perl" && $uri->rel($base) eq "basic") {
+	  return ("ok 12", "xyzzy");
+      } else {
+          return undef;
+      }
+   }
+}
+$req = new HTTP::Request GET => url("/basic", $base);
+$res = MyUA->new->request($req);
+#print $res->as_string;
+
+print "not " unless $res->is_success;
+print $res->content;
+
+# Let's try with a $ua that does not pass out credentials
+$res = $ua->request($req);
+print "not " unless $res->code == 401;
+print "ok 13\n";
+
+# Let's try to set credentials for this realm
+$ua->credentials($req->url->netloc, "libwww-perl", "ok 12", "xyzzy");
+$res = $ua->request($req);
+print "not " unless $res->is_success;
+print "ok 14\n";
+
+# Then illegal credentials
+$ua->credentials($req->url->netloc, "libwww-perl", "user", "passwd");
+$res = $ua->request($req);
+print "not " unless $res->code == 401;
+print "ok 15\n";
+
+
+#----------------------------------------------------------------
+print "Check proxy...\n";
+sub httpd_get_proxy
+{
+   my($c,$r) = @_;
+   if ($r->method eq "GET" and
+       $r->url->scheme eq "ftp") {
+       $c->send_basic_header(200);
+       $c->send_crlf;
+   } else {
+       $c->send_error;
+   }
+}
+
+$ua->proxy(ftp => $base);
+$req = new HTTP::Request GET => "ftp://ftp.perl.com/proxy";
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->is_success;
+print "ok 16\n";
+
+#----------------------------------------------------------------
+print "Check POSTing...\n";
+sub httpd_post_echo
+{
+   my($c,$r) = @_;
+   $c->send_basic_header;
+   $c->print("Content-Type: text/plain");
+   $c->send_crlf;
+   $c->send_crlf;
+   $c->print($r->as_string);
+}
+
+$req = new HTTP::Request POST => url("/echo/foo", $base);
+$req->content_type("application/x-www-form-urlencoded");
+$req->content("foo=bar&bar=test");
+$res = $ua->request($req);
+#print $res->as_string;
+
+$_ = $res->content;
+print "not " unless $res->is_success
+                and /^Content-Length:\s*16$/mi
+		and /^Content-Type:\s*application\/x-www-form-urlencoded$/mi
+		and /^foo=bar&bar=test/m;
+print "ok 17\n";		
+
+#----------------------------------------------------------------
+print "Terminating server...\n";
+sub httpd_get_quit
+{
+    my($c) = @_;
+    $c->send_error(503, "Bye, bye");
+    exit;  # terminate HTTP server
+}
+
+$req = new HTTP::Request GET => url("/quit", $base);
+$res = $ua->request($req);
+
+print "not " unless $res->code == 503 and $res->content =~ /Bye, bye/;
+print "ok 18\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/protosub.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/protosub.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/local/protosub.t	(revision 21112)
@@ -0,0 +1,73 @@
+#!perl
+
+print "1..6\n";
+
+# This test tries to make a custom protocol implementation by
+# subclassing of LWP::Protocol.
+
+
+use LWP::UserAgent ();
+use LWP::Protocol ();
+
+LWP::Protocol::implementor(http => 'myhttp');
+
+$ua = LWP::UserAgent->new;
+$ua->proxy('ftp' => "http://www.sn.no/");
+
+$req = HTTP::Request->new(GET => 'ftp://foo/');
+$req->header(Cookie => "perl=cool");
+
+$res = $ua->request($req);
+
+print $res->as_string;
+
+print "not " unless $res->code == 200;
+print "ok 5\n";
+print "not " unless $res->content eq "Howdy\n";
+print "ok 6\n";
+exit;
+
+
+#----------------------------------
+package myhttp;
+
+BEGIN {
+   @ISA=qw(LWP::Protocol);
+}
+
+sub new
+{
+    my $class = shift;
+    print "CTOR: $class->new(@_)\n";
+    my($prot) = @_;
+    print "not " unless $prot eq "http";
+    print "ok 1\n";
+    my $self = $class->SUPER::new(@_);
+    for (keys %$self) {
+	print "$_: $self->{$_}\n";
+    }
+    $self;
+}
+
+sub request
+{
+    my $self = shift;
+    print "REQUEST: $self->request(",
+       join(",", (map defined($_)? $_ : "UNDEF", @_)), ")\n";
+
+    my($request, $proxy, $arg, $size, $timeout) = @_;
+    print $request->as_string;
+
+    print "not " unless $proxy eq "http://www.sn.no/";
+    print "ok 2\n";
+    print "not " unless $request->url eq "ftp://foo/";
+    print "ok 3\n";
+    print "not " unless $request->header("cookie") eq "perl=cool";
+    print "ok 4\n";
+
+    my $res = HTTP::Response->new(200 => "OK");
+    $res->content_type("text/plain");
+    $res->date(time);
+    $self->collect_once($arg, $res, "Howdy\n");
+    $res;
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/moved
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/moved	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/moved	(revision 21112)
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+echo "Location: http://$SERVER_NAME:$SERVER_PORT/"
+echo
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/nph-slowdata
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/nph-slowdata	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/nph-slowdata	(revision 21112)
@@ -0,0 +1,26 @@
+#!/usr/local/bin/perl
+
+# This script outputs some data slowly. It can be used to check that
+# pipelined processing of response content really works.  We use syswrite
+# so that the script will notice when the connection is broken.
+
+out("HTTP/1.0 200 OK\r
+Content-Type: text/plain\r
+\r
+");
+
+for (1..5) {
+    out("The number is now $_\n");
+    sleep(1);
+}
+exit;
+
+
+sub out
+{
+    my $data = shift;
+    my $l = length $data;
+    if (syswrite(STDOUT, $data, $l) != $l) {
+	exit 1;
+    }
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/slowread
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/slowread	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/slowread	(revision 21112)
@@ -0,0 +1,31 @@
+#!/usr/local/bin/perl
+
+# You might post large amount of data to this script.  It will read
+# it slowly.
+
+{ local($!) = 1; print "Content-Type: text/plain\n\n"; }
+
+$len = $ENV{CONTENT_LENGTH};
+
+unless ($len) {
+    system "env";
+    exit;
+}
+
+$size = 20;  # chunk size
+
+$content = '';
+$bytes = 0;
+
+sleep(1);
+while ($len > 0) {
+    $n = sysread(STDIN, $b, $size);
+    last if $n <= 0;
+    $len -= $n;
+    $bytes += $n;
+    $content .= $b;
+    sleep(1);
+}
+print "$bytes bytes read\n";
+
+	
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/test
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/test	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/test	(revision 21112)
@@ -0,0 +1,28 @@
+#!/usr/local/bin/perl
+
+$| = 1;
+print "Content-type: text/plain
+
+";
+
+if (@ARGV) {
+    print "ARGS: ";
+    print join(", ", map { $_ = qq{"$_"} } @ARGV);
+    print "\n\n";
+} else {
+    print "No command line arguments passed to script\n\n";
+}
+
+while (($key,$val) = each %ENV) {
+   print "$key=$val\n";
+}
+
+if ($ENV{CONTENT_LENGTH}) {
+    $len = $ENV{CONTENT_LENGTH};
+    while ($len) {
+	$n = sysread(STDIN, $content, $len, length $content);
+	last unless defined $n;
+	$len -= $n;
+    }
+    print "\nContent\n-------\n$content";
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/timeout
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/timeout	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/cgi-bin/timeout	(revision 21112)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+sleep 20;
+
+echo "Content-type: text/plain"
+echo
+echo "Test"
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/config.pl.dist
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/config.pl.dist	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/config.pl.dist	(revision 21112)
@@ -0,0 +1,10 @@
+package net;
+
+# Configure these for you local system
+$httpserver  = "localhost:80";
+$cgidir      = "/cgi-bin/lwp";
+
+# Used for proxy test
+$ftp_proxy = "http://localhost/";
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-get.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-get.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-get.t	(revision 21112)
@@ -0,0 +1,43 @@
+#!/usr/local/bin/perl -w
+#
+# Check GET via HTTP.
+#
+
+print "1..2\n";
+
+require "net/config.pl";
+require LWP::Protocol::http;
+require LWP::UserAgent;
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+$netloc = $net::httpserver;
+$script = $net::cgidir . "/test";
+
+$url = new URI::URL("http://$netloc$script?query");
+
+my $request = new HTTP::Request('GET', $url);
+
+print "GET $url\n\n";
+
+my $response = $ua->request($request, undef, undef);
+
+my $str = $response->as_string;
+
+print "$str\n";
+
+if ($response->is_success and $str =~ /^REQUEST_METHOD=GET$/m) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+if ($str =~ /^QUERY_STRING=query$/m) {
+    print "ok 2\n";
+} else {
+    print "not ok 2\n";
+}
+
+# avoid -w warning
+$dummy = $net::httpserver;
+$dummy = $net::cgidir;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-post.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-post.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-post.t	(revision 21112)
@@ -0,0 +1,44 @@
+#!/usr/local/bin/perl -w
+#
+# Check POST via HTTP.
+#
+
+print "1..2\n";
+
+require "net/config.pl";
+require LWP::Protocol::http;
+require LWP::UserAgent;
+
+$netloc = $net::httpserver;
+$script = $net::cgidir . "/test";
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+$url = new URI::URL("http://$netloc$script");
+
+my $form = 'searchtype=Substring';
+
+my $request = new HTTP::Request('POST', $url, undef, $form);
+$request->header('Content-Type', 'application/x-www-form-urlencoded');
+
+my $response = $ua->request($request, undef, undef);
+
+my $str = $response->as_string;
+
+print "$str\n";
+
+if ($response->is_success and $str =~ /^REQUEST_METHOD=POST$/m) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+if ($str =~ /^CONTENT_LENGTH=(\d+)$/m && $1 == length($form)) {
+    print "ok 2\n";
+} else {
+    print "not ok 2\n";
+}
+
+# avoid -w warning
+$dummy = $net::httpserver;
+$dummy = $net::cgidir;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-timeout.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-timeout.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/http-timeout.t	(revision 21112)
@@ -0,0 +1,41 @@
+#
+# Check timeouts via HTTP.
+#
+
+print "1..1\n";
+
+require "net/config.pl";
+require HTTP::Status;
+require LWP::Protocol::http;
+require LWP::UserAgent;
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+$ua->timeout(4);
+
+$netloc = $net::httpserver;
+$script = $net::cgidir . "/timeout";
+
+$url = new URI::URL("http://$netloc$script");
+
+my $request = new HTTP::Request('GET', $url);
+
+print $request->as_string;
+
+my $response = $ua->request($request, undef);
+
+my $str = $response->as_string;
+
+print "$str\n";
+
+if ($response->is_error and
+    $str =~ /timeout/) {
+    print "ok 1\n";
+}
+else {
+    print "nok ok 1\n";
+}
+
+# avoid -w warning
+$dummy = $net::httpserver;
+$dummy = $net::cgidir;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/mirror.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/mirror.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/mirror.t	(revision 21112)
@@ -0,0 +1,34 @@
+#
+# Test mirroring a file
+#
+
+require "net/config.pl";
+require LWP::Protocol::http;
+require LWP::UserAgent;
+require HTTP::Status;
+
+print "1..2\n";
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+my $url = "http://$net::httpserver/";
+my $copy = "lwp-test-$$"; # downloaded copy
+
+my $response = $ua->mirror($url, $copy);
+
+if ($response->code == &HTTP::Status::RC_OK) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+# OK, so now do it again, should get Not-Modified
+$response = $ua->mirror($url, $copy);
+if ($response->code == &HTTP::Status::RC_NOT_MODIFIED) {
+    print "ok 2\n";
+} else {
+    print "nok ok 2\n";
+}
+unlink($copy);
+
+$net::httpserver = $net::httpserver;  # avoid -w warning
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/moved.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/moved.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/moved.t	(revision 21112)
@@ -0,0 +1,34 @@
+#!/usr/local/bin/perl -w
+#
+
+print "1..1\n";
+
+require "net/config.pl";
+require LWP::Debug;
+require LWP::UserAgent;
+
+$url = "http://$net::httpserver$net::cgidir/moved";
+
+#LWP::Debug::level('+trace');
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+$ua->timeout(30);               # timeout in seconds
+
+my $request = new HTTP::Request('GET', $url);
+
+print $request->as_string;
+
+my $response = $ua->request($request, undef, undef);
+
+print $response->as_string, "\n";
+
+if ($response->is_success) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
+
+
+# avoid -w warning
+$dummy = $net::httpserver;
+$dummy = $net::cgidir;
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/proxy.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/proxy.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/net/proxy.t	(revision 21112)
@@ -0,0 +1,36 @@
+#!/usr/local/bin/perl -w
+#
+# Test retrieving a file with a 'ftp://' URL,
+# via a HTTP proxy.
+#
+
+print "1..1\n";
+
+require "net/config.pl";
+unless (defined $net::ftp_proxy) {
+    print "not ok 1\n";
+    exit 0;
+}
+
+require LWP::Debug;
+require LWP::UserAgent;
+
+#LWP::Debug::level('+');
+
+my $ua = new LWP::UserAgent;    # create a useragent to test
+
+$ua->proxy('ftp', $net::ftp_proxy);
+
+my $url = new URI::URL('ftp://ftp.uninett.no/');
+
+my $request = new HTTP::Request('GET', $url);
+
+my $response = $ua->request($request, undef, undef);
+
+my $str = $response->as_string;
+
+if ($response->is_success) {
+    print "ok 1\n";
+} else {
+    print "not ok 1\n";
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules-dbm.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules-dbm.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules-dbm.t	(revision 21112)
@@ -0,0 +1,128 @@
+
+print "1..13\n";
+
+
+use WWW::RobotRules::AnyDBM_File;
+
+$file = "test-$$";
+
+$r = new WWW::RobotRules::AnyDBM_File "myrobot/2.0", $file;
+
+$r->parse("http://www.aas.no/robots.txt", "");
+
+$r->visit("www.aas.no");
+
+print "not " if $r->no_visits("www.aas.no") != 1;
+print "ok 1\n";
+
+
+$r->push_rules("www.sn.no", "/aas", "/per");
+$r->push_rules("www.sn.no", "/god", "/old");
+
+@r = $r->rules("www.sn.no");
+print "Rules: @r\n";
+
+print "not " if "@r" ne "/aas /per /god /old";
+print "ok 2\n";
+
+$r->clear_rules("per");
+$r->clear_rules("www.sn.no");
+
+@r = $r->rules("www.sn.no");
+print "Rules: @r\n";
+
+print "not " if "@r" ne "";
+print "ok 3\n";
+
+$r->visit("www.aas.no", time+10);
+$r->visit("www.sn.no");
+
+print "No visits: ", $r->no_visits("www.aas.no"), "\n";
+print "Last visit: ", $r->last_visit("www.aas.no"), "\n";
+print "Fresh until: ", $r->fresh_until("www.aas.no"), "\n";
+
+print "not " if $r->no_visits("www.aas.no") != 2;
+print "ok 4\n";
+
+print "not " if abs($r->last_visit("www.sn.no") - time) > 2;
+print "ok 5\n";
+
+$r = undef;
+
+# Try to reopen the database without a name specified
+$r = new WWW::RobotRules::AnyDBM_File undef, $file;
+$r->visit("www.aas.no");
+
+print "not " if $r->no_visits("www.aas.no") != 3;
+print "ok 6\n";
+
+print "Agent-Name: ", $r->agent, "\n";
+print "not " if $r->agent ne "myrobot";
+print "ok 7\n";
+
+$r = undef;
+
+print "*** Dump of database ***\n";
+tie(%cat, AnyDBM_File, $file, 0, 0644) or die "Can't tie: $!";
+while (($key,$val) = each(%cat)) {
+    print "$key\t$val\n";
+}
+print "******\n";
+
+untie %cat;
+
+# Try to open database with a different agent name
+$r = new WWW::RobotRules::AnyDBM_File "MOMSpider/2.0", $file;
+
+print "not " if $r->no_visits("www.sn.no");
+print "ok 8\n";
+
+# Try parsing
+$r->parse("http://www.sn.no:8080/robots.txt", <<EOT, (time + 3));
+
+User-Agent: *
+Disallow: /
+
+User-Agent: Momspider
+Disallow: /foo
+Disallow: /bar
+
+EOT
+
+@r = $r->rules("www.sn.no:8080");
+print "not " if "@r" ne "/foo /bar";
+print "ok 9\n";
+
+print "not " if $r->allowed("http://www.sn.no") >= 0;
+print "ok 10\n";
+
+print "not " if $r->allowed("http://www.sn.no:8080/foo/gisle");
+print "ok 11\n";
+
+sleep(5);  # wait until file has expired
+print "not " if $r->allowed("http://www.sn.no:8080/foo/gisle") >= 0;
+print "ok 12\n";
+
+
+$r = undef;
+
+print "*** Dump of database ***\n";
+tie(%cat, AnyDBM_File, $file, 0, 0644) or die "Can't tie: $!";
+while (($key,$val) = each(%cat)) {
+    print "$key\t$val\n";
+}
+print "******\n";
+
+untie %cat;			# Otherwise the next line fails on DOSish
+
+unlink "$file", "$file.pag", "$file.dir", "$file.db";
+
+# Try open a an emty database without specifying a name
+eval { 
+   $r = new WWW::RobotRules::AnyDBM_File undef, $file;
+};
+print $@;
+print "not " unless $@;  # should fail
+print "ok 13\n";
+
+unlink "$file", "$file.pag", "$file.dir", "$file.db";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/rules.t	(revision 21112)
@@ -0,0 +1,138 @@
+#!/local/bin/perl
+
+=head1 NAME
+
+robot-rules.t
+
+=head1 DESCRIPTION
+
+Test a number of different A</robots.txt> files against a number
+of different User-agents.
+
+=cut
+
+require WWW::RobotRules;
+use Carp;
+use strict;
+
+print "1..26\n"; # for Test::Harness
+
+# We test a number of different /robots.txt files,
+#
+
+my $content1 = <<EOM;
+# http://foo/robots.txt
+User-agent: *
+Disallow: /private
+Disallow: http://foo/also_private
+
+User-agent: MOMspider
+Disallow:
+EOM
+
+my $content2 = <<EOM;
+# http://foo/robots.txt
+User-agent: MOMspider
+ # comment which should be ignored
+Disallow: /private
+EOM
+
+my $content3 = <<EOM;
+# http://foo/robots.txt
+EOM
+
+my $content4 = <<EOM;
+# http://foo/robots.txt
+User-agent: *
+Disallow: /private
+
+User-agent: MOMspider
+Disallow: /this
+
+User-agent: Another
+Disallow: /that
+EOM
+
+# and a number of different robots:
+
+my @tests1 = (
+	   [$content1, 'MOMspider' =>
+	    1 => 'http://foo/private' => 1,
+	    2 => 'http://foo/also_private' => 1,
+	   ],
+
+	   [$content1, 'Wubble' =>
+	    3 => 'http://foo/private' => 0,
+	    4 => 'http://foo/also_private' => 0,
+	    5 => 'http://foo/other' => 1,
+	   ],
+
+	   [$content2, 'MOMspider' =>
+	    6 => 'http://foo/private' => 0,
+	    7 => 'http://foo/other' => 1,
+	   ],
+
+	   [$content2, 'Wubble' =>
+	    8  => 'http://foo/private' => 1,
+	    9  => 'http://foo/also_private' => 1,
+	    10 => 'http://foo/other' => 1,
+	   ],
+
+	   [$content3, 'MOMspider' =>
+	    11 => 'http://foo/private' => 1,
+	    12 => 'http://foo/other' => 1,
+	   ],
+
+	   [$content3, 'Wubble' =>
+	    13 => 'http://foo/private' => 1,
+	    14 => 'http://foo/other' => 1,
+	   ],
+
+	   [$content4, 'MOMspider' =>
+	    15 => 'http://foo/private' => 1,
+	    16 => 'http://foo/this' => 0,
+	    17 => 'http://foo/that' => 1,
+	   ],
+
+	   [$content4, 'Another' =>
+	    18 => 'http://foo/private' => 1,
+	    19 => 'http://foo/this' => 1,
+	    20 => 'http://foo/that' => 0,
+	   ],
+
+	   [$content4, 'Wubble' =>
+	    21 => 'http://foo/private' => 0,
+	    22 => 'http://foo/this' => 1,
+	    23 => 'http://foo/that' => 1,
+	   ],
+
+	   [$content4, 'Another/1.0' =>
+	    24 => 'http://foo/private' => 1,
+	    25 => 'http://foo/this' => 1,
+	    26 => 'http://foo/that' => 0,
+	   ],
+
+	   # when adding tests, remember to increase
+	   # the maximum at the top
+
+	  );
+
+my $t;
+
+for $t (@tests1) {
+    my ($content, $ua) = splice(@$t, 0, 2);
+
+    my $robotsrules = new WWW::RobotRules($ua);
+    $robotsrules->parse('http://foo/robots.txt', $content);
+
+    my ($num, $path, $expected);
+    while(($num, $path, $expected) = splice(@$t, 0, 3)) {
+	my $allowed = $robotsrules->allowed($path);
+	$allowed = 1 if $allowed;
+	if($allowed != $expected) {
+	    $robotsrules->dump;
+	    confess "Test Failed: $ua => $path ($allowed != $expected)";
+	}
+	print "ok $num\n";
+    }
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/ua.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/ua.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/robot/ua.t	(revision 21112)
@@ -0,0 +1,135 @@
+$| = 1; # autoflush
+
+require IO::Socket;  # make sure this work before we try to make a HTTP::Daemon
+
+# First we make ourself a daemon in another process
+my $D = shift || '';
+if ($D eq 'daemon') {
+
+    require HTTP::Daemon;
+
+    my $d = new HTTP::Daemon Timeout => 10;
+
+    print "Please to meet you at: <URL:", $d->url, ">\n";
+    open(STDOUT, $^O eq 'MSWin32' ? ">nul" : ">/dev/null");
+
+    while ($c = $d->accept) {
+	$r = $c->get_request;
+	if ($r) {
+	    my $p = ($r->url->path_components)[1];
+	    $p =~ s/\W//g;
+	    my $func = lc("httpd_" . $r->method . "_$p");
+	    #print STDERR "Calling $func...\n";
+            if (defined &$func) {
+		&$func($c, $r);
+	    } else {
+		$c->send_error(404);
+	    }
+	}
+	$c = undef;  # close connection
+    }
+    print STDERR "HTTP Server terminated\n";
+    exit;
+}
+else {
+    use Config;
+    open(DAEMON , "$Config{'perlpath'} robot/ua.t daemon |") or die "Can't exec daemon: $!";
+}
+
+print "1..7\n";
+
+
+$greating = <DAEMON>;
+$greating =~ /(<[^>]+>)/;
+
+require URI::URL;
+URI::URL->import;
+my $base = new URI::URL $1;
+
+print "Will access HTTP server at $base\n";
+
+require LWP::RobotUA;
+require HTTP::Request;
+$ua = new LWP::RobotUA 'lwp-spider/0.1', 'gisle@aas.no';
+$ua->delay(0.05);  # rather quick robot
+
+#----------------------------------------------------------------
+sub httpd_get_robotstxt
+{
+   my($c,$r) = @_;
+   $c->send_basic_header;
+   $c->print("Content-Type: text/plain");
+   $c->send_crlf;
+   $c->send_crlf;
+   $c->print("User-Agent: *
+Disallow: /private
+
+");
+}
+
+sub httpd_get_someplace
+{
+   my($c,$r) = @_;
+   $c->send_basic_header;
+   $c->print("Content-Type: text/plain");
+   $c->send_crlf;
+   $c->send_crlf;
+   $c->print("Okidok\n");
+}
+
+$req = new HTTP::Request GET => url("/someplace", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->is_success;
+print "ok 1\n";
+
+$req = new HTTP::Request GET => url("/private/place", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->code == 403
+                and $res->message =~ /robots.txt/;
+print "ok 2\n";
+
+$req = new HTTP::Request GET => url("/foo", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->code == 404;  # not found
+print "ok 3\n";
+
+# Let the robotua generate "Service unavailable/Retry After response";
+$ua->delay(1);
+$ua->use_sleep(0);
+$req = new HTTP::Request GET => url("/foo", $base);
+$res = $ua->request($req);
+#print $res->as_string;
+print "not " unless $res->code == 503   # Unavailable
+                and $res->header("Retry-After");
+print "ok 4\n";
+
+#----------------------------------------------------------------
+print "Terminating server...\n";
+sub httpd_get_quit
+{
+    my($c) = @_;
+    $c->send_error(503, "Bye, bye");
+    exit;  # terminate HTTP server
+}
+
+$ua->delay(0);
+$req = new HTTP::Request GET => url("/quit", $base);
+$res = $ua->request($req);
+
+print "not " unless $res->code == 503 and $res->content =~ /Bye, bye/;
+print "ok 5\n";
+
+#---------------------------------------------------------------
+$ua->delay(1);
+
+# host_wait() should be around 60s now
+print "not " unless abs($ua->host_wait($base->netloc) - 60) < 5;
+print "ok 6\n";
+
+# Number of visits to this place should be 
+print "not " unless $ua->no_visits($base->netloc) == 4;
+print "ok 7\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/abs-config.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/abs-config.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/abs-config.t	(revision 21112)
@@ -0,0 +1,35 @@
+print "1..6\n";
+
+use URI::URL qw(url);
+
+# Test configuration via some global variables.
+
+$URI::URL::ABS_REMOTE_LEADING_DOTS = 1;
+$URI::URL::ABS_ALLOW_RELATIVE_SCHEME = 1;
+
+$u1 = url("../../../../abc", "http://web/a/b");
+
+print "not " unless $u1->abs->as_string eq "http://web/abc";
+print "ok 1\n";
+
+{
+    local $URI::URL::ABS_REMOTE_LEADING_DOTS;
+    print "not " unless $u1->abs->as_string eq "http://web/../../../abc";
+    print "ok 2\n";
+}
+
+
+$u1 = url("http:../../../../abc", "http://web/a/b");
+print "not " unless $u1->abs->as_string eq "http://web/abc";
+print "ok 3\n";
+
+{
+   local $URI::URL::ABS_ALLOW_RELATIVE_SCHEME;
+   print "not " unless $u1->abs->as_string eq "http:../../../../abc";
+   print "ok 4\n";
+   print "not " unless $u1->abs(undef,1)->as_string eq "http://web/abc";
+   print "ok 5\n";
+}
+
+print "not " unless $u1->abs(undef,0)->as_string eq "http:../../../../abc";
+print "ok 6\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/base.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/base.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/base.t	(revision 21112)
@@ -0,0 +1,955 @@
+#!/local/bin/perl -w
+
+use URI::URL qw(url);
+use URI::Escape qw(uri_escape uri_unescape);
+
+use Carp;
+
+# _expect()
+#
+# Handy low-level object method tester which we insert as a method
+# in the URI::URL class
+#
+sub URI::URL::_expect {
+    my($self, $method, $expect, @args) = @_;
+    my $result = $self->$method(@args);
+    $expect = 'UNDEF' unless defined $expect;
+    $result = 'UNDEF' unless defined $result;
+    return 1 if $expect eq $result;
+    warn "'$self'->$method(@args) = '$result' " .
+		"(expected '$expect')\n";
+    $self->print_on('STDERR');
+    confess "Test Failed";
+}
+
+package main;
+
+# Must ensure that there is no relative paths in @INC because we will
+# chdir in the newlocal tests.
+chomp($pwd = ($^O =~ /mswin32/i ? `cd` : `pwd`));
+for (@INC) {
+    next if m|^/| or $^O =~ /os2|mswin32/i and m|^\w:[\\/]|;
+    print "Turn lib path $_ into $pwd/$_\n";
+    $_ = "$pwd/$_";
+
+}
+
+$| = 1;
+
+print "1..8\n";  # for Test::Harness
+
+# Do basic tests first.
+# Dies if an error has been detected, prints "ok" otherwise.
+
+print "Self tests for URI::URL version $URI::URL::VERSION...\n";
+
+scheme_parse_test();
+print "ok 1\n";
+
+parts_test();
+print "ok 2\n";
+
+escape_test();
+print "ok 3\n";
+
+newlocal_test();
+print "ok 4\n";
+
+absolute_test();
+print "ok 5\n";
+
+eq_test();
+print "ok 6\n";
+
+# Let's test makeing our own things
+URI::URL::strict(0);
+# This should work after URI::URL::strict(0)
+$url = new URI::URL "x-myscheme:something";
+# Since no implementor is registered for 'x-myscheme' then it will
+# be handeled by the URI::URL::_generic class
+$url->_expect('as_string' => 'x-myscheme:something');
+$url->_expect('path' => 'something');
+URI::URL::strict(1);
+
+# Let's try to make our URL subclass
+{
+    package MyURL;
+    @ISA = URI::URL::implementor();
+
+    sub _parse {
+	my($self, $init) = @_;
+	$self->URI::URL::_generic::_parse($init, qw(netloc path));
+    }
+
+    sub foo {
+	my $self = shift;
+	print ref($self)."->foo called for $self\n";
+    }
+}
+# Let's say that it implements the 'x-a+b.c' scheme (alias 'x-foo')
+URI::URL::implementor('x-a+b.c', 'MyURL');
+URI::URL::implementor('x-foo', 'MyURL');
+
+# Now we are ready to try our new URL scheme
+$url = new URI::URL 'x-a+b.c://foo/bar;a?b';
+$url->_expect('as_string', 'x-a+b.c://foo/bar;a?b');
+$url->_expect('path', '/bar;a?b');
+$url->foo;
+$newurl = new URI::URL 'xxx', $url;
+$newurl->foo;
+$url = new URI::URL 'yyy', 'x-foo:';
+$url->foo;
+
+print "ok 7\n";
+
+# Test the new wash&go constructor
+print "not " if url("../foo.html", "http://www.sn.no/a/b")->abs->as_string
+		ne 'http://www.sn.no/foo.html';
+print "ok 8\n";
+
+print "URI::URL version $URI::URL::VERSION ok\n";
+
+exit 0;
+
+
+
+
+#####################################################################
+#
+# scheme_parse_test()
+#
+# test parsing and retrieval methods
+
+sub scheme_parse_test {
+
+    print "scheme_parse_test:\n";
+
+    $tests = {
+	'hTTp://web1.net/a/b/c/welcome#intro'
+	=> {    'scheme'=>'http', 'host'=>'web1.net', 'port'=>80,
+		'path'=>'/a/b/c/welcome', 'frag'=>'intro','query'=>undef,
+		'epath'=>'/a/b/c/welcome', 'equery'=>undef,
+		'params'=>undef, 'eparams'=>undef,
+		'as_string'=>'http://web1.net/a/b/c/welcome#intro',
+		'full_path' => '/a/b/c/welcome' },
+
+	'http://web:1/a?query+text'
+	=> {    'scheme'=>'http', 'host'=>'web', 'port'=>1,
+		'path'=>'/a', 'frag'=>undef, 'query'=>'query+text' },
+
+	'http://web.net/'
+	=> {    'scheme'=>'http', 'host'=>'web.net', 'port'=>80,
+		'path'=>'/', 'frag'=>undef, 'query'=>undef,
+		'full_path' => '/',
+		'as_string' => 'http://web.net/' },
+
+	'http://web.net'
+	=> {    'scheme'=>'http', 'host'=>'web.net', 'port'=>80,
+		'path'=>'/', 'frag'=>undef, 'query'=>undef,
+		'full_path' => '/',
+		'as_string' => 'http://web.net/' },
+
+	'http:0'
+	 => {   'scheme'=>'http', 'path'=>'0', 'query'=>undef,
+		'as_string'=>'http:0', 'full_path'=>'0', },
+
+	'http:/0?0'
+	 => {   'scheme'=>'http', 'path'=>'/0', 'query'=>'0',
+		'as_string'=>'http:/0?0', 'full_path'=>'/0?0', },
+
+	'http://0:0/0/0;0?0#0'
+	 => {   'scheme'=>'http', 'host'=>'0', 'port'=>'0',
+		'path' => '/0/0', 'query'=>'0', 'params'=>'0',
+		'netloc'=>'0:0',
+		'frag'=>0, 'as_string'=>'http://0:0/0/0;0?0#0' },
+
+	'ftp://0%3A:%40@h:0/0?0'
+	=>  {   'scheme'=>'ftp', 'user'=>'0:', 'password'=>'@',
+		'host'=>'h', 'port'=>'0', 'path'=>'/0?0',
+		'query'=>undef, params=>undef,
+		'netloc'=>'0%3A:%40@h:0',
+		'as_string'=>'ftp://0%3A:%40@h:0/0?0' },
+
+	'ftp://usr:pswd@web:1234/a/b;type=i'
+	=> {    'host'=>'web', 'port'=>1234, 'path'=>'/a/b',
+		'user'=>'usr', 'password'=>'pswd',
+		'params'=>'type=i',
+		'as_string'=>'ftp://usr:pswd@web:1234/a/b;type=i' },
+
+	'ftp://host/a/b'
+	=> {    'host'=>'host', 'port'=>21, 'path'=>'/a/b',
+		'user'=>'anonymous',
+		'as_string'=>'ftp://host/a/b' },
+
+	'file://host/fseg/fs?g/fseg'
+	# don't escape ? for file: scheme
+	=> {    'host'=>'host', 'path'=>'/fseg/fs?g/fseg',
+		'as_string'=>'file://host/fseg/fs?g/fseg' },
+
+	'gopher://host'
+	=> {     'gtype'=>'1', 'as_string' => 'gopher://host/', },
+
+	'gopher://host/'
+	=> {     'gtype'=>'1', 'as_string' => 'gopher://host/', },
+
+	'gopher://gopher/2a_selector'
+	=> {    'gtype'=>'2', 'selector'=>'a_selector',
+		'as_string' => 'gopher://gopher/2a_selector', },
+
+	'mailto:libwww-perl@ics.uci.edu'
+	=> {    'address'       => 'libwww-perl@ics.uci.edu',
+		'encoded822addr'=> 'libwww-perl@ics.uci.edu',
+		'user'          => 'libwww-perl',
+		'host'          => 'ics.uci.edu',
+		'as_string'     => 'mailto:libwww-perl@ics.uci.edu', },
+
+	'news:*'
+	=> {    'groupart'=>'*', 'group'=>'*', as_string=>'news:*' },
+	'news:comp.lang.perl'
+	=> {    'group'=>'comp.lang.perl' },
+	'news:perl-faq/module-list-1-794455075@ig.co.uk'
+	=> {    'article'=>
+		    'perl-faq/module-list-1-794455075@ig.co.uk' },
+
+	'nntp://news.com/comp.lang.perl/42'
+	=> {    'group'=>'comp.lang.perl', 'digits'=>42 },
+
+	'telnet://usr:pswd@web:12345/'
+	=> {    'user'=>'usr', 'password'=>'pswd', 'host'=>'web' },
+	'rlogin://aas@a.sn.no'
+	=> {    'user'=>'aas', 'host'=>'a.sn.no' },
+	'tn3270://aas@ibm'
+	=> {    'user'=>'aas', 'host'=>'ibm',
+		'as_string'=>'tn3270://aas@ibm/'},
+
+	'wais://web.net/db'
+	=> { 'database'=>'db' },
+	'wais://web.net/db?query'
+	=> { 'database'=>'db', 'query'=>'query' },
+	'wais://usr:pswd@web.net/db/wt/wp'
+	=> {    'database'=>'db', 'wtype'=>'wt', 'wpath'=>'wp',
+		'password'=>'pswd' },
+    };
+
+    foreach $url_str (sort keys %$tests ){
+	print "Testing '$url_str'\n";
+	my $url = new URI::URL $url_str;
+	my $tests = $tests->{$url_str};
+	while( ($method, $exp) = each %$tests ){
+	    $exp = 'UNDEF' unless defined $exp;
+	    $url->_expect($method, $exp);
+	}
+    }
+}
+
+
+#####################################################################
+#
+# parts_test()          (calls netloc_test test)
+#
+# Test individual component part access functions
+#
+sub parts_test {
+    print "parts_test:\n";
+
+    # test storage part access/edit methods (netloc, user, password,
+    # host and port are tested by &netloc_test)
+
+    $url = new URI::URL 'file://web/orig/path';
+    $url->scheme('http');
+    $url->path('1info');
+    $url->query('key words');
+    $url->frag('this');
+    $url->_expect('as_string' => 'http://web/1info?key%20words#this');
+
+    $url->epath('%2f/%2f');
+    $url->equery('a=%26');
+    $url->_expect('full_path' => '/%2f/%2f?a=%26');
+
+    # At this point it should be impossible to access the members path()
+    # and query() without complaints.
+    eval { my $p = $url->path; print "Path is $p\n"; };
+    die "Path exception failed" unless $@;
+    eval { my $p = $url->query; print "Query is $p\n"; };
+    die "Query exception failed" unless $@;
+
+    # but we should still be able to set it 
+    $url->path("howdy");
+    $url->_expect('as_string' => 'http://web/howdy?a=%26#this');
+
+    # Test the path_components function
+    $url = new URI::URL 'file:%2f/%2f';
+    my $p;
+    $p = join('-', $url->path_components);
+    die "\$url->path_components returns '$p', expected '/-/'"
+      unless $p eq "/-/";
+    $url->host("localhost");
+    $p = join('-', $url->path_components);
+    die "\$url->path_components returns '$p', expected '-/-/'"
+      unless $p eq "-/-/";
+    $url->epath("/foo/bar/");
+    $p = join('-', $url->path_components);
+    die "\$url->path_components returns '$p', expected '-foo-bar-'"
+      unless $p eq "-foo-bar-";
+    $url->path_components("", "/etc", "\0", "..", "øse", "");
+    $url->_expect('full_path' => '/%2Fetc/%00/../%F8se/');
+
+    # Setting undef
+    $url = new URI::URL 'http://web/p;p?q#f';
+    $url->epath(undef);
+    $url->equery(undef);
+    $url->eparams(undef);
+    $url->frag(undef);
+    $url->_expect('as_string' => 'http://web/');
+
+    # Test http query access methods
+    $url->keywords('dog');
+    $url->_expect('as_string' => 'http://web/?dog');
+    $url->keywords(qw(dog bones));
+    $url->_expect('as_string' => 'http://web/?dog+bones');
+    $url->keywords(0,0);
+    $url->_expect('as_string' => 'http://web/?0+0');
+    $url->keywords('dog', 'bones', '#+=');
+    $url->_expect('as_string' => 'http://web/?dog+bones+%23%2B%3D');
+    $a = join(":", $url->keywords);
+    die "\$url->keywords did not work (returned '$a')" unless $a eq 'dog:bones:#+=';
+    # calling query_form is an error
+    eval { my $foo = $url->query_form; };
+    die "\$url->query_form should croak since query contains keywords not a form."
+      unless $@;
+
+    $url->query_form(a => 'foo', b => 'bar');
+    $url->_expect('as_string' => 'http://web/?a=foo&b=bar');
+    my %a = $url->query_form;
+    die "\$url->query_form did not work"
+      unless $a{a} eq 'foo' && $a{b} eq 'bar';
+
+    $url->query_form(a => undef, a => 'foo', '&=' => '&=+');
+    $url->_expect('as_string' => 'http://web/?a=&a=foo&%26%3D=%26%3D%2B');
+
+    my @a = $url->query_form;
+    die "Wrong length" unless @a == 6;
+    die "Bad keys from query_form"
+      unless $a[0] eq 'a' && $a[2] eq 'a' && $a[4] eq '&=';
+    die "Bad values from query_form"
+      unless $a[1] eq '' && $a[3] eq 'foo' && $a[5] eq '&=+';
+
+    # calling keywords is an error
+    eval { my $foo = $url->keywords; };
+    die "\$url->keywords should croak when query is a form"
+      unless $@;
+    # Try this odd one
+    $url->equery('&=&=b&a=&a&a=b=c&&a=b');
+    @a = $url->query_form;
+    #print join(":", @a), "\n";
+    die "Wrong length" unless @a == 16;
+    die "Wrong sequence" unless $a[4]  eq ""  && $a[5]  eq "b" &&
+                                $a[10] eq "a" && $a[11] eq "b=c";
+
+    # Try array ref values in the key value pairs
+    $url->query_form(a => ['foo', 'bar'], b => 'foo', c => ['bar', 'foo']);
+    $url->_expect('as_string', 'http://web/?a=foo&a=bar&b=foo&c=bar&c=foo');
+
+
+    netloc_test();
+    port_test();
+
+    $url->query(undef);
+    $url->_expect('query', undef);
+
+    $url = new URI::URL 'gopher://gopher/';
+    $url->port(33);
+    $url->gtype("3");
+    $url->selector("S");
+    $url->search("query");
+    $url->_expect('as_string', 'gopher://gopher:33/3S%09query');
+
+    $url->epath("45%09a");
+    $url->_expect('gtype' => '4');
+    $url->_expect('selector' => '5');
+    $url->_expect('search' => 'a');
+    $url->_expect('string' => undef);
+    $url->_expect('path' => "/45\ta");
+    $url->path("00\t%09gisle");
+    $url->_expect('search', '%09gisle');
+
+    # Let's test som other URL schemes
+    $url = new URI::URL 'news:';
+    $url->group("comp.lang.perl.misc");
+    $url->_expect('as_string' => 'news:comp.lang.perl.misc');
+    $url->article('<1234@a.sn.no>');
+    $url->_expect('as_string' => 'news:1234@a.sn.no'); # "<" and ">" are gone
+    # This one should be illegal
+    eval { $url->article("no.perl"); };
+    die "This one should really complain" unless $@;
+
+    $url = new URI::URL 'mailto:';
+    $url->user("aas");
+    $url->host("a.sn.no");
+    $url->_expect("as_string" => 'mailto:aas@a.sn.no');
+    $url->address('foo@bar');
+    $url->_expect("host" => 'bar');
+    $url->_expect("user" => 'foo');
+
+    $url = new URI::URL 'wais://host/database/wt/wpath';
+    $url->database('foo');
+    $url->_expect('as_string' => 'wais://host/foo/wt/wpath');
+    $url->wtype('bar');
+    $url->_expect('as_string' => 'wais://host/foo/bar/wpath');
+
+    # Test crack method for various URLs
+    my(@crack, $crack);
+    @crack = URI::URL->new("http://host/path;param?query#frag")->crack;
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "http*UNDEF*UNDEF*host*80*/path*param*query*frag";
+
+    @crack = URI::URL->new("foo/bar", "ftp://aas\@ftp.sn.no/")->crack;
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "ftp*UNDEF*UNDEF*UNDEF*21*foo/bar*UNDEF*UNDEF*UNDEF";
+
+    @crack = URI::URL->new('ftp://u:p@host/q?path')->crack;
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "ftp*u*p*host*21*/q?path*UNDEF*UNDEF*UNDEF";
+
+    @crack = URI::URL->new("ftp://ftp.sn.no/pub")->crack;    # Test anon ftp
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    die "No passwd in anonymous crack" unless $crack[2];
+    $crack[2] = 'passwd';  # easier to test when we know what it is
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "ftp*anonymous*passwd*ftp.sn.no*21*/pub*UNDEF*UNDEF*UNDEF";
+
+    @crack = URI::URL->new('mailto:aas@sn.no')->crack;
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "mailto*aas*UNDEF*sn.no*UNDEF*aas\@sn.no*UNDEF*UNDEF*UNDEF";
+
+    @crack = URI::URL->new('news:comp.lang.perl.misc')->crack;
+    die "Cracked result should be 9 elements" unless @crack == 9;
+    $crack = join("*", map { defined($_) ? $_ : "UNDEF" } @crack);
+    print "Cracked result: $crack\n";
+    die "Bad crack result" unless
+      $crack eq "news*UNDEF*UNDEF*UNDEF*UNDEF*comp.lang.perl.misc*UNDEF*UNDEF*UNDEF";
+}
+
+#
+# netloc_test()
+#
+# Test automatic netloc synchronisation
+#
+sub netloc_test {
+    print "netloc_test:\n";
+
+    my $url = new URI::URL 'ftp://anonymous:p%61ss@håst:12345';
+    $url->_expect('user', 'anonymous');
+    $url->_expect('password', 'pass');
+    $url->_expect('host', 'håst');
+    $url->_expect('port', 12345);
+    # Can't really know how netloc is represented since it is partially escaped
+    #$url->_expect('netloc', 'anonymous:pass@hst:12345');
+    $url->_expect('as_string' => 'ftp://anonymous:p%61ss@h%E5st:12345/');
+
+    # The '0' is sometimes tricky to get right
+    $url->user(0);
+    $url->password(0);
+    $url->host(0);
+    $url->port(0);
+    $url->_expect('netloc' => '0:0@0:0');
+    $url->host(undef);
+    $url->_expect('netloc' => '');
+    $url->host('h');
+    $url->user(undef);
+    $url->_expect('netloc' => 'h:0');
+    $url->user('');
+    $url->_expect('netloc' => ':0@h:0');
+    $url->password('');
+    $url->_expect('netloc' => ':@h:0');
+    $url->user('foo');
+    $url->_expect('netloc' => 'foo:@h:0');
+
+    # Let's try a simple one
+    $url->user('nemo');
+    $url->password('p2');
+    $url->host('hst2');
+    $url->port(2);
+    $url->_expect('netloc' => 'nemo:p2@hst2:2');
+
+    $url->user(undef);
+    $url->password(undef);
+    $url->port(undef);
+    $url->_expect('netloc' => 'hst2');
+    $url->_expect('port' => '21');  # the default ftp port
+
+    $url->port(21);
+    $url->_expect('netloc' => 'hst2');
+
+    # Let's try some reserved chars
+    $url->user("@");
+    $url->password(":-#-;-/-?");
+    $url->_expect('as_string' => 'ftp://%40:%3A-%23-%3B-%2F-%3F@hst2/');
+
+}
+
+#
+# port_test()
+#
+# Test port behaviour
+#
+sub port_test {
+    print "port_test:\n";
+
+    $url = URI::URL->new('http://foo/root/dir/');
+    my $port = $url->port;
+    die "Port undefined" unless defined $port;
+    die "Wrong port $port" unless $port == 80;
+    die "Wrong string" unless $url->as_string eq
+	'http://foo/root/dir/';
+
+    $url->port(8001);
+    $port = $url->port;
+    die "Port undefined" unless defined $port;
+    die "Wrong port $port" unless $port == 8001;
+    die "Wrong string" unless $url->as_string eq
+	'http://foo:8001/root/dir/';
+
+    $url->port(80);
+    $port = $url->port;
+    die "Port undefined" unless defined $port;
+    die "Wrong port $port" unless $port == 80;
+    die "Wrong string" unless $url->as_string eq
+	'http://foo/root/dir/';
+
+    $url->port(8001);
+    $url->port(undef);
+    $port = $url->port;
+    die "Port undefined" unless defined $port;
+    die "Wrong port $port" unless $port == 80;
+    die "Wrong string" unless $url->as_string eq
+	'http://foo/root/dir/';
+}
+
+
+#####################################################################
+#
+# escape_test()
+#
+# escaping functions
+
+sub escape_test {
+    print "escape_test:\n";
+
+    # supply escaped URL
+    $url = new URI::URL 'http://web/this%20has%20spaces';
+    # check component is unescaped
+    $url->_expect('path', '/this has spaces');
+
+    # modify the unescaped form
+    $url->path('this ALSO has spaces');
+    # check whole url is escaped
+    $url->_expect('as_string',
+		  'http://web/this%20ALSO%20has%20spaces');
+
+    $url = new URI::URL uri_escape('http://web/try %?#" those');
+    $url->_expect('as_string',
+		  'http://web/try%20%25%3F%23%22%20those');
+
+    my $all = pack('c*',0..255);
+    my $esc = uri_escape($all);
+    my $new = uri_unescape($esc);
+    die "uri_escape->uri_unescape mismatch" unless $all eq $new;
+    $url->path($all);
+    $url->_expect('full_path' => '/%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13%14%15%16%17%18%19%1A%1B%1C%1D%1E%1F%20!%22%23$%25%26\'()*%2B,-./0123456789%3A%3B%3C%3D%3E%3F%40ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%7F%80%81%82%83%84%85%86%87%88%89%8A%8B%8C%8D%8E%8F%90%91%92%93%94%95%96%97%98%99%9A%9B%9C%9D%9E%9F%A0%A1%A2%A3%A4%A5%A6%A7%A8%A9%AA%AB%AC%AD%AE%AF%B0%B1%B2%B3%B4%B5%B6%B7%B8%B9%BA%BB%BC%BD%BE%BF%C0%C1%C2%C3%C4%C5%C6%C7%C8%C9%CA%CB%CC%CD%CE%CF%D0%D1%D2%D3%D4%D5%D6%D7%D8%D9%DA%DB%DC%DD%DE%DF%E0%E1%E2%E3%E4%E5%E6%E7%E8%E9%EA%EB%EC%ED%EE%EF%F0%F1%F2%F3%F4%F5%F6%F7%F8%F9%FA%FB%FC%FD%FE%FF');
+
+    # test escaping uses uppercase (preferred by rfc1837)
+    $url = new URI::URL 'file://h/';
+    $url->path(chr(0x7F));
+    $url->_expect('as_string', 'file://h/%7F');
+
+    # reserved characters differ per scheme
+
+    ## XXX is this '?' allowed to be unescaped
+    $url = new URI::URL 'file://h/test?ing';
+    $url->_expect('path', '/test?ing');
+
+    $url = new URI::URL 'file://h/';
+    $url->epath('question?mark');
+    $url->_expect('as_string', 'file://h/question?mark');
+    # XXX Why should this be any different???
+    #     Perhaps we should not expect too much :-)
+    $url->path('question?mark');
+    $url->_expect('as_string', 'file://h/question%3Fmark');
+
+    # See what happens when set different elements to this ugly sting
+    my $reserved = ';/?:@&=#%';
+    $url->path($reserved . "foo");
+    $url->_expect('as_string', 'file://h/%3B/%3F%3A%40%26%3D%23%25foo');
+
+    $url->scheme('http');
+    $url->path('');
+    $url->_expect('as_string', 'http://h/');
+    $url->query($reserved);
+    $url->params($reserved);
+    $url->frag($reserved);
+    $url->_expect('as_string', 'http://h/;%3B%2F%3F%3A%40&=%23%25?%3B%2F%3F%3A%40&=%23%25#;/?:@&=#%');
+
+    $str = $url->as_string;
+    $url = new URI::URL $str;
+    die "URL changed" if $str ne $url->as_string;
+
+    $url = new URI::URL 'ftp:foo';
+    $url->user($reserved);
+    $url->host($reserved);
+    $url->_expect('as_string', 'ftp://%3B%2F%3F%3A%40%26%3D%23%25@%3B%2F%3F%3A%40%26%3D%23%25/foo');
+
+}
+
+
+#####################################################################
+#
+# newlocal_test()
+#
+
+sub newlocal_test {
+    print "newlocal_test:\n";
+    my $pwd = ($^O eq 'MSWin32' ? 'cd' : (-e '/bin/pwd' ? '/bin/pwd' : 'pwd'));
+    my $tmpdir = ($^O eq 'MSWin32' ? $ENV{TEMP} : '/tmp');
+    $tmpdir =~ tr|\\|/|;
+
+    my $savedir = `$pwd`;     # we don't use Cwd.pm because we want to check
+			      # that it get require'd corretly by URL.pm
+    chomp $savedir;
+
+    # cwd
+    chdir($tmpdir) or die $!;
+    my $dir = `$pwd`; $dir =~ tr|\\|/|;
+    chomp $dir; $dir = uri_escape($dir, ':');
+    $url = newlocal URI::URL;
+    $url->_expect('as_string', URI::URL->new("file:$dir/")->as_string);
+
+    print "Local directory is ". $url->local_path . "\n";
+
+    # absolute dir
+    chdir('/') or die $!;
+    $url = newlocal URI::URL '/usr/';
+    $url->_expect('as_string', 'file:/usr/');
+
+    # absolute file
+    $url = newlocal URI::URL '/vmunix';
+    $url->_expect('as_string', 'file:/vmunix');
+
+    # relative file
+    chdir($tmpdir) or die $!;
+    $dir = `$pwd`; $dir =~ tr|\\|/|;
+    chomp $dir; $dir = uri_escape($dir, ':');
+    $url = newlocal URI::URL 'foo';
+    $url->_expect('as_string', "file:$dir/foo");
+
+    # relative dir
+    chdir($tmpdir) or die $!;
+    $dir = `$pwd`; $dir =~ tr|\\|/|;
+    chomp $dir; $dir = uri_escape($dir, ':');
+    $url = newlocal URI::URL 'bar/';
+    $url->_expect('as_string', "file:$dir/bar/");
+
+    # 0
+    chdir('/') or die $!;
+    $dir = `$pwd`; $dir =~ tr|\\|/|;
+    chomp $dir; $dir = uri_escape($dir, ':');
+    $url = newlocal URI::URL '0';
+    $url->_expect('as_string', "file:${dir}0");
+
+    # Test access methods for file URLs
+    $url = new URI::URL 'file:/c:/dos';
+    $url->_expect('dos_path', 'C:\\DOS');
+    $url->_expect('unix_path', '/c:/dos');
+    $url->_expect('vms_path', '[C:]DOS');
+    eval { $url->mac_path; };  # This is not allowed since we have ':' in path
+    die "Mac_path generated when it should fail." unless $@;
+    $url = new URI::URL 'file:/foo/bar';
+    $url->_expect('unix_path', '/foo/bar');
+    $url->_expect('mac_path', 'foo:bar');
+
+    # Some edge cases
+    $url = new URI::URL 'file:';
+    $url->_expect('unix_path', '/');
+    $url = new URI::URL 'file:/';
+    $url->_expect('unix_path', '/');
+    $url = new URI::URL 'file:.';
+    $url->_expect('unix_path', '.');
+    $url = new URI::URL 'file:./foo';
+    $url->_expect('unix_path', 'foo');
+    $url = new URI::URL 'file:0';
+    $url->_expect('unix_path', '0');
+    $url = new URI::URL 'file:../../foo';
+    $url->_expect('unix_path', '../../foo');
+    $url = new URI::URL 'file:foo/../bar';
+    $url->_expect('unix_path', 'bar');
+
+    # Relative files
+    $url = new URI::URL 'file:foo/b%61r/Note.txt';
+    $url->_expect('unix_path', 'foo/bar/Note.txt');
+    $url->_expect('mac_path', ':foo:bar:Note.txt');
+    $url->_expect('dos_path', 'FOO\\BAR\\NOTE.TXT');
+    $url->_expect('vms_path', '[.FOO.BAR]NOTE.TXT');
+
+    # The VMS path found in RFC 1738 (section 3.10)
+    $url = new URI::URL 'file://vms.host.edu/disk$user/my/notes/note12345.txt';
+    $url->_expect('vms_path', 'DISK$USER:[MY.NOTES]NOTE12345.TXT');
+    $url->_expect('mac_path', 'disk$user:my:notes:note12345.txt');
+
+    chdir($savedir) or die $!;
+}
+
+
+#####################################################################
+#
+# absolute_test()
+#
+sub absolute_test {
+
+    print "Test relative/absolute URI::URL parsing:\n";
+
+    # Tests from draft-ietf-uri-relative-url-06.txt
+    # Copied verbatim from the draft, parsed below
+
+    @URI::URL::g::ISA = qw(URI::URL::_generic); # for these tests
+
+    my $base = 'http://a/b/c/d;p?q#f';
+
+    $absolute_tests = <<EOM;
+5.1.  Normal Examples
+
+      g:h        = <URL:g:h>
+      g          = <URL:http://a/b/c/g>
+      ./g        = <URL:http://a/b/c/g>
+      g/         = <URL:http://a/b/c/g/>
+      /g         = <URL:http://a/g>
+      //g        = <URL:http://g>
+      ?y         = <URL:http://a/b/c/d;p?y>
+      g?y        = <URL:http://a/b/c/g?y>
+      g?y/./x    = <URL:http://a/b/c/g?y/./x>
+      #s         = <URL:http://a/b/c/d;p?q#s>
+      g#s        = <URL:http://a/b/c/g#s>
+      g#s/./x    = <URL:http://a/b/c/g#s/./x>
+      g?y#s      = <URL:http://a/b/c/g?y#s>
+      ;x         = <URL:http://a/b/c/d;x>
+      g;x        = <URL:http://a/b/c/g;x>
+      g;x?y#s    = <URL:http://a/b/c/g;x?y#s>
+      .          = <URL:http://a/b/c/>
+      ./         = <URL:http://a/b/c/>
+      ..         = <URL:http://a/b/>
+      ../        = <URL:http://a/b/>
+      ../g       = <URL:http://a/b/g>
+      ../..      = <URL:http://a/>
+      ../../     = <URL:http://a/>
+      ../../g    = <URL:http://a/g>
+
+5.2.  Abnormal Examples
+
+   Although the following abnormal examples are unlikely to occur
+   in normal practice, all URL parsers should be capable of resolving
+   them consistently.  Each example uses the same base as above.
+
+   An empty reference resolves to the complete base URL:
+
+      <>         = <URL:http://a/b/c/d;p?q#f>
+
+   Parsers must be careful in handling the case where there are more
+   relative path ".." segments than there are hierarchical levels in
+   the base URL's path.  Note that the ".." syntax cannot be used to
+   change the <net_loc> of a URL.
+
+     ../../../g = <URL:http://a/../g>
+     ../../../../g = <URL:http://a/../../g>
+
+   Similarly, parsers must avoid treating "." and ".." as special
+   when they are not complete components of a relative path.
+
+      /./g       = <URL:http://a/./g>
+      /../g      = <URL:http://a/../g>
+      g.         = <URL:http://a/b/c/g.>
+      .g         = <URL:http://a/b/c/.g>
+      g..        = <URL:http://a/b/c/g..>
+      ..g        = <URL:http://a/b/c/..g>
+
+   Less likely are cases where the relative URL uses unnecessary or
+   nonsensical forms of the "." and ".." complete path segments.
+
+      ./../g     = <URL:http://a/b/g>
+      ./g/.      = <URL:http://a/b/c/g/>
+      g/./h      = <URL:http://a/b/c/g/h>
+      g/../h     = <URL:http://a/b/c/h>
+
+   Finally, some older parsers allow the scheme name to be present in
+   a relative URL if it is the same as the base URL scheme.  This is
+   considered to be a loophole in prior specifications of partial
+   URLs [1] and should be avoided by future parsers.
+
+      http:g     = <URL:http:g>
+      http:      = <URL:http:>
+EOM
+    # convert text to list like
+    # @absolute_tests = ( ['g:h' => 'g:h'], ...)
+
+    for $line (split("\n", $absolute_tests)) {
+	next unless $line =~ /^\s{6}/;
+	if ($line =~ /^\s+(\S+)\s*=\s*<URL:([^>]*)>/) {
+	    my($rel, $abs) = ($1, $2);
+	    $rel = '' if $rel eq '<>';
+	    push(@absolute_tests, [$rel, $abs]);
+	}
+	else {
+	    warn "illegal line '$line'";
+	}
+    }
+
+    # add some extra ones for good measure
+
+    push(@absolute_tests, ['x/y//../z' => 'http://a/b/c/x/y/z'],
+			  ['1'         => 'http://a/b/c/1'    ],
+			  ['0'         => 'http://a/b/c/0'    ],
+			  ['/0'        => 'http://a/0'        ],
+			  ['%2e/a'     => 'http://a/b/c/%2e/a'],  # %2e is '.'
+			  ['%2e%2e/a'  => 'http://a/b/c/%2e%2e/a'],
+	);
+
+    print "  Relative    +  Base  =>  Expected Absolute URL\n";
+    print "================================================\n";
+    for $test (@absolute_tests) {
+	my($rel, $abs) = @$test;
+	my $abs_url = new URI::URL $abs;
+	my $abs_str = $abs_url->as_string;
+
+	printf("  %-10s  +  $base  =>  %s\n", $rel, $abs);
+	my $u   = new URI::URL $rel, $base;
+	my $got = $u->abs;
+	$got->_expect('as_string', $abs_str);
+    }
+
+    # bug found and fixed in 1.9 by "J.E. Fritz" <FRITZ@gems.vcu.edu>
+    $base = new URI::URL 'http://host/directory/file';
+    my $relative = new URI::URL 'file', $base;
+    my $result = $relative->abs;
+
+    my ($a, $b) = ($base->path, $result->path);
+	die "'$a' and '$b' should be the same" unless $a eq $b;
+
+    # Counter the expectation of least surprise,
+    # section 6 of the draft says the URL should
+    # be canonicalised, rather than making a simple
+    # substitution of the last component.
+    # Better doublecheck someone hasn't "fixed this bug" :-)
+    $base = new URI::URL 'http://host/dir1/../dir2/file';
+    $relative = new URI::URL 'file', $base;
+    $result = $relative->abs;
+    die 'URL not canonicalised' unless $result eq 'http://host/dir2/file';
+
+    print "--------\n";
+    # Test various other kinds of URLs and how they like to be absolutized
+    for (["http://abc/", "news:45664545", "http://abc/"],
+	 ["news:abc",    "http://abc/",   "news:abc"],
+	 ["abc",         "file:/test?aas", "file:/abc"],
+	 ["gopher:",     "",               "gopher:"],
+	 ["?foo",        "http://abc/a",   "http://abc/a?foo"],
+	 ["?foo",        "file:/abc",      "file:/?foo"],
+	 ["#foo",        "http://abc/a",   "http://abc/a#foo"],
+	 ["#foo",        "file:a",         "file:a#foo"],
+	 ["#foo",        "file:/a",         "file:/a#foo"],
+	 ["#foo",        "file:/a",         "file:/a#foo"],
+	 ["#foo",        "file://localhost/a", "file://localhost/a#foo"],
+	 ['123@sn.no',   "news:comp.lang.perl.misc", 'news:123@sn.no'],
+	 ['no.perl',     'news:123@sn.no',           'news:no.perl'],
+	 ['mailto:aas@a.sn.no', "http://www.sn.no/", 'mailto:aas@a.sn.no'],
+
+	 # Test absolutizing with old behaviour.
+	 ['http:foo',     'http://h/a/b',   'http://h/a/foo'],
+	 ['http:/foo',    'http://h/a/b',   'http://h/foo'],
+	 ['http:?foo',    'http://h/a/b',   'http://h/a/b?foo'],
+	 ['http:#foo',    'http://h/a/b',   'http://h/a/b#foo'],
+	 ['http:?foo#bar','http://h/a/b',   'http://h/a/b?foo#bar'],
+	 ['file:/foo',    'http://h/a/b',   'file:/foo'],
+
+	)
+    {
+	my($url, $base, $expected_abs) = @$_;
+	my $rel = new URI::URL $url, $base;
+	my $abs = $rel->abs($base, 1);
+	printf("  %-12s+  $base  =>  %s\n", $rel, $abs);
+	$abs->_expect('as_string', $expected_abs);
+    }
+    print "absolute test ok\n";
+
+    # Test relative function
+    for (
+	 ["http://abc/a",   "http://abc",        "a"],
+	 ["http://abc/a",   "http://abc/b",      "a"],
+	 ["http://abc/a?q", "http://abc/b",      "a?q"],
+	 ["http://abc/a;p", "http://abc/b",      "a;p"],
+	 ["http://abc/a",   "http://abc/a/b/c/", "../../../a"],
+         ["http://abc/a/",  "http://abc/a/",     "./"],
+         ["http://abc/a#f", "http://abc/a",      "#f"],
+
+	 ["file:/etc/motd", "file:/",            "etc/motd"],
+	 ["file:/etc/motd", "file:/etc/passwd",  "motd"],
+	 ["file:/etc/motd", "file:/etc/rc2.d/",  "../motd"],
+	 ["file:/etc/motd", "file:/usr/lib/doc", "../../etc/motd"],
+         ["file:",          "file:/etc/",        "../"],
+         ["file:foo",       "file:/etc/",        "../foo"],
+
+	 ["mailto:aas",     "http://abc",        "mailto:aas"],
+
+	 # Nicolai Langfeldt's original example
+	 ["http://www.math.uio.no/doc/mail/top.html",
+	  "http://www.math.uio.no/doc/linux/", "../mail/top.html"],
+        )
+    {
+	my($abs, $base, $expect) = @$_;
+	printf "url('$abs', '$base')->rel eq '$expect'\n";
+	my $rel = URI::URL->new($abs, $base)->rel;
+	$rel->_expect('as_string', $expect);
+    }
+    print "relative test ok\n";
+}
+
+
+sub eq_test
+{
+    my $u1 = new URI::URL 'http://abc.com:80/~smith/home.html';
+    my $u2 = new URI::URL 'http://ABC.com/%7Esmith/home.html';
+    my $u3 = new URI::URL 'http://ABC.com:/%7esmith/home.html';
+
+    # Test all permutations of these tree
+    $u1->eq($u2) or die "1: $u1 ne $u2";
+    $u1->eq($u3) or die "2: $u1 ne $u3";
+    $u2->eq($u1) or die "3: $u2 ne $u1";
+    $u2->eq($u3) or die "4: $u2 ne $u3";
+    $u3->eq($u1) or die "5: $u3 ne $u1";
+    $u3->eq($u2) or die "6: $u3 ne $u2";
+
+    # Test empty path
+    my $u4 = new URI::URL 'http://www.sn.no';
+    $u4->eq("HTTP://WWW.SN.NO:80/") or die "7: $u4";
+    $u4->eq("http://www.sn.no:81") and die "8: $u4";
+
+    # Test mailto
+    my $u5 = new URI::URL 'mailto:AAS@SN.no';
+    $u5->eq('mailto:aas@sn.no') or die "9: $u5";
+
+    # Test reserved char
+    my $u6 = new URI::URL 'ftp://ftp/%2Fetc';
+    $u6->eq("ftp://ftp/%2fetc") or die "10: $u6";
+    $u6->eq("ftp://ftp://etc") and die "11: $u6";
+}
+
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/heuristic.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/heuristic.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/heuristic.t	(revision 21112)
@@ -0,0 +1,61 @@
+print "1..14\n";
+
+use URI::Heuristic qw(uf_urlstr uf_url);
+$URI::Heuristic::DEBUG++;
+open(STDERR, ">&STDOUT");  # redirect STDERR
+
+print "not " unless uf_urlstr("http://www.sn.no/") eq "http://www.sn.no/";
+print "ok 1\n";
+
+print "not " unless uf_urlstr("/etc/passwd") eq "file:/etc/passwd";
+print "ok 2\n";
+
+print "not " unless uf_urlstr("./foo.txt") eq "file:./foo.txt";
+print "ok 3\n";
+
+print "not " unless uf_urlstr("ftp.aas.no/lwp.tar.gz") eq "ftp://ftp.aas.no/lwp.tar.gz";
+print "ok 4\n";
+
+print "not " unless uf_urlstr("C:\\CONFIG.SYS") eq "file:C:\\CONFIG.SYS";
+print "ok 5\n";
+
+if (gethostbyname("www.netscape.com")) {
+    # DNS probably work, lets run test 6..8
+
+    $URI::Heuristic::MY_COUNTRY = "no";
+    print "not " unless uf_urlstr("perl/camel.gif") eq "http://www.perl.com/camel.gif";
+    print "ok 6\n";
+
+    $URI::Heuristic::MY_COUNTRY = "uk";
+    print "not " unless uf_urlstr("perl/camel.gif") eq "http://www.perl.co.uk/camel.gif";
+    print "ok 7\n";
+   
+    $ENV{URL_GUESS_PATTERN} = "www.ACME.org www.ACME.com";
+    print "not " unless uf_urlstr("perl") eq "http://www.perl.org";
+    print "ok 8\n";
+
+} else {
+    # don't make the inocent worry
+    print "Skipping test 6-8 because DNS does not work\n";
+    for (6..8) { print "ok $_\n"; }
+
+}
+
+$ENV{URL_GUESS_PATTERN} = "";
+print "not " unless uf_urlstr("perl") eq "http://perl";
+print "ok 9\n";
+
+print "not " unless uf_urlstr("http:80") eq "http:80";
+print "ok 10\n";
+
+print "not " unless uf_urlstr("mailto:gisle\@aas.no") eq "mailto:gisle\@aas.no";
+print "ok 11\n";
+
+print "not " unless uf_urlstr("gisle\@aas.no") eq "mailto:gisle\@aas.no";
+print "ok 12\n";
+
+print "not " unless uf_urlstr("Gisle.Aas\@aas.perl.org") eq "mailto:Gisle.Aas\@aas.perl.org";
+print "ok 13\n";
+
+print "not " unless uf_url("gopher.sn.no")->scheme eq "gopher";
+print "ok 14\n";
Index: /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/rel-base.t
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/rel-base.t	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/libwww-perl-5.36/t/uri/rel-base.t	(revision 21112)
@@ -0,0 +1,24 @@
+print "1..3\n";
+
+use URI::URL;
+
+# We used to have problems with URLs that used a base that was
+# not absolute itself.
+
+$u1 = url("/foo/bar", "http://www.acme.com/");
+$u2 = url("../foo/", $u1);
+$u3 = url("zoo/foo", $u2);
+
+$a1 = $u1->abs->as_string;
+$a2 = $u2->abs->as_string;
+$a3 = $u3->abs->as_string;
+
+print "$a1\n$a2\n$a3\n";
+
+print "not " unless $a1 eq "http://www.acme.com/foo/bar";
+print "ok 1\n";
+print "not " unless $a2 eq "http://www.acme.com/foo/";
+print "ok 2\n";
+print "not " unless $a3 eq "http://www.acme.com/foo/zoo/foo";
+print "ok 3\n";
+
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Artistic
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Artistic	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Artistic	(revision 21112)
@@ -0,0 +1,131 @@
+
+
+
+
+			 The "Artistic License"
+
+				Preamble
+
+The intent of this document is to state the conditions under which a
+Package may be copied, such that the Copyright Holder maintains some
+semblance of artistic control over the development of the package,
+while giving the users of the package the right to use and distribute
+the Package in a more-or-less customary fashion, plus the right to make
+reasonable modifications.
+
+Definitions:
+
+	"Package" refers to the collection of files distributed by the
+	Copyright Holder, and derivatives of that collection of files
+	created through textual modification.
+
+	"Standard Version" refers to such a Package if it has not been
+	modified, or has been modified in accordance with the wishes
+	of the Copyright Holder as specified below.
+
+	"Copyright Holder" is whoever is named in the copyright or
+	copyrights for the package.
+
+	"You" is you, if you're thinking about copying or distributing
+	this Package.
+
+	"Reasonable copying fee" is whatever you can justify on the
+	basis of media cost, duplication charges, time of people involved,
+	and so on.  (You will not be required to justify it to the
+	Copyright Holder, but only to the computing community at large
+	as a market that must bear the fee.)
+
+	"Freely Available" means that no fee is charged for the item
+	itself, though there may be fees involved in handling the item.
+	It also means that recipients of the item may redistribute it
+	under the same conditions they received it.
+
+1. You may make and give away verbatim copies of the source form of the
+Standard Version of this Package without restriction, provided that you
+duplicate all of the original copyright notices and associated disclaimers.
+
+2. You may apply bug fixes, portability fixes and other modifications
+derived from the Public Domain or from the Copyright Holder.  A Package
+modified in such a way shall still be considered the Standard Version.
+
+3. You may otherwise modify your copy of this Package in any way, provided
+that you insert a prominent notice in each changed file stating how and
+when you changed that file, and provided that you do at least ONE of the
+following:
+
+    a) place your modifications in the Public Domain or otherwise make them
+    Freely Available, such as by posting said modifications to Usenet or
+    an equivalent medium, or placing the modifications on a major archive
+    site such as uunet.uu.net, or by allowing the Copyright Holder to include
+    your modifications in the Standard Version of the Package.
+
+    b) use the modified Package only within your corporation or organization.
+
+    c) rename any non-standard executables so the names do not conflict
+    with standard executables, which must also be provided, and provide
+    a separate manual page for each non-standard executable that clearly
+    documents how it differs from the Standard Version.
+
+    d) make other distribution arrangements with the Copyright Holder.
+
+4. You may distribute the programs of this Package in object code or
+executable form, provided that you do at least ONE of the following:
+
+    a) distribute a Standard Version of the executables and library files,
+    together with instructions (in the manual page or equivalent) on where
+    to get the Standard Version.
+
+    b) accompany the distribution with the machine-readable source of
+    the Package with your modifications.
+
+    c) give non-standard executables non-standard names, and clearly
+    document the differences in manual pages (or equivalent), together
+    with instructions on where to get the Standard Version.
+
+    d) make other distribution arrangements with the Copyright Holder.
+
+5. You may charge a reasonable copying fee for any distribution of this
+Package.  You may charge any fee you choose for support of this
+Package.  You may not charge a fee for this Package itself.  However,
+you may distribute this Package in aggregate with other (possibly
+commercial) programs as part of a larger (possibly commercial) software
+distribution provided that you do not advertise this Package as a
+product of your own.  You may embed this Package's interpreter within
+an executable of yours (by linking); this shall be construed as a mere
+form of aggregation, provided that the complete Standard Version of the
+interpreter is so embedded.
+
+6. The scripts and library files supplied as input to or produced as
+output from the programs of this Package do not automatically fall
+under the copyright of this Package, but belong to whomever generated
+them, and may be sold commercially, and may be aggregated with this
+Package.  If such scripts or library files are aggregated with this
+Package via the so-called "undump" or "unexec" methods of producing a
+binary executable image, then distribution of such an image shall
+neither be construed as a distribution of this Package nor shall it
+fall under the restrictions of Paragraphs 3 and 4, provided that you do
+not represent such an executable image as a Standard Version of this
+Package.
+
+7. C subroutines (or comparably compiled subroutines in other
+languages) supplied by you and linked into this Package in order to
+emulate subroutines and variables of the language defined by this
+Package shall not be considered part of this Package, but are the
+equivalent of input as in Paragraph 6, provided these subroutines do
+not change the language in any way that would cause it to fail the
+regression tests for the language.
+
+8. Aggregation of this Package with a commercial distribution is always
+permitted provided that the use of this Package is embedded; that is,
+when no overt attempt is made to make this Package's interfaces visible
+to the end user of the commercial distribution.  Such use shall not be
+construed as a distribution of this Package.
+
+9. The name of the Copyright Holder may not be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+				The End
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Changes
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Changes	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Changes	(revision 21112)
@@ -0,0 +1,51 @@
+Fixes in 1.0.6:
+- -f works with binary files
+- w3mir will no longer use 'Referer: (commandline)' when a URL is derived
+  from the commandline/config file.
+- More errormessage fixes
+- Fixed install instructions to warn against LWP 5.40.
+
+Fixes in 1.0.5: 
+- Better error-messaged in -quiet mode 
+- Fixed quoting. Both " and ' quotes in documents are accepted. 
+- Fixed timeout handling. 
+- Fixed broken connection handling. Incomplete documents are 
+  deleted and retried.
+- Fixed URL mangling problem. In 1.0.3 URLs like ``foo.html#bar'' would
+  be rewritten to ``#bar''. This is quite bad and makes a lot of mirrors
+  un-useable. 
+- -v now prints LWPs version as well as w3mirs version
+
+Features/bugs in 1.0.5 
+- As earlier, w3mir does not support cookies. I have received a patch for
+  basic cookie support but have not yet incorporated it due to
+  time-restraints.
+- There actually exists web servers that break the HTTP standard rather
+  thoroughly by not sending a Content-type header (this is mandatory in
+  the standard). W3mir will not discover that a HTML file is HTML in this
+  case and will not extract URLs from it.
+- If you rewrite the ending filename URLs with Apply rules (as in 
+  index.htm -> index.html) and there are links to index.htm#foo the link
+  will be incorrect after mirroring. The solution is to avoid those
+  kinds of rules. This was meant to be fixed in 1.0.3 (which broke it as
+  described above), I will fix it when I get more time. The fix is to
+  remove the #fragment part of URLs when they pass thru the apply
+  machinery. Patches accepted. 
+
+Fixes in 1.0.3: 
+- The -R/remove option to remove files is no longer more destructive
+  than intended.
+- Files with 'unsafe' characters in their filename is now saved as
+  "foo bar" instead of "foo%20bar" 
+- The -B switch works once again. 
+
+Fixes in 1.0.2: 
+- W3mir now only newline converts HTML files, all other files are handled
+  as binary. This should ensure file-integrety even when working with
+   mis-configured servers. 
+
+Fixes in 1.0.1: 
+-fa option and corresponding configuration file directive works again. 
+- Added -nnc option and a corresponding configuration file directive. It
+  disables newline conversions, which could corrupt files the Webserver
+  mis-declares. The HOWTO explains the uses of -nnc 
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL	(revision 21112)
@@ -0,0 +1,117 @@
+This is how you install w3mir on a Unix system. Please see the file
+INSTALL.w32 for install instructions for win32 systems.
+
+INSTALLING PREREQUISITES:
+
+The prerequisites are the perl packages libwww-perl, and MIME-Base64.
+The w3mir install procedure will check for their presence and complain
+if they are not there, so if you are unsure what you have you might
+skip this part for now.
+
+- Get libwww-perl from CPAN 
+  (http://www.perl.com/CPAN/modules/by-module/WWW/).  W3mir does not work
+  with LWP 5.40 and later versions.  Please get an earlier version.
+  It also does not work with the new URI package.  Use the URI package
+  enclosed in the older LWP.
+
+- Unpack it:
+
+    gzip -dc libwww-perl-*.tar.gz | tar xvBf -
+
+- Install it.  W3mir does not require the whole libwww-perl package and
+  all the packages that goes with it to work so you may follow these
+  install instructions instead of the libwww-perl install instructions
+  to get an easier install which will work with w3mir.
+
+    cd libwww-perl-*
+
+  Are you going to install it in the standard perl place (you need to
+  be the system administrator to do this)? If so:
+
+    perl Makefile.PL
+
+  If not (if you are a user):
+
+    perl Makefile.PL PREFIX=/some/dir
+
+  /some/dir could be ~ (as in your home directory).
+
+  IF you are installing an old LWP just for w3mir you can install both
+  the old LWP, the Base64 module and w3mir in a special PREFIX
+  separate from your usual install PREFIX.  This ensures that w3mir will
+  access the older version of LWP and that other software will access
+  the newer version.  If you do this make sure you install all of LWP,
+  Base64 and w3mir with the same PREFIX.
+
+  This step might produce some complaints;  A standard perl
+  installation will typically get messages about missing these
+  modules: IO::Socket, NET::FTP and MD5.  None of these need be
+  installed to run w3mir and so any error messages about these may
+  safely be ignored.
+
+    make
+    make test      (robot/ua and local/http might fail, that's OK)
+    make install
+
+  Now libwww-perl should be installed and working.
+
+- You also have to get the MIME-Base64 package
+  (http://www.perl.com/CPAN/modules/by-module/MIME/)
+
+- Unpack it:
+
+    gzip -dc MIME-Base64-*.tar.gz | tar xvBf -
+
+- Install it:
+
+    cd MIME-Base64-*
+
+  Then, as above, either
+
+    perl Makefile.PL
+
+  or
+
+    perl Makefile.PL PREFIX=/some/dir
+
+  Continue:
+
+    make
+    make test      (it should print 'All tests successful.')
+    make install
+
+  Now MIME-Base64 should be installed and working.
+
+INSTALLING W3MIR:
+
+- Get w3mir (http://www.math.uio.no/~janl/w3mir/ or from CPAN)
+
+- Unpack it:
+
+     gzip -dc w3mir-*.tar.gz | tar xvBf -
+
+- Install it:
+
+     cd w3mir-*
+
+  Are you going to install it in the standard perl place? (You
+  probably need to be the system administrator to do this.)  If so:
+
+    perl Makefile.PL
+
+  If not (if you're a user):
+
+    perl Makefile.PL PREFIX=/some/dir
+
+  /some/dir could be ~ (as in your home directory).
+
+  then
+
+     make
+     make install
+
+w3mir should now be installed correctly.  And if it's installed in
+your path you can even run it.  If you use csh or tcsh you want to run
+'rehash' before the shell will find it.
+
+1999/04/14 - Nicolai Langfeldt
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL.w32
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL.w32	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/INSTALL.w32	(revision 21112)
@@ -0,0 +1,92 @@
+This file contains installation instructions for two different ports
+of Perl to win32.  One port is known as the 'ActivePerl' port, the
+other is known as 'the GS port'.
+
+********************************* ActivePerl **********************************
+
+This was written and submitted by Roland Bauer.  Thanks.  I have
+edited them slightly.
+
+How to install w3mir on Win95/AS-port (April 1999):
+===================================================
+
++ get and install winzip from http://www.winzip.com/
++ get and install ActivePerl (now Build 509) from http://www.activestate.com/
++ get nmake.exe from 
+  ftp://ftp.microsoft.com/Softlib/MSLFILES/nmake15.exe
+  (or set switch make=nmake to make=dmake in Config.pm
+  and get dmake from GS port)
+
++ perl makefile.pl
++ nmake
++ nmake install
+
++ perl w3mir -h
++ perl w3mir http://xx.zz.zz/dir/
+
+
+Caveats:
+========
+
++ archive 1.0.6 from CPAN seems to be corrupted
+
++ &mkdir does not seem to work with dos-paths
+like 'D:/DIR1/DIR2'
+(so it is not possible to save to another harddisk)
+
+*********************************** GS Perl ***********************************
+
+These win32 installation instructions were written by Paul Hsiao and
+slightly edited by me. My thanks to Paul. It was current as of
+22-May-1998.
+
+
+WINZIP DOWNLOAD INSTRUCTIONS
+
+    The WinZip program can be accessed from the "Start" button, if the
+    program came with your system.  This software can be obtained from:
+ 
+                http://www.winzip.com/
+ 
+    Follow the installation instructions for the WinZip software as needed.
+
+ 
+PERL DOWNLOAD INSTRUCTIONS
+ 
+    The w3mir code will need access to version 5.003 or higher of the 
+    Perl interpreter and LWP library extensions in order for it to run. 
+    This software can be obtained from:
+ 
+                ftp://sunsite.uio.no/pub/languages/perl/CPAN/ports/win32/Standard
+ 
+    From this URL, select perl5.00402-bindist04-bc.tar.gz and download
+    to your PC into a folder of your choice.  In the folder, run WinZip
+    on this file.  For MS Windows 95, you need to update the registration: 
+    open C:\autoexec.bat  and add a line
+
+                path %path%;C:\Perl\bin;       
+    (C:\Perl\ is the path of the perl folder)  
+
+
+W3MIR DOWNLOAD AND INSTALLATION INSTRUCTIONS
+
+    You already have w3mir. Right? Unpack the .tar.gz file with
+    winzip.
+ 
+    Open a DOS prompt screen and go to the w3mir folder:
+ 
+        cd C:\w3mir-1.0.3
+ 
+    Type the install command for w3mir:
+ 
+        perl.exe Makefile.PL
+        make
+        make install
+ 
+    To test w3mir, type:
+ 
+    perl w3mir -h
+ 
+    If the screen displays all options (about 25 options), then 
+    w3mir is installed correctly.
+
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/MANIFEST
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/MANIFEST	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/MANIFEST	(revision 21112)
@@ -0,0 +1,15 @@
+Artistic
+Changes
+INSTALL
+INSTALL.w32
+MANIFEST
+Makefile.PL
+README
+example.cfg
+multiscope.cfg
+w3mir.PL
+w3mir-HOWTO.html
+w3mfix.PL
+htmlop.pm
+w3http.pm
+w3pdfuri.pm
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Makefile.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Makefile.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/Makefile.PL	(revision 21112)
@@ -0,0 +1,116 @@
+# -*-perl-*-
+# Makefile.PL for w3mir.
+# Note that the file 'MANIFEST' is quite significant in this context
+#
+# Notes:  Should unpack any tar.gz files in current directory and install
+#   them with identical arguments to Makefile.PL to this invocation.
+#   This is especially for making installation of w3mir easy 
+#   by installing libwww-perl and mime-base64 for the user before
+#   installing w3mir.  Cool eh?  That would require some changes.
+#
+# - janl 18/4/1997
+#
+# 19/04/99 janl: Added 'clean' target on tip from  Lupe Christoph
+#	and LWP max-version check to the LWP detection.
+# 
+
+# First just write the Makefile
+use ExtUtils::MakeMaker;
+
+WriteMakefile(
+    NAME	=> 'w3mir',
+    VERSION_FROM => 'w3mir.PL', # finds $VERSION
+    EXE_FILES => [ 'w3mir', 'w3mfix' ],
+    clean => { FILES => 'w3mir w3mfix' },
+    dist => { COMPRESS => 'gzip -f9', SUFFIX => 'gz' },
+);
+
+print "\n";
+
+# Find value of INSTALLSITELIB
+
+&read_makefile;
+$islib = resolve_make_var('INSTALLSITELIB');
+
+# Check for prerequisites
+
+$| = 1;
+my $missing_modules = 0;
+
+# libwww-perl
+print "Checking for libwww-perl:";
+
+eval "use lib '$islib'; require LWP;";
+
+if (!defined($LWP::VERSION)) {
+  print ' failed
+
+libwww seems to be missing.
+
+';
+  $missing_modules++;
+  sleep(2);
+} elsif ($LWP::VERSION<5.08 or $LWP::VERSION>=5.40) {
+  print ' failed
+
+libwww is version ',$LWP::VERSION,', I need version 5.08 to 5.39.
+Please see the INSTALL file.
+
+';
+  $missing_modules++;
+  sleep(2);
+} else {  
+  print " ok\n\n";
+}
+
+# MIME::Base64
+print "Checking for MIME::Base64:";
+
+eval "use lib '$islib'; require MIME::Base64;";
+
+if (!defined(&MIME::Base64::encode)) {
+  print ' failed
+
+MIME::Base64 seems to be missing.
+
+';
+  $missing_modules++;
+  sleep(2);
+} else {
+  print " ok\n\n";
+}
+
+
+print 'Please see the INSTALL file for instructions on how to get the
+missing/too old modules.
+
+' if $missing_modules;
+
+
+sub resolve_make_var ($) {
+
+  my($var) = shift @_;
+  my($val) = $make{$var};
+
+#  print "Resolving: ",$var,"=",$val,"\n";
+  
+  while ($val =~ s~\$\((\S+)\)~$make{$1}~g) {}
+#  print "Resolved: $var: $make{$var} -> $val\n";
+  $val;
+}
+
+
+sub read_makefile {
+
+  open(MAKEFILE, 'Makefile') || 
+    die "Could not open Makefile for reading: $!\n";
+
+  while (<MAKEFILE>) {
+    chomp;
+    next unless m/^([A-Z]+)\s*=\s*(\S+)$/;
+    $make{$1}=$2;
+#    print "Makevar: $1 = $2\n";
+  }
+
+  close(MAKEFILE)
+}
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/README
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/README	(revision 21112)
@@ -0,0 +1,133 @@
+PROPAGANDA:
+
+See http://www.math.uio.no/~janl/w3mir/ for propaganda.
+
+--------------------------------------------------------------------------
+Q: Where can I get a new version of w3mir?
+A: The w3mir homepage is at http://www.math.uio.no/~janl/w3mir/.
+   W3mir is also distributed on CPAN: http://www.perl.com/
+
+Q: Are there any mailing lists?
+A: Yes, see below.
+
+Q: Should I subscribe to any of the mailinglists?
+A: Yes, if you use w3mir at all you should subscribe to 
+   w3mir-info@usit.uio.no, send e-mail to janl@math.uio.no to be
+   subscribed.
+
+Q: I found a bug!
+A: See below.
+
+Q: How do it...
+A: There is a w3mir-HOWTO.html file in the distribution.  Read it.
+
+--------------------------------------------------------------------------
+BUGS:
+
+- -lc switch does not work too well.
+
+Please see below for how to report bugs.
+
+--------------------------------------------------------------------------
+FEATURES (NOT bugs):
+
+- URLs with two /es ('//') in the path component does not work
+  as some might expect.  According to my reading of the http/url spec.
+  it is an illegal construct, which is a Good Thing, because I don't
+  know how to handle it if it's legal.
+- If you start at http://foo/bar/ then index.html might be gotten twice.
+- Some documents point to a point above the server root, i.e.,
+  http://some.server/../stuff.html.  Netscape, and other browsers, in
+  defiance of the URL standard documents will change the URL to
+  http://some.server/stuff.html.  W3mir will not.
+
+--------------------------------------------------------------------------
+MAIL LISTS, REPORTING BUGS:
+
+Please send ideas (see todo lists further down please) and bug reports
+to w3mir-core@usit.uio.no, please include URL and command line/config
+file that triggered the bug. w3mir-info@usit.uio.no is mainly used for
+announcing new versions or bugs to w3mir users. You will get more
+information following that list than anywhere else. And it's very low
+volume. To subscribe to these lists email janl@math.uio.no. The
+w3mir-core list is intended for w3mir hackers only.
+
+--------------------------------------------------------------------------
+COPYRIGTHS:
+
+w3mir, w3http.pm, w3pdfuri.pm and htmlop.pm are free but it is
+Copyrighted by the various involved hackers.  If you want to copy,
+hack or distribte w3mir you can do that providing you comply with the
+'Artistic License' enclosed in the w3mir distribution in the file
+named Artistic.
+
+--------------------------------------------------------------------------
+CREDITS:
+
+- Oscar Nierstrasz: Wrote htget
+- Gorm Haug Eriksen: Started w3mir on the foundations of htget,
+	contributed code later.
+- Nicolai Langfeldt: Learning from Oscar and Gorms mistakes, rewrote
+  everything.
+- Chris Szurgot: Adapting to win32, good ideas and code contribs, 
+  Debugging.  And criticism.
+- Rik Faith: Uses w3mir extensively, not shy about complaining and
+  commenting and suggesting.  
+- The libwww-perl author(s) that made adding some new featres
+  ridicolously easy.
+
+--------------------------------------------------------------------------
+TODO LIST:
+
+* TODO, version 1.1:
+
+Some of these are speculative, some others are very useful.
+
+- CSS parsing/support at the same level as HTML
+- Full support for APPLETS/OBJECT tags.
+- Alias rules.  These would enable w3mir to map ukoln.bath.ac.uk and 
+  bubl.bath.ac.uk to www.ukoln.ac.uk and know that the objects contained
+  in these are all the same.  Another use would be to mirror from a mirror
+  instead of the _real_ site, since the original site to which you have
+  references are on a slow link while the mirror is on a fast link.
+- FTP support (easy if through a http style ftp proxy, but is that what
+  we want?)
+- SHTTP/SSL support
+- Add a 3rd argument to URL/Also that denotes the depth of documents to
+  be retrived under each, this would autimatically generate fetch/ignore
+  rules.  How well these work would be dependent on the order of the
+  URL/Also directives though.  Must be documented.
+- Retrive recursively until N-th order links are reached.  This
+  differs siginificantly from directory recursion which we do now.  When
+  this is done w3mir should also know the difference between inline and
+  external links, inline links should always be retrived.  Trivia question:
+  What order is needed to reach every findable document on the web from
+  Yahoo?
+- Integrate with cvs or rcs (or other version controll system) to make
+  retriver able to reproduce mirrored site for any given date.
+- Some text processing:  Adding and removing text/sgml comments when suitable
+  options and tags are found.
+- Put retrival date-stamps as comments in html files, to document the when
+  and how of how this document was retrived.
+- Example: If you're mirroring a site primarily to get to the papers, but
+  the site has n versions of each paper: foo.ps.gz, foo.ps.Z, foo.dvi.gz
+  foo.dvi.Z, foo.tar.gz, foo.zip and you only need one version.  Implement
+  a way to get only one version of documents provided in multipele versions,
+  something like multi axis preference list to get only the most attractive
+  version of the doc.
+- Logging of retrivals to file, need to change every print to a functioncall.
+- Your suggestion here.
+
+* TODO, http related
+- Use Keep-alive.  Then we should probably stop using 30 second pauses
+  between document retrivals.
+- HTTP/1.1?  HTTP/1.1 servers should do keep-alive even with 1.0 requests.
+- Separate quenes for each server, interleave requests.
+- Authentication is only done if challenged now.  If a document in a
+  directory needs authentication it is very likely that all other documents
+  in that directory and subdirectories require authentication.
+
+* If perl gets threads:
+- Make the retrival and analysis engines separate threads, and have each
+  one retrival thread pr. method/server/port and do paralell retrivals.
+- This makes http/1.1 easier too.
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/example.cfg
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/example.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/example.cfg	(revision 21112)
@@ -0,0 +1,120 @@
+# This is a example w3mir config file.  The file and options are fully
+# documented in the w3mir man page, and a lot is spelled out in the 
+# w3mir-HOWTO.  This file might not be completely updated, but the man
+# page should be.
+#
+# Comment lines start with '#'.  Comments are not allowed at end of
+# lines.  The keywords are not casesensitive, but some settings are,
+# notably fetch/ignore/apply settings, as well as the urls given.
+#
+# Available options: recurse, no-date-check, only-nonexistent, list-urls,
+# lowercase.  It is not default to recurse.  Options must be the
+# first directive to appear in the config file.
+# The options directive needs to be the first directive in the config file
+# and there can only be one.
+Options: recurse
+# Multipele options:
+# Options: recurse, only-nonexistent
+# This directive was previously named 'Fetch-options'
+#
+# Start here:
+URL: http://www.stud.ifi.uio.no/~janl/
+#
+#
+# How much noise to make, one of quiet, brief or chatty.  Default is brief
+#Verbosity: brief
+#
+# Initial referer.  Some servers are very picky....
+#Initial-Referer: http://foo.bar/
+#
+# Headers to disable for anonymitys sake: 
+#Disable-headers: referer, user
+#
+# Nicenes, robotrules can be true or false
+#Robot-Rules: true
+# Default pause between http connections is now 0 seconds 
+# Pause: 0
+# Pause between retries, in seconds.  If w3mir runs as a once-a-day thing
+# this can and should be high.  I suggest at least one hour (3600 seconds).
+#Retry-Pause: 600
+# How many times a failed retrival is retried.
+#Retries: 3
+#
+# URL selection.  Order is significant, first Fetch/Ignore rule overrides
+# later rules.  Supply as many as you want.  If not matched by ignore rules
+# a doc is gotten.  
+#Fetch: *.html
+#
+# Ignore urls we can't copy anyway.  Actually w3mir can copy
+# many cgi things with no problems.  But not always.
+Ignore: *.cgi
+Ignore: *-cgi
+Ignore: *.map
+#
+# RE variants are full perl REs.  Mix freely with the others.  These
+# must be legal perl REs, including the starting m and any
+# options you want, like 'i'.
+#
+#Fetch-RE: m/jpeg/
+#Ignore-RE: m/\.gif$/
+#
+# URL manipulation.  _All_ rules are applied in order.  If lowercasing
+# options is given that is done first.  This must be a legal perl
+# substitution RE.  Including the starting s and any options you want,
+# like 'g' and 'i'.
+#
+# This changes the name of all .gif files to .fif
+#Apply: s/\.gif$/.fif/
+#
+# HTTP Proxy server:
+#HTTP-Proxy: www-cache.uio.no:3128
+#
+# HTTP Proxy options: 
+# - 'revalidate': Force proxy to revalidate cached documents.
+# - 'refresh': Force proxy to get cached documents from server _again_.
+# - 'no-store': Use if the contents of the documents is sensitive.
+# - 'no-pragma': Force use of HTTP/1.1 'Cache-Control:' header.
+# If only 'refresh' is specified the default is to use the 'Pragma: no-cache'
+# HTTP/1.0 header, this can be overruled with the 'no-pragma' option.
+# You can be fairly sure any proxy will understand 'refresh', ONLY HTTP/1.1
+# proxies will understand 'revalidate' and 'no-store'.
+#proxy-options: refresh
+#
+# HTTP Proxy authentication.  The mechanism  is simpleminded, we either need
+# to authenticate ourselves with the proxy or not, and there can only be
+# one user/password tuple, domain/realm information is ignored...
+# HTTP-Proxy-user: foo
+# HTTP-Proxy-passwd: bar
+#
+# What to do with files, one of save, stdout or forget.
+#File-disposition: save
+#
+# Debug level
+#Debug: 0|1|2
+#
+# Umask, if not the default
+#Umask: 077
+#
+# HTML document editing options
+#Header: Mirrored from http://www.ifi.uio.no/~janl/ts/
+#Remove-nomirror: true
+#
+# Authentication:
+#Auth-domain: server:port/realm
+#Auth-user: me
+#Auth-passwd: my_password
+#
+# You can use '*' in place of server:port and/or realm:
+#Auth-domain: */*
+#Auth-user: otherme
+#Auth-passwd: otherpassword
+#
+
+# 'Fixup' is done by the external w3mfix program.  If you want a
+# mirror that's browseable in a filesystem (as opposed to via a http
+# server) just put this in:
+Fixup: run
+
+# Related is this, what is the name of the file the http server serves
+# when a directory URL is requested?  w3mir default is index.html.
+#Index-name: welcome.html
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/htmlop.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/htmlop.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/htmlop.pm	(revision 21112)
@@ -0,0 +1,656 @@
+# -*-perl-*-
+# htmlop.pl: Do operations on html documents.
+$VERSION=0.2.4;
+#
+# Original source from Bjørn Borud, without it I would not have atempted
+# this. In this incarnation it bears no resemblance to Bjørns code.
+# - Nicolai Langfeldt 18/11/95.
+# 
+# htmlop.pl does operations on html files, possebly many at a time.
+# Operations:
+# - Absolitify urls
+# - Relativify urls
+# - Gather list of urls
+# - Callback with url
+# - Canonify document w.r.t. SGML.
+#
+# Authors:
+# - Nicolai Langfeldt (janl@ifi.uio.no)
+# - Chris Szurgot (szurgot@itribe.net)
+#
+# Changes:
+# janl    18/11/95 - Initial version
+# szurgot 09/02/96 - Code in htmlop'process to remove <BASE> Tag if we are 
+#                    returning the form. An unchanged base destroys local 
+#                    fragments.
+# janl    22/02/96 - Added URLSUB functions.  <BASE> will only be removed from
+#		     the returned doc, never from the origninal doc.
+# janl    16/05/96 - Removed I* options and added single NODOC option/modifier
+#		     to replace them.
+# janl	  09/09/96 - Added URLPROC and NREL opcodes, for a better URL
+#		     processing model. -> 0.1.7
+# janl    24/09/96 - Various cosmetics, no longer inserting a !SGML tag,
+#		     only !DOCTYPE and HTML tags. -> 0.1.8
+# janl    11/10/96 - URLs in HTML 3.2 tags are now found.
+# janl     6/11/96 - Added netscape SCRIPT tag.
+# janl    20/11/96 - 'BORDER' is different from 'BORDER=0' -> 0.1.10
+# janl    06/11/96 - Added userdata argument to URLPROC.
+#   			Added SAVEURLS and USESAVED opcodes. -> 0.1.11
+# janl    11/04/97 - Changed to URI::URL and got into strict harness
+# janl	  13/04/97 - Fixed comment processing, extended it to processing
+#			directives. Bug reported by Chris Johnson. -> 0.1.12
+# janl    08/04/97 - Made ISMAP into ISMAP=, bad! -> 0.1.13
+# janl	  23/05/97 - Treating <script> and <style> as verbatim -> 0.1.14
+# janl	  06/06/97 - Canonify: <!SGML tag confused it and !DOCTYPE was mangled
+#			-> 0.1.15
+# janl	  30/07/97 - Now supporting <BASE> tag.
+#		   - Added TAGCALLBACK opcode -> 0.1.16
+# janl    16/10/97 - Empty string must be quoted, reported by Bart Barenburg
+#			-> 0.1.17
+# janl	  01/12/97 - More HTML URL tags/attributes recognized (Greg Lindhorst) 
+# janl	  13/12/97 - Can't delete the whole <BASE> tag.  Netscape has
+#		     extended it for frame use.  So only delete the HREF
+#		     attr -> 0.1.18
+# janl    01/01/98 - Realized applet/object support can't work and changed
+#		     so it's able to work, if conditions are right.
+# janl    04/01/98 - Added html 4.0 tags and attributes.
+# janl    08/02/98 - Hacked for speed.  Went from 43s to 9s on a 170K
+#		     document -> 0.2.  Thanks to Rune Frøysa who taunted me.
+# janl	  09/04/98 - More tolerant about what constitutes a newline -> 0.2.1
+# janl	  09/05/98 - Export %isdir -> 0.2.2
+# janl    13/04/99 - Remove leading /../ sequences in path component in
+#		     ABS code. -> 0.2.3
+# janl    28/05/99 - The code was buggy, now it's not. -> 0.2.4
+
+package htmlop;
+
+use URI::URL;
+
+use strict;
+# Global variables
+use vars qw($ABS $REL $LIST $CANON $URLSUB $NODOC $URLPROC $NREL);
+use vars qw($SAVEURL $USESAVED $TAGCALLBACK $debug %isdir);
+
+# These are for the smartrel routines
+my $url_origin;
+my $doc_top;
+my $doc_top_re;
+my $choped_url_or;
+
+# HTML operation codes for process_html.  The first argument is the
+# html document to do operations on.  It will not be changed.  If
+# NODOC is not passed a new document edited as specified will be
+# returned.  No more than one document will be returned by one
+# invocation of htmlop.
+
+$ABS = 1;			# Absolutify urls. Arg: Origin. The
+                                # ABS function absolutifies URLS,
+                                # assuming they are relative to the
+                                # argument.
+				# If a <BASE> tag is found the origin
+				# given by it will be used instead of
+				# the arguemnt given.
+
+$REL = 2;			# Relativize all urls. Arg: Origin.
+                                # The REL function simply removes the
+                                # argument string from any urls
+                                # matching it.  For this reason the
+                                # origin string is interprated as a
+                                # RE, which may have unexpected
+                                # results, unless RE specials are
+                                # escaped, which they should be.
+
+$LIST = 3;			# List all urls, Ret: urls
+
+$CANON = 4;			# See that missing opening (and
+                                # ending) SGML tags are injected.
+
+$URLSUB = 5;			# Do regular expression substitution
+                                # on urls.  Arg: RE, substitute.
+
+$NODOC = 6;			# Do not return a rebuilt document.
+                                # Saves memory, and time
+
+# NOTE: I SUSPECT URLPROC IS BROKEN IF A BASE TAG APPEARS IN THE TEXT.
+# THE URL PROCESSOR NEEDS TO BE PASSED htmlop::process' IDEA OF WHAT THE
+# BASE URL IS.
+
+$URLPROC = 7;			# Apply function on urls (process
+                                # urls).  Arg: Pointer to function to
+				# apply, userdata.  The function will
+				# be passed the url, modified by any
+				# previous operations and the
+				# userdata.  The function must return
+				# the new url.
+
+$NREL = 8;			# New relativisation function, works
+                                # much better than the old one. Arg:
+                                # Origin, Top.
+
+$SAVEURL = 9;			# Save urls in tag with modified name.
+				# Arg: attribute prefix.
+				# Example: <a href=foo> becomes
+				# <a href=foo w3mir-href=foo> if no other
+				# processing of the url is done.
+
+$USESAVED = 10;			# Use saved urls. Arg: attribute prefix
+
+$TAGCALLBACK = 11;		# Procedure to call for each Tag.
+				# Args: procedure, userdata (one item)
+
+				# Args to procedure: userdata, Base
+				# URL, tag name, reference to array of
+				# URL attributes, reference to hash of
+				# all attributes.  The base url is
+				# derived from the one used in ABS or
+				# the BASE tag.
+
+$debug=0;			# Debugging level in this package
+
+# process_html returns a array.  The first component of the array is
+# the new html document. The rest of the array is the urls.  If a
+# document is not to be returned a empty string is returned. If a url
+# list is not to be returned a empty array is returned.
+
+# HERE BE DRAGONS:
+
+# Where to find URLs in various tags.  The second compoent is a array
+# reference.
+
+my(%urls) = (
+	HEAD    => [ 'PROFILE' ],
+        BLOCKQUOTE => [ 'CITE' ],
+        Q 	=> [ 'CITE' ],
+        INS	=> [ 'CITE' ],
+	DEL	=> [ 'CITE' ],
+	A	=> [ 'HREF' ] ,
+	IMG	=> [ 'SRC' ,'LOWSRC' ,'USEMAP', 'LONGDESC' ] ,
+	EMBED	=> [ 'SRC' ],
+	FRAME	=> [ 'SRC', 'LONGDESC' ],
+        IFRAME  => [ 'SRC', 'LONGDESC' ],
+	BODY	=> [ 'BACKGROUND' ],
+	AREA	=> [ 'HREF' ],
+	LINK	=> [ 'HREF' ],
+
+	# The APPLET and OBJECT tags do not fit into my model for URL
+	# manipulation.  Just looking at CODEBASE might work, if the
+	# URL it names is a browseable directory...
+	APPLET	=> [ 'CODEBASE' ],  # If the codebase dir is browseable
+	OBJECT  => [ 'CODEBASE' ],  # Ditto.  Can't handle DATA attribute now
+
+	INPUT	=> [ 'SRC', 'USEMAP' ],
+	MAP	=> [ 'HREF' ],
+	SCRIPT	=> [ 'SRC', 'FOR' ],# 'FOR's semantics is not defined, the
+				    # attribute is just reserved for possible
+				    # future use...
+	BGSOUND => [ 'SRC' ],
+	FORM	=> [ 'ACTION' ],    # Is this asking for trouble?
+				    # Maybe it should just be absolutized...
+	     			    # On the other hand: It's CGI...
+	);
+
+my(%relative) = (
+	# Identify URL attributes containing urls that are relative to
+	# the named URL attribute.   When processing these they should
+        # be absolitized and then relativized relative to the BASE attribute.
+        # This is just window dressing for now; it is not used for anything.
+
+	# ARCHIVE is really a URI _list_.
+	CODEBASE => [ 'CLASSID', 'DATA', 'CODE', 'ARCHIVE' ],
+        );
+
+%isdir = (
+	# These tags refer to directories:
+	CODEBASE => 1
+	);
+
+# Tags that enclose bits we want to leave absolutely alone because they
+# are not very like HTML, or some such.
+
+# The material between the start and end tags is copied with no
+# processing at all.  The end tag is left to be processed.
+# The endtag match is case insensitive.
+my(%verbatim) = (
+        SCRIPT	=> quotemeta('</SCRIPT>'),	# Embeded scripts
+	STYLE	=> quotemeta('</STYLE>'),	# Embeded stylesheet
+	);
+
+# These are the functions that pick the HTML to pieces.  It will not
+# work esp. good on a random SGML document since the HTML application
+# of SGML has simpler quoting than it might.
+
+sub gettoken {
+  # Get one token from the argument, removing it from the argument.
+  # BUG: There should be whitespace at the end of the examined string.
+  my($c,$token,$i);
+  
+  # Skip whitespace and newlines
+  return '' unless defined(@_) && defined($_[0]);
+  $_[0] =~ s/^[\r\n\s]*//;
+  
+  return '' if ($_[0] eq '');
+  
+  $c = substr($_[0],0,1);
+  substr($_[0],0,1)='';
+  
+  if ($c eq '"' || $c eq "\'") { # Quoted material
+    $i=index($_[0],$c);
+    # End-quote missing, just gobble the rest of the doc
+    $i=length($_[0]) if $i == -1;
+    # Extract and remove token
+    $token=substr($_[0],0,$i);
+    substr($_[0],0,$i+1)='';
+  } elsif ($c eq '=') {
+    $token='=';
+  } else {			# Non-quoted material, ends in whitespace or =
+    $_[0] =~ m/[=\s\n\r]/;
+    $_[0] = $&.$';
+    $token=$c.$`;
+  }
+  # print "Token: '$token'\t\tRest: '",$_[0],"'\n";
+  return $token;
+}
+
+
+sub tagtoken {
+  # Pick the tag to pieces (also knonw as tokens). Return an
+  # associative array of attributes.  The attribute-names are changed
+  # to uppercase.  The attribute-values are left as is.
+  my($tok,$lasttok,%tokens);
+  # Append a space, gettoken needs it - silly? Yes! 
+  # Change it to test on boundrary things rather than ...?
+  $_[0].=' ';
+
+  $lasttok='';
+  while (1) {
+    last if (($tok=uc &gettoken($_[0])) eq '');
+
+    if ($tok eq '=') {
+#      print STDERR " -bad html-" if ($lasttok eq '');
+      $tokens{$lasttok}.=&gettoken($_[0]);
+      print STDERR "STORED: $lasttok = ",$tokens{$lasttok},"\n" if $debug;
+    } else {
+      $tokens{$tok}=undef;
+      $lasttok=$tok;
+    }
+  }
+  return %tokens;
+}
+
+
+sub gettag {
+  # Pick out the following things from the remaining html doc:
+  # Everything leading up to the first tag.  The first tag, and its
+  # contents.  Modify @_ directly to reduce number of copies of
+  # possebly huge documents kept in memory at once.  Return the body,
+  # the tag name, and the attributes (associative array)
+  my(%attr,$tagn,$tagc,$body,$tag,$doc);
+
+  $doc=\$_[0];
+
+  my($start,$end,$length);
+  
+  $start=index($$doc,'<');
+
+  if ($start<$[) {
+    # EOF
+    $body=$$doc;
+    $$doc='';
+    return ($body,'',());
+  }
+
+  $end=index($$doc,'>',$start+1);
+  
+  if ($end<$[) {
+    # This sucks, found no end of the tag...
+    $body=$$doc;
+    $$doc='';
+    return ($body,'',());
+  }
+
+  $length=$end-$start-1;
+
+  $body=substr($$doc,0,$start);
+  $tag=substr($$doc,$start+1,$length);
+
+  # This shortens the string in each itteration, some kind of mechanism
+  # to do it once in a while would speed things up further.  HOWEVER, when
+  # I tried to code this all I got was a _nasty_ memory leak.
+  substr($$doc,0,$end+1)='';
+
+#  print STDERR "------\n";
+  
+#  print STDERR "BODY: /$body/\n";
+  print STDERR "COMPLETE TAG: /$tag/\n" if $debug;
+#  print STDERR "REST: /",substr($$doc,0,20),"/\n";
+
+#  print STDERR "------\n";
+
+  # Examine tag contents
+  if ($tag =~ /^([!?]--)/ || $tag =~ /^(!\w+)/) {
+    # Comment or processing dicective, handle specially
+    $tagn=$&;
+    $tagc=$';
+    return ($body,$tagn,("$tagc",undef));
+  }
+
+  # Everything else
+  ($tagn,$tagc) = split(/[\s\n\r]+/,$tag,2);
+  $tagn="\U$tagn";
+
+  return ($body,$tagn,()) if !defined($tagc);
+  
+  return ($body,$tagn,&tagtoken($tagc));
+}
+
+
+# This is meant for general consumption:
+
+sub process {
+  # Process a html file.  Into one end you put a html file.  Out
+  # of the other end you get something dependent on the operations
+  # you specified.
+  # I cannot gobble my arguments. I need to examine them several times.
+  
+  my($arg,$doc,$i,$retdoc,$canon,$newdoc,@urllist,$Q,$cont);
+  my($origin,$baseurl);
+  
+  $origin=$baseurl='';
+
+  $retdoc=$canon=0;
+  
+  # Get the document from the argument list
+  $doc=shift(@_);
+  
+  $retdoc=1;
+  $i=0;
+  # Argument checking
+  while (defined($arg=$_[$i++])) {
+    if (! ($arg =~ /^\d+$/)) {
+      print STDERR "ERROR IN HTMLOP::process:\n";
+      print STDERR "Args: ",join(',',@_),"\n";
+      print STDERR "ARG: $arg is not a opcode.\n";
+      exit(1);
+    }
+    if ($arg == $ABS) {
+      $baseurl=$origin=$_[$i++];
+    } elsif ($arg == $REL || $arg == $SAVEURL || $arg == $USESAVED) {
+      $i++; # Skip one arg
+    } elsif ($arg == $URLSUB || $arg == $NREL || $arg == $URLPROC ||
+	     $arg == $TAGCALLBACK ) {
+      $i += 2; # Skip two args
+    } elsif ($arg == $LIST) { 
+      # do nothing
+    } elsif ($arg == $CANON) {
+      $canon=1;
+    } elsif ($arg == $NODOC) {
+      $retdoc=0;
+    } else {
+      die "htmlop: Incorrect invocation of html_process\n";
+    }
+  }
+  
+  my($endhtml)=0;	# Have we seen </html>?
+  my($SGML)=1;		# 1: !DOCTYPE not seen, 2: !DOCTYPE seen, -1: it's OK
+
+  # These are used to store the tag components
+  my($textpart);
+  my($tagname);
+  my(%attrval);
+
+  my($wholetag);	# The whole tag, put together again
+  my($moretag);		# TMP storage of tag attributes
+  my($attr);		# Looping thru attributes, misnomer.
+  my($RE);
+  my($subst);
+  my($prefix);
+  my($fun);		# Function to apply to url
+  my($verbatim)='';
+  my($url_o,$path);		# URL object Used in ABS to remove leading ..
+
+  # Welcome to the machine
+
+  $newdoc='';
+  while ($doc ne '') {
+    ($textpart,$tagname,%attrval)=&gettag($doc);
+    
+    # Process the tag
+    { # Need a way to get out of this, last is my friend
+      # 'Canonize'
+      if ($canon) {
+	$endhtml=1 if ($tagname eq '/HTML');
+	if ($SGML!=-1 && $canon && $tagname ne '!--' && $tagname ne '!SGML') {
+	  $wholetag=$tagname;
+	  $moretag=join('',keys %attrval);
+	  $wholetag.=' '.$moretag if ($moretag ne '');
+	  if ($SGML==1) {
+	    $SGML=2;
+	    if ($tagname eq '!DOCTYPE') {
+	      $tagname=$wholetag;
+	      %attrval=();
+	      last;
+	    }
+
+	    # We have no idea what DTD this doc follows so we put in
+	    # something kinda non-comitting
+	    $newdoc.='<!DOCTYPE HTML PUBLIC "html.dtd">'."\n";
+	  }
+	  if ($SGML==2) {
+	    $SGML=-1;
+	    if ($tagname eq 'HTML') {
+	      $tagname=$wholetag;
+	      %attrval=();
+	      last;
+	    }
+	    $newdoc.='<HTML>'."\n";
+	  }
+	}			# sgml!=-1
+      }				#canon
+
+      # Tack on text part before I bail out, if wanted
+      $newdoc.=$textpart if $retdoc;
+
+      if ($tagname eq 'BASE') {
+	if (exists($attrval{'HREF'})) {
+	  if ($origin) {
+	    $baseurl=(url($attrval{'HREF'})->abs($origin,1))->as_string;
+	  } else {
+	    $baseurl=$attrval{'HREF'};
+	  }
+	  # Get rid of the HREF attribute.  Netscape 4.0 puts (other) stuff
+	  # into BASE that is not found in the HTML 4.0 spec.
+	  delete $attrval{'HREF'};
+	  print STDERR "\nBase tag: $baseurl\n" if $debug;
+	}
+      }
+
+      # URL processing
+      $i=0;
+      while (defined($arg=$_[$i])) {
+	if (! ($arg =~ /^\d+$/)) {
+	  print STDERR "Args: ",join(',',@_),"\n";
+	  print STDERR "ARG: $arg ($i) is not a opcode.\n";
+	  exit(1);
+	}
+	$i++;
+	if ($arg == $ABS) {
+	  $origin=$baseurl || $_[$i++];
+	  $i++;
+	  # Want it to be a URL object
+	  $origin=url $origin unless ref $origin;
+	  print STDERR 'ABS: ',$origin->as_string,"\n" if $debug;
+	  next unless defined($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    if (defined($attrval{$attr})) {
+	      # Ugly: Remove leading /../ sequences in path component
+	      $url_o=url($attrval{$attr})->abs($origin,1);
+	      if (defined($path=$url_o->path)) {
+		# mailto: URLs does not have path components
+		$path =~ s~/\.\.(?=/)~~g;
+		$url_o->path($path);
+	      }
+	      $attrval{$attr}=$url_o->as_string;
+	    }
+	  }
+	} elsif ($arg == $REL) {
+	  $origin=$_[$i++];
+	  # Want it to be a string
+	  $origin=$origin->as_string if ref $origin;
+	  print STDERR 'REL: ',$origin,"\n" if $debug;
+	  next unless defined($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    $attrval{$attr} =~ s/^$origin//
+	      if defined($attrval{$attr});
+	  }
+	} elsif ($arg == $URLSUB) {
+	  $RE=$_[$i++];
+	  $subst=$_[$i++];
+	  warn "URLSUB: $RE -> $subst\n" if $debug;
+	  next unless defined($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    $attrval{$attr} =~ s/$RE/$subst/
+	      if defined($attrval{$attr});
+	  }
+	} elsif ($arg == $LIST) {
+	  warn "LIST;\n" if $debug;
+	  next unless exists($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    if (exists($attrval{$attr})) {
+	      $attrval{$attr}.='/'
+		if $isdir{$attr} && $attrval{$attr} =~ m~[^/]$~;
+	      push(@urllist,$attrval{$attr});
+	    }
+	  }
+	} elsif ($arg == $SAVEURL) {
+	  warn "SAVEURL;\n" if $debug;
+	  $prefix=$_[$i++];
+	  next unless exists($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    $attrval{"$prefix$attr"}=$attrval{$attr}
+	      if defined($attrval{$attr});
+	  }
+	} elsif ($arg == $USESAVED) {
+	  warn "USESAVED;\n" if $debug;
+	  $prefix=$_[$i++];
+	  next unless exists($urls{$tagname});
+	  foreach $attr (@{$urls{$tagname}}) {
+	    $attrval{$attr}=$attrval{"$prefix$attr"}
+	      if (defined($attrval{"$prefix$attr"}));
+	    # bug compatability, drop sometime after w3mir 1.0
+	    $attrval{$attr}=$attrval{"$prefix-$attr"}
+	      if (defined($attrval{"$prefix-$attr"}));
+	  }
+	} elsif ($arg == $NODOC) {
+	  warn "NODOC;\n" if $debug;
+	} elsif ($arg == $CANON) {
+	  warn "CANON;\n" if $debug;
+	} elsif ($arg == $URLPROC || $arg == $NREL) {
+	  # Apply a function to all urls.
+	  # NREL = Special case of $URLPROC, apply internal function.
+	  if ($arg == $URLPROC) {
+	    warn "URLPROC;\n" if $debug;
+	    $fun=$_[$i++];
+	    $arg=$_[$i++];
+	  } else {
+	    $fun=\&smartrel;
+	    $arg=undef;
+	    $url_origin=$_[$i++];
+	    $doc_top=$_[$i++];
+	    $doc_top_re=quotemeta $doc_top;
+	    ($choped_url_or=$url_origin) =~ s/^$doc_top_re//;
+	    warn "NREL $url_origin $doc_top\n" if $debug;
+	  }
+
+	  next unless (defined($urls{$tagname}));
+
+	  foreach $attr (@{$urls{$tagname}}) {
+	    $attrval{$attr}=&$fun($attrval{$attr},$arg)
+	      if defined($attrval{$attr});
+	  }
+	} elsif ($arg == $TAGCALLBACK) {
+	  $fun=$_[$i++];
+	  $arg=$_[$i++];
+	  warn "TAGCALLBACK($tagname);\n" if $debug;
+	  &$fun($arg,$baseurl,$tagname, 
+		(defined($urls{$tagname})?($urls{$tagname}):undef),
+		\%attrval);
+	} else {
+	  print STDERR "Internal error. opcode: $arg, i: $i, args: ",
+	  join(',',@_),"\n";
+	}
+      }
+      last;
+    }
+
+    # That ends URL processing
+
+    # Was this a verbatim leadin tag?  If yes, atempt to fish out text
+    # between here and the end tag, (minimal match) and substitute it
+    # with nothing.  The end tag is kept.  And the fished out text is
+    # re-inserted in the result with no changes.
+
+    $verbatim=$1 
+      if defined($verbatim{$tagname}) &&
+	($doc =~ s/^(.*?)($verbatim{$tagname})/$2/is);
+    
+    # Tack on the tag, if wanted.
+    if ($retdoc) {
+      if ($tagname) {
+	$newdoc.='<'.$tagname;
+	foreach $attr (keys %attrval) {
+	  $newdoc.=' '.$attr;
+	  if (defined($cont=$attrval{$attr})) {
+	    $Q='"';
+	    $Q="'" if $cont =~ m/\"/;
+	    $newdoc.='='.$Q.$cont.$Q;
+	  }
+	}
+	$newdoc.='>'.$verbatim;
+	$verbatim='';
+      }
+    }
+    print STDERR "NEW: $newdoc\n" if $debug > 2;
+  }
+  $newdoc.="</HTML>\n" if ($canon && !$endhtml);
+  return ($newdoc,@urllist);
+}
+
+
+sub smartrel {
+  # 'smart' relativisation function, uses .. to refer to things up to
+  # $doc_top level.  Outside that scope it stays absolute.
+  #
+  # The rel function itself is now in libwww-perl
+  
+  my($url_o)=@_;
+
+  return if (!(defined($url_o) && $url_o));
+  
+  $url_o=url $url_o unless ref $url_o;
+
+#  print STDERR "\nsmartrel: $url_o->as_string\n",
+#                 "    from: $url_origin\n",
+#                 "  within: $doc_top\n";
+  
+  # Check if within scope of our doc
+  $url_o = $url_o->rel($url_origin)
+    if $url_o->as_string =~ m/^$doc_top_re/;
+
+  return $url_o->as_string;
+}
+
+
+sub smart_setup {
+  # Setup routine for smartrel.
+
+  ($url_origin,$doc_top) = @_;
+
+  $url_origin=$url_origin->as_string if ref $url_origin;
+  $doc_top=$doc_top->as_string if ref $doc_top;
+
+  $doc_top_re=quotemeta $doc_top;
+
+  ($choped_url_or=$url_origin) =~ s/^$doc_top_re//;
+}
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/install.sh
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/install.sh	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/install.sh	(revision 21112)
@@ -0,0 +1,9 @@
+#!/bin/sh -v
+
+export PERL5LIB=$GSDLHOME/perllib/cpan/lib:$PERL5LIB
+export PATH=$GSDLHOME/perllib/cpan/bin:$PATH
+
+perl Makefile.PL PREFIX=$GSDLHOME/perllib/cpan
+
+make
+make install
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/multiscope.cfg
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/multiscope.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/multiscope.cfg	(revision 21112)
@@ -0,0 +1,46 @@
+# This file is meant to illustrate multiscope retrival.
+#
+# Available options: recurse, no-date-check, only-nonexistent, list-urls,
+# lowercase.  It is not default to recurse.  options must be the 
+# first directive in the the config file.
+Options: recurse
+# This directive was previously known as 'Fetch-options'
+#
+# This is the two argument form of URL:.  It fetches the first into the second
+URL: http://www.math.uio.no/~janl/ math/janl
+#
+# These says that any documents refered to that lives under these places
+# should be gotten too.  Into the named directories.  Two arguments are
+# required for 'Also:'.
+Also: http://www.math.uio.no/drift/personer/ math/drift
+Also: http://www.stud.ifi.uio.no/~janl/ ifi/janl
+Also: http://www.mi.uib.no/~nicolai/ math-uib/nicolai
+#
+# A more typical use would perhaps be:
+#   URL: http://www.math.uio.no/~janl/ janl
+#   Also: http://www.math.uio.no/images/ images
+# which would retrive the images refered to under
+# http://www.math.uio.no/~janl/ into a directory called images
+#
+# CD here before doing any work
+#cd: janlhomes
+#
+# Default pause is now 0.
+#Pause: 0
+#
+# Ignore urls we can't copy anyway.  Actually w3mir can copy
+# many cgi things with no problems.  But not always.
+Ignore: *.cgi
+Ignore: *-cgi
+Ignore: *.map
+#
+# Run fixup to get a filesystem browseable mirror
+Fixup: run
+# The options above will result in this directory hierarchy under
+# where you started w3mir:
+#
+# w3mir/math/janl		files from http://www.math.uio.no/~janl
+# w3mir/math/drift		from http://www.math.uio.no/drift/personer/
+# w3mir/ifi/janl		from http://www.stud.ifi.uio.no/~janl/
+# w3mir/math-uib/nicolai	from http://www.mi.uib.no/~nicolai/
+#
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3http.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3http.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3http.pm	(revision 21112)
@@ -0,0 +1,765 @@
+# -*- perl -*-
+# w3http.pm	--- send http requests, janl's 12" mix for w3mir
+#
+$VERSION=1.0.22;
+#
+# This implements http/1.0 requests.  We'll have problems with http/0.9
+# This is in no way specific to w3mir.
+#
+# IMPORTANT: The caller should initialize the C locale for some of the
+#   things here to work correctly (specifically the strftime function).
+#
+# This is a rewrite of http.pl by Oscar Nierstrasz; I copied the code he he
+# copied from the camel book.  Some functions written by Gorm Haug Eriksen
+# (gorm@usit.uio.no) has been used as is.
+#
+# Contributors:
+#   Nicolai Langfeldt, janl@ifi.uio.no
+#   Gorm Haug Eriksen, gorm@usit.uio.no
+#   Chris Szurgot, szurgot@itribe.net
+#   Bernhard Weisshuh, bkw@weisshuhn.de
+# Copying and modification is governed by the "Artistic License" enclosed in
+# the w3mir distribution
+#
+# gorm :
+# &w3http::get_last_modified  return the last modified stamp on a file in
+#                         the right format for use with http
+#
+# janl:
+# &http::query: Send a http query.  A completely general function to send a
+#   http query.  Will extract header values, http response code and, optionaly,
+#   convert text files to local linefeed format.
+#
+# Variables to examine after a query
+# $w3http::document: The document returned by the query, if any.
+# $w3http::doclen: The length of the document
+# $w3http::result: The numerical http result code.  It may take these values:
+# - Normal HTTP reply codes
+# -  98: OS error, permanent, errormessage in $!
+# -  99: Other permanent error: see $restext.
+# - 100: Transient error: Timeout/connection broken
+# $w3http::restext: The english(?) HTTP result or w3http generated message
+# $w3http::header: The http header returned.
+# $w3http::plaintexthtml: 1 if this doc is non-content-encoded text/html
+#	(as opposed to content-encoding: compressed content-type: text/html
+#       which needs decompression before we can inspect the html) 
+#	The tests are somewhat longwinded so I do it just once here.
+# %w3http::headval: Associative array of header values
+# $w3http::headval{'CONTENT-TYPE'}: Derived content type, stripped of charset
+#		qualifiers and other distractions.
+# $w3http::xfbytes: Transfered bytes, cumulative.  Document part only.
+# $w3http::headbytes: Bytes of headers received, cumulative.
+#
+# Variables that change http's behaviour/requests:
+# $w3http::agent: User agent, default is basename of $0
+# $w3http::from: Request is from, default is user@host
+# $w3http::version: The http version to use, only 1.0 is known to me.
+# $w3http::timeout: How long to wait for new data to arrive, default is 600sec
+# $w3http::buflen: Network read buffer size, default is 4096.  It might give a
+#	speedup to tune this for specific servers' so it matches their send
+#	size.  This size can be detected if we want to, I think.
+# $w3http::debug: 1 debuging output, 2, more, 3 queries and replies
+# $w3http::verbose: 0: say nothing, 1: print progress info
+# $w3http::convert: Convert text/* documents to local newline convention?
+#	The default is to do it.
+# $w3http::proxyserver: The name of the proxyserver to use.
+# $w3http::proxyport: The port of the proxyserver to use. 0 if no proxyserver.
+# $w3http::proxyuser: If this is set proxy authentication will be used.
+# $w3http::proxypasswd: The password for proxy authentication
+#
+# Things gotten from main:
+# - $main::win32: 1 if win32 restrictions apply to this system
+# - $main::nulldevice: Bit sink file/device on this system.
+#
+# History (european date format dd/mm/yy):
+#     janl ??/??/95 -- Rewrite finished
+#  szurgot ??/??/95 -- Win32 compatability
+#     janl 16/05/96 -- Added SAVEBIN option, based on idea by szurgot
+#  szurgot 03/05/96 -- Corrected typo in check for content-length against
+#                      retreive document length. Added test for zero-length
+#		       documents (Not retreived because not-modified)
+#  szurgot 19/05/96 -- Win32 adaptions, fixes.
+#     janl 19/05/96 -- Chris won an argument, and janl simplified http 
+#		       retrival loop (-> version 1.0.4)
+#     janl 09/09/96 -- Incorporated a patch submited by Michael Kriby -> 1.0.5
+#     janl 16/09/96 -- Support for authorization. -> 1.0.6
+#     janl 27/09/96 -- Support for Accept header, lack pointed out by
+# 			charles@ermine.ox.ac.uk: ... HTTP/1.1 (§14.1) says 
+#			``If no Accept header field is present, then it is
+#			assumed that the client accepts all media types,
+#			earlier versions of the protocol suggest that only
+#			text/plain and text/html will be offered by default.''
+#			This contradicts my memory of a http/1.0 draft.
+#			Also added $ACCEPT option.
+#     janl 20/10/96 -- Now uses HTTP::Date to produce HTTP timestamps -> 1.0.7
+#     janl 27/10/96 -- Didn't use to check if gethostbyname worked -> 1.0.8
+#     janl 02/12/96 -- Forgot a unlink when renaming temporary files.
+#     janl 21/02/97 -- Multipele $ACCEPT options work. -> 1.0.9
+#     janl 19/03/97 -- Now issues Host: header -> 1.0.10
+#     janl 10/04/97 -- Changed from wwwurl to URI::URL, and various related
+#                      changes. -> 1.0.11
+#     janl 09/05/97 -- Microsoft ISS servers are _so_ broken -> 1.0.12
+#                      (don't close the write end of the HTTP socket after
+#			sending a query to them)
+#     janl 12/05/97 -- New version of perl caught some typos, fixed
+#			longstanding bug in the newline conversion bit.
+#			-> 1.0.13
+#     janl 06/06/97 -- Demand Loading of MIME::BASE64 -> 1.0.14
+#     janl 01/12/97 -- FAT filesystems drops LSB of modtime.  Patch from
+#			Greg Lindhorst (gregli@microsoft.com)
+#                   -- whoami does not exist on win32, hardwire a default
+#			value (unknown) (also Greg L.) -> 1.0.15
+#     janl 01/22/98 -- Proxy authentication as outlined by Christian Geuer
+#     janl 02/20/98 -- Complex 'content-type' headers handled. -> 1.0.17
+#     janl 04/20/98 -- Only newline convert text/html, everything else is
+#			handled as binary. -> 1.0.18
+#     janl 12/05/98 -- Store tmpfile in its final destination directory
+#			avoiding asking movefile move it across filesystems.
+#			-> 1.0.19
+#     janl 01/08/98 -- Timeout fix from Michael Gusev, also flag short doc
+#			as error.
+#     janl 24/09/98 -- Better error handling -> 1.0.20
+#     bkw  17/12/98 -- Fixed problem with tempfile-generation when
+#                      running in forget-mode (-f)
+#     janl 05/01/99 -- Referer: dropped if argument not true -> 1.0.21
+#     janl 13/04/99 -- Added workaround for broken win32 perl resolving.
+
+package w3http;
+
+require 5.002;
+use Socket;
+use HTTP::Date;
+use Sys::Hostname;
+use URI::URL;
+
+# Suplementary libwww-perl:
+sub URI::URL::_generic::basename {
+  my $self = shift;
+  my @p = $self->path_components;
+  my $old = $p[-1];
+  if (@_) {
+    splice(@p, -1, 1, shift);
+    $self->path_components(@p)
+  }
+  $old;
+}
+
+
+END { 
+  # Remove tmp file and such in here.  That means that main:: gotta catch
+  # interrupt signals and exit on them, so ENDs are executed.
+}
+
+use strict;
+# Global variables, we want to share them:
+use vars qw($GET $HEAD $GETURL $HEADURL $IFMOD $IFMODF $AUTHORIZ $REFERER);
+use vars qw($SAVEBIN $ACCEPT $NOUSER $FREEHEAD $agent $version $timeout);
+use vars qw($debug $convert $proxyserver $proxyport $xfbytes $headbytes);
+use vars qw($verbose $result $restext $header $document);
+use vars qw($plaintexthtml %headval $progress $doclen $proxyuser);
+use vars qw($proxypasswd);
+
+my $hasAlarm;   # Win32 does not have any alarm
+my $chime;	# Has the alarm gone off yet?
+my %address;	# My own DNS cache
+my $savALRM;	# Saved ALRM handler
+my $savPIPE;	# Saved PIPE handler
+
+# The main:: program should detect if we're running on win32 or not,
+# somehow
+if ($main::win32) {
+  warn "win32\n";
+  # Compensate for lacks of win32 perl. 
+  $hasAlarm=0;
+  # Seems to be unavailable in win32/perl5.001.  It has to be in 5.003!
+#  eval "sub sockaddr_in {
+#	($port, $thataddr) = @_;
+#	$sockaddr = 'S n a4 x8';
+#	return pack($sockaddr, &AF_INET, $port, $thataddr);
+#    }";
+} else {
+  $hasAlarm=1;
+}
+
+
+# Find out some things
+my $thishost = hostname();
+my $proto = getprotobyname("tcp");
+
+(my $name, undef) = gethostbyname($thishost);
+chomp(my $user = $ENV{'LOGNAME'} || $ENV{'USER'} || `whoami` || 'unknown');
+my $from   = "$user\@$name";
+
+my $nl = "\r\n";
+# Default values, change by assignment in using-program.
+$agent  = $0; $agent =~ s~.*/~~; # Basename 
+$version= "1.0";
+$timeout= 600;			# Timeout while waiting for data/connection
+my $buflen = 4096;		# recv buffer length
+$debug = 0;			# Debuging output?
+$convert = 1;			# Convert newlines of text docs to local format
+$proxyserver='';		# Proxy server.
+$proxyport=0;			# Proxy server port. 0 if no proxy.
+$proxyuser='';			# Username for proxy authentication
+$proxypasswd='';		# Password for proxy authentication
+$xfbytes=0;			# 0 bytes transfered, cumulative
+$headbytes=0;			# 0 bytes of headers, cumulative
+$doclen=0;			# 0 bytes in doc, pr. document
+my $tmpfile="w3mir$$.tmp";	# Temporary filename
+$verbose=0;			# Verbosenes, 0: silent, 1: progress info
+
+# Query opcodes
+$GET = 1;			# GET query. Arg: host,port,path
+$HEAD = 2;			# HEAD query. Arg: host,port,path
+$GETURL = 3;			# GET query. Arg: url
+$HEADURL = 4;			# HEAD query. Arg: url
+# Here we lack PUT, which is not implemented
+# Modify query thus:
+$IFMOD = 101;			# If-modified after: Arg: HTTP-date-str
+$IFMODF = 102;			# If-modified after file: Arg: local-file-name
+$AUTHORIZ= 103;			# Basic authorization. Arg: 'user:password'
+$REFERER = 104;			# Referer: Arg: Referer 
+$SAVEBIN = 105;			# Write binary files to disk. Arg: File name
+				# If this opcode is used then main must provide
+				# a &main::movefile(oldname,newname) procedure
+				# that handles moving the tmp file to the
+				# final name/location.
+$ACCEPT  = 106;			# Accept header value: Arg: value
+$NOUSER  = 107;			# Don't insert user header.  Arg: none
+$FREEHEAD= 999;			# Freeform header, one line.  Arg: header
+
+sub query {
+  # Build and send a HTTP query.  And also receive response - janl 95/09/18
+  #
+  # Return codes: 0 if it didn't work.  1 if it did work.
+  # HTTP style result code in w3http::$result and message in w3http::$restext
+  
+  # We do next to no argument type checking btw.
+  
+  my($host,$port,$request,$query,$method,$inp,$linp,$saveto,$save,$arg);
+  my($start,$wantbytes,$thataddr,$err,$headb,$tmpf,$ldoc,$nouser,$q,$accept);
+  my($origreq,$req_o,$plaintext);
+  
+  # Something ought to be said
+  $result=99;
+  $restext='w3http: internal error';
+  $nouser=0;
+  
+  if ($version ne '1.0') {
+    warn "Unknown HTTP version $version, no request sent\n";
+    return 0;
+  }
+  
+  $accept=$saveto=$query='';
+  
+  # Find out what to ask for
+  
+  while (defined($arg=shift)) {
+    if ($arg == $GET) {
+      $host=shift;
+      $port=shift;
+      $request=shift;
+      $req_o=url 'http://'.$host.':'.$port.$request;
+      if ($proxyport) {
+	$query.='GET http://'.$req_o->as_string;
+      } else {
+	$query.='GET '.$req_o->epath;
+      }
+      $query.=' HTTP/'.$version.$nl;
+    } elsif ($arg == $HEAD) {
+      $host=shift;
+      $port=shift;
+      $request=shift;
+      $req_o=url 'http://'.$host.':'.$port.$request;
+      if ($proxyport) {
+	$query.='HEAD '.$req_o->as_string;
+      } else {
+	$query.='HEAD '.$req_o->epath;
+      }
+      $query.=' HTTP/'.$version.$nl;
+    } elsif ($arg == $GETURL) {
+      $req_o=shift;
+      $req_o=url $req_o unless ref $req_o;
+      ($method,undef,undef,$host,$port,$request,undef,$q) = $req_o->crack;
+      if ($proxyport) {
+	$query.='GET '.$req_o->as_string;
+      } else {
+	$q=$req_o->equery;
+	$query.='GET '.$request.($q?"?$q":'');
+      }
+      $query.=' HTTP/'.$version.$nl;
+    } elsif ($arg == $HEADURL) {
+      $req_o=shift;
+      $req_o=url $req_o unless ref $req_o;
+      if ($proxyport) {
+	$query.='HEAD '.$req_o->as_string;
+      } else {
+	$q=$req_o->equery;
+	$query.='HEAD '.$req_o->epath.($q?"?$q":'');
+      }
+      $query.=' HTTP/'.$version.$nl;
+    } elsif ($arg == $IFMOD) {
+      $query.='If-Modified-Since: '.(shift).$nl;
+    } elsif ($arg == $IFMODF) {
+      $query.='If-Modified-Since: '.&last_modified(shift).$nl;
+    } elsif ($arg == $AUTHORIZ) {
+      # Demand-load MIME::Base64
+      if (!defined(&MIME::Base64::encode)) {
+	eval "use MIME::Base64;";
+	die "w3http: Could not load MIME::Base64 module necessary for authentication\n"
+	  unless defined(&MIME::Base64::encode);
+      }
+      $query.='Authorization: Basic '.MIME::Base64::encode(shift,'').$nl;
+    } elsif ($arg == $REFERER) {
+      my($referer)=shift;
+      $query.='Referer: '.$referer.$nl if $referer;
+    } elsif ($arg == $SAVEBIN) {
+      $saveto=shift;
+    } elsif ($arg == $ACCEPT) {
+      $accept.='Accept: '.(shift).$nl;
+    } elsif ($arg == $NOUSER) {
+      $nouser=1;
+    } elsif ($arg == $FREEHEAD) {
+      $query.=(shift).$nl;
+    } else {
+      warn "Unknown http query opcode: $arg\n";
+    } 
+    # Insert the last parts of the query:
+  }
+  
+  $query.='Host: '.$req_o->netloc.$nl;
+  $query.='From: '.$from.$nl unless $nouser;
+
+  $accept='Accept: */*'.$nl unless $accept;
+
+  if ($proxyport) {
+    # Use proxy instead of originserver
+    $host=$proxyserver;
+    $port=$proxyport;
+
+    # Add authentication stuff to query
+    if ($proxyuser) {
+      # Demand-load MIME::Base64
+      if (!defined(&MIME::Base64::encode)) {
+	eval "use MIME::Base64;";
+	die "w3http: Could not load MIME::Base64 module necessary for authentication\n"
+	  unless defined(&MIME::Base64::encode);
+      }
+      
+      $query.='Proxy-Authorization: Basic '.
+	MIME::Base64::encode($proxyuser.':'.$proxypasswd);
+
+      print STDERR "\nProxyuser: [$proxyuser]\nProxypasswd: [$proxypasswd]\n"
+	if $debug>=2;
+    }
+  }
+  
+  $query.='User-Agent: '.$agent.$nl.$accept.$nl;
+  
+  # If we're using proxy then set up things...
+  print STDERR "\nQUERY:\n",$query,"---\n" if $debug>=2;
+
+  # win32 fix: this should be added in case of troubles with
+  # gethostbyname. possible reason: nameserver down?
+  if ($host =~ /^\d+(\.\d+){3}$/) {
+    # in case gethostbyname will not work ... ;-)
+    $address{$host} = pack 'C4', (split /\./, $host);
+  }
+  
+  # Find out who to ask, check if we know already
+  if (exists($address{$host})) {
+    # We know
+    $thataddr=$address{$host};
+  } else {
+    # Cache miss, get and remember.
+    (my $fqdn, undef, undef, undef, $thataddr) = gethostbyname($host);
+    # Hostname lookup failure?  Cache even misses.
+    if (defined($fqdn)) {
+      print STDERR "Lookup of $host:\nFQDN: $fqdn\n"
+	if $debug;
+      $address{$host}=$thataddr;
+      $address{$fqdn}=$thataddr if $fqdn ne $host;
+    } else {
+      $thataddr=$address{$host}=undef;
+    }
+  }    
+
+  # Check if lookup failure, return
+  if (!defined($thataddr)) {
+    $restext='Host lookup failure';
+    return;
+  }
+
+  $port=80 unless defined($port) && $port;
+
+  # When connected we might receive SIGPIPE.  I'm not sure if the
+  # default behaviour of dying is beneficial in that case.  If we get
+  # alarm a timeout has expired.
+  $savPIPE = $SIG{'PIPE'};
+  $savALRM = $SIG{'ALRM'};
+
+  $chime=0;			# There has been no alarm yet
+  $SIG{'ALRM'} = \&timeout;
+  $SIG{'PIPE'} = \&ignore;
+
+  # Close the socket, just in case, and ignore error returns
+  close(FS);
+  
+  socket(FS, AF_INET, SOCK_STREAM, $proto) or return &oserror;
+  warn "Got my socks on\n" if $debug;
+  
+  my $paddr = sockaddr_in($port, $thataddr);
+  connect(FS, $paddr) or return &oserror;
+  warn "Connected\n" if $debug;
+  
+  # Arrange timeout
+  alarm($timeout) if $hasAlarm;
+  
+  # We have, in fact, received SIGPIPE on this line:
+  send(FS,$query,0) or return &oserror;
+
+  if ($chime) {
+    $result=100;
+    $restext='timeout sending query';
+    return &resetsign;
+  }
+  
+  $header='';
+  $document='';
+  $inp=' 'x$buflen;
+  $doclen=$chime=$plaintext=$plaintexthtml=$save=0;
+
+  # Breaks some M$ ISS servers:
+  # shutdown(FS,1);  # Half-close socket, sending now not allowed
+  
+  print STDERR ", receiving header" if $verbose>0;
+  
+  # Retrive HTTP response HEADER.  Why do I use recv and not <FS>?
+  # Because then the timeout can work correctly!
+  while (1) {
+    # Set up alarm to ensure recv returns within a reasonable timeframe
+    alarm($timeout) if $hasAlarm;
+    $err = recv(FS,$inp,$buflen,0);
+    # recv returned, cancel alarm.
+    alarm(0) if $hasAlarm;
+    
+    # If there has been a timeout, then we quit now.  The recv man page
+    # does not seem to allow recv to return the bytes received up to
+    # the timeout.
+    if ($chime) {
+      $result=100;  
+      $restext='timeout fetching document';
+      $!=0;
+      if ($save) {
+	unlink($tmpf) || 
+	  warn "Could not unlink $tmpf: $!\n";
+      }
+      return &resetsign;
+    }
+    
+    # recv returnes the undefined value on error
+    if (!defined($err)) {
+      warn "Error in recv: $!\n";
+      last;
+    }
+    
+    $linp=length($inp);
+    
+    # If the returned input was 0 in length then we've gotten to the 
+    # end of the response.
+    last unless $linp;
+    
+    # Accounting
+    $xfbytes += $linp;
+    $doclen += $linp;
+    
+    # Accumulate input
+    $header.=$inp;
+
+    # eof(SOCKET) has strange semantics it seems
+    # last if eof(FS);
+    
+    # Check if header is complete
+    last if ($header =~ m/(\r?\n\r?\n)/);
+  }
+  
+  if (length($header)==0) {
+    $restext='the HTTP reply header is empty!';
+    return &resetsign;
+  }
+  
+  if ($header =~ m/(\r?\n\r?\n)/) {
+    $header=$`;
+    $document=$';
+  } else {
+    $header=$document;
+  }
+  
+  # Adjust accounting
+  $headb = length($header)+length($1);
+  $headbytes += $headb;
+  $xfbytes -= $headb;
+  $doclen -= $headb;
+  
+  # Pick headers to pieces
+  ($result,$restext,%headval)=&analyze_header($header);
+
+  print STDERR "REPLY:\n",$header,"\n---\n" if $debug>=2;
+
+  # Check if the document is a non-encoded text document. The contents
+  # could be (x-)?compress or (x-)gzip coded (compressed in other
+  # words).
+  
+  $plaintext=defined($headval{'CONTENT-TYPE'}) &&
+    (substr($headval{'CONTENT-TYPE'},0,5) eq 'text/' || 0) &&
+      !defined($headval{'content-encoding'});
+  $plaintexthtml=$plaintext && 
+    ($headval{'CONTENT-TYPE'} eq 'text/html');
+  
+  if ($result==200) {
+    
+    # Save this to a file, or not?  Never save html files.
+    if ($saveto && !$plaintexthtml) {
+      # We're going to save this document directly into a file.  This
+      # stresses the VM less when getting the large binares so often
+      # found at cool sites.
+      $save=1;
+
+      # Find a temporary filename
+      $tmpf=url "file:$saveto";
+      $tmpf->basename($tmpfile);
+      $tmpf=$tmpf->unix_path;
+      
+      # Find suitable final filename, one with no URL escapes
+      $saveto=(url "file:$saveto")->unix_path;
+
+      # If output to stdout then send it directly there rather than
+      # using disk unnecesarily.
+      $tmpf='-' if ($saveto eq '-');
+
+      # If output is nulldevice (running -f), use it also for tmpfile,
+      # since it would otherwise try to create it in /dev under unix.
+      $tmpf=$main::nulldevice if ($saveto eq $main::nulldevice);
+      
+      warn "USING TMPFILE: $tmpf\n" if $debug;
+      
+      open(SAVE,">$tmpf") || 
+	die "Could not open tmp file: $tmpf: $!\n";
+      binmode SAVE;		# It's a binary file...
+    }
+    
+    if ($verbose>0) {
+      print STDERR ", document";
+      print STDERR "->disk" if $save;
+    }
+    
+    # Now retrive document itself.  Se comments in header loop
+    $start=time;
+    $wantbytes = defined($headval{'content-length'})?
+      $headval{'content-length'}:0;
+    
+    $ldoc=length($document);
+    
+    while (1) {
+      alarm($timeout) if $hasAlarm;
+      recv(FS,$inp,$buflen,0);
+      alarm(0) if $hasAlarm;
+      
+      if ($chime) {
+	$result=100;
+	$restext='timeout fetching document';
+	$!=0;
+	if ($save) {
+	  unlink($tmpf) || 
+	    warn "Could not unlink $tmpf: $!\n";
+	}
+	return &resetsign;
+      }
+      
+      $linp=length($inp);
+      
+      last unless $linp || $ldoc;
+      $ldoc = 0;
+      
+      $xfbytes += $linp;
+      $doclen += $linp;
+      
+      if ($verbose>0 && time-$start>5) {
+	# Write progress info ... 
+	if ($wantbytes) {
+	  $progress = sprintf " %3d%%", $doclen/$wantbytes*100;
+	} else {
+	  $progress = sprintf " %d", $doclen;
+	}
+	print STDERR $progress, "\ch"x(length($progress));
+	# ...every 5 seconds
+	$start=time;
+      }
+      
+      $document.=$inp;
+      
+      if ($save) {
+	$err = print SAVE $document;
+	die "Error writing $tmpf: $!\n" unless $err;
+	$document='';
+      }
+
+      # The eof test seems to work very oddly for sockets.
+      # last if eof(FS);
+    }
+
+    close(FS);  # Close socket completely
+    
+    print STDERR "DOCUMENT:\n----\n",$document,"\n----\n" if $debug>=255;
+  
+    if ($wantbytes &&
+	$wantbytes != $doclen) {
+      $result=100;
+      $restext='transfer error; too many bytes in document';
+      $restext='document was incomplete' if ($wantbytes > $doclen) ;
+      print STDERR "SHORT DOCUMENT" if $debug>=16;
+      if ($save) {
+	unlink($tmpf) || warn "Could not unlink $tmpf: $!\n";
+      }
+      return &resetsign;
+    }
+    
+    # warn "XFB: $xfbytes, DL: $doclen\n";
+    if ($save) {
+      close(SAVE);
+      &main::movefile($tmpf,$saveto);
+    }
+    
+    # If this is a non-encoded text file and we're supposed to convert
+    # foreign newlines then we do it. It would be faster to do this
+    # with each chunk of input in the input loop, but this gives us
+    # two problems:
+    # - A \r\n newline could be split into two chunks.  Thus escaping
+    #   newline conversion.
+    # - It messes up the received bytes accounting rather badly.
+    #
+    # This used to be a test for $plaintext, the problem is that too
+    # many documents were typed as text/plain and so we corrupted
+    # binary files. This is bad. So now we're more paranoid about it:
+    # Only HTML gets converted.
+    if ($convert && $plaintexthtml) {
+      # Change non unix newlines to unix newlines. bare \r is known
+      # from macintosh (they hadta be different didn't they?), \r\n is
+      # known as 'network format' and from numerous systems, among
+      # them ms-dos.
+      $document =~ s~\r~\n~g unless $document =~ s~\r\n~\n~g;
+      warn "Newlines converted(?)\n" if $debug;
+    }
+    
+  }				# if $result == 200
+  
+  &resetsign;
+  return 1;
+}
+
+
+sub analyze_header {
+  my($header)=@_;
+  my($result,$restext,%headval,$hdln,$key,$value);
+  
+  # Summary of the http spec on headers (with my comments):
+  # - Each header line ends in CRLF (or just LF, or maybe even just CR,
+  #   anyways, it's easier if all is LF).
+  $header =~ s/\r/\n/mg unless $header =~ s/\r\n/\n/mg;
+  # - If a line starts with space then it's a continuation of the previous
+  #   line (these I fold into one line).
+  $header =~ s/\n\s/ /mg;
+  # - The header field names are case insensitive (so I convert them to
+  #   lowercase)
+  # - A field may appear twice, that is equivalent to listing the values
+  #   in a comma separated list (so I fold them into a comma separated list)n
+  # - The field name and the field value are separated by ': '
+  ($result,$restext) = $header =~ m~^HTTP/\d\.\d (\d\d\d) (.*)~;
+  # Shave off http result code from the header
+  $header =~ s~^.*\n~~;
+  
+  warn "Header:\n$header\n---\n" if $debug>=3;
+  
+  warn "Result: $result, Text: $restext\n" if $debug>=2;
+  
+  %headval=();
+  
+  foreach $hdln (split(/\r?\n/m,$header)) {
+    ($key,$value)=split(': ',$hdln,2);
+    $key="\L$key";
+    # Strip leading&trailing space off the reply, some servers use
+    # copious space after.
+    $value =~ s/^\s+|\s+$//g;
+    print STDERR "K: '$key', V: '$value'\n" if $debug>=2;
+    if (defined($headval{$key})) {
+      $headval{$key}.=", ".$value;
+    } else {
+      $headval{$key}=$value;
+    }
+  }
+
+  # See if there are any type parameters in the content-type header
+  # and if so remove them.
+  if (defined($headval{'content-type'})) {
+    my $val=$headval{'content-type'};
+    ($val,undef)=split(';',$val,2) if ($val =~ /;/);
+    $headval{'CONTENT-TYPE'}=$val;
+  }
+  
+  return ($result,$restext,%headval);
+}
+
+
+sub last_modified {
+  # will return the last modified time for a local file as a HTTP
+  # timestamp.
+  
+  my(@tmp) = stat($_[0]);	# file doesn't exist ok to fetch
+
+  # FAT file systems strip the LSB of the file time.  Add it back in
+  # here before asking the server about a modified file.  The only way
+  # this can fail is if the newer server file was saved one second
+  # after the first version (very unlikely).  This isn't needed for
+  # NTFS file systems, but there is no good portable Perl way to
+  # determine the file system type.
+  $tmp[9] = $tmp[9] | 1 if ( $main::win32 );
+
+  # now we got the last modified in a 32 bit integer.  time to convert
+  # it and return
+  return time2str($tmp[9]);
+}
+
+
+sub timeout {
+  # Set timeout flag.  The using procedure has to set other result codes.
+  $chime=1;			# When this is 1 then the alarm has gone off
+  print STDERR "TIMEOUT!!!!\n" if $debug>=16;
+}
+
+
+sub ignore {
+  warn "I got SIGPIPE, ignoring it...\n";
+}
+
+
+sub resetsign {
+  return 0 if !defined($savALRM);
+  $SIG{'ALRM'}=$savALRM;
+  undef $savALRM;
+#  $SIG{'PIPE'}=$savPIPE;
+  return 0;
+}
+
+
+sub oserror {
+  
+  resetsign;
+
+  $result=98;
+  $restext='w3http: OS error';
+  return 0;
+
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mfix.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mfix.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mfix.PL	(revision 21112)
@@ -0,0 +1,1070 @@
+# -*-perl-*-
+
+use Config;
+
+&read_makefile;
+$fullperl = resolve_make_var('FULLPERL') || $Config{'perlpath'};
+$islib = resolve_make_var('INSTALLSITELIB');
+
+$name = $0;
+$name =~ s~^.*/~~;
+$name =~ s~.PL$~~;
+
+open(OUT,"> $name") || 
+  die "Could open $name for writing: $!\n";
+
+print "writing $name\n";
+
+while (<DATA>) {
+  if (m~^\#!/.*/perl.*$~o) {
+    # This substitutes the path perl was installed at on this system
+    # _and_ removed any (-w) options.
+    print OUT "#!",$fullperl,$1,"\n";
+    next;
+  }
+  if (/^use lib/o) {
+    # This substitutes the actuall library install path
+    print OUT "use lib '$islib';\n";
+    next;
+  }
+  print OUT;
+}
+
+close(OUT);
+
+# Make it executable too, and writeable
+chmod 0755, $name;
+
+#### The library
+
+sub resolve_make_var ($) {
+
+  my($var) = shift @_;
+  my($val) = $make{$var};
+
+#  print "Resolving: ",$var,"=",$val,"\n";
+  
+  while ($val =~ s~\$\((\S+)\)~$make{$1}~g) {}
+#  print "Resolved: $var: $make{$var} -> $val\n";
+  $val;
+}
+
+
+sub read_makefile {
+
+  open(MAKEFILE, 'Makefile') || 
+    die "Could not open Makefile for reading: $!\n";
+
+  while (<MAKEFILE>) {
+    chomp;
+    next unless m/^([A-Z]+)\s*=\s*(\S+)$/;
+    $make{$1}=$2;
+#    print "Makevar: $1 = $2\n";
+  }
+
+  close(MAKEFILE)
+}
+
+__END__
+#!/local/bin/perl5 -w
+# Perl 5.002 or later.  w3mir is mostly tested with perl 5.004
+#
+# You might want to change or comment out this:
+use lib '/hom/janl/lib/perl';
+#
+# Can perform the following fixes:
+# - Rewrite redirected to URLs
+# - change .../ into .../index.html (or .../Welcome.html)
+# - change external links to point to some helpfull .html file ---- NOT IMPLEMENTED
+# - change links to documents not retrived to point to some helpfull
+#   .html file --- NOT IMPLEMENTED
+# - After adding a 'Also' directive we can edit the urls pointing to within
+#   the new space of the retrival scope, making the pointers consistent.
+
+# Method: 
+# 1 Gather list of all URLs to be rewritten:
+#   - Redirects: Just read the .redirs file
+#   - .../ into .../index.html: All can be found in the .referers file
+#     Remember to rewrite missing / redirects too...
+#   - external links: in the .referers file too
+#   - non-retrived documents: What files should be here (according to
+#	.referers file) but are not?
+# 2 Gather list of all documents needing editing
+# 3 Edit them
+#
+# 09/05/98 janl		- Only rewrite .../ to .../$indexname if the attribute
+#			  does not refer to a directory. -> 0.6.2
+# 12/05/98 janl		- Use ->local_path to determine filename -> 0.6.3
+# 22/05/98 janl		- .../#foo was not being rewritten to 
+#			  .../$indexname#foo -> 0.6.4
+#
+require 5.002;
+
+use vars qw($win32);
+
+# To figure out what kind of system this is
+BEGIN {
+  use Config;
+  $win32 = ( $Config{'osname'} eq 'MSWin32' ); 
+}
+
+use Carp;
+use htmlop;
+use URI::URL;
+use URI::Escape;
+
+use strict;
+
+my $VERSION;
+$VERSION='0.6.5';
+
+my $debug=0;			# Not debugging
+my $verbose=0;
+
+my $indexname='index.html';
+my $chdirto='';			# Place to chdir to after reading
+                                # config file
+
+my $infoloss=0;			# 1 if any URL translations (which
+                                # cause information loss) are in
+                                # effect.  If this is true we use the
+                                # SAVEURL operation.  What to get, and
+                                # not.  Text of user supplied
+                                # fetch/ignore rules
+
+my $doindex=1;			# append $indexname to /$ ?
+
+my $editthis='';		# Edit references matching this expression.
+
+my $files=0;			# How many files have I edited?
+my $rc='';
+
+my $rule_text="# User defined fetch/ignore rules\n";
+# Code ref to the rule procedure
+my $rule_code;
+
+# Code to prefix and postfix the generated code.  Prefix should make
+# $_ contain the url to match.  Postfix should return 1, the default
+# is to get the url/file.
+my $rule_prefix='$rule_code = sub { local($_) = shift;'."\n";
+my $rule_postfix=' return 1; } ';
+
+# Scope tests generated by URL/Also directives in cfg. The scope code
+# is just like the rule code, but used for program generated
+# fetch/ignore rules related to multiscope retrival.
+my $scope_fetch="# Automatic fetch rules for multiscope retrival\n";
+my $scope_ignore="# Automatic ignore rules for multiscope retrival\n";
+my $scope_code;
+
+my $scope_prefix='$scope_code = sub { local($_) = shift;'."\n";
+my $scope_postfix=' return 0; } ';
+
+# Function to apply to urls, se rule comments.
+my $user_apply_code;	# User specified apply code
+my $apply_code;		# w3mirs apply code
+my $apply_prefix='$apply_code = sub { local($_) = @_;'."\n";
+my $apply_lc=' $_ = lc $_; ';
+my $apply_postfix=' return $_; } ';
+my @user_apply;		# List of users apply rules.
+my @internal_apply;	# List of w3mirs apply rules.
+
+
+my $iinline='';			# inline RE code to make RE caseinsensitive 
+my $ipost='';			# RE postfix to make it caseinsensitive
+my $lc=0;			# Convert urls/filenames to lowercase?
+my $abs=0;			# Absolutify URLs?
+my $fixrc='';			# Name of w3mfix config file
+my $fixup=0;			# Do things needed to run fixup
+my $r=0;			# Recurse? no recursion = absolutify links
+my %rum_referers=();		# Array of referers, key: rum_url
+my %rum_redirected=();		# Array of redirected url: key: original url
+my %lf_edited=();		# Edited this file yet?
+my $list;			# List url on STDOUT?
+
+my %stat=();			# stat($lf_url): 'd' for dir, 'f' for others
+
+# ######################### Libwww-perl addons:
+
+sub URI::URL::_generic::unix_path {
+  my $self = (shift)->clone;
+
+  $self->frag(undef);
+  $self->scheme("file");
+  $self->unix_path;
+}
+
+
+sub URI::URL::_generic::basename {
+  my $self = shift;
+  my @p = $self->path_components;
+  my $old = $p[-1];
+  if (@_) {
+    splice(@p, -1, 1, shift);
+    $self->path_components(@p)
+  }
+  $old;
+}
+
+
+sub URI::URL::_generic::path_components {
+    my $self = shift;
+    my $old = $self->{'path'};
+    $old = '' unless defined $old;
+    $old = "/$old" if $old !~ m|^/| && defined $self->{'netloc'};
+    if (@_) {
+	$self->_elem('path',
+		     join("/", map { uri_escape($_,$URI::URL::reserved)
+				   } @_));
+    }
+    map { uri_unescape($_) } split("/", $old, -1);
+}
+
+
+# ######################### Configuration/argument parsing
+
+sub parse_args {
+  my $f;
+  my $i;
+
+  $i=0;
+
+  while ($f=shift) {
+    $i++;
+    # This is a demonstration against Getopts::Long.
+    if ($f =~ s/^-+//) {
+      $verbose=-1,next if $f eq 'q'; 		# Quiet
+      $verbose=1,next if $f eq 'c';		# Chatty
+      die "w3mfix version $VERSION\n" if $f eq 'v';	# Version
+      die "rtfm\n" if ($f eq 'help' || $f eq 'h' || $f eq '?');
+
+      if ($f eq 'editref') {
+	die "Sorry, can only have one -editref pr. run\n"
+	  if $editthis;
+	$editthis=quotemeta(shift);
+	next;
+      }
+
+      if ($f eq 'd') {	# Debugging level
+	$f=shift;
+	unless (($debug = $f) > 0) {
+	  die "w3mfix: debug level must be a number greater than zero.\n";
+	}
+	next;
+      }
+
+      # Those were all the options...
+      warn "w3mfix: Unknown option: -$f.  Use -h for usage info.\n";
+      exit(1);
+    } else {
+      # If we get this far then ... it's a configuration file name:
+      $rc = $f;
+      die "w3mfix: Got a non-option argument that wasn't the name of a\n".
+	"(configuration) file either\n"
+	  unless -f $f;
+    }
+  }
+}
+
+
+sub parse_cfg_file {
+  # Read the configuration file.  Aborts on errors.
+  # Ignores w3mir options w3mfix does not need itself.
+
+  my ( $file ) = @_ ;
+  my ($key, $value, $authserver,$authrealm,$authuser,$authpasswd);
+  my $i;
+
+  die "w3mfix: config file $file is not a file.\n" unless -f $file;
+  open(CFGF, $file) || die "Could not open config file $file: $!\n";
+
+  # print STDERR "Reading $file\n";
+
+  $i=0;
+
+  while (<CFGF>) {
+    # Trim off various junk
+    chomp;
+    s/^#.*//;
+    s/^\s+|\s$//g;
+    # Anything left?
+    next if $_ eq '';
+    # Examine remains
+    $i++;
+    ($key, $value) = split(/\s*:\s*/,$_,2);
+    $key = lc $key;
+
+    # These are no-ops in w3mfix
+    next if ( $key eq 'initial-referer' );
+    next if ( $key eq 'header' );
+    next if ( $key eq 'pause' );
+    next if ( $key eq 'retry-pause' );
+    next if ( $key eq 'retries' );
+    next if ( $key eq 'robot-rules' );
+    next if ( $key eq 'remove-nomirror' );
+    next if ( $key eq 'file-disposition' );
+    next if ( $key eq 'http-proxy' );
+    next if ( $key eq 'proxy-options' );
+    next if ( $key eq 'auth-domain' );
+    next if ( $key eq 'auth-user' );
+    next if ( $key eq 'auth-passwd' );
+    next if ( $key eq 'disable-headers' );
+    next if ( $key eq 'agent' );
+
+    $debug=numeric($value),next if ( $key eq 'debug' );
+    umask(numeric($value)),next if ( $key eq 'umask' );
+    $indexname=$value,next if ($key eq 'index-name');
+    $verbose=nway($value,'quiet','brief','chatty')-1,next
+      if ( $key eq 'verbosity' );
+
+    if ( $key eq 'cd' ) {
+      $chdirto=$value;
+      next;
+    }
+
+    if ($key eq 'url') {
+      my ($rum_url_o,$lf_dir,$rum_reurl,$rum_rebase);
+
+      # A two argument URL: line?
+      if ($value =~ m/^(.+)\s+(.+)/i) {
+	# Two arguments.
+	$rum_url_o=url $1;
+	# The last is a directory, it must end in /
+	$lf_dir=$2;
+	$lf_dir.='/' unless $lf_dir =~ m~/$~;
+
+	# The first is a URL, make it more canonical, find the base.
+	# The namespace confusion in this section is correct.(??)
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+	# print "URL: ",$rum_url_o->as_string,"\n";
+	# print "Base: $rum_rebase\n";
+
+	# Translate from rum space to lf space:
+	push(@internal_apply,"s/^".$rum_rebase."/".quotemeta($lf_dir)."/");
+
+	# That translation could lead to information loss.
+	$infoloss=1;
+
+	# Fetch rules tests the rum_url_o->as_string.  Fetch whatever
+	# matches the base.
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+
+	# Ignore whatever did not match the base.
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+
+      } else {
+	# $rum_url_o=root_quene($value);
+
+	$rum_url_o=url $value;
+
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+	# Translate from rum space to lf space:
+	push(@internal_apply,"s/^".$rum_rebase."//");
+
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+      }
+      next;
+    }
+
+    if ($key eq 'also' || $key eq 'also-quene') {
+      if ($value =~ m/^(.+)\s+(.+)/i) {
+	my ($rum_url_o,$lf_dir,$rum_reurl,$rum_rebase);
+	# Two arguments.
+	$rum_url_o=url $1;
+	$rum_url_o->host(lc $rum_url_o->host);
+	# The last is a directory, it must end in /
+	$lf_dir=$2;
+	$lf_dir.='/' unless $lf_dir =~ m~/$~;
+
+	# The first is a URL, find the base
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+	# Ok, now we can transform and select stuff the right way
+	push(@internal_apply,"s/^".$rum_rebase."/".quotemeta($lf_dir)."/");
+	$infoloss=1;
+
+	# Fetch rules tests the rum_url_o->as_string.  Fetch whatever
+	# matches the base.
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+
+	# Ignore whatever did not match the base.  This cures problem
+	# with '..' from base in in rum space pointing within the the
+	# scope in ra space.  We introduced a extra level (or more) of
+	# directories with the apply above.  Must do same with 'Also:'
+	# directives.
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+      } else {
+	die "Also: requires 2 arguments\n";
+      }
+      next;
+    }
+
+    if ($key eq 'quene') {
+      # root_quene($value);
+      next;
+    }
+
+    if ($key eq 'ignore-re' || $key eq 'fetch-re') {
+      # Check that it's a re, better that I am strict than for perl to
+      # make compilation errors.
+      unless ($value =~ /^m(.).*\1[gimosx]*$/) {
+	print STDERR "w3mfix: $value is not a recognized regular expression\n";
+	exit 1;
+      }
+    }
+
+    if ($key eq 'fetch' || $key eq 'fetch-re') {
+      my $expr=$value;
+      $expr = wild_re($expr).$ipost if ($key eq 'fetch');
+      $rule_text.=' return 1 if '.$expr.";\n";
+      next;
+    }
+
+    if ($key eq 'ignore' || $key eq 'ignore-re') {
+      my $expr=$value;
+      $expr = wild_re($expr).$ipost if ($key eq 'ignore');
+      $rule_text.=' return 0 if '.$expr.";\n";
+      next;
+    }
+
+
+    if ($key eq 'apply') {
+      unless ($value =~ /^s(.).*\1.*\1[gimosxe]*$/) {
+	print STDERR
+	  "w3mfix: '$value' is not a recognized regular expression\n";
+	exit 1;
+      }
+      push(@user_apply,$value) ;
+      $infoloss=1;
+      next;
+    }
+
+    if ($key eq 'options') {
+      
+      my($val,$nval);
+      foreach $val (split(/\s*,\s*/,lc $value)) {
+	if ($i==1) {
+	  $nval=nway($val,'recurse','no-date-check','only-nonexistent',
+		     'list-urls','lowercase','remove','batch','read-urls',
+		     'abs');
+	  $r=1,next if $nval==0;
+	  next if $nval==1;
+	  next if $nval==2;
+	  $list=1,next if $nval==3;
+	  if ($nval==4) {
+	    $lc=1;
+	    $iinline=($lc?"(?i)":"");
+	    $ipost=($lc?"i":"");
+	    next ;
+	  }
+	  next if $nval==5;
+	  next if $nval==6;
+	  next if $nval==7;
+	  $abs=1,next if $nval==8;
+        } else {
+	  die "w3mfix: options must be the first directive in the config file.\n";
+	}
+      }
+    }
+
+    if ($key eq 'fixup') {
+
+#      chomp($fixrc=`pwd` || '.');
+#      $fixrc.="/$file";
+#      warn "Fixrc: $fixrc\n";
+#      $fixup=1;
+
+      my($val,$nval);
+      foreach $val (split(/\s*,\s*/,lc $value)) {
+	$nval=nway($val,'on','run','noindex');
+	next if $nval==0;
+	next if $nval==1;
+	$doindex=0 if $nval==2;
+	# Ignore everyting else
+      }
+    }
+
+  }
+  close(CFGF);
+}
+
+sub wild_re {
+  # Here we translate unix wildcard subset to to perlre
+  local($_) = shift;
+  s#\*#\.\*#;
+  s#\?#\.#;
+  s#([\/\(\)\\\|\{\}\+)\$\^])#\\$1#g;
+  return $_ = '/'.$_.'/';
+}
+
+
+sub numeric {
+  # Check if argument is numeric?
+  my ( $number ) = @_ ;
+  return oct($number) if ($number =~ /\d+/ || $number =~ /\d+.\d+/);
+  die "Expected a number, got \"$number\"\n";
+}
+
+
+sub boolean {
+  my ( $boolean ) = @_ ;
+
+  $boolean = lc $boolean;
+
+  return 0 if ($boolean eq 'false' || $boolean eq 'off' || $boolean eq '0');
+  return 1 if ($boolean eq 'true' || $boolean eq 'on' || $boolean eq '1');
+  die "Expected a boolean, got \"$boolean\"\n";
+}
+
+
+sub nway {
+  my ( $value ) = shift;
+  my ( @values ) = @_;
+  my ( $val ) = 0;
+
+  $value = lc $value;
+  while (@_) {
+    return $val if $value eq shift;
+    $val++;
+  }
+  die "Expected one of ".join(", ",@values).", got \"$value\"\n";
+}
+
+
+sub stat {
+
+  my $file = shift;
+
+  if (exists($stat{$file})) {
+    print STDERR "++Cache hit: $file\n" if $debug;
+  } else {
+    stat($file);
+    if (-e _) {
+      $stat{$file}=(-d _)?'d':'f';
+    } else {
+      $stat{$file}='n';
+    }
+    print STDERR "--Cache miss: $file\n" if $debug;
+  }
+  return $stat{$file};
+}
+
+# ######################## Read 'state' files ##############################
+
+sub read_state {
+
+  my $reffile='.referers';
+  my $refered;
+  my @referers;
+
+  $reffile="referers" if $win32;
+
+  print STDERR "reading $reffile\n" if $verbose>0;
+  open(REFERERS,"< $reffile") ||
+    die "Could not open $reffile for reading: $!\n";
+
+  while (<REFERERS>) {
+    chomp;
+    ($refered,undef,@referers) = split(/\s+/);
+    $rum_referers{$refered}= [ @referers ];
+    # print STDERR $refered," <- ",join(' and ',@referers),"\n";
+  }
+
+  close(REFERERS);
+
+
+  # Read redirection report
+
+  my $redirfile='.redirs';
+  my $wrong;
+  my $right;
+  my $tmp;
+
+  $redirfile="redirs" if $win32;
+
+  print STDERR "reading $redirfile\n" if $verbose>0;
+  open(REDIRS,"< $redirfile") ||
+    die "Could not open $redirfile for reading: $!\n";
+
+  while (<REDIRS>) {
+    chomp;
+    ($wrong,undef,$right) = split(/\s+/);
+
+    $rum_redirected{$wrong}=$right;
+  }
+
+  close(REDIRS);
+}
+
+# ######################### Process every single tag ########################
+
+sub process_tag {
+  # Process a tag in html file
+  my $lf_referer = shift;
+  my $base_url = shift;
+  my $tag_name = shift;
+  my $url_attrs = shift;
+
+  # Retrun quickly if no URL attributes
+  return unless defined($url_attrs);
+
+  my $attrs = shift;
+
+  # Information loss through apply or processing in this procedure?
+  my $il = $infoloss; 
+  my $redirs;
+  my $stat;
+
+  my $rum_url;	# The absolute URL
+  my $lf_url;	# The local filesystem url
+  my $lf_url_o; # ... and it's object
+  my $key;
+  my $orig_rum_url;
+#  my $debug = 1;
+
+#  print STDERR "\nProcess Tag: $tag_name, URL attributes: ", join(', ',@{$url_attrs}),"\nOrigin:",$base_url,"\n"; # if $debug>2;
+
+  substr($lf_referer,0,0)='./' unless substr($lf_referer,0,1) eq '/';
+
+  foreach $key (@{$url_attrs}) {
+    if (defined($$attrs{$key})) {
+      $orig_rum_url=$rum_url=$$attrs{$key};
+
+      print STDERR "\n$key = $rum_url\n" if $debug;
+
+      # Apply redirects:
+      $redirs=0;
+      if (exists($rum_redirected{$rum_url})) {
+	$il=1;
+	while (exists($rum_redirected{$rum_url})) {
+	  die "Too many redirects in a row\n" if $redirs++>32;
+	  print STDERR "$rum_url -> ".$rum_redirected{$rum_url}."\n"
+	    if $debug;
+	  $rum_url=$rum_redirected{$rum_url};
+	}
+      }
+
+      # Apply program/user apply rules
+      $lf_url=apply($rum_url);
+
+      if (defined($lf_url)) {
+	# Apply directory/file check here
+	$stat=&stat($lf_url);
+
+	if ($stat eq 'f' && $lf_url =~ m~/$~) {
+	  # It's a file, remove trailing /
+	  print STDERR "****** File / fixup of $lf_url\n" if $debug;
+	  $il=1;
+	  substr($lf_url,length($lf_url)) = '';
+	} elsif ($stat eq 'd' && !($lf_url =~ m~/$~) ) {
+	  # It's a directory, add a trailing /
+	  print STDERR "****** Directory / fixup of $lf_url\n" if $debug;
+	  $il=1;
+	  $lf_url .= '/';
+	}
+
+	substr($lf_url,0,0)='./' unless substr($lf_url,0,1) eq '/';
+
+	$lf_url_o=url $lf_url;
+	my $tmp=$lf_url_o->clone;
+	$tmp->basename($indexname);
+
+	if ( $doindex && $lf_url_o->basename eq "" && 
+	     &stat($tmp->unix_path) eq 'f' &&
+	     !$htmlop::isdir{$key}) {
+	  $lf_url_o=$tmp;
+	  $il=1;
+	  print STDERR "indexname adjusted to ",$lf_url_o->as_string,"\n"
+	    if $debug;
+	}
+		 
+	# Save new value in the hash, make it a file url to get 'rel' working
+	$lf_url_o->scheme('file');
+	$$attrs{$key} = ($lf_url_o->rel("file:".$lf_referer))->as_string;
+
+	printf STDERR "Saved ".$$attrs{$key}."\n" if $debug;
+
+	# If there is potential information loss save the old value too
+	$$attrs{"W3MIR".$key}=$orig_rum_url if $il;
+
+      } elsif ($redirs>0) {
+	$$attrs{$key}=$rum_url;
+	print STDERR "Saved ".$$attrs{$key}."\n" if $debug;
+	$$attrs{"W3MIR".$key}=$orig_rum_url;
+      }
+    }
+  }
+}
+
+
+# ###################### Edit the URLs in one file... ########################
+
+sub edit_html_file {
+  # Check if it's a html file.  I know this tag is in all html
+  # files, because w3mir put it there.
+
+  my($lf_url)=shift;
+  my($rum_url)=shift;
+
+  # Figure out the filename for our local filesystem.
+  $lf_url.=$indexname if $lf_url =~ m~/$~ || $lf_url eq '';
+
+  # Stuff in need of unquoting?
+  $lf_url = (url "file:$lf_url")->local_path if $lf_url =~ /\%\d\d/;
+
+  if (exists($lf_edited{$lf_url})) {
+    print STDERR "Already edited $lf_url\n" if $debug;
+    return ;
+  }
+
+  $lf_edited{$lf_url}=1;
+
+  my $page;
+  my $newpage;
+  my $read;
+  my $atime;
+  my $mtime;
+
+  # dev  uno   mode  nlink uid   gid   rdev  size  atime mtime
+  (undef,undef,undef,undef,undef,undef,undef,undef,$atime,$mtime)
+    = stat($lf_url);
+
+  if (!open(TMPF,"< $lf_url\n")) {
+    warn "Cannot read $lf_url: $!\n" if $verbose>=0;
+    return;
+  }
+
+  $read=sysread(TMPF,$page,10240,0);
+  close(TMPF);
+
+  if (! $page =~ /<HTML/i) {
+    print STDERR "$lf_url is not html\n" if $verbose>0;
+    return ;
+  }
+
+  $files++;
+
+  print STDERR "w3mfix: $lf_url" if $verbose>=0;
+  
+  print STDERR "$lf_url is a html file\n" if $debug;
+
+  print STDERR " reading" if $verbose>0;
+  
+  open(TMPF,$lf_url) || 
+    die "Could not open $lf_url for reading: $!\n";
+  # read the whole file.
+  {
+    local($/)=undef;
+    $page = <TMPF>;
+  }
+  close(TMPF);
+
+  print STDERR " ",length($page)," bytes" if $verbose>0;
+
+  # It's a html document
+
+  print STDERR ", editing" if $verbose>0;
+
+  ($newpage,undef) = 
+    &htmlop::process($page, # $htmlop::NODOC,
+		     $htmlop::ABS,$rum_url,
+		     $htmlop::USESAVED,'W3MIR',
+		     $htmlop::TAGCALLBACK,\&process_tag,$lf_url);
+
+  open(TMPF,">$lf_url") ||
+    die "\nCould not open $lf_url for writing: $!\n";
+  
+  print STDERR ", saving" if $verbose>0;
+
+  if (length($newpage)) {
+    # This is ODD: close does not seem to flush the buffers.  So we
+    # force the issue.
+    local($|)=1;
+    print TMPF $newpage || 
+      die "\nCould not write to $lf_url (disk full?): $!\n";
+  }
+
+  close(TMPF) || 
+    die "\nCould not close $lf_url after writing: $!\n";
+
+  # Set times back to what they were.
+  utime $atime,$mtime,$lf_url;
+
+  print STDERR ".\n" if $verbose>=0;
+}
+
+# ############################### Scope test
+
+sub want_this {
+  # Find out if we want the url passed.  Just pass it on to the
+  # generated functions.
+  my($rum_url)=shift;
+
+  # Does scope rule want this?
+  return &$scope_code($rum_url) &&
+    # Does user rule want this too?
+    &$rule_code($rum_url) 
+  
+}
+
+# ############################### Apply the apply rules
+
+sub user_apply {
+  # Apply the user apply rules
+  return &$user_apply_code(shift);
+}
+
+
+sub internal_apply {
+  # Apply the w3mir generated apply rules
+  return &$apply_code(shift);
+}
+
+
+sub apply {
+  # Apply the user apply rules.  Then if URL is wanted return result of
+  # w3mir apply rules.  Return the undefined value otherwise.
+
+  my $url = user_apply(shift);
+
+  return undef unless want_this($url);
+
+  internal_apply($url);
+}
+
+# ############################### Decide what URLs to edit
+
+sub edit_as_needed {
+  my $rum_redirected;
+  my $rum_url;
+  my $o_rum_url;
+  my $rum_referer;
+  my $lf_url;
+  my $foo;
+  my $redirs;
+
+  if ($editthis) {
+
+    # Find the URLs that match $edithis
+    foreach $o_rum_url (keys %rum_referers) {
+      # Work on them if they (now) fall within the scope of retrival
+      $redirs=0;
+      $rum_url=$o_rum_url;
+      if (exists($rum_redirected{$o_rum_url})) {
+	while (exists($rum_redirected{$rum_url})) {
+	  die "Too many redirects in a row\n" if $redirs++>32;
+	  print STDERR "$rum_url -> ".$rum_redirected{$rum_url}."\n"
+	    if $debug;
+	  $rum_url=$rum_redirected{$rum_url};
+	}
+      }
+      next unless $rum_url =~ /$editthis/io;
+      next unless want_this($rum_url);
+      # Find and edit the documents containing references to $o_rum_url
+      foreach $rum_referer (@{$rum_referers{$o_rum_url}}) {
+	next if $rum_referer eq '(commandline)';
+	$lf_url=apply($rum_referer);
+	next unless defined($lf_url);
+	edit_html_file($lf_url,$rum_referer);
+      }
+    }
+    
+    # Don't do anything else when invoked thus
+    return;
+  }
+
+  if ($doindex) {
+    # Edit everything that refers anything with trailing /
+    foreach $rum_url (grep(/\/$/,keys %rum_referers)) {
+      foreach $rum_referer (@{$rum_referers{$rum_url}}) {
+	next if $rum_referer eq '(commandline)';
+	$lf_url=apply($rum_referer);
+	next unless defined($lf_url);
+	edit_html_file($lf_url,$rum_referer);
+      }
+    }
+  }
+
+  # Edit only redirected stuff
+  foreach $rum_redirected (keys %rum_redirected) {
+    # print "Redirected $rum_redirected\n";
+    foreach $rum_url (@{$rum_referers{$rum_redirected}}) {
+      # print "- Found in $rum_url\n";
+      $lf_url=apply($rum_url);
+      next unless defined($lf_url);
+      edit_html_file($lf_url,$rum_url);
+    }
+  }
+}
+
+# ############################### 'main'
+
+&parse_args(@ARGV);
+
+if (!$rc) {
+  $rc='.w3mirc';
+  $rc='w3mir.ini' if $win32;
+}
+
+print STDERR "w3mfix: rc file: $rc\n" if $verbose>0;
+
+&parse_cfg_file($rc);
+
+# $chdirto is ignored, w3mir already did it
+
+# Compile second order code
+
+# - The rum scope tests
+my $full_rules=$scope_prefix.$scope_fetch.$scope_ignore.$scope_postfix;
+eval $full_rules;
+
+# warn "Scope rules:\n-------------\n$full_rules\n---------------\n";
+
+die "Program generated rules did not compile.  The code is:\n----\n".
+  $full_rules."\n----\n"
+  if !defined($scope_code);
+
+$full_rules=$rule_prefix.$rule_text.$rule_postfix;
+eval $full_rules;
+
+# warn "User rules:\n-------------\n$full_rules\n---------------\n";
+
+# - The user specified rum tests
+die "Ignore/Fetch rules did not compile.  The code is:\n----\n".
+  $full_rules."\n----\n"
+  if !defined($rule_code);
+
+# - The user specified apply rules
+
+my $full_apply=$apply_prefix.($lc?$apply_lc:'').
+  join($ipost.";\n",@user_apply).(($#user_apply>=0)?$ipost:"").";\n".
+  $apply_postfix;
+eval $full_apply;
+
+die "User apply rules did not compile. The code is:
+----
+".$full_apply."
+----\n" if !defined($apply_code);
+
+$user_apply_code=$apply_code;
+
+# - The w3mir generated apply rules
+
+$full_apply=$apply_prefix.($lc?$apply_lc:'').
+  join($ipost.";\n",@internal_apply).(($#internal_apply>=0)?$ipost:"").";\n".
+  $apply_postfix;
+eval $full_apply;
+
+die "Internal apply rules did not compile.  The code is:
+----
+".$full_apply."
+----\n" if !defined($apply_code);
+
+&read_state;
+
+&edit_as_needed;
+
+exit 0;
+__END__
+# -*- perl -*-  There must be a blank line here:
+
+=head1 NAME
+
+w3mfix - fixup program for w3mir
+
+=head1 SYNOPSIS
+
+B<w3mfix> [B<options>] [B<configuration-file>]
+
+=head1 DESCRIPTION
+
+B<w3mfix> is the companion program to L<w3mir>.  It can be used for
+several URL editing operations usefull in different situations.  
+
+When starting B<w3mfix> will read it's configuration file.  It's name is 
+either .w3mirc (w3mir.ini on win32) or specified on the commandline.
+
+B<w3mfix> is controlled by the 'Fixup' directive of the configuration
+file (described in the L<w3mir> documentation).  B<w3mfix> is also
+affected by 'Index-name' and the one special commandline option it
+knows, as well as the directives/options controlling verbosity and
+debugging information.
+
+=head1 DESCRIPTION
+
+B<w3mfix> can rewrite URLs in these ways:
+
+=over 4
+
+=item * Rewrite URLs that resutled in redirects to point to the place
+redirected to.  This is needed in all cases and will always be fixed
+by B<w3mfix>.
+
+=item * Change URLs ending in .../ into .../index.html (or
+.../Welcome.html).  This is, probably, not needed when the mirror is
+meant do be used with a web-server.  It is usefull for browsing
+directly from disk or CDROM, but in this case it's, most often,
+required.  To disable this specify the I<noindex> option with the
+'Fixup' directive.
+
+The default is to transform URLs ending in .../ into .../index.html.
+
+To controll the name of the index file use the I<Index-name> directive
+as documented in L<w3mir>
+
+=item * Change URL links to documents outside the mirror to point to
+some local document.  Could be usefull if the mirror is destined for a
+CDROM to be used on a unconnected machine.
+
+THIS IS NOT YET IMPLEMENTED
+
+=item * Change URL links to documents that L<w3mir> was
+unable/forbidden to retrive to point to some local document.  Pointing
+these to a nice informative document is probably better than random
+error messages from the browser.
+
+THIS IS NOT YET IMPLEMENTED
+
+=item * And, least, but far from last, B<w3mfix> can be used to
+prepare an established mirror for enlargement.
+
+This feature is used thus: Add the new site or subsite to be mirrored
+on the configuration file (by adding B<Also:> and B<Also-quene:>
+directives).  Then run B<w3mfix> with the B<-editref> option.  When
+the B<-editref> option is specified B<w3mfix> will not perform any
+other editing tasks.
+
+E.g.; To add I<http://www.yahoo.com/Science/Artificial_Life/> to your
+mirror add something like
+
+	  Also: http://www.yahoo.com/Science/Artificial_Life/ yahoo
+
+to the configuration file, then run w3mfix:
+
+	  w3mfix -editref www.yahoo.com/Science/Artificial_Life
+
+This will cause all references to
+I<www.yahoo.com/Science/Artificial_Life> (and under) to be edited so they
+point to within the mirror.  After B<w3mfix> has finished you can run
+L<w3mir> in the normal manner.
+
+=back
+
+=head1 BUGS
+
+Naah.
+
+=head1 SEE ALSO
+
+L<w3mir>
+
+=head1 AUTHORS
+
+B<w3mir>s authors can be reached at I<w3mir-core@usit.uio.no>.
+B<w3mir>s home page is at http://www.math.uio.no/~janl/w3mir/
+
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir-HOWTO.html
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir-HOWTO.html	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir-HOWTO.html	(revision 21112)
@@ -0,0 +1,943 @@
+<!doctype html public "-//W3C//DTD HTML 4.0//EN">
+<html>
+<head>
+<title>W3MIR HOWTO</title>
+<style type="text/css">
+<!--
+  body { background-color: white }
+  h1, h2, h3, b { font-family: sans-serif }
+  .red { color: red }
+-->
+</style>
+<body>
+<h1>W3MIR HOWTO</h1>
+
+<p><b>Corresponding to w3mir version 1.0.2 and above</b>
+
+<p>W3mir is an all purpose WWW copying and mirroring program.  Its
+main focus is copying complete directory structures keeping your copy
+browseable through a web server, or directly off a disk or CDROM if
+you want.  W3mir will fix URLs that are redirected and everything else
+that needs to be fixed to make your copy browseable.  But it also does
+odd jobs, retrieving single documents, batch getting several documents
+and more.  You may tell w3mir not to change anything in the retrieved
+documents.  W3mir has been in development quite a long time so you
+find options to do a lot of things needed when copying things off the
+web.
+
+<p>With w3mir you may copy the entire contents a web server.  Or just
+a directory hierarchy, or several related hierarchies off as many
+servers as you like.  They don't even have to be related.
+
+<p>W3mir supports HTML4, and has partial support for CSS, Java,
+ActiveX and Adobe Acrobat (PDF) files.  And it works on Win32
+machines.
+
+<p><b>Warning:</b> W3mir enables you to copy a lot of things off the
+Web, but remember, the things you retrieve might be copyrighted and
+the copy you make with w3mir might in fact be illegal to make and
+posses.
+
+<hr>
+
+<h2><a name="contents">Contents</a></h2>
+
+<p><a href="#intro">README</a> (You want to read this!  <b
+class="red">Really!</b>)
+
+<p><b>How do I...</b>
+<ol>
+  <li><p><a href="#copy">copy a file?</a>
+  <li><p><a href="#recurse">copy a directory hierarchy?</a>
+  <li><p><a href="#resources">copy the needed resource files from another
+    directory hierarchy?</a>
+  <li><p><a href="#ignore">avoid copying files I don't want or copy only
+    files I want?</a>
+  <li><p><a href="#rm">remove the files that are no longer on the
+    original site from the mirror?</a>
+  <li><p><a href="#depth">limit how deep w3mir will recurse?</a>
+  <li><p><a href="#memory">limit w3mirs memory usage?</a>
+  <li><p><a href="#multi">copy files from multiple sites?</a>
+  <li><p><a href="#alias">copy files from one server with several names?</a>
+  <li><p><a href="#aborted">restart a mirror process after stopping it
+    prematurely?</a>
+  <li><p><a href="#enlarge">enlarge or prune an established mirror?</a>
+  <li><p><a href="#cat">'cat' a file?</a>
+  <li><p><a href="#list">list URLs in a document?</a>
+  <li><p><a href="#robots">disable robots.txt obedience?</a>
+  <li><p><a href="#corrupt">stop w3mir from corrupting binary files?</a>
+  <li><p><a href="#auth">copy a site that wants user-name and password?</a>
+  <li><p><a href="#mauth">access a site that wants several different
+    user-names and passwords?</a>
+  <li><p><a href="#proxy">use a proxy server?</a>
+  <li><p><a href="#pauth">authenticate myself to a proxy server?</a>
+  <li><p><a href="#proxytweak">ensure that the proxy server ...?</a>
+  <li><p><a href="#batchget">batch get files with w3mir?</a>
+  <li><p><a href="#cgi">handle CGI?</a>
+  <li><p><a href="#imap">handle server side image-maps?</a>
+  <li><p><a href="#java">handle Java and ActiveX?</a>
+  <li><p><a href="#script">handle java-script and other script languages?</a>
+  <li><p><a href="#css">handle the other things with 'partial support'?</a>
+  <li><p><a href="#anon">keep my identity secret?</a>
+  <li><p><a href="#ns">pretend that I'm using Netscape, Internet
+    Explorer or Lynx?</a>
+  <li><p><a href="#other">do other things?</a>
+</ol>
+
+<hr>
+
+<h2><a name="intro">README</a></h2>
+
+<p>W3mir may be used in two, main, ways:
+
+<ul>
+  <li><p>To copy something random once.
+  <li><p>To keep a local mirror of some remote site
+</ul>
+
+<p>To copy something random once there is a high likeliness you can
+just start w3mir with some simple options and it will do the job you
+want it to.  Providing that the remote site is not too complex and
+your expectations of the copy aren't high :-) This is what wget, the
+gnu w3 mirroring program, does and is good at.
+
+<h3>Configuration file</h3>
+
+<p>Once you want to keep a copy of a remote site up-to-date over time,
+mirror something with server side image-maps, redirects or
+authentication you have to write a configuration file for w3mir.  This
+is what w3mir is good at, compared to wget.  Writing the file is not
+hard, and there are two example files in the w3mir distribution.  It
+will also be explained here.  The configuration file is typically
+called <tt>.w3mirc</tt> (<tt>w3mir.ini</tt> on win32 machines), and
+can be written with a simple text editor.  It is kept in the top
+directory of the mirror, where w3mir will find it when it starts.
+Please refer to the <a href="#contents">contents</a> for how to handle
+a specific problem with a configuration file.
+
+<hr>
+
+<h2>The answers:</h2>
+
+<hr>
+
+<h3><a name="copy">How do I copy a file?</a></h3>
+
+<p>To copy the top page off www.starwars.com:
+
+<p><tt>w3mir http://www.starwars.com/</tt>
+
+<p><b>Note:</b> it is <em>important</em> that you give the trailing
+slash for server names and directories.
+
+<hr>
+
+<h3><a name="recurse">How to I copy a directory hierarchy?</a></h3>
+
+<p>To copy the entire stuff about episode I from www.starwars.com
+which is stored in <tt>http://www.starwars.com/episode-i/</tt> (I don't
+recommend this, it's quite a lot of data):
+
+<p><tt>w3mir -r http://www.starwars.com/episode-i/</tt>
+
+<p>The corresponding configuration file is simple:
+
+<pre>
+Options: recurse
+URL: http://www.starwars.com/episode-i/
+Fixup: run
+</pre>
+
+<p>The <tt>-r</tt> option makes w3mir recurse down from the starting
+point.  It will only copy all the documents under
+http://www.starwars.com/episode-i/ that it sees referenced from those
+same documents.  W3mir will <em>not</em> retrieve documents from
+http://www.starwars.com/ because it is considered to be 'over' the
+starting point.
+
+<p>The command-line will get you a copy that is definitely browseable
+via a WEB server, and possibly browseable directly from a CDROM or
+hard-disk.  To ensure that it is browseable from CDROM and disk you
+need to use a configuration file with the <tt>Fixup: run</tt> line in.
+It causes w3mir to edit anything that needs editing after the mirror
+has completed, including fixing URLs that caused redirects.  The dirty
+work is done by w3mirs helper program w3mfix.  The directive will
+cause w3mfix to be run each time w3mir completes the mirror.
+
+<p><b>Note:</b> it is <em>important</em> that you give the trailing
+slash after the directory name.  Specifying
+<tt>http://www.starwars.com/episode-i</tt> and
+<tt>http://www.starwars.com/episode-i/</tt> is quite different in
+w3mirs eyes.  In the former case episode-i is considered to be a
+document within the / (top) directory of www.starwars.com and w3mir
+will recurse from /, which is a lot more than you wanted.  In the
+latter case w3mir understands that episode-i is a directory and will
+consider that directory to be the staring point, which is what you
+wanted.
+
+<hr>
+
+<h3><a name="resources">How do I copy the needed resource files from
+another directory hierarchy?</a></h3>
+
+<p>Some sites store their documents in one place, and puts their
+banners, icons and such in a separate directory called
+<tt>/images</tt>, <tt>/banners</tt>, <tt>/icons</tt>,
+<tt>/resources</tt> or some such.  Unless you retrieve these as well as
+the documents things will probably not be too colorful.  So, imagine
+that the starwars site stored all the images in one holding directory
+called <tt>/imagery</tt> and you want to copy all the stuff in it that
+the episode-i pages need.  Then you do this:
+
+<pre>
+Options: recurse
+URL: http://www.starwars.com/episode-i/ episode-i
+Also: http://www.starwars.com/imagery/ imagery
+Fixup: run
+</pre>
+
+<p>There are two changes here compared to the simpler file we started
+with: There is an extra argument at the end of the URL directive.  It
+tells w3mir to store everything gotten from
+<tt>http://www.starwars.com/episode-i/</tt> in the subdirectory
+<tt>episode-i</tt>.  The directory can be omitted, but I think its
+neater this way.  Then the new directive 'Also:'.  It tells w3mir that
+you also want whatever the documents under
+<tt>http://www.starwars.com/episode-i/</tt> references under
+<tt>http://www.starwars.com/imagery/</tt>.
+
+<p><b>Note:</b> this will only get stuff that was used by the
+documents under <tt>http://www.starwars.com/episode-i/</tt>, anything
+stored under <tt>http://www.starwars.com/imagery/</tt> which is not
+used will not be retrieved. If you want everything under
+<tt>imagery</tt> to be retrived use the <tt>Also-quene:</tt>
+directive.
+
+<hr>
+
+<h3><a name="ignore">How do I avoid copying files I don't want or copy
+only files I want?</a></h3>
+
+<p>To control what files w3mir copies you can use the
+<tt>Ignore:</tt>, <tt>Fetch:</tt>, <tt>Ignore-RE:</tt> and
+<tt>Fetch-RE:</tt> directives in the configuration file.  The embeded
+references to any file you chose to ignore, i.e., not copy, will point
+at the original site, <em>not</em> to the mirror.  This means that the
+mirror user may still get ahold of the file from the original source
+by simply clicking if she so desires.
+
+<p>If a site contains huge .wav audio files that you are not
+interested in you put
+
+<pre>
+Ignore: *.wav
+</pre>
+
+<p>in the configuration file.  You may ignore as many different
+filename patterns as you want.  If you are mirroring a site you want
+very few, specific files from, say all HTML (named
+<em>something</em><tt>.html</tt>) and all Mpeg video files (named
+<em>something</em><tt>.mpg</tt>) you can write this:
+
+<pre>
+Fetch: *.html
+Fetch: *.mpg
+Ignore: *
+</pre>
+
+<p>W3mir will test each filename against each Fetch/Ignore rule in
+sequence.  A html file will match the first line and be fetched.  Any
+mpg file will match the second line and be fetched.  All other files
+will match the third line, and be ignored.  This last line is needed
+because the default is to get any files which are not ignored.  By
+arranging fetch and ignore rules carefully you may retrieve exactly
+the filename patterns you want and not retrieve anything else.
+
+<p>If you decide you also want all Mpeg Layer 3 audio files
+(<em>something.</em><tt>mp3</tt>) from the site, after the mirror has
+been established.  Then you add this:
+
+<pre>
+Fetch: *.mp3
+</pre>
+
+<p>as the third line, making the <tt>Ignore: *</tt> line the forth and
+last.  Then you must fix all references to .mp3 files within the
+mirror by running w3mfix thus:
+
+<pre>
+w3mfix -editref .mp3
+</pre>
+
+<p>which will edit all references to .mp3 files, pointing them the
+right place, on your disk. Ditto when you remove a fetch rule, or add
+or remove an ignore rule. See the answer about <a
+href="#enlarge">enlarging and pruning</a> mirrors for more examples of
+using <tt>w3mfix -editme ...</tt>
+
+<p><b>Note:</b> when retrieving only a very limited set of files, as
+in the example above, you <em>must</em> retrieve the html files,
+because how else will w3mir find URLs of files to retrieve? Only html
+files contain links to other files.
+
+<p>Similarly, you may chose to not mirror whole branches of the
+original site.  If you for example mirror my home-pages, and you decide
+not to mirror the comics pages you can put
+
+<pre>
+Ignore: /ts/
+</pre>
+
+<p>or more precisely
+
+<pre>
+Ignore: http://www.ifi.uio.no/~janl/ts/
+</pre>
+
+<p>in the configuration file.  If you do this after having established
+the mirror you use w3mfix to fix the references:
+
+<pre>
+w3mfix -editref /ts/
+</pre>
+
+<p><tt>Fetch:</tt> and <tt>Ignore:</tt> rules can only use a very
+limited subset of the Unix wild-cards.  w3mir understands only '?',
+'*', and '[a-z]' ranges.
+
+<p><tt>Ignore-RE:</tt> and <tt>Fetch-RE:</tt> are the same as
+<tt>Fetch:</tt> and <tt>Ignore:</tt> except that they give you access
+to the full power of Regular Expressions to make rules for that to get
+or not to get. They support perls superset of the normal Unix regular
+expression syntax.  They must be completely specified, including the
+prefixed m, a delimiter of your choice (except the paired delimiters:
+parenthesis, brackets and braces), and any of the RE modifiers. I.e.,
+
+<pre>
+Ignore-RE: m/.gif$/i
+</pre>
+
+<p>or
+
+<pre>
+Ignore-RE: m~/.*/.*/.*/~
+</pre>
+
+<p>and so on.  "#" cannot be used as delimiter as it is the comment
+character in the configuration file.
+
+<p>There are some traps when using <tt>Ignore-RE</tt> and
+<tt>Fetch-RE</tt>, please see their documentation in <tt>mandoc
+w3mir</tt> for a more complete explanation.
+
+<hr>
+
+<h3><a name="depth">How do I limit how deep w3mir will recurse?</a></h3>
+
+<p>W3mir has no explicit mechanism to limit the depth of recursion,
+but the same result can be achieved with a simple <tt>Ignore</tt> rule:
+
+<pre>
+Ignore: /*/*/*/*/*/*/
+</pre>
+
+<p>This will ignore any URLs that contain at least 7 slashes ("/").
+Note that a URL contains three slashes that does not have anything to
+do with depth:
+
+<pre>
+http://www.ifi.uio.no/
+</pre>
+
+<p>so only the surplus slashes are used for depth in this match.  In the
+example above the limit is 4 levels from the top.  The
+<tt>Ignore:</tt> rule that is used to limit recursion depth must be
+listed before any <tt>Fetch:</tt> rules to be effective.
+
+<hr>
+
+<h3><a name="memory">How do I limit w3mirs memory usage?</a></h3>
+
+<p>In a mirror consisting of <em>many</em> files, such as a archive of
+an active mailinglist w3mir will build a very large referer table, in
+part for w3mir to use in the <tt>Referer:</tt> header and in part for
+w3mfix to use in fixing references.
+
+<p>If you disable both the <tt>Referer:</tt> header and don't use
+w3mfix w3mir will not build a referer table.  You do this in the
+configuration file:
+
+<pre>
+Disable-headers: referer
+Fixup: off
+</pre>
+
+<p>Please note the potential problems of turning off fixup described
+earlier in this howto.  There are normaly no problems associated with
+simple sites, but if there are redirects fixup <em>is</em> needed for
+a consistent mirror.
+
+<hr>
+
+<h3><a name="rm">How do I remove the files are no longer on the
+original site from the mirror?</a></h3>
+
+<p>Over time the site you mirror will add files, and quite possibly
+remove files.  Or you might introduce new <tt>Ignore:</tt> rules after
+establishing the mirror that reduces the files wanted in the mirror.
+
+<p>By default w3mir will not delete such old files, some people might
+want to keep the files even if they are removed from the original
+site.  To remove the old/unwanted files you add 'remove' to the
+<tt>Options:</tt> line.
+
+<hr>
+
+<h3><a name="multi">How do I copy files from multiple sites?</a></h3>
+
+<p>In the answer to the previous question we see how to mirror several
+related sites.  For example, say you want to mirror all my home-pages
+into one mirror:
+
+<pre>
+Option: recurse
+URL: http://www.math.uio.no/~janl/ math/janl
+Also: http://www.math.uio.no/drift/personer/ math/drift
+Also: http://www.ifi.uio.no/~janl/ ifi/janl
+Also: http://www.mi.uib.no/~nicolai/ math-uib/nicolai
+</pre>
+
+<p>As in the previous example this will only get documents that are
+referenced.  Any documents that are stored at these location but to
+which w3mir finds no references will not be retrieved.  So this will
+fail if the sites are not in any way related, or if you wanted
+<em>everything</em> stored at each site.
+
+<p>To mirror unrelated sites, or get it all you may specify that the
+given URL should be considered a starting-point as well:
+
+<pre>
+Also-quene: http://www.math.uio.no/drift/personer/ math/drift
+</pre>
+
+<p>and, if you want to add an additional starting-point within a already
+named site:
+
+<pre>
+Quene: http://www.math.uio.no/drift/personer/foo.html
+</pre>
+
+<p>Armed with that you should be able to get pretty much anything you
+like.
+
+<hr>
+
+<h3><a name="alias">How do I copy files from one server with several
+names?</a></h3>
+
+<p>Simple, the same way you mirror several servers with different
+names. The math department at University of Oslo has a web server
+known under two names: math-www.uio.no and www.math.uio.no, and both
+names are used in documents stored on it. To copy the whole server,
+one time only, give these URL and Also lines:
+
+<pre>
+URL: http://www.math.uio.no/ .
+Also: http://math-www.uio.no/ .
+</pre>
+
+<p>Note the period/dot (.) at the end of each line.  It means that
+w3mir will store the files in the current directory, i.e. documents
+from both servers will be stored in the same place.  But since w3mir
+asks to only get documents that are newer than the ones it already has
+any document gotten from the server under the www.math.uio.no name
+will not be gotten from the math-www.uio.no name as well.  ... w3mir
+will ask for the document, but the server will tell w3mir that its
+copy is current and there will be no additional transfer of the
+document.
+
+<hr>
+
+<h3><a name="enlarge">How do I enlarge or prune an established
+mirror?</a></h3>
+
+<p>This only works if you use a configuration file.
+
+<p>If you want to add a site or directory to a mirror you simply add
+the needed <tt>Also:</tt> or <tt>Also-Quene:</tt> to the configuration
+file and then you run w3mfix manually, with the -editref option.  If,
+you for example have established a mirror of my home-pages, but want to
+add my wife's home-page you add this
+
+<pre>
+Also: http://www.ifi.uio.no/~annen/ ifi/annen
+</pre>
+
+<p>to the configuration shown earlier.  Then you run w3mfix, and you want
+it to fix all URLs referencing her home-page, the distinguishing
+characteristic is the name 'annen':
+
+<pre>
+w3mfix -editref annen
+</pre>
+
+<p>but 
+
+<pre>
+w3mirx -editref http://www.ifi.uio.no/~annen/
+</pre>
+
+<p>would work too, but it's a lot more to type.  This fixes all the
+references to her home-page so that they point to the mirror instead of
+the original pages.
+
+<p>To prune (cut out something) a mirror you do the same.  Make the
+change in the configuration file and run 'w3mfix -editme ...' to fix
+the references to that which you removed.
+
+<hr>
+
+<h3><a name="cat">How do I 'cat' a file?</a></h3>
+
+<p>W3mir will output the fetched document to its standard output
+(normally your screen/window) if you specify the '-s' command line
+option.  The corresponding configuration file directive is 
+
+<pre>
+File-Disposition: stdout
+</pre>
+
+<hr>
+
+<h3><a name="list">How do I list URLs in a document?</a></h3>
+
+<p>To list the URLs in http://www.math.uio.no/:
+
+<pre>
+w3mir -q -f -l http://www.math.uio.no/
+</pre>
+
+<p>The <tt>-q</tt> switch causes w3mir to produce no other output
+which would disturb the URL listing.  The <tt>-f</tt> switch tells
+w3mir to forget the document once it has been analyzed, i.e., not save
+it on disk.  And finally, the <tt>-l</tt> switch makes w3mir list the
+URLs in the document.  You may combine <tt>-l</tt> with <tt>-r</tt>
+and you need not use it with <tt>-f</tt>.
+
+<p>In the configuration file you put <tt>list</tt> on the
+<tt>Options:</tt> line.
+
+<hr>
+
+<h3><a name="aborted">How to I restart a mirror process after stopping
+it prematurely?</a></h3>
+
+<p>You may just rerun the same command once more.  But that makes
+w3mir request all the documents you have already once more to see if a
+more recent version is available on the server.  You can save time by
+using the <tt>-fs</tt> (Fetch Some) option.  This makes w3mir only
+request documents it does not find on your disk.  E.g.:
+
+<p><tt>w3mir -fs -r http://www.starwars.com/</tt>
+
+<p>This is not something you would normally put in the configuration
+file, but you can, by adding 'only-nonexistent' on the 'Options:' line.
+
+<hr>
+
+<h3><a name="robots">How do I disable robots.txt obedience?</a></h3>
+
+<p>Normally w3mir will read and obey each sites robots.txt file,
+because w3mir wants to be a nice tool. However robots.txt was designed
+with something slightly different than the normal use of w3mir in
+mind, so if you want w3mir to disregard the robot rules you can use
+<tt>-drr</tt> (Disable Robot Rules) on the command-line, or the line
+
+<pre>
+Robot-Rules: off
+</pre>
+
+<p>in the configuration file.  The robot exclusion standard is
+described in <a
+href="http://info.webcrawler.com/mak/projects/robots/norobots.html">http://info.webcrawler.com/mak/projects/robots/norobots.htm</a>.
+
+<hr>
+
+<h3><a name="corrupt">How do I stop w3mir from corrupting binary
+files?</a></h3>
+     
+<p>During the normal course of events w3mir converts the newline
+format of fetched HTML documents to your systems native newline
+format. On Unix a newline consists of a single ASCII LF character, on
+Macintoshes it's a single ASCII CR character and on Dos/Windows it's a
+ASCII CR/LF pair. W3mir understands all these and all HTML files are
+saved in the format your operating system prefers.
+
+<p>If, and this is very unlikely, a web server identifies a binary
+file as HTML w3mir will very likely corrupt the file. If you discover
+a file which is obviously ruined in the mirror, but is not ruined when
+you view it on the original site do this:
+
+<ol>
+
+<li>Notify the webmaster on the original site that the file has the
+wrong MIME type
+
+<li>Use the <tt>-nnc</tt> (No Newline Conversion) option on the
+command line, or
+
+<pre>
+Options: no-newline-conv
+</pre>
+
+in the configuration file.
+
+<li>Remove the corrupt file(s).
+
+<li>Run "<tt>w3mir -fs</tt>...", to fetch only the deleted file(s)
+again.
+
+</ol>
+
+<hr>
+
+<h3><a name="auth">How do I copy a site that wants user-name and
+password?</a></h3>
+
+<p>This can only be done with a configuration file.  Being able to
+give this on the command-line would give the user-name and password away
+to other users of the system, so the ability to give authentication
+information that way has not been put in w3mir.
+
+<p>In the configuration file you put:
+
+<pre>
+Auth-domain: */*
+Auth-user: me
+Auth-passwd: my-password
+</pre>
+
+<p>This will cause w3mir to give the user-name and password each time
+the server asks.  There is no way to make w3mir give the user-name and
+password each time no matter if the server asks or not.
+
+<hr>
+
+<h3><a name="mauth">How do I access a site that wants several
+different user-names and passwords?</a></h3>
+
+<p>If you have several user-names and passwords across
+the server(s) that are copied you need a slightly more advanced
+version of this that associates each user-name/password with a
+authentication "domain".  "Domain" is a HTTP concept.  It is simply a
+grouping of files and documents within a "realm".  One file or a whole
+directory hierarchy can belong to a realm.  One server may have many
+realms.  A user may have separate passwords for each realm, or the
+same password for all the realms the user has access to.  A
+combination of a server name, server port and a realm is called a
+domain.
+
+<pre>
+Auth-domain: theserver:theport/therealm
+Auth-user: me
+Auth-passwd: my-password
+
+Auth-domain: theserver:theport/otherrealm
+Auth-user: other-me
+Auth-password: other-password
+</pre>
+
+W3mir will tell you what the name of the realm is if it is unable to
+authenticate itself with the server. You may also use '*' as the realm
+name if you only copy documents from one realm on that server.
+
+<hr>
+
+<h3><a name="proxy">How do I use a proxy server?</a></h3>
+
+<p>On some secured sites you have to access the Internet through proxy
+servers to get out of the internal network.
+
+<p>A proxy server has a host name, and a port you must use. On the
+command line you simply specify <tt>-P proxy-host-name:proxy-port</tt>. In
+the configuration file you put this:
+
+<pre>
+HTTP-Proxy: proxy-host-name:proxyport
+</pre>
+
+<p>The main advantage of working through proxy servers other than
+security is that you take advantage of any caching the proxy server
+which can speed up retrievals enormously.
+
+<p>Another use of the proxy option is to "prime" the proxy servers
+cache.  I.e. you can use w3mir to fetch the documents through the proxy
+server to ensure that the documents are cached there later when you
+want to read them with your browser.  If you also specify 
+
+<pre>
+File-Disposition: forget
+</pre>
+
+<p>it won't even use any space on your disk, w3mir will just process
+the documents looking for URLs and then <em>not</em> save them.
+
+<hr>
+
+<h3><a name="pauth">How do I authenticate myself to a proxy
+server?</a></h3>
+
+<p>Some proxy servers demands a user-name and password to let you use
+them.  W3mir does not support the domain concept in connection with
+proxy authentication because the author cannot imagine that it will be
+needed.  You need to put this in your configuration file:
+
+<pre>
+HTTP-Proxy-user: proxy-username
+HTTP-Proxy-passwd: proxy-password
+</pre>
+
+<hr>
+
+<h3><a name="proxytweak">How do I ensure that the proxy server
+...?</a></h3>
+
+<p>HTTP/1.0 proxy servers may be told to not use its current copy of
+a document if you specify the <tt>-pflush</tt> command-line option.  Or 
+
+<pre>
+Proxy-Options: refresh
+</pre>
+
+<p>in the configuration file.  This is useful if the proxy has an old
+copy of some document and does not realize that a newer version exists
+on the origin site.  W3mir uses the HTTP/1.0 version of this command
+by default.  You can force w3mir to use the HTTP/1.1 version by adding
+<tt>no-pragma</tt> to the line.  If you do this it will not work at
+all as you want unless the server knows the HTTP/1.1 protocol.
+
+
+<p>HTTP/1.1 proxy servers can be manipulated in a few more ways.  The
+configuration file <tt>Proxy-Options:</tt> directive also takes
+<tt>revalidate</tt> and <tt>no-store</tt> options.  The former tells
+the proxy server to check if there is any newer version available.
+This is, in principle, more network friendly than the <tt>refresh</tt>
+option since it will only cause a copy if there is a newer file
+available.  The <tt>no-store</tt> option tells the proxy server to not
+store the documents you transfer.  This might be useful if the
+documents are 'sensitive' or something like that, but if the proxy
+server does not understand HTTP/1.1 it will not obey this option, and
+it might store the document anyway because the functionality is not
+implemented, so you should not count on this to work.
+
+<hr>
+
+<h3><a name="batchget">How do I batch get files with w3mir?</a></h3>
+
+<p>Normally when fetching files w3mir will process each html (and PDF)
+file to find URLs in them for further retrievals.  This is
+time-consuming, and not always wanted.  Sometimes you simply want to
+get a file, or more, and save it, untouched:
+
+<pre>
+w3mir -B http://www.starwars.com/ http://www.ifi.uio.no/~janl/
+</pre>
+
+<p>There is a companion switch for <tt>-B</tt>, namely <tt>-I</tt>, it
+makes w3mir read URLs from its standard input, one pr. line.  Thus you
+can use w3mir in a pipe to batch get several files whose URLs you find
+in some way.  This is a stupid example:
+
+<pre>
+w3mir -q -l -f http://www.ifi.uio.no/ | w3mir -I -B
+</pre>
+
+<p><tt>-B</tt> may also be used with <tt>-r</tt>, but the only effect
+it will have then is to save the html files unchanged on disk, because
+to recurse w3mir <em>has</em> to examine all the html the documents
+for URLs.
+
+<p><b>Please note</b> that using <tt>-B</tt> combined with <tt>-r</tt>
+for mirroring will probably lead to a unstable mirror, because w3mir
+does not get a chance to manipulate the URLs in the documents as it
+needs to be able to maintain a mirror later, and most important of
+all, w3mir needs all html files to contain a &lt;HTML&gt; tag to be
+able to recognize a HTML file as a HTML file. When running with the
+<tt>-B</tt> switch w3mir will not ensure the presence of this and thus
+we must rely on the original documents author to be nice. This is a
+bad bet. In other words, <b>don't use <tt>-B</tt> for recursive
+mirroring</b>, only for batch copying/mirroring of single documents.
+
+<hr>
+
+<h3><a name="cgi">How do I handle CGI?</a></h3>
+
+<p>There is no way w3mir can duplicate the process that happens on the
+Web server when it comes to CGI.  For some CGI programs w3mir can
+simply copy the output and store on disk.  For other CGI programs this
+is not possible, and the only way out is to make w3mir not get the
+involved files using Ignore rules in the configuration file.  These
+will avoid a lot of cgi programs:
+
+<pre>
+Ignore: *.cgi
+Ignore: *-cgi
+</pre>
+
+<p>You might have to add other/more rules for some sites if they have
+other naming conventions or if it's simply impossible to tell from the
+file-name if it's a CGI or not.
+
+<p>When you add ignore rules this causes two things:
+
+<ol>
+<li><p>W3mir will not retrieve documents matching the rules
+<li><p>W3mir will make all references to matching documents point to
+  the site you mirrored from instead of pointing to a non-existent
+  file in the mirror.
+</ol>
+
+<hr>
+
+<h3><a name="imap">How do I handle server side image-maps?</a></h3>
+
+<p>Server side image-maps is yet another thing it's impossible for
+w3mir to relate to.  w3mir simply cannot handle them.  Put ignore
+rules in the configuration file:
+
+<pre>
+Ignore: *.map
+</pre>
+
+<p>W3mir has full support for client side image-maps though.
+
+<hr>
+
+<h3><a name="java">How do I handle Java and ActiveX?</a></h3>
+
+<p>Java and Active X objects are are included in html pages with a
+<tt>&lt;OBJECT&gt;</tt> or <tt>&lt;APPLET&gt;</tt> tag.  W3mir can
+handle these on one condition: The CODEBASE attribute names the
+directory where the program stores its resources (such as
+subprograms, graphic files, sound, text, and so on) and w3mir must
+have read access to this directory.  Otherwise w3mir is without hope,
+it's impossible to extract the name of the resources the program needs
+in any reliable way.
+
+<p>HTML4 supports a attribute that enumerates the resources the
+program needs, w3mir is not able to use this yet.
+
+<hr>
+
+<h3><a name="script">How do I handle java-script and other script
+languages?</a></h3>
+
+<p>W3mir does its best to pass scripts (java-script, perl-script,
+etc...) embedded in the HTML undamaged.  It cannot, however, extract
+any URLs the script generates and the browser would cause the document
+to refer to or embed in a page. 
+
+<p>It will however work if the script generates relative references
+and there is some other way for w3mir to access the referenced file in
+some other manner.  Or if the script generates absolute references and
+the person browsing the mirror has access to the site named, then the
+user will be able to browse the referenced documents via that other
+server.
+
+<hr>
+
+<h3><a name="css">How to I handle the other things with 'partial
+support'</a></h3>
+
+<p>W3mir has partial support for CSS.  This means that
+<tt>&lt;style&gt</tt> tags and the enclosed style data are passed
+undamaged by w3mir.  W3mir will also retrieve the external CSSes named
+in HTML documents.  But w3mir will <em>not</em> (yet) analyze the
+CSSes data to find URLs of other resources (such as fonts) named in
+these.
+
+<p>W3mir also has partial support for Adobe Acrobat (PDF) files.  This
+means that w3mir can extract URLs from PDF files, and get the named
+documents if you want them.  But w3mir cannot edit those URLs so that
+the PDF files point to the mirror instead of wherever on the original
+site they were pointing.  If the PDF files contain absolute URLs they
+will continue pointing to where they were pointing before.  However,
+if the PDF files contain relative references things will work out.
+
+<p>The reason that URLs in PDF files cannot be edited is that they are
+binary and contain byte pointers.  If the URLs length is changed the
+byte pointers will point to the wrong place in the document.  Writing
+code to correct these pointers would be quite complex.  But if you
+write it I will use it.
+
+<hr>
+
+<h3><a name="anon">How do I keep my identity secret?</a></h3>
+
+<p>The HTTP protocol has a header, <tt>User:</tt> which is recommended
+to use by robots, such as w3mir.  Another way to track you is looking
+at the 'Referer:' header w3mir gives in HTTP requests.  Both can be
+disabled:
+
+<pre>
+Disable-headers: referer, user
+</pre>
+
+<p>If you in addition use a proxy server that many other users use
+there is little probability you can be tracked (easily) by the server
+you are copying things from.  You are however much easier to track
+from the logs in the proxy server.  And a court order is quite likely
+to get you tracked in spite of any precautions you take.  
+
+<p>W3mir does not support cookies and thus you cannot be tracked with
+the help of that mechanism.
+
+<hr>
+
+<h3><a name="ns">How do I pretend that I'm using Netscape, Internet
+Explorer or Lynx?</a></h3>
+
+<p>Some web sites give you different documents when you ask for a
+specific URL based on what browser you use, or even what OS you appear
+to be using.  w3mir identifies itself with a string that looks like
+this:
+
+<p><tt>w3mir/<em>version</em>-<em>release-date</em></tt>
+
+<p>Netscape identifies itself with strings that look something like
+this:
+
+<p><tt>Mozilla/3.01 (X11; I; Linux 2.0.30 i586)</tt>
+
+<p>and Internet Explorer says it's something like this:
+
+<p><tt>Mozilla/2.0 (compatible; MSIE 3.02; Windows NT)</tt>
+
+<p>and Lynx says something like this
+
+<p><tt>Lynx/2.6  libwww-FM/2.14</tt>
+
+<p>You can change w3mirs identification with <tt>-agent 'string'</tt>
+on the command line.  In the configuration file you put
+
+<pre>
+Agent: Mozilla/3.01 (X11; I; Linux 2.0.30 i586)
+</pre>
+
+<p>to pretend w3mir is netscape 3.01.
+
+<hr>
+
+<h3><a name="other">How do I do other things?</a></h3>
+
+<p>This document is by no means a complete list of the things you can
+do with w3mir.  The w3mir man page (<tt>man w3mir</tt> or <tt>perldoc
+w3mir</tt> lists more things, and goes into more detail of how things
+work so you can use the knowledge to do neat things.  There are
+several things mentioned only in the man-page that helps you with
+tricky multi-server mirroring, and gives you better control of what to
+get and not to get and under what name to save it on disk.  And a
+couple of other things...
+
+<hr>
+<address>Nicolai Langfeldt 9/7/1998</address>
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir.PL
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir.PL	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3mir.PL	(revision 21112)
@@ -0,0 +1,3291 @@
+# -*-perl-*-
+
+use Config;
+
+&read_makefile;
+$fullperl = resolve_make_var('FULLPERL') || $Config{'perlpath'};
+$islib = resolve_make_var('INSTALLSITELIB');
+
+$name = $0;
+$name =~ s~^.*/~~;
+$name =~ s~.PL$~~;
+
+open(OUT,"> $name") || 
+  die "Could open $name for writing: $!\n";
+
+print "writing $name\n";
+
+while (<DATA>) {
+  if (m~^\#!/.*/perl.*$~o) {
+    # This substitutes the path perl was installed at on this system
+    # _and_ removed any (-w) options.
+    print OUT "#!",$fullperl,$1,"\n";
+    next;
+  }
+  if (/^use lib/o) {
+    # This substitutes the actuall library install path
+    print OUT "use lib '$islib';\n";
+    next;
+  }
+  print OUT;
+}
+
+close(OUT);
+
+# Make it executable too, and writeable
+chmod 0755, $name;
+
+#### The library
+
+sub resolve_make_var ($) {
+
+  my($var) = shift @_;
+  my($val) = $make{$var};
+
+#  print "Resolving: ",$var,"=",$val,"\n";
+  
+  while ($val =~ s~\$\((\S+)\)~$make{$1}~g) {}
+#  print "Resolved: $var: $make{$var} -> $val\n";
+  $val;
+}
+
+
+sub read_makefile {
+
+  open(MAKEFILE, 'Makefile') || 
+    die "Could not open Makefile for reading: $!\n";
+
+  while (<MAKEFILE>) {
+    chomp;
+    next unless m/^([A-Z]+)\s*=\s*(\S+)$/;
+    $make{$1}=$2;
+#    print "Makevar: $1 = $2\n";
+  }
+
+  close(MAKEFILE)
+}
+
+__END__
+#!/local/bin/perl5 -w
+# Perl 5.002 or later.  w3mir is mostly tested with perl 5.004
+#
+# You might want to change or comment out this:
+use lib '/hom/janl/lib/perl';
+#
+# Once upon a long time ago this was Oscar Nierstrasz's
+# <oscar@cui.unige.ch> htget script.
+#
+# Retrieves HTML pages, creating local copies in the _current_ 
+# directory. The script will check for the last-modified stamp on the
+# document, and will not fetch it if the document isn't changed.
+#
+# Bug list is in w3mir-README.
+#
+# Test cases for janl to use:
+#   w3mir -r -fs http://www.eff.org/ - infinite recursion!
+#     --- but cursory examination seems to indicate confused server...
+#   http://java.sun.com/progGuide/index.html check out the img things.
+#
+# Copyright Holders:
+#   Nicolai Langfeldt, janl@ifi.uio.no
+#   Gorm Haug Eriksen, gorm@usit.uio.no
+#   Chris Szurgot, szurgot@itribe.net
+#   Ed Jordan, ed@olympus.itl.net
+#   Alex Knowles, aknowles@avs.com aka ark.
+# Copying and modification is governed by the "Artistic License" enclosed in
+# the w3mir distribution
+#
+# History (European format date: dd/mm/yy):
+#  oscar 25/03/94 -- added -s option to send output to stdout
+#  oscar 28/03/94 -- made HTTP 1.0 the default
+#  oscar 30/05/94 -- special handling of directory URLs missing a trailing "/"
+#   gorm 20/02/95 -- added mirror capacity + fixed a couple of bugs
+#   janl 28/03/95 -- added a working commandline parser.
+#   janl 18/09/95 -- Changed to use a net http library.  Removed dependency of
+#		     url.pl.
+#   janl 19/09/95 -- Extensive rewrite. Simplified a lot, works better. 
+#		     HTML files are now saved in a new and improved manner,
+#		     which means they can be recognized as such w/o fancy
+#		     filename extention type rules.
+# szurgot 27/01/96-- Added "Plaintextmode" wrapper to binmode PAGE. 
+#                    binmode page is required under Win32, but broke modified
+#                    checking
+#                 -- Minor change added ; to "# '" strings for Emacs cperl-mode
+# szurgot 07/02/96-- When reading in local file for checking of URLs changed
+#                    local ($/) =0; to equal undef;
+#   janl 08/02/96 -- Added szurgot's changes and changed them :-)
+# szurgot 09/02/96-- Added code to strip /#.*$/ from urls when reading from 
+#                    local file
+#                 -- Added hasAlarm variable to w3http.pl. Set to 1 if you have
+#                    alarm(). 0 otherwise.
+#                 -- Moved code setting up the valid extensions list into the
+#                    args processing where it belonged
+#   janl 20/02/96 -- Added szurgot changes again.
+#		  -- Make timeout code work.
+#   		  -- and made another win32 test.
+#   janl 19/03/96 -- Worked through the code for handling not-modified 
+#    		     documents, it was a bit shabby after htmlop was intro'ed.
+#   janl 20/03/96 -- -l fix
+#   janl 23/04/96 -- Added -fs by request (by Rik Faith)
+#   janl 16/05/96 -- Made -R mandatory, added use and support for
+#			w3http::SAVEBIN
+# szurgot 19/05/96-- Win95 adaptions.
+#   janl 19/05/96 -- -C did not exactly work as expected.  Thanks to Petr
+#		     Novak for bug descriptions.
+#   janl 19/05/96 -- Changed logic for @didntget, @got and so on to use
+#		     @queue and %urlstat.
+#   janl 09/09/96 -- Removed -R switch.
+#   janl 14/09/96 -- Added ir (initial referer) switch
+#   janl 21/09/96 -- Made retry code saner.  There probably needs to be a
+#                    sleep before retry comencing switch.  When no tty is
+#                    present it should be fairly long.
+#   gorm 15/09/96 -- Added cr (check robot) switch. Default to 1 (on)
+#   janl 22/09/96 -- Modified gorms patch to use WWW::RobotRules.  Changed
+#		     robot switch to be consistent with current w3mir
+#                    practice.
+#   janl 27/09/96 -- Spelling corrections from charles.curran@ox.ac.uk
+#                 -- Folded in manual diffs from ark.
+#    ark 24/09/96 -- Simple facilities to edit the incomming file(s)
+#   janl 27/09/96 -- Added switch to enable <!--NOMIRROR--> editing and
+#		     foolproofed ark's patch a bit.
+#   janl 02/10/96 -- Added -umask switch.
+#		  -- Redirected documents did not have a meaningful referer
+#		     value (it was undefined).
+#		  -- Got w3mir into strict discipline, found some typos...
+#   janl 20/10/96 -- Mtime is preserved
+#   janl 21/10/96 -- -lc switch added.  Mtime preservation works better.
+#   janl 06/11/96 -- Treat 301 like 302.
+#   janl 02/12/96 -- Added config file code, fetch/ignore rules, apply
+#   janl 04/12/96 -- Better checking of config input.
+#   janl 06/12/96 -- Putting together the URL selection/editing brains.
+#   janl 07/12/96 -- Checking out some bugs.  Adding multiscope options.
+#   janl 12/12/96 -- Adding to and defeaturing the multiscope options.
+#   janl 13/12/96 -- Continuing work in multiscope stuff
+#                 -- Unreferenced file and empty directory removal works.
+#   janl 19/02/97 -- Can extract urls from adobe acrobat pdf files :-)
+#		     Important: It does _not_ edit urls, so they still
+#		     point at the original site(s).
+#   janl 21/02/97 -- Fix -lc bug related to case and the apply things.
+#		  -- only use SAVEURL if needed
+#   janl 11/03/97 -- Finish work on SAVEURL conditional.
+#		  -- Fixed directory removal code.
+#                 -- parse_args did not abort when unknown option/argument
+#		     was specified.
+#   janl 12/03/97 -- Made test case for -lc.  Didn't work.  Fixed it.  I think.
+#		     Realized we have bug w.r.t. hostname caseing.
+#   janl 13/03/97 -- All redirected to URLs within scope are now queued.  
+#		     That should make the mirror more complete, but it won't
+#		     help browsability when it comes to the redirected doc.
+#		  -- Moved robot retrival to the inside of the mirror loop
+#		     since we now possebly mirror several sites.
+#		  -- Changed 'fetch-options' to 'options'.
+#                 -- Added 'proxy-options'/-pflush to controll proxy server(s).
+#   janl 09/04/97 -- Started using URI::URL.
+#   janl 11/04/97 -- Debugging and using URI::URL more correctly various places
+#   janl 09/05/97 -- Added --agent switch
+#   janl 12/05/97 -- Simplified scope checks for root URL, changed URL 'apply'
+#		     processing.
+#		  -- Small output formating fix in the robot rules code.
+#		  -- Version is now 0.99
+#   janl 14/05/97 -- htmpop no-longer puts '<!DOCTYPE...' into doc, so check 
+#		     for '<HTML' instead
+#   janl 11/06/97 -- Made :port optional in server part of auth-domain.
+#			Always removing :80 from server part to match netloc.
+#   janl 22/07/97 -- More debugging of rewrite for new features -B, -I.
+#   janl 01/08/97 -- Fixed bug in RE quoting for Ignore/Fetch
+#   janl 04/08/97 -- s/writepage/write_page/g
+#   janl 07/09/97 -- 0.99b1 is released
+#   janl 19/09/97 -- Kaj Hejer discovers omissions in non-html-url-mining code.
+#                 -- 0.99b2 is released
+#   janl 24/09/97 -- Matt Chapman found bug in realm-name extraction.
+#   janl 10/10/97 -- Referer: header supression supressed User: header instead
+#		  -- Added fixup handling, writes .redirs and .referers
+#		     (no dot in win32)
+#  		  -- Read .w3mirc (w3mir.ini on win32) if present
+#		  -- Stop file removal code from removing these files
+#   janl 16/10/97 -- process_tag was mangling url attributes in tags with more
+#		     than one of them.  Problem found by Robert L. Binkley
+#   janl 04/12/97 -- Fixed problem with authentication, misplaced +
+#		  -- default inter-docuent pause is 0.  I figure it's better
+#		     to keep one httpd occupied in a steady stream than to
+#		     wait for it to die before we talk to it again.
+#   janl 13/12/97 -- The arguments to index.html in the form of index.html/foo
+#		     handling code was incomplete.  To make it complete would
+#		     have been hard, so it was removed.
+#		  -- If a URL changes from file to directory or vice versa
+#		     this is now handled.
+#   janl 11/01/98 -- PDF files with no URLs does not cause warnings now.
+#		  -- Close REFERERS and REDIRECTS before calling w3mfix
+#   janl 22/01/98 -- Proxy authentication as outlined by Christian Geuer
+#   janl 04/02/98 -- Version 1pre1
+#   janl 18/02/98 -- Fixed wild_re after tip by Prentiss Riddle.
+#		  -- Version 1pre2
+#   janl 20/02/98 -- w3http updated to handle complex content-types.
+#		  -- Fix wild_re more, bug noted by James Dumser
+#		  -- 1.0pre3
+#   janl 18/03/98 -- Version 1.0 is released
+#   janl 09/04/98 -- Added feature so user can disable newline conversion.
+#   janl 20/04/98 -- Only convert newlines in HTML files. -> 1.0.2
+#   janl 09/05/98 -- More carefull clean_disk code.
+#		  -- Check if the redirected URL was a root url, if so
+#		     issue a warning and exit.
+#   janl 12/05/98 -- use ->unix_path instead of ->as_string to derive local
+#		     filename.
+#   janl 25/05/98 -- -B didn't work too well.
+#   janl 09/07/98 -- Redirect to fragment broke us, less broken now -> 1.0.4
+#   janl 24/09/98 -- Better errormessages on errors -> 1.0.5
+#   janl 21/11/98 -- Fix errormessages better.
+#   janl 05/01/99 -- Drop 'Referer: (commandline)'
+#   janl 13/04/99 -- Add initial referer to root urls in batch mode.
+#
+# Variable name discipline:
+# - remote, umodified URL.  Variables prefixed 'rum_'
+# - local, filesystem.  Variables prefixed 'lf_'.
+# Use these prefixes so we know what we're working with at all times.
+# Also, URL objects are postfixed _o
+#
+# The apply rules and scope rules work this way:
+# - First apply the user rules to the remote url.
+# - Check if document is within scope after this.
+# - Then apply w3mir's rules to the result.  This results is the local,
+#   filesystem, name.
+#
+# We use features introduced in 5.002.
+require 5.002;
+
+# win32 and $nulldevice need to be globals, other modules use them.
+use vars qw($win32 $nulldevice);
+
+# To figure out what kind of system this is
+BEGIN {
+  use Config;
+  $win32 = ( $Config{'osname'} eq 'MSWin32' ); 
+}
+# More ways to die:
+use Carp;
+# Http module:
+use w3http;
+# html url extraction and manupulation:
+use htmlop;
+# Extract urls from adobe acrobat pdf files:
+use w3pdfuri;
+# Date computer:
+use HTTP::Date;
+# URLs:
+use URI::URL;
+# For flush method
+use FileHandle;
+
+# Full discipline:
+use strict;
+
+# Set params in the http package, HTTP protocol version:
+$w3http::version="1.0";
+
+# The defaults should be for a robotic http agent on good behaviour.
+my $debug=0;			# Debug level
+my $verbose=0;			# Verbosity level, -1 = quiet, 0 = normal, 1...
+my $pause=0;			# Pause between http requests
+my $retryPause=600;		# Pause between retries. 10 minutes.
+my $retry=3;			# Max 3 stabs pr. url.
+my $r=0;			# Recurse? no recursion = absolutify links
+my $remove=0;			# Remove files that are not there?
+my $s=0;			# 0: save on disk 1: stdout 2: just forget 'em
+my $useauth=0;			# Use authorization
+my %authdata;			# Authorization data
+my $check_robottxt = 1;		# Check robots.txt
+my $do_referer = 1;		# Send referers header
+my $do_user = 1;		# Send user header
+my $cache_header = '';		# The cache-control/pragma: no-cache header
+my $using_proxy = 0;		# Using proxy server or not?
+my $batch=0;			# Batch get URLs?
+my $read_urls=0;		# Get urls from STDIN?
+my $abs=0;			# Absolutify URLs?
+my $immediate_redir=0;		# Immediately follow a redirect?
+my @root_urls;			# This is where we start, the root documents
+my @root_dirs;			# The corresponding directories.  for remove
+my $chdirto='';			# Place to chdir to after reading config file
+my %nodelete=();		# Files that should not be deleted
+my $numarg=0;			# Number of arguments accepted.
+
+# Fixup related things
+my $fixrc='';			# Name of w3mfix config file
+my $fixup=1;			# Do things needed to run fixup
+my $runfix=0;			# Run w3mfix for user?
+my $fixopen=0;			# Fixup files open?
+
+my $indexname='index.html';
+
+my $VERSION;
+$VERSION='1.0.8';
+$w3http::agent = my $w3mir_agent = "w3mir/$VERSION-1999-05-28";
+my $iref='';			# Initial referer.  Must evaluate to false
+
+# Derived settings	
+my $mine_urls=0;		# Mine URLs from documents?
+my $process_urls=0;		# Perform (URL) processing of documents?
+
+# Queue of urls to get.
+my @rum_queue = ();
+my @urls = ();
+# URL status map.
+my %rum_urlstat = ();
+# Status codes:
+my $QUEUED   = 0;		# Queued but not gotten yet.
+my $TERROR   = 100;		# Transient error, retry later
+my $HLERR    = 101;		# Permanent error, give up
+my $GOTIT    = 200;		# Gotten. Note similarity to http result code
+my $NOTMOD   = 304;		# Not modified.
+# Negative codes for nonexistent files, easier to check.
+my $NEVERMIND= -1;		# Don't want it
+my $REDIR    = -302;		# Does not exist, redirected
+my $ENOTFND  = -404;		# Does not exist.
+my $OTHERERR = -600;		# Some other error happened
+my $FROBOTS  = -601;		# Forbidden by robots.txt rule
+
+# Directory/files survey:
+my %lf_file;		# What files are present in FS?  Disposition?  One of:
+my $FILEDEL=0;		# Delete file
+my $FILEHERE=1;		# File present in filesystem only
+my $FILETHERE=2;	# File present on server too.
+my %lf_dir;		# Number of files/dirs in dir.  If 0 dir is 
+			# eligible for deletion.
+
+my %fiddled=();		# If a file becomes a directory or a directory
+                        # becomes a file it is considered fiddled and
+                        # w3mir will not fiddle with it again in this
+                        # run.
+
+# Bitbucket device, very OS dependent.
+$nulldevice='/dev/null';
+$nulldevice='nul:' if ($win32);
+
+# What to get, and not.
+# Text of user supplied fetch/ignore rules
+my $rule_text=" # User defined fetch/ignore rules\n";
+# Code ref to the rule procedure
+my $rule_code;
+
+# Code to prefix and postfix the generated code.  Prefix should make
+# $_ contain the url to match.  Postfix should return 1, the default
+# is to get the url/file.
+my $rule_prefix='$rule_code = sub { local($_) = shift;'."\n";
+my $rule_postfix=" return 1;\n}";
+
+# Scope tests generated by URL/Also directives in cfg. The scope code
+# is just like the rule code, but used for program generated
+# fetch/ignore rules related to multiscope retrival.
+my $scope_fetch=" # Automatic fetch rules for multiscope retrival\n";
+my $scope_ignore=" # Automatic ignore rules for multiscope retrival\n";
+my $scope_code;
+
+my $scope_prefix='$scope_code = sub { local($_) = shift;'."\n";
+my $scope_postfix=" return 0;\n}";
+
+# Function to apply to urls, se rule comments.
+my $user_apply_code;	# User specified apply code
+my $apply_code;		# w3mirs apply code
+my $apply_prefix='$apply_code = sub { local($_) = @_;'."\n";
+my $apply_lc=' $_ = lc $_; ';
+my $apply_postfix='  return $_;'."\n}";
+my @user_apply;		# List of users apply rules.
+my @internal_apply;	# List of w3mirs apply rules.
+
+my $infoloss=0;		# 1 if any URL translations (which cause
+                        # information loss) are in effect.  If this is
+                        # true we use the SAVEURL operation.
+my $list;		# List url on STDOUT?
+my $edit;		# Edit doc? Remove <!--NOMIRROR>...<!--/NOMIRROR-->
+my $header;		# Text to insert in header
+my $lc=0;		# Convert urls/filenames to lowercase?
+my $fetch=0;		# What to fetch: -1: Some, 0: not modified 1: all
+my $convertnl=1;	# Convert newlines?
+
+# Non text/html formats we can extract urls from.  Function must take one
+# argument: the filename.
+my %knownformats = ( 'application/pdf', \&w3pdfuri::list,
+		     'application/x-pdf', \&w3pdfuri::list,
+		   );
+
+# Known 'magic numbers' of the known formats.  The value is used as
+# key in %knownformats.  the key part is a exact match for the
+# following <string> beginning at the first byte of the file.
+# This should probably be made more flexible, but not until we need it.
+
+my %knownmagic = ( '%PDF-', 'application/pdf' );
+
+my $iinline='';	# inline RE code to make RE caseinsensitive 
+my $ipost='';	# RE postfix to make it caseinsensitive
+
+usage() unless parse_args(@ARGV);
+
+{
+  my $w3mirc='.w3mirc';
+  
+  $w3mirc='w3mir.ini' if $win32;
+  
+  if (-f $w3mirc) {
+    parse_cfg_file($w3mirc);
+    $nodelete{$w3mirc}=1;
+  }
+}
+
+# Check arguments and options
+if ($#root_urls>=0) {
+  # OK
+} else {
+  print "URLs: $#rum_queue\n";
+  usage("No URLs given");
+}
+
+# Are we converting newlines today?
+$w3http::convert=0 unless $convertnl;
+
+if ($chdirto) {
+  &mkdir($chdirto.'/this-is-not-created-odd-or-what');
+  chdir($chdirto) || 
+    die "w3mir: Can't change working directory to '$chdirto': $!\n";
+}
+
+$SIG{'INT'}=sub { print STDERR "\nCaught SIGINT!\n"; exit 1; };
+$SIG{'QUIT'}=sub { print STDERR "\nCaught SIGQUIT!\n"; exit 1; };
+$SIG{'HUP'}=sub { print STDERR "\nCaught SIGHUP!\n"; exit 1; };
+
+&open_fixup if $fixup;
+
+# Derive how much document processing we should do.
+$mine_urls=( $r || $list );
+$process_urls=(!$batch && !$edit && !$header);
+# $abs can be set explicitly with -abs, and implicitly if not recursing
+$abs = 1 unless $r;
+print "Absolute references\n" if $abs && $debug;
+
+# Cache_controll specified but proxy not in use?
+die "w3mir: If you want to control a cache, use a proxy server!\n"
+  if ($cache_header && !$using_proxy);
+
+# Compile the second order code
+
+# - The rum scope tests
+my $full_rules=$scope_prefix.$scope_fetch.$scope_ignore.$scope_postfix;
+# warn "Scope rules:\n-------------\n$full_rules\n---------------\n";
+eval $full_rules;
+
+die "w3mir: Program generated rules did not compile.\nPlease report to w3mir-core\@usit.uio.no.  The code is:\n----\n".
+  $full_rules."\n----\n"
+  if !defined($scope_code);
+
+$full_rules=$rule_prefix.$rule_text.$rule_postfix;
+# warn "Fetch rules:\n-------------\n$full_rules\n---------------\n";
+eval $full_rules;
+
+# - The user specified rum tests
+die "w3mir: Ignore/Fetch rules did not compile.\nPlease report to w3mir-core\@usit.uio.no.  The code is:\n----\n".
+  $full_rules."\n----\n"
+  if !defined($rule_code);
+
+# - The user specified apply rules
+
+my $full_apply=$apply_prefix.($lc?$apply_lc:'').
+  join($ipost.";\n",@user_apply).(($#user_apply>=0)?$ipost:"").";\n".
+  $apply_postfix;
+eval $full_apply;
+
+die "w3mir: User apply rules did not compile.\nPlease report to w3mir-core\@usit.uio.no. The code is:
+----
+".$full_apply."
+----\n" if !defined($apply_code);
+
+$user_apply_code=$apply_code;
+
+# - The w3mir generated apply rules
+
+$full_apply=$apply_prefix.($lc?$apply_lc:'').
+  join($ipost.";\n",@internal_apply).(($#internal_apply>=0)?$ipost:"").";\n".
+  $apply_postfix;
+eval $full_apply;
+
+die "Internal apply rules did not compile.  The code is:
+----
+".$full_apply."
+----\n" if !defined($apply_code);
+
+# - Information loss via -lc?  There are other sources as well.
+$infoloss=1 if $lc;
+
+warn "Infoloss is $infoloss\n" if $debug;
+
+# More setup:
+
+$w3http::debug=$debug;
+
+$w3http::verbose=$verbose;
+
+my %rum_referers=();	# Array of referers, key: rum_url
+my $Robot_Blob;		# WWW::RobotsRules object, decides if rum_url is 
+			# forbidden to access for us.
+my $rum_url_o;		# rum url, mostly the current, the one we're getting
+my %gotrobots;		# Did I get robots.txt from site? key: url->netloc
+my($authuser,$authpass);# Username and password for authentication with server
+my @rum_newurls;	# List of rum_urls in document
+
+if ($check_robottxt) {
+  # Eval is only way to defer loading of module until we know it's needed?
+  eval 'use WWW::RobotRules;';
+
+  die "Could not load WWW::RobotRules, try -drr switch\n"
+    unless defined(&WWW::RobotRules::parse);
+
+  $Robot_Blob = new WWW::RobotRules $w3mir_agent;
+}
+
+# We have several main-modes of operation.  Here we select one
+if ($r) {
+
+  die "w3mir: No URLs?  Try 'w3mir -h' for help.\n"
+    if $#root_urls==-1;
+
+  warn "Recursive retrival comencing\n" if $debug;
+
+  die "w3mir: Sorry, you cannot combine -r/recurse with -I/read_urls\n"
+    if $read_urls;
+
+  # Recursive
+  my $url;
+  foreach $url (@root_urls) {
+    warn "Root url dequeued: $url\n" if $debug;
+    if (want_this($url)) {
+      queue($url);
+      &add_referer($url,$iref);
+    } else {
+      die "w3mir: Inconsistent configuration: Specified $url is not inside retrival scope\n";
+    }
+  }
+  mirror();
+
+} else {
+  if ($batch) {
+    warn "Batch retrival commencing\n" if $debug;
+    # Batch get
+    if ($read_urls) { 
+      # Get URLs from <STDIN>
+      while (<STDIN>) {
+	chomp;
+	&add_referer($_,$iref);
+	batch_get($_);
+      }
+    } else {
+      # Get URLs from commandline
+      my $url;
+      foreach $url (@root_urls) {
+	&add_referer($url,$iref);
+      }
+      foreach $url (@root_urls) {
+	batch_get($url);
+      }
+    }
+  } else {
+    warn "Single url retrival commencing\n" if $debug;
+
+    # A single URL, with all processing on
+    die "w3mir: You specified several URLs and not -B/batch\n"
+      if $#root_urls>0;
+    queue($root_urls[0]);
+    &add_referer($root_urls[0],$iref);
+    mirror();
+  }
+}
+
+&close_fixup if $fixup;
+
+# This should clean up files:
+&clean_disk if $remove;
+
+warn "w3mir: That's all (".$w3http::xfbytes.'+',$w3http::headbytes.
+  " bytes of it).\n" unless $verbose<0;
+
+if ($runfix) {
+  eval 'use Config;';
+  warn "Running w3mfix\n";
+  if ($win32) {
+    system($Config{'perlpath'}." w3mfix $fixrc");
+  } else {
+    system("w3mfix $fixrc");
+  }
+}
+
+exit 0;
+
+sub get_document {
+  # Get one document by HTTP ($1/rum_url_o).  Save in given filename ($2).
+  # Possebly returning references found in the document.  Caller must
+  # set up referer array, check wantedness and everything else.  We
+  # handle authentication here though.
+
+  my($rum_url_o)=shift;
+  my($lf_url)=shift;
+  croak("\$rum_url_o is empty") if !defined($rum_url_o) || !$rum_url_o;
+  croak("$lf_url is empty") if !defined($lf_url) || !$lf_url;
+
+  # Make sure it's an object
+  $rum_url_o = url $rum_url_o 
+    unless ref $rum_url_o;
+  
+  # Derive a filename from the url, the filename contains no URL-quoting
+  my($lf_name) = (url "file:$lf_url")->unix_path;
+
+  # Make all intermediate directories
+  &mkdir($lf_name) if $s==0;
+
+  my($rum_as_string) = $rum_url_o->as_string;
+
+  print STDERR "GET_DOCUMENT: '",$rum_as_string,"' -> '",$lf_name,"'\n"
+    if $debug;
+
+  my $hostport;
+  my $www_auth='';	# Value of that http reply header
+  my $page_ref;
+  my @rum_newurls;	# List of URLs extracted
+  my $url_extractor;
+  my $do_query;		# Do query or not?
+
+  if (defined($rum_urlstat{$rum_as_string}) && 
+      $rum_urlstat{$rum_as_string}>0) {
+    warn "w3mir: Internal error, ".$rum_as_string.
+      " queued several times\n";
+    next;
+  }
+
+  # Goto here if we want to retry b/c of authentication
+  try_again:
+
+  # Start building the extra http::query arguments again
+  my @EXTRASTUFF=();
+
+  # We'll start by assuming that we're doing the query.
+  $do_query=1;
+    
+  # If we're not checking the timestamp, or the file does not exist
+  # then we get the file unconditionally.  Otherwise we only want it
+  # if it's updated.
+
+  if ($fetch==1) {
+    # Nothing do do?
+  } else {
+    if (-f $lf_name) {
+      if ($fetch==-1) {
+	print STDERR "w3mir: ".($infoloss?$rum_as_string:$lf_name).
+	  ", already have it" if $verbose>=0;
+	if (!$mine_urls) {
+	  # If -fs and the file exists and we don't need to mine URLs
+	  # we're finished!
+	  warn "Already have it, no mining, returning!\n" if $debug;
+	  print STDERR "\n" if $verbose>=0;
+	  return;
+	}
+	$w3http::result=1304;	# Pretend it was 'not modified'
+	$do_query=0;
+      } else {
+	push(@EXTRASTUFF,$w3http::IFMODF,$lf_name);
+      }
+    }
+  }
+
+  if ($do_query) {
+
+    # Does the server want authorization for this file?  $www_auth is
+    # only set if authentication was requested the first time around.
+
+    # For testing:
+    # $www_auth='Basic realm="foo"';
+
+    if ($www_auth) {
+      my($authdata,$method,$realm);
+
+      ($method,$realm)= $www_auth =~ m/^(\S+)\s+realm=\"([^\"]+)\"/i;
+      $method=lc $method;
+      $realm=lc $realm;
+      die "w3mir: '$method' authentication needed, don't know that.\n"
+	if ($method ne 'basic');
+      
+      $hostport = $rum_url_o->netloc;
+      $authdata=$authdata{$hostport}{$realm} || $authdata{$hostport}{'*'} ||
+	$authdata{'*'}{$realm} || $authdata{'*'}{'*'};
+
+      if ($authdata) {
+	push(@EXTRASTUFF,$w3http::AUTHORIZ,$authdata);
+      } else {
+	print STDERR "w3mir: No authorization data for $hostport/$realm\n";
+	$rum_urlstat{$rum_as_string}=$NEVERMIND;
+	next;
+      }
+    }
+    
+    push(@EXTRASTUFF,$w3http::FREEHEAD,$cache_header)
+      if ($cache_header);
+
+    # Insert referer header data if at all
+    push(@EXTRASTUFF,$w3http::REFERER,$rum_referers{$rum_as_string}[0])
+      if ($do_referer && exists($rum_referers{$rum_as_string}));
+
+    push(@EXTRASTUFF,$w3http::NOUSER)
+      unless ($do_user);
+
+    # YES, $lf_url is right, w3http::query handles this like an url so
+    # the quoting must all be in place.
+    my $binfile=$lf_url;
+    $binfile='-' if $s==1;
+    $binfile=$nulldevice if $s==2;
+
+    if ($pause) {
+      print STDERR "w3mir: sleeping\n" if $verbose>0;
+      sleep($pause);
+    }
+
+    print STDERR "w3mir: ".($infoloss?$rum_as_string:$lf_name) 
+      unless $verbose<0;
+    print STDERR "\nFile: $lf_name\n" if $debug;
+
+    &w3http::query($w3http::GETURL,$rum_as_string,
+		   $w3http::SAVEBIN,$binfile,
+		   @EXTRASTUFF);
+
+    print STDERR "w3http::result: '",$w3http::result,
+    "' doc size: ", length($w3http::document),
+    " doc type: -",$w3http::headval{'CONTENT-TYPE'},
+    "- plaintexthtml: ",$w3http::plaintexthtml,"\n"
+      if $debug;
+
+    print "Result: ",$w3http::result," Recurse: $r, html: ",
+    $w3http::plaintexthtml,"\n" 
+      if $debug;
+
+  } # if $do_query
+    
+  if ($w3http::result==200) { # 200 OK 
+    $rum_urlstat{$rum_as_string}=$GOTIT;
+
+    if ($mine_urls || $process_urls) {
+
+      if ($w3http::plaintexthtml) {
+	# Only do URL manipulations if this is a html document with no
+	# special content-encoding.  We do not handle encodings, yet.
+
+	my $page;
+
+	print STDERR ($process_urls)?", processing":", url mining"
+	  if $verbose>0;
+	  
+	print STDERR "\nurl:'$lf_url'\n" 
+	  if $debug;
+
+	print "\nMining URLs: $mine_urls, Process: $process_urls\n" 
+	  if $debug;
+	    
+	($page,@rum_newurls) = 
+	  &htmlop::process($w3http::document,
+			   # Only get a new document if wanted
+			   $process_urls?():($htmlop::NODOC),
+			   $htmlop::CANON,
+			   $htmlop::ABS,$rum_url_o,
+			   # Only list urls if wanted
+			   $mine_urls?($htmlop::LIST):(),
+
+			   # If user wants absolute URLs do not
+			   # relativize them
+
+			   $abs?
+			   ():
+			   (
+			    $htmlop::TAGCALLBACK,\&process_tag,$lf_url,
+			    )
+			  );
+
+#	print "URL: ",join("\nURL: ",@rum_newurls),"\n";
+
+	if ($process_urls) {
+	  $page_ref=\$page;
+	  $w3http::document='';
+	} else {
+	  $page_ref=\$w3http::document;
+	}
+
+      } elsif ($s == 0 &&
+	       ($url_extractor = 
+		$knownformats{$w3http::headval{'CONTENT-TYPE'}})) {
+
+	# The knownformats extractors only work on disk files so write
+	# doc to disk if not there already (non-html text will not be)
+	write_page($lf_name,$w3http::document,1);
+
+	# Now we try our hand at fetching URIs from non-html files.
+	print STDERR ", mining URLs" if $verbose>=1;
+	@rum_newurls = &$url_extractor($lf_name);
+	# warn "URLs from PDF: ",join(', ',@rum_newurls),"\n";
+      }
+      
+      
+    } # if ($mine_urls || $process_urls)
+
+#    print "page_ref defined: ",defined($page_ref),"\n";
+#    print "plaintext: ",$w3http::plaintext,"\n";
+
+    $page_ref=\$w3http::document 
+      if !defined($page_ref) && $w3http::plaintexthtml;
+
+    if ($w3http::plaintexthtml) {
+      # ark: this is where I want to do my changes to the page strip
+      # out the <!--NOMIRROR-->...<!--/NOMIRROR--> Stuff.
+      $$page_ref=~ s/<(!--)?\s*NO\s*MIRROR\s*(--)?>[^\000]*?<(!--)?\s*\/NO\s*MIRROR\s*(--)?>//g 
+	if $edit;
+	  
+      if ($header) {
+	# ark: insert a header string at the start of the page
+	my $mirrorstr=$header;
+	$mirrorstr =~ s/\$url/$rum_as_string/g;
+	insert_at_start( $mirrorstr, $page_ref );
+      }
+    }
+
+    write_page($lf_name,$page_ref,0);
+
+    # print "New urls: ",join("\n",@rum_newurls),"\n";
+
+    return @rum_newurls;
+  }
+
+  if ($w3http::result==304 || #  304 Not modified
+      $w3http::result==1304) { # 1304 Have it
+
+    { 
+      # last = out of nesting
+
+      my $rum_urlstat;
+      my $rum_newurls;
+      
+      @rum_newurls=();
+      
+      print STDERR ", not modified"
+	if $verbose>=0 && $w3http::result==304;
+      
+      $rum_urlstat{$rum_as_string}=$NOTMOD;
+
+      last unless $mine_urls;
+      
+      $rum_newurls=get_references($lf_name);
+
+      # print "New urls: ",ref($rum_newurls),"\n";
+      
+      if (!ref($rum_newurls)) {
+	last;
+      } elsif (ref($rum_newurls) eq 'SCALAR') {
+	$page_ref=$rum_newurls;
+      } elsif (ref($rum_newurls) eq 'ARRAY') {
+	@rum_newurls=@$rum_newurls;
+	last;
+      } else {
+	die "\nw3mir: internal error: Unknown return type from get_references\n";
+      }
+      
+      # Check if it's a html file.  I know this tag is in all html
+      # files, because I put it there as I pull them in.
+      last unless $$page_ref =~ /<HTML/i;
+      
+      warn "$lf_name is a html file\n" if $debug;
+      
+      # It's a html document
+      print STDERR ", mining URLs" if $verbose>=1;
+      
+      # This will give us a list of absolute urls
+      (undef,@rum_newurls) = 
+	&htmlop::process($$page_ref,$htmlop::NODOC,
+			 $htmlop::ABS,$rum_as_string,
+			 $htmlop::USESAVED,'W3MIR',
+			 $htmlop::LIST);
+    }
+
+    print STDERR "\n" if $verbose>=0;
+    return @rum_newurls;
+  }
+	
+  if ($w3http::result==302 || $w3http::result==301) { # Redirect
+    # Cern and NCSA httpd sends 302 'redirect' if a ending / is
+    # forgotten on a url.  More recent httpds send 301 'permanent
+    # redirect' in this case.  Here we check if the difference in URLs
+    # is just a / and if so push the url again with the / added.  This
+    # code only works if the http server has the right idea about its
+    # own name.
+    #
+    # 18/3/97: Added code to queue redirected-to-URLs that are within
+    # the scope of the retrival.
+    my $new_rum_url;
+
+    $rum_urlstat{$rum_as_string}=$REDIR;
+
+    # Absolutify the new url, it might be relative to the requested
+    # document.  That's a ugly wart on some servers/admins.
+    $new_rum_url=url $w3http::headval{'location'};
+    $new_rum_url=$new_rum_url->abs($rum_url_o);
+
+    print REDIRS $rum_as_string,' -> ',$new_rum_url->as_string,"\n"
+      if $fixup;
+
+    if ($immediate_redir) {
+      print STDERR " =>> ",$new_rum_url->as_string,", getting that instead\n";
+      return get_document($new_rum_url,$lf_url);
+    }
+
+    # Some redirect to a fragment of another doc...
+    $new_rum_url->frag(undef);
+    $new_rum_url=$new_rum_url->as_string;
+
+    if ($rum_as_string.'/' eq $new_rum_url) {
+      if (grep { $rum_as_string eq $_; } @root_urls) {
+	print STDERR "\nw3mir: missing / in a start URL detected.  Please fix commandline/config file.\n";
+	exit(1);
+      }
+      print STDERR ", missing /\n";
+      queue($new_rum_url);
+      # Initialize referer to something meaningful
+      $rum_referers{$new_rum_url}=$rum_referers{$rum_as_string};
+    } else {
+      print STDERR " =>> $new_rum_url";
+      if (want_this($new_rum_url)) {
+	print STDERR ", getting that\n";
+	queue($new_rum_url);
+	$rum_referers{$new_rum_url}=$rum_referers{$rum_as_string};
+      } else {
+	print STDERR ", don't want it\n";
+      }
+    }
+    return ();
+  }
+
+  if ($w3http::result==403 || # Forbidden
+      $w3http::result==404 || # Not found
+      $w3http::result==406 || # Not Acceptable, hmm, belongs here?
+      $w3http::result==410) { # Gone - no forwarding address known
+
+    $rum_urlstat{$rum_as_string}=$ENOTFND;
+    &handleerror;
+    print STDERR "Was refered from: ",
+      join(',',@{$rum_referers{$rum_as_string}}),
+      "\n" if defined(@{$rum_referers{$rum_as_string}});
+    return ();
+  }
+
+  if ($w3http::result==407) {
+    # Proxy authentication requested
+    die "Proxy server requests authentication but failed to return the\n".
+      "REQUIRED Proxy-Authenticate header for this condition\n"
+	unless exists($w3http::headval{'proxy-authenticate'});
+
+    die "Proxy authentication is required for ".$w3http::headval{'proxy-authenticate'}."\n";
+  }
+
+  if ($w3http::result==401) {
+    # A www-authenticate reply header should acompany a 401 message.
+    if (!exists($w3http::headval{'www-authenticate'})) {
+      warn "w3mir: Server indicated authentication failure but gave no www-authenticate reply\n";
+      $rum_urlstat{$rum_as_string}=$NEVERMIND;
+    } else {
+      # Unauthorized
+      if ($www_auth) {
+	# Failed when authorization data was supplied.
+	$rum_urlstat{$rum_as_string}=$NEVERMIND;
+	print STDERR ", authorization failed data needed for ",
+	$w3http::headval{'www-authenticate'},"\n"
+	  if ($verbose>=0);
+      } else {
+	if ($useauth) {
+	  # First time failure, send back and retry at once with some known
+	  # user/passwd.
+	  $www_auth=$w3http::headval{'www-authenticate'};
+	  print STDERR ", retrying with authorization\n" unless $verbose<0;
+	  goto try_again;
+	} else {
+	  print ", authorization needed: ",
+	    $w3http::headval{'www-authenticate'},"\n";
+	  $rum_urlstat{$rum_as_string}=$NEVERMIND;
+	}
+      }
+    }
+    return ();
+  }
+   
+  # Something else.  
+  &handleerror;
+}
+
+
+sub robot_check {
+  # Check if URL is allowed by robots.txt, if we respect it at all
+  # that is.  Return 1 it allowed, 0 otherwise.
+
+  my($rum_url_o)=shift;
+  my $hostport;
+
+  if ($check_robottxt) {
+    
+    $hostport = $rum_url_o->netloc;
+    if (!exists($gotrobots{$hostport})) {
+      # Get robots.txt from the server
+      $gotrobots{$hostport}=1;
+      my $robourl="http://$hostport/robots.txt";
+      print STDERR "w3mir: $robourl" if ($verbose>=0);
+      &w3http::query($w3http::GETURL,$robourl);
+      $w3http::document='' if ($w3http::result != 200);
+      print STDERR ", processing" if $verbose>=1;
+      print STDERR "\n" if ($verbose>=0);
+      $Robot_Blob->parse($robourl,$w3http::document);
+    }
+
+    if (!$Robot_Blob->allowed($rum_url_o->as_string)) {
+      # It is forbidden
+      $rum_urlstat{$rum_url_o->as_string}=$FROBOTS;
+      warn "w3mir: ",$rum_url_o->as_string,": forbidden by robots.txt\n";
+      return 0;
+    }
+  }
+  return 1;
+}
+
+
+
+sub batch_get {
+  # Batch get _one_ document.
+  my $rum_url=shift;
+  my $lf_url;
+
+  $rum_url_o = url $rum_url;
+
+  return unless robot_check($rum_url_o);
+
+  ($lf_url=$rum_url) =~ s~.*/~~;
+  if (!defined($lf_url) || $lf_url eq '') {
+    ($lf_url=$rum_url) =~ s~/$~~;
+    $lf_url =~ s~.*/~~;
+    $lf_url .= "-$indexname";
+  }
+
+  warn "Batch get: $rum_url -> $lf_url\n" if $debug;
+
+  $immediate_redir=1; # Do follow redirects immediately
+
+  get_document($rum_url,$lf_url);
+}
+
+
+
+sub mirror {
+  # Mirror (or get) the requested url(s). Possibly recursively.
+  # Working from whatever cwd is at invocation we'll retrieve all
+  # files under it in the file hierarchy.
+  
+  my $rum_url;	# URL of the document we're getting now, defined at main level
+  my $lf_url;	# rum_url after apply - and 
+  my $new_lf_url;
+  my @new_rum_urls;
+  my $rum_ref;
+  
+  while (defined($rum_url = pop(@rum_queue))) {
+
+    warn "mirror: Poped $rum_url from queue\n" if $debug;
+
+    # Unwanted URLs should not be queued
+    die "Found url $rum_url that I don't want in queue!\n" 
+      unless defined($lf_url=apply($rum_url));
+
+    $rum_url_o = url $rum_url;
+
+    next unless robot_check($rum_url_o);
+
+    # Figure out the filename for our local filesystem.
+    $lf_url.=$indexname if $lf_url =~ m~/$~ || $lf_url eq '';
+
+    @new_rum_urls = get_document($rum_url_o,$lf_url);
+
+    print join("\n",@new_rum_urls),"\n" if ($list);
+
+    if ($r) {
+      foreach $rum_ref (@new_rum_urls) {
+	# warn "Recursive url: $rum_ref\n";
+	$new_lf_url=apply($rum_ref);
+	next unless $new_lf_url;
+
+	# warn "Want it\n";
+	$rum_ref =~ s/\#.*$//s;		# Clip off section marks
+	
+	add_referer($rum_ref,$rum_url_o->as_string);
+	queue($rum_ref);
+      }
+    }
+
+    @new_rum_urls=();
+
+    # Is the URL queue empty? Are there outstanding retries?  Refill
+    # the queue from the retry list.
+    if ($#rum_queue<0 && $retry-->0) {
+      foreach $rum_url_o (keys %rum_urlstat) {
+	$rum_url_o = url $rum_url_o;
+	if ($rum_urlstat{$rum_url_o->as_string}==100) {
+	  push(@rum_queue,$rum_url_o->as_string);
+	  $rum_urlstat{$rum_url_o->as_string}=0;
+	}
+      }
+      if ($#rum_queue>=0) {
+	warn "w3mir: Sleeping before retrying.  $retry more times left\n"
+	  if $verbose>=0;
+	sleep($retryPause);
+      }
+    }
+
+  }
+}
+
+
+sub get_references {
+  # Get references from a non-html-on-disk file.  Return references if
+  # we know how to find them.  Return reference do the complete page
+  # if it's html.  Return single numerical 0 if unknown format.
+
+  my($lf_url)=shift;
+  my($urlextractor)=shift;
+
+  my $read; # Buffer of stuff read from file to check filetype
+  my $magic;
+  my $url_extractor;
+  my $rum_ref;
+  my $page;
+
+  warn "w3mir: Looking at local $lf_url\n" if $debug;
+	
+  # Open file and read the first 10kilobytes for file-type-test
+  # purposes.
+  if (!open(TMPF,$lf_url)) {
+    warn "Unable to open $lf_url for reading: $!\n";
+    last;
+  }
+	
+  $page=' 'x10240;
+  $read=sysread(TMPF,$page,length($page),0);
+  close(TMPF);
+	
+  die "Error reading $lf_url: $!\n" if (!defined($read));
+
+  if (!defined($url_extractor)) {
+    $url_extractor=0;
+
+    # Check file against list of magic numbers.
+    foreach $magic (keys %knownmagic) {
+      if (substr($page,0,length($magic)) eq $magic) {
+	$url_extractor = $knownformats{$knownmagic{$magic}};
+	last;
+      }
+    }
+  }
+
+  # Found a extraction method, apply.
+  if ($url_extractor) {
+    print STDERR ", mining URLs" if $verbose>=1;
+    return [&$url_extractor($lf_url)];
+  }
+
+  if ($page =~ /<HTML/i) {
+    open(TMPF,$lf_url) || 
+      die "Could not open $lf_url for reading: $!\n";
+    # read the whole file.
+    local($/)=undef;
+    $page = <TMPF>;
+    close(TMPF);
+    return \$page;
+  }
+
+  return 0;
+}
+
+
+sub open_fixup {
+  # Open the referers and redirects files
+
+  my $reffile='.referers';
+  my $redirfile='.redirs';
+
+  if ($win32) {
+    $reffile="referers";
+    $redirfile="redirs";
+  }
+
+  $nodelete{$reffile} = $nodelete{$redirfile} = 1;
+
+  open(REDIRS,"> $redirfile") || 
+    die "Could not open $redirfile for writing: $!\n";
+
+  autoflush REDIRS 1;
+
+  open(REFERERS,"> $reffile") ||
+    die "Could not open $reffile for writing: $!\n";
+
+  $fixopen=1;
+  eval 'END { close_fixup; 0; }';
+}
+
+
+sub close_fixup {
+  # Close the fixup data files.  In the case of the referer file also
+  # write the entire content
+
+  return unless $fixopen;
+
+  my $referer;
+
+  foreach $referer (keys %rum_referers) {
+    print REFERERS $referer," <- ",join(' ',@{$rum_referers{$referer}}),"\n";
+  }
+
+  close(REFERERS) || warn "Error closing referers file: $!\n";
+  close(REDIRS) || warn "Error closing redirects file: $!\n";
+  $fixopen=0;
+}
+
+
+sub clean_disk {
+  # This procedure removes files that are not present on the server(s)
+  # anymore.
+  
+  # - To avoid removing files that were not fetched due to network
+  #   problems we only do blanket removal IFF all documents were
+  #   fetched w/o problems, eventually.  
+  # - In any case we can remove files the server said were not found
+
+  # The strategy has three main parts:
+  # 1. Find all files we have
+  # 2. Find what files we ought to have
+  # 3. Remove the difference
+  
+  my $complete_retrival=1;	# Flag saying IFF all documents were fetched
+  my $urlstat;			# Tmp storage
+  my $rum_url;
+  my $lf_url;
+  my $lf_dir;
+  my $dirs_to_remove;
+
+  # For fileremoval code
+  eval "use File::Find;" unless defined(&find);
+
+  die "w3mir: Could not load File::Find module.  Don't use -R switch.\n"
+    unless defined(&find);
+  
+  # This to shut up -w
+  $lf_dir=$File::Find::dir;
+
+  # ***** 1. Find out what files we have *****
+  #
+  # This does two things:  For each file or directory found:
+  # - Increases entry count for the container directory
+  # - If it's a file: $lf_file{relative_path}=$FILEHERE;
+
+  chop(@root_dirs);
+  print STDERR "Looking in: ",join(", ",@root_dirs),"\n" if $debug;
+
+  find(\&find_files,@root_dirs);
+
+  # ***** 2. Find out what files we ought to have *****
+  #
+  # First we loop over %rum_urlstat to determine what files are not
+  # present on the server(s).
+  foreach $rum_url (keys %rum_urlstat) {
+    # Figure out name of local file from rum_url
+    next unless defined($lf_url=apply($rum_url));
+
+    $lf_url.=$indexname if $lf_url =~ m~/$~ || $lf_url eq '';
+
+    # find prefixes ./, we must too.
+    $lf_url="./".$lf_url unless substr($lf_url,0,1) eq '/';
+
+    # Ignore if file does not exist here.
+    next unless exists($lf_file{$lf_url});
+    
+    # The apply rules can map several remote files to same local
+    # file.  If we decided to keep file already we stay with that.
+    next if $lf_file{$lf_url}==$FILETHERE;
+    
+    $urlstat=$rum_urlstat{$rum_url};
+
+    # Figure out the status code.
+    if ($urlstat==$GOTIT || $urlstat==$NOTMOD) {
+      # Present on server.  Keep.
+      $lf_file{$lf_url}=$FILETHERE;
+      next;
+    } elsif ($urlstat==$ENOTFND || $urlstat==$NEVERMIND ) {
+      # One of: not on server, can't get, don't want, access forbiden:
+      # Schedule for removal.
+      $lf_file{$lf_url}=$FILEDEL if exists($lf_file{$lf_url});
+      next;
+    } elsif ($urlstat==$OTHERERR || $urlstat==$TERROR) {
+      # Some error occured transfering.
+      $complete_retrival=0;	# The retrival was not complete. Delete less
+    } elsif ($urlstat==$QUEUED) {
+      warn "w3mir: Internal inconsistency, $rum_url marked as queued after retrival terminated\n";
+      $complete_retrival=0;	# Fishy. Be conservative about removing
+    } else {
+      $complete_retrival=0;
+      warn "w3mir: Warning: $rum_url is marked as $urlstat.\n".
+	"w3mir: Please report to w3mir-core\@usit.uio.no.\n";
+    }
+  }				# foreach %rum_urlstat
+
+  # ***** 3. Remove the difference *****
+
+  # Loop over all found files:
+  # - Should we have this file?
+  # - If not: Remove file and decrease directory entry count
+  # Loop as long as there are directories with 0 entry count:
+  # - Loop over all directories with 0 entry count:
+  #   - Remove directory
+  #   - Decrease entry count of parent
+  
+  warn "w3mir: Some error occured, conservative file removal\n"
+    if !$complete_retrival && $verbose>=0;
+
+  # Remove all files we don't want removed from list of files present:
+  foreach $lf_url (keys %nodelete) {
+    print STDERR "Not deleting: $lf_url\n" if $verbose>=1;
+    delete $lf_file{$lf_url} || delete $lf_file{'./'.$lf_url};
+  }
+  
+  # Remove files
+  foreach $lf_url (keys %lf_file) {
+    if (($complete_retrival && $lf_file{$lf_url}==$FILEHERE) ||
+	($lf_file{$lf_url} == $FILEDEL)) {
+      if (unlink $lf_url) {
+	($lf_dir)= $lf_url =~ m~^(.+)/~;
+	$lf_dir{$lf_dir}--;
+	$dirs_to_remove=1 if ($lf_dir{$lf_dir}==0);
+	warn "w3mir: removed file $lf_url\n" if $verbose>=0;
+      } else {
+	warn "w3mir: removal of file $lf_url failed: $!\n";
+      }
+    }
+  }
+  
+  # Remove empty directories
+  while ($dirs_to_remove) {
+    $dirs_to_remove=0;
+    foreach $lf_url (keys %lf_dir) {
+      next if $lf_url eq '.';
+      if ($lf_dir{$lf_url}==0) {
+	if (rmdir($lf_url)) {
+	  warn "w3mir: removed directory $lf_dir\n" if $verbose>=0;
+	  delete $lf_dir{$lf_url};
+	  ($lf_dir)= $lf_url =~ m~^(.+)/~;
+	  $lf_dir{$lf_dir}--;
+	  $dirs_to_remove=1 if ($lf_dir{$lf_dir}==0);
+	} else {
+	  warn "w3mir: removal of directory $lf_dir failed: $!\n";
+	}
+      }
+    }
+  }
+}
+
+
+sub find_files {
+  # This is called by the find procedure for every file/dir found.
+  
+  # This builds two hashes:
+  # lf_file{<file>}: 1: file exists
+  # lf_dir{<dir>): Number of files in directory.
+  
+  lstat($_);
+
+  $lf_dir{$File::Find::dir}++;
+
+  if (-f _) {
+    $lf_file{$File::Find::name}=$FILEHERE;
+  } elsif (-d _) {
+    # null
+    # Bug: If an empty directory exists it will not be removed
+  } else {
+    warn "w3mir: File $File::Find::name has unknown type.  Ignoring.\n";
+  }
+  return 0;
+  
+}
+
+
+sub handleerror {
+  # Handle error status of last http connection, will set the rum_urlstat
+  # appropriately and print a error message.
+
+  my $msg;
+  
+  if ($verbose<0) {
+    $msg="w3mir: ".$rum_url_o->as_string.": ";
+  } else {
+    $msg=": ";
+  }
+
+  if ($w3http::result == 98) {
+    # OS/Network error
+    $msg .= "$!";
+    $rum_urlstat{$rum_url_o->as_string}=$OTHERERR;
+  } elsif ($w3http::result == 100) {
+    # Some kind of error connecting or sending request
+    $msg .= $w3http::restext || "Timeout";
+    $rum_urlstat{$rum_url_o->as_string}=$TERROR;
+  } else {
+    # Other HTTP error
+    $rum_urlstat{$rum_url_o->as_string}=$OTHERERR;
+    $msg .= " ".$w3http::result." ".$w3http::restext;
+    $msg .= " =>> ".$w3http::headval{'location'}
+    if (defined($w3http::headval{'location'}));
+  }
+  print STDERR "$msg\n";
+}
+
+
+sub queue {
+  # Queue given url if appropriate and create a status entry for it
+  my($rum_url_o)=url $_[0];
+
+  croak("BUG: undefined \$rum_url_o")
+    if !defined($rum_url_o);
+
+  croak("BUG: undefined \$rum_url_o->as_string")
+    if !defined($rum_url_o->as_string);
+
+  croak("BUG: ".$rum_url_o->as_string." (fragnent) queued")
+    if $rum_url_o->as_string =~ /\#/;
+  
+  return if exists($rum_urlstat{$rum_url_o->as_string});
+  return unless want_this($rum_url_o->as_string);
+
+  warn "QUEUED: ",$rum_url_o->as_string,"\n" if $debug;
+  
+  # Note lack of scope checks.
+  $rum_urlstat{$rum_url_o->as_string}=$QUEUED;
+  push(@rum_queue,$rum_url_o->as_string);
+}
+
+
+sub root_queue {
+  # Queue function for root urls and directories.  One or the other might
+  # be boolean false, in that case, don't queue it.
+
+  my $root_url_o;
+
+  my($root_url)=shift;
+  my($root_dir)=shift;
+
+  die "w3mir: No fragments in start URLs :".$root_url."\n"
+    if $root_url =~ /\#/;
+
+  if ($root_dir) {
+    print "Root dir: $root_dir\n" if $debug;
+    $root_dir="./$root_dir" unless substr($root_dir,0,1) eq '/' or
+      substr($root_dir,0,2) eq './';
+    push(@root_dirs,$root_dir);
+  }
+
+
+  if ($root_url) {
+    $root_url_o=url $root_url;
+
+    # URL canonification, or what we do of it at least.
+    $root_url_o->host($root_url_o->host);
+
+    warn "Root queue: ".$root_url_o->as_string."\n" if $debug;
+
+    push(@root_urls,$root_url_o->as_string); 
+
+    return $root_url_o;
+ }
+
+}
+
+
+sub write_page {
+  # write a retrieved page to wherever it's supposed to be written.
+  # Added difficulty: all files but plaintext files have already been
+  # written to disk in w3http.
+  
+  # $s == 0 save to disk
+  # $s == 1 dump to stdout
+  # $s == 2 forget
+
+  my($lf_name,$page_ref,$silent) = @_;
+  my($verb);
+
+  if ($silent) {
+    $verb=-1;
+  } else {
+    $verb=$verbose;
+  }
+
+#  confess("\n\$page_ref undefined") if !defined($page_ref);
+  
+  if ($w3http::plaintexthtml) {
+    # I have it in memory
+    if ($s==0) {
+      print STDERR ", saving" if $verb>0;
+
+      while (-d $lf_name) {
+	# This will run once, maybe twice, $fiddled will be canged the
+	# first time
+	if (exists($fiddled{$lf_name})) {
+	  warn "Cannot save $lf_name, there is a directory in the way\n";
+	  return;
+	}
+
+	$fiddled{$lf_name}=1;
+
+	rm_rf($lf_name);
+	print STDERR "w3mir: $lf_name" if $verbose>=0;
+      }
+
+      if (!open(PAGE,">$lf_name")) {
+	warn "\nw3mir: can't open $lf_name for writing: $!\n";
+	return;
+      }
+      if (!$convertnl) {
+	binmode PAGE;
+	warn "BINMODE\n" if $debug;
+      }
+      if ($$page_ref ne '') {
+	print PAGE $$page_ref || die "w3mir: Error writing $lf_name: $!\n";
+      }
+      close(PAGE) || die "w3mir: Error closing $lf_name: $!\n";
+      print STDERR ": ", length($$page_ref), " bytes\n"
+	if $verb>=0;
+      setmtime($lf_name,$w3http::headval{'last-modified'})
+	if exists($w3http::headval{'last-modified'});
+    } elsif ($s==1) {
+      print $$page_ref ;
+    } elsif ($s==2) {
+      print STDERR ", got and forgot it.\n" unless $verb<0;
+    }
+  } else {
+    # Already written by http module, just emit a message if wanted
+    if ($s==0) {
+      print STDERR ": ",$w3http::doclen," bytes\n"
+	if $verb>=0;
+      setmtime($lf_name,$w3http::headval{'last-modified'})
+	  if exists($w3http::headval{'last-modified'});
+    } elsif ($s==2) {
+      print STDERR ", got and forgot it.\n" if $verb>=0;
+    }
+  }
+}
+
+
+sub setmtime {
+  # Set mtime of the given file
+  my($file,$time)=@_;
+  my($tm_sec,$tm_min,$tm_hour,$tm_mday,$tm_mon,$tm_year,$tm_wday,$tm_yday,
+     $tm_isdst,$tics);
+
+  $tm_isdst=0;
+  $tm_yday=-1;
+
+  carp("\$time is undefined"),return if !defined($time);
+
+  $tics=str2time($time);
+  utime(time, $tics, $file) || 
+      warn "Could not change mtime of $file: $!\n";
+}
+
+
+sub movefile {
+  # Rename a file.  Note that copy is not a good alternative, since
+  # copying over NFS is something we want to Avoid.
+
+  # Returns 0 if failure and 1 in case of sucess.
+
+  (my $old,my $new) = @_;
+
+  # Remove anything that might have the name already.
+  if (-d $new) {
+    print STDERR "\n" if $verbose>=0;
+    rm_rf($new);
+    $fiddled{$new}=1;
+    print STDERR "w3mir: $new" if $verbose>=0;
+  } elsif (-e $new) {
+    $fiddled{$new}=1;
+    if (unlink($new)) {
+      print STDERR "\nw3mir: removed $new\nw3mir: $new"
+	if $verbose>=0;
+    } else {
+      return 0;
+    }
+
+  }
+
+  if ($new ne '-' && $new ne $nulldevice) {
+    warn "MOVING $old -> $new\n" if $debug;
+    rename($old,$new) || 
+      warn "Could not rename $old to $new: $!\n",return 0;
+  }
+  return 1;
+}
+
+
+sub mkdir {
+  # Make all intermediate directories needed for a file, the file name
+  # is expected to be included in the argument!
+
+  # Reasons for not using File::Path::mkpath:
+  # - I already wrote this.
+  # - I get to be able to produce as good and precise errormessages as
+  #   unix and perl will allow me.  mkpath will not.
+  # - It's easier to find out if it worked or not.
+  
+  my($file) = @_;
+  my(@dirs) = split("/",$file);
+  my $path;
+  my $dir;
+  my $moved=0;
+
+  if (!$dirs[0]) {
+    shift @dirs;
+    $path='';
+  } else {
+    $path = '.';
+  }
+
+  # This removes the last element of the array, it's meant to shave
+  # off the file name leaving only the directory name, as a
+  # convenience, for the caller.
+  pop @dirs;
+  foreach $dir (@dirs) {
+    $path .= "/$dir";
+    stat($path);
+    # only make if it isn't already there
+    next if -d _;
+
+    while (!-d _) {
+      if (exists($fiddled{$path})) {
+	warn "Cannot make directory $path, there is a file in the way.\n";
+	return;
+      }
+
+      $fiddled{$path}=1;
+
+      if (!-e _) {
+	mkdir($path,0777);
+	last;
+      }
+
+      if (unlink($path)) {
+	warn "w3mir: removed file $path\n" if $verbose>=0;
+      } else {
+	warn "Unable to remove $path: $!\n";
+	next;
+      }
+    
+      warn "mkdir $path\n" if $debug;
+      mkdir($path,0777) ||
+	warn "Unable to create directory $path: $!\n";
+
+      stat($path);
+    }
+  }
+}
+
+
+sub add_referer {
+  # Add a referer to the list of referers of a document.  Unless it's
+  # already there.
+  # Don't mail me if you (only) think this is a bit like a toungetwiser:
+
+  # Don't remember referers if BOTH fixup and referer header is disabled.
+  return if $fixup==0 && $do_referer==0;
+
+  my($rum_referee,$rum_referer) = @_ ;
+  my $re_rum_referer;
+
+  if (exists($rum_referers{$rum_referee})) {
+    $re_rum_referer=quotemeta $rum_referer;
+    if (!grep(m/^$re_rum_referer$/,@{$rum_referers{$rum_referee}})) {
+      push(@{$rum_referers{$rum_referee}},$rum_referer);
+      # warn "$rum_referee <- $rum_referer pushed\n";
+    } else {
+      # warn "$rum_referee <- $rum_referer NOT pushed\n";
+    }
+  } else {
+    $rum_referers{$rum_referee}=[$rum_referer];
+    # warn "$rum_referee <- $rum_referer pushed\n";
+  }
+}
+
+
+sub user_apply {
+  # Apply the user apply rules
+
+  return &$user_apply_code(shift);
+
+# Debug version:
+#  my ($foo,$bar);
+#  $foo=shift;
+#  $bar=&$apply_code($foo);
+#  print STDERR "Apply: $foo -> $bar\n";
+#  return $bar;
+}
+
+sub internal_apply {
+  # Apply the w3mir generated apply rules
+
+  return &$apply_code(shift);
+}
+
+
+sub apply {
+  # Apply the user apply rules.  Then if URL is wanted return result of
+  # w3mir apply rules.  Return the undefined value otherwise.
+
+  my $url = user_apply(shift);
+
+  return undef unless want_this($url);
+
+  internal_apply($url);
+}
+
+
+sub want_this {
+  # Find out if we want the url passed.  Just pass it on to the
+  # generated functions.
+  my($rum_url)=shift;
+
+  # What about robot rules?
+
+  # Does scope rule want this?
+  return &$scope_code($rum_url) &&
+    # Does user rule want this too?
+    &$rule_code($rum_url) 
+  
+}
+
+
+sub process_tag {
+  # Process a tag in html file
+  my $lf_referer = shift;	# User argument
+  my $base_url = shift;		# Not used... why not?
+  my $tag_name = shift;
+  my $url_attrs = shift;
+
+  # Retrun quickly if no URL attributes
+  return unless defined($url_attrs);
+
+  my $attrs = shift;
+
+  my $rum_url;	# The absolute URL
+  my $lf_url;	# The local filesystem url
+  my $lf_url_o; # ... and it's object
+  my $key;
+
+  print STDERR "\nProcess Tag: $tag_name, URL attributes: ", 
+  join(', ',@{$url_attrs}),"\nbase_url: ",$base_url,"\nlf_referer: ",
+  $lf_referer,"\n" 
+    if $debug>2;
+
+  $lf_referer =~ s~^/~~;
+  $lf_referer = "file:/$lf_referer";
+
+  foreach $key (@{$url_attrs}) {
+    if (defined($$attrs{$key})) {
+      $rum_url=$$attrs{$key};
+      printf STDERR "$key = $rum_url\n" if $debug;
+      $lf_url=apply($rum_url);
+      if (defined($lf_url)) {
+
+	printf STDERR "Transformed to $lf_url\n" if $debug>2;
+
+	$lf_url =~ s~^/~~;  # Remove leading / to avoid doubeling
+	$lf_url_o=url "file:/$lf_url";
+
+	# Save new value in the hash
+	$$attrs{$key}=($lf_url_o->rel($lf_referer))->as_string;
+	print STDERR "New value: ",$$attrs{$key},"\n" if $debug>2;
+
+	# If there is potential information loss save the old value too
+	$$attrs{"W3MIR".$key}=$rum_url if $infoloss;
+      }
+    }
+  }
+}
+
+
+sub version {
+  eval 'require LWP;';
+  print $w3mir_agent,"\n";
+  print "LWP version ",$LWP::VERSION,"\n" if defined $LWP::VERSION;
+  print "Perl version: ",$],"\n";
+  exit(0);
+}
+
+
+sub parse_args {
+  my $f;
+  my $i;
+
+  $i=0;
+
+  while ($f=shift) {
+    $i++;
+    $numarg++;
+    # This is a demonstration against Getopts::Long.
+    if ($f =~ s/^-+//) {
+      $s=1,next if $f eq 's';			# Stdout
+      $r=1,next if $f eq 'r';			# Recurse
+      $fetch=1,next if $f eq 'fa';		# Fetch all, no date test
+      $fetch=-1,next if $f eq 'fs';	# Fetch those we don't already have.
+      $verbose=-1,next if $f eq 'q'; 		# Quiet
+      $verbose=1,next if $f eq 'c';		# Chatty
+      &version,next if $f eq 'v';		# Version
+      $pause=shift,next if $f eq 'p';		# Pause between requests
+      $retryPause=shift,next if $f eq 'rp';	# Pause between retries.
+      $s=2,$convertnl=0,next if $f eq 'f';	# Forget
+      $retry=shift,next if $f eq 't';		# reTry
+      $list=1,next if $f eq 'l';		# List urls
+      $iref=shift,next if $f eq 'ir';		# Initial referer
+      $check_robottxt = 0,next if $f eq 'drr';	# Disable robots.txt rules.
+      umask(oct(shift)),next if $f eq 'umask';
+      parse_cfg_file(shift),next if $f eq 'cfgfile';
+      usage(),exit 0 if ($f eq 'help' || $f eq 'h' || $f eq '?');
+      $remove=1,next if $f eq 'R';
+      $cache_header = 'Pragma: no-cache',next if $f eq 'pflush';
+      $w3http::agent=$w3mir_agent=shift,next if $f eq 'agent';
+      $abs=1,next if $f eq 'abs';
+      $convertnl=0,$batch=1,next if $f eq 'B';
+      $read_urls = 1,next if $f eq 'I';
+      $convertnl=0,next if $f eq 'nnc';
+
+      if ($f eq 'lc') {
+	if ($i == 1) {
+	  $lc=1;
+	  $iinline=($lc?"(?i)":"");
+	  $ipost=($lc?"i":"");
+	  next;
+	} else {
+	  die "w3mir: -lc must be the first argument on the commandline.\n";
+	}
+      }
+
+      if ($f eq 'P') {				# Proxy
+	($w3http::proxyserver,$w3http::proxyport)= 
+	  shift =~ /([^:]+):?(\d+)?/;
+	$w3http::proxyport=80 unless $w3http::proxyport;
+	$using_proxy=1;
+	next;
+      }
+    
+      if ($f eq 'd') {		# Debugging level
+	$f=shift;
+	unless (($debug = $f) > 0) {
+	  die "w3mir: debug level must be a number greater than zero.\n";
+	}
+	next;
+      }
+
+      # Those were all the options...
+      warn "w3mir: Unknown option: -$f.  Use -h for usage info.\n";
+      exit(1);
+      
+    } elsif ($f =~ /^http:/) {
+      my ($rum_url_o,$rum_reurl,$rum_rebase,$server);
+
+      $rum_url_o=root_queue($f,'./');
+
+      $rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+      push(@internal_apply,"s/^".$rum_rebase."//");
+      $scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+      $scope_ignore.="return 0 if m/^".
+	quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+
+    } else {
+      # If we get this far then the commandline is broken
+      warn "Unknown commandline argument: $f.  Use -h for usage info.\n";
+      $numarg--;
+      exit(1);
+    }
+  }
+  return 1;
+}
+
+
+sub parse_cfg_file {
+  # Read the configuration file.  Aborts on errors.  Not good to
+  # mirror something using the wrong config.
+
+  my ( $file ) = @_ ;
+  my ($key, $value, $authserver,$authrealm,$authuser,$authpasswd);
+  my $i;
+
+  die "w3mir: config file $file is not a file.\n" unless -f $file;
+  open(CFGF, $file) || die "Could not open config file $file: $!\n";
+
+  $i=0;
+
+  while (<CFGF>) {
+    # Trim off various junk
+    chomp;
+    s/^#.*//;
+    s/^\s+|\s$//g;
+    # Anything left?
+    next if $_ eq '';
+    # Examine remains
+    $i++;
+    $numarg++;
+
+    ($key, $value) = split(/\s*:\s*/,$_,2);
+    $key = lc $key;
+
+    $iref=$value,next if ( $key eq 'initial-referer' );
+    $header=$value,next if ( $key eq 'header' );
+    $pause=numeric($value),next if ( $key eq 'pause' );
+    $retryPause=numeric($value),next if ( $key eq 'retry-pause' );
+    $debug=numeric($value),next if ( $key eq 'debug' );
+    $retry=numeric($value),next if ( $key eq 'retries' );
+    umask(numeric($value)),next if ( $key eq 'umask' );
+    $check_robottxt=boolean($value),next if ( $key eq 'robot-rules' );
+    $edit=boolean($value),next if ($key eq 'remove-nomirror');
+    $indexname=$value,next if ($key eq 'index-name');
+    $s=nway($value,'save','stdout','forget'),next 
+      if ( $key eq 'file-disposition' );
+    $verbose=nway($value,'quiet','brief','chatty')-1,next
+      if ( $key eq 'verbosity' );
+    $w3http::proxyuser=$value,next if $key eq 'http-proxy-user';
+    $w3http::proxypasswd=$value,next if $key eq 'http-proxy-passwd';
+
+    if ( $key eq 'cd' ) {
+      $chdirto=$value;
+      warn "Use of 'cd' is discouraged\n" unless $verbose==-1;
+      next;
+    }
+    
+    if ($key eq 'http-proxy') {
+      ($w3http::proxyserver,$w3http::proxyport)= 
+	$value =~ /([^:]+):?(\d+)?/;
+      $w3http::proxyport=80 unless $w3http::proxyport;
+      $using_proxy=1;
+      next;
+    }
+
+    if ($key eq 'proxy-options') {
+      my($val,$nval,@popts,$pragma);
+      $pragma=1;
+      foreach $val (split(/\s*,\*/,lc $value)) {
+	$nval=nway($val,'no-pragma','revalidate','refresh','no-store',);
+	# Force use of Cache-control: header
+	$pragma=0 if ($nval==0);
+	# use to force proxy to revalidate
+	$pragma=0,push(@popts,'max-age=0') if ($nval==1);
+	# use to force proxy to refresh
+	push(@popts,'no-cache') if ($nval==2);
+	# use if information transfered is sensitive
+	$pragma=0,push(@popts,'no-store') if ($nval==3);
+      }
+      $cache_header=($pragma?'Pragma: ':'Cache-control: ').join(', ',@popts);
+      next;
+    }
+
+
+    if ($key eq 'url') {
+      my ($rum_url_o,$lf_dir,$rum_reurl,$rum_rebase);
+
+      # A two argument URL: line?
+      if ($value =~ m/^(.+)\s+(.+)/i) {
+	# Two arguments.
+	# The last is a directory, it must end in /
+	$lf_dir=$2;
+	$lf_dir.='/' unless $lf_dir =~ m~/$~;
+
+	$rum_url_o=root_queue($1,$lf_dir);
+
+	# The first is a URL, make it more canonical, find the base.
+	# The namespace confusion in this section is correct.(??)
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+	# print "URL: ",$rum_url_o->as_string,"\n";
+	# print "Base: $rum_rebase\n";
+
+	# Translate from rum space to lf space:
+	push(@internal_apply,"s/^".$rum_rebase."/".quotemeta($lf_dir)."/");
+
+	# That translation could lead to information loss.
+	$infoloss=1;
+
+	# Fetch rules tests the rum_url_o->as_string.  Fetch whatever
+	# matches the base.
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+
+	# Ignore whatever did not match the base.
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+
+      } else {
+	$rum_url_o=root_queue($value,'./');
+
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+	# Translate from rum space to lf space:
+	push(@internal_apply,"s/^".$rum_rebase."//");
+
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+      }
+      next;
+    }
+
+    if ($key eq 'also-quene') {
+      print STDERR 
+	"Found 'also-quene' keyword, please replace with 'also-queue'\n";
+      $key='also-queue';
+    }
+
+    if ($key eq 'also' || $key eq 'also-queue') {
+      if ($value =~ m/^(.+)\s+(.+)/i) {
+	my ($rum_url_o,$lf_dir,$rum_reurl,$rum_rebase);
+	# Two arguments.
+	# The last is a directory, it must end in /
+	# print STDERR "URL ",$1," DIR ",$2,"\n";
+	$lf_dir=$2;
+	$lf_dir.='/' unless $lf_dir =~ m~/$~;
+
+	if ($key eq 'also-queue') {
+	  $rum_url_o=root_queue($1,$lf_dir);
+	} else {
+	  root_queue("",$lf_dir);
+	  $rum_url_o=url $1;
+	  $rum_url_o->host(lc $rum_url_o->host);
+	}
+
+	# The first is a URL, find the base
+	$rum_rebase = quotemeta( ($rum_url_o->as_string =~ m~^(.*/)~)[0] );
+
+#	print "URL: $rum_url_o->as_string\n";
+#	print "Base: $rum_rebase\n";
+#	print "Server: $server\n";
+
+	# Ok, now we can transform and select stuff the right way
+	push(@internal_apply,"s/^".$rum_rebase."/".quotemeta($lf_dir)."/");
+	$infoloss=1;
+
+	# Fetch rules tests the rum_url_o->as_string.  Fetch whatever
+	# matches the base.
+	$scope_fetch.="return 1 if m/^".$rum_rebase."/".$ipost.";\n";
+
+	# Ignore whatever did not match the base.  This cures problem
+	# with '..' from base in in rum space pointing within the the
+	# scope in ra space.  We introduced a extra level (or more) of
+	# directories with the apply above.  Must do same with 'Also:'
+	# directives.
+	$scope_ignore.="return 0 if m/^".
+	  quotemeta("http://".$rum_url_o->netloc."/")."/".$ipost.";\n";
+      } else {
+	die "Also: requires 2 arguments\n";
+      }
+      next;
+    }
+
+    if ($key eq 'quene') {
+      print STDERR "Found 'quene' keyword, please replace with 'queue'\n";
+      $key='queue';
+    }
+
+    if ($key eq 'queue') {
+      root_queue($value,"");
+      next;
+    }
+
+    if ($key eq 'ignore-re' || $key eq 'fetch-re') {
+      # Check that it's a re, better that I am strict than for perl to
+      # make compilation errors.
+      unless ($value =~ /^m(.).*\1[gimosx]*$/) {
+	print STDERR "w3mir: $value is not a recognized regular expression\n";
+	exit 1;
+      }
+      # Fall-through to next cases!
+    }
+
+    if ($key eq 'fetch' || $key eq 'fetch-re') {
+      my $expr=$value;
+      $expr = wild_re($expr).$ipost if ($key eq 'fetch');
+      $rule_text.=' return 1 if '.$expr.";\n";
+      next;
+    }
+
+    if ($key eq 'ignore' || $key eq 'ignore-re') {
+      my $expr=$value;
+      $expr = wild_re($expr).$ipost if ($key eq 'ignore');
+      # print STDERR "Ignore expression: $expr\n";
+      $rule_text.=' return 0 if '.$expr.";\n";
+      next;
+    }
+
+
+    if ($key eq 'apply') {
+      unless ($value =~ /^s(.).*\1.*\1[gimosxe]*$/) {
+	print STDERR
+	  "w3mir: '$value' is not a recognized regular expression\n";
+	exit 1;
+      }
+      push(@user_apply,$value) ;
+      $infoloss=1;
+      next;
+    }
+
+    if ($key eq 'agent') {
+      $w3http::agent=$w3mir_agent=$value;
+      next;
+    }
+
+    # The authorization stuff:
+    if ($key eq 'auth-domain') {
+      $useauth=1;
+      ($authserver, $authrealm) = split('/',$value,2);
+      die "w3mir: server part of auth-domain has format server[:port]\n"
+	unless $authserver =~ /^(\S+(:\d+)?)$|^\*$/;
+      $authserver =~ s/:80$//;
+      die "w3mir: auth-domain '$value' is not valid\n"
+	if !defined($authserver) || !defined($authrealm);
+      $authrealm=lc $authrealm;
+    }
+
+    $authuser=$value if ($key eq 'auth-user');
+    $authpasswd=$value if ($key eq 'auth-passwd');
+
+    # Got a full authentication spec?
+    if ($authserver && $authrealm && $authuser && $authpasswd) {
+      $authdata{$authserver}{$authrealm}=$authuser.":".$authpasswd;
+      print "Authentication for $authserver/$authrealm is ".
+	"$authuser/$authpasswd\n" if $verbose>=0;
+      # exit;
+      # Invalidate tmp vars
+      $authserver=$authrealm=$authuser=$authpasswd=undef;
+      next;
+    }
+
+    next if $key eq 'auth-user' || $key eq 'auth-passwd' || 
+      $key eq 'auth-domain';
+
+    if ($key eq 'fetch-options') {
+      warn "w3mir: The 'fetch-options' directive has been renamed to 'options'\nw3mir: Please change your configuration file.\n";
+      $key='options';
+      # Fall through to 'options'!
+    }
+
+    if ($key eq 'options') {
+      
+      my($val,$nval);
+      foreach $val (split(/\s*,\s*/,lc $value)) {
+	if ($i==1) {
+	  $nval=nway($val,'recurse','no-date-check','only-nonexistent',
+		     'list-urls','lowercase','remove','batch','read-urls',
+		     'abs','no-newline-conv');
+	  $r=1,next if $nval==0;
+	  $fetch=1,next if $nval==1;
+	  $fetch=-1,next if $nval==2;
+	  $list=1,next if $nval==3;
+	  if ($nval==4) {
+	    $lc=1;
+	    $iinline=($lc?"(?i)":"");
+	    $ipost=($lc?"i":"");
+	    next ;
+	  }
+	  $remove=1,next if $nval==5;
+	  $convertnl=0,$batch=1,next if $nval==6;
+	  $read_urls=1,next if $nval==7;
+	  $abs=1,next if $nval==8;
+	  $convertnl=0,next if $nval==9;
+        } else {
+	    die "w3mir: options must be the first directive in the config file.\n";
+	}
+      }
+      next;
+    }
+
+    if ($key eq 'disable-headers') {
+      my($val,$nval);
+      foreach $val (split(/\s*,\s*/,lc $value)) {
+	$nval=nway($val,'referer','user');
+	$do_referer=0,next if $nval==0;
+	$do_user=0,next if $nval==1;
+      }
+      next;
+    }
+
+
+    if ($key eq 'fixup') {
+
+      $fixrc="$file";
+      # warn "Fixrc: $fixrc\n";
+
+      my($val,$nval);
+      foreach $val (split(/\s*,\s*/,lc $value)) {
+	$nval=nway($val,'on','run','noindex','off');
+	$runfix=1,next if $nval==1;
+	# Disable fixup
+	$fixup=0,next if $nval==3;
+	# Ignore everyting else
+      }
+      next;
+    }
+
+    die "w3mir: Unrecognized directive ('$key') in config file $file at line $.\n";
+
+  }
+  close(CFGF);
+
+  if (defined($w3http::proxypasswd) && $w3http::proxyuser) {
+    warn "Proxy authentication: ".$w3http::proxyuser.":".
+      $w3http::proxypasswd."\n" if $verbose>=0;
+  }
+
+}
+
+
+sub wild_re {
+  # Here we translate unix wildcard subset to to perlre
+  local($_) = shift;
+
+  # Quote anything that's RE and not wildcard: / ( ) \ | { } + $ ^
+  s~([\/\(\)\\\|\{\}\+)\$\^])~\\$1~g;
+  # . -> \.
+  s~\.~\\.~g;
+  # * -> .*
+  s~\*~\.\*~g;
+  # ? -> .
+  s~\?~\.~g;
+
+  # print STDERR "wild_re: $_\n";
+
+  return $_ = '/'.$_.'/';
+}
+
+
+sub numeric {
+  # Check if argument is numeric?
+  my ( $number ) = @_ ;
+  return oct($number) if ($number =~ /\d+/ || $number =~ /\d+.\d+/);
+  die "Expected a number, got \"$number\"\n";
+}
+
+
+sub boolean {
+  my ( $boolean ) = @_ ;
+
+  $boolean = lc $boolean;
+
+  return 0 if ($boolean eq 'false' || $boolean eq 'off' || $boolean eq '0');
+  return 1 if ($boolean eq 'true' || $boolean eq 'on' || $boolean eq '1');
+  die "Expected a boolean, got \"$boolean\"\n";
+}
+
+
+sub nway {
+  my ( $value ) = shift;
+  my ( @values ) = @_;
+  my ( $val ) = 0;
+
+  $value = lc $value;
+  while (@_) {
+    return $val if $value eq shift;
+    $val++;
+  }
+  die "Expected one of ".join(", ",@values).", got \"$value\"\n";
+}
+
+
+sub insert_at_start {
+  # ark: inserts the first arg at the top of the html in the second arg
+  # janl: The second arg must be a reference to a scalar.
+  my( $str, $text_ref ) = @_;
+  my( @possible ) =("<BODY.*?>", "</HEAD.*?>", "</TITLE.*?>", "<HTML.*?>" );
+  my( $f, $done );
+  
+  $done=0;
+  @_=@possible;
+  
+  while( $done!=1 && ($f=shift) ){
+    print "Searching for: $f\n";
+    if( $$text_ref =~ /$f/i ){
+      print "found it!\n";
+      $$text_ref =~ s/($f)/$1\n$str/i;
+      $done=1;
+    }
+  }
+}
+
+
+
+sub rm_rf {
+  # Recursively remove directories and other files
+  # File::Path::rmtree does a similar thing but the messages are wrong
+
+  my($remove)=shift;
+
+  eval "use File::Find;" unless defined(&finddepth);
+
+  die "w3mir: Could not load File::Find module when trying to remove $remove\n"
+    unless defined(&find);
+
+  finddepth(\&remove_everything,$remove);
+
+  if (rmdir($remove)) {
+    print STDERR "\nw3mir: removed directory $remove\n" if $verbose>=0;
+  } else {
+    print STDERR "w3mir: could not remove $remove: $!\n";
+  }
+}
+
+
+sub remove_everything {
+  # This does the removal
+  ((-d && rmdir($_)) || unlink($_)) && $verbose>=0 &&
+    print STDERR "w3mir: removed $File::Find::name\n";
+}
+
+
+
+sub usage {
+  my($message)=shift @_;
+
+  print STDERR "w3mir: $message\n" if $message;
+  
+  die 'w3mir: usage: w3mir [options] <single-http-url>
+          or: w3mir -B [-I] [options] [<http-urls>]
+
+    Options :
+	-agent <agent>	- Set the agent name.  Default is w3mir
+	-abs		- Force all URLs to be absolute.
+	-B		- Batch-get documents.
+	-I		- The URLs to get are read from standard input.
+	-c		- be more Chatty.
+	-cfgfile <file> - Read config from file
+	-d <debug-level>- set debug level to 1 or 2
+	-drr		- Disable robots.txt rules.
+	-f		- Forget all files, nothing is saved to disk.
+	-fa		- Fetch All, will not check timestamps.
+	-fs		- Fetch Some, do not fetch the files we already have.
+	-ir <referer>	- Initial referer.  For picky servers.
+	-l		- List URLs in the documents retrived.
+	-lc		- Convert all URLs (and filenames) to lowercase.
+				This does not work reliably.
+	-p <n>		- Pause n seconds before retriving each doc.
+	-q		- Quiet, error-messages only
+        -rp <n>		- Retry Pause in seconds.
+	-P <server:port>- Use host/port for proxy http requests
+        -pflush		- Flush proxy server.
+	-r		- Recursive mirroring.
+        -R		- Remove files not referenced or not present on server.
+	-s		- Send output to stdout instead of file
+	-t <n>		- How many times to (re)try getting a failed doc?
+        -umask <umask>  - Set umask for mirroring, must be usual octal format.
+	-nnc		- No Newline Conversion.  Disable newline conversions.
+	-v		- Show w3mir version.
+';
+}
+__END__
+# -*- perl -*- There must be a blank line here
+
+=head1 NAME
+
+w3mir - all purpose HTTP-copying and mirroring tool
+
+=head1 SYNOPSIS
+
+B<w3mir> [B<options>] [I<HTTP-URL>]
+
+B<w3mir> B<-B> [B<options>] <I<HTTP-URLS>>
+
+B<w3mir> is a all purpose HTTP copying and mirroring tool.  The
+main focus of B<w3mir> is to create and maintain a browsable copy of
+one, or several, remote WWW site(s).
+
+Used to the max w3mir can retrive the contents of several related
+sites and leave the mirror browseable via a local web server, or from
+a filesystem, such as directly from a CDROM.  
+
+B<w3mir> has options for all operations that are simple enough for
+options.  For authentication and passwords, multiple site retrievals
+and such you will have to resort to a L</CONFIGURATION-FILE>.  If
+browsing from a filesystem references ending in '/' needs to be
+rewritten to end in '/index.html', and in any case, if there are URLs
+that are redirected will need to be changed to make the mirror
+browseable, see the documentation of B<Fixup> in the
+L</CONFIGURATION-FILE> secton.
+
+B<w3mir>s default behavior is to do as little as possible and to be as
+nice as possible to the server(s) it is getting documents from.  You
+will need to read through the options list to make B<w3mir> do more
+complex, and, useful things.  Most of the things B<w3mir> can do is
+also documented in the w3mir-HOWTO which is available at the B<w3mir>
+home-page (F<http://www.math.uio.no/~janl/w3mir/>) as well as in the
+w3mir distribution bundle.
+
+=head1 DESCRIPTION
+
+You may specify many options and one HTTP-URL on the w3mir
+command line.
+
+A single HTTP URL I<must> be specified either on the command line or
+in a B<URL> directive in a configuration file.  If the URL refers to a
+directory it I<must> end with a "/", otherwise you might get surprised
+at what gets retrieved (e.g. rather more than you expect).
+
+Options must be prefixed with at least one - as shown below, you can
+use more if you want to. B<-cfgfile> is equivalent to B<--cfgfile> or
+even B<------cfgfile>.  Options cannot be I<clustered>, i.e., B<-r -R>
+is not equivalent to B<-rR>.
+
+=over 4
+
+=item B<-h> | B<-help> | B<-?>
+
+prints a brief summary of all command line options and exits.
+
+=item B<-cfgfile> F<file>
+
+Makes B<w3mir> read the given configuration file.  See the next section
+for how to write such a file.
+
+=item B<-r>
+
+Puts B<w3mir> into recursive mode.  The default is to fetch only one
+document and then quit.  'I<recursive>' mode means that all the
+documents linked to the given document that are fetched, and all they
+link to in turn and so on.  But only I<Iff> they are in the same
+directory or under the same directory as the start document.  Any
+document that is in or under the starting documents directory is said
+to be within the I<scope of retrieval>.
+
+=item B<-fa>
+
+Fetch All.  Normally B<w3mir> will only get the document if it has been
+updated since the last time it was fetched.  This switch turns that
+check off.
+
+=item B<-fs>
+
+Fetch Some.  Not the opposite of B<-fa>, but rather, fetch the ones we
+don't have already.  This is handy to restart copying of a site
+incompletely copied by earlier, interrupted, runs of B<w3mir>.
+
+=item B<-p> I<n>
+
+Pause for I<n> seconds between getting each document.  The default is
+30 seconds.
+
+=item B<-rp> I<n>
+
+Retry Pause, in seconds.  When B<w3mir> fails to get a document for some
+technical reason (timeout mainly) the document will be queued for a
+later retry.  The retry pause is how long B<w3mir> waits between
+finishing a mirror pass before starting a new one to get the still
+missing documents.  This should be a long time, so network conditions
+have a chance to get better.  The default is 600 seconds (10 minutes),
+which might be a bit too short, for batch running B<w3mir> I would
+suggest an hour (3600 seconds) or more.
+
+=item B<-t> I<n>
+
+Number of reTries.  If B<w3mir> cannot get all the documents by the
+I<n>th retry B<w3mir> gives up.  The default is 3.
+
+=item B<-drr>
+
+Disable Robot Rules.  The robot exclusion standard is described in
+http://info.webcrawler.com/mak/projects/robots/norobots.html.  By
+default B<w3mir> honors this standard.  This option causes B<w3mir> to
+ignore it.
+
+=item B<-nnc>
+
+No Newline Conversion.  Normally w3mir converts the newline format of
+all files that the web server says is a text file.  However, not all
+web servers are reliable, and so binary files may become corrupted due
+to the newline conversion w3mir performs.  Use this option to stop
+w3mir from converting newlines.  This also causes the file to be
+regarded as binary when written to disk, to disable the implicit
+newline conversion when saving text files on most non-Unix systems.
+
+This will probably be on by default in version 1.1 of w3mir, but not
+in version 1.0.
+
+=item B<-R>
+
+Remove files.  Normally B<w3mir> will not remove files that are no
+longer on the server/part of the retrieved web of files.  When this
+option is specified all files no longer needed or found on the servers
+will be removed.  If B<w3mir> fails to get a document for I<any> other
+reason the file will not be removed.
+
+=item B<-B>
+
+Batch fetch documents whose URLs are given on the commandline.
+
+In combination with the B<-r> and/or B<-l> switch all HTML and PDF
+documents will be mined for URLs, but the documents will be saved on
+disk unchanged.  When used with the B<-r> switch only one single URL
+is allowed.  When not used with the B<-r> switch no HTML/URL
+processing will be performed at all.  When the B<-B> switch is used
+with B<-r> w3mir will not do repeated mirrorings reliably since the
+changes w3mir needs to do, in the documents, to work reliably are not
+done.  In any case it's best not to use B<-R> in combination with
+B<-B> since that can result in deleting rather more documents than
+expected.  Hwowever, if the person writing the documents being copied
+is good about making references relative and placing the <HTML> tag at
+the beginning of documents there is a fair chance that things will
+work even so.  But I wouln't bet on it.  It will, however, work
+reliably for repeated mirroring if the B<-r> switch is not used.
+
+When the B<-B> switch is specified redirects for a given document will
+be followed no matter where they point.  The redirected-to document
+will be retrieved in the place of the original document.  This is a
+potential weakness, since w3mir can be directed to fetch any document
+anywhere on the web.
+
+Unless used with B<-r> all retrived files will be stored in one
+directory using the remote filename as the local filename.  I.e.,
+F<http://foo/bar/gazonk.html> will be saved as F<gazonk.html>.
+F<http://foo/bar/> will be saved as F<bar-index.html> so as to avoid
+name colitions for the common case of URLs ending in /.
+
+=item B<-I>
+
+This switch can only be used with the B<-B> switch, and only after it
+on the commandline or configuration file.  When given w3mir will get
+URLs from standard input (i.e., w3mir can be used as the end of a pipe
+that produces URLs.)  There should only be one URL pr. line of input.
+
+=item B<-q>
+
+Quiet.  Turns off all informational messages, only errors will be
+output.
+
+=item B<-c>
+
+Chatty.  B<w3mir> will output more progress information.  This can be
+used if you're watching B<w3mir> work.
+
+=item B<-v>
+
+Version.  Output B<w3mir>s version.
+
+=item B<-s>
+
+Copy the given document(s) to STDOUT.
+
+=item B<-f>
+
+Forget.  The retrieved documents are not saved on disk, they are just
+forgotten.  This can be used to prime the cache in proxy servers, or
+not save documents you just want to list the URLs in (see B<-l>).
+
+=item B<-l>
+
+List the URLs referred to in the retrieved document(s) on STDOUT.
+
+=item B<-umask> I<n>
+
+Sets the umask, i.e., the permission bits of all retrieved files.  The
+number is taken as octal unless it starts with a 0x, in which case
+it's taken as hexadecimal.  No matter what you set this to make sure
+you get write as well as read access to created files and directories.
+
+Typical values are: 
+
+=over 8
+
+=item 022
+
+let everyone read the files (and directories), only you can change
+them. 
+
+=item 027
+
+you and everyone in the same file-group as you can read, only you can
+change them.
+
+=item 077
+
+only you can read the files, only you can change them.
+
+=item 0
+
+everyone can read, write and change everything.
+
+=back
+
+The default is whatever was set when B<w3mir> was invoked.  022 is a
+reasonable value.
+
+This option has no meaning, or effect, on Win32 platforms.
+
+=item B<-P> I<server:port>
+
+Use the given server and port is a HTTP proxy server.  If no port is
+given port 80 is assumed (this is the normal HTTP port).  This is
+useful if you are inside a firewall, or use a proxy server to save
+bandwidth.
+
+=item B<-pflush>
+
+Proxy flush, force the proxy server to flush it's cache and re-get the
+document from the source.  The I<Pragma: no-cache> HTTP/1.0 header is
+used to implement this.
+
+=item B<-ir> I<referrer>
+
+Initial Referrer.  Set the referrer of the first retrieved document.
+Some servers are reluctant to serve certain documents unless this is
+set right.
+
+=item B<-agent> I<agent>
+
+Set the HTTP User-Agent fields value.  Some servers will serve
+different documents according to the WWW browsers capabilities.
+B<w3mir> normally has B<w3mir>/I<version> in this header field.
+Netscape uses things like B<Mozilla/3.01 (X11; I; Linux 2.0.30 i586)>
+and MSIE uses things like B<Mozilla/2.0 (compatible; MSIE 3.02;
+Windows NT)> (remember to enclose agent strings with spaces in with
+double quotes ("))
+
+=item B<-lc>
+
+Lower Case URLs. Some OSes, like W95 and NT, are not case sensitive
+when it comes to filenames.  Thus web masters using such OSes can case
+filenames differently in different places (apps.html, Apps.html,
+APPS.HTML).  If you mirror to a Unix machine this can result in one
+file on the server becoming many in the mirror.  This option
+lowercases all filenames so the mirror corresponds better with the
+server.
+
+If given it must be the first option on the command line.
+
+This option does not work perfectly.  Most especially for mixed case
+host-names.
+
+=item B<-d> I<n>
+
+Set the debug level.  A debug level higher than 0 will produce lots of
+extra output for debugging purposes.
+
+=item B<-abs>
+
+Force all URLs to be absolute.  If you retrive
+F<http://www.ifi.uio.no/~janl/index.html> and it references foo.html
+the referense is absolutified into
+F<http://www.ifi.uio.no/~janl/foo.html>.  In other words, you get
+absolute references to the origin site if you use this option.
+
+=back
+
+=head1 CONFIGURATION-FILE
+
+Most things can be mirrored with a (long) command line.  But multi
+server mirroring, authentication and some other things are only
+available through a configuration file.  A configuration file can
+either be specified with the B<-cfgfile> switch, but w3mir also looks
+for .w3mirc (w3mir.ini on Win32 platforms) in the directory where
+w3mir is started from.
+
+The configuration file consists of lines of comments and directives.
+A directive consists of a keyword followed by a colon (:) and then one
+or several arguments.
+
+ # This is a comment.  And the next line is a directive:
+ Options: recurse, remove
+
+A comment can only start at the beginning of a line.  The directive
+keywords are not case-sensitive, but the arguments I<might> be.
+
+=over 4
+
+=item Options: I<recurse> | I<no-date-check> | I<only-nonexistent> | I<list-urls> | I<lowercase> | I<remove> | I<batch> | I<input-urls> | I<no-newline-conv>
+
+This must be the first directive in a configuration file.  
+
+=over 8
+
+=item I<recurse>
+
+see B<-r> switch.
+
+=item I<no-date-check>
+
+see B<-fa> switch.
+
+=item I<only-nonexistent>
+
+see B<-fs> switch.
+
+=item I<list-urls>
+
+see B<-l> option.
+
+=item I<lowercase>
+
+see B<-lc> option. 
+
+=item I<remove>
+
+see B<-R> option.
+
+=item I<batch>
+
+see B<-B> option.
+
+=item I<input-urls>
+
+see B<-I> option.
+
+=item I<no-newline-conv>
+
+see B<-nnc> option.
+
+=back
+
+=item URL: I<HTTP-URL> [I<target-directory>]
+
+The URL directive may only appear once in any configuration file.
+
+Without the optional target directory argument it corresponds directly
+to the I<single-HTTP-URL> argument on the command line.  
+
+If the optional target directory is given all documents from under the
+given URL will be stored in that directory, and under.  The target
+directory is most likely only specified if the B<Also> directive is
+also specified.
+
+If the URL given refers to a directory it I<must> end in a "/",
+otherwise you might get quite surprised at what gets retrieved.
+
+Either one URL: directive or the single-HTTP-URL at the command-line
+I<must> be given.
+
+=item Also: I<HTTP-URL directory>
+
+This directive is only meaningful if the I<recurse> (or B<-r>)
+option is given.
+
+The directive enlarges the scope of a recursive retrieval to contain
+the given HTTP-URL and all documents in the same directory or under.
+Any documents retrieved because of this directive will be stored in the
+given directory of the mirror.
+
+In practice this means that if the documents to be retrieved are stored
+on several servers, or in several hierarchies on one server or any
+combination of those.  Then the B<Also> directive ensures that we get
+everything into one single mirror.
+
+This also means that if you're retrieving
+
+  URL: http://www.foo.org/gazonk/
+
+but it has inline icons or images stored in http://www.foo.org/icons/
+which you will also want to get, then that will be retrieved as well by
+entering
+
+  Also: http://www.foo.org/icons/ icons
+
+As with the URL directive, if the URL refers to a directory it I<must>
+end in a "/".
+
+Another use for it is when mirroring sites that have several names
+that all refer to the same (logical) server:
+
+  URL: http://www.midifest.com/
+  Also: http://midifest.com/ .
+
+At this point in time B<w3mir> has no mechanism to easily enlarge the
+scope of a mirror after it has been established.  That means that you
+should survey the documents you are going to retrieve to find out what
+icons, graphics and other things they refer to that you want.  And
+what other sites you might like to retrieve.  If you find out that
+something is missing you will have to delete the whole mirror, add the
+needed B<Also> directives and then reestablish the mirror.  This lack
+of flexibility in what to retrieve will be addressed at a later date.
+
+See also the B<Also-quene> directive.
+
+=item Also-quene: I<HTTP-URL directory>
+
+This is like Also, except that the URL itself is also quened.  The
+Also directive will not cause any documents to be retrived UNLESS they
+are referenced by some other document w3mir has already retrived.
+
+=item Quene: I<HTTP-URL>
+
+This is quenes the URL for retrival, but does not enlarge the scope of
+the retrival.  If the URL is outside the scope of retrival it will not
+be retrived anyway.
+
+The observant reader will see that B<Also-quene> is like B<Also>
+combined with B<Quene>.
+
+=item Initial-referer: I<referer>
+
+see B<-ir> option.
+
+=item Ignore: F<wildcard>
+
+=item Fetch: F<wildcard>
+
+=item Ignore-RE: F<regular-expression>
+
+=item Fetch-RE: F<regular-expression>
+
+These four are used to set up rules about which documents, within the
+scope of retrieval, should be gotten and which not.  The default is to
+get I<anything> that is within the scope of retrieval.  That may not
+be practical though.  This goes for CGI scripts, and especially server
+side image maps and other things that are executed/evaluated on the
+server.  There might be other things you want unfetched as well.
+
+B<w3mir> stores the I<Ignore>/I<Fetch> rules in a list.  When a
+document is considered for retrieval the URL is checked against the
+list in the same order that the rules appeared in the configuration
+file.  If the URL matches any rule the search stops at once.  If it
+matched a I<Ignore> rule the document is not fetched and any URLs in
+other documents pointing to it will point to the document at the
+original server (not inside the mirror).  If it matched a I<Fetch>
+rule the document is gotten.  If not matched by any ruøes the document
+is gotten.
+
+The F<wildcard>s are a very limited subset of Unix-wildcards.
+B<w3mir> understands only 'I<?>', 'I<*>', and 'I<[x-y]>' ranges.
+
+The F<perl-regular-expression> is perls superset of the normal Unix
+regular expression syntax.  They must be completely specified,
+including the prefixed m, a delimiter of your choice (except the
+paired delimiters: parenthesis, brackets and braces), and any of the
+RE modifiers. E.g.,
+
+  Ignore-RE: m/.gif$/i
+
+or
+
+  Ignore-RE: m~/.*/.*/.*/~
+
+and so on.  "#" cannot be used as delimiter as it is the comment
+character in the configuration file.  This also has the bad
+side-effect of making you unable to match fragment names (#foobar)
+directly.  Fortunately perl allows writing ``#'' as ``\043''.
+
+You must be very carefull of using the RE anchors (``^'' and ``$''
+with the RE versions of these and the I<Apply> directive. Given the
+rules:
+
+  Fetch-RE: m/foobar.cgi$/
+  Ignore: *.cgi
+
+the all files called ``foobar.cgi'' will be fetched. However, if the
+file is referenced as ``foobar.cgi?query=mp3'' it will I<not> be
+fetched since the ``$'' anchor will prevent it from matching the
+I<Fetch-RE> directive and then it will match the I<Ignore> directive
+instead. If you want to match ``foobar.cgi'' but not ``foobar.cgifu''
+you can use perls ``\b'' character class which matches a word
+boundrary:
+
+  Fetch-RE: m/foobar.cgi\b/
+  Ignore: *.cgi
+
+which will get ``foobar.cgi'' as well as ``foobar.cgi?query=mp3'' but
+not ``foobar.cgifu''. BUT, you must keep in mind that a lot of
+diffetent characters make a word boundrary, maybe something more
+subtle is needed.
+
+=item Apply: I<regular-expression>
+
+This is used to change a URL into another URL.  It is a potentially
+I<very> powerful feature, and it also provides ample chance for you to
+shoot your own foot. The whole aparatus is somewhat tenative, if you
+find there is a need for changes in how Apply rules work please
+E-mail. If you are going to use this feature please read the
+documentation for I<Fetch-RE> and I<Ignore-RE> first.
+
+The B<Apply> expressions are applied, in sequence, to the URLs in
+their absolute form. I.e., with the whole
+http://host:port/dir/ec/tory/file URL. It is only after this B<w3mir>
+checks if a document is within the scope of retrieval or not. That
+means that B<Apply> rules can be used to change certain URLs to fall
+inside the scope of retrieval, and vice versa.
+
+The I<regular-expression> is perls superset of the usual Unix regular
+expressions for substitution.  As with I<Fetch> and I<Ignore> rules it
+must be specified fully, with the I<s> and delimiting character.  It
+has the same restrictions with regards to delimiters. E.g.,
+
+  Apply: s~/foo/~/bar/~i
+
+to translate the path element I<foo> to I<bar> in all URLs.
+
+"#" cannot be used as delimiter as it is the comment character in the
+configuration file.
+
+Please note that w3mir expects that URLs identifying 'directories'
+keep idenfifying directories after application of Apply rules.  Ditto
+for files.
+
+=item Agent: I<agent>
+
+see B<-agent> option.
+
+=item Pause: I<n>
+
+see B<-p> option.
+
+=item Retry-Pause: I<n>
+
+see B<-rp> option.
+
+=item Retries: I<n>
+
+see B<-t> option.
+
+=item debug: I<n>
+
+see B<-d> option.
+
+=item umask I<n>
+
+see B<-umask> option.
+
+=item Robot-Rules: I<on> | I<off>
+
+Turn robot rules on of off.  See B<-drr> option.
+
+=item Remove-Nomirror: I<on> | I<off>
+
+If this is enabled sections between two consecutive
+
+  <!--NO MIRROR-->
+
+comments in a mirrored document will be removed.  This editing is
+performed even if batch getting is specified.
+
+=item Header: I<html/text>
+
+Insert this I<complete> html/text into the start of the document.
+This will be done even if batch is specified.
+
+=item File-Disposition: I<save> | I<stdout> | I<forget>
+
+What to do with a retrieved file.  The I<save> alternative is default.
+The two others correspond to the B<-s> and B<-f> options.  Only one
+may be specified.
+
+=item Verbosity: I<quiet> | I<brief> | I<chatty>
+
+How much B<w3mir> informs you of it's progress.  I<Brief> is the
+default.  The two others correspond to the B<-q> and B<-c> switches.
+
+=item Cd: I<directory>
+
+Change to given directory before starting work.  If it does not exist
+it will be quietly created.  Using this option breaks the 'fixup'
+code so consider not using it, ever.
+
+=item HTTP-Proxy: I<server:port>
+
+see the B<-P> switch.
+
+=item HTTP-Proxy-user: I<username>
+
+=item HTTP-Proxy-passwd: I<password>
+
+These two are is used to activate authentication with the proxy
+server.  L<w3mir> only supports I<basic> proxy autentication, and is
+quite simpleminded about it, if proxy authentication is on L<w3mir>
+will always give it to the proxy.  The domain concept is not supported
+with proxy-authentication.
+
+=item Proxy-Options: I<no-pragma> | I<revalidate> | I<refresh> | I<no-store>
+
+Set proxy options.  There are two ways to pass proxy options, HTTP/1.0
+compatible and HTTP/1.1 compatible.  Newer proxy-servers will
+understand the 1.1 way as well as 1.0.  With old proxy-servers only
+the 1.0 way will work.  L<w3mir> will prefer the 1.0 way.
+
+The only 1.0 compatible proxy-option is I<refresh>, it corresponds to
+the B<-pflush> option and forces the proxy server to pass the request
+to a upstream server to retrieve a I<fresh> copy of the document. 
+
+The I<no-pragma> option forces w3mir to use the HTTP/1.1 proxy
+control header, use this only with servers you know to be new,
+otherwise it won't work at all.  Use of any option but I<refresh> will
+also cause HTTP/1.1 to be used.  
+
+I<revalidate> forces the proxy server to contact the upstream server
+to validate that it has a fresh copy of the document.  This is nicer
+to the net than I<refresh> option which forces re-get of the document
+no matter if the server has a fresh copy already.
+
+I<no-store> forbids the proxy from storing the document in other than
+in transient storage.  This can be used when transferring sensitive
+documents, but is by no means any warranty that the document can't be
+found on any storage device on the proxy-server after the transfer.
+Cryptography, if legal in your contry, is the solution if you want the
+contents to be secret.
+
+I<refresh> corresponds to the HTTP/1.0 header I<Pragma: no-cache> or
+the identical HTTP/1.1 I<Cache-control> option.  I<revalidate> and
+I<no-store> corresponds to I<max-age=0> and I<no-store> respectively.
+
+=item Authorization
+
+B<w3mir> supports only the I<basic> authentication of HTTP/1.0.  This
+method can assign a password to a given user/server/I<realm>.  The
+"user" is your user-name on the server.  The "server" is the server.
+The I<realm> is a HTTP concept.  It is simply a grouping of files and
+documents.  One file or a whole directory hierarchy can belong to a
+realm.  One server may have many realms.  A user may have separate
+passwords for each realm, or the same password for all the realms the
+user has access to.  A combination of a server and a realm is called a
+I<domain>.
+
+=over 8
+
+=item Auth-Domain: I<server:port/realm>
+
+Give the server and port, and the belonging realm (making a domain)
+that the following authentication data holds for.  You may specify "*"
+wildcard for either of I<server:port> and I<realm>, this will work
+well if you only have one usernme and password on all the servers
+mirrored.
+
+=item Auth-User: I<user>
+
+Your user-name.
+
+=item Auth-Passwd: I<password>
+
+Your password.
+
+=back
+
+These three directives may be repeated, in clusters, as many times as
+needed to give the necessary authentication information
+
+=item Disable-Headers: I<referer> | I<user>
+
+Stop B<w3mir> from sending the given headers.  This can be used for
+anonymity, making your retrievals harder to track.  It will be even
+harder if you specify a generic B<Agent>, like Netscape.
+
+=item Fixup: I<...>
+
+This directive controls some aspects of the separate program w3mfix.
+w3mfix uses the same configuration file as w3mir since it needs a lot
+of the information in the B<w3mir> configuration file to do it's work
+correctly.  B<w3mfix> is used to make mirrors more browseable on
+filesystems (disk or CDROM), and to fix redirected URLs and some other
+URL editing.  If you want a mirror to be browseable of disk or CDROM
+you almost certainly need to run w3mfix.  In many cases it is not
+necessary when you run a mirror to be used through a WWW server.
+
+To make B<w3mir> write the data files B<w3mfix> needs, and do nothing
+else, simply put
+
+=over 8
+
+  Fixup: on
+
+=back
+
+in the configuration file.  To make B<w3mir> run B<w3mfix>
+automatically after each time B<w3mir> has completed a mirror run
+specify
+
+=over 8
+
+  Fixup: run
+
+=back
+
+L<w3mfix> is documented in a separate man page in a effort to not
+prolong I<this> manpage unnecessarily.
+
+=item Index-name: I<name-of-index-file>
+
+When retriving URLs ending in '/' w3mir needs to append a filename to
+store it localy.  The default value for this is 'index.html' (this is
+the most used, its use originated in the NCSA HTTPD as far as I know).
+Some WWW servers use the filename 'Welcome.html' or 'welcome.html'
+instead (this was the default in the old CERN HTTPD).  And servers
+running on limited OSes frequently use 'index.htm'.  To keep things
+consistent and sane w3mir and the server should use the same name.
+Put
+
+  Index-name: welcome.html
+
+when mirroring from a site that uses that convention.
+
+When doing a multiserver retrival where the servers use two or more
+different names for this you should use B<Apply> rules to make the
+names consistent within the mirror.
+
+When making a mirror for use with a WWW server, the mirror should use
+the same name as the new server for this, to acomplish that
+B<Index-name> should be combined with B<Apply>.
+
+Here is an example of use in the to latter cases when Welcome.html is
+the prefered I<index> name:
+
+  Index-name: Welcome.html
+  Apply: s~/index.html$~/Welcome.html~
+
+Similarly, if index.html is the prefered I<index> name.
+
+  Apply: s~/Welcome.html~/index.html~
+
+I<Index-name> is not needed since index.html is the default index name.
+
+=back
+
+=head1 EXAMPLES
+
+=over 4
+
+=item * Just get the latest Dr-Fun if it has been changed since the last
+time
+
+ w3mir http://sunsite.unc.edu/Dave/Dr-Fun/latest.jpg
+
+=item * Recursively fetch everything on the Star Wars site, remove
+what is no longer at the server from the mirror:
+
+ w3mir -R -r http://www.starwars.com/
+
+=item * Fetch the contents of the Sega site through a proxy, pausing
+for 30 seconds between each document
+
+ w3mir -r -p 30 -P www.foo.org:4321 http://www.sega.com/
+
+=item * Do everything according to F<w3mir.cfg>
+
+ w3mir -cfgfile w3mir.cfg
+
+=item * A simple configuration file
+
+ # Remember, options first, as many as you like, comma separated
+ Options: recurse, remove
+ #
+ # Start here:
+ URL: http://www.starwars.com/
+ #
+ # Speed things up
+ Pause: 0
+ #
+ # Don't get junk
+ Ignore: *.cgi
+ Ignore: *-cgi
+ Ignore: *.map
+ #
+ # Proxy:
+ HTTP-Proxy: www.foo.org:4321
+ #
+ # You _should_ cd away from the directory where the config file is.
+ cd: starwars
+ #
+ # Authentication:
+ Auth-domain: server:port/realm
+ Auth-user: me
+ Auth-passwd: my_password
+ #
+ # You can use '*' in place of server:port and/or realm:
+ Auth-domain: */*
+ Auth-user: otherme
+ Auth-user: otherpassword
+
+=item Also:
+
+ # Retrive all of janl's home pages:
+ Options: recurse
+ #
+ # This is the two argument form of URL:.  It fetches the first into the second
+ URL: http://www.math.uio.no/~janl/ math/janl
+ #
+ # These says that any documents refered to that lives under these places
+ # should be gotten too.  Into the named directories.  Two arguments are
+ # required for 'Also:'.
+ Also: http://www.math.uio.no/drift/personer/ math/drift
+ Also: http://www.ifi.uio.no/~janl/ ifi/janl
+ Also: http://www.mi.uib.no/~nicolai/ math-uib/nicolai
+ #
+ # The options above will result in this directory hierarchy under
+ # where you started w3mir:
+ # w3mir/math/janl		files from http://www.math.uio.no/~janl
+ # w3mir/math/drift		from http://www.math.uio.no/drift/personer/
+ # w3mir/ifi/janl		from http://www.ifi.uio.no/~janl/
+ # w3mir/math-uib/nicolai	from http://www.mi.uib.no/~nicolai/
+
+=item Ignore-RE and Fetch-RE
+
+ # Get only jpeg/jpg files, no gifs
+ Fetch-RE: m/\.jp(e)?g$/
+ Ignore-RE: m/\.gif$/
+
+=item Apply
+
+As I said earlier, B<Apply> has not been used for Real Work yet, that
+I know of.  But B<Apply> I<could>, be used to map all web servers at
+the university of Oslo inside the scope of retrieval very easily:
+
+  # Start at the main server
+  URL: http://www.uio.no/
+  # Change http://*.uio.no and http://129.240.* to be a subdirectory
+  # of http://www.uio.no/.
+  Apply: s~^http://(.*\.uio\.no(?:\d+)?)/~http://www.uio.no/$1/~i
+  Apply: s~^http://(129\.240\.[^:]*(?:\d+)?)/~http://www.uio.no/$1/~i
+
+
+=back
+
+There are two rather extensive example files in the B<w3mir> distribution.
+
+=head1 BUGS
+
+=over 4
+
+=item The -lc switch does not work too well.
+
+=back
+
+=head1 FEATURES
+
+These are not bugs.
+
+=over 4
+
+=item URLs with two /es ('//') in the path component does not work as
+some might expect.  According to my reading of the URL spec. it is an
+illegal construct, which is a Good Thing, because I don't know how to
+handle it if it's legal.
+
+=item If you start at http://foo/bar/ then index.html might be gotten
+twice.
+
+=item Some documents point to a point above the server root, i.e.,
+http://some.server/../stuff.html.  Netscape, and other browsers, in
+defiance of the URL standard documents will change the URL to
+http://some.server/stuff.html.  W3mir will not.
+
+=item Authentication is I<only> tried if the server requests it.  This
+might lead to a lot of extra connections going up and down, but that's
+the way it's gotta work for now.
+
+=back
+
+=head1 SEE ALSO
+
+L<w3mfix>
+
+=head1 AUTHORS
+
+B<w3mir>s authors can be reached at I<w3mir-core@usit.uio.no>.
+B<w3mir>s home page is at http://www.math.uio.no/~janl/w3mir/
Index: /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3pdfuri.pm
===================================================================
--- /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3pdfuri.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/w3mir/w3mir-1.0.8/w3pdfuri.pm	(revision 21112)
@@ -0,0 +1,64 @@
+# -*- perl -*-
+#
+# w3pdfuri.pm -- extract uris from a adobe acrobat pdf file.
+#     version 0.1
+$VERSION=0.1;
+#
+# Assumptions:
+# - File starts with %PDF
+# - All URLS match this RE: ^\/URI\s+\(([^)]*)\)\s*$  (found by inspection of
+#   one (1) pdf file (the reader.pdf in the acrobat distribution).
+# - pdf files are potentially very large, so we read it from disk
+#   record by record.
+# - Record separator is ^M
+#
+# History:
+# - 19/02/97 janl: Version 0.1, seems to work with PDF-1.2
+#
+
+package w3pdfuri;
+
+use strict;
+
+sub list ($) {
+  my ($file) = @_;
+
+  my @urls=();
+  local($/)="\r";
+
+  unless (open(PDF,"< $file")) {
+    warn "Could not open $file for input: $!\n";
+    return;
+  }
+
+  $_ = <PDF>;
+  unless (/^%PDF-/) {
+    warn "$file is not a PDF file.\n";
+    close(PDF);
+    return;
+  }
+  
+  while (<PDF>) {
+    chomp;
+    push(@urls,$1) if (m~^/URI\s+\(([^)]*)\)\s*$~);
+  }
+
+  close(PDF);
+
+  return @urls;
+}
+
+1;
+
+__END__
+
+# Test code
+print "reader.pdf: ",
+  join(",",list("/local/lib/acrobat/Reader/help/reader.pdf")),"\n";
+print "Acrobat.pdf: ",
+  join(",",list("/local/lib/acrobat/Reader/Acrobat.pdf")),"\n";
+print "License.pdf: ",
+  join(",",list("/local/lib/acrobat/Reader/License.pdf")),"\n";
+print "MapTypes.pdf: ",
+  join(",",list("/local/lib/acrobat/Reader/MapTypes.pdf")),"\n";
+list("/etc/services");
Index: /main/tags/3.01/gsdl/packages/wget/README
===================================================================
--- /main/tags/3.01/gsdl/packages/wget/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/wget/README	(revision 21112)
@@ -0,0 +1,19 @@
+wget - web site mirroring software
+
+This directory contains a version of the GNU wget program that was written by
+Hrvoje Niksic and others and distributed under the GNU General Public Licence
+
+The GNU wget homepage is http://www.cg.tuwien.ac.at/~prikryl/wget.html
+
+A couple of very small changes were made to this package by Stefan Boddie
+(sjboddie@cs.waikato.ac.nz). The USE_STDARG preprocessor definition was
+added to the VC++ makefile as it appeared to need it to compile on either
+VC++ 4.2 or VC++ 6.0. A change was also made to prevent backslashes from
+mistakenly being converted to "@5c" on windows.
+
+2003/11/27 - sjboddie@cs.waikato.ac.nz made a few more small changes to fix
+h_errno bug preventing code from compiling on latest versions of GCC.
+
+2004/06/30 - kjdon@cs.waikato.ac.nz replaced version 1.5.3 with version 1.9.
+It compiled as is under Windows XP, using VC++ 6.0, under Linux using GCC 3.2.3 and GCC 2.95.3. So I haven't made any changes. 
+Under Windows, I haven't compiled it with openssl, so it doesn't support https://. OpenSSL comes with a warning about it being illegal in some countries to distribute cryptographic technology, so I haven't used it. Look at wget-1.9/windows/README for Windows compiling instructions if you want to add it in.
Index: /main/tags/3.01/gsdl/packages/windows/crypt/README.TXT
===================================================================
--- /main/tags/3.01/gsdl/packages/windows/crypt/README.TXT	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/windows/crypt/README.TXT	(revision 21112)
@@ -0,0 +1,7 @@
+This directory contains a win32 port of the crypt library (see the enclosed
+README files for details). To compile with Microsoft Visual C++ unzip
+crypt.zip and run "nmake /f win32.mak" in the top level directory.
+
+Link crypt.lib with your project when compiling Greenstone.
+
+Stefan Boddie (sjboddie@cs.waikato.ac.nz)
Index: /main/tags/3.01/gsdl/packages/windows/gdbm/README.TXT
===================================================================
--- /main/tags/3.01/gsdl/packages/windows/gdbm/README.TXT	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/windows/gdbm/README.TXT	(revision 21112)
@@ -0,0 +1,7 @@
+This directory contains a win32 port of gdbm. To compile with Microsoft
+Visual C++ unzip gdbm.zip and run "nmake /f win32.mak" in the top level
+directory.
+
+Link gdbm.lib with your project when compiling Greenstone.
+
+Stefan Boddie (sjboddie@cs.waikato.ac.nz)
Index: /main/tags/3.01/gsdl/packages/wv/README
===================================================================
--- /main/tags/3.01/gsdl/packages/wv/README	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/wv/README	(revision 21112)
@@ -0,0 +1,16 @@
+wv - Microsoft Word converter
+
+This directory contains a version of the wv program.  It was originally
+written by Caolan McNamara and distributed under the GNU General Public
+License.  wv is now maintained by Dom Lachowicz and Martin Vermeer.  wv
+incorporates code from a variety of GPL programs from other sources.
+Thank you all.
+
+The wv homepage is http://wvware.sourceforge.net/
+
+We have included the source code here.  It is a slimmed-down version
+of wv version 0.7.1. It should handle documents created with Word 2 up
+to Word 2000. We have tested it on Linux (slackware 7), darwin (OS X
+10.1.3), cygwin, and solaris 2.8 (via sourceforge's compile farm).
+
+John McPherson -- Apr 2002
Index: /main/tags/3.01/gsdl/packages/xlhtml/README.txt
===================================================================
--- /main/tags/3.01/gsdl/packages/xlhtml/README.txt	(revision 21112)
+++ /main/tags/3.01/gsdl/packages/xlhtml/README.txt	(revision 21112)
@@ -0,0 +1,19 @@
+This package contains two programs:
+* ppthtml, which converts Microsoft PowerPoint 95 and 97 files
+into html or xml, and
+* xlhtml, which converts Microsoft Excel 95 and 97 files.
+
+It has been slightly modified for greenstone use as we only want the
+ppt to html program and the xls to html program, and none of the other
+programs (summary/diagnostics, man pages, etc).
+
+This version (0.4.9.0) was the latest development snapshot at the
+time of writing.
+
+See http://www.xlhtml.org/ for newer versions or more info.
+
+xlhtml is released under the GNU GPL.
+
+20 Feb 2002
+John McPherson
+NZDL.
Index: /main/tags/3.01/gsdl/perllib/DateExtract.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/DateExtract.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/DateExtract.pm	(revision 21112)
@@ -0,0 +1,275 @@
+package DateExtract;
+
+##use BasPlug; ## no, DON'T use BasPlug, BasPlug uses us....
+use sorttools;
+use util;
+
+#75% of the instances of the word century use full name ordinals
+my %ordinals = ("first" => 1, "second" => 2, "third" => 3, "fourth" => 4,
+             "fifth" => 5, "sixth" => 6, "seventh" => 7, "eighth" => 8,
+             "ninth" => 9, "tenth" => 10, "eleventh" => 11, "twelfth" => 12,
+             "thirteenth" => 13, "fourteenth" => 14, "fifteenth" => 15,
+             "sixteenth" => 16, "seventeenth" => 17, "eighteenth" => 18,
+             "nineteenth" => 19, "twentieth" => 20);
+
+
+             
+
+#definitions for a date grammar.
+my $fulOrd = join('|',(keys %ordinals));
+
+my @months = ("january","february","march","april","may","june","july",
+              "august","september","october","november","december");
+
+my $shortmth = "";
+foreach $m (@months) { $shortmth .= (substr($m,0,3)."\\.?|"); }
+chop($shortmth);
+
+my $longmth = join('|',@months);
+
+
+my $Qualifier = "(B(\\.)?C(\\.)?(E(\\.)?)?)|(A(\\.)?D(\\.)?)|(C(\\.)E(\\.)?)";
+my $Century = "Cent(\\.|ur(y|ies))";
+my $Ord = "st|nd|rd|th";
+my $seasonref = "(spring|fall|autumn|winter|summer)";
+my $sep = " ?- ?";
+my $tri_digit = "(in|since|(the year)|($seasonref of)) \\d{3}\\D( ($Qualifier))?";
+my $centurydate = "(((\\d{1,2})($Ord))|($fulOrd)) ($Century)( ($Qualifier))?";
+my $qualified = "(($Qualifier) ?(\\d{1,4}))|((\\d{1,4}) ?($Qualifier))";
+my $millenium = "[1-9]\\d{3}";
+my $range = "(($millenium)($sep)($millenium))|(($millenium)($sep)(\\d{1,2}))|(($qualified)($sep)($qualified))|(($Qualifier) ?(\\d{1,3})($sep)(\\d{1,3}))|((\\d{1,3})($sep)(\\d{1,3}) ?($Qualifier))";
+
+my $pgnum = "(p(p|g)?\\.? ?\\d+((-|,)(\\d+))?)|(page \\d+)|(pages \\d+((-|,)(\\d+))?)";
+my $lgnum = "\\d{1,3},(\\d{3})+";
+my $colon = ":\\d+";
+my $money = "\$(\\d{1,3}(((,\\d{3})+)|\\d+))";  $money = "\\" . $money; 
+my $microfilm = " reel([^\\.\\)])*(\\.|\\))";
+my $lastaltered = "last (edited|updated)";
+my $references = "reference(s?)(:?)\\n";
+my $cited = "work(s?) cited";
+my $biblio = "bibliography(:?)( ?)\\n";
+
+my $direction = "(N(\\.|o\\.|th\\.|orth))|(S(\\.|th\\.|outh))|(E(\\.|ast))|(W(\\.|est))";
+my $street_id = "(st\\.)|(street[^A-Za-z])|(ave\\.)|(avenue[^A-Za-z])|(boulevard[^A-Za-z])|(blvd\\.)|(rd\\.)|(road[^A-Za-z])";
+my $streetname = "(\\d{1,2}(<SUP>)?($Ord)(</SUP>)?)|([A-Za-z]+( [A-Za-z]+)?)|([A-Za-z]+-[A-Za-z]+($Ord))";
+my $address = "\\d{1,4} (($direction) )?($streetname) ($street_id)";
+
+my $bracket = "\\($millenium\\)";
+my $ref_end = ", ?$millenium\\)";
+my $colonsp = ": $millenium";
+my $reprint = "[Rr]eprint of \\d{4} edition";
+my $comma = ", $millenium\\.";
+my $fullstop = "\\. $millenium\\.";
+my $semi = "; $millenium\\.";
+
+my $lookalikes = "($pgnum)|($lgnum)|($colon)|($money)|($microfilm)|($address)";
+my $spurious = "($lastaltered)";
+my $bibheader = "($references)|($cited)|($biblio)";
+
+
+sub get_date_metadata {
+    #get the text of the document, the "document object" concerned,
+    #and the current section within the document
+    local ($text, $doc, $cursection, $keep_bib, $max_year, $max_century) = @_;
+    
+    #format a prechristian maximum century value to be negative so that it can
+    #be used in numeric comparison
+    if($max_century =~ /B/)
+    {
+        $max_century = $`;
+        $max_century =~ /\d+/;
+        $max_century = $&;
+        $max_century *=-1
+    }
+   
+    local $extr = &remove_excess($text);    
+    #print "EXTRACTION TEXT:\n $extr";
+    $extr = &remove_tags($extr);
+    if(!$keep_bib){
+        $extr = &remove_biblio($extr);
+    }
+  
+
+    local @datelist = ();
+    while($extr =~ m!($range)|($millenium)|($qualified)|($centurydate)|($tri_digit)!i)
+    { 
+        $extr = $';
+        my $fulldate = $&;
+        if ($fulldate =~ /$centurydate/i)
+        {
+            if($max_century!=-1)
+            {
+
+                local $date = $fulldate; if($date =~ /\d+/) {$date = $&;} 
+                else 
+                {
+                    $date=$fulldate; $date =~ m! ($Century)!i; $date = $`;
+                    $date =~ tr/A-Z/a-z/;
+                    $date = $ordinals{$date};
+                }
+                if($max_century >= $date){
+                    $date = ($date-1)*100 +1;
+                    #if it BC, make it negative
+                    $date = &convert_bc($fulldate,$date);
+                    $end = $date + 99; 
+                    @century = ($date..$end);
+                    @datelist = (@datelist,@century);
+                }
+            }
+        }
+        
+        elsif($fulldate =~ /$range/)
+        {
+            $fulldate =~ /$sep/;
+            my @addlist = ();
+            #print "Range: $fulldate\n";
+            $fullfirst = $`; 
+            $fullsecond = $'; 
+            $fullfirst =~ /\d+/; $first = $&; 
+            $fullsecond =~ /\d+/; $second = $&;
+            $len1 = length($first);
+            $len2 = length($second);
+            $second = (substr($first,0,($len1-$len2))).$second;
+            $first = &convert_bc($fullfirst,$first);
+            $second = &convert_bc($fullsecond,$second);
+            @addlist = ($first..$second);
+            @datelist = (@datelist,@addlist);
+            
+        }
+        else {
+            
+            my $date = $fulldate; $date =~ /\d+/; $date = $&;  
+            $date = &convert_bc($fulldate,$date);
+            #add the date metadata
+            push(@datelist,$date);
+            #print "datelist @datelist\n"
+        }
+        
+    }
+    
+    if(@datelist){
+        @datelist = sort { $a <=> $b } @datelist;
+        @datelist = &post_process($max_year, @datelist);
+        foreach $date (@datelist)
+        {
+            if($date>0){
+                $doc->add_metadata($cursection,"Coverage",$date);}
+            else{
+                $doc->add_metadata($cursection,"Coverage","bc".(-1*$date));}
+                
+        }
+    }
+}
+sub convert_bc {
+    local ($full,$num) = @_;
+    if ($full =~ /B/) { $num *= -1; }
+    $num;
+}
+
+sub post_process {
+    local ($max_year, @list) = @_;
+    @cleanlist = ();
+    $prev = 0;
+    foreach $e (@list) {
+        if ($e!=$prev && $e <= $max_year) {
+            push(@cleanlist, $e);
+        }
+        $prev = $e;
+    }
+    @cleanlist;
+}
+
+
+#removes all html tags from that data, as they will not contain dates which
+#are part of the content of the document, and therefore interesting, but do
+#contain date lookalikes
+sub remove_tags {
+    local ($tmp) = @_;
+   
+    local $parsed = "";
+    #while there is still text to be parsed and tags are still found
+    while($tmp=~ m!<([^>])*(>|$)! && $tmp ne "")
+    {
+        $parsed .= $`;#keep all that is not in a tag
+        $tmp = $';    #restart the search after then end of the tag
+    }
+    $parsed .= $tmp; #add anything after the last match
+    $parsed;
+}
+
+
+sub remove_excess {
+    local ($tmp) = @_;
+    local $parsed = "";
+   
+
+    if(($tmp =~ m!($spurious)|($lookalikes)!i) == 0 )
+    { 
+        $parsed = $tmp;
+    }
+    else { 
+        while ($tmp =~ m!($spurious)|($lookalikes)!i 
+               && $tmp ne "")
+        {
+            $parsed .= $`;
+            $storage = $&;
+            $tmp = $';
+            #match the pattern which indicates most recent alteration 
+            if ($storage =~ m!$lastaltered!i)
+            {
+                #match a four digit year or up until the first / 
+                #(as in last edited 3/97). 
+                $tmp =~ m!($millenium)|(\/)!;
+                $tmp = $';
+            }
+                
+        }
+        
+        $parsed .= $tmp;
+           
+    }
+    #print "Parsed:\n $parsed\n\n";
+    $parsed;
+    
+}
+
+sub remove_biblio{
+    local ($tmp) = @_;
+    local $parsed = "";
+    
+    if($tmp =~ m!$bibheader!i)
+    {
+        $tmp=$`;
+    }
+    
+    $tmp =~ s/( |\t)+/ /g;
+    if(($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|($seasonref) ($millenium)!i) == 0)
+    {
+        $parsed = $tmp;
+    }
+    else{
+
+        #print "removing bib\n";
+        while ($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|(($seasonref) ($millenium))|($bibheader)!i && $tmp ne "")
+        {
+            
+            $parsed .= $`;
+            $tmp = $';
+            if($&=~m!($comma)|($fullstop)!)
+            {
+                
+                local  $date = $&;
+                if($parsed =~ m!((\d)($Ord)$)|(($shortmth)$)|(($longmth)$)!i)
+                {
+                    $parsed .= $date;
+                }
+            }  
+            
+        }
+                   $parsed .= $tmp;
+    }
+    $parsed;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/Kea.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/Kea.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/Kea.pm	(revision 21112)
@@ -0,0 +1,124 @@
+package Kea;
+
+# This function is called by BasPlug.pm when a flag in a collection
+# configuration document specifies that keyphrase metadata must be gathered for
+# that collection.
+# It is passed as arguments, the documents text and possibly some options for
+# how the keyphrase data is to be collected if the keyphrase option flag was
+# set in the collection configuration file.  This module then writes the
+# documents text to a file in a temporary directory because the stand-alone program Kea which will be
+# called to do the actual extraction of the keyphrases expects a directory with one or more files as argument.
+# Once Kea has been called upon, the file containing the keyphrase data
+# gathered by Kea should be stored in gsdl/tmp and this file is read, the data
+# we are interested in is extracted and passed back to BasPlug.pm in an
+# appropriate format.
+
+# returns a string containing comma-separated keyphrases
+sub extract_KeyPhrases {
+
+    # Parsing arguments of the function
+    my $doc = shift(@_); # documents text  
+    my $args = shift(@_); # any options
+    my @optionlist = split(/\s+/, $args) if (defined($args)); #list of options
+
+    # Specifying directory names
+    my $gsdlhome = $ENV{'GSDLHOME'}; 
+    my $keahome = "$gsdlhome/packages/kea/kea-3.0";
+    my $defaultmodel = "$keahome/CSTR";
+    
+    # Initializing variables:
+    my $command = ""; 
+    my @keylist; 
+    my @options = ();
+    $modelspec = 0;
+
+    # Settings for the java executable: 
+
+    # CLASSPATH:
+    $java_classpath = ".:$keahome";
+
+    # See if java executable is on path
+    my $java_exec="";
+    if (system("which java >/dev/null 2>/dev/null")==0) {
+	$java_exec=`which java`;
+	chomp $java_exec;
+    } else {
+	$java_exec="$java_home/bin/java";
+    }
+    
+    # The actual java command is based on these other variables:
+    $java_command = "$java_exec -classpath \"$java_classpath\"";
+    
+    # end of java settings
+
+    # Parsing options for keyphrase extraction:
+    if (@optionlist) {
+	foreach $element (@optionlist){ #for each option
+	    if (length($element) == 1) {
+		push(@options, "-$element");
+	    } else {
+		$option = substr($element, 0, 1);
+		$value = substr($element,1);
+		if (($option eq "m") && (-e "$keahome/$value")) {
+		    $modelspec = 1;
+		    push(@options, "-$option $keahome/$value");
+		} elsif ($option eq "m") {		    
+		    $modelspec = 1;
+		    print STDERR "Couldn't find model $value. Using the default model instead\n";
+		    push(@options, "-$option $defaultmodel");
+		} else {
+		    push(@options, "-$option $value");
+		}
+		
+	    }
+	}
+	# if none of the option specifies the model, set the default one:
+	if ($modelspec != 1) {
+	    push(@options, "-m $defaultmodel");	
+	}
+	$options = join(" ",@options);
+	# print STDERR "OPTIONS: $options\n";
+    } else {
+	# If no options were specified: Set default value for the model
+	$options = "-m $defaultmodel";	
+    }
+
+    # Remove all HTML tags from the original text
+    $doc =~ s/<P[^>]*>/\n/sgi;
+    $doc =~ s/<H[^>]*>/\n/sgi;
+    $doc =~ s/<[^>]*>//sgi;
+    $doc =~ tr/\n/\n/s;
+
+    # Write text to a temporary file doc.txt 
+    open(OUT, ">$gsdlhome/tmp/doc.txt") or die "In Kea.pm doc.txt could not be created\n";  
+    print OUT $doc;
+    close(OUT);
+
+
+    # EXECUTE KEA with specific options:
+    $command = "$java_command KEAKeyphraseExtractor -l $gsdlhome/tmp $options";
+    system ("$command");
+
+    # Read the resulting doc.key, which contains keyphrases:
+
+    open(IN, "<$gsdlhome/tmp/doc.key") or return "";
+                                          #this means doc.key does not exist
+                                          #either because an option was wrongly specified
+	                                  #or no keyphrases were found
+    while(<IN>){
+	chomp;
+	push(@keylist,$_);
+    }
+    close(IN);
+
+    $keylist = join(", ", @keylist);
+
+    # Delete doc.key so that in future it will not be opened and read.
+    # Otherwise KEA sees it as more keyphrases!
+
+    unlink("$gsdlhome/tmp/doc.key");
+	
+    return $keylist;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/XMLParser.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/XMLParser.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/XMLParser.pm	(revision 21112)
@@ -0,0 +1,33 @@
+# Wrapper that ensures the right version of XML::Parser is loaded given
+# the version of Perl being used.  Need to distinguish between Perl 5.6 and
+# Perl 5.8
+
+
+sub BEGIN {
+    my $perl_dir;
+
+    # Note: $] encodes the version number of perl
+    if ($]>5.008) { 
+	# perl 5.8.1 or above
+	$perl_dir = "perl-5.8";
+    }
+    elsif ($]<5.008) {
+	# assume perl 5.6
+	$perl_dir = "perl-5.6";
+    }
+    else {
+	print STDERR "Warning: Perl 5.8.0 is not a maintained release.\n";
+	print STDERR "         Please upgrade to a newer version of Perl.\n";
+	$perl_dir = "perl-5.8";
+    }
+
+    if ($ENV{'GSDLOS'} !~ /^windows$/i) {
+	# Use push to put this on the end, so an existing XML::Parser will be used by default
+	push (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir");
+    }
+}
+
+use XML::Parser;
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/acronym.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/acronym.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/acronym.pm	(revision 21112)
@@ -0,0 +1,697 @@
+##########################################################################
+#
+# acronym.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+#    class to handle acronyms
+###########################################################################
+
+eval "require diagnostics"; # some perl distros (eg mac) don't have this
+
+package acronym;
+
+###########################################################################
+#    global variables
+###########################################################################
+# valiables to control the recall/precision tradeoff
+
+#the maximum range to look for acronyms 
+my $max_offset = 30;
+#acronyms must be upper case
+my $upper_case = 1;
+#acronym case must match
+my $case_match = 1;
+#minimum acronym length
+my $min_def_length = 3;
+#minimum acronym length
+my $min_acro_length = 3;
+#minimum acronym length saving
+my $min_length_saving = 4;
+#allow recusive acronyms
+my $allow_recursive = 0;
+#let definitions be all capitals
+my $allow_all_caps = 0;
+
+my @stop_words = split / /, "OF AT THE IN TO AND";
+
+#the text split into an array, one word per element
+my @split_text = ();
+my @acronym_list = ();
+
+#the file to collate acronyms into
+my $acronym_accumulate_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_definitions.pm";
+my $acronym_options_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_options.pm";
+
+my %acronyms_found_in_collection = ();
+my %acronyms_banned_from_collection = ();
+
+my $writing_acronyms = 1;
+my $accumulate_acronyms = 1;
+my $markup_accumulate_acronyms = 1;
+my $markup_local_acronyms = 1;
+
+
+
+###########################################################################
+#   file saving / loading stuff
+###########################################################################
+
+sub initialise_acronyms {
+
+    my $local_max_offset = $max_offset;
+    my $local_upper_case = $upper_case;
+    my $local_case_match = $case_match ;
+    my $local_min_def_length = $min_def_length;
+    my $local_min_acro_length = $min_acro_length;
+    my $local_min_length_saving = $min_length_saving;
+    my $local_allow_recursive = $allow_recursive;
+    my $local_allow_all_caps = $allow_all_caps;
+    my @local_stop_words = @stop_words;
+    my $local_acronym_accumulate_file = $acronym_accumulate_file;
+    
+    
+    my $file_text = "";
+    if (open ACRONYM_HANDLE, "<$acronym_options_file")
+    {
+	$file_text = do { local $/; <ACRONYM_HANDLE> };  
+    }
+    if ($file_text eq "")
+    {
+	print STDERR "failed to open $acronym_options_file\n";
+	open ACRONYM_HANDLE, ">$acronym_options_file\n";
+	print ACRONYM_HANDLE "#Config file for acronym extraction. EDIT THIS FILE, it should\n";
+	print ACRONYM_HANDLE "#not be overridden by the software. It's read by GSDL using perl's\n";
+	print ACRONYM_HANDLE "#'eval' function, so pretty much anything that's valid in perl is \n";
+	print ACRONYM_HANDLE "#valid here.\n\n";
+	print ACRONYM_HANDLE "#Quite a few things here are defined in terms of recall and precision\n";
+	print ACRONYM_HANDLE "#which are the key measures from Information Retreval (IR). If you\n";
+	print ACRONYM_HANDLE "#don't understand recall and precision, any good IR textbook should\n";
+	print ACRONYM_HANDLE "#explain them fully \n\n";
+	print ACRONYM_HANDLE "#the maximum range to look for acronyms (raise to raise precision)\n"; 
+	print ACRONYM_HANDLE "\$local_max_offset = $max_offset;\n\n"; 
+	print ACRONYM_HANDLE "#acronyms must be upper case (0 = false, 1 = true (high precision))\n";
+	print ACRONYM_HANDLE "\$local_upper_case = $upper_case;\n\n";
+	print ACRONYM_HANDLE "#acronym case must match (0 = false, 1 = true (high precision))\n";
+	print ACRONYM_HANDLE "\$local_case_match = $case_match;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$local_min_def_length = $min_def_length;\n\n";
+	print ACRONYM_HANDLE "#let definitions be all capitals\n";
+	print ACRONYM_HANDLE "\$local_allow_all_caps = $allow_all_caps;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$local_min_acro_length = 3;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length saving (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$local_min_length_saving = 4;\n\n";
+	print ACRONYM_HANDLE "#allow recusive acronyms (0 = false (high precision), 1 = true)\n";
+	print ACRONYM_HANDLE "\$local_allow_recursive = 0;\n\n";
+	print ACRONYM_HANDLE "#stop words-words allowed in acronyms (the multi-lingual version\n";
+	print ACRONYM_HANDLE "#slows down acronym extraction slightly so is not the default)\n";
+	print ACRONYM_HANDLE "#\@local_stop_words = split / /, \"A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND DE DU A LA LE LES L DANS ET S\";\n";
+	print ACRONYM_HANDLE "\@local_stop_words = split / /, \"OF AT THE IN TO AND\";\n"; 
+	print ACRONYM_HANDLE "\n"; 
+	print ACRONYM_HANDLE "#the file to collate acronyms into\n";
+	print ACRONYM_HANDLE "\$local_acronym_accumulate_file = \$ENV{'GSDLCOLLECTDIR'} . \"/etc/acronym_definitions.pm\";\n";
+	print ACRONYM_HANDLE "\n";
+	print ACRONYM_HANDLE "# any acronym definitions which should always be marked up can be copied here\n";
+	print ACRONYM_HANDLE "# from the acronym_accumulate_file file ...\n";
+	print ACRONYM_HANDLE "# \n";
+	print ACRONYM_HANDLE "# \n";
+	print ACRONYM_HANDLE "# \n";
+	print STDERR "written new options file to $acronym_options_file...\n";
+    } else {
+	print STDERR "read file $acronym_options_file...\n";
+	eval $file_text ;
+	warn $@ if $@;
+	print STDERR "evaluated file $acronym_options_file...\n";
+    }
+
+    $max_offset = $local_max_offset;
+    $upper_case = $local_upper_case;
+    $case_match = $local_case_match ;
+    $min_def_length = $local_min_def_length;
+    $min_acro_length = $local_min_acro_length;
+    $min_length_saving = $local_min_length_saving;
+    $allow_recursive = $local_allow_recursive;
+    $allow_all_caps = $local_allow_all_caps;
+    @stop_words = @local_stop_words;
+    
+    $local_acronym_accumulate_file = $local_acronym_accumulate_file;
+    
+
+    &read_all_acronyms_from_file();
+#    rename $acronym_file, $acronym_file . "." . int(rand (2<<7)). 
+#	int(rand (2<<7)). int(rand (2<<7)). int(rand (2<<7));
+    if ($writing_acronyms && open ACRONYM_HANDLE, ">$acronym_accumulate_file")
+    {
+	print ACRONYM_HANDLE "#This is an automatically generated file.\n";
+	print ACRONYM_HANDLE "#\n";
+	print ACRONYM_HANDLE "#If you edit this file and it will be overwritten the next\n";
+	print ACRONYM_HANDLE "#time the acronym code runs unless you set the file to \n";
+	print ACRONYM_HANDLE "#read-only. \n";
+	print ACRONYM_HANDLE "#\n";
+	print ACRONYM_HANDLE "#start of acronyms...\n";
+	$writing_acronyms = 1;
+    } else {
+	warn "failed to open $acronym_accumulate_file for writing\n";
+	$writing_acronyms = 0;
+    }
+}
+
+#close the list of accumulated acronyms
+sub finalise_acronyms {
+    if ($writing_acronyms)
+    {
+	print ACRONYM_HANDLE "#end of acronyms.\n";  
+	close ACRONYM_HANDLE;
+    }
+}
+
+#eval a file of accumulated acronyms
+sub read_all_acronyms_from_file {
+    
+    my $file_text = " ";
+    if (open ACRONYM_HANDLE, "<$acronym_accumulate_file")
+    {
+	$file_text = do { local $/; <ACRONYM_HANDLE> };  
+    } else {
+	print STDERR "failed to open $acronym_accumulate_file for reading (this is the first pass?).\n";
+    }
+    eval $file_text;
+    #promotes warnings/errors from evaluated file to current context 
+    warn $@ if $@;
+}
+
+#called from within the file of accumulated acronyms to indicate a good acronym
+sub add {
+    my $self = shift (@_);
+    if (defined ($acronyms_found_in_collection{$self->[0]}))
+    {
+	my $def = $self->to_def_string();
+	if ($acronyms_found_in_collection{$self->[0]} =~ m/(^|\|)$def(\||$)/)
+	{
+	    return;
+	}
+	$acronyms_found_in_collection{$self->[0]} = 
+	    $acronyms_found_in_collection{$self->[0]} . "|" . $self->to_def_string();
+    } else {
+	$acronyms_found_in_collection{$self->[0]} = $self->to_def_string();
+    }
+}
+
+#called from within the file of accumulated acronyms to indicate a bad acronym
+sub ban {
+    my $self = shift (@_);
+    
+    if (!defined $acronyms_banned_from_collection{$self->[0]})
+    {
+	$acronyms_banned_from_collection{$self->[0]} = $self->to_def_string();
+    } else {
+	$acronyms_banned_from_collection{$self->[0]} = $acronyms_banned_from_collection{$self->[0]} . "|" . $self->to_def_string();
+    }
+}
+
+
+#write a good acronym to the accumulated acronyms file
+sub write_to_file {
+    my $self = shift (@_);
+    if ($writing_acronyms)
+    {
+	print ACRONYM_HANDLE "new acronym(\"$self->[0]\",\"" . 
+	    $self->to_def_string() . 
+		"\")->add();\n";
+    }
+}
+
+
+###########################################################################
+# mark functionality    
+###########################################################################
+
+#small routine to sort by length 
+sub sort_by_length {
+    length($b) <=> length($a) or $a cmp $b
+}
+
+sub markup_acronyms {
+    my  $text = shift (@_);
+    my  $verbosity_obj = shift (@_);
+    if (defined $text)
+    {
+	for my $acro (sort sort_by_length keys %acronyms_found_in_collection)
+	{
+	    $text  =~ s/^((?:[^\<\n]|(?:\<[^\>\n]*\>))*)$acro([^\<A-Z])/$1$acro\<img src=\"\" width=8 height=8 alt=\"$acronyms_found_in_collection{$acro}\"\>$2/gm;
+	    printf STDERR " " .  $acro . ","
+		if ($verbosity_obj->{'verbosity'} >= 2);
+	}
+    }
+    return $text;
+}
+
+
+
+###########################################################################
+#    member functions 
+###########################################################################
+
+
+sub new {
+    my $trash = shift (@_); 
+    my $acro = shift (@_); 
+    my $def  = shift (@_); 
+    
+    my $self = [
+		"", # 0 acronym
+		[], # 1 definition
+               ];
+    
+    $self->[0] = $acro                    if defined $acro;
+    push @{$self->[1]},  split / /, $def  if defined $def;
+    
+    bless $self;
+}
+
+sub clone  {
+    my $self = shift (@_);
+
+    my $copy = new acronym;
+
+    $copy->[0] = $self->[0];
+    push @{$copy->[1]}, @{$self->[1]};
+    bless $copy;
+
+    return $copy;
+}
+
+#return the acronym
+sub to_acronym {
+    my $self = shift (@_);
+    my @array = @{$self->[1]};
+
+    return $self->[0];
+}
+
+#return the number of words in the acronym definition
+sub words_in_acronym_definition {
+    my $self = shift (@_);
+    my @array = @{$self->[1]};
+
+    return $#array + 1;
+}
+
+#return the number of letters in the acronym definition
+sub letters_in_acronym_definition {
+    my $self = shift (@_);
+
+    return length($self->to_def_string());
+}
+
+#return the number of letters in the acronym definition
+sub letters_in_acronym {
+    my $self = shift (@_);
+
+    return length($self->to_acronym());
+}
+
+#return the acronym definition
+sub to_def_string {
+    my $self = shift (@_);
+
+    my $result = "";
+
+    # do the definition
+    my @array = @{$self->[1]};
+    my $i = 0;
+    while ($i <= $#array)
+    {
+	$result = $result . $array[$i];
+
+	if ($i+1 <= $#array)
+	{
+	    $result = $result . " ";
+	}
+	$i++;
+    }
+    return $result;
+}
+
+
+#print out the kwic for the acronym
+sub to_string_kwic {
+    my $self = shift (@_);
+
+    # the list of all possible combinations
+    my @list = ();
+
+    my $result = "";
+
+    my $j = 0;
+    my @array = @{$self->[1]};
+    while ($j <= $#array)
+    {
+
+	# do the definition
+	my $i = 0;
+
+	#add the key word
+	$result = "<td halign=left>"  . $array[$j] . "</td><td halign=right>";
+
+	#add the proceeding words
+	while ($i < $j)
+	{
+	    $result = $result .  $array[$i] . " ";
+	    $i++;
+	}
+	#add the key word
+	$result = $result . "</td><td halign=left>"  . $array[$j] . 
+	    "</td><td halign=left>";
+
+	#add the trailing words
+	$i++;
+	while ($i <= $#array )
+	{
+	    $result = $result .  $array[$i] . " ";
+	    $i++;
+	}
+
+	#add the actual acronym
+
+	$result = $result . "</td><td halign=left>" . $self->[0] . "</td>";
+
+	push @list, $result;
+	$j++;
+    }
+    return @list;
+}
+
+#this is the one used when building the collection ...
+sub to_string {
+    my $self = shift (@_);
+
+    my $result = $self->[0] . " ";
+
+    # do the definition
+    my @array = @{$self->[1]};
+    my $i = 0;
+    while ($i <= $#array)
+    {
+	$result = $result . $array[$i];
+	if ($i+1 <= $#array)
+	{
+	    $result = $result . " ";
+	}
+	$i++;
+    }
+    return $result;
+}
+
+sub check {
+    my $self = shift (@_);
+
+    if (length($self->to_acronym()) < $min_acro_length) 
+    {
+#	print "acronym " . $self->to_string() . " rejected (too short I)\n";
+	return 0;
+    }
+    if ($self->words_in_acronym_definition() < $min_def_length) 
+    {
+#	print "acronym " . $self->to_string() . " rejected (too short II)\n";
+	return 0;
+    }
+    if ($min_length_saving * $self->letters_in_acronym() > 
+	$self->letters_in_acronym_definition()) 
+    {
+#	print "acronym " . $self->to_string() . " rejected (too short III)\n";
+#	print "" . $min_length_saving .
+#	    "|" . $self->letters_in_acronym() .
+#	    "|" . $self->letters_in_acronym_definition() . "\n";
+	return 0;
+    }
+    if (!$allow_all_caps && 
+	$self->to_def_string() eq uc($self->to_def_string()))
+    {
+#	print "acronym " . $self->to_string() . " rejected (all upper)\n";
+	return 0;
+    }
+    if (!$allow_all_caps)
+    {
+	my $upper_count = 0;
+	my $lower_count = 0;
+	my @letters = $self->to_def_string();
+	for my $letter (split //, $self->to_def_string())
+	{
+	    if ($letter eq uc($letter))
+	    { 
+		$upper_count++;
+	    } else {
+		$lower_count++;
+	    }		
+	}
+	return 0 if ($upper_count > $lower_count);
+    }
+    if (!$allow_recursive && $self->to_def_string() =~ /$self->to_acronym()/i )
+    {
+	return 0;
+    }
+#    print "acronym " . $self->to_string() . " not rejected\n";
+    return 1;
+}
+
+###########################################################################
+#    static functions 
+###########################################################################
+
+sub recurse {
+    my ($acro_offset,       #offset of word we're finding acronyms for
+	$text_offset,       
+	$letter_offset, 
+	@def_so_far) = @_;
+
+    my $word = $split_text[$text_offset];
+    my $acro = $split_text[$acro_offset];
+    $word = "" if !defined $word;
+    $acro = "" if !defined $acro;
+    
+#    print "recurse(" . $acro_offset . ", " . $text_offset . ", " . 
+#	$letter_offset  . ", " . @def_so_far . ")\n";
+
+    #check for termination ...
+    if ($letter_offset >= length($acro))
+    {	
+	my $acronym = new acronym();
+	$acronym->[0] = $acro;
+	push @{$acronym->[1]}, @def_so_far;
+	if ($acronym->check())
+	{
+	    push @acronym_list, ( $acronym );
+	}
+#	print "acronym created\n";
+	return;
+    }
+    #check for recursion
+    if (!$allow_recursive)
+    {
+	if ($word eq $acro)
+	{
+#	    print "recursion detected\n";
+	    return;
+	}
+    }
+    
+    #skip a stop-word
+    my $i = 0;
+    if ($letter_offset != 0)
+    {
+	while ($i <= $#stop_words)
+	{
+	    if ($stop_words[$i] eq uc($word))
+	    {
+#	    print "::found stop word::" . $stop_words[$i] . "\n";
+		&recurse($acro_offset,
+			 $text_offset+1,
+			 $letter_offset,
+			 @def_so_far, $word);
+	    }
+	    $i++;
+	}
+    }
+    $i = 1;
+    #using the first $i letters ...
+    while ($letter_offset+$i <= length($acro) )
+    {
+#	print "". ((substr $word, 0, $i) . " " . 
+#	    (substr $acro, $letter_offset, $i) . "\n");
+	if (((!$case_match) &&
+	     (uc(substr $word, 0, $i) eq
+	      uc(substr $acro, $letter_offset, $i)))
+	    ||
+	    (($case_match) &&
+	     ((substr $word, 0, $i) eq
+	      (substr $acro, $letter_offset, $i))))
+	{
+#	    print "::match::\n";
+#	    print "" . ((substr $word, 0, $i) . " " . 
+#		   (substr $acro, $letter_offset, $i) . "\n");
+	    &recurse($acro_offset,
+		     $text_offset+1,
+		     $letter_offset+$i,
+		     @def_so_far, $word);
+	} else {
+	    return;
+	}	    
+	$i++;
+    }
+    return;
+}
+
+#the main
+sub acronyms {
+    #clean up the text
+    my $processed_text =  shift @_;
+    $$processed_text =~ s/<[^>]*>/ /g;
+    $$processed_text =~ s/\W/ /g;
+    $$processed_text =~ s/[0-9_]/ /g;
+    $$processed_text =~ s/\s+/ /g;
+    $$processed_text =~ s/(\n|\>)References.*/ /i;
+    $$processed_text =~ s/(\n|\>)Bibliography.*/ /i;
+    $$processed_text =~ s/(\n|\>)(Cited Works?).*/ /i;
+    $$processed_text =~ s/(\n|\>)(Works? Cited).*/ /i;
+
+    #clear some global variables
+    @split_text = ();
+    @acronym_list = ();
+
+    return &acronyms_from_clean_text($processed_text);
+}
+
+sub acronyms_from_clean_text {
+    my ($processed_text) = @_;
+
+    @split_text = split / /, $$processed_text;
+
+#    my $i = 0;
+#    while ($i <= $#split_text)
+#    {
+#	print $split_text[$i] . "\n";
+#	$i++;
+#    }
+
+    my $first = 0;
+    my $last = $#split_text +1;
+    my $acro_counter = $first;
+    
+    while ($acro_counter < $last)
+    {
+	my $word = $split_text[$acro_counter];
+
+	if ((!$upper_case) ||
+	    (uc($word) eq $word))
+	{
+	    
+	    if (length $word >= $min_acro_length)
+	    {
+		my $def_counter = 0;
+		if ($acro_counter - $max_offset > 0)
+		{
+		    $def_counter = $acro_counter - $max_offset;
+		}
+		my $local_last = $acro_counter  + $max_offset;
+		if ($local_last > $last)
+		{
+		    $local_last = $last;
+		}
+		while ($def_counter <= $local_last)
+		{
+		    &recurse($acro_counter,$def_counter,0,());
+		    $def_counter++;
+		}
+	    }
+	}
+	$acro_counter++;
+    }
+
+    return \@acronym_list;
+}
+
+
+
+sub test {
+
+#    my $blarg = new acronym();
+#    my $simple;
+#    $simple = 10;
+#    $blarg->initialise($simple, $simple, $simple);
+#    my $blarg2 = $blarg->clone();
+#    print $blarg->to_string();
+#    print $blarg2;
+#    print "\n";
+#    
+    my $tla = new acronym();
+    $tla->[0] = "TLA";
+
+    my @array = ("Three", "Letter", "Acronym");
+#    my $i = 0;
+#    while ($i <= $#array)
+#    {
+#	print @array[$i] . "\n";
+#	$i++;
+#    }
+
+    print "\n";
+    push @{$tla->[1]}, ("Three" );
+    push @{$tla->[1]}, ("Letter" );
+    push @{$tla->[1]}, ("Acronym" );
+    print $tla->to_string(). "\n";
+    print "\n";
+    print "\n";
+    my $tla2 = $tla->clone();
+    push @{$tla2->[1]}, ("One");
+    push @{$tla2->[1]}, ("Two");
+    $tla2->[0] = "ALT";
+    print $tla->to_string(). "\n";
+    print $tla2->to_string(). "\n";
+
+    print "\n";
+    print "\n";
+
+    print "Testing recursion ...\n";
+    my $acros = &acronyms("TLA Three Letter Acronym in tla TlA");
+    
+    foreach my $acro (@$acros)
+    {
+	if ($acro->check)
+        {
+           print "accepted: " .$acro->to_string() . "\n";
+#           print "|" .  $acro->to_acronym() . "|" .  $acro->to_def_string() .
+#              "|" .  $acro->words_in_acronym_definition() .
+#              "|" .  $acro->letters_in_acronym_definition() . 
+#              "|" .  $acro->letters_in_acronym() . "|\n";
+        } else {
+#          print "found but rejected: " .$acro->to_string() . "\n";
+        }
+    }
+}
+
+#uncomment this line to test this package
+#&test();
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/arcinfo.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/arcinfo.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/arcinfo.pm	(revision 21112)
@@ -0,0 +1,226 @@
+###########################################################################
+#
+# arcinfo.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This module stores information about the archives. At the moment
+# this information just consists of the file name (relative to the
+# directory the archives information file is in) and its OID.
+
+# This module assumes there is a one to one correspondance between
+# a file in the archives directory and an OID.
+
+
+package arcinfo;
+
+use constant ORDER_OID_INDEX  => 0;
+use constant ORDER_SORT_INDEX => 1;
+
+use constant INFO_FILE_INDEX    => 0;
+use constant INFO_STATUS_INDEX  => 1;
+
+# File format read in: OID <tab> Filename <tab> Optional-Index-Status
+
+# Index status can be:
+#  I = Index for the first time
+#  R = Reindex
+#  D = Delete
+#  B = Been indexed
+
+sub new {
+    my ($class) = @_;
+    my $self = {'info'=>{},
+		'order'=>[]};
+
+    return bless $self, $class;
+}
+
+sub load_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    $self->{'info'} = {};
+ 
+    if (-e $filename) {
+	open (INFILE, $filename) || 
+	    die "arcinfo::load_info couldn't read $filename\n";
+
+	my ($line, @line);
+	while (defined ($line = <INFILE>)) {
+	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	    @line = split ("\t", $line); # filename, 
+	    if (scalar(@line) >= 2) {
+		$self->add_info (@line);
+	    }
+	}
+	close (INFILE);
+    }
+}
+
+sub save_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my ($OID, $info);
+
+    open (OUTFILE, ">$filename") || 
+	die "arcinfo::save_info couldn't write $filename\n";
+  
+    foreach $info (@{$self->get_OID_list()}) {
+	if (defined $info) {
+	    print OUTFILE join("\t", @$info), "\n";
+	}
+    }
+    close (OUTFILE);
+}
+
+sub delete_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	delete $self->{'info'}->{$OID};
+	
+	my $i = 0;
+	while ($i < scalar (@{$self->{'order'}})) {
+	    if ($self->{'order'}->[$i]->[ORDER_OID_INDEX] eq $OID) {
+		splice (@{$self->{'order'}}, $i, 1);
+		last;
+	    }
+	    
+	    $i ++;
+	}
+    }
+}
+
+sub add_info {
+    my $self = shift (@_);
+    my ($OID, $doc_file, $index_status, $sortmeta) = @_;
+    $sortmeta = "" unless defined $sortmeta;
+    $index_status = "I" unless defined $index_status; # I = needs indexing
+
+    if (! defined($OID)) {
+	# only happens when no files can be processed?
+	return undef;
+    }
+
+    $self->delete_info ($OID);
+    $self->{'info'}->{$OID} = [$doc_file,$index_status];
+    push (@{$self->{'order'}}, [$OID, $sortmeta]);
+}
+
+sub set_status_info {
+    my $self = shift (@_);
+    my ($OID, $index_status) = @_;
+
+    my $OID_info = $self->{'info'}->{$OID};
+    $OID_info->[INFO_STATUS_INDEX] = $index_status;
+}
+
+
+sub get_status_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    my $index_status = undef;
+
+    my $OID_info = $self->{'info'}->{$OID};
+    if (defined $OID_info) {
+	$index_status = $OID_info->[INFO_STATUS_INDEX];
+    }
+    else {
+	die "Unable to find document id $OID\n";
+    }
+
+    return $index_status;
+
+}
+
+
+# returns a list of the form [[OID, doc_file, index_status], ...]
+sub get_OID_list 
+{
+    my $self = shift (@_);
+
+    my $order = $self->{'order'};
+
+    my @sorted_order 
+	= sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
+
+    my @list = ();
+
+    foreach my $OID_order (@sorted_order) {
+	my $OID = $OID_order->[ORDER_OID_INDEX];
+	my $OID_info = $self->{'info'}->{$OID};
+
+	push (@list, [$OID, $OID_info->[INFO_FILE_INDEX], 
+		      $OID_info->[INFO_STATUS_INDEX]]);
+    }
+
+    return \@list;
+}
+
+# returns a list of the form [[doc_file, OID], ...]
+sub get_file_list {
+    my $self = shift (@_);
+
+    my $order = $self->{'order'};
+
+    my @sorted_order 
+	= sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
+
+    my @list = ();
+
+    foreach $OID_order (@sorted_order) {
+	my $OID = $OID_order->[ORDER_OID_INDEX];
+	my $OID_info = $self->{'info'}->{$OID};
+
+	push (@list, [$OID_info->[INFO_FILE_INDEX], $OID]);
+    }
+
+    return \@list;
+}
+
+
+# returns a list of the form [doc_file]
+sub get_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	return $self->{'info'}->{$OID};
+    }
+
+    return undef;
+}
+
+
+# returns the number of documents so far
+sub size {
+    my $self = shift (@_);
+    return (scalar(@{$self->{'order'}}));
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/basebuilder.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/basebuilder.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/basebuilder.pm	(revision 21112)
@@ -0,0 +1,625 @@
+###########################################################################
+#
+# basebuilder.pm -- base class for collection builders
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package basebuilder;
+
+use classify;
+use cfgread;
+use colcfg;
+use plugin;
+use util;
+use FileHandle;
+
+BEGIN {
+    # set autoflush on for STDERR and STDOUT so that mgpp
+    # doesn't get out of sync with plugins
+    STDOUT->autoflush(1);
+    STDERR->autoflush(1);
+}
+
+END {
+    STDOUT->autoflush(0);
+    STDERR->autoflush(0);
+}
+
+our $maxdocsize = 12000;
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, $verbosity, 
+	$maxdocs, $debug, $keepold, $remove_empty_classifications, 
+	$outhandle, $no_text, $failhandle, $gli) = @_;
+
+    $outhandle = STDERR unless defined $outhandle;
+    $no_text = 0 unless defined $no_text;
+    $failhandle = STDERR unless defined $failhandle;
+
+    # create a builder object
+    my $self = bless {'collection'=>$collection,
+		      'source_dir'=>$source_dir,
+		      'build_dir'=>$build_dir,
+		      'verbosity'=>$verbosity,
+		      'maxdocs'=>$maxdocs,
+		      'debug'=>$debug,
+		      'keepold'=>$keepold,
+		      'remove_empty_classifications'=>$remove_empty_classifications,
+		      'outhandle'=>$outhandle,
+		      'no_text'=>$no_text,
+		      'failhandle'=>$failhandle,
+		      'notbuilt'=>{},    # indexes not built
+		      'gli'=>$gli
+		      }, $class;
+
+    $self->{'gli'} = 0 unless defined $self->{'gli'};
+
+    # read in the collection configuration file
+    my $colcfgname = "$ENV{'GSDLCOLLECTDIR'}/etc/collect.cfg";
+    if (!-e $colcfgname) {
+	die "mgbuilder::new - couldn't find collect.cfg for collection $collection\n";
+    }
+    $self->{'collect_cfg'} = &colcfg::read_collect_cfg ($colcfgname);
+    
+    # get the list of plugins for this collection
+    my $plugins = [];
+    if (defined $self->{'collect_cfg'}->{'plugin'}) {
+	$plugins = $self->{'collect_cfg'}->{'plugin'};
+    }
+    
+    # load all the plugins
+
+    #build up the extra global options for the plugins
+    my @global_opts = ();
+    if (defined $self->{'collect_cfg'}->{'separate_cjk'} && $self->{'collect_cfg'}->{'separate_cjk'} =~ /^true$/i) {
+	push @global_opts, "-separate_cjk";
+    }
+    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle, $failhandle, \@global_opts, $keepold);
+   
+    if (scalar(@{$self->{'pluginfo'}}) == 0) {
+	print $outhandle "No plugins were loaded.\n";
+	die "\n";
+    }
+
+    # get the list of classifiers for this collection
+    my $classifiers = [];
+    if (defined $self->{'collect_cfg'}->{'classify'}) {
+	$classifiers = $self->{'collect_cfg'}->{'classify'};
+    }
+    
+    # load all the classifiers
+    $self->{'classifiers'} = &classify::load_classifiers ($classifiers, $build_dir, $outhandle);
+
+    # load up any dontgdbm fields
+    $self->{'dontgdbm'} = {};
+    if (defined ($self->{'collect_cfg'}->{'dontgdbm'})) {
+	foreach my $dg (@{$self->{'collect_cfg'}->{'dontgdbm'}}) {
+	    $self->{'dontgdbm'}->{$dg} = 1;
+	}
+    }
+
+    return $self;
+}
+
+# stuff has been moved here from new, so we can use subclass methods
+sub init {
+    my $self = shift(@_);
+    
+    $self->generate_index_list();
+    
+    # sort out subcollection indexes
+    if (defined $self->{'collect_cfg'}->{'indexsubcollections'}) {
+	my $indexes = $self->{'collect_cfg'}->{'indexes'};
+	$self->{'collect_cfg'}->{'indexes'} = [];
+	foreach my $subcollection (@{$self->{'collect_cfg'}->{'indexsubcollections'}}) {
+	    foreach my $index (@$indexes) {
+		push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$subcollection");
+	    }
+	}
+    }
+
+    # sort out language subindexes
+    if (defined $self->{'collect_cfg'}->{'languages'}) {
+	my $indexes = $self->{'collect_cfg'}->{'indexes'};
+	$self->{'collect_cfg'}->{'indexes'} = [];
+	foreach my $language (@{$self->{'collect_cfg'}->{'languages'}}) {
+	    foreach my $index (@$indexes) {
+		if (defined ($self->{'collect_cfg'}->{'indexsubcollections'})) {
+		    push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$language");
+		}
+		else { # add in an empty subcollection field
+		    push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index\:\:$language");
+		}
+	    }
+	}
+    }
+
+    if (defined($self->{'collect_cfg'}->{'indexes'})) {
+	# make sure that the same index isn't specified more than once
+	my %tmphash = ();
+	my @tmparray = @{$self->{'collect_cfg'}->{'indexes'}};
+	$self->{'collect_cfg'}->{'indexes'} = [];
+	foreach my $i (@tmparray) {
+	    if (!defined ($tmphash{$i})) {
+		push (@{$self->{'collect_cfg'}->{'indexes'}}, $i);
+		$tmphash{$i} = 1;
+	    }
+	}
+    } else {
+	$self->{'collect_cfg'}->{'indexes'} = [];
+    }
+
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    my ($buildprocdir, $buildproctype);
+    my $collection = $self->{'collection'};
+    if (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}buildproc.pm") {
+	$buildprocdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
+	$buildproctype = "${collection}buildproc";
+    } else {
+	$buildprocdir = "$ENV{'GSDLHOME'}/perllib";
+	$buildproctype = $self->default_buildproc();
+    }
+    require "$buildprocdir/$buildproctype.pm";
+
+    eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
+	 "\$source_dir, \$build_dir, \$keepold, \$verbosity, \$outhandle)");
+    die "$@" if $@;
+
+    if (!$self->{'debug'} && !$self->{'keepold'}) {
+	# remove any old builds
+	&util::rm_r($self->{'build_dir'});
+	&util::mk_all_dir($self->{'build_dir'});
+        
+	# make the text directory
+	my $textdir = "$self->{'build_dir'}/text";
+	&util::mk_all_dir($textdir);
+    }
+    
+}
+
+sub deinit {
+    my $self = shift (@_);
+    
+    &plugin::deinit($self->{'pluginfo'},$self->{'buildproc'});
+}
+
+sub set_sections_index_document_metadata {
+    my $self = shift (@_);
+    my ($index) = @_;
+  
+    $self->{'buildproc'}->set_sections_index_document_metadata($index);
+}
+
+sub set_strip_html {
+    my $self = shift (@_);
+    my ($strip) = @_;
+    
+    $self->{'strip_html'} = $strip;
+    $self->{'buildproc'}->set_strip_html($strip);
+}
+
+sub compress_text {
+    my $self = shift (@_);
+    my ($textindex) = @_;
+
+    print STDERR "compress_text() should be implemented in subclass!!";
+    return;
+}
+
+
+sub build_indexes {
+    my $self = shift (@_);
+    my ($indexname) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $indexes = [];
+    if (defined $indexname && $indexname =~ /\w/) {
+	push @$indexes, $indexname;
+    } else {
+	$indexes = $self->{'collect_cfg'}->{'indexes'};
+    }
+
+    # create the mapping between the index descriptions 
+    # and their directory names (includes subcolls and langs)
+    $self->{'index_mapping'} = $self->create_index_mapping ($indexes);
+
+    # build each of the indexes
+    foreach my $index (@$indexes) {
+	if ($self->want_built($index)) {
+	    print $outhandle "\n*** building index $index in subdirectory " . 
+		"$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
+	    print STDERR "<Stage name='Index' source='$index'>\n" if $self->{'gli'};
+	    $self->build_index($index);
+	} else {
+	    print $outhandle "\n*** ignoring index $index\n" if ($self->{'verbosity'} >= 1);
+	}
+    }
+
+    $self->build_indexes_extra();
+
+}
+
+sub build_indexes_extra {
+    my $self = shift(@_);
+    
+}
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+    
+    print STDERR "build_index should be implemented in subclass\n";
+    return;
+}
+
+sub make_infodatabase {
+    my $self = shift (@_);
+    my $outhandle = $self->{'outhandle'};
+
+    my $textdir = &util::filename_cat($self->{'build_dir'}, "text");
+    my $assocdir = &util::filename_cat($self->{'build_dir'}, "assoc");
+    &util::mk_all_dir ($textdir);
+    &util::mk_all_dir ($assocdir);
+
+    # get db name
+    my $dbext = ".bdb";
+    $dbext = ".ldb" if &util::is_little_endian();
+    my $fulldbname = &util::filename_cat ($textdir, "$self->{'collection'}$dbext");
+    $fulldbname =~ s/\//\\/g if ($ENV{'GSDLOS'} =~ /^windows$/i);
+
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $txt2db_exe = &util::filename_cat($exedir, "txt2db$exe");
+
+    print $outhandle "\n*** creating the info database and processing associated files\n" 
+	if ($self->{'verbosity'} >= 1);
+    print STDERR "<Stage name='CreateInfoData'>\n" if $self->{'gli'};
+
+    # init all the classifiers
+    &classify::init_classifiers ($self->{'classifiers'});
+    
+    # set up the document processor
+    my ($handle);
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$txt2db_exe" || !open (PIPEOUT, "| txt2db$exe \"$fulldbname\"")) {
+	    print STDERR "<FatalError name='NoRunText2DB'/>\n</Stage>\n" if $self->{'gli'};
+	    die "builder::make_infodatabase - couldn't run $txt2db_exe\n";
+	}
+	$handle = basebuilder::PIPEOUT;
+    }
+    
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('infodb');
+    $self->{'buildproc'}->set_assocdir ($assocdir);
+    $self->{'buildproc'}->set_dontgdbm ($self->{'dontgdbm'});
+    $self->{'buildproc'}->set_classifiers ($self->{'classifiers'});
+    $self->{'buildproc'}->set_indexing_text (0);
+    $self->{'buildproc'}->set_store_text(1);
+    # make_infodatabase does not support incremental build 
+    # => full reset needed
+    $self->{'buildproc'}->zero_reset();
+   
+    # do we need this anymore if we get coll meta from config file??
+    $self->output_collection_meta($handle);
+
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
+    
+    # output classification information
+    &classify::output_classify_info ($self->{'classifiers'}, $handle,
+				     $self->{'remove_empty_classifications'}, 
+				     $self->{'gli'});
+
+
+    #output doclist
+    my @doclist = $self->{'buildproc'}->get_doc_list();
+    my $docs = join (";",@doclist);
+    print $handle "[browselist]\n";
+    print $handle "<hastxt>0\n";
+    print $handle "<childtype>VList\n";
+    print $handle "<numleafdocs>" . ($#doclist+1) . "\n";
+    print $handle "<thistype>Invisible\n";
+    print $handle "<contains>$docs";
+    print $handle "\n" . ('-' x 70) . "\n";
+
+    close ($handle) if !$self->{'debug'};
+
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+sub make_auxiliary_files {
+    my $self = shift (@_);
+    my ($index);
+    my $build_cfg = {};
+    # subclasses may have already defined stuff in here
+    if (defined $self->{'build_cfg'}) {
+	$build_cfg = $self->{'build_cfg'};
+    }
+
+    my $outhandle = $self->{'outhandle'};
+
+    print $outhandle "\n*** creating auxiliary files \n" if ($self->{'verbosity'} >= 1);
+    print STDERR "<Stage name='CreatingAuxilary'>\n" if $self->{'gli'};
+
+    # get the text directory
+    &util::mk_all_dir ($self->{'build_dir'});
+
+    # store the build date
+    $build_cfg->{'builddate'} = time;
+    $build_cfg->{'buildtype'} = $self->{'buildtype'};
+    $build_cfg->{'indexstem'} = $self->{'collection'};
+    # store the number of documents and number of bytes
+    $build_cfg->{'numdocs'} = $self->{'buildproc'}->get_num_docs();
+    $build_cfg->{'numbytes'} = $self->{'buildproc'}->get_num_bytes();
+
+    
+    # store the mapping between the index names and the directory names
+    # the index map is used to determine what indexes there are, so any that are not built should not be put into the map.
+    my @indexmap = ();
+    foreach my $index (@{$self->{'index_mapping'}->{'indexmaporder'}}) {
+	if (not defined ($self->{'notbuilt'}->{$index})) {
+	    push (@indexmap, "$index\-\>$self->{'index_mapping'}->{'indexmap'}->{$index}");
+	}
+    }
+    $build_cfg->{'indexmap'} = \@indexmap;
+
+    my @subcollectionmap = ();
+    foreach my $subcollection (@{$self->{'index_mapping'}->{'subcollectionmaporder'}}) {
+	push (@subcollectionmap, "$subcollection\-\>" .
+	      $self->{'index_mapping'}->{'subcollectionmap'}->{$subcollection});
+    }
+    $build_cfg->{'subcollectionmap'} = \@subcollectionmap if scalar (@subcollectionmap);
+
+    my @languagemap = ();
+    foreach my $language (@{$self->{'index_mapping'}->{'languagemaporder'}}) {
+	push (@languagemap, "$language\-\>" .
+	      $self->{'index_mapping'}->{'languagemap'}->{$language});
+    }
+    $build_cfg->{'languagemap'} = \@languagemap if scalar (@languagemap);
+
+    my @notbuilt = ();
+    foreach my $nb (keys %{$self->{'notbuilt'}}) {
+	push (@notbuilt, $nb);
+    }
+    $build_cfg->{'notbuilt'} = \@notbuilt if scalar (@notbuilt);
+
+    $build_cfg->{'maxnumeric'} = 4;
+    if (defined($self->{'collect_cfg'}->{'maxnumeric'}) &&
+	$self->{'collect_cfg'}->{'maxnumeric'} =~ /^\d+$/) {
+	$build_cfg->{'maxnumeric'} = $self->{'collect_cfg'}->{'maxnumeric'};
+    }
+
+    $self->build_cfg_extra($build_cfg);
+    
+    $self->write_cfg_file($build_cfg);
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+sub collect_specific {
+    my $self = shift (@_);
+}
+
+sub want_built {
+    my $self = shift (@_);
+    my ($index) = @_;
+
+    if (defined ($self->{'collect_cfg'}->{'dontbuild'})) {
+	foreach my $checkstr (@{$self->{'collect_cfg'}->{'dontbuild'}}) {
+	    if ($index =~ /^$checkstr$/) {
+		$self->{'notbuilt'}->{$index} = 1;
+		return 0;
+	    }
+	}
+    }
+
+    return 1;
+}
+
+sub create_index_mapping {
+    my $self = shift (@_);
+    my ($indexes) = @_;
+
+    print STDERR "create_index_mapping should be implemented in subclass\n";
+    my %mapping = ();
+    return \%mapping;
+}
+
+# returns a processed version of a field.
+# if the field has only one component the processed
+# version will contain the first character and next consonant
+# of that componant - otherwise it will contain the first 
+# character of the first two components 
+sub process_field {
+    my $self = shift (@_);
+    my ($field) = @_;
+ 
+    return "" unless (defined ($field) && $field =~ /\w/);
+
+    my @components = split /,/, $field;
+    if (scalar @components >= 2) {
+	splice (@components, 2);
+	map {s/^(.).*$/$1/;} @components;
+	return join("", @components);
+    } else {
+	my ($a, $b) = $field =~ /^(.).*?([bcdfghjklmnpqrstvwxyz])/i;
+	($a, $b) = $field =~ /^(.)(.)/ unless defined $a && defined $b;
+	return "$a$b";
+    }
+}
+
+sub get_next_version {
+    my $self = shift (@_);
+    my ($nameref) = @_;
+    my $num=0;
+    if ($$nameref =~ /(\d\d)$/) {
+	$num = $1; $num ++;
+	$$nameref =~ s/\d\d$/$num/;
+    } elsif ($$nameref =~ /(\d)$/) {
+	$num = $1;
+	if ($num == 9) {$$nameref =~ s/\d$/10/;}
+	else {$num ++; $$nameref =~ s/\d$/$num/;}
+    } else {
+	$$nameref =~ s/.$/0/;
+    }
+}
+
+# implement this in subclass if want to add extra stuff to build.cfg
+sub build_cfg_extra {
+   my $self = shift(@_);
+   my ($build_cfg) = @_;
+   
+}
+
+sub write_cfg_file {
+    my $self = shift(@_);
+    my ($build_cfg) = @_;
+    
+    # write out the build information
+    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
+			     '^(builddate|buildtype|numdocs|numbytes|numwords|numsections|maxnumeric|indexstem)$', 
+                             '^(indexmap|subcollectionmap|languagemap|notbuilt)$');
+
+}
+
+
+sub output_collection_meta {
+    my $self = shift(@_);
+    my ($handle) = @_;
+    my $outhandle = $self->{'outhandle'};
+    if (defined $self->{'collect_cfg'}->{'collectionmeta'}) {
+	
+        if (!defined $self->{'index_mapping'}) {
+	    $self->{'index_mapping'} = 
+		$self->create_index_mapping ($self->{'collect_cfg'}->{'indexes'});
+	}
+	
+	print $handle "[collection]\n";
+     
+	foreach my $cmeta (keys (%{$self->{'collect_cfg'}->{'collectionmeta'}})) {
+	    my $defaultfound=0;
+	    my $first=1;
+	    my $metadata_entry = "";
+	    my $default="";
+	    my $cmetamap = "";
+	    if ($cmeta =~ s/^\.//) {
+    		if (defined $self->{'index_mapping'}->{$cmeta}) {
+		    $cmetamap = $self->{'index_mapping'}->{$cmeta};
+		    $cmeta = ".$cmeta";
+		}
+		else {
+		    print $outhandle "mgbuilder: warning bad collectionmeta option '$cmeta' - ignored\n";
+		    next; #ignore this one
+		}
+	    }
+	    else {
+		$cmetamap = $cmeta; # just using the same name
+	    }
+	    #iterate through the languages
+	    foreach my $lang (keys (%{$self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}})) {
+		if ($first) {
+		    $first=0;
+		    #set the default default to the first entry
+		    $default=$self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}->{$lang};
+		}
+		if ($lang =~ /default/) {
+		    $defaultfound=1;
+		    #the default entry goes first
+		    $metadata_entry = "<$cmetamap>" .
+			$self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}->{'default'} . "\n" . $metadata_entry;
+		}
+		else {
+		    my ($l) = $lang =~ /^\[l=(\w*)\]$/;
+		    if ($l) {
+			$metadata_entry .= "<$cmetamap:$l>" . 
+			    $self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}->{$lang} . "\n";
+
+			# Use the English value as the default if no default is specified
+			if ($l =~ /en/i) {
+			    $default=$self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}->{$lang};
+			}
+		    }
+		}
+	    }
+	    #if we haven't found a default, put one in
+	    if (!$defaultfound) {
+		$metadata_entry = "<$cmetamap>$default\n" . $metadata_entry;
+	    }
+	#write the entry to the file
+	print $handle $metadata_entry;
+
+	}
+	
+	print $handle "\n" . ('-' x 70) . "\n";
+    }
+    
+}
+  
+sub print_stats {
+    my $self = shift (@_);
+
+    my $outhandle = $self->{'outhandle'};
+    my $indexing_text = $self->{'buildproc'}->get_indexing_text();
+    my $index = $self->{'buildproc'}->get_index();
+    my $num_bytes = $self->{'buildproc'}->get_num_bytes();
+    my $num_processed_bytes = $self->{'buildproc'}->get_num_processed_bytes();
+
+    if ($indexing_text) {
+	print $outhandle "Stats (Creating index $index)\n";
+    } else {
+	print $outhandle "Stats (Compressing text from $index)\n";
+    }
+    print $outhandle "Total bytes in collection: $num_bytes\n";
+    print $outhandle "Total bytes in $index: $num_processed_bytes\n";
+
+    if ($num_processed_bytes < 50 && ($indexing_text || !$self->{'no_text'})) {
+	
+	if ($self->{'keepold'}) {
+	    if ($num_processed_bytes == 0) {
+		if ($indexing_text) {
+		    print $outhandle "No additional text was added to $index\n";
+		} elsif (!$self->{'no_text'}) {
+		    print $outhandle "No additional text was compressed\n";
+		}	
+	    }	
+	}
+	else {
+	    print $outhandle "***************\n";
+	    if ($indexing_text) {
+		print $outhandle "WARNING: There is very little or no text to process for $index\n";
+	    } elsif (!$self->{'no_text'}) {
+		print $outhandle "WARNING: There is very little or no text to compress\n";
+	    }	   
+	    print $outhandle "         Was this your intention?\n";
+	    print $outhandle "***************\n";
+	}
+
+    }
+
+}
+
+  
+1;
+
Index: /main/tags/3.01/gsdl/perllib/basebuildproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/basebuildproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/basebuildproc.pm	(revision 21112)
@@ -0,0 +1,596 @@
+###########################################################################
+#
+# basebuildproc.pm -- 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor outputs a document for indexing (should be 
+# implemented by subclass) and storing in gdbm database
+
+package basebuildproc;
+
+eval {require bytes};
+
+use classify;
+use doc;
+use docproc;
+use util;
+
+BEGIN {
+    @basebuildproc::ISA = ('docproc');
+}
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, $keepold,
+	$verbosity, $outhandle) = @_; 
+    my $self = new docproc ();
+
+    # outhandle is where all the debugging info goes
+    # output_handle is where the output of the plugins is piped
+    # to (i.e. mg, gdbm etc.)
+    $outhandle = STDERR unless defined $outhandle;
+
+    $self->{'collection'} = $collection;
+    $self->{'source_dir'} = $source_dir;
+    $self->{'build_dir'}  = $build_dir;
+    $self->{'keepold'}    = $keepold;
+    $self->{'verbosity'}  = $verbosity;
+    $self->{'outhandle'}  = $outhandle;
+
+    $self->{'classifiers'} = [];
+    $self->{'mode'} = "text";
+    $self->{'assocdir'} = $build_dir;
+    $self->{'dontgdbm'} = {};
+
+    $self->{'index'} = "section:text";
+    $self->{'indexexparr'} = [];
+
+    my $found_num_data = 0;
+    my $buildconfigfile = undef;
+
+    if ($keepold) {
+	# For incremental building need to seed num_docs etc from values
+	# stored in build.cfg (if present)
+
+	$buildconfigfile = &util::filename_cat($build_dir, "build.cfg");
+
+	if (-e $buildconfigfile) {
+	    $found_num_data = 1;
+	}
+	else {
+	    # try the index dir
+	    $buildconfigfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
+						   "index", "build.cfg");
+	    if (-e $buildconfigfile) {
+		$found_num_data = 1;
+	    }
+	}
+
+    } 
+
+    if ($found_num_data) {
+	my $buildcfg = &colcfg::read_build_cfg($buildconfigfile);
+	
+	$self->{'starting_num_docs'}     = $buildcfg->{'numdocs'};
+	$self->{'starting_num_sections'} = $buildcfg->{'numsections'};
+	$self->{'starting_num_bytes'}    = $buildcfg->{'numbytes'};
+    }
+    else {
+	$self->{'starting_num_docs'}     = 0;
+	$self->{'starting_num_sections'} = 0;
+	$self->{'starting_num_bytes'}    = 0;
+    }
+
+    $self->{'output_handle'} = "STDOUT";
+    $self->{'num_docs'}      = $self->{'starting_num_docs'};
+    $self->{'num_sections'}  = $self->{'starting_num_sections'};
+    $self->{'num_bytes'}     = $self->{'starting_num_bytes'};
+
+    $self->{'num_processed_bytes'} = 0;
+    $self->{'store_text'} = 1;
+
+    # what level (section/document) the gdbm database - indexer intersection is
+    $self->{'gdbm_level'} = "section";
+    #used by browse interface
+    $self->{'doclist'} = [];
+
+    $self->{'indexing_text'} = 0;
+
+    return bless $self, $class;
+
+}
+
+sub reset {
+    my $self = shift (@_);
+
+    $self->{'num_docs'}      = $self->{'starting_num_docs'};
+    $self->{'num_sections'}  = $self->{'starting_num_sections'};
+    $self->{'num_bytes'}     = $self->{'starting_num_bytes'};
+    
+    $self->{'num_processed_bytes'} = 0;
+}
+
+sub zero_reset {
+    my $self = shift (@_);
+
+    $self->{'num_docs'}      = 0;
+    $self->{'num_sections'}  = 0;
+    $self->{'num_bytes'}     = 0;
+    
+    $self->{'num_processed_bytes'} = 0;
+}
+
+sub is_incremental_capable
+{
+    # By default we return 'no' as the answer
+    # Safer to assume non-incremental to start with, and then override in
+    # inherited classes that are.
+
+    return 0;
+}
+
+sub get_num_docs {
+    my $self = shift (@_);
+
+    return $self->{'num_docs'};
+}
+
+sub get_num_sections {
+    my $self = shift (@_);
+
+    return $self->{'num_sections'};
+}
+
+# num_bytes is the actual number of bytes in the collection
+# this is normally the same as what's processed during text compression
+sub get_num_bytes {
+    my $self = shift (@_);
+
+    return $self->{'num_bytes'};
+}
+
+# num_processed_bytes is the number of bytes actually passed
+# to mg for the current index
+sub get_num_processed_bytes {
+    my $self = shift (@_);
+
+    return $self->{'num_processed_bytes'};
+}
+
+sub set_output_handle {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    $self->{'output_handle'} = $handle;
+}
+
+
+sub set_mode {
+    my $self = shift (@_);
+    my ($mode) = @_;
+
+    $self->{'mode'} = $mode;
+}
+
+sub get_mode {
+    my $self = shift (@_);
+
+    return $self->{'mode'};
+}
+
+sub set_assocdir {
+    my $self = shift (@_);
+    my ($assocdir) = @_;
+
+    $self->{'assocdir'} = $assocdir;
+}
+
+sub set_dontgdbm {
+    my $self = shift (@_);
+    my ($dontgdbm) = @_;
+
+    $self->{'dontgdbm'} = $dontgdbm;
+}
+
+sub set_index {
+    my $self = shift (@_);
+    my ($index, $indexexparr) = @_;
+
+    $self->{'index'} = $index;
+    $self->{'indexexparr'} = $indexexparr if defined $indexexparr;
+}
+
+sub set_index_languages {
+    my $self = shift (@_);
+    my ($lang_meta, $langarr) = @_;
+    $self->{'lang_meta'} = $lang_meta;
+    $self->{'langarr'} = $langarr;
+}
+
+sub get_index {
+    my $self = shift (@_);
+
+    return $self->{'index'};
+}
+
+sub set_classifiers {
+    my $self = shift (@_);
+    my ($classifiers) = @_;
+
+    $self->{'classifiers'} = $classifiers;
+}
+
+sub set_indexing_text {
+    my $self = shift (@_);
+    my ($indexing_text) = @_;
+
+    $self->{'indexing_text'} = $indexing_text;
+}
+
+sub get_indexing_text {
+    my $self = shift (@_);
+
+    return $self->{'indexing_text'};
+}
+
+sub set_store_text {
+    my $self = shift (@_);
+    my ($store_text) = @_;
+
+    $self->{'store_text'} = $store_text;
+}
+sub get_doc_list {
+    my $self = shift(@_);
+    
+    return @{$self->{'doclist'}};
+}
+
+# the standard gdbm level is section, but you may want to change it to document
+sub set_gdbm_level {
+    my $self= shift (@_);
+    my ($gdbm_level) = @_;
+
+    $self->{'gdbm_level'} = $gdbm_level;
+}
+
+sub set_sections_index_document_metadata {
+    my $self= shift (@_);
+    my ($index_type) = @_;
+    
+    $self->{'sections_index_document_metadata'} = $index_type;
+}
+sub process {
+    my $self = shift (@_);
+    my $method = $self->{'mode'};
+
+    $self->$method(@_);
+}
+
+sub infodb {
+    my $self = shift (@_);
+    my ($doc_obj, $filename) = @_;
+    my $handle = $self->{'output_handle'};
+
+    my $doctype = $doc_obj->get_doc_type();
+
+    # only output this document if it is one to be indexed
+    return if ($doctype ne "indexed_doc");
+
+    my ($archivedir) = $filename =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
+    $archivedir = "" unless defined $archivedir;
+    $archivedir =~ s/\\/\//g;
+    $archivedir =~ s/^\/+//;
+    $archivedir =~ s/\/+$//;
+
+    # resolve the final filenames of the files associated with this document
+    $self->assoc_files ($doc_obj, $archivedir);
+
+    #GRB: moved 1/06/2004 from GRB01062004
+    #add this document to the browse structure
+    push(@{$self->{'doclist'}},$doc_obj->get_OID()) 
+	unless ($doctype eq "classification");
+
+    # classify this document
+    &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
+    #GRB: end of moved block
+
+    # this is another document
+    $self->{'num_docs'} += 1 unless ($doctype eq "classification");
+
+    # is this a paged or a hierarchical document
+    my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
+
+    my $section = $doc_obj->get_top_section ();
+    my $doc_OID = $doc_obj->get_OID();
+    my $first = 1;
+    my $url = "";
+    while (defined $section) {
+	# update a few statistics
+	$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	$self->{'num_sections'} += 1 unless ($doctype eq "classification");
+
+	# output the section name
+	if ($section eq "") { print $handle "[$doc_OID]\n"; }
+	else { print $handle "[$doc_OID.$section]\n"; }
+
+	# output the fact that this document is a document (unless doctype
+	# has been set to something else from within a plugin
+	my $dtype = $doc_obj->get_metadata_element ($section, "doctype");
+	if (!defined $dtype || $dtype !~ /\w/) {
+	    print $handle "<doctype>doc\n";
+	}
+
+	# output whether this node contains text
+	if ($doc_obj->get_text_length($section) > 0) {
+	    print $handle "<hastxt>1\n";
+	} else {
+	    print $handle "<hastxt>0\n";
+	}
+
+	# output all the section metadata
+	my $metadata = $doc_obj->get_all_metadata ($section);
+	foreach my $pair (@$metadata) {
+	    my ($field, $value) = (@$pair);
+
+	    if ($field ne "Identifier" && $field !~ /^gsdl/ && 
+		defined $value && $value ne "") {	    
+
+		# escape problematic stuff
+		$value =~ s/\\/\\\\/g;
+		$value =~ s/\n/\\n/g;
+		$value =~ s/\r/\\r/g;
+		if ($value =~ /-{70,}/) {
+		    # if value contains 70 or more hyphens in a row we need
+		    # to escape them to prevent txt2db from treating them
+		    # as a separator
+		    $value =~ s/-/&\#045;/gi;
+		}
+
+		# special case for URL metadata
+		if ($field =~ /^URL$/i) {
+                    $url .= "[$value]\n";
+                    if ($section eq "") {$url .= "<section>$doc_OID\n";}
+                    else {$url .= "<section>$doc_OID.$section\n";}
+                    $url .= '-' x 70 . "\n";
+		}
+
+		if (!defined $self->{'dontgdbm'}->{$field}) {
+		    print $handle "<$field>$value\n";
+		}
+	    }
+	}
+
+	# output archivedir if at top level
+	if ($section eq $doc_obj->get_top_section()) {
+	    print $handle "<archivedir>$archivedir\n";
+	}
+
+	# output document display type
+	if ($first) {
+	    print $handle "<thistype>$thistype\n";
+	}
+
+	if ($self->{'gdbm_level'} eq "document") {
+	    # doc num is num_docs not num_sections
+	    # output the matching document number
+	    print $handle "<docnum>$self->{'num_docs'}\n";
+	} else {
+	    # output a list of children
+	    my $children = $doc_obj->get_children ($section);
+	    if (scalar(@$children) > 0) {
+		print $handle "<childtype>$childtype\n";
+		print $handle "<contains>";
+		my $firstchild = 1;
+		foreach my $child (@$children) {
+		    print $handle ";" unless $firstchild;
+		    $firstchild = 0;
+		    if ($child =~ /^.*?\.(\d+)$/) {
+			print $handle "\".$1";
+		    } else {
+			print $handle "\".$child";
+		    }
+#		if ($child eq "") { print $handle "$doc_OID"; }
+#		elsif ($section eq "") { print $handle "$doc_OID.$child"; }
+#		else { print $handle "$doc_OID.$section.$child"; }
+		}
+		print $handle "\n";
+	    }
+	    #output the matching doc number
+	    print $handle "<docnum>$self->{'num_sections'}\n";
+	    
+	} 
+	
+	print $handle '-' x 70, "\n";
+
+	
+	# output a database entry for the document number
+	if ($self->{'gdbm_level'} eq "document") {
+	    print $handle "[$self->{'num_docs'}]\n";
+	    print $handle "<section>$doc_OID\n";
+	}
+	else {
+	    print $handle "[$self->{'num_sections'}]\n";
+	    if ($section eq "") { print $handle "<section>$doc_OID\n"; }
+	    else { print $handle "<section>$doc_OID.$section\n"; }
+	}
+	print $handle '-' x 70, "\n";
+	
+        # output entry for url
+        if ($url ne "") {
+            print $handle $url;
+        }
+
+	$first = 0;
+	$section = $doc_obj->get_next_section($section);
+	last if ($self->{'gdbm_level'} eq "document"); # if no sections wanted, only gdbm the docs
+    }
+
+    #GRB01062004: see code above moved from here
+}
+
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    
+    my $handle = $self->{'outhandle'};
+    print $handle "basebuildproc::text function must be implemented in sub classes\n";
+    die "\n";
+}
+
+# should the document be indexed - according to the subcollection and language
+# specification.
+sub is_subcollection_doc {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    
+    my $indexed_doc = 1;
+    foreach my $indexexp (@{$self->{'indexexparr'}}) {
+	$indexed_doc = 0;
+	my ($field, $exp, $options) = split /\//, $indexexp;
+	if (defined ($field) && defined ($exp)) {
+	    my ($bool) = $field =~ /^(.)/;
+	    $field =~ s/^.// if $bool eq '!';
+	    my @metadata_values;
+	    if ($field =~ /^filename$/i) {
+		push(@metadata_values, $doc_obj->get_source_filename());
+	    }
+	    else {
+		@metadata_values = @{$doc_obj->get_metadata($doc_obj->get_top_section(), $field)};
+	    }
+	    next unless @metadata_values;
+	    foreach my $metadata_value (@metadata_values) {
+		if ($bool eq '!') {
+		    if ($options =~ /^i$/i) {
+			if ($metadata_value !~ /$exp/i) {$indexed_doc = 1; last;}
+		    } else {
+			if ($metadata_value !~ /$exp/) {$indexed_doc = 1; last;}
+		    }
+		} else {
+		    if ($options =~ /^i$/i) {
+			if ($metadata_value =~ /$exp/i) {$indexed_doc = 1; last;}
+		    } else {
+			if ($metadata_value =~ /$exp/) {$indexed_doc = 1; last;}
+		    }
+		}
+	    }
+
+	    last if ($indexed_doc == 1);
+	}
+    }
+    
+    # if this doc is so far in the sub collection, and we have lang info, 
+    # now we check the languages to see if it matches
+    if($indexed_doc && defined $self->{'lang_meta'}) {
+	$indexed_doc = 0;
+	my $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'lang_meta'});
+	if (defined $field) {
+	    foreach my $lang (@{$self->{'langarr'}}) {
+		my ($bool) = $lang =~ /^(.)/;
+		if ($bool eq '!') {
+		    $lang =~ s/^.//;
+		    if ($field !~ /$lang/) {
+			$indexed_doc = 1; last;
+		    }
+		} else {
+		    if ($field =~ /$lang/) {
+			$indexed_doc = 1; last;
+		    }
+		}
+	    }
+	} 
+    }
+    return $indexed_doc;
+    
+}
+
+# use 'Paged' if document has no more than 2 levels
+# and each section at second level has a number for
+# Title metadata
+# also use Paged if gsdlthistype metadata is set to Paged
+sub get_document_type {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $thistype = "VList";
+    my $childtype = "VList";
+    my $title;
+    my @tmp = ();
+    
+    my $section = $doc_obj->get_top_section ();
+    
+    my $gsdlthistype = $doc_obj->get_metadata_element ($section, "gsdlthistype");
+    if (defined $gsdlthistype) {
+	if ($gsdlthistype eq "Paged") {
+	    $childtype = "Paged";
+	    if ($doc_obj->get_text_length ($doc_obj->get_top_section())) {
+		$thistype = "Paged";
+	    } else {
+		$thistype = "Invisible";
+	    }
+	    
+	    return ($thistype, $childtype);
+	} elsif ($gsdlthistype eq "Hierarchy") {
+	    return ($thistype, $childtype); # use VList, VList
+	}
+    }
+    my $first = 1;
+    while (defined $section) {
+	@tmp = split /\./, $section;
+	if (scalar(@tmp) > 1) {
+	    return ($thistype, $childtype);
+	}
+	if (!$first) {
+	    $title = $doc_obj->get_metadata_element ($section, "Title");
+	    if (!defined $title || $title !~ /^\d+$/) {
+		return ($thistype, $childtype);
+	    }
+	}
+	$first = 0;
+	$section = $doc_obj->get_next_section($section);
+    }
+    if ($doc_obj->get_text_length ($doc_obj->get_top_section())) {
+	$thistype = "Paged";
+    } else {
+	$thistype = "Invisible";
+    }
+    $childtype = "Paged";
+    return ($thistype, $childtype);
+}
+
+sub assoc_files {
+    my $self = shift (@_);
+    my ($doc_obj, $archivedir) = @_;
+    my ($afile);
+    
+    foreach my $assoc_file (@{$doc_obj->get_assoc_files()}) {
+	# if assoc file starts with a slash, we put it relative to the assoc
+	# dir, otherwise it is relative to the HASH... directory
+	if ($assoc_file->[1] =~ m@^[/\\]@) {
+	    $afile = &util::filename_cat($self->{'assocdir'},$assoc_file->[1]);
+	} else {
+	    $afile = &util::filename_cat($self->{'assocdir'}, $archivedir, $assoc_file->[1]);
+	}
+	&util::hard_link ($assoc_file->[0], $afile);
+    }
+}
+
Index: /main/tags/3.01/gsdl/perllib/cfgread.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cfgread.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cfgread.pm	(revision 21112)
@@ -0,0 +1,178 @@
+###########################################################################
+#
+# cfgread.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# reads in configuration files
+
+package cfgread;
+
+
+sub read_cfg_line {
+    my ($handle) = @_;
+    my $line = "";
+    my @line = ();
+    my $linecontinues = 0;
+
+    while (defined($line = <$handle>)) {
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	# Merge with following line if a quoted phrase is left un-closed.
+	if ($line =~ m/[^\\]([\"\'])/) {
+	    my $quote=$1;
+	    while ($line !~ m/$quote(.*?[^\\])?(\\\\)*$quote/) {
+		my $nextline = <$handle>;
+		if (defined($nextline)) {
+		    $nextline =~ s/\r?\n//; # remove end-of-line
+		    $line .= " " . $nextline;
+		} else {
+		    return undef; # parse error?
+		}
+	    }
+	}
+	$linecontinues = $line =~ s/\\$//;
+
+	while ($line =~ s/\s*(\".*?[^\\](\\\\)*\"|\'.*?[^\\](\\\\)*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute an environment variables
+##		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+
+	if (scalar(@line) > 0 && !$linecontinues) {
+#	    print STDERR "line: \"" . join ("\" \"", @line) . "\"\n";
+	    return \@line;
+	}
+    }
+
+    return undef;
+}
+
+sub write_cfg_line {
+    my ($handle, $line) = @_;
+    print $handle join ("\t", @$line), "\n";
+}
+
+
+# stringexp, arrayexp, hashexp,arrayarrayexp and hashhashexp
+# should be something like '^(this|that)$'
+sub read_cfg_file {
+    my ($filename, $stringexp, $arrayexp, $hashexp, $arrayarrayexp,
+	$hashhashexp) = @_;
+    my ($line);
+    my $data = {};
+
+    if (open (COLCFG, $filename)) {
+	while (defined ($line = &read_cfg_line('COLCFG'))) {
+	    if (scalar(@$line) >= 2) {
+		my $key = shift (@$line);
+		if (defined $stringexp && $key =~ /$stringexp/) {
+		    $data->{$key} = shift (@$line);
+
+		} elsif (defined $arrayexp && $key =~ /$arrayexp/) {
+		    push (@{$data->{$key}}, @$line);
+
+		} elsif (defined $hashexp && $key =~ /$hashexp/) {
+		    my $k = shift @$line;
+		    my $v = shift @$line;
+		    $data->{$key}->{$k} = $v;
+		} elsif (defined $arrayarrayexp && $key =~ /$arrayarrayexp/) {
+		    if (!defined $data->{$key}) {
+			$data->{$key} = [];
+		    }
+		    push (@{$data->{$key}}, $line);
+		}
+		elsif (defined $hashhashexp && $key =~ /$hashhashexp/) {
+		    my $k = shift @$line;
+		    my $p = shift @$line;
+		    my $v = shift @$line;
+		    if (!defined $v) {
+			$v = $p;
+			$p = 'default';
+		    }
+		    $data->{$key}->{$k}->{$p} = $v;
+		}
+	    }
+	}
+	close (COLCFG);
+
+    } else {
+	print STDERR "cfgread::read_cfg_file couldn't read the cfg file $filename\n";
+    }
+
+    return $data;
+}
+
+
+# stringexp, arrayexp, hashexp and arrayarrayexp 
+# should be something like '^(this|that)$'
+sub write_cfg_file {
+    my ($filename, $data, $stringexp, $arrayexp, $hashexp, $arrayarrayexp,
+	$hashhashexp) = @_;
+    
+    if (open (COLCFG, ">$filename")) {
+	foreach $key (sort(keys(%$data))) {
+	    if ($key =~ /$stringexp/) {
+		&write_cfg_line ('COLCFG', [$key, $data->{$key}]);
+	    } elsif ($key =~ /$arrayexp/) {
+		&write_cfg_line ('COLCFG', [$key, @{$data->{$key}}]);
+	    } elsif ($key =~ /$hashexp/) {
+		foreach $k (keys (%{$data->{$key}})) {
+		    &write_cfg_line ('COLCFG', [$key, $k, $data->{$key}->{$k}]);
+		}
+	    } elsif ($key =~ /$arrayarrayexp/) {
+		foreach $k (@{$data->{$key}}) {
+		    &write_cfg_line ('COLCFG', [$key, @$k]);
+		}
+	    } elsif ($key =~ /$hashhashexp/) {
+		foreach $k (keys (%{$data->{$key}})) {
+		    foreach $p (keys (%{$data->{$key}->{$k}})) {
+			if ($p =~ /default/) {
+			    &write_cfg_line ('COLCFG', 
+					     [$key, $k, $data->{$key}->{$k}]);
+			}
+			else {
+			    &write_cfg_line ('COLCFG', 
+			       [$key, $k, $p, $data->{$key}->{$k}->{$p}]);
+			}
+		    }
+		}
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "cfgread::write_cfg_file couldn't write the cfg file $filename\n";
+    }
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify.pm	(revision 21112)
@@ -0,0 +1,289 @@
+###########################################################################
+#
+# classify.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# functions to handle classifiers
+
+package classify;
+
+require util;
+require AllList;
+use gsprintf;
+
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+$next_classify_num = 1;
+sub load_classifier_for_info {
+    my ($classifier) = shift @_;
+
+    # find the classifier
+    my $colclassname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},
+					   "perllib/classify", 
+					   "${classifier}.pm");
+    my $mainclassname = &util::filename_cat($ENV{'GSDLHOME'},
+					    "perllib/classify", 
+					    "${classifier}.pm");
+
+    if (-e $colclassname) { require $colclassname; }
+    elsif (-e $mainclassname) { require $mainclassname; }
+    else { 
+	&gsprintf(STDERR, "{classify.could_not_find_classifier}\n", $classifier) && die "\n";
+    }
+    my ($classobj);
+    my $options = "-gsdlinfo";
+    eval ("\$classobj = new \$classifier([],[$options])");
+    die "$@" if $@;
+
+    return $classobj;
+}
+
+sub load_classifiers {
+    my ($classify_list, $build_dir, $outhandle) = @_;
+    my @classify_objects = ();
+    my $classify_number  = 1;
+    
+    foreach $classifyoption (@$classify_list) {
+
+	# get the classifier name
+	my $classname = shift @$classifyoption;
+	next unless defined $classname;
+
+	# find the classifier
+	my $colclassname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/classify", 
+					       "${classname}.pm");
+	my $mainclassname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/classify", 
+						"${classname}.pm");
+
+	if (-e $colclassname) { require $colclassname; }
+	elsif (-e $mainclassname) { require $mainclassname; }
+	else { &gsprintf(STDERR, "{classify.could_not_find_classifier}\n", $classname) && die "\n";
+	       # die "ERROR - couldn't find classifier \"$classname\"\n";
+	   }
+
+	# create the classify object
+	my ($classobj);
+
+	my @newoptions;
+
+	# do these first so they can be overriden by user supplied options
+	push @newoptions, "-builddir", "$build_dir" if ($build_dir);
+	push @newoptions, "-outhandle", "$outhandle" if ($outhandle);
+	push @newoptions, "-verbosity", "2";
+
+	# backwards compatability hack: if the classifier options are 
+	# in "x=y" format, convert them to parsearg ("-x y") format.
+	my ($opt, $key, $value);
+	foreach $opt (@$classifyoption) {
+	    if ($opt =~ /^(\w+)=(.*)$/) {
+		push @newoptions, "-$1", $2;
+	    } else {
+		push @newoptions, $opt;
+	    }
+	}
+
+	map { $_ = "\"$_\""; } @newoptions;
+	my $options .= join (",", @newoptions);
+
+
+	eval ("\$classobj = new \$classname([],[$options])");
+	die "$@" if $@;
+
+	$classobj->set_number($classify_number);
+	$classify_number ++;
+
+       	# add this object to the list
+	push (@classify_objects, $classobj);
+    }
+
+    my ($classobj);
+    eval ("\$classobj = new AllList()");
+    die "$@" if $@;
+    push (@classify_objects, $classobj);
+
+    return \@classify_objects;
+}
+
+# init_classifiers resets all the classifiers and readys them to process
+# the documents.
+sub init_classifiers {
+    my ($classifiers) = @_;
+    
+    foreach $classobj (@$classifiers) {
+	$classobj->init();
+    }
+}
+
+# classify_doc lets each of the classifiers classify a document
+sub classify_doc {
+    my ($classifiers, $doc_obj) = @_;
+    
+    foreach $classobj (@$classifiers) {
+	my $title = $classobj->{'title'};
+	$classobj->classify($doc_obj);
+    }
+}
+
+# output_classify_info outputs all the info needed for the classification
+# to the gdbm
+sub output_classify_info {
+    my ($classifiers, $handle, $remove_empty_classifications, $gli) = @_;
+#    $handle = "main::STDOUT";
+
+    $gli = 0 unless defined $gli;
+
+    # create a classification containing all the info
+    my $classifyinfo = {'classifyOID'=>'browse',
+			'contains'=>[]};
+
+    # get each of the classifications
+    foreach $classobj (@$classifiers) {
+	my $tempinfo = $classobj->get_classify_info($gli);
+	my $classID = $tempinfo->{'classifyOID'};
+
+	$tempinfo->{'classifyOID'} = "CL$next_classify_num" unless defined($tempinfo->{'classifyOID'});
+	$next_classify_num++;
+	push (@{$classifyinfo->{'contains'}}, $tempinfo);
+    }
+
+    &print_classify_info ($handle, $classifyinfo, "", $remove_empty_classifications);
+}
+
+sub print_classify_info {
+    my ($handle, $classifyinfo, $OID, $remove_empty_classifications) = @_;
+
+    $OID =~ s/^\.+//; # just for good luck
+
+    # book information is printed elsewhere
+    return if (defined ($classifyinfo->{'OID'}));
+ 
+    # don't want empty classifications
+    return if (&check_contents ($classifyinfo, $remove_empty_classifications) == 0 && $remove_empty_classifications);
+   
+    $OID = $classifyinfo->{'classifyOID'} if defined ($classifyinfo->{'classifyOID'});
+	
+    my $outputtext = "[$OID]\n";
+    $outputtext .= "<doctype>classify\n";
+    $outputtext .= "<hastxt>0\n";
+    $outputtext .= "<childtype>$classifyinfo->{'childtype'}\n" 
+	if defined $classifyinfo->{'childtype'};
+    $outputtext .= "<Title>$classifyinfo->{'Title'}\n" 
+	if defined $classifyinfo->{'Title'};
+    $outputtext .= "<numleafdocs>$classifyinfo->{'numleafdocs'}\n" 
+	if defined $classifyinfo->{'numleafdocs'};
+    $outputtext .= "<thistype>$classifyinfo->{'thistype'}\n" 
+	if defined $classifyinfo->{'thistype'};
+    $outputtext .= "<parameters>$classifyinfo->{'parameters'}\n" 
+	if defined $classifyinfo->{'parameters'};
+    $outputtext .= "<supportsmemberof>$classifyinfo->{'supportsmemberof'}\n"
+	if defined $classifyinfo->{'supportsmemberof'};
+    
+    my $contains_text = "<contains>";
+    my $mdoffset_text = "<mdoffset>";
+    
+    my $next_subOID = 1;
+    my $first = 1;
+    foreach $tempinfo (@{$classifyinfo->{'contains'}}) {
+	# empty contents were made undefined by clean_contents()
+	next unless defined $tempinfo;
+	
+	if (!defined ($tempinfo->{'classifyOID'}) ||
+	    $tempinfo->{'classifyOID'} ne "oai") {
+	    $contains_text .= ";" unless $first;
+	}
+	$mdoffset_text .= ";" unless $first;
+	$first = 0;
+	
+	if (defined ($tempinfo->{'classifyOID'})) {
+	    if ($tempinfo->{'classifyOID'} ne "oai") {
+		$contains_text .= $tempinfo->{'classifyOID'};
+	    }
+	    &print_classify_info ($handle, $tempinfo, $tempinfo->{'classifyOID'},
+				  $remove_empty_classifications);
+	} elsif (defined ($tempinfo->{'OID'})) {
+	    $contains_text .= $tempinfo->{'OID'};
+	    $mdoffset_text .= $tempinfo->{'offset'}
+	    if (defined ($tempinfo->{'offset'}))
+		# note: we don't want to print the contents of the books
+	    } else {
+		
+		# Supress having top-level node in Collage classifier
+		# so no bookshelf icon appears, top-level, along with the
+		# applet
+		
+		if (!defined ($tempinfo->{'Title'}) || $tempinfo->{'Title'} ne "Collage") {
+		    $contains_text .= "\".$next_subOID"; 
+		}
+		
+		&print_classify_info ($handle, $tempinfo, "$OID.$next_subOID",
+				      $remove_empty_classifications);
+		$next_subOID++;
+	    }
+    }
+    
+    $outputtext .= "$contains_text\n";
+    $outputtext .= "<mdtype>$classifyinfo->{'mdtype'}\n" 
+	if defined $classifyinfo->{'mdtype'};
+    $outputtext .= "$mdoffset_text\n"
+	if ($mdoffset_text !~ m/^<mdoffset>;+$/);
+    
+    $outputtext .= '-' x 70 . "\n";
+    
+    print $handle $outputtext;
+
+}
+
+sub check_contents {
+    my ($classifyinfo,$remove_empty_classifications) = @_;
+    $remove_empty_classifications = 0 unless ($remove_empty_classifications);
+    my $num_leaf_docs = 0;
+    my $sub_num_leaf_docs = 0;
+
+    return $classifyinfo->{'numleafdocs'} if (defined $classifyinfo->{'numleafdocs'});
+
+    foreach $content (@{$classifyinfo->{'contains'}}) {
+	if (defined $content->{'OID'}) {
+	    # found a book
+	    $num_leaf_docs ++;
+	} elsif (($sub_num_leaf_docs = &check_contents ($content,$remove_empty_classifications)) > 0) {
+	    # there's a book somewhere below
+	    $num_leaf_docs += $sub_num_leaf_docs;
+	} else {
+	    if ($remove_empty_classifications){
+		# section contains no books so we want to remove 
+		# it from its parents contents
+		$content = undef;
+	    }
+	}
+    }
+
+    $classifyinfo->{'numleafdocs'} = $num_leaf_docs;
+    return $num_leaf_docs;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/AZCompactList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/AZCompactList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/AZCompactList.pm	(revision 21112)
@@ -0,0 +1,827 @@
+###########################################################################
+#
+# AZCompactList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# classifier plugin for sorting alphabetically
+# options are:
+
+package AZCompactList;
+
+use BasClas;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @AZCompactList::ISA = ('BasClas');
+}
+
+my $doclevel_list = 
+    [ {	'name' => "top",
+ 	'desc' => "{AZCompactList.doclevel.top}" },
+      { 'name' => "section",
+	'desc' => "{AZCompactList.doclevel.section}" } ];
+
+my $arguments = 
+    [ { 'name' => "metadata",
+	'desc' => "{AZCompactList.metadata}",
+	'type' => "metadata",
+	'reqd' => "yes" },
+      { 'name' => "firstvalueonly",
+	'desc' => "{AZCompactList.firstvalueonly}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "allvalues",
+	'desc' => "{AZCompactList.allvalues}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "buttonname",
+  	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "sort",
+	'desc' => "{AZCompactList.sort}",
+	'type' => "string",
+#	'deft' => "Title",
+	'reqd' => "no" },
+      { 'name' => "removeprefix",
+	'desc' => "{BasClas.removeprefix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "removesuffix",
+	'desc' => "{BasClas.removesuffix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "mingroup",
+	'desc' => "{AZCompactList.mingroup}",
+	'type' => "int",
+	'deft' => "2",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "minnesting",
+	'desc' => "{AZCompactList.minnesting}",
+	'type' => "int",
+	'deft' => "20",
+	'range' => "2,",
+	'reqd' => "no" },
+      { 'name' => "mincompact",
+	'desc' => "{AZCompactList.mincompact}",
+	'type' => "int",
+	'deft' => "10",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "maxcompact",
+	'desc' => "{AZCompactList.maxcompact}",
+	'type' => "int",
+	'deft' => "30",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "doclevel",
+	'desc' => "{AZCompactList.doclevel}",
+	'type' => "enum",
+	'list' => $doclevel_list,
+	'deft' => "top",
+	'reqd' => "no" },
+      { 'name' => "freqsort",
+	'desc' => "{AZCompactList.freqsort}",
+	'type' => "flag"},
+      { 'name' => "recopt",
+	'desc' => "{AZCompactList.recopt}",
+	'type' => "flag",
+	'reqd' => "no" } ];
+
+my $options = 
+{ 	'name'     => "AZCompactList",
+	'desc'     => "{AZCompactList.desc}",
+	'abstract' => "no",
+	'inherits' => "yes",
+	'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($pluginlist,$inputargs,$hashArgOptLists): new BasClas($pluginlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+    
+    if (!$self->{"metadata"}) {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "AZCompactList Error: required option -metadata not supplied\n";
+	$self->print_txt_usage("");  
+	die "AZCompactList Error: required option -metadata not supplied\n";
+    }
+
+    # Manually set $self parameters.
+    $self->{'list'} = {};
+    $self->{'listmetavalue'} = {};
+    $self->{'list_mvpair'} = {};
+    $self->{'reclassify'} = {};
+    $self->{'reclassifylist'} = {};
+
+    $self->{'buttonname'} = $self->generate_title_from_metadata($self->{'metadata'}) unless ($self->{'buttonname'});
+    
+    if (defined($self->{"removeprefix"}) && $self->{"removeprefix"}) {
+	$self->{"removeprefix"} =~ s/^\^//; # don't need a leading ^
+    }
+    if (defined($self->{"removesuffix"}) && $self->{"removesuffix"}) {
+	$self->{"removesuffix"} =~ s/\$$//; # don't need a trailing $
+    }
+	
+    $self->{'recopt'} = ($self->{'recopt'} == 0) ? undef : "on";
+
+    # Clean out the unused keys
+    if($self->{"removeprefix"} eq "") {delete $self->{"removeprefix"};}
+    if($self->{"removesuffix"} eq "") {delete $self->{"removesuffix"};}
+
+    return bless $self, $class;
+}
+
+sub init 
+{
+    my $self = shift (@_);
+    
+    $self->{'list'} = {};
+    $self->{'listmetavalue'} = {};
+    $self->{'list_mvpair'} = {};
+    $self->{'reclassify'} = {};
+    $self->{'reclassifylist'} = {};
+}
+
+my $tmp = 0;
+
+sub classify
+{
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    my @sectionlist = ();
+    my $topsection = $doc_obj->get_top_section();
+    my $metaname = $self->{'metadata'};
+    my $outhandle = $self->{'outhandle'};
+
+    $metaname =~ s/(\/|\|).*//; # grab first name in n1/n2/n3 or n1|n2|n3 list
+    my @commameta_list = split(/,/, $metaname);
+
+    if ($self->{'doclevel'} =~ /^top(level)?/i)
+    {
+	push(@sectionlist,$topsection);
+    }
+    else
+    {
+	my $thissection = $doc_obj->get_next_section($topsection);
+	while (defined $thissection) 
+	{
+	    push(@sectionlist,$thissection);
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+
+    my $thissection;
+    foreach $thissection (@sectionlist)
+    {
+	my $full_doc_OID 
+	    = ($thissection ne "") ? "$doc_OID.$thissection" : $doc_OID;
+	if (defined $self->{'list_mvpair'}->{$full_doc_OID}) 
+	{
+	    print $outhandle "WARNING: AZCompactList::classify called multiple times for $full_doc_OID\n";
+	} 
+	$self->{'list'}->{$full_doc_OID} = [];	
+	$self->{'listmetavalue'}->{$full_doc_OID} = [];
+	$self->{'list_mvpair'}->{$full_doc_OID} = [];
+
+	my $metavalues = [];
+	foreach my $cmn (@commameta_list) {
+	    my $cmvalues = $doc_obj->get_metadata($thissection,$cmn);
+	    push(@$metavalues,@$cmvalues) if (@{$cmvalues});
+	    last if (@{$cmvalues} && !$self->{'allvalues'});
+	}
+
+	my $metavalue;
+	foreach $metavalue (@$metavalues) 
+	{
+	    # Tidy up use of white space in metavalue for classifying
+	    $metavalue =~ s/^\s*//s;
+	    $metavalue =~ s/\s*$//s;
+	    $metavalue =~ s/\n/ /s;
+	    $metavalue =~ s/\s{2,}/ /s;
+
+	    # if this document doesn't contain the metadata element we're
+	    # sorting by we won't include it in this classification
+	    if (defined $metavalue && $metavalue =~ /\w/) 
+	    {
+		if (defined($self->{'removeprefix'}) &&
+		    length($self->{'removeprefix'})) {
+		    $metavalue =~ s/^$self->{'removeprefix'}//;
+
+		    # check that it's not now empty
+		    if (!$metavalue) {next;}
+		}
+
+		if (defined($self->{'removesuffix'}) &&
+		    length($self->{'removesuffix'})) {
+		    $metavalue =~ s/$self->{'removesuffix'}$//;
+
+		    # check that it's not now empty
+		    if (!$metavalue) {next;}
+		}
+
+		my $formatted_metavalue;
+		if ($self->{'no_metadata_formatting'}) {
+		    $formatted_metavalue = $metavalue;
+		} else {
+		    $formatted_metavalue = &sorttools::format_metadata_for_sorting($self->{'metadata'},	 $metavalue, $doc_obj);
+		}
+		
+		#### prefix-str
+		if (! defined($formatted_metavalue)) {
+		    print $outhandle "Warning: AZCompactList: metavalue is ";
+		    print $outhandle "empty\n";
+		    $formatted_metavalue="";
+		}
+
+		my $mv_pair = { 'mv' => $metavalue, 'fmv' => $formatted_metavalue };
+		push(@{$self->{'list'}->{$full_doc_OID}},$formatted_metavalue);
+		push(@{$self->{'listmetavalue'}->{$full_doc_OID}} ,$metavalue);
+		push(@{$self->{'list_mvpair'}->{$full_doc_OID}},$mv_pair);
+
+
+		last if ($self->{'firstvalueonly'});
+	    }
+	}
+
+	# This is used in reclassify below for AZCompactSectionList
+	my $sortmeta = $doc_obj->get_metadata_element($thissection, $self->{'sort'});
+	$self->{'reclassify'}->{$full_doc_OID} = [$doc_obj,$sortmeta];
+    }
+}
+
+sub reinit 
+{
+    my ($self,$classlist_ref) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    my %mtfreq = ();
+    my @single_classlist = ();
+    my @multiple_classlist = ();
+
+    # find out how often each metavalue occurs
+    map 
+    { 
+	foreach my $mvp (@{$self->{'list_mvpair'}->{$_}} )
+	{
+###	    print STDERR "*** plain mv = $mvp->{'mv'}\n";
+###	    print STDERR "*** format mv = $mvp->{'fmv'}\n";
+
+	    my $metavalue = $mvp->{'mv'};
+	    $metavalue =~ s!^-!\\-!; # in case it starts with "-"
+	    $mtfreq{$metavalue}++;
+	}
+    } @$classlist_ref;
+
+    # use this information to split the list: single metavalue/repeated value
+    map
+    { 
+	my $i = 1;
+	my $metavalue;
+	foreach my $mvp (@{$self->{'list_mvpair'}->{$_}})
+	{
+	    my $metavalue = $mvp->{'mv'};
+	    $metavalue =~ s!^-!\\-!; # in case it starts with "-"
+	    my $cs_metavalue = $mvp->{'mv'}; # case sensative
+	    if ($mtfreq{$metavalue} >= $self->{'mingroup'})
+	    {
+###		print STDERR "*** pushing on $cs_metavalue\n";
+		push(@multiple_classlist,[$_,$i,$metavalue]); 
+	    } 
+	    else
+	    {
+		push(@single_classlist,[$_,$cs_metavalue]); 
+		$metavalue =~ tr/[A-Z]/[a-z]/;
+		$self->{'reclassifylist'}->{"Metavalue_$i.$_"} = $metavalue;
+	    }
+	    $i++;
+	}
+    } @$classlist_ref;
+    
+    
+    # Setup sub-classifiers for multiple list
+
+    $self->{'classifiers'} = {};
+
+    my $pm;
+    foreach $pm ("List", "SectionList")
+    {
+	my $listname 
+	    = &util::filename_cat($ENV{'GSDLHOME'},"perllib/classify/$pm.pm");
+	if (-e $listname) { require $listname; }
+	else 
+	{ 
+	    print $outhandle "AZCompactList ERROR - couldn't find classifier \"$listname\"\n"; 
+	    die "\n";
+	}
+    }
+
+    # Create classifiers objects for each entry >= mingroup
+    my $metavalue;
+    my $doclevel = $self->{'doclevel'};
+    my $mingroup = $self->{'mingroup'};
+    my @metaname_list = split(/\/|\|/,$self->{'metadata'});
+    my $metaname = shift(@metaname_list);
+    my $hierarchical = 0;
+    if (scalar(@metaname_list) > 1) {
+	$hierarchical = 1;
+	$metaname = join('/',@metaname_list);
+    }
+    foreach $metavalue (keys %mtfreq)
+    {
+	if ($mtfreq{$metavalue} >= $mingroup)
+	{
+	    # occurs more often than minimum required to compact into a group
+	    my $listclassobj;
+	    
+	    if (!$hierarchical) 
+	    {	
+		my @args;
+		push @args, ("-metadata", "$metaname");
+		# buttonname is also used for the node's title
+		push @args, ("-buttonname", "$metavalue");
+		push @args, ("-sort", $self->{'sort'});
+
+		my $ptArgs = \@args;
+		if ($doclevel =~ m/^top(level)?/i)
+		{
+		    eval ("\$listclassobj = new List([],\$ptArgs)"); 
+		}
+		else
+		{
+		    eval ("\$listclassobj = new SectionList([],\$ptArgs)");
+		}
+	    }
+	    else
+	    {
+		
+		my @args;
+		push @args, ("-metadata", "$metaname");
+		# buttonname is also used for the node's title
+		push @args, ("-buttonname", "$metavalue");
+		push @args, ("-sort", $self->{'sort'});
+
+		if (defined $self->{'removeprefix'}) {
+		    push @args, ("-removeprefix", $self->{'removeprefix'});
+		}
+		if (defined $self->{'removesuffix'}) {
+		    push @args, ("-removesuffix", $self->{'removesuffix'});
+		}
+		
+		push @args, ("-doclevel", "$doclevel");
+		push @args, ("-mingroup", $mingroup);
+		push @args, "-recopt ";
+
+		my $ptArgs = \@args;
+		eval ("\$listclassobj = new AZCompactList([],\$ptArgs)");
+	    }
+	    
+	    if ($@) {
+		print $outhandle "$@";
+		die "\n";
+	    }
+	    
+	    $listclassobj->init();
+
+	    if (defined $metavalue && $metavalue =~ /\w/) 
+	    {
+		my $formatted_node = $metavalue;
+
+		if (defined($self->{'removeprefix'}) &&
+		    length($self->{'removeprefix'})) {
+		    $formatted_node =~ s/^$self->{'removeprefix'}//;
+		    # check that it's not now empty
+		    if (!$formatted_node) {next;}
+		}
+		if (defined($self->{'removesuffix'}) &&
+		    length($self->{'removesuffix'})) {
+		    $formatted_node =~ s/$self->{'removesuffix'}$//;
+		    # check that it's not now empty
+		    if (!$formatted_node) {next;}
+		}
+
+		$formatted_node = &sorttools::format_metadata_for_sorting($self->{'metadata'}, $formatted_node) unless $self->{'no_metadata_formatting'};
+
+		# In case our formatted string is empty...
+		if (! defined($formatted_node)) {
+		    print $outhandle "Warning: AZCompactList: metavalue is ";
+		    print $outhandle "empty\n";
+		    $formatted_node="";
+		}
+
+		# use the lower case, for speed of lookup.
+		my $meta_lc=lc($metavalue);
+		$self->{'classifiers'}->{$meta_lc}
+		= { 'classifyobj'   => $listclassobj,
+		    'formattednode' => $formatted_node };
+	    }
+	}
+    }
+
+
+    return (\@single_classlist,\@multiple_classlist);
+}
+
+
+sub reclassify 
+{
+    my ($self,$multiple_cl_ref) = @_;
+
+    # Entries in the current classify list that are "book nodes"
+    # should be recursively classified.
+    #--
+    foreach my $dm_pair (@$multiple_cl_ref) 
+    {
+	my ($doc_OID,$mdoffset,$metavalue,$cs_metavalue) = @$dm_pair;
+        my $listclassobj;
+
+	# find metavalue in list of sub-classifiers
+	# check if we have a key (lower case) for this metadata value
+	my $node_name=lc($metavalue);
+	if (exists $self->{'classifiers'}->{$node_name})
+	{
+	    my ($doc_obj, $sortmeta) = @{$self->{'reclassify'}->{$doc_OID}};
+
+	    # record the metadata value offset temporarily, so eg AZList can
+	    # get the correct metadata value (for multi-valued metadata fields)
+	    $doc_obj->{'mdoffset'}=$mdoffset;
+
+	    if ($doc_OID =~ m/^[^\.]*\.([\d\.]+)$/)
+	    {
+		my $section=$1;
+		if ($self->{'doclevel'} =~ m/^top/i) { # toplevel
+		    $self->{'classifiers'}->{$node_name}->{'classifyobj'}
+		    ->classify($doc_obj, "Section=$section");
+		} else { # section level
+		    # Thanks to Don Gourley for this...
+		    # classify can't handle multi-level section
+		    $self->{'classifiers'}->{$node_name}->{'classifyobj'}
+		    ->classify_section($section, $doc_obj, $sortmeta);
+		}
+	    }
+	    else
+	    {
+		$self->{'classifiers'}->{$node_name}->{'classifyobj'}
+		->classify($doc_obj); 
+	    }
+	} else { # this key is not in the hash
+	    my $outhandle=$self->{outhandle};
+	    print $outhandle "Warning: AZCompactList::reclassify ";
+	    print $outhandle "could not find sub-node for metadata=`$metavalue' with doc_OID $doc_OID\n";
+	}
+    }
+}
+
+
+
+sub get_reclassify_info 
+{
+    my $self = shift (@_);
+    
+    my $node_name;
+    foreach $node_name (keys %{$self->{'classifiers'}}) 
+    {
+        my $classifyinfo 
+	    = $self->{'classifiers'}->{$node_name}->{'classifyobj'}
+	        ->get_classify_info();
+        $self->{'classifiers'}->{$node_name}->{'classifyinfo'} 
+	    = $classifyinfo;
+        $self->{'reclassifylist'}->{"CLASSIFY.$node_name"} 
+	    = $self->{'classifiers'}->{$node_name}->{'formattednode'};
+    }
+}
+
+
+sub alpha_numeric_cmp
+{
+    my ($self,$a,$b) = @_;
+
+    my $title_a = $self->{'reclassifylist'}->{$a};
+    my $title_b = $self->{'reclassifylist'}->{$b};
+    
+    if ($title_a =~ m/^(\d+(\.\d+)?)/)
+    {
+	my $val_a = $1;
+	if ($title_b =~ m/^(\d+(\.\d+)?)/)
+	{
+	    my $val_b = $1;
+	    if ($val_a != $val_b)
+	    {
+		return ($val_a <=> $val_b);
+	    }
+	}
+    }
+    
+    return ($title_a cmp $title_b);
+}
+
+sub frequency_cmp
+{
+    my ($self,$a,$b) = @_;
+
+
+    my $title_a = $self->{'reclassifylist'}->{$a};
+    my $title_b = $self->{'reclassifylist'}->{$b};
+
+    my $a_freq = 1;
+    my $b_freq = 1;
+
+    if ($a =~ m/^CLASSIFY\.(.*)$/)
+    {
+	my $a_node = $1;
+	my $a_nodeinfo = $self->{'classifiers'}->{$a_node}->{'classifyinfo'};
+	$a_freq = scalar(@{$a_nodeinfo->{'contains'}});
+    }
+    
+    if ($b =~ m/^CLASSIFY\.(.*)$/)
+    {
+	my $b_node = $1;
+	my $b_nodeinfo = $self->{'classifiers'}->{$b_node}->{'classifyinfo'};
+	$b_freq = scalar(@{$b_nodeinfo->{'contains'}});
+    }
+
+    return $b_freq <=> $a_freq;
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist =keys %{$self->{'list_mvpair'}}; # list all doc oids
+
+    my ($single_cl_ref,$multiple_cl_ref) = $self->reinit(\@classlist);
+    $self->reclassify($multiple_cl_ref);
+    $self->get_reclassify_info();
+
+    my @reclassified_classlist;
+    if ($self->{'freqsort'})
+    {
+	@reclassified_classlist 
+	    = sort { $self->frequency_cmp($a,$b) } keys %{$self->{'reclassifylist'}};
+	# supress sub-grouping by alphabet
+	map { $self->{'reclassifylist'}->{$_} = "A".$self->{'reclassifylist'}; } keys %{$self->{'reclassifylist'}};
+    }
+    else
+    {
+#	@reclassified_classlist 
+#	    = sort {$self->{'reclassifylist'}->{$a} cmp $self->{'reclassifylist'}->{$b};} keys %{$self->{'reclassifylist'}};
+
+	# alpha_numeric_cmp is slower than "cmp" but handles numbers better ...
+
+	@reclassified_classlist
+	    = sort { $self->alpha_numeric_cmp($a,$b) } keys %{$self->{'reclassifylist'}};
+
+    }
+
+    return $self->splitlist (\@reclassified_classlist);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $metaname, $thistype) = @_;
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+                        'Title'=>$title,
+                        'contains'=>[],
+			'mdtype'=>$metaname);
+
+    $classifyinfo{'thistype'} = $thistype
+        if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+
+
+# splitlist takes an ordered list of classifications (@$classlistref) and 
+# splits it up into alphabetical sub-sections.
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my @metanames = split(/\/|\|/,$self->{'metadata'});
+    my $metaname = shift(@metanames);
+
+    my $childtype = "HList";
+    $childtype = "VList" if (scalar (@$classlistref) <= $self->{'minnesting'});
+
+    my $title = $self->{'buttonname'}; # should always be defined by now.
+    my $classifyinfo;
+    if (!defined($self->{'recopt'}))
+    {
+	$classifyinfo 
+	    = $self->get_entry ($title, $childtype, $metaname, "Invisible");
+    }
+    else
+    {
+	$classifyinfo 
+	    = $self->get_entry ($title, $childtype, $metaname, "VList");
+    }
+
+    # don't need to do any splitting if there are less than 'minnesting' classifications
+    if ((scalar @$classlistref) <= $self->{'minnesting'}) {
+	foreach my $subOID (@$classlistref) {
+            if ($subOID =~ /^CLASSIFY\.(.*)$/ 
+		&& defined $self->{'classifiers'}->{$1}) 
+	    {
+                push (@{$classifyinfo->{'contains'}}, 
+		      $self->{'classifiers'}->{$1}->{'classifyinfo'});
+            } 
+	    else 
+	    {
+		$subOID =~ s/^Metavalue_(\d+)\.//;
+		my $metaname_offset = $1 -1;
+		my $oid_rec = {'OID'=>$subOID, 'offset'=>$metaname_offset};
+		push (@{$classifyinfo->{'contains'}}, $oid_rec);
+	    }
+	}
+	return $classifyinfo;
+    }
+	
+    # first split up the list into separate A-Z and 0-9 classifications
+    foreach my $classification (@$classlistref) {
+	my $title = $self->{'reclassifylist'}->{$classification};
+	$title =~ s/&(.){2,4};//g; # remove any HTML special chars
+	$title =~ s/^\W+//g; # remove leading non-word chars
+
+	# only want first character for classification
+	$title =~ m/^(.)/; # always a char, or first byte of wide char?
+	if (defined($1) && $1 ne "") {
+	    $title=$1;
+	} else {
+	    print STDERR "no first character found for \"$title\" - \"" .
+		$self->{'reclassifylist'}->{$classification} . "\"\n";
+	}
+	$title =~ tr/[a-z]/[A-Z]/;
+
+	if ($title =~ /^[0-9]$/) {$title = '0-9';}
+	elsif ($title !~ /^[A-Z]$/) {
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "AZCompactList: WARNING $classification has badly formatted title ($title)\n";
+	}
+	$classhash->{$title} = [] unless defined $classhash->{$title};
+	push (@{$classhash->{$title}}, $classification);
+    }
+    $classhash = $self->compactlist ($classhash);
+
+    my @tmparr = ();
+    foreach my $subsection (sort keys (%$classhash)) {
+	push (@tmparr, $subsection);
+    }
+    
+    # if there's a 0-9 section it will have been sorted to the beginning
+    # but we want it at the end
+    if ($tmparr[0] eq '0-9') {
+	shift @tmparr;
+	push (@tmparr, '0-9');
+    }
+    foreach my $subclass (@tmparr) 
+    {
+	my $tempclassify 
+	    = (scalar(@tmparr)==1) 
+		? ($self->get_entry(" ", "VList", $metaname))
+		: ($self->get_entry($subclass, "VList", $metaname));
+
+
+	foreach my $subsubOID (@{$classhash->{$subclass}}) 
+	{
+            if ($subsubOID =~ /^CLASSIFY\.(.*)$/ 
+		&& defined $self->{'classifiers'}->{$1}) 
+	    {
+		# this is a "bookshelf" node... >1 entry compacted
+                push (@{$tempclassify->{'contains'}}, 
+		      $self->{'classifiers'}->{$1}->{'classifyinfo'});
+		# set the metadata field name, for mdoffset to work
+		$self->{'classifiers'}->{$1}->{'classifyinfo'}->{'mdtype'}=
+		    $metaname;
+	    }
+	    else
+	    {
+		$subsubOID =~ s/^Metavalue_(\d+)\.//;
+		# record the offset if this metadata type has multiple values
+		my $metaname_offset = $1 -1;
+		my $oid_rec = {'OID'=>$subsubOID, 'offset'=>$metaname_offset};
+		push (@{$tempclassify->{'contains'}}, $oid_rec);
+	    }
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";
+    my $currentlastletter = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = $self->{'mincompact'}; 
+    my $max = $self->{'maxcompact'};
+
+    foreach my $subsection (sort keys %$classhashref) {
+	if ($subsection eq '0-9') {
+	    @{$compactedhash->{$subsection}} = @{$classhashref->{$subsection}};
+	    next;
+	}
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+		$lastkey=$subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+
+    # don't add if there aren't any oids
+    if (! scalar (@currentOIDs)) {return $compactedhash;}
+
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(.)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/classify/AZCompactSectionList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/AZCompactSectionList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/AZCompactSectionList.pm	(revision 21112)
@@ -0,0 +1,234 @@
+###########################################################################
+#
+# AZCompactSectionList.pm --
+#
+# Experimental AZCompactList with fixes to handle section-level metadata
+#
+###########################################################################
+
+package AZCompactSectionList;
+
+use AZCompactList;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @AZCompactSectionList::ISA = ('AZCompactList');
+}
+
+my $arguments = [
+		 ];
+my $options = 
+{ 	'name'     => "AZCompactSectionList",
+	'desc'     => "{AZCompactSectionList.desc}",
+	'abstract' => "no",
+	'inherits' => "yes",
+	'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new AZCompactList($classifierslist,$inputargs,$hashArgOptLists): new AZCompactList($classifierslist,$inputargs);
+
+    return bless $self, $class;
+}
+
+#
+# override reinit() & reclassify() to demonstrate possible bug fixes
+# (search for SECTIONFIX? to see lines changed from AZCompactList.pm)
+#
+sub reinit 
+{
+    my ($self,$classlist_ref) = @_;
+    
+    my %mtfreq = ();
+    my @single_classlist = ();
+    my @multiple_classlist = ();
+
+    # find out how often each metavalue occurs
+    map 
+    { 
+	my $mv;
+	foreach $mv (@{$self->{'listmetavalue'}->{$_}} )
+	{
+	    $mtfreq{$mv}++; 
+	}
+    } @$classlist_ref;
+
+    # use this information to split the list: single metavalue/repeated value
+    map
+    { 
+	my $i = 1;
+	my $metavalue;
+	foreach $metavalue (@{$self->{'listmetavalue'}->{$_}})
+	{
+	    if ($mtfreq{$metavalue} >= $self->{'mingroup'})
+	    {
+		push(@multiple_classlist,[$_,$i,$metavalue]); 
+	    } 
+	    else
+	    {
+		push(@single_classlist,[$_,$metavalue]); 
+		$metavalue =~ tr/[A-Z]/[a-z]/;
+		$self->{'reclassifylist'}->{"Metavalue_$i.$_"} = $metavalue;
+	    }
+	    $i++;
+	}
+    } @$classlist_ref;
+    
+    
+    # Setup sub-classifiers for multiple list
+
+    $self->{'classifiers'} = {};
+
+    my $pm;
+    foreach $pm ("List", "SectionList")
+    {
+	my $listname 
+	    = &util::filename_cat($ENV{'GSDLHOME'},"perllib/classify/$pm.pm");
+	if (-e $listname) { require $listname; }
+	else 
+	{ 
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "AZCompactList ERROR - couldn't find classifier \"$listname\"\n"; 
+	    die "\n";
+	}
+    }
+
+    # Create classifiers objects for each entry >= mingroup
+    my $metavalue;
+    foreach $metavalue (keys %mtfreq)
+    {
+	if ($mtfreq{$metavalue} >= $self->{'mingroup'})
+	{
+	    my $listclassobj;
+	    my $doclevel = $self->{'doclevel'};
+	    my $metaname  = $self->{'metaname'};
+	    my @metaname_list = split('/',$metaname);
+	    $metaname = shift(@metaname_list);
+	    if (@metaname_list==0)
+	    {
+		my @args;
+		push @args, ("-metadata", "$metaname");
+# buttonname is also used for the node's title
+		push @args, ("-buttonname", "$metavalue");
+		push @args, ("-sort", $self->{'sort'});
+
+		my $ptArgs = \@args;
+		if ($doclevel =~ m/^top(level)?/i)
+		{
+		    eval ("\$listclassobj = new List([],\$ptArgs)"); warn $@ if $@;
+		}
+		else
+		{
+		    # SECTIONFIX?
+		    #eval ("\$listclassobj = new SectionList($args)");
+		    eval ("\$listclassobj = new SectionList([],\$ptArgs)");
+		}
+	    }
+	    else
+	    {
+		$metaname = join('/',@metaname_list);
+		
+		my @args;
+		push @args, ("-metadata", "$metaname");
+# buttonname is also used for the node's title
+		push @args, ("-buttonname", "$metavalue");
+		push @args, ("-doclevel", "$doclevel");
+		push @args, "-recopt";
+
+		# SECTIONFIX?
+		#eval ("\$listclassobj = new AZCompactList($args)");
+		my $ptArgs = \@args;
+		eval ("\$listclassobj = new AZCompactList([],\$ptArgs)");
+	    }
+	    if ($@) {
+		my $outhandle = $self->{'outhandle'};
+		print $outhandle "$@";
+		die "\n";
+	    }
+	    
+	    $listclassobj->init();
+
+	    if (defined $metavalue && $metavalue =~ /\w/) 
+	    {
+		my $formatted_node = $metavalue;
+		if ($self->{'metaname'} =~ m/^Creator(:.*)?$/)
+		{
+		    &sorttools::format_string_name_en(\$formatted_node);
+		} 
+		else 
+		{
+		    &sorttools::format_string_en(\$formatted_node);
+		}
+		
+		$self->{'classifiers'}->{$metavalue} 
+		= { 'classifyobj'   => $listclassobj,
+		    'formattednode' => $formatted_node };
+	    }
+	}
+    }
+
+
+    return (\@single_classlist,\@multiple_classlist);
+}
+
+
+sub reclassify 
+{
+    my ($self,$multiple_cl_ref) = @_;
+
+    # Entries in the current classify list that are "book nodes"
+    # should be recursively classified.
+    #--
+    foreach my $dm_pair (@$multiple_cl_ref) 
+    {
+	my ($doc_OID,$mdoffset,$metavalue) = @$dm_pair;
+        my $listclassobj;
+
+	# find metavalue in list of sub-classifiers
+	my $found = 0;
+	my $node_name;
+	foreach $node_name (keys %{$self->{'classifiers'}}) 
+	{
+	    my $resafe_node_name = $node_name;
+	    $resafe_node_name =~ s/(\(|\)|\[|\]|\{|\}|\^|\$|\.|\+|\*|\?|\|)/\\$1/g;
+	    if ($metavalue =~ m/^$resafe_node_name$/i) 
+	    {
+		my ($doc_obj, $sortmeta) = @{$self->{'reclassify'}->{$doc_OID}};
+
+		# SECTIONFIX?  section must include multiple levels, e.g. '1.12'
+		#if ($doc_OID =~ m/^.*\.(\d+)$/)
+		if ($doc_OID =~ m/^[^\.]*\.([\d\.]+)$/)
+		{
+		    $self->{'classifiers'}->{$node_name}->{'classifyobj'}
+		    # SECTIONFIX? classify can't handle multi-level section
+		    #->classify($doc_obj, "Section=$1"); 
+		    ->classify_section($1, $doc_obj, $sortmeta);
+		}
+		else
+		{
+		    $self->{'classifiers'}->{$node_name}->{'classifyobj'}
+		    ->classify($doc_obj); 
+		}
+		
+		$found = 1;
+		last;
+	    }
+	}
+	
+	if (!$found)
+	{
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "Warning: AZCompactList::reclassify ";
+	    print $outhandle "could not find sub-node for $metavalue with doc_OID $doc_OID\n";
+	}
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/AZList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/AZList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/AZList.pm	(revision 21112)
@@ -0,0 +1,368 @@
+###########################################################################
+#
+# AZList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting alphabetically
+
+package AZList;
+
+use BasClas;
+use sorttools;
+use iso639;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @AZList::ISA = ('BasClas');
+}
+
+
+my $arguments =
+    [ { 'name' => "metadata",
+	'desc' => "{AZList.metadata}",
+	'type' => "metadata",
+	'reqd' => "yes" } ,
+      { 'name' => "buttonname",
+  	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "removeprefix",
+	'desc' => "{BasClas.removeprefix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" } ,
+      { 'name' => "removesuffix",
+	'desc' => "{BasClas.removesuffix}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" } 
+      ];
+
+my $options = { 'name'     => "AZList",
+		'desc'     => "{AZList.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    if (!$self->{"metadata"}) {
+	print STDERR "AZList Error: required option -metadata not supplied \n";
+	$self->print_txt_usage("");  # Use default resource bundle
+	
+	die "AZList Error: required option -metadata not supplied\n";
+    }
+        
+    # Manually set $self parameters.
+    $self->{'list'} = {};
+
+    # Transfer value from Auto Parsing to the variavle name that used in previous GreenStone.
+    my @meta_list = split(/,/, $self->{"metadata"});
+    $self->{'meta_list'} = \@meta_list;
+
+    $self->{'buttonname'} = $self->generate_title_from_metadata($self->{'metadata'}) unless ($self->{'buttonname'});
+
+    # Further setup 
+    if (defined($self->{"removeprefix"}) && $self->{"removeprefix"}) {
+	$self->{"removeprefix"} =~ s/^\^//; # don't need a leading ^
+    }
+    if (defined($self->{"removesuffix"}) && $self->{"removesuffix"}) {
+	$self->{"removesuffix"} =~ s/\$$//; # don't need a trailing $
+    }
+
+    # Clean out the unused keys
+    delete $self->{"metadata"}; # Delete this key
+
+    if($self->{"removeprefix"} eq "") {delete $self->{"removeprefix"};}
+    if($self->{"removesuffix"} eq "") {delete $self->{"removesuffix"};}
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $outhandle = $self->{'outhandle'};
+
+    my $metavalue;
+    my $metaname;
+    # should we extend this to use all available metadata not just the first?
+    if (!defined $self->{'meta_list'}) {
+	# just in case
+	return;
+    }
+
+    # find the first available metadata
+    foreach my $m (@{$self->{'meta_list'}}) {
+	$metavalue = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $m);
+	$metaname = $m;
+	last if defined $metavalue;
+    } 
+    
+    #if we haven't found a metavalue here, then the doc shouldn't be included
+    if (!defined $metavalue || $metavalue eq "") {
+	print $outhandle "WARNING: AZList: $doc_OID metadata is empty - not classifying\n";
+	return;
+    }
+
+    if (defined($self->{'removeprefix'}) &&
+	length($self->{'removeprefix'})) {
+	$metavalue =~ s/^$self->{'removeprefix'}//;
+    }
+    if (defined($self->{'removesuffix'}) &&
+	length($self->{'removesuffix'})) {
+	$metavalue =~ s/$self->{'removesuffix'}$//;
+    }
+    
+    
+    $metavalue = &sorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj) unless $self->{'no_metadata_formatting'};
+	
+    if (defined $self->{'list'}->{$doc_OID}) {
+	print $outhandle "WARNING: AZList::classify called multiple times for $doc_OID\n";
+    } 
+    if ($metavalue) {
+	$self->{'list'}->{$doc_OID} = $metavalue;
+    } else {
+	# the formatting has made it empty
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "WARNING: AZList: $doc_OID metadata has become empty - not classifying\n";
+    }
+    
+}
+
+sub alpha_numeric_cmp
+{
+    my ($self,$a,$b) = @_;
+
+    my $title_a = $self->{'list'}->{$a};
+    my $title_b = $self->{'list'}->{$b};
+
+    if ($title_a =~ m/^(\d+(\.\d+)?)/)
+    {
+	my $val_a = $1;
+	if ($title_b =~ m/^(\d+(\.\d+)?)/)
+	{
+	    my $val_b = $1;
+	    if ($val_a != $val_b)
+	    {
+		return ($val_a <=> $val_b);
+	    }
+	}
+    }
+    
+    return ($title_a cmp $title_b);
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist 
+	= sort { $self->alpha_numeric_cmp($a,$b) } keys %{$self->{'list'}};
+
+    return $self->splitlist (\@classlist);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and splits it
+# up into alphabetical sub-sections.
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $childtype = "HList";
+    if (scalar (@$classlistref) <= 39) {$childtype = "VList";}
+    my $classifyinfo = $self->get_entry ($self->{'buttonname'}, $childtype, "Invisible");
+
+    # don't need to do any splitting if there are less than 39 (max + min -1) classifications
+    if ((scalar @$classlistref) <= 39) {
+	foreach my $subOID (@$classlistref) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	return $classifyinfo;
+    }
+	
+    # first split up the list into separate A-Z and 0-9 classifications
+    foreach my $classification (@$classlistref) {
+	my $title = $self->{'list'}->{$classification};
+
+	$title =~ s/^(&.{1,6};|<[^>]>|[^a-zA-Z0-9])//g; # remove any unwanted stuff
+	# only need first char for classification
+	$title =~ m/^(.)/; $title=$1;
+	$title =~ tr/[a-z]/[A-Z]/;
+	if ($title =~ /^[0-9]$/) {$title = '0-9';}
+	elsif ($title !~ /^[A-Z]$/) {
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "AZList: WARNING $classification has badly formatted title ($title)\n";
+	}
+	$classhash->{$title} = [] unless defined $classhash->{$title};
+	push (@{$classhash->{$title}}, $classification);
+    }
+    $classhash = $self->compactlist ($classhash);
+
+    my @tmparr = ();
+    foreach my $subsection (sort keys (%$classhash)) {
+	push (@tmparr, $subsection);
+    }
+    #if there is only one entry here, we suppress the buckets
+    if ((scalar @tmparr) == 1) {
+	$classifyinfo->{'childtype'} = "VList";
+	foreach my $OID (@{$classhash->{$tmparr[0]}}) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$OID});
+	}
+	return $classifyinfo;
+    }
+    
+    # if there's a 0-9 section it will have been sorted to the beginning
+    # but we want it at the end
+    if ($tmparr[0] eq '0-9') {
+	shift @tmparr;
+	push (@tmparr, '0-9');
+    }
+
+    foreach my $subclass (@tmparr) {
+	my $tempclassify = $self->get_entry($subclass, "VList");
+	foreach my $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";  # start of working bin
+    my $currentlastletter = "";   # end of working bin
+    my $lastkey = "";             # the name of the last completed key
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+
+    foreach my $subsection (sort keys %$classhashref) {
+	if ($subsection eq '0-9') {
+	    # leave this bin as-is... copy it straight across
+	    @{$compactedhash->{$subsection}} = @{$classhashref->{$subsection}};
+	    next;
+	}
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    # add this letter to the bin and continue
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+	    # too many or too few for a separate bin
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		# this key is now complete. Start a new one
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+		$lastkey = $subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    # BUG FIX: don't add anything if there are no currentOIDs (thanks to Don Gourley)
+    if (! scalar(@currentOIDs)) {return $compactedhash;}
+
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(.)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	}
+	else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} =
+		@currentOIDs;
+	}
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/AZSectionList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/AZSectionList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/AZSectionList.pm	(revision 21112)
@@ -0,0 +1,124 @@
+###########################################################################
+#
+# AZSectionList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting sections alphabetically
+
+# this is very similar to AZList except it sorts by
+# section level metadata (excluding the top level)
+# instead of just top level metadata
+
+# the only change is to the classify() subroutine which 
+# must now iterate through each section, adding each
+# to the classification
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package AZSectionList;
+
+use AZList;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @AZSectionList::ISA = ('AZList');
+}
+
+my $arguments = [
+		 ];
+my $options = { 'name'     => "AZSectionList",
+		'desc'     => "{AZSectionList.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" }; 
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new AZList($classifierslist,$inputargs,$hashArgOptLists): new AZList($classifierslist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $thissection = $doc_obj->get_next_section ($doc_obj->get_top_section());
+
+    while (defined $thissection) {
+	$self->classify_section ($thissection, $doc_obj);
+	$thissection = $doc_obj->get_next_section ($thissection);
+    }
+}
+
+sub classify_section {
+    my $self = shift (@_);
+    my ($section, $doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    my $metavalue;
+    my $metaname;
+
+    if (!defined $self->{'meta_list'}) {
+	# just in case
+	return;
+    }
+
+    # find the first available metadata
+    foreach my $m (@{$self->{'meta_list'}}) {
+	$metavalue = $doc_obj->get_metadata_element($section, $m);
+	$metaname = $m;
+	last if defined $metavalue;
+    } 
+
+    # if this section doesn't contain the metadata element we're
+    # sorting by we won't include it in this classification
+
+    if (defined $metavalue && $metavalue ne "") {
+	if ($self->{'removeprefix'}) {
+	    $metavalue =~ s/^$self->{'removeprefix'}//;
+	}
+	
+	$metavalue = &sorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj) unless $self->{'no_metadata_formatting'};
+	if (defined $self->{'list'}->{"$doc_OID.$section"}) {
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "WARNING: AZSectionList::classify called multiple times " .
+		"for $doc_OID.$section\n";
+	} 
+	$self->{'list'}->{"$doc_OID.$section"} = $metavalue;
+    }
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/AllList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/AllList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/AllList.pm	(revision 21112)
@@ -0,0 +1,85 @@
+package AllList;
+
+use BasClas;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @AllList::ISA = ('BasClas');
+}
+
+my $arguments = 
+    [
+     ];
+
+my $options = { 'name'     => "AllList",
+	        'abstract' => "no",
+	        'inherits' => "yes" };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+    
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Manually set $self parameters.
+    $self->{'list'} = [];
+    $self->{'buttonname'} = "All Documents";
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    
+    push (@{$self->{'list'}}, $doc_OID);
+
+    return;
+}
+
+sub get_classify_info {
+    my $self = shift(@_);
+    my ($no_thistype) = @_;
+
+    my %classifyinfo = ('childtype'   =>'VList',
+			'Title'       =>$self->{'buttonname'},
+			'contains'    =>[],
+			'classifyOID' =>"oai");
+    $classifyinfo{'thistype'} = 'Invisible';
+    my @list = @{$self->{'list'}};
+
+    my $seqNo = 0;
+    foreach my $OID (@list) {
+	my $hashref={};
+	$hashref->{'OID'}=$OID;
+       
+	my %tempinfo=('childtype'=>'VList',
+		      'Title'=>$self->{'buttonname'},
+		      'classifyOID' =>"oai.$seqNo",
+		      'contains'    =>[]);
+
+	push (@{$tempinfo{'contains'}}, $hashref);
+
+	push (@{$classifyinfo{'contains'}}, \%tempinfo);
+	$seqNo ++;
+    }
+
+    return \%classifyinfo;
+}
Index: /main/tags/3.01/gsdl/perllib/classify/BasClas.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/BasClas.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/BasClas.pm	(revision 21112)
@@ -0,0 +1,330 @@
+###########################################################################
+#
+# BasClas.pm -- base class for all classifiers
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package BasClas;
+
+# How a classifier works.  
+#
+# For each classifier requested in the collect.cfg file, buildcol.pl creates
+# a new classifier object (a subclass of BasClas).  Later, it passes each 
+# document object to each classifier in turn for classification.
+#
+# Four primary functions are used:
+#
+# 1. "new" is called before the documents are processed to set up the
+#    classifier.
+#
+# 2. "init" is called after buildcol.pl has created the indexes etc but
+#    before the documents are classified in order that the classifier might
+#    set any variables it requires, etc.
+#
+# 3. "classify" is called once for each document object.  The classifier
+#    "classifies" each document and updates its local data accordingly.
+#
+# 4. "get_classify_info" is called after every document has been
+#    classified.  It collates the information about the documents and
+#    stores a reference to the classifier so that Greenstone can later
+#    display it.
+
+# 09/05/02 Added usage datastructure - John Thompson
+# 28/11/03 Commented out verbosity argument - John Thompson
+
+use gsprintf;
+use printusage;
+use parse2;
+
+# suppress the annoying "subroutine redefined" warning that various
+# classifiers cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use strict;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+my $arguments = 
+    [ 
+      { 'name' => "no_metadata_formatting",
+        'desc' => "{BasClas.no_metadata_formatting}",
+        'type' => "flag" },
+      { 'name' => "builddir",
+	'desc' => "{BasClas.builddir}",
+	'type' => "string",
+	'deft' => "" },
+      { 'name' => "outhandle",
+	'desc' => "{BasClas.outhandle}",
+	'type' => "string",
+	'deft' => "STDERR" },
+      { 'name' => "verbosity",
+	'desc' => "{BasClas.verbosity}",
+#	'type' => "enum",
+	'type' => "int",
+	'deft' => "2",
+	'reqd' => "no" }
+      
+#      { 'name' => "ignore_namespace",
+#	'desc' => "{BasClas.ignore_namespace}",
+#	'type' => "flag"} 
+      ];
+
+my $options = { 'name'     => "BasClas",
+		'desc'     => "{BasClas.desc}",
+		'abstract' => "yes",
+		'inherits' => "no",
+		'args'     => $arguments };
+
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+sub print_xml_usage
+{
+    my $self = shift(@_);
+
+    # XML output is always in UTF-8
+    &gsprintf::output_strings_in_UTF8;
+
+    &PrintUsage::print_xml_header();
+    $self->print_xml();
+}
+
+
+sub print_xml
+{
+    my $self = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $classifieroptions = shift(@$optionlistref);
+    return if (!defined($classifieroptions));
+
+    &gsprintf(STDERR, "<ClassInfo>\n");
+    &gsprintf(STDERR, "  <Name>$classifieroptions->{'name'}</Name>\n");
+    my $desc = &gsprintf::lookup_string($classifieroptions->{'desc'});
+    $desc =~ s/</&amp;lt;/g; # doubly escaped
+    $desc =~ s/>/&amp;gt;/g;
+    &gsprintf(STDERR, "  <Desc>$desc</Desc>\n");
+    &gsprintf(STDERR, "  <Abstract>$classifieroptions->{'abstract'}</Abstract>\n");
+    &gsprintf(STDERR, "  <Inherits>$classifieroptions->{'inherits'}</Inherits>\n");
+    &gsprintf(STDERR, "  <Arguments>\n");
+    if (defined($classifieroptions->{'args'})) {
+	&PrintUsage::print_options_xml($classifieroptions->{'args'});
+    }
+
+    # Recurse up the classifier hierarchy
+    $self->print_xml();
+
+    &gsprintf(STDERR, "  </Arguments>\n");
+    &gsprintf(STDERR, "</ClassInfo>\n");
+}
+
+
+sub print_txt_usage
+{
+    my $self = shift(@_);
+
+    # Print the usage message for a classifier (recursively)
+    my $descoffset = $self->determine_description_offset(0);
+    $self->print_classifier_usage($descoffset, 1);
+}
+
+
+sub determine_description_offset
+{
+    my $self = shift(@_);
+    my $maxoffset = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $classifieroptions = pop(@$optionlistref);
+    return $maxoffset if (!defined($classifieroptions));
+
+    # Find the length of the longest option string of this classifier
+    my $classifierargs = $classifieroptions->{'args'};
+    if (defined($classifierargs)) {
+	my $longest = &PrintUsage::find_longest_option_string($classifierargs);
+	if ($longest > $maxoffset) {
+	    $maxoffset = $longest;
+	}
+    }
+
+    # Recurse up the classifier hierarchy
+    $maxoffset = $self->determine_description_offset($maxoffset);
+    $self->{'option_list'} = \@optionlist;
+    return $maxoffset;
+}
+
+
+sub print_classifier_usage
+{
+    my $self = shift(@_);
+    my $descoffset = shift(@_);
+    my $isleafclass = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $classifieroptions = shift(@$optionlistref);
+    return if (!defined($classifieroptions));
+
+    my $classifiername = $classifieroptions->{'name'};
+    my $classifierargs = $classifieroptions->{'args'};
+    my $classifierdesc = $classifieroptions->{'desc'};
+    # Produce the usage information using the data structure above
+    if ($isleafclass) {
+	if (defined($classifierdesc)) {
+	    &gsprintf(STDERR, "$classifierdesc\n\n");
+	}
+	&gsprintf(STDERR, " {common.usage}: classify $classifiername [{common.options}]\n\n");
+	
+    }
+
+    # Display the classifier options, if there are some
+    if (defined($classifierargs)) {
+	# Calculate the column offset of the option descriptions
+	my $optiondescoffset = $descoffset + 2;  # 2 spaces between options & descriptions
+
+	if ($isleafclass) {
+	    &gsprintf(STDERR, " {common.specific_options}:\n");
+	}
+	else {
+	    &gsprintf(STDERR, " {common.general_options}:\n", $classifiername);
+	}
+
+	# Display the classifier options
+	&PrintUsage::print_options_txt($classifierargs, $optiondescoffset);
+    }
+
+    # Recurse up the classifier hierarchy
+    $self->print_classifier_usage($descoffset, 0);
+    $self->{'option_list'} = \@optionlist;
+}
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$args,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+    my $classifier_name = (defined $classifierslist->[0]) ? $classifierslist->[0] : $class;
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+
+    # Manually set $self parameters.
+    my $self = {};
+    $self->{'outhandle'} = STDERR;
+    $self->{'idnum'} = -1;
+    $self->{'option_list'} = $hashArgOptLists->{"OptList"};
+    $self->{"info_only"} = 0;
+
+    # Check if gsdlinfo is in the argument list or not - if it is, don't parse 
+    # the args, just return the object.  
+    foreach my $strArg (@{$args})
+    {
+	if($strArg eq "-gsdlinfo")
+	{
+	    $self->{"info_only"} = 1;
+	    return bless $self, $class;
+	}
+    }
+
+    # general options available to all classifiers
+    if(!parse2::parse($args,$hashArgOptLists->{"ArgList"},$self))
+    {
+	#print out the text usage of this classifier. 
+	my $classTempClass = bless $self, $class;
+	print STDERR "<BadClassifier c=$classifier_name>\n";
+
+	&gsprintf(STDERR, "\n{BasClas.bad_general_option}\n", $classifier_name);
+	$classTempClass->print_txt_usage("");  # Use default resource bundle
+	die "\n";
+    }
+    
+    delete $self->{"info_only"};
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'supportsmemberof'} = &supports_memberof();
+}
+
+sub set_number {
+    my $self = shift (@_);
+    my ($id)   = @_;
+    $self->{'idnum'} = $id;
+}
+
+sub get_number {
+    my $self = shift (@_);
+    return $self->{'idnum'};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    &gsprintf($outhandle, "BasClass::classify {common.must_be_implemented}\n");
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my $outhandle = $self->{'outhandle'};
+    &gsprintf($outhandle, "BasClass::get_classify_info {common.must_be_implemented}\n");
+}
+
+sub supports_memberof {
+    my $self = shift(@_);
+
+    return "false";
+}
+
+# previously, if a buttonname wasn't specified, we just use the metadata value,
+# but with a list of metadata, we want to do something a bit nicer so that
+# eg -metadata dc.Title,Title will end up with Title as the buttonname
+
+# current algorithm - use the first element, but strip its namespace
+sub generate_title_from_metadata {
+    
+    my $self = shift (@_);
+    my $metadata = shift (@_);
+
+    return "" unless defined $metadata && $metadata =~ /\S/;
+    
+    my @metalist = split(/,/, $metadata);
+    my $firstmeta = $metalist[0];
+    if ($firstmeta =~ /\./) {
+	$firstmeta =~ s/^\w+\.//;
+    }
+    return $firstmeta; 
+}
+
+1; 
Index: /main/tags/3.01/gsdl/perllib/classify/Browse.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/Browse.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/Browse.pm	(revision 21112)
@@ -0,0 +1,101 @@
+###########################################################################
+#
+# List.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package Browse;
+
+use BasClas;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @Browse::ISA = ('BasClas');
+}
+
+my $arguments = [
+		 ];
+my $options = { 'name'     => "Browse",
+		'desc'     => "{Browse.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes" };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Manually set $self parameters.
+    $self->{'collection'} = $ENV{'GSDLCOLLECTION'}; # classifier information
+    $self->{'buttonname'} = "Browse";
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    
+}
+
+sub classify {
+    my $self = shift (@_);
+   
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+  
+
+    # Return the information about the classifier that we'll later want to
+    # use to create macros when the Phind classifier document is displayed.
+    my %classifyinfo = ('thistype'=>'Invisible', 
+                        'Title'=>$self->{'buttonname'},
+			'contains'=>[]);
+    
+    my $collection = $self->{'collection'};
+    my $url = "library?a=br&c=collection";
+    push (@{$classifyinfo{'contains'}}, {'OID'=>$url});
+   
+    return \%classifyinfo;
+
+ 
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/Collage.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/Collage.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/Collage.pm	(revision 21112)
@@ -0,0 +1,263 @@
+##########################################################################
+#
+# Collage.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package Collage;
+
+use BasClas;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @Collage::ISA = ('BasClas');
+}
+
+my $arguments = 
+    [ { 'name' => "buttonname",
+  	'desc' => "{Collage.buttonname}",
+	'type' => "string",
+	'deft' => "Collage",
+	'reqd' => "no" }, 
+      { 'name' => "geometry",
+  	'desc' => "{Collage.geometry}",
+	'type' => "string",
+	'deft' => "600x300",
+	'reqd' => "no" },
+      { 'name' => "verbosity",
+  	'desc' => "{BasClas.verbosity}",
+	'type' => "string",
+	'deft' => "5",
+	'reqd' => "no" },
+      { 'name' => "maxDepth",
+  	'desc' => "{Collage.maxDepth}",
+	'type' => "string",
+	'deft' => "500"},
+#      { 'name' => "maxDownloads",
+#  	'desc' => "{BasClas.maxDownloads}",
+#	'type' => "string",
+#	'deft' => "",
+#	'reqd' => "no" },
+      { 'name' => "maxDisplay",
+  	'desc' => "{Collage.maxDisplay}",
+	'type' => "string",
+	'deft' => "25",
+	'reqd' => "no" },
+      { 'name' => "imageType",
+  	'desc' => "{Collage.imageType}",
+	'type' => "string",
+	'deft' => ".jpg%%.png",
+	'reqd' => "no" },
+      { 'name' => "bgcolor",
+  	'desc' => "{Collage.bgcolor}",
+	'type' => "string",
+	'deft' => "#96c29a",
+	'reqd' => "no" },
+      { 'name' => "refreshDelay",
+  	'desc' => "{Collage.refreshDelay}",
+	'type' => "string",
+	'deft' => "1500",
+	'reqd' => "no" },
+      { 'name' => "isJava2",
+  	'desc' => "{Collage.isJava2}",
+	'type' => "string",
+	'deft' => "auto",
+	'reqd' => "no" },
+      { 'name' => "imageMustNotHave",
+  	'desc' => "{Collage.imageMustNotHave}",
+	'type' => "string",
+	'deft' => "hl=%%x=%%gt=%%gc=%%.pr",
+	'reqd' => "no" },
+      { 'name' => "caption",
+  	'desc' => "{Collage.caption}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" }
+      ];
+
+
+
+
+my $options = { 'name'     => "Collage",
+		'desc'     => "{Collage.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+    
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Manually set $self parameters.
+    $self->{'list'} = {};
+    $self->{'buttonname'} = 'Collage' unless ($self->{'buttonname'});
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = [];
+}
+
+sub classify {
+   
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $has_image_type = 0;
+
+    my @image_type_split = split(/%/,$self->{'imageType'});
+    my @image_type_ext = map { "($_)" } @image_type_split;
+    my $image_type_re = join("|",@image_type_ext);
+    $image_type_re =~ s/\./\\\./g;
+
+    my $assoc_files = $doc_obj->{'associated_files'};
+
+    foreach my $af ( @$assoc_files ) {
+	my ($real_filename, $assoc_filename, $mime_type, $section) = @$af;
+	if ($assoc_filename =~ m/$image_type_re/) {
+	    $has_image_type = 1;
+	    last;
+	}
+    }
+
+    if ($has_image_type) {
+	my $doc_OID = $doc_obj->get_OID();
+	push (@{$self->{'list'}}, $doc_OID);
+    }
+
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+    
+    my $items_per_page = 2; 
+    my $max_items_per_page = 20;
+
+    my @list      = @{$self->{'list'}};
+
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    if ($verbosity>1) {
+	print $outhandle ("$self->{'buttonname'}\n");
+    }
+
+    my $collage_head = $self->get_entry ($self->{'buttonname'}, "Collage", "Invisible");
+    my $collage_curr = $self->get_entry("Collage","VList");
+    push (@{$collage_head->{'contains'}},$collage_curr);
+
+    my $global_c=1;
+    my $within_page_c=1;
+
+    foreach my $oid (@list) {
+	if ($within_page_c>$items_per_page) {
+	    my $title = "Items $global_c+";
+	    my $nested_node = $self->get_entry($title,"VList");
+	    push (@{$collage_curr->{'contains'}}, $nested_node);
+	    $collage_curr = $nested_node;
+
+	    $within_page_c=1;
+	    
+	    $items_per_page++ if ($items_per_page < $max_items_per_page);
+	}
+
+	push (@{$collage_curr->{'contains'}}, {'OID'=>$oid});
+	$global_c++;
+	$within_page_c++;
+    }
+   
+    return $collage_head;
+}
+
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+
+    $classifyinfo{'thistype'} = $thistype if (defined $thistype);
+
+    if ($childtype eq "Collage") {
+	my $geometry = $self->{'geometry'};
+	my ($x_dim,$y_dim) = ($geometry =~ m/^(.*)x(.*)$/);
+	my $verbosity = $self->{'verbosity'};
+	my $maxDepth = $self->{'maxDepth'};
+#	my $maxDownloads = $self->{'maxDownloads'};
+	my $maxDisplay = $self->{'maxDisplay'};
+	my $imageType = $self->{'imageType'};
+	my $bgcolor = $self->{'bgcolor'};
+	my $refreshDelay = $self->{'refreshDelay'};
+	my $isJava2 = $self->{'isJava2'};
+	my $imageMustNotHave = $self->{'imageMustNotHave'};
+	my $caption = $self->{'caption'};
+	
+	#if (!defined($maxDownloads)) {
+	#    $maxDownloads="";
+	#}
+
+	my $parameters;
+
+	$parameters = "xdim=".$x_dim;
+	$parameters .= ";ydim=".$y_dim;
+	$parameters .= ";geometry=".$self->{'geometry'};
+	$parameters .= ";verbosity=".$self->{'verbosity'};
+	$parameters .= ";maxDepth=".$self->{'maxDepth'};
+#	$parameters .= ";maxDownloads=".$maxDownloads;
+	$parameters .= ";maxDisplay=".$self->{'maxDisplay'};
+	$parameters .= ";imageType=".$self->{'imageType'};
+	$parameters .= ";bgcolor=".$self->{'bgcolor'};
+	$parameters .= ";refreshDelay=".$self->{'refreshDelay'};
+	$parameters .= ";isJava2=".$self->{'isJava2'};
+	$parameters .= ";caption=".$self->{'caption'};
+
+#	$parameters .= ";imageMustNotHave=".$self->{'imageMustNotHave'};
+
+    
+	$classifyinfo{'parameters'} = $parameters;
+    }
+
+    return \%classifyinfo;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/DateList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/DateList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/DateList.pm	(revision 21112)
@@ -0,0 +1,332 @@
+###########################################################################
+#
+# DateList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting by date
+
+# date is assumed to be in the form yyyymmdd
+
+# at present dates are split by year - this should change
+# jrm21 - added option "bymonth", which splits by year and month.
+
+# 23/09/03 Added some more options -kjdon.
+# these include:
+# -nogroup, which makes each year (or year+month) an individual entry in 
+# the horizontal list and prevents compaction
+# -metadata, use a different metadata for the date (instead of Date), still expects yyyymmdd format. this affects display cos greenstone displays Date metadata as dd month yyyy, whereas any other date metadata is displayed as yyyymmdd - this needs fixing
+# -sort specifies an additional metadata to use in sorting, will take affect when two docs have the same date.
+
+package DateList;
+
+use BasClas;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @DateList::ISA = ('BasClas');
+}
+
+my $arguments =
+    [ { 'name' => "metadata",
+        'desc' => "{DateList.metadata}",
+        'type' => "metadata",
+	'deft' => "Date",
+        'reqd' => "no" } ,
+      { 'name' => "sort",
+        'desc' => "{DateList.sort}",
+        'type' => "metadata",
+	'reqd' => "no" } ,
+      { 'name' => "reverse_sort",
+	'desc' => "{DateList.reverse_sort}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "bymonth",
+	'desc' => "{DateList.bymonth}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "nogroup",
+	'desc' => "{DateList.nogroup}",
+	'type' => "flag",
+	'reqd' => "no" }
+      ];
+
+my $options = { 'name'     => "DateList",
+		'desc'     => "{DateList.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Manually set $self parameters.
+    $self->{'list'} = {};
+
+    if (!defined $self->{"metadata"} || $self->{"metadata"} eq "") {
+	$self->{'metadata'} = "Date";
+    }
+    
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $date = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), $self->{'metadata'});
+    
+    my $sort_other = "";
+    if (defined $self->{'sort'} && $self->{'sort'} ne "") {
+	$sort_other = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), $self->{'sort'});
+	$sort_other = &sorttools::format_metadata_for_sorting($self->{'sort'}, $sort_other, $doc_obj) unless $self->{'no_metadata_formatting'};
+    }
+    # if this document doesn't contain Date element we won't 
+    # include it in this classification
+    if (defined $date && $date =~ /\d/) {
+	if (defined $self->{'list'}->{$doc_OID}) {
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "WARNING: DateList::classify called multiple times for $doc_OID\n";
+	} 
+
+	$self->{'list'}->{$doc_OID} = "$date$sort_other";
+    }
+}
+
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist = sort {$self->{'list'}->{$a} cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+    if ($self->{'reverse_sort'}) {
+	@classlist = reverse @classlist;
+    }
+
+    return $self->splitlist (\@classlist);
+}
+
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[],
+			'mdtype'=>$self->{'metadata'});
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and
+# splits it up into sub-sections by date
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $childtype = "HList";
+
+    if (scalar (@$classlistref) <= 39 &&
+	!$self->{'nogroup'}) {$childtype = "DateList";}
+    my $classifyinfo = $self->get_entry ("Date", $childtype, "Invisible");
+    # don't need to do any splitting if there are less than 39 (max + min -1)
+    # classifications, unless nogroup is specified
+    if ((scalar @$classlistref) <= 39 && !$self->{'nogroup'}) {
+	foreach my $subOID (@$classlistref) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	return $classifyinfo;
+    }
+
+
+    if ($self->{'bymonth'}) {
+	# first split up the list into separate year+month classifications
+
+	if (!$self->{'nogroup'}) { # hlist of year+month pairs
+	    # single level of classifications
+	    foreach my $classification (@$classlistref) {
+		my $date = $self->{'list'}->{$classification};
+		$date =~ s/^(\d\d\d\d)(\d\d).*$/$1&nbsp;_textmonth$2_/;
+		# sanity check if month is zero
+		if ($date =~ /00_$/) {
+		    $date =~ s/^(\d\d\d\d).*$/$1/g;
+		}
+		$classhash->{$date} = [] unless defined $classhash->{$date};
+		push (@{$classhash->{$date}}, $classification);
+	    }
+	    
+	} else { # don't group - individual years and months
+	    foreach my $classification (@$classlistref) {
+		my $date = $self->{'list'}->{$classification};
+		$date =~ s/^(\d\d\d\d)(\d\d).*$/$1&nbsp;_textmonth$2_/;
+		my ($year, $month)=($1,$2);
+		# sanity check if month is zero
+		if ($date =~ /00_$/) {
+		    $date =~ s/^(\d\d\d\d).*$/$1/g;
+		}
+		# create subclass if it doesn't already exist
+		$classhash->{$year} = () unless defined $classhash->{$year};
+		$classhash->{$year}->{$month} = []
+		    unless defined $classhash->{$year}->{$month};
+		push (@{$classhash->{$year}->{$month}}, $classification);
+	    }
+	    # create hlist of years containing hlists of months
+
+	    foreach my $subclass (sort {$a <=> $b} keys %$classhash) {
+		my $yearclassify = $self->get_entry($subclass, "HList");
+		foreach my $subsubclass (sort {$a <=> $b} 
+					 (keys %{$classhash->{$subclass}})) {
+		    my $monthname=$subsubclass;
+		    if ($monthname >= 1 && $monthname <= 12) {
+			$monthname="_textmonth" . $monthname . "_";
+		    }
+		    my $monthclassify=$self->get_entry($monthname, "DateList");
+		    push (@{$yearclassify->{'contains'}}, $monthclassify);
+		
+		    foreach my $subsubOID 
+			(@{$classhash->{$subclass}->{$subsubclass}}) {
+			    push (@{$monthclassify->{'contains'}},
+				  {'OID'=>$subsubOID});
+			}
+		}
+		push (@{$classifyinfo->{'contains'}}, $yearclassify);
+	    }
+	    return $classifyinfo;
+	} # nogroup
+    } else {
+	# not by month
+	# first split up the list into separate year classifications
+	foreach my $classification (@$classlistref) {
+	    my $date = $self->{'list'}->{$classification};
+	    $date =~ s/^(\d\d\d\d).*$/$1/;
+	    $classhash->{$date} = [] unless defined $classhash->{$date};
+	    push (@{$classhash->{$date}}, $classification);
+	}
+    }
+    
+    # only compact the list if nogroup not specified
+    if (!$self->{'nogroup'}) {
+	$classhash = $self->compactlist ($classhash);
+    }
+    foreach my $subclass (sort keys %$classhash) {
+	my $tempclassify = $self->get_entry($subclass, "DateList");
+	foreach my $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstdate = "";
+    my $currentlastdate = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+    foreach my $subsection (sort keys %$classhashref) {
+	$currentfirstdate = $subsection if $currentfirstdate eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastdate = $subsection;
+	} else {
+
+	    if ($currentfirstdate eq $currentlastdate) {
+		@{$compactedhash->{$currentfirstdate}} = @currentOIDs;
+		$lastkey = $currentfirstdate;
+	    } else {
+		@{$compactedhash->{"$currentfirstdate-$currentlastdate"}} = @currentOIDs;
+		$lastkey = "$currentfirstdate-$currentlastdate";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstdate = "";
+		$lastkey = $subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstdate = $subsection;
+		$currentlastdate = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    if ((scalar (@currentOIDs) < $min) && (scalar (@currentOIDs) > 0)) {
+	# want every thing in previous up to the dash
+	my ($newkey) = $lastkey =~ /^([^\-]+)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastdate"}} = @currentOIDs;	
+    } else {
+	if ($currentfirstdate eq $currentlastdate) {
+	    @{$compactedhash->{$currentfirstdate}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstdate-$currentlastdate"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/GenericList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/GenericList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/GenericList.pm	(revision 21112)
@@ -0,0 +1,494 @@
+###########################################################################
+#
+# GenericList.pm -- A general and flexible list classifier with most of
+#                   the abilities of AZCompactList, and better Unicode,
+#                   metadata and sorting capabilities.
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+#
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# TO DO: Remove punctuation from metadata values before sorting.
+#
+###########################################################################
+
+package GenericList;
+
+
+use BasClas;
+
+use strict;
+no strict 'refs';  # Allow filehandles to be variables and viceversa
+
+
+sub BEGIN {
+    @GenericList::ISA = ('BasClas');
+}
+
+
+my $arguments = 
+    [ { 'name' => "metadata",
+	'desc' => "{GenericList.metadata}",
+	'type' => "metadata",
+	'reqd' => "yes" },
+      { 'name' => "buttonname",
+  	'desc' => "{BasClas.buttonname}",
+	'type' => "string" },
+
+      # The interesting options
+      { 'name' => "always_bookshelf_last_level",
+	'desc' => "{GenericList.always_bookshelf_last_level}",
+	'type' => "flag" },
+      { 'name' => "classify_sections",
+	'desc' => "{GenericList.classify_sections}",
+	'type' => "flag" },
+      { 'name' => "partition_type_within_level",
+	'desc' => "{GenericList.partition_type_within_level}",
+	'type' => "string",
+	'deft' => "none" },
+      { 'name' => "partition_size_within_level",
+	'desc' => "{GenericList.partition_size_within_level}",
+	'type' => "string" },
+      { 'name' => "sort_leaf_nodes_using",
+	'desc' => "{GenericList.sort_leaf_nodes_using}",
+	'type' => "metadata",
+	'deft' => "Title" },
+      { 'name' => "use_hlist_for",
+	'desc' => "{GenericList.use_hlist_for}",
+	'type' => "string" } ];
+
+my $options = { 'name'     => "GenericList",
+		'desc'     => "{GenericList.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+		'args'     => $arguments };
+
+
+sub new
+{
+    my ($class) = shift(@_);
+    my ($classifierslist, $inputargs, $hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if (defined $arguments) { push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); }
+    if (defined $options) { push(@{$hashArgOptLists->{"OptList"}}, $options); }
+
+    my $self = (defined $hashArgOptLists) ? new BasClas($classifierslist, $inputargs, $hashArgOptLists) : new BasClas($classifierslist, $inputargs);
+    $self->{'OIDlist'} = [];
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $metadata = $self->{'metadata'};
+    my $buttonname = $self->{'buttonname'};
+    my $partition_type_within_level = $self->{'partition_type_within_level'};
+    my $partition_size_within_level = $self->{'partition_size_within_level'};
+    my $sort_leaf_nodes_using = $self->{'sort_leaf_nodes_using'};
+    my $use_hlist_for = $self->{'use_hlist_for'};
+
+    # The metadata elements to use (required)
+    # print STDERR "Creating new GenericList for $metadata...\n";
+    if (!$metadata) {
+	die "Error: No metadata fields specified for GenericList.\n";
+    }
+    my @metalist = split(/\//, $metadata);
+    $self->{'metalist'} = \@metalist;
+
+    # The classifier button name
+    if (!$buttonname) {
+	# Default: the first metadata element specified
+	my $firstmetagroupfirstelem = (split(/\;/, $metalist[0]))[0];
+	$buttonname = $self->generate_title_from_metadata($firstmetagroupfirstelem);
+    }
+    $self->{'title'} = $buttonname;
+
+    # Whether to group single items into a bookshelf (must be true for all metadata fields except the last)
+    foreach my $metagroup (@metalist) {
+	$self->{$metagroup . ".always_bookshelf"} = "t";
+    }
+    if (!$self->{'always_bookshelf_last_level'}) {
+	# Default: leave leafnodes ungrouped (equivalent to AZCompactList -mingroup 2)
+	my $lastmetagroup = $metalist[$#metalist];
+	$self->{$lastmetagroup . ".always_bookshelf"} = "f";
+    }
+
+    # Whether to use an hlist or a vlist for each level in the hierarchy
+    foreach my $metagroup (@metalist) {
+	$self->{$metagroup . ".use_hlist"} = "f";
+    }
+    foreach my $metagroup (split(/\,/, $use_hlist_for)) {
+	$self->{$metagroup . ".use_hlist"} = "t";
+    }
+
+    # How the items are grouped into partitions
+    if (!$partition_type_within_level) {
+	# Default: none
+	$partition_type_within_level = "none";
+    }
+    $self->{"partition_type_within_level"} = $partition_type_within_level;
+
+    # The number of items in each partition
+    if (!$partition_size_within_level) {
+	# Default: 20
+	foreach my $metagroup (@metalist) {
+	    $self->{$metagroup . ".partition_size_within_level"} = 20;
+	}
+    }
+    else {
+	my @partition_size_within_levellist = split(/\//, $partition_size_within_level);
+
+	# Assign values based on the partition_size_within_level parameter
+	foreach my $metagroup (@metalist) {
+	    my $partition_size_within_levelelem = shift(@partition_size_within_levellist);
+	    if (defined($partition_size_within_levelelem)) {
+		$self->{$metagroup . ".partition_size_within_level"} = $partition_size_within_levelelem;
+	    }
+	    else {
+		$self->{$metagroup . ".partition_size_within_level"} = $self->{$metalist[0] . ".partition_size_within_level"};
+	    }
+	}
+    }
+
+    # The metadata elements to use to sort the leaf nodes
+    my @sort_leaf_nodes_usingmetalist = ( "Title" );
+    if ($sort_leaf_nodes_using) {
+	@sort_leaf_nodes_usingmetalist = split(/\|/, $sort_leaf_nodes_using);
+    }
+    $self->{'sort_leaf_nodes_usingmetalist'} = \@sort_leaf_nodes_usingmetalist;
+
+    return bless $self, $class;
+}
+
+
+sub init
+{
+    # Nothing to do...
+}
+
+
+sub classify
+{
+    my $self = shift(@_);
+    my $doc_obj = shift(@_);
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    if ($self->{'classify_sections'}) {
+	my $section = $doc_obj->get_next_section($doc_obj->get_top_section());
+	while (defined $section) {
+	    $self->classify_section($doc_obj, $doc_OID . ".$section", $section);
+	    $section = $doc_obj->get_next_section($section);
+	}
+    }
+    else {
+	$self->classify_section($doc_obj, $doc_OID, $doc_obj->get_top_section());
+    }
+}
+
+
+sub classify_section
+{
+    my $self = shift(@_);
+    my $doc_obj = shift(@_);
+    my $doc_OID = shift(@_);
+    my $section = shift(@_);
+
+    my @metalist = @{$self->{'metalist'}};
+
+    # Only classify the document if it has a value for one of the metadata elements in the first group
+    foreach my $firstmetagroupelem (split(/\;/, $metalist[0])) {
+	my $firstmetagroupelemvalue = $doc_obj->get_metadata_element($section, $firstmetagroupelem);
+	if (defined($firstmetagroupelemvalue) && $firstmetagroupelemvalue ne "") {
+	    push(@{$self->{'OIDlist'}}, $doc_OID);
+
+	    # Create a hash for the metadata values of each metadata element we're interested in
+	    my %metagroupsdone = ();
+	    foreach my $metagroup (@metalist, @{$self->{'sort_leaf_nodes_usingmetalist'}}) {
+		# Take care not to do a metadata group more than once
+		unless ($metagroupsdone{$metagroup}) {
+		    foreach my $metaelem (split(/\;/, $metagroup)) {
+			my @metavalues = @{$doc_obj->get_metadata($section, $metaelem)};
+			foreach my $metavalue (@metavalues) {
+			    # Strip leading and trailing whitespace
+			    $metavalue =~ s/^\s*//;
+			    $metavalue =~ s/\s*$//;
+			    push(@{$self->{$metagroup . ".list"}->{$doc_OID}}, $metavalue);
+			}
+			last if (@metavalues > 0);
+		    }
+
+		    $metagroupsdone{$metagroup} = 1;
+		}
+	    }
+
+	    last;
+	}
+    }
+}
+
+
+sub get_classify_info
+{
+    my $self = shift(@_);
+
+    # The metadata elements to classify by
+    my @metalist = @{$self->{'metalist'}};
+    my $firstmetagroup = $metalist[0];
+
+    # The OID values of the documents to include in the classification
+    my @OIDlist = @{$self->{'OIDlist'}};
+
+    # The root node of the classification hierarchy
+    my $childtype = (($self->{$firstmetagroup . ".use_hlist"} eq "t") ? "HList" : "VList");
+    my %classifyinfo = ( 'thistype' => "Invisible",
+			 'childtype' => $childtype,
+			 'Title' => $self->{'title'},
+			 'contains' => [] );
+
+    # Recursively create the classification hierarchy, one level for each metadata element
+    &add_az_list($self, \@metalist, \@OIDlist, \%classifyinfo);
+    return \%classifyinfo;
+}
+
+
+sub add_az_list
+{
+    my $self = shift(@_);
+    my @metalist = @{shift(@_)};
+    my @OIDlist = @{shift(@_)};
+    my $classifyinfo = shift(@_);
+    # print STDERR "\nAdding AZ list for " . $classifyinfo->{'Title'} . "\n";
+
+    my $metagroup = $metalist[0];
+    # print STDERR "Processing metadata group: " . $metagroup . "\n";
+    # print STDERR "Number of OID values: " . @OIDlist . "\n";
+
+    my %OIDtometavaluehash = %{$self->{$metagroup . ".list"}};
+
+    # Create a mapping from metadata value to OID
+    my %metavaluetoOIDhash = ();
+    foreach my $OID (@OIDlist) {
+	if ($OIDtometavaluehash{$OID}) {
+	    my @metavalues = @{$OIDtometavaluehash{$OID}};
+	    foreach my $metavalue (@metavalues) {
+		push(@{$metavaluetoOIDhash{$metavalue}}, $OID);
+	    }
+	}
+    }
+    # print STDERR "Number of distinct values: " . scalar(keys %metavaluetoOIDhash) . "\n";
+
+    # Partition the values (if necessary)
+    my $partition_type_within_level = $self->{"partition_type_within_level"};
+    if ($partition_type_within_level =~ /^per_letter$/i) {
+	# Generate one hlist for each letter
+	my @sortedmetavalues = sort(keys %metavaluetoOIDhash);
+	my %metavaluetoOIDsubhash = ();
+
+	my $lastpartition = &unicode::substr($sortedmetavalues[0], 0, 1);
+	foreach my $metavalue (@sortedmetavalues) {
+	    my $metavaluepartition = &unicode::substr($metavalue, 0, 1);
+
+	    # Is this the start of a new partition?
+	    if ($metavaluepartition ne $lastpartition) {
+		&add_hlist_partition($self, \@metalist, $classifyinfo, $lastpartition, \%metavaluetoOIDsubhash);
+		%metavaluetoOIDsubhash = ();
+		$lastpartition = $metavaluepartition;
+	    }
+
+	    $metavaluetoOIDsubhash{$metavalue} = $metavaluetoOIDhash{$metavalue};	    
+	}
+
+	# Don't forget to add the last partition
+	&add_hlist_partition($self, \@metalist, $classifyinfo, $lastpartition, \%metavaluetoOIDsubhash);
+
+	# The partitions are stored in an HList
+	$classifyinfo->{'childtype'} = "HList";
+    }
+
+    else {
+	# Generate hlists of a certain size
+	my $partition_size_within_level = $self->{$metagroup . ".partition_size_within_level"};
+	if ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metavaluetoOIDhash) > $partition_size_within_level) {
+	    my @sortedmetavalues = sort(keys %metavaluetoOIDhash);
+	    my $itemsdone = 0;
+	    my %metavaluetoOIDsubhash = ();
+	    my $lastpartitionend = "";
+	    my $partitionstart;
+	    foreach my $metavalue (@sortedmetavalues) {
+		$metavaluetoOIDsubhash{$metavalue} = $metavaluetoOIDhash{$metavalue};
+		$itemsdone++;
+		my $itemsinpartition = scalar(keys %metavaluetoOIDsubhash);
+
+		# Is this the start of a new partition?
+		if ($itemsinpartition == 1) {
+		    $partitionstart = &generate_partition_start($metavalue, $lastpartitionend);
+		}
+
+		# Is this the end of the partition?
+		if ($itemsinpartition == $partition_size_within_level || $itemsdone == @sortedmetavalues) {
+		    my $partitionend = &generate_partition_end($metavalue, $partitionstart);
+		    my $partitionname = $partitionstart;
+		    if ($partitionend ne $partitionstart) {
+			$partitionname = $partitionname . "-" . $partitionend;
+		    }
+
+		    &add_hlist_partition($self, \@metalist, $classifyinfo, $partitionname, \%metavaluetoOIDsubhash);
+		    %metavaluetoOIDsubhash = ();
+		    $lastpartitionend = $partitionend;
+		}
+	    }
+
+	    # The partitions are stored in an HList
+	    $classifyinfo->{'childtype'} = "HList";
+	}
+
+	# Otherwise just add all the values to a VList
+	else {
+	    &add_vlist($self, \@metalist, $classifyinfo, \%metavaluetoOIDhash);
+	}
+    }
+}
+
+
+sub unicode_length
+{
+    my $utf8string = shift(@_);
+
+    my @unicodestring = @{&unicode::utf82unicode($utf8string)};
+    return scalar(@unicodestring);
+}
+
+
+sub generate_partition_start
+{
+    my $metavalue = shift(@_);
+    my $lastpartitionend = shift(@_);
+
+    my $partitionstart = &unicode::substr($metavalue, 0, 1);
+    if ($partitionstart le $lastpartitionend) {
+	$partitionstart = &unicode::substr($metavalue, 0, 2);
+	# Give up after three characters
+	if ($partitionstart le $lastpartitionend) {
+	    $partitionstart = &unicode::substr($metavalue, 0, 3);
+	}
+    }
+
+    return $partitionstart;
+}
+
+
+sub generate_partition_end
+{
+    my $metavalue = shift(@_);
+    my $partitionstart = shift(@_);
+
+    my $partitionend = &unicode::substr($metavalue, 0, &unicode_length($partitionstart));
+    if ($partitionend gt $partitionstart) {
+	$partitionend = &unicode::substr($metavalue, 0, 1);
+	if ($partitionend le $partitionstart) {
+	    $partitionend = &unicode::substr($metavalue, 0, 2);
+	    # Give up after three characters
+	    if ($partitionend le $partitionstart) {
+		$partitionend = &unicode::substr($metavalue, 0, 3);
+	    }
+	}
+    }
+
+    return $partitionend;
+}
+
+
+sub add_hlist_partition
+{
+    my $self = shift(@_);
+    my @metalist = @{shift(@_)};
+    my $classifyinfo = shift(@_);
+    my $partitionname = shift(@_);
+    my %metavaluetoOIDhash = %{shift(@_)};
+
+    # Create an hlist partition
+    my %subclassifyinfo = ( 'Title' => $partitionname,
+			    'childtype' => "VList",
+			    'contains' => [] );
+
+    # Add the children to the hlist partition
+    &add_vlist($self, \@metalist, \%subclassifyinfo, \%metavaluetoOIDhash);
+    push(@{$classifyinfo->{'contains'}}, \%subclassifyinfo);
+}
+
+
+sub add_vlist
+{
+    my $self = shift(@_);
+    my @metalist = @{shift(@_)};
+    my $classifyinfo = shift(@_);
+    my %metavaluetoOIDhash = %{shift(@_)};
+
+    my $metagroup = shift(@metalist);
+
+    # Create an entry in the vlist for each value
+    foreach my $metavalue (sort(keys %metavaluetoOIDhash)) {
+	my @OIDlist = @{$metavaluetoOIDhash{$metavalue}};
+
+	# If there is only one item and 'always_bookshelf' is false, add the item to the list
+	if (@OIDlist == 1 && $self->{$metagroup . ".always_bookshelf"} eq "f") {
+	    push(@{$classifyinfo->{'contains'}}, { 'OID' => $OIDlist[0] });
+	}
+
+	# Otherwise create a sublist (bookshelf) for the metadata value
+	else {
+	    my %subclassifyinfo = ( 'Title' => $metavalue,
+				    'childtype' => "VList",
+				    'contains' => [] );
+
+	    # If there are metadata elements remaining, recursively apply the process
+	    if (@metalist > 0) {
+		my $nextmetagroup = $metalist[0];
+		my $childtype = (($self->{$nextmetagroup . ".use_hlist"} eq "t") ? "HList" : "VList");
+		$subclassifyinfo{'childtype'} = $childtype;
+		&add_az_list($self, \@metalist, \@OIDlist, \%subclassifyinfo);
+	    }
+	    # Otherwise just add the documents as children of this list
+	    else {
+		# Sort the leaf nodes by the metadata elements specified with -sort_leaf_nodes_using
+		my @sort_leaf_nodes_usingmetalist = @{$self->{'sort_leaf_nodes_usingmetalist'}};
+		foreach my $sort_leaf_nodes_usingmetaelem (reverse @sort_leaf_nodes_usingmetalist) {
+		    my %OIDtometavaluehash = %{$self->{$sort_leaf_nodes_usingmetaelem . ".list"}};
+		    # Force a stable sort (Perl 5.6's sort isn't stable)
+		    # !! The [0] bits aren't ideal (multiple metadata values) !! 
+		    @OIDlist = @OIDlist[ sort { $OIDtometavaluehash{$OIDlist[$a]}[0] cmp $OIDtometavaluehash{$OIDlist[$b]}[0] || $a <=> $b; } 0..$#OIDlist ];
+		}
+
+  		foreach my $OID (@OIDlist) {
+  		    push(@{$subclassifyinfo{'contains'}}, { 'OID' => $OID });
+  		}
+	    }
+
+	    # Add the sublist to the list
+	    push(@{$classifyinfo->{'contains'}}, \%subclassifyinfo);
+	}
+    }
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/HFileHierarchy.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/HFileHierarchy.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/HFileHierarchy.pm	(revision 21112)
@@ -0,0 +1,328 @@
+###########################################################################
+#
+# HFileHierarchy.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for generating hierarchical classifications
+
+
+package HFileHierarchy;
+
+use BasClas;
+use util;
+use cfgread;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @HFileHierarchy::ISA = ('BasClas');
+}
+
+my $arguments =
+    [ { 'name' => "metadata",
+	'desc' => "{AZCompactList.metadata}",
+	'type' => "metadata",
+	'reqd' => "yes" },
+      { 'name' => "firstvalueonly",
+	'desc' => "{AZCompactList.firstvalueonly}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "allvalues",
+	'desc' => "{AZCompactList.allvalues}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "hfile",
+	'desc' => "{Hierarchy.hfile}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "buttonname",
+  	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "sort",
+	'desc' => "{Hierarchy.sort}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "reverse_sort",
+	'desc' => "{Hierarchy.reverse_sort}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "hlist_at_top",
+	'desc' => "{Hierarchy.hlist_at_top}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "documents_last",
+	'desc' => "{Hierarchy.documents_last}",
+	'type' => "flag",
+	'reqd' => "no"}
+      ];
+
+my $options = 
+{ 	'name'     => "HFileHierarchy",
+	'desc'     => "{HFileHierarchy.desc}",
+	'abstract' => "yes",
+	'inherits' => "yes", 
+	'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs); 
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $metadata = $self->{'metadata'};
+    if (!$metadata) {
+	print STDERR "$class Error: required option -metadata not supplied\n";
+	$self->print_txt_usage("");  # Use default resource bundle
+	
+	die "$class Error: required option -metadata not supplied\n";
+    }
+    
+    $self->{'buttonname'} = $self->generate_title_from_metadata($metadata) unless ($self->{'buttonname'});
+
+    #$self->{'metaname'} = $metadata;
+    my @meta_list = split(/,/, $metadata);
+    $self->{'meta_list'} = \@meta_list;
+
+    # sort = undef in this case is the same as sort=nosort
+    if ($self->{'sort'} eq "nosort") {
+	$self->{'sort'} = undef;
+    }
+
+    # sortname is handled a bit differently - kjdon
+    # undef means to sort, but use the metadata value from -metadata
+    # because there is no one metadata value to get for sorting when 
+    # we have a list of possible metadata.
+    # To get no sorting, set sortname = 'nosort'
+    # we don't need to set it to undef if its not defined do we???
+
+#    if (!$self->{'sort'}) {
+#	if (defined ($metadata)) {
+#	    $sortname = undef;
+#	} else {
+#	    $sortname = "nosort";
+#	}
+#    }
+ #   $self->{'sortname'} = $sortname;
+
+    #if (defined $self->{'sort'} && $reverse_sort) {
+#	$self->{'reverse_sort'} = 1;
+ #   }
+	
+    if ($self->{'hfile'}) {
+	my $hfile = $self->{'hfile'};
+	my $subjectfile;
+	$subjectfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"etc", $hfile);
+	if (!-e $subjectfile) {
+	    my $collfile = $subjectfile;
+	    $subjectfile = &util::filename_cat($ENV{'GSDLHOME'},"etc", $hfile);
+	    if (!-e $subjectfile) {
+		my $outhandle = $self->{'outhandle'};
+		print STDERR "\nHFileHierarchy Error: Can't locate subject file $hfile\n";
+		print STDERR "This file should be in $collfile or $subjectfile\n";
+		$self->print_txt_usage("");  # Use default resource bundle
+		print STDERR "\nHFileHierarchy Error: Can't locate subject file $hfile\n";
+		print STDERR "This file should be in $collfile or $subjectfile\n";
+		die "\n";
+	    }
+	}
+	$self->{'descriptorlist'} = {}; # first field in subject file
+	$self->{'locatorlist'} = {}; # second field in subject file
+	$self->{'subjectfile'} = $subjectfile;
+    }
+    
+
+   # $self->{'firstvalueonly'} = $firstvalueonly;
+   # $self->{'allvalues'} = $allvalues;
+
+    #$self->{'hlist_at_top'} = $hlist_at_top;
+
+    # Clean out the unused keys
+    delete $self->{'metadata'};
+    delete $self->{'hfile'};
+    
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    my $subjectfile = $self->{'subjectfile'};
+    if (defined $subjectfile) {
+	# read in the subject file
+	my $list = &cfgread::read_cfg_file ($self->{'subjectfile'}, undef, '^[^#]?\S');
+	# $list is a hash that is indexed by the descriptor. The contents of this
+	# hash is a list of two items. The first item is the OID and the second item
+	# is the title
+	foreach my $descriptor (keys (%$list)) {
+	    $self->{'descriptorlist'}->{$descriptor} = $list->{$descriptor}->[0];
+	    unless (defined $self->{'locatorlist'}->{$list->{$descriptor}->[0]}) {
+		$self->{'locatorlist'}->{$list->{$descriptor}->[0]}->{'title'} = $list->{$descriptor}->[1];
+		$self->{'locatorlist'}->{$list->{$descriptor}->[0]}->{'contents'} = [];
+	    }
+	}
+    }
+}
+
+sub hfile_classify
+{
+    my $self = shift (@_);
+    my ($doc_obj,$sortmeta,$metavalues) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    foreach my $metaelement (@$metavalues) {
+	if ((defined $self->{'descriptorlist'}->{$metaelement}) &&
+	    (defined $self->{'locatorlist'}->{$self->{'descriptorlist'}->{$metaelement}})) {
+	    push (@{$self->{'locatorlist'}->{$self->{'descriptorlist'}->{$metaelement}}->{'contents'}}, 
+		  [$doc_OID, $sortmeta]);
+	    my $localid = $self->{'descriptorlist'}->{$metaelement};
+	    my $classid = $self->get_number();
+
+	    $doc_obj->add_metadata($doc_obj->get_top_section(), "memberof", "CL$classid.$localid");
+	}
+    }
+}
+
+
+
+
+sub hfile_get_classify_info {
+    my $self = shift (@_);
+    
+    my ($classifyinfo) = @_;
+
+    my $list = $self->{'locatorlist'};
+
+    # sorted the keys - otherwise funny things happen - kjdon 03/01/03
+    foreach my $OID (sort keys (%$list)) {
+	my $tempinfo = $self->get_OID_entry ($OID, $classifyinfo, $list->{$OID}->{'title'}, "VList");
+	
+	if (defined $self->{'sort'}) {
+	    if ($self->{'reverse_sort'}) {
+		foreach my $subOID (sort {$b->[1] cmp $a->[1];} @{$list->{$OID}->{'contents'}}) {
+		    push (@{$tempinfo->{'contains'}}, {'OID'=>$subOID->[0]});
+		}
+	    }
+	    else {
+		foreach my $subOID (sort {$a->[1] cmp $b->[1];} @{$list->{$OID}->{'contents'}}) {
+		    push (@{$tempinfo->{'contains'}}, {'OID'=>$subOID->[0]});
+		}
+	    }
+	}
+	else {
+	    foreach my $subOID (@{$list->{$OID}->{'contents'}}) {
+		push (@{$tempinfo->{'contains'}}, {'OID'=>$subOID->[0]});
+	    }
+	}
+    }
+    
+    return $classifyinfo;
+}
+
+
+sub supports_memberof {
+    my $self = shift(@_);
+
+    return "true";
+}
+
+sub get_OID_entry {
+    my $self = shift (@_);
+    my ($OID, $classifyinfo, $title, $classifytype) = @_;
+
+    $OID = "" unless defined $OID;
+    $OID =~ s/^\.+//;
+
+    my ($headOID, $tailOID) = $OID =~ /^(\d+)(.*)$/;
+    $tailOID = "" unless defined $tailOID;
+
+    if (!defined $headOID) {
+	$classifyinfo->{'Title'} = $title;
+	$classifyinfo->{'classifytype'} = $classifytype;
+	return $classifyinfo;
+    }
+
+    $classifyinfo->{'contains'} = [] unless defined $classifyinfo->{'contains'};
+    if ($self->{'documents_last'}) {
+	# documents should come after nodes in the classifier
+
+	my $doc_pos = 0;
+	foreach my $thing (@{$classifyinfo->{'contains'}}) {
+	    last if defined $thing->{'OID'};
+	    $doc_pos++;
+	}
+	
+	while ($doc_pos < $headOID) {
+	    splice(@{$classifyinfo->{'contains'}}, $doc_pos, 0, $self->get_entry("", $classifytype));
+	    $doc_pos++;
+	}
+
+	return $self->get_OID_entry ($tailOID, $classifyinfo->{'contains'}->[($headOID-1)], $title, $classifytype);
+
+    }
+    
+    # else, documents come before nodes
+    my $offset = 0;
+    foreach my $thing (@{$classifyinfo->{'contains'}}) {
+	$offset ++ if defined $thing->{'OID'};
+    }
+
+    while (scalar(@{$classifyinfo->{'contains'}}) < ($headOID+$offset)) { 
+	push (@{$classifyinfo->{'contains'}}, $self->get_entry("", $classifytype));
+    }
+
+    return $self->get_OID_entry ($tailOID, $classifyinfo->{'contains'}->[($headOID+$offset-1)], $title, $classifytype);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    my $memberof = &supports_memberof();
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'supportsmemberof'=>$memberof,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/HTML.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/HTML.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/HTML.pm	(revision 21112)
@@ -0,0 +1,107 @@
+###########################################################################
+#
+# HTML.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# html classifier plugin - creates an empty classification
+# that's simply a link to a web page
+
+package HTML;
+
+use BasClas;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @HTML::ISA = ('BasClas');
+}
+
+my $arguments = 
+    [ { 'name' => "url",
+	'desc' => "{HTML.url}",
+	'type' => "string",
+	'reqd' => "yes" } ,
+      { 'name' => "buttonname",
+	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'deft' => "Browse",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "HTML",
+		'desc'     => "{HTML.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    if (!$self->{'url'}) {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "HTML Error: required option -url not supplied\n";
+	$self->print_txt_usage("");
+	die "HTML Error: required option -url not supplied\n";
+    }
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    # we don't do anything for individual documents
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my %classifyinfo = ('thistype'=>'Invisible',
+			'childtype'=>'HTML',
+			'Title'=>$self->{'buttonname'},
+			'contains'=>[]);
+
+    push (@{$classifyinfo{'contains'}}, {'OID'=>$self->{'url'}});
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/Hierarchy.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/Hierarchy.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/Hierarchy.pm	(revision 21112)
@@ -0,0 +1,349 @@
+###########################################################################
+#
+# Hierarchy.pm -- classifier that enables a Hierarchy to beformed without 
+#                 the need for a hierarchy file (like HFileHierarchy). Used
+#                 to be called AutoHierarchy.  Inherits from HFileHierarchy
+#                 so can also do everything that does as well.
+#                 Created by Imene, modified by Katherine and David.
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# An advanced Hierarchical classifier
+# to see the options, run "perl -S classinfo.pl Hierarchy"
+
+package Hierarchy;
+
+use HFileHierarchy;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @Hierarchy::ISA = ('HFileHierarchy');
+}
+
+my $arguments = 
+    [ { 'name' => "separator",
+	'desc' => "{Hierarchy.separator}",
+	'type' => "regexp",
+	'deft' => "[\\\\\\\/|\\\\\\\|]",
+	'reqd' => "no" },
+      { 'name' => "suppresslastlevel",
+	'desc' => "{Hierarchy.suppresslastlevel}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "suppressfirstlevel",
+	'desc' => "{Hierarchy.suppressfirstlevel}",
+	'type' => "flag",
+	'reqd' => "no" }
+      ];
+
+my $options = { 'name'     => "Hierarchy",
+		'desc'     => "{Hierarchy.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new HFileHierarchy($classifierslist,$inputargs,$hashArgOptLists): new HFileHierarchy($classifierslist,$inputargs); 
+    
+    # the hash that we use to build up the hierarchy
+    $self->{'path_hash'}= {};
+    
+    return bless $self, $class;
+}
+
+
+sub auto_classify {
+    my $self = shift (@_);
+    my ($doc_obj,$nosort,$sortmeta,$metavalues) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    
+    #Add all the metadata values to the hash
+    my $path_hash;
+    my $current_pos;
+    
+    
+    foreach my $metavalue (@$metavalues) {
+	$path_hash = $self->{'path_hash'};
+	my @chunks = split (/$self->{'separator'}/, $metavalue);
+	if ($self->{'suppresslastlevel'}) {
+	    pop(@chunks); # remove the last element from the end
+	}
+	if ($self->{'suppressfirstlevel'}) {
+	    shift(@chunks);
+	}
+	foreach my $folderName (@chunks) 
+	{
+	    if ($folderName ne ""){ #sometimes the tokens are empty
+		$current_pos = $self->add_To_Hash($path_hash, $folderName, $nosort);
+		$path_hash = $current_pos->{'nodes'};
+	    } 
+	}
+	# now add the document, with sort meta if needed
+	if ($nosort) {
+	    push(@{$current_pos->{'docs'}}, $doc_OID);
+	} else {
+	    $current_pos->{'docs'}->{$doc_OID} = $sortmeta;
+	    
+	    #if (defined $sortmeta) {
+	#	# can you ever get the same doc twice in one classification??
+	#	$current_pos->{'docs'}->{$doc_OID} = $sortmeta;
+	#    } else {
+	#	$current_pos->{'docs'}->{$doc_OID} = $metavalue;
+	#    }
+	}
+    } # foreach metadata
+
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    # are we sorting the list??
+    my $nosort = 0;
+    if (!defined $self->{'sort'}) {
+	$nosort = 1;
+    }
+    
+    my $metavalues = [];
+    # find all the metadata values
+    foreach my $m (@{$self->{'meta_list'}}) {
+	my $mvalues = $doc_obj->get_metadata($doc_obj->get_top_section(), $m);
+	next unless (@{$mvalues});
+	if ($self->{'firstvalueonly'}) {
+	    # we only want the first metadata value
+	    push (@$metavalues, $mvalues->[0]);
+	    last;
+	}
+	push (@$metavalues, @$mvalues);
+	last if (!$self->{'allvalues'}); # we don't want to try other elements
+ 	                                 # cos we have already found some 
+    } 
+    
+    return unless (@$metavalues);
+
+    #check for a sort element other than our metadata
+    my $sortmeta = undef;
+    if (!$nosort) {
+	if ($self->{'sort'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'sort'});
+	    if (defined $sortmeta && !$self->{'no_metadata_formatting'}) {
+		$sortmeta = &sorttools::format_metadata_for_sorting($self->{'sort'}, $sortmeta, $doc_obj);
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+    }
+
+    if (defined $self->{'subjectfile'}) {
+	$self->hfile_classify($doc_obj,$sortmeta,$metavalues);
+    }
+    else {
+	$self->auto_classify($doc_obj,$nosort,$sortmeta,$metavalues);
+    }
+}
+
+sub add_To_Hash {
+    my $self = shift (@_);
+    my ($myhash, $k, $nosort) = @_;
+    
+    if (!defined $myhash->{$k}){
+	$myhash->{$k}={};
+	$myhash->{$k}->{'nodes'}={};
+	if ($nosort) {
+	    $myhash->{$k}->{'docs'}=[];
+	} else {
+	    $myhash->{$k}->{'docs'} = {};
+	}
+    }
+    return $myhash->{$k}; 
+}
+
+sub print_Hash{
+    my $self = shift (@_); 
+    my ($myHash, $num_spaces) = @_;
+
+    foreach my $key (keys %{$myHash}){
+	print "\n";
+	$self->print_spaces($num_spaces);
+	print STDERR "$key*";
+	$self->print_Hash($myHash->{$key}, $num_spaces + 2);
+    }    
+}
+
+sub print_spaces{
+    my $self = shift (@_);
+    my ($num_spaces) = @_;
+    
+    for (my $i = 0; $i < $num_spaces; $i++){
+	print STDERR " ";
+    }
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[],);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+    
+    return \%classifyinfo;
+}
+
+sub process_hash {
+    my $self = shift (@_);
+    my ($top_hash, $top_entry) = @_;   
+    my ($entry);
+    
+    my $hash = {};
+    foreach my $key (sort keys %{$top_hash}) {
+	$entry = $self->get_entry($key,"VList","VList");
+	my $has_content = 0;
+	my @doc_list;
+	# generate a sorted list of doc ids
+	if (not (defined ($self->{'sort'})) && scalar(@{$top_hash->{$key}->{'docs'}})) {
+	    @doc_list = @{$top_hash->{$key}->{'docs'}};
+	} elsif (defined ($self->{'sort'}) && (keys %{$top_hash->{$key}->{'docs'}})) {
+	    @doc_list = sort {$top_hash->{$key}->{'docs'}->{$a} 
+			      cmp $top_hash->{$key}->{'docs'}->{$b};} keys %{$top_hash->{$key}->{'docs'}};
+	    
+	}
+
+	if ($self->{'documents_last'}) {
+	    # add nodes, then documents
+	    # if this key has nodes, add them
+	    if (scalar(keys %{$top_hash->{$key}->{'nodes'}})) {
+		$has_content = 1;
+		$self->process_hash($top_hash->{$key}->{'nodes'}, $entry); 
+	    }
+
+	    # if this key has documents, add them
+	    if (@doc_list) {
+		$has_content = 1;
+		foreach my $d (@doc_list) {
+		    push (@{$entry->{'contains'}}, {'OID'=>$d});
+		}    
+	    }
+
+	} else {
+	    # add documents then nodes
+	    # if this key has documents, add them
+	    if (@doc_list) {
+		$has_content = 1;
+		foreach my $d (@doc_list) {
+		    push (@{$entry->{'contains'}}, {'OID'=>$d});
+		}    
+	    }
+	    # if this key has nodes, add them
+	    if (scalar(keys %{$top_hash->{$key}->{'nodes'}})) {
+		$has_content = 1;
+		$self->process_hash($top_hash->{$key}->{'nodes'}, $entry); 
+	    }
+	}
+	
+	# if we have found some content, add the new entry for this key into the parent node
+	if ($has_content) {
+	    push (@{$top_entry->{'contains'}}, $entry);
+	}
+
+    }    
+}
+
+sub auto_get_classify_info {
+    my $self = shift (@_);
+    my ($no_thistype) = @_;
+    $no_thistype = 0 unless defined $no_thistype;
+
+    my ($classification);
+    my $top_h = $self->{'path_hash'};
+
+    if ($self->{'path_hash'}) {
+	if ($self->{'hlist_at_top'}) {
+	    $classification = $self->get_entry ($self->{'buttonname'}, "HList", "Invisible");
+	}
+	else {
+	    $classification = $self->get_entry ($self->{'buttonname'}, "VList", "Invisible");
+	}
+    }
+
+    $self->process_hash($top_h, $classification);
+   
+    return  $classification;
+
+}
+
+sub auto_get_classify_info
+{
+    my $self = shift (@_);
+    my ($classifyinfo) = @_;
+
+    $self->process_hash($self->{'path_hash'}, $classifyinfo);
+
+    return $classifyinfo;
+}
+
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my ($classifyinfo);
+
+    if ($self->{'hlist_at_top'}) {
+	$classifyinfo = $self->get_entry ($self->{'buttonname'}, "HList", "Invisible");
+    }
+    else {
+	$classifyinfo = $self->get_entry ($self->{'buttonname'}, "VList", "Invisible");
+    }
+
+    if (defined $self->{'subjectfile'}) {
+	return $self->hfile_get_classify_info($classifyinfo);
+    }
+    else {
+	return $self->auto_get_classify_info($classifyinfo);
+    }
+}
+
+
+1;
+
+
+
Index: /main/tags/3.01/gsdl/perllib/classify/List.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/List.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/List.pm	(revision 21112)
@@ -0,0 +1,249 @@
+###########################################################################
+#
+# List.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# simple list classifier plugin 
+# to see the options, run "perl -S classinfo.pl List"
+
+use BasClas;
+package List;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use sorttools;
+
+sub BEGIN {
+    @List::ISA = ('BasClas');
+}
+
+my $arguments = 
+    [ { 'name' => "metadata",
+	'desc' => "{List.metadata}",
+	'type' => "metadata",
+	'reqd' => "no" },
+      { 'name' => "buttonname",
+  	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "sort",
+	'desc' => "{List.sort}",
+	'type' => "string",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "List",
+		'desc'     => "{List.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    # Transfer value from Auto Parsing to the variable name that used in previous GreenStone.
+    my (@meta_list,$meta1);
+    if ($self->{"metadata"}) {
+	@meta_list = split(/,/, $self->{"metadata"});
+	$meta1 = $meta_list[0];
+	$self->{'meta_list'} = \@meta_list;
+    } else {
+	$meta1=undef;
+	@meta_list = undef;
+    }
+
+    if (!$self->{"buttonname"}) {
+	if (defined ($self->{'metadata'})) {
+	    $self->{"buttonname"} = $self->generate_title_from_metadata($self->{'metadata'});
+	} else {
+	    $self->{"buttonname"} = 'List';
+	}
+    }
+
+    # Further setup
+    # $self->{"sort"} is handled a bit differently - kjdon
+    # undef means to sort, but use the metadata value from -metadata
+    # because there is no one metadata value to get for sorting when 
+    # we have a list of possible metadata
+    # to get no sorting, set $self->{"sort"} = 'nosort'
+    if (!$self->{"sort"}) {
+	if (defined ($self->{"metadata"})) {
+	    $self->{"sort"} = undef;
+	} else {
+	    $self->{"sort"} = "nosort";
+	}
+    }    
+    if (defined  $self->{"sort"} &&  $self->{"sort"} eq "nosort") {
+	$self->{'list'} = [];
+    } else {
+	$self->{'list'} = {};
+    }
+        
+    # Clean out the unused keys
+    delete $self->{"metadata"}; # Delete this key
+    
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    # are we sorting the list??
+    my $nosort = 0;
+    if (defined $self->{'sort'} && $self->{'sort'} eq "nosort") {
+	$nosort = 1;
+    }
+    
+    my $metavalue;
+    my $metaname;
+    if (defined $self->{'meta_list'}) {
+	my $topsection=$doc_obj->get_top_section();
+
+	# find the correct bit of metadata, if multi-valued metadata field
+	if (exists $doc_obj->{'mdoffset'}) { # set by AZCompactList
+
+	    my $mdoffset=$doc_obj->{'mdoffset'} - 1;
+	    
+	    foreach my $m (@{$self->{'meta_list'}}) {
+		my $values_listref=
+		    $doc_obj->get_metadata($topsection, $m);
+		my $array_size = scalar(@{$values_listref});
+		if ($array_size==0 || $array_size < $mdoffset+1) {
+		    $mdoffset = $mdoffset - $array_size;
+		} else {
+		    $metaname = $m;
+		    # get the correct value using the offset
+		    $metavalue=@$values_listref[$mdoffset];
+                    # use a special format for docOID...
+		    $doc_OID .= ".offset$mdoffset";
+		    last;
+		}
+	    }
+	} else {
+	    # use the first available metadata
+	    foreach my $m (@{$self->{'meta_list'}}) {
+		$metavalue = $doc_obj->
+		    get_metadata_element($topsection, $m);
+		$metaname = $m;
+		last if defined $metavalue;
+	    }
+	} 
+	# if we haven't found a metavalue, then the doc shouldn't be included
+	return unless defined $metavalue;
+    }
+    
+    # we know the doc should be included, add it now if we are not sorting
+    if ($nosort) {
+	push (@{$self->{'list'}}, $doc_OID);
+	return;
+    }
+
+    #check for a sort element other than our metadata
+    if (defined $self->{'sort'}) {
+	my $sortmeta;
+	if ($self->{'sort'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'sort'});
+	    if (defined $sortmeta && !$self->{'no_metadata_formatting'}) {
+		$sortmeta = &sorttools::format_metadata_for_sorting($self->{'sort'}, $sortmeta, $doc_obj);
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+	$self->{'list'}->{$doc_OID} = $sortmeta;
+    } else {
+	# we add to the list based on metadata value
+	# but we need to do the same formatting as for sort value
+	($metavalue) = &sorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj) unless $self->{'no_metadata_formatting'};
+	$self->{'list'}->{$doc_OID} = $metavalue;
+    }
+    my $id = $self->get_number();
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "memberof", "CL$id");
+}
+
+
+sub get_classify_info {
+    my $self = shift (@_);
+    my ($no_thistype) = @_;
+    $no_thistype = 0 unless defined $no_thistype;
+    my $memberof = &supports_memberof();
+
+    my @list = ();
+    if (defined $self->{'sort'} && $self->{'sort'} eq "nosort") {
+	@list = @{$self->{'list'}};
+    } else {
+	if (keys %{$self->{'list'}}) {
+	    @list = sort {$self->{'list'}->{$a} 
+			  cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+	}	
+    }
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>'VList',
+			'Title'=>$self->{'buttonname'},
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = 'Invisible' unless $no_thistype;
+    # always supports memberof
+    $classifyinfo{'supportsmemberof'} = $memberof;
+
+    foreach my $OID (@list) {
+	my $hashref={};
+	# special oid format, if using offsets (from AZCompactList)
+	if ($OID =~ s/\.offset(\d+)$//) {
+	    $hashref->{'offset'}=$1;
+	}
+	$hashref->{'OID'}=$OID;
+
+	push (@{$classifyinfo{'contains'}}, $hashref);
+    }
+
+    return \%classifyinfo;
+}
+
+sub supports_memberof {
+    my $self = shift(@_);
+
+    return "true";
+}
+
+1;
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/classify/Phind.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/Phind.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/Phind.pm	(revision 21112)
@@ -0,0 +1,1503 @@
+###########################################################################
+#
+# Phind.pm -- the Phind classifier
+#
+# Copyright (C) 2000 Gordon W. Paynter
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# The Phind clasifier plugin. 
+# Type "classinfo.pl Phind" at the command line for a summary.
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package Phind;
+
+use BasClas;
+use util;
+use ghtml;
+use unicode;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+my @removedirs = ();
+
+my %wanted_index_files = ('td'=>1,
+			  't'=>1,
+			  'ti'=>1,
+			  'tl'=>1,
+			  'tsd'=>1,
+			  'idb'=>1,
+			  'ib1'=>1,
+			  'ib2'=>1,
+			  'ib3'=>1,
+			  'i'=>1,
+			  'il'=>1,
+			  'w'=>1,
+			  'wa'=>1);
+
+sub BEGIN {
+    @Phind::ISA = ('BasClas');
+}
+
+sub END {
+    
+    # Tidy up stray files - we do this here as there's some weird problem
+    # preventing us from doing it in the get_classify_info() function (on
+    # windows at least) where the close() appears to fail on txthandle and
+    # dochandle, thus preventing us from deleting those files
+
+    foreach my $dir (@removedirs) {
+	if (-d $dir && opendir (DIR, $dir)) {
+	    my @files = readdir DIR;
+	    closedir DIR;
+	
+	    foreach my $file (@files) {
+		next if $file =~ /^\.\.?$/;
+		my ($suffix) = $file =~ /\.([^\.]+)$/;
+		if (!defined $suffix || !defined $wanted_index_files{$suffix}) {
+		    # delete it!
+		    &util::rm (&util::filename_cat ($dir, $file));
+		}
+	    }
+	}
+    }
+}
+
+my $arguments =
+    [ { 'name' => "text",
+	'desc' => "{Phind.text}",
+	'type' => "string",
+	'deft' => "section:Title,section:text",
+	'reqd' => "no" },
+      { 'name' => "title",
+	'desc' => "{Phind.title}",
+	'type' => "metadata",
+	'deft' => "Title",
+	'reqd' => "no" },
+      { 'name' => "buttonname",
+	'desc' => "{BasClas.buttonname}",
+	'type' => "string",
+	'deft' => "Phrase",
+	'reqd' => "no" },
+      { 'name' => "language",
+	'desc' => "{Phind.language}",
+	'type' => "string",
+	'deft' => "en",
+	'reqd' => "no" },
+      { 'name' => "savephrases",
+	'desc' => "{Phind.savephrases}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "suffixmode",
+	'desc' => "{Phind.suffixmode}",
+	'type' => "int",
+	'deft' => "1",
+	'range' => "0,1",
+	'reqd' => "no" },
+      { 'name' => "min_occurs",
+	'desc' => "{Phind.min_occurs}",
+	'type' => "int",
+	'deft' => "2",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "thesaurus",
+	'desc' => "{Phind.thesaurus}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "untidy",
+	'desc' => "{Phind.untidy}",
+	'type' => "flag",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "Phind",
+		'desc'     => "{Phind.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+# Phrase delimiter symbols - these should be abstracted out someplace
+
+my $colstart = "COLLECTIONSTART";
+my $colend   = "COLLECTIONEND";
+my $doclimit = "DOCUMENTLIMIT";
+my $senlimit = "SENTENCELIMIT";
+my @delimiters = ($colstart, $colend, $doclimit, $senlimit);
+
+
+# Create a new Phind browser based on collect.cfg
+
+sub new {
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasClas($classifierslist,$inputargs,$hashArgOptLists): new BasClas($classifierslist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Ensure the Phind generate scripts are in place
+    my $file1 = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "suffix");
+    $file1 .= ".exe" if $ENV{'GSDLOS'} =~ /^windows$/;
+    my $src = &util::filename_cat($ENV{'GSDLHOME'}, "src", "phind", "generate");
+    if (!(-e $file1)) {
+	print STDERR "Phind.pm: ERROR: The Phind \"suffix\" program is not installed.\n\n";
+	exit(1);
+    }
+    
+    # Transfer value from Auto Parsing to the variable name that used in previous GreenStone.
+    $self->{"indexes"} = $self->{"text"};
+
+    # Further setup
+    $self->{'collection'} = $ENV{'GSDLCOLLECTION'}; # classifier information
+    $self->{'collectiondir'} = $ENV{'GSDLCOLLECTDIR'}; # collection directories
+    if (! defined $self->{'builddir'}) {
+	$self->{'builddir'} = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "building");
+    } 
+    $self->{'total'} = 0;
+    
+    # Clean out the unused keys
+    delete $self->{"text"};
+
+    return bless $self, $class;
+}
+
+
+# Initialise the Phind classifier
+
+sub init {
+    my $self = shift (@_);
+
+    # ensure we have a build directory
+    my $builddir = $self->{'builddir'};
+    die unless (-e "$builddir");
+
+    # create Phind directory
+    my $phnumber = 1;
+    my $phinddir = &util::filename_cat($builddir, "phind1");
+    while (-e "$phinddir") {
+	$phnumber++;
+	$phinddir = &util::filename_cat($builddir, "phind$phnumber");
+    }
+    &util::mk_dir("$phinddir");
+    $self->{'phinddir'} = $phinddir;
+    $self->{'phindnumber'} = $phnumber;
+
+    push(@removedirs, $phinddir) unless $self->{'untidy'};
+
+    # open filehandles for documents and text
+    my $clausefile =  &util::filename_cat("$phinddir", "clauses");
+    &util::rm($clausefile) if (-e $clausefile);
+
+    my $txthandle = 'TEXT' . $phnumber;
+    open($txthandle, ">$clausefile") || die "Cannot open $clausefile: $!";
+    $self->{'txthandle'} = $txthandle;
+
+    my $docfile = &util::filename_cat("$phinddir", "docs.txt");
+    &util::rm($docfile) if (-e $docfile);
+
+    my $dochandle = 'DOC' . $phnumber;
+    open($dochandle, ">$docfile") || die "Cannot open $docfile: $!";
+    $self->{'dochandle'} = $dochandle;
+    
+}
+
+
+# Classify each document.
+#
+# Each document is passed here in turn.  The classifier extracts the 
+# text of each and stores it in the clauses file.  Document details are
+# stored in the docs.txt file. 
+
+sub classify {
+    my $self = shift (@_);
+    my $doc_obj = shift @_;
+
+    my $verbosity = $self->{'verbosity'};
+    my $top_section = $doc_obj->get_top_section();
+
+    my $titlefield = $self->{'title'};
+    
+    my $title = $doc_obj->get_metadata_element ($top_section, $titlefield);
+    if (!defined($title)) {
+	$title = "";
+	print STDERR "Phind: document has no title\n";
+    }
+    print "process: $title\n" if ($verbosity > 2);
+
+    # Only consider the file if it is in the correct language
+    my $doclanguage = $doc_obj->get_metadata_element ($top_section, "Language");
+    my $phrlanguage = $self->{'language'};
+    return if ($doclanguage && ($doclanguage !~ /$phrlanguage/i));
+    
+    # record this file
+    $self->{'total'} ++;
+    # what is $file ???
+    # print "file $self->{'total'}: $file\n" if ($self->{'$verbosity'});
+
+    # Store document details
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+    my $dochandle = $self->{'dochandle'};
+    print $dochandle "<Document>\t$OID\t$title\n";
+    
+    # Store the text occuring in this object
+
+    # output the document delimiter
+    my $txthandle = $self->{'txthandle'};
+    print $txthandle "$doclimit\n";
+
+    # iterate over the required indexes and store their text
+    my $indexes = $self->{'indexes'};
+    my $text = "";
+    my ($part, $level, $field, $section, $data, $dataref);
+
+    foreach $part (split(/,/, $indexes)) {
+
+	# Each field has a level and a data element ((e.g. document:Title)
+	($level, $field) = split(/:/, $part);
+	die unless ($level && $field);
+	
+	# Extract the text from every section
+	# (In phind, document:text and section:text are equivalent)
+	if ($field eq "text") {
+	    $data = "";
+	    $section = $doc_obj->get_top_section();
+	    while (defined($section)) {
+		$data .= $doc_obj->get_text($section) . "\n";
+		$section = $doc_obj->get_next_section($section);
+	    }
+	    $text .= convert_gml_to_tokens($phrlanguage, $data) . "\n";
+	}
+	
+	# Extract a metadata field from a document
+	# (If there is more than one element of the given type, get them all.)
+	elsif ($level eq "document") {
+	    $dataref = $doc_obj->get_metadata($doc_obj->get_top_section(), $field);
+	    foreach $data (@$dataref) {
+		$text .= convert_gml_to_tokens($phrlanguage, $data) . "\n";
+	    } 
+	}
+
+	# Extract metadata from every section in a document
+	elsif ($level eq "section") {
+	    $data = "";
+	    $section = $doc_obj->get_top_section();
+	    while (defined($section)) {
+		$dataref = $doc_obj->get_metadata($section, $field);
+		$data .= join("\n", @$dataref) . "\n";
+		$section = $doc_obj->get_next_section($section);
+	    }
+	    $text .= convert_gml_to_tokens($phrlanguage, $data) . "\n";
+	} 
+	
+	# Some sort of specification which I don't understand
+	else {
+	    die "Unknown level ($level) in Phind index ($part)\n";
+	}
+	
+    }
+
+    # output the text
+    $text =~ tr/\n//s;
+    print $txthandle "$text";
+}
+
+
+# Construct the classifier from the information already gathered
+#
+# When get_classify_info is called, the clauses and docs.txt files have
+# already been constructed in the Phind directory.  This function will
+# translate them into compressed, indexed MGPP files that can be read by
+# the phindcgi script.  It will also register our classifier so that it
+# shows up in the navigation bar.
+
+sub get_classify_info {
+    my $self = shift (@_);
+    my ($gli) = @_;    
+
+    close $self->{'dochandle'};
+    close $self->{'txthandle'};
+    my $verbosity = $self->{'verbosity'};
+    my $out = $self->{'outhandle'};
+    my $phinddir = $self->{'phinddir'};
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} !~ /^windows$/i) {
+	$osextra = " -d /";
+    }
+
+    print STDERR "</Stage>\n" if $gli;
+
+    if ($verbosity) {
+	print $out "\n*** Phind.pm generating indexes for ", $self->{'indexes'}, "\n";
+	print $out "***          in ", $self->{'phinddir'}, "\n";
+    }
+
+    print STDERR "<Stage name='Phind'>\n" if $gli;
+
+    # Construct phind indexes
+    my $suffixmode = $self->{'suffixmode'};
+    my $min_occurs = $self->{'min_occurs'};
+    my ($command, $status);
+    
+    # Generate the vocabulary, symbol statistics, and numbers file
+    # from the clauses file
+    print $out "\nExtracting vocabulary and statistics\n" if $verbosity;
+    print STDERR "<Phase name='ExtractingVocab'/>\n" if $gli;
+    &extract_vocabulary($self);
+ 
+    # Use the suffix program to generate the phind/phrases file
+    print $out "\nExtracting phrases from processed text (with suffix)\n" if $verbosity;
+    print STDERR "<Phase name='ExtractingPhrase'/>\n" if $gli;
+    &execute("suffix \"$phinddir\" $suffixmode $min_occurs $verbosity", $verbosity, $out);
+
+    # check that we generated some files. It's not necessarily an error if
+    # we didn't (execute() would have quit on error), but we can't go on.
+    my $phrasesfile=&util::filename_cat($self->{'phinddir'}, 'phrases');
+    if (! -r $phrasesfile) {
+	print STDERR "<Warning name='NoPhrasesFound'/>\n" if $gli;
+	print $out "\nNo phrases found for Phind classifier!\n";
+	return;
+    }   
+
+    # Create the phrase file and put phrase numbers in phind/phrases
+    print $out "\nSorting and renumbering phrases for input to mgpp\n" if $verbosity;
+    print STDERR "<Phase name='SortAndRenumber'/>\n" if $gli;
+    &renumber_phrases($self);
+
+    print $out "\nCreating phrase databases\n";
+    print STDERR "<Phase name='PhraseDatabases'/>\n" if $gli;
+    my $mg_input = &util::filename_cat($phinddir, "pdata.txt");
+    my $mg_stem = &util::filename_cat($phinddir, "pdata");
+
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T1 \"$mg_input\"", $verbosity, $out);
+    &execute("mgpp_compression_dict $osextra -f \"$mg_stem\"", $verbosity, $out);
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T2 \"$mg_input\"", $verbosity, $out);
+
+    # create the mg index of words
+    print $out "\nCreating word-level search indexes\n";
+    print STDERR "<Phase name='WordLevelIndexes'/>\n" if $gli;
+    $mg_input = &util::filename_cat($phinddir, "pword.txt");
+    $mg_stem = &util::filename_cat($phinddir, "pword");
+
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T1 -I1 \"$mg_input\"", $verbosity, $out);
+    &execute("mgpp_compression_dict $osextra -f \"$mg_stem\"", $verbosity, $out);
+    &execute("mgpp_perf_hash_build $osextra -f \"$mg_stem\"", $verbosity, $out);
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T2 -I2 \"$mg_input\"", $verbosity, $out);
+    &execute("mgpp_weights_build $osextra -f \"$mg_stem\"", $verbosity, $out);
+    &execute("mgpp_invf_dict $osextra -f \"$mg_stem\"", $verbosity, $out);
+
+    &execute("mgpp_stem_idx $osextra -f \"$mg_stem\" -s 1", $verbosity, $out);
+    &execute("mgpp_stem_idx $osextra -f \"$mg_stem\" -s 2", $verbosity, $out);
+    &execute("mgpp_stem_idx $osextra -f \"$mg_stem\" -s 3", $verbosity, $out);
+
+    # create the mg document information database
+    print $out "\nCreating document information databases\n";
+    print STDERR "<Phase name='DocInfoDatabases'/>\n" if $gli;
+    $mg_input = &util::filename_cat($phinddir, "docs.txt");
+    $mg_stem = &util::filename_cat($phinddir, "docs");
+
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T1 \"$mg_input\"", $verbosity, $out);
+    &execute("mgpp_compression_dict $osextra -f \"$mg_stem\"", $verbosity, $out);
+    &execute("mgpp_passes $osextra -f \"$mg_stem\" -T2 \"$mg_input\"", $verbosity, $out);
+
+    # Return the information about the classifier that we'll later want to
+    # use to create macros when the Phind classifier document is displayed.
+    my %classifyinfo = ('thistype'=>'Invisible',
+                        'childtype'=>'Phind', 
+                        'Title'=>$self->{'buttonname'},
+                        'parameters'=>"phindnumber=$self->{'phindnumber'}",
+			'contains'=>[]);
+    
+    my $collection = $self->{'collection'};
+    my $url = "library?a=p&p=phind&c=$collection";
+    push (@{$classifyinfo{'contains'}}, {'OID'=>$url});
+   
+    return \%classifyinfo;
+}
+
+
+
+sub convert_gml_to_tokens {
+    
+    my ($language_exp, $text) = @_;
+
+    # escape any magic words... - jrm21
+    foreach my $delim (@delimiters) {
+	my $replacement=lc($delim);
+	my $num= $text=~ s/$delim/$replacement/g;
+	if (!$num) {$num=0;}
+    }
+
+    if ($language_exp =~ /en/) {
+	return &convert_gml_to_tokens_EN($text);
+    }
+
+    if ($language_exp =~ /zh/) {
+	return &convert_gml_to_tokens_ZH($text);
+    }  
+    
+    $_ = $text;
+
+    # 1. remove GML tags
+
+    # Remove everything that is in a tag
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isgo;
+    s/\s*<br>\s*/ LINEBREAK /isgo;
+    s/<[^>]*>/ /sgo;
+
+    # Now we have the text, but it may contain HTML 
+    # elements coded as &gt; etc.  Remove these tags. 
+    s/&amp;/&/sgo;
+    s/&lt;/</sgo;
+    s/&gt;/>/sgo;
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isgo;
+    s/\s*<br>\s*/ LINEBREAK /isgo;
+    s/<[^>]*>/ /sgo;
+
+    # replace<p> and <br> placeholders with clause break symbol (\n)
+    s/\s+/ /gso;
+    s/PARAGRAPHBREAK/\n/sgo;
+    s/LINEBREAK/\n/sgo;
+
+    
+
+
+    # 2. Split the remaining text into space-delimited tokens
+
+    # Convert any HTML special characters (like &quot;) to their UTF8 equivalent
+    s/&([^;]+);/&unicode::ascii2utf8(\&ghtml::getcharequiv($1,1))/gse;
+
+    # Split text at word boundaries
+    s/\b/ /go;
+
+    # 3. Convert the remaining text to "clause format"
+
+    # Insert newline if the end of a sentence is detected
+    # (delimter is:  "[\.\?\!]\s")
+    # s/\s*[\.\?\!]\s+/\n/go; 
+
+    # remove unnecessary punctuation and replace with clause break symbol (\n)
+    s/[^\w ]/\n/go;
+
+    # remove extraneous whitespace
+    s/ +/ /sgo;
+    s/^\s+//mgo;
+    s/\s*$/\n/mgo;
+
+    # remove lines that contain one word or less
+    s/^\S*$//mgo;
+    s/^\s*$//mgo;
+    tr/\n//s;
+
+    return $_;
+}
+
+# a chinese version
+sub convert_gml_to_tokens_ZH {
+
+    $_ = shift @_;
+
+    # Replace all whitespace with a simple space
+    s/\s+/ /gs;
+    # Remove everything that is in a tag
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # Now we have the text, but it may contain HTML 
+    # elements coded as &gt; etc.  Remove these tags. 
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+
+    s/\s+/ /sg;
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # remove &amp; and other miscellaneous markup tags
+    s/&amp;/&/sg;
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+    s/&amp;/&/sg;
+
+    # replace<p> and <br> placeholders with carriage returns
+    s/PARAGRAPHBREAK/\n/sg;
+    s/LINEBREAK/\n/sg;
+
+    
+#    print STDERR "text:$_\n";
+    return $_;
+}
+
+# A version of convert_gml_to_tokens that is fine-tuned to the English language.
+
+sub convert_gml_to_tokens_EN {
+    $_ = shift @_;
+
+    # FIRST, remove GML tags
+
+    # Replace all whitespace with a simple space
+    s/\s+/ /gs;
+
+    # Remove everything that is in a tag
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # Now we have the text, but it may contain HTML 
+    # elements coded as &gt; etc.  Remove these tags. 
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+
+    s/\s+/ /sg;
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # remove &amp; and other miscellaneous markup tags
+    s/&amp;/&/sg;
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+    s/&amp;/&/sg;
+
+    # replace<p> and <br> placeholders with carriage returns
+    s/PARAGRAPHBREAK/\n/sg;
+    s/LINEBREAK/\n/sg;
+
+
+    # Exceptional punctuation
+    # 
+    # We make special cases of some punctuation
+
+    # remove any apostrophe that indicates omitted letters 
+    s/(\w+)\'(\w*\s)/ $1$2 /g;
+
+    # remove period that appears in a person's initals
+    s/\s([A-Z])\./ $1 /g;
+
+    # replace hyphens in hypheanted words and names with a space
+    s/([A-Za-z])-\s*([A-Za-z])/$1 $2/g;
+
+    # Convert the remaining text to "clause format",
+    # This means removing all excess punctuation and garbage text,
+    # normalising valid punctuation to fullstops and commas,
+    # then putting one cluse on each line.
+
+    # Insert newline when the end of a sentence is detected
+    # (delimter is:  "[\.\?\!]\s")
+    s/\s*[\.\?\!]\s+/\n/g; 
+
+    # split numbers after four digits
+    s/(\d\d\d\d)/$1 /g;
+
+    # split words after 32 characters
+
+    # squash repeated punctuation 
+    tr/A-Za-z0-9 //cs;
+
+    # save email addresses
+    # s/\w+@\w+\.[\w\.]+/EMAIL/g;
+
+    # normalise clause breaks (mostly punctuation symbols) to commas
+    s/[^A-Za-z0-9 \n]+/ , /g;
+
+    # Remove repeated commas, and replace with newline
+    s/\s*,[, ]+/\n/g;
+
+    # remove extra whitespace
+    s/ +/ /sg;
+    s/^\s+//mg;
+    s/\s*$/\n/mg;
+
+    # remove lines that contain one word or less
+    s/^\w*$//mg;
+    s/^\s*$//mg;
+    tr/\n//s;
+
+    return $_;
+
+}
+
+
+
+# Execute a system command
+
+sub execute {
+    my ($command, $verbosity, $outhandle) = @_;
+    print $outhandle "Executing: $command\n"  if ($verbosity > 2);
+    $! = 0;
+    my $status = system($command);
+    if ($status != 0) {
+	print STDERR "Phind - Error executing '$command': $!\n";
+	exit($status);  # this causes the build to fail... 
+    }
+}
+
+
+# Generate the vocabulary, symbol statistics, and numbers file from the
+# clauses file.  This is legacy code, so is a bit messy and probably wont
+# run under windows.
+
+sub extract_vocabulary {
+    my ($self) = @_;
+    
+    my $verbosity = $self->{'verbosity'};
+    my $out = $self->{'outhandle'};
+
+    my $collectiondir = $self->{'collectiondir'};
+    my $phinddir = $self->{'phinddir'};
+
+    my $language_exp = $self->{'language'};
+
+    my ($w, $l, $line, $word);
+    
+    my ($first_delimiter, $last_delimiter,
+	$first_stopword, $last_stopword,
+	$first_extractword, $last_extractword,
+	$first_contentword, $last_contentword,
+	$phrasedelimiter);
+    
+    my $thesaurus = $self->{'thesaurus'};
+    my ($thesaurus_links, $thesaurus_terms, 
+	%thesaurus, $first_thesaurusword, $last_thesaurusword);
+
+    my %symbol;
+    my (%freq);
+
+    print $out "Calculating vocabulary\n" if ($verbosity > 1);
+
+    # Read and store the stopwords
+    my $stopdir = &util::filename_cat($ENV{'GSDLHOME'}, "etc", "packages", "phind", "stopword");
+    my $stopword_files = ();
+    my ($language, $language_dir, $file, $file_name);
+    my %stopwords;
+
+    # Examine each directory in the stopword directory
+    opendir(STOPDIR, $stopdir);
+    foreach $language (readdir STOPDIR) {
+
+	# Ignore entries that do not match the classifier's language
+	next unless ($language =~ /$language_exp/);
+	$language_dir = &util::filename_cat($stopdir, $language);
+	next unless (-d "$language_dir");
+
+	opendir(LANGDIR, $language_dir);
+	foreach $file (readdir LANGDIR) {
+
+	    # Ignore entries that are not stopword files
+	    next unless ($file =~ /sw$/);
+	    $file_name = &util::filename_cat($language_dir, $file);
+	    next unless (-f "$file_name");
+
+	    # Read the stopwords
+	    open(STOPFILE, "<$file_name");
+	    while (<STOPFILE>) {
+		s/^\s+//;
+		s/\s.*//;
+		$word = $_;
+		$l = lc($word);
+		$stopwords{$l} = $word;
+	    }
+	    close STOPFILE;
+
+	}
+	closedir LANGDIR;
+    }
+    closedir STOPDIR;
+
+    # Read thesaurus information
+    if ($thesaurus) {
+
+	# link file exists
+	$thesaurus_links = &util::filename_cat($collectiondir, "etc", "$thesaurus.lnk");
+	die "Cannot find thesaurus link file" unless (-e "$thesaurus_links");
+
+	# ensure term file exists in the correct language
+	if ($language_exp =~ /^([a-z][a-z])/) {
+	    $language = $1;
+	} else {
+	    $language = 'en';
+	}
+	$thesaurus_terms = &util::filename_cat($collectiondir, "etc", "$thesaurus.$language");
+	die "Cannot find thesaurus term file" unless (-e "$thesaurus_terms");
+	
+
+	# Read the thesaurus terms
+	open(TH, "<$thesaurus_terms");
+	while(<TH>) {
+	    s/^\d+ //;
+	    s/\(.*\)//;
+	    foreach $w (split(/\s+/, $_)) {
+		$thesaurus{lc($w)} = $w;
+	    }
+	}
+	close TH;
+    }
+
+    # Read words in the text and count occurences
+    open(TXT, "<$phinddir/clauses");
+
+    my @words;
+    while(<TXT>) {
+	$line = $_;
+	next unless ($line =~ /./);
+	
+	@words = split(/\s+/, $line);
+	foreach $w (@words) {
+	    $l = lc($w);
+	    $w = $l if ((defined $stopwords{$l}) || (defined $thesaurus{$l}));
+	    $freq{$w}++;
+	}
+	$freq{$senlimit}++;
+    }
+    
+    close TXT;
+
+    # Calculate the "best" form of each word
+    my (%bestform, %totalfreq, %bestfreq);
+
+    foreach $w (sort (keys %freq)) {
+	$l = lc($w);
+	
+	# totalfreq is the number of times a term appears in any form
+	$totalfreq{$l} += $freq{$w};
+	
+	if (defined $stopwords{$l}) {
+	    $bestform{$l} = $stopwords{$l};
+	    
+	} elsif (defined $thesaurus{$l}) {
+	    $bestform{$l} = $thesaurus{$l};
+	    
+	} elsif (!$bestform{$l} || ($freq{$w} > $bestfreq{$l})) {
+	    $bestfreq{$l} = $freq{$w};
+	    $bestform{$l} = $w;
+	}
+    }
+    undef %freq;
+    undef %bestfreq;
+    
+
+    # Assign symbol numbers to tokens
+    my $nextsymbol = 1;
+    my (@vocab);
+    
+    # Delimiters
+    $first_delimiter = 1;
+    
+    foreach $word (@delimiters) {
+
+#	$word = lc($word); # jrm21
+	$word = uc($word);
+	$bestform{$word} = $word;
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_delimiter = $nextsymbol - 1;
+    # Stopwords
+    $first_stopword = $nextsymbol;
+    
+    foreach my $word (sort keys %stopwords) {
+	# don't include stopword unless it occurs in the text
+	$word = lc($word);
+	next unless ($totalfreq{$word});
+	next if ($symbol{$word});
+	
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_stopword = $nextsymbol - 1;
+    $first_contentword = $nextsymbol;
+    
+    # Thesaurus terms
+    if ($thesaurus) {
+	$first_thesaurusword = $nextsymbol;
+    
+	foreach my $word (sort keys %thesaurus) {
+	    
+	    $word = lc($word);
+	    next if ($symbol{$word});
+	    $bestform{$word} = $thesaurus{$word};
+	    
+	    $vocab[$nextsymbol] = $word;
+	    $symbol{$word} = $nextsymbol;
+	    $nextsymbol++;
+	    
+	}
+	$last_thesaurusword = $nextsymbol - 1;
+    }
+
+    # Other content words
+    $first_extractword = $nextsymbol;
+    
+    foreach my $word (sort (keys %bestform)) {
+	
+	next if ($symbol{$word});
+	
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_extractword = $nextsymbol - 1;
+    $last_contentword = $nextsymbol - 1;
+    
+    # Outut the words
+    print $out "Saving vocabulary in $phinddir/clauses.vocab\n" if ($verbosity > 1);
+    open(VOC, ">$phinddir/clauses.vocab");
+
+    for (my $i = 1; $i < $nextsymbol; $i++) {
+	$w = $vocab[$i];
+
+	print VOC "$bestform{$w}\n";
+	$totalfreq{$w} = 0 unless ($totalfreq{$w});
+    }
+    close VOC;
+
+
+    # Create statistics file
+    # Output statistics about the vocablary
+    print $out "Saving statistics in $phinddir/clauses.stats\n" if ($verbosity > 1);
+    &util::rm("$phinddir/clauses.stats") if (-e "$phinddir/clauses.stats");
+
+    open(STAT, ">$phinddir/clauses.stats")
+	|| die "Cannot open $phinddir/clauses.stats: $!";
+
+    print STAT "first_delimiter $first_delimiter\n";
+    print STAT "last_delimiter $last_delimiter\n";
+    print STAT "first_stopword $first_stopword\n";
+    print STAT "last_stopword $last_stopword\n";
+    if ($thesaurus) {
+	print STAT "first_thesaurusword $first_thesaurusword\n";
+	print STAT "last_thesaurusword $last_thesaurusword\n";
+    }
+    print STAT "first_extractword $first_extractword\n";
+    print STAT "last_extractword $last_extractword\n";
+    print STAT "first_contentword $first_contentword\n";
+    print STAT "last_contentword $last_contentword\n";
+    print STAT "first_symbol $first_delimiter\n";
+    print STAT "last_symbol $last_contentword\n";
+    print STAT "first_word $first_stopword\n";
+    print STAT "last_word $last_contentword\n";
+    close STAT;
+
+    undef @vocab;
+
+
+    # Create numbers file
+    # Save text as symbol numbers
+    print $out "Saving text as numbers in $phinddir/clauses.numbers\n" if ($verbosity > 1);
+    
+    open(TXT, "<$phinddir/clauses");
+    open(NUM, ">$phinddir/clauses.numbers");
+    
+##    $phrasedelimiter = $symbol{lc($senlimit)}; # jrm21
+##    print NUM "$symbol{lc($colstart)}\n"; # jrm21
+    $phrasedelimiter = $symbol{$senlimit};
+    print NUM "$symbol{$colstart}\n";
+    
+    # set up the special symbols that delimit documents and sentences
+    while(<TXT>) {
+	
+	# split sentence into a list of tokens
+	$line = $_;
+	next unless ($line =~ /./);
+	@words = split(/\s+/, $line);
+	
+	# output one token at a time
+	foreach $word (@words) {
+# don't lower-case special delimiters - jrm21
+	    if (!map {if ($word eq $_) {1} else {()}} @delimiters) {
+		$word = lc($word);
+	    }
+	    print NUM "$symbol{$word}\n";
+	}
+	
+	# output phrase delimiter
+	print NUM "$phrasedelimiter\n";
+    }
+    
+    close TXT;
+#    print NUM "$symbol{lc($colend)}\n";# jrm21
+    print NUM "$symbol{$colend}\n";
+    close NUM;
+
+    # Save thesaurus  data in one convienient file
+    if ($thesaurus) {
+
+	my $thesaurusfile = &util::filename_cat($phinddir, "$thesaurus.numbers");
+
+
+	print $out "Saving thesaurus as numbers in $thesaurusfile\n" 
+	    if ($verbosity > 1);
+
+	# Read the thesaurus terms
+	my ($num, $text, %thes_symbols);
+	
+	open(TH, "<$thesaurus_terms");
+	while(<TH>) {
+	    chomp;
+	    @words = split(/\s+/, $_);
+	    $num = shift @words;
+	    $text = "";
+
+	    # translate words into symbol numbers
+	    foreach $word (@words) {
+		$word = lc($word);
+		if ($symbol{$word}) {
+		    $text .= "s$symbol{$word} ";
+		} elsif ($verbosity) {
+		    print $out "Phind: No thesaurus symbol, ignoring \"$word\"\n";
+		}
+	    }
+	    $text =~ s/ $//;
+	    $thes_symbols{$num} = $text;
+	}
+	close TH;
+
+	# Read the thesaurus links and write the corresponding data
+	open(TH, "<$thesaurus_links");
+	open(THOUT, ">$thesaurusfile");
+
+	while(<TH>) {
+	    chomp;
+	    ($num, $text) = split(/:/, $_);
+
+	    if (defined($thes_symbols{$num})) {
+		print THOUT "$num:$thes_symbols{$num}:$text\n";
+	    } else {
+		print THOUT "$num:untranslated:$text\n";
+	    }
+	}
+	close TH;    
+	close THOUT;
+    }
+
+
+
+
+}
+
+
+# renumber_phrases
+#
+# Prepare the phrases file to be input to mgpp.  The biggest problem is
+# reconciling the phrase identifiers used by the suffix program (which
+# we'll call suffix-id numbers) with the numbers used in the thesaurus
+# (theesaurus-id) to create a ciommon set of phind id numbers (phind-id).
+# Phind-id numbers must be sorted by frequency of occurance.
+#
+# Start creating a set of phind-id numbers from the sorted suffix-id
+# numbers and (if required) the thesaurus-id numbers.  Then add any other
+# phrases occuring in the thesaurus.
+#
+# The last thing we have to do is restore the vocabulary information to the
+# phrase file so that the phrases are stored as words, not as symbol
+# numbers.
+
+# The original phrases file looks something like this:
+#   159396-1:s5175:4:1:116149-2:3:d2240,2;d2253;d2254
+#   159409-1:s5263:6:1:159410-2:6:d2122;d2128;d2129;d2130;d2215;d2380
+#   159415-1:s5267:9:1:159418-2:8:d3,2;d632;d633;d668;d1934;d2010;d2281;d2374
+#   159426-1:s5273:5:2:159429-2,115168-17:5:d252;d815;d938;d939;d2361
+
+
+sub renumber_phrases {
+    my ($self) = @_;
+
+    renumber_suffix_data($self);
+    renumber_thesaurus_data($self);
+    restore_vocabulary_data($self);
+
+}
+
+
+
+# renumber_suffix_data
+#
+# Translate phrases file to phrases.2 using phind keys instead
+# of suffix keys and sorting the expansion data.
+
+sub renumber_suffix_data {
+    my ($self) = @_;
+    
+    my $verbosity = $self->{'verbosity'};
+    my $out = $self->{'outhandle'};
+    print $out "Translate phrases: suffix-ids become phind-id's\n" 
+	if ($verbosity);
+    
+    my $phinddir = $self->{'phinddir'};
+    my $infile = &util::filename_cat($phinddir, 'phrases');
+    my $outfile = &util::filename_cat($phinddir, 'phrases.2');
+
+    # Read the phrase file.  Calculate initial set of phind-id
+    # numbers and store (suffixid -> frequency) relation.
+
+    my %suffixtophind;
+    my @totalfrequency;
+    my (@fields, $suffixid);
+    my $nextphind = 1;
+    
+    open(IN, "<$infile");
+    while(<IN>) {
+
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# get next suffixid and phindid
+	$suffixid = shift @fields;
+	$suffixtophind{$suffixid} = $nextphind;
+
+	# store total frequency
+	shift @fields;
+	$totalfrequency[$nextphind] = shift @fields;
+
+	$nextphind++;
+    }
+    close IN;
+
+
+    # Translate phrases file to phrases.2.  Use phind keys (not suffix
+    # keys), sort expansion and document occurance data in order of
+    # descending frequency..
+    open(IN, "<$infile");
+    open(OUT, ">$outfile");
+    
+    my ($phindid, $text, $tf, $countexp, $expansions, $countdocs, $documents);
+    my (@documwents, @newexp, $k, $n);
+    my $linenumber = 0;
+
+    while(<IN>) {
+	
+	# read the line
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# get a phrase number for this line
+	$suffixid = shift @fields;
+	die unless (defined($suffixtophind{$suffixid}));
+	$phindid = $suffixtophind{$suffixid};
+	
+	# get the symbols in the phrase
+	$text = shift @fields;
+
+	# output status information
+	$linenumber++;
+	if ($verbosity > 2) {
+	    if ($linenumber % 1000 == 0) {
+		print $out "line $linenumber:\t$phindid\t$suffixid\t($text)\n";
+	    }
+	    # what are $num and $key??
+	    #print $out "$num: $key\t($text)\n" if ($verbosity > 3);
+	}
+
+	# get the phrase frequency
+	$tf = shift @fields;
+	
+	# get the number of expansions
+	$countexp = shift @fields;
+	
+	# get the expansions, convert them into phind-id numbers, and sort them
+	$expansions = shift @fields;
+	@newexp = ();
+	foreach $k (split(/,/, $expansions)) {
+	    die "ERROR - no phindid for: $k" unless (defined($suffixtophind{$k}));
+	    $n = $suffixtophind{$k};
+	    push @newexp, $n;
+	}
+	@newexp = sort {$totalfrequency[$b] <=> $totalfrequency[$a]} @newexp;
+
+	# get the number of documents
+	$countdocs = shift @fields;
+	
+	# get the documents and sort them
+	$documents = shift @fields;
+	$documents =~ s/d//g;
+	my @documents = split(/;/, $documents);
+	@documents = sort by_doc_frequency @documents;
+
+	# output the phrase data
+ 	print OUT "$phindid:$text:$tf:$countexp:$countdocs:";
+	print OUT join(",", @newexp), ",:", join(";", @documents), ";\n";
+	
+    }
+
+    close IN;
+    close OUT;
+}
+
+
+# renumber_thesaurus_data
+#
+# Translate phrases.2 to phrases.3, adding thesaurus data if available.
+
+sub renumber_thesaurus_data {
+    my ($self) = @_;
+  
+    my $out = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+    my $thesaurus = $self->{'thesaurus'};
+
+    my $phinddir = $self->{'phinddir'};
+    my $infile = &util::filename_cat($phinddir, "phrases.2");
+    my $outfile = &util::filename_cat($phinddir, "phrases.3");
+
+
+    # If no thesaurus is defined, simply move the phrases file.
+    if (!$thesaurus) {
+	print $out "Translate phrases.2: no thesaurus data\n" 
+	    if ($verbosity);
+	&util::mv($infile, $outfile);
+	return;
+    }
+
+    print $out "Translate phrases.2: add thesaurus data\n" 
+	if ($verbosity);
+
+    # 1.
+    # Read thesaurus file and store (symbols->thesaurusid) mapping
+    my $thesaurusfile = &util::filename_cat($phinddir, "$thesaurus.numbers");
+    my %symbolstothesid;
+    my (@fields, $thesid, $symbols);
+    
+    open(TH, "<$thesaurusfile");
+
+    while (<TH>) {
+	
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# get id and text
+	$thesid = shift @fields;
+	$symbols = shift @fields;
+	$symbolstothesid{$symbols} = $thesid;
+    }
+    close TH;    
+
+    # 2. 
+    # Read phrases file to find thesaurus entries that already
+    # have a phindid.  Store their phind-ids for later translation,
+    # and store their frequency for later sorting.
+    my %thesaurustophindid;
+    my %phindidtofrequency;
+    my ($phindid, $freq);
+
+    open(IN, "<$infile");
+    
+    while(<IN>) {
+	
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# phindid and symbols for this line
+	$phindid = shift @fields;
+	$symbols = shift @fields;
+	$freq = shift @fields;
+
+	# do we have a thesaurus id corresponding to this phrase?
+	if (defined($symbolstothesid{$symbols})) {
+	    $thesid = $symbolstothesid{$symbols};
+	    $thesaurustophindid{$thesid} = $phindid;
+	    $phindidtofrequency{$phindid} = $freq;
+	}
+    }
+    close IN;
+  
+    undef %symbolstothesid;
+
+    # 3.
+    # Create phind-id numbers for remaining thesaurus entries,
+    # and note that their frequency is 0 for later sorting.
+    my $nextphindid = $phindid + 1;
+
+    open(TH, "<$thesaurusfile");
+    while(<TH>) {
+	
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# read thesaurus-id and ensure it has a corresponding phind-id
+	$thesid = shift @fields;
+	if (!defined($thesaurustophindid{$thesid})) {
+	    $thesaurustophindid{$thesid} = $nextphindid;
+	    $phindidtofrequency{$nextphindid} = 0;
+	    $nextphindid++;
+	}
+    }
+    close TH;
+
+    # 4.
+    # Translate thesaurus file, replacing thesaurus-id numbers with 
+    # phind-id numbers.
+    my $newthesaurusfile = &util::filename_cat($phinddir, "$thesaurus.phindid");
+    my ($relations, $linkcounter, $linktext, $linktype, @linkdata);
+    my (@links, $linkid, %linkidtotype, $newrelation);
+
+    open(TH, "<$thesaurusfile");
+    open(TO, ">$newthesaurusfile");
+    while(<TH>) {
+	
+	chomp;
+	@fields = split(/:/, $_);
+	
+	# phindid and symbols for this line
+	($thesid, $symbols, $relations) = @fields;
+	
+	die unless ($thesid && $symbols);
+	die unless $thesaurustophindid{$thesid};
+	$phindid = $thesaurustophindid{$thesid};
+
+	# convert each part of the relation string to use phind-id numbers
+	# at the same time, we want to sort the list by frequency.
+	undef %linkidtotype;
+	
+	foreach $linktext (split(/;/, $relations)) {
+	    @linkdata = split(/,/, $linktext);
+	    
+	    # remember the linktype (e.g. BT, NT)
+	    $linktype = shift @linkdata;
+	    
+	    # store the type of each link
+	    foreach $thesid (@linkdata) {
+		die unless (defined($thesaurustophindid{$thesid}));
+		$linkidtotype{$thesaurustophindid{$thesid}} = $linktype;
+	    }
+	}
+
+	# sort the list of links, first by frequency, then by type.
+	@links = sort { ($phindidtofrequency{$b} <=> $phindidtofrequency{$a}) 
+                        or ($linkidtotype{$a} cmp $linkidtotype{$b}) } (keys %linkidtotype);
+	$linkcounter = (scalar @links);
+
+	# create a string describing the link information
+	$linktype = $linkidtotype{$links[0]};
+	$newrelation = $linktype;
+	foreach $linkid (@links) {
+	    if ($linkidtotype{$linkid} ne $linktype) {
+		$linktype = $linkidtotype{$linkid};
+		$newrelation .= ";" . $linktype;
+	    }
+	    $newrelation .= "," . $linkid;
+	}
+	$newrelation .= ";";
+	
+
+	# output the new line
+	print TO "$phindid:$symbols:$linkcounter:$newrelation:\n";
+    }
+    close TH;
+    close TO;
+
+    undef %thesaurustophindid;
+    undef %linkidtotype;
+    undef %phindidtofrequency;
+
+    # 5.
+    # Read thesaurus data (in phind-id format) into memory
+    my %thesaurusdata;
+
+    open(TH, "<$newthesaurusfile");
+    while(<TH>) {
+	chomp;
+	($phindid, $symbols, $linkcounter, $relations) = split(/:/, $_);
+	die unless ($phindid && $symbols);
+	$thesaurusdata{$phindid} = "$symbols:$linkcounter:$relations";
+    }
+    close TH;
+
+    # 6. 
+    # Add thesaurus data to phrases file
+    my ($text, $tf, $countexp, $expansions, $countdocs, $documents);
+    my (@documwents, @newexp, $k, $n);
+    my $linenumber = 0;
+
+    open(IN, "<$infile");
+    open(OUT, ">$outfile");
+
+    # Update existing phrases
+    while(<IN>) {
+	
+	chomp;
+	@fields = split(/:/, $_);
+		
+	# get data for this line
+	$phindid = shift @fields;
+	
+	# output the phrase data, with thesaurus information
+ 	print OUT "$phindid:", join(":", @fields);
+
+	# add thesaurus data
+	if (defined($thesaurusdata{$phindid})) {
+	    @fields = split(/:/, $thesaurusdata{$phindid});
+	    shift @fields;
+	    $linkcounter = shift @fields;
+	    $relations = shift @fields;
+
+	    print OUT ":$linkcounter:$relations";
+	    $thesaurusdata{$phindid} = "";
+	}
+	print OUT "\n";
+    }
+    close IN;
+
+    # Add phrases that aren't already in the file
+    foreach $phindid (sort numerically keys %thesaurusdata) {
+	next unless ($thesaurusdata{$phindid});
+
+	@fields = split(/:/, $thesaurusdata{$phindid});
+	$symbols = shift @fields;
+	$linkcounter = shift @fields;
+	$relations = shift @fields;
+
+	print OUT "$phindid:$symbols:0:0:0:::$linkcounter:$relations\n";
+    }
+    close OUT;
+
+}
+
+# restore_vocabulary_data
+#
+# Read phrases.3 and restore vocabulary information. Then write 
+# this data to the MGPP input files (pword.txt and pdata.txt) and
+# (if requested) to the saved phrases file.
+
+sub restore_vocabulary_data {
+    my ($self) = @_;
+ 
+    my $out = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+    print $out "Translate phrases.3: restore vocabulary\n" if ($verbosity);
+
+    my $phinddir = $self->{'phinddir'};
+    my $infile = &util::filename_cat($phinddir, 'phrases.3');
+    my $vocabfile = &util::filename_cat($phinddir, 'clauses.vocab');
+    my $datafile = &util::filename_cat($phinddir, 'pdata.txt');
+    my $wordfile = &util::filename_cat($phinddir, 'pword.txt');
+
+    my $savephrases = $self->{'savephrases'};
+
+    # 1.
+    # Read the vocabulary file
+    open(V, "<$vocabfile")
+	|| die "Cannot open $vocabfile: $!";
+    my @symbol;
+    my $i = 1;
+    while(<V>) {
+	chomp;
+	$symbol[$i++] = $_;
+    }
+    close V;
+
+    # 2. 
+    # Translate phrases.3 to MGPP input files
+    my ($key, $text, $word, $isThesaurus, $line);
+    my @fields;
+    my $linenumber = 0;
+
+    open(IN, "<$infile");
+    open(DATA, ">$datafile");
+    open(WORD, ">$wordfile");
+
+    # Save the phrases in a separate text file
+    if ($savephrases) {
+	print $out "Saving phrases in $savephrases\n" if ($verbosity);
+	open(SAVE, ">$savephrases");
+    }
+
+    while(<IN>) {
+	
+	# read the line
+	chomp;
+	$line = $_;
+	@fields = split(/:/, $line);
+	
+	# get a phrase number for this line
+	$key = shift @fields;
+	
+	# restore the text of the phrase
+	$text = shift @fields;
+	$text =~ s/s(\d+)/$symbol[$1]/g;
+	if ($text =~ / /) {
+	    $word = "";
+	} elsif ($text ne 'untranslated') {
+	    $word = $text;
+	}
+
+	# output the phrase data
+	print DATA "<Document>";
+ 	print DATA "$key:$text:", join(":", @fields), ":\n";
+	
+	# output the word index search data
+	print WORD "<Document>$word\n";
+
+	# output the phrases to a text file
+	if ($savephrases) {
+	    if ((scalar @fields) == 7) {
+		$isThesaurus = 1;
+	    } else {
+		$isThesaurus = 0;
+	    }
+	    print SAVE $fields[0], "\t", $fields[2], "\t$isThesaurus\t$text\n";
+	}
+    }
+    close IN;
+    close WORD;
+    close DATA;
+    close SAVE if ($savephrases);
+
+}
+
+
+
+# sort routines used to renumber phrases
+
+sub numerically { $a <=> $b }
+
+sub by_doc_frequency { 
+    my $fa = 1;
+    if ($a =~ /,/) {
+	$fa = $a;
+	$fa =~ s/\d+,//;
+    }
+    my $fb = 1;
+    if ($b =~ /,/) {
+	$fb = $b;
+	$fb =~ s/\d+,//;
+    }
+
+    return ($fb <=> $fa);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/classify/SectionList.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/classify/SectionList.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/classify/SectionList.pm	(revision 21112)
@@ -0,0 +1,153 @@
+###########################################################################
+#
+# SectionList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Same as List classifier but includes all sections of document
+# (excluding top level) rather than just top level document 
+# itself
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package SectionList;
+
+use List;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @SectionList::ISA = ('List');
+}
+
+my $arguments = [];
+my $options = { 'name'     => "SectionList",
+		'desc'     => "{SectionList.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+
+
+sub new { 
+    my ($class) = shift (@_);
+    my ($classifierslist,$inputargs,$hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new List($classifierslist,$inputargs,$hashArgOptLists): new List($classifierslist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj, @options) = @_;
+    
+    # @options used by AZCompactList when is uses SectionList internally
+    # are we sorting the list??
+    my $nosort = 0;
+    if (defined $self->{'sort'} && $self->{'sort'} eq "nosort") {
+	$nosort = 1;
+    }
+
+    my $thissection = undef;
+
+    foreach my $option (@options)
+    {
+	if ($option =~ m/^section=(\d+)$/i) 
+	{
+	    $thissection = $1;
+	}
+    }
+
+    my $sortmeta = "";
+    if (!$nosort && defined $self->{'sort'}) {
+	if ($self->{'sort'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'sort'});
+	    if (defined $sortmeta && !$self->{'no_metadata_formatting'}) {
+		$sortmeta = &sorttools::format_metadata_for_sorting($self->{'sort'}, $sortmeta, $doc_obj);
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+    }
+
+    if (defined $thissection) {
+	# just classify the one section
+	$self->classify_section($thissection, $doc_obj, $sortmeta, $nosort);
+    } else    
+    {
+	$thissection = $doc_obj->get_next_section ($doc_obj->get_top_section());
+	while (defined $thissection) {
+	    $self->classify_section($thissection, $doc_obj, $sortmeta, $nosort);
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+}
+
+sub classify_section {
+    my $self = shift (@_);
+    my ($section, $doc_obj, $sortmeta, $nosort) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    $nosort = 0 unless defined $nosort;
+    $sortmeta = "" unless defined $sortmeta;
+
+    my $metavalue;
+    my $metaname;
+    if (defined $self->{'meta_list'}) {
+	# find the first available metadata
+	foreach my $m (@{$self->{'meta_list'}}) {
+	    $metavalue = $doc_obj->get_metadata_element($section, $m);
+	    $metaname = $m;
+	    last if defined $metavalue;
+	} 
+	#if we haven't found a metavalue here, then the section shouldn't be included
+	return unless defined $metavalue;
+    }
+    
+    # we know the section should be included, add it now if we are not sorting
+    if ($nosort) {
+	push (@{$self->{'list'}}, "$doc_OID.$section");
+	return;
+    }
+    # check that it hasn't been added already
+    if (defined $self->{'list'}->{"$doc_OID.$section"}) {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "WARNING: SectionList::classify called multiple times for $doc_OID.$section\n";
+    } 
+
+    if (defined $self->{'sort'}) {
+	# sorting on alternative metadata
+	$self->{'list'}->{"$doc_OID.$section"} = $sortmeta;
+    } else {
+	# sorting on the classification metadata
+	# do the same formatting on the meta value as for sort meta
+	$metavalue = &sorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj) unless $self->{'no_metadata_formatting'};
+	$self->{'list'}->{"$doc_OID.$section"} = $metavalue;
+    }
+}
+1;
Index: /main/tags/3.01/gsdl/perllib/cnseg.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cnseg.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cnseg.pm	(revision 21112)
@@ -0,0 +1,74 @@
+###########################################################################
+#
+# cnseg.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# this package segments a chinese UTF-8 encoded Unicode
+# string into words.
+
+package cnseg;
+
+use unicode;
+
+
+# 'segment' takes a UTF-8 encoded Unicode Chinese-language
+# string and places U-200B between words -- the ZERO
+# WIDTH SPACE. Each line is treated as a separate 
+# paragraph, so lines in one paragraph should
+# be joined before using this method (normally a single
+# word might span more than one line within a paragraph).
+#
+# 'segment' is currently written in Perl, however, I (Rodger)
+# plan to use C++ (via pipes) once a more complex (and useful!)
+# algorithm is being used. Currently, each Chinese character
+# is treated as a seperate word.
+
+sub segment {
+    my ($in) = @_;
+    my ($c);
+
+    my $uniin = &unicode::utf82unicode($in);
+    my $out = [];
+
+    my $space = 1; # start doesn't need a space
+    foreach $c (@$uniin) {
+	if (($c >= 0x4e00 && $c <= 0x9fa5) ||
+	    ($c >= 0xf900 && $c <= 0xfa2d)) {
+	    # Chinese character
+	    push (@$out, 0x200b) unless $space;
+	    push (@$out, $c);
+	    push (@$out, 0x200b);
+	    $space = 1;
+
+	} else {
+	    # non-Chinese character
+	    push (@$out, $c);
+	    $space = 0;
+	}
+    }
+
+    return &unicode::unicode2utf8($out);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/colcfg.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/colcfg.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/colcfg.pm	(revision 21112)
@@ -0,0 +1,146 @@
+###########################################################################
+#
+# colcfg.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# reads in a collection configuration file
+
+package colcfg;
+
+use cfgread;
+use strict;
+
+# the collection configuration file data is stored in the form
+#
+# {'creator'}->string
+# {'public'}->string
+# {'defaultindex'}->string
+# {'importdir'}->string
+# {'archivedir'}->string
+# {'cachedir'}->string
+# {'builddir'}->string
+# {'removeold'}->string
+# {'textcompress'}->string
+# {'buildtype'}->string
+# {'maxnumeric'}->string
+# {'separate_cjk'}->string
+# {'sections_index_document_metadata'}->string (always, unless_section_metadata_exists)
+# {'maintainer'}->array of strings
+# {'languages'}->array of strings
+# {'indexsubcollections'}->array of strings
+# {'indexes'}->array of strings
+# {'dontbuild'}->array of strings
+# {'dontgdbm'}->array of strings
+# {'mirror'}->array of strings
+# {'phind'}->array of strings
+# {'levels'}->array of strings (for mgpp eg Section, Paragraph)
+# {'searchtype'}->array of strings (for mgpp, form or plain)
+
+# {'subcollection'}->hash of key-value pairs
+
+# {'acquire'}->array of arrays of strings
+# {'plugin'}->array of arrays of strings
+# {'classify'}->array of arrays of strings
+
+# {'collectionmeta'}->hash of key->hash of param-value -used 
+# for language specification
+#    for example, collectionmeta->collectionname->default->demo
+#                                               ->mi->maori demo
+
+sub read_collect_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+				    q/^(creator|public|defaultindex|importdir|/ .
+					q/archivedir|cachedir|builddir|removeold|/ .
+					q/textcompress|buildtype|no_text|keepold|gzip|/ .
+					q/verbosity|remove_empty_classifications|OIDtype|/ .
+					q/groupsize|maxdocs|debug|mode|saveas|/ .
+					q/sortmeta|removesuffix|removeprefix|create_images|/ .
+					q/maxnumeric|separate_cjk|language_metadata|/ .
+					q/no_strip_html|index|sections_index_document_metadata)$/,
+				    q/(maintainer|languages|indexsubcollections|/ .
+				       q/indexes|dontbuild|dontgdbm|mirror|levels|/ .
+				       q/searchtype|searchtypes)$/,
+				    q/^(subcollection|format)$/,
+				    q/^(acquire|plugin|classify)$/,
+				    q/^(collectionmeta)$/);
+}
+
+sub write_collect_cfg {
+    my ($filename, $data) = @_;
+    
+    &cfgread::write_cfg_file($filename, $data,
+			     q/^(creator|public|defaultindex|importdir|/ .
+				 q/archivedir|cachedir|builddir|removeold|/ .
+				 q/textcompress|buildtype|no_text|keepold|gzip|/ .
+				 q/verbosity|remove_empty_classifications|OIDtype|/.
+				 q/groupsize|maxdocs|debug|mode|saveas|/ .
+				 q/sortmeta|removesuffix|removeprefix|create_images|/ .
+				 q/maxnumeric|separate_cjk|language_metadata/ .
+				 q/no_strip_html|index|sections_index_document_metadata)$/,
+			     q/^(maintainer|languages|indexsubcollections|/ .
+				 q/indexes|dontbuild|dontgdbm|mirror|levels|/.
+				 q/searchtype|searchtypes)$/,
+			     q/^(subcollection|format)$/,
+			     q/^(acquire|plugin|classify)$/,
+			     q/^(collectionmeta)$/);
+}
+
+
+# the build configuration file data is stored in the form
+#
+# {'builddate'}->string
+# {'buildtype'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numsections'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+# {'maxnumeric'}->string
+# {'indexfields'}->array of strings
+# {'indexfieldmap'}->array of strings in the form "field->FI"
+# {'indexmap'} -> array of strings
+# {'indexlevels'} -> array of strings
+# {'textlevel'}->string
+# {'levelmap'} -> array of strings in the form "level->shortname"
+sub read_build_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+			    q/^(builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem)$/, 
+				    q/^(metadata|languages|indexfields|indexfieldmap|indexmap|indexlevels|levelmap)$/);
+}
+
+sub write_build_cfg {
+    my ($filename, $data) = @_;
+
+    &cfgread::write_cfg_file($filename, $data,
+	       q/^(builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem)$/, 
+	       q/^(metadata|languages|indexfieldsindexfieldmap|indexmap|indexlevels|levelmap)$/);
+}
+
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/Batch.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/Batch.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/Batch.pm	(revision 21112)
@@ -0,0 +1,155 @@
+package MARC::Batch;
+
+=head1 NAME
+
+MARC::Batch - Perl module for handling files of MARC::Record objects
+
+=cut
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+=head1 SYNOPSIS
+
+MARC::Batch hides all the file handling of files of C<MARC::Record>s.
+C<MARC::Record> still does the file I/O, but C<MARC::Batch> handles the
+multiple-file aspects.
+
+    use MARC::Batch;
+
+    my $batch = new MARC::Batch( 'USMARC', @files );
+    while ( my $marc = $batch->next ) {
+	print $marc->subfield(245,"a"), "\n";
+    }
+
+=head1 EXPORT
+
+None.  Everything is a class method.
+
+=head1 METHODS
+
+=head2 new( $type, [@files] )
+
+Create a C<MARC::Batch> object that will process C<@files>.
+
+C<$type> must be either "USMARC" or "MicroLIF".  If you want to specify 
+"MARC::File::USMARC" or "MARC::File::MicroLIF", that's OK, too.
+
+=cut
+
+sub new {
+    my $class = shift;
+    my $type = shift;
+
+    my $marcclass = ($type =~ /^MARC::File/) ? $type : "MARC::File::$type";
+
+    eval "require $marcclass";
+    die $@ if $@;
+
+    my @files = @_;
+
+    my $self = {
+	filelist =>	[@files],
+	filestack =>	[@files],
+	filename =>	undef,
+	marcclass =>	$marcclass,
+	file =>		undef,
+    };
+
+    bless $self, $class;
+
+    return $self;
+} # new()
+
+
+=head2 next()
+
+Read the next record from the files.  If the current file is at EOF, close
+it and open the next one.
+
+=cut
+
+sub next {
+    my $self = shift;
+
+    # If we have an open file, just use it and go.
+    if ( $self->{file} ) {
+	my $rec = $self->{file}->next();
+	return $rec if $rec;
+    }
+
+    $self->{file} = undef;
+
+    # Get the next file off the stack, if there is one
+    $self->{filename} = shift @{$self->{filestack}} or return undef;
+
+    # Instantiate a filename for it
+    my $marcclass = $self->{marcclass};
+    $self->{file} = $marcclass->in( $self->{filename} ) or return undef;
+
+    return $self->{file}->next();
+}
+
+=head2 filename()
+
+Returns the currently open filename
+
+=cut
+
+sub filename {
+    my $self = shift;
+
+    return $self->{filename};
+}
+
+=head2 warnings() 
+
+Returns any warnings that have accumulated while processing a particular 
+batch file. As a side effect the warning buffer will be cleared.
+
+=cut
+
+sub warnings {
+    my $self = shift;
+    my $file = $self->{file}; 
+    return(undef) if !$file;
+    return ($file->warnings());
+}
+
+
+1;
+
+__END__
+
+=head1 RELATED MODULES
+
+L<MARC::Record>, L<MARC::Lint>
+
+=head1 TODO
+
+None yet.  Send me your ideas and needs.
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/Field.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/Field.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/Field.pm	(revision 21112)
@@ -0,0 +1,487 @@
+package MARC::Field;
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+use constant SUBFIELD_INDICATOR => "\x1F";
+use constant END_OF_FIELD       => "\x1E";
+
+use vars qw( $ERROR );
+
+=head1 NAME
+
+MARC::Field - Perl extension for handling MARC fields
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+=head1 SYNOPSIS
+
+  use MARC::Field;
+
+  my $field = 
+  	MARC::Field->new( 
+		245, '1', '0',
+			'a' => 'Raccoons and ripe corn / ',
+			'c' => 'Jim Arnosky.'
+		);
+  $field->add_subfields( "a", "1st ed." );
+
+=head1 DESCRIPTION
+
+Defines MARC fields for use in the MARC::Record module.  I suppose
+you could use them on their own, but that wouldn't be very interesting.
+
+=head1 EXPORT
+
+None by default.  Any errors are stored in C<$MARC::Field::ERROR>, which
+C<$MARC::Record> usually bubbles up to C<$MARC::Record::ERROR>.
+
+=head1 METHODS
+
+=head2 C<new(tag,indicator1,indicator2,code,data[,code,data...])>
+
+  my $record = 
+  	MARC::Field->new( 
+		245, '1', '0',
+			'a' => 'Raccoons and ripe corn / ',
+			'c' => 'Jim Arnosky.'
+		);
+
+=cut
+
+sub new($) {
+	my $class = shift;
+	$class = ref($class) || $class;
+
+	my $tagno = shift;
+	($tagno =~ /^\d\d\d$/)
+		or return _gripe( "Tag \"$tagno\" is not a valid tag number." );
+
+	my $self = bless {
+		_tag => $tagno,
+		_warnings => [],
+		}, $class;
+	
+	if ( $tagno < 10 ) { 
+		$self->{_data} = shift;
+	} else {
+		for my $indcode ( qw( _ind1 _ind2 ) ) {
+			my $indicator = shift;
+			if ( $indicator !~ /^[0-9 ]$/ ) {
+				$self->_warn( "Invalid indicator \"$indicator\" forced to blank" ) unless ($indicator eq "");
+				$indicator = " ";
+			}
+			$self->{$indcode} = $indicator;
+		} # for
+		
+		(@_ >= 2)
+			or return _gripe( "Field $tagno must have at least one subfield" );
+
+		# Normally, we go thru add_subfields(), but internally we can cheat
+		$self->{_subfields} = [@_];
+	}
+
+	return $self;
+} # new()
+
+=head2 C<clone()>
+
+Makes a copy of the field.  Note that this is not just the same as saying
+
+    my $newfield = $field;
+
+since that just makes a copy of the reference.  To get a new object, you must
+
+    my $newfield = $field->clone;
+
+=cut
+
+sub clone {
+    my $self = shift;
+
+    my $tagno = $self->tag;
+
+    my $clone = 
+	bless {
+	    _tag => $tagno,
+	    _warnings => [],
+	}, ref($self);
+
+    if ( $tagno < 10 ) {
+	$clone->{_data} = $self->{_data};
+    } else {
+	$clone->{_ind1} = $self->{_ind1};
+	$clone->{_ind2} = $self->{_ind2};
+	$clone->{_subfields} = [@{$self->{_subfields}}]; 
+    }
+
+    return $clone;
+}
+
+=head2 C<update()>
+
+Allows you to change the values of the field. You can update indicators
+and subfields like this:
+
+  $field->update( ind2 => '4', a => 'The ballad of Abe Lincoln');
+
+The amount of items modified will be returned to you as a result of the
+method call.
+
+If you want to update a field that has no indicators or subfields (000-009)
+just call update() with one argument, the string that you would like to 
+set the field to. 
+
+  $field = $record->field( '003' );
+  $field->update('IMchF');
+
+Note: when doing subfield updates be aware that C<update()> will only 
+update the first occurrence. If you need to do anything more complicated
+you need to create a new field and use C<replace_with()>. 
+
+=cut
+
+sub update {
+
+  my $self = shift;
+
+  ## tags 000 - 009 don't have indicators or subfields
+  if ( $self->tag() < 10 ) {
+    $self->{_data} = shift;
+    return(1);
+  }
+  
+  ## otherwise we need to update subfields and indicators
+  my @data = @{$self->{_subfields}}; 
+  my $changes = 0;
+
+  while ( @_ ) {
+
+    my $arg = shift;
+    my $val = shift;
+
+    ## indicator update
+    if ($arg =~ /^ind[12]$/) {
+      $self->{"_$arg"} = $val;
+      $changes++;
+    }
+    ## subfield update
+    else {
+      for (my $i=0; $i<@data; $i=$i+2) {
+	if ($data[$i] eq $arg) {
+	  $data[$i+1] = $val;
+	  $changes++;
+	  last;
+	}
+      }
+    }
+
+  }
+
+  ## synchronize our subfields 
+  $self->{_subfields} = \@data;
+  return($changes);
+
+}
+
+=head2 C<replace_with()> 
+
+Allows you to replace an existing field with a new one. You need to pass 
+C<replace()> a MARC::Field object to replace the existing field with. For 
+example:
+
+  $field = $record->field('245');
+  my $new_field = new MARC::Field('245','0','4','The ballad of Abe Lincoln.');
+  $field->replace_with($new_field);
+
+=cut 
+
+sub replace_with {
+
+  my ($self,$new) = @_;
+  ref($new) =~ /^MARC::Field$/ 
+    or return _gripe("Must pass a MARC::Field object");
+
+  %$self = %$new;
+    
+}
+
+
+=head2 C<tag()>
+
+Returns the three digit tag for the field.
+
+=cut
+
+sub tag {
+	my $self = shift;
+	return $self->{_tag};
+}
+
+=head2 C<indicator(indno)>
+
+Returns the specified indicator.  Returns C<undef> and sets 
+C<$MARC::Field::ERROR> if the I<indno> is not 1 or 2, or if 
+the tag doesn't have indicators.
+
+=cut
+
+sub indicator($) {
+	my $self = shift;
+	my $indno = shift;
+
+	($self->tag >= 10)
+		or return _gripe( "Fields below 010 do not have indicators" );
+
+	if ( $indno == 1 ) {
+		return $self->{_ind1};
+	} elsif ( $indno == 2 ) {
+		return $self->{_ind2};
+	} else {
+		return _gripe( "Indicator number must be 1 or 2" );
+	}
+}
+
+
+
+=head2 C<subfield(code)>
+
+Returns the text from the first subfield matching the subfield code.
+If no matching subfields are found, C<undef> is returned.
+
+If the tag is less than an 010, C<undef> is returned and
+C<$MARC::Field::ERROR> is set.
+
+=cut
+
+sub subfield {
+	my $self = shift;
+	my $code_wanted = shift;
+
+	($self->tag >= 10)
+		or return _gripe( "Fields below 010 do not have subfields" );
+
+	my @data = @{$self->{_subfields}};
+	while ( defined( my $code = shift @data ) ) {
+		return shift @data if ( $code eq $code_wanted );
+		shift @data;
+	}
+
+	return undef;
+}
+
+=head2 C<subfields()>
+
+Returns all the subfields in the field.  What's returned is a list of 
+lists, where the inner list is a subfield code and the subfield data. 
+
+For example, this might be the subfields from a 245 field:
+
+	[
+	  [ 'a', 'Perl in a nutshell :' ],
+	  [ 'b', 'A desktop quick reference.' ],
+	]
+
+=cut
+
+sub subfields {
+	my $self = shift;
+
+	($self->tag >= 10)
+		or return _gripe( "Fields below 010 do not have subfields" );
+
+	my @list;
+	my @data = @{$self->{_subfields}};
+	while ( defined( my $code = shift @data ) ) {
+		push( @list, [$code, shift @data] );
+	}
+	return @list;
+}
+
+sub _gripe(@) {
+	$ERROR = join( "", @_ );
+
+	warn $ERROR;
+
+	return undef;
+}
+
+=head2 C<data()>
+
+Returns the data part of the field, if the tag number is less than 10.
+
+=cut
+
+sub data($) {
+	my $self = shift;
+
+	($self->{_tag} < 10)
+		or return _gripe( "data() is only for tags less than 10" );
+		
+	my $data = shift;
+	$self->{_data} = $data if defined( $data );
+
+	return $self->{_data};
+}
+
+=head2 C<add_subfields(code,text[,code,text ...])>
+
+Adds subfields to the end of the subfield list.
+
+Returns the number of subfields added, or C<undef> if there was an error.
+
+=cut
+
+sub add_subfields(@) {
+	my $self = shift;
+
+	($self->{_tag} >= 10)
+		or return _gripe( "Subfields are only for tags >= 10" );
+
+	push( @{$self->{_subfields}}, @_ );
+	return @_/2;
+}
+
+=head2 C<as_string()>
+
+Returns a string of all subfields run together, without the tag number.
+
+=cut
+
+sub as_string() {
+	my $self = shift;
+
+	return $self->{_data} if $self->tag < 10;
+
+	my @subs;
+
+	my @subdata = @{$self->{_subfields}};
+	while ( @subdata ) {
+		my $code = shift @subdata;
+		my $text = shift @subdata;
+		push( @subs, $text );
+	} # for
+
+	return join( " ", @subs );
+}
+
+
+=head2 C<as_formatted()>
+
+Returns a pretty string for printing in a MARC dump.
+
+=cut
+
+sub as_formatted() {
+	my $self = shift;
+
+	my @lines;
+
+	if ( $self->tag < 10 ) {
+		push( @lines, sprintf( "%03d     %s", $self->{_tag}, $self->{_data} ) );
+	} else {
+		my $hanger = sprintf( "%03d %1.1s%1.1s", $self->{_tag}, $self->{_ind1}, $self->{_ind2} );
+
+		my @subdata = @{$self->{_subfields}};
+		while ( @subdata ) {
+			my $code = shift @subdata;
+			my $text = shift @subdata;
+			push( @lines, sprintf( "%-6.6s _%1.1s%s", $hanger, $code, $text ) );
+			$hanger = "";
+		} # for
+	}
+
+
+
+	return join( "\n", @lines );
+}
+
+
+=head2 C<as_usmarc()>
+
+Returns a string for putting into a USMARC file.  It's really only
+useful by C<MARC::Record::as_usmarc()>.
+
+=cut
+
+sub as_usmarc() {
+	my $self = shift;
+
+	# Tags < 010 are pretty easy
+	if ( $self->tag < 10 ) {
+		return $self->data . END_OF_FIELD;
+	} else {
+		my @subs;
+		my @subdata = @{$self->{_subfields}};
+		while ( @subdata ) {
+			push( @subs, join( "", SUBFIELD_INDICATOR, shift @subdata, shift @subdata ) );
+		} # while
+
+		return join( "", 
+			$self->indicator(1),
+			$self->indicator(2),
+			@subs,
+			END_OF_FIELD,
+			);
+	}
+}
+
+=head2 C<warnings()>
+
+Returns the warnings that were created when the record was read.
+These are things like "Invalid indicators converted to blanks".
+
+The warnings are items that you might be interested in, or might
+not.  It depends on how stringently you're checking data.  If
+you're doing some grunt data analysis, you probably don't care.
+
+=cut
+
+sub warnings() {
+	my $self = shift;
+
+	return @{$self->{_warnings}};
+}
+
+# NOTE: _warn is an object method
+sub _warn($) {
+	my $self = shift;
+
+	push( @{$self->{_warnings}}, join( "", @_ ) );
+}
+
+1;
+
+__END__
+
+=head1 SEE ALSO
+
+See the "SEE ALSO" section for L<MARC::Record>.
+
+=head1 TODO
+
+See the "TODO" section for L<MARC::Record>.
+
+=cut
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/File.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/File.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/File.pm	(revision 21112)
@@ -0,0 +1,202 @@
+package MARC::File;
+
+=head1 NAME
+
+MARC::File - Base class for files of MARC records
+
+=cut
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+use vars qw( $ERROR );
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+=head1 SYNOPSIS
+
+    use MARC::File::USMARC;
+
+    my $file = MARC::File::USMARC->in( $filename );
+    
+    while ( my $marc = $file->next() ) {
+	# Do something
+    }
+    $file->close();
+    undef $file;
+
+=head1 EXPORT
+
+None.  
+
+=head1 METHODS
+
+=head2 in()
+
+Opens a file for input.
+
+=cut
+
+sub in {
+    my $class = shift;
+    my $filename = shift;
+
+    my $self = {
+	filename => $filename,
+	_warnings => [],
+    };
+
+    bless $self, $class;
+
+    my $fh = eval { local *FH; open( FH, $filename ) or die; *FH{IO}; };
+
+    if ( $@ ) {
+	undef $self;
+	$MARC::File::ERROR = "Couldn't open $filename: $@";
+    } else {
+	$self->{fh} = $fh;
+    }
+	
+
+    return $self;
+} # new()
+
+sub out {
+    die "Not yet written";
+}
+
+=head2 next()
+
+Reads the next record from the file handle passed in. 
+
+=cut
+
+sub next {
+    my $self = shift;
+    my $rec = $self->_next();
+    return $rec ? $self->decode($rec) : undef;
+}
+
+=head2 skip()
+
+Skips over the next record in the file.  Same as C<next()>,
+without the overhead of parsing a record you're going to throw away
+anyway.
+
+Returns 1 or undef.
+
+=cut
+
+sub skip {
+    my $self = shift;
+
+    my $rec = $self->_next();
+
+    return $rec ? 1 : undef;
+}
+
+=head2 warnings()
+
+Simlilar to MARC::Record and MARC::Batch, warnings() will return any 
+warnings that have accumulated while processing this file; and as a 
+side-effect will clear the warnings buffer.
+
+=cut 
+
+sub warnings {
+    my $self = shift;
+    my @warnings = @{ $self->{_warnings} };
+    $self->{_warnings} = [];
+    return(@warnings);
+}
+
+sub close {
+    my $self = shift;
+
+    close( $self->{fh} );
+    delete $self->{fh};
+    delete $self->{filename};
+
+    return;
+}
+
+sub _unimplemented() {
+    my $self = shift;
+    my $method = shift;
+
+    warn "Method $method must be overridden";
+}
+
+sub write   { $_[0]->_unimplemented("write"); }
+sub decode  { $_[0]->_unimplemented("decode"); }
+
+# NOTE: _warn must be called as an object method
+
+sub _warn {
+    my ($self,$warning) = @_;
+    push( @{ $self->{_warnings} }, $warning );
+    return(undef);
+}
+
+# NOTE: _gripe can be called as an object method, or not.  Your choice.
+sub _gripe(@) {
+    my @parms = @_;
+    if ( @parms ) {
+	my $self = shift @parms;
+
+	if ( ref($self) =~ /^MARC::File/ ) {
+	    push( @parms, " at byte ", tell($self->{fh}) ) if $self->{fh};
+	    push( @parms, " in file ", $self->{filename} ) if $self->{filename};
+	} else {
+	    unshift( @parms, $self );
+	}
+
+	$ERROR = join( "", @parms );
+	warn $ERROR;
+    }
+
+    return(undef);
+}
+
+1;
+
+__END__
+
+=head1 RELATED MODULES
+
+L<MARC::Record>
+
+=head1 TODO
+
+=over 4
+
+=item * C<out()> method
+
+We only handle files for input right now.
+
+=back
+
+=cut
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/File/MicroLIF.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/File/MicroLIF.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/File/MicroLIF.pm	(revision 21112)
@@ -0,0 +1,128 @@
+package MARC::File::MicroLIF;
+
+=head1 NAME
+
+MARC::File::MicroLIF - MicroLIF-specific file handling
+
+=cut
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+use vars qw( $ERROR );
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+use MARC::File;
+use vars qw( @ISA ); @ISA = qw( MARC::File );
+
+use MARC::Record qw( LEADER_LEN );
+
+=head1 SYNOPSIS
+
+    use MARC::File::MicroLIF;
+
+    my $file = MARC::File::MicroLIF::in( $filename );
+    
+    while ( my $marc = $file->next() ) {
+	# Do something
+    }
+    $file->close();
+    undef $file;
+
+=head1 EXPORT
+
+None.  
+
+=head1 METHODS
+
+=cut
+
+sub _next {
+    my $self = shift;
+
+    my $fh = $self->{fh};
+
+    local $/ = "`\n";
+    
+    my $lifrec = <$fh>;
+
+    return $lifrec;
+}
+
+sub decode {
+    my $text = shift;
+    $text = shift if (ref($text)||$text) =~ /^MARC::File/; # Handle being called as a method
+
+    my $marc = MARC::Record->new();
+
+    my @lines = split( /\n/, $text );
+    for my $line ( @lines ) {
+	# Ignore the file header if the calling program hasn't already dealt with it
+	next if $line =~ /^HDR/;
+
+	($line =~ s/^(\d\d\d|LDR)//) or
+	return $marc->_gripe( "Invalid tag number: ", substr( $line, 0, 3 ) );
+	my $tagno = $1;
+
+	($line =~ s/\^$//) or $marc->_warn( "Tag $tagno is missing a trailing caret." );
+
+	if ( $tagno eq "LDR" ) {
+	    $marc->leader( substr( $line, 0, LEADER_LEN ) );
+	} elsif ( $tagno < 10 ) {
+	    $marc->add_fields( $tagno, $line );
+	} else {
+	    $line =~ s/^(.)(.)//;
+	    my ($ind1,$ind2) = ($1,$2);
+	    my @subfields;
+	    my @subfield_data_pairs = split( /_(?=[a-z0-9])/, $line );
+	    shift @subfield_data_pairs; # Leading _ makes an empty pair
+	    for my $pair ( @subfield_data_pairs ) {
+		my ($subfield,$data) = (substr( $pair, 0, 1 ), substr( $pair, 1 ));
+		push( @subfields, $subfield, $data );
+	    }
+	    $marc->add_fields( $tagno, $ind1, $ind2, @subfields );
+	}
+    } # for
+
+    return $marc;
+}
+
+1;
+
+__END__
+
+=head1 TODO
+
+=over 4
+
+=item * Squawks about the final field missing a caret
+
+=back
+
+=head1 RELATED MODULES
+
+L<MARC::File>
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/File/USMARC.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/File/USMARC.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/File/USMARC.pm	(revision 21112)
@@ -0,0 +1,287 @@
+package MARC::File::USMARC;
+
+=head1 NAME
+
+MARC::File::USMARC - USMARC-specific file handling
+
+=cut
+
+use strict;
+use integer;
+eval 'use bytes'    if $] >= 5.006;
+eval 'use warnings' if $] >= 5.006;
+
+use vars qw( $ERROR );
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+use MARC::File;
+use vars qw( @ISA ); @ISA = qw( MARC::File );
+
+use MARC::Record qw( LEADER_LEN );
+use constant SUBFIELD_INDICATOR	    => "\x1F";
+use constant END_OF_FIELD	    => "\x1E";
+use constant END_OF_RECORD	    => "\x1D";
+use constant DIRECTORY_ENTRY_LEN    => 12;
+
+=head1 SYNOPSIS
+
+    use MARC::File::USMARC;
+
+    my $file = MARC::File::USMARC::in( $filename );
+    
+    while ( my $marc = $file->next() ) {
+	# Do something
+    }
+    $file->close();
+    undef $file;
+
+=head1 EXPORT
+
+None.  
+
+=head1 METHODS
+
+=for internal
+
+Internal function to get the next raw record out of a file.
+
+=cut
+
+sub _next {
+    my $self = shift;
+
+    my $fh = $self->{fh};
+
+    my $reclen;
+    return undef if eof($fh);
+
+    local $/ = END_OF_RECORD;
+    my $usmarc = <$fh>;
+
+    if ( length($usmarc) < 5 ) {
+	$self->_warn( "Couldn't find record length" );
+	return $self->_next();
+    }
+
+    $reclen = substr($usmarc,0,5);
+
+    if ( $reclen !~ /^\d{5}$/ or $reclen != length($usmarc) ) {
+	$self->_warn( "Invalid record length \"$reclen\"" );
+	return $self->_next();
+    }
+
+    return $usmarc;
+}
+
+=head2 decode()
+
+Constructor for handling data from a USMARC file.  This function takes care of all
+the tag directory parsing & mangling.
+
+Any warnings or coercions can be checked in the C<warnings()> function.
+
+=cut
+
+sub decode {
+    my $text = shift;
+    $text = shift if (ref($text)||$text) =~ /^MARC::File/;
+
+    my $marc = MARC::Record->new();
+
+    # Check for an all-numeric record length
+    ($text =~ /^(\d{5})/)
+	or return $marc->_warn( "Record length \"", substr( $text, 0, 5 ), "\" is not numeric" );
+
+    my $reclen = $1;
+    ($reclen == length($text))
+	or return $marc->_gripe( "Invalid record length: Leader says $reclen bytes, but it's actually ", length( $text ) );
+
+    $marc->leader( substr( $text, 0, LEADER_LEN ) );
+    my @fields = split( END_OF_FIELD, substr( $text, LEADER_LEN ) );
+    my $dir = shift @fields or return $marc->_warn( "No directory found" );
+
+    (length($dir) % 12 == 0)
+	or return $marc->_gripe( "Invalid directory length" );
+    my $nfields = length($dir)/12;
+
+    my $finalfield = pop @fields;
+    # Check for the record terminator, and ignore it
+    ($finalfield eq END_OF_RECORD)
+    	or $marc->_gripe( "Invalid record terminator: \"$finalfield\"" );
+
+    # Walk thru the directories, and shift off the fields while we're at it
+    # Shouldn't be any non-digits anywhere in any directory entry
+    my @directory = unpack( "A3 A4 A5" x $nfields, $dir );
+    my @bad = grep /\D/, @directory;
+    if ( @bad ) { 
+	return $marc->_gripe( "Non-numeric entries in the tag directory: ", join( ", ", map { "\"$_\"" } @bad ) );
+    }
+
+    my $databytesused = 0;
+    while ( @directory ) {
+	my $tagno = shift @directory;
+	my $len = shift @directory;
+	my $offset = shift @directory;
+	my $tagdata = shift @fields;
+	warn "Specs: ", join( "|", $tagno, $len, $offset, $tagdata ), "\n" if $MARC::Record::DEBUG;
+
+	# Check directory validity
+	($tagno =~ /^\d\d\d$/)
+	    or return $marc->_gripe( "Invalid field number in directory: \"$tagno\"" );
+
+	($len == length($tagdata) + 1)
+	    or $marc->_warn( "Invalid length in the directory for tag $tagno" );
+
+	($offset == $databytesused)
+	    or $marc->_warn( "Directory offsets are out of whack" );
+	$databytesused += $len;
+
+	if ( $tagno < 10 ) {
+	    $marc->add_fields( $tagno, $tagdata )
+		or return undef; # We're relying on add_fields() having set $MARC::Record::ERROR
+	} else {
+	    my @subfields = split( SUBFIELD_INDICATOR, $tagdata );
+	    my $indicators = shift @subfields
+		or return $marc->_gripe( "No subfields found." );
+	    my ($ind1,$ind2);
+	    if ( $indicators =~ /^([0-9 ])([0-9 ])$/ ) {
+		($ind1,$ind2) = ($1,$2);
+	    } else {
+		$marc->_warn( "Invalid indicators \"$indicators\" forced to blanks\n" );
+		($ind1,$ind2) = (" "," ");
+	    }
+
+	    # Split the subfield data into subfield name and data pairs
+	    my @subfield_data = map { (substr($_,0,1),substr($_,1)) } @subfields;
+	    $marc->add_fields( $tagno, $ind1, $ind2, @subfield_data )
+		or return undef;
+	}
+    } # while
+
+    # Once we're done, there shouldn't be any fields left over: They should all have shifted off.
+    (@fields == 0)
+    	or return $marc->_gripe( "I've got leftover fields that weren't in the directory" );
+
+    return $marc;
+}
+
+=head2 update_leader()
+
+If any changes get made to the MARC record, the first 5 bytes of the
+leader (the length) will be invalid.  This function updates the 
+leader with the correct length of the record as it would be if
+written out to a file.
+
+=cut
+
+sub update_leader() {
+	my $self = shift;
+
+	my (undef,undef,$reclen,$baseaddress) = $self->_build_tag_directory();
+
+	$self->_set_leader_lengths( $reclen, $baseaddress );
+}
+
+=head2 _build_tag_directory()
+
+Function for internal use only: Builds the tag directory that gets
+put in front of the data in a MARC record.
+
+Returns two array references, and two lengths: The tag directory, and the data fields themselves,
+the length of all data (including the Leader that we expect will be added),
+and the size of the Leader and tag directory.
+
+=cut
+
+sub _build_tag_directory {
+	my $marc = shift;
+	$marc = shift if (ref($marc)||$marc) =~ /^MARC::File/;
+	die "Wanted a MARC::Record but got a ", ref($marc) unless ref($marc) eq "MARC::Record";
+
+	my @fields;
+	my @directory;
+
+	my $dataend = 0;
+	for my $field ( $marc->fields() ) {
+		# Dump data into proper format
+		my $str = $field->as_usmarc;
+		push( @fields, $str );
+
+		# Create directory entry
+		my $len = length $str;
+		my $direntry = sprintf( "%03d%04d%05d", $field->tag, $len, $dataend );
+		push( @directory, $direntry );
+		$dataend += $len;
+	}
+
+	my $baseaddress = 
+		LEADER_LEN +    # better be 24
+		( @directory * DIRECTORY_ENTRY_LEN ) +
+				# all the directory entries
+		1;           	# end-of-field marker
+
+
+	my $total = 
+		$baseaddress +	# stuff before first field
+		$dataend + 	# Length of the fields
+		1;		# End-of-record marker
+
+
+
+	return (\@fields, \@directory, $total, $baseaddress);
+}
+
+=head2 encode()
+
+Returns a string of characters suitable for writing out to a USMARC file,
+including the leader, directory and all the fields.
+
+=cut
+
+sub encode() {
+    my $marc = shift;
+    $marc = shift if (ref($marc)||$marc) =~ /^MARC::File/;
+
+    my ($fields,$directory,$reclen,$baseaddress) = _build_tag_directory($marc);
+    $marc->set_leader_lengths( $reclen, $baseaddress );
+
+    # Glomp it all together
+    return join("",$marc->leader, @$directory, END_OF_FIELD, @$fields, END_OF_RECORD);
+}
+
+1;
+
+__END__
+
+=head1 RELATED MODULES
+
+L<MARC::Record>
+
+=head1 TODO
+
+Make some sort of autodispatch so that you don't have to explicitly
+specify the MARC::File::X subclass, sort of like how DBI knows to
+use DBD::Oracle or DBD::Mysql.
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/Lint.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/Lint.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/Lint.pm	(revision 21112)
@@ -0,0 +1,764 @@
+package MARC::Lint;
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+=head1 NAME
+
+MARC::Lint - Perl extension for checking validity of MARC records
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+use MARC::Record;
+use MARC::Field;
+
+=head1 SYNOPSIS
+
+  use MARC::Record;
+  use MARC::Lint;
+
+  my $linter = new MARC::Lint;
+  my $filename = shift;
+
+  open( IN, "<", $filename ) or die "Couldn't open $filename: $!\n";
+  binmode( IN ); # for the Windows folks
+  while ( !eof(IN) ) {
+  	my $marc = MARC::Record::next_from_file( *IN );
+	die $MARC::Record::ERROR unless $marc;
+
+	$linter->check_record( $marc );
+
+
+	# Print the title tag
+	print $marc->subfield(245,"a"), "\n";
+
+	# Print the errors that were found
+	print join( "\n", $linter->warnings ), "\n";
+  } # while
+
+  close IN or die "Error closing $filename: $!\n";
+
+Given the following MARC record:
+
+	LDR 00000nam  22002538a 4500
+	100 14 _aWall, Larry.
+	110 1  _aO'Reilly & Associates.
+	245 90 _aProgramming Perl /
+               _aBig Book of Perl /
+               _cLarry Wall, Tom Christiansen & Jon Orwant.
+	250    _a3rd ed.
+	250    _a3rd ed.
+	260    _aCambridge, Mass. :
+	       _bO'Reilly,
+	       _r2000.
+	590 4  _aPersonally signed by Larry.
+	856 43 _uhttp://www.perl.com/
+
+the following errors are generated:
+
+	1XX: Only one 1XX tag is allowed, but I found 2 of them.
+	100: Indicator 2 must be blank
+	245: Indicator 1 must be 0 or 1
+	245: Subfield _a is not repeatable.
+	250: Field is not repeatable.
+	260: Subfield _r is not valid.
+	260: Must have a subfield _c.
+	590: Indicator 1 must be blank
+	856: Indicator 2 must be blank, 0, 1, 2 or 8
+
+=head1 DESCRIPTION
+
+Module for checking validity of MARC records.  99% of the users will want to do 
+something like is shown in the synopsis.  The other intrepid 1% will overload the
+C<MARC::Lint> module's methods and provide their own special field-level checking.
+
+What this means is that if you have certain requirements, such as making sure that
+all 952 tags have a certain call number in them, you can write a function that 
+checks for that, and still get all the benefits of the MARC::Lint framework.
+
+=head1 EXPORT
+
+None.  Everything is done through objects.
+
+=head1 METHODS
+
+=head2 C<new()>
+
+No parms needed.  The C<MARC::Lint> object is little more than a list of warnings 
+and a bunch of rules.
+
+=cut
+
+sub new() {
+	my $class = shift;
+	$class = ref($class) || $class;
+
+	my $self = {
+		_warnings => [],
+	};
+	bless $self, $class;
+
+	$self->_read_rules();
+
+	return $self;
+}
+
+=head2 C<warnings()>
+
+Returns a list of warnings found by C<check_record()> and its brethren.
+
+=cut
+
+sub warnings {
+	my $self = shift;
+
+	return wantarray ? @{$self->{_warnings}} : scalar @{$self->{_warnings}};
+}
+
+=head2 C<clear_warnings()>
+
+Clear the list of warnings for this linter object.  It's automatically called
+when you call C<check_record()>.
+
+=cut
+
+sub clear_warnings {
+	my $self = shift;
+
+	$self->{_warnings} = [];
+}
+
+=head2 C<warn(str[,str...])>
+
+Create a warning message, built from strings passed, like a C<print> statement.
+
+Typically, you'll leave this to C<check_record()>, but industrious programmers
+may want to do their own checking as well.
+
+=cut
+
+sub warn {
+	my $self = shift;
+
+	push( @{$self->{_warnings}}, join( "", @_ ) );
+
+	return undef;
+}
+
+=head2 C<check_record(marc)>
+
+Does all sorts of lint-like checks on the MARC record I<marc>, both on the record as a whole,
+and on the individual fields & subfields.
+
+=cut
+
+sub check_record {
+	my $self = shift;
+	my $marc = shift;
+
+	$self->clear_warnings();
+
+	(ref($marc) eq "MARC::Record")
+		or return $self->warn( "Must pass a MARC::Record object to check_record" );
+
+	if ( (my @_1xx = $marc->field( "1XX" )) > 1 ) {
+		$self->warn( "1XX: Only one 1XX tag is allowed, but I found ", scalar @_1xx, " of them." );
+	}
+
+	if ( not $marc->field( 245 ) ) {
+		$self->warn( "245: No 245 tag." );
+	}
+
+
+	my %field_seen;
+	my $rules = $self->{_rules};
+	for my $field ( $marc->fields ) {
+		my $tagno = $field->tag;
+		my $tagrules = $rules->{$tagno} or next;
+
+		if ( $tagrules->{NR} && $field_seen{$tagno} ) { 
+			$self->warn( "$tagno: Field is not repeatable." );
+		}
+
+		if ( $tagno >= 10 ) {
+			for my $ind ( 1..2 ) {
+				my $indvalue = $field->indicator($ind);
+				if ( not ($indvalue =~ $tagrules->{"ind$ind" . "_regex"}) ) {
+					$self->warn( 
+						"$tagno: Indicator $ind must be ", 
+						$tagrules->{"ind$ind" . "_desc"}, 
+						" but it's \"$indvalue\"" 
+					);
+				}
+			}
+			
+			my %sub_seen;
+			for my $subfield ( $field->subfields ) {
+				my ($code,$data) = @$subfield;
+
+				my $rule = $tagrules->{$code};
+				if ( not defined $rule ) {
+					$self->warn( "$tagno: Subfield _$code is not allowed." );
+				} elsif ( ($rule eq "NR") && $sub_seen{$code} ) {
+					$self->warn( "$tagno: Subfield _$code is not repeatable." );
+				}
+
+				if ( $data =~ /[\t\r\n]/ ) {
+					$self->warn( "$tagno: Subfield _$code has an invalid control character" );
+				}
+
+				++$sub_seen{$code};
+			}
+		}
+
+		# Check to see if a check_xxx() function exists, and call it on the field if it does
+		my $checker = "check_$tagno";
+		if ( $self->can( $checker ) ) {
+			$self->$checker( $field );
+		}
+
+		++$field_seen{$tagno};
+	} # for
+
+	return;
+}
+
+=head2 C<check_I<xxx>(field)>
+
+Various functions to check the different fields.  If the function doesn't exist, 
+then it doesn't get checked.
+
+=cut
+
+sub check_245 {
+	my $self = shift;
+	my $field = shift;
+
+	if ( not $field->subfield( "a" ) ) {
+		$self->warn( "245: Must have a subfield _a." );
+	}
+}
+
+sub check_260 {
+	my $self = shift;
+	my $field = shift;
+
+	if ( not $field->subfield( "c" ) ) {
+		$self->warn( "260: Must have a subfield _c." );
+	}
+}
+
+
+=head1 SEE ALSO
+
+Check the docs for L<MARC::Record>.  All software links are there.
+
+=head1 TODO
+
+=over 4
+
+=item * ISBN and ISSN checking
+
+We can check the 020 and 022 fields with the C<Business::ISBN> and 
+C<Business::ISSN> modules, respectively.
+
+=back
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
+# Used only to read the stuff from __DATA__
+sub _read_rules() {
+	my $self = shift;
+	
+	my $tell = tell(DATA);  # Stash the position so we can reset it for next time
+
+	local $/ = "";
+	while ( my $lines = <DATA> ) {
+		$lines =~ s/\s+$//;
+		my @keyvals = split( /\s+/, $lines );
+
+		my $tagno = shift @keyvals;
+		my $repeatable = shift @keyvals;
+		
+		my @tag_range = ($tagno);
+		if ( $tagno =~ /^(\d\d)X/ ) {
+			my $base = $1;
+			@tag_range = ( "${base}0" .. "${base}9" );
+		}
+
+		# Handle the ranges of tags.
+		for my $currtag ( @tag_range ) {
+			$self->_parse_tag_rules( $currtag, $repeatable, @keyvals );
+		} # for
+		# I guess I could just have multiple references to the same tag, but I'm not that worried about memory
+	} # while
+
+	seek( DATA, $tell, 0 );
+}
+
+sub _parse_tag_rules {
+	my $self = shift;
+	my $tagno = shift;
+	my $repeatable = shift;
+	my @keyvals = @_;
+
+	my $rules = ($self->{_rules}->{$tagno} ||= {});
+	$rules->{$repeatable} = $repeatable;
+
+	while ( @keyvals ) {
+		my $key = shift @keyvals;
+		my $val = shift @keyvals;
+		
+		$rules->{$key} = $val;
+
+		# Do magic for indicators
+		if ( $key =~ /^ind/ ) {
+			my $desc;
+			my $regex;
+			
+			if ( $val eq "blank" ) {
+				$desc = "blank";
+				$regex = qr/^ $/;
+			} else {
+				$desc = _nice_list($val);
+				$val =~ s/^b/ /;
+				$regex = qr/^[$val]$/;
+			}
+
+		$rules->{$key."_desc"} = $desc;
+		$rules->{$key."_regex"} = $regex;
+		} # if indicator
+	} # while
+}
+
+
+sub _nice_list($) {
+	my $str = shift;
+
+	if ( $str =~ s/(\d)-(\d)/$1 thru $2/ ) {
+		return $str;
+	}
+
+	my @digits = split( //, $str );
+	$digits[0] = "blank" if $digits[0] eq "b";
+	my $last = pop @digits;
+	return join( ", ", @digits ) . " or $last";
+}
+
+sub _ind_regex($) {
+	my $str = shift;
+
+	return qr/^ $/ if $str eq "blank";
+
+	return qr/^[$str]$/;
+}
+
+
+1;
+
+__DATA__
+010	NR
+ind1	blank
+ind2	blank
+a	NR
+z	NR
+
+016	R
+ind1	b7
+ind2	blank
+a	NR
+z	R
+2	NR
+
+020	R
+ind1	blank
+ind2	blank
+a	R
+c	R
+
+022	R
+ind1	blank
+ind2	blank
+a	NR
+
+040	NR
+ind1	blank
+ind2	blank
+a	NR
+c	NR
+d	R
+
+100	NR
+ind1	013
+ind2	blank
+a	NR
+q	NR
+b	R
+c	R
+d	NR
+e	R
+
+110	NR
+ind1	012
+ind2	blank
+a	NR
+b	R
+
+111	NR
+ind1	012
+ind2	blank
+a	NR
+n	R	
+d	NR
+c	NR
+e	R
+
+130	NR
+ind1	0-9
+ind2	blank
+a	NR
+n	R
+p	R
+h	NR
+l	NR
+k	R
+s	NR
+f	NR
+
+240	NR
+ind1	01
+ind2	0-9
+
+245	NR
+ind1	01
+ind2	0-9
+a	NR
+n	R
+p	R
+h	NR
+b	NR
+s	NR
+c	NR
+
+246	NR
+ind1	0123
+ind2	012345678
+a	NR
+h	NR
+b	NR
+n	R
+p	R
+i	NR
+f	NR
+
+250	NR
+ind1	blank
+ind2	blank
+a	NR
+b	NR
+
+260	NR
+ind1	blank
+ind2	blank
+a	R
+b	R
+c	R
+
+300	NR
+ind1	blank
+ind2	blank
+a	R
+b	NR
+c	R
+e	NR
+
+440	R
+ind1	blank
+ind2	0-9
+a	NR
+n	R
+p	R
+v	NR
+
+490	R
+ind1	01
+ind2	blank
+a	R
+v	R
+
+500	R
+ind1	blank
+ind2	blank
+a	NR
+
+504	R
+ind1	blank
+ind2	blank
+a	NR
+
+505	R
+ind1	0128
+ind2	b0
+a	NR
+g	R
+r	R
+t	R
+
+520	R
+ind1	b018
+ind2	blank
+a	R
+b	R
+
+521	R
+ind1	b012348
+ind2	blank
+a	R
+b	NR
+
+526	R
+ind1	08
+ind2	blank
+a	NR
+b	NR
+c	NR
+d	NR
+i	NR
+x	R
+z	R
+
+538	R
+ind1	blank
+ind2	blank
+a	NR
+
+546	R
+ind1	blank
+ind2	blank
+a	NR
+
+586	R
+ind1	b8
+ind2	blank
+a	NR
+
+59X	R
+ind1	blank
+ind2	blank
+a	NR
+
+600	R
+ind1	013
+ind2	012567
+a	NR
+q	NR
+b	R
+c	R
+d	NR
+t	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+610	R
+ind1	012
+ind2	012567
+a	NR
+b	R
+t	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+611	R
+ind1	012
+ind2	012567
+a	NR
+n	R
+d	NR
+c	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+630	R
+ind1	0-9
+ind2	012567
+a	NR
+n	R
+p	R
+l	NR
+k	R
+s	NR
+f	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+650	R
+ind1	blank
+ind2	012567
+a	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+651	R
+ind1	blank
+ind2	012567
+a	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+655	R
+ind1	blank
+ind2	7
+a	NR
+v	R
+x	R
+y	R
+z	R
+2	NR
+
+658	R
+ind1	blank
+ind2	blank
+a	NR
+b	R
+c	NR
+d	NR
+2	NR
+
+69X	R
+ind1	blank
+ind2	blank
+a	NR
+v	R
+x	R
+y	R
+z	R
+
+700	R
+ind1	013
+ind2	b2
+a	NR
+q	NR
+b	R
+c	R
+d	NR
+k	R
+t	NR
+e	R
+f	NR
+
+710	R
+ind1	012
+ind2	b2
+a	NR
+b	R
+e	R
+t	NR
+
+711	R
+ind1	012
+ind2	b2
+a	NR
+n	R
+d	NR
+c	NR
+t	NR
+e	R
+
+730	R
+ind1	0-9
+ind2	b2
+a	NR
+n	R
+p	R
+h	NR
+l	NR
+k	R
+s	NR
+f	NR
+
+740	R
+ind1	0-9
+ind2	b2
+a	NR
+h	NR
+n	R
+p	R
+
+
+800	R
+ind1	013
+ind2	blank
+a	NR
+q	NR
+b	NR
+c	R
+d	NR
+t	NR
+e	R
+v	NR
+
+852	R
+ind1	b01234568
+ind2	b012
+a	NR
+b	R
+h	NR
+i	R
+k	NR
+m	NR
+t	NR
+p	NR
+9	NR
+
+856	R
+ind1	b012347
+ind2	b0128
+a	R
+b	NR
+d	R
+f	R
+h	NR
+i	R
+u	R
+x	R
+z	R
+2	NR
+3	NR
Index: /main/tags/3.01/gsdl/perllib/cpan/MARC/Record.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MARC/Record.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MARC/Record.pm	(revision 21112)
@@ -0,0 +1,705 @@
+package MARC::Record;
+
+=head1 NAME
+
+MARC::Record - Perl extension for handling MARC records
+
+=cut
+
+use strict;
+use integer;
+eval 'use warnings' if $] >= 5.006;
+
+use vars qw( $ERROR );
+
+use MARC::Field;
+
+=head1 VERSION
+
+Version 1.10
+
+    $Id$
+
+=cut
+
+use vars '$VERSION'; $VERSION = '1.10';
+
+use Exporter;
+use vars qw( @ISA @EXPORTS @EXPORT_OK );
+@ISA = qw( Exporter );
+@EXPORTS = qw();
+@EXPORT_OK = qw( LEADER_LEN );
+
+use vars qw( $DEBUG ); $DEBUG = 0;
+
+use constant LEADER_LEN	=> 24;
+
+=head1 DESCRIPTION
+
+Module for handling MARC records as objects.  The file-handling stuff is
+in MARC::File::*.
+
+=head1 EXPORT
+
+None.  
+
+=head1 ERROR HANDLING
+
+Any errors generated are stored in C<$MARC::Record::ERROR>. 
+Warnings are kept with the record and accessible in the C<warnings()> method. 
+
+=head1 METHODS
+
+=head2 new()
+
+Base constructor for the class.  It just returns a completely empty record.
+To get real data, you'll need to populate it with fields, or use one of
+the MARC::File::* modules to read from a file.
+
+=cut
+
+sub new {
+    my $class = shift;
+    $class = ref($class) || $class; # Handle cloning
+    my $self = {
+	_leader => ' ' x 24,
+	_fields => [],
+	_warnings => [],
+    };
+    return bless $self, $class;
+} # new()
+
+
+
+
+=head2 fields()
+
+Returns a list of all the fields in the record. The list contains 
+a MARC::Field object for each field in the record.
+
+=cut
+
+sub fields() {
+    my $self = shift;
+    return @{$self->{_fields}};
+}
+
+=head2 field(tagspec(s))
+
+Returns a list of tags that match the field specifier, or in scalar
+context, just the first matching tag.
+
+The field specifier can be a simple number (i.e. "245"), or use the "X" 
+notation of wildcarding (i.e. subject tags are "6XX").
+
+=cut
+
+my %field_regex;
+
+sub field {
+    my $self = shift;
+    my @specs = @_;
+
+    my @list = ();
+    for my $tag ( @specs ) {
+	my $regex = $field_regex{ $tag };
+
+	# Compile & stash it if necessary
+	if ( not defined $regex ) {
+	    my $pattern = $tag;
+	    $pattern =~ s/X/\\d/g;
+	    $regex = qr/^$pattern$/;
+	    $field_regex{ $tag } = $regex;
+	} # not defined
+
+	for my $maybe ( $self->fields ) {
+	    if ( $maybe->tag =~ $regex ) {
+		return $maybe unless wantarray;
+
+		push( @list, $maybe );
+	    } # if
+	} # for $maybe
+    } # for $tag
+
+    return @list;
+}
+
+=head2 subfield(tag,subfield)
+
+Shortcut method for getting just a subfield for a tag.  These are equivalent:
+
+  my $title = $marc->field(245)->subfield("a");
+  my $title = $marc->subfield(245,"a");
+
+If either the field or subfield can't be found, C<undef> is returned.
+
+=cut
+
+sub subfield($$) {
+    my $self = shift;
+    my $tag = shift;
+    my $subfield = shift;
+
+    my $field = $self->field($tag) or return undef;
+    return $field->subfield($subfield);
+} # subfield()
+
+=head2 insert_grouped_field(C<$field>)
+
+Will insert the specified MARC::Field object into the record in 'grouped
+order' and return true (1) on success, and false (undef) on failure.
+For example, if a '650' field is inserted with insert_grouped_field()
+it will be inserted at the end of the 6XX group of tags. After some discussion
+on the perl4lib list it was descided that this is ordinarily what you will 
+want. If you would like to insert at a specific point in the record you can use 
+insert_fields_after() and insert_fields_before() methods which are described 
+below. 
+
+=cut 
+
+sub insert_grouped_field {
+    my ($self,$new) = @_;
+    _all_parms_are_fields($new) or return(_gripe('Argument must be MARC::Fieldobject'));
+
+    ## try to find the end of the field group and insert it there
+    my $limit = int($new->tag() / 100);
+    my $found = 0;
+    foreach my $field ($self->fields()) {
+	if ( int($field->tag() / 100) > $limit ) {
+	    $self->insert_field_before($field,$new);
+	    $found = 1;
+	    last;
+	}
+    }
+
+    ## if we couldn't find the end of the group, then we must not have 
+    ## any tags this high yet, so just append it
+    if (!$found) {
+	$self->append_fields($new); 
+    }
+
+    return(1);
+
+}
+
+=for internal
+
+=cut
+
+sub _all_parms_are_fields {
+    for ( @_ ) {
+	return 0 unless ref($_) eq 'MARC::Field';
+    }
+    return 1;
+}
+
+=head2 append_field(C<@fields>)
+
+Appends the field specified by C<$field> to the end of the record. 
+C<@fields> need to be MARC::Field objects.
+
+    my $field = MARC::Field->new('590','','','a' => 'My local note.');
+    $record->append_field($field);
+
+Returns the number of fields appended.
+
+=cut
+
+sub append_fields {
+    my $self = shift;
+
+    _all_parms_are_fields(@_) or return(_gripe('Arguments must be MARC::Field objects'));
+
+    push(@{ $self->{_fields} }, @_); 
+    return scalar @_;
+}
+
+=head2 insert_fields_before($before_field,@new_fields)
+
+Inserts the field specified by C<$new_field> before the field C<$before_field>.
+Returns the number of fields inserted, or undef on failures.
+Both C<$before_field> and all C<@new_fields> need to be MARC::Field objects.
+
+    my $before_field = $record->field('260');
+    my $new_field = MARC::Field->new('250','','','a' => '2nd ed.');
+    $record->insert_fields_before($before_field,$new_field);
+
+=cut
+
+sub insert_fields_before {
+    my $self = shift;
+
+    _all_parms_are_fields(@_) or return(_gripe('All arguments must be MARC::Field objects'));
+    my ($before,@new) = @_;
+
+    ## find position of $before
+    my $fields = $self->{_fields};
+    my $pos = 0;
+    foreach my $f (@$fields) {
+	last if ($f == $before);
+	$pos++;
+    }
+
+    ## insert before $before 
+    if ($pos >= @$fields) {
+	return(_gripe("Couldn't find field to insert before"));
+    }
+    splice(@$fields,$pos,0,@new);
+    return scalar @new;
+
+}
+
+=head2 insert_fields_after($after_field,@new_fields)
+
+Identical to L<insert_fields_before()>, but fields are added after C<$after_field>.
+
+=cut
+
+sub insert_fields_after {
+    my $self = shift;
+
+    _all_parms_are_fields(@_) or return(_gripe('All arguments must be MARC::Field objects'));
+    my ($after,@new) = @_;
+
+    ## find position of $after
+    my $fields = $self->{_fields};
+    my $pos = 0;
+    foreach my $f (@$fields) {
+	last if ($f == $after);
+	$pos++;
+    }
+
+    ## insert after $after
+    if ($pos+1 >= @$fields) { 
+	return(_gripe("Couldn't find field to insert after"));
+    }
+    splice(@$fields,$pos+1,0,@new);
+    return scalar @new;
+}
+
+=head2 delete_field(C<$field>)
+
+Deletes a field from the record.
+
+The field must have been retrieved from the record using the 
+C<field()> method.  For example, to delete a 526 tag if it exists:
+
+    my $tag526 = $marc->field( "526" );
+    if ( $tag526 ) {
+	$marc->delete_field( $tag526 );
+    }
+
+C<delete_field()> returns the number of fields that were deleted.
+This shouldn't be 0 unless you didn't get the tag properly.
+
+=cut
+
+sub delete_field {
+    my $self = shift;
+    my $deleter = shift;
+    my $list = $self->{_fields};
+
+    my $old_count = @$list;
+    @$list = grep { $_ != $deleter } @$list;
+    return $old_count - @$list;
+}
+
+=head2 as_usmarc()
+
+This is a wrapper around C<MARC::File::USMARC::encode()> for compatibility with
+older versions of MARC::Record.
+
+=cut
+
+sub as_usmarc() {
+    my $self = shift;
+
+    require MARC::File::USMARC;
+
+    return MARC::File::USMARC::encode( $self );
+}
+
+=head2 as_formatted()
+
+Returns a pretty string for printing in a MARC dump.
+
+=cut
+
+sub as_formatted() {
+    my $self = shift;
+	    
+    my @lines = ( "LDR " . ($self->{_leader} || "") );
+    for my $field ( @{$self->{_fields}} ) {
+	    push( @lines, $field->as_formatted() );
+    }
+
+    return join( "\n", @lines );
+} # as_formatted
+
+=head2 title()
+
+Returns the title from the 245 tag.
+Note that it is a string, not a MARC::Field record.
+
+=cut
+
+sub title() {
+    my $self = shift;
+
+    my $field = $self->field(245) or return "<no 245 tag found>";
+
+    return $field->as_string;
+}
+
+=head2 author()
+
+Returns the author from the 100, 110 or 111 tag.
+Note that it is a string, not a MARC::Field record.
+
+=cut
+
+sub author() {
+    my $self = shift;
+
+    for my $tag ( qw( 100 110 111 ) ) {
+	my $field = $self->field($tag);
+	return $field->as_string() if $field;
+    }
+
+    return "<No author tag found>";
+}
+
+=head2 leader([text])
+
+Returns the leader for the record.  Sets the leader if I<text> is defined.
+No error checking is done on the validity of the leader.
+
+=cut
+
+sub leader {
+    my $self = shift;
+    my $text = shift;
+
+    if ( defined $text ) {
+    	(length($text) eq 24)
+	    or $self->_warn( "Leader must be 24 bytes long" );
+	$self->{_leader} = $text;
+    } # set the leader
+
+    return $self->{_leader};
+} # leader()
+
+=head2 set_leader_lengths( $reclen, $baseaddr )
+
+Internal function for updating the leader's length and base address.
+
+=cut
+
+sub set_leader_lengths {
+    my $self = shift;
+    my $reclen = shift;
+    my $baseaddr = shift;
+    substr($self->{_leader},0,5)  = sprintf("%05d",$reclen);
+    substr($self->{_leader},12,5) = sprintf("%05d",$baseaddr);
+}
+
+=head2 clone( [field specs] )
+
+The C<clone()> method makes a copy of an existing MARC record and returns
+the new version.  Note that you cannot just say:
+
+    my $newmarc = $oldmarc;
+
+This just makes a copy of the reference, not a new object.  You must use
+the C<clone()> method like so:
+
+    my $newmarc = $oldmarc->clone;
+
+You can also specify field specs to filter down only a 
+certain subset of fields.  For instance, if you only wanted the
+title and ISBN tags from a record, you could do this:
+
+    my $small_marc = $marc->clone( 245, '020' );
+
+The order of the fields is preserved as it was in the original record.
+
+=cut
+
+sub clone {
+    my $self = shift;
+    my @keeper_tags = @_;
+
+    my $clone = $self->new();
+    $clone->{_leader} = $self->{_leader};
+
+    my $filtered = @keeper_tags ? [$self->field( @keeper_tags )] : undef;
+
+    for my $field ( $self->fields() ) {
+        if ( !$filtered || (grep {$field==$_} @$filtered ) ) {
+	    $clone->add_fields( $field->clone );
+        }
+    }
+
+    # XXX FIX THIS $clone->update_leader();
+
+    return $clone;
+}
+
+=head2 warnings()
+
+Returns the warnings that were created when the record was read.
+These are things like "Invalid indicators converted to blanks".
+
+The warnings are items that you might be interested in, or might
+not.  It depends on how stringently you're checking data.  If
+you're doing some grunt data analysis, you probably don't care.
+
+A side effect of calling warnings() is that the warning buffer will
+be cleared.
+
+=cut
+
+sub warnings() {
+    my $self = shift;
+    my @warnings = @{$self->{_warnings}};
+    $self->{_warnings} = []; 
+    return @warnings;
+}
+
+=head2 add_fields()
+
+C<add_fields()> is now deprecated, and users are encouraged to use C<append_field()>,
+C<insert_field_after()>, and C<insert_field_before()> since they do what you want 
+probably. It is still here though, for backwards compatability.
+
+C<add_fields()> adds MARC::Field objects to the end of the list.  Returns the 
+number of fields added, or C<undef> if there was an error.
+
+There are three ways of calling C<add_fields()> to add data to the record.
+
+=over 4
+
+=item 1 Create a MARC::Field object and add it
+
+  my $author = MARC::Field->new(
+	        100, "1", " ", a => "Arnosky, Jim."
+	        );
+  $marc->add_fields( $author );
+
+=item 2 Add the data fields directly, and let C<add_fields()> take care of the objectifying.
+
+  $marc->add_fields(
+        245, "1", "0",
+                a => "Raccoons and ripe corn /",
+                c => "Jim Arnosky.",
+        	);
+
+=item 3 Same as #2 above, but pass multiple fields of data in anonymous lists
+
+  $marc->add_fields(
+	[ 250, " ", " ", a => "1st ed." ],
+	[ 650, "1", " ", a => "Raccoons." ],
+	);
+
+=back
+
+=cut
+
+sub add_fields {
+    my $self = shift;
+
+    my $nfields = 0;
+    my $fields = $self->{_fields};
+
+    while ( my $parm = shift ) {
+	# User handed us a list of data (most common possibility)
+	if ( ref($parm) eq "" ) {
+	    my $field = MARC::Field->new( $parm, @_ )
+		    or return _gripe( $MARC::Field::ERROR );
+	    push( @$fields, $field );
+	    ++$nfields;
+	    last; # Bail out, we're done eating parms
+
+	# User handed us an object.
+	} elsif ( ref($parm) eq "MARC::Field" ) {
+	    push( @$fields, $parm );
+	    ++$nfields;
+
+	# User handed us an anonymous list of parms
+	} elsif ( ref($parm) eq "ARRAY" ) {
+	    my $field = MARC::Field->new(@$parm) 
+		or return _gripe( $MARC::Field::ERROR );
+	    push( @$fields, $field );
+	    ++$nfields;
+
+	} else {
+	    return _gripe( "Unknown parm of type", ref($parm), " passed to add_fields()" );
+	} # if
+
+    } # while
+
+    return $nfields;
+}
+
+=head2 new_from_usmarc( $marcblob )
+
+This is a wrapper around C<MARC::File::USMARC::decode()> for compatibility with
+older versions of MARC::Record.
+
+=cut
+
+sub new_from_usmarc {
+    my $blob = shift;
+    $blob = shift if (ref($blob) || ($blob eq "MARC::Record"));
+
+    require MARC::File::USMARC;
+
+    return MARC::File::USMARC::decode( $blob );
+}
+
+
+# NOTE: _warn is an object method
+sub _warn {
+    my $self = shift;
+
+    push( @{$self->{_warnings}}, join( "", @_ ) );
+}
+
+
+# NOTE: _gripe is NOT an object method
+sub _gripe {
+    $ERROR = join( "", @_ );
+
+    warn $ERROR;
+
+    return undef;
+}
+
+
+1;
+
+__END__
+
+=head1 DESIGN NOTES
+
+A brief discussion of why MARC::Record is done the way it is:
+
+=over 4
+
+=item * It's built for quick prototyping
+
+One of the areas Perl excels is in allowing the programmer to 
+create easy solutions quickly.  MARC::Record is designed along
+those same lines.  You want a program to dump all the 6XX
+tags in a file?  MARC::Record is your friend.
+
+=item * It's built for extensibility
+
+Currently, I'm using MARC::Record for analyzing bibliographic
+data, but who knows what might happen in the future?  MARC::Record
+needs to be just as adept at authority data, too.
+
+=item * It's designed around accessor methods
+
+I use method calls everywhere, and I expect calling programs to do
+the same, rather than accessing internal data directly.  If you
+access an object's hash fields on your own, future releases may
+break your code.
+
+=item * It's not built for speed
+
+One of the tradeoffs in using accessor methods is some overhead
+in the method calls.  Is this slow?  I don't know, I haven't measured.
+I would suggest that if you're a cycle junkie that you use
+Benchmark.pm to check to see where your bottlenecks are, and then
+decide if MARC::Record is for you.
+
+=back
+
+=head1 RELATED MODULES
+
+L<MARC::Record>, L<MARC::Lint>
+
+=head1 SEE ALSO
+
+=over 4
+
+=item * perl4lib (L<http://www.rice.edu/perl4lib/>)
+
+A mailing list devoted to the use of Perl in libraries.
+
+=item * Library Of Congress MARC pages (L<http://www.loc.gov/marc/>)
+
+The definitive source for all things MARC.
+
+
+=item * I<Understanding MARC Bibliographic> (L<http://lcweb.loc.gov/marc/umb/>)
+
+Online version of the free booklet.  An excellent overview of the MARC format.  Essential.
+
+
+=item * Tag Of The Month (L<http://www.tagofthemonth.com/>)
+
+Follett Software Company's
+(L<http://www.fsc.follett.com/>) monthly discussion of various MARC tags.
+
+=back
+
+=head1 TODO
+
+=over 4
+
+=item * Incorporate MARC.pm in the distribution.
+
+Combine MARC.pm and MARC::* into one distribution.
+
+=item * Podify MARC.pm
+
+=item * Allow regexes across the entire tag
+
+Imagine something like this:
+
+  my @sears_headings = $marc->tag_grep( /Sears/ );
+
+(from Mike O'Regan)
+
+=item * Insert a field in an arbitrary place in the record
+
+=item * Allow deleting a field
+
+  for my $field ( $record->field( "856" ) ) {
+	$record->delete_field( $field ) unless useful($field);
+	} # for
+
+(from Anne Highsmith hismith@tamu.edu)
+
+
+=item * Modifying an existing field
+
+=back
+
+=head1 IDEAS
+
+Ideas are things that have been considered, but nobody's actually asked for.
+
+=over 4
+
+=item * Create multiple output formats.
+
+These could be ASCII, XML, or MarcMaker.
+
+=back
+
+=head1 LICENSE
+
+This code may be distributed under the same terms as Perl itself. 
+
+Please note that these modules are not products of or supported by the
+employers of the various contributors to the code.
+
+=head1 AUTHOR
+
+Andy Lester, E<lt>marc@petdance.comE<gt> or E<lt>alester@flr.follett.comE<gt>
+
+=cut
+
Index: /main/tags/3.01/gsdl/perllib/cpan/MP3/Info.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/MP3/Info.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/MP3/Info.pm	(revision 21112)
@@ -0,0 +1,1563 @@
+package MP3::Info;
+use overload;
+use strict;
+use Carp;
+use Symbol;
+
+use vars qw(
+	@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION $REVISION
+	@mp3_genres %mp3_genres @winamp_genres %winamp_genres $try_harder
+	@t_bitrate @t_sampling_freq @frequency_tbl %v1_tag_fields
+	@v1_tag_names %v2_tag_names %v2_to_v1_names $AUTOLOAD
+	@mp3_info_fields
+);
+
+@ISA = 'Exporter';
+@EXPORT = qw(
+	set_mp3tag get_mp3tag get_mp3info remove_mp3tag
+	use_winamp_genres
+);
+@EXPORT_OK = qw(@mp3_genres %mp3_genres use_mp3_utf8);
+%EXPORT_TAGS = (
+	genres	=> [qw(@mp3_genres %mp3_genres)],
+	utf8	=> [qw(use_mp3_utf8)],
+	all	=> [@EXPORT, @EXPORT_OK]
+);
+
+# $Id$
+($REVISION) = ' $Revision$ ' =~ /\$Revision:\s+([^\s]+)/;
+$VERSION = '1.02';
+
+=pod
+
+=head1 NAME
+
+MP3::Info - Manipulate / fetch info from MP3 audio files
+
+=head1 SYNOPSIS
+
+	#!perl -w
+	use MP3::Info;
+	my $file = 'Pearls_Before_Swine.mp3';
+	set_mp3tag($file, 'Pearls Before Swine', q"77's",
+		'Sticks and Stones', '1990',
+		q"(c) 1990 77's LTD.", 'rock & roll');
+
+	my $tag = get_mp3tag($file) or die "No TAG info";
+	$tag->{GENRE} = 'rock';
+	set_mp3tag($file, $tag);
+
+	my $info = get_mp3info($file);
+	printf "$file length is %d:%d\n", $info->{MM}, $info->{SS};
+
+=cut
+
+{
+	my $c = -1;
+	# set all lower-case and regular-cased versions of genres as keys
+	# with index as value of each key
+	%mp3_genres = map {($_, ++$c, lc, $c)} @mp3_genres;
+
+	# do it again for winamp genres
+	$c = -1;
+	%winamp_genres = map {($_, ++$c, lc, $c)} @winamp_genres;
+}
+
+=pod
+
+	my $mp3 = new MP3::Info $file;
+	$mp3->title('Perls Before Swine');
+	printf "$file length is %s, title is %s\n",
+		$mp3->time, $mp3->title;
+
+
+=head1 DESCRIPTION
+
+=over 4
+
+=item $mp3 = MP3::Info-E<gt>new(FILE)
+
+OOP interface to the rest of the module.  The same keys
+available via get_mp3info and get_mp3tag are available
+via the returned object (using upper case or lower case;
+but note that all-caps "VERSION" will return the module
+version, not the MP3 version).
+
+Passing a value to one of the methods will set the value
+for that tag in the MP3 file, if applicable.
+
+=cut
+
+sub new {
+	my($pack, $file) = @_;
+
+	my $info = get_mp3info($file) or return undef;
+	my $tags = get_mp3tag($file) || { map { ($_ => undef) } @v1_tag_names };
+	my %self = (
+		FILE		=> $file,
+		TRY_HARDER	=> 0
+	);
+
+	@self{@mp3_info_fields, @v1_tag_names, 'file'} = (
+		@{$info}{@mp3_info_fields},
+		@{$tags}{@v1_tag_names},
+		$file
+	);
+
+	return bless \%self, $pack;
+}
+
+sub can {
+	my $self = shift;
+	return $self->SUPER::can(@_) unless ref $self;
+	my $name = uc shift;
+	return sub { $self->$name(@_) } if exists $self->{$name};
+	return undef;
+}
+
+sub AUTOLOAD {
+	my($self) = @_;
+	(my $name = uc $AUTOLOAD) =~ s/^.*://;
+
+	if (exists $self->{$name}) {
+		my $sub = exists $v1_tag_fields{$name}
+			? sub {
+				if (defined $_[1]) {
+					$_[0]->{$name} = $_[1];
+					set_mp3tag($_[0]->{FILE}, $_[0]);
+				}
+				return $_[0]->{$name};
+			}
+			: sub {
+				return $_[0]->{$name}
+			};
+
+		no strict 'refs';
+		*{$AUTOLOAD} = $sub;
+		goto &$AUTOLOAD;
+
+	} else {
+		carp(sprintf "No method '$name' available in package %s.",
+			__PACKAGE__);
+	}
+}
+
+sub DESTROY {
+
+}
+
+
+=item use_mp3_utf8([STATUS])
+
+Tells MP3::Info to (or not) return TAG info in UTF-8.
+TRUE is 1, FALSE is 0.  Default is FALSE.
+
+Will only be able to it on if Unicode::String is available.  ID3v2
+tags will be converted to UTF-8 according to the encoding specified
+in each tag; ID3v1 tags will be assumed Latin-1 and converted
+to UTF-8.
+
+Function returns status (TRUE/FALSE).  If no argument is supplied,
+or an unaccepted argument is supplied, function merely returns status.
+
+This function is not exported by default, but may be exported
+with the C<:utf8> or C<:all> export tag.
+
+=cut
+
+my $unicode_module = eval { require Unicode::String };
+my $UNICODE = 0;
+
+sub use_mp3_utf8 {
+	my($val) = @_;
+	if ($val == 1) {
+		$UNICODE = 1 if $unicode_module;
+	} elsif ($val == 0) {
+		$UNICODE = 0;
+	}
+	return $UNICODE;
+}
+
+=pod
+
+=item use_winamp_genres()
+
+Puts WinAmp genres into C<@mp3_genres> and C<%mp3_genres>
+(adds 68 additional genres to the default list of 80).
+This is a separate function because these are non-standard
+genres, but they are included because they are widely used.
+
+You can import the data structures with one of:
+
+	use MP3::Info qw(:genres);
+	use MP3::Info qw(:DEFAULT :genres);
+	use MP3::Info qw(:all);
+
+=cut
+
+sub use_winamp_genres {
+	%mp3_genres = %winamp_genres;
+	@mp3_genres = @winamp_genres;
+	return 1;
+}
+
+=pod
+
+=item remove_mp3tag (FILE [, VERSION, BUFFER])
+
+Can remove ID3v1 or ID3v2 tags.  VERSION should be C<1> for ID3v1,
+C<2> for ID3v2, and C<ALL> for both.
+
+For ID3v1, removes last 128 bytes from file if those last 128 bytes begin
+with the text 'TAG'.  File will be 128 bytes shorter.
+
+For ID3v2, removes ID3v2 tag.  Because an ID3v2 tag is at the
+beginning of the file, we rewrite the file after removing the tag data.
+The buffer for rewriting the file is 4MB.  BUFFER (in bytes) ca
+change the buffer size.
+
+Returns the number of bytes removed, or -1 if no tag removed,
+or undef if there is an error.
+
+=cut
+
+sub remove_mp3tag {
+	my($file, $version, $buf) = @_;
+	my($fh, $return);
+
+	$buf ||= 4096*1024;  # the bigger the faster
+	$version ||= 1;
+
+	if (not (defined $file && $file ne '')) {
+		$@ = "No file specified";
+		return undef;
+	}
+
+	if (not -s $file) {
+		$@ = "File is empty";
+		return undef;
+	}
+
+	if (ref $file) { # filehandle passed
+		$fh = $file;
+	} else {
+		$fh = gensym;
+		if (not open $fh, "+< $file\0") {
+			$@ = "Can't open $file: $!";
+			return undef;
+		}
+	}
+
+	binmode $fh;
+
+	if ($version eq 1 || $version eq 'ALL') {
+		seek $fh, -128, 2;
+		my $tell = tell $fh;
+		if (<$fh> =~ /^TAG/) {
+			truncate $fh, $tell or carp "Can't truncate '$file': $!";
+			$return += 128;
+		}
+	}
+
+	if ($version eq 2 || $version eq 'ALL') {
+		my $h = _get_v2head($fh);
+		if ($h) {
+			local $\;
+			seek $fh, 0, 2;
+			my $eof = tell $fh;
+			my $off = $h->{tag_size};
+
+			while ($off < $eof) {
+				seek $fh, $off, 0;
+				read $fh, my($bytes), $buf;
+				seek $fh, $off - $h->{tag_size}, 0;
+				print $fh $bytes;
+				$off += $buf;
+			}
+
+			truncate $fh, $eof - $h->{tag_size}
+				or carp "Can't truncate '$file': $!";
+			$return += $h->{tag_size};
+		}
+	}
+
+	_close($file, $fh);
+
+	return $return || -1;
+}
+
+
+=pod
+
+=item set_mp3tag (FILE, TITLE, ARTIST, ALBUM, YEAR, COMMENT, GENRE [, TRACKNUM])
+
+=item set_mp3tag (FILE, $HASHREF)
+
+Adds/changes tag information in an MP3 audio file.  Will clobber
+any existing information in file.
+
+Fields are TITLE, ARTIST, ALBUM, YEAR, COMMENT, GENRE.  All fields have
+a 30-byte limit, except for YEAR, which has a four-byte limit, and GENRE,
+which is one byte in the file.  The GENRE passed in the function is a
+case-insensitive text string representing a genre found in C<@mp3_genres>.
+
+Will accept either a list of values, or a hashref of the type
+returned by C<get_mp3tag>.
+
+If TRACKNUM is present (for ID3v1.1), then the COMMENT field can only be
+28 bytes.
+
+ID3v2 support may come eventually.  Note that if you set a tag on a file
+with ID3v2, the set tag will be for ID3v1[.1] only, and if you call
+C<get_mp3_tag> on the file, it will show you the (unchanged) ID3v2 tags,
+unless you specify ID3v1.
+
+=cut
+
+sub set_mp3tag {
+	my($file, $title, $artist, $album, $year, $comment, $genre, $tracknum) = @_;
+	my(%info, $oldfh, $ref, $fh);
+	local %v1_tag_fields = %v1_tag_fields;
+
+	# set each to '' if undef
+	for ($title, $artist, $album, $year, $comment, $tracknum, $genre,
+		(@info{@v1_tag_names}))
+		{$_ = defined() ? $_ : ''}
+
+	($ref) = (overload::StrVal($title) =~ /^(?:.*\=)?([^=]*)\((?:[^\(]*)\)$/)
+		if ref $title;
+	# populate data to hashref if hashref is not passed
+	if (!$ref) {
+		(@info{@v1_tag_names}) =
+			($title, $artist, $album, $year, $comment, $tracknum, $genre);
+
+	# put data from hashref into hashref if hashref is passed
+	} elsif ($ref eq 'HASH') {
+		%info = %$title;
+
+	# return otherwise
+	} else {
+		carp(<<'EOT');
+Usage: set_mp3tag (FILE, TITLE, ARTIST, ALBUM, YEAR, COMMENT, GENRE [, TRACKNUM])
+       set_mp3tag (FILE, $HASHREF)
+EOT
+		return undef;
+	}
+
+	if (not (defined $file && $file ne '')) {
+		$@ = "No file specified";
+		return undef;
+	}
+
+	if (not -s $file) {
+		$@ = "File is empty";
+		return undef;
+	}
+
+	# comment field length 28 if ID3v1.1
+	$v1_tag_fields{COMMENT} = 28 if $info{TRACKNUM};
+
+
+	# only if -w is on
+	if ($^W) {
+		# warn if fields too long
+		foreach my $field (keys %v1_tag_fields) {
+			$info{$field} = '' unless defined $info{$field};
+			if (length($info{$field}) > $v1_tag_fields{$field}) {
+				carp "Data too long for field $field: truncated to " .
+					 "$v1_tag_fields{$field}";
+			}
+		}
+
+		if ($info{GENRE}) {
+			carp "Genre `$info{GENRE}' does not exist\n"
+				unless exists $mp3_genres{$info{GENRE}};
+		}
+	}
+
+	if ($info{TRACKNUM}) {
+		$info{TRACKNUM} =~ s/^(\d+)\/(\d+)$/$1/;
+		unless ($info{TRACKNUM} =~ /^\d+$/ &&
+			$info{TRACKNUM} > 0 && $info{TRACKNUM} < 256) {
+			carp "Tracknum `$info{TRACKNUM}' must be an integer " .
+				"from 1 and 255\n" if $^W;
+			$info{TRACKNUM} = '';
+		}
+	}
+
+	if (ref $file) { # filehandle passed
+		$fh = $file;
+	} else {
+		$fh = gensym;
+		if (not open $fh, "+< $file\0") {
+			$@ = "Can't open $file: $!";
+			return undef;
+		}
+	}
+
+	binmode $fh;
+	$oldfh = select $fh;
+	seek $fh, -128, 2;
+	# go to end of file if no tag, beginning of file if tag
+	seek $fh, (<$fh> =~ /^TAG/ ? -128 : 0), 2;
+
+	# get genre value
+	$info{GENRE} = $info{GENRE} && exists $mp3_genres{$info{GENRE}} ?
+		$mp3_genres{$info{GENRE}} : 255;  # some default genre
+
+	local $\;
+	# print TAG to file
+	if ($info{TRACKNUM}) {
+		print pack "a3a30a30a30a4a28xCC", 'TAG', @info{@v1_tag_names};
+	} else {
+		print pack "a3a30a30a30a4a30C", 'TAG', @info{@v1_tag_names[0..4, 6]};
+	}
+
+	select $oldfh;
+
+	_close($file, $fh);
+
+	return 1;
+}
+
+=pod
+
+=item get_mp3tag (FILE [, VERSION, RAW_V2])
+
+Returns hash reference containing tag information in MP3 file.  The keys
+returned are the same as those supplied for C<set_mp3tag>, except in the
+case of RAW_V2 being set.
+
+If VERSION is C<1>, the information is taken from the ID3v1 tag (if present).
+If VERSION is C<2>, the information is taken from the ID3v2 tag (if present).
+If VERSION is not supplied, or is false, the ID3v1 tag is read if present, and
+then, if present, the ID3v2 tag information will override any existing ID3v1
+tag info.
+
+If RAW_V2 is C<1>, the raw ID3v2 tag data is returned, without any manipulation
+of text encoding.  The key name is the same as the frame ID (ID to name mappings
+are in the global %v2_tag_names).
+
+If RAW_V2 is C<2>, the ID3v2 tag data is returned, manipulating for Unicode if
+necessary, etc.  It also takes multiple values for a given key (such as comments)
+and puts them in an arrayref.
+
+If the ID3v2 version is older than ID3v2.2.0 or newer than ID3v2.4.0, it will
+not be read.
+
+Strings returned will be in Latin-1, unless UTF-8 is specified (L<use_mp3_utf8>),
+(unless RAW_V2 is C<1>).
+
+Also returns a TAGVERSION key, containing the ID3 version used for the returned
+data (if TAGVERSION argument is C<0>, may contain two versions).
+
+=cut
+
+sub get_mp3tag {
+	my($file, $ver, $raw_v2) = @_;
+	my($tag, $v1, $v2, $v2h, %info, @array, $fh);
+	$raw_v2 ||= 0;
+	$ver = !$ver ? 0 : ($ver == 2 || $ver == 1) ? $ver : 0;
+
+	if (not (defined $file && $file ne '')) {
+		$@ = "No file specified";
+		return undef;
+	}
+
+	if (not -s $file) {
+		$@ = "File is empty";
+		return undef;
+	}
+
+	if (ref $file) { # filehandle passed
+		$fh = $file;
+	} else {
+		$fh = gensym;
+		if (not open $fh, "< $file\0") {
+			$@ = "Can't open $file: $!";
+			return undef;
+		}
+	}
+
+	binmode $fh;
+
+	if ($ver < 2) {
+		seek $fh, -128, 2;
+		while(defined(my $line = <$fh>)) { $tag .= $line }
+
+		if ($tag =~ /^TAG/) {
+			$v1 = 1;
+			if (substr($tag, -3, 2) =~ /\000[^\000]/) {
+				(undef, @info{@v1_tag_names}) =
+					(unpack('a3a30a30a30a4a28', $tag),
+					ord(substr($tag, -2, 1)),
+					$mp3_genres[ord(substr $tag, -1)]);
+				$info{TAGVERSION} = 'ID3v1.1';
+			} else {
+				(undef, @info{@v1_tag_names[0..4, 6]}) =
+					(unpack('a3a30a30a30a4a30', $tag),
+					$mp3_genres[ord(substr $tag, -1)]);
+				$info{TAGVERSION} = 'ID3v1';
+			}
+			if ($UNICODE) {
+				for my $key (keys %info) {
+					next unless $info{$key};
+					my $u = Unicode::String::latin1($info{$key});
+					$info{$key} = $u->utf8;
+				}
+			}
+		} elsif ($ver == 1) {
+			_close($file, $fh);
+			$@ = "No ID3v1 tag found";
+			return undef;
+		}
+	}
+
+	($v2, $v2h) = _get_v2tag($fh);
+
+	unless ($v1 || $v2) {
+		_close($file, $fh);
+		$@ = "No ID3 tag found";
+		return undef;
+	}
+
+	if (($ver == 0 || $ver == 2) && $v2) {
+		if ($raw_v2 == 1 && $ver == 2) {
+			%info = %$v2;
+			$info{TAGVERSION} = $v2h->{version};
+		} else {
+			my $hash = $raw_v2 == 2 ? { map { ($_, $_) } keys %v2_tag_names } : \%v2_to_v1_names;
+			for my $id (keys %$hash) {
+				if (exists $v2->{$id}) {
+					if ($id =~ /^TCON?$/ && $v2->{$id} =~ /^.?\((\d+)\)/) {
+						$info{$hash->{$id}} = $mp3_genres[$1];
+					} else {
+						my $data1 = $v2->{$id};
+
+						# this is tricky ... if this is an arrayref,
+						# we want to only return one, so we pick the
+						# first one.  but if it is a comment, we pick
+						# the first one where the first charcter after
+						# the language is NULL and not an additional
+						# sub-comment, because that is most likely to be
+						# the user-supplied comment
+						if (ref $data1 && !$raw_v2) {
+							if ($id =~ /^COMM?$/) {
+								my($newdata) = grep /^(....\000)/, @{$data1};
+								$data1 = $newdata || $data1->[0];
+							} else {
+								$data1 = $data1->[0];
+							}
+						}
+
+						$data1 = [ $data1 ] if ! ref $data1;
+
+						for my $data (@$data1) {
+							$data =~ s/^(.)//; # strip first char (text encoding)
+							my $encoding = $1;
+							my $desc;
+							if ($id =~ /^COM[M ]?$/) {
+								$data =~ s/^(?:...)//;		# strip language
+								$data =~ s/^(.*?)\000+//;	# strip up to first NULL(s),
+												# for sub-comment
+								$desc = $1;
+							}
+
+							if ($UNICODE) {
+								if ($encoding eq "\001" || $encoding eq "\002") {  # UTF-16, UTF-16BE
+									my $u = Unicode::String::utf16($data);
+									$data = $u->utf8;
+									$data =~ s/^\xEF\xBB\xBF//;	# strip BOM
+								} elsif ($encoding eq "\000") {
+									my $u = Unicode::String::latin1($data);
+									$data = $u->utf8;
+								}
+							}
+
+							if ($raw_v2 == 2 && $desc) {
+								$data = { $desc => $data };
+							}
+
+							if ($raw_v2 == 2 && exists $info{$hash->{$id}}) {
+								if (ref $info{$hash->{$id}} eq 'ARRAY') {
+									push @{$info{$hash->{$id}}}, $data;
+								} else {
+									$info{$hash->{$id}} = [ $info{$hash->{$id}}, $data ];
+								}
+							} else {
+								$info{$hash->{$id}} = $data;
+							}
+						}
+					}
+				}
+			}
+			if ($ver == 0 && $info{TAGVERSION}) {
+				$info{TAGVERSION} .= ' / ' . $v2h->{version};
+			} else {
+				$info{TAGVERSION} = $v2h->{version};
+			}
+		}
+	}
+
+	unless ($raw_v2 && $ver == 2) {
+		foreach my $key (keys %info) {
+			if (defined $info{$key}) {
+				$info{$key} =~ s/\000+.*//g;
+				$info{$key} =~ s/\s+$//;
+			}
+		}
+
+		for (@v1_tag_names) {
+			$info{$_} = '' unless defined $info{$_};
+		}
+	}
+
+	if (keys %info && exists $info{GENRE} && ! defined $info{GENRE}) {
+		$info{GENRE} = '';
+	}
+
+	_close($file, $fh);
+
+	return keys %info ? {%info} : undef;
+}
+
+sub _get_v2tag {
+	my($fh) = @_;
+	my($off, $myseek, $myseek_22, $myseek_23, $v2, $h, $hlen, $num);
+	$h = {};
+
+	$v2 = _get_v2head($fh) or return;
+	if ($v2->{major_version} < 2) {
+		carp "This is $v2->{version}; " .
+		     "ID3v2 versions older than ID3v2.2.0 not supported\n"
+		     if $^W;
+		return;
+	}
+
+	if ($v2->{major_version} == 2) {
+		$hlen = 6;
+		$num = 3;
+	} else {
+		$hlen = 10;
+		$num = 4;
+	}
+
+	$myseek = sub {
+		seek $fh, $off, 0;
+		read $fh, my($bytes), $hlen;
+		return unless $bytes =~ /^([A-Z0-9]{$num})/
+			|| ($num == 4 && $bytes =~ /^(COM )/);  # stupid iTunes
+		my($id, $size) = ($1, $hlen);
+		my @bytes = reverse unpack "C$num", substr($bytes, $num, $num);
+		for my $i (0 .. ($num - 1)) {
+			$size += $bytes[$i] * 256 ** $i;
+		}
+		return($id, $size);
+	};
+
+	$off = $v2->{ext_header_size} + 10;
+
+	while ($off < $v2->{tag_size}) {
+		my($id, $size) = &$myseek or last;
+		seek $fh, $off + $hlen, 0;
+		read $fh, my($bytes), $size - $hlen;
+		if (exists $h->{$id}) {
+			if (ref $h->{$id} eq 'ARRAY') {
+				push @{$h->{$id}}, $bytes;
+			} else {
+				$h->{$id} = [$h->{$id}, $bytes];
+			}
+		} else {
+			$h->{$id} = $bytes;
+		}
+		$off += $size;
+	}
+
+	return($h, $v2);
+}
+
+
+=pod
+
+=item get_mp3info (FILE)
+
+Returns hash reference containing file information for MP3 file.
+This data cannot be changed.  Returned data:
+
+	VERSION		MPEG audio version (1, 2, 2.5)
+	LAYER		MPEG layer description (1, 2, 3)
+	STEREO		boolean for audio is in stereo
+
+	VBR		boolean for variable bitrate
+	BITRATE		bitrate in kbps (average for VBR files)
+	FREQUENCY	frequency in kHz
+	SIZE		bytes in audio stream
+
+	SECS		total seconds
+	MM		minutes
+	SS		leftover seconds
+	MS		leftover milliseconds
+	TIME		time in MM:SS
+
+	COPYRIGHT	boolean for audio is copyrighted
+	PADDING		boolean for MP3 frames are padded
+	MODE		channel mode (0 = stereo, 1 = joint stereo,
+			2 = dual channel, 3 = single channel)
+	FRAMES		approximate number of frames
+	FRAME_LENGTH	approximate length of a frame
+	VBR_SCALE	VBR scale from VBR header
+
+On error, returns nothing and sets C<$@>.
+
+=cut
+
+sub get_mp3info {
+	my($file) = @_;
+	my($off, $myseek, $byte, $eof, $h, $tot, $fh);
+
+	if (not (defined $file && $file ne '')) {
+		$@ = "No file specified";
+		return undef;
+	}
+
+	if (not -s $file) {
+		$@ = "File is empty";
+		return undef;
+	}
+
+	if (ref $file) { # filehandle passed
+		$fh = $file;
+	} else {
+		$fh = gensym;
+		if (not open $fh, "< $file\0") {
+			$@ = "Can't open $file: $!";
+			return undef;
+		}
+	}
+
+	$off = 0;
+	$tot = 4096;
+
+	$myseek = sub {
+		seek $fh, $off, 0;
+		read $fh, $byte, 4;
+	};
+
+	binmode $fh;
+	&$myseek;
+
+	if ($off == 0) {
+		if (my $id3v2 = _get_v2head($fh)) {
+			$tot += $off += $id3v2->{tag_size};
+			&$myseek;
+		}
+	}
+
+	$h = _get_head($byte);
+	until (_is_mp3($h)) {
+		$off++;
+		&$myseek;
+		$h = _get_head($byte);
+		if ($off > $tot && !$try_harder) {
+			_close($file, $fh);
+			$@ = "Couldn't find MP3 header (perhaps set " .
+			     '$MP3::Info::try_harder and retry)';
+			return undef;
+		}
+	}
+
+	my $vbr = _get_vbr($fh, $h, \$off);
+
+	seek $fh, 0, 2;
+	$eof = tell $fh;
+	seek $fh, -128, 2;
+	$off += 128 if <$fh> =~ /^TAG/ ? 1 : 0;
+
+	_close($file, $fh);
+
+	$h->{size} = $eof - $off;
+
+	return _get_info($h, $vbr);
+}
+
+sub _get_info {
+	my($h, $vbr) = @_;
+	my $i;
+
+	$i->{VERSION}	= $h->{IDR} == 2 ? 2 : $h->{IDR} == 3 ? 1 :
+				$h->{IDR} == 0 ? 2.5 : 0;
+	$i->{LAYER}	= 4 - $h->{layer};
+	$i->{VBR}	= defined $vbr ? 1 : 0;
+
+	$i->{COPYRIGHT}	= $h->{copyright} ? 1 : 0;
+	$i->{PADDING}	= $h->{padding_bit} ? 1 : 0;
+	$i->{STEREO}	= $h->{mode} == 3 ? 0 : 1;
+	$i->{MODE}	= $h->{mode};
+
+	$i->{SIZE}	= $vbr && $vbr->{bytes} ? $vbr->{bytes} : $h->{size};
+
+	my $mfs		= $h->{fs} / ($h->{ID} ? 144000 : 72000);
+	$i->{FRAMES}	= int($vbr && $vbr->{frames}
+				? $vbr->{frames}
+				: $i->{SIZE} / $h->{bitrate} / $mfs
+			  );
+
+	if ($vbr) {
+		$i->{VBR_SCALE}	= $vbr->{scale} if $vbr->{scale};
+		$h->{bitrate}	= $i->{SIZE} / $i->{FRAMES} * $mfs;
+		if (not $h->{bitrate}) {
+			$@ = "Couldn't determine VBR bitrate";
+			return undef;
+		}
+	}
+
+	$h->{'length'}	= ($i->{SIZE} * 8) / $h->{bitrate} / 10;
+	$i->{SECS}	= $h->{'length'} / 100;
+	$i->{MM}	= int $i->{SECS} / 60;
+	$i->{SS}	= int $i->{SECS} % 60;
+	$i->{MS}	= (($i->{SECS} - ($i->{MM} * 60) - $i->{SS}) * 1000);
+#	$i->{LF}	= ($i->{MS} / 1000) * ($i->{FRAMES} / $i->{SECS});
+#	int($i->{MS} / 100 * 75);  # is this right?
+	$i->{TIME}	= sprintf "%.2d:%.2d", @{$i}{'MM', 'SS'};
+
+	$i->{BITRATE}		= int $h->{bitrate};
+	# should we just return if ! FRAMES?
+	$i->{FRAME_LENGTH}	= int($h->{size} / $i->{FRAMES}) if $i->{FRAMES};
+	$i->{FREQUENCY}		= $frequency_tbl[3 * $h->{IDR} + $h->{sampling_freq}];
+
+	return $i;
+}
+
+sub _get_head {
+	my($byte) = @_;
+	my($bytes, $h);
+
+	$bytes = _unpack_head($byte);
+	@$h{qw(IDR ID layer protection_bit
+		bitrate_index sampling_freq padding_bit private_bit
+		mode mode_extension copyright original
+		emphasis version_index bytes)} = (
+		($bytes>>19)&3, ($bytes>>19)&1, ($bytes>>17)&3, ($bytes>>16)&1,
+		($bytes>>12)&15, ($bytes>>10)&3, ($bytes>>9)&1, ($bytes>>8)&1,
+		($bytes>>6)&3, ($bytes>>4)&3, ($bytes>>3)&1, ($bytes>>2)&1,
+		$bytes&3, ($bytes>>19)&3, $bytes
+	);
+
+	$h->{bitrate} = $t_bitrate[$h->{ID}][3 - $h->{layer}][$h->{bitrate_index}];
+	$h->{fs} = $t_sampling_freq[$h->{IDR}][$h->{sampling_freq}];
+
+	return $h;
+}
+
+sub _is_mp3 {
+	my $h = $_[0] or return undef;
+	return ! (	# all below must be false
+		 $h->{bitrate_index} == 0
+			||
+		 $h->{version_index} == 1
+			||
+		($h->{bytes} & 0xFFE00000) != 0xFFE00000
+			||
+		!$h->{fs}
+			||
+		!$h->{bitrate}
+			||
+		 $h->{bitrate_index} == 15
+			||
+		!$h->{layer}
+			||
+		 $h->{sampling_freq} == 3
+			||
+		 $h->{emphasis} == 2
+			||
+		!$h->{bitrate_index}
+			||
+		($h->{bytes} & 0xFFFF0000) == 0xFFFE0000
+			||
+		($h->{ID} == 1 && $h->{layer} == 3 && $h->{protection_bit} == 1)
+			||
+		($h->{mode_extension} != 0 && $h->{mode} != 1)
+	);
+}
+
+sub _get_vbr {
+	my($fh, $h, $roff) = @_;
+	my($off, $bytes, @bytes, $myseek, %vbr);
+
+	$off = $$roff;
+	@_ = ();	# closure confused if we don't do this
+
+	$myseek = sub {
+		my $n = $_[0] || 4;
+		seek $fh, $off, 0;
+		read $fh, $bytes, $n;
+		$off += $n;
+	};
+
+	$off += 4;
+
+	if ($h->{ID}) {	# MPEG1
+		$off += $h->{mode} == 3 ? 17 : 32;
+	} else {	# MPEG2
+		$off += $h->{mode} == 3 ? 9 : 17;
+	}
+
+	&$myseek;
+	return unless $bytes eq 'Xing';
+
+	&$myseek;
+	$vbr{flags} = _unpack_head($bytes);
+
+	if ($vbr{flags} & 1) {
+		&$myseek;
+		$vbr{frames} = _unpack_head($bytes);
+	}
+
+	if ($vbr{flags} & 2) {
+		&$myseek;
+		$vbr{bytes} = _unpack_head($bytes);
+	}
+
+	if ($vbr{flags} & 4) {
+		$myseek->(100);
+# Not used right now ...
+#		$vbr{toc} = _unpack_head($bytes);
+	}
+
+	if ($vbr{flags} & 8) { # (quality ind., 0=best 100=worst)
+		&$myseek;
+		$vbr{scale} = _unpack_head($bytes);
+	} else {
+		$vbr{scale} = -1;
+	}
+
+	$$roff = $off;
+	return \%vbr;
+}
+
+sub _get_v2head {
+	my $fh = $_[0] or return;
+	my($h, $bytes, @bytes);
+
+	# check first three bytes for 'ID3'
+	seek $fh, 0, 0;
+	read $fh, $bytes, 3;
+	return unless $bytes eq 'ID3';
+
+	# get version
+	read $fh, $bytes, 2;
+	$h->{version} = sprintf "ID3v2.%d.%d",
+		@$h{qw[major_version minor_version]} =
+			unpack 'c2', $bytes;
+
+	# get flags
+	read $fh, $bytes, 1;
+	if ($h->{major_version} == 2) {
+		@$h{qw[unsync compression]} =
+			(unpack 'b8', $bytes)[7, 6];
+		$h->{ext_header} = 0;
+		$h->{experimental} = 0;
+	} else {
+		@$h{qw[unsync ext_header experimental]} =
+			(unpack 'b8', $bytes)[7, 6, 5];
+	}
+
+	# get ID3v2 tag length from bytes 7-10
+	$h->{tag_size} = 10;	# include ID3v2 header size
+	read $fh, $bytes, 4;
+	@bytes = reverse unpack 'C4', $bytes;
+	foreach my $i (0 .. 3) {
+		# whoaaaaaa nellllllyyyyyy!
+		$h->{tag_size} += $bytes[$i] * 128 ** $i;
+	}
+
+	# get extended header size
+	$h->{ext_header_size} = 0;
+	if ($h->{ext_header}) {
+		$h->{ext_header_size} += 10;
+		read $fh, $bytes, 4;
+		@bytes = reverse unpack 'C4', $bytes;
+		for my $i (0..3) {
+			$h->{ext_header_size} += $bytes[$i] * 256 ** $i;
+		}
+	}
+
+	return $h;
+}
+
+sub _unpack_head {
+	unpack('l', pack('L', unpack('N', $_[0])));
+}
+
+sub _close {
+	my($file, $fh) = @_;
+	unless (ref $file) { # filehandle not passed
+		close $fh or carp "Problem closing '$file': $!";
+	}
+}
+
+BEGIN {
+	@mp3_genres = (
+		'Blues',
+		'Classic Rock',
+		'Country',
+		'Dance',
+		'Disco',
+		'Funk',
+		'Grunge',
+		'Hip-Hop',
+		'Jazz',
+		'Metal',
+		'New Age',
+		'Oldies',
+		'Other',
+		'Pop',
+		'R&B',
+		'Rap',
+		'Reggae',
+		'Rock',
+		'Techno',
+		'Industrial',
+		'Alternative',
+		'Ska',
+		'Death Metal',
+		'Pranks',
+		'Soundtrack',
+		'Euro-Techno',
+		'Ambient',
+		'Trip-Hop',
+		'Vocal',
+		'Jazz+Funk',
+		'Fusion',
+		'Trance',
+		'Classical',
+		'Instrumental',
+		'Acid',
+		'House',
+		'Game',
+		'Sound Clip',
+		'Gospel',
+		'Noise',
+		'AlternRock',
+		'Bass',
+		'Soul',
+		'Punk',
+		'Space',
+		'Meditative',
+		'Instrumental Pop',
+		'Instrumental Rock',
+		'Ethnic',
+		'Gothic',
+		'Darkwave',
+		'Techno-Industrial',
+		'Electronic',
+		'Pop-Folk',
+		'Eurodance',
+		'Dream',
+		'Southern Rock',
+		'Comedy',
+		'Cult',
+		'Gangsta',
+		'Top 40',
+		'Christian Rap',
+		'Pop/Funk',
+		'Jungle',
+		'Native American',
+		'Cabaret',
+		'New Wave',
+		'Psychadelic',
+		'Rave',
+		'Showtunes',
+		'Trailer',
+		'Lo-Fi',
+		'Tribal',
+		'Acid Punk',
+		'Acid Jazz',
+		'Polka',
+		'Retro',
+		'Musical',
+		'Rock & Roll',
+		'Hard Rock',
+	);
+
+	@winamp_genres = (
+		@mp3_genres,
+		'Folk',
+		'Folk-Rock',
+		'National Folk',
+		'Swing',
+		'Fast Fusion',
+		'Bebob',
+		'Latin',
+		'Revival',
+		'Celtic',
+		'Bluegrass',
+		'Avantgarde',
+		'Gothic Rock',
+		'Progressive Rock',
+		'Psychedelic Rock',
+		'Symphonic Rock',
+		'Slow Rock',
+		'Big Band',
+		'Chorus',
+		'Easy Listening',
+		'Acoustic',
+		'Humour',
+		'Speech',
+		'Chanson',
+		'Opera',
+		'Chamber Music',
+		'Sonata',
+		'Symphony',
+		'Booty Bass',
+		'Primus',
+		'Porn Groove',
+		'Satire',
+		'Slow Jam',
+		'Club',
+		'Tango',
+		'Samba',
+		'Folklore',
+		'Ballad',
+		'Power Ballad',
+		'Rhythmic Soul',
+		'Freestyle',
+		'Duet',
+		'Punk Rock',
+		'Drum Solo',
+		'Acapella',
+		'Euro-House',
+		'Dance Hall',
+		'Goa',
+		'Drum & Bass',
+		'Club-House',
+		'Hardcore',
+		'Terror',
+		'Indie',
+		'BritPop',
+		'Negerpunk',
+		'Polsk Punk',
+		'Beat',
+		'Christian Gangsta Rap',
+		'Heavy Metal',
+		'Black Metal',
+		'Crossover',
+		'Contemporary Christian',
+		'Christian Rock',
+		'Merengue',
+		'Salsa',
+		'Thrash Metal',
+		'Anime',
+		'JPop',
+		'Synthpop',
+	);
+
+	@t_bitrate = ([
+		[0, 32, 48, 56,  64,  80,  96, 112, 128, 144, 160, 176, 192, 224, 256],
+		[0,  8, 16, 24,  32,  40,  48,  56,  64,  80,  96, 112, 128, 144, 160],
+		[0,  8, 16, 24,  32,  40,  48,  56,  64,  80,  96, 112, 128, 144, 160]
+	],[
+		[0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448],
+		[0, 32, 48, 56,  64,  80,  96, 112, 128, 160, 192, 224, 256, 320, 384],
+		[0, 32, 40, 48,  56,  64,  80,  96, 112, 128, 160, 192, 224, 256, 320]
+	]);
+
+	@t_sampling_freq = (
+		[11025, 12000,  8000],
+		[undef, undef, undef],	# reserved
+		[22050, 24000, 16000],
+		[44100, 48000, 32000]
+	);
+
+	@frequency_tbl = map { $_ ? eval "${_}e-3" : 0 }
+		map { @$_ } @t_sampling_freq;
+
+	@mp3_info_fields = qw(
+		VERSION
+		LAYER
+		STEREO
+		VBR
+		BITRATE
+		FREQUENCY
+		SIZE
+		SECS
+		MM
+		SS
+		MS
+		TIME
+		COPYRIGHT
+		PADDING
+		MODE
+		FRAMES
+		FRAME_LENGTH
+		VBR_SCALE
+	);
+
+	%v1_tag_fields =
+		(TITLE => 30, ARTIST => 30, ALBUM => 30, COMMENT => 30, YEAR => 4);
+
+	@v1_tag_names = qw(TITLE ARTIST ALBUM YEAR COMMENT TRACKNUM GENRE);
+
+	%v2_to_v1_names = (
+		# v2.2 tags
+		'TT2' => 'TITLE',
+		'TP1' => 'ARTIST',
+		'TAL' => 'ALBUM',
+		'TYE' => 'YEAR',
+		'COM' => 'COMMENT',
+		'TRK' => 'TRACKNUM',
+		'TCO' => 'GENRE', # not clean mapping, but ...
+		# v2.3 tags
+		'TIT2' => 'TITLE',
+		'TPE1' => 'ARTIST',
+		'TALB' => 'ALBUM',
+		'TYER' => 'YEAR',
+		'COMM' => 'COMMENT',
+		'TRCK' => 'TRACKNUM',
+		'TCON' => 'GENRE',
+	);
+
+	%v2_tag_names = (
+		# v2.2 tags
+		'BUF' => 'Recommended buffer size',
+		'CNT' => 'Play counter',
+		'COM' => 'Comments',
+		'CRA' => 'Audio encryption',
+		'CRM' => 'Encrypted meta frame',
+		'ETC' => 'Event timing codes',
+		'EQU' => 'Equalization',
+		'GEO' => 'General encapsulated object',
+		'IPL' => 'Involved people list',
+		'LNK' => 'Linked information',
+		'MCI' => 'Music CD Identifier',
+		'MLL' => 'MPEG location lookup table',
+		'PIC' => 'Attached picture',
+		'POP' => 'Popularimeter',
+		'REV' => 'Reverb',
+		'RVA' => 'Relative volume adjustment',
+		'SLT' => 'Synchronized lyric/text',
+		'STC' => 'Synced tempo codes',
+		'TAL' => 'Album/Movie/Show title',
+		'TBP' => 'BPM (Beats Per Minute)',
+		'TCM' => 'Composer',
+		'TCO' => 'Content type',
+		'TCR' => 'Copyright message',
+		'TDA' => 'Date',
+		'TDY' => 'Playlist delay',
+		'TEN' => 'Encoded by',
+		'TFT' => 'File type',
+		'TIM' => 'Time',
+		'TKE' => 'Initial key',
+		'TLA' => 'Language(s)',
+		'TLE' => 'Length',
+		'TMT' => 'Media type',
+		'TOA' => 'Original artist(s)/performer(s)',
+		'TOF' => 'Original filename',
+		'TOL' => 'Original Lyricist(s)/text writer(s)',
+		'TOR' => 'Original release year',
+		'TOT' => 'Original album/Movie/Show title',
+		'TP1' => 'Lead artist(s)/Lead performer(s)/Soloist(s)/Performing group',
+		'TP2' => 'Band/Orchestra/Accompaniment',
+		'TP3' => 'Conductor/Performer refinement',
+		'TP4' => 'Interpreted, remixed, or otherwise modified by',
+		'TPA' => 'Part of a set',
+		'TPB' => 'Publisher',
+		'TRC' => 'ISRC (International Standard Recording Code)',
+		'TRD' => 'Recording dates',
+		'TRK' => 'Track number/Position in set',
+		'TSI' => 'Size',
+		'TSS' => 'Software/hardware and settings used for encoding',
+		'TT1' => 'Content group description',
+		'TT2' => 'Title/Songname/Content description',
+		'TT3' => 'Subtitle/Description refinement',
+		'TXT' => 'Lyricist/text writer',
+		'TXX' => 'User defined text information frame',
+		'TYE' => 'Year',
+		'UFI' => 'Unique file identifier',
+		'ULT' => 'Unsychronized lyric/text transcription',
+		'WAF' => 'Official audio file webpage',
+		'WAR' => 'Official artist/performer webpage',
+		'WAS' => 'Official audio source webpage',
+		'WCM' => 'Commercial information',
+		'WCP' => 'Copyright/Legal information',
+		'WPB' => 'Publishers official webpage',
+		'WXX' => 'User defined URL link frame',
+
+		# v2.3 tags
+		'AENC' => 'Audio encryption',
+		'APIC' => 'Attached picture',
+		'COMM' => 'Comments',
+		'COMR' => 'Commercial frame',
+		'ENCR' => 'Encryption method registration',
+		'EQUA' => 'Equalization',
+		'ETCO' => 'Event timing codes',
+		'GEOB' => 'General encapsulated object',
+		'GRID' => 'Group identification registration',
+		'IPLS' => 'Involved people list',
+		'LINK' => 'Linked information',
+		'MCDI' => 'Music CD identifier',
+		'MLLT' => 'MPEG location lookup table',
+		'OWNE' => 'Ownership frame',
+		'PCNT' => 'Play counter',
+		'POPM' => 'Popularimeter',
+		'POSS' => 'Position synchronisation frame',
+		'PRIV' => 'Private frame',
+		'RBUF' => 'Recommended buffer size',
+		'RVAD' => 'Relative volume adjustment',
+		'RVRB' => 'Reverb',
+		'SYLT' => 'Synchronized lyric/text',
+		'SYTC' => 'Synchronized tempo codes',
+		'TALB' => 'Album/Movie/Show title',
+		'TBPM' => 'BPM (beats per minute)',
+		'TCOM' => 'Composer',
+		'TCON' => 'Content type',
+		'TCOP' => 'Copyright message',
+		'TDAT' => 'Date',
+		'TDLY' => 'Playlist delay',
+		'TENC' => 'Encoded by',
+		'TEXT' => 'Lyricist/Text writer',
+		'TFLT' => 'File type',
+		'TIME' => 'Time',
+		'TIT1' => 'Content group description',
+		'TIT2' => 'Title/songname/content description',
+		'TIT3' => 'Subtitle/Description refinement',
+		'TKEY' => 'Initial key',
+		'TLAN' => 'Language(s)',
+		'TLEN' => 'Length',
+		'TMED' => 'Media type',
+		'TOAL' => 'Original album/movie/show title',
+		'TOFN' => 'Original filename',
+		'TOLY' => 'Original lyricist(s)/text writer(s)',
+		'TOPE' => 'Original artist(s)/performer(s)',
+		'TORY' => 'Original release year',
+		'TOWN' => 'File owner/licensee',
+		'TPE1' => 'Lead performer(s)/Soloist(s)',
+		'TPE2' => 'Band/orchestra/accompaniment',
+		'TPE3' => 'Conductor/performer refinement',
+		'TPE4' => 'Interpreted, remixed, or otherwise modified by',
+		'TPOS' => 'Part of a set',
+		'TPUB' => 'Publisher',
+		'TRCK' => 'Track number/Position in set',
+		'TRDA' => 'Recording dates',
+		'TRSN' => 'Internet radio station name',
+		'TRSO' => 'Internet radio station owner',
+		'TSIZ' => 'Size',
+		'TSRC' => 'ISRC (international standard recording code)',
+		'TSSE' => 'Software/Hardware and settings used for encoding',
+		'TXXX' => 'User defined text information frame',
+		'TYER' => 'Year',
+		'UFID' => 'Unique file identifier',
+		'USER' => 'Terms of use',
+		'USLT' => 'Unsychronized lyric/text transcription',
+		'WCOM' => 'Commercial information',
+		'WCOP' => 'Copyright/Legal information',
+		'WOAF' => 'Official audio file webpage',
+		'WOAR' => 'Official artist/performer webpage',
+		'WOAS' => 'Official audio source webpage',
+		'WORS' => 'Official internet radio station homepage',
+		'WPAY' => 'Payment',
+		'WPUB' => 'Publishers official webpage',
+		'WXXX' => 'User defined URL link frame',
+
+		# v2.4 additional tags
+		# note that we don't restrict tags from 2.3 or 2.4,
+		'ASPI' => 'Audio seek point index',
+		'EQU2' => 'Equalisation (2)',
+		'RVA2' => 'Relative volume adjustment (2)',
+		'SEEK' => 'Seek frame',
+		'SIGN' => 'Signature frame',
+		'TDEN' => 'Encoding time',
+		'TDOR' => 'Original release time',
+		'TDRC' => 'Recording time',
+		'TDRL' => 'Release time',
+		'TDTG' => 'Tagging time',
+		'TIPL' => 'Involved people list',
+		'TMCL' => 'Musician credits list',
+		'TMOO' => 'Mood',
+		'TPRO' => 'Produced notice',
+		'TSOA' => 'Album sort order',
+		'TSOP' => 'Performer sort order',
+		'TSOT' => 'Title sort order',
+		'TSST' => 'Set subtitle',
+
+		# grrrrrrr
+		'COM ' => 'Broken iTunes comments',
+	);
+}
+
+1;
+
+__END__
+
+=pod
+
+=back
+
+=head1 TROUBLESHOOTING
+
+If you find a bug, please send me a patch (see the project page in L<"SEE ALSO">).
+If you cannot figure out why it does not work for you, please put the MP3 file in
+a place where I can get it (preferably via FTP, or HTTP, or .Mac iDisk) and send me
+mail regarding where I can get the file, with a detailed description of the problem.
+
+If I download the file, after debugging the problem I will not keep the MP3 file
+if it is not legal for me to have it.  Just let me know if it is legal for me to
+keep it or not.
+
+
+=head1 TODO
+
+=over 4
+
+=item ID3v2 Support
+
+Still need to do more for reading tags, such as using Compress::Zlib to decompress
+compressed tags.  But until I see this in use more, I won't bother.  If something
+does not work properly with reading, follow the instructions above for
+troubleshooting.
+
+ID3v2 I<writing> is coming soon.
+
+=item Get data from scalar
+
+Instead of passing a file spec or filehandle, pass the
+data itself.  Would take some work, converting the seeks, etc.
+
+=item Padding bit ?
+
+Do something with padding bit.
+
+=item Test suite
+
+Test suite could use a bit of an overhaul and update.  Patches very welcome.
+
+=over 4
+
+=item *
+
+Revamp getset.t.  Test all the various get_mp3tag args.
+
+=item *
+
+Test Unicode.
+
+=item *
+
+Test OOP API.
+
+=item *
+
+Test error handling, check more for missing files, bad MP3s, etc.
+
+=back
+
+=item Other VBR
+
+Right now, only Xing VBR is supported.
+
+=back
+
+
+=head1 THANKS
+
+Edward Allen E<lt>allenej@c51844-a.spokn1.wa.home.comE<gt>,
+Vittorio Bertola E<lt>v.bertola@vitaminic.comE<gt>,
+Michael Blakeley E<lt>mike@blakeley.comE<gt>,
+Per Bolmstedt E<lt>tomten@kol14.comE<gt>,
+Tony Bowden E<lt>tony@tmtm.comE<gt>,
+Tom Brown E<lt>thecap@usa.netE<gt>,
+Sergio Camarena E<lt>scamarena@users.sourceforge.netE<gt>,
+Chris Dawson E<lt>cdawson@webiphany.comE<gt>,
+Luke Drumm E<lt>lukedrumm@mypad.comE<gt>,
+Kyle Farrell E<lt>kyle@cantametrix.comE<gt>,
+Jeffrey Friedl E<lt>jfriedl@yahoo.comE<gt>,
+brian d foy E<lt>comdog@panix.comE<gt>,
+Ben Gertzfield E<lt>che@debian.orgE<gt>,
+Brian Goodwin E<lt>brian@fuddmain.comE<gt>,
+Todd Hanneken E<lt>thanneken@hds.harvard.eduE<gt>,
+Todd Harris E<lt>harris@cshl.orgE<gt>,
+Woodrow Hill E<lt>asim@mindspring.comE<gt>,
+Kee Hinckley E<lt>nazgul@somewhere.comE<gt>,
+Roman Hodek E<lt>Roman.Hodek@informatik.uni-erlangen.deE<gt>,
+Peter Kovacs E<lt>kovacsp@egr.uri.eduE<gt>,
+Johann Lindvall,
+Peter Marschall E<lt>peter.marschall@mayn.deE<gt>,
+Trond Michelsen E<lt>mike@crusaders.noE<gt>,
+Dave O'Neill E<lt>dave@nexus.carleton.caE<gt>,
+Christoph Oberauer E<lt>christoph.oberauer@sbg.ac.atE<gt>,
+Jake Palmer E<lt>jake.palmer@db.comE<gt>,
+Andrew Phillips E<lt>asp@wasteland.orgE<gt>,
+David Reuteler E<lt>reuteler@visi.comE<gt>,
+John Ruttenberg E<lt>rutt@chezrutt.comE<gt>,
+Matthew Sachs E<lt>matthewg@zevils.comE<gt>,
+E<lt>scfc_de@users.sf.netE<gt>,
+Hermann Schwaerzler E<lt>Hermann.Schwaerzler@uibk.ac.atE<gt>,
+Chris Sidi E<lt>sidi@angband.orgE<gt>,
+Roland Steinbach E<lt>roland@support-system.comE<gt>,
+Stuart E<lt>schneis@users.sourceforge.netE<gt>,
+Jeffery Sumler E<lt>jsumler@mediaone.netE<gt>,
+Predrag Supurovic E<lt>mpgtools@dv.co.yuE<gt>,
+Bogdan Surdu E<lt>tim@go.roE<gt>,
+E<lt>tim@tim-landscheidt.deE<gt>,
+Pass F. B. Travis E<lt>pftravis@bellsouth.netE<gt>,
+Tobias Wagener E<lt>tobias@wagener.nuE<gt>,
+Ronan Waide E<lt>waider@stepstone.ieE<gt>,
+Andy Waite E<lt>andy@mailroute.comE<gt>,
+Ken Williams E<lt>ken@forum.swarthmore.eduE<gt>,
+Meng Weng Wong E<lt>mengwong@pobox.comE<gt>.
+
+
+=head1 AUTHOR AND COPYRIGHT
+
+Chris Nandor E<lt>pudge@pobox.comE<gt>, http://pudge.net/
+
+Copyright (c) 1998-2003 Chris Nandor.  All rights reserved.  This program is
+free software; you can redistribute it and/or modify it under the terms
+of the Artistic License, distributed with Perl.
+
+
+=head1 SEE ALSO
+
+=over 4
+
+=item MP3::Info Project Page
+
+	http://projects.pudge.net/
+
+=item mp3tools
+
+	http://www.zevils.com/linux/mp3tools/
+
+=item mpgtools
+
+	http://www.dv.co.yu/mpgscript/mpgtools.htm
+	http://www.dv.co.yu/mpgscript/mpeghdr.htm
+
+=item mp3tool
+
+	http://www.dtek.chalmers.se/~d2linjo/mp3/mp3tool.html
+
+=item ID3v2
+
+	http://www.id3.org/
+
+=item Xing Variable Bitrate
+
+	http://www.xingtech.com/support/partner_developer/mp3/vbr_sdk/
+
+=item MP3Ext
+
+	http://rupert.informatik.uni-stuttgart.de/~mutschml/MP3ext/
+
+=item Xmms
+
+	http://www.xmms.org/
+
+
+=back
+
+=head1 VERSION
+
+v1.02, Sunday, March 2, 2003
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/Mail/Sendmail.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/Mail/Sendmail.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/Mail/Sendmail.pm	(revision 21112)
@@ -0,0 +1,753 @@
+package Mail::Sendmail;
+# Mail::Sendmail by Milivoj Ivkovic <mi@alma.ch>
+# see embedded POD documentation after __END__
+# or http://alma.ch/perl/mail.htm
+
+=head1 NAME
+
+Mail::Sendmail v. 0.78 - Simple platform independent mailer
+
+=cut
+
+$VERSION = '0.78';
+
+# *************** Configuration you may want to change *******************
+# You probably want to set your SMTP server here (unless you specify it in
+# every script), and leave the rest as is. See pod documentation for details
+
+%mailcfg = (
+    # List of SMTP servers:
+    'smtp'    => [],
+
+    'from'    => '', # default sender e-mail, used when no From header in mail
+
+    'mime'    => 1, # use MIME encoding by default
+
+    'retries' => 1, # number of retries on smtp connect failure
+    'delay'   => 1, # delay in seconds between retries
+
+    'tz'      => '', # only to override automatic detection
+    'port'    => 25, # change it if you always use a non-standard port
+    'debug'   => 0 # prints stuff to STDERR
+);
+
+# *******************************************************************
+
+require Exporter;
+use strict;
+use vars qw(
+            $VERSION
+            @ISA
+            @EXPORT
+            @EXPORT_OK
+            %mailcfg
+            $default_smtp_server
+            $default_smtp_port
+            $default_sender
+            $TZ
+            $use_MIME
+            $address_rx
+            $debug
+            $log
+            $error
+            $retry_delay
+            $connect_retries
+           );
+
+use Socket;
+use Time::Local; # for automatic time zone detection
+
+# use MIME::QuotedPrint if available and configured in %mailcfg
+eval("use MIME::QuotedPrint");
+$mailcfg{'mime'} &&= (!$@);
+
+@ISA        = qw(Exporter);
+@EXPORT     = qw(&sendmail);
+@EXPORT_OK  = qw(
+                 %mailcfg
+                 time_to_date
+                 $default_smtp_server
+                 $default_smtp_port
+                 $default_sender
+                 $TZ
+                 $address_rx
+                 $debug
+                 $log
+                 $error
+                );
+
+# regex for e-mail addresses where full=$1, user=$2, domain=$3
+# see pod documentation about this regex
+
+my $word_rx = '[\x21\x23-\x27\x2A-\x2B\x2D\w\x3D\x3F]+';
+my $user_rx = $word_rx         # valid chars
+             .'(?:\.' . $word_rx . ')*' # possibly more words preceded by a dot
+             ;
+my $dom_rx = '\w[-\w]*(?:\.\w[-\w]*)*'; # less valid chars in domain names
+my $ip_rx = '\[\d{1,3}(?:\.\d{1,3}){3}\]';
+
+$address_rx = '((' . $user_rx . ')\@(' . $dom_rx . '|' . $ip_rx . '))';
+; # v. 0.6
+
+sub time_to_date {
+    # convert a time() value to a date-time string according to RFC 822
+
+    my $time = $_[0] || time(); # default to now if no argument
+
+    my @months = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
+    my @wdays  = qw(Sun Mon Tue Wed Thu Fri Sat);
+
+    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)
+        = localtime($time);
+
+    $TZ ||= $mailcfg{'tz'};
+
+    if ( $TZ eq "" ) {
+        # offset in hours
+        my $offset  = sprintf "%.1f", (timegm(localtime) - time) / 3600;
+        my $minutes = sprintf "%02d", ( $offset - int($offset) ) * 60;
+        $TZ  = sprintf("%+03d", int($offset)) . $minutes;
+    }
+    return join(" ",
+                    ($wdays[$wday] . ','),
+                     $mday,
+                     $months[$mon],
+                     $year+1900,
+                     sprintf("%02d", $hour) . ":" . sprintf("%02d", $min),
+                     $TZ
+               );
+} # end sub time_to_date
+
+sub sendmail {
+    # original sendmail 1.21 by Christian Mallwitz.
+    # Modified and 'modulized' by mi@alma.ch
+
+    $error = '';
+    $log = "Mail::Sendmail v. $VERSION - "    . scalar(localtime()) . "\n";
+
+    local $_;
+    local $/ = "\015\012";
+
+    my (%mail, $k,
+        $smtp, $server, $port, $connected, $localhost,
+        $message, $fromaddr, $recip, @recipients, $to, $header,
+       );
+
+    sub fail {
+        # things to do before returning a sendmail failure
+        print STDERR @_ if $^W;
+        $error .= join(" ", @_) . "\n";
+        close S;
+        return 0;
+    }
+
+    # all config keys to lowercase, to prevent typo errors
+    foreach $k (keys %mailcfg) {
+        if ($k =~ /[A-Z]/) {
+            $mailcfg{lc($k)} = $mailcfg{$k};
+        }
+    }
+
+    # redo hash, arranging keys case etc...
+    while (@_) {
+        # arrange keys case
+        $k = ucfirst lc(shift @_);
+
+        if (!$k and $^W) {
+            warn "Received false mail hash key: \'$k\'. Did you forget to put it in quotes?\n";
+        }
+
+        $k =~ s/\s*:\s*$//o; # kill colon (and possible spaces) at end, we add it later.
+        $mail{$k} = shift @_;
+    }
+
+    $smtp = $mail{'Smtp'} || $mail{'Server'} || $default_smtp_server;
+#    unshift @{$mailcfg{'smtp'}}, $smtp if ($smtp and $mailcfg{'smtp'}->[0] ne $smtp);
+    unshift @{$mailcfg{'smtp'}}, $smtp if $smtp; # Change here to avoid undefined value
+                                                 # with our empty mailcfg{'smtp'} - Stefan
+
+    # delete non-header keys, so we don't send them later as mail headers
+    # I like this syntax, but it doesn't seem to work with AS port 5.003_07:
+    # delete @mail{'Smtp', 'Server'};
+    # so instead:
+    delete $mail{'Smtp'}; delete $mail{'Server'};
+
+    $mailcfg{'port'} = $mail{'Port'} || $default_smtp_port || $mailcfg{'port'} || 25;
+    delete $mail{'Port'};
+
+    # for backward compatibility only
+    $mailcfg{'retries'} = $connect_retries if defined($connect_retries);
+    $mailcfg{'delay'} = $retry_delay if defined($retry_delay);
+
+    {    # don't warn for undefined values below
+        local $^W = 0;
+        $message = join("", $mail{'Message'}, $mail{'Body'}, $mail{'Text'});
+    }
+
+    # delete @mail{'Message', 'Body', 'Text'};
+    delete $mail{'Message'}; delete $mail{'Body'}; delete $mail{'Text'};
+
+    # Extract 'From:' e-mail address
+
+    $fromaddr = $mail{'From'} || $default_sender || $mailcfg{'from'};
+    unless ($fromaddr =~ /$address_rx/) {
+        return fail("Bad or missing From address: \'$fromaddr\'");
+    }
+    $fromaddr = $1;
+
+    # add Date header if needed
+    $mail{Date} ||= time_to_date() ;
+    $log .= "Date: $mail{Date}\n";
+
+    # cleanup message, and encode if needed
+    $message =~ s/^\./\.\./gom;     # handle . as first character
+    $message =~ s/\r\n/\n/go;     # normalize line endings, step 1 of 2 (next step after MIME encoding)
+
+    $mail{'Mime-version'} ||= '1.0';
+    $mail{'Content-type'} ||= 'text/plain; charset="iso-8859-1"';
+
+    unless ( $mail{'Content-transfer-encoding'}
+          || $mail{'Content-type'} =~ /multipart/io )
+    {
+        if ($mailcfg{'mime'}) {
+            $mail{'Content-transfer-encoding'} = 'quoted-printable';
+            $message = encode_qp($message);
+        }
+        else {
+            $mail{'Content-transfer-encoding'} = '8bit';
+            if ($message =~ /[\x80-\xFF]/o) {
+                $error .= "MIME::QuotedPrint not present!\nSending 8bit characters, hoping it will come across OK.\n";
+                warn "MIME::QuotedPrint not present!\n",
+                     "Sending 8bit characters, hoping it will come across OK.\n"
+                     if $^W;
+            }
+        }
+    }
+
+    $message =~ s/\n/\015\012/go; # normalize line endings, step 2.
+
+    # Get recipients
+    {    # don't warn for undefined values below
+        local $^W = 0;
+        $recip = join(", ", $mail{To}, $mail{Cc}, $mail{Bcc});
+    }
+
+    delete $mail{'Bcc'};
+
+    @recipients = ();
+    while ($recip =~ /$address_rx/go) {
+        push @recipients, $1;
+    }
+    unless (@recipients) {
+        return fail("No recipient!")
+    }
+
+    # get local hostname for polite HELO
+    $localhost = (gethostbyname('localhost'))[0] || 'localhost';
+
+    foreach $server ( @{$mailcfg{'smtp'}} ) {
+        # open socket needs to be inside this foreach loop on Linux,
+        # otherwise all servers fail if 1st one fails !??! why?
+        unless ( socket S, AF_INET, SOCK_STREAM, (getprotobyname 'tcp')[2] ) {
+            return fail("socket failed ($!)")
+        }
+
+        print "- trying $server\n" if $mailcfg{'debug'} > 1;
+
+        $server =~ s/\s+//go; # remove spaces just in case of a typo
+        # extract port if server name like "mail.domain.com:2525"
+        ($server =~ s/:(.+)$//o) ? $port = $1    : $port = $mailcfg{'port'};
+        $smtp = $server; # save $server for use outside foreach loop
+
+        my $smtpaddr = inet_aton $server;
+        unless ($smtpaddr) {
+            $error .= "$server not found\n";
+            next; # next server
+        }
+
+        my $retried = 0; # reset retries for each server
+        while ( ( not $connected = connect S, pack_sockaddr_in($port, $smtpaddr) )
+            and ( $retried < $mailcfg{'retries'} )
+              ) {
+            $retried++;
+            $error .= "connect to $server failed ($!)\n";
+            print "- connect to $server failed ($!)\n" if $mailcfg{'debug'} > 1;
+            print "retrying in $mailcfg{'delay'} seconds...\n";
+            sleep $mailcfg{'delay'};
+        }
+
+        if ( $connected ) {
+            print "- connected to $server\n" if $mailcfg{'debug'} > 3;
+            last;
+        }
+        else {
+            $error .= "connect to $server failed\n";
+            print "- connect to $server failed, next server...\n" if $mailcfg{'debug'} > 1;
+            next; # next server
+        }
+    }
+
+    unless ( $connected ) {
+        return fail("connect to $smtp failed ($!) no (more) retries!")
+    };
+
+    {
+        local $^W = 0; # don't warn on undefined variables
+        # Add info to log variable
+        $log .= "Server: $smtp Port: $port\n"
+              . "From: $fromaddr\n"
+              . "Subject: $mail{Subject}\n"
+              . "To: ";
+    }
+
+    my($oldfh) = select(S); $| = 1; select($oldfh);
+
+    chomp($_ = <S>);
+    if (/^[45]/ or !$_) {
+        return fail("Connection error from $smtp on port $port ($_)")
+    }
+
+    print S "HELO $localhost\015\012";
+    chomp($_ = <S>);
+    if (/^[45]/ or !$_) {
+        return fail("HELO error ($_)")
+    }
+    
+    print S "mail from: <$fromaddr>\015\012";
+    chomp($_ = <S>);
+    if (/^[45]/ or !$_) {
+        return fail("mail From: error ($_)")
+    }
+
+    foreach $to (@recipients) {
+        if ($debug) { print STDERR "sending to: <$to>\n"; }
+        print S "rcpt to: <$to>\015\012";
+        chomp($_ = <S>);
+        if (/^[45]/ or !$_) {
+            $log .= "!Failed: $to\n    ";
+            return fail("Error sending to <$to> ($_)\n");
+        }
+        else {
+            $log .= "$to\n    ";
+        }
+    }
+
+    # start data part
+    print S "data\015\012";
+    chomp($_ = <S>);
+    if (/^[45]/ or !$_) {
+           return fail("Cannot send data ($_)");
+    }
+
+    # print headers
+    foreach $header (keys %mail) {
+        $mail{$header} =~ s/\s+$//o; # kill possible trailing garbage
+        print S "$header: ", $mail{$header}, "\015\012";
+    };
+
+    #- test diconnecting from network here, to see what happens
+    #- print STDERR "DISCONNECT NOW!\n";
+    #- sleep 4;
+    #- print STDERR "trying to continue, expecting an error... \n";
+
+    # send message body
+    print S "\015\012",
+            $message,
+            "\015\012.\015\012";
+
+    chomp($_ = <S>);
+    if (/^[45]/ or !$_) {
+           return fail("message transmission failed ($_)");
+    }
+
+    # finish
+    print S "quit\015\012";
+    $_ = <S>;
+    close S;
+
+    return 1;
+} # end sub sendmail
+
+1;
+__END__
+
+=head1 SYNOPSIS
+
+  use Mail::Sendmail;
+
+  %mail = ( To      => 'you@there.com',
+            From    => 'me@here.com',
+            Message => "This is a very short message"
+           );
+
+  sendmail(%mail) or die $Mail::Sendmail::error;
+
+  print "OK. Log says:\n", $Mail::Sendmail::log;
+
+=head1 DESCRIPTION
+
+Simple platform independent e-mail from your perl script. Only requires
+Perl 5 and a network connection.
+
+After struggling for some time with various command-line mailing programs
+which never did exactly what I wanted, I put together this Perl only
+solution.
+
+Mail::Sendmail contains mainly &sendmail, which takes a hash with the
+message to send and sends it. It is intended to be very easy to setup and
+use.
+
+=head1 INSTALLATION
+
+=over 4
+
+=item Best
+
+perl -MCPAN -e "install Mail::Sendmail"
+
+=item Traditional
+
+    perl Makefile.PL
+    make
+    make test
+    make install
+
+=item Manual
+
+Copy Sendmail.pm to Mail/ in your Perl lib directory.
+
+    (eg. c:\Perl\lib\Mail\, c:\Perl\site\lib\Mail\,
+     /usr/lib/perl5/site_perl/Mail/, ... or whatever it
+     is on your system)
+
+=item ActivePerl's PPM
+
+ppm install --location=http://alma.ch/perl/ppm Mail-Sendmail
+
+But this way you don't get a chance to have a look at other files (Changes, 
+Todo, test.pl, ...) and PPM doesn't run the test script (test.pl).
+
+=back
+
+At the top of Sendmail.pm, set your default SMTP server, unless you specify
+it with each message, or want to use the default.
+
+Install MIME::QuotedPrint. This is not required but strongly recommended.
+
+=head1 FEATURES
+
+Automatic time zone detection, Date: header, MIME quoted-printable encoding 
+(if MIME::QuotedPrint installed), all of which can be overridden.
+
+Internal Bcc: and Cc: support (even on broken servers)
+
+Allows real names in From: and To: fields
+
+Doesn't send unwanted headers, and allows you to send any header(s) you
+want
+
+Configurable retries and use of alternate servers if your mail server is
+down
+
+Good plain text error reporting
+
+=head1 LIMITATIONS
+
+Doesn't work on OpenVMS.
+
+Headers are not encoded, even if they have accented characters.
+
+Since the whole message is in memory (twice!), it's not suitable for 
+sending very big attached files.
+
+The SMTP server has to be set manually in Sendmail.pm or in your script,
+unless you have a mail server on localhost.
+
+=head1 CONFIGURATION
+
+=over 4
+
+=item Default SMTP server(s)
+
+This is probably all you want to configure. It is usually done through
+I<$mailcfg{smtp}>, which you can edit at the top of the Sendmail.pm file.
+This is a reference to a list of SMTP servers. You can also set it from
+your script:
+
+C<unshift @{$Mail::Sendmail::mailcfg{'smtp'}} , 'my.mail.server';>
+
+Alternatively, you can specify the server in the I<%mail> hash you send
+from your script, which will do the same thing:
+
+C<$mail{smtp} = 'my.mail.server';>
+
+A future version will try to set useful defaults for you during the
+Makefile.PL.
+
+=item Other configuration settings
+
+See I<%mailcfg> under L<"DETAILS"> below for other configuration options.
+
+=back
+
+=head1 DETAILS
+
+=head2 sendmail()
+
+sendmail is the only thing exported to your namespace by default
+
+C<sendmail(%mail) || print "Error sending mail: $Mail::Sendmail::error\n";>
+
+It takes a hash containing the full message, with keys for all headers,
+body, and optionally for another non-default SMTP server and/or port.
+
+It returns 1 on success or 0 on error, and rewrites
+C<$Mail::Sendmail::error> and C<$Mail::Sendmail::log>.
+
+Keys are NOT case-sensitive.
+
+The colon after headers is not necessary.
+
+The Body part key can be called 'Body', 'Message' or 'Text'. The SMTP
+server key can be called 'Smtp' or 'Server'.
+
+The following headers are added unless you specify them yourself:
+
+    Mime-version: 1.0
+    Content-type: 'text/plain; charset="iso-8859-1"'
+
+    Content-transfer-encoding: quoted-printable
+    or (if MIME::QuotedPrint not installed)
+    Content-transfer-encoding: 8bit
+
+    Date: [string returned by time_to_date()]
+
+The following are not exported by default, but you can still access them
+with their full name, or request their export on the use line like in:
+C<use Mail::Sendmail qw($address_rx time_to_date);>
+
+=head2 Mail::Sendmail::time_to_date()
+
+convert time ( as from C<time()> ) to an RFC 822 compliant string for the
+Date header. See also L<"%Mail::Sendmail::mailcfg">.
+
+=head2 $Mail::Sendmail::error
+
+When you don't run with the B<-w> flag, the module sends no errors to
+STDERR, but puts anything it has to complain about in here. You should
+probably always check if it says something.
+
+=head2 $Mail::Sendmail::log
+
+A summary that you could write to a log file after each send
+
+=head2 $Mail::Sendmail::address_rx
+
+A handy regex to recognize e-mail addresses.
+
+A correct regex for valid e-mail addresses was written by one of the judges
+in the obfuscated Perl contest... :-) It is quite big. This one is an
+attempt to a reasonable compromise, and should accept all real-world
+internet style addresses. The domain part is required and comments or
+characters that would need to be quoted are not supported.
+
+  Example:
+    $rx = $Mail::Sendmail::address_rx;
+    if (/$rx/) {
+      $address=$1;
+      $user=$2;
+      $domain=$3;
+    }
+
+=head2 %Mail::Sendmail::mailcfg
+
+This hash contains all configuration options. You normally edit it once (if
+ever) in Sendmail.pm and forget about it, but you could also access it from
+your scripts. For readability, I'll assume you have imported it.
+
+The keys are not case-sensitive: they are all converted to lowercase before
+use. Writing C<$mailcfg{Port} = 2525;> is OK: the default $mailcfg{port}
+(25) will be deleted and replaced with your new value of 2525.
+
+=over 4
+
+=item $mailcfg{smtp}
+
+C<$mailcfg{smtp} = [qw(localhost my.other.mail.server)];>
+
+This is a reference to a list of smtp servers, so if your main server is
+down, the module tries the next one. If one of your servers uses a special
+port, add it to the server name with a colon in front, to override the
+default port (like in my.special.server:2525).
+
+Default: localhost. (the previous version also had smtp.site1.csi.com which
+was an open relay, but it isn't anymore)
+
+=item $mailcfg{from}
+
+C<$mailcfg{from} = 'Mailing script me@mydomain.com';>
+
+From address used if you don't supply one in your script. Should not be of
+type 'user@localhost' since that may not be valid on the recipient's
+host.
+
+Default: undefined.
+
+=item $mailcfg{mime}
+
+C<$mailcfg{mime} = 1;>
+
+Set this to 0 if you don't want any automatic MIME encoding. You normally
+don't need this, the module should 'Do the right thing' anyway.
+
+Default: 1;
+
+=item $mailcfg{retries}
+
+C<$mailcfg{retries} = 1;>
+
+How many times should the connection to the same SMTP server be retried in
+case of a failure.
+
+Default: 1;
+
+=item $mailcfg{delay}
+
+C<$mailcfg{delay} = 1;>
+
+Number of seconds to wait between retries. This delay also happens before
+trying the next server in the list, if the retries for the current server
+have been exhausted. For CGI scripts, you want few retries and short delays
+to return with a results page before the http connection times out. For
+unattended scripts, you may want to use many retries and long delays to
+have a good chance of your mail being sent even with temporary failures on
+your network.
+
+Default: 1 (second);
+
+=item $mailcfg{tz}
+
+C<$mailcfg{tz} = '+0800';>
+
+Normally, your time zone is set automatically, from the difference between
+C<time()> and C<gmtime()>. This allows you to override automatic detection
+in cases where your system is confused (such as some Win32 systems in zones
+which do not use daylight savings time: see Microsoft KB article Q148681)
+
+Default: undefined (automatic detection at run-time).
+
+=item $mailcfg{port}
+
+C<$mailcfg{port} = 25;>
+
+Port used when none is specified in the server name.
+
+Default: 25.
+
+=item $mailcfg{debug}
+
+C<$mailcfg{debug} => 0;>
+
+Prints stuff to STDERR. Not used much, and what is printed may change
+without notice. Don't count on it.
+
+Default: 0;
+
+=back
+
+=head2 $Mail::Sendmail::VERSION
+
+The package version number (you can not import this one)
+
+=head2 Configuration variables from previous versions
+
+The following global variables were used in version 0.74 for configuration. 
+They should still work, but will not in a future version (unless you 
+complain loudly). Please use I<%mailcfg> if you need to access the 
+configuration from your scripts.
+
+=over 4
+
+=item $Mail::Sendmail::default_smtp_server
+
+=item $Mail::Sendmail::default_smtp_port
+
+=item $Mail::Sendmail::default_sender
+
+=item $Mail::Sendmail::TZ
+
+=item $Mail::Sendmail::connect_retries
+
+=item $Mail::Sendmail::retry_delay
+
+=item $Mail::Sendmail::use_MIME
+
+This one couldn't really be used in the previous version, so I just dropped it.
+It is replaced by I<$mailcfg{mime}> which works.
+
+=back
+
+=head1 ANOTHER EXAMPLE
+
+  use Mail::Sendmail;
+
+  print "Testing Mail::Sendmail version $Mail::Sendmail::VERSION\n";
+  print "Default server: $Mail::Sendmail::mailcfg{smtp}->[0]\n";
+  print "Default sender: $Mail::Sendmail::mailcfg{from}\n";
+
+  %mail = (
+      #To      => 'No to field this time, only Bcc and Cc',
+      #From    => 'not needed, use default',
+      Bcc     => 'Someone <him@there.com>, Someone else her@there.com',
+      # only addresses are extracted from Bcc, real names disregarded
+      Cc      => 'Yet someone else <xz@whatever.com>',
+      # Cc will appear in the header. (Bcc will not)
+      Subject => 'Test message',
+      'X-Mailer' => "Mail::Sendmail version $Mail::Sendmail::VERSION",
+  );
+
+
+  $mail{Smtp} = 'special_server.for-this-message-only.domain.com';
+  $mail{'X-custom'} = 'My custom additionnal header';
+  $mail{'mESSaGE : '} = "The message key looks terrible, but works.";
+  # cheat on the date:
+  $mail{Date} = Mail::Sendmail::time_to_date( time() - 86400 ),
+
+  if (sendmail %mail) { print "Mail sent OK.\n" }
+  else { print "Error sending mail: $Mail::Sendmail::error \n" }
+
+  print "\n\$Mail::Sendmail::log says:\n", $Mail::Sendmail::log;
+
+=head1 CHANGES
+
+Single-letter host names bug fixed since version 0.77. See the F<Changes> file for
+the full history.
+
+=head1 AUTHOR
+
+Milivoj Ivkovic mi@alma.ch or ivkovic@bluewin.ch
+
+=head1 NOTES
+
+MIME::QuotedPrint is used by default on every message if available. It 
+allows reliable sending of accented characters, and also takes care of 
+too long lines (which can happen in HTML mails). It is available in the 
+MIME-Base64 package at http://www.perl.com/CPAN/modules/by-module/MIME/ or 
+through PPM.
+
+Look at http://alma.ch/perl/Mail-Sendmail-FAQ.htm for additional info 
+(CGI, examples of sending attachments, HTML mail etc...)
+
+You can use it freely. (Someone complained this is too vague. So, more
+precisely: do whatever you want with it, but be warned that terrible things
+will happen to you if you use it badly, like for sending spam, claiming you
+wrote it alone, or ...?)
+
+I would appreciate a short (or long) e-mail note if you use this (and even
+if you don't, especially if you care to say why). And of course,
+bug-reports and/or suggestions are welcome.
+
+Last revision: 25.09.2000. Latest version should be available at
+http://alma.ch/perl/mail.htm , and a few days later on CPAN.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/Ogg/Vorbis/Header/PurePerl.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/Ogg/Vorbis/Header/PurePerl.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/Ogg/Vorbis/Header/PurePerl.pm	(revision 21112)
@@ -0,0 +1,674 @@
+package Ogg::Vorbis::Header::PurePerl;
+
+use 5.005;
+use strict;
+use warnings;
+
+use Fcntl qw/SEEK_END/;
+
+our $VERSION = '0.07';
+
+sub new 
+{
+    my $class = shift;
+    my $file = shift;
+
+    return load($class, $file);
+}
+
+sub load 
+{
+    my $class    = shift;
+    my $file     = shift;
+    my $from_new = shift;
+    my %data;
+    my $self;
+
+    # there must be a better way...
+    if ($class eq 'Ogg::Vorbis::Header::PurePerl')
+    {
+	$self = bless \%data, $class;
+    }
+    else
+    {
+	$self = $class;
+    }
+
+    if ($self->{'FILE_LOADED'})
+    {
+	return $self;
+    }
+
+    $self->{'FILE_LOADED'} = 1;
+
+    # check that the file exists and is readable
+    unless ( -e $file && -r _ )
+    {
+	warn "File does not exist or cannot be read.";
+	# file does not exist, can't do anything
+	return undef;
+    }
+    # open up the file
+    open FILE, $file;
+    # make sure dos-type systems can handle it...
+    binmode FILE;
+
+    $data{'filename'} = $file;
+    $data{'fileHandle'} = \*FILE;
+
+    _init(\%data);
+    _loadInfo(\%data);
+    _loadComments(\%data);
+    _calculateTrackLength(\%data);
+
+    close FILE;
+
+    return $self;
+}
+
+sub info 
+{
+    my $self = shift;
+    my $key = shift;
+
+    # if the user did not supply a key, return the entire hash
+    unless ($key)
+    {
+	return $self->{'INFO'};
+    }
+
+    # otherwise, return the value for the given key
+    return $self->{'INFO'}{lc $key};
+}
+
+sub comment_tags 
+{
+    my $self = shift;
+
+    return @{$self->{'COMMENT_KEYS'}};
+}
+
+sub comment 
+{
+    my $self = shift;
+    my $key = shift;
+
+    # if the user supplied key does not exist, return undef
+    unless($self->{'COMMENTS'}{lc $key})
+    {
+	return undef;
+    }
+
+    return @{$self->{'COMMENTS'}{lc $key}};
+}
+
+sub add_comments 
+{
+    warn "Ogg::Vorbis::Header::PurePerl add_comments() unimplemented.";
+}
+
+sub edit_comment 
+{
+    warn "Ogg::Vorbis::Header::PurePerl edit_comment() unimplemented.";
+}
+
+sub delete_comment 
+{
+    warn "Ogg::Vorbis::Header::PurePerl delete_comment() unimplemented.";
+}
+
+sub clear_comments 
+{
+    warn "Ogg::Vorbis::Header::PurePerl clear_comments() unimplemented.";
+}
+
+sub path 
+{
+    my $self = shift;
+
+    return $self->{'fileName'};
+}
+
+sub write_vorbis
+{
+    warn "Ogg::Vorbis::Header::PurePerl write_vorbis unimplemented.";
+}
+
+# "private" methods
+
+sub _init
+{
+    my $data = shift;
+    my $fh = $data->{'fileHandle'};
+    my $byteCount = 0;
+
+    # check the header to make sure this is actually an Ogg-Vorbis file
+    $byteCount = _checkHeader($data);
+
+    unless($byteCount)
+    {
+	# if it's not, we can't do anything
+	return undef;
+    }
+
+    $data->{'startInfoHeader'} = $byteCount;
+}
+
+sub _checkHeader
+{
+    my $data = shift;
+    my $fh = $data->{'fileHandle'};
+    my $buffer;
+    my $pageSegCount;
+    my $byteCount = 0; # stores how far into the file we've read,
+                       # so later reads into the file can skip right
+                       # past all of the header stuff
+
+    # check that the first four bytes are 'OggS'
+    read($fh, $buffer, 4);
+    if ($buffer ne 'OggS')
+    {
+	warn "This is not an Ogg bitstream (no OggS header).";
+	return undef;
+    }
+    $byteCount += 4;
+
+    # check the stream structure version (1 byte, should be 0x00)
+    read($fh, $buffer, 1);
+    if (ord($buffer) != 0x00)
+    {
+	warn "This is not an Ogg bitstream (invalid structure version).";
+	return undef;
+    }
+    $byteCount += 1;
+
+    # check the header type flag 
+    # This is a bitfield, so technically we should check all of the bits
+    # that could potentially be set. However, the only value this should
+    # possibly have at the beginning of a proper Ogg-Vorbis file is 0x02,
+    # so we just check for that. If it's not that, we go on anyway, but
+    # give a warning (this behavior may (should?) be modified in the future.
+    read($fh, $buffer, 1);
+    if (ord($buffer) != 0x02)
+    {
+	warn "Invalid header type flag (trying to go ahead anyway).";
+    }
+    $byteCount += 1;
+
+    # skip to the page_segments count
+    read($fh, $buffer, 20);
+    $byteCount += 20;
+    # we do nothing with this data
+
+    # read the number of page segments
+    read($fh, $buffer, 1);
+    $pageSegCount = ord($buffer);
+    $byteCount += 1;
+
+    # read $pageSegCount bytes, then throw 'em out
+    read($fh, $buffer, $pageSegCount);
+    $byteCount += $pageSegCount;
+
+    # check packet type. Should be 0x01 (for indentification header)
+    read($fh, $buffer, 1);
+    if (ord($buffer) != 0x01)
+    {
+	warn "Wrong vorbis header type, giving up.";
+	return undef;
+    }
+    $byteCount += 1;
+
+    # check that the packet identifies itself as 'vorbis'
+    read($fh, $buffer, 6);
+    if ($buffer ne 'vorbis')
+    {
+	warn "This does not appear to be a vorbis stream, giving up.";
+	return undef;
+    }
+    $byteCount += 6;
+
+    # at this point, we assume the bitstream is valid
+    return $byteCount;
+}
+
+sub _loadInfo
+{
+    my $data = shift;
+    my $start = $data->{'startInfoHeader'};
+    my $fh = $data->{'fileHandle'};
+    my $buffer;
+    my $byteCount = $start;
+    my %info;
+    
+    seek $fh, $start, 0;
+    
+    # read the vorbis version
+    read($fh, $buffer, 4);
+    $info{'version'} = _decodeInt($buffer);
+    $byteCount += 4;
+    
+    # read the number of audio channels
+    read($fh, $buffer, 1);
+    $info{'channels'} = ord($buffer);
+    $byteCount += 1;
+
+    # read the sample rate
+    read($fh, $buffer, 4);
+    $info{'rate'} = _decodeInt($buffer);
+    $byteCount += 4;
+    
+    # read the bitrate maximum
+    read($fh, $buffer, 4);
+    $info{'bitrate_upper'} = _decodeInt($buffer);
+    $byteCount += 4;
+
+    # read the bitrate nominal
+    read($fh, $buffer, 4);
+    $info{'bitrate_nominal'} = _decodeInt($buffer);
+    $byteCount += 4;
+
+    # read the bitrate minimal
+    read($fh, $buffer, 4);
+    $info{'bitrate_lower'} = _decodeInt($buffer);
+    $byteCount += 4;
+
+    # read the blocksize_0 and blocksize_1
+    read($fh, $buffer, 1);
+    # these are each 4 bit fields, whose actual value is 2 to the power
+    # of the value of the field
+    $info{'blocksize_0'} = 2 << ((ord($buffer) & 0xF0) >> 4);
+    $info{'blocksize_1'} = 2 << (ord($buffer) & 0x0F);
+    $byteCount += 1;
+
+    # read the framing_flag
+    read($fh, $buffer, 1);
+    $info{'framing_flag'} = ord($buffer);
+    $byteCount += 1;
+
+    # bitrate_window is -1 in the current version of vorbisfile
+    $info{'bitrate_window'} = -1;
+
+    $data->{'startCommentHeader'} = $byteCount;
+
+    $data->{'INFO'} = \%info;
+}
+
+sub _loadComments
+{
+    my $data = shift;
+    my $fh = $data->{'fileHandle'};
+    my $start = $data->{'startCommentHeader'};
+    my $buffer;
+    my $page_segments;
+    my $vendor_length;
+    my $user_comment_count;
+    my $byteCount = $start;
+    my %comments;
+
+    seek $fh, $start, 0;
+
+    # check that the first four bytes are 'OggS'
+    read($fh, $buffer, 4);
+    if ($buffer ne 'OggS')
+    {
+	warn "No comment header?";
+	return undef;
+    }
+    $byteCount += 4;
+
+    # skip over next ten bytes
+    read($fh, $buffer, 10);
+    $byteCount += 10;
+
+    # read the stream serial number
+    read($fh, $buffer, 4);
+    push @{$data->{'commentSerialNumber'}}, _decodeInt($buffer);
+    $byteCount += 4;
+
+    # read the page sequence number (should be 0x01)
+    read($fh, $buffer, 4);
+    if (_decodeInt($buffer) != 0x01)
+    {
+	warn "Comment header page sequence number is not 0x01: " + 
+	    _decodeInt($buffer);
+	warn "Going to keep going anyway.";
+    }
+    $byteCount += 4;
+
+    # and ignore the page checksum for now
+    read($fh, $buffer, 4);
+    $byteCount += 4;
+
+    # get the number of entries in the segment_table...
+    read($fh, $buffer, 1);
+    $page_segments = _decodeInt($buffer);
+    $byteCount += 1;
+    # then skip on past it
+    read($fh, $buffer, $page_segments);
+    $byteCount += $page_segments;
+
+    # check the header type (should be 0x03)
+    read($fh, $buffer, 1);
+    if (ord($buffer) != 0x03)
+    {
+	warn "Wrong header type: " . ord($buffer);
+    }    
+    $byteCount += 1;
+
+    # now we should see 'vorbis'
+    read($fh, $buffer, 6);
+    if ($buffer ne 'vorbis')
+    {
+	warn "Missing comment header. Should have found 'vorbis', found " .
+	    $buffer;
+    }
+    $byteCount += 6;
+
+    # get the vendor length
+    read($fh, $buffer, 4);
+    $vendor_length = _decodeInt($buffer);
+    $byteCount += 4;
+
+    # read in the vendor
+    read($fh, $buffer, $vendor_length);
+    $comments{'vendor'} = $buffer;
+    $byteCount += $vendor_length;
+
+    # read in the number of user comments
+    read($fh, $buffer, 4);
+    $user_comment_count = _decodeInt($buffer);
+    $byteCount += 4;
+
+    $data->{'COMMENT_KEYS'} = [];
+
+    # finally, read the comments
+    for (my $i = 0; $i < $user_comment_count; $i++)
+    {
+	# first read the length
+	read($fh, $buffer, 4);
+	my $comment_length = _decodeInt($buffer);
+	$byteCount += 4;
+
+	# then the comment itself
+	read($fh, $buffer, $comment_length);
+	$byteCount += $comment_length;
+
+	my ($key) = $buffer =~ /^([^=]+)/;
+	my ($value) = $buffer =~ /=(.*)$/;
+
+	push @{$comments{lc $key}}, $value;
+	push @{$data->{'COMMENT_KEYS'}}, lc $key;
+    }
+    
+    # read past the framing_bit
+    read($fh, $buffer, 1);
+    $byteCount += 1;
+
+    $data->{'INFO'}{'offset'} = $byteCount;
+
+    $data->{'COMMENTS'} = \%comments;
+}
+
+sub _calculateTrackLength
+{
+    my $data = shift;
+    my $fh = $data->{'fileHandle'};
+    my $buffer;
+    my $pageSize;
+    my $granule_position;
+
+    seek($fh,-8500,SEEK_END);  # that magic number is from vorbisfile.c
+    # in the constant CHUNKSIZE, which comes
+    # with the comment /* a shade over 8k;
+    # anyone using pages well over 8k gets
+    # what they deserve */
+
+    # we just keep looking through the headers until we get to the last one
+    # (there might be a couple of blocks here)
+    while(_findPage($fh))
+    {
+	# stream structure version - must be 0x00
+	read($fh, $buffer, 1);
+	if (ord($buffer) != 0x00)
+	{
+	    warn "Invalid stream structure version: " .
+		sprintf("%x", ord($buffer));
+	    return;
+ 	}
+
+ 	# header type flag
+ 	read($fh, $buffer, 1);
+ 	# we should check this, but for now we'll just ignore it
+
+ 	# absolute granule position - this is what we need!
+ 	read($fh, $buffer, 8);
+ 	$granule_position = _decodeInt($buffer);
+
+	# skip past stream_serial_number, page_sequence_number, and crc
+	read($fh, $buffer, 12);
+
+	# page_segments
+	read($fh, $buffer, 1);
+	my $page_segments = ord($buffer);
+
+	# reset pageSize
+	$pageSize = 0;
+	
+	# calculate approx. page size
+	for (my $i = 0; $i < $page_segments; $i++)
+	{
+	    read($fh, $buffer, 1);
+	    $pageSize += ord($buffer);
+	}
+
+	seek $fh, $pageSize, 1;
+    }
+
+    $data->{'INFO'}{'length'} = 
+	int($granule_position / $data->{'INFO'}{'rate'});
+}
+
+sub _findPage
+{
+    # search forward in the file for the 'OggS' page header
+    my $fh = shift;
+    my $char;
+    my $curStr = '';
+
+    while (read($fh, $char, 1))
+    {
+	$curStr = $char . $curStr;
+	$curStr = substr($curStr, 0, 4);
+
+	# we are actually looking for the string 'SggO' because we
+	# tack character on to our test string backwards, to make
+	# trimming it to 4 characters easier.
+	if ($curStr eq 'SggO')
+	{
+	    return 1;
+	}
+    }
+
+    return undef;
+}
+
+sub _decodeInt
+{
+    my $bytes = shift;
+    my $num = 0;
+    my @byteList = split //, $bytes;
+    my $numBytes = @byteList;
+    my $mult = 1;
+
+    for (my $i = 0; $i < $numBytes; $i ++)
+    {
+	$num += ord($byteList[$i]) * $mult;
+	$mult *= 256;
+    }
+
+    return $num;
+}
+
+sub _decodeInt5Bit
+{
+    my $byte = ord(shift);
+
+    $byte = $byte & 0xF8; # clear out the bottm 3 bits
+    $byte = $byte >> 3; # and shifted down to where it belongs
+
+    return $byte;
+}
+
+sub _decodeInt4Bit
+{
+    my $byte = ord(shift);
+
+    $byte = $byte & 0xFC; # clear out the bottm 4 bits
+    $byte = $byte >> 4; # and shifted down to where it belongs
+
+    return $byte;
+}
+
+sub _ilog
+{
+    my $x = shift;
+    my $ret = 0;
+
+    unless ($x > 0)
+    {
+	return 0;
+    }
+
+    while ($x > 0)
+    {
+	$ret++;
+	$x = $x >> 1;
+    }
+
+    return $ret;
+}
+
+1;
+__DATA__
+
+=head1 NAME
+
+Ogg::Vorbis::Header::PurePerl - An object-oriented interface to Ogg Vorbis
+information and comment fields, implemented entirely in Perl.  Intended to be 
+a drop in replacement for Ogg::Vobis::Header.
+
+Unlike Ogg::Vorbis::Header, this module will go ahead and fill in all of the
+information fields as soon as you construct the object.  In other words,
+the C<new> and C<load> constructors have identical behavior. 
+
+=head1 SYNOPSIS
+
+	use Ogg::Vorbis::Header::PurePerl;
+	my $ogg = Ogg::Vorbis::Header::PurePerl->new("song.ogg");
+	while (my ($k, $v) = each %{$ogg->info}) {
+		print "$k: $v\n";
+	}
+	foreach my $com ($ogg->comment_tags) {
+		print "$com: $_\n" foreach $ogg->comment($com);
+	}
+
+=head1 DESCRIPTION
+
+This module is intended to be a drop in replacement for Ogg::Vorbis::Header,
+implemented entirely in Perl.  It provides an object-oriented interface to
+Ogg Vorbis information and comment fields.  (NOTE: This module currently 
+supports only read operations).
+
+=head1 CONSTRUCTORS
+
+=head2 C<new ($filename)>
+
+Opens an Ogg Vorbis file, ensuring that it exists and is actually an
+Ogg Vorbis stream.  This method does not actually read any of the
+information or comment fields, and closes the file immediately. 
+
+=head2 C<load ([$filename])>
+
+Opens an Ogg Vorbis file, ensuring that it exists and is actually an
+Ogg Vorbis stream, then loads the information and comment fields.  This
+method can also be used without a filename to load the information
+and fields of an already constructed instance.
+
+=head1 INSTANCE METHODS
+
+=head2 C<info ([$key])>
+
+Returns a hashref containing information about the Ogg Vorbis file from
+the file's information header.  Hash fields are: version, channels, rate,
+bitrate_upper, bitrate_nominal, bitrate_lower, bitrate_window, and length.
+The bitrate_window value is not currently used by the vorbis codec, and 
+will always be -1.  
+
+The optional parameter, key, allows you to retrieve a single value from
+the object's hash.  Returns C<undef> if the key is not found.
+
+=head2 C<comment_tags ()>
+
+Returns an array containing the key values for the comment fields. 
+These values can then be passed to C<comment> to retrieve their values.
+
+=head2 C<comment ($key)>
+
+Returns an array of comment values associated with the given key.
+
+=head2 C<add_comments ($key, $value, [$key, $value, ...])>
+
+Unimplemented.
+
+=head2 C<edit_comment ($key, $value, [$num])>
+
+Unimplemented.
+
+=head2 C<delete_comment ($key, [$num])>
+
+Unimplemented.
+
+=head2 C<clear_comments ([@keys])>
+
+Unimplemented.
+
+=head2 C<write_vorbis ()>
+
+Unimplemented.
+
+=head2 C<path ()>
+
+Returns the path/filename of the file the object represents.
+
+=head1 NOTE
+
+This is ALPHA SOFTWARE.  It may very well be very broken.  Do not use it in
+a production environment.  You have been warned.
+
+=head1 ACKNOWLEDGEMENTS
+
+Dave Brown <cpan@dagbrown.com> made this module significantly faster
+at calculating the length of ogg files.
+
+Robert Moser II <rlmoser@earthlink.net> fixed a problem with files that
+have no comments.
+
+=head1 AUTHOR
+
+Andrew Molloy E<lt>amolloy@kaizolabs.comE<gt>
+
+=head1 COPYRIGHT
+
+Copyright (c) 2003, Andrew Molloy.  All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at
+your option) any later version.  A copy of this license is included
+with this module (LICENSE.GPL).
+
+=head1 SEE ALSO
+
+L<Ogg::Vorbis::Header>, L<Ogg::Vorbis::Decoder>
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/Win32/Shortcut.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/Win32/Shortcut.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/Win32/Shortcut.pm	(revision 21112)
@@ -0,0 +1,352 @@
+package Win32::Shortcut;
+#######################################################################
+#
+# Win32::Shortcut - Perl Module for Shell Link Interface
+# ^^^^^^^^^^^^^^^
+# This module creates an object oriented interface to the Win32
+# Shell Links (IShellLink interface).
+#
+# Version: 0.03 (07 Apr 1997)
+#
+#######################################################################
+
+require Exporter;       # to export the constants to the main:: space
+require DynaLoader;     # to dynuhlode the module.
+
+@ISA= qw( Exporter DynaLoader );
+@EXPORT = qw(
+    SW_SHOWMAXIMIZED
+    SW_SHOWMINNOACTIVE
+    SW_SHOWNORMAL
+);
+
+
+#######################################################################
+# This AUTOLOAD is used to 'autoload' constants from the constant()
+# XS function.  If a constant is not found then control is passed
+# to the AUTOLOAD in AutoLoader.
+#
+
+sub AUTOLOAD {
+    my($constname);
+    ($constname = $AUTOLOAD) =~ s/.*:://;
+    #reset $! to zero to reset any current errors.
+    local $! = 0;
+    my $val = constant($constname, @_ ? $_[0] : 0);
+    if ($! != 0) {
+
+    # [dada] This results in an ugly Autoloader error
+
+    #if ($! =~ /Invalid/) {
+    #    $AutoLoader::AUTOLOAD = $AUTOLOAD;
+    #    goto &AutoLoader::AUTOLOAD;
+    #} else {
+    
+    # [dada] ... I prefer this one :)
+
+        ($pack, $file, $line) = caller;
+        undef $pack; # [dada] and get rid of "used only once" warning...
+        die "Win32::Shortcut::$constname is not defined, used at $file line $line.";
+
+    #}
+    }
+    eval "sub $AUTOLOAD { $val }";
+    goto &$AUTOLOAD;
+}
+
+
+#######################################################################
+# STATIC OBJECT PROPERTIES
+#
+$VERSION = "0.03";
+
+#######################################################################
+# PUBLIC METHODS
+#
+
+#======== ### CLASS CONSTRUCTOR
+sub new {
+#========
+    my($class, $file) = @_;
+    my $self = {}; 
+    my $ilink = 0; 
+    my $ifile = 0;
+
+    ($ilink, $ifile) = _Instance();
+
+    if($ilink and $ifile) {
+        $self->{'ilink'} = $ilink;
+        $self->{'ifile'} = $ifile;
+        bless $self;
+        # Initialize properties
+        $self->{'File'}             = "";
+        $self->{'Path'}             = "";
+        $self->{'Arguments'}        = "";
+        $self->{'WorkingDirectory'} = "";
+        $self->{'Description'}      = "";
+        $self->{'ShowCmd'}          = 0;
+        $self->{'Hotkey'}           = 0;
+        $self->{'IconLocation'}     = "";
+        $self->{'IconNumber'}       = 0;
+
+        $self->Load($file) if $file;
+        
+    } else {
+        return undef;
+    }
+    $self;
+}  
+
+#=========
+sub Load {
+#=========
+    my($self, $file) = @_;
+    return undef unless ref($self);
+  
+    my $result = _Load($self->{'ilink'}, $self->{'ifile'}, $file);
+
+    if(defined($result)) {
+  
+        # fill the properties of $self
+        $self->{'File'} = $file;
+        $self->{'Path'} = _GetPath($self->{'ilink'}, $self->{'ifile'},0);
+        $self->{'ShortPath'} = _GetPath($self->{'ilink'}, $self->{'ifile'},1);
+        $self->{'Arguments'} = _GetArguments($self->{'ilink'}, $self->{'ifile'});
+        $self->{'WorkingDirectory'} = _GetWorkingDirectory($self->{'ilink'}, $self->{'ifile'});
+        $self->{'Description'} = _GetDescription($self->{'ilink'}, $self->{'ifile'});
+        $self->{'ShowCmd'} = _GetShowCmd($self->{'ilink'}, $self->{'ifile'});
+        $self->{'Hotkey'} = _GetHotkey($self->{'ilink'}, $self->{'ifile'});
+        ($self->{'IconLocation'},
+         $self->{'IconNumber'}) = _GetIconLocation($self->{'ilink'}, $self->{'ifile'});
+    }
+    return $result;
+}
+
+
+#========
+sub Set {
+#========
+    my($self, $path, $arguments, $dir, $description, $show, $hotkey, 
+       $iconlocation, $iconnumber) = @_;
+    return undef unless ref($self);
+
+    $self->{'Path'}             = $path;
+    $self->{'Arguments'}        = $arguments;
+    $self->{'WorkingDirectory'} = $dir;
+    $self->{'Description'}      = $description;
+    $self->{'ShowCmd'}          = $show;
+    $self->{'Hotkey'}           = $hotkey;
+    $self->{'IconLocation'}     = $iconlocation;
+    $self->{'IconNumber'}       = $iconnumber;
+    return 1;
+}
+
+
+#=========
+sub Save {
+#=========
+    my($self, $file) = @_;
+    return undef unless ref($self);
+
+    return undef if not $file and not $self->{'File'};
+    $file = $self->{'File'} if not $file;
+
+    _SetPath($self->{'ilink'}, $self->{'ifile'}, $self->{'Path'});
+    _SetArguments($self->{'ilink'}, $self->{'ifile'}, $self->{'Arguments'});
+    _SetWorkingDirectory($self->{'ilink'}, $self->{'ifile'}, $self->{'WorkingDirectory'});
+    _SetDescription($self->{'ilink'}, $self->{'ifile'}, $self->{'Description'});
+    _SetShowCmd($self->{'ilink'}, $self->{'ifile'}, $self->{'ShowCmd'});
+    _SetHotkey($self->{'ilink'}, $self->{'ifile'}, $self->{'Hotkey'});
+    _SetIconLocation($self->{'ilink'}, $self->{'ifile'},
+                     $self->{'IconLocation'}, $self->{'IconNumber'});
+
+    my $result = _Save($self->{'ilink'}, $self->{'ifile'}, $file);
+    return $result;
+}
+
+#============
+sub Resolve {
+#============
+    my($self, $flags) = @_;
+    return undef unless ref($self);
+    $flags = 1 unless defined($flags);
+    my $result = _Resolve($self->{'ilink'}, $self->{'ifile'}, $flags);
+    return $result;
+}
+
+
+#==========
+sub Close {
+#==========
+    my($self) = @_;
+    return undef unless ref($self);
+
+    my $result = _Release($self->{'ilink'}, $self->{'ifile'});
+    $self->{'released'} = 1;
+    return $result;
+}
+
+#=========
+sub Path {
+#=========
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'Path'};
+    } else {
+        $self->{'Path'} = $value;
+    }
+    return $self->{'Path'};
+}
+
+#==============
+sub ShortPath {
+#==============
+    my($self) = @_;
+    return undef unless ref($self);
+    return $self->{'ShortPath'};
+}
+
+#==============
+sub Arguments {
+#==============
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'Arguments'};
+    } else {
+        $self->{'Arguments'} = $value;
+    }
+    return $self->{'Arguments'};
+}
+
+#=====================
+sub WorkingDirectory {
+#=====================
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'WorkingDirectory'};
+    } else {
+        $self->{'WorkingDirectory'} = $value;
+    }
+    return $self->{'WorkingDirectory'};
+}
+
+
+#================
+sub Description {
+#================
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'Description'};
+    } else {
+        $self->{'Description'} = $value;
+    }
+    return $self->{'Description'};
+}
+
+#============
+sub ShowCmd {
+#============
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'ShowCmd'};
+    } else {
+        $self->{'ShowCmd'} = $value;
+    }
+    return $self->{'ShowCmd'};
+}
+
+#===========
+sub Hotkey {
+#===========
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'Hotkey'};
+    } else {
+        $self->{'Hotkey'} = $value;
+    }
+    return $self->{'Hotkey'};
+}
+
+#=================
+sub IconLocation {
+#=================
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'IconLocation'};
+    } else {
+        $self->{'IconLocation'} = $value;
+    }
+    return $self->{'IconLocation'};
+}
+
+#===============
+sub IconNumber {
+#===============
+    my($self, $value) = @_;
+    return undef unless ref($self);
+
+    if(not defined($value)) {
+        return $self->{'IconNumber'};
+    } else {
+        $self->{'IconNumber'} = $value;
+    }
+    return $self->{'IconNumber'};
+}
+
+#============
+sub Version {
+#============
+    # [dada] to get rid of the "used only once" warning...
+    return $VERSION;
+}
+
+
+#######################################################################
+# PRIVATE METHODS
+#
+
+#============ ### CLASS DESTRUCTOR
+sub DESTROY {
+#============
+    my($self) = @_;
+
+    if(not $self->{'released'}) {
+        _Release($self->{'ilink'}, $self->{'ifile'});
+    }
+}
+
+#======== ### PACKAGE DESTRUCTOR
+sub END { 
+#========
+    # print "Exiting...\n";
+    _Exit(); 
+}
+
+#######################################################################
+# dynamically load in the Shortcut.pll module.
+#
+
+bootstrap Win32::Shortcut;
+
+# Preloaded methods go here.
+
+#Currently Autoloading is not implemented in Perl for win32
+# Autoload methods go after __END__, and are processed by the autosplit program.
+
+1;
+__END__
+
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath.pm	(revision 21112)
@@ -0,0 +1,553 @@
+# $Id$
+
+package XML::XPath;
+
+use strict;
+use vars qw($VERSION $AUTOLOAD $revision);
+
+$VERSION = '1.13';
+
+$XML::XPath::Namespaces = 1;
+$XML::XPath::Debug = 0;
+
+use XML::XPath::XMLParser;
+use XML::XPath::Parser;
+use IO::File;
+
+# For testing
+#use Data::Dumper;
+#$Data::Dumper::Indent = 1;
+
+# Parameters for new()
+my @options = qw(
+        filename
+        parser
+        xml
+        ioref
+        context
+        );
+
+sub new {
+    my $proto = shift;
+    my $class = ref($proto) || $proto;
+
+    my(%args);
+    # Try to figure out what the user passed
+    if ($#_ == 0) { # passed a scalar
+        my $string = $_[0];
+        if ($string =~ m{<.*?>}s) { # it's an XML string
+            $args{'xml'} = $string;
+        } elsif (ref($string)) {    # read XML from file handle
+            $args{'ioref'} = $string;
+        } elsif ($string eq '-') {  # read XML from stdin
+            $args{'ioref'} = IO::File->new($string);
+        } else {                    # read XML from a file
+            $args{'filename'} = $string;
+        }
+    } else {        # passed a hash or hash reference
+        # just pass the parameters on to the XPath constructor
+        %args = ((ref($_[0]) eq "HASH") ? %{$_[0]} : @_);
+    }
+
+    if ($args{filename} && (!-e $args{filename} || !-r $args{filename})) {
+        die "Cannot open file '$args{filename}'";
+    }
+    my %hash = map(( "_$_" => $args{$_} ), @options);
+    $hash{path_parser} = XML::XPath::Parser->new();
+    return bless \%hash, $class;
+}
+
+sub find {
+    my $self = shift;
+    my $path = shift;
+    my $context = shift;
+    die "No path to find" unless $path;
+    
+    if (!defined $context) {
+        $context = $self->get_context;
+    }
+    if (!defined $context) {
+        # Still no context? Need to parse...
+        my $parser = XML::XPath::XMLParser->new(
+                filename => $self->get_filename,
+                xml => $self->get_xml,
+                ioref => $self->get_ioref,
+                parser => $self->get_parser,
+                );
+        $context = $parser->parse;
+        $self->set_context($context);
+#        warn "CONTEXT:\n", Data::Dumper->Dumpxs([$context], ['context']);
+    }
+    
+    my $parsed_path = $self->{path_parser}->parse($path);
+#    warn "\n\nPATH: ", $parsed_path->as_string, "\n\n";
+    
+#    warn "evaluating path\n";
+    return $parsed_path->evaluate($context);
+}
+
+# sub memsize {
+#     print STDERR @_, "\t";
+#     open(FH, '/proc/self/status');
+#     while(<FH>) {
+#         print STDERR $_ if /^VmSize/;
+#     }
+#     close FH;
+# }
+# 
+sub findnodes {
+    my $self = shift;
+    my ($path, $context) = @_;
+    
+    my $results = $self->find($path, $context);
+    
+    if ($results->isa('XML::XPath::NodeSet')) {
+        return wantarray ? $results->get_nodelist : $results;
+#        return $results->get_nodelist;
+    }
+    
+#    warn("findnodes returned a ", ref($results), " object\n") if $XML::XPath::Debug;
+    return wantarray ? () : XML::XPath::NodeSet->new();
+}
+
+sub matches {
+    my $self = shift;
+    my ($node, $path, $context) = @_;
+
+    my @nodes = $self->findnodes($path, $context);
+
+    if (grep { "$node" eq "$_" } @nodes) {
+        return 1;
+    }
+    return;
+}
+
+sub findnodes_as_string {
+    my $self = shift;
+    my ($path, $context) = @_;
+    
+    my $results = $self->find($path, $context);
+    
+    if ($results->isa('XML::XPath::NodeSet')) {
+        return join('', map { $_->toString } $results->get_nodelist);
+    }
+    elsif ($results->isa('XML::XPath::Node')) {
+        return $results->toString;
+    }
+    else {
+        return XML::XPath::Node::XMLescape($results->value);
+    }
+}
+
+sub findvalue {
+    my $self = shift;
+    my ($path, $context) = @_;
+    
+    my $results = $self->find($path, $context);
+    
+    if ($results->isa('XML::XPath::NodeSet')) {
+        return $results->to_literal;
+    }
+    
+    return $results;
+}
+
+sub exists
+{
+    my $self = shift;
+    my ($path, $context) = @_;
+    $path = '/' if (!defined $path);
+    my @nodeset = $self->findnodes($path, $context);
+    return 1 if (scalar( @nodeset ));
+    return 0;
+}
+
+sub getNodeAsXML {
+  my $self = shift;
+  my $node_path = shift;
+  $node_path = '/' if (!defined $node_path);
+  if (ref($node_path)) {
+    return $node_path->as_string();
+  } else {
+    return $self->findnodes_as_string($node_path);
+  }
+}
+
+sub getNodeText {
+  my $self = shift;
+  my $node_path = shift;
+  if (ref($node_path)) {
+    return $node_path->string_value();
+  } else {
+    return $self->findvalue($node_path);
+  }
+}
+
+sub setNodeText {
+  my $self = shift;
+  my($node_path, $new_text) = @_;
+  my $nodeset = $self->findnodes($node_path);
+  return undef if (!defined $nodeset); # could not find node
+  my @nodes = $nodeset->get_nodelist;
+  if ($#nodes < 0) {
+    if ($node_path =~ m|/@([^/]+)$|) {
+      # attribute not found, so try to create it
+      my $parent_path = $`;
+      my $attr = $1;
+      $nodeset = $self->findnodes($parent_path);
+      return undef if (!defined $nodeset); # could not find node
+      foreach my $node ($nodeset->get_nodelist) {
+        my $newnode = XML::XPath::Node::Attribute->new($attr, $new_text);
+        return undef if (!defined $newnode); # could not create new node
+        $node->appendAttribute($newnode);
+      }
+    } else {
+      return undef; # could not find node
+    }
+  }
+  foreach my $node (@nodes) {
+    if ($node->getNodeType == XML::XPath::Node::ATTRIBUTE_NODE) {
+      $node->setNodeValue($new_text);
+    } else {
+      foreach my $delnode ($node->getChildNodes()) {
+        $node->removeChild($delnode);
+      }
+      my $newnode = XML::XPath::Node::Text->new($new_text);
+      return undef if (!defined $newnode); # could not create new node
+      $node->appendChild($newnode);
+    }
+  }
+  return 1;
+}
+
+sub createNode {
+  my $self = shift;
+  my($node_path) = @_;
+  my $path_steps = $self->{path_parser}->parse($node_path);
+  my @path_steps = ();
+  foreach my $step (@{$path_steps->get_lhs()}) {
+    my $string = $step->as_string();
+    push(@path_steps, $string) if (defined $string && $string ne "");
+  }
+  my $prev_node = undef;
+  my $nodeset = undef;
+  my $nodes = undef;
+  my $p = undef;
+  my $test_path = "";
+  # Start with the deepest node, working up the path (right to left),
+  # trying to find a node that exists.
+  for ($p = $#path_steps; $p >= 0; $p--) {
+    my $path = $path_steps[$p];
+    $test_path = "(/" . join("/", @path_steps[0..$p]) . ")";
+    $nodeset = $self->findnodes($test_path);
+    return undef if (!defined $nodeset); # error looking for node
+    $nodes = $nodeset->size;
+    return undef if ($nodes > 1); # too many paths - path not specific enough
+    if ($nodes == 1) { # found a node -- need to create nodes below it
+      $prev_node = $nodeset->get_node(1);
+      last;
+    }
+  }
+  if (!defined $prev_node) {
+    my @root_nodes = $self->findnodes('/')->get_nodelist();
+    $prev_node = $root_nodes[0];
+  }
+  # We found a node that exists, or we'll start at the root.
+  # Create all lower nodes working left to right along the path.
+  for ($p++ ; $p <= $#path_steps; $p++) {
+    my $path = $path_steps[$p];
+    my $newnode = undef;
+    my($axis,$name) = ($path =~ /^(.*?)::(.*)$/);
+    if ($axis =~ /^child$/i) {
+      $newnode = XML::XPath::Node::Element->new($name);
+      return undef if (!defined $newnode); # could not create new node
+      $prev_node->appendChild($newnode);
+    } elsif ($axis =~ /^attribute$/i) {
+      $newnode = XML::XPath::Node::Attribute->new($name, "");
+      return undef if (!defined $newnode); # could not create new node
+      $prev_node->appendAttribute($newnode);
+    }
+    $prev_node = $newnode;
+  }
+  return $prev_node;
+}
+
+sub get_filename {
+    my $self = shift;
+    $self->{_filename};
+}
+
+sub set_filename {
+    my $self = shift;
+    $self->{_filename} = shift;
+}
+
+sub get_parser {
+    my $self = shift;
+    $self->{_parser};
+}
+
+sub set_parser {
+    my $self = shift;
+    $self->{_parser} = shift;
+}
+
+sub get_xml {
+    my $self = shift;
+    $self->{_xml};
+}
+
+sub set_xml {
+    my $self = shift;
+    $self->{_xml} = shift;
+}
+
+sub get_ioref {
+    my $self = shift;
+    $self->{_ioref};
+}
+
+sub set_ioref {
+    my $self = shift;
+    $self->{_ioref} = shift;
+}
+
+sub get_context {
+    my $self = shift;
+    $self->{_context};
+}
+
+sub set_context {
+    my $self = shift;
+    $self->{_context} = shift;
+}
+
+sub cleanup {
+    my $self = shift;
+    if ($XML::XPath::SafeMode) {
+        my $context = $self->get_context;
+        return unless $context;
+        $context->dispose;
+    }
+}
+
+sub set_namespace {
+    my $self = shift;
+    my ($prefix, $expanded) = @_;
+    $self->{path_parser}->set_namespace($prefix, $expanded);
+}
+
+sub clear_namespaces {
+    my $self = shift;
+    $self->{path_parser}->clear_namespaces();
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath - a set of modules for parsing and evaluating XPath statements
+
+=head1 DESCRIPTION
+
+This module aims to comply exactly to the XPath specification at
+http://www.w3.org/TR/xpath and yet allow extensions to be added in the
+form of functions. Modules such as XSLT and XPointer may need to do
+this as they support functionality beyond XPath.
+
+=head1 SYNOPSIS
+
+    use XML::XPath;
+    use XML::XPath::XMLParser;
+    
+    my $xp = XML::XPath->new(filename => 'test.xhtml');
+    
+    my $nodeset = $xp->find('/html/body/p'); # find all paragraphs
+    
+    foreach my $node ($nodeset->get_nodelist) {
+        print "FOUND\n\n", 
+            XML::XPath::XMLParser::as_string($node),
+            "\n\n";
+    }
+
+=head1 DETAILS
+
+There's an awful lot to all of this, so bear with it - if you stick it
+out it should be worth it. Please get a good understanding of XPath
+by reading the spec before asking me questions. All of the classes
+and parts herein are named to be synonimous with the names in the
+specification, so consult that if you don't understand why I'm doing
+something in the code.
+
+=head1 API
+
+The API of XML::XPath itself is extremely simple to allow you to get
+going almost immediately. The deeper API's are more complex, but you
+shouldn't have to touch most of that.
+
+=head2 new()
+
+This constructor follows the often seen named parameter method call.
+Parameters you can use are: filename, parser, xml, ioref and context.
+The filename parameter specifies an XML file to parse. The xml
+parameter specifies a string to parse, and the ioref parameter
+specifies an ioref to parse. The context option allows you to 
+specify a context node. The context node has to be in the format 
+of a node as specified in L<XML::XPath::XMLParser>. The 4 parameters
+filename, xml, ioref and context are mutually exclusive - you should
+only specify one (if you specify anything other than context, the
+context node is the root of your document).
+The parser option allows you to pass in an already prepared 
+XML::Parser object, to save you having to create more than one
+in your application (if, for example, you're doing more than just XPath).
+
+    my $xp = XML::XPath->new( context => $node );
+
+It is very much recommended that you use only 1 XPath object throughout 
+the life of your application. This is because the object (and it's sub-objects)
+maintain certain bits of state information that will be useful (such
+as XPath variables) to later calls to find(). It's also a good idea because
+you'll use less memory this way.
+
+=head2 I<nodeset> = find($path, [$context])
+
+The find function takes an XPath expression (a string) and returns either an
+XML::XPath::NodeSet object containing the nodes it found (or empty if
+no nodes matched the path), or one of XML::XPath::Literal (a string),
+XML::XPath::Number, or XML::XPath::Boolean. It should always return 
+something - and you can use ->isa() to find out what it returned. If you
+need to check how many nodes it found you should check $nodeset->size.
+See L<XML::XPath::NodeSet>. An optional second parameter of a context
+node allows you to use this method repeatedly, for example XSLT needs
+to do this.
+
+=head2 findnodes($path, [$context])
+
+Returns a list of nodes found by $path, optionally in context $context. 
+In scalar context returns an XML::XPath::NodeSet object.
+
+=head2 findnodes_as_string($path, [$context])
+
+Returns the nodes found reproduced as XML. The result is not guaranteed
+to be valid XML though.
+
+=head2 findvalue($path, [$context])
+
+Returns either a C<XML::XPath::Literal>, a C<XML::XPath::Boolean> or a
+C<XML::XPath::Number> object. If the path returns a NodeSet,
+$nodeset->to_literal is called automatically for you (and thus a
+C<XML::XPath::Literal> is returned). Note that
+for each of the objects stringification is overloaded, so you can just
+print the value found, or manipulate it in the ways you would a normal
+perl value (e.g. using regular expressions).
+
+=head2 exists($path, [$context])
+
+Returns true if the given path exists.
+
+=head2 matches($node, $path, [$context])
+
+Returns true if the node matches the path (optionally in context $context).
+
+=head2 getNodeText($path)
+
+Returns the text string for a particular XML node.  Returns a string,
+or undef if the node doesn't exist.
+
+=head2 setNodeText($path, $text)
+
+Sets the text string for a particular XML node.  The node can be an
+element or an attribute.  If the node to be set is an attribute, and
+the attribute node does not exist, it will be created automatically.
+
+=head2 createNode($path)
+
+Creates the node matching the path given.  If part of the path given, or
+all of the path do not exist, the necessary nodes will be created
+automatically.
+
+=head2 set_namespace($prefix, $uri)
+
+Sets the namespace prefix mapping to the uri.
+
+Normally in XML::XPath the prefixes in XPath node tests take their
+context from the current node. This means that foo:bar will always
+match an element <foo:bar> regardless of the namespace that the prefix
+foo is mapped to (which might even change within the document, resulting
+in unexpected results). In order to make prefixes in XPath node tests
+actually map to a real URI, you need to enable that via a call
+to the set_namespace method of your XML::XPath object.
+
+=head2 clear_namespaces()
+
+Clears all previously set namespace mappings.
+
+=head2 $XML::XPath::Namespaces
+
+Set this to 0 if you I<don't> want namespace processing to occur. This
+will make everything a little (tiny) bit faster, but you'll suffer for it,
+probably.
+
+=head1 Node Object Model
+
+See L<XML::XPath::Node>, L<XML::XPath::Node::Element>, 
+L<XML::XPath::Node::Text>, L<XML::XPath::Node::Comment>,
+L<XML::XPath::Node::Attribute>, L<XML::XPath::Node::Namespace>,
+and L<XML::XPath::Node::PI>.
+
+=head1 On Garbage Collection
+
+XPath nodes work in a special way that allows circular references, and 
+yet still lets Perl's reference counting garbage collector to clean up
+the nodes after use. This should be totally transparent to the user,
+with one caveat: B<If you free your tree before letting go of a sub-tree,
+consider that playing with fire and you may get burned>. What does this
+mean to the average user? Not much. Provided you don't free (or let go
+out of scope) either the tree you passed to XML::XPath->new, or if you
+didn't pass a tree, and passed a filename or IO-ref, then provided you
+don't let the XML::XPath object go out of scope before you let results
+of find() and its friends go out of scope, then you'll be fine. Even if
+you B<do> let the tree go out of scope before results, you'll probably
+still be fine. The only case where you may get stung is when the last
+part of your path/query is either an ancestor or parent axis. In that
+case the worst that will happen is you'll end up with a circular reference
+that won't get cleared until interpreter destruction time. You can get
+around that by explicitly calling $node->DESTROY on each of your result
+nodes, if you really need to do that.
+
+Mail me direct if that's not clear. Note that it's not doom and gloom. It's
+by no means perfect, but the worst that will happen is a long running process
+could leak memory. Most long running processes will therefore be able to
+explicitly be careful not to free the tree (or XML::XPath object) before
+freeing results. AxKit, an application that uses XML::XPath, does this and
+I didn't have to make any changes to the code - it's already sensible
+programming.
+
+If you I<really> don't want all this to happen, then set the variable
+$XML::XPath::SafeMode, and call $xp->cleanup() on the XML::XPath object
+when you're finished, or $tree->dispose() if you have a tree instead.
+
+=head1 Example
+
+Please see the test files in t/ for examples on how to use XPath.
+
+=head1 Support/Author
+
+This module is copyright 2000 AxKit.com Ltd. This is free
+software, and as such comes with NO WARRANTY. No dates are used in this
+module. You may distribute this module under the terms of either the
+Gnu GPL,  or the Artistic License (the same terms as Perl itself).
+
+For support, please subscribe to the Perl-XML mailing list at the URL 
+http://listserv.activestate.com/mailman/listinfo/perl-xml
+
+Matt Sergeant, matt@sergeant.org
+
+=head1 SEE ALSO
+
+L<XML::XPath::Literal>, L<XML::XPath::Boolean>, L<XML::XPath::Number>,
+L<XML::XPath::XMLParser>, L<XML::XPath::NodeSet>, L<XML::XPath::PerlSAX>,
+L<XML::XPath::Builder>.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Boolean.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Boolean.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Boolean.pm	(revision 21112)
@@ -0,0 +1,73 @@
+# $Id$
+
+package XML::XPath::Boolean;
+use XML::XPath::Number;
+use XML::XPath::Literal;
+use strict;
+
+use overload
+		'""' => \&value,
+		'<=>' => \&cmp;
+
+sub True {
+	my $class = shift;
+	my $val = 1;
+	bless \$val, $class;
+}
+
+sub False {
+	my $class = shift;
+	my $val = 0;
+	bless \$val, $class;
+}
+
+sub value {
+	my $self = shift;
+	$$self;
+}
+
+sub cmp {
+	my $self = shift;
+	my ($other, $swap) = @_;
+	if ($swap) {
+		return $other <=> $$self;
+	}
+	return $$self <=> $other;
+}
+
+sub to_number { XML::XPath::Number->new($_[0]->value); }
+sub to_boolean { $_[0]; }
+sub to_literal { XML::XPath::Literal->new($_[0]->value ? "true" : "false"); }
+
+sub string_value { return $_[0]->to_literal->value; }
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath::Boolean - Boolean true/false values
+
+=head1 DESCRIPTION
+
+XML::XPath::Boolean objects implement simple boolean true/false objects.
+
+=head1 API
+
+=head2 XML::XPath::Boolean->True
+
+Creates a new Boolean object with a true value.
+
+=head2 XML::XPath::Boolean->False
+
+Creates a new Boolean object with a false value.
+
+=head2 value()
+
+Returns true or false.
+
+=head2 to_literal()
+
+Returns the string "true" or "false".
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Builder.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Builder.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Builder.pm	(revision 21112)
@@ -0,0 +1,198 @@
+# $Id$
+
+package XML::XPath::Builder;
+
+use strict;
+
+# to get array index constants
+use XML::XPath::Node;
+use XML::XPath::Node::Element;
+use XML::XPath::Node::Attribute;
+use XML::XPath::Node::Namespace;
+use XML::XPath::Node::Text;
+use XML::XPath::Node::PI;
+use XML::XPath::Node::Comment;
+
+use vars qw/$xmlns_ns $xml_ns/;
+
+$xmlns_ns = "http://www.w3.org/2000/xmlns/";
+$xml_ns = "http://www.w3.org/XML/1998/namespace";
+
+sub new {
+    my $class = shift;
+    my $self = ($#_ == 0) ? { %{ (shift) } } : { @_ };
+
+    bless $self, $class;
+}
+
+sub start_document {
+    my $self = shift;
+
+    $self->{IdNames} = {};
+    $self->{InScopeNamespaceStack} = [ { 
+            '_Default' => undef,
+            'xmlns' => $xmlns_ns,
+            'xml' => $xml_ns,
+        } ];
+    
+    $self->{NodeStack} = [ ];
+    
+    my $document = XML::XPath::Node::Element->new();
+    my $newns = XML::XPath::Node::Namespace->new('xml', $xml_ns);
+    $document->appendNamespace($newns);
+    $self->{current} = $self->{DOC_Node} = $document;
+}
+
+sub end_document {
+    my $self = shift;
+    
+    return $self->{DOC_Node};
+}
+
+sub characters {
+    my $self = shift;
+    my $sarg = shift;
+    my $text = $sarg->{Data};
+    
+    my $parent = $self->{current};
+    
+    my $last = $parent->getLastChild;
+    if ($last && $last->isTextNode) {
+        # append to previous text node
+        $last->appendText($text);
+        return;
+    }
+    
+    my $node = XML::XPath::Node::Text->new($text);
+    $parent->appendChild($node, 1);
+}
+
+sub start_element {
+    my $self = shift;
+    my $sarg = shift;
+    my $tag  = $sarg->{'Name'};
+    my $attr = $sarg->{'Attributes'};
+
+    push @{ $self->{InScopeNamespaceStack} },
+         { %{ $self->{InScopeNamespaceStack}[-1] } };
+    $self->_scan_namespaces(@_);
+    
+    my ($prefix, $namespace) = $self->_namespace($tag);
+    
+    my $node = XML::XPath::Node::Element->new($tag, $prefix);
+    
+    foreach my $name (keys %$attr) {
+	my $value = $attr->{$name};
+
+        if ($name =~ /^xmlns(:(.*))?$/) {
+            # namespace node
+            my $prefix = $2 || '#default';
+#            warn "Creating NS node: $prefix = $value\n";
+            my $newns = XML::XPath::Node::Namespace->new($prefix, $value);
+            $node->appendNamespace($newns);
+        }
+        else {
+	    my ($prefix, $namespace) = $self->_namespace($name);
+            undef $namespace unless $prefix;
+
+            my $newattr = XML::XPath::Node::Attribute->new($name, $value, $prefix);
+            $node->appendAttribute($newattr, 1);
+            if (exists($self->{IdNames}{$tag}) && ($self->{IdNames}{$tag} eq $name)) {
+    #            warn "appending Id Element: $val for ", $node->getName, "\n";
+                $self->{DOC_Node}->appendIdElement($value, $node);
+            }
+        }
+    }
+        
+    $self->{current}->appendChild($node, 1);
+    $self->{current} = $node;
+}
+
+sub end_element {
+    my $self = shift;
+    $self->{current} = $self->{current}->getParentNode;
+}
+
+sub processing_instruction {
+    my $self = shift;
+    my $pi = shift;
+    my $node = XML::XPath::Node::PI->new($pi->{Target}, $pi->{Data});
+    $self->{current}->appendChild($node, 1);
+}
+
+sub comment {
+    my $self = shift;
+    my $comment = shift;
+    my $node = XML::XPath::Node::Comment->new($comment->{Data});
+    $self->{current}->appendChild($node, 1);
+}
+
+sub _scan_namespaces {
+    my ($self, %attributes) = @_;
+
+    while (my ($attr_name, $value) = each %attributes) {
+	if ($attr_name eq 'xmlns') {
+	    $self->{InScopeNamespaceStack}[-1]{'_Default'} = $value;
+	} elsif ($attr_name =~ /^xmlns:(.*)$/) {
+	    my $prefix = $1;
+	    $self->{InScopeNamespaceStack}[-1]{$prefix} = $value;
+	}
+    }
+}
+
+sub _namespace {
+    my ($self, $name) = @_;
+
+    my ($prefix, $localname) = split(/:/, $name);
+    if (!defined($localname)) {
+	if ($prefix eq 'xmlns') {
+	    return '', undef;
+	} else {
+	    return '', $self->{InScopeNamespaceStack}[-1]{'_Default'};
+	}
+    } else {
+	return $prefix, $self->{InScopeNamespaceStack}[-1]{$prefix};
+    }
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::XPath::Builder - SAX handler for building an XPath tree
+
+=head1 SYNOPSIS
+
+ use AnySAXParser;
+ use XML::XPath::Builder;
+
+ $builder = XML::XPath::Builder->new();
+ $parser = AnySAXParser->new( Handler => $builder );
+
+ $root_node = $parser->parse( Source => [SOURCE] );
+
+=head1 DESCRIPTION
+
+C<XML::XPath::Builder> is a SAX handler for building an XML::XPath
+tree.
+
+C<XML::XPath::Builder> is used by creating a new instance of
+C<XML::XPath::Builder> and providing it as the Handler for a SAX
+parser.  Calling `C<parse()>' on the SAX parser will return the
+root node of the tree built from that parse.
+
+=head1 AUTHOR
+
+Ken MacLeod, <ken@bitsko.slc.ut.us>
+
+=head1 SEE ALSO
+
+perl(1), XML::XPath(3)
+
+PerlSAX.pod in libxml-perl
+
+Extensible Markup Language (XML) <http://www.w3c.org/XML>
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Expr.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Expr.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Expr.pm	(revision 21112)
@@ -0,0 +1,619 @@
+# $Id$
+
+package XML::XPath::Expr;
+use strict;
+
+sub new {
+    my $class = shift;
+    my ($pp) = @_;
+    bless { predicates => [], pp => $pp }, $class;
+}
+
+sub as_string {
+    my $self = shift;
+    local $^W; # Use of uninitialized value! grrr
+    my $string = "(" . $self->{lhs}->as_string;
+    $string .= " " . $self->{op} . " " if defined $self->{op};
+    $string .= $self->{rhs}->as_string if defined $self->{rhs};
+    $string .= ")";
+    foreach my $predicate (@{$self->{predicates}}) {
+        $string .= "[" . $predicate->as_string . "]";
+    }
+    return $string;
+}
+
+sub as_xml {
+    my $self = shift;
+    local $^W; # Use of uninitialized value! grrr
+    my $string;
+    if (defined $self->{op}) {
+        $string .= $self->op_xml();
+    }
+    else {
+        $string .= $self->{lhs}->as_xml();
+    }
+    foreach my $predicate (@{$self->{predicates}}) {
+        $string .= "<Predicate>\n" . $predicate->as_xml() . "</Predicate>\n";
+    }
+    return $string;
+}
+
+sub op_xml {
+    my $self = shift;
+    my $op = $self->{op};
+
+    my $tag;    
+    for ($op) {
+        /^or$/    && do {
+                    $tag = "Or";
+                };
+        /^and$/    && do {
+                    $tag = "And";
+                };
+        /^=$/    && do {
+                    $tag = "Equals";
+                };
+        /^!=$/    && do {
+                    $tag = "NotEquals";
+                };
+        /^<=$/    && do {
+                    $tag = "LessThanOrEquals";
+                };
+        /^>=$/    && do {
+                    $tag = "GreaterThanOrEquals";
+                };
+        /^>$/    && do {
+                    $tag = "GreaterThan";
+                };
+        /^<$/    && do {
+                    $tag = "LessThan";
+                };
+        /^\+$/    && do {
+                    $tag = "Plus";
+                };
+        /^-$/    && do {
+                    $tag = "Minus";
+                };
+        /^div$/    && do {
+                    $tag = "Div";
+                };
+        /^mod$/    && do {
+                    $tag = "Mod";
+                };
+        /^\*$/    && do {
+                    $tag = "Multiply";
+                };
+        /^\|$/    && do {
+                    $tag = "Union";
+                };
+    }
+    
+    return "<$tag>\n" . $self->{lhs}->as_xml() . $self->{rhs}->as_xml() . "</$tag>\n";
+}
+
+sub set_lhs {
+    my $self = shift;
+    $self->{lhs} = $_[0];
+}
+
+sub set_op {
+    my $self = shift;
+    $self->{op} = $_[0];
+}
+
+sub set_rhs {
+    my $self = shift;
+    $self->{rhs} = $_[0];
+}
+
+sub push_predicate {
+    my $self = shift;
+    
+    die "Only 1 predicate allowed on FilterExpr in W3C XPath 1.0"
+            if @{$self->{predicates}};
+    
+    push @{$self->{predicates}}, $_[0];
+}
+
+sub get_lhs { $_[0]->{lhs}; }
+sub get_rhs { $_[0]->{rhs}; }
+sub get_op { $_[0]->{op}; }
+
+sub evaluate {
+    my $self = shift;
+    my $node = shift;
+    
+    # If there's an op, result is result of that op.
+    # If no op, just resolve Expr
+    
+#    warn "Evaluate Expr: ", $self->as_string, "\n";
+    
+    my $results;
+    
+    if ($self->{op}) {
+        die ("No RHS of ", $self->as_string) unless $self->{rhs};
+        $results = $self->op_eval($node);
+    }
+    else {
+        $results = $self->{lhs}->evaluate($node);
+    }
+    
+    if (my @predicates = @{$self->{predicates}}) {
+        if (!$results->isa('XML::XPath::NodeSet')) {
+            die "Can't have predicates execute on object type: " . ref($results);
+        }
+        
+        # filter initial nodeset by each predicate
+        foreach my $predicate (@{$self->{predicates}}) {
+            $results = $self->filter_by_predicate($results, $predicate);
+        }
+    }
+    
+    return $results;
+}
+
+sub op_eval {
+    my $self = shift;
+    my $node = shift;
+    
+    my $op = $self->{op};
+    
+    for ($op) {
+        /^or$/    && do {
+                    return op_or($node, $self->{lhs}, $self->{rhs});
+                };
+        /^and$/    && do {
+                    return op_and($node, $self->{lhs}, $self->{rhs});
+                };
+        /^=$/    && do {
+                    return op_equals($node, $self->{lhs}, $self->{rhs});
+                };
+        /^!=$/    && do {
+                    return op_nequals($node, $self->{lhs}, $self->{rhs});
+                };
+        /^<=$/    && do {
+                    return op_le($node, $self->{lhs}, $self->{rhs});
+                };
+        /^>=$/    && do {
+                    return op_ge($node, $self->{lhs}, $self->{rhs});
+                };
+        /^>$/    && do {
+                    return op_gt($node, $self->{lhs}, $self->{rhs});
+                };
+        /^<$/    && do {
+                    return op_lt($node, $self->{lhs}, $self->{rhs});
+                };
+        /^\+$/    && do {
+                    return op_plus($node, $self->{lhs}, $self->{rhs});
+                };
+        /^-$/    && do {
+                    return op_minus($node, $self->{lhs}, $self->{rhs});
+                };
+        /^div$/    && do {
+                    return op_div($node, $self->{lhs}, $self->{rhs});
+                };
+        /^mod$/    && do {
+                    return op_mod($node, $self->{lhs}, $self->{rhs});
+                };
+        /^\*$/    && do {
+                    return op_mult($node, $self->{lhs}, $self->{rhs});
+                };
+        /^\|$/    && do {
+                    return op_union($node, $self->{lhs}, $self->{rhs});
+                };
+        
+        die "No such operator, or operator unimplemented in ", $self->as_string, "\n";
+    }
+}
+
+# Operators
+
+use XML::XPath::Boolean;
+
+sub op_or {
+    my ($node, $lhs, $rhs) = @_;
+    if($lhs->evaluate($node)->to_boolean->value) {
+        return XML::XPath::Boolean->True;
+    }
+    else {
+        return $rhs->evaluate($node)->to_boolean;
+    }
+}
+
+sub op_and {
+    my ($node, $lhs, $rhs) = @_;
+    if( ! $lhs->evaluate($node)->to_boolean->value ) {
+        return XML::XPath::Boolean->False;
+    }
+    else {
+        return $rhs->evaluate($node)->to_boolean;
+    }
+}
+
+sub op_equals {
+    my ($node, $lhs, $rhs) = @_;
+
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    if ($lh_results->isa('XML::XPath::NodeSet') &&
+            $rh_results->isa('XML::XPath::NodeSet')) {
+        # True if and only if there is a node in the
+        # first set and a node in the second set such
+        # that the result of performing the comparison
+        # on the string-values of the two nodes is true.
+        foreach my $lhnode ($lh_results->get_nodelist) {
+            foreach my $rhnode ($rh_results->get_nodelist) {
+                if ($lhnode->string_value eq $rhnode->string_value) {
+                    return XML::XPath::Boolean->True;
+                }
+            }
+        }
+        return XML::XPath::Boolean->False;
+    }
+    elsif (($lh_results->isa('XML::XPath::NodeSet') ||
+            $rh_results->isa('XML::XPath::NodeSet')) &&
+            (!$lh_results->isa('XML::XPath::NodeSet') ||
+             !$rh_results->isa('XML::XPath::NodeSet'))) {
+        # (that says: one is a nodeset, and one is not a nodeset)
+        
+        my ($nodeset, $other);
+        if ($lh_results->isa('XML::XPath::NodeSet')) {
+            $nodeset = $lh_results;
+            $other = $rh_results;
+        }
+        else {
+            $nodeset = $rh_results;
+            $other = $lh_results;
+        }
+        
+        # True if and only if there is a node in the
+        # nodeset such that the result of performing
+        # the comparison on <type>(string_value($node))
+        # is true.
+        if ($other->isa('XML::XPath::Number')) {
+            foreach my $node ($nodeset->get_nodelist) {
+                if ($node->string_value == $other->value) {
+                    return XML::XPath::Boolean->True;
+                }
+            }
+        }
+        elsif ($other->isa('XML::XPath::Literal')) {
+            foreach my $node ($nodeset->get_nodelist) {
+                if ($node->string_value eq $other->value) {
+                    return XML::XPath::Boolean->True;
+                }
+            }
+        }
+        elsif ($other->isa('XML::XPath::Boolean')) {
+            if ($nodeset->to_boolean->value == $other->value) {
+                return XML::XPath::Boolean->True;
+            }
+        }
+
+        return XML::XPath::Boolean->False;
+    }
+    else { # Neither is a nodeset
+        if ($lh_results->isa('XML::XPath::Boolean') ||
+            $rh_results->isa('XML::XPath::Boolean')) {
+            # if either is a boolean
+            if ($lh_results->to_boolean->value == $rh_results->to_boolean->value) {
+                return XML::XPath::Boolean->True;
+            }
+            return XML::XPath::Boolean->False;
+        }
+        elsif ($lh_results->isa('XML::XPath::Number') ||
+                $rh_results->isa('XML::XPath::Number')) {
+            # if either is a number
+            local $^W; # 'number' might result in undef
+            if ($lh_results->to_number->value == $rh_results->to_number->value) {
+                return XML::XPath::Boolean->True;
+            }
+            return XML::XPath::Boolean->False;
+        }
+        else {
+            if ($lh_results->to_literal->value eq $rh_results->to_literal->value) {
+                return XML::XPath::Boolean->True;
+            }
+            return XML::XPath::Boolean->False;
+        }
+    }
+}
+
+sub op_nequals {
+    my ($node, $lhs, $rhs) = @_;
+    if (op_equals($node, $lhs, $rhs)->value) {
+        return XML::XPath::Boolean->False;
+    }
+    return XML::XPath::Boolean->True;
+}
+
+sub op_le {
+    my ($node, $lhs, $rhs) = @_;
+    op_gt($node, $rhs, $lhs);
+}
+
+sub op_ge {
+    my ($node, $lhs, $rhs) = @_;
+
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    if ($lh_results->isa('XML::XPath::NodeSet') &&
+        $rh_results->isa('XML::XPath::NodeSet')) {
+
+        foreach my $lhnode ($lh_results->get_nodelist) {
+            foreach my $rhnode ($rh_results->get_nodelist) {
+                my $lhNum = XML::XPath::Number->new($lhnode->string_value);
+                my $rhNum = XML::XPath::Number->new($rhnode->string_value);
+                if ($lhNum->value >= $rhNum->value) {
+                    return XML::XPath::Boolean->True;
+                }
+            }
+        }
+        return XML::XPath::Boolean->False;
+    }
+    elsif (($lh_results->isa('XML::XPath::NodeSet') ||
+            $rh_results->isa('XML::XPath::NodeSet')) &&
+            (!$lh_results->isa('XML::XPath::NodeSet') ||
+             !$rh_results->isa('XML::XPath::NodeSet'))) {
+        # (that says: one is a nodeset, and one is not a nodeset)
+
+        my ($nodeset, $other);
+        my ($true, $false);
+        if ($lh_results->isa('XML::XPath::NodeSet')) {
+            $nodeset = $lh_results;
+            $other = $rh_results;
+            # we do this because unlike ==, these ops are direction dependant
+            ($false, $true) = (XML::XPath::Boolean->False, XML::XPath::Boolean->True);
+        }
+        else {
+            $nodeset = $rh_results;
+            $other = $lh_results;
+            # ditto above comment
+            ($true, $false) = (XML::XPath::Boolean->False, XML::XPath::Boolean->True);
+        }
+        
+        # True if and only if there is a node in the
+        # nodeset such that the result of performing
+        # the comparison on <type>(string_value($node))
+        # is true.
+        foreach my $node ($nodeset->get_nodelist) {
+            if ($node->to_number->value >= $other->to_number->value) {
+                return $true;
+            }
+        }
+        return $false;
+    }
+    else { # Neither is a nodeset
+        if ($lh_results->isa('XML::XPath::Boolean') ||
+            $rh_results->isa('XML::XPath::Boolean')) {
+            # if either is a boolean
+            if ($lh_results->to_boolean->to_number->value
+                    >= $rh_results->to_boolean->to_number->value) {
+                return XML::XPath::Boolean->True;
+            }
+        }
+        else {
+            if ($lh_results->to_number->value >= $rh_results->to_number->value) {
+                return XML::XPath::Boolean->True;
+            }
+        }
+        return XML::XPath::Boolean->False;
+    }
+}
+
+sub op_gt {
+    my ($node, $lhs, $rhs) = @_;
+
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    if ($lh_results->isa('XML::XPath::NodeSet') &&
+        $rh_results->isa('XML::XPath::NodeSet')) {
+
+        foreach my $lhnode ($lh_results->get_nodelist) {
+            foreach my $rhnode ($rh_results->get_nodelist) {
+                my $lhNum = XML::XPath::Number->new($lhnode->string_value);
+                my $rhNum = XML::XPath::Number->new($rhnode->string_value);
+                if ($lhNum->value > $rhNum->value) {
+                    return XML::XPath::Boolean->True;
+                }
+            }
+        }
+        return XML::XPath::Boolean->False;
+    }
+    elsif (($lh_results->isa('XML::XPath::NodeSet') ||
+            $rh_results->isa('XML::XPath::NodeSet')) &&
+            (!$lh_results->isa('XML::XPath::NodeSet') ||
+             !$rh_results->isa('XML::XPath::NodeSet'))) {
+        # (that says: one is a nodeset, and one is not a nodeset)
+
+        my ($nodeset, $other);
+        my ($true, $false);
+        if ($lh_results->isa('XML::XPath::NodeSet')) {
+            $nodeset = $lh_results;
+            $other = $rh_results;
+            # we do this because unlike ==, these ops are direction dependant
+            ($false, $true) = (XML::XPath::Boolean->False, XML::XPath::Boolean->True);
+        }
+        else {
+            $nodeset = $rh_results;
+            $other = $lh_results;
+            # ditto above comment
+            ($true, $false) = (XML::XPath::Boolean->False, XML::XPath::Boolean->True);
+        }
+        
+        # True if and only if there is a node in the
+        # nodeset such that the result of performing
+        # the comparison on <type>(string_value($node))
+        # is true.
+        foreach my $node ($nodeset->get_nodelist) {
+            if ($node->to_number->value > $other->to_number->value) {
+                return $true;
+            }
+        }
+        return $false;
+    }
+    else { # Neither is a nodeset
+        if ($lh_results->isa('XML::XPath::Boolean') ||
+            $rh_results->isa('XML::XPath::Boolean')) {
+            # if either is a boolean
+            if ($lh_results->to_boolean->value > $rh_results->to_boolean->value) {
+                return XML::XPath::Boolean->True;
+            }
+        }
+        else {
+            if ($lh_results->to_number->value > $rh_results->to_number->value) {
+                return XML::XPath::Boolean->True;
+            }
+        }
+        return XML::XPath::Boolean->False;
+    }
+}
+
+sub op_lt {
+    my ($node, $lhs, $rhs) = @_;
+    op_gt($node, $rhs, $lhs);
+}
+
+sub op_plus {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    my $result =
+        $lh_results->to_number->value
+            +
+        $rh_results->to_number->value
+            ;
+    return XML::XPath::Number->new($result);
+}
+
+sub op_minus {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    my $result =
+        $lh_results->to_number->value
+            -
+        $rh_results->to_number->value
+            ;
+    return XML::XPath::Number->new($result);
+}
+
+sub op_div {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+
+    my $result = eval {
+        $lh_results->to_number->value
+            /
+        $rh_results->to_number->value
+            ;
+    };
+    if ($@) {
+        # assume divide by zero
+        # This is probably a terrible way to handle this! 
+        # Ah well... who wants to live forever...
+        return XML::XPath::Literal->new('Infinity');
+    }
+    return XML::XPath::Number->new($result);
+}
+
+sub op_mod {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    my $result =
+        $lh_results->to_number->value
+            %
+        $rh_results->to_number->value
+            ;
+    return XML::XPath::Number->new($result);
+}
+
+sub op_mult {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_results = $lhs->evaluate($node);
+    my $rh_results = $rhs->evaluate($node);
+    
+    my $result =
+        $lh_results->to_number->value
+            *
+        $rh_results->to_number->value
+            ;
+    return XML::XPath::Number->new($result);
+}
+
+sub op_union {
+    my ($node, $lhs, $rhs) = @_;
+    my $lh_result = $lhs->evaluate($node);
+    my $rh_result = $rhs->evaluate($node);
+    
+    if ($lh_result->isa('XML::XPath::NodeSet') &&
+            $rh_result->isa('XML::XPath::NodeSet')) {
+        my %found;
+        my $results = XML::XPath::NodeSet->new;
+        foreach my $lhnode ($lh_result->get_nodelist) {
+            $found{"$lhnode"}++;
+            $results->push($lhnode);
+        }
+        foreach my $rhnode ($rh_result->get_nodelist) {
+            $results->push($rhnode)
+                    unless exists $found{"$rhnode"};
+        }
+                $results->sort;
+        return $results;
+    }
+    die "Both sides of a union must be Node Sets\n";
+}
+
+sub filter_by_predicate {
+    my $self = shift;
+    my ($nodeset, $predicate) = @_;
+    
+    # See spec section 2.4, paragraphs 2 & 3:
+    # For each node in the node-set to be filtered, the predicate Expr
+    # is evaluated with that node as the context node, with the number
+    # of nodes in the node set as the context size, and with the
+    # proximity position of the node in the node set with respect to
+    # the axis as the context position.
+    
+    if (!ref($nodeset)) { # use ref because nodeset has a bool context
+        die "No nodeset!!!";
+    }
+    
+#    warn "Filter by predicate: $predicate\n";
+    
+    my $newset = XML::XPath::NodeSet->new();
+    
+    for(my $i = 1; $i <= $nodeset->size; $i++) {
+        # set context set each time 'cos a loc-path in the expr could change it
+        $self->{pp}->set_context_set($nodeset);
+        $self->{pp}->set_context_pos($i);
+        my $result = $predicate->evaluate($nodeset->get_node($i));
+        if ($result->isa('XML::XPath::Boolean')) {
+            if ($result->value) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+        elsif ($result->isa('XML::XPath::Number')) {
+            if ($result->value == $i) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+        else {
+            if ($result->to_boolean->value) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+    }
+    
+    return $newset;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Function.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Function.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Function.pm	(revision 21112)
@@ -0,0 +1,392 @@
+# $Id$
+
+package XML::XPath::Function;
+use XML::XPath::Number;
+use XML::XPath::Literal;
+use XML::XPath::Boolean;
+use XML::XPath::NodeSet;
+use XML::XPath::Node::Attribute;
+use strict;
+
+sub new {
+    my $class = shift;
+    my ($pp, $name, $params) = @_;
+    bless { 
+        pp => $pp, 
+        name => $name, 
+        params => $params 
+        }, $class;
+}
+
+sub as_string {
+    my $self = shift;
+    my $string = $self->{name} . "(";
+    my $second;
+    foreach (@{$self->{params}}) {
+        $string .= "," if $second++;
+        $string .= $_->as_string;
+    }
+    $string .= ")";
+    return $string;
+}
+
+sub as_xml {
+    my $self = shift;
+    my $string = "<Function name=\"$self->{name}\"";
+    my $params = "";
+    foreach (@{$self->{params}}) {
+        $params .= "<Param>" . $_->as_string . "</Param>\n";
+    }
+    if ($params) {
+        $string .= ">\n$params</Function>\n";
+    }
+    else {
+        $string .= " />\n";
+    }
+    
+    return $string;
+}
+
+sub evaluate {
+    my $self = shift;
+    my $node = shift;
+    if ($node->isa('XML::XPath::NodeSet')) {
+        $node = $node->get_node(1);
+    }
+    my @params;
+    foreach my $param (@{$self->{params}}) {
+        my $results = $param->evaluate($node);
+        push @params, $results;
+    }
+    $self->_execute($self->{name}, $node, @params);
+}
+
+sub _execute {
+    my $self = shift;
+    my ($name, $node, @params) = @_;
+    $name =~ s/-/_/g;
+    no strict 'refs';
+    $self->$name($node, @params);
+}
+
+# All functions should return one of:
+# XML::XPath::Number
+# XML::XPath::Literal (string)
+# XML::XPath::NodeSet
+# XML::XPath::Boolean
+
+### NODESET FUNCTIONS ###
+
+sub last {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "last: function doesn't take parameters\n" if (@params);
+    return XML::XPath::Number->new($self->{pp}->get_context_size);
+}
+
+sub position {
+    my $self = shift;
+    my ($node, @params) = @_;
+    if (@params) {
+        die "position: function doesn't take parameters [ ", @params, " ]\n";
+    }
+    # return pos relative to axis direction
+    return XML::XPath::Number->new($self->{pp}->get_context_pos);
+}
+
+sub count {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "count: Parameter must be a NodeSet\n" unless $params[0]->isa('XML::XPath::NodeSet');
+    return XML::XPath::Number->new($params[0]->size);
+}
+
+sub id {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "id: Function takes 1 parameter\n" unless @params == 1;
+    my $results = XML::XPath::NodeSet->new();
+    if ($params[0]->isa('XML::XPath::NodeSet')) {
+        # result is the union of applying id() to the
+        # string value of each node in the nodeset.
+        foreach my $node ($params[0]->get_nodelist) {
+            my $string = $node->string_value;
+            $results->append($self->id($node, XML::XPath::Literal->new($string)));
+        }
+    }
+    else { # The actual id() function...
+        my $string = $self->string($node, $params[0]);
+        $_ = $string->value; # get perl scalar
+        my @ids = split; # splits $_
+        foreach my $id (@ids) {
+            if (my $found = $node->getElementById($id)) {
+                $results->push($found);
+            }
+        }
+    }
+    return $results;
+}
+
+sub local_name {
+    my $self = shift;
+    my ($node, @params) = @_;
+    if (@params > 1) {
+        die "name() function takes one or no parameters\n";
+    }
+    elsif (@params) {
+        my $nodeset = shift(@params);
+        $node = $nodeset->get_node(1);
+    }
+    
+    return XML::XPath::Literal->new($node->getLocalName);
+}
+
+sub namespace_uri {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "namespace-uri: Function not supported\n";
+}
+
+sub name {
+    my $self = shift;
+    my ($node, @params) = @_;
+    if (@params > 1) {
+        die "name() function takes one or no parameters\n";
+    }
+    elsif (@params) {
+        my $nodeset = shift(@params);
+        $node = $nodeset->get_node(1);
+    }
+    
+    return XML::XPath::Literal->new($node->getName);
+}
+
+### STRING FUNCTIONS ###
+
+sub string {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "string: Too many parameters\n" if @params > 1;
+    if (@params) {
+        return XML::XPath::Literal->new($params[0]->string_value);
+    }
+    
+    # TODO - this MUST be wrong! - not sure now. -matt
+    return XML::XPath::Literal->new($node->string_value);
+    # default to nodeset with just $node in.
+}
+
+sub concat {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "concat: Too few parameters\n" if @params < 2;
+    my $string = join('', map {$_->string_value} @params);
+    return XML::XPath::Literal->new($string);
+}
+
+sub starts_with {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "starts-with: incorrect number of params\n" unless @params == 2;
+    my ($string1, $string2) = ($params[0]->string_value, $params[1]->string_value);
+    if (substr($string1, 0, length($string2)) eq $string2) {
+        return XML::XPath::Boolean->True;
+    }
+    return XML::XPath::Boolean->False;
+}
+
+sub contains {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "starts-with: incorrect number of params\n" unless @params == 2;
+    my $value = $params[1]->string_value;
+    if ($params[0]->string_value =~ /(.*?)\Q$value\E(.*)/) {
+        # $1 and $2 stored for substring funcs below
+        # TODO: Fix this nasty implementation!
+        return XML::XPath::Boolean->True;
+    }
+    return XML::XPath::Boolean->False;
+}
+
+sub substring_before {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "starts-with: incorrect number of params\n" unless @params == 2;
+    if ($self->contains($node, @params)->value) {
+        return XML::XPath::Literal->new($1); # hope that works!
+    }
+    else {
+        return XML::XPath::Literal->new('');
+    }
+}
+
+sub substring_after {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "starts-with: incorrect number of params\n" unless @params == 2;
+    if ($self->contains($node, @params)->value) {
+        return XML::XPath::Literal->new($2);
+    }
+    else {
+        return XML::XPath::Literal->new('');
+    }
+}
+
+sub substring {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "substring: Wrong number of parameters\n" if (@params < 2 || @params > 3);
+    my ($str, $offset, $len);
+    $str = $params[0]->string_value;
+    $offset = $params[1]->value;
+    $offset--; # uses 1 based offsets
+    if (@params == 3) {
+        $len = $params[2]->value;
+    }
+    return XML::XPath::Literal->new(substr($str, $offset, $len));
+}
+
+sub string_length {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "string-length: Wrong number of params\n" if @params > 1;
+    if (@params) {
+        return XML::XPath::Number->new(length($params[0]->string_value));
+    }
+    else {
+        return XML::XPath::Number->new(
+                length($node->string_value)
+                );
+    }
+}
+
+sub normalize_space {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "normalize-space: Wrong number of params\n" if @params > 1;
+    my $str;
+    if (@params) {
+        $str = $params[0]->string_value;
+    }
+    else {
+        $str = $node->string_value;
+    }
+    $str =~ s/^\s*//;
+    $str =~ s/\s*$//;
+    $str =~ s/\s+/ /g;
+    return XML::XPath::Literal->new($str);
+}
+
+sub translate {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "translate: Wrong number of params\n" if @params != 3;
+    local $_ = $params[0]->string_value;
+    my $find = $params[1]->string_value;
+    my $repl = $params[2]->string_value;
+    eval "tr/\\Q$find\\E/\\Q$repl\\E/d, 1" or die $@;
+    return XML::XPath::Literal->new($_);
+}
+
+### BOOLEAN FUNCTIONS ###
+
+sub boolean {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "boolean: Incorrect number of parameters\n" if @params != 1;
+    return $params[0]->to_boolean;
+}
+
+sub not {
+    my $self = shift;
+    my ($node, @params) = @_;
+    $params[0] = $params[0]->to_boolean unless $params[0]->isa('XML::XPath::Boolean');
+    $params[0]->value ? XML::XPath::Boolean->False : XML::XPath::Boolean->True;
+}
+
+sub true {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "true: function takes no parameters\n" if @params > 0;
+    XML::XPath::Boolean->True;
+}
+
+sub false {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "true: function takes no parameters\n" if @params > 0;
+    XML::XPath::Boolean->False;
+}
+
+sub lang {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "lang: function takes 1 parameter\n" if @params != 1;
+    my $lang = $node->findvalue('(ancestor-or-self::*[@xml:lang]/@xml:lang)[last()]');
+    my $lclang = lc($params[0]->string_value);
+    # warn("Looking for lang($lclang) in $lang\n");
+    if (substr(lc($lang), 0, length($lclang)) eq $lclang) {
+        return XML::XPath::Boolean->True;
+    }
+    else {
+        return XML::XPath::Boolean->False;
+    }
+}
+
+### NUMBER FUNCTIONS ###
+
+sub number {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "number: Too many parameters\n" if @params > 1;
+    if (@params) {
+        if ($params[0]->isa('XML::XPath::Node')) {
+            return XML::XPath::Number->new(
+                    $params[0]->string_value
+                    );
+        }
+        return $params[0]->to_number;
+    }
+    
+    return XML::XPath::Number->new( $node->string_value );
+}
+
+sub sum {
+    my $self = shift;
+    my ($node, @params) = @_;
+    die "sum: Parameter must be a NodeSet\n" unless $params[0]->isa('XML::XPath::NodeSet');
+    my $sum = 0;
+    foreach my $node ($params[0]->get_nodelist) {
+        $sum += $self->number($node)->value;
+    }
+    return XML::XPath::Number->new($sum);
+}
+
+sub floor {
+    my $self = shift;
+    my ($node, @params) = @_;
+    require POSIX;
+    my $num = $self->number($node, @params);
+    return XML::XPath::Number->new(
+            POSIX::floor($num->value));
+}
+
+sub ceiling {
+    my $self = shift;
+    my ($node, @params) = @_;
+    require POSIX;
+    my $num = $self->number($node, @params);
+    return XML::XPath::Number->new(
+            POSIX::ceil($num->value));
+}
+
+sub round {
+    my $self = shift;
+    my ($node, @params) = @_;
+    my $num = $self->number($node, @params);
+    require POSIX;
+    return XML::XPath::Number->new(
+            POSIX::floor($num->value + 0.5)); # Yes, I know the spec says don't do this...
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Literal.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Literal.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Literal.pm	(revision 21112)
@@ -0,0 +1,99 @@
+# $Id$
+
+package XML::XPath::Literal;
+use XML::XPath::Boolean;
+use XML::XPath::Number;
+use strict;
+
+use overload 
+		'""' => \&value,
+		'cmp' => \&cmp;
+
+sub new {
+	my $class = shift;
+	my ($string) = @_;
+	
+#	$string =~ s/&quot;/"/g;
+#	$string =~ s/&apos;/'/g;
+	
+	bless \$string, $class;
+}
+
+sub as_string {
+	my $self = shift;
+	my $string = $$self;
+	$string =~ s/'/&apos;/g;
+	return "'$string'";
+}
+
+sub as_xml {
+    my $self = shift;
+    my $string = $$self;
+    return "<Literal>$string</Literal>\n";
+}
+
+sub value {
+	my $self = shift;
+	$$self;
+}
+
+sub cmp {
+	my $self = shift;
+	my ($cmp, $swap) = @_;
+	if ($swap) {
+		return $cmp cmp $$self;
+	}
+	return $$self cmp $cmp;
+}
+
+sub evaluate {
+	my $self = shift;
+	$self;
+}
+
+sub to_boolean {
+	my $self = shift;
+	return (length($$self) > 0) ? XML::XPath::Boolean->True : XML::XPath::Boolean->False;
+}
+
+sub to_number { return XML::XPath::Number->new($_[0]->value); }
+sub to_literal { return $_[0]; }
+
+sub string_value { return $_[0]->value; }
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath::Literal - Simple string values.
+
+=head1 DESCRIPTION
+
+In XPath terms a Literal is what we know as a string.
+
+=head1 API
+
+=head2 new($string)
+
+Create a new Literal object with the value in $string. Note that &quot; and
+&apos; will be converted to " and ' respectively. That is not part of the XPath
+specification, but I consider it useful. Note though that you have to go
+to extraordinary lengths in an XML template file (be it XSLT or whatever) to
+make use of this:
+
+	<xsl:value-of select="&quot;I'm feeling &amp;quot;sad&amp;quot;&quot;"/>
+
+Which produces a Literal of:
+
+	I'm feeling "sad"
+
+=head2 value()
+
+Also overloaded as stringification, simply returns the literal string value.
+
+=head2 cmp($literal)
+
+Returns the equivalent of perl's cmp operator against the given $literal.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/LocationPath.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/LocationPath.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/LocationPath.pm	(revision 21112)
@@ -0,0 +1,61 @@
+# $Id$
+
+package XML::XPath::LocationPath;
+use XML::XPath::Root;
+use strict;
+
+sub new {
+	my $class = shift;
+	my $self = [];
+	bless $self, $class;
+}
+
+sub as_string {
+	my $self = shift;
+	my $string;
+	for (my $i = 0; $i < @$self; $i++) {
+		$string .= $self->[$i]->as_string;
+		$string .= "/" if $self->[$i+1];
+	}
+	return $string;
+}
+
+sub as_xml {
+    my $self = shift;
+    my $string = "<LocationPath>\n";
+    
+    for (my $i = 0; $i < @$self; $i++) {
+        $string .= $self->[$i]->as_xml;
+    }
+    
+    $string .= "</LocationPath>\n";
+    return $string;
+}
+
+sub set_root {
+	my $self = shift;
+	unshift @$self, XML::XPath::Root->new();
+}
+
+sub evaluate {
+	my $self = shift;
+	# context _MUST_ be a single node
+	my $context = shift;
+	die "No context" unless $context;
+	
+	# I _think_ this is how it should work :)
+	
+	my $nodeset = XML::XPath::NodeSet->new();
+	$nodeset->push($context);
+	
+	foreach my $step (@$self) {
+		# For each step
+		# evaluate the step with the nodeset
+		my $pos = 1;
+		$nodeset = $step->evaluate($nodeset);
+	}
+	
+	return $nodeset;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node.pm	(revision 21112)
@@ -0,0 +1,592 @@
+# $Id$
+
+package XML::XPath::Node;
+
+use strict;
+use vars qw(@ISA @EXPORT $AUTOLOAD %EXPORT_TAGS @EXPORT_OK);
+use Exporter;
+use Carp;
+@ISA = ('Exporter');
+
+sub UNKNOWN_NODE () {0;}
+sub ELEMENT_NODE () {1;}
+sub ATTRIBUTE_NODE () {2;}
+sub TEXT_NODE () {3;}
+sub CDATA_SECTION_NODE () {4;}
+sub ENTITY_REFERENCE_NODE () {5;}
+sub ENTITY_NODE () {6;}
+sub PROCESSING_INSTRUCTION_NODE () {7;}
+sub COMMENT_NODE () {8;}
+sub DOCUMENT_NODE () {9;}
+sub DOCUMENT_TYPE_NODE () {10;}
+sub DOCUMENT_FRAGMENT_NODE () {11;}
+sub NOTATION_NODE () {12;}
+
+# Non core DOM stuff
+sub ELEMENT_DECL_NODE () {13;}
+sub ATT_DEF_NODE () {14;}
+sub XML_DECL_NODE () {15;}
+sub ATTLIST_DECL_NODE () {16;}
+sub NAMESPACE_NODE () {17;}
+
+# per-node constants
+
+# All
+sub node_parent () { 0; }
+sub node_pos () { 1; }
+sub node_global_pos () { 2; }
+
+# Element
+sub node_prefix () { 3; }
+sub node_children () { 4; }
+sub node_name () { 5; }
+sub node_attribs () { 6; }
+sub node_namespaces () { 7; }
+sub node_ids () { 8; }
+
+# Char
+sub node_text () { 3; }
+
+# PI
+sub node_target () { 3; }
+sub node_data () { 4; }
+
+# Comment
+sub node_comment () { 3; }
+
+# Attribute
+# sub node_prefix () { 3; }
+sub node_key () { 4; }
+sub node_value () { 5; }
+
+# Namespaces
+# sub node_prefix () { 3; }
+sub node_expanded () { 4; }
+
+@EXPORT = qw(
+    UNKNOWN_NODE
+    ELEMENT_NODE
+    ATTRIBUTE_NODE
+    TEXT_NODE
+    CDATA_SECTION_NODE
+    ENTITY_REFERENCE_NODE
+    ENTITY_NODE
+    PROCESSING_INSTRUCTION_NODE
+    COMMENT_NODE
+    DOCUMENT_NODE
+    DOCUMENT_TYPE_NODE
+    DOCUMENT_FRAGMENT_NODE
+    NOTATION_NODE
+    ELEMENT_DECL_NODE
+    ATT_DEF_NODE
+    XML_DECL_NODE
+    ATTLIST_DECL_NODE
+    NAMESPACE_NODE
+    );
+
+@EXPORT_OK = qw(
+            node_parent
+            node_pos
+            node_global_pos
+            node_prefix
+            node_children
+            node_name
+            node_attribs
+            node_namespaces
+            node_text
+            node_target
+            node_data
+            node_comment
+            node_key
+            node_value
+            node_expanded
+                        node_ids
+        );
+
+%EXPORT_TAGS = (
+    'node_keys' => [
+        qw(
+            node_parent
+            node_pos
+            node_global_pos
+            node_prefix
+            node_children
+            node_name
+            node_attribs
+            node_namespaces
+            node_text
+            node_target
+            node_data
+            node_comment
+            node_key
+            node_value
+            node_expanded
+                        node_ids
+        ), @EXPORT,
+    ],
+);
+
+
+my $global_pos = 0;
+
+sub nextPos {
+    my $class = shift;
+    return $global_pos += 5;
+}
+
+sub resetPos {
+    $global_pos = 0;
+}
+
+my %DecodeDefaultEntity =
+(
+ '"' => "&quot;",
+ ">" => "&gt;",
+ "<" => "&lt;",
+ "'" => "&apos;",
+ "&" => "&amp;"
+);
+
+sub XMLescape {
+    my ($str, $default) = @_;
+    return undef unless defined $str;
+    $default ||= '';
+    
+    if ($XML::XPath::EncodeUtf8AsEntity) {
+        $str =~ s/([\xC0-\xDF].|[\xE0-\xEF]..|[\xF0-\xFF]...)|([$default])|(]]>)/
+        defined($1) ? XmlUtf8Decode ($1) : 
+        defined ($2) ? $DecodeDefaultEntity{$2} : "]]&gt;" /egsx;
+    }
+    else {
+        $str =~ s/([$default])|(]]>)/
+        defined ($1) ? $DecodeDefaultEntity{$1} : ']]&gt;' /gsex;
+    }
+
+#?? could there be references that should not be expanded?
+# e.g. should not replace &#nn; &#xAF; and &abc;
+#    $str =~ s/&(?!($ReName|#[0-9]+|#x[0-9a-fA-F]+);)/&amp;/go;
+
+    $str;
+}
+
+#
+# Opposite of XmlUtf8Decode plus it adds prefix "&#" or "&#x" and suffix ";"
+# The 2nd parameter ($hex) indicates whether the result is hex encoded or not.
+#
+sub XmlUtf8Decode
+{
+    my ($str, $hex) = @_;
+    my $len = length ($str);
+    my $n;
+
+    if ($len == 2) {
+        my @n = unpack "C2", $str;
+        $n = (($n[0] & 0x3f) << 6) + ($n[1] & 0x3f);
+    }
+    elsif ($len == 3) {
+        my @n = unpack "C3", $str;
+        $n = (($n[0] & 0x1f) << 12) + (($n[1] & 0x3f) << 6) + 
+            ($n[2] & 0x3f);
+    }
+    elsif ($len == 4) {
+        my @n = unpack "C4", $str;
+        $n = (($n[0] & 0x0f) << 18) + (($n[1] & 0x3f) << 12) + 
+            (($n[2] & 0x3f) << 6) + ($n[3] & 0x3f);
+    }
+    elsif ($len == 1) {    # just to be complete...
+        $n = ord ($str);
+    }
+    else {
+        die "bad value [$str] for XmlUtf8Decode";
+    }
+    $hex ? sprintf ("&#x%x;", $n) : "&#$n;";
+}
+
+sub new {
+    my $class = shift;
+    no strict 'refs';
+    my $impl = $class . "Impl";
+    my $this = $impl->new(@_);
+    if ($XML::XPath::SafeMode) {
+        return $this;
+    }
+    my $self = \$this;
+    return bless $self, $class;
+}
+
+sub AUTOLOAD {
+    my $method = $AUTOLOAD;
+    $method =~ s/.*:://;
+#    warn "AUTOLOAD $method!\n";
+    no strict 'refs';
+    *{$AUTOLOAD} = sub { 
+        my $self = shift;
+        my $olderror = $@; # store previous exceptions
+        my $obj = eval { $$self };
+        if ($@) {
+            if ($@ =~ /Not a SCALAR reference/) {
+                croak("No such method $method in " . ref($self));
+            }
+            croak $@;
+        }
+        if ($obj) {
+            # make sure $@ propogates if this method call was the result
+            # of losing scope because of a die().
+            if ($method =~ /^(DESTROY|del_parent_link)$/) {
+                $obj->$method(@_);
+                $@ = $olderror if $olderror;
+                return;
+            }
+            return $obj->$method(@_);
+        }
+    };
+    goto &$AUTOLOAD;
+}
+
+package XML::XPath::NodeImpl;
+
+use vars qw/@ISA $AUTOLOAD/;
+@ISA = ('XML::XPath::Node');
+
+sub new {
+    die "Virtual base method";
+}
+
+sub getNodeType {
+    my $self = shift;
+    return XML::XPath::Node::UNKNOWN_NODE;
+}
+
+sub isElementNode {}
+sub isAttributeNode {}
+sub isNamespaceNode {}
+sub isTextNode {}
+sub isProcessingInstructionNode {}
+sub isPINode {}
+sub isCommentNode {}
+
+sub getNodeValue {
+    return;
+}
+
+sub getValue {
+    shift->getNodeValue(@_);
+}
+
+sub setNodeValue {
+    return;
+}
+
+sub setValue {
+    shift->setNodeValue(@_);
+}
+
+sub getParentNode {
+    my $self = shift;
+    return $self->[XML::XPath::Node::node_parent];
+}
+
+sub getRootNode {
+    my $self = shift;
+    while (my $parent = $self->getParentNode) {
+        $self = $parent;
+    }
+    return $self;
+}
+
+sub getElementById {
+    my $self = shift;
+    my ($id) = @_;
+#    warn "getElementById: $id\n";
+    my $root = $self->getRootNode;
+    my $node = $root->[XML::XPath::Node::node_ids]{$id};
+#    warn "returning node: ", $node->getName, "\n";
+    return $node;
+}
+
+sub getName { }
+sub getData { }
+
+sub getChildNodes {
+    return wantarray ? () : [];
+}
+
+sub getChildNode {
+    return;
+}
+
+sub getAttribute {
+    return;
+}
+
+sub getAttributes {
+    return wantarray ? () : [];
+}
+
+sub getAttributeNodes {
+    shift->getAttributes(@_);
+}
+
+sub getNamespaceNodes {
+    return wantarray ? () : [];
+}
+
+sub getNamespace {
+    return;
+}
+
+sub getLocalName {
+    return;
+}
+
+sub string_value { return; }
+
+sub get_pos {
+    my $self = shift;
+    return $self->[XML::XPath::Node::node_pos];
+}
+
+sub set_pos {
+    my $self = shift;
+    $self->[XML::XPath::Node::node_pos] = shift;
+}
+
+sub get_global_pos {
+    my $self = shift;
+    return $self->[XML::XPath::Node::node_global_pos];
+}
+
+sub set_global_pos {
+    my $self = shift;
+    $self->[XML::XPath::Node::node_global_pos] = shift;
+}
+
+sub renumber {
+    my $self = shift;
+    my $search = shift;
+    my $diff = shift;
+    
+    foreach my $node ($self->findnodes($search)) {
+        $node->set_global_pos(
+                $node->get_global_pos + $diff
+                );
+    }
+}
+    
+sub insertAfter {
+    my $self = shift;
+    my $newnode = shift;
+    my $posnode = shift;
+
+    my $pos_number = eval { $posnode->[XML::XPath::Node::node_children][-1]->get_global_pos() + 1; };
+    if (!defined $pos_number) {
+        $pos_number = $posnode->get_global_pos() + 1;
+    }
+    
+    eval {
+        if ($pos_number == 
+                $posnode->findnodes(
+                    'following::node()'
+                    )->get_node(1)->get_global_pos()) {
+            $posnode->renumber('following::node()', +5);
+        }
+    };
+    
+    my $pos = $posnode->get_pos;
+    
+    $newnode->setParentNode($self);
+    splice @{$self->[XML::XPath::Node::node_children]}, $pos + 1, 0, $newnode;
+    
+    for (my $i = $pos + 1; $i < @{$self->[XML::XPath::Node::node_children]}; $i++) {
+        $self->[XML::XPath::Node::node_children][$i]->set_pos($i);
+    }
+    
+    $newnode->set_global_pos($pos_number);
+}
+
+sub insertBefore {
+    my $self = shift;
+    my $newnode = shift;
+    my $posnode = shift;
+    
+    my $pos_number = ($posnode->getPreviousSibling() || $posnode->getParentNode)->get_global_pos();
+    if ($pos_number == $posnode->get_global_pos()) {
+        $posnode->renumber('self::node() | descendant::node() | following::node()', +5);
+    }
+    
+    my $pos = $posnode->get_pos;
+    
+    $newnode->setParentNode($self);
+    splice @{$self->[XML::XPath::Node::node_children]}, $pos, 0, $newnode;
+    
+    for (my $i = $pos; $i < @{$self->[XML::XPath::Node::node_children]}; $i++) {
+        $self->[XML::XPath::Node::node_children][$i]->set_pos($i);
+    }
+    
+    $newnode->set_global_pos($pos_number);
+}
+
+sub getPreviousSibling {
+    my $self = shift;
+    my $pos = $self->[XML::XPath::Node::node_pos];
+    return unless $self->[XML::XPath::Node::node_parent];
+    return $self->[XML::XPath::Node::node_parent]->getChildNode($pos);
+}
+
+sub getNextSibling {
+    my $self = shift;
+    my $pos = $self->[XML::XPath::Node::node_pos];
+    return unless $self->[XML::XPath::Node::node_parent];
+    return $self->[XML::XPath::Node::node_parent]->getChildNode($pos + 2);
+}
+
+sub setParentNode {
+    my $self = shift;
+    my $parent = shift;
+#    warn "SetParent of ", ref($self), " to ", $parent->[XML::XPath::Node::node_name], "\n";
+    $self->[XML::XPath::Node::node_parent] = $parent;
+}
+
+sub del_parent_link {
+    my $self = shift;
+    $self->[XML::XPath::Node::node_parent] = undef;
+}
+
+sub dispose {
+    my $self = shift;
+    foreach my $kid ($self->getChildNodes) {
+        $kid->dispose;
+    }
+    foreach my $kid ($self->getAttributeNodes) {
+        $kid->dispose;
+    }
+    foreach my $kid ($self->getNamespaceNodes) {
+        $kid->dispose;
+    }
+    $self->[XML::XPath::Node::node_parent] = undef;
+}
+
+sub to_number {
+    my $num = shift->string_value;
+    return XML::XPath::Number->new($num);
+}
+
+sub find {
+    my $node = shift;
+    my ($path) = @_;
+    my $xp = XML::XPath->new(); # new is v. lightweight
+    return $xp->find($path, $node);
+}
+
+sub findvalue {
+    my $node = shift;
+    my ($path) = @_;
+    my $xp = XML::XPath->new();
+    return $xp->findvalue($path, $node);
+}
+
+sub findnodes {
+    my $node = shift;
+    my ($path) = @_;
+    my $xp = XML::XPath->new();
+    return $xp->findnodes($path, $node);
+}
+
+sub matches {
+    my $node = shift;
+    my ($path, $context) = @_;
+    my $xp = XML::XPath->new();
+    return $xp->matches($node, $path, $context);
+}
+
+sub to_sax {
+    my $self = shift;
+    unshift @_, 'Handler' if @_ == 1;
+    my %handlers = @_;
+    
+    my $doch = $handlers{DocumentHandler} || $handlers{Handler};
+    my $dtdh = $handlers{DTDHandler} || $handlers{Handler};
+    my $enth = $handlers{EntityResolver} || $handlers{Handler};
+
+    $self->_to_sax ($doch, $dtdh, $enth);
+}
+
+sub DESTROY {}
+
+use Carp;
+
+sub _to_sax {
+    carp "_to_sax not implemented in ", ref($_[0]);
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath::Node - internal representation of a node
+
+=head1 API
+
+The Node API aims to emulate DOM to some extent, however the API
+isn't quite compatible with DOM. This is to ease transition from
+XML::DOM programming to XML::XPath. Compatibility with DOM may
+arise once XML::DOM gets namespace support.
+
+=head2 new
+
+Creates a new node. See the sub-classes for parameters to pass to new().
+
+=head2 getNodeType
+
+Returns one of ELEMENT_NODE, TEXT_NODE, COMMENT_NODE, ATTRIBUTE_NODE,
+PROCESSING_INSTRUCTION_NODE or NAMESPACE_NODE. UNKNOWN_NODE is returned
+if the sub-class doesn't implement getNodeType - but that means
+something is broken! The constants are exported by default from
+XML::XPath::Node. The constants have the same numeric value as the
+XML::DOM versions.
+
+=head2 getParentNode
+
+Returns the parent of this node, or undef if this is the root node. Note
+that the root node is the root node in terms of XPath - not the root
+element node.
+
+=head2 to_sax ( $handler | %handlers )
+
+Generates sax calls to the handler or handlers. See the PerlSAX docs for
+details (not yet implemented correctly).
+
+=head1 MORE INFO
+
+See the sub-classes for the meaning of the rest of the API:
+
+=over 4
+
+=item *
+
+L<XML::XPath::Node::Element>
+
+=item *
+
+L<XML::XPath::Node::Attribute>
+
+=item *
+
+L<XML::XPath::Node::Namespace>
+
+=item *
+
+L<XML::XPath::Node::Text>
+
+=item *
+
+L<XML::XPath::Node::Comment>
+
+=item *
+
+L<XML::XPath::Node::PI>
+
+=back
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Attribute.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Attribute.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Attribute.pm	(revision 21112)
@@ -0,0 +1,135 @@
+# $Id$
+
+package XML::XPath::Node::Attribute;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::AttributeImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::Attribute');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+	my $class = shift;
+	my ($key, $val, $prefix) = @_;
+	
+        my $pos = XML::XPath::Node->nextPos;
+        
+        my @vals;
+        @vals[node_global_pos, node_prefix, node_key, node_value] =
+                ($pos, $prefix, $key, $val);
+	my $self = \@vals;
+        
+	bless $self, $class;
+	
+}
+
+sub getNodeType { ATTRIBUTE_NODE }
+
+sub isAttributeNode { 1; }
+
+sub getName {
+    my $self = shift;
+    $self->[node_key];
+}
+
+sub getLocalName {
+    my $self = shift;
+    my $local = $self->[node_key];
+    $local =~ s/.*://;
+    return $local;
+}
+
+sub getNodeValue {
+    my $self = shift;
+    $self->[node_value];
+}
+
+sub getData {
+    shift->getNodeValue(@_);
+}
+
+sub setNodeValue {
+    my $self = shift;
+    $self->[node_value] = shift;
+}
+
+sub getPrefix {
+	my $self = shift;
+	$self->[node_prefix];
+}
+
+sub string_value {
+	my $self = shift;
+	return $self->[node_value];
+}
+
+sub toString {
+	my $self = shift;
+	my $string = ' ';
+# 	if ($self->[node_prefix]) {
+# 		$string .= $self->[node_prefix] . ':';
+# 	}
+	$string .= join('',
+					$self->[node_key],
+					'="',
+					XML::XPath::Node::XMLescape($self->[node_value], '"&><'),
+					'"');
+	return $string;
+}
+
+sub getNamespace {
+    my $self = shift;
+    my ($prefix) = @_;
+    $prefix ||= $self->getPrefix;
+    if (my $parent = $self->getParentNode) {
+        return $parent->getNamespace($prefix);
+    }
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Attribute - a single attribute
+
+=head1 API
+
+=head2 new ( key, value, prefix )
+
+Create a new attribute node.
+
+=head2 getName
+
+Returns the key for the attribute
+
+=head2 getLocalName
+
+As getName above, but without namespace information
+
+=head2 getNodeValue / getData
+
+Returns the value
+
+=head2 setNodeValue
+
+Sets the value of the attribute node.
+
+=head2 getPrefix
+
+Returns the prefix
+
+=head2 getNamespace
+
+Return the namespace.
+
+=head2 toString
+
+Generates key="value", encoded correctly.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Comment.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Comment.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Comment.pm	(revision 21112)
@@ -0,0 +1,91 @@
+# $Id$
+
+package XML::XPath::Node::Comment;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::CommentImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::Comment');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+    my $class = shift;
+    my ($comment) = @_;
+    
+        my $pos = XML::XPath::Node->nextPos;
+        
+        my @vals;
+        @vals[node_global_pos, node_comment] =
+                ($pos, $comment);
+    my $self = \@vals;
+        
+    bless $self, $class;
+}
+
+sub getNodeType { COMMENT_NODE }
+
+sub isCommentNode { 1; }
+
+sub getNodeValue {
+    return shift->[node_comment];
+}
+
+sub getData {
+    shift->getNodeValue;
+}
+
+sub setNodeValue {
+    shift->[node_comment] = shift;
+}
+
+sub _to_sax {
+    my $self = shift;
+    my ($doch, $dtdh, $enth) = @_;
+    
+    $doch->comment( { Data => $self->getValue } );
+}
+
+sub comment_escape {
+    my $data = shift;
+    $data =~ s/--/&#45;&#45;/g;
+    return $data;
+}
+
+sub string_value {
+    my $self = shift;
+    return $self->[node_comment];
+}
+
+sub toString {
+    my $self = shift;
+    return '<!--' . comment_escape($self->[node_comment]) . '-->';
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Comment - an XML comment: <!--comment-->
+
+=head1 API
+
+=head2 new ( data )
+
+Create a new comment node.
+
+=head2 getValue / getData
+
+Returns the value in the comment
+
+=head2 toString
+
+Returns the comment with -- encoded as a numeric entity (if it
+exists in the comment text).
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Element.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Element.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Element.pm	(revision 21112)
@@ -0,0 +1,503 @@
+# $Id$
+
+package XML::XPath::Node::Element;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::ElementImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::Element');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+    my $class = shift;
+    my ($tag, $prefix) = @_;
+        
+    my $pos = XML::XPath::Node->nextPos;
+
+    my @vals;
+    @vals[node_global_pos, node_prefix, node_children, node_name, node_attribs] =
+            ($pos, $prefix, [], $tag, []);
+        
+    my $self = \@vals;
+    bless $self, $class;
+}
+
+sub getNodeType { ELEMENT_NODE }
+
+sub isElementNode { 1; }
+
+sub appendChild {
+    my $self = shift;
+    my $newnode = shift;
+    if (shift) { # called from internal to XML::XPath
+#    warn "AppendChild $newnode to $self\n";
+        push @{$self->[node_children]}, $newnode;
+        $newnode->setParentNode($self);
+        $newnode->set_pos($#{$self->[node_children]});
+    }
+    else {
+        if (@{$self->[node_children]}) {
+            $self->insertAfter($newnode, $self->[node_children][-1]);
+        }
+        else {
+            my $pos_number = $self->get_global_pos() + 1;
+            
+            if (my $brother = $self->getNextSibling()) { # optimisation
+                if ($pos_number == $brother->get_global_pos()) {
+                    $self->renumber('following::node()', +5);
+                }
+            }
+            else {
+                eval {
+                    if ($pos_number == 
+                            $self->findnodes(
+                                'following::node()'
+                                )->get_node(1)->get_global_pos()) {
+                        $self->renumber('following::node()', +5);
+                    }
+                };
+            }
+            
+            push @{$self->[node_children]}, $newnode;
+            $newnode->setParentNode($self);
+            $newnode->set_pos($#{$self->[node_children]});
+            $newnode->set_global_pos($pos_number);
+        }
+    }
+}
+
+sub removeChild {
+    my $self = shift;
+    my $delnode = shift;
+    
+    my $pos = $delnode->get_pos;
+    
+#    warn "removeChild: $pos\n";
+    
+#    warn "children: ", scalar @{$self->[node_children]}, "\n";
+    
+#    my $node = $self->[node_children][$pos];
+#    warn "child at $pos is: $node\n";
+    
+    splice @{$self->[node_children]}, $pos, 1;
+    
+#    warn "children now: ", scalar @{$self->[node_children]}, "\n";
+    
+    for (my $i = $pos; $i < @{$self->[node_children]}; $i++) {
+#        warn "Changing pos of child: $i\n";
+        $self->[node_children][$i]->set_pos($i);
+    }
+    
+    $delnode->del_parent_link;
+    
+}
+
+sub appendIdElement {
+    my $self = shift;
+    my ($val, $element) = @_;
+#    warn "Adding '$val' to ID hash\n";
+    $self->[node_ids]{$val} = $element;
+}
+
+sub DESTROY {
+    my $self = shift;
+#    warn "DESTROY ELEMENT: ", $self->[node_name], "\n";
+#    warn "DESTROY ROOT\n" unless $self->[node_name];
+    
+    foreach my $kid ($self->getChildNodes) {
+        $kid && $kid->del_parent_link;
+    }
+    foreach my $attr ($self->getAttributeNodes) {
+        $attr && $attr->del_parent_link;
+    }
+    foreach my $ns ($self->getNamespaceNodes) {
+        $ns && $ns->del_parent_link;
+    }
+#     $self->[node_children] = undef;
+#     $self->[node_attribs] = undef;
+#     $self->[node_namespaces] = undef;
+}
+
+sub getName {
+    my $self = shift;
+    $self->[node_name];
+}
+
+sub getTagName {
+    shift->getName(@_);
+}
+
+sub getLocalName {
+    my $self = shift;
+    my $local = $self->[node_name];
+    $local =~ s/.*://;
+    return $local;
+}
+
+sub getChildNodes {
+    my $self = shift;
+    return wantarray ? @{$self->[node_children]} : $self->[node_children];
+}
+
+sub getChildNode {
+    my $self = shift;
+    my ($pos) = @_;
+    if ($pos < 1 || $pos > @{$self->[node_children]}) {
+        return;
+    }
+    return $self->[node_children][$pos - 1];
+}
+
+sub getFirstChild {
+    my $self = shift;
+    return unless @{$self->[node_children]};
+    return $self->[node_children][0];
+}
+
+sub getLastChild {
+    my $self = shift;
+    return unless @{$self->[node_children]};
+    return $self->[node_children][-1];
+}
+
+sub getAttributeNode {
+    my $self = shift;
+    my ($name) = @_;
+    my $attribs = $self->[node_attribs];
+    foreach my $attr (@$attribs) {
+        return $attr if $attr->getName eq $name;
+    }
+}
+
+sub getAttribute {
+    my $self = shift;
+    my $attr = $self->getAttributeNode(@_);
+    if ($attr) {
+        return $attr->getValue;
+    }
+}
+
+sub getAttributes {
+    my $self = shift;
+    if ($self->[node_attribs]) {
+        return wantarray ? @{$self->[node_attribs]} : $self->[node_attribs];
+    }
+    return wantarray ? () : [];
+}
+
+sub appendAttribute {
+    my $self = shift;
+    my $attribute = shift;
+    
+    if (shift) { # internal call
+        push @{$self->[node_attribs]}, $attribute;
+        $attribute->setParentNode($self);
+        $attribute->set_pos($#{$self->[node_attribs]});
+    }
+    else {
+        my $node_num;
+        if (@{$self->[node_attribs]}) {
+            $node_num = $self->[node_attribs][-1]->get_global_pos() + 1;
+        }
+        else {
+            $node_num = $self->get_global_pos() + 1;
+        }
+        
+        eval {
+            if (@{$self->[node_children]}) {
+                if ($node_num == $self->[node_children][-1]->get_global_pos()) {
+                    $self->renumber('descendant::node() | following::node()', +5);
+                }
+            }
+            elsif ($node_num == 
+                    $self->findnodes('following::node()')->get_node(1)->get_global_pos()) {
+                $self->renumber('following::node()', +5);
+            }
+        };
+        
+        push @{$self->[node_attribs]}, $attribute;
+        $attribute->setParentNode($self);
+        $attribute->set_pos($#{$self->[node_attribs]});
+        $attribute->set_global_pos($node_num);
+        
+    }
+}
+
+sub removeAttribute {
+    my $self = shift;
+    my $attrib = shift;
+    
+    if (!ref($attrib)) {
+        $attrib = $self->getAttributeNode($attrib);
+    }
+    
+    my $pos = $attrib->get_pos;
+    
+    splice @{$self->[node_attribs]}, $pos, 1;
+    
+    for (my $i = $pos; $i < @{$self->[node_attribs]}; $i++) {
+        $self->[node_attribs][$i]->set_pos($i);
+    }
+    
+    $attrib->del_parent_link;
+}
+
+sub setAttribute {
+    my $self = shift;
+    my ($name, $value) = @_;
+    
+    if (my $attrib = $self->getAttributeNode($name)) {
+        $attrib->setNodeValue($value);
+        return $attrib;
+    }
+    
+    my ($nsprefix) = ($name =~ /^($XML::XPath::Parser::NCName):($XML::XPath::Parser::NCName)$/o);
+    
+    if ($nsprefix && !$self->getNamespace($nsprefix)) {
+        die "No namespace matches prefix: $nsprefix";
+    }
+    
+    my $newnode = XML::XPath::Node::Attribute->new($name, $value, $nsprefix);
+    $self->appendAttribute($newnode);
+}
+
+sub setAttributeNode {
+    my $self = shift;
+    my ($node) = @_;
+    
+    if (my $attrib = $self->getAttributeNode($node->getName)) {
+        $attrib->setNodeValue($node->getValue);
+        return $attrib;
+    }
+    
+    my ($nsprefix) = ($node->getName() =~ /^($XML::XPath::Parser::NCName):($XML::XPath::Parser::NCName)$/o);
+    
+    if ($nsprefix && !$self->getNamespace($nsprefix)) {
+        die "No namespace matches prefix: $nsprefix";
+    }
+    
+    $self->appendAttribute($node);
+}
+
+sub getNamespace {
+    my $self = shift;
+    my ($prefix) = @_;
+    $prefix ||= $self->getPrefix || '#default';
+    my $namespaces = $self->[node_namespaces] || [];
+    foreach my $ns (@$namespaces) {
+        return $ns if $ns->getPrefix eq $prefix;
+    }
+    my $parent = $self->getParentNode;
+    
+    return $parent->getNamespace($prefix) if $parent;
+}
+
+sub getNamespaces {
+    my $self = shift;
+    if ($self->[node_namespaces]) {
+        return wantarray ? @{$self->[node_namespaces]} : $self->[node_namespaces];
+    }
+    return wantarray ? () : [];
+}
+
+sub getNamespaceNodes { goto &getNamespaces }
+
+sub appendNamespace {
+    my $self = shift;
+    my ($ns) = @_;
+    push @{$self->[node_namespaces]}, $ns;
+    $ns->setParentNode($self);
+    $ns->set_pos($#{$self->[node_namespaces]});
+}
+
+sub getPrefix {
+    my $self = shift;
+    $self->[node_prefix];
+}
+
+sub getExpandedName {
+    my $self = shift;
+    warn "Expanded name not implemented for ", ref($self), "\n";
+    return;
+}
+
+sub _to_sax {
+    my $self = shift;
+    my ($doch, $dtdh, $enth) = @_;
+    
+    my $tag = $self->getName;
+    my @attr;
+    
+    for my $attr ($self->getAttributes) {
+        push @attr, $attr->getName, $attr->getValue;
+    }
+    
+    my $ns = $self->getNamespace($self->[node_prefix]);
+    if ($ns) {
+        $doch->start_element( 
+                { 
+                Name => $tag,
+                Attributes => { @attr },
+                NamespaceURI => $ns->getExpanded,
+                Prefix => $ns->getPrefix,
+                LocalName => $self->getLocalName,
+                }
+            );
+    }
+    else {
+        $doch->start_element(
+                {
+                Name => $tag,
+                Attributes => { @attr },
+                }
+            );
+    }
+    
+    for my $kid ($self->getChildNodes) {
+        $kid->_to_sax($doch, $dtdh, $enth);
+    }
+    
+    if ($ns) {
+        $doch->end_element( 
+                {
+                Name => $tag,
+                NamespaceURI => $ns->getExpanded,
+                Prefix => $ns->getPrefix,
+                LocalName => $self->getLocalName
+                }
+            );
+    }
+    else {
+        $doch->end_element( { Name => $tag } );
+    }
+}
+
+sub string_value {
+    my $self = shift;
+    my $string = '';
+    foreach my $kid (@{$self->[node_children]}) {
+        if ($kid->getNodeType == ELEMENT_NODE
+                || $kid->getNodeType == TEXT_NODE) {
+            $string .= $kid->string_value;
+        }
+    }
+    return $string;
+}
+
+sub toString {
+    my $self = shift;
+    my $norecurse = shift;
+    my $string = '';
+    if (! $self->[node_name] ) {
+            # root node
+            return join('', map { $_->toString($norecurse) } @{$self->[node_children]});
+    }
+    $string .= "<" . $self->[node_name];
+    
+        $string .= join('', map { $_->toString } @{$self->[node_namespaces]});
+    
+        $string .= join('', map { $_->toString } @{$self->[node_attribs]});
+    
+    if (@{$self->[node_children]}) {
+        $string .= ">";
+
+        if (!$norecurse) {
+                        $string .= join('', map { $_->toString($norecurse) } @{$self->[node_children]});
+        }
+        
+        $string .= "</" . $self->[node_name] . ">";
+    }
+    else {
+        $string .= " />";
+    }
+    
+    return $string;
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Element - an <element>
+
+=head1 API
+
+=head2 new ( name, prefix )
+
+Create a new Element node with name "name" and prefix "prefix". The name
+be "prefix:local" if prefix is defined. I know that sounds wierd, but it
+works ;-)
+
+=head2 getName
+
+Returns the name (including "prefix:" if defined) of this element.
+
+=head2 getLocalName
+
+Returns just the local part of the name (the bit after "prefix:").
+
+=head2 getChildNodes
+
+Returns the children of this element. In list context returns a list. In
+scalar context returns an array ref.
+
+=head2 getChildNode ( pos )
+
+Returns the child at position pos.
+
+=head2 appendChild ( childnode )
+
+Appends the child node to the list of current child nodes.
+
+=head2 getAttribute ( name )
+
+Returns the attribute node with key name.
+
+=head2 getAttributes / getAttributeNodes
+
+Returns the attribute nodes. In list context returns a list. In scalar
+context returns an array ref.
+
+=head2 appendAttribute ( attrib_node)
+
+Appends the attribute node to the list of attributes (XML::XPath stores
+attributes in order).
+
+=head2 getNamespace ( prefix )
+
+Returns the namespace node by the given prefix
+
+=head2 getNamespaces / getNamespaceNodes
+
+Returns the namespace nodes. In list context returns a list. In scalar
+context returns an array ref.
+
+=head2 appendNamespace ( ns_node )
+
+Appends the namespace node to the list of namespaces.
+
+=head2 getPrefix
+
+Returns the prefix of this element
+
+=head2 getExpandedName
+
+Returns the expanded name of this element (not yet implemented right).
+
+=head2 string_value
+
+For elements, the string_value is the concatenation of all string_values
+of all text-descendants of the element node in document order.
+
+=head2 toString ( [ norecurse ] )
+
+Output (and all children) the node to a string. Doesn't process children
+if the norecurse option is a true value.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Namespace.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Namespace.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Namespace.pm	(revision 21112)
@@ -0,0 +1,99 @@
+# $Id$
+
+package XML::XPath::Node::Namespace;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::NamespaceImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::Namespace');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+	my $class = shift;
+	my ($prefix, $expanded) = @_;
+	
+        my $pos = XML::XPath::Node->nextPos;
+        
+        my @vals;
+        @vals[node_global_pos, node_prefix, node_expanded] =
+                ($pos, $prefix, $expanded);
+	my $self = \@vals;
+        
+	bless $self, $class;
+}
+
+sub getNodeType { NAMESPACE_NODE }
+
+sub isNamespaceNode { 1; }
+
+sub getPrefix {
+	my $self = shift;
+	$self->[node_prefix];
+}
+
+sub getExpanded {
+	my $self = shift;
+	$self->[node_expanded];
+}
+
+sub getValue {
+	my $self = shift;
+	$self->[node_expanded];
+}
+
+sub getData {
+	my $self = shift;
+	$self->[node_expanded];
+}
+
+sub string_value {
+	my $self = shift;
+	$self->[node_expanded];
+}
+
+sub toString {
+	my $self = shift;
+	my $string = '';
+	return '' unless defined $self->[node_expanded];
+	if ($self->[node_prefix] eq '#default') {
+		$string .= ' xmlns="';
+	}
+	else {
+		$string .= ' xmlns:' . $self->[node_prefix] . '="';
+	}
+	$string .= XML::XPath::Node::XMLescape($self->[node_expanded], '"&<');
+	$string .= '"';
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Namespace - an XML namespace node
+
+=head1 API
+
+=head2 new ( prefix, expanded )
+
+Create a new namespace node, expanded is the expanded namespace URI.
+
+=head2 getPrefix
+
+Returns the prefix
+
+=head2 getExpanded
+
+Returns the expanded URI
+
+=head2 toString
+
+Returns a string that you can add to the list
+of attributes of an element: xmlns:prefix="expanded"
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/PI.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/PI.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/PI.pm	(revision 21112)
@@ -0,0 +1,81 @@
+# $Id$
+
+package XML::XPath::Node::PI;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::PIImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::PI');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+	my $class = shift;
+	my ($target, $data) = @_;
+	
+        my $pos = XML::XPath::Node->nextPos;
+        
+        my @vals;
+        @vals[node_global_pos, node_target, node_data] = 
+                ($pos, $target, $data);
+	my $self = \@vals;
+	bless $self, $class;
+}
+
+sub getNodeType { PROCESSING_INSTRUCTION_NODE }
+
+sub isPINode { 1; }
+sub isProcessingInstructionNode { 1; }
+
+sub getTarget {
+	my $self = shift;
+	$self->[node_target];
+}
+
+sub getData {
+	my $self = shift;
+	$self->[node_data];
+}
+
+sub _to_sax {
+	my $self = shift;
+	my ($doch, $dtdh, $enth) = @_;
+	# PI's not supported in PerlSAX 1
+}
+
+sub string_value {
+	my $self = shift;
+	return $self->[node_data];
+}
+
+sub toString {
+	my $self = shift;
+	return "<?" . $self->[node_target] . " " . XML::XPath::Node::XMLescape($self->[node_data], ">") . "?>";
+}
+
+1;
+__END__
+
+=head1 NAME
+
+PI - an XML processing instruction node
+
+=head1 API
+
+=head2 new ( target, data )
+
+Create a new PI node.
+
+=head2 getTarget
+
+Returns the target
+
+=head2 getData
+
+Returns the data
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Text.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Text.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Node/Text.pm	(revision 21112)
@@ -0,0 +1,96 @@
+# $Id$
+
+package XML::XPath::Node::Text;
+
+use strict;
+use vars qw/@ISA/;
+
+@ISA = ('XML::XPath::Node');
+
+package XML::XPath::Node::TextImpl;
+
+use vars qw/@ISA/;
+@ISA = ('XML::XPath::NodeImpl', 'XML::XPath::Node::Text');
+use XML::XPath::Node ':node_keys';
+
+sub new {
+    my $class = shift;
+    my ($text) = @_;
+    
+        my $pos = XML::XPath::Node->nextPos;
+        
+        my @vals;
+        @vals[node_global_pos, node_text] = ($pos, $text);
+    my $self = \@vals;
+        
+    bless $self, $class;
+}
+
+sub getNodeType { TEXT_NODE }
+
+sub isTextNode { 1; }
+
+sub appendText {
+    my $self = shift;
+    my ($text) = @_;
+    $self->[node_text] .= $text;
+}
+
+sub getNodeValue {
+    my $self = shift;
+    $self->[node_text];
+}
+
+sub getData {
+    my $self = shift;
+    $self->[node_text];
+}
+
+sub setNodeValue {
+    my $self = shift;
+    $self->[node_text] = shift;
+}
+
+sub _to_sax {
+    my $self = shift;
+    my ($doch, $dtdh, $enth) = @_;
+    
+    $doch->characters( { Data => $self->getValue } );
+}
+
+sub string_value {
+    my $self = shift;
+    $self->[node_text];
+}
+
+sub toString {
+    my $self = shift;
+    XML::XPath::Node::XMLescape($self->[node_text], "<&");
+}
+
+1;
+__END__
+
+=head1 NAME
+
+Text - an XML text node
+
+=head1 API
+
+=head2 new ( text )
+
+Create a new text node.
+
+=head2 getValue / getData
+
+Returns the text
+
+=head2 string_value
+
+Returns the text
+
+=head2 appendText ( text )
+
+Adds the given text string to this node.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/NodeSet.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/NodeSet.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/NodeSet.pm	(revision 21112)
@@ -0,0 +1,184 @@
+# $Id$
+
+package XML::XPath::NodeSet;
+use strict;
+
+use XML::XPath::Boolean;
+
+use overload 
+		'""' => \&to_literal,
+                'bool' => \&to_boolean,
+        ;
+
+sub new {
+	my $class = shift;
+	bless [], $class;
+}
+
+sub sort {
+    my $self = CORE::shift;
+    @$self = CORE::sort { $a->get_global_pos <=> $b->get_global_pos } @$self;
+    return $self;
+}
+
+sub pop {
+	my $self = CORE::shift;
+	CORE::pop @$self;
+}
+
+sub push {
+	my $self = CORE::shift;
+	my (@nodes) = @_;
+	CORE::push @$self, @nodes;
+}
+
+sub append {
+	my $self = CORE::shift;
+	my ($nodeset) = @_;
+	CORE::push @$self, $nodeset->get_nodelist;
+}
+
+sub shift {
+	my $self = CORE::shift;
+	CORE::shift @$self;
+}
+
+sub unshift {
+	my $self = CORE::shift;
+	my (@nodes) = @_;
+	CORE::unshift @$self, @nodes;
+}
+
+sub prepend {
+	my $self = CORE::shift;
+	my ($nodeset) = @_;
+	CORE::unshift @$self, $nodeset->get_nodelist;
+}
+
+sub size {
+	my $self = CORE::shift;
+	scalar @$self;
+}
+
+sub get_node { # uses array index starting at 1, not 0
+	my $self = CORE::shift;
+	my ($pos) = @_;
+	$self->[$pos - 1];
+}
+
+sub getRootNode {
+    my $self = CORE::shift;
+    return $self->[0]->getRootNode;
+}
+
+sub get_nodelist {
+	my $self = CORE::shift;
+	@$self;
+}
+
+sub to_boolean {
+	my $self = CORE::shift;
+	return (@$self > 0) ? XML::XPath::Boolean->True : XML::XPath::Boolean->False;
+}
+
+sub string_value {
+	my $self = CORE::shift;
+	return '' unless @$self;
+	return $self->[0]->string_value;
+}
+
+sub to_literal {
+	my $self = CORE::shift;
+	return XML::XPath::Literal->new(
+			join('', map { $_->string_value } @$self)
+			);
+}
+
+sub to_number {
+	my $self = CORE::shift;
+	return XML::XPath::Number->new(
+			$self->to_literal
+			);
+}
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath::NodeSet - a list of XML document nodes
+
+=head1 DESCRIPTION
+
+An XML::XPath::NodeSet object contains an ordered list of nodes. The nodes
+each take the same format as described in L<XML::XPath::XMLParser>.
+
+=head1 SYNOPSIS
+
+	my $results = $xp->find('//someelement');
+	if (!$results->isa('XML::XPath::NodeSet')) {
+		print "Found $results\n";
+		exit;
+	}
+	foreach my $context ($results->get_nodelist) {
+		my $newresults = $xp->find('./other/element', $context);
+		...
+	}
+
+=head1 API
+
+=head2 new()
+
+You will almost never have to create a new NodeSet object, as it is all
+done for you by XPath.
+
+=head2 get_nodelist()
+
+Returns a list of nodes. See L<XML::XPath::XMLParser> for the format of
+the nodes.
+
+=head2 string_value()
+
+Returns the string-value of the first node in the list.
+See the XPath specification for what "string-value" means.
+
+=head2 to_literal()
+
+Returns the concatenation of all the string-values of all
+the nodes in the list.
+
+=head2 get_node($pos)
+
+Returns the node at $pos. The node position in XPath is based at 1, not 0.
+
+=head2 size()
+
+Returns the number of nodes in the NodeSet.
+
+=head2 pop()
+
+Equivalent to perl's pop function.
+
+=head2 push(@nodes)
+
+Equivalent to perl's push function.
+
+=head2 append($nodeset)
+
+Given a nodeset, appends the list of nodes in $nodeset to the end of the
+current list.
+
+=head2 shift()
+
+Equivalent to perl's shift function.
+
+=head2 unshift(@nodes)
+
+Equivalent to perl's unshift function.
+
+=head2 prepend($nodeset)
+
+Given a nodeset, prepends the list of nodes in $nodeset to the front of
+the current list.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Number.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Number.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Number.pm	(revision 21112)
@@ -0,0 +1,87 @@
+# $Id$
+
+package XML::XPath::Number;
+use XML::XPath::Boolean;
+use XML::XPath::Literal;
+use strict;
+
+use overload
+        '""' => \&value,
+        '0+' => \&value,
+        '<=>' => \&cmp;
+
+sub new {
+    my $class = shift;
+    my $number = shift;
+    if ($number !~ /^\s*[+-]?(\d+(\.\d*)?|\.\d+)\s*$/) {
+        $number = undef;
+    }
+    else {
+        $number =~ s/^\s*(.*)\s*$/$1/;
+    }
+    bless \$number, $class;
+}
+
+sub as_string {
+    my $self = shift;
+    defined $$self ? $$self : 'NaN';
+}
+
+sub as_xml {
+    my $self = shift;
+    return "<Number>" . (defined($$self) ? $$self : 'NaN') . "</Number>\n";
+}
+
+sub value {
+    my $self = shift;
+    $$self;
+}
+
+sub cmp {
+    my $self = shift;
+    my ($other, $swap) = @_;
+    if ($swap) {
+        return $other <=> $$self;
+    }
+    return $$self <=> $other;
+}
+
+sub evaluate {
+    my $self = shift;
+    $self;
+}
+
+sub to_boolean {
+    my $self = shift;
+    return $$self ? XML::XPath::Boolean->True : XML::XPath::Boolean->False;
+}
+
+sub to_literal { XML::XPath::Literal->new($_[0]->as_string); }
+sub to_number { $_[0]; }
+
+sub string_value { return $_[0]->value }
+
+1;
+__END__
+
+=head1 NAME
+
+XML::XPath::Number - Simple numeric values.
+
+=head1 DESCRIPTION
+
+This class holds simple numeric values. It doesn't support -0, +/- Infinity,
+or NaN, as the XPath spec says it should, but I'm not hurting anyone I don't think.
+
+=head1 API
+
+=head2 new($num)
+
+Creates a new XML::XPath::Number object, with the value in $num. Does some
+rudimentary numeric checking on $num to ensure it actually is a number.
+
+=head2 value()
+
+Also as overloaded stringification. Returns the numeric value held.
+
+=cut
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Parser.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Parser.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Parser.pm	(revision 21112)
@@ -0,0 +1,821 @@
+# $Id$
+
+package XML::XPath::Parser;
+
+use strict;
+use vars qw/
+        $NCName 
+        $QName 
+        $NCWild
+        $QNWild
+        $NUMBER_RE 
+        $NODE_TYPE 
+        $AXIS_NAME 
+        %AXES 
+        $LITERAL
+        %CACHE/;
+
+use XML::XPath::XMLParser;
+use XML::XPath::Step;
+use XML::XPath::Expr;
+use XML::XPath::Function;
+use XML::XPath::LocationPath;
+use XML::XPath::Variable;
+use XML::XPath::Literal;
+use XML::XPath::Number;
+use XML::XPath::NodeSet;
+
+# Axis name to principal node type mapping
+%AXES = (
+        'ancestor' => 'element',
+        'ancestor-or-self' => 'element',
+        'attribute' => 'attribute',
+        'namespace' => 'namespace',
+        'child' => 'element',
+        'descendant' => 'element',
+        'descendant-or-self' => 'element',
+        'following' => 'element',
+        'following-sibling' => 'element',
+        'parent' => 'element',
+        'preceding' => 'element',
+        'preceding-sibling' => 'element',
+        'self' => 'element',
+        );
+
+$NCName = '([A-Za-z_][\w\\.\\-]*)';
+$QName = "($NCName:)?$NCName";
+$NCWild = "${NCName}:\\*";
+$QNWild = "\\*";
+$NODE_TYPE = '((text|comment|processing-instruction|node)\\(\\))';
+$AXIS_NAME = '(' . join('|', keys %AXES) . ')::';
+$NUMBER_RE = '\d+(\\.\d*)?|\\.\d+';
+$LITERAL = '\\"[^\\"]*\\"|\\\'[^\\\']*\\\'';
+
+sub new {
+    my $class = shift;
+    my $self = bless {}, $class;
+    debug("New Parser being created.\n");
+    $self->{context_set} = XML::XPath::NodeSet->new();
+    $self->{context_pos} = undef; # 1 based position in array context
+    $self->{context_size} = 0; # total size of context
+    $self->clear_namespaces();
+    $self->{vars} = {};
+    $self->{direction} = 'forward';
+    $self->{cache} = {};
+    return $self;
+}
+
+sub get_var {
+    my $self = shift;
+    my $var = shift;
+    $self->{vars}->{$var};
+}
+
+sub set_var {
+    my $self = shift;
+    my $var = shift;
+    my $val = shift;
+    $self->{vars}->{$var} = $val;
+}
+
+sub set_namespace {
+    my $self = shift;
+    my ($prefix, $expanded) = @_;
+    $self->{namespaces}{$prefix} = $expanded;
+}
+
+sub clear_namespaces {
+    my $self = shift;
+    $self->{namespaces} = {};
+}
+
+sub get_namespace {
+    my $self = shift;
+    my ($prefix, $node) = @_;
+    if (my $ns = $self->{namespaces}{$prefix}) {
+        return $ns;
+    }
+    if (my $nsnode = $node->getNamespace($prefix)) {
+        return $nsnode->getValue();
+    }
+}
+
+sub get_context_set { $_[0]->{context_set}; }
+sub set_context_set { $_[0]->{context_set} = $_[1]; }
+sub get_context_pos { $_[0]->{context_pos}; }
+sub set_context_pos { $_[0]->{context_pos} = $_[1]; }
+sub get_context_size { $_[0]->{context_set}->size; }
+sub get_context_node { $_[0]->{context_set}->get_node($_[0]->{context_pos}); }
+
+sub my_sub {
+    return (caller(1))[3];
+}
+
+sub parse {
+    my $self = shift;
+    my $path = shift;
+    if ($CACHE{$path}) {
+        return $CACHE{$path};
+    }
+    my $tokens = $self->tokenize($path);
+
+    $self->{_tokpos} = 0;
+    my $tree = $self->analyze($tokens);
+    
+    if ($self->{_tokpos} < scalar(@$tokens)) {
+        # didn't manage to parse entire expression - throw an exception
+        die "Parse of expression $path failed - junk after end of expression: $tokens->[$self->{_tokpos}]";
+    }
+    
+    $CACHE{$path} = $tree;
+    
+    debug("PARSED Expr to:\n", $tree->as_string, "\n") if $XML::XPath::Debug;
+    
+    return $tree;
+}
+
+sub tokenize {
+    my $self = shift;
+    my $path = shift;
+    study $path;
+    
+    my @tokens;
+    
+    debug("Parsing: $path\n");
+    
+    # Bug: We don't allow "'@' NodeType" which is in the grammar, but I think is just plain stupid.
+
+    while($path =~ m/\G
+        \s* # ignore all whitespace
+        ( # tokens
+            $LITERAL|
+            $NUMBER_RE| # Match digits
+            \.\.| # match parent
+            \.| # match current
+            ($AXIS_NAME)?$NODE_TYPE| # match tests
+            processing-instruction|
+            \@($NCWild|$QName|$QNWild)| # match attrib
+            \$$QName| # match variable reference
+            ($AXIS_NAME)?($NCWild|$QName|$QNWild)| # match NCName,NodeType,Axis::Test
+            \!=|<=|\-|>=|\/\/|and|or|mod|div| # multi-char seps
+            [,\+=\|<>\/\(\[\]\)]| # single char seps
+            (?<!(\@|\(|\[))\*| # multiply operator rules (see xpath spec)
+            (?<!::)\*|
+            $ # match end of query
+        )
+        \s* # ignore all whitespace
+        /gcxso) {
+
+        my ($token) = ($1);
+
+        if (length($token)) {
+            debug("TOKEN: $token\n");
+            push @tokens, $token;
+        }
+        
+    }
+    
+    if (pos($path) < length($path)) {
+        my $marker = ("." x (pos($path)-1));
+        $path = substr($path, 0, pos($path) + 8) . "...";
+        $path =~ s/\n/ /g;
+        $path =~ s/\t/ /g;
+        die "Query:\n",
+            "$path\n",
+            $marker, "^^^\n",
+            "Invalid query somewhere around here (I think)\n";
+    }
+    
+    return \@tokens;
+}
+
+sub analyze {
+    my $self = shift;
+    my $tokens = shift;
+    # lexical analysis
+    
+    return Expr($self, $tokens);
+}
+
+sub match {
+    my ($self, $tokens, $match, $fatal) = @_;
+    
+    $self->{_curr_match} = '';
+    return 0 unless $self->{_tokpos} < @$tokens;
+
+    local $^W;
+    
+#    debug ("match: $match\n");
+    
+    if ($tokens->[$self->{_tokpos}] =~ /^$match$/) {
+        $self->{_curr_match} = $tokens->[$self->{_tokpos}];
+        $self->{_tokpos}++;
+        return 1;
+    }
+    else {
+        if ($fatal) {
+            die "Invalid token: ", $tokens->[$self->{_tokpos}], "\n";
+        }
+        else {
+            return 0;
+        }
+    }
+}
+
+sub Expr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    return OrExpr($self, $tokens);
+}
+
+sub OrExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = AndExpr($self, $tokens); 
+    while (match($self, $tokens, 'or')) {
+        my $or_expr = XML::XPath::Expr->new($self);
+        $or_expr->set_lhs($expr);
+        $or_expr->set_op('or');
+
+        my $rhs = AndExpr($self, $tokens);
+
+        $or_expr->set_rhs($rhs);
+        $expr = $or_expr;
+    }
+    
+    return $expr;
+}
+
+sub AndExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = EqualityExpr($self, $tokens);
+    while (match($self, $tokens, 'and')) {
+        my $and_expr = XML::XPath::Expr->new($self);
+        $and_expr->set_lhs($expr);
+        $and_expr->set_op('and');
+        
+        my $rhs = EqualityExpr($self, $tokens);
+        
+        $and_expr->set_rhs($rhs);
+        $expr = $and_expr;
+    }
+    
+    return $expr;
+}
+
+sub EqualityExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = RelationalExpr($self, $tokens);
+    while (match($self, $tokens, '!?=')) {
+        my $eq_expr = XML::XPath::Expr->new($self);
+        $eq_expr->set_lhs($expr);
+        $eq_expr->set_op($self->{_curr_match});
+        
+        my $rhs = RelationalExpr($self, $tokens);
+        
+        $eq_expr->set_rhs($rhs);
+        $expr = $eq_expr;
+    }
+    
+    return $expr;
+}
+
+sub RelationalExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = AdditiveExpr($self, $tokens);
+    while (match($self, $tokens, '(<|>|<=|>=)')) {
+        my $rel_expr = XML::XPath::Expr->new($self);
+        $rel_expr->set_lhs($expr);
+        $rel_expr->set_op($self->{_curr_match});
+        
+        my $rhs = AdditiveExpr($self, $tokens);
+        
+        $rel_expr->set_rhs($rhs);
+        $expr = $rel_expr;
+    }
+    
+    return $expr;
+}
+
+sub AdditiveExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = MultiplicativeExpr($self, $tokens);
+    while (match($self, $tokens, '[\\+\\-]')) {
+        my $add_expr = XML::XPath::Expr->new($self);
+        $add_expr->set_lhs($expr);
+        $add_expr->set_op($self->{_curr_match});
+        
+        my $rhs = MultiplicativeExpr($self, $tokens);
+        
+        $add_expr->set_rhs($rhs);
+        $expr = $add_expr;
+    }
+    
+    return $expr;
+}
+
+sub MultiplicativeExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = UnaryExpr($self, $tokens);
+    while (match($self, $tokens, '(\\*|div|mod)')) {
+        my $mult_expr = XML::XPath::Expr->new($self);
+        $mult_expr->set_lhs($expr);
+        $mult_expr->set_op($self->{_curr_match});
+        
+        my $rhs = UnaryExpr($self, $tokens);
+        
+        $mult_expr->set_rhs($rhs);
+        $expr = $mult_expr;
+    }
+    
+    return $expr;
+}
+
+sub UnaryExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    if (match($self, $tokens, '-')) {
+        my $expr = XML::XPath::Expr->new($self);
+        $expr->set_lhs(XML::XPath::Number->new(0));
+        $expr->set_op('-');
+        $expr->set_rhs(UnaryExpr($self, $tokens));
+        return $expr;
+    }
+    else {
+        return UnionExpr($self, $tokens);
+    }
+}
+
+sub UnionExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = PathExpr($self, $tokens);
+    while (match($self, $tokens, '\\|')) {
+        my $un_expr = XML::XPath::Expr->new($self);
+        $un_expr->set_lhs($expr);
+        $un_expr->set_op('|');
+        
+        my $rhs = PathExpr($self, $tokens);
+        
+        $un_expr->set_rhs($rhs);
+        $expr = $un_expr;
+    }
+    
+    return $expr;
+}
+
+sub PathExpr {
+    my ($self, $tokens) = @_;
+
+    debug("in SUB\n");
+    
+    # PathExpr is LocationPath | FilterExpr | FilterExpr '//?' RelativeLocationPath
+    
+    # Since we are being predictive we need to find out which function to call next, then.
+        
+    # LocationPath either starts with "/", "//", ".", ".." or a proper Step.
+    
+    my $expr = XML::XPath::Expr->new($self);
+    
+    my $test = $tokens->[$self->{_tokpos}];
+    
+    # Test for AbsoluteLocationPath and AbbreviatedRelativeLocationPath
+    if ($test =~ /^(\/\/?|\.\.?)$/) {
+        # LocationPath
+        $expr->set_lhs(LocationPath($self, $tokens));
+    }
+    # Test for AxisName::...
+    elsif (is_step($self, $tokens)) {
+        $expr->set_lhs(LocationPath($self, $tokens));
+    }
+    else {
+        # Not a LocationPath
+        # Use FilterExpr instead:
+        
+        $expr = FilterExpr($self, $tokens);
+        if (match($self, $tokens, '//?')) {
+            my $loc_path = XML::XPath::LocationPath->new();
+            push @$loc_path, $expr;
+            if ($self->{_curr_match} eq '//') {
+                push @$loc_path, XML::XPath::Step->new($self, 'descendant-or-self', 
+                                        XML::XPath::Step::test_nt_node);
+            }
+            push @$loc_path, RelativeLocationPath($self, $tokens);
+            my $new_expr = XML::XPath::Expr->new($self);
+            $new_expr->set_lhs($loc_path);
+            return $new_expr;
+        }
+    }
+    
+    return $expr;
+}
+
+sub FilterExpr {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $expr = PrimaryExpr($self, $tokens);
+    while (match($self, $tokens, '\\[')) {
+        # really PredicateExpr...
+        $expr->push_predicate(Expr($self, $tokens));
+        match($self, $tokens, '\\]', 1);
+    }
+    
+    return $expr;
+}
+
+sub PrimaryExpr {
+    my ($self, $tokens) = @_;
+
+    debug("in SUB\n");
+    
+    my $expr = XML::XPath::Expr->new($self);
+    
+    if (match($self, $tokens, $LITERAL)) {
+        # new Literal with $self->{_curr_match}...
+        $self->{_curr_match} =~ m/^(["'])(.*)\1$/;
+        $expr->set_lhs(XML::XPath::Literal->new($2));
+    }
+    elsif (match($self, $tokens, $NUMBER_RE)) {
+        # new Number with $self->{_curr_match}...
+        $expr->set_lhs(XML::XPath::Number->new($self->{_curr_match}));
+    }
+    elsif (match($self, $tokens, '\\(')) {
+        $expr->set_lhs(Expr($self, $tokens));
+        match($self, $tokens, '\\)', 1);
+    }
+    elsif (match($self, $tokens, "\\\$$QName")) {
+        # new Variable with $self->{_curr_match}...
+        $self->{_curr_match} =~ /^\$(.*)$/;
+        $expr->set_lhs(XML::XPath::Variable->new($self, $1));
+    }
+    elsif (match($self, $tokens, $QName)) {
+        # check match not Node_Type - done in lexer...
+        # new Function
+        my $func_name = $self->{_curr_match};
+        match($self, $tokens, '\\(', 1);
+        $expr->set_lhs(
+                XML::XPath::Function->new(
+                    $self,
+                    $func_name,
+                    Arguments($self, $tokens)
+                )
+            );
+        match($self, $tokens, '\\)', 1);
+    }
+    else {
+        die "Not a PrimaryExpr at ", $tokens->[$self->{_tokpos}], "\n";
+    }
+    
+    return $expr;
+}
+
+sub Arguments {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my @args;
+    
+    if($tokens->[$self->{_tokpos}] eq ')') {
+        return \@args;
+    }
+    
+    push @args, Expr($self, $tokens);
+    while (match($self, $tokens, ',')) {
+        push @args, Expr($self, $tokens);
+    }
+    
+    return \@args;
+}
+
+sub LocationPath {
+    my ($self, $tokens) = @_;
+
+    debug("in SUB\n");
+    
+    my $loc_path = XML::XPath::LocationPath->new();
+    
+    if (match($self, $tokens, '/')) {
+        # root
+        debug("SUB: Matched root\n");
+        push @$loc_path, XML::XPath::Root->new();
+        if (is_step($self, $tokens)) {
+            debug("Next is step\n");
+            push @$loc_path, RelativeLocationPath($self, $tokens);
+        }
+    }
+    elsif (match($self, $tokens, '//')) {
+        # root
+        push @$loc_path, XML::XPath::Root->new();
+        my $optimised = optimise_descendant_or_self($self, $tokens);
+        if (!$optimised) {
+            push @$loc_path, XML::XPath::Step->new($self, 'descendant-or-self',
+                                XML::XPath::Step::test_nt_node);
+            push @$loc_path, RelativeLocationPath($self, $tokens);
+        }
+        else {
+            push @$loc_path, $optimised, RelativeLocationPath($self, $tokens);
+        }
+    }
+    else {
+        push @$loc_path, RelativeLocationPath($self, $tokens);
+    }
+    
+    return $loc_path;
+}
+
+sub optimise_descendant_or_self {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my $tokpos = $self->{_tokpos};
+    
+    # // must be followed by a Step.
+    if ($tokens->[$tokpos+1] && $tokens->[$tokpos+1] eq '[') {
+        # next token is a predicate
+        return;
+    }
+    elsif ($tokens->[$tokpos] =~ /^\.\.?$/) {
+        # abbreviatedStep - can't optimise.
+        return;
+    }                                                                                              
+    else {
+        debug("Trying to optimise //\n");
+        my $step = Step($self, $tokens);
+        if ($step->{axis} ne 'child') {
+            # can't optimise axes other than child for now...
+            $self->{_tokpos} = $tokpos;
+            return;
+        }
+        $step->{axis} = 'descendant';
+        $step->{axis_method} = 'axis_descendant';
+        $self->{_tokpos}--;
+        $tokens->[$self->{_tokpos}] = '.';
+        return $step;
+    }
+}
+
+sub RelativeLocationPath {
+    my ($self, $tokens) = @_;
+    
+    debug("in SUB\n");
+    
+    my @steps;
+    
+    push @steps, Step($self, $tokens);
+    while (match($self, $tokens, '//?')) {
+        if ($self->{_curr_match} eq '//') {
+            my $optimised = optimise_descendant_or_self($self, $tokens);
+            if (!$optimised) {
+                push @steps, XML::XPath::Step->new($self, 'descendant-or-self',
+                                        XML::XPath::Step::test_nt_node);
+            }
+            else {
+                push @steps, $optimised;
+            }
+        }
+        push @steps, Step($self, $tokens);
+        if (@steps > 1 && 
+                $steps[-1]->{axis} eq 'self' && 
+                $steps[-1]->{test} == XML::XPath::Step::test_nt_node) {
+            pop @steps;
+        }
+    }
+    
+    return @steps;
+}
+
+sub Step {
+    my ($self, $tokens) = @_;
+
+    debug("in SUB\n");
+    
+    if (match($self, $tokens, '\\.')) {
+        # self::node()
+        return XML::XPath::Step->new($self, 'self', XML::XPath::Step::test_nt_node);
+    }
+    elsif (match($self, $tokens, '\\.\\.')) {
+        # parent::node()
+        return XML::XPath::Step->new($self, 'parent', XML::XPath::Step::test_nt_node);
+    }
+    else {
+        # AxisSpecifier NodeTest Predicate(s?)
+        my $token = $tokens->[$self->{_tokpos}];
+        
+        debug("SUB: Checking $token\n");
+        
+        my $step;
+        if ($token eq 'processing-instruction') {
+            $self->{_tokpos}++;
+            match($self, $tokens, '\\(', 1);
+            match($self, $tokens, $LITERAL);
+            $self->{_curr_match} =~ /^["'](.*)["']$/;
+            $step = XML::XPath::Step->new($self, 'child',
+                                    XML::XPath::Step::test_nt_pi,
+                        XML::XPath::Literal->new($1));
+            match($self, $tokens, '\\)', 1);
+        }
+        elsif ($token =~ /^\@($NCWild|$QName|$QNWild)$/o) {
+            $self->{_tokpos}++;
+                        if ($token eq '@*') {
+                            $step = XML::XPath::Step->new($self,
+                                    'attribute',
+                                    XML::XPath::Step::test_attr_any,
+                                    '*');
+                        }
+                        elsif ($token =~ /^\@($NCName):\*$/o) {
+                            $step = XML::XPath::Step->new($self,
+                                    'attribute',
+                                    XML::XPath::Step::test_attr_ncwild,
+                                    $1);
+                        }
+                        elsif ($token =~ /^\@($QName)$/o) {
+                            $step = XML::XPath::Step->new($self,
+                                    'attribute',
+                                    XML::XPath::Step::test_attr_qname,
+                                    $1);
+                        }
+        }
+        elsif ($token =~ /^($NCName):\*$/o) { # ns:*
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child', 
+                                XML::XPath::Step::test_ncwild,
+                                $1);
+        }
+        elsif ($token =~ /^$QNWild$/o) { # *
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child', 
+                                XML::XPath::Step::test_any,
+                                $token);
+        }
+        elsif ($token =~ /^$QName$/o) { # name:name
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child', 
+                                XML::XPath::Step::test_qname,
+                                $token);
+        }
+        elsif ($token eq 'comment()') {
+                    $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child',
+                            XML::XPath::Step::test_nt_comment);
+        }
+        elsif ($token eq 'text()') {
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child',
+                    XML::XPath::Step::test_nt_text);
+        }
+        elsif ($token eq 'node()') {
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child',
+                    XML::XPath::Step::test_nt_node);
+        }
+        elsif ($token eq 'processing-instruction()') {
+            $self->{_tokpos}++;
+            $step = XML::XPath::Step->new($self, 'child',
+                    XML::XPath::Step::test_nt_pi);
+        }
+        elsif ($token =~ /^$AXIS_NAME($NCWild|$QName|$QNWild|$NODE_TYPE)$/o) {
+                    my $axis = $1;
+                    $self->{_tokpos}++;
+                    $token = $2;
+            if ($token eq 'processing-instruction') {
+                match($self, $tokens, '\\(', 1);
+                match($self, $tokens, $LITERAL);
+                $self->{_curr_match} =~ /^["'](.*)["']$/;
+                $step = XML::XPath::Step->new($self, $axis,
+                                        XML::XPath::Step::test_nt_pi,
+                            XML::XPath::Literal->new($1));
+                match($self, $tokens, '\\)', 1);
+            }
+            elsif ($token =~ /^($NCName):\*$/o) { # ns:*
+                $step = XML::XPath::Step->new($self, $axis, 
+                                    (($axis eq 'attribute') ? 
+                                    XML::XPath::Step::test_attr_ncwild
+                                        :
+                                    XML::XPath::Step::test_ncwild),
+                                    $1);
+            }
+            elsif ($token =~ /^$QNWild$/o) { # *
+                $step = XML::XPath::Step->new($self, $axis, 
+                                    (($axis eq 'attribute') ?
+                                    XML::XPath::Step::test_attr_any
+                                        :
+                                    XML::XPath::Step::test_any),
+                                    $token);
+            }
+            elsif ($token =~ /^$QName$/o) { # name:name
+                $step = XML::XPath::Step->new($self, $axis, 
+                                    (($axis eq 'attribute') ?
+                                    XML::XPath::Step::test_attr_qname
+                                        :
+                                    XML::XPath::Step::test_qname),
+                                    $token);
+            }
+            elsif ($token eq 'comment()') {
+                $step = XML::XPath::Step->new($self, $axis,
+                                XML::XPath::Step::test_nt_comment);
+            }
+            elsif ($token eq 'text()') {
+                $step = XML::XPath::Step->new($self, $axis,
+                        XML::XPath::Step::test_nt_text);
+            }
+            elsif ($token eq 'node()') {
+                $step = XML::XPath::Step->new($self, $axis,
+                        XML::XPath::Step::test_nt_node);
+            }
+            elsif ($token eq 'processing-instruction()') {
+                $step = XML::XPath::Step->new($self, $axis,
+                        XML::XPath::Step::test_nt_pi);
+            }
+            else {
+                die "Shouldn't get here";
+            }
+        }
+        else {
+            die "token $token doesn't match format of a 'Step'\n";
+        }
+        
+        while (match($self, $tokens, '\\[')) {
+            push @{$step->{predicates}}, Expr($self, $tokens);
+            match($self, $tokens, '\\]', 1);
+        }
+        
+        return $step;
+    }
+}
+
+sub is_step {
+    my ($self, $tokens) = @_;
+    
+    my $token = $tokens->[$self->{_tokpos}];
+    
+    return unless defined $token;
+        
+    debug("SUB: Checking if '$token' is a step\n");
+    
+        local $^W;
+        
+    if ($token eq 'processing-instruction') {
+        return 1;
+    }
+    elsif ($token =~ /^\@($NCWild|$QName|$QNWild)$/o) {
+        return 1;
+    }
+    elsif ($token =~ /^($NCWild|$QName|$QNWild)$/o && $tokens->[$self->{_tokpos}+1] ne '(') {
+        return 1;
+    }
+    elsif ($token =~ /^$NODE_TYPE$/o) {
+        return 1;
+    }
+    elsif ($token =~ /^$AXIS_NAME($NCWild|$QName|$QNWild|$NODE_TYPE)$/o) {
+        return 1;
+    }
+    
+    debug("SUB: '$token' not a step\n");
+
+    return;
+}
+
+sub debug {
+    return unless $XML::XPath::Debug;
+    
+    my ($pkg, $file, $line, $sub) = caller(1);
+    
+    $sub =~ s/^$pkg\:://;
+    
+    while (@_) {
+        my $x = shift;
+        $x =~ s/\bPKG\b/$pkg/g;
+        $x =~ s/\bLINE\b/$line/g;
+        $x =~ s/\bSUB\b/$sub/g;
+        print STDERR $x;
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/PerlSAX.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/PerlSAX.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/PerlSAX.pm	(revision 21112)
@@ -0,0 +1,166 @@
+# $Id$
+
+package XML::XPath::PerlSAX;
+use XML::XPath::XMLParser;
+use strict;
+
+sub new {
+	my $class = shift;
+	my %args = @_;
+	bless \%args, $class;
+}
+
+sub parse {
+	my $self = shift;
+
+	die "XML::XPath::PerlSAX: parser instance ($self) already parsing\n"
+		if (defined $self->{ParseOptions});
+
+	# If there's one arg and it's an array ref, assume it's a node we're parsing
+	my $args;
+	if (@_ == 1 && ref($_[0]) =~ /^(text|comment|element|namespace|attribute|pi)$/) {
+#		warn "Parsing node\n";
+		my $node = shift;
+#		warn "PARSING: $node ", XML::XPath::XMLParser::as_string($node), "\n\n";
+		$args = { Source => { Node => $node } };
+	}
+	else {
+		$args = (@_ == 1) ? shift : { @_ };
+	}
+
+	my $parse_options = { %$self, %$args };
+	$self->{ParseOptions} = $parse_options;
+
+	# ensure that we have at least one source
+	if (!defined $parse_options->{Source} ||
+		!defined $parse_options->{Source}{Node}) {
+		die "XML::XPath::PerlSAX: no source defined for parse\n";
+	}
+
+	# assign default Handler to any undefined handlers
+	if (defined $parse_options->{Handler}) {
+		$parse_options->{DocumentHandler} = $parse_options->{Handler}
+			if (!defined $parse_options->{DocumentHandler});
+	}
+
+	# ensure that we have a DocumentHandler
+	if (!defined $parse_options->{DocumentHandler}) {
+		die "XML::XPath::PerlSAX: no Handler or DocumentHandler defined for parse\n";
+	}
+
+	# cache DocumentHandler in self for callbacks
+	$self->{DocumentHandler} = $parse_options->{DocumentHandler};
+
+	if ((ref($parse_options->{Source}{Node}) eq 'element') &&
+			!($parse_options->{Source}{Node}->[node_parent])) {
+		# Got root node
+		$self->{DocumentHandler}->start_document( { } );
+		$self->parse_node($parse_options->{Source}{Node});
+		return $self->{DocumentHandler}->end_document( { } );
+	}
+	else {
+		$self->parse_node($parse_options->{Source}{Node});
+	}
+
+	# clean up parser instance
+	delete $self->{ParseOptions};
+	delete $self->{DocumentHandler};
+
+}
+
+sub parse_node {
+	my $self = shift;
+	my $node = shift;
+#	warn "parse_node $node\n";
+	if (ref($node) eq 'element' && $node->[node_parent]) {
+		# bundle up attributes
+		my @attribs;
+		foreach my $attr (@{$node->[node_attribs]}) {
+			if ($attr->[node_prefix]) {
+				push @attribs, $attr->[node_prefix] . ":" . $attr->[node_key];
+			}
+			else {
+				push @attribs, $attr->[node_key];
+			}
+			push @attribs, $attr->[node_value];
+		}
+		
+		$self->{DocumentHandler}->start_element(
+				{ Name => $node->[node_name],
+				  Attributes => \@attribs,
+				}
+			);
+		foreach my $kid (@{$node->[node_children]}) {
+			$self->parse_node($kid);
+		}
+		$self->{DocumentHandler}->end_element(
+				{
+					Name => $node->[node_name],
+				}
+			);
+	}
+	elsif (ref($node) eq 'text') {
+		$self->{DocumentHandler}->characters($node->[node_text]);
+	}
+	elsif (ref($node) eq 'comment') {
+		$self->{DocumentHandler}->comment($node->[node_comment]);
+	}
+	elsif (ref($node) eq 'pi') {
+		$self->{DocumentHandler}->processing_instruction(
+				{
+					Target => $node->[node_target],
+					Data => $node->[node_data]
+				}
+			);
+	}
+	elsif (ref($node) eq 'element') { # root node
+		# just do kids
+		foreach my $kid (@{$node->[node_children]}) {
+			$self->parse_node($kid);
+		}
+	}
+	else {
+		die "Unknown node type: '", ref($node), "' ", scalar(@$node), "\n";
+	}
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::XPath::PerlSAX - A PerlSAX event generator for my wierd node structure
+
+=head1 SYNOPSIS
+
+	use XML::XPath;
+	use XML::XPath::PerlSAX;
+	use XML::DOM::PerlSAX;
+	
+	my $xp = XML::XPath->new(filename => 'test.xhtml');
+	my $paras = $xp->find('/html/body/p');
+	
+	my $handler = XML::DOM::PerlSAX->new();
+	my $generator = XML::XPath::PerlSAX->new( Handler => $handler );
+	
+	foreach my $node ($paras->get_nodelist) {
+		my $domtree = $generator->parse($node);
+		# do something with $domtree
+	}
+
+=head1 DESCRIPTION
+
+This module generates PerlSAX events to pass to a PerlSAX handler such
+as XML::DOM::PerlSAX. It operates specifically on my wierd tree format.
+
+Unfortunately SAX doesn't seem to cope with namespaces, so these are
+lost completely. I believe SAX2 is doing namespaces.
+
+=head1 Other
+
+The XML::DOM::PerlSAX handler I tried was completely broken (didn't even
+compile before I patched it a bit), so I don't know how correct this
+is or how far it will work.
+
+This software may only be distributed as part of the XML::XPath package.
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Root.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Root.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Root.pm	(revision 21112)
@@ -0,0 +1,36 @@
+# $Id$
+
+package XML::XPath::Root;
+use strict;
+use XML::XPath::XMLParser;
+use XML::XPath::NodeSet;
+
+sub new {
+	my $class = shift;
+	my $self; # actually don't need anything here - just a placeholder
+	bless \$self, $class;
+}
+
+sub as_string {
+	# do nothing
+}
+
+sub as_xml {
+    return "<Root/>\n";
+}
+
+sub evaluate {
+	my $self = shift;
+	my $nodeset = shift;
+	
+#	warn "Eval ROOT\n";
+	
+	# must only ever occur on 1 node
+	die "Can't go to root on > 1 node!" unless $nodeset->size == 1;
+	
+	my $newset = XML::XPath::NodeSet->new();
+	$newset->push($nodeset->get_node(1)->getRootNode());
+	return $newset;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Step.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Step.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Step.pm	(revision 21112)
@@ -0,0 +1,519 @@
+# $Id$
+
+package XML::XPath::Step;
+use XML::XPath::Parser;
+use XML::XPath::Node;
+use strict;
+
+# the beginnings of using XS for this file...
+# require DynaLoader;
+# use vars qw/$VERSION @ISA/;
+# $VERSION = '1.0';
+# @ISA = qw(DynaLoader);
+# 
+# bootstrap XML::XPath::Step $VERSION;
+
+sub test_qname () { 0; } # Full name
+sub test_ncwild () { 1; } # NCName:*
+sub test_any () { 2; } # *
+
+sub test_attr_qname () { 3; } # @ns:attrib
+sub test_attr_ncwild () { 4; } # @nc:*
+sub test_attr_any () { 5; } # @*
+
+sub test_nt_comment () { 6; } # comment()
+sub test_nt_text () { 7; } # text()
+sub test_nt_pi () { 8; } # processing-instruction()
+sub test_nt_node () { 9; } # node()
+
+sub new {
+    my $class = shift;
+    my ($pp, $axis, $test, $literal) = @_;
+    my $axis_method = "axis_$axis";
+    $axis_method =~ tr/-/_/;
+    my $self = {
+        pp => $pp, # the XML::XPath::Parser class
+        axis => $axis,
+        axis_method => $axis_method,
+        test => $test,
+        literal => $literal,
+        predicates => [],
+        };
+    bless $self, $class;
+}
+
+sub as_string {
+    my $self = shift;
+    my $string = $self->{axis} . "::";
+
+    my $test = $self->{test};
+        
+    if ($test == test_nt_pi) {
+        $string .= 'processing-instruction(';
+        if ($self->{literal}->value) {
+            $string .= $self->{literal}->as_string;
+        }
+        $string .= ")";
+    }
+    elsif ($test == test_nt_comment) {
+        $string .= 'comment()';
+    }
+    elsif ($test == test_nt_text) {
+        $string .= 'text()';
+    }
+    elsif ($test == test_nt_node) {
+        $string .= 'node()';
+    }
+    elsif ($test == test_ncwild || $test == test_attr_ncwild) {
+        $string .= $self->{literal} . ':*';
+    }
+    else {
+        $string .= $self->{literal};
+    }
+    
+    foreach (@{$self->{predicates}}) {
+        next unless defined $_;
+        $string .= "[" . $_->as_string . "]";
+    }
+    return $string;
+}
+
+sub as_xml {
+    my $self = shift;
+    my $string = "<Step>\n";
+    $string .= "<Axis>" . $self->{axis} . "</Axis>\n";
+    my $test = $self->{test};
+    
+    $string .= "<Test>";
+    
+    if ($test == test_nt_pi) {
+        $string .= '<processing-instruction';
+        if ($self->{literal}->value) {
+            $string .= '>';
+            $string .= $self->{literal}->as_string;
+            $string .= '</processing-instruction>';
+        }
+        else {
+            $string .= '/>';
+        }
+    }
+    elsif ($test == test_nt_comment) {
+        $string .= '<comment/>';
+    }
+    elsif ($test == test_nt_text) {
+        $string .= '<text/>';
+    }
+    elsif ($test == test_nt_node) {
+        $string .= '<node/>';
+    }
+    elsif ($test == test_ncwild || $test == test_attr_ncwild) {
+        $string .= '<namespace-prefix>' . $self->{literal} . '</namespace-prefix>';
+    }
+    else {
+        $string .= '<nametest>' . $self->{literal} . '</nametest>';
+    }
+    
+    $string .= "</Test>\n";
+    
+    foreach (@{$self->{predicates}}) {
+        next unless defined $_;
+        $string .= "<Predicate>\n" . $_->as_xml() . "</Predicate>\n";
+    }
+    
+    $string .= "</Step>\n";
+    
+    return $string;
+}
+
+sub evaluate {
+    my $self = shift;
+    my $from = shift; # context nodeset
+    
+#    warn "Step::evaluate called with ", $from->size, " length nodeset\n";
+    
+    $self->{pp}->set_context_set($from);
+    
+    my $initial_nodeset = XML::XPath::NodeSet->new();
+    
+    # See spec section 2.1, paragraphs 3,4,5:
+    # The node-set selected by the location step is the node-set
+    # that results from generating an initial node set from the
+    # axis and node-test, and then filtering that node-set by
+    # each of the predicates in turn.
+    
+    # Make each node in the nodeset be the context node, one by one
+    for(my $i = 1; $i <= $from->size; $i++) {
+        $self->{pp}->set_context_pos($i);
+        $initial_nodeset->append($self->evaluate_node($from->get_node($i)));
+    }
+    
+#    warn "Step::evaluate initial nodeset size: ", $initial_nodeset->size, "\n";
+    
+    $self->{pp}->set_context_set(undef);
+
+    $initial_nodeset->sort;
+        
+    return $initial_nodeset;
+}
+
+# Evaluate the step against a particular node
+sub evaluate_node {
+    my $self = shift;
+    my $context = shift;
+    
+#    warn "Evaluate node: $self->{axis}\n";
+    
+#    warn "Node: ", $context->[node_name], "\n";
+    
+    my $method = $self->{axis_method};
+    
+    my $results = XML::XPath::NodeSet->new();
+    no strict 'refs';
+    eval {
+        $method->($self, $context, $results);
+    };
+    if ($@) {
+        die "axis $method not implemented [$@]\n";
+    }
+    
+#    warn("results: ", join('><', map {$_->string_value} @$results), "\n");
+    # filter initial nodeset by each predicate
+    foreach my $predicate (@{$self->{predicates}}) {
+        $results = $self->filter_by_predicate($results, $predicate);
+    }
+    
+    return $results;
+}
+
+sub axis_ancestor {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    my $parent = $context->getParentNode;
+        
+    START:
+    return $results unless $parent;
+    if (node_test($self, $parent)) {
+        $results->push($parent);
+    }
+    $parent = $parent->getParentNode;
+    goto START;
+}
+
+sub axis_ancestor_or_self {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    START:
+    return $results unless $context;
+    if (node_test($self, $context)) {
+        $results->push($context);
+    }
+    $context = $context->getParentNode;
+    goto START;
+}
+
+sub axis_attribute {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    foreach my $attrib (@{$context->getAttributes}) {
+        if ($self->test_attribute($attrib)) {
+            $results->push($attrib);
+        }
+    }
+}
+
+sub axis_child {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    foreach my $node (@{$context->getChildNodes}) {
+        if (node_test($self, $node)) {
+            $results->push($node);
+        }
+    }
+}
+
+sub axis_descendant {
+    my $self = shift;
+    my ($context, $results) = @_;
+
+    my @stack = $context->getChildNodes;
+
+    while (@stack) {
+        my $node = pop @stack;
+        if (node_test($self, $node)) {
+            $results->unshift($node);
+        }
+        push @stack, $node->getChildNodes;
+    }
+}
+
+sub axis_descendant_or_self {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    my @stack = ($context);
+    
+    while (@stack) {
+        my $node = pop @stack;
+        if (node_test($self, $node)) {
+            $results->unshift($node);
+        }
+        push @stack, $node->getChildNodes;
+    }
+}
+
+sub axis_following {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    START:
+
+    my $parent = $context->getParentNode;
+    return $results unless $parent;
+        
+    while ($context = $context->getNextSibling) {
+        axis_descendant_or_self($self, $context, $results);
+    }
+
+    $context = $parent;
+    goto START;
+}
+
+sub axis_following_sibling {
+    my $self = shift;
+    my ($context, $results) = @_;
+
+    while ($context = $context->getNextSibling) {
+        if (node_test($self, $context)) {
+            $results->push($context);
+        }
+    }
+}
+
+sub axis_namespace {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    return $results unless $context->isElementNode;
+    foreach my $ns (@{$context->getNamespaces}) {
+        if ($self->test_namespace($ns)) {
+            $results->push($ns);
+        }
+    }
+}
+
+sub axis_parent {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    my $parent = $context->getParentNode;
+    return $results unless $parent;
+    if (node_test($self, $parent)) {
+        $results->push($parent);
+    }
+}
+
+sub axis_preceding {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    # all preceding nodes in document order, except ancestors
+    
+    START:
+
+    my $parent = $context->getParentNode;
+    return $results unless $parent;
+
+    while ($context = $context->getPreviousSibling) {
+        axis_descendant_or_self($self, $context, $results);
+    }
+    
+    $context = $parent;
+    goto START;
+}
+
+sub axis_preceding_sibling {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    while ($context = $context->getPreviousSibling) {
+        if (node_test($self, $context)) {
+            $results->push($context);
+        }
+    }
+}
+
+sub axis_self {
+    my $self = shift;
+    my ($context, $results) = @_;
+    
+    if (node_test($self, $context)) {
+        $results->push($context);
+    }
+}
+    
+sub node_test {
+    my $self = shift;
+    my $node = shift;
+    
+    # if node passes test, return true
+    
+    my $test = $self->{test};
+
+    return 1 if $test == test_nt_node;
+        
+    if ($test == test_any) {
+        return 1 if $node->isElementNode && defined $node->getName;
+    }
+        
+    local $^W;
+
+    if ($test == test_ncwild) {
+        return unless $node->isElementNode;
+        my $match_ns = $self->{pp}->get_namespace($self->{literal}, $node);
+        if (my $node_nsnode = $node->getNamespace()) {
+            return 1 if $match_ns eq $node_nsnode->getValue;
+        }
+    }
+    elsif ($test == test_qname) {
+        return unless $node->isElementNode;
+        if ($self->{literal} =~ /:/) {
+            my ($prefix, $name) = split(':', $self->{literal}, 2);
+            my $match_ns = $self->{pp}->get_namespace($prefix, $node);
+            if (my $node_nsnode = $node->getNamespace()) {
+#                warn "match: '$self->{literal}' match NS: '$match_ns' got NS: '", $node_nsnode->getValue, "'\n";
+                return 1 if ($match_ns eq $node_nsnode->getValue) &&
+                        ($name eq $node->getLocalName);
+            }
+        }
+        else {
+#            warn "Node test: ", $node->getName, "\n";
+            return 1 if $node->getName eq $self->{literal};
+        }
+    }
+    elsif ($test == test_nt_text) {
+        return 1 if $node->isTextNode;
+    }
+    elsif ($test == test_nt_comment) {
+        return 1 if $node->isCommentNode;
+    }
+#     elsif ($test == test_nt_pi && !$self->{literal}) {
+#         warn "Unreachable code???";
+#         return 1 if $node->isPINode;
+#     }
+    elsif ($test == test_nt_pi) {
+        return unless $node->isPINode;
+        if (my $val = $self->{literal}->value) {
+            return 1 if $node->getTarget eq $val;
+        }
+        else {
+            return 1;
+        }
+    }
+    
+    return; # fallthrough returns false
+}
+
+sub test_attribute {
+    my $self = shift;
+    my $node = shift;
+    
+#    warn "test_attrib: '$self->{test}' against: ", $node->getName, "\n";
+#    warn "node type: $node->[node_type]\n";
+    
+    my $test = $self->{test};
+    
+    return 1 if ($test == test_attr_any) || ($test == test_nt_node);
+        
+    if ($test == test_attr_ncwild) {
+        my $match_ns = $self->{pp}->get_namespace($self->{literal}, $node);
+        if (my $node_nsnode = $node->getNamespace()) {
+            return 1 if $match_ns eq $node_nsnode->getValue;
+        }
+    }
+    elsif ($test == test_attr_qname) {
+        if ($self->{literal} =~ /:/) {
+            my ($prefix, $name) = split(':', $self->{literal}, 2);
+            my $match_ns = $self->{pp}->get_namespace($prefix, $node);
+            if (my $node_nsnode = $node->getNamespace()) {
+                return 1 if ($match_ns eq $node_nsnode->getValue) &&
+                        ($name eq $node->getLocalName);
+            }
+        }
+        else {
+            return 1 if $node->getName eq $self->{literal};
+        }
+    }
+    
+    return; # fallthrough returns false
+}
+
+sub test_namespace {
+    my $self = shift;
+    my $node = shift;
+    
+    # Not sure if this is correct. The spec seems very unclear on what
+    # constitutes a namespace test... bah!
+    
+    my $test = $self->{test};
+    
+    return 1 if $test == test_any; # True for all nodes of principal type
+    
+    if ($test == test_any) {
+        return 1;
+    }
+    elsif ($self->{literal} eq $node->getExpanded) {
+        return 1;
+    }
+    
+    return;
+}
+
+sub filter_by_predicate {
+    my $self = shift;
+    my ($nodeset, $predicate) = @_;
+    
+    # See spec section 2.4, paragraphs 2 & 3:
+    # For each node in the node-set to be filtered, the predicate Expr
+    # is evaluated with that node as the context node, with the number
+    # of nodes in the node set as the context size, and with the
+    # proximity position of the node in the node set with respect to
+    # the axis as the context position.
+    
+    if (!ref($nodeset)) { # use ref because nodeset has a bool context
+        die "No nodeset!!!";
+    }
+    
+#    warn "Filter by predicate: $predicate\n";
+    
+    my $newset = XML::XPath::NodeSet->new();
+    
+    for(my $i = 1; $i <= $nodeset->size; $i++) {
+        # set context set each time 'cos a loc-path in the expr could change it
+        $self->{pp}->set_context_set($nodeset);
+        $self->{pp}->set_context_pos($i);
+        my $result = $predicate->evaluate($nodeset->get_node($i));
+        if ($result->isa('XML::XPath::Boolean')) {
+            if ($result->value) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+        elsif ($result->isa('XML::XPath::Number')) {
+            if ($result->value == $i) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+        else {
+            if ($result->to_boolean->value) {
+                $newset->push($nodeset->get_node($i));
+            }
+        }
+    }
+    
+    return $newset;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Twig.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Twig.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Twig.pm	(revision 21112)
@@ -0,0 +1,10940 @@
+# $Id$
+#
+# Copyright (c) 1999-2004 Michel Rodriguez
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the same terms as Perl itself.
+#
+
+# This is created in the caller's space
+BEGIN
+{ sub ::PCDATA { '#PCDATA' } 
+  sub ::CDATA  { '#CDATA'  } 
+}
+
+
+######################################################################
+package XML::Twig;
+######################################################################
+
+require 5.004;
+use strict; 
+use UNIVERSAL qw( isa ) ;
+
+use vars qw($VERSION @ISA %valid_option);
+use Carp;
+
+#start-extract twig_global
+
+# constants: element types
+use constant (PCDATA  => '#PCDATA');
+use constant (CDATA   => '#CDATA');
+use constant (PI      => '#PI');
+use constant (COMMENT => '#COMMENT');
+use constant (ENT     => '#ENT');
+
+# element classes
+use constant (ELT     => '#ELT');
+use constant (TEXT    => '#TEXT');
+
+# element properties
+use constant (ASIS    => '#ASIS');
+use constant (EMPTY   => '#EMPTY');
+
+#end-extract twig_global
+
+# used in parseurl to set the buffer size to the same size as in XML::Parser::Expat
+use constant (BUFSIZE => 32768);
+
+
+# used to store the gi's
+my %gi2index;   # gi => index
+my @index2gi;   # list of gi's
+my $SPECIAL_GI; # first non-special gi;
+my %base_ent;   # base entity character => replacement
+
+# flag, set to true if the weaken sub is available
+use vars qw( $weakrefs);
+
+#start-extract twig_global
+my $REG_NAME       = q{(?:(?:[^\W\d]|[:#])(?:[\w.-]*:)?[\w.-]*)};     # xml name
+my $REG_NAME_W     = q{(?:(?:[^\W\d_]|[:#])(?:[\w.-]*:)?[\w.-]*|\*)}; # name or wildcard (* or '')
+my $REG_REGEXP     = q{(?:/(?:[^\\/]|\\.)*/[eimsox]*)};   # regexp
+my $REG_REGEXP_EXP = q{(?:(?:[^\\/]|\\.)*)};              # content of a regexp
+my $REG_REGEXP_MOD = q{(?:[eimso]*)};                     # regexp modifiers
+my $REG_MATCH      = q{[!=]~};                            # match (or not)
+my $REG_STRING     = q{(?:"(?:[^\\"]|\\.)*"|'(?:[^\\']|\\.)*')};  # string (simple or double quoted)
+my $REG_NUMBER     = q{(?:\d+(?:\.\d*)?|\.\d+)};          # number
+my $REG_VALUE      = qq{(?:$REG_STRING|$REG_NUMBER)};     # value
+my $REG_OP         = q{=|==|!=|>|<|>=|<=|eq|ne|lt|gt|le|ge}; # op
+
+#end-extract twig_global
+
+my $parser_version;
+my( $FB_HTMLCREF, $FB_XMLCREF);
+
+
+BEGIN
+{ 
+$VERSION = '3.15';
+
+use XML::Parser;
+my $needVersion = '2.23';
+$parser_version= $XML::Parser::VERSION;
+croak "need at least XML::Parser version $needVersion" unless $parser_version >= $needVersion;
+
+if( $] >= 5.008) 
+  { eval "use Encode qw( :fallback_all)";
+    $FB_XMLCREF  = 0x0400; # Encode::FB_XMLCREF;
+    $FB_HTMLCREF = 0x0200; #Encode::FB_HTMLCREF;
+  }
+
+# test whether we can use weak references
+# set local empty signal handler to trap error messages
+{ local $SIG{__DIE__};
+  if( eval 'require Scalar::Util') { import Scalar::Util qw(weaken); $weakrefs= 1; }
+  elsif( eval 'require WeakRef')   { import WeakRef; $weakrefs= 1;                 }
+  else                             { $weakrefs= 0;                                 } 
+}
+
+import XML::Twig::Elt;
+import XML::Twig::Entity;
+import XML::Twig::Entity_list;
+
+# used to store the gi's
+# should be set for each twig really, at least when there are several
+# the init ensures that special gi's are always the same
+
+# gi => index
+# do NOT use => or the constants become quoted!
+%XML::Twig::gi2index=( PCDATA, 0, CDATA, 1, PI, 2, COMMENT, 3, ENT, 4); 
+# list of gi's
+@XML::Twig::index2gi=( PCDATA, CDATA, PI, COMMENT, ENT);
+
+# gi's under this value are special 
+$XML::Twig::SPECIAL_GI= @XML::Twig::index2gi;
+
+%XML::Twig::base_ent= ( '>' => '&gt;', '<' => '&lt;', '&' => '&amp;', "'" => '&apos;', '"' => '&quot;',);
+
+# now set some aliases
+*find_nodes           = *get_xpath;               # same as XML::XPath
+*findnodes            = *get_xpath;               # same as XML::LibXML
+*getElementsByTagName = *descendants;
+*descendants_or_self  = *descendants;             # valid in XML::Twig, not in XML::Twig::Elt
+*find_by_tag_name     = *descendants;
+*getElementById       = *elt_id;
+*getEltById           = *elt_id;
+*toString             = *sprint;
+}
+
+@ISA = qw(XML::Parser);
+
+# fake gi's used in twig_handlers and start_tag_handlers
+my $ALL    = '_all_';     # the associated function is always called
+my $DEFAULT= '_default_'; # the function is called if no other handler has been
+
+# some defaults
+my $COMMENTS_DEFAULT= 'keep';
+my $PI_DEFAULT      = 'keep';
+
+
+# handlers used in regular mode
+my %twig_handlers=( Start      => \&_twig_start, 
+                    End        => \&_twig_end, 
+                    Char       => \&_twig_char, 
+                    Entity     => \&_twig_entity, 
+                    XMLDecl    => \&_twig_xmldecl, 
+                    Doctype    => \&_twig_doctype, 
+                    Element    => \&_twig_element, 
+                    Attlist    => \&_twig_attlist, 
+                    CdataStart => \&_twig_cdatastart, 
+                    CdataEnd   => \&_twig_cdataend, 
+                    Proc       => \&_twig_pi,
+                    Comment    => \&_twig_comment,
+                    Default    => \&_twig_default,
+      );
+
+# handlers used when twig_roots is used and we are outside of the roots
+my %twig_handlers_roots=
+  ( Start      => \&_twig_start_check_roots, 
+    End        => \&_twig_end_check_roots, 
+    Doctype    => \&_twig_doctype, 
+    Char       => undef, Entity     => undef, XMLDecl    => \&_twig_xmldecl, 
+    Element    => undef, Attlist    => undef, CdataStart => undef, 
+    CdataEnd   => undef, Proc       => undef, Comment    => undef, 
+    Proc       => \&_twig_pi_check_roots,
+    Default    =>  sub {}, # hack needed for XML::Parser 2.27
+  );
+
+# handlers used when twig_roots and print_outside_roots are used and we are
+# outside of the roots
+my %twig_handlers_roots_print_2_30=
+  ( Start      => \&_twig_start_check_roots_print, 
+    End        => \&_twig_end_check_roots_print, 
+    Char       => \&_twig_print, 
+    # I have no idea why I should not be using this handler!
+    Entity     => \&_twig_print_entity, 
+    XMLDecl    => \&_twig_print,
+    Doctype   =>  \&_twig_print_doctype, # because recognized_string is broken here
+    # Element    => \&_twig_print, Attlist    => \&_twig_print, 
+    CdataStart => \&_twig_print, CdataEnd   => \&_twig_print, 
+    Proc       => \&_twig_print, Comment    => \&_twig_print, 
+    Default    => \&_twig_print_check_doctype,
+  );
+
+# handlers used when twig_roots, print_outside_roots and keep_encoding are used
+# and we are outside of the roots
+my %twig_handlers_roots_print_original_2_30=
+  ( Start      => \&_twig_start_check_roots_print_original, 
+    End        => \&_twig_end_check_roots_print_original, 
+    Char       => \&_twig_print_original, 
+    # I have no idea why I should not be using this handler!
+    #Entity     => \&_twig_print_original, 
+    ExternEnt  => \&_twig_print_entity,
+    XMLDecl    => \&_twig_print_original, 
+    Doctype    => \&_twig_print_original_doctype,  # because original_string is broken here
+    Element    => \&_twig_print_original, Attlist   => \&_twig_print_original,
+    CdataStart => \&_twig_print_original, CdataEnd  => \&_twig_print_original,
+    Proc       => \&_twig_print_original, Comment   => \&_twig_print_original,
+    Default    => \&_twig_print_original_check_doctype, 
+  );
+
+# handlers used when twig_roots and print_outside_roots are used and we are
+# outside of the roots
+my %twig_handlers_roots_print_2_27=
+  ( Start      => \&_twig_start_check_roots_print, 
+    End        => \&_twig_end_check_roots_print, 
+    Char       => \&_twig_print, 
+    # I have no idea why I should not be using this handler!
+    #Entity     => \&_twig_print, 
+    XMLDecl    => \&_twig_print, Doctype    => \&_twig_print, 
+    CdataStart => \&_twig_print, CdataEnd   => \&_twig_print, 
+    Proc       => \&_twig_print, Comment    => \&_twig_print, 
+    Default    => \&_twig_print, 
+  );
+
+# handlers used when twig_roots, print_outside_roots and keep_encoding are used
+# and we are outside of the roots
+my %twig_handlers_roots_print_original_2_27=
+  ( Start      => \&_twig_start_check_roots_print_original, 
+    End        => \&_twig_end_check_roots_print_original, 
+    Char       => \&_twig_print_original, 
+    # for some reason original_string is wrong here 
+    # this can be a problem if the doctype includes non ascii characters
+    XMLDecl    => \&_twig_print, Doctype    => \&_twig_print,
+    # I have no idea why I should not be using this handler!
+    Entity     => \&_twig_print, 
+    #Element    => undef, Attlist   => undef,
+    CdataStart => \&_twig_print_original, CdataEnd  => \&_twig_print_original,
+    Proc       => \&_twig_print_original, Comment   => \&_twig_print_original,
+    Default    => \&_twig_print_default, #  _twig_print_original does not work
+  );
+
+
+my %twig_handlers_roots_print= $parser_version > 2.27 
+                               ? %twig_handlers_roots_print_2_30 
+                               : %twig_handlers_roots_print_2_27; 
+my %twig_handlers_roots_print_original= $parser_version > 2.27 
+                               ? %twig_handlers_roots_print_original_2_30 
+                               : %twig_handlers_roots_print_original_2_27; 
+
+
+# handlers used when the finish_print method has been called
+my %twig_handlers_finish_print=
+  ( Start      => \&_twig_print, 
+    End        => \&_twig_print_end, Char       => \&_twig_print, 
+    Entity     => \&_twig_print, XMLDecl    => \&_twig_print, 
+    Doctype    => \&_twig_print, Element    => \&_twig_print, 
+    Attlist    => \&_twig_print, CdataStart => \&_twig_print, 
+    CdataEnd   => \&_twig_print, Proc       => \&_twig_print, 
+    Comment    => \&_twig_print, Default    => \&_twig_print, 
+  );
+
+# handlers used when the finish_print method has been called and the keep_encoding
+# option is used
+my %twig_handlers_finish_print_original=
+  ( Start      => \&_twig_print_original, End      => \&_twig_print_end_original, 
+    Char       => \&_twig_print_original, Entity   => \&_twig_print_original, 
+    XMLDecl    => \&_twig_print_original, Doctype  => \&_twig_print_original, 
+    Element    => \&_twig_print_original, Attlist  => \&_twig_print_original, 
+    CdataStart => \&_twig_print_original, CdataEnd => \&_twig_print_original, 
+    Proc       => \&_twig_print_original, Comment  => \&_twig_print_original, 
+    Default    => \&_twig_print_original, 
+  );
+
+# handlers used whithin ignored elements
+my %twig_handlers_ignore=
+  ( Start      => \&_twig_ignore_start, 
+    End        => \&_twig_ignore_end, 
+    Char       => undef, Entity     => undef, XMLDecl    => undef, 
+    Doctype    => undef, Element    => undef, Attlist    => undef, 
+    CdataStart => undef, CdataEnd   => undef, Proc       => undef, 
+    Comment    => undef, Default    => undef,
+  );
+
+
+# those handlers are only used if the entities are NOT to be expanded
+my %twig_noexpand_handlers= ( Default => \&_twig_default );
+
+my @saved_default_handler;
+
+my $ID= 'id'; # default value, set by the Id argument
+
+# all allowed options
+%valid_option=
+    ( # XML::Twig options
+      TwigHandlers          => 1, Id                    => 1,
+      TwigRoots             => 1, TwigPrintOutsideRoots => 1,
+      StartTagHandlers      => 1, EndTagHandlers        => 1,
+      ForceEndTagHandlersUsage => 1,
+      DoNotChainHandlers    => 1,
+      IgnoreElts            => 1,
+      CharHandler           => 1, KeepEncoding          => 1,
+      ParseStartTag         => 1, KeepAttsOrder         => 1,
+      LoadDTD               => 1, DTDHandler            => 1,
+      DoNotOutputDTD        => 1, NoProlog              => 1,
+      ExpandExternalEnts    => 1,
+      DiscardSpaces         => 1, KeepSpaces            => 1, 
+      DiscardSpacesIn       => 1, KeepSpacesIn          => 1, 
+      PrettyPrint           => 1, EmptyTags             => 1, 
+      Comments              => 1, Pi                    => 1, 
+      OutputFilter          => 1, InputFilter           => 1,
+      OutputTextFilter      => 1, 
+      OutputEncoding        => 1, 
+      EltClass              => 1,
+      MapXmlns              => 1, KeepOriginalPrefix    => 1,
+      # XML::Parser options
+      ErrorContext          => 1, ProtocolEncoding      => 1,
+      Namespaces            => 1, NoExpand              => 1,
+      Stream_Delimiter      => 1, ParseParamEnt         => 1,
+      NoLWP                 => 1, Non_Expat_Options     => 1,
+      Xmlns                 => 1,
+    );
+
+# predefined input and output filters
+use vars qw( %filter);
+%filter= ( html       => \&html_encode,
+           safe       => \&safe_encode,
+           safe_hex   => \&safe_encode_hex,
+         );
+
+sub new
+  { my ($class, %args) = @_;
+    my $handlers;
+
+    # change all nice_perlish_names into nicePerlishNames
+    %args= _normalize_args( %args);
+
+    # check options
+    unless( $args{MoreOptions})
+      { foreach my $arg (keys %args)
+        { carp "invalid option $arg" unless $valid_option{$arg}; }
+      }
+     
+    # a twig is really an XML::Parser
+    # my $self= XML::Parser->new(%args);
+    my $self;
+    $self= XML::Parser->new(%args);   
+    
+    bless $self, $class;
+
+    if( exists $args{TwigHandlers})
+      { $handlers= $args{TwigHandlers};
+        $self->setTwigHandlers( $handlers);
+        delete $args{TwigHandlers};
+      }
+
+    # take care of twig-specific arguments
+    if( exists $args{StartTagHandlers})
+      { $self->setStartTagHandlers( $args{StartTagHandlers});
+        delete $args{StartTagHandlers};
+      }
+
+    if( exists $args{DoNotChainHandlers})
+      { $self->{twig_do_not_chain_handlers}=  $args{DoNotChainHandlers}; }
+
+    if(  exists $args{IgnoreElts})
+      { $self->setIgnoreEltsHandlers( $args{IgnoreElts});
+        delete $args{IgnoreElts};
+      }
+
+    $self->{twig_elt_class}= $args{EltClass} || 'XML::Twig::Elt';
+    if( exists( $args{EltClass})) { delete $args{EltClass}; }
+
+    if( exists( $args{MapXmlns}))
+      { $self->{twig_map_xmlns}=  $args{MapXmlns};
+        $self->{Namespaces}=1;
+        delete $args{MapXmlns};
+      }
+
+    if( exists( $args{KeepOriginalPrefix}))
+      { $self->{twig_keep_original_prefix}= $args{KeepOriginalPrefix};
+        delete $args{KeepOriginalPrefix};
+      }
+
+    $self->{twig_dtd_handler}= $args{DTDHandler};
+    delete $args{DTDHandler};
+
+    if( $args{CharHandler})
+      { $self->setCharHandler( $args{CharHandler});
+        delete $args{CharHandler};
+      }
+
+    if( $args{LoadDTD})
+      { $self->{twig_read_external_dtd}= 1;
+        delete $args{LoadDTD};
+      }
+      
+    if( $args{ExpandExternalEnts})
+      { $self->set_expand_external_entities( 1);
+        $self->{twig_read_external_dtd}= 1; # implied by ExpandExternalEnts
+        delete $args{LoadDTD};
+        delete $args{ExpandExternalEnts};
+      }
+
+    # deal with TwigRoots argument, a hash of elements for which
+    # subtrees will be built (and associated handlers)
+     
+    if( $args{TwigRoots})
+      { $self->setTwigRoots( $args{TwigRoots});
+        delete $args{TwigRoots}; 
+      }
+    
+    if( $args{EndTagHandlers})
+      { unless ($self->{twig_roots} || $args{ForceEndTagHandlersUsage})
+          { croak "you should not use EndTagHandlers without TwigRoots\n",
+                  "if you want to use it anyway, normally because you have ",
+                  "a start_tag_handlers that calls 'ignore' and you want to ",
+                  "call an ent_tag_handlers at the end of the element, then ",
+                  "pass 'force_end_tag_handlers_usage => 1' as an argument ",
+                  "to new";
+          }
+                  
+        $self->setEndTagHandlers( $args{EndTagHandlers});
+        delete $args{EndTagHandlers};
+      }
+      
+    if( $args{TwigPrintOutsideRoots})
+      { croak "cannot use TwigPrintOutsideRoots without TwigRoots"
+          unless( $self->{twig_roots});
+        # if the arg is a filehandle then store it
+        if( _is_fh( $args{TwigPrintOutsideRoots}) )
+          { $self->{twig_output_fh}= $args{TwigPrintOutsideRoots}; }
+        $self->{twig_default_print}= $args{TwigPrintOutsideRoots};
+      }
+
+    if( $args{PrettyPrint})
+      { $self->set_pretty_print( $args{PrettyPrint}); }
+
+    if( $args{EmptyTags})
+      { $self->set_empty_tag_style( $args{EmptyTags}); }
+
+    # space policy
+    if( $args{KeepSpaces})
+      { croak "cannot use both keep_spaces and discard_spaces" if( $args{DiscardSpaces});
+        croak "cannot use both keep_spaces and keep_spaces_in" if( $args{KeepSpacesIn});
+        $self->{twig_keep_spaces}=1;
+        delete $args{KeepSpaces}; 
+      }
+    if( $args{DiscardSpaces})
+      { croak "cannot use both discard_spaces and keep_spaces_in" if( $args{KeepSpacesIn});
+        $self->{twig_discard_spaces}=1; 
+        delete $args{DiscardSpaces}; 
+      }
+    if( $args{KeepSpacesIn})
+      { croak "cannot use both keep_spaces_in and discard_spaces_in" if( $args{DiscardSpacesIn});
+        $self->{twig_discard_spaces}=1; 
+        $self->{twig_keep_spaces_in}={}; 
+        my @tags= @{$args{KeepSpacesIn}}; 
+        foreach my $tag (@tags) { $self->{twig_keep_spaces_in}->{$tag}=1; } 
+        delete $args{KeepSpacesIn}; 
+      }
+    if( $args{DiscardSpacesIn})
+      { $self->{twig_keep_spaces}=1; 
+        $self->{twig_discard_spaces_in}={}; 
+        my @tags= @{$args{DiscardSpacesIn}};
+        foreach my $tag (@tags) { $self->{twig_discard_spaces_in}->{$tag}=1; } 
+        delete $args{DiscardSpacesIn}; 
+      }
+    # discard spaces by default 
+    $self->{twig_discard_spaces}= 1 unless(  $self->{twig_keep_spaces});
+
+    $args{Comments}||= $COMMENTS_DEFAULT;
+    if( $args{Comments} eq 'drop')       { $self->{twig_keep_comments}= 0;    }
+    elsif( $args{Comments} eq 'keep')    { $self->{twig_keep_comments}= 1;    }
+    elsif( $args{Comments} eq 'process') { $self->{twig_process_comments}= 1; }
+    else { croak "wrong value for comments argument: '$args{Comments}' (should be 'drop', 'keep' or 'process')"; }
+    delete $args{Comments};
+
+    $args{Pi}||= $PI_DEFAULT;
+    if( $args{Pi} eq 'drop')       { $self->{twig_keep_pi}= 0;    }
+    elsif( $args{Pi} eq 'keep')    { $self->{twig_keep_pi}= 1;    }
+    elsif( $args{Pi} eq 'process') { $self->{twig_process_pi}= 1; }
+    else { croak "wrong value for pi argument: '$args{Pi}' (should be 'drop', 'keep' or 'process')"; }
+    delete $args{Pi};
+
+    if( $args{KeepEncoding})
+      { $self->{twig_keep_encoding}= $args{KeepEncoding};
+        # set it in XML::Twig::Elt so print functions know what to do
+        $self->set_keep_encoding( 1); 
+        $self->{parse_start_tag}= $args{ParseStartTag} || \&_parse_start_tag; 
+        delete $args{ParseStartTag} if defined( $args{ParseStartTag}) ;
+        delete $args{KeepEncoding};
+        $self->{NoExpand}= 1;
+      }
+    else
+      { $self->set_keep_encoding( 0);  
+        $self->{parse_start_tag}= $args{ParseStartTag} if( $args{ParseStartTag}); 
+      }
+
+    if( $args{OutputFilter})
+      { $self->set_output_filter( $args{OutputFilter}); 
+        delete $args{OutputFilter};
+      }
+    else
+      { $self->set_output_filter( 0); }
+
+    if( $args{OutputTextFilter})
+      { $self->set_output_text_filter( $args{OutputTextFilter}); 
+        delete $args{OutputTextFilter};
+      }
+    else
+      { $self->set_output_text_filter( 0); }
+
+
+    if( exists $args{KeepAttsOrder})
+      { $self->{keep_atts_order}= $args{KeepAttsOrder};
+        if( eval 'require Tie::IxHash') 
+          { import Tie::IxHash; 
+            $self->set_keep_atts_order(  $self->{keep_atts_order}); 
+          }
+        else 
+          { carp "Tie::IxHash not available, option  keep_atts_order not allowed"; }
+      }
+    else
+      { $self->set_keep_atts_order( 0); }
+
+    if( my $output_encoding= $args{OutputEncoding})
+      { $self->set_output_encoding( $output_encoding);
+        delete $args{OutputFilter};
+      }
+
+    if( $args{InputFilter})
+      { $self->set_input_filter(  $args{InputFilter}); 
+        delete  $args{InputFilter}; 
+      }
+
+    if( exists $args{Id}) { $ID= $args{Id}; delete $args{ID}; }
+
+    if( $args{NoExpand})
+      { $self->setHandlers( %twig_noexpand_handlers);
+        $self->{twig_no_expand}=1;
+      }
+
+    if( $args{NoProlog})
+      { $self->{no_prolog}= 1; 
+        delete $args{NoProlog}; 
+      }
+
+    if( $args{DoNotOutputDTD})
+      { $self->{no_dtd_output}= 1; 
+        delete $args{DoNotOutputDTD}; 
+      }
+
+    # set handlers
+    if( $self->{twig_roots})
+      { if( $self->{twig_default_print})
+          { if( $self->{twig_keep_encoding})
+              { $self->setHandlers( %twig_handlers_roots_print_original); }
+            else
+              { $self->setHandlers( %twig_handlers_roots_print);  }
+          }
+        else
+          { $self->setHandlers( %twig_handlers_roots); }
+      }
+    else
+      { $self->setHandlers( %twig_handlers); }
+
+    # XML::Parser::Expat does not like these handler to be set. So in order to 
+    # use the various sets of handlers on XML::Parser or XML::Parser::Expat
+    # objects when needed, these ones have to be set only once, here, at 
+    # XML::Parser level
+    $self->setHandlers( Init => \&_twig_init, Final => \&_twig_final);
+
+    $self->{twig_entity_list}= XML::Twig::Entity_list->new; 
+
+    $self->{twig_id}= $ID; 
+    $self->{twig_stored_spaces}='';
+
+    $self->{twig}= $self;
+    weaken( $self->{twig}) if( $weakrefs);
+
+    return $self;
+  }
+
+
+sub parseurl
+  { my $t= shift;
+    return $t->_parseurl( 0, @_);
+  }
+
+sub safe_parseurl
+  { my $t= shift;
+    return $t->_parseurl( 1, @_);
+  }
+
+# I should really add extra options to allow better configuration of the 
+# LWP::UserAgent object
+# this method forks: 
+#   - the child gets the data and copies it to the pipe,
+#   - the parent reads the stream and sends it to XML::Parser
+# the data is cut it chunks the size of the XML::Parser::Expat buffer
+# the method returns the twig and the status
+sub _parseurl
+  { my( $t, $safe, $url, $agent)= @_;
+    pipe( README, WRITEME) or croak  "cannot create connected pipes: $!";
+    if( my $pid= fork)
+      { # parent code: parse the incoming file
+        close WRITEME; # no need to write
+        my $result= $safe ? $t->safe_parse( \*README) : $t->parse( \*README);
+        close README;
+        return $@ ? 0 : $t;
+      }
+    else
+     { # child
+        close README; # no need to read
+        require LWP;  # so we can get LWP::UserAgent and HTTP::Request
+        $|=1;
+        $agent    ||= LWP::UserAgent->new;
+        my $request  = HTTP::Request->new( GET => $url);
+        # _pass_url_content is called with chunks of data the same size as
+        # the XML::Parser buffer 
+        my $response = $agent->request( $request, 
+                         sub { _pass_url_content( \*WRITEME, @_); }, BUFSIZE);
+        $response->is_success or croak "$url ", $response->message;
+        close WRITEME;
+        CORE::exit(); # CORE is there for mod_perl (which redefines exit)
+      }
+  }
+
+# get the (hopefully!) XML data from the URL and 
+sub _pass_url_content
+  { my( $fh, $data, $response, $protocol)= @_;
+    print $fh $data;
+  }
+
+sub add_options
+  { my %args= map { $_, 1 } @_;
+    %args= _normalize_args( %args);
+    foreach (keys %args) { $valid_option{$_}++; } 
+  }
+
+sub _twig_store_internal_dtd
+  { 
+    my( $p, $string)= @_;
+    my $t= $p->{twig};
+    $string= $p->original_string() if( $t->{twig_keep_encoding});
+    $t->{twig_doctype}->{internal} .= $string;
+  }
+
+sub _twig_stop_storing_internal_dtd
+  { my $p= shift;
+    if( @saved_default_handler && defined $saved_default_handler[1])
+      { $p->setHandlers( @saved_default_handler); }
+    else
+      { my $t= $p->{twig};
+        $p->setHandlers( Default => undef);
+      }
+  }
+
+
+sub _normalize_args
+  { my %normalized_args;
+    while( my $key= shift )
+      { $key= join '', map { ucfirst } split /_/, $key;
+        #$key= "Twig".$key unless( substr( $key, 0, 4) eq 'Twig');
+        $normalized_args{$key}= shift ;
+      }
+    return %normalized_args;
+  }    
+
+sub _is_fh { return unless $_[0]; return isa( $_[0], 'GLOB') || isa( $_[0], 'IO::Scalar'); }
+
+sub _set_handler
+  { my( $handlers, $path, $handler)= @_;
+
+    $handlers ||= {}; # create the handlers struct if necessary
+
+    my $prev_handler= $handlers->{handlers}->{$path} || undef;
+
+       _set_gi_handler              ( $handlers, $path, $handler, $prev_handler)
+    || _set_path_handler            ( $handlers, $path, $handler, $prev_handler)
+    || _set_subpath_handler         ( $handlers, $path, $handler, $prev_handler)
+    || _set_attribute_handler       ( $handlers, $path, $handler, $prev_handler)
+    || _set_star_att_handler        ( $handlers, $path, $handler, $prev_handler)
+    || _set_star_att_regexp_handler ( $handlers, $path, $handler, $prev_handler)
+    || _set_string_handler          ( $handlers, $path, $handler, $prev_handler)
+    || _set_attribute_regexp_handler( $handlers, $path, $handler, $prev_handler)
+    || _set_string_regexp_handler   ( $handlers, $path, $handler, $prev_handler)
+    || _set_pi_handler              ( $handlers, $path, $handler, $prev_handler)
+    || croak "unrecognized expression in handler: '$path'";
+
+
+    # this both takes care of the simple (gi) handlers and store
+    # the handler code reference for other handlers
+    $handlers->{handlers}->{$path}= $handler;
+
+    return $prev_handler;
+  }
+
+
+sub _set_gi_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    if( $path =~ m{^\s*($REG_NAME)\s*$}o )
+      { my $gi= $1;
+        $handlers->{handlers}->{gi}->{$gi}= $handler; 
+        return 1;
+      }
+    else 
+      { return 0; }
+  }
+
+sub _set_path_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    if( $path=~ m{^\s*(?:/$REG_NAME)*/($REG_NAME)\s*$}o)
+      { # a full path has been defined
+        # update the path_handlers count, knowing that
+        # either the previous or the new handler can be undef
+        $handlers->{path_handlers}->{gi}->{$1}-- if( $prev_handler);
+        if( $handler)
+         { $handlers->{path_handlers}->{gi}->{$1}++;
+           $handlers->{path_handlers}->{path}->{$path}= $handler;
+         }
+        return 1;
+      }
+    else 
+      { return 0; }
+  }
+
+sub _set_subpath_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    if( $path=~ m{^\s*(?:$REG_NAME/)+($REG_NAME)\s*$}o)
+      { # a partial path has been defined
+        # $1 is the "final" gi
+        $handlers->{subpath_handlers}->{gi}->{$1}-- if( $prev_handler);
+        if( $handler)
+         { $handlers->{subpath_handlers}->{gi}->{$1}++;
+           $handlers->{subpath_handlers}->{path}->{$path}= $handler;
+         }
+        return 1;
+      }
+    else 
+      { return 0; }
+  }
+
+sub _set_attribute_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for attribute conditions
+    if( $path=~ m{^\s*($REG_NAME)          # elt
+                 \s*\[\s*\@                #    [@
+                 ($REG_NAME)\s*            #      att
+                 (?:=\s*($REG_STRING)\s*)? #           = value (optional)         
+                 \]\s*$}xo)                #                             ]  
+      { my( $gi, $att, $val)= ($1, $2, $3);
+        $val= substr( $val, 1, -1) if( defined $val); # remove the quotes
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{attcond_handlers_exp}->{$gi}})
+             { if( ($exp->{att} eq $att) && ( _eq( $exp->{val}, $val)) )
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler; }
+                   else          # remove the handler
+                     { $handlers->{attcond_handlers}->{$gi}--;
+                       splice( @{$handlers->{attcond_handlers_exp}->{$gi}}, $i, 1);
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            $handlers->{attcond_handlers}->{$gi}++;
+            my $exp={att => $att, val => $val, handler => $handler};
+            $handlers->{attcond_handlers_exp}->{$gi} ||= [];
+            push @{$handlers->{attcond_handlers_exp}->{$gi}}, $exp;
+          }
+        return 1;
+      }
+    else 
+      { return 0; }
+  }
+
+
+sub _set_attribute_regexp_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for attribute regexp conditions
+    if( $path=~ m{^\s*($REG_NAME)     # elt
+                 \s*\[\s*\@           #    [@
+                 ($REG_NAME)          #      att
+                 \s*=~\s*             #          =~
+                 /($REG_REGEXP_EXP)/  #             /regexp/
+                 ($REG_REGEXP_MOD)    #                     mods
+                 \s*]\s*$}gxo)        #                         ] 
+      { my( $gi, $att, $regexp, $mods)= ($1, $2, $3, $4);
+        $regexp= qr/(?$mods:$regexp)/;
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{attregexp_handlers_exp}->{$gi}})
+             { if( ($exp->{att} eq $att) && ($exp->{regexp} eq $regexp) )
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler; }
+                   else          # remove the handler
+                     { $handlers->{attregexp_handlers}->{$gi}--;
+                       splice( @{$handlers->{attregexp_handlers_exp}->{$gi}}, $i, 1);
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            $handlers->{attregexp_handlers}->{$gi}++;
+            my $exp={att => $att, regexp => $regexp, handler => $handler};
+            $handlers->{attregexp_handlers_exp}->{$gi} ||= [];
+            push @{$handlers->{attregexp_handlers_exp}->{$gi}}, $exp;
+          }
+        return 1;
+      }
+    else 
+      { return 0; }
+  }
+
+sub _set_string_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for string conditions
+    if( $path=~/^\s*($REG_NAME)            # elt
+                 \s*\[\s*string            #    [string
+                 \s*\(\s*($REG_NAME)?\s*\) #           (sub_elt)
+                 \s*=\s*                   #                     =
+                 ($REG_STRING)             #                       "text" (or 'text')
+                 \s*\]\s*$/ox)             #                              ] 
+      { my( $gi, $sub_elt, $text)= ($1, $2, $3);
+        $text= substr( $text, 1, -1) if( defined $text); # remove the quotes
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{text_handlers_exp}->{$gi}})
+             { if( ($exp->{text} eq $text) &&
+                   ( !$exp->{sub_elt} || ($exp->{sub_elt} eq $sub_elt) )
+                 )
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler; }
+                   else          # remove the handler
+                     { $handlers->{text_handlers}->{$gi}--;
+                       splice( @{$handlers->{text_handlers_exp}->{$gi}}, $i, 1);
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            $handlers->{text_handlers}->{$gi}++;
+            my $exp={sub_elt => $sub_elt, text => $text, handler => $handler};
+            $handlers->{text_handlers_exp}->{$gi} ||= [];
+            push @{$handlers->{text_handlers_exp}->{$gi}}, $exp;
+          }
+        return 1;
+      }
+    else 
+      { return 0; 
+      }
+  }
+
+
+sub _set_string_regexp_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for string regexp conditions
+    if( $path=~m{^\s*($REG_NAME)        # (elt)
+                 \s*\[\s*string         #    [string
+                 \s*\(\s*($REG_NAME?)\) #           (sub_elt)
+                 \s*=~\s*               #              =~ 
+                 /($REG_REGEXP_EXP)/    #                 /(regexp)/
+                 \s*($REG_REGEXP_MOD)?  #                         (mods)
+                 \s*\]\s*$}ox)          #                             ]   (or ')
+      { my( $gi, $sub_elt, $regexp, $mods)= ($1, $2, $3, $4);
+        $mods||="";
+        $regexp= qr/(?$mods:$regexp)/;
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{regexp_handlers_exp}->{$gi}})
+             { if( ($exp->{regexp} eq $regexp) &&
+                   ( !$exp->{sub_elt} || ($exp->{sub_elt} eq $sub_elt) )
+                 )
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler;  
+                     }
+                   else          # remove the handler
+                     { $handlers->{regexp_handlers}->{$gi}--;
+                       splice( @{$handlers->{regexp_handlers_exp}->{$gi}}, $i, 1);
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            $handlers->{regexp_handlers}->{$gi}++;
+            my $exp= {sub_elt => $sub_elt, regexp => $regexp, handler => $handler};
+            $handlers->{regexp_handlers_exp}->{$gi} ||= [];
+            push @{$handlers->{regexp_handlers_exp}->{$gi}}, $exp;
+          }
+        return 1;
+      }
+    else 
+      { return 0; 
+      }
+  }
+
+
+sub _set_star_att_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for *[@att="val"] or *[@att] conditions
+    if( $path=~/^(?:\s*\*)?         # * (optional)
+                 \s*\[\s*\@         #    [@
+                 ($REG_NAME)        #      att
+                 (?:\s*=\s*         #         = 
+                 ($REG_STRING))?    #           string
+                     \s*\]\s*$/ox)  #                 ]  
+      { my( $att, $val)= ($1, $2);
+        $val= substr( $val, 1, -1) if( defined $val); # remove the quotes from the string
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{att_handlers_exp}->{$att}})
+             { if( ($exp->{att} eq $att) && ( !defined( $val) || _eq( $exp->{val}, $val) ) )
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler; }
+                   else          # remove the handler
+                     { splice( @{$handlers->{att_handlers_exp}->{$att}}, $i, 1);
+                   $handlers->{att_handlers}->{$att}--;
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            $handlers->{att_handlers}->{$att}++;
+            my $exp={att => $att, val => $val, handler => $handler};
+            $handlers->{att_handlers_exp}->{$att} ||= [];
+            push @{$handlers->{att_handlers_exp}->{$att}}, $exp;
+          }
+        return 1;
+      }
+    else 
+      { return 0; 
+      }
+  }
+
+sub _set_star_att_regexp_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # check for *[@att=~ /regexp/] conditions
+    if( $path=~ m{^(?:\s*\*)?             # * (optional)
+                   \s*\[\s*\@             #  [@
+                   ($REG_NAME)            #    att
+                   \s*=~\s*               #        =~ 
+                   /($REG_REGEXP_EXP)/    #           /(regexp)/
+                   \s*($REG_REGEXP_MOD)?  #                     (mods)
+                   \s*\]\s*$}ox)          #                           ]  
+      { my( $att, $regexp, $mods)= ($1, $2, $3);
+        $mods||="";
+        $regexp= qr/(?$mods:$regexp)/;
+        if( $prev_handler)
+          { # replace or remove the previous handler
+            my $i=0; # so we can splice the array if need be
+            foreach my $exp ( @{$handlers->{att_regexp_handlers_exp}->{$att}})
+             { if( $exp->{regexp} eq $regexp)
+                 { if( $handler) # just replace the handler
+                     { $exp->{handler}= $handler;  
+                     }
+                   else          # remove the handler
+                     { splice( @{$handlers->{att_regexp_handlers_exp}->{$att}}, $i, 1);
+                   $handlers->{att_regexp_handlers}--;
+                       last;
+                     }
+                 }
+               $i++;
+             }
+          }
+        elsif( $handler)
+          { # new handler only
+            my $exp= { regexp => $regexp, handler => $handler};
+            $handlers->{regexp_handlers_exp}->{$att} ||= [];
+            push @{$handlers->{att_regexp_handlers_exp}->{$att}}, $exp;
+            $handlers->{att_regexp_handlers}++;
+          }
+        return 1;
+      }
+    else 
+      { return 0; 
+      }
+  }
+
+
+sub _set_pi_handler
+  { my( $handlers, $path, $handler, $prev_handler)= @_;
+    # PI conditions ( '?target' => \&handler or '?' => \&handler
+    #             or '#PItarget' => \&handler or '#PI' => \&handler)
+    if( $path=~ /^\s*(?:\?|#PI)\s*(?:([^\s]*)\s*)$/)
+      { my $target= $1 || '';
+        # update the path_handlers count, knowing that
+        # either the previous or the new handler can be undef
+        $handlers->{pi_handlers}->{$1}= $handler;
+        return 1;
+      }
+    else 
+      { return 0; 
+      }
+  }
+
+# just like eq except that undef values do not trigger warnings
+sub _eq
+  { my( $val1, $val2)= @_;
+    if( !defined $val1) { return !defined $val2 }
+    if( !defined $val2) { return 0; }
+    return $val1 eq $val2;
+  }
+
+sub setCharHandler
+  { my( $t, $handler)= @_;
+    $t->{twig_char_handler}= $handler;
+  }
+
+
+sub _reset_handlers
+  { my $handlers= shift;
+    delete $handlers->{handlers};
+    delete $handlers->{path_handlers};
+    delete $handlers->{subpath_handlers};
+    $handlers->{attcond_handlers_exp}=[] if( $handlers->{attcond_handlers});
+    delete $handlers->{attcond_handlers};
+  }
+  
+sub _set_handlers
+  { my $handlers= shift || return;
+    my $set_handlers= {};
+    foreach my $path (keys %{$handlers})
+      { _set_handler( $set_handlers, $path, $handlers->{$path}); }
+    return $set_handlers;
+  }
+    
+
+sub setTwigHandler
+  { my( $t, $path, $handler)= @_;
+    $t->{twig_handlers} ||={};
+    return _set_handler( $t->{twig_handlers}, $path, $handler);
+  }
+
+sub setTwigHandlers
+  { my( $t, $handlers)= @_;
+    my $previous_handlers= $t->{twig_handlers} || undef;
+    _reset_handlers( $t->{twig_handlers});
+    $t->{twig_handlers}= _set_handlers( $handlers);
+    return $previous_handlers;
+  }
+
+sub setStartTagHandler
+  { my( $t, $path, $handler)= @_;
+    $t->{twig_starttag_handlers}||={};
+    return _set_handler( $t->{twig_starttag_handlers}, $path, $handler);
+  }
+
+sub setStartTagHandlers
+  { my( $t, $handlers)= @_;
+    my $previous_handlers= $t->{twig_starttag_handlers} || undef;
+    _reset_handlers( $t->{twig_starttag_handlers});
+    $t->{twig_starttag_handlers}= _set_handlers( $handlers);
+    return $previous_handlers;
+   }
+
+sub setIgnoreEltsHandler
+  { my( $t, $path, $action)= @_;
+    $t->{twig_ignore_elts_handlers}||={};
+    return _set_handler( $t->{twig_ignore_elts_handlers}, $path, $action );
+  }
+
+sub setIgnoreEltsHandlers
+  { my( $t, $handlers)= @_;
+    my $previous_handlers= $t->{twig_ignore_elts_handlers} || undef;
+    _reset_handlers( $t->{twig_ignore_elts_handlers});
+    $t->{twig_ignore_elts_handlers}= _set_handlers( $handlers);
+    return $previous_handlers;
+   }
+
+sub setEndTagHandler
+  { my( $t, $path, $handler)= @_;
+    $t->{twig_endtag_handlers}||={};
+    return _set_handler( $t->{twig_endtag_handlers}, $path,$handler);
+  }
+
+sub setEndTagHandlers
+  { my( $t, $handlers)= @_;
+    my $previous_handlers= $t->{twig_endtag_handlers} || undef;
+    _reset_handlers( $t->{twig_endtag_handlers});
+    $t->{twig_endtag_handlers}= _set_handlers( $handlers);
+    return $previous_handlers;
+   }
+
+# a little more complex: set the twig_handlers only if a code ref is given
+sub setTwigRoots
+  { my( $t, $handlers)= @_;
+    my $previous_roots= $t->{twig_roots} || undef;
+    _reset_handlers($t->{twig_roots});
+    $t->{twig_roots}= _set_handlers( $handlers);
+    foreach my $path (keys %{$handlers})
+      { $t->{twig_handlers}||= {};
+        _set_handler( $t->{twig_handlers}, $path, $handlers->{$path})
+          if( isa( $handlers->{$path}, 'CODE')); 
+      }
+    return $previous_roots;
+  }
+
+# just store the reference to the expat object in the twig
+sub _twig_init
+  { 
+    my $p= shift;
+    my $t=$p->{twig};
+    $t->{twig_parser}= $p; 
+    weaken( $t->{twig_parser}) if( $weakrefs);
+    $t->{twig_parsing}=1;
+    # in case they had been created by a previous parse
+    delete $t->{twig_dtd};
+    delete $t->{twig_doctype};
+    delete $t->{twig_xmldecl};
+    # if needed set the output filehandle
+    $t->_set_fh_to_twig_output_fh();
+  }
+
+# uses eval to catch the parser's death
+sub safe_parse
+  { my( $t, $str)= @_;
+    eval { $t->parse( $str); } ;
+    return $@ ? 0 : $t;
+  }
+
+sub safe_parsefile
+  { my( $t, $file)= @_;
+    eval { $t->parsefile( $file); } ;
+    return $@ ? 0 : $t;
+  }
+
+
+sub _add_or_discard_stored_spaces
+  { my $t= shift;
+    my %option= @_;
+    
+    if( $t->{twig_stored_spaces} || $option{force} || $t->{twig_preserve_space})
+      { if( $t->{twig_current}->is_pcdata)
+          { $t->{twig_current}->append_pcdata($t->{twig_stored_spaces}); }
+        else
+          { my $current_gi= $t->{twig_current}->gi;
+            unless( defined( $t->{twig_space_policy}->{$current_gi}))
+              { $t->{twig_space_policy}->{$current_gi}= _space_policy( $t, $current_gi); }
+
+            if( $t->{twig_space_policy}->{$current_gi} ||  $option{force} || $t->{twig_preserve_space})
+              { _insert_pcdata( $t, $t->{twig_stored_spaces} ); }
+
+            $t->{twig_stored_spaces}='';
+          }
+      }
+  }
+
+# the default twig handlers, which build the tree
+sub _twig_start
+  { 
+    my ($p, $gi, @att)= @_;
+    my $t=$p->{twig};
+
+    # empty the stored pcdata (space stored in case they are really part of 
+    # a pcdata element) or stored it if the space policy dictades so
+    # create a pcdata element with the spaces if need be
+    _add_or_discard_stored_spaces( $t);
+    my $parent= $t->{twig_current};
+
+    # if we were parsing PCDATA then we exit the pcdata
+    if( $t->{twig_in_pcdata})
+      { $t->{twig_in_pcdata}= 0;
+        delete $parent->{'twig_current'};
+        $parent= $parent->{parent};
+      }
+
+    # if we choose to keep the encoding then we need to parse the tag
+    if( my $func = $t->{parse_start_tag})
+      { ($gi, @att)= &$func($p->original_string); }
+    
+    # filter the input data if need be  
+    if( my $filter= $t->{twig_input_filter})
+      { $gi= $filter->( $gi);
+        @att= map { $filter->($_) } @att; 
+      }
+
+    if( $t->{twig_map_xmlns}) 
+      { 
+        foreach my $new_prefix ( $p->new_ns_prefixes)
+          { my $uri= $t->parser->expand_ns_prefix( $new_prefix);
+            # replace the prefix if it is mapped
+            if( !$t->{twig_keep_original_prefix} && (my $mapped_prefix= $t->{twig_map_xmlns}->{$uri}))
+              { $new_prefix= $mapped_prefix; }
+            # now put the namespace declaration back in the element
+            if( $new_prefix eq '#default')
+              { push @att, "xmlns" =>  $uri; } 
+            else
+              { push @att, "xmlns:$new_prefix" =>  $uri; } 
+          }
+
+        if( $t->{twig_keep_original_prefix})
+          { # things become more complex: we need to find the original prefix
+            # and store both prefixes
+            my $ns_info= $t->_ns_info( $gi);
+            my $map_att;
+            if( $ns_info->{mapped_prefix})
+              { $gi= "$ns_info->{mapped_prefix}:$gi";
+                $map_att->{$ns_info->{mapped_prefix}}= $ns_info->{prefix};
+              }
+            my $att_name=1;
+            foreach( @att) 
+              { if( $att_name) 
+                  { 
+                    my $ns_info= $t->_ns_info( $_);
+                    if( $ns_info->{mapped_prefix})
+                      { $_= "$ns_info->{mapped_prefix}:$_";
+                        $map_att->{$ns_info->{mapped_prefix}}= $ns_info->{prefix};
+                      }
+                    $att_name=0; 
+                  }
+                else           
+                  {  $att_name=1; }
+              }
+            push @att, '#original_gi', $map_att if( $map_att);
+          }
+        else
+          { $gi= $t->_replace_prefix( $gi); 
+            my $att_name=1;
+            foreach( @att) 
+              { if( $att_name) { $_= $t->_replace_prefix( $_); $att_name=0; }
+                else           {  $att_name=1; }
+              }
+          }
+      }
+
+    my $elt= $t->{twig_elt_class}->new( $gi);
+    $elt->set_atts( @att);
+ 
+    delete $parent->{'twig_current'} if( $parent);
+    $t->{twig_current}= $elt;
+    $elt->{'twig_current'}=1;
+
+    if( $parent)
+      { my $prev_sibling= $parent->{last_child};
+        if( $prev_sibling) 
+          { $prev_sibling->{next_sibling}=  $elt; 
+            $elt->set_prev_sibling( $prev_sibling);
+          }
+
+        $elt->set_parent( $parent);
+        $parent->{first_child}=  $elt unless( $parent->{first_child}); 
+        $parent->set_last_child( $elt);
+      }
+    else 
+      { # processing root
+        $t->set_root( $elt);
+        # call dtd handlerif need be
+        $t->{twig_dtd_handler}->($t, $t->{twig_dtd})
+          if( defined $t->{twig_dtd_handler});
+      
+        # set this so we can catch external entities
+        # (the handler was modified during DTD processing)
+        if( $t->{twig_default_print})
+          { $p->setHandlers( Default => \&_twig_print); }
+        elsif( $t->{twig_roots})
+          { $p->setHandlers( Default => sub { return }); }
+        else
+          { $p->setHandlers( Default => \&_twig_default); }
+      }
+   
+    $elt->{empty}=  $p->recognized_string=~ m{/\s*>$}s ? 1 : 0;
+
+    $elt->{extra_data}= $t->{extra_data} if( $t->{extra_data});
+    $t->{extra_data}='';
+
+    # if the element is ID-ed then store that info
+    my $id= $elt->{'att'}->{$ID};
+    if( $id) { $t->{twig_id_list}->{$id}= $elt; 
+               weaken( $t->{twig_id_list}->{$id}) if( $weakrefs);
+             }
+
+
+    # call user handler if need be
+    if( $t->{twig_starttag_handlers})
+      { # call all appropriate handlers
+        my @handlers= _handler( $t, $t->{twig_starttag_handlers}, $gi, $elt);
+    
+        local $_= $elt;
+    
+        foreach my $handler ( @handlers)
+          { $handler->($t, $elt) || last; }
+        # call _all_ handler if needed
+        if( my $all= $t->{twig_starttag_handlers}->{handlers}->{$ALL})
+          { $all->($t, $elt); }
+      }
+
+    # check if the tag is in the list of tags to be ignored
+    if( $t->{twig_ignore_elts_handlers})
+      { my @handlers= _handler( $t, $t->{twig_ignore_elts_handlers}, $gi, $elt);
+        # only the first handler counts, it contains the action (discard/print/string)
+        if( @handlers) { my $action= shift @handlers; $t->ignore( $action); }
+      }
+
+    if( $elt->{'att'}->{'xml:space'} && (  $elt->{'att'}->{'xml:space'} eq 'preserve')) { $t->{twig_preserve_space}++; }
+
+  }
+
+# extract prefix, local_name, uri, mapped_prefix from a name
+# will only work if called from a start or end tag handler
+sub _ns_info
+  { my( $t, $name)= @_;
+    my $ns_info={};
+    my $p= $t->parser;
+    $ns_info->{uri}= $p->namespace( $name); 
+    return $ns_info unless( $ns_info->{uri});
+
+    $ns_info->{prefix}= _a_proper_ns_prefix( $p, $ns_info->{uri});
+    $ns_info->{mapped_prefix}= $t->{twig_map_xmlns}->{$ns_info->{uri}} || $ns_info->{prefix};
+
+    return $ns_info;
+  }
+    
+sub _a_proper_ns_prefix
+  { my( $p, $uri)= @_;
+    foreach my $prefix ($p->current_ns_prefixes)
+      { if( $p->expand_ns_prefix( $prefix) eq $uri)
+          { return $prefix; }
+      }
+  }
+
+
+# the default function to parse a start tag (in keep_encoding mode)
+# can be overridden with the parse_start_tag method
+# only works for 1-byte character sets
+sub _parse_start_tag
+  { my $string= shift;
+    my( $gi, @atts);
+
+    # get the gi (between < and the first space, / or > character)
+    #if( $string=~ s{^<\s*([^\s>/]*)[\s>/]*}{}s)
+    if( $string=~ s{^<\s*($REG_NAME)[\s>/]}{}s)
+      { $gi= $1; }
+    else
+      { croak "error parsing tag '$string'"; }
+    while( $string=~ s{^([^\s=]*)\s*=\s*(["'])(.*?)\2\s*}{}s)
+      { push @atts, $1, $3; }
+    return $gi, @atts;
+  }
+
+sub set_root
+  { my( $t, $elt)= @_;
+    $t->{twig_root}= $elt;
+    $elt->{twig}= $t;
+    weaken(  $elt->{twig}) if( $weakrefs);
+  }
+
+sub _twig_end($$;@)
+  { 
+    my ($p, $gi)  = @_;
+    my $t=$p->{twig};
+
+    if( $t->{twig_map_xmlns}) { $gi= $t->_replace_prefix( $gi); }
+  
+    _add_or_discard_stored_spaces( $t);
+ 
+    # the new twig_current is the parent
+    my $elt= $t->{twig_current};
+    delete $elt->{'twig_current'};
+
+    # if we were parsing PCDATA then we exit the pcdata too
+    if( $t->{twig_in_pcdata})
+      { $t->{twig_in_pcdata}= 0;
+        $elt= $elt->{parent} if($elt->{parent});
+        delete $elt->{'twig_current'};
+      }
+
+    # parent is the new current element
+    my $parent= $elt->{parent};
+    $parent->{'twig_current'}=1 if( $parent);
+    $t->{twig_current}= $parent;
+
+    $elt->{extra_data_before_end_tag}= $t->{extra_data} if( $t->{extra_data}); 
+    $t->{extra_data}='';
+
+    if( $t->{twig_handlers})
+      { # look for handlers
+        my @handlers= _handler( $t, $t->{twig_handlers}, $gi, $elt);
+
+        local $_= $elt; # so we can use $_ in the handlers
+    
+        foreach my $handler ( @handlers)
+          { $handler->($t, $elt) || last; }
+        # call _all_ handler if needed
+        if( my $all= $t->{twig_handlers}->{handlers}->{$ALL})
+          { $all->($t, $elt); }
+      }
+
+    # if twig_roots is set for the element then set appropriate handler
+    if(  $t->{twig_root_depth} and ($p->depth == $t->{twig_root_depth}) )
+      { if( $t->{twig_default_print})
+          { # select the proper fh (and store the currently selected one)
+            $t->_set_fh_to_twig_output_fh(); 
+            if( $t->{twig_keep_encoding})
+              { $p->setHandlers( %twig_handlers_roots_print_original); }
+            else
+              { $p->setHandlers( %twig_handlers_roots_print); }
+          }
+        else
+          { $p->setHandlers( %twig_handlers_roots); }
+      }
+
+    if( $elt->{'att'}->{'xml:space'} && (  $elt->{'att'}->{'xml:space'} eq 'preserve')) { $t->{twig_preserve_space}--; }
+  }
+
+# return the list of handler that can be activated for an element 
+# (either of CODE ref's or 1's for twig_roots)
+
+sub _handler
+  { my( $t, $handlers, $gi, $elt)= @_;
+
+    my @found_handlers=();
+    my $found_handler;
+
+    # warning: $elt can be either 
+    # - a regular element
+    # - a ref to the attribute hash (when called for an element 
+    #   for which the XML::Twig::Elt has not been built, outside 
+    #   of the twig_roots)
+    # - a string (case of an entity in keep_encoding mode)
+
+    # check for an attribute expression with no gi
+    if( $handlers->{att_handlers})
+      { my %att_handlers= %{$handlers->{att_handlers_exp}};
+        foreach my $att ( keys %att_handlers)
+          { my $att_val;
+            # get the attribute value
+            if( ref $elt eq 'HASH')
+              { $att_val= $elt->{$att}; }     # $elt is the atts hash
+            elsif( isa( $elt,'XML::Twig::Elt'))
+              { $att_val= $elt->{'att'}->{$att}; }  # $elt is an element
+                if( defined $att_val)
+                  { my @cond= @{$handlers->{att_handlers_exp}->{$att}};
+                    foreach my $cond (@cond)
+                      {  # 2 cases: either there is a val and the att value should be equal to it
+                         #          or there is no val (condition was gi[@att]), just for the att to be defined 
+                    if( !defined $cond->{val} || ($att_val eq $cond->{val}) )  
+                      { push @found_handlers, $cond->{handler};}
+                  }
+              }
+          }
+      }
+
+    # check for an attribute regexp expression with no gi
+    if( $handlers->{att_regexp_handlers})
+      { my %att_handlers= %{$handlers->{att_regexp_handlers_exp}};
+        foreach my $att ( keys %att_handlers)
+          { my $att_val;
+            # get the attribute value
+            if( ref $elt eq 'HASH')
+              { $att_val= $elt->{$att}; }     # $elt is the atts hash
+            elsif( isa( $elt,'XML::Twig::Elt'))
+              { $att_val= $elt->{'att'}->{$att}; }  # $elt is an element
+
+            if( defined $att_val)
+              { my @cond= @{$handlers->{att_regexp_handlers_exp}->{$att}};
+                foreach my $cond (@cond)
+                  { if( $att_val=~ $cond->{regexp})  
+                      { push @found_handlers, $cond->{handler};}
+                  }
+              }
+          }
+      }
+
+    # check for a text expression
+    if( $handlers->{text_handlers}->{$gi})
+      { my @text_handlers= @{$handlers->{text_handlers_exp}->{$gi}};
+        foreach my $exp ( @text_handlers)
+          { if (!$exp->{sub_elt})
+              { push @found_handlers, $exp->{handler}
+                  if $elt->text eq $exp->{text};
+              }
+            else
+              { foreach my $child ($elt->children($exp->{sub_elt}))
+                  { if( $child->text eq $exp->{text})
+                      { push @found_handlers, $exp->{handler};
+                        last;
+                      }
+                  }
+              }
+          }
+      }
+
+    # check for a text regexp expression
+    if( $handlers->{regexp_handlers}->{$gi})
+      { my @regexp_handlers= @{$handlers->{regexp_handlers_exp}->{$gi}};
+        foreach my $exp ( @regexp_handlers)
+          { if( !$exp->{sub_elt})
+              { push @found_handlers, $exp->{handler}
+                  if $elt->text =~ $exp->{regexp};
+              }
+            else
+              { foreach my $child ($elt->children($exp->{sub_elt}))
+                  { if( $child->text =~ $exp->{regexp})
+                      { push @found_handlers, $exp->{handler};
+                        last;
+                      }
+                  }
+              }
+          }
+      }
+
+    # check for an attribute expression
+    if( $handlers->{attcond_handlers}->{$gi})
+      { my @attcond_handlers= @{$handlers->{attcond_handlers_exp}->{$gi}};
+        foreach my $exp ( @attcond_handlers)
+          { my $att_val;
+        # get the attribute value
+        if( ref $elt eq 'HASH')
+          { $att_val= $elt->{$exp->{att}}; }    # $elt is the atts hash
+        else
+          { $att_val= $elt->{'att'}->{$exp->{att}}; }# $elt is an element
+
+        # 2 cases: either there is a val and the att value should be equal to it
+        #          or there is no val (condition was gi[@att]), just for the att to be defined 
+        if( defined $att_val && ( !defined $exp->{val} || ($att_val eq $exp->{val}) ) ) 
+              { push @found_handlers, $exp->{handler}; }
+          }
+      }
+
+    # check for an attribute regexp
+    if( $handlers->{attregexp_handlers}->{$gi})
+      { my @attregexp_handlers= @{$handlers->{attregexp_handlers_exp}->{$gi}};
+        foreach my $exp ( @attregexp_handlers)
+          { my $att_val;
+        # get the attribute value
+        if( ref $elt eq 'HASH')
+          { $att_val= $elt->{$exp->{att}}; }    # $elt is the atts hash
+        else
+          { $att_val= $elt->{'att'}->{$exp->{att}}; }# $elt is an element
+
+        if( defined $att_val && ( ($att_val=~  $exp->{regexp}) ) ) 
+              { push @found_handlers, $exp->{handler}; }
+          }
+      }
+
+    # check for a full path
+    if( defined $handlers->{path_handlers}->{gi}->{$gi})
+      { my $path= $t->path( $gi); 
+        if( defined( $found_handler= $handlers->{path_handlers}->{path}->{$path}) )
+          { push @found_handlers, $found_handler; }
+      }
+
+    # check for a partial path
+    if( $handlers->{subpath_handlers}->{gi}->{$gi})
+      { my $path= $t->path( $gi);
+        while( $path)
+          { # test each sub path
+            if( defined( $found_handler= $handlers->{subpath_handlers}->{path}->{$path}) )
+              { push @found_handlers, $found_handler; }
+             $path=~ s{^[^/]*/?}{}; # remove initial gi and /
+          }
+      }
+
+    # check for a gi (simple gi's are stored directly in the handlers field)
+    if( defined $handlers->{handlers}->{gi}->{$gi})
+      { push @found_handlers, $handlers->{handlers}->{gi}->{$gi}; }
+
+    # if no handler found call default handler if defined
+    if( !@found_handlers && defined $handlers->{handlers}->{$DEFAULT})
+      { push @found_handlers, $handlers->{handlers}->{$DEFAULT}; }
+
+    if( @found_handlers and $t->{twig_do_not_chain_handlers}) 
+      { @found_handlers= ($found_handlers[0]); }
+
+    return @found_handlers; # empty if no handler found
+
+  }
+
+
+sub _replace_prefix
+  { my( $t, $name)= @_;
+    my $p= $t->parser;
+    my $uri= $p->namespace( $name);
+    # try to get the namespace rom default if none is found (for attributes)
+    # this should probably be an option
+    if( !$uri and( $name!~/^xml/)) { $uri= $p->expand_ns_prefix( '#default'); }
+    if( $uri)
+      { if (my $mapped_prefix= $t->{twig_map_xmlns}->{$uri})
+          { return "$mapped_prefix:$name"; }
+        else
+          { my $prefix= _a_proper_ns_prefix( $p, $uri);
+            return $prefix ? "$prefix:$name" : $name; 
+          }
+      }
+    else
+      { return $name; }
+  }
+
+sub _twig_char
+  { 
+    my ($p, $string)= @_;
+    my $t=$p->{twig}; 
+
+    # if keep_encoding was set then use the original string instead of
+    # the parsed (UTF-8 converted) one
+    if( $t->{twig_keep_encoding})
+      { $string= $p->original_string(); }
+
+    if( $t->{twig_input_filter})
+      { $string= $t->{twig_input_filter}->( $string); }
+
+    if( $t->{twig_char_handler})
+      { $string= $t->{twig_char_handler}->( $string); }
+
+    my $elt= $t->{twig_current};
+
+    if(    $t->{twig_in_cdata})
+      { # text is the continuation of a previously created pcdata
+        $elt->{cdata}.=  $t->{twig_stored_spaces}.$string; } 
+    elsif( $t->{twig_in_pcdata})
+      { # text is the continuation of a previously created cdata
+        $elt->{pcdata}.=  $string; 
+      } 
+    else
+      { # text is just space, which might be discarded later
+        if( $string=~/\A\s*\Z/s)
+          { if( $t->{extra_data})
+              { # we got extra data (comment, pi), lets add the spaces to it
+                $t->{extra_data} .= $string; 
+              }
+            else
+              { # no extra data, just store the spaces
+                $t->{twig_stored_spaces}.= $string;
+              }
+          } 
+        else
+          { my $new_elt= _insert_pcdata( $t, $t->{twig_stored_spaces}.$string);
+            delete $elt->{'twig_current'};
+            $new_elt->{'twig_current'}=1;
+            $t->{twig_current}= $new_elt;
+            $t->{twig_in_pcdata}=1;
+            $new_elt->{extra_data}= $t->{extra_data} if( $t->{extra_data});
+            $t->{extra_data}='';
+          }
+      }
+  }
+
+sub _twig_cdatastart
+  { 
+    my $p= shift;
+    my $t=$p->{twig};
+
+    $t->{twig_in_cdata}=1;
+    my $cdata=  $t->{twig_elt_class}->new( '#CDATA');
+    my $twig_current= $t->{twig_current};
+
+    if( $t->{twig_in_pcdata})
+      { # create the node as a sibling of the #PCDATA
+        $cdata->set_prev_sibling( $twig_current);
+        $twig_current->{next_sibling}=  $cdata;
+        my $parent= $twig_current->{parent};
+        $cdata->set_parent( $parent);
+        $parent->set_last_child( $cdata);
+        $t->{twig_in_pcdata}=0;
+      }
+    else
+      { # we have to create a PCDATA element if we need to store spaces
+        if( $t->_space_policy($XML::Twig::index2gi[$twig_current->{'gi'}]) && $t->{twig_stored_spaces})
+          { _insert_pcdata( $t, $t->{twig_stored_spaces}); }
+        $t->{twig_stored_spaces}='';
+      
+        # create the node as a child of the current element      
+        $cdata->set_parent( $twig_current);
+        if( my $prev_sibling= $twig_current->{last_child})
+          { $cdata->set_prev_sibling( $prev_sibling);
+            $prev_sibling->{next_sibling}=  $cdata;
+          }
+        else
+          { $twig_current->{first_child}=  $cdata; }
+        $twig_current->set_last_child( $cdata);
+      
+      }
+
+    delete $twig_current->{'twig_current'};
+    $t->{twig_current}= $cdata;
+    $cdata->{'twig_current'}=1;
+  }
+
+sub _twig_cdataend
+  { 
+    my $p= shift;
+    my $t=$p->{twig};
+
+    $t->{twig_in_cdata}=0;
+
+    my $elt= $t->{twig_current};
+    delete $elt->{'twig_current'};
+    my $cdata= $elt->{cdata};
+    $elt->{cdata}=  $cdata;
+
+    if( $t->{twig_handlers})
+      { # look for handlers
+        my @handlers= _handler( $t, $t->{twig_handlers}, CDATA, $elt);
+        local $_= $elt; # so we can use $_ in the handlers
+        foreach my $handler ( @handlers) { $handler->($t, $elt) || last; }
+      }
+
+    $elt= $elt->{parent};
+    $t->{twig_current}= $elt;
+    $elt->{'twig_current'}=1;
+  }
+
+sub _twig_pi
+  { 
+    my( $p, $target, $data)= @_;
+    my $t=$p->{twig};
+    return unless( $t->{twig_process_pi} || $t->{twig_keep_pi});
+
+    if( $t->{twig_input_filter})
+      { $target = $t->{twig_input_filter}->( $target) ;
+        $data   = $t->{twig_input_filter}->( $data)   ;
+      }
+
+    my $twig_current= $t->{twig_current};    # always defined
+
+    # if pi's are to be kept then we piggiback them to the current element
+    if( $t->{twig_keep_pi})
+      {  
+        if( my $handler= $t->{twig_handlers}->{pi_handlers}->{$target})
+          { $t->{extra_data}.= $handler->( $t, $target, $data); }
+        elsif( $handler= $t->{twig_handlers}->{pi_handlers}->{''})
+          { $t->{extra_data}.= $handler->( $t, $target, $data); }
+        else
+          { if( $t->{twig_stored_spaces})
+              { $t->{extra_data}.= $t->{twig_stored_spaces};
+                $t->{twig_stored_spaces}= '';
+              }
+            # then the recognized/original string (input filtered if needed)
+            my $extra_data= $XML::Twig::Elt::keep_encoding ?  $p->recognized_string() : $p->original_string();
+            $extra_data= $t->{twig_input_filter}->( $extra_data) if( $t->{twig_input_filter});
+            $t->{extra_data}.= $extra_data;
+            return;
+          }
+
+      }
+    else
+      { # pi's are processed
+        my $pi=  $t->{twig_elt_class}->new( PI);
+        $pi->set_pi( $target, $data);
+
+        unless( $t->root)
+          {  _pi_handlers( $t, $pi, $target);
+            _add_prolog_data( $t, $pi); 
+            return;
+          }
+
+        if( $t->{twig_in_pcdata})
+          { # create the node as a sibling of the #PCDATA
+            $pi->paste_after( $twig_current);
+            $t->{twig_in_pcdata}=0;
+          }
+        else
+          { # we have to create a PCDATA element if we need to store spaces
+            if( $t->_space_policy($XML::Twig::index2gi[$twig_current->{'gi'}]) && $t->{twig_stored_spaces})
+              { _insert_pcdata( $t, $t->{twig_stored_spaces}); }
+            $t->{twig_stored_spaces}='';
+            # create the node as a child of the current element
+            $pi->paste_last_child( $twig_current);
+          }
+    
+        delete $twig_current->{'twig_current'};
+        my $parent= $pi->{parent}; 
+        $t->{twig_current}= $parent;
+        $parent->{'twig_current'}=1;
+
+        _pi_handlers( $t, $pi, $target);
+      }
+
+  }
+
+sub _pi_handlers
+  { my( $t, $pi, $target)= @_;
+    if( my $handler= $t->{twig_handlers}->{pi_handlers}->{$target})
+      { local $_= $pi; $handler->( $t, $pi); }
+    elsif( $handler= $t->{twig_handlers}->{pi_handlers}->{''})
+      { local $_= $pi; $handler->( $t, $pi); }
+  }
+
+
+sub _twig_comment
+  { 
+    my( $p, $comment_text)= @_;
+    my $t=$p->{twig};
+    return unless( $t->{twig_process_comments} || $t->{twig_keep_comments});
+
+    my $twig_current= $t->{twig_current};    # always defined
+
+    # if comments are to be kept then we piggiback them to the current element
+    if( $t->{twig_keep_comments})
+      { # first add spaces
+        if( $t->{twig_stored_spaces})
+              { $t->{extra_data}.= $t->{twig_stored_spaces};
+                $t->{twig_stored_spaces}= '';
+              }
+        # then the recognized/original string (input filtered if needed)
+        my $extra_data= $XML::Twig::Elt::keep_encoding ?  $p->recognized_string() : $p->original_string();
+        $extra_data= $t->{twig_input_filter}->( $extra_data) if( $t->{twig_input_filter});
+        $t->{extra_data}.= $extra_data;
+        return;
+      }
+
+    $comment_text= $t->{twig_input_filter}->( $comment_text) if( $t->{twig_input_filter});
+
+    my $comment=  $t->{twig_elt_class}->new( COMMENT);
+    $comment->{comment}=  $comment_text;
+
+    unless( $t->root) 
+      { _add_prolog_data( $t, $comment);
+        _comment_handler( $t, $comment);
+        return;
+      }
+
+    if( $t->{twig_in_pcdata})
+      { # create the node as a sibling of the #PCDATA
+        $comment->paste_after( $twig_current);
+        $t->{twig_in_pcdata}=0;
+      }
+    else
+      { # we have to create a PCDATA element if we need to store spaces
+        if( $t->_space_policy($XML::Twig::index2gi[$twig_current->{'gi'}]) && $t->{twig_stored_spaces})
+          { _insert_pcdata( $t, $t->{twig_stored_spaces}); }
+        $t->{twig_stored_spaces}='';
+        # create the node as a child of the current element
+        $comment->paste_last_child( $twig_current);
+
+      }
+    _comment_handler( $t, $comment);
+
+    delete $twig_current->{'twig_current'};
+
+    my $parent= $comment->{parent};
+    $t->{twig_current}= $parent;
+    $parent->{'twig_current'}=1;
+
+  }
+
+sub _comment_handler
+  { my( $t, $comment)= @_;
+    if( $t->{twig_handlers}->{handlers}->{gi}->{'#COMMENT'})
+      { # look for handlers
+        local $_= $comment;
+        my @handlers= _handler( $t, $t->{twig_handlers}, '#COMMENT', $comment);
+        foreach my $handler ( @handlers)
+          { $handler->($t, $comment) || last; }
+      }
+  }
+
+
+sub _add_prolog_data
+  { my($t, $prolog_data)= @_;
+    # comment before the first element
+    $t->{prolog_data} ||= $t->{twig_elt_class}->new( '#PROLOG_DATA');
+    # create the node as a child of the current element
+    $prolog_data->paste_last_child( $t->{prolog_data});
+  }
+  
+sub _twig_final
+  { 
+    my $p= shift;
+    my $t=$p->{twig};
+
+    # restore the selected filehandle if needed
+    $t->_set_fh_to_selected_fh();
+
+    select $t->{twig_original_selected_fh} if($t->{twig_original_selected_fh}); # probably dodgy
+
+    # tries to clean-up (probably not very well at the moment)
+    undef $p->{twig};
+    undef $t->{twig_parser};
+
+    undef $t->{twig_parsing};
+
+    return $t;
+  }
+
+sub _insert_pcdata
+  { my( $t, $string)= @_;
+    # create a new #PCDATA element
+    my $parent= $t->{twig_current};    # always defined
+    my $elt=  $t->{twig_elt_class}->new( PCDATA);
+    $elt->{pcdata}=  $string;
+    my $prev_sibling= $parent->{last_child};
+    if( $prev_sibling) 
+      { $prev_sibling->{next_sibling}=  $elt; 
+        $elt->set_prev_sibling( $prev_sibling);
+      }
+    else
+      { $parent->{first_child}=  $elt; }
+
+    $elt->set_parent( $parent);
+    $parent->set_last_child( $elt);
+    $t->{twig_stored_spaces}='';
+    return $elt;
+  }
+
+sub _space_policy
+  { my( $t, $gi)= @_;
+    my $policy;
+    $policy=0 if( $t->{twig_discard_spaces});
+    $policy=1 if( $t->{twig_keep_spaces});
+    $policy=1 if( $t->{twig_keep_spaces_in}
+               && $t->{twig_keep_spaces_in}->{$gi});
+    $policy=0 if( $t->{twig_discard_spaces_in} 
+               && $t->{twig_discard_spaces_in}->{$gi});
+    return $policy;
+  }
+
+
+sub _twig_entity($$$$$$)
+  { 
+    my( $p, $name, $val, $sysid, $pubid, $ndata)= @_;
+    my $t=$p->{twig};
+    my $ent=XML::Twig::Entity->new( $name, $val, $sysid, $pubid, $ndata);
+    $t->{twig_entity_list}->add( $ent);
+    if( $parser_version > 2.27) 
+      { # this is really ugly, but with some versions of XML::Parser the value 
+        # of the entity is not properly returned by the default handler
+        my $ent_decl= $ent->text;
+        if( $t->{twig_keep_encoding})
+          { if( defined $ent->{val} && ($ent_decl !~ /["']/))
+              { my $val=  $ent->{val};
+                $ent_decl .= $val =~ /"/ ? qq{'$val' } : qq{"$val" }; 
+              }
+				    # for my solaris box (perl 5.6.1, XML::Parser 2.31, expat?)
+            $t->{twig_doctype}->{internal}=~ s{<!ENTITY\s+$name\s+$}{substr( $ent_decl, 0, -1)}e;
+          }
+        $t->{twig_doctype}->{internal} .= $ent_decl 
+          unless( $t->{twig_doctype}->{internal}=~ m{<!ENTITY\s+$name\s+});
+      }
+  }
+
+sub _twig_xmldecl
+  { 
+    my $p= shift;
+    my $t=$p->{twig};
+    $t->{twig_xmldecl}||={};                 # could have been set by set_output_encoding
+    $t->{twig_xmldecl}->{version}= shift;
+    $t->{twig_xmldecl}->{encoding}= shift; 
+    $t->{twig_xmldecl}->{standalone}= shift;
+  }
+
+sub _twig_doctype
+  { 
+    my( $p, $name, $sysid, $pub, $internal)= @_;
+    my $t=$p->{twig};
+    $t->{twig_doctype}||= {};                   # create 
+    $t->{twig_doctype}->{name}= $name;          # always there
+    $t->{twig_doctype}->{sysid}= $sysid;        #  
+    $t->{twig_doctype}->{pub}= $pub;            #  
+
+    # now let's try to cope with XML::Parser 2.28 and above
+    if( $parser_version > 2.27)
+      { @saved_default_handler= $p->setHandlers( Default     => \&_twig_store_internal_dtd,
+                                                 Entity      => \&_twig_entity,
+                                               );
+      $p->setHandlers( DoctypeFin  => \&_twig_stop_storing_internal_dtd);
+      $t->{twig_doctype}->{internal}='';
+      }
+    else            
+      # for XML::Parser before 2.28
+      { $t->{twig_doctype}->{internal}=$internal; }
+
+    # now check if we want to get the DTD info
+    if( $t->{twig_read_external_dtd} && $sysid)
+      { # let's build a fake document with an internal DTD
+        # is this portable?
+        my $tmpfile= "twig_tmp$$";
+        open( TMP, ">$tmpfile") 
+          or croak "cannot create temp file $tmpfile: $!";
+        print TMP "<!DOCTYPE $name [\n";   # print the doctype
+        # slurp the DTD
+          { open( DTD, "<$sysid") 
+              or croak "cannot open dtd file $sysid: $!";
+            local $/= undef;
+            my $dtd= <DTD>;
+            close DTD;
+            print TMP $dtd;                 # add the dtd
+          }
+        print TMP "]>";                     # close the dtd
+        print TMP "<dummy></dummy>\n";      # XML::Parser needs an element
+
+        close TMP;
+       
+        $t->save_global_state();            # save the globals (they will be reset by the following new)  
+        my $t_dtd= XML::Twig->new;          # create a temp twig
+        $t->restore_global_state();
+        $t_dtd->parsefile( $tmpfile);       # parse it
+        $t->{twig_dtd}= $t_dtd->{twig_dtd}; # grab the dtd info
+        #$t->{twig_dtd_is_external}=1;
+        $t->{twig_entity_list}= $t_dtd->{twig_entity_list}; # grab the entity info
+
+        unlink $tmpfile;
+      }
+
+  }
+
+sub _twig_element
+  { 
+    my( $p, $name, $model)= @_;
+    my $t=$p->{twig};
+    $t->{twig_dtd}||= {};                      # may create the dtd 
+    $t->{twig_dtd}->{model}||= {};             # may create the model hash 
+    $t->{twig_dtd}->{elt_list}||= [];          # ordered list of elements 
+    push @{$t->{twig_dtd}->{elt_list}}, $name; # store the elt
+    $t->{twig_dtd}->{model}->{$name}= $model;  # store the model
+    if( ($parser_version > 2.27) && ($t->{twig_doctype}->{internal}=~ m{(^|>)\s*$}) ) 
+      { my $text= $XML::Twig::Elt::keep_encoding ? $p->original_string : $p->recognized_string; 
+        unless( $text)
+          { # this version of XML::Parser does not return the text in the *_string method
+            # we need to rebuild it
+            $text= "<!ELEMENT $name $model>";
+          }
+        $t->{twig_doctype}->{internal} .= $text;
+      }
+  }
+
+sub _twig_attlist
+  { 
+    my( $p, $el, $att, $type, $default, $fixed)= @_;
+    my $t=$p->{twig};
+    $t->{twig_dtd}||= {};                      # create dtd if need be 
+    $t->{twig_dtd}->{$el}||= {};               # create elt if need be 
+    #$t->{twig_dtd}->{$el}->{att}||= {};        # create att if need be 
+    if( ($parser_version > 2.27) && ($t->{twig_doctype}->{internal}=~ m{(^|>)\s*$}) ) 
+      { my $text= $XML::Twig::Elt::keep_encoding ? $p->original_string : $p->recognized_string; 
+        unless( $text)
+          { # this version of XML::Parser does not return the text in the *_string method
+            # we need to rebuild it
+            my $att_decl="$att $type";
+            $att_decl .= " #FIXED"   if( $fixed);
+            $att_decl .= " $default" if( $default);
+            # 2 cases: there is already an attlist on that element or not
+            if( $t->{twig_dtd}->{att}->{$el})
+              { # there is already an attlist, add to it
+                $t->{twig_doctype}->{internal}=~ s{(<!ATTLIST\s*$el )(.*?)\n?>}
+                                                  { "$1$2\n" . ' ' x length( $1) . "$att_decl\n>"}es;
+              }
+            else
+              { # create the attlist
+                 $t->{twig_doctype}->{internal}.= "<!ATTLIST $el $att_decl>"
+              }
+          }
+      }
+    $t->{twig_dtd}->{att}->{$el}->{$att}= {} ;
+    $t->{twig_dtd}->{att}->{$el}->{$att}->{type}= $type; 
+    $t->{twig_dtd}->{att}->{$el}->{$att}->{default}= $default; 
+    $t->{twig_dtd}->{att}->{$el}->{$att}->{fixed}= $fixed; 
+  }
+
+sub _twig_default
+  { 
+    my( $p, $string)= @_;
+    
+    my $t= $p->{twig};
+    
+    # process only if we have an entity
+    return unless( $string=~ m{^&([^;]*);$});
+    # the entity has to be pure pcdata, or we have a problem
+    my $ent;
+    if( $t->{twig_keep_encoding}) 
+      { _twig_char( $p, $string); 
+        $ent= substr( $string, 1, -1);
+      }
+    else
+      { $ent= _twig_insert_ent( $t, $string); 
+      }
+  }
+    
+sub _twig_insert_ent
+  { 
+    my( $t, $string)=@_;
+
+    my $twig_current= $t->{twig_current};
+
+    my $ent=  $t->{twig_elt_class}->new( '#ENT');
+    $ent->{ent}=  $string;
+
+    _add_or_discard_stored_spaces( $t, force => 0);
+    
+    if( $t->{twig_in_pcdata})
+      { # create the node as a sibling of the #PCDATA
+
+        $ent->set_prev_sibling( $twig_current);
+        $twig_current->{next_sibling}=  $ent;
+        my $parent= $twig_current->{parent};
+        $ent->set_parent( $parent);
+        $parent->set_last_child( $ent);
+        # the twig_current is now the parent
+        delete $twig_current->{'twig_current'};
+        $t->{twig_current}= $parent;
+        # we left pcdata
+        $t->{twig_in_pcdata}=0;
+      }
+    else
+      { # create the node as a child of the current element
+        $ent->set_parent( $twig_current);
+        if( my $prev_sibling= $twig_current->{last_child})
+          { $ent->set_prev_sibling( $prev_sibling);
+            $prev_sibling->{next_sibling}=  $ent;
+          }
+        else
+          { $twig_current->{first_child}=  $ent; }
+        $twig_current->set_last_child( $ent);
+      }
+
+    return $ent;
+  }
+
+sub parser
+  { return $_[0]->{twig_parser}; }
+
+# returns the declaration text (or a default one)
+sub xmldecl
+  { my $t= shift;
+    return '' unless( $t->{twig_xmldecl} || $t->{output_encoding});
+    my $decl_string;
+    my $decl= $t->{twig_xmldecl};
+    if( $decl)
+      { my $version= $decl->{version};
+        $decl_string= q{<?xml};
+        $decl_string .= qq{ version="$version"};
+
+        # encoding can either have been set (in $decl->{output_encoding})
+        # or come from the document (in $decl->{encoding})
+        if( $t->{output_encoding})
+          { my $encoding= $t->{output_encoding};
+            $decl_string .= qq{ encoding="$encoding"};
+          }
+        elsif( $decl->{encoding})
+          { my $encoding= $decl->{encoding};
+            $decl_string .= qq{ encoding="$encoding"};
+          }
+    
+        if( defined( $decl->{standalone}))
+          { $decl_string .= q{ standalone="};  
+            $decl_string .= $decl->{standalone} ? "yes" : "no";  
+            $decl_string .= q{"}; 
+          }
+      
+        $decl_string .= "?>\n";
+      }
+    else
+      { my $encoding= $t->{output_encoding};
+        $decl_string= qq{<?xml version="1.0" encoding="$encoding"?>};
+      }
+      
+    my $output_filter= XML::Twig::Elt::output_filter();
+    return $output_filter ? $output_filter->( $decl_string) : $decl_string;
+  }
+
+# returns the doctype text (or none)
+# that's the doctype just has it was in the original document
+sub doctype
+  { my $t= shift;
+    my $doctype= $t->{twig_doctype} or return '';
+    my $string= "<!DOCTYPE " . $doctype->{name};
+    $string  .= qq{ SYSTEM "$doctype->{sysid}"} if( $doctype->{sysid});
+    $string  .= qq{ PUBLIC  "$doctype->{pub}" } if( $doctype->{pub});
+    if( $doctype->{internal})
+      { # !@#$%^&* code to deal with various expat/XML::Parser versions
+        $string.= " [" unless( $doctype->{internal}=~ m{^\s*\[});
+        $string.= " " if( $doctype->{internal}=~ m{^\[});
+        $string  .= $doctype->{internal};
+        $string=~ s{\n?]?>?$}{\n]>};
+      }
+    return $string;
+  }
+
+sub set_doctype
+  { my( $t, $name, $system, $public, $internal)= @_;
+    $t->{twig_doctype}= {};
+    my $doctype= $t->{twig_doctype};
+    $doctype->{name}     = $name     if( defined $name);
+    $doctype->{sysid}    = $system   if( defined $system);
+    $doctype->{pub}      = $public   if( defined $public);
+    $doctype->{internal} = $internal if( defined $internal);
+  }
+
+# return the dtd object
+sub dtd
+  { my $t= shift;
+    return $t->{twig_dtd};
+  }
+
+# return an element model, or the list of element models
+sub model
+  { my $t= shift;
+    my $elt= shift;
+    return $t->dtd->{model}->{$elt} if( $elt);
+    return sort keys %{$t->dtd->{model}};
+  }
+
+        
+# return the entity_list object 
+sub entity_list($)
+  { my $t= shift;
+    return $t->{twig_entity_list};
+  }
+
+# return the list of entity names 
+sub entity_names($)
+  { my $t= shift;
+    return sort keys %{$t->{twig_entity_list}} ;
+  }
+
+# return the entity object 
+sub entity($$)
+  { my $t= shift;
+    my $entity_name= shift;
+    return $t->{twig_entity_list}->{$entity_name};
+  }
+
+
+sub print_prolog
+  { my $t= shift;
+    my $fh=  _is_fh($_[0])  ? shift : undef;
+    if( $fh) { print $fh $t->prolog( @_); }
+    else     { print $t->prolog( @_);     }
+  }
+
+sub prolog
+  { my $t= shift;
+    my %args= _normalize_args( @_);
+    my $prolog='';
+
+    return $prolog if( $t->{no_prolog});
+
+    my $update_dtd = $args{UpdateDTD} || '';
+
+    $prolog .= $t->xmldecl;
+    return $prolog if( defined( $t->{no_dtd_output}) or !(defined $t->{twig_doctype}));
+    my $doctype= $t->{twig_doctype};
+    if( $update_dtd)
+      { 
+        if( defined $doctype->{sysid}  )  
+          { $prolog .= "<!DOCTYPE ".$doctype->{name};
+            $prolog .= " PUBLIC  \"$doctype->{pub}\""  if( $doctype->{pub});
+            $prolog .= " SYSTEM \"$doctype->{sysid}\"" if( $doctype->{sysid} && !$doctype->{pub});
+            $prolog .= "[\n";
+            $prolog .= $t->{twig_entity_list}->text;
+            $prolog .= "]>\n";
+          }
+        else
+          { my $dtd= $t->{'twig_dtd'};
+            $prolog .= $t->dtd_text;
+          }            
+      }
+    else
+      { 
+        $prolog .= "<!DOCTYPE ". $doctype->{name}  if( $doctype->{name});
+        $prolog .= " PUBLIC \"$doctype->{pub}\""  if( $doctype->{pub});
+        $prolog .= " SYSTEM" if( $doctype->{sysid} && !$doctype->{pub});
+        $prolog .= ' "' . $doctype->{sysid} . '"'  if( $doctype->{sysid}); 
+        if( my $internal= $doctype->{internal}) 
+          { # add opening and closing brackets if not already there
+            # plus some spaces and newlines for a nice formating
+            # I test it here because I can't remember which version of
+            # XML::Parser need it or not, so this is actually quite safe
+            $internal=~ s{^\s*(\[\s*)?}{ [\n};
+            $internal=~ s{\s*(\]\s*(>\s*)?)?\s*$}{\n]>\n};
+            $prolog .=  $internal; 
+          }
+      }
+
+    # terrible hack, as I can't figure out in which case the darn prolog
+    # should get an extra >
+    $prolog=~ s/(>\s*)*$/>\n/;
+
+    my $output_filter= XML::Twig::Elt::output_filter();
+    return $output_filter ? $output_filter->( $prolog) : $prolog;
+  }
+
+sub _print_prolog_data
+  { my $t= shift;
+    my $fh=  _is_fh($_[0])  ? shift : undef;
+    if( $fh) { print $fh $t->_prolog_data( @_); }
+    else     { print $t->_prolog_data( @_);     }
+  }
+
+sub _prolog_data
+  { my $t= shift;
+    return''  unless( $t->{prolog_data});
+    my $prolog_data_text='';
+    foreach ( $t->{prolog_data}->children)
+      { $prolog_data_text .= $_->sprint . "\n"; }
+    return$ prolog_data_text;
+  }
+
+sub print
+  { my $t= shift;
+    my $fh=  _is_fh( $_[0])  ? shift : undef;
+    my %args= _normalize_args( @_);
+
+    my $old_pretty;
+    if( defined $args{PrettyPrint})
+      { $old_pretty= $t->set_pretty_print( $args{PrettyPrint}); 
+        delete $args{PrettyPrint};
+      }
+
+     my $old_empty_tag_style;
+     if( defined $args{EmptyTags})
+      { $old_empty_tag_style= $t->set_empty_tag_style( $args{EmptyTags}); 
+        delete $args{EmptyTags};
+      }
+
+    if( $fh) 
+      { $t->print_prolog( $fh, %args); 
+        $t->_print_prolog_data( $fh, %args);
+      }
+    else 
+      { $t->print_prolog( %args);
+        $t->_print_prolog_data( %args);
+      }
+
+    $t->{twig_root}->print( $fh) if( $t->{twig_root});
+    $t->set_pretty_print( $old_pretty) if( defined $old_pretty); 
+    $t->set_empty_tag_style( $old_empty_tag_style) if( defined $old_empty_tag_style); 
+  }
+
+
+sub flush
+  { my $t= shift;
+    my $fh=  _is_fh( $_[0]) ? shift : undef;
+    my $old_select= defined $fh ? select $fh : undef;
+    my $up_to= ref $_[0] ? shift : undef;
+    my %args= _normalize_args( @_);
+
+    my $old_pretty;
+    if( defined $args{PrettyPrint})
+      { $old_pretty= $t->set_pretty_print( $args{PrettyPrint}); 
+        delete $args{PrettyPrint};
+      }
+
+     my $old_empty_tag_style;
+     if( $args{EmptyTags})
+      { $old_empty_tag_style= $t->set_empty_tag_style( $args{EmptyTags}); 
+        delete $args{EmptyTags};
+      }
+
+
+    # the "real" last element processed, as _twig_end has closed it
+    my $last_elt;
+    if( $up_to)
+      { $last_elt= $up_to; }
+    elsif( $t->{twig_current})
+      { $last_elt= $t->{twig_current}->_last_child; }
+    else
+      { $last_elt= $t->{twig_root}; }
+
+    # flush the DTD unless it has ready flushed (ie root has been flushed)
+    my $elt= $t->{twig_root};
+    $t->print_prolog( %args) unless( $elt->_flushed);
+
+    while( $elt)
+      { my $next_elt; 
+        if( $last_elt && $last_elt->in( $elt))
+          { 
+            unless( $elt->_flushed) 
+              { # just output the front tag
+                print $elt->start_tag();
+                $elt->_set_flushed;
+              }
+            $next_elt= $elt->{first_child};
+          }
+        else
+          { # an element before the last one or the last one,
+            $next_elt= $elt->{next_sibling};  
+            $elt->_flush();
+            $elt->delete; 
+            last if( $last_elt && ($elt == $last_elt));
+          }
+        $elt= $next_elt;
+      }
+    select $old_select if( defined $old_select);
+    $t->set_pretty_print( $old_pretty) if( defined $old_pretty); 
+    $t->set_empty_tag_style( $old_empty_tag_style) if( defined $old_empty_tag_style); 
+  }
+
+
+# flushes up to an element
+# this method just reorders the arguments and calls flush
+sub flush_up_to
+  { my $t= shift;
+    my $up_to= shift;
+    if( _is_fh( $_[0]))
+      { my $fh=  shift;
+        $t->flush( $fh, $up_to, @_);
+      }
+    else
+      { $t->flush( $up_to, @_); }
+  }
+
+    
+# same as print except the entire document text is returned as a string
+sub sprint
+  { my $t= shift;
+    my %args= _normalize_args( @_);
+
+    my $old_pretty;
+    if( defined $args{PrettyPrint})
+      { $old_pretty= $t->set_pretty_print( $args{PrettyPrint}); 
+        delete $args{PrettyPrint};
+      }
+
+     my $old_empty_tag_style;
+     if( defined $args{EmptyTags})
+      { $old_empty_tag_style= $t->set_empty_tag_style( $args{EmptyTags}); 
+        delete $args{EmptyTags};
+      }
+      
+    my $prolog= $t->prolog( %args) || '';
+    my $prolog_data= $t->_prolog_data( %args) || '';
+    
+    my $string=  $prolog . $prolog_data . $t->{twig_root}->sprint;
+
+    $t->set_pretty_print( $old_pretty) if( defined $old_pretty); 
+    $t->set_empty_tag_style( $old_empty_tag_style) if( defined $old_empty_tag_style); 
+
+    return $string;
+  }
+    
+
+# this method discards useless elements in a tree
+# it does the same thing as a flush except it does not print it
+# the second argument is an element, the last purged element
+# (this argument is usually set through the purge_up_to method)
+sub purge
+  { my $t= shift;
+    my $up_to= shift;
+
+    # the "real" last element processed, as _twig_end has closed it
+    my $last_elt;
+    if( $up_to)
+      { $last_elt= $up_to; }
+    elsif( $t->{twig_current})
+      { $last_elt= $t->{twig_current}->_last_child; }
+    else
+      { $last_elt= $t->{twig_root}; }
+    
+    my $elt= $t->{twig_root};
+
+    while( $elt)
+      { my $next_elt; 
+        if( $last_elt && $last_elt->in( $elt))
+          { $elt->_set_flushed;
+            $next_elt= $elt->{first_child};
+          }
+        else
+          { # an element before the last one or the last one,
+            $next_elt= $elt->{next_sibling};  
+            $elt->delete; 
+            last if( $elt == $last_elt);
+          }
+        $elt= $next_elt;
+      }
+  }
+    
+# flushes up to an element. This method just calls purge
+sub purge_up_to
+  { my $t= shift;
+    my $up_to= shift;
+    $t->purge( $up_to);
+  }
+
+sub root
+  { return $_[0]->{twig_root}; }
+
+#start-extract twig_document (used to generate XML::(DOM|GDOME)::Twig)
+sub first_elt
+  { my( $t, $cond)= @_;
+    my $root= $t->root || return undef;
+    return $root if( $root->passes( $cond));
+    return $root->next_elt( $cond); 
+  }
+
+sub next_n_elt
+  { my( $t, $offset, $cond)= @_;
+    $offset -- if( $t->root->matches( $cond) );
+    return $t->root->next_n_elt( $offset, $cond);
+  }
+
+sub get_xpath
+  { my $twig= shift;
+    if( isa( $_[0], 'ARRAY'))
+      { my $elt_array= shift;
+        return _unique_elts( map { $_->get_xpath( @_) } @$elt_array);
+      }
+    else
+      { return $twig->root->get_xpath( @_); }
+  }
+
+# get a list of elts and return a sorted list of unique elts
+sub _unique_elts
+  { my @sorted= sort { $a ->cmp( $b) } @_;
+    my @unique;
+    while( my $current= shift @sorted)
+      { push @unique, $current unless( @unique && ($unique[-1] == $current)); }
+    return @unique;
+  }
+
+sub findvalue
+  { my $twig= shift;
+    if( isa( $_[0], 'ARRAY'))
+      { my $elt_array= shift;
+        return join( '', map { $_->findvalue( @_) } @$elt_array);
+      }
+    else
+      { return $twig->root->findvalue( @_); }
+  }
+
+sub set_id_seed
+  { my $t= shift;
+    XML::Twig::Elt->set_id_seed( @_);
+  }
+
+
+# return a list with just the root
+# if a condition is given then return an empty list unless the root matches
+sub children
+  { my( $t, $cond)= @_;
+    my $root= $t->root;
+    unless( $cond && !($root->passes( $cond)) )
+      { return ($root); }
+    else
+      { return (); }
+  }
+
+sub _children
+  { return ($_[0]->root); }
+
+sub descendants
+  { my( $t, $cond)= @_;
+    my $root= $t->root;
+    if( $root->passes( $cond) )
+      { return ($root, $root->descendants( $cond)); }
+    else
+      { return ( $root->descendants( $cond)); }
+  }
+
+sub simplify
+  { my $t= shift;
+    $t->root->simplify( @_);
+  }
+
+sub subs_text
+  { my $t= shift;
+    $t->root->subs_text( @_);
+  }
+
+#end-extract twig_document
+
+sub set_keep_encoding
+  { return XML::Twig::Elt::set_keep_encoding( @_); }
+
+sub set_expand_external_entities
+  { return XML::Twig::Elt::set_expand_external_entities( @_); }
+
+# WARNING: at the moment the id list is not updated reliably
+sub elt_id
+  { return $_[0]->{twig_id_list}->{$_[1]}; }
+
+# change it in ALL twigs at the moment
+sub change_gi 
+  { my( $twig, $old_gi, $new_gi)= @_;
+    my $index;
+    return unless($index= $XML::Twig::gi2index{$old_gi});
+    $XML::Twig::index2gi[$index]= $new_gi;
+    delete $XML::Twig::gi2index{$old_gi};
+    $XML::Twig::gi2index{$new_gi}= $index;
+  }
+
+
+# builds the DTD from the stored (possibly updated) data
+sub dtd_text
+  { my $t= shift;
+    my $dtd= $t->{twig_dtd};
+    my $doctype= $t->{twig_doctype} or return '';
+    my $string= "<!DOCTYPE ".$doctype->{name};
+
+    unless( $parser_version > 3.27) { $string .= " [\n"; }
+
+    foreach my $gi (@{$dtd->{elt_list}})
+      { $string.= "<!ELEMENT $gi ".$dtd->{model}->{$gi}.">\n" ;
+        if( $dtd->{att}->{$gi})
+          { my $attlist= $dtd->{att}->{$gi};
+            $string.= "<!ATTLIST $gi\n";
+            foreach my $att ( sort keys %{$attlist})
+              { $string.= "   $att $attlist->{$att}->{type} ".
+                            "$attlist->{$att}->{default}"; 
+                if( $attlist->{$att}->{fixed})
+                  { $string .= " #FIXED"};
+                $string.= "\n";
+              }
+            $string.= ">\n";
+          }
+      }
+    $string.= $t->entity_list->text if( $t->entity_list);
+    $string.= "\n]>\n";
+    return $string;
+  }
+        
+# prints the DTD from the stored (possibly updated) data
+sub dtd_print
+  { my $t= shift;
+    my $fh=  _is_fh( $_[0])  ? shift : undef;
+    if( $fh) { print $fh $t->dtd_text; }
+    else     { print $t->dtd_text; }
+  }
+
+# build the subs that call directly expat
+BEGIN
+  { my @expat_methods= qw( depth in_element within_element context
+                           current_line current_column current_byte
+                           recognized_string original_string 
+                           xpcroak xpcarp 
+                           xml_escape
+                           base current_element element_index 
+                           position_in_context);
+    foreach my $method (@expat_methods)
+      { no strict 'refs';
+        *{$method}= sub { my $t= shift;
+                          croak "calling $method after parsing is finished" 
+                                 unless( $t->{twig_parsing}); 
+                          return $t->{twig_parser}->$method(@_); 
+                        };
+      }
+  }
+
+sub path
+  { my( $t, $gi)= @_;
+    return "/" . join( "/", ($t->{twig_parser}->context, $gi));
+  }
+
+sub finish
+  { my $t= shift;
+    return $t->{twig_parser}->finish;
+  }
+
+# just finish the parse by printing the rest of the document
+sub finish_print
+  { my( $t, $fh)= @_;
+    my $old_fh;
+    unless( defined $fh)
+      { $t->_set_fh_to_twig_output_fh(); }
+    elsif( defined $fh)
+      { $old_fh= select $fh; 
+        $t->{twig_original_selected_fh}= $old_fh if( $old_fh); 
+      }
+    
+    my $p=$t->{twig_parser};
+    if( $t->{twig_keep_encoding})
+      { $p->setHandlers( %twig_handlers_finish_print); }
+    else
+      { $p->setHandlers( %twig_handlers_finish_print_original); }
+  }
+
+sub output_filter { return XML::Twig::Elt::output_filter( @_); }
+sub set_output_filter { return XML::Twig::Elt::set_output_filter( @_); }
+
+sub output_text_filter { return XML::Twig::Elt::output_text_filter( @_); }
+sub set_output_text_filter { return XML::Twig::Elt::set_output_text_filter( @_); }
+
+sub set_input_filter
+  { my( $t, $input_filter)= @_;
+    my $old_filter= $t->{twig_input_filter};
+      if( !$input_filter || isa( $input_filter, 'CODE') )
+        { $t->{twig_input_filter}= $input_filter; }
+      elsif( $input_filter eq 'latin1')
+        {  $t->{twig_input_filter}= latin1(); }
+      elsif( $filter{$input_filter})
+        {  $t->{twig_input_filter}= $filter{$input_filter}; }
+      else
+        { croak "invalid input filter: $input_filter"; }
+      
+      return $old_filter;
+    }
+
+sub set_empty_tag_style
+  { return XML::Twig::Elt::set_empty_tag_style( @_); }
+
+sub set_pretty_print
+  { return XML::Twig::Elt::set_pretty_print( @_); }
+
+sub set_quote
+  { return XML::Twig::Elt::set_quote( @_); }
+
+sub set_indent
+  { return XML::Twig::Elt::set_indent( @_); }
+
+sub set_keep_atts_order
+  { shift; return XML::Twig::Elt::set_keep_atts_order( @_); }
+
+sub keep_atts_order
+  { return XML::Twig::Elt::keep_atts_order( @_); }
+
+# save and restore package globals (the ones in XML::Twig::Elt)
+sub save_global_state
+  { my $t= shift;
+    $t->{twig_saved_state}= XML::Twig::Elt::global_state();
+  }
+
+sub restore_global_state
+  { my $t= shift;
+    XML::Twig::Elt::set_global_state( $t->{twig_saved_state});
+  }
+
+sub global_state
+  { return XML::Twig::Elt::global_state(); }
+
+sub set_global_state
+  {  return XML::Twig::Elt::set_global_state( $_[1]); }
+
+sub dispose
+  { my $t= shift;
+    $t->DESTROY;
+  }
+  
+sub DESTROY
+  { my $t= shift;
+    if( $t->{twig_root} && isa(  $t->{twig_root}, 'XML::Twig')) 
+      { $t->{twig_root}->delete } 
+
+# added to break circular references
+    undef $t->{twig};
+    undef $t->{twig_root}->{twig} if( $t->{twig_root});
+    undef $t->{twig_parser};
+    
+    $t={}; # prevents memory leaks (especially when using mod_perl)
+    undef $t;
+  }        
+
+
+#
+#  non standard handlers
+#
+
+# kludge: expat 1.95.2 calls both Default AND Doctype handlers
+# so if the default handler finds '<!DOCTYPE' then it must 
+# unset itself (_twig_print_doctype will reset it)
+sub _twig_print_check_doctype
+  { 
+    my $p= shift;
+    my $string= $p->recognized_string();
+    if( $string eq '<!DOCTYPE') 
+      { $p->setHandlers( Default => undef); 
+        $p->{twig}->{expat_1_95_2}=1; 
+      }
+    else                        
+      { print $string; }
+    
+  }
+
+#sub twig_buffer_original
+#  { my $p= shift; 
+#    my $t= $p->{twig};
+#    $t->{twig_buffered_string}= $p->original_string(); 
+#  }
+
+sub _twig_print
+  { print $_[0]->recognized_string(); }
+
+sub _twig_print_default
+  { my( $p, $string)= @_;
+    print $p->recognized_string();
+  }
+
+# recognized_string does not seem to work for entities, go figure!
+# so this handler is not used 
+sub _twig_print_entity
+  { my $p= shift; }
+
+# account for the case where the element is empty
+sub _twig_print_end
+  { my $p= shift; 
+    print $p->recognized_string();
+  }
+
+
+# kludge: expat 1.95.2 calls both Default AND Doctype handlers
+# so if the default handler finds '<!DOCTYPE' then it must 
+# unset itself (_twig_print_doctype will reset it)
+sub _twig_print_original_check_doctype
+  { 
+    my $p= shift;
+    my $string= $p->original_string();
+    if( $string eq '<!DOCTYPE') 
+      { $p->setHandlers( Default => undef); 
+        $p->{twig}->{expat_1_95_2}=1; 
+      }
+    else                        
+      { print $string; }
+    
+  }
+
+sub _twig_print_original
+  { print $_[0]->original_string(); }
+
+
+sub _twig_print_original_doctype
+  { 
+    my(  $p, $name, $sysid, $pubid, $internal)= @_;
+    if( $name)
+      { # with recent versions of XML::Parser original_string does not work,
+        # hence we need to rebuild the doctype declaration
+        my $doctype='';
+        $doctype .= qq{<!DOCTYPE $name}    if( $name);
+        $doctype .=  qq{ PUBLIC  "$pubid"}  if( $pubid);
+        $doctype .=  qq{ SYSTEM}            if( $sysid && !$pubid);
+        $doctype .=  qq{ "$sysid"}          if( $sysid); 
+        $doctype .=  qq{>} unless( $p->{twig}->{expat_1_95_2});
+        print $doctype;
+      }
+    $p->setHandlers( Default => \&_twig_print_original);
+  }
+
+sub _twig_print_doctype
+  { 
+    my(  $p, $name, $sysid, $pubid, $internal)= @_;
+    if( $name)
+      { # with recent versions of XML::Parser original_string does not work,
+        # hence we need to rebuild the doctype declaration
+        my $doctype='';
+        $doctype .= qq{<!DOCTYPE $name}    if( $name);
+        $doctype .=  qq{ PUBLIC  "$pubid"}  if( $pubid);
+        $doctype .=  qq{ SYSTEM}            if( $sysid && !$pubid);
+        $doctype .=  qq{ "$sysid"}          if( $sysid); 
+        $doctype .=  qq{>} unless( $p->{twig}->{expat_1_95_2});
+        print $doctype;
+      }
+    $p->setHandlers( Default => \&_twig_print_original);
+  }
+
+
+
+
+sub _twig_print_original_default
+  { 
+    my $p= shift;
+    print $p->original_string();
+  }
+
+# account for the case where the element is empty
+sub _twig_print_end_original
+  { my $p= shift;
+    print $p->original_string();
+  }
+
+sub _twig_start_check_roots
+  { 
+    my( $p, $gi, %att)= @_;
+    my $t= $p->{twig};
+
+    if( _handler( $t, $t->{twig_roots}, $gi, \%att))
+      { 
+        $p->setHandlers( %twig_handlers); # restore regular handlers
+        $t->{twig_root_depth}= $p->depth; 
+        _twig_start( $p, $gi, %att);
+      }
+    elsif( $p->depth == 0)
+      { _twig_start( $p, $gi, %att); }
+    elsif( $t->{twig_starttag_handlers})
+      { # look for start tag handlers
+        my @handlers= _handler( $t, $t->{twig_starttag_handlers}, $gi, \%att);
+        foreach my $handler ( @handlers)
+          { $handler->($t, $gi, %att) || last; }
+      }
+  }
+
+sub _twig_start_check_roots_print
+  { 
+    my( $p, $gi, %att)= @_;
+    my $t= $p->{twig};
+
+    if( _handler( $t, $t->{twig_roots}, $gi, \%att))
+      { $p->setHandlers( %twig_handlers); # restore regular handlers
+        $t->_set_fh_to_selected_fh();      # select the proper output fh
+        $t->{twig_root_depth}= $p->depth; 
+        _twig_start( $p, $gi, %att);
+      }
+    elsif( $p->depth == 0)
+      { if( my $fh= $t->{twig_output_fh})
+          { print $fh $p->recognized_string(); }
+        else
+          { print $p->recognized_string(); }
+      
+        _twig_start( $p, $gi, %att);
+        $t->_set_fh_to_twig_output_fh();
+      }
+    elsif( $t->{twig_starttag_handlers})
+      { # look for start tag handlers
+        my @handlers= _handler( $t, $t->{twig_starttag_handlers}, $gi, \%att);
+        my $last_handler_res;
+        $t->_set_fh_to_selected_fh() if( @handlers);     # select the proper output fh
+        foreach my $handler ( @handlers)
+          { $last_handler_res= $handler->($t, $gi, %att);
+            last unless $last_handler_res;
+          }
+        $t->_set_fh_to_twig_output_fh() if( @handlers);  # re-select the twig output fh
+        print $p->recognized_string() if( !@handlers || $last_handler_res);   
+      }
+    else
+      { print $p->recognized_string(); }  
+  }
+
+sub _twig_start_check_roots_print_original
+  { 
+    my( $p, $gi, %att)= @_;
+    my $t= $p->{twig};
+
+    if( _handler( $t, $t->{twig_roots}, $gi, \%att))
+      { $p->setHandlers( %twig_handlers); # restore regular handlers
+        $t->_set_fh_to_selected_fh();      # select the proper output fh
+        $t->{twig_root_depth}= $p->depth; 
+        _twig_start( $p, $gi, %att);
+      }
+    elsif( $p->depth == 0)
+      { if( my $fh= $t->{twig_output_fh})
+          { print $fh $p->original_string(); }
+        else
+          { print $p->original_string(); }
+
+        _twig_start( $p, $gi, %att);
+        $t->_set_fh_to_twig_output_fh();
+      }
+    elsif( $t->{twig_starttag_handlers})
+      { # look for start tag handlers
+        my @handlers= _handler( $t, $t->{twig_starttag_handlers}, $gi, \%att);
+        my $last_handler_res;
+        $t->_set_fh_to_selected_fh() if( @handlers);     # select the proper output fh
+        foreach my $handler ( @handlers)
+          { $last_handler_res= $handler->($t, $gi, %att);
+            last unless( $last_handler_res);
+          }
+        $t->_set_fh_to_twig_output_fh() if( @handlers);  # re-select the twig output fh
+        print $p->original_string() if( !@handlers || $last_handler_res);   
+      }
+    else
+      { print $p->original_string(); }
+  }
+
+sub _twig_end_check_roots
+  { 
+    my( $p, $gi)= @_;
+    my $t= $p->{twig};
+    if( $t->{twig_endtag_handlers})
+      { # look for end tag handlers
+        my @handlers= _handler( $t, $t->{twig_endtag_handlers}, $gi, {});
+        my $last_handler_res=1;
+        foreach my $handler ( @handlers)
+          { $last_handler_res= $handler->($t, $gi) || last; }
+      }
+
+    if( $p->depth == 0)
+      { _twig_end( $p, $gi); }
+  }
+
+sub _twig_end_check_roots_print
+  { 
+    my( $p, $gi, %att)= @_;
+    my $t= $p->{twig};
+    if( $t->{twig_endtag_handlers})
+      { # look for start tag handlers
+        my @handlers= _handler( $t, $t->{twig_endtag_handlers}, $gi, {});
+        my $last_handler_res=1;
+        $t->_set_fh_to_selected_fh() if( @handlers);     # select the proper output fh
+        foreach my $handler ( @handlers)
+          { $last_handler_res= $handler->($t, $gi) || last; }
+        $t->_set_fh_to_twig_output_fh() if( @handlers);  # re-select the twig output fh
+        return unless $last_handler_res;
+      }
+    print $p->recognized_string();  
+    $t->_set_fh_to_selected_fh() if( $p->depth == 0);
+    if( $p->depth == 0)
+      { _twig_end( $p, $gi);  }
+  }
+
+sub _twig_end_check_roots_print_original
+  { 
+    my( $p, $gi, %att)= @_;
+    my $t= $p->{twig};
+    if( $t->{twig_endtag_handlers})
+      { # look for start tag handlers
+        my @handlers= _handler( $t, $t->{twig_endtag_handlers}, $gi, {});
+        my $last_handler_res=1;
+        $t->_set_fh_to_selected_fh() if( @handlers);     # select the proper output fh
+        foreach my $handler ( @handlers)
+          { $last_handler_res= $handler->($t, $gi) || last; }
+        $t->_set_fh_to_twig_output_fh() if( @handlers);  # re-select the twig output fh
+        return unless $last_handler_res;
+      }
+    print $p->original_string();
+    $t->_set_fh_to_selected_fh() if( $p->depth == 0);
+    if( $p->depth == 0)
+      { _twig_end( $p, $gi);  }
+  }
+
+sub _twig_pi_check_roots
+  { my( $p, $target, $data)= @_;
+    my $t= $p->{twig};
+    if( my $handler=    $t->{twig_handlers}->{pi_handlers}->{$target}
+                     || $t->{twig_handlers}->{pi_handlers}->{''}
+      )
+      { _twig_pi( @_); }
+  }
+
+
+sub _twig_ignore_start
+  { 
+    my( $p, $gi)= @_;
+    my $t= $p->{twig};
+    return unless( $gi eq $t->{twig_ignore_gi});
+    $t->{twig_ignore_level}++;
+    my $action= $t->{twig_ignore_action};
+    if( $action eq 'print' )
+      { _twig_print_original( @_); }
+#    elsif( $action eq 'string' )
+#      { $t->{twig_buffered_string} .= $p->original_string(); }
+  }
+
+sub _twig_ignore_end
+  { 
+    my( $p, $gi)= @_;
+    my $t= $p->{twig};
+
+    my $action= $t->{twig_ignore_action};
+
+    if( $action eq 'print')
+      { _twig_print_original( $p, $gi); }
+#    elsif( $action eq 'string')
+#      { $t->{twig_buffered_string} .= $p->original_string(); }
+
+    return unless( $gi eq $t->{twig_ignore_gi});
+
+    $t->{twig_ignore_level}--;
+
+    unless( $t->{twig_ignore_level})
+      { $t->{twig_ignore_elt}->delete; 
+        $p->setHandlers( @{$t->{twig_saved_handlers}});
+        # test for handlers
+        if( $t->{twig_endtag_handlers})
+          { # look for end tag handlers
+            my @handlers= _handler( $t, $t->{twig_endtag_handlers}, $gi, {});
+            my $last_handler_res=1;
+            foreach my $handler ( @handlers)
+              { $last_handler_res= $handler->($t, $gi) || last; }
+          }
+      };
+  }
+    
+sub ignore
+  { my $t= shift;
+    my $elt;
+
+    # get the element (default: current elt)
+    if( $_[0] && isa( $_[0], 'XML::Twig::Elt'))
+      { $elt= shift; }
+    else
+      { $elt = $t->{twig_current}; }
+
+    my $action= shift || 1; 
+    $t->{twig_ignore_action}= $action;
+
+    $t->{twig_ignore_elt}= $elt;     # save it
+    $t->{twig_ignore_gi}= $XML::Twig::index2gi[$elt->{'gi'}];  # save its gi
+    $t->{twig_ignore_level}++;
+    my $p= $t->{twig_parser};
+    my @saved_handlers= $p->setHandlers( %twig_handlers_ignore); # set handlers
+    if( $action eq 'print')
+      { $p->setHandlers( Default => \&_twig_print_original); }
+#    elsif( $action eq 'string')
+#      { # not used at the moment
+#        $t->{twig_buffered_string}='';
+#        $p->setHandlers( Default => \&twig_buffer_original);
+#      }
+
+    $t->{twig_saved_handlers}= \@saved_handlers;        # save current handlers
+  }
+
+# select $t->{twig_output_fh} and store the current selected fh 
+sub _set_fh_to_twig_output_fh
+  { my $t= shift;
+    my $output_fh= $t->{twig_output_fh};
+    if( $output_fh && !$t->{twig_output_fh_selected})
+      { # there is an output fh
+        $t->{twig_selected_fh}= select(); # store the currently selected fh
+        $t->{twig_output_fh_selected}=1;
+        select $output_fh;                # select the output fh for the twig
+      }
+  }
+
+# select the fh that was stored in $t->{twig_selected_fh} 
+# (before $t->{twig_output_fh} was selected)
+sub _set_fh_to_selected_fh
+  { my $t= shift;
+    return unless( $t->{twig_output_fh});
+    my $selected_fh= $t->{twig_selected_fh};
+    $t->{twig_output_fh_selected}=0;
+    select $selected_fh;
+    return;
+  }
+  
+
+sub encoding
+  { return $_[0]->{twig_xmldecl}->{encoding} if( $_[0]->{twig_xmldecl}); }
+
+sub set_encoding
+  { my( $t, $encoding)= @_;
+    $t->{twig_xmldecl} ||={};
+    $t->set_xml_version( "1.0") unless( $t->xml_version);
+    $t->{twig_xmldecl}->{encoding}= $encoding;
+    return $t;
+  }
+
+sub output_encoding
+  { return $_[0]->{output_encoding}; }
+  
+sub set_output_encoding
+  { my( $t, $encoding)= @_;
+    $t->set_output_filter( _encoding_filter( $encoding)) if( $encoding);
+    return $t->{output_encoding}= $encoding;
+  }
+
+sub xml_version
+  { return $_[0]->{twig_xmldecl}->{version} if( $_[0]->{twig_xmldecl}); }
+
+sub set_xml_version
+  { my( $t, $version)= @_;
+    $t->{twig_xmldecl} ||={};
+    return $t->{twig_xmldecl}->{version}= $version;
+  }
+
+sub standalone
+  { return $_[0]->{twig_xmldecl}->{standalone} if( $_[0]->{twig_xmldecl}); }
+
+sub set_standalone
+  { my( $t, $standalone)= @_;
+    $t->{twig_xmldecl} ||={};
+    $t->set_xml_version( "1.0") unless( $t->xml_version);
+    return $t->{twig_xmldecl}->{standalone}= $standalone;
+  }
+
+
+# SAX methods
+
+sub toSAX1
+  { croak "cannot use toSAX1 while parsing (use flush_toSAX1)" if (defined $_[0]->{twig_parser});
+    shift(@_)->_toSAX(@_, \&XML::Twig::Elt::_start_tag_data_SAX1,
+                          \&XML::Twig::Elt::_end_tag_data_SAX1
+             ); }
+
+sub toSAX2
+  { croak "cannot use toSAX2 while parsing (use flush_toSAX2)" if (defined $_[0]->{twig_parser});
+    shift(@_)->_toSAX(@_, \&XML::Twig::Elt::_start_tag_data_SAX2,
+                          \&XML::Twig::Elt::_end_tag_data_SAX2
+             ); }
+
+
+sub _toSAX
+  { my( $t, $handler, $start_tag_data, $end_tag_data) = @_;
+
+    if( my $start_document =  $handler->can( 'start_document'))
+      { $start_document->( $handler); }
+    
+    $t->_prolog_toSAX( $handler);
+    
+    $t->root->_toSAX( $handler, $start_tag_data, $end_tag_data)  if( $t->root);
+    if( my $end_document =  $handler->can( 'end_document'))
+      { $end_document->( $handler); }
+  }
+
+
+sub flush_toSAX1
+  { shift(@_)->_flush_toSAX(@_, \&XML::Twig::Elt::_start_tag_data_SAX1,
+                               \&XML::Twig::Elt::_end_tag_data_SAX1
+             ); }
+
+sub flush_toSAX2
+  { shift(@_)->_flush_toSAX(@_, \&XML::Twig::Elt::_start_tag_data_SAX2,
+                               \&XML::Twig::Elt::_end_tag_data_SAX2
+             ); }
+
+sub _flush_toSAX
+  { my( $t, $handler, $start_tag_data, $end_tag_data, $up_to)= @_;
+
+    # the "real" last element processed, as _twig_end has closed it
+    my $last_elt;
+    if( $up_to)
+      { $last_elt= $up_to; }
+    elsif( $t->{twig_current})
+      { $last_elt= $t->{twig_current}->_last_child; }
+    else
+      { $last_elt= $t->{twig_root}; }
+
+    my $elt= $t->{twig_root};
+    unless( $elt->_flushed)
+      { # init unless already done (ie root has been flushed)
+        if( my $start_document =  $handler->can( 'start_document'))
+          { $start_document->( $handler); }
+        # flush the DTD
+        $t->_prolog_toSAX( $handler) 
+      }
+
+    while( $elt)
+      { my $next_elt; 
+        if( $last_elt && $last_elt->in( $elt))
+          { 
+            unless( $elt->_flushed) 
+              { # just output the front tag
+                if( my $start_element = $handler->can( 'start_element'))
+                 { if( my $tag_data= $start_tag_data->( $elt))
+                     { $start_element->( $handler, $tag_data); }
+                 }
+                $elt->_set_flushed;
+              }
+            $next_elt= $elt->{first_child};
+          }
+        else
+          { # an element before the last one or the last one,
+            $next_elt= $elt->{next_sibling};  
+            $elt->_toSAX( $handler, $start_tag_data, $end_tag_data);
+            $elt->delete; 
+            last if( $last_elt && ($elt == $last_elt));
+          }
+        $elt= $next_elt;
+      }
+    if( !$t->{twig_parsing}) 
+      { if( my $end_document =  $handler->can( 'end_document'))
+          { $end_document->( $handler); }
+      }
+  }
+
+
+sub _prolog_toSAX
+  { my( $t, $handler)= @_;
+    $t->_xmldecl_toSAX( $handler);
+    $t->_DTD_toSAX( $handler);
+  }
+
+sub _xmldecl_toSAX
+  { my( $t, $handler)= @_;
+    my $decl= $t->{twig_xmldecl};
+    my $data= { Version    => $decl->{version},
+                Encoding   => $decl->{encoding},
+                Standalone => $decl->{standalone},
+          };
+    if( my $xml_decl= $handler->can( 'xml_decl'))
+      { $xml_decl->( $handler, $data); }
+  }
+                
+sub _DTD_toSAX
+  { my( $t, $handler)= @_;
+    my $doctype= $t->{twig_doctype};
+    return unless( $doctype);
+    my $data= { Name     => $doctype->{name},
+                PublicId => $doctype->{pub},
+                SystemId => $doctype->{sysid},
+              };
+
+    if( my $start_dtd= $handler->can( 'start_dtd'))
+      { $start_dtd->( $handler, $data); }
+
+    # I should call code to export the internal subset here 
+    
+    if( my $end_dtd= $handler->can( 'end_dtd'))
+      { $end_dtd->( $handler); }
+  }
+
+# input/output filters
+
+sub latin1 
+  { local $SIG{__DIE__};
+    if( eval 'require Encode')
+      { import Encode; 
+        return encode_convert( 'ISO-8859-15');
+      }
+    elsif( eval 'require Text::Iconv;')
+      { 
+        return iconv_convert( 'ISO-8859-15');
+      }
+    elsif( eval 'require Unicode::Map8 && require Unicode::String;')
+      { 
+        return unicode_convert( 'ISO-8859-15'); 
+      }
+    else
+      { return \&regexp2latin1; }
+  }
+
+sub _encoding_filter
+  { 
+      { local $SIG{__DIE__};
+        my $encoding= $_[1] || $_[0];
+        if( eval 'require Encode')
+          { import Encode; 
+            my $sub= encode_convert( $encoding);
+            return $sub;
+          }
+        if( eval 'require Text::Iconv;')
+          { return iconv_convert( $encoding); }
+        elsif( eval 'require Unicode::Map8 && require Unicode::String;')
+          { return unicode_convert( $encoding); }
+        }
+    croak "Encode, Text::Iconv or Unicode::Map8 and Unicode::String need to be installed ",
+          "in order to use encoding options";
+  }
+
+# shamelessly lifted from XML::TyePYX (works only with XML::Parse 2.27)
+sub regexp2latin1
+  { my $text=shift;
+    $text=~s{([\xc0-\xc3])(.)}{ my $hi = ord($1);
+                                my $lo = ord($2);
+                                chr((($hi & 0x03) <<6) | ($lo & 0x3F))
+                              }ge;
+    return $text;
+  }
+
+
+sub html_encode
+  { require HTML::Entities;
+    return HTML::Entities::encode_entities($_[0] );
+  }
+
+sub safe_encode
+  {   my $str= shift;
+      if( $] < 5.008)
+        { $str =~ s{([\xC0-\xDF].|[\xE0-\xEF]..|[\xF0-\xFF]...)}
+                   {_XmlUtf8Decode($1)}egs; 
+        }
+      else
+        { $str= encode( ascii => $str, $FB_HTMLCREF); }
+      return $str;
+  }
+
+sub safe_encode_hex
+  {   my $str= shift;
+      if( $] < 5.008)
+        { $str =~ s{([\xC0-\xDF].|[\xE0-\xEF]..|[\xF0-\xFF]...)}
+                   {_XmlUtf8Decode($1, 1)}egs; 
+        }
+      else
+        { $str= encode( ascii => $str, $FB_XMLCREF); }
+      return $str;
+  }
+
+# this one shamelessly lifted from XML::DOM
+# does NOT work on 5.8.0
+sub _XmlUtf8Decode
+  { my ($str, $hex) = @_;
+    my $len = length ($str);
+    my $n;
+
+    if ($len == 2)
+      { my @n = unpack "C2", $str;
+        $n = (($n[0] & 0x3f) << 6) + ($n[1] & 0x3f);
+      }
+    elsif ($len == 3)
+      { my @n = unpack "C3", $str;
+        $n = (($n[0] & 0x1f) << 12) + (($n[1] & 0x3f) << 6) + ($n[2] & 0x3f);
+      }
+    elsif ($len == 4)
+      { my @n = unpack "C4", $str;
+        $n = (($n[0] & 0x0f) << 18) + (($n[1] & 0x3f) << 12) 
+           + (($n[2] & 0x3f) << 6) + ($n[3] & 0x3f);
+      }
+    elsif ($len == 1)    # just to be complete...
+      { $n = ord ($str); }
+    else
+      { croak "bad value [$str] for _XmlUtf8Decode"; }
+
+    my $char= $hex ? sprintf ("&#x%x;", $n) : "&#$n;";
+    return $char;
+}
+
+
+sub unicode_convert
+  { my $enc= $_[1] ? $_[1] : $_[0]; # so the method can be called on the twig or directly
+    require Unicode::Map8;
+    require Unicode::String;
+    import Unicode::String qw(utf8);
+    my $sub= eval q{
+            { my $cnv;
+              BEGIN {  $cnv= Unicode::Map8->new($enc) 
+                           or croak "Can't create converter to $enc";
+                    }
+              sub { return  $cnv->to8 (utf8($_[0])->ucs2); } 
+            } 
+                   };
+    unless( $sub) { croak $@; }
+    return $sub;
+  }
+
+sub iconv_convert
+  { my $enc= $_[1] ? $_[1] : $_[0]; # so the method can be called on the twig or directly
+    require Text::Iconv;
+    my $sub= eval q{
+            { my $cnv;
+              BEGIN { $cnv = Text::Iconv->new( 'utf8', $enc) 
+                           or croak "Can't create iconv converter to $enc";
+                    }
+              sub { return  $cnv->convert( $_[0]); } 
+            }       
+                   };
+    unless( $sub)
+      { if( $@=~ m{^Unsupported conversion: Invalid argument})
+          { croak "Unsupported encoding: $enc"; }
+        else
+          { croak $@; }
+      }
+
+    return $sub;
+  }
+
+sub encode_convert
+  { my $enc= $_[1] ? $_[1] : $_[0]; # so the method can be called on the twig or directly
+    my $sub=  eval qq{sub { return encode( "$enc", \$_[0]); } };
+    croak "can't create Encode-based filter: $@" unless( $sub);
+    return $sub;
+  }
+
+
+# XML::XPath compatibility
+sub getRootNode        { return $_[0]; }
+sub getParentNode      { return undef; }
+sub getChildNodes      { my @children= ($_[0]->root); return wantarray ? @children : \@children; }
+
+
+1;
+
+######################################################################
+package XML::Twig::Entity_list;
+######################################################################
+use UNIVERSAL qw( isa ) ;
+
+sub new
+  { my $class = shift;
+    my $self={};
+
+    bless $self, $class;
+    return $self;
+
+  }
+
+sub add_new_ent
+  { my $list= shift;
+    my $ent= XML::Twig::Entity->new( @_);
+    $list->add( $ent);
+  }
+
+sub add
+  { my( $list, $ent)= @_;
+    $list->{$ent->{name}}= $ent;
+  }
+
+# can be called with an entity or with an entity name
+sub delete
+  { my $list= shift;
+    if( isa( ref $_[0], 'XML::Twig::Entity'))
+      { # the second arg is an entity
+        my $ent= shift;
+        delete $list->{$ent->{name}};
+      }
+    else
+      { # the second arg was not entity, must be a string then
+        my $name= shift;
+        delete $list->{$name};
+      }
+  }
+
+sub print
+  { my ($ent_list, $fh)= @_;
+    my $old_select= defined $fh ? select $fh : undef;
+
+    foreach my $ent_name ( sort keys %{$ent_list})
+      { my $ent= $ent_list->{$ent_name};
+        # we have to test what the entity is or un-defined entities can creep in
+        $ent->print() if( isa( $ent, 'XML::Twig::Entity'));
+      }
+    select $old_select if( defined $old_select);
+  }
+
+sub text
+  { my ($ent_list)= @_;
+    return join "\n", map { $ent_list->{$_}->text} sort keys %{$ent_list};
+  }
+
+sub list
+  { my ($ent_list)= @_;
+    return map { $ent_list->{$_} } sort keys %$ent_list;
+  }
+
+1;
+
+######################################################################
+package XML::Twig::Entity;
+######################################################################
+use UNIVERSAL qw( isa ) ;
+
+sub new
+  { my( $ent, $name, $val, $sysid, $pubid, $ndata)= @_;
+
+    my $self={};
+
+    $self->{name}= $name;
+    if( $val)
+      { $self->{val}= $val; }
+    else
+      { $self->{sysid}= $sysid;
+        $self->{pubid}= $pubid;
+        $self->{ndata}= $ndata;
+      }
+    bless $self;
+    return $self;
+  }
+
+sub name  { return $_[0]->{name}; }
+sub val   { return $_[0]->{val}; }
+sub sysid { return $_[0]->{sysid}; }
+sub pubid { return $_[0]->{pubid}; }
+sub ndata { return $_[0]->{ndata}; }
+
+sub print
+  { my ($ent, $fh)= @_;
+    if( $fh) { print $fh $ent->text . "\n"; }
+    else     { print $ent->text . "\n"; }
+  }
+
+
+sub text
+  { my ($ent)= @_;
+    if( exists $ent->{'val'})
+      { if( $ent->{'val'}=~ /"/)
+          { return "<!ENTITY $ent->{'name'} '$ent->{'val'}'>"; }
+        return "<!ENTITY $ent->{'name'} \"$ent->{'val'}\">";
+      }
+    elsif( $ent->{'sysid'})
+      { my $text= "<!ENTITY $ent->{'name'} ";
+        $text .= "SYSTEM \"$ent->{'sysid'}\" " if( $ent->{'sysid'});
+        $text .= "PUBLIC \"$ent->{'pubid'}\" " if( $ent->{'pubid'});
+        $text .= "NDATA $ent->{'ndata'}"        if( $ent->{'ndata'});
+        $text .= ">";
+        return $text;
+      }
+  }
+
+                
+1;
+
+######################################################################
+package XML::Twig::Elt;
+######################################################################
+use Carp;
+use UNIVERSAL qw( isa ) ;
+
+use constant  PCDATA  => '#PCDATA'; 
+use constant  CDATA   => '#CDATA'; 
+use constant  PI      => '#PI'; 
+use constant  COMMENT => '#COMMENT'; 
+use constant  ENT     => '#ENT'; 
+
+use constant  ASIS    => '#ASIS';    # pcdata elements not to be XML-escaped
+
+use constant  ELT     => '#ELT'; 
+use constant  TEXT    => '#TEXT'; 
+use constant  EMPTY   => '#EMPTY'; 
+
+use constant CDATA_START    => "<![CDATA[";
+use constant CDATA_END      => "]]>";
+use constant PI_START       => "<?";
+use constant PI_END         => "?>";
+use constant COMMENT_START  => "<!--";
+use constant COMMENT_END    => "-->";
+
+use constant XMLNS_URI      => 'http://www.w3.org/2000/xmlns/';
+my $XMLNS_URI               = XMLNS_URI;
+
+
+BEGIN
+  { # set some aliases for methods
+    *tag           = *gi; 
+    *name          = *gi; 
+    *set_tag       = *set_gi; 
+    *set_name      = *set_gi; 
+    *find_nodes    = *get_xpath; # as in XML::DOM
+    *findnodes     = *get_xpath; # as in XML::LibXML
+    *field         = *first_child_text;
+    *trimmed_field = *first_child_trimmed_text;
+    *is_field      = *contains_only_text;
+    *is            = *passes;
+    *matches       = *passes;
+    *has_child     = *first_child;
+    *has_children  = *first_child;
+    *all_children_pass = *all_children_are;
+    *all_children_match= *all_children_are;
+    *getElementsByTagName= *descendants;
+    *find_by_tag_name= *descendants_or_self;
+  
+    *first_child_is  = *first_child_matches;
+    *last_child_is   = *last_child_matches;
+    *next_sibling_is = *next_sibling_matches;
+    *prev_sibling_is = *prev_sibling_matches;
+    *next_elt_is     = *next_elt_matches;
+    *prev_elt_is     = *prev_elt_matches;
+    *parent_is       = *parent_matches;
+    *child_is        = *child_matches;
+
+    *sort_children_by_value= *sort_children_on_value;
+
+    *has_atts= *att_nb;
+
+    # imports from XML::Twig
+    *_is_fh= *XML::Twig::_is_fh;
+
+    # XML::XPath compatibility
+    *string_value       = *text;
+    *toString           = *sprint;
+    *getName            = *gi;
+    *getRootNode        = *twig;  
+    *getNextSibling     = *_next_sibling;
+    *getPreviousSibling = *_prev_sibling;
+    *isElementNode      = *is_elt;
+    *isTextNode         = *is_text;
+    *isPI               = *is_pi;
+    *isPINode           = *is_pi;
+    *isProcessingInstructionNode= *is_pi;
+    *isComment          = *is_comment;
+    *isCommentNode      = *is_comment;
+    *getTarget          = *target;
+
+    # try using weak references
+    # test whether we can use weak references
+    { local $SIG{__DIE__};
+      if( eval 'require Scalar::Util')
+        { import Scalar::Util qw(weaken); }
+      elsif( eval 'require WeakRef')
+        { import WeakRef; }
+    }
+}
+
+ 
+# can be called as XML::Twig::Elt->new( [[$gi, $atts, [@content]])
+# - gi is an optional gi given to the element
+# - $atts is a hashref to attributes for the element
+# - @content is an optional list of text and elements that will
+#   be inserted under the element 
+sub new 
+  { my $class= shift;
+    $class= ref $class || $class;
+    my $elt  = {};
+    bless ($elt, $class);
+
+    return $elt unless @_;
+
+    # if a gi is passed then use it
+    my $gi= shift;
+    $elt->set_gi( $gi);
+
+
+    my $atts= ref $_[0] eq 'HASH' ? shift : undef;
+
+    if( $gi eq PCDATA)
+      { $elt->{pcdata}=  shift; }
+    elsif( $gi eq ENT)
+      { $elt->{ent}=  shift; }
+    elsif( $gi eq CDATA)
+      { $elt->{cdata}=  shift; }
+    elsif( $gi eq COMMENT)
+      { $elt->{comment}=  shift; }
+    elsif( $gi eq PI)
+      { $elt->set_pi( shift, shift); }
+    else
+      { # the rest of the arguments are the content of the element
+        if( @_)
+          { $elt->set_content( @_); }
+        else
+          { $elt->{empty}=  1;    }
+      }
+
+    if( $atts)
+      { # the attribute hash can be used to pass the asis status 
+        if( defined $atts->{'#ASIS'})  { $elt->set_asis(  $atts->{'#ASIS'} ); delete $atts->{'#ASIS'};  }
+        if( defined $atts->{'#EMPTY'}) { $elt->{empty}=  $atts->{'#EMPTY'}; delete $atts->{'#EMPTY'}; }
+        $elt->set_atts( $atts) if( keys %$atts);
+      }
+
+    return $elt;
+  }
+
+# this function creates an XM:::Twig::Elt from a string
+# it is quite clumsy at the moment, as it just creates a
+# new twig then returns its root
+# there might also be memory leaks there
+# additional arguments are passed to new XML::Twig
+sub parse
+  { my $class= shift;
+    my $string= shift;
+    my %args= @_;
+    my $t= XML::Twig->new(%args);
+    $t->parse( $string);
+    my $elt= $t->root;
+    # clean-up the node 
+    delete $elt->{twig};         # get rid of the twig data
+    delete $elt->{twig_current}; # better get rid of this too
+    return $elt;
+  }
+    
+
+sub set_gi 
+  { my ($elt, $gi)= @_;
+    unless( defined $XML::Twig::gi2index{$gi})
+      { # new gi, create entries in %gi2index and @index2gi
+        push  @XML::Twig::index2gi, $gi;
+        $XML::Twig::gi2index{$gi}= $#XML::Twig::index2gi;
+      }
+    $elt->{gi}= $XML::Twig::gi2index{$gi};
+    return $elt; 
+  }
+
+sub gi  { return $XML::Twig::index2gi[$_[0]->{gi}]; }
+
+sub local_name 
+  { my $elt= shift;
+    return _local_name( $XML::Twig::index2gi[$elt->{'gi'}]);
+  }
+
+sub ns_prefix
+  { my $elt= shift;
+    return _ns_prefix( $XML::Twig::index2gi[$elt->{'gi'}]);
+  }
+
+# namespace prefix for any qname (can be used for elements or attributes)
+sub _ns_prefix
+  { my $qname= shift;
+    if( $qname=~ m{^([^:]*):})
+      { return $1; }
+    else
+      { return( ''); } # should it be '' ?
+  }
+
+# local name for any qname (can be used for elements or attributes)
+sub _local_name
+  { my $qname= shift;
+    (my $local= $qname)=~ s{^[^:]*:}{};
+    return $local;
+  }
+
+BEGIN 
+  { my %DEFAULT_NS= ( xml   => "http://www.w3.org/XML/1998/namespace",
+                      xmlns => "http://www.w3.org/2000/xmlns/",
+                    );
+ 
+    #sub get_namespace
+    sub namespace
+      { my $elt= shift;
+        my $prefix= defined $_[0] ? shift() : $elt->ns_prefix;
+        my $ns_att= $prefix ? "xmlns:$prefix" : "xmlns";
+        my $expanded= $DEFAULT_NS{$prefix} || $elt->inherit_att( $ns_att) || '';
+        return $expanded;
+      }
+  }
+
+
+# return #ELT for an element and #PCDATA... for others
+sub get_type
+  { my $gi_nb= $_[0]->{gi}; # the number, not the string
+    return ELT if( $gi_nb >= $XML::Twig::SPECIAL_GI);
+    return $_[0]->gi;
+  }
+
+# return the gi if it's a "real" element, 0 otherwise
+sub is_elt
+  { return $_[0]->gi if(  $_[0]->{gi} >=  $XML::Twig::SPECIAL_GI);
+    return 0;
+  }
+
+
+sub is_pcdata
+  { my $elt= shift;
+    return (exists $elt->{'pcdata'});
+  }
+
+sub is_cdata
+  { my $elt= shift;
+    return (exists $elt->{'cdata'});
+  }
+
+sub is_pi
+  { my $elt= shift;
+    return (exists $elt->{'target'});
+  }
+
+sub is_comment
+  { my $elt= shift;
+    return (exists $elt->{'comment'});
+  }
+
+sub is_ent
+  { my $elt= shift;
+    return (exists $elt->{ent} || $elt->{ent_name});
+  }
+
+
+sub is_text
+  { my $elt= shift;
+    return (exists( $elt->{'pcdata'}) || (exists $elt->{'cdata'}));
+  }
+
+sub is_empty
+  { return $_[0]->{empty} || 0; }
+
+sub set_empty
+  { $_[0]->{empty}= defined( $_[1]) ? $_[1] : 1; return $_[0]; }
+
+sub set_not_empty
+  { delete $_[0]->{empty} if( ($_[0]->{'empty'} || 0)); return $_[0]; }
+
+
+sub set_asis
+  { my $elt=shift;
+
+    foreach my $descendant ($elt, $elt->_descendants )
+      { $descendant->{asis}= 1;
+        if( (exists $descendant->{'cdata'}))
+          { $descendant->set_gi( PCDATA);
+            $descendant->{pcdata}=  $descendant->{cdata};
+          }
+
+      }
+    return $elt;
+  }
+
+sub set_not_asis
+  { my $elt=shift;
+    foreach my $descendant ($elt, $elt->descendants)
+      { delete $descendant->{asis} if $descendant->{asis};}
+    return $elt;
+  }
+
+sub is_asis
+  { return $_[0]->{asis}; }
+
+sub closed 
+  { my $elt= shift;
+    my $t= $elt->twig || return;
+    my $curr_elt= $t->{twig_current};
+    return unless( $curr_elt);
+    return $curr_elt->in( $elt);
+  }
+
+sub set_pcdata 
+  { delete $_[0]->{empty};
+    $_[0]->{'pcdata'}= $_[1];
+    return $_[0]; 
+  }
+
+sub append_pcdata
+  { delete $_[0]->{empty};
+    $_[0]->{'pcdata'}.= $_[1];
+    return $_[0]; 
+  }
+
+sub pcdata        { return $_[0]->{pcdata}; }
+
+
+sub append_extra_data 
+  {  $_[0]->{extra_data}.= $_[1];
+     return $_[0]; 
+  }
+sub set_extra_data 
+  { $_[0]->{extra_data}= $_[1];
+    return $_[0]; 
+  }
+sub extra_data { return $_[0]->{extra_data}; }
+
+sub set_target 
+  { $_[0]->{'target'}= $_[1];
+    return $_[0]; 
+     
+  }
+sub target { return $_[0]->{target}; }
+
+sub set_data 
+  { $_[0]->{'data'}= $_[1]; 
+    return $_[0];
+  }
+sub data { return $_[0]->{data}; }
+
+sub set_pi
+  { $_[0]->{target}=  $_[1];
+    $_[0]->{data}=  $_[2];
+    return $_[0]; 
+  }
+
+sub pi_string { return PI_START . $_[0]->{target} . " " . $_[0]->{data} . PI_END; }
+
+sub set_comment    { $_[0]->{comment}= $_[1]; return $_[0]; }
+sub comment        { return $_[0]->{comment}; }
+sub comment_string { return COMMENT_START . $_[0]->{comment} . COMMENT_END; }
+
+sub set_ent  { $_[0]->{ent}= $_[1]; return $_[0]; }
+sub ent      { return $_[0]->{ent}; }
+sub ent_name { return substr( $_[0]->{ent}, 1, -1);}
+
+sub set_cdata 
+  { delete $_[0]->{empty};
+    $_[0]->{'cdata'}= $_[1]; 
+    return $_[0];
+  }
+
+sub append_cdata
+  { $_[0]->{'cdata'}.= $_[1]; 
+    return $_[0];
+  }
+sub cdata { return $_[0]->{'cdata'}; }
+sub cdata_string
+  { my $cdata= $_[0]->{cdata};
+    return defined $cdata ? CDATA_START . $cdata . CDATA_END : '';
+  }
+
+#start-extract twig_node
+sub contains_only_text
+  { my $elt= shift;
+    return 0 unless $elt->is_elt;
+    foreach my $child ($elt->children)
+      { return 0 if $child->is_elt; }
+    return $elt;
+  } 
+  
+sub contains_only
+  { my( $elt, $exp)= @_;
+    my @children= $elt->children;
+    foreach my $child (@children)
+      { return 0 unless $child->is( $exp); }
+    return @children;
+  } 
+
+sub contains_a_single
+  { my( $elt, $exp)= @_;
+    my $child= $elt->{first_child} or return 0;
+    return 0 unless $child->matches( $exp);
+    return 0 if( $child->{next_sibling});
+    return $child;
+  } 
+
+
+
+sub root 
+  { my $elt= shift;
+    while( $elt->{parent}) { $elt= $elt->{parent}; }
+    return $elt;
+  }
+#end-extract twig_node
+
+sub twig 
+  { my $elt= shift;
+    my $root= $elt->root;
+    return $root->{twig};
+  }
+
+
+#start-extract twig_node
+
+# returns undef or the element, depending on whether $elt passes $cond
+# $cond can be
+# - empty: the element passes the condition
+# - ELT ('#ELT'): the element passes the condition if it is a "real" element
+# - TEXT ('#TEXT'): the element passes if it is a CDATA or PCDATA element
+# - a string with an XPath condition (only a subset of XPath is actually
+#   supported).
+# - a regexp: the element passes if its gi matches the regexp
+# - a code ref: the element passes if the code, applied on the element,
+#   returns true
+
+my %cond_cache; # expression => coderef
+
+sub reset_cond_cache { %cond_cache=(); }
+
+{ 
+   sub _install_cond
+    { my $cond= shift;
+      my $sub;
+      my $test;
+
+      my $original_cond= $cond;
+
+      my $not= ($cond=~ s{^\s*!}{}) ? '!' : '';
+
+      if( ref $cond eq 'CODE') { return $cond; }
+    
+      if( ref $cond eq 'Regexp')
+        { $test = qq{(\$_[0]->gi=~ /$cond/)}; }
+      else
+        { # the condition is a string
+          if( $cond eq ELT)     
+            { $test = qq{\$_[0]->is_elt}; }
+          elsif( $cond eq TEXT) 
+            { $test = qq{\$_[0]->is_text}; }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)\s*$}o)                  
+            { # gi
+              if( $1 ne '*')
+                { # 2 options, depending on whether the gi exists in gi2index
+                  # start optimization
+                  my $gi= $XML::Twig::gi2index{$1};
+                  if( $gi)
+                    { # the gi exists, use its index as a faster shortcut
+                      $test = qq{ \$_[0]->{gi} eq "$XML::Twig::gi2index{$1}"};
+                    }
+                  else
+                  # end optimization
+                    { # it does not exist (but might be created later), compare the strings
+                      $test = qq{ \$_[0]->gi eq "$1"}; 
+                    }
+                }
+              else
+                { $test = qq{ (1) } }
+            }
+          elsif( $cond=~ m{^\s*($REG_REGEXP)\s*$}o)
+            { # /regexp/
+              $test = qq{ \$_[0]->gi=~ $1 }; 
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*\[\s*(\!\s*)?\@($REG_NAME)\s*\]\s*$}o)
+            { # gi[@att]
+              my( $gi, $not, $att)= ($1, $2, $3);
+              $not||='';
+              if( $gi && ($gi ne '*'))
+                { $test = qq{    (\$_[0]->gi eq "$gi") 
+                              && $not(defined \$_[0]->{'att'}->{"$att"})
+                            };
+                }
+              else
+                { $test = qq{ $not (defined \$_[0]->{'att'}->{"$att"})}; }
+             }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*  # $1
+                           \[\s*(-?)\s*(\d+)\s*\] #   [$2]
+                           \s*$}xo
+               )
+            { my( $gi, $neg, $index)= ($1, $2, $3);
+              my $siblings= $neg ? q{$_[0]->_next_siblings} : q{$_[0]->_prev_siblings};
+              if( $gi && ($gi ne '*')) 
+                { $test= qq{((\$_[0]->gi eq "$gi") && (scalar( grep { \$_->gi eq "$gi" } $siblings) + 1 == $index))}; }
+              else
+                { $test= qq{(scalar( $siblings) + 1 == $index)}; }
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*  # $1
+                           \[\@($REG_NAME)        #   [@$2
+                           \s*($REG_OP)\s*        #        = (or other op) $3
+                          ($REG_VALUE)            #          "$4" or '$4'
+                          \s*\]\s*$}xo)           #                       ]
+            { # gi[@att="val"]
+              my( $gi, $att, $op, $string)= ($1, $2, _op( $3), $4);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{    (\$_[0]->gi eq "$gi") 
+                              && (defined \$_[0]->{'att'}->{"$att"}) 
+                              && ( \$_[0]->{'att'}->{"$att"} $op $string) 
+                            }; 
+                }
+              else
+                { $test = qq{    (defined \$_[0]->{'att'}->{"$att"}) 
+                              && ( \$_[0]->{'att'}->{"$att"} $op $string) 
+                            };
+                }
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*  # $1
+                           \[\@($REG_NAME)        #   [@$2
+                           \s*($REG_OP)\s*        #        = (or other op) $3
+                           ($REG_VALUE)           #          "$4" or '$4'
+                           \s*(and|or)\s*         #              and or or ($5)
+                           \@($REG_NAME)          #                  @$6
+                           \s*($REG_OP)\s*        #                      = (or other op) $7
+                           ($REG_VALUE)           #                        "$8" or '$8'
+                           \s*\]\s*$}xo)          #                       ]
+            { # gi[@att="val"]
+              my( $gi, $att1, $op1, $string1, $connector, $att2, $op2, $string2)= ($1, $2, _op( $3), $4, $5, $6, _op( $7), $8);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{ (   (\$_[0]->gi eq "$gi") 
+                                and ( ( (defined \$_[0]->{'att'}->{"$att1"})  and( \$_[0]->{'att'}->{"$att1"} $op1 $string1) )
+                                        $connector 
+                                      ( (defined \$_[0]->{'att'}->{"$att2"})  and( \$_[0]->{'att'}->{"$att2"} $op2 $string2) )
+                                    )
+                              )
+                            }; 
+                }
+              else
+                { $test = qq{ ( ( (defined \$_[0]->{'att'}->{"$att1"}) and ( \$_[0]->{'att'}->{"$att1"} $op1 $string1) )
+                               $connector 
+                                ( (defined \$_[0]->{'att'}->{"$att2"})  and( \$_[0]->{'att'}->{"$att2"} $op2 $string2) )
+                              )
+                            };
+                }
+            }
+          elsif( $cond=~ m{^\s*\.([\w-]+)\s*$}o)
+            { # .class
+              my $class= $1;
+              $test = qq{(\$_[0]->in_class( "$class")) }; 
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*  # $1
+                           \[\@($REG_NAME)        #   [@$2
+                           \s*($REG_MATCH)\s*     #        =~ or !~ ($3)
+                           ($REG_REGEXP)          #           /$4/
+                           \s*\]\s*$}xo)          #                ]
+            { # gi[@att=~ /regexp/] or gi[@att!~/regexp/]
+              my( $gi, $att, $match, $regexp)= ($1, $2, $3, $4);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{    (\$_[0]->gi eq "$gi") 
+                              && (   defined \$_[0]->{'att'}->{"$att"}) 
+                              && ( \$_[0]->{'att'}->{"$att"} $match $regexp)
+                            }; 
+                }
+              else
+                { # *[@att=~/regexp/ or *[@att!~/regexp/
+                  $test = qq{(    defined \$_[0]->{'att'}->{"$att"}) 
+                               && ( \$_[0]->{'att'}->{"$att"} $match $regexp)
+                            };
+                }
+            }
+          elsif( $cond=~ m{^\s*\@($REG_NAME)\s*$}o)
+            { # @att (or !@att)
+              my( $att)= ($1);
+              $test = qq{ (defined \$_[0]->{'att'}->{"$att"})}; 
+            }
+          elsif( $cond=~ m{^\s*                   
+                           \@($REG_NAME)        #   @$1
+                           \s*($REG_OP)\s*      #       = (or other op) $2
+                           ($REG_VALUE)         #         "$3" or '$3'
+                           \s*$}xo)                                 
+            { # @att="val"
+              my( $att, $op, $string)= ( $1, _op( $2), $3);
+              $test = qq{    (defined \$_[0]->{'att'}->{"$att"}) 
+                          && ( \$_[0]->{'att'}->{"$att"} $op $string) 
+                        };
+             }
+          elsif( $cond=~ m{^\s*                   
+                           \@($REG_NAME)        #   @$1
+                           \s*($REG_OP)\s*      #       = (or other op) $2
+                           ($REG_VALUE)         #         "$3" or '$3'
+                           \s*(and|or)\s*       #              and or or ($4)
+                           \@($REG_NAME)        #   @$5
+                           \s*($REG_OP)\s*      #       = (or other op) $6
+                           ($REG_VALUE)         #         "$7" or '$7'
+                           \s*$}xo)                                 
+            { # @att="val"
+              my( $att1, $op1, $string1, $connector, $att2, $op2, $string2 )= ( $1, _op( $2), $3, $4, $5, _op( $6), $7);
+              $test = qq{ ( (    (defined \$_[0]->{'att'}->{"$att1"}) 
+                              && ( \$_[0]->{'att'}->{"$att1"} $op1 $string1) 
+                            )
+                            $connector
+                            (    (defined \$_[0]->{'att'}->{"$att2"}) 
+                              && ( \$_[0]->{'att'}->{"$att2"} $op2 $string2) 
+                            )
+                          )
+                        };
+             }
+          elsif( $cond=~ m{^\s*
+                           \@($REG_NAME)        #   [@$1
+                           \s*(=~|!~)\s*        #        =~ or !~ ($2)
+                          ($REG_REGEXP)         #           /$3/
+                          \s*\s*$}xo)           #                ]
+            { # @att=~ /regexp/ or @att!~/regexp/
+              my( $att, $match, $regexp)= ( $2, $3, $4);
+              $test = qq{(    defined \$_[0]->{'att'}->{"$att"}) 
+                           && ( \$_[0]->{'att'}->{"$att"} $match $regexp)
+                        };
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*            # $1
+                           \[\s*(?:text|string)(?:\(\s*\))? #   [string()
+                           \s*($REG_OP)\s*                  #            = or other op ($2)
+                           ($REG_VALUE)                     #              "$3" or '$3'
+                           \s*\]\s*$}xo)                    #                          ]
+            { # gi[string()= "val"]
+              my ($gi, $op, $text)= ($1, _op( $2), $3);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{(\$_[0]->gi eq "$gi") && ( \$_[0]->text eq $text)}; }
+              else
+                { $test = qq{ \$_[0]->text eq $text }; }
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*            # $1
+                           \[\s*(?:text|string)(?:\(\s*\))? #   [string()
+                           \s*($REG_MATCH)\s*               #             =~ or !~ ($2)
+                           ($REG_REGEXP)                    #               /$3/
+                           \s*\]\s*$}xo)                    #                   ]
+            { # gi[string()=~ /regexp/]
+              my( $gi, $match, $regexp)= ($1, $2, $3);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{(\$_[0]->gi eq "$gi") && ( \$_[0]->text $match $regexp) }; }
+              else
+                { $test = qq{ \$_[0]->text $match $regexp }; }
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*            # $1
+                           \[\s*(?:text|string)\s*\(\s*     #   [string(
+                           ($REG_NAME)\s*\)                 #            $2)
+                           \s*($REG_OP)\s*                  #                = or other op $3
+                           ($REG_VALUE)                     #                  "$4" or '$4'
+                           \s*\]\s*$}xo)                    #                      ]
+            { # gi[string(gi2)= "text"]
+              my ($gi, $gi2, $op, $text)= ($1, $2, _op($3), $4);
+              $text=~ s/([{}])/\\$1/g;
+              if( $gi && ($gi ne '*'))
+                { $test = qq{    (\$_[0]->gi eq "$gi") 
+                              && ( \$_[0]->first_child( qq{$gi2\[text() $op $text]}))
+                            };
+                }
+              else
+                { $test = qq{ \$_[0]->first_child(qq{$gi2\[text() $op $text]}) } ; }
+            }
+          elsif( $cond=~ m{^\s*($REG_NAME_W)?\s*     # $1
+                           \[\s*(?:text|string)\(\s* #   [string(
+                           ($REG_NAME)\s*\)          #         $2)
+                           \s*(=~|!~)\s*             #            =~ or !~ ($3)
+                           ($REG_REGEXP)             #              /$4/
+                           \s*\]\s*$}xo)             #                  ]
+            { # gi[string(gi2)=~ /regexp/]
+              my( $gi, $gi2, $match, $regexp)= ($1, $2, $3, $4);
+              if( $gi && ($gi ne '*'))
+                { $test = qq{   (\$_[0]->gi eq "$gi") 
+                             && ( \$_[0]->field( "$gi2") $match $regexp)
+                            };
+                }
+              else
+                { $test = qq{\$_[0]->field( "$gi2") $match $regexp}; }
+            }
+          else
+            { croak "wrong condition '$original_cond'"; }
+        }
+
+      my $s= eval "sub { return \$_[0] if( $not($test)) }";
+      if( $@) 
+        { croak "wrong navigation condition $original_cond ($@);" }
+      return $s;
+    }
+
+  sub _op
+    { my $op= shift;
+      if(    $op eq '=')  { $op= 'eq'; }
+      elsif( $op eq '!=') { $op= 'ne'; }
+      return $op;
+    }
+ 
+  sub passes
+    { my( $elt, $cond)= @_;
+      return $elt unless $cond;
+      my $sub= ($cond_cache{$cond} ||= _install_cond( $cond));
+      return $sub->( $elt);
+    }
+}
+# end-extract twig_nodes
+
+sub set_parent 
+  { $_[0]->{parent}= $_[1];
+    weaken( $_[0]->{parent}) if( $XML::Twig::weakrefs);
+  }
+
+#start-extract twig_node
+sub parent
+  { my $elt= shift;
+    my $cond= shift || return $elt->{parent};
+    do { $elt= $elt->{parent} || return; } until (!$elt || $elt->passes( $cond));
+    return $elt;
+  }
+#end-extract twig_node
+
+sub set_first_child 
+  { delete $_[0]->{empty};
+    $_[0]->{'first_child'}= $_[1]; 
+  }
+
+#start-extract twig_node
+sub first_child
+  { my $elt= shift;
+    my $cond= shift || return $elt->{first_child};
+    my $child= $elt->{first_child};
+    my $test_cond= ($cond_cache{$cond} ||= _install_cond( $cond));
+    while( $child && !$test_cond->( $child)) 
+       { $child= $child->{next_sibling}; }
+    return $child;
+  }
+#end-extract twig_node
+  
+sub _first_child   { return $_[0]->{first_child};  }
+sub _last_child    { return $_[0]->{last_child};   }
+sub _next_sibling  { return $_[0]->{next_sibling}; }
+sub _prev_sibling  { return $_[0]->{prev_sibling}; }
+sub _parent        { return $_[0]->{parent};       }
+sub _next_siblings { my $elt= shift; my @siblings; while( $elt= $elt->{next_sibling}) { push @siblings, $elt; } return @siblings; }
+sub _prev_siblings { my $elt= shift; my @siblings; while( $elt= $elt->{prev_sibling}) { push @siblings, $elt; } return @siblings; }
+
+# sets a field
+# arguments $record, $cond, @content
+sub set_field
+  { my $record = shift;
+    my $cond = shift;
+    my $child= $record->first_child( $cond);
+    if( $child)
+      { $child->set_content( @_); }
+    else
+      { if( $cond=~ m{^\s*($REG_NAME)})
+          { my $gi= $1;
+            $child= $record->insert_new_elt( last_child => $gi, @_); 
+          }
+        else
+          { croak "can't create a field name from $cond"; }
+      } 
+    return $child;
+  }
+
+sub set_last_child 
+  { delete $_[0]->{empty};
+    $_[0]->{'last_child'}= $_[1];
+    weaken( $_[0]->{'last_child'}) if( $XML::Twig::weakrefs);
+  }
+
+#start-extract twig_node
+sub last_child
+  { my $elt= shift;
+    my $cond= shift || return $elt->{last_child};
+    my $test_cond= ($cond_cache{$cond} ||= _install_cond( $cond));
+    my $child= $elt->{last_child};
+    while( $child && !$test_cond->( $child) )
+      { $child= $child->{prev_sibling}; }
+    return $child
+  }
+#end-extract twig_node
+
+
+sub set_prev_sibling 
+  { $_[0]->{'prev_sibling'}= $_[1]; 
+    weaken( $_[0]->{'prev_sibling'}) if( $XML::Twig::weakrefs); 
+  }
+
+#start-extract twig_node
+sub prev_sibling
+  { my $elt= shift;
+    my $cond= shift || return $elt->{prev_sibling};
+    my $test_cond= ($cond_cache{$cond} ||= _install_cond( $cond));
+    my $sibling= $elt->{prev_sibling};
+    while( $sibling && !$test_cond->( $sibling) )
+          { $sibling= $sibling->{prev_sibling}; }
+    return $sibling;
+  }
+#end-extract twig_node
+
+sub set_next_sibling { $_[0]->{'next_sibling'}= $_[1]; }
+
+#start-extract twig_node
+sub next_sibling
+  { my $elt= shift;
+    my $cond= shift || return $elt->{next_sibling};
+    my $test_cond= ($cond_cache{$cond} ||= _install_cond( $cond));
+    my $sibling= $elt->{next_sibling};
+    while( $sibling && !$test_cond->( $sibling) )
+          { $sibling= $sibling->{next_sibling}; }
+    return $sibling;
+  }
+
+# methods dealing with the class attribute, convenient if you work with xhtml
+sub class     { my( $elt)= @_; return $elt->{'att'}->{'class'}; }
+sub set_class { my( $elt, $class)= @_; $elt->set_att( class => $class); }
+
+# adds a class to an element
+sub add_to_class
+  { my( $elt, $new_class)= @_;
+    return $elt unless $new_class;
+    my $class= $elt->class;
+    my %class= $class ? map { $_ => 1 } split /\s+/, $class : ();
+    $class{$new_class}= 1;
+    $elt->set_class( join( ' ', sort keys %class));
+  }
+
+sub att_to_class      { my( $elt, $att)= @_; $elt->set_class( $elt->{'att'}->{$att}); }
+sub add_att_to_class  { my( $elt, $att)= @_; $elt->add_to_class( $elt->{'att'}->{$att}); }
+sub move_att_to_class { my( $elt, $att)= @_; $elt->add_to_class( $elt->{'att'}->{$att});
+                        $elt->del_att( $att); 
+                      }
+sub tag_to_class      { my( $elt)= @_; $elt->set_class( $elt->tag);    }
+sub add_tag_to_class  { my( $elt)= @_; $elt->add_to_class( $elt->tag); }
+sub set_tag_class     { my( $elt, $new_tag)= @_; $elt->add_tag_to_class; $elt->set_tag( $new_tag); }
+
+sub in_class          
+  { my( $elt, $class)= @_;
+    my $elt_class= $elt->class;
+    return unless( defined $elt_class);
+    return $elt->class=~ m{(?:^|\s)\Q$class\E(?:\s|$)} ? $elt : 0;
+  }
+
+#end-extract twig_node
+
+# get or set all attributes
+sub set_atts 
+  { my $elt= shift;
+    my %atts;
+    tie %atts, 'Tie::IxHash' if( keep_atts_order());
+    if( isa( $_[0] || '', 'HASH'))
+      { while( my( $att, $value)= each %{$_[0]}) { $atts{$att}= $value; } 
+      }
+    else
+      { %atts= @_; }
+    $elt->{'att'}= \%atts;
+    return $elt;
+  }
+
+sub atts      { return $_[0]->{att};           }
+sub att_names { return sort keys %{$_[0]->{att}};   }
+sub del_atts  { $_[0]->{att}={}; return $_[0]; }
+
+# get or set a single attribute (set works for several atts)
+sub set_att 
+  { my $elt= shift;
+    
+    unless( $elt->{att})
+      { $elt->{att}={};
+        tie %{$elt->{att}}, 'Tie::IxHash' if( keep_atts_order());
+      }
+
+    while(@_) { my( $att, $val)= (shift, shift);
+                $elt->{att}->{$att}= $val;
+          }
+    return $elt;
+  }
+sub att { return $_[0]->{att}->{$_[1]}; }
+sub del_att 
+  { my $elt= shift;
+    while( @_) { delete $elt->{'att'}->{shift()}; }
+    return $elt;
+  }
+
+# delete an attribute from all descendants of an element
+sub strip_att
+  { my( $elt, $att)= @_;
+    $_->del_att( $att) foreach ($elt->descendants_or_self( qq{*[\@$att]}));
+  }
+
+sub change_att_name
+  { my( $elt, $old_name, $new_name)= @_;
+    my $value= $elt->{'att'}->{$old_name};
+    return $elt unless( defined $value);
+    $elt->del_att( $old_name)
+        ->set_att( $new_name => $value);
+    return $elt;
+  }
+
+sub set_twig_current { $_[0]->{twig_current}=1; }
+sub del_twig_current { delete $_[0]->{twig_current}; }
+
+
+# get or set the id attribute
+sub set_id 
+  { my( $elt, $id)= @_;
+    $elt->set_att($ID, $id); 
+    my $t= $elt->twig || return;
+    $elt->twig->{twig_id_list}->{$id}= $elt;
+    weaken(  $elt->twig->{twig_id_list}->{$id}) if( $XML::Twig::weakrefs);
+  }
+
+sub id { return $_[0]->{att}->{$ID}; }
+
+# methods used to add ids to elements that don't have one
+BEGIN 
+{ my $id_nb   = "0001";
+  my $id_seed = "twig_id_";
+
+  sub set_id_seed
+    { $id_seed= $_[1]; $id_nb=1; }
+
+  sub add_id
+    { my $elt= shift;
+      $elt->set_id( $id_seed . $id_nb++) unless( $elt->{'att'}->{$ID});
+    }
+
+}
+
+
+
+# delete the id attribute and remove the element from the id list
+sub del_id 
+  { my $elt= shift;
+    my $id= $elt->{'att'}->{$ID} or return;
+    my $t= $elt->twig && delete $elt->twig->{twig_id_list}->{$id};
+    delete $elt->{'att'}->{$ID}; 
+  }
+
+# return the list of children
+#start-extract twig_node
+sub children
+  { my $elt= shift;
+    my @children;
+    my $child= $elt->first_child( @_);
+    while( $child) 
+      { push @children, $child;
+        $child= $child->next_sibling( @_);
+      } 
+    return @children;
+  }
+
+sub _children
+  { my $elt= shift;
+    my @children=();
+    my $child= $elt->_first_child();
+    while( $child) 
+      { push @children, $child;
+        $child= $child->{next_sibling};
+      } 
+    return @children;
+  }
+
+sub children_copy
+  { my $elt= shift;
+    my @children;
+    my $child= $elt->first_child( @_);
+    while( $child) 
+      { push @children, $child->copy;
+        $child= $child->next_sibling( @_);
+      } 
+    return @children;
+  }
+
+
+sub children_count
+  { my $elt= shift;
+    my $cond= shift;
+    my $count=0;
+    my $child= $elt->{first_child};
+    while( $child)
+      { $count++ if( $child->passes( $cond)); 
+        $child= $child->{next_sibling};
+      }
+    return $count;
+  }
+
+sub children_text
+  { my $elt= shift;
+    return map { $_->text} $elt->children( @_);
+  }
+
+sub all_children_are
+  { my( $parent, $cond)= @_;
+    foreach my $child ($parent->children)
+      { return 0 unless( $child->passes( $cond)); }
+    return 1;
+  }
+
+
+sub ancestors
+  { my( $elt, $cond)= @_;
+    my @ancestors;
+    while( $elt->{parent})
+      { $elt= $elt->{parent};
+        push @ancestors, $elt
+          if( $elt->passes( $cond));
+      }
+    return @ancestors;
+  }
+
+sub ancestors_or_self
+  { my( $elt, $cond)= @_;
+    my @ancestors;
+    while( $elt)
+      { push @ancestors, $elt
+          if( $elt->passes( $cond));
+        $elt= $elt->{parent};
+      }
+    return @ancestors;
+  }
+
+
+sub _ancestors
+  { my( $elt, $include_self)= @_;
+    my @ancestors= $include_self ? ($elt) : ();
+    while( $elt= $elt->{parent})
+      { push @ancestors, $elt;
+      }
+    return @ancestors;
+  }
+
+
+sub inherit_att
+  { my $elt= shift;
+    my $att= shift;
+    my %tags= map { ($_, 1) } @_;
+
+    do 
+      { if(   (defined $elt->{'att'}->{$att})
+           && ( !%tags || $tags{$XML::Twig::index2gi[$elt->{'gi'}]})
+          )
+          { return $elt->{'att'}->{$att}; }
+      } while( $elt= $elt->{parent});
+    return undef;
+  }
+
+
+sub current_ns_prefixes
+  { my $elt= shift;
+    my %prefix;
+    $prefix{''}=1 if( $elt->namespace( ''));
+    while( $elt)
+      { my @ns= grep { !m{^xml} } map { m{^([^:]+):} } ($XML::Twig::index2gi[$elt->{'gi'}], $elt->att_names);
+        $prefix{$_}=1 foreach (@ns);
+        $elt= $elt->{parent};
+      }
+
+    return sort keys %prefix;
+  }
+
+# kinda counter-intuitive actually:
+# the next element is found by looking for the next open tag after from the
+# current one, which is the first child, if it exists, or the next sibling
+# or the first next sibling of an ancestor
+# optional arguments are: 
+#   - $subtree_root: a reference to an element, when the next element is not 
+#                    within $subtree_root anymore then next_elt returns undef
+#   - $gi: a gi, next_elt returns the next element of this gi
+                 
+sub next_elt
+  { my $elt= shift;
+    my $subtree_root= 0;
+    $subtree_root= shift if( defined $_[0] and (isa( $_[0], 'XML::Twig::Elt')));
+    my $cond= shift;
+    my $next_elt;
+
+    my $ind;                                                              # optimization
+    my $test_cond;
+    if( $cond)                                                            # optimization
+      { unless( defined( $ind= $XML::Twig::gi2index{$cond}) )             # optimization
+          { $test_cond= ($cond_cache{$cond} ||= _install_cond( $cond)); } # optimization
+      }                                                                   # optimization
+    
+    do
+      { if( $next_elt= $elt->{first_child})
+          { # simplest case: the elt has a child
+          }
+         elsif( $next_elt= $elt->{next_sibling}) 
+          { # no child but a next sibling (just check we stay within the subtree)
+          
+            # case where elt is subtree_root, is empty and has a sibling
+            return undef if( $subtree_root && ($elt == $subtree_root));
+            
+          }
+        else
+          { # case where the element has no child and no next sibling:
+            # get the first next sibling of an ancestor, checking subtree_root 
+          
+            # case where elt is subtree_root, is empty and has no sibling
+            return undef if( $subtree_root && ($elt == $subtree_root));
+             
+            # backtrack until we find a parent with a next sibling
+            $next_elt= $elt->{parent} || return undef;
+            until( $next_elt->{next_sibling})
+              { return undef if( $subtree_root && ($subtree_root == $next_elt));
+                $next_elt= $next_elt->{parent} || return undef;
+              }
+            return undef if( $subtree_root && ($subtree_root == $next_elt)); 
+            $next_elt= $next_elt->{next_sibling};   
+          }  
+      $elt= $next_elt;                   # just in case we need to loop
+    } until(    ! defined $elt 
+             || ! defined $cond 
+         || (defined $ind       && ($elt->{gi} eq $ind))   # optimization
+         || (defined $test_cond && ($test_cond->( $elt)))
+               );
+    
+      return $elt;
+      }
+
+# counter-intuitive too:
+# the previous element is found by looking
+# for the first open tag backwards from the current one
+# it's the last descendant of the previous sibling 
+# if it exists, otherwise it's simply the parent
+sub prev_elt
+  { my $elt= shift;
+    my $cond= shift;
+    # get prev elt
+    my $prev_elt;
+    do
+      { if( $prev_elt= $elt->{prev_sibling})
+          { while( $prev_elt->{last_child})
+              { $prev_elt= $prev_elt->{last_child}; }
+          }
+        else
+          { $prev_elt= $elt->{parent} || return; }
+        $elt= $prev_elt;     # in case we need to loop 
+      } until( $elt->passes( $cond));
+
+    return $prev_elt;
+  }
+
+
+sub next_n_elt
+  { my $elt= shift;
+    my $offset= shift;
+    foreach (1..$offset)
+      { $elt= $elt->next_elt( @_) || return undef; }
+    return $elt;
+  }
+
+# checks whether $elt is included in $ancestor, returns 1 in that case
+sub in
+  { my ($elt, $ancestor)= @_;
+    if( isa( $ancestor, 'XML::Twig::Elt'))
+      { # element
+        while( $elt= $elt->{parent}) { return $elt if( $elt ==  $ancestor); } 
+      }
+    else
+      { # condition
+        while( $elt= $elt->{parent}) { return $elt if( $elt->matches( $ancestor)); } 
+      }
+    return 0;           
+  }
+
+sub first_child_text  
+  { my $elt= shift;
+    my $dest=$elt->first_child(@_) or return '';
+    return $dest->text;
+  }
+  
+sub first_child_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->first_child(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub first_child_matches
+  { my $elt= shift;
+    my $dest= $elt->{first_child} or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub last_child_text  
+  { my $elt= shift;
+    my $dest=$elt->last_child(@_) or return '';
+    return $dest->text;
+  }
+  
+sub last_child_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->last_child(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub last_child_matches
+  { my $elt= shift;
+    my $dest= $elt->{last_child} or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub child_text
+  { my $elt= shift;
+    my $dest=$elt->child(@_) or return '';
+    return $dest->text;
+  }
+  
+sub child_trimmed_text
+  { my $elt= shift;
+    my $dest=$elt->child(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub child_matches
+  { my $elt= shift;
+    my $nb= shift;
+    my $dest= $elt->child( $nb) or return undef;
+    return $dest->passes( @_);
+  }
+
+sub prev_sibling_text  
+  { my $elt= shift;
+    my $dest=$elt->prev_sibling(@_) or return '';
+    return $dest->text;
+  }
+  
+sub prev_sibling_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->prev_sibling(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub prev_sibling_matches
+  { my $elt= shift;
+    my $dest= $elt->{prev_sibling} or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub next_sibling_text  
+  { my $elt= shift;
+    my $dest=$elt->next_sibling(@_) or return '';
+    return $dest->text;
+  }
+  
+sub next_sibling_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->next_sibling(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub next_sibling_matches
+  { my $elt= shift;
+    my $dest= $elt->{next_sibling} or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub prev_elt_text  
+  { my $elt= shift;
+    my $dest=$elt->prev_elt(@_) or return '';
+    return $dest->text;
+  }
+  
+sub prev_elt_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->prev_elt(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub prev_elt_matches
+  { my $elt= shift;
+    my $dest= $elt->prev_elt or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub next_elt_text  
+  { my $elt= shift;
+    my $dest=$elt->next_elt(@_) or return '';
+    return $dest->text;
+  }
+  
+sub next_elt_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->next_elt(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub next_elt_matches
+  { my $elt= shift;
+    my $dest= $elt->next_elt or return undef;
+    return $dest->passes( @_);
+  }
+  
+sub parent_text  
+  { my $elt= shift;
+    my $dest=$elt->parent(@_) or return '';
+    return $dest->text;
+  }
+  
+sub parent_trimmed_text  
+  { my $elt= shift;
+    my $dest=$elt->parent(@_) or return '';
+    return $dest->trimmed_text;
+  }
+  
+sub parent_matches
+  { my $elt= shift;
+    my $dest= $elt->{parent} or return undef;
+    return $dest->passes( @_);
+  }
+ 
+sub is_first_child
+  { my $elt= shift;
+    my $parent= $elt->{parent} or return 0;
+    my $first_child= $parent->first_child( @_) or return 0;
+    return ($first_child == $elt) ? $elt : 0;
+  }
+ 
+sub is_last_child
+  { my $elt= shift;
+    my $parent= $elt->{parent} or return 0;
+    my $last_child= $parent->last_child( @_) or return 0;
+    return ($last_child == $elt) ? $elt : 0;
+  }
+
+# returns the depth level of the element
+# if 2 parameter are used then counts the 2cd element name in the
+# ancestors list
+sub level
+  { my( $elt, $cond)= @_;
+   
+    my $level=0;
+    my $name=shift || '';
+    while( $elt= $elt->{parent}) { $level++ if( !$cond || $elt->matches( $cond)); }
+    return $level;           
+  }
+
+# checks whether $elt has an ancestor that satisfies $cond, returns the ancestor
+sub in_context
+  { my ($elt, $cond, $level)= @_;
+    $level= -1 unless( $level) ;  # $level-- will never hit 0
+
+    while( $level)
+      { $elt= $elt->{parent} or return;
+        if( $elt->matches( $cond)) { return $elt; }
+        $level--;
+      }
+  }
+
+
+sub _descendants
+  { my( $subtree_root, $include_self)= @_;
+    my @descendants= $include_self ? ($subtree_root) : ();
+
+    my $elt= $subtree_root; 
+    my $next_elt;   
+ 
+    MAIN: while( 1)  
+      { if( $next_elt= $elt->{first_child})
+          { # simplest case: the elt has a child
+          }
+        elsif( $next_elt= $elt->{next_sibling}) 
+          { # no child but a next sibling (just check we stay within the subtree)
+          
+            # case where elt is subtree_root, is empty and has a sibling
+            last MAIN if( $elt == $subtree_root);
+          }
+        else
+          { # case where the element has no child and no next sibling:
+            # get the first next sibling of an ancestor, checking subtree_root 
+                
+            # case where elt is subtree_root, is empty and has no sibling
+            last MAIN if( $elt == $subtree_root);
+               
+            # backtrack until we find a parent with a next sibling
+            $next_elt= $elt->{parent} || last;
+            until( $next_elt->{next_sibling})
+              { last MAIN if( $subtree_root == $next_elt);
+                $next_elt= $next_elt->{parent} || last MAIN;
+              }
+            last MAIN if( $subtree_root == $next_elt); 
+            $next_elt= $next_elt->{next_sibling};   
+          }  
+        $elt= $next_elt || last MAIN;
+        push @descendants, $elt;
+      }
+    return @descendants;
+  }
+
+
+sub descendants
+  { my( $subtree_root, $cond)= @_;
+    my @descendants=(); 
+    my $elt= $subtree_root;
+    
+    # this branch is pure optimisation for speed: if $cond is a gi replace it
+    # by the index of the gi and loop here 
+    # start optimization
+    my $ind;
+    if( !$cond || ( defined ( $ind= $XML::Twig::gi2index{$cond})) )
+      {
+        my $next_elt;
+
+        while( 1)  
+          { if( $next_elt= $elt->{first_child})
+                { # simplest case: the elt has a child
+                }
+             elsif( $next_elt= $elt->{next_sibling}) 
+              { # no child but a next sibling (just check we stay within the subtree)
+           
+                # case where elt is subtree_root, is empty and has a sibling
+                last if( $subtree_root && ($elt == $subtree_root));
+              }
+            else
+              { # case where the element has no child and no next sibling:
+                # get the first next sibling of an ancestor, checking subtree_root 
+                
+                # case where elt is subtree_root, is empty and has no sibling
+                last if( $subtree_root && ($elt == $subtree_root));
+               
+                # backtrack until we find a parent with a next sibling
+                $next_elt= $elt->{parent} || last undef;
+                until( $next_elt->{next_sibling})
+                  { last if( $subtree_root && ($subtree_root == $next_elt));
+                    $next_elt= $next_elt->{parent} || last;
+                  }
+                last if( $subtree_root && ($subtree_root == $next_elt)); 
+                $next_elt= $next_elt->{next_sibling};   
+              }  
+            $elt= $next_elt || last;
+            push @descendants, $elt if( !$cond || ($elt->{gi} eq $ind));
+          }
+      }
+    else
+    # end optimization
+      { # branch for a complex condition: use the regular (slow but simple) way
+        while( $elt= $elt->next_elt( $subtree_root, $cond))
+          { push @descendants, $elt; }
+      }
+    return @descendants;
+  }
+
+ 
+sub descendants_or_self
+  { my( $elt, $cond)= @_;
+    my @descendants= $elt->passes( $cond) ? ($elt) : (); 
+    push @descendants, $elt->descendants( $cond);
+    return @descendants;
+  }
+  
+sub sibling
+  { my $elt= shift;
+    my $nb= shift;
+    if( $nb > 0)
+      { foreach( 1..$nb)
+          { $elt= $elt->next_sibling( @_) or return undef; }
+      }
+    elsif( $nb < 0)
+      { foreach( 1..(-$nb))
+          { $elt= $elt->prev_sibling( @_) or return undef; }
+      }
+    else # $nb == 0
+      { return $elt->passes( $_[0]); }
+    return $elt;
+  }
+
+sub sibling_text
+  { my $elt= sibling( @_);
+    return $elt ? $elt->text : undef;
+  }
+
+
+sub child
+  { my $elt= shift;
+    my $nb= shift;
+    if( $nb >= 0)
+      { $elt= $elt->first_child( @_) or return undef;
+        foreach( 1..$nb)
+          { $elt= $elt->next_sibling( @_) or return undef; }
+      }
+    else
+      { $elt= $elt->last_child( @_) or return undef;
+        foreach( 2..(-$nb))
+          { $elt= $elt->prev_sibling( @_) or return undef; }
+      }
+    return $elt;
+  }
+
+sub prev_siblings
+  { my $elt= shift;
+    my @siblings=();
+    while( $elt= $elt->prev_sibling( @_))
+      { unshift @siblings, $elt; }
+    return @siblings;
+  }
+
+sub pos
+  { my $elt= shift;
+    return 0 if ($_[0] && !$elt->matches( @_));
+    my $pos=1;
+    $pos++ while( $elt= $elt->prev_sibling( @_));
+    return $pos;
+  }
+
+
+sub next_siblings
+  { my $elt= shift;
+    my @siblings=();
+    while( $elt= $elt->next_sibling( @_))
+      { push @siblings, $elt; }
+    return @siblings;
+  }
+
+# used by get_xpath: parses the xpath expression and generates a sub that performs the
+# search
+sub _install_xpath
+  { my( $xpath_exp, $type)= @_;
+    my $original_exp= $xpath_exp;
+    my $sub= 'my $elt= shift; my @results;';
+    
+    # grab the root if expression starts with a /
+    if( $xpath_exp=~ s{^/}{})
+      { $sub .= '@results= ($elt->twig);'; }
+    elsif( $xpath_exp=~ s{^\./}{})
+      { $sub .= '@results= ($elt);'; }
+    else
+      { $sub .= '@results= ($elt);'; }
+
+
+    while( $xpath_exp &&
+           $xpath_exp=~s{^\s*(/?)                            
+                          # the xxx=~/regexp/ is a pain as it includes /  
+                          (\s*((\*|$REG_NAME|\.)\s*)?\[\s*(string\(\s*\)|\@$REG_NAME)\s*$REG_MATCH  
+                            \s*$REG_REGEXP\s*\]\s* 
+                          # or a regular condition, with no / excepts \/
+                          |([^\\/]|\\.)*
+                          )
+                          (/|$)}{}xo)
+
+      { my $wildcard= $1;
+        my $sub_exp= $2; 
+        
+        # grab a parent
+        if( $sub_exp eq '..')
+          { croak "error in xpath expression $original_exp" if( $wildcard);
+            $sub .= '@results= map { $_->{parent}} @results;';
+          }
+    # test the element itself
+    elsif( $sub_exp=~ m{^\.(.*)$}s)
+      { $sub .= "\@results= grep { \$_->matches( q{$1}) } \@results;" }
+        # grab children
+        elsif( $sub_exp=~ m{($REG_NAME_W)?\s*                # * or a gi    ($1)
+                            (?:
+                \[\s*                             #  [
+                  (
+                (?:string\(\s*\)|\@$REG_NAME) #    regexp condition 
+                   \s*$REG_MATCH\s*$REG_REGEXP\s*  # or
+                        |[^\]]*                       #    regular condition 
+                  )
+                            \]                                #   ]
+                             )?\s*$}xs)        
+          { my $gi= $1; 
+            if( !$1 or $1 eq '*') { $gi=''; }
+            my $cond= $2; 
+            if( $cond) { $cond=~ s{^\s*}{}; $cond=~ s{\s*$}{}; }
+            my $function;
+            
+
+            # "special" conditions, that return just one element
+            if( $cond && ($cond =~ m{^((-\s*)?\d+)$}) )
+              { my $offset= $1;
+                $offset-- if( $offset > 0);
+                $function=  $wildcard ? "next_n_elt( $offset, '$gi')" 
+                                      : "child( $offset, '$gi')";
+                $sub .= "\@results= map { \$_->$function } \@results;"
+              }
+            elsif( $cond && ($cond =~ m{^last\s*\(\s*\)$}) )
+              { croak "error in xpath expression $original_exp, cant use // and last()"
+                  if( $wildcard);
+                 $sub .= "\@results= map { \$_->last_child( '$gi') } \@results;";
+              }
+            else
+              { # go down and get the children or descendants
+                unless ( defined $gi)
+                  { if( $wildcard)
+                      { $sub .= '@results= map { $_->descendants  } @results;' }
+                    else
+                      { $sub .= '@results= map { $_->children } @results;'; }
+                  }
+                else
+                  { if( $wildcard)
+                      { $sub .= "\@results= map { \$_->descendants( '$gi')  } \@results;";  }            
+                    else
+                      { $sub .= "\@results= map { \$_->children( '$gi')  } \@results;"; }
+                  } 
+                # now filter using the condition
+                if( $cond)
+                  { my $op='';
+                    my $test="";
+                    do
+                      { if( $op)
+                          { $cond=~ s{^\s*$op\s*}{};
+                            $op= lc( $op);
+                            $test .= $op;
+                          }
+                       if( $cond =~ s{^string\(\s*\)\s*=\s*($REG_STRING)\s*}{}o)  # string()="string" cond
+                          { $test .= "\$_->text eq $1"; 
+                          }
+                       elsif( $cond =~ s{^string\(\s*\)\s*($REG_MATCH)\s*($REG_REGEXP)\s*}{}o)  # string()=~/regex/ cond
+                          { my( $match, $regexp)= ($1, $2);
+                            $test .= "\$_->text $match $regexp"; 
+                          }
+                       elsif( $cond=~ s{^@($REG_NAME)\s*($REG_OP)\s*($REG_STRING|$REG_NUMBER)}{}o)  # @att="val" cond
+                          { my( $att, $oper, $val)= ($1, _op( $2), $3);
+                            $test .= qq{((defined \$_->{'att'}->{"$att"})  && (\$_->{'att'}->{"$att"} $oper $val))};
+                          }
+                       elsif( $cond =~ s{^@($REG_NAME)\s*($REG_MATCH)\s*($REG_REGEXP)\s*}{}o)  # @att=~/regex/ cond XXX
+                          { my( $att, $match, $regexp)= ($1, $2, $3);
+                            $test .= qq{((defined \$_->{'att'}->{"$att"})  && (\$_->{'att'}->{"$att"} $match $regexp))};; 
+                          }
+                       elsif( $cond=~ s{^@($REG_NAME)\s*}{}o)                      # @att cond
+                          { 
+                            $test .= qq{(defined \$_->{'att'}->{"$1"})};
+                          }
+                       elsif( $cond=~ s{^\s*(\d+)\s*}{}o)                          # positive number condition
+                          { 
+                            $test .= qq{(\$_->prev_siblings( \$_->gi) == $1)};
+                          }
+                       elsif( $cond=~ s{^\s*-\s*(\d+)\s*}{}o)                      # negative number condition
+                          { 
+                            $test .= qq{(\$_->next_siblings( \$_->gi) == $1)};
+                          }
+                       } while( ($op)=($cond=~ m{^\s*(and|or)\s*}i));
+                     croak "error in xpath expression $original_exp at $cond" if( $cond);
+                     $sub .= "\@results= grep { $test } \@results;";
+                   }
+              }
+          }
+      }
+
+    if( $xpath_exp)
+      { croak "error in xpath expression $original_exp around $xpath_exp"; }
+      
+    $sub .= "return \@results; ";
+    my $s= eval "sub { $sub }";
+    if( $@) { croak "error in xpath expression $original_exp ($@);" }
+    return( $s); 
+   }
+        
+{ # extremely elaborate caching mechanism
+  my %xpath; # xpath_expression => subroutine_code;  
+  sub get_xpath
+    { my( $elt, $xpath_exp, $offset)= @_;
+      my $sub= ($xpath{$xpath_exp} ||= _install_xpath( $xpath_exp));
+      return $sub->( $elt) unless( defined $offset); 
+      my @res= $sub->( $elt);
+      return $res[$offset];
+    }
+    1; # so the module returns 1 as this is the last BEGIN block in the file
+}
+
+
+sub findvalue
+  { my $elt= shift;
+    return join '', map { $_->text } $elt->get_xpath( @_);
+  }
+
+#end-extract twig_node
+
+
+# XML::XPath compatibility
+sub getElementById     { return $_[0]->twig->elt_id( $_[1]); }
+sub getChildNodes      { my @children= $_[0]->children; return wantarray ? @children : \@children; }
+
+sub _flushed     { return $_[0]->{flushed}; }
+sub _set_flushed { $_[0]->{flushed}=1;      }
+sub _del_flushed { delete $_[0]->{flushed}; }
+
+
+sub cut
+  { my $elt= shift;
+    my( $parent, $prev_sibling, $next_sibling, $last_elt);
+
+    # you can't cut the root, sorry
+    unless( $parent= $elt->{parent}) 
+      { return; }
+    # it we cut the current element then its parent becomes the current elt
+    if( $elt->{twig_current})
+      { my $twig_current= $elt->{parent};
+        my $t= $elt->twig;
+        $t->{twig_current}= $twig_current;
+        $twig_current->{'twig_current'}=1;
+        delete $elt->{'twig_current'};
+      }
+
+    if( $parent->{first_child} == $elt)
+      { $parent->{first_child}=  $elt->{next_sibling};
+        $parent->{empty}= 1 unless( $elt->{next_sibling});
+      }
+    $parent->set_last_child( $elt->{prev_sibling}) 
+      if( $parent->{last_child} == $elt);
+
+    if( $prev_sibling= $elt->{prev_sibling})
+      { $prev_sibling->{next_sibling}=  $elt->{next_sibling}; }
+    if( $next_sibling= $elt->{next_sibling})
+      { $next_sibling->set_prev_sibling( $elt->{prev_sibling}); }
+
+
+    $elt->set_parent( undef);
+    $elt->set_prev_sibling( undef);
+    $elt->{next_sibling}=  undef;
+
+    return $elt;
+  }
+
+sub cut_children
+  { my( $elt, $exp)= @_;
+    my @children= $elt->children( $exp);
+    foreach (@children) { $_->cut; }
+    return @children;
+  }
+
+sub erase
+  { my $elt= shift;
+    #you cannot erase the current element
+    if( $elt->{twig_current})
+      { croak "trying to erase an element before it has been completely parsed"; }
+    unless( $elt->{parent})
+      { # trying to erase the root (of a twig or of a cut/new element)
+        my @children= $elt->children;
+        unless( @children == 1)
+          { croak "can only erase an element with no parent if it has a single child"; }
+        my $child= shift @children;
+        $child->set_parent( undef);
+        my $twig= $elt->twig;
+        $twig->set_root( $child);
+      }
+    else     
+      { # normal case
+        my @children= $elt->children;
+        if( @children)
+          { # elt has children, move them up
+            if( $elt->{prev_sibling})
+              { # connect first child to previous sibling
+                $elt->{first_child}->set_prev_sibling( $elt->{prev_sibling});      
+                $elt->{prev_sibling}->set_next_sibling( $elt->{first_child}); 
+              }
+            else
+              { # elt was the first child
+                $elt->{parent}->set_first_child( $elt->{first_child});
+              }
+            if( $elt->{next_sibling})
+              { # connect last child to next sibling
+                $elt->{last_child}->set_next_sibling( $elt->{next_sibling});      
+                $elt->{next_sibling}->set_prev_sibling( $elt->{last_child}); 
+              }
+            else
+              { # elt was the last child
+                $elt->{parent}->set_last_child( $elt->{last_child});
+              }
+            # update parent for all siblings
+            foreach my $child (@children)
+              { $child->set_parent( $elt->{parent}); }
+            # elt is not referenced any more, so it will be DESTROYed
+            # so we'd better break the links to its children
+            undef $elt->{'first_child'};
+            undef $elt->{'last_child'};
+            undef $elt->{'parent'};
+            undef $elt->{'prev_sibling'};
+            undef $elt->{'next_sibling'};
+          }
+          { # elt had no child, delete it
+             $elt->delete;
+          }
+    
+        return $elt;
+      }
+
+  }
+        
+
+BEGIN
+  { my %method= ( before      => \&paste_before,
+                  after       => \&paste_after,
+                  first_child => \&paste_first_child,
+                  last_child  => \&paste_last_child,
+                  within      => \&paste_within,
+        );
+    
+    # paste elt somewhere around ref
+    # pos can be first_child (default), last_child, before, after or within
+    sub paste
+      { my $elt= shift;
+        if( $elt->{parent}) 
+          { croak "cannot paste an element that belongs to a tree"; }
+        my $pos;
+        my $ref;
+        if( ref $_[0]) 
+          { $pos= 'first_child'; 
+            croak "wrong argument order in paste, should be $_[1] first" if($_[1]); 
+          }
+        else
+          { $pos= shift; }
+
+        if( my $method= $method{$pos})
+          { unless( ref $_[0]) { croak "Destination undefined in paste"; }
+            unless( isa( $_[0], "XML::Twig::Elt"))
+              { croak "wrong argument type in paste, should be XML::Twig::Elt"; }
+            # check here so error message lists the caller file/line
+            if( !$ref->{parent} && ($method=~ m{^(before|after)}) ) { croak "cannot paste $1 root"; }
+            $elt->$method( @_); 
+          }
+        else
+          { croak "tried to paste in wrong position '$pos', allowed positions " . 
+              " are 'first_child', 'last_child', 'before', 'after' and "    .
+              "'within'";
+          }
+        return $elt;
+      }
+  
+
+    sub paste_before
+      { my( $elt, $ref)= @_;
+        my( $parent, $prev_sibling, $next_sibling );
+        unless( $ref->{parent}) { croak "cannot paste before root"; }
+        $parent= $ref->{parent};
+        $prev_sibling= $ref->{prev_sibling};
+        $next_sibling= $ref;
+
+        $elt->set_parent( $parent);
+        $parent->{first_child}=  $elt if( $parent->{first_child} == $ref);
+
+        $prev_sibling->{next_sibling}=  $elt if( $prev_sibling);
+        $elt->set_prev_sibling( $prev_sibling);
+
+        $next_sibling->set_prev_sibling( $elt);
+        $elt->{next_sibling}=  $ref;
+        return $elt;
+      }
+     
+     sub paste_after
+      { my( $elt, $ref)= @_;
+        my( $parent, $prev_sibling, $next_sibling );
+        unless( $ref->{parent}) { croak "cannot paste after root"; }
+        $parent= $ref->{parent};
+        $prev_sibling= $ref;
+        $next_sibling= $ref->{next_sibling};
+
+        $elt->set_parent( $parent);
+        $parent->set_last_child( $elt) if( $parent->{last_child}== $ref);
+
+        $prev_sibling->{next_sibling}=  $elt;
+        $elt->set_prev_sibling( $prev_sibling);
+
+        $next_sibling->set_prev_sibling( $elt) if( $next_sibling);
+        $elt->{next_sibling}=  $next_sibling;
+        return $elt;
+
+      }
+
+    sub paste_first_child
+      { my( $elt, $ref)= @_;
+        my( $parent, $prev_sibling, $next_sibling );
+        $parent= $ref;
+        $next_sibling= $ref->{first_child};
+        delete $ref->{empty};
+
+        $elt->set_parent( $parent);
+        $parent->{first_child}=  $elt;
+        $parent->set_last_child( $elt) unless( $parent->{last_child});
+
+        $elt->set_prev_sibling( undef);
+
+        $next_sibling->set_prev_sibling( $elt) if( $next_sibling);
+        $elt->{next_sibling}=  $next_sibling;
+        return $elt;
+      }
+      
+    sub paste_last_child
+      { my( $elt, $ref)= @_;
+        my( $parent, $prev_sibling, $next_sibling );
+        $parent= $ref;
+        $prev_sibling= $ref->{last_child};
+        delete $ref->{empty};
+
+        $elt->set_parent( $parent);
+        $parent->set_last_child( $elt);
+        $parent->{first_child}=  $elt unless( $parent->{first_child});
+
+        $elt->set_prev_sibling( $prev_sibling);
+        $prev_sibling->{next_sibling}=  $elt if( $prev_sibling);
+
+        $elt->{next_sibling}=  undef;
+        return $elt;
+      }
+
+    sub paste_within
+      { my( $elt, $ref, $offset)= @_;
+        my $text= $ref->is_text ? $ref : $ref->next_elt( '#TEXT', $ref);
+        my $new= $text->split_at( $offset);
+        $elt->paste_before( $new);
+        return $elt;
+      }
+  }
+
+# load an element into a structure similar to XML::Simple's
+sub simplify
+  { my $elt= shift;
+
+    # normalize option names
+    my %options= @_;
+    %options= map { my ($key, $val)= ($_, $options{$_});
+                       $key=~ s{(\w)([A-Z])}{$1_\L$2}g;
+                       $key => $val
+                     } keys %options;
+
+    # check options
+    my @allowed_options= qw( keyattr forcearray noattr content_key
+                             var var_regexp variables var_attr 
+                             erase group_tags forcecontent
+                             normalise_space normalize_space
+                   );
+    my %allowed_options= map { $_ => 1 } @allowed_options;
+    foreach my $option (keys %options)
+      { warn "invalid option $option\n" unless( $allowed_options{$option}); }
+
+    $options{normalise_space} ||= $options{normalize_space} || 0;
+    
+    $options{content_key} ||= 'content';
+    if( $options{content_key}=~ m{^-})
+      { # need to remove the - and to activate extra folding
+        $options{content_key}=~ s{^-}{};
+        $options{extra_folding}= 1;
+      }
+    else
+      { $options{extra_folding}= 0; }
+   
+    $options{forcearray} ||=0; 
+    if( isa( $options{forcearray}, 'ARRAY'))
+      { my %forcearray_tags= map { $_ => 1 } @{$options{forcearray}};
+        $options{forcearray_tags}= \%forcearray_tags;
+        $options{forcearray}= 0;
+      }
+
+    $options{erase}||= $options{group_tags}; # for compat with XML::Simple
+    if( isa( $options{erase}, 'ARRAY'))
+      { my %erase_tags= map { $_ => 1 } @{$options{erase}};
+        $options{erase}= \%erase_tags;
+      }
+
+    $options{keyattr}     ||= ['name', 'key', 'id'];
+    if( ref $options{keyattr} eq 'ARRAY')
+      { foreach my $keyattr (@{$options{keyattr}})
+          { my( $prefix, $att)= ($keyattr=~ m{^([+-])?(.*)});
+            $prefix ||= '';
+            $options{key_for_all}->{$att}= 1;
+            $options{remove_key_for_all}->{$att}=1 unless( $prefix eq '+');
+            $options{prefix_key_for_all}->{$att}=1 if( $prefix eq '-');
+          }
+      }
+    elsif( ref $options{keyattr} eq 'HASH')
+      { while( my( $elt, $keyattr)= each %{$options{keyattr}})
+         { my( $prefix, $att)= ($keyattr=~ m{^([+-])?(.*)});
+           $prefix ||='';
+           $options{key_for_elt}->{$elt}= $att;
+           $options{remove_key_for_elt}->{"$elt#$att"}=1 unless( $prefix);
+           $options{prefix_key_for_elt}->{"$elt#$att"}=1 if( $prefix eq '-');
+         }
+      }
+       
+
+    $options{var}||= $options{var_attr}; # for compat with XML::Simple
+    if( $options{var}) { $options{var_values}= {}; }
+    else               { $options{var}='';         }
+
+    if( $options{variables}) 
+      { $options{var}||= 1;
+        $options{var_values}= $options{variables};
+      }
+
+    if( $options{var_regexp} and !$options{var})
+      { warn "var option not used, var_regexp option ignored\n"; }
+    $options{var_regexp} ||= '\$\{?(\w+)\}?';
+      
+    $elt->_simplify( \%options);
+ 
+ }
+
+sub _simplify
+  { my( $elt, $options)= @_;
+
+    my $data;
+
+    my $gi= $XML::Twig::index2gi[$elt->{'gi'}];
+    my @children= $elt->children;
+    my %atts= $options->{noattr} ? () : %{$elt->atts};
+    my $nb_atts= keys %atts;
+    my $nb_children= $elt->children_count + $nb_atts;
+
+    my %nb_children;
+    foreach (@children)   { $nb_children{$_->tag}++; }
+    foreach (keys %atts)  { $nb_children{$_}++;      }
+
+    my $arrays; # tag => array where elements are stored
+
+
+    # store children
+    foreach my $child (@children)
+      { if( $child->is_text)
+          { # generate with a content key
+            my $text= $elt->_text_with_vars( $options);
+            $text= _normalize_space( $text) if( $options->{normalise_space} >= 2);
+            if(    $options->{force_content}
+                || $nb_atts 
+              )
+              { $data->{$options->{content_key}}= $text; }
+            else
+              { $data= $text; }
+          }
+        else
+          { # element with sub elements
+            my $child_gi= $XML::Twig::index2gi[$child->{'gi'}];
+
+            my $child_data= $child->_simplify( $options);
+
+            # first see if we need to simplify further the child data
+            # because of grouped tags
+            if( my $grouped_tag= $options->{group_tags}->{$child_gi})
+              { # check that the child data is a hash with a single field
+                unless(    (ref( $child_data) eq 'HASH')
+                        && (keys %$child_data == 1)
+                        && defined ( my $grouped_child_data= $child_data->{$grouped_tag})
+                      )
+                  { die "error in grouped tag $child_gi"; }
+                else
+                  { $child_data=  $grouped_child_data; }
+              }
+            # because of extra folding
+            if( $options->{extra_folding})
+              { if(    (ref( $child_data) eq 'HASH')
+                    && (keys %$child_data == 1)
+                    && defined( my $content= $child_data->{$options->{content_key}})
+                  )
+                  { $child_data= $content; }
+              }
+
+
+            if( my $keyatt= $child->_key_attr( $options))
+              { # element with key
+                my $key= $child->{'att'}->{$keyatt};
+                $key= _normalize_space( $key) if( $options->{normalise_space} >= 1);
+                $data->{$child_gi}->{$key}= $child_data;
+              }
+            elsif(      $options->{forcearray}
+                   ||   $options->{forcearray_tags}->{$child_gi}
+                   || ( $nb_children{$child_gi} > 1)
+                 )
+              { # element to store in an array
+                $data->{$child_gi} ||= [];
+                push @{$data->{$child_gi}}, $child_data;
+              }
+            else
+              { # element to store as a hash field
+                $data->{$child_gi}= $child_data;
+              }
+
+          }
+    }
+
+    # store atts
+    # TODO: deal with att that already have an element by that name
+    foreach my $att (keys %atts)
+      { # do not store if the att is a key that needs to be removed
+        if(    $options->{remove_key_for_all}->{$att}
+            || $options->{remove_key_for_elt}->{"$gi#$att"}
+          )
+          { next; }
+
+        my $att_text= _replace_vars_in_text( $atts{$att}, $options);
+        $att_text= _normalize_space( $att_text) if( $options->{normalise_space} >= 2);
+        
+        if(    $options->{prefix_key_for_all}->{$att}
+            || $options->{prefix_key_for_elt}->{"$gi#$att"}
+          )
+          { # prefix the att
+            $data->{"-$att"}= $att_text;
+          } 
+        else
+          { # normal case
+            $data->{$att}= $att_text; 
+          }
+      }
+    
+    return $data;
+  }
+
+sub _key_attr
+  { my( $elt, $options)=@_;
+    return if( $options->{noattr});
+    if( $options->{key_for_all})
+      { foreach my $att ($elt->att_names)
+          { if( $options->{key_for_all}->{$att})
+              { return $att; }
+          }
+      }
+    elsif( $options->{key_for_elt})
+      { if( my $key_for_elt= $options->{key_for_elt}->{$XML::Twig::index2gi[$elt->{'gi'}]} )
+          { return $key_for_elt if( defined( $elt->{'att'}->{$key_for_elt})); }
+      }
+    return;
+  }
+
+sub _text_with_vars
+  { my( $elt, $options)= @_;
+    my $text;
+    if( $options->{var}) 
+      { $text= _replace_vars_in_text( $elt->text, $options); 
+        $elt->_store_var( $options);
+      }
+     else
+      { $text= $elt->text; }
+    return $text;
+  }
+
+
+sub _normalize_space
+  { my $text= shift;
+    $text=~ s{\s+}{ }sg;
+    $text=~ s{^\s}{};
+    $text=~ s{\s$}{};
+    return $text;
+  }
+
+
+sub att_nb
+  { return 0 unless( my $atts= $_[0]->atts);
+    return scalar keys %$atts;
+  }
+    
+sub has_no_atts
+  { return 1 unless( my $atts= $_[0]->atts);
+    return scalar keys %$atts ? 0 : 1;
+ }
+    
+sub _replace_vars_in_text
+  { my( $text, $options)= @_;
+
+    $text=~ s{($options->{var_regexp})}
+             { if( defined( my $value= $options->{var_values}->{$2}))
+                 { $value }
+               else
+                 { warn "unknown variable $2\n";
+                   $1
+                 }
+             }gex;
+    return $text;
+  }
+
+sub _store_var
+  { my( $elt, $options)= @_;
+    if( defined (my $var_name= $elt->{'att'}->{$options->{var}}))
+       { $options->{var_values}->{$var_name}= $elt->text; 
+       }
+  }
+
+
+# split a text element at a given offset
+sub split_at
+  { my( $elt, $offset)= @_;
+    my $text_elt= $elt->is_text ? $elt : $elt->first_child( TEXT) || return '';
+    my $string= $text_elt->text; 
+    my $left_string= substr( $string, 0, $offset);
+    my $right_string= substr( $string, $offset);
+    $text_elt->{pcdata}=  $left_string;
+    my $new_elt= XML::Twig::Elt->new( $XML::Twig::index2gi[$elt->{'gi'}], $right_string);
+    $new_elt->paste( after => $elt);
+    return $new_elt;
+  }
+
+    
+# split an element or its text descendants into several, in place
+# all elements (new and untouched) are returned
+sub split    
+  { my $elt= shift;
+    my @text_chunks;
+    my @result;
+    if( $elt->is_text) { @text_chunks= ($elt); }
+    else               { @text_chunks= $elt->descendants( '#TEXT'); }
+    foreach my $text_chunk (@text_chunks)
+      { push @result, $text_chunk->_split( 1, @_); }
+    return @result;
+  }
+
+# split an element or its text descendants into several, in place
+# created elements (those which match the regexp) are returned
+sub mark
+  { my $elt= shift;
+    my @text_chunks;
+    my @result;
+    if( $elt->is_text) { @text_chunks= ($elt); }
+    else               { @text_chunks= $elt->descendants( '#TEXT'); }
+    foreach my $text_chunk (@text_chunks)
+      { push @result, $text_chunk->_split( 0, @_); }
+    return @result;
+  }
+
+# split a single text element
+# return_all defines what is returned: if it is true 
+# only returns the elements created by matches in the split regexp
+# otherwise all elements (new and untouched) are returned
+
+{ my $encode_is_loaded=0;   # so we only load Encode once in 5.8.0+
+ 
+  sub _split
+    { my $elt= shift;
+      my $return_all= shift;
+      my( $regexp, $tag, $atts)= @_;
+      my @result;                                 # the returned list of elements
+      my $text= $elt->text;
+      my $gi= $XML::Twig::index2gi[$elt->{'gi'}];
+      $tag||= $elt->parent( '#ELT')->gi;          # default: same tag as the current one
+      $atts ||= {};                               # default: no attributes
+  
+      # 2 uses: if split matches then the first substring reuses $elt
+      #         once a split has occured then the last match needs to be put in
+      #         a new element      
+      my $replaced= 0;
+
+      while( $text=~/(.*?)(?:($regexp)|$)/sg)
+        { # $1 is the pre-regexp match
+          # $2 is the string that matches the regexp (if true then the regexp 
+          #    matched, otherwise no match)
+          # $3...$n are captured by the regexp (and should be wrapped in tag)
+          if( defined $2 || $replaced)
+            { # the regexp matched this time ($2) or previously ($replaced)
+              unless( $replaced)
+                { # first match in $elt, re-use $elt for the first sub-string
+                  $elt->set_text( _utf8_ify( $1));
+                  $replaced++;                     # note that there was a match
+                  push @result, $elt if( $return_all);
+                }
+              else
+                { # match, not the first one, create a new text ($gi) element
+                  my $pre_match= _utf8_ify( $1);
+                  my $new_text= XML::Twig::Elt->new( $gi, $pre_match);
+                  $new_text->paste( 'after', $elt); # paste it after $elt
+                  $elt= $new_text;                  # $elt: element to paste after
+                  push @result, $elt if( $return_all);
+                }
+              # now deal with matches captured in the regexp
+              no strict( 'refs');                   # needed to access $3...$n
+              for( my $match_nb=3; defined $$match_nb; $match_nb++)
+                { # create new element, text is the match
+                  my %atts= %$atts; # or the same atts is used for all matches!
+                  my $match= _utf8_ify( $$match_nb);
+                  my $new_text= XML::Twig::Elt->new( $tag, \%atts, $match);
+                  $new_text->paste( 'after', $elt); # paste it after the current
+                  $elt= $new_text;                  # it becomes the current elt
+                  push @result, $elt;
+                }
+            }
+          else
+            { # the regep did not match at all, $elt is not changed
+              push @result, $elt if( $return_all);
+            }
+        } 
+      return @result; # return all elements
+   }
+
+  # evil hack needed in 5.8.0, the utf flag is not set on $<n>...
+  sub _utf8_ify
+    { my $string= shift;
+      if( $] > 5.007 and !keep_encoding()) 
+        { unless( $encode_is_loaded) { require Encode; import Encode; $encode_is_loaded++; }
+          Encode::_utf8_on( $string); # the flag should be set but is not
+        }
+      return $string;
+    }
+
+
+}
+
+{ my %replace_sub; # cache for complex expressions (expression => sub)
+
+  sub subs_text
+    { my( $elt, $regexp, $replace)= @_;
+  
+      my $replacement_string;
+      my $is_string= _is_string( $replace);
+      foreach my $text_elt ($elt->descendants_or_self( '#TEXT'))
+        { if( $is_string)
+            { my $text= $text_elt->text;
+              $text=~ s{$regexp}{ _replace_var( $replace, $1, $2, $3, $4, $5, $6, $7, $8, $9)}egx;
+              $text_elt->set_text( $text);
+           }
+          else
+            { my $replace_sub= ( $replace_sub{$replace} ||= _install_replace_sub( $replace)); 
+              my $text= $text_elt->text;
+              my $pos=0;  # used to skip text that was previously matched
+              while( my( $pre_match_string, $match_string, @var)= ($text=~ m{(.*?)($regexp)}g))
+                { my $match_start  = length( $pre_match_string);
+                  my $match        = $text_elt->split_at( $match_start + $pos);
+                  my $match_length = length( $match_string);
+                  my $post_match   = $match->split_at( $match_length);
+                  $replace_sub->( $match, @var);
+                  # merge previous text with current one
+                  my $next_sibling;
+                  if(    ($next_sibling= $text_elt->{next_sibling})
+                      && ($XML::Twig::index2gi[$text_elt->{'gi'}] eq $XML::Twig::index2gi[$next_sibling->{'gi'}])
+                    )
+                    { $text_elt->merge_text( $next_sibling); }
+                  # go to next 
+                  $text_elt= $post_match;
+                  $text= $post_match->text;
+                  # merge last text element with next one if needed,
+                  # the match will be against the non-matched text,
+                  # so $pos is used to skip the merged part
+                  my $prev_sibling;
+                  if(    ($prev_sibling=  $post_match->{prev_sibling})
+                      && ($XML::Twig::index2gi[$post_match->{'gi'}] eq $XML::Twig::index2gi[$prev_sibling->{'gi'}])
+                    )
+                    { $pos= length( $prev_sibling->text);
+                      $post_match->merge_text(  $prev_sibling);
+                    }
+                }
+            }
+        }
+      return $elt;
+    }
+
+
+  sub _is_string
+    { return ($_[0]=~ m{&e[ln]t}) ? 0: 1 }
+
+  sub _replace_var
+    { my( $string, @var)= @_;
+      unshift @var, undef;
+      $string=~ s{\$(\d)}{$var[$1]}g;
+      return $string;
+    }
+
+  sub _install_replace_sub
+    { my $replace_exp= shift;
+      my @item= split m{(&e[ln]t\s*\([^)]*\))}, $replace_exp;
+      my $sub= q{ my( $match, @var)= @_; unshift @var, undef; my $new; };
+      my( $gi, $exp);
+      foreach my $item (@item)
+        { if(    $item=~ m{^&elt\s*\(([^)]*)\)})
+            { $exp= $1;
+            }
+          elsif( $item=~ m{^&ent\s*\(\s*([^\s)]*)\s*\)})
+            { $exp= " '#ENT' => $1"; }
+          else
+            { $exp= qq{ '#PCDATA' => "$item"}; }
+          $exp=~ s{\$(\d)}{\$var[$1]}g; # replace references to matches
+          $sub.= qq{ \$new= XML::Twig::Elt->new( $exp); };
+          $sub .= q{ $new->paste( before => $match); };
+        }
+      $sub .= q{ $match->delete; };
+      #$sub=~ s/;/;\n/g;
+      my $coderef= eval "sub { $sub }";
+      if( $@) { croak( "invalid replacement expression $replace_exp: ",$@); }
+      return $coderef;
+    }
+
+  }
+
+
+sub merge_text
+  { my( $e1, $e2)= @_;
+    croak "invalid merge: can only merge 2 elements" 
+        unless( isa( $e2, 'XML::Twig::Elt'));
+    croak "invalid merge: can only merge 2 text elements" 
+        unless( $e1->is_text && $e2->is_text && ($e1->gi eq $e2->gi));
+    $e1->set_text( $e1->text . $e2->text);
+    $e2->delete;
+    return $e1;
+  }
+
+
+# recursively copy an element and returns the copy (can be huge and long)
+sub copy
+  { my $elt= shift;
+    my $copy= XML::Twig::Elt->new( $XML::Twig::index2gi[$elt->{'gi'}]);
+
+    $copy->set_extra_data( $elt->extra_data) if( $elt->extra_data);
+    $copy->set_asis                          if( $elt->is_asis);
+    if( ($elt->{'empty'} || 0)) { $copy->{empty}= 1; } # do not swap or speedup will mess up this                         
+
+    if( (exists $elt->{'pcdata'}))
+      { $copy->{pcdata}=  $elt->{pcdata}; }
+    elsif( (exists $elt->{'cdata'}))
+      { $copy->{cdata}=  $elt->{cdata}; }
+    elsif( (exists $elt->{'target'}))
+      { $copy->set_pi( $elt->{target}, $elt->{data}); }
+    elsif( (exists $elt->{'comment'}))
+      { $copy->{comment}=  $elt->{comment}; }
+    elsif( (exists $elt->{'ent'}))
+      { $copy->{ent}=  $elt->{ent}; }
+    else
+      { my @children= $elt->children;
+        if( my $atts= $elt->atts)
+          { my %atts= %{$atts}; # we want to do a real copy of the attributes
+            $copy->set_atts( \%atts);
+          }
+        foreach my $child (@children)
+          { my $child_copy= $child->copy;
+            $child_copy->paste( 'last_child', $copy);
+          }
+      }
+    return $copy;
+  }
+
+sub delete
+  { my $elt= shift;
+    $elt->cut;
+    $elt->DESTROY unless( $XML::Twig::weakrefs);
+    return undef;
+  }
+
+{ my $current_twig;
+  sub DESTROY
+    { my $elt= shift;
+
+      return if( $XML::Twig::weakrefs);
+
+      foreach( @{[$elt->children]}) { $_->XML::Twig::Elt::DESTROY; }
+      # the id reference needs to be destroyed
+      $elt->del_id if( $ID && exists $elt->{att}->{$ID});
+      undef $elt;
+    }
+}
+
+
+# to be called only from a start_tag_handler: ignores the current element
+sub ignore
+  { my $elt= shift;
+    my $t= $elt->twig;
+    $t->ignore( $elt, @_);
+  }
+
+BEGIN {
+  my $pretty=0;
+  my $quote='"';
+  my $INDENT= '  ';
+  my $empty_tag_style= 0;
+  my $keep_encoding= 0;
+  my $expand_external_entities= 0;
+  my $keep_atts_order=0;
+
+  my ($NSGMLS, $NICE, $INDENTED, $INDENTEDC, $RECORD1, $RECORD2)= (1..6);
+
+  my %pretty_print_style=
+    ( none       => 0,          # no added \n
+      nsgmls     => $NSGMLS,    # nsgmls-style, \n in tags
+      # below this line styles are UNSAFE (the generated XML can be invalid)
+      nice       => $NICE,      # \n after open/close tags except when the 
+                                # element starts with text
+      indented   => $INDENTED,  # nice plus idented
+      indented_c => $INDENTEDC, # slightly more compact than indented (closing
+                                # tags are on the same line)
+      record_c   => $RECORD1,   # for record-like data (compact)
+      record     => $RECORD2,   # for record-like data  (not so compact)
+    );
+
+  my ($HTML, $EXPAND)= (1..2);
+  my %empty_tag_style=
+    ( normal => 0,        # <tag/>
+      html   => $HTML,    # <tag />
+      xhtml  => $HTML,    # <tag />
+      expand => $EXPAND,  # <tag></tag>
+    );
+
+  my %quote_style=
+    ( double  => '"',    
+      single  => "'", 
+      # smart  => "smart", 
+    );
+
+  my $xml_space_preserve; # set when an element includes xml:space="preserve"
+
+  my $output_filter;      # filters the entire output (including < and >)
+  my $output_text_filter; # filters only the text part (tag names, attributes, pcdata)
+
+  # returns those pesky "global" variables so you can switch between twigs 
+  sub global_state
+    { return
+       { pretty                    => $pretty,
+         quote                     => $quote,
+         indent                    => $INDENT,
+         empty_tag_style           => $empty_tag_style,
+         keep_encoding             => $keep_encoding,
+         expand_external_entities  => $expand_external_entities,
+         output_filter             => $output_filter,
+         output_text_filter        => $output_text_filter,
+         keep_atts_order           => $keep_atts_order,
+        };
+    }
+
+  # restores the global variables
+  sub set_global_state
+    { my $state= shift;
+      $pretty                   = $state->{pretty};
+      $quote                    = $state->{quote};
+      $INDENT                   = $state->{indent};
+      $empty_tag_style          = $state->{empty_tag_style};
+      $keep_encoding            = $state->{keep_encoding};
+      $expand_external_entities = $state->{expand_external_entities};
+      $output_filter            = $state->{output_filter};
+      $output_text_filter       = $state->{output_text_filter};
+      $keep_atts_order          = $state->{keep_atts_order};
+    }
+
+  # set the pretty_print style (in $pretty) and returns the old one
+  # can be called from outside the package with 2 arguments (elt, style)
+  # or from inside with only one argument (style)
+  # the style can be either a string (one of the keys of %pretty_print_style
+  # or a number (presumably an old value saved)
+  sub set_pretty_print
+    { my $style= lc( defined $_[1] ? $_[1] : $_[0]); # so we cover both cases 
+      my $old_pretty= $pretty;
+      if( $style=~ /^\d+$/)
+        { croak "invalid pretty print style $style"
+        unless( $style < keys %pretty_print_style);
+        $pretty= $style;
+    }
+      else
+        { croak "invalid pretty print style '$style'"
+            unless( exists $pretty_print_style{$style});
+          $pretty= $pretty_print_style{$style};
+    }
+      return $old_pretty;
+    }
+  
+  
+  # set the empty tag style (in $empty_tag_style) and returns the old one
+  # can be called from outside the package with 2 arguments (elt, style)
+  # or from inside with only one argument (style)
+  # the style can be either a string (one of the keys of %empty_tag_style
+  # or a number (presumably an old value saved)
+  sub set_empty_tag_style
+    { my $style= lc( defined $_[1] ? $_[1] : $_[0]); # so we cover both cases 
+      my $old_style= $empty_tag_style;
+      if( $style=~ /^\d+$/)
+        { croak "invalid empty tag style $style"
+        unless( $style < keys %empty_tag_style);
+        $empty_tag_style= $style;
+    }
+      else
+        { croak "invalid empty tag style '$style'"
+            unless( exists $empty_tag_style{$style});
+          $empty_tag_style= $empty_tag_style{$style};
+    }
+      return $old_style;
+    }
+      
+  sub set_quote
+    { my $style= $_[1] || $_[0];
+      my $old_quote= $quote;
+      croak "invalid quote '$style'" unless( exists $quote_style{$style});
+      $quote= $quote_style{$style};
+      return $old_quote;
+    }
+      
+  sub set_indent
+    { my $new_value= defined $_[1] ? $_[1] : $_[0];
+      my $old_value= $INDENT;
+      $INDENT= $new_value;
+      return $old_value;
+    }
+       
+  sub set_keep_encoding
+    { my $new_value= defined $_[1] ? $_[1] : $_[0];
+      my $old_value= $keep_encoding;
+      $keep_encoding= $new_value;
+      return $old_value;
+    
+   }
+
+  sub keep_encoding { return $keep_encoding; } # so I can use elsewhere in the module
+
+  sub output_filter      { return $output_filter; }
+  sub output_text_filter { return $output_text_filter; }
+
+  sub set_output_filter
+    { my $new_value= defined $_[1] ? $_[1] : $_[0]; # can be called in object/non-object mode
+      # if called in object mode with no argument, the filter is undefined
+      if( isa( $new_value, 'XML::Twig::Elt') || isa( $new_value, 'XML::Twig')) { undef $new_value; }
+      my $old_value= $output_filter;
+      if( !$new_value || isa( $new_value, 'CODE') )
+        { $output_filter= $new_value; }
+      elsif( $new_value eq 'latin1')
+        { $output_filter= XML::Twig::latin1();
+        }
+      elsif( $XML::Twig::filter{$new_value})
+        {  $output_filter= $XML::Twig::filter{$new_value}; }
+      else
+        { croak "invalid output filter '$new_value'"; }
+      
+      return $old_value;
+    }
+       
+  sub set_output_text_filter
+    { my $new_value= defined $_[1] ? $_[1] : $_[0]; # can be called in object/non-object mode
+      # if called in object mode with no argument, the filter is undefined
+      if( isa( $new_value, 'XML::Twig::Elt') || isa( $new_value, 'XML::Twig')) { undef $new_value; }
+      my $old_value= $output_text_filter;
+      if( !$new_value || isa( $new_value, 'CODE') )
+        { $output_text_filter= $new_value; }
+      elsif( $new_value eq 'latin1')
+        { $output_text_filter= XML::Twig::latin1();
+        }
+      elsif( $XML::Twig::filter{$new_value})
+        {  $output_text_filter= $XML::Twig::filter{$new_value}; }
+      else
+        { croak "invalid output text filter '$new_value'"; }
+      
+      return $old_value;
+    }
+       
+  sub set_expand_external_entities
+    { my $new_value= defined $_[1] ? $_[1] : $_[0];
+      my $old_value= $expand_external_entities;
+      $expand_external_entities= $new_value;
+      return $old_value;
+    }
+       
+  sub set_keep_atts_order
+    { my $new_value= defined $_[1] ? $_[1] : $_[0];
+      my $old_value= $keep_atts_order;
+      $keep_atts_order= $new_value;
+      return $old_value;
+    
+   }
+
+  sub keep_atts_order { return $keep_atts_order; } # so I can use elsewhere in the module
+
+  # $elt is an element to print
+  # $pretty is an optional value, if true a \n is printed after the <
+
+  my %empty_should_be_expanded= ( script => 1);
+
+  sub start_tag
+    { my $elt= shift;
+  
+      return if( $elt->{gi}<$XML::Twig::SPECIAL_GI);
+
+      my $extra_data= $elt->{extra_data} || '';
+
+      my $gi= $XML::Twig::index2gi[$elt->{'gi'}];
+
+      my $ns_map= $elt->{'att'}->{'#original_gi'};
+      if( $ns_map) { $gi= _restore_original_prefix( $ns_map, $gi); }
+ 
+      if( $output_text_filter) { $gi= $output_text_filter->( $gi); }
+
+      my $tag="<" . $gi;
+  
+      # get the attribute and their values
+      my $att= $elt->atts;
+      if( $att)
+        { foreach my $att_name ( $keep_atts_order ?  keys %{$att} : sort keys %{$att}) 
+           { # skip private attributes (they start with #)
+             next if( (substr( $att_name, 0, 1) eq '#'));
+
+             $tag .=  $pretty==$NSGMLS ? "\n" : ' ';
+
+             my $output_att_name= $ns_map ? _restore_original_prefix( $ns_map, $att_name) : $att_name;
+             if( $output_text_filter) { $output_att_name=  $output_text_filter->( $output_att_name); }
+
+             $tag .= $output_att_name . '=' . $quote . $elt->att_xml_string( $att_name, $quote) . $quote; 
+           }
+        } 
+  
+      $tag .= "\n" if($pretty==$NSGMLS);
+
+      if( $elt->{empty})
+        { if( !$empty_tag_style)
+            { $tag .= "/>";     }
+          elsif( ($empty_tag_style eq $HTML) && ! $empty_should_be_expanded{$XML::Twig::index2gi[$elt->{'gi'}]})
+            { $tag .= " />";  }
+          else #  $empty_tag_style eq $EXPAND
+            { $tag .= "></" . $XML::Twig::index2gi[$elt->{'gi'}] .">";  }
+        }
+      else
+        { $tag .= ">"; }
+
+      if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#')) { $tag= ''; }
+
+      unless( $pretty) { return $extra_data . $tag  }
+
+      my $prefix='';
+      my $return='';   # '' or \n is to be printed before the tag
+      my $indent=0;    # number of indents before the tag
+
+      if( $pretty==$RECORD1)
+        { my $level= $elt->level;
+          $return= "\n" if( $level < 2);
+          $indent= 1 if( $level == 1);
+        }
+
+     elsif( $pretty==$RECORD2)
+        { $return= "\n";
+          $indent= $elt->level;
+        }
+
+      elsif( $pretty==$NICE)
+        { my $parent= $elt->{parent};
+          unless( !$parent || $parent->{contains_text}) 
+            { $return= "\n"; }
+          $elt->{contains_text}= 1 if( ($parent && $parent->{contains_text})
+                                     || $elt->contains_text);
+        }
+
+      elsif( ($pretty==$INDENTED) || ($pretty==$INDENTEDC))
+        { my $parent= $elt->{parent};
+          unless( !$parent || $parent->{contains_text}) 
+            { $return= "\n"; 
+              $indent= $elt->level; 
+            }
+          $elt->{contains_text}= 1 if( ($parent && $parent->{contains_text})
+                                     || $elt->contains_text);
+        }
+
+      if( $return || $indent)
+        { # check for elements in which spaces should be kept
+          my $t= $elt->twig;
+          return $extra_data . $tag if( $xml_space_preserve);
+          if( $t && $t->{twig_keep_spaces_in})
+            { foreach my $ancestor ($elt->ancestors)
+                { return $extra_data . $tag if( $t->{twig_keep_spaces_in}->{$XML::Twig::index2gi[$ancestor->{'gi'}]}) }
+                }
+        
+              $prefix= $INDENT x $indent;
+              if( $extra_data)
+                { $extra_data=~ s{\s+$}{};
+                  $extra_data= $prefix .  $extra_data . $return;
+                }
+        }
+
+      return $return . $extra_data . $prefix . $tag;
+    }
+  
+  sub end_tag
+    { my $elt= shift;
+      return  '' if( ($elt->{gi}<$XML::Twig::SPECIAL_GI) || ($elt->{'empty'} || 0));
+      my $tag= "<";
+      $tag.= "\n" if($pretty==$NSGMLS);
+
+      my $gi= $XML::Twig::index2gi[$elt->{'gi'}];
+
+      if( my $map= $elt->{'att'}->{'#original_gi'}) { $gi= _restore_original_prefix( $map, $gi); }
+
+      if( $output_text_filter) { $gi= $output_text_filter->( $XML::Twig::index2gi[$elt->{'gi'}]); } 
+      $tag .=  "/$gi>";
+
+      $tag = ($elt->{extra_data_before_end_tag} || '') . $tag;
+
+      if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#')) { $tag= ''; }
+
+      return $tag unless $pretty;
+
+      my $prefix='';
+      my $return=0;    # 1 if a \n is to be printed before the tag
+      my $indent=0;    # number of indents before the tag
+
+      if( $pretty==$RECORD1)
+        { $return= 1 if( $elt->level == 0);
+        }
+
+     elsif( $pretty==$RECORD2)
+        { unless( $elt->contains_text)
+            { $return= 1 ;
+              $indent= $elt->level;
+            }
+        }
+
+      elsif( $pretty==$NICE)
+        { my $parent= $elt->{parent};
+          if( (    ($parent && !$parent->{contains_text}) || !$parent )
+            && ( !$elt->{contains_text}  
+             && ($elt->{has_flushed_child} || $elt->_first_child())           
+           )
+         )
+            { $return= 1; }
+        }
+
+      elsif( $pretty==$INDENTED)
+        { my $parent= $elt->{parent};
+          if( (    ($parent && !$parent->{contains_text}) || !$parent )
+            && ( !$elt->{contains_text}  
+             && ($elt->{has_flushed_child} || $elt->_first_child())           
+           )
+         )
+            { $return= 1; 
+              $indent= $elt->level; 
+            }
+        }
+
+      if( $return || $indent)
+        { # check for elements in which spaces should be kept
+          my $t= $elt->twig;
+          return $tag if( $xml_space_preserve);
+          if( $t && $t->{twig_keep_spaces_in})
+            { foreach my $ancestor ($elt, $elt->ancestors)
+                { return $tag if( $t->{twig_keep_spaces_in}->{$XML::Twig::index2gi[$ancestor->{'gi'}]}) }
+            }
+      
+          $prefix= "\n" if( $return);
+          $prefix.= $INDENT x $indent;
+    }
+
+      # add a \n at the end of the document (after the root element)
+      $tag .= "\n" unless( $elt->{parent});
+  
+      return $prefix . $tag;
+    }
+
+  sub _restore_original_prefix
+    { my( $map, $name)= @_;
+      my $prefix= _ns_prefix( $name);
+      if( my $original_prefix= $map->{$prefix})
+        { if( $original_prefix eq '#default')
+            { $name=~ s{^$prefix:}{}; }
+          else
+            { $name=~ s{^$prefix(?=:)}{$original_prefix}; }
+        }
+      return $name;
+    }
+
+  # $elt is an element to print
+  # $fh is an optional filehandle to print to
+  # $pretty is an optional value, if true a \n is printed after the < of the
+  # opening tag
+  sub print
+    { my $elt= shift;
+  
+      my $pretty;
+      my $fh= _is_fh( $_[0]) ? shift : undef;
+      my $old_select= defined $fh ? select $fh : undef;
+      my $old_pretty= defined ($pretty= shift) ? set_pretty_print( $pretty) : undef;
+
+      $xml_space_preserve= 1 if( ($elt->inherit_att( 'xml:space') || '') eq 'preserve');
+ 
+      #$elt->_print;
+      print $elt->_sprint;
+
+      $xml_space_preserve= 0;
+    
+      select $old_select if( defined $old_select);
+      set_pretty_print( $old_pretty) if( defined $old_pretty);
+    }
+      
+  
+  # same as output but does not output the start tag if the element
+  # is marked as flushed
+  sub flush
+    { my $elt= shift;
+      $elt->twig->flush( @_);
+    }
+  
+  sub _flush
+    { my $elt= shift;
+  
+      my $pretty;
+      my $fh=  _is_fh( $_[0]) ? shift : undef;
+      my $old_select= defined $fh ? select $fh : undef;
+      my $old_pretty= defined ($pretty= shift) ? set_pretty_print( $pretty) : undef;
+
+      $xml_space_preserve= 1 if( ($elt->inherit_att( 'xml:space') || '') eq 'preserve');
+
+      $elt->__flush();
+
+      $xml_space_preserve= 0;
+
+      select $old_select if( defined $old_select);
+      set_pretty_print( $old_pretty) if( defined $old_pretty);
+    }
+
+  sub __flush
+    { my $elt= shift;
+  
+      # in case there's some comments or PI's piggybacking
+      #if( $elt->{extra_data})
+      #  { print $output_filter ? $output_filter->($elt->{extra_data}) 
+      #                         : $elt->{extra_data};
+      #  }
+
+      if( $elt->{gi} >= $XML::Twig::SPECIAL_GI)
+        { my $preserve= ($elt->{'att'}->{'xml:space'} || '') eq 'preserve';
+          $xml_space_preserve++ if $preserve;
+          unless( $elt->_flushed)
+            { print $elt->start_tag();
+            }
+      
+          # flush the children
+          my @children= $elt->children;
+          foreach my $child (@children)
+            { $child->_flush( $pretty); 
+        }
+          print $elt->end_tag;
+          $xml_space_preserve-- if $preserve;
+          # used for pretty printing
+          if( my $parent= $elt->{parent}) { $parent->{has_flushed_child}= 1; }
+        }
+      else # text or special element
+        { my $text;
+          if( (exists $elt->{'pcdata'}))     { $text= $elt->pcdata_xml_string; 
+                                     if( my $parent= $elt->{parent}) 
+                                       { $parent->{contains_text}= 1; }
+                                   }
+          elsif( (exists $elt->{'cdata'}))   { $text= $elt->cdata_string;        
+                                     if( my $parent= $elt->{parent}) 
+                                       { $parent->{contains_text}= 1; }
+                                   }
+          elsif( (exists $elt->{'target'}))      { $text= $elt->pi_string;          }
+          elsif( (exists $elt->{'comment'})) { $text= $elt->comment_string;     }
+          elsif( (exists $elt->{'ent'}))     { $text= $elt->ent_string;         }
+
+          print $output_filter ? $output_filter->( $text) : $text;
+        }
+    }
+  
+
+  sub xml_text
+    { my $elt= shift;
+      my $string='';
+
+      if( $elt->{gi} >= $XML::Twig::SPECIAL_GI)
+        { # sprint the children
+          my $child= $elt->{first_child}||'';
+          while( $child)
+            { $string.= $child->xml_text;
+              $child= $child->{next_sibling};
+            }
+        }
+      elsif( (exists $elt->{'pcdata'}))  { $string .= $output_filter ?  $output_filter->($elt->pcdata_xml_string) 
+                                                           : $elt->pcdata_xml_string; 
+                               }
+      elsif( (exists $elt->{'cdata'}))   { $string .= $output_filter ?  $output_filter->($elt->cdata_xml_string)  
+                                                           : $elt->cdata_string;      
+                               }
+      elsif( (exists $elt->{'ent'}))     { $string .= $elt->ent_string; }
+
+      return $string;
+    }
+
+
+  # same as print but except... it does not print but rather returns the string
+  # if the second parameter is set then only the content is returned, not the
+  # start and end tags of the element (but the tags of the included elements are
+  # returned)
+  sub sprint
+    { my $elt= shift;
+
+      $xml_space_preserve= 1 if( ($elt->inherit_att( 'xml:space') || '') eq 'preserve');
+
+      my $sprint= $output_filter ? $output_filter->( $elt->_sprint( @_)) : $elt->_sprint( @_);
+
+      $xml_space_preserve= 0;
+
+      return $sprint;
+
+    }
+  
+  sub _sprint
+    { my $elt= shift;
+      my $no_tag= shift || 0;
+      # in case there's some comments or PI's piggybacking
+      my $string='';
+      #if( $elt->{extra_data} && !$no_tag)
+      #  { $string= $elt->{extra_data};
+      #  }
+
+      if( $elt->{gi} >= $XML::Twig::SPECIAL_GI)
+        {
+          my $preserve= ($elt->{'att'}->{'xml:space'} || '') eq 'preserve';
+          $xml_space_preserve++ if $preserve;
+
+          $string.= $elt->start_tag unless( $no_tag);
+      
+          # sprint the children
+          my $child= $elt->{first_child}||'';
+          while( $child)
+            { $string.= $child->_sprint;
+              $child= $child->{next_sibling};
+            }
+          $string.= $elt->end_tag unless( $no_tag);
+          $xml_space_preserve-- if $preserve;
+        }
+      else
+        { $string .= $elt->{extra_data} || '';
+             if( (exists $elt->{'pcdata'}))  { $string .= $elt->pcdata_xml_string; }
+          elsif( (exists $elt->{'cdata'}))   { $string .= $elt->cdata_string;      }
+          elsif( (exists $elt->{'target'}))      { $string .= $elt->pi_string;         }
+          elsif( (exists $elt->{'comment'})) { $string .= $elt->comment_string;    }
+          elsif( (exists $elt->{'ent'}))     { $string .= $elt->ent_string;        }
+        }
+
+      return $string;
+    }
+
+  # just a shortcut to $elt->sprint( 1)
+  sub xml_string
+    { $_[0]->sprint( 1); }
+
+  sub pcdata_xml_string 
+    { my $string='';
+      if( defined( $string= $_[0]->{pcdata}) )
+        { $string=~ s/([&<])/$XML::Twig::base_ent{$1}/g
+            unless( $keep_encoding || $_[0]->{asis}); 
+        }
+      return $output_text_filter ? $output_text_filter->( $string) : $string;
+    }
+  
+  sub att_xml_string 
+    { my $elt= shift;
+      my $att= shift;
+      my $quote= shift || '"';
+      my $string='';
+      if( defined ($string= $elt->{att}->{$att}))
+        { $string=~ s/([$quote<&])/$XML::Twig::base_ent{$1}/g
+            unless( $keep_encoding); 
+        }
+      return $output_text_filter ? $output_text_filter->( $string) : $string;
+    }
+
+  sub ent_string 
+    { my $ent= shift;
+      my $ent_text= $ent->{ent};
+      my( $t, $el, $ent_string);
+      if(    $expand_external_entities
+          && ($t= $ent->twig) 
+          && ($el= $t->entity_list)
+          && ($ent_string= $el->{$ent->ent_name}->{val})
+        )
+       { return $ent_string; }
+  
+       return $ent_text; 
+    }
+
+  # returns just the text, no tags, for an element
+  sub text
+    { my $elt= shift;
+      my $string;
+  
+      if( (exists $elt->{'pcdata'}))     { return  $elt->{pcdata};   }
+      elsif( (exists $elt->{'cdata'}))   { return  $elt->{cdata};    }
+      elsif( (exists $elt->{'target'}))      { return  $elt->pi_string;}
+      elsif( (exists $elt->{'comment'})) { return  $elt->{comment};  }
+      elsif( (exists $elt->{'ent'}))     { return  $elt->{ent} ;     }
+  
+      my $child= $elt->{first_child} ||'';
+      while( $child)
+        { $string.= defined($child->text) ? $child->text : '';
+          $child= $child->{next_sibling};
+        }
+      unless( defined $string) { $string=''; }
+  
+      return $output_text_filter ? $output_text_filter->( $string) : $string;
+    }
+
+  sub trimmed_text
+    { my $elt= shift;
+      my $text= $elt->text;
+      $text=~ s{\s+}{ }sg;
+      $text=~ s{^\s*}{};
+      $text=~ s{\s*$}{};
+      return $text;
+    }
+
+  # remove cdata sections (turns them into regular pcdata) in an element 
+  sub remove_cdata 
+    { my $elt= shift;
+      foreach my $cdata ($elt->descendants_or_self( CDATA))
+        { if( $keep_encoding)
+            { my $data= $cdata->{cdata};
+              $data=~ s{([&<"'])}{$XML::Twig::base_ent{$1}}g;
+              $cdata->{pcdata}=  $data;
+            }
+          else
+            { $cdata->{pcdata}=  $cdata->{cdata}; }
+          $cdata->set_gi( PCDATA);
+          undef $cdata->{cdata};
+        }
+    }
+
+sub _is_private      { return _is_private_name( $_[0]->gi); }
+sub _is_private_name { return $_[0]=~ m{^#};                }
+
+
+} # end of block containing package globals ($pretty_print, $quotes, keep_encoding...)
+
+
+# SAX export methods
+sub toSAX1
+  { _toSAX(@_, \&_start_tag_data_SAX1, \&_end_tag_data_SAX1); }
+
+sub toSAX2
+  { _toSAX(@_, \&_start_tag_data_SAX2, \&_end_tag_data_SAX2); }
+
+sub _toSAX
+  { my( $elt, $handler, $start_tag_data, $end_tag_data)= @_;
+    if( $elt->{gi} >= $XML::Twig::SPECIAL_GI)
+      { my $data= $start_tag_data->( $elt);
+        _start_prefix_mapping( $elt, $handler, $data);
+        if( $data && (my $start_element = $handler->can( 'start_element')))
+          { $start_element->( $handler, $data) unless( $elt->_flushed); }
+      
+        foreach my $child ($elt->children)
+          { $child->_toSAX( $handler, $start_tag_data, $end_tag_data); }
+
+        if( (my $data= $end_tag_data->( $elt)) && (my $end_element = $handler->can( 'end_element')) )
+          { $end_element->( $handler, $data); }
+        _end_prefix_mapping( $elt, $handler);
+      }
+    else # text or special element
+      { if( (exists $elt->{'pcdata'}) && (my $characters= $handler->can( 'characters')))
+          { $characters->( $handler, { Data => $elt->{pcdata} });  }
+        elsif( (exists $elt->{'cdata'}))  
+          { if( my $start_cdata= $handler->can( 'start_cdata'))
+              { $start_cdata->( $handler); }
+            if( my $characters= $handler->can( 'characters'))
+              { $characters->( $handler, {Data => $elt->{cdata} });  }
+            if( my $end_cdata= $handler->can( 'end_cdata'))
+              { $end_cdata->( $handler); }
+          }
+        elsif( ((exists $elt->{'target'}))  && (my $pi= $handler->can( 'processing_instruction')))
+          { $pi->( $handler, { Target =>$elt->{target}, Data => $elt->{data} });  }
+        elsif( ((exists $elt->{'comment'}))  && (my $comment= $handler->can( 'comment')))
+          { $comment->( $handler, { Data => $elt->{comment} });  }
+        elsif( ((exists $elt->{'ent'})))
+          { 
+            if( my $se=   $handler->can( 'skipped_entity'))
+              { $se->( $handler, { Name => $elt->ent_name });  }
+            elsif( my $characters= $handler->can( 'characters'))
+              { if( defined $elt->ent_string)
+                  { $characters->( $handler, {Data => $elt->ent_string});  }
+                else
+                  { $characters->( $handler, {Data => $elt->ent_name});  }
+              }
+          }
+      
+      }
+  }
+  
+sub _start_tag_data_SAX1
+  { my( $elt)= @_;
+    my $name= $XML::Twig::index2gi[$elt->{'gi'}];
+    return if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#'));
+    my $attributes={};
+    my $atts= $elt->atts;
+    while( my( $att, $value)= each %$atts)
+      { $attributes->{$att}= $value unless( (substr( $att, 0, 1) eq '#')); }
+    my $data= { Name => $name, Attributes => $attributes};
+    return $data;
+  }
+
+sub _end_tag_data_SAX1
+  { my( $elt)= @_;
+    return if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#'));
+    return  { Name => $XML::Twig::index2gi[$elt->{'gi'}] };
+  } 
+  
+sub _start_tag_data_SAX2
+  { my( $elt)= @_;
+    my $data={};
+    
+    my $name= $XML::Twig::index2gi[$elt->{'gi'}];
+    return if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#'));
+    $data->{Name}         = $name;
+    $data->{Prefix}       = $elt->ns_prefix; 
+    $data->{LocalName}    = $elt->local_name;
+    $data->{NamespaceURI} = $elt->namespace;
+
+    # save a copy of the data so we can re-use it for the end tag
+    my %sax2_data= %$data;
+    $elt->{twig_elt_SAX2_data}= \%sax2_data;
+   
+    # add the attributes
+    $data->{Attributes}= $elt->_atts_to_SAX2;
+
+    return $data;
+  }
+
+sub _atts_to_SAX2
+  { my $elt= shift;
+    my $SAX2_atts= {};
+    foreach my $att (keys %{$elt->atts})
+      { 
+        next if( (substr( $att, 0, 1) eq '#'));
+        my $SAX2_att={};
+        $SAX2_att->{Name}         = $att;
+        $SAX2_att->{Prefix}       = _ns_prefix( $att); 
+        $SAX2_att->{LocalName}    = _local_name( $att);
+        $SAX2_att->{NamespaceURI} = $elt->namespace( $SAX2_att->{Prefix});
+        $SAX2_att->{Value}        = $elt->{'att'}->{$att};
+        my $SAX2_att_name= "{$SAX2_att->{NamespaceURI}}$SAX2_att->{LocalName}";
+
+        $SAX2_atts->{$SAX2_att_name}= $SAX2_att;
+      }
+    return $SAX2_atts;
+  }
+
+sub _start_prefix_mapping
+  { my( $elt, $handler, $data)= @_;
+    if( my $start_prefix_mapping= $handler->can( 'start_prefix_mapping')
+        and my @new_prefix_mappings= grep { /^\{[^}]*\}xmlns/ || /^\{$XMLNS_URI\}/ } keys %{$data->{Attributes}}
+      )
+      { foreach my $prefix (@new_prefix_mappings)
+          { my $prefix_string= $data->{Attributes}->{$prefix}->{LocalName};
+            if( $prefix_string eq 'xmlns') { $prefix_string=''; }
+            my $prefix_data=
+              {  Prefix       => $prefix_string,
+                 NamespaceURI => $data->{Attributes}->{$prefix}->{Value}
+              };
+            $start_prefix_mapping->( $handler, $prefix_data);
+            $elt->{twig_end_prefix_mapping} ||= [];
+            push @{$elt->{twig_end_prefix_mapping}}, $prefix_string;
+          }
+      }
+  }
+
+sub _end_prefix_mapping
+  { my( $elt, $handler)= @_;
+    if( my $end_prefix_mapping= $handler->can( 'end_prefix_mapping'))
+      { foreach my $prefix (@{$elt->{twig_end_prefix_mapping}})
+          { $end_prefix_mapping->( $handler, { Prefix => $prefix} ); }
+      }
+  }
+             
+sub _end_tag_data_SAX2
+  { my( $elt)= @_;
+    return if( (substr( $XML::Twig::index2gi[$elt->{'gi'}], 0, 1) eq '#'));
+    return $elt->{twig_elt_SAX2_data};
+  } 
+
+
+
+#start-extract twig_node
+sub contains_text
+  { my $elt= shift;
+    my $child= $elt->{first_child};
+    while ($child)
+      { return 1 if( $child->is_text || (exists $child->{'ent'})); 
+        $child= $child->{next_sibling};
+      }
+    return 0;
+  }
+
+#end-extract twig_node
+
+# creates a single pcdata element containing the text as child of the element
+# options: 
+#   - force_pcdata: when set to a true value forces the text to be in a#PCDATA
+#                   even if the original element was a #CDATA
+sub set_text
+  { my( $elt, $string, %option)= @_;
+
+    if( $XML::Twig::index2gi[$elt->{'gi'}] eq PCDATA) 
+      { return $elt->{pcdata}=  $string; }
+    elsif( $XML::Twig::index2gi[$elt->{'gi'}] eq CDATA)  
+      { if( $option{force_pcdata})
+          { $elt->set_gi( PCDATA);
+            $elt->{cdata}= '';
+            return $elt->{pcdata}=  $string;
+      }
+    else
+      { return $elt->{cdata}=  $string; }
+      }
+
+    foreach my $child (@{[$elt->children]})
+      { $child->delete; }
+
+    my $pcdata= XML::Twig::Elt->new( PCDATA, $string);
+    $pcdata->paste( $elt);
+
+    delete $elt->{empty};
+
+    return $elt;
+  }
+
+# set the content of an element from a list of strings and elements
+sub set_content
+  { my $elt= shift;
+
+    return $elt unless defined $_[0];
+
+    # attributes can be given as a hash (passed by ref)
+    if( ref $_[0] eq 'HASH')
+      { my $atts= shift;
+        $elt->del_atts; # usually useless but better safe than sorry
+        $elt->set_atts( $atts);
+        return  $elt unless defined $_[0];
+      }
+
+    # check next argument for #EMPTY
+    if( !(ref $_[0]) && ($_[0] eq EMPTY) ) 
+      { $elt->{empty}= 1; return $elt; }
+
+    # case where we really want to do a set_text, the element is '#PCDATA'
+    # and we only want to add text in it
+    if( ($XML::Twig::index2gi[$elt->{'gi'}] eq PCDATA) && ($#_ == 0) && !( ref $_[0]))
+      { $elt->{pcdata}=  $_[0];
+        return $elt;
+      }
+    elsif( ($XML::Twig::index2gi[$elt->{'gi'}] eq CDATA) && ($#_ == 0) && !( ref $_[0]))
+      { $elt->{cdata}=  $_[0];
+        return $elt;
+      }
+
+    # delete the children
+    # WARNING: potential problem here if the children are used
+    # somewhere else (where?). Will be solved when I use weak refs
+    foreach my $child (@{[$elt->children]})
+      { $child->delete; }
+
+    foreach my $child (@_)
+      { if( isa( $child, 'XML::Twig::Elt'))
+          { # argument is an element
+            $child->paste( 'last_child', $elt);
+          }
+        else
+          { # argument is a string
+            if( (my $pcdata= $elt->{last_child}) && $elt->{last_child}->is_pcdata)
+              { # previous child is also pcdata: just concatenate
+                $pcdata->{pcdata}=  $pcdata->{pcdata} . $child 
+              }
+            else
+              { # previous child is not a string: creat a new pcdata element
+                $pcdata= XML::Twig::Elt->new( PCDATA, $child);
+                $pcdata->paste( 'last_child', $elt);  
+              }
+          }
+      }
+
+    delete $elt->{empty};
+
+    return $elt;
+  }
+
+# inserts an element (whose gi is given) as child of the element
+# all children of the element are now children of the new element
+# returns the new element
+sub insert
+  { my ($elt, @args)= @_;
+    # first cut the children
+    my @children= $elt->children;
+    foreach my $child (@children)
+      { $child->cut; }
+    # insert elements
+    while( my $gi= shift @args)
+      { my $new_elt= XML::Twig::Elt->new( $gi);
+        # add attributes if needed
+        if( defined( $args[0]) && ( isa( $args[0], 'HASH')) )
+          { $new_elt->set_atts( shift @args); }
+        # paste the element
+        $new_elt->paste( $elt);
+        delete $elt->{empty};
+        $elt= $new_elt;
+      }
+    # paste back the children
+    foreach my $child (@children)
+      { $child->paste( 'last_child', $elt); }
+    return $elt;
+  }
+
+# insert a new element 
+# $elt->insert_new_element( $opt_position, $gi, $opt_atts_hash, @opt_content); 
+# the element is created with the same syntax as new
+# position is the same as in paste, first_child by default
+sub insert_new_elt
+  { my $elt= shift;
+    my $position= $_[0];
+    if(     ($position eq 'before') || ($position eq 'after')
+         || ($position eq 'first_child') || ($position eq 'last_child'))
+      { shift; }
+    else
+      { $position= 'first_child'; }
+
+    my $new_elt= $elt->new( @_);
+    $new_elt->paste( $position, $elt);
+    return $new_elt;
+  }
+
+# wraps an element in elements which gi's are given as arguments
+# $elt->wrap_in( 'td', 'tr', 'table') wraps the element as a single
+# cell in a table for example
+# returns the new element
+sub wrap_in
+  { my $elt= shift;
+    while( my $gi = shift @_)
+      { my $new_elt = XML::Twig::Elt->new( $gi);
+        if( $elt->{twig_current})
+          { my $t= $elt->twig;
+            $t->{twig_current}= $new_elt;
+            delete $elt->{'twig_current'};
+            $new_elt->{'twig_current'}=1;
+          }
+
+        if( my $parent= $elt->{parent})
+          { $new_elt->set_parent( $parent); 
+            $parent->{first_child}=  $new_elt if( $parent->{first_child} == $elt);
+            $parent->set_last_child( $new_elt)  if( $parent->{last_child} == $elt);
+          }
+        else
+          { # wrapping the root
+            my $twig= $elt->twig;
+            if( $twig && $twig->root && ($twig->root eq $elt) )
+              { $twig->{twig_root}= $new_elt; }
+          }
+
+        if( my $prev_sibling= $elt->{prev_sibling})
+          { $new_elt->set_prev_sibling( $prev_sibling);
+            $prev_sibling->{next_sibling}=  $new_elt;
+          }
+
+        if( my $next_sibling= $elt->{next_sibling})
+          { $new_elt->{next_sibling}=  $next_sibling;
+            $next_sibling->set_prev_sibling( $new_elt);
+          }
+        $new_elt->{first_child}=  $elt;
+        $new_elt->set_last_child( $elt);
+
+        $elt->set_parent( $new_elt);
+        $elt->set_prev_sibling( undef);
+        $elt->{next_sibling}=  undef;
+
+        # add the attributes if the next argument is a hash ref
+        if( defined( $_[0]) && (isa( $_[0], 'HASH')) )
+          { $new_elt->set_atts( shift @_); }
+
+        $elt= $new_elt;
+      }
+      
+    return $elt;
+  }
+
+sub replace
+  { my( $elt, $ref)= @_;
+    if( my $parent= $ref->{parent})
+      { $elt->set_parent( $parent);
+        $parent->{first_child}=  $elt if( $parent->{first_child} == $ref);
+        $parent->set_last_child( $elt)  if( $parent->{last_child} == $ref);
+      }
+    if( my $prev_sibling= $ref->{prev_sibling})
+      { $elt->set_prev_sibling( $prev_sibling);
+        $prev_sibling->{next_sibling}=  $elt;
+      }
+    if( my $next_sibling= $ref->{next_sibling})
+      { $elt->{next_sibling}=  $next_sibling;
+        $next_sibling->set_prev_sibling( $elt);
+      }
+   
+    $ref->set_parent( undef);
+    $ref->set_prev_sibling( undef);
+    $ref->{next_sibling}=  undef;
+    return $ref;
+  }
+
+sub replace_with
+  { my $ref= shift;
+    my $elt= shift;
+    $elt->replace( $ref);
+    foreach my $new_elt (reverse @_)
+      { $new_elt->paste( after => $elt); }
+    return $elt;
+  }
+
+
+#start-extract twig_node
+# move an element, same syntax as paste, except the element is first cut
+sub move
+  { my $elt= shift;
+    $elt->cut;
+    $elt->paste( @_);
+    return $elt;
+  }
+#end-extract twig_node
+
+
+# adds a prefix to an element, creating a pcdata child if needed
+sub prefix
+  { my ($elt, $prefix, $option)= @_;
+    my $asis= ($option && ($option eq 'asis')) ? 1 : 0;
+    if( (exists $elt->{'pcdata'}) 
+        && (($asis && $elt->{asis}) || (!$asis && ! $elt->{asis}))
+      )
+      { $elt->{pcdata}=  $prefix . $elt->{pcdata}; }
+    elsif( $elt->{first_child} && $elt->{first_child}->is_pcdata
+        && (   ($asis && $elt->{first_child}->{asis}) 
+            || (!$asis && ! $elt->{first_child}->{asis}))
+         )
+      { $elt->{first_child}->set_pcdata( $prefix . $elt->{first_child}->pcdata); }
+    else
+      { my $new_elt= XML::Twig::Elt->new( PCDATA, $prefix);
+        $new_elt->paste( $elt);
+        $new_elt->set_asis if( $asis);
+      }
+    return $elt;
+  }
+
+# adds a suffix to an element, creating a pcdata child if needed
+sub suffix
+  { my ($elt, $suffix, $option)= @_;
+    my $asis= ($option && ($option eq 'asis')) ? 1 : 0;
+    if( (exists $elt->{'pcdata'})
+        && (($asis && $elt->{asis}) || (!$asis && ! $elt->{asis}))
+      )
+      { $elt->{pcdata}=  $elt->{pcdata} . $suffix; }
+    elsif( $elt->{last_child} && $elt->{last_child}->is_pcdata
+        && (   ($asis && $elt->{last_child}->{asis}) 
+            || (!$asis && ! $elt->{last_child}->{asis}))
+         )
+      { $elt->{last_child}->set_pcdata( $elt->{last_child}->pcdata . $suffix); }
+    else
+      { my $new_elt= XML::Twig::Elt->new( PCDATA, $suffix);
+        $new_elt->paste( 'last_child', $elt);
+        $new_elt->set_asis if( $asis);
+      }
+    return $elt;
+  }
+
+#start-extract twig_node
+# create a path to an element ('/root/.../gi)
+sub path
+  { my $elt= shift;
+    my @context= ( $elt, $elt->ancestors);
+    return "/" . join( "/", reverse map {$_->gi} @context);
+  }
+
+sub xpath
+  { my $elt= shift;
+    my $xpath;
+    foreach my $ancestor (reverse $elt->ancestors_or_self)
+      { my $gi= $XML::Twig::index2gi[$ancestor->{'gi'}];
+        $xpath.= "/$gi";
+        my $index= $ancestor->prev_siblings( $gi) + 1;
+        unless( ($index == 1) && !$ancestor->next_sibling( $gi))
+          { $xpath.= "[$index]"; }
+      }
+    return $xpath;
+  }
+
+# methods used mainly by wrap_children
+
+# return a string with the 
+# for an element <foo><elt att="val">...</elt><elt2/><elt>...</elt></foo>
+# returns '<elt att="val"><elt2><elt>'
+sub _stringify_struct
+  { my( $elt, %opt)= @_;
+    my $string;
+    my $pretty_print= XML::Twig::Elt::set_pretty_print( 'none');
+    foreach my $child ($elt->children)
+      { $child->add_id; $string .= $child->start_tag; }
+    XML::Twig::Elt::set_pretty_print( $pretty_print);
+    return $string;
+  }
+
+# wrap a series of elements in a new one
+sub _wrap_range
+  { my $elt= shift;
+    my $gi= shift;
+    my $atts= isa( $_[0], 'HASH') ? shift : undef;
+    my $range= shift; # the string with the tags to wrap
+
+    my $t= $elt->twig;
+
+    # get the tags to wrap
+    my @to_wrap;
+    while( $range=~ m{<\w+\s+[^>]*id=("[^"]*"|'[^']*')[^>]*>}g)
+      { push @to_wrap, $t->elt_id( substr( $1, 1, -1)); }
+
+    return '' unless @to_wrap;
+    
+    my $to_wrap= shift @to_wrap;
+    my %atts= %$atts;
+    my $new_elt= $to_wrap->wrap_in( $gi, \%atts);
+    $_->move( last_child => $new_elt) foreach (@to_wrap);
+
+    return '';
+  }
+    
+# wrap children matching a regexp in a new element
+sub wrap_children
+  { my( $elt, $regexp, $gi, $atts)= @_;
+
+    $atts ||={};
+
+    my $elt_as_string= $elt->_stringify_struct; # stringify the elt structure
+    $regexp=~ s{(<[^>]*>)}{_match_expr( $1)}eg; # in the regexp, replace gi's by the proper regexp 
+    $elt_as_string=~ s{($regexp)}{$elt->_wrap_range( $gi, $atts, $1)}eg; # then do the actual replace
+  
+    return $elt; 
+  }
+
+sub _match_expr
+  { my $tag= shift;
+    my( $gi, %atts)= XML::Twig::_parse_start_tag( $tag);
+    return _match_tag( $gi, %atts);
+  }
+
+
+sub _match_tag
+  { my( $elt, %atts)= @_;
+    my $string= "<$elt\\b";
+    foreach my $key (sort keys %atts)
+      { my $val= $atts{$key};
+        $val= "\Q$val\E";
+       # my $val= qq{\Q$atts{$key}\E};
+        $string.= qq{[^>]*$key=(?:"$val"|'$val')};
+      }
+    $string.=  qq{[^>]*>};
+    return "(?:$string)";
+  }
+
+sub field_to_att
+  { my( $elt, $cond, $att)= @_;
+    $att ||= $cond;
+    my $child= $elt->first_child( $cond) or return undef;
+    $elt->set_att( $att => $child->text);
+    $child->cut;
+    return $elt;
+  }
+
+sub att_to_field
+  { my( $elt, $att, $tag)= @_;
+    $tag ||= $att;
+    my $child= $elt->insert_new_elt( first_child => $tag, $elt->{'att'}->{$att});
+    $elt->del_att( $att);
+    return $elt;
+  }
+
+# sort children methods
+
+sub sort_children_on_field
+  { my $elt   = shift;
+    my $field = shift;
+    my $get_key= sub { return $_[0]->field( $field) };
+    return $elt->sort_children( $get_key, @_); 
+  }
+
+sub sort_children_on_att
+  { my $elt = shift;
+    my $att = shift;
+    my $get_key= sub { return $_[0]->{'att'}->{$att} };
+    return $elt->sort_children( $get_key, @_); 
+  }
+
+sub sort_children_on_value
+  { my $elt   = shift;
+    #my $get_key= eval qq{ sub { return \$_[0]->text } };
+    my $get_key= \&text;
+    return $elt->sort_children( $get_key, @_); 
+  }
+
+
+sub sort_children
+  { my( $elt, $get_key, %opt)=@_;
+    $opt{order} ||= 'normal';
+    $opt{type}  ||= 'alpha';
+    my( $par_a, $par_b)= ($opt{order} eq 'reverse') ? qw( b a) : qw ( a b) ;
+    my $op= ($opt{type} eq 'numeric') ? '<=>' : 'cmp' ;
+    my @children= $elt->cut_children;
+    if( $opt{type} eq 'numeric')
+      {  @children= map  { $_->[1] }
+                    sort { $a->[0] <=> $b->[0] }
+                    map  { [ $get_key->( $_), $_] } @children;
+      }
+    elsif( $opt{type} eq 'alpha')
+      {  @children= map  { $_->[1] }
+                    sort { $a->[0] cmp $b->[0] }
+                    map  { [ $get_key->( $_), $_] } @children;
+      }
+    else
+      { croak "wrong sort type '$opt{type}', should be either 'alpha' or 'numeric'"; }
+
+    @children= reverse @children if( $opt{order} eq 'reverse');
+    $elt->set_content( @children);
+  }
+
+
+# comparison methods
+
+sub before
+  { my( $a, $b)=@_;
+    if( $a->cmp( $b) == -1) { return 1; } else { return 0; }
+  }
+
+sub after
+  { my( $a, $b)=@_;
+    if( $a->cmp( $b) == 1) { return 1; } else { return 0; }
+  }
+
+sub lt
+  { my( $a, $b)=@_;
+    return 1 if( $a->cmp( $b) == -1);
+    return 0;
+  }
+
+sub le
+  { my( $a, $b)=@_;
+    return 1 unless( $a->cmp( $b) == 1);
+    return 0;
+  }
+
+sub gt
+  { my( $a, $b)=@_;
+    return 1 if( $a->cmp( $b) == 1);
+    return 0;
+  }
+
+sub ge
+  { my( $a, $b)=@_;
+    return 1 unless( $a->cmp( $b) == -1);
+    return 0;
+  }
+
+
+sub cmp
+  { my( $a, $b)=@_;
+
+    # easy cases
+    return  0 if( $a == $b);    
+    return  1 if( $a->in($b)); # a starts after b 
+    return -1 if( $b->in($a)); # a starts before b
+
+    # ancestors does not include the element itself
+    my @a_pile= ($a, $a->ancestors); 
+    my @b_pile= ($b, $b->ancestors);
+
+    # the 2 elements are not in the same twig
+    return undef unless( $a_pile[-1] == $b_pile[-1]);
+
+    # find the first non common ancestors (they are siblings)
+    my $a_anc= pop @a_pile;
+    my $b_anc= pop @b_pile;
+
+    while( $a_anc == $b_anc) 
+      { $a_anc= pop @a_pile;
+        $b_anc= pop @b_pile;
+      }
+
+    # from there move left and right and figure out the order
+    my( $a_prev, $a_next, $b_prev, $b_next)= ($a_anc, $a_anc, $b_anc, $b_anc);
+    while()
+      { $a_prev= $a_prev->{prev_sibling} || return( -1);
+        return 1 if( $a_prev == $b_next);
+        $a_next= $a_next->{next_sibling} || return( 1);
+        return -1 if( $a_next == $b_prev);
+        $b_prev= $b_prev->{prev_sibling} || return( 1);
+        return -1 if( $b_prev == $a_next);
+        $b_next= $b_next->{next_sibling} || return( -1);
+        return 1 if( $b_next == $a_prev);
+      }
+  }
+    
+#end-extract twig_node
+
+1;
+
+
+__END__
+
+=head1 NAME
+
+XML::Twig - A perl module for processing huge XML documents in tree mode.
+
+=head1 SYNOPSIS
+
+Small documents (loaded in memory as a tree):
+
+  my $twig=XML::Twig->new();    # create the twig
+  $twig->parsefile( 'doc.xml'); # build it
+  my_process( $twig);           # use twig methods to process it 
+  $twig->print;                 # output the twig
+
+Huge documents (processed in combined stream/tree mode):
+
+  # at most one div will be loaded in memory
+  my $twig=XML::Twig->new(   
+    twig_handlers => 
+      { title   => sub { $_->set_gi( 'h2') }, # change title tags to h2
+        para    => sub { $_->set_gi( 'p')  }, # change para to p
+        hidden  => sub { $_->delete;       }, # remove hidden elements
+        list    => \&my_list_process,         # process list elements
+        div     => sub { $_[0]->flush;     }, # output and free memory
+      },
+    pretty_print => 'indented',               # output will be nicely formatted
+    empty_tags   => 'html',                   # outputs <empty_tag />
+                         );
+    $twig->flush;                             # flush the end of the document
+
+See L<XML::Twig 101|XML::Twig 101> for other ways to use the module, as a 
+filter for example
+
+Note that this documentation is intended as a reference to the module. A 
+tutorial is available at http://www.xmltwig.com/xmltwig/tutorial/index.html
+and a FAQ is at http://www.xmltwig.com/xmltwig/XML-Twig-FAQ.html
+
+=head1 DESCRIPTION
+
+This module provides a way to process XML documents. It is build on top
+of C<XML::Parser>.
+
+The module offers a tree interface to the document, while allowing you
+to output the parts of it that have been completely processed.
+
+It allows minimal resource (CPU and memory) usage by building the tree
+only for the parts of the documents that need actual processing, through the 
+use of the C<L<twig_roots|twig_roots> > and 
+C<L<twig_print_outside_roots|twig_print_outside_roots> > options. The 
+C<L<finish|finish> > and C<L<finish_print|finish_print> > methods also help 
+to increase performances.
+
+XML::Twig tries to make simple things easy so it tries its best to takes care 
+of a lot of the (usually) annoying (but sometimes necessary) features that 
+come with XML and XML::Parser.
+
+=head1 XML::Twig 101
+
+XML::Twig can be used either on "small" XML documents (that fit in memory)
+or on huge ones, by processing parts of the document and outputting or
+discarding them once they are processed.
+
+
+=head2 Loading an XML document and processing it
+
+  my $t= XML::Twig->new();
+  $t->parse( '<d><title>title</title><para>p 1</para><para>p 2</para></d>');
+  my $root= $t->root;
+  $root->set_gi( 'html');               # change doc to html
+  $title= $root->first_child( 'title'); # get the title
+  $title->set_gi( 'h1');                # turn it into h1
+  my @para= $root->children( 'para');   # get the para children
+  foreach my $para (@para)
+    { $para->set_gi( 'p'); }            # turn them into p
+  $t->print;                            # output the document
+
+Other useful methods include:
+
+L<att|att>: C<< $elt->{'att'}->{'foo'} >> return the C<foo> attribute for an 
+element,
+
+L<set_att|set_att> : C<< $elt->set_att( foo => "bar") >> sets the C<foo> 
+attribute to the C<bar> value,
+
+L<next_sibling|next_sibling>: C<< $elt->{next_sibling} >> return the next sibling
+in the document (in the example C<< $title->{next_sibling} >> is the first
+C<para>, you can also (and actually should) use 
+C<< $elt->next_sibling( 'para') >> to get it 
+
+The document can also be transformed through the use of the L<cut|cut>, 
+L<copy|copy>, L<paste|paste> and L<move|move> methods: 
+C<< $title->cut; $title->paste( after => $p); >> for example
+
+And much, much more, see L<Elt|"Elt">.
+
+=head2 Processing an XML document chunk by chunk
+
+One of the strengths of XML::Twig is that it let you work with files that do 
+not fit in memory (BTW storing an XML document in memory as a tree is quite
+memory-expensive, the expansion factor being often around 10).
+
+To do this you can define handlers, that will be called once a specific 
+element has been completely parsed. In these handlers you can access the
+element and process it as you see fit, using the navigation and the
+cut-n-paste methods, plus lots of convenient ones like C<L<prefix|prefix> >.
+Once the element is completely processed you can then C<L<flush|flush> > it, 
+which will output it and free the memory. You can also C<L<purge|purge> > it 
+if you don't need to output it (if you are just extracting some data from 
+the document for example). The handler will be called again once the next 
+relevant element has been parsed.
+
+  my $t= XML::Twig->new( twig_handlers => 
+                          { section => \&section,
+                            para   => sub { $_->set_tag( 'p');
+                          },
+                       );
+  $t->parsefile( 'doc.xml');
+  $t->flush; # don't forget to flush one last time in the end or anything
+             # after the last </section> tag will not be output 
+    
+  # the handler is called once a section is completely parsed, ie when 
+  # the end tag for section is found, it receives the twig itself and
+  # the element (including all its sub-elements) as arguments
+  sub section 
+    { my( $t, $section)= @_;      # arguments for all twig_handlers
+      $section->set_tag( 'div');  # change the tag name.4, my favourite method...
+      # let's use the attribute nb as a prefix to the title
+      my $title= $section->first_child( 'title'); # find the title
+      my $nb= $title->{'att'}->{'nb'}; # get the attribute
+      $title->prefix( "$nb - ");  # easy isn't it?
+      $section->flush;            # outputs the section and frees memory
+    }
+
+        
+There is of course more to it: you can trigger handlers on more elaborate 
+conditions than just the name of the element, C<section/title> for example.
+
+  my $t= XML::Twig->new( twig_handlers => 
+                           { 'section/title' => sub { $_->print } }
+                       )
+                  ->parsefile( 'doc.xml');
+
+Here C<< sub { $_->print } >> simply prints the current element (C<$_> is aliased
+to the element in the handler).
+
+You can also trigger a handler on a test on an attribute:
+
+  my $t= XML::Twig->new( twig_handlers => 
+                      { 'section[@level="1"]' => sub { $_->print } }
+                       );
+                  ->parsefile( 'doc.xml');
+
+You can also use C<L<start_tag_handlers|start_tag_handlers> > to process an 
+element as soon as the start tag is found. Besides C<L<prefix|prefix> > you
+can also use C<L<suffix|suffix> >, 
+
+=head2 Processing just parts of an XML document
+
+The twig_roots mode builds only the required sub-trees from the document
+Anything outside of the twig roots will just be ignored:
+
+  my $t= XML::Twig->new( 
+       # the twig will include just the root and selected titles 
+           twig_roots   => { 'section/title' => \&print_n_purge,
+                             'annex/title'   => \&print_n_purge
+           }
+                      );
+  $t->parsefile( 'doc.xml');
+  
+  sub print_n_purge 
+    { my( $t, $elt)= @_;
+      print $elt->text;    # print the text (including sub-element texts)
+      $t->purge;           # frees the memory
+    }
+
+You can use that mode when you want to process parts of a documents but are
+not interested in the rest and you don't want to pay the price, either in
+time or memory, to build the tree for the it.
+
+
+=head2 Building an XML filter
+
+You can combine the C<twig_roots> and the C<twig_print_outside_roots> options to 
+build filters, which let you modify selected elements and will output the rest 
+of the document as is.
+
+This would convert prices in $ to prices in Euro in a document:
+
+  my $t= XML::Twig->new( 
+           twig_roots   => { 'price' => \&convert, },   # process prices 
+           twig_print_outside_roots => 1,               # print the rest
+                      );
+  $t->parsefile( 'doc.xml');
+ 
+  sub convert 
+    { my( $t, $price)= @_;
+      my $currency=  $price->{'att'}->{'currency'};          # get the currency
+      if( $currency eq 'USD')
+        { $usd_price= $price->text;                     # get the price
+          # %rate is just a conversion table 
+          my $euro_price= $usd_price * $rate{usd2euro};
+          $price->set_text( $euro_price);               # set the new price
+          $price->set_att( currency => 'EUR');          # don't forget this!
+        }
+      $price->print;                                    # output the price
+    }
+
+=head2 XML::Twig and various versions of Perl, XML::Parser and expat:
+
+Before being uploaded to CPAN, XML::Twig 3.14 has been tested under the 
+following environments:
+
+=over 4
+
+=item linux-x86
+
+perl 5.6.2 to 5.9.1, expat 1.95.2 to 1.95.7, XML::Parser 2.31, 2.33 and 2.34
+perl 5.6.2, XML::Parser 2.27 (which comes with its own version of expat)
+
+=item Mac OS X (10.2)
+
+Mac OS X: same as linux-x86, plus perl 5.5.4
+
+=item Solaris
+
+perl 5.6.1, expat 1.95.2, XML::Parser 2.31
+
+=item Windows 98
+
+perl 5.6.1 (Activestate build 635), XML::Parser 2.27
+perl 5.8.2 (Activestate build 808), XML::Parser 2.34
+
+Note that with Windows 98 and Perl 5.6.1 C<nmake> may freeze while trying to copy
+the tools (xml_grep, xml_print and xml_spellcheck), so you have to answer no 
+when asked if you want to install them.
+
+=back
+
+See L<http://testers.cpan.org/search?request=dist&dist=XML-Twig> for the
+CPAN testers reports on XML::Twig
+
+XML::Twig does B<NOT> work with expat 1.95.4
+XML::Twig only works with XML::Parser 2.27 in perl 5.6.*  
+XML::Parser 2.28 does not really work
+
+When in doubt, upgrade expat, XML::Parser and Scalar::Util
+
+=head1 Simplifying XML processing
+
+=over 4
+
+=item Whitespaces
+
+Whitespaces that look non-significant are discarded, this behaviour can be 
+controlled using the C<L<keep_spaces|keep_spaces> >, 
+C<L<keep_spaces_in|keep_spaces_in> > and 
+C<L<discard_spaces_in|discard_spaces_in> > options.
+
+=item Encoding
+
+You can specify that you want the output in the same encoding as the input
+(provided you have valid XML, which means you have to specify the encoding
+either in the document or when you create the Twig object) using the 
+C<L<keep_encoding|keep_encoding> > option
+
+You can also use C<L<output_encoding>> to convert the internal UTF-8 format
+to the required encoding.
+
+=item Comments and Processing Instructions (PI)
+
+Comments and PI's can be hidden from the processing, but still appear in the
+output (they are carried by the "real" element closer to them)
+
+=item Pretty Printing
+
+XML::Twig can output the document pretty printed so it is easier to read for
+us humans.
+
+=item Surviving an untimely death
+
+XML parsers are supposed to react violently when fed improper XML. 
+XML::Parser just dies.
+
+XML::Twig provides the C<L<safe_parse|safe_parse> > and the 
+C<L<safe_parsefile|safe_parsefile> > methods which wrap the parse in an eval
+and return either the parsed twig or 0 in case of failure.
+
+=item Private attributes
+
+Attributes with a name starting with # (illegal in XML) will not be
+output, so you can safely use them to store temporary values during
+processing. Note that you can store anything in a private attribute, 
+not just text, it's just a regular Perl variable, so a reference to
+an object or a huge data structure is perfectly fine.
+
+=back
+
+=head1 CLASSES
+
+XML::Twig uses a very limited number of classes. The ones you are most likely to use
+are C<L<XML::Twig>> of course, which represents a complete XML document, including the 
+document itself (the root of the document itself is C<L<root>>), its handlers, its
+input or output filters... The other main class is C<L<XML::Twig::Elt>>, which models 
+an XML element. Element here has a very wide definition: it can be a regular element, or
+but also text, with an element C<L<tag>> of C<#PCDATA> (or C<#CDATA>), an entity (tag is
+C<#ENT>), a Processing Instruction (C<#PI>), a comment (C<#COMMENT>). 
+
+Those are the 2 commonly used classes.
+
+You might want to look the C<L<elt_class>> option if you want to subclass C<XML::Twig::Elt>.
+
+Attributes are just attached to their parent element, they are not objects per se. (Please
+use the provided methods C<L<att>> and C<L<set_att>> to access them, if you access them
+as a hash, then your code becomes implementaion de[endant and might break in the future).
+
+Other classes that are seldom used are C<L<XML::Twig::Entity_list>> and C<L<XML::Twig::Entity>>.
+
+If you use C<L<XML::Twig::XPath>> instead of C<XML::Twig>, elements are then created as
+C<L<XML::Twig::XPath::Elt>>
+
+
+=head1 METHODS
+
+=head2 XML::Twig 
+
+A twig is a subclass of XML::Parser, so all XML::Parser methods can be
+called on a twig object, including parse and parsefile.
+C<setHandlers> on the other hand cannot be used, see C<L<BUGS|BUGS> >
+
+
+=over 4
+
+=item new 
+
+This is a class method, the constructor for XML::Twig. Options are passed
+as keyword value pairs. Recognized options are the same as XML::Parser,
+plus some XML::Twig specifics.
+
+New Options:
+
+=over 4
+
+=item twig_handlers
+
+This argument replaces the corresponding XML::Parser argument. It consists
+of a hash C<{ expression => \&handler}> where expression is a 
+I<generic_attribute_condition>, I<string_condition>,
+an I<attribute_condition>,I<full_path>, a I<partial_path>, a I<gi>,
+I<_default_> or I<_all_>.
+
+The idea is to support a usefull but efficient (thus limited) subset of
+XPATH. A fuller expression set will be supported in the future, as users
+ask for more and as I manage to implement it efficiently. This will never
+encompass all of XPATH due to the streaming nature of parsing (no lookahead
+after the element end tag).
+
+A B<generic_attribute_condition> is a condition on an attribute, in the form
+C<*[@att="val"]> or C<*[@att]>, simple quotes can be used instead of double 
+quotes and the leading '*' is actually optional. No matter what the gi of the
+element is, the handler will be triggered either if the attribute has the 
+specified value or if it just exists. 
+
+A B<string_condition> is a condition on the content of an element, in the form
+C<gi[string()="foo"]>, simple quotes can be used instead of double quotes, at 
+the moment you cannot escape the quotes (this will be added as soon as I
+dig out my copy of Mastering Regular Expressions from its storage box).
+The text returned is, as per what I (and Matt Sergeant!) understood from
+the XPATH spec the concatenation of all the text in the element, excluding
+all markup. Thus to call a handler on the elementC<< <p>text <b>bold</b></p> >>
+the appropriate condition is C<p[string()="text bold"]>. Note that this is not
+exactly conformant to the XPATH spec, it just tries to mimic it while being
+still quite concise. 
+
+A extension of that notation is C<gi[string(B<child_gi>)="foo"]> where the
+handler will be called if a child of a C<gi> element has a text value of 
+C<foo>.  At the moment only direct children of the C<gi> element are checked.
+If you need to test on descendants of the element let me know. The fix is
+trivial but would slow down the checks, so I'd like to keep it the way it is.
+
+A B<regexp_condition> is a condition on the content of an element, in the form
+C<gi[string()=~ /foo/"]>. This is the same as a string condition except that
+the text of the element is matched to the regexp. The C<i>, C<m>, C<s> and C<o>
+modifiers can be used on the regexp.
+
+The C<< gi[string(B<child_gi>)=~ /foo/"] >> extension is also supported.
+
+An B<attribute_condition> is a simple condition of an attribute of the
+current element in the form C<gi[@att="val"]> (simple quotes can be used
+instead of double quotes, you can escape quotes either). 
+If several attribute_condition are true the same element all the handlers
+can be called in turn (in the order in which they were first defined).
+If the C<="val"> part is ommited ( the condition is then C<gi[@att]>) then
+the handler is triggered if the attribute actually exists for the element,
+no matter what it's value is.
+
+A B<full_path> looks like C<'/doc/section/chapter/title'>, it starts with
+a / then gives all the gi's to the element. The handler will be called if
+the path to the current element (in the input document) is exactly as
+defined by the C<full_path>.
+
+A B<partial_path> is like a full_path except it does not start with a /:
+C<'chapter/title'> for example. The handler will be called if the path to
+the element (in the input document) ends as defined in the C<partial_path>.
+
+B<WARNING>: (hopefully temporary) at the moment C<string_condition>, 
+C<regexp_condition> and C<attribute_condition> are only supported on a 
+simple gi, not on a path.
+
+A B<gi> (generic identifier) is just a tag name.
+
+#CDATA can be used to call a handler for a CDATA.
+
+A special gi B<_all_> is used to call a function for each element.
+The special gi B<_default_> is used to call a handler for each element
+that does NOT have a specific handler.
+
+The order of precedence to trigger a handler is: 
+I<generic_attribute_condition>, I<string_condition>, I<regexp_condition>, 
+I<attribute_condition>, I<full_path>, longer I<partial_path>, shorter 
+I<partial_path>, I<gi>, I<_default_> . 
+
+B<Important>: once a handler has been triggered if it returns 0 then no other
+handler is called, exept a C<_all_> handler which will be called anyway.
+
+If a handler returns a true value and other handlers apply, then the next
+applicable handler will be called. Repeat, rince, lather..; The exception
+to that rule is when the C<L<do_not_chain_handlers|do_not_chain_handlers>>
+option is set, in which case only the first handler will be called.
+
+Note that it might be a good idea to explicitely return a short true value
+(like 1) from handlers: this ensures that other applicable handlers are 
+called even if the last statement for the handler happens to evaluate to
+false. This might also speedup the code by avoiding the result of the last 
+statement of the code to be copied and passed to the code managing handlers.
+It can really pay to have 1 instead of a long string returned.
+
+When an element is CLOSED the corresponding handler is called, with 2
+arguments: the twig and the C<L</Element|/Element> >. The twig includes the 
+document tree that has been built so far, the element is the complete sub-tree
+for the element. This means that handlers for inner elements are called before
+handlers for outer elements.
+
+C<$_> is also set to the element, so it is easy to write inline handlers like
+
+  para => sub { $_->change_gi( 'p'); }
+
+Text is stored in elements where gi is #PCDATA (due to mixed content, text
+and sub-element in an element there is no way to store the text as just an
+attribute of the enclosing element).
+
+B<Warning>: if you have used purge or flush on the twig the element might not
+be complete, some of its children might have been entirely flushed or purged,
+and the start tag might even have been printed (by C<flush>) already, so changing
+its gi might not give the expected result.
+
+More generally, the I<full_path>, I<partial_path> and I<gi> expressions are
+evaluated against the input document. Which means that even if you have changed
+the gi of an element (changing the gi of a parent element from a handler for
+example) the change will not impact the expression evaluation. Attributes in
+I<attribute_condition> are different though. As the initial value of attribute
+is not stored the handler will be triggered if the B<current> attribute/value
+pair is found when the element end tag is found. Although this can be quite
+confusing it should not impact most of users, and allow others to play clever
+tricks with temporary attributes. Let me know if this is a problem for you.
+
+
+=item twig_roots
+
+This argument let's you build the tree only for those elements you are
+interested in. 
+
+  Example: my $t= XML::Twig->new( twig_roots => { title => 1, subtitle => 1});
+           $t->parsefile( file);
+           my $t= XML::Twig->new( twig_roots => { 'section/title' => 1});
+           $t->parsefile( file);
+
+
+return a twig containing a document including only C<title> and C<subtitle> 
+elements, as children of the root element.
+
+You can use I<generic_attribute_condition>, I<attribute_condition>,
+I<full_path>, I<partial_path>, I<gi>, I<_default_> and I<_all_> to 
+trigger the building of the twig. 
+I<string_condition> and I<regexp_condition> cannot be used as the content 
+of the element, and the string, have not yet been parsed when the condition
+is checked.
+
+B<WARNING>: path are checked for the document. Even if the C<twig_roots> option
+is used they will be checked against the full document tree, not the virtual
+tree created by XML::Twig
+
+
+B<WARNING>: twig_roots elements should NOT be nested, that would hopelessly
+confuse XML::Twig ;--(
+
+Note: you can set handlers (twig_handlers) using twig_roots
+  Example: my $t= XML::Twig->new( twig_roots => 
+                                   { title    => sub { $_{1]->print;}, 
+                                     subtitle => \&process_subtitle 
+                                   }
+                               );
+           $t->parsefile( file);
+ 
+
+=item twig_print_outside_roots
+
+To be used in conjunction with the C<twig_roots> argument. When set to a true 
+value this will print the document outside of the C<twig_roots> elements.
+
+ Example: my $t= XML::Twig->new( twig_roots => { title => \&number_title },
+                                twig_print_outside_roots => 1,
+                               );
+           $t->parsefile( file);
+           { my $nb;
+           sub number_title
+             { my( $twig, $title);
+               $nb++;
+               $title->prefix( "$nb "; }
+               $title->print;
+             }
+           }
+               
+
+This example prints the document outside of the title element, calls 
+C<number_title> for each C<title> element, prints it, and then resumes printing
+the document. The twig is built only for the C<title> elements. 
+
+If the value is a reference to a file handle then the document outside the
+C<twig_roots> elements will be output to this file handle:
+
+  open( OUT, ">out_file") or die "cannot open out file out_file:$!";
+  my $t= XML::Twig->new( twig_roots => { title => \&number_title },
+                         # default output to OUT
+                         twig_print_outside_roots => \*OUT, 
+                       );
+
+         { my $nb;
+           sub number_title
+             { my( $twig, $title);
+               $nb++;
+               $title->prefix( "$nb "; }
+               $title->print( \*OUT);    # you have to print to \*OUT here
+             }
+           }
+
+
+=item start_tag_handlers
+
+A hash C<{ expression => \&handler}>. Sets element handlers that are called when
+the element is open (at the end of the XML::Parser C<Start> handler). The handlers
+are called with 2 params: the twig and the element. The element is empty at 
+that point, its attributes are created though. 
+
+You can use I<generic_attribute_condition>, I<attribute_condition>,
+I<full_path>, I<partial_path>, I<gi>, I<_default_>  and I<_all_> to trigger 
+the handler. 
+
+I<string_condition> and I<regexp_condition> cannot be used as the content of 
+the element, and the string, have not yet been parsed when the condition is 
+checked.
+
+The main uses for those handlers are to change the tag name (you might have to 
+do it as soon as you find the open tag if you plan to C<flush> the twig at some
+point in the element, and to create temporary attributes that will be used
+when processing sub-element with C<twig_hanlders>. 
+
+You should also use it to change tags if you use C<flush>. If you change the tag 
+in a regular C<twig_handler> then the start tag might already have been flushed. 
+
+B<Note>: C<start_tag> handlers can be called outside of C<twig_roots> if this 
+argument is used, in this case handlers are called with the following arguments:
+C<$t> (the twig), C<$gi> (the gi of the element) and C<%att> (a hash of the 
+attributes of the element). 
+
+If the C<twig_print_outside_roots> argument is also used then the start tag
+will be printed if the last handler called returns a C<true> value, if it
+does not then the start tag will B<not> be printed (so you can print a
+modified string yourself for example);
+
+Note that you can use the L<ignore|ignore> method in C<start_tag_handlers> 
+(and only there). 
+
+=item end_tag_handlers
+
+A hash C<{ expression => \&handler}>. Sets element handlers that are called when
+the element is closed (at the end of the XML::Parser C<End> handler). The handlers
+are called with 2 params: the twig and the gi of the element. 
+
+I<twig_handlers> are called when an element is completely parsed, so why have 
+this redundant option? There is only one use for C<end_tag_handlers>: when using
+the C<twig_roots> option, to trigger a handler for an element B<outside> the roots.
+It is for example very useful to number titles in a document using nested 
+sections: 
+
+  my @no= (0);
+  my $no;
+  my $t= XML::Twig->new( 
+          start_tag_handlers => 
+           { section => sub { $no[$#no]++; $no= join '.', @no; push @no, 0; } },
+          twig_roots         => 
+           { title   => sub { $_[1]->prefix( $no); $_[1]->print; } },
+          end_tag_handlers   => { section => sub { pop @no;  } },
+          twig_print_outside_roots => 1
+                      );
+   $t->parsefile( $file);
+
+Using the C<end_tag_handlers> argument without C<twig_roots> will result in an
+error.
+
+=item do_not_chain_handlers
+
+If this option is set to a true value, then only one handler will be called for
+each element, even if several satisfy the condition
+
+Note that the C<_all_> handler will still be called regardeless
+
+=item ignore_elts
+
+This option lets you ignore elements when building the twig. This is useful 
+in cases where you cannot use C<twig_roots> to ignore elements, for example if
+the element to ignore is a sibling of elements you are interested in.
+
+Example:
+
+  my $twig= XML::Twig->new( ignore_elts => { elt => 1 });
+  $twig->parsefile( 'doc.xml');
+
+This will build the complete twig for the document, except that all C<elt> 
+elements (and their children) will be left out.
+
+
+=item char_handler
+
+A reference to a subroutine that will be called every time C<PCDATA> is found.
+
+=item elt_class
+
+The name of a class used to store elements. this class should inherit from
+C<XML::Twig::Elt> (and by default it is C<XML::Twig::Elt>). This option is used
+to subclass the element class and extend it with new methods.
+
+This option is needed because during the parsing of the XML, elements are created
+by C<XML::Twig>, without any control from the user code.
+
+=item keep_atts_order
+
+Setting this option to a true value causes the attribute hash to be tied to
+a Tie::IxHash object.
+This means that Tie::IxHash needs to be installe for this option to be 
+available. It also means that the hash keeps its order, so you will get 
+the attributes in order. This allows outputing the attributes in the same 
+order as they were in the original document.
+
+=item keep_encoding
+
+This is a (slightly?) evil option: if the XML document is not UTF-8 encoded and
+you want to keep it that way, then setting keep_encoding will use theC<Expat> 
+original_string method for character, thus keeping the original encoding, as 
+well as the original entities in the strings.
+
+See the C<t/test6.t> test file to see what results you can expect from the 
+various encoding options.
+
+B<WARNING>: if the original encoding is multi-byte then attribute parsing will
+be EXTREMELY unsafe under any Perl before 5.6, as it uses regular expressions
+which do not deal properly with multi-byte characters. You can specify an 
+alternate function to parse the start tags with the C<parse_start_tag> option 
+(see below)
+
+B<WARNING>: this option is NOT used when parsing with the non-blocking parser 
+(C<parse_start>, C<parse_more>, parse_done methods) which you probably should 
+not use with XML::Twig anyway as they are totally untested!
+
+=item output_encoding
+
+This option generates an output_filter using C<Encode>,  C<Text::Iconv> or 
+C<Unicode::Map8> and C<Unicode::Strings>, and sets the encoding in the XML
+declaration. This is the easiest way to deal with encodings, if you need 
+more sophisticated features, look at C<output_filter> below
+
+
+=item output_filter
+
+This option is used to convert the character encoding of the output document.
+It is passed either a string corresponding to a predefined filter or
+a subroutine reference. The filter will be called every time a document or 
+element is processed by the "print" functions (C<print>, C<sprint>, C<flush>). 
+
+Pre-defined filters: 
+
+=over 4 
+
+=item latin1 
+
+uses either C<Encode>, C<Text::Iconv> or C<Unicode::Map8> and C<Unicode::String>
+or a regexp (which works only with XML::Parser 2.27), in this order, to convert 
+all characters to ISO-8859-1 (aka latin1)
+
+=item html
+
+does the same conversion as C<latin1>, plus encodes entities using
+C<HTML::Entities> (oddly enough you will need to have HTML::Entities intalled 
+for it to be available). This should only be used if the tags and attribute 
+names themselves are in US-ASCII, or they will be converted and the output will
+not be valid XML any more
+
+=item safe
+
+converts the output to ASCII (US) only  plus I<character entities> (C<&#nnn;>) 
+this should be used only if the tags and attribute names themselves are in 
+US-ASCII, or they will be converted and the output will not be valid XML any 
+more
+
+=item safe_hex
+
+same as C<safe> except that the character entities are in hexa (C<&#xnnn;>)
+
+=item encode_convert ($encoding)
+
+Return a subref that can be used to convert utf8 strings to C<$encoding>).
+Uses C<Encode>.
+
+   my $conv = XML::Twig::encode_convert( 'latin1');
+   my $t = XML::Twig->new(output_filter => $conv);
+
+=item iconv_convert ($encoding)
+
+this function is used to create a filter subroutine that will be used to 
+convert the characters to the target encoding using C<Text::Iconv> (which needs
+to be installed, look at the documentation for the module and for the
+C<iconv> library to find out which encodings are available on your system)
+
+   my $conv = XML::Twig::iconv_convert( 'latin1');
+   my $t = XML::Twig->new(output_filter => $conv);
+
+=item unicode_convert ($encoding)
+
+this function is used to create a filter subroutine that will be used to 
+convert the characters to the target encoding using  C<Unicode::Strings> 
+and C<Unicode::Map8> (which need to be installed, look at the documentation 
+for the modules to find out which encodings are available on your system)
+
+   my $conv = XML::Twig::unicode_convert( 'latin1');
+   my $t = XML::Twig->new(output_filter => $conv);
+
+=back
+
+The C<text> and C<att> methods do not use the filter, so their 
+result are always in unicode.
+
+Those predeclared filters are based on subroutines that can be used
+by themselves (as C<XML::Twig::foo>). 
+
+=over 4
+
+=item html_encode ($string)
+
+Use C<HTML::Entities> to encode a utf8 string
+
+=item safe_encode ($string)
+
+Use either a regexp (perl < 5.8) or C<Encode> to encode non-ascii characters
+in the string in C<< &#<nnnn>; >> format
+
+=item safe_encode_hex ($string)
+
+Use either a regexp (perl < 5.8) or C<Encode> to encode non-ascii characters
+in the string in C<< &#x<nnnn>; >> format
+
+=item regexp2latin1 ($string)
+
+Use a regexp to encode a utf8 string into latin 1 (ISO-8859-1). Does not
+work with Perl 5.8.0!
+
+=back
+
+=item output_text_filter
+
+same as output_filter, except it doesn't apply to the brackets and quotes 
+around attribute values. This is useful for all filters that could change
+the tagging, basically anything that does not just change the encoding of
+the output. C<html>, C<safe> and C<safe_hex> are better used with this option.
+
+=item input_filter
+
+This option is similar to C<output_filter> except the filter is applied to 
+the characters before they are stored in the twig, at parsing time.
+
+=item parse_start_tag
+
+If you use the C<keep_encoding> option then this option can be used to replace
+the default parsing function. You should provide a coderef (a reference to a 
+subroutine) as the argument, this subroutine takes the original tag (given
+by XML::Parser::Expat C<original_string()> method) and returns a gi and the
+attributes in a hash (or in a list attribute_name/attribute value).
+
+=item expand_external_ents
+
+When this option is used external entities (that are defined) are expanded
+when the document is output using "print" functions such as C<L<print> >,
+C<L<sprint|sprint> >, C<L<flush|flush> > and C<L<xml_string|xml_string> >. 
+Note that in the twig the entity will be stored as an element whith a 
+gi 'C<#ENT>', the entity will not be expanded there, so you might want to 
+process the entities before outputting it. 
+
+=item load_DTD
+
+If this argument is set to a true value, C<parse> or C<parsefile> on the twig
+will load  the DTD information. This information can then be accessed through 
+the twig, in a C<DTD_handler> for example. This will load even an external DTD.
+
+Note that to do this the module will generate a temporary file in the current
+directory. If this is a problem let me know and I will add an option to
+specify an alternate directory.
+
+See L<DTD Handling|DTD Handling> for more information
+
+=item DTD_handler
+
+Set a handler that will be called once the doctype (and the DTD) have been 
+loaded, with 2 arguments, the twig and the DTD.
+
+=item no_prolog
+
+Does not output a prolog (XML declaration and DTD)
+
+=item id
+
+This optional argument gives the name of an attribute that can be used as
+an ID in the document. Elements whose ID is known can be accessed through
+the elt_id method. id defaults to 'id'.
+See C<L<BUGS|BUGS> >
+
+=item discard_spaces
+
+If this optional argument is set to a true value then spaces are discarded
+when they look non-significant: strings containing only spaces are discarded.
+This argument is set to true by default.
+
+=item keep_spaces
+
+If this optional argument is set to a true value then all spaces in the
+document are kept, and stored as C<PCDATA>.
+C<keep_spaces> and C<discard_spaces> cannot be both set.
+
+=item discard_spaces_in
+
+This argument sets C<keep_spaces> to true but will cause the twig builder to
+discard spaces in the elements listed.
+
+The syntax for using this argument is:
+ 
+  XML::Twig->new( discard_spaces_in => [ 'elt1', 'elt2']);
+
+=item keep_spaces_in
+
+This argument sets C<discard_spaces> to true but will cause the twig builder to
+keep spaces in the elements listed.
+
+The syntax for using this argument is: 
+
+  XML::Twig->new( keep_spaces_in => [ 'elt1', 'elt2']);
+
+=item pretty_print
+
+Set the pretty print method, amongst 'C<none>' (default), 'C<nsgmls>', 
+'C<nice>', 'C<indented>', 'C<indented_c>', 'C<record>' and 'C<record_c>'
+
+pretty_print formats:
+
+=over 4
+
+=item none
+
+The document is output as one ling string, with no line breaks except those 
+found within text elements
+
+=item nsgmls
+
+Line breaks are inserted in safe places: that is within tags, between a tag 
+and an attribute, between attributes and before the > at the end of a tag.
+
+This is quite ugly but better than C<none>, and it is very safe, the document 
+will still be valid (conforming to its DTD).
+
+This is how the SGML parser C<sgmls> splits documents, hence the name.
+
+=item nice
+
+This option inserts line breaks before any tag that does not contain text (so
+element with textual content are not broken as the \n is the significant).
+
+B<WARNING>: this option leaves the document well-formed but might make it
+invalid (not conformant to its DTD). If you have elements declared as
+
+  <!ELEMENT foo (#PCDATA|bar)>
+
+then a C<foo> element including a C<bar> one will be printed as
+
+  <foo>
+  <bar>bar is just pcdata</bar>
+  </foo>
+
+This is invalid, as the parser will take the line break after the C<foo> tag 
+as a sign that the element contains PCDATA, it will then die when it finds the 
+C<bar> tag. This may or may not be important for you, but be aware of it!
+
+=item indented
+
+Same as C<nice> (and with the same warning) but indents elements according to 
+their level 
+
+=item indented_c
+
+Same as C<indented> but a little more compact: the closing tags are on the 
+same line as the preceeding text
+
+=item record
+
+This is a record-oriented pretty print, that display data in records, one field 
+per line (which looks a LOT like C<indented>)
+
+=item record_c
+
+Stands for record compact, one record per line
+
+=back
+
+
+=item empty_tags
+
+Set the empty tag display style ('C<normal>', 'C<html>' or 'C<expand>').
+
+=item comments
+
+Set the way comments are processed: 'C<drop>' (default), 'C<keep>' or 
+'C<process>' 
+
+Comments processing options:
+
+=over 4
+
+=item drop
+
+drops the comments, they are not read, nor printed to the output
+
+=item keep
+
+comments are loaded and will appear on the output, they are not 
+accessible within the twig and will not interfere with processing
+though
+
+B<Bug>: comments in the middle of a text element such as 
+
+  <p>text <!-- comment --> more text --></p>
+
+are output at the end of the text:
+
+  <p>text  more text <!-- comment --></p>
+
+=item process
+
+comments are loaded in the twig and will be treated as regular elements 
+(their C<gi> is C<#COMMENT>) this can interfere with processing if you
+expect C<< $elt->{first_child} >> to be an element but find a comment there.
+Validation will not protect you from this as comments can happen anywhere.
+You can use C<< $elt->first_child( 'gi') >> (which is a good habit anyway)
+to get where you want. 
+
+Consider using C<process> if you are outputing SAX events from XML::Twig.
+
+=back
+
+=item pi
+
+Set the way processing instructions are processed: 'C<drop>', 'C<keep>' 
+(default) or 'C<process>'
+
+Note that you can also set PI handlers in the C<twig_handlers> option: 
+
+  '?'       => \&handler
+  '?target' => \&handler 2
+
+The handlers will be called with 2 parameters, the twig and the PI element if
+C<pi> is set to C<process>, and with 3, the twig, the target and the data if
+C<pi> is set to C<keep>. Of course they will not be called if C<pi> is set to 
+C<drop>.
+
+If C<pi> is set to C<keep> the handler should return a string that will be used
+as-is as the PI text (it should look like "C< <?target data?> >" or '' if you
+want to remove the PI), 
+
+Only one handler will be called, C<?target> or C<?> if no specific handler for
+that target is available.
+
+=item map_xmlns 
+
+This option is passed a hashref that maps uri's to prefixes. The prefixes in
+the document will be replaced by the ones in the map. The mapped prefixes can
+(actually have to) be used to trigger handlers, navigate or query the document.
+
+Here is an example:
+
+  my $t= XML::Twig->new( map_xmlns => {'http://www.w3.org/2000/svg' => "svg"},
+                         twig_handlers => 
+                           { 'svg:circle' => sub { $_->set_att( r => 20) } },
+                         pretty_print => 'indented', 
+                       )
+                  ->parse( '<doc xmlns:gr="http://www.w3.org/2000/svg">
+                              <gr:circle cx="10" cy="90" r="10"/>
+                           </doc>'
+                         )
+                  ->print;
+
+This will output:
+
+  <doc xmlns:svg="http://www.w3.org/2000/svg">
+     <svg:circle cx="10" cy="90" r="20"/>
+  </doc>
+
+Note that this only works when working on the entire twig. If you use C<twig_roots>
+then the namespace processing is not done. I have to keep something to do for the
+next release ;--)
+
+
+=item keep_original_prefix
+
+When used with C<L<map_xmlns>> this option will make C<XML::Twig> use the original
+namespace prefixes when outputing a document. The mapped prefix will still be used
+for triggering handlers and in navigation and query methods.
+
+  my $t= XML::Twig->new( map_xmlns => {'http://www.w3.org/2000/svg' => "svg"},
+                         twig_handlers => 
+                           { 'svg:circle' => sub { $_->set_att( r => 20) } },
+                         keep_original_prefix => 1,
+                         pretty_print => 'indented', 
+                       )
+                  ->parse( '<doc xmlns:gr="http://www.w3.org/2000/svg">
+                              <gr:circle cx="10" cy="90" r="10"/>
+                           </doc>'
+                         )
+                  ->print;
+
+This will output:
+
+  <doc xmlns:gr="http://www.w3.org/2000/svg">
+     <gr:circle cx="10" cy="90" r="20"/>
+  </doc>
+
+=back
+
+B<Note>: I _HATE_ the Java-like name of arguments used by most XML modules.
+So in pure TIMTOWTDI fashion all arguments can be written either as
+C<UglyJavaLikeName> or as C<readable_perl_name>: C<twig_print_outside_roots>
+or C<TwigPrintOutsideRoots> (or even C<twigPrintOutsideRoots> {shudder}). 
+XML::Twig normalizes them before processing them.
+
+=item parse (SOURCE [, OPT => OPT_VALUE [...]])
+
+This method is inherited from XML::Parser.
+The C<SOURCE> parameter should either be a string containing the whole XML
+document, or it should be an open C<IO::Handle>. Constructor options to
+C<XML::Parser::Expat> given as keyword-value pairs may follow theC<SOURCE> 
+parameter. These override, for this call, any options or attributes passed
+through from the XML::Parser instance.
+
+A die call is thrown if a parse error occurs. Otherwise it will return 
+the twig built by the parse. Use C<safe_parse> if you want the parsing
+to return even when an error occurs.
+
+=item parsestring
+
+This is just an alias for C<parse> for backwards compatibility.
+
+=item parsefile (FILE [, OPT => OPT_VALUE [...]])
+
+This method is inherited from XML::Parser.
+
+Open C<FILE> for reading, then call C<parse> with the open handle. The file
+is closed no matter how C<parse> returns. 
+
+A C<die> call is thrown if a parse error occurs. Otherwise it will return 
+the twig built by the parse. Use C<safe_parsefile> if you want the parsing
+to return even when an error occurs.
+
+=item parseurl ($url $optional_user_agent)
+
+Gets the data from C<$url> and parse it. Note that the data is piped to the
+parser in chunks the size of the XML::Parser::Expat buffer, so memory 
+consumption and hopefully speed are optimal.
+
+If the C<$optional_user_agent> argument is used then it is used, otherwise a
+new one is created.
+
+=item safe_parse ( SOURCE [, OPT => OPT_VALUE [...]])
+
+This method is similar to C<parse> except that it wraps the parsing in an
+C<eval> block. It returns the twig on success and 0 on failure (the twig object
+also contains the parsed twig). C<$@> contains the error message on failure.
+
+Note that the parsing still stops as soon as an error is detected, there is
+no way to keep going after an error.
+
+=item safe_parsefile (FILE [, OPT => OPT_VALUE [...]])
+
+This method is similar to C<parsefile> except that it wraps the parsing in an
+C<eval> block. It returns the twig on success and 0 on failure (the twig object
+also contains the parsed twig) . C<$@> contains the error message on failure
+
+Note that the parsing still stops as soon as an error is detected, there is
+no way to keep going after an error.
+
+=item safe_parseurl ($url $optional_user_agent)
+
+Same as C<parseurl> except that it wraps the parsing in an C<eval> block. It 
+returns the twig on success and 0 on failure (the twig object also contains
+the parsed twig) . C<$@> contains the error message on failure
+
+=item parser
+
+This method returns the C<expat> object (actually the XML::Parser::Expat object) 
+used during parsing. It is useful for example to call XML::Parser::Expat methods
+on it. To get the line of a tag for example use C<< $t->parser->current_line >>.
+
+=item setTwigHandlers ($handlers)
+
+Set the twig_handlers. C<$handlers> is a reference to a hash similar to the
+one in the C<twig_handlers> option of new. All previous handlers are unset.
+The method returns the reference to the previous handlers.
+
+=item setTwigHandler ($exp $handler)
+
+Set a single twig_handler for elements matching C<$exp>. C<$handler> is a 
+reference to a subroutine. If the handler was previously set then the reference 
+to the previous handler is returned.
+
+=item setStartTagHandlers ($handlers)
+
+Set the start_tag handlers. C<$handlers> is a reference to a hash similar to the
+one in the C<start_tag_handlers> option of new. All previous handlers are unset.
+The method returns the reference to the previous handlers.
+
+=item setStartTagHandler ($exp $handler)
+
+Set a single start_tag handlers for elements matching C<$exp>. C<$handler> is a 
+reference to a subroutine. If the handler was previously set then the reference
+to the previous handler is returned.
+
+=item setEndTagHandlers ($handlers)
+
+Set the end_tag handlers. C<$handlers> is a reference to a hash similar to the
+one in the C<end_tag_handlers> option of new. All previous handlers are unset.
+The method returns the reference to the previous handlers.
+
+=item setEndTagHandler ($exp $handler)
+
+Set a single end_tag handlers for elements matching C<$exp>. C<$handler> is a 
+reference to a subroutine. If the handler was previously set then the 
+reference to the previous handler is returned.
+
+=item setTwigRoots ($handlers)
+
+Same as using the C<L<twig_roots>> option when creating the twig
+
+=item setCharHandler ($exp $handler)
+
+Set a C<char_handler>
+
+=item setIgnoreEltsHandler ($exp)
+
+Set a C<ignore_elt> handler (elements that match C<$exp> will be ignored
+
+=item setIgnoreEltsHandlers ($exp)
+
+Set all C<ignore_elt> handlers (previous handlers are replaced)
+
+=item dtd
+
+Return the dtd (an L<XML::Twig::DTD> object) of a twig
+
+=item xmldecl
+
+Return the XML declaration for the document, or a default one if it doesn't
+have one
+
+=item doctype
+
+Return the doctype for the document
+
+=item dtd_text
+
+Return the DTD text
+
+=item dtd_print
+
+Print the DTD
+
+=item model ($tag)
+
+Return the model (in the DTD) for the element C<$tag>
+
+=item root
+
+Return the root element of a twig
+
+=item set_root ($elt)
+
+Set the root of a twig
+
+=item first_elt ($optional_condition)
+
+Return the first element matching C<$optional_condition> of a twig, if
+no condition is given then the root is returned
+
+=item elt_id        ($id)
+
+Return the element whose C<id> attribute is $id
+
+=item getEltById
+
+Same as C<L<elt_id>>
+
+=item encoding
+
+This method returns the encoding of the XML document, as defined by the 
+C<encoding> attribute in the XML declaration (ie it is C<undef> if the attribute
+is not defined)
+
+=item set_encoding
+
+This method sets the value of the C<encoding> attribute in the XML declaration. 
+Note that if the document did not have a declaration it is generated (with
+an XML version of 1.0)
+
+=item xml_version
+
+This method returns the XML version, as defined by the C<version> attribute in 
+the XML declaration (ie it is C<undef> if the attribute is not defined)
+
+=item set_xml_version
+
+This method sets the value of the C<version> attribute in the XML declaration. 
+If the declaration did not exist it is created.
+
+=item standalone
+
+This method returns the value of the C<standalone> declaration for the document
+
+=item set_standalone
+
+This method sets the value of the C<standalone> attribute in the XML 
+declaration.  Note that if the document did not have a declaration it is 
+generated (with an XML version of 1.0)
+
+=item set_output_encoding
+
+Set the C<encoding> "attribute" in the XML declaration
+
+=item set_doctype ($name, $system, $public, $internal)
+
+Set the doctype of the element. If an argument is C<undef> (or not present)
+then its former value is retained, if a false ('' or 0) value is passed then
+the former value is deleted;
+
+=item entity_list
+
+Return the entity list of a twig
+
+=item entity_names
+
+Return the list of all defined entities
+
+=item entity ($entity_name)
+
+Return the entity 
+
+=item change_gi      ($old_gi, $new_gi)
+
+Performs a (very fast) global change. All elements C<$old_gi> are now 
+C<$new_gi>.
+
+See C<L<BUGS|BUGS> >
+
+=item flush            ($optional_filehandle, $options)
+
+Flushes a twig up to (and including) the current element, then deletes
+all unnecessary elements from the tree that's kept in memory.
+C<flush> keeps track of which elements need to be open/closed, so if you
+flush from handlers you don't have to worry about anything. Just keep 
+flushing the twig every time you're done with a sub-tree and it will
+come out well-formed. After the whole parsing don't forget toC<flush> 
+one more time to print the end of the document.
+The doctype and entity declarations are also printed.
+
+flush take an optional filehandle as an argument.
+
+options: use the C<update_DTD> option if you have updated the (internal) DTD 
+and/or the entity list and you want the updated DTD to be output 
+
+The C<pretty_print> option sets the pretty printing of the document.
+
+   Example: $t->flush( Update_DTD => 1);
+            $t->flush( \*FILE, Update_DTD => 1);
+            $t->flush( \*FILE);
+
+
+=item flush_up_to ($elt, $optional_filehandle, %options)
+
+Flushes up to the C<$elt> element. This allows you to keep part of the
+tree in memory when you C<flush>.
+
+options: see flush.
+
+=item purge
+
+Does the same as a C<flush> except it does not print the twig. It just deletes
+all elements that have been completely parsed so far.
+
+=item purge_up_to ($elt)
+
+Purges up to the C<$elt> element. This allows you to keep part of the tree in 
+memory when you C<purge>.
+
+=item print            ($optional_filehandle, %options)
+
+Prints the whole document associated with the twig. To be used only AFTER the
+parse.
+ 
+options: see C<flush>.
+
+=item sprint
+
+Return the text of the whole document associated with the twig. To be used only
+AFTER the parse.
+
+options: see C<flush>.
+
+=item toSAX1 ($handler)
+
+Send SAX events for the twig to the SAX1 handler C<$handler>
+
+=item toSAX2 ($handler)
+
+Send SAX events for the twig to the SAX2 handler C<$handler>
+
+=item flush_toSAX1 ($handler)
+
+Same as flush, except that SAX events are sent to the SAX1 handler
+C<$handler> instead of the twig being printed
+
+=item flush_toSAX2 ($handler)
+
+Same as flush, except that SAX events are sent to the SAX2 handler
+C<$handler> instead of the twig being printed
+
+=item ignore
+
+This method can B<only> be called in C<start_tag_handlers>. It causes the 
+element to be skipped during the parsing: the twig is not built for this 
+element, it will not be accessible during parsing or after it. The element 
+will not take up any memory and parsing will be faster.
+
+Note that this method can also be called on an element. If the element is a 
+parent of the current element then this element will be ignored (the twig will
+not be built any more for it and what has already been built will be deleted)
+
+
+=item set_pretty_print  ($style)
+
+Set the pretty print method, amongst 'C<none>' (default), 'C<nsgmls>', 
+'C<nice>', 'C<indented>', 'C<record>' and 'C<record_c>'
+
+B<WARNING:> the pretty print style is a B<GLOBAL> variable, so once set it's
+applied to B<ALL> C<print>'s (and C<sprint>'s). Same goes if you use XML::Twig
+with C<mod_perl> . This should not be a problem as the XML that's generated 
+is valid anyway, and XML processors (as well as HTML processors, including
+browsers) should not care. Let me know if this is a big problem, but at the
+moment the performance/cleanliness trade-off clearly favors the global 
+approach.
+
+=item set_empty_tag_style  ($style)
+
+Set the empty tag display style ('C<normal>', 'C<html>' or 'C<expand>'). As 
+with C<L<set_pretty_print>> this sets a global flag.  
+
+C<normal> outputs an empty tag 'C<< <tag/> >>', C<html> adds a space 
+'C<< <tag /> >>' and C<expand> outputs 'C<< <tag></tag> >>'
+
+=item print_prolog     ($optional_filehandle, %options)
+
+Prints the prolog (XML declaration + DTD + entity declarations) of a document.
+
+options: see C<L<flush>>.
+
+=item prolog     ($optional_filehandle, %options)
+
+Return the prolog (XML declaration + DTD + entity declarations) of a document.
+
+options: see C<L<flush>>.
+
+=item finish
+
+Call Expat C<finish> method.
+Unsets all handlers (including internal ones that set context), but expat
+continues parsing to the end of the document or until it finds an error.
+It should finish up a lot faster than with the handlers set.
+
+=item finish_print
+
+Stop twig processing, flush the twig and proceed to finish printing the 
+document as fast as possible. Use this method when modifying a document and 
+the modification is done. 
+
+=item set_expand_external_entities
+
+Same as using the C<L<expand_external_ents>> option when creating the twig
+
+=item set_input_filter
+
+Same as using the C<L<input_filter>> option when creating the twig
+
+=item set_keep_atts_order
+
+Same as using the C<L<keep_atts_order>> option when creating the twig
+
+=item set_keep_encoding
+
+Same as using the C<L<keep_encoding>> option when creating the twig
+
+=item set_output_filter
+
+Same as using the C<L<output_filter>> option when creating the twig
+
+=item set_output_text_filter
+
+Same as using the C<L<output_text_filter>> option when creating the twig
+
+=item Methods inherited from XML::Parser::Expat
+
+A twig inherits all the relevant methods from XML::Parser::Expat. These 
+methods can only be used during the parsing phase (they will generate
+a fatal error otherwise).
+
+Inherited methods are:
+
+=over 4
+
+=item depth
+
+Returns the size of the context list.
+
+=item in_element
+
+Returns true if NAME is equal to the name of the innermost curâ
+rently opened element. If namespace processing is being used and
+you want to check against a name that may be in a namespace, then
+use the generate_ns_name method to create the NAME argument.
+
+=item within_element
+
+Returns the number of times the given name appears in the context
+list.  If namespace processing is being used and you want to check
+against a name that may be in a namespace, then use the generâ
+ate_ns_name method to create the NAME argument.
+
+=item context
+
+Returns a list of element names that represent open elements, with
+the last one being the innermost. Inside start and end tag hanâ
+dlers, this will be the tag of the parent element.
+
+=item current_line
+
+Returns the line number of the current position of the parse.
+
+=item current_column
+
+Returns the column number of the current position of the parse.
+
+=item current_byte
+
+Returns the current position of the parse.
+
+=item position_in_context
+
+Returns a string that shows the current parse position. LINES
+should be an integer >= 0 that represents the number of lines on
+either side of the current parse line to place into the returned
+string.
+
+=item base ([NEWBASE])
+
+Returns the current value of the base for resolving relative URIs.
+If NEWBASE is supplied, changes the base to that value.
+
+=item current_element
+
+Returns the name of the innermost currently opened element. Inside
+start or end handlers, returns the parent of the element associated
+with those tags.
+
+=item element_index
+
+Returns an integer that is the depth-first visit order of the curâ
+rent element. This will be zero outside of the root element. For
+example, this will return 1 when called from the start handler for
+the root element start tag.
+
+=item recognized_string
+
+Returns the string from the document that was recognized in order
+to call the current handler. For instance, when called from a start
+handler, it will give us the the start-tag string. The string is
+encoded in UTF-8.  This method doesn't return a meaningful string
+inside declaration handlers.
+
+=item original_string
+
+Returns the verbatim string from the document that was recognized
+in order to call the current handler. The string is in the original
+document encoding. This method doesn't return a meaningful string
+inside declaration handlers.
+
+=item xpcroak
+
+Concatenate onto the given message the current line number within
+the XML document plus the message implied by ErrorContext. Then
+croak with the formed message.
+
+=item xpcarp 
+
+Concatenate onto the given message the current line number within
+the XML document plus the message implied by ErrorContext. Then
+carp with the formed message.
+
+=item xml_escape(TEXT [, CHAR [, CHAR ...]])
+
+Returns TEXT with markup characters turned into character entities.
+Any additional characters provided as arguments are also turned
+into character references where found in TEXT.
+
+(this method is broken on some versions of expat/XML::Parser)
+
+=back
+
+=item path ($gi)
+
+Return the element context in a form similar to XPath's short
+form: 'C</root/gi1/../gi>'
+
+=item get_xpath  ( $optional_array_ref, $xpath, $optional_offset)
+
+Performs a C<get_xpath> on the document root (see <Elt|"Elt">)
+
+If the C<$optional_array_ref> argument is used the array must contain
+elements. The C<$xpath> expression is applied to each element in turn 
+and the result is union of all results. This way a first query can be
+refined in further steps.
+
+
+=item find_nodes ( $optional_array_ref, $xpath, $optional_offset)
+
+same as C<get_xpath> 
+
+=item findnodes ( $optional_array_ref, $xpath, $optional_offset)
+
+same as C<get_xpath> (similar to the XML::LibXML method)
+
+=item findvalue ( $optional_array_ref, $xpath, $optional_offset)
+
+Return the C<join> of all texts of the results of appling C<L<get_xpath>>
+to the node (similar to the XML::LibXML method)
+
+=item subs_text ($regexp, $replace)
+
+subs_text does text substitution on the whole document, similar to perl's 
+C< s///> operator.
+
+=item dispose
+
+Useful only if you don't have C<Scalar::Util> or C<WeakRef> installed.
+
+Reclaims properly the memory used by an XML::Twig object. As the object has
+circular references it never goes out of scope, so if you want to parse lots 
+of XML documents then the memory leak becomes a problem. Use
+C<< $twig->dispose >> to clear this problem.
+
+=back 
+
+
+=head2 XML::Twig::Elt
+
+=over 4
+
+=item new          ($optional_gi, $optional_atts, @optional_content)
+
+The C<gi> is optional (but then you can't have a content ), the C<$optional_atts> 
+argument is a refreference to a hash of attributes, the content can be just a 
+string or a list of strings and element. A content of 'C<#EMPTY>' creates an empty 
+element;
+
+ Examples: my $elt= XML::Twig::Elt->new();
+           my $elt= XML::Twig::Elt->new( para => { align => 'center' });  
+           my $elt= XML::Twig::Elt->new( para => { align => 'center' }, 'foo');  
+           my $elt= XML::Twig::Elt->new( br   => '#EMPTY');
+           my $elt= XML::Twig::Elt->new( 'para');
+           my $elt= XML::Twig::Elt->new( para => 'this is a para');  
+           my $elt= XML::Twig::Elt->new( para => $elt3, 'another para'); 
+
+The strings are not parsed, the element is not attached to any twig.
+
+B<WARNING>: if you rely on ID's then you will have to set the id yourself. At
+this point the element does not belong to a twig yet, so the ID attribute
+is not known so it won't be strored in the ID list.
+
+Note that C<#COMMENT>, C<#PCDATA> or C<#CDATA> are valid tag names, that will 
+create text elements.
+
+To create an element C<foo> containing a CDATA section:
+
+           my $foo= XML::Twig::Elt->new( '#CDATA' => "content of the CDATA section")
+                                  ->wrap_in( 'foo');
+
+=item parse         ($string, %args)
+
+Creates an element from an XML string. The string is actually
+parsed as a new twig, then the root of that twig is returned.
+The arguments in C<%args> are passed to the twig.
+As always if the parse fails the parser will die, so use an
+eval if you want to trap syntax errors.
+
+As obviously the element does not exist beforehand this method has to be 
+called on the class: 
+
+  my $elt= parse XML::Twig::Elt( "<a> string to parse, with <sub/>
+                                  <elements>, actually tons of </elements>
+                  h</a>");
+
+=item print         ($optional_filehandle, $optional_pretty_print_style)
+
+Prints an entire element, including the tags, optionally to a 
+C<$optional_filehandle>, optionally with a C<$pretty_print_style>.
+
+The print outputs XML data so base entities are escaped.
+
+=item sprint       ($elt, $optional_no_enclosing_tag)
+
+Return the xml string for an entire element, including the tags. 
+If the optional second argument is true then only the string inside the 
+element is returned (the start and end tag for $elt are not).
+The text is XML-escaped: base entities (& and < in text, & < and " in
+attribute values) are turned into entities.
+
+=item gi                       
+
+Return the gi of the element (the gi is the C<generic identifier> the tag
+name in SGML parlance).
+
+C<tag> and C<name> are synonyms of C<gi>.
+
+=item tag
+
+Same as C<L<gi|gi>>
+
+=item name
+
+Same as C<L<gi|gi>>
+
+=item set_gi         ($gi)
+
+Set the gi (tag) of an element
+
+=item set_tag        ($tag)
+
+Set the tag (=C<L<gi|gi>>) of an element
+
+=item set_name       ($name)
+
+Set the name (=C<L<gi|gi>>) of an element
+
+=item root 
+
+Return the root of the twig in which the element is contained.
+
+=item twig 
+
+Return the twig containing the element. 
+
+=item parent        ($optional_condition)
+
+Return the parent of the element, or the first ancestor matching the 
+C<$optional_condition>
+
+=item first_child   ($optional_condition)
+
+Return the first child of the element, or the first child matching the 
+C<$optional_condition>
+
+=item has_child ($optional_condition)
+
+Return the first child of the element, or the first child matching the 
+C<$optional_condition> (same as L<first_child>)
+
+=item has_children ($optional_condition)
+
+Return the first child of the element, or the first child matching the 
+C<$optional_condition> (same as L<first_child>)
+
+
+=item first_child_text   ($optional_condition)
+
+Return the text of the first child of the element, or the first child
+ matching the C<$optional_condition>
+If there is no first_child then returns ''. This avoids getting the
+child, checking for its existence then getting the text for trivial cases.
+
+Similar methods are available for the other navigation methods: 
+
+=over 4
+
+=item last_child_text
+
+=item prev_sibling_text
+
+=item next_sibling_text
+
+=item prev_elt_text
+
+=item next_elt_text
+
+=item child_text
+
+=item parent_text
+
+=back
+
+All this methods also exist in "trimmed" variant: 
+
+=over 4
+
+=item first_child_trimmed_text
+
+=item last_child_trimmed_text
+
+=item prev_sibling_trimmed_text
+
+=item next_sibling_trimmed_text
+
+=item prev_elt_trimmed_text
+
+=item next_elt_trimmed_text
+
+=item child_trimmed_text
+
+=item parent_trimmed_text
+
+=back
+
+=item field         ($optional_condition)
+
+Same method as C<first_child_text> with a different name
+
+=item trimmed_field         ($optional_condition)
+
+Same method as C<first_child_trimmed_text> with a different name
+
+=item set_field ($condition, $optional_atts, @list_of_elt_and_strings)
+
+Set the content of the first child of the element that matches
+C<$condition>, the rest of the arguments is tha same as for C<L<set_content>>
+
+If no child matches C<$condition> _and_ if C<$condition> is a valid
+XML element name, then a new element by that name is created and 
+inserted as the last child.
+
+=item first_child_matches   ($optional_condition)
+
+Return the element if the first child of the element (if it exists) passes
+the C<$optional_condition> C<undef> otherwise
+
+  if( $elt->first_child_matches( 'title')) ... 
+
+is equivalent to
+
+  if( $elt->{first_child} && $elt->{first_child}->passes( 'title')) 
+
+C<first_child_is> is an other name for this method
+
+Similar methods are available for the other navigation methods: 
+
+=over 4
+
+=item last_child_matches
+
+=item prev_sibling_matches
+
+=item next_sibling_matches
+
+=item prev_elt_matches
+
+=item next_elt_matches
+
+=item child_matches
+
+=item parent_matches
+
+=back
+
+=item is_first_child ($optional_condition)
+
+returns true (the element) if the element is the first child of its parent
+(optionaly that satisfies the C<$optional_condition>)
+
+=item is_last_child ($optional_condition)
+
+returns true (the element) if the element is the first child of its parent
+(optionaly that satisfies the C<$optional_condition>)
+
+=item prev_sibling  ($optional_condition)
+
+Return the previous sibling of the element, or the previous sibling matching
+C<$optional_condition>
+
+=item next_sibling  ($optional_condition)
+
+Return the next sibling of the element, or the first one matching 
+C<$optional_condition>.
+
+=item next_elt     ($optional_elt, $optional_condition)
+
+Return the next elt (optionally matching C<$optional_condition>) of the element. This 
+is defined as the next element which opens after the current element opens.
+Which usually means the first child of the element.
+Counter-intuitive as it might look this allows you to loop through the
+whole document by starting from the root.
+
+The C<$optional_elt> is the root of a subtree. When the C<next_elt> is out of the
+subtree then the method returns undef. You can then walk a sub tree with:
+
+  my $elt= $subtree_root;
+  while( $elt= $elt->next_elt( $subtree_root)
+    { # insert processing code here
+    }
+
+=item prev_elt     ($optional_condition)
+
+Return the previous elt (optionally matching C<$optional_condition>) of the
+element. This is the first element which opens before the current one.
+It is usually either the last descendant of the previous sibling or
+simply the parent
+
+=item next_n_elt   ($offset, $optional_condition)
+
+Return the C<$offset>-th element that matches the C<$optional_condition> 
+
+=item children     ($optional_condition)
+
+Return the list of children (optionally which matches C<$optional_condition>) of 
+the element. The list is in document order.
+
+=item children_count ($optional_condition)
+
+Return the number of children of the element (optionally which matches 
+C<$optional_condition>)
+
+=item children_text ($optional_condition)
+
+Return an array containing the text of children of the element (optionally 
+which matches C<$optional_condition>)
+
+=item children_copy ($optional_condition)
+
+Return a list of elements that are copies of the children of the element, 
+optionally which matches C<$optional_condition>
+
+=item descendants     ($optional_condition)
+
+Return the list of all descendants (optionally which matches 
+C<$optional_condition>) of the element. This is the equivalent of the 
+C<getElementsByTagName> of the DOM (by the way, if you are really a DOM 
+addict, you can use C<getElementsByTagName> instead)
+
+=item getElementsByTagName ($optional_condition)
+
+Same as C<L<descendants>>
+
+=item find_by_tag_name ($optional_condition)
+
+Same as C<L<descendants>>
+
+=item descendants_or_self ($optional_condition)
+
+Same as C<L<descendants>> except that the element itself is included in the list
+if it matches the C<$optional_condition> 
+
+=item ancestors    ($optional_condition)
+
+Return the list of ancestors (optionally matching C<$optional_condition>) of the 
+element.  The list is ordered from the innermost ancestor to the outtermost one
+
+NOTE: the element itself is not part of the list, in order to include it 
+you will have to use ancestors_or_self
+
+=item ancestors_or_self     ($optional_condition)
+
+Return the list of ancestors (optionally matching C<$optional_condition>) of the 
+element, including the element (if it matches the condition>).  
+The list is ordered from the innermost ancestor to the outtermost one
+
+=item passes ($condition)
+
+Return the element if it passes the C<$condition> 
+
+=item att          ($att)
+
+Return the value of attribute C<$att> or C<undef>
+
+=item set_att      ($att, $att_value)
+
+Set the attribute of the element to the given value
+
+You can actually set several attributes this way:
+
+  $elt->set_att( att1 => "val1", att2 => "val2");
+
+=item del_att      ($att)
+
+Delete the attribute for the element
+
+You can actually delete several attributes at once:
+
+  $elt->del_att( 'att1', 'att2', 'att3');
+
+=item cut
+
+Cut the element from the tree. The element still exists, it can be copied
+or pasted somewhere else, it is just not attached to the tree anymore.
+
+=item cut_children ($optional_condition)
+
+Cut all the children of the element (or all of those which satisfy the
+C<$optional_condition>).
+
+Return the list of children 
+
+=item copy        ($elt)
+
+Return a copy of the element. The copy is a "deep" copy: all sub elements of 
+the element are duplicated.
+
+=item paste       ($optional_position, $ref)
+
+Paste a (previously C<cut> or newly generated) element. Die if the element
+already belongs to a tree.
+
+The element is pasted 
+
+Example:
+
+  my $t= XML::Twig->new->parse( 'doc.xml')
+  my $toc= $t->root->new( 'toc');
+  $toc->paste( $t->root); # $toc is pasted as first child of the root 
+  foreach my $title ($t->findnodes( '/doc/section/title'))
+    { my $title_toc= $title->copy;
+      # paste $title_toc as the last child of toc
+      $title_toc->paste( last_child => $toc) 
+    }
+
+Position options:
+
+=over 4
+
+=item first_child (default)
+
+The element is pasted as the first child of C<$ref>
+
+=item last_child
+
+The element is pasted as the last child of C<$ref>
+
+=item before
+
+The element is pasted before C<$ref>, as its previous sibling.
+
+=item after
+
+The element is pasted after C<$ref>, as its next sibling.
+
+=item within
+
+In this case an extra argument, C<$offset>, should be supplied. The element
+will be pasted in the reference element (or in its first text child) at the
+given offset. To achieve this the reference element will be split at the 
+offset.
+
+=back
+
+Note that you can call directly the underlying method:
+
+=over 4
+
+=item paste_before
+
+=item paste_after
+
+=item paste_first_child
+
+=item paste_last_child
+
+=item paste_within
+
+=back
+
+=item move       ($optional_position, $ref)
+
+Move an element in the tree.
+This is just a C<cut> then a C<paste>.  The syntax is the same as C<paste>.
+
+=item replace       ($ref)
+
+Replaces an element in the tree. Sometimes it is just not possible toC<cut> 
+an element then C<paste> another in its place, so C<replace> comes in handy.
+The calling element replaces C<$ref>.
+
+=item replace_with   (@elts)
+
+Replaces the calling element with one or more elements 
+
+=item delete
+
+Cut the element and frees the memory.
+
+=item prefix       ($text, $optional_option)
+
+Add a prefix to an element. If the element is a C<PCDATA> element the text
+is added to the pcdata, if the elements first child is a C<PCDATA> then the
+text is added to it's pcdata, otherwise a new C<PCDATA> element is created 
+and pasted as the first child of the element.
+
+If the option is C<asis> then the prefix is added asis: it is created in
+a separate C<PCDATA> element with an C<asis> property. You can then write:
+
+  $elt1->prefix( '<b>', 'asis');
+
+to create a C<< <b> >> in the output of C<print>.
+
+=item suffix       ($text, $optional_option)
+
+Add a suffix to an element. If the element is a C<PCDATA> element the text
+is added to the pcdata, if the elements last child is a C<PCDATA> then the
+text is added to it's pcdata, otherwise a new PCDATA element is created 
+and pasted as the last child of the element.
+
+If the option is C<asis> then the suffix is added asis: it is created in
+a separate C<PCDATA> element with an C<asis> property. You can then write:
+
+  $elt2->suffix( '</b>', 'asis');
+
+=item simplify (%options)
+
+Return a data structure suspiciously similar to XML::Simple's. Options are
+identical to XMLin options, see XML::Simple doc for more details (or use
+DATA::dumper or YAML to dump the data structure)
+
+=over 4
+
+=item keyattr 
+
+=item forcearray 
+
+=item noattr 
+
+=item content_key
+                             
+=item variables (%var_hash)
+
+%var_hash is a hash { name => value }
+
+This option allows variables in the XML to be expanded when the file is read. (there is no facility for putting the variable names back if you regenerate XML using XMLout).
+
+A 'variable' is any text of the form ${name} (or $name) which occurs in an attribute value or in the text content of an element. If 'name' matches a key in the supplied hashref, ${name} will be replaced with the corresponding value from the hashref. If no matching key is found, the variable will not be replaced. 
+
+=item var_att ($attribute_name)
+
+This option gives the name of an attribute that will be used to create 
+variables in the XML:
+
+  <dirs>
+    <dir name="prefix">/usr/local</dir>
+    <dir name="exec_prefix">$prefix/bin</dir>
+  </dirs>
+
+use C<< var => 'name' >> to get $prefix replaced by /usr/local in the
+generated data structure  
+
+By default variables are captured by the following regexp: /$(\w+)/
+    
+=item var_regexp (regexp)
+
+This option changes the regexp used to capture variables. The variable
+name should be in $1
+
+=item group_tags ([<tag1>, <tag2>...])
+
+Option used to simplify the structure: elements listed will not be used.
+Their children will be, they will be considered children of the element
+parent.
+
+If the element is:
+
+  <config host="laptop.xmltwig.com">
+    <server>localhost</server>
+    <dirs>
+      <dir name="base">/home/mrodrigu/standards</dir>
+      <dir name="tools">$base/tools</dir>
+    </dirs>
+    <templates>
+      <template name="std_def">std_def.templ</template>
+      <template name="dummy">dummy</template>
+    </templates>
+  </config>
+
+Then callin simplify with C<< erase => [ 'dirs', 'templates'] >> makes the
+data structure be exactly as if the start and end tags for C<dirs> and
+C<templates> were not there.
+
+A YAML dump of the structure 
+
+  base: '/home/mrodrigu/standards'
+  host: laptop.xmltwig.com
+  server: localhost
+  template:
+    - std_def.templ
+    - dummy.templ
+  tools: '$base/tools'
+
+
+=back
+
+=item split_at        ($offset)
+
+Split a text (C<PCDATA> or C<CDATA>) element in 2 at C<$offset>, the original
+element now holds the first part of the string and a new element holds the
+right part. The new element is returned
+
+If the element is not a text element then the first text child of the element
+is split
+
+=item split        ( $optional_regexp, $optional_tag, $optional_attribute_ref)
+
+Split the text descendants of an element in place, the text is split using 
+the regexp, if the regexp includes () then the matched separators will be 
+wrapped in C<$optional_tag>, with C<$optional_attribute_ref> attributes
+
+if $elt is C<< <p>tati tata <b>tutu tati titi</b> tata tati tata</p> >>
+
+  $elt->split( qr/(ta)ti/, 'foo', {type => 'toto'} )
+
+will change $elt to
+
+  <p><foo type="toto">ta</foo> tata <b>tutu <foo type="toto">ta</foo>
+      titi</b> tata <foo type="toto">ta</foo> tata</p> 
+
+The regexp can be passed either as a string or as C<qr//> (perl 5.005 and 
+later), it defaults to \s+ just as the C<split> built-in (but this would be 
+quite a useless behaviour without the C<$optional_tag> parameter)
+
+C<$optional_tag> defaults to PCDATA or CDATA, depending on the initial element
+type
+
+The list of descendants is returned (including un-touched original elements 
+and newly created ones)
+
+=item mark        ( $regexp, $optional_tag, $optional_attribute_ref)
+
+This method behaves exactly as L<split|split>, except only the newly created 
+elements are returned
+
+=item wrap_children ( $regexp_string, $tag, $optional_att, $optional_value)
+
+Wrap the children of the element that match the regexp in an element C<$tag>.
+If C<$optional_att> and C<$optional_value> are passed then the new element will
+have an attribute C<$optional_att> with a value C<$optional_value>.
+
+Note that elements might get extra C<id> attributes in the process. See L<add_id>.
+Use L<strip_att> to remove unwanted id's. 
+
+Here is an example:
+
+If the element C<$elt> has the following content:
+
+  <elt>
+   <p>para 1</p>
+   <l_l1_1>list 1 item 1 para 1</l_l1_1>
+     <l_l1>list 1 item 1 para 2</l_l1>
+   <l_l1_n>list 1 item 2 para 1 (only para)</l_l1_n>
+   <l_l1_n>list 1 item 3 para 1</l_l1_n>
+     <l_l1>list 1 item 3 para 2</l_l1>
+     <l_l1>list 1 item 3 para 3</l_l1>
+   <l_l1_1>list 2 item 1 para 1</l_l1_1>
+     <l_l1>list 2 item 1 para 2</l_l1>
+   <l_l1_n>list 2 item 2 para 1 (only para)</l_l1_n>
+   <l_l1_n>list 2 item 3 para 1</l_l1_n>
+     <l_l1>list 2 item 3 para 2</l_l1>
+     <l_l1>list 2 item 3 para 3</l_l1>
+  </elt>
+
+Then the code
+
+  $elt->wrap_children( q{<l_l1_1><l_l1>*} , li => { type => "ul1" });
+  $elt->wrap_children( q{<l_l1_n><l_l1>*} , li => { type => "ul" });
+
+  $elt->wrap_children( q{<li type="ul1"><li type="ul">+}, "ul");
+  $elt->strip_att( 'id');
+  $elt->strip_att( 'type');
+  $elt->print;
+
+will output:
+
+  <elt>
+     <p>para 1</p>
+     <ul>
+       <li>
+         <l_l1_1>list 1 item 1 para 1</l_l1_1>
+         <l_l1>list 1 item 1 para 2</l_l1>
+       </li>
+       <li>
+         <l_l1_n>list 1 item 2 para 1 (only para)</l_l1_n>
+       </li>
+       <li>
+         <l_l1_n>list 1 item 3 para 1</l_l1_n>
+         <l_l1>list 1 item 3 para 2</l_l1>
+         <l_l1>list 1 item 3 para 3</l_l1>
+       </li>
+     </ul>
+     <ul>
+       <li>
+         <l_l1_1>list 2 item 1 para 1</l_l1_1>
+         <l_l1>list 2 item 1 para 2</l_l1>
+       </li>
+       <li>
+         <l_l1_n>list 2 item 2 para 1 (only para)</l_l1_n>
+       </li>
+       <li>
+         <l_l1_n>list 2 item 3 para 1</l_l1_n>
+         <l_l1>list 2 item 3 para 2</l_l1>
+         <l_l1>list 2 item 3 para 3</l_l1>
+       </li>
+     </ul>
+  </elt>
+
+=item subs_text ($regexp, $replace)
+
+subs_text does text substitution, similar to perl's C< s///> operator.
+
+C<$regexp> must be a perl regexp, created with the C<qr> operatot.
+
+C<$replace> can include C<$1, $2>... from the C<$regexp>. It can also be
+used to create element and entities, by using 
+C<< &elt( tag => { att => val }, text) >> (similar syntax as C<L<new>>) and
+C<< &ent( name) >>.
+
+Here is a rather complex example:
+
+  $elt->subs_text( qr{(?<!do not )link to (http://([^\s,]*))},
+                   'see &elt( a =>{ href => $1 }, $2)'
+                 );
+
+This will replace text like I<link to http://www.xmltwig.com> by 
+I<< see <a href="www.xmltwig.com">www.xmltwig.com</a> >>, but not
+I<do not link to...>
+
+Generating entities (here replacing spaces with &nbsp;):
+
+  $elt->subs_text( qr{ }, '&ent( "&nbsp;")');
+
+or, using a variable:
+
+  my $ent="&nbsp;";
+  $elt->subs_text( qr{ }, "&ent( '$ent')");
+
+Note that the substitution is always global, as in using the C<g> modifier
+in a perl substitution, and that it is performed on all text descendants
+of the element.
+
+=item add_id
+
+Add an id to the element.
+
+The id is an attribute (C<id> by default, see the C<id> option for XML::Twig
+C<new> to change it. Use an id starting with C<#> to get an id that's not 
+output by L<print>, L<flush> or L<sprint>) that allows you to use the
+L<elt_id> method to get the element easily.
+
+=item set_id_seed ($prefix)
+
+by default the id generated by C<L<add_id>> is C<< twig_id_<nnnn> >>, 
+C<set_id_seed> changes the prefix to C<$prefix> and resets the number
+to 1
+
+=item strip_att ($att)
+
+Remove the attribute C<$att> from all descendants of the element (including 
+the element)
+
+=item change_att_name ($old_name, $new_name)
+
+Change the name of the attribute from C<$old_name> to C<$new_name>. If there is no
+attribute C<$old_name> nothing happens.
+
+=item sort_children_on_value( %options)
+
+Sort the children of the element in place according to their text.
+All children are sorted. 
+
+Return the element, with its children sorted.
+
+
+L<%options> are
+
+  type  : numeric |  alpha     (default: alpha)
+  order : normal  |  reverse   (default: normal)
+
+Return the element, with its children sorted
+
+
+=item sort_children_on_att ($att, %options)
+
+Sort the children of the  element in place according to attribute C<$att>. 
+C<%options> are the same as for L<C<sort_children_on_value>>
+
+Return the element.
+
+
+=item sort_children_on_field ($gi, %options)
+
+Sort the children of the element in place, according to the field C<$gi> (the 
+text of the first child of the child with this gi). C<%options> are the same
+as for L<C<sort_children_on_value>>.
+
+Return the element, with its children sorted
+
+
+=item sort_children( $get_key, %options) 
+
+Sort the children of the element in place. The C<$get_key> argument is
+a reference to a function that returns the sort key when passed an element.
+
+For example:
+
+  $elt->sort_children( sub { $_[0]->{'att'}->{"nb"} + $_[0]->text }, 
+                       type => 'numeric', order => 'reverse'
+                     );
+
+=item field_to_att ($cond, $att)
+
+Turn the text of the first sub-element matched by C<$cond> into the value of 
+attribute C<$att> of the element. If C<$att> is ommited then C<$cond> is used 
+as the name of the attribute, which makes sense only if C<$cond> is a valid
+element (and attribute) name.
+
+The sub-element is then cut.
+
+=item att_to_field ($att, $gi)
+
+Take the value of attribute C<$att> and create a sub-element C<$gi> as first
+child of the element. If C<$gi> is ommited then C<$att> is used as the name of
+the sub-element. 
+
+
+=item get_xpath  ($xpath, $optional_offset)
+
+Return a list of elements satisfying the C<$xpath>. C<$xpath> is an XPATH-like 
+expression.
+
+A subset of the XPATH abbreviated syntax is covered:
+
+  gi
+  gi[1] (or any other positive number)
+  gi[last()]
+  gi[@att] (the attribute exists for the element)
+  gi[@att="val"]
+  gi[@att=~ /regexp/]
+  gi[att1="val1" and att2="val2"]
+  gi[att1="val1" or att2="val2"]
+  gi[string()="toto"] (returns gi elements which text (as per the text method) 
+                       is toto)
+  gi[string()=~/regexp/] (returns gi elements which text (as per the text 
+                          method) matches regexp)
+  expressions can start with / (search starts at the document root)
+  expressions can start with . (search starts at the current element)
+  // can be used to get all descendants instead of just direct children
+  * matches any gi
+  
+So the following examples from the 
+F<XPath recommendationL<http://www.w3.org/TR/xpath.html#path-abbrev>> work:
+
+  para selects the para element children of the context node
+  * selects all element children of the context node
+  para[1] selects the first para child of the context node
+  para[last()] selects the last para child of the context node
+  */para selects all para grandchildren of the context node
+  /doc/chapter[5]/section[2] selects the second section of the fifth chapter 
+     of the doc 
+  chapter//para selects the para element descendants of the chapter element 
+     children of the context node
+  //para selects all the para descendants of the document root and thus selects
+     all para elements in the same document as the context node
+  //olist/item selects all the item elements in the same document as the 
+     context node that have an olist parent
+  .//para selects the para element descendants of the context node
+  .. selects the parent of the context node
+  para[@type="warning"] selects all para children of the context node that have
+     a type attribute with value warning 
+  employee[@secretary and @assistant] selects all the employee children of the
+     context node that have both a secretary attribute and an assistant 
+     attribute
+
+
+The elements will be returned in the document order.
+
+If C<$optional_offset> is used then only one element will be returned, the one 
+with the appropriate offset in the list, starting at 0
+
+Quoting and interpolating variables can be a pain when the Perl syntax and the 
+XPATH syntax collide, so here are some more examples to get you started:
+
+  my $p1= "p1";
+  my $p2= "p2";
+  my @res= $t->get_xpath( "p[string( '$p1') or string( '$p2')]");
+
+  my $a= "a1";
+  my @res= $t->get_xpath( "//*[@att=\"$a\"]);
+
+  my $val= "a1";
+  my $exp= "//p[ \@att='$val']"; # you need to use \@ or you will get a warning
+  my @res= $t->get_xpath( $exp);
+
+Note that the only supported regexps delimiters are / and that you must 
+backslash all / in regexps AND in regular strings.
+
+XML::Twig does not provide natively full XPATH support, but you can use 
+XML::Twig does not provide natively full XPATH support, but you can use 
+C<L<XML::Twig::XPath>> to get C<findnodes> to use C<XML::XPath> as the
+XPath engine, with full coverage of the spec.
+
+C<L<XML::Twig::XPath>> to get C<findnodes> to use C<XML::XPath> as the
+XPath engine, with full coverage of the spec.
+
+=item find_nodes
+
+same asC<get_xpath> 
+
+=item findnodes
+
+same asC<get_xpath> 
+
+
+=item text
+
+Return a string consisting of all the C<PCDATA> and C<CDATA> in an element, 
+without any tags. The text is not XML-escaped: base entities such as C<&> 
+and C<< < >> are not escaped.
+
+=item trimmed_text
+
+Same as C<text> except that the text is trimmed: leading and trailing spaces
+are discarded, consecutive spaces are collapsed
+
+=item set_text        ($string)
+
+Set the text for the element: if the element is a C<PCDATA>, just set its
+text, otherwise cut all the children of the element and create a single
+C<PCDATA> child for it, which holds the text.
+
+=item insert         ($gi1, [$optional_atts1], $gi2, [$optional_atts2],...)
+
+For each gi in the list inserts an element C<$gi> as the only child of the 
+element.  The element gets the optional attributes inC<< $optional_atts<n>. >> 
+All children of the element are set as children of the new element.
+The upper level element is returned.
+
+  $p->insert( table => { border=> 1}, 'tr', 'td') 
+
+put C<$p> in a table with a visible border, a single C<tr> and a single C<td> 
+and return the C<table> element:
+
+  <p><table border="1"><tr><td>original content of p</td></tr></table></p>
+
+=item wrap_in        (@gi)
+
+Wrap elements C<$gi> as the successive ancestors of the element, returns the 
+new element.
+$elt->wrap_in( 'td', 'tr', 'table') wraps the element as a single cell in a 
+table for example.
+
+=item insert_new_elt ($opt_position, $gi, $opt_atts_hashref, @opt_content)
+
+Combines a C<L<new|new> > and a C<L<paste|paste> >: creates a new element using 
+C<$gi>, C<$opt_atts_hashref >and C<@opt_content> which are arguments similar 
+to those for C<new>, then paste it, using C<$opt_position> or C<'first_child'>,
+relative to C<$elt>.
+
+Return the newly created element
+
+=item erase
+
+Erase the element: the element is deleted and all of its children are
+pasted in its place.
+
+=item set_content    ( $optional_atts, @list_of_elt_and_strings)
+                     ( $optional_atts, '#EMPTY')
+
+Set the content for the element, from a list of strings and
+elements.  Cuts all the element children, then pastes the list
+elements as the children.  This method will create a C<PCDATA> element
+for any strings in the list.
+
+The C<$optional_atts> argument is the ref of a hash of attributes. If this
+argument is used then the previous attributes are deleted, otherwise they
+are left untouched. 
+
+B<WARNING>: if you rely on ID's then you will have to set the id yourself. At
+this point the element does not belong to a twig yet, so the ID attribute
+is not known so it won't be strored in the ID list.
+
+A content of 'C<#EMPTY>' creates an empty element;
+
+=item namespace ($optional_prefix)
+
+Return the URI of the namespace that C<$optional_prefix> or the element name
+belongs to. If the name doesn't belong to any namespace, C<undef> is returned.
+
+=item local_name
+
+Return the local name (without the prefix) for the element
+
+=item ns_prefix
+
+Return the namespace prefix for the element
+
+=item current_ns_prefixes
+
+Returna list of namespace prefixes valid for the element. The order of the
+prefixes in the list has no meaning. If the default namespace is currently 
+bound, '' appears in the list.
+
+
+=item inherit_att  ($att, @optional_gi_list)
+
+Return the value of an attribute inherited from parent tags. The value
+returned is found by looking for the attribute in the element then in turn
+in each of its ancestors. If the C<@optional_gi_list> is supplied only those
+ancestors whose gi is in the list will be checked. 
+
+=item all_children_are ($optional_condition)
+
+return 1 if all children of the element pass the C<$optional_condition>, 
+0 otherwise
+
+=item level       ($optional_condition)
+
+Return the depth of the element in the twig (root is 0).
+If C<$optional_condition> is given then only ancestors that match the condition are 
+counted.
+ 
+B<WARNING>: in a tree created using the C<twig_roots> option this will not return
+the level in the document tree, level 0 will be the document root, level 1 
+will be the C<twig_roots> elements. During the parsing (in a C<twig_handler>)
+you can use the C<depth> method on the twig object to get the real parsing depth.
+
+=item in           ($potential_parent)
+
+Return true if the element is in the potential_parent (C<$potential_parent> is 
+an element)
+
+=item in_context   ($cond, $optional_level)
+
+Return true if the element is included in an element which passes C<$cond>
+optionally within C<$optional_level> levels. The returned value is the 
+including element.
+
+=item pcdata
+
+Return the text of a C<PCDATA> element or C<undef> if the element is not 
+C<PCDATA>.
+
+=item pcdata_xml_string
+
+Return the text of a PCDATA element or undef if the element is not PCDATA. 
+The text is "XML-escaped" ('&' and '<' are replaced by '&amp;' and '&lt;')
+
+=item set_pcdata     ($text)
+
+Set the text of a C<PCDATA> element. 
+
+=item append_pcdata  ($text)
+
+Add the text at the end of a C<PCDATA> element.
+
+=item is_cdata
+
+Return 1 if the element is a C<CDATA> element, returns 0 otherwise.
+
+=item is_text
+
+Return 1 if the element is a C<CDATA> or C<PCDATA> element, returns 0 otherwise.
+
+=item cdata
+
+Return the text of a C<CDATA> element or C<undef> if the element is not 
+C<CDATA>.
+
+=item cdata_string
+
+Return the XML string of a C<CDATA> element, including the opening and
+closing markers.
+
+=item set_cdata     ($text)
+
+Set the text of a C<CDATA> element. 
+
+=item append_cdata  ($text)
+
+Add the text at the end of a C<CDATA> element.
+
+=item remove_cdata
+
+Turns all C<CDATA> sections in the element into regular C<PCDATA> elements. This is useful
+when converting XML to HTML, as browsers do not support CDATA sections. 
+
+=item extra_data 
+
+Return the extra_data (comments and PI's) attached to an element
+
+=item set_extra_data     ($extra_data)
+
+Set the extra_data (comments and PI's) attached to an element
+
+=item append_extra_data  ($extra_data)
+
+Append extra_data to the existing extra_data before the element (if no
+previous extra_data exists then it is created)
+
+=item set_asis
+
+Set a property of the element that causes it to be output without being XML
+escaped by the print functions: if it contains C<< a < b >> it will be output
+as such and not as C<< a &lt; b >>. This can be useful to create text elements
+that will be output as markup. Note that all C<PCDATA> descendants of the 
+element are also marked as having the property (they are the ones taht are
+actually impacted by the change).
+
+If the element is a C<CDATA> element it will also be output asis, without the
+C<CDATA> markers. The same goes for any C<CDATA> descendant of the element
+
+=item set_not_asis
+
+Unsets the C<asis> property for the element and its text descendants.
+
+=item is_asis
+
+Return the C<asis> property status of the element ( 1 or C<undef>)
+
+=item closed                   
+
+Return true if the element has been closed. Might be usefull if you are
+somewhere in the tree, during the parse, and have no idea whether a parent
+element is completely loaded or not.
+
+=item get_type
+
+Return the type of the element: 'C<#ELT>' for "real" elements, or 'C<#PCDATA>',
+'C<#CDATA>', 'C<#COMMENT>', 'C<#ENT>', 'C<#PI>'
+
+=item is_elt
+
+Return the gi if the element is a "real" element, or 0 if it is C<PCDATA>, 
+C<CDATA>...
+
+=item contains_only_text
+
+Return 1 if the element does not contain any other "real" element
+
+=item contains_only ($exp)
+
+Return the list of children if all children of the element match
+the expression C<$exp> 
+
+  if( $para->contains_only( 'tt')) { ... }
+
+=item contains_a_single ($exp)
+
+If the element contains a single child that matches the expression C<$exp>
+returns that element. Otherwise returns 0.
+
+=item is_field
+
+same as C<contains_only_text> 
+
+=item is_pcdata
+
+Return 1 if the element is a C<PCDATA> element, returns 0 otherwise.
+
+=item is_ent
+
+Return 1 if the element is an entity (an unexpanded entity) element, 
+return 0 otherwise.
+
+=item is_empty
+
+Return 1 if the element is empty, 0 otherwise
+
+=item set_empty
+
+Flags the element as empty. No further check is made, so if the element
+is actually not empty the output will be messed. The only effect of this 
+method is that the output will be C<< <gi att="value""/> >>.
+
+=item set_not_empty
+
+Flags the element as not empty. if it is actually empty then the element will
+be output as C<< <gi att="value""></gi> >>
+
+=item is_pi
+
+Return 1 if the element is a processing instruction (C<#PI>) element,
+return 0 otherwise.
+
+=item target
+
+Return the target of a processing instruction
+
+=item set_target ($target)
+
+Set the target of a processing instruction
+
+=item data
+
+Return the data part of a processing instruction
+
+=item set_data ($data)
+
+Set the data of a processing instruction
+
+=item set_pi ($target, $data)
+
+Set the target and data of a processing instruction
+
+=item pi_string
+
+Return the string form of a processing instruction
+(C<< <?target data?> >>)
+
+=item is_comment
+
+Return 1 if the element is a comment (C<#COMMENT>) element,
+return 0 otherwise.
+
+=item set_comment ($comment_text)
+
+Set the text for a comment
+
+=item comment
+
+Return the content of a comment (just the text, not the C<< <!-- >>
+and C<< --> >>)
+
+=item comment_string 
+
+Return the XML string for a comment (C<< <!-- comment --> >>)
+
+=item set_ent ($entity)
+
+Set an (non-expanded) entity (C<#ENT>). C<$entity>) is the entity
+text (C<&ent;>)
+
+=item ent
+
+Return the entity for an entity (C<#ENT>) element (C<&ent;>)
+
+=item ent_name
+
+Return the entity name for an entity (C<#ENT>) element (C<ent>)
+
+=item ent_string
+
+Return the entity, either expanded if the expanded version is available,
+or non-expanded (C<&ent;>) otherwise
+
+=item child ($offset, $optional_condition)
+
+Return the C<$offset>-th child of the element, optionally the C<$offset>-th 
+child that matches C<$optional_condition>. The children are treated as a list, so 
+C<< $elt->child( 0) >> is the first child, while C<< $elt->child( -1) >> is 
+the last child.
+
+=item child_text ($offset, $optional_condition)
+
+Return the text of a child or C<undef> if the sibling does not exist. Arguments
+are the same as child.
+
+=item last_child    ($optional_condition)
+
+Return the last child of the element, or the last child matching 
+C<$optional_condition> (ie the last of the element children matching
+the condition).
+
+=item last_child_text   ($optional_condition)
+
+Same as C<first_child_text> but for the last child.
+
+=item sibling  ($offset, $optional_condition)
+
+Return the next or previous C<$offset>-th sibling of the element, or the 
+C<$offset>-th one matching C<$optional_condition>. If C<$offset> is negative then a 
+previous sibling is returned, if $offset is positive then  a next sibling is 
+returned. C<$offset=0> returns the element if there is no condition or
+if the element matches the condition>, C<undef> otherwise.
+
+=item sibling_text ($offset, $optional_condition)
+
+Return the text of a sibling or C<undef> if the sibling does not exist. 
+Arguments are the same as C<sibling>.
+
+=item prev_siblings ($optional_condition)
+
+Return the list of previous siblings (optionaly matching C<$optional_condition>)
+for the element. The elements are ordered in document order.
+
+=item next_siblings ($optional_condition)
+
+Return the list of siblings (optionaly matching C<$optional_condition>)
+following the element. The elements are ordered in document order.
+
+=item pos ($optional_condition)
+
+Return the position of the element in the children list. The first child has a
+position of 1 (as in XPath).
+
+If the C<$optional_condition> is given then only siblings that match the condition 
+are counted. If the element itself does not match the  condition then
+0 is returned.
+
+=item atts
+
+Return a hash ref containing the element attributes
+
+=item set_atts      ({att1=>$att1_val, att2=> $att2_val... })
+
+Set the element attributes with the hash ref supplied as the argument
+
+=item del_atts
+
+Deletes all the element attributes.
+
+=item att_nb
+
+Return the number of attributes for the element
+
+=item has_atts
+
+Return true if the element has attributes (in fact return the number of
+attributes, thus being an alias to C<L<att_nb>>
+
+=item has_no_atts
+
+Return true if the element has no attributes, false (0) otherwise
+
+=item att_names
+
+return a list of the attribute names for the element
+
+=item att_xml_string ($att, $optional_quote)
+
+Return the attribute value, where '&', '<' and $quote (" by default)
+are XML-escaped
+
+if C<$optional_quote> is passed then it is used as the quote.
+
+=item set_id       ($id)
+
+Set the C<id> attribute of the element to the value.
+See C<L<elt_id|elt_id> > to change the id attribute name
+
+=item id
+
+Gets the id attribute value
+
+=item del_id       ($id)
+
+Deletes the C<id> attribute of the element and remove it from the id list
+for the document
+
+=item class
+
+Return the C<class> attribute for the element (methods on the C<class>
+attribute are quite convenient when dealing with XHTML, or plain XML that
+will eventually be displayed using CSS)
+
+=item set_class ($class)
+
+Set the C<class> attribute for the element to C<$class>
+
+=item add_to_class ($class)
+
+Add C<$class> to the element C<class> attribute: the new class is added
+only if it is not already present. Note that classes are sorted alphabetically,
+so the C<class> attribute can be changed even if the class is already there
+
+=item att_to_class ($att)
+
+Set the C<class> attribute to the value of attribute C<$att>
+
+=item add_att_to_class ($att)
+
+Add the value of attribute C<$att> to the C<class> attribute of the element
+
+=item move_att_to_class ($att)
+
+Add the value of attribute C<$att> to the C<class> attribute of the element
+and delete the attribute
+
+=item tag_to_class
+
+Set the C<class> attribute of the element to the element tag
+
+=item add_tag_to_class
+
+Add the element tag to its C<class> attribute
+
+=item set_tag_class ($new_tag)
+
+Add the element tag to its C<class> attribute and sets the tag to C<$new_tag>
+
+=item in_class ($class)
+
+Return true (C<1>) if the element is in the class C<$class> (if C<$class> is
+one of the tokens in the element C<class> attribute)
+
+=item DESTROY
+
+Frees the element from memory.
+
+=item start_tag
+
+Return the string for the start tag for the element, including 
+the C<< /> >> at the end of an empty element tag
+
+=item end_tag
+
+Return the string for the end tag of an element.  For an empty
+element, this returns the empty string ('').
+
+=item xml_string
+
+Equivalent to C<< $elt->sprint( 1) >>, returns the string for the entire 
+element, excluding the element's tags (but nested element tags are present)
+
+=item xml_text 
+
+Return the text of the element, encoded (and processed by the current 
+C<L<output_filter>> or C<L<output_encoding>> options, without any tag.
+
+=item set_pretty_print ($style)
+
+Set the pretty print method, amongst 'C<none>' (default), 'C<nsgmls>', 
+'C<nice>', 'C<indented>', 'C<record>' and 'C<record_c>'
+
+pretty_print styles:
+
+=over 4
+
+=item none
+
+the default, no C<\n> is used
+
+=item nsgmls
+
+nsgmls style, with C<\n> added within tags
+
+=item nice
+
+adds C<\n> wherever possible (NOT SAFE, can lead to invalid XML)
+
+=item indented
+
+same as C<nice> plus indents elements (NOT SAFE, can lead to invalid XML) 
+
+=item record
+
+table-oriented pretty print, one field per line 
+
+=item record_c
+
+table-oriented pretty print, more compact than C<record>, one record per line 
+
+=back
+
+=item set_empty_tag_style ($style)
+
+Set the method to output empty tags, amongst 'C<normal>' (default), 'C<html>',
+and 'C<expand>', 
+
+=item set_indent ($string)
+
+Set the indentation for the indented pretty print style (default is 2 spaces)
+
+=item set_quote ($quote)
+
+Set the quotes used for attributes. can be 'C<double>' (default) or 'C<single>'
+
+=item cmp       ($elt)
+
+  Compare the order of the 2 elements in a twig.
+
+  C<$a> is the <A>..</A> element, C<$b> is the <B>...</B> element
+  
+  document                        $a->cmp( $b)
+  <A> ... </A> ... <B>  ... </B>     -1
+  <A> ... <B>  ... </B> ... </A>     -1
+  <B> ... </B> ... <A>  ... </A>      1
+  <B> ... <A>  ... </A> ... </B>      1
+   $a == $b                           0
+   $a and $b not in the same tree   undef
+
+=item before       ($elt)
+
+Return 1 if C<$elt> starts before the element, 0 otherwise. If the 2 elements 
+are not in the same twig then return C<undef>.
+
+    if( $a->cmp( $b) == -1) { return 1; } else { return 0; }
+
+=item after       ($elt)
+
+Return 1 if $elt starts after the element, 0 otherwise. If the 2 elements 
+are not in the same twig then return C<undef>.
+
+    if( $a->cmp( $b) == -1) { return 1; } else { return 0; }
+
+=item other comparison methods
+
+=over 4
+
+=item lt
+
+=item le
+
+=item gt
+
+=item ge
+
+=back
+
+=item path
+
+Return the element context in a form similar to XPath's short
+form: 'C</root/gi1/../gi>'
+
+=item xpath
+
+Return a unique XPath expression that can be used to find the element
+again. 
+
+It looks like C</doc/sect[3]/title>: unique elements do not have an index,
+the others do.
+
+=item private methods
+
+Low-level methods on the twig:
+
+=over 4
+
+=item set_parent        ($parent)
+
+=item set_first_child   ($first_child)
+
+=item set_last_child    ($last_child)
+
+=item set_prev_sibling  ($prev_sibling)
+
+=item set_next_sibling  ($next_sibling)
+
+=item set_twig_current
+
+=item del_twig_current
+
+=item twig_current
+
+=item flush
+
+This method should NOT be used, always flush the twig, not an element.
+
+=item contains_text
+
+=back
+
+Those methods should not be used, unless of course you find some creative 
+and interesting, not to mention useful, ways to do it.
+
+=back
+
+=head2 cond
+
+Most of the navigation functions accept a condition as an optional argument
+The first element (or all elements for C<L<children|children> > or 
+C<L<ancestors|ancestors> >) that passes the condition is returned.
+
+The condition is a single step of an XPath expression using the XPath subset
+defined by C<L<get_xpath>>. Additional conditions are:
+
+The condition can be 
+
+=over 4
+
+=item #ELT
+
+return a "real" element (not a PCDATA, CDATA, comment or pi element) 
+
+=item #TEXT
+
+return a PCDATA or CDATA element
+
+=item regular expression
+
+return an element whose gi matches the regexp. The regexp has to be created 
+with C<qr//> (hence this is available only on perl 5.005 and above)
+
+=item code reference
+
+applies the code, passing the current element as argument, if the code returns
+true then the element is returned, if it returns false then the code is applied
+to the next candidate.
+
+=back
+
+=head2 XML::Twig::XPath
+
+XML::Twig implements a subset of XPath through the C<L<get_xpath>> method. 
+
+If you want to use the whole XPath power, then you can use C<XML::Twig::XPath>
+instead. In this case C<XML::Twig> uses C<XML::XPath> to execute XPath queries.
+You will of course need C<XML::XPath> installed to be able to use C<XML::Twig::XPath>.
+
+See L<XML::XPath> for more information.
+
+The methods you can use are:
+
+=over 4
+
+=item findnodes              ($path)
+
+return a list of nodes found by C<$path>.
+
+=item findnodes_as_string    ($path)
+
+return the nodes found reproduced as XML. The result is not guaranteed
+to be valid XML though.
+
+=item findvalue              ($path)
+
+return the concatenation of the text content of the result nodes
+
+=back
+
+In order for C<XML::XPath> to be used as the XPath engine the following methods
+are included in C<XML::Twig>:
+
+in XML::Twig
+
+=over 4
+
+=item getRootNode
+
+=item getParentNode
+
+=item getChildNodes 
+
+=back
+
+in XML::Twig::Elt
+
+=over 4
+
+=item string_value
+
+=item toString
+
+=item getName
+
+=item getRootNode
+
+=item getNextSibling
+
+=item getPreviousSibling
+
+=item isElementNode
+
+=item isTextNode
+
+=item isPI
+
+=item isPINode
+
+=item isProcessingInstructionNode
+
+=item isComment
+
+=item isCommentNode
+
+=item getTarget 
+
+=item getChildNodes 
+
+=item getElementById
+
+=back
+
+=head2 XML::Twig::XPath::Elt
+
+The methods you can use are the same as on C<XML::Twig::XPath> elements:
+
+=over 4
+
+=item findnodes              ($path)
+
+return a list of nodes found by C<$path>.
+
+=item findnodes_as_string    ($path)
+
+return the nodes found reproduced as XML. The result is not guaranteed
+to be valid XML though.
+
+=item findvalue              ($path)
+
+return the concatenation of the text content of the result nodes
+
+=back
+
+
+=head2 XML::Twig::Entity_list
+
+=over 4
+
+=item new
+
+Create an entity list.
+
+=item add         ($ent)
+
+Add an entity to an entity list.
+
+=item add_new_ent ($name, $val, $sysid, $pubid, $ndata)
+
+Create a new entity and add it to the entity list
+
+=item delete     ($ent or $gi).
+
+Delete an entity (defined by its name or by the Entity object)
+from the list.
+
+=item print      ($optional_filehandle)
+
+Print the entity list.
+
+=item list
+
+Return the list as an array
+
+=back
+
+
+=head2 XML::Twig::Entity
+
+=over 4
+
+=item new        ($name, $val, $sysid, $pubid, $ndata)
+
+Same arguments as the Entity handler for XML::Parser.
+
+=item print       ($optional_filehandle)
+
+Print an entity declaration.
+
+=item name 
+
+Return the name of the entity
+
+=item val  
+
+Return the value of the entity
+
+=item sysid
+
+Return the system id for the entity (for NDATA entities)
+
+=item pubid
+
+Return the public id for the entity (for NDATA entities)
+
+=item ndata
+
+Return true if the entity is an NDATA entity
+
+=item text
+
+Return the entity declaration text.
+
+=back
+
+
+=head1 EXAMPLES
+
+Additional examples (and a complete tutorial) can be found  on the
+F<XML::Twig PageL<http://www.xmltwig.com/xmltwig/>>
+
+To figure out what flush does call the following script with an
+XML file and an element name as arguments
+
+  use XML::Twig;
+
+  my ($file, $elt)= @ARGV;
+  my $t= XML::Twig->new( twig_handlers => 
+      { $elt => sub {$_[0]->flush; print "\n[flushed here]\n";} });
+  $t->parsefile( $file, ErrorContext => 2);
+  $t->flush;
+  print "\n";
+
+
+=head1 NOTES
+
+=head2 Subclassing XML::Twig
+
+Useful methods:
+
+=over 4
+
+=item elt_class
+
+In order to subclass C<XML::Twig> you will probably need to subclass also
+C<L<XML::Twig::Elt>>. Use the C<elt_class> option when you create the
+C<XML::Twig> object to get the elements created in a different class
+(which should be a subclass of C<XML::Twig::Elt>.
+
+=item add_options
+
+If you inherit C<XML::Twig> new method but want to add more options to it
+you can use this method to prevent XML::Twig to issue warnings for those
+additional options.
+
+=back
+
+=head2 DTD Handling
+
+There are 3 possibilities here.  They are:
+
+=over 4
+
+=item No DTD
+
+No doctype, no DTD information, no entity information, the world is simple...
+
+=item Internal DTD
+
+The XML document includes an internal DTD, and maybe entity declarations.
+
+If you use the load_DTD option when creating the twig the DTD information and
+the entity declarations can be accessed.
+
+The DTD and the entity declarations will be C<flush>'ed (or C<print>'ed) either
+as is (if they have not been modified) or as reconstructed (poorly, comments 
+are lost, order is not kept, due to it's content this DTD should not be viewed 
+by anyone) if they have been modified. You can also modify them directly by 
+changing the C<< $twig->{twig_doctype}->{internal} >> field (straight from 
+XML::Parser, see the C<Doctype> handler doc)
+
+=item External DTD
+
+The XML document includes a reference to an external DTD, and maybe entity 
+declarations.
+
+If you use the C<load_DTD> when creating the twig the DTD information and the 
+entity declarations can be accessed. The entity declarations will be
+C<flush>'ed (or C<print>'ed) either as is (if they have not been modified) or
+as reconstructed (badly, comments are lost, order is not kept).
+
+You can change the doctype through the C<< $twig->set_doctype >> method and 
+print the dtd through the C<< $twig->dtd_text >> or C<< $twig->dtd_print >>
+ methods.
+
+If you need to modify the entity list this is probably the easiest way to do it.
+
+=back
+
+
+=head2 Flush
+
+If you set handlers and use C<flush>, do not forget to flush the twig one
+last time AFTER the parsing, or you might be missing the end of the document.
+
+Remember that element handlers are called when the element is CLOSED, so
+if you have handlers for nested elements the inner handlers will be called
+first. It makes it for example trickier than it would seem to number nested
+clauses.
+
+
+
+=head1 BUGS
+
+=over 4
+
+=item entity handling
+
+Due to XML::Parser behaviour, non-base entities in attribute values disappear:
+C<att="val&ent;"> will be turned into C<< att => val >>, unless you use the 
+C<keep_encoding> argument to C<< XML::Twig->new >> 
+
+=item DTD handling
+
+Basically the DTD handling methods are competely bugged. No one uses them and
+it seems very difficult to get them to work in all cases, including with 
+several slightly incompatible versions of XML::Parser and of libexpat.
+
+So use XML::Twig with standalone documents, or with documents refering to an
+external DTD, but don't expect it to properly parse and even output back the
+DTD.
+
+=item memory leak
+
+If you use a lot of twigs you might find that you leak quite a lot of memory
+(about 2Ks per twig). You can use the C<L<dispose|dispose> > method to free 
+that memory after you are done.
+
+If you create elements the same thing might happen, use the C<L<delete|delete>>
+method to get rid of them.
+
+Alternatively installing the C<Scalar::Util> (or C<WeakRef>) module on a version 
+of Perl that supports it (>5.6.0) will get rid of the memory leaks automagically.
+
+=item ID list
+
+The ID list is NOT updated when ID's are modified or elements cut or
+deleted.
+
+=item change_gi
+
+This method will not function properly if you do:
+
+     $twig->change_gi( $old1, $new);
+     $twig->change_gi( $old2, $new);
+     $twig->change_gi( $new, $even_newer);
+
+=item sanity check on XML::Parser method calls
+
+XML::Twig should really prevent calls to some XML::Parser methods, especially 
+the C<setHandlers> method.
+
+=item pretty printing
+
+Pretty printing (at least using the 'C<indented>' style) is hard to get right! 
+Only elements that belong to the document will be properly indented. Printing 
+elements that do not belong to the twig makes it impossible for XML::Twig to 
+figure out their depth, and thus their indentation level.
+
+Also there is an anavoidable bug when using C<flush> and pretty printing for
+elements with mixed content that start with an embedded element:
+
+  <elt><b>b</b>toto<b>bold</b></elt>
+
+  will be output as 
+
+  <elt>
+    <b>b</b>toto<b>bold</b></elt>
+
+if you flush the twig when you find the C<< <b> >> element
+  
+
+=back
+
+=head1 Globals
+
+These are the things that can mess up calling code, especially if threaded.
+They might also cause problem under mod_perl. 
+
+=over 4
+
+=item Exported constants
+
+Whether you want them or not you get them! These are subroutines to use
+as constant when creating or testing elements
+
+  PCDATA  return '#PCDATA'
+  CDATA   return '#CDATA'
+  PI      return '#PI', I had the choice between PROC and PI :--(
+
+=item Module scoped values: constants
+
+these should cause no trouble:
+
+  %base_ent= ( '>' => '&gt;',
+               '<' => '&lt;',
+               '&' => '&amp;',
+               "'" => '&apos;',
+               '"' => '&quot;',
+             );
+  CDATA_START   = "<![CDATA[";
+  CDATA_END     = "]]>";
+  PI_START      = "<?";
+  PI_END        = "?>";
+  COMMENT_START = "<!--";
+  COMMENT_END   = "-->";
+
+pretty print styles
+
+  ( $NSGMLS, $NICE, $INDENTED, $RECORD1, $RECORD2)= (1..5);
+
+empty tag output style
+
+  ( $HTML, $EXPAND)= (1..2);
+
+=item Module scoped values: might be changed
+
+Most of these deal with pretty printing, so the worst that can
+happen is probably that XML output does not look right, but is
+still valid and processed identically by XML processors.
+
+C<$empty_tag_style> can mess up HTML bowsers though and changing C<$ID> 
+would most likely create problems.
+
+  $pretty=0;           # pretty print style
+  $quote='"';          # quote for attributes
+  $INDENT= '  ';       # indent for indented pretty print
+  $empty_tag_style= 0; # how to display empty tags
+  $ID                  # attribute used as a gi ('id' by default)
+
+=item Module scoped values: definitely changed
+
+These 2 variables are used to replace gi's by an index, thus 
+saving some space when creating a twig. If they really cause
+you too much trouble, let me know, it is probably possible to
+create either a switch or at least a version of XML::Twig that 
+does not perform this optimisation.
+
+  %gi2index;     # gi => index
+  @index2gi;     # list of gi's
+
+=back
+
+If you need to manipulate all those values, you can use the following methods on the
+XML::Twig object:
+
+=over 4
+
+=item global_state
+
+Return a hasref with all the global variables used by XML::Twig
+
+The hash has the following fields:  C<pretty>, C<quote>, C<indent>, 
+C<empty_tag_style>, C<keep_encoding>, C<expand_external_entities>, 
+C<output_filter>, C<output_text_filter>, C<keep_atts_order>
+
+=item set_global_state ($state)
+
+Set the global state, C<$state> is a hashref
+
+=item save_global_state
+
+Save the current global state
+
+=item restore_global_state
+
+Restore the previously saved (using C<Lsave_global_state>> state
+
+=back
+
+=head1 TODO 
+
+=over 4
+
+=item SAX handlers
+
+Allowing XML::Twig to work on top of any SAX parser
+
+=item multiple twigs are not well supported
+
+A number of twig features are just global at the moment. These include
+the ID list and the "gi pool" (if you use C<change_gi> then you change the gi 
+for ALL twigs).
+
+A future version will try to support this while trying not to be to
+hard on performance (at least when a single twig is used!).
+
+
+=back
+
+
+=head1 AUTHOR
+
+Michel Rodriguez <mirod@xmltwig.com>
+
+=head1 LICENSE
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself.
+
+Bug reports should be sent using:
+F<RTL<http://rt.cpan.org/NoAuth/Bugs.html?Dist=XML-Twig>>
+
+Comments can be sent to mirod@xmltwig.com
+
+The XML::Twig page is at L<http://www.xmltwig.com/xmltwig/>
+It includes the development version of the module, a slightly better version 
+of the documentation, examples, a tutorial and a: 
+F<Processing XML efficiently with Perl and XML::Twig: 
+L<http://www.xmltwig.com/xmltwig/tutorial/index.html>>
+
+=head1 SEE ALSO
+
+XML::Parser,XML::Parser::Expat, Encode, Text::Iconv, Scalar::Utils
+
+
+=cut
+
+
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Variable.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Variable.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/Variable.pm	(revision 21112)
@@ -0,0 +1,43 @@
+# $Id$
+
+package XML::XPath::Variable;
+use strict;
+
+# This class does NOT contain 1 instance of a variable
+# see the XML::XPath::Parser class for the instances
+# This class simply holds the name of the var
+
+sub new {
+    my $class = shift;
+    my ($pp, $name) = @_;
+    bless { name => $name, path_parser => $pp }, $class;
+}
+
+sub as_string {
+    my $self = shift;
+    '\$' . $self->{name};
+}
+
+sub as_xml {
+    my $self = shift;
+    return "<Variable>" . $self->{name} . "</Variable>\n";
+}
+
+sub get_value {
+    my $self = shift;
+    $self->{path_parser}->get_var($self->{name});
+}
+
+sub set_value {
+    my $self = shift;
+    my ($val) = @_;
+    $self->{path_parser}->set_var($self->{name}, $val);
+}
+
+sub evaluate {
+    my $self = shift;
+    my $val = $self->get_value;
+    return $val;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/XMLParser.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/XMLParser.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/XML/XPath/XMLParser.pm	(revision 21112)
@@ -0,0 +1,385 @@
+# $Id$
+
+package XML::XPath::XMLParser;
+
+use strict;
+
+use XML::Parser;
+#use XML::XPath;
+use XML::XPath::Node;
+use XML::XPath::Node::Element;
+use XML::XPath::Node::Text;
+use XML::XPath::Node::Comment;
+use XML::XPath::Node::PI;
+use XML::XPath::Node::Attribute;
+use XML::XPath::Node::Namespace;
+
+my @options = qw(
+        filename
+        xml
+        parser
+        ioref
+        );
+
+my ($_current, $_namespaces_on);
+my %IdNames;
+
+use vars qw/$xmlns_ns $xml_ns/;
+
+$xmlns_ns = "http://www.w3.org/2000/xmlns/";
+$xml_ns = "http://www.w3.org/XML/1998/namespace";
+
+sub new {
+    my $proto = shift;
+    my $class = ref($proto) || $proto;
+    my %args = @_;
+    my %hash = map(( "_$_" => $args{$_} ), @options);
+    bless \%hash, $class;
+}
+
+sub parse {
+    my $self = shift;
+    
+    $self->{IdNames} = {};
+    $self->{InScopeNamespaceStack} = [ { 
+            '_Default' => undef,
+            'xmlns' => $xmlns_ns,
+            'xml' => $xml_ns,
+        } ];
+    
+    $self->{NodeStack} = [ ];
+    
+    $self->set_xml($_[0]) if $_[0];
+    
+    my $parser = $self->get_parser || XML::Parser->new(
+            ErrorContext => 2,
+            ParseParamEnt => 1,
+            );
+    
+    $parser->setHandlers(
+            Init => sub { $self->parse_init(@_) },
+            Char => sub { $self->parse_char(@_) },
+            Start => sub { $self->parse_start(@_) },
+            End => sub { $self->parse_end(@_) },
+            Final => sub { $self->parse_final(@_) },
+            Proc => sub { $self->parse_pi(@_) },
+            Comment => sub { $self->parse_comment(@_) },
+            Attlist => sub { $self->parse_attlist(@_) },
+            );
+    
+    my $toparse;
+    if ($toparse = $self->get_filename) {
+        return $parser->parsefile($toparse);
+    }
+    else {
+        return $parser->parse($self->get_xml || $self->get_ioref);
+    }
+}
+
+sub parsefile {
+    my $self = shift;
+    my ($filename) = @_;
+    $self->set_filename($filename);
+    $self->parse;
+}
+
+sub parse_init {
+    my $self = shift;
+    my $e = shift;
+    my $document = XML::XPath::Node::Element->new();
+    my $newns = XML::XPath::Node::Namespace->new('xml', $xml_ns);
+    $document->appendNamespace($newns);
+    $self->{current} = $self->{DOC_Node} = $document;
+}
+
+sub parse_final {
+    my $self = shift;
+    return $self->{DOC_Node};
+}
+
+sub parse_char {
+    my $self = shift;
+    my $e = shift;
+    my $text = shift;
+    
+    my $parent = $self->{current};
+    
+    my $last = $parent->getLastChild;
+    if ($last && $last->isTextNode) {
+        # append to previous text node
+        $last->appendText($text);
+        return;
+    }
+    
+    my $node = XML::XPath::Node::Text->new($text);
+    $parent->appendChild($node, 1);
+}
+
+sub parse_start {
+    my $self = shift;
+    my $e = shift;
+    my $tag = shift;
+    
+    push @{ $self->{InScopeNamespaceStack} },
+         { %{ $self->{InScopeNamespaceStack}[-1] } };
+    $self->_scan_namespaces(@_);
+    
+    my ($prefix, $namespace) = $self->_namespace($tag);
+    
+    my $node = XML::XPath::Node::Element->new($tag, $prefix);
+    
+    my @attributes;
+    for (my $ii = 0; $ii < $#_; $ii += 2) {
+	my ($name, $value) = ($_[$ii], $_[$ii+1]);
+        if ($name =~ /^xmlns(:(.*))?$/) {
+            # namespace node
+            my $prefix = $2 || '#default';
+#            warn "Creating NS node: $prefix = $value\n";
+            my $newns = XML::XPath::Node::Namespace->new($prefix, $value);
+            $node->appendNamespace($newns);
+        }
+        else {
+	    my ($prefix, $namespace) = $self->_namespace($name);
+            undef $namespace unless $prefix;
+
+            my $newattr = XML::XPath::Node::Attribute->new($name, $value, $prefix);
+            $node->appendAttribute($newattr, 1);
+            if (exists($self->{IdNames}{$tag}) && ($self->{IdNames}{$tag} eq $name)) {
+    #            warn "appending Id Element: $val for ", $node->getName, "\n";
+                $self->{DOC_Node}->appendIdElement($value, $node);
+            }
+        }
+    }
+        
+    $self->{current}->appendChild($node, 1);
+    $self->{current} = $node;
+}
+
+sub parse_end {
+    my $self = shift;
+    my $e = shift;
+    $self->{current} = $self->{current}->getParentNode;
+}
+
+sub parse_pi {
+    my $self = shift;
+    my $e = shift;
+    my ($target, $data) = @_;
+    my $node = XML::XPath::Node::PI->new($target, $data);
+    $self->{current}->appendChild($node, 1);
+}
+
+sub parse_comment {
+    my $self = shift;
+    my $e = shift;
+    my ($data) = @_;
+    my $node = XML::XPath::Node::Comment->new($data);
+    $self->{current}->appendChild($node, 1);
+}
+
+sub parse_attlist {
+    my $self = shift;
+    my $e = shift;
+    my ($elname, $attname, $type, $default, $fixed) = @_;
+    if ($type eq 'ID') {
+        $self->{IdNames}{$elname} = $attname;
+    }
+}
+
+sub _scan_namespaces {
+    my ($self, %attributes) = @_;
+
+    while (my ($attr_name, $value) = each %attributes) {
+	if ($attr_name eq 'xmlns') {
+	    $self->{InScopeNamespaceStack}[-1]{'_Default'} = $value;
+	} elsif ($attr_name =~ /^xmlns:(.*)$/) {
+	    my $prefix = $1;
+	    $self->{InScopeNamespaceStack}[-1]{$prefix} = $value;
+	}
+    }
+}
+
+sub _namespace {
+    my ($self, $name) = @_;
+
+    my ($prefix, $localname) = split(/:/, $name);
+    if (!defined($localname)) {
+	if ($prefix eq 'xmlns') {
+	    return '', undef;
+	} else {
+	    return '', $self->{InScopeNamespaceStack}[-1]{'_Default'};
+	}
+    } else {
+	return $prefix, $self->{InScopeNamespaceStack}[-1]{$prefix};
+    }
+}
+
+sub as_string {
+    my $node = shift;
+    $node->toString;
+}
+
+sub get_parser { shift->{_parser}; }
+sub get_filename { shift->{_filename}; }
+sub get_xml { shift->{_xml}; }
+sub get_ioref { shift->{_ioref}; }
+
+sub set_parser { $_[0]->{_parser} = $_[1]; }
+sub set_filename { $_[0]->{_filename} = $_[1]; }
+sub set_xml { $_[0]->{_xml} = $_[1]; }
+sub set_ioref { $_[0]->{_ioref} = $_[1]; }
+
+1;
+
+__END__
+
+=head1 NAME
+
+XML::XPath::XMLParser - The default XML parsing class that produces a node tree
+
+=head1 SYNOPSIS
+
+    my $parser = XML::XPath::XMLParser->new(
+                filename => $self->get_filename,
+                xml => $self->get_xml,
+                ioref => $self->get_ioref,
+                parser => $self->get_parser,
+            );
+    my $root_node = $parser->parse;
+
+=head1 DESCRIPTION
+
+This module generates a node tree for use as the context node for XPath processing.
+It aims to be a quick parser, nothing fancy, and yet has to store more information
+than most parsers. To achieve this I've used array refs everywhere - no hashes.
+I don't have any performance figures for the speedups achieved, so I make no
+appologies for anyone not used to using arrays instead of hashes. I think they
+make good sense here where we know the attributes of each type of node.
+
+=head1 Node Structure
+
+All nodes have the same first 2 entries in the array: node_parent
+and node_pos. The type of the node is determined using the ref() function.
+The node_parent always contains an entry for the parent of the current
+node - except for the root node which has undef in there. And node_pos is the
+position of this node in the array that it is in (think: 
+$node == $node->[node_parent]->[node_children]->[$node->[node_pos]] )
+
+Nodes are structured as follows:
+
+=head2 Root Node
+
+The root node is just an element node with no parent.
+
+    [
+      undef, # node_parent - check for undef to identify root node
+      undef, # node_pos
+      undef, # node_prefix
+      [ ... ], # node_children (see below)
+    ]
+
+=head2 Element Node
+
+    [
+      $parent, # node_parent
+      <position in current array>, # node_pos
+      'xxx', # node_prefix - namespace prefix on this element
+      [ ... ], # node_children
+      'yyy', # node_name - element tag name
+      [ ... ], # node_attribs - attributes on this element
+      [ ... ], # node_namespaces - namespaces currently in scope
+    ]
+
+=head2 Attribute Node
+
+    [
+      $parent, # node_parent - the element node
+      <position in current array>, # node_pos
+      'xxx', # node_prefix - namespace prefix on this element
+      'href', # node_key - attribute name
+      'ftp://ftp.com/', # node_value - value in the node
+    ]
+
+=head2 Namespace Nodes
+
+Each element has an associated set of namespace nodes that are currently
+in scope. Each namespace node stores a prefix and the expanded name (retrieved
+from the xmlns:prefix="..." attribute).
+
+    [
+      $parent,
+      <pos>,
+      'a', # node_prefix - the namespace as it was written as a prefix
+      'http://my.namespace.com', # node_expanded - the expanded name.
+    ]
+
+=head2 Text Nodes
+
+    [
+      $parent,
+      <pos>,
+      'This is some text' # node_text - the text in the node
+    ]
+
+=head2 Comment Nodes
+
+    [
+      $parent,
+      <pos>,
+      'This is a comment' # node_comment
+    ]
+
+=head2 Processing Instruction Nodes
+
+    [
+      $parent,
+      <pos>,
+      'target', # node_target
+      'data', # node_data
+    ]
+
+=head1 Usage
+
+If you feel the need to use this module outside of XML::XPath (for example
+you might use this module directly so that you can cache parsed trees), you
+can follow the following API:
+
+=head2 new
+
+The new method takes either no parameters, or any of the following parameters:
+
+        filename
+        xml
+        parser
+        ioref
+
+This uses the familiar hash syntax, so an example might be:
+
+    use XML::XPath::XMLParser;
+    
+    my $parser = XML::XPath::XMLParser->new(filename => 'example.xml');
+
+The parameters represent a filename, a string containing XML, an XML::Parser
+instance and an open filehandle ref respectively. You can also set or get all
+of these properties using the get_ and set_ functions that have the same
+name as the property: e.g. get_filename, set_ioref, etc.
+
+=head2 parse
+
+The parse method generally takes no parameters, however you are free to
+pass either an open filehandle reference or an XML string if you so require.
+The return value is a tree that XML::XPath can use. The parse method will
+die if there is an error in your XML, so be sure to use perl's exception
+handling mechanism (eval{};) if you want to avoid this.
+
+=head2 parsefile
+
+The parsefile method is identical to parse() except it expects a single
+parameter that is a string naming a file to open and parse. Again it
+returns a tree and also dies if there are XML errors.
+
+=head1 NOTICES
+
+This file is distributed as part of the XML::XPath module, and is copyright
+2000 Fastnet Software Ltd. Please see the documentation for the module as a
+whole for licencing information.
Index: /main/tags/3.01/gsdl/perllib/cpan/rm/Header/PurePerl.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/cpan/rm/Header/PurePerl.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/cpan/rm/Header/PurePerl.pm	(revision 21112)
@@ -0,0 +1,228 @@
+package rm::Header::PurePerl;
+
+use 5.005;
+use strict;
+use warnings;
+
+use Fcntl qw/SEEK_END/;
+
+our $VERSION = '0.07';
+
+sub new 
+{
+    my $class = shift;
+    my $file = shift;
+
+    return load($class, $file);
+}
+
+sub load 
+{
+    my $class    = shift;
+    my $file     = shift;
+    my $from_new = shift;
+    my %data;
+    my $self;
+
+    # there must be a better way...
+    if ($class eq 'rm::Header::PurePerl')
+    {
+	$self = bless \%data, $class;
+    }
+    else
+    {
+	$self = $class;
+    }
+
+    if ($self->{'FILE_LOADED'})
+    {
+	return $self;
+    }
+
+    $self->{'FILE_LOADED'} = 1;
+
+    # check that the file exists and is readable
+    unless ( -e $file && -r _ )
+    {
+	warn "File does not exist or cannot be read.";
+	# file does not exist, can't do anything
+	return undef;
+    }
+    # open up the file
+    open FILE, $file;
+    # make sure dos-type systems can handle it...
+    binmode FILE;
+
+    $data{'filename'} = $file;
+    $data{'fileHandle'} = \*FILE;
+
+    _loadInfo(\%data);
+
+    close FILE;
+
+    return $self;
+}
+
+sub info 
+{
+    my $self = shift;
+    my $key = shift;
+
+    # if the user did not supply a key, return the entire hash
+    unless ($key)
+    {
+	return $self->{'INFO'};
+    }
+
+    # otherwise, return the value for the given key
+    return $self->{'INFO'}{lc $key};
+}
+
+sub _loadInfo
+{
+    my $data = shift;
+    my $start = 0;
+    my $fh = $data->{'fileHandle'};
+    my $buffer;
+    my $byteCount = $start;
+    my %info;
+    
+    # check that the first four bytes are '.RMF'
+    read($fh, $buffer, 4);
+    if ($buffer ne '.RMF')
+    {
+	warn "No RMF header?";
+	return undef;
+    }
+
+$buffer='';
+my $char;
+
+#find the header
+my $bytes = "DATA";
+my @byteList = split //, $bytes;
+my $numBytes = @byteList;
+my $i;
+
+LINE:   while (1){
+  INNER:  for ($i = 0; $i < $numBytes; $i ++)
+    {
+  unless ( read($fh, $char, 1) ) {last LINE ;}
+	# Find out all of char
+	$buffer= $buffer.$char;  
+
+if (ord($char) !=  ord($byteList[$i]) ) 
+{last  INNER ;}
+    }
+if ($i == $numBytes) {last LINE ;} #jump out the while loop
+        }
+
+#find the tail
+ $bytes = "INDX";
+ @byteList = split //, $bytes;
+ $numBytes = @byteList;
+
+my $isrecord=0;
+LINE:  while (read($fh, $char, 1)){
+   if ($isrecord) {
+	# Find out all of char
+	$buffer= $buffer.$char;  
+   }else
+        {
+         INNER:  for ($i = 0; $i < $numBytes; $i ++)
+                     {
+                      if (ord($char) !=  ord($byteList[$i]) ) {last  INNER ;}
+                      unless ( read($fh, $char, 1) ) {last LINE ;}
+                     }
+if ($i == $numBytes) {$isrecord = 1;} #start record
+        }
+       }
+
+my @cliptype = (
+
+#add clip type here
+"Comments",
+"Keywords",
+"Category",
+"MimeType",# title
+"Lyrics",
+"Artist",
+"CD Track #",
+"Album",
+"Extension",
+"Genre",
+"Statistics",
+"PROP",
+"MDPR",
+"Target Audiences",
+"Audio Format",
+"Creation Date",
+"Modification Date",
+"Generated By",
+"Abstract",
+"Content Rating",
+"File ID",
+"CONT",
+"Audio Stream",
+"Video Stream",
+"Title"
+);
+
+for my $j ( 1 .. scalar(@cliptype) ) {
+$info{$cliptype[$j - 1]} = _loadInfor($buffer,$cliptype[$j - 1]);
+}
+
+    $data->{'INFO'} = \%info;
+}
+
+#search for the element name and value
+sub _loadInfor
+{my $data = shift;
+my $item = shift;
+
+my @byteList = split //, $data;
+my $startbyte = 0;
+
+
+my $isrecord;
+my $data2 = "";
+my $char;
+my $item2 = "";
+if ( $item eq "Title") {$item2 = $item; $item = "MimeType";}
+
+
+OUT: while(index($data, $item, $startbyte) != -1){
+$startbyte = index($data, $item, $startbyte);
+$isrecord=0;
+
+$startbyte += length($item);
+
+if(ord($byteList[$startbyte]) == 0 or ord($byteList[$startbyte]) == 0x14){
+  if ( $item eq "Album" or $item eq "Artist" or $item2 eq "Title"){
+     if ( index($data,"Name",$startbyte) != -1) {
+     $startbyte = index($data,"Name",$startbyte);
+     $startbyte += length("Name");
+     }else {next OUT;}
+}
+
+  if ($data2 ne "") {$data2 = $data2."; ";}
+
+LINE:   while (1){
+        $char = $byteList[++$startbyte];
+
+	if (ord($char) >= 32 and ord($char) <= 126)
+	{
+        $isrecord=1; #record the string started
+	$data2 = $data2.$char;  
+	}else{
+             if ( $isrecord == 1 ) {last LINE ;}# stop at the end of string
+             }
+        }# end LINE: while
+
+ }# end if 
+}# end while
+
+return $data2;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/doc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/doc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/doc.pm	(revision 21112)
@@ -0,0 +1,1580 @@
+###########################################################################
+#
+# doc.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistr   te it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# base class to hold documents
+
+package doc;
+eval {require bytes};
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/dynamic/lib/site_perl/5.005/i686-linux");
+}
+
+use unicode;
+use util;
+use ghtml;
+use File::stat;
+##use hashdoc;
+
+# the document type may be indexed_doc, nonindexed_doc, or
+# classification
+
+my $OIDcount = 0;
+
+sub new {
+    my $class = shift (@_);
+    my ($source_filename, $doc_type) = @_;
+
+    my $self = bless {'associated_files'=>[],
+		      'subsection_order'=>[],
+		      'next_subsection'=>1,
+		      'subsections'=>{},
+		      'metadata'=>[],
+		      'text'=>"",
+		      'OIDtype'=>"hash"}, $class;
+
+    # used to set lastmodified here, but this can screw up the HASH ids, so
+    # the docsave processor now calls set_lastmodified
+    
+    if (defined $source_filename) {
+	my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
+
+	if (defined $collect_dir) {
+	    my $dirsep = &util::get_dirsep();
+	    if ($collect_dir !~ m/$dirsep$/) {
+		$collect_dir .= $dirsep;
+	    }
+
+	    $collect_dir =~ s/\\/\\\\/g; # escape DOS style file separator
+
+	    # if from within GSDLCOLLECTDIR, then remove directory prefix
+	    # so source_filename is realative to it.  This is done to aid
+	    # portability, i.e. the collection can be moved to somewhere
+	    # else on the file system and the archives directory will still
+	    # work.  This is needed, for example in the applet version of
+	    # GLI where GSDLHOME/collect on the server will be different to
+	    # the collect directory of the remove user.  Of course,
+	    # GSDLCOLLECTDIR subsequently needs to be put back on to turn
+	    # it back into a full pathname.
+
+	    if ($source_filename =~ /^$collect_dir(.*)$/) {
+		$source_filename = $1;
+	    }
+	}
+
+	$self->set_source_filename ($source_filename);
+    }
+
+    $self->set_doc_type ($doc_type) if defined $doc_type;
+
+    return $self;
+}
+# set lastmodified for OAI purposes, added by GRB, moved by kjdon
+sub set_lastmodified {
+    my $self = shift (@_);
+
+    my $source_filename = $self->get_source_filename();
+    if ((defined $self->get_metadata_element ($self->get_top_section(), "gsdldoctype")) &&
+	(defined $source_filename) && (-e $source_filename)) {
+	my $file_stat = stat($source_filename);
+	my $mtime = $file_stat->mtime;
+	$self->add_utf8_metadata($self->get_top_section(), "lastmodified", $file_stat->mtime);
+    } 
+}
+
+# clone the $self object
+sub duplicate {
+    my $self = shift (@_);
+
+    my $newobj = {};
+    
+    foreach my $k (keys %$self) {
+	$newobj->{$k} = &clone ($self->{$k});
+    }
+
+    bless $newobj, ref($self);
+    return $newobj;
+}
+
+sub clone {
+    my ($from) = @_;
+    my $type = ref ($from);
+
+    if ($type eq "HASH") {
+	my $to = {};
+	foreach my $key (keys %$from) {
+	    $to->{$key} = &clone ($from->{$key});
+	}
+	return $to;
+    } elsif ($type eq "ARRAY") {
+	my $to = [];
+	foreach my $v (@$from) {
+	    push (@$to, &clone ($v));
+	}
+	return $to;
+    } else {
+	return $from;
+    }
+}
+
+sub set_OIDtype {
+    my $self = shift (@_);
+    my ($type) = @_;
+
+    if (defined $type && $type =~ /^(hash|incremental|dirname|assigned)$/) {
+	$self->{'OIDtype'} = $type;
+    } else {
+	$self->{'OIDtype'} = "hash";
+    }
+}
+
+sub set_source_filename {
+    my $self = shift (@_);
+    my ($source_filename) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdlsourcefilename", 
+				 $source_filename);
+}
+
+sub set_converted_filename {
+    my $self = shift (@_);
+    my ($converted_filename) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdlconvertedfilename", 
+				 $converted_filename);
+}
+
+
+# returns the source_filename as it was provided
+sub get_source_filename {
+    my $self = shift (@_);
+
+    return $self->get_metadata_element ($self->get_top_section(), "gsdlsourcefilename");
+}
+
+# returns converted filename if available else returns source filename
+sub get_filename_for_hashing {
+    my $self = shift (@_);
+
+    my $filename = $self->get_metadata_element ($self->get_top_section(), "gsdlconvertedfilename");
+
+    if (!defined $filename) {
+	$filename = $self->get_metadata_element ($self->get_top_section(), "gsdlsourcefilename");
+    }
+    return $filename;
+}
+
+sub set_doc_type {
+    my $self = shift (@_);
+    my ($doc_type) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdldoctype", 
+				 $doc_type);
+}
+
+# returns the gsdldoctype as it was provided
+# the default of "indexed_doc" is used if no document
+# type was provided
+sub get_doc_type {
+    my $self = shift (@_);
+
+    my $doc_type = $self->get_metadata_element ($self->get_top_section(), "gsdldoctype");
+    return $doc_type if (defined $doc_type);
+    return "indexed_doc";
+}
+
+sub _escape_text {
+    my ($text) = @_;
+
+    # special characters in the gml encoding
+    $text =~ s/&/&amp;/g; # this has to be first...
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/\"/&quot;/g;
+
+    return $text;
+}
+
+sub buffer_section_xml {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section ($section);
+    return "" unless defined $section_ptr;
+
+    my $all_text = "<Section>\n";
+    $all_text .= "  <Description>\n";
+    
+    # output metadata
+    foreach my $data (@{$section_ptr->{'metadata'}}) {
+	my $escaped_value = &_escape_text($data->[1]);
+	$all_text .= '    <Metadata name="' . $data->[0] . '">' . $escaped_value . "</Metadata>\n";
+    }
+
+    $all_text .= "  </Description>\n";
+
+    # output the text
+    $all_text .= "  <Content>";
+    $all_text .= &_escape_text($section_ptr->{'text'});
+    $all_text .= "</Content>\n";
+    
+    # output all the subsections
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
+	$all_text .= $self->buffer_section_xml("$section.$subsection");
+    }
+    
+    $all_text .=  "</Section>\n";
+
+    # make sure no nasty control characters have snuck through
+    # (XML::Parser will barf on anything it doesn't consider to be
+    # valid UTF-8 text, including things like \c@, \cC etc.)
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+sub buffer_txt_section_xml {
+    my $self = shift(@_);
+    my ($section) = @_;
+ 
+    my $section_ptr = $self->_lookup_section ($section);
+    
+    return "" unless defined $section_ptr;
+   
+    my $all_text = "<Section>\n";
+   
+    ##output the text
+    #$all_text .= "   <Content>";
+    $all_text .= &_escape_text($section_ptr->{'text'});
+    #$all_text .= "   </Content>\n";
+    
+   
+    #output all the subsections
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}){
+       $all_text .= $self->buffer_txt_section_xml("$section.$subsection");
+     }
+
+     $all_text .= "</Section>\n";
+
+     
+     $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+     return $all_text;
+}
+
+sub buffer_mets_fileSection_section_xml() {
+    my $self = shift(@_);
+    my ($section,$version) = @_;
+
+    #$section="" unless defined $section;
+    
+
+    my $section_ptr=$self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+ 
+
+    # output fileSection by sections
+    my $section_num ="1". $section;
+      
+    my $filePath = 'doctxt.xml';
+
+    my $opt_owner_id = "";
+    if ($version eq "fedora") {
+	$opt_owner_id = "OWNERID=\"M\"";
+    }
+
+    # output the fileSection details
+    my $all_text = '  <mets:fileGrp ID="FILEGROUP_PRELUDE' . $section_num . '">'. "\n";
+    $all_text .= "    <mets:file MIMETYPE=\"text/xml\" ID=\"FILE$section_num\" $opt_owner_id >\n";
+    $all_text .= '      <mets:FLocat LOCTYPE="URL" xlink:href="file:'.$filePath.'#xpointer(/Section[';
+    
+    my $xpath = "1".$section;
+    $xpath =~ s/\./]\/Section[/g;
+   
+    $all_text .=  $xpath;
+
+    $all_text .= ']/text())" xlink:title="Hierarchical Document Structure"/>' . "\n";
+    $all_text .= "    </mets:file>\n";
+    $all_text .= "  </mets:fileGrp>\n";
+
+
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}){
+	$all_text .= $self->buffer_mets_fileSection_section_xml("$section.$subsection",$version);
+    }
+    
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+sub buffer_mets_fileWhole_section_xml(){
+    my $self = shift(@_);
+    my ($section,$version,$working_dir) = @_;
+
+    my $section_ptr = $self-> _lookup_section($section);
+    return "" unless defined $section_ptr;
+    
+    my $all_text="" unless defined $all_txt;
+
+    my $fileID=0;
+
+    # Output the fileSection for the whole section
+    #  => get the sourcefile and associative file
+
+    my $id_root = "";
+    my $opt_owner_id = "";    
+
+    if ($version eq "fedora") {
+	$opt_owner_id = "OWNERID=\"M\"";
+    }
+    else {
+	$id_root = "default";
+    }
+
+    if ($version ne "fedora") {
+	$all_text .= "  <mets:fileGrp ID=\"$id_root\">\n";
+    }
+
+    foreach my $data (@{$section_ptr->{'metadata'}}){
+       my $escaped_value = &_escape_text($data->[1]);
+
+       if (($data->[0] eq "gsdlsourcefilename") && ($version ne "fedora")) { 
+          my ($dirPath) = $escaped_value =~ m/^(.*)[\/\\][^\/\\]*$/;
+
+          ++$fileID;	  
+          $all_text .= "    <mets:file MIMETYPE=\"text/xml\" ID=\"$id_root.$fileID\" $opt_owner_id >\n";
+        
+	  $all_text .= '      <mets:FLocat LOCTYPE="URL" xlink:href="file:'.$data->[1].'" />'."\n";
+	  
+          $all_text .= "    </mets:file>\n";
+       }
+       
+       if ($data->[0] eq "gsdlassocfile"){
+	   
+	   $escaped_value =~ m/^(.*?):(.*):(.*)$/;
+	   my $assoc_file = $1;
+	   my $mime_type  = $2;
+	   my $assoc_dir  = $3;
+	   
+	   if ($version eq "fedora") {
+	       $id_root = $assoc_file;
+	       $id_root =~ s/\//_/g;
+	       $all_text .= "  <mets:fileGrp ID=\"$id_root\">\n";
+	   }
+	   
+	   my $assfilePath = ($assoc_dir eq "") ? $assoc_file : "$assoc_dir/$assoc_file";
+	   ++$fileID;
+	   
+	   my $mime_attr   = "MIMETYPE=\"$mime_type\"";
+	   my $xlink_title = "xlink:title=\"$assoc_file\"";
+
+	   my $id_attr;
+	   my $xlink_href;
+
+	   if ($version eq "fedora") {
+	       $id_attr = "ID=\"$id_root.0\"";
+
+	       my $fedora_prefix = $ENV{'FEDORA_PREFIX'};
+	       if (!defined $fedora_prefix) {
+		   $xlink_href  = "xlink:href=\"$assfilePath\"";
+	       }
+	       else
+	       {
+		   my $gsdlhome = $ENV{'GSDLHOME'};
+		   my $gsdl_href = "$working_dir/$assfilePath";
+
+		   $gsdl_href =~ s/^$gsdlhome(\/)?//;
+		   $gsdl_href = "/gsdl/$gsdl_href";
+
+		   my $fserver = $ENV{'FEDORA_HOSTNAME'};
+		   my $fport = $ENV{'FEDORA_SERVER_PORT'};
+
+		   my $fdomain = "http://$fserver:$fport";
+		   $xlink_href  = "xlink:href=\"$fdomain$gsdl_href\"";
+	       }
+
+	       my $top_section = $self->get_top_section();
+	       my $id = $self->get_metadata_element($top_section,"Identifier");
+	   }
+	   else {
+	       $id_attr = "ID=\"$id_root.$fileID\"";
+	       $xlink_href  = "xlink:href=\"$assfilePath\"";
+	   }
+
+	   $all_text .= "    <mets:file $mime_attr $id_attr $opt_owner_id >\n";
+	   $all_text .= "      <mets:FLocat LOCTYPE=\"URL\" $xlink_href $xlink_title />\n";
+	   
+	   $all_text .= "    </mets:file>\n";
+	   
+	   if ($version eq "fedora") {
+	       $all_text .= "  </mets:fileGrp>\n";
+	   }
+	   
+       }
+   }
+    
+    if ($version ne "fedora") {
+	$all_text .= "  </mets:fileGrp>\n";
+    }         
+    
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+    
+    return $all_text;
+}
+
+sub buffer_mets_StructMapSection_section_xml(){
+    my $self = shift(@_);
+    my ($section, $order_numref) = @_;
+
+    $section="" unless defined $section;
+    
+    
+    my $section_ptr=$self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+
+    # output fileSection by Sections
+    my $section_num ="1". $section;
+    my $dmd_num = $section_num;
+
+    ##**output the dmdSection details  
+    #if ($section_num eq "1") {
+    #	$dmd_num = "0";
+    #}
+
+    #**output the StructMap details
+ 
+    my $dmdid_attr = "DM$dmd_num";
+
+    my $all_text = "  <mets:div ID=\"DS$section_num\" TYPE=\"Section\" \n";
+    $all_text .= '      ORDER="'.$$order_numref++.'" ORDERLABEL="'. $section_num .'" '."\n";
+    $all_text .= "      LABEL=\"$section_num\" DMDID=\"$dmdid_attr\">\n";
+   
+    $all_text .= '    <mets:fptr FILEID="FILEGROUP_PRELUDE'.$section_num.'" />'. "\n";
+
+
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}){
+       $all_text .= $self->buffer_mets_StructMapSection_section_xml("$section.$subsection", $order_numref);
+    }
+    
+    $all_text .= "  </mets:div>\n";
+
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+
+sub buffer_mets_StructMapWhole_section_xml(){
+    my $self = shift(@_);
+    my ($section) = @_;
+    
+    my $section_ptr = $self-> _lookup_section($section);
+    return "" unless defined $section_ptr;
+    
+    my $all_text="" unless defined $all_txt;   
+    my $fileID=0;
+    my $order_num = 0;
+
+    $all_text .= '  <mets:div ID="DSAll" TYPE="Document" ORDER="'.$order_num.'" ORDERLABEL="All" LABEL="Whole Documemt" DMDID="DM1">' . "\n";
+  
+    #** output the StructMapSection for the whole section
+    #  get the sourcefile and associative file
+
+    foreach my $data (@{$section_ptr->{'metadata'}}){
+       my $escaped_value = &_escape_text($data->[1]);
+   
+       if ($data->[0] eq "gsdlsourcefilename") { 
+          ++$fileID;
+	  $all_text .= '    <mets:fptr FILEID="default.'.$fileID.'" />'."\n";
+       }
+       
+       if ($data->[0] eq "gsdlassocfile"){
+          ++$fileID;
+	  $all_text .= '    <mets:fptr FILEID="default.'.$fileID. '" />'. "\n";
+       }
+    }
+    $all_text .= "  </mets:div>\n";
+    
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+    
+    return $all_text;
+}
+
+
+sub buffer_mets_dmdSection_section_xml(){
+    my $self = shift(@_);
+    my ($section,$version) = @_;
+   
+    $section="" unless defined $section;
+    
+    my $section_ptr=$self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+    # convert section number
+    my $section_num ="1". $section;
+    my $dmd_num = $section_num;
+
+    # #**output the dmdSection details  
+    # if ($section_num eq "1") {
+    #	$dmd_num = "0";
+    # }
+
+
+    my $all_text = "";
+
+    my $label_attr = "";
+    if ($version eq "fedora") {
+	$all_text .= "<mets:amdSec ID=\"DC\" >\n";
+	$all_text .= "  <mets:techMD ID=\"DC.0\">\n"; # .0 fedora version number?
+
+	$label_attr = "LABEL=\"Dublin Core Metadata\"";
+    }
+    else {
+	print STDERR "***** Check that GROUPID in dmdSec is valid!!!\n";
+	print STDERR "***** Check to see if <techMD> required\n";
+	# if it isn't allowed, go back and set $mdTag = dmdSec/amdSec
+
+	$all_text .= "<mets:dmdSec ID=\"DM$dmd_num\" GROUPID=\"$section_num\">\n";
+    }
+
+    $all_text .= "  <mets:mdWrap $label_attr MDTYPE=\"OTHER\" OTHERMDTYPE=\"gsdl3\" ID=\"gsdl$section_num\">\n";
+    $all_text .= "    <mets:xmlData>\n";
+
+    if ($version eq "fedora") {
+	my $dc_namespace = "";
+	$dc_namespace .= "xmlns:dc=\"http://purl.org/dc/elements/1.1/\"";
+	$dc_namespace .= " xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\">\n";
+
+	$all_text .= "  <oai_dc:dc $dc_namespace>\n";
+
+	$all_text .= $self->buffer_dc_section($section,"oai_dc");
+	$all_text .= "  </oai_dc:dc>\n";
+    }
+    else {
+	foreach my $data (@{$section_ptr->{'metadata'}}){
+	    my $escaped_value = &_escape_text($data->[1]);
+	    $all_text .= '      <gsdl3:Metadata name="'. $data->[0].'">'. $escaped_value. "</gsdl3:Metadata>\n";
+	    if ($data->[0] eq "dc.Title") {
+		$all_text .= '      <gsdl3:Metadata name="Title">'. $escaped_value."</gsdl3:Metadata>\n";
+	    }
+	}
+    }
+   
+    $all_text .= "    </mets:xmlData>\n";
+    $all_text .= "  </mets:mdWrap>\n";
+    
+    if ($version eq "fedora") {
+	$all_text .= "  </mets:techMD>\n";
+	$all_text .= "</mets:amdSec>\n";
+    }
+    else {
+	$all_text .= "</mets:dmdSec>\n";    
+    }
+
+
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}){
+       $all_text .= $self->buffer_mets_dmdSection_section_xml("$section.$subsection",$version);
+    }
+    
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+sub output_section {
+    my $self = shift (@_);
+    my ($handle, $section) = @_;
+
+    print $handle $self->buffer_section_xml($section);
+}
+
+#  print out DSpace dublin_core metadata section
+sub output_dspace_section {
+    my $self = shift (@_);
+    my ($handle, $section) = @_;
+
+    my $section_ptr = $self->_lookup_section ($section);
+    return "" unless defined $section_ptr;
+
+    my $all_text = "<Section>\n";
+    $all_text .= "  <Description>\n";
+    
+    # output metadata
+    foreach my $data (@{$section_ptr->{'metadata'}}) {
+	my $escaped_value = &_escape_text($data->[1]);
+	$all_text .= '    <Metadata name="' . $data->[0] . '">' . $escaped_value . "</Metadata>\n";
+    }
+
+    $all_text .= "  </Description>\n";
+    $all_text .=  "</Section>\n";
+
+    # make sure no nasty control characters have snuck through
+    # (XML::Parser will barf on anything it doesn't consider to be
+    # valid UTF-8 text, including things like \c@, \cC etc.)
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+#  print out doctxt.xml file
+sub output_txt_section {
+    my $self = shift (@_);
+    my ($handle, $section) = @_;
+
+    print $handle $self->buffer_txt_section_xml($section);
+}
+
+#  print out docmets.xml file
+sub output_mets_section {
+    my $self = shift(@_);
+    my ($handle, $section, $version, $working_dir) = @_;
+
+    # print out the dmdSection
+    print $handle $self->buffer_mets_dmdSection_section_xml($section, $version);
+
+    print $handle "<mets:fileSec>\n";
+    if ($version eq "fedora") {
+	print $handle "  <mets:fileGrp ID=\"DATASTREAMS\">\n";
+    }
+
+    # print out the fileSection by sections
+    print $handle $self->buffer_mets_fileSection_section_xml($section,$version);
+
+    # print out the whole fileSection
+    print $handle $self->buffer_mets_fileWhole_section_xml($section,$version,$working_dir); 
+
+    if ($version eq "fedora") {
+	print $handle "  </mets:fileGrp>\n";
+    }
+    print $handle "</mets:fileSec>\n";
+  
+    # print out the StructMapSection by sections
+
+    my $struct_type;
+    if ($version eq "fedora") {
+	$struct_type = "fedora:dsBindingMap";
+    }
+    else {
+	$struct_type = "Section";
+    }
+
+    if ($version ne "fedora") {
+	print $handle "<mets:structMap ID=\"Section\" TYPE=\"$struct_type\" LABEL=\"Section\">\n";
+	my $order_num=0;
+	print $handle $self->buffer_mets_StructMapSection_section_xml($section, \$order_num);
+	print $handle "</mets:structMap>\n";
+
+	print $handle '<mets:structMap ID="All" TYPE="Whole Document" LABEL="All">'."\n";
+	print $handle $self->buffer_mets_StructMapWhole_section_xml($section);
+	print $handle "</mets:structMap>\n";
+    }
+  
+}
+
+my $dc_set = { Title => 1,       
+	       Creator => 1, 
+	       Subject => 1, 
+	       Description => 1, 
+	       Publisher => 1, 
+	       Contributors => 1, 
+	       Date => 1, 
+	       Type => 1, 
+	       Format => 1, 
+	       Identifier => 1, 
+	       Source => 1, 
+	       Language => 1, 
+	       Relation => 1, 
+	       Coverage => 1, 
+	       Rights => 1};
+
+
+
+
+# Build up dublin_core metadata.  Priority given to dc.* over ex.*
+
+sub buffer_dc_section {
+    my $self = shift(@_);
+    my ($section, $version) = @_;
+    
+    # build up string of dublin core metadata
+    $section="" unless defined $section;
+    
+    my $section_ptr=$self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+    my $explicit_dc = {};
+    my $explicit_ex = {};
+
+    my $all_text="";
+    foreach my $data (@{$section_ptr->{'metadata'}}){
+	my $escaped_value = &_escape_text($data->[1]);
+	if ($data->[0]=~ m/^dc\./) {
+	    $data->[0] =~ tr/[A-Z]/[a-z]/;
+
+	    $data->[0] =~ m/^dc\.(.*)/;
+	    my $dc_element =  $1;
+
+	    if (!defined $explicit_dc->{$dc_element}) {
+		$explicit_dc->{$dc_element} = [];
+	    }
+	    push(@{$explicit_dc->{$dc_element}},$escaped_value);
+
+	    #$all_text .= '   <dcvalue element="'. $data->[0].'" qualifier="#####">'. $escaped_value. "</dcvalue>\n";
+	    if (defined $version && ($version eq "oai_dc")) {
+		$all_text .= "   <dc:$dc_element>$escaped_value</dc:$dc_element>\n";
+	    }
+	    else {
+		$all_text .= '   <dcvalue element="'. $dc_element.'">'. $escaped_value. "</dcvalue>\n";
+	    }
+
+	} 
+	elsif (($data->[0] =~ m/^ex\./) || ($data->[0] !~ m/\./)) {
+	    $data->[0] =~ m/^(ex\.)?(.*)/;
+	    my $ex_element =  $2;
+	    my $lc_ex_element = lc($ex_element);
+
+	    if (defined $dc_set->{$ex_element}) {
+		if (!defined $explicit_ex->{$lc_ex_element}) {
+		    $explicit_ex->{$lc_ex_element} = [];
+		}
+		push(@{$explicit_ex->{$lc_ex_element}},$escaped_value);
+	    }
+	}
+    }
+
+    # go through dc_set and for any element *not* defined in explicit_dc
+    # that do exist in explicit_ex, add it in as metadata
+    foreach my $k ( keys %$dc_set ) {
+	my $lc_k = lc($k);
+
+	if (!defined $explicit_dc->{$lc_k}) {
+	    if (defined $explicit_ex->{$lc_k}) {
+
+		foreach my $v (@{$explicit_ex->{$lc_k}}) {
+		    my $dc_element    = $lc_k;
+		    my $escaped_value = $v;
+
+		    if (defined $version && ($version eq "oai_dc")) {
+			$all_text .= "   <dc:$dc_element>$escaped_value</dc:$dc_element>\n";
+		    }
+		    else {
+			$all_text .= '   <dcvalue element="'. $dc_element.'">'. $escaped_value. "</dcvalue>\n";
+		    }
+		    
+		}
+	    }
+	}
+    }
+
+    if ($all_text eq "") {
+	$all_text .= "   There is no Dublin Core metatdata in this document\n";
+    }	
+    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
+
+    return $all_text;
+}
+
+
+# Print out dublin_core metadata
+sub output_dc_section {
+    my $self = shift(@_);
+    my ($handle, $section, $version) = @_;
+    
+    my $all_text = $self->buffer_dc_section($section,$version);
+
+    print $handle $all_text;
+}
+
+
+# look up the reference to the a particular section
+sub _lookup_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my ($num);
+    my $sectionref = $self;
+
+    while (defined $section && $section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num && defined $sectionref->{'subsections'}->{$num}) {
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+	} else {
+	    return undef;
+	}
+    }
+    
+    return $sectionref;
+}
+
+# calculate OID by hashing the contents of the document
+sub _calc_OID {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my $osexe = &util::get_os_exe();
+
+    my $hashfile_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",
+					   $ENV{'GSDLOS'},"hashfile$osexe");
+
+    my $result = "NULL";
+    
+    if (-e "$hashfile_exe") {
+#	$result = `\"$hashfile_exe\" \"$filename\"`;
+	$result = `hashfile$osexe \"$filename\"`;
+	($result) = $result =~ /:\s*([0-9a-f]+)/i;
+
+    } else {
+	print STDERR "doc::_calc_OID $hashfile_exe could not be found\n";
+    }
+    return "HASH$result";
+}
+
+# methods dealing with OID, not groups of them.
+
+# if $OID is not provided one is calculated
+sub set_OID {
+    my $self = shift (@_);
+    my ($OID) = @_;
+    
+    my $use_hash_oid = 0;
+    # if an OID wasn't provided claculate one
+    if (!defined $OID) {
+	$OID = "NULL";
+	if ($self->{'OIDtype'} eq "hash") {
+	    $use_hash_oid = 1;
+	} elsif ($self->{'OIDtype'} eq "incremental") {
+	    $OID = "D" . $OIDcount;
+	    $OIDcount ++;
+	  
+	} elsif ($self->{'OIDtype'} eq "dirname") {
+	    $OID = 'J';
+	    my $filename = $self->get_source_filename();
+	    if (defined($filename)) { # && -e $filename) {
+		$OID = &File::Basename::dirname($filename);
+		if (defined $OID) {
+		    $OID = 'J'.&File::Basename::basename($OID);
+		    $OID =~ s/\.//; #remove any periods
+		} else {
+		    print STDERR "Failed to find base for filename ($filename)...generating hash id\n";
+		    $use_hash_oid = 1;
+		}
+	    } else {
+		print STDERR "Failed to find filename, generating hash id\n";
+		$use_hash_oid = 1;
+	    }    
+	    
+	} elsif ($self->{'OIDtype'} eq "assigned") {
+	    my $identifier = $self->get_metadata_element ($self->get_top_section(), "dc.Identifier");
+	    if (defined $identifier && $identifier ne "") {
+		$OID = "D" . $self->get_metadata_element ($self->get_top_section(), "dc.Identifier");
+		$OID =~ s/\.//; #remove any periods
+	    } else {
+		# need a hash id
+		print STDERR "no dc.Identifier found, generating hash id\n";
+		$use_hash_oid = 1;
+	    }
+	    	    
+	} else {
+	    $use_hash_oid = 1;
+	}
+
+	if ($use_hash_oid) {
+	    
+	    # "hash" OID - feed file to hashfile.exe
+	    #my $filename = $self->get_source_filename();
+	    # we want to use the converted file for hashing if available
+	    # cos its quicker
+	    my $filename = $self->get_filename_for_hashing();
+
+	    # -z: don't want to hash on the file if it is zero size
+	    if (defined($filename) && -e $filename && !-z $filename) {
+		$OID = $self->_calc_OID ($filename);
+	    } else {
+		$filename = &util::get_tmp_filename();
+		if (!open (OUTFILE, ">$filename")) {
+		    print STDERR "doc::set_OID could not write to $filename\n";
+		} else {
+		    $self->output_section('OUTFILE', $self->get_top_section(), 
+					  undef, 1);
+		    close (OUTFILE);
+		}
+		$OID = $self->_calc_OID ($filename);
+		&util::rm ($filename);
+	    }
+	}
+    }
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", $OID);
+}
+
+# this uses hashdoc (embedded c thingy) which is faster but still 
+# needs a little work to be suffiently stable
+sub ___set_OID {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    # if an OID wasn't provided then calculate hash value based on document
+    if (!defined $OID) 
+    {
+	my $hash_text = $self->buffer_section_gml($self->get_top_section(),
+						  undef, 1);
+	my $hash_len = length($hash_text);
+
+        $OID = &hashdoc::buffer($hash_text,$hash_len);
+    }
+
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", $OID);
+}
+
+# returns the OID for this document
+sub get_OID {
+    my $self = shift (@_);
+    my $OID = $self->get_metadata_element ($self->get_top_section(), "Identifier");
+    return $OID if (defined $OID);
+    return "NULL";
+}
+
+sub delete_OID {
+    my $self = shift (@_);
+    
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", "NULL");
+}
+
+
+# methods for manipulating section names
+
+# returns the name of the top-most section (the top
+# level of the document
+sub get_top_section {
+    my $self = shift (@_);
+    
+    return "";
+}
+
+# returns a section
+sub get_parent_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    $section =~ s/(^|\.)\d+$//;
+
+    return $section;
+}
+
+# returns the first child section (or the end child 
+# if there isn't any)
+sub get_begin_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    return $self->get_end_child ($section);
+}
+
+# returns the next child of a parent section
+sub get_next_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+    
+    my $parent_section = $self->get_parent_section($section);
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    return undef unless defined $parent_section_ptr;
+
+    my ($section_num) = $section =~ /(\d+)$/;
+    return undef unless defined $section_num;
+
+    my $i = 0;
+    my $section_order = $parent_section_ptr->{'subsection_order'};
+    while ($i < scalar(@$section_order)) {
+	last if $section_order->[$i] eq $section_num;
+	$i++;
+    }
+
+    $i++; # the next child
+    if ($i < scalar(@$section_order)) {
+	return $section_order->[$i] if $parent_section eq "";
+	return "$parent_section.$section_order->[$i]";
+    }
+
+    # no more sections in this level
+    return undef;
+}
+
+# returns a reference to a list of children
+sub get_children {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return [] unless defined $section_ptr;
+
+    my @children = @{$section_ptr->{'subsection_order'}};
+
+    map {$_ = "$section.$_"; $_ =~ s/^\.+//;} @children;
+    return \@children;
+}
+
+# returns the child section one past the last one (which
+# is coded as "0")
+sub get_end_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return $section . ".0" unless $section eq "";
+    return "0";
+}
+
+# returns the next section in book order
+sub get_next_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return undef unless defined $section;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return undef unless defined $section_ptr;
+
+    # first try to find first child
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return $section_ptr->{'subsection_order'}->[0] if ($section eq "");
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    do {
+	# try to find sibling
+	my $next_child = $self->get_next_child ($section);
+	return $next_child if (defined $next_child);
+
+	# move up one level
+	$section = $self->get_parent_section ($section);
+    } while $section =~ /\d/;
+
+    return undef;
+}
+
+sub is_leaf_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return 1 unless defined $section_ptr;
+
+    return (scalar (@{$section_ptr->{'subsection_order'}}) == 0);
+}
+
+# methods for dealing with sections
+
+# returns the name of the inserted section
+sub insert_section {
+    my $self = shift (@_);
+    my ($before_section) = @_;
+
+    # get the child to insert before and its parent section
+    my $parent_section = "";
+    my $before_child = "0";
+    my @before_section = split (/\./, $before_section);
+    if (scalar(@before_section) > 0) {
+	$before_child = pop (@before_section);
+	$parent_section = join (".", @before_section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::insert_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # get the next section number
+    my $section_num = $parent_section_ptr->{'next_subsection'}++;
+
+    my $i = 0;
+    while ($i < scalar(@{$parent_section_ptr->{'subsection_order'}}) &&
+	   $parent_section_ptr->{'subsection_order'}->[$i] ne $before_child) {
+	$i++;
+    }
+    
+    # insert the section number into the order list
+    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 0, $section_num);
+
+    # add this section to the parent section
+    my $section_ptr = {'subsection_order'=>[],
+		       'next_subsection'=>1,
+		       'subsections'=>{},
+		       'metadata'=>[],
+		       'text'=>""};
+    $parent_section_ptr->{'subsections'}->{$section_num} = $section_ptr;
+
+    # work out the full section number
+    my $section = $parent_section;
+    $section .= "." unless $section eq "";
+    $section .= $section_num;
+    
+    return $section;
+}
+
+# creates a pre-named section
+sub create_named_section {
+    my $self = shift (@_);
+    my ($mastersection) = @_;
+
+    my ($num);
+    my $section = $mastersection;
+    my $sectionref = $self;
+
+    while ($section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num) {
+	    if (!defined $sectionref->{'subsections'}->{$num}) {
+		push (@{$sectionref->{'subsection_order'}}, $num);
+		$sectionref->{'subsections'}->{$num} = {'subsection_order'=>[],
+							'next_subsection'=>1,
+							'subsections'=>{},
+							'metadata'=>[],
+							'text'=>""};
+		if ($num >= $sectionref->{'next_subsection'}) {
+		    $sectionref->{'next_subsection'} = $num + 1;
+		}
+	    }
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+
+	} else {
+	    print STDERR "doc::create_named_section couldn't create section ";
+	    print STDERR "$mastersection\n";
+	    last;
+	}
+    }
+}
+
+# returns a reference to a list of subsections
+sub list_subsections {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section ($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::list_subsections couldn't find section $section\n";
+	return [];
+    }
+
+    return [@{$section_ptr->{'subsection_order'}}];
+}
+
+sub delete_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+#    my $section_ptr = {'subsection_order'=>[],
+#		       'next_subsection'=>1,
+#		       'subsections'=>{},
+#		       'metadata'=>[],
+#		       'text'=>""};
+
+    # if this is the top section reset everything
+    if ($section eq "") {
+	$self->{'subsection_order'} = [];
+	$self->{'subsections'} = {};
+	$self->{'metadata'} = [];
+	$self->{'text'} = "";
+	return;
+    }
+
+    # find the parent of the section to delete
+    my $parent_section = "";
+    my $child = "0";
+    my @section = split (/\./, $section);
+    if (scalar(@section) > 0) {
+	$child = pop (@section);
+	$parent_section = join (".", @section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::delete_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # remove this section from the subsection_order list
+    my $i = 0;
+    while ($i < scalar (@{$parent_section_ptr->{'subsection_order'}})) {
+	if ($parent_section_ptr->{'subsection_order'}->[$i] eq $child) {
+	    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 1);
+	    last;
+	}
+	$i++;
+    }
+
+    # remove this section from the subsection hash
+    if (defined ($parent_section_ptr->{'subsections'}->{$child})) {
+	undef $parent_section_ptr->{'subsections'}->{$child};
+    }
+}
+
+#--
+# methods for dealing with metadata
+
+# set_metadata_element and get_metadata_element are for metadata
+# which should only have one value. add_meta_data and get_metadata
+# are for metadata which can have more than one value.
+
+# returns the first metadata value which matches field
+
+# This version of get metadata element works much like the one above,
+# except it allows for the namespace portion of a metadata element to
+# be ignored, thus if you are searching for dc.Title, the first piece
+# of matching metadata ending with the name Title (once any namespace
+# is removed) would be returned.
+# 28-11-2003 John Thompson
+sub get_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $ignore_namespace) = @_;
+    my ($data);
+
+    $ignore_namespace = 0 unless defined $ignore_namespace;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_metadata_element couldn't find section ", $section, "\n";
+	return;
+    }
+
+    # Remove the any namespace if we are being told to ignore them
+    if($ignore_namespace) {
+	$field =~ s/^\w*\.//;
+    }
+
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+
+	my $data_name = $data->[0];
+	# Remove the any namespace if we are being told to ignore them
+	if($ignore_namespace) {
+	    $data_name =~ s/^\w*\.//;
+	}
+	
+	return $data->[1] if (scalar(@$data) >= 2 && $data_name eq $field);
+    }
+	
+    return undef; # was not found
+}
+
+# returns a list of the form [value1, value2, ...]
+sub get_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $ignore_namespace) = @_;
+    my ($data);
+
+    $ignore_namespace = 0 unless defined $ignore_namespace;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+        print STDERR "doc::get_metadata couldn't find section ",
+	    $section, "\n";
+        return;
+    }
+
+    # Remove the any namespace if we are being told to ignore them
+    if($ignore_namespace) {
+	$field =~ s/^\w*\.//;
+    }
+
+    my @metadata = ();
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+
+	my $data_name = $data->[0];
+	# Remove the any namespace if we are being told to ignore them
+	if($ignore_namespace) {
+	    $data_name =~ s/^\w*\.//;
+	}
+
+        push (@metadata, $data->[1]) if ($data_name eq $field);
+    }
+
+    return \@metadata;
+}
+
+# returns a list of the form [[field,value],[field,value],...]
+sub get_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_all_metadata couldn't find section ", $section, "\n";
+	return;
+    }
+    
+    return $section_ptr->{'metadata'};
+}
+
+# $value is optional
+sub delete_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_metadata couldn't find section ", $section, "\n";
+	return;
+    }
+
+    my $i = 0;
+    while ($i < scalar (@{$section_ptr->{'metadata'}})) {
+	if (($section_ptr->{'metadata'}->[$i]->[0] eq $field) &&
+	    (!defined $value || $section_ptr->{'metadata'}->[$i]->[1] eq $value)) {
+	    splice (@{$section_ptr->{'metadata'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+sub delete_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_all_metadata couldn't find section ", $section, "\n";
+	return;
+    }
+    
+    $section_ptr->{'metadata'} = [];
+}
+
+sub set_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->set_utf8_metadata_element ($section, $field, 
+				      &unicode::ascii2utf8(\$value));
+}
+
+# set_utf8_metadata_element assumes the text has already been
+# converted to the UTF-8 encoding.
+sub set_utf8_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->delete_metadata ($section, $field);
+    $self->add_utf8_metadata ($section, $field, $value);
+}
+
+
+# add_metadata assumes the text is in (extended) ascii form. For
+# text which has already been converted to the UTF-8 format use
+# add_utf8_metadata.
+sub add_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->add_utf8_metadata ($section, $field,
+			      &unicode::ascii2utf8(\$value));
+}
+
+sub add_utf8_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_utf8_metadata couldn't find section ", $section, "\n";
+	return;
+    }
+    if (!defined $value) {
+	print STDERR "doc::add_utf8_metadata undefined value for $field\n";
+	return;
+    }
+    if (!defined $field) {
+	print STDERR "doc::add_utf8_metadata undefined metadata type \n";
+	return;
+    }
+    
+    #print STDERR "###$field=$value\n";
+    # double check that the value is utf-8
+    if (unicode::ensure_utf8(\$value)) {
+	print STDERR "doc::add_utf8_metadata: warning: '$field' wasn't utf8\n";
+    }
+
+    push (@{$section_ptr->{'metadata'}}, [$field, $value]);
+}
+
+
+# methods for dealing with text
+
+# returns the text for a section
+sub get_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text couldn't find section " .
+	    "$section\n";
+	return "";
+    }
+
+    return $section_ptr->{'text'};
+}
+
+# returns the (utf-8 encoded) length of the text for a section
+sub get_text_length {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text_length couldn't find section " .
+	    "$section\n";
+	return 0;
+    }
+
+    return length ($section_ptr->{'text'});
+}
+
+sub delete_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} = "";
+}
+
+# add_text assumes the text is in (extended) ascii form. For
+# text which has been already converted to the UTF-8 format
+# use add_utf8_text.
+sub add_text {
+    my $self = shift (@_);
+    my ($section, $text) = @_;
+
+    # convert the text to UTF-8 encoded unicode characters
+    # and add the text
+    $self->add_utf8_text($section, &unicode::ascii2utf8(\$text));
+}
+
+
+# add_utf8_text assumes the text to be added has already
+# been converted to the UTF-8 encoding. For ascii text use
+# add_text
+sub add_utf8_text {
+    my $self = shift (@_);
+    my ($section, $text) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_utf8_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} .= $text;
+}
+
+
+# methods for dealing with associated files
+
+# a file is associated with a document, NOT a section.
+# if section is defined it is noted in the data structure
+# only so that files associated from a particular section
+# may be removed later (using delete_section_assoc_files)
+sub associate_file {
+    my $self = shift (@_);
+    my ($real_filename, $assoc_filename, $mime_type, $section) = @_;
+    $mime_type = &ghtml::guess_mime_type ($real_filename) unless defined $mime_type;
+
+    # remove all associated files with the same name
+    $self->delete_assoc_file ($assoc_filename);
+    
+    push (@{$self->{'associated_files'}}, 
+	  [$real_filename, $assoc_filename, $mime_type, $section]);
+}
+
+# returns a list of associated files in the form
+#   [[real_filename, assoc_filename, mimetype], ...]
+sub get_assoc_files {
+    my $self = shift (@_);
+
+    return $self->{'associated_files'};
+}
+
+sub delete_section_assoc_files {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $i=0;
+    while ($i < scalar (@{$self->{'associated_files'}})) {
+	if (defined $self->{'associated_files'}->[$i]->[3] &&
+	    $self->{'associated_files'}->[$i]->[3] eq $section) {
+	    splice (@{$self->{'associated_files'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+sub delete_assoc_file {
+    my $self = shift (@_);
+    my ($assoc_filename) = @_;
+
+    my $i=0;
+    while ($i < scalar (@{$self->{'associated_files'}})) {
+	if ($self->{'associated_files'}->[$i]->[1] eq $assoc_filename) {
+	    splice (@{$self->{'associated_files'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+sub reset_nextsection_ptr {
+    my $self = shift (@_);
+    my ($section) = @_;
+    
+    my $section_ptr = $self->_lookup_section($section);
+    $section_ptr->{'next_subsection'} = 1;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/docprint.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/docprint.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/docprint.pm	(revision 21112)
@@ -0,0 +1,67 @@
+###########################################################################
+#
+# docprint.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor prints a document out to STDOUT
+
+
+package docprint;
+
+use arcinfo;
+use docproc;
+use util;
+
+
+sub BEGIN {
+    @docprint::ISA = ('docproc');
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new docproc ();
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    # add associated files as metadata to the document
+    my @assoc_files = ();
+    foreach my $assoc_file (@{$doc_obj->get_assoc_files()}) {
+	if (-e $assoc_file->[0]) {
+	    $doc_obj->add_metadata ($doc_obj->get_top_section(), 
+				    "gsdlassocfile", 
+				    "$assoc_file->[1]:$assoc_file->[2]");
+	}
+    }
+    
+    # output document
+    $doc_obj->output_section(STDOUT, $doc_obj->get_top_section(),
+			     undef, undef, 0);
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/docproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/docproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/docproc.pm	(revision 21112)
@@ -0,0 +1,84 @@
+###########################################################################
+#
+# docproc.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# document processors are used by the document reader plugins
+# to do some processing on some documents
+
+package docproc;
+
+sub new {
+    my ($class) = @_;
+    my $self = {};
+
+    $self->{'OIDtype'} = "hash";
+    $self->{'saveas'} = "GA";  # default
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    die "docproc::process function must be implemented in sub classes\n";
+}
+
+# OIDtype may be "hash" or "incremental" or "dirname" or "assigned"
+sub set_OIDtype {
+    my $self = shift (@_);
+    my ($type) = @_;
+
+    if ($type =~ /^(hash|incremental|dirname|assigned)$/) {
+	$self->{'OIDtype'} = $type;
+    } else {
+	$self->{'OIDtype'} = "hash";
+    }
+}
+
+sub set_saveas {
+    my $self = shift (@_);
+    my ($saveas) = @_;
+
+    $self->{'saveas'} = $saveas;
+
+}
+
+sub set_saveas_version {
+    my $self = shift (@_);
+    my ($saveas_version) = @_;
+
+    $self->{'saveas_version'} = $saveas_version;
+
+}
+
+
+1;
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/docsave.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/docsave.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/docsave.pm	(revision 21112)
@@ -0,0 +1,671 @@
+###########################################################################
+#
+# docsave.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor saves a document in the 
+# archives directory of a collection (as xml)
+
+use strict;
+no strict 'refs';
+
+package docsave;
+
+eval {require bytes};
+
+use arcinfo;
+use docproc;
+use util;
+
+
+sub BEGIN {
+    @docsave::ISA = ('docproc');
+}
+
+
+sub calcoutputdir 
+{
+    my ($service,$collectdir,$outhandle) = @_;
+
+    my $output_dir = undef;
+
+    if (($service eq "import") || ($service eq "unbuild")) {
+	$output_dir = &util::filename_cat ($collectdir, "archives");
+    }
+    elsif ($service eq "export") {
+	$output_dir = &util::filename_cat($collectdir, "export");
+    }
+    else {
+	print $outhandle "docsave::calcoutputdir did not recognise service ";
+	print $outhandle " '$service'. No output directory set.\n";
+    }
+
+    return $output_dir;
+}
+
+
+sub new {
+    my ($class, $collection, $info, $verbosity, 
+	$gzip, $groupsize, $outhandle, $service, $saveas) = @_;
+    my $self = new docproc ();
+    
+    my $collectdir = $ENV{'GSDLCOLLECTDIR'};
+
+    $outhandle = 'STDERR' unless (defined $outhandle);
+    $service   = "import" unless (defined $service);
+    $saveas    = "GA" unless (defined $saveas);
+    $groupsize = 1 unless (defined $groupsize);
+
+    $self->{'collection'} = $collection;
+
+    $self->{'output_info'} = $info;
+    $self->{'output_dir'} = undef;
+
+    $self->{'verbosity'} = $verbosity;
+    $self->{'gzip'} = $gzip;
+    $self->{'keepimportstructure'} = 0;
+    $self->{'groupsize'} = $groupsize;
+    $self->{'gs_count'}  = 0;
+
+    $self->{'outhandle'} = $outhandle;
+    $self->{'service'}   = $service;
+    $self->{'saveas'}    = $saveas;
+
+    $self->{'sortmeta'} = undef;
+    
+    return bless $self, $class;
+}
+
+
+
+sub setoutputdir 
+{
+    my $self = shift @_;
+    my ($output_dir) = @_;
+
+    &util::mk_all_dir ($output_dir) unless -e $output_dir;
+
+    $self->{'output_dir'} = $output_dir;
+}
+
+sub getoutputinfo 
+{
+    my $self = shift (@_);
+
+    return $self->{'output_info'};
+}
+
+sub getoutputdir 
+{
+    my $self = shift (@_);
+
+    return $self->{'output_dir'};
+}
+
+
+sub set_sortmeta {
+    my $self = shift (@_);
+    my ($sortmeta, $removeprefix, $removesuffix) = @_;
+    
+    $self->{'sortmeta'} = $sortmeta;
+    if (defined ($removeprefix) && $removeprefix ) {
+	$removeprefix =~ s/^\^//; # don't need a leading ^
+	$self->{'removeprefix'} = $removeprefix;
+    }
+    if (defined ($removesuffix) && $removesuffix) {
+	$removesuffix =~ s/\$$//; # don't need a trailing $
+	$self->{'removesuffix'} = $removesuffix;
+    }
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+ 
+    my $outhandle = $self->{'outhandle'};
+    my $service = $self->{'service'};
+
+    # Define the SaveAs Type
+    my $save_as = $self->{'saveas'};
+    my $collection = $self->{'collection'};
+
+    # set the lastmodified time before we write it out.
+    $doc_obj->set_lastmodified();
+
+    if ($self->{'groupsize'} > 1) {
+	$self->group_process ($doc_obj);
+	return;
+    }
+
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+
+    my $top_section = $doc_obj->get_top_section();
+
+    # get document's directory
+    my $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
+
+    # groupsize is 1 (i.e. one document per XML file) so sortmeta 
+    # may be used
+    
+    my $output_info = $self->getoutputinfo();
+    return if (!defined $output_info);
+
+    my $output_dir = $self->getoutputdir();
+    my $working_dir = &util::filename_cat ($output_dir, $doc_dir);
+
+    # copy all the associated files, add this information as metadata
+    # to the document
+    if ($save_as eq "DSpace") {
+
+	# Genereate handle file 
+	# (Note: this section of code would benefit from being restructured)
+	my $doc_handle_file = &util::filename_cat ($working_dir, "handle");
+
+	my $env_hp = $ENV{'DSPACE_HANDLE_PREFIX'};
+	my $handle_prefix = (defined $env_hp) ? $env_hp : "123456789";
+
+	if (!open(OUTDOC_HANDLE,">$doc_handle_file")){
+	    print $outhandle "docsave::process could not write collection handle to file $doc_handle_file\n";
+	    return;
+	} 
+
+	my ($handle) = ($doc_dir =~ m/^(.*)\.dir$/);
+	print OUTDOC_HANDLE "$handle_prefix/$handle\n";
+
+	close OUTDOC_HANDLE;
+
+	# Generate contents file
+	my $doc_contents_file = &util::filename_cat ($working_dir, "contents");
+	
+	if (!open(OUTDOC_CONTENTS,">$doc_contents_file")){
+	    print $outhandle "docsave::process could not write collection contents to file $doc_contents_file\n";
+	    return;
+	} 
+	$self->process_assoc_files ($doc_obj, $doc_dir, 'docsave::OUTDOC_CONTENTS');
+
+	close OUTDOC_CONTENTS;
+
+    } else {
+	$self->process_assoc_files ($doc_obj, $doc_dir, '');
+    }
+        
+    # Save the document in the requested 'save_as' format
+
+    if ($save_as eq "GA") {
+
+	my $doc_file = &util::filename_cat ($working_dir, "doc.xml");
+
+	if (!open (OUTDOC, ">$doc_file")) {
+	    print $outhandle "docsave::process could not write to file $doc_file\n";
+	    return;
+	}
+	
+	# save this document
+	$self->output_xml_header('docsave::OUTDOC');
+	$doc_obj->output_section('docsave::OUTDOC',$top_section);
+	$self->output_xml_footer('docsave::OUTDOC');
+	
+	close OUTDOC;
+    } 
+    elsif ($save_as eq "METS") {
+	
+	my $doc_txt_file = &util::filename_cat ($working_dir,"doctxt.xml");	
+
+	if (!open(OUTDOC_TXT, ">$doc_txt_file")){
+	    print $outhandle "docsave::process could not write to file $doc_txt_file\n";
+	    return;
+	}
+	
+	$self->output_txt_xml_header('docsave::OUTDOC_TXT');
+	$doc_obj->output_txt_section('docsave::OUTDOC_TXT', $top_section);
+	$self->output_txt_xml_footer('docsave::OUTDOC_TXT');
+	
+	close OUTDOC_TXT;
+	
+	# Now save the document with metadata and text structure to docmets.xml
+	
+	my $doc_mets_file = &util::filename_cat ($working_dir, "docmets.xml");
+
+	my $doc_title = $doc_obj->get_metadata_element($top_section,"dc.Title");
+	if (!defined $doc_title) {
+	    $doc_title = $doc_obj->get_metadata_element($top_section,"Title");
+	}
+
+	if (!open(OUTDOC_METS,">$doc_mets_file")){
+	    print $outhandle "docsave::process could not write to file $doc_mets_file\n";
+	    return;
+	}
+	
+	my $saveas_version = $self->{'saveas_version'};
+	$self->output_mets_xml_header('docsave::OUTDOC_METS', $OID, $doc_title);
+	$doc_obj->output_mets_section('docsave::OUTDOC_METS',$top_section,$saveas_version,$working_dir);
+	$self->output_mets_xml_footer('docsave::OUTDOC_METS');
+	
+	close OUTDOC_METS;	
+    } 
+    elsif ($save_as eq "DSpace") {
+
+	# Generate dublin_core.xml file
+	my $doc_dc_file = &util::filename_cat ($working_dir, "dublin_core.xml");
+	
+	if (!open(OUTDOC_DC,">$doc_dc_file")){
+	    print $outhandle "docsave::process could not write dublin core to file $doc_dc_file\n";
+	    return;
+	}   
+	
+	my $saveas_version = $self->{'saveas_version'};
+
+	$self->output_dc_xml_header('docsave::OUTDOC_DC', $OID);
+	$doc_obj->output_dc_section('docsave::OUTDOC_DC',$top_section);
+	$self->output_dc_xml_footer('docsave::OUTDOC_DC');
+	
+	close OUTDOC_DC;
+    } else { # save_as isn't one of the recognised types
+	print $outhandle "docsave::process unrecognised saveas type, $save_as\n";
+	return;
+    }
+
+
+    my $short_doc_file;
+
+    if ($save_as eq "GA") {
+	$short_doc_file = util::filename_cat ($doc_dir, "doc.xml");
+    } elsif ($save_as eq "METS") {
+	$short_doc_file = &util::filename_cat ($doc_dir, "docmets.xml");
+    } elsif ($save_as eq "DSpace") {
+	$short_doc_file=&util::filename_cat ($doc_dir, "dublin_core.xml");
+    } else {
+	return;
+    }
+
+    #save for later (for close_file_output())
+    $self->{'short_doc_file'} = $short_doc_file;	
+ 
+    if ($self->{'gzip'}) {
+	my $doc_file = $self->{'gs_filename'};
+	`gzip $doc_file`;
+	$doc_file .= ".gz";
+	$short_doc_file .= ".gz";
+	if (!-e $doc_file) {
+	    print $outhandle "error while gzipping: $doc_file doesn't exist\n";
+	    return 0;
+	}
+    }
+
+    # do the sortmeta thing
+    my ($metadata);	
+    if (defined ($self->{'sortmeta'})) {
+	$metadata = $doc_obj->get_metadata_element($top_section,$self->{'sortmeta'});
+    }
+    if (defined ($metadata) && $metadata) {
+	# do remove prefix/suffix
+	if (defined($self->{'removeprefix'})) {
+	    $metadata =~ s/^$self->{'removeprefix'}//;
+	}
+	if (defined($self->{'removesuffix'})) {
+	    $metadata =~ s/$self->{'removesuffix'}$//;
+	}
+	$metadata = &sorttools::format_metadata_for_sorting($self->{'sortmeta'}, $metadata, $doc_obj);
+    }
+
+    # store reference in the output_info
+    $output_info->add_info($OID, $short_doc_file, undef, $metadata);
+}
+
+      
+sub group_process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    
+    my $outhandle = $self->{'outhandle'};
+
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+
+    my $groupsize = $self->{'groupsize'};
+    my $gs_count = $self->{'gs_count'};
+    my $open_new_file = (($gs_count % $groupsize)==0);
+
+    # opening a new file, or document has assoicated files => directory needed
+    if (($open_new_file) || (scalar(@{$doc_obj->get_assoc_files()})>0)) {
+
+	# get document's directory
+	my $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
+
+	# copy all the associated files, add this information as metadata
+	# to the document
+	$self->process_assoc_files ($doc_obj, $doc_dir);
+
+
+	if ($open_new_file) {
+	    # only if opening new file
+	    my $output_dir = $self->getoutputdir();
+	    my $doc_file 
+		= &util::filename_cat ($output_dir, $doc_dir, "doc.xml");
+	    my $short_doc_file = &util::filename_cat ($doc_dir, "doc.xml");
+	    
+	    if ($gs_count>0)
+	    {
+		return if (!$self->close_file_output());
+	    }
+	    
+	    if (!open (OUTDOC, ">$doc_file")) {
+		print $outhandle "docsave::group_process could not write to file $doc_file\n";
+		return;
+	    }
+	    $self->{'gs_filename'} = $doc_file;
+	    $self->{'gs_short_filename'} = $short_doc_file;
+	    $self->{'gs_OID'} = $OID;
+
+	    $self->output_xml_header('docsave::OUTDOC');
+	}
+    }
+
+    # save this document
+    $doc_obj->output_section('docsave::OUTDOC', $doc_obj->get_top_section());
+
+    $self->{'gs_count'}++;
+}
+
+sub get_doc_dir {
+    my $self = shift (@_);
+    my ($OID, $source_filename) = @_;
+
+    my $service = $self-> {'service'};
+
+    my $working_dir  = $self->getoutputdir();
+    my $working_info = $self->getoutputinfo();
+    return if (!defined $working_info);
+
+    my $doc_info = $working_info->get_info($OID);
+    my $doc_dir = '';
+
+    if (defined $doc_info && scalar(@$doc_info) >= 1) {
+	# this OID already has an assigned directory, use the
+	# same one.
+	$doc_dir = $doc_info->[0];
+	$doc_dir =~ s/\/?((doc(mets)?)|(dublin_core))\.xml(\.gz)?$//;
+    } elsif ($self->{'keepimportstructure'}) {
+	$source_filename = &File::Basename::dirname($source_filename);
+	$source_filename =~ s/[\\\/]+/\//g;
+	$source_filename =~ s/\/$//;
+ 
+	
+	#print STDERR "Source filename: $source_filename; \nImport dir:",$ENV{'GSDLIMPORTDIR'}, "\n";
+	$doc_dir = substr($source_filename, length($ENV{'GSDLIMPORTDIR'}) + 1);
+
+    }
+    if ($doc_dir eq "") {
+	# have to get a new document directory
+
+	if (($service eq "import") || ($service eq "unbuild")) {
+	    my $doc_dir_rest = $OID;
+	    my $doc_dir_num = 0;
+
+	    do {
+		$doc_dir .= "/" if $doc_dir_num > 0;
+		if ($doc_dir_rest =~ s/^(.{1,8})//) {
+		    $doc_dir .= $1;
+		    $doc_dir_num++;
+		}
+	    } while ($doc_dir_rest ne "" && 
+		     ((-d &util::filename_cat ($working_dir, "$doc_dir.dir")) || 
+		      ($working_info->size() >= 1024 && $doc_dir_num < 2)));
+	}
+	else {
+	    # Export formats such as DSpace need the directory structure to
+	    # be flat.  This is simple to arrange (set 'doc_dir' to be the
+	    # documents OID) but breaks Windows 3.1 file system compliance.
+	    # Such a loss is not a big thing in this situation as such
+	    # systems don't run on Windows 3.1 anyway.
+
+	    $doc_dir = $OID;
+	}
+
+
+	$doc_dir .= ".dir";
+	&util::mk_all_dir (&util::filename_cat ($working_dir, $doc_dir));
+    }
+    return $doc_dir;
+}
+
+sub process_assoc_files {
+    my $self = shift (@_);
+    my ($doc_obj, $doc_dir, $handle) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $service = $self->{'service'};
+    my $save_as = $self->{'saveas'};
+
+    my $output_dir = $self->getoutputdir();
+    return if (!defined $output_dir);
+
+    my $working_dir = &util::filename_cat($output_dir, $doc_dir);
+
+    my @assoc_files = ();
+    my $filename;;
+
+    my $source_filename = $doc_obj->get_source_filename();
+
+    my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
+
+    if (defined $collect_dir) {
+	my $dirsep_regexp = &util::get_os_dirsep();
+
+	if ($collect_dir !~ /$dirsep_regexp$/) {
+	    $collect_dir .= &util::get_dirsep(); # ensure there is a slash at the end
+	}
+
+	# This test is never going to fail on Windows -- is this a problem?
+	if ($source_filename !~ /^$dirsep_regexp/) {
+	    $source_filename = &util::filename_cat($collect_dir, $source_filename);
+	}
+    }
+
+
+    if ($save_as eq "DSpace") {
+	my ($tail_filename) = ($source_filename =~ m/\/([^\/\\]*)$/);
+
+	print $handle "$tail_filename\n";
+
+	$filename = &util::filename_cat($working_dir, $tail_filename);
+	&util::hard_link ($source_filename, $filename);
+    }
+         
+    # set the assocfile path (even if we have no assoc files - need this for lucene)
+    $doc_obj->set_utf8_metadata_element ($doc_obj->get_top_section(),
+					 "assocfilepath",
+					 "$doc_dir");
+    foreach my $assoc_file_rec (@{$doc_obj->get_assoc_files()}) {
+	my ($dir, $afile) = $assoc_file_rec->[1] =~ /^(.*?)([^\/\\]+)$/;
+	$dir = "" unless defined $dir;
+	    
+	
+	my $real_filename = $assoc_file_rec->[0];
+	# for some reasons the image associate file has / before the full path
+	$real_filename =~ s/^\\(.*)/$1/i;
+	if (-e $real_filename) {
+
+
+	    if ($save_as eq "DSpace") {
+		if ($real_filename =~ m/$source_filename$/) {
+		    next;
+		}
+		else {
+		    my $bundle = "bundle:ORIGINAL";
+
+		    if ($afile =~ m/^thumbnail\./) {
+			$bundle = "bundle:THUMBNAIL";
+		    }
+
+		    # Store the associated file to the "contents" file
+		    print $handle "$assoc_file_rec->[1]\t$bundle\n";
+		}
+	    }
+
+	    $filename = &util::filename_cat($working_dir, $afile);
+
+
+	    &util::hard_link ($real_filename, $filename);
+	    
+	    $doc_obj->add_utf8_metadata ($doc_obj->get_top_section(),
+					 "gsdlassocfile",
+					 "$afile:$assoc_file_rec->[2]:$dir");
+	} elsif ($self->{'verbosity'} > 2) {
+	    print $outhandle "docsave::process couldn't copy the associated file " .
+		"$real_filename to $afile\n";
+	}
+    }
+}
+    
+
+sub close_file_output
+{
+    my ($self) = @_;
+    my $service =$self->{'service'};
+
+    # make sure that the handle has been opened - it won't be if we failed
+    # to import any documents...
+    if (defined(fileno(docsave::OUTDOC))) {
+	$self->output_xml_footer('docsave::OUTDOC');    
+	close OUTDOC;
+    }
+
+    my $OID = $self->{'gs_OID'};
+    my $short_doc_file;
+    # can we use 'short_doc_file' for GA too?
+    if (exists($self->{'saveas'}) && $self->{'saveas'} eq "METS") {
+	$short_doc_file=$self->{'short_doc_file'};
+    } elsif ($self->{'saveas'} eq "GA") { # "GA"
+	$short_doc_file=$self->{'gs_short_filename'};
+    } else { # "DSpace"
+    }
+    
+    if ($self->{'gzip'}) {
+	my $doc_file = $self->{'gs_filename'};
+	`gzip $doc_file`;
+	$doc_file .= ".gz";
+	$short_doc_file .= ".gz";
+	if (!-e $doc_file) {
+	    my $outhandle = $self->{'outhandle'};
+	    print $outhandle "error while gzipping: $doc_file doesn't exist\n";
+	    return 0;
+	}
+    }
+
+    # store reference in output_info
+    my $output_info = $self->getoutputinfo();
+    return 0 if (!defined $output_info);
+    $output_info->add_info($OID, $short_doc_file, undef, undef);
+
+    return 1;
+}
+
+sub output_xml_header {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    print $handle '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' . "\n";
+   
+    print $handle '<!DOCTYPE Archive SYSTEM "http://greenstone.org/dtd/Archive/1.0/Archive.dtd">' . "\n";
+    print $handle "<Archive>\n";
+}
+
+sub output_xml_footer {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    print $handle "</Archive>\n";
+}
+
+sub output_txt_xml_header{
+    my $self = shift (@_);
+    my ($handle) = @_;
+    print $handle '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' . "\n";
+    print $handle '<!DOCTYPE Archive SYSTEM "http://greenstone.org/dtd/Archive/1.0/Archive.dtd">' . "\n";
+}
+
+sub output_txt_xml_footer{
+    my $self = shift(@_);
+    my ($handle) = @_;
+    # Nothing needs to be output at present
+}
+
+sub output_mets_xml_header(){
+    my $self = shift(@_);
+    my ($handle, $OID, $doc_title) = @_;
+
+    my $version = $self->{'saveas_version'};
+
+    my $extra_attr = "";
+    if ($version eq "fedora") {	
+	my $fnamespace = $ENV{'FEDORA_PID_NAMESPACE'};
+	my $oid_namespace = (defined $fnamespace) ? $fnamespace : "test";
+
+	$extra_attr = "OBJID=\"$oid_namespace:$OID\" TYPE=\"FedoraObject\" LABEL=\"$doc_title\"";
+    }
+    else {
+	# Greenstone METS profile
+	$extra_attr = "OBJID=\"$OID:2\"";
+    }
+
+
+    print $handle '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' . "\n";
+    print $handle '<mets:mets xmlns:mets="http://www.loc.gov/METS/"' . "\n";
+    print $handle '           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' . "\n";
+    print $handle '           xmlns:gsdl3="http://www.greenstone.org/namespace/gsdlmetadata/1.0/"' . "\n";
+    print $handle '           xmlns:xlink="http://www.w3.org/TR/xlink"' ."\n";
+    print $handle '           xsi:schemaLocation="http://www.loc.gov/METS/' . "\n";
+    print $handle '           http://www.loc.gov/standards/mets/mets.xsd' . "\n";
+    print $handle '           http://www.greenstone.org/namespace/gsdlmetadata/1.0/' . "\n";
+    print $handle '           http://www.greenstone.org/namespace/gsdlmetadata/1.0/gsdl_metadata.xsd"' . "\n";
+    print $handle "  $extra_attr>\n";
+
+    if ($version eq "fedora") {
+	print $handle '<mets:metsHdr RECORDSTATUS="A"/>'. "\n"; # A = active
+    }
+
+}
+
+sub output_mets_xml_footer() {
+    my $self = shift(@_);
+    my ($handle) = @_;
+    print $handle '</mets:mets>' . "\n";
+}
+
+sub output_dc_xml_header(){
+    my $self = shift(@_);
+    my ($handle, $OID) = @_;
+
+    print $handle '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' . "\n";
+#    print $handle '<!DOCTYPE Archive SYSTEM "http://greenstone.org/dtd/Archive/1.0/Archive.dtd">'."\n";
+    print $handle '<dublin_core>' . "\n";
+}
+
+sub output_dc_xml_footer() {
+    my $self = shift(@_);
+    my ($handle) = @_;
+    print $handle '</dublin_core>' . "\n";
+}
+1;
Index: /main/tags/3.01/gsdl/perllib/encodings.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/encodings.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/encodings.pm	(revision 21112)
@@ -0,0 +1,113 @@
+###########################################################################
+#
+# encodings.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Each encoding supported by the Greenstone build-time software should be
+# specified in the following hash table ($encodings).
+
+package encodings;
+
+# $encodings takes the form:
+# --> identifier --> name      --> The full display name of the encoding.
+#                --> mapfile   --> The ump file associated with the encoding
+#                --> double    --> 1 if it's a double byte encoding
+#                --> converter --> If the encoding needs a specialized conversion
+#                                  routine this is the name of that routine.
+
+$encodings::encodings = {
+    'iso_8859_1' => {'name' => 'Latin1 (western languages)', 'mapfile' => '8859_1.ump'},
+
+    'iso_8859_2' => {'name' => 'Latin2 (central and eastern european languages)',
+		     'mapfile' => '8859_2.ump'},
+
+    'iso_8859_3' => {'name' => 'Latin3', 'mapfile' => '8859_3.ump'},
+
+    'iso_8859_4' => {'name' => 'Latin4', 'mapfile' => '8859_4.ump'},
+
+    'iso_8859_5' => {'name' => 'Cyrillic', 'mapfile' => '8859_5.ump'},
+
+    'iso_8859_6' => {'name' => 'Arabic', 'mapfile' => '8859_6.ump'},
+
+    'iso_8859_7' => {'name' => 'Greek', 'mapfile' => '8859_7.ump'},
+
+    'iso_8859_8' => {'name' => 'Hebrew', 'mapfile' => '8859_8.ump'},
+
+    'iso_8859_9' => {'name' => 'Turkish', 'mapfile' => '8859_9.ump'},
+
+    'iso_8859_15' => {'name' => 'Latin15 (revised western)', 'mapfile' => '8859_15.ump'},
+
+    'windows_1250' => {'name' => 'Windows codepage 1250 (WinLatin2)',
+		       'mapfile' => 'win1250.ump'},
+
+    'windows_1251' => {'name' => 'Windows codepage 1251 (WinCyrillic)',
+		       'mapfile' => 'win1251.ump'},
+
+    'windows_1252' => {'name' => 'Windows codepage 1252 (WinLatin1)',
+		       'mapfile' => 'win1252.ump'},
+
+    'windows_1253' => {'name' => 'Windows codepage 1253 (WinGreek)', 
+		       'mapfile' => 'win1253.ump'},
+
+    'windows_1254' => {'name' => 'Windows codepage 1254 (WinTurkish)',
+		       'mapfile' => 'win1254.ump'},
+
+    'windows_1255' => {'name' => 'Windows codepage 1255 (WinHebrew)', 
+		       'mapfile' => 'win1255.ump'},
+
+    'windows_1256' => {'name' => 'Windows codepage 1256 (WinArabic)', 
+		       'mapfile' => 'win1256.ump'},
+
+    'windows_1257' => {'name' => 'Windows codepage 1257 (WinBaltic)',
+		       'mapfile' => 'win1257.ump'},
+
+    'windows_1258' => {'name' => 'Windows codepage 1258 (Vietnamese)',
+		       'mapfile' => 'win1258.ump'},
+
+    'windows_874' => {'name' => 'Windows codepage 874 (Thai)', 'mapfile' => 'win874.ump'},
+
+    'dos_437' => {'name' => 'DOS codepage 437 (US English)', 'mapfile' => 'dos437.ump'},
+
+    'dos_850' => {'name' => 'DOS codepage 850 (Latin 1)', 'mapfile' => 'dos850.ump'},
+
+    'dos_866' => {'name' => 'DOS codepage 866 (Cyrillic)', 'mapfile' => 'dos866.ump'},
+
+    'koi8_r' => {'name' => 'Cyrillic', 'mapfile' => 'koi8_r.ump'},
+
+    'koi8_u' => {'name' => 'Cyrillic (Ukrainian)', 'mapfile' => 'koi8_u.ump'},
+
+    'iscii_de' => {'name' => 'ISCII Devanagari', 'mapfile' => 'iscii_de.ump'},
+
+    'shift_jis' => {'name' => 'Japanese (Shift-JIS)', 'mapfile' => 'shiftjis.ump',
+		    'converter' => 'shiftjis2unicode'},
+
+    'euc_jp' => {'name' => 'Japanese (EUC)', 'mapfile' => 'euc_jp.ump'},
+
+    'korean' => {'name' => 'Korean (Unified Hangul Code - i.e. a superset of EUC-KR)',
+		 'mapfile' => 'uhc.ump'},
+
+    'gb' => {'name' => 'Chinese Simplified (GB)', 'mapfile' => 'gbk.ump'},
+
+    'big5' => {'name' => 'Chinese Traditional (Big5)', 'mapfile' => 'big5.ump'}
+
+};
Index: /main/tags/3.01/gsdl/perllib/expinfo.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/expinfo.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/expinfo.pm	(revision 21112)
@@ -0,0 +1,163 @@
+###########################################################################
+#
+# expinfo.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# modified by: Chi-Yu Huang
+# This module stores information about the export directory. At the moment
+# this information just consists of the file name (relative to the
+# directory the export information file is in) and its OID.
+
+# This module assumes there is a one to one correspondance between
+# a file in the export directory and an OID.
+
+
+package expinfo;
+
+sub new {
+    my ($class) = @_;
+    my $self = {'info'=>{},
+		'order'=>[]};
+
+    return bless $self, $class;
+}
+
+sub load_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    $self->{'info'} = {};
+ 
+    if (-e $filename) {
+	open (INFILE, $filename) || 
+	    die "expinfo::load_info couldn't read $filename\n";
+
+	my ($line, @line);
+	while (defined ($line = <INFILE>)) {
+	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	    @line = split ("\t", $line); # filename, 
+	    if (scalar(@line) >= 2) {
+		$self->add_info (@line);
+	    }
+	}
+	close (INFILE);
+    }
+}
+
+sub save_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my ($OID, $info);
+
+    open (OUTFILE, ">$filename") || 
+	die "expinfo::save_info couldn't write $filename\n";
+
+    foreach $info (@{$self->get_OID_list()}) {
+	if (defined $info) {
+	    print OUTFILE join("\t", @$info), "\n";
+	}
+    }
+    close (OUTFILE);
+}
+
+sub delete_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	delete $self->{'info'}->{$OID};
+	
+	my $i = 0;
+	while ($i < scalar (@{$self->{'order'}})) {
+	    if ($self->{'order'}->[$i]->[0] eq $OID) {
+		splice (@{$self->{'order'}}, $i, 1);
+		last;
+	    }
+	    $i ++;
+	}
+    }
+}
+
+sub add_info {
+    my $self = shift (@_);
+    my ($OID, $doc_file, $sortmeta) = @_;
+    $sortmeta = "" unless defined $sortmeta;
+
+    if (! defined($OID)) {
+	# only happens when no files can be processed?
+	return undef;
+    }
+    $self->delete_info ($OID);
+    $self->{'info'}->{$OID} = [$doc_file];
+    push (@{$self->{'order'}}, [$OID, $sortmeta]);
+}
+
+# returns a list of the form [[OID, doc_file], ...]
+sub get_OID_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (sort {$a->[1] cmp $b->[1]} @{$self->{'order'}}) {
+	push (@list, [$OID->[0], $self->{'info'}->{$OID->[0]}->[0]]);
+    }
+    return \@list;
+}
+
+# returns a list of the form [[doc_file, OID], ...]
+sub get_file_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (sort {$a->[1] cmp $b->[1]} @{$self->{'order'}}) {
+	push (@list, [$self->{'info'}->{$OID->[0]}->[0], $OID->[0]]);
+    }
+    return \@list;
+}
+
+
+# returns a list of the form [doc_file]
+sub get_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	return $self->{'info'}->{$OID};
+    }
+
+    return undef;
+}
+
+
+# returns the number of documents so far
+sub size {
+    my $self = shift (@_);
+    return (scalar(@{$self->{'order'}}));
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/gflock.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/gflock.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/gflock.pm	(revision 21112)
@@ -0,0 +1,54 @@
+###########################################################################
+#
+# gflock.pm - an attempt to use flock only on those platforms
+# on which it's supported (i.e. everything but Windows 95/98)
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package gflock;
+
+use Fcntl ':flock';
+
+# returns true if successful (or if OS is a form of windows other than NT)
+sub lock {
+    my ($handle) = @_;
+
+    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
+	return 1;
+    } else {
+	return flock($handle, LOCK_EX);
+    }
+}
+
+# returns true if successful (or if OS is a form of windows other than NT)
+sub unlock {
+    my ($handle) = @_;
+
+    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
+	return 1;
+    } else {
+	return flock($handle, LOCK_UN);
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/ghtml.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/ghtml.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/ghtml.pm	(revision 21112)
@@ -0,0 +1,305 @@
+###########################################################################
+#
+# ghtml.pm -- this used to be called html.pm but it clashed
+# with the existing html module under windows
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package ghtml;
+
+use unicode;
+
+# htmlsafe(TEXT)
+# 
+# Converts SGML meta characters in TEXT to entity references.
+# 
+sub htmlsafe
+{
+    $_[0] =~ s/&/&amp;/osg;
+    $_[0] =~ s/</&lt;/osg;
+    $_[0] =~ s/>/&gt;/osg;
+    $_[0] =~ s/\"/&quot;/osg;
+}
+
+# unescape_html(TEXT)
+# 
+# Converts HTML entities into their original form.
+# 
+sub unescape_html
+{
+    my ($html) = @_;
+
+    $html =~ s/&amp;/&/osg;
+    $html =~ s/&lt;/</osg;
+    $html =~ s/&gt;/>/osg;
+    $html =~ s/&quot;/\"/osg;
+
+    return $html;
+}
+
+# urlsafe(TEXT)
+# 
+# Converts characters not allowed in a URL to their hex representation.
+# 
+sub urlsafe
+{
+    $_[0] =~ s/[\x09\x20\x22\x3c\x3e\x5b\x5c\x5d\x5e\x60\x7b\x7c\x7d\x7e\?\=\&\+_\/]/sprintf("%%%2x", ord($&))/gse;
+}
+
+# dmsafe
+#
+# Converts characters which could make display.pl or html crack to their entity references.
+# Don't use this on dm macros as they will no longer be recognisable by dm (which
+# by the way, is the whole idea).
+# -- Rodger 12/1/98
+sub dmsafe {
+    my ($s) = $_[0];
+    $s =~ s/&/&amp;/osg; # for html
+    $s =~ s/</&lt;/osg;  # for html
+    $s =~ s/>/&gt;/osg;  # for html
+    $s =~ s/\_/&#095;/osg; # for dm (we have a convention of starting macros with _
+    $s =~ s/\"/&quot;/osg; # for html (don't want to be interpreted as a quote)
+    $s =~ s/\{/&#123;/osg; # for dm blocks
+    $s =~ s/\}/&#125;/osg; # for dm blocks
+    $s =~ s/\\/&#092;/osg; # for dm (dm removes naturally occurring backquotes)
+    return $s;
+}
+
+
+# named entry to the standard html font
+my %charnetosf = ("Agrave"=> "192",  "Aacute"=> "193",  "Acirc" => "194",  "Atilde"=> "195",
+	       "Auml"  => "196",  "Aring" => "197",  "AElig" => "198",  "Ccedil"=> "199",
+	       "Egrave"=> "200",  "Eacute"=> "201",  "Ecirc" => "202",  "Euml"  => "203",
+	       "Igrave"=> "204",  "Iacute"=> "205",  "Icirc" => "206",  "Iuml"  => "207",
+	       "ETH"   => "208",  "Ntilde"=> "209",  "Ograve"=> "210",  "Oacute"=> "211",
+	       "Ocirc" => "212",  "Otilde"=> "213",  "Ouml"  => "214",  
+	       "Oslash"=> "216",  "Ugrave"=> "217",  "Uacute"=> "218",  "Ucirc" => "219",
+	       "Uuml"  => "220",  "Yacute"=> "221",  "THORN" => "222",  "szlig" => "223",
+	       "agrave"=> "224",  "aacute"=> "225",  "acirc" => "226",  "atilde"=> "227",
+	       "auml"  => "228",  "aring" => "229",  "aelig" => "230",  "ccedil"=> "231",
+	       "egrave"=> "232",  "eacute"=> "233",  "ecirc" => "234",  "euml"  => "235",
+	       "igrave"=> "236",  "iacute"=> "237",  "icirc" => "238",  "iuml"  => "239",
+	       "eth"   => "240",  "ntilde"=> "241",  "ograve"=> "242",  "oacute"=> "243",
+	       "ocirc" => "244",  "otilde"=> "245",  "ouml"  => "246",  
+	       "oslash"=> "248",  "ugrave"=> "249",  "uacute"=> "250",  "ucirc" => "251",
+	       "uuml"  => "252",  "yacute"=> "253",  "thorn" => "254",  "yuml"  => "255");
+
+my %symnetosf = ("quot"  => "34",   "amp"   => "38",   "lt"    => "60",   "gt"    => "62",
+	      "nbsp"  => "160",  "iexcl" => "161",  "cent"  => "162",  "pound" => "163",
+	      "curren"=> "164",  "yen"   => "165",  "brvbar"=> "166",  "sect"  => "167",
+	      "uml"   => "168",  "copy"  => "169",  "ordf"  => "170",  "laquo" => "171",
+	      "not"   => "172",  "shy"   => "173",  "reg"   => "174",  "macr"  => "175",
+	      "deg"   => "176",  "plusmn"=> "177",  "sup2"  => "178",  "sup3"  => "179",
+	      "acute" => "180",  "micro" => "181",  "para"  => "182",  "middot"=> "183",
+	      "cedil" => "184",  "sup1"  => "185",  "ordm"  => "186",  "raquo" => "187",
+	      "frac14"=> "188",  "frac12"=> "189",  "frac34"=> "190",  "iquest"=> "191",
+	      "times" => "215",  "divide"=> "247");
+
+
+
+# standard font to plain text
+my %sftotxt = ("32"  => " ",  "33"  => "!",  "34"  => "\"",  "35"  => "\#",  "36"  => "\$",
+	    "37"  => "\%", "38"  => "&",  "39"  => "'",   "40"  => "(",   "41"  => ")",
+	    "42"  => "*",  "43"  => "+",  "44"  => ",",   "45"  => "-",   "46"  => ".",
+	    "47"  => "/",  "48"  => "0",  "49"  => "1",   "50"  => "2",   "51"  => "3",
+	    "52"  => "4",  "53"  => "5",  "54"  => "6",   "55"  => "7",   "56"  => "8",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "60"  => "<",   "61"  => "=",
+	    "62"  => ">",  "63"  => "?",  "64"  => "\@",  "65"  => "A",   "66"  => "B",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "61"  => "=",
+	    "63"  => "?",  "64"  => "\@", "65"  => "A",  "66"  => "B",
+	    "67"  => "C",  "68"  => "D",  "69"  => "E",   "70"  => "F",   "71"  => "G",
+	    "72"  => "H",  "73"  => "I",  "74"  => "J",   "75"  => "K",   "76"  => "L",
+	    "77"  => "M",  "78"  => "N",  "79"  => "O",   "80"  => "P",   "81"  => "Q",
+	    "82"  => "R",  "83"  => "S",  "84"  => "T",   "85"  => "U",   "86"  => "V",
+	    "87"  => "W",  "88"  => "X",  "89"  => "Y",   "90"  => "Z",   "91"  => "[",
+	    "92"  => "\\", "93"  => "]",  "94"  => "^",   "95"  => "_",   "96"  => "`",
+	    "97"  => "a",  "98"  => "b",  "99"  => "c",   "100" => "d",   "101" => "e",
+	    "102" => "f",  "103" => "g",  "104" => "h",   "105" => "i",   "106" => "j",
+	    "107" => "k",  "108" => "l",  "109" => "m",   "110" => "n",   "111" => "o",
+	    "112" => "p",  "113" => "q",  "114" => "r",   "115" => "s",   "116" => "t",
+	    "117" => "u",  "118" => "v",  "119" => "w",   "120" => "x",   "121" => "y",
+	    "122" => "z",  "123" => "{",  "124" => "|",   "125" => "}",   "126" => "~",
+	    "130" => ",",  "131" => "f",  "132" => "\"",   "133" => "...", "139" => "<",
+	    "140" => "OE", "145" => "'",  "146" => "'",   "147" => "\"",   "148" => "\"",
+	    "149" => "o",  "150" => "--", "151" => "-",   "152" => "~",   "153" => "TM",
+	    "155" => ">",  "156" => "oe", "159" => "Y",   "160" => " ",   "178" => "2",
+	    "179" => "3",  "185" => "1",  "188" => "1/4", "189" => "1/2", "190" => "3/4",
+	    "192" => "A",  "193" => "A",  "194" => "A",   "195" => "A",   "196" => "A",
+	    "197" => "A",  "198" => "AE", "199" => "C",   "200" => "E",   "201" => "E",
+	    "202" => "E",  "203" => "E",  "204" => "I",   "205" => "I",   "206" => "I",
+	    "207" => "I",  "208" => "D",  "209" => "N",   "210" => "O",   "211" => "O",
+	    "212" => "O",  "213" => "O",  "214" => "O",   "215" => "*",   "216" => "O",
+	    "217" => "U",  "218" => "U",  "219" => "U",   "220" => "U",   "221" => "Y",
+	    "223" => "ss", "224" => "a",  "225" => "a",   "226" => "a",   "227" => "a",
+	    "228" => "a",  "229" => "a",  "230" => "ae",  "231" => "c",   "232" => "e",
+	    "233" => "e",  "234" => "e",  "235" => "e",   "236" => "i",   "237" => "i",
+	    "238" => "i",  "239" => "i",  "241" => "n",   "242" => "o",   "243" => "o",
+	    "244" => "o",  "245" => "o",  "246" => "o",   "247" => "/",   "248" => "o",
+	    "249" => "u",  "250" => "u",  "251" => "u",   "252" => "u",   "253" => "y",
+	    "255" => "y",  "8218" => ",");
+
+
+# returns the character as a raw utf-8 character. It assumes that the
+# & and ; have been stripped off the string.
+sub getcharequiv {
+    my ($entity, $convertsymbols) = @_;
+
+    # a numeric entity
+    if ($entity =~ /^\#0*(\d+)/) {
+	my $code=$1;
+	# non-standard Microsoft breakage, as usual
+	if ($code < 0x9f) { # code page 1252 uses reserved bytes
+	    if ($code == 0x91) {$code=0x2018} # 145 = single left quote
+	    elsif ($code == 0x92) {$code=0x2019} # 146 = single right quote
+	    elsif ($code == 0x93) {$code=0x201c} # 147 = double left quote
+	    elsif ($code == 0x94) {$code=0x201d} # 148 = double right quote
+	    # ...
+	}
+	return &unicode::unicode2utf8([$code]);
+    }
+    
+    # a named character entity
+    if (defined $charnetosf{$entity}) {
+	return &unicode::unicode2utf8([$charnetosf{$entity}]);
+    }
+
+    # a named symbol entity
+    if ($convertsymbols && defined $symnetosf{$entity}) {
+	return &unicode::unicode2utf8([$symnetosf{$entity}]);
+    }
+
+    return "&$entity;"; # unknown character
+}
+
+# convert character entities from named equivalents to html font
+sub convertcharentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,0)/gse;
+}
+
+# convert any entities from named equivalents to html font
+sub convertallentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,1)/gse;
+}
+
+sub html2txt {
+    # args: the text that you want converted to ascii, 
+    # and whether to strip out sgml tags
+
+    # strip out sgml tags if needed
+    $_[0] =~ s/<[^>]*>//g if $_[1];
+
+    # convert the char entities to the standard html font
+    &convertcharentities($_[0]); 
+    
+    # convert the html character set to a plain ascii character set
+    my $pos = 0;
+    while ($pos < length($_[0])) {
+	my $charnum = ord(substr($_[0], $pos, 1));
+	if ($charnum >= 32) { # only convert characters above #32
+	    my $replacechars = " ";
+	    $replacechars = $sftotxt{$charnum} if defined $sftotxt{$charnum};
+	    substr($_[0], $pos, 1) = $replacechars;
+	    $pos += length ($replacechars);
+
+	} else {
+	    $pos ++;
+	}
+    }
+}  
+
+
+# look for mime.types (eg in /etc, or apache/conf directories), or have a look
+# at <ftp://ftp.iana.org/in-notes/iana/assignments/media-types/> for defaults.
+sub guess_mime_type {
+    my ($filename) = @_;
+
+    my ($fileext) = $filename =~ /\.(\w+)$/;
+    return "unknown" unless defined $fileext;
+
+    my %mime_type = ("ai"=>"application/postscript", "aif"=>"audio/x-aiff", 
+		     "aifc"=>"audio/x-aiff", "aiff"=>"audio/x-aiff", 
+		     "au"=>"audio/basic", "avi"=>"video/x-msvideo",
+		     "bcpio"=>"application/x-bcpio", "bin"=>"application/octet-stream", 
+		     "cdf"=>"application/x-netcdf", "class"=>"application/octet-stream", 
+		     "cpio"=>"application/x-cpio", "cpt"=>"application/mac-compactpro",
+		     "csh"=>"application/x-csh", "dcr"=>"application/x-director", 
+		     "dir"=>"application/x-director", "dms"=>"application/octet-stream", 
+		     "doc"=>"application/msword", "dvi"=>"application/x-dvi",
+		     "dxr"=>"application/x-director", "eps"=>"application/postscript", 
+		     "etx"=>"text/x-setext",
+		     "exe"=>"application/octet-stream", "gif"=>"image/gif",
+		     "gtar"=>"application/x-gtar", "hdf"=>"application/x-hdf",
+		     "hqx"=>"application/mac-binhex40", "htm"=>"text/html",
+		     "html"=>"text/html", "ice"=>"x-conference/x-cooltalk",
+		     "ief"=>"image/ief", "jpe"=>"image/jpeg",
+		     "jpeg"=>"image/jpeg", "jpg"=>"image/jpeg",
+		     "kar"=>"audio/midi", "latex"=>"application/x-latex",
+		     "lha"=>"application/octet-stream", "lzh"=>"application/octet-stream",
+		     "man"=>"application/x-troff-man", "mcf"=>"image/vasa",
+		     "me"=>"application/x-troff-me", "mid"=>"audio/midi",
+		     "midi"=>"audio/midi", "mif"=>"application/x-mif",
+		     "mov"=>"video/quicktime", "movie"=>"video/x-sgi-movie",
+		     "mp2"=>"audio/mpeg", "mpe"=>"video/mpeg",
+		     "mpeg"=>"video/mpeg", "mpg"=>"video/mpeg",
+		     "mpga"=>"audio/mpeg", "ms"=>"application/x-troff-ms",
+		     "nc"=>"application/x-netcdf", "oda"=>"application/oda",
+		     "pbm"=>"image/x-portable-bitmap", "pdb"=>"chemical/x-pdb",
+		     "pdf"=>"application/pdf", "pgm"=>"image/x-portable-graymap",
+		     "png"=>"image/png", "pnm"=>"image/x-portable-anymap",
+		     "ppm"=>"image/x-portable-pixmap",
+		     "ppt"=>"application/vnd.ms-powerpoint",
+		     "ps"=>"application/postscript", "qt"=>"video/quicktime",
+		     "ra"=>"audio/x-realaudio", "ram"=>"audio/x-pn-realaudio",
+		     "ras"=>"image/x-cmu-raster", "rgb"=>"image/x-rgb",
+		     "roff"=>"application/x-troff", "rpm"=>"audio/x-pn-realaudio-plugin",
+		     "rtf"=>"application/rtf", "rtx"=>"text/richtext",
+		     "sgm"=>"text/x-sgml", "sgml"=>"text/x-sgml",
+		     "sh"=>"application/x-sh", "shar"=>"application/x-shar",
+		     "sit"=>"application/x-stuffit", "skd"=>"application/x-koan",
+		     "skm"=>"application/x-koan", "skp"=>"application/x-koan",
+		     "skt"=>"application/x-koan", "snd"=>"audio/basic",
+		     "src"=>"application/x-wais-source", "sv4cpio"=>"application/x-sv4cpio",
+		     "sv4crc"=>"application/x-sv4crc", "t"=>"application/x-troff",
+		     "tar"=>"application/x-tar", "tcl"=>"application/x-tcl",
+		     "tex"=>"application/x-tex", "texi"=>"application/x-texinfo",
+		     "texinfo"=>"application/x-texinfo", "tif"=>"image/tiff",
+		     "tiff"=>"image/tiff", "tr"=>"application/x-troff",
+		     "tsv"=>"text/tab-separated-values", "txt"=>"text/plain",
+		     "ustar"=>"application/x-ustar", "vcd"=>"application/x-cdlink",
+		     "vrml"=>"x-world/x-vrml", "wav"=>"audio/x-wav",
+		     "wrl"=>"x-world/x-vrml", "xbm"=>"image/x-xbitmap",
+		     "xls"=>"application/vnd.ms-excel",
+		     "xpm"=>"image/x-xpixmap", "xwd"=>"image/x-xwindowdump",
+		     "xyz"=>"chemical/x-pdb", "zip"=>"application/zip");
+
+    return $mime_type{$fileext} if (defined $mime_type{$fileext});
+
+    return "unknown";
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/giget.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/giget.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/giget.pm	(revision 21112)
@@ -0,0 +1,263 @@
+
+sub readin_html 
+{
+    my ($html_fname) = @_;
+
+    open(HIN,"<$html_fname") 
+	|| die "Unable to open $html_fname: $!\n";
+    
+    my $html_text;
+    my $line;
+    while (defined ($line=<HIN>)) {
+	$html_text .= $line;
+    }
+    close(HIN);
+
+    return $html_text;
+}
+
+sub stripout_anchortags
+{
+    my ($html_text) = @_;
+
+    my @anchor_tags = ($html_text =~ m/(<a\s+.*?>)+/gs);
+
+    return @anchor_tags;
+} 
+
+
+sub print_tags
+{
+    my (@tags) = @_;
+
+    my $a;
+    foreach $a ( @tags) {
+	print "$a\n";
+    }
+}
+
+sub filter_tags
+{
+    my ($filter_text,@tags) = @_;
+
+    my @filtered_tags = ();
+
+    my $t;
+    foreach $t (@tags) {
+	if ($t =~ m/$filter_text/x) {
+	    push(@filtered_tags,$t);
+	}
+    }
+
+    return @filtered_tags;
+}
+
+sub extract_urls {
+    my (@tags) = @_;
+
+    my @urls = ();
+
+    my $t;
+    foreach $t (@tags) {
+	if ($t =~ m/href=([^ ]+)/i) {
+	    my $url = $1;
+	    $url =~ s/&amp;/&/g;
+	    push(@urls,$url);
+	}
+    }
+
+    return @urls;
+}
+
+sub get_gi_page
+{
+    my ($cgi_base,$cgi_call,$downloadto_fname) = @_;
+
+    my $full_url = "$cgi_base$cgi_call";
+    
+    if ((!-e $downloadto_fname) || (-z $downloadto_fname)) {
+	my $cmd = "wget -nv -T 10 -nc -U \"Mozilla\" -O \"$downloadto_fname\" \"$full_url\"";
+##	print STDERR "*** wget cmd:\n $cmd\n";
+
+    `$cmd`;
+    }
+
+    if (-z $downloadto_fname) { 
+	print STDERR "Warning: downloaded file 0 bytes!\n";
+    }
+}
+
+
+sub parse_gi_search_page 
+{
+    my ($ga_base,$search_term_dir,$downloaded_fname,$currpage_url) = @_;
+
+    my $nextpage_url = undef;
+
+    my @imgref_urls = ();
+
+    my $downloaded_text = readin_html($downloaded_fname);
+    if (defined $downloaded_text) {
+	my @anchor_tags = stripout_anchortags($downloaded_text);
+	
+	my @thumbimg_tags = filter_tags("imgres\\?",@anchor_tags);
+	my @nextpage_tags = filter_tags("images\\?.*?start=\\d+",@anchor_tags);
+	
+	my @thumbimg_urls = extract_urls(@thumbimg_tags);
+	my @nextpage_urls = extract_urls(@nextpage_tags);
+
+	my $curr_start = 0;
+	if ($currpage_url =~ m/start=(\d+)/) {
+	    $curr_start = $1;
+	}
+
+	my $pot_url;
+	foreach $pot_url (@nextpage_urls) {
+	
+	    my ($next_start) = ($pot_url =~ m/start=(\d+)/);
+	    if ($next_start>$curr_start) {
+		$nextpage_url = $pot_url;
+		last;
+	    }
+	}
+
+#	print "-" x 40, "\n";
+	my $c = 1;
+	my $p = 1;
+
+	foreach my $tvu (@thumbimg_urls) {
+	    my ($img_url) = ($tvu =~ m/imgurl=([^&]*)/);
+	    $img_url =~ s/%25/%/g;
+
+	    my ($imgref_url) = ($tvu =~ m/imgrefurl=([^&]*)/);
+##	    print STDERR "****imgref_url = $imgref_url\n";
+	    $imgref_url =~ s/%25/%/g;
+	    
+	    my ($img_ext) = ($img_url =~ m/\.(\w+)$/);
+	    $img_ext = lc($img_ext);
+
+	    # remove http:// if there, so later we can explicitly add it in
+	    $img_url =~ s/^http:\/\///; 
+
+	    print "Downloading image url http://$img_url\n";
+	    my $output_fname = "$search_term_dir/img_$c.$img_ext";
+
+	    get_gi_page("http://",$img_url,$output_fname);
+
+	    if (-s $output_fname == 0) {
+		unlink $output_fname;
+	    }
+	    elsif (system("identify \"$output_fname\"") > 0 ) {
+		print STDERR "**** NOT JPEG: output_fname \n";
+		unlink $output_fname;
+	    }
+	    else {
+
+		my $command = "identify \"$output_fname\" 2>&1";
+		my $result = `$command`;
+
+		my $type =   'unknown';
+		my $width =  'unknown';
+		my $height = 'unknown';
+		
+		my $image_safe = quotemeta $output_fname;
+		if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) {
+		    $type = $1;
+		    $width = $2;
+		    $height = $3;
+		}
+
+		if (($width ne "unknown") && ($height ne "unknown")) {
+		    if (($width>200) || ($height>200)) {
+			`convert \"$output_fname\" -resize 200x200 /tmp/x.jpg`;
+			`/bin/mv /tmp/x.jpg \"$output_fname\"`;
+		    }
+		}
+		$c++;
+	    }
+
+	    push(@imgref_urls,$imgref_url);
+
+	    last if ($c==3); # Only take first 2
+
+	    $p++;
+
+	    if ($p==20) {
+		print STDERR "*** Unable to get enough images after 20 passes\n";
+		last;
+	    }
+
+
+	}
+
+	if (defined $nextpage_url) {
+	    print "Next page URL:\n";
+	    print_tags($nextpage_url);
+	}
+#	print "-" x 40, "\n";
+    }
+
+    return ($nextpage_url, \@imgref_urls);
+}
+
+sub make_search_term_safe
+{
+    my ($search_terms) = @_;
+
+    my $search_term_safe = join("+",@$search_terms);
+    $search_term_safe =~ s/\"/%22/g;
+    $search_term_safe =~ s/ /+/g;
+
+    return $search_term_safe;
+}
+
+sub gi_query_url
+{
+    my ($search_term) = @_;
+
+    my $search_term_safe = make_search_term_safe($search_term);
+
+    my $nextpage_url 
+	= "/images?as_filetype=jpg&imgc=color\&ie=UTF-8\&oe=UTF-8\&hl=en\&btnG=Google+Search";
+    $nextpage_url .= "\&q=$search_term_safe";
+
+    return $nextpage_url;
+}
+
+sub gi_url_base
+{
+    return "http://images.google.com";
+}
+
+sub giget
+{
+    my ($search_terms,$output_dir) = @_;
+    my $imgref_urls = [];
+
+    if (!-e $output_dir) {
+	mkdir($output_dir);
+    
+    }
+	
+    print STDERR "Searching Google Images for: ", join(", ",@$search_terms), "\n";
+
+    my $gi_base = gi_url_base();
+    my $nextpage_url = gi_query_url($search_terms);
+    
+    $respage_fname = "$output_dir/respage1.html";
+    get_gi_page($gi_base,$nextpage_url,$respage_fname);
+    
+    ($nextpage_url, $imgref_urls) 
+	= parse_gi_search_page($gi_base,$output_dir,
+			       $respage_fname,$nextpage_url);
+#    else {
+#	print STDERR "  Images already mirrored\n";
+#    }
+
+    print STDERR "-" x 40, "\n";
+
+    return $imgref_urls;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/gsprintf.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/gsprintf.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/gsprintf.pm	(revision 21112)
@@ -0,0 +1,239 @@
+###########################################################################
+#
+# gsprintf.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+use strict;
+no strict 'refs';
+
+package gsprintf;
+require Exporter;
+@gsprintf::ISA=qw(Exporter);
+
+use unicode;
+use util;
+
+@gsprintf::EXPORT_OK = qw'gsprintf'; # functions we can export into namespace
+
+
+# Language-specific resource bundle
+my %specialresourcebundle = ();
+our $specialoutputencoding; # our, so that it can be changed outside.
+
+# Default resource bundle
+my %defaultresourcebundle;
+my $defaultoutputencoding;
+
+# English resource bundle
+my %englishresourcebundle;
+my $englishoutputencoding;
+
+# Ignore the OutputEncoding strings in the resource bundles and output all text in UTF-8
+my $outputstringsinUTF8 = 0;
+
+
+sub gsprintf
+{
+    my ($handle, $text_string, @text_arguments) = @_;
+
+    # Return unless the required arguments were supplied
+    return unless (defined($handle) && defined($text_string));
+
+    # Look up all the strings in the dictionary
+    $text_string =~ s/(\{[^\}]+\})/&lookup_string($1)/eg;
+
+    # Resolve the string arguments using sprintf, then write out to the handle
+    print $handle sprintf($text_string, @text_arguments);
+}
+
+
+sub lookup_string
+{
+    my ($stringkey) = @_;
+    return "" unless defined $stringkey;
+    # Try the language-specific resource bundle first
+    my $utf8string = $specialresourcebundle{$stringkey};
+    my $outputencoding = $specialoutputencoding;
+
+    # Try the default resource bundle next
+    if (!defined($utf8string)) {
+	# Load the default resource bundle if it is not already loaded
+	&load_default_resource_bundle() if (!%defaultresourcebundle);
+
+	$utf8string = $defaultresourcebundle{$stringkey};
+	$outputencoding = $defaultoutputencoding;
+    }
+
+    # Try the English resource bundle last
+    if (!defined($utf8string)) {
+	# Load the English resource bundle if it is not already loaded
+	&load_english_resource_bundle() if (!%englishresourcebundle);
+
+	$utf8string = $englishresourcebundle{$stringkey};
+	$outputencoding = $englishoutputencoding;
+    }
+
+    # No matching string was found, so just return the key
+    if (!defined($utf8string)) {
+	return $stringkey;
+    }
+
+    # Return the string matching the key
+    return $utf8string if (!defined($outputencoding) || $outputstringsinUTF8
+			   || $outputencoding eq "utf8");
+
+    # If an 8-bit output encoding has been defined, encode the string appropriately
+    my $encoded=unicode::unicode2singlebyte(&unicode::utf82unicode($utf8string), $outputencoding);
+    
+    # If we successfully encoded it, return it
+    if ($encoded) { return $encoded }
+
+    # Otherwise, we can't convert to the requested encoding. return the utf8?
+    $specialoutputencoding='utf8';
+    return $utf8string;
+}
+
+
+sub load_language_specific_resource_bundle
+{
+    my $language = shift(@_);
+
+    # Read the specified resource bundle
+    my $resourcebundlehome = &util::filename_cat("$ENV{'GSDLHOME'}", "perllib");
+    my $resourcebundlename = "strings_" . $language . ".rb";
+    my $resourcebundlefile = &util::filename_cat($resourcebundlehome, $resourcebundlename);
+
+    %specialresourcebundle = &read_resource_bundle($resourcebundlefile);
+    return if (!%specialresourcebundle);
+
+    # Read the output encoding to use from the resource bundle
+    if ($ENV{'GSDLOS'} =~ /windows/) {
+	$specialoutputencoding = $specialresourcebundle{"{OutputEncoding.windows}"};
+    }
+    else {
+	# see if there is an encoding set in the appropriate locale env var
+
+	foreach my $envvar ('LC_ALL', 'LANG') {
+	    if (!exists $ENV{$envvar}) { next }
+	    my $locale=$ENV{$envvar};
+	    if ($locale !~ /^\w+\.(.+)$/) { next }
+	    my $enc=lc($1);
+	    $enc =~ s/-/_/g;
+	    if ($enc eq 'utf_8') { $enc='utf8' } # normalise to this name
+	    $specialoutputencoding = $enc;
+	    return;
+	}
+	$specialoutputencoding = $specialresourcebundle{"{OutputEncoding.unix}"};
+    }
+}
+
+
+sub load_default_resource_bundle
+{
+    # Read the default resource bundle
+    my $resourcebundlehome = &util::filename_cat("$ENV{'GSDLHOME'}", "perllib");
+    my $resourcebundlename = "strings.rb";
+    my $resourcebundlefile = &util::filename_cat($resourcebundlehome, $resourcebundlename);
+
+    %defaultresourcebundle = &read_resource_bundle($resourcebundlefile);
+    return if (!%defaultresourcebundle);
+
+    # Read the output encoding to use from the resource bundle
+    if ($ENV{'GSDLOS'} =~ /windows/) {
+	$defaultoutputencoding = $defaultresourcebundle{"{OutputEncoding.windows}"};
+    }
+    else {
+	$defaultoutputencoding = $defaultresourcebundle{"{OutputEncoding.unix}"};
+    }
+}
+
+
+sub load_english_resource_bundle
+{
+    # Ensure the English resource bundle hasn't already been loaded
+    if (%specialresourcebundle && $specialresourcebundle{"{Language.code}"} eq "en") {
+	%englishresourcebundle = %specialresourcebundle;
+	$englishoutputencoding = $specialoutputencoding;
+    }
+    if ($defaultresourcebundle{"{Language.code}"} eq "en") {
+	%englishresourcebundle = %defaultresourcebundle;
+	$englishoutputencoding = $defaultoutputencoding;
+    }
+
+    # Read the English resource bundle
+    my $resourcebundlehome = &util::filename_cat("$ENV{'GSDLHOME'}", "perllib");
+    my $resourcebundlename = "strings_en.rb";
+    my $resourcebundlefile = &util::filename_cat($resourcebundlehome, $resourcebundlename);
+
+    %englishresourcebundle = &read_resource_bundle($resourcebundlefile);
+    return if (!%englishresourcebundle);
+
+    # Read the output encoding to use from the resource bundle
+    if ($ENV{'GSDLOS'} =~ /windows/) {
+	$englishoutputencoding = $englishresourcebundle{"{OutputEncoding.windows}"};
+    }
+    else {
+	$englishoutputencoding = $englishresourcebundle{"{OutputEncoding.unix}"};
+    }
+}
+
+
+sub read_resource_bundle
+{
+    my ($resourcebundlefilepath) = shift(@_);
+
+    # Return an empty hash if the specified resource bundle could not be read
+    return () if (!open(RESOURCE_BUNDLE, "<$resourcebundlefilepath"));
+
+    # Load this resource bundle
+    my @resourcebundlelines = <RESOURCE_BUNDLE>;
+    close(RESOURCE_BUNDLE);
+
+    # Parse the resource bundle
+    my %resourcebundle = ();
+    foreach my $line (@resourcebundlelines) {
+	# Remove any trailing whitespace
+	$line =~ s/(\s*)$//;
+
+	# Ignore comments and empty lines
+	if ($line !~ /^\#/ && $line ne "") {
+	    # Parse key (everything up to the first colon)
+	    $line =~ /^([^:]+):(.+)$/;
+	    my $linekey = "{" . $1 . "}";
+	    my $linetext = $2;
+
+	    # Map key to text
+	    $resourcebundle{$linekey} = $linetext;
+	}
+    }
+
+    return %resourcebundle;
+}
+
+
+sub output_strings_in_UTF8
+{
+    $outputstringsinUTF8 = 1;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/iso639.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/iso639.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/iso639.pm	(revision 21112)
@@ -0,0 +1,339 @@
+# mapping to and from iso 639 two letter language symbols
+
+package iso639;
+
+%fromiso639 = (
+	       'aa' => 'Afar',
+	       'ab' => 'Abkhazian',
+	       'af' => 'Afrikaans',
+	       'am' => 'Amharic',
+	       'ar' => 'Arabic',
+	       'as' => 'Assamese',
+	       'ay' => 'Aymara',
+	       'az' => 'Azerbaijani',
+
+	       'ba' => 'Bashkir',
+	       'be' => 'Byelorussian',
+	       'bg' => 'Bulgarian',
+	       'bh' => 'Bihari',
+	       'bi' => 'Bislama',
+	       'bn' => 'Bengali; Bangla',
+	       'bo' => 'Tibetan',
+	       'br' => 'Breton',
+
+	       'ca' => 'Catalan',
+	       'co' => 'Corsican',
+	       'cs' => 'Czech',
+	       'cy' => 'Welsh',
+
+	       'da' => 'Danish',
+	       'de' => 'German',
+	       'dz' => 'Bhutani',
+
+	       'el' => 'Greek',
+	       'en' => 'English',
+	       'eo' => 'Esperanto',
+	       'es' => 'Spanish',
+	       'et' => 'Estonian',
+	       'eu' => 'Basque',
+
+	       'fa' => 'Persian',
+	       'fi' => 'Finnish',
+	       'fj' => 'Fiji',
+	       'fo' => 'Faroese',
+	       'fr' => 'French',
+	       'fy' => 'Frisian',
+
+	       'ga' => 'Irish',
+	       'gd' => 'Scots Gaelic',
+	       'gl' => 'Galician',
+	       'gn' => 'Guarani',
+	       'gu' => 'Gujarati',
+
+	       'ha' => 'Hausa',
+	       'he' => 'Hebrew',
+	       'hi' => 'Hindi',
+	       'hr' => 'Croatian',
+	       'hu' => 'Hungarian',
+	       'hy' => 'Armenian',
+
+	       'ia' => 'Interlingua',
+	       'id' => 'Indonesian',
+	       'ie' => 'Interlingue',
+	       'ik' => 'Inupiak',
+	       'is' => 'Icelandic',
+	       'it' => 'Italian',
+	       'iu' => 'Inuktitut',
+
+	       'ja' => 'Japanese',
+	       'jv' => 'Javanese',
+
+	       'ka' => 'Georgian',
+	       'kk' => 'Kazakh',
+	       'kl' => 'Greenlandic',
+	       'km' => 'Cambodian',
+	       'kn' => 'Kannada',
+	       'ko' => 'Korean',
+	       'ks' => 'Kashmiri',
+	       'ku' => 'Kurdish',
+	       'ky' => 'Kirghiz',
+
+	       'la' => 'Latin',
+	       'ln' => 'Lingala',
+	       'lo' => 'Laothian',
+	       'lt' => 'Lithuanian',
+	       'lv' => 'Latvian, Lettish',
+
+	       'mg' => 'Malagasy',
+	       'mi' => 'Maori',
+	       'mk' => 'Macedonian',
+	       'ml' => 'Malayalam',
+	       'mn' => 'Mongolian',
+	       'mo' => 'Moldavian',
+	       'mr' => 'Marathi',
+	       'ms' => 'Malay',
+	       'mt' => 'Maltese',
+	       'my' => 'Burmese',
+
+	       'na' => 'Nauru',
+	       'ne' => 'Nepali',
+	       'nl' => 'Dutch',
+	       'no' => 'Norwegian',
+
+	       'oc' => 'Occitan',
+	       'om' => '(Afan) Oromo',
+	       'or' => 'Oriya',
+
+	       'pa' => 'Punjabi',
+	       'pl' => 'Polish',
+	       'ps' => 'Pashto, Pushto',
+	       'pt' => 'Portuguese',
+
+	       'qu' => 'Quechua',
+	       'rm' => 'Rhaeto-Romance',
+	       'rn' => 'Kirundi',
+	       'ro' => 'Romanian',
+	       'ru' => 'Russian',
+	       'rw' => 'Kinyarwanda',
+
+	       'sa' => 'Sanskrit',
+	       'sd' => 'Sindhi',
+	       'sg' => 'Sangro',
+	       'sh' => 'Serbo-Croatian',
+	       'si' => 'Singhalese',
+	       'sk' => 'Slovak',
+	       'sl' => 'Slovenian',
+	       'sm' => 'Samoan',
+	       'sn' => 'Shona',
+	       'so' => 'Somali',
+	       'sq' => 'Albanian',
+	       'sr' => 'Serbian',
+	       'ss' => 'Siswati',
+	       'st' => 'Sesotho',
+	       'su' => 'Sudanese',
+	       'sv' => 'Swedish',
+	       'sw' => 'Swahili',
+
+	       'ta' => 'Tamil',
+	       'te' => 'Tegulu',
+	       'tg' => 'Tajik',
+	       'th' => 'Thai',
+	       'ti' => 'Tigrinya',
+	       'tk' => 'Turkmen',
+	       'tl' => 'Tagalog',
+	       'tn' => 'Setswana',
+	       'to' => 'Tonga',
+	       'tr' => 'Turkish',
+	       'ts' => 'Tsonga',
+	       'tt' => 'Tatar',
+	       'tw' => 'Twi',
+
+	       'ug' => 'Uigur',
+	       'uk' => 'Ukrainian',
+	       'ur' => 'Urdu',
+	       'uz' => 'Uzbek',
+
+	       'vi' => 'Vietnamese',
+	       'vo' => 'Volapuk',
+
+	       'wo' => 'Wolof',
+
+	       'xh' => 'Xhosa',
+
+	       'yi' => 'Yiddish',
+	       'yo' => 'Yoruba',
+
+	       'za' => 'Zhuang',
+	       'zh' => 'Chinese',
+	       'zu' => 'Zulu'
+	       );
+
+%toiso639 = (
+	     'afar' => 'aa',
+	     'abkhazian' => 'ab',
+	     'afrikaans' => 'af',
+	     'amharic' => 'am',
+	     'arabic' => 'ar',
+	     'assamese' => 'as',
+	     'aymara' => 'ay',
+	     'azerbaijani' => 'az',
+
+	     'bashkir' => 'ba',
+	     'byelorussian' => 'be',
+	     'bulgarian' => 'bg',
+	     'bihari' => 'bh',
+	     'bislama' => 'bi',
+	     'bengali' => 'bn',
+	     'bangla' => 'bn',
+	     'tibetan' => 'bo',
+	     'breton' => 'br',
+
+	     'catalan' => 'ca',
+	     'corsican' => 'co',
+	     'czech' => 'cs',
+	     'welsh' => 'cy',
+
+	     'danish' => 'da',
+	     'german' => 'de',
+	     'bhutani' => 'dz',
+
+	     'greek' => 'el',
+	     'english' => 'en',
+	     'esperanto' => 'eo',
+	     'spanish' => 'es',
+	     'estonian' => 'et',
+	     'basque' => 'eu',
+
+	     'persian' => 'fa',
+	     'finnish' => 'fi',
+	     'fiji' => 'fj',
+	     'faroese' => 'fo',
+	     'french' => 'fr',
+	     'frisian' => 'fy',
+
+	     'irish' => 'ga',
+	     'scots_gaelic' => 'gd',
+	     'galician' => 'gl',
+	     'guarani' => 'gn',
+	     'gujarati' => 'gu',
+
+	     'hausa' => 'ha',
+	     'hebrew' => 'he',
+	     'hindi' => 'hi',
+	     'croatian' => 'hr',
+	     'hungarian' => 'hu',
+	     'armenian' => 'hy',
+
+	     'interlingua' => 'ia',
+	     'indonesian' => 'id',
+	     'interlingue' => 'ie',
+	     'inupiak' => 'ik',
+	     'icelandic' => 'is',
+	     'italian' => 'it',
+	     'inuktitut' => 'iu',
+
+	     'japanese' => 'ja',
+	     'javanese' => 'jv',
+
+	     'georgian' => 'ka',
+	     'kazakh' => 'kk',
+	     'greenlandic' => 'kl',
+	     'cambodian' => 'km',
+	     'kannada' => 'kn',
+	     'korean' => 'ko',
+	     'kashmiri' => 'ks',
+	     'kurdish' => 'ku',
+	     'kirghiz' => 'ky',
+
+	     'latin' => 'la',
+	     'lingala' => 'ln',
+	     'laothian' => 'lo',
+	     'lithuanian' => 'lt',
+	     'latvian' => 'lv',
+	     'lettish' => 'lv',
+
+	     'malagasy' => 'mg',
+	     'maori' => 'mi',
+	     'macedonian' => 'mk',
+	     'malayalam' => 'ml',
+	     'mongolian' => 'mn',
+	     'moldavian' => 'mo',
+	     'marathi' => 'mr',
+	     'malay' => 'ms',
+	     'maltese' => 'mt',
+	     'burmese' => 'my',
+
+	     'nauru' => 'na',
+	     'nepali' => 'ne',
+	     'dutch' => 'nl',
+	     'norwegian' => 'no',
+
+	     'occitan' => 'oc',
+	     '(afan)_oromo' => 'om',
+	     'oriya' => 'or',
+
+	     'punjabi' => 'pa',
+	     'polish' => 'pl',
+	     'pashto' => 'ps',
+	     'pushto' => 'ps',
+	     'portuguese' => 'pt',
+
+	     'quechua' => 'qu',
+
+	     'rhaeto-romance' => 'rm',
+	     'kirundi' => 'rn',
+	     'romanian' => 'ro',
+	     'russian' => 'ru',
+	     'kinyarwanda' => 'rw',
+	     
+	     'sanskrit' => 'sa',
+	     'sindhi' => 'sd',
+	     'sangro' => 'sg',
+	     'serbo-croatian' => 'sh',
+	     'singhalese' => 'si',
+	     'slovak' => 'sk',
+	     'slovenian' => 'sl',
+	     'samoan' => 'sm',
+	     'shona' => 'sn',
+	     'somali' => 'so',
+	     'albanian' => 'sq',
+	     'serbian' => 'sr',
+	     'siswati' => 'ss',
+	     'sesotho' => 'st',
+	     'sudanese' => 'su',
+	     'swedish' => 'sv',
+	     'swahili' => 'sw',
+
+	     'tamil' => 'ta',
+	     'tegulu' => 'te',
+	     'tajik' => 'tg',
+	     'thai' => 'th',
+	     'tigrinya' => 'ti',
+	     'turkmen' => 'tk',
+	     'tagalog' => 'tl',
+	     'setswana' => 'tn',
+	     'tonga' => 'to',
+	     'turkish' => 'tr',
+	     'tsonga' => 'ts',
+	     'tatar' => 'tt',
+	     'twi' => 'tw',
+
+	     'uigur' => 'ug',
+	     'ukrainian' => 'uk',
+	     'urdu' => 'ur',
+	     'uzbek' => 'uz',
+
+	     'vietnamese' => 'vi',
+	     'volapuk' => 'vo',
+
+	     'wolof' => 'wo',
+	     
+	     'xhosa' => 'xh',
+
+	     'yiddish' => 'yi',
+	     'yoruba' => 'yo',
+
+	     'zhuang' => 'za',
+	     'chinese' => 'zh',
+	     'zul' => 'zu',
+	     );
Index: /main/tags/3.01/gsdl/perllib/lang.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/lang.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/lang.pm	(revision 21112)
@@ -0,0 +1,201 @@
+###########################################################################
+#
+# lang.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# this module translates language descriptions to and from
+# the ISO 639 standard
+
+package lang;
+
+@lang::trans = (["ab", "Abkhazian"],      ["om", "Afan Oromo"],
+	  ["om", "Oromo"],          ["aa", "Afar"],
+	  ["af", "Afrikaans"],      ["sq", "Albanian"],
+	  ["am", "Amharic"],        ["ar", "Arabic"],
+	  ["hy", "Armenian"],       ["as", "Assamese"],
+	  ["ay", "Aymara"],         ["az", "Azerbaijani"],
+
+	  ["ba", "Bashkir"],        ["eu", "Basque"],
+	  ["bn", "Bengali"],        ["bn", "Bangla"],
+	  ["dz", "Bhutani"],        ["bh", "Bihari"],
+	  ["bi", "Bislama"],        ["br", "Breton"],
+	  ["bg", "Bulgarian"],      ["my", "Burmese"],
+	  ["be", "Byelorussian"],
+
+	  ["km", "Cambodian"],      ["ca", "Catalan"],
+	  ["zh", "Chinese"],        ["co", "Corsican"],
+	  ["hr", "Croatian"],       ["cs", "Czech"],
+
+	  ["da", "Danish"],         ["nl", "Dutch"],
+
+	  ["en", "English"],        ["eo", "Esperanto"],
+	  ["et", "Estonian"],       ["fo", "Faroese"],
+	  ["fj", "Fijian"],         ["fi", "Finnish"],
+	  ["fr", "French"],         ["fy", "Frisian"],
+
+	  ["gl", "Galician"],       ["ka", "Georgian"],
+	  ["de", "German"],         ["el", "Greek"],
+	  ["kl", "Greenlandic"],    ["gn", "Guarani"],
+	  ["gu", "Gujarati"],
+
+	  ["ha", "Hausa"],          ["iw", "Hebrew"],
+	  ["hi", "Hindi"],          ["hu", "Hungarian"],
+
+	  ["is", "Icelandic"],      ["in", "Indonesian"],
+	  ["ia", "Interlingua"],    ["ie", "Interlingue"],
+	  ["ik", "Inupiak"],        ["ga", "Irish"],
+	  ["it", "Italian"],
+
+	  ["ja", "Japanese"],       ["jv", "Javanese"],
+
+	  ["kn", "Kannada"],        ["ks", "Kashmiri"],
+	  ["kk", "Kazakh"],         ["rw", "Kinyarwanda"],
+	  ["ky", "Kirghiz"],        ["rn", "Kirundi"],
+	  ["ko", "Korean"],         ["ku", "Kurdish"],
+
+	  ["lo", "Laothian"],       ["la", "Latin"],
+	  ["lv", "Latvian"],        ["lv", "Lettish"],
+	  ["ln", "Lingala"],        ["lt", "Lithuanian"],
+
+	  ["mk", "Macedonian"],     ["mg", "Malagasy"],
+	  ["ms", "Malay"],          ["ml", "Malayalam"],
+	  ["mt", "Maltese"],        ["mi", "Maori"],
+	  ["mr", "Marathi"],        ["mo", "Moldavian"],
+	  ["mn", "Mongolian"],
+
+	  ["na", "Nauru"],          ["ne", "Nepali"],
+	  ["no", "Norwegian"],
+
+	  ["oc", "Occitan"],        ["or", "Oriya"],
+
+	  ["ps", "Pashto"],         ["ps", "Pushto"],
+	  ["fa", "Persian"],        ["pl", "Polish"],
+	  ["pt", "Portuguese"],     ["pa", "Punjabi"],
+
+	  ["qu", "Quechua"],
+
+	  ["rm", "Rhaeto-Romance"], ["ro", "Romanian"],
+	  ["ru", "Russian"],
+
+	  ["sm", "Samoan"],         ["sg", "Sangho"],
+	  ["sa", "Sanskrit"],       ["gd", "Scots Gaelic"],
+	  ["sr", "Serbian"],        ["sh", "Serbo-Croatian"],
+	  ["st", "Sesotho"],        ["tn", "Setswana"],
+	  ["sn", "Shona"],          ["sd", "Sindhi"],
+	  ["si", "Singhalese"],     ["ss", "Siswati"],
+	  ["sk", "Slovak"],         ["sl", "Slovenian"],
+	  ["so", "Somali"],         ["es", "Spanish"],
+	  ["su", "Sundanese"],      ["sw", "Swahili"],
+	  ["sv", "Swedish"],
+
+	  ["tl", "Tagalog"],        ["tg", "Tajik"],
+	  ["ta", "Tamil"],          ["tt", "Tatar"],
+	  ["te", "Telugu"],         ["th", "Thai"],
+	  ["bo", "Tibetan"],        ["ti", "Tigrinya"],
+	  ["to", "Tonga"],          ["ts", "Tsonga"],
+	  ["tr", "Turkish"],        ["tk", "Turkmen"],
+	  ["tw", "Twi"],
+
+	  ["uk", "Ukrainian"],      ["ur", "Urdu"],
+	  ["uz", "Uzbek"],
+
+	  ["vi", "Vietnamese"],     ["vo", "Volapuk"],
+
+	  ["cy", "Welsh"],          ["wo", "Wolof"],
+
+	  ["xh", "Xhosa"],
+
+	  ["ji", "Yiddish"],        ["yo", "Yoruba"],
+
+	  ["zu", "Zulu"]);
+
+
+sub one_english_to_iso639 {
+    my ($english) = @_;
+    my $iso639 = "";
+
+    foreach my $lang (@lang::trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($english =~ /^\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $iso639 = $code;
+	    last;
+
+	}
+    }
+
+    return $iso639;
+}
+
+
+sub english_to_iso639 {
+    my ($englishlist) = @_;
+    my ($english);
+
+    my @englishlist = split (",", $englishlist);
+    my @iso639list = ();
+
+    foreach my $english (@englishlist) {
+	push (@iso639list, &one_english_to_iso639($english));
+    }
+
+    return join (",", @iso639list);
+}
+
+
+sub one_iso639_to_english {
+    my ($iso639) = @_;
+    my $english = "";
+
+    foreach my $lang (@lang::trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($iso639 =~ /\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $english = $enname;
+	    last;
+
+	}
+    }
+
+    return $english;
+}
+
+
+sub iso639_to_english {
+    my ($iso639list) = @_;
+    my ($iso639);
+
+    my @iso639list = split (",", $iso639list);
+    my @englishlist = ();
+
+    foreach $iso639 (@iso639list) {
+	push (@englishlist, &one_iso639_to_english($iso639));
+    }
+
+    return join (",", @englishlist);
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/lucenebuilder.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/lucenebuilder.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/lucenebuilder.pm	(revision 21112)
@@ -0,0 +1,312 @@
+###########################################################################
+#
+# lucenebuilder.pm -- perl wrapper for building index with Lucene
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package lucenebuilder;
+
+# Use same basic XML structure setup by mgppbuilder/mgppbuildproc
+
+use mgppbuilder; 
+
+sub BEGIN {
+    @lucenebuilder::ISA = ('mgppbuilder');
+}
+
+
+sub new {
+    my $class = shift(@_);
+    my ($collection, $source_dir, $build_dir, $verbosity, 
+	$maxdocs, $debug, $keepold, $allclassifications, 
+	$outhandle, $no_text, $gli) = @_;
+
+    my $self = new mgppbuilder (@_);
+    $self = bless $self, $class;
+
+    $self->{'buildtype'} = "lucene";
+
+    return $self;
+}
+
+sub default_buildproc {
+    my $self  = shift (@_);
+
+    return "lucenebuildproc";
+}
+
+# this writes a nice version of the text docs
+sub compress_text {
+
+    my $self = shift (@_);
+    my ($textindex) = @_;
+    my $outhandle = $self->{'outhandle'};
+    print STDERR "Saving the document text\n";
+    # the text directory
+    my $text_dir = &util::filename_cat($self->{'build_dir'}, "text");
+    my $build_dir = &util::filename_cat($self->{'build_dir'},"");
+    &util::mk_all_dir ($text_dir);
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$text_dir =~ s@/@\\@g;
+    } else {
+	if ($outhandle ne "STDERR") {
+	    # so lucene_passes doesn't print to stderr if we redirect output
+	    $osextra .= " 2>/dev/null";
+	}
+    }
+	
+
+    # get any os specific stuff
+    my $scriptdir = "$ENV{'GSDLHOME'}/bin/script";
+
+    my $lucene_passes_exe = &util::filename_cat($scriptdir, "lucene_passes.pl");
+    my $full_lucene_passes_exe = $lucene_passes_exe;
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$full_lucene_passes_exe = "perl.exe -S \"$lucene_passes_exe\"";
+    }
+    my $lucene_passes_sections = "Doc";
+
+    my ($handle);
+
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$lucene_passes_exe" || 
+	    !open (PIPEOUT, "| $full_lucene_passes_exe text $lucene_passes_sections \"$build_dir\" \"dummy\"   $osextra")) {
+	    print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";
+	}
+	$handle = lucenebuilder::PIPEOUT;
+    }
+    my $levels = $self->{'levels'};
+    my $gdbm_level = "document";
+    if ($levels->{'section'}) {
+	$gdbm_level = "section";
+    }
+
+    undef $levels->{'paragraph'}; # get rid of para if we had it.
+    # set up the document processr
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($textindex);
+    $self->{'buildproc'}->set_indexing_text (0);
+    $self->{'buildproc'}->set_store_text(1);
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($levels);                       
+    $self->{'buildproc'}->set_gdbm_level ($gdbm_level);                       
+    $self->{'buildproc'}->reset();
+    &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   $self->{'buildproc'}, $self->{'maxdocs'});    
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    &plugin::end($self->{'pluginfo'});
+    close ($handle) unless $self->{'debug'};
+    close PIPEOUT;
+    $self->print_stats();
+
+    print STDERR "</Stage>\n" if $self->{'gli'};
+
+}
+
+sub build_indexes {
+    my $self = shift (@_);
+    my ($indexname) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $indexes = [];
+    if (defined $indexname && $indexname =~ /\w/) {
+	push @$indexes, $indexname;
+    } else {
+	$indexes = $self->{'collect_cfg'}->{'indexes'};
+    }
+
+    # create the mapping between the index descriptions 
+    # and their directory names (includes subcolls and langs)
+    $self->{'index_mapping'} = $self->create_index_mapping ($indexes);
+
+    # build each of the indexes
+    foreach $index (@$indexes) {
+	if ($self->want_built($index)) {
+
+	    my $idx = $self->{'index_mapping'}->{$index};
+	    foreach my $level (keys %{$self->{'levels'}}) {
+		next if $level =~ /paragraph/; # we don't do para indexing
+		my ($pindex) = $level =~ /^(.)/;
+		# should probably check that new name with level 
+		# is unique ... but currently (with doc sec and para)
+		# each has unique first letter.
+		$self->{'index_mapping'}->{$index} = $pindex.$idx;
+
+		my $llevel = $mgppbuilder::level_map{$level};
+		print $outhandle "\n*** building index $index at level $llevel in subdirectory " . 
+		    "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
+		print STDERR "<Stage name='Index' source='$index' level=$llevel>\n" if $self->{'gli'};
+
+		$self->build_index($index,$llevel);
+	    }
+	    $self->{'index_mapping'}->{$index} = $idx;
+
+	} else {
+	    print $outhandle "\n*** ignoring index $index\n" if ($self->{'verbosity'} >= 1);
+	}
+    }
+
+    #define the final field lists
+    $self->make_final_field_list();
+}
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index,$llevel) = @_;
+    my $outhandle = $self->{'outhandle'};
+    my $build_dir = $self->{'build_dir'};
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($build_dir, $indexdir));
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $scriptdir = "$ENV{'GSDLHOME'}/bin/script";
+
+    my $exe = &util::get_os_exe ();
+    my $lucene_passes_exe = &util::filename_cat($scriptdir, "lucene_passes.pl");
+    my $full_lucene_passes_exe = $lucene_passes_exe;
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$full_lucene_passes_exe = "perl.exe -S \"$lucene_passes_exe\"";
+    }
+
+    # define the section names for lucenepasses
+    # define the section names and possibly the doc name for lucenepasses
+    my $lucene_passes_sections = $llevel;
+
+    my $opt_create_index = ($self->{'keepold'}) ? "" : "-create";
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$build_dir =~ s@/@\\@g;
+    } else {
+	if ($outhandle ne "STDERR") {
+	    # so lucene_passes doesn't print to stderr if we redirect output
+	    $osextra .= " 2>/dev/null";
+	}
+    }
+ 
+    # get the index expression if this index belongs
+    # to a subcollection
+    my $indexexparr = [];
+    my $langarr = [];
+
+    # there may be subcollection info, and language info. 
+    my ($fields, $subcollection, $language) = split (":", $index);
+    my @subcollections = ();
+    @subcollections = split /,/, $subcollection if (defined $subcollection);
+
+    foreach $subcollection (@subcollections) {
+	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
+	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
+	} 
+    }
+    
+    # add expressions for languages if this index belongs to
+    # a language subcollection - only put languages expressions for the 
+    # ones we want in the index
+    
+    my @languages = ();
+    my $language_metadata = "Language";
+    if (defined ($self->{'collect_cfg'}->{'language_metadata'})) {
+	$language_metadata = $self->{'collect_cfg'}->{'language_metadata'};
+    }
+    @languages = split /,/, $language if (defined $language);
+    foreach my $language (@languages) {
+	my $not=0;
+	if ($language =~ s/^\!//) {
+	    $not = 1;
+	}
+	if($not) {
+	    push (@$langarr, "!$language");
+	} else {
+	    push (@$langarr, "$language");
+	}
+    }
+
+    # Build index dictionary. Uses verbatim stem method
+    print $outhandle "\n    creating index dictionary (lucene_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
+    my ($handle);
+
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$lucene_passes_exe" || 
+	    !open (PIPEOUT, "| $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
+	    print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";
+	}
+	$handle = lucenebuilder::PIPEOUT;
+    }
+	
+    my $store_levels = $self->{'levels'};
+    my $gdbm_level = "document";
+    if ($store_levels->{'section'}) {
+	$gdbm_level = "section";
+    }
+    
+    my $dom_level = "";
+    foreach my $key (keys %$store_levels) {
+	if ($mgppbuilder::level_map{$key} eq $llevel) {
+	    $dom_level = $key;
+	}
+    }
+    if ($dom_level eq "") {
+	print STDERR "Warning: unrecognized tag level $llevel\n";
+	$dom_level = "document";
+    }
+
+    my $local_levels = { $dom_level => 1 }; # work on one level at a time
+
+    # set up the document processr
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index, $indexexparr);
+    $self->{'buildproc'}->set_index_languages ($language_metadata, $langarr) if (defined $language);
+    $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($local_levels);                       
+    $self->{'buildproc'}->set_gdbm_level($gdbm_level);
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    $self->{'buildproc'}->set_levels ($store_levels);                       
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}   
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/lucenebuildproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/lucenebuildproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/lucenebuildproc.pm	(revision 21112)
@@ -0,0 +1,333 @@
+###########################################################################
+#
+# lucenebuildproc.pm -- perl wrapper for building index with Lucene
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package lucenebuildproc;
+
+# This document processor outputs a document
+# for lucene to process
+
+# Use same basic XML structure setup by mgppbuilder/mgppbuildproc
+
+use mgppbuildproc; 
+use ghtml;
+
+sub BEGIN {
+    @lucenebuildproc::ISA = ('mgppbuildproc');
+}
+
+
+sub new {
+    my $class = shift @_;
+    my $self = new mgppbuildproc (@_);
+
+    return bless $self, $class;
+}
+
+
+sub is_incremental_capable
+{
+    my $self = shift (@_);
+
+    # Unlike MG and MGPP, Lucene supports incremental building
+    return 1;
+}
+
+
+sub preprocess_text {
+    my $self = shift (@_);
+    my ($text, $strip_html, $para) = @_;
+
+    my ($outtext) = "";
+    if ($strip_html) { 
+	while ($text =~ /<([^>]*)>/ && $text ne "") {
+	    
+	    my $tag = $1;
+	    $outtext .= $`." "; #add everything before the matched tag
+	    $text = $'; #'everything after the matched tag
+	    if ($para && $tag =~ /^\s*p\s/i) {
+		$outtext .= $para;
+	    }
+	    elsif ($tag =~ /^pre$/) { # a pre tag
+		$text =~ /<\/pre>/; # find the closing pre tag
+		my $tmp_text = $`; #everything before the closing pre tag
+		$text = $'; #'everything after the </pre>
+		$tmp_text =~ s/[<>]//g; # remove all < and >
+		$outtext.= $tmp_text . " ";
+	    }
+	}
+    
+	$outtext .= $text; # add any remaining text
+    } #if strip_html
+    else {
+	$outtext = $text;
+    }
+    #if ($para) {
+	#$text =~ s/(<p\b)/$para$1/gi;
+	#return $text;
+    #}
+
+    # remove entities
+    $outtext =~ s/&\w{1,10};//g; 
+
+    return $outtext;
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj,$file) = @_;
+    my $handle = $self->{'output_handle'};
+    my $outhandle = $self->{'outhandle'};
+    my $indexed_doc = 1;
+
+    # only output this document if it is one to be indexed
+    return if ($doc_obj->get_doc_type() ne "indexed_doc");
+
+    # see if this document belongs to this subcollection
+    foreach my $indexexp (@{$self->{'indexexparr'}}) {
+	$indexed_doc = 0;
+	my ($field, $exp, $options) = split /\//, $indexexp;
+	if (defined ($field) && defined ($exp)) {
+	    my ($bool) = $field =~ /^(.)/;
+	    $field =~ s/^.// if $bool eq '!';
+	    if ($field =~ /^filename$/i) {
+		$field = $doc_obj->get_source_filename();
+	    } else {
+		$field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
+	    }
+	    next unless defined $field;
+	    if ($bool eq '!') {
+		if ($options =~ /^i$/i) {
+		    if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
+		} else {
+		    if ($field !~ /$exp/) {$indexed_doc = 1; last;}
+		}
+	    } else {
+		if ($options =~ /^i$/i) {
+		    if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
+		} else {
+		    if ($field =~ /$exp/) {$indexed_doc = 1; last;}
+		}
+	    }
+	}
+    }
+
+    # if this doc is so far in the sub collection, and we have lang info, 
+    # now we check the languages to see if it matches
+    if($indexed_doc && defined $self->{'lang_meta'}) {
+	$indexed_doc = 0;
+	my $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'lang_meta'});
+	if (defined $field) {
+	    foreach my $lang (@{$self->{'langarr'}}) {
+		my ($bool) = $lang =~ /^(.)/;
+		if ($bool eq '!') {
+		    $lang =~ s/^.//;
+		    if ($field !~ /$lang/) {
+			$indexed_doc = 1; last;
+		    }
+		} else {
+		    if ($field =~ /$lang/) {
+			$indexed_doc = 1; last;
+		    }
+		}
+	    }
+	} 
+    }
+
+    # this is another document
+    $self->{'num_docs'} += 1;
+
+    # get the parameters for the output
+    # split on : just in case there is subcoll and lang stuff
+    my ($fields) = split (/:/, $self->{'index'});
+
+    my $doc_level = $mgppbuildproc::level_map{'document'};
+    my $gs2ns = 'xmlns:gs2="http://www.greenstone.org/gs2"';
+
+    my $levels = $self->{'levels'};
+    my $ldoc_level = $levels->{'document'};
+    my $lsec_level = $levels->{'section'};
+    my $lpar_level = $levels->{'paragraph'};
+
+    my $docid="";
+    if ($ldoc_level) {
+	if ($self->{'gdbm_level'} eq 'document') {
+	    my $doc_sec_num = $self->{'num_docs'};	
+	    $docid = "gs2:id=\"$doc_sec_num\"";
+	} else  {
+	    # default is section level
+	    my $doc_sec_num = $self->{'num_sections'}+1;	
+	    $docid = "gs2:id=\"$doc_sec_num\"";
+	}
+    }
+    my $documenttag = "<$doc_level $gs2ns file=\"$file\" $docid >\n";
+    my $documentendtag = "\n</$doc_level>\n";
+    
+    my ($sectiontag) = "";    
+    if ($lsec_level) {
+	$sectiontag = $mgppbuildproc::level_map{'section'};
+    }
+    my ($parastarttag) = "";
+    my ($paraendtag) = "";
+    if ($self->{'levels'}->{'paragraph'}) {
+	if ($self->{'strip_html'}) {
+	    $parastarttag = "<".$mgppbuildproc::level_map{'paragraph'}.">";
+	    $paraendtag = "</".$mgppbuildproc::level_map{'paragraph'}.">";
+	} else {
+	    print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
+	}
+    }
+    
+    my $doc_section = 0; # just for this document
+
+    my $text = "";
+    $text .= $documenttag;
+    # get the text for this document
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$doc_section++;
+	$self->{'num_sections'} += 1;
+ 
+	if ($sectiontag ne "") {
+	    my $secid = "gs2:id=\"".$self->{'num_sections'}."\"";	    
+	    $text .= "\n<$sectiontag $secid >\n";
+	}
+
+	# if we are doing subcollections, then some docs shouldn't be indexed.
+	# but we need to put the section tag placeholders in there so the 
+	# sections match up with gdbm db
+	if ($indexed_doc) {
+	    #if ($self->{'indexing_text'}) {
+	#	$text .= "$parastarttag"; # only add para tags for indexing
+		# note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text
+	   # }
+	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	    foreach my $field (split (/,/, $fields)) {
+		# only deal with this field if it doesn't start with top or
+		# this is the first section
+		my $real_field = $field;
+		if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
+		    my $new_text = ""; 
+		    my $tmp_text = "";
+		    if ($real_field eq "text") {
+			if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed
+			    $new_text .= "$parastarttag<TX index=\"1\">\n";
+			    $tmp_text .= $doc_obj->get_text ($section);
+			    if ($parastarttag =~ "") {
+				# we don't want to individually tag each paragraph if not doing para indexing
+				$tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "");
+			    } else {
+				$tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</TX>$paraendtag$parastarttag<TX index=\"1\">");
+			    }
+			    
+			    $new_text .= "$tmp_text</TX>$paraendtag\n";
+			    #if (!defined $self->{'indexfields'}->{'TextOnly'}) {
+				#$self->{'indexfields'}->{'TextOnly'} = 1;   
+			    #}
+			}
+			else { # leave html stuff in, but escape the tags, and dont add Paragraph tags - never retrieve paras at the moment
+			    if ($self->{'store_text'}) {
+				$tmp_text .= $doc_obj->get_text ($section);
+				&ghtml::htmlsafe($tmp_text);
+				$new_text .= $tmp_text;
+			    }
+			}
+		    } else { # metadata field
+			if ($real_field eq "allfields") { #ignore
+			}
+			elsif ($real_field eq "metadata") { # insert all metadata 
+			    #except gsdl stuff
+			    my $shortname = "";
+			    my $metadata = $doc_obj->get_all_metadata ($section);
+			    foreach $pair (@$metadata) {
+				my ($mfield, $mvalue) = (@$pair);
+				# check fields here, maybe others dont want - change to use dontindex!!
+				if ($mfield ne "Identifier" 
+				    && $mfield !~ /^gsdl/ 
+				    && $mfield ne "classifytype"
+				    && $mfield ne "assocfilepath"
+				    && defined $mvalue && $mvalue ne "") {
+				    
+				    if (defined $self->{'indexfieldmap'}->{$mfield}) {
+					$shortname = $self->{'indexfieldmap'}->{$mfield};
+				    }
+				    else {
+					$shortname = $self->create_shortname($mfield);
+					$self->{'indexfieldmap'}->{$mfield} = $shortname;
+					$self->{'indexfieldmap'}->{$shortname} = 1;
+				    }	   
+				    $new_text .= "$parastarttag<$shortname index=\"1\">$mvalue</$shortname>$paraendtag\n";
+				    if (!defined $self->{'indexfields'}->{$mfield}) {
+					$self->{'indexfields'}->{$mfield} = 1;
+				    }				    
+				}
+			    }
+			    
+			}
+			else { #individual metadata specified
+			    my $shortname="";
+			    #if (!defined $self->{'indexfields'}->{$real_field}) {
+				#$self->{'indexfields'}->{$real_field} = 1;
+			    #}
+			    if (defined $self->{'indexfieldmap'}->{$real_field}) {
+				$shortname = $self->{'indexfieldmap'}->{$real_field};
+			    }
+			    else {
+				$shortname = $self->create_shortname($real_field);
+				$self->{'indexfieldmap'}->{$real_field} = $shortname;
+				$self->{'indexfieldmap'}->{$shortname} = 1;
+			    }
+			    my @section_metadata = @{$doc_obj->get_metadata ($section, $real_field)};
+			    if ($self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
+				if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
+				    push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $real_field)});
+				}
+			    }
+			    foreach $item (@section_metadata) {
+				$new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n";
+				# remove entities
+				$new_text =~ s/&\w{1,10};//g; 
+			    }
+			}
+
+		    }
+		    
+		    # filter the text
+		    $self->filter_text ($field, $new_text);
+		    $self->{'num_processed_bytes'} += length ($new_text);
+		    $text .= "$new_text";
+		}
+	    }
+	} # if (indexed_doc)
+	
+	$text .= "\n</$sectiontag>\n" if ($sectiontag ne "");
+
+	$section = $doc_obj->get_next_section($section);
+    } #while defined section
+    print $handle "$text\n$documentendtag"; 
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/mgbuilder.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/mgbuilder.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/mgbuilder.pm	(revision 21112)
@@ -0,0 +1,515 @@
+###########################################################################
+#
+# mgbuilder.pm -- MGBuilder object
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package mgbuilder;
+
+use basebuilder;
+use classify;
+use cfgread;
+use colcfg;
+use plugin;
+use util;
+use FileHandle;
+
+BEGIN {
+    @mgbuilder::ISA = ('basebuilder');
+}
+
+
+my %wanted_index_files = ('td'=>1,
+		       't'=>1,
+		       'idb'=>1,
+		       'ib1'=>1,
+		       'ib2'=>1,
+		       'ib3'=>1,
+		       'i'=>1,
+		       'ip'=>1,
+		       'tiw'=>1,
+		       'wa'=>1);
+
+my $maxdocsize = $basebuilder::maxdocsize;
+
+
+sub new {
+    my $class = shift(@_);
+
+    my ($collection, $source_dir, $build_dir, $verbosity, 
+	$maxdocs, $debug, $keepold, $remove_empty_classifications, 
+	$outhandle, $no_text, $failhandle, $gli) = @_;
+
+    my $self = new basebuilder (@_);
+    $self = bless $self, $class;
+
+    $self->{'buildtype'} = "mg";
+    return $self;
+}
+
+sub default_buildproc {
+    my $self  = shift (@_);
+
+    return "mgbuildproc";
+}
+
+sub generate_index_list {
+    my $self = shift (@_);
+
+    if (!defined($self->{'collect_cfg'}->{'indexes'})) {
+	$self->{'collect_cfg'}->{'indexes'} = [];
+    }
+    if (scalar(@{$self->{'collect_cfg'}->{'indexes'}}) == 0) {
+	# no indexes have been specified so we'll build a "dummy:text" index
+	push (@{$self->{'collect_cfg'}->{'indexes'}}, "dummy:text");	
+    }
+
+}
+
+
+sub compress_text {
+    my $self = shift (@_);
+    my ($textindex) = @_;
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
+    my $mg_compression_dict_exe = &util::filename_cat($exedir, "mg_compression_dict$exe");
+    my $outhandle = $self->{'outhandle'};
+
+    my $maxnumeric = 4;
+    if (defined($self->{'collect_cfg'}->{'maxnumeric'}) &&
+	$self->{'collect_cfg'}->{'maxnumeric'} =~ /^\d+$/) {
+	$maxnumeric = $self->{'collect_cfg'}->{'maxnumeric'};
+    }
+
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
+    my $basefilename = "text/$self->{'collection'}";
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, $basefilename);
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fulltextprefix =~ s@/@\\@g;
+    } else {
+	$osextra = " -d /";
+    }
+
+    print $outhandle "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
+    print STDERR "<Stage name='CompressText'>\n" if $self->{'gli'};
+
+    # collect the statistics for the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print $outhandle "\n    collecting text statistics\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CollectTextStats'/>\n" if $self->{'gli'};
+
+    my ($handle);
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$mg_passes_exe" || 
+	    !open (PIPEOUT, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
+	}
+	$handle = mgbuilder::PIPEOUT;
+    }
+
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($textindex);
+    $self->{'buildproc'}->set_indexing_text (0);
+
+
+    if ($self->{'no_text'}) {
+	$self->{'buildproc'}->set_store_text(0);
+    } else {
+	$self->{'buildproc'}->set_store_text(1);
+    }
+    $self->{'buildproc'}->reset();
+
+    &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   $self->{'buildproc'}, $self->{'maxdocs'});
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    &plugin::end($self->{'pluginfo'});
+    
+
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    # create the compression dictionary
+    # the compression dictionary is built by assuming the stats are from a seed
+    # dictionary (-S), if a novel word is encountered it is spelled out (-H),
+    # and the resulting dictionary must be less than 5 meg with the most frequent
+    # words being put into the dictionary first (-2 -k 5120)
+    if (!$self->{'debug'}) {
+	print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'};
+	if (!-e "$mg_compression_dict_exe") {
+	    die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
+	}
+	system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
+
+	# -b $maxdocsize sets the maximum document size to be 12 meg
+	if (!-e "$mg_passes_exe" || 
+	    !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
+	}
+    }
+    else {
+	print STDERR "<Phase name='SkipCreatingComp'/>\n" if $self->{'gli'};
+    }
+
+    $self->{'buildproc'}->reset();
+    # compress the text
+    print $outhandle "\n    compressing the text\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CompressingText'/>\n" if $self->{'gli'};
+
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+
+# creates directory names for each of the index descriptions
+sub create_index_mapping {
+    my $self = shift (@_);
+    my ($indexes) = @_;
+
+    my %mapping = ();
+    $mapping{'indexmaporder'} = [];
+    $mapping{'subcollectionmaporder'} = [];
+    $mapping{'languagemaporder'} = [];
+    
+    # dirnames is used to check for collisions. Start this off
+    # with the manditory directory names
+    my %dirnames = ('text'=>'text',
+		    'extra'=>'extra');
+    my %pnames = ('index' => {}, 'subcollection' => {}, 'languages' => {});
+    foreach my $index (@$indexes) {
+	my ($level, $gran, $subcollection, $languages) = split (":", $index);
+
+	# the directory name starts with the first character of the index level
+	my ($pindex) = $level =~ /^(.)/;
+
+	# next comes a processed version of the index
+	$pindex .= $self->process_field ($gran); 
+	$pindex = lc ($pindex);
+
+	# next comes a processed version of the subcollection if there is one.
+	my $psub = $self->process_field ($subcollection);
+	$psub = lc ($psub);
+
+	# next comes a processed version of the language if there is one.
+	my $plang = $self->process_field ($languages);
+	$plang = lc ($plang);
+
+	my $dirname = $pindex . $psub . $plang;
+
+	# check to be sure all index names are unique
+	while (defined ($dirnames{$dirname})) {
+	    $dirname = $self->make_unique (\%pnames, $index, \$pindex, \$psub, \$plang);
+	}
+	$mapping{$index} = $dirname;
+
+	# store the mapping orders as well as the maps
+	# also put index, subcollection and language fields into the mapping thing - 
+	# (the full index name (eg document:text:subcol:lang) is not used on
+	# the query page) -these are used for collectionmeta later on
+	if (!defined $mapping{'indexmap'}{"$level:$gran"}) {
+	    $mapping{'indexmap'}{"$level:$gran"} = $pindex;
+	    push (@{$mapping{'indexmaporder'}}, "$level:$gran");
+	    if (!defined $mapping{"$level:$gran"}) {
+		$mapping{"$level:$gran"} = $pindex;
+	    }
+	}
+	if ($psub =~ /\w/ && !defined ($mapping{'subcollectionmap'}{$subcollection})) {
+	    $mapping{'subcollectionmap'}{$subcollection} = $psub;
+	    push (@{$mapping{'subcollectionmaporder'}}, $subcollection);
+	    $mapping{$subcollection} = $psub;
+	}
+	if ($plang =~ /\w/ && !defined ($mapping{'languagemap'}{$languages})) {
+	    $mapping{'languagemap'}{$languages} = $plang;
+	    push (@{$mapping{'languagemaporder'}}, $languages);
+	    $mapping{$languages} = $plang;
+	}
+	$dirnames{$dirname} = $index;
+	$pnames{'index'}->{$pindex} = "$level:$gran";
+	$pnames{'subcollection'}->{$psub} = $subcollection;
+	$pnames{'languages'}->{$plang} = $languages;
+    }
+
+    return \%mapping;
+}
+
+
+sub make_unique {
+    my $self = shift (@_);
+    my ($namehash, $index, $indexref, $subref, $langref) = @_;
+    my ($level, $gran, $subcollection, $languages) = split (":", $index);
+
+    if ($namehash->{'index'}->{$$indexref} ne "$level:$gran") {
+	$self->get_next_version ($indexref);
+    } elsif ($namehash->{'subcollection'}->{$$subref} ne $subcollection) {
+	$self->get_next_version ($subref);
+    } elsif ($namehash->{'languages'}->{$$langref} ne $languages) {
+	$self->get_next_version ($langref);
+    }
+    return "$$indexref$$subref$$langref";
+}	
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, $indexdir));
+    my $fullindexprefix = &util::filename_cat ($self->{'build_dir'}, $indexdir, 
+					       $self->{'collection'});
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, "text", 
+					       $self->{'collection'});
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
+    my $mg_perf_hash_build_exe = 
+	&util::filename_cat($exedir, "mg_perf_hash_build$exe");
+    my $mg_weights_build_exe = 
+	&util::filename_cat ($exedir, "mg_weights_build$exe");
+    my $mg_invf_dict_exe = 
+	&util::filename_cat ($exedir, "mg_invf_dict$exe");
+    my $mg_stem_idx_exe =
+	&util::filename_cat ($exedir, "mg_stem_idx$exe");
+
+    my $maxnumeric = 4;
+    if (defined($self->{'collect_cfg'}->{'maxnumeric'}) &&
+	$self->{'collect_cfg'}->{'maxnumeric'} =~ /^\d+$/) {
+	$maxnumeric = $self->{'collect_cfg'}->{'maxnumeric'};
+    }
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fullindexprefix =~ s@/@\\@g;
+    } else {
+	$osextra = " -d /";
+	if ($outhandle ne "STDERR") {
+	    # so mg_passes doesn't print to stderr if we redirect output
+	    $osextra .= " 2>/dev/null";
+	}
+    }
+
+    # get the index level from the index description
+    # the index will be level 2 unless we are building a
+    # paragraph level index
+    my $index_level = 2;
+    $index_level = 3 if $index =~ /^paragraph/i;
+
+    # get the index expression if this index belongs
+    # to a subcollection
+    my $indexexparr = [];
+    my $langarr = [];
+    # there may be subcollection info, and language info. 
+    my ($level, $fields, $subcollection, $language) = split (":", $index);
+    my @subcollections = ();
+    @subcollections = split /,/, $subcollection if (defined $subcollection);
+
+    foreach my $subcollection (@subcollections) {
+	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
+	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
+	} 
+    }
+    
+    # add expressions for languages if this index belongs to
+    # a language subcollection - only put languages expressions for the 
+    # ones we want in the index
+
+    my @languages = ();
+    my $language_metadata = "Language";
+    if (defined ($self->{'collect_cfg'}->{'language_metadata'})) {
+	$language_metadata = $self->{'collect_cfg'}->{'language_metadata'};
+    }
+    @languages = split /,/, $language if (defined $language);
+    foreach my $language (@languages) {
+	my $not=0;
+	if ($language =~ s/^\!//) {
+	    $not = 1;
+	}
+	if($not) {
+	    push (@$langarr, "!$language");
+	} else {
+	    push (@$langarr, "$language");
+	}
+    }
+    
+    # Build index dictionary. Uses verbatim stem method
+    print $outhandle "\n    creating index dictionary\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
+    my ($handle);
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$mg_passes_exe" || 
+	    !open (PIPEOUT, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
+		   "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
+	}
+	$handle = mgbuilder::PIPEOUT;
+    }
+	
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index, $indexexparr);
+    $self->{'buildproc'}->set_index_languages ($language_metadata, $langarr) if (defined $language);
+    $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
+
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    # now we check to see if the required files have been produced - if not we quit building this index so the whole process doesn't crap out.
+    # we check on the .id file - index dictionary
+    my $dict_file = "$fullindexprefix.id";
+    if (!-e $dict_file) {
+	print $outhandle "mgbuilder::build_index - Couldn't create index $index\n";
+	$self->{'notbuilt'}->{$index}=1;
+	return;
+    }
+    if (!$self->{'debug'}) {
+	# create the perfect hash function
+	if (!-e "$mg_perf_hash_build_exe") {
+	    print STDERR "<FatalError name='NoRunMGHash'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
+	}
+	system ("mg_perf_hash_build$exe -f \"$fullindexprefix\" $osextra");
+
+	if (!-e "$mg_passes_exe" || 
+	    !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
+		   "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
+	}
+    }
+    
+    # invert the text
+    print $outhandle "\n    inverting the text\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='InvertingText'/>\n" if $self->{'gli'};
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
+
+   
+    $self->print_stats ();
+
+    if (!$self->{'debug'}) {
+
+	close ($handle);
+	
+	# create the weights file
+	print $outhandle "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreateTheWeights'/>\n" if $self->{'gli'};
+	if (!-e "$mg_weights_build_exe") {
+	    print STDERR "<FatalError name='NoRunMGWeights'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
+	}
+	system ("mg_weights_build$exe -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
+
+	# create 'on-disk' stemmed dictionary
+	print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreateStemmedDic'/>\n" if $self->{'gli'};
+	if (!-e "$mg_invf_dict_exe") {
+	    print STDERR "<FatalError name='NoRunMGInvf'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
+	}
+	system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");
+
+
+	# creates stem index files for the various stemming methods
+	print $outhandle "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreatingStemIndx'/>\n" if $self->{'gli'};
+	if (!-e "$mg_stem_idx_exe") {
+	    print STDERR "<FatalError name='NoRunMGStem'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
+	}
+	system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+	system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+	system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
+    
+	# remove unwanted files
+	my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
+	opendir (DIR, $tmpdir) || die
+	    "mgbuilder::build_index - couldn't read directory $tmpdir\n";
+	foreach my $file (readdir(DIR)) {
+	    next if $file =~ /^\./;
+	    my ($suffix) = $file =~ /\.([^\.]+)$/;
+	    if (defined $suffix && !defined $wanted_index_files{$suffix}) {
+		# delete it!
+		print $outhandle "deleting $file\n" if $self->{'verbosity'} > 2;
+		&util::rm (&util::filename_cat ($tmpdir, $file));
+	    }
+	}
+	closedir (DIR);
+    }
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+sub build_cfg_extra {
+   my $self = shift(@_);
+   my ($build_cfg) = @_;
+   
+    # get additional stats from mg
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mgstat_exe = &util::filename_cat($exedir, "mgstat$exe");
+    my $input_file = &util::filename_cat ("text", $self->{'collection'});
+    if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
+	print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
+    } else {
+	my $line = "";
+	while (defined ($line = <PIPEIN>)) {
+	    if ($line =~ /^Words in collection \[dict\]\s+:\s+(\d+)/) {
+		($build_cfg->{'numwords'}) = $1;
+	    } elsif ($line =~ /^Documents\s+:\s+(\d+)/) {
+		($build_cfg->{'numsections'}) = $1;
+	    }
+	}
+	close PIPEIN;
+    }
+}
+
+1;
+
+
+
Index: /main/tags/3.01/gsdl/perllib/mgbuildproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/mgbuildproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/mgbuildproc.pm	(revision 21112)
@@ -0,0 +1,147 @@
+###########################################################################
+#
+# mgbuildproc.pm -- 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor outputs a document
+# for mg to process
+
+
+package mgbuildproc;
+
+use basebuildproc;
+
+BEGIN {
+    @mgbuildproc::ISA = ('basebuildproc');
+}
+
+sub new {
+    my $class = shift @_;
+    my $self = new basebuildproc (@_);
+    return bless $self, $class;
+}
+
+
+sub find_paragraphs {
+    $_[1] =~ s/(<p\b)/\cC$1/gi;
+}
+
+sub filter_text {
+    # $self->filter_text ($field, $new_text);
+    # don't want to do anything for this version, however,
+    # in a particular collection you might want to override
+    # this method to post-process certain fields depending on
+    # the field, or whether we are outputting it for indexing
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $handle = $self->{'output_handle'};
+    
+    # only output this document if it is one to be indexed
+    return if ($doc_obj->get_doc_type() ne "indexed_doc");
+    
+    # see if this document belongs to this subcollection
+    my $indexed_doc = $self->is_subcollection_doc($doc_obj);
+
+    # this is another document
+    $self->{'num_docs'} += 1;
+
+    # get the parameters for the output
+    my ($level, $fields) = split (/:/, $self->{'index'});
+    $fields =~ s/\ball\b/Title,Creator,text/;
+    $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;
+
+    my $doc_section = 0; # just for this document
+    my $text = "";
+    my $text_extra = "";
+
+    # get the text for this document
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$doc_section++;
+	$self->{'num_sections'} += 1;
+
+	if ($indexed_doc) {
+	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	    foreach my $field (split (/,/, $fields)) {
+		# only deal with this field if it doesn't start with top or
+		# this is the first section
+		my $real_field = $field;
+		if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
+		    my $new_text = "";
+		    if ($level eq "dummy") {
+			# a dummy index is a special case used when no
+			# indexes are specified (since there must always be
+			# at least one index or we can't retrieve the
+			# compressed text) - we add a small amount of text
+			# to these dummy indexes which will never be seen
+			# but will overcome mg's problems with building
+			# empty indexes
+			$new_text = "this is dummy text to stop mg barfing";
+			$self->{'num_processed_bytes'} += length ($new_text);
+
+		    } elsif ($real_field eq "text") {
+			$new_text = $doc_obj->get_text ($section) if $self->{'store_text'};
+			$self->{'num_processed_bytes'} += length ($new_text);
+			$new_text =~ s/[\cB\cC]//g;
+			$self->find_paragraphs($new_text);
+			
+		    } else {
+			my $first = 1;
+			my @section_metadata = @{$doc_obj->get_metadata ($section, $real_field)};
+			if ($level eq "section" && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
+			    if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
+				push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $real_field)});
+			    }
+			}
+			foreach my $meta (@section_metadata) {
+			    $meta =~ s/[\cB\cC]//g;
+			    $self->{'num_processed_bytes'} += length ($meta);
+			    $new_text .= "\cC" unless $first;
+			    $new_text .= $meta if $self->{'store_text'};
+			    $first = 0;
+			}
+		    }
+		    
+		    # filter the text
+		    $self->filter_text ($field, $new_text);
+
+		    $text .= "$new_text\cC";
+		}
+	    }
+	}
+	
+	if ($level eq "document") { $text_extra .= "\cB"; }
+	else { $text .= "\cB"; }
+	
+	$section = $doc_obj->get_next_section($section);
+    }
+
+    print $handle "$text$text_extra";
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/mgppbuilder.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/mgppbuilder.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/mgppbuilder.pm	(revision 21112)
@@ -0,0 +1,887 @@
+###########################################################################
+#
+# mgppbuilder.pm -- MGBuilder object
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package mgppbuilder;
+
+use basebuilder;
+use classify;
+use cfgread;
+use colcfg;
+use plugin;
+use util;
+use FileHandle;
+
+sub BEGIN {
+    @mgppbuilder::ISA = ('basebuilder');
+}
+
+
+
+our %level_map = ('document'=>'Doc',
+	      'section'=>'Sec',
+	      'paragraph'=>'Para',
+	      'Doc'=>'_textdocument_',
+	      'Sec'=>'_textsection_',
+	      'Para'=>'_textparagraph_');
+
+#$doc_level = "Doc";
+#$sec_level = "Sec";
+#$para_level = "Para";
+
+our %wanted_index_files = ('td'=>1,
+		       't'=>1,
+		       'tl'=>1,
+		       'ti'=>1,
+		       'idb'=>1,
+		       'ib1'=>1,
+		       'ib2'=>1,
+		       'ib3'=>1,
+		       'i'=>1,
+		       'il'=>1,
+		       'w'=>1,
+		       'wa'=>1);
+
+# change this so a user can add their own ones in via a file or cfg
+#add AND, OR, NOT NEAR to this list - these cannot be used as field names
+#also add the level names (Doc, Sec, Para)
+our %static_indexfield_map = ('Title'=>'TI',
+			  'TI'=>1,
+			  'Subject'=>'SU',
+			  'SU'=>1,
+			  'Creator'=>'CR',
+			  'CR'=>1,
+			  'Organization'=>'ORG',
+			  'ORG'=>1,
+			  'Source'=>'SO',
+			  'SO'=>1,
+			  'Howto'=>'HT',
+			  'HT'=>1,
+			  'ItemTitle'=>'IT',
+			  'IT'=>1,
+			  'ProgNumber'=>'PN',
+			  'PN'=>1,
+			  'People'=>'PE',
+			  'PE'=>1,
+			  'Coverage'=>'CO',
+			  'CO'=>1,
+			  'allfields'=>'ZZ',
+			  'ZZ'=>1,
+			  'text'=>'TX',
+			  'TX'=>1,
+			  'AND'=>1,
+			  'OR'=>1,
+			  'NOT'=>1,
+			  'NEAR'=>1,
+			  'Doc'=>1,
+			  'Sec'=>1,
+			  'Para'=>1);
+
+my $maxdocsize = $basebuilder::maxdocsize;
+
+sub new {
+    my $class = shift(@_);
+
+    my ($collection, $source_dir, $build_dir, $verbosity, 
+	$maxdocs, $debug, $keepold, $remove_empty_classifications, 
+	$outhandle, $no_text, $failhandle, $gli) = @_;
+
+    my $self = new basebuilder (@_);
+    $self = bless $self, $class;
+
+    $self->{'indexfieldmap'} = \%static_indexfield_map;
+
+    # get the levels (Section, Paragraph) for indexing and compression
+    $self->{'levels'} = {};
+    $self->{'levelorder'} = ();
+    if (defined $self->{'collect_cfg'}->{'levels'}) {
+        foreach my $level ( @{$self->{'collect_cfg'}->{'levels'}} ){
+	    $level =~ tr/A-Z/a-z/;
+            $self->{'levels'}->{$level} = 1;
+	    push (@{$self->{'levelorder'}}, $level);
+        }
+    } else { # default to document
+	$self->{'levels'}->{'document'} = 1;
+	push (@{$self->{'levelorder'}}, 'document');
+    }
+    
+    $self->{'doc_level'} = "document";
+    if (! $self->{'levels'}->{'document'}) {
+	if ($self->{'levels'}->{'section'}) {
+	    $self->{'doc_level'} = "section";
+	} else {
+	    die "you must have either document or section level specified!!\n";
+	}
+    }
+
+    $self->{'buildtype'} = "mgpp";
+
+    return $self;
+}
+
+sub generate_index_list {
+    my $self  = shift (@_);
+    
+    # sort out the indexes
+    #indexes are specified with spaces, but we put them into one index
+    my $indexes = $self->{'collect_cfg'}->{'indexes'};
+    $self->{'collect_cfg'}->{'indexes'} = [];
+    push (@{$self->{'collect_cfg'}->{'indexes'}}, join(',', @$indexes));
+}
+
+sub default_buildproc {
+    my $self  = shift (@_);
+
+    return "mgppbuildproc";
+}
+
+sub compress_text {
+
+    my $self = shift (@_);
+    my ($textindex) = @_;
+
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
+    my $mgpp_compression_dict_exe = &util::filename_cat($exedir, "mgpp_compression_dict$exe");
+    my $outhandle = $self->{'outhandle'};
+
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
+
+    my $basefilename = "text/$self->{'collection'}";
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, $basefilename);
+    
+   my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fulltextprefix =~ s@/@\\@g;
+    } 
+    else {
+	$osextra = " -d /";
+    }
+
+
+    # define the section names and possibly the doc name for mgpasses
+    # the compressor doesn't need to know about paragraphs - never want to 
+    # retrieve them
+    my $mgpp_passes_sections = "";
+    my ($doc_level) = $self->{'doc_level'};
+    $mgpp_passes_sections .= "-J " . $level_map{$doc_level} . " ";
+    foreach my $level (keys %{$self->{'levels'}}) {
+	if ($level ne $doc_level && $level ne "paragraph") {
+	    $mgpp_passes_sections .= "-K " . $level_map{$level} . " ";
+	}
+    }
+    $mgpp_passes_sections .= "-K SENT ";
+    print $outhandle "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
+    print STDERR "<Stage name='CompressText'>\n" if $self->{'gli'};
+
+    # collect the statistics for the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print $outhandle "\n    collecting text statistics (mgpp_passes -T1)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CollectTextStats'/>\n" if $self->{'gli'};
+
+    my ($handle);
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	#print $outhandle "trying to run (compress 1) mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra\n";
+	if (!-e "$mgpp_passes_exe" || 
+	    !open (PIPEOUT, "| mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
+	}
+	$handle = mgppbuilder::PIPEOUT;
+    }
+
+    # gdbm_level
+    my $gdbm_level = "document";
+    if ($self->{'levels'}->{'section'}) {
+	$gdbm_level = "section";
+    }
+    
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($textindex);
+    $self->{'buildproc'}->set_indexing_text (0);
+    if ($self->{'no_text'}) {
+	$self->{'buildproc'}->set_store_text(0);
+    } else {
+	$self->{'buildproc'}->set_store_text(1);
+    }
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($self->{'levels'});                      
+    $self->{'buildproc'}->set_gdbm_level ($gdbm_level);                       
+    $self->{'buildproc'}->reset();
+    &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   $self->{'buildproc'}, $self->{'maxdocs'});
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    &plugin::end($self->{'pluginfo'});
+    close (PIPEOUT);
+
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    # create the compression dictionary
+    # the compression dictionary is built by assuming the stats are from a seed
+    # dictionary (-S), if a novel word is encountered it is spelled out (-H),
+    # and the resulting dictionary must be less than 5 meg with the most 
+    # frequent words being put into the dictionary first (-2 -k 5120)
+    # note: these options are left over from mg version
+    if (!$self->{'debug'}) {
+	print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'};
+	if (!-e "$mgpp_compression_dict_exe") {
+	    print STDERR "<FatalError name='NoRunMGCompress'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::compress_text - couldn't run $mgpp_compression_dict_exe\n";
+	}
+	system ("mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
+
+	if (!$self->{'debug'}) {
+	    #print $outhandle "trying to run (compress 2) mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra\n";
+	    if (!-e "$mgpp_passes_exe" || 
+		!open ($handle, "| mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra")) {
+		print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+		die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
+	    }
+	}
+    }
+    else {
+	print STDERR "<Phase name='SkipCreatingComp'/>\n" if $self->{'gli'};
+    }
+
+    $self->{'buildproc'}->reset();
+    # compress the text
+    print $outhandle "\n    compressing the text (mgpp_passes -T2)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CompressingText'/>\n" if $self->{'gli'};
+
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+
+sub build_indexes_extra {
+    my $self = shift(@_);
+    #define the final field lists
+    $self->make_final_field_list();
+}    
+
+# creates directory names for each of the index descriptions
+sub create_index_mapping {
+    my $self = shift (@_);
+    my ($indexes) = @_;
+
+    my %mapping = ();
+
+    $mapping{'indexmaporder'} = [];
+    $mapping{'subcollectionmaporder'} = [];
+    $mapping{'languagemaporder'} = [];
+    
+    # dirnames is used to check for collisions. Start this off
+    # with the manditory directory names
+    my %dirnames = ('text'=>'text',
+		    'extra'=>'extra');
+    my %pnames = ('index' => {}, 'subcollection' => {}, 'languages' => {});
+
+    foreach my $index (@$indexes) {
+	my ($fields, $subcollection, $languages) = split (":", $index);
+	# the directory name starts with a processed version of index fields
+	#my ($pindex) = $self->process_field($fields);
+	#$pindex = lc ($pindex);
+	# now we only ever have one index, and its called 'idx'
+	my $pindex = 'idx';
+	
+	# next comes a processed version of the subcollection if there is one.
+	my $psub = $self->process_field ($subcollection);
+	$psub = lc ($psub);
+
+	# next comes a processed version of the language if there is one.
+	my $plang = $self->process_field ($languages);
+	$plang = lc ($plang);
+
+	my $dirname = $pindex . $psub . $plang;
+
+	# check to be sure all index names are unique
+	while (defined ($dirnames{$dirname})) {
+	    $dirname = $self->make_unique (\%pnames, $index, \$pindex, \$psub, \$plang);
+	}
+
+	$mapping{$index} = $dirname;
+
+	# store the mapping orders as well as the maps
+	# also put index, subcollection and language fields into the mapping thing - 
+	# (the full index name (eg text:subcol:lang) is not used on
+	# the query page) -these are used for collectionmeta later on
+	if (!defined $mapping{'indexmap'}{"$fields"}) {
+	    $mapping{'indexmap'}{"$fields"} = $pindex;
+	    push (@{$mapping{'indexmaporder'}}, "$fields");
+	    if (!defined $mapping{"$fields"}) {
+		$mapping{"$fields"} = $pindex;
+	    }	
+	}
+	if ($psub =~ /\w/ && !defined ($mapping{'subcollectionmap'}{$subcollection})) {
+	    $mapping{'subcollectionmap'}{$subcollection} = $psub;
+	    push (@{$mapping{'subcollectionmaporder'}}, $subcollection);
+	    $mapping{$subcollection} = $psub;
+	}
+	if ($plang =~ /\w/ && !defined ($mapping{'languagemap'}{$languages})) {
+	    $mapping{'languagemap'}{$languages} = $plang;
+	    push (@{$mapping{'languagemaporder'}}, $languages);
+	    $mapping{$languages} = $plang;
+	}
+	$dirnames{$dirname} = $index;
+	$pnames{'index'}->{$pindex} = "$fields";
+	$pnames{'subcollection'}->{$psub} = $subcollection;
+	$pnames{'languages'}->{$plang} = $languages;
+    }
+
+    return \%mapping;
+}
+
+sub make_unique {
+    my $self = shift (@_);
+    my ($namehash, $index, $indexref, $subref, $langref) = @_;
+    my ($fields, $subcollection, $languages) = split (":", $index);
+
+    if ($namehash->{'index'}->{$$indexref} ne "$fields") {
+	$self->get_next_version ($indexref);
+    } elsif ($namehash->{'subcollection'}->{$$subref} ne $subcollection) {
+	$self->get_next_version ($subref);
+    } elsif ($namehash->{'languages'}->{$$langref} ne $languages) {
+	$self->get_next_version ($langref);
+    }
+    return "$$indexref$$subref$$langref";
+}	
+
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, $indexdir));
+    my $fullindexprefix = &util::filename_cat ($self->{'build_dir'}, 
+					       $indexdir, 
+					       $self->{'collection'});
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, "text", 
+					       $self->{'collection'});
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+
+    my $exe = &util::get_os_exe ();
+    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
+
+    # define the section names for mgpasses
+    # define the section names and possibly the doc name for mgpasses
+    my $mgpp_passes_sections = "";
+    my ($doc_level) = $self->{'doc_level'};
+    $mgpp_passes_sections .= "-J " . $level_map{$doc_level} ." ";
+    
+    foreach my $level (keys %{$self->{'levels'}}) {
+	if ($level ne $doc_level) {
+	    $mgpp_passes_sections .= "-K " . $level_map{$level}. " ";
+	}
+    }
+
+    my $mgpp_perf_hash_build_exe = 
+	&util::filename_cat($exedir, "mgpp_perf_hash_build$exe");
+    my $mgpp_weights_build_exe = 
+	&util::filename_cat ($exedir, "mgpp_weights_build$exe");
+    my $mgpp_invf_dict_exe = 
+	&util::filename_cat ($exedir, "mgpp_invf_dict$exe");
+    my $mgpp_stem_idx_exe =
+	&util::filename_cat ($exedir, "mgpp_stem_idx$exe");
+
+     my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fullindexprefix =~ s@/@\\@g;
+    } else {
+	$osextra = " -d /";
+	if ($outhandle ne "STDERR") {
+	    # so mgpp_passes doesn't print to stderr if we redirect output
+	    $osextra .= " 2>/dev/null";
+	}
+    }
+ 
+    # get the index expression if this index belongs
+    # to a subcollection
+    my $indexexparr = [];
+    my $langarr = [];
+    # there may be subcollection info, and language info. 
+    my ($fields, $subcollection, $language) = split (":", $index);
+    my @subcollections = ();
+    @subcollections = split /,/, $subcollection if (defined $subcollection);
+
+    foreach $subcollection (@subcollections) {
+	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
+	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
+	} 
+    }
+    
+    # add expressions for languages if this index belongs to
+    # a language subcollection - only put languages expressions for the 
+    # ones we want in the index
+    
+    my @languages = ();
+    my $language_metadata = "Language";
+    if (defined ($self->{'collect_cfg'}->{'language_metadata'})) {
+	$language_metadata = $self->{'collect_cfg'}->{'language_metadata'};
+    }
+    @languages = split /,/, $language if (defined $language);
+    foreach my $language (@languages) {
+	my $not=0;
+	if ($language =~ s/^\!//) {
+	    $not = 1;
+	}
+	if($not) {
+	    push (@$langarr, "!$language");
+	} else {
+	    push (@$langarr, "$language");
+	}
+    }
+
+    # Build index dictionary. Uses verbatim stem method
+    print $outhandle "\n    creating index dictionary (mgpp_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
+    my ($handle);
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	if (!-e "$mgpp_passes_exe" || 
+	    !open (PIPEOUT, "| mgpp_passes$exe $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
+	}
+	$handle = mgppbuilder::PIPEOUT;
+    }
+	    
+    # gdbm_level
+    my $gdbm_level = "document";
+    if ($self->{'levels'}->{'section'}) {
+	$gdbm_level = "section";
+    }
+
+    # set up the document processr
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index, $indexexparr);
+    $self->{'buildproc'}->set_index_languages ($language_metadata, $langarr) if (defined $language);
+    $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($self->{'levels'}); 
+    $self->{'buildproc'}->set_gdbm_level ($gdbm_level);   
+    
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    # now we check to see if the required files have been produced - if not we quit building this index so the whole process doesn't crap out.
+    # we check on the .id file - index dictionary
+    my $dict_file = "$fullindexprefix.id";
+    if (!-e $dict_file) {
+	print $outhandle "mgppbuilder::build_index - Couldn't create index $index\n";
+	print STDERR "<Warning name='NoIndex'/>\n</Stage>\n" if $self->{'gli'};
+	$self->{'notbuilt'}->{$index}=1;
+	return;
+    }
+
+    if (!$self->{'debug'}) {
+	# create the perfect hash function
+	if (!-e "$mgpp_perf_hash_build_exe") {
+	    print STDERR "<FatalError name='NoRunMGHash'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_perf_hash_build_exe\n";
+	}
+	system ("mgpp_perf_hash_build$exe -f \"$fullindexprefix\" $osextra");
+
+	if (!-e "$mgpp_passes_exe" || 
+	    !open ($handle, "| mgpp_passes$exe $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra")) {
+	    print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
+	}
+    }
+    
+    # invert the text
+    print $outhandle "\n    inverting the text (mgpp_passes -I2)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='InvertingText'/>\n" if $self->{'gli'};
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+
+    $self->print_stats ();
+    
+    if (!$self->{'debug'}) {
+
+	close ($handle);
+	
+	# create the weights file
+	print $outhandle "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreateTheWeights'/>\n" if $self->{'gli'};
+	if (!-e "$mgpp_weights_build_exe") {
+	    print STDERR "<FatalError name='NoRunMGWeights'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_weights_build_exe\n";
+	}
+	system ("mgpp_weights_build$exe -f \"$fullindexprefix\" $osextra");
+
+	# create 'on-disk' stemmed dictionary
+	print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
+	if (!-e "$mgpp_invf_dict_exe") {
+	    print STDERR "<FatalError name='NoRunMGInvf'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_invf_dict_exe\n";
+	}
+	system ("mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra" );
+
+
+	# creates stem index files for the various stemming methods
+	print $outhandle "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
+	print STDERR "<Phase name='CreatingStemIndx'/>\n" if $self->{'gli'};
+	if (!-e "$mgpp_stem_idx_exe") {
+	    print STDERR "<FatalError name='NoRunMGStem'/>\n</Stage>\n" if $self->{'gli'};
+	    die "mgppbuilder::build_index - couldn't run $mgpp_stem_idx_exe\n";
+	}
+	system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+	system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+	system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
+	
+	# remove unwanted files
+	my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
+	opendir (DIR, $tmpdir) || die
+	    "mgppbuilder::build_index - couldn't read directory $tmpdir\n";
+	foreach my $file (readdir(DIR)) {
+	    next if $file =~ /^\./;
+	    my ($suffix) = $file =~ /\.([^\.]+)$/;
+	    if (defined $suffix && !defined $wanted_index_files{$suffix}) {
+		# delete it!
+		print $outhandle "deleting $file\n" if $self->{'verbosity'} > 2;
+		#&util::rm (&util::filename_cat ($tmpdir, $file));
+	    }
+	}
+	closedir (DIR);
+    }
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}   
+
+
+sub output_collection_meta {
+    my $self = shift(@_);
+    my ($handle) = @_;
+
+    # define the indexed field mapping if not already done so (ie if infodb called separately from build_index)
+    if (!defined $self->{'build_cfg'}) {
+	$self->read_final_field_list();
+    }
+
+    # do the collection info
+    print $handle "[collection]\n";
+    
+    # first do the collection meta stuff - everything without a dot
+    my $collmetadefined = 0;
+    my $metadata_entry;
+    if (defined $self->{'collect_cfg'}->{'collectionmeta'}) {
+	$collmetadefined = 1;
+	foreach my $cmeta (keys (%{$self->{'collect_cfg'}->{'collectionmeta'}})) {
+	    next if ($cmeta =~ /^\./); # for now, ignore ones with dots
+	    $metadata_entry = $self->create_language_db_map($cmeta, $cmeta);
+	    #write the entry to the file
+	    print $handle $metadata_entry;
+	    
+	} # foreach collmeta key
+    }
+    #add the index field macros to [collection]
+    # eg <TI>Title
+    #    <SU>Subject
+    # these now come from collection meta. if that is not defined, usses the metadata name
+    my $field_entry="";
+    my $collmeta = "";
+    foreach my $longfield (@{$self->{'build_cfg'}->{'indexfields'}}){
+	my $shortfield = $self->{'buildproc'}->{'indexfieldmap'}->{$longfield};
+	next if $shortfield eq 1;
+	
+	# we need to check if some coll meta has been defined
+	$collmeta = ".$longfield";
+	if ($collmetadefined && defined $self->{'collect_cfg'}->{'collectionmeta'}->{$collmeta}) {
+	    $metadata_entry = $self->create_language_db_map($collmeta, $shortfield);
+	    $field_entry .= $metadata_entry;
+	} else { #use the metadata names, or the text macros for allfields and textonly
+	    if ($longfield eq "allfields") {
+		$field_entry .= "<$shortfield>_query:textallfields_\n";
+	    } elsif ($longfield eq "text") {
+		$field_entry .= "<$shortfield>_query:texttextonly_\n";
+	    } else {
+		$field_entry .= "<$shortfield>$longfield\n";
+	    }
+	}
+    }
+    print $handle $field_entry;
+    
+    # now add the level names
+    my $level_entry = "";
+    foreach my $level (@{$self->{'collect_cfg'}->{'levels'}}) {
+	$collmeta = ".$level"; # based on the original specification
+	$level =~ tr/A-Z/a-z/; # make it lower case
+	my $levelid = $level_map{$level}; # find the actual value we used in the index
+	if ($collmetadefined && defined $self->{'collect_cfg'}->{'collectionmeta'}->{$collmeta}) {
+	    $metadata_entry = $self->create_language_db_map($collmeta, $levelid);
+	    $level_entry .= $metadata_entry;
+	} else {
+	    # use the default macro
+	    $level_entry .= "<$levelid>" . $level_map{$levelid} . "\n";
+	}
+    }
+    print $handle $level_entry;
+    
+    # now add subcoll meta
+    my $subcoll_entry = "";
+    my $shortname = "";
+    my $one_entry = "";
+    foreach my $subcoll (@{$self->{'index_mapping'}->{'subcollectionmaporder'}}) {
+	if (defined $self->{'collect_cfg'}->{'collectionmeta'}->{".$subcoll"}) {
+	    $shortname = $self->{'index_mapping'}->{$subcoll};
+	    $one_entry = $self->create_language_db_map(".$subcoll", $shortname);
+	    $subcoll_entry .= $one_entry;
+	} else {
+	    $subcoll_entry .= "<$shortname>$subcoll\n";
+	}
+    }
+    print $handle $subcoll_entry;
+
+    # now add language meta
+    my $lang_entry = "";
+    foreach my $lang (@{$self->{'index_mapping'}->{'languagemaporder'}}) {
+	if (defined $self->{'collect_cfg'}->{'collectionmeta'}->{".$lang"}) {
+	    $shortname = $self->{'index_mapping'}->{$lang};
+	    $one_entry = $self->create_language_db_map(".$lang", $shortname);
+	    $lang_entry .= $one_entry;
+	} else {
+	    $lang_entry .= "<$shortname>$lang\n";
+	}
+    }
+    print $handle $lang_entry;
+    # end the collection entry
+    print $handle "\n" . ('-' x 70) . "\n";        
+
+
+}
+sub create_language_db_map {
+    my $self = shift (@_);
+    my ($metaname, $mapname) = @_;
+    my $outhandle =  $self->{'outhandle'};
+    my $defaultfound=0;
+    my $first=1;
+    my $metadata_entry = "";
+    my $default="";
+    #iterate through the languages
+    foreach my $lang (keys (%{$self->{'collect_cfg'}->{'collectionmeta'}->{$metaname}})) {
+	if ($first) {
+	    $first=0;
+	    #set the default default to the first entry
+	    $default=$self->{'collect_cfg'}->{'collectionmeta'}->{$metaname}->{$lang};
+	}
+	if ($lang =~ /default/) {
+	    $defaultfound=1;
+	    #the default entry goes first
+	    $metadata_entry = "<$mapname>" .
+		$self->{'collect_cfg'}->{'collectionmeta'}->{$metaname}->{'default'} . "\n" . $metadata_entry;
+	}
+	else {
+	    my ($l) = $lang =~ /^\[l=(\w*)\]$/;
+	    if ($l) {
+		$metadata_entry .= "<$mapname:$l>" . 
+		    $self->{'collect_cfg'}->{'collectionmeta'}->{$metaname}->{$lang} . "\n";
+
+		# Use the English value as the default if no default is specified
+		if ($l =~ /en/i) {
+		    $default=$self->{'collect_cfg'}->{'collectionmeta'}->{$metaname}->{$lang};
+		}
+	    }
+	}
+    } #foreach lang
+    #if we haven't found a default, put one in
+    if (!$defaultfound) {
+	$metadata_entry = "<$mapname>$default\n" . $metadata_entry;
+    }
+    return $metadata_entry;
+    
+}
+
+# at the end of building, we have an indexfieldmap with all the mappings,
+# plus some extras, and indexmap with any indexes in it that weren't
+# specified in the index definition.  we want to make an ordered list of
+# fields that are indexed, and a list of mappings that are used. this will
+# be used for the build.cfg file, and for collection meta definition we
+# store these in a build.cfg bit
+sub make_final_field_list {
+    my $self = shift (@_);
+    
+    $self->{'build_cfg'} = {};
+    
+    # store the indexfieldmap information
+    my @indexfieldmap = ();
+    my @indexfields = ();
+    my $specifiedfields = {};
+    my @specifiedfieldorder = ();
+
+    # go through the index definition and add each thing to a map, so we
+    # can easily check if it is already specified - when doing the
+    # metadata, we print out all the individual fields, but some may
+    # already be specified in the index definition, so we dont want to add
+    # those again.
+
+    my $field;
+    foreach $field (@{$self->{'collect_cfg'}->{'indexes'}}) {
+	# remove subcoll stuff
+	my $parts = $field;
+	$parts =~ s/:.*$//;
+	my @fs = split(',', $parts);
+	foreach my $f(@fs) {
+	    if (!defined $specifiedfields->{$f}) {
+		$specifiedfields->{$f}=1;
+		push (@specifiedfieldorder, "$f");
+	    }
+	}
+    }
+    
+    #add all fields bit 
+    foreach $field (@specifiedfieldorder) {
+	if ($field eq "metadata") {
+	    foreach my $newfield (keys %{$self->{'buildproc'}->{'indexfields'}}) {
+		if (!defined $specifiedfields->{$newfield}) {
+		    push (@indexfieldmap, "$newfield\-\>$self->{'buildproc'}->{'indexfieldmap'}->{$newfield}");
+		    push (@indexfields, "$newfield");
+		}
+	    }
+
+	} elsif ($field eq 'text') {
+	    push (@indexfieldmap, "text\-\>TX");
+	    push (@indexfields, "text");
+	} elsif ($field eq 'allfields') {
+	    push (@indexfieldmap, "allfields\-\>ZZ");
+	    push (@indexfields, "allfields");
+	} else {
+	    push (@indexfieldmap, "$field\-\>$self->{'buildproc'}->{'indexfieldmap'}->{$field}");
+	    push (@indexfields, "$field");
+	    
+	}
+    }
+
+    $self->{'build_cfg'}->{'indexfieldmap'} = \@indexfieldmap;
+    $self->{'build_cfg'}->{'indexfields'} = \@indexfields;
+}
+
+
+# recreate the field list from the build.cfg file, look first in building,
+# then in index to find it. if there is no build.cfg, we can't do the field
+# list (there is unlikely to be any index anyway.)
+sub read_final_field_list {
+    my $self = shift (@_);
+    $self->{'build_cfg'} = {};
+    my @indexfieldmap = ();
+    my @indexfields = ();
+    
+    if (scalar(keys %{$self->{'buildproc'}->{'indexfieldmap'}}) == 0) {
+	# set the default mapping
+	$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    }
+    # we read the stuff in from the build.cfg file - if its there
+    my $buildconfigfile = &util::filename_cat($self->{'build_dir'}, "build.cfg");
+    
+    if (!-e $buildconfigfile) {
+	# try the index dir - but do we know where it is?? try here
+	$buildconfigfile  = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "index", "build.cfg");
+	if (!-e $buildconfigfile) {
+	    #we cant find a config file - just ignore the field list
+	    return;
+	}
+    } 
+
+    my $buildcfg = &colcfg::read_build_cfg( $buildconfigfile);
+    my $field;
+    if (defined $buildcfg->{'indexfields'}) {
+	foreach $field (@{$buildcfg->{'indexfields'}}) {
+	    push (@indexfields, "$field");
+	}
+    }
+
+    if (defined $buildcfg->{'indexfieldmap'}) {
+	foreach $field (@{$buildcfg->{'indexfieldmap'}}) {
+	    push (@indexfieldmap, "$field");
+	    my ($f, $v) = $field =~ /^(.*)\-\>(.*)$/;
+	    $self->{'buildproc'}->{'indexfieldmap'}->{$f} = $v;
+	}
+    }	    
+
+    $self->{'build_cfg'}->{'indexfieldmap'} = \@indexfieldmap;
+    $self->{'build_cfg'}->{'indexfields'} = \@indexfields;
+}
+
+
+sub write_cfg_file {
+    my $self = shift(@_);
+    my ($build_cfg) = @_;
+
+    # write out the build information
+    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
+			     '^(builddate|buildtype|numdocs|numsections|numbytes|textlevel|indexstem|maxnumeric)$', 
+                             '^(indexmap|subcollectionmap|languagemap|indexfieldmap|notbuilt|indexfields|indexlevels|levelmap)$');
+
+}
+
+sub build_cfg_extra {
+    my $self = shift (@_);
+    my ($build_cfg) = @_;
+
+    $build_cfg->{'numsections'} = $self->{'buildproc'}->get_num_sections();
+    
+    # store the level info
+    my @indexlevels = ();
+    my @levelmap = ();
+    foreach my $l (@{$self->{'levelorder'}}) {
+	push (@indexlevels, $level_map{$l});
+	push (@levelmap, "$l\-\>$level_map{$l}");
+    }
+    $build_cfg->{'indexlevels'} = \@indexlevels;
+    $build_cfg->{'levelmap'} = \@levelmap;
+
+    if ($self->{'levels'}->{'section'}) {
+	$build_cfg->{'textlevel'} = $level_map{'section'};
+    } else {	
+	$build_cfg->{'textlevel'} = $level_map{'document'};
+    }
+   
+}
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/mgppbuildproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/mgppbuildproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/mgppbuildproc.pm	(revision 21112)
@@ -0,0 +1,338 @@
+###########################################################################
+#
+# mgppbuildproc.pm -- 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor outputs a document
+# for mgpp to process
+
+
+package mgppbuildproc;
+
+use basebuildproc;
+
+BEGIN {
+    @mgppbuildproc::ISA = ('basebuildproc');
+}
+
+#this must be the same as in mgppbuilder
+our %level_map = ('document'=>'Doc',
+		  'section'=>'Sec',
+		  'paragraph'=>'Para');
+
+sub new {
+    my $class = shift @_;
+    my $self = new basebuildproc (@_);
+
+    # use a different index specification to the default
+    $self->{'index'} = "text";
+
+    $self->{'dontindex'} = {};
+    $self->{'indexfieldmap'} = {};
+    $self->{'indexfields'} = {}; # only put in the ones that are not specified directly in the index
+    $self->{'strip_html'}=1;
+    
+    return bless $self, $class;
+}
+
+
+sub set_indexfieldmap {
+    my $self = shift (@_);
+    my ($indexmap) = @_;
+
+    $self->{'indexfieldmap'} = $indexmap;
+}
+
+sub get_indexfieldmap {
+    my $self = shift (@_);
+
+    return $self->{'indexfieldmap'};
+}
+
+sub set_levels {
+    my $self = shift (@_);
+    my ($levels) = @_;
+
+    $self->{'levels'} = $levels;
+}
+
+sub set_strip_html {
+    my $self = shift (@_);
+    my ($strip) = @_;
+    $self->{'strip_html'}=$strip;
+}
+
+
+sub get_gdbm_level {
+    my $self = shift (@_);
+    
+    #if a Section level index is not built, the gdbm file should be at doc 
+    #level not Section
+    if ($self->{'levels'}->{'section'}) {
+	return "section";
+    }
+    return "document";
+}
+
+
+#sub find_paragraphs {
+#    $_[1] =~ s/(<p\b)/<Paragraph>$1/gi;
+#}
+
+#this function strips the html tags from the doc if ($strip_html) and
+# if ($para) replaces <p> with <Paragraph> tags.
+# if both are false, the original text is returned
+#assumes that <pre> and </pre> have no spaces, and removes all < and > inside
+#these tags
+sub preprocess_text {
+    my $self = shift (@_);
+    my ($text, $strip_html, $para) = @_;
+    my ($outtext) = "";
+    if ($strip_html) { 
+	while ($text =~ /<([^>]*)>/ && $text ne "") {
+	    
+	    my $tag = $1;
+	    $outtext .= $`." "; #add everything before the matched tag
+	    $text = $'; #'everything after the matched tag
+	    if ($para && $tag =~ /^\s*p\s/i) {
+		$outtext .= $para;
+	    }
+	    elsif ($tag =~ /^pre$/) { # a pre tag
+		$text =~ /<\/pre>/; # find the closing pre tag
+		my $tmp_text = $`; #everything before the closing pre tag
+		$text = $'; #'everything after the </pre>
+		$tmp_text =~ s/[<>]//g; # remove all < and >
+		$outtext.= $tmp_text . " ";
+	    }
+	}
+    
+	$outtext .= $text; # add any remaining text
+	return $outtext;
+    } #if strip_html
+
+    #if ($para) {
+	#$text =~ s/(<p\b)/$para$1/gi;
+	#return $text;
+   # }
+    return $text;
+}
+	
+    
+
+sub filter_text {
+    # $self->filter_text ($field, $new_text);
+    # don't want to do anything for this version, however,
+    # in a particular collection you might want to override
+    # this method to post-process certain fields depending on
+    # the field, or whether we are outputting it for indexing
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $handle = $self->{'output_handle'};
+    my $outhandle = $self->{'outhandle'};
+
+    # only output this document if it is one to be indexed
+    return if ($doc_obj->get_doc_type() ne "indexed_doc");
+
+    my $indexed_doc = $self->is_subcollection_doc($doc_obj);
+    
+    # this is another document
+    $self->{'num_docs'} += 1;
+
+    # get the parameters for the output
+    # split on : just in case there is subcoll and lang stuff
+    my ($fields) = split (/:/, $self->{'index'});
+
+    my ($documenttag) = "";
+    my($documentendtag) = "";
+    if ($self->{'levels'}->{'document'}) {
+	$documenttag = "\n<". $level_map{'document'} . ">\n";
+	$documentendtag = "\n</". $level_map{'document'} . ">\n";
+    }
+    my ($sectiontag) = "";
+    if ($self->{'levels'}->{'section'}) {
+	$sectiontag = "\n<". $level_map{'section'} . ">\n";
+    }
+    my ($paratag) = "";
+    
+    if ($self->{'levels'}->{'paragraph'}) {
+	if ($self->{'strip_html'}) {
+	    $paratag = "<". $level_map{'paragraph'} . ">";
+	} else {
+	    print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
+	}
+    }
+
+    my $doc_section = 0; # just for this document
+    
+    my $text = $documenttag;
+   
+    # get the text for this document
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$doc_section++;
+	$self->{'num_sections'} += 1;
+	$text .= "$sectiontag";
+
+	if ($indexed_doc) {
+	    if ($self->{'indexing_text'}) {
+		$text .= "$paratag"; # only add para tags for indexing
+		# note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text
+	    }
+	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	    foreach my $field (split (/,/, $fields)) {
+		# only deal with this field if it doesn't start with top or
+		# this is the first section
+		my $real_field = $field;
+		if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
+		    my $new_text = ""; 
+		    my $tmp_text = "";
+		    if ($real_field eq "text") {
+			if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed
+			    $new_text .= "$paratag<TX>\n";
+			    $tmp_text .= $doc_obj->get_text ($section);
+			    $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</TX>$paratag<TX>");
+			    
+			    $new_text .= "$tmp_text</TX>\n";
+			    #if (!defined $self->{'indexfields'}->{'TextOnly'}) {
+				#$self->{'indexfields'}->{'TextOnly'} = 1;   
+			    #}
+			}
+			else { # leave html stuff in, and dont add Paragraph tags - never retrieve paras at the moment
+			    $new_text .= $doc_obj->get_text ($section) if $self->{'store_text'};
+			}
+		    } else { # metadata field
+			if ($real_field eq "allfields") { #ignore
+			}
+			elsif ($real_field eq "metadata") { # insert all metadata 
+			    #except gsdl stuff
+			    my $shortname = "";
+			    my $metadata = $doc_obj->get_all_metadata ($section);
+			    foreach my $pair (@$metadata) {
+				my ($mfield, $mvalue) = (@$pair);
+				# check fields here, maybe others dont want - change to use dontindex!!
+				if ($mfield ne "Identifier" 
+				    && $mfield !~ /^gsdl/ 
+				    && $mfield ne "classifytype"
+				    && $mfield ne "assocfilepath"
+				    && defined $mvalue && $mvalue ne "") {
+				    
+				    if (defined $self->{'indexfieldmap'}->{$mfield}) {
+					$shortname = $self->{'indexfieldmap'}->{$mfield};
+				    }
+				    else {
+					$shortname = $self->create_shortname($mfield);
+					$self->{'indexfieldmap'}->{$mfield} = $shortname;
+					$self->{'indexfieldmap'}->{$shortname} = 1;
+				    }	   
+				    $new_text .= "$paratag<$shortname>$mvalue</$shortname>\n";
+				    if (!defined $self->{'indexfields'}->{$mfield}) {
+					$self->{'indexfields'}->{$mfield} = 1;
+				    }				    
+				}
+			    }
+			    
+			}
+			else { #individual metadata specified
+			    my $shortname="";
+			    #if (!defined $self->{'indexfields'}->{$real_field}) {
+				#$self->{'indexfields'}->{$real_field} = 1;
+			    #}
+			    if (defined $self->{'indexfieldmap'}->{$real_field}) {
+				$shortname = $self->{'indexfieldmap'}->{$real_field};
+			    }
+			    else {
+				$shortname = $self->create_shortname($real_field);
+				$self->{'indexfieldmap'}->{$real_field} = $shortname;
+				$self->{'indexfieldmap'}->{$shortname} = 1;
+			    }
+			    my @section_metadata = @{$doc_obj->get_metadata ($section, $real_field)};
+			    if ($self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
+				if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
+				    push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $real_field)});
+				}
+			    }
+			    foreach my $item (@section_metadata) {
+			    #foreach my $item (@{$doc_obj->get_metadata ($section, $real_field)}) {
+				$new_text .= "$paratag<$shortname>$item</$shortname>\n";
+			    }
+			}
+
+		    }
+		    
+		    # filter the text
+		    $self->filter_text ($field, $new_text);
+
+		    $self->{'num_processed_bytes'} += length ($new_text);
+		    $text .= "$new_text";
+		}
+	    }
+	} # if (indexed_doc)
+	
+	$section = $doc_obj->get_next_section($section);
+    } #while defined section
+    print $handle "$text\n$documentendtag"; 
+    
+}
+
+#chooses the first two letters or digits for the shortname
+#now ignores non-letdig characters
+sub create_shortname {
+    my $self = shift(@_);
+    
+    my ($realname) = @_;
+    #take the first two chars
+    my $shortname;
+    if ($realname =~ /^[^\w]*(\w)[^\w]*(\w)/) {
+	$shortname = "$1$2";
+    } else {
+	# there aren't two letdig's in the field - try arbitrary combinations
+	$realname = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+	$shortname = "AB";
+    }
+    $shortname =~ tr/a-z/A-Z/;
+
+    #if already used, take the first and third letdigs and so on
+    my $count = 1;
+    while (defined $self->{'indexfieldmap'}->{$shortname}) {
+	if ($realname =~ /^[^\w]*(\w)([^\w]*\w){$count}[^\w]*(\w)/) {
+	    $shortname = "$1$3";
+	    $count++;
+	    $shortname =~ tr/a-z/A-Z/;
+	
+	}
+	else {
+	    #remove up to and incl the first letdig
+	    $realname =~ s/^[^\w]*\w//;
+	    $count = 0;
+	}
+    }
+
+    return $shortname;
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/multiread.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/multiread.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/multiread.pm	(revision 21112)
@@ -0,0 +1,313 @@
+###########################################################################
+#
+# multiread.pm --
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ
+# Copyright (C) 2005 New Zealand Digital Library project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# the multiread object will read in a number of encodings,
+# the results are always returned in the utf-8 format
+
+# encodings currently supported are
+#
+# utf8             - either utf8 or unicode (automatically detected)
+# unicode          - 2-byte UCS (does endian detection)
+#
+# plus all encodings in the "encodings" package
+
+package multiread;
+
+eval {require bytes};
+
+use unicode;
+
+sub new {
+    my ($class) = @_;
+
+    my $self = {'handle'    => "",
+		'first'     => 1,
+		'encoding'  => "utf8",
+		'bigendian' => 1};
+
+    return bless $self, $class;
+}
+
+# set_handle expects the file to be already open but
+# not read yet
+sub set_handle {
+    my $self = shift;
+    $self->{'handle'} = shift;
+    binmode( $self->{'handle'} );
+    $self->{'first'} = 1;
+    $self->{'encoding'} = "utf8";
+    $self->{'bigendian'} = 1;
+}
+
+# set_encoding should be called after set_handle
+sub set_encoding {
+    my $self = shift;
+    $self->{'encoding'} = shift;
+}
+
+sub get_encoding {
+    my $self = shift (@_);
+    return $self->{'encoding'};
+}
+
+# undef will be returned if the eof has been reached
+# the result will always be returned in utf-8
+
+sub read_unicode_char {
+    my $self = shift (@_);
+
+    # make sure we have a file handle
+    return undef if ($self->{'handle'} eq "");
+    my $handle = $self->{'handle'};
+
+    if ($self->{'encoding'} eq "utf8") {
+	# utf-8 text, how many characters we get depends 
+	# on what we find
+	my $c1 = "";
+	my $c2 = "";
+	my $c3 = "";
+
+	while (!eof ($handle)) {
+	    $c1 = ord (getc ($handle));
+
+	    if ($c1 <= 0x7f) {
+		# one byte character
+		return chr ($c1);
+
+	    } elsif ($c1 >= 0xc0 && $c1 <= 0xdf) {
+		# two byte character
+		$c2 = getc ($handle) if (!eof ($handle));
+		return chr ($c1) . $c2;
+
+	    } elsif ($c1 >= 0xe0 && $c1 <= 0xef) {
+		# three byte character
+		$c2 = getc ($handle) if (!eof ($handle));
+		$c3 = getc ($handle) if (!eof ($handle));
+		return chr ($c1) . $c2 . $c3;
+	    }
+
+	    # if we get here there was an error in the file, we should
+	    # be able to recover from it however, maybe the file is in
+	    # another encoding
+	}
+
+	return undef if (eof ($handle));
+    }
+
+    if ($self->{'encoding'} eq "unicode") {
+	# unicode text, get the next two characters
+	return undef if (eof ($handle));
+	my $c1 = ord (getc ($handle));
+	return undef if (eof ($handle));
+	my $c2 = ord (getc ($handle));
+
+	return &unicode::unicode2utf8 ([(($self->{'bigendian'}) ? ($c1*256+$c2) : ($c2*256+$c1))]);
+    }
+
+    return undef;
+}
+
+
+# undef will be returned if the eof has been reached
+# the result will always be returned in utf-8
+sub read_line {
+    my $self = shift (@_);
+
+    # make sure we have a file handle
+    return undef if ($self->{'handle'} eq "");
+
+    my $handle = $self->{'handle'};
+
+    if ($self->{'encoding'} eq "utf8") {
+	# utf-8 line
+	return <$handle>;
+    }
+
+    if ($self->{'encoding'} eq "unicode") {
+	# unicode line
+	my $c = "";
+	my ($c1, $c2) = ("", "");
+	my $out = "";
+	while (read ($handle, $c, 2) == 2) {
+	    $c1 = ord (substr ($c, 0, 1));
+	    $c2 = ord (substr ($c, 1, 1));
+	    $c = &unicode::unicode2utf8([(($self->{'bigendian'}) ? ($c1*256+$c2) : ($c2*256+$c1))]);
+	    $out .= $c;
+	    last if ($c eq "\n");
+	}
+
+	return $out if (length ($out) > 0);
+	return undef;
+    }
+
+    if ($self->{'encoding'} eq "iso_8859_1") {
+	# we'll use ascii2utf8() for this as it's faster than going
+	# through convert2unicode()
+	my $line = "";
+	if (defined ($line = <$handle>)) {
+	    return &unicode::ascii2utf8 (\$line);
+	}
+    }
+
+    # everything else uses unicode::convert2unicode
+    my $line = "";
+    if (defined ($line = <$handle>)) {
+	return &unicode::unicode2utf8 (&unicode::convert2unicode ($self->{'encoding'}, \$line));
+    }
+
+    return undef;
+}
+
+
+
+# this will look for a Byte Order Marker at the start of the file, and
+# set the encoding appropriately if there is one, returning any
+# non-marker text on the first line (or returns undef).
+sub find_unicode_bom {
+    my $self=shift;
+
+    my $non_bom_text=""; # to return if we read in 'real' text
+
+    if ($self->{'first'} == 0) { return }
+
+    # make sure we have a file handle
+    return if ($self->{'handle'} eq "");
+    my $handle = $self->{'handle'};
+
+    $self->{'first'} = 0;
+
+    my $b1 = ord(getc ($handle));
+    my $b2;
+    my $b3;
+
+    if ($b1 == 0xfe || $b1 == 0xff) {
+	$b2 = ord (getc ($handle)) if (!eof ($handle));
+	if ($b1 == 0xff && $b2 == 0xfe) {
+	    $self->{'encoding'} = "unicode";
+	    $self->{'bigendian'} = 0;
+	    return;
+	} elsif ($b1 == 0xfe && $b2 == 0xff) {
+	    $self->{'encoding'} = "unicode";
+	    $self->{'bigendian'} = 1;
+	    return;
+	} elsif ($b1 == 0xef && $b2 == 0xbb) {
+	    $b3 = ord(getc($handle));
+	    if ($b3 == 0xbf) {
+		$self->{'encoding'} = "utf8";
+		$self->{'bigendian'} = 1;
+		return;
+	    }
+	}
+    } else { # $b1 != fe or ff
+	$handle->ungetc($b1); return;
+    }
+    # if here, we have removed some chars and they aren't a BOM
+    if ($self->{'encoding'} eq "unicode") { # return the 2byte char
+	if (defined ($b3)) { # we looked at this... return it
+	    $handle->ungetc($b3);
+	}
+	return &unicode::unicode2utf8([$self->{'bigendian'}?
+				       ($b1*256+$b2) : ($b2*256+$b1)]);
+    }
+    # if here, it's utf-8
+    if ($b2 < 0x80) {
+	if (defined ($b3)) { # we grabbed this, but don't need it now
+	    $handle->ungetc($b3);
+	}
+	return ($b1 . $b2);
+    }
+    # if here, we have taken part of a multi-byte char. we need to make
+    # sure we return the entire character
+    if (defined($b3) && $b3 < 0x80) { # we have all we need
+	$handle->ungetc($b3);
+	return ($b1 . $b2);
+    }
+    my $c=$b1.$b2.$b3;
+    my $b4=$handle->getc();
+    while ($b4 > 0x7f) { # note - this will return consecutive mb utf8 chars
+	$c .= $b4;
+	$b4=$handle->getc();
+	if (eof($handle)) { last }
+    }
+    if (! eof($handle)) {
+	$handle->ungetc($b4); # this byte is an ascii byte
+    }
+    return $c;
+}
+
+
+
+# will convert entire contents of file to utf8 and append result to $outputref
+# this may be a slightly faster way to get the contents of a file than by 
+# recursively calling read_line()
+sub read_file {
+    my $self = shift (@_);
+    my ($outputref) = @_;
+
+    # make sure we have a file handle
+    return if ($self->{'handle'} eq "");
+
+    my $handle = $self->{'handle'};
+
+    # if encoding is set to utf8 or unicode, sniff to see if there is a 
+    # byte order marker
+    if ($self->{'first'} &&
+	($self->{'encoding'} eq "utf8" || $self->{'encoding'} eq 'unicode')) {
+	# this will change $self's encoding if there is a BOM
+	my $read_text = $self->find_unicode_bom();
+	$$outputref .= $read_text if (defined($read_text));
+    }
+
+    if ($self->{'encoding'} eq "utf8") {
+	undef $/;
+	$$outputref .=  <$handle>;
+	$/ = "\n";
+	return;
+    }
+
+    if ($self->{'encoding'} eq "unicode") {
+	my $line = "";
+	while (defined ($line = $self->read_line())) {
+	    $$outputref .= $line;
+	}
+	return;
+    }
+
+    if ($self->{'encoding'} eq "iso_8859_1") {
+	# we'll use ascii2utf8() for this as it's faster than going
+	# through convert2unicode()
+	undef $/;
+	my $text = <$handle>;
+	$/ = "\n";
+	$$outputref .= &unicode::ascii2utf8 (\$text);
+	return;
+    }
+
+    # everything else uses unicode::convert2unicode
+    undef $/;
+    my $text = <$handle>;
+    $/ = "\n";
+    $$outputref .= &unicode::unicode2utf8 (&unicode::convert2unicode ($self->{'encoding'}, \$text));
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/muread.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/muread.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/muread.pm	(revision 21112)
@@ -0,0 +1,151 @@
+###########################################################################
+#
+# muread.pm -- read a marked-up file
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package muread;
+
+use unicode;
+use multiread;
+
+sub new {
+    my ($class) = @_;
+
+    my $self = {'filename'=>"",
+		'encoding'=>"",
+		'handle'=>"",
+		'reader'=>"",
+		'buffer'=>""};
+
+    return bless $self, $class;
+}
+
+# returns a new tag with a tag name and any options
+sub parse_tag {
+    my $self = shift (@_);
+    my ($orgtagtext) = @_;
+    my $tagtext = $orgtagtext;
+    my $newtag = {};
+    my $misformed = 0;
+
+#    print STDERR "parsing \"$tagtext\"\n";
+
+    # get tag name (if there is one)
+    if ($tagtext =~ /^(\w+)/) {
+	$newtag->{'_tagname'} = $1;
+	$tagtext =~ s/^(\w+)//;
+    } else {
+	print STDERR "muread::parse_tag error - no tag name found\n";
+    }
+
+    # get the tag arguments
+    while ($tagtext =~ /\S/) {
+	$tagtext =~ s/^\s+//s;
+	if ($tagtext =~ /^(\w+)\s*=\s*\"([^\"]*)\"/s) {
+	    $newtag->{$1} = (defined $2) ? $2 : "";
+	    $tagtext =~ s/^\w+\s*=\s*\"[^\"]*\"//s;
+
+	} else {
+	    if (!$misformed) {
+		print STDERR "muread::parse_tag error - miss-formed tag <$orgtagtext>\n";
+		$misformed = 1;
+	    }
+	    $tagtext =~ s/^\S+//s;
+	}
+    }
+
+    return $newtag;
+}
+
+sub read_tag_content {
+    my $self = shift (@_);
+    my ($tag) = @_;
+
+    # all tags contain a _tagname except the tag for the document
+
+    my $line = "";
+    while (1) {
+	# deal with preceeding text
+	if ($self->{'buffer'} =~ /^([^<]+)</s) {
+	    # add preceeding text
+	    $tag->{'_contains'} = [] unless defined $tag->{'_contains'};
+	    push (@{$tag->{'_contains'}}, {'_text'=>$1});
+
+	    $self->{'buffer'} =~ s/^[^<]+</</s;
+	}
+
+	if ($self->{'buffer'} =~ /^<([^>\/]+)>/s) {
+	    # add info from this tag
+	    my $tagtext = $1;
+	    my $newtag = $self->parse_tag ($tagtext);
+	    push (@{$tag->{'_contains'}}, $newtag);
+	    $self->{'buffer'} =~ s/^<[^>\/]+>//s;
+	    
+	    # deal with the contents of this tag
+	    $self->read_tag_content ($newtag);
+
+	} elsif ($self->{'buffer'} =~ /^<\/([^>\/]+)>/s) {
+	    my $tagname = $1;
+	    $self->{'buffer'} =~ s/^<\/[^>\/]+>//s;
+
+	    # check that this tag is the right tag
+	    if (!defined $tag->{'_tagname'} || $tag->{'_tagname'} ne $tagname) {
+		print STDERR "muread::read_tag_content error - mismatched tag </$tagname>, " .
+		    "expected </$tag->{'_tagname'}>\n";
+	    } else {
+		return;
+	    }
+	} elsif (defined ($line = $self->{'reader'}->read_line())) {
+	    $self->{'buffer'} .= $line;
+	} else {
+	    if ($self->{'buffer'} =~ /\S/) {
+		print STDERR "muread::read_tag_content error - can't parse text \"$self->{'buffer'}\"\n";
+	    }
+	    last;
+	}
+    }
+    
+    if (defined $tag->{'_tagname'}) {
+	print STDERR "muread::read_tag_content error - eof reached before closing " .
+	    "tag \"$tag->{'_tagname'}\" found\n";
+    }
+}
+
+sub read_file {
+    my $self = shift (@_);
+    ($self->{'handle'}, $self->{'filename'}, $self->{'encoding'}) = @_;
+    $self->{'encoding'} = "utf8" unless defined $self->{'encoding'};
+
+    my $doc = {};
+
+    # get reader set up
+    $self->{'reader'} = new multiread ();
+    $self->{'reader'}->set_handle ($self->{'handle'});
+    $self->{'reader'}->set_encoding ($self->{'encoding'});
+
+    # read in the file
+    $self->read_tag_content ($doc);
+
+    $self->{'handle'} = "";
+    return $doc;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/parsargv.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/parsargv.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/parsargv.pm	(revision 21112)
@@ -0,0 +1,238 @@
+###########################################################################
+#
+# parseargv.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package parsargv;
+# 
+# parse(ARGVREF, [SPEC, VARREF] ...)
+# 
+# Parse command line arguments.
+# 
+# ARGVREF is an array reference, usually to @ARGV.  The remaining
+# arguments are paired (SPEC, VARREF).  SPEC is a specification string
+# for a particular argument; VARREF is a variable reference that will
+# receive the argument.
+# 
+# SPEC is in one of the following forms:
+# 
+# ARG/REGEX/DEFAULT ARG is the name of command line argument.  REGEX is
+#     a regular expression that gives legal values for the argument.
+#     DEFAULT is the default value assigned to VARREF if the option does
+#     not appear on the command line.  Example
+# 
+# 
+# ARG/REGEX ARG and REGEX are as above.  Since no default is given, ARG
+#     must appear on the command line; if it doesn't, parse() returns 0.
+# 
+# ARG ARG is as above.  ARG is a boolean option.  VARREF is assigned 0 if ARG
+#     is not on the command line; 1 otherwise.
+# 
+# SPEC may start with a punctuation character, in which case this
+# character will be used instead of '/' as a delimiter.  Useful when '/'
+# is needed in the REGEX part.
+# 
+# VARREF is a reference to a scalar or an array.  If VARREF is an array
+# reference, then multiple command line options are allowed an append.  Example:
+# 
+# 	Command line: -day mon -day fri
+# 
+# 	parse(\@ARGV, "day/(mon|tue|wed|thu|fri)", \@days)
+# 
+# 	days => ('mon', 'fri')
+# 
+# Returns 0 if there was an error, nonzero otherwise.
+# 
+
+ 
+sub parse
+{
+    my $arglist = shift;
+    my ($spec, $var);
+    my %option;
+
+    my @rest = @_;
+
+
+    # if the last argument is the string "allow_extra_options" then options
+    # in \@rest without a corresponding SPEC will be ignored (i.e. the "$arg is
+    # not a valid option" error won't occur)\n";
+    my $allow_extra_options = pop @rest;
+    if (defined ($allow_extra_options)) {
+	if ($allow_extra_options eq "allow_extra_options") {
+	    $allow_extra_options = 1;
+	} else {
+	    # put it back where we got it
+	    push (@rest, $allow_extra_options);
+	    $allow_extra_options = 0;
+	}
+    } else {
+	$allow_extra_options = 0;
+    }
+
+    while (($spec, $var) = splice(@rest, 0, 2))
+        {
+	  
+	die "Variable for $spec is not a valid type."
+	    unless ref($var) eq 'SCALAR' || ref($var) eq 'ARRAY'
+	    || (ref($var) eq 'REF' && ref($$var) eq 'GLOB');
+
+	my $delimiter;
+	if ($spec !~ /^\w/)
+	{
+	    $delimiter = substr($spec, 0, 1);
+	    $spec = substr($spec, 1);
+	}
+	else
+	{
+	    $delimiter = '/';
+	}
+	my ($name, $regex, $default) = split(/$delimiter/, $spec, 3);
+	
+	
+	if ($name)
+	{   
+	    if ($default && $default !~ /$regex/)
+	    {
+		die "Default value for $name doesn't match regex ($spec).";
+	    }
+	    $option{$name} = {'name' => $name,
+			      'regex' => $regex,
+			      'default' => $default,
+			      'varref' => $var,
+			      'set' => 0};
+	}
+	else
+	{
+	    die "Invalid argument ($spec) for parsargv.";
+	}
+    }
+
+    my @argv;
+    my $arg;
+    my $parse_options = 1;
+    my $errors = 0;
+
+    while ($arg = shift(@$arglist))
+    {
+	if ($parse_options && $arg eq '--')
+	{
+	    $parse_options = 0;
+	    next;
+	}
+
+	if ($parse_options && $arg =~ /^-+\w/)
+	{
+	    $arg =~ s/^-+//;
+
+	    if (defined $option{$arg})
+	    {
+		&process_arg($option{$arg}, $arglist, \$errors);
+	    }
+	    elsif (!$allow_extra_options)
+	    {
+		print STDOUT "$arg is not a valid option.\n";
+		$errors++;
+	    }
+	}
+	else
+	{
+	    push(@argv, $arg);
+	}
+    }
+    @$arglist = @argv;
+
+    foreach $arg (keys %option)
+    {
+	if ($option{$arg}->{'set'} == 0)
+	{
+	    if (defined $option{$arg}->{'default'})
+	    {
+		&set_var($option{$arg}, $option{$arg}->{'default'});
+	    }
+	    elsif (!$option{$arg}->{'regex'})
+	    {
+		&set_var($option{$arg}, 0)
+	    }
+	    elsif (ref($option{$arg}->{'varref'}) ne 'ARRAY')
+	    {
+		print STDOUT "Missing command line argument -$arg.\n";
+		$errors++;
+	    }
+	}
+    }
+    return $errors == 0;
+}
+
+sub process_arg
+{
+    my ($option, $arglist, $errors) = @_;
+
+    if ($option->{'regex'} && @$arglist > 0 && $arglist->[0] !~ /^-+\w/)
+    {
+	if ($arglist->[0] =~ /$option->{'regex'}/)
+	{
+	    &set_var($option, shift(@$arglist));
+	}
+	else
+	{
+	    print STDOUT  "Bad value for -$option->{'name'} argument.\n";
+	    $$errors++;
+	}
+    }
+    elsif (!$option->{'regex'})
+    {
+	&set_var($option, 1);
+    }
+    else
+    {
+	print STDOUT "No value given for -$option->{'name'}.\n";
+	$$errors++;
+    }
+}
+
+sub set_var
+{
+    my ($option, $value) = @_;
+    my $type = ref($option->{'varref'});
+
+    if ($type eq 'SCALAR')
+    {
+	${$option->{'varref'}} = $value;
+    }
+    elsif ($type eq 'ARRAY')
+    {
+	push(@{$option->{'varref'}}, $value);
+    }
+    $option->{'set'} = 1;
+}
+
+1;
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/parse2.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/parse2.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/parse2.pm	(revision 21112)
@@ -0,0 +1,340 @@
+#Last: Keeping doing the processArg for handing different type of arguments
+
+#parse2(\@_,$arguments,$self )
+
+package parse2;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use util;
+
+
+
+#--Local Util Functions----------------------------
+#-----------------------------------------
+# Name: transformArg
+# Parameters: 1.(Array pointer of plugin pre-defined argument list)
+# Pre-condition: Call this function and pass a array pointer of argument list.
+# Post-condition: This function will transform the array to a hash table 
+#                 with "Argument name" as its key
+# Return value: Return a hash table of plugin pre-defined argument 
+#               list with "argument name" as the key
+#-----------------------------------------
+sub transformArg
+{
+    my ($aryptSysArguList) = @_;
+    my %hashArg;
+    
+    foreach my $hashOneArg (@{$aryptSysArguList})
+    {
+	if(!(defined $hashArg{$hashOneArg->{"name"}}))
+	{
+	    $hashArg{$hashOneArg->{"name"}} = $hashOneArg;
+	}
+    }
+    return %hashArg;
+}
+
+sub checkRange
+{
+    my ($strRange,$intInputArg,$strArgName) = @_;
+    my @aryRange = split(",",$strRange);
+    if(defined $aryRange[0])
+    {
+	if($intInputArg < $aryRange[0])
+	{
+	    print STDERR " Parameter Parsing Error (Incorrect Range): when parse argument parameter for \"-$strArgName\"\n";
+	    return 0;
+	}
+	else
+	{
+	    if(scalar(@aryRange) == 2)
+	    { 
+		if($intInputArg > $aryRange[1])
+		{
+		    print STDERR " Parameter Parsing Error (Incorrect Range): when parse argument parameter for \"-$strArgName\"\n";
+		    return 0;
+		}
+	    }
+	}
+    }
+    else{ die " System error: minimum range is not defined. Possible mistyping in Argument list for $strArgName\n";}
+    return 1;
+}
+
+sub checkCharLength
+{
+    my ($intCharLength,$intInputArg,$strArgName) = @_;
+    if($intCharLength =~ m/\d/)
+    {
+	if(length($intInputArg) != $intCharLength)
+	{ 
+	    print STDERR " Parameter Parsing Error (Incorrect Char_Length): when parse argument parameter for \"-$strArgName\"\n";
+	    return 0;
+	}
+    }
+    else
+    {
+	die " System error: incorrect char_length. Possible mistyping in Argument list for $strArgName\n";
+    }
+    return 1;
+}
+#-----------------------------------------
+# Name: processArg
+# Parameters: 1.(Hash pointer of one argument)
+#             2.(Array pointer of the user given argument)
+#             3.(Hash pointer of user given arguments' values)
+# Pre-condition: Given a argument ($hashOneArg) 
+# Post-condition: System will check whether it need to get parameter 
+#                 from $aryptInputArguList or not, and also check the 
+#                 given parameter is following the argument description
+# Return value: 1 is parsing successful, 0 is failed.
+#-----------------------------------------
+sub processArg
+{
+    my ($hashOneArg,$aryptInputArguList,$hashInputArg) = @_;
+
+    # Since these two variables are going to be 
+    # used a lot, store them with some better names.
+    my $strArgName = $hashOneArg->{"name"};
+    my $strArgType = $hashOneArg->{"type"};
+
+    # If the argument type is "flag" then 
+    # set it to 1(which is "true") 
+    if($strArgType eq "flag")
+    {
+	$hashInputArg->{$strArgName} = 1;
+    }
+
+    # If the argument type is "int" then
+    # gets the next argument from $aryptInputArguList 
+    # and check whether it is a digit
+    # TODO: check its "range" and "char_length"
+    elsif($strArgType eq "int")
+    {
+	my $intInputArg = shift(@{$aryptInputArguList});
+	if ($intInputArg =~ /\d+/)
+	{
+	    $hashInputArg->{$strArgName} = $intInputArg;
+	}
+	else
+	{
+	    print STDERR " Error: occur in parse2.pm::processArg()\n Unmatched Argument: -$strArgName with type $strArgType\n";
+	    return 0;
+	}
+    }
+
+    # If the argument type is "enum" then
+    elsif($strArgType eq "enum")
+    {
+	if(defined $hashOneArg->{"list"})
+	{
+	    my $aryptList = $hashOneArg->{"list"};
+	    my $blnCheckInList = "false";
+	    my $strInputArg = shift(@{$aryptInputArguList});
+	    foreach my $hashEachItem (@$aryptList)
+	    {
+		if($strInputArg eq $hashEachItem->{"name"})
+		{
+		    $blnCheckInList = "true";
+		}
+		last if($blnCheckInList eq "true");
+	    }
+	    if($blnCheckInList ne "true")
+	    {
+		print STDERR " Error: occur in parse2.pm::processArg()\n Unknown Enum List Type: -$strArgName with parameter: $strInputArg\n";
+		return 0;
+	    } else {
+		$hashInputArg->{$strArgName} = $strInputArg;
+	    }
+	    
+	}
+	else
+	{
+	    print STDERR " Error: occur in parse2.pm::processArg(2)\n Unknown Type: -$strArgName with type $strArgType\n";
+	    return 0;
+	}
+    }
+
+    # If the argument type is "string" or "metadata" then
+    # just shift the next argument from $aryptInputArguList
+    # TODO: make sure if there is any checking required for this two types
+    elsif($strArgType eq "string" || $strArgType eq "metadata" || $strArgType eq "regexp")
+    {
+	$hashInputArg->{$strArgName}= shift(@{$aryptInputArguList});
+    }
+    
+    # Report any undefined types
+    else
+    {
+	print STDERR " Error: occur in parse2.pm::processArg(3)\n Unknown Type: -$strArgName with type $strArgType\n";
+	return 0;
+    }
+    
+    return 1;
+}
+
+#--Main Parsing Function----------------------------
+#-----------------------------------------
+# Name: parse
+# Parameters: 1.(Array pointer of the user given argument)
+#             2.(Array pointer of plugin pre-defined argument list)  
+#             3.(Self pointer, where we store all the argument value)
+# Pre-condition: Plugin gives the parameters to parse function in parse2 
+# Post-condition: Store all the default or user given values to the $self->{$ArgumentName}.
+#                 Since $self is a pointer from plugin, plugin will have every values we set.
+# Return value: 1 is parsing successful, 0 is failed.
+#-----------------------------------------
+sub parse
+{
+    # Get the user supplied arguments pointer "\@_"
+    my $aryptUserArguList = shift; 
+
+    # Check if allow extra arguments
+    my $blnAllowExtraOptions = "false";
+    
+    if(scalar(@_) == 3)
+    { 
+	my $strAllowExtraOptions = pop @_;
+	
+	if ($strAllowExtraOptions eq "allow_extra_options") 
+	{
+	    $blnAllowExtraOptions = "true";    
+	} 
+    }
+    
+    my ($aryptSysArguList,$self) = @_;
+    my %hashArg;
+    my %hashInputArg;
+    my @ExtraOption;
+
+    # Transform the system argument (predefined the code) 
+    # from array to hash table for increasing performance
+    %hashArg = &transformArg($aryptSysArguList);
+
+    # Process each User input argument and store the 
+    # information into hashInputArg
+    while (my $strOneArg = shift(@{$aryptUserArguList}))
+    {
+	# Check whether it start with a "-" sign
+	if ($strOneArg =~ /^-+\w/)
+	{
+	    # If it is start with a "-" sign then take it off
+	    $strOneArg =~ s/^-+//;
+
+	    # If the inputed argument is defined in the argument 
+	    # list from this plugin then process
+	    
+	    if(defined $hashArg{$strOneArg})
+	    {
+		#$%^
+		#print "($strOneArg) is processed\n";
+		# Process this argument and store the related 
+		# information in %hashInputArg
+		if(processArg($hashArg{$strOneArg},$aryptUserArguList,\%hashInputArg) == 0){ 
+		    print STDERR "<BadArgumentValue a=$strOneArg>\n";
+		    return 0;}
+	    }
+	    
+	    # Else check if it allows extra options, if yes 
+	    # then push it to a new array, else return fault
+	    else
+	    {
+		if($blnAllowExtraOptions eq "true")
+		{
+		    push(@ExtraOption,"-$strOneArg");
+		}
+		else
+		{
+		    print STDERR "<BadArgument a=$strOneArg>\n";
+		    print STDERR " Error: occur in parse2.pm::parse()\n Extra Arguments: $strOneArg\n";
+		    return 0;
+		}
+	    }
+	}
+	
+	# This part follow the previous parsing system. 
+	# It doesn't return error message even user
+	# gave a invalid argument.
+	else
+	{
+	    if($blnAllowExtraOptions eq "true")
+	    {
+		push(@ExtraOption,$strOneArg);
+	    }
+	    else
+	    {
+		print STDERR " Error: occur in parse2.pm::parse()\n Invalid Argument: $strOneArg\n";
+		return 0;
+	    }
+	}
+    }
+
+    # Store the extra option back 
+    # to the user given argument list.
+    @$aryptUserArguList = @ExtraOption;
+    
+    # Now we go through all the pre defined arguments, 
+    # if the user has specified the arguments then just 
+    # set to whatever they set. Otherwise use the default value
+    foreach my $hashOneArg (@{$aryptSysArguList})
+    {
+	my $strArgName = $hashOneArg->{"name"};
+
+	# If the strArgName has defined in the %hashInputArg, 
+	# this means users has give this argument, store the 
+	# user given to self->{"$strArgName"}
+	if(defined $hashInputArg{$strArgName})
+	{
+	    if(defined $hashOneArg->{"range"}) 
+	    {
+		if(checkRange($hashOneArg->{"range"},$hashInputArg{$strArgName},$strArgName) == 0){ return 0;}
+	    }
+	    if(defined $hashOneArg->{"char_length"})
+	    {
+		if(checkCharLength($hashOneArg->{"char_length"},$hashInputArg{$strArgName},$strArgName) == 0){ return 0;}
+	    }
+	    $self->{"$strArgName"} = $hashInputArg{"$strArgName"};
+	}
+	elsif (!defined $self->{$strArgName}) 
+	{
+	    # don't want to override default with superclass value
+	  
+	    # Else use the default value of the arguments, 
+	    # if there is no default value, then it must be a flag, 
+	    # then set it to 0 (which is false)
+	
+	    if(defined $hashOneArg->{"deft"})
+	    {
+		$self->{"$strArgName"} = $hashOneArg->{"deft"};
+	    }
+	    else
+	    {
+		if($hashOneArg->{"type"} eq "flag"){ 
+		    $self->{"$strArgName"} = 0;
+		}
+		else {
+		    # all other cases, use "" as default
+		    $self->{"$strArgName"} = "";
+		}
+	    }
+	}
+    }
+
+    # If allow_extra_options is set, then return the number of arguments left in the argument list.
+    if($blnAllowExtraOptions eq "true")
+    {
+	return scalar(@$aryptUserArguList);
+    }
+    else
+    {
+	return 1;
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugin.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugin.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugin.pm	(revision 21112)
@@ -0,0 +1,329 @@
+###########################################################################
+#
+# plugin.pm -- functions to handle using plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package plugin;
+
+use strict; # to pick up typos and undeclared variables...
+no strict 'refs'; # ...but allow filehandles to be variables and vice versa
+no strict 'subs';
+
+require util;
+use gsprintf 'gsprintf';
+
+# global variables
+my $stats = {'num_processed' => 0,
+	     'num_blocked' => 0,
+	     'num_not_processed' => 0,
+	     'num_not_recognised' => 0,
+	     'num_archives' => 0
+	     };
+
+#globaloptions contains any options that should be passed to all plugins
+my ($verbosity, $outhandle, $failhandle, $globaloptions);
+
+sub load_plugin_for_info {
+    my ($pluginname) = shift @_;
+
+    # find the plugin
+    my $colplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugins", 
+					  "${pluginname}.pm");
+    my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/plugins", 
+					   "${pluginname}.pm");
+    if (-e $colplugname) { require $colplugname; }
+    elsif (-e $mainplugname) { require $mainplugname; }
+    else {
+	&gsprintf(STDERR, "{plugin.could_not_find_plugin}\n",
+		 $pluginname);
+	die "\n";
+    }
+
+    # create a plugin object
+    my ($plugobj);
+    my $options = "-gsdlinfo";
+    
+    eval ("\$plugobj = new \$pluginname([],[$options])");
+    die "$@" if $@;
+
+    return $plugobj;
+}
+
+sub load_plugins {
+    my ($plugin_list) = shift @_;
+    my $keepold;
+    ($verbosity, $outhandle, $failhandle, $globaloptions, $keepold) = @_; # globals
+    my @plugin_objects = ();
+    $keepold = 0 unless (defined $keepold && $keepold == 1);
+    $verbosity = 2 unless defined $verbosity;
+    $outhandle = 'STDERR' unless defined $outhandle;
+    $failhandle = 'STDERR' unless defined $failhandle;
+
+    map { $_ = "\"$_\""; } @$globaloptions;
+    my $globals = join (",", @$globaloptions);
+
+    foreach my $pluginoptions (@$plugin_list) {
+	my $pluginname = shift @$pluginoptions;
+	next unless defined $pluginname;
+
+	# find the plugin
+	my $colplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugins", 
+					      "${pluginname}.pm");
+	my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/plugins", 
+					       "${pluginname}.pm");
+	if (-e $colplugname) { require $colplugname; }
+	elsif (-e $mainplugname) { require $mainplugname; }
+	else {
+	    gsprintf($outhandle, "{plugin.could_not_find_plugin}\n",
+		     $pluginname);
+	    die "\n";
+	}
+
+	# create a plugin object
+	my ($plugobj);
+	map { $_ = "\"$_\""; } @$pluginoptions;
+	my $options = join (",", @$pluginoptions);
+	if ($globals) {
+	    if (@$pluginoptions) {
+		$options .= ",";
+	    }
+	    $options .= "$globals";
+	}
+	$options =~ s/\$/\\\$/g;
+
+	eval ("\$plugobj = new \$pluginname([],[$options])");
+	die "$@" if $@;
+	
+	# initialize plugin
+	$plugobj->init($verbosity, $outhandle, $failhandle);
+	
+	$plugobj->set_keepold($keepold);
+
+	# add this object to the list
+	push (@plugin_objects, $plugobj);
+    }
+
+    return \@plugin_objects;
+}
+
+
+sub begin {
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+   
+
+    map { $_->begin($pluginfo, $base_dir, $processor, $maxdocs); } @$pluginfo;
+}
+
+
+sub metadata_read {
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli, $aux) = @_;
+
+    $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
+    $gli = 0 unless defined $gli;
+
+    my $rv = 0;
+    my $glifile = $file;
+    
+    $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
+    
+    # Announce to GLI that we are handling a file
+    print STDERR "<File n='$glifile'>\n" if $gli;
+    
+    # the .kill file is a handy (if not very elegant) way of aborting 
+    # an import.pl or buildcol.pl process
+    if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
+	gsprintf($outhandle, "{plugin.kill_file}\n");
+	die "\n";
+    }
+
+    my $had_error = 0;
+    # pass this file by each of the plugins in turn until one
+    # is found which will process it
+    # read must return:
+    # undef - could not recognise
+    # -1 - tried but error
+    # 0 - blocked
+    # anything else for successful processing
+	
+    foreach my $plugobj (@$pluginfo) {
+
+	$rv = $plugobj->metadata_read($pluginfo, $base_dir, $file, 
+			     $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli, $aux);
+
+	if (defined $rv) {
+	    if ($rv == -1) {
+	        # an error has occurred
+		$had_error = 1;
+		print STDERR "<ProcessingError n='$glifile'>\n" if $gli;
+	    } else {
+        	return $rv;
+	    }
+	} # else undefined - was not recognised by the plugin
+    }
+
+    return 0;
+}
+
+sub read {
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli, $aux) = @_;
+
+    $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
+    $total_count = 0 unless defined $total_count && $total_count =~ /\d/;
+    $gli = 0 unless defined $gli;
+
+    my $rv = 0;
+    my $glifile = $file;
+    
+    $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
+    
+    # Announce to GLI that we are handling a file
+    print STDERR "<File n='$glifile'>\n" if $gli;
+    
+    # the .kill file is a handy (if not very elegant) way of aborting 
+    # an import.pl or buildcol.pl process
+    if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
+	gsprintf($outhandle, "{plugin.kill_file}\n");
+	die "\n";
+    }
+
+    my $had_error = 0;
+    # pass this file by each of the plugins in turn until one
+    # is found which will process it
+    # read must return:
+    # undef - could not recognise
+    # -1 - tried but error
+    # 0 - blocked
+    # anything else for successful processing
+	
+    foreach my $plugobj (@$pluginfo) {
+
+      	$rv = $plugobj->read($pluginfo, $base_dir, $file, 
+			     $metadata, $processor, $maxdocs, $total_count, $gli, $aux);
+
+	if (defined $rv) {
+	    if ($rv == -1) {
+	        # an error has occurred
+		$had_error = 1;
+	    } else {
+        	return $rv;
+	    }
+	} # else undefined - was not recognised by the plugin
+    }
+
+    if ($had_error) {
+	# was recognised but couldn't be processed
+	if ($verbosity >= 2) {
+	    gsprintf($outhandle, "{plugin.no_plugin_could_process}\n", $file);
+	}
+	# tell the GLI that it was not processed
+	print STDERR "<NonProcessedFile n='$glifile'>\n" if $gli;
+      
+	$file =~ s/.*?([^\\\/]+)$/$1/;
+	gsprintf($failhandle, "$file: {plugin.no_plugin_could_process_this_file}\n");
+	$stats->{'num_not_processed'} ++;
+    } else {
+	# was not recognised
+	if ($verbosity >= 2) {
+	    gsprintf($outhandle, "{plugin.no_plugin_could_recognise}\n",$file);
+	}
+	# tell the GLI that it was not processed
+	print STDERR "<NonRecognisedFile n='$glifile'>\n" if $gli;
+	
+	$file =~ s/.*?([^\\\/]+)$/$1/;
+	gsprintf($failhandle, "$file: {plugin.no_plugin_could_recognise_this_file}\n");
+	$stats->{'num_not_recognised'} ++;
+    }
+    return 0;
+}
+
+# write out some general stats that the plugins have compiled - note that
+# the buildcol.pl process doesn't currently call this process so the stats
+# are only output after import.pl -
+sub write_stats {
+    my ($pluginfo, $statshandle, $faillog, $gli) = @_;
+
+    $gli = 0 unless defined $gli;
+
+    foreach my $plugobj (@$pluginfo) {
+	$plugobj->compile_stats($stats);
+    }
+
+    my $total = $stats->{'num_processed'} + $stats->{'num_blocked'} + 
+	$stats->{'num_not_processed'} + $stats->{'num_not_recognised'};
+
+    print STDERR "<ImportComplete considered='$total' processed='$stats->{'num_processed'}' blocked='$stats->{'num_blocked'}' ignored='$stats->{'num_not_recognised'}' failed='$stats->{'num_not_processed'}'>\n" if $gli;
+
+    if ($total == 1) {
+	gsprintf($statshandle, "* {plugin.one_considered}\n");
+    } else {
+	gsprintf($statshandle, "* {plugin.n_considered}\n", $total);
+    }
+    if ($stats->{'num_archives'}) {
+	if ($stats->{'num_archives'} == 1) {
+	    gsprintf($statshandle, "   ({plugin.including_archive})\n");
+	}
+	else {
+	    gsprintf($statshandle, "   ({plugin.including_archives})\n",
+		     $stats->{'num_archives'});
+	}
+    }
+    if ($stats->{'num_processed'} == 1) {
+	gsprintf($statshandle, "* {plugin.one_included}\n");
+    } else {
+	gsprintf($statshandle, "* {plugin.n_included}\n", $stats->{'num_processed'});
+    }
+    if ($stats->{'num_not_recognised'}) {
+	if ($stats->{'num_not_recognised'} == 1) {
+	    gsprintf($statshandle, "* {plugin.one_unrecognised}\n");
+	} else {
+	    gsprintf($statshandle, "* {plugin.n_unrecognised}\n",
+		     $stats->{'num_not_recognised'});
+	}
+
+    }
+    if ($stats->{'num_not_processed'}) {
+	if ($stats->{'num_not_processed'} == 1) {
+	    gsprintf($statshandle, "* {plugin.one_rejected}\n");
+	} else {
+	    gsprintf($statshandle, "* {plugin.n_rejected}\n",
+		     $stats->{'num_not_processed'});
+	}
+    }
+    if ($stats->{'num_not_processed'} || $stats->{'num_not_recognised'}) {
+	gsprintf($statshandle, " {plugin.see_faillog}\n", $faillog);
+    }
+}
+
+sub end {
+    my ($pluginfo, $processor) = @_;
+    map { $_->end($processor); } @$pluginfo;
+}
+
+sub deinit {
+    my ($pluginfo, $processor) = @_;
+   
+
+    map { $_->deinit($processor); } @$pluginfo;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ArcPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ArcPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ArcPlug.pm	(revision 21112)
@@ -0,0 +1,180 @@
+###########################################################################
+#
+# ArcPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which recurses through an archives.inf file
+# (i.e. the file generated in the archives directory
+# when an import is done), processing each file it finds 
+
+# 12-05-02 Added usage datastructure - John Thompson
+
+package ArcPlug;
+
+use util;
+use BasPlug;
+use plugin;
+use arcinfo;
+use gsprintf;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+BEGIN {
+    @ArcPlug::ISA = ('BasPlug');
+}
+
+my $arguments = [
+		 ];
+
+my $options = { 'name'     => "ArcPlug",
+		'desc'     => "{ArcPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+         
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub deinit {
+    my ($self) = @_;
+
+    my $archive_info = $self->{'archive_info'};
+
+    if (defined $archive_info) {
+	my $archive_info_filename = $self->{'archive_info_filename'};
+
+       	my $file_list = $archive_info->get_file_list();
+
+	# change each file to "Been Indexed"
+
+	foreach my $subfile (@$file_list) {
+	    my $doc_oid = $subfile->[1];
+	    # why do we get this when it is not used???
+	    my $index_status = $archive_info->get_status_info($doc_oid);
+	    $archive_info->set_status_info($doc_oid,"B");
+	}
+
+	$archive_info->save_info($archive_info_filename);
+    }
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs,$total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $count = 0;
+
+    # see if this has a archives information file within it
+    my $archive_info_filename = &util::filename_cat($base_dir,$file,"archives.inf");
+    $self->{'archive_info_filename'} = $archive_info_filename;
+
+    if (-e $archive_info_filename) {
+
+	# found an archives.inf file
+	&gsprintf($outhandle, "ArcPlug: {common.processing} $archive_info_filename\n") if $self->{'verbosity'} > 1;
+
+	# read in the archives information file
+	my $archive_info = new arcinfo ();
+	$self->{'archive_info'} = $archive_info;
+
+	$archive_info->load_info ($archive_info_filename);
+	
+	my $file_list = $archive_info->get_file_list();
+
+	# process each file
+	foreach my $subfile (@$file_list) {
+	    last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs);
+
+	    my $tmp = &util::filename_cat ($file, $subfile->[0]);
+	    next if $tmp eq $file;
+
+	    # Decide if file needs to be processed
+	    my $doc_oid = $subfile->[1];
+	    my $index_status = $archive_info->get_status_info($doc_oid);
+
+	    my $process_file = 0;
+	    
+	    if ($processor->get_mode() eq "infodb") {
+		$process_file = 1;
+	    }
+	    else {
+		if (!$processor->is_incremental_capable() || !$self->{'keepold'}) {
+		    $process_file = 1;
+		}
+		else {
+		    # is incremental
+
+		    # check to see if file needs to be indexed
+		    $index_status = $archive_info->get_status_info($doc_oid);
+
+		    if ($index_status eq "I") {
+			$process_file = 1;
+		    }
+		}
+	    }
+
+
+	    if ($process_file) {
+		# note: metadata is not carried on to the next level
+		$count += &plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor, $maxdocs, ($total_count+$count), $gli);
+	    }
+
+	}
+
+	return $count;
+    }
+
+    # wasn't an archives directory, someone else will have to process it
+    return undef;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/BNContentePlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/BNContentePlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/BNContentePlug.pm	(revision 21112)
@@ -0,0 +1,321 @@
+###########################################################################
+#
+# BNContentePlug.pm -- plugin for import the BN-Portugal Collection
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# BNContentePlug - 11/2004
+# 
+#
+# This plugin takes "mets.xml" and "record/NCB_***.xml: the file contain MARC details
+# about BN-Portugal ccllection. The intension is to import such a collection into GS2.
+
+package BNContentePlug;
+
+use BasPlug;
+use plugin;
+#use ghtml;
+use XMLParser;
+use XML::Parser;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @BNContentePlug::ISA = ('BasPlug');
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "only_first_doc",
+	'desc' => "{BNContentePlug.only_first_doc}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "first_inoder_ext",
+	'desc' => "{BNContentePlug.first_inorder_ext}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "first_inorder_mime",
+	'desc' => "{BNContentePlug.first_inorder_mime}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "string",
+	'deft' => &get_default_block_exp(),
+	'reqd' => "no" }];
+
+my $options = { 'name'     => "BNContentePlug",
+		'desc'     => "{BNContentePlug.desc}",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+# Important variation to regular plugin structure.  Need to desclare
+# $self as global variable to file so XMLParser callback routines
+# can access the content of the object. 
+my ($self); 
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)(metsHTML\.xml)$^;
+}
+
+# block files
+sub get_default_block_exp {
+    my $self = shift (@_);
+    
+    # Block all files besides contents
+    #return q^(?i)(metsHTML\.xml|)$^;
+    return q^(?i)((.*?)\.(.*?))$^;
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    
+    #create XML::Parser object for parsing metsHTML.xml, NCB_???.xml files
+    my $mets_parser = new XML::Parser('Style' => 'Stream',
+				      'Handlers' => {'Doctype' => \&METS_Doctype,
+						     'Start' => \&METS_StartTag,
+						     'End' => \&METS_EndTag
+						     });
+    
+    my $marc_parser = new XML::Parser('Style' => 'Stream',
+				      'Handlers' => {'Char' => \&Char,
+						     'Doctype' => \&MARC_Doctype,
+						     'Start' => \&MARC_StartTag,
+						     'End' => \&MARC_EndTag
+						     });
+    $self->{'mets_parser'} = $mets_parser;
+    $self->{'marc_parser'} = $marc_parser;
+
+    $self->{'index_file'} = "";
+
+    return bless $self, $class;
+}
+
+sub read_marc_content {
+    my $self = shift (@_);
+    my ($marc_file) = @_;
+    
+    # parse the Marc_file: NCB_???.xml
+    eval{
+	$self->{'marc_parser'}->parsefile($marc_file);
+    };
+    
+    if ($@) {
+	die "BNContentePlug: ERROR $marc_file is not a well formed XML file ($@)\n";
+    }  
+}
+
+# Read metsHTML.xml from BN-Portugal collection 
+sub metadata_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    if ($filename !~ /metsHTML\.xml$/ || !-f $filename) {
+	if ($filename =~ /\.xml$/i || $filename =~ /log\.txt$/i || $filename =~ /isbd\.html$/i) {
+	    $self->{'file_blocks'}->{$filename}=1;
+	}
+	return undef;
+    } else {
+	$self->{'file_blocks'}->{$filename}=1;	
+    }
+  
+    print $outhandle "BNContentePlug: extracting metadata from $filename\n"
+	if $self->{'verbosity'} > 1;
+   
+    my ($dir) = $filename =~ /^(.*?)[^\/\\]*$/;
+    $self->{'dir'} = $dir;
+
+    eval {
+	$self->{'mets_parser'}->parsefile($filename);
+    };
+    
+    if ($@) {
+	die "BNContentePlug: ERROR $filename is not a well formed XML file ($@)\n";
+    }
+    
+    # read NCB_???.xml to parse MARC records and save as metadata
+    my $marc_file = &util::filename_cat($dir,$self->{'marc_file'});
+    $self->read_marc_content ($marc_file);
+
+    if (defined $self->{'index_file'} && $self->{'index_file'} ne "") {
+	my $index_file = $self->{'index_file'};
+	push(@$extrametakeys,$index_file);
+	$extrametadata->{$index_file} = $self->{'saved_metadata'};
+    } else {
+	print STDERR "####Warning can't find main index file\n";
+    }
+    return 1;
+}
+
+# The BNContentePlug read() function. This function does all the right things
+# to make general options work for a given plugin. It calls the process()
+# function which does all the work specific to a plugin (like the old
+# read functions used to do). Most plugins should define their own 
+# process() function and let this read() function keep control.
+#
+# Return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+
+sub readxxx {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    my $filename = &util::filename_cat($base_dir, $file);
+    #return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    
+    #return 0 if ($filename =~ /metsHTML\.xml$/);
+    return 0 if ($filename =~ /\.xml$/);
+    return 0 if (defined $self->{'file_blocks'}->{'filename'});
+    
+    return undef;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+sub METS_Doctype {
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+    die if ($name !~ /^metsHTML\.xml$/);
+}
+
+sub MARC_Doctype {
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+    print STDERR "###MARC Name =$name\n";
+    #die if ($name !~ /^metsHTML\.xml$/);
+    #die if (!$name);
+}
+
+sub METS_StartTag {
+    my ($expat, $element, %attr) = @_;
+    my @file_blocks;
+    if ($element eq "dmdSec") {
+	$self->{'marc_file'} = "";
+    } elsif ($element eq "mdRef") {
+	my $marc_href = $attr{'xlink:href'};
+	$self->{'marc_file'} = $marc_href;
+    } elsif ($element eq "FLocat"){
+ 	my $assocfiles = $attr{'xlink:href'};
+ 	if ($assocfiles =~ /index\.html$/) {
+ 	    my $index_file = &util::filename_cat($self->{'dir'}, $assocfiles);
+ 	    $self->{'index_file'} = $index_file;
+ 	} else {
+	    my $link = &util::filename_cat($self->{'dir'}, $assocfiles);
+	    #$self->{'file_blocks'}->{$link} = 1;
+	}
+    }
+}
+
+sub METS_EndTag {
+    my ($expat, $element, %attr) = @_;
+}
+
+sub MARC_StartTag {
+    my ($expat, $element, %attr) = @_;
+
+    if ($element eq "record") {
+	$self->{'saved_metadata'} = {};
+    }
+    elsif ($element eq "datafield") {
+	$self->{'metaname'} = $element;
+	$self->{'datafield'} = $attr{'tag'};
+    }
+    elsif ($element eq "subfield") {
+	$self->{'subfield'} = $attr{'code'};
+	$self->{'text'} = "";
+    }
+}
+
+sub MARC_EndTag {
+    my ($expat, $element) = @_;
+
+    if ($element eq "datafield") {
+	$self->{'metaname'} = "";
+    }
+    elsif ($element eq "subfield") {
+	my $mvalue = $self->{'text'};
+	my $mname = $self->{'datafield'}."^".$self->{'subfield'};
+	#print STDERR "**** $mname = $mvalue\n";
+	
+	$mvalue =~ s/\[/&\#91;/g;
+	$mvalue =~ s/\[/&\#93;/g;
+	
+
+	if (defined $self->{'saved_metadata'}->{$mname}) {
+	    # accumulate - add value to existing value(s)
+	    if (ref ($self->{'saved_metadata'}->{$mname}) eq "ARRAY") {
+		push (@{$self->{'saved_metadata'}->{$mname}}, $mvalue);
+	    } else {
+		$self->{'saved_metadata'}->{$mname} = 
+		    [$self->{'saved_metadata'}->{$mname}, $mvalue];
+	    }
+	} else {
+	    # accumulate - add value into (currently empty) array
+	    $self->{'saved_metadata'}->{$mname} = [$mvalue];
+	}
+	# store something here 
+	$self->{'subfield'} = "";
+	$self->{'text'} = "";
+    }
+
+}
+
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {
+    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
+    $_[0]->{'Text'} .= $_[1];
+    if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
+	$self->{'text'} .= $_[1];
+    }
+    return undef;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/BasPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/BasPlug.pm	(revision 21112)
@@ -0,0 +1,1489 @@
+###########################################################################
+#
+# BasPlug.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package BasPlug;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+}
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use strict; 
+no strict 'subs';
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use File::Basename;
+
+use Kea;
+use multiread;
+use encodings;
+use cnseg;
+use acronym;
+use textcat;
+use doc;
+eval "require diagnostics"; # some perl distros (eg mac) don't have this
+use DateExtract;
+use ghtml;
+use gsprintf 'gsprintf';
+use printusage;
+use parse2;
+
+
+use GISBasPlug;
+
+@BasPlug::ISA = ( GISBasPlug );
+
+my $unicode_list =
+    [ { 'name' => "ascii",
+	'desc' => "{BasPlug.input_encoding.ascii}" },
+      { 'name' => "utf8",
+	'desc' => "{BasPlug.input_encoding.utf8}" },
+      { 'name' => "unicode",
+	'desc' => "{BasPlug.input_encoding.unicode}" } ];
+
+my $auto_unicode_list = 
+    [ { 'name' => "auto",
+	'desc' => "{BasPlug.input_encoding.auto}" } ];
+
+my $e = $encodings::encodings;
+foreach my $enc (sort {$e->{$a}->{'name'} cmp $e->{$b}->{'name'}} keys (%$e)) 
+{
+    my $hashEncode =
+    {'name' => $enc,
+     'desc' => $e->{$enc}->{'name'}};
+    
+    push(@{$unicode_list},$hashEncode);
+}
+
+push(@{$auto_unicode_list},@{$unicode_list});
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "smart_block",
+	'desc' => "{BasPlug.smart_block}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "associate_ext",
+	'desc' => "{BasPlug.associate_ext}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "input_encoding",
+	'desc' => "{BasPlug.input_encoding}",
+	'type' => "enum",
+	'list' => $auto_unicode_list,
+	'reqd' => "no" ,
+	'deft' => "auto" } ,
+      { 'name' => "default_encoding",
+	'desc' => "{BasPlug.default_encoding}",
+	'type' => "enum",
+	'list' => $unicode_list,
+	'reqd' => "no",
+        'deft' => "utf8" },
+      { 'name' => "extract_language",
+	'desc' => "{BasPlug.extract_language}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "default_language",
+	'desc' => "{BasPlug.default_language}",
+	'type' => "string",
+	'deft' => "en",
+	'reqd' => "no" },
+      { 'name' => "extract_acronyms",
+	'desc' => "{BasPlug.extract_acronyms}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "markup_acronyms",
+	'desc' => "{BasPlug.markup_acronyms}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "extract_keyphrases",
+	'desc' => "{BasPlug.extract_keyphrases}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "extract_keyphrase_options",
+	'desc' => "{BasPlug.extract_keyphrase_options}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "first",
+	'desc' => "{BasPlug.first}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "extract_email",
+	'desc' => "{BasPlug.extract_email}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "extract_historical_years",
+	'desc' => "{BasPlug.extract_historical_years}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "maximum_year",
+	'desc' => "{BasPlug.maximum_year}",
+	'type' => "int",
+	'deft' => (localtime)[5]+1900,
+	'char_length' => "4",
+	#'range' => "2,100",
+	'reqd' => "no"},
+      { 'name' => "maximum_century",
+	'desc' => "{BasPlug.maximum_century}",
+	'type' => "string",
+	'deft' => "-1",
+	'reqd' => "no" },
+      { 'name' => "no_bibliography",
+	'desc' => "{BasPlug.no_bibliography}",
+	'type' => "flag",
+	'reqd' => "no"},
+      { 'name' => "no_cover_image",
+	'desc' => "{BasPlug.no_cover_image}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "extract_keyphrases",
+	'desc' => "{BasPlug.extract_keyphrases}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "extract_keyphrase_options",
+	'desc' => "{BasPlug.extract_keyphrase_options}",
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "separate_cjk",
+	'desc' => "{BasPlug.separate_cjk}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "smart_block",
+	'desc' => "{BasPlug.smart_block}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "new_extract_email",
+	'desc' => "",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" } ];
+
+my $gis_arguments = 
+    [ { 'name' => "extract_placenames",
+	'desc' => "{GISBasPlug.extract_placenames}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "gazetteer",
+	'desc' => "{GISBasPlug.gazetteer}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "place_list",
+	'desc' => "{GISBasPlug.place_list}",
+	'type' => "flag",
+	'reqd' => "no" } ];
+
+
+my $options = { 'name'     => "BasPlug",
+		'desc'     => "{BasPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "no",
+		'args'     => $arguments };
+
+
+sub set_keepold {
+    my $self = shift(@_);
+    my ($keepold) = @_;
+
+    $self->{'keepold'} = $keepold;
+}
+
+sub get_arguments
+{
+    my $self = shift(@_);
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $pluginoptions = pop(@$optionlistref);
+    my $pluginarguments = $pluginoptions->{'args'};
+    return $pluginarguments;
+}
+
+
+sub print_xml_usage
+{
+    my $self = shift(@_);
+
+    # XML output is always in UTF-8
+    gsprintf::output_strings_in_UTF8;
+
+    PrintUsage::print_xml_header();
+    $self->print_xml();
+}
+
+
+sub print_xml
+{
+    my $self = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $pluginoptions = shift(@$optionlistref);
+    return if (!defined($pluginoptions));
+
+    gsprintf(STDERR, "<PlugInfo>\n");
+    gsprintf(STDERR, "  <Name>$pluginoptions->{'name'}</Name>\n");
+    my $desc = gsprintf::lookup_string($pluginoptions->{'desc'});
+    $desc =~ s/</&amp;lt;/g; # doubly escaped
+    $desc =~ s/>/&amp;gt;/g;
+    
+    gsprintf(STDERR, "  <Desc>$desc</Desc>\n");
+    gsprintf(STDERR, "  <Abstract>$pluginoptions->{'abstract'}</Abstract>\n");
+    gsprintf(STDERR, "  <Inherits>$pluginoptions->{'inherits'}</Inherits>\n");
+    gsprintf(STDERR, "  <Explodes>" . ($pluginoptions->{'explodes'} || "no") . "</Explodes>\n");
+    gsprintf(STDERR, "  <Arguments>\n");
+    if (defined($pluginoptions->{'args'})) {
+	&PrintUsage::print_options_xml($pluginoptions->{'args'});
+    }
+
+    # Recurse up the plugin hierarchy
+    $self->print_xml();
+
+    gsprintf(STDERR, "  </Arguments>\n");
+    gsprintf(STDERR, "</PlugInfo>\n");
+}
+
+
+sub print_txt_usage
+{
+    my $self = shift(@_);
+    # Print the usage message for a plugin (recursively)
+    my $descoffset = $self->determine_description_offset(0);
+    $self->print_plugin_usage($descoffset, 1);
+}
+
+
+sub determine_description_offset
+{
+    my $self = shift(@_);
+    my $maxoffset = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $pluginoptions = shift(@$optionlistref);
+    return $maxoffset if (!defined($pluginoptions));
+
+    # Find the length of the longest option string of this plugin
+    my $pluginargs = $pluginoptions->{'args'};
+    if (defined($pluginargs)) {
+	my $longest = &PrintUsage::find_longest_option_string($pluginargs);
+	if ($longest > $maxoffset) {
+	    $maxoffset = $longest;
+	}
+    }
+
+    # Recurse up the plugin hierarchy
+    $maxoffset = $self->determine_description_offset($maxoffset);
+    $self->{'option_list'} = \@optionlist;
+    return $maxoffset;
+}
+
+
+sub print_plugin_usage
+{
+    my $self = shift(@_);
+    my $descoffset = shift(@_);
+    my $isleafclass = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $pluginoptions = shift(@$optionlistref);
+    return if (!defined($pluginoptions));
+
+    my $pluginname = $pluginoptions->{'name'};
+    my $pluginargs = $pluginoptions->{'args'};
+    my $plugindesc = $pluginoptions->{'desc'};
+
+    # Produce the usage information using the data structure above
+    if ($isleafclass) {
+	if (defined($plugindesc)) {
+	    gsprintf(STDERR, "$plugindesc\n\n");
+	}
+	gsprintf(STDERR, " {common.usage}: plugin $pluginname [{common.options}]\n\n");
+    }
+
+    # Display the plugin options, if there are some
+    if (defined($pluginargs)) {
+	# Calculate the column offset of the option descriptions
+	my $optiondescoffset = $descoffset + 2;  # 2 spaces between options & descriptions
+
+	if ($isleafclass) {
+	    gsprintf(STDERR, " {common.specific_options}:\n");
+	}
+	else {
+	    gsprintf(STDERR, " {common.general_options}:\n", $pluginname);
+	}
+
+	# Display the plugin options
+	&PrintUsage::print_options_txt($pluginargs, $optiondescoffset);
+    }
+
+    # Recurse up the plugin hierarchy
+    $self->print_plugin_usage($descoffset, 0);
+    $self->{'option_list'} = \@optionlist;
+}
+
+
+sub new {
+    # Set Encodings to the list!!
+
+
+    # Start the BasPlug Constructor
+    my $class = shift (@_);
+    my ($pluginlist,$args,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    my $plugin_name = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    if (GISBasPlug::has_mapdata()) {
+	push(@$arguments,@$gis_arguments);
+    }
+    
+    my $self = {};
+    $self->{'outhandle'} = STDERR;
+    $self->{'option_list'} = $hashArgOptLists->{"OptList"};
+    $self->{"info_only"} = 0;
+
+    # Check if gsdlinfo is in the argument list or not - if it is, don't parse 
+    # the args, just return the object.  
+    foreach my $strArg (@{$args})
+    {
+	if($strArg eq "-gsdlinfo")
+	{
+	    $self->{"info_only"} = 1;
+	    return bless $self, $class;
+	}
+    }
+
+    if(!parse2::parse($args,$hashArgOptLists->{"ArgList"},$self))
+    {
+	my $classTempClass = bless $self, $class;
+	print STDERR "<BadPlugin p=$plugin_name>\n";
+	&gsprintf(STDERR, "\n{BasPlug.bad_general_option}\n", $plugin_name);
+	$classTempClass->print_txt_usage("");  # Use default resource bundle
+	die "\n";
+    }
+
+
+    delete $self->{"info_only"};
+    # else parsing was successful.
+
+    $self->{'plugin_type'} = $plugin_name;
+    #$self->{'outhandle'} = STDERR;
+    $self->{'textcat'} = new textcat();
+    $self->{'num_processed'} = 0;
+    $self->{'num_not_processed'} = 0;
+    $self->{'num_blocked'} = 0;
+    $self->{'num_archives'} = 0;
+    $self->{'cover_image'} = 1; # cover image is on by default
+    $self->{'cover_image'} = 0 if ($self->{'no_cover_image'});
+    $self->{'file_blocks'} = {};
+    #$self->{'option_list'} = $hashArgOptLists->{"OptList"};
+    
+    my $associate_ext = $self->{'associate_ext'};
+    if ((defined $associate_ext) && ($associate_ext ne "")) {
+	my @exts = split(/,/,$associate_ext);
+
+	my %associate_ext_lookup = ();
+	foreach my $e (@exts) {
+	    $associate_ext_lookup{$e} = 1;
+	}
+
+	$self->{'associate_ext_lookup'} = \%associate_ext_lookup;
+    }
+
+    $self->{'shared_fileroot'} = {};
+    $self->{'file_blocks'} = {};
+
+    if ($self->{'extract_placenames'}) {
+
+	my $outhandle = $self->{'outhandle'};
+	
+	my $places_ref 
+	    = GISBasPlug::loadGISDatabase($outhandle,$self->{'gazetteer'});
+	
+	if (!defined $places_ref) {
+	    print $outhandle "Warning: Error loading mapdata gazetteer \"$self->{'gazetteer'}\"\n";
+	    print $outhandle "         No placename extraction will take place.\n";
+	    $self->{'extract_placenames'} = undef;
+	}
+	else {
+	    $self->{'places'} = $places_ref;
+	}
+    }
+    return bless $self, $class;
+    
+}
+
+# initialize BasPlug options
+# if init() is overridden in a sub-class, remember to call BasPlug::init()
+sub init {
+    my $self = shift (@_);
+    my ($verbosity, $outhandle, $failhandle) = @_;
+
+    # verbosity is passed through from the processor
+    $self->{'verbosity'} = $verbosity;
+
+    # as are the outhandle and failhandle
+    $self->{'outhandle'} = $outhandle if defined $outhandle;
+    $self->{'failhandle'} = $failhandle;
+
+    # set process_exp and block_exp to defaults unless they were
+    # explicitly set
+
+    if ((!$self->is_recursive()) and 
+	(!defined $self->{'process_exp'}) || ($self->{'process_exp'} eq "")) {
+
+	$self->{'process_exp'} = $self->get_default_process_exp ();
+	if ($self->{'process_exp'} eq "") {
+	    warn ref($self) . " Warning: Non-recursive plugin has no process_exp\n";
+	}
+    }
+
+    if ((!defined $self->{'block_exp'}) || ($self->{'block_exp'} eq "")) {
+	$self->{'block_exp'} = $self->get_default_block_exp ();
+    }
+}
+
+sub begin {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+    $self->initialise_extractors();
+}
+
+sub end {
+    # potentially called at the end of each plugin pass 
+    # import.pl only has one plugin pass, but buildcol.pl has multiple ones
+
+    my ($self) = @_;
+    $self->finalise_extractors();
+}
+
+sub deinit {
+    # called only once, after all plugin passes have been done
+
+    my ($self) = @_;
+}
+
+# this function should be overridden to return 1
+# in recursive plugins
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; 
+}
+
+sub get_default_block_exp {
+    my $self = shift (@_);
+
+    return "";
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return "";
+}
+
+# default implementation is to do nothing. 
+sub store_block_files
+{
+    my $self =shift (@_);
+    my ($filename) = @_;
+    return;
+}
+
+#default implementation is to block a file with same name as this, but extension jpg or JPG, if cover_images is on.
+sub block_cover_image
+{
+    my $self =shift;
+    my $filename = shift;
+
+    if (exists $self->{'file_blocks'}->{$filename}) {
+	# we've already checked if this file exists, so don't stat() again.
+	# (eg for multi-sectioned files using SplitPlug)
+	return;
+    }
+
+    if ($self->{'cover_image'}) {
+	my $coverfile = $filename;
+	$coverfile =~ s/\.[^\\\/\.]+$/\.jpg/;
+	if (!-e $coverfile) {
+	    $coverfile =~ s/jpg$/JPG/;
+	} 	
+	if (-e $coverfile) {
+	    $self->{'file_blocks'}->{$coverfile} = 1;
+	} else {
+	    $self->{'file_blocks'}->{$coverfile} = 0;
+	}
+    }
+
+    return;
+}
+    
+sub metadata_read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    # Keep track of filenames with same root but different extensions
+    # Used to support -associate_ext
+
+    my $associate_ext = $self->{'associate_ext'};
+    if ((defined $associate_ext) && ($associate_ext ne "")) {
+
+	my ($file_prefix,$file_ext) = ($file =~ m/^(.*)\.(.*?)$/);
+	if ((defined $file_prefix) && (defined $file_ext)) {
+	    
+	    my $shared_fileroot = $self->{'shared_fileroot'};
+	    if (!defined $shared_fileroot->{$file_prefix}) {
+		my $file_prefix_rec = { 'tie_to' => undef, 'exts' => {} };
+		$shared_fileroot->{$file_prefix} = $file_prefix_rec;
+	    }
+	    
+	    my $file_prefix_rec = $shared_fileroot->{$file_prefix};
+
+	    my $process_exp = $self->{'process_exp'};
+
+	    if ($file =~ m/$self->{'process_exp'}/) {
+		# This is the document the others should be tied to
+		$file_prefix_rec->{'tie_to'} = $file_ext;
+	    }
+	    else {
+		if (defined $self->{'associate_ext_lookup'}->{$file_ext}) {
+		    $file_prefix_rec->{'exts'}->{$file_ext} = 1;
+		}
+	    }
+	}
+    }
+    
+    # now check whether we are actually processing this
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    if ($self->{'process_exp'} eq "" || $filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef; # can't recognise
+    }
+
+    # do smart blocking if appropriate
+    if (defined $self->{'smart_block'}) {
+	$self->block_cover_image($filename);
+	$self->store_block_files($filename);
+    }
+       
+    return 1;
+}
+
+sub tie_to_filename
+{
+    my $self = shift (@_);  
+  
+    my ($file_ext,$file_prefix_rec) = @_;
+
+    if (defined $file_prefix_rec) {
+	my $tie_to = $file_prefix_rec->{'tie_to'};
+
+	if (defined $tie_to) {
+	    if ($tie_to eq $file_ext) {
+		return 1;
+	    }
+	}
+    }
+
+    return 0;
+}
+
+sub tie_to_assoc_file
+{
+    my $self = shift (@_);    
+    my ($file_ext,$file_prefix_rec) = @_;
+
+    if (defined $file_prefix_rec) {
+	my $tie_to = $file_prefix_rec->{'tie_to'};
+	if (defined $tie_to) {
+
+	    my $exts = $file_prefix_rec->{'exts'};
+
+	    my $has_file_ext = $exts->{$file_ext};
+	    
+	    if ($has_file_ext) {
+		return 1;
+	    }
+	}
+    }
+
+    return 0;
+}
+
+
+sub associate_with
+{
+    my $self = shift (@_);    
+    my ($file, $filename, $metadata) = @_;
+
+    my $associate_ext = $self->{'associate_ext'};
+
+
+    return 0 if (!$associate_ext);
+
+    # If file, see if matches with "tie_to" doc or is one of the
+    # associated filename extensions.
+
+    my ($file_prefix,$file_ext) = ($file =~ m/^(.*)\.(.*?)$/);
+    if ((defined $file_prefix) && (defined $file_ext)) {
+
+	my $file_prefix_rec = $self->{'shared_fileroot'}->{$file_prefix};
+	
+	if ($self->tie_to_filename($file_ext,$file_prefix_rec)) {
+
+	    # Set up gsdlassocfile_tobe
+
+	    my $exts = $file_prefix_rec->{'exts'};
+		
+	    if (!defined $metadata->{'gsdlassocfile_tobe'}) {
+		$metadata->{'gsdlassocfile_tobe'} = [];
+	    }
+
+	    my $assoc_tobe = $metadata->{'gsdlassocfile_tobe'};
+			
+	    my ($full_prefix) = ($filename =~ m/^(.*)\..*?$/);
+	    foreach my $e (keys %$exts) {		
+		my $assoc_file = "$full_prefix.$e";
+		my $mime_type = ""; # let system auto detect this
+		push(@$assoc_tobe,"$assoc_file:$mime_type:"); 
+	    }
+	}
+	elsif ($self->tie_to_assoc_file($file_ext,$file_prefix_rec)) {
+	    # a form of smart block
+	    
+	    return 1;
+	}
+    }
+
+    return 0;
+}
+
+
+sub read_block {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($self->associate_with($file,$filename,$metadata)) {
+	# a form of smart block
+	$self->{'num_blocked'} ++;
+	return (0,undef); # blocked
+    }
+
+    my $smart_block = $self->{'smart_block'};
+    my $smart_block_BN = $self->{'smart_block_BN'};
+
+    if ($smart_block || $smart_block_BN) {
+	if (defined $self->{'file_blocks'}->{$filename} && $self->{'file_blocks'}->{$filename} == 1){
+	    $self->{'num_blocked'} ++;
+	    return (0,undef); # blocked
+	}
+    } elsif ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return (0,undef); # blocked
+    }
+    
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return (undef,undef); # can't recognise
+    }
+    
+    return (1,$filename);
+}
+
+sub read_tidy_file {
+
+    my $self = shift (@_);  
+  
+    my ($file) = @_;
+
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    return $file;
+}
+
+
+
+# The BasPlug read_into_doc_obj() function. This function does all the
+# right things to make general options work for a given plugin.  It reads in
+# a file and sets up a slew of metadata all saved in doc_obj, which
+# it then returns as part of a tuple (process_status,doc_obj)
+#
+# Much of this functionality used to reside in read, but it was broken
+# down into a supporting routine to make the code more flexible.  
+#
+# recursive plugins (e.g. RecPlug) and specialized plugins like those
+# capable of processing many documents within a single file (e.g.
+# GMLPlug) will normally want to implement their own version of
+# read_into_doc_obj()
+#
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read_into_doc_obj {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    if ($self->is_recursive()) {
+	gsprintf(STDERR, "{BasPlug.read_must_be_implemented}") && die "\n";
+    }
+
+    my $outhandle   = $self->{'outhandle'};
+
+    my ($block_status,$filename) = $self->read_block(@_);    
+    return $block_status if ((!defined $block_status) || ($block_status==0));
+    $file = $self->read_tidy_file($file);
+
+    # Do encoding stuff
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    my ($filemeta) = $file =~ /([^\\\/]+)$/;
+    # how do we know what encoding the filename is in?
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+    if ($self->{'cover_image'}) {
+	$self->associate_cover_image($doc_obj, $filename);
+    }
+    
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($filename, $encoding, $language, \$text);
+
+    if (!length ($text)) {
+	my $plugin_name = ref ($self);
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='File contains no text'>\n";
+	}
+	gsprintf($outhandle, "$plugin_name: {BasPlug.file_has_no_text}\n", $filename) if $self->{'verbosity'};
+
+	my $failhandle = $self->{'failhandle'};
+	gsprintf($failhandle, "$file: " . ref($self) . ": {BasPlug.empty_file}\n");
+	# print $failhandle "$file: " . ref($self) . ": file contains no text\n";
+	$self->{'num_not_processed'} ++;
+
+	return (0,undef); # what should we return here?? error but don't want to pass it on
+    }
+   
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
+	$text = '';
+	undef $text;
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return (-1,undef);
+    }
+    $text='';
+    undef $text;
+   
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+   
+    # add an OID
+    # see if there is a plugin-specific set_OID function...
+    if (defined ($self->can('set_OID'))) {
+	# it will need $doc_obj to set the Identifier metadata...
+	$self->set_OID($doc_obj);
+    } else {
+	# use the default set_OID() in doc.pm
+	$doc_obj->set_OID();
+    }
+    
+    return (1,$doc_obj);
+}
+
+
+# The BasPlug read() function. This function calls read_into_doc_obj()
+# to ensure all the right things to make general options work for a
+# given plugin are done. It then calls the process() function which
+# does all the work specific to a plugin (like the old read functions
+# used to do). Most plugins should define their own process() function
+# and let this read() function keep control.  
+#
+# recursive plugins (e.g. RecPlug) and specialized plugins like those
+# capable of processing many documents within a single file (e.g.
+# GMLPlug) might want to implement their own version of read(), but
+# more likely need to implement their own version of read_into_doc_obj()
+#
+# Return number of files processed, undef if can't recognise, -1 if can't
+# process
+
+sub read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
+    
+    if ((defined $process_status) && ($process_status == 1)) {
+	# process the document
+	$processor->process($doc_obj);
+	
+	if(defined($self->{'places_filename'})){
+	    &util::rm($self->{'places_filename'});
+	    $self->{'places_filename'} = undef;
+	}
+	
+	$self->{'num_processed'} ++;
+	undef $doc_obj; 
+    }
+
+    # if process_status == 1, then the file has been processed.
+    return $process_status;
+
+}
+
+# returns undef if file is rejected by the plugin
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+
+    gsprintf(STDERR, "BasPlug::process {common.must_be_implemented}\n") && die "\n";
+    # die "Basplug::process function must be implemented in sub-class\n";
+
+    return undef; # never gets here
+}
+
+# uses the multiread package to read in the entire file pointed to
+# by filename and loads the resulting text into $$textref. Input text
+# may be in any of the encodings handled by multiread, output text
+# will be in utf8
+sub read_file {
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    if (!-r $filename)
+    {
+	my $outhandle = $self->{'outhandle'};
+	gsprintf($outhandle, "{BasPlug.read_denied}\n", $filename) if $self->{'verbosity'};
+	# print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
+	return;
+    }
+    $$textref = "";
+    if (!open (FILE, $filename)) {
+	gsprintf(STDERR, "BasPlug::read_file {BasPlug.could_not_open_for_reading} ($!)\n", $filename);
+	die "\n";
+    }
+     
+    if ($encoding eq "ascii") {
+	undef $/;
+	$$textref = <FILE>;
+	$/ = "\n";
+    } else {
+	my $reader = new multiread();
+	$reader->set_handle ('BasPlug::FILE');
+	$reader->set_encoding ($encoding);
+	$reader->read_file ($textref);
+        #Now segments chinese if the separate_cjk option is set
+	if ($self->{'separate_cjk'}) {
+	    # segment the Chinese words
+	    $$textref = &cnseg::segment($$textref);
+	}
+    }
+    close FILE;
+}
+
+# write_file -- used by ConvertToPlug, for example in post processing
+#
+sub utf8_write_file {
+    my $self = shift (@_);
+    my ($textref, $filename) = @_;
+    
+    if (!open (FILE, ">$filename")) {
+	gsprintf(STDERR, "ConvertToPlug::write_file {ConvertToPlug.could_not_open_for_writing} ($!)\n", $filename);
+	 die "\n";
+     }
+    print FILE $$textref;
+    
+    close FILE;
+}
+
+
+sub filename_based_title
+{
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    my $file_derived_title = $file;
+    $file_derived_title =~ s/_/ /g;
+    $file_derived_title =~ s/\..*?$//;
+
+    return $file_derived_title;
+}
+
+
+sub title_fallback
+{
+    my $self = shift (@_);
+    my ($doc_obj,$section,$file) = @_;
+
+    if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
+
+	my $file_derived_title = $self->filename_based_title($file);
+	$doc_obj->add_metadata ($section, "Title", $file_derived_title);
+    }
+}
+
+sub textcat_get_language_encoding {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    
+    my ($language, $encoding, $extracted_encoding);
+    if ($self->{'input_encoding'} eq "auto") {
+        # use textcat to automatically work out the input encoding and language
+        ($language, $encoding) = $self->get_language_encoding ($filename);
+    } elsif ($self->{'extract_language'}) {
+        # use textcat to get language metadata
+        ($language, $extracted_encoding) = $self->get_language_encoding ($filename);
+        $encoding = $self->{'input_encoding'};
+	# don't print this message for english... english in utf8 is identical
+	# to english in iso-8859-1 (except for some punctuation). We don't have
+	# a language model for en_utf8, so textcat always says iso-8859-1!
+        if ($extracted_encoding ne $encoding && $language ne "en"
+	    && $self->{'verbosity'}) {
+	    my $plugin_name = ref ($self);
+	    my $outhandle = $self->{'outhandle'};
+	    gsprintf($outhandle, "$plugin_name: {BasPlug.wrong_encoding}\n", $filename, $encoding, $extracted_encoding);
+            # print $outhandle "$plugin_name: WARNING: $filename was read using $encoding encoding but ";
+            # print $outhandle "appears to be encoded as $extracted_encoding.\n";
+        }
+    } else {
+        $language = $self->{'default_language'};
+        $encoding = $self->{'input_encoding'};
+    }
+
+    return ($language, $encoding);
+}
+
+# Uses textcat to work out the encoding and language of the text in 
+# $filename. All html tags are removed before processing.
+# returns an array containing "language" and "encoding"
+sub get_language_encoding {
+    my $self = shift (@_);
+    my ($filename) = @_;
+    my $outhandle = $self->{'outhandle'};
+    my $unicode_format = "";
+    # read in file
+    open (FILE, $filename) || (gsprintf(STDERR, "BasPlug::get_language_encoding {BasPlug.could_not_open_for_reading} ($!)\n", $filename) && die "\n"); # die "BasPlug::get_language_encoding could not open $filename for reading ($!)\n";
+    undef $/;
+    my $text = <FILE>;
+    $/ = "\n";
+    close FILE;
+
+    # check if first few bytes have a Byte Order Marker
+    my $bom=substr($text,0,2); # check 16bit unicode
+    if ($bom eq "\xff\xfe") { # little endian 16bit unicode
+	$unicode_format="unicode";
+    } elsif ($bom eq "\xfe\xff") { # big endian 16bit unicode
+	$unicode_format="unicode";
+    } else {
+	$bom=substr($text,0,3); # check utf-8
+	if ($bom eq "\xef\xbb\xbf") { # utf-8 coded FEFF bom
+	    $unicode_format="utf8";
+#	} elsif ($bom eq "\xef\xbf\xbe") { # utf-8 coded FFFE bom. Error!?
+#	    $unicode_format="utf8";
+	}
+    }
+    
+    # VB scripting generated Word to HTML file
+    if ($text =~ /charset=(windows.*)[\"]/ig){
+	my $vbhtml_encoding = $1;
+	$vbhtml_encoding =~ s/-+/_/g;
+	$self->{'input_encoding'} = $vbhtml_encoding;
+    }
+    
+    # remove <title>stuff</title> -- as titles tend often to be in English
+    # for foreign language documents
+    $text =~ s/<title>(.|\n)*?<\/title>//i;
+
+    # remove all HTML tags
+    # XXX this doesn't match plugins derived from HTMLPlug (except ConvertTo)
+    if (ref($self) eq 'HTMLPlug' ||
+	(exists $self->{'converted_to'} && $self->{'converted_to'} eq 'HTML')){
+	$text =~ s/<[^>]*>//sg;
+    }
+
+    # get the language/encoding
+    my $results = $self->{'textcat'}->classify(\$text);
+
+    # if textcat returns 3 or less possibilities we'll use the 
+    # first one in the list - otherwise use the defaults
+    if (scalar @$results > 3) {
+	my $best_encoding="";
+	if ($unicode_format) { # in case the first had a BOM
+	    $best_encoding=$unicode_format;
+	} else {
+	    my %guessed_encodings = ();
+	    foreach my $result (@$results) {
+		$result =~ /([^\-]+)$/;
+		my $enc=$1;
+		if (!defined($guessed_encodings{$enc})) {
+		    $guessed_encodings{$enc}=0;
+		}
+		$guessed_encodings{$enc}++;
+	    }
+
+	    $guessed_encodings{""}=-1; # for default best_encoding of ""
+	    foreach my $enc (keys %guessed_encodings) {
+		if ($guessed_encodings{$enc} >
+		    $guessed_encodings{$best_encoding}){
+		    $best_encoding=$enc;
+		}
+	    }
+	}
+
+	if ($self->{'input_encoding'} ne 'auto') {
+	    if ($self->{'extract_language'} && ($self->{'verbosity'}>2)) {
+		gsprintf($outhandle,
+			 "BasPlug: {BasPlug.could_not_extract_language}\n",
+			 $filename, $self->{'default_language'});
+	    }		
+	    return ($self->{'default_language'}, $self->{'input_encoding'});
+
+	} else {
+	    if ($self->{'verbosity'}>2) {
+		gsprintf($outhandle,
+			 "BasPlug: {BasPlug.could_not_extract_language}\n",
+			 $filename, $self->{'default_language'});
+	    }
+	    return ($self->{'default_language'}, $best_encoding);
+	}
+    }
+
+    # format language/encoding
+    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
+    if (!defined $language) {
+	if ($self->{'verbosity'}>2) {
+	    gsprintf($outhandle,
+		     "BasPlug: {BasPlug.could_not_extract_language}\n",
+		     $filename, $self->{'default_language'});
+	}
+	$language = $self->{'default_language'};
+    }
+    if (!defined $encoding) {
+	if ($self->{'verbosity'}>2) {
+	    gsprintf($outhandle,
+		     "BasPlug: {BasPlug.could_not_extract_encoding}\n",
+		     $filename, $self->{'default_encoding'});
+	}
+	$encoding = $self->{'default_encoding'};
+    }
+
+
+    # check for equivalents where textcat doesn't have some encodings...
+    # eg MS versions of standard encodings
+    if ($encoding =~ /^iso_8859_(\d+)/) {
+	my $iso = $1; # which variant of the iso standard?
+	# iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
+	if ($text =~ /[\x80-\x9f]/) {
+	    # Western Europe
+	    if ($iso == 1 or $iso == 15) { $encoding = 'windows_1252' }
+	    elsif ($iso == 2) { $encoding = 'windows_1250' } # Central Europe
+	    elsif ($iso == 5) { $encoding = 'windows_1251' } # Cyrillic
+	    elsif ($iso == 6) { $encoding = 'windows_1256' } # Arabic
+	    elsif ($iso == 7) { $encoding = 'windows_1253' } # Greek
+	    elsif ($iso == 8) { $encoding = 'windows_1255' } # Hebrew
+	    elsif ($iso == 9) { $encoding = 'windows_1254' } # Turkish
+	}
+    }
+
+    if ($encoding !~ /^(ascii|utf8|unicode)$/ &&
+	!defined $encodings::encodings->{$encoding}) {
+	if ($self->{'verbosity'}) {
+	    gsprintf($outhandle, "BasPlug: {BasPlug.unsupported_encoding}\n",
+		     $filename, $encoding, $self->{'default_encoding'});
+	}
+	$encoding = $self->{'default_encoding'};
+    }
+
+    return ($language, $encoding);
+}
+
+# add any extra metadata that's been passed around from one
+# plugin to another.
+# extra_metadata uses add_utf8_metadata so it expects metadata values
+# to already be in utf8
+sub extra_metadata {
+    my $self = shift (@_);
+    my ($doc_obj, $cursection, $metadata) = @_;
+
+    foreach my $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if ($field eq "gsdlassocfile_tobe") {
+	    # 'gsdlassocfile_tobe' is artificially introduced metadata
+	    # that is used to signal that certain additional files should
+	    # be tied to this document.  Useful in situations where a
+	    # metadata pass in the plugin pipeline works out some files
+	    # need to be associated with a document, but the document hasn't
+	    # been formed yet.
+	    
+	    my $equiv_form = "";
+	    foreach my $gaf (@{$metadata->{$field}}) {
+		my ($full_filename,$mimetype) = ($gaf =~ m/^(.*):(.*):$/);
+		my ($tail_filename) = ($full_filename =~ /^.*[\/\\](.+?)$/);
+		my $filename = $full_filename;
+			      
+		$doc_obj->associate_file($full_filename,$tail_filename,$mimetype);
+		
+		my ($doc_ext) = ($tail_filename =~ m/^.*\.(.*)$/);
+		my $start_doclink = "<a href=\"_httpcollection_/index/assoc/{Or}{[parent(Top):archivedir],[archivedir]}/$tail_filename\">";
+		my $srcicon = "_icon".$doc_ext."_";
+		my $end_doclink = "</a>";
+
+		$equiv_form .= " $start_doclink\{If\}{$srcicon,$srcicon,$doc_ext\}$end_doclink";		
+	    }
+	    $doc_obj->add_utf8_metadata ($cursection, "equivlink", $equiv_form); 
+	}
+	elsif (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_utf8_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+}
+
+# initialise metadata extractors
+sub initialise_extractors {
+    my $self = shift (@_);
+
+    if ($self->{'extract_acronyms'} || $self->{'markup_acronyms'}) {
+	&acronym::initialise_acronyms();
+    }
+}
+
+# finalise metadata extractors
+sub finalise_extractors {
+    my $self = shift (@_);
+
+    if ($self->{'extract_acronyms'} || $self->{'markup_acronyms'}) {
+	&acronym::finalise_acronyms();
+    }
+}
+
+# FIRSTNNN: extract the first NNN characters as metadata
+sub extract_first_NNNN_characters {
+    my $self = shift (@_);
+    my ($textref, $doc_obj, $thissection) = @_;
+    
+    foreach my $size (split /,/, $self->{'first'}) {
+	my $tmptext =  $$textref;
+	$tmptext =~ s/^\s+//;
+	$tmptext =~ s/\s+$//;
+	$tmptext =~ s/\s+/ /gs;
+	$tmptext = substr ($tmptext, 0, $size);
+	$tmptext =~ s/\s\S*$/&#8230;/;
+	$doc_obj->add_utf8_metadata ($thissection, "First$size", $tmptext);
+    }
+}
+
+sub extract_email {
+    my $self = shift (@_);
+    my ($textref, $doc_obj, $thissection) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    gsprintf($outhandle, " {BasPlug.extracting_emails}...\n")
+	if ($self->{'verbosity'} > 2);
+    
+    my @email = ($$textref =~ m/([-a-z0-9\.@+_=]+@(?:[-a-z0-9]+\.)+(?:com|org|edu|mil|int|net|[a-z][a-z]))/g);
+    @email = sort @email;
+    
+#    if($self->{"new_extract_email"} == 0)
+#    {
+#    my @email2 = ();
+#    foreach my $address (@email) 
+#   {
+#	if (!(join(" ",@email2) =~ m/(^| )$address( |$)/ )) 
+#	    {
+#		push @email2, $address;
+#		$doc_obj->add_utf8_metadata ($thissection, "emailAddress", $address);
+#		# print $outhandle "  extracting $address\n" 
+#		&gsprintf($outhandle, "  {BasPlug.extracting} $address\n")
+#		    if ($self->{'verbosity'} > 3);
+#	    }
+#	}
+#    }
+#    else
+#    {
+    my $hashExistMail = {};
+    foreach my $address (@email) {
+	if (!(defined $hashExistMail->{$address}))
+	{
+	    $hashExistMail->{$address} = 1;
+	    $doc_obj->add_utf8_metadata ($thissection, "emailAddress", $address);
+	    gsprintf($outhandle, "  {BasPlug.extracting} $address\n")
+		if ($self->{'verbosity'} > 3);
+	}
+    }
+    gsprintf($outhandle, " {BasPlug.done_email_extract}\n")
+	if ($self->{'verbosity'} > 2);
+}
+
+# extract metadata
+sub auto_extract_metadata {
+
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    
+    if ($self->{'extract_email'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $self->extract_email (\$text, $doc_obj, $thissection) if $text =~ /./;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    } 
+    if ($self->{'extract_placenames'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $self->extract_placenames (\$text, $doc_obj, $thissection) if $text =~ /./;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    } 
+
+    #adding kea keyphrases
+    if ($self->{'extract_keyphrases'}) {  
+	
+	my $thissection = $doc_obj->get_top_section();
+	my $text = "";
+	my $list;
+
+	#loop through sections to gather whole doc
+	while (defined $thissection) { 
+	    my $sectiontext = $doc_obj->get_text($thissection);   
+	    $text = $text.$sectiontext;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	} 
+       
+	if($self->{'extract_keyphrase_options'}) { #if kea options flag is set, call Kea with specified options 
+	    $list = &Kea::extract_KeyPhrases ($text, $self->{'extract_keyphrase_options'}); 
+	} else { #otherwise call Kea with no options
+	    $list = &Kea::extract_KeyPhrases ($text); 
+	}
+	 
+	if ($list){
+	    # if a list of kea keyphrases was returned (ie not empty)
+	    if ($self->{'verbosity'}) {
+		gsprintf(STDERR, "{BasPlug.keyphrases}: $list\n");
+	    }
+
+	    #add metadata to top section
+	    $thissection = $doc_obj->get_top_section(); 
+
+	    # add all key phrases as one metadata
+	    $doc_obj->add_metadata($thissection, "Keyphrases", $list);
+
+	    # add individual key phrases as multiple metadata
+	    foreach my $keyphrase (split(',', $list)) {
+	    	$keyphrase =~ s/^\s+|\s+$//g;
+		$doc_obj->add_metadata($thissection, "Keyphrase", $keyphrase);
+	    }
+	}
+    } #end of kea 
+
+    if ($self->{'first'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $self->extract_first_NNNN_characters (\$text, $doc_obj, $thissection) if $text =~ /./;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }   
+    
+    if ($self->{'extract_acronyms'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $self->extract_acronyms (\$text, $doc_obj, $thissection) if $text =~ /./;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+    
+    if ($self->{'markup_acronyms'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $text = $self->markup_acronyms ($text, $doc_obj, $thissection);
+	    $doc_obj->delete_text($thissection);
+	    $doc_obj->add_text($thissection, $text);
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+
+    if($self->{'extract_historical_years'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+
+	    my $text = $doc_obj->get_text($thissection);
+	    &DateExtract::get_date_metadata($text, $doc_obj, 
+					    $thissection, 
+					    $self->{'no_bibliography'}, 
+					    $self->{'maximum_year'}, 
+					    $self->{'maximum_century'});
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+}
+
+# extract acronyms from a section in a document. progress is 
+# reported to outhandle based on the verbosity. both the Acronym
+# and the AcronymKWIC metadata items are created. 
+
+sub extract_acronyms {
+    my $self = shift (@_);
+    my ($textref, $doc_obj, $thissection) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # print $outhandle " extracting acronyms ...\n" 
+    gsprintf($outhandle, " {BasPlug.extracting_acronyms}...\n")
+	if ($self->{'verbosity'} > 2);
+
+    my $acro_array =  &acronym::acronyms($textref);
+    
+    foreach my $acro (@$acro_array) {
+
+	#check that this is the first time ...
+	my $seen_before = "false";
+	my $previous_data = $doc_obj->get_metadata($thissection, "Acronym");
+	foreach my $thisAcro (@$previous_data) {
+	    if ($thisAcro eq $acro->to_string()) {
+		$seen_before = "true";
+		if ($self->{'verbosity'} >= 4) {
+		    gsprintf($outhandle, " {BasPlug.already_seen} " .
+			     $acro->to_string() . "\n");
+		}
+	    }
+	}
+
+	if ($seen_before eq "false") {
+	    #write it to the file ...
+	    $acro->write_to_file();
+
+	    #do the normal acronym
+	    $doc_obj->add_utf8_metadata($thissection, "Acronym",  $acro->to_string());
+	    gsprintf($outhandle, " {BasPlug.adding} ".$acro->to_string()."\n")
+		if ($self->{'verbosity'} > 3);
+	}
+    }
+
+    gsprintf($outhandle, " {BasPlug.done_acronym_extract}\n")
+	if ($self->{'verbosity'} > 2);
+}
+
+sub markup_acronyms {
+    my $self = shift (@_);
+    my ($text, $doc_obj, $thissection) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    gsprintf($outhandle, " {BasPlug.marking_up_acronyms}...\n")
+	if ($self->{'verbosity'} > 2);
+
+    #self is passed in to check for verbosity ...
+    $text = &acronym::markup_acronyms($text, $self);
+
+    gsprintf($outhandle, " {BasPlug.done_acronym_markup}\n")
+	if ($self->{'verbosity'} > 2);
+
+    return $text;
+}
+
+sub compile_stats {
+    my $self = shift(@_);
+    my ($stats) = @_;
+
+    $stats->{'num_processed'} += $self->{'num_processed'};
+    $stats->{'num_not_processed'} += $self->{'num_not_processed'};
+    $stats->{'num_archives'} += $self->{'num_archives'};
+
+}
+
+sub associate_cover_image {
+    my $self = shift;
+    my ($doc_obj, $filename) = @_;
+
+    $filename =~ s/\.[^\\\/\.]+$/\.jpg/;
+    if (exists $self->{'covers_missing_cache'}->{$filename}) {
+	# don't stat() for existence eg for multiple document input files
+	# (eg SplitPlug)
+	return;
+    }
+
+    my $top_section=$doc_obj->get_top_section();
+
+    if (-e $filename) {
+	$doc_obj->associate_file($filename, "cover.jpg", "image/jpeg");
+	$doc_obj->add_utf8_metadata($top_section, "hascover",  1);
+    } else {
+	my $upper_filename = $filename;
+	$upper_filename =~ s/jpg$/JPG/;
+	if (-e $upper_filename) {
+	    $doc_obj->associate_file($upper_filename, "cover.jpg",
+				     "image/jpeg");
+	    $doc_obj->add_utf8_metadata($top_section, "hascover",  1);
+	} else {
+	    # file doesn't exist, so record the fact that it's missing so
+	    # we don't stat() again (stat is slow)
+	    $self->{'covers_missing_cache'}->{$filename} = 1;
+	}
+    }
+
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/BibTexPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/BibTexPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/BibTexPlug.pm	(revision 21112)
@@ -0,0 +1,821 @@
+###########################################################################
+#
+# BibTexPlug.pm - a plugin for bibliography records in BibTex format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2000 Gordon W. Paynter
+# Copyright 1999-2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# BibTexPlug reads bibliography files in BibTex format.
+#
+# by Gordon W. Paynter (gwp@cs.waikato.ac.nz), November 2000
+# Based on ReferPlug.  See ReferPlug for geneology.
+#
+# BibTexPlug creates a document object for every reference a the file.
+# It is a subclass of SplitPlug, so if there are multiple records, all
+# are read.
+#
+# Modified Dec 2001 by John McPherson:
+#  *  some modifications submitted by Sergey Yevtushenko
+#                      <sergey@intellektik.informatik.tu-darmstadt.de>
+#  *  some non-ascii char support (ie mostly Latin)
+
+
+package BibTexPlug;
+
+use SplitPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+# BibTexPlug is a sub-class of BasPlug.
+sub BEGIN {
+    @BibTexPlug::ISA = ('SplitPlug');
+}
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_split_exp(),
+	'reqd' => "no" }
+      ];
+
+my $options = { 'name'     => "BibTexPlug",
+		'desc'     => "{BibTexPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'explodes' => "yes",
+		'args'     => $arguments };
+
+# This plugin processes files with the suffix ".bib"
+sub get_default_process_exp {
+    return q^(?i)\.bib$^;
+}
+
+# This plugin splits the input text at blank lines
+sub get_default_split_exp {
+    return q^\n+(?=@)^;
+}
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+
+
+# The process function reads a single bibliographic record and stores
+# it as a new document.
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $cursection = $doc_obj->get_top_section();
+    $self->{'key'} = "default";
+
+    # Check that we're dealing with a valid BibTex record
+    return undef unless ($$textref =~ /^@\w+\{.*\}/s);
+
+    # Ignore things we can't use
+    return 0 if ($$textref =~ /^\@String/);
+
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='BibTexPlug'>\n" if ($gli);
+    print $outhandle "BibTexPlug: processing $file\n"
+	if ($self->{'verbosity'}) > 1;
+
+
+    # This hash translates BibTex field names into metadata names.  The
+    # BibTex names are taken from the "Local Guide to Latex" Graeme
+    # McKinstry.  Metadata names are consistent with ReferPlug.
+
+    # The author metadata will be stored as one "Creator" entry, but will
+    # also be split up into several individual "Author" fields.
+
+    my %field = (
+		 'address', 'PublisherAddress',
+		 'author', 'Creator',
+		  
+		 'booktitle', 'Booktitle',
+		 'chapter', 'Chapter',
+		 'edition', 'Edition',
+		 'editor', 'Editor', 
+		 'institution', 'Publisher',
+		 'journal', 'Journal',
+		 'month', 'Month',
+		 'number', 'Number',
+		 'pages', 'Pages',
+		 'publisher', 'Publisher',
+		 'school', 'Publisher',
+		 'title', 'Title',
+		 'volume', 'Volume',
+		 'year', 'Year', # Can't use "Date" as this implies DDDDMMYY!
+
+		 'keywords', 'Keywords',
+		 'abstract', 'Abstract',
+		 'copyright', 'Copyright',
+		 'note', 'Note',
+		 );
+
+    # Metadata fields 
+    my %metadata;
+    my ($EntryType, $Creator, $Keywords, $text);
+
+    my $verbosity = $self->{'verbosity'};
+    $verbosity = 0 unless $verbosity;
+
+    # Make sure the text has exactly one entry per line -
+    # append line to previous if it doesn't start with "  <key> = "
+
+    my @input_lines=split(/\r?\n/, $$textref);
+    my @all_lines;
+    my $entry_line=shift @input_lines;
+    foreach my $input_line (@input_lines) {
+	if ($input_line =~ m/^\s*\w+\s*=\s*/) {
+	    # this is a new key
+	    push(@all_lines, $entry_line);
+	    $entry_line=$input_line;
+	} else {
+	    # this is a continuation of previous line
+	    $entry_line .= " " . $input_line;
+	}
+	
+    }
+    # add final line, removing trailing '}'
+    $entry_line =~ s/\}\s*$//;
+    push(@all_lines, $entry_line);
+    push(@all_lines, "}");
+
+    # Read and process each line in the bib file.
+    my ($entryname, $name, $value, $line);
+    foreach $line (@all_lines) {
+
+	# Add each line.  Most lines consist of a field identifer and
+	# then data, and we simply store them, though we treat some
+	# of the fields a bit differently.
+
+	$line =~ s/\s+/ /g;
+	$text .= "$line\n";
+
+
+	print "Processing line = $line \n" if $verbosity>=4;
+
+	# The first line is special, it contains the reference type and OID
+	if ($line =~ /\@(\w+)\W*\{\W*([\*\.\w\d:-]*)\W*$/) {
+	    $EntryType = $1;
+	    my $EntryID = (defined $2) ? $2 : "default";
+	    print "** $EntryType - \"$EntryID\" \n"
+		if ($verbosity >= 4);
+
+	    next;
+	}
+	if ($line =~ /\@/) {
+	    print $outhandle "bibtexplug: suspect line in bibtex file: $line\n"
+		if ($verbosity >= 2);
+	    print $outhandle "bibtexplug: if that's the start of a new bibtex record ammend regexp in bibtexplug::process()\n"
+		if ($verbosity >= 2);
+	}
+	
+	# otherwise, parse the metadata out of this line
+	next unless ($line =~ /^\s*(\w+)\s*=\s*(.*)/);
+	$entryname = lc($1);
+	$value = $2;
+	$value =~ s/,?\s*$//; # remove trailing comma and space
+	if ($value =~ /^"/ && $value =~ /"$/) {
+	    # remove surrounding " marks
+	    $value =~ s/^"//; $value =~ s/"$//;
+	}
+	$value = &process_latex($value);
+
+	# Add this line of metadata
+	$metadata{$entryname} .= "$value";
+	   
+    }
+
+    # Add the Entry type as metadata
+    $doc_obj->add_utf8_metadata ($cursection, "EntryType", $EntryType);
+    
+    #Add the fileformat as metadata
+    $doc_obj->add_metadata($cursection, "FileFormat", "BibTex");
+
+
+    # Add the various field as metadata
+    foreach my $entryname (keys %metadata) {
+	next unless (defined $field{$entryname});
+	next unless (defined $metadata{$entryname});	
+	
+	$name = $field{$entryname};
+	$value = $metadata{$entryname};
+
+	if ($name =~ /^Month/) {
+	    $value=expand_month($value);
+	}
+
+	# Several special operatons on metadata follow
+	
+	# Add individual keywords.
+	# The full set of keywords will be added, in due course, as "Keywords".
+	# However, we also want to add them as individual "Keyword" metadata elements.
+	if ($entryname eq "keywords") {
+	    my @keywordlist = split(/,/, $value);
+	    foreach my $k (@keywordlist) {
+		$k = lc($k); 
+		$k =~ s/\s*$//; 
+		$k =~ s/^\s*//; 
+		if ($k =~ /\w/) {
+		    $doc_obj->add_utf8_metadata ($cursection, "Keyword", $k);
+		}
+	    } 
+	}
+	
+	# Add individual authors
+	# The author metadata will be stored as one "Creator" entry, but we
+	# also want to split it into several individual "Author" fields in
+	# "Lastname, Firstnames" format so we can browse it.
+	if ($entryname eq "author") { #added also comparison with editor
+	   
+	    # take care of "et al."...
+	    my $etal='';
+	    if ($value =~ s/\s+(and\s+others|et\.?\s+al\.?)\s*$//i) {
+		$etal=' <em>et. al.</em>';
+	    }
+	    # und here for german language...
+	    # don't use brackets in pattern, else the matched bit becomes
+	    # an element in the list!
+	    my @authorlist = split(/\s+and\s+|\s+und\s+/, $value); 
+	    my @formattedlist = ();
+	    foreach $a (@authorlist) {
+		$a =~ s/\s*$//; 
+		$a =~ s/^\s*//; 
+		# Reformat and add author name
+		next if $a=~ /^\s*$/;
+		
+		# names are "First von Last", "von Last, First"
+		# or "von Last, Jr, First". See the "BibTeXing" manual, page 16
+		my $first="";
+		my $vonlast="";
+		my $jr="";
+
+		if ($a =~ /,/) {
+		    my @parts=split(/,\s*/, $a);
+		    $first = pop @parts;
+		    if (scalar(@parts) == 2) {
+			$jr = pop @parts;
+		    }
+		    $vonlast=shift @parts;
+		    if (scalar(@parts) > 0) {
+			print $outhandle "BibTexPlug: couldn't parse name $a\n";
+			# but we continue anyway...
+		    }
+		} else { # First von Last
+		    my @words = split(/ /, $a);
+		    while (scalar(@words) > 1 && $words[0] !~ /^[a-z]{2..}/) {
+			$first .= " " . shift (@words);
+		    }
+		    $first =~ s/^\s//;
+		    $vonlast = join (' ', @words); # whatever's left...
+		}
+		my $von="";
+		my $last="";
+		if ($vonlast =~ m/^[a-z]/) { # lowercase implies "von"
+		    $vonlast =~ s/^(([a-z]\w+\s+)+)//;
+		    $von = $1;
+		    if (!defined ($von)) {
+			# some non-English names do start with lowercase
+			# eg "Marie desJardins". Also we can get typos...
+			print $outhandle "BibTexPlug: couldn't parse surname $vonlast\n";
+			$von="";
+			if ($vonlast =~ /^[a-z]+$/) {
+			    # if it's all lowercase, uppercase 1st.
+			    $vonlast =~ s/^(.)/\u$1/;
+
+			}
+		    }
+		    $von =~ s/\s*$//;
+		    $last=$vonlast;
+		} else {
+		    $last=$vonlast;
+		}
+		my $wholename="$first $von $last $jr";
+		$wholename =~ s/\s+/ /g; # squeeze multiple spaces
+		$wholename =~ s/ $//;
+		push (@formattedlist, $wholename);
+		my $fullname = "$last";
+		$fullname .= " $jr" if ($jr);
+		$fullname .= ", $first";
+		$fullname .= " $von" if ($von);
+
+		# Add each name to set of Authors
+		$doc_obj->add_utf8_metadata ($cursection, "Author", $fullname);
+	    }
+
+	    # Only want at most one "and" in the Creator field
+	    if (scalar(@formattedlist) > 2) {
+		my $lastauthor=pop @formattedlist;
+		$value=join(', ', @formattedlist);
+		$value.=" and $lastauthor";
+	    } else { # 1 or 2 authors...
+		$value=join(" and ",@formattedlist);
+	    }
+	    $value.=$etal; # if there was "et al."
+	}
+
+	# Books and Journals are additionally marked for display purposes
+	if ($entryname eq "booktitle") {
+	    $doc_obj->add_utf8_metadata($cursection, "BookConfOnly", 1);
+	} elsif ($entryname eq "journal") {
+	    $doc_obj->add_utf8_metadata($cursection, "JournalsOnly", 1); 
+	}
+
+	# Add the various fields as metadata	
+	$doc_obj->add_utf8_metadata ($cursection, $name, $value); 
+    }
+
+    # for books and journals...
+    if (!exists $metadata{'title'}) {
+	my $name=$field{'title'}; # get Greenstone metadata name
+	my $value;
+	if (exists $metadata{'booktitle'}) {
+	    $value=$metadata{'booktitle'};
+	} elsif (exists $metadata{'journal'}) {
+	    $value=$metadata{'journal'};
+	}
+	if ($value) {
+	    $doc_obj->add_utf8_metadata ($cursection, $name, $value); 
+	}
+    }
+
+    # Add Date (yyyymmdd) metadata
+    if (defined ($metadata{'year'}) ) {
+	my $date=$metadata{'year'};
+	chomp $date;
+	my $month=$metadata{'month'};
+	if (defined($month)) {
+	    # month is currently 3 letter code or a range...
+	    $month = expand_month($month);
+	    # take the first month found... might not find one!
+	    $month =~ m/_textmonth(\d\d)_/;
+	    $month = $1;
+	}
+	if (!defined($month)) {
+	    $month="00";
+	}
+	$date .= "${month}00";
+	$doc_obj->add_utf8_metadata($cursection, "Date", $date); 
+}
+
+#    # Add the text in BibTex format (all fields)
+    if ($text =~ /\w/) {
+
+	$text =~ s@&@&amp;@g;
+	$text =~ s@<@&lt;@g;
+	$text =~ s@>@&gt;@g;
+	$text =~ s@\n@<br/>\n@g;
+	$text =~ s@\\@\\\\@g;
+
+	$doc_obj->add_utf8_text ($cursection, $text);
+    }
+
+    return 1;
+}
+
+
+
+
+# convert email addresses and URLs into links
+sub convert_urls_into_links{
+   my ($text) = @_;
+ 
+   $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
+   $text =~ s/(http:\/\/[\w\d\.\-]+[\/\w\d\.\-]*)/<a href=\"$1\">$1<\/a>/g;
+
+   return $text;
+}
+
+# Clean up whitespace and convert \n charaters to <BR> or <P>
+sub clean_up_whitespaces{
+   my ($text) = @_;
+
+   $text =~ s/%%%%%/<BR> <BR>/g;
+   $text =~ s/ +/ /g;
+   $text =~ s/\s*$//; 
+   $text =~ s/^\s*//; 
+   $text =~ s/\n/\n<BR>/g;
+   $text =~ s/<BR>\s*<BR>/<P>/g;
+
+   return $text;
+}
+
+
+sub convert_problem_characters_without_ampersand{
+    my ($text) = @_;
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    
+    $text =~ s/\'\'/\"/g; #Latex -specific conversion
+    $text =~ s/\`\`/\"/g; #Latex -specific conversion
+
+    $text =~ s/\"/&quot;/g;
+    $text =~ s/\'/&#8217;/g;
+    $text =~ s/\`/&#8216;/g;
+
+#    $text =~ s/\+/ /g;
+#    $text =~ s/\(/ /g;
+#    $text =~ s/\)/ /g;
+
+    $text =~ s/\\/\\\\/g;
+
+#    $text =~ s/\./\\\./g;
+
+    return $text;
+}
+
+# Convert a text string into HTML.
+
+# The HTML is going to be inserted into a GML file, so we have to be
+# careful not to use symbols like ">", which occurs frequently in email
+# messages (and use &gt instead.
+
+# This function also turns URLs and email addresses into links, and
+# replaces carriage returns with <BR> tags (and multiple carriage returns
+# with <P> tags).
+
+sub text_into_html {
+    my ($text) = @_;
+
+    # Convert problem characters into HTML symbols
+    $text =~ s/&/&amp;/g;
+
+    $text = &convert_problem_characters_without_ampersand( $text );
+
+    # convert email addresses and URLs into links
+    $text = &convert_urls_into_links( $text );
+
+    $text = &clean_up_whitespaces( $text );
+
+    return $text;
+}
+
+
+sub expand_month {
+    my $text=shift;
+
+    # bibtex style files expand abbreviations for months.
+    # Entries can contain more than one month (eg ' month = jun # "-" # aug, ')
+    $text =~ s/jan/_textmonth01_/g;
+    $text =~ s/feb/_textmonth02_/g;
+    $text =~ s/mar/_textmonth03_/g;
+    $text =~ s/apr/_textmonth04_/g;
+    $text =~ s/may/_textmonth05_/g;
+    $text =~ s/jun/_textmonth06_/g;
+    $text =~ s/jul/_textmonth07_/g;
+    $text =~ s/aug/_textmonth08_/g;
+    $text =~ s/sep/_textmonth09_/g;
+    $text =~ s/oct/_textmonth10_/g;
+    $text =~ s/nov/_textmonth11_/g;
+    $text =~ s/dec/_textmonth12_/g;
+
+    return $text;
+}
+
+
+# Convert accented characters, remove { }, interprete some commands....
+# Note!! This is not comprehensive! Also assumes Latin -> Unicode!
+sub process_latex {
+    my ($text) = @_;
+      
+    # note - this is really ugly, but it works. There may be a prettier way
+    # of mapping latex accented chars to utf8, but we just brute force it here.
+    # Also, this isn't complete - not every single possible accented letter
+    # is in here yet, but most of the common ones are.
+
+    my %utf8_chars =
+	(
+	 # acutes
+	 '\'a' => chr(0xc3).chr(0xa1),
+	 '\'c' => chr(0xc4).chr(0x87),
+	 '\'e' => chr(0xc3).chr(0xa9),
+	 '\'i' => chr(0xc3).chr(0xad),
+	 '\'l' => chr(0xc3).chr(0xba),
+	 '\'n' => chr(0xc3).chr(0x84),
+	 '\'o' => chr(0xc3).chr(0xb3),
+	 '\'r' => chr(0xc5).chr(0x95),
+	 '\'s' => chr(0xc5).chr(0x9b),
+	 '\'u' => chr(0xc3).chr(0xba),
+	 '\'y' => chr(0xc3).chr(0xbd),
+	 '\'z' => chr(0xc5).chr(0xba),
+	 # graves
+	 '`a' => chr(0xc3).chr(0xa0),
+	 '`A' => chr(0xc3).chr(0x80),
+	 '`e' => chr(0xc3).chr(0xa8),
+	 '`E' => chr(0xc3).chr(0x88),
+	 '`i' => chr(0xc3).chr(0xac),
+	 '`I' => chr(0xc3).chr(0x8c),
+	 '`o' => chr(0xc3).chr(0xb2),
+	 '`O' => chr(0xc3).chr(0x92),
+	 '`u' => chr(0xc3).chr(0xb9),
+	 '`U' => chr(0xc3).chr(0x99),
+	 # circumflex
+	 '^a' => chr(0xc3).chr(0xa2),
+	 '^A' => chr(0xc3).chr(0x82),
+	 '^c' => chr(0xc4).chr(0x89),
+	 '^C' => chr(0xc4).chr(0x88),
+	 '^e' => chr(0xc3).chr(0xaa),
+	 '^E' => chr(0xc3).chr(0x8a),
+	 '^g' => chr(0xc4).chr(0x9d),
+	 '^G' => chr(0xc4).chr(0x9c),
+	 '^h' => chr(0xc4).chr(0xa5),
+	 '^H' => chr(0xc4).chr(0xa4),
+	 '^i' => chr(0xc3).chr(0xae),
+	 '^I' => chr(0xc3).chr(0x8e),
+	 '^j' => chr(0xc4).chr(0xb5),
+	 '^J' => chr(0xc4).chr(0xb4),
+	 '^o' => chr(0xc3).chr(0xb4),
+	 '^O' => chr(0xc3).chr(0x94),
+	 '^s' => chr(0xc5).chr(0x9d),
+	 '^S' => chr(0xc5).chr(0x9c),
+	 '^u' => chr(0xc3).chr(0xa2),
+	 '^U' => chr(0xc3).chr(0xbb),
+	 '^w' => chr(0xc5).chr(0xb5),
+	 '^W' => chr(0xc5).chr(0xb4),
+	 '^y' => chr(0xc5).chr(0xb7),
+	 '^Y' => chr(0xc5).chr(0xb6),
+	 
+	 # diaeresis
+	 '"a' => chr(0xc3).chr(0xa4),
+	 '"A' => chr(0xc3).chr(0x84),
+	 '"e' => chr(0xc3).chr(0xab),
+	 '"E' => chr(0xc3).chr(0x8b),
+	 '"\\\\i' => chr(0xc3).chr(0xaf),
+	 '"\\\\I' => chr(0xc3).chr(0x8f),
+	 '"o' => chr(0xc3).chr(0xb6),
+	 '"O' => chr(0xc3).chr(0x96),
+	 '"u' => chr(0xc3).chr(0xbc),
+	 '"U' => chr(0xc3).chr(0x9c),
+	 '"y' => chr(0xc3).chr(0xbf),
+	 '"Y' => chr(0xc3).chr(0xb8),
+	 # tilde
+	 '~A' => chr(0xc3).chr(0x83),
+	 '~N' => chr(0xc3).chr(0x91),
+	 '~O' => chr(0xc3).chr(0x95),
+	 '~a' => chr(0xc3).chr(0xa3),
+	 '~n' => chr(0xc3).chr(0xb1),
+	 '~o' => chr(0xc3).chr(0xb5),
+	 # caron - handled specially
+	 # double acute
+	 # ring
+	 # dot
+	 '.c' => chr(0xc4).chr(0x8b),
+	 '.C' => chr(0xc4).chr(0x8a),
+	 '.e' => chr(0xc4).chr(0x97),
+	 '.E' => chr(0xc4).chr(0x96),
+	 '.g' => chr(0xc4).chr(0xa1),
+	 '.G' => chr(0xc4).chr(0xa0),
+	 '.I' => chr(0xc4).chr(0xb0),
+	 '.z' => chr(0xc5).chr(0xbc),
+	 '.Z' => chr(0xc5).chr(0xbb),
+	 # macron
+	 '=a' => chr(0xc4).chr(0x81),
+	 '=A' => chr(0xc4).chr(0x80),
+	 '=e' => chr(0xc4).chr(0x93),
+	 '=E' => chr(0xc4).chr(0x92),
+	 '=i' => chr(0xc4).chr(0xab),
+	 '=I' => chr(0xc4).chr(0xaa),
+	 '=o' => chr(0xc4).chr(0x8d),
+	 '=O' => chr(0xc4).chr(0x8c),
+	 '=u' => chr(0xc4).chr(0xab),
+	 '=U' => chr(0xc4).chr(0xaa),
+	 
+	 # stroke - handled specially - see below
+	 
+	 # cedilla - handled specially
+	 );
+    
+# these are one letter latex commands - we make sure they're not a longer
+# command name. eg {\d} is d+stroke, so careful of \d
+    my %special_utf8_chars = 
+	(
+	 # breve
+	 'u g' => chr(0xc4).chr(0x9f),
+	 'u G' => chr(0xc4).chr(0x9e),
+	 'u i' => chr(0xc4).chr(0xad),
+	 'u I' => chr(0xc4).chr(0xac),
+	 'u o' => chr(0xc5).chr(0x8f),
+	 'u O' => chr(0xc5).chr(0x8e),
+	 'u u' => chr(0xc5).chr(0xad),
+	 'u U' => chr(0xc5).chr(0xac),
+	 # caron
+	 'v c' => chr(0xc4).chr(0x8d),
+	 'v C' => chr(0xc4).chr(0x8c),
+	 'v n' => chr(0xc5).chr(0x88),
+	 'v N' => chr(0xc5).chr(0x87),
+	 'v s' => chr(0xc5).chr(0xa1),
+	 'v S' => chr(0xc5).chr(0xa5),
+	 'v z' => chr(0xc5).chr(0xbe),
+	 'v Z' => chr(0xc5).chr(0xbd),
+	 # cedilla
+	 'c c' => chr(0xc3).chr(0xa7),
+	 'c C' => chr(0xc3).chr(0x87),
+	 'c g' => chr(0xc4).chr(0xa3),
+	 'c G' => chr(0xc4).chr(0xa2),
+	 'c k' => chr(0xc4).chr(0xb7),
+	 'c K' => chr(0xc4).chr(0xb6),
+	 'c l' => chr(0xc4).chr(0xbc),
+	 'c L' => chr(0xc4).chr(0xbb),
+	 'c n' => chr(0xc5).chr(0x86),
+	 'c N' => chr(0xc5).chr(0x85),
+	 'c r' => chr(0xc5).chr(0x97),
+	 'c R' => chr(0xc5).chr(0x96),
+	 'c s' => chr(0xc5).chr(0x9f),
+	 'c S' => chr(0xc5).chr(0x9e),
+	 'c t' => chr(0xc5).chr(0xa3),
+	 'c T' => chr(0xc5).chr(0xa2),
+	 # double acute / Hungarian accent
+	 'H O' => chr(0xc5).chr(0x90),
+	 'H o' => chr(0xc5).chr(0x91),
+	 'H U' => chr(0xc5).chr(0xb0),
+	 'H u' => chr(0xc5).chr(0xb1),
+	 
+	 # stroke 
+	 'd' => chr(0xc4).chr(0x91),
+	 'D' => chr(0xc4).chr(0x90),
+	 'h' => chr(0xc4).chr(0xa7),
+#	 'H' => chr(0xc4).chr(0xa6), # !! this normally(!!?) means Hung. umlaut
+	 'i' => chr(0xc4).chr(0xb1), # dotless lowercase i
+	 'l' => chr(0xc5).chr(0x82),
+	 'L' => chr(0xc5).chr(0x81),
+	 'o' => chr(0xc3).chr(0xb8),
+	 'O' => chr(0xc3).chr(0x98),
+	 't' => chr(0xc5).chr(0xa7),
+	 'T' => chr(0xc5).chr(0xa6),
+	 # german ss/szlig/sharp s
+	 'ss' => chr(0xc3).chr(0x9f),
+	 );
+    
+    # convert latex-style accented characters.
+
+    # remove space (if any) between \ and letter to accent (eg {\' a})
+
+    $text =~ s@(\\[`'="^~\.])\s(\w)@$1$2@g;
+
+    # remove {} around a single character (eg \'{e})
+    $text =~ s@(\\[`'="^~\.]){(\w)}@{$1$2}@g;
+
+    # \, is another way of doing cedilla \c
+    $text =~ s@\\,(.)@\\c $1@g;
+
+    # remove {} around a single character for special 1 letter commands -
+    # need to insert a space. Eg \v{s}  ->  {\v s}
+    $text =~ s@(\\[uvcH]){(\w)}@{$1 $2}@g;
+
+    # only do if the text contains a '\' character. 
+    if ($text =~ m|\\|) {
+	# "normal" accents - ie non-alpha latex tag
+	while ($text =~ m@\\([`'="^~\.])([\w])@) {
+	    my $tex="$1$2"; my $char="$2";
+	    my $replacement=$utf8_chars{$tex};
+	    if (!defined($replacement)) {
+		print STDERR "BibTexPlug: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
+		$replacement=$char;
+	    }
+	    $text =~ s/\\\Q$tex/$replacement/g;
+	}
+
+        # where the following letter matters (eg "sm\o rrebr\o d", \ss{})
+        # only do the change if immediately followed by a space, }, {, or \
+	# one letter accents ( + ss)
+        while ($text =~ m@\\([DdhiLlOoTt]|ss)[{}\s\"\\]@) {
+	    my $tex=$1;
+	    my $replacement=$special_utf8_chars{$tex};
+	    if (!defined($replacement)) {
+		print STDERR "BibTexPlug: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
+		$replacement=$tex;
+	    }
+	    $text =~ s/\\$tex([{}\s\"\\])/$replacement$1/g;
+
+	}
+
+	# one letter latex accent commands that affect following letter
+        while ($text =~ m@\\([uvcH]) ([\w])@) {
+          my $tex="$1 $2"; my $char="$2";
+          my $replacement=$special_utf8_chars{$tex};
+          if (!defined($replacement)) {
+	      print STDERR "BibTexPlug: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
+	      $replacement=$char;
+	  }
+          $text =~ s/\\$tex/$replacement/g;
+        }
+    }
+
+    # escape html-sensitive characters
+    $text =~ s@&@&amp;@g;
+    $text =~ s@<@&lt;@g;
+    $text =~ s@>@&gt;@g;
+    $text =~ s/''/"/g; # Latex-specific
+    $text =~ s/``/"/g; # Latex-specific
+    # greenstone-specific
+    $text =~ s@\[@&\#91;@g;
+    $text =~ s@\]@&\#93;@g;
+
+    # remove latex commands
+
+    # explicitly recognised commands
+    $text =~ s@\\ldots@&hellip;@g;
+
+    # maths mode
+    $text =~ s@\$(.*?)\$@&process_latex_math($1)@ge;
+
+    # remove all other commands with optional arguments...
+    $text =~ s@\\\w+(\[.*?\])?\s*@@g;
+    # $text =~ s@\\noopsort{[^}]+\}@@g;
+    # $text =~ s@\\\w+{(([^}]*[^\\])*)}@$1@g; # all other commands
+    
+    # remove latex groupings { } (but not \{ or \} )
+    while ($text =~ s/([^\\])[\{\}]/$1/g) {;} # needed for "...}{..."
+    $text =~ s/^\{//; # remove { if first char
+    
+    # latex characters
+    # spaces - nobr space (~), opt break (\-), append ("#" - bibtex only)
+    $text =~ s/([^\\])~+/$1 /g; # non-breaking space  "~"
+    # optional break "\-"
+    if ($text =~ m/[^&]\#/) { # concat macros (bibtex) but not HTML codes
+	# the non-macro bits have quotes around them - we just remove quotes
+	$text =~ s/([^&])[\"\#]/$1/g;
+    }
+    # dashes. Convert (m|n)-dash into single dash for html.
+    $text =~ s@\-\-+@\-@g;
+
+    # quoted { } chars
+    $text =~ s@\\{@{@g;
+    $text =~ s@\\}@}@g;
+
+    # finally to protect against macro language...
+    $text =~ s@\\@\\\\@g;
+
+    return $text;
+}
+
+
+sub process_latex_math {
+    my $text = shift;
+
+    $text =~ s@\\infty@infinity@g;       # or unicode 0x221E...
+    $text =~ s@\^{(.*?)}@<sup>$1</sup>@g; # superscript
+    $text =~ s@\^([^\{])@<sup>$1</sup>@g;
+    $text =~ s@\_{(.*?)}@<sub>$1</sub>@g; # subscript
+    $text =~ s@\_([^\{])@<sub>$1</sub>@g;
+
+    # put all other command names in italics
+    $text =~ s@\\([\w]+)@<i>$1</i>@g;
+
+    return $text;
+}
+
+sub set_OID {
+    my $self = shift (@_);
+    my ($doc_obj, $id, $segment_number) = @_;
+    
+    if ( $self->{'key'} eq "default") {
+	$doc_obj->set_OID("$id\_$segment_number");
+    } else {
+	$doc_obj->set_OID($self->{'key'});
+    }
+}
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/BookPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/BookPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/BookPlug.pm	(revision 21112)
@@ -0,0 +1,248 @@
+###########################################################################
+#
+# BookPlug.pm (formally called HBSPlug) -- plugin for processing simple
+# html (or text) books
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates multi-level document from document containing 
+# <<TOC>> level tags. Metadata for each section is taken from any
+# other tags on the same line as the <<TOC>>. e.g. <<Title>>xxxx<</Title>>
+# sets Title metadata.
+
+# Everything else between TOC tags is treated as simple html (i.e. no 
+# processing of html links or any other HTMLPlug type stuff is done).
+
+# expects input files to have a .hb file extension by default (this can be 
+# changed by adding a -process_exp option
+
+# a file with the same name as the hb file but a .jpg extension is
+# taken as the cover image (jpg files are blocked by this plugin)
+
+# BookPlug is a simplification (and extension) of the HBPlug used
+# by the Humanity Library collections. BookPlug is faster as it expects
+# the input files to be cleaner (The input to the HDL collections
+# contains lots of excess html tags around <<TOC>> tags, uses <<I>>
+# tags to specify images, and simply takes all text between <<TOC>>
+# tags and start of text to be Title metadata). If you're marking up
+# documents to be displayed in the same way as the HDL collections,
+# use this plugin instead of HBPlug.
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package BookPlug;
+
+use BasPlug;
+use util;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @BookPlug::ISA = ('BasPlug');
+}
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_block_exp() } ];
+
+my $options = { 'name'     => "BookPlug",
+		'desc'     => "{BookPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_block_exp {
+    my $self = shift (@_);
+
+    return q^\.jpg$^;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.hb$^;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print STDERR "<Processing n='$file' p='BookPlug'>\n" if ($gli);
+    print $outhandle "BookPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+    
+    my $cursection = $doc_obj->get_top_section();
+
+    # Add FileFormat as the metadata
+    $doc_obj->add_metadata($doc_obj->get_top_section(),"FileFormat", "Book");
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $absdir = $filename;
+    $absdir =~ s/[^\/\\]*$//;
+
+    # add the cover image
+    my $coverimage = $filename;
+    $coverimage =~ s/\.[^\.]*$/\.jpg/i;
+    $doc_obj->associate_file($coverimage, "cover.jpg", "image/jpeg");
+
+    my $title = "";
+
+    # remove any leading rubbish
+    $$textref =~ s/^.*?(<<TOC)/$1/ios;
+    
+    my $curtoclevel = 1;
+    my $firstsection = 1;
+    my $toccount = 0;
+    while ($$textref =~ /\w/) {
+	$$textref =~ s/^<<TOC(\d+)>>([^\n]*)\n(.*?)(<<TOC|\Z)/$4/ios;
+	my $toclevel = $1;
+	my $metadata = $2;
+	my $sectiontext = $3;
+
+	if ($toclevel == 2) {
+	    $toccount ++;
+	}
+	
+	# close any sections below the current level and
+	# create a new section (special case for the firstsection)
+	while (($curtoclevel > $toclevel) ||
+	       (!$firstsection && $curtoclevel == $toclevel)) {
+	    $cursection = $doc_obj->get_parent_section ($cursection);
+	    $curtoclevel--;
+	}
+	if ($curtoclevel+1 < $toclevel) {
+	    print $outhandle "WARNING - jump in toc levels in $filename " . 
+		"from $curtoclevel to $toclevel\n";
+	}
+	while ($curtoclevel < $toclevel) {
+	    $curtoclevel++;
+	    $cursection = 
+		$doc_obj->insert_section($doc_obj->get_end_child($cursection));
+	}
+
+	# sort out metadata
+	while ($metadata =~ s/^.*?<<([^>]*)>>(.*?)<<[^>]*>>//) {
+	    my $metakey = $1;
+	    my $metavalue = $2;
+
+	    if ($metavalue ne "" && $metakey ne "") {
+		# make sure key fits in with gsdl naming scheme 
+		$metakey =~ tr/[A-Z]/[a-z]/;
+		$metakey = ucfirst ($metakey);
+		$doc_obj->add_utf8_metadata ($cursection, $metakey, $metavalue);
+	    }
+	}
+
+	# remove header rubbish
+	$sectiontext =~ s/^.*?<body[^>]*>//ios;
+
+	# and any other unwanted tags
+	$sectiontext =~ s/<(\/p|\/html|\/body)>//isg;
+
+	# fix up the image links
+	$sectiontext =~ s/(<img[^>]*?src\s*=\s*\"?)([^\">]+)(\"?[^>]*>)/
+	    &replace_image_links($absdir, $doc_obj, $1, $2, $3)/isge;
+
+	# add the text
+	$doc_obj->add_utf8_text($cursection, $sectiontext);
+
+	$firstsection = 0;
+
+	$$textref =~ s/^\s+//s;
+    }
+
+    return 1;
+}
+
+sub replace_image_links {
+    my $self = shift (@_);
+    my ($dir, $doc_obj, $front, $link, $back) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my ($filename, $error);
+    my $foundimage = 0;
+    
+    $link =~ s/\/\///;
+    my ($imagetype) = $link =~ /([^\.]*)$/;
+    $imagetype =~ tr/[A-Z]/[a-z]/;
+    if ($imagetype eq "jpg") {$imagetype = "jpeg";}
+    if ($imagetype !~ /^(jpeg|gif|png)$/) {
+	print $outhandle "BookPlug: Warning - unknown image type ($imagetype)\n";
+    }
+    my ($imagefile) = $link =~ /([^\/]*)$/;
+    my ($imagepath) = $link =~ /^[^\/]*(.*)$/;
+
+    if (defined $imagepath && $imagepath =~ /\w/) {
+	# relative link
+	$filename = &util::filename_cat ($dir, $imagepath);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");
+	    $foundimage = 1;
+	} else {
+	    $error = "BookPlug: Warning - couldn't find image file $imagefile in either $filename or";
+	}
+    }
+
+    if (!$foundimage) {
+	$filename = &util::filename_cat ($dir, $imagefile);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");    
+	    $foundimage = 1;
+	} elsif (defined $error) {
+	    print $outhandle "$error $filename\n";
+	} else {
+	    print $outhandle "BookPlug: Warning - couldn't find image file $imagefile in $filename\n";
+	}
+    }
+
+    if ($foundimage) {
+	return "${front}_httpdocimg_/${imagefile}${back}";
+    } else {
+	return "";
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ConvertToPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ConvertToPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ConvertToPlug.pm	(revision 21112)
@@ -0,0 +1,458 @@
+###########################################################################
+#
+# ConvertToPlug.pm -- plugin that inherits from BasPlug
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This plugin is inherited by such plugins as WordPlug, PPTPlug, PSPlug, RTFPlug 
+# and PDFPlug. It facilitates the conversion of these document types to either 
+# HTML, Text or auto (allow user to choose which format to convert to).
+# It works by dynamically inheriting BasPlug and base on the plugin type in
+# secondary_plugins to devide which format to 'convert_to'. If the argument is 
+# not present, the default is to inherit auto.
+package ConvertToPlug;
+
+use BasPlug;
+use ghtml;
+use HTMLPlug;
+use TEXTPlug;
+use PagedImgPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @ConvertToPlug::ISA = ('BasPlug');
+}
+
+my $convert_to_list =
+    [ {	'name' => "auto",
+	'desc' => "{ConvertToPlug.convert_to.auto}" },
+      {	'name' => "html",
+	'desc' => "{ConvertToPlug.convert_to.html}" },
+      {	'name' => "text",
+	'desc' => "{ConvertToPlug.convert_to.text}" }
+      ];
+
+my $arguments =
+    [ { 'name' => "convert_to",
+	'desc' => "{ConvertToPlug.convert_to}",
+	'type' => "enum",
+	'reqd' => "yes",
+	'list' => $convert_to_list, 
+	'deft' => "auto" },
+      { 'name' => "title_sub",
+	'desc' => "{HTMLPlug.title_sub}",
+	'type' => "string", 
+	#'type' => "regexp",
+	'deft' => "" },
+      { 'name' => "use_strings",
+	'desc' => "{ConvertToPlug.use_strings}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "extract_keyphrases",
+	'desc' => "{BasPlug.extract_keyphrases}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes" },
+      { 'name' => "extract_keyphrase_options",
+	'desc' => "{BasPlug.extract_keyphrase_options}",
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes" } ];
+
+my $options = { 'name'     => "ConvertToPlug",
+		'desc'     => "{ConvertToPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub load_secondary_plugins
+{
+    my $self = shift (@_);
+    my ($class,$input_args,$hashArgOptLists) = @_;
+
+    my @convert_to_list = split(",",$self->{'convert_to'});
+    my $secondary_plugins = {};
+
+    foreach my $convert_to (@convert_to_list) {
+	# load in "convert_to" plugin package
+	my $plugin_class = $convert_to."Plug";
+	my $plugin_package = $plugin_class.".pm";
+
+	require $plugin_package;
+
+	# call its constructor with extra options that we've worked out!
+	my $arglist = $input_args->{$plugin_class};
+#	my $secondary_plugin = new $plugin_class([],$arglist, $hashArgOptLists);
+	# secondary plugins don';t need to know their calling plugin's options
+	my $secondary_plugin = new $plugin_class([],$arglist);
+	$secondary_plugins->{$plugin_class} = $secondary_plugin;
+    }
+    $self->{'secondary_plugins'} = $secondary_plugins;
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    my $classPluginName = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    
+    my $convert_to_type = $self->{'convert_to'};
+    if (!defined $convert_to_type || $convert_to_type eq "") {
+	$convert_to_type = "auto";
+    }
+    my $windows_scripting = $self->{'windows_scripting'};
+    if ($classPluginName eq "PDFPlug") {
+	if ($convert_to_type eq "text" && 
+	    $ENV{'GSDLOS'} =~ /^windows$/i) {
+	    print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
+	    $convert_to_type = "html";
+	}
+    } elsif ($classPluginName eq "WordPlug") {
+	if (defined $self->{'windows_scripting'} && $ENV{'GSDLOS'} =~ /^windows$/i && $convert_to_type =~ /^(html|auto)$/) {
+	    # we use structured HTML, not normal html
+	    $convert_to_type = "structuredhtml";
+	} 
+    } elsif ($classPluginName eq "PPTPlug") {
+	if (defined $self->{'windows_scripting'} && $ENV{'GSDLOS'} =~ /^windows$/i && $convert_to_type eq "auto") {
+	    # we use paged img
+	    $convert_to_type = "pagedimg_jpg";
+	} 
+    } elsif ($classPluginName eq "PSPlug") {
+	if ($convert_to_type eq "auto") {
+	    # we use text
+	    $convert_to_type = "text";
+	}
+    }
+    
+    if ($convert_to_type eq "auto") {
+	# choose html for now - should choose a format based on doc type
+	$convert_to_type = "html";
+    }
+    
+    if ($convert_to_type eq "html") {
+	$self->{'convert_to'} = "HTML";
+	$self->{'convert_to_ext'} = "html";
+    } elsif ($convert_to_type eq "text") {
+	$self->{'convert_to'} = "TEXT";
+	$self->{'convert_to_ext'} = "txt";
+    } elsif ($convert_to_type eq "structuredhtml") {
+	$self->{'convert_to'} = "StructuredHTML";
+	$self->{'convert_to_ext'} = "html";
+    } elsif ($convert_to_type =~ /^pagedimg/) {
+	$self->{'convert_to'} = "PagedImg";
+	my ($convert_to_ext) = $convert_to_type =~ /pagedimg\_(jpg|gif|png)/i;
+	$convert_to_ext = 'jpg' unless defined $convert_to_ext;
+	$self->{'convert_to_ext'} = $convert_to_ext;
+    }
+
+    return bless $self, $class;
+}
+
+
+sub init {
+    my $self = shift (@_);
+    my ($verbosity, $outhandle, $failhandle) = @_;
+
+    $self->SUPER::init($verbosity,$outhandle,$failhandle);
+
+    my $secondary_plugins =  $self->{'secondary_plugins'};
+
+    foreach my $plug_name (keys %$secondary_plugins) {
+	my $plugin = $secondary_plugins->{$plug_name};
+	$plugin->init($verbosity,$outhandle,$failhandle);
+    }
+}
+
+sub deinit {
+    # called only once, after all plugin passes have been done
+
+    my ($self) = @_;
+
+    my $secondary_plugins =  $self->{'secondary_plugins'};
+
+    foreach my $plug_name (keys %$secondary_plugins) {
+	my $plugin = $secondary_plugins->{$plug_name};
+	$plugin->deinit();
+    }
+}
+
+sub convert_post_process
+{
+    # by default do no post processing
+    return;
+}
+
+
+# Run conversion utility on the input file.  
+#
+# The conversion takes place in a collection specific 'tmp' directory so 
+# that we don't accidentally damage the input.
+#
+# The desired output type is indicated by $output_ext.  This is usually
+# something like "html" or "word", but can be "best" (or the empty string)
+# to indicate that the conversion utility should do the best it can.
+sub tmp_area_convert_file {
+    my $self = shift (@_);
+    my ($output_ext, $input_filename, $textref) = @_;
+    
+    my $outhandle = $self->{'outhandle'};
+    my $convert_to = $self->{'convert_to'};
+    my $failhandle = $self->{'failhandle'};
+    my $convert_to_ext = $self->{'convert_to_ext'};
+    
+    # softlink to collection tmp dir
+    my $tmp_dirname 
+	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
+
+    # derive tmp filename from input filename
+    my ($tailname, $dirname, $suffix)
+	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
+
+    # Remove any white space from filename -- no risk of name collision, and
+    # makes later conversion by utils simpler. Leave spaces in path...
+    # tidy up the filename with space, dot, hyphen between
+    $tailname =~ s/\s+//g; 
+    $tailname =~ s/\.+//g;
+    $tailname =~ s/\-+//g;
+    $suffix = lc($suffix);
+    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+    &util::soft_link($input_filename, $tmp_filename);
+    my $verbosity = $self->{'verbosity'};
+    if ($verbosity > 0) {
+	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
+    }
+
+    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
+    
+    # Execute the conversion command and get the type of the result,
+    # making sure the converter gives us the appropriate output type
+    my $output_type="";
+    if ($convert_to =~ m/PagedImg/i) {
+	$output_type = lc($convert_to)."_".lc($convert_to_ext);
+    } else {
+	$output_type = lc($convert_to);
+    }
+
+    my $cmd = "perl -S gsConvert.pl -verbose $verbosity ";
+    if (defined $self->{'convert_options'}) {
+	$cmd .= $self->{'convert_options'} . " ";
+    }
+    if ($self->{'use_strings'}) {
+      $cmd .= "-use_strings ";
+    }
+    $cmd .= "-errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
+
+    $output_type = `$cmd`;
+
+    # remove symbolic link to original file
+    &util::rm($tmp_filename);
+
+    # Check STDERR here
+    chomp $output_type;
+    if ($output_type eq "fail") {
+	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
+	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
+	$self->{'num_not_processed'} ++;
+	if (-s "$errlog") {
+	    open(ERRLOG, "$errlog");
+	    while (<ERRLOG>) {
+		print $outhandle "$_";
+	    }
+	    print $outhandle "\n";
+	    close ERRLOG;
+	}
+	&util::rm("$errlog") if (-e "$errlog");
+	return "";
+    }
+
+    # store the *actual* output type and return the output filename
+    # it's possible we requested conversion to html, but only to text succeeded
+    #$self->{'convert_to_ext'} = $output_type;
+    if ($output_type =~ /html/i) {
+	$self->{'converted_to'} = "HTML";
+    } elsif ($output_type =~ /te?xt/i) {
+	$self->{'converted_to'} = "TEXT";
+    } elsif ($output_type =~ /item/i){
+	$self->{'converted_to'} = "PagedImg";
+    }
+    
+    my $output_filename = $tmp_filename;
+    if ($output_type =~ /item/i) {
+	# running under windows
+	if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	    $output_filename = $tmp_dirname . "\\$tailname\\" . $tailname . ".$output_type";
+	} else {
+	    $output_filename = $tmp_dirname . "\/$tailname\/" . $tailname . ".$output_type";
+	}
+    } else {
+	$output_filename =~ s/$suffix$/.$output_type/;
+    }
+    return $output_filename;
+}
+
+
+# Override BasPlug read
+# We don't want to get language encoding stuff until after we've converted
+# our file to either TEXT or HTML or PagedImage.
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    #if ($self->is_recursive()) {
+    #   die "BasPlug::read function must be implemented in sub-class for recursive plugins\n";
+    # }
+
+    my $outhandle = $self->{'outhandle'};
+    
+    my ($block_status,$filename) = $self->read_block(@_);
+    return $block_status if ((!defined $block_status) || ($block_status==0));
+    $file = $self->read_tidy_file($file);
+    
+    my $output_ext = $self->{'convert_to_ext'};
+    my $conv_filename = "";
+    $conv_filename = $self->tmp_area_convert_file($output_ext, $filename);
+    
+    if ("$conv_filename" eq "") {return -1;} # had an error, will be passed down pipeline 
+    if (! -e "$conv_filename") {return -1;} 
+    $self->{'conv_filename'} = $conv_filename;
+    $self->convert_post_process($conv_filename);
+    
+    my $secondary_plugins =  $self->{'secondary_plugins'};
+    my $num_secondary_plugins = scalar(keys %$secondary_plugins);
+
+    if ($num_secondary_plugins == 0) {
+	print $outhandle "Warning: No secondary plugin to use in conversion.  Skipping $file\n";
+	return 0; # effectively block it
+    }
+
+    my @plugin_names = keys %$secondary_plugins;
+    my $plugin_name = shift @plugin_names;
+	
+    if ($num_secondary_plugins > 1) {
+	print $outhandle "Warning: Multiple secondary plugins not supported yet!  Choosing $plugin_name\n.";
+    }
+   
+    my $secondary_plugin = $secondary_plugins->{$plugin_name};
+
+    # note: metadata is not carried on to the next level
+    my ($rv,$doc_obj) 
+	= $secondary_plugin->read_into_doc_obj ($pluginfo,"", $conv_filename, 
+						$metadata, $processor, $maxdocs, $total_count,
+						$gli);
+
+    if ((!defined $rv) || ($rv<1)) {
+	# wasn't processed
+	return $rv;
+    }
+    
+    # Override previous gsdlsourcefilename set by secondary plugin
+    my $collect_file = &util::filename_within_collection($filename);
+    my $collect_conv_file = &util::filename_within_collection($conv_filename);
+    $doc_obj->set_source_filename ($collect_file); 
+    $doc_obj->set_converted_filename($collect_conv_file);
+
+    my ($filemeta) = $file =~ /([^\\\/]+)$/;
+    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    if ($self->{'cover_image'}) {
+	$self->associate_cover_image($doc_obj, $filename);
+    }
+
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process(undef, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1;
+    }
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    # have we found a Title??
+    $self->title_fallback($doc_obj,$doc_obj->get_top_section(),$filemeta);
+
+    # add an OID
+    $doc_obj->set_OID();
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'} ++;
+
+    return 1;
+}
+
+
+# do plugin specific processing of doc_obj for doc_ext type
+sub process_type {
+    my $self = shift (@_);
+    my ($doc_ext, $base_dir, $file, $doc_obj) = @_;
+    
+    # associate original file with doc object
+    my $cursection = $doc_obj->get_top_section();
+    my $filename = &util::filename_cat($base_dir, $file);
+    $doc_obj->associate_file($filename, "doc.$doc_ext", undef, $cursection);
+
+    my $file_type;
+
+    if ($doc_ext eq "doc") {
+    	$file_type = "Word";
+    } elsif ($doc_ext eq "xls") {
+	$file_type = "Excel";
+    } elsif ($doc_ext eq "ppt") {
+	$file_type = "PPT"; 
+    } elsif ($doc_ext eq "pdf") {
+	$file_type = "PDF"; 
+    } elsif ($doc_ext eq "rtf") {
+	$file_type = "RTF";
+    } elsif ($doc_ext eq "ps") {
+	$file_type = "PS";
+    }
+
+    my $file_format = $file_type || "unknown";
+
+    # We use set instead of add here because we only want one value
+    $doc_obj->set_utf8_metadata_element($cursection, "FileFormat", $file_format);
+
+    my $doclink = "<a href=\"_httpcollection_/index/assoc/[archivedir]/doc.$doc_ext\">";
+    $doc_obj->add_utf8_metadata ($cursection, "srclink",  $doclink); 
+    $doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icon".$doc_ext."_"); 
+    $doc_obj->add_utf8_metadata ($cursection, "/srclink", "</a>"); 
+
+    return 1;
+}
+
+1;
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/ConvertToRogPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ConvertToRogPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ConvertToRogPlug.pm	(revision 21112)
@@ -0,0 +1,463 @@
+###########################################################################
+#
+# ConvertToRogPlug.pm -- plugin that inherits from RogPlug
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package ConvertToRogPlug;
+
+use BasPlug;
+use RogPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @ConvertToRogPlug::ISA = ('RogPlug');
+}
+
+my $arguments = [
+		 ];
+my $options = { 'name'     => "ConvertToRogPlug",
+		'desc'     => "{ConvertToRogPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes" };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new RogPlug($pluginlist,$inputargs,$hashArgOptLists): new RogPlug($pluginlist,$inputargs);
+
+    $self->{'convert_to'} = "Rog";
+    $self->{'convert_to_ext'} = "rog";
+
+    return bless $self, $class;
+}
+
+
+sub begin {
+    my $self = shift (@_);
+    
+    $self->SUPER::begin(@_);
+
+    $self->{'docnum'} = 0;
+}
+
+sub end {
+    my ($self) = @_;
+
+    # nothing to do, but keep symmetric with begin function
+    $self->SUPER::end(@_);
+}
+
+
+# Run conversion utility on the input file.  
+#
+# The conversion takes place in a collection specific 'tmp' directory so 
+# that we don't accidentally damage the input.
+#
+# The desired output type is indicated by $output_ext.  This is usually
+# something like "html" or "word", but can be "best" (or the empty string)
+# to indicate that the conversion utility should do the best it can.
+
+sub tmp_area_convert_file {
+    my $self = shift (@_);
+    my ($output_ext, $input_filename, $textref) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $convert_to = $self->{'convert_to'};
+    my $failhandle = $self->{'failhandle'};
+
+    # softlink to collection tmp dir
+    my $tmp_dirname 
+	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
+
+    # derive tmp filename from input filename
+    my ($tailname, $dirname, $suffix)
+	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
+
+    # Remove any white space from filename -- no risk of name collision, and
+    # makes later conversion by utils simpler. Leave spaces in path...
+    $tailname =~ s/\s+//g; 
+
+    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+
+    &util::soft_link($input_filename, $tmp_filename);
+
+    my $verbosity = $self->{'verbosity'};
+    if ($verbosity > 0) {
+	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
+    }
+
+    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
+    
+    # Execute the conversion command and get the type of the result,
+    # making sure the converter gives us the appropriate output type
+    my $output_type = lc($convert_to);
+    my $cmd = "perl -S gsMusicConvert.pl -verbose $verbosity -errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
+    $output_type = `$cmd`;
+
+    # remove symbolic link to original file
+    &util::rm($tmp_filename);
+
+    # Check STDERR here
+    chomp $output_type;
+    if ($output_type eq "fail") {
+	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
+	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
+	$self->{'num_not_processed'} ++;
+	if (-s "$errlog") {
+	    open(ERRLOG, "$errlog");
+	    while (<ERRLOG>) {
+		print $outhandle "$_";
+	    }
+	    print $outhandle "\n";
+	    close ERRLOG;
+	}
+	&util::rm("$errlog") if (-e "$errlog");
+	return "";
+    }
+
+    # store the *actual* output type and return the output filename
+    # it's possible we requested conversion to html, but only to text succeeded
+
+    $self->{'convert_to_ext'} = $output_type;
+    $self->{'converted_to'} = "Rog";
+
+    my $output_filename = $tmp_filename;
+
+    $output_filename =~ s/$suffix$//;
+
+    return $output_filename;
+}
+
+
+# Remove collection specific tmp directory and all its contents.
+
+sub cleanup_tmp_area {
+    my $self = shift (@_);
+
+    my $tmp_dirname 
+	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    &util::rm_r($tmp_dirname);
+    &util::mk_dir($tmp_dirname);
+}
+
+
+# Exact copy of read_rog_record from RogPlug 
+# Needed for FILE in right scope
+
+sub read_rog_record
+{
+    my ($self,$file_buffer, $docnum, $seclevel) = @_;
+
+    my $next_line = $file_buffer->{'next_line'};
+
+    return 0 if (!defined $next_line);
+
+    if ($next_line eq "")
+    {
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	    $next_line = $line;
+	    last;
+	}
+    }
+	    
+    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
+    {
+	print STDERR "Error: Malformed Rog file: $next_line";
+	return 0;
+    }
+    else
+    {
+	# init default values
+	$file_buffer->{'song'}->{'tempo'}    = 120;
+	$file_buffer->{'song'}->{'ks_type'}  = 0;
+	$file_buffer->{'song'}->{'ks_num'}   = 0;
+	$file_buffer->{'song'}->{'metadata'} = [];
+	$file_buffer->{'song'}->{'content'}  = "";
+	
+	$file_buffer->{'song'}->{'subcol'} = $2;
+	$file_buffer->{'song'}->{'title'}  = $4;
+	$file_buffer->{'song'}->{'tval'}   = $6;
+
+	chomp($next_line);
+	my $content = $next_line;
+	if (defined $docnum) 
+	{
+	    $content.= " $docnum $seclevel";
+	}
+	$content .= "\n";
+
+	$file_buffer->{'song'}->{'content'} = $content;
+
+
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	
+	    if ($line =~ m/^song/) 
+	    {	
+		$file_buffer->{'next_line'} = $line;
+		return 1;
+	    }
+	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'tempo'} = $2;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    } 
+	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ks_type'} = $2;
+		$file_buffer->{'song'}->{'ks_num'}  = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;	   
+	    } 
+	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ts_numer'} = $2;
+		$file_buffer->{'song'}->{'ts_denom'} = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
+	    {
+		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    else
+	    {
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	}
+	
+	$file_buffer->{'next_line'} = undef;
+    }
+
+    return 1;
+}
+
+# Override RogPlug function so rog files are stored as sections (not docs)
+
+sub process_rog_record
+{
+    my ($self,$doc_obj,$cursection,$song) = @_;
+
+    $cursection = 
+	$doc_obj->insert_section($cursection);
+    $self->{'docnum'}++;
+
+    my $title = $song->{'title'};
+    my $title_safe = $title;
+    $title_safe =~ s/\'/\\\\&apos;/g;
+
+    # add metadata 
+    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
+    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
+    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
+    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
+    $doc_obj->add_metadata($cursection, "Title",         $title); 
+    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
+    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
+
+    foreach my $md ( @{$song->{'metadata'}} )
+    {
+	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
+    }
+
+    # add contents as text
+    $doc_obj->add_text($cursection,$song->{'content'});
+
+    return $cursection;
+}
+
+
+
+# Override BasPlug read
+# We don't want to get language encoding stuff until after we've converted
+# our file to Rog format
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+#    if ($self->is_recursive()) {
+#        die "BasPlug::read function must be implemented in sub-class for recursive plugins\n";
+#    }
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return 0;
+    }
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+        return undef;
+    }
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+
+    my $output_ext = $self->{'convert_to_ext'};
+    my $conv_filename = $self->tmp_area_convert_file($output_ext, $filename);
+
+    if ("$conv_filename" eq "") {return 0;} # allows continue on errors
+    $self->{'conv_filename'} = $conv_filename;
+
+
+    # create a new document
+    #my $doc_obj = new doc ($conv_filename, "indexed_doc");
+    # the original filename is used now
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    # the converted filename is set separately
+    $doc_obj->set_converted_filename($conv_filename);
+
+    my $topsection = $doc_obj->get_top_section();
+    my $cursection = $topsection;
+
+    $self->{'docnum'}++;
+    my $docnum = $self->{'docnum'};
+
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+    my ($filemeta) = $file =~ /([^\\\/]+)$/;
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+    if ($self->{'cover_image'}) {
+	$self->associate_cover_image($doc_obj, $filename);
+    }
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    my $track_no = "1";
+    my $rog_filename = "$conv_filename$track_no.$output_ext";
+    while (1)
+    {
+	last unless open (FILE, $rog_filename) ;
+
+	my $file_buffer = { line_no => 0, next_line => "", song => {} };
+	
+	while ($self->read_rog_record($file_buffer, $docnum, $track_no))
+	{
+	    my $song = $file_buffer->{'song'};
+	    my $content = $song->{'content'};
+	    $content =~ s/^song\w+(.*)$/song $1 X.$track_no/;
+	    
+	    $cursection 
+		= $self->process_rog_record($doc_obj,$cursection,
+					    $file_buffer->{'song'});
+	}
+
+	close FILE;
+
+	$track_no++;
+	$rog_filename = "$conv_filename$track_no.$output_ext";
+    }
+
+    print STDERR "\n";
+
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1;
+    }
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+    # add an OID
+    $doc_obj->set_OID();
+
+    my $oid = $doc_obj->get_OID();
+    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
+
+    $doc_obj->add_utf8_metadata ($topsection, "audiolink",  $appletlink); 
+    $doc_obj->add_utf8_metadata ($topsection, "audioicon",  "_iconaudio_"); 
+    $doc_obj->add_utf8_metadata ($topsection, "/audiolink", "</a>"); 
+
+    # if no title metadata defined, set it to filename minus extension
+    my $existing_title = $doc_obj->get_metadata_element($topsection,"Title");
+    if (!defined $existing_title) 
+    {
+	my $title = $doc_obj->get_metadata_element($topsection,"Source");
+	$title =~ s/\..*?$//g;
+	$doc_obj->add_utf8_metadata ($topsection, "Title", $title); 
+
+	my $title_safe = $title;
+	$title_safe =~ s/\'/\\\\&apos;/g;
+	$doc_obj->add_utf8_metadata ($topsection, "TitleSafe", $title_safe); 
+    }
+
+    # process the document
+    $processor->process($doc_obj);
+    $self->cleanup_tmp_area();
+
+    $self->{'num_processed'} ++;
+
+    return 1;
+}
+
+
+# do plugin specific processing of doc_obj for HTML type
+sub process_type {
+    my $self = shift (@_);
+    my ($doc_ext, $textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    
+    my $conv_filename = $self->{'conv_filename'};
+    my $tmp_dirname = File::Basename::dirname($conv_filename);
+    my $tmp_tailname = File::Basename::basename($conv_filename);
+    
+    my $converted_to = $self->{'converted_to'};
+    my $ret_val = 1;    
+
+#   $ret_val = &RogPlug::process($self, $textref, $pluginfo,
+#				 $tmp_dirname, $tmp_tailname,
+#				 $metadata, $doc_obj);
+
+    # associate original file with doc object
+    my $cursection = $doc_obj->get_top_section();
+    my $filename = &util::filename_cat($base_dir, $file);
+    $doc_obj->associate_file($filename, "doc.$doc_ext", undef, $cursection);
+
+    my $doclink = "<a href=\"_httpcollection_/index/assoc/[archivedir]/doc.$doc_ext\">";
+    $doc_obj->add_utf8_metadata ($cursection, "srclink",  $doclink); 
+    $doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icon".$doc_ext."_"); 
+    $doc_obj->add_utf8_metadata ($cursection, "/srclink", "</a>"); 
+    return $ret_val;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/DBPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/DBPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/DBPlug.pm	(revision 21112)
@@ -0,0 +1,354 @@
+###########################################################################
+#
+# DBPlug.pm -- plugin to import records from a database
+# 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2003 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+#
+# See <GSDLHOME>/etc/packages/example.dbi for an example config file!!
+#
+
+# Written by John McPherson for the NZDL project
+# Mar, Apr 2003
+
+package DBPlug;
+
+use strict;
+no strict 'refs'; # allow variable as a filehandle
+
+use BasPlug;
+use unicode;
+
+use DBI; # database independent stuff
+
+sub BEGIN {
+    @DBPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" }];
+
+my $options = { 'name'     => "DBPlug",
+		'desc'     => "{DBPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.dbi$^;
+}
+# we don't have a per-greenstone document process() function!
+sub process {
+
+}
+
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    # see if we can handle the passed file...
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	# this plugin can't process this file type...
+        return undef;
+    }
+
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    print $outhandle "DBPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+    
+    # calculate the document hash, for document ids
+    my $hash="";
+
+    my $osexe = &util::get_os_exe();
+    my $hashfile_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",
+					   $ENV{'GSDLOS'},"hashfile$osexe");
+    if (-e "$hashfile_exe") {
+	$hash = `hashfile$osexe \"$filename\"`;
+	$hash =~ /:\s*([0-9a-f]+)/i;
+	$hash="HASH$1";
+    }
+
+
+    # default options - may be overridden by config file
+    my $language=undef;
+    my $encoding=undef;
+    my $dbplug_debug=0;
+    my $username='';
+    my $password='';
+
+    # these settings must be set by the config file:
+    my $db=undef;
+
+# get id of pages from "nonempty", get latest version number from "recent", and
+# then get pagename from "page" and content from "version" !
+
+    my $sql_query = undef ;
+
+    my %db_to_greenstone_fields=();
+    my %callbacks=();
+
+    # read in config file.
+    if (!open (CONF, $filename)) {
+	    print $outhandle "DBPlug: can't read $filename: $!\n";
+	    return 0;
+    }
+    my $line;
+    my $statement="";
+    my $callback="";
+    while (defined($line=<CONF>)) {
+	chomp $line;
+	$line =~ s/\s*\#.*$//mg; # remove comments
+	$statement .= $line;
+
+	if ($line =~ /^\}\s*$/ && $callback) { # ends the callback
+	    $callback .= $statement ; $statement = "";
+	    # try to check that the function is "safe"
+	    if ($callback  =~ /\b(?:system|open|pipe|readpipe|qx|kill|eval|do|use|require|exec|fork)\b/ ||
+		$callback =~ /[\`]|\|\-/) {
+		# no backticks or functions that start new processes allowed
+		print $outhandle "DBPlug: bad function in callback\n";
+		return 0;
+	    }
+	    $callback =~ s/sub (\w+?)_callback/sub/;
+	    my $fieldname=$1;
+	    eval "\$callbacks{'$fieldname'} = $callback ; 1";
+	    $callback="";
+	} elsif ($callback) {
+	    # add this line to the callback function
+	    $callback .= $statement;
+	    $statement = "";
+	} elsif ($statement =~ m/;\s*$/) { # ends with ";"
+	    # check that it is safe
+	    # assignment
+	    if ($statement =~ m~(\$\w+)\s* = \s*
+		(\d		# digits
+		 | ".*?(?<!\\)" # " up to the next " not preceded by a \
+		 | '.*?(?<!\\)' # ' up to the next ' not preceded by a \
+		)\s*;~x ||      # /x means ignore comments and whitespace in rx
+		$statement =~ m~(\%\w+)\s*=\s*(\([\w\s\"\',:=>]+\))\s*;~ ) {   
+		# evaluate the assignment, return 1 on success "
+		if (!eval "$1=$2; 1") {
+		    my $err=$@;
+		    chomp $err;
+		    $err =~ s/\.$//; # remove a trailing .
+		    print $outhandle "DBPlug: error evaluating `$statement'\n";
+		    print $outhandle " $err (in $filename)\n";
+		    return 0; # there was an error reading the config file
+		}
+	    } elsif ($statement =~ /sub \w+_callback/) {
+		# this is the start of a callback function definition
+		$callback = $statement;
+		$statement = "";
+	    } else {
+		print $outhandle "DBPlug: skipping statement `$statement'\n";
+	    }
+	    $statement = "";
+	}
+    }
+    close CONF;
+
+    if (!defined($db)) {
+	print $outhandle "DBPlug: error: $filename does not specify a db!\n";
+	return 0;
+    }
+    if (!defined($sql_query)) {
+    	print $outhandle "DBPlug: error: no SQL query specified!\n";
+	return 0;
+    }
+    # connect to database
+    my $dbhandle=DBI->connect($db, $username, $password);
+
+    if (!defined($dbhandle)) {
+	die "DBPlug: could not connect to database, exiting.\n";
+    }
+    if (defined($dbplug_debug) && $dbplug_debug==1) {
+	print $outhandle "DBPlug (debug): connected ok\n";
+    }
+
+    my $statement_hand=$dbhandle->prepare($sql_query);
+    $statement_hand->execute;
+
+    # get the array-ref for the field names and cast it to array
+    my @field_names;
+    @field_names=@{ $statement_hand->{NAME} };
+
+    foreach my $fieldname (@field_names) {
+	if (defined($db_to_greenstone_fields{$fieldname})) {
+	    if (defined($dbplug_debug) && $dbplug_debug==1) {
+		print $outhandle "DBPlug (debug): mapping db field "
+		    . "'$fieldname' to "
+			. $db_to_greenstone_fields{$fieldname} . "\n";
+	    }
+	    $fieldname=$db_to_greenstone_fields{$fieldname};
+	}
+    }
+
+#    print "DBPlug: names: " . join (", ", @field_names) . ".\n";
+    # get rows
+
+    my $count = 0;
+    my @row_array;
+
+    @row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
+    
+    while (scalar(@row_array)) {
+	if (defined($dbplug_debug) && $dbplug_debug==1) {
+	    print $outhandle "DBPlug (debug): retrieved a row from query\n";
+	}
+
+	# create a new document
+	my $doc_obj = new doc ($filename, "indexed_doc");
+	$doc_obj->set_OIDtype ($processor->{'OIDtype'});
+	my $cursection = $doc_obj->get_top_section();
+
+	# if $language not set in config file, will use BasPlug's default
+	if (defined($language)) {
+	    $doc_obj->add_utf8_metadata($cursection, "Language", $language);
+	}
+	# if $encoding not set in config file, will use BasPlug's default
+	if (defined($encoding)) {
+	    # allow some common aliases
+	    if ($encoding =~ m/^utf[-_]8$/i) {$encoding="utf8"}
+	    $encoding =~ s/-/_/g; # greenstone uses eg iso_8859_1
+	    $doc_obj->add_utf8_metadata($cursection, "Encoding", $encoding);
+	}
+	$doc_obj->add_utf8_metadata($cursection,
+				    "Source", &ghtml::dmsafe($db));
+	if ($self->{'cover_image'}) {
+	    $self->associate_cover_image($doc_obj, $filename);
+	}
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+
+	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "DB");
+
+	# include any metadata passed in from previous plugins 
+	# note that this metadata is associated with the top level section
+	$self->extra_metadata ($doc_obj, $cursection,
+			       $metadata);
+
+	# do any automatic metadata extraction
+	$self->auto_extract_metadata ($doc_obj);
+
+	my $unique_id=undef;
+
+	foreach my $fieldname (@field_names) {
+	    my $fielddata=shift @row_array;
+	    # use the specified encoding, defaulting to utf-8
+	    if (defined($encoding) && $encoding ne "ascii"
+	    	&& $encoding ne "utf8") {
+	      $fielddata=&unicode::unicode2utf8(
+	      	&unicode::convert2unicode($encoding, \$fielddata)
+					       );
+	    }
+	    # see if we have a ****_callback() function defined
+	    if (exists $callbacks{$fieldname}) {
+		my $funcptr = $callbacks{$fieldname};
+		$fielddata = &$funcptr($fielddata);
+	    }
+
+	    if ($fieldname eq "text") {
+		# add as document text
+		$fielddata=~s@<@&lt;@g;
+		$fielddata=~s@>@&gt;@g; # for xml protection...
+		$fielddata=~s@_@\\_@g; # for macro language protection...
+		$doc_obj->add_utf8_text($cursection, $fielddata);
+	    } elsif ($fieldname eq "Identifier") {
+		# use as greenstone's unique record id
+		if ($fielddata =~ /^\d+$/) {
+		    # don't allow IDs that are completely numeric
+		    $unique_id="id" . $fielddata;
+		} else {
+		    $unique_id=$fielddata;
+		}
+	    } else {
+		# add as document metadata
+		$fielddata=~s/\[/&#91;/g;
+		$fielddata=~s/\]/&#93;/g;
+		$doc_obj->add_utf8_metadata($cursection,
+					    $fieldname, $fielddata);
+
+	    }
+	}
+
+	if (!defined $unique_id) {
+	    $doc_obj->set_OID($hash . "s$count");
+	} else {
+	    # use our id from the database...
+	    $doc_obj->set_OID($unique_id);
+	}
+
+
+        # process the document
+	$processor->process($doc_obj);
+
+
+	$count++;
+
+	# get next row
+	@row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
+    } # end of row_array is not empty
+
+    # check "$sth->err" if empty array for error
+    if ($statement_hand->err) {
+	print $outhandle "DBPlug: received error: \"" .
+	    $statement_hand->errstr . "\"\n";
+    }
+
+    # clean up connection to database
+    $statement_hand->finish();
+    $dbhandle->disconnect();
+
+    # num of input files, rather than documents created?
+    $self->{'num_processed'}++;
+
+    if (defined($dbplug_debug) && $dbplug_debug==1) {
+        print $outhandle "DBPlug: imported $count DB records as documents.\n";
+    }
+    $count;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/DSpacePlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/DSpacePlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/DSpacePlug.pm	(revision 21112)
@@ -0,0 +1,409 @@
+
+###########################################################################
+#
+# DSpacePlug.pm -- plugin for import the collection from DSpace 
+# 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# DSpace Plug - 10/2004
+# 
+#
+# This plugin takes "contents" and dublin_core.xml file, which contain 
+# Metadata and lists of associated files for a particular document
+# and produces a document containing sections, one for each page.
+# The files should be named "contents" and "dublin_core.xml".  For each of 
+# document in DSpace, it is stored in one directory
+#
+# The format of the "contents" file is as follows:
+# 
+# File.type      bundle:ORIGINAL
+# license.txt    bundle:LICENSE
+# The format of the "dublin_core.xml" file is as follows:
+# The first line contains any metadata for the whole document
+# <dublin_core>
+# eg.
+# <dcvalue element="Title" qualifier="">Snail farming</dcvalue>
+# <dcvalue element="date" qualifier="">2004-10-15</dcvalue>
+#
+
+package DSpacePlug;
+
+use BasPlug;
+use plugin;
+#use ghtml;
+use XMLParser;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @DSpacePlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "only_first_doc",
+	'desc' => "{DSpacePlug.only_first_doc}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "first_inorder_ext",
+	'desc' => "{DSpacePlug.first_inorder_ext}",
+	'type' => "string",
+	'reqd' => "no" },
+      { 'name' => "first_inorder_mime",
+	'desc' => "{DSpacePlug.first_inorder_mime}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_block_exp(),
+	'reqd' => "no" }];
+
+
+my $options = { 'name'     => "DSpacePlug",
+		'desc'     => "{DSpacePlug.desc}",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+my $primary_doc_lookup = { 'text/html' => '(?i)\.(gif|jpe?g|jpe|jpg|png|css)$' };
+
+# Important variation to regular plugin structure.  Need to declare
+# $self as global variable to file so XMLParser callback routines
+# can access the content of the object. 
+my ($self); 
+
+sub new {
+    my ($class) = shift (@_); 
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    
+    #create XML::Parser object for parsing dublin_core.xml files
+    my $parser = new XML::Parser('Style' => 'Stream',
+				 'Handlers' => {'Char' => \&Char,
+						'Doctype' => \&Doctype
+						});
+    $self->{'parser'} = $parser;
+    $self->{'extra_blocks'} = {};
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)contents$^;
+}
+
+# want to block all files except the "contents"
+sub get_default_block_exp {
+    my $self = shift (@_);
+    
+    # Block all files besides contents
+    return q^(?i)(handle|dublin_core\.xml|\.tx?t)$^;
+}
+
+sub read_content
+{
+    my $self = shift (@_);
+    my ($dir, $only_first_doc, $first_inorder_ext, $first_inorder_mime, $mimetype_list) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my @fnamemime_list = ();
+    my @assocmime_list = ();
+
+    my $content_fname = &util::filename_cat($dir,"contents");
+
+    open(CIN,"<$content_fname") 
+	|| die "Unable to open $content_fname: $!\n";
+    
+    my $line;
+    my $pos = 0;
+
+    while (defined ($line = <CIN>)) {
+	if ($line =~ m/^(.*)\s+bundle:ORIGINAL\s*$/) {
+	    my $fname = $1;
+	    my $mtype = $mimetype_list->[$pos];
+	    my $fm_rec = { 'file' => $fname, 'mimetype' => $mtype};
+	    push(@fnamemime_list,$fm_rec);
+	    $pos++;
+	}
+    }
+    close CIN;
+
+    if ($only_first_doc){
+	my ($first_fname, @rest_fnames) = @fnamemime_list;
+	@fnamemime_list = ($first_fname);
+	@assocmime_list = @rest_fnames;
+    }
+
+    # allow user to specify the types of files (inorder)they would like to assign as
+    # a primary bitstream
+    if ($first_inorder_ext) {
+	# parse user-define file extension names
+	my @extfiles_list = split /,/, $first_inorder_ext;
+	my (@rest_fnames) = @fnamemime_list;
+	my @matched_list = ();
+	foreach my $file_ext (@extfiles_list) {
+	    $pos = 0;
+	    foreach my $allfiles (@fnamemime_list){
+		$allfiles->{'file'} =~ /^(.*)\.(.*?)$/;
+		my $allfiles_ext = $2;
+
+		if ($allfiles_ext =~ /$file_ext/) {
+		    print $outhandle "Existing file:$allfiles->{'file'} match the user-define File Extension:$file_ext\n";
+		    push (@matched_list, $allfiles);
+
+		    # delete the matched extension file from the array
+		    splice(@rest_fnames,$pos,1);
+
+		    return (\@matched_list, \@rest_fnames);
+
+		}
+		$pos++;
+	    }
+	}
+    }
+    
+    if ($first_inorder_mime) {
+	# parse user-define file mimetype
+	my @file_mime_list = split /,/, $first_inorder_mime;
+	my (@rest_fnames) = @fnamemime_list;
+	my @matched_list = ();
+	foreach my $file_mime (@file_mime_list) {
+	    $pos = 0;
+	    foreach my $allfiles (@fnamemime_list){
+		my $allfiles_mime = $allfiles->{'mimetype'};
+
+		if ($allfiles_mime =~ /$file_mime/) {
+		    print $outhandle "Existing file:$allfiles->{'file'} match the user-defined File MimeType:$file_mime\n";
+		    push (@matched_list, $allfiles);
+
+		    # delete the matched MIMEType file from the array
+		    splice(@rest_fnames,$pos,1);
+		    return (\@matched_list, \@rest_fnames);
+		}
+		$pos++;
+	    }
+	}
+    }
+    return (\@fnamemime_list, \@assocmime_list);
+}
+
+
+sub filemime_list_to_re
+{
+    my $self = shift (@_);
+    my ($fnamemime_list) = @_;
+
+    my @fname_list = map { "(".$_->{'file'}.")" } @$fnamemime_list;
+    my $fname_re = join("|",@fname_list);
+    $fname_re =~ s/\./\\\./g;
+
+    return $fname_re;
+}
+
+# Read dublin_core metadata from DSpace collection 
+sub metadata_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    my $only_first_doc = $self->{'only_first_doc'};
+    my $first_inorder_ext = $self->{'first_inorder_ext'};
+    my $first_inorder_mime = $self->{'first_inorder_mime'};
+    
+    my $outhandle = $self->{'outhandle'};
+    
+    my $filename = &util::filename_cat($base_dir, $file);
+    # return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    
+    if ($filename !~ /dublin_core\.xml$/ || !-f $filename) {
+	return undef;
+    }
+    
+    print $outhandle "DSpacePlug: extracting metadata from $file\n"
+	if $self->{'verbosity'} > 1;
+    
+    my ($dir) = $filename =~ /^(.*?)[^\/\\]*$/;
+
+    eval {
+	$self->{'parser'}->parsefile($filename);
+    };
+    
+    if ($@) {
+	die "DSpacePlug: ERROR $filename is not a well formed dublin_core.xml file ($@)\n";
+    }
+
+    my $mimetype_list = $self->{'saved_metadata'}->{'dc.Format^mimetype'};
+    my ($doc_file_mimes, $assoc_file_mimes) = $self->read_content($dir, $only_first_doc, $first_inorder_ext, 
+								  $first_inorder_mime, $mimetype_list);
+
+    my $file_re = $self->filemime_list_to_re($doc_file_mimes);
+
+    if ($only_first_doc || $first_inorder_ext || $first_inorder_mime) {	
+	foreach my $afm ( @$assoc_file_mimes ) {
+ 	    my $full_af = &util::filename_cat($dir,$afm->{'file'});
+	    $self->{'extra_blocks'}->{$full_af} = 1;
+	}
+    }
+    push(@$extrametakeys,$file_re);
+
+    if (defined $self->{'saved_metadata'}->{'dc.Format^extent'}) {
+	delete $self->{'saved_metadata'}->{'dc.Format^extent'};
+    }
+
+    if (defined $mimetype_list) {
+	delete $self->{'saved_metadata'}->{'dc.Format^mimetype'};
+
+	# Temporarily store associate file info in metadata table
+	# This will be removed in 'extra_metadata' in BasPlug and used
+	# to perform the actual file association (once the doc obj has
+	# been formed
+
+	my $main_doc = $doc_file_mimes->[0];
+	my $md_mimetype = $main_doc->{'mimetype'};
+
+	my $pd_lookup = $primary_doc_lookup->{$md_mimetype};
+
+	if (defined $pd_lookup) {
+	    my $filter_re = $pd_lookup;
+	    @$assoc_file_mimes = grep { $_->{'file'} !~ m/$filter_re/ }  @$assoc_file_mimes;
+	}
+
+	my @gsdlassocfile_tobe 
+	    = map { &util::filename_cat($dir,$_->{'file'}).":".$_->{'mimetype'}.":" } @$assoc_file_mimes;
+	$self->{'saved_metadata'}->{'gsdlassocfile_tobe'} = \@gsdlassocfile_tobe;
+
+    }
+    
+    $extrametadata->{$file_re} = $self->{'saved_metadata'};
+
+    return 1;
+}
+
+
+# The DSpacePlug read() function. This function does all the right things
+# to make general options work for a given plugin. It calls the process()
+# function which does all the work specific to a plugin (like the old
+# read functions used to do). Most plugins should define their own 
+# process() function and let this read() function keep control.
+#
+# DSpace overrides read() because there is no need to read the actual 
+# text of the file in, because the contents of the file is not text...
+#
+# Return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    # Block all files except contents
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+
+    my $assocfile = $metadata->{'assocfile'};
+
+    return 0 if (($filename =~ /dublin_core\.xml$/) || ($filename =~ /contents$/));
+    return 0 if (defined $self->{'extra_blocks'}->{$filename});
+    return undef;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+sub Doctype {
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+
+    die if ($name !~ /^dublin_core$/);
+}
+
+sub StartTag {
+    my ($expat, $element) = @_;
+    if ($element eq "dublin_core") {
+	$self->{'saved_metadata'} = {};
+    } elsif ($element eq "dcvalue") {
+	my $metaname = $_{'element'};
+	my $qualifier = $_{'qualifier'}||"";
+	if ($metaname ne "description") {
+	    $metaname .= "^$qualifier" if ($qualifier ne "none" && $qualifier ne "");
+	    $self->{'metaname'} = "dc.\u$metaname";
+	}
+    }
+}
+
+sub EndTag {
+    my ($expat, $element) = @_;
+    
+    if ($element eq "dcvalue") {
+	$self->{'metaname'} = "";
+    }
+}
+
+sub Text {
+    if (defined ($self->{'metaname'}) && $self->{'metaname'} ne "") {
+	# $_ == Metadata content
+	my $mname = $self->{'metaname'};
+	if (defined $self->{'saved_metadata'}->{$mname}) {
+	    # accumulate - add value to existing value(s)
+	    if (ref ($self->{'saved_metadata'}->{$mname}) eq "ARRAY") {
+		push (@{$self->{'saved_metadata'}->{$mname}}, $_);
+	    } else {
+		$self->{'saved_metadata'}->{$mname} = 
+		    [$self->{'saved_metadata'}->{$mname}, $_];
+	    }
+	} else {
+	    # accumulate - add value into (currently empty) array
+	    $self->{'saved_metadata'}->{$mname} = [$_];
+	}
+
+    }
+}
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {
+    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
+    $_[0]->{'Text'} .= $_[1];
+    return undef;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/EMAILPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/EMAILPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/EMAILPlug.pm	(revision 21112)
@@ -0,0 +1,1122 @@
+###########################################################################
+#
+# EMAILPlug.pm - a plugin for parsing email files
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2002 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+
+# EMAILPlug
+#
+# by Gordon Paynter (gwp@cs.waikato.ac.nz)
+#
+# Email plug reads email files.  These are named with a simple
+# number (i.e. as they appear in maildir folders) or with the 
+# extension .mbx (for mbox mail file format)
+#
+# Document text:
+#   The document text consists of all the text 
+#   after the first blank line in the document.
+#
+# Metadata (not Dublin Core!):
+#   $Headers      All the header content (optional, not stored by default)
+#   $Subject      Subject: header
+#   $To           To: header
+#   $From         From: header
+#   $FromName     Name of sender (where available)
+#   $FromAddr     E-mail address of sender
+#   $DateText     Date: header
+#   $Date         Date: header in GSDL format (eg: 19990924)
+#
+#   $Title	  made up of Subject, Date and Sender (for default formatting)
+#
+#
+# John McPherson - June/July 2001
+# added (basic) MIME support and quoted-printable and base64 decodings.
+# Minor fixes for names that are actually email addresses (ie <...> was lost)
+#
+# See:  * RFC 822  - ARPA Internet Text Messages
+#       * RFC 2045 - Multipurpose Internet Mail Extensions (MIME) -part1
+#       * RFC 2046 - MIME (part 2)  Media Types (and multipart messages)
+#       * RFC 2047 - MIME (part 3)  Message Header Extensions
+#       * RFC 1806 - Content Dispositions (ie inline/attachment)
+
+# 12/05/02 Added usage datastructure - John Thompson
+package EMAILPlug;
+
+use strict;
+no strict "refs"; # so we can use a variable as a filehandle for print $out
+
+
+use SplitPlug;
+use unicode;  # gs conv functions
+use gsprintf 'gsprintf'; # translations
+
+use sorttools;
+use util;
+
+sub BEGIN {
+    @EMAILPlug::ISA = ('SplitPlug');
+}
+
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "no_attachments",
+	'desc' => "{EMAILPlug.no_attachments}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "headers",
+	'desc' => "{EMAILPlug.headers}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "split_exp",
+	'desc' => "{EMAILPlug.split_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_split_exp() } 
+      ];
+
+my $options = { 'name'     => "EMAILPlug",
+		'desc'     => "{EMAILPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+# Create a new EMAILPlug object with which to parse a file.
+# Accomplished by creating a new BasPlug and using bless to 
+# turn it into an EMAILPlug.
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    $self->{'assoc_filenames'} = {}; # to save attach names so we don't clobber
+
+    # this might not actually be true at read-time, but after processing
+    # it should all be utf8.
+    $self->{'input_encoding'}="utf8";
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    # mbx/email for mailbox file format, \d+ for maildir (each message is
+    # in a separate file, with a unique number for filename)
+    # mozilla and IE will save individual mbx format files with a ".eml" ext.
+    return q@([\\/]\d+|\.(mbx|email|eml))$@;
+}
+
+# This plugin splits the mbox mail files at lines starting with From<sp>
+# It is supposed to be "\n\nFrom ", but this isn't always used.
+# add \d{4} so that the line ends in a year (in case the text has an
+# unescaped "From " at the start of a line).
+sub get_default_split_exp {
+    return q^\nFrom .*\d{4}\n^;
+    
+}
+
+
+# do plugin specific processing of doc_obj
+sub process {
+
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # Check that we're dealing with a valid mail file
+    # mbox message files start with "From "
+    # maildir messages usually start with Return-Path and Delivered-To
+    # mh is very similar to maildir
+    my $startoffile=substr($$textref,0,256);
+    if (($startoffile !~ /^(From )/) &&
+	($startoffile !~ /^(From|To|Envelope.*|Received|Return-Path|Date|Subject|Content\-.*|MIME-Version|Forwarded):/im)) {
+	return undef;
+    }
+
+
+    print STDERR "<Processing n='$file' p='EMAILPlug'>\n" if ($gli);
+
+    gsprintf($outhandle, "EMAILPlug: {common.processing} $file\n")
+	if $self->{'verbosity'} > 1;
+
+    my $cursection = $doc_obj->get_top_section();
+
+    #
+    # Parse the document's text and extract metadata
+    #
+
+    # Protect backslashes
+    $$textref =~ s@\\@\\\\@g;
+
+    # Separate header from body of message
+    my $Headers = $$textref;
+    $Headers =~ s/\r?\n\r?\n(.*)$//s;
+    $$textref = $1;
+    $Headers .= "\n";
+    
+    # Unfold headers - see rfc822
+    $Headers =~ s/\r?\n[\t\ ]+/ /gs;
+    # Extract basic metadata from header
+    my @headers = ("From", "To", "Subject", "Date");
+    my %raw;
+    foreach my $name (@headers) {
+	$raw{$name} = "No $name value";
+    }
+
+    # Get a default encoding for the header - RFC says should be ascii...
+    my $default_header_encoding="iso_8859_1";
+
+    # We don't know what character set is the user's default...
+    # We could use textcat to guess... for now we'll look at mime content-type
+#    if ($Headers =~ /([[:^ascii:]])/) {
+#    }
+    if ($Headers =~ /^Content\-type:.*charset=\"?([a-z0-9\-_]+)/mi) {
+	$default_header_encoding=$1;
+	$default_header_encoding =~ s@\-@_@g;
+	$default_header_encoding =~ tr/A-Z/a-z/;
+    }
+
+
+    # Examine each line of the headers
+    my ($line, $name, $value);
+    my @parts;
+    foreach $line (split(/\n/, $Headers)) {
+	
+	# Ignore lines with no content or which begin with whitespace
+	next unless ($line =~ /:/);
+	next if ($line =~ /^\s/);
+
+	# Find out what metadata is on this line
+	@parts = split(/:/, $line);
+	$name = shift @parts;
+        # get fieldname in canonical form - first cap, then lower case.
+	$name =~ tr/A-Z/a-z/;
+        # uppercase the first character according to the current locale
+	$name=~s/(.+)/\u$1/;
+	next unless $name;
+	next unless ($raw{$name});
+
+	# Find the value of that metadata
+	$value = join(":", @parts);
+	$value =~ s/^\s+//;
+	$value =~ s/\s+$//;
+	# decode headers if stored using =?<charset>?[BQ]?<data>?= (rfc2047)
+	if ($value =~ /=\?.*\?[BbQq]\?.*\?=/) {
+	    my $original_value=$value;
+	    my $encoded=$value;
+	    $value="";
+	    # we should ignore spaces between consecutive encoded-texts
+	    $encoded =~ s@\?=\s+=\?@\?==\?@g;
+	    while ($encoded =~ s/(.*?)=\?([^\?]*)\?([bq])\?([^\?]+)\?=//i) {
+		my ($charset, $encoding, $data)=($2,$3,$4);
+		my ($decoded_data);
+		$value.="$1"; # any leading chars
+		$data=~s/^\s*//; $data=~s/\s*$//; # strip whitespace from ends
+		chomp $data;
+		$encoding =~ tr/BQ/bq/;
+		if ($encoding eq "q") { # quoted printable
+		    $data =~ s/_/\ /g;  # from rfc2047 (sec 4.2.2)
+		    $decoded_data=qp_decode($data);
+		    # qp_decode adds \n, which is default for body text
+		    chomp($decoded_data);
+		} else { # base 64
+		    $decoded_data=base64_decode($data);
+		}
+		$self->convert2unicode($charset, \$decoded_data);
+		$value .= $decoded_data;
+	      } # end of while loop
+	    
+	    # get any trailing characters
+	    $self->convert2unicode($default_header_encoding, \$encoded);
+	    $value.=$encoded;
+
+	    if ($value =~ /^\s*$/) { # we couldn't extract anything...
+	      $self->convert2unicode($default_header_encoding,
+				     \$original_value);
+	      $value=$original_value;
+	    }
+        } # end of if =?...?=
+
+        # In the absense of other charset information, assume the
+        # header is the default (usually "iso_8859_1") and convert to unicode.
+        else {
+	    $self->convert2unicode($default_header_encoding, \$value);
+	}
+	
+	# Store the metadata
+	$value =~ s@_@\\_@g; # protect against GS macro language
+	$raw{$name} = $value;
+    }
+
+    # Extract the name and e-mail address from the From metadata
+    my $frommeta = $raw{"From"};
+    my $fromnamemeta;
+    my $fromaddrmeta;
+
+    $frommeta =~ s/\s*$//;  # Remove trailing space, if any
+
+    if ($frommeta =~ m/(.+)\s*<(.+)>/) {
+	$fromnamemeta=$1;
+	$fromaddrmeta=$2;
+    } elsif ($frommeta =~ m/(.+@.+)\s+\((.*)\)/) {
+	$fromnamemeta=$2;
+	$fromaddrmeta=$1;
+    } 
+    if (!defined($fromaddrmeta)) {
+	$fromaddrmeta=$frommeta;
+    }
+    $fromaddrmeta=~s/<//; $fromaddrmeta=~s/>//;
+    # minor attempt to prevent spam-bots from harvesting addresses...
+    $fromaddrmeta=~s/@/&#64;/;
+
+    $doc_obj->add_utf8_metadata ($cursection, "FromAddr", $fromaddrmeta);
+
+    if (defined($fromnamemeta) && $fromnamemeta) { # must be > 0 long
+	$fromnamemeta =~ s/\"//g;  # remove quotes
+	$fromnamemeta =~ s/\s+$//; # remove trailing whitespace
+    }
+    else {
+	$fromnamemeta = $fromaddrmeta;
+    }
+    # if name is an address
+    $fromnamemeta =~ s/<//g; $fromnamemeta =~ s/>//g;
+    $fromnamemeta=~s/@/&#64\;/;
+    $doc_obj->add_utf8_metadata ($cursection, "FromName", $fromnamemeta);
+
+    $raw{"From"}=$frommeta;
+
+    # Process Date information
+    if ($raw{"Date"} !~ /No Date/) {
+	$raw{"DateText"} = $raw{"Date"};
+	
+	# Convert the date text to internal date format
+	$value = $raw{"Date"};
+	my ($day, $month, $year) = $value =~ /(\d?\d)\s([A-Z][a-z][a-z])\s(\d\d\d?\d?)/;
+	# make some assumptions about the year formatting...
+	# some (old) software thinks 2001 is 101, some think 2001 is 01
+	if ($year < 20) { $year += 2000; } # assume not really 1920...
+	elsif ($year < 150) { $year += 1900; } # assume not really 2150...
+	$raw{"Date"} = &sorttools::format_date($day, $month, $year);
+	
+    } else {
+	# We have not extracted a date
+	$raw{"DateText"} = "Unknown.";
+	$raw{"Date"} = "19000000";
+    }
+
+    # Add extracted metadata to document object
+    foreach my $name (keys %raw) {
+	$value = $raw{$name};
+	if ($value) {
+	    # assume subject, etc headers have no special HTML meaning.
+	    $value = &text_into_html($value);
+	    # escape [] so it isn't re-interpreted as metadata
+	    $value =~ s/\[/&#91;/g; $value =~ s/\]/&#93;/g;
+	} else {
+	    $value = "No $name field";
+	}
+	$doc_obj->add_utf8_metadata ($cursection, $name, $value);
+    }
+
+
+    # extract a message ID from the headers, if there is one, and we'll use
+    # that as the greenstone doc ID. Having a predictable ID means we can
+    # link to other messages, eg from In-Reply-To or References headers...
+    if ($Headers =~ m@^Message-ID:(.+)$@mi) {
+	my $id=escape_msg_id($1);
+	$doc_obj->{'msgid'}=$id;
+    }
+    # link to another message, if this is a reply
+    if ($Headers =~ m@^In-Reply-To:(.+)$@mi) {
+	my $id=escape_msg_id($1);
+	$doc_obj->add_utf8_metadata ($cursection, 'InReplyTo', $id);
+    } elsif ($Headers =~ m@^References:.*\s([^\s]+)$@mi) {
+	# References can have multiple, get the last one
+	my $id=escape_msg_id($1);
+	# not necessarily in-reply-to, but same thread...
+	$doc_obj->add_utf8_metadata ($cursection, 'InReplyTo', $id);
+    }
+
+
+
+    my $mimetype="text/plain";
+    my $mimeinfo="";
+    my $charset = $default_header_encoding;
+    # Do MIME and encoding stuff. Allow \s in mimeinfo in case there is
+    # more than one parameter given to Content-type.
+    # eg: Content-type: text/plain; charset="us-ascii"; format="flowed"
+    if ($Headers =~ m@^content\-type:\s*([\w\.\-/]+)\s*(\;\s*.+)?\s*$@mi)
+	{
+	    $mimetype=$1;
+	    $mimetype =~ tr/[A-Z]/[a-z]/;
+
+	    if ($mimetype eq "text") { # for pre-RFC2045 messages (c. 1996)
+		$mimetype = "text/plain";
+	    }
+
+	    $mimeinfo=$2;
+	    if (!defined $mimeinfo) {
+		$mimeinfo="";
+	    } else { # strip leading and trailing stuff
+		$mimeinfo =~ s/^\;\s*//;
+		$mimeinfo =~ s/\s*$//;
+	    }
+	    if ($mimeinfo =~ /charset=\"([^\"]+)\"/i) {
+	      $charset = $1;
+	    }
+	}
+
+    my $transfer_encoding="7bit";
+    if ($Headers =~ /^content-transfer-encoding:\s*([^\s]+)\s*$/mi) {
+	$transfer_encoding=$1;
+    }
+
+    if ($mimetype eq "text/html") {
+	$$textref= $self->text_from_part($$textref, $Headers);
+    } elsif ($mimetype ne "text/plain") {
+	$self->{'doc_obj'} = $doc_obj; # in case we need to associate files...
+	$$textref=$self->text_from_mime_message($mimetype,$mimeinfo,$$textref);
+    } else { # mimetype eq text/plain
+
+	if ($transfer_encoding =~ /quoted\-printable/) {
+	    $$textref=qp_decode($$textref);
+	} elsif ($transfer_encoding =~ /base64/) {
+	    $$textref=base64_decode($$textref);
+	}
+	$self->convert2unicode($charset, $textref);
+
+	$$textref = &text_into_html($$textref);
+	$$textref =~ s@_@\\_@g; # protect against GS macro language
+
+    }
+
+    
+    if ($self->{'headers'} && $self->{'headers'} == 1) {
+	# Add "All headers" metadata
+	$Headers = &text_into_html($Headers);
+
+	$Headers = "No headers" unless ($Headers =~ /\w/);
+	$Headers =~ s/@/&#64\;/g;
+	# escape [] so it isn't re-interpreted as metadata
+	$Headers =~ s/\[/&#91;/g; $Headers =~ s/\]/&#93;/g;
+	$self->convert2unicode($charset, \$Headers);
+
+	$Headers =~ s@_@\\_@g; # protect against GS macro language
+	$doc_obj->add_utf8_metadata ($cursection, "Headers", $Headers);
+    }
+
+
+    # Add Title metadata
+    my $Title = text_into_html($raw{'Subject'});
+    $Title .= "<br>From: " . text_into_html($fromnamemeta);
+    $Title .= "<br>Date: " . text_into_html($raw{'DateText'});
+    $Title =~ s/\[/&#91;/g; $Title =~ s/\]/&#93;/g;
+
+    $doc_obj->add_utf8_metadata ($cursection, "Title", $Title);
+	
+    # Add FileFormat metadata
+    $doc_obj->add_metadata($cursection, "FileFormat", "EMAIL");
+
+    # Add text to document object
+    $$textref = "No message" unless ($$textref =~ /\w/);
+
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    return 1;
+}
+
+
+# Convert a text string into HTML.
+#
+# The HTML is going to be inserted into a GML file, so 
+# we have to be careful not to use symbols like ">",
+# which ocurs frequently in email messages (and use
+# &gt instead.
+#
+# This function also turns links and email addresses into hyperlinks,
+# and replaces carriage returns with <BR> tags (and multiple carriage
+# returns with <P> tags).
+
+
+sub text_into_html {
+    my ($text) = @_;
+
+    # Convert problem characters into HTML symbols
+    $text =~ s/&/&amp;/g;
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/\"/&quot;/g;
+
+    # convert email addresses and URIs into links
+# don't markup email addresses for now
+#    $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
+
+    # try to munge email addresses a little bit...
+    $text =~ s/@/&#64;/;
+    # assume hostnames are \.\w\- only, then might have a trailing '/.*'
+    # assume URI doesn't finish with a '.'
+    $text =~ s@((http|ftp|https)://[\w\-]+(\.[\w\-]+)*/?((&amp;|\.|\%[a-f0-9]{2})?[\w\?\=\-_/~]+)*(\#[\w\.\-_]*)?)@<a href=\"$1\">$1<\/a>@gi;
+
+
+    # Clean up whitespace and convert \n charaters to <BR> or <P>
+    $text =~ s/ +/ /g;
+    $text =~ s/\s*$//g; 
+    $text =~ s/^\s*//g;
+    $text =~ s/\n/\n<br>/g;
+    $text =~ s/<br>\s*<br>/<p>/gi;
+
+    return $text;
+}
+
+
+
+
+#Process a MIME message.
+# the textref we are given DOES NOT include the header.
+sub text_from_mime_message {
+    my $self = shift(@_);
+    my ($mimetype,$mimeinfo,$text)=(@_);
+    my $outhandle=$self->{'outhandle'};
+    # Check for multiparts - $mimeinfo will be a boundary
+    if ($mimetype =~ /multipart/) {
+	my $boundary="";
+	if ($mimeinfo =~ m@boundary=(\"[^\"]+\"|[^\s]+)\s*$@im) {
+	    $boundary=$1;
+	    if ($boundary =~ m@^\"@) {
+		$boundary =~ s@^\"@@; $boundary =~ s@\"$@@;
+	    }
+	} else {
+	    print $outhandle "EMAILPlug: (warning) couldn't parse MIME boundary\n";
+	}
+	# parts start with "--$boundary"
+	# message ends with "--$boundary--"
+	# RFC says boundary is <70 chars, [A-Za-z'()+_,-./:=?], so escape any
+	# that perl might want to interpolate. Also allows spaces...
+	$boundary=~s/\\/\\\\/g;
+	$boundary=~s/([\?\+\.\(\)\:\/\'])/\\$1/g;
+	my @message_parts = split("\r?\n\-\-$boundary", "\n$text");
+	# remove first "part" and last "part" (final --)
+	shift @message_parts;
+	my $last=pop @message_parts;
+	# if our boundaries are a bit dodgy and we only found 1 part...
+	if (!defined($last)) {$last="";}
+	# make sure it is only -- and whitespace
+	if ($last !~ /^\-\-\s*$/ms) {
+	    print $outhandle "EMAILPlug: (warning) last part of MIME message isn't empty\n";
+	}
+	foreach my $message_part (@message_parts) {
+	    # remove the leading newline left from split.
+	    $message_part=~s/^\r?\n//;
+	}
+	if ($mimetype eq "multipart/alternative") {
+	    # check for an HTML version first, then TEXT, otherwise use first.
+	    my $part_text="";
+	    foreach my $message_part (@message_parts) {
+		if ($message_part =~ m@^content\-type:\s*text/html@i)
+		{
+		    # Use the HTML version
+		    $part_text = $self->text_from_part($message_part);
+		    $mimetype="text/html";
+		    last;
+		}
+	    }
+	    if ($part_text eq "") { # try getting a text part instead
+		foreach my $message_part (@message_parts) {
+		    if ($message_part =~ m@^content\-type:\s*text/plain@i)
+		    {
+			# Use the plain version
+			$part_text = $self->text_from_part($message_part);
+			if ($part_text =~/[^\s]/) {
+			    $part_text = text_into_html($part_text);
+			}
+			$mimetype="text/plain";
+			last;
+		    }
+		}
+	    }
+	    if ($part_text eq "") { #use first part (no html/text part found)
+		$part_text = $self->text_from_part(shift @message_parts);
+		$part_text = text_into_html($part_text);
+	    }
+	    if ($part_text eq "") { # we couldn't get anything!!!
+		# or it was an empty message...
+		# do nothing...
+		gsprintf($outhandle, "{BasPlug.empty_file} - empty body?\n");
+	    } else {
+		$text = $part_text;
+	    }
+	} elsif ($mimetype =~ m@multipart/(mixed|digest|related|signed)@) {
+	    $text = "";
+	    # signed is for PGP/GPG messages... the last part is a hash
+	    if ($mimetype =~ m@multipart/signed@) {
+		pop @message_parts;
+	    }
+	    my $is_first_part=1;
+	    foreach my $message_part (@message_parts) {
+		if ($is_first_part && $text ne "") {$is_first_part=0;}
+
+		if ($mimetype eq "multipart/digest") {
+		    # default type - RTFRFC!! Set if not already set
+		    $message_part =~ m@^(.*)\n\r?\n@s;
+		    my $part_header=$1;
+		    if ($part_header !~ m@^content-type@mi) {
+			$message_part="Content-type: message/rfc822\n"
+			    . $message_part; # prepend default type
+		    }
+		}
+
+		$text .= $self->process_multipart_part($message_part,
+						       $is_first_part);
+	    } # foreach message part.
+	} else {
+	    # we can't handle this multipart type (not mixed or alternative)
+	    # the RFC also mentions "parallel".
+	}
+    } # end of ($mimetype =~ multipart)
+    elsif ($mimetype =~ m@message/rfc822@) {
+	my $msg_header = $text;
+	$msg_header =~ s/\r?\n\r?\n(.*)$//s;
+	$text = $1;
+
+	if ($msg_header =~ /^content\-type:\s*([\w\.\-\/]+)\s*\;?\s*(.+?)\s*$/mi)
+	{
+	    $mimetype=$1;
+	    $mimeinfo=$2;
+	    $mimetype =~ tr/[A-Z]/[a-z]/;
+
+	    my $msg_text;
+	    if ($mimetype =~ m@multipart/@) {
+		$msg_text = text_from_mime_message($self, $mimetype, $mimeinfo,
+						   $text);
+	    } else {
+		$msg_text=$self->text_from_part($text,$msg_header);
+	    }
+
+	    my $brief_header=text_into_html(get_brief_headers($msg_header));
+	    $text= "\n<b>&lt;&lt;attached message&gt;&gt;</b><br>";
+	    $text.= "<table><tr><td width=\"5%\"> </td>\n";
+	    $text.="<td>" . $brief_header . "\n</p>" . $msg_text 
+		. "</td></tr></table>";
+	}
+    } else {
+	# we don't do any processing of the content.
+    }
+
+    return $text;
+}
+
+
+
+# used for turning a message id into a more friendly string for greenstone
+sub escape_msg_id {
+#msgid
+    my $id=shift;
+    chomp $id; $id =~ s!\s!!g; # remove spaces
+    $id =~ s![<>\[\]]!!g; # remove [ ] < and >
+    $id =~ s![_&]!-!g; # replace symbols that might cause problems
+    $id =~ s!\.!-!g; # . means section to greenstone doc ids!
+    $id =~ s!@!-!g; # replace @ symbol, to avoid spambots
+    return $id;
+}
+
+
+
+sub process_multipart_part {
+    my $self = shift;
+    my $message_part = shift;
+    my $is_first_part = shift;
+
+    my $return_text="";
+    my $part_header=$message_part;
+    my $part_body;
+    if ($message_part=~ /^\s*\n/) {
+	# no header... use defaults
+	$part_body=$message_part;
+	$part_header="Content-type: text/plain; charset=us-ascii";
+    } elsif ($part_header=~s/\r?\n\r?\n(.*)$//s) {
+	$part_body=$1;
+    } else {
+	# something's gone wrong...
+	$part_header="";
+	$part_body=$message_part;
+    }
+    
+    $part_header =~ s/\r?\n[\t\ ]+/ /gs; #unfold
+    my $part_content_type="";
+    my $part_content_info="";
+
+    if ($part_header =~ m@^content\-type:\s*([\w\.\-/]+)\s*(\;.*)?$@mi) {
+	$part_content_type=$1; $part_content_type =~ tr/A-Z/a-z/;
+	$part_content_info=$2;
+	if (!defined($part_content_info)) {
+	    $part_content_info="";
+	} else {
+	    $part_content_info =~ s/^\;\s*//;
+	    $part_content_info =~ s/\s*$//;
+	}
+    }
+    my $filename="";
+    if ($part_header =~ m@name=\"?([^\"\n]+)\"?@mis) {
+	$filename=$1;
+	$filename =~ s@\r?\s*$@@; # remove trailing space, if any
+    }
+    
+    # disposition - either inline or attachment.
+    # NOT CURRENTLY USED - we display all text types instead...
+    # $part_header =~ /^content\-disposition:\s*([\w+])/mis;
+    
+    # add <<attachment>> to each part except the first...
+    if (!$is_first_part) {
+	$return_text.="\n<p><hr><strong>&lt;&lt;attachment&gt;&gt;";
+	# add part info header
+	my $header_text = "<br>Type: $part_content_type<br>\n";
+	if ($filename ne "") {
+	    $header_text .= "Filename: $filename\n";
+	}
+	$header_text =~ s@_@\\_@g;
+	$return_text .= $header_text . "</strong></p>\n<p>\n";
+    }
+
+    if ($part_content_type =~ m@text/@)
+    {
+	# $message_part includes the mime part headers
+	my $part_text = $self->text_from_part($message_part);
+	if ($part_content_type !~ m@text/(ht|x)ml@) {
+	    $part_text = text_into_html($part_text);
+	}
+	if ($part_text eq "") {
+	    $part_text = ' ';
+	}
+	$return_text .= $part_text;
+    } elsif ($part_content_type =~ m@message/rfc822@) {
+	# This is a forwarded message
+	my $message_part_headers=$part_body;
+	$message_part_headers=~s/\r?\n\r?\n(.*)$//s;
+	my $message_part_body=$1;
+	$message_part_headers =~ s/\r?\n[\t\ ]+/ /gs; #unfold
+	
+	my $rfc822_formatted_body=""; # put result in here
+	if ($message_part_headers =~
+	    /^content\-type:\s*([\w\.\-\/]+)\s*(\;.*)?$/ims)
+	{
+	    # The message header uses MIME flags
+	    my $message_content_type=$1;
+	    my $message_content_info=$2;
+	    if (!defined($message_content_info)) {
+		$message_content_info="";
+	    } else {
+		$message_content_info =~ s/^\;\s*//;
+		$message_content_info =~ s/\s*$//;
+	    }
+	    $message_content_type =~ tr/A-Z/a-z/;
+	    if ($message_content_type =~ /multipart/) {
+		$rfc822_formatted_body=
+		    $self->text_from_mime_message($message_content_type,
+						  $message_content_info,
+						  $message_part_body);
+	    } else {
+		$message_part_body=$self->text_from_part($part_body,
+							$message_part_headers);
+		$rfc822_formatted_body=text_into_html($message_part_body);
+	    }
+	} else {
+	    # message doesn't use MIME flags
+	    $rfc822_formatted_body=text_into_html($message_part_body);
+	    $rfc822_formatted_body =~ s@_@\\_@g;
+	}
+	# Add the returned text to the output
+	# don't put all the headers...
+#	$message_part_headers =~ s/^(X\-.*|received|message\-id|return\-path):.*\n//img;
+	my $brief_headers=get_brief_headers($message_part_headers);
+	$return_text.=text_into_html($brief_headers);
+	$return_text.="</p><p>\n";
+	$return_text.=$rfc822_formatted_body;
+	$return_text.="</p>\n";
+	# end of message/rfc822
+    } elsif ($part_content_type =~ /multipart/) { 
+	# recurse again
+	
+	my $tmptext= $self->text_from_mime_message($part_content_type,
+						   $part_content_info,
+						   $part_body);
+	$return_text.=$tmptext;
+    } else {
+	# this part isn't text/* or another message...
+	if ($is_first_part) {
+	    # this is the first part of a multipart, or only part!
+	    $return_text="\n<p><hr><strong>&lt;&lt;attachment&gt;&gt;";
+	    # add part info header
+	    my $header_text="<br>Type: $part_content_type<br>\n";
+	    $header_text.="Filename: $filename</strong></p>\n<p>\n";
+	    $header_text =~ s@_@\\_@g;
+	    $return_text.=$header_text;
+	}
+	
+	# save attachment by default
+	if (!$self->{'no_attachments'}
+	    && $filename ne "") { # this part has a file...
+	    my $encoding="8bit";
+	    if ($part_header =~
+		/^content-transfer-encoding:\s*(\w+)/mi ) {
+		$encoding=$1; $encoding =~ tr/A-Z/a-z/;
+	    }
+	    my $tmpdir=$ENV{'GSDLHOME'} . "/tmp";
+	    my $save_filename=$filename;
+	    
+	    # make sure we don't clobber files with same name;
+	    # need to keep state between .mbx files
+	    my $assoc_files=$self->{'assoc_filenames'};
+	    if ($assoc_files->{$filename}) { # it's been set...
+		$assoc_files->{$filename}++;
+		$filename =~ m/(.+)\.(\w+)$/;
+		my ($filestem, $ext)=($1,$2);
+		$save_filename="${filestem}_"
+		    . $assoc_files->{$filename} . ".$ext";
+	    } else { # first file with this name
+		$assoc_files->{$filename}=1;
+	    }
+	    open (SAVE, ">$tmpdir/$save_filename") ||
+		warn "EMAILPlug: Can't save attachment as $tmpdir/$save_filename: $!";
+	    my $part_text = $message_part;
+	    $part_text =~ s/(.*?)\r?\n\r?\n//s; # remove header
+	    if ($encoding eq "base64") {
+		print SAVE base64_decode($part_text);
+	    } elsif ($encoding eq "quoted-printable") {
+		print SAVE qp_decode($part_text);
+	    } else { # 7bit, 8bit, binary, etc...
+		print SAVE $part_text;
+	    }
+	    close SAVE;
+	    my $doc_obj=$self->{'doc_obj'};
+	    $doc_obj->associate_file("$tmpdir/$save_filename",
+				     "$save_filename",
+				     $part_content_type # mimetype
+				     );
+	    # clean up tmp area...
+	    # Can't do this as it hasn't been copied/linked yet!!!
+#			&util::rm("$tmpdir/$save_filename");
+	    my $outhandle=$self->{'outhandle'};
+	    print $outhandle "EMAILPlug: saving attachment \"$filename\"\n"; #
+	    
+	    # be nice if "download" was a translatable macro :(
+	    $return_text .="<a href=\"_httpdocimg_/$save_filename\">download</a>";
+	} # end of save attachment
+    } # end of !text/message part
+
+
+    return $return_text;
+}
+
+
+# Return only the "important" headers from a set of message headers
+sub get_brief_headers {
+    my $msg_header = shift;
+    my $brief_header = "";
+
+    # Order matters!
+    if ($msg_header =~ /^(From:.*)$/im) {$brief_header.="$1\n";}
+    if ($msg_header =~ /^(To:.*)$/im) {$brief_header.="$1\n";}
+    if ($msg_header =~ /^(Cc:.*)$/im) {$brief_header.="$1\n";}
+    if ($msg_header =~ /^(Subject:.*)$/im) {$brief_header.="$1\n";}
+    if ($msg_header =~ /^(Date:.*)$/im) {$brief_header.="$1\n";}
+
+    return $brief_header;
+}
+
+
+# Process a MIME part. Return "" if we can't decode it.
+# should only be called for parts with type "text/*" ?
+# Either pass the entire mime part (including the part's header),
+# or pass the mime part's text and optionally the part's header.
+sub text_from_part {
+    my $self = shift;
+    my $text = shift || '';
+    my $part_header = shift;
+
+
+    my $type="text/plain"; # default, overridden from part header
+    my $charset=undef;     # convert2unicode() will guess if necessary
+
+    if (! $part_header) { # no header argument was given. check the body
+	$part_header = $text;
+	# check for empty part header (leading blank line)
+	if ($text =~ /^\s*\r?\n/) {
+	    $part_header="Content-type: text/plain; charset=us-ascii";
+	} else {
+	    $part_header =~ s/\r?\n\r?\n(.*)$//s;
+	    $text=$1; if (!defined($text)) {$text="";}
+	}
+	$part_header =~ s/\r?\n[\t ]+/ /gs; #unfold
+    }
+
+    if ($part_header =~
+	/content\-type:\s*([\w\.\-\/]+).*?charset=\"?([^\;\"\s]+)\"?/is) {
+	$type=$1;
+	$charset=$2;
+    }
+    my $encoding="";
+    if ($part_header =~ /^content\-transfer\-encoding:\s*([^\s]+)/mis) {
+	$encoding=$1; $encoding=~tr/A-Z/a-z/;
+    }
+    # Content-Transfer-Encoding is per-part
+    if ($encoding ne "") {
+	if ($encoding =~ /quoted\-printable/) {
+	    $text=qp_decode($text);
+	} elsif ($encoding =~ /base64/) {
+	    $text=base64_decode($text);
+	} elsif ($encoding !~ /[78]bit/) { # leave 7/8 bit as is.
+	    # rfc2045 also allows binary, which we ignore (for now).
+	    my $outhandle=$self->{'outhandle'};
+	    print $outhandle "EMAILPlug: unknown transfer encoding: $encoding\n";
+	    return "";
+	}
+    }
+
+    if ($type eq "text/html") {
+	# only get stuff between <body> tags, or <html> tags.
+	$text =~ s@^.*<html[^>]*>@@is;
+	$text =~ s@</html>.*$@@is;
+	$text =~ s/^.*?<body[^>]*>//si;
+	$text =~ s/<\/body>.*$//si;
+    }
+    elsif ($type eq "text/xml") {
+	$text=~s/</&lt;/g;$text=~s/>/&gt;/g;
+	$text="<pre>\n$text\n</pre>\n";
+    }
+    # convert to unicode
+    $self->convert2unicode($charset, \$text);
+    $text =~ s@_@\\_@g; # protect against GS macro language
+    return $text;
+}
+
+
+
+
+# decode quoted-printable text
+sub qp_decode {
+    my $text=shift;
+
+    # if a line ends with "=\s*", it is a soft line break, otherwise
+    # keep in any newline characters.
+
+    $text =~ s/=\s*\r?\n//mg;
+    $text =~ s/=([0-9A-Fa-f]{2})/chr (hex "0x$1")/eg;
+    return $text;
+}
+
+# decode base64 text. This is fairly slow (since it's interpreted perl rather
+# than compiled XS stuff like in the ::MIME modules, but this is more portable
+# for us at least).
+# see rfc2045 for description, but basically, bits 7 and 8 are set to zero;
+# 4 bytes of encoded text become 3 bytes of binary - remove 2 highest bits
+# from each encoded byte.
+
+
+sub base64_decode {
+    my $enc_text = shift;
+# A=>0, B=>1, ..., '+'=>62, '/'=>63
+# also '=' is used for padding at the end, but we remove it anyway.
+    my $mimechars="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+# map each MIME char into it's value, for more efficient lookup.
+    my %index;
+    map { $index{$_} = index ($mimechars, $_) } (split ('', $mimechars));
+# remove all non-base64 chars. eval to get variable in transliteration...
+# also remove '=' - we'll assume (!!) that there are no errors in the encoding
+    eval "\$enc_text =~ tr|$mimechars||cd";
+    my $decoded="";
+    while (length ($enc_text)>3)
+    { 
+	my $fourchars=substr($enc_text,0,4,"");
+	my @chars=(split '',$fourchars);
+	$decoded.=chr( $index{$chars[0]}        << 2 | $index{$chars[1]} >> 4);
+	$decoded.=chr( ($index{$chars[1]} & 15) << 4 | $index{$chars[2]} >> 2);
+	$decoded.=chr( ($index{$chars[2]} & 3 ) << 6 |  $index{$chars[3]});
+    } 
+# if there are any input chars left, there are either
+# 2 encoded bytes (-> 1 raw byte) left or 3 encoded (-> 2 raw) bytes left.
+    my @chars=(split '',$enc_text);
+    if (length($enc_text)) {
+	$decoded.=chr($index{$chars[0]} << 2 | (int $index{$chars[1]} >> 4));
+    } 
+    if (length($enc_text)==3) {
+	$decoded.=chr( ($index{$chars[1]} & 15) << 4 | $index{$chars[2]} >> 2);
+    }
+    return $decoded;
+}
+
+# returns 0 if valid utf-8, 1 if invalid
+sub is_utf8 {
+    my $self = shift;
+    my $textref = shift;
+
+    $$textref =~ m/^/g; # to set \G
+    my $badbytesfound=0;
+    while ($$textref =~ m!\G.*?([\x80-\xff]+)!sg) {
+	my $highbytes=$1;
+	my $highbyteslength=length($highbytes);
+	# replace any non utf8 complaint bytes
+	$highbytes =~ /^/g; # set pos()
+	while ($highbytes =~
+	       m!\G (?: [\xc0-\xdf][\x80-\xbf] | # 2 byte utf-8
+		     [\xe0-\xef][\x80-\xbf]{2} | # 3 byte
+		     [\xf0-\xf7][\x80-\xbf]{3} | # 4 byte
+		     [\xf8-\xfb][\x80-\xbf]{4} | # 5 byte
+		     [\xfc-\xfd][\x80-\xbf]{5}   # 6 byte
+		     )*([\x80-\xff])? !xg
+	       ) {
+	    my $badbyte=$1;
+	    if (!defined $badbyte) {next} # hit end of string
+	    return 1;
+	}
+    }
+    return 0;
+}
+
+
+
+sub convert2unicode {
+  my $self = shift(@_);
+  my ($charset, $textref) = @_;
+
+  if (!$$textref) {
+      # nothing to do!
+      return;
+  }
+
+  if (! defined $charset) {
+      # check if we have valid utf-8
+      if ($self->is_utf8($textref)) { $charset = "utf8" }
+
+      # default to latin
+      $charset = "iso_8859_1" if ! defined($charset);
+  }
+
+  # first get our character encoding name in the right form.
+  $charset =~ tr/A-Z/a-z/; # lowercase
+  $charset =~ s/\-/_/g;
+  if ($charset =~ /gb_?2312/) { $charset="gb" }
+  # assumes EUC-KR, not ISO-2022 !?
+  $charset =~ s/^ks_c_5601_1987/korean/;
+  if ($charset eq 'utf_8') {$charset='utf8'}
+
+  my $outhandle = $self->{'outhandle'};
+
+  if ($charset eq "utf8") {
+      # no conversion needed, but lets check that it's valid utf8
+      # see utf-8 manpage for valid ranges
+      $$textref =~ m/^/g; # to set \G
+      my $badbytesfound=0;
+      while ($$textref =~ m!\G.*?([\x80-\xff]+)!sg) {
+	  my $highbytes=$1;
+	  my $highbyteslength=length($highbytes);
+	  # replace any non utf8 complaint bytes
+	  $highbytes =~ /^/g; # set pos()
+	  while ($highbytes =~
+		 m!\G (?: [\xc0-\xdf][\x80-\xbf] | # 2 byte utf-8
+		       [\xe0-\xef][\x80-\xbf]{2} | # 3 byte
+		       [\xf0-\xf7][\x80-\xbf]{3} | # 4 byte
+		       [\xf8-\xfb][\x80-\xbf]{4} | # 5 byte
+		       [\xfc-\xfd][\x80-\xbf]{5}   # 6 byte
+		       )*([\x80-\xff])? !xg
+		 ) {
+	      my $badbyte=$1;
+	      if (!defined $badbyte) {next} # hit end of string
+	      my $pos=pos($highbytes);
+	      substr($highbytes, $pos-1, 1, "\xc2\x80");
+	      # update the position to continue searching (for \G)
+	      pos($highbytes) = $pos+1; # set to just after the \x80
+	      $badbytesfound=1;
+	  }
+	  if ($badbytesfound==1) {
+	      # claims to be utf8, but it isn't!
+	      print $outhandle "EMAILPlug: Headers claim utf-8 but bad bytes "
+		  . "detected and removed.\n";
+
+	      my $replength=length($highbytes);
+	      my $textpos=pos($$textref);
+	      # replace bad bytes with good bytes
+	      substr( $$textref, $textpos-$replength,
+		      $replength, $highbytes);
+	      # update the position to continue searching (for \G)
+	      pos($$textref)=$textpos+($replength-$highbyteslength);
+	  }
+      }
+      return;
+  }
+
+  # It appears that we can't always trust ascii text so we'll treat it
+  # as iso-8859-1 (letting characters above 0x80 through without
+  # converting them to utf-8 will result in invalid XML documents
+  # which can't be parsed at build time).
+  $charset = "iso_8859_1" if ($charset eq "us_ascii" || $charset eq "ascii");
+
+  if ($charset eq "iso_8859_1") {
+      # test if the mailer lied, and it has win1252 chars in it...
+      # 1252 has characters between 0x80 and 0x9f, 8859-1 doesn't
+      if ($$textref =~ m/[\x80-\x9f]/) {
+	  print $outhandle "EMAILPlug: Headers claim ISO charset but MS ";
+	  print $outhandle "codepage 1252 detected.\n";
+	  $charset = "windows_1252";
+      }
+  }
+  my $utf8_text=&unicode::unicode2utf8(&unicode::convert2unicode($charset,$textref));
+
+  if ($utf8_text ne "") {
+      $$textref=$utf8_text;
+  } else {
+      # we didn't get any text... unsupported encoding perhaps? Or it is
+      # empty anyway. We'll try to continue, assuming 8859-1. We could strip
+      # characters out here if this causes problems...
+      my $outhandle=$self->{'outhandle'};
+      print $outhandle "EMAILPlug: falling back to iso-8859-1\n";
+      $$textref=&unicode::unicode2utf8(&unicode::convert2unicode("iso_8859_1",$textref));
+
+  }
+}
+
+
+sub set_OID {
+    my $self = shift (@_);
+    my ($doc_obj, $id, $segment_number) = @_;
+    
+    if ( exists $doc_obj->{'msgid'} ) {
+	$doc_obj->set_OID($doc_obj->{'msgid'});
+    } else {
+	$doc_obj->set_OID("$id\_$segment_number");
+    }
+}
+
+
+# Perl packages have to return true if they are run.
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ExcelPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ExcelPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ExcelPlug.pm	(revision 21112)
@@ -0,0 +1,115 @@
+###########################################################################
+#
+# ExcelPlug.pm -- plugin for importing Microsoft Excel files.
+#  (currently only versions 95 and 97)
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2002 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package ExcelPlug;
+
+use ConvertToPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @ExcelPlug::ISA = ('ConvertToPlug');
+}
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() }
+      ];
+
+my $options = { 'name'     => "ExcelPlug",
+		'desc'     => "{ExcelPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+    if (!defined $secondary_plugin_options->{'HTMLPlug'}) {
+	$secondary_plugin_options->{'HTMLPlug'} = [];
+    }
+    my $html_options = $secondary_plugin_options->{'HTMLPlug'};
+    
+    #$self->{'input_encoding'} = "utf8";
+    #$self->{'extract_language'} = 1;
+    push(@$html_options, "-input_encoding", "utf8");
+    push(@$html_options,"-extract_language");
+    $self = bless $self, $class;
+    
+    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
+    return bless $self;    
+}
+
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+    
+    my $outhandle=$self->{'outhandle'};
+    
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+    
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($conv_filename, $encoding, $language, \$text);
+    
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+    #unicode::ensure_utf8(\$text);
+    
+    # Write it out again!
+    #$self->utf8_write_file (\$text, $conv_filename);
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    return q^(?i)\.xls$^;
+}
+    
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    return $self->process_type("xls",$base_dir,$file,$doc_obj);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/FOXPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/FOXPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/FOXPlug.pm	(revision 21112)
@@ -0,0 +1,295 @@
+###########################################################################
+#
+# FOXPlug.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin to process a Foxbase dbt file. This plugin provides the basic
+# functionality to read in the dbt and dbf files and process each record.
+# This general plugin should be overridden for a particular database to process
+# the appropriate fields in the file.
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package FOXPlug;
+
+use BasPlug;
+use util;
+use doc;
+use unicode;
+use cnseg;
+# use gb;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+
+sub BEGIN {
+    @FOXPlug::ISA = ('BasPlug');
+}
+
+my $arguments = [
+		 ];
+
+my $options = { 'name'     => "FOXPlug",
+		'desc'     => "{FOXPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $fullname = &util::filename_cat ($base_dir, $file);
+
+    # dbt files are processed at the same time as dbf files
+    return 0 if ($fullname =~ /\.dbt$/i);
+
+    # see if this is a foxbase database
+    return undef unless (-f $fullname && $fullname =~ /\.dbf$/i);
+
+    my ($parent_dir) = $fullname =~ /^(.*)\/[^\/]+\.dbf$/i;
+
+    # open the file
+    if (!open (FOXBASEIN, $fullname)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read $fullname'>\n";
+	}
+	print STDERR "FOXPlug::read - couldn't read $fullname\n";
+	return -1; # error in processing
+    }
+
+    print STDERR "<Processing n='$file' p='FOXPlug'>\n" if ($gli);
+    
+    print STDERR "FOXPlug: processing $file\n";
+
+    # read in the database header
+    my ($temp, %dbf);
+    
+    # read in information about dbt file
+    if (read (FOXBASEIN, $temp, 32) < 32) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='EOF while reading database header'>\n";
+	}
+	print STDERR "FOXPlug::read - eof while reading database header\n";
+	close (FOXBASEIN);
+	return -1;
+    }
+    
+    # unpack the header
+    ($dbf{'hasdbt'},
+     $dbf{'modyear'}, $dbf{'modmonth'}, $dbf{'modday'},
+     $dbf{'numrecords'}, $dbf{'headerlen'}, 
+     $dbf{'recordlen'}) = unpack ("CCCCVvv", $temp);
+    
+    # process hasdbt
+    if ($dbf{'hasdbt'} == 131) {
+	$dbf{'hasdbt'} = 1;
+    } elsif ($dbf{'hasdbt'} == 3 || $dbf{'hasdbt'} == 48) {
+	$dbf{'hasdbt'} = 0;
+    } else {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Does not seem to be a Foxbase file'>\n";
+	}
+	print STDERR "FOXPlug:read - $fullname doesn't seem to be a Foxbase file\n";
+	return -1;
+    }
+
+    # read in the field description
+    $dbf{'numfields'} = 0;
+    $dbf{'fieldinfo'} = [];
+    while (read (FOXBASEIN, $temp, 1) > 0) {
+	last if ($temp eq "\x0d");
+	last if (read (FOXBASEIN, $temp, 31, 1) < 31);
+
+	my %field = ();
+	$field{'name'} = $self->extracttext($temp, 11);
+	($field{'type'}, $field{'pos'}, $field{'len'}, $field{'dp'}) 
+	    = unpack ("x11a1VCC", $temp);
+
+	push (@{$dbf{'fieldinfo'}}, \%field);
+
+	$dbf{'numfields'} ++;
+    }
+
+    # open the dbt file if we need to
+    my $dbtfullname = $fullname;
+    if ($fullname =~ /f$/) {
+	$dbtfullname =~ s/f$/t/;
+    } else {
+	$dbtfullname =~ s/F$/T/;
+    }
+    if ($dbf{'hasdbt'} && !open (DBTIN, $dbtfullname)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read $dbtfullname'>\n";
+	}
+	print STDERR "FOXPlug::read - couldn't read $dbtfullname\n";
+	close (FOXBASEIN);
+	return -1;
+    }
+
+    # read in and process each record in the database
+    my $numrecords = 0;
+    while (($numrecords < $dbf{'numrecords'}) && 
+	   (read (FOXBASEIN, $temp, $dbf{'recordlen'}) == $dbf{'recordlen'})) {
+
+	# create a new record
+	my $record = [];
+	
+	foreach my $field (@{$dbf{'fieldinfo'}}) {
+	    my $fieldvalue = "";
+	    
+	    if ($field->{'type'} eq "M" && $dbf{'hasdbt'}) {
+		# a memo field, look up this field in the dbt file
+		my $seekpos = substr ($temp, $field->{'pos'}, $field->{'len'});
+
+		$seekpos =~ s/^\s*//;
+		$seekpos = 0 unless $seekpos =~ /^\d+$/;
+		
+		$seekpos = $seekpos * 512;
+
+		if ($seekpos == 0) {
+		    # there is no memo field
+
+		} elsif (seek (DBTIN, $seekpos, 0)) {
+		    while (read (DBTIN, $fieldvalue, 512, length($fieldvalue)) > 0) {
+			last if ($fieldvalue =~ /\cZ/);
+		    }
+
+		    # remove everything after the control-Z
+		    substr($fieldvalue, index($fieldvalue, "\cZ")) = "";
+
+		} else {
+		    print STDERR "\nERROR - seek (to $seekpos) failed\n";
+		}
+
+	    } else {
+		# a normal field
+		$fieldvalue = substr ($temp, $field->{'pos'}, $field->{'len'});
+	    }
+
+	    push (@$record, {%$field, 'value'=>$fieldvalue});
+	}
+
+	# process this record
+	$self->process_record ($pluginfo, $base_dir, $file, $metadata, $processor, 
+			       $numrecords, $record);
+	
+	# finished another record...
+	$numrecords++;
+    }
+
+    # close the dbt file if we need to
+    if ($dbf{'hasdbt'}) {
+	close (DBTIN);
+    }
+
+    # close the dbf file
+    close (FOXBASEIN);
+
+    # finished processing
+    return 1;
+}
+
+
+# will extract a string from some larger string, making it
+# conform to a number of constraints
+sub extracttext {
+    my $self = shift (@_);
+    my ($text, $maxlen, $offset, $stopstr) = @_;
+    $offset = 0 unless defined $offset;
+    $stopstr = "\x00" unless defined $stopstr;
+    
+    # decide where the string finishes
+    my $end = index ($text, $stopstr, $offset);
+    $end = length ($text) if $end < 0;
+    $end = $offset+$maxlen if (defined $maxlen) && ($end-$offset > $maxlen);
+    
+    return "" if ($end <= $offset);
+    return substr ($text, $offset, $end-$offset);
+}
+
+
+# process_record should be overriden for a particular type
+# of database. This default version outputs an html document
+# containing all the fields in the record as a table.
+# It also assumes that the text is in utf-8.
+sub process_record {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $numrecords, $record) = @_;
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    my $section = $doc_obj->get_top_section();
+
+    $doc_obj->add_metadata($section, "FileFormat", "FOX");
+    $doc_obj->add_metadata($section, "FileSize",   (-s $file));
+
+    # start of document
+    $doc_obj->add_utf8_text($section, "<table>\n");
+
+    # add each field
+    foreach my $field (@$record) {
+	if (defined ($field->{'name'}) && defined ($field->{'value'})) {
+	    $doc_obj->add_utf8_text($section, "  <tr>\n");
+	    $doc_obj->add_utf8_text($section, "    <td>$field->{'name'}</td>\n");
+	    $doc_obj->add_utf8_text($section, "    <td>$field->{'value'}</td>\n");
+	    $doc_obj->add_utf8_text($section, "  </tr>\n");
+	}
+    }
+
+    # end of document
+    $doc_obj->add_utf8_text($section, "</table>\n");
+
+    # add an object id
+    $doc_obj->set_OID();
+
+    # process the document
+    $processor->process($doc_obj);
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/FavouritesPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/FavouritesPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/FavouritesPlug.pm	(revision 21112)
@@ -0,0 +1,113 @@
+###########################################################################
+#
+# FavouritesPlug.pm -- Plugin for Internet Explorer Favourites files
+#                      By Stephen De Gabrielle
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# hacked together by Stephen De Gabrielle from various plugins 
+# especially SRCPlug by John McPherson Nov 2000
+
+package FavouritesPlug;
+
+use BasPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @FavouritesPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "FavouritesPlug",
+		'desc'     => "FavouritesPlug imports Internet Explorer style Favourites. Favourites are often found in the \"C:\\Documents and Settings\\[your username]\\Favorites\" folder on your computer, but can also be made by dragging a bookmark or location from your browser (any) to the desktop.",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift(@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+
+sub get_default_process_exp
+{
+    # URL is extension for single bookmarks under windows.
+    return q^(?i)\.URL$^;
+}
+
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $section = $doc_obj->get_top_section();
+    print STDERR "<Processing n='$file' p='FavouritesPlug'>\n" if ($gli);
+    print $outhandle "FavouritesPlug: processing $file\n" if $self->{'verbosity'} > 1;
+
+    # don't want mg to turn escape chars into actual values
+    $$textref =~ s/\\/\\\\/g;
+
+    # use filename (minus the .url extension) as the title
+    my $title = $file;
+    $title =~ s/.url$//i;
+    $doc_obj->add_utf8_metadata($section, "Title", $title);
+
+    # get the URL from the file
+    my ($url) = ($$textref =~ m/^URL=(http.+)/mg);
+    $doc_obj->add_metadata($section, "URL", $url);
+
+    # Add srclink metadata for an automatic link to the webpage
+    $doc_obj->add_utf8_metadata($section, "srclink", "<a href=\"$url\">");
+    $doc_obj->add_utf8_metadata($section, "srcicon", "_iconworld_");
+    $doc_obj->add_utf8_metadata($section, "/srclink", "</a>");
+
+    # Tidy up the favourite text to look a bit nicer
+    $$textref =~ s/^\\n/<p>/g;
+    $$textref =~ s/\[/<p><strong>/g;
+    $$textref =~ s/\]/<\/strong><p>/g;
+    $$textref =~ s/^Modified=(.+)$/<strong>Modified<\/strong>$1<p>/g;
+    $doc_obj->add_utf8_text($section, "$$textref");
+
+    $doc_obj->add_metadata($section, "FileFormat", "Favourite");
+    return 1;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/GAPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/GAPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/GAPlug.pm	(revision 21112)
@@ -0,0 +1,197 @@
+###########################################################################
+#
+# GAPlug.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Processes GreenstoneArchive XML documents. Note that this plugin does no
+# syntax checking (though the XML::Parser module tests for
+# well-formedness). It's assumed that the GreenstoneArchive files conform
+# to their DTD.
+
+package GAPlug;
+
+use XMLPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @GAPlug::ISA = ('XMLPlug');
+}
+
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)doc\.xml$^;
+}
+
+my $arguments = [
+		 ];
+
+my $options = { 'name'     => "GAPlug",
+		'desc'     => "{GAPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new XMLPlug($pluginlist,$inputargs,$hashArgOptLists): new XMLPlug($pluginlist,$inputargs);
+
+    $self->{'section'} = "";
+    $self->{'section_level'} = 0;
+    $self->{'metadata_name'} = "";
+    $self->{'metadata_value'} = "";
+    $self->{'content'} = "";
+   
+    return bless $self, $class;
+}
+
+sub xml_start_document {
+}
+
+sub xml_end_document {
+}
+
+sub xml_doctype {
+    my $self = shift(@_);
+
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+
+    # allow the short-lived and badly named "GreenstoneArchive" files to be processed
+    # as well as the "Archive" files which should now be created by import.pl
+    die "" if ($name !~ /^(Greenstone)?Archive$/);
+
+    my $outhandle = $self->{'outhandle'};
+    print $outhandle "GAPlug: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
+    print STDERR "<Processing n='$self->{'file'}' p='GAPlug'>\n" if $self->{'gli'};
+
+}
+
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    $self->{'element'} = $element;
+    if ($element eq "Section") {
+	if ($self->{'section_level'} == 0) {
+	    $self->open_document();
+       	} else {
+	    my $doc_obj = $self->{'doc_obj'};
+	    $self->{'section'} = 
+		$doc_obj->insert_section($doc_obj->get_end_child($self->{'section'}));
+	}
+	
+	$self->{'section_level'} ++;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'metadata_name'} = $_{'name'};
+    }
+}
+
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    if ($element eq "Section") {
+	$self->{'section_level'} --;
+	$self->{'section'} = $self->{'doc_obj'}->get_parent_section ($self->{'section'});
+	$self->close_document() if $self->{'section_level'} == 0;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, $self->{'metadata_name'},$self->{'metadata_value'});
+	$self->{'metadata_name'} = "";
+	$self->{'metadata_value'} = "";
+    }
+    elsif ($element eq "Content" && $self->{'content'} ne "") {
+	$self->{'doc_obj'}->add_utf8_text($self->{'section'}, $self->{'content'});
+	$self->{'content'} = "";
+    }
+    
+    $self->{'element'} = "";
+}
+
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    if ($self->{'element'} eq "Metadata") {
+	$self->{'metadata_value'} .= $_;
+    }
+    elsif ($self->{'element'} eq "Content") {
+	$self->{'content'} .= $_;
+    }
+}
+
+sub open_document {
+    my $self = shift(@_);
+
+    # create a new document
+    $self->{'doc_obj'} = new doc ();
+    $self->{'section'} = "";
+}
+
+sub close_document {
+    my $self = shift(@_);
+    
+    # add the associated files
+    my $assoc_files = 
+	$self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    # for when "assocfilepath" isn't the same directory that doc.xml is in...
+    my $assoc_filepath_list= $self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "assocfilepath");
+
+    my $assoc_filepath=shift (@$assoc_filepath_list);
+    if (defined ($assoc_filepath)) {
+	# make absolute rather than relative...
+	$self->{'filename'} =~ m@^(.*[\\/]archives)@;
+	$assoc_filepath = "$1/$assoc_filepath/";
+    } else {
+	$assoc_filepath = $self->{'filename'};
+	$assoc_filepath =~ s/[^\\\/]*$//;
+    }
+
+    foreach my $assoc_file_info (@$assoc_files) {
+	my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
+	my $real_dir = &util::filename_cat($assoc_filepath, $assoc_file),
+	my $assoc_dir = (defined $dir && $dir ne "") 
+	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
+	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
+    }
+    $self->{'doc_obj'}->delete_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    # process the document
+    $self->{'processor'}->process($self->{'doc_obj'}, $self->{'file'});
+}
+
+
+1;
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/GISBasPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/GISBasPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/GISBasPlug.pm	(revision 21112)
@@ -0,0 +1,300 @@
+###########################################################################
+#
+# GISBasPlug.pm -- base class to enhance plugins with GIS capabilities
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package GISBasPlug;
+
+use util;
+use locale;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+#field categories in DataBase files
+#$LAT = 3;
+#$LONG = 4;
+my $FC = 9;
+my $DSG = 10;
+#$CC1 = 12;
+my $FULL_NAME = 22;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+}
+
+
+use BasPlug; # uses BasPlug, but is not inherited
+
+
+my $options = { 'name'     => "GISBasPlug",
+		'desc'     => "{GISBasPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "no" };
+
+
+sub new {
+    my $class = shift (@_);
+    my $plugin_name = shift (@_);
+
+    my $self = {};
+    $self->{'plugin_type'} = "GISBasPlug";
+
+    $self->{'option_list'} = [ $options ];
+
+    return bless $self, $class;
+}
+
+sub init {
+}
+
+sub print_xml_usage
+{
+    BasPlug::print_xml_usage(@_);
+}
+
+sub print_xml
+{
+    BasPlug::print_xml(@_);
+}
+
+sub print_txt_usage
+{
+   BasPlug::print_txt_usage(@_);
+}
+
+sub set_keepold
+{
+    BasPlug::set_keepold(@_);
+}
+
+sub has_mapdata
+{
+    my $db_dir = &util::filename_cat($ENV{'GSDLHOME'}, "lamp", "data");
+    return ( -d $db_dir );
+}
+
+
+#returns a hash table of names from database files (specified in collect.cfg). 
+sub loadGISDatabase {
+    my ($outhandle,$datasets) = @_;
+    my @dbase = map{$_ = $_ . ".txt";} split(/,/, $datasets);
+    if(scalar(@dbase)==0) { #default is to include all databases
+	@dbase=("UK.txt", "NZ.txt", "NF.txt", "CA.txt", "AS.txt", "GM.txt", "BE.txt", "IN.txt", "JA.txt", "USA.txt");
+    }
+    my $counter=0;
+    my %places = ();
+    my @cats = ();	
+    while($counter <= $#dbase){ #loop through all the databases
+	my $folder = $dbase[$counter];
+	$folder =~ s/(.*?)\.txt/$1/;
+####		my $dbName = &util::filename_cat($ENV{'GSDLHOME'}, "etc", "mapdata", "data", $folder, $dbase[$counter]);
+	my $dbName = &util::filename_cat($ENV{'GSDLHOME'}, "lamp", "data", $folder, $dbase[$counter]);
+	if (!open(FILEIN, "<$dbName")) {
+	    print $outhandle "Unable to open database $dbName: $!\n";
+	    return undef;
+	}
+	
+	my $line = <FILEIN>; #database category details.
+	my @catdetails = split("\t", $line);
+	
+	while ( defined($line = <FILEIN>)){ #stores all place names in file as keys in hash array %places
+	    @cats = split("\t", $line);
+	    if( #eliminating "bad" place names without missing real ones
+		($cats[$FC] eq "A" && !($cats[$DSG] =~ /ADMD|ADM2|PRSH/))
+		||($cats[$FC] eq "P" && ($cats[$DSG] =~ /PPLA|PPLC|PPLS/)) 
+		||($cats[$FC] eq "H" && ($cats[$DSG] =~ /BAY|LK/)) 
+		||($cats[$FC] eq "L" && !($cats[$DSG] =~ /LCTY|RGN/)) 
+		||($cats[$FC] eq "T" && ($cats[$DSG] =~ /CAPE|ISL|GRGE/))
+		||($dbase[$counter] eq "USA.txt" && ($cats[$DSG] =~ /ppl|island/))
+		){$places{$cats[$FULL_NAME]} = [@cats];}
+			@cats = ();	
+	}	
+	close(FILEIN);
+	$counter++;	
+    }	
+    return \%places;
+}
+
+#returns a unique hash array of all the places found in a document, along with coordinates, description and country data
+sub getPlaces {
+    my $self = shift @_;
+
+    my ($textref, $places) = @_;
+	my %tempPlaces = ();	
+	
+	foreach my $plc (%$places){ #search for an occurrence of each place in the text
+		if($$textref =~ m/(\W)$plc(\W)/){
+			$tempPlaces{$plc} = $places->{$plc};
+		}
+	}
+	#make sure each place is only there once
+	my %uniquePlaces = ();
+	foreach my $p (keys %tempPlaces) {	
+		if(!defined($uniquePlaces{$p})){
+			$uniquePlaces{$p} = $tempPlaces{$p};
+		}
+	}
+	return \%uniquePlaces;
+}
+
+
+
+
+#returns a lowercase version of the place, with no spaces
+sub placename_to_anchorname {
+	my ($placename) = @_;
+	my $p_tag = lc($placename);
+	$p_tag =~ s/\s+//g;
+	return $p_tag;
+}
+
+#takes a place from the text and wraps an anchor tag, a hyperlink tag and an image tag around it
+sub anchor_wrapper {
+	my ($p, $p_counter_ref, $path) = @_;
+	my $image = "/gsdlgis/gisimages/nextplace.gif";
+	my $endTag = "</a>";
+	my $hrefTag = "<a href=";
+	$$p_counter_ref++;
+	my $next = $$p_counter_ref + 1;
+	my $p_tag = placename_to_anchorname($p);
+	my $place_anchor = "<a name=\"" . $p_tag . $$p_counter_ref . "\">" . $endTag; 
+	my $popup_anchor = $hrefTag . "'" . $path . "'>" . $p . $endTag;
+	my $image_anchor = $hrefTag . "#" . $p_tag . $next . "><img src=\"" . $image . "\" name=\"img" . $p_tag . $$p_counter_ref . "\" border=\"0\">" . $endTag;
+	return $place_anchor . $popup_anchor . $image_anchor;
+}
+
+#takes dangerous place names and checks if they are part of another placename or not.
+sub place_name_check {
+	my ($pre, $preSpace, $p, $postSpace, $post, $p_counter_ref, $path, $y) = @_;
+	if($pre =~ /$y/ || $post =~ /$y/) {return $pre . $preSpace . $p . $postSpace . $post;}
+	$pre = $pre . $preSpace;
+	$post = $postSpace . $post;
+	return $pre . &anchor_wrapper("", $p, "", $p_counter_ref, $path) . $post;
+}
+
+sub extract_placenames {
+    my $self = shift (@_);
+    my ($textref, $doc_obj, $thissection) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $GSDLHOME = $ENV{'GSDLHOME'};
+    #field categories in DataBase file for extract_placenames.
+    my $LAT = 3;
+    my $LONG = 4;
+    my $CC1 = 12;
+
+    
+    &gsprintf($outhandle, " {BasPlug.extracting_placenames}...\n")
+	if ($self->{'verbosity'} > 2);
+
+    #get all the places found in the document	
+    my $uniquePlaces = $self->getPlaces($textref, $self->{'places'});
+       
+    #finds 'dangerous' placenames (eg York and New York). Dangerous because program will find "York" within "New York"  
+    my %danger = ();    
+    foreach my $x (keys %$uniquePlaces){
+	foreach my $y (keys %$uniquePlaces){
+		if(($y =~ m/ /) && ($y =~ m/$x/) && ($y ne $x)){
+			$y =~ s/($x\s)|(\s$x)//;
+			$danger{$x} = $y;
+		}
+	}
+    }  
+      
+    #creates a list of clickable placenames at top of page, linked to first occurrence of name, and reads them into a file 
+    my $tempfname = $doc_obj;
+    $tempfname =~ s/.*\(0x(.*)\)/$1/;
+    my $names = "";
+    my $filename = "tmpfile" . $tempfname;
+    my $tempfile = &util::filename_cat($GSDLHOME, "tmp", $filename);
+    open(FOUT, ">$tempfile") || die "Unable to create a temp file: $!";	
+    foreach my $name (sort (keys %$uniquePlaces)){
+	if(!defined($danger{$name})){
+		my $name_tag = placename_to_anchorname($name);
+		print FOUT "$name\t" . $uniquePlaces->{$name}->[$LONG] . "\t" . $uniquePlaces->{$name}->[$LAT] . "\n";
+		if($self->{'place_list'}) {$names = $names . "<a href=\"#" . $name_tag . "1\">" . $name . "</a>" . "\n";}
+	}
+    }
+    close(FOUT);
+    $doc_obj->associate_file($tempfile, "places.txt", "text/plain");
+    $self->{'places_filename'} = $tempfile;
+    my %countries = ();
+    
+    foreach my $p (keys %$uniquePlaces){
+	my $place = $p;
+	$place =~ s/\s+|\n+|\r+|\t+/(\\s+)/g;
+	my $cap_place = uc($place);	
+	my $long = $uniquePlaces->{$p}->[$LONG];
+	my $lat = $uniquePlaces->{$p}->[$LAT];
+	my $country = $uniquePlaces->{$p}->[$CC1];
+	my $path = "javascript:popUp(\"$long\",\"$lat\",\"$p\",\"$country\")";
+	my $p_counter = 0;
+	
+	if(!defined($danger{$p})){	
+		#adds html tags to each place name
+		$$textref =~ s/\b($place|$cap_place)\b/&anchor_wrapper($1,\$p_counter,$path)/sge;
+	} 
+	#else {
+	#$y = $danger{$p};
+	#$$textref =~ s/(\w+)(\s+?)($place|$cap_place)(\s+?)(\w+)/&place_name_check($1,$2,$3,$4,$5,\$p_counter,$path, $y)/sge;  
+	#}
+	
+	#edits the last place's image, and removes image if place only occurres once.
+	my $p_tag = placename_to_anchorname($p);
+	$p_counter++;
+	$$textref =~ s/#$p_tag$p_counter(><img src="\/gsdl\/images\/)nextplace.gif/#${p_tag}1$1firstplace.gif/;
+	$$textref =~ s/<img src="\/gsdl\/images\/firstplace.gif" name="img$p_tag(1)" border="0">//;
+	
+	#this line removes apostrophes from placenames (they break the javascript function)
+	$$textref =~ s/(javascript:popUp.*?)(\w)'(\w)/$1$2$3/g;
+		
+	#for displaying map of document, count num of places from each country
+	if(defined($countries{$country})){$countries{$country}++;}
+	else{$countries{$country} = 1;}
+
+	#adds placename to metadata
+	$doc_obj->add_utf8_metadata ($thissection, "Placename", $p);	
+	&gsprintf($outhandle, "  {BasPlug.extracting} $p\n")
+		if ($self->{'verbosity'} > 3); 
+    }
+    #finding the country that most places are from, in order to display map of the document
+    my $max = 0;
+    my $CNTRY = "";
+    foreach my $c_key (keys %countries){
+	if($countries{$c_key} > $max){
+		$max = $countries{$c_key};
+		$CNTRY = $c_key;
+	}
+    }
+    #allows user to view map with all places from the document on it
+####    my $places_filename = &util::filename_cat($GSDLHOME, "collect", "_cgiargc_", "index", "assoc", "_thisOID_", "places.txt");
+    my $places_filename = &util::filename_cat("collect", "_cgiargc_", "index", "assoc", "_thisOID_", "places.txt");
+    my $docmap = "<a href='javascript:popUp(\"$CNTRY\",\"$places_filename\")'>View map for this document<\/a><br><br>\n";
+    $$textref = $docmap . $names . "<br>" . $$textref;
+    
+    $doc_obj->delete_text($thissection);
+    $doc_obj->add_utf8_text($thissection, $$textref);
+    &gsprintf($outhandle, " {BasPlug.done_places_extract}\n")
+	if ($self->{'verbosity'} > 2);
+}
Index: /main/tags/3.01/gsdl/perllib/plugins/GMLPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/GMLPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/GMLPlug.pm	(revision 21112)
@@ -0,0 +1,254 @@
+###########################################################################
+#
+# GMLPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which processes a GML format document
+# assumes that gml tags are all in lower-case.
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package GMLPlug;
+
+use BasPlug;
+use util;
+use doc;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @GMLPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' =>  &get_default_process_exp() } 
+    ];
+
+my $options = { 'name'     => "GMLPlug",
+		'desc'     => "{GMLPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args' => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.gml?$^;
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    print STDERR "<Processing n='$file' p='GMLPlug'>\n" if ($gli);
+    print $outhandle "GMLPlug: processing $file\n";
+
+    my $parent_dir = $file;
+    $parent_dir =~ s/[^\\\/]*$//;
+    $parent_dir = &util::filename_cat ($base_dir, $parent_dir);
+
+    if (!open (INFILE, $filename)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read $filename'>\n";
+	}
+	print $outhandle "GMLPlug::read - couldn't read $filename\n";
+	return -1;
+    }
+
+    undef $/;
+    my $gml = <INFILE>;
+    $/ = "\n";
+    close (INFILE);
+    
+    my @gml_sections = split("</gsdlsection>",$gml);
+    $gml = shift(@gml_sections);
+
+    my $no_docs = 0;
+
+    while (1) {
+	# create a new document
+	my $doc_obj = new doc ();
+	my $section = $doc_obj->get_top_section();
+
+	# process the document
+	my $firstsection = 1;
+	while (1) {
+	    my ($tags, $text) = ("", "");
+
+	    my @indenting_sections = split("<gsdlsection", $gml);
+	    shift(@indenting_sections); # skips over xml header if present
+
+	    foreach $gml (@indenting_sections) {
+
+		if ($gml =~ /^\s*([^>]*)>(.*)$/so) {
+		    $tags = $1 if defined $1;
+		    $text = &GMLPlug::_unescape_text($2);
+
+		} else {
+		    print $outhandle "GMLPlug::read - error in file $filename\n";
+		    print $outhandle "text: \"$gml\"\n";
+		    last;
+		}
+
+		# create the section (unless this is the first section)
+		if ($firstsection) {
+ 		    $firstsection = 0;
+#		    $tags =~ /gsdlsourcefilename\s*=\s*(?:\"([^\"]*)\")/o;
+#		    $src_filename = $2 || $3;
+
+		} else {
+
+		    $tags =~ s/gsdlnum\s*=\s*\"?(\d+)\"?//o;
+		    if (defined $1) {
+			$section .= ".$1";
+			$doc_obj->create_named_section($section);
+		    } else {
+			$section = $doc_obj->insert_section($doc_obj->get_end_child($section));
+		    }
+		}
+	    
+		# add the metadata
+		# could be stored as either attributes or ....
+		while ((defined $tags) && ($tags =~ s/^\s*(\S+)=\"([^\"]*)\"//o)) {
+		    $doc_obj->add_utf8_metadata($section, $1, &GMLPlug::_unescape_text($2)) 
+			if (defined $1 and defined $2);
+
+		}
+		
+		# ... or tags (xml compliant)
+		if ($text =~ m/^\s*<metadata>/)
+		{
+		    my ($metadata, $tagname, $tagvalue);
+		    ($metadata,$text) 
+			= ($text =~ m/\s*<metadata>\s*(<.*)\s*<\/metadata>(.*)$/s);
+
+		    # note: \1 refers to 1st match within regexp, so we can
+		    # handle the unescaped text here...
+		    while ((defined $metadata) 
+			   && ($metadata =~ s/<(.*?)>(.*?)<\/\1>//s))
+		    {
+			if (defined $1 && defined $2) 
+			{
+			    $tagname = $1;
+			    $tagvalue = $2;
+			    
+			    # if tagname begins with '/' it will be escaped
+			    $tagname =~ s/^&\#47;/\//;
+			    
+			    $doc_obj->add_utf8_metadata($section, $tagname, &GMLPlug::_unescape_text($tagvalue)); 
+			}
+		    }
+		}
+
+		# add the text
+
+		$doc_obj->add_utf8_text($section, $text) 
+		    if ((defined $text) && ($text ne ""));		
+	    }
+
+	    $gml = shift(@gml_sections); # get next bit of data
+	    last unless defined $gml;
+ 	    last if $section eq ""; # back to top level again (more than one document in gml file)
+	    $section = $doc_obj->get_parent_section ($section);
+	} # while (1) section level
+	
+	# add the FileFormat as the metadata
+	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "GML");
+
+	# add the associated files
+	my $assoc_files = $doc_obj->get_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+	my ($assoc_file_info);
+	
+	foreach $assoc_file_info (@$assoc_files) 
+	{
+	    my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
+	    my $real_dir = &util::filename_cat($parent_dir, $assoc_file),
+	    my $assoc_dir = (defined $dir && $dir ne "") 
+		? &util::filename_cat($dir, $assoc_file) : $assoc_file;
+	    $doc_obj->associate_file($real_dir, $assoc_dir, $mime_type);
+	    
+	}
+	$doc_obj->delete_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+
+	# add metadata passed in from elsewhere
+	$self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+	
+	# do any automatic metadata extraction
+	$self->auto_extract_metadata ($doc_obj);
+
+	# assume the document has an OID already
+
+	# process the document
+	$processor->process($doc_obj, $file);
+	
+	$no_docs++;
+	last if ($maxdocs > -1 && ($total_count+$no_docs) >= $maxdocs);
+	last unless defined $gml && $gml =~ /\w/;
+    } # while(1) document level
+    
+    return $no_docs; # no of docs processed
+}
+
+sub _unescape_text {
+    my ($text) = @_;
+
+    # special characters in the gml encoding
+    $text =~ s/&lt;/</g;
+    $text =~ s/&gt;/>/g;
+    $text =~ s/&quot;/\"/g;
+    $text =~ s/&amp;/&/g; # this has to be last...
+
+    return $text;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/HBPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/HBPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/HBPlug.pm	(revision 21112)
@@ -0,0 +1,358 @@
+###########################################################################
+#
+# HBPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which processes an HTML book directory
+
+# This plugin is used by the Humanity Library collections and does not handle
+# input encodings other than ascii or extended ascii
+
+# this code is kind of ugly and could no doubt be made to run faster, by leaving
+# it in this state I hope to encourage people to make their collections use
+# HBSPlug instead ;-)
+
+# Use HBSPlug if creating a new collection and marking up files like the
+# Humanity Library collections. HBSPlug accepts all input encodings but
+# expects the marked up files to be cleaner than those used by the
+# Humanity Library collections
+
+package HBPlug;
+
+use ghtml;
+use BasPlug;
+use unicode;
+use util;
+use doc;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @HBPlug::ISA = ('BasPlug');
+}
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() }
+      ];
+
+my $options = { 'name'     => "HBPlug",
+		'desc'     => "{HBPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+    my ($verbosity, $outhandle) = @_;
+
+    $self->BasPlug::init($verbosity, $outhandle);
+    $self->{'input_encoding'} = "iso_8859_1";
+
+    # this plugin only handles ascii encodings
+    if ($self->{'input_encoding'} !~ /^(iso_8859_1|ascii)$/) {
+	die "ERROR: HBPlug can handle only iso_8859_1 or ascii encodings.\n" .
+	    $self->{'input_encoding'} . " is not an acceptable input_encoding value\n";
+    } 
+}
+
+# this is included only to prevent warnings being printed out 
+# from BasPlug::init. The process_exp is not used by this plugin
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return "This plugin does not use a process_exp\n";
+}
+
+
+sub HB_read_html_file {
+    my $self = shift (@_);
+    my ($htmlfile, $text) = @_;
+
+    # load in the file
+    if (!open (FILE, $htmlfile)) {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "ERROR - could not open $htmlfile\n";
+	return;
+    }
+
+    my $foundbody = 0;
+    $self->HB_gettext (\$foundbody, $text, "FILE");
+    close FILE;
+
+    # just in case there was no <body> tag
+    if (!$foundbody) {
+	$foundbody = 1;
+	open (FILE, $htmlfile) || return;
+	$self->HB_gettext (\$foundbody, $text, "FILE");	
+	close FILE;
+    }
+    # text is in utf8
+}
+
+# converts the text to utf8, as ghtml does that for &eacute; etc.
+sub HB_gettext {
+    my $self = shift (@_);
+    my ($foundbody, $text, $handle) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	# look for body tag
+	if (!$$foundbody) {
+	    if ($line =~ s/^.*<body[^>]*>//i) {
+		$$foundbody = 1;
+	    } else {
+		next;
+	    }
+	}
+	
+	# check for symbol fonts
+	if ($line =~ /<font [^>]*?face\s*=\s*\"?(\w+)\"?/i) {
+	    my $font = $1;
+	    print $outhandle "HBPlug::HB_gettext - warning removed font $font\n" 
+		if ($font !~ /^arial$/i);
+	}
+
+	$line =~ s/<\/p>//ig;   # remove </p> tags
+	$line =~ s/<\/?(body|html|font)\b[^>]*>//ig; # remove any unwanted tags
+
+	$$text .= $line;
+    }
+    #
+    if ($self->{'input_encoding'} eq "iso_8859_1") {
+	# convert to utf-8
+	$$text=&unicode::unicode2utf8(&unicode::convert2unicode("iso_8859_1", $text));
+    }
+    # convert any alphanumeric character entities to their utf-8
+    # equivalent for indexing purposes
+    &ghtml::convertcharentities ($$text);
+
+    $$text =~ s/\s+/ /g; # remove \n's
+}
+
+sub HB_clean_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    # remove tags without a starting tag from the section
+    my ($tag, $tagstart);
+    while ($section =~ /<\/([^>]{1,10})>/) {
+	$tag = $1;
+	$tagstart = index($section, "<$tag");
+	last if (($tagstart >= 0) && ($tagstart < index($section, "<\/$tag")));
+	$section =~ s/<\/$tag>//;
+    }
+    
+    # remove extra paragraph tags
+    while ($section =~ s/<p\b[^>]*>\s*<p\b/<p/ig) {}
+    
+    # remove extra stuff at the end of the section
+    while ($section =~ s/(<u>|<i>|<b>|<p\b[^>]*>|&nbsp;|\s)$//i) {}
+    
+    # add a newline at the beginning of each paragraph
+    $section =~ s/(.)\s*<p\b/$1\n\n<p/gi;
+    
+    # add a newline every 80 characters at a word boundary
+    # Note: this regular expression puts a line feed before
+    # the last word in each section, even when it is not
+    # needed.
+    $section =~ s/(.{1,80})\s/$1\n/g;
+    
+    # fix up the image links
+    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/
+	<center><img src=\"_httpdocimg_\/$1\"><\/center><br>/ig;
+    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/
+	<center><img src=\"_httpdocimg_\/$1\"><\/center><br>/ig;
+
+    return $section;
+}
+
+
+sub shorten {
+    my $self = shift (@_);
+    my ($text) = @_;
+
+    return "\"$text\"" if (length($text) < 100);
+
+    return "\"" . substr ($text, 0, 50) . "\" ... \"" . 
+	substr ($text, length($text)-50) . "\"";
+}
+
+# if input_encoding is ascii we can call add_utf8_metadata
+# directly but if it's iso_8859_1 (the default) we need to call
+# add_metadata so that the ascii2utf8 conversion is done first
+# this should speed things up a little if processing an ascii only
+# document with input_encoding set to ascii
+sub HB_add_metadata {
+    my $self = shift (@_);
+    my ($doc_obj, $cursection, $field, $value) = @_;
+
+# All text should now be in utf-8
+#    if ($self->{'input_encoding'} eq "ascii") {
+    $doc_obj->add_utf8_metadata ($cursection, $field, $value);
+#    } else {
+#	$doc_obj->add_metadata ($cursection, $field, $value);
+#    }
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # get the html filename and see if this is an HTML Book...
+    my $jobnumber = $file;
+    if ($file =~ /[\\\/]/) {
+	($jobnumber) = $file =~ /[\\\/]([^\\\/]+)$/;
+    }
+    return undef unless defined $jobnumber;
+    my $htmlfile = &util::filename_cat($base_dir, $file, "$jobnumber.htm");
+    return undef unless -e $htmlfile;
+
+    print STDERR "<Processing n='$file' p='HBPlug'>\n" if ($gli);
+    print $outhandle "HBPlug: processing $file\n";
+
+    # read in the file and do basic html cleaning (removing header etc)
+    my $html = "";
+    $self->HB_read_html_file ($htmlfile, \$html);
+    # html is in utf8
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+
+    # copy the book cover if it exists
+    my $bookcover = &util::filename_cat($base_dir, $file, "$jobnumber.jpg");
+    $doc_obj->associate_file($bookcover, "cover.jpg", "image/jpeg");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileFormat", "HB");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $htmlfile));
+
+    my $cursection = $doc_obj->get_top_section();
+    
+    # add metadata for top level of document
+    foreach my $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$self->HB_add_metadata ($doc_obj, $cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $self->HB_add_metadata ($doc_obj, $cursection, $field, $metadata->{$field}); 
+	}
+    }
+
+    # process the file one section at a time
+    my $curtoclevel = 1;
+    my $firstsection = 1;
+    while (length ($html) > 0) {
+	if ($html =~ s/^.*?(?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC(\d+)&gt;&gt;\s*(.*?)<p\b/<p/i) {
+	    my $toclevel = $3;
+	    my $title = $4;
+	    my $sectiontext = "";
+	    if ($html =~ s/^(.*?)((?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC\d+&gt;&gt;)/$2/i) {
+		$sectiontext = $1;
+	    } else {
+		$sectiontext = $html;
+		$html = "";
+	    }
+
+	    # remove tags and extra spaces from the title
+	    $title =~ s/<\/?[^>]+>//g;
+	    $title =~ s/^\s+|\s+$//g;
+
+	    # close any sections below the current level and
+	    # create a new section (special case for the firstsection)
+	    while (($curtoclevel > $toclevel) ||
+		   (!$firstsection && $curtoclevel == $toclevel)) {
+		$cursection = $doc_obj->get_parent_section ($cursection);
+		$curtoclevel--;
+	    }
+	    if ($curtoclevel+1 < $toclevel) {
+		print $outhandle "WARNING - jump in toc levels in $htmlfile " . 
+		    "from $curtoclevel to $toclevel\n";
+	    }
+	    while ($curtoclevel < $toclevel) {
+		$curtoclevel++;
+		$cursection = 
+		    $doc_obj->insert_section($doc_obj->get_end_child($cursection));
+	    }
+
+	    # add the metadata to this section
+	    $self->HB_add_metadata ($doc_obj, $cursection, "Title", $title);
+
+	    # clean up the section html
+	    $sectiontext = $self->HB_clean_section($sectiontext);
+
+	    # associate any files
+	    map { $doc_obj->associate_file(&util::filename_cat ($base_dir, $file, $1), $1)
+		      if /_httpdocimg_\/([^\"]+)\"/; 0; }
+	         split (/(_httpdocimg_\/[^\"]+\")/, $sectiontext);
+
+	    # add the text for this section
+# All read text should now be in utf-8
+#	    if ($self->{'input_encoding'} eq "ascii") {
+	    $doc_obj->add_utf8_text ($cursection, $sectiontext);
+#	    } else {
+#		$doc_obj->add_text ($cursection, $sectiontext);
+#	    }
+	} else {
+	    print $outhandle "WARNING - leftover text\n" , $self->shorten($html), 
+	    "\nin $htmlfile\n";
+	    last;
+	}
+	$firstsection = 0;
+    }
+
+    # add a OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj, &util::filename_cat($file, "$jobnumber.htm"));
+        
+    return 1; # processed the file
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/HTMLPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/HTMLPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/HTMLPlug.pm	(revision 21112)
@@ -0,0 +1,996 @@
+###########################################################################
+#
+# HTMLPlug.pm -- basic html plugin
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# 
+# Note that this plugin handles frames only in a very simple way
+# i.e. each frame is treated as a separate document. This means
+# search results will contain links to individual frames rather
+# than linking to the top level frameset.
+# There may also be some problems caused by the _parent target
+# (it's removed by this plugin)
+#
+
+package HTMLPlug;
+
+use BasPlug;
+use ghtml;
+use unicode;
+use util;
+use XMLParser;
+
+sub BEGIN {
+    @HTMLPlug::ISA = ('BasPlug');
+}
+
+use strict; # every perl program should have this!
+no strict 'refs'; # make an exception so we can use variables as filehandles
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' =>  &get_default_process_exp() },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => 'regexp',
+	'deft' =>  &get_default_block_exp() },
+      { 'name' => "nolinks",
+	'desc' => "{HTMLPlug.nolinks}",
+	'type' => "flag" },
+      { 'name' => "keep_head",
+	'desc' => "{HTMLPlug.keep_head}",
+	'type' => "flag" },
+      { 'name' => "no_metadata",
+	'desc' => "{HTMLPlug.no_metadata}",
+	'type' => "flag" },
+      { 'name' => "metadata_fields",
+	'desc' => "{HTMLPlug.metadata_fields}",
+	'type' => "string",
+	'deft' => "Title" },
+      { 'name' => "hunt_creator_metadata",
+	'desc' => "{HTMLPlug.hunt_creator_metadata}",
+	'type' => "flag" },
+      { 'name' => "file_is_url",
+	'desc' => "{HTMLPlug.file_is_url}",
+	'type' => "flag" },
+      { 'name' => "assoc_files",
+	'desc' => "{HTMLPlug.assoc_files}",
+	'type' => "regexp",
+	'deft' => &get_default_block_exp() },
+      { 'name' => "rename_assoc_files",
+	'desc' => "{HTMLPlug.rename_assoc_files}",
+	'type' => "flag" },
+      { 'name' => "title_sub",
+	'desc' => "{HTMLPlug.title_sub}",
+	'type' => "string", 
+	'deft' => "" },
+      { 'name' => "description_tags",
+	'desc' => "{HTMLPlug.description_tags}",
+	'type' => "flag" },
+      # retain this for backward compatibility (w3mir option was replaced by
+      # file_is_url)
+      { 'name' => "w3mir",
+#	'desc' => "{HTMLPlug.w3mir}",
+	'type' => "flag",
+	'hiddengli' => "yes"},
+      { 'name' => "no_strip_metadata_html",
+	'desc' => "{HTMLPlug.no_strip_metadata_html}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no"},
+      { 'name' => "sectionalise_using_h_tags",
+	'desc' => "{HTMLPlug.sectionalise_using_h_tags}",
+	'type' => "flag" }
+      ];
+
+my $options = { 'name'     => "HTMLPlug",
+		'desc'     => "{HTMLPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+   
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    
+    if ($self->{'w3mir'}) {
+	$self->{'file_is_url'} = 1;
+    }
+    $self->{'aux_files'} = {};
+    $self->{'dir_num'} = 0;
+    $self->{'file_num'} = 0;
+    
+    return bless $self, $class;
+}
+
+# may want to use (?i)\.(gif|jpe?g|jpe|png|css|js(?:@.*)?)$
+# if have eg <script language="javascript" src="img/lib.js@123">
+sub get_default_block_exp {
+    my $self = shift (@_);
+    
+    return q^(?i)\.(gif|jpe?g|jpe|jpg|png|css)$^;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    
+    # the last option is an attempt to encode the concept of an html query ...
+    return q^(?i)(\.html?|\.shtml|\.shm|\.asp|\.php\d?|\.cgi|.+\?.+=.*)$^;
+}
+
+sub store_block_files
+{
+    my $self =shift (@_);
+    my ($filename) = @_;
+    my $html_fname = $filename;
+    my @file_blocks;
+    
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($filename, $encoding, $language, \$text);
+    my $textref = \$text;
+    my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
+    my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
+    $$textref =~ s/$opencom(.*?)$closecom//gs;
+
+    my $attval = "\\\"[^\\\"]+\\\"|[^\\s>]+";
+    my @img_matches = ($$textref =~ m/<img[^>]*?src\s*=\s*($attval)[^>]*>/igs);
+    my @usemap_matches = ($$textref =~ m/<img[^>]*?usemap\s*=\s*($attval)[^>]*>/igs);
+    my @link_matches = ($$textref =~ m/<link[^>]*?href\s*=\s*($attval)[^>]*>/igs);
+    my @embed_matches = ($$textref =~ m/<embed[^>]*?src\s*=\s*($attval)[^>]*>/igs);
+    my @tabbg_matches = ($$textref =~ m/<(?:table|tr|td)[^>]*?background\s*=\s*($attval)[^>]*>/igs);
+
+    foreach my $link (@img_matches, @usemap_matches, @link_matches, @embed_matches, @tabbg_matches) {
+
+	# remove quotes from link at start and end if necessary
+	if ($link=~/^\"/) {
+	    $link=~s/^\"//;
+	    $link=~s/\"$//;
+	}
+
+	$link =~ s/\#.*$//s; # remove any anchor names, e.g. foo.html#name becomes foo.html
+
+	if ($link !~ m@^/@ && $link !~ m/^([A-Z]:?)\\/) {
+	    # Turn relative file path into full path
+	    my $dirname = &File::Basename::dirname($filename);
+	    $link = &util::filename_cat($dirname, $link);
+	}
+	$link = $self->eval_dir_dots($link);
+	
+	$self->{'file_blocks'}->{$link} = 1;
+    }
+}
+
+  
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print STDERR "<Processing n='$file' p='HTMLPlug'>\n" if ($gli);
+
+    print $outhandle "HTMLPlug: processing $file\n"
+	if $self->{'verbosity'} > 1;
+
+    if ($ENV{'GSDLOS'} =~ /^windows/i) {
+      # this makes life so much easier... perl can cope with unix-style '/'s.
+      $base_dir =~ s@(\\)+@/@g;
+      $file =~ s@(\\)+@/@g; 
+    }
+    
+    # reset per-doc stuff...
+    $self->{'aux_files'} = {};
+    $self->{'dir_num'} = 0;
+    $self->{'file_num'} = 0;
+
+    # process an HTML file where sections are divided by headings tags (H1, H2 ...)
+    # you can also include metadata in the format (X can be any number)
+    # <hX>Title<!--gsdl-metadata
+    #	<Metadata name="name1">value1</Metadata>
+    #	...
+    #	<Metadata name="nameN">valueN</Metadata>
+    #--></hX>
+    if ($self->{'sectionalise_using_h_tags'}) {
+	# description_tags should allways be activated because we convert headings to description tags
+	$self->{'description_tags'} = 1;
+
+	my $arrSections = [];
+	$$textref =~ s/<h([0-9]+)[^>]*>(.*?)<\/h[0-9]+>/$self->process_heading($1, $2, $arrSections, $file)/isge;
+
+	if (scalar(@$arrSections)) {
+	    my $strMetadata = $self->update_section_data($arrSections, -1);
+	    if (length($strMetadata)) {
+		$strMetadata = '<!--' . $strMetadata . "\n-->\n</body>";
+		$$textref =~ s/<\/body>/$strMetadata/ig;
+	    }
+	}
+    }
+
+    my $cursection = $doc_obj->get_top_section();
+
+    $self->extract_metadata ($textref, $metadata, $doc_obj, $cursection)
+	unless $self->{'no_metadata'} || $self->{'description_tags'};
+
+    # Store URL for page as metadata - this can be used for an
+    # altavista style search interface. The URL won't be valid
+    # unless the file structure contains the domain name (i.e.
+    # like when w3mir is used to download a website).
+
+    # URL metadata (even invalid ones) are used to support internal
+    # links, so even if 'file_is_url' is off, still need to store info
+
+    my $web_url = "http://$file";
+    $doc_obj->add_metadata($cursection, "URL", $web_url);
+
+    if ($self->{'file_is_url'}) {
+	$doc_obj->add_metadata($cursection, "weblink", "<a href=\"$web_url\">");
+	$doc_obj->add_metadata($cursection, "webicon", "_iconworld_");
+	$doc_obj->add_metadata($cursection, "/weblink", "</a>");
+    }
+
+    if ($self->{'description_tags'}) {
+	# remove the html header - note that doing this here means any
+	# sections defined within the header will be lost (so all <Section>
+	# tags must appear within the body of the HTML)
+	my ($head_keep) = ($$textref =~ m/^(.*?)<body[^>]*>/is);
+
+	$$textref =~ s/^.*?<body[^>]*>//is;
+	$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+
+	my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
+	my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
+
+	my $lt = '(?:<|&lt;)';
+	my $gt = '(?:>|&gt;)';
+	my $quot = '(?:"|&quot;|&rdquo;|&ldquo;)';
+
+	my $dont_strip = '';
+	if ($self->{'no_strip_metadata_html'}) {
+	    ($dont_strip = $self->{'no_strip_metadata_html'}) =~ s{,}{|}g;
+	}
+
+	my $found_something = 0; my $top = 1;
+	while ($$textref =~ s/^(.*?)$opencom(.*?)$closecom//s) {
+	    my $text = $1;
+	    my $comment = $2;
+	    if (defined $text) {
+		# text before a comment - note that getting to here
+		# doesn't necessarily mean there are Section tags in
+		# the document
+		$self->process_section(\$text, $base_dir, $file, $doc_obj, $cursection);
+	    }
+	    while ($comment =~ s/$lt(.*?)$gt//s) {
+		my $tag = $1;
+		if ($tag eq "Section") {
+		    $found_something = 1;
+		    $cursection = $doc_obj->insert_section($doc_obj->get_end_child($cursection)) unless $top;
+		    $top = 0;
+		} elsif ($tag eq "/Section") {
+		    $found_something = 1;
+		    $cursection = $doc_obj->get_parent_section ($cursection);
+		} elsif ($tag =~ /^Metadata name=$quot(.*?)$quot/s) {
+		    my $metaname = $1;
+		    my $accumulate = $tag =~ /mode=${quot}accumulate${quot}/ ? 1 : 0;
+		    $comment =~ s/^(.*?)$lt\/Metadata$gt//s;
+		    my $metavalue = $1;
+		    $metavalue =~ s/^\s+//;
+		    $metavalue =~ s/\s+$//;
+                    # assume that no metadata value intentionally includes
+                    # carriage returns or HTML tags (if they're there they
+                    # were probably introduced when converting to HTML from
+                    # some other format).
+		    # actually some people want to have html tags in their
+		    # metadata.
+		    $metavalue =~ s/[\cJ\cM]/ /sg;
+		    $metavalue =~ s/<[^>]+>//sg
+			unless $dont_strip && ($dont_strip eq 'all' || $metaname =~ /^($dont_strip)$/);
+		    $metavalue =~ s/\s+/ /sg;
+		    if ($accumulate) {
+			$doc_obj->add_utf8_metadata($cursection, $metaname, $metavalue);
+		    } else {
+			$doc_obj->set_utf8_metadata_element($cursection, $metaname, $metavalue);	
+		    }
+		} elsif ($tag eq "Description" || $tag eq "/Description") {
+		    # do nothing with containing Description tags
+		} else {
+		    # simple HTML tag (probably created by the conversion
+		    # to HTML from some other format) - we'll ignore it and
+		    # hope for the best ;-)
+		}
+	    }
+	}
+	if ($cursection ne "") {
+	    print $outhandle "HTMLPlug: WARNING: $file contains unmatched <Section></Section> tags\n";
+	}
+
+	$$textref =~ s/^.*?<body[^>]*>//is;
+	$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+	if ($$textref =~ /\S/) {
+	    if (!$found_something) {
+		if ($self->{'verbosity'} > 2) {
+		    print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags so\n";
+		    print $outhandle "          will be processed as a single section document\n";
+		}
+
+		# go ahead and process single-section document
+		$self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
+
+		# if document contains no Section tags we'll go ahead
+		# and extract metadata (this won't have been done
+		# above as the -description_tags option prevents it)
+		my $complete_text = $head_keep.$doc_obj->get_text($cursection);
+		$self->extract_metadata (\$complete_text, $metadata, $doc_obj, $cursection) 
+		    unless $self->{'no_metadata'};
+
+	    } else {
+		print $outhandle "HTMLPlug: WARNING: $file contains the following text outside\n";
+		print $outhandle "          of the final closing </Section> tag. This text will\n";
+		print $outhandle "          be ignored.";
+
+		my ($text);
+		if (length($$textref) > 30) {
+		    $text = substr($$textref, 0, 30) . "...";
+		} else {
+		    $text = $$textref;
+		}
+		$text =~ s/\n/ /isg;
+		print $outhandle " ($text)\n";
+	    }
+	} elsif (!$found_something) {
+
+	    if ($self->{'verbosity'} > 2) {
+		# may get to here if document contained no valid Section
+		# tags but did contain some comments. The text will have
+		# been processed already but we should print the warning
+		# as above and extract metadata
+		print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags and\n";
+		print $outhandle "          is blank or empty.  Metadata will be assigned if present.\n";
+	    }
+
+	    my $complete_text = $head_keep.$doc_obj->get_text($cursection);
+	    $self->extract_metadata (\$complete_text, $metadata, $doc_obj, $cursection) 
+		unless $self->{'no_metadata'};
+	}
+	
+    } else {
+
+	# remove header and footer
+	if (!$self->{'keep_head'} || $self->{'description_tags'}) {
+	    $$textref =~ s/^.*?<body[^>]*>//is;
+	    $$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+	}
+
+	# single section document
+	$self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
+    }
+    return 1;
+}
+
+
+sub process_heading
+{
+    my ($self, $nHeadNo, $strHeadingText, $arrSections, $file) = @_;
+    $strHeadingText = '' if (!defined($strHeadingText));
+
+    my $strMetadata = $self->update_section_data($arrSections, int($nHeadNo));
+
+    my $strSecMetadata = '';
+    while ($strHeadingText =~ s/<!--gsdl-metadata(.*?)-->//is)
+    {
+	$strSecMetadata .= $1;
+    }
+
+    $strHeadingText =~ s/^\s+//g;
+    $strHeadingText =~ s/\s+$//g;
+    $strSecMetadata =~ s/^\s+//g;
+    $strSecMetadata =~ s/\s+$//g;
+
+    $strMetadata .= "\n<Section>\n\t<Description>\n\t\t<Metadata name=\"Title\">" . $strHeadingText . "</Metadata>\n";
+
+    if (length($strSecMetadata)) {
+	$strMetadata .= "\t\t" . $strSecMetadata . "\n";
+    }
+
+    $strMetadata .= "\t</Description>\n";
+
+    return "<!--" . $strMetadata . "-->";
+}
+
+
+sub update_section_data
+{
+    my ($self, $arrSections, $nCurTocNo) = @_;
+    my ($strBuffer, $nLast, $nSections) = ('', 0, scalar(@$arrSections));
+
+    if ($nSections == 0) {
+	push @$arrSections, $nCurTocNo;
+	return $strBuffer;
+    }
+    $nLast = $arrSections->[$nSections - 1];
+    if ($nCurTocNo > $nLast) {
+	push @$arrSections, $nCurTocNo;
+	return $strBuffer;
+    }
+    for(my $i = $nSections - 1; $i >= 0; $i--) {
+	if ($nCurTocNo <= $arrSections->[$i]) {
+	    $strBuffer .= "\n</Section>";
+	    pop @$arrSections;
+	}
+    }
+    push @$arrSections, $nCurTocNo;
+    return $strBuffer;
+}
+
+
+# note that process_section may be called multiple times for a single
+# section (relying on the fact that add_utf8_text appends the text to any
+# that may exist already).
+sub process_section {
+    my $self = shift (@_);
+    my ($textref, $base_dir, $file, $doc_obj, $cursection) = @_;
+    # trap links
+    if (!$self->{'nolinks'}) {
+
+	# usemap="./#index" not handled correctly => change to "#index"
+	$$textref =~ s/(<img[^>]*?usemap\s*=\s*\"?)([^\">\s]+)(\"?[^>]*>)/
+	    $self->replace_usemap_links($1, $2, $3)/isge;
+
+	$$textref =~ s/(<(?:a|area|frame|link|script)\s+[^>]*?\s*(?:href|src)\s*=\s*\"?)([^\">\s]+)(\"?[^>]*>)/
+	    $self->replace_href_links ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
+    }
+
+    # trap images
+
+    # allow spaces if inside quotes - jrm21
+    $$textref =~ s/(<(?:img|embed|table|tr|td)[^>]*?(?:src|background)\s*=\s*)(\"[^\"]+\"|[^\s>]+)([^>]*>)/
+	$self->replace_images ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
+
+    # add text to document object
+    # turn \ into \\ so that the rest of greenstone doesn't think there
+    # is an escape code following. (Macro parsing loses them...)
+    $$textref =~ s/\\/\\\\/go;
+    
+    $doc_obj->add_utf8_text($cursection, $$textref);
+}
+
+sub replace_images {
+    my $self = shift (@_);
+    my ($front, $link, $back, $base_dir, 
+	$file, $doc_obj, $section) = @_;
+
+    # remove quotes from link at start and end if necessary
+    if ($link=~/^\"/) {
+	$link=~s/^\"//;$link=~s/\"$//;
+	$front.='"';
+	$back="\"$back";
+    }
+
+    $link =~ s/\n/ /g;
+
+    # Hack to overcome Windows wv 0.7.1 bug that causes embedded images to be broken
+    # If the Word file path has spaces in it, wv messes up and you end up with
+    #   absolute paths for the images, and without the "file://" prefix
+    # So check for this special case and massage the data to be correct
+    if ($ENV{'GSDLOS'} =~ /^windows/i && $self->{'plugin_type'} eq "WordPlug" && $link =~ /^[A-Za-z]\:\\/) {
+	$link =~ s/^.*\\([^\\]+)$/$1/;
+    }
+
+    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
+
+    my $img_file =  $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section);
+
+    my $anchor_name = $img_file;
+    $anchor_name =~ s/^.*\///;
+    $anchor_name = "<a name=\"$anchor_name\">";
+
+    return $front . $img_file . $back . $anchor_name;
+}
+
+sub replace_href_links {
+    my $self = shift (@_);
+    my ($front, $link, $back, $base_dir, $file, $doc_obj, $section) = @_;
+
+    # attempt to sort out targets - frames are not handled 
+    # well in this plugin and some cases will screw things
+    # up - e.g. the _parent target (so we'll just remove 
+    # them all ;-)
+    $front =~ s/(target=\"?)_top(\"?)/$1_gsdltop_$2/is;
+    $back =~ s/(target=\"?)_top(\"?)/$1_gsdltop_$2/is;
+    $front =~ s/target=\"?_parent\"?//is;
+    $back =~ s/target=\"?_parent\"?//is;
+
+    return $front . $link . $back if $link =~ /^\#/s;
+    $link =~ s/\n/ /g;
+
+    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
+    # href may use '\'s where '/'s should be on Windows
+    $href =~ s/\\/\//g;
+
+    my ($filename) = $href =~ /^(?:.*?):(?:\/\/)?(.*)/;
+    
+
+    ##### leave all these links alone (they won't be picked up by intermediate 
+    ##### pages). I think that's safest when dealing with frames, targets etc.
+    ##### (at least until I think of a better way to do it). Problems occur with
+    ##### mailto links from within small frames, the intermediate page is displayed
+    ##### within that frame and can't be seen. There is still potential for this to
+    ##### happen even with html pages - the solution seems to be to somehow tell
+    ##### the browser from the server side to display the page being sent (i.e. 
+    ##### the intermediate page) in the top level window - I'm not sure if that's 
+    ##### possible - the following line should probably be deleted if that can be done
+    return $front . $link . $back if $href =~ /^(mailto|news|gopher|nntp|telnet|javascript):/is;
+
+
+    if (($rl == 0) || ($filename =~ /$self->{'process_exp'}/) || 
+	($href =~ /\/$/) || ($href =~ /^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
+	&ghtml::urlsafe ($href);
+	return $front . "_httpextlink_&rl=" . $rl . "&href=" . $href . $hash_part . $back;
+    } else {
+	# link is to some other type of file (eg image) so we'll
+	# need to associate that file
+	return $front . $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) . $back;
+    }
+}
+
+sub add_file {
+    my $self = shift (@_);
+    my ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) = @_;
+    my ($newname);
+
+    my $filename = $href;
+    if ($base_dir eq "") {
+	# remove http:/ thereby leaving one slash at the start
+	$filename =~ s/^[^:]*:\///;
+    }
+    else {
+	# remove http://
+	$filename =~ s/^[^:]*:\/\///;
+    }
+
+    $filename = &util::filename_cat($base_dir, $filename);
+       
+    # Replace %20's in URL with a space if required. Note that the filename
+    # may include the %20 in some situations
+    if ($filename =~ /\%20/) {
+	if (!-e $filename) {
+	    $filename =~ s/\%20/ /g;
+	}
+    }
+
+    my ($ext) = $filename =~ /(\.[^\.]*)$/;
+
+    if ($rl == 0) {
+	if ((!defined $ext) || ($ext !~ /$self->{'assoc_files'}/)) {
+	    return "_httpextlink_&rl=0&el=prompt&href=" . $href . $hash_part;
+	}
+	else {
+	    return "_httpextlink_&rl=0&el=direct&href=" . $href . $hash_part;
+	}
+    }
+
+    if ((!defined $ext) || ($ext !~ /$self->{'assoc_files'}/)) {
+	return "_httpextlink_&rl=" . $rl . "&href=" . $href . $hash_part;
+    }
+    if ($self->{'rename_assoc_files'}) {
+	if (defined $self->{'aux_files'}->{$href}) {
+	    $newname = $self->{'aux_files'}->{$href}->{'dir_num'} . "/" .
+		$self->{'aux_files'}->{$href}->{'file_num'} . $ext;
+	} else {
+	    $newname = $self->{'dir_num'} . "/" . $self->{'file_num'} . $ext;
+	    $self->{'aux_files'}->{$href} = {'dir_num' => $self->{'dir_num'}, 'file_num' => $self->{'file_num'}};
+	    $self->inc_filecount ();
+	}
+	$doc_obj->associate_file($filename, $newname, undef, $section);
+	return "_httpdocimg_/$newname";
+    } else {
+	($newname) = $filename =~ /([^\/\\]*)$/;
+	$doc_obj->associate_file($filename, $newname, undef, $section);
+	return "_httpdocimg_/$newname";
+    }
+}
+
+
+sub format_link {
+    my $self = shift (@_);
+    my ($link, $base_dir, $file) = @_;
+
+    my ($before_hash, $hash_part) = $link =~ /^([^\#]*)(\#?.*)$/;
+    
+    $hash_part = "" if !defined $hash_part;
+    if (!defined $before_hash || $before_hash !~ /[\w\.\/]/) {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "HTMLPlug: ERROR - badly formatted tag ignored ($link)\n"
+	    if $self->{'verbosity'};
+	return ($link, "", 0);
+    }
+
+    if ($before_hash =~ s@^((?:http|ftp|file)://)@@i) {
+	my $type = $1;
+
+	if ($link =~ /^(http|ftp):/i) {
+	    # Turn url (using /) into file name (possibly using \ on windows)
+	    my @http_dir_split = split('/', $before_hash);
+	    $before_hash = &util::filename_cat(@http_dir_split);
+	}
+
+	$before_hash = $self->eval_dir_dots($before_hash);
+					 
+	my $linkfilename = &util::filename_cat ($base_dir, $before_hash);
+	
+	my $rl = 0;
+	$rl = 1 if (-e $linkfilename);
+
+	# make sure there's a slash on the end if it's a directory
+	if ($before_hash !~ /\/$/) {
+	    $before_hash .= "/" if (-d $linkfilename);
+	}
+
+	return ($type . $before_hash, $hash_part, $rl);
+    
+    } elsif ($link !~ /^(mailto|news|gopher|nntp|telnet|javascript):/i && $link !~ /^\//) {
+	if ($before_hash =~ s@^/@@ || $before_hash =~ /\\/) {
+
+	    # the first directory will be the domain name if file_is_url
+	    # to generate archives, otherwise we'll assume all files are
+	    # from the same site and base_dir is the root
+
+	    if ($self->{'file_is_url'}) {
+		my @dirs = split /[\/\\]/, $file;
+		my $domname = shift (@dirs);
+		$before_hash = &util::filename_cat($domname, $before_hash);
+		$before_hash =~ s@\\@/@g; # for windows
+	    }
+	    else
+	    {
+		# see if link shares directory with source document
+		# => turn into relative link if this is so!
+		
+		if ($ENV{'GSDLOS'} =~ /^windows/i) {
+		    # too difficult doing a pattern match with embedded '\'s...
+		    my $win_before_hash=$before_hash;
+		    $win_before_hash =~ s@(\\)+@/@g;
+		    # $base_dir is already similarly "converted" on windows.
+		    if ($win_before_hash =~ s@^$base_dir/@@o) {
+		      # if this is true, we removed a prefix
+		      $before_hash=$win_before_hash;
+		    }
+		}
+		else {
+		    # before_hash has lost leading slash by this point,
+		    # -> add back in prior to substitution with $base_dir
+		    $before_hash = "/$before_hash"; 
+
+		    $before_hash = &util::filename_cat("",$before_hash);
+		    $before_hash =~ s@^$base_dir/@@;
+		}
+	    }
+	} else {
+	    # Turn relative file path into full path
+	    my $dirname = &File::Basename::dirname($file);
+	    $before_hash = &util::filename_cat($dirname, $before_hash);
+	    $before_hash = $self->eval_dir_dots($before_hash);
+	}
+
+	my $linkfilename = &util::filename_cat ($base_dir, $before_hash); 
+	# make sure there's a slash on the end if it's a directory
+	if ($before_hash !~ /\/$/) {
+	    $before_hash .= "/" if (-d $linkfilename);
+	}
+	return ("http://" . $before_hash, $hash_part, 1);
+    } else {
+	# mailto, news, nntp, telnet, javascript or gopher link
+	return ($before_hash, "", 0);
+    }
+}
+
+sub extract_first_NNNN_characters {
+    my $self = shift (@_);
+    my ($textref, $doc_obj, $thissection) = @_;
+    
+    foreach my $size (split /,/, $self->{'first'}) {
+	my $tmptext =  $$textref;
+	# skip to the body
+	$tmptext =~ s/.*<body[^>]*>//i;
+	# remove javascript
+	$tmptext =~ s@<script.*?</script>@ @sig;
+	$tmptext =~ s/<[^>]*>/ /g;
+	$tmptext =~ s/&nbsp;/ /g;
+	$tmptext =~ s/^\s+//;
+	$tmptext =~ s/\s+$//;
+	$tmptext =~ s/\s+/ /gs;
+	$tmptext = &unicode::substr ($tmptext, 0, $size);
+	$tmptext =~ s/\s\S*$/&#8230;/; # adds an ellipse (...)
+	$doc_obj->add_utf8_metadata ($thissection, "First$size", $tmptext);
+    }
+}
+
+
+sub extract_metadata {
+    my $self = shift (@_);
+    my ($textref, $metadata, $doc_obj, $section) = @_;
+    my $outhandle = $self->{'outhandle'};
+    # if we don't want metadata, we may as well not be here ...
+    return if (!defined $self->{'metadata_fields'});
+
+    # metadata fields to extract/save. 'key' is the (lowercase) name of the
+    # html meta, 'value' is the metadata name for greenstone to use
+    my %find_fields = ();
+
+    my %creator_fields = (); # short-cut for lookups
+
+
+    foreach my $field (split /,/, $self->{'metadata_fields'}) {
+	# support tag<tagname>
+	if ($field =~ /^(.*?)<(.*?)>$/) {
+	    # "$2" is the user's preferred gs metadata name
+	    $find_fields{lc($1)}=$2; # lc = lowercase
+	} else { # no <tagname> for mapping
+	    # "$field" is the user's preferred gs metadata name
+	    $find_fields{lc($field)}=$field; # lc = lowercase
+	}
+    }
+
+    if (defined $self->{'hunt_creator_metadata'} &&
+	$self->{'hunt_creator_metadata'} == 1 ) {
+	my @extra_fields =
+	    (
+	     'author',
+	     'author.email',
+	     'creator',
+	     'dc.creator',
+	     'dc.creator.corporatename',
+	     );
+
+	# add the creator_metadata fields to search for
+	foreach my $field (@extra_fields) {
+	    $creator_fields{$field}=0; # add to lookup hash
+	}
+    }
+
+
+    # find the header in the html file, which has the meta tags
+    $$textref =~ m@<head>(.*?)</head>@si;
+
+    my $html_header=$1;
+
+    # go through every <meta... tag defined in the html and see if it is
+    # one of the tags we want to match.
+    
+    # special case for title - we want to remember if its been found
+    my $found_title = 0;
+    # this assumes that ">" won't appear. (I don't think it's allowed to...)
+    $html_header =~ /^/; # match the start of the string, for \G assertion
+ 
+    while ($html_header =~ m/\G.*?<meta(.*?)>/sig) {
+	my $metatag=$1;
+	my ($tag, $value);
+
+	# find the tag name
+	$metatag =~ /(?:name|http-equiv)\s*=\s*([\"\'])?(.*?)\1/is;
+	$tag=$2;
+	# in case they're not using " or ', but they should...
+	if (! $tag) {
+	    $metatag =~ /(?:name|http-equiv)\s*=\s*([^\s\>]+)/is;
+	    $tag=$1;
+	}
+
+	if (!defined $tag) {
+	    print $outhandle "HTMLPlug: can't find NAME in \"$metatag\"\n";
+	    next;
+	}
+	
+	# don't need to assign this field if it was passed in from a previous 
+	# (recursive) plugin
+	if (defined $metadata->{$tag}) {next}
+
+	# find the tag content
+	$metatag =~ /content\s*=\s*([\"\'])?(.*?)\1/is;
+	$value=$2;
+
+	if (! $value) {
+	    $metatag =~ /(?:name|http-equiv)\s*=\s*([^\s\>]+)/is;
+	    $value=$1;
+	}
+	if (!defined $value) {
+	    print $outhandle "HTMLPlug: can't find VALUE in \"$metatag\"\n";
+	    next;
+	}
+
+	# clean up and add
+	$value =~ s/\s+/ /gs;
+	chomp($value); # remove trailing \n, if any
+	if (exists $creator_fields{lc($tag)}) {
+	    # map this value onto greenstone's "Creator" metadata
+	    $tag='Creator';
+	} elsif (!exists $find_fields{lc($tag)}) {
+	     next; # don't want this tag
+	} else {
+	    # get the user's preferred capitalisation
+	    $tag = $find_fields{lc($tag)};
+	}
+	if (lc($tag) eq "title") {
+	    $found_title = 1;
+	}
+	print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
+	    if ($self->{'verbosity'} > 2);
+	if ($tag =~ /date.*/i){
+	    $tag = lc($tag);
+	}
+	$doc_obj->add_utf8_metadata($section, $tag, $value);
+
+    }
+    
+    # TITLE: extract the document title
+    if (exists $find_fields{'title'} && !$found_title) {
+	# we want a title, and didn't find one in the meta tags
+	# see if there's a <title> tag
+	my $title;
+	my $from = ""; # for debugging output only
+	if ($html_header =~ /<title[^>]*>([^<]+)<\/title[^>]*>/is) {
+	    $title = $1;
+	    $from = "<title> tags";
+	}
+
+	if (!defined $title) {
+	    $from = "first 100 chars";
+	    # if no title use first 100 or so characters
+	    $title = $$textref;
+	    $title =~ s/^\xFE\xFF//; # Remove unicode byte order mark	    
+	    $title =~ s/^.*?<body>//si;
+	    # ignore javascript!
+	    $title =~ s@<script.*?</script>@ @sig;
+	    $title =~ s/<\/([^>]+)><\1>//g; # (eg) </b><b> - no space
+	    $title =~ s/<[^>]*>/ /g; # remove all HTML tags
+	    $title = substr ($title, 0, 100);
+	    $title =~ s/\s\S*$/.../;
+	}
+	$title =~ s/<[^>]*>/ /g; # remove html tags
+	$title =~ s/&nbsp;/ /g;
+	$title =~ s/(?:&nbsp;|\xc2\xa0)/ /g; # utf-8 for nbsp...
+	$title =~ s/\s+/ /gs; # collapse multiple spaces
+	$title =~ s/^\s*//;   # remove leading spaces
+	$title =~ s/\s*$//;   # remove trailing spaces
+
+	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
+	$title =~ s/^\s+//s; # in case title_sub introduced any...
+	$doc_obj->add_utf8_metadata ($section, 'Title', $title);
+	print $outhandle " extracted Title metadata \"$title\" from $from\n" 
+	    if ($self->{'verbosity'} > 2);
+    } 
+    
+    # add FileFormat metadata
+    $doc_obj->add_metadata($section,"FileFormat", "HTML");
+
+    # Special, for metadata names such as tagH1 - extracts
+    # the text between the first <H1> and </H1> tags into "H1" metadata.
+
+    foreach my $field (keys %find_fields) {
+	if ($field !~ /^tag([a-z0-9]+)$/i) {next}
+	my $tag = $1;
+	if ($$textref =~ m@<$tag[^>]*>(.*?)</$tag[^>]*>@g) {
+	    my $content = $1;
+	    $content =~ s/&nbsp;/ /g;
+	    $content =~ s/<[^>]*>/ /g;
+	    $content =~ s/^\s+//;
+	    $content =~ s/\s+$//;
+	    $content =~ s/\s+/ /gs;
+	    if ($content) {
+		$tag=$find_fields{"tag$tag"}; # get the user's capitalisation
+		$tag =~ s/^tag//i;
+		$doc_obj->add_utf8_metadata ($section, $tag, $content);
+		print $outhandle " extracted \"$tag\" metadata \"$content\"\n" 
+		    if ($self->{'verbosity'} > 2);
+	    }
+	}
+    }    
+}
+
+
+# evaluate any "../" to next directory up
+# evaluate any "./" as here
+sub eval_dir_dots {
+    my $self = shift (@_);
+    my ($filename) = @_;
+    my $dirsep_os = &util::get_os_dirsep();
+    my @dirsep = split(/$dirsep_os/,$filename);
+
+    my @eval_dirs = ();
+    foreach my $d (@dirsep) {
+	if ($d eq "..") {
+	    pop(@eval_dirs);
+	    
+	} elsif ($d eq ".") {
+	    # do nothing!
+
+	} else {
+	    push(@eval_dirs,$d);
+	}
+    }
+
+    # Need to fiddle with number of elements in @eval_dirs if the
+    # first one is the empty string.  This is because of a
+    # modification to util::filename_cat that supresses the addition
+    # of a leading '/' character (or \ if windows) (intended to help
+    # filename cat with relative paths) if the first entry in the
+    # array is the empty string.  Making the array start with *two*
+    # empty strings is a way to defeat this "smart" option.
+    #
+    if (scalar(@eval_dirs) > 0) {
+	if ($eval_dirs[0] eq ""){
+	    unshift(@eval_dirs,"");
+	}
+    }
+    return &util::filename_cat(@eval_dirs);
+}
+
+sub replace_usemap_links {
+    my $self = shift (@_);
+    my ($front, $link, $back) = @_;
+
+    $link =~ s/^\.\///;
+    return $front . $link . $back;
+}
+
+sub inc_filecount {
+    my $self = shift (@_);
+
+    if ($self->{'file_num'} == 1000) {
+	$self->{'dir_num'} ++;
+	$self->{'file_num'} = 0;
+    } else {
+	$self->{'file_num'} ++;
+    }
+}
+
+
+# Extend the BasPlug read_file so that strings like &eacute; are
+# converted to UTF8 internally.  
+#
+# We don't convert &lt; or &gt; or &amp; or &quot; in case
+# they interfere with the GML files
+
+sub read_file {
+    my ($self, $filename, $encoding, $language, $textref) = @_;
+
+    &BasPlug::read_file($self, $filename, $encoding, $language, $textref);
+
+    # Convert entities to their UTF8 equivalents
+    $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
+    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1)/gseo;
+    $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ISISPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ISISPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ISISPlug.pm	(revision 21112)
@@ -0,0 +1,288 @@
+###########################################################################
+#
+# ISISPlug.pm -- A plugin for CDS/ISIS databases
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 1999-2004 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package ISISPlug;
+
+
+use multiread;
+use SplitPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+# ISISPlug is a sub-class of SplitPlug.
+sub BEGIN {
+    @ISISPlug::ISA = ('SplitPlug');
+}
+
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_block_exp() },
+      { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_split_exp() },
+
+      # The interesting options
+      { 'name' => "entry_separator",
+	'desc' => "{ISISPlug.entry_separator}",
+	'type' => "string",
+	'reqd' => "no",
+	'deft' => "<br>" },
+      { 'name' => "subfield_separator",
+	'desc' => "{ISISPlug.subfield_separator}",
+	'type' => "string",
+	'reqd' => "no",
+	'deft' => ", " }
+      ];
+
+my $options = { 'name'     => "ISISPlug",
+		'desc'     => "{ISISPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'explodes' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes files with the suffix ".mst"
+sub get_default_process_exp {
+    return q^(?i)(\.mst)$^;
+}
+
+
+# This plugin blocks files with the suffix ".fdt" and ".xrf"
+sub get_default_block_exp {
+    return q^(?i)(\.fdt|\.xrf)$^;
+}
+
+    
+# This plugin splits the input text at the "----------" lines
+sub get_default_split_exp {
+    return q^\r?\n----------\r?\n^;
+}
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+
+sub read_file
+{
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    my ($databasename) = ($filename =~ /(.*)\.mst$/i);
+
+    # Check the associated .fdt and .xrf files exist
+    # These files must have a lowercase extension for the IsisGdl program to work
+    # Bailing out because of this is kind of crappy but it is only an issue on Unix
+    my $fdtfilename = $databasename . ".fdt";
+    if (! -e $fdtfilename) {
+	die "Error: Could not find ISIS FDT file $fdtfilename.\n";
+    }
+    my $xrffilename = $databasename . ".xrf";
+    if (! -e $xrffilename) {
+	die "Error: Could not find ISIS XRF file $xrffilename.\n";
+    }
+
+    # The text to split is exported from the database by the IsisGdl program
+    open(FILE, "IsisGdl \"$filename\" |");
+
+    my $reader = new multiread();
+    $reader->set_handle('ISISPlug::FILE');
+    $reader->set_encoding($encoding);
+    $reader->read_file($textref);
+
+    close(FILE);
+
+    # Parse the associated ISIS database Field Definition Table file (.fdt)
+    my %fdtmapping = &parse_field_definition_table($fdtfilename);
+
+    # Map the tag numbers to tag names, using the FDT mapping
+    $$textref =~ s/\r?\ntag=(\d+) /\ntag=$fdtmapping{$1}{'title'} /g;
+
+    # Remove the line at the start so it is split and processed properly
+    $$textref =~ s/^----------\n//;
+}
+
+
+sub process
+{
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $cursection = $doc_obj->get_top_section();
+    my $subfield_separator = $self->{'subfield_separator'};
+    my $entry_separator = $self->{'entry_separator'};
+
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='ISISPlug'>\n" if ($gli);
+    print $outhandle "IsisPlug: processing $file\n"
+	if ($self->{'verbosity'}) > 1;
+
+    # Process each line of the ISIS record, one at a time
+    foreach my $line (split(/\n/, $$textref)) {
+	$line =~ /^tag=(.*) data=(.+)$/;
+	my $rawtagname = $1;
+	my $rawtagdata = $2;
+	# print STDERR "Raw tag: $rawtagname, Raw data: $rawtagdata\n";
+	next if ($rawtagname eq "");
+
+	# Metadata field names: title case, then remove spaces
+	my $tagname = "";
+	foreach my $word (split(/\s+/, $rawtagname)) {
+	    substr($word, 0, 1) =~ tr/a-z/A-Z/;
+	    $tagname .= $word;
+	}
+
+	# Make sure there is nothing bad in the tag names
+	$tagname =~ s/&//g;
+
+	# Handle each piece of metadata ('%' separated)
+	my $completetagvalue = "";
+	foreach my $rawtagvalue (split(/%/, $rawtagdata)) {
+	    $completetagvalue .= $entry_separator unless ($completetagvalue eq "");
+
+	    # Metadata field values: take care with subfields
+	    my $completeentryvalue = "";
+	    while ($rawtagvalue ne "") {
+		# If there is a subfield specifier, parse it off
+		my $subfieldname = "";
+		if ($rawtagvalue =~ s/^\^// && $rawtagvalue =~ s/([a-z])//) {
+		    $subfieldname = "^$1";
+		}
+
+		# Parse the metadata value off
+		$rawtagvalue =~ s/^([^\^]*)//;
+		my $metadatafieldname = $tagname . $subfieldname;
+		my $metadatafieldvalue = $1;
+		
+		# Handle Keywords specially
+		if ($metadatafieldname eq "Keywords") {
+		    my $keywordmetadatavalue = $metadatafieldvalue;
+		    my $keywordlist = "";
+		    while ($keywordmetadatavalue =~ s/\<(.+?)\>//) {
+			my $keyword = $1;
+			$doc_obj->add_utf8_metadata($cursection, $metadatafieldname, $keyword); 
+			$keywordlist .= ", " unless ($keywordlist eq "");
+			$keywordlist .= $keyword;
+		    }
+
+		    $metadatafieldvalue = $keywordlist;
+		}
+
+		# Escape any '<' and '>' characters so they appear correctly in the final collection
+		$metadatafieldvalue =~ s/\</&lt;/g;
+		$metadatafieldvalue =~ s/\>/&gt;/g;
+
+		# We have already added Keywords metadata above
+		unless ($metadatafieldname eq "Keywords") {
+		    $doc_obj->add_utf8_metadata($cursection, $metadatafieldname, $metadatafieldvalue); 
+		}
+
+		$completeentryvalue .= $subfield_separator unless ($completeentryvalue eq "");
+		$completeentryvalue .= $metadatafieldvalue;
+	    }
+
+	    $completetagvalue .= $completeentryvalue;
+	}
+
+	$doc_obj->add_utf8_metadata($cursection, $tagname . "^all", $completetagvalue); 
+    }
+
+    # Add the full record as the document text
+    $$textref =~ s/\</&lt;/g;
+    $$textref =~ s/\>/&gt;/g;
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    # Add FileFormat metadata
+    $doc_obj->add_utf8_metadata($cursection, "FileFormat", "CDS/ISIS");
+
+    # Record was processed successfully (and there was no document obtained)
+    return 1;
+}
+
+
+sub parse_field_definition_table
+{
+    my $fdtfilename = shift(@_);
+
+    my %fdtmapping = ();
+
+    open(FDT_FILE, "<$fdtfilename") || die "Error: Could not open file $fdtfilename.\n";
+
+    my $amongstdefinitions = 0;
+    foreach my $fdtfileline (<FDT_FILE>) {
+	$fdtfileline =~ s/(\s*)$//;  # Remove any nasty spaces at the end of the lines
+
+	if ($amongstdefinitions) {
+	    my $fieldtitle     = substr($fdtfileline,  0, 30);
+	    my $fieldsubfields = substr($fdtfileline, 30, 20);
+	    my $fieldspecs     = substr($fdtfileline, 50);
+
+	    # Remove extra spaces
+	    $fieldtitle =~ s/(\s*)$//;
+	    $fieldsubfields =~ s/(\s*)$//;
+
+	    # Map from tag number to metadata field title and subfields
+	    my $fieldtag = (split(/ /, $fieldspecs))[0];
+	    $fdtmapping{$fieldtag} = { 'title' => $fieldtitle,
+				       'subfields' => $fieldsubfields };
+	}
+	elsif ($fdtfileline eq "***") {
+	    $amongstdefinitions = 1;
+	}
+    }
+
+    close(FDT_FILE);
+
+    return %fdtmapping;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ImagePlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ImagePlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ImagePlug.pm	(revision 21112)
@@ -0,0 +1,473 @@
+###########################################################################
+#
+# ImagePlug.pm -- simple text plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package ImagePlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @ImagePlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "noscaleup",
+	'desc' => "{ImagePlug.noscaleup}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "thumbnailsize",
+	'desc' => "{ImagePlug.thumbnailsize}",
+	'type' => "int",
+	'deft' => "100",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "thumbnailtype",
+	'desc' => "{ImagePlug.thumbnailtype}",
+	'type' => "string",
+	'deft' => "gif",
+	'reqd' => "no" },
+      { 'name' => "screenviewsize",
+	'desc' => "{ImagePlug.screenviewsize}",
+	'type' => "int",
+	'deft' => "0",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "screenviewtype",
+	'desc' => "{ImagePlug.screenviewtype}",
+	'type' => "string",
+	'deft' => "jpg",
+	'reqd' => "no" },
+      { 'name' => "converttotype",
+	'desc' => "{ImagePlug.converttotype}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "minimumsize",
+	'desc' => "{ImagePlug.minimumsize}",
+	'type' => "int",
+	'deft' => "100",
+	'range' => "1,",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "ImagePlug",
+		'desc'     => "{ImagePlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    # Check that ImageMagick is installed and available on the path (except for Windows 95/98)
+    if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
+	my $result = `identify 2>&1`;
+	if ($? == -1 || $? == 256) {  # Linux and Windows return different values for "program not found"
+	    $self->{'imagemagick_not_installed'} = 1;
+	}
+    }
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)(\.jpe?g|\.gif|\.png|\.bmp|\.xbm|\.tif?f)$^; 
+}
+
+# this makes no sense for images
+sub block_cover_image
+{
+    my $self =shift (@_);
+    my ($filename) = @_;
+
+    return;
+}
+# Create the thumbnail and screenview images, and discover the Image's
+# size, width, and height using the convert utility.
+
+sub run_convert {
+    my $self = shift (@_);
+    my $filename = shift (@_);   # filename with full path
+    my $file = shift (@_);       # filename without path
+    my $doc_obj = shift (@_);
+    my $section = $doc_obj->get_top_section();
+    
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+
+    # check the filename is okay
+    return 0 if ($file eq "" || $filename eq "");
+
+#    Code now extended to quote filenames in 'convert' commnads
+#    Allows spaces in filenames, but note needs spaces to be escaped in URL as well
+#    if ($filename =~ m/ /) {
+#	print $outhandle "ImagePlug: \"$filename\" contains a space. choking.\n";
+#	return undef;
+#    }
+
+    my $minimumsize = $self->{'minimumsize'};
+    if (defined $minimumsize && (-s $filename < $minimumsize)) {
+        print $outhandle "ImagePlug: \"$filename\" too small, skipping\n"
+	    if ($verbosity > 1);
+    }
+
+
+    # Convert the image to a new type (if required).
+    my $converttotype = $self->{'converttotype'};
+    my $originalfilename = "";  # only set if we do a conversion
+    my $type = "unknown";
+
+
+    if ($converttotype ne "" && $filename =~ m/$converttotype$/) {
+
+	$originalfilename = $filename;
+	$filename = &util::get_tmp_filename() . ".$converttotype";
+	$self->{'tmp_filename'} = $filename;
+
+	my $command = "convert -interlace plane -verbose \"$originalfilename\" \"$filename\"";
+	print $outhandle "$command\n" if ($verbosity > 2);
+	my $result = '';
+	$result = `$command`;
+	print $outhandle "RESULT = $result\n" if ($verbosity > 2);
+
+	$type = $converttotype;
+    }
+    
+
+    # Add the image metadata
+    my $url = $file;
+    $url =~ s/ /%20/g;
+
+    $doc_obj->add_metadata ($section, "Image", $url);
+
+    # Also want to set filename as 'Source' metadata to be
+    # consistent with other plugins
+    $doc_obj->add_metadata ($section, "Source", $url);
+
+    my ($image_type, $image_width, $image_height, $image_size) 
+	= &identify($filename, $outhandle, $verbosity);
+
+    if ($image_type ne " ") {
+	$type = $image_type;
+    }
+    
+    $doc_obj->add_metadata ($section, "FileFormat", $type);
+    $doc_obj->add_metadata ($section, "FileSize",   $image_size);
+
+    $doc_obj->add_metadata ($section, "ImageType",   $image_type);
+    $doc_obj->add_metadata ($section, "ImageWidth",  $image_width);
+    $doc_obj->add_metadata ($section, "ImageHeight", $image_height);
+    $doc_obj->add_metadata ($section, "ImageSize",   $image_size);
+
+    $doc_obj->add_metadata ($section, "srclink", 
+			    "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\">");
+    $doc_obj->add_metadata ($section, "/srclink", "</a>");
+
+    $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\" width=100>");
+
+    
+    # Add the image as an associated file
+    $doc_obj->associate_file($filename,$file,"image/$type",$section);
+
+
+    # Make the thumbnail image
+    my $thumbnailsize = $self->{'thumbnailsize'} || 100;
+    my $thumbnailtype = $self->{'thumbnailtype'} || 'gif';
+	
+    my $thumbnailfile = &util::get_tmp_filename() . ".$thumbnailtype";
+    $self->{'tmp_filename2'} = $thumbnailfile;
+
+    # Generate the thumbnail with convert
+    my $command = "convert -interlace plane -verbose -geometry $thumbnailsize" 
+	. "x$thumbnailsize \"$filename\" \"$thumbnailfile\"";
+    print $outhandle "THUMBNAIL: $command\n" if ($verbosity > 2);
+    my $result = '';
+    $result = `$command 2>&1` ;
+    print $outhandle "THUMB RESULT: $result\n" if ($verbosity > 2);
+
+    # Add the thumbnail as an associated file ...
+    if (-e "$thumbnailfile") { 
+	$doc_obj->associate_file("$thumbnailfile", "thumbnail.$thumbnailtype", 
+				 "image/$thumbnailtype",$section);
+	$doc_obj->add_metadata ($section, "ThumbType", $thumbnailtype);
+	$doc_obj->add_metadata ($section, "Thumb", "thumbnail.$thumbnailtype");
+
+	$doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Thumb]\" width=[ThumbWidth] height=[ThumbHeight]>");
+    }
+
+    # Extract Thumnail metadata from convert output
+    if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
+	$doc_obj->add_metadata ($section, "ThumbWidth", $1);
+	$doc_obj->add_metadata ($section, "ThumbHeight", $2);
+    }
+
+    # Make a screen-sized version of the picture if requested
+    if ($self->{'screenviewsize'}) {
+
+	# To do: if the actual image smaller than the screenview size,
+	# we should use the original !
+
+	my $screenviewsize = $self->{'screenviewsize'};
+	my $screenviewtype = $self->{'screenviewtype'} || 'jpeg';
+	my $screenviewfilename = &util::get_tmp_filename() . ".$screenviewtype";
+	$self->{'tmp_filename3'} = $screenviewfilename;
+
+	# make the screenview image
+	my $command = "convert -interlace plane -verbose -geometry $screenviewsize" 
+	    . "x$screenviewsize \"$filename\" \"$screenviewfilename\"";
+	print $outhandle "$command\n" if ($verbosity > 2);
+	my $result = "";
+	$result = `$command  2>&1` ;
+	print $outhandle "$result\n" if ($verbosity > 3);
+
+	# get screenview dimensions, size and type
+        if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
+	    $doc_obj->add_metadata ($section, "ScreenWidth", $1);
+	    $doc_obj->add_metadata ($section, "ScreenHeight", $2);
+	}
+	else {
+	    $doc_obj->add_metadata ($section, "ScreenWidth", $image_width);
+	    $doc_obj->add_metadata ($section, "ScreenHeight", $image_height);
+	}
+
+	#add the screenview as an associated file ...
+	if (-e "$screenviewfilename") { 
+	    $doc_obj->associate_file("$screenviewfilename", "screenview.$screenviewtype",
+				     "image/$screenviewtype",$section);
+	    $doc_obj->add_metadata ($section, "ScreenType", $screenviewtype);
+	    $doc_obj->add_metadata ($section, "Screen", "screenview.$screenviewtype");
+
+	    $doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>");
+	} else {
+	    print $outhandle "ImagePlug: couldn't find \"$screenviewfilename\"\n";
+	}
+    }
+
+    return $type;
+
+
+}
+
+
+
+# Discover the characteristics of an image file with the ImageMagick
+# "identify" command.
+
+sub identify { 
+    my ($image, $outhandle, $verbosity) = @_;
+
+    # Use the ImageMagick "identify" command to get the file specs
+    my $command = "identify \"$image\" 2>&1";
+    print $outhandle "$command\n" if ($verbosity > 2);
+    my $result = '';
+    $result = `$command`;
+    print $outhandle "$result\n" if ($verbosity > 3);
+
+    # Read the type, width, and height
+    my $type =   'unknown';
+    my $width =  'unknown';
+    my $height = 'unknown';
+
+    my $image_safe = quotemeta $image;
+    if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) {
+	$type = $1;
+	$width = $2;
+	$height = $3;
+    }
+
+    # Read the size
+    my $size = "unknown";
+    if ($result =~ m/^.* ([0-9]+)b/) {
+	$size = $1;
+    }
+    elsif ($result =~ m/^.* ([0-9]+)(\.([0-9]+))?kb?/) {
+	$size = 1024 * $1;
+	if (defined($2)) {
+	    $size = $size + (1024 * $2);
+	    # Truncate size (it isn't going to be very accurate anyway)
+	    $size = int($size);
+	}
+    }
+
+    print $outhandle "file: $image:\t $type, $width, $height, $size\n" 
+	if ($verbosity > 2);
+
+    # Return the specs
+    return ($type, $width, $height, $size);
+}
+
+
+# The ImagePlug read() function. This function does all the right things
+# to make general options work for a given plugin. It calls the process()
+# function which does all the work specific to a plugin (like the old
+# read functions used to do). Most plugins should define their own 
+# process() function and let this read() function keep control.
+#
+# ImagePlug overrides read() because there is no need to read the actual 
+# text of the file in, because the contents of the file is not text...
+#
+# Return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+
+    print STDERR "<Processing n='$file' p='ImagePlug'>\n" if ($gli);
+    print $outhandle "ImagePlug processing $file\n"
+	    if $self->{'verbosity'} > 1;
+
+    # None of this works very well on Windows 95/98...
+    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Windows 95/98 not supported'>\n";
+	}
+	print $outhandle "ImagePlug: Windows 95/98 not supported\n";
+	return -1;
+    }
+
+    # None of this is going to work very well without ImageMagick...
+    if ($self->{'imagemagick_not_installed'}) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='ImageMagick not installed'>\n";
+	}
+	print $outhandle "ImagePlug: ImageMagick not installed\n";
+	return -1;
+    }
+
+    #if there's a leading directory name, eat it...
+    $file =~ s/^.*[\/\\]//;
+    
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});    
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+
+    #run convert to get the thumbnail and extract size and type info
+    my $result = run_convert($self, $filename, $file, $doc_obj);
+    
+    if (!defined $result)
+    {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file'>\n";
+	}
+	print $outhandle "ImagePlug: couldn't process \"$filename\"\n";
+	return -1; # error during processing
+    }
+
+    #create an empty text string so we don't break downstream plugins 
+    my $text = &gsprintf::lookup_string("{BasPlug.dummy_text}");
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1;
+    }
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    # if we haven't found any Title so far, assign one
+    # this was shifted to here from inside read()
+    $self->title_fallback($doc_obj,$section,$file);
+    # add an OID
+    $doc_obj->set_OID();
+    $doc_obj->add_text($section, $text);
+
+    # process the document
+    $processor->process($doc_obj);
+
+    # clean up temporary files - we do this here instead of in  
+    # run_convert becuase associated files aren't actually copied 
+    # until after process has been run.
+    if (defined $self->{'tmp_filename'} && 
+	-e $self->{'tmp_filename'}) {
+      &util::rm($self->{'tmp_filename'})
+    }
+    if (defined $self->{'tmp_filename2'} && 
+	-e $self->{'tmp_filename2'}) {
+      &util::rm($self->{'tmp_filename2'})
+    }
+    if (defined $self->{'tmp_filename3'} && 
+	-e $self->{'tmp_filename3'}) {
+      &util::rm($self->{'tmp_filename3'})
+    }
+
+    $self->{'num_processed'}++;
+
+    return 1;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/IndexPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/IndexPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/IndexPlug.pm	(revision 21112)
@@ -0,0 +1,160 @@
+###########################################################################
+#
+# IndexPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This recursive plugin processes an index.txt file. 
+# The index.txt file should contain the list of files to be
+# included in the collection followed by any extra metadata to
+# be associated with each file.
+
+# The index.txt file should be formatted as follows:
+# The first line may be a key (beginning with key:)
+# to name the metadata fields
+# (e.g. key: Subject Organization Date) 
+# The following lines will contain a filename followed 
+# by the value that metadata entry is to be set to.
+# (e.g. 'irma/iw097e 3.2 unesco 1993' will associate the
+# metadata Subject=3.2, Organization=unesco, and Date=1993
+# with the file irma/iw097e if the above key line was used)
+
+# Note that if any of the metadata fields use the Hierarchy
+# classifier plugin then the value they're set to should
+# correspond to the first field (the descriptor) in the
+# appropriate classification file.
+
+# Metadata values may be named separately using a tag 
+# (e.g. <Subject>3.2) and this will override any name 
+# given to them by the key line.
+# If there's no key line any unnamed metadata value will be
+# named 'Subject'.
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package IndexPlug;
+
+use plugin;
+use BasPlug;
+use doc;
+use util;
+use cfgread;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @IndexPlug::ISA = ('BasPlug');
+}
+
+my $arguments = [
+		 ];
+
+my $options = { 'name'     => "IndexPlug",
+		'desc'     => "{IndexPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+    
+    return 1;
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $indexfile = &util::filename_cat($base_dir, $file, "index.txt");
+    if (!-f $indexfile) {
+	# not a directory containing an index file
+	return undef;
+    }
+
+    # found an index.txt file
+    print STDERR "<Processing n='$file' p='IndexPlug'>\n" if ($gli);
+    print $outhandle "IndexPlug: processing $indexfile\n";
+
+    # read in the index.txt
+    my $list = &cfgread::read_cfg_file ($indexfile, undef, '^[^#]\w');
+    my @fields = ();
+    # see if there's a 'key:' line
+    if (defined $list->{'key:'}) {
+	@fields = @{$list->{'key:'}};
+    }
+
+    my $index_base_dir = &util::filename_cat($base_dir, $file);
+
+    # process each document
+    my $count = 0;
+    foreach my $docfile (keys (%$list)) {
+	last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs);
+	$metadata = {}; # at present we can do this as metadata
+	                # will always be empty when it arrives
+	                # at this plugin - this might cause 
+	                # problems if things change though
+
+	# note that $list->{$docfile} is an array reference
+	if ($docfile !~ /key:/i) {
+	    my $i = 0;
+	    for ($i = 0; $i < scalar (@{$list->{$docfile}}); $i ++) {
+		if ($list->{$docfile}->[$i] =~ /^<([^>]+)>(.+)$/) {
+		    unless (defined ($metadata->{$1})) {
+			$metadata->{$1} = [];
+		    } 
+		    push (@{$metadata->{$1}}, $2);
+		} elsif (scalar @fields >= $i) {
+		    unless (defined ($metadata->{$fields[$i]})) {
+			$metadata->{$fields[$i]} = [];
+		    } 
+		    push (@{$metadata->{$fields[$i]}}, $list->{$docfile}->[$i]);
+		} else {
+		    $metadata->{'Subject'} = $list->{$docfile};
+		}
+	    }
+	    $count += &plugin::read ($pluginfo, $index_base_dir, $docfile, $metadata, $processor, $maxdocs, ($total_count +$count), $gli);
+	}
+    }
+
+    return $count; # was processed
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/LaTeXPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/LaTeXPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/LaTeXPlug.pm	(revision 21112)
@@ -0,0 +1,755 @@
+###########################################################################
+#
+# LaTeXPlug.pm
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Written by John McPherson
+# Copyright (C) 2004 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+###########################################################################
+
+# todo:
+#  \includegraphics
+#  parse/remove tex \if ... macros
+
+package LaTeXPlug;
+
+# System complains about $arguments if the strict is set
+use strict;
+no strict 'refs'; # so we can print to a handle named by a variable
+
+# greenstone packages
+use BasPlug;
+use unicode;
+use util;
+
+my $arguments = [
+		 ];
+my $options={ 'name' => 'LaTeXPlug',
+	      'desc' => '{LaTeXPlug.desc}',
+	      'abstract' => 'no',
+	      'inherits' => 'yes' };
+sub BEGIN {
+    @LaTeXPlug::ISA = ('BasPlug');
+}
+
+sub print_usage {
+    print STDERR "\n  usage: plugin LaTeXPlug [options]\n\n";
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    $self->{'aux_files'} = {};
+    $self->{'dir_num'} = 0;
+    $self->{'file_num'} = 0;
+    return bless $self, $class;
+}
+
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    return q^\.tex$^;
+}
+
+sub get_default_block_exp {
+    # assume any .eps files are part of the latex stuff
+    return '\.(?:eps)$';
+}
+
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    my $start=substr($$textref, 0, 200); # first 200 bytes
+
+    if ($start !~ m~\\ (?:documentclass | documentstyle | input | section 
+			| chapter | contents | begin) ~x) {
+	# this doesn't look like latex...
+	return undef;
+    }
+    my $outhandle = $self->{'outhandle'};
+    if ($gli) {
+	print STDERR "<Processing n='$file' p='LaTeXPlug'>\n";
+    } elsif ($self->{'verbosity'} > 1) {
+	print $outhandle "LaTeXPlug: processing $file\n" 
+    }
+    my $cursection = $doc_obj->get_top_section();
+
+    ###### clean up text ######
+    $$textref =~ s/\r$//mg;  # remove dos ^M
+    $$textref =~ s/%.*$//mg; # remove comments
+
+    # convert to utf-8 if not already - assume non ascii => iso-8859-1/latin
+
+    $$textref =~ s@(?<=[[:ascii:]])\xA0+@\xc2\xa0@g; # latin nonbreaking space
+    # check that both sides are ascii, so we don't screw up utf-8 chars
+    $$textref =~ s@ (?<=[[:ascii:]])([\x80-\xff])(?=[[:ascii:]]) @
+	unicode::ascii2utf8($1) @egx; # takes "extended ascii" (ie latin)
+
+
+    ###### find metadata ######
+
+    ## FileFormat metadata ##
+    $doc_obj->add_metadata($cursection, "FileFormat", "LaTeX");
+
+    ### title metadata ###
+    $$textref =~ m@\\title\s*{(.*?)}@s;
+    my $title = $1;
+    if (!$title) {
+	# no title tag. look for a chapter/section heading
+	$$textref =~ m@\\(?:chapter|section)\s*{(.*?)}@s; # will get 1st match
+	$title = $1;
+    }
+    if (!$title) {
+	# no chapter/section heading tags either... use filename
+	$title = $file;
+	$title =~ s/\.tex$//i;
+	$title =~ s/[-_.]/ /g; # turn punctuation into spaces
+    }
+    if ($title) {
+	$title =~ s@\\\\@ @g; # embedded newlines
+	$title = $self->process_latex($title); # no "-html" for title eg in browser
+	$doc_obj->add_utf8_metadata($cursection, "Title", $title);
+    }
+
+    ### creator/individual author metadata ###
+    $$textref =~ m@\\author\s*{((?:{.*?}|.*?)+)}\s*$@sm;
+    my $authors=$1;
+    if ($authors) {
+	# take care of "et al."...
+	$authors =~ s/(\s+et\.?\s+al\.?)\s*$//;
+	my $etal=$1;
+	$etal="" if (!defined ($etal));
+
+	my @authorlist=parse_authors($self, $authors);
+
+	foreach my $author (@authorlist) {
+	    # Add each name to set of Authors
+	    $doc_obj->add_utf8_metadata ($cursection, "Author", $author);
+	}
+
+	# Only want at most one "and" in the Creator field
+	my $creator_str="";
+	if (scalar(@authorlist) > 2) {
+	    my $lastauthor=pop @authorlist;
+	    $creator_str=join(', ', @authorlist);
+	    $creator_str.=" and $lastauthor";
+	} else { # 1 or 2 authors...
+	    $creator_str=join(" and ",@authorlist);
+	}
+	$creator_str.=$etal; # if there was "et al."
+    	$doc_obj->add_utf8_metadata($cursection, "Creator", $creator_str);
+    }
+    ### end of author metadata ###
+
+    ###### process latex for the main text ######
+    $$textref =~ s/^.*?\\begin{document}//s;
+    $$textref =~ s/\\end{document}.*?$//s;
+    $$textref = $self->process_latex("-html",$$textref);
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    return 1;
+}
+
+
+# returns a list of author names
+sub parse_authors {
+    my $self=shift;
+    my $authors=shift;
+
+    my $outhandle=$self->{'outhandle'};
+
+    $authors =~ s/\n/ /g; # remove newlines
+	
+    # some people do this for affiliation footnote/dagger
+    $authors =~ s@\$.*?\$@@g; # remove maths from author :(
+
+    # und here for german language...
+    # don't use brackets in pattern, else the matched bit becomes
+    # an element in the list!
+    my @authorlist = split(/\s+and\s+|\s+und\s+/, $authors);
+    my @formattedlist = ();
+    foreach my $author (@authorlist) {
+	$author =~ s/\s*$//; 
+	$author =~ s/^\s*//; 
+	# Reformat and add author name
+	next if $author=~ /^\s*$/;
+
+	# names are "First von Last", "von Last, First"
+	# or "von Last, Jr, First". See the "BibTeXing" manual, page 16
+	my $first="";
+	my $vonlast="";
+	my $jr="";
+	    
+	if ($author =~ /,/) {
+	    my @parts=split(/,\s*/, $author);
+	    $first = pop @parts;
+	    if (scalar(@parts) == 2) {
+		$jr = pop @parts;
+	    }
+	    $vonlast=shift @parts;
+	    if (scalar(@parts) > 0) {
+		print $outhandle $self->{'plugin_type'} .
+		    ": couldn't parse name $author\n";
+		# but we continue anyway...
+	    }
+	} else { # First von Last
+	    my @words = split(/ /, $author);
+	    while (scalar(@words) > 1 && $words[0] !~ /^[a-z]{2..}/) {
+		$first .= " " . shift (@words);
+	    }
+	    $first =~ s/^\s//;
+	    $vonlast = join (' ', @words); # whatever's left...
+	}
+	my $von="";
+	my $last="";
+	if ($vonlast =~ m/^[a-z]/) { # lowercase implies "von"
+	    $vonlast =~ s/^(([a-z]\w+\s+)+)//;
+	    $von = $1;
+	    if (!defined ($von)) {
+		# some non-English names do start with lowercase
+		# eg "Marie desJardins". Also we can get typos...
+		print $outhandle "BibTexPlug: couldn't parse surname $vonlast\n";
+		$von="";
+		if ($vonlast =~ /^[a-z]+$/) {
+		    # if it's all lowercase, uppercase 1st.
+		    $vonlast =~ s/^(.)/\u$1/;
+		}
+	    }
+	    $von =~ s/\s*$//;
+	    $last=$vonlast;
+	} else {
+	    $last=$vonlast;
+	}
+	my $wholename="$first $von $last $jr";
+	$wholename =~ s/ $//; $wholename =~ s/\s+/ /g;
+#	my $fullname = "$last";
+#	$fullname .= " $jr" if ($jr);
+#	$fullname .= ", $first";
+#	$fullname .= " $von" if ($von);
+	push (@formattedlist, $wholename);
+    }
+    return @formattedlist;
+}
+
+
+## following functions based on bibtex plugin ##
+# not actually used at the moment, but might be useful in future?
+sub expand_month {
+    my $text=shift;
+
+    # bibtex style files expand abbreviations for months.
+    # Entries can contain more than one month (eg ' month = jun # "-" # aug, ')
+    $text =~ s/jan/_textmonth01_/g;
+    $text =~ s/feb/_textmonth02_/g;
+    $text =~ s/mar/_textmonth03_/g;
+    $text =~ s/apr/_textmonth04_/g;
+    $text =~ s/may/_textmonth05_/g;
+    $text =~ s/jun/_textmonth06_/g;
+    $text =~ s/jul/_textmonth07_/g;
+    $text =~ s/aug/_textmonth08_/g;
+    $text =~ s/sep/_textmonth09_/g;
+    $text =~ s/oct/_textmonth10_/g;
+    $text =~ s/nov/_textmonth11_/g;
+    $text =~ s/dec/_textmonth12_/g;
+
+    return $text;
+}
+
+
+# If you want basic html formatting (eg \emph -> <em>, \bf, etc), give "-html"
+# as the first argument to this function.
+#
+# Convert accented characters, remove { }, interprete some commands....
+# Note!! This is not comprehensive! Also assumes Latin -> Unicode!
+
+# Also, it sucks quite a bit for complicated/nested commands since it doesn't
+# match { with the corresponding }, only the nearest }
+
+sub process_latex {
+    my $self=shift;
+    my $text=shift;
+
+    my $outhandle=$self->{'outhandle'};
+
+    my $html_markup=0;
+    if ($text =~ /^\-html/) {
+	$html_markup=1;
+	$text=shift;
+    }
+
+    if (! $text) {
+	return $text;
+    }
+    # escape html-sensitive characters
+    $text =~ s@&@&amp;@g;
+    $text =~ s@<@&lt;@g;
+    $text =~ s@>@&gt;@g;
+ 
+    # do this before accents, since \= means something different in tabbing
+    # also \> is a tab stop too, and \\ is newline
+    sub do_tabbing {
+	my $tabbing=shift;
+	$tabbing =~ s!^.*\\kill\s*$!!mg; # \kill sets tab stops, kills line
+	$tabbing =~ s~\\(?:=|&gt;)~\xc2\xa0~g; # replace with nbsp
+	$tabbing =~ s~[\\][\\](?:\[.*?\])?\s*$~<br/>~mg;
+	return "<br/>" . $tabbing . "<br/>\n";
+    }
+    $text =~ s@\\begin{tabbing}(.*?)\\end{tabbing}@do_tabbing($1)@ges;
+    sub do_tabular {
+	my $tabular=shift;
+	$tabular =~ s~(?<!\\)\s*&amp;\s*~</td><td>~g;
+	$tabular =~ s~[\\][\\]\s*~</td></tr>\n <tr><td>~g;
+	$tabular =~ s~\\hline~~g; # for now...
+	$tabular =~ s~<td>\s*\\multicolumn{(\d+)}{.*?}~<td colspan="$1">~g;
+	return "<table border=\"1\">\n <tr><td>"
+	    . $tabular . "</td></tr></table>\n";
+    }
+    $text =~ s@\\begin{tabular}(?:\[.*?\])?{.*?}(.*?)\\end{tabular} @
+    		do_tabular($1)  @xges;
+
+    $text =~ s@[\\][\\]\s*\n@ @g; # fold lines ending with \\
+
+    # process maths mode before accents... things like \, mean different!
+    # maths mode
+    $text =~ s@\$\$(.*?)\$\$ 
+	@ process_latex_math($html_markup,$1)
+	@xsge; # multi-line maths: $$ .... $$
+
+    $text =~ s@([^\\])\$(.*?[^\\])\$
+	@$1.process_latex_math($html_markup,$2)@xsge;
+
+
+    # is this an amstext environment, or just custom for that input file?
+    $text =~ s@\\begin{(algorithm)}(.*?)\\end{\1}@remove_equals($2)@ges;
+
+    # convert latex-style accented characters.
+    $self->latex_accents_to_utf8(\$text);
+
+    # replace quotes with utf-8
+
+    $text =~ s/``/\xe2\xc0\x9c/g; # Latex-specific, left-dbl quote (&ldquo;)
+    $text =~ s/''/\xe2\xc0\x9d/g; # Latex-specific, right-dbl quote (&rdquo;)
+    $text =~ s/`/\xe2\xc0\x98/g; # single left quote
+    $text =~ s/'/\xe2\xc0\x99/g; # single right quote
+
+    ###### remove/replace latex commands ######
+    ### commands that expand to something that gets displayed ###
+    $text =~ s~\\ldots~&hellip;~g;
+    $text =~ s~\\hrule~<hr/>\n~g;
+    $text =~ s~\\maketitle~ ~;
+    ### space commands ###
+    $text =~ s~\\[vh]skip\s+\S+~~g;
+    $text =~ s~\\vspace\*?{.*?}~<div>&nbsp;</div>~g; # vertical space
+    $text =~ s~\\\w+skip~ ~g; # \smallskip \medskip \bigskip \baselineskip etc
+    $text =~ s~\\noindent\b~~g;
+    # newpage, etc
+    $text =~ s~\\(?:clearemptydoublepage|newpage)~~g;
+    ### counters, contents, environments, labels, etc ###
+    $text =~ s~\\(?:addcontentsline){.*?}{.*?}{.*}~~g;
+    $text =~ s~\s*\\begin{itemize}\s*~\n<ul>\n~g;
+    $text =~ s~\s*\\end{itemize}\s*~</li></ul>\n~g;
+    $text =~ s~\s*\\begin{enumerate}\s*~<ol>\n~g;
+    $text =~ s~\s*\\end{enumerate}\s*~</li></ol>\n~g;
+    if ($text =~ s~\s*\\item~</li>\n<li>~g) {
+	# (count for first list item)
+	$text =~ s~<([ou])l>\s*</li>\s*~<$1l>~g;
+    }
+    $text =~ s~\\(?:label|begin|end){.*?}\s*\n?~ ~g; # remove tag and contents
+    $text =~ s~\\(?:tableofcontents|listoffigures)~ ~g;
+    ### font sizes/styles ###
+    $text =~ s~\\(?:tiny|small|footnotesize|normalsize|large|Large|huge|Huge)\b~~g;
+
+    if ($html_markup) {
+	$text =~ s~\\section\*?{([^\}]+)}\s*\n?~<H1>$1</H1>\n~g;
+	$text =~ s~\\subsection\*?{(.*?)}\s*\n?~<H2>$1</H2>\n~g;
+	$text =~ s~{\\tt\s*(.*?)}~<tt>$1</tt>~g;
+	$text =~ s~\\(?:texttt|tt|ttseries)\s*{(.*?)}~<tt>$1</tt>~g;
+	$text =~ s~\\emph{(.*?)}~<em>$1</em>~g;
+	$text =~ s~{\\(?:em|it)\s*(.*?)}~<em>$1</em>~g;
+	$text =~ s~{\\(?:bf|bfseries)\s*(.*?)}~<strong>$1</strong>~g;
+	$text =~ s~\\(?:textbf|bf|bfseries)\s*{(.*?)}~<strong>$1</strong>~g;
+    } else {
+	# remove tags for text-only
+	$text =~ s~\\(?:textbf|bf|bfseries|em|emph|tt|rm|texttt)\b~ ~g;
+    }
+    $text =~ s ~ {\\sc\s+(.*?)} ~
+		{<span style="font-variant:\ small-caps">$1</span>} ~gx;
+    # ignore these font tags (if there are any left)
+    # sf is sans-serif
+    $text =~ s~\\(?:mdseries|textmd|bfseries|textbf|sffamily|sf|sc)\b~ ~;
+    #### end font-related stuff ####
+
+    ### remove all other commands with optional arguments... ###
+    # don't remove commands without { }....
+    # $text =~ s~\\\w+(\[.*?\])?\s*~~g;
+    # $text =~ s~\\noopsort{[^}]+\}~~g;
+    # verbatim
+    $text =~ s~\\verb(.)(.*?)\1~verb_text($1)~ge;
+    # remove tags, keep contents for \tag[optional]{contents}
+    while ($text =~ s~\\\w+(\[.*?\])?{([^}]+)}~$2 ~g) {;} # all other commands
+    
+    # remove latex groupings { } (but not \{ or \} )
+    while ($text =~ s/([^\\])[\{\}]/$1/g) {;} # needed for "...}{..."
+    $text =~ s/^\{//; # remove { if first char
+
+    # latex characters
+    # spaces - nobr space (~), opt break (\-), append ("#" - bibtex only)
+    $text =~ s/([^\\])~+/$1 /g; # non-breaking space  "~"
+    # optional break "\-"
+    if ($text =~ m/[^&]\#/) { # concat macros (bibtex) but not HTML codes
+	# the non-macro bits have quotes around them - we just remove quotes
+	# XXX bibtex and latex differ here (for the '#' char)...
+	$text =~ s/([^&])[\"\#]/$1/g;
+    }
+    # dashes. Convert (m|n)-dash into single dash for html.
+    $text =~ s~\-\-+~\-~g;
+
+    # quoted { } chars
+    $text =~ s~\\{~{~g;
+    $text =~ s~\\}~}~g;
+
+    # spaces
+    $text =~ s~\\ ~ ~g;
+
+    # finally to protect against macro language...
+    # greenstone-specific
+    $text =~ s~\[~&\#91;~g;
+    $text =~ s~\]~&\#93;~g;
+    $text =~ s~(?<!\\)([\\_])~\\$1~g;
+
+    if ($html_markup) {
+	$text =~ s~\n{2,}~\n</p>\n<p>~g;
+	return "<p>$text</p>";
+    }
+
+    return $text;
+}
+
+# only used by process_latex for \verb....
+sub verb_text {
+    my $verbatim=shift;
+    $verbatim =~ s/([{}_])/\\$1/g;
+    return $verbatim;
+}
+# only used by process_latex_math
+# returns a unicode char if applicable, otherwise ascii
+sub math_fraction {
+    my $num=$1;
+    my $denom=$2;
+
+    if ($num==1 && $denom==2) {return chr(0xc2).chr(0xbd)}
+    if ($num==1 && $denom==4) {return chr(0xc2).chr(0xbc)}
+    if ($num==3 && $denom==4) {return chr(0xc2).chr(0xbe)}
+    return "$num/$denom";
+}
+
+sub process_latex_math {
+
+    my $text = pop; # if given one or two args, this is the last one...
+    my $html_markup=pop; # if given two args, this is the first one else undef
+
+    $text =~ s~\\,~ ~g; # forces a space?
+    $text =~ s~\\infty~infinity~g;             # or unicode 0x221E...
+
+# use this one when more things can read 3-byte utf8 values like this!
+#    $text =~ s~\\cup\b~\xe2\xc8\xaa~g; # union operator - unicode 0x222a
+    $text =~ s~\\cup\b~ U ~g;
+
+    $text =~ s~\\frac\s*{(.+?)}{(.+?)}~math_fraction($1,$2)~ge;
+
+    if ($html_markup) {
+	$text =~ s~\^{(.*?)}~<sup>$1</sup>~g;  # a^b superscript
+	$text =~ s~\^([^\{])~<sup>$1</sup>~g;
+	$text =~ s~\_{(.*?)}~<sub>$1</sub>~g;  # a_b subscript
+	$text =~ s~\_([^\{])~<sub>$1</sub>~g;
+	
+	$text =~ s~\\ldots~&hellip;~g;         # use html named entity for now
+
+	# put all other command names in italics for now
+	$text =~ s~\\([\w]+)~<i>$1</i> ~g;
+    }
+
+    # special cases, for some input files
+    if ($text =~ m~^\\\w+$~) {
+	$text="{" . $text . "}";
+    }
+
+    return $text;
+}
+
+
+
+sub latex_accents_to_utf8 {
+
+    # note - this is really ugly, but it works. There may be a prettier way
+    # of mapping latex accented chars to utf8, but we just brute force it here.
+    # Also, this isn't complete - not every single possible accented letter
+    # is in here yet, but most of the common ones are.
+
+    my %utf8_chars =
+	(
+	 # acutes
+	 '\'a' => chr(0xc3).chr(0xa1),
+	 '\'c' => chr(0xc4).chr(0x87),
+	 '\'e' => chr(0xc3).chr(0xa9),
+	 '\'i' => chr(0xc3).chr(0xad),
+	 '\'l' => chr(0xc3).chr(0xba),
+	 '\'n' => chr(0xc3).chr(0x84),
+	 '\'o' => chr(0xc3).chr(0xb3),
+	 '\'r' => chr(0xc5).chr(0x95),
+	 '\'s' => chr(0xc5).chr(0x9b),
+	 '\'u' => chr(0xc3).chr(0xba),
+	 '\'y' => chr(0xc3).chr(0xbd),
+	 '\'z' => chr(0xc5).chr(0xba),
+	 # graves
+	 '`a' => chr(0xc3).chr(0xa0),
+	 '`A' => chr(0xc3).chr(0x80),
+	 '`e' => chr(0xc3).chr(0xa8),
+	 '`E' => chr(0xc3).chr(0x88),
+	 '`i' => chr(0xc3).chr(0xac),
+	 '`I' => chr(0xc3).chr(0x8c),
+	 '`o' => chr(0xc3).chr(0xb2),
+	 '`O' => chr(0xc3).chr(0x92),
+	 '`u' => chr(0xc3).chr(0xb9),
+	 '`U' => chr(0xc3).chr(0x99),
+	 # circumflex
+	 '^a' => chr(0xc3).chr(0xa2),
+	 '^A' => chr(0xc3).chr(0x82),
+	 '^c' => chr(0xc4).chr(0x89),
+	 '^C' => chr(0xc4).chr(0x88),
+	 '^e' => chr(0xc3).chr(0xaa),
+	 '^E' => chr(0xc3).chr(0x8a),
+	 '^g' => chr(0xc4).chr(0x9d),
+	 '^G' => chr(0xc4).chr(0x9c),
+	 '^h' => chr(0xc4).chr(0xa5),
+	 '^H' => chr(0xc4).chr(0xa4),
+	 '^i' => chr(0xc3).chr(0xae),
+	 '^I' => chr(0xc3).chr(0x8e),
+	 '^j' => chr(0xc4).chr(0xb5),
+	 '^J' => chr(0xc4).chr(0xb4),
+	 '^o' => chr(0xc3).chr(0xb4),
+	 '^O' => chr(0xc3).chr(0x94),
+	 '^s' => chr(0xc5).chr(0x9d),
+	 '^S' => chr(0xc5).chr(0x9c),
+	 '^u' => chr(0xc3).chr(0xa2),
+	 '^U' => chr(0xc3).chr(0xbb),
+	 '^w' => chr(0xc5).chr(0xb5),
+	 '^W' => chr(0xc5).chr(0xb4),
+	 '^y' => chr(0xc5).chr(0xb7),
+	 '^Y' => chr(0xc5).chr(0xb6),
+	 
+	 # diaeresis
+	 '"a' => chr(0xc3).chr(0xa4),
+	 '"A' => chr(0xc3).chr(0x84),
+	 '"e' => chr(0xc3).chr(0xab),
+	 '"E' => chr(0xc3).chr(0x8b),
+	 '"i' => chr(0xc3).chr(0xaf),
+	 '"I' => chr(0xc3).chr(0x8f),
+	 '"\\\\i' => chr(0xc3).chr(0xaf),
+	 '"\\\\I' => chr(0xc3).chr(0x8f),
+	 '"o' => chr(0xc3).chr(0xb6),
+	 '"O' => chr(0xc3).chr(0x96),
+	 '"u' => chr(0xc3).chr(0xbc),
+	 '"U' => chr(0xc3).chr(0x9c),
+	 '"y' => chr(0xc3).chr(0xbf),
+	 '"Y' => chr(0xc3).chr(0xb8),
+	 # tilde
+	 '~A' => chr(0xc3).chr(0x83),
+	 '~N' => chr(0xc3).chr(0x91),
+	 '~O' => chr(0xc3).chr(0x95),
+	 '~a' => chr(0xc3).chr(0xa3),
+	 '~n' => chr(0xc3).chr(0xb1),
+	 '~o' => chr(0xc3).chr(0xb5),
+	 # caron - handled specially
+	 # double acute
+	 # ring
+	 # dot
+	 '.c' => chr(0xc4).chr(0x8b),
+	 '.C' => chr(0xc4).chr(0x8a),
+	 '.e' => chr(0xc4).chr(0x97),
+	 '.E' => chr(0xc4).chr(0x96),
+	 '.g' => chr(0xc4).chr(0xa1),
+	 '.G' => chr(0xc4).chr(0xa0),
+	 '.I' => chr(0xc4).chr(0xb0),
+	 '.z' => chr(0xc5).chr(0xbc),
+	 '.Z' => chr(0xc5).chr(0xbb),
+	 # macron
+	 '=a' => chr(0xc4).chr(0x81),
+	 '=A' => chr(0xc4).chr(0x80),
+	 '=e' => chr(0xc4).chr(0x93),
+	 '=E' => chr(0xc4).chr(0x92),
+	 '=i' => chr(0xc4).chr(0xab),
+	 '=I' => chr(0xc4).chr(0xaa),
+	 '=o' => chr(0xc4).chr(0x8d),
+	 '=O' => chr(0xc4).chr(0x8c),
+	 '=u' => chr(0xc4).chr(0xab),
+	 '=U' => chr(0xc4).chr(0xaa),
+	 
+	 # stroke - handled specially - see below
+	 
+	 # cedilla - handled specially
+	 );
+    
+# these are one letter latex commands - we make sure they're not a longer
+# command name. eg {\d} is d+stroke, so careful of \d
+    my %special_utf8_chars = 
+	(
+	 # breve
+	 'u g' => chr(0xc4).chr(0x9f),
+	 'u G' => chr(0xc4).chr(0x9e),
+	 'u i' => chr(0xc4).chr(0xad),
+	 'u I' => chr(0xc4).chr(0xac),
+	 'u o' => chr(0xc5).chr(0x8f),
+	 'u O' => chr(0xc5).chr(0x8e),
+	 'u u' => chr(0xc5).chr(0xad),
+	 'u U' => chr(0xc5).chr(0xac),
+	 'u z' => chr(0xc5).chr(0xbe), # !!! no such char, but common mistake
+	 'u Z' => chr(0xc5).chr(0xbd), # used instead of v Z !!!
+	 # caron
+	 'v c' => chr(0xc4).chr(0x8d),
+	 'v C' => chr(0xc4).chr(0x8c),
+	 'v n' => chr(0xc5).chr(0x88),
+	 'v N' => chr(0xc5).chr(0x87),
+	 'v s' => chr(0xc5).chr(0xa1),
+	 'v S' => chr(0xc5).chr(0xa5),
+	 'v z' => chr(0xc5).chr(0xbe),
+	 'v Z' => chr(0xc5).chr(0xbd),
+	 # cedilla
+	 'c c' => chr(0xc3).chr(0xa7),
+	 'c C' => chr(0xc3).chr(0x87),
+	 'c g' => chr(0xc4).chr(0xa3),
+	 'c G' => chr(0xc4).chr(0xa2),
+	 'c k' => chr(0xc4).chr(0xb7),
+	 'c K' => chr(0xc4).chr(0xb6),
+	 'c l' => chr(0xc4).chr(0xbc),
+	 'c L' => chr(0xc4).chr(0xbb),
+	 'c n' => chr(0xc5).chr(0x86),
+	 'c N' => chr(0xc5).chr(0x85),
+	 'c r' => chr(0xc5).chr(0x97),
+	 'c R' => chr(0xc5).chr(0x96),
+	 'c s' => chr(0xc5).chr(0x9f),
+	 'c S' => chr(0xc5).chr(0x9e),
+	 'c t' => chr(0xc5).chr(0xa3),
+	 'c T' => chr(0xc5).chr(0xa2),
+	 # double acute / Hungarian accent
+	 'H O' => chr(0xc5).chr(0x90),
+	 'H o' => chr(0xc5).chr(0x91),
+	 'H U' => chr(0xc5).chr(0xb0),
+	 'H u' => chr(0xc5).chr(0xb1),
+	 
+	 # stroke 
+	 'd' => chr(0xc4).chr(0x91),
+	 'D' => chr(0xc4).chr(0x90),
+	 'h' => chr(0xc4).chr(0xa7),
+#	 'H' => chr(0xc4).chr(0xa6), # !! this normally(!!?) means Hung. umlaut
+	 'i' => chr(0xc4).chr(0xb1), # dotless lowercase i
+	 'l' => chr(0xc5).chr(0x82),
+	 'L' => chr(0xc5).chr(0x81),
+	 'o' => chr(0xc3).chr(0xb8),
+	 'O' => chr(0xc3).chr(0x98),
+	 't' => chr(0xc5).chr(0xa7),
+	 'T' => chr(0xc5).chr(0xa6),
+
+         # other special chars
+         'ss' => chr(0xc3).chr(0x9f), # german ss/szlig/sharp s
+         'aa' =>,chr(0xc3).chr(0xa5), # scandanavian/latin a with ring
+	 );
+
+    my $self=shift;
+    my $textref=shift;
+
+    my $outhandle=$self->{'outhandle'};
+    my $text=$$textref;
+    
+    # remove space (if any) between \ and letter to accent (eg {\' a})
+    $text =~ s!(\\[`'="^~\.])\s(\w)\b!$1$2!g;   # for emacs indenting... `]);
+
+    # remove {} around a single character (eg \'{e})
+    $text =~ s!(\\[`'="^~\.]){(\w)}!{$1$2}!g;  # for emacs indenting... `]);
+
+    ## only in bibtex... not in latex proper?!
+    ### \, is another way of doing cedilla \c
+    ##$text =~ s~\\,(.)~\\c $1~g;
+
+    # remove {} around a single character for special 1 letter commands -
+    # need to insert a space. Eg \v{s}  ->  {\v s}
+    $text =~ s~(\\[uvcH]){(\w)}~{$1 $2}~g;
+
+    # only do if the text contains a '\' character. 
+    if ($text =~ m|\\|) {
+	# "normal" accents - ie non-alpha latex tag
+	# xxx used to have ([\w]\b)@ (for word boundary)
+	while ($text =~ m/\\([`'="^~\.])([\w])/) {      # for emacs `])){
+	    my $tex="$1$2"; my $char="$2";
+	    my $replacement=$utf8_chars{$tex};
+	    if (!defined($replacement)) {
+		$text =~ m~(.{20}\\\Q$tex\E.{20})~s;
+		print $outhandle . $self->{'plugin_type'} .
+		    ": Warning: unknown latex accent \"$tex\""
+		    . " in \"$1\"\n";
+		$replacement=$char;
+	    }
+	    $text =~ s/\\\Q$tex/$replacement/g;
+	}
+
+        # where the following letter matters (eg "sm\o rrebr\o d", \ss{})
+        # only do the change if immediately followed by a space, }, {, or \
+	# one letter accents ( + ss / aa)
+        while ($text =~ m~\\([DdhiLlOoTt]|ss|aa)[{}\s\"\\]~) {
+	    my $tex=$1;
+	    my $replacement=$special_utf8_chars{$tex};
+	    if (!defined($replacement)) {
+		$text =~ m~(.{20}\\\Q$tex\E.{20})~s;
+		print $outhandle $self->{'plugin_type'} .
+		    ": Warning: unknown latex accent \"$tex\""
+		    . " in \"$1\"\n";
+		$replacement=$tex;
+	    }
+	    ($text =~ s/{\\$tex}/$replacement/g) or
+		$text =~ s/\\$tex([{}\s\"\\])/$replacement$1/g;
+
+	}
+
+	# one letter latex accent commands that affect following letter
+        while ($text =~ m~\\([uvcH]) ([\w])~) {
+          my $tex="$1 $2"; my $char="$2";
+          my $replacement=$special_utf8_chars{$tex};
+          if (!defined($replacement)) {
+	      $text =~ m~(.{20}\\\Q$tex\E.{20})~s;
+	      print  $outhandle $self->{'plugin_type'} .
+		  ": Warning: unknown latex accent \"$tex\""
+		  . " in \"$1\"\n";
+	      $replacement=$char;
+	  }
+          $text =~ s/\\$tex/$replacement/g;
+        }
+    }
+    $textref=\$text;
+}
+
+
+# modules must return true
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/MACROPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/MACROPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/MACROPlug.pm	(revision 21112)
@@ -0,0 +1,536 @@
+###########################################################################
+#
+# MACROPlug plugin - to process .dm files for language translation
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package MACROPlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @MACROPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "MACROPlug",
+		'desc'     => "{MACROPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub load_language_table
+{
+    my $lang_table = {};
+
+    my $lang_fname = util::filename_cat($ENV{'GSDLHOME'},"tmp","lang",
+					"package_forms","languages.log");
+    open (LANGFILE, "<$lang_fname") 
+	|| die ("Unable to open $lang_fname: $!\n");
+
+    my $full_name;
+    my $abbr_name;
+
+    while (defined ($full_name=<LANGFILE>)) {
+	chomp($full_name);
+	
+	$abbr_name = <LANGFILE>;
+	chomp($abbr_name);
+
+	$lang_table->{$full_name} = $abbr_name;
+
+	my $fourchar_name = substr($full_name,0,4);
+	if (!defined $lang_table->{$fourchar_name}) {
+	    $lang_table->{$fourchar_name} = $abbr_name;
+	}
+	else {
+	    print STDERR "Warning: Clash on four character abbreviation for language $fourchar_name\n";
+	}
+    }
+
+    close LANGFILE;
+
+    return $lang_table;
+}
+
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    # $self->{'lang_abbr'} = load_language_table();
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.dm$^;
+}
+
+
+sub read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $lang_table = $self->{'lang_abbr'};
+    my $fn = $file;
+    $fn =~ s/.*\/(.*)\..*/$1/;
+    $fn =~ s/\d+$//; # remove any digits from end of filename
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ((!-d $filename) && ($file !~ m/doc.xml$/) && (!defined $lang_table->{$fn})) {
+	print $outhandle "MACROPlug: blocking $file\n" 
+	    if $self->{'verbosity'} > 2;
+	$self->{'num_blocked'} ++;
+	return 0;
+    }
+
+    return $self->SUPER::read(@_);
+}
+
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print STDERR "<Processing n='$file' p='MACROPlug'>\n" if ($gli);
+    print $outhandle "MACROPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+   
+    my $thissection = $doc_obj->get_top_section();
+
+    $self->extract_macronames (\$textref, $doc_obj, $thissection, $file);
+    # we need to escape the escape character, or else mg will convert into
+    # eg literal newlines, instead of leaving the text as '\n'
+    
+    
+    my ($filemeta) = $file =~ /([^\\\/]+)$/;
+    $doc_obj->add_utf8_metadata($thissection, "Title", &ghtml::dmsafe($filemeta));
+    
+    # FileFormat metadata
+    $doc_obj->add_metadata($thissection, "FileFormat", "MACRO");
+    $doc_obj->add_metadata($thissection, "FileSize", (-s $file));
+
+    return 1;
+}
+
+
+sub extract_macronames {
+
+    my $self = shift(@_);
+
+    my ($textref, $doc_obj, $thissection, $file) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print $outhandle " extracting macronames ...\n"
+	if ($self->{'verbosity'}>3);
+    
+    my @textarray = split ("\n", $$$textref)
+	if ($self->{'verbosity'}>3);
+
+    my $macro_text = "";
+    my $image_macro = "false";
+    my $norm_macro = "false";
+    my $currpackage = "";
+    
+    #print STDERR "FILE@@@@@ $file\n";
+
+    #foreach my $ta (@textarray) {
+	
+    for (my $k = 0; $k < scalar(@textarray); $k++) {
+	
+	my $ta = $textarray[$k];
+
+        #print STDERR "$ta\n" if ($file =~ m/port/);
+
+	if ($ta =~ m/^package /) {
+	    $currpackage = $ta;
+	    chomp($currpackage);
+	    $currpackage =~ s/^package //;
+	}
+	
+	
+	elsif ($image_macro eq "true") {
+	    unless ($ta =~ m/\S+/) {
+		$image_macro = "false";
+		
+		$macro_text =~ s/\_/\\_/osg; # for dm (we have a convention of starting macros with _
+	    
+		#print STDERR "$macro_text\n\n" if ($file =~ m/spanish/);
+
+		$doc_obj->add_utf8_text($thissection, $macro_text);
+
+		my @names = split(/\s*\#\#\s*/, $macro_text);
+		
+		my ($title) = $names[1];
+		
+		if (length($title) > 100) {
+		    $title = substr ($title, 0, 100) . "...";
+		}
+
+		$title =~ s/</\&lt;/g;
+		$title =~ s/>/\&gt;/g;
+		
+		#print STDERR "$title\n" if ($file =~ m/port/);
+
+		$doc_obj->add_utf8_metadata ($thissection, "SecTitle", $title);
+		$macro_text = "";
+	    }
+	    $ta .= "<br>";
+	    $macro_text .= $ta;
+	    
+	}
+
+	elsif ($norm_macro eq "true") {
+
+	    $macro_text .= $ta;
+
+	    if ($ta =~ m/\}\s*(\#.*)*\Z/) {
+		
+		$norm_macro = "false";
+		
+		$macro_text =~ s/\A(_\w+_)\s//;
+		
+		#print STDERR "$macro_text\n" if($file =~ m/spanish/i);
+
+		$doc_obj->add_utf8_text($thissection, $macro_text);
+		
+		my ($title) = $macro_text;
+		
+		$title =~ s/\_/\\_/osg;
+		$title =~ s/\{//;
+		$title =~ s/\}\s*(\#.*)*\Z//;
+		$title =~ s/\<br\>|\<p\>/\n/osg;
+		$title =~ s/\n/ /osg;
+		$title =~ s/</\&lt;/g;
+		$title =~ s/>/\&gt;/g;
+
+		$title =~ s/\(/\\\(/g;
+		$title =~ s/\)/\\\)/g;
+		
+		if (length($title) > 100) {
+		    $title = substr ($title, 0, 100) . "...";
+		}
+		#print STDERR "$title\n\n\n" if($file =~ m/spanish/i);
+
+		$doc_obj->add_utf8_metadata ($thissection, "SecTitle", $title);
+		$macro_text = "";
+	    }
+	    
+	}
+	elsif(($ta =~ m/\A(_\w+_)\s*/) && ($image_macro eq "false")) {
+	    
+	    my $new_macro = $1;
+	    
+	    my $new_cursection = $doc_obj->get_next_child($thissection);
+	    $thissection = $doc_obj->insert_section("");
+	    
+	    $new_macro =~ s/\_/\\_/osg;
+	    
+	    $new_macro = $currpackage . "::" . $new_macro;
+	    
+	    #print STDERR "$new_macro\n" if ($file =~ m/spanish/);
+
+	    $doc_obj->add_utf8_metadata ($thissection, "Macroname", $new_macro);
+	    
+	    $macro_text = $ta;
+	    
+	    if ($ta =~ m/\}\s*(\#.*)*\Z/) {
+		
+		$macro_text =~ s/\A(_\w+_)\s//;
+		$macro_text =~ s/\n/ /osg;
+		
+		#print STDERR "$macro_text\n" if($file =~ m/spanish/i);
+
+		$doc_obj->add_utf8_text($thissection, $macro_text);
+		
+		my ($title) = $macro_text;
+		$title =~ s/\{//;
+		$title =~ s/\}\s*(\#.*)*\Z//;
+		$title =~ s/\<br\>|\<p\>/\n/osg;
+		$title =~ s/\n/ /osg;
+		$title =~ s/</\&lt;/g;
+		$title =~ s/>/\&gt;/g;
+		
+		if (length($title) > 100) {
+		    $title = substr ($title, 0, 100) . "...";
+		}
+		#print STDERR "$title\n\n\n" if($file =~ m/spanish/i);
+
+		$doc_obj->add_utf8_metadata ($thissection, "SecTitle", $title);
+	    }
+	    else {
+		$norm_macro = "true";
+	    }
+	
+	    $macro_text = "";
+	    $macro_text .= $ta;
+	}
+	
+	#line in format ## "sometext" ## macro ## macroname ##
+	elsif ($ta =~ m/^\#\# .*/) {
+	    my $macroname = $ta;
+	    $image_macro = "true";
+	    $macro_text = "";
+
+	    #NEED S.T LIKE THIS TO PICK UP HOARY ONES OVER 2 LINES
+	    unless ($macroname =~ m/^\#\# .*\#\#/) {		
+		$macroname = $ta;
+		chomp($macroname);
+		$macro_text .= $macroname;
+		$ta .= $textarray[++$k];
+		$macroname .= $ta;
+	    }
+
+	    my @names = split(/\s*\#\#\s*/, $macroname);
+	    $macroname = $names[(scalar @names) - 1];
+	    my $key = $currpackage . "::" . $macroname; # key to the hash and the database
+
+	    my $new_cursection = $doc_obj->get_next_child($thissection);
+	    $thissection = $doc_obj->insert_section("");
+	    
+	    #print STDERR "$key \n" if ($file =~ m/(port)|(chin)|(engl)/);
+
+	    $doc_obj->add_utf8_metadata ($thissection, "Macroname", $key);
+	    
+	    $macro_text = $ta . "<br>";
+	    
+	}
+    }
+   
+    print $outhandle "done extracting macros\n"
+	if ($self->{'verbosity'}>3);
+
+}
+
+
+sub get_language_encoding_old {
+    my $self = shift (@_);
+    my ($filename) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # read in file
+    open (FILE, $filename) || die "BasPlug::get_language_encoding could not open $filename for reading ($!)\n";
+    undef $/;
+    my $text = <FILE>;
+    $/ = "\n";
+    close FILE;
+
+    # remove <title>stuff</title> -- as titles tend often to be in English
+    # for foreign language documents
+    $text =~ s/<title>.*?<\/title>//i;
+
+    # remove all HTML tags
+    $text =~ s/<[^>]*>//sg;
+    my $results = [];
+
+    # get the language/encoding
+    $results = $self->{'textcat'}->classify(\$text);
+
+    foreach my $r (@$results) {
+	print $outhandle "Results: $r\n";
+	
+    }
+   
+    # if textcat returns 3 or less possibilities we'll use the 
+    # first one in the list - otherwise use the defaults
+    if (scalar @$results > 3) {
+	                            
+        my $lang_fname = util::filename_cat($ENV{'GSDLHOME'},"tmp","lang","package_forms",
+					    "languages.log");
+	open (LANGFILE, "<$lang_fname") or die ("Unable to open $lang_fname: $!\n");
+
+	while (<LANGFILE>) {
+	    
+	    my $line = $_;
+	    chomp($line);
+
+	    my $fn = $filename;
+	    $fn =~ s/.*\/(.*)\..*/$1/;
+
+
+	    if ($line eq $fn) {
+		print $outhandle "BINGO $line $fn\n";
+		$line = <LANGFILE>;
+		chomp($line);
+		print $outhandle "language code is $line\n";
+
+		foreach my $r (@$results) {
+		    print $outhandle "MY1 $r\n";
+
+		    my $roar = $& if($r =~ m/../);
+
+		    if ($roar eq $line) {
+			print $outhandle "WE HAVE A WINNER $r\n";
+		    }
+		}
+
+
+	    }
+	    else {
+		$line = <LANGFILE>;
+	    }
+	}
+
+	close LANGFILE;
+
+	if ($self->{'input_encoding'} ne 'auto') {
+	    if ($self->{'extract_language'} && $self->{'verbosity'}) {
+		print $outhandle "BasPlug: WARNING: language could not be extracted from $filename - ";
+		print $outhandle "defaulting to $self->{'default_language'}\n";
+	    }		
+	    return ($self->{'default_language'}, $self->{'input_encoding'});
+
+	} else {
+	    if ($self->{'verbosity'}) {
+		print $outhandle "BASPlug: WARNING: language/encoding could not be extracted from $filename - ";
+		print $outhandle "defaulting to $self->{'default_language'}/$self->{'default_encoding'}\n";
+	    }
+	    return ($self->{'default_language'}, $self->{'default_encoding'});
+	}
+    }
+    
+
+    # format language/encoding
+    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
+    if (!defined $language) {
+	if ($self->{'verbosity'}) {
+	    print $outhandle "BasPlug: WARNING: language could not be extracted from $filename - ";
+	    print $outhandle "defaulting to $self->{'default_language'}\n";
+	}
+	$language = $self->{'default_language'};
+    }
+    if (!defined $encoding) {
+	if ($self->{'verbosity'}) {
+	    print $outhandle "BasPlug: WARNING: encoding could not be extracted from $filename - ";
+	    print $outhandle "defaulting to $self->{'default_encoding'}\n";
+	}
+	$encoding = $self->{'default_encoding'};
+    }
+
+    if ($encoding !~ /^(ascii|utf8|unicode)$/ &&
+	!defined $encodings::encodings->{$encoding}) {
+	if ($self->{'verbosity'}) {
+	    print $outhandle "BasPlug: WARNING: $filename appears to be encoded in an unsupported encoding ($encoding) - ";
+	    print $outhandle "using $self->{'default_encoding'}\n";
+	}
+	$encoding = $self->{'default_encoding'};
+    }
+
+    print STDERR "**** forcing encoding to be utf8\n";
+    $encoding = "utf8";
+
+    print STDERR "**** forcing language to be first two letters\n";
+    my $lfname = $filename;
+    $lfname =~ s/^.*\///;
+    $language = substr($lfname,0,2);
+
+    print $outhandle "RETURNING VALUES $language $encoding\n";
+
+    return ($language, $encoding);
+}
+
+
+sub find_language {
+    my ($self,$fn) = @_;
+
+    my $lang_table = $self->{'lang_abbr'};
+
+    if (!defined $lang_table->{$fn}) {
+
+	# try and find it with shorter string name
+	
+	my $try_len = length($fn);
+
+	while ($try_len>=4) {
+	    my $try_fn = substr($fn,0,$try_len);
+
+	    if (defined $lang_table->{$try_fn}) {
+		$fn = $try_fn;
+		last;
+	    }
+	    $try_len--;
+	}
+    }
+
+    return $fn;
+}
+
+
+sub get_language_encoding {
+    my $self = shift (@_);
+    my ($filename) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $fn = $filename;
+    $fn =~ s/.*\/(.*)\..*/$1/;
+    $fn =~ s/\d+$//; # remove any digits from end of filename
+
+    my $language;
+    my $encoding = "utf8";
+
+    ## my $lang_lookup = $self->find_language($fn);
+
+    my $lang_table = $self->{'lang_abbr'};
+
+    if (!defined $lang_table->{$fn}) {
+
+	print $outhandle "Warning: Macro file name $filename not in list of languages.\n";
+	print $outhandle "         Using default language.\n";
+	$language = $self->{'default_language'};
+    }
+    else {
+	$language = $lang_table->{$fn};
+    }
+
+    ## print $outhandle "Storing $filename as $language $encoding\n";
+
+    return ($language, $encoding);
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/MARCPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/MARCPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/MARCPlug.pm	(revision 21112)
@@ -0,0 +1,311 @@
+###########################################################################
+#
+# MARCPlug.pm -- basic MARC plugin
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2002 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package MARCPlug;
+
+use SplitPlug;
+
+use unicode;
+use util;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @MARCPlug::ISA = ('SplitPlug');
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+my $arguments = 
+    [ { 'name' => "metadata_mapping",
+	'desc' => "{MARCPlug.metadata_mapping}",
+	'type' => "string",
+	'deft' => "marctodc.txt",
+	'reqd' => "no" },
+      { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_split_exp() } 
+      ];
+
+my $options = { 'name'     => "MARCPlug",
+		'desc'     => "{MARCPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+		'explodes' => "yes",
+		'args'     => $arguments };
+
+require MARC::Record;  
+require MARC::Batch;  
+#use MARC::Record;  
+#use MARC::Batch;
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+    my ($verbosity, $outhandle, $failhandle) = @_;
+
+    my @metadata_mapping = ();
+
+    # read in the metadata mapping file
+    my $mm_file =
+	&util::filename_cat( $ENV{'GSDLHOME'}, "etc", $self->{'metadata_mapping'} );
+
+    if (!-e $mm_file)
+    {
+
+	my $msg = "MARCPlug ERROR: Can't locate mapping file \"" .
+	    $self->{'metadata_mapping'} . "\".\n    This file should be at $mm_file\n" .
+		"    No marc files can be processed.\n";
+
+	print $outhandle $msg;
+	print $failhandle $msg;
+	$self->{'metadata_mapping'} = undef;
+	# We pick up the error in process() if there is no $mm_file
+	# If we exit here, then pluginfo.pl will exit too!
+    }
+    elsif (open(MMIN, "<$mm_file"))
+    {
+	my $l=1;
+	my $line;
+	while (defined($line=<MMIN>))
+	{
+	    chomp $line;
+	    if ($line =~ m/^(\d+)\s*->\s*(\w+)$/)
+	    {
+		my $marc_info = $1;
+		my $gsdl_info = $2;
+		my $mapping = { 'marc' => $marc_info, 'gsdl' => $gsdl_info };
+		push(@metadata_mapping,$mapping);
+	    }
+	    elsif ($line !~ m/^\#/       # allow comments (# in first column)
+		   && $line !~ m/^\s*$/) # allow blank lines
+	    {
+		print $outhandle "Parse error on line $l of $mm_file:\n";
+		print $outhandle "  \"$line\"\n";
+	    }
+	    $l++
+	}
+	close(MMIN);
+    }
+    else
+    {
+	print STDERR "Unable to open $mm_file: $!\n";
+    }
+
+    $self->{'metadata_mapping'} = \@metadata_mapping;
+
+    $self->SUPER::init(@_);
+}
+    
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)(\.marc)$^;
+}
+
+
+sub get_default_split_exp {
+    # \r\n for msdos eol, \n for unix
+    return q^\r?\n\s*\r?\n^;
+}
+
+
+
+# The bulk of this function is based on read_line in multiread.pm
+# Unable to use read_line original because it expects to get its input
+# from a file.  Here the line to be converted is passed in as a string
+
+sub to_utf8
+{
+    my $self = shift (@_);
+    my ($encoding, $line) = @_;
+
+    if ($encoding eq "iso_8859_1") {
+	# we'll use ascii2utf8() for this as it's faster than going
+	# through convert2unicode()
+	return &unicode::ascii2utf8 (\$line);
+    }
+
+    # everything else uses unicode::convert2unicode
+    return &unicode::unicode2utf8 (&unicode::convert2unicode ($encoding, \$line));
+}
+
+
+sub read_file {
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    $self->{'readfile_encoding'}->{$filename} = $encoding;
+
+    my @marc_entries = ();
+
+    if (!-r $filename)
+    {
+	my $outhandle = $self->{'outhandle'};
+	print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
+	return;
+    }
+
+    my $batch = new MARC::Batch( 'USMARC', $filename );
+    while ( my $marc = $batch->next ) 
+    {
+	push(@marc_entries,$marc);
+	$$textref .= $marc->as_formatted();
+	$$textref .= "\n\n"; # for SplitPlug - see default_split_exp above...
+    }
+
+    $self->{'marc_entries'}->{$filename} = \@marc_entries;
+}
+
+
+
+# do plugin specific processing of doc_obj
+# This gets done for each record found by SplitPlug in marc files.
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    if (! defined($self->{'metadata_mapping'}))
+    {
+	print $outhandle "MARCPlug: no metadata file! Can't process $file\n";
+	return undef;
+    }
+
+    print STDERR "<Processing n='$file' p='MARCPlug'>\n" if ($gli);
+    print $outhandle "MARCPlug: processing $file\n"
+	if $self->{'verbosity'} > 1;
+
+    my $cursection = $doc_obj->get_top_section();
+
+    # Add fileFormat as the metadata
+    $doc_obj->add_metadata($cursection, "FileFormat", "MARC");
+    
+    my $marc_entries = $self->{'marc_entries'}->{$filename};
+    my $marc = shift(@$marc_entries);
+
+    my $encoding = $self->{'readfile_encoding'}->{$filename};
+    $self->extract_metadata ($marc, $metadata, $encoding, $doc_obj, $cursection);
+
+    # add spaces after the sub-field markers, for word boundaries
+    $$textref =~ s/^(.{6} _\w)/$1 /gm;
+
+    # add text to document object
+    $$textref =~ s/</&lt;/g;
+    $$textref =~ s/>/&gt;/g;
+
+    $$textref = $self->to_utf8($encoding,$$textref);
+
+    print $outhandle "  Adding Marc Record:\n",substr($$textref,0,40), " ...\n"
+	if $self->{'verbosity'} > 2;
+
+    # line wrapping
+    $$textref = &wrap_text_in_columns($$textref, 64);
+    $$textref = "<pre>\n" . $$textref . "</pre>\n"; # HTML formatting...
+
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    return 1;
+}
+
+sub wrap_text_in_columns
+{
+    my ($text, $columnwidth) = @_;
+    my $newtext = "";
+    my $linelength = 0;
+    
+    # Break the text into words, and display one at a time
+    my @words = split(/ /, $text);
+
+    foreach my $word (@words) {
+	# If printing this word would exceed the column end, start a new line
+	if (($linelength + length($word)) >= $columnwidth) {
+	    $newtext .= "\n";
+	    $linelength = 0;
+	}
+	
+	# Write the word
+	$newtext .= " $word";
+	if ($word =~ /\n/) {
+	    $linelength = 0;
+	} else {
+	    $linelength = $linelength + length(" $word");
+	}
+    }
+
+    $newtext .= "\n";
+    return $newtext;
+}
+
+
+sub extract_metadata 
+{
+    my $self = shift (@_);
+    my ($marc, $metadata, $encoding, $doc_obj, $section) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $metadata_mapping = $self->{'metadata_mapping'};
+    my $mm;
+    foreach $mm ( @$metadata_mapping )
+    {
+	my $marc_field = $mm->{'marc'};
+	my @metavalues = $marc->field($marc_field);
+
+	if (scalar(@metavalues)>0)
+	{
+	    my $metaname = $mm->{'gsdl'};
+	    my $metavalue;
+	    foreach $metavalue ( @metavalues )
+	    {
+		my $metavalue_str = $self->to_utf8($encoding,$metavalue->as_string());
+		$doc_obj->add_utf8_metadata ($section, $metaname, $metavalue_str);
+	    }
+	}
+    }
+}
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/METSPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/METSPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/METSPlug.pm	(revision 21112)
@@ -0,0 +1,320 @@
+###########################################################################
+#
+# METSPlug.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Processes GreenstoneArchive METS documents. Note that this plugin does no
+# syntax checking (though the XML::Parser module tests for
+# well-formedness). It's assumed that the GreenstoneArchive files conform
+# to their DTD.
+
+# 28/07/04 Modified to METSPlug - CHI-YU HUANG
+
+package METSPlug;
+
+use ghtml;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use XMLPlug;
+use XML::XPath;
+use XML::XPath::XMLParser;
+
+sub BEGIN {
+    @METSPlug::ISA = ('XMLPlug');
+}
+
+my $arguments = [
+		 ];
+my $options = { 'name'     => "METSPlug",
+		'desc'     => "{METSPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)docmets\.xml$^;
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new XMLPlug($pluginlist,$inputargs,$hashArgOptLists): new XMLPlug($pluginlist,$inputargs);
+
+    $self->{'section'} = "";
+    $self->{'section_level'} = 0;
+    $self->{'metadata_name'} = "";
+    $self->{'metadata_value'} = "";
+    $self->{'content'} = "";
+
+    return bless $self, $class;
+}
+
+sub xml_start_document {
+    my $self = shift (@_);
+    my ($expat, $element) =  @_;
+
+    $self->{'section'} = "";
+    $self->{'section_level'} = 0;
+    $self->{'metadata_name'} = "";
+    $self->{'metadata_value'} = "";
+    $self->{'content'} = "";
+
+    #**defined a dmdSection Table
+    $self->{'dmdSec_table'}={};
+
+    #**defined a fileSection Table
+    $self->{'fileSec_table'}={};
+
+    #***open doctxt.xml and read the data in 
+    my $filename = $self->{'filename'};
+    
+    $filename =~ s/docmets.xml$/doctxt.xml/;
+   
+    if (!open (FILEIN,"<$filename")){
+   	print STDERR "Warning: unable to open the $filename\n";
+	$self->{'xmltxt'} = "";
+    }
+    else {
+	my $xml_text = "";
+	while (defined (my $line = <FILEIN>)) {
+	    if ($line !~ m/^<!DOCTYPE.*>/) {
+		$xml_text .= $line;
+	    }
+	}
+        my $xml_parser = XML::XPath->new (xml=> $xml_text);
+	#my $xml_tree = $xml_parser->parse ($xml_text);
+
+	#eval {$self->{'parser_text'}->parse};
+	$self->{'parsed_xml'} = $xml_parser;
+    }
+    my $outhandle = $self->{'outhandle'};
+    print $outhandle "METSPlug: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
+    print STDERR "<Processing n='$self->{'file'}' p='METSPlug'>\n" if ($self->{'gli'});
+
+}
+
+sub xml_end_document {
+}
+
+sub xml_doctype {
+}
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    $self->{'element'} = $element;
+
+    #**deal with dmdSection
+    if ($element eq "mets:dmdSec" || $element eq "gsdl3:Metadata"){    
+	$self->xml_dmd_start_tag (@_);
+    } elsif ($element eq "mets:file") {
+	$_{'ID'} =~ m/FILE(.*)/;
+	$self->{'file_Id'} = $1;
+    } elsif ($element eq "mets:FLocat"){
+	#***deal with fileSection
+	$self->xml_fileloc_start_tag (@_);
+    } elsif ($element eq "mets:div"){
+	#***deal with StrucMap Section
+	$self->xml_strucMap_start_tag (@_);
+    }
+}
+
+sub xml_dmd_start_tag {
+    my $self = shift (@_);
+    my ($expat, $element) = @_;
+
+    if ($element eq "mets:dmdSec"){
+  	my ($section_num) = ($_{'ID'} =~ m/DM(.*)/);
+	$self->{'dmdSec_table'}->{"$section_num"}=[];
+	$self->{'dmdSec_table'}->{'section_num'}=$section_num;
+    } elsif ($element eq "gsdl3:Metadata") {
+	$self->{'metadata_name'} = $_{'name'};
+    }
+}
+
+sub xml_fileloc_start_tag {
+    my $self = shift (@_);
+    my ($expat, $element) = @_;
+
+    my $xlink = $_{'xlink:href'};
+    #my ($section_num) = ($_{'ID'} =~ m/^FLOCAT(.*)$/);
+    my $section_num = $self->{'file_Id'};
+
+    return if (!defined $section_num); 
+    #**return if the section_num is not defined or not deal with the whole section (ID="default.*")
+    
+    $self->{'fileSec_table'}->{"$section_num"}=[];
+    $self->{'fileSec_table'}->{'section_num'}=$section_num;
+    
+    my ($filename,$xpath_expr)=($xlink =~ m/^file:(.*)\#xpointer\((.*)\)$/);
+    
+    my $nodeset = $self->{'parsed_xml'}->findnodes ($xpath_expr);
+    my $node_size= $nodeset->size;
+    
+    if ($node_size==0) {
+	print STDERR "Warning: no text associated with XPATH $xpath_expr\n";
+    }
+    else {
+	foreach my $node ($nodeset->get_nodelist) {
+	    my $xml_content = XML::XPath::XMLParser::as_string($node);
+	    my $unescaped_xml_content = &ghtml::unescape_html($xml_content);
+
+	    my $section_content={'section_content'=> $unescaped_xml_content};
+	    
+	    my $content_list = $self->{'fileSec_table'}->{"$section_num"};
+	    push (@$content_list, $section_content);
+	}
+    }
+}
+
+sub xml_strucMap_start_tag {
+    my $self = shift (@_);
+    my ($expat, $element) = @_;
+
+
+    my ($section_num) = ($_{'ID'} =~ m/DS(.*)/);
+
+    if ($_{'ID'} ne "DSAll"){
+	if ($self->{'section_level'}==0) {
+	    $self->open_document();
+	} else {
+	    my $doc_obj = $self->{'doc_obj'};
+	    $self->{'section'}=
+		$doc_obj->insert_section($doc_obj->get_end_child($self->{'section'}));
+	}
+	$self->{'section_level'}++;
+	
+	#***Add metadata from dmdSection
+	my $md_list = $self->{'dmdSec_table'}->{"$section_num"};
+	
+	foreach my $md_pair (@$md_list){
+	    my $metadata_name = $md_pair->{'metadata_name'};
+	    my $metadata_value = $md_pair->{'metadata_value'};
+	    $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, $metadata_name, $metadata_value);
+	}
+	
+	#*** Add content from fileSection
+	my $content_list = $self->{'fileSec_table'}->{"$section_num"};
+	
+	foreach my $section_content (@$content_list){
+	    my $content = $section_content->{'section_content'};
+	    $self->{'doc_obj'}->add_utf8_text($self->{'section'},$content);
+	}
+    }
+}
+
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    if ($element eq "gsdl3:Metadata") {
+	my $section_num = $self->{'dmdSec_table'}->{'section_num'};
+	my $metadata_name=$self->{'metadata_name'};
+	my $metadata_value=$self->{'metadata_value'};
+
+	my $md_pair={'metadata_name' => $metadata_name,
+		     'metadata_value'=> $metadata_value};
+
+	my $md_list = $self->{'dmdSec_table'}->{"$section_num"};
+
+	push(@$md_list,$md_pair);
+      
+	$self->{'metadata_name'} = "";
+	$self->{'metadata_value'} = "";
+    } elsif ($element eq "mets:file"){
+	$self->{'file_id'} = "";
+    }
+    
+    
+    #*** StrucMap Section
+    if ($element eq "mets:div") {
+	$self->{'section_level'}--;
+	$self->{'section'} = $self->{'doc_obj'}->get_parent_section($self->{'section'});
+	$self->close_document() if $self->{'section_level'}==0;
+    }
+    $self->{'element'} = "";
+}
+
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_; 
+
+    if ($self->{'element'} eq "gsdl3:Metadata") {
+   	$self->{'metadata_value'} .= $_;
+    }
+}
+
+sub open_document {
+    my $self = shift(@_);
+
+    # create a new document
+    $self->{'doc_obj'} = new doc ();
+    $self->{'section'} = "";
+}
+
+sub close_document {
+    my $self = shift(@_);
+    
+    # add the associated files
+    my $assoc_files = 
+	$self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    # for when "assocfilepath" isn't the same directory that doc.xml is in...
+    my $assoc_filepath_list= $self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "assocfilepath");
+
+    my $assoc_filepath=shift (@$assoc_filepath_list);
+    if (defined ($assoc_filepath)) {
+	# make absolute rather than relative...
+	$self->{'filename'} =~ m@^(.*[\\/]archives)@;
+	$assoc_filepath = "$1/$assoc_filepath/";
+    } else {
+	$assoc_filepath = $self->{'filename'};
+	$assoc_filepath =~ s/[^\\\/]*$//;
+    }
+
+    foreach my $assoc_file_info (@$assoc_files) {
+	my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
+	my $real_dir = &util::filename_cat($assoc_filepath, $assoc_file),
+	my $assoc_dir = (defined $dir && $dir ne "") 
+	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
+	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
+    }
+    $self->{'doc_obj'}->delete_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    # process the document
+    $self->{'processor'}->process($self->{'doc_obj'}, $self->{'file'});
+}
+
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/plugins/MP3Plug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/MP3Plug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/MP3Plug.pm	(revision 21112)
@@ -0,0 +1,396 @@
+###########################################################################
+#
+# MP3Plug.pm -- Plugin for MP3 files (MPEG audio layer 3).
+#
+# A component of the Greenstone digital library software from the New
+# Zealand Digital Library Project at the University of Waikato, New
+# Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package MP3Plug;
+
+use UnknownPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use MP3::Info;
+
+require giget;
+
+sub BEGIN {
+    @MP3Plug::ISA = ('UnknownPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "assoc_images",
+        'desc' => "{MP3Plug.assoc_images}",
+        'type' => "flag",
+        'deft' => "",
+        'reqd' => "no" },
+      { 'name' => "applet_metadata",
+	'desc' => "{MP3Plug.applet_metadata}",
+	'type' => "flag",
+	'deft' => "" },
+      { 'name' => "metadata_fields",
+	'desc' => "{MP3Plug.metadata_fields}",
+	'type' => "string",
+	'deft' => "Title,Artist,Genre" } ];
+
+my $options = { 'name'     => "MP3Plug",
+		'desc'     => "{MP3Plug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new UnknownPlug($pluginlist,$inputargs,$hashArgOptLists): new UnknownPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    return q^(?i)\.mp3$^;
+}
+
+sub gen_mp3applet {
+
+    my ($mp3_filename) = @_;
+
+    my $applet_html = '
+<OBJECT classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
+        WIDTH = "59"
+        HEIGHT = "32"
+        codebase="http://java.sun.com/products/plugin/1.3/jinstall-13-win32.cab#Version=1,3,0,0">
+<PARAM NAME = CODE VALUE = "javazoom.jlGui.TinyPlayer" >
+<PARAM NAME = ARCHIVE VALUE = "_httpcollection_/tinyplayer.jar,_httpcollection_/jl10.jar" >
+<PARAM NAME="type" VALUE="application/x-java-applet;version=1.3">
+<PARAM NAME="scriptable" VALUE="false">
+<PARAM NAME = "skin" VALUE ="_httpcollection_/skins/Digitalized">
+<PARAM NAME = "autoplay" VALUE ="yes">
+<PARAM NAME = "bgcolor" VALUE ="638182">
+<PARAM NAME = "audioURL" VALUE ="MP3FILENAME">
+<COMMENT>
+<EMBED type="application/x-java-applet;version=1.3"
+       CODE = "javazoom.jlGui.TinyPlayer"
+       ARCHIVE = "_httpcollection_/tinyplayer.jar,_httpcollection_/jl10.jar"
+       WIDTH = "59"
+       HEIGHT = "32"
+       skin = "_httpcollection_/skins/Digitalized"
+       autoplay = "yes"
+       bgcolor = "638182"
+       audioURL = "MP3FILENAME"
+       scriptable=false
+       pluginspage="http://java.sun.com/products/plugin/1.3/plugin-install.html">
+<NOEMBED>
+</COMMENT>
+</NOEMBED></EMBED>
+</OBJECT>
+';
+
+    $applet_html =~ s/MP3FILENAME/$mp3_filename/g;
+
+    return $applet_html;
+}
+
+
+
+# Associate the mp3 file with the new document
+
+sub associate_mp3_file {
+    my $self = shift (@_);
+    my $filename = shift (@_);   # filename with full path
+    my $file = shift (@_);       # filename without path
+    my $doc_obj = shift (@_);
+    
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+
+    # check the filename is okay
+    return 0 if ($file eq "" || $filename eq "");
+
+    # Add the file metadata
+    my $assoc_url = $file;
+#    $assoc_url =~ s/ /%20/g; # probably need to do more escaping than this!!
+    $assoc_url =~ s/ /_/g; # workaround for now
+    my $dst_file = $file;
+    $dst_file =~ s/ /_/g;
+
+    # Add the file as an associated file ...
+    my $section = $doc_obj->get_top_section();
+    my $mime_type = $self->{'mime_type'} || "audio/mp3";
+    my $assoc_field = $self->{'assoc_field'} || "mp3";
+    my $assoc_name = $file;
+    $assoc_name =~ s/\.mp3$//;
+
+    $doc_obj->associate_file($filename, $dst_file, $mime_type, $section);
+    $doc_obj->add_metadata ($section, "Source", $file);
+    $doc_obj->add_metadata ($section, $assoc_field, $assoc_name);
+    $doc_obj->add_metadata ($section, "srcurl", $assoc_url);
+
+    my $mp3_info = get_mp3info($filename);
+    my $mp3_tags = get_mp3tag($filename);
+
+    my $metadata_fields = $self->{'metadata_fields'};
+
+    if ($metadata_fields eq "*") {
+	# Locate all info and tag metadata
+
+	foreach my $ki ( keys %$mp3_info ) {
+	    my $mp3_metavalue = $mp3_info->{$ki};
+
+	    if ($mp3_metavalue !~ m/^\s*$/s) {
+		my $mp3_metaname = "mp3:".lc($ki);
+		$doc_obj->add_metadata ($section, $mp3_metaname, $mp3_metavalue);
+	    }
+	}
+
+	foreach my $kt ( keys %$mp3_tags ) {
+	    my $mp3_metavalue = $mp3_tags->{$kt};
+	    
+	    if ($mp3_metavalue !~ m/^\s*$/s) {
+		my $kt_len = length($kt);
+		my $kt_initial_cap = uc(substr($kt,0,1)).lc(substr($kt,1,$kt_len-1));
+		my $mp3_metaname = "mp3:".$kt_initial_cap;
+		
+		$doc_obj->add_metadata ($section, $mp3_metaname, $mp3_metavalue);
+	    }
+	}
+    }
+    else {
+
+	# Restrict metadata to that specifically given
+	foreach my $field (split /,/, $metadata_fields) {
+
+	    # check info
+	    if (defined $mp3_info->{$field}) {
+
+		my $mp3i_metavalue = $mp3_info->{$field};
+		
+		if ($mp3i_metavalue !~ m/^\s*$/s) {
+		    my $mp3i_metaname = "mp3:".lc($field);
+		    $doc_obj->add_metadata ($section, $mp3i_metaname, $mp3i_metavalue);
+		}
+	    }
+
+	    # check tags
+	    if (defined $mp3_tags->{uc($field)}) {
+
+		my $mp3t_metavalue = $mp3_tags->{uc($field)};
+		
+		if ($mp3t_metavalue !~ m/^\s*$/s) {
+		    my $mp3t_metaname = "mp3:".$field;
+		    
+		    $doc_obj->add_metadata ($section, $mp3t_metaname, $mp3t_metavalue);
+		}
+	    }
+	    
+	}
+    }
+
+    $doc_obj->add_metadata ($section, "FileFormat", "MP3");
+    
+    $doc_obj->add_metadata ($section, "srclink", 
+			    "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[srcurl]\">");
+    $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpimg_/mp3icon.gif\">");
+    $doc_obj->add_metadata ($section, "/srclink", "</a>");
+
+    my $applet_metadata = $self->{'applet_metadata'};
+    if (defined $applet_metadata && $applet_metadata ) {
+	my $applet_html 
+	    = gen_mp3applet("_httpcollection_/index/assoc/[assocfilepath]/[srcurl]");
+	$doc_obj->add_metadata ($section, "mp3applet", $applet_html);
+    }
+
+    my $assoc_images = $self->{'assoc_images'};
+    if (defined $assoc_images && $assoc_images) {
+	my @search_terms = ();
+
+	my $title  = $mp3_tags->{'TITLE'};
+	my $artist = $mp3_tags->{'ARTIST'};
+
+	if (defined $title && $title ne "") {
+
+	    push(@search_terms,$title);
+
+	    if (defined $artist && $artist ne "") {
+		push(@search_terms,$artist);
+	    }
+	}
+	else {
+	    push(@search_terms,$assoc_name);
+	}
+
+	push(@search_terms,"song");
+
+	my $output_dir = $filename;
+	$output_dir =~ s/\.\w+$//;
+
+	my ($imgref_urls) = giget(\@search_terms,$output_dir);
+
+	my $gi_base = gi_url_base();
+	my $gi_query_url = gi_query_url(\@search_terms);
+
+	$doc_obj->add_metadata ($section, "giquery", "<a href=\"$gi_base$gi_query_url\" target=giwindow>");
+	$doc_obj->add_metadata ($section, "/giquery", "</a>");
+
+	for (my $i=1; $i<=2; $i++) {
+	    my $img_filename = "$output_dir/img_$i.jpg";
+	    my $dst_file = "img_$i.jpg";
+
+	    if (-e $img_filename) {
+		$doc_obj->associate_file($img_filename, $dst_file, "image/jpeg", $section);
+
+		my $srcurl = "src=\"_httpcollection_/index/assoc/[assocfilepath]/$dst_file\"";
+
+		$doc_obj->add_metadata ($section, "img$i", 
+					"<img $srcurl>");
+		$doc_obj->add_metadata ($section, "smallimg$i", 
+					"<img $srcurl width=100>");
+
+		my $imgref_url = $imgref_urls->[$i-1];
+
+		$doc_obj->add_metadata ($section, "imgref$i", "<a href=\"$imgref_url\" target=giwindow>");
+		$doc_obj->add_metadata ($section, "/imgref$i", "</a>");
+	    }
+
+	}
+
+
+    }
+
+    return 1;
+}
+
+
+
+# The MP3Plug read() function is based on UnknownPlug read().  This
+# function does all the right things to make general options work for
+# a given plugin. 
+
+my $mp3_doc_count = 0; ## is this used anywhere now !!???
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    # Make sure we're processing the correct file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    print STDERR "<Processing n='$file' p='MP3Plug'>\n" if ($gli);
+    print $outhandle "MP3Plug processing \"$filename\"\n"
+	    if $self->{'verbosity'} > 1;
+
+    #if there's a leading directory name, eat it...
+    $file =~ s/^.*[\/\\]//;
+    
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $mp3_doc_count++;
+    
+##    $doc_obj->set_OIDtype ($processor->{'OIDtype'});    
+    $doc_obj->set_OIDtype ("incremental");    # this is done to avoid hashing content of file
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    # associate the file with the document
+    if (associate_mp3_file($self, $filename, $file, $doc_obj) != 1)
+    {
+	print "MP3Plug: couldn't process \"$filename\"\n";
+	return 0;
+    }
+
+    #create an empty text string so we don't break downstream plugins 
+    my $text = &gsprintf::lookup_string("{BasPlug.dummy_text}");
+    $text .= "[img1]<br>";
+    $text .= "[img2]<br>";
+
+    # include any metadata passed in from previous plugins 
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+
+    $self->title_fallback($doc_obj,$section,$file);
+
+    # do plugin specific processing of doc_obj
+    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, 
+						 $file, $metadata, $doc_obj));
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    # add an OID
+    $doc_obj->set_OID();
+    $doc_obj->add_text($section, $text);
+
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'} ++;
+    return 1;
+}
+
+
+sub title_fallback
+{
+    my $self = shift (@_);
+    my ($doc_obj,$section,$file) = @_;
+
+    if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
+	my $mp3_title = $doc_obj->get_metadata_element ($section, "mp3:Title");
+	if (defined $mp3_title) {
+	    $doc_obj->add_metadata ($section, "Title", $mp3_title);
+	}
+	else {
+	    &BasPlug::title_fallback($self, $doc_obj, $section, $file);
+	}
+    }
+}
+
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/MetadataPass.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/MetadataPass.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/MetadataPass.pm	(revision 21112)
@@ -0,0 +1,158 @@
+###########################################################################
+#
+# MetadataPass.pm -- class to enhance plugins with generic metadata capabilities
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package MetadataPass;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use BasPlug; # uses BasPlug, but is not inherited
+
+
+my $options = { 'name'     => "MetadataPass",
+		'desc'     => "{MetadataPass.desc}",
+		'abstract' => "yes",
+		'inherits' => "no" };
+
+
+sub new {
+    my $class = shift (@_);
+    my $plugin_name = shift (@_);
+
+    my $self = {};
+    $self->{'plugin_type'} = "MetadataPass";
+
+    $self->{'option_list'} = [ $options ];
+
+    return bless $self, $class;
+}
+
+sub init {
+}
+
+sub print_xml_usage
+{
+    BasPlug::print_xml_usage(@_);
+}
+
+sub print_xml
+{
+    BasPlug::print_xml(@_);
+}
+
+sub set_keepold
+{
+    BasPlug::set_keepold(@_);
+}
+
+sub reset_saved_metadata {
+    my $self = shift(@_);
+
+    $self->{'saved_metadata'} = {};
+}
+
+
+sub get_saved_metadata {
+    my $self = shift(@_);
+    
+    return $self->{'saved_metadata'};
+}
+
+sub open_prettyprint_metadata_table
+{
+    my $self = shift(@_);
+
+    my $att   = "width=100% cellspacing=2";
+    my $style = "style=\'border-bottom: 4px solid #000080\'";
+
+    $self->{'ppmd_table'} = "\n<table $att $style>";
+}
+
+sub add_prettyprint_metadata_line 
+{
+    my $self = shift(@_);
+    my ($metaname, $metavalue_utf8) = @_;
+
+    $metavalue_utf8 = &util::hyperlink_text($metavalue_utf8);
+
+    $self->{'ppmd_table'} .= "  <tr bgcolor=#b5d3cd>\n";
+    $self->{'ppmd_table'} .= "    <td width=30%>\n";
+    $self->{'ppmd_table'} .= "      $metaname\n";
+    $self->{'ppmd_table'} .= "    </td>\n";
+    $self->{'ppmd_table'} .= "    <td>\n";
+    $self->{'ppmd_table'} .= "      $metavalue_utf8\n";
+    $self->{'ppmd_table'} .= "    </td>\n";
+    $self->{'ppmd_table'} .= "  </tr>\n";
+
+}
+
+sub close_prettyprint_metadata_table
+{
+    my $self = shift(@_);
+    $self->{'ppmd_table'} .= "</table>\n";
+
+    $self->set_filere_metadata("prettymd",$self->{'ppmd_table'});
+    $self->{'ppmd_table'} = undef;
+}
+
+sub set_filere_metadata
+{
+    my $self = shift(@_);
+    my ($full_mname,$md_content) = @_;
+
+    if (defined $self->{'saved_metadata'}->{$full_mname}) {
+	# accumulate - add value to existing value(s)
+	if (ref ($self->{'saved_metadata'}->{$full_mname}) eq "ARRAY") {
+	    push (@{$self->{'saved_metadata'}->{$full_mname}}, $md_content);
+	} else {
+	    $self->{'saved_metadata'}->{$full_mname} = 
+		[$self->{'saved_metadata'}->{$full_mname}, $md_content];
+	}
+    } else {
+	# accumulate - add value into (currently empty) array
+	$self->{'saved_metadata'}->{$full_mname} = [$md_content];
+    }
+}
+
+
+sub get_filere_metadata
+{
+    my $self = shift(@_);
+    my ($full_mname) = @_;
+
+    return $self->{'saved_metadata'}->{$full_mname};
+}
+
+sub get_filere_metadata_head
+{
+    my $self = shift(@_);
+    my ($full_mname) = @_;
+    
+    return $self->{'saved_metadata'}->{$full_mname}->[0];
+}
+
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/NULPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/NULPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/NULPlug.pm	(revision 21112)
@@ -0,0 +1,165 @@
+###########################################################################
+#
+# NULPlug.pm -- Plugin for dummy (.nul) files
+#
+# A component of the Greenstone digital library software from the New
+# Zealand Digital Library Project at the University of Waikato, New
+# Zealand.
+#
+# Copyright (C) 2005 Katherine Don
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# NULPlug - a plugin for dummy files
+
+# This is a simple Plugin for importing dummy files, along with
+# their metadata.  A fictional document will
+# be created for every such file, and the metadata added to it.
+
+# This is used mainly for the null files resulting from exploding metadata
+# databases
+
+package NULPlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @NULPlug::ISA = ('BasPlug');
+}
+
+my $arguments = [
+		 ];
+
+my $options = { 'name'     => "NULPlug",
+		'desc'     => "{NULPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes" };
+	#	'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    return '(?i)\.nul$';
+}
+
+# The NULPlug read() function. This function does all the right
+# things to make general options work for a given plugin.  NULPlug
+# overrides read() because there is no need to read the actual text of
+# the file in, because the contents of the file is not text...
+#
+#
+# Return number of files processed, undef if can't process 
+#
+# Note that $base_dir might be "" and that $file might include directories
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    # Make sure we're processing the correct file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    print STDERR "<Processing n='$file' p='NULPlug'>\n" if ($gli);
+    print $outhandle "NULPlug processing \"$filename\"\n"
+	    if $self->{'verbosity'} > 1;
+
+    #if there's a leading directory name, eat it...
+    $file =~ s/^.*[\/\\]//;
+    
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});    
+    #$doc_obj->set_OIDtype ("incremental");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($file)); # set the filename as Source metadata to be consistent with other plugins
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    #create an empty text string so we don't break downstream plugins 
+    my $text = &gsprintf::lookup_string("{BasPlug.dummy_text}");
+    
+    # include any metadata passed in from previous plugins 
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+    
+    $self->title_fallback($doc_obj,$section,$file);
+    
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1;
+    }
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    # add an OID
+    $doc_obj->set_OID();
+    $doc_obj->add_text($section, $text);
+    
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'} ++;
+    return 1;
+}
+
+
+# NULPlug processing of doc_obj.  In practice we don't need to do
+# anything here because the read function takes care of everything.
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/OAIPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/OAIPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/OAIPlug.pm	(revision 21112)
@@ -0,0 +1,375 @@
+###########################################################################
+#
+# OAIPlug.pm -- basic Open Archives Initiative (OAI) plugin
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package OAIPlug;
+
+use BasPlug;
+use unicode;
+use util;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use XMLPlug;
+
+sub BEGIN {
+    @OAIPlug::ISA = ('XMLPlug');
+}
+
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      ];
+
+my $options = { 'name'     => "OAIPlug",
+		'desc'     => "{OAIPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new XMLPlug($pluginlist,$inputargs,$hashArgOptLists): new XMLPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)(\.oai)$^;
+}
+
+sub xml_start_document {
+    my $self = shift (@_);
+    $self->{'in_metadata_node'} = 0;
+    $self->{'rawxml'} = "";
+}
+
+sub xml_end_document {
+}
+
+sub xml_doctype {
+    my $self = shift(@_);
+
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+
+    # allow the short-lived and badly named "GreenstoneArchive" files to be processed
+    # as well as the "Archive" files which should now be created by import.pl
+    die "" if ($name !~ /^OAI-PMH$/);
+
+    my $outhandle = $self->{'outhandle'};
+    print $outhandle "OAIPlug: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
+    print STDERR "<Processing n='$self->{'file'}' p='OAIPlug'>\n" if $self->{'gli'};
+
+}
+
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat,$element) = @_;
+
+    my %attr_hash = %_;
+
+    my $attr = "";
+    map { $attr .= " $_=$attr_hash{$_}"; } keys %attr_hash;
+
+    $self->{'rawxml'} .= "<$element$attr>";
+
+    if ($element eq "metadata") {
+	$self->{'in_metadata_node'} = 1;
+	$self->{'metadata_xml'} = "";
+    }
+
+    if ($self->{'in_metadata_node'}) {
+	$self->{'metadata_xml'} .= "<$element$attr>";
+    }
+}
+
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    $self->{'rawxml'} .= "</$element>";
+
+    if ($self->{'in_metadata_node'}) {
+	$self->{'metadata_xml'} .= "</$element>";
+    }
+
+    if ($element eq "metadata") {
+	my $textref = \$self->{'metadata_xml'};
+	my $metadata = $self->{'metadata'};
+	$self->extract_oai_metadata($textref,$metadata);
+
+	$self->{'in_metadata_node'} = 0;	
+    }
+
+
+}
+
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    $self->{'rawxml'} .= $_;
+
+    if ($self->{'in_metadata_node'}) {
+	$self->{'metadata_xml'} .= $_;
+    }
+}
+
+
+
+
+sub read {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    return 0 if ((-d $filename) && ($filename =~ m/srcdocs$/));
+
+    if ($self->SUPER::read(@_)) {
+
+	# Do encoding stuff
+	my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+	
+	my $url_array = $metadata->{'URL'};
+	my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
+	
+	my $srcdoc_exists = 0;
+	my $srcdoc_pos = 0;
+	my $filename_dir = &util::filename_head($filename);
+	
+	for (my $i=0; $i<$num_urls; $i++) {
+	    
+	    if ($url_array->[$i] !~ m/^(http|ftp):/) {
+		
+		my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
+		
+		if (-e $src_filename) {
+		    $srcdoc_pos = $i;
+		    $srcdoc_exists = 1;
+		}
+	    }
+	}
+	
+	if ($srcdoc_exists)
+	{
+	    print $outhandle "OAIPlug: passing metadata on to $url_array->[0]\n"
+		if ($self->{'verbosity'}>1);
+	    
+	    
+	    # Make pretty print metadata table stick with src filename
+	    my $ppmd_table = $self->{'ppmd_table'};
+	    $metadata->{'prettymd'} = [ $ppmd_table ];
+	    $self->{'ppmd_table'} = undef;
+	    
+	    return &plugin::read ($pluginfo, $filename_dir, $url_array->[0],
+				  $metadata, $processor, $maxdocs, $total_count, $gli);
+	}
+	else
+	{
+	    # create a new document
+	    my $doc_obj = new doc ($filename, "indexed_doc");
+	    my $top_section = $doc_obj->get_top_section;
+	    my $plugin_type = $self->{'plugin_type'};
+	    
+	    $doc_obj->add_utf8_metadata($top_section, "Language", $language);
+	    $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+	    $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
+	    $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
+	    $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
+	    
+	    # include any metadata passed in from previous plugins 
+	    # note that this metadata is associated with the top level section
+	    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+	    
+	    # do plugin specific processing of doc_obj
+	    my $textref = \$self->{'rawxml'};
+	    unless (defined ($self->process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+		print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+		return -1;
+	    }
+	    
+	    # do any automatic metadata extraction
+	    $self->auto_extract_metadata ($doc_obj);
+	    
+	    # add an OID
+	    $doc_obj->set_OID();
+	    
+	    my $ppmd_table = $self->{'ppmd_table'};
+	    $doc_obj->add_utf8_metadata($top_section,"prettymd",$ppmd_table);
+	    $self->{'ppmd_table'} = undef;
+	    
+	    # process the document
+	    $processor->process($doc_obj);
+	    
+	    $self->{'num_processed'} ++;
+	    
+	    return 1; # processed the file
+	}
+    }
+    else {
+	return undef;
+    }
+}
+
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print STDERR "<Processing n='$file' p='OAIPlug'>\n" if ($gli);
+    print $outhandle "OAIPlug: processing $file\n"
+	if $self->{'verbosity'} > 1;
+
+    my $cursection = $doc_obj->get_top_section();
+
+##    $self->extract_metadata ($textref, $metadata, $doc_obj, $cursection);
+
+    # add text to document object
+
+#    $$textref =~ s/<(.*?)>/$1 /g;
+    $$textref =~ s/</&lt;/g;
+    $$textref =~ s/>/&gt;/g;
+
+##    print STDERR "*** adding text: $$textref\n";
+    
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    return 1;
+}
+
+
+# Improvement is to merge this with newer version in MetadataPass
+
+sub open_prettyprint_metadata_table
+{
+    my $self = shift(@_);
+
+    my $att   = "width=100% cellspacing=2";
+    my $style = "style=\'border-bottom: 4px solid #000080\'";
+
+    $self->{'ppmd_table'} = "\n<table $att $style>";
+}
+
+sub add_prettyprint_metadata_line 
+{
+    my $self = shift(@_);
+    my ($metaname, $metavalue_utf8) = @_;
+
+    $metavalue_utf8 =~ s/hdl\.handle\.net/mcgonagall.cs.waikato.ac.nz:8080\/dspace\/handle/;
+    $metavalue_utf8 = &util::hyperlink_text($metavalue_utf8);
+
+    $self->{'ppmd_table'} .= "  <tr bgcolor=#b5d3cd>\n";
+    $self->{'ppmd_table'} .= "    <td width=30%>\n";
+    $self->{'ppmd_table'} .= "      $metaname\n";
+    $self->{'ppmd_table'} .= "    </td>\n";
+    $self->{'ppmd_table'} .= "    <td>\n";
+    $self->{'ppmd_table'} .= "      $metavalue_utf8\n";
+    $self->{'ppmd_table'} .= "    </td>\n";
+    $self->{'ppmd_table'} .= "  </tr>\n";
+
+}
+
+sub close_prettyprint_metadata_table
+{
+    my $self = shift(@_);
+
+    $self->{'ppmd_table'} .= "</table>\n";
+}
+
+
+
+
+sub extract_oai_metadata {
+    my $self = shift (@_);
+    my ($textref, $metadata) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # Only handles DC metadata
+
+    $self->open_prettyprint_metadata_table();
+
+    if ($$textref =~ m/<metadata\s*>(.*?)<\/metadata\s*>/s)
+    {
+	my $metadata_text = $1;
+	$metadata_text =~ s/^.*?<(oai_dc:)?dc.*?>(.*?)<\/(oai_dc:)?dc>.*?/$2/s;
+
+	while ($metadata_text =~ m/<(.*?)>(.*?)<\/(.*?)>(.*)/s)
+	{
+	    # if URL given for document as identifier metadata, store it ...
+	    # $doc_obj->add_utf8_metadata($cursection, "URL", $web_url);
+
+	    my $metaname = $1;
+	    my $metavalue = $2;
+	    $metadata_text = $4;
+	    
+	    $metaname =~ s/^(dc:)?(.)/\u$2/;
+
+	    if ($metaname eq "Identifier")
+	    {
+		# name clashes with GSDL reserved metadata name for hash id
+		$metaname = "URL";
+	    }
+
+	    if (defined $metadata->{$metaname})
+	    {
+		push(@{$metadata->{$metaname}},$metavalue);
+
+	    }
+	    else
+	    {
+		$metadata->{$metaname} = [ $metavalue ];
+	    }
+
+	    $self->add_prettyprint_metadata_line($metaname, $metavalue);
+	    
+	}
+    }
+
+    $self->close_prettyprint_metadata_table();
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/OggVorbisPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/OggVorbisPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/OggVorbisPlug.pm	(revision 21112)
@@ -0,0 +1,178 @@
+###########################################################################
+#
+# OggVorbisPlug.pm -- A plugin for Ogg Vorbis audio files
+#
+# Original code by Christy Kuo
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 1999-2004 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package OggVorbisPlug;
+
+
+use UnknownPlug;
+use Ogg::Vorbis::Header::PurePerl;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @OggVorbisPlug::ISA = ('UnknownPlug');
+}
+
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "add_technical_metadata",
+	'desc' => "{OggVorbisPlug.add_technical_metadata}",
+	'type' => "flag",
+	'deft' => "" } ];
+
+my $options = { 'name'     => "OggVorbisPlug",
+		'desc'     => "{OggVorbisPlug.desc}",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes exported Ogg Vorbis files with the suffix ".ogg"
+sub get_default_process_exp
+{
+    return q^(?i)(\.ogg)$^;
+}
+
+
+sub new
+{
+    my ($class) = shift(@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = (defined $hashArgOptLists)? new UnknownPlug($pluginlist,$inputargs,$hashArgOptLists): new UnknownPlug($pluginlist,$inputargs);
+    
+    return bless $self, $class;
+}
+
+
+sub read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    # filename is the full pathname of the file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{block_exp} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='OggVorbisPlug'>\n" if ($gli);
+    print $outhandle "OggVorbisPlug: processing $file\n"
+  	if ($self->{'verbosity'}) > 1;
+    
+    # file is just the name of the file (need to get rid off any leading directory names)
+    $file =~ s/^.*[\/\\]//;
+
+    # create a new index document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ("incremental");    # this is done to avoid hashing content of file
+    my $section = $doc_obj->get_top_section();
+    
+    # replace spaces in filename with %20 in url for metadata entry
+    my $url = $file;
+    $url =~ s/ /%20/g;
+    # Source (filename) to be consistent with other plugins
+    $doc_obj->add_metadata ($section, "Source", $url);
+
+    # Extract metadata
+    my $ogg = Ogg::Vorbis::Header::PurePerl->new($filename);
+
+    # Comments added to the file
+    foreach my $key ($ogg->comment_tags())
+    {
+	# Convert key to title case
+	my $keytc = uc(substr($key, 0, 1)) . substr($key, 1, length($key));
+	foreach my $value ($ogg->comment($key))
+	{
+	    if (defined $value && $value ne "") {
+		$doc_obj->add_metadata($section, $keytc, $value);
+	    }
+	}
+    }
+
+    # Technical data (optional)
+    if ($self->{'add_technical_metadata'}) {
+	foreach my $key (keys %{$ogg->info})
+	{
+	    # Convert key to title case
+	    my $keytc = uc(substr($key, 0, 1)) . substr($key, 1, length($key));
+	    my $value = $ogg->info->{$key};
+	    if (defined $value && $value ne "") {
+		$doc_obj->add_metadata($section, $keytc, $value);
+	    }
+	}
+    }
+
+    # srclink
+    $doc_obj->add_metadata ($section, "FileFormat", "OggVorbis");
+    $doc_obj->add_metadata ($section, "srclink", "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[Source]\">");
+    $doc_obj->add_metadata ($section, "/srclink", "</a>");
+    # srcicon (need to include "ogg.png" in the greenstone images directory
+    $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpprefix_/images/ogg.png\" title=\"Download\" border=0>");
+
+    # Add the actual file as an associated file
+    $doc_obj->associate_file($filename, $file, "VORBIS", $section);
+
+    # Create an empty text string so we don't break downstream plugins 
+    my $text = &gsprintf::lookup_string("{BasPlug.dummy_text}");
+
+    # include any metadata passed in from previous plugins
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+
+    # do plugin specific processing of doc_obj
+    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata($doc_obj);
+
+    # add an OID
+    $doc_obj->set_OID();
+    $doc_obj->add_text($section, $text);
+
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'}++;
+    return 1;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/PDFPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/PDFPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/PDFPlug.pm	(revision 21112)
@@ -0,0 +1,330 @@
+###########################################################################
+#
+# PDFPlug.pm -- reasonably with-it pdf plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+package PDFPlug;
+
+use ConvertToPlug;
+use unicode;
+use strict;
+no strict 'refs'; # so we can use a var for filehandles (eg STDERR)
+
+sub BEGIN {
+    @PDFPlug::ISA = ('ConvertToPlug');
+}
+
+my $convert_to_list =
+    [ {	'name' => "auto",
+	'desc' => "{ConvertToPlug.convert_to.auto}" },
+      {	'name' => "html",
+	'desc' => "{ConvertToPlug.convert_to.html}" },
+      {	'name' => "text",
+	'desc' => "{ConvertToPlug.convert_to.text}" },
+      { 'name' => "pagedimg_jpg",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_jpg}"},
+      { 'name' => "pagedimg_gif",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_gif}"},
+      { 'name' => "pagedimg_png",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_png}"}, 
+      ];
+
+
+my $arguments = 
+    [
+     { 'name' => "convert_to",
+       'desc' => "{ConvertToPlug.convert_to}",
+       'type' => "enum",
+       'reqd' => "yes",
+       'list' => $convert_to_list, 
+       'deft' => "html" },	 
+     { 'name' => "process_exp",
+       'desc' => "{BasPlug.process_exp}",
+       'type' => "regexp",
+       'deft' => &get_default_process_exp(),
+       'reqd' => "no" },
+     { 'name' => "block_exp",
+       'desc' => "{BasPlug.block_exp}",
+       'type' => "regexp",
+       'deft' => &get_default_block_exp() },
+     { 'name' => "metadata_fields",
+       'desc' => "{HTMLPlug.metadata_fields}",
+       'type' => "string",
+       'deft' => "" },
+     { 'name' => "noimages",
+       'desc' => "{PDFPlug.noimages}",
+       'type' => "flag" },
+     { 'name' => "allowimagesonly",
+       'desc' => "{PDFPlug.allowimagesonly}",
+       'type' => "flag" },
+     { 'name' => "complex",
+       'desc' => "{PDFPlug.complex}",
+       'type' => "flag" },
+     { 'name' => "nohidden",
+       'desc' => "{PDFPlug.nohidden}",
+       'type' => "flag" },
+     { 'name' => "zoom",
+       'desc' => "{PDFPlug.zoom}",
+       'deft' => "2",
+       'range' => "1,3", # actually the range is 0.5-3 
+       'type' => "int" },
+     { 'name' => "use_sections",
+       'desc' => "{PDFPlug.use_sections}",
+       'type' => "flag" },
+     { 'name' => "description_tags",
+       'desc' => "{HTMLPlug.description_tags}",
+       'type' => "flag" }
+     ];
+
+my $options = { 'name'     => "PDFPlug",
+		'desc'     => "{PDFPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@$inputargs,"-title_sub");
+    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my @arg_array = @$inputargs;
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+    
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # these are passed through to gsConvert.pl by ConvertToPlug.pm
+    my $zoom = $self->{"zoom"};
+    $self->{'convert_options'} = "-pdf_zoom $zoom";
+    $self->{'convert_options'} .= " -pdf_complex" if $self->{"complex"};
+    $self->{'convert_options'} .= " -pdf_nohidden" if $self->{"nohidden"};
+    $self->{'convert_options'} .= " -pdf_ignore_images" if $self->{"noimages"};
+    $self->{'convert_options'} .= " -pdf_allow_images_only" if $self->{"allowimagesonly"};
+
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+
+    if (!defined $secondary_plugin_options->{'HTMLPlug'}) {
+	$secondary_plugin_options->{'HTMLPlug'} = [];
+    }
+    if (!defined $secondary_plugin_options->{'TEXTPlug'}) {
+	$secondary_plugin_options->{'TEXTPlug'} = [];
+    }
+    if (defined $self->{'convert_to'} && $self->{'convert_to'} =~ /pagedimg.*/i) {
+	if (!defined $secondary_plugin_options->{'PagedImgPlug'}){
+	    $secondary_plugin_options->{'PagedImgPlug'} = [];
+	    my $pagedimg_options = $secondary_plugin_options->{'PagedImgPlug'}; 
+	    push(@$pagedimg_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+	}
+    }
+    my $html_options = $secondary_plugin_options->{'HTMLPlug'};
+    my $text_options = $secondary_plugin_options->{'TEXTPlug'};
+   
+    if ($self->{'input_encoding'} eq "auto") {
+	# pdftohtml will always produce html files encoded as utf-8
+	# => restrict primary PDFPlug and secondary HTML plugin to use 
+	# utf8 and extract language.
+	$self->{'input_encoding'} = "utf8";
+	$self->{'extract_language'} = 1;
+
+	push(@$html_options,"-extract_language");
+    }
+    # if pdftohtml is always producing utf8, then htmlplug always needs this option
+    push(@$html_options,"-input_encoding", "utf8");
+
+    # Instruct HTMLPlug (when eventually accessed through read_into_doc_obj) 
+    # to extract these metadata fields from the HEAD META fields
+    my $required_metadata;
+    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'} =~ /\S/) {
+	push(@$html_options,"-metadata_fields",$self->{'metadata_fields'});
+    } else {
+	push(@$html_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+    }
+    #push(@$html_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+        
+    if ($self->{'use_sections'} || $self->{'description_tags'}) {
+	$self->{'description_tags'} = 1;
+	push(@$html_options,"-description_tags");
+    }
+
+    # following title_sub removes "Page 1" added by pdftohtml, and a leading
+    # "1", which is often the page number at the top of the page. Bad Luck
+    # if your document title actually starts with "1 " - is there a better way?
+    push(@$html_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+    push(@$text_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+ 
+    $self = bless $self, $class;
+    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
+    return $self;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.pdf$^;
+}
+
+# so we don't inherit HTMLPlug's block exp...
+sub get_default_block_exp {
+    return "";
+}
+    
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+
+    my $outhandle=$self->{'outhandle'};
+
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($conv_filename, $encoding, $language, \$text);
+
+    # Calculate number of pages based on <a ...> tags (we have a <a name=1> etc
+    # for each page).  Metadata based on this calculation not set until process()
+    # 
+    # Note: this is done even if we are not breaking to document into pages as it might
+    # be useful to give an indication of document length in browser through setting
+    # num_pages as metadata.
+    my @pages = ($text =~ /\<[Aa] name=\"?\w+\"?>/ig);
+    my $num_pages = scalar(@pages);
+    $self->{'num_pages'} = $num_pages;
+
+    if ($self->{'use_sections'}
+	&& $self->{'converted_to'} eq "HTML") {
+
+	print $outhandle "PDFPlug: Calculating sections...\n";
+
+	# we have "<a name=1></a>" etc for each page
+	# it may be <A name=
+	my @sections = split('<[Aa] name=', $text);
+
+	my $top_section = "";
+
+	if (scalar (@sections) == 1) { #only one section - no split!
+	    print $outhandle "PDFPlug: warning - no sections found\n";
+	} else {
+	    $top_section .= shift @sections; # keep HTML header etc as top_section
+	}
+
+	# handle first section specially for title? Or all use first 100...
+	
+	my $title = $sections[0];
+	$title =~ s/^\"?\w+\"?>//; # specific for pdftohtml...
+	$title =~ s/<\/([^>]+)><\1>//g; # (eg) </b><b> - no space
+	$title =~ s/<[^>]*>/ /g;
+	$title =~ s/(?:&nbsp;|\xc2\xa0)/ /g; # utf-8 for nbsp...
+	$title =~ s/^\s+//s;
+	$title =~ s/\s+$//;
+	$title =~ s/\s+/ /gs;
+	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
+	$title =~ s/^\s+//s; # in case title_sub introduced any...
+	$title = substr ($title, 0, 100);
+	$title =~ s/\s\S*$/.../;
+
+
+	if (scalar (@sections) == 1) { # no sections found
+	    $top_section .= $sections[0];
+	    @sections=();
+	} else {
+	    $top_section .= "<!--<Section>\n<Metadata name=\"Title\">$title</Metadata>\n-->\n <!--</Section>-->\n";
+	}
+
+	# add metadata per section...
+	foreach my $section (@sections) {
+	    # section names are not always just digits, may be like "outline"
+	    $section =~ s@^\"?(\w+)\"?></a>@@; # leftover from split expression...
+
+	    $title = $1; # Greenstone does magic if sections are titled digits
+	    if (! defined($title) ) {
+		print STDERR "no title: $section\n";
+		$title = " "; # get rid of the undefined warning in next line
+	    }
+	    my $newsection = "<!-- from PDFPlug -->\n<!-- <Section>\n";
+	    $newsection .= "<Metadata name=\"Title\">" . $title
+		. "</Metadata>\n--><p>\n";
+	    $newsection .= $section;
+	    $newsection .= "<!--</Section>-->\n";
+	    $section = $newsection;
+	}
+
+	$text=join('', ($top_section, @sections));
+    }
+
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+    unicode::ensure_utf8(\$text);
+
+    # Write it out again!
+    $self->utf8_write_file (\$text, $conv_filename);
+}
+
+
+# do plugin specific processing of doc_obj for HTML type
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    my $result = $self->process_type("pdf",$base_dir,$file,$doc_obj);
+
+    # fix up the extracted date metadata to be in Greenstone date format,
+    # and fix the capitalisation of 'date'
+    my $cursection = $doc_obj->get_top_section();
+    foreach my $datemeta (@{$doc_obj->get_metadata($cursection, "date")}) {
+	$doc_obj->delete_metadata($cursection, "date", $datemeta);
+
+	# We're just interested in the date bit, not the time
+	# some pdf creators (eg "Acrobat 5.0 Scan Plug-in for Windows")
+	# set a /CreationDate, and set /ModDate to 000000000. pdftohtml
+	# extracts the ModDate, so it is 0...
+	$datemeta =~ /(\d+)-(\d+)-(\d+)/;
+	my ($year, $month, $day) = ($1,$2,$3);
+	if (defined($year) && defined($month) && defined($day)) {
+	    if ($year == 0) {next}
+	    if ($year < 100) {$year += 1900} # just to be safe
+	    if ($month =~ /^\d$/) {$month="0$month"} # single digit
+	    if ($day =~ /^\d$/) {$day="0$day"} # single digit
+	    my $date="$year$month$day";
+	    $doc_obj->add_utf8_metadata($cursection, "Date", $date);
+	}
+    }
+
+    $doc_obj->add_utf8_metadata($cursection, "NumPages", $self->{'num_pages'});
+    
+    if ($self->{'use_sections'} && $self->{'converted_to'} eq "HTML") {
+	# we explicitly make it a paged document, cos greenstone won't get it
+	# right if any section has an empty title, or one with letters in it
+	$doc_obj->set_utf8_metadata_element ($cursection, "gsdlthistype", "Paged");
+    }
+
+    return $result;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/PPTPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/PPTPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/PPTPlug.pm	(revision 21112)
@@ -0,0 +1,171 @@
+###########################################################################
+#
+# PPTPlug.pm -- plugin for importing Microsoft PowerPoint files.
+#  (currently only versions 95 and 97)
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2002 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package PPTPlug;
+
+use ConvertToPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @PPTPlug::ISA = ('ConvertToPlug');
+}
+
+my $convert_to_list =
+    [ {	'name' => "auto",
+	'desc' => "{ConvertToPlug.convert_to.auto}" },
+      {	'name' => "html",
+	'desc' => "{ConvertToPlug.convert_to.html}" },
+      {	'name' => "text",
+	'desc' => "{ConvertToPlug.convert_to.text}" },
+      { 'name' => "pagedimg_jpg",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_jpg}" },
+      { 'name' => "pagedimg_gif",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_gif}" },
+      { 'name' => "pagedimg_png",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_png}" }
+      ];
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp()}
+      ];
+
+my $options = { 'name'     => "PPTPlug",
+		'desc'     => "{PPTPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+	        'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
+	my $ws_arg =[{ 'name' => "convert_to",
+		       'desc' => "{ConvertToPlug.convert_to}",
+		       'type' => "enum",
+		       'reqd' => "yes",
+		       'list' => $convert_to_list, 
+		       'deft' => "html" },
+		     { 'name' => "windows_scripting",
+		       'desc' => "{PPTPlug.windows_scripting}",
+		       'type' => "flag",
+		       'reqd' => "no" }
+		     ];
+	push(@$arguments,@$ws_arg);
+    }
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+
+    my @arg_array = @$inputargs;
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+ 
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # ppthtml outputs utf-8 already.
+    #these are passed through to gsConvert.pl by ConvertToPlug.pm
+    $self->{'convert_options'} = "-windows_scripting" if $self->{'windows_scripting'};
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+
+    if ($self->{'windows_scripting'} && ($self->{'convert_to'} eq "PagedImg")) {
+	$secondary_plugin_options->{'PagedImgPlug'} = [];
+    } else {
+	$secondary_plugin_options->{'HTMLPlug'} = [];
+    }
+    my $html_options = $secondary_plugin_options->{'HTMLPlug'};
+    my $pageimg_options = $secondary_plugin_options->{'PagedImgPlug'};
+
+    if ($self->{'input_encoding'} eq "auto") {
+	$self->{'input_encoding'} = "utf8";
+	$self->{'extract_language'} = 1;
+	if (defined $secondary_plugin_options->{'HTMLPlug'}){
+	    push(@$html_options,"-input_encoding", "utf8");
+	    push(@$html_options,"-extract_language");
+
+	    # Instruct HTMLPlug (when eventually accessed through read_into_doc_obj) 
+	    # to extract these metadata fields from the HEAD META fields
+	    push(@$html_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+	} 
+	if (defined $secondary_plugin_options->{'PagedImgPlug'}){
+	    push(@$pageimg_options,"-input_encoding", "utf8");
+	    push(@$pageimg_options,"-extract_language");
+	}
+    }
+
+    $self = bless $self, $class;
+
+    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
+    return $self;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    return q^(?i)\.ppt$^;
+}
+
+sub get_file_type {
+    my $self = shift (@_);
+    my $file_type = "PPT";
+    return $file_type;
+}
+
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+
+    my $outhandle=$self->{'outhandle'};
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($conv_filename, $encoding, $language, \$text);
+ 
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+    unicode::ensure_utf8(\$text);
+    # Write it out again!
+    $self->utf8_write_file (\$text, $conv_filename);
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    return $self->process_type("ppt",$base_dir,$file,$doc_obj);
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/plugins/PSPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/PSPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/PSPlug.pm	(revision 21112)
@@ -0,0 +1,257 @@
+###########################################################################
+#
+# PSPlug.pm -- this might look VERY similar to the PDF plugin...
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package PSPlug;
+
+use ConvertToPlug;
+use sorttools;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @PSPlug::ISA = ('ConvertToPlug');
+}
+
+my $convert_to_list =
+    [ {	'name' => "auto",
+	'desc' => "{ConvertToPlug.convert_to.auto}" },
+      {	'name' => "text",
+	'desc' => "{ConvertToPlug.convert_to.text}" },
+      { 'name' => "pagedimg_jpg",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_jpg}" },
+      { 'name' => "pagedimg_gif",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_gif}" },
+      { 'name' => "pagedimg_png",
+	'desc' => "{ConvertToPlug.convert_to.pagedimg_png}" }
+      ];
+
+my $arguments =
+    [ { 'name' => "convert_to",
+	'desc' => "{ConvertToPlug.convert_to}",
+	'type' => "enum",
+	'reqd' => "yes",
+	'list' => $convert_to_list, 
+	'deft' => "text" },
+      { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => 'regexp',
+	'deft' => &get_default_block_exp() },
+      { 'name' => "extract_date",
+	'desc' => "{PSPlug.extract_date}",
+	'type' => "flag" },
+      { 'name' => "extract_pages",
+	'desc' => "{PSPlug.extract_pages}",
+	'type' => "flag" },
+      { 'name' => "extract_title",
+	'desc' => "{PSPlug.extract_title}",
+	'type' => "flag" } ];
+
+my $options = { 'name'     => "PSPlug",
+		'desc'     => "{PSPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    #push(@$inputargs,"-convert_to");
+    #push(@$inputargs,"text");
+    push(@$inputargs,"-title_sub");
+    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+
+    if (!defined $secondary_plugin_options->{'TEXTPlug'}) {
+	$secondary_plugin_options->{'TEXTPlug'} = [];
+    }
+
+    my $text_options = $secondary_plugin_options->{'TEXTPlug'};
+
+    if (defined $self->{'convert_to'} && $self->{'convert_to'} =~ /pagedimg.*/i) {
+	if (!defined $secondary_plugin_options->{'PagedImgPlug'}){
+	    $secondary_plugin_options->{'PagedImgPlug'} = [];
+	    my $pagedimg_options = $secondary_plugin_options->{'PagedImgPlug'}; 
+	    push(@$pagedimg_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+	}
+    }
+    # following title_sub removes "Page 1" added by ps2ascii, and a leading
+    # "1", which is often the page number at the top of the page. Bad Luck
+    # if your document title actually starts with "1 " - is there a better way?
+    #$self->{'input_encoding'} = "utf8";
+    #$self->{'extract_language'} = 1;
+    push(@$text_options, "-input_encoding", "utf8");
+    push(@$text_options,"-extract_language");
+    push(@$text_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+
+    $self = bless $self, $class;
+
+    $self->load_secondary_plugins($class,$secondary_plugin_options, $hashArgOptLists);
+
+    return $self;
+}
+
+
+sub get_default_block_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.(eps)$^;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.ps$^;
+}
+
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+    
+    my $outhandle=$self->{'outhandle'};
+    
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+    
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($conv_filename, $encoding, $language, \$text);
+    
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+    unicode::ensure_utf8(\$text);
+    
+    # Write it out again!
+    $self->utf8_write_file (\$text, $conv_filename);
+}
+
+sub extract_metadata_from_postscript {
+    my $self = shift (@_);
+
+    my ($filename,$doc) = @_;
+
+    my $section = $doc->get_top_section();
+
+    my $title_found = 0;
+    my $pages_found = 0;
+    my $date_found = 0;
+
+    print STDERR "PSPlug: extracting PostScript metadata from \"$filename\"\n" 
+	if $self->{'verbosity'} > 1;
+
+    open(INPUT, "<$filename");
+    my $date;
+
+    while(my $line =<INPUT>) {
+	if ($self->{'extract_title'} && !$title_found) {
+	    foreach my $word ($line =~ m|Title: ([-A-Za-z0-9@/\/\(\):,. ]*)|g) {
+		my $new_word = $word; 
+		$new_word =~ s/\(Untitled\)//i;
+		$new_word =~ s/\(Microsoft Word\)//i;
+		$new_word =~ s/Microsoft Word//i;
+		$new_word =~ s/^\(//i;
+		$new_word =~ s/\)$//i;
+		$new_word =~ s/^ - //i;
+		if ($new_word ne "") {
+		    $doc->add_utf8_metadata($section, "Title", $new_word );
+		    $title_found = 1;
+		}
+	    }
+	}
+	if ($self->{'extract_date'} && !$date_found) {
+            foreach my $word ($line =~ m/(Creation[-A-Za-z0-9@\/\(\):,. ]*)/g) {
+                if ($word =~ m/ ([A-Za-z][A-Za-z][A-Za-z]) ([0-9 ][0-9])  ?[0-9: ]+ ([0-9]{4})/) {
+                    $date = &sorttools::format_date($2,$1,$3);
+		    if (defined $date) {
+			$doc->add_utf8_metadata($section, "Date", $date );
+		    }
+                }
+                if ($word =~ m/D:([0-9]{4})([0-9]{2})([0-9]{2})[0-9]{6}\)/) {
+                    $date = &sorttools::format_date($3,$2,$1);
+		    if (defined $date) {
+			$doc->add_utf8_metadata($section, "Date", $date );
+		    }
+                }
+                if ($word =~ m/CreationDate: ([0-9]{4}) ([A-Za-z][A-Za-z][A-Za-z]) ([0-9 ][0-9]) [0-9:]*/) {
+                    $date = &sorttools::format_date($3,$2,$1);
+		    if (defined $date) {
+			$doc->add_utf8_metadata($section, "Date", $date );
+		    }
+                }
+		$date_found = 1;
+            }
+	}
+	if ($self->{'extract_pages'} && !$pages_found) {
+	    foreach my $word ($line =~ m/(Pages: [0-9]*)/g) {
+                my $digits = $word;
+                $digits =~ s/[^0-9]//g;
+		if ($digits ne "" && $digits ne "0") {
+		    $doc->add_utf8_metadata($section, "Pages", $digits );
+		    $pages_found = 1;
+		}
+            }
+	}
+    }
+}
+
+# do plugin specific processing of doc_obj for HTML type
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+#    my $outhandle = $self->{'outhandle'};
+
+#    print $outhandle "PSPlug: passing $file on to $self->{'converted_to'}Plug\n" 
+#	if $self->{'verbosity'} > 1;
+#    print STDERR "<Processing n='$file' p='PSPlug'>\n" if ($gli);
+
+    my $filename = &util::filename_cat($base_dir,$file);
+    $self->extract_metadata_from_postscript($filename, $doc_obj);
+
+    return $self->process_type("ps",$base_dir,$file,$doc_obj);
+}
+
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/plugins/PagedImgPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/PagedImgPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/PagedImgPlug.pm	(revision 21112)
@@ -0,0 +1,937 @@
+###########################################################################
+#
+# PagedImgPlug.pm -- plugin for sets of images and OCR text that
+#  make up a document
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# PagedImgPlug
+# processes sequences of images, with optional OCR text
+#
+# This plugin takes *.item files, which contain metadata and lists of image 
+# files, and produces a document containing sections, one for each page.
+# The files should be named something.item, then you can have more than one 
+# book in a directory. You will need to create these files, one for each
+# document/book.
+#
+#There are two formats for the item files: a plain text format, and an xml 
+#format. You can use either format, and can have both formats in the same 
+#collection if you like. If you use the plain format, you must not start the 
+#file off with <PagedDocument>
+
+#### PLAIN FORMAT
+# The format of the xxx.item file is as follows:
+# The first lines contain any metadata for the whole document
+# <metadata-name>metadata-value
+# eg.
+# <Title>Snail farming
+# <Date>19230102
+# Then comes a list of pages, one page per line, each line has the format
+#
+# pagenum:imagefile:textfile:r
+#
+# page num and imagefile are required. pagenum is used for the Title 
+# of the section, and in the display is shown as page <pagenum>. 
+# imagefile is the image for the page. textfile is an optional text 
+# file containing the OCR (or any) text for the page - this gets added 
+# as the text for the section. r is optional, and signals that the image 
+# should be rotated 180deg. Eg use this if the image has been made upside down.
+# So an example item file looks like:
+# <Title>Snail farming
+# <Date>19960403
+# 1:p1.gif:p1.txt:
+# 2:p2.gif::
+# 3:p3.gif:p3.txt:
+# 3b:p3b.gif:p3b.txt:r
+# The second page has no text, the fourth page is a back page, and 
+# should be rotated.
+# 
+
+#### XML FORMAT
+# The xml format looks like the following
+#<PagedDocument>
+#<Metadata name="Title">The Title of the entire document</Metadata>
+#<Page pagenum="1" imgfile="xxx.jpg" txtfile="yyy.jpg">
+#<Metadata name="Title">The Title of this page</Metadata>
+#</Page>
+#... more pages
+#</PagedDocument>
+#PagedDocument contains a list of Pages, Metadata and PageGroups. Any metadata 
+#that is not inside another tag will belong to the document.
+#Each Page has a pagenum (not used at the moment), an imgfile and/or a txtfile.
+#These are both optional - if neither is used, the section will have no content.
+#Pages can also have metadata associated with them.
+#PageGroups can be introduced at any point - they can contain Metadata and Pages and other PageGroups. They are used to introduce hierarchical structure into the document.
+#For example
+#<PagedDocument>
+#<PageGroup>
+#<Page>
+#<Page>
+#</PageGroup>
+#<Page>
+#</PagedDocument>
+#would generate a structure like
+#X
+#--X
+#  --X
+#  --X
+#--X
+#PageGroup tags can also have imgfile/textfile metadata if you like - this way they get some content themselves.
+
+#Currently the XML structure doesn't work very well with the paged document type, unless you use numerical Titles for each section.
+#There is still a bit of work to do on this format:
+#* enable other text file types, eg html, pdf etc
+#* make the document paging work properly
+#* add pagenum as Title unless a Title is present?
+
+# All the supplemetary image amd text files should be in the same folder as 
+# the .item file.
+#
+# To display the images instead of the document text, you can use [srcicon] 
+# in the DocumentText format statement.
+# For example, 
+#
+# format DocumentText "<center><table width=_pagewidth_><tr><td>[srcicon]</td></tr></table></center>"
+#
+# To have it create thumbnail size images, use the '-thumbnail' option.
+# To have it create medium size images for display, use the '-screenview'
+# option. As usual, running 
+# 'perl -S pluginfo.pl PagedImgPlug' will list all the options.
+
+# If you want the resulting documents to be presented with a table of 
+# contents, use '-documenttype hierarchy', otherwise they will have 
+# next and previous arrows, and a goto page X box. 
+
+# If you have used -screenview, you can also use [screenicon] in the format
+# statement to display the smaller image.  Here is an example  that switches 
+# between the two: 
+#
+# format DocumentText "<center><table width=_pagewidth_><tr><td>{If}{_cgiargp_ eq full,<a href='_httpdocument_&d=_cgiargd_&p=small'>Switch to small version.</a>,<a href='_httpdocument_&d=_cgiargd_&p=full'>Switch to fullsize version</a>}</td></tr><tr><td>{If}{_cgiargp_ eq full,<a href='_httpdocument_&d=_cgiargd_&p=small' title='Switch to small version'>[srcicon]</a>,<a href='_httpdocument_&d=_cgiargd_&p=full' title='Switch to fullsize version'>[screenicon]</a>}</td></tr></table></center>"
+#
+# Additional metadata can be added into the .item files, alternatively you can 
+# use normal metadata.xml files, with the name of the xxx.item file as the 
+# FileName (only for document level metadata).
+
+package PagedImgPlug;
+
+use XMLPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @PagedImgPlug::ISA = ('XMLPlug');
+}
+
+my $type_list =
+    [ { 'name' => "paged",
+        'desc' => "{PagedImgPlug.documenttype.paged}" },
+      { 'name' => "hierarchy",
+        'desc' => "{PagedImgPlug.documenttype.hierarchy}" } ];
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "string",
+	'deft' => &get_default_block_exp(),
+	'reqd' => "no" },
+      { 'name' => "title_sub",
+	'desc' => "{HTMLPlug.title_sub}",
+	'type' => "string", 
+	'deft' => "" },
+      { 'name' => "noscaleup",
+	'desc' => "{ImagePlug.noscaleup}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "thumbnail",
+	'desc' => "{PagedImgPlug.thumbnail}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "thumbnailsize",
+	'desc' => "{ImagePlug.thumbnailsize}",
+	'type' => "int",
+	'deft' => "100",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "thumbnailtype",
+	'desc' => "{ImagePlug.thumbnailtype}",
+	'type' => "string",
+	'deft' => "gif",
+	'reqd' => "no" },
+      { 'name' => "screenview",
+	'desc' => "{PagedImgPlug.screenview}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "screenviewsize",
+	'desc' => "{PagedImgPlug.screenviewsize}",
+	'type' => "int",
+        'deft' => "500",
+        'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "screenviewtype",
+	'desc' => "{PagedImgPlug.screenviewtype}",
+	'type' => "string",
+	'deft' => "jpg",
+	'reqd' => "no" },
+      { 'name' => "converttotype",
+	'desc' => "{ImagePlug.converttotype}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "minimumsize",
+	'desc' => "{ImagePlug.minimumsize}",
+	'type' => "int",
+	'deft' => "100",
+	'range' => "1,",
+	'reqd' => "no" },
+      { 'name' => "headerpage",
+	'desc' => "{PagedImgPlug.headerpage}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "documenttype",
+	'desc' => "{PagedImgPlug.documenttype}",
+	'type' => "enum",
+	'list' => $type_list,
+	'deft' => "paged",
+	'reqd' => "no" } ];
+
+
+my $options = { 'name'     => "PagedImgPlug",
+		'desc'     => "{PagedImgPlug.desc}",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = (defined $hashArgOptLists)? new XMLPlug($pluginlist,$inputargs,$hashArgOptLists): new XMLPlug($pluginlist,$inputargs);
+    
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^\.item$^;
+}
+
+# want to block everything except the .item ones 
+# but instead we will block images and txt files
+sub get_default_block_exp {
+    my $self = shift (@_);
+    
+    return q^(?i)(\.jpe?g|\.gif|\.png|\.tif?f|\.te?xt|~)$^
+}
+
+# Create the thumbnail and screenview images, and discover the Image's
+# size, width, and height using the convert utility.
+sub process_image {
+    my $self = shift (@_);
+    my $filename = shift (@_); # filename with full path
+    my $srcfile = shift (@_); # filename without path
+    my $doc_obj = shift (@_);
+    my $section = shift (@_); #the current section
+    my $rotate = shift (@_); # whether to rotate the image or not
+    $rotate = 0 unless defined $rotate;
+    
+    my $top=0;
+    if ($section eq $doc_obj->get_top_section()) {
+	$top=1;
+    }
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+
+    # check the filename is okay
+    return 0 if ($srcfile eq "" || $filename eq "");
+    
+    my $minimumsize = $self->{'minimumsize'};
+    if (defined $minimumsize && (-s $filename < $minimumsize)) {
+        print $outhandle "PagedImgPlug: \"$filename\" too small, skipping\n"
+	    if ($verbosity > 1);
+    }
+
+    # Convert the image to a new type (if required), and rotate if required.
+    my $converttotype = $self->{'converttotype'};
+    my $originalfilename = "";  # only set if we do a conversion
+    my $type = "unknown";
+    my $converted = 0;
+    my $rotated=0;
+
+    if ($converttotype ne "" && $filename !~ /$converttotype$/) {
+	$converted=1;
+	$originalfilename = $filename;
+	my $filehead = &util::get_tmp_filename();
+	$filename = $filehead . ".$converttotype";
+	my $n = 1;
+	while (-e $filename) {
+	    $filename = "$filehead$n\.$converttotype";
+	    $n++;
+	}
+	$self->{'tmp_filename1'} = $filename;
+
+	my $rotate_option = "";
+	if ($rotate eq "r") {
+	    $rotate_option = "-rotate 180 ";
+	}
+	
+	my $command = "convert -verbose \"$originalfilename\" $rotate_option \"$filename\"";
+	print $outhandle "CONVERT: $command\n" if ($verbosity > 2);
+	my $result = '';
+	$result = `$command`;
+	print $outhandle "CONVERT RESULT = $result\n" if ($verbosity > 2);
+
+	$type = $converttotype;
+    } elsif ($rotate eq "r") {
+	$rotated=1;
+	$originalfilename = $filename;
+	$filename  = &util::get_tmp_filename();
+	
+	my $command = "convert \"$originalfilename\" -rotate 180 \"$filename\"";
+	print $outhandle "ROTATE: $command\n" if ($verbosity > 2);
+	my $result = '';
+	$result = `$command`;
+	print $outhandle "ROTATE RESULT = $result\n" if ($verbosity > 2);
+	
+    } 
+    
+
+    # Add the image metadata
+    my $file; # the new file name
+    my $id = $srcfile;
+    $id =~ s/\.([^\.]*)$//;  # the new file name without an extension
+    if ($converted) {
+	# we have converted the image
+	# add on the new extension
+	$file .= "$id.$converttotype";
+    } else {
+	$file = $srcfile;
+    }
+    
+    my $url =$file; # the new file name prepared for a url
+    my $srcurl = $srcfile;
+    $url =~ s/ /%20/g;
+    $srcurl =~ s/ /%20/g;
+    
+    $doc_obj->add_metadata ($section, "Image", $url);
+
+    # Also want to set filename as 'Source' metadata to be
+    # consistent with other plugins
+    $doc_obj->add_metadata ($section, "Source", $srcurl);
+
+    my ($image_type, $image_width, $image_height, $image_size) 
+	= &identify($filename, $outhandle, $verbosity);
+
+    $doc_obj->add_metadata ($section, "ImageType",   $image_type);
+    $doc_obj->add_metadata ($section, "ImageWidth",  $image_width);
+    $doc_obj->add_metadata ($section, "ImageHeight", $image_height);
+    $doc_obj->add_metadata ($section, "ImageSize",   $image_size);
+    $doc_obj->add_metadata ($section, "FileFormat", "PagedImg");
+    
+    if ($type eq "unknown" && $image_type) {
+	$type = $image_type;
+    }
+
+    if ($top) {
+	$doc_obj->add_metadata ($section, "srclink", 
+				"<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\">");
+	$doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\">");
+
+    } else {
+	$doc_obj->add_metadata ($section, "srclink", 
+				"<a href=\"_httpcollection_/index/assoc/[parent(Top):assocfilepath]/[Image]\">");
+	$doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpcollection_/index/assoc/[parent(Top):assocfilepath]/[Image]\">");
+
+    }
+    $doc_obj->add_metadata ($section, "/srclink", "</a>");
+
+
+    # Add the image as an associated file
+    $doc_obj->associate_file($filename,$file,"image/$type",$section);
+    print $outhandle "associating file $filename as name $file\n" if ($verbosity > 2);
+
+    if ($self->{'thumbnail'}) {
+	# Make the thumbnail image
+	my $thumbnailsize = $self->{'thumbnailsize'} || 100;
+	my $thumbnailtype = $self->{'thumbnailtype'} || 'gif';
+	
+	my $filehead = &util::get_tmp_filename();
+	my $thumbnailfile = $filehead . ".$thumbnailtype";
+	my $n=1;
+	while (-e $thumbnailfile) {
+	    $thumbnailfile = $filehead . $n . ".$thumbnailtype";
+	    $n++;
+	}
+	
+	$self->{'tmp_filename2'} = $thumbnailfile;
+	
+	# Generate the thumbnail with convert
+	my $command = "convert  -verbose -geometry $thumbnailsize" 
+	    . "x$thumbnailsize \"$filename\" \"$thumbnailfile\"";
+	print $outhandle "THUMBNAIL: $command\n" if ($verbosity > 2);
+	my $result = '';
+	$result = `$command 2>&1` ;
+	print $outhandle "THUMB RESULT: $result\n" if ($verbosity > 2);
+	
+	# Add the thumbnail as an associated file ...
+	if (-e "$thumbnailfile") { 
+	    $doc_obj->associate_file("$thumbnailfile", $id."thumb.$thumbnailtype", "image/$thumbnailtype",$section);
+	    $doc_obj->add_metadata ($section, "ThumbType", $thumbnailtype);
+	    $doc_obj->add_metadata ($section, "Thumb", $id."thumb.$thumbnailtype");
+	    if ($top) {
+		$doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Thumb]\" width=[ThumbWidth] height=[ThumbHeight]>");
+	    } else {
+		$doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpcollection_/index/assoc/[parent(Top):assocfilepath]/[Thumb]\" width=[ThumbWidth] height=[ThumbHeight]>");
+	    }
+	}
+	
+	# Extract Thumnail metadata from convert output
+	if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
+	    $doc_obj->add_metadata ($section, "ThumbWidth", $1);
+	    $doc_obj->add_metadata ($section, "ThumbHeight", $2);
+	}
+    }
+    # Make a screen-sized version of the picture if requested
+    if ($self->{'screenview'}) {
+	
+	# To do: if the actual image is smaller than the screenview size,
+	# we should use the original !
+
+	my $screenviewsize = $self->{'screenviewsize'} || 500;
+	my $screenviewtype = $self->{'screenviewtype'} || 'jpeg';
+	my $filehead = &util::get_tmp_filename();
+	my $screenviewfilename = $filehead . ".$screenviewtype";
+	my $n=1;
+	while (-e $screenviewfilename) {
+	    $screenviewfilename = "$filehead$n\.$screenviewtype";
+	    $n++;
+	}
+	$self->{'tmp_filename3'} = $screenviewfilename;
+
+	# make the screenview image
+	my $command = "convert  -verbose -geometry $screenviewsize" 
+	    . "x$screenviewsize \"$filename\" \"$screenviewfilename\"";
+	print $outhandle "SCREENVIEW: $command\n" if ($verbosity > 2);
+	my $result = "";
+	$result = `$command  2>&1` ;
+	print $outhandle "SCREENVIEW RESULT: $result\n" if ($verbosity > 3);
+
+	# get screenview dimensions, size and type
+        if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
+	    $doc_obj->add_metadata ($section, "ScreenWidth", $1);
+	    $doc_obj->add_metadata ($section, "ScreenHeight", $2);
+	}elsif ($result =~ m/([0-9]+)x([0-9]+)/) {
+	    #if the image hasn't changed size,  the previous regex doesn't match
+	    $doc_obj->add_metadata ($section, "ScreenWidth", $1);
+	    $doc_obj->add_metadata ($section, "ScreenHeight", $2);
+	}
+    
+	#add the screenview as an associated file ...
+	if (-e "$screenviewfilename") { 
+	    $doc_obj->associate_file("$screenviewfilename", $id."sv.$screenviewtype",
+				     "image/$screenviewtype",$section);
+	    print $outhandle "associating screen file $screenviewfilename as name $id sv.$screenviewtype\n" if ($verbosity > 2);
+	    
+	    $doc_obj->add_metadata ($section, "ScreenType", $screenviewtype);
+	    $doc_obj->add_metadata ($section, "Screen", $id."sv.$screenviewtype");
+
+	    if ($top) {
+		$doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>");
+	    } else {
+		$doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpcollection_/index/assoc/[parent(Top):assocfilepath]/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>");
+
+	    }
+	} else {
+	    print $outhandle "PagedImgPlug: couldn't find \"$screenviewfilename\"\n";
+	}
+    }
+
+    return $type;
+
+
+}
+
+
+
+# Discover the characteristics of an image file with the ImageMagick
+# "identify" command.
+
+sub identify { 
+    my ($image, $outhandle, $verbosity) = @_;
+
+    # Use the ImageMagick "identify" command to get the file specs
+    my $command = "identify \"$image\" 2>&1";
+    print $outhandle "$command\n" if ($verbosity > 2);
+    my $result = '';
+    $result = `$command`;
+    print $outhandle "$result\n" if ($verbosity > 3);
+
+    # Read the type, width, and height
+    my $type =   'unknown';
+    my $width =  'unknown';
+    my $height = 'unknown';
+
+    my $image_safe = quotemeta $image;
+    if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) {
+	$type = $1;
+	$width = $2;
+	$height = $3;
+    }
+	
+    # Read the size
+    my $size = "unknown";
+    if ($result =~ m/^.* ([0-9]+)b/) {
+	$size = $1;
+    } elsif ($result =~ m/^.* ([0-9]+)kb/) {
+	$size = 1024 * $1;
+    }
+
+    print $outhandle "file: $image:\t $type, $width, $height, $size\n" 
+	if ($verbosity > 3);
+
+    # Return the specs
+    return ($type, $width, $height, $size);
+}
+
+
+# The PagedImgPlug read() function. This function does all the right things
+# to make general options work for a given plugin. It calls the process()
+# function which does all the work specific to a plugin (like the old
+# read functions used to do). Most plugins should define their own 
+# process() function and let this read() function keep control.
+#
+# PagedImgPlug overrides read() because there is no need to read the actual 
+# text of the file in, because the contents of the file is not text...
+#
+# Return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+
+sub read_into_doc_obj {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle   = $self->{'outhandle'};
+    my $smart_block = $self->{'smart_block'};
+    
+    my $filename = &util::filename_cat($base_dir, $file);
+    
+    if ($self->associate_with($file,$filename,$metadata)) {
+	# a form of smart block
+	$self->{'num_blocked'} ++;
+	return 0; # blocked
+    }
+    
+    if ($smart_block) {
+	if (defined $self->{'file_blocks'}->{$filename} && $self->{'file_blocks'}->{$filename} == 1){
+	    $self->{'num_blocked'} ++;
+	    return 0; # blocked
+	}
+    } elsif ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return 0; # blocked
+    }
+    
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    
+    print $outhandle "PagedImgPlug processing \"$filename\"\n"
+	if $self->{'verbosity'} > 1;
+    print STDERR "<Processing n='$file' p='PagedImgPlug'>\n" if ($gli);
+    
+    # here we need to decide if we have an old text .item file, or a new xml 
+    # .item file - for now the test is if the first non-empty line is 
+    # <PagedDocument> then its xml
+    my $xml_version = 0;
+    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
+    
+    my $backup_filename = "backup.item";
+    open (BACKUP,">$backup_filename")|| die "couldn't write to $backup_filename\n";
+    my $line = "";
+    my $num = 0;
+    $line = <ITEMFILE>;
+    while ($line !~ /\w/) {
+	$line = <ITEMFILE>;
+    }
+    chomp $line;
+    if ($line =~ /<PagedDocument/) {
+	$xml_version = 1;
+    }
+    close ITEMFILE;
+    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
+    #Tidy up the item file some metadata title contains \vt-vertical tab
+    while ($line = <ITEMFILE>) {
+	$line =~ s/\x0B+//ig;
+	$line =~ s/&/&amp;/g;
+	print BACKUP ($line);
+    }
+    close ITEMFILE;
+    close BACKUP;
+    &File::Copy::copy ($backup_filename, $filename);
+    &util::rm($backup_filename);
+    #print STDERR "xml version = $xml_version\n";
+    my $doc_obj;
+    if ($xml_version) {
+	$file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+	$self->{'file'} = $file;
+	$self->{'filename'} = $filename;
+	$self->{'processor'} = $processor;
+	$self->{'metadata'} = $metadata;
+	$self->{'gli'} = $gli;
+	eval {
+	    $@ = "";
+	    my $xslt = $self->{'xslt'};
+	    if (defined $xslt && ($xslt ne "")) {
+		# perform xslt
+		my $transformed_xml = $self->apply_xslt($xslt,$filename);
+		
+		# feed transformed file (now in memory as string) into XML parser
+		#$self->{'parser'}->parse($transformed_xml);
+		$self->parse_string($transformed_xml);
+	    }
+	    else {
+		#$self->{'parser'}->parsefile($filename);
+		$self->parse_file($filename);
+	    }
+	};
+	
+	
+	
+	if ($@) {
+	    
+	    # parsefile may either croak somewhere in XML::Parser (e.g. because
+	    # the document is not well formed) or die somewhere in XMLPlug or a
+	    # derived plugin (e.g. because we're attempting to process a
+	    # document whose DOCTYPE is not meant for this plugin). For the
+	    # first case we'll print a warning and continue, for the second
+	    # we'll just continue quietly
+	    
+	    print STDERR "**** XML Parse Error is: $@\n";
+	    
+	    my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
+	    if (defined $msg) {	
+		my $outhandle = $self->{'outhandle'};
+		my $plugin_name = ref ($self);
+		print $outhandle "$plugin_name failed to process $file ($msg)\n";
+	    }
+
+	    # reset ourself for the next document
+	    $self->{'section_level'}=0;
+	    print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	    return -1; # error during processing
+	}
+	$doc_obj = $self->{'doc_obj'};
+    } else {
+	my ($dir);
+	($dir, $file) = $filename =~ /^(.*?)([^\/\\]*)$/;
+
+	#process the .item file
+	$doc_obj = $self->process_item($filename, $dir, $file, $processor);
+	
+    }
+    
+    if ($self->{'cover_image'}) {
+	$self->associate_cover_image($doc_obj, $filename);
+    }
+
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+    #my $text="";
+    # do plugin specific processing of doc_obj
+    #unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+	#print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	#return -1;
+    #}
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    $self->{'num_processed'}++;
+    return (1,$doc_obj);
+}
+
+sub read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
+
+    if ((defined $process_status) && ($process_status == 1)) {
+	# process the document
+	$processor->process($doc_obj);
+
+	#if(defined($self->{'places_filename'})){
+	#    &util::rm($self->{'places_filename'});
+	#    $self->{'places_filename'} = undef;
+	#}
+	#$self->{'num_processed'} ++;
+	undef $doc_obj; 
+    }
+    
+    # clean up temporary files - we do this here instead of in  
+    # process_image becuase associated files aren't actually copied 
+    # until after process has been run.
+    if (defined $self->{'tmp_filename1'} && 
+	-e $self->{'tmp_filename1'}) {
+	&util::rm($self->{'tmp_filename1'})
+    }
+    if (defined $self->{'tmp_filename2'} && 
+	-e $self->{'tmp_filename2'}) {
+	&util::rm($self->{'tmp_filename2'})
+    }
+    if (defined $self->{'tmp_filename3'} && 
+	-e $self->{'tmp_filename3'}) {
+      &util::rm($self->{'tmp_filename3'})
+    }
+    # if process_status == 1, then the file has been processed.
+    return $process_status;
+}
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+    $self->{'element'} = $element;
+    
+    my $doc_obj = $self->{'doc_obj'};
+    if ($element eq "PagedDocument") {
+	$self->{'current_section'} = $doc_obj->get_top_section();
+    } elsif ($element eq "PageGroup" || $element eq "Page") {
+	# create a new section as a child
+	$self->{'current_section'} = $doc_obj->insert_section($doc_obj->get_end_child($self->{'current_section'}));
+	$self->{'num_pages'}++;
+	# assign pagenum as  what??
+	my $pagenum = $_{'pagenum'}; #TODO!!
+	$doc_obj->set_utf8_metadata_element($self->{'current_section'}, 'PageNum', $pagenum);
+	my ($imgfile) = $_{'imgfile'};
+	if (defined $imgfile) {
+	    $self->process_image($self->{'base_dir'}.$imgfile, $imgfile, $doc_obj, $self->{'current_section'});
+	}
+	my ($txtfile) = $_{'txtfile'};
+	if (defined($txtfile)&& $txtfile ne "") {
+	    $self->process_text ($self->{'base_dir'}.$txtfile, $txtfile, $doc_obj, $self->{'current_section'});
+	} else {
+	    # otherwise add in some dummy text 
+	    $doc_obj->add_text($self->{'current_section'}, &gsprintf::lookup_string("{BasPlug.dummy_text}"));
+	}
+    } elsif ($element eq "Metadata") {
+	$self->{'metadata_name'} = $_{'name'};
+    }
+}
+
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+    
+    my $doc_obj = $self->{'doc_obj'};
+    if ($element eq "Page" || $element eq "PageGroup") {
+	# if Title hasn't been assigned, set PageNum as Title
+	if (!defined $doc_obj->get_metadata_element ($self->{'current_section'}, "Title") && defined $doc_obj->get_metadata_element ($self->{'current_section'}, "PageNum" )) {
+	    $doc_obj->add_utf8_metadata ($self->{'current_section'}, "Title", $doc_obj->get_metadata_element ($self->{'current_section'}, "PageNum" ));
+	}
+	# move the current section back to the parent
+	$self->{'current_section'} = $doc_obj->get_parent_section($self->{'current_section'});
+    } elsif ($element eq "Metadata") {
+	
+	$doc_obj->add_utf8_metadata ($self->{'current_section'}, $self->{'metadata_name'}, $self->{'metadata_value'});
+	$self->{'metadata_name'} = "";
+	$self->{'metadata_value'} = "";
+
+    }
+    # otherwise we ignore the end tag
+}
+
+
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_; 
+
+    if ($self->{'element'} eq "Metadata") {
+   	$self->{'metadata_value'} .= $_;
+    }
+}
+
+sub xml_doctype {
+}
+
+sub open_document {
+    my $self = shift(@_);
+    
+    # create a new document
+    $self->{'doc_obj'} = new doc ($self->{'filename'}, "indexed_doc");
+    my $doc_obj = $self->{'doc_obj'};
+    $doc_obj->set_OIDtype ($self->{'processor'}->{'OIDtype'});
+    my ($dir, $file) = $self->{'filename'} =~ /^(.*?)([^\/\\]*)$/;
+    $self->{'base_dir'} = $dir;
+    $self->{'num_pages'} = 0;
+    my $topsection = $doc_obj->get_top_section();
+    if ($self->{'documenttype'} eq 'paged') {
+	# set the gsdlthistype metadata to Paged - this ensures this document will
+	# be treated as a Paged doc, even if Titles are not numeric
+    
+	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Paged");
+    } else {
+	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Hierarchy");
+    }
+
+    $doc_obj->add_metadata ($topsection, "Source", $file);
+    if ($self->{'headerpage'}) {
+	$doc_obj->add_text($topsection, &gsprintf::lookup_string("{BasPlug.dummy_text}"));
+    }
+
+}
+
+sub close_document {
+    my $self = shift(@_);
+    my $doc_obj = $self->{'doc_obj'};
+        
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "PagedImg");
+    
+    # add numpages metadata
+    $doc_obj->set_utf8_metadata_element ($doc_obj->get_top_section(), 'NumPages', $self->{'num_pages'});
+
+    # add an OID
+    $doc_obj->set_OID();
+    
+}
+
+sub process_item {
+    my $self = shift (@_);
+    my ($filename, $dir, $file, $processor) = @_;
+
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+    my $topsection = $doc_obj->get_top_section();
+
+    if ($self->{'documenttype'} eq 'paged') {
+	# set the gsdlthistype metadata to Paged - this ensures this document will
+	# be treated as a Paged doc, even if Titles are not numeric
+	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Paged");
+    } else {
+	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Hierarchy");
+    }
+
+    $doc_obj->add_metadata ($topsection, "Source", $file);
+
+    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
+    my $line = "";
+    my $num = 0;
+    while (defined ($line = <ITEMFILE>)) {
+	next unless $line =~ /\w/;
+	chomp $line;
+	if ($line =~ /^<([^>]*)>\s*(.*?)\s*$/) {
+	    $doc_obj->set_utf8_metadata_element ($topsection, $1, $2);
+	    #$meta->{$1} = $2;
+	} else {
+	    $num++;
+	    # line should be like page:imagefilename:textfilename:r - the r is optional -> means rotate the image 180 deg
+	    $line =~ s/^\s+//; #remove space at the front
+	    $line =~ s/\s+$//; #remove space at the end
+	    my ($pagenum, $imgname, $txtname, $rotate) = split /:/, $line;
+	    
+	    # create a new section for each image file
+	    my $cursection = $doc_obj->insert_section($doc_obj->get_end_child($topsection));
+	    # the page number becomes the Title
+	    $doc_obj->set_utf8_metadata_element($cursection, 'Title', $pagenum);
+	    # process the image for this page
+	    my $result = $self->process_image($dir.$imgname, $imgname, $doc_obj, $cursection, $rotate);
+	    
+	    if (!defined $result)
+	    {
+		print "PagedImgPlug: couldn't process image \"$dir.$imgname\" for item \"$filename\"\n";
+	    }
+	    
+	    # process the text file if one is there
+	    if (defined $txtname && $txtname ne "") {
+		$result = undef;
+		$result = $self->process_text ($dir.$txtname, $txtname, $doc_obj, $cursection);
+		if (!defined $result) {
+		    print "PagedImgPlug: couldn't process text file \"$dir.$txtname\" for item \"$filename\"\n";
+		}
+	    } else {
+		# otherwise add in some dummy text 
+		$doc_obj->add_text($cursection, &gsprintf::lookup_string("{BasPlug.dummy_text}"));
+	    }
+	}
+    }
+    
+    close ITEMFILE;
+
+    # if we want a header page, we need to add some text into the top section, otherwise this section will become invisible
+    if ($self->{'headerpage'}) {
+	$doc_obj->add_text($topsection, &gsprintf::lookup_string("{BasPlug.dummy_text}"));
+    }
+    $file =~ s/\.item//i;
+    $doc_obj->set_OID ();
+    # add numpages metadata
+    $doc_obj->set_utf8_metadata_element ($topsection, 'NumPages', "$num");
+    return $doc_obj;
+}
+
+sub process_text {
+    my $self = shift (@_);
+    my ($fullpath, $file, $doc_obj, $cursection) = @_;
+
+    # Do encoding stuff
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($fullpath);
+
+    my $text="";
+    &BasPlug::read_file($self, $fullpath, $encoding, $language, \$text);
+    if (!length ($text)) {
+        my $plugin_name = ref ($self);
+        print "PagedImgPlug: ERROR: $fullpath contains no text\n" if $self->{'verbosity'};
+        return 0;
+    }
+
+    # we need to escape the escape character, or else mg will convert into
+    # eg literal newlines, instead of leaving the text as '\n'
+    $text =~ s/\\/\\\\/g; # macro language
+    $text =~ s/_/\\_/g; # macro language
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+ 
+    # insert preformat tags and add text to document object
+    $doc_obj->add_utf8_text($cursection, "<pre>\n$text\n</pre>");
+    
+    return 1;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ProCitePlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ProCitePlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ProCitePlug.pm	(revision 21112)
@@ -0,0 +1,241 @@
+###########################################################################
+#
+# ProCitePlug.pm -- A plugin for (exported) ProCite databases
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 1999-2004 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package ProCitePlug;
+
+
+use multiread;
+use SplitPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+# ProCitePlug is a sub-class of SplitPlug
+sub BEGIN {
+    @ProCitePlug::ISA = ('SplitPlug');
+}
+
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_split_exp(),
+	'reqd' => "no" }
+      ];
+
+my $options = { 'name'     => "ProCitePlug",
+		'desc'     => "{ProCitePlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes exported ProCite files with the suffix ".txt"
+sub get_default_process_exp
+{
+    return q^(?i)(\.txt)$^;
+}
+
+
+# This plugin splits the input text at every line
+sub get_default_split_exp
+{
+    return q^\n^;
+}
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+
+my %crazy_workform_mapping =
+    ( "A", "Book, Long Form",
+      "B", "Book, Short Form",
+      "C", "Journal, Long Form",
+      "D", "Journal, Short Form",
+      "E", "Report",
+      "F", "Newspaper",
+      "G", "Dissertation",
+      "H", "Trade Catalog",
+      "I", "Letter (Correspondence)",
+      "J", "Manuscript",
+      "K", "Conference Proceedings",
+      "L", "Map",
+      "M", "Music Score",
+      "N", "Sound Recording",
+      "O", "Motion Picture",
+      "P", "Audiovisual Material",
+      "Q", "Video Recording",
+      "R", "Art Work",
+      "S", "Computer Program",
+      "T", "Data File" );
+
+
+sub read_file
+{
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    # Store the workform definitions for this file
+    my %workform_definitions = ();
+
+    # Read the contents of the file into $textref
+    open(PROCITE_FILE, "<$filename");
+    my $reader = new multiread();
+    $reader->set_handle ('ProCitePlug::PROCITE_FILE');
+    $reader->set_encoding ($encoding);
+    $reader->read_file ($textref);
+    close(PROCITE_FILE);
+
+    # Read the workform definitions at the start of the file
+    while ($$textref =~ /^\<Workform Definition\>/) {
+	# Remove the workform definition line so it is not processed later as a record
+	$$textref =~ s/^\<Workform Definition\>(.*)\n//;
+	my $workform_definition = $1;
+
+	# Parse the workform definitions and store them for later
+	$workform_definition =~ s/^\"([^\"]*)\",//;
+	my $workform_name = $1;
+
+	my @workform_values;
+	while ($workform_definition !~ /^$/) {
+	    $workform_definition =~ s/^\"([^\"]*)\",?//;
+	    my $workform_field = $1;
+	    push(@workform_values, $workform_field);
+	}
+
+	# Remember this workform definition for when we're reading the records
+	$workform_definitions{$workform_name} = \@workform_values;
+    }
+
+    $self->{'workform_definitions'}->{$filename} = \%workform_definitions;
+}
+
+
+sub process
+{
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $cursection = $doc_obj->get_top_section();
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='ProCitePlug'>\n" if ($gli);
+    print $outhandle "ProCitePlug: processing $file\n"
+  	if ($self->{'verbosity'}) > 1;
+
+    # Build up an HTML view of the record for easy display at run-time
+    my $html_record = "";
+
+    # Read the record's workform indicator and record number
+    $$textref =~ s/^\"([^\"]*)\",\"([^\"]*)\",//;
+    my $workform_indicator = $1;
+    my $recordnum = $2;
+
+    # If necessary, map the workform indicator into something useful
+    if ($crazy_workform_mapping{$workform_indicator}) {
+	$workform_indicator = $crazy_workform_mapping{$workform_indicator};
+    }
+
+    # Check we know about the workform of this record
+    my %workform_definitions = %{$self->{'workform_definitions'}->{$filename}};
+    if (!$workform_definitions{$workform_indicator}) {
+	print STDERR "Unknown workform!\n";
+	return 0;
+    }
+
+    # Store the full record as the document text
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    # Store workform and record number as metadata
+    $doc_obj->add_utf8_metadata($cursection, "pc.Workform", $workform_indicator);
+    $doc_obj->add_utf8_metadata($cursection, "pc.RecordNumber", $recordnum);
+    
+    # Store FileFormat metadata
+    $doc_obj->add_metadata($cursection, "FileFormat", "ProCite");
+
+    $html_record .= "<tr><td valign=top><b>Record Number: </b></td><td valign=top>$recordnum</td></tr>";
+
+    my @workform_values = @{$workform_definitions{$workform_indicator}};
+
+    # Read each field (surrounded by quotes) of the record
+    my $fieldnum = 0;
+    while ($$textref !~ /^$/) {
+	$$textref =~ s/^\"([^\"]*)\",?//;
+	my $field_value_raw = $1;
+
+	# Add non-empty metadata values to the document
+	unless ($field_value_raw eq "") {
+	    # Add the display name of the metadata field for format statement convenience
+	    my $field_name = $workform_values[$fieldnum];
+	    unless ($field_name eq "---") {
+		my $meta_name = "pc.Field" . ($fieldnum + 1) . "Name";
+		$doc_obj->add_utf8_metadata($cursection, $meta_name, $field_name);
+	    }
+
+	    $html_record .= "<tr><td valign=top><b>$field_name: </b></td><td valign=top>";
+
+	    # Multiple metadata values are separated with "//"
+	    foreach my $field_value (split(/\/\//, $field_value_raw)) {
+		my $meta_name = "pc.Field" . ($fieldnum + 1) . "Value";
+		$doc_obj->add_utf8_metadata($cursection, $meta_name, $field_value);
+
+		$html_record .= $field_value . "<br>";
+	    }
+
+	    $html_record .= "</td></tr>";
+	}
+
+	$fieldnum++;
+    }
+
+    # Store HTML view of record as metadata
+    $doc_obj->add_utf8_metadata($cursection, "HTMLDisplay", $html_record);
+
+    # Record was processed successfully
+    return 1;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/RTFPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/RTFPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/RTFPlug.pm	(revision 21112)
@@ -0,0 +1,109 @@
+###########################################################################
+#
+# RTFPlug.pm -- plugin for importing Rich Text Format files.
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package RTFPlug;
+
+use ConvertToPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @RTFPlug::ISA = ('ConvertToPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "description_tags",
+	'desc' => "{HTMLPlug.description_tags}",
+	'type' => "flag" }
+];
+
+my $options = { 'name'     => "RTFPlug",
+		'desc'     => "{RTFPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+ 
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+    if (!defined $secondary_plugin_options->{'TEXTPlug'}) {
+	$secondary_plugin_options->{'TEXTPlug'} = [];
+    }
+    if (!defined $secondary_plugin_options->{'HTMLPlug'}) {
+	$secondary_plugin_options->{'HTMLPlug'} = [];
+    }
+    my $text_options = $secondary_plugin_options->{'TEXTPlug'};
+    my $html_options = $secondary_plugin_options->{'HTMLPlug'};
+    
+    #$self->{'input_encoding'} = "utf8";
+    #$self->{'extract_language'} = 1;
+    push(@$text_options, "-input_encoding", "utf8");
+    push(@$text_options,"-extract_language");
+    if ($self->{'description_tags'} == 1) {
+	push(@$html_options, "-description_tags");
+    }
+    
+
+    $self = bless $self, $class;
+
+    $self->load_secondary_plugins($class,$secondary_plugin_options, $hashArgOptLists);
+
+    return $self;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+    return q^(?i)\.rtf$^;
+}
+    
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    return $self->process_type("rtf",$base_dir,$file,$doc_obj);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/RealMediaPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/RealMediaPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/RealMediaPlug.pm	(revision 21112)
@@ -0,0 +1,152 @@
+###########################################################################
+#
+# RealMediaPlug.pm -- Extract metadata from Real Media files
+#
+# Original code by Xin Gao
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package RealMediaPlug;
+
+
+use UnknownPlug;
+use rm::Header::PurePerl;
+
+use strict;
+no strict 'refs'; # make an exception so we can use variables as filehandles
+
+
+sub BEGIN {
+    @RealMediaPlug::ISA = ('UnknownPlug');
+}
+
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "RealMediaPlug",
+		'desc'     => "{RealMediaPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes Real Media files with the suffixes ".rm" and ".rmvb"
+sub get_default_process_exp
+{
+    return q^(?i)(\.rm|rmvb)$^;
+}
+
+
+sub new
+{
+    my ($class) = shift(@_);
+    my ($pluginlist, $inputargs, $hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if (defined $arguments) { push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); }
+    if (defined $options) { push(@{$hashArgOptLists->{"OptList"}}, $options); }
+
+    my $self = (defined $hashArgOptLists) ? new UnknownPlug($pluginlist, $inputargs, $hashArgOptLists) : new UnknownPlug($pluginlist, $inputargs);
+    
+    return bless $self, $class;
+}
+
+
+# do plugin specific processing of doc_obj
+sub read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    # filename is the full pathname of the file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{block_exp} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='RealMediaPlug'>\n" if ($gli);
+    print $outhandle "RealMediaPlug: processing $file\n"
+  	if ($self->{'verbosity'}) > 1;
+
+    # create a new index document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ("incremental");    # this is done to avoid hashing content of file
+    my $top_section = $doc_obj->get_top_section();
+
+    # replace spaces in filename with %20 in url for metadata entry
+    my $url = $file;
+    $url =~ s/ /%20/g;
+    # Source (filename) to be consistent with other plugins
+    $doc_obj->add_metadata($top_section, "Source", $url);
+
+    my $text = "";
+    my $real_media = rm::Header::PurePerl->new($filename);
+    foreach my $key (keys %{$real_media->info})
+    {
+	my $value = $real_media->info->{$key};
+	$doc_obj->add_metadata($top_section, $key, $value);
+	$text .= "$key: $value\n";
+    }
+
+    $doc_obj->add_utf8_text($top_section, "<pre>\n$text\n</pre>");
+
+    # srclink
+    $doc_obj->add_metadata($top_section, "FileFormat", "RealMedia");
+    $doc_obj->add_metadata($top_section, "srclink", "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[Source]\">");
+    $doc_obj->add_metadata($top_section, "/srclink", "</a>");
+    # srcicon (need to include "irmvideo.gif" in the greenstone images directory
+    $doc_obj->add_metadata($top_section, "srcicon", "<img src=\"_httpprefix_/images/irmvideo.gif\" title=\"Download\" border=0>");
+
+    # Add the actual file as an associated file
+    $doc_obj->associate_file($filename, $file, "RealMedia", $top_section);
+
+    # include any metadata passed in from previous plugins
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+
+    # do plugin specific processing of doc_obj
+    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata($doc_obj);
+
+    # add an OID
+    $doc_obj->set_OID();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'}++;
+    return 1;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/RecPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/RecPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/RecPlug.pm	(revision 21112)
@@ -0,0 +1,666 @@
+###########################################################################
+#
+# RecPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# RecPlug is a plugin which recurses through directories processing
+# each file it finds.
+
+# RecPlug has one option: use_metadata_files.  When this is set, it will
+# check each directory for an XML file called "metadata.xml" that specifies
+# metadata for the files (and subdirectories) in the directory.
+#
+# Here's an example of a metadata file that uses three FileSet structures
+# (ignore the # characters):
+
+#<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+#<!DOCTYPE DirectoryMetadata SYSTEM "http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd">
+#<DirectoryMetadata>
+#  <FileSet>
+#    <FileName>nugget.*</FileName>
+#    <Description>
+#      <Metadata name="Title">Nugget Point, The Catlins</Metadata>
+#      <Metadata name="Place" mode="accumulate">Nugget Point</Metadata>
+#    </Description>
+#  </FileSet>
+#  <FileSet>
+#    <FileName>nugget-point-1.jpg</FileName>
+#    <Description>
+#      <Metadata name="Title">Nugget Point Lighthouse, The Catlins</Metadata>
+#      <Metadata name="Subject">Lighthouse</Metadata>
+#    </Description>
+#  </FileSet>
+#  <FileSet>
+#    <FileName>kaka-point-dir</FileName>
+#    <Description>
+#      <Metadata name="Title">Kaka Point, The Catlins</Metadata>
+#    </Description>
+#  </FileSet>
+#</DirectoryMetadata>
+
+# Metadata elements are read and applied to files in the order they appear
+# in the file.
+#
+# The FileName element describes the subfiles in the directory that the
+# metadata applies to as a perl regular expression (a FileSet group may
+# contain multiple FileName elements). So, <FileName>nugget.*</FileName>
+# indicates that the metadata records in the following Description block
+# apply to every subfile that starts with "nugget".  For these files, a
+# Title metadata element is set, overriding any old value that the Title
+# might have had.
+#
+# Occasionally, we want to have multiple metadata values applied to a
+# document; in this case we use the "mode=accumulate" attribute of the
+# particular Metadata element.  In the second metadata element of the first
+# FileSet above, the "Place" metadata is accumulating, and may therefore be
+# given several values.  If we wanted to override these values and use a
+# single metadata element again, we could set the mode attribute to
+# "override" instead.  Remember: every element is assumed to be in override
+# mode unless you specify otherwise, so if you want to accumulate metadata
+# for some field, every occurance must have "mode=accumulate" specified.
+#
+# The second FileSet element above applies to a specific file, called
+# nugget-point-1.jpg.  This element overrides the Title metadata set in the
+# first FileSet, and adds a "Subject" metadata field.
+#
+# The third and final FileSet sets metadata for a subdirectory rather than
+# a file.  The metadata specified (a Title) will be passed into the
+# subdirectory and applied to every file that occurs in the subdirectory
+# (and to every subsubdirectory and its contents, and so on) unless the
+# metadata is explictly overridden later in the import.
+
+
+
+package RecPlug;
+
+use BasPlug;
+use plugin;
+use util;
+
+use File::Basename;
+use strict;
+no strict 'refs';
+
+BEGIN {
+    @RecPlug::ISA = ('BasPlug');
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use XMLParser;
+
+my $arguments =
+    [ { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_block_exp(),
+	'reqd' => "no" },
+      { 'name' => "use_metadata_files",
+	'desc' => "{RecPlug.use_metadata_files}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "recheck_directories",
+	'desc' => "{RecPlug.recheck_directories}",
+	'type' => "flag",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "RecPlug",
+		'desc'     => "{RecPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+my ($self);
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+    if ($self->{'use_metadata_files'}) {
+	# create XML::Parser object for parsing metadata.xml files
+	my $parser = new XML::Parser('Style' => 'Stream',
+				     'Handlers' => {'Char' => \&Char,
+						    'Doctype' => \&Doctype
+						    });
+
+	$self->{'parser'} = $parser;
+	$self->{'in_filename'} = 0;
+    }
+    
+    $self->{'subdir_extrametakeys'} = {};
+
+    return bless $self, $class;
+}
+
+sub begin {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+
+    my $proc_package_name = ref $processor;
+
+    if ($proc_package_name !~ /buildproc$/) {
+
+	# Only lookup timestamp info for import.pl
+
+	my $output_dir = $processor->getoutputdir();
+	my $archives_inf = &util::filename_cat($output_dir,"archives.inf");
+    
+	if ( -e $archives_inf ) {
+	    $self->{'inf_timestamp'} = -M $archives_inf;
+	}
+    }
+
+    $self->SUPER::begin($pluginfo, $base_dir, $processor, $maxdocs);
+}
+
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+    
+    return 1;
+}
+
+sub get_default_block_exp {
+    my $self = shift (@_);
+    
+    return 'CVS';
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+
+# This function passes around metadata hash structures.  Metadata hash
+# structures are hashes that map from a (scalar) key (the metadata element
+# name) to either a scalar metadata value or a reference to an array of
+# such values.
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $in_metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+    my $read_metadata_files = $self->{'use_metadata_files'};
+
+    # Calculate the directory name and ensure it is a directory and
+    # that it is not explicitly blocked.
+    my $dirname = $file;
+    $dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    return undef unless (-d $dirname);
+    return 0 if ($self->{'block_exp'} ne "" && $dirname =~ /$self->{'block_exp'}/);
+
+    # check to make sure we're not reading the archives or index directory
+    my $gsdlhome = quotemeta($ENV{'GSDLHOME'});
+    if ($dirname =~ m/^$gsdlhome\/.*?\/import.*?\/(archives|index)$/) {
+	print $outhandle "RecPlug: $dirname appears to be a reference to a Greenstone collection, skipping.\n";
+        return 0;
+    }
+    
+    # check to see we haven't got a cyclic path...
+    if ($dirname =~ m%(/.*){,41}%) {
+	print $outhandle "RecPlug: $dirname is 40 directories deep, is this a recursive path? if not increase constant in RecPlug.pm.\n";
+	return 0;
+    }
+    
+    # check to see we haven't got a cyclic path...
+    if ($dirname =~ m%.*?import/(.+?)/import/\1.*%) {
+	print $outhandle "RecPlug: $dirname appears to be in a recursive loop...\n";
+	return 0;
+    }
+    
+    if (($verbosity > 2) && ((scalar keys %$in_metadata) > 0)) {
+        print $outhandle "RecPlug: metadata passed in: ", 
+	join(", ", keys %$in_metadata), "\n";
+    }
+    
+    # Recur over directory contents.
+    my (@dir, $subfile);
+    my $count = 0;
+    
+    print $outhandle "RecPlug: getting directory $dirname\n" if ($verbosity);
+    
+    # find all the files in the directory
+    if (!opendir (DIR, $dirname)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
+	}
+	print $outhandle "RecPlug: WARNING - couldn't read directory $dirname\n";
+	return -1; # error in processing
+    }
+    @dir = readdir (DIR);
+    closedir (DIR);
+
+    # Re-order the files in the list so any directories ending with .all are moved to the end
+    for (my $i = scalar(@dir) - 1; $i >= 0; $i--) {
+	if (-d &util::filename_cat($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
+	    push(@dir, splice(@dir, $i, 1));
+	}
+    }
+
+    # read XML metadata files (if supplied)
+    my $additionalmetadata = 0;      # is there extra metadata available?
+    my %extrametadata;               # maps from filespec to extra metadata keys
+    my @extrametakeys;               # keys of %extrametadata in order read
+
+    my $dirsepre = &util::get_re_dirsep();
+    my $dirsep    = &util::get_dirsep();
+    my $local_dirname = $dirname;
+    $local_dirname =~ s/^$base_dir($dirsepre)//;
+    $local_dirname .= $dirsep;
+
+    if (defined $self->{'subdir_extrametakeys'}->{$local_dirname}) {
+	my $extrakeys = $self->{'subdir_extrametakeys'}->{$local_dirname};
+	foreach my $ek (@$extrakeys) {
+	    my $extrakeys_re  = $ek->{'re'};
+	    my $extrakeys_md  = $ek->{'md'};
+	    push(@extrametakeys,$extrakeys_re);
+	    $extrametadata{$extrakeys_re} = $extrakeys_md;
+	}
+	delete($self->{'subdir_extrametakeys'}->{$local_dirname});
+    }
+
+    if ($read_metadata_files) {
+	#read the directory "metadata.xml" file
+	my $metadatafile = &util::filename_cat ($dirname, 'metadata.xml');
+	if (-e $metadatafile) {
+	    print $outhandle "RecPlug: found metadata in $metadatafile\n" 
+		if ($verbosity);
+	    $self->read_metadata_xml_file($metadatafile, \%extrametadata, \@extrametakeys);
+	    $additionalmetadata = 1;
+	}
+    }
+    
+    # apply metadata pass for each of the files in the directory
+    my $out_metadata; 
+    my $num_files = scalar(@dir);
+    for (my $i = 0; $i < scalar(@dir); $i++) {
+	my $subfile = $dir[$i];
+	my $this_file_base_dir = $base_dir;
+	last if ($maxdocs != -1 && $count >= $maxdocs);
+	next if ($subfile =~ m/^\.\.?$/);
+	#next if ($read_metadata_files && $subfile =~ /metadata\.xml$/);
+
+	# Recursively read each $subfile
+	print $outhandle "RecPlug metadata recurring: $subfile\n" if ($verbosity > 2);
+	
+	$count += &plugin::metadata_read ($pluginfo, $this_file_base_dir,
+					  &util::filename_cat($file, $subfile),
+					  $out_metadata, \@extrametakeys, \%extrametadata,
+					  $processor, $maxdocs, $gli);
+	$additionalmetadata = 1;
+    }
+   
+    # filter out any extrametakeys that mention subdirectories and store
+    # for later use (i.e. when that sub-directory is being processed)
+
+    foreach my $ek (@extrametakeys) {
+	my ($subdir_re,$extrakey_dir) = &File::Basename::fileparse($ek);
+	$extrakey_dir =~ s/\\\./\./g; # remove RE syntax
+
+	my $dirsep_re = &util::get_re_dirsep();
+
+	if ($ek =~ m/$dirsep_re/) { # specifies at least one directory
+	    my $md = $extrametadata{$ek};
+
+	    my $subdir_extrametakeys = $self->{'subdir_extrametakeys'};
+
+	    my $subdir_rec = { 're' => $subdir_re, 'md' => $md };
+	    push(@{$subdir_extrametakeys->{$extrakey_dir}},$subdir_rec);
+	}
+    }
+    
+    # import each of the files in the directory
+    $count=0;
+    for (my $i = 0; $i <= scalar(@dir); $i++) {
+	# When every file in the directory has been done, pause for a moment (figuratively!)
+	# If the -recheck_directories argument hasn't been provided, stop now (default)
+	# Otherwise, re-read the contents of the directory to check for new files
+	#   Any new files are added to the @dir list and are processed as normal
+	#   This is necessary when documents to be indexed are specified in bibliographic DBs
+	#   These files are copied/downloaded and stored in a new folder at import time
+	if ($i == $num_files) {
+	    last unless $self->{'recheck_directories'};
+
+	    # Re-read the files in the directory to see if there are any new files
+	    last if (!opendir (DIR, $dirname));
+	    my @dirnow = readdir (DIR);
+	    closedir (DIR);
+
+	    # We're only interested if there are more files than there were before
+	    last if (scalar(@dirnow) <= scalar(@dir));
+
+	    # Any new files are added to the end of @dir to get processed by the loop
+	    my $j;
+	    foreach my $subfilenow (@dirnow) {
+		for ($j = 0; $j < $num_files; $j++) {
+		    last if ($subfilenow eq $dir[$j]);
+		}
+		if ($j == $num_files) {
+		    # New file
+		    push(@dir, $subfilenow);
+		}
+	    }
+	    # When the new files have been processed, check again
+	    $num_files = scalar(@dir);
+	}
+
+	my $subfile = $dir[$i];
+	my $this_file_base_dir = $base_dir;
+	last if ($maxdocs != -1 && ($count + $total_count) >= $maxdocs);
+	next if ($subfile =~ /^\.\.?$/);
+	next if ($read_metadata_files && $subfile =~ /metadata\.xml$/);
+
+	# Follow Windows shortcuts
+	if ($subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
+	    require Win32::Shortcut;
+	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $subfile));
+	    if ($shortcut) {
+		# The file to be processed is now the target of the shortcut
+		$this_file_base_dir = "";
+		$file = "";
+		$subfile = $shortcut->Path;
+	    }
+	}
+
+	# check for a symlink pointing back to a leading directory
+	if (-d "$dirname/$subfile" && -l "$dirname/$subfile") {
+	    # readlink gives a "fatal error" on systems that don't implement
+	    # symlinks. This assumes the the -l test above would fail on those.
+	    my $linkdest=readlink "$dirname/$subfile";
+	    if (!defined ($linkdest)) {
+		# system error - file not found?
+		warn "RecPlug: symlink problem - $!";
+	    } else {
+		# see if link points to current or a parent directory
+		if ($linkdest =~ m@^[\./\\]+$@ ||
+		    index($dirname, $linkdest) != -1) {
+		    warn "RecPlug: Ignoring recursive symlink ($dirname/$subfile -> $linkdest)\n";
+		    next;
+		    ;
+		}
+	    }
+	}
+
+	print $outhandle "RecPlug: preparing metadata for $subfile\n" if ($verbosity > 2);
+
+	# Make a copy of $in_metadata to pass to $subfile
+	$out_metadata = {};
+	&combine_metadata_structures($out_metadata, $in_metadata);
+
+	# Next add metadata read in XML files (if it is supplied)
+	if ($additionalmetadata == 1) {
+	    
+	    my ($filespec, $mdref);
+	    foreach $filespec (@extrametakeys) {
+		if ($subfile =~ /^$filespec$/) {
+		    print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
+			if ($verbosity > 2);
+		    $mdref = $extrametadata{$filespec};
+		    &combine_metadata_structures($out_metadata, $mdref);
+		}
+	    }
+	}
+
+
+	my $file_subfile = &util::filename_cat($file, $subfile);
+	my $filename_subfile 
+	    = &util::filename_cat($this_file_base_dir,$file_subfile);
+	if (defined $self->{'inf_timestamp'}) {
+	    my $inf_timestamp = $self->{'inf_timestamp'};
+
+	    if (! -d $filename_subfile) {
+		my $filename_timestamp = -M $filename_subfile;
+		if ($filename_timestamp > $inf_timestamp) {
+		    # filename has been around for longer than inf
+#####		    print $outhandle "**** Skipping $subfile\n";
+		    next;
+		}
+	    }
+	}
+
+	# Recursively read each $subfile
+	print $outhandle "RecPlug recurring: $subfile\n" if ($verbosity > 2);
+	
+	$count += &plugin::read ($pluginfo, $this_file_base_dir,
+				 $file_subfile,
+				 $out_metadata, $processor, $maxdocs, ($total_count + $count), $gli);
+    }
+
+    return $count;
+}
+
+
+
+# Read a manually-constructed metadata file and store the data
+# it contains in the $metadataref structure.
+#
+# (metadataref is a reference to a hash whose keys are filenames
+# and whose values are metadata hash structures.)  
+
+sub read_metadata_xml_file {
+    my $self = shift(@_);
+    my ($filename, $metadataref, $metakeysref) = @_;
+    $self->{'metadataref'} = $metadataref;
+    $self->{'metakeysref'} = $metakeysref;
+    
+    eval {
+	$self->{'parser'}->parsefile($filename);
+    };
+
+    if ($@) {
+	die "RecPlug: ERROR $filename is not a well formed metadata.xml file ($@)\n";
+    }
+}
+
+sub Doctype {
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+
+    # allow the short-lived and badly named "GreenstoneDirectoryMetadata" files 
+    # to be processed as well as the "DirectoryMetadata" files which should now
+    # be created by import.pl
+    die if ($name !~ /^(Greenstone)?DirectoryMetadata$/);
+}
+
+sub StartTag {
+    my ($expat, $element) = @_;
+
+    if ($element eq "FileSet") {
+	$self->{'saved_targets'} = [];
+	$self->{'saved_metadata'} = {};
+    }
+    elsif ($element eq "FileName") {
+	$self->{'in_filename'} = 1;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'metadata_name'} = $_{'name'};
+	if ((defined $_{'mode'}) && ($_{'mode'} eq "accumulate")) {
+	    $self->{'metadata_accumulate'} = 1;
+	} else {
+	    $self->{'metadata_accumulate'} = 0;
+	}
+    }
+}
+
+sub EndTag {
+    my ($expat, $element) = @_;
+
+    if ($element eq "FileSet") {
+	push (@{$self->{'metakeysref'}}, @{$self->{'saved_targets'}});
+	foreach my $target (@{$self->{'saved_targets'}}) {
+	    my $file_metadata = $self->{'metadataref'}->{$target};
+	    my $saved_metadata = $self->{'saved_metadata'};
+	    if (!defined $file_metadata) {
+		$self->{'metadataref'}->{$target} = $saved_metadata;
+	    }
+	    else {
+		$self->combine_metadata_structures($file_metadata,$saved_metadata);
+	    }
+	}
+    }
+    elsif ($element eq "FileName") {
+	$self->{'in_filename'} = 0;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'metadata_name'} = "";
+    }
+
+}
+
+sub store_saved_metadata
+{
+    my $self = shift(@_);
+    my ($mname,$mvalue,$md_accumulate) = @_;
+
+    if (defined $self->{'saved_metadata'}->{$mname}) {
+	if ($md_accumulate) {
+	    # accumulate mode - add value to existing value(s)
+	    if (ref ($self->{'saved_metadata'}->{$mname}) eq "ARRAY") {
+		push (@{$self->{'saved_metadata'}->{$mname}}, $mvalue);
+	    } else {
+		$self->{'saved_metadata'}->{$mname} = 
+		    [$self->{'saved_metadata'}->{$mname}, $mvalue];
+	    }
+	} else {
+	    # override mode
+	    $self->{'saved_metadata'}->{$mname} = $mvalue;
+	}
+    } else {
+	if ($md_accumulate) {
+	    # accumulate mode - add value into (currently empty) array
+	    $self->{'saved_metadata'}->{$mname} = [$mvalue];
+	} else {
+	    # override mode
+	    $self->{'saved_metadata'}->{$mname} = $mvalue;
+	}
+    }
+}
+
+
+sub Text {
+
+    if ($self->{'in_filename'}) {
+	# $_ == FileName content
+	push (@{$self->{'saved_targets'}}, $_);
+    }
+    elsif (defined ($self->{'metadata_name'}) && $self->{'metadata_name'} ne "") {
+	# $_ == Metadata content
+	my $mname = $self->{'metadata_name'};
+	my $mvalue = $_;
+	my $md_accumulate = $self->{'metadata_accumulate'};
+	$self->store_saved_metadata($mname,$mvalue,$md_accumulate);
+    }
+}
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {
+    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
+    $_[0]->{'Text'} .= $_[1];
+    return undef;
+}
+
+# Combine two metadata structures.  Given two references to metadata
+# element structures, add every field of the second ($mdref2) to the first
+# ($mdref1).  
+#
+# Afterwards $mdref1 will be updated, and $mdref2 will be unchanged.
+#
+# We have to be careful about the way we merge metadata when one metadata
+# structure is in "override" mode and one is in "merge" mode.  In fact, we
+# use the mode from the second structure, $mdref2, because it is generally
+# defined later (lower in the directory structure) and is therefore more
+# "local" to the document concerned.
+#
+# Another issue is the use of references to pass metadata around.  If we
+# simply copy one metadata structure reference to another, then we're
+# effectively just copyinga pointer, and changes to the new referene
+# will affect the old (copied) one also.  This also applies to ARRAY
+# references used as metadata element values (hence the "clonedata"
+# function below).
+
+sub combine_metadata_structures { 
+    my ($mdref1, $mdref2) = @_;
+    my ($key, $value1, $value2);
+
+    foreach $key (keys %$mdref2) {
+
+	$value1 = $mdref1->{$key};
+	$value2 = $mdref2->{$key};
+	
+	# If there is no existing value for this metadata field in 
+	# $mdref1, so we simply copy the value from $mdref2 over.
+	if (!defined $value1) {
+	    $mdref1->{$key} = &clonedata($value2);
+	} 
+	# Otherwise we have to add the new values to the existing ones.
+	# If the second structure is accumulated, then acculate all the
+	# values into the first structure
+	elsif ((ref $value2) eq "ARRAY") {
+	    # If the first metadata element is a scalar we have to 
+	    # convert it into an array before we add anything more.
+	    if ((ref $value1) ne 'ARRAY') {
+		$mdref1->{$key} = [$value1];
+		$value1 = $mdref1->{$key};
+	    }
+	    # Now add the value(s) from the second array to the first
+	    $value2 = &clonedata($value2);
+	    push @$value1, @$value2;
+	} 
+	# Finally, If the second structure is not an array erference, we
+	# know it is in override mode, so override the first structure.
+	else {
+	    $mdref1->{$key} = &clonedata($value2);
+	}
+    }
+}
+
+
+# Make a "cloned" copy of a metadata value.  
+# This is trivial for a simple scalar value,
+# but not for an array reference.
+
+sub clonedata {
+    my ($value) = @_;
+    my $result;
+
+    if ((ref $value) eq 'ARRAY') {
+	$result = [];
+	foreach my $item (@$value) {
+	    push @$result, $item;
+	}
+    } else {
+	$result = $value;
+    }
+    return $result;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/ReferPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ReferPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ReferPlug.pm	(revision 21112)
@@ -0,0 +1,313 @@
+###########################################################################
+#
+# ReferPlug.pm - a plugin for bibliography records in Refer format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2000 Gordon W. Paynter
+# Copyright 1999-2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# ReferPlug reads bibliography files in Refer format.
+#
+# by Gordon W. Paynter (gwp@cs.waikato.ac.nz), November 2000
+#
+# Loosely based on hcibib2Plug by Steve Jones (stevej@cs.waikato.ac.nz).
+# Which was based on EMAILPlug by Gordon Paynter (gwp@cs.waikato.ac.nz).
+# Which was based on old versions of HTMLplug and HCIBIBPlugby by Stefan
+# Boddie and others -- it's hard to tell what came from where, now.
+#
+#
+# ReferPlug creates a document object for every reference in the file.
+# It is a subclass of SplitPlug, so if there are multiple records, all
+# are read.
+#
+# Document text:
+#   The document text consists of the reference in Refer format
+#
+# Metadata:
+#	$Creator	%A	Author name
+#	$Title		%T	Title of article of book
+#	$Journal	%J	Title of Journal
+#	$Booktitle	%B	Title of book containing the publication
+#	$Report		%R	Type of Report, paper or thesis
+#	$Volume		%V	Volume Number of Journal
+#	$Number		%N	Number of Journal within Volume
+#	$Editor		%E	Editor name
+#	$Pages		%P	Page Number of article
+#	$Publisher	%I	Name of Publisher
+#	$Publisheraddr	%C	Publisher's address
+#	$Date		%D	Date of publication
+#	$Keywords	%K	Keywords associated with publication 
+#	$Abstract	%X	Abstract of publication
+#	$Copyright	%*	Copyright information for the article
+#
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package ReferPlug;
+
+use SplitPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+# ReferPlug is a sub-class of BasPlug.
+sub BEGIN {
+    @ReferPlug::ISA = ('SplitPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_split_exp() } 
+      ];
+
+my $options = { 'name'     => "ReferPlug",
+		'desc'     => "{ReferPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+# This plugin processes files with the suffix ".bib"
+sub get_default_process_exp {
+    return q^(?i)\.bib$^;
+}
+
+# This plugin splits the input text at blank lines
+sub get_default_split_exp {
+    return q^\n\s*\n^;
+}
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new SplitPlug($pluginlist,$inputargs,$hashArgOptLists): new SplitPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+# The process function reads a single bibliogrphic record and stores
+# it as a new document.
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    # Check that we're dealing with a valid Refer file
+    return undef unless ($$textref =~ /^\s*%/);
+
+    my $cursection = $doc_obj->get_top_section();
+    # Report that we're processing the file
+    print STDERR "<Processing n='$file' p='ReferPlug'>\n" if ($gli);
+    print $outhandle "ReferPlug: processing $file\n"
+	if ($self->{'verbosity'}) > 1;
+
+    my %field = ('H', 'Header',
+		 'A', 'Creator',
+		 'T', 'Title',
+		 'J', 'Journal',
+		 'B', 'Booktitle',
+		 'R', 'Report',
+		 'V', 'Volume',
+		 'N', 'Number',
+		 'E', 'Editor',
+		 'P', 'Pages',
+		 'I', 'Publisher',
+		 'C', 'PublisherAddress',
+		 'D', 'Date',
+		 'O', 'OtherInformation',
+		 'K', 'Keywords',
+		 'X', 'Abstract',
+		 '*', 'Copyright');
+
+    # Metadata fields 
+    my %metadata;
+    my ($id, $Creator, $Keywords, $text);
+    my @lines = split(/\n+/, $$textref);
+
+    
+    # Read and process each line in the bib file.
+    # Each file consists of a set of metadata items, one to each line
+    # with the Refer key followed by a space then the associated data
+    foreach my $line (@lines) {
+	
+	# Add each line.  Most lines consist of a field identifer and
+	# then data, and we simply store them, though we treat some
+	# of the fields a bit differently.
+
+	$line =~ s/\s+/ /g;
+	$text .= "$line\n";
+	# $ReferFormat .= "$line\n"; # what is this???
+	
+	next unless ($line =~ /^%[A-Z\*]/);
+	$id = substr($line,1,1);
+	$line =~ s/^%. //;
+   	
+	# Add individual authors in "Lastname, Firstname" format.
+	# (The full set of authors will be added below as "Creator".)
+	if ($id eq "A") {
+
+	    # Reformat and add author name
+	    my @words = split(/ /, $line);
+	    my $lastname = pop @words;
+	    my $firstname = join(" ",  @words);
+	    my $fullname = $lastname . ", " . $firstname;
+	    
+	    # Add each name to set of Authors
+	    if ($fullname =~ /\w/) {
+		$fullname = &text_into_html($fullname);
+		$doc_obj->add_metadata ($cursection, "Author", $fullname);
+	    }
+	}
+	# Add FileFormat as the metadata
+	$doc_obj->add_metadata($cursection,"FileFormat","Refer");
+
+	# Add individual keywords.
+	# (The full set of authors will be added below as "Keywords".)
+	if ($id eq "K") {
+	    my @keywordlist = split(/,/, $line);
+	    foreach my $k (@keywordlist) {
+		$k = lc($k);
+		$k =~ s/\s*$//; 
+		$k =~ s/^\s*//; 
+		if ($k =~ /\w/) {
+		    $k = &text_into_html($k);
+		    $doc_obj->add_metadata ($cursection, "Keyword", $k);
+		}
+	    } 
+	}
+	
+	# Add this line of metadata
+	$metadata{$id} .= "$line\n";
+    }
+
+
+
+    # Add the various field as metadata
+    my ($f, $name, $value);
+    foreach $f (keys %metadata) {
+	
+	next unless (defined $field{$f});
+	next unless (defined $metadata{$f});	
+
+	$name = $field{$f};
+	$value = $metadata{$f};
+
+	# Add the various field as metadata	
+	
+	# The Creator metadata is found by concatenating authors.
+	if ($f eq "A") {
+
+	    my @authorlist = split(/\n/, $value);
+	    my $lastauthor = pop @authorlist;
+	    my $Creator = "";
+	    if (scalar @authorlist) {
+		$Creator = join(", ", @authorlist) . "and $lastauthor";
+	    } else {
+		$Creator = $lastauthor;
+	    }
+
+	    if ($Creator =~ /\w/) {
+		$Creator = &text_into_html($Creator);
+		$doc_obj->add_metadata ($cursection, "Creator", $Creator);
+	    }
+	}
+
+	# The rest are added in a standard way
+	else {
+	    $value = &text_into_html($value);
+	    $doc_obj->add_metadata ($cursection, $name, $value);
+	}
+
+	# Books and Journals are additionally marked for display purposes
+	if ($f eq "B") {
+	    $doc_obj->add_metadata($cursection, "BookConfOnly", 1);
+	} elsif ($f eq "J") {
+	    $doc_obj->add_metadata($cursection, "JournalsOnly", 1); 
+	}
+
+
+    }
+
+    # Add the text in refer format(all fields)
+    if ($text =~ /\w/) {
+	$text = &text_into_html($text);
+	$doc_obj->add_text ($cursection, $text);
+    }
+
+    return 1; # processed the file
+}
+
+1;
+#
+# Convert a text string into HTML.
+#
+# The HTML is going to be inserted into a GML file, so 
+# we have to be careful not to use symbols like ">",
+# which ocurs frequently in email messages (and use
+# &gt instead.
+#
+# This function also turns links and email addresses into hyperlinks,
+# and replaces carriage returns with <BR> tags (and multiple carriage
+# returns with <P> tags).
+#
+
+sub text_into_html {
+    my ($text) = @_;
+
+
+    # Convert problem charaters into HTML symbols
+    $text =~ s/&/&amp;/g;
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/\"/&quot;/g;
+    $text =~ s/\'/ /g;
+    $text =~ s/\+/ /g;
+    $text =~ s/\(/ /g;
+    $text =~ s/\)/ /g;
+
+    # convert email addresses and URLs into links
+    $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
+    $text =~ s/(http:\/\/[\w\d\.\-]+[\/\w\d\.\-]*)/<a href=\"$1">$1<\/a>/g;
+
+    # Clean up whitespace and convert \n charaters to <BR> or <P>
+    $text =~ s/ +/ /g;
+    $text =~ s/\s*$//; 
+    $text =~ s/^\s*//; 
+    $text =~ s/\n/\n<BR>/g;
+    $text =~ s/<BR>\s*<BR>/<P>/g;
+
+    return $text;
+}
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/RogPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/RogPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/RogPlug.pm	(revision 21112)
@@ -0,0 +1,288 @@
+###########################################################################
+#
+# RogPlug.pm -- simple text plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .rog or .mdb files 
+
+package RogPlug;
+
+use BasPlug;
+use sorttools;
+use doc; 
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @RogPlug::ISA = ('BasPlug');
+}
+
+my $arguments = 
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      ];
+
+my $options = { 'name'     => "RogPlug",
+		'desc'     => "{RogPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+		'args' => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# This plugin processes files with the suffix ".mdb" or ".rog"
+sub get_default_process_exp {
+    return q^(?i)\.(mdb|rog)$^;
+}
+
+sub read_rog_record
+{
+    my ($self,$file_buffer, $seclevel) = @_;
+
+    my $next_line = $file_buffer->{'next_line'};
+
+    return 0 if (!defined $next_line);
+
+    if ($next_line eq "")
+    {
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	    $next_line = $line;
+	    last;
+	}
+    }
+	    
+    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
+    {
+	print STDERR "Error: Malformed Rog file: $next_line";
+	return 0;
+    }
+    else
+    {
+	# init default values
+	$file_buffer->{'song'}->{'tempo'}    = 120;
+	$file_buffer->{'song'}->{'ks_type'}  = 0;
+	$file_buffer->{'song'}->{'ks_num'}   = 0;
+	$file_buffer->{'song'}->{'metadata'} = [];
+	$file_buffer->{'song'}->{'content'}  = "";
+	
+	$file_buffer->{'song'}->{'subcol'} = $2;
+	$file_buffer->{'song'}->{'title'}  = $4;
+	$file_buffer->{'song'}->{'tval'}   = $6;
+
+	chomp($next_line);
+	my $content = $next_line;
+	if (defined $seclevel) 
+	{
+	    $content.= " $seclevel";
+	}
+	$content .= "\n";
+
+	$file_buffer->{'song'}->{'content'} = $content;
+
+
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	
+	    if ($line =~ m/^song/) 
+	    {	
+		$file_buffer->{'next_line'} = $line;
+		return 1;
+	    }
+	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'tempo'} = $2;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    } 
+	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ks_type'} = $2;
+		$file_buffer->{'song'}->{'ks_num'}  = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;	   
+	    } 
+	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ts_numer'} = $2;
+		$file_buffer->{'song'}->{'ts_denom'} = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
+	    {
+		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    else
+	    {
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	}
+	
+	$file_buffer->{'next_line'} = undef;
+    }
+
+    return 1;
+}
+
+sub process_rog_record
+{
+    my ($self,$file,$metadata,$song,$processor) = @_;
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    my $cursection = $doc_obj->get_top_section();
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+
+    my $title = $song->{'title'};
+    my $title_safe = $title;
+    $title_safe =~ s/\'/\\\\&apos;/g;
+
+    # add metadata 
+    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
+    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
+    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
+    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
+    $doc_obj->add_metadata($cursection, "Title",         $title); 
+    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
+    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
+    $doc_obj->add_metadata($cursection, "FileFormat",    "Rog");
+    $doc_obj->add_metadata($cursection, "FileSize",      (-s $file));
+
+    foreach my $md ( @{$song->{'metadata'}} )
+    {
+	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
+    }
+
+    # add contents as text
+    $doc_obj->add_text($cursection,$song->{'content'});
+
+    $self->extra_metadata($doc_obj,$cursection, $metadata);
+    
+    # add OID
+    $doc_obj->set_OID ();
+
+    my $oid = $doc_obj->get_OID();
+    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
+
+    $doc_obj->add_utf8_metadata ($cursection, "audiolink",  $appletlink); 
+    $doc_obj->add_utf8_metadata ($cursection, "audioicon",  "_iconaudio_"); 
+    $doc_obj->add_utf8_metadata ($cursection, "/audiolink", "</a>"); 
+
+    # process the document
+    $processor->process($doc_obj);
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    return undef unless ($filename =~ /\.((rog|mdb)(\.gz)?)$/i && (-e $filename));
+
+    my $gz = (defined $3) ? 1: 0;
+
+    	print STDERR "<Processing n='$file' p='RogPlug'>\n" if ($gli);
+    print STDERR "RogPlug: processing $filename\n" if $processor->{'verbosity'};
+    
+    if ($gz) {
+	open (FILE, "zcat $filename |") 
+	    || die "RogPlug::read - zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) 
+	    || die "RogPlug::read - can't open $filename\n";
+    }
+
+    my $doc_count = 0;
+    my $dot_count = 0;
+    my $file_buffer = { line_no => 0, next_line => "", song => {} };
+
+    while ($self->read_rog_record($file_buffer))
+    {
+	$self->process_rog_record($file,$metadata,$file_buffer->{'song'},$processor);
+	$doc_count++;
+
+	last if ($maxdocs !=-1 && ($total_count+$doc_count) >= $maxdocs);
+	
+	if (($doc_count % 10) == 0)
+	{
+	    print STDERR ".";
+	    $dot_count++;
+	    print STDERR "\n" if (($dot_count % 80) == 0);
+	}
+    }
+
+    close FILE;
+
+    print STDERR "\n";
+
+    $self->{'num_processed'} = $doc_count;
+
+    return 1; # processed the file
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/SRCPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/SRCPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/SRCPlug.pm	(revision 21112)
@@ -0,0 +1,249 @@
+###########################################################################
+#
+# SRCPlug.pm -- source code plugin
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# John McPherson Nov 2000
+# originally based on TEXTPlug
+
+# filename is currently used for Title ( optionally minus some prefix )
+
+# Current languages:
+#   text: READMEs/Makefiles
+#   C/C++   (currently extracts #include statements and C++ class decls)
+#   Perl    (currently only done as text)
+#   Shell   (currently only done as text)
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package SRCPlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @SRCPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ,
+      { 'name' => "block_exp",
+	'desc' => "{BasPlug.block_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_block_exp(),
+	'reqd' => "no" },
+      { 'name' => "remove_prefix",
+	'desc' => "{SRCPlug.remove_prefix}",
+	'type' => "regexp",
+	'deft' => "^.*[/\\]",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "SRCPlug",
+		'desc'     => "{SRCPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_block_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.(o|obj|a|so|dll)$^;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+#    return q^(?i)\.te?xt$^;
+    return q^(Makefile.*|README.*|(?i)\.(c|cc|cpp|C|h|hpp|pl|pm|sh))$^;
+}
+
+
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    	print STDERR "<Processing n='$file' p='SRCPlug'>\n" if ($gli);
+    print $outhandle "SRCPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+    
+    my $cursection = $doc_obj->get_top_section();
+
+    my $filetype="text";  # Makefiles, READMEs, ...
+    if ($file =~ /\.(cc|h|cpp|C)$/) {$filetype="C++";} # assume all .h files...
+    elsif ($file =~ /\.c$/)         {$filetype="C";}
+    elsif ($file =~ /\.p(l|m)$/)    {$filetype="perl";}
+    elsif ($file =~ /\.sh$/)        {$filetype="sh";}
+
+    # modify '<' and '>' for GML... (even though inside <pre> tags!!)
+    $$textref =~ s/</&lt;/g;
+    $$textref =~ s/>/&gt;/g;
+    $$textref =~ s/_/&#95;/g;
+    # try _escape_text($text) from doc.pm....
+
+    # don't want mg to turn escape chars into actual values
+    $$textref =~ s/\\/\\\\/g;
+
+    # use filename (minus any prefix) as the title.
+    my $title;
+    if ($self->{'remove_prefix' ne ""}) {
+	($title = $file) =~ s/^$self->{'remove_prefix'}//;
+    } else {
+	($title = $file) =~ s@^.*[/\\]@@; # remove pathname by default
+    }
+    $doc_obj->add_utf8_metadata ($cursection, "Title", $title);
+    $doc_obj->add_metadata ($cursection, "FileFormat", "SRC");
+
+    # remove the gsdl prefix from the filename
+    my $relative_filename=$file;
+    $relative_filename =~ s@^.*?gsdl[/\\]@@;
+    $doc_obj->add_utf8_metadata ($cursection, "filename", $relative_filename);
+
+    # class information from .h and .cc and .C and .cpp files
+    if ($filetype eq "C++") 
+    {
+	process_c_plus_plus($textref,$pluginfo, $base_dir, 
+				   $file, $metadata, $doc_obj);
+    } elsif ($filetype eq "C")
+    {
+	get_includes_metadata($textref, $doc_obj);
+    }
+
+
+     # default operation...
+     # insert preformat tags and add text to document object
+    $doc_obj->add_utf8_text($cursection, "<pre>\n$$textref\n</pre>");
+    
+    return 1;
+}
+
+
+
+
+sub get_includes_metadata {
+    my ($textref, $doc_obj) = @_;
+    
+    my $topsection = $doc_obj->get_top_section();
+
+    # Get '#include' directives for metadata
+    if ($$textref !~ /\#\s*include\b/) {
+	return;
+    }
+
+    my @includes =
+	($$textref =~ m/^\s*\#\s*include\s*(?:\"|&lt;)(.*?)(?:\"|&gt;)/mg);
+    
+    my $incs_done_ref=$doc_obj->get_metadata($topsection, "includes");
+    my @incs_done;
+    if (defined($incs_done_ref)) {
+	@incs_done=@$incs_done_ref;
+    } else {
+	@incs_done=();
+    }
+
+    foreach my $inc (@includes) {
+	# add entries, but only if they don't already exist
+	if (!join('', map {$_ eq "$inc"?1:""} @incs_done)) {
+	    push @incs_done, $inc;
+	    $doc_obj->add_utf8_metadata($topsection, "includes", $inc);
+	}
+    }
+}
+
+
+
+sub process_c_plus_plus {
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+
+    my $topsection = $doc_obj->get_top_section();
+
+
+    # Check for include metadata
+    get_includes_metadata($textref, $doc_obj);
+
+
+
+    # Get class declarations (but not forward declarations...) as metadata
+    if ($$textref =~ /\bclass\b/ ) {
+	my $classnames=$$textref;
+	
+	# remove commented lines
+	$classnames =~ s@/\*.*?\*/@@sg;
+	$classnames =~ s@//.*$@@mg;
+	while ($classnames =~ /\bclass\b/) {
+
+	    # delete all lines up to the next "class"
+	    while ($classnames !~ /^[^\n]*\bclass\b[^\n]*\n/)
+	    {$classnames =~ s/.*\n//;}
+	    
+#	    $classnames =~ s/^([^c][^l])*(.)?$//mg; # delete unneccessary lines
+
+	    # get the line including the next "class" and remove it from
+	    # our tmp text.
+	    $classnames =~ s/^(.*\bclass\b.*)$//m;
+
+	    # don't index if merely a reference/fwd decl. of another class
+	    if ($1 !~ /(friend\Wclass)|(class\W\w+\W?\;)|(\/\/.*class)/) {
+		# $1 is still the whole line - eg:
+		# "class StaffSystem: public BaseStaffSystem"
+		my $wholeline=$1;
+		my $classname=$1;
+		$classname =~ s/.*class\W(\w+).*/$1/;
+		my $classes=$doc_obj->get_metadata($topsection, "class");
+		foreach my $elem (@$classes) {
+		    if ("$elem" eq "$classname") {goto class_done;}
+		}
+		$doc_obj->add_utf8_metadata($topsection, "class", $classname);
+	      class_done:
+		$doc_obj->add_utf8_metadata($topsection, "classdecl", $wholeline);
+	    }
+	}
+    } # end of "class"
+
+    return 1;
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/plugins/SplitPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/SplitPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/SplitPlug.pm	(revision 21112)
@@ -0,0 +1,284 @@
+###########################################################################
+#
+# SplitPlug.pm - a plugin for splitting input files into segments that
+#                will then be individually processed.
+#
+#
+# Copyright 2000 Gordon W. Paynter (gwp@cs.waikato.ac.nz)
+# Copyright 2000 The New Zealand Digital Library Project
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# SplitPlug is a plugin for splitting input files into segments that will
+# then be individually processed.  
+
+# This plugin should not be called directly.  Instead, if you need to
+# process input files that contain several documents, you should write a
+# plugin with a process function that will handle one of those documents
+# and have it inherit from SplitPlug.  See ReferPlug for an example.
+
+
+package SplitPlug;
+
+use BasPlug;
+use gsprintf 'gsprintf';
+use util;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+# SplitPlug is a sub-class of BasPlug.
+sub BEGIN {
+    @SplitPlug::ISA = ('BasPlug');
+}
+
+
+my $arguments =
+    [ { 'name' => "split_exp",
+	'desc' => "{SplitPlug.split_exp}",
+	'type' => "regexp",
+	#'deft' => &get_default_split_exp(),
+	'deft' => "",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "SplitPlug",
+		'desc'     => "{SplitPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes",
+	        'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    $self->{'textcat_store'} = {};
+    $self->{'metapass_srcdoc'} = {}; # which segments have valid metadata_srcdoc
+    return bless $self, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+    my ($verbosity, $outhandle, $failhandle) = @_;
+
+    $self->BasPlug::init($verbosity, $outhandle, $failhandle);
+
+    if ((!defined $self->{'process_exp'}) || ($self->{'process_exp'} eq "")) {
+
+	$self->{'process_exp'} = $self->get_default_process_exp ();
+	if ($self->{'process_exp'} eq "") {
+	    warn ref($self) . " Warning: Non-recursive plugin has no process_exp\n";
+	}
+    }
+
+
+    # set split_exp to default unless explicitly set
+    if (!$self->{'split_exp'}) {
+	$self->{'split_exp'} = $self->get_default_split_exp ();
+    }
+
+}
+
+# This plugin recurs over the segments it finds
+sub is_recursive {
+    return 1;
+}
+
+# By default, we split the input text at blank lines
+sub get_default_split_exp {
+    return q^\n\s*\n^;
+}
+
+sub metadata_read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    my $matched = $self->SUPER::metadata_read($pluginfo, $base_dir, $file, 
+					      $metadata, $extrametakeys, 
+					      $extrametadata, $processor, 
+					      $maxdocs, $gli);
+    my $split_matched = undef;
+
+    if ($matched) {
+
+	my $outhandle = $self->{'outhandle'};
+	my $filename = &util::filename_cat($base_dir, $file);
+
+	my $plugin_name = ref ($self);
+	$file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+	$self->{'metapass_srcdoc'}->{$file} = {};
+
+	# Do encoding stuff
+	my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+	my $le_rec = { 'language' => $language, 'encoding' => $encoding };
+	$self->{'textcat_store'}->{$file} = $le_rec;
+
+	# Read in file ($text will be in utf8)
+	my $text = "";
+	$self->read_file ($filename, $encoding, $language, \$text);
+
+	if ($text !~ /\w/) {
+	    gsprintf($outhandle, "$plugin_name: {BasPlug.file_has_no_text}\n",
+		     $file)
+		if $self->{'verbosity'};
+	    
+	    my $failhandle = $self->{'failhandle'};
+	    print $failhandle "$file: " . ref($self) . ": file contains no text\n";
+	    $self->{'num_not_processed'} ++;
+
+	    $self->{'textcat_store'}->{$file} = undef;
+
+	    return 0; 
+	}
+    
+    
+	# Split the text into several smaller segments
+	my $split_exp = $self->{'split_exp'};
+	my @segments = split(/$split_exp/, $text);
+	print $outhandle "SplitPlug found " . (scalar @segments) . " documents in $filename\n" 
+	    if $self->{'verbosity'};
+	
+	$self->{'split_segments'}->{$file} = \@segments;
+	$split_matched = scalar(@segments);
+    }
+    
+    return $split_matched;
+}
+
+
+
+# The read function opens a file and splits it into parts. 
+# Each part is sent to the process function
+#
+# Returns: Number of document objects created (or undef if it fails)
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    # Figure out the exact filename of this file (and maybe block it)
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $block_exp = $self->{'block_exp'};
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    my $le_rec = $self->{'textcat_store'}->{$file};
+    if (!defined $le_rec) {
+	# means no text was found;
+	return 0; # not processed but no point in passing it on
+    }
+
+    my $language = $le_rec->{'language'};
+    my $encoding = $le_rec->{'encoding'};
+    $self->{'textcat_store'}->{$file} = undef;
+
+    my $segments = $self->{'split_segments'}->{$file};
+    $self->{'split_segments'}->{$file} = undef;
+
+    # Process each segment in turn
+    my ($count, $segment, $segtext, $status, $id);
+    $segment = 0;
+    $count = 0;
+    foreach $segtext (@$segments) {
+	$segment++;
+
+	if (defined $self->{'metapass_srcdoc'}->{$file}->{$segment}) {
+	    # metadata is attached to a srcdoc
+	    next;
+	}
+
+	# create a new document
+	my $doc_obj = new doc ($filename, "indexed_doc");
+	$doc_obj->set_OIDtype ($processor->{'OIDtype'});
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
+	my ($filemeta) = $file =~ /([^\\\/]+)$/;
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$segment");
+	if ($self->{'cover_image'}) {
+	  $self->associate_cover_image($doc_obj, $filename);
+	}
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+	#$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "Split");
+
+	# Calculate a "base" document ID.
+	if (!defined $id) {
+	    $doc_obj->set_OID();
+	    $id = $doc_obj->get_OID();
+	}
+    
+	# include any metadata passed in from previous plugins 
+	# note that this metadata is associated with the top level section
+	$self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+
+	# do plugin specific processing of doc_obj
+	print $outhandle "segment $segment - " if ($self->{'verbosity'});
+	$status = $self->process (\$segtext, $pluginfo, $base_dir, $file, $metadata, $doc_obj);
+	if (!defined $status) {
+	    print $outhandle "WARNING - no plugin could process segment $segment of $file\n" 
+		if ($verbosity >= 2);
+	    next;
+	}
+	# If the plugin returned 0, it threw away this part
+	if ($status == 0) {
+	    next;
+	}
+	$count += $status;
+
+	# do any automatic metadata extraction
+	$self->auto_extract_metadata ($doc_obj);
+
+	# add an OID
+	$self->set_OID($doc_obj, $id, $segment);
+
+	# process the document
+	$processor->process($doc_obj);
+
+	$self->{'num_processed'} ++;
+    }
+
+    delete $self->{'metapass_srcdoc'}->{$file};
+
+    # Return number of document objects produced
+    return $count; 
+}
+
+sub set_OID {
+    my $self = shift (@_);
+    my ($doc_obj, $id, $segment_number) = @_;
+    
+    $doc_obj->set_OID($id . "s" . $segment_number);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/StructuredHTMLPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/StructuredHTMLPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/StructuredHTMLPlug.pm	(revision 21112)
@@ -0,0 +1,422 @@
+###########################################################################
+#
+# StructuredHTMLPlug.pm -- html plugin with extra facilities for teasing out 
+# hierarchical structure (such as h1, h2, h3, or user-defined tags) in an 
+# HTML document
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# This plugin is to process an HTML file where sections are divided by 
+# user-defined headings tags. As it is difficult to predict what user's definition
+# this plugin allows to detect the user-defined titles up to three levels (level1, level2, level3...)
+# as well as allows to get rid of user-defined Table of Content (TOC)...
+# format:e.g. level1 (Abstract_title|ChapterTitle|Referencing Heading) level2(SectionHeading)...
+
+package StructuredHTMLPlug;
+
+use HTMLPlug;
+use ImagePlug;
+
+#use strict; # every perl program should have this!
+#no strict 'refs'; # make an exception so we can use variables as filehandles
+
+sub BEGIN {
+    @StructuredHTMLPlug::ISA = ('HTMLPlug');
+}
+
+my $arguments = [];
+
+my $options = { 'name'     => "StructuredHTMLPlug",
+		'desc'     => "{StructuredHTMLPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = (defined $hashArgOptLists)? new HTMLPlug($pluginlist,$inputargs,$hashArgOptLists): new HTMLPlug($pluginlist,$inputargs);
+    
+    return bless $self, $class;
+}
+
+sub read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($filename =~ m/\.html?$/) {
+	my $poss_doc_filename = $filename;
+	$poss_doc_filename =~ s/\.html?$/.doc/;
+
+	if (-e $poss_doc_filename) {
+	    # this file has already been processed by Word plugin
+	    return 0;
+	}
+    }
+    return $self->SUPER::read(@_);
+}
+
+sub process {
+    my $self = shift (@_);
+    #my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print $outhandle "StructuredHTMLPlug: processing $file\n"
+        if $self->{'verbosity'} > 1;
+    
+    my @head_and_body = split(/<body/i,$$textref);
+    my $head = shift(@head_and_body);
+    my $body_text = join("<body", @head_and_body);
+    $head =~ m/<title>(.+)<\/title>/i;
+    my $doctitle = $1 if defined $1;
+    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'}=~ /\S/) {
+	my @doc_properties = split(/<xml>/i,$head);
+	my $doc_heading = shift(@doc_properties);
+	my $rest_doc_properties = join(" ", @doc_properties);
+	my @extracted_metadata = split(/<\/xml>/i, $rest_doc_properties);
+	my $extracted_metadata = shift (@extracted_metadata);
+	$self->extract_metadata($extracted_metadata, $metadata, $doc_obj);
+    }
+    
+    # If delete_toc is enables, it means to get rid of toc and tof contents.
+    # get rid of TOC and TOF sections and their title
+    if ($self->{'delete_toc'} == 1){
+	if (defined $self->{'toc_header'}&& $self->{'toc_header'} =~ /\S/){
+	    $body_text =~ s/<p class=(($self->{'toc_header'})[^>]*)>(.+?)<\/p>//isg;
+	}
+	if (defined $self->{'tof_header'}&& $self->{'tof_header'}=~ /\S/) {
+	    $body_text =~ s/<p class=(($self->{'tof_header'})[^>]*)>(.+?)<\/p>//isg;
+	}
+    }
+    
+    if (defined $self->{'title_header'} && $self->{'title_header'}=~ /\S/){
+	$self->{'title_header'} =~ s/^(\()(.*)(\))/$2/is;
+	#$body_text =~ s/<p class=(($self->{'title_header'})[^>]*)>(.+?)<\/p>/<p class=$1><title>$3<\/title><\/p>/isg;
+	#$doctitle = $3;
+	$body_text =~ s/<p class=(($self->{'title_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h1>$3<\/h1><\/p>/isg;
+	#$body_text =~ m/<p class=(($self->{'title_header'})[^>]*)>(.+?)<\/p>/isg;
+	#$doctitle = "<h1>".$3."<\/h1>" if defined $3;
+    }
+
+    if (defined $self->{'level1_header'} && $self->{'level1_header'}=~ /\S/ ){
+     	$self->{'level1_header'} =~ s/^\((.*)\)/$1/i;
+	$body_text =~ s/<p class=(($self->{'level1_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h1>$3<\/h1><\/p>/isg;
+    }
+    
+    if (defined $self->{'level2_header'} && $self->{'level2_header'}=~ /\S/){
+	$self->{'level2_header'} =~ s/^\((.*)\)/$1/i;
+	$body_text =~ s/<p class=(($self->{'level2_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h2>$3<\/h2><\/p>/isg;
+    }
+    
+    if (defined $self->{'level3_header'} && $self->{'level3_header'}=~ /\S/ ){
+	$self->{'level3_header'} =~ s/^\((.*)\)/$1/is;
+	$body_text =~ s/<p class=(($self->{'level3_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h3>$3<\/h3><\/p>/isg;
+    }
+    # Tidy up extra new lines
+    $body_text =~ s/(<p[^>]*><span[^>]*><o:p>&nbsp;<\/o:p><\/span><\/p>)//isg;
+    $body_text =~ s/(<p[^>]*><o:p>&nbsp;<\/o:p><\/p>)//isg;
+    
+    $section_text .= "<!--\n<Section>\n-->\n";
+    #my $top_section_tag = "<!--\n<Section>\n-->\n";
+    #$body_text =~ s/(<div.*)/$top_section_text$doctitle$1/i;
+    #$body_text =~ s/(<div.*)/$top_section_tag$1/i;
+    my $body = "<body".$body_text;
+    
+    my $section_text = $head;
+    
+    # split HTML text on <h1>, <h2> etc tags
+    my @h_split = split(/<h/i,$body);
+    
+    my $hnum = 0;
+
+    my $sectionh1 = 0;
+    $section_text .= shift(@h_split);
+    
+    my $hc;
+    foreach $hc ( @h_split )
+    {
+	if ($hc =~ m/^([1-3])\s*.*?>(.*)$/s)
+	{
+	    my $new_hnum = $1;
+	    my $hc_after = $2;
+	    
+	    if ($hc_after =~ m/^(.*?)<\/h$new_hnum>/is)
+	    {
+		my $h_text = $1;
+		$hc =~ s/^(\&nbsp\;)+/\&nbsp\;/g;
+		# boil HTML down to some interesting text
+		$h_text =~ s/^[1-3]>//;
+		$h_text =~ s/<\/?.*?>//sg;
+		$h_text =~ s/\s+/ /sg;
+		$h_text =~ s/^\s$//s;
+		$h_text =~ s/(&nbsp;)+\W*/&nbsp;/sg;
+		
+		if ($h_text =~ m/\w+/)
+		{
+		    if ($new_hnum > $hnum)
+		    {
+			# increase section nesting
+			$hnum++;
+			while ($hnum < $new_hnum)
+			{
+			    my $spacing = "  " x $hnum;
+			    $section_text .= "<!--\n";
+			    $section_text .= $spacing."<Section>\n";
+			    $section_text .= "-->\n";
+			    $hnum++;
+			}
+		    }
+		    else # ($new_hnum <= $hnum)
+		    {
+			# descrease section nesting
+			while ($hnum >= $new_hnum)
+			{
+			    my $spacing = "  " x $hnum;
+			    $section_text .= "<!--\n";
+			    $section_text .= $spacing."</Section>\n";
+			    $section_text .= "-->\n";
+			    $hnum--;
+			}
+			$hnum++;
+		    }
+
+		    my $spacing = "  " x $hnum;
+		    $section_text .= "<!--\n";
+		    $section_text .= $spacing."<Section>\n";
+		    $section_text .= $spacing."  <Description>\n";
+		    $section_text .= $spacing."    <Metadata name=\"Title\">$h_text</Metadata>";
+		    $section_text .= $spacing."  </Description>\n";
+		    $section_text .= "-->\n";
+		    
+		    print $outhandle $spacing."$h_text\n"
+			if $self->{'verbosity'} > 2;
+		    
+		    $sectionh1++ if ($hnum==1);
+		}
+	    }
+	    else {
+###		print STDERR "***** hc = <h$hc\n\n";
+	    }
+	    $section_text .= "<h$hc";
+	}
+	else
+	{
+	    $section_text .= "<h$hc";
+	}
+    }
+
+    while ($hnum >= 1)
+    {
+	my $spacing = "  " x $hnum;
+	$section_text .= "<!--\n";
+	$section_text .= $spacing."</Section>\n";
+	$section_text .= "-->\n";
+	$hnum--;
+    }
+
+    $section_text .= "<!--\n</Section>\n-->\n";
+
+    $$textref = $section_text;
+    
+    # should be textref not testref???
+    #$$testref =~ s/<h(\d+)>(.*?)<\/h$1>/<Section><Metadata name=\"Title\">$1<\/Metadata></Section><h$1><\/h$1>/gi;
+    
+    if ($sectionh1>0)
+    {
+	print $outhandle "  Located section headings ..."
+	    if $self->{'verbosity'} > 1;
+    }
+    print $outhandle "  Passing on the HTMLPlug\n"
+	if $self->{'verbosity'} > 1;
+    
+    $$textref =~ s/<!\[if !vml\]>/<![if vml]>/g;
+    
+    $$textref =~ s/(&nbsp;)+/&nbsp;/sg;    
+    
+    ## $$textref =~ s/<o:p>&nbsp;<\/o:p>//g; # used with VML to space figures?
+    
+    $self->SUPER::process(@_);
+    
+    # associate original file with doc object
+    my $cursection = $doc_obj->get_top_section();
+    my $filename = &util::filename_cat($base_dir, $file);
+    if (-e $filename)
+    {
+	print $outhandle "  Adding associated Word document\n"
+	    if $self->{'verbosity'} > 1;
+	
+	$doc_obj->associate_file($filename, "doc.doc", undef, $cursection);
+	
+	my $doclink = "<a href=_httpcollection_/index/assoc/[archivedir]/doc.doc>";
+	$doc_obj->add_utf8_metadata ($cursection, "srclink",  $doclink);
+	$doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icondoc_");
+	$doc_obj->add_utf8_metadata ($cursection, "/srclink", "</a>");
+	$doc_obj->add_utf8_metadata ($cursection, "Title", $doctitle);
+   	my $file_size = -s $filename;
+	if ($file_size>1024)
+	{
+	    my $fs_kbytes = sprintf("%d",$file_size/1024);
+	    $doc_obj->add_utf8_metadata ($cursection, "filesize", "$fs_kbytes Kb");
+	}
+	else
+	{
+	    $doc_obj->add_utf8_metadata ($cursection, "filesize", "$file_size bytes");
+	}
+
+	if ($file_size > 200000)
+	{
+	    $doc_obj->add_utf8_metadata ($cursection, "fswarning", "1");
+	}
+    }
+}
+
+
+sub resize_if_necessary
+{
+    my ($self,$front,$back,$base_dir,$href) = @_;
+    
+    # dig out width and height of image, if there
+    my $img_attributes = "$front back";
+    my ($img_width)  = ($img_attributes =~ m/\s+width=\"?(\d+)\"?/i);
+    my ($img_height) = ($img_attributes =~ m/\s+height=\"?(\d+)\"?/i);
+    
+    # derive local filename for image based on its URL
+    my $img_filename = $href;
+    $img_filename =~ s/^[^:]*:\/\///;
+    $img_filename = &util::filename_cat($base_dir, $img_filename);
+    
+    # Replace %20's in URL with a space if required. Note that the filename
+    # may include the %20 in some situations
+    if ($img_filename =~ /\%20/) {
+	if (!-e $img_filename) {
+	    $img_filename =~ s/\%20/ /g;
+	}
+    }
+    if ((-e $img_filename) && (defined $img_width) && (defined $img_height)) {
+	# get image info on width and height
+	
+	my $outhandle = $self->{'outhandle'};
+	my $verbosity = $self->{'verbosity'};
+
+	my ($image_type, $actual_width, $actual_height, $image_size) 
+	    = &ImagePlug::identify($img_filename, $outhandle, $verbosity);
+	
+	#print STDERR "**** $actual_width x $actual_height";
+	#print STDERR " (requested: $img_width x $img_height)\n";
+
+	if (($img_width < $actual_width) || ($img_height < $actual_height)) {
+	    print $outhandle "Resizing $img_filename\n" if ($verbosity > 0);
+	    
+	    # derive new image name based on current image
+	    my ($tailname, $dirname, $suffix)
+		= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
+	    
+	    my $resized_filename 
+		= &util::filename_cat($dirname, $tailname."_resized".$suffix);
+	    
+	    #print STDERR "**** suffix = $suffix\n";
+	    
+	    # Generate smaller image with convert
+	    my $newsize = "$img_widthx$image_height";
+	    my $command = "convert -interlace plane -verbose "
+		."-geometry $newsize \"img_$filename\" \"$resized_filename\"";
+	    print $outhandle "ImageResize: $command\n" if ($verbosity > 2);
+	    my $result = '';
+	    print $outhandle "ImageResize result: $result\n" if ($verbosity > 2);
+	}
+    }
+    return $href;
+}
+
+sub replace_images {
+    my $self = shift (@_);
+    my ($front, $link, $back, $base_dir, 
+	$file, $doc_obj, $section) = @_;
+    # remove quotes from link at start and end if necessary
+    if ($link=~/^\"/) {
+	$link=~s/^\"//;$link=~s/\"$//;
+	$front.='"';
+	$back="\"$back";
+    }
+    
+    $link =~ s/\n/ /g;
+    
+    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
+    
+##    $href = $self->resize_if_necessary($front,$back,$base_dir,$href);
+    
+    my $middle = $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section);
+    
+    return $front . $middle . $back;
+}
+
+sub extract_metadata 
+{
+    my $self = shift (@_);
+    my ($textref, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    # metadata fields to extract/save. 'key' is the (lowercase) name of the
+    # html meta, 'value' is the metadata name for greenstone to use
+    my %find_fields = ();
+    my ($tag,$value);
+
+    my $orig_field = "";
+    foreach my $field (split /,/, $self->{'metadata_fields'}) {
+	# support tag<tagname>
+	if ($field =~ /^(.*?)<(.*?)>$/) {
+	    # "$2" is the user's preferred gs metadata name
+	    $find_fields{lc($1)}=$2; # lc = lowercase
+	    $orig_field = $1;
+	} else { # no <tagname> for mapping
+	    # "$field" is the user's preferred gs metadata name
+	    $find_fields{lc($field)}=$field; # lc = lowercase
+	    $orig_field = $field;
+	}
+	if ($textref =~ m/<o:$orig_field>(.*)<\/o:$orig_field>/i){
+	    $tag = $orig_field;
+	    $value = $1;
+	    if (!defined $value || !defined $tag){
+		print $outhandle "StructuredHTMLPlug: can't find VALUE in \"$tag\"\n";
+		next;
+	    } else {
+		# clean up and add
+		chomp($value); # remove trailing \n, if any
+		$tag = $find_fields{lc($tag)};
+		print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
+		    if ($self->{'verbosity'} > 2);
+		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $value);
+	    }
+	}
+    }
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/TEXTPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/TEXTPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/TEXTPlug.pm	(revision 21112)
@@ -0,0 +1,137 @@
+###########################################################################
+#
+# TEXTPlug.pm -- simple text plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document. Adds Title metadata 
+# of first line of text (up to 100 characters long).
+
+# 12/05/02 Added usage datastructure - John Thompson
+
+package TEXTPlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @TEXTPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ,
+      { 'name' => "title_sub",
+	'desc' => "{TEXTPlug.title_sub}",
+	'type' => "regexp",
+	'deft' => "",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "TEXTPlug",
+		'desc'     => "{TEXTPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.te?xt$^;
+}
+
+# do plugin specific processing of doc_obj
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print STDERR "<Processing n='$file' p='TEXTPlug'>\n" if ($gli);
+    print $outhandle "TEXTPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+    
+    my $cursection = $doc_obj->get_top_section();
+    # get title metadata
+    # (don't need to get title if it has been passed
+    # in from another plugin)
+    if (!defined $metadata->{'Title'}) {
+	my ($title) = $$textref;
+	$title =~ /^\s+/s;
+	if (defined $self->{'title_sub'} &&
+           $self->{'title_sub'}) {$title =~ s/$self->{'title_sub'}//;}
+	$title =~ /^\s*([^\n]*)/s; $title=$1;
+	if (length($title) > 100) {
+	    $title = substr ($title, 0, 100) . "...";
+	}
+        $title =~ s/\[/&#91;/g;
+        $title =~ s/\[/&#93;/g;
+        $title =~ s/\</&#60;/g;
+        $title =~ s/\>/&#62;/g;
+	$doc_obj->add_utf8_metadata ($cursection, "Title", $title);
+    }
+    # Add FileFormat metadata
+    $doc_obj->add_metadata($cursection, "FileFormat", "TEXT");
+    
+
+    # we need to escape the escape character, or else mg will convert into
+    # eg literal newlines, instead of leaving the text as '\n'
+    $$textref =~ s/\\/\\\\/g; # macro language
+    $$textref =~ s/_/\\_/g; # macro language
+    $$textref =~ s/</&lt;/g;
+    $$textref =~ s/>/&gt;/g;
+
+    # insert preformat tags and add text to document object
+    $doc_obj->add_utf8_text($cursection, "<pre>\n$$textref\n</pre>");
+
+    return 1;
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/UnknownPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/UnknownPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/UnknownPlug.pm	(revision 21112)
@@ -0,0 +1,257 @@
+###########################################################################
+#
+# UnknownPlug.pm -- Plugin for files you know about but Greenstone doesn't
+#
+# A component of the Greenstone digital library software from the New
+# Zealand Digital Library Project at the University of Waikato, New
+# Zealand.
+#
+# Copyright (C) 2001 Gordon W. Paynter
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# UnknownPlug - a plugin for unknown files
+
+# This is a simple Plugin for importing files in formats that
+# Greenstone doesn't know anything about.  A fictional document will
+# be created for every such file, and the file itself will be passed
+# to Greenstone as the "associated file" of the document.
+
+# Here's an example where it is useful: I have a collection of
+# pictures that include a couple of quicktime movie files with names
+# like DCP_0163.MOV.  Rather than write a new plugin for quicktime
+# movies, I add this line to the collection configuration file:
+
+# plugin UnknownPlug -process_exp "*.MOV" -assoc_field "movie"
+
+# A document is created for each movie, with the associated movie
+# file's name in the "movie" metadata field.  In the collection's
+# format strings, I use the {If} macro to output different text for
+# each type of file, like this:
+
+# {If}{[movie],<HTML for displaying movie>}{If}{[Image],<HTML for displaying image>}
+
+# You can also add extra metadata, such as the Title, Subject, and
+# Duration, with metadata.xml files and RecPlug.  (If you want to use
+# UnknownPlug with more than one type of file, you will have to add
+# some sort of distinguishing metadata in this way.)
+
+
+
+package UnknownPlug;
+
+use BasPlug;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @UnknownPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "assoc_field",
+	'desc' => "{UnknownPlug.assoc_field}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "file_format",
+	'desc' => "{UnknownPlug.file_format}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "mime_type",
+	'desc' => "{UnknownPlug.mime_type}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "process_extension",
+	'desc' => "{UnknownPlug.process_extension}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "UnknownPlug",
+		'desc'     => "{UnknownPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    # "-process_extension" is a simpler alternative to -process_exp for non-regexp people
+    if (!$self->{'process_exp'} && $self->{'process_extension'}) {
+	$self->{'process_exp'} = "\\." . $self->{'process_extension'} . "\$";
+    }
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    return '';
+}
+
+
+# Associate the unknown file with the new document
+
+sub associate_unknown_file {
+    my $self = shift (@_);
+    my $filename = shift (@_);   # filename with full path
+    my $file = shift (@_);       # filename without path
+    my $doc_obj = shift (@_);
+    
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+
+    # check the filename is okay
+    return 0 if ($file eq "" || $filename eq "");
+
+    # Add the image metadata
+    my $url = $file;
+    $url =~ s/ /%20/g;
+
+    # Add the file as an associated file ...
+    my $section = $doc_obj->get_top_section();
+    my $file_format = $self->{'file_format'} || "unknown";
+    my $mime_type = $self->{'mime_type'} || "unknown/unknown";
+    my $assoc_field = $self->{'assoc_field'} || "unknown_file";
+
+    $doc_obj->associate_file($filename, $file, $mime_type, $section);
+    $doc_obj->add_metadata ($section, "FileFormat", $file_format);
+    $doc_obj->add_metadata ($section, "MimeType", $mime_type);
+    $doc_obj->add_metadata ($section, $assoc_field, $file);
+    
+    $doc_obj->add_metadata ($section, "srclink", 
+			    "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[$assoc_field]\">");
+    $doc_obj->add_metadata ($section, "srcicon", "_iconunknown_");
+    $doc_obj->add_metadata ($section, "/srclink", "</a>");
+    
+    return 1;
+}
+
+
+
+# The UnknownPlug read() function. This function does all the right
+# things to make general options work for a given plugin.  UnknownPlug
+# overrides read() because there is no need to read the actual text of
+# the file in, because the contents of the file is not text...
+#
+#
+# Return number of files processed, undef if can't process 
+#
+# Note that $base_dir might be "" and that $file might include directories
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    # Make sure we're processing the correct file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    print STDERR "<Processing n='$file' p='UnknownPlug'>\n" if ($gli);
+    print $outhandle "UnknownPlug processing \"$filename\"\n"
+	    if $self->{'verbosity'} > 1;
+
+    #if there's a leading directory name, eat it...
+    $file =~ s/^.*[\/\\]//;
+    
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});    
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($file)); # set the filename as Source metadata to be consistent with other plugins
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename));
+
+    # associate the file with the document
+    if (associate_unknown_file($self, $filename, $file, $doc_obj) != 1)
+    {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file'>\n";
+	}
+	print $outhandle "UnknownPlug: couldn't process \"$filename\"\n";
+	return -1; # error during processing
+    }
+
+    #create an empty text string so we don't break downstream plugins 
+    my $text = &gsprintf::lookup_string("{BasPlug.dummy_text}");
+
+    # include any metadata passed in from previous plugins 
+    my $section = $doc_obj->get_top_section();
+    $self->extra_metadata ($doc_obj, $section, $metadata);
+
+    $self->title_fallback($doc_obj,$section,$file);
+
+    # do plugin specific processing of doc_obj
+    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1;
+    }
+
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+
+    # add an OID
+    $doc_obj->set_OID();
+    $doc_obj->add_text($section, $text);
+
+    # process the document
+    $processor->process($doc_obj);
+
+    $self->{'num_processed'} ++;
+    return 1;
+}
+
+
+# UnknownPlug processing of doc_obj.  In practice we don't need to do
+# anything here because the read function takes care of everything.
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return 1;
+}
+
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/W3ImgPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/W3ImgPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/W3ImgPlug.pm	(revision 21112)
@@ -0,0 +1,1068 @@
+###########################################################################
+#
+# W3ImgPlug.pm -- Context-based image indexing plugin for HTML documents
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# DESCRIPTION:
+#
+#  Extracts images and associated text and metadata from
+#  web pages as individual documents for indexing. Thumbnails
+#  are created from each image for browsing purposes.
+#
+#  Options are available for configuring the aggressiveness of the
+#  associated text extraction mechanisms. A higher level of
+#  aggressiveness will extract more text and consequently
+#  may mean lower accuracy (precision); however, it may also 
+#  retrieve more of the relevant images from the collection (recall). 
+#  Lower levels of aggressiveness maybe result in slightly faster
+#  collection builds at the import stage.
+#
+#  W3ImgPlug is a subclass of HTMLPlug (i.e. it will index pages also 
+#  if required). It can be used in place of HTMLPlug to index both
+#  pages and their images.
+#
+# REQUIREMENTS:
+#   
+#  The ImageMagick image manipulation is used to create
+#  thumbnails and extract some image metadata. (Available 
+#  from http://www.imagemagick.org/)
+#
+#  Unix:
+#    Many Linux distributions contain ImageMagick.
+#
+#  Windows:
+#    ImageMagick can be downloaded from the website above.
+#    Make sure the system path includes the ImageMagick binaries
+#    before using W3ImgPlug.
+#
+#    NOTE: NT/2000/XP contain a filesystem utility 'convert.exe' 
+#    with the same name as the image conversion utility. The
+#    ImageMagick FAQ recommends renaming the filesystem
+#    utility (e.g. to 'fsconvert.exe') to avoid this clash.
+#
+# USAGE:  
+#
+#  An image document consists of metadata elements:
+#
+#   OriginalFilename, FilePath, Filename, FileExt, FileSize,
+#   Width, Height, URL, PageURL, ThumbURL, CacheURL, CachePageURL
+#   ImageText, PageTitle
+#
+#  Most of these are only useful in format strings (e.g. ThumbURL, 
+#  Filename, URL, PageURL, CachePageURL). 
+#
+#  ImageText, as the name suggests contains the indexable text.
+#  (unless using the -document_text plugin option)
+#
+#  Since image documents are made up of metadata elements 
+#  alone, format strings are needed to display them properly. 
+#  NOTE: The receptionist will only display results (e.g. thumbnails)
+#  in 4 columns if the format string begins with "<td><table>".
+#
+#  The example below takes the user to the image within the
+#  source HTML document rather than using a format string
+#  on DocumentText to display the image document itself.
+#
+#  Example collect.cfg:
+#
+#   ...
+#
+#   indexes document:ImageText document:text
+#   defaultindex document:ImageText
+#  
+#   collectionmeta .document:ImageText "images"
+#   collectionmeta .document:text "documents"
+#
+#   ...
+#
+#   plugin W3ImgPlug -index_pages -aggressiveness 6
+#
+#   ...
+#  
+#   format SearchVList '<td>{If}{[Title],[link][icon]&nbsp;[Title][[/link],
+#    <table><tr><td align="center"><a href="[CachePageURL]">
+#    <img src="[ThumbURL]"></a></td></tr><tr><td align="center">
+#    <a href="[CachePageURL]"><font size="-1">[OriginalFilename]</font></a>
+#    <br>[Width]x[Height]</td></tr></table>}</td>'
+#
+#   ...
+#
+ 
+package W3ImgPlug;
+
+use HTMLPlug;
+use ghtml;
+use unicode;
+use util;
+use strict; # 'subs';
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @W3ImgPlug::ISA = qw( HTMLPlug );
+}
+
+my $aggressiveness_list = 
+    [ { 'name' => "1",
+	'desc' => "{W3ImgPlug.aggressiveness.1}" },
+      { 'name' => "2",
+	'desc' => "{W3ImgPlug.aggressiveness.2}" },
+      { 'name' => "3",
+	'desc' => "{W3ImgPlug.aggressiveness.3}" },
+      { 'name' => "4",
+	'desc' => "{W3ImgPlug.aggressiveness.4}" },
+      { 'name' => "5",
+	'desc' => "{W3ImgPlug.aggressiveness.5}" },
+      { 'name' => "6",
+	'desc' => "{W3ImgPlug.aggressiveness.6}" },
+      { 'name' => "7",
+	'desc' => "{W3ImgPlug.aggressiveness.7}" },
+      { 'name' => "8",
+	'desc' => "{W3ImgPlug.aggressiveness.8}" },
+      { 'name' => "9",
+	'desc' => "{W3ImgPlug.aggressiveness.9}" } ];
+
+my $arguments =
+    [ { 'name' => "aggressiveness",
+	'desc' => "{W3ImgPlug.aggressiveness}",
+	'type' => "int",
+	'list' => $aggressiveness_list,
+	'deft' => "3",
+	'reqd' => "no" },
+      { 'name' => "index_pages",
+	'desc' => "{W3ImgPlug.index_pages}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "no_cache_images",
+	'desc' => "{W3ImgPlug.no_cache_images}",
+	'type' => "flag",
+	'reqd' => "no" },
+      { 'name' => "min_size",
+	'desc' => "{W3ImgPlug.min_size}",
+	'type' => "int",
+	'deft' => "2000",
+	'reqd' => "no" },
+      { 'name' => "min_width",
+	'desc' => "{W3ImgPlug.min_width}",
+	'type' => "int",
+	'deft' => "50",
+	'reqd' => "no" },
+      { 'name' => "min_height",
+	'desc' => "{W3ImgPlug.min_height}",
+	'type' => "int",
+	'deft' => "50",
+	'reqd' => "no" },
+      { 'name' => "thumb_size",
+	'desc' => "{W3ImgPlug.thumb_size}",
+	'type' => "int",
+	'deft' => "100",
+	'reqd' => "no" },
+      { 'name' => "convert_params",
+	'desc' => "{W3ImgPlug.convert_params}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" },
+      { 'name' => "min_near_text",
+	'desc' => "{W3ImgPlug.min_near_text}",
+	'type' => "int",
+	'deft' => "10",
+	'reqd' => "no" },
+      { 'name' => "max_near_text",
+	'desc' => "{W3ImgPlug.max_near_text}",
+	'type' => "int",
+	'deft' => "400",
+	'reqd' => "no" },
+      { 'name' => "smallpage_threshold",
+	'desc' => "{W3ImgPlug.smallpage_threshold}",
+	'type' => "int",
+	'deft' => "2048",
+	'reqd' => "no" },
+      { 'name' => "textrefs_threshold",
+	'desc' => "{W3ImgPlug.textrefs_threshold}",
+	'type' => "int",
+	'deft' => "2",
+	'reqd' => "no" },
+      { 'name' => "caption_length",
+	'desc' => "{W3ImgPlug.caption_length}",
+	'type' => "int",
+	'deft' => "80",
+	'reqd' => "no" },
+      { 'name' => "neartext_length",
+	'desc' => "{W3ImgPlug.neartext_length}",
+	'type' => "int",
+	'deft' => "300",
+	'reqd' => "no" },
+      { 'name' => "document_text",
+	'desc' => "{W3ImgPlug.document_text}",
+	'type' => "flag",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "W3ImgPlug",
+		'desc'     => "{W3ImgPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new HTMLPlug($pluginlist,$inputargs,$hashArgOptLists): new HTMLPlug($pluginlist,$inputargs);
+
+    # init class variables
+    $self->{'textref'} = undef; # init by read_file fn
+    $self->{'htdoc_obj'} = undef; # init by process fn
+    $self->{'htpath'} = undef; # init by process fn
+    $self->{'hturl'} = undef; # init by process fn
+    $self->{'plaintext'} = undef; # HTML stripped version - only init if needed by raw_neartext sub
+    $self->{'smallpage'} = 0; # set by process fn
+    $self->{'images_indexed'} = undef; # num of images indexed - if 1 or 2 then we know page is small
+    $self->{'initialised'} = undef; # flag (see set_extraction_options())
+
+    return bless $self, $class;
+}
+
+# if indexing pages, let HTMLPlug do it's stuff
+# image extraction done through read()
+sub process {
+    my ($self, $textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    $self->{'imglist'} = ();
+    if ( $self->{'index_pages'} ) {
+	my $ok = $self->SUPER::process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj);
+	if ( ! $ok ) { return $ok }
+	$self->{'htdoc_obj'} = $doc_obj;
+    }
+    # else use URL for referencing
+    #if ( $file =~ /(.*)[\/\\]/ ) { $self->{'htpath'} = $1; } else { $self->{'htpath'} = $file; }
+
+    $self->{'htpath'} = $base_dir if (-d $base_dir);
+    if ( $file =~ /(.*)[\/\\]/ ) { $self->{'htpath'} .= "/$1"; }
+    $self->{'htpath'} =~ s/\\/\//g;  # replace \ with /
+
+    $self->{'hturl'} = "http://$file";
+    $self->{'hturl'} =~ s/\\/\//g; # for windows
+    ($self->{'filename'}) = $file =~ /.*[\/\\](.*)/;
+    ($self->{'base_path'}) = $file =~ /(.*)[\/\\]/i;
+    if ( ( -s "$base_dir/$file") <= $self->{'smallpage_threshold'} ) {
+	$self->{'smallpage'} = 1;
+    } else { $self->{'smallpage'} = 0; }
+
+    if ( defined($self->{'initialised'}) ) { return 1; }
+    else {
+	$self->{'initialised'} = $self->set_extraction_options($base_dir =~ /^(.*?)\/import/i);
+	return $self->{'initialised'};
+    }
+}
+
+# get complex configuration options from configuration files
+# -- $GSDLCOLLECTION/etc/W3ImgPlug.cfg (tag sets for aggr 2+)
+# -- $GSDLHOME/etc/packages/phind/stopword/en/brown.sw (stopwords for aggr 5+)
+
+# If there's no W3ImgPlug.cfg file we'll use the following default values
+my $defaultcfg = '
+<delimitertagset>
+  <setname>Caption</setname>
+  <taggroup>font</taggroup>
+  <taggroup>tt</taggroup>
+  <taggroup>small</taggroup>
+  <taggroup>b</taggroup>
+  <taggroup>i</taggroup>
+  <taggroup>u</taggroup>
+  <taggroup>em</taggroup>
+  <taggroup>td</taggroup>
+  <taggroup>li</taggroup>
+  <taggroup>a</taggroup>
+  <taggroup>p</taggroup>
+  <taggroup>tr</taggroup>
+  <taggroup>center</taggroup>
+  <taggroup>div</taggroup>
+  <taggroup>caption</taggroup>
+  <taggroup>br</taggroup>
+  <taggroup>ul</taggroup>
+  <taggroup>ol</taggroup>
+  <taggroup>table</taggroup>
+  <taggroup>hr</taggroup>
+</delimitertagset>
+
+<delimitertagset>
+  <setname>Neartext</setname>
+  <taggroup>tr|hr|table|h\d|img|body</taggroup>
+  <taggroup>td|tr|hr|table|h\d|img|body</taggroup>
+  <taggroup>p|br|td|tr|hr|table|h\d|img|body</taggroup>
+  <taggroup>font|p|i|b|em|img</taggroup>
+</delimitertagset>
+';
+
+sub set_extraction_options() {
+    my ($self, $collpath) = @_;
+    my ($filepath);
+
+    print {$self->{'outhandle'}} "W3ImgPlug: Initialising\n"
+	if $self->{'verbosity'} > 1;
+    # etc/W3ImgPlug.cfg (XML)
+    # tag sets for captions and neartext
+    if ( $self->{'aggressiveness'} > 1 && $self->{'aggressiveness'} != 9 ) {
+	$self->{'delims'} = [];
+	$self->{'cdelims'} = [];
+	my ($cfg, @tagsets, $tagset, $type, @delims);
+
+	$filepath = "$collpath/etc/W3ImgPlug.cfg";
+	if ( open CFG, "<$filepath" ) {
+	    while (<CFG>) { $cfg .= $_ }
+	    close CFG;
+	} else {
+	    $cfg = $defaultcfg;
+	}
+
+	(@tagsets) = 
+	    $cfg =~ /<delimitertagset>(.*?)<\/delimitertagset>/igs;
+	foreach $tagset ( @tagsets ) {
+	    ($type) = $tagset =~ /<setname>(.*?)<\/setname>/i;
+	    if ( lc($type) eq "caption" ) {
+		(@{$self->{'cdelims'}}) = $tagset =~ /<taggroup>(.*?)<\/taggroup>/igs;
+	    }
+	    elsif ( lc($type) eq "neartext" ) {
+		(@{$self->{'delims'}}) = $tagset =~ /<taggroup>(.*?)<\/taggroup>/igs;
+	    }
+	}
+
+	# output a warning if there seem to be no delimiters
+	if ( scalar(@{$self->{'cdelims'}} == 0)) {
+	    print {$self->{'outhandle'}} "W3ImgPlug: Warning: no caption delimiters found in $filepath\n";
+	}
+	if ( scalar(@{$self->{'delims'}} == 0)) {
+	    print {$self->{'outhandle'}} "W3ImgPlug: Warning: no neartext delimiters found in $filepath\n";
+	}
+    }
+    
+    # get stop words for textual reference extraction
+    # TODO: warnings scroll off. Would be best to output them again at end of import
+    if ( $self->{'aggressiveness'} >=5 && $self->{'aggressiveness'} != 9 ) {
+	$self->{'stopwords'} = ();
+	$filepath = &util::filename_cat($ENV{'GSDLHOME'}, "etc", "packages", "phind", "stopword", "en", "brown.sw");
+	if ( open STOPWORDS, "<$filepath" ) {
+	    while ( <STOPWORDS> ) {
+		chomp;
+		$self->{'stopwords'}{$_} = 1;
+	    }
+	    close STOPWORDS;
+	} else {
+	    print {$self->{'outhandle'}} "W3ImgPlug: Warning: couldn't open stopwords file at $filepath ($!)\n";
+	}
+	
+    }
+
+    if ( $self->{'neartext_length'} > $self->{'max_near_text'} ) {
+	$self->{'max_near_text'} = $self->{'neartext_length'} * 1.33;
+	print {$self->{'outhandle'}} "W3ImgPlug: Warning: adjusted max_text to $self->{'max_near_text'}\n";
+    } 
+    if ( $self->{'caption_length'} > $self->{'max_near_text'} ) {
+	$self->{'max_near_text'} = $self->{'caption_length'} * 1.33;
+	print {$self->{'outhandle'}} "W3ImgPlug: Warning: adjusted max_text to $self->{'max_near_text'}\n";
+    }
+
+    return 1;
+}
+
+# return number of files processed, undef if can't recognise, -1 if 
+# cant process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my ($self, $pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = (@_);
+    my ($doc_obj, $section, $filepath, $imgtag, $pos, $context, $numdocs, $tndir, $imgs);
+    # forward normal read (runs HTMLPlug if index_pages T)
+    my $ok =  $self->SUPER::read($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli); 
+    if ( ! $ok ) { return $ok } # what is this returning??
+
+    my $outhandle = $self->{'outhandle'};
+    my $textref = $self->{'textref'};
+    my $htdoc_obj = $self->{'htdoc_obj'};
+    $numdocs = 0;
+    $base_dir =~ /(.*)\/.*/;
+    $tndir = "$1/archives/thumbnails"; # TODO: this path shouldn't be hardcoded?
+    &util::mk_all_dir($tndir) unless -e "$tndir"; 
+
+    $imgs = \%{$self->{'imglist'}};
+    my $nimgs = $self->get_img_list($textref);
+    $self->{'images_indexed'} = $nimgs;
+    if ( $nimgs > 0 ) {
+	my @fplist = (sort { $imgs->{$a}{'pos'} <=> $imgs->{$b}{'pos'} } keys %{$imgs});
+	my $i = 0;
+	foreach $filepath ( @fplist ) {
+	    $pos = $imgs->{$filepath}{'pos'}; 
+	    $context = substr ($$textref, $pos - 50, $pos + 50); # grab context (quicker)
+	    ($imgtag) = ($context =~ /(<(?:img|a|body)\s[^>]*$filepath[^>]*>)/is );
+	    if (! defined($imgtag)) { $imgtag = $filepath }
+	    print $outhandle "W3ImgPlug: extracting $filepath\n"
+		if ( $self->{'verbosity'} > 1 );
+	    $doc_obj = new doc ("", "indexed_doc");
+	    $section = $doc_obj->get_top_section();
+	    my $prevpos = ( $i == 0 ? 0 : $imgs->{$fplist[$i - 1]}{'pos'});
+	    my $nextpos = ( $i >= ($nimgs -1) ? -1 : $imgs->{$fplist[$i + 1]}{'pos'} );
+
+	    $self->extract_image_info($imgtag, $filepath, $textref, $doc_obj, $section, $tndir, $prevpos, $nextpos);
+            $processor->process($doc_obj);
+	    $numdocs++;
+	    $i++;
+	}
+	return $numdocs;
+    } else {
+	print $outhandle "W3ImgPlug: No images from $file indexed\n"
+	    if ( $self->{'verbosity'} > 2 );
+	return 1;
+    }
+    
+}
+
+# for every valid image tag
+# 1. extract related text and image metadata
+# 2. add this as document meta-data
+# 3. add assoc image(s) as files
+#
+sub extract_image_info {
+    my $self = shift (@_);
+    my ($tag, $id, $textref, $doc_obj, $section, $tndir, $prevpos, $nextpos) = (@_);
+    my ($filename, $orig_fp, $fn, $ext, $reltext, $relreltext, $crcid, $imgs,
+	$thumbfp, $pagetitle, $alttext,	$filepath, $aggr);
+    $aggr = $self->{'aggressiveness'};
+    $imgs = \%{$self->{'imglist'}};
+    $filepath = $imgs->{$id}{'relpath'}; 
+    ($filename) = $filepath =~ /([^\/\\]+)$/s;
+    ($orig_fp) = "$self->{'base_path'}/$filepath"; 
+    $orig_fp =~ tr/+/ /;
+    $orig_fp =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; # translate %2E to space, etc
+    $orig_fp =~ s/\\/\//g;
+    $filepath = "$self->{'htpath'}/$filepath";
+    my ($onlyfn) = $filename =~ /([^\\\/]*)$/;
+    ($fn, $ext) = $onlyfn =~ /(.*)\.(.*)/;
+    $fn = lc $fn; $ext = lc $ext;
+    ($reltext) = "<tr><td>GifComment</td><td>" . `identify $filepath -ping -format "%c"` . "</td></tr>\n"
+        if ($ext eq "gif");
+    $reltext .= "<tr><td>FilePath</td><td>$orig_fp</td></tr>\n";
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$crcid = "$fn.$ext." . $self->{'next_crcid'}++;
+    } else { ($crcid) = `cksum $filepath` =~ /^(\d+)/; }
+    $thumbfp = "$tndir/tn_$crcid.jpg";
+    `convert -flatten -filter Hanning $self->{'convert_params'} -geometry "$self->{'thumb_size'}x$self->{'thumb_size'}>" $filepath $thumbfp` unless -e $thumbfp;
+    if ( ! (-e $thumbfp) ) {
+	print STDERR "W3ImgPlug: 'convert' failed. Check ImageMagicK binaries are installed and working correctly\n"; return 0; 
+    }
+    
+    # shove in full text (tag stripped or unstripped) if settings require it
+    if ( $aggr == 10) {
+	$reltext = "<tr><td>AllPage</td><td>" . $$textref . "</td><tr>\n";   # level 10 (all text, verbatim)
+    } else {
+	$pagetitle = $self->get_meta_value("title", $textref);
+	($alttext) = $tag =~ /\salt\s*=\s*(?:\"|\')(.+?)(?:\"|\')/is;
+	if ( defined($alttext) && length($alttext) > 1) {
+	    $reltext .= "<tr><td>ALTtext</td><td>$alttext</td></tr>\n"; }
+	$reltext .= "<tr><td>SplitCapitalisation</td><td>" . 
+	    $self->split_filepath($orig_fp) . "</td></tr>\n";
+
+	# get caption/tag based near text (if appropriate)
+	if ( $aggr > 1 ) {
+	    if ( $aggr >= 2 ) {
+		$reltext .= 
+		    $self->extract_caption_text($tag, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos);
+		$relreltext = $reltext;
+	    } 
+	    # repeat the filepath, alt-text, caption, etc
+	    if ( $aggr == 8 ) {
+		$reltext .= $relreltext; 
+	    }
+	    if ( $aggr >= 3 ) {
+		$reltext .= 
+		    $self->extract_near_text($tag, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos);
+	    }
+	
+	    # get page metadata (if appropriate)
+	    if ( $aggr >= 6 || ( $aggr >= 2 && 
+						     ( $self->{'images_indexed'} < 2 || 
+						       ($self->{'smallpage'} == 1 && $self->{'images_indexed'} < 6 )))) {	
+		$reltext .= $self->get_page_metadata($textref);
+	    }
+	    # textual references
+	    if ( $aggr  == 5 || $aggr >= 7) {
+		if ( length($relreltext) > ($self->{'caption_length'} * 2) )  {
+		    $reltext .= $self->get_textrefs($relreltext, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos); }
+		else {
+		    $reltext .= $self->get_textrefs($reltext, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos); 
+		}
+	    }
+	} # aggr > 1
+    } # aggr != 10
+    
+    $doc_obj->set_OID($crcid); 
+    $doc_obj->associate_file($thumbfp, "$fn.thumb.jpg", undef, $section);
+    $doc_obj->add_metadata($section, "OriginalFilename", $filename);
+    $doc_obj->add_metadata($section, "FilePath", $orig_fp);
+    $doc_obj->add_metadata($section, "Filename", $fn);
+    $doc_obj->add_metadata($section, "FileExt", $ext);
+    $doc_obj->add_metadata($section, "FileSize", $imgs->{$id}{'filesize'});
+    $doc_obj->add_metadata($section, "Width", $imgs->{$id}{'width'});
+    $doc_obj->add_metadata($section, "Height", $imgs->{$id}{'height'});
+    $doc_obj->add_metadata($section, "URL", "http://$orig_fp");
+    $doc_obj->add_metadata($section, "PageURL", $self->{'hturl'});
+    $doc_obj->add_metadata($section, "PageTitle", $pagetitle);
+    $doc_obj->add_metadata($section, "ThumbURL", 
+			   "_httpcollection_/index/assoc/[archivedir]/$fn.thumb.jpg");
+    $doc_obj->add_metadata($section, "FileFormat", "W3Img");
+
+    if ( $self->{'document_text'} ) {
+	$doc_obj->add_utf8_text($section, "<table border=1>\n$reltext</table>");
+    } else {
+	$doc_obj->add_metadata($section, "ImageText", "<table border=1>\n$reltext</table>\n");
+    }
+    
+    if ( $self->{'index_pages'} ) {
+	my ($cache_url) = "_httpdoc_&d=" . $self->{'htdoc_obj'}->get_OID();
+	if ( $imgs->{$id}{'anchored'} ) {
+	    my $a_name = $id;
+	    $a_name =~ s/[\/\\\:\&]/_/g;
+	    $cache_url .=  "#gsdl_$a_name" ;
+	}
+	$doc_obj->add_utf8_metadata($section, "CachePageURL", $cache_url);
+    }
+    if ( ! $self->{'no_cache_images'} ) {
+	$onlyfn = lc $onlyfn;
+	$doc_obj->associate_file($filepath, $onlyfn, undef, $section);
+	$doc_obj->add_utf8_metadata($section, "CacheURL", 
+			       "_httpcollection_/index/assoc/[archivedir]/$onlyfn");
+    }
+    return 1;
+}
+
+sub get_page_metadata {
+    my ($self, $textref) = (@_);
+    my (@rval);
+    $rval[0] = $self->get_meta_value("title", $textref);
+    $rval[1] = $self->get_meta_value("keywords", $textref);
+    $rval[2] = $self->get_meta_value("description", $textref);
+    $rval[3] = $self->{'filename'};
+
+    return wantarray ? @rval : "<tr><td>PageMeta</td><td>@rval</td></tr>\n" ;
+}
+
+# turns LargeCatFish into Large,Cat,Fish so MG sees the separate words
+sub split_filepath {
+    my ($self, $filepath) = (@_);
+    my (@words) = $filepath =~ /([A-Z][a-z]+)/g; 
+    return join(',', @words);
+}
+
+# finds and extracts sentences
+# that seem to be on the same topic
+# as other related text (correlations)
+# and textual references (e.g. in figure 3 ...)
+sub get_textrefs {
+    my ($self, $reltext, $textref, $prevpos, $pos, $nextpos) = (@_);
+    my ($maxtext, $mintext, $startpos, $context_size, $context);
+
+    my (@relwords, @refwords, %sentences, @pagemeta);
+
+    # extract larger context
+    $maxtext = $self->{'max_near_text'};
+    $startpos = $pos - ($maxtext * 4);
+    $context_size = $maxtext*10;
+    if ($startpos < $prevpos ) { $startpos = $prevpos }
+    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) { $context_size = ($nextpos - $startpos) }
+    $context = substr ( $$textref, $startpos, $context_size );
+    $context =~ s/<.*?>//gs;
+    $context =~ s/^.*>(.*)/$1/gs;
+    $context =~ s/(.*)<.*$/$1/gs;
+
+    # get page meta-data (if not already included)
+    if ( $self->{'aggressiveness'} == 5 && ! $self->{'smallpage'} ) {
+	@pagemeta = $self->get_page_metadata($textref);
+	foreach my $value ( @pagemeta ) {
+	    $context .= "$value."; # make each into psuedo-sentence
+	}
+    }
+
+    # TODO: this list is not exhaustive
+    @refwords = ( '(?:is|are)? ?(?:show(?:s|n)|demonstrate(?:d|s)|explains|features) (?:in|by|below|above|here)', 
+		  '(?:see)? (?:figure|table)? (?:below|above)');    
+
+    # extract general references
+    foreach my $rw ( @refwords ) {
+	while ( $context =~ /[\.\?\!\,](.*?$rw\W.*?[\.\?\!\,])/ig ) {
+	    my $sentence = $1;
+	    $sentence =~ s/\s+/ /g;
+	    $sentences{$sentence}+=2;
+	}
+    }
+    # extract specific (figure, table) references by number
+    my ($fignum) = $context =~ /[\.\?\!].*?(?:figure|table)s?[\-\_\ \.](\d+\w*)\W.*?[\.\?\!]/ig;
+    if ( $fignum ) {
+	foreach my $rw ( @refwords ) {
+	    while ( $context =~ /[\.\?\!](.*?(figure|table)[\-\_\ \.]$fignum\W.*?[\.\?\!])/ig ) {
+		my $sentence = $1;
+		$sentence =~ s/\s+/ /g;
+		$sentences{$sentence}+=4;
+	    }
+	}
+    }
+
+    # sentences with occurances of important words
+    @relwords = $reltext =~ /([a-zA-Z]{4,})/g; # take out small words
+    foreach my $word ( @relwords ) {
+	if ( $self->{'stopwords'}{$word} ) { next } # skip stop words
+	while ( $context =~ /([^\.\?\!]*?$word\W.*?[\.\?\!])/ig ) {
+	    my $sentence = $1;
+	    $sentence =~ s/\s+/ /g;
+	    $sentences{$sentence}++;
+	}
+    }
+    foreach my $sentence ( keys %sentences ) {
+	if ($sentences{$sentence} < $self->{'textrefs_threshold'}) {
+	    delete $sentences{$sentence};
+	}
+    }
+    my ($rval) = join "<br>\n", (keys %sentences);
+    if ( $rval && length($rval) > 5 ) {
+	return ( "<tr><td>TextualReferences</td><td>" . $rval . "</td></tr>\n") }
+    else { return "" }
+}
+
+# handles caption extraction
+# calling the extractor with different
+# tags and choosing the best candidate caption
+sub extract_caption_text {
+    my ($self, $tag, $textref, $prevpos, $pos, $nextpos) = (@_);
+    my (@neartext, $len, $hdelim, $mintext, $goodlen,
+	$startpos, $context, $context_size);
+    
+    $mintext = $self->{'min_near_text'};
+    $goodlen = $self->{'caption_length'};
+
+    # extract a context to extract near text from (faster)
+    $context_size = $self->{'max_near_text'}*3;
+    $startpos = $pos - ($context_size / 2);
+    if ($startpos < $prevpos ) { $startpos = $prevpos }
+    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) 
+    { $context_size = ($nextpos - $startpos) }
+
+    $context = substr ( $$textref, $startpos, $context_size );
+    $context =~ s/<!--.*?-->//gs;
+    $context =~ s/^.*-->(.*)/$1/gs;
+    $context =~ s/(.*)<!--.*$/$1/gs;
+
+    # try stepping through markup delimiter sets
+    # and selecting the best one
+    foreach $hdelim ( @{ $self->{'cdelims'} } ) {
+	@neartext = $self->extract_caption($tag, $hdelim, \$context);
+	$len = length(join("", @neartext));
+	last if ($len >= $mintext && $len <= $goodlen);
+    }
+    # reject if well over reasonable length
+    if ( $len > $goodlen ) {
+	@neartext = [];
+    }
+    $neartext[0] = " " if (! defined $neartext[0]);
+    $neartext[1] = " " if (! defined $neartext[1]);
+    return "<tr><td>Caption</td><td>" . (join ",",  @neartext) . "</td></tr>\n"; # TODO: the | is for testing purposes
+} # end extract_caption_text
+
+# the previous section header often gives a bit
+# of context to the section that the image is
+# in (invariably the header is before/above the image)
+# so extract the text of the closest header above the image
+#
+# this fn just gets all the headers above the image, within the context window
+sub get_prev_header {
+    my ($self, $pos, $textref) = (@_);
+    my ($rhtext);
+    while ( $$textref =~ /<h\d>(.*?)<\/h\d>/sig ) {
+	# only headers before image
+	if ((pos $$textref) < $pos) {
+	    $rhtext .= "$1, ";
+	}
+    } 
+    if ( $rhtext ) { return "Header($rhtext)" }
+    else { return "" }
+}
+
+# not the most robust tag stripping 
+# regexps (see perl.com FAQ) but good enough
+#
+# used by caption & tag-based near text algorithms
+sub strip_tags {
+    my ( $self, $value ) = @_;
+    if ( ! defined($value) ) { $value = "" } # handle nulls
+    else {
+	$value =~ s/<.*?>//gs; # strip all html tags
+	$value =~ s/\s+/\ /g; # remove extra whitespace
+	$value =~ s/\&\w+\;//g; # remove &nbsp; etc
+    }
+    return $value;
+}
+
+# uses the given tag(s) to identify
+# the caption near to the image
+# (below, above or both below and above)
+sub extract_caption {
+    my ($self, $tag, $bound_tag, $contextref) = (@_);
+    my (@nt, $n, $etag, $gotcap);
+    return ("", "") if ( ! ($$contextref =~ /\Q$tag/) );
+
+    $nt[0] = $`;
+    $nt[1] = $';
+    $gotcap = 0;
+
+    # look before the image for a boundary tag
+    ($etag, $nt[0]) = $nt[0] =~ /<($bound_tag)[\s]?.*?>(.*?)$/is;
+    # if bound_tag too far from the image, then prob not caption
+    # (note: have to allow for tags, so multiply by 3
+    if ( $etag && length($nt[0]) < ($self->{'caption_length'} * 3) ) { 
+	if ( $nt[0] =~ /<\/$etag>/si ) {
+	    # the whole caption is above the image: <tag>text</tag><img>
+	    ($nt[0]) =~ /<(?:$etag)[\s]?.*?>(.*?)<\/$etag>/is;
+	    $nt[0] = $self->strip_tags($nt[0]);
+	    if ( length($nt[0]) > $self->{'min_near_text'} ) { 
+		$gotcap = 1;
+		$nt[1] = ""; 
+	    }
+
+	} elsif ( $nt[1] =~ /<\/$etag>/si) {
+	    # the caption tag covers image: <tag>text?<img>text?</tag>
+	    ($nt[1]) = $nt[1] =~ /(.*?)<\/$etag>/si;
+	    $nt[0] = $self->strip_tags($nt[0] . $nt[1]);
+	    if ( length($nt[0]) > $self->{'min_near_text'} ) { 
+		$gotcap = 2;
+		$nt[1] = "";
+	    }
+	}
+    }
+    # else try below the image
+    if ( ! $gotcap ) { 
+	# the caption is after the image: <img><tag>text</tag>
+	($etag, $nt[1]) = $nt[1] =~ /^.*?<($bound_tag)[\s]?.*?>(.*)/is;
+	if ( $etag && $nt[1] =~ /<\/$etag>/s) {
+	    ($nt[1]) = $nt[1] =~ /(.*?)<\/$etag>/si;
+	    $gotcap = 3;
+	    $nt[0] = "";
+	    $nt[1] = $self->strip_tags($nt[1]);
+	} 
+    }
+    if ( ! $gotcap ) { $nt[0] = $nt[1] = "" }
+    else {
+	# strip part-tags
+	$nt[0] =~ s/^.*>//s;
+	$nt[1] =~ s/<.*$//s;
+    }
+    my ($type);
+    if ( $gotcap == 0 ) { return ("nocaption", "") }
+    elsif ( $gotcap == 1 ) { $type = "captionabove:" }
+    elsif ( $gotcap == 2 ) { $type = "captioncovering:" }
+    elsif ( $gotcap == 3 ) { $type = "captionbelow:" }
+    return ($type, $nt[0], $nt[1]);
+}
+
+# tag-based near text
+# 
+# tries different tag sets
+# and chooses the best one
+sub extract_near_text {
+    my ($self, $tag, $textref, $prevpos, $pos, $nextpos) = (@_);
+    my (@neartext, $len, $hdelim, $maxtext, $mintext, $goodlen,
+	@bestlen, @best, $startpos, $context, $context_size,
+	$dist, $bdist, $best1, $i, $nt);
+    $bestlen[0] = $bestlen[1] = 0; $bestlen[2] = $bdist = 999999;
+    $best[0] = $best[1] = $best[2] = "";
+    $maxtext = $self->{'max_near_text'};
+    $mintext = $self->{'min_near_text'};
+    $goodlen = $self->{'neartext_length'}; 
+
+    # extract a context to extract near text from (faster)
+    $context_size = $maxtext*4;
+    $startpos = $pos - ($context_size / 2);
+    if ($startpos < $prevpos ) { $startpos = $prevpos }
+    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) 
+    { $context_size = ($nextpos - $startpos) }
+    $context = substr ( $$textref, $startpos, $context_size );
+    $context =~ s/<!--.*?-->//gs;
+    $context =~ s/^.*-->(.*)/$1/gs;
+    $context =~ s/(.*)<!--.*$/$1/gs;
+
+    # try stepping through markup delimiter sets
+    # and selecting the best one
+    foreach $hdelim ( @{ $self->{'delims'} } ) {
+	@neartext = $self->extract_tagged_neartext($tag, $hdelim, \$context);
+	$nt = join("", @neartext);
+	$len = length($nt);
+	# Priorities:
+	# 1. Greater than mintext 
+	# 2. Less than maxtext 
+	# 3. Closest to goodlen
+	if ( $len <= $goodlen && $len > $bestlen[0] ) {
+	    $bestlen[0] = $len;
+	    $best[0] = $hdelim;
+	} elsif ( $len >= $maxtext && $len < $bestlen[2] ) {
+	    $bestlen[2] = $len;
+	    $best[2] = $hdelim;
+	} elsif ( $len >= $bestlen[0] && $len <= $bestlen[2] ) {
+	    $dist = abs($goodlen - $len);
+	    if ( $dist < $bdist ) {
+		$bestlen[1] = $len;
+		$best[1] = $hdelim;
+		$bdist = $dist;
+	    }
+	}
+    }
+    $best1 = 2;
+    foreach $i ( 0..2 ) {
+	if ( $bestlen[$i] == 999999 ) { $bestlen[$i] = 0 }
+	$dist = abs($goodlen - $bestlen[$i]);
+	if ( $bestlen[$i] > $mintext && $dist <= $bdist ) {
+	    $best1 = $i;
+	    $bdist = $dist;
+	}
+    }
+    @neartext = $self->extract_tagged_neartext($tag, $best[$best1], \$context);
+    if ( $bestlen[$best1] > $maxtext ) {
+	# truncate on word boundary if too much text
+	my $hmax = $maxtext / 2;
+	($neartext[0]) = $neartext[0] =~ /([^\s]*.{1,$hmax})$/s;
+	($neartext[1]) = $neartext[1] =~ /^(.{1,$hmax}[^\s]*)/s;
+    } elsif ( $bestlen[$best1] < $mintext ) {
+	# use plain text extraction if tags failed (e.g. usable tag outside context)
+	print {$self->{'outhandle'}} "W3ImgPlug: Fallback to plain-text extraction for $tag\n" 
+	    if $self->{'verbosity'} > 2;
+	$neartext[0] = "<tr><td>RawNeartext</td><td>" . $self->extract_raw_neartext($tag, $textref) . "</td></tr>";
+	$neartext[1] = "";
+    }
+    # get previous header if available
+    $neartext[0] .= "<br>\n" . 
+	$self->get_prev_header($pos, \$context) if ( $self->{'aggressiveness'} >= 4 );
+    $neartext[0] = " " if (! defined $neartext[0]);
+    $neartext[1] = " " if (! defined $neartext[1]);
+
+    return "<tr><td>NearText</td><td>" . (join "|",  @neartext) . "</td></tr>\n"; # TODO: the | is for testing purposes
+} # end extract_near_text
+
+# actually captures tag-based
+# near-text given a tag set
+sub extract_tagged_neartext {
+    my ($self, $tag, $bound_tag, $textref) = (@_);
+    return "" if ( ! ($$textref =~ /\Q$tag/) );
+    my (@nt, $delim, $pre_tag, $n);
+    $nt[0] = $`;
+    $nt[1] = $';
+
+    # get text after previous image tag
+    $nt[0] =~ s/.*<($bound_tag)[^>]*>(.*)/$2/is; # get rid of preceding text
+    if (defined($1)) { $delim = $1 }    
+    $pre_tag = $bound_tag;
+
+    if (defined($delim)) {
+	# we want to try and use the end tag of the previous delimiter
+	# (put it on the front of the list)
+	$pre_tag =~ s/(^|\|)($delim)($|\|)//i; # take it out
+	$pre_tag =~ s/\|\|/\|/i; # replace || with |
+	$pre_tag = $delim . "|" . $pre_tag; # put it on the front
+    }
+    
+    # get text before next image tag
+    $nt[1] =~ s/<\/?(?:$pre_tag)[^>]*>.*//is; # get rid of stuff after first delimiter
+
+    # process related text
+    for $n (0..1) {
+	if ( defined($nt[$n]) ) {
+	    $nt[$n] =~ s/<.*?>//gs; # strip all html tags
+	    $nt[$n] =~ s/\s+/\ /gs; # remove extra whitespace
+	    $nt[$n] =~ s/\&\w+\;//sg; # remove &nbsp; etc
+	    # strip part-tags
+	    if ( $n == 0 ) { $nt[0] =~ s/^.*>//s }
+	    if ( $n == 1 ) { $nt[1] =~ s/<.*$//s }
+	} else { $nt[$n] = ""; } # handle nulls
+    }
+    return @nt;
+}
+
+# this function is fall-back
+# if tags aren't suitable.
+#
+# extracts a fixed length of characters
+# either side of image tag (on word boundary)
+sub extract_raw_neartext {
+    my ($self, $tag, $textref) = (@_);
+    my ($rawtext, $startpos, $fp);
+    my $imgs = \%{$self->{'imglist'}};
+    ($fp) = $tag =~ /([\w\\\/]+\.(?:gif|jpe?g|png))/is;
+    if (! $fp) { return " " };
+    # if the cached, plain-text version isn't there, then create it
+    $self->init_plaintext($textref) unless defined($self->{'plaintext'});
+
+    # take the closest maxtext/2 characters 
+    # either side of the tag (by word boundary)
+    return "" if ( ! exists $imgs->{$fp}{'rawpos'} );
+    $startpos = $imgs->{$fp}{'rawpos'} - (($self->{'max_near_text'} / 2) + 20);
+    if ( $startpos < 0 ) { $startpos = 0 }
+    $rawtext = substr $self->{'plaintext'}, $startpos, $self->{'max_near_text'} + 20;
+    $rawtext =~ s/\s\s/ /g;
+
+    return $rawtext;
+}
+
+# init plaintext variable for HTML-stripped version 
+# (for full text index/raw assoc text extraction)
+sub init_plaintext {
+    my ($self, $textref) = (@_);
+    my ($page, $fp);
+    my $imgs = \%{$self->{'imglist'}};
+    $page = $$textref; # make a copy of original
+
+    # strip tags around image filenames so they don't get zapped
+    $page =~ s/<\w+\s+.*?([\w\/\\]+\.(?:gif|jpe?g|png))[^>]*>/\"$1\"/gsi;
+    $page =~ s/<.*?>//gs;
+    $page =~ s/&nbsp;/ /gs;
+    $page =~ s/&amp;/&/gs; #TODO: more &zzz; replacements (except &lt;, $gt;)
+
+    # get positions and strip images
+    while ( $page =~ /([^\s\'\"]+\.(jpe?g|gif|png))/ig ) {
+	$fp = $1;
+	if ( $imgs->{$fp}{'exists'} ) {
+	    $imgs->{$fp}{'rawpos'} = pos $page;
+	}
+	$page =~ s/\"$fp\"//gs;
+    }
+    $self->{'plaintext'} = $page;
+}
+
+# finds and filters images based on size 
+# (dimensions, height, filesize) and existence
+#
+# looks for image filenames (.jpg, .gif, etc)
+# and checks for existence on disk 
+# (hence supports most JavaScript images)
+sub get_img_list {
+    my $self = shift (@_);
+    my ($textref) = (@_);
+    my ($filepath, $relpath, $abspath, $pos, $num, $width, $height, $filesize);
+    my $imgs = \%{$self->{'imglist'}};
+    while ( $$textref =~ /([^\s\'\"]+\.(jpe?g|gif|png))/ig ) {
+	$filepath = $1;
+	$pos = pos $$textref;
+	next if ( $imgs->{$filepath}{'relpath'} );
+        $relpath = $filepath;
+	$relpath =~ s/^http\:\/\///; # remove http:// in case we have mirrored it
+	$relpath =~ s/\\/\//g;  # replace \ with /
+	$relpath =~ s/^\.\///s; # make "./filepath" into "filepath"
+	$imgs->{$filepath}{'relpath'} = $relpath;
+	$abspath = "$self->{'htpath'}/$relpath";
+
+	if (! -e $abspath) { next }
+
+	# can't modify real filepath var because it
+	# then can't be located in the page for tag recognition later
+	($width, $height) = 
+	    `identify $abspath -ping -format "%wx%h"` =~ /^(\d*)x(\d*)$/m;
+	if (! ($width && $height)) { 
+	    print STDERR "W3ImgPlug: ($abspath) 'identify' failed. Check ImageMagicK binaries are installed and working correctly\n"; next;
+	}
+	$filesize = (-s $abspath);
+	if ( $filesize >= $self->{'min_size'} 
+	    && ( $width >= $self->{'min_width'} ) 
+	    && ( $height >= $self->{'min_height'} ) ) {
+	    
+	   $imgs->{$filepath}{'exists'} = 1;
+	   $imgs->{$filepath}{'pos'} = $pos;
+	   $imgs->{$filepath}{'width'} = $width;
+	   $imgs->{$filepath}{'height'} = $height;
+	   $imgs->{$filepath}{'filesize'} = $filesize;
+       } else {
+	   print {$self->{'outhandle'}} "W3ImgPlug: skipping $self->{'base_path'}/$relpath: $filesize, $width x $height\n" 
+	       if $self->{'verbosity'} > 2;
+       }
+    }
+    $num = 0;
+    foreach my $i ( keys %{$imgs} ) {
+	if ( $imgs->{$i}{'pos'} ) {
+	    $num++;
+	} else { delete $imgs->{$i} }
+    }
+    return $num;
+}
+
+# make the text available to the read function
+# by making it an object variable
+sub read_file {
+    my ($self, $filename, $encoding, $language, $textref) = @_;
+    $self->SUPER::read_file($filename, $encoding, $language, $textref);
+
+    # if HTMLplug has run through, then it will
+    # have replaced references so we have to 
+    # make a copy of the text before processing
+    if ( $self->{'index_pages'} ) {
+	$self->{'text'} = $$textref;
+	$self->{'textref'} = \($self->{'text'});
+    } else {
+	$self->{'textref'} = $textref;
+    }
+    $self->{'plaintext'} = undef;
+}
+
+# HTMLPlug only extracts meta-data if it is specified in plugin options
+# hence a special function to do it here
+sub get_meta_value {
+    my ($self, $name, $textref) = @_;
+    my ($value);
+    $name = lc $name;
+    if ($name eq "title") {
+	($value) = $$textref =~ /<title>(.*?)<\/title>/is
+    } else {
+	my $qm = "(?:\"|\')";
+	($value) = $$textref =~ /<meta name\s*=\s*$qm?$name$qm?\s+content\s*=\s*$qm?(.*?)$qm?\s*>/is
+    }
+    $value = "" unless $value;
+    return $value;
+}
+
+# make filename an anchor reference
+# so we can go straight to the image
+# within the cached version of the source page
+# (augment's HTMLPlug sub)
+sub replace_images {
+    my $self = shift (@_);
+    my ($front, $link, $back, $base_dir, 
+	$file, $doc_obj, $section) = @_;
+    $link =~ s/\"//g;
+    my ($a_name) = $link;
+    $a_name =~ s/[\/\\\:\&]/_/g;
+    # keep a list so we don't repeat the same anchor
+    if ( ! $self->{'imglist'}{$link}{'anchored'} ) {
+	$front = "<a name=\"gsdl_$a_name\">$front";
+	$back = "$back</a>";
+	$self->{'imglist'}{$link}{'anchored'} = 1;
+    }
+    return $self->SUPER::replace_images($front, $link, $back, $base_dir, 
+				     $file, $doc_obj, $section);
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/plugins/WordPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/WordPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/WordPlug.pm	(revision 21112)
@@ -0,0 +1,240 @@
+###########################################################################
+#
+# WordPlug.pm -- plugin for importing Microsoft Word documents
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# 12/05/02 Added usage datastructure - John Thompson
+
+package WordPlug;
+
+use ConvertToPlug;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @WordPlug::ISA = ('ConvertToPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "description_tags",
+	'desc' => "{HTMLPlug.description_tags}",
+	'type' => "flag" }
+      ];
+
+my $options = { 'name'     => "WordPlug",
+		'desc'     => "{WordPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
+	my $ws_arg = [ { 'name' => "windows_scripting",
+			 'desc' => "{WordPlug.windows_scripting}",
+			 'type' => "flag",
+		         'reqd' => "no" },
+		       { 'name' => "delete_toc",
+			 'desc' => "{WordPlug.delete_toc}",
+			 'type' => "flag",
+			 'reqd' => "no"},
+		       { 'name' => "level1_header",
+			 'desc' => "{WordPlug.level1_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "level2_header",
+			 'desc' => "{WordPlug.level2_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "level3_header",
+			 'desc' => "{WordPlug.level3_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "title_header",
+			 'desc' => "{WordPlug.title_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "toc_header",
+			 'desc' => "{WordPlug.toc_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "tof_header",
+			 'desc' => "{WordPlug.tof_header}",
+			 'type' => "regexp",
+			 'reqd' => "no",
+			 'deft' => "" },
+		       { 'name' => "metadata_fields",
+			 'type' => "string",
+			 'deft' => "Title"}];
+	
+	push(@$arguments,@$ws_arg);
+    }
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new ConvertToPlug($pluginlist,$inputargs,$hashArgOptLists): new ConvertToPlug($pluginlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    #this is passed through to gsConvert.pl by ConvertToPlug.pm
+    $self->{'convert_options'} = "-windows_scripting" if $self->{'windows_scripting'};
+
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+    if (defined $self->{'windows_scripting'}) { 
+	if (!defined $secondary_plugin_options->{'StructuredHTMLPlug'}){
+	    $secondary_plugin_options->{'StructuredHTMLPlug'} = [];
+	    my $structhtml_options = $secondary_plugin_options->{'StructuredHTMLPlug'};
+	    if ($self->{'input_encoding'} eq "auto") {
+		$self->{'input_encoding'} = "utf8";
+		$self->{'extract_language'} = 1;
+		push(@$structhtml_options,"-input_encoding", "utf8");
+		push(@$structhtml_options,"-extract_language");
+	    }
+				
+	    # Instruct HTMLPlug (when eventually accessed through read_into_doc_obj)
+	    # to extract these metadata fields from the HEAD META fields
+	    push (@$structhtml_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+	    push (@$structhtml_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+	    push (@$structhtml_options, "-description_tags") if $self->{'windows_scripting'};
+	    push (@$structhtml_options, "-delete_toc") if $self->{'delete_toc'};
+	    push (@$structhtml_options, "-toc_header", $self->{'toc_header'}) if $self->{'toc_header'};
+	    push (@$structhtml_options, "-tof_header", $self->{'tof_header'}) if $self->{'tof_header'};
+	    push (@$structhtml_options, "-title_header", $self->{'title_header'}) if $self->{'title_header'};
+	    push (@$structhtml_options, "-level1_header", $self->{'level1_header'}) if $self->{'level1_header'};
+	    push (@$structhtml_options, "-level2_header", $self->{'level2_header'})if $self->{'level2_header'};
+	    push (@$structhtml_options, "-level3_header", $self->{'level3_header'}) if $self->{'level3_header'};
+	    push (@$structhtml_options, "-metadata_fields", $self->{'metadata_fields'}) if $self->{'metadata_fields'};
+	}
+    }
+    if (!defined $secondary_plugin_options->{'HTMLPlug'}) {
+	$secondary_plugin_options->{'HTMLPlug'} = [];
+    }
+    if (!defined $secondary_plugin_options->{'TEXTPlug'}) {
+	$secondary_plugin_options->{'TEXTPlug'} = [];
+    }
+
+    my $html_options = $secondary_plugin_options->{'HTMLPlug'};
+    my $text_options = $secondary_plugin_options->{'TextPlug'};
+    
+    # wvWare will always produce html files encoded as utf-8, so make sure the secondary HTMLPlug knows this
+    push(@$html_options,"-input_encoding", "utf8");
+    if ($self->{'input_encoding'} eq "auto") {
+	$self->{'input_encoding'} = "utf8";
+	$self->{'extract_language'} = 1;
+	push(@$html_options,"-extract_language");
+    }
+    if ($self->{'description_tags'} == 1) {
+	push(@$html_options, "-description_tags");
+    }
+    # Instruct HTMLPlug (when eventually accessed through read_into_doc_obj)
+    # to extract these metadata fields from the HEAD META fields
+    push(@$html_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+    push(@$html_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+       
+    $self = bless $self, $class;
+    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
+
+    return bless $self;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.(doc|dot)$^;
+}
+
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+
+    my $outhandle=$self->{'outhandle'};
+     
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($conv_filename, $encoding, $language, \$text);
+
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+    #unicode::ensure_utf8(\$text);
+    
+    # Write it out again!
+    #$self->utf8_write_file (\$text, $conv_filename);
+}
+
+sub get_file_type {
+    my $self = shift (@_);
+    my $file_type = "Word";
+    return $file_type;
+}
+
+# Modified to cache HTML files for efficieny reasons rather
+# than delete all.  HTML is modified not to use IE's VML.
+# VML uses WML files, so these can be deleted.
+sub cleanup_tmp_area {
+    my ($self) = @_;
+    if (defined $self->{'files_dir'}) {
+	my $html_files_dir = $self->{'files_dir'};
+
+	if (opendir(DIN,$html_files_dir)) {
+	    my @wmz_files = grep( /\.wmz$/, readdir(DIN));
+	    foreach my $f (@wmz_files) {
+		my $full_f = &util::filename_cat($html_files_dir,$f);
+		&util::rm($full_f);
+	    }
+	    closedir(DIN);
+	}
+	else {
+	    # if HTML file has no supporting images, then no _files dir made
+	    # => do nothing
+	}
+    }
+}
+
+# do plugin specific processing of doc_obj for HTML type
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    return $self->process_type("doc", $base_dir, $file, $doc_obj);
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/plugins/XMLPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/XMLPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/XMLPlug.pm	(revision 21112)
@@ -0,0 +1,360 @@
+###########################################################################
+#
+# XMLPlug.pm -- base class for XML plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package XMLPlug;
+
+use BasPlug;
+use doc;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+sub BEGIN {
+    @XMLPlug::ISA = ('BasPlug');
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use XMLParser;
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "xslt",
+	'desc' => "{XMLPlug.xslt}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "XMLPlug",
+		'desc'     => "{XMLPlug.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+our ($self);
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    # $self is global for use within subroutines called by XML::Parser
+    $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    my $parser = new XML::Parser('Style' => 'Stream',
+				 'Handlers' => {'Char' => \&Char,
+						'XMLDecl' => \&XMLDecl,
+						'Entity' => \&Entity,
+						'Doctype' => \&Doctype,
+						'Default' => \&Default
+						});  
+    $self->{'parser'} = $parser;
+
+    return bless $self, $class;
+}
+
+sub apply_xslt
+{
+    my $self = shift @_;
+    my ($xslt,$filename) = @_;
+    
+    my $outhandle = $self->{'outhandle'};
+
+    my $xslt_filename = $xslt;
+
+    if (! -e $xslt_filename) {
+	# Look in main site directory
+	my $gsdlhome = $ENV{'GSDLHOME'};
+	$xslt_filename = &util::filename_cat($gsdlhome,$xslt);
+    }
+
+    if (! -e $xslt_filename) {
+	# Look in collection directory
+	my $coldir = $ENV{'COLLECTDIR'};
+	$xslt_filename = &util::filename_cat($coldir,$xslt);
+    }
+
+    if (! -e $xslt_filename) {
+	print $outhandle "Warning: Unable to find XSLT $xslt\n";
+	if (open(XMLIN,"<$filename")) {
+
+	    my $untransformed_xml = "";
+	    while (defined (my $line = <XMLIN>)) {
+
+		$untransformed_xml .= $line;
+	    }
+	    close(XMLIN);
+	    
+	    return $untransformed_xml;
+	}
+	else {
+	    print $outhandle "Error: Unable to open file $filename\n";
+	    print $outhandle "       $!\n";
+	    return "";
+	}
+	
+    }
+
+    my $bin_java = &util::filename_cat($ENV{'GSDLHOME'},"bin","java");
+    my $jar_filename = &util::filename_cat($bin_java,"xalan.jar");
+    my $xslt_base_cmd = "java -jar $jar_filename";
+    my $xslt_cmd = "$xslt_base_cmd -IN \"$filename\" -XSL \"$xslt_filename\"";
+
+    my $transformed_xml = "";
+
+    if (open(XSLT_IN,"$xslt_cmd |")) {
+	while (defined (my $line = <XSLT_IN>)) {
+
+	    $transformed_xml .= $line;
+	}
+	close(XSLT_IN);
+    }
+    else {
+	print $outhandle "Error: Unable to run command $xslt_cmd\n";
+	print $outhandle "       $!\n";
+    }
+
+    return $transformed_xml;
+
+}
+
+
+sub read {
+    # $self must be global to work with XML callback routines.
+    $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return 0;
+    }
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+    $self->{'file'} = $file;
+    $self->{'filename'} = $filename;
+    $self->{'processor'} = $processor;
+    $self->{'metadata'} = $metadata;
+    $self->{'gli'} = $gli;
+
+    eval {
+	my $xslt = $self->{'xslt'};
+	if (defined $xslt && ($xslt ne "")) {
+	    # perform xslt
+	    my $transformed_xml = $self->apply_xslt($xslt,$filename);
+
+	    # feed transformed file (now in memory as string) into XML parser
+	    $self->{'parser'}->parse($transformed_xml);
+	}
+	else {
+	    $self->{'parser'}->parsefile($filename);
+	}
+    };
+  
+    if ($@) {
+
+	# parsefile may either croak somewhere in XML::Parser (e.g. because
+	# the document is not well formed) or die somewhere in XMLPlug or a
+	# derived plugin (e.g. because we're attempting to process a
+	# document whose DOCTYPE is not meant for this plugin). For the
+	# first case we'll print a warning and continue, for the second
+	# we'll just continue quietly
+
+	print STDERR "**** Error is: $@\n";
+
+	my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
+	if (defined $msg) {	
+	    my $outhandle = $self->{'outhandle'};
+	    my $plugin_name = ref ($self);
+	    print $outhandle "$plugin_name failed to process $file ($msg)\n";
+	}
+
+	# reset ourself for the next document
+	$self->{'section_level'}=0;
+	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+	return -1; # error during processing
+    }
+
+    return 1; # processed the file
+}
+
+# the following two methods are for if you want to do the parsing from a
+# plugin that inherits from this. it seems that you can't call the parse 
+# methods directly. WHY???
+sub parse_file {
+    $self = shift (@_); 
+    my ($filename) = @_;
+    $self->{'parser'}->parsefile($filename);
+}
+
+sub parse_string {
+    $self = shift (@_); 
+    my ($xml_string) = @_;
+    $self->{'parser'}->parse($xml_string);
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.xml$^;
+}
+
+sub StartDocument {$self->xml_start_document(@_);}
+sub XMLDecl {$self->xml_xmldecl(@_);}
+sub Entity {$self->xml_entity(@_);}
+sub Doctype {$self->xml_doctype(@_);}
+sub StartTag {$self->xml_start_tag(@_);}
+sub EndTag {$self->xml_end_tag(@_);}
+sub Text {$self->xml_text(@_);}
+sub PI {$self->xml_pi(@_);}
+sub EndDocument {$self->xml_end_document(@_);}
+sub Default {$self->xml_default(@_);}
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {
+    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
+    $_[0]->{'Text'} .= $_[1];
+    return undef;
+}
+
+# Called at the beginning of the XML document.
+sub xml_start_document {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    $self->open_document();
+}
+
+# Called for XML declarations
+sub xml_xmldecl {
+    my $self = shift(@_);
+    my ($expat, $version, $encoding, $standalone) = @_;
+}
+
+# Called for XML entities
+sub xml_entity {
+  my $self = shift(@_);
+  my ($expat, $name, $val, $sysid, $pubid, $ndata) = @_;
+}
+
+# Called for DOCTYPE declarations - use die to bail out if this doctype
+# is not meant for this plugin
+sub xml_doctype {
+    my $self = shift(@_);
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+    die "XMLPlug Cannot process XML document with DOCTYPE of $name";
+}
+
+# Called for every start tag. The $_ variable will contain a copy of the
+# tag and the %_ variable will contain the element's attributes.
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+}
+
+# Called for every end tag. The $_ variable will contain a copy of the tag.
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+}
+
+# Called just before start or end tags with accumulated non-markup text in
+# the $_ variable.
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_;
+}
+
+# Called for processing instructions. The $_ variable will contain a copy
+# of the pi.
+sub xml_pi {
+    my $self = shift(@_);
+    my ($expat, $target, $data) = @_;
+}
+
+# Called at the end of the XML document.
+sub xml_end_document {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    $self->close_document();
+}
+
+# Called for any characters not handled by the above functions.
+sub xml_default {
+    my $self = shift(@_);
+    my ($expat, $text) = @_;
+}
+
+sub open_document {
+    my $self = shift(@_);
+
+    # create a new document
+    $self->{'doc_obj'} = new doc ($self->{'filename'}, "indexed_doc");
+    $self->{'doc_obj'}->set_OIDtype ($self->{'processor'}->{'OIDtype'});
+}
+
+sub close_document {
+    my $self = shift(@_);
+    my $doc_obj = $self->{'doc_obj'};
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, 
+			   $doc_obj->get_top_section(), 
+			   $self->{'metadata'});
+   
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+   
+    # add an OID
+    $doc_obj->set_OID();
+    
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "XML");
+
+    # process the document
+    $self->{'processor'}->process($doc_obj);
+    
+    $self->{'num_processed'} ++;
+}
+
+1;
+
+
+
+
Index: /main/tags/3.01/gsdl/perllib/plugins/ZIPPlug.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/plugins/ZIPPlug.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/plugins/ZIPPlug.pm	(revision 21112)
@@ -0,0 +1,208 @@
+###########################################################################
+#
+# ZIPPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which handles compressed and/or archived input formats
+#
+# currently handled formats and file extensions are:
+# 
+# gzip (.gz, .z, .tgz, .taz)
+# bzip (.bz)
+# bzip2 (.bz2)
+# zip (.zip .jar)
+# tar (.tar)
+#
+# this plugin relies on the following utilities being present 
+# (if trying to process the corresponding formats)
+#
+# gunzip (for gzip)
+# bunzip (for bzip)
+# bunzip2 
+# unzip (for zip)
+# tar (for tar) 
+
+
+package ZIPPlug;
+
+use BasPlug;
+use plugin;
+use util;
+use Cwd;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+BEGIN {
+    @ZIPPlug::ISA = ('BasPlug');
+}
+
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BasPlug.process_exp}",
+	'type' => "string",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" } ];
+
+my $options = { 'name'     => "ZIPPlug",
+		'desc'     => "{ZIPPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+sub new {
+
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
+
+    # BasPlug is explicitly set not to set process_exp if recursive plugin
+    # Not sure of this reasoning.  Want it to be set in ZIPPlug, so explicitly
+    # pass it in as default value
+    
+    ## do we need this???? --kjdon
+    if (!$self->{'process_exp'}) {
+	$self->{'process_exp'} = get_default_process_exp();
+    }
+
+    return bless $self, $class;
+}
+
+# this is a recursive plugin
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+sub get_default_process_exp {
+    return q^(?i)\.(gz|tgz|z|taz|bz|bz2|zip|jar|tar)$^;
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    if ($file =~ /$self->{'process_exp'}/) {
+
+	my $filename = $file;
+	$filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+	if (!-e $filename) {
+	    print $outhandle "ZIPPLug: WARNING: $filename does not exist\n";
+	    return undef;
+	}
+	
+	my ($file_only) = $file =~ /([^\\\/]*)$/;
+	my $tmpdir = &util::get_tmp_filename ();
+	&util::mk_all_dir ($tmpdir);
+
+	print $outhandle "ZIPPlug: extracting $file_only to $tmpdir\n"
+	    if $self->{'verbosity'} > 1;
+
+	# save current working directory
+	my $cwd = cwd();
+	chdir ($tmpdir) || die "Unable to change to $tmpdir";
+	&util::cp ($filename, $tmpdir);
+	
+	if ($file =~ /\.bz$/i) {
+	    $self->bunzip ($file_only);
+	} elsif ($file =~ /\.bz2$/i) {
+	    $self->bunzip2 ($file_only);
+	} elsif ($file =~ /\.(zip|jar)$/i) {
+	    $self->unzip ($file_only);
+	} elsif ($file =~ /\.tar$/i) {
+	    $self->untar ($file_only);
+	} else {
+	    $self->gunzip ($file_only);
+	}
+
+	chdir ($cwd) || die "Unable to change back to $cwd";
+
+	my $numdocs = &plugin::read ($pluginfo, "", $tmpdir, $metadata, $processor, $maxdocs, $total_count, $gli);
+	&util::rm_r ($tmpdir);
+
+	$self->{'num_archives'} ++;
+
+	return $numdocs;
+
+    } else {
+	return undef;
+    }
+}
+
+sub bunzip {
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    if (system ("bunzip $file")!=0)
+    {
+	&util::rm ($file);
+    }
+}
+
+sub bunzip2 {
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    if (system ("bunzip2 $file")!=0)
+    {
+	&util::rm ($file);
+    }
+}
+
+sub unzip {
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    system ("unzip $file");
+    &util::rm ($file) if -e $file;
+}
+
+sub untar {
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    system ("tar xf $file");
+    &util::rm ($file) if -e $file;
+}
+
+sub gunzip {
+    my $self = shift (@_);
+    my ($file) = @_;
+
+    if (system ("gunzip $file")!=0)
+    {
+	&util::rm ($file);
+    };
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/printusage.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/printusage.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/printusage.pm	(revision 21112)
@@ -0,0 +1,333 @@
+###########################################################################
+#
+# printusage.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package PrintUsage;
+
+
+use gsprintf;
+use strict;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+
+sub gsprintf
+{
+    return &gsprintf::gsprintf(@_);
+}
+
+
+sub print_xml_usage
+{
+    my $options = shift(@_);
+
+    # XML output is always in UTF-8
+    &gsprintf::output_strings_in_UTF8;
+
+    &print_xml_header();
+
+    &gsprintf(STDERR, "<Info>\n");
+    &gsprintf(STDERR, "  <Name>$options->{'name'}</Name>\n");
+    &gsprintf(STDERR, "  <Desc>$options->{'desc'}</Desc>\n");
+    &gsprintf(STDERR, "  <Arguments>\n");
+    if (defined($options->{'args'})) {
+	&print_options_xml($options->{'args'});
+    }
+    &gsprintf(STDERR, "  </Arguments>\n");
+    &gsprintf(STDERR, "</Info>\n");
+}
+
+
+sub print_xml_header
+{
+    &gsprintf(STDERR, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+    &gsprintf(STDERR, "<!DOCTYPE Info [\n");
+    &gsprintf(STDERR, "  <!ELEMENT Info      (Name, Desc, Arguments)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Arguments (Option*)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Option    (Name, Desc, Type, Required, Range, Default?, List?)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Name      (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Desc      (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Type      (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Required  (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Range     (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Default   (#PCDATA)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT List      (Value*)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT Value     (Name, Desc?)>\n");
+    &gsprintf(STDERR, "  <!ELEMENT HiddenGLI (#PCDATA)>\n");
+    &gsprintf(STDERR, "]>\n\n");
+}
+
+
+sub print_options_xml
+{
+    my $options = shift(@_);
+
+    foreach my $option (@$options) {
+	my $optionname = $option->{'name'};
+	my $optiondesc = &gsprintf::lookup_string($option->{'desc'});
+
+	# Escape '<' and '>' characters
+	$optiondesc =~ s/</&amp;lt;/g; # doubly escaped
+	$optiondesc =~ s/>/&amp;gt;/g;
+
+	# Display option name, description and type
+	&gsprintf(STDERR, "    <Option>\n");
+	&gsprintf(STDERR, "      <Name>$optionname</Name>\n");
+	&gsprintf(STDERR, "      <Desc>$optiondesc</Desc>\n");
+	&gsprintf(STDERR, "      <Type>$option->{'type'}</Type>\n");
+
+	# If the option has a required field, display this
+	if (defined($option->{'reqd'})) {
+	    &gsprintf(STDERR, "      <Required>$option->{'reqd'}</Required>\n");
+	}
+
+	# If the option has a charactor length field, display this
+	if (defined($option->{'char_length'})) {
+	    &gsprintf(STDERR, "      <CharactorLength>$option->{'char_length'}</CharactorLength>\n");
+	}
+
+	# If the option has a range field, display this
+	if (defined($option->{'range'})) {
+	    &gsprintf(STDERR, "      <Range>$option->{'range'}</Range>\n");
+	}
+
+	# If the option has a list of possible values, display these
+	if (defined $option->{'list'}) {
+	    &gsprintf(STDERR, "      <List>\n");
+	    my $optionvalueslist = $option->{'list'};
+	    foreach my $optionvalue (@$optionvalueslist) {
+		&gsprintf(STDERR, "        <Value>\n");
+		&gsprintf(STDERR, "          <Name>$optionvalue->{'name'}</Name>\n");
+		if (defined $optionvalue->{'desc'}) {
+		    my $optionvaluedesc = &gsprintf::lookup_string($optionvalue->{'desc'});
+
+		    # Escape '<' and '>' characters
+		    $optionvaluedesc =~ s/</&amp;lt;/g; #doubly escaped
+		    $optionvaluedesc =~ s/>/&amp;gt;/g;
+
+		    &gsprintf(STDERR, "          <Desc>$optionvaluedesc</Desc>\n");
+		}
+		&gsprintf(STDERR, "        </Value>\n");
+	    }
+
+#	    # Special case for 'input_encoding'
+#	    if ($optionname =~ m/^input_encoding$/i) {
+#		my $e = $encodings::encodings;
+#		foreach my $enc (sort {$e->{$a}->{'name'} cmp $e->{$b}->{'name'}} keys (%$e)) {
+#		    &gsprintf(STDERR, "        <Value>\n");
+#		    &gsprintf(STDERR, "          <Name>$enc</Name>\n");
+#		    &gsprintf(STDERR, "          <Desc>$e->{$enc}->{'name'}</Desc>\n");
+#		    &gsprintf(STDERR, "        </Value>\n");
+#		}
+#	    }
+
+	    &gsprintf(STDERR, "      </List>\n");
+	}
+
+	# Show the default value for the option, if there is one
+	if (defined $option->{'deft'}) {
+	    my $optiondeftvalue = $option->{'deft'};
+
+	    # Escape '<' and '>' characters
+	    $optiondeftvalue =~ s/</&amp;lt;/g; #doubly escaped
+	    $optiondeftvalue =~ s/>/&amp;gt;/g;
+
+	    &gsprintf(STDERR, "      <Default>$optiondeftvalue</Default>\n");
+	}
+
+	# If the option is noted as being hidden in GLI, add that to the printout
+	if (defined($option->{'hiddengli'})) {
+	    &gsprintf(STDERR, "      <HiddenGLI>$option->{'hiddengli'}</HiddenGLI>\n");
+	}
+	# If the argument is not hidden then print out the lowest detail mode it is visible in
+	if (defined($option->{'modegli'})) {
+	    &gsprintf(STDERR, "      <ModeGLI>$option->{'modegli'}</ModeGLI>\n");
+	}
+
+	&gsprintf(STDERR, "    </Option>\n");
+    }
+}
+
+
+sub print_txt_usage
+{
+    my $options = shift(@_);
+    my $params = shift(@_);
+    my $programdesc = shift(@_);
+
+    my $programname = $options->{'name'};
+    my $programargs = $options->{'args'};
+
+    # Find the length of the longest option string
+    my $descoffset = 0;
+    if (defined($programargs)) {
+	$descoffset = &find_longest_option_string($programargs);
+    }
+
+    # Produce the usage information using the data structure above
+    if ($programdesc) {
+	&gsprintf(STDERR, $programname . ": $options->{'desc'}\n\n");
+    }
+    &gsprintf(STDERR, " {common.usage}: $programname $params\n\n");
+
+    # Display the program options, if there are some
+    if (defined($programargs)) {
+	# Calculate the column offset of the option descriptions
+	my $optiondescoffset = $descoffset + 2;  # 2 spaces between options & descriptions
+
+	&gsprintf(STDERR, " {common.options}:\n");
+
+	# Display the program options
+	&print_options_txt($programargs, $optiondescoffset);
+    }
+}
+
+
+sub print_options_txt
+{
+    my $options = shift(@_);
+    my $optiondescoffset = shift(@_);
+
+    foreach my $option (@$options) {
+	# Display option name
+	my $optionname = $option->{'name'};
+	&gsprintf(STDERR, "  -$optionname");
+	my $optionstringlength = length("  -$optionname");
+
+	# Display option type, if the option is not a flag
+	my $optiontype = $option->{'type'};
+	if ($optiontype ne "flag") {
+	    &gsprintf(STDERR, " <$optiontype>");
+	    $optionstringlength = $optionstringlength + length(" <$optiontype>");
+	}
+
+	# Display the option description
+	my $optiondesc = &gsprintf::lookup_string($option->{'desc'});
+	my $optionreqd = $option->{'reqd'};
+	if (defined($optionreqd) && $optionreqd eq "yes") {
+	    $optiondesc = "(" . &gsprintf::lookup_string("{PrintUsage.required}") . ") " . $optiondesc;
+	}
+	&display_text_in_column($optiondesc, $optiondescoffset, $optionstringlength, 80);
+
+	# Show the default value for the option, if there is one
+	my $optiondefault = $option->{'deft'};
+	if (defined($optiondefault)) {
+	    &gsprintf(STDERR, " " x $optiondescoffset);
+	    &gsprintf(STDERR, "{PrintUsage.default}: $optiondefault\n");
+	}
+
+	# If the option has a list of possible values, display these
+	my $optionvalueslist = $option->{'list'};
+	if (defined($optionvalueslist)) {
+	    &gsprintf(STDERR, "\n");
+	    foreach my $optionvalue (@$optionvalueslist) {
+		my $optionvaluename = $optionvalue->{'name'};
+		&gsprintf(STDERR, " " x $optiondescoffset);
+		&gsprintf(STDERR, "$optionvaluename:");
+
+		my $optionvaluedesc = &gsprintf::lookup_string($optionvalue->{'desc'});
+		&display_text_in_column($optionvaluedesc, $optiondescoffset + 2,
+					$optiondescoffset + length($optionvaluename), 80);
+	    }
+	}
+
+#	# Special case for 'input_encoding'
+#	if ($optionname =~ m/^input_encoding$/i) {
+#	    my $e = $encodings::encodings;
+#	    foreach my $enc (sort {$e->{$a}->{'name'} cmp $e->{$b}->{'name'}} keys (%$e)) {
+#		&gsprintf(STDERR, " " x $optiondescoffset);
+#		&gsprintf(STDERR, "$enc:");
+#
+#		my $encodingdesc = $e->{$enc}->{'name'};
+#		&display_text_in_column($encodingdesc, $optiondescoffset + 2,
+#					$optiondescoffset + length("$enc:"), 80);
+#	    }
+#	}
+
+	# Add a blank line to separate options
+	&gsprintf(STDERR, "\n");
+    }
+}
+
+
+sub display_text_in_column
+{
+    my ($text, $columnbeg, $firstlineoffset, $columnend) = @_;
+
+    # Spaces are put *before* words, so treat the column beginning as 1 smaller than it is
+    $columnbeg = $columnbeg - 1;
+
+    # Add some padding (if needed) for the first line
+    my $linelength = $columnbeg;
+    if ($firstlineoffset < $columnbeg) {
+	&gsprintf(STDERR, " " x ($columnbeg - $firstlineoffset));
+    }
+    else {
+	$linelength = $firstlineoffset;
+    }
+
+    # Break the text into words, and display one at a time
+    my @words = split(/ /, $text);
+
+    foreach my $word (@words) {
+	# If printing this word would exceed the column end, start a new line
+	if (($linelength + length($word)) >= $columnend) {
+	    &gsprintf(STDERR, "\n");
+	    &gsprintf(STDERR, " " x $columnbeg);
+	    $linelength = $columnbeg;
+	}
+
+	# Write the word
+	&gsprintf(STDERR, " $word");
+	$linelength = $linelength + length(" $word");
+    }
+
+    &gsprintf(STDERR, "\n");
+}
+
+
+sub find_longest_option_string
+{
+    my $options = shift(@_);
+
+    my $maxlength = 0;
+    foreach my $option (@$options) {
+	my $optionname = $option->{'name'};
+	my $optiontype = $option->{'type'};
+
+	my $optionlength = length("  -$optionname");
+	if ($optiontype ne "flag") {
+	    $optionlength = $optionlength + length(" <$optiontype>");
+	}
+
+	# Remember the longest
+	if ($optionlength > $maxlength) {
+	    $maxlength = $optionlength;
+	}
+    }
+    return $maxlength;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/remproc.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/remproc.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/remproc.pm	(revision 21112)
@@ -0,0 +1,59 @@
+###########################################################################
+#
+# remproc.pm -- document processor used to cache all documents in memory
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package remproc;
+
+use classify;
+use doc;
+use docproc;
+
+BEGIN {
+    @remproc::ISA = ('docproc');
+}
+
+
+sub new {
+    my ($class) = @_;
+    my $self = {'docs'=>[]};
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj, $filename) = @_;
+
+    push (@{$self->{'docs'}}, {'doc_obj'=>$doc_obj,
+			       'filename'=>$filename});
+}
+
+sub get_doc_list {
+    my $self = shift (@_);
+
+    return $self->{'docs'};
+}
+
+1;
+
Index: /main/tags/3.01/gsdl/perllib/scriptutil.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/scriptutil.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/scriptutil.pm	(revision 21112)
@@ -0,0 +1,62 @@
+###########################################################################
+#
+# scriptutil.pm -- various useful utilities for the scripts
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package scriptutil;
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+no strict 'refs'; # ...but allow filehandles to be variables and vice versa
+
+use gsprintf 'gsprintf';
+
+# returns $removeold, $keepold
+sub check_removeold_and_keepold {
+
+    my ($removeold, $keepold, $dir, $collectcfg) = @_;
+
+    if ($keepold && $removeold) {
+	gsprintf(STDERR, "{scripts.both_old_options}\n", $dir);
+	sleep(3); #just in case
+	return (1,0); 
+	
+    } 
+
+    if (!$keepold && !$removeold) {
+	# we only look at config file options if we dont have these on the command line
+	if (defined $collectcfg->{'removeold'} && $collectcfg->{'removeold'} =~ /^true$/i ) {
+	    $removeold = 1;
+	} elsif (defined $collectcfg->{'keepold'} && $collectcfg->{'keepold'} =~ /^true$/i) {
+	    $keepold = 1;
+	}
+    }
+    if (!$keepold && !$removeold) {
+	gsprintf(STDERR, "{scripts.no_old_options} \n", $dir);
+	sleep(3); #just in case
+	return (1,0);
+    }
+    
+    return ($removeold, $keepold);
+
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/sorttools.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/sorttools.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/sorttools.pm	(revision 21112)
@@ -0,0 +1,195 @@
+###########################################################################
+#
+# sorttools.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# various subroutines to format strings
+# for sorting
+
+package sorttools;
+
+# moved here from BasClas so import can share it
+sub format_metadata_for_sorting {
+    my ($metaname, $metavalue, $doc_obj) = @_;
+    if (!defined $metaname || $metaname !~ /\S/ || ! defined $metavalue || $metavalue !~ /\S/) {
+	return "";
+    }
+
+    if ($metaname eq "Language") {
+	$metavalue = $iso639::fromiso639{$metavalue};
+	return $metavalue;
+    } 
+    
+    my $lang;
+    if (defined $doc_obj) {
+	$lang = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), 'Language');
+    }
+    $lang = 'en' unless defined $lang;
+    
+    # is this metadata likely to be a name?
+    my $function_name="format_string_name_$lang";
+    if ($metaname =~ /^(?:\w+\.)?(?:Creator|Author|Editor)(?::.*)?$/
+	&& exists &$function_name) {
+	no strict 'refs';
+	&$function_name(\$metavalue);
+    } else {
+	$function_name="format_string_$lang";
+	if (exists &$function_name) {
+	    no strict 'refs';
+	    &$function_name(\$metavalue);
+	}
+    }
+    
+    return $metavalue;
+}
+
+### language-specific sorting functions (called by format_metadata_for_sorting)
+
+## format_string_$lang() converts to lowercase (where appropriate), and
+# removes punctuation, articles from the start of string, etc
+## format_string_name_$lang() converts to lowercase, puts the surname first,
+# removes punctuation, etc
+
+sub format_string_en {
+    my $stringref = shift;
+    $$stringref = lc($$stringref);
+    $$stringref =~ s/&[^\;]+\;//g; # html entities
+    $$stringref =~ s/^\s*(the|a|an)\b//; # articles
+    $$stringref =~ s/[^[:alnum:]]//g;
+    $$stringref =~ s/\s+/ /g;
+    $$stringref =~ s/^\s+//;
+    $$stringref =~ s/\s+$//;
+}
+
+sub format_string_name_en {
+    my ($stringref) = @_;
+
+    $$stringref =~ tr/A-Z/a-z/;
+    $$stringref =~ s/&\S+;//g;
+
+    my $comma_format = ($$stringref =~ m/^.+,.+$/);
+    $$stringref =~ s/[[:punct:]]//g;
+    $$stringref =~ s/\s+/ /g;
+    $$stringref =~ s/^\s+//;
+    $$stringref =~ s/\s+$//;
+
+    
+    if (!$comma_format) {
+	# No commas in name => name in 'firstname surname' format
+	# need to sort by surname
+	my @names = split / /, $$stringref;
+	my $surname = pop @names;
+	while (scalar @names && $surname =~ /^(jnr|snr)$/i) {
+	    $surname = pop @names;
+	}
+	$$stringref = $surname . " " . $$stringref;
+    }
+}
+
+
+sub format_string_fr {
+    my $stringref = shift;
+
+    $$stringref = lc($$stringref);
+    $$stringref =~ s/&[^\;]+\;//g; # html entities
+    $$stringref =~ s/[^[:alpha:]]//g;
+    $$stringref =~ s/^\s*(les?|la|une?)\b//; # articles
+    $$stringref =~ s/\s+/ /g;
+    $$stringref =~ s/^\s+//;
+    $$stringref =~ s/\s+$//;
+}
+
+sub format_string_es {
+    my $stringref = shift;
+
+    $$stringref = lc($$stringref);
+    $$stringref =~ s/&[^\;]+\;//g; # html entities
+    $$stringref =~ s/[^[:alpha:]]//g;
+    $$stringref =~ s/^\s*(la|el)\b//; # articles
+    $$stringref =~ s/\s+/ /g;
+    $$stringref =~ s/^\s+//;
+    $$stringref =~ s/\s+$//;
+}
+
+### end of language-specific functions
+
+# takes arguments of day, month, year and converts to
+# date of form yyyymmdd. month may be full (e.g. "January"),
+# abbreviated (e.g. "Jan"), or a number (1-12). Years like "86" 
+# will be assumed to be "1986".
+sub format_date {
+    my ($day, $month, $year) = @_;
+
+    my %months = ('january' => '01', 'jan' => '01', 'february' => '02', 'feb' => '02',
+		  'march' => '03', 'mar' => '03', 'april' => '04', 'apr' => '04',
+		  'may' => '05', 'june' => '06', 'jun' => '06', 'july' => '07', 
+		  'jul' => '07', 'august' => '08', 'aug' => '08', 'september' => '09', 
+		  'sep' => '09', 'october' => '10', 'oct' => '10', 'november' => '11', 
+		  'nov' => '11', 'december' => '12', 'dec' => '12');
+
+    $month =~ tr/A-Z/a-z/;
+    
+    if ($day < 1) { 
+	print STDERR "sorttools::format_date WARNING day $day out of range\n";
+	$day = "01";
+    } elsif ($day > 31) {
+	print STDERR "sorttools::format_date WARNING day $day out of range\n";
+	$day = "31";
+    }
+
+    $day = "0$day" if (length($day) == 1);
+
+    if ($month =~ /^\d\d?$/) {
+	if ($month < 1) {
+	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	    $month = "01";
+	} elsif ($month > 12) {
+	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	    $month = "12";
+	}
+	if ($month =~ /^\d$/) {
+	    $month = "0" . $month;
+	}
+    } elsif (!defined $months{$month}) {
+	print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	$month = "01";
+    } else {
+	$month = $months{$month};
+    }
+    
+    if ($year !~ /^\d\d\d\d$/) {
+	if ($year !~ /^\d\d$/) {
+	    my $newyear = 1900 + $year;
+	    print STDERR "sorttools::format_date WARNING year $year assumed to be $newyear\n";
+	    $year=$newyear;
+	} else {
+	    print STDERR "sorttools::format_date WARNING year $year out of range - reset to 1900\n";
+	    $year = "1900";
+	}
+    }
+
+    return "$year$month$day";
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/strings.rb
===================================================================
--- /main/tags/3.01/gsdl/perllib/strings.rb	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/strings.rb	(revision 21112)
@@ -0,0 +1,935 @@
+#
+# Resource bundle description
+#
+
+Language.code:en
+Language.name:English
+
+OutputEncoding.unix:iso_8859_1
+OutputEncoding.windows:iso_8859_1
+
+
+#
+# Common output messages
+#
+
+common.cannot_create_file:ERROR: Can't create file %s
+
+common.cannot_find_cfg_file:ERROR: Can't find the configuration file %s
+
+common.cannot_open:ERROR: Can't open %s
+
+common.cannot_open_fail_log:ERROR: Can't open fail log %s
+
+common.cannot_open_output_file:ERROR: Can't open output file %s
+
+common.cannot_read:ERROR: Can't read %s
+
+common.cannot_read_file:ERROR: Can't read file %s
+
+common.general_options:general options (for %s)
+
+common.must_be_implemented:function must be implemented in sub-class
+
+common.options:options
+
+common.processing:processing
+
+common.specific_options:specific options
+
+common.usage:Usage
+
+
+#
+# Script option descriptions and output messages
+#
+
+scripts.language:Language to display option descriptions in (eg. 'en_US' specifies American English). Requires translations of the option descriptions to exist in the perllib/strings_language-code.rb file.
+
+scripts.xml:Produces the information in an XML form, without 'pretty' comments but with much more detail.
+
+scripts.listall:Lists all items known about.
+
+scripts.both_old_options:WARNING: Both -removeold and -keepold were specified, defaulting to -removeold. Current contents of %s directory will be deleted.
+
+scripts.no_old_options:WARNING: Neither -removeold or -keepold were specified, defaulting to -removeold. Current contents of %s directory will be deleted.
+
+# -- buildcol.pl --
+
+buildcol.archivedir:Where the archives live.
+
+buildcol.builddir:Where to put the built indexes.
+
+buildcol.cachedir:Collection will be temporarily built here before being copied to the build directory.
+
+buildcol.cannot_open_cfg_file:WARNING: Can't open config file for updating: %s
+
+buildcol.collectdir:Collection directory.
+
+buildcol.copying_back_cached_build:Copying back the cached build
+
+buildcol.create_images:Attempt to create default images for new collection. This relies on the Gimp being installed along with relevant perl modules to allow scripting from perl.
+
+buildcol.debug:Print output to STDOUT.
+
+buildcol.desc:PERL script used to build a greenstone collection from GML documents.
+
+buildcol.faillog:Fail log filename. This log receives the filenames of any files which fail to be processed.
+
+buildcol.index:Index to build (will build all in config file if not set).
+
+buildcol.keepold:Will not destroy the current contents of the building directory.
+
+buildcol.maxdocs:Maximum number of documents to build.
+
+buildcol.mode:The parts of the building process to carry out.
+buildcol.mode.all:Do everything.
+buildcol.mode.build_index:Just index the text.
+buildcol.mode.compress_text:Just compress the text.
+buildcol.mode.infodb:Just build the metadata database.
+
+buildcol.no_default_images:Default images will not be generated.
+
+buildcol.no_image_script:WARNING: Image making script could not be found: %s
+
+buildcol.no_strip_html:Do not strip the html tags from the indexed text (only used for mgpp collections).
+
+buildcol.no_text:Don't store compressed text. This option is useful for minimizing the size of the built indexes if you intend always to display the original documents at run time (i.e. you won't be able to retrieve the compressed text version).
+
+buildcol.sections_index_document_metadata:Index document level metadata at section level
+buildcol.sections_index_document_metadata.never:Don't index any document metadata at section level.
+buildcol.sections_index_document_metadata.always:Add all specified document level metadata even if section level metadata of that name exists.
+buildcol.sections_index_document_metadata.unless_section_metadata_exists:Only add document level metadata if no section level metadata of that name exists.
+
+buildcol.out:Filename or handle to print output status to.
+
+buildcol.params:[options] collection-name
+
+buildcol.remove_empty_classifications:Hide empty classifiers and classification nodes (those that contain no documents).
+
+buildcol.removeold:Will remove the old contents of the building directory.
+
+buildcol.unlinked_col_images:Collection images may not be linked correctly.
+
+buildcol.unknown_mode:Unknown mode: %s
+
+buildcol.updating_archive_cache:Updating archive cache
+
+buildcol.verbosity:Controls the quantity of output. 0=none, 3=lots.
+
+
+# -- classinfo.pl --
+
+classinfo.collect:Giving a collection name will make classinfo.pl look in collect/collection-name/perllib/classify first. If the classifier is not found there it will look in the general perllib/classify directory.
+
+classinfo.desc:Prints information about a classifier.
+
+classinfo.general_options:General options are inherited from parent classes of the classifier.
+
+classinfo.info:info
+
+classinfo.no_classifier_name:ERROR: You must provide a classifier name.
+
+classinfo.option_types:Classifiers may take two types of options
+
+classinfo.params:[options] classifier-name
+
+classinfo.passing_options:Options may be passed to any classifier by including them in your collect.cfg configuration file.
+
+classinfo.specific_options:Specific options are defined within the classifier itself, and are available only to this particular classifier.
+
+# -- explode_metadata_database.pl --
+explode.desc:Explode a metadata database
+
+explode.document_field:The metadata element specifying the file name of documents to obtain and include in the collection.
+
+explode.document_prefix:A prefix for the document locations (for use with the document_field option).
+
+explode.document_suffix:A suffix for the document locations (for use with the document_field option).
+
+explode.encoding:Encoding to use when reading in the database file
+explode.filename_field:Field to use for filenames for the generated null files (if no document_field or that field is empty)
+explode.metadata_set:Metadata set (namespace) to export all metadata as
+explode.plugin: Plugin to use for exploding
+explode.params: [options] filename
+# -- exportcol.pl --
+
+exportcol.out:Filename or handle to print output status to.
+exportcol.cddir:The name of the directory that the CD contents are exported to.
+exportcol.cdname:The name of the CD-ROM -- this is what will appear in the start menu once the CD-ROM is installed.
+exportcol.desc:PERL script used to export one or more collections to a Windows CD-ROM. 
+exportcol.params:[options] collection-name1 collection-name2 ...
+exportcol.coll_not_found:Ignoring invalid collection %s: collection not found at %s.
+exportcol.coll_dirs_not_found:Ignoring invalid collection %s: one of the following directories not found:
+exportcol.fail:exportcol.pl failed:
+exportcol.no_valid_colls:No valid collections specified to export.
+exportcol.couldnt_create_dir:Could not create directory %s.
+exportcol.couldnt_create_file:Could not create %s.
+exportcol.instructions:To create a self-installing Windows CD-ROM, write the contents of this folder out to a CD-ROM.
+exportcol.non_exist_files:One or more of the following necessary files and directories does not exist:
+exportcol.success:exportcol.pl succeeded:
+exportcol.output_dir:The exported collections (%s) are in %s.
+exportcol.nonetscape:Do not bundle netscape with exported CD-ROM.
+exportcol.export_coll_not_installed:The Export to CD-ROM functionality has not been installed.
+
+# -- import.pl --
+
+import.archivedir:Where the converted material ends up.
+
+import.cannot_open_stats_file:WARNING: Couldn't open stats file %s.
+
+import.cannot_open_fail_log:ERROR: Couldn't open fail log %s
+
+import.cannot_sort:WARNING: import.pl cannot sort documents when groupsize > 1. sortmeta option will be ignored.
+
+import.collectdir:Collection directory.
+
+import.complete:Import complete
+
+import.debug:Print imported text to STDOUT.
+
+import.desc:PERL script used to import files into a GML format ready for building.
+
+import.faillog:Fail log filename. This log receives the filenames of any files which fail to be processed.
+
+import.groupsize:Number of import documents to group into one XML file.
+
+import.gzip:Use gzip to compress resulting xml documents (don't forget to include ZIPPlug in your plugin list when building from compressed documents).
+
+import.importdir:Where the original material lives.
+
+import.keepold:Will not destroy the current contents of the archives directory.
+
+import.maxdocs:Maximum number of documents to import.
+
+import.no_plugins_loaded:ERROR: No plugins loaded.
+
+import.OIDtype:The method to use when generating unique identifiers for each document.
+import.OIDtype.hash:Hashes the contents of the file. Document identifier will be the same every time the collection is imported.
+
+import.OIDtype.incremental:A simple document count that is significantly faster than "hash". It is not guaranteed to always assign the same identifier to a given document though and does not allow further documents to be added to existing xml archives.
+
+import.OIDtype.assigned:Uses 'D' plus the value of dc.Identifier as the document identifier. dc.Identifiers should be unique. If no dc.Identifier is assigned to the document, a hash id will be used instead.
+
+import.OIDtype.dirname:Uses 'J' plus the parent directory name as the identifier. This relies on there being only one document per directory, and all directory names being unique. E.g. import/b13as/h15ef/page.html will get an identifier of Jh15ef.
+
+import.saveas:This is to decide the archives format to be generated. The default setting is to GA.
+
+import.saveas.GA:Will generate Greenstone Archive format.
+
+import.saveas.METS:Will generate METS format.
+
+import.out:Filename or handle to print output status to.
+
+import.params:[options] collection-name
+
+import.removeold:Will remove the old contents of the archives directory.
+
+import.removing_archives:Removing current contents of the archives directory...
+
+import.removing_tmpdir:Removing contents of the collection "tmp" directory...
+
+import.sortmeta:Sort documents alphabetically by metadata for building. Search results for boolean queries will be displayed in this order. This will be disabled if groupsize > 1.
+
+import.statsfile:Filename or handle to print import statistics to.
+
+import.stats_backup:Will print stats to STDERR instead.
+
+import.verbosity:Controls the quantity of output. 0=none, 3=lots.
+
+
+# -- export.pl --
+
+export.exportdir:Where the export material ends up.
+
+export.cannot_open_stats_file:WARNING: Couldn't open stats file %s.
+
+export.cannot_open_fail_log:ERROR: Couldn't open fail log %s
+
+export.cannot_sort:WARNING: export.pl cannot sort documents when groupsize > 1. sortmeta option will be ignored.
+
+export.collectdir:Collection directory.
+
+export.complete:Export complete
+
+export.debug:Print exported text to STDOUT.
+
+export.desc:PERL script used to export files in a Greenstone collection to another format.
+
+export.faillog:Fail log filename. This log receives the filenames of any files which fail to be processed. (Default: collectdir/collect/collname/etc/fail.log)
+
+export.groupsize:Number of documents to group into one XML file.
+
+export.gzip:Use gzip to compress resulting xml documents (don't forget to include ZIPPlug in your plugin list when building from compressed documents).
+
+export.importdir:Where the original material lives.
+
+export.keepold:Will not destroy the current contents of the export directory. 
+
+export.maxdocs:Maximum number of documents to export.
+
+export.no_plugins_loaded:ERROR: No plugins loaded.
+
+export.OIDtype:The method to use when generating unique identifiers for each document.
+export.OIDtype.hash:Hashes the contents of the file. Document identifier will be the same every time the collection is imported.
+export.OIDtype.incremental:A simple document count that is significantly faster than "hash". It is not guaranteed to always assign the same identifier to a given document though and does not allow further documents to be added to existing xml archives.
+
+export.saveas:Format to export documents as.
+
+export.saveas.DSpace:DSpace Archive format.
+
+export.saveas.METS:METS format using the Greenstone profile.
+
+export.out:Filename or handle to print output status to.
+
+export.params:[options] collection-name1, collection-name2...
+
+export.removeold:Will remove the old contents of the export directory. 
+
+export.removing_export:Removing current contents of the export directory...
+
+export.sortmeta:Sort documents alphabetically by metadata for building. This will be disabled if groupsize > 1.
+
+export.statsfile:Filename or handle to print export statistics to.
+
+export.stats_backup:Will print stats to STDERR instead.
+
+export.verbosity:Controls the quantity of output. 0=none, 3=lots.
+
+
+# -- mkcol.pl --
+
+mkcol.about:The about text for the collection.
+
+mkcol.bad_name_cvs:ERROR: No collection can be named CVS as this may interfere with directories created by the CVS versioning system.
+
+mkcol.bad_name_modelcol:ERROR: No collection can be named modelcol as this is the name of the model collection.
+
+mkcol.cannot_find_modelcol:ERROR: Cannot find the model collection %s
+
+mkcol.col_already_exists:ERROR: This collection already exists.
+
+mkcol.collectdir:Directory where new collection will be created.
+
+mkcol.creating_col:Creating the collection %s
+
+mkcol.creator:The collection creator's e-mail address.
+
+mkcol.creator_undefined:ERROR: The creator was not defined. This variable is needed to recognise duplicate collection names.
+
+mkcol.desc:PERL script used to create the directory structure for a new Greenstone collection.
+
+mkcol.doing_replacements:doing replacements for %s
+
+mkcol.long_colname:ERROR: The collection name must be less than 8 characters so compatibility with earlier filesystems can be maintained.
+
+mkcol.maintainer:The collection maintainer's email address (if different from the creator).
+
+mkcol.no_collectdir:ERROR: The collect dir doesn't exist: %s
+
+mkcol.no_colname:ERROR: No collection name was specified.
+
+mkcol.optionfile:Get options from file, useful on systems where long command lines may cause problems.
+
+mkcol.params:[options] collection-name
+
+mkcol.plugin:Perl plugin module to use (there may be multiple plugin entries).
+
+mkcol.public:If this collection has anonymous access (true/false).
+
+mkcol.quiet:Operate quietly.
+
+mkcol.success:The new collection was created successfully at %s
+
+mkcol.title:The title of the collection.
+
+mkcol.win31compat:Whether or not the named collection directory must conform to Windows 3.1 file conventions or not (i.e. 8 characters long).
+
+# -- pluginfo.pl --
+
+pluginfo.collect:Giving a collection name will make pluginfo.pl look in collect/collection-name/perllib/plugins first. If the plugin is not found there it will look in the general perllib/plugins directory.
+
+pluginfo.desc:Prints information about a plugin.
+
+pluginfo.general_options:General options are inherited from parent classes of the plugin.
+
+pluginfo.info:info
+
+pluginfo.no_plugin_name:ERROR: You must provide a plugin name.
+
+pluginfo.option_types:Plugins may take two types of options
+
+pluginfo.params:[options] plugin-name
+
+pluginfo.passing_options:Options may be passed to any plugin by including them in your collect.cfg configuration file.
+
+pluginfo.specific_options:Specific options are defined within the plugin itself, and are available only to this particular plugin.
+
+
+#
+# Classifier option descriptions
+#
+
+AZCompactList.allvalues:Use all metadata values found.
+
+AZCompactList.desc:Classifier plugin for sorting alphabetically
+
+AZCompactList.doclevel:Level to process document at.
+AZCompactList.doclevel.top:Whole document.
+AZCompactList.doclevel.section:By sections.
+
+AZCompactList.firstvalueonly:Use only the first metadata value found.
+
+AZCompactList.freqsort:Sort by node frequency rather than alpha-numeric.
+
+AZCompactList.maxcompact:Maximum number of documents to be displayed per page.
+
+AZCompactList.metadata:A single Metadata field, or a comma separated list of Metadata fields, used for classification. If a list is specified, the first metadata type that has values will be used. May be used in conjunction with the -firstvalueonly and -allvalues flags, to select only the first value, or all metadata values from the list.
+
+AZCompactList.mincompact:Minimum number of documents to be displayed per page.
+
+AZCompactList.mingroup:The smallest value that will cause a group in the hierarchy to form.
+
+AZCompactList.minnesting:The smallest value that will cause a list to be converted into a nested list.
+
+AZCompactList.recopt:Used in nested metadata such as -metadata Year/Organisation.
+AZCompactList.sort:Metadata field to sort the leaf nodes by. 
+
+AZCompactSectionList.desc:Variation on AZCompactList that classifies sections rather than documents. Entries are sorted by section-level metadata.
+
+AZList.desc:Classifier plugin for sorting alphabetically
+
+AZList.metadata:A single Metadata field or a comma separated list of Metadata fields used for classification. Following the order indicated by the list, the first field that contains a Metadata value will be used. List will be sorted by this element.
+
+AZSectionList.desc:Classifier plugin for sorting alphabetically. This is very similar to AZList except it sorts by section level metadata (excluding the top level) instead of just top level metadata. The only change is to the classify() subroutine which must now iterate through each section, adding each to the classification.
+
+BasClas.bad_general_option:The %s classifier uses an incorrect general option (general options are those available to all classifiers). Check your collect.cfg configuration file.
+
+BasClas.builddir:Where to put the built indexes.
+
+BasClas.buttonname:The label for the classifier screen and button in navigation bar. The default is the metadata element specified with -metadata.
+
+BasClas.desc:Base class for all the classifiers.
+
+BasClas.no_metadata_formatting:Don't do any automatic metadata formatting (for sorting.)
+
+BasClas.outhandle:The file handle to write output to.
+
+BasClas.removeprefix:A prefix to ignore in metadata values when sorting.
+
+BasClas.removesuffix:A suffix to ignore in metadata values when sorting.
+
+BasClas.verbosity:Controls the quantity of output. 0=none, 3=lots.
+
+Browse.desc:.
+
+DateList.bymonth:Classify by year and month instead of only year.
+
+DateList.desc:Classifier plugin for sorting by date. By default, sorts by 'Date' metadata. Date is assumed to be in the form yyyymmdd.
+
+DateList.metadata:The metadata that contains the dates to classify by. The format is expected to be yyyymmdd.
+
+DateList.reverse_sort:Sort the documents in reverse chronological order (newest first).
+
+DateList.nogroup:Make each year an individual entry in the horizontal list, instead of spanning years with few entries. (This can also be used with the -bymonth option to make each month a separate entry instead of merging).
+
+DateList.sort:An extra metadata field to sort by in the case where two documents have the same date.
+
+GenericList.always_bookshelf_last_level:Create a bookshelf icon even if there is only one item in each group at the leaf nodes.
+GenericList.classify_sections:Classify sections instead of documents.
+GenericList.desc:A general and flexible list classifier with most of the abilities of AZCompactList, but with better Unicode, metadata and sorting capabilities.
+GenericList.metadata:Metadata fields used for classification. Use '/' to separate the levels in the hierarchy and ';' to separate metadata fields within each level.
+GenericList.partition_size_within_level:The number of items in each partition (only applies when partition_type_within_level is set to 'constant_size').
+GenericList.partition_type_within_level:The type of partitioning done: either 'per_letter', 'constant_size', or 'none'.
+GenericList.sort_leaf_nodes_using:Metadata fields used for sorting the leaf nodes. Use '|' to separate the metadata groups to stable sort and ';' to separate metadata fields within each group.
+GenericList.use_hlist_for:Metadata fields to use a hlist rather than a vlist. Use ',' to separate the metadata groups and ';' to separate the metadata fields within each group.
+
+HFileHierarchy.desc:Classifier plugin for generating hierarchical classifications based on a supplementary structure file.
+
+Hierarchy.desc:Classifier plugin for generating a hierarchical classification. This may be based on structured metadata, or may use a supplementary structure file (use the -hfile option).
+
+Hierarchy.documents_last:Display document nodes after classifier nodes.
+
+Hierarchy.hfile:Use the specified classification structure file.
+
+Hierarchy.hlist_at_top:Display the first level of the classification horizontally.
+
+Hierarchy.reverse_sort:Sort leaf nodes in reverse order (use with -sort).
+
+Hierarchy.separator:Regular expression used for the separator, if using structured metadata.
+
+Hierarchy.sort:Metadata field to sort leaf nodes by. Leaves will not be sorted if not specified.
+
+Hierarchy.suppressfirstlevel:Ignore the first part of the metadata value. This is useful for metadata where the first element is common, such as the import directory in gsdlsourcefilename.
+
+Hierarchy.suppresslastlevel:Ignore the final part of the metadata value. This is useful for metadata where each value is unique, such as file paths.
+
+HTML.desc:Creates an empty classification that's simply a link to a web page.
+
+HTML.url:The url of the web page to link to.
+
+List.desc:Simple list classifier plugin. 
+
+List.metadata:A single Metadata field or a comma separated list of Metadata fields used for classification. Following the order indicated by the list, the first field that contains a Metadata value will be used. List will be sorted by this element, unless -sort is used. If no metadata is specified, then all documents will be included in the list, otherwise only documents that contain a metadata value will be included.
+
+List.sort:Metadata field to sort by. Use '-sort nosort' for no sorting. 
+
+Phind.desc:The Phind clasifier plugin.
+
+Phind.language:Language or languages to use building hierarchy. Languages are identified by two-letter country codes like en (English), es (Spanish), and fr (French). Language is a regular expression, so 'en|fr' (English or French) and '..' (match any language) are valid.
+
+Phind.min_occurs:The minimum number of times a phrase must appear in the text to be included in the phrase hierarchy.
+
+Phind.savephrases:If set, the phrase infomation will be stored in the given file as text. It is probably a good idea to use an absolute path.
+
+Phind.suffixmode:The smode parameter to the phrase extraction program. A value of 0 means that stopwords are ignored, and of 1 means that stopwords are used.
+
+Phind.text:The text used to build the phrase hierarchy.
+
+Phind.thesaurus:Name of a thesaurus stored in Phind format in the collection's etc directory.
+
+Phind.title:The metadata field used to describe each document.
+
+Phind.untidy:Don't remove working files.
+
+SectionList.desc:Same as List classifier but includes all sections of document (excluding top level) rather than just top level document itself.
+
+Collage.desc:The Collage classifier plugin.
+
+Collage.geometry:The dimensions of the Collage canvas. For a canvas 600 pixels wide by 400 pixels high, for example, specify geometry as 600x400
+
+Collage.maxDepth:Images for collaging are drawn from mirroring the underlying browse classifier. This controls the maximum depth of the mirroring process.
+
+Collage.maxDisplay:The maximum number of images to show in the collage at any one time.
+
+Collage.imageType:Used to control, by expressing file name extensions, which file types are used in the collage.  A list of file name extensions is separated by the percent (%%) symbol.
+
+Collage.bgcolor:The background color of the collage canvas, specified in hexadecimal form (for example #008000 results in a forest green background).
+
+Collage.buttonname:The label for the classifier screen and button in navigation bar.
+
+Collage.refreshDelay:Rate, in milliseconds, that the collage canvas is refreshed.
+
+Collage.isJava2:Used to control which run-time classes of Java are used. More advanced version of Java (i.e. Java 1.2 onwards) include more sophisticated support for controlling transparency in images, this flag helps control what happens, however the built-in Java runtime for some browsers is version 1.1.  The applet is designed to, by default, auto-detect which version of Java the browser is running and act accordingly.
+
+Collage.imageMustNotHave:Used to suppress images that should not appear in the collage, such as image buttons that make up the navigation bar.
+
+Collage.caption:Optional captions to display below the collage canvas.
+
+#
+# Plugin option descriptions
+#
+
+ArcPlug.desc:Plugin which recurses through an archives.inf file (i.e. the file generated in the archives directory when an import is done), processing each file it finds.
+
+BasPlug.adding:adding
+
+BasPlug.already_seen:already seen
+
+BasPlug.bad_general_option:The %s plugin uses an incorrect general option (general options are those available to all plugins). Check your collect.cfg configuration file.
+
+BasPlug.block_exp:Files matching this regular expression will be blocked from being passed to any later plugins in the list. This has no real effect other than to prevent lots of warning messages about input files you don't care about. Each plugin might have a default block_exp. e.g. by default HTMLPlug blocks any files with .gif, .jpg, .jpeg, .png or .css file extensions.
+
+BasPlug.associate_ext:Causes files with the same root filename as the document being processed by the plugin AND a filename extension from the comma separated list provided by this argument to be associated with the document being processed rather than handled as a separate list.
+
+BasPlug.could_not_extract_encoding:WARNING: encoding could not be extracted from %s - defaulting to %s
+
+BasPlug.could_not_extract_language:WARNING: language could not be extracted from %s - defaulting to %s
+
+BasPlug.could_not_open_for_reading:could not open %s for reading
+
+BasPlug.no_cover_image:Do not look for a prefix.jpg file (where prefix is the same prefix as the file being processed) and associate it as a cover image.
+
+BasPlug.default_encoding:Use this encoding if -input_encoding is set to 'auto' and the text categorization algorithm fails to extract the encoding or extracts an encoding unsupported by Greenstone. This option can take the same values as -input_encoding.
+
+BasPlug.default_language:If Greenstone fails to work out what language a document is the 'Language' metadata element will be set to this value. The default is 'en' (ISO 639 language symbols are used: en = English). Note that if -input_encoding is not set to 'auto' and -extract_language is not set, all documents will have their 'Language' metadata set to this value.
+
+BasPlug.desc:Base class for all the import plugins.
+
+BasPlug.done_acronym_extract:done extracting acronyms.
+
+BasPlug.done_acronym_markup:done acronym markup.
+
+BasPlug.done_email_extract:done extracting e-mail addresses.
+
+BasPlug.dummy_text:This document has no text.
+
+BasPlug.empty_file:file contains no text
+
+BasPlug.extract_acronyms:Extract acronyms from within text and set as metadata.
+
+BasPlug.extract_email:Extract email addresses as metadata.
+
+BasPlug.extract_historical_years:Extract time-period information from historical documents.  This is stored as metadata with the document. There is a search interface for this metadata, which you can include in your collection by adding the statement, "format QueryInterface DateSearch" to your collection configuration file.
+
+BasPlug.extract_language:Identify the language of each document and set 'Language' metadata. Note that this will be done automatically if -input_encoding is 'auto'.
+
+BasPlug.extracting:extracting
+
+BasPlug.extracting_acronyms:extracting acronyms
+
+BasPlug.extract_keyphrases:Extract keyphrases automatically with Kea (default settings).
+
+BasPlug.extract_keyphrase_options:Options for keyphrase extraction with Kea. For example: mALIWEB - use ALIWEB extraction model; n5 - extract 5 keyphrase;, eGBK - use GBK encoding.
+
+BasPlug.extracting_emails:extracting e-mail addresses
+
+BasPlug.file_has_no_text:ERROR: %s contains no text
+
+BasPlug.first:Comma separated list of first sizes to extract from the text into a metadata field. The field is called 'FirstNNN'.
+
+BasPlug.input_encoding:The encoding of the source documents. Documents will be converted from these encodings and stored internally as utf8.
+BasPlug.input_encoding.ascii:Plain 7 bit ascii. This may be a bit faster than using iso_8859_1. Beware of using this on a collection of documents that may contain characters outside the plain 7 bit ascii set though (e.g. German or French documents containing accents), use iso_8859_1 instead.
+BasPlug.input_encoding.auto:Use text categorization algorithm to automatically identify the encoding of each source document. This will be slower than explicitly setting the encoding but will work where more than one encoding is used within the same collection.
+
+BasPlug.input_encoding.unicode:Just unicode.
+
+BasPlug.input_encoding.utf8:Either utf8 or unicode -- automatically detected.
+
+BasPlug.keyphrases:keyphrases
+
+BasPlug.marking_up_acronyms:marking up acronyms
+
+BasPlug.markup_acronyms:Add acronym metadata into document text.
+
+BasPlug.maximum_century:The maximum named century to be extracted as historical metadata (e.g. 14 will extract all references up to the 14th century).
+
+BasPlug.maximum_year:The maximum historical date to be used as metadata (in a Common Era date, such as 1950).
+
+BasPlug.must_be_implemented:BasPlug::read function must be implemented in sub-class for recursive plugins
+
+BasPlug.no_bibliography:Do not try to block bibliographic dates when extracting historical dates.
+
+BasPlug.process_exp:A perl regular expression to match against filenames. Matching filenames will be processed by this plugin. For example, using '(?i).html?\$' matches all documents ending in .htm or .html (case-insensitive).
+
+BasPlug.read_denied:Read permission denied for %s
+
+BasPlug.separate_cjk:Insert spaces between Chinese/Japanese/Korean characters to make each character a word. Use if text is not segmented.
+
+BasPlug.smart_block:Block files in a smarter way than just looking at filenames.
+
+BasPlug.stems:stems
+
+BasPlug.unsupported_encoding:WARNING: %s appears to be encoded in an unsupported encoding (%s) - using %s
+
+BasPlug.wrong_encoding:WARNING: %s was read using %s encoding but appears to be encoded as %s.
+
+BibTexPlug.desc:BibTexPlug reads bibliography files in BibTex format. BibTexPlug creates a document object for every reference in the file. It is a subclass of SplitPlug, so if there are multiple records, all are read.
+
+BookPlug.desc:Creates multi-level document from document containing <<TOC>> level tags. Metadata for each section is taken from any other tags on the same line as the <<TOC>>. e.g. <<Title>>xxxx<</Title>> sets Title metadata. Everything else between TOC tags is treated as simple html (i.e. no processing of html links or any other HTMLPlug type stuff is done). Expects input files to have a .hb file extension by default (this can be changed by adding a -process_exp option a file with the same name as the hb file but a .jpg extension is taken as the cover image (jpg files are blocked by this plugin). BookPlug is a simplification (and extension) of the HBPlug used by the Humanity Library collections. BookPlug is faster as it expects the input files to be cleaner (The input to the HDL collections contains lots of excess html tags around <<TOC>> tags, uses <<I>> tags to specify images, and simply takes all text between <<TOC>> tags and start of text to be Title metadata). If you're marking up documents to be displayed in the same way as the HDL collections, use this plugin instead of HBPlug.
+
+BRSPlug.desc: BRSPlug deals with the BBC collections. There are three BBC collections we deliver, which are National Sound Archive (nsa), British Film Institute (bfi), and BBC Other (bbcother) (everything does not fit into one of the first two). Four options are provided which are all-build all collections, nsa-build the NSA collections, bfi-build the BFI collections, bbcother-build the OTHER collections.
+
+ConvertToPlug.convert_to:Plugin converts to TEXT or HTML or various types of Image (e.g. JPEG, GIF, PNG).
+ConvertToPlug.convert_to.auto:Automatically select the format converted too.  Format chosen depends on input document type, for example Word will automatically be converted to HTML, whereas PowerPoint will be converted to Greenstone's PagedImage format.
+ConvertToPlug.convert_to.html:HTML format.
+ConvertToPlug.convert_to.text:Plain text format.
+ConvertToPlug.convert_to.pagedimg_jpg:JPEG format.
+ConvertToPlug.convert_to.pagedimg_gif:GIF format.
+ConvertToPlug.convert_to.pagedimg_png:PNG format. 
+
+ConvertToPlug.desc:The plugin is inherited by such plugins as WordPlug and PDFPlug. It facilitates the conversion of these document types to either HTML or TEXT by setting up variable that instruct ConvertToBasPlug how to work. It works by dynamically inheriting HTMLPlug or TEXTPlug based on the plugin argument 'convert_to'. If the argument is not present, the default is to inherit HTMLPlug.
+
+ConvertToPlug.use_strings:If set, a simple strings function will be called to extract text if the conversion utility fails.
+
+ConvertToRogPlug.desc:A plugin that inherits from RogPlug.
+
+DBPlug.desc:Uses records from a database as documents.
+
+DBPlug.title_sub:Substitution expression to modify string stored as Title. Used by, for example, PSPlug to remove "Page 1" etc from text used as the title.
+
+DSpacePlug.desc:DSpacePlug reads the DSpace Export collection and its dublin_core metadata file, and convert to GSII compilance format.	
+DSpacePlug.first_inorder_ext: This is used to identify the primary stream of DSpace collection document. With this option, the system will treat the defined ext types of document in sequence to look for the possible primary stream. 
+DSpacePlug.first_inorder_mime:This is used to identify the primary data stream of DSpace collection document.With this option, the system will treat the defined mime types of document in sequence to look for the possible primary stream. 
+DSpacePlug.only_first_doc:This is used to identify the primary data stream of DSpace collection document.With this option, the system will treat the first document in dublic_core metadata file as the possible primary stream. 
+
+EMAILPlug.desc:Email plug reads email files.  These are named with a simple number (i.e. as they appear in maildir folders) or with the extension .mbx (for mbox mail file format).\nDocument text: The document text consists of all the text after the first blank line in the document.\nMetadata (not Dublin Core!):\n\t\$Headers      All the header content (optional, not stored by default)\n\t\$Subject      Subject: header\n\t\$To           To: header\n\t\$From         From: header\n\t\$FromName     Name of sender (where available)\n\t\$FromAddr     E-mail address of sender\n\t\$DateText     Date: header\n\t\$Date         Date: header in GSDL format (eg: 19990924)
+
+EMAILPlug.no_attachments:Do not save message attachments.
+
+EMAILPlug.headers:Store email headers as "Headers" metadata.
+
+EMAILPlug.split_exp:A perl regular expression used to split files containing many messages into individual documents.
+
+ExcelPlug.desc:A plugin for importing Microsoft Excel files.
+
+FOXPlug.desc:Plugin to process a Foxbase dbt file. This plugin provides the basic functionality to read in the dbt and dbf files and process each record. This general plugin should be overridden for a particular database to process the appropriate fields in the file.
+
+GAPlug.desc:Processes GreenstoneArchive XML documents. Note that this plugin does no syntax checking (though the XML::Parser module tests for well-formedness). It's assumed that the GreenstoneArchive files conform to their DTD.
+
+GISBasPlug.extract_placenames:Extract placenames from within text and set as metadata. Requires GIS extension to Greenstone.
+
+GISBasPlug.gazetteer:Gazetteer to use to extract placenames from within text and set as metadata. Requires GIS extension to Greenstone.
+
+GISBasPlug.place_list:When extracting placements, include list of placenames at start of the document. Requires GIS extension to Greenstone.
+
+GMLPlug.desc:Plugin which processes a GML format document assumes that gml tags are all in lower-case.
+
+HBPlug.desc:Plugin which processes an HTML book directory. This plugin is used by the Humanity Library collections and does not handle input encodings other than ascii or extended ascii. This code is kind of ugly and could no doubt be made to run faster, by leaving it in this state I hope to encourage people to make their collections use HBSPlug instead ;-)\n\nUse HBSPlug if creating a new collection and marking up files like the Humanity Library collections. HBSPlug accepts all input encodings but expects the marked up files to be cleaner than those used by the Humanity Library collections
+
+HTMLPlug.assoc_files:Perl regular expression of file extensions to associate with html documents.
+
+HTMLPlug.desc:This plugin processes HTML files
+
+HTMLPlug.description_tags:Split document into sub-sections where <Section> tags occur. Note that by setting this option you implicitly set -no_metadata, as all metadata should be included within the <Section> tags. Also, '-keep_head' will have no effect when this option is set.
+
+HTMLPlug.file_is_url:Set if input filenames make up url of original source documents e.g. if a web mirroring tool was used to create the import directory structure.
+
+HTMLPlug.hunt_creator_metadata:Find as much metadata as possible on authorship and place it in the 'Creator' field. Requires the -metadata_fields flag.
+
+HTMLPlug.keep_head:Don't remove headers from html files.
+
+HTMLPlug.metadata_fields:Comma separated list of metadata fields to attempt to extract. Use 'tag<tagname>' to have the contents of the first <tagname> pair put in a metadata element called 'tagname'. Capitalise this as you want the metadata capitalised in Greenstone, since the tag extraction is case insensitive.
+
+HTMLPlug.no_metadata:Don't attempt to extract any metadata from files.
+
+HTMLPlug.no_strip_metadata_html:Comma separated list of metadata names, or 'all'. Used with -description_tags, it prevents stripping of HTML tags from the values for the specified metadata.
+
+HTMLPlug.nolinks:Don't make any attempt to trap links (setting this flag may improve speed of building/importing but any relative links within documents will be broken).
+
+HTMLPlug.rename_assoc_files:Renames files associated with documents (e.g. images). Also creates much shallower directory structure (useful when creating collections to go on cd-rom).
+
+HTMLPlug.sectionalise_using_h_tags:Automatically create a sectioned document using h1, h2, ... hX tags.
+
+HTMLPlug.title_sub:Substitution expression to modify string stored as Title. Used by, for example, PDFPlug to remove "Page 1", etc from text used as the title.
+
+ImagePlug.converttotype:Convert main image to format 's'.
+
+ImagePlug.desc:This plugin processes images, adding basic metadata
+
+ImagePlug.minimumsize:Ignore images smaller than n bytes.
+
+ImagePlug.noscaleup:Don't scale up small images when making thumbnails.
+
+ImagePlug.screenviewsize:If set, makes an image of size n for screen display and sets Screen, ScreenSize, ScreenWidth and ScreenHeight metadata.  By default it is not set.
+
+ImagePlug.screenviewtype:If -screenviewsize is set, this sets the screen display image type.
+
+ImagePlug.thumbnailsize:Make thumbnails of size nxn.
+
+ImagePlug.thumbnailtype:Make thumbnails in format 's'.
+
+IndexPlug.desc:This recursive plugin processes an index.txt file. The index.txt file should contain the list of files to be included in the collection followed by any extra metadata to be associated with each file.\n\nThe index.txt file should be formatted as follows: The first line may be a key (beginning with key:) to name the metadata fields (e.g. key: Subject Organization Date). The following lines will contain a filename followed by the value that metadata entry is to be set to. (e.g. 'irma/iw097e 3.2 unesco 1993' will associate the metadata Subject=3.2, Organization=unesco, and Date=1993 with the file irma/iw097e if the above key line was used)\n\nNote that if any of the metadata fields use the Hierarchy classifier plugin then the value they're set to should correspond to the first field (the descriptor) in the appropriate classification file.\n\nMetadata values may be named separately using a tag (e.g. <Subject>3.2) and this will override any name given to them by the key line. If there's no key line any unnamed metadata value will be named 'Subject'.
+
+ISISPlug.desc:This plugin processes CDS/ISIS databases. For each CDS/ISIS database processed, three files must exist in the collection's import folder: the Master file (.mst), the Field Definition Table (.fdt), and the Cross-Reference File (.xrf).
+
+ISISPlug.subfield_separator:The string used to separate subfields in CDS/ISIS database records.
+
+ISISPlug.entry_separator:The string used to separate multiple values for single metadata fields in CDS/ISIS database records.
+
+LaTeXPlug.desc:Plugin for LaTeX documents.
+
+MACROPlug.desc:Creates simple single-level document. Adds Title metadata of first line of text (up to 100 characters long).
+
+MARCPlug.desc:Basic MARC plugin.
+
+MARCPlug.metadata_mapping:Name of file that includes mapping details from MARC values to Greenstone metadata names. Defaults to 'marctodc.txt' found in the site's etc directory.
+
+MetadataPass.desc:On-the-side base class to BasPlug that supports metadata plugins utilise metadata_read pass of import.pl
+GISBasPlug.desc: On-the-side base class to BasPlug that supports GIS capabilities
+
+NULPlug.desc:Dummy (.nul) file plugin. Used with the files produced by exploding metadata database files.
+
+OAIPlug.desc:Basic Open Archives Initiate (OAI) plugin.
+
+OggVorbisPlug.add_technical_metadata:Add technical (eg. bitrate) metadata.
+
+OggVorbisPlug.desc:A plugin for importing Ogg Vorbis audio files.
+
+PagedImgPlug.desc:Plugin for documents made up of a sequence of images, with optional OCR text for each image.
+
+PagedImgPlug.documenttype:Set the document type (used for display)
+PagedImgPlug.documenttype.paged:Paged documents have next and previous arrows and a 'go to page X' box
+PagedImgPlug.documenttype.hierarchy:Hierarchical documents have a table of contents
+
+PagedImgPlug.headerpage:Add a top level header page (that contains no image) to each document.
+
+PagedImgPlug.screenview:Produce a screenview image for each image, and set Screen, ScreenSize, ScreenWidth and ScreenHeight metadata.
+
+PagedImgPlug.screenviewsize:Make screenview images of size nxn.
+
+PagedImgPlug.screenviewtype:Make screenview images in format 's'.
+
+PagedImgPlug.thumbnail:Produce a thumbnail for each image
+
+PDFPlug.allowimagesonly:Allow PDF files with no extractable text. Avoids the need to have -complex set. Only useful with convert_to html.
+PDFPlug.complex:Create more complex output. With this option set the output html will look much more like the original PDF file. For this to function properly you Ghostscript installed (for *nix gs should be on your path while for windows you must have gswin32c.exe on your path).
+
+PDFPlug.desc:Reasonably with-it pdf plugin.
+
+PDFPlug.nohidden:Prevent pdftohtml from attempting to extract hidden text. This is only useful if the -complex option is also set.
+
+PDFPlug.noimages:Don't attempt to extract images from PDF.
+
+PDFPlug.use_sections:Create a separate section for each page of the PDF file.
+
+PDFPlug.zoom:The factor by which to zoom the PDF for output (this is only useful if -complex is set).
+
+PPTPlug.desc:A plugin for importing Microsoft PowerPoint files.
+
+PPTPlug.windows_scripting:Use MicroSoft Windows scripting technology (Visual Basic for Applications) to get PPT to convert document to various image types (e.g. JPEG,PNG,GIF) rather than rely on the open source package ppttohtml. 
+
+PSPlug.desc:This is a \"poor man's\" ps to text converter. If you are serious, consider using the PRESCRIPT package, which is available for download at http://www.nzdl.org/html/software.html
+
+PSPlug.extract_date:Extract date from PS header.
+
+PSPlug.extract_pages:Extract pages from PS header.
+
+PSPlug.extract_title:Extract title from PS header.
+
+RealMediaPlug.desc:A plugin for processing Real Media files.
+
+RecPlug.desc:RecPlug is a plugin which recurses through directories processing each file it finds.
+
+RecPlug.recheck_directories:After the files in an import directory have been processed, re-read the directory to discover any new files created.
+
+RecPlug.use_metadata_files:Read metadata from metadata XML files.
+
+ReferPlug.desc:ReferPlug reads bibliography files in Refer format.
+
+ReferPlug.longdesc:ReferPlug reads bibliography files in Refer format.\nBy Gordon W. Paynter (gwp\@cs.waikato.ac.nz), November 2000\n\nLoosely based on hcibib2Plug by Steve Jones (stevej\@cs.waikato.ac.nz). Which was based on EMAILPlug by Gordon Paynter (gwp\@cs.waikato.ac.nz). Which was based on old versions of HTMLplug and HCIBIBPlugby by Stefan Boddie and others -- it's hard to tell what came from where, now.\n\nReferPlug creates a document object for every reference in the file. It is a subclass of SplitPlug, so if there are multiple records, all are read.\n\nDocument text:\n\tThe document text consists of the reference in Refer format.\nMetadata:\n\t\$Creator	\%A	Author name\n\t\$Title		\%T	Title of article of book\n\t\$Journal	\%J	Title of Journal\n\t\$Booktitle	\%B	Title of book containing the publication\n\t\$Report		\%R	Type of Report, paper or thesis\n\t\$Volume		\%V	Volume Number of Journal\n\t\$Number		\%N	Number of Journal within Volume\n\t\$Editor		\%E	Editor name\n\t\$Pages		\%P	Page Number of article\n\t\$Publisher	\%I	Name of Publisher\n\t\$Publisheraddr	\%C	Publisher's address\n\t\$Date		\%D	Date of publication\n\t\$Keywords	\%K	Keywords associated with publication\n\t\$Abstract	\%X	Abstract of publication\n\t\$Copyright\t\%*	Copyright information for the article
+
+RogPlug.desc:Creates simple single-level documents from .rog or .mdb files.
+
+RTFPlug.desc:Plugin for importing Rich Text Format files.
+
+SRCPlug.desc:Filename is currently used for Title ( optionally minus some prefix ). Current languages:\ntext: READMEs/Makefiles\nC/C++   (currently extracts #include statements and C++ class decls)\nPerl    (currently only done as text)\nShell   (currently only done as text)
+
+SRCPlug.remove_prefix:Remove this leading pattern from the filename (eg -remove_prefix /tmp/XX/src/). The default is to remove the whole path from the filename.
+
+SplitPlug.desc:SplitPlug is a plugin for splitting input files into segments that will then be individually processed. This plugin should not be called directly.  Instead, if you need to process input files that contain several documents, you should write a plugin with a process function that will handle one of those documents and have it inherit from SplitPlug.  See ReferPlug for an example.
+
+SplitPlug.split_exp:A perl regular expression to split input files into segments.
+
+TEXTPlug.desc:Creates simple single-level document. Adds Title metadata of first line of text (up to 100 characters long).
+
+TEXTPlug.title_sub:Substitution expression to modify string stored as Title. Used by, for example, PSPlug to remove "Page 1" etc from text used as the title.
+
+UnknownPlug.assoc_field:Name of the metadata field that will hold the associated file's name.
+
+UnknownPlug.desc:This is a simple Plugin for importing files in formats that Greenstone doesn't know anything about.  A fictional document will be created for every such file, and the file itself will be passed to Greenstone as the \"associated file\" of the document.
+
+UnknownPlug.file_format:Type of the file (e.g. MPEG, MIDI, ...)
+
+UnknownPlug.mime_type:Mime type of the file (e.g. image/gif).
+
+UnknownPlug.process_extension:Process files with this file extension. This option is an alternative to process_exp that is simpler to use but less flexible.
+
+MP3Plug.desc:Plugin for processing MP3 files.
+
+MP3Plug.assoc_images:Use google image search to locate images related to MP3 file based on ID3 Title and Artist metadata.
+
+MP3Plug.assoc_images:Used to store [applet] metadata for each document that contains the necessary HTML for an MP3 audio player applet to play that file.
+
+MP3Plug.metadata_fields:Comma separated list of metadata fields to extract (assuming present) in an MP3 file. Use \"*\" to extract all the fields.
+
+W3ImgPlug.aggressiveness:Range of related text extraction techniques to use.
+W3ImgPlug.aggressiveness.1:Filename, path, ALT text only.
+W3ImgPlug.aggressiveness.2:All of 1, plus caption where available.
+W3ImgPlug.aggressiveness.3:All of 2, plus near paragraphs where available.
+W3ImgPlug.aggressiveness.4:All of 3, plus previous headers (<h1>, <h2>...) where available.
+W3ImgPlug.aggressiveness.5:All of 4, plus textual references where available.
+W3ImgPlug.aggressiveness.6:All of 4, plus page metatags (title, keywords, etc).
+W3ImgPlug.aggressiveness.7:All of 6, 5 and 4 combined.
+W3ImgPlug.aggressiveness.8:All of 7, plus repeat caption, filename, etc (raise ranking of more relevant results).
+W3ImgPlug.aggressiveness.9:All of 1, plus full text of source page.
+
+W3ImgPlug.caption_length:Maximum length of captions (in characters).
+
+W3ImgPlug.convert_params:Additional parameters for ImageMagicK convert on thumbnail creation. For example, '-raise' will give a three dimensional effect to thumbnail images.
+
+W3ImgPlug.desc:A plugin for extracting images and associated text from webpages.
+
+W3ImgPlug.document_text:Add image text as document:text (otherwise IndexedText metadata field).
+
+W3ImgPlug.index_pages:Index the pages along with the images. Otherwise reference the pages at the source URL.
+
+W3ImgPlug.max_near_text:Maximum characters near images to extract.
+
+W3ImgPlug.min_height:Pixels. Skip images shorter than this.
+
+W3ImgPlug.min_near_text:Minimum characters of near text or caption to extract.
+
+W3ImgPlug.min_size:Bytes. Skip images smaller than this.
+
+W3ImgPlug.min_width:Pixels. Skip images narrower than this.
+
+W3ImgPlug.neartext_length:Target length of near text (in characters).
+
+W3ImgPlug.no_cache_images:Don't cache images (point to URL of original).
+
+W3ImgPlug.smallpage_threshold:Images on pages smaller than this (bytes) will have the page (title, keywords, etc) meta-data added.
+
+W3ImgPlug.textrefs_threshold:Threshold for textual references. Lower values mean the algorithm is less strict.
+
+W3ImgPlug.thumb_size:Max thumbnail size. Both width and height.
+
+WordPlug.desc:A plugin for importing Microsoft Word documents.
+
+WordPlug.windows_scripting:Use MicroSoft Windows scripting technology (Visual Basic for Applications) to get Word to convert document to HTML rather than rely on the open source package WvWare.  Causes Word application to open on screen if not already running.
+
+WordPlug.delete_toc:Remove the "Table of Contents" and "List of Tables" from the converted HTML file. 
+
+WordPlug.title_header:set up the possible user-defined style for the title_header.
+
+WordPlug.level1_header:possible user-defined styles for the level1 header in the HTML document (equivalent to <h1>).
+
+WordPlug.level2_header:possible user-defined styles for the level2 header in the HTML document (equivalent to <h2>).
+
+WordPlug.level3_header:possible user-defined styles for the level3 header in the HTML document (equivalent <h3>).
+
+WordPlug.toc_header:possible user-defined header styles for the TOC.
+
+WordPlug.tof_header:possible user-defined header styles for the TOF
+
+WordPlug.extracted_word_metadata: This is to retrieve metadata from the HTML document converted by VB scripting. It allows users to define comma separated list of metadata fields to attempt to extract. Use 'tag<tagname>' to have the contents of the first <tagname> pair put in a metadata element called 'tagname'. Capitalise this as you want the metadata capitalised in Greenstone, since the tag extraction is case insensitive
+
+XMLPlug.desc:Base class for XML plugins.
+
+XMLPlug.xslt:Transform a matching input document with the XSLT in the named file.  A relative filename is assumed to be in the collection's file area, for instance etc/mods2dc.xsl.
+ZIPPlug.desc:Plugin which handles compressed and/or archived input formats currently handled formats and file extensions are:\ngzip (.gz, .z, .tgz, .taz)\nbzip (.bz)\nbzip2 (.bz2)\nzip (.zip .jar)\ntar (.tar)\n\nThis plugin relies on the following utilities being present (if trying to process the corresponding formats):\ngunzip (for gzip)\nbunzip (for bzip)\nbunzip2 \nunzip (for zip)\ntar (for tar)
+
+
+#
+# Perl module strings
+#
+
+classify.could_not_find_classifier:ERROR: Could not find classifier \"%s\"
+
+plugin.could_not_find_plugin:ERROR: Could not find plugin \"%s\"
+
+plugin.including_archive:including the contents of 1 ZIP/TAR archive
+
+plugin.including_archives:including the contents of %d ZIP/TAR archives
+
+plugin.kill_file:Process killed by .kill file
+
+plugin.n_considered:%d documents were considered for processing
+
+plugin.n_included:%d were processed and included in the collection
+
+plugin.n_rejected:%d were rejected
+
+plugin.n_unrecognised:%d were unrecognised
+
+plugin.no_plugin_could_process:WARNING: No plugin could process %s
+
+plugin.no_plugin_could_recognise:WARNING: No plugin could recognise %s
+
+plugin.no_plugin_could_process_this_file:no plugin could process this file
+
+plugin.no_plugin_could_recognise_this_file:no plugin could recognise this file
+
+plugin.one_considered:1 document was considered for processing
+
+plugin.one_included:1 was processed and included in the collection
+
+plugin.one_rejected:1 was rejected
+
+plugin.one_unrecognised:1 was unrecognised
+
+plugin.see_faillog:See %s for a list of unrecognised and/or rejected documents
+
+PrintUsage.default:Default
+
+PrintUsage.required:REQUIRED
Index: /main/tags/3.01/gsdl/perllib/strings_es.rb
===================================================================
--- /main/tags/3.01/gsdl/perllib/strings_es.rb	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/strings_es.rb	(revision 21112)
@@ -0,0 +1,1275 @@
+#
+# Resource bundle description
+#
+
+Language.code:es
+
+Language.name:EspaÃ±ol
+
+
+OutputEncoding.unix:iso_8859_1
+
+OutputEncoding.windows:iso_8859_1
+
+
+
+#
+# Common output messages
+#
+
+common.cannot_create_file:ERROR: no puede crear archivo %s
+
+
+common.cannot_find_cfg_file:ERROR: no puede encontrar el archivo de configuraciÃ³n %s
+
+
+common.cannot_open:ERROR: no puede abrir %s
+
+
+common.cannot_open_fail_log:ERROR: no puede abrir el fail log %s
+
+
+common.cannot_open_output_file:ERROR: no puede abrir el archivo del rendimiento output %s
+
+
+common.cannot_read:ERROR: no puede leer %s
+
+
+common.cannot_read_file:ERROR: no puede leer el archivo %s
+
+
+common.general_options:las opciones generales (para % s)
+
+
+common.must_be_implemented:la funciÃ³n debe implementarse en clase-subalterno
+
+
+common.options:opciones
+
+
+common.processing:procesando
+
+
+common.specific_options:opciones especÃ­ficas
+
+
+common.usage:uso
+
+
+
+#
+# Script option descriptions and output messages
+#
+
+scripts.language:El idioma para desplegar las descripciones de la opciÃ³n en ('en_US' especifica inglÃ©s americano). Requiere las traducciones de las descripciones de la opciÃ³n existente en el archivo de perllib/strings_language-code.rb.
+
+
+scripts.xml:Produce la informaciÃ³n en un formato XML, sin comentarios 'elegantes' pero con mucho mÃ¡s detalle.
+
+
+scripts.listall:Listar todos los Ã­tems conocidos.
+
+
+
+# -- buildcol.pl --
+
+buildcol.archivedir:Donde los archivos residen.
+
+
+buildcol.builddir:DÃ³nde poner los Ã­ndices construidos.
+
+
+buildcol.cachedir:La colecciÃ³n se construirÃ¡ aquÃ­ temporalmente antes de copiarse al directorio construido.
+
+
+buildcol.cannot_open_cfg_file:ADVERTENCIA: no puede abrir los archivos config a efecto de poner al dÃ­a: %s
+
+
+buildcol.collectdir:El directorio de colecciÃ³n. 
+
+
+buildcol.copying_back_cached_build:Copiando la figura anteriormente construida
+
+
+buildcol.create_images:Intente crear las imÃ¡genes predefinidas para la nueva colecciÃ³n. Esto confÃ­a en el Gimp instalÃ¡ndose a lo largo con los mÃ³dulos perl pertinentes que permitan el scripting del perl.
+
+
+buildcol.debug:Imprima el output a STDOUT.
+
+
+buildcol.desc:La escritura de PERL construÃ­a una colecciÃ³n del greenstone de los documentos GML.
+
+
+buildcol.faillog:Falla en el archivo de login. Este log recibe los nombres de archivos de cualquier archivo que haya fallado al ser procesado.
+
+
+buildcol.index:Indice para construir (construirÃ¡ todos los archivos en el config si no estÃ¡ determinado).
+
+
+buildcol.keepold:No destruirÃ¡ los volÃºmenes actuales del directorio construido.
+
+
+buildcol.maxdocs:El nÃºmero mÃ¡ximo de documentos para construir.
+
+
+buildcol.mode:Las partes del proceso construido para llevar a cabo.
+
+buildcol.mode.all:Realizar todo.
+
+buildcol.mode.build_index:Simplemente ponga en un Ã­ndice el texto.
+
+buildcol.mode.compress_text:Simplemente comprima el texto.
+
+buildcol.mode.infodb:Simplemente construya la base de datos del metadata.
+
+
+buildcol.no_default_images:No se generarÃ¡n las imÃ¡genes predefinidas.
+
+
+buildcol.no_image_script:ADVIRTIENDO: Imagen que hace la escritura no encontrada: %s
+
+
+buildcol.no_strip_html:No despoje las etiquetas html del texto puesto en un Ã­ndice (sÃ³lo utilice para las colecciones del mgpp).
+
+
+buildcol.no_text:No guarde el texto comprimido. Esta opciÃ³n es Ãºtil para minimizar el tamaÃ±o de los Ã­ndices construidos si usted siempre piensa desplegar los documentos originales en momento que estÃ¡ utilizando el programa (es decir usted no podrÃ¡ recuperar la versiÃ³n del texto comprimido).
+
+
+buildcol.out:Nombre de archivo o palanca para imprimir el estado del rendimiento a.
+
+
+buildcol.params:[las opciones] collection-name
+
+
+buildcol.remove_empty_classifications:Ocultar los clasificadores y los nodos de clasificaciÃ³n vacÃ­os (aquellos que no contienen documentos).
+
+
+buildcol.unlinked_col_images:No pueden unirse las imÃ¡genes de la colecciÃ³n de manera correcta.
+
+
+buildcol.unknown_mode:Modo desconocido: %s
+
+
+buildcol.updating_archive_cache:El archivo poniendo al dÃ­a actualizando
+
+
+buildcol.verbosity:Los mandos la cantidad de rendimiento. El 0=ninguno, 3=muchos.
+
+
+
+# -- classinfo.pl --
+
+classinfo.collect:Dar un nombre de la colecciÃ³n harÃ¡n que classinfo.pl aparezca primero clasifique en el collect/collection-name/perllib/classify. Si el clasificador no se encuentra allÃ­ buscarÃ¡ en el directorio general perllib/classify.
+
+
+classinfo.desc:Imprime la informaciÃ³n de las impresiones sobre un clasificador.
+
+
+classinfo.general_options:Se heredan las opciones generales de las clases paternales del clasificador.
+
+
+classinfo.info:info
+
+
+classinfo.no_classifier_name:ERROR: Usted debe proporcionar un nombre del clasificador.
+
+
+classinfo.option_types:Los clasificadores pueden tomar dos tipos de opciones
+
+
+classinfo.params:[opciones] nombre-clasificador
+
+
+classinfo.passing_options:Las opciones pueden pasarse a cualquier clasificador incluyÃ©ndolos en su archivo de configuraciÃ³n collect.cfg.
+
+
+classinfo.specific_options:Se definen las opciones especÃ­ficas dentro del propio clasificador, y sÃ³lo estÃ¡ disponible a este clasificador particular.
+
+
+# -- explode_metadata_database.pl --
+explode.desc:Explotar una base de datos de metadatos.
+
+explode.document_field:El elemento de los meta-datos que especifica los nombres de archivo de los documentos a obtener e incluir en la colecciÃ³n.
+
+
+explode.document_prefix:Un prefijo para las ubicaciones de los documentos (para usar con la opciÃ³n document_field).
+
+
+explode.document_suffix:Un sufijo para las ubicaciones de los documentos (para usar con la opciÃ³n document_field).
+
+
+explode.encoding:CodificaciÃ³n a utilizar al leer el archivo de la base de datos
+explode.filename_field:Campo a utilizar para los nombres de los archivos nulos generados (si no existe document_field o ese campo estÃ¡ vacÃ­o)
+explode.metadata_set:Conjunto de metadatos (espacio de nombres) para exportar todos los metadatos
+explode.plugin:Plugin a utilizar para explotar
+explode.params:[opciones] nombre de archivo
+# -- exportcol.pl --
+
+exportcol.out:Nombre del archivo o identificador para imprimir el estado de los resultados. 
+
+exportcol.cdname:El nombre del CD-ROM. Esto es lo que aparecerÃ¡ en el menÃº de inicio una vez que el CD-ROM se haya instalado.
+
+exportcol.desc:GuiÃ³n PERL usado para exportar una o mÃ¡s colecciones a un CD-ROM para Windows. 
+
+exportcol.params:[opciones] nombre de la colecciÃ³n 1 nombre de la colecciÃ³n 2 ...
+
+exportcol.coll_not_found:El %s de la colecciÃ³n no es vÃ¡lido: la colecciÃ³n no se encontrÃ³ en %s.
+
+exportcol.coll_dirs_not_found:El %s de la colecciÃ³n no es vÃ¡lido: no se pudo encontrar uno de los siguientes directorios: 
+
+exportcol.fail:exportcol.pl no se pudo ejecutar:
+
+exportcol.no_valid_colls:Las colecciones especificadas no son vÃ¡lidas y por lo tanto no se pueden exportar.
+
+exportcol.couldnt_create_dir:No se pudo crear el directorio %s.
+
+exportcol.couldnt_create_file:No se pudo crear %s.
+
+exportcol.instructions:Para crear un CD-ROM en Windows que se podrÃ¡ instalar automÃ¡ticamente, grabe el contenido de esta carpeta en un CD-ROM.
+
+exportcol.non_exist_files:Uno o mÃ¡s de los siguientes archivos y directorios no existe:
+
+exportcol.success:exportcol.pl se ejecutÃ³ con Ã©xito:
+
+exportcol.output_dir:Las colecciones que se exportaron (%s) se encuentran en %s
+
+exportcol.nonetscape:No adjuntar netscape al CD-ROM exportado.
+exportcol.export_coll_not_installed:La funcionalidad de 'Exportar a un CD-ROM' no ha estado instalada.
+
+
+# -- import.pl --
+
+import.archivedir:Lugar donde los materiales convertidos finalizan.
+
+
+import.cannot_open_stats_file:ADVIRTIENDO: No podrÃ­a abrir el archivos stats %s.
+
+
+import.cannot_open_fail_log:ERROR: No se pudo abrir el registro %s
+
+
+import.cannot_sort:ADVIRTIENDO: import.pl no puede ordenar los documentos cuando el tamaÃ±o del grupo > 1. la opciÃ³n sortmeta se ignorarÃ¡.
+
+
+import.collectdir:El directorio de la colecciÃ³n.
+
+
+import.complete:La importaciÃ³n ha sido completada
+
+
+import.debug:La impresiÃ³n importÃ³ el texto a STDOUT.
+
+
+import.desc:La escritura de PERL utilizada para importar los archivos en un formato de GML listo para utilizar.
+
+
+import.faillog:FallÃ³ log el nombre del archivo . Este log recibe los nombres de archivo de cualquier archivo que haya fallado al ser procesado.
+
+
+import.groupsize:El nÃºmero de documentos de importaciÃ³n para agruparse en un archivo de XML.
+
+
+import.gzip:Use el gzip para comprimir documentos xml (no se olvide de incluir ZIPPlug en su lista plugin al construir de los documentos comprimidos).
+
+
+import.importdir:Donde el material original reside.
+
+
+import.keepold:No destruirÃ¡ los volÃºmenes actuales del directorio de los archivos (el valor predeterminado).
+
+
+import.maxdocs:El nÃºmero mÃ¡ximo de documentos para importar.
+
+
+import.no_plugins_loaded:ERROR: NingÃºn plugins ha sido cargado.
+
+
+import.OIDtype:El mÃ©todo a utilizar al generar los Ãºnicos identificadores para cada documento.
+
+import.OIDtype.hash:Conjuga los volÃºmenes del archivo. El identificador del documento serÃ¡ el mismo cada vez que la colecciÃ³n es importada.
+
+
+import.OIDtype.incremental:Una cuenta del documento simple que es significativamente mÃ¡s rÃ¡pido que hash. No se garantiza siempre asignar el mismo identificador a un documento dado aunque y no permite agregar los documentos extensos a los archivos xml existentes.
+
+
+import.OIDtype.assigned:Utiliza 'D' mÃ¡s el valor de dc.Identifier como el identificador del documento. Los dc.Identifiers deben ser Ãºnicos. Si al documento no se le ha asignado un dc.Identifier, se procede a utilizar una identificaciÃ³n generada por hash.
+
+import.OIDtype.dirname:Utiliza como identificador 'J' mÃ¡s el nombre del directorio padre. Se parte de la base de que sÃ³lo existe un documento por directorio y que todos los nombres de directorio son Ãºnicos. Por ejemplo: import/b13as/h15ef/page.html producirÃ¡ Jh15ef como identificador.
+
+
+import.saveas:Establece el fomato de archivo a generar. La preselecciÃ³n es GA.
+
+
+import.saveas.GA:GenerarÃ¡ formato de archivo Greenstone.
+
+
+import.saveas.METS:GenerarÃ¡ formato METS:
+
+
+import.out:Nombre de archivo o maneja para imprimir el estado del rendimiento.
+
+
+import.params:[opciones] colecciÃ³n-nombre
+
+
+import.removeold:EliminarÃ¡ los volÃºmenes antiguos del directorio de los archivos--utilice con cuidado.
+
+
+import.removing_archives:Los volÃºmenes actuales eliminados del directorio de los archivos...
+
+
+import.sortmeta:Ordena documentos alfabÃ©ticamente por metadatos para la construcciÃ³n. Los resultados de las bÃºsquedas booleanas se mostrarÃ¡n con este orden. Se desactivarÃ¡ si groupsize > 1.
+
+
+import.statsfile:Nombre de archivo o habilidad para imprimir las estadÃ­sticas de importaciÃ³n.
+
+
+import.stats_backup:ImprimirÃ¡ el stats en lugar de STDERR.
+
+
+import.verbosity:Controla la cantidad de output rendimiento. El 0=ninguno, 3=muchos.
+
+
+
+# -- export.pl --
+
+export.exportdir:LocalizaciÃ³n final del material exportado.
+
+
+export.cannot_open_stats_file:ATENCIÃN:No se puede abrir el archivo de estadÃ­sticas %s.
+
+
+export.cannot_open_fail_log:ERROR: No se puede abrir el registro de errores %s
+
+
+export.cannot_sort:ATENCIÃN: export.pl no puede ordenar los documentos cuando groupsize>1. Se ignorarÃ¡ la opciÃ³n sortmeta.
+
+
+export.collectdir:Directorio de la colecciÃ³n
+
+
+export.complete:ExportaciÃ³n completa
+
+
+export.debug:Imprimir el texto exportado a STDOUT.
+
+
+export.desc:Script PERL usado para importar archivos al formato GML preparado para la construcciÃ³n.
+
+
+export.faillog:Nombre de archivo del registro de fallos. Este registro contiene el nombre de cualquier archivo que falle al ser procesado.
+
+
+export.groupsize:NÃºmero de documentos importados a agrupar en un archivo XML.
+
+
+export.gzip:Use gzip para comprimir los documentos xml resultantes (no olvide incluir ZIPPlug en su lista de plugins cuando construya desde documentos comprimidos).
+
+
+export.importdir:LocalizaciÃ³n del material original.
+
+
+export.keepold:No destruirÃ¡ el contenido actual del directorio export (predefinido).
+
+
+export.maxdocs:NÃºmero mÃ¡ximo de documentos a importar.
+
+
+export.no_plugins_loaded:ERROR: no se han cargado plugins.
+
+
+export.OIDtype:MÃ©todo usado cuando se generan identificadores Ãºnicos para cada documento.
+
+export.OIDtype.hash:Marca el contenido del archivo. El identificador del documento serÃ¡ el mismo cada vez que se importe la colecciÃ³n.
+
+export.OIDtype.incremental:Contador simple de documentos bastante mÃ¡s rÃ¡pido que "hash". No se garantiza que siempre asigne el mismo identificador a un documento dado, y no permite que se aÃ±adan nuevos documentos a archivos xml ya existentes.
+
+
+export.saveas:Sirve para decidir el formato de exportaciÃ³n a generar. La selecciÃ³n previa es METS.
+
+
+export.saveas.DSpace:GenerarÃ¡ formato de archivo Dspace.
+
+
+export.saveas.METS:GenerarÃ¡ formato METS.
+
+
+export.out:archivo o dispositivo en el que imprimir el estado de salida.
+
+
+export.params:[options] nombre de colecciÃ³n
+
+
+export.removeold:BorrarÃ¡ los contenidos previos del directorio export -- usar con cuidado.
+
+
+export.removing_export:Borrando los contenidos actuales del directorio exportâŠ
+
+
+export.sortmeta:Ordenar los documentos alfabÃ©ticamente por metadatos para construir. Se desactivarÃ¡ si groupsize > 1.
+
+
+export.statsfile:archivo o dispositivo en el que imprimiar las estadÃ­sticas de exportaciÃ³n.
+
+
+export.stats_backup:ImrprimirÃ¡ las estadÃ­sticas en STDERR en su lugar.
+
+
+export.verbosity:Controla la cantidad de salida. 0=nada, 3=mucha.
+
+
+
+# -- mkcol.pl --
+
+mkcol.about:Sobre el texto para la colecciÃ³n.
+
+
+mkcol.bad_name_cvs:ERROR: Ninguna colecciÃ³n puede nombrarse CVS puesto que puede interferir con directorios creados por la versiÃ³n del sistema CVS.
+
+
+mkcol.bad_name_modelcol:ERROR: Ninguna colecciÃ³n puede nombrarse modelcol puesto que este es el nombre de la colecciÃ³n model
+
+
+mkcol.cannot_find_modelcol:ERROR: Ninguna colecciÃ³n puede nombrarse el modelcol puesto que este es el nombre del ejemplo de colecciÃ³n %s
+
+
+mkcol.col_already_exists:ERROR: Esta colecciÃ³n ya existe.
+
+
+mkcol.collectdir:Directorio dÃ³nde la nueva colecciÃ³n serÃ¡ creada.
+
+
+mkcol.creating_col:Creando la colecciÃ³n %s
+
+
+mkcol.creator:La direcciÃ³n de correo electrÃ³nico del creador de la colecciÃ³n.
+
+
+mkcol.creator_undefined:ERROR: El creador no fue definido. Esta variable se necesita reconocer los nombres duplicados de la colecciÃ³n.
+
+
+mkcol.desc:La escritura de PERL utilizada para crear la estructura del directorio para una nueva colecciÃ³n de Greenstone.
+
+
+mkcol.doing_replacements:realizando reemplazos %s
+
+
+mkcol.long_colname:ERROR: El nombre de la colecciÃ³n debe estar menor de 8 caracteres para que la compatibilidad con el sistema de archivo anterior pueda mantenerse.
+
+
+mkcol.maintainer:La direcciÃ³n del correo electrÃ³nico de la persona que proporciona mantenimiento a la colecciÃ³n (si es diferente a la del creador).
+
+
+mkcol.no_collectdir:ERROR: El dir del coleccionador no existe: %s
+
+
+mkcol.no_colname:ERROR: NingÃºn nombre de la colecciÃ³n fue especificado.
+
+
+mkcol.optionfile:Reciba las opciones del archivo, Ãºtil en sistemas donde las lÃ­neas de orden largas pueden causar problemas.
+
+
+mkcol.params:[opciones] colecciÃ³n-nombre
+
+
+mkcol.plugin:El mÃ³dulo Perl plugin para usar (puede haber entradas mÃºltiples plugin).
+
+
+mkcol.public:Si esta colecciÃ³n tiene acceso anÃ³nimo (verdadero/falso).
+
+
+mkcol.quiet:Opere de manera silenciosa
+
+
+mkcol.success:La nueva colecciÃ³n se creada exitosamente a %s
+
+
+mkcol.title:El tÃ­tulo de la colecciÃ³n.
+
+
+mkcol.win31compat:Establece si el directorio de la colecciÃ³n indicada debe ser conforme a las convenciones de archivo de Windows 3.1 (ej.: longitud de 8 caracteres).
+
+
+# -- pluginfo.pl --
+
+pluginfo.collect:Al dar un nombre a la colecciÃ³n realizarÃ¡ que un pluginfo.pl revise primero en el collect/collection-name/perllib/plugins. Si el plugin general no se encuentra allÃ­ que aparecerÃ¡ en el directorio del perllib/plugins.
+
+
+pluginfo.desc:Imprime informaciÃ³n sobre un plugin.
+
+
+pluginfo.general_options:Se toman las opciones generales de las clases paternal inicial del plugin.
+
+
+pluginfo.info:info
+
+
+pluginfo.no_plugin_name:ERROR: Usted debe proporcionar un nombre plug.
+
+
+pluginfo.option_types:Plugins podrÃ¡n tomar dos tipos de opciones
+
+
+pluginfo.params:[opciones] el nombre-plugin
+
+
+pluginfo.passing_options:Pueden pasarse las opciones a cualquier plugin incluyÃ©ndolos en su archivo de configuraciÃ³n collect.cfg.
+
+
+pluginfo.specific_options:Se definen las opciones especÃ­ficas dentro del propio plugin, y sÃ³lo estÃ¡n disponibles en este plugin en particular.
+
+
+
+#
+# Classifier option descriptions
+#
+
+AZCompactList.allvalues:Utilizar todos los valores de meta-datos encontrados
+
+
+AZCompactList.desc:Plugin para clasificar por orden alfabÃ©tica
+
+
+AZCompactList.doclevel:Nivel para procesar el documento a
+
+AZCompactList.doclevel.top:Documento completo
+
+AZCompactList.doclevel.section:Por secciones
+
+
+AZCompactList.firstvalueonly:Utilizar Ãºnicamente el primer valor de meta-datos encontrado.
+
+
+AZCompactList.freqsort:Clasificar por la frecuencia del nodo en lugar de alfa-numÃ©rico.
+
+
+AZCompactList.maxcompact:NÃºmero mÃ¡ximo de documentos que serÃ¡n desplegados por pÃ¡gina.
+
+
+AZCompactList.metadata:Un solo campo de Meta-datos, o una lista de campos de Meta-datos separada por comas, utilizados para clasificaciÃ³n. Si se especifica una lista, serÃ¡ utilizado el primer tipo de meta-datos que tenga valores. Puede utilizarse al tiempo con las banderas -firstvalueonly y -allvalues, para seleccionar Ãºnicamente el primer valor o todos los valores de la lista.
+
+
+AZCompactList.mincompact:NÃºmero mÃ­nimo de documentos que serÃ¡n desplegados por pÃ¡gina.
+
+
+AZCompactList.mingroup:El valor mÃ¡s pequeÃ±o que causarÃ¡ un grupo en formato jerÃ¡rquico
+
+
+AZCompactList.minnesting:El valor mÃ¡s pequeÃ±o que causarÃ¡ una lista a ser convertida en una lista anidada.
+
+
+AZCompactList.recopt:Usado en metadatos anidados como -metadato AÃ±o/OrganizaciÃ³n.
+
+AZCompactList.sort:Campo de metadatos por medio del cual ordenar los nodos anidados.
+
+
+AZCompactSectionList.desc:VariaciÃ³n de AZCompactList que clasifica secciones en vez de documentos. Las entradas se ordenan por medio de metadatos a nivel de secciÃ³n.
+
+
+AZList.desc:El plugin clasificador alfabÃ©tico.
+
+
+AZList.metadata:Un solo campo de Metadatos o una lista de campos de Metadatos separados por una coma y usados con fines de clasificaciÃ³n. Siguiendo el orden indicado por la lista, se utilizarÃ¡ el primer campo que contiene un valor. La lista se ordenarÃ¡ de acuerdo a este elemento.
+
+
+AZSectionList.desc:El plugin del clasificador para ordenar de manera alfabÃ©tica. Esto es muy similar a AZList excepto que clasifica por secciones el nivel del metadata (excluyendo el nivel superior) en lugar de sÃ³lo el nivel superior del metadata. El Ãºnico cambio es a la subrutina de clasificaciÃ³n () que debe repetir ahora a travÃ©s de cada secciÃ³n, agregando cada uno a la clasificaciÃ³n.
+
+
+BasClas.bad_general_option:El clasificador de porcentajes (%s) utiliza una opciÃ³n general incorrecta (las opciones generales estÃ¡n disponibles a todos los clasificadores). Revise su archivo de configuraciÃ³n collect.cfg.
+
+
+BasClas.builddir:En dÃ³nde poner los Ã­ndices construidos.
+
+
+BasClas.buttonname:Etiqueta para clasificador de pantalla y botÃ³n en la barra de navegaciÃ³n.
+
+
+BasClas.desc:Clase base para todos los clasificadores.
+
+
+BasClas.outhandle:El archivo para escribir el output (o salida) para.
+
+
+BasClas.removeprefix:Un prefijo que al clasificar se debe ignorar en los valores de los metadatos.
+
+BasClas.removesuffix:Un sufijo que al clasificar se debe ignorar en los valores de los metadatos.
+
+BasClas.verbosity:Controla la cantidad de (output) rendimiento. 0=ninguno, 3=muchos.
+
+
+Browse.desc:.
+
+
+DateList.bymonth:Clasificar por aÃ±o y mes en vez de sÃ³lo aÃ±o.
+
+
+DateList.desc:El plugin clasificador ordenador por fecha. El metadata por defaul clasifica por 'Fecha'. Se asume que la fecha estÃ¡ en el formulario aaaammdd
+
+
+DateList.metadata:El metadata que contiene las fechas para clasificar. Se espera que el formato sea el aaaammdd. (aÃ±o, mes, dÃ­a)
+
+
+DateList.reverse_sort:Ordenar los documentos en orden cronolÃ³gico inverso (el mÃ¡s nuevo al comienzo).
+
+
+DateList.nogroup:Haga de cada aÃ±o una entrada individual en la lista horizontal, en vez de extender los aÃ±os con pocas entradas. (Esto tambiÃ©n se puede usar con la opciÃ³n -bymonth para hacer que cada mes tenga una entrada separada en vez de combinarlas).
+
+
+DateList.sort:Un campo de metadata extra para ordenar por en el caso dÃ³nde dos documentos tienen la misma fecha.
+
+
+HFileHierarchy.desc:Plugin clasificador para generar clasificaciones jerÃ¡rquicas basadas en una estructura existente en un archivo suplementario.
+
+
+Hierarchy.hfile:Archivo de clasificaciÃ³n de estructura.
+
+
+Hierarchy.hlist_at_top:Despliega el primer nivel de clasificaciÃ³n horizontalmente
+
+Hierarchy.reverse_sort:Ordenar hojas de nodos en orden inverso (usar con -sort).
+
+Hierarchy.sort:Campo de Metadata para clasificar. Use '-sort nosort' para no clasificar.
+
+
+Hierarchy.separator:ExpresiÃ³n regular utilizada por el separador
+
+
+Hierarchy.suppresslastlevel:Ignorar la parte final del valor del metadato. Es Ãºtil para metadatos en los cuales cada valor es Ãºnico, como en la rutas de archivos.
+
+
+HTML.desc:Crea una clasificaciÃ³n vacÃ­a que simplemente es un link a una pÃ¡gina web.
+
+
+HTML.url:El url de la pÃ¡gina web para unirse a.
+
+
+List.desc:El plugin clasificador de lista simple.
+
+
+List.metadata:Un solo campo de Metadatos o una lista de campos de Metadatos separados por una coma y usados con fines de clasificaciÃ³n. Siguiendo el orden indicado por la lista, se utilizarÃ¡ el primer campo que contiene un valor. La lista se ordenarÃ¡ por este elemento, a menos que estÃ© utilizando otro clasificador.
+
+
+List.sort:Campo de Metadata a partir del cual se harÃ¡ la ordenaciÃ³n. Use '-sort nosort' para no clasificar.
+
+
+Phind.desc:Plugin clasificador Phind.
+
+
+Phind.language:Idioma o idiomas utilizados para construir una jerarquÃ­a. Los idiomas son identificados por un cÃ³digo de paÃ­s de dos letras: en (inglÃ©s), es (espaÃ±ol), y fr (francÃ©s). El lenguaje es una expresiÃ³n normal, de modo que 'en|fr' (inglÃ©s o francÃ©s) y '.. ' (iguala cualquier idioma) son vÃ¡lidos.
+
+
+Phind.min_occurs:MÃ­nimo nÃºmero de veces que debe aparecer una frase en el texto para que sea incluida en la jerarquÃ­a de frases.
+
+
+Phind.savephrases:Si estÃ¡ determinado, la informaciÃ³n de la frase se guardarÃ¡ en el archivo como texto. Probablemente es una buena idea para utilizar una ruta absoluto.
+
+
+Phind.suffixmode:El parÃ¡metro smode para el programa de extracciÃ³n de frase. Un valor de 0 significa que los stopwords serÃ¡n ignorados, y 1 significa que los stopwords serÃ¡n utilizadas.
+
+
+Phind.text:El texto utilizado para construir la jerarquÃ­a de la frase.
+
+
+Phind.thesaurus:El nombre de un thesaurus guardado en formato Phind en el directorio etc de la colecciÃ³n.
+
+
+Phind.title:El campo del metadata para describir cada documento.
+
+
+Phind.untidy:No elimine los archivos activos.
+
+
+SectionList.desc:Igual que el clasificador de la lista pero incluye todas las secciones del documento (excluyendo el nivel superior) y no sÃ³lo el propio documento del nivel superior.
+
+Collage.buttonname:Etiqueta para clasificador de pantalla y botÃ³n en la barra de navegaciÃ³n.
+
+
+Collage.desc:El plugin clasificador del collage.
+
+
+Collage.geometry:Las dimensiones del lienzo del collage. Para un lienzo de 600 pixels de ancho y 400 pixels de alto, por ejemplo, especifique la geometrÃ­a como 600x400
+
+
+Collage.maxDepth:Las imÃ¡genes para el collage se dibujan mediante un espejo del clasificador de navegaciÃ³n subyacente. Esto controla la profundidad mÃ¡xima del proceso de espejo.
+
+
+Collage.maxDisplay:El mÃ¡ximo nÃºmero de imÃ¡genes a mostrar en el collage en cualquier momento.
+
+
+Collage.imageType:Utilizado para controlar, empleando las extensiones del nombre de archivo, cuales tipos de archivo se utilizan en el collage. Una lista de las extensiones del nombre de archivo se separa con el sÃ­mbolo (%%).
+
+
+Collage.bgcolor:El color de fondo del lienzo del collage, especificado en hexadecimal (por ejemplo, #008000 produce un fondo verde bosque).
+
+
+Collage.refreshDelay:Rata, en milisegundos, de refresco del lienzo del collage.
+
+
+Collage.isJava2:Utilizado para controlar cuales clases del run-time de Java se usan. Las versiones mÃ¡s avanzadas de Java (desde Java 1.2 en adelante) incluyen un soporte mas sofisticado para controlar la transparencia de las imÃ¡genes. Esta bandera ayuda a controlar los que pasa, sin embargo, el run-time de Java integrado con algunos navegadores es versiÃ³n 1.1. El applet estÃ¡ disÃ±ado, por defecto, para auto-detectar cual versiÃ³n de Java estÃ¡ utilizando el navegador y actuar en conformidad.
+
+
+Collage.imageMustNotHave:Utilizado para suprimir las imÃ¡genes que no deben aparecer en el collage, tales como las imÃ¡genes de los botones que forman la barra de navegaciÃ³n.
+
+
+Collage.caption:TÃ­tulos opcionales para desplegar bajo el lienzo del collage.
+
+
+#
+# Plugin option descriptions
+#
+
+ArcPlug.desc:Plugin que reproduce a travÃ©s de un archivo de archivos.inf file (es decir el archivo generado en el directorio de archivos cuando realiza una importaciÃ³n), procesando cada archivo que encuentra.
+
+
+BasPlug.adding:agregando
+
+
+BasPlug.already_seen:ya ha sido visto
+
+
+BasPlug.bad_general_option:El % del plugin utiliza una opciÃ³n general incorrecta (las opciones generales estÃ¡n disponibles a todos los plugins). Revise su archivo de configuraciÃ³n collect.cfg.
+
+
+BasPlug.block_exp:Los archivos que sean igual a esta expresiÃ³n regular serÃ¡n bloqueados al pasarse a cualquier plugins posterior en el listado. Esto no tiene otro efecto real de manera que previene muchos mensajes de alerta sobre los archivos input que a usted no le interesen. Cada plugin podrÃ­a tener un block_exp predefinido, por ejemplo por defaul HTMLPlug bloquea cualquier archivo gif. jpg. jpeg. png o los archivos css.
+
+
+BasPlug.associate_ext:Hace que los archivos cuyos nombres contienen la misma parte inicial que el documento procesado por el plugin Y una extensiÃ³n que figura en la lista separada por comas que provee este argumento, sean asociados con el documento que se estÃ¡ procesando en vez de ser manejados como una lista separada.
+
+BasPlug.could_not_extract_encoding:ALERTA: codificando no podrÃ­an extraerse de % s - predefiniendo a % s
+
+
+BasPlug.could_not_extract_language:ALERTA: el idioma no podrÃ­a extraerse de % s - predefiniendo a % s
+
+
+BasPlug.could_not_open_for_reading:no podrÃ¡ abrir % s ni leerlos
+
+
+BasPlug.no_cover_image:No buscar un archivo prefix.jpg (en el cual prefix es el mismo prefix que el del archivo que estÃ¡ siendo procesado) y asociarlo como imagen de portada.
+
+
+BasPlug.default_encoding:Utilice esta codificaciÃ³n si -input_encoding estÃ¡ puesto en âautoâ y el algoritmo de categorizaciÃ³n de texto no puede extraer el cÃ³digo o extrae un cÃ³digo que Greenstone no soporte. Esta opciÃ³n puede tomar los mismos valores que -input_encoding.
+
+
+BasPlug.default_language:Si Greenstone no identifica el idioma del documento el elemento del metadata serÃ¡ colocado a este valor. El valor predeterminado es el 'en' (ISO 639 los sÃ­mbolos del idioma son: el en = inglÃ©s). Nota que si - el input_encoding no estÃ¡ en 'automÃ¡tico' y - el extract_language no estÃ¡ programado, todos los documentos tendrÃ¡n su 'Idioma' el metadata estÃ¡ programado a este valor.
+
+
+BasPlug.desc:La clase base para todos los plugins de importaciÃ³n.
+
+
+BasPlug.done_acronym_extract:terminÃ³ de extraer las siglas.
+
+
+BasPlug.done_acronym_markup:TerminÃ³ el marcado de sigla (o acrÃ³nimo).
+
+
+BasPlug.done_email_extract:terminÃ³ de extraer las direcciones electrÃ³nicas.
+
+
+BasPlug.dummy_text:Este documento no tiene contenido.
+
+
+BasPlug.empty_file:el archivo no contiene ningÃºn texto
+
+
+BasPlug.extract_acronyms:Extrae las siglas desde dentro el texto y las coloca como metadata.
+
+
+BasPlug.extract_email:Extrae direcciones de correo electrÃ³nico como metadata.
+
+
+BasPlug.extract_historical_years:Extrae informaciÃ³n periÃ³dica de documentos histÃ³ricos. Esto se guarda como documento de metadata. Existe un interfaz de bÃºsqueda para este metadata que usted puede incluir en su colecciÃ³n agregando la frase, âformat QueryInterface DateSearchâ a su archivo de configuraciÃ³n colectivo.
+
+
+BasPlug.extract_language:Identifica el idioma de cada documento y establece el 'Idioma' del metadata. Note que esto se harÃ¡ automÃ¡ticamente si - el input_encoding esta en 'automÃ¡tico'.
+
+
+BasPlug.extracting:extrayendo
+
+
+BasPlug.extracting_acronyms:extrayendo siglas
+
+
+BasPlug.extract_keyphrases:Extraer automÃ¡ticamente frases clave con Kea (opciones por defecto).
+
+BasPlug.extract_keyphrase_options:Opciones para extracciÃ³n de frases clave con Kea. Por ejemplo: mALIWEB - utilizar el modelo de extracciÃ³n ALIWEB; n5 - extraer 5 frases clave;, eGBK - utilizar codificaciÃ³n GBK.
+
+BasPlug.extracting_emails:extrayendo direcciones electrÃ³nicas	
+
+
+BasPlug.file_has_no_text:ERROR: % s no contiene texto alguno
+
+
+BasPlug.first:La coma separa el listado de primeros tamaÃ±os para extraer del texto en un campo del metadata. El campo se llama 'FirstNNN'.
+
+
+BasPlug.input_encoding:La codificaciÃ³n de los documentos de la fuente. Los documentos se convertirÃ¡n de estas codificaciones y se guardarÃ¡n internamente como utf8.
+
+BasPlug.input_encoding.ascii:Conjunto de 7 bits ascii simples. Ãste puede resultar mÃ¡s rÃ¡pido que usando el iso_8859_1. Tenga cuidado con usar esto en una colecciÃ³n de documentos que pueden contener caracteres fuera de los 7 ascii simples (por ejemplo documentos en AlemÃ¡n o FrancÃ©s que contengan acentos), mejor utilice el iso_8859_1.
+
+BasPlug.input_encoding.auto:Use el algoritmo de categorizaciÃ³n de texto para identificar la codificaciÃ³n de cada fuente de documento automÃ¡ticamente. Esto serÃ¡ mÃ¡s lento que poniendo la codificaciÃ³n explÃ­citamente pero trabajarÃ¡ en donde mÃ¡s de una codificaciÃ³n se use dentro de la misma colecciÃ³n.
+
+BasPlug.input_encoding.unicode:SÃ³lo unicode.
+
+BasPlug.input_encoding.utf8:Ya sea Utf8 o unicodeâdetectada automÃ¡ticamente
+
+
+BasPlug.keyphrases:frases clave
+
+
+BasPlug.marking_up_acronyms:marcando siglas
+
+
+BasPlug.markup_acronyms:Agregue el metadato de sigla en el texto del documento.
+
+
+BasPlug.maximum_century:El mÃ¡ximo siglo a que se hace referencia que serÃ¡ extraÃ­do como metadata histÃ³rico (por ejemplo 14 extraerÃ¡ todas las referencias hasta el siglo14).
+
+
+BasPlug.maximum_year:La fecha histÃ³rica mÃ¡xima a ser usada como metadata (en una fecha referida a un periÃ³do comÃºn, como 1950).
+
+
+BasPlug.must_be_implemented:La funciÃ³n de BasPlug::read deberÃ¡ ser implementada en la subclase para los plugins recursivos
+
+
+BasPlug.no_bibliography:No intente bloquear las fechas bibliogrÃ¡ficas al extraer las fechas histÃ³ricas.
+
+
+BasPlug.process_exp:La expresiÃ³n perl regular para igualar contra nombres de archivos. Para igualar nombres de archivo se procesarÃ¡n por este plugin. Por ejemplo, usando '(? i). html? \ $' aparecerÃ¡n todos los documentos con terminaciÃ³n. htm o el .html (caso-insensible).
+
+
+BasPlug.read_denied:Lea permiso negado para % s
+
+
+BasPlug.smart_block:Ablocar los archivos en una forma mÃ¡s inteligente que simplemente examinando sus nombres.
+
+BasPlug.stems:tallos (inicios, bases)
+
+
+BasPlug.unsupported_encoding:ALERTA: % s aparece para ser codificado en un cÃ³digo sin apoyo (% s) â usando % s
+
+
+BasPlug.wrong_encoding:ALERTA: %s fue leÃ­do utilizando la codificaciÃ³n %s pero parece haber sido codificado como % s.
+
+
+BibTexPlug.desc:BibTexPlug lee los archivos de bibliografÃ­a en el formato de BibTex. BibTexPlug crea un objeto del documento para cada referencia en el archivo. Es una subclase de SplitPlug, por lo que si hay archivos mÃºltiples, todos se leen.
+
+
+BookPlug.desc:Crea documentos multi-nivel a partir de documentos que contengan etiquetas de nivel <<TOC>>. El meta-dato para cada secciÃ³n se toma de cualquier otra etiqueta que estÃ© en la misma lÃ­nea que el << TOC >>, por ejemplo, <<Title>> xxxx <</Title>> establece el meta-dato Title. Todo lo demÃ¡s que aparezca entre las etiquetas TOC se tratarÃ¡ como html simple (es decir, no se establece ningÃºn proceso de vÃ­nculos html o cualquier otro tipo HTMLPlug). Espera que los archivos de entrada por defecto tengan una extensiÃ³n .hb (esto puede cambiarse agregando una opciÃ³n -process_exp), un archivo con el mismo nombre que el archivo hb pero con una extensiÃ³n .jpg se toma como la imagen de la portada (los archivos jpg son bloqueados por este plugin). BookPlug es una simplificaciÃ³n (y extensiÃ³n) del HBPlug utilizado por las colecciones de la Biblioteca de la Humanidad. BookPlug es mÃ¡s rÃ¡pido en tanto que  espera la entrada de archivos mÃ¡s limpios (La entrada a las colecciones de HDL contiene muchas etiquetas html adicionales alrededor de etiquetas << TOC >>, utiliza etiquetas << I >> para especificar las imÃ¡genes, y simplemente toma como meta-datos del TÃ­tulo todo el texto entre las etiquetas << TOC >> y el inicio del texto). Si usted estÃ¡ colocando mark-up en documentos para que sean desplegados de la misma manera que las colecciones HDL, use este plugin en lugar de HBPlug.
+
+
+ConvertToPlug.convert_to:Plugin convierte a TEXTO o HTML.
+
+ConvertToPlug.convert_to.html:Formato HTML
+
+ConvertToPlug.convert_to.text:Formato texto simple.
+
+
+ConvertToPlug.desc:El plugin se hereda por dichos plugins como WordPlug y PDFPlug. Facilita la conversiÃ³n de Ã©stos tipos de documentos ya sea HTML o TEXTO variable que le dice a ConvertToBasPlug cÃ³mo trabajar. Funciona heredando HTMLPlug o TEXTPlug dinÃ¡micamente basado en el argumento del plugin 'convert_to'. Si el argumento no estÃ¡ presente, el valor predeterminado es heredar HTMLPlug.
+
+
+ConvertToPlug.use_strings:Si estÃ¡ definido, una funciÃ³n de cadenas simple serÃ¡ llamada para extraer el texto si la conversiÃ³n de utilidad falla.
+
+
+ConvertToRogPlug.desc:Un plugin que hereda de RogPlug.
+
+
+DBPlug.desc:Usa de archivos desde una base de datos como documentos.
+
+
+DBPlug.title_sub:La substituciÃ³n de la expresiÃ³n para modificar cordÃ³n guardado como TÃ­tulo. Por ejemplo, utilizado por PSPlug para quitar PÃ¡gina 1 etc de texto usado como tÃ­tulo.
+
+
+EMAILPlug.desc:El lector del correo electrÃ³nico lee los archivos electrÃ³nicos.  Ãstos se nombran con un nÃºmero simple (es decir cuando ellos aparecen en las carpetas del maildir) o con la extensiÃ³n. mbx (para el mbox mande por correo el formato del archivo). el texto del \nDocument text: El texto del documento consiste en todo el texto despuÃ©s de la primera lÃ­nea en blanco en el documento. el \nMetadata (not DublÃ­n Core!):\n\t\$Headers Todo el contenido del Asunto content (opcional, no se almacenan si no se indica)\n\t\$Subject:  header\n\t\$To A: el header\n\t\$From De: los header\n\t\$FromName Nombran al remitente (donde los available)\n\t\$FromAddr Mandan electrÃ³nicamente a la direcciÃ³n de Fecha del sender\n\t\$DateText: los header\n\t\$Date Fecha: el tÃ­tulo en GSDL estructura (ejemplo: 19990924)
+
+
+EMAILPlug.no_attachments:No guarde los anexos del mensaje.
+
+
+EMAILPlug.headers:Almacenar encabezados de email como metadatos "Headers".
+
+
+EMAILPlug.split_exp:Un expresiÃ³n perl regular se utiliza para separar archivos que contengan muchos mensajes en documentos individuales.
+
+
+ExcelPlug.desc:Un plugin para importador los archivos de Microsoft Excel.
+
+
+FOXPlug.desc:Plugin para procesar un archivo Foxbase dbt. Este plugin proporciona la funcionalidad bÃ¡sica para leer los archivos dbt y dbf y procesa cada registro. Este plugin generales debe doblemente leÃ­do para una base de datos particular para procesar los campos apropiados en el archivo.
+
+
+GAPlug.desc:Procesa los documentos XML del GreenstoneArchive. Note que este plugin no realiza ninguna sintaxis verificativa (aunque el mÃ³dulo de XML::Parser prueba para buen-formateo). Se asume que los archivos de GreenstoneArchive conforman a su DTD.
+
+
+GISBasPlug.extract_placenames:Extraer del texto nombres de lugares y establecerlos como metadata. Requiere extensiÃ³n de Greenstone a GIS.
+
+GISBasPlug.gazetteer:Mirar para extrarer del texto nombres de lugares y establecerlos como metadata. Requiere extensiÃ³n de Greenstone a GIS.
+
+GISBasPlug.place_list:Al extractar ubicaciones, incluya la lista de nombres de lugares al comienzo del documento. Requiere extensiÃ³n de Greenstone a GIS.
+
+GMLPlug.desc:Plugin que procesa un formato en documento GML asume que las etiquetas gml estÃ¡n todos escritas en letras minÃºsculas.
+
+
+HBPlug.desc: Plugin que procesa un directorio de libro HTML. Este plugin se usa por las colecciones de la Biblioteca de Humanidades y no se ocupa de codificaciones de la entrada mas que de ascii o el ascii extendido. Este cÃ³digo es algo pesado y podrÃ­a hacerse correr mÃ¡s rÃ¡pidamente indudablemente, dejÃ¡ndolo en este estado espero animar a las personas para hagan sus colecciones utilizando el HBSPlug en cambio ;-)\n\nUse utilice HBSPlug si estÃ¡ creando una nueva colecciÃ³n y marcando a los archivos como las colecciones de la Biblioteca de Humanidades. HBSPlug acepta todas las codificaciones de entrada pero espera los archivos marcados sean mas limpios que aquellos usados por las colecciones de la Biblioteca de Humanidades
+
+
+HTMLPlug.assoc_files:ExpresiÃ³n Perl normal de las extensiones de archivo para asociar con documentos html.
+
+
+HTMLPlug.desc:Este plugin procesa archivos HTML
+
+
+HTMLPlug.description_tags:Divide el documento en las sub-secciones dÃ³nde aparecen las etiquetas de <SecciÃ³n>. Note que al utilizar esta opciÃ³n usted implÃ­citamente fija -no_metadata, cuando todos los metadata deben ser incluidos dentro de las etiquetas <SecciÃ³n>. TambiÃ©n, '-keep_head' no surtirÃ¡ efecto cuando esta opciÃ³n estÃ¡ definida.
+
+
+HTMLPlug.file_is_url:Utilice y ponga si los nombres de los archivos de la entrada constituyen el url de los documentos fuente originales, por ejemplo si una herramienta para espejos web fuera utilizada para crear la estructura del directorio de importaciÃ³n.
+
+
+HTMLPlug.hunt_creator_metadata:Localiza tantos metadatos como sea posible en autor y los coloca en el campo 'Creator'. Requiere la bandera de -metadata_fields (campos de metadata)
+
+
+HTMLPlug.keep_head:No elimine los tÃ­tulos de los archivos html.
+
+
+HTMLPlug.metadata_fields:Listado de campos de metadata separado por coma para intentar extraer. Use 'tag<tagname>' para tener los contenidos del primer par <tagname> en un elemento del metadata llamado 'tagname' (nombre de etiqueta). Ponga en mayÃºsculas esto si desea el metadata en mayÃºsculas en Greenstone, ya que la extracciÃ³n de la etiqueta no tiene la sensibilidad para hacerlo.
+
+
+HTMLPlug.no_metadata:No intente extraer metadata de los archivos.
+
+
+HTMLPlug.no_strip_metadata_html:Lista de nombres de metadatos separados por comas o 'all'. Utilizado con -description_tags, evita la supresiÃ³n de las etiquetas HTML de los valores de los metadatos especificados.
+
+
+HTMLPlug.nolinks:No se esfuerce por tomar links (poniendo esta bandera puede mejorar la velocidad para crear/importar pero cualquier link relativo dentro de los documentos se romperÃ¡).
+
+
+HTMLPlug.rename_assoc_files:Renombra archivos asociados con documentos (por ejemplo imÃ¡genes). TambiÃ©n crea mucha estructura de directorios de poco peso (Ãºtil al crear las colecciones cd-rom).
+
+
+HTMLPlug.title_sub:ExpresiÃ³n sustituta para modificar una cadena almacenada como TÃ­tulo. Usada, por ejemplo, en PDFPlug para eliminar "PÃ¡gina 1", etc de un texto usado como tÃ­tulo.
+
+
+ImagePlug.converttotype:Convertir la imagen principal al formato 's'.
+
+
+ImagePlug.desc:Este plugin procesa las imÃ¡genes, agregando el metadata bÃ¡sico
+
+
+ImagePlug.minimumsize:Ignore las imÃ¡genes menores que n bytes.
+
+
+ImagePlug.noscaleup:No sobreponga las imÃ¡genes pequeÃ±as al hacer thumbnails .
+
+
+ImagePlug.screenviewsize:Si estÃ¡ fijo, hace un imagen de tamaÃ±o n para el despliegue de la pantalla y fija el metadato de Pantalla, ScreenSize, (tamaÃ±o de pantalla) ScreenWidth (ancho de Pantalla) y ScreenHeight (altura de pantalla). No estÃ¡ predeterminado por defaul.
+
+
+ImagePlug.screenviewtype:Si - el screenviewsize (tamaÃ±o de lector de pantalla) estÃ¡ establecido, Ã©ste coloca el despliegue de imagen.
+
+
+ImagePlug.thumbnailsize:Realiza iconos pequeÃ±os tamaÃ±o nxn.
+
+
+ImagePlug.thumbnailtype:Realiza iconos pequeÃ±os en el formato 's'.
+
+
+IndexPlug.desc:Este plugin recursivo procesa un archivo index.txt. El archivo index.txt debe contener la lista de archivos que deberÃ¡n ser incluidos en la colecciÃ³n, seguida por cualesquiera meta-datos extras a ser asociados con cada archivo.\n\n El archivo index.txt debe estructurarse de la siguiente manera: La primera lÃ­nea puede ser una llave (empezando con key:)  para nombrar los archivos de meta-datos (por ejemplo, key: Asunto OrganizaciÃ³n Fecha). Las lÃ­neas siguientes contendrÃ¡n un nombre de archivo seguido por el valor que se asignarÃ¡ a la entrada del meta-dato. (por ejemplo, 'irma/iw097e 3.2 unesco 1993' asociarÃ¡ el meta-dato Asunto=3.2, OrganizaciÃ³n=unesco, y Fecha=1993 con el archivo irma/iw097e si se utilizÃ³ la lÃ­nea anterior)\n\n Note que si cualquiera de los campos de los meta-datos utiliza el plugin de clasificador de JerarquÃ­a, entonces el valor de ellos debe corresponder al primer campo (el descriptor) en el archivo de la clasificaciÃ³n apropiado.\n\nLos valores de los Meta-datos pueden nombrarse utilizando una etiqueta de manera separada (por ejemplo <Asunto>3.2) y esto prevalecerÃ¡ sobre cualquier nombre que se les haya asignado en la lÃ­nea que contiene la "key". Si no hay ninguna lÃ­nea que contenga una "key", cualquier valor sin nombre de los meta-datos se denominarÃ¡ 'Asunto'.
+
+
+ISISPlug.desc:Este plugin procesa las bases de datos CDS/ISIS. Por cada base de datos CDS/ISIS procesada deberÃ¡n existir tres archivos en la carpeta de importaciÃ³n de la colecciÃ³n: el archivo Maestro (.mst), la Tabla de DefiniciÃ³n de Campos (.fdt) y el Archivo de Referencias Cruzadas (.xrf)
+
+
+ISISPlug.subfield_separator:La cadena de caracteres que se usa para separar subcampos en los registros de la base de datos CDS/ISIS.
+
+
+ISISPlug.entry_separator:La cadena de caracteres que se usa para separar mÃºltiples valores en los campos de metadatos de los registros de la base de datos CDS/ISIS.
+
+
+LaTeXPlug.desc:Plugin para documentos LaTeX.
+
+
+MACROPlug.desc:Crea el documento simple-nivelado. Agrega TÃ­tulo de metadata en primera lÃ­nea de texto (hasta 100 caracteres).
+
+
+MARCPlug.desc:Plugin MARC bÃ¡sico.
+
+
+MARCPlug.metadata_mapping:Nombre de archivo que incluye la cartografÃ­a detallada de valores MARC para los nombres de los metadatos de Greenstone. Predetermina a 'marctodc.txt' encontrado en el directorio etc del sitio.
+
+
+MetadataPass.desc:Clase base paralela a BasPlug que soporta plugins de metadata y usa el paso metadata_read de import.pl
+
+GISBasPlug.desc:Clase base paralela a BasPlug que soporta capacidades de GIS
+
+NULPlug.desc:Plugin de archivo postizo (.nul). Utilizado con los archivos producidos al explotar los archivos de la base de datos de metadatos.
+
+OAIPlug.desc:Plugin Inicio de Archivos Abiertos BÃ¡sicos (OAI)
+
+
+OggVorbisPlug.add_technical_metadata:AÃ±adir metadatos tÃ©cnicos (ej. bitrate)
+
+
+OggVorbisPlug.desc:Plugin para importar archivos de audio Ogg Vorbis.
+
+
+PagedImgPlug.desc:El plugin para documentos consiste de secuencias de imÃ¡genes, con texto opcional de OCR para cada imagen.
+
+
+PagedImgPlug.documenttype:Establecer el tipo de documento (usado para despliegue)
+
+PagedImgPlug.documenttype.paged:Los documentos paginados tienen flechas "siguiente" y "anterior", asÃ­ como una opciÃ³n para "ir a la pÃ¡gina X"
+
+PagedImgPlug.documenttype.hierarchy:Los documentos jerÃ¡rquicos tienen una tabla de contenido
+
+
+PagedImgPlug.headerpage:AÃ±adir una cabecera de pÃ¡gina de nivel principal (que no contenga imagen) a cada documento.
+
+
+PagedImgPlug.screenview:Producir una imagen "screenview"  para cada imagen, asÃ­ como metadatos Screen, ScreenSize, ScreenWidth y ScreenHeight.
+
+
+PagedImgPlug.screenviewsize:Producir imÃ¡genes "screenview" de tamaÃ±o nxn.
+
+
+PagedImgPlug.screenviewtype:Producir imÃ¡genes "screenview" en formato 's'.
+
+
+PagedImgPlug.thumbnail:Producir una vista en miniatura de cada imagen
+
+
+PDFPlug.complex:Crea un output mÃ¡s complejo. Con esta opciÃ³n fijada los html resultantes se parecerÃ¡n mucho mÃ¡s al archivo PDF original. Para el apropiado funcionamiento Ghostscript instalado (para *nix gs deben estar en su trazado mientras que para windows se debe tener gswin32c.exe).
+
+
+PDFPlug.desc:Razonablemente con un plugin pdf.
+
+
+PDFPlug.nohidden:Prevenga al pdftohtml de intentar extraer texto oculto. Esto sÃ³lo es Ãºtil si la opciÃ³n -complex estÃ¡ determinada.
+
+
+PDFPlug.noimages:No intente extraer las imÃ¡genes de PDF.
+
+
+PDFPlug.use_sections:Cree una secciÃ³n separada para cada pÃ¡gina del archivo de PDF.
+
+
+PDFPlug.zoom:El factor por el cual se hace acercamiento al PDF para la salida (esto sÃ³lo es Ãºtil si el -complex esta determinado).
+
+
+PPTPlug.desc:Un plugin para importar archivos de PowerPoint Microsoft.
+
+
+PSPlug.desc:Este es un ps \"poor manâs\" para convertidor de texto. Si si plan es serio, considere utilizar el paquete PRESCRIPT, para bajarlo lo podrÃ¡ encontrar en http://www.nzdl.org/html/software.html
+
+
+PSPlug.extract_date:Extrae la fecha del encabezado de PS.
+
+
+PSPlug.extract_pages:Extrae las pÃ¡ginas del encabezado de PS.
+
+
+PSPlug.extract_title:Extrae el encabezado de PS inicial.
+
+
+RecPlug.desc:RecPlug es un plugin que recorre a travÃ©s de directorios procesando cada archivo encontrado.
+
+
+RecPlug.recheck_directories:DespuÃ©s de procesar los archivos en un directorio de importaciÃ³n, vuelva a leer el directorio para descubrir los nuevos archivos creados.
+
+
+RecPlug.use_metadata_files:Lea el metadato archivos de XML metadata.
+
+
+ReferPlug.desc:ReferPlug lee archivos bibliogrÃ¡ficos en formato Refer.
+
+
+ReferPlug.longdesc:ReferPlug lee archivos bibliogrÃ¡ficos en formato Refer.\nBy Gordon W. Paynter (gwp\@cs.waikato.ac.nz), Noviembre 2000\n\nLoosely basado en el hcibib2Plug por Steve Jones (stevej\@cs.waikato.ac.nz). Que a su vez estaba basado en EMAILPlug por Gordon Paynter (gwp\@cs.waikato.ac.nz). Basado en las versiones anteriores de HTMLplug y HCIBIBPlugby por Stefan Boddie y otros--es difÃ­cil decir quÃ© viene de dÃ³nde, ahora .\n\nReferPlug crea un objeto del documento para cada referencia en el archivo. Es una subclase de SplitPlug, por lo que si hay archivos mÃºltiples, todos se leen .\n\nDocument text:\n\t El texto del documento consiste en la referencia en Refer format .\nMetadata:\n\t\$Creator \% A Autor name\n\t\$Title del Autor \% T Titulo de artÃ­culo de book\n\t\$Journal \%J Titulo de Journal\n\t\$Booktitle \%B Titulo de libro que contiene publicaciÃ³n publication\n\t\$Report \%R Tipo de Informe, ponencia o tesis thesis\n\t\$Volume \%V NÃºmero de Volumen Journal\n\t\$Number \%N NÃºmero de Revista dentro de volumen Volume\n\t\$Editor \%E Nombre del Editor name\n\t\$Pages \%P NÃºmero de hojas en artÃ­culo de article\n\t\$Publisher \%I Nombre de editor Publisher\n\t\$Publisheraddr \% address\n\t\$Fecha de PublicaciÃ³n C \% D fecha de publicaciÃ³n publication\n\t\$Keywords \% las Palabras claves de la publicaciÃ³n\n\t\$Abstract \%X Abstract de la publicaciÃ³n\n\t\$Copyright\t \% * informaciÃ³n de derechos de propiedad literaria para el artÃ­culo
+
+
+RogPlug.desc:Crea los documentos simples single-level (de nivel Ãºnico) a partir de archivos .rog o .mdb.
+
+
+RTFPlug.desc:Plugin para importar archivos de Formato Rich Text.
+
+
+SRCPlug.desc:Nombre de Archivo utilizado actualmente para TÃ­tulo (opcionalmente menos algÃºn prefijo). Languages actuales:\ntextl: READMEs/Makefiles\nC/C++ (actualmente los extractos #incluye las declaraciones y clase C++ decls)\nPerl (actualmente sÃ³lo elaborado como texto)\nShell (actualmente sÃ³lo elaborado como texto)
+
+
+SRCPlug.remove_prefix:Elimine este modelo principal del nombre del archivo (eg - remove_prefix /tmp/XX/src/). El valor predeterminado es eliminar la ruta completa del nombre del archivo.
+
+
+SplitPlug.desc:SplitPlug es un plugin para los archivos de entrada que divide en segmentos que se procesarÃ¡n de manera individual. Este plugin no debe llamarse de manera directa.  En cambio, si usted necesita procesar archivos input que contienen varios documentos, debe escribir un plugin con una funciÃ³n de proceso que manejarÃ¡ uno de esos documentos habiÃ©ndolo tomado de SplitPlug. Vea ReferPlug para un ejemplo.
+
+
+SplitPlug.split_exp:La expresiÃ³n perl regular para separar la entrada de archivos en segmentos.
+
+
+TEXTPlug.desc:Crea un documento simple de nivel Ãºnico (single-level). Agrega el metadato TÃ­tulo de la primera lÃ­nea de texto (hasta 100 caracteres).
+
+
+TEXTPlug.title_sub:ExpresiÃ³n de substituciÃ³n para modificar la cadena guardada como TÃ­tulo. Por ejemplo, PSPlug para eliminar "PÃ¡gina 1" etc del texto utilizado como tÃ­tulo.
+
+
+UnknownPlug.assoc_field:El nombre del campo del metadata que sostendrÃ¡ el nombre del archivo asociado.
+
+
+UnknownPlug.desc:Ãste es un Plugin simple para los archivos importados en los formatos que Greenstone no tiene. Un documento de ficciÃ³n serÃ¡ creado para cada uno de esos archivos, y el propio archivo se pasarÃ¡ a Greenstone como archivo asociado \âassociated file\â del documento.
+
+
+UnknownPlug.file_format:Tipo de archivo (ej: MPEG, MIDI, âŠ)
+
+
+UnknownPlug.mime_type:Tipo mime del archivo (por ejemplo image/gif).
+
+
+UnknownPlug.process_extension:Procesar archivos con esta extensiÃ³n de archivo. Esta oopciÃ³n es una alternativa a process_exp mÃ¡s sencilla de usar pero menos flexible.
+
+
+MP3Plug.desc:Plugin para procesar archivos MP3.
+
+
+MP3Plug.assoc_images:Use google image search to locate images related to MP3 file based on ID3 Title and Artist metadata.
+
+MP3Plug.assoc_images:Usado para almacenar [applet] metadatos para cada documento que contiene el HTML necesario para que un applet de audio MP3 pueda reproducir ese archivo.
+
+
+MP3Plug.metadata_fields:Lista de campos de metadatos separados por comas, a extraer (asumiendo los existentes) de un archivo MP3. Use \"*\" para extraer todos los campos.
+
+
+W3ImgPlug.aggressiveness:El rango de tÃ©cnicas de extracciÃ³n de textos relacionados a utilizar.
+
+W3ImgPlug.aggressiveness.1:Nombre de Archivo, ruta, ALT sÃ³lo texto.
+
+W3ImgPlug.aggressiveness.2:Todos de 1, mÃ¡s el subtÃ­tulo si hubiese disponible.
+
+W3ImgPlug.aggressiveness.3:Todos de 2, mÃ¡s los pÃ¡rrafos disponibles.
+
+W3ImgPlug.aggressiveness.4:Todos de 3, mÃ¡s los encabezados anteriores (<h1>, <h2>...) disponibles.
+
+W3ImgPlug.aggressiveness.5:Todos de 4, mÃ¡s las referencias textuales disponibles.
+
+W3ImgPlug.aggressiveness.6:Todos de 4, mÃ¡s la metaetiqueta de la pÃ¡gina (tÃ­tulo, las palabras clave, etc).
+
+W3ImgPlug.aggressiveness.7:Todos de 6, 5 y 4 combinados.
+
+W3ImgPlug.aggressiveness.8:Todos de 7, mÃ¡s la repeticiÃ³n de subtÃ­tulo, el nombre del archivo, etc (la clasificaciÃ³n jerÃ¡rquica de resultados mÃ¡s relevantes).
+
+W3ImgPlug.aggressiveness.9:Todos de 1, mÃ¡s el texto completo de la pÃ¡gina fuente.
+
+
+W3ImgPlug.caption_length:La longitud mÃ¡xima de subtÃ­tulos (en caracteres).
+
+
+W3ImgPlug.convert_params:ParÃ¡metros adicionales para la que ImageMagicK convierta el Ã­cono. Por ejemplo, '-raise' darÃ¡ un efecto tridimensional a las imÃ¡genes en miniatura.
+
+
+W3ImgPlug.desc:Plugin para extraer imÃ¡genes y el texto asociado de pÃ¡ginas web.
+
+
+W3ImgPlug.document_text:Agrega el texto de imagen como document:text (por otra parte los metadata de IndexedText se presentan).
+
+
+W3ImgPlug.index_pages:Indice las pÃ¡ginas junto con las imÃ¡genes. Por otra parte referencia las pÃ¡ginas a la fuente URL.
+
+
+W3ImgPlug.max_near_text:Caracteres mÃ¡ximos cerca de las imÃ¡genes a extraer.
+
+
+W3ImgPlug.min_height:Pixeles. Salta las imÃ¡genes mÃ¡s pequeÃ±as que Ã©sta.
+
+
+W3ImgPlug.min_near_text:Caracteres mÃ­nimos cerca del texto o subtitulo para extraer.
+
+
+W3ImgPlug.min_size:Bytes. Brinca las imÃ¡genes mÃ¡s pequeÃ±as que estas.
+
+
+W3ImgPlug.min_width:Pixeles. Salta imÃ¡genes mas breves que Ã©sta.
+
+
+W3ImgPlug.neartext_length:Longitud designada del texto mas cercano (en caracteres)
+
+
+W3ImgPlug.no_cache_images:No esconda las imÃ¡genes (apunte al URL original).
+
+
+W3ImgPlug.smallpage_threshold:ImÃ¡genes en las pÃ¡ginas mÃ¡s pequeÃ±as que esto (los bytes) tendrÃ¡n la pÃ¡gina (el tÃ­tulo, las palabras claves, etc) con meta-datos agregados.
+
+
+W3ImgPlug.textrefs_threshold:El umbral para las referencias textuales. Los valores mÃ¡s bajos significan que el algoritmo es menos estricto.
+
+
+W3ImgPlug.thumb_size:Max el tamaÃ±o en miniatura. Ambos ancho y altura.
+
+
+WordPlug.desc:Plugin importador para los documentos de Microsoft Word.
+
+
+XMLPlug.desc:La clase base para plugins XML.
+
+
+ZIPPlug.desc:Plugin que se ocupa de los formatos comprimidos y/o archivados de entrada los formatos actualmente manejados y las extensiones de archivo son are:\ngzip de (.gz. z. el tgz. taz)\nbzip (.bz)\nbzip2 (.bz2)\nzip (.zip. .jar)\ntar\n\n Este plugin confÃ­a en las utilerÃ­as siguientes que estÃ¡n presentes (si estÃ¡ intentando procesar formatos correspondientes) :\ngunzip (para gzip)\nbunzip (para bzip)\nbunzip2 \nunzip (para zip)\ntar (para tar)
+
+
+
+#
+# Perl module strings
+#
+
+classify.could_not_find_classifier:ERROR: no encontrÃ³ el clasificador \â% s\â
+
+
+plugin.could_not_find_plugin:ERROR: No podrÃ¡ encontrar el plugin \"% s\"
+
+
+plugin.including_archive:inclusive los volÃºmenes de 1 archivo de ZIP/TAR
+
+
+plugin.including_archives:inclusive los volÃºmenes de %d los archivos de ZIP/TAR
+
+
+plugin.kill_file:Proceso finiquitados por el archivo .kill
+
+
+plugin.n_considered:%d de los documentos fueron considerados a efecto de ser procesados
+
+
+plugin.n_included:%d se procesaron e incluidos en la colecciÃ³n
+
+
+plugin.n_rejected:%d rechazados
+
+
+plugin.n_unrecognised:%d no fueron reconocidos
+
+
+plugin.no_plugin_could_process:ADVIRTIENDO: NingÃºn plugin podrÃ¡ ser procesado %s
+
+
+plugin.no_plugin_could_recognise:ATENCIÃN: NingÃºn plugin pudo reconocer %s
+
+
+plugin.no_plugin_could_process_this_file:ningÃºn plugin podrÃ¡ procesar este archivo
+
+
+plugin.no_plugin_could_recognise_this_file:ningÃºn plugin pudo reconocer este archivo
+
+
+plugin.one_considered:1 documento fue considerado a efecto de ser procesado
+
+
+plugin.one_included:1 fue procesado e incluido en la colecciÃ³n
+
+
+plugin.one_rejected:1 fue rechazado
+
+
+plugin.one_unrecognised:1 no fue reconocido
+
+
+plugin.see_faillog:Vea en %s una lista de documentos no reconocidos y/o rechazados
+
+
+PrintUsage.default:Valor predeterminado (default)
+
+
+PrintUsage.required:REQUERIDO
+
Index: /main/tags/3.01/gsdl/perllib/strings_fr.rb
===================================================================
--- /main/tags/3.01/gsdl/perllib/strings_fr.rb	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/strings_fr.rb	(revision 21112)
@@ -0,0 +1,1269 @@
+#
+# Resource bundle description
+#
+
+Language.code:fr
+
+Language.name:FranÃ§ais
+
+
+OutputEncoding.unix:iso_8859_1
+
+OutputEncoding.windows:iso_8859_1
+
+
+
+#
+# Common output messages
+#
+
+common.cannot_create_file:ERREUR: impossible de crÃ©er le fichier %s
+
+
+common.cannot_find_cfg_file:ERREUR: fichier de configuration introuvable %s
+
+
+common.cannot_open:ERREUR: impossible d'ouvrir %s
+
+
+common.cannot_open_fail_log:ERREUR: Impossible d'ouvrir le log d'Ã©chec %s
+
+
+common.cannot_open_output_file:ERREUR: impossible d'ouvrir le fichier output %s
+
+
+common.cannot_read:ERREUR: impossible de lire %s
+
+
+common.cannot_read_file:ERREUR: impossible de lire le fichier %s
+
+
+common.general_options:options gÃ©nÃ©rales (pour %s)
+
+
+common.must_be_implemented:fonction doit Ãªtre implÃ©mentÃ©e dans sous-classe
+
+
+common.options:options
+
+
+common.processing:traitement
+
+
+common.specific_options:options spÃ©cifiques
+
+
+common.usage:utilisation
+
+
+
+#
+# Script option descriptions and output messages
+#
+
+scripts.language:Langue dans laquelle les descriptions d'option sont affichÃ©es (ex. 'en_US' spÃ©cifie l'anglais amÃ©ricain). NÃ©cessite des traductions des descriptions d'options qui sont prÃ©sentes dans le fichier perllib/strings_language-code.rb.
+
+
+scripts.xml:Produit l'information sous forme XML, sans 'jolis' commentaires mais avec beaucoup plus de dÃ©tails.
+
+
+scripts.listall:Liste tous les Ã©lÃ©ments connus.
+
+
+
+# -- buildcol.pl --
+
+buildcol.archivedir:LÃ  oÃ¹ se trouvent les archives.
+
+
+buildcol.builddir:LÃ  oÃ¹ mettre les indexes construits.
+
+
+buildcol.cachedir:La collection sera temporairement construite ici avant d'Ãªtre copiÃ©e dans le le rÃ©pertoire de construction.
+
+
+buildcol.cannot_open_cfg_file:ATTENTION: Impossible d'ouvrir le fichier de configuration pour mise Ã  jour: %s
+
+
+buildcol.collectdir:RÃ©pertoire de la collection.
+
+
+buildcol.copying_back_cached_build:Recopie de la construction mise en cache
+
+
+buildcol.create_images:Tente de crÃ©er des images par dÃ©faut pour la nouvelle collection. Cela dÃ©pend de l'installation de Gimp en mÃªme temps que les modules qui vont avec pour permettre l'Ã©criture de scripts Ã  partir de perl.
+
+
+buildcol.debug:Imprimer la sortie sur STDOUT.
+
+
+buildcol.desc:Script PERL utilisÃ© pour bÃ¢tir une collection Greenstone Ã  pertir de documents GML.
+
+
+buildcol.faillog:Nom de fichier log d'Ã©chec. Ce fichier log reÃ§oit les noms de fichiers de tous les fichiers qu'on n'a pas pu traiter.
+
+
+buildcol.index:Index Ã  construire (va construire toust dans le fichier de configuration si pas dÃ©fini).
+
+
+buildcol.keepold:Ne dÃ©truira pas les contenus actuels du rÃ©pertoire de construction.
+
+
+buildcol.maxdocs:Nombre maximum de documents Ã  construire.
+
+
+buildcol.mode:Les parties du processus de construction Ã  mener.
+
+buildcol.mode.all:Tout faire.
+
+buildcol.mode.build_index:Indexer le texte uniquement.
+
+buildcol.mode.compress_text:Compresser le texte uniquement.
+
+buildcol.mode.infodb:Construire la base de donnÃ©es des mÃ©ta-donnÃ©es uniquement.
+
+
+buildcol.no_default_images:Des images par dÃ©faut ne seront gÃ©nÃ©rÃ©es.
+
+
+buildcol.no_image_script:ATTENTION: script de fabrication d'images introuvable: %s
+
+
+buildcol.no_strip_html:Ne pas enlever les balises html du texte indexÃ© (utilisÃ© seulement pour les collections mgpp).
+
+
+buildcol.no_text:Ne pas stocker de texte compressÃ©. Cette option est utile pour la rÃ©duction de la taille des indexes incorporÃ©s si vous voulez toujours afficher les documents originaux en temps voulu (c'est Ã  dire que vous ne pouvez pas retrouver la version du texte compressÃ©).
+
+
+buildcol.out:Nom de fichier ou de contrÃŽle vers lequel envoyer le statut de sortie.
+
+
+buildcol.params:[options] nom de collection
+
+
+buildcol.remove_empty_classifications:Cacher les nÅuds de classeurs et classification (ceux qui sont vides, sans documents)
+
+
+buildcol.unlinked_col_images:Les images de la collection peuvent ne pas Ãªtre correctement Ãªtre liÃ©es.
+
+
+buildcol.unknown_mode:Mode inconnu: %s
+
+
+buildcol.updating_archive_cache:Mise Ã  jour du cache d'archive
+
+
+buildcol.verbosity:ContrÃŽle la quantitÃ© de sortie. 0=aucun, 3= beaucoup.
+
+
+
+# -- classinfo.pl --
+
+classinfo.collect:En donnant un nom Ã  la collection fera que classinfo.pl va regarder d'abord dans collect/nom-de-collection/perlib/classify. Si le classificateur n'est pas trouvÃ©, il va alors chercher dans le rÃ©pertoire gÃ©nÃ©ral perlib/classify.
+
+
+classinfo.desc:Impime l'information sur un classificateur.
+
+
+classinfo.general_options:Options gÃ©nÃ©rales hÃ©ritÃ©es de classes parentes du classificateur.
+
+
+classinfo.info:info
+
+
+classinfo.no_classifier_name:ERREUR: Vous devez donner un nom de classificateur.
+
+
+classinfo.option_types:Les classificateurs peuvent prendre deux types d'options
+
+
+classinfo.params:[options] nom de classificateur
+
+
+classinfo.passing_options:Les options peuvent Ãªtre passÃ©es Ã  tout classificateur en les intÃ©grant au fichier de configuration de votre collection collect.cfg.
+
+
+classinfo.specific_options:Les options spÃ©cifiques sont dÃ©finies dans le classificateur lui mÃªme, et sont disponibles seulement pour ce classificateur particulier.
+
+
+# -- explode_metadata_database.pl --
+explode.desc:Ãclater la base de donnÃ©es mÃ©ta-donnÃ©es.
+
+explode.document_field:L'Ã©lÃ©ment de metadata indiquant le nom de fichier des documents Ã  obtenir et inclure dans la collection.
+
+
+explode.document_prefix:PrÃ©fixe pour localiser les documents (s'utilise avec l'option "document_field").
+
+
+explode.document_suffix:Suffixe pour localiser les documents (s'utilise avec l'option "document_field").
+
+
+explode.encoding:Encodage Ã  utiliser lors de la lecture de la base de donnÃ©es
+explode.filename_field:Champ Ã  utiliser pour les noms de fichier factices (null) gÃ©nÃ©rÃ©s (si pas de document_field ou ce champ est vide)
+explode.metadata_set:Jeu mÃ©ta-donnÃ©es (espace nom) pour exporter toutes les mÃ©ta-donnÃ©es comme
+explode.plugin:Plugin Ã  utiliser pour Ã©clater
+explode.params:[options] nom de fichier
+# -- exportcol.pl --
+
+exportcol.out:Nom de fichier ou de contrÃŽle vers lequel envoyer le statut de sortie.
+
+exportcol.cdname:Le nom du CD-ROM -- c'est ce qui va apparaÃ®tre dans le menu de dÃ©marrage une fois que le CD-ROM est installÃ©.
+
+exportcol.desc:Script PERL utilisÃ© pour exporter une ou plusieurs collections vers un CD-ROM Windows.
+
+exportcol.params:[options] nom de collection1 nom de collection 2âŠ
+
+exportcol.coll_not_found:Collection non valide %s ignorÃ©e: collection introuvable Ã  %s.
+
+exportcol.coll_dirs_not_found:Collection non valide %s ignorÃ©e: l'un des rÃ©pertoires suivants n'a pas Ã©tÃ© trouvÃ©:
+
+exportcol.fail:echec de export.pl:
+
+exportcol.no_valid_colls:Pas de collection valide spÃ©cifiÃ©e pour exportation
+
+exportcol.couldnt_create_dir:N'a pas pu crÃ©er rÃ©pertoire %s.
+
+exportcol.couldnt_create_file:N'a pas pu crÃ©er %s.
+
+exportcol.instructions:Pour crÃ©er un CD-ROM d'auto-installation pour Windows, Ã©crire les contenus de ce fichier dans un CD-ROM.
+
+exportcol.non_exist_files:Un ou plusieurs des fichiers et rÃ©pertoires nÃ©cessaires suivants n'existent pas:
+
+exportcol.success:succÃšs de export.pl:
+
+exportcol.output_dir:Les collections exportÃ©es (%s) sont dans %s.
+
+exportcol.nonetscape:Ne mettez pas netscape sur le CD-ROM
+exportcol.export_coll_not_installed:La fonctionnalitÃ© de 'Exporter vers CD-Rom' n'a pas Ã©tÃ© installÃ©e.
+
+
+# -- import.pl --
+
+import.archivedir:LÃ  oÃ¹ finissent le matÃ©riau converti.
+
+
+import.cannot_open_stats_file:ATTENTION: impossible d'ouvrir le fichier stats %s.
+
+
+import.cannot_open_fail_log:Erreur: N'a pas pu ouvrir le journal des Ã©checs %s
+
+
+import.cannot_sort:ATTENTION: le plugin d'importation ne peut pas trier les documents si groupsize > 1. L'option sortmeta sera ignorÃ©e.
+
+
+import.collectdir:RÃ©pertoire de la collection.
+
+
+import.complete:Fin de l'importation.
+
+
+import.debug:Imprimer le texte importÃ© vers STDOUT.
+
+
+import.desc:Script PERL utilisÃ© pour importer les fichiers dans le formÃ¹at GML prÃªt pour la construction.
+
+
+import.faillog:Nom de fichier du log d'Ã©chec. Ce log reÃ§oit les noms de fichier de tous les fichiers qui n'ont pas pu Ãªtre traitÃ©s
+
+
+import.groupsize:Nombre de documents importÃ©s Ã  regrouper dans un fichier XML.
+
+
+import.gzip:Utiliser gzip pour compresser les documents xml de sortie (ne pas oublier d'intÃ©grer ZIPPLug dans votre liste de plugins en construisant Ã  partir de documents compressÃ©s.
+
+
+import.importdir:lÃ  oÃ¹ se trouve le matÃ©riau original.
+
+
+import.keepold:Ne dÃ©truira pas les contenus actuels du rÃ©pertoire des archives (par dÃ©faut).
+
+
+import.maxdocs:Nombre maximum de documents Ã  importer.
+
+
+import.no_plugins_loaded:ERREUR: plugins non chargÃ©s
+
+
+import.OIDtype:La mÃ©thode Ã  utiliser pour produire des identificateurs uniques pour chaque document
+
+import.OIDtype.hash:Hache les contenus du fichier. L'identificateur du document sera le mÃªme Ã  chaque fois que la collection est importÃ©e.
+
+
+import.OIDtype.incremental:Un compteur de document simple beaucoup plus rapide que ''hash". Ce n'est pas garanti de toujours assigner le mÃªme identificateur Ã  un document donnÃ© et ne permet pas l'addition d'aures documents dans des archives xml existants.
+
+
+import.OIDtype.assigned:Utilise 'D' plus la valeur de ds.Idendifier comme identificateur de document. Dc.Identifier doit Ãªtre unique. Si aucun dc.Idenfier n'est associÃ© au document, un id calculÃ© sera utilisÃ©.
+
+import.OIDtype.dirname:Utilise 'J' plus le nom du rÃ©pertoire parent comme identificateur. Cela suppose qu'il y a juste un document par rÃ©pertoire, et que tous les noms de rÃ©pertoire sont uniques.C.a.d. import/b13as/h15ef/page.html obtiendra l'identificateur Jh15ef.
+"
+	
+source::plugin.one_unrecognised	"1 was unrecognised
+
+
+import.saveas:Format Ã  gÃ©nÃ©rer pour l'exportation. L'option par dÃ©faut est GA.
+
+
+import.saveas.GA:Va gÃ©nÃ©rer le format archive Greenstone.
+
+
+import.saveas.METS:Va gÃ©nÃ©rer le format METS
+
+
+import.out:Nom de fichier ou contrÃŽle vers lequel envoyer le statut.
+
+
+import.params:[options] nom de collection
+
+
+import.removeold:Va supprimer les anciens contenus du rÃ©pertoire d'archives -- Ã  utiliser avec prudence.
+
+
+import.removing_archives:Suppression des contenus du rÃ©pertoire d'archivesâŠ
+
+
+import.sortmeta:Trier les documents par ordre alphabÃ©tique et par mÃ©ta-donnÃ©e pour la construction. Ceci sera dÃ©sactivÃ© si groupsize > 1.
+
+
+import.statsfile:Nom de fichier ou contrÃŽle vers lequel envoyer les statistiques d'importation.
+
+
+import.stats_backup:Va plutÃŽt imprimer les stats vers STDERR.
+
+
+import.verbosity:ContrÃŽle la quantitÃ© de sortie. 0=aucun, 3=beaucoup.
+
+
+
+# -- export.pl --
+
+export.exportdir:Ou le matÃ©riel exportÃ© se termine
+
+
+export.cannot_open_stats_file:ATTENTION: Impossible d'ouvrir le fichier stats %s
+
+
+export.cannot_open_fail_log:ERREUR: Impossible d'ouvrir la log des Ã©checs %s
+
+
+export.cannot_sort:ATTENTION: export.pl ne peut trier les documents lorsque groupsize>1. L'option sortmeta sera ignorÃ©e.
+
+
+export.collectdir:RÃ©pertoire collection.
+
+
+export.complete:Exportation terminÃ©e
+
+
+export.debug:Imprimer le texte exportÃ© sur STDOUT.
+
+
+export.desc:Les scripts PERL utilisÃ©s pour importer des fichier dans un format GML sont prÃªts pour la construction
+
+
+export.faillog:Nom de fichier de la log des Ã©checs. Cette log reÃ§oit les noms de fichier de tous les fichiers qui n'ont pu Ãªtre traitÃ©.
+
+
+export.groupsize:Nombre de documents importÃ©s Ã  grouper dans un fichier XML.
+
+
+export.gzip:Utiliser gzip pour compresser les documents xml rÃ©sultats (Noubliez pas d'insÃ©rer ZIPPlug dans votre liste de plugins lorsque vous construisez Ã  partir de documents comprÃ©ssÃ©s.
+
+
+export.importdir:La ou le matÃ©riel original rÃ©side.
+
+
+export.keepold:Ne dÃ©truira pas les contenus du rÃ©pertoire "export" (par dÃ©faut) 
+
+
+export.maxdocs:Nombre maximum de documents Ã  importer.
+
+
+export.no_plugins_loaded:ERREUR: Pas de plugins chargÃ©s.
+
+
+export.OIDtype:MÃ©thode Ã  utiliser pour gÃ©nÃ©rer un identificateur unique pour chaque document.
+
+export.OIDtype.hash:Calculer une valeur ("hash") Ã  partir du contenu du fichier. L'idendificateur document restera le mÃªme Ã  chaque fois que la collection est importÃ©e. 
+
+export.OIDtype.incremental:Un comptage simple des documents plus rapdide que la mÃ©thode "hash". Cette mÃ©thode ne garantit pas d'affecter toujours le mÃªme identificateur Ã  un document donnÃ© et ne permet donc pas d'ajouter d'autres documents Ã  des archives xml existantes.
+
+
+export.saveas:Format Ã  gÃ©nÃ©rer pour l'exportation. L'option par dÃ©faut est METS.
+
+
+export.saveas.DSpace:Va gÃ©nÃ©rer le format archive DSPACE
+
+
+export.saveas.METS:Va gÃ©nÃ©rer le format METS
+
+
+export.out:Nom de fichier ou sortie pour imprimer l'Ã©tat.
+
+
+export.params:[options] nom-collection
+
+
+export.removeold:EnlÃ©vera le contenu du rÃ©pertoire "export" -- Ã  utiliser avec prÃ©caution.
+
+
+export.removing_export:Suppression des contenus du rÃ©pertoire exportâŠ
+
+
+export.sortmeta:Trier les documents en ordre alphabÃ©tique des mÃ©ta-donnÃ©es pour la construction. DÃ©sactivÃ© si groupsize > 1.
+
+
+export.statsfile:Nom de fichier ou sortie pour imprimer les statistiques d'exportation.
+
+
+export.stats_backup:Va imprimer les stats sur STDERR
+
+
+export.verbosity:ContrÃŽle du volume de sortie. 0=aucun, 3=beaucoup.
+
+
+
+# -- mkcol.pl --
+
+mkcol.about:Le texte Ã  propos pour la collection.
+
+
+mkcol.bad_name_cvs:ERREUR: Aucune collection ne peut Ãªtre nommÃ©e CVS car cela pourrait interfÃ©rer avec des rÃ©pertoires de gestion de version de CVS.
+
+
+mkcol.bad_name_modelcol:ERREUR: Aucune collection ne peut Ãªtre nommÃ©e modelcol puisque c'est le nom de la collection modÃšle.
+
+
+mkcol.cannot_find_modelcol:ERREUR: impossible de trouver la collection modÃšle %s
+
+
+mkcol.col_already_exists:ERREUR: Cette collection existe dÃ©jÃ .
+
+
+mkcol.collectdir:Le rÃ©pertoire dans lequel la nouvelle collection va Ãªtre crÃ©Ã©e.
+
+
+mkcol.creating_col:CrÃ©ation de la collection %s
+
+
+mkcol.creator:L'adresse e-mail du crÃ©ateur de la collection.
+
+
+mkcol.creator_undefined:ERREUR: Le crÃ©ateur n'a pas Ã©tÃ© dÃ©fini. Cette variable est nÃ©cesaire pour reconnaÃ®tre les noms de collection dupliquÃ©s.
+
+
+mkcol.desc:Script PERL utilisÃ© pour crÃ©er la structure de rÃ©pertoire pour une nouvelle collection Greenstone.
+
+
+mkcol.doing_replacements:en train d'effectuer des remplacements pour %s
+
+
+mkcol.long_colname:ERREUR: Le nom de collection doit Ãªtre de moins de 8 caractÃšres afin de maintenir la compatibilitÃ© avec des sytÃšmes prÃ©cÃ©dents.
+
+
+mkcol.maintainer:L'adresse e-mail du maintenancier de la collection (si diffÃ©rent du crÃ©ateur).
+
+
+mkcol.no_collectdir:ERREUR: Le rÃ©pertoire de la collection n'existe pas: %s
+
+
+mkcol.no_colname:ERREUR: Aucun nom de collection n'a Ã©tÃ© spÃ©cifiÃ©.
+
+
+mkcol.optionfile:Prendre des options Ã  partir du fichier, utile dans des systÃšmes oÃ¹ des longues lignes de commandes peuvent poser des problÃšmes.
+
+
+mkcol.params:[option] nom de collection
+
+
+mkcol.plugin:Module plugin Perl Ã  utiliser (il peut y avoir plusieurs entrÃ©es de plugins).
+
+
+mkcol.public:Si cette collection a des accÃšs anonymes (vrai/faux).
+
+
+mkcol.quiet:Fonctionne en silence.
+
+
+mkcol.success:La nouvelle collection a Ã©tÃ© crÃ©e avec succÃšs Ã  %s
+
+
+mkcol.title:Le titre de la collection
+
+
+mkcol.win31compat:Est-ce que les noms de rÃ©pertoires doivent suivre le modÃšle Windows 3.1 ou non (c.a.d. 8 caractÃšres max).
+
+
+# -- pluginfo.pl --
+
+pluginfo.collect:L'attribution d'un nom de colection fera que pluginfo.pl va rechercher d'abord dans collect/nom_de_collection/perllib/plugins. Si le plugin n'y est pas trouvÃ© il va alors chercher dans le rÃ©pertoire gÃ©nÃ©ral de logiciels/perlib.
+
+
+pluginfo.desc:Imprime l'information concernant un plugin.
+
+
+pluginfo.general_options:Les options gÃ©nÃ©rales sont hÃ©ritÃ©es des classes parentes du plugin.
+
+
+pluginfo.info:info
+
+
+pluginfo.no_plugin_name:ERREUR: Vous devez donner un nom de plugin.
+
+
+pluginfo.option_types:Les plugins peuvent prendre deux types d'options.
+
+
+pluginfo.params:[options] nom de plugin
+
+
+pluginfo.passing_options:Des options peuvent Ãªtre passÃ©es Ã  tous les plugins en les intÃ©grant dans votre fichier de configuration collect.cfg.
+
+
+pluginfo.specific_options:Les options spÃ©cifiques sont dÃ©finies au sein du plugin mÃªme, et sont disponibles uniquement pour ce plugin particulier.
+
+
+
+#
+# Classifier option descriptions
+#
+
+AZCompactList.allvalues:Utiliser toutes les valeurs de mÃ©ta-donnÃ©es trouvÃ©es.
+
+AZCompactList.desc:Plugin de classification pour tri par ordre alphabÃ©tique
+
+
+AZCompactList.doclevel:Niveau oÃ¹ le document est traitÃ©.
+
+AZCompactList.doclevel.top:Tout le document.
+
+AZCompactList.doclevel.section:Par sections.
+
+
+AZCompactList.firstvalueonly:Utiliser uniquement la premiÃšre valeure de mÃ©ta-donnÃ©e trouvÃ©e
+
+
+AZCompactList.freqsort:Classer par frÃ©quence nodale plutÃŽt que de maniÃšre alpha-numÃ©rique.
+
+
+AZCompactList.maxcompact:Nombre maximum de documents Ã  afficher par page.
+
+
+AZCompactList.metadata:Champ de mÃ©ta-donnÃ©e, ou liste de champs  de mÃ©ta-donnÃ©e sÃ©parÃ©s par des virgules, utilisÃ© pour la classification. Si une liste est spÃ©cifiiÃ©e, le premier type de mÃ©ta-donnÃ©e qui a une valeure sera utilisÃ©. Peut Ãªtre aussi utilisÃ© avec les options -firstvalueonly et -allvalues, pour sÃ©lectioner uniquement la premiÃšre valeur, ou toutes les valeurs de mÃ©ta-donnÃ©es de la liste.
+
+
+AZCompactList.mincompact:Nombre minimum de documents Ã  afficher par page.
+
+
+AZCompactList.mingroup:Valeur minimale qui va causer la formation d'un groupe dans la hierarchie.
+
+
+AZCompactList.minnesting:La valeur minimale qui va entraÃ®ner la conversion d'une liste en liste emboÃ®tÃ©e.
+
+
+AZCompactList.recopt:UtilisÃ© dans des mÃ©ta-donnÃ©es imbriquÃ©es tel que AnnÃ©e/Organisation
+
+AZCompactList.sort:Champ de mÃ©ta-donnÃ©e Ã  partir duquel les nÅuds fils seront triÃ©s.
+
+
+AZCompactSectionList.desc:Variation sur AZCompactList qui classifie des sections plutÃŽt que des documents. Les entrÃ©es sont triÃ©es par mÃ©ta-donnÃ©es de niveau section.
+
+
+AZList.desc:Plugin de classification pour le tri par ordre alphabÃ©tique
+
+
+AZList.metadata:Un champ de mÃ©ta-donnÃ©e unique ou une liste de champs de mÃ©ta-donnÃ©e sÃ©parÃ©s par des virgules utilisÃ© pour classification. En suivant l'ordre indiquÃ© par la liste, le premier champ qui contient  une valeur de mÃ©ta-donnÃ©e sera utilisÃ©. La liste sera triÃ©e suivant cet Ã©lÃ©ment.
+
+
+AZSectionList.desc:Plugin de classification pour classer par ordre alphabÃ©tique. Ceci est trÃšs similaire Ã  AZList sauf qu'il classe par niveau de mÃ©ta-donnÃ©es (en excluant le sommet) au lieu de ne considÃ©rer que les seules mÃ©ta-donnÃ©es des niveaux supÃ©rieurs. Le seul changement est au niveau sous-programme classify () qui doit procÃ©der Ã  une itÃ©ration Ã  travers chaque section, ajoutant chacune d'elle Ã  la classification.
+
+
+BasClas.bad_general_option:Le classificateur %s utilise une option gÃ©nÃ©rale incorrecte (les options gÃ©nÃ©rales sont celles qui sont disponibles pour tous les classificateurs). VÃ©rifiez votre fichier de configuration collect.cfg.
+
+
+BasClas.builddir:Endroit oÃ¹ les indexes construits vont Ãªtre mis.
+
+
+BasClas.buttonname:Le label pour l'Ã©cran et le bouton dans la barre de navigation du classificateur.
+
+
+BasClas.desc:Classe de base pour tous les classificateurs.
+
+
+BasClas.outhandle:Le fichier qui va recevoir les rÃ©sultats.
+
+
+BasClas.removeprefix:PrÃ©fixe Ã  ignorer dans les valeurs de mÃ©ta-donnÃ©es lors du tri. 
+
+BasClas.removesuffix:Suffixe Ã  ignorer dans les valeurs de mÃ©ta-donnÃ©es lors du tri.
+
+BasClas.verbosity:ContrÃŽle la quantitÃ© d'Ã©lÃ©ments produits. 0=aucun, 3=beaucoup.
+
+
+Browse.desc:.
+
+
+DateList.bymonth:Classifier par annÃ©e et par mois plutÃŽt que par annÃ©e seulement.
+
+
+DateList.desc:Plugin de classification utilisÃ© pour trier par date. Par dÃ©faut, trie suivant la mÃ©ta-donnÃ©e 'Date'. La date doit Ãªtre sous la forme yyyymmdd (aaaammjj).
+
+
+DateList.metadata:Le mÃ©ta-donnÃ©e contient qui contient les dates dont on se sert pour classissifier. Le format doit Ãªtre yyyymmdd (aaaammjj).
+
+
+DateList.reverse_sort:Trier les documents par ordre chronologique inverse (plus rÃ©cent en premier).
+
+
+DateList.nogroup:Faire de chaque annÃ©e une entrÃ©e individuelle dans la liste horizontale plutÃŽt que d'utiliser quelques entrÃ©es pour plusieurs annÃ©es
+
+
+DateList.sort:Champ de mÃ©ta-donnÃ©e supplÃ©mentaire pour le tri au oÃ¹ deux documents ont la mÃªme date.
+
+HFileHierarchy.desc:Plugin pour gÃ©nÃ©rer des classifications hiÃ©rarchiques basÃ© sur un fichier structurel supplÃ©mentaire.
+
+Hierarchy.hfile:Le fichier de structure de classificateur.
+
+Hierarchy.hlist_at_top:Affiche horizontalement le premier niveau de la classification.
+
+Hierarchy.reverse_sort:Trier les nÅuds feuilles en ordre inverse (utiliser avec -sort).
+
+Hierarchy.separator:Expression rÃ©guliÃšre utilisÃ©e pour le sÃ©parateur
+
+Hierarchy.sort:Champ de mÃ©ta-donnÃ©e suivant lequel trier. Utiliser '-sort nosort' pour dÃ©sactiver le tri.
+
+Hierarchy.suppresslastlevel:Ignorer la partie finale de la valeur de mÃ©ta-donnÃ©e. C'est particuliÃšrement utile pour des mÃ©ta-donnÃ©es dont chaque valeur est unique, telles que les chemins de fichiers.
+
+
+HTML.desc:CrÃ©e une classification vide qui qui est tout simplement un lien vers une page web.
+
+
+HTML.url:L'url de la page web de destination.
+
+
+List.desc:Plugin de classificateur de liste simple.
+
+
+List.metadata:Un champ de mÃ©ta-donnÃ©e unique ou une liste de champs de mÃ©ta-donnÃ©e sÃ©parÃ©s par des virgules utilisÃ© pour classification. En suivant l'ordre indiquÃ© par la liste, le premier champ qui contient  une valeur de mÃ©ta-donnÃ©e sera utilisÃ©. La liste va Ãªtre triÃ©Ã©e suivant cet Ã©lÃ©ment, Ã  moins que -sort soit utilisÃ©.
+
+
+List.sort:Champ de mÃ©ta-donnÃ©e suivant lequel trier. Utiliser '-sort nosort' pour dÃ©sactiver le tri.
+
+
+Phind.desc:Le plugin du classificateur Phind.
+
+
+Phind.language:Langue ou langues Ã  utiliser pour la construction d'une hiÃ©rarchie. Les langues sont identifiÃ©es par des codes de pays Ã  deux lettres comme en (anglais), es (espagnole), fr (franÃ§ais). Langue est une expression ordinaire, donc 'en|fr' (anglais ou franÃ§ais) et '..' (toute langue) sont valides.
+
+
+Phind.min_occurs:Le nombre minimum de fois qu'une phrase doit apparaÃ®tre dans le texte pour faire partie de la hiÃ©rarchie des phrases.
+
+
+Phind.savephrases:Lorsque dÃ©finie, l'information d'expression sera stockÃ©e comme texte dans le fichier spÃ©cifiÃ©. C'est probablement une bonne idÃ©e d'utiliser une chemin absolu.
+
+
+Phind.suffixmode:Le paramÃštre smode du programme d'extraction d'expression. Une valeur 0 signifie que les mots d'arrÃªt sont ignorÃ©s et 1 signifie que les mots d'arrÃªts sont utilisÃ©s.
+
+
+Phind.text:Le texte utilisÃ© pour construire la hierarchie d'expression.
+
+
+Phind.thesaurus:Nom d'un thÃ©saurus stockÃ© au format Phind dans le repertoire etc de la collection.
+
+
+Phind.title:Le champ de mÃ©ta-donnÃ©e utilisÃ© pour dÃ©crire chaque document.
+
+
+Phind.untidy:Ne pas supprimer les fichiers actifs.
+
+
+SectionList.desc:Comme le classificateur List, mais inclut toutes les sections du document (Ã  l'exception du niveau supÃ©rieur) plutÃŽt que le document de niveau supÃ©rieur seulement.
+
+Collage.buttonname:Le label pour l'Ã©cran et le bouton dans la barre de navigation du classificateur.
+
+Collage.desc:Le plugin du classeur de collage
+
+
+Collage.geometry:Les dimensions du fond de Collage.  Pour un fond de 600 pixels de large et 400 pixels de haut par example, indiquez la gÃ©omÃ©trie comme 600x400
+
+
+Collage.maxDepth:Les images pour le collage sont obtenues en reflÃ©tant ("mirroring") le classeur utilisÃ©. Ceci commande la profondeur maximum du processus de reflÃ©t ("mirroring").
+
+
+Collage.maxDisplay:Le nombre maximum d'images Ã  montrer dans le collage n'importe quand.
+
+
+Collage.imageType:UtilisÃ© pour contrÃŽler, en spÃ©cifiant les extensions de nom de fichier, quels types de fichier sont untilisÃ©s pour le collage. Une liste de noms de fichier est sÃ©parÃ©e par le caractÃšre pourcent (%%).
+
+Collage.bgcolor:La couleur d'arriÃšre plan de la toile de fond, indiquÃ©e sous forme hexadÃ©cimale (par exemple #008000 a comme consÃ©quence un arriÃšre plan vert forÃªt).
+
+
+Collage.refreshDelay:Taux, en millisecondes, de rafraichissement du fond de collage.
+
+
+Collage.isJava2:UtilisÃ© pour contrÃŽler l'utilisation des classes "Java runtime".  Les versions plus rÃ©centes de Java (c.-Ã -d. supÃ©rieures Ã  Java 1.2) incluent des fonctions plus avancÃ©es pour contrÃŽler la transparence des images, qui permettent un contrÃŽle plus fin, toutefois le "Java runtime"   intÃ©grÃ© dans certain navigateurs  est la version 1,1.  L'applet est conÃ§u pour dÃ©tecter automatiquement la version du "Java runtime" utilisÃ©e par le navigateur et rÃ©agir en consÃ©quence.
+
+
+Collage.imageMustNotHave:UtilisÃ© pour supprimer les images qui ne devraient pas apparaÃ®tre dans le collage, telles que les images des boutons qui composent la barre de navigation.
+
+
+Collage.caption:Sous-titres facultatifs Ã  afficher au-dessous de la toile de fond.
+
+
+#
+# Plugin option descriptions
+#
+
+ArcPlug.desc:Plugin rÃ©cursif Ã  travers un fichier archives.inf (ex: le fichier gÃ©nÃ©rÃ© par l'index des archives lorsqu'une importation est faite), traitant chaque fichier trouvÃ©.
+
+
+BasPlug.adding:ajout
+
+
+BasPlug.already_seen:dÃ©jÃ  vu
+
+
+BasPlug.bad_general_option:Le plugin %s utilise une option gÃ©nÃ©rale incomplÃšte (les options gÃ©nÃ©rales sont celles qui sont disponibles pour tous les plugins). VÃ©rifiez votre fichier de configuration collect.cfg.
+
+
+BasPlug.block_exp:Les fichiers qui correspondent Ã  cette expression ordinaire ne pourront pas Ãªtre passÃ©s Ã  d'autres plugins dans la liste. Cela n'a pas d'autres vÃ©ritables effets que d'empÃªcher qu'il y ait plusieurs messages d'avertissements concernant des fichiers d'entrÃ©e dont vous n'aurez pas Ã  vous soucier. Chaque plugin pourrait avoir un bloc_exp par dÃ©faut, p.ex: par dÃ©faut HTMLPlug bloque tout fichier avec les extensions de fichiers .gif, .,jpg, ,.jpeg, .png ou .css.
+
+
+BasPlug.associate_ext:Modifie les fichiers ayant le  mÃªme nom de fichier racine que le document traitÃ© par le plugin ET une extension de fichier provenant de la liste d'arguments sÃ©parÃ©s par une virgule  donnÃ© par cet argument pour les associer au document traitÃ© plutÃŽt que manipulÃ© comme une liste sÃ©parÃ©e
+
+BasPlug.could_not_extract_encoding:ATTENTION: encodage n'a pas pu Ãªtre extrait de %s - utilisation de %s par dÃ©faut
+
+
+BasPlug.could_not_extract_language:ATTENTION: langue n'a pas pu Ãªtre extraite de %s - utilisation de %s par dÃ©faut
+
+
+BasPlug.could_not_open_for_reading:impossible d'ouvrir %s pour lecture
+
+
+BasPlug.no_cover_image:Ne pas rechercher un fichier prÃ©fix.jpg ( prÃ©fix Ã©tant le mÃªme prÃ©fix que le fichier traitÃ©) et considÃ©rer le comme image de couverture.
+
+
+BasPlug.default_encoding:Utiliser cet encodage si -input_ encoding est placÃ© Ã  'auto' et l'algorithme de catÃ©gorisation de texte ne rÃ©ussit pas Ã  extraire l'encodage ou lorsqu'il extrait un encodage non supportÃ© par Greenstone. Cette option peut prendre la mÃªme valeur que -input_ encoding.
+
+
+BasPlug.default_language:Si Greenstone ne rÃ©ussit pas Ã  identifier la langue du document l'Ã©lÃ©ment de mÃ©ta-donnÃ©e 'Langue' sera portÃ© Ã  cette valeur. Le dÃ©faut est 'en' (les symboles de langue ISO 639 sont utilisÃ©s: en= anglais). Notez que si -imput_encoding n'est pas mis Ã  'auto' et -extract_language n'est pas dÃ©fini, tous les documents auront leurs mÃ©ta-donnÃ©es 'Langue' portÃ©es Ã  cette valeur.
+
+
+BasPlug.desc:Classe de base pour tous les plugins d'importation.
+
+
+BasPlug.done_acronym_extract:extraction des acronymes terminÃ©e.
+
+
+BasPlug.done_acronym_markup:marquage des acronymes terminÃ©.
+
+
+BasPlug.done_email_extract:extraction des adresses e-mail terminÃ©e.
+
+
+BasPlug.dummy_text:Ce document n'a pas de texte.
+
+
+BasPlug.empty_file:le fichier ne contient aucun texte
+
+
+BasPlug.extract_acronyms:Extraire les acronymes du texte et les dÃ©finir comme mÃ©ta-donnÃ©es.
+
+
+BasPlug.extract_email:Extraire les adresses e-mail comme mÃ©ta-donnÃ©es.
+
+
+BasPlug.extract_historical_years:Extraire des documents historiques l'information sur les pÃ©riodes. Ceci est stockÃ© comme mÃ©ta-donnÃ©e avec le document. Il y a une interface de recherche pour cette mÃ©ta-donnÃ©e que vous pouvez inclure dans votre collection en ajoutant la formule ''format QueryInterface DateSearch'' au fichier de configuration de votre collection.
+
+
+BasPlug.extract_language:Identifier la langue de chaque document et dÃ©finir la mÃ©ta-donnÃ©e 'Langue'. Notez que cela se fera automatiquement si -imput_encoding est Ã  'auto'.
+
+
+BasPlug.extracting:Extraction
+
+
+BasPlug.extracting_acronyms:extraction d'acronymes
+
+
+BasPlug.extract_keyphrases:Extraction automatique des phrases clÃ©s avec Kea (par dÃ©faut).
+
+BasPlug.extract_keyphrase_options:Options pour l'extraction phrase clÃ© avec Kea. Par exemple: mALIWEB - utilise ALIWEB modÃšle d'extraction; n5 - extraire 5 phrases clÃ©; eGBK - utiliser l'encodage GBK
+
+BasPlug.extracting_emails:extraction d'adresses e-mail
+
+
+BasPlug.file_has_no_text:ERREUR: %s ne contient pas de texte
+
+
+BasPlug.first:Liste des premiÃšres tailles sÃ©parrÃ©es par une virgule Ã  extraire du texte vers le champ de mÃ©ta-donnÃ©e. Ce champ est appelÃ© 'FirstNNN'.
+
+
+BasPlug.input_encoding:Encodage des documents sources. Les documents vont Ãªtre convertis de ces encodages et stoquÃ©s en interne au format utf8.
+
+BasPlug.input_encoding.ascii:ASCII 7 bits pur. Ceci peut Ãªtre plus rapide que d'utiliser iso_8859_1. Veillez Ã  ne pas utiliser ceci sur une collection de documents ayant des caractÃšres non compris dans ASCCI 7 bits pur (ex: les documents franÃ§ais et allemands qui ont des accents). Utiliser iso_8859_1 dans ce cas.
+
+BasPlug.input_encoding.auto:Utilisez un algorithme de catÃ©gorisation de texte pour identifier automatiquement le codage de chaque document source, cela sera plus lent que l'opÃ©ration consistant Ã  mettre explicitement le codage mais est efficace lorsqu'il y a plusieurs codes dans une mÃªme collection.
+
+BasPlug.input_encoding.unicode:Unicode seulement.
+
+BasPlug.input_encoding.utf8:Utf8 ou unicode -- dÃ©tectÃ© automatique
+
+
+BasPlug.keyphrases:expressions clÃ©s
+
+
+BasPlug.marking_up_acronyms:acronymes de marquage
+
+
+BasPlug.markup_acronyms:Ajouter des mÃ©ta-data d'acromyme au texte du document
+
+
+BasPlug.maximum_century:Le siÃšcle le plus citÃ© Ã  extraire comme mÃ©ta-donnÃ©e historique (ex: 14 va extraire toutes les rÃ©fÃ©rences jusqu'au 14 Ãšme siÃšcle).
+
+
+BasPlug.maximum_year:La date historique la plus Ã©levÃ©e Ã  utiliser comme mÃ©ta-donnÃ©e (dans une Ãšre commune, comme 1950).
+
+
+BasPlug.must_be_implemented:BasPlug: :la fonction de lecture doit Ãªtre exÃ©cutÃ©e dans une sous-classe pour les plugins rÃ©cursifs
+
+
+BasPlug.no_bibliography:Ne cherchez pas Ã  bloquer les dates bibliographiques lors de l'extraction de dates historiques.
+
+
+BasPlug.process_exp:Une expression de Perl Ã  faire correspondre Ã  des noms de fichiers. La correspondance avec les noms de fichiers sera traitÃ©e avec ce plugin. Par exemple, l'usage de '(?i).html?\$' fait correspondre tous les documents se terminant par .htm ou .html (indÃ©pendemment de la casse).
+
+
+BasPlug.read_denied:Permission de lecture refusÃ©e pour %s
+
+
+BasPlug.smart_block:Bloquer les fichiers d'une faÃ§on plus flexible qu'en regardant le nom du fichier.
+
+BasPlug.stems:troncs
+
+
+BasPlug.unsupported_encoding:ATTENTION: %s semble encodÃ© dans un encodage non supportÃ© (%s) - utilisant %
+
+
+BasPlug.wrong_encoding:ATTENTION: %s a Ã©tÃ© lu en utilisant l'encodage %s mais semble avoir Ã©tÃ© encodÃ© comme %s
+
+
+BibTexPlug.desc:BibTexPlug lit les fichiers bibliographiques dans le format BibTex. BibTexPlug crÃ©e un objet document pour chaque rÃ©fÃ©rence dans le fichier. C'est une sous-classe de SplitPlug, donc s'il y a plusieurs Ã©lÃ©ments, ils sont tous lus.
+
+
+BookPlug.desc:CrÃ©e un document Ã  niveaux multiples Ã  partir d'un document contenant les balises de niveau <<TOC>>. Les mÃ©ta-donnÃ©es pour chaque section sont extraites Ã  partir des autres balises se trouvant sur la mÃªme ligne que <<TOC>>. Par exemple, <<Title>>xxxx<</Title>> dÃ©finit une mÃ©ta-donnÃ©e de titre. Tout le reste entre des balises TOC est traitÃ© comme du simple html (c.-Ã -d. qu'aucun lien html ou tout autre Ã©lÃ©ment de type HTMLPlug n'est traitÃ©). Les fichiers d'entrÃ©e sont supposÃ©s avoir l'extension .hb par dÃ©faut (cela peut Ãªtre changÃ© en ajoutant une option -processus_exp de mÃªme que le fichier hb mais une extension .jpg est prise comme image de couverture (les fichiers jpg sont bloquÃ©s par ce plugin). BookPlug est une simplification (et une extension) de HPPlug utilisÃ© par Humanity Library Collections. BookPlug est plus rapide et il s'attend Ã  ce que les fichiers d'entrÃ©e soient  propres (l'entrÃ©e aux collections HDL contient un nombre excessif de balises html autour de <<TOC>>, et utilise des balises <<I>> pour spÃ©cifier les images, et prend tout simplement tout le texte compris entre les balises <<TOC>> et le dÃ©but du texte comme Ã©tant la mÃ©ta-donnÃ©e de titre). Si vous dÃ©sirez baliser un document pour qu'il soit affichÃ© de la mÃªme maniÃšre que les collections HDL, utilisez ce plugin au lieu de HBPlug.
+
+
+ConvertToPlug.convert_to:Ce plugin convertit vers du TEXT ou HTML
+
+ConvertToPlug.convert_to.html:Format HTML
+
+ConvertToPlug.convert_to.text:Format de texte simple
+
+
+ConvertToPlug.desc:Ce plugin est hÃ©ritÃ© par des plugins comme WORDPlug et PDFPlug. Il facilite la conversion de ces types de documents en HTML ou Text en dÃ©finissant une variable qui instruit ConvertTBasPlug comme procÃ©der. Il fonctionne en hÃ©ritant de maniÃšre dynamique HTMLPlug ou TextPlug, basÃ© sur un argument plugin 'convert to' Si l'argument n'est pas prÃ©sent, HTMLPlug est hÃ©ritÃ© par dÃ©faut.
+
+
+ConvertToPlug.use_strings:Si dÃ©fini, une simple fonction de chaÃ®ne sera appelÃ©e pour extraire le texte si l'utilitaire de conversion Ã©choue.
+
+
+ConvertToRogPlug.desc:Un plugiciel qui hÃ©rite de RogPlug.
+
+
+DBPlug.desc:Utilise des enregistrements d'une base de donnÃ©es comme documents.
+
+
+DBPlug.title_sub:Expression de substitution pour modifier une chaÃ®ne enregistrÃ©e titre. UtilisÃ©e, par exemple, par PSPlug pour supprimer ''Page1'' etc du texte utilisÃ© comme titre.
+
+
+EMAILPlug.desc:Email plug lit les fichiers email. Ceux-ci sont nommÃ©s simplement avec un numÃ©ro (c'est-Ã -dire comme ils apparaissent dans les dossiers maildir) ou avec l'extension .mbx (pour format de fichier courrier mbox).\n Texte de document: Le texte de document est composÃ© de tous le texte situÃ© aprÃšs la premiÃšre ligne vide du document.\nMÃ©ta-donnÃ©es (pas Dublin Core!):\n\t\$Headers  Tout le contenu de l'en-tÃªte (optionel, n'est pas emmagasinÃ© par defaut)\n\t\$Subject   Objet: header\n\t\$To    A: header\n\t\$From   De: header/n/t/$FromName    Nom de l'expÃ©diteur (si disponible)\n\t\$FromAddr    Adresse Ã©lectronique de l'expÃ©diteur\n\t\$DateText     Date: header\n\t\$Date   Date: en-tÃªte au format GSDL (ex 19990924)
+
+
+EMAILPlug.no_attachments:Ne pas sauvegarder fichiers attachÃ©s aux messages.
+
+
+EMAILPlug.headers:Emmagasine les titres des email comme "Headers" dans les mÃ©ta-donnÃ©es.
+
+
+EMAILPlug.split_exp:Une expression ordinaire perl utilisÃ©e pour scinder les fichiers contenant plusieurs messages en documents sÃ©parÃ©s.
+
+
+ExcelPlug.desc:Un Plugin pour importer des fichiers Microsoft Excel.
+
+
+FOXPlug.desc:Plugin pour traiter un fichier Foxbase dbt. Ce plugin fournit une fonctionnalitÃ© de base pour lire dans les fichiers dbt et dbf et traiter chaque enregistrement. Ce plugin gÃ©nÃ©ral devrait Ãªtre remplacÃ© pour une base de donnÃ©es particuliÃšre afin de traiter les champs appropriÃ©s.
+
+
+GAPlug.desc:Traite les documents XML de GreenstoneArchive. Notez que ce plugin ne corrige pas la syntaxe (bien que le module XML: :Parser vÃ©rifie l'exactitude de la forme). Il est supposÃ© que les fichiers GreenstoneArchive sont conformes Ã  leur DTD.
+
+
+GISBasPlug.extract_placenames:Extrait le nom des emplacements du texte et les considÃ©rer comme mÃ©ta-donnÃ©es. NÃ©cessite l'installation d'une extension GIS Ã  Greenstone.
+
+GISBasPlug.gazetteer:Atlas Ã  utiliser pour extraire le nom des emplacements du texte et les considÃ©rer comme mÃ©ta-donnÃ©es. NÃ©cessite l'installation d'une extension GIS Ã  Greenstone.
+
+GISBasPlug.place_list:Lors de l'extraction des emplacements, contient la liste des noms d'emplacements au dÃ©but du document. NÃ©cessite l'installation d'une extension GIS Ã  Greenstone.
+
+GMLPlug.desc:Plugin qui traite un document au format GML suppose que toutes les balises GML sont en minuscule.
+
+
+HBPlug.desc:Plugin qui traite un rÃ©pertoire de livres en HTML. Ce plugin est utilisÃ© par les collections Humanity Library et ne traite pas d' encodages d'entrÃ©e autres ascii et ascii Ã©tendu. Ce code n'est pas du tout joli et on pourrait le rendre plus rapide. En le laissant tel quel, j'espÃšre encourager les gens Ã  Ã  construire leurs collections en utilisant plutÃŽt HBSPlug ;-) \n\n Utilisez HBSPlug pour crÃ©er une nouvelle collection et pour marquer des fichiers comme les collections Humanity Library Collections. HBSPlug accepte tous les encodages d'entrÃ©e mais exige que les fichiers marquÃ©s soient plus propres que ceux utilisÃ©s par les collections Human Library.
+
+
+HTMLPlug.assoc_files:Expression perl d'extensions de fichiers Ã  associer avec des documents html.
+
+
+HTMLPlug.desc:Ce plugin traite les fichiers HTML
+
+
+HTMLPlug.description_tags:Divise le document en sous-sections oÃ¹ apparaissent des balises <Section>. Notez qu'en dÃ©finissant cette option, vous dÃ©finissez de faÃ§on implicite -no_metadata puisque toutes les mÃ©ta-donnÃ©es doivent Ãªtre incluses entre les signes <Sections>. '-keep_head' n'aura non plus aucun effet si cette option est dÃ©finie.
+
+
+HTMLPlug.file_is_url:DÃ©fini si les noms de fichiers d'entrÃ©e constituent l'url des documents source d'origine, p.ex. si un outil de "mirroring" a Ã©tÃ© utilisÃ© pour crÃ©er la structure du rÃ©pertoire d'importation.
+
+
+HTMLPlug.hunt_creator_metadata:Trouver autant de mÃ©ta-donnÃ©es possibles sur l'auteur et les placer dans le champ 'Creator'. NÃ©cessite le drapeau -metadata_fields.
+
+
+HTMLPlug.keep_head:Ne pas supprimer les en-tÃªtes des fichiers html.
+
+
+HTMLPlug.metadata_fields:Liste de champs de mÃ©ta-donnÃ©es sÃ©parÃ©s par des virgules Ã  essayer Ã  extraire. Utiliser balise 'tag<nom_balise>' pour obtenir le contenu de la premiÃšre paire de <nom_balise> placÃ©e dans un Ã©lÃ©ment de mÃ©ta-donnÃ©e 'tagname'. Mettre en majuscules si vous voulez les mÃ©ta-donnÃ©es en en lettres majuscules dans Greenstone, car l'extraction des balises ne tient pas compte de la casse.
+
+
+HTMLPlug.no_metadata:Ne cherchez pas Ã  extraire des mÃ©tadata des fichiers.
+
+
+HTMLPlug.no_strip_metadata_html:Liste de noms de mÃ©ta-donnÃ©es sÃ©parÃ©es par des virgules, ou 'all'. UtilisÃ© avec -description_tags, cela Ã©vite d'Ã©liminer les balises HTML des valeurs pour les mÃ©ta-donnÃ©es spÃ©cifiÃ©es.
+
+HTMLPlug.nolinks:Ne pas essayer de piÃ©ger les liens (la dÃ©finition de ce drapeau peut amÃ©liorer la vitesse de construction/importation mais tout lien relatif dans le document sera brisÃ©).
+
+
+HTMLPlug.rename_assoc_files:Renomme les fichiers associÃ©s aux documents (ex. les images). CrÃ©e aussi une structure de rÃ©pertoire beaucoup plus petite (utile pour la crÃ©ation de collections sur cd-rom).
+
+
+HTMLPlug.title_sub:Expression de substitution pour modifier la chaine enregistrÃ©e comme titre. UtilisÃ© par exemple par PDFPlug pour supprimer ''Page 1'', etc du texte utilisÃ© comme titre.
+
+
+ImagePlug.converttotype:Convertit lâimage pricipale en une format 's'.
+
+
+ImagePlug.desc:Ce plugin traite les images, en ajoutant des mÃ©ta-donnÃ©es de base
+
+
+ImagePlug.minimumsize:Ignorer les images de moins de n octets.
+
+
+ImagePlug.noscaleup:Ne pas rÃ©Ã©chelonner les petites images en crÃ©ant des vignettes.
+
+
+ImagePlug.screenviewsize:Si dÃ©fini, crÃ©e une image de taille n pour l'affichage Ã  l'Ã©cran et dÃ©finit les mÃ©ta-donnÃ©es Screen, ScreenSize, ScreenWidth et ScreenHeigth. N'est pas dÃ©fini par dÃ©faut.
+
+
+ImagePlug.screenviewtype:Si -screenviewsize est dÃ©fini, ceci dÃ©finit le type d'mage d'affichage.
+
+
+ImagePlug.thumbnailsize:CrÃ©e des vignettes de taille nxn.
+
+
+ImagePlug.thumbnailtype:CrÃ©e des vignettes au format 's'.
+
+
+IndexPlug.desc:Ce plugin rÃ©cursif traite un fichier index.txt. Le fichier index.txt doit contenir la liste des fichiers Ã  inclure dans la collection, suivie par toute mÃ©ta-donnÃ©e supplÃ©mentaire Ã  associer Ã  chaque fichier\n\nLe fichier index.txt doit Ãªtre formatÃ© comme suit: la premiÃšre ligne peut Ãªtre une clÃ© (commenÃ§ant par key:) pour nommer les champs de mÃ©ta-donnÃ©e (ex. key: Subject Organization Date). Les lignes suivantes contiennent un nom de fichier suivi de la valeur de la mÃ©ta-donnÃ©e (ex 'irma/iwo097e 3.2 unesco 93â va associer les mÃ©ta-donnÃ©es Subject= 3.2, Organization=unesco, et Date=1993 avec le fichier irma/iwo097e si la ligne de clÃ© ci dessus a Ã©tÃ© utilisÃ©e)\n\nNotez que si l'un des champs de mÃ©ta-donnÃ©e utilise le plugin de classification de HiÃ©rarchie, alors sa valeur doit correspondre au premier champ (le descripteur) dans le fichier de classification appropriÃ©.\n\nLes valeurs de mÃ©ta-donnÃ©e peuvent Ãªtre nommÃ©es sÃ©parÃ©ment en utilisant une balise (ex. >Sujet<3.2) et cela va remplacer tout nom qui leur a Ã©tÃ© attribuÃ© par la ligne de clÃ©. S'il n'y a pas de ligne de clÃ©, toute valeur de  mÃ©ta-donnÃ©e sans nom sera nommÃ©e 'Subject'.
+
+
+ISISPlug.desc:Ce plugin traite les bases de donnÃ©es CDS/ISIS. Pour chaque base de donnÃ©es CDS/ISIS traitÃ©e, il doit y avoir trois fichiers dans le dossier d'importation de la collection: le Fichier de Base (.mst), le Tableau de DÃ©finition de Champ (.fdt), et le Fichier de RÃ©fÃ©rences croisÃ©es (.xrf).
+
+
+ISISPlug.subfield_separator:La chaÃ®ne utilisÃ©e pour sÃ©parer les sous champs dans les enregistrements de bases de donnÃ©es CDS/ISIS.
+
+
+ISISPlug.entry_separator:La chaÃ®ne utilisÃ©e pour sÃ©parer les valeurs multiples des champs de mÃ©ta-donnÃ©es individuelles dans les enregistrements de base de donnÃ©es CDS/ISIS.
+
+
+LaTeXPlug.desc:Plugin pour documents  LaTeX .
+
+
+MACROPlug.desc:CrÃ©e un document simple Ã  un seul niveau. Ajoute une mÃ©ta-donnÃ©e de Titre de la premiÃšre ligne de texte (jusqu'Ã  100 caratÃšres).
+
+
+MARCPlug.desc:Plugin MARC de base.
+
+
+MARCPlug.metadata_mapping:Nom de fichier qui inclut des dÃ©tails de de correspondance de valeurs MARC vers les noms de mÃ©ta-donnÃ©es Greenstone. Le dÃ©faut est 'marctdoc.txt' trouvÃ© dans rÃ©pertoire etc du site.
+
+
+MetadataPass.desc:Classe de base Ã  BasPlug qui permet l'utilisation des plugins mÃ©ta-donnÃ©e, utilise la passe metadata_read de import.pl
+
+# -- Missing translation: GISBasPlug.desc
+
+NULPlug.desc:Fichier plugin factice (.nul). UtilisÃ© avec les fichiers gÃ©nÃ©rÃ©s lors de l'Ã©clatement des bases de donnÃ©es mÃ©ta-donnÃ©es.
+
+OAIPlug.desc:Plugin Basic Open Archives Initiate (OAI)
+
+
+OggVorbisPlug.add_technical_metadata:Ajouter des mÃ©ta-donnÃ©es techniques (c.a.d. bitrate)
+
+
+OggVorbisPlug.desc:Un plugin pour importer des fichiers audio Ogg Vorbis.
+
+
+PagedImgPlug.desc:Ajouter Plugin pour les documents construits a partir d'une chaine d'images, avec l'option texte OCR pour chaque image.
+
+
+PagedImgPlug.documenttype:Met le type de document (utilisÃ© pour le display)
+
+PagedImgPlug.documenttype.paged:Les documents avec pages numÃ©rotÃ©es ont des flÃšches qui indiquent la prochaine et la derniÃšre page, ainsi qu'une boite permettant d'aller directement la page X
+
+PagedImgPlug.documenttype.hierarchy:Les documents hiÃ©rarchiques ont une table de contenu.
+
+
+PagedImgPlug.headerpage:Ajouter une page d'entÃªte au niveau supÃ©rieur( sans image) Ã  chaque document.
+
+
+PagedImgPlug.screenview:Produit une vue Ã©cran pour chaque image, et programme les mÃ©ta-donnÃ©es de Screen, ScreenSize, ScreenWidth et ScreenHeight.
+
+
+PagedImgPlug.screenviewsize:CrÃ©e des images vue-Ã©cran de taille nxn.
+
+
+PagedImgPlug.screenviewtype:CrÃ©e des images vue-Ã©cran en format 's'.
+
+
+PagedImgPlug.thumbnail:CrÃ©e une vignette pour chaque image
+
+
+PDFPlug.complex:CrÃ©e une sortie plus complexe. Avec la dÃ©finition de cette option, la sortie html va beaucoup plus ressembler au fichier PDF original. Pour que cela fonctionne correctement, il faut que Ghostscript soit installÃ© (pour *nix, gs doit Ãªtre dans la dÃ©finition de chemins, pendant que pour windows, vous devez avoir gswin32c.exe sur votre chemin).
+
+
+PDFPlug.desc:Plugin pdf "reasonably with-it"
+
+
+PDFPlug.nohidden:EmpÃªche pdftohtml de chercher Ã  extraire du texte cachÃ©. Cela n'est utile que si l'option -complex est Ã©galement dÃ©finie.
+
+
+PDFPlug.noimages:Ne pas chercher Ã  extraire des images de PDF.
+
+
+PDFPlug.use_sections:CreÃ©e une section sÃ©parÃ©e pour chaque page du fichier PDF.
+
+
+PDFPlug.zoom:Le facteur avec lequel zoomer PDF pour la sortie (ceci n'est utile que si -comlex est dÃ©fini).
+
+
+PPTPlug.desc:Un plugin pour importer des fichiers Microsoft PowerPoint.
+
+
+PSPlug.desc:Ceci est un convertisseur de texte ps \''bon marchÃ©\''. Si vous Ãªtes sÃ©rieux, pensez Ã  utiliser le paquetage PRESCRIT, qui est disponible pour tÃ©lÃ©chargement Ã  http://www.nzdl/software.html
+
+
+PSPlug.extract_date:Extrait date de l'en-tÃªte PS.
+
+
+PSPlug.extract_pages:Extrait pages de l'en-tÃªte PS.
+
+
+PSPlug.extract_title:Extrait titre de l'entÃªte PS.
+
+
+RecPlug.desc:RecPlug est un plugin rÃ©cursif Ã  travers les rÃ©pertoires, traitant chaque fichier trouvÃ©.
+
+
+RecPlug.recheck_directories:Une fois le traitement terminÃ© des fichiers dans un rÃ©pertoire d'importation, vÃ©rifiez le rÃ©pertoire pour dÃ©couvrir les nouveaux fichiers crÃ©Ã©s.
+
+
+RecPlug.use_metadata_files:Lit les mÃ©ta-donnÃ©es Ã  partir de fichiers XML.
+
+
+ReferPlug.desc:ReferPlug lit les fichiers de bibliographie au format Refer.
+
+
+ReferPlug.longdesc:ReferPlug lit les fichiers de bibliographie au format Refer.\nPar Gordon W. Paynter (gwp\@cs.waikato.ac.nz), Novembre 2000\n\n BasÃ© approximativement sur hcibib2Plug de Steve Jones (stevej\@cs.waikato.ac.nz) qui Ã©tait basÃ© sur EmailPlug de Gordon Paynter (gwp\@cs.waikato.ac.nz), qui Ã©tait basÃ© sur d'anciennes versions de HTMLPlug et HCIBIBPlug de Stefan Boddie et autres ... il est actuellement difficile de dire quoi vient de quoi\n\nReferPlug crÃ©e un objet document pour chaque rÃ©fÃ©rence dans le fichier. C'est une sous-classe de SplitPlug, donc s'il y a plusieurs enregistrements, ils sont tous lus.\n\nLe texte du document constituÃ© de la rÃ©fÃ©rence au format Refer.\nMeta-donnÃ©es::\n\t\$Creator \%A Nom de l'auteur\n\t\$Title  \%T Titre d'article du livre\n\t\$Journal \%J Titre de la revue\n\t\$Booktitle \%B Titre du livre contenant la publication\n\t\$Report  \%R Type de rapport, article ou thÃšse\n\t\$Volume  \%V NumÃ©ro de volume de la revue\n\t\$Number  \%N Nombre de revues dans le volume\n\t\$Editor  \%E Editor name\n\t\$Pages  \%P Page Number of article\n\t\$Publisher \%I Name of Publisher\n\t\$Publisheraddr \%C Adresse de l'Ã©diteur\n\t\$Date  \%D Date de publication\n\t\$Keywords \%K Mots-clÃ©s associÃ©s Ã  la publication\n\t\$Abstract \%X Abstract de la publication\n\t\$Copyright\t\%* Information de droits d'auteur pour l'article
+
+
+RogPlug.desc:CrÃ©e des documents Ã  un seul niveau Ã  partir de fichiers .rog ou .mdb.
+
+
+RTFPlug.desc:Plugin pour l'importation de fichiers Rich Text Format.
+
+
+SRCPlug.desc:Le nom du fichier est prÃ©sentement utilisÃ© pour Titre (optionnellement moins un certain prÃ©fixe). Languages actuels:\ntext:READMEs/Makefiles\nC/C++ (extrait actuellement les dÃ©clarations #include et les dÃ©clarations de classe C++)\n Perl (pour le moment, effectuÃ© uniquement comme texte)\nShell (pour le moment, effectuÃ© uniquement comme texte)
+
+
+SRCPlug.remove_prefix:Enlever cette chaine de dÃ©but du nom de fichier (ex.: -remove_prefix /tmp/XX/src). Par dÃ©faut, le chemin complet est supprimÃ© du nom de fichier.
+
+
+SplitPlug.desc:SplitPlug est un plugin pour Ã©clater des fichiers d'entrÃ© en en segments qui vont Ãªtre traitÃ©s individuellement. Ce plugin ne doit pas Ãªtre invoquÃ© directement. Si vous voulez traiter des fichiers d'entrÃ©e qui contiennent plusieurs documents, vous devez plutÃŽt Ã©crire un plugin avec une fonction de traitement qui va traiter un de ces documents et hÃ©riter de SplitPlug. Voir ReferPlug en guise d'exemple.
+
+
+SplitPlug.split_exp:Une expression ordinaire Perl pour Ã©clater les fichiers d'entrÃ©e en segments.
+
+
+TEXTPlug.desc:CrÃ©e un document Ã  niveau unique. Ajoute une mÃ©ta-donnÃ©e de Titre de premiÃšre ligne de texte (jusqu'Ã  100 caractÃšres).
+
+
+TEXTPlug.title_sub:Expression de substitution pour modifier la chaine stockÃ©e comme Titre. UtilisÃ©e par exemple par PSPlug pour supprimer ''Page 1'' etc du texte utilisÃ© comme titre.
+
+
+UnknownPlug.assoc_field:Nom du champ de mÃ©ta-donnÃ©e qui va contenir le nom de fichier associÃ©.
+
+
+UnknownPlug.desc:Ceci est un plugin simple d'importation de fichiers aux formats inconnus de Greenstone. Un document fictif va Ãªtre crÃ©Ã© pour chacun de ses fichiers et le fichier lui mÃªme va Ãªtre passÃ© Ã  Greenstone comme \"fichier associÃ©\" du document.
+
+
+UnknownPlug.file_format:Type Mime du fichier (ex. image/gif).
+
+
+UnknownPlug.mime_type:Type Mime du fichier (c.a.d. image/gif).
+
+
+UnknownPlug.process_extension:Traiter les fichiers avec cette extension. Cette option est une alternative Ã  process_exp qui est plus simple Ã  utiliser mais moins flexible.
+
+
+MP3Plug.desc:Plugin pour le traitement des fichiers MP3.
+
+
+MP3Plug.assoc_images:Use google image search to locate images related to MP3 file based on ID3 Title and Artist metadata.
+
+MP3Plug.assoc_images:UtilisÃ© pour enregistrer [applet] des mÃ©ta-donnÃ©es pour chaque document qui contient le code HTML d'un applet MP3 audio player pour jouer ce fichier.
+
+
+MP3Plug.metadata_fields:Liste de champs mÃ©ta-donnÃ©es sÃ©parÃ©s par des virgules pour extraction dans un fichier MP3 (supposÃ© prÃ©sent) . Utiliser \"*\" pour extraire tous les champs.
+
+
+W3ImgPlug.aggressiveness:SÃ©rie de techniques d'extraction de textes du mÃªme type Ã  utiliser.
+
+W3ImgPlug.aggressiveness.1:Nom de fichier, chemin, texte ALT seulement.
+
+W3ImgPlug.aggressiveness.2:Tout de 1, plus lÃ©gende ou image s'il y a lieu.
+
+W3ImgPlug.aggressiveness.3:Tout de 2, plus paragraphes proches si disponibles.
+
+W3ImgPlug.aggressiveness.4:Tout de 3, plus en-tÃªtes prÃ©cÃ©dents (<h1>,<h2>,,,) si disponibles.
+
+W3ImgPlug.aggressiveness.5:Tout de 4, plus les rÃ©fÃ©rences textuelles si disponibles.
+
+W3ImgPlug.aggressiveness.6:Tout de 4, plus page de mÃ©ta-balises (titre, mots clÃ©s, etc).
+
+W3ImgPlug.aggressiveness.7:Tout de 6, 5 et 4 combinÃ©s.
+
+W3ImgPlug.aggressiveness.8:Tout de 7, plus lÃ©gende de rÃ©pÃ©tition, nom de fichier, etc (Ã©lever le rang des rÃ©sultats les plus pertinents).
+
+W3ImgPlug.aggressiveness.9:Tout de 1, plus texte complet de la page source.
+
+
+W3ImgPlug.caption_length:Longueur maximum des lÃ©gendes (en caractÃšres).
+
+
+W3ImgPlug.convert_params:ParamÃštres additionnels de conversion d'ImageMagick pour la crÃ©ation de vignettes. Par exemple, '-raise' va donner un effet tridimensionnel aux vignettes.
+
+
+W3ImgPlug.desc:Un plugin pour extraire des images et le texte associÃ© Ã  partir de pages web.
+
+
+W3ImgPlug.document_text:Ajoute un texte d'image comme document:text (autrement champ de mÃ©ta-donnÃ©e IndexedText).
+
+
+W3ImgPlug.index_pages:Indexe les pages en mÃªme temps que les images. Ou alors rÃ©fÃ©rencie les pages Ã  la source URL.
+
+
+W3ImgPlug.max_near_text:Nombre maximum de caractÃšres prÃšs des images Ã  extraire.
+
+
+W3ImgPlug.min_height:Pixels. Sauter les images plus courtes que ceci.
+
+
+W3ImgPlug.min_near_text:Nombre minimum de caractÃšres prÃšs du texte ou de la lÃ©gende Ã  extraire.
+
+
+W3ImgPlug.min_size:Bytes. Sauter les images plus petites que ceci.
+
+
+W3ImgPlug.min_width:Pixels. Sauter les images plus petites que ceci.
+
+
+W3ImgPlug.neartext_length:Longueur cible du texte proche (en caractÃšres).
+
+
+W3ImgPlug.no_cache_images:Ne pas mettre les images en cache (pointer vers URL de l'original).
+
+
+W3ImgPlug.smallpage_threshold:Les images sur les pages plus petites que ceci (bytes) auront les mÃ©ta-donnÃ©es de la page (titre, mots clÃ©s, etc) ajoutÃ©es.
+
+
+W3ImgPlug.textrefs_threshold:Seuil de rÃ©fÃ©rences textuelles. Des valeurs plus faibles indiquent que l'algorithme est moins strict.
+
+
+W3ImgPlug.thumb_size:Taille maximale de vignette. Largeur comme hauteur.
+
+
+WordPlug.desc:Un plugin pour des documents Microsoft Word.
+
+
+XMLPlug.desc:Classe de base pour les plugins XML.
+
+
+ZIPPlug.desc:Plugin qui traite les formats d'entrÃ©e compressÃ©s et/ou les formats d'entrÃ©e archivÃ©s actuellement traitÃ©s et les extensions de fichiers sont:\ngzip (.gz, .z, .tgz, .taz)\nbzip (.bz)\nbzip2 (.bz2)\nzip (.zip .jar)\ntar (.tar)\n\n. Ce plugin dÃ©pend de la prÃ©sence des utilaires suivants (lorsqu'il essaie de traiter les formats correspondants):\ngunzip (pour gzip)\nbunzip (pour bzip)\nbunzip2 \nunzip (pour zip)\ntar (pour tar)
+
+
+
+#
+# Perl module strings
+#
+
+classify.could_not_find_classifier:ERREUR: Classificateur \''%s\'' introuvable
+
+
+plugin.could_not_find_plugin:ERREUR: impossible de trouver le plugin /''%s''
+
+
+plugin.including_archive:intÃ©gration des contenus d'une archive ZIP/TAR
+
+
+plugin.including_archives:intÃ©gration des contenus de %d archives ZIP/TAR
+
+
+plugin.kill_file:Processus tuÃ© par le fichier .kill
+
+
+plugin.n_considered:%d documents ont Ã©tÃ© pris en compte pour traitement
+
+
+plugin.n_included:%d ont Ã©tÃ© traitÃ©s et intÃ©grÃ©s dans la collection
+
+
+plugin.n_rejected:%d ont Ã©tÃ© rejetÃ©s
+
+
+plugin.n_unrecognised:%d n'ont pas Ã©tÃ© reconnus
+
+
+plugin.no_plugin_could_process:ATTENTION: Aucun plugin n'a pu traiter %s
+
+
+plugin.no_plugin_could_recognise:ATTENTION: Aucun plugin n'a pu reconnaÃ®tre %s
+
+
+plugin.no_plugin_could_process_this_file:aucun plugin n'a pu traiter ce fichier
+
+
+plugin.no_plugin_could_recognise_this_file:Aucun plugin n'a pu reconnaÃ®tre ce fichier
+
+
+plugin.one_considered:1 document a Ã©tÃ© pris en compte pour traitement
+
+
+plugin.one_included:1 a Ã©tÃ© traitÃ© et intÃ©grÃ© dans la collection
+
+
+plugin.one_rejected:1 a Ã©tÃ© rejetÃ©
+
+
+plugin.one_unrecognised:1 n'a pas Ã©tÃ© reconnu
+
+
+plugin.see_faillog:Voir %s pour une liste des documents rejetÃ©s
+
+
+PrintUsage.default:DÃ©faut
+
+
+PrintUsage.required:REQUIS
+
Index: /main/tags/3.01/gsdl/perllib/strings_ru.rb
===================================================================
--- /main/tags/3.01/gsdl/perllib/strings_ru.rb	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/strings_ru.rb	(revision 21112)
@@ -0,0 +1,1557 @@
+#
+# Resource bundle description
+#
+
+Language.code:ru
+
+Language.name:Russian
+
+
+OutputEncoding.unix:koi8_r
+
+OutputEncoding.windows:dos_850
+
+
+
+#
+# Common output messages
+#
+
+common.cannot_create_file:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐŸÐ·ÐŽÐ°ÑÑ ÑÐ°Ð¹Ð» %s
+
+
+common.cannot_find_cfg_file:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐ°Ð¹ÑÐž ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» %s
+
+
+common.cannot_open:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ %s
+
+
+common.cannot_open_fail_log:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ ÑÐµÐ³ÐžÑÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» %s
+
+
+common.cannot_open_output_file:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ Ð²ÑÑ
+ÐŸÐŽÐœÐŸÐ¹ ÑÐ°Ð¹Ð» %s.
+
+
+common.cannot_read:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐžÑÐ°ÑÑ %s
+
+
+common.cannot_read_file:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ Ð¿ÑÐŸÑÐµÑÑÑ ÑÐ°Ð¹Ð» %s
+
+
+common.general_options:ÐŸÐ±ÑÐžÐµ Ð²Ð°ÑÐžÐ°ÐœÑÑ (ÐŽÐ»Ñ %s)
+
+
+common.must_be_implemented:ÑÑÐœÐºÑÐžÑ ÐŽÐŸÐ»Ð¶ÐœÐ° Ð±ÑÑÑ ÐŸÑÑÑÐµÑÑÐ²Ð»ÐµÐœÐ° Ð² Ð¿ÐŸÐŽÐºÐ»Ð°ÑÑÐµ
+
+
+common.options:ÐŸÐ¿ÑÐžÐž
+
+
+common.processing:ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐ°
+
+
+common.specific_options:ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐµ ÐŸÐ¿ÑÐžÐž
+
+
+common.usage:ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµ
+
+
+
+#
+# Script option descriptions and output messages
+#
+
+scripts.language:Ð¯Ð·ÑÐº ÐŽÐ»Ñ ÐŸÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐŸÐ¿ÐžÑÐ°ÐœÐžÐ¹ ÐŸÐ¿ÑÐžÐ¹ Ð² (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, 'en_US' ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÐµÑ Ð°ÐŒÐµÑÐžÐºÐ°ÐœÑÐºÐžÐ¹ Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐžÐ¹ ÑÐ·ÑÐº). Ð¢ÑÐµÐ±ÑÐµÑ Ð¿ÐµÑÐµÐ²ÐŸÐŽÐŸÐ² ÐŸÐ¿ÐžÑÐ°ÐœÐžÐ¹ ÐŸÐ¿ÑÐžÐ¹ ÐŽÐ»Ñ Ð¿ÑÐžÑÑÑÑÑÐ²ÐžÑ Ð² ÑÐ°Ð¹Ð»Ðµ perllib/strings_language-code.rb.
+
+
+scripts.xml:ÐÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ Ð² ÑÐŸÑÐŒÐ°ÑÐµ XML Ð±ÐµÐ· 'ÑÐžÐŒÐ¿Ð°ÑÐžÑÐœÑÑ
+' ÐºÐŸÐŒÐŒÐµÐœÑÐ°ÑÐžÐµÐ², ÐœÐŸ Ñ ÐœÐ°ÐŒÐœÐŸÐ³ÐŸ Ð±ÐŸÐ»ÑÑÐžÐŒ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ ÐŽÐµÑÐ°Ð»ÐµÐ¹.
+
+
+scripts.listall:Ð¡Ð¿ÐžÑÐŸÐº Ð²ÑÐµÑ
+ ÐžÐ·Ð²ÐµÑÑÐœÑÑ
+ ÐµÐŽÐžÐœÐžÑ.
+
+
+
+# -- buildcol.pl --
+
+buildcol.archivedir:ÐÐŽÐµ ÐœÐ°Ñ
+ÐŸÐŽÑÑÑÑ Ð°ÑÑ
+ÐžÐ²Ñ.
+
+
+buildcol.builddir:ÐÐŽÐµ Ð¿ÐŸÐŒÐµÑÑÐžÑÑ ÑÐŸÐ·ÐŽÐ°ÐœÐœÑÐµ ÐžÐœÐŽÐµÐºÑÑ.
+
+
+buildcol.cachedir:ÐÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐŽÐµÑ Ð²ÑÐµÐŒÐµÐœÐœÐŸ ÑÐŸÐ·ÐŽÐ°ÐœÐ° Ð·ÐŽÐµÑÑ, Ð¿ÐµÑÐµÐŽ ÑÐµÐŒ ÐºÐ°Ðº Ð±ÑÐŽÐµÑ ÑÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐ° Ð² ÑÐŸÐ·ÐŽÐ°ÐœÐœÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÑ.
+
+
+buildcol.cannot_open_cfg_file:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: ÐÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÐŸÑÐºÑÑÑÑ ÐŽÐ»Ñ ÐŸÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÑ: %s
+
+
+buildcol.collectdir:ÐÐžÑÐµÐºÑÐŸÑÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+buildcol.copying_back_cached_build:ÐÐ±ÑÐ°ÑÐœÐŸÐµ ÐºÐŸÐ¿ÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐºÑÑÐžÑÐŸÐ²Ð°ÐœÐœÐŸÐ¹ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ.
+
+
+buildcol.create_images:ÐÐŸÐ¿ÑÑÐºÐ° ÑÐŸÐ·ÐŽÐ°ÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐŽÐ»Ñ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. ÐÐœÐ° ÐŸÑÐœÐŸÐ²Ð°ÐœÐ° ÐœÐ° Gimp, ÐºÐŸÑÐŸÑÑÐ¹ ÐžÐœÑÑÐ°Ð»Ð»ÐžÑÑÐµÑÑÑ ÐœÐ°ÑÑÐŽÑ Ñ Ð¿ÐŸÐŽÑ
+ÐŸÐŽÑÑÐžÐŒÐž ÐŒÐŸÐŽÑÐ»ÑÐŒÐž perl, ÑÑÐŸÐ±Ñ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÐžÑÑ ÐœÐ°Ð¿ÐžÑÐ°ÐœÐžÐµ ÑÐºÑÐžÐ¿ÑÐŸÐ² ÐŸÑ perl.
+
+
+buildcol.debug:ÐÐ°Ð¿ÐžÑÐ°ÑÑ Ð¿ÑÐŸÐŽÑÐºÑÐžÑ Ð² STDOUT.
+
+
+buildcol.desc:Ð¡ÐºÑÐžÐ¿Ñ PERL, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² GML.
+
+
+buildcol.faillog:ÐÐµÑÐŽÐ°ÑÐœÐ°Ñ ÑÐµÐ³ÐžÑÑÑÐ°ÑÐžÑ ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð°. Ð­ÑÐŸÑ Ð¶ÑÑÐœÐ°Ð» ÑÐµÐ³ÐžÑÑÑÐ°ÑÐžÑ Ð¿ÐŸÐ»ÑÑÐžÐ» ÐžÐŒÐµÐœÐ° ÑÐ°Ð¹Ð»ÐŸÐ² ÐžÐ· Ð»ÑÐ±ÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÑ
+
+
+buildcol.index:ÐÐœÐŽÐµÐºÑ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ( Ð²ÑÐµ Ð±ÑÐŽÐµÑ ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÑÑÑÑ Ð² ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ, ÐµÑÐ»Ðž ÐŸÐœ ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ).
+
+
+buildcol.keepold:ÐÐµ Ð±ÑÐŽÐµÑ ÑÐœÐžÑÑÐŸÐ¶Ð°ÑÑ ÑÐµÐºÑÑÐµÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÐŸÐ·ÐŽÐ°Ð²Ð°ÐµÐŒÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž.
+
+
+buildcol.maxdocs:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ.
+
+
+buildcol.mode:Ð§Ð°ÑÑÐž Ð¿ÑÐŸÑÐµÑÑÐ° Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐºÐŸÑÐŸÑÑÐµ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ Ð²ÑÐ¿ÐŸÐ»ÐœÐžÑÑ.
+
+buildcol.mode.all:ÐÐµÐ»Ð°ÑÑ Ð²ÑÐµ.
+
+buildcol.mode.build_index:Ð¢ÐŸÐ»ÑÐºÐŸ ÐžÐœÐŽÐµÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÑÐµÐºÑÑ.
+
+buildcol.mode.compress_text:Ð¢ÐŸÐ»ÑÐºÐŸ ÑÐ¶Ð°ÑÑ ÑÐµÐºÑÑ.
+
+buildcol.mode.infodb:Ð¢ÐŸÐ»ÑÐºÐŸ ÑÐŸÐ·ÐŽÐ°ÑÑ Ð±Ð°Ð·Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.
+
+
+buildcol.no_default_images:ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐœÐµ Ð±ÑÐŽÑÑ ÑÐŸÐ·ÐŽÐ°ÐœÑ.
+
+
+buildcol.no_image_script:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: Ð¡ÐºÑÐžÐ¿Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐœÐµ ÐŒÐŸÐ³ Ð±ÑÑÑ ÐœÐ°Ð¹ÐŽÐµÐœ: %s
+
+
+buildcol.no_strip_html:ÐÐµ ÑÐŽÐ°Ð»ÑÑÑ ÑÑÐ³Ðž HTML ÐžÐ· ÐžÐœÐŽÐµÐºÑÐžÑÐŸÐ²Ð°ÐœÐœÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ° (ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐŽÐ»Ñ mgpp ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹).
+
+
+buildcol.no_text:ÐÐµ Ñ
+ÑÐ°ÐœÐžÑÑ ÑÐ¶Ð°ÑÑÐ¹ ÑÐµÐºÑÑ. Ð­ÑÐŸÑ ÐŸÐ¿ÑÐžÑ Ð¿ÐŸÐ»ÐµÐ·ÐœÐ° ÐŽÐ»Ñ ÐŒÐžÐœÐžÐŒÐžÐ·Ð°ÑÐžÐž ÑÐ°Ð·ÐŒÐµÑÐ° Ð¿ÐŸÑÑÑÐŸÐµÐœÐœÑÑ
+ ÐžÐœÐŽÐµÐºÑÐŸÐ², ÐµÑÐ»Ðž ÐÑ ÐœÐ°ÐŒÐµÑÐµÐ²Ð°ÐµÑÐµÑÑ Ð²ÑÐµÐ³ÐŽÐ° Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÑÑ ÐŸÑÐžÐ³ÐžÐœÐ°Ð»Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° Ð² ÑÐµÐºÑÑÐµÐµ Ð²ÑÐµÐŒÑ (ÑÐŸ ÐµÑÑÑ ÐÑ ÐœÐµ Ð±ÑÐŽÐµÑÐµ ÑÐ¿ÐŸÑÐŸÐ±ÐœÑ Ð²ÐŸÑÑÑÐ°ÐœÐŸÐ²ÐžÑÑ ÑÐ¶Ð°ÑÑÑ Ð²ÐµÑÑÐžÑ ÑÐµÐºÑÑÐ°).
+
+
+buildcol.out:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐžÐœÐŽÐµÐºÑ ÐŽÐ»Ñ Ð¿ÐµÑÐ°ÑÐž ÑÑÐ°ÑÑÑÐ° Ð¿ÑÐŸÐŽÑÐºÑÐžÐž Ð².
+
+
+buildcol.params:[Ð²Ð°ÑÐžÐ°ÐœÑÑ] ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+
+
+buildcol.remove_empty_classifications:Ð¡ÐºÑÑÑÑ ÐœÐµÐ·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÐœÑÐµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ Ðž ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÐµ ÑÐ·Ð»Ñ (Ð±ÐµÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²).
+
+
+buildcol.unlinked_col_images:ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÑÐ²ÑÐ·Ð°ÐœÑ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸ.
+
+
+buildcol.unknown_mode:ÐÐµÐžÐ·Ð²ÐµÑÑÐœÑÐ¹ ÑÐ¿ÐŸÑÐŸÐ±: %s
+
+
+buildcol.updating_archive_cache:ÐÐ±ÐœÐŸÐ²Ð»ÐµÐœÐžÐµ ÐºÑÑÐ° Ð°ÑÑ
+ÐžÐ²Ð°
+
+
+buildcol.verbosity:Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ Ð¿ÑÐŸÐŽÑÐºÑÐžÐž. 0=ÐœÐžÑÐµÐ³ÐŸ, 3=Ð±ÐŸÐ»ÑÑÐŸÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸ.
+
+
+
+# -- classinfo.pl --
+
+classinfo.collect:ÐÑÐž Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÐµÐœÐžÐž ÐžÐŒÐµÐœÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐ°ÐŽÐŸ Ð±ÑÐŽÐµÑ Ð¿ÑÐµÐ¶ÐŽÐµ Ð²ÑÐµÐ³ÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ classinfo.pl Ð² collect/collection-name/perllib/classify. ÐÑÐ»Ðž ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ ÑÐ°ÐŒ ÐœÐµ ÐœÐ°Ð¹ÐŽÐµÐœ, ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐŸ Ð±ÑÐŽÐµÑ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ ÐŸÐ±ÑÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÑ perllib/classify.
+
+
+classinfo.desc:ÐÐ°Ð¿ÐžÑÐ°ÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐµ.
+
+
+classinfo.general_options:ÐÐ±ÑÐžÐµ ÐŸÐ¿ÑÐžÐž ÑÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÑ ÐŸÑ ÑÐŸÐŽÐžÑÐµÐ»ÑÑÐºÐžÑ
+ ÐºÐ»Ð°ÑÑÐŸÐ² ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°.
+
+
+classinfo.info:ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ
+
+
+classinfo.no_classifier_name:ÐÐšÐÐÐÐ: ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÑÐµÐŽÐŸÑÑÐ°Ð²ÐžÑÑ ÐžÐŒÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°.
+
+
+classinfo.option_types:ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ ÐŒÐŸÐ³ÑÑ ÐžÐŒÐµÑÑ ÐŽÐ²Ð° ÑÐžÐ¿Ð° ÐŸÐ¿ÑÐžÐ¹.
+
+
+classinfo.params:[Ð²Ð°ÑÐžÐ°ÐœÑÑ] ÐžÐŒÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°
+
+
+classinfo.passing_options:ÐÐ¿ÑÐžÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð¿ÐµÑÐµÐŽÐ°ÐœÑ Ð»ÑÐ±ÐŸÐŒÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ Ð²ÐºÐ»ÑÑÐµÐœÐžÐµÐŒ ÐžÑ
+ Ð² ÐÐ°Ñ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» collect.cfg.
+
+
+classinfo.specific_options:ÐÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐµ ÐŸÐ¿ÑÐžÐž ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÑÐ°ÐŒÐŸÐ³ÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Ðž ÐŽÐŸÑÑÑÐ¿ÐœÑ ÑÐŸÐ»ÑÐºÐŸ ÑÑÐŸÐŒÑ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐŒÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ.
+
+
+# -- explode_metadata_database.pl --
+explode.desc:Ð Ð°ÑÑÐžÑÐžÑÑ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+Ñ
+
+explode.document_field: Ð­Ð»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑÐžÐ¹ ÐžÐŒÑ ÑÐ°Ð¹Ð»Ð° ÐŽÐ»Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐŸÐ»ÑÑÐµÐœÑ Ðž Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ.
+
+
+explode.document_prefix: ÐÑÐžÑÑÐ°Ð²ÐºÐ° ÐŽÐ»Ñ ÐŒÐµÑÑÐŸÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° (ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ñ ÐŸÐ¿ÑÐžÐµÐ¹ document_field).
+
+
+explode.document_suffix:Ð¡ÑÑÑÐžÐºÑ ÐŽÐ»Ñ ÐŒÐµÑÑÐŸÐ¿ÐŸÐ»ÐŸÐ¶ÐµÐœÐžÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° (ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ñ ÐŸÐ¿ÑÐžÐµÐ¹ document_field).
+
+
+explode.encoding:ÐÐŸÐŽÐžÑÐŸÐ²ÐºÐ° ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ð¿ÑÐž ÑÑÐµÐœÐžÐž ÑÐ°Ð¹Ð»ÐŸÐ² Ð² Ð±Ð°Ð·Ðµ ÐŽÐ°ÐœÐœÑÑ
+
+
+explode.filename_field:ÐÐŸÐ»Ðµ Ñ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑÐŒÐž ÑÐ°Ð¹Ð»ÐŸÐ², ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐŒÐž Ð¿ÑÐž ÑÐŸÐ·ÐŽÐ°ÐœÐžÐž ÐœÑÐ»ÐµÐ²ÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² (ÐµÑÐ»Ðž ÐœÐµÑ document_field ÐžÐ»Ðž ÑÑÐŸ Ð¿ÐŸÐ»Ðµ Ð¿ÑÑÑÐŸ)
+explode.metadata_set:ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐœÐ°Ð±ÐŸÑÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð²ÑÐµÑ
+ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+
+explode.plugin:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ Ð¿ÑÐž ÑÐ°ÑÑÐžÑÐµÐœÐžÐž
+explode.params: [ÐŸÐ¿ÑÐžÐž] ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°
+
+# -- exportcol.pl --
+
+exportcol.out:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐžÐœÐŽÐµÐºÑ ÐŽÐ»Ñ Ð¿ÐµÑÐ°ÑÐž ÑÑÐ°ÑÑÑÐ° Ð¿ÑÐŸÐŽÑÐºÑÐžÐž Ð².
+
+exportcol.cdname:ÐÐŒÑ CD-ROM - ÑÑÐŸ ÑÐŸ, ÑÑÐŸ Ð¿ÐŸÑÐ²ÐžÑÑÑ Ð² ÑÑÐ°ÑÑÐŸÐ²ÐŸÐŒ ÐŒÐµÐœÑ, ÐºÐ°Ðº ÑÐŸÐ»ÑÐºÐŸ CD-ROM ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ.
+
+exportcol.desc:Ð¡ÐºÑÐžÐ¿Ñ PERL ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°Ð»ÑÑ ÐŽÐ»Ñ ÑÐºÑÐ¿ÐŸÑÑÐ° ÐŸÐŽÐœÐŸÐ¹ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹ Ð² Windows CD-ROM. 
+
+exportcol.params:[ÐŸÐ¿ÑÐžÐž] ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ-ÐžÐŒÑ1 ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ-ÐžÐŒÑ2...
+
+exportcol.coll_not_found:ÐÐµ Ð¿ÑÐžÐœÐžÐŒÐ°ÑÑ Ð²ÐŸ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ ÐœÐµÐŽÐµÐ¹ÑÑÐ²ÐžÑÐµÐ»ÑÐœÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ %s:ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµ ÐœÐ°Ð¹ÐŽÐµÐœÐ° Ð² %s.
+
+exportcol.coll_dirs_not_found:ÐÐµ Ð¿ÑÐžÐœÐžÐŒÐ°ÑÑ Ð²ÐŸ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ ÐœÐµÐŽÐµÐ¹ÑÑÐ²ÐžÑÐµÐ»ÑÐœÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ %s: ÐŸÐŽÐœÐ° ÐžÐ· ÑÐ»ÐµÐŽÑÑÑÐžÑ
+ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐ¹ ÐœÐµ ÐœÐ°Ð¹ÐŽÐµÐœÐ°.
+
+exportcol.fail:exportcol.pl ÐœÐµ ÑÐŽÐ°Ð»ÑÑ:
+
+exportcol.no_valid_colls:ÐÐžÐºÐ°ÐºÐžÑ
+ Ð¿ÑÐžÐ³ÐŸÐŽÐœÑÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐºÑÐ¿ÐŸÑÑÐ°.
+
+exportcol.couldnt_create_dir:ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐŸÐ·ÐŽÐ°ÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÑ %s.
+
+exportcol.couldnt_create_file:ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÑÐŸÐ·ÐŽÐ°ÑÑ %s.
+
+exportcol.instructions:Ð§ÑÐŸÐ±Ñ ÑÐŸÐ·ÐŽÐ°ÑÑ ÑÐ°ÐŒÐŸÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÑÑÐžÐ¹ÑÑ CD-ROM ÐŽÐ»Ñ Windows, Ð·Ð°Ð¿ÐžÑÐžÑÐµ ÐœÐ° CD-ROM ÑÐŸÐŽÐµÑÐ¶ÐžÐŒÐŸÐµ ÑÑÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐž.
+
+exportcol.non_exist_files:ÐÐŽÐžÐœ ÐžÐ»Ðž Ð±ÐŸÐ»ÐµÐµ ÐžÐ· ÑÐ»ÐµÐŽÑÑÑÐžÑ
+ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ðž ÐŽÐžÑÐµÐºÑÐŸÑÐžÐ¹ ÐœÐµ ÑÑÑÐµÑÑÐ²ÑÐµÑ:
+
+exportcol.success:exportcol.pl ÑÐ»ÐµÐŽÐŸÐ²Ð°Ð²ÑÐžÐ¹:
+
+exportcol.output_dir:Ð­ÐºÑÐ¿ÐŸÑÑÐžÑÑÐµÐŒÑÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž (%s) ÐœÐ°Ñ
+ÐŸÐŽÑÑÑÑ Ð² %s.
+
+exportcol.nonetscape:ÐÐµ Ð²ÐºÐ»ÑÑÐ°ÑÑ Netscape ÐœÐ° ÑÐºÑÐ¿ÐŸÑÑÐžÑÑÐµÐŒÑÐ¹ CD-ROM.
+exportcol.export_coll_not_installed:Ð€ÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐµ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÐž âÐ­ÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ Ð² CD-ROMâ ÐœÐµ Ð±ÑÐ»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ.
+
+
+# -- import.pl --
+
+import.archivedir:ÐÐŽÐµ Ð·Ð°ÐºÐ°ÐœÑÐžÐ²Ð°ÐµÑÑÑ ÐºÐŸÐœÐ²ÐµÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐŒÐ°ÑÐµÑÐžÐ°Ð» Ð·Ð°ÐºÐ°ÐœÑÐžÐ²Ð°ÐµÑÑÑ.
+
+
+import.cannot_open_stats_file:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ ÑÐ°Ð¹Ð» ÑÐŸÑÑÐŸÑÐœÐžÑ %s.
+
+
+import.cannot_open_fail_log:ÐÐšÐÐÐÐ: Ð ÐµÐ³ÐžÑÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» ÐœÐµ ÐŒÐŸÐ³ Ð±ÑÑÑ ÐŸÑÐºÑÑÑ %s
+
+
+import.cannot_sort:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: import.pl ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÑÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ, ÐºÐŸÐ³ÐŽÐ° ÑÐ°Ð·ÐŒÐµÑ Ð³ÑÑÐ¿Ð¿Ñ> 1. sortmeta ÐÐ¿ÑÐžÑ ÐŒÐµÑÐ°ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð±ÑÐŽÐµÑ ÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÑÑÑÑ.
+
+
+import.collectdir:ÐÐžÑÐµÐºÑÐŸÑÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+import.complete:ÐÐŒÐ¿ÐŸÑÑ Ð·Ð°Ð²ÐµÑÑÐµÐœ
+
+
+import.debug:ÐÐ°Ð¿ÐžÑÐ°ÑÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÑÐµÐºÑÑ Ð² STDOUT.
+
+
+import.desc:Ð¡ÐºÑÐžÐ¿Ñ PERL, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ð² ÑÐŸÑÐŒÐ°Ñ GML, Ð³ÐŸÑÐŸÐ² ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ.
+
+
+import.faillog:ÐÐµÑÐŽÐ°ÑÐœÐ°Ñ ÑÐµÐ³ÐžÑÑÑÐ°ÑÐžÑ ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð°. Ð­ÑÐŸÑ Ð¶ÑÑÐœÐ°Ð» ÑÐµÐ³ÐžÑÑÑÐ°ÑÐžÐž Ð¿ÐŸÐ»ÑÑÐ°ÐµÑ ÐžÐŒÐµÐœÐ° ÑÐ°Ð¹Ð»Ð° Ð»ÑÐ±ÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÑ.
+
+
+import.groupsize:Ð§ÐžÑÐ»ÐŸ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÐµÐŒÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²ÐºÐž Ð² ÐŸÐŽÐžÐœ XML ÑÐ°Ð¹Ð».
+
+
+import.gzip:ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ gzip ÐŽÐ»Ñ ÑÐ¶Ð°ÑÐžÑ ÑÐµÐ·ÑÐ»ÑÑÐžÑÑÑÑÐžÑ
+ xml ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² (ÐœÐµ Ð·Ð°Ð±ÑÐŽÑÑÐµ Ð²ÐºÐ»ÑÑÐžÑÑ ZIPPlug Ð² ÐÐ°Ñ ÑÐ¿ÐžÑÐŸÐº Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ², ÐºÐŸÐ³ÐŽÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÑÐŸÐ·ÐŽÐ°ÐµÑÑÑ ÐžÐ· ÑÐ¶Ð°ÑÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²).
+
+
+import.importdir:ÐÐŽÐµ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÐžÑÑ
+ÐŸÐŽÐœÑÐ¹ ÐŒÐ°ÑÐµÑÐžÐ°Ð».
+
+
+import.keepold:ÐÐµ Ð±ÑÐŽÐµÑ ÑÐœÐžÑÑÐŸÐ¶Ð°ÑÑ ÑÐµÐºÑÑÐµÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ Ð°ÑÑ
+ÐžÐ²ÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž (Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ).
+
+
+import.maxdocs:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ.
+
+
+import.no_plugins_loaded:ÐÐšÐÐÐÐ: ÐÐžÐºÐ°ÐºÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐœÐµ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÑ.
+
+
+import.OIDtype:ÐÐµÑÐŸÐŽ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð¿ÑÐž Ð³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÑÐœÐžÐºÐ°Ð»ÑÐœÑÑ
+ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+import.OIDtype.hash:Ð¥ÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐ°Ð¹Ð»Ð°. ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° Ð±ÑÐŽÐµÑ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐºÐ°Ð¶ÐŽÑÐ¹ ÑÐ°Ð·, ÐºÐŸÐ³ÐŽÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐŒÐ¿ÐŸÑÑÐžÑÑÐµÑÑÑ.
+
+
+import.OIDtype.incremental:ÐÑÐŸÑÑÐŸÐ¹ Ð¿ÐŸÐŽÑÑÐµÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐºÐŸÑÐŸÑÑÐ¹ ÑÐ²Ð»ÑÐµÑÑÑ Ð·ÐœÐ°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð±ÐŸÐ»ÐµÐµ Ð±ÑÑÑÑÑÐŒ, ÑÐµÐŒ "Ñ
+ÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐµ". Ð­ÑÐŸ ÐœÐµ Ð³Ð°ÑÐ°ÐœÑÐžÑÑÑÑ ÑÐŸÐ³ÐŸ, ÑÑÐŸ Ð²ÑÐµÐ³ÐŽÐ° Ð±ÑÐŽÐµÑ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœ ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ Ðº ÐŽÐ°ÐœÐœÐŸÐŒÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ, Ñ
+ÐŸÑÑ Ðž ÐœÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ Ð¿ÐŸÑÐ»ÐµÐŽÑÑÑÐžÐŒ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒ Ð±ÑÑÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐœÑÐŒÐž Ðº ÑÑÑÐµÑÑÐ²ÑÑÑÐžÐŒ  xml Ð°ÑÑ
+ÐžÐ²Ð°ÐŒ.
+
+
+import.OIDtype.assigned:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ 'D' Ð¿Ð»ÑÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð² Ð¿ÐŸÐ»Ðµ dc.Identifier Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°. ÐÐŸÐ»Ðµ dc.Identifiers ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð±ÑÑÑ ÐŸÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÑÐŒ. ÐÑÐ»Ðž dc.Identifier ÐœÐµ Ð·Ð°Ð¿ÐŸÐ»ÐœÐµÐœÐŸ ÐŽÐ»Ñ ÐŽÐ°ÐœÐœÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ Ð·Ð°ÐŒÐµÐœÑ Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°Ðœ hash id.
+
+import.OIDtype.dirname:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ 'J' Ð¿Ð»ÑÑ ÑÐŸÐŽÐžÑÐµÐ»ÑÑÐºÑÑ Ð¿Ð°Ð¿ÐºÑ as the identifier. ÐÐ»Ñ ÑÑÐŸÐ³ÐŸ Ð² ÐºÐ°Ð¶ÐŽÐŸÐ¹ Ð¿Ð°Ð¿ÐºÐµ ÐŽÐŸÐ»Ð¶ÐµÐœ Ð±ÑÑÑ ÑÐŸÐ»ÑÐºÐŸ ÐŸÐŽÐžÐœ ÑÐ°Ð¹Ð», Ð° ÐµÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŽÐŸÐ»Ð¶ÐœÐŸ Ð±ÑÑÑ ÐŸÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÑÐŒ. E.g. import/b13as/h15ef/page.html Ð¿ÐŸÐ»ÑÑÐžÑ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ Jh15ef.
+
+
+import.saveas:Ð ÐºÐ°ÐºÐŸÐŒ ÑÐŸÑÐŒÐ°ÑÐµ Ð³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÑÑ Ð°ÑÑ
+ÐžÐ². ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ GA.
+
+
+import.saveas.GA:ÐÐµÐœÐµÑÐžÑÑÐµÑ ÑÐŸÑÐŒÐ°Ñ Greenstone Archive.
+
+
+import.saveas.METS:ÐÐµÐœÐµÑÐžÑÑÐµÑ ÑÐŸÑÐŒÐ°Ñ METS.
+
+
+import.out:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐžÐœÐŽÐµÐºÑ ÐŽÐ»Ñ Ð·Ð°Ð¿ÐžÑÐž Ð²ÑÑ
+ÐŸÐŽÐœÐŸÐ³ÐŸ ÑÑÐ°ÑÑÑÐ° Ð².
+
+
+import.params:[ÐŸÐ¿ÑÐžÐž] ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+
+
+import.removeold:Ð£ÐŽÐ°Ð»ÐžÑ ÑÑÐ°ÑÐŸÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ Ð°ÑÑ
+ÐžÐ²ÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž - ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ñ ÐŸÑÑÐŸÑÐŸÐ¶ÐœÐŸÑÑÑÑ.
+
+
+import.removing_archives:Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÑÐµÐºÑÑÐµÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ Ð°ÑÑ
+ÐžÐ²ÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž...
+
+
+import.sortmeta:Ð¡ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð ÐµÐ·ÑÐ»ÑÑÐ°ÑÑ Ð¿ÐŸÐžÑÐºÐ° Ð±ÑÐ»ÐµÐ²ÑÑ
+ Ð·Ð°Ð¿ÑÐŸÑÐŸÐ² Ð±ÑÐŽÑÑ Ð²ÑÐŽÐ°ÐœÑ Ð² ÑÑÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ. Ð­ÑÐŸ Ð±ÑÐŽÐµÑ Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÐŸ, ÐµÑÐ»Ðž ÑÐ°Ð·ÐŒÐµÑ Ð³ÑÑÐ¿Ð¿Ñ> 1.
+
+
+import.statsfile:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐžÐœÐŽÐµÐºÑ ÐŽÐ»Ñ Ð·Ð°Ð¿ÐžÑÐž ÑÑÐ°ÑÐžÑÑÐžÐºÐž ÐžÐŒÐ¿ÐŸÑÑÐ° Ð².
+
+
+import.stats_backup:ÐÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ Ð±ÑÐŽÐµÑ Ð·Ð°Ð¿ÐžÑÑÐ²Ð°ÑÑ stats Ð² STDERR.
+
+
+import.verbosity:Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ Ð¿ÑÐŸÐŽÑÐºÑÐžÐž. 0=ÐœÐžÑÐµÐ³ÐŸ, 3=Ð±ÐŸÐ»ÑÑÐŸÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸ.
+
+
+
+# -- export.pl --
+
+export.exportdir:ÐÐŽÐµ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐŒÐ°ÑÐµÑÐžÐ°Ð».
+
+
+export.cannot_open_stats_file:ÐÐÐÐÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ ÑÐ°Ð¹Ð» ÑÑÐ°ÑÑ %s
+
+
+export.cannot_open_fail_log:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐŸÑÐºÑÑÑÑ ÑÐ°Ð¹Ð» Ð»ÐŸÐ³ %s
+
+
+export.cannot_sort:ÐÐÐÐÐÐÐÐ: export.pl ÐœÐµ ÐŒÐŸÐ¶ÐµÑ ÑÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ, ÐµÑÐ»Ðž ÑÐ°Ð·ÐŒÐµÑ Ð³ÑÑÐ¿Ð¿Ñ > 1. ÐÐµÑÐ°-ÐŸÐ¿ÑÐžÑ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÑ Ð±ÑÐŽÐµÑ Ð¿ÑÐŸÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÐœÐ°.
+
+
+export.collectdir:ÐÐžÑÐµÐºÑÐŸÑÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+export.complete:Ð­ÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐµ ÐŸÐºÐŸÐœÑÐµÐœÐœÐŸ.
+
+
+export.debug:ÐÑÐ²ÐµÑÑÐž ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÑÐµÐºÑÑ ÐœÐ° ÑÐºÑÐ°Ðœ.
+
+
+export.desc:PERL ÑÐºÑÐžÐ¿Ñ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ð² ÑÐŸÑÐŒÐ°Ñ GML Ð³ÐŸÑÐŸÐ² ÐŽÐ»Ñ ÐºÐŸÐœÑÑÑÑÐºÑÐžÐž.
+
+
+export.faillog:ÐÐµÑÐŽÐ°Ð»ÐŸÑÑ Ð·Ð°Ð¿ÐžÑÐ°ÑÑ Ð² Ð»ÐŸÐ³ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°. Ð­ÑÐŸÑ Ð»ÐŸÐ³ Ð·Ð°Ð¿ÐžÑÑÐ²Ð°ÐµÑ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÑÐŒÐŸÐ³Ð»Ðž Ð±ÑÑÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÐœÑ.
+
+
+export.groupsize:Ð§ÐžÑÐ»ÐŸ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² ÐŽÐ»Ñ Ð³ÑÑÐ¿Ð¿ÐžÑÐŸÐ²ÐºÐž Ð² ÐŸÐŽÐœÐŸÐŒ XML ÑÐ°Ð¹Ð»Ðµ.
+
+
+export.gzip:ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÑ gzip ÐŽÐ»Ñ ÐºÐŸÐŒÐ¿ÑÐžÐŒÐžÑÐŸÐ²Ð°ÐœÐžÑ ÑÐµÐ·ÑÐ»ÑÑÐžÑÑÑÑÐžÑ
+ xml ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² (ÐœÐµ Ð·Ð°Ð±ÑÐŽÑÑÐµ Ð²ÐºÐ»ÑÑÐžÑÑ ZIPPlug Ð² ÑÐ¿ÐžÑÐŸÐº Ð²Ð°ÑÐžÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² Ð¿ÑÐž ÐºÐŸÐœÑÑÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐžÐ· ÐºÐŸÐŒÐ¿ÑÐžÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²).
+
+
+export.importdir:ÐÐŽÐµ ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ ÐžÑÑ
+ÐŸÐŽÐœÑÐ¹ ÐŒÐ°ÑÐµÑÐžÐ°Ð».
+
+
+export.keepold:ÐÐµ ÑÐœÐžÑÑÐŸÐ¶ÐžÑ ÐœÐ°ÑÑÐŸÑÑÐµÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž (Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ).
+
+
+export.maxdocs:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÑ
+ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐ°. 
+
+
+export.no_plugins_loaded:ÐÐšÐÐÐÐ: ÐÐµÑ Ð·Ð°Ð³ÑÑÐ¶ÐµÐœÐœÑÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ².
+
+
+export.OIDtype:ÐÐµÑÐŸÐŽ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ Ð¿ÑÐž Ð³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐŸÐŽÐœÐŸÐ·ÐœÐ°ÑÐœÑÑ
+ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ² ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+export.OIDtype.hash:Ð¡ÐŸÑ
+ÑÐ°ÐœÑÐµÑ Ð² ÐŸÐ¿ÐµÑÐ°ÑÐžÐŸÐœÐœÐŸÐ¹ Ð¿Ð°ÐŒÑÑÐž ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»Ð°. ÐÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐŸÑÑÐ°ÐµÑÑÑ ÑÐµÐŒ Ð¶Ðµ Ð¿ÑÐž ÐºÐ°Ð¶ÐŽÐŸÐŒ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+export.OIDtype.incremental:ÐÑÐŸÑÑÐŸÐ¹ ÑÐ¿ÐŸÑÐŸÐ± Ð¿ÐŸÐŽÑÑÐžÑÑÐ²Ð°ÐœÐžÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐœÐ°ÐŒÐœÐŸÐ³ÐŸ Ð±ÑÑÑÑÐµÐµ, ÑÐµÐŒ "hash". ÐÐŽÐœÐ°ÐºÐŸ, ÐŸÐœ ÐœÐµ Ð³Ð°ÑÐ°ÐœÑÐžÑÐµÑ, ÑÑÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÐºÐ°Ð¶ÐŽÑÐ¹ ÑÐ°Ð· Ð¿ÐŸÐ»ÑÑÐžÑ ÑÐŸÑ Ð¶Ðµ ÑÐ°ÐŒÑÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ, Ðž ÐœÐµ Ð¿ÐŸÐ·Ð²ÐŸÐ»ÑÐµÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑÑ ÐœÐŸÐ²ÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ðº ÑÑÑÐµÑÑÐ²ÑÑÑÐµÐŒÑ xml Ð°ÑÑ
+ÐžÐ²Ñ.
+
+
+export.saveas:Ð ÐºÐ°ÐºÐŸÐŒ ÑÐŸÑÐŒÐ°ÑÐµ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ. ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ METS.
+
+
+export.saveas.DSpace:ÐÐµÐœÐµÑÐžÑÑÐµÑ ÑÐŸÑÐŒÐ°Ñ DSpace Archive.
+
+
+export.saveas.METS:ÐÐµÐœÐµÑÐžÑÑÐµÑ ÑÐŸÑÐŒÐ°Ñ METS.
+
+
+export.out:ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐµÐ³ÐŸ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ, Ð² ÐºÐŸÑÐŸÑÑÐ¹ Ð±ÑÐŽÐµÑ Ð·Ð°Ð¿ÐžÑÐ°Ðœ ÑÑÐ°ÑÑÑ Ð²ÑÐ²ÐŸÐŽÐ°.
+
+
+export.params:[ÐŸÐ¿ÑÐžÐž] ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+
+
+export.removeold:Ð£ÐŽÐ°Ð»ÐžÑ ÑÑÐ°ÑÐŸÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž -- ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ ÐŸÑÑÐŸÑÐŸÐ¶ÐœÐŸ.
+
+
+export.removing_export:Ð£ÐŽÐ°Ð»ÐµÐœÐžÐµ ÐœÐ°ÑÑÐŸÑÑÐµÐ³ÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ ÑÐºÑÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž...
+
+
+export.sortmeta:Ð¡ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµÐŒ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž. Ð­ÑÐŸ Ð±ÑÐŽÐµÑ Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÐŸ, ÐµÑÐ»Ðž ÑÐ°Ð·ÐŒÐµÑ Ð³ÑÑÐ¿Ð¿Ñ> 1.
+
+
+export.statsfile:ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÐžÐ»Ðž ÐµÐ³ÐŸ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐŸÑ, Ð² ÐºÐŸÑÐŸÑÑÐ¹ Ð±ÑÐŽÑÑ Ð·Ð°Ð¿ÐžÑÐ°ÐœÑ ÑÑÐ°ÑÐžÑÑÐžÐºÐž ÑÐºÑÐ¿ÐŸÑÑÐ°.
+
+
+export.stats_backup:ÐÐ·Ð°ÐŒÐµÐœ Ð²ÑÐ²ÐŸÐŽÐžÑ ÐœÐ° STDERR (Ð¿ÑÐŸÑÐŸÐºÐŸÐ» ÐŸÑÐžÐ±ÐŸÐº).
+
+
+export.verbosity:ÐÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÑ ÐºÐŸÐ»Ð»ÐžÑÐµÑÑÐ²ÐŸ Ð²ÑÐ²ÐŸÐŽÐ°.  0=ÐœÐžÑÐµÐ³ÐŸ, 3=ÐŒÐœÐŸÐ³ÐŸ.
+
+
+
+# -- mkcol.pl --
+
+mkcol.about:Ð ÑÐµÐºÑÑÐµ ÐŽÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+mkcol.bad_name_cvs:ÐÐšÐÐÐÐ: ÐÐžÐºÐ°ÐºÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐœÐ°Ð·Ð²Ð°ÐœÐ° CVS, Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ ÑÑÐŸ ÐŒÐŸÐ¶ÐµÑ ÐœÐ°Ð»ÐŸÐ¶ÐžÑÑÑÑ ÐœÐ° ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž, ÑÐŸÐ·ÐŽÐ°ÐœÐœÑÐµ Ð²ÐµÑÑÐžÐµÐ¹ CVS ÑÐžÑÑÐµÐŒÑ.
+
+
+mkcol.bad_name_modelcol:ÐÐšÐÐÐÐ: ÐÐžÐºÐ°ÐºÐ°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐœÐµ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐœÐ°Ð·Ð²Ð°ÐœÐ° modelcol, Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ ÑÑÐŸ ÐžÐŒÑ ÐŒÐŸÐŽÐµÐ»ÑÐœÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+mkcol.cannot_find_modelcol:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐ°Ð¹ÑÐž ÐŒÐŸÐŽÐµÐ»ÑÐœÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ %s
+
+
+mkcol.col_already_exists:ÐÐšÐÐÐÐ: Ð­ÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÑÐ¶Ðµ ÑÑÑÐµÑÑÐ²ÑÐµÑ.
+
+
+mkcol.collectdir:ÐÐžÑÐµÐºÑÐŸÑÐžÑ, Ð³ÐŽÐµ ÐœÐŸÐ²Ð°Ñ ÐºÐŸÐ»ÐµÐºÑÐžÑ Ð±ÑÐŽÐµÑ ÑÐŸÐ·ÐŽÐ°ÐœÐ°.
+
+
+mkcol.creating_col:Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž %s
+
+
+mkcol.creator:ÐÐŽÑÐµÑ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ ÑÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+mkcol.creator_undefined:ÐÐšÐÐÐÐ: ÑÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ ÐœÐµ Ð±ÑÐ» ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœ. Ð­ÑÐ° Ð¿ÐµÑÐµÐŒÐµÐœÐœÐ°Ñ ÐœÐµÐŸÐ±Ñ
+ÐŸÐŽÐžÐŒÐ°, ÑÑÐŸÐ±Ñ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÑÑ ÐŽÑÐ±Ð»ÐžÐºÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+mkcol.desc:Ð¡ÐºÑÐžÐ¿Ñ PERL, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ ÐŽÐ»Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÑÑÑÑÐºÑÑÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž ÐŽÐ»Ñ ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Greenstone.
+
+
+mkcol.doing_replacements:Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÐµ Ð·Ð°ÐŒÐµÐœ ÐŽÐ»Ñ %s
+
+
+mkcol.long_colname:ÐÐšÐÐÐÐ: ÐÐŒÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŽÐŸÐ»Ð¶ÐœÐŸ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ ÐŒÐµÐœÐµÐµ 8 Ð·ÐœÐ°ÐºÐŸÐ², ÑÑÐŸÐ±Ñ Ð±ÑÑÑ ÑÐŸÐ²ÐŒÐµÑÑÐžÐŒÑÐŒ  Ñ Ð±ÐŸÐ»ÐµÐµ ÑÐ°ÐœÐœÐžÐŒÐž ÑÐ°Ð¹Ð»ÐŸÐ²ÑÐŒÐž ÑÐžÑÑÐµÐŒÐ°ÐŒÐž Ðž ÐžÐŒÐµÑÑ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÑ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐºÐž.
+
+
+mkcol.maintainer:ÐÐŽÑÐµÑ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ Ð»ÐžÑÐ°, Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÑÑÐµÐ³ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ (ÐµÑÐ»Ðž ÐŸÐœ ÐŸÑÐ»ÐžÑÐ°ÐµÑÑÑ ÐŸÑ ÑÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ).
+
+
+mkcol.no_collectdir:ÐÐšÐÐÐÐ: ÐÐžÑÐµÐºÑÐŸÑÐžÑ collect ÐœÐµ ÑÑÑÐµÑÑÐ²ÑÐµÑ: %s
+
+
+mkcol.no_colname:ÐÐšÐÐÐÐ: ÐÐžÐºÐ°ÐºÐŸÐ³ÐŸ ÐžÐŒÐµÐœÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐœÐµ Ð±ÑÐ»ÐŸ Ð·Ð°ÐŽÐ°ÐœÐŸ.
+
+
+mkcol.optionfile:ÐÐŸÐ»ÑÑÐžÑÑ ÐŸÐ¿ÑÐžÐž ÑÐ°Ð¹Ð»Ð°, Ð¿ÐŸÐ»ÐµÐ·ÐœÐŸÐ³ÐŸ ÐŽÐ»Ñ ÑÐžÑÑÐµÐŒ, Ð³ÐŽÐµ ÐŽÐ»ÐžÐœÐœÑÐµ ÐºÐŸÐŒÐ°ÐœÐŽÐœÑÐµ ÑÑÑÐŸÐºÐž ÐŒÐŸÐ³ÑÑ Ð²ÑÐ·ÑÐ²Ð°ÑÑ Ð¿ÑÐŸÐ±Ð»ÐµÐŒÑ.
+
+
+mkcol.params:[ÐŸÐ¿ÑÐžÐž] ÐžÐŒÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž
+
+
+mkcol.plugin:ÐÐŸÐŽÑÐ»Ñ Ð¿Ð»Ð°Ð³ÐžÐœÐ° Perl ÐŽÐ»Ñ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ (Ð·ÐŽÐµÑÑ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐŒÐœÐŸÐ³ÐŸÐºÑÐ°ÑÐœÑÐµ Ð²Ñ
+ÐŸÐ¶ÐŽÐµÐœÐžÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ°).
+
+
+mkcol.public:ÐÑÐ»Ðž ÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ ÐžÐŒÐµÐµÑ Ð°ÐœÐŸÐœÐžÐŒÐœÑÐ¹  ÐŽÐŸÑÑÑÐ¿ (Ð²ÐµÑÐœÑÐ¹/Ð»ÐŸÐ¶ÐœÑÐ¹).
+
+
+mkcol.quiet:Ð Ð°Ð±ÐŸÑÐ°ÑÑ ÑÐ¿ÐŸÐºÐŸÐ¹ÐœÐŸ.
+
+
+mkcol.success:ÐÐŸÐ²Ð°Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ Ð±ÑÐ»Ð° ÑÐŸÐ·ÐŽÐ°ÐœÐ° ÑÑÐ¿ÐµÑÐœÐŸ Ð² %s
+
+
+mkcol.title:ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+mkcol.win31compat:Ð£ÑÐžÑÑÐ²Ð°ÑÑ Ð»Ðž ÐºÐŸÐœÐ²ÐµÐœÑÐžÐž Windows 3.1, Ð¿ÑÐž Ð²ÑÐ±ÐŸÑÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÐŽÐ»Ñ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐœÐµ Ð±ÐŸÐ»ÐµÐµ 8 Ð·ÐœÐ°ÐºÐŸÐ²).
+
+
+# -- pluginfo.pl --
+
+pluginfo.collect:ÐÑÐµÐŽÐŸÑÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐžÐŒÐµÐœÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ð±ÑÐŽÐµÑ Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐŸ Ð¿ÑÐž Ð¿ÑÐŸÑÐŒÐŸÑÑÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² Ð¿ÑÐµÐ¶ÐŽÐµ Ð²ÑÐµÐ³ÐŸ  Ð² collect/collection-name/perllib/plugins.  ÐÑÐ»Ðž Ð¿Ð»Ð°Ð³ÐžÐœ ÑÐ°ÐŒ ÐœÐµ ÐœÐ°Ð¹ÐŽÐµÐœ, ÑÐŸ ÐœÐ°ÐŽÐŸ Ð±ÑÐŽÐµÑ Ð¿ÑÐŸÑÐŒÐŸÑÑÐµÑÑ Ð² ÐŸÐ±ÑÐµÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž perllib/plugins.
+
+
+pluginfo.desc:ÐÐ°Ð¿ÐžÑÐ°ÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð¿Ð»Ð°Ð³ÐžÐœÐ°Ñ
+.
+
+
+pluginfo.general_options:ÐÐ±ÑÐžÐµ ÐŸÐ¿ÑÐžÐž, ÑÐœÐ°ÑÐ»ÐµÐŽÐŸÐ²Ð°ÐœÐœÑÐµ ÐŸÑ ÑÐŸÐŽÐžÑÐµÐ»ÑÑÐºÐžÑ
+ ÐºÐ»Ð°ÑÑÐŸÐ² Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ².
+
+
+pluginfo.info:ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ
+
+
+pluginfo.no_plugin_name:ÐÐšÐÐÐÐ: ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ Ð·Ð°ÐŽÐ°ÑÑ ÐžÐŒÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ°.
+
+
+pluginfo.option_types:ÐÐ»Ð°Ð³ÐžÐœÑ ÐŒÐŸÐ³ÑÑ ÐžÐŒÐµÑÑ ÐŽÐ²Ð° ÑÐžÐ¿Ð° ÐŸÐ¿ÑÐžÐ¹.
+
+
+pluginfo.params:[ÐŸÐ¿ÑÐžÐž] ÐžÐŒÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ°
+
+
+pluginfo.passing_options:ÐÐ¿ÑÐžÐž ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ Ð¿ÐµÑÐµÐŽÐ°ÐœÑ Ð»ÑÐ±ÑÐŒ Ð¿Ð»Ð°Ð³ÐžÐœÐ°ÐŒ Ð²ÐºÐ»ÑÑÐµÐœÐžÐµÐŒ ÐžÑ
+ Ð² ÐÐ°Ñ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» collect.cfg.
+
+
+pluginfo.specific_options:Ð¡Ð¿ÐµÑÐžÑÐžÑÐµÑÐºÐžÐµ ÐŸÐ¿ÑÐžÐž ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÑÐ°ÐŒÐžÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² Ðž ÐŽÐŸÑÑÑÐ¿ÐœÑ ÑÐŸÐ»ÑÐºÐŸ ÑÑÐŸÐŒÑ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐŒÑ Ð¿Ð»Ð°Ð³ÐžÐœÑ.
+
+
+
+#
+# Classifier option descriptions
+#
+
+AZCompactList.allvalues:ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð²ÑÐµ ÐœÐ°Ð¹ÐŽÐµÐœÐœÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+.
+
+AZCompactList.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ
+
+
+AZCompactList.doclevel:Ð£ÑÐŸÐ²ÐµÐœÑ ÐŽÐ»Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° Ð².
+
+AZCompactList.doclevel.top:ÐŠÐµÐ»ÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ.
+
+AZCompactList.doclevel.section:Ð Ð°Ð·ÐŽÐµÐ»Ð°ÐŒÐž.
+
+
+AZCompactList.firstvalueonly:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÑÐŸÐ»ÑÐºÐŸ Ð¿ÐµÑÐ²ÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.
+
+
+AZCompactList.freqsort:Ð¡ÐŸÑÑÐžÑÐŸÐ²Ð°ÑÑ Ð¿ÐŸ ÑÐ°ÑÑÐŸÑÐµ ÑÐ·Ð»Ð°, Ð° ÐœÐµ Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸ-ÑÐžÑÑÐŸÐ²ÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ.
+
+
+AZCompactList.maxcompact:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÐœÐ° ÑÑÑÐ°ÐœÐžÑÐµ.
+
+
+AZCompactList.metadata:ÐÐŽÐžÐœÑÑÐ²ÐµÐœÐœÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÐ»Ðž Ð·Ð°Ð¿ÑÑÐ°Ñ ÑÐ°Ð·ÐŽÐµÐ»ÐžÑÐµÐ»Ñ Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž. ÐÑÐ»Ðž ÑÐ¿ÐžÑÐŸÐº ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœ, Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð¿ÐµÑÐ²ÑÐ¹ ÑÐžÐ¿ ÐŒÐµÑÐ°Ð»ÐŽÐ°ÐœÐœÑÑ
+, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ±Ð»Ð°ÐŽÐ°ÐµÑ Ð·ÐœÐ°ÑÐµÐœÐžÑÐŒÐž. ÐÐŸÐ¶ÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ Ð² ÑÐŸÐµÐŽÐžÐœÐµÐœÐžÐž Ñ -firstvalueonly Ðž ÑÐ»Ð°Ð³Ð°ÐŒÐž -allvalues, ÐŽÐ»Ñ Ð²ÑÐ±ÐŸÑÐ° ÑÐŸÐ»ÑÐºÐŸ Ð¿ÐµÑÐ²ÐŸÐ³ÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÑ, ÐžÐ»Ðž Ð²ÑÐµÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ· ÑÐ¿ÐžÑÐºÐ°.
+
+
+AZCompactList.mincompact:ÐÐžÐœÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÐœÐ° ÑÑÑÐ°ÐœÐžÑÐµ.
+
+
+AZCompactList.mingroup:ÐÐ°ÐžÐŒÐµÐœÑÑÐ°Ñ ÑÐžÑÐ»ÐŸ, ÐºÐŸÑÐŸÑÐŸÐµ Ð²ÑÐ·ÑÐ²Ð°ÐµÑ ÑÐŸÑÐŒÐžÑÐŸÐ²Ð°ÐœÐžÐµ Ð³ÑÑÐ¿Ð¿ Ð² ÐžÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ.
+
+
+AZCompactList.minnesting:ÐÐ°ÐžÐŒÐµÐœÑÑÐµÐµ ÑÐžÑÐ»ÐŸ, ÐºÐŸÑÐŸÑÐŸÐµ Ð²ÑÐ·ÐŸÐ²ÐµÑ Ð¿ÑÐµÐŸÐ±ÑÐ°Ð·ÐŸÐ²Ð°ÐœÐžÐµ ÑÐ¿ÐžÑÐºÐ° Ð²ÐŸ Ð²Ð»ÐŸÐ¶ÐµÐœÐœÑÐ¹ ÑÐ¿ÐžÑÐŸÐº.
+
+
+AZCompactList.recopt:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ Ð²ÐŸ Ð²Ð»ÐŸÐ¶ÐµÐœÐœÑÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÑÐžÐ¿Ð° -ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÐŸÐŽÐ°/ÐÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÐž.
+
+AZCompactList.sort:ÐÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž ÐŽÐŸÑÐµÑÐœÐžÑ
+ ÑÐ·Ð»ÐŸÐ² Ñ Ð¿ÐŸÐŒÐŸÑÑÑ
+
+
+AZCompactSectionList.desc:ÐÐ°ÑÐžÐ°ÑÐžÐž Ð² AZCompactList, ÐºÐŸÑÐŸÑÑÐ¹ ÐºÐ»Ð°ÑÑÐžÑÐžÑÐžÑÑÐµÑ ÑÐ°Ð·ÐŽÐµÐ»Ñ, Ð° ÐœÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ. ÐÑ
+ÐŸÐ¶ÐŽÐµÐœÐžÑ ÐŸÑÑÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÑ Ð¿ÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒ ÑÑÐŸÐ²ÐœÑ ÑÐ°Ð·ÐŽÐµÐ»Ð°.
+
+
+AZList.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ
+
+
+AZList.metadata:ÐÐŽÐžÐœÑÑÐ²ÐµÐœÐœÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ»Ðž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐœÑÐ¹ Ð·Ð°Ð¿ÑÑÐŸÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÑ
+ ÐŽÐ»Ñ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž. Ð¡Ð»ÐµÐŽÑÑ Ð¿ÐŸÑÑÐŽÐºÑ, ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐŸÐŒÑ Ð² ÑÐ¿ÐžÑÐºÐµ, Ð¿ÐµÑÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ, ÐºÐŸÑÐŸÑÐŸÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ. Ð¡Ð¿ÐžÑÐŸÐº Ð±ÑÐŽÐµÑ ÐŸÑÑÐŸÑÑÐžÑÐŸÐ²Ð°Ðœ Ð¿ÐŸ ÑÑÐŸÐŒÑ ÑÐ»ÐµÐŒÐµÐœÑÑ.
+
+
+AZSectionList.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð² Ð°Ð»ÑÐ°Ð²ÐžÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ. ÐÐœ ÐŸÑÐµÐœÑ Ð¿ÐŸÑ
+ÐŸÐ¶ ÐœÐ° AZList, Ð·Ð° ÐžÑÐºÐ»ÑÑÐµÐœÐžÐµÐŒ ÑÐŸÐ³ÐŸ, ÑÑÐŸ ÐŸÐœ ÑÐŸÑÑÐžÑÑÐµÑ Ð¿ÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒ ÑÑÐŸÐ²ÐœÑ ÑÐ°Ð·ÐŽÐµÐ»Ð° (ÐžÑÐºÐ»ÑÑÐ°Ñ Ð²ÐµÑÑ
+ÐœÐžÐ¹ ÑÑÐŸÐ²ÐµÐœÑ) Ð²ÐŒÐµÑÑÐŸ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž ÑÐŸÐ»ÑÐºÐŸ  Ð¿ÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒ Ð²ÐµÑÑ
+ÐœÐµÐ³ÐŸ ÑÑÐŸÐ²ÐœÑ. ÐÐŽÐžÐœÑÑÐ²ÐµÐœÐœÐŸÐµ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÐµ ÐŸÑÐœÐŸÑÐžÑÑÑ Ðº Ð¿ÐŸÐŽÐ¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐµ  () ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž, ÐºÐŸÑÐŸÑÐ°Ñ  ÑÐµÐ¿ÐµÑÑ ÐŽÐŸÐ»Ð¶ÐœÐ° Ð¿ÑÐŸÐ²ÐŸÐŽÐžÑÑ ÐžÑÐµÑÐ°ÑÐžÑ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð·ÐŽÐµÐ»Ð°, ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑ ÐºÐ°Ð¶ÐŽÑÐ¹ Ð² ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÑ.
+
+
+BasClas.bad_general_option:ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ %s  ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ ÐœÐµÐºÐŸÑÑÐµÐºÑÐœÑÑ ÐŸÐ±ÑÑÑ ÐŸÐ¿ÑÐžÑ (ÐŸÐ±ÑÐžÐµ ÐŸÐ¿ÑÐžÐž  - ÑÑÐŸ ÑÐ°ÐºÐžÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÑÑÑÐ¿ÐœÑ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ²). ÐÑÐŸÐ²ÐµÑÑÑÐµ ÐÐ°Ñ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» collect.cfg.
+
+
+BasClas.builddir:ÐÐŽÐµ Ð¿ÐŸÐŒÐµÑÑÐžÑÑ Ð¿ÐŸÑÑÑÐŸÐµÐœÐœÑÐµ ÐžÐœÐŽÐµÐºÑÑ.
+
+
+BasClas.buttonname:Ð¯ÑÐ»ÑÐº ÐŽÐ»Ñ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Ð¿ÐŸÐºÐ°Ð·Ð° ÐœÐ° ÑÐºÑÐ°ÐœÐµ Ðž ÐºÐœÐŸÐ¿ÐºÐ° Ð² ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐŸÐœÐœÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ.
+
+
+BasClas.desc:ÐÑÐœÐŸÐ²ÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑ ÐŽÐ»Ñ Ð²ÑÐµÑ
+ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ².
+
+
+BasClas.outhandle:ÐÐœÐŽÐµÐºÑÐœÑÐ¹ ÑÐ°Ð¹Ð» ÐŽÐ»Ñ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ ÑÐµÐ·ÑÐ»ÑÑÐ°ÑÐ°.
+
+
+BasClas.removeprefix:ÐÑÐµÑÐžÐºÑ, Ñ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐœÐ°ÑÐžÐœÐ°ÑÑÑÑ ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ ÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð¿ÑÐž ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž.
+
+BasClas.removesuffix:Ð¡ÑÑÑÐžÐºÑ, Ñ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐœÐ°ÑÐžÐœÐ°ÑÑÑÑ ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ ÐžÐ³ÐœÐŸÑÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð¿ÑÐž ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž.
+
+BasClas.verbosity:Ð£Ð¿ÑÐ°Ð²Ð»ÐµÐœÐžÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸÐŒ Ð¿ÑÐŸÐŽÑÐºÑÐžÐž. 0=ÐŸÑÑÑÑÑÑÐ²ÐžÐµ, 3=Ð±ÐŸÐ»ÑÑÐžÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²Ð°.
+
+
+Browse.desc:.
+
+
+DateList.bymonth:ÐÐ»Ð°ÑÑÐžÑÐžÑÐžÑÐŸÐ²Ð°ÑÑ Ð¿ÐŸ Ð³ÐŸÐŽÑ Ðž ÐŒÐµÑÑÑÑ Ð²ÐŒÐµÑÑÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐŸÐ»ÑÐºÐŸ Ð¿ÐŸ Ð³ÐŸÐŽÑ
+
+
+DateList.desc:ÐÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐŸÐœÐœÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð¿ÐŸ ÐŽÐ°ÑÐ°ÐŒ. ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐ° Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ Ð¿ÐŸ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒ 'ÐÐ°ÑÐ°'. ÐÑÐžÐœÑÑÐŸ, ÑÑÐŸ ÐŽÐ°ÑÐ° ÐŽÐŸÐ»Ð¶ÐœÐ° ÐžÐŒÐµÑÑ Ð²ÐžÐŽ Ð³Ð³Ð³Ð³ÐŒÐŒÐŽÐŽ.
+
+
+DateList.metadata:ÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÐŽÐ°ÑÑ Ñ ÑÐµÐ»ÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž. ÐÑÐµÐŽÐ¿ÐŸÐ»Ð°Ð³Ð°ÐµÑÑÑ, ÑÑÐŸ ÑÐŸÑÐŒÐ°Ñ Ð±ÑÐŽÐµÑ Ð³Ð³Ð³Ð³ÐŒÐŒÐŽÐŽ.
+
+
+DateList.reverse_sort:Ð¡ÐŸÑÑÐžÑÐŸÐ²ÐºÐ° ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ð² ÐŸÐ±ÑÐ°ÑÐœÐŸÐŒ Ñ
+ÑÐŸÐœÐŸÐ»ÐŸÐ³ÐžÑÐµÑÐºÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ (ÐœÐŸÐ²ÐµÐ¹ÑÐžÐ¹ Ð²ÐœÐ°ÑÐ°Ð»Ðµ).
+
+
+DateList.nogroup:Ð¡ÐŽÐµÐ»Ð°ÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ Ð³ÐŸÐŽÐ° ÐžÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÐ¹ Ð²Ñ
+ÐŸÐŽ Ð² Ð³ÐŸÑÐžÐ·ÐŸÐœÑÐ°Ð»ÑÐœÐŸÐŒ ÑÐ¿ÐžÑÐºÐµ, Ð²ÐŒÐµÑÑÐŸ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ ÐŸÑ
+Ð²Ð°ÑÐžÑÑ Ð³ÐŸÐŽÑ ÐœÐµÑÐºÐŸÐ»ÑÐºÐžÐŒÐž Ð²Ñ
+ÐŸÐŽÐ°ÐŒÐž. (Ð­ÑÐŸ ÐŒÐŸÐ¶ÐµÑ ÑÐ°ÐºÐ¶Ðµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ ÐŽÐ»Ñ ÐŽÐ²ÑÑ
+ÐŒÐµÑÑÑÐœÐŸÐ¹ ÐŸÐ¿ÑÐžÐž, ÐŽÐµÐ»Ð°Ñ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŒÐµÑÑÑÐ° ÐŸÑÐŽÐµÐ»ÑÐœÑÐ¹ Ð²Ñ
+ÐŸÐŽ Ð²ÐŒÐµÑÑÐŸ ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÐŸÐ³ÐŸ).
+
+
+DateList.sort:ÐÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž Ð² ÑÐ»ÑÑÐ°Ðµ, ÐºÐŸÐ³ÐŽÐ° ÐŽÐ²Ð° ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐžÐŒÐµÑÑ ÐŸÐŽÐœÑ Ðž ÑÑ Ð¶Ðµ  ÐŽÐ°ÑÑ.
+
+
+HFileHierarchy.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° ÐŽÐ»Ñ Ð³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐžÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐžÑ
+ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐ¹, Ð¿ÐŸÑÑÑÐŸÐµÐœÐœÑÑ
+ ÐœÐ° ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÑÐŸ ÑÑÑÑÐºÑÑÑÐŸÐ¹.
+
+Hierarchy.hfile:Ð¡ÑÑÑÐºÑÑÑÐ° ÑÐ°Ð¹Ð»Ð° ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž.
+
+Hierarchy.hlist_at_top:ÐÐŸÐºÐ°Ð·Ð°ÑÑ Ð¿ÐµÑÐ²ÑÐ¹ ÑÑÐŸÐ²ÐµÐœÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž Ð³ÐŸÑÐžÐ·ÐŸÐœÑÐ°Ð»ÑÐœÐŸ.
+
+Hierarchy.reverse_sort:Ð¡ÐŸÑÑÐžÑÐŸÐ²ÐºÐ° Ð»ÐžÑÑÑÐµÐ² ÐŽÐµÑÐµÐ²Ð° (ÐºÐŸÐœÑÐµÐ²ÑÑ
+ Ð²ÐµÑÑÐžÐœ) Ð² ÐŸÐ±ÑÐ°ÑÐœÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ (ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ñ -sort).
+
+Hierarchy.separator:Ð ÐµÐ³ÑÐ»ÑÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐœÐŸÐµ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÑÐ°Ð·ÐŽÐµÐ»ÐžÑÐµÐ»Ñ.
+
+
+Hierarchy.sort:ÐÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ'-sort nosort', ÐµÑÐ»Ðž ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐ° ÐœÐµ ÐœÑÐ¶ÐœÐ°.
+
+Hierarchy.suppresslastlevel:ÐÐ³ÐœÐŸÑÐžÑÑÐ¹ÑÐµ Ð¿ÐŸÑÐ»ÐµÐŽÐœÑÑ ÑÐ°ÑÑÑ Ð² Ð·ÐœÐ°ÑÐµÐœÐžÐž ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+. Ð­ÑÐŸ Ð¿ÐŸÐ»ÐµÐ·ÐœÐŸ ÐŽÐ»Ñ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, Ð² ÐºÐŸÑÐŸÑÑÑ
+ ÐºÐ°Ð¶ÐŽÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŸÐŽÐžÐœÐ°ÐºÐŸÐ²ÐŸ, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð²ÐµÑÐ²Ñ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž.
+
+HTML.desc:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑ Ð¿ÑÑÑÑÑ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÑÐ²Ð»ÑÐµÑÑÑ Ð¿ÑÐŸÑÑÐŸ ÑÑÑÐ»ÐºÐŸÐ¹ ÐœÐ° Ð²ÐµÐ±-ÑÑÑÐ°ÐœÐžÑÑ.
+
+
+HTML.url:URL Ð²ÐµÐ±-ÑÑÑÐ°ÐœÐžÑÑ ÐŽÐ»Ñ ÑÐ²ÑÐ·Ðž Ñ.
+
+
+List.desc:ÐÑÐŸÑÑÐŸÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž.
+
+
+List.metadata:ÐÐŽÐžÐœÑÑÐ²ÐµÐœÐœÐŸÐµ Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÐ»Ðž ÑÐ°Ð·ÐŽÐµÐ»ÐµÐœÐœÑÐ¹ Ð·Ð°Ð¿ÑÑÐŸÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÑ
+ ÐŽÐ»Ñ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž. Ð¡Ð»ÐµÐŽÑÑ Ð¿ÐŸÑÑÐŽÐºÑ, ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐµÐœÐœÐŸÐŒÑ Ð² ÑÐ¿ÐžÑÐºÐµ, Ð¿ÐµÑÐ²ÐŸÐµ Ð¿ÐŸÐ»Ðµ, ÐºÐŸÑÐŸÑÐŸÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, Ð±ÑÐŽÐµÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑÑÑ. Ð¡Ð¿ÐžÑÐŸÐº Ð±ÑÐŽÐµÑ ÐŸÑÑÐŸÑÑÐžÑÐŸÐ²Ð°Ðœ ÑÑÐžÐŒ ÑÐ»ÐµÐŒÐµÐœÑÐŸÐŒ, ÐµÑÐ»Ðž ÐœÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ -sort.
+
+
+List.sort:ÐÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐ»Ñ ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐž. ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ'-sort nosort', ÐµÑÐ»Ðž ÑÐŸÑÑÐžÑÐŸÐ²ÐºÐ° ÐœÐµ ÐœÑÐ¶ÐœÐ°.
+
+
+Phind.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Phind.
+
+
+Phind.language:Ð¯Ð·ÑÐº ÐžÐ»Ðž ÑÐ·ÑÐºÐž, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐµ ÐŽÐ»Ñ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ ÐžÐµÑÐ°ÑÑ
+ÐžÐž. Ð¯Ð·ÑÐºÐž ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐ°ÑÑÑÑ ÐŽÐ²ÑÑ
+Ð±ÑÐºÐ²ÐµÐœÐœÑÐŒÐž ÐºÐŸÐŽÐ°ÐŒÐž ÑÑÑÐ°Ðœ, Ðº Ð¿ÑÐžÐŒÐµÑÑ, en (Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐžÐ¹ ÑÐ·ÑÐº), es (ÐžÑÐ¿Ð°ÐœÑÐºÐžÐ¹ ÑÐ·ÑÐº) Ðž fr (ÑÑÐ°ÐœÑÑÐ·ÑÐºÐžÐ¹ ÑÐ·ÑÐº). Ð¯Ð·ÑÐº ÑÐ²Ð»ÑÐµÑÑÑ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÑÐŒ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµÐŒ, ÑÐ°Ðº 'en|fr' (Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐžÐ¹ ÐžÐ»Ðž ÑÑÐ°ÐœÑÑÐ·ÑÐºÐžÐ¹ ÑÐ·ÑÐº) Ðž '..' (ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸ, Ð»ÑÐ±ÐŸÐ¹ ÑÐ·ÑÐº) Ð¿ÑÐžÐ³ÐŸÐŽÐœÑ.
+
+
+Phind.min_occurs:ÐÐžÐœÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐŸÐ±ÐŸÐ·ÐœÐ°ÑÐ°ÑÑÐµÐµ, ÑÐºÐŸÐ»ÐºÐŸ ÑÐ°Ð· ÑÑÐ°Ð·Ð° ÐŽÐŸÐ»Ð¶ÐœÐ° Ð²ÑÑÑÐµÑÐžÑÑÑÑ Ð² ÑÐµÐºÑÑÐµ, ÑÑÐŸÐ±Ñ Ð±ÑÑÑ Ð²ÐºÐ»ÑÑÐµÐœÐœÐŸÐ¹ Ð²ÐŸ ÑÑÐ°Ð·ÐŸÐ²ÑÑ ÐžÐµÑÐ°ÑÑ
+ÐžÑ.
+
+
+Phind.savephrases:ÐÑÐ»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐŸ, ÑÐŸ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÐŸÐœÐœÐ°Ñ ÑÑÐ°Ð·Ð° Ð±ÑÐŽÐµÑ Ñ
+ÑÐ°ÐœÐžÑÑÑÑ Ð² Ð·Ð°ÐŽÐ°ÐœÐœÐŸÐŒ ÑÐ°Ð¹Ð»Ðµ ÐºÐ°Ðº ÑÐµÐºÑÑ. Ð­ÑÐŸ  Ð²ÐµÑÐŸÑÑÐœÐŸ Ñ
+ÐŸÑÐŸÑÐ°Ñ ÐžÐŽÐµÑ - ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ð°Ð±ÑÐŸÐ»ÑÑÐœÑÐ¹ Ð¿ÑÑÑ ÐŽÐŸÑÑÑÐ¿Ð°.
+
+
+Phind.suffixmode:Smode Ð¿Ð°ÑÐ°ÐŒÐµÑÑ Ðº Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐµ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÑÑÐ°Ð·Ñ. ÐÐœÐ°ÑÐµÐœÐžÐµ 0 ÐŸÐ·ÐœÐ°ÑÐ°ÐµÑ, ÑÑÐŸ ÑÑÐŸÐ¿-ÑÐ»ÐŸÐ²Ð° ÐžÐ³ÐœÐŸÑÐžÑÑÑÑÑÑ, Ð° 1  -  ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ.
+
+
+Phind.text:Ð¢ÐµÐºÑÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÐ¹ ÐŽÐ»Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÐžÐµÑÐ°ÑÑ
+ÐžÐž ÑÑÐ°Ð·.
+
+
+Phind.thesaurus:ÐÐŒÑ ÑÐµÐ·Ð°ÑÑÑÑÐ°, Ñ
+ÑÐ°ÐœÑÑÐµÐ³ÐŸÑÑ Ð² Phind-ÑÐŸÑÐŒÐ°ÑÐµ Ð² ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ñ.ÐŽ.
+
+
+Phind.title:ÐÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐµ ÐŽÐ»Ñ ÐŸÐ¿ÐžÑÐ°ÐœÐžÑ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+
+Phind.untidy:ÐÐµ ÑÐŽÐ°Ð»ÑÑÑ ÑÐ°Ð±ÐŸÑÐžÐµ ÑÐ°Ð¹Ð»Ñ.
+
+
+SectionList.desc:Ð¢ÐŸ Ð¶Ðµ ÑÐ°ÐŒÐŸÐµ, ÑÑÐŸ Ðž ÑÐ¿ÐžÑÐŸÐº ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐŸÐ², ÐœÐŸ Ð²ÐºÐ»ÑÑÐ°ÐµÑ Ð²ÑÐµ ÑÐ°Ð·ÐŽÐµÐ»Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° (ÐžÑÐºÐ»ÑÑÐ°Ñ Ð²ÑÑÑÐžÐ¹ ÑÑÐŸÐ²ÐµÐœÑ), Ð° ÐœÐµ ÑÐŸÐ»ÑÐºÐŸ Ð²ÑÑÑÐžÐ¹ ÑÑÐŸÐ²ÐµÐœÑ ÑÐ°ÐŒÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+Collage.buttonname:Ð¯ÑÐ»ÑÐº ÐŽÐ»Ñ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ° Ð¿ÐŸÐºÐ°Ð·Ð° ÐœÐ° ÑÐºÑÐ°ÐœÐµ Ðž ÐºÐœÐŸÐ¿ÐºÐ° Ð² ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐŸÐœÐœÐŸÐ¹ Ð»ÐžÐœÐµÐ¹ÐºÐµ.
+
+Collage.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ ÐºÐŸÐ»Ð»Ð°Ð¶Ð°.
+
+
+Collage.geometry:ÐÐ·ÐŒÐµÑÐµÐœÐžÑ ÑÐ°Ð·ÐŒÐµÑÐ° ÐºÐŸÐ»Ð»Ð°Ð¶Ð°. ÐÐ»Ñ ÐºÐŸÐ»Ð»Ð°Ð¶Ð° Ð² 600 Ð¿ÐžÐºÑÐµÐ»ÐµÐ¹ ÑÐžÑÐžÐœÐŸÐ¹ Ðž 400 Ð¿ÐžÐºÑÐµÐ»ÐµÐ¹ Ð²ÑÑÐŸÑÐŸÐ¹, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐŸÐ¿ÑÐµÐŽÐµÐ»ÑÑÑ Ð³ÐµÐŸÐŒÐµÑÑÐžÑ ÐºÐ°Ðº 600x400
+
+
+Collage.maxDepth:ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐŽÐ»Ñ ÐºÐŸÐ»Ð»Ð°Ð¶Ð° Ð¿ÑÐŸÑÐžÑÐŸÐ²Ð°ÐœÑ ÐŸÑ Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ Ðº ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÑ Ð±ÑÐ°ÑÐ·ÐµÑÐ°. Ð­ÑÐŸ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÑÐµÑ ÐŒÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÑÑ Ð³Ð»ÑÐ±ÐžÐœÑ Ð¿ÑÐŸÑÐµÑÑÐ° Ð·ÐµÑÐºÐ°Ð»ÐžÑÐŸÐ²Ð°ÐœÐžÑ.
+
+
+Collage.maxDisplay:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐžÑÐ»ÐŸ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹ Ð¿ÐŸÐºÐ°Ð·ÑÐ²Ð°ÐµÐŒÑÑ
+ ÐŸÐŽÐœÐŸÐ²ÑÐµÐŒÐµÐœÐœÐŸ Ð² ÐºÐŸÐ»Ð»Ð°Ð¶Ðµ.
+
+
+Collage.imageType: ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ, ÑÑÐŸÐ±Ñ ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑ, Ð²ÑÑÐ°Ð¶Ð°Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð°, ÑÐµÑ
+ ÑÐžÐ¿ÐŸÐ² ÑÐ°Ð¹Ð»ÐŸÐ² ÐºÐŸÑÐŸÑÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ Ð² ÐºÐŸÐ»Ð»Ð°Ð¶Ðµ. Ð¡Ð¿ÐžÑÐŸÐº ÑÐ°ÑÑÐžÑÐµÐœÐžÐ¹ ÐžÐŒÐµÐœ ÑÐ°Ð¹Ð»ÐŸÐ² ÐŸÑÐŽÐµÐ»ÐµÐœ ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐŒ Ð¿ÑÐŸÑÐµÐœÑÐ° (%%).
+
+
+Collage.bgcolor:ÐŠÐ²ÐµÑ ÑÐŸÐœÐ° Ñ
+ÐŸÐ»ÑÑÐ° ÐºÐŸÐ»Ð»Ð°Ð¶Ð°, ÑÐºÐ°Ð·Ð°ÐœÐœÐŸÐ³ÐŸ Ð² ÑÐµÑÑÐœÐ°ÐŽÑÐ°ÑÐžÑÐžÑÐœÐŸÐ¹ ÑÐŸÑÐŒÐµ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ #008000 ÑÐµÐ·ÑÐ»ÑÑÐ°Ñ: ÑÐµÐŒÐœÐŸ-Ð·ÐµÐ»ÐµÐœÑÐ¹ ÑÐŸÐœ).
+
+
+Collage.refreshDelay:ÐÐŸÑÐŒÐ°, Ð² ÐŒÐžÐ»Ð»ÐžÑÐµÐºÑÐœÐŽÐ°Ñ
+, Ñ ÐºÐŸÑÐŸÑÐŸÐ¹ ÐŸÐ±ÐœÐŸÐ²Ð»ÑÐµÑÑÑ ÐºÐŸÐ»Ð»Ð°Ð¶.
+
+
+Collage.isJava2:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ, ÑÑÐŸÐ±Ñ ÑÐ¿ÑÐ°Ð²Ð»ÑÑÑ, ÐœÐµÐºÐŸÑÐŸÑÑÐŒÐž  ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÐžÐŒÐžÑÑ ÐºÐ»Ð°ÑÑÐ°ÐŒÐž-Ð²ÑÐ¿ÐŸÐ»ÐœÐµÐœÐžÑ Java. ÐÐŸÐ»ÐµÐµ Ð¿ÑÐŸÐŽÐ²ÐžÐœÑÑÐ°Ñ Ð²ÐµÑÑÐžÑ Java (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ: Java 1.2 Ðž ÐŽÐ°Ð»ÐµÐµ) Ð²ÐºÐ»ÑÑÐ°ÐµÑ Ð±ÐŸÐ»ÐµÐµ ÐžÑÐºÑÑÐµÐœÐœÑÑ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐºÑ ÐŽÐ»Ñ ÑÐ¿ÑÐ°Ð²Ð»ÐµÐœÐžÑ Ð¿ÑÐŸÐ·ÑÐ°ÑÐœÐŸÑÑÑÑ Ð² ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑÑ
+, ÑÑÐŸ Ð¿ÐŸÐŒÐŸÐ³Ð°ÐµÑ ÐºÐŸÐœÑÑÐŸÐ»ÐžÑÐŸÐ²Ð°ÑÑ ÑÑÐŸ ÑÐ»ÑÑÐ°ÐµÑÑÑ Ñ ÑÐ»Ð°Ð³ÐŸÐŒ, ÐŸÐŽÐœÐ°ÐºÐŸ Ð²ÑÑÑÐŸÐµÐœÐœÐ°Ñ Ð²ÐµÑÑÐžÑ Java ÐŽÐ»Ñ ÐœÐµÐºÐŸÑÐŸÑÑÑ
+ Ð±ÑÐ°ÑÐ·ÐµÑÐŸÐ² - Ð²ÐµÑÑÐžÑ 1.1. ÐÐ¿Ð¿Ð»ÐµÑ ÑÐ°Ð·ÑÐ°Ð±ÐŸÑÐ°Ðœ ÑÐ°Ðº, ÑÑÐŸÐ±Ñ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ,  ÐŸÐ±ÐœÐ°ÑÑÐ¶ÐžÐ²Ð°Ð»ÑÑ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž, ÐŽÐ»Ñ ÑÐŸÐ¹ Ð²ÐµÑÑÐžÐž Java ÐºÐŸÑÐŸÑÐŸÐ¹ ÐŸÐ±Ð»Ð°ÐŽÐ°ÐµÑ Ð±ÑÐ°ÑÐ·ÐµÑ Ð¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÐµÐ»Ñ, ÑÑÐŸ Ð·Ð°ÑÑÐ°Ð²Ð»ÑÐµÑ Ð²ÑÐ¿ÐŸÐ»ÐœÑÑÑÑÑ Ðž ÐŽÐµÐ¹ÑÑÐ²ÐŸÐ²Ð°ÑÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐµÐœÐœÐŸ.
+
+
+Collage.imageMustNotHave:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÑÐ¶Ð°ÑÐžÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹, ÐºÐŸÑÐŸÑÑÐµ ÐœÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð¿ÐŸÑÐ²ÐžÑÑÑÑ Ð² ÐºÐŸÐ»Ð»Ð°Ð¶Ðµ, ÑÐžÐ¿Ð° ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹ ÐºÐœÐŸÐ¿ÐŸÑÐµÐº, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÑÑÐ°Ð²Ð»ÑÑÑ ÐœÐ°Ð²ÐžÐ³Ð°ÑÐžÐŸÐœÐœÑÑ Ð¿Ð°ÐœÐµÐ»Ñ.
+
+
+Collage.caption:ÐÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÐµ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐž, Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÐœÐžÐ¶Ðµ ÐºÐŸÐ»Ð»Ð°Ð¶Ð°. 
+
+
+#
+# Plugin option descriptions
+#
+
+ArcPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐ²ÑÐŸÑÑÐµÑÑÑ Ð² ÑÐ°Ð¹Ð»Ðµ archives.inf (ÑÐŸ ÐµÑÑÑ ÑÐ°Ð¹Ð», Ð³ÐµÐœÐµÑÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ Ð² Ð°ÑÑ
+ÐžÐ²ÐœÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž, ÐºÐŸÐ³ÐŽÐ°  Ð²ÑÐ¿ÐŸÐ»ÐœÑÐ»ÑÑ ÐžÐŒÐ¿ÐŸÑÑ ), ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°Ñ ÐºÐ°Ð¶ÐŽÑÐ¹ ÐœÐ°Ð¹ÐŽÐµÐœÐœÑÐ¹ ÑÐ°Ð¹Ð».
+
+
+BasPlug.adding:ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµ
+
+
+BasPlug.already_seen:ÑÐ¶Ðµ Ð·Ð°ÐŒÐµÑÐµÐœÐœÑÐ¹
+
+
+BasPlug.bad_general_option:ÐÐ»Ð°Ð³ÐžÐœ %s ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ ÐœÐµÐºÐŸÑÑÐµÐºÑÐœÑÑ ÐŸÐ±ÑÑÑ ÐŸÐ¿ÑÐžÑ (ÐŸÐ±ÑÐžÐµ ÐŸÐ¿ÑÐžÐž -  ÑÑÐŸ ÑÐµ, ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÑÑÑÐ¿ÐœÑ Ð²ÑÐµÐŒ Ð¿Ð»Ð°Ð³ÐžÐœÐ°ÐŒ). ÐÑÐŸÐ²ÐµÑÑÑÐµ Ð²Ð°Ñ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÑÐ¹ ÑÐ°Ð¹Ð» collect.cfg.
+
+
+BasPlug.block_exp:Ð€Ð°Ð¹Ð»Ñ, ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐµ ÑÑÐŸÐŒÑ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐŒÑ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÑ, Ð±ÑÐŽÑÑ Ð·Ð°Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÐœÑ ÐŸÑ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ Ð±ÑÑÑ Ð¿ÐµÑÐµÐŽÐ°ÐœÐœÑÐŒÐž Ð»ÑÐ±ÐŸÐŒÑ ÐŽÐ°Ð»ÑÐœÐµÐ¹ÑÐµÐŒÑ Ð¿Ð»Ð°Ð³ÐžÐœÑ Ð² ÑÐ¿ÐžÑÐºÐµ. Ð­ÑÐŸ ÐœÐµ ÐžÐŒÐµÐµÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ ÐŽÑÑÐ³ÐŸÐ³ÐŸ ÑÐµÐ°Ð»ÑÐœÐŸÐ³ÐŸ ÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐžÑ, ÑÐµÐŒ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐžÑÑ Ð±ÐŸÐ»ÑÑÐŸÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸ Ð¿ÑÐµÐŽÑÐ¿ÑÐµÐ¶ÐŽÐ°ÑÑÐžÑ
+ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹ ÐŸ Ð²Ñ
+ÐŸÐŽÐœÑÑ
+ ÑÐ°Ð¹Ð»Ð°Ñ
+, ÐºÐŸÑÐŸÑÑÐŒ ÐÑ ÐœÐµ ÑÐŽÐµÐ»ÑÐµÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÑ. ÐÐ°Ð¶ÐŽÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŒÐŸÐ³ Ð±Ñ ÐžÐŒÐµÑÑ Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ block_exp., ÐºÐ°Ðº ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, HTMLPlug  Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ Ð±Ð»ÐŸÐºÐžÑÑÐµÑ Ð»ÑÐ±ÑÐµ ÑÐ°Ð¹Ð»Ñ Ñ  ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ .gif, .jpg, .jpeg, .png ÐžÐ»Ðž .css. 
+
+
+BasPlug.associate_ext:ÐÑÐ·ÑÐ²Ð°ÐµÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ñ ÑÐµÐŒ Ð¶Ðµ ÐºÐŸÑÐœÐµÐ²ÑÐŒ ÐºÐ°ÑÐ°Ð»ÐŸÐ³ÐŸÐŒ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐŒ  AND Ð°ÑÑÐŸÑÐžÐžÑÑÐµÑ Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒÐž ÑÐ¿ÐžÑÐŸÐº ÑÐ°ÑÑÐžÑÐµÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ², Ð¿ÐµÑÐµÑÐžÑÐ»ÐµÐœÐœÑÑ
+ ÑÐµÑÐµÐ· Ð·Ð°Ð¿ÑÑÑÑ, Ð¿ÐµÑÐµÐŽÐ°Ð²Ð°ÐµÐŒÑÐ¹ Ð·ÐŽÐµÑÑ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ Ð°ÑÐ³ÑÐŒÐµÐœÑÐ°.
+
+
+BasPlug.could_not_extract_encoding:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐœÐµ ÑÐŒÐŸÐ³Ð»ÐŸ Ð±ÑÑÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐŸ ÐžÐ· %s - Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ Ðº %s
+
+
+BasPlug.could_not_extract_language:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: ÑÐ·ÑÐº ÐœÐµ ÑÐŒÐŸÐ³ Ð±ÑÑÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœ ÐžÐ· %s - Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ Ðº %s
+
+
+BasPlug.could_not_open_for_reading:ÐœÐµ ÑÐŒÐŸÐ³ ÐŸÑÐºÑÑÑÑ %s ÐŽÐ»Ñ ÑÑÐµÐœÐžÑ
+
+
+BasPlug.no_cover_image:ÐÐµ ÐŸÐ±ÑÐ°ÑÐ°Ð¹ÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÑ ÐœÐ° ÑÐ°Ð¹Ð» prefix.jpg (Ð³ÐŽÐµ prefix - ÑÑÐŸ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÐŒÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°). Ð­ÑÐŸ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐŸÐ±Ð»ÐŸÐ¶ÐºÐž.
+
+
+BasPlug.default_encoding:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐŸ ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐžÐµ, ÐµÑÐ»Ðž -input_encoding ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ ÐœÐ° 'Ð°Ð²ÑÐŸ', Ð° Ð°Ð»Ð³ÐŸÑÐžÑÐŒ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐµÐºÑÑÐ° ÐœÐµ Ð² ÑÐŸÑÑÐŸÑÐœÐžÐž ÐžÐ·Ð²Ð»ÐµÑÑ Ð·Ð°ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ ÐžÐ»Ðž ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÑ Ð·Ð°ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ, ÐœÐŸ ÐœÐµÐ¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÐŒÑÐµ Greenstone. Ð­ÑÐ° ÐŸÐ¿ÑÐžÑ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÐ°ÑÑ ÑÐµ Ð¶Ðµ ÑÐ°ÐŒÑÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ, ÐºÐ°Ðº Ðž -input_encoding.
+
+
+BasPlug.default_language:ÐÑÐ»Ðž Greenstone ÐœÐµ Ð² ÑÐŸÑÑÐŸÑÐœÐžÐž ÑÐµÑÐžÑÑ, ÐºÐ°ÐºÐžÐŒ ÑÐ²Ð»ÑÐµÑÑÑ ÑÐ·ÑÐº ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ 'Ð¯Ð·ÑÐº' Ð±ÑÐŽÐµÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ ÐœÐ° ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ. ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ -  ÑÑÐŸ 'en' (ÐÐ¡Ð ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ 639 ÑÐžÐŒÐ²ÐŸÐ»ÐŸÐ² ÑÐ·ÑÐºÐ°: en = Ð°ÐœÐ³Ð»ÐžÐ¹ÑÐºÐžÐ¹ ÑÐ·ÑÐº). ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ, ÑÑÐŸ ÐµÑÐ»Ðž -input_encoding ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ ÐœÐ° 'Ð°Ð²ÑÐŸ', Ðž -extract_language ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ, Ð²ÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ Ð±ÑÐŽÑÑ ÐžÐŒÐµÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÑ
+ 'Ð¯Ð·ÑÐºÐ°' ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐœÑÐŒÐž ÐœÐ° ÑÑÐŸ Ð·ÐœÐ°ÑÐµÐœÐžÐµ.
+
+
+BasPlug.desc:ÐÑÐœÐŸÐ²ÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑ ÐŽÐ»Ñ Ð²ÑÐµÐ³ÐŸ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² ÐžÐŒÐ¿ÐŸÑÑÐ°.
+
+
+BasPlug.done_acronym_extract:ÐÑÐŸÐžÐ·Ð²ÐµÐŽÐµÐœÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð°ÐºÑÐŸÐœÐžÐŒÐŸÐ².
+
+
+BasPlug.done_acronym_markup:ÐÑÐ¿ÐŸÐ»ÐœÐµÐœÐ° Ð¿ÐŸÐŒÐµÑÐºÐ° Ð°ÐºÑÐŸÐœÐžÐŒÐ°.
+
+
+BasPlug.done_email_extract:ÐÑÐ¿ÐŸÐ»ÐœÐµÐœÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð°ÐŽÑÐµÑÐŸÐ² ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ.
+
+
+BasPlug.dummy_text:ÐÐ°ÐœÐœÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÐœÐµ ÐžÐŒÐµÐµÑ ÑÐµÐºÑÑÐ°.
+
+
+BasPlug.empty_file:ÑÐ°Ð¹Ð» ÐœÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ°
+
+
+BasPlug.extract_acronyms:ÐÐºÑÐŸÐœÐžÐŒÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÑ ÐžÐ· ÑÐµÐºÑÑÐ° Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ  Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.
+
+
+BasPlug.extract_email:ÐÐ·Ð²Ð»ÐµÑÑ Ð°ÐŽÑÐµÑÐ° ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+.
+
+
+BasPlug.extract_historical_years:ÐÐ·Ð²Ð»ÐµÑÑ ÐžÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸ Ð²ÑÐµÐŒÐµÐœÐœÐŸÐŒ Ð¿ÐµÑÐžÐŸÐŽÐµ ÐžÐ· ÐžÑÑÐŸÑÐžÑÐµÑÐºÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ². ÐÐœÐ° Ñ
+ÑÐ°ÐœÐžÑÑÑ Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐŒ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐŒÐµÐµÑÑÑ ÐžÐœÑÐµÑÑÐµÐ¹Ñ Ð¿ÐŸÐžÑÐºÐ° ÐŽÐ»Ñ ÑÑÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐºÐŸÑÐŸÑÑÐ¹ ÐÑ ÐŒÐŸÐ¶ÐµÑÐµ Ð²ÐºÐ»ÑÑÐžÑÑ Ð² ÐÐ°ÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑ ÐŸÐ¿ÐµÑÐ°ÑÐŸÑ"format QueryInterface DateSearch" Ðº ÐÐ°ÑÐµÐŒÑ ÐºÐŸÐœÑÐžÐ³ÑÑÐ°ÑÐžÐŸÐœÐœÐŸÐŒÑ ÑÐ°Ð¹Ð»Ñ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+
+BasPlug.extract_language:ÐÐŽÐµÐœÑÐžÑÐžÑÐžÑÑÐ¹ÑÐµ ÑÐ·ÑÐº ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ 'Ð¯Ð·ÑÐº' . ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ, ÑÑÐŸ ÑÑÐŸ Ð±ÑÐŽÐµÑ ÑÐŽÐµÐ»Ð°ÐœÐŸ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž, ÐµÑÐ»Ðž -input_encoding ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ ÐœÐ° 'Ð°Ð²ÑÐŸ'.
+
+
+BasPlug.extracting:ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ
+
+
+BasPlug.extracting_acronyms:ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð°ÐºÑÐŸÐœÐžÐŒÐŸÐ²
+
+
+BasPlug.extract_keyphrases:ÐÐ·Ð²Ð»ÐµÑÑ ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÑÐ°Ð·Ñ  Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Kea (ÐŸÐ¿ÑÐžÐž Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ)
+
+BasPlug.extract_keyphrase_options:ÐÐ¿ÑÐžÐž ÐŽÐ»Ñ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÑÐ°Ð· Ñ Ð¿ÐŸÐŒÐŸÑÑÑ Kea. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, mALIWEB - ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ ÐŒÐŸÐŽÐµÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ALIWEB; n5 - ÐžÐ·Ð²Ð»ÐµÐºÐ°Ð¹ Ð¿ÐŸ 5 ÐºÐ»ÑÑÐµÐ²ÑÑ
+ ÑÐ»ÐŸÐ² ÐžÐ· ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°; eGBK - ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ ÐºÐŸÐŽÐžÑÐŸÐ²ÐºÑ GBK.
+
+BasPlug.extracting_emails:ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ Ð°ÐŽÑÐµÑÐŸÐ² ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ
+
+
+BasPlug.file_has_no_text:ÐÐšÐÐÐÐ: %s ÐœÐµ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ°
+
+
+BasPlug.first:ÐÑÐŽÐµÐ»ÐµÐœÐœÑÐ¹ Ð·Ð°Ð¿ÑÑÐŸÐ¹ ÑÐ¿ÐžÑÐŸÐº Ð¿ÐµÑÐ²ÑÑ
+ Ð²ÐµÐ»ÐžÑÐžÐœ ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÐžÐ· ÑÐµÐºÑÑÐ° Ð² Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+. ÐÐŸÐ»Ðµ ÐœÐ°Ð·Ð²Ð°ÐœÐŸ 'FirstNNN'.
+
+
+BasPlug.input_encoding:ÐÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐžÐµ ÐžÑÑ
+ÐŸÐŽÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ². ÐÐŸÐºÑÐŒÐµÐœÑÑ Ð±ÑÐŽÑÑ ÐºÐŸÐœÐ²ÐµÑÑÐžÑÐŸÐ²Ð°ÐœÑ ÐžÐ· ÑÑÐžÑ
+ ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÐ¹ Ðž Ñ
+ÑÐ°ÐœÐžÑÑÑÑ Ð²ÐœÑÑÑÐž ÑÐžÑÑÐµÐŒÑ Ð² Ð²ÐžÐŽÐµ utf8.
+
+BasPlug.input_encoding.ascii:ÐÑÐŸÑÑÐŸÐ¹ 7-Ð±ÐžÑÐœÑÐ¹ ascii. ÐÐœ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐœÐµÐŒÐœÐŸÐ³ÐŸ Ð±ÑÑÑÑÐµÐµ, ÑÐµÐŒ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµ iso_8859_1. ÐÑÑÐµÑÐµÐ³Ð°Ð¹ÑÐµÑÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÑ ÑÑÐŸÐ³ÐŸ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŒÐŸÐ³ÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ ÑÐžÐŒÐ²ÐŸÐ»Ñ Ð²ÐœÐµ Ð¿ÑÐŸÑÑÑÑ
+ 7-Ð±ÐžÑÐœÑÑ
+ ÐœÐ°Ð±ÐŸÑÐŸÐ² ascii (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐœÐµÐŒÐµÑÐºÐžÐµ ÐžÐ»Ðž ÑÑÐ°ÐœÑÑÐ·ÑÐºÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ Ð·ÐœÐ°ÐºÐž ÑÐŽÐ°ÑÐµÐœÐžÑ), ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð²ÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ iso_8859_1.
+
+BasPlug.input_encoding.auto:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ Ð°Ð»Ð³ÐŸÑÐžÑÐŒ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž ÑÐµÐºÑÑÐ° ÐŽÐ»Ñ Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐŸÐ¹ ÐžÐŽÐµÐœÑÐžÑÐžÐºÐ°ÑÐžÐž ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐžÑÑ
+ÐŸÐŽÐœÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°. Ð­ÑÐŸ Ð±ÑÐŽÐµÑ ÐŒÐµÐŽÐ»ÐµÐœÐœÐµÐµ, ÑÐµÐŒ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÑ Ð² ÑÐ²ÐœÐŸÐŒ Ð²ÐžÐŽÐµ, ÐœÐŸ Ð±ÑÐŽÐµÑ ÑÐ°Ð±ÐŸÑÐ°ÑÑ ÑÐ°ÐŒ, Ð³ÐŽÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ Ð±ÐŸÐ»ÑÑÐµ ÑÐµÐŒ ÐŸÐŽÐžÐœ Ð²ÐžÐŽ ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÑ  Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÑÐŸÐ¹ Ð¶Ðµ ÑÐ°ÐŒÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž.
+
+BasPlug.input_encoding.unicode:Ð¢ÐŸÐ»ÑÐºÐŸ ÑÐœÐžÐºÐŸÐŽ.
+
+BasPlug.input_encoding.utf8:ÐÐ»Ðž utf8, ÐžÐ»Ðž ÑÐœÐžÐºÐŸÐŽ - Ð°Ð²ÑÐŸÐŒÐ°ÑÐžÑÐµÑÐºÐž ÐŸÐ±ÐœÐ°ÑÑÐ¶ÐµÐœÐœÑÐµ.
+
+
+BasPlug.keyphrases:ÐÐ»ÑÑÐµÐ²ÑÐµ ÑÑÐ°Ð·Ñ
+
+
+BasPlug.marking_up_acronyms:Ð¿ÐŸÐŒÐµÑÐºÐ° Ð°ÐºÑÐŸÐœÐžÐŒÐŸÐ²
+
+
+BasPlug.markup_acronyms:ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð°ÐºÑÐŸÐœÐžÐŒÐ° Ð² ÑÐµÐºÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+
+BasPlug.maximum_century:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐœÐŸÐ³ÐŸ ÑÑÐŸÐ»ÐµÑÐžÑ, ÐºÐŸÑÐŸÑÐŸÐµ Ð±ÑÐŽÐµÑ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐŸ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐžÑÑÐŸÑÐžÑÐµÑÐºÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, 14 ÐžÐ·Ð²Ð»ÐµÑÐµÑ Ð²ÑÐµ ÑÑÑÐ»ÐºÐž, ÐŸÑÐœÐŸÑÑÑÐžÐµÑÑ Ðº 14-ÐŸÐŒÑ ÑÑÐŸÐ»ÐµÑÐžÑ).
+
+
+BasPlug.maximum_year:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐ°Ñ ÐžÑÑÐŸÑÐžÑÐµÑÐºÐ°Ñ ÐŽÐ°ÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐœÐ°Ñ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ (ÐŽÐ°ÑÐ° ÑÑÑ, ÑÐžÐ¿Ð° 1950).
+
+
+BasPlug.must_be_implemented:BasPlug::ÑÐžÑÐ°ÐµÐŒÐ°Ñ ÑÑÐœÐºÑÐžÑ, ÐºÐŸÑÐŸÑÐ°Ñ ÐŽÐŸÐ»Ð¶ÐœÐ° Ð±ÑÑÑ ÑÐµÐ°Ð»ÐžÐ·ÐŸÐ²Ð°ÐœÐ° Ð² Ð¿ÐŸÐŽÐºÐ»Ð°ÑÑÐµ ÐŽÐ»Ñ ÑÐµÐºÑÑÑÐžÐ²ÐœÑÑ
+ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ².
+
+
+BasPlug.no_bibliography:ÐÐµ Ð¿ÑÐŸÐ±ÑÐ¹ÑÐµ Ð±Ð»ÐŸÐºÐžÑÐŸÐ²Ð°ÑÑ Ð±ÐžÐ±Ð»ÐžÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÐŽÐ°ÑÑ Ð¿ÑÐž ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÐž ÐžÑÑÐŸÑÐžÑÐµÑÐºÐžÑ
+ ÐŽÐ°Ñ.
+
+
+BasPlug.process_exp:ÐÐ±ÑÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ Perl ÐŽÐ»Ñ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÑ  ÐžÐŒÐµÐœÐ°ÐŒ ÑÐ°Ð¹Ð»Ð°. Ð¡ÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐžÐµ ÐžÐŒÐµÐœÐ°ÐŒ ÑÐ°Ð¹Ð»Ð° Ð±ÑÐŽÐµÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÐŸ ÑÑÐžÐŒ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐŒ. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµ '(?i).html?\$' ÑÐŸÑÐ²ÐµÑÑÑÐ²ÑÐµÑ Ð²ÑÐµÐŒ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒ, Ð·Ð°ÐºÐ°ÐœÑÐžÐ²Ð°ÑÑÐžÐŒÑÑ ÐœÐ° .htm ÐžÐ»Ðž .html (Ð±ÐµÐ· ÑÐ°Ð·Ð»ÐžÑÐµÐœÐžÑ Ð·Ð°Ð³Ð»Ð°Ð²ÐœÑÑ
+ Ðž ÑÑÑÐŸÑÐœÑÑ
+ Ð±ÑÐºÐ²).
+
+
+BasPlug.read_denied:Ð Ð°Ð·ÑÐµÑÐµÐœÐžÐµ ÑÑÐµÐœÐžÑ,  ÐŸÑÐºÐ»ÐŸÐœÐµÐœÐœÐŸÐµ ÐŽÐ»Ñ %s
+
+
+BasPlug.smart_block:ÐÐ»ÐŸÐºÐžÑÐŸÐ²ÐºÐ° ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐ»ÑÑÑÐµÐœÐœÑÐŒ ÐŒÐµÑÐŸÐŽÐŸÐŒ, Ð²ÐŒÐµÑÑÐŸ Ð¿ÑÐŸÑÑÐŸÐ³ÐŸ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ° ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÑÐ°Ð¹Ð»ÐŸÐ².
+
+BasPlug.stems:ÑÑÐµÐ±Ð»Ðž
+
+
+BasPlug.unsupported_encoding:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: %s Ð¿ÐŸÑÐ²Ð»ÑÐµÑÑÑ, ÑÑÐŸÐ±Ñ Ð±ÑÑÑ Ð·Ð°ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐœÑÐŒ Ð² ÐœÐµÐ¿ÐŸÐŽÐŽÐµÑÐ¶Ð°ÐœÐœÐŸÐŒ ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÐž (%s) - ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ %s
+
+
+BasPlug.wrong_encoding:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: %s ÑÐžÑÐ°Ð»ÐŸÑÑ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ %s ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÐµ, ÐœÐŸ  Ð¿ÐŸÑÐ²Ð»ÑÐ»ÐŸÑÑ ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐœÑÐŒ ÐºÐ°Ðº %s.
+
+
+BibTexPlug.desc:BibTexPlug ÑÐžÑÐ°ÐµÑ Ð±ÐžÐ±Ð»ÐžÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÑÐŸÑÐŒÐ°ÑÐµ BibTex. BibTexPlug ÑÐŸÐ·ÐŽÐ°ÐµÑ ÐŸÐ±ÑÐ°Ð· ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÑÑÑÐ»ÐºÐž Ð² ÑÐ°Ð¹Ð»Ðµ. Ð­ÑÐŸ - Ð¿ÐŸÐŽÐºÐ»Ð°ÑÑ SplitPlug, ÑÐ°Ðº ÑÑÐŸ ÐµÑÐ»Ðž ÐžÐŒÐµÑÑÑÑ ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÐµ Ð·Ð°Ð¿ÐžÑÐž, ÑÐŸ Ð²ÑÐµ ÐŸÐœÐž ÑÐžÑÐ°ÑÑÑÑ.
+
+
+BookPlug.desc:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑÑÑ ÐŒÐœÐŸÐ³ÐŸÑÑÐŸÐ²ÐœÐµÐ²ÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÐžÐ· ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ³ÐŸ ÑÑÐ³Ðž ÑÑÐŸÐ²ÐœÑ <<TOC>>. ÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ  ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°Ð·ÐŽÐµÐ»Ð° Ð²Ð·ÑÑÑ ÐžÐ· Ð»ÑÐ±ÐŸÐ³ÐŸ ÐŽÑÑÐ³ÐŸÐ³ÐŸ ÑÑÐ³Ð° Ð² ÑÐŸÐ¹ Ð¶Ðµ ÑÐ°ÐŒÐŸÐ¹ ÑÑÑÐŸÐºÐµ, ÐºÐ°Ðº Ðž <<TOC>>. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, <<Title>>xxxx<</Title>>  ÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ. ÐÑÐµ ÐŸÑÑÐ°Ð»ÑÐœÐŸÐµ ÐŒÐµÐ¶ÐŽÑ ÑÑÐ³Ð°ÐŒÐž ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑÑÑ ÐºÐ°Ðº Ð¿ÑÐŸÑÑÐŸÐ¹ html (ÑÐŸ ÐµÑÑÑ ÐœÐžÐºÐ°ÐºÐŸÐ¹ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž ÑÐ²ÑÐ·ÐµÐ¹ html ÐžÐ»Ðž Ð»ÑÐ±ÑÑ
+ ÐŽÑÑÐ³ÐžÑ
+ Ð²ÐžÐŽÐŸÐ² HTMLPlug  ÐœÐµ Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐŽÐžÑÑÑ). ÐÐ¶ÐžÐŽÐ°ÐµÑÑÑ, ÑÑÐŸ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐžÐŒÐµÑÑ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ .hb Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ. Ð­ÑÐŸ ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐžÐ·ÐŒÐµÐœÐµÐœÐŸ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÐµÐŒ ÐŸÐ¿ÑÐžÐž -process_exp Ðº  ÑÐ°Ð¹Ð»Ñ Ñ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÐžÐŒÐµÐœÐµÐŒ, ÑÑÐŸ Ðž hb ÑÐ°Ð¹Ð», ÐœÐŸ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ .jpg Ð²Ð·ÑÑÐŸ ÐŽÐ»Ñ ÐŸÐ±Ð»ÐŸÐ¶ÐºÐž ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ (jpg ÑÐ°Ð¹Ð»Ñ Ð±Ð»ÐŸÐºÐžÑÑÑÑÑÑ ÑÑÐžÐŒ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐŒ). BookPlug ÑÐ²Ð»ÑÐµÑÑÑ ÑÐ¿ÑÐŸÑÐµÐœÐžÐµÐŒ (Ðž ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ) HBPlug, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒÐž ÐÑÐŒÐ°ÐœÐžÑÐ°ÑÐœÐŸÐ¹ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž. BookPlug ÑÐ²Ð»ÑÐµÑÑÑ Ð±ÐŸÐ»ÐµÐµ Ð±ÑÑÑÑÑÐŒ, Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ Ð¿ÑÐµÐŽÐ¿ÐŸÐ»Ð°Ð³Ð°ÐµÑÑÑ, ÑÑÐŸ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÑÐ°Ð¹Ð»Ñ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ Ð±ÐŸÐ»ÐµÐµ ÑÐžÑÑÑÐŒÐž (Ð²Ñ
+ÐŸÐŽ Ðº HDL ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒ ÑÐŸÐŽÐµÑÐ¶ÐžÑ ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐŸ ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÑ
+ ÑÑÐ³ÐŸÐ² html Ð²ÐŸÐºÑÑÐ³  ÑÑÐ³ÐŸÐ² <<TOC>>, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÑÑÐ³Ðž <<I>>, ÑÑÐŸÐ±Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐžÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, Ðž Ð¿ÑÐŸÑÑÐŸ Ð±ÐµÑÑÑ Ð²ÐµÑÑ ÑÐµÐºÑÑ ÐŒÐµÐ¶ÐŽÑ ÑÑÐ³Ð°ÐŒÐž <<TOC>>, Ð° ÐœÐ°ÑÐ°Ð»ÐŸÐŒ ÑÐµÐºÑÑÐ° ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÐ°Ð·Ð²Ð°ÐœÐžÑ). ÐÑÐ»Ðž ÐÑ Ð¿ÐŸÐŒÐµÑÐ°ÐµÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐŸÐºÐ°Ð·Ð°ÐœÑ ÑÐµÐŒ Ð¶Ðµ ÑÐ°ÐŒÑÐŒ ÑÐ¿ÐŸÑÐŸÐ±ÐŸÐŒ, ÑÑÐŸ Ðž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ HDL, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ ÑÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ Ð²ÐŒÐµÑÑÐŸ HBPlug.
+
+
+ConvertToPlug.convert_to:ÐÐ»Ð°Ð³ÐžÐœ ÐºÐŸÐœÐ²ÐµÑÑÐžÑÑÐµÑ Ðº Ð¢ÐÐÐ¡Ð¢Ð£ ÐžÐ»Ðž HTML.
+
+ConvertToPlug.convert_to.html:Ð€ÐŸÑÐŒÐ°Ñ HTML.
+
+ConvertToPlug.convert_to.text:Ð€ÐŸÑÐŒÐ°Ñ Ð¿ÑÐŸÑÑÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ°.
+
+
+ConvertToPlug.desc:The plugin is inherited by such plugins as WordPlug and PDFPlug. It facilitates the conversion of these document types to either HTML or TEXT by setting up variable that instruct ConvertToBasPlug how to work. It works by dynamically inheriting HTMLPlug or TEXTPlug based on the plugin argument 'convert_to'.  If the argument is not present, the default is to inherit  .
+
+
+ConvertToPlug.use_strings:ÐÑÐ»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐŸ, ÑÐŸ Ð¿ÑÐŸÑÑÐ°Ñ  ÑÑÑÐŸÐºÐŸÐ²Ð°Ñ ÑÑÐœÐºÑÐžÑ Ð±ÑÐŽÐµÑ Ð²ÑÐ·ÑÐ²Ð°ÑÑÑÑ ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÑÐµÐºÑÑÐ°, ÐµÑÐ»Ðž ÑÑÐžÐ»ÐžÑÐ° ÐºÐŸÐœÐ²ÐµÑÑÐžÐž ÐœÐµ ÑÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ.
+
+
+ConvertToRogPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ ÐœÐ°ÑÐ»ÐµÐŽÑÐµÑ RogPlug.
+
+
+DBPlug.desc:ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ Ð·Ð°Ð¿ÐžÑÐž ÐžÐ· Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+  ÐºÐ°Ðº ÐŽÐŸÐºÑÐŒÐµÐœÑÑ.
+
+
+DBPlug.title_sub:ÐÑÑÐ°Ð¶ÐµÐœÐžÐµ Ð·Ð°ÐŒÐµÐœÑ, ÑÑÐŸÐ±Ñ ÐžÐ·ÐŒÐµÐœÐžÑÑ ÑÑÑÐŸÐºÑ, ÑÐŸÑ
+ÑÐ°ÐœÐµÐœÐœÑÑ ÐºÐ°Ðº ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ. ÐÑÐžÐŒÐµÐœÑÐµÑÑÑ, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐŒ PSPlug, ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ "Ð¡ÑÑÐ°ÐœÐžÑÑ 1" Ðž Ñ.ÐŽ ÐžÐ· ÑÐµÐºÑÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ.
+
+
+EMAILPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ Email ÑÐžÑÐ°ÐµÑ ÑÐ°Ð¹Ð»Ñ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ. ÐÐœÐž ÐžÐŒÐµÐœÑÑÑÑÑ Ð¿ÑÐŸÑÑÑÐŒÐž ÐœÐŸÐŒÐµÑÐ°ÐŒÐž (Ñ.Ðµ.Ð² ÑÐŸÐŒ Ð¿ÐŸÑÑÐŽÐºÐµ,ÐºÐ°Ðº ÐŸÐœÐž Ð¿ÐŸÑÐ²Ð»ÑÑÑÑÑ Ð² Ð¿Ð°Ð¿ÐºÐ°Ñ
+ Ð¿ÐŸÑÑÐŸÐ²ÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž) ÐžÐ»Ðž Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ .mbx (ÑÐŸÑÐŒÐ°Ñ ÑÐ°Ð¹Ð»Ð° ÐŽÐ»Ñ Ð¿ÐžÑÐµÐŒ mbox).\nÐ¢ÐµÐºÑÑ ÐŽÐŸÐŸÐºÑÐŒÐµÐœÑÐ°: Ð¢ÐµÐºÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÑÐŸÑÑÐŸÐžÑ ÐžÐ· Ð²ÑÐµÐ³ÐŸ ÑÐµÐºÑÑÐ° Ð¿ÐŸÑÐ»Ðµ Ð¿ÐµÑÐ²ÐŸÐ¹ Ð¿ÑÑÑÐŸÐ¹ ÑÑÑÐŸÐºÐž Ð² ÐŽÐŸÐºÑÐŒÐµÐœÑÐµ.\nÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ (ÐœÐµ Dublin Core!):\n\t\$Headers ÐÑÐµ ÑÑÐŸ ÑÐŸÐŽÐµÑÐ¶ÐžÑ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº (ÐœÐµÐŸÐ±ÑÐ·Ð°ÑÐµÐ»ÑÐœÐŸ, Ð¿ÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐœÐµ ÑÐŸÑ
+ÑÐ°ÐœÑÐµÑÑÑ)\n\t\$Subject ÐÑÐµÐŽÐŒÐµÑ: Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº\n\t\$To  ÐÐŸÐŒÑ:Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº\n\t\$From ÐºÐŸÐ³ÐŸ   ÐÑ ÐºÐŸÐ³ÐŸ: Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº\n\t\$FromName ÐŸÑ ÐºÐŸÐ³ÐŸ  ÐÐŒÑ ÐŸÑÐ¿ÑÐ°Ð²ÐžÑÐµÐ»Ñ(ÐºÐŸÐ³ÐŽÐŽÐ° ÐŸÐœÐŸ ÐžÐŒÐµÐµÑÑÑ Ð² ÐœÐ°Ð»ÐžÑÐžÐž)\n\t\$FromAddr ÐŸÑ ÐºÐŸÐ³ÐŸ  Ð°ÐŽÑÐµÑ ÑÐ»ÐµÐºÑÑÐŸÐœÐœÐŸÐ¹ Ð¿ÐŸÑÑÑ ÐŸÑÐ¿ÑÐ°Ð²ÐžÑÐµÐ»Ñ\n\t\$DateText ÐŽÐ°ÑÑ  ÐÐ°ÑÐ°: Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº\n\t\$Date   ÐÐ°ÑÐ°: Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº Ð² ÑÐŸÑÐŒÐ°ÑÐµ ÑÐžÑÑÐŸÐ²ÐŸÐ¹ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž Greenstone (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ: 19990924)
+
+
+EMAILPlug.no_attachments:ÐÐµ ÑÐŸÑ
+ÑÐ°ÐœÑÑÑ Ð¿ÑÐžÐ»ÐŸÐ¶ÐµÐœÐžÑ ÑÐŸÐŸÐ±ÑÐµÐœÐžÑ.
+
+
+EMAILPlug.headers:Ð¡ÐŸÑ
+ÑÐ°ÐœÑÑÑ Ð¿ÐŸÑÑÐŸÐ²ÑÐµ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐž ÐºÐ°Ðº ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ "Headers"
+
+
+EMAILPlug.split_exp:ÐÐ±ÑÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ  perl ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÑÐ°ÑÑÐµÐ¿Ð»ÐµÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ², ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐžÑ
+ ÐžÐŒÐµÐ»ÐŸ ÐŸÐ±ÑÐºÐœÐŸÐ²ÐµÐœÐžÐµ ÑÐ°ÑÐºÐ°Ð»ÑÐ²Ð°ÑÑ ÑÐ°Ð¹Ð»Ñ, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐžÐµ ÐŒÐœÐŸÐ³ÐŸ ÑÐŸÐŸÐ±ÑÐµÐœÐžÐ¹ Ð² ÐžÐœÐŽÐžÐ²ÐžÐŽÑÐ°Ð»ÑÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°Ñ
+.
+
+
+ExcelPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐ° ÑÐ°Ð¹Ð»Ð° Microsoft Excel.
+
+
+FOXPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž ÑÐ°Ð¹Ð»Ð° Foxbase dbt. Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ±ÐµÑÐ¿ÐµÑÐžÐ²Ð°ÐµÑ ÐŸÑÐœÐŸÐ²ÐœÑÐµ ÑÑÐœÐºÑÐžÐŸÐœÐ°Ð»ÑÐœÑÐµ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÐž ÑÑÐµÐœÐžÑ dbt Ðž dbf ÑÐ°Ð¹Ð»ÐŸÐ² Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž ÐºÐ°Ð¶ÐŽÐŸÐ¹ Ð·Ð°Ð¿ÐžÑÐž. Ð­ÑÐŸÑ ÐŸÐ±ÑÐžÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŽÐŸÐ»Ð¶ÐµÐœ Ð±ÑÑÑ Ð·Ð°ÐŒÐµÑÐµÐœ ÐŽÐ»Ñ ÑÐ¿ÐµÑÐžÑÐžÑÐµÑÐºÐŸÐ¹ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸÐ±Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÑÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐµ Ð¿ÐŸÐ»Ñ Ð² ÑÐ°Ð¹Ð»Ðµ.
+
+
+GAPlug.desc:ÐÐ±ÑÐ°Ð±ÐŸÑÐºÐ° XML ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ð°ÑÑ
+ÐžÐ²Ð° Greenstone. ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ, ÑÑÐŸ ÑÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐœÐµ ÐŽÐµÐ»Ð°ÐµÑ ÐœÐžÐºÐ°ÐºÐŸÐ¹ Ð¿ÑÐŸÐ²ÐµÑÐºÐž ÑÐžÐœÑÐ°ÐºÑÐžÑÐ° (Ñ
+ÐŸÑÑ  Ð°ÐœÐ°Ð»ÐžÐ·ÐžÑÑÑÑÐžÐ¹ ÐŒÐŸÐŽÑÐ»Ñ XML::   Ð¿ÑÐŸÐ²ÐµÑÑÐµÑ Ð¿ÑÐ°Ð²ÐžÐ»ÑÐœÐŸÑÑÑ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ). ÐÑÐµÐŽÐ¿ÐŸÐ»Ð°Ð³Ð°ÐµÑÑÑ, ÑÑÐŸ ÑÐ°Ð¹Ð»Ñ Ð°ÑÑ
+ÐžÐ²Ð° Greenstone ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑ ÐžÑ
+ DTD .
+
+
+GISBasPlug.extract_placenames:Ð Ð°ÑÐ¿ÐŸÐ·ÐœÐŸÐ²Ð°ÑÑ Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ Ð² ÑÐµÐºÑÑÐµ Ðž Ð²ÐºÐ»ÑÑÐžÑÑ ÐžÑ
+ Ð² ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ. Ð¢ÑÐµÐ±ÑÐµÑ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÐÐÐ¡ Ð² ÐÑÐžÐœÑÑÐŸÑÐœ.
+
+GISBasPlug.gazetteer:ÐÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐ¹ ÑÐ¿ÑÐ°Ð²ÐŸÑÐœÐžÐº ÐŽÐ»Ñ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐœÐžÑ Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ Ð² ÑÐµÐºÑÑÐµ Ðž ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐžÑ ÐžÑ
+ Ð² ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+. Ð¢ÑÐµÐ±ÑÐµÑ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÐÐÐ¡ Ð² ÐÑÐžÐœÑÑÐŸÑÐœ.
+
+GISBasPlug.place_list:ÐÑÐž ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐœÐžÐž Ð³ÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹, Ð¿Ð»Ð°Ð³ÐžÐœ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐžÑ
+ ÑÐ¿ÐžÑÐŸÐº Ð² ÐœÐ°ÑÐ°Ð»ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°. Ð¢ÑÐµÐ±ÑÐµÑ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµ ÐÐÐ¡ Ð² ÐÑÐžÐœÑÑÐŸÑÐœ.
+
+GMLPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÐŽÐŸÐºÑÐŒÐµÐœÑ ÑÐŸÑÐŒÐ°ÑÐ° GML, Ð¿ÑÐµÐŽÐ¿ÐŸÐ»Ð°Ð³Ð°Ñ, ÑÑÐŸ Ð²ÑÐµ ÑÑÐ³Ðž gml Ð² ÑÑÑÐŸÑÐœÑÑ
+ Ð±ÑÐºÐ²Ð°Ñ
+.
+
+
+HBPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ HTML ÐºÐœÐžÐ¶ÐœÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÑ. Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒÐž ÐÑÐŒÐ°ÐœÐžÑÐ°ÑÐœÐŸÐ¹ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž Ðž ÐœÐµ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÐŽÑÑÐ³ÐžÐµ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÑÐžÑÑÐŸÐ²Ð°ÐœÐžÑ, ÐºÑÐŸÐŒÐµ ascii ÐžÐ»Ðž Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ ascii. Ð­ÑÐ° Ð¿ÑÐŸÐ³ÑÐ°ÐŒÐŒÐ° ÑÐ²Ð»ÑÐµÑÑÑ Ð»ÑÑÑÐµÐ¹ ÐžÐ· Ñ
+ÑÐŽÑÐžÑ
+ Ðž ÐŒÐŸÐ³Ð»Ð° Ð±Ñ Ð±ÐµÐ· ÑÐŸÐŒÐœÐµÐœÐžÑ ÐŽÐŸÑÐ°Ð±ÐŸÑÐ°ÐœÐ° ÑÐ°Ðº, ÑÑÐŸÐ±Ñ Ð·Ð°Ð³ÑÑÐ¶Ð°ÑÑÑÑ Ð±ÑÑÑÑÐµÐµ. ÐÑÑÐ°Ð²Ð»ÑÑ ÐµÐµ Ð² ÑÑÐŸÐŒ ÑÐŸÑÑÐŸÑÐœÐžÐž, Ñ ÐœÐ°ÐŽÐµÑÑÑ Ð¿ÐŸÐŸÑÑÐžÑÑ Ð»ÑÐŽÐµÐ¹ ÐŽÐµÐ»Ð°ÑÑ ÐžÑ
+ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ Ð²ÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ Ð¿Ð»Ð°Ð³ÐžÐœ HBSPlug. ;-)\n\n ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ HBSPlug Ð¿ÑÐž ÑÐŸÐ·ÐŽÐ°ÐœÐžÐž ÐœÐŸÐ²ÐŸÐ¹ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž Ðž Ð¿ÐŸÐŒÐµÑÐµÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ², Ð¿ÐŸÐŽÐŸÐ±ÐœÐŸ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒ ÐÑÐŒÐ°ÐœÐžÑÐ°ÑÐœÐŸÐ¹ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž. HBSPlug Ð¿ÑÐžÐœÐžÐŒÐ°ÐµÑ Ð²ÑÐµ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÐºÐŸÐŽÐžÑÐŸÐ²Ð°ÐœÐžÑ, ÐœÐŸ ÐŸÐ¶ÐžÐŽÐ°ÐµÑ, ÑÑÐŸ Ð¿ÐŸÐŒÐµÑÐµÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ Ð±ÑÐŽÑÑ Ð±ÐŸÐ»ÐµÐµ ÑÐžÑÑÑÐŒÐž, ÑÐµÐŒ ÑÐµ, ÑÑÐŸ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑÑÑÑ ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑÐŒÐž ÐÑÐŒÐ°ÐœÐžÑÐ°ÑÐœÐŸÐ¹ Ð±ÐžÐ±Ð»ÐžÐŸÑÐµÐºÐž.
+
+
+HTMLPlug.assoc_files:ÐÐ±ÑÑÐœÐŸÐµ Perl Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ ÑÐ°Ð¹Ð»Ð° ÑÐ°ÑÑÐžÑÐµÐœÐžÐ¹ ÐŽÐ»Ñ ÑÐ²ÑÐ·Ðž Ñ  html -ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒÐž.
+
+
+HTMLPlug.desc:Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÑÐ°Ð¹Ð»Ñ HTML
+
+
+HTMLPlug.description_tags:Ð Ð°Ð·Ð±ÐžÐµÐœÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐœÐ° Ð¿ÐŸÐŽÑÐ°Ð·ÐŽÐµÐ»Ñ, Ð³ÐŽÐµ ÐžÐŒÐµÑÑÑÑ ÑÑÐ³Ðž <Ð Ð°Ð·ÐŽÐµÐ»>. ÐÐ±ÑÐ°ÑÐžÑÐµ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ, ÑÑÐŸ Ð¿ÑÐž ÑÑÑÐ°ÐœÐŸÐ²ÐºÐµ ÑÑÐŸÐ¹ ÐŸÐ¿ÑÐžÐž, ÐÑ ÐœÐµÑÐ²ÐœÐŸ ÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑÐµ -no_metadata (ÐœÐžÐºÐ°ÐºÐžÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+), Ð¿ÐŸÑÐºÐŸÐ»ÑÐºÑ Ð²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÑÑÐ³ÐŸÐ² <Ð Ð°Ð·ÐŽÐµÐ»>. ÐÑÐŸÐŒÐµ ÑÐŸÐ³ÐŸ '-keep_head' ÐœÐµ Ð±ÑÐŽÐµÑ ÐžÐŒÐµÑÑ ÐœÐžÐºÐ°ÐºÐŸÐ³ÐŸ ÐŽÐµÐ¹ÑÑÐ²ÐžÑ, ÐºÐŸÐ³ÐŽÐ° ÑÑÐ° ÐŸÐ¿ÑÐžÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐ°.
+
+
+HTMLPlug.file_is_url:Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÑ, ÐµÑÐ»Ðž Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÐžÐŒÐµÐœÐ° ÑÐ°Ð¹Ð»ÐŸÐ² ÑÐŸÐŽÐµÑÐ¶Ð°Ñ URL Ð¿ÐµÑÐ²ÐŸÐœÐ°ÑÐ°Ð»ÑÐœÑÑ
+ ÐžÑÑ
+ÐŸÐŽÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐµÑÐ»Ðž ÑÑÐµÐŽÑÑÐ²Ð° Ð²ÐµÐ±-Ð·ÐµÑÐºÐ°Ð»Ð° ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°Ð»ÐžÑÑ ÐŽÐ»Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐžÑ ÑÑÑÑÐºÑÑÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž ÐžÐŒÐ¿ÐŸÑÑÐ°.
+
+
+HTMLPlug.hunt_creator_metadata:ÐÐ°Ð¹ÑÐž ÐœÐ°ÑÐºÐŸÐ»ÑÐºÐŸ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ Ð±ÐŸÐ»ÑÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² Ð°Ð²ÑÐŸÑÑÑÐ²Ðµ Ðž ÑÐ°Ð·ÐŒÐµÑÑÐžÑÑ ÐžÑ
+ Ð² Ð¿ÐŸÐ»Ðµ 'CÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ'. Ð¢ÑÐµÐ±ÑÐµÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ»Ð°Ð³Ð° ÐœÐ° -metadata_fields (ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð¿ÐŸÐ»ÐµÐ¹).
+
+
+HTMLPlug.keep_head:ÐÐµ ÑÐŽÐ°Ð»ÑÑÑ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐž ÐžÐ· html-ÑÐ°Ð¹Ð»ÐŸÐ².
+
+
+HTMLPlug.metadata_fields:ÐÐ°Ð¿ÑÑÐ°Ñ ÐŸÑÐŽÐµÐ»ÑÐµÑ ÑÐ¿ÐžÑÐŸÐº Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÑÑÐŸÐ±Ñ Ð¿ÐŸÐ¿ÑÑÐ°ÑÑÑÑ  ÐœÐ°ÑÐ°ÑÑ ÑÐºÑÑÑÐ°ÐºÑÐžÑ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ'ÑÑÐ³ <tagname>', ÑÑÐŸÐ±Ñ Ð¿ÐŸÐ»ÑÑÐžÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÐµ Ð¿ÐµÑÐ²ÐŸÐ¹  Ð¿Ð°ÑÑ <tagname>, Ð²ÑÑÐ°Ð²Ð»ÐµÐœÐœÐŸÐ¹ Ð² ÑÐ»ÐµÐŒÐµÐœÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐœÐ°Ð·ÑÐ²Ð°ÐµÐŒÑÐ¹ 'tagname'. ÐÐ°ÐŒÐµÐœÐžÑÐµ Ð·ÐŽÐµÑÑ ÑÑÑÐŸÑÐœÑÐµ Ð±ÑÐºÐ²Ñ Ð·Ð°Ð³Ð»Ð°Ð²ÐœÑÐŒÐž, ÐµÑÐ»Ðž ÐÑ Ñ
+ÐŸÑÐžÑÐµ, ÑÑÐŸÐ±Ñ ÑÑÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Ð±ÑÐ»Ðž Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÑ Ð² Greenstone Ð·Ð°Ð³Ð»Ð°Ð²ÐœÑÐŒÐž Ð±ÑÐºÐ²Ð°ÐŒÐž, ÑÐ°Ðº ÐºÐ°Ðº ÑÐºÑÑÑÐ°ÐºÑÐžÑ ÑÑÐ³ÐŸÐ² ÑÐ²Ð»ÑÐµÑÑÑ ÐŒÐ°Ð»ÐŸÑÑÐ²ÑÑÐ²ÐžÑÐµÐ»ÑÐœÑÐŒ ÐŽÐµÐ¹ÑÑÐ²ÐžÐµÐŒ.
+
+
+HTMLPlug.no_metadata:ÐÐµ Ð¿ÑÑÐ°Ð¹ÑÐµÑÑ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÑÑ Ð»ÑÐ±ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ².
+
+
+HTMLPlug.no_strip_metadata_html:Ð¡Ð¿ÐžÑÐŸÐº ÐœÐ°Ð·Ð²Ð°ÐœÐžÐ¹ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, Ð¿ÐµÑÐµÑÐžÑÐ»ÐµÐœÐœÑÑ
+ ÑÐµÑÐµÐ· Ð·Ð°Ð¿ÑÑÑÑ, ÐžÐ»Ðž 'all' (Ð²ÑÐµ). ÐÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÑÑ Ñ -description_tags, ÑÑÐŸ Ð¿ÑÐµÐŽÐŸÑÐ²ÑÐ°ÑÐ°ÐµÑ ÑÐŽÐ°Ð»ÐµÐœÐžÐµ HTML ÑÐ°Ð³ÐŸÐ² ÐžÐ· Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+.
+
+HTMLPlug.nolinks:ÐÐµ ÐŽÐµÐ»Ð°Ð¹ÑÐµ ÐœÐžÐºÐ°ÐºÐžÑ
+ Ð¿ÐŸÐ¿ÑÑÐŸÐº  ÑÐ»Ð°Ð²Ð»ÐžÐ²Ð°ÑÑ Ð»ÐžÐœÐºÐž (ÑÑÑÐ°ÐœÐŸÐ²ÐºÐ° ÑÐ°ÐºÐŸÐ³ÐŸ ÑÐ»Ð°Ð³Ð° ÐŒÐŸÐ¶ÐµÑ ÑÐ»ÑÑÑÐžÑÑ ÑÐºÐŸÑÐŸÑÑÑ Ð¿ÐŸÑÑÑÐŸÐµÐœÐžÑ/ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ, ÐœÐŸ Ð»ÑÐ±ÑÐµ ÐŸÑÐœÐŸÑÐžÑÐµÐ»ÑÐœÑÐµ ÑÐ²ÑÐ·Ðž Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Ð±ÑÐŽÑÑ ÐœÐ°ÑÑÑÐµÐœÑ).
+
+
+HTMLPlug.rename_assoc_files:ÐÐµÑÐµÐžÐŒÐµÐœÐŸÐ²ÑÐ²Ð°ÐµÑ ÑÐ°Ð¹Ð»Ñ, ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°ÐŒÐž (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑÐŒÐž). Ð¢Ð°ÐºÐ¶Ðµ ÑÐŸÐ·ÐŽÐ°ÐµÑ ÐœÐ°ÐŒÐœÐŸÐ³ÐŸ Ð±ÐŸÐ»ÐµÐµ ÐŒÐµÐ»ÐºÑÑ ÑÑÑÑÐºÑÑÑÑ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž (Ð¿ÐŸÐ»ÐµÐ·ÐœÐŸ Ð¿ÑÐž ÑÐŸÐ·ÐŽÐ°ÐœÐžÐž ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐ¹, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐµÑÐµÐœÐµÑÐµÐœÑ ÐœÐ° CD-ROM).
+
+
+HTMLPlug.title_sub:ÐÑÑÐ°Ð¶ÐµÐœÐžÐµ Ð·Ð°ÐŒÐµÐœÑ ÐŽÐ»Ñ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ ÑÑÑÐŸÐºÐž, Ñ
+ÑÐ°ÐœÑÑÐµÐ¹ÑÑ ÐºÐ°Ðº ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ.ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐŒ PDFPlug, ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ "Ð¡ÑÑÐ°ÐœÐžÑÑ 1 ", Ðž Ñ.ÐŽ ÐžÐ· ÑÐµÐºÑÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ ÐºÐ°Ðº ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ.
+
+
+ImagePlug.converttotype:ÐÑÐµÐŸÐ±ÑÐ°Ð·ÐŸÐ²Ð°ÑÑ Ð³Ð»Ð°Ð²ÐœÐŸÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ Ðº ÑÐŸÑÐŒÐ°ÑÑ 's'
+
+
+ImagePlug.desc:Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, ÐŽÐŸÐ±Ð°Ð²Ð»ÑÑ Ð±Ð°Ð·ÐŸÐ²ÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ
+
+
+ImagePlug.minimumsize:ÐÐµ Ð¿ÑÐžÐœÐžÐŒÐ°ÑÑ Ð²ÐŸ Ð²ÐœÐžÐŒÐ°ÐœÐžÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, ÐŒÐµÐœÑÑÐžÐµ, ÑÐµÐŒ n Ð±Ð°Ð¹Ñ.
+
+
+ImagePlug.noscaleup:ÐÐµ ÑÐ²ÐµÐ»ÐžÑÐžÐ²Ð°ÑÑ ÐŒÐ°Ð»ÐµÐœÑÐºÐžÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð¿ÑÐž ÑÐŒÐµÐœÑÑÐµÐœÐžÑÑ
+.
+
+
+ImagePlug.screenviewsize:ÐÑÐ»Ðž ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐŸ, ÐŽÐµÐ»Ð°ÐµÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÑÐ°Ð·ÐŒÐµÑÐŸÐŒ n ÐŽÐ»Ñ Ð¿ÐŸÐºÐ°Ð·Ð° ÑÐºÑÐ°ÐœÐ° Ðž ÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ, Ð Ð°Ð·ÐŒÐµÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, ÐšÐžÑÐžÐœÐ° ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ðž ÐÑÑÐŸÑÐ° ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ.ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐŸÐœÐŸ ÐœÐµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐŸ.
+
+
+ImagePlug.screenviewtype:ÐÑÐ»Ðž  -screenviewsize ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ,  ÑÐŸ ÑÑÐŸ ÑÑÑÐ°ÐœÐ°Ð²Ð»ÐžÐ²Ð°ÐµÑ ÑÐžÐ¿ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐŽÐ»Ñ  Ð¿ÐŸÐºÐ°Ð·Ð° ÐœÐ° ÑÐºÑÐ°ÐœÐµ.
+
+
+ImagePlug.thumbnailsize:ÐÑÐ¿ÐŸÐ»ÐœÐžÑÑ ÑÐŒÐµÐœÑÑÐµÐœÐžÐµ ÑÐ°Ð·ÐŒÐµÑÐ° nxn.
+
+
+ImagePlug.thumbnailtype:ÐÑÐ¿ÐŸÐ»ÐœÐžÑÑ ÑÐŒÐµÐœÑÑÐµÐœÐžÐµ Ð² ÑÐŸÑÐŒÐ°ÑÐµ 's'.
+
+
+IndexPlug.desc:Ð­ÑÐŸÑ ÑÐµÐºÑÑÑÐžÐ²ÐœÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÑÐ°Ð¹Ð» index.txt. Ð€Ð°Ð¹Ð» index.txt ÐŽÐŸÐ»Ð¶ÐµÐœ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ ÑÐ¿ÐžÑÐŸÐº ÑÐ°Ð¹Ð»ÐŸÐ², ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ, ÑÐŸÐ¿ÑÐŸÐ²ÐŸÐ¶ÐŽÐ°ÐµÐŒÑÐ¹ Ð»ÑÐ±ÑÐŒÐž ÐŽÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÐŒÐž ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐŒÐž, ÑÐ²ÑÐ·Ð°ÐœÐœÑÐŒÐž Ñ ÐºÐ°Ð¶ÐŽÑÐŒ ÑÐ°Ð¹Ð»ÐŸÐŒ \n\n Ð€Ð°Ð¹Ð» index.txt ÐŽÐŸÐ»Ð¶ÐµÐœ Ð±ÑÑÑ ÑÐŸÑÐŒÐ°ÑÐžÑÐŸÐ²Ð°Ðœ ÑÐ»ÐµÐŽÑÑÑÐžÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ: Ð¿ÐµÑÐ²Ð°Ñ ÑÑÑÐŸÐºÐ° ÐŒÐŸÐ¶ÐµÑ Ð±ÑÑÑ ÐºÐ»ÑÑÐŸÐŒ (ÐœÐ°ÑÐžÐœÐ°Ñ Ñ ÐºÐ»ÑÑÐ°:), ÑÑÐŸÐ±Ñ ÐœÐ°Ð·Ð²Ð°ÑÑ Ð¿ÐŸÐ»Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐºÐ»ÑÑ: ÐÑÐµÐŽÐŒÐµÑ, ÐÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ, ÐÐ°ÑÐ°). Ð¡Ð»ÐµÐŽÑÑÑÐžÐµ ÑÑÑÐŸÐºÐž Ð±ÑÐŽÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°ÑÑ ÐžÐŒÑ ÑÐ°Ð¹Ð»Ð°, Ð·Ð°ÑÐµÐŒ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÐµÑ
+ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð²Ñ
+ÐŸÐŽÐ°, ÐºÐŸÑÐŸÑÑÐµ ÐŽÐŸÐ»Ð¶ÐœÑ Ð±ÑÑÑ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ. (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, 'irma/iw097e 3.2 unesco 1993' ÑÐ²ÑÐ¶ÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÑÐµÐŽÐŒÐµÑ=3.2, ÐÑÐ³Ð°ÐœÐžÐ·Ð°ÑÐžÑ=unesco, Ðž ÐÐ°ÑÐ°=1993 Ñ ÑÐ°Ð¹Ð»ÐŸÐŒ irma/iw097e, ÐµÑÐ»Ðž Ð²ÑÑÐµÑÐ¿ÐŸÐŒÑÐœÑÑÐ°Ñ ÑÑÑÐŸÐºÐ° ÐºÐ»ÑÑÐ° ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐ°). \n\nÐÐ°ÐŒÐµÑÑÑÐµ, ÑÑÐŸ ÐµÑÐ»Ðž Ð»ÑÐ±ÐŸÐµ ÐžÐ· Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐžÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐŸÐ³ÐŸ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑÐ°, ÐºÐŸÐ³ÐŽÐ° ÐžÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÑ , ÐŸÐœ ÐŽÐŸÐ»Ð¶ÐµÐœ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÐŸÐ²Ð°ÑÑ Ð¿ÐµÑÐ²ÐŸÐŒÑ Ð¿ÐŸÐ»Ñ (ÐŽÐµÑÐºÑÐžÐ¿ÑÐŸÑ) Ð² ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐµÐŒ ÑÐ°Ð¹Ð»Ðµ ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐžÐž. \n\nÐÐœÐ°ÑÐµÐœÐžÑ  ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ ÐŒÐŸÐ³ÑÑ Ð±ÑÑÑ ÐœÐ°Ð·Ð²Ð°ÐœÑ ÐŸÑÐŽÐµÐ»ÑÐœÐŸ, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÑ ÑÑÐ³ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ,>ÐÑÐµÐŽÐŒÐµÑ<3.2) Ðž ÑÑÐŸ Ð·Ð°ÐŒÐµÐœÐžÑ Ð»ÑÐ±ÐŸÐµ ÐžÐŒÑ, Ð¿ÑÐžÑÐ²ÐŸÐµÐœÐœÐŸÐµ ÐžÐŒ ÐºÐ»ÑÑÐµÐ²ÐŸÐ¹ ÑÑÑÐŸÐºÐŸÐ¹. ÐÑÐ»Ðž ÐºÐ»ÑÑÐµÐ²Ð°Ñ ÑÑÑÐŸÐºÐ° ÐŸÑÑÑÑÑÑÐ²ÑÐµÑ, Ð»ÑÐ±ÐŸÐµ ÐœÐµÐœÐ°Ð·Ð²Ð°ÐœÐœÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð±ÑÐŽÐµÑ ÐœÐ°Ð·Ð²Ð°ÐœÐŸ 'ÐÑÐµÐŽÐŒÐµÑ'.
+
+
+ISISPlug.desc:Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+ CDS/ISIS. ÐÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÑÐ°ÐºÐŸÐ¹ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÐœÐŸÐ¹ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+ ÐŽÐŸÐ»Ð¶ÐœÑ ÑÑÑÐµÑÑÐ²ÐŸÐ²Ð°ÑÑ ÑÑÐž ÑÐ°Ð¹Ð»Ð° Ð² Ð¿Ð°Ð¿ÐºÐµ ÐžÐŒÐ¿ÐŸÑÑÐ° ÐºÐŸÐ»Ð»ÐµÐºÑÐžÐž: ÐŸÑÐœÐŸÐ²ÐœÐŸÐ¹ ÑÐ°Ð¹Ð» (.mst), ÑÐ°Ð±Ð»ÐžÑÐ° ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸÐ»ÐµÐ¹ (.fdt), Ðž ÑÐ°Ð¹Ð» Ð¿ÐµÑÐµÐºÑÐµÑÑÐœÑÑ
+ ÑÑÑÐ»ÐŸÐº (.xrf).
+
+
+ISISPlug.subfield_separator:Ð¡ÑÑÐŸÐºÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐ°Ñ ÐŽÐ»Ñ ÐŸÑÐŽÐµÐ»ÐµÐœÐžÑ Ð¿ÐŸÐŽÐ¿ÐŸÐ»ÐµÐ¹  Ð² Ð·Ð°Ð¿ÐžÑÑÑ
+ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+ CDS/ISIS.
+
+
+ISISPlug.entry_separator:Ð¡ÑÑÐŸÐºÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐ°Ñ ÐŽÐ»Ñ ÐŸÑÐŽÐµÐ»ÐµÐœÐžÑ ÐŒÐœÐŸÐ¶ÐµÑÑÐ²ÐµÐœÐœÑÑ
+ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐŽÐ»Ñ ÐŸÑÐŽÐµÐ»ÑÐœÑÑ
+ Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Ð² Ð·Ð°Ð¿ÐžÑÑÑ
+ Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÑ
+ CDS/ISIS.
+
+
+LaTeXPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² LaTeX. 
+
+
+MACROPlug.desc:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑ Ð¿ÑÐŸÑÑÐŸÐ¹ ÐŸÐŽÐœÐŸ-ÑÑÐŸÐ²ÐœÐµÐ²ÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ. ÐÐŸÐ±Ð°Ð²Ð»ÑÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÐ°Ð·Ð²Ð°ÐœÐžÑ ÐžÐ· Ð¿ÐµÑÐ²ÐŸÐ¹ ÑÑÑÐŸÐºÐž ÑÐµÐºÑÑÐ° (ÐŽÐŸ 100 Ð·ÐœÐ°ÐºÐŸÐ² ÐŽÐ»ÐžÐœÐŸÐ¹).
+
+
+MARCPlug.desc:ÐÑÐœÐŸÐ²ÐœÐŸÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ  MARC.
+
+
+MARCPlug.metadata_mapping:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð°, ÐºÐŸÑÐŸÑÐŸÐµ Ð²ÐºÐ»ÑÑÐ°ÐµÑ ÐŽÐµÑÐ°Ð»Ðž ÐŸÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð·ÐœÐ°ÑÐµÐœÐžÐ¹ ÐžÐ· MARC Ð² ÐžÐŒÐµÐœÐ° ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ Greenstone. ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ 'marctodc.txt' ÐœÐ°Ñ
+ÐŸÐŽÐžÑÑÑ Ð²  ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž ÑÐ°Ð¹ÑÐŸÐ².
+
+
+MetadataPass.desc:ÐÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÐ¹ ÐºÐ»Ð°ÑÑ Ðº BasPlug, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ Ð¿Ð°ÑÐ°ÐŒÐµÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+ metadata_read Ð² import.pl
+
+GISBasPlug.desc:ÐÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÐ¹ ÐºÐ»Ð°ÑÑ Ðº BasPlug, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐŽÐŽÐµÑÐ¶ÐžÐ²Ð°ÐµÑ Ð²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸÑÑÐž ÐÐµÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÑ
+ ÐÐœÑÐŸÑÐŒÐ°ÑÐžÐŸÐœÑÑ
+ Ð¡ÐžÑÑÐµÐŒ (ÐÐÐ¡)
+
+NULPlug.desc:Ð¥ÐŸÐ»ÐŸÑÑÐŸÐµ Ð·ÐœÐ°ÑÐµÐœÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐŽÐ»Ñ ÑÐ°Ð¹Ð»Ð° (.nul). ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ Ñ ÑÐ°Ð¹Ð»Ð°ÐŒÐž, ÑÐŸÐ·ÐŽÐ°ÐœÐœÑÐŒÐž Ð¿ÑÐž ÑÐ°ÑÑÐžÑÐµÐœÐžÐž Ð±Ð°Ð·Ñ ÐŽÐ°ÐœÐœÑÐ¹ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐŒÐž ÑÐ°Ð¹Ð»ÐŸÐ².
+
+OAIPlug.desc:ÐÑÐœÐŸÐ²ÐœÐŸÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐžÑÑ
+ÐŸÐŽÐœÑÑ
+ ÐŸÑÐºÑÑÑÑÑ
+ Ð°ÑÑ
+ÐžÐ²ÐŸÐ² (OAI).
+
+
+OggVorbisPlug.add_technical_metadata:ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐµÑ
+ÐœÐžÑÐµÑÐºÐžÐµ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÑÐºÐŸÑÐŸÑÑÑ Ð¿ÐµÑÐµÐŽÐ°ÑÐž).
+
+
+OggVorbisPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ Ð·Ð²ÑÐºÐŸÐ²ÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² Ogg Vorbis.
+
+
+PagedImgPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÑÐŸÑÑÐ°Ð²Ð»ÐµÐœÐœÑÑ
+ ÐžÐ· Ð¿ÐŸÑÐ»ÐµÐŽÐŸÐ²Ð°ÑÐµÐ»ÑÐœÐŸÑÑÐž ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹, Ñ ÐœÐµÐŸÐ±ÑÐ·Ð°ÑÐµÐ»ÑÐœÑÐŒ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐœÐœÑÐŒ ÑÐµÐºÑÑÐŸÐŒ Ðº ÐºÐ°Ð¶ÐŽÐŸÐŒÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ.
+
+
+PagedImgPlug.documenttype:Ð£ÑÑÐ°ÐœÐŸÐ²ÐžÑÐµ ÑÐžÐ¿ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° (ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ Ð¿ÑÐŸÑÐŒÐŸÑÑÐ°).
+
+PagedImgPlug.documenttype.paged:Ð¡ÑÑÐ°ÐœÐžÑÐœÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÑÑÑÐµÐ»ÐºÐž 'Ð¡Ð»ÐµÐŽÑÑÑÐ°Ñ' Ðž 'ÐÑÐµÐŽÑÐŽÑÑÐ°Ñ' Ðž ÐºÐœÐŸÐ¿ÐºÑ 'ÐÐµÑÐµÐ¹ÑÐž Ðº ÑÑÑÐ°ÐœÐžÑÐµ X'.
+
+PagedImgPlug.documenttype.hierarchy:ÐÐµÑÐ°ÑÑ
+ÐžÑÐµÑÐºÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÐŸÐ³Ð»Ð°Ð²Ð»ÐµÐœÐžÐµ.
+
+
+PagedImgPlug.headerpage:ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° Ð·Ð°Ð³Ð»Ð°Ð²ÐœÑÑ ÑÑÑÐ°ÐœÐžÑÐºÑ (Ð±ÐµÐ· ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ).
+
+
+PagedImgPlug.screenview:Ð¡ÐŸÐ·ÐŽÐ°Ð¹ÑÐµ ÑÐºÑÐ°ÐœÐœÐŸÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, Ðž ÑÑÑÐ°ÐœÐŸÐ²ÐžÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ Screen (ÑÐºÑÐ°Ðœ), ScreenSize (ÑÐ°Ð·ÐŒÐµÑ ÑÐºÑÐ°ÐœÐ°), ScreenWidth (ÑÐžÑÐžÐœÐ° ÑÐºÑÐ°ÐœÐ°) Ðž ScreenHeight (Ð²ÑÑÐŸÑÐ° ÑÐºÑÐ°ÐœÐ°).
+
+
+PagedImgPlug.screenviewsize:Ð¡ÐŸÐ·ÐŽÐ°Ð¹ÑÐµ ÑÐºÑÐ°ÐœÐœÑÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÑÐ°Ð·ÐŒÐµÑÐ° nxn.
+
+
+PagedImgPlug.screenviewtype:Ð¡ÐŸÐ·ÐŽÐ°Ð¹ÑÐµ ÑÐºÑÐ°ÐœÐœÑÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð² ÑÐŸÑÐŒÐ°ÑÐµ 's'.
+
+
+PagedImgPlug.thumbnail:Ð¡ÐŸÐ·ÐŽÐ°Ð¹ÑÐµ ÐžÐºÐŸÐœÐºÑ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ.
+
+
+PDFPlug.complex:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑ Ð±ÐŸÐ»ÐµÐµ ÑÐ»ÐŸÐ¶ÐœÑÑ Ð¿ÑÐŸÐŽÑÐºÑÐžÑ. Ð¡ ÑÑÐŸÐ¹ ÐŸÐ¿ÑÐžÐµÐ¹ ÑÑÑÐ°ÐœÐŸÐ²ÐŸÐº Ð¿ÑÐŸÐŽÑÐºÑÐžÑ Ð² ÑÐŸÑÐŒÐ°ÑÐµ HTML Ð±ÑÐŽÐµÑ Ð²ÑÐ³Ð»ÑÐŽÐµÑÑ Ð·ÐœÐ°ÑÐžÑÐµÐ»ÑÐœÐŸ Ð±ÐŸÐ»ÐµÐµ Ð¿ÐŸÑ
+ÐŸÐ¶ÐµÐ¹ ÐœÐ° ÐŸÑÐžÐ³ÐžÐœÐ°Ð»ÑÐœÑÐ¹ PDF-ÑÐ°Ð¹Ð». ÐÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ ÑÑÐœÐºÑÐžÐŸÐœÐžÑÐŸÐ²Ð°ÑÑ ÐŽÐŸÐ»Ð¶ÐœÑÐŒ ÐŸÐ±ÑÐ°Ð·ÐŸÐŒ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ Ghostscript (ÐŽÐ»Ñ *nix gs ÐŽÐŸÐ»Ð¶ÐµÐœ Ð±ÑÑÑ ÐœÐ° Ð²Ð°ÑÐµÐŒ Ð¿ÑÑÐž ÐŽÐŸÑÑÑÐ¿Ð°, Ð² ÑÐŸ Ð²ÑÐµÐŒÑ ÐºÐ°Ðº ÐŽÐ»Ñ Windows Ñ ÐÐ°Ñ ÐŽÐŸÐ»Ð¶ÐµÐœ Ð±ÑÑÑ gswin32c.exe ÐœÐ° Ð¿ÑÑÐž ÐŽÐŸÑÑÑÐ¿Ð°).
+
+
+PDFPlug.desc:ÐÑÐžÐµÐŒÐ»ÐµÐŒÑÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ with-it pdf.
+
+
+PDFPlug.nohidden:ÐÑÐµÐ¿ÑÑÑÑÐ²ÑÐµÑ Ð¿ÐŸÐ¿ÑÑÐºÐ°ÐŒ pdftohtml ÐžÐ·Ð²Ð»ÐµÑÑ ÑÐºÑÑÑÑÐ¹ ÑÐµÐºÑÑ. Ð­ÑÐŸ Ð¿ÐŸÐ»ÐµÐ·ÐœÐŸ ÑÐŸÐ»ÑÐºÐŸ, ÐµÑÐ»Ðž ÐŸÐ¿ÑÐžÑ  -complex ÑÐ°ÐºÐ¶Ðµ ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœÐ°.
+
+
+PDFPlug.noimages:ÐÐµ Ð¿ÑÑÐ°Ð¹ÑÐµÑÑ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐžÐ· PDF.
+
+
+PDFPlug.use_sections:Ð¡ÐŸÐ·ÐŽÐ°ÑÑ ÐŸÑÐŽÐµÐ»ÑÐœÑÐ¹ ÑÐ°Ð·ÐŽÐµÐ» ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÑ PDF-ÑÐ°Ð¹Ð»Ð°.
+
+
+PDFPlug.zoom:Ð€Ð°ÐºÑÐŸÑ, c Ð¿ÐŸÐŒÐŸÑÑÑ ÐºÐŸÑÐŸÑÐŸÐ³ÐŸ ÐžÐ·ÐŒÐµÐœÑÐµÑÑÑ ÐŒÐ°ÑÑÑÐ°Ð± ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ PDF (ÑÑÐŸ Ð¿ÐŸÐ»ÐµÐ·ÐœÐŸ ÑÐŸÐ»ÑÐºÐŸ, ÐµÑÐ»Ðž   ÑÑÑÐ°ÐœÐŸÐ²Ð»ÐµÐœ -complex).
+
+
+PPTPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐ° ÑÐ°Ð¹Ð»ÐŸÐ² Microsoft PowerPoint.
+
+
+PSPlug.desc:Ð­ÑÐŸ - Ð¿ÐŸÑÑÑÐºÑÐžÐ¿ÑÑÐŒ \"Ð±ÐµÐŽÐœÑÐµ Ð»ÑÐŽÐž\" ÐŽÐ»Ñ ÑÐµÐºÑÑÐŸÐ²ÐŸÐ³ÐŸ ÐºÐŸÐœÐ²ÐµÑÑÐµÑÐ°.. ÐÑÐ»Ðž ÑÐµÑÑÐµÐ·ÐœÐŸ, ÑÐ°ÑÑÐŒÐŸÑÑÐžÑÐµ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÐŸÐ²Ð°ÐœÐžÐµ Ð¿Ð°ÐºÐµÑÐ° PRESCRIPT, ÐºÐŸÑÐŸÑÑÐ¹ ÐŽÐŸÑÑÑÐ¿ÐµÐœ ÐŽÐ»Ñ Ð·Ð°Ð³ÑÑÐ·ÐºÐž Ñ http://www.nzdl.org/html/software.html
+
+
+PSPlug.extract_date:ÐÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ ÐŽÐ°ÑÑ ÐžÐ· Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐ° PS.
+
+
+PSPlug.extract_pages:ÐÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ ÑÑÑÐ°ÐœÐžÑ ÐžÐ· Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐ° PS.
+
+
+PSPlug.extract_title:ÐÐ·Ð²Ð»ÐµÑÐµÐœÐžÐµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ ÐžÐ· Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐ° PS.
+
+
+RecPlug.desc:RecPlug - ÑÑÐŸ Ð¿Ð»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ Ð¿ÐŸÐ²ÑÐŸÑÑÐµÑÑÑ Ð² ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÐŽÐžÑÐµÐºÑÐŸÑÐžÐž, ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°Ñ ÐºÐ°Ð¶ÐŽÑÐ¹ ÑÐ°Ð¹Ð», ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐœ ÐœÐ°Ñ
+ÐŸÐŽÐžÑ.
+
+
+RecPlug.recheck_directories:ÐÐŸÑÐ»Ðµ ÑÐŸÐ³ÐŸ, ÐºÐ°Ðº ÑÐ°Ð¹Ð»Ñ Ð² Ð¿Ð°Ð¿ÐºÐµ ÐžÐŒÐ¿ÐŸÑÑÐ° Ð±ÑÐ»Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÑ, Ð¿Ð°Ð¿ÐºÐ° Ð¿ÐŸÐ²ÑÐŸÑÐœÐŸ Ð¿ÑÐŸÑÐžÑÑÐ²Ð°ÐµÑÑÑ, ÑÑÐŸÐ±Ñ ÐŸÐ±ÐœÐ°ÑÑÐ¶ÐžÑÑ Ð»ÑÐ±ÑÐµ Ð²ÐœÐŸÐ²Ñ ÑÐŸÐ·ÐŽÐ°ÐœÐœÑÐµ ÑÐ°Ð¹Ð»Ñ.
+
+
+RecPlug.use_metadata_files:Ð§ÐžÑÐ°ÑÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐžÐ· ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ XML-ÑÐ°Ð¹Ð»ÐŸÐ².
+
+
+ReferPlug.desc:ReferPlug ÑÐžÑÐ°ÐµÑ Ð±ÐžÐ±Ð»ÐžÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÑÐµÑÐµÑÐ°ÑÐžÐ²ÐœÐŸÐŒ ÑÐŸÑÐŒÐ°ÑÐµ.
+
+
+ReferPlug.longdesc:ReferPlug ÑÐžÑÐ°ÐµÑ Ð±ÐžÐ±Ð»ÐžÐŸÐ³ÑÐ°ÑÐžÑÐµÑÐºÐžÐµ ÑÐ°Ð¹Ð»Ñ Ð² ÑÐµÑÐµÑÐ°ÑÐžÐ²ÐœÐŸÐŒ ÑÐŸÑÐŒÐ°ÑÐµ. \nÐÐŸÑÐŽÐŸÐœ Ð.ÐÑÐ¹ÐœÑÐµÑ (gwp\@cs.waikato.ac.nz), ÐœÐŸÑÐ±ÑÑ 2000 Ð³.\n\nÐ¡Ð²ÐŸÐ±ÐŸÐŽÐœÐŸ ÐŸÑÐœÐŸÐ²Ð°ÐœÐœÑÐ¹ ÐœÐ° hcibib2Plug Ð¡ÑÐžÐ²ÐŸÐŒ ÐÐ¶ÐŸÑÐœÑÐŸÐŒ (stevej\@cs.waikato.ac.nz). ÐÐŸÑÐŸÑÑÐ¹ Ð±Ð°Ð·ÐžÑÐŸÐ²Ð°Ð»ÑÑ ÐœÐ° EMAILPlug ÐÐŸÑÐŽÐŸÐœÐ° ÐÑÐ¹ÐœÑÐµÑÐ° (gwp\@cs.waikato.ac.nz). ÐÐŸÑÐŸÑÑÐ¹ Ð±Ð°Ð·ÐžÑÐŸÐ²Ð°Ð»ÑÑ ÐœÐ° ÑÑÐ°ÑÑÑ
+ Ð²ÐµÑÑÐžÑÑ
+ HTMLplug Ðž HCIBIBPlugby ÐšÑÐµÑÐ°ÐœÐ° ÐÐŸÐŽÐŽÐž, Ð° ÐŽÑÑÐ³ÐžÐµ -  ÑÐµÐ¿ÐµÑÑ ÑÑÑÐŽÐœÐŸ ÑÐºÐ°Ð·Ð°ÑÑ, ÑÑÐŸ ÐŸÑÐºÑÐŽÐ° Ð²Ð·ÑÐ»ÐŸÑÑ.\n\nReferPlug ÑÐŸÐ·ÐŽÐ°ÐµÑ ÐŸÑÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ¹ ÑÑÑÐ»ÐºÐž Ð² ÑÐ°Ð¹Ð»Ðµ. ÐÐœ ÑÐ²Ð»ÑÐµÑÑÑ Ð¿ÐŸÐŽÐºÐ»Ð°ÑÑÐŸÐŒ SplitPlug, ÑÐ°Ðº ÑÑÐŸ, ÐµÑÐ»Ðž ÐµÑÑÑ ÐŒÐœÐŸÐ³ÐŸÐºÑÐ°ÑÐœÑÐµ Ð·Ð°Ð¿ÐžÑÐž, ÑÐŸ Ð²ÑÐµ ÐŸÐœÐžÑÐžÑÐ°ÑÑÑÑ. \n\n ÐÐŸÐºÑÐŒÐµÐœÑ, ÑÐµÐºÑÑ:\n\tÐ¢ÐµÐºÑÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ° ÑÐŸÑÑÐŸÐžÑ ÐžÐ· ÑÑÑÐ»ÐºÐž Ð² ÑÐµÑÐµÑÐ°ÑÐžÐ²ÐœÐŸÐŒ ÑÐŸÑÐŒÐ°ÑÐµ.\nÐÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ:\n\t\$Ð¡ÐŸÐ·ÐŽÐ°ÑÐµÐ»Ñ\%A ÐÐŒÑ Ð°Ð²ÑÐŸÑÐ°\n\t\$ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ \%T ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÑÑÐ°ÑÑÐž ÐžÐ· ÐºÐœÐžÐ³Ðž\n\t\$ÐÑÑÐœÐ°Ð» \%J ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ Ð¶ÑÑÐœÐ°Ð»Ð°\n\t\$ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐœÐžÐ³Ðž \%B ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ ÐºÐœÐžÐ³Ðž, ÑÐŸÐŽÐµÑÐ¶Ð°ÑÐµÐ¹ Ð¿ÑÐ±Ð»ÐžÐºÐ°ÑÐžÑ\n\t\$Ð¡ÐŸÐŸÐ±ÑÐµÐœÐžÐµ \%R Ð¢ÐžÐ¿ ÑÐŸÐŸÐ±ÑÐµÐœÐžÑ, ÑÑÐ°ÑÑÑ ÐžÐ»Ðž ÑÐµÐ·ÐžÑÑ\n\t\$Ð¢ÐŸÐŒ \%V ÐÐŸÐŒÐµÑ ÑÐŸÐŒÐ° Ð¶ÑÑÐœÐ°Ð»Ð°\n\t\$ÐÐŸÐŒÐµÑ \%N ÐÐŸÐŒÐµÑ Ð¶ÑÑÐœÐ°Ð»Ð° Ð² Ð¿ÑÐµÐŽÐµÐ»Ð°Ñ
+ ÑÐŸÐŒÐ°\n\t\$Ð ÐµÐŽÐ°ÐºÑÐŸÑ \%E ÐÐŒÑ ÑÐµÐŽÐ°ÐºÑÐŸÑÐ°\n\t\$Ð¡ÑÑÐ°ÐœÐžÑÑ \%P ÐÐŸÐŒÐµÑ ÑÑÑÐ°ÐœÐžÑÑ ÑÑÐ°ÑÑÐž\n\t\$ÐÐ·ÐŽÐ°ÑÐµÐ»Ñ \%I ÐÐŒÑÐžÐ·ÐŽÐ°ÑÐµÐ»Ñ\n\t\$ÐÐŽÑÐµÑ ÐžÐ·ÐŽÐ°ÑÐµÐ»Ñ \%C Ð°ÐŽÑÐµÑ\n\t\$ÐÐ°ÑÐ° \%D ÐÐ°ÑÐ° Ð¿ÑÐ±Ð»ÐžÐºÐ°ÑÐžÐž\n\t\$ÐÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð° \%K ÐÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð°, ÑÐ²ÑÐ·Ð°ÐœÐœÑÐµ Ñ Ð¿ÑÐ±Ð»ÐžÐºÐ°ÑÐžÐµÐ¹\n\t\$Ð ÐµÐ·ÑÐŒÐµ \%X Ð ÐµÐ·ÑÐŒÐµ Ð¿ÑÐ±Ð»ÐžÐºÐ°ÑÐžÐž\n\t\$ÐÐŸÐ¿ÐžÑÐ°Ð¹Ñ t\%* ÐÐœÑÐŸÑÐŒÐ°ÑÐžÑ ÐŸÐ± Ð°Ð²ÑÐŸÑÑÐºÐŸÐŒ Ð¿ÑÐ°Ð²Ðµ ÐœÐ° ÑÑÐ°ÑÑÑ.
+
+
+RogPlug.desc:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑ Ð¿ÑÐŸÑÑÑÐµ ÐŸÐŽÐœÐŸ-ÑÑÐŸÐ²ÐœÐµÐ²ÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ ÐžÐ· ÑÐ°Ð¹Ð»ÐŸÐ² .rog ÐžÐ»Ðž .mdb.
+
+
+RTFPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² RTF.
+
+
+SRCPlug.desc:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð° Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ ÐŽÐ»Ñ ÐÐ°Ð·Ð²Ð°ÐœÐžÑ (Ð¿ÑÐŸÐžÐ·Ð²ÐŸÐ»ÑÐœÐŸ Ð²ÑÐ±ÑÐ°ÐœÐœÐŸÐµ ÐŒÐžÐœÑÑ ÐœÐµÐºÐŸÑÐŸÑÑÑ Ð¿ÑÐžÑÑÐ°Ð²ÐºÑ). Ð¢ÐµÐºÑÑÐžÐµ ÑÐ·ÑÐºÐž:\nÑÐµÐºÑÑ:READMEs/Ð¡ÐŸÐ·ÐŽÐ°ÐœÐžÐµÑÐ°Ð¹Ð»ÐŸÐ²\nC/C ++ (Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÑ #Ð²ÐºÐ»ÑÑÐµÐœÐœÑÐµ ÐŸÐ¿ÐµÑÐ°ÑÐŸÑÑ Ðž decls ÐºÐ»Ð°ÑÑÐ° C ++ ) \nPerl (Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÑÐ¹ ÑÐŸÐ»ÑÐºÐŸ Ð² Ð²ÐžÐŽÐµ ÑÐµÐºÑÑÐ°) \nÐÐ±ÐŸÐ»ÐŸÑÐºÐ° (Ð² ÐœÐ°ÑÑÐŸÑÑÐµÐµ Ð²ÑÐµÐŒÑ Ð¿ÑÐµÐŽÑÑÐ°Ð²Ð»ÐµÐœÐœÐ°Ñ ÑÐŸÐ»ÑÐºÐŸ Ð² Ð²ÐžÐŽÐµ ÑÐµÐºÑÑÐ°).
+
+
+SRCPlug.remove_prefix:Ð£ÐŽÐ°Ð»ÐžÑÑ ÑÑÐŸÑ Ð²ÐµÐŽÑÑÐžÐ¹ ÐŸÐ±ÑÐ°Ð·ÐµÑ ÐžÐ· ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð° (eg-remove_prefix/tmp/xx/src/). ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ ÐŽÐŸÐ»Ð¶ÐµÐœ Ð²ÐµÑÑ Ð¿ÑÑÑ ÐŽÐŸÑÑÑÐ¿Ð° ÐŸÑ ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð°.
+
+
+SplitPlug.desc:SplitPlug - ÑÑÐŸ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ Ð²Ñ
+ÐŸÐŽÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐœÐ° ÑÐµÐ³ÐŒÐµÐœÑÑ, ÐºÐŸÑÐŸÑÑÐµ Ð±ÑÐŽÑÑ Ð¿ÐŸÑÐŸÐŒ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÑÑÑÑ Ð¿ÐŸ ÐŸÑÐŽÐµÐ»ÑÐœÐŸÑÑÐž.Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐœÐµÐ»ÑÐ·Ñ Ð²ÑÐ·Ð²Ð°ÑÑ ÐœÐµÐ¿ÐŸÑÑÐµÐŽÑÑÐ²ÐµÐœÐœÐŸ. ÐÐŒÐµÑÑÐŸ ÑÑÐŸÐ³ÐŸ, ÐµÑÐ»Ðž ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÑÑ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÑÐ°Ð¹Ð»Ñ, ÐºÐŸÑÐŸÑÑÐµ ÑÐŸÐŽÐµÑÐ¶Ð°Ñ ÐœÐµÑÐºÐŸÐ»ÑÐºÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐÑ ÐŽÐŸÐ»Ð¶ÐœÑ ÐœÐ°Ð¿ÐžÑÐ°ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœ Ñ ÑÑÐœÐºÑÐžÐµÐ¹ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž, ÐºÐŸÑÐŸÑÑÐ¹ Ð±ÑÐŽÐµÑ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÑÑ ÐŸÐŽÐžÐœ ÐžÐ· ÑÑÐžÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ², ÐœÐ°ÑÐ»ÐµÐŽÑÑ ÐµÐ³ÐŸ ÐžÐ· SplitPlug. Ð¡ÐŒ. ReferPlug ÐŽÐ»Ñ Ð¿ÑÐžÐŒÐµÑÐ°.
+
+
+SplitPlug.split_exp:ÐÑÐ°Ð²ÐžÐ»ÑÐœÐŸÐµ Ð²ÑÑÐ°Ð¶ÐµÐœÐžÐµ perl ÐŽÐ»Ñ ÑÐ°Ð·Ð±ÐžÐµÐœÐžÑ Ð²Ñ
+ÐŸÐŽÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ² ÐœÐ° ÑÐ°ÑÑÐž.
+
+
+TEXTPlug.desc:Ð¡ÐŸÐ·ÐŽÐ°ÐµÑ Ð¿ÑÐŸÑÑÐŸÐ¹ ÐŸÐŽÐœÐŸ-ÑÑÐŸÐ²ÐœÐµÐ²ÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ. ÐÐŸÐ±Ð°Ð²Ð»ÑÐµÑ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÐÐ°Ð·Ð²Ð°ÐœÐžÑ ÐžÐ· Ð¿ÐµÑÐ²ÐŸÐ¹ ÑÑÑÐŸÐºÐž ÑÐµÐºÑÑÐ° (ÐŽÐ»ÐžÐœÐŸÐ¹ ÐŽÐŸ 100 Ð·ÐœÐ°ÐºÐŸÐ².
+
+
+TEXTPlug.title_sub:ÐÑÑÐ°Ð¶ÐµÐœÐžÐµ Ð·Ð°ÐŒÐµÐœÑ ÐŽÐ»Ñ ÐžÐ·ÐŒÐµÐœÐµÐœÐžÑ ÑÑÑÐŸÐºÐž, Ñ
+ÑÐ°ÐœÑÑÐµÐ¹ÑÑ ÐºÐ°Ðº ÐÐ°Ð·Ð²Ð°ÐœÐžÐµ. ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÑÑÑ, ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, PSPlug, ÑÑÐŸÐ±Ñ ÑÐŽÐ°Ð»ÐžÑÑ "Ð¡ÑÑÐ°ÐœÐžÑÑ 1" Ðž Ñ.ÐŽ ÐžÐ· ÑÐµÐºÑÑÐ°, ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÐŸÐ³ÐŸ Ð² ÐºÐ°ÑÐµÑÑÐ²Ðµ ÐœÐ°Ð·Ð²Ð°ÐœÐžÑ.
+
+
+UnknownPlug.assoc_field:ÐÐŒÑ Ð¿ÐŸÐ»Ñ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+, ÐºÐŸÑÐŸÑÐŸÐµ Ð±ÑÐŽÐµÑ ÐŽÐµÑÐ¶Ð°ÑÑ ÐžÐŒÐµÐœÐ° Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÑ
+ ÑÐ°Ð¹Ð»ÐŸÐ².
+
+
+UnknownPlug.desc:Ð­ÑÐŸ Ð¿ÑÐŸÑÑÐŸÐ¹ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÑÐ°Ð¹Ð»ÐŸÐ² Ð² ÑÐŸÑÐŒÐ°ÑÐ°Ñ
+, ÐœÐµÐžÐ·Ð²ÐµÑÑÐœÑÑ
+ ÐŽÐ»Ñ Greenstone.  ÐÑÐŒÑÑÐ»ÐµÐœÐœÑÐ¹ ÐŽÐŸÐºÑÐŒÐµÐœÑ Ð±ÑÐŽÐµÑ ÑÐŸÐ·ÐŽÐ°Ðœ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÑÐ°ÐºÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð°, Ðž ÑÐ°ÐŒ ÑÐ°Ð¹Ð» Ð±ÑÐŽÐµÑ Ð¿ÐµÑÐµÐŽÐ°Ðœ Ð² Greenstone ÐºÐ°Ðº \"Ð°ÑÑÐŸÑÐžÐžÑÐŸÐ²Ð°ÐœÐœÑÐ¹ ÑÐ°Ð¹Ð»\" ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°.
+
+
+UnknownPlug.file_format:Ð¢ÐžÐ¿ ÑÐ°Ð¹Ð»Ð° (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, MPEG, MIDI, ...).
+
+
+UnknownPlug.mime_type:Ð¢ÐžÐ¿ ÑÐ°Ð¹Ð»Ð°  mime (ÐœÐ°Ð¿ÑÐžÐŒÐµÑ, ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐµ/gif).
+
+
+UnknownPlug.process_extension:ÐÐ±ÑÐ°Ð±ÐŸÑÐºÐ° ÑÐ°Ð¹Ð»ÐŸÐ² Ñ ÐŸÐ¿ÑÐµÐŽÐµÐ»ÐµÐœÐœÑÐŒ ÑÐ°ÑÑÐžÑÐµÐœÐžÐµÐŒ. Ð­ÑÐ° ÐŸÐ¿ÑÐžÑ - Ð°Ð»ÑÑÐµÑÐœÐ°ÑÐžÐ²Ð° Ðº process_exp, ÐºÐŸÑÐŸÑÐ°Ñ Ð¿ÑÐŸÑÐµ, ÐœÐŸ ÐŒÐµÐœÐµÐµ Ð¿ÑÐžÑÐ¿ÐŸÑÐ°Ð±Ð»ÐžÐ²Ð°ÐµÐŒÐ°Ñ.
+
+
+MP3Plug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž MP3 ÑÐ°Ð¹Ð»ÐŸÐ².
+
+
+MP3Plug.assoc_images:Use google image search to locate images related to MP3 file based on ID3 Title and Artist metadata.
+
+MP3Plug.assoc_images:ÐÑÐ¿ÐŸÐ»ÑÐ·Ð°Ð²Ð°ÐœÐœÐŸ ÐŽÐ»Ñ ÑÐŸÐ³ÐŸ, ÑÑÐŸÐ±Ñ Ñ
+ÑÐ°ÐœÐžÑÑ [Ð°Ð¿Ð¿Ð»ÐµÑ] ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÐµ ÐŽÐ»Ñ ÐºÐ°Ð¶ÐŽÐŸÐ³ÐŸ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°, Ð² ÐºÐŸÑÐŸÑÑÑ
+ Ð·Ð°Ð¿ÐžÑÐ°Ðœ HTML ÐŽÐ»Ñ Ð¿ÑÐŸÐžÐ³ÑÑÐ²Ð°ÐœÐžÑ ÑÑÐŸÐ³ÐŸ ÑÐ°Ð¹Ð»Ð° Ð² MP3 Ð¿ÑÐŸÐžÐ³ÑÑÐ²Ð°ÑÐµÐ»Ðµ.
+
+
+MP3Plug.metadata_fields:Ð¡Ð¿ÐžÑÐŸÐº Ð¿ÐŸÐ»ÐµÐ¹ ÐŒÐµÑÐ°-ÐŽÐ°ÐœÐœÑÑ
+, Ð¿ÐµÑÐµÑÐžÑÐ»ÐµÐœÐœÑÑ
+ ÑÐµÑÐµÐ· Ð·Ð°Ð¿ÑÑÑÑ, ÐºÐŸÑÐŸÑÑÐµ Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐµÐœÑ ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÐžÐ· MP3 ÑÐ°Ð¹Ð»Ð° (ÐµÑÐ»Ðž ÐŽÐŸÑÑÑÐ¿ÐµÐœ). ÐÑÐ¿ÐŸÐ»ÑÐ·ÑÐ¹ÑÐµ \"*\" ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ Ð²ÑÐµÑ
+ Ð¿ÐŸÐ»ÐµÐ¹.
+
+
+W3ImgPlug.aggressiveness:Ð Ð°ÐœÐ³ ÑÐ²ÑÐ·Ð°ÐœÐœÑÑ
+ ÐŒÐµÑÐŸÐŽÐŸÐ², ÐžÑÐ¿ÐŸÐ»ÑÐ·ÑÐµÐŒÑÑ
+ ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÑÐµÐºÑÑÐ°.
+
+W3ImgPlug.aggressiveness.1:ÐÐŒÑ ÑÐ°Ð¹Ð»Ð°, Ð¿ÑÑÑ, ÐŒÐŸÐŽÐžÑÐžÐºÐ°ÑÐžÑ (ALT) ÑÐŸÐ»ÑÐºÐŸ ÑÐµÐºÑÑÐ°.
+
+W3ImgPlug.aggressiveness.2:ÐÑÐµ ÐžÐ· 1 Ð¿Ð»ÑÑ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐŸÐº, Ð³ÐŽÐµ ÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ.
+
+W3ImgPlug.aggressiveness.3:ÐÑÐµ  ÐžÐ· 2 Ð¿Ð»ÑÑ Ð±Ð»ÐžÐ¶Ð°Ð¹ÑÐžÐµ Ð¿Ð°ÑÐ°Ð³ÑÐ°ÑÑ, Ð³ÐŽÐµ ÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ.
+
+W3ImgPlug.aggressiveness.4:ÐÑÐµ ÐžÐ· 3 Ð¿Ð»ÑÑ Ð¿ÑÐµÐŽÑÐŽÑÑÐžÐµ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐž (<h1>, <h2> ...), Ð³ÐŽÐµ ÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ.
+
+W3ImgPlug.aggressiveness.5:ÐÑÐµ ÐžÐ· 4 Ð¿Ð»ÑÑ ÑÐµÐºÑÑÐŸÐ²ÑÐµ ÑÑÑÐ»ÐºÐž, Ð³ÐŽÐµ ÑÑÐŸ ÐŽÐŸÑÑÑÐ¿ÐœÐŸ.
+
+W3ImgPlug.aggressiveness.6:ÐÑÐµ ÐžÐ· 4 Ð¿Ð»ÑÑ ÑÑÑÐ°ÐœÐžÑÐ° ÐŒÐµÑÐ°ÑÑÐ³ÐŸÐ² (ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ, ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð° Ðž Ñ.ÐŽ).
+
+W3ImgPlug.aggressiveness.7:ÐÑÐµ  ÐžÐ· ÐŸÐ±ÑÐµÐŽÐžÐœÐµÐœÐœÑÑ
+ 6, 5 Ðž 4.
+
+W3ImgPlug.aggressiveness.8:ÐÑÐµ ÐžÐ· 7 Ð¿Ð»ÑÑ Ð¿ÐŸÐ²ÑÐŸÑÐµÐœÐžÐµ Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐ°, ÐžÐŒÐµÐœÐž ÑÐ°Ð¹Ð»Ð° Ðž Ñ.ÐŽ ( Ð¿ÐŸÐ²ÑÑÐµÐœÐžÐµ ÑÐ°ÐœÐ³Ð° Ð±ÐŸÐ»ÐµÐµ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÑ
+ ÑÐµÐ·ÑÐ»ÑÑÐ°ÑÐŸÐ²).
+
+W3ImgPlug.aggressiveness.9:ÐÑÐµ ÐžÐ· 1 Ð¿Ð»ÑÑ Ð¿ÐŸÐ»ÐœÑÐ¹ ÑÐµÐºÑÑ ÐžÑÑ
+ÐŸÐŽÐœÐŸÐ¹ ÑÑÑÐ°ÐœÐžÑÑ.
+
+
+W3ImgPlug.caption_length:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐ°Ñ ÐŽÐ»ÐžÐœÐ° Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐŸÐ² (Ð² Ð·ÐœÐ°ÐºÐ°Ñ
+).
+
+
+W3ImgPlug.convert_params: ÐÐŸÐ¿ÐŸÐ»ÐœÐžÑÐµÐ»ÑÐœÑÐµ Ð¿Ð°ÑÐ°ÐŒÐµÑÑÑ ÐŽÐ»Ñ ImageMagicK ÐºÐŸÐœÐ²ÐµÑÑÐžÑÑÑÑ ÑÐŒÐµÐœÑÑÐµÐœÐœÑÐµ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ. ÐÐ°Ð¿ÑÐžÐŒÐµÑ, '-Ð¿ÐŸÐ²ÑÑÐµÐœÐžÐµ' ÐŽÐ°ÑÑ ÑÑÐµÑ
+ÐŒÐµÑÐœÑÐ¹ Ð²ÐžÐŽ ÑÐŒÐµÐœÑÑÐµÐœÐœÑÐŒ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑÐŒ.
+
+
+W3ImgPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐ·Ð²Ð»ÐµÑÐµÐœÐžÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹ Ðž ÑÐ²ÑÐ·Ð°ÐœÐœÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ° ÐžÐ· Ð²ÐµÐ±-ÑÑÑÐ°ÐœÐžÑ.
+
+
+W3ImgPlug.document_text:ÐÐŸÐ±Ð°Ð²ÐžÑÑ ÑÐµÐºÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ Ð² Ð²ÐžÐŽÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÐ°:ÑÐµÐºÑÑ (ÐžÐœÐ°ÑÐµ, Ð¿ÐŸÐ»Ðµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÑ
+ IndexedText).
+
+
+W3ImgPlug.index_pages:ÐÐœÐŽÐµÐºÑÐžÑÐŸÐ²Ð°ÑÑ ÑÑÑÐ°ÐœÐžÑÑ ÐœÐ°ÑÑÐŽÑ Ñ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑÐŒÐž. ÐÐœÐ°ÑÐµ, ÑÐŸÑÐ»Ð°ÑÑÑÑ ÐœÐ° ÑÑÑÐ°ÐœÐžÑÑ Ð² ÐžÑÑ
+ÐŸÐŽÐœÐŸÐŒ URL.
+
+
+W3ImgPlug.max_near_text:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸ Ð·ÐœÐ°ÐºÐŸÐ² Ñ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÐŒÑÑ
+ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹.
+
+
+W3ImgPlug.min_height:ÐÐžÐºÑÐµÐ»Ñ. ÐÑÐŸÐ¿ÑÑÑÐžÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, ÐŒÐµÐœÑÑÐžÐµ, ÑÐµÐŒ ÑÑÐŸ.
+
+
+W3ImgPlug.min_near_text:ÐÐžÐœÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÐºÐŸÐ»ÐžÑÐµÑÑÐ²ÐŸ Ð·ÐœÐ°ÐºÐŸÐ² Ñ ÐžÐ·Ð²Ð»ÐµÐºÐ°ÐµÐŒÐŸÐ³ÐŸ ÑÐµÐºÑÑÐ° ÐžÐ»Ðž Ð·Ð°Ð³ÐŸÐ»ÐŸÐ²ÐºÐ°.
+
+
+W3ImgPlug.min_size:ÐÐ°Ð¹ÑÑ. ÐÑÐŸÐ¿ÑÑÑÐžÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, ÐŒÐµÐœÑÑÐžÐµ, ÑÐµÐŒ ÑÑÐŸ.
+
+
+W3ImgPlug.min_width:ÐÐžÐºÑÐµÐ»Ñ. ÐÑÐŸÐ¿ÑÑÑÐžÑÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ, Ð±ÐŸÐ»ÐµÐµ ÑÐ·ÐºÐžÐµ, ÑÐµÐŒ ÑÑÐŸ.
+
+
+W3ImgPlug.neartext_length:ÐÐ»ÐžÐœÐ° Ð°ÐŽÑÐµÑÐ° Ñ ÑÐµÐºÑÑÐ° (Ð² Ð·ÐœÐ°ÐºÐ°Ñ
+).
+
+
+W3ImgPlug.no_cache_images:ÐÐµ ÐŽÐµÐ»Ð°ÑÑ ÐºÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐžÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÐ¹ (ÑÐºÐ°Ð·Ð°ÑÑ URL ÐŸÑÐžÐ³ÐžÐœÐ°Ð»Ð°).
+
+
+W3ImgPlug.smallpage_threshold:ÐÐ·ÐŸÐ±ÑÐ°Ð¶ÐµÐœÐžÑ ÐœÐ° ÑÑÑÐ°ÐœÐžÑÐ°Ñ
+, ÐŒÐµÐœÑÑÐžÑ
+ ÑÐµÐŒ ÑÑÐ° ( Ð² Ð±Ð°Ð¹ÑÐ°Ñ
+), Ð±ÑÐŽÑÑ ÐžÐŒÐµÑÑ ÐŽÐŸÐ±Ð°Ð²Ð»ÐµÐœÐœÑÐµ ÐŒÐµÑÐ°ÐŽÐ°ÐœÐœÑÐµ ÑÑÑÐ°ÐœÐžÑÑ (ÐœÐ°Ð·Ð²Ð°ÐœÐžÐµ, ÐºÐ»ÑÑÐµÐ²ÑÐµ ÑÐ»ÐŸÐ²Ð° Ðž Ñ.ÐŽ).
+
+
+W3ImgPlug.textrefs_threshold:ÐÐŸÑÐŸÐ³ ÐŽÐ»Ñ ÑÐµÐºÑÑÐŸÐ²ÑÑ
+ ÑÑÑÐ»ÐŸÐº. ÐÐŸÐ»ÐµÐµ ÐœÐžÐ·ÐºÐžÐµ Ð·ÐœÐ°ÑÐµÐœÐžÑ Ð¿ÐŸÐŽÑÐ°Ð·ÑÐŒÐµÐ²Ð°ÑÑ, ÑÑÐŸ Ð°Ð»Ð³ÐŸÑÐžÑÐŒ ÐŒÐµÐœÐµÐµ ÑÑÑÐŸÐ³.
+
+
+W3ImgPlug.thumb_size:ÐÐ°ÐºÑÐžÐŒÐ°Ð»ÑÐœÐŸÐµ ÑÐŒÐµÐœÑÑÐµÐœÐžÐµ ÑÐ°Ð·ÐŒÐµÑÐ° - Ðž ÑÐžÑÐžÐœÑ, Ðž Ð²ÑÑÐŸÑÑ.
+
+
+WordPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ ÐŽÐ»Ñ ÐžÐŒÐ¿ÐŸÑÑÐžÑÐŸÐ²Ð°ÐœÐžÑ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ² Microsoft Word.
+
+
+XMLPlug.desc:ÐÑÐœÐŸÐ²ÐœÐŸÐ¹ ÐºÐ»Ð°ÑÑ Ð¿Ð»Ð°Ð³ÐžÐœÐŸÐ² XML.
+
+
+ZIPPlug.desc:ÐÐ»Ð°Ð³ÐžÐœ, ÐºÐŸÑÐŸÑÑÐ¹ ÐŸÐ±ÑÐ°Ð±Ð°ÑÑÐ²Ð°ÐµÑ ÑÐ¶Ð°ÑÑÐµ Ðž/ÐžÐ»Ðž Ð°ÑÑ
+ÐžÐ²ÐžÑÐŸÐ²Ð°ÐœÐœÑÐµ Ð²Ñ
+ÐŸÐŽÐœÑÐµ ÑÐŸÑÐŒÐ°ÑÑ, ÑÐµÐºÑÑÐžÐµ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÐœÑÐµ ÑÐŸÑÐŒÐ°ÑÑ Ðž ÑÐ°Ð¹Ð»Ñ Ñ ÑÐ°ÑÑÐžÑÐµÐœÐžÑÐŒÐž:\ngzip (.gz, .z, .tgz, .taz)\nbzip (.bz)\nbzip2 (.bz2)\nzip (.zip .jar)\ntar (.tar)\n\n Ð­ÑÐŸÑ Ð¿Ð»Ð°Ð³ÐžÐœ ÐŸÐ¿ÐžÑÐ°ÐµÑÑÑ ÐœÐ° ÑÐ»ÐµÐŽÑÑÑÐžÐµ Ð¿ÑÐžÑÑÑÑÑÐ²ÑÑÑÐžÐµ ÑÑÐžÐ»ÐžÑÑ (ÐµÑÐ»Ðž Ð¿ÑÑÐ°ÑÑÑÑ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÑÑ ÑÐŸÐŸÑÐ²ÐµÑÑÑÐ²ÑÑÑÐžÐµ ÑÐŸÑÐŒÐ°ÑÑ):\ngunzip (ÐŽÐ»Ñ gzip)\nbunzip (ÐŽÐ»Ñ bzip)\nbunzip2 \nunzip (ÐŽÐ»Ñzip)\ntar (ÐŽÐ»Ñ tar).
+
+
+
+#
+# Perl module strings
+#
+
+classify.could_not_find_classifier:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐ°Ð¹ÑÐž ÐºÐ»Ð°ÑÑÐžÑÐžÐºÐ°ÑÐŸÑ \"%s\"
+
+
+plugin.could_not_find_plugin:ÐÐšÐÐÐÐ: ÐÐµÐ²ÐŸÐ·ÐŒÐŸÐ¶ÐœÐŸ ÐœÐ°Ð¹ÑÐž Ð¿Ð»Ð°Ð³ÐžÐœ \"%s\"
+
+
+plugin.including_archive:Ð²ÐºÐ»ÑÑÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ 1 Ð°ÑÑ
+ÐžÐ²Ð° ZIP/TAR. 
+
+
+plugin.including_archives:Ð²ÐºÐ»ÑÑÐµÐœÐžÐµ ÑÐŸÐŽÐµÑÐ¶Ð°ÐœÐžÑ  Ð°ÑÑ
+ÐžÐ²ÐŸÐ² %d ZIP/TAR 
+
+
+plugin.kill_file:ÐÐ±ÑÐ°Ð±ÐŸÑÐºÐ°, ÑÐœÐžÑÑÐŸÐ¶ÐµÐœÐœÐ°Ñ ÑÐ°Ð¹Ð»ÐŸÐŒ .kill.
+
+
+plugin.n_considered:%d ÐŽÐŸÐºÑÐŒÐµÐœÑÑ, Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐµÐœÐœÑÐµ ÐŽÐ»Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž
+
+
+plugin.n_included:%d Ð±ÑÐ»Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÐœÑ Ðž Ð²ÐºÐ»ÑÑÐµÐœÑ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+
+
+plugin.n_rejected:%d Ð±ÑÐ»Ðž ÐŸÑÐºÐ»ÐŸÐœÐµÐœÑ
+
+
+plugin.n_unrecognised:%d Ð·ÐŽÐµÑÑ ÐœÐµÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐœÐŸ
+
+
+plugin.no_plugin_could_process:ÐÐ ÐÐÐ£ÐÐ ÐÐÐÐÐÐÐ: ÐÐžÐºÐ°ÐºÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐœÐµ ÑÐŒÐŸÐ³Ð»Ðž ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÑÑ %s
+
+
+plugin.no_plugin_could_recognise:ÐÐÐÐÐÐÐÐ: ÐÐµÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐºÐŸÑÐŸÑÑÐ¹ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐµÑ %s
+
+
+plugin.no_plugin_could_process_this_file:ÐœÐžÐºÐ°ÐºÐžÐµ Ð¿Ð»Ð°Ð³ÐžÐœÑ ÐœÐµ ÑÐŒÐŸÐ³Ð»Ðž  ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°ÑÑ ÑÑÐŸÑ ÑÐ°Ð¹Ð»
+
+
+plugin.no_plugin_could_recognise_this_file:ÐœÐµÑ Ð¿Ð»Ð°Ð³ÐžÐœÐ° ÐºÐŸÑÐŸÑÑÐ¹ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐµÑ ÑÑÐŸÑ ÑÐ°Ð¹Ð»
+
+
+plugin.one_considered:1 ÐŽÐŸÐºÑÐŒÐµÐœÑ Ð¿ÑÐµÐŽÐœÐ°Ð·ÐœÐ°ÑÐ°Ð»ÑÑ ÐŽÐ»Ñ ÐŸÐ±ÑÐ°Ð±ÐŸÑÐºÐž
+
+
+plugin.one_included:1 Ð±ÑÐ» ÐŸÐ±ÑÐ°Ð±ÐŸÑÐ°Ðœ Ðž Ð²ÐºÐ»ÑÑÐµÐœ Ð² ÐºÐŸÐ»Ð»ÐµÐºÑÐžÑ
+
+
+plugin.one_rejected:1 Ð±ÑÐ» ÐŸÑÐºÐ»ÐŸÐœÐµÐœ
+
+
+plugin.one_unrecognised:1 Ð±ÑÐ» ÐœÐµ ÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°Ðœ
+
+plugin.see_faillog:Ð¡ÐŒ. %s ÐŽÐ»Ñ ÑÐ¿ÐžÑÐºÐ° ÐœÐµÑÐ°ÑÐ¿ÐŸÐ·ÐœÐ°ÐœÐœÑÑ
+  Ðž/ÐžÐ»Ðž ÐŸÑÐºÐ»ÐŸÐœÐµÐœÐœÑÑ
+ ÐŽÐŸÐºÑÐŒÐµÐœÑÐŸÐ²
+
+
+PrintUsage.default:ÐÐŸ ÑÐŒÐŸÐ»ÑÐ°ÐœÐžÑ
+
+
+PrintUsage.required:Ð¢Ð ÐÐÐ£ÐÐÐ«Ð
+
Index: /main/tags/3.01/gsdl/perllib/textcat.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/textcat.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/textcat.pm	(revision 21112)
@@ -0,0 +1,149 @@
+###########################################################################
+#
+# textcat.pm -- Identify the language of a piece of text
+#
+#
+# This file is based on TextCat version 1.08 by Gertjan van Noord
+# Copyright (C) 1997 Gertjan van Noord (vannoord@let.rug.nl)
+# TextCat is available from: http://odur.let.rug.nl/~vannoord/TextCat 
+#
+# It was modified by Gordon Paynter (gwp@cs.waikato.ac.nz) and turned
+# into a package for use in Greenstone digital library system.  Most of
+# the modifications consist of commenting out or deleting functionality
+# I don't need.  
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package textcat;
+
+# OPTIONS
+my $model_dir = $ENV{'GSDLHOME'} . "/perllib/textcat";
+
+my $opt_f = 1;                # Ngrams which occur <= this number of times are removed
+my $opt_t = 400;              # topmost number of ngrams that should be used
+my $opt_u = 1.05;             # how much worse result must be before it is ignored
+
+my $non_word_characters = '0-9\s';
+
+sub new {
+    my $class = shift (@_);
+    my $self = {};
+
+    # open directory to find which languages are supported
+    opendir DIR, "$model_dir" or die "directory $model_dir: $!\n";
+    my @languages = sort(grep { s/\.lm// && -r "$model_dir/$_.lm" } readdir(DIR));
+    closedir DIR;
+    @languages or die "sorry, can't read any language models from $model_dir\n" .
+	"language models must reside in files with .lm ending\n";
+
+    # load model and count for each language.
+    foreach my $language (@languages) {
+	my %ngram=();
+	my $rang=1;
+	open(LM, "$model_dir/$language.lm") || die "cannot open $language.lm: $!\n";
+	while (<LM>) {
+	    chomp;
+	    # only use lines starting with appropriate character. Others are ignored.
+	    if (/^[^$non_word_characters]+/o) {
+		$self->{'ngrams'}->{$language}->{$&} = $rang++;
+	    } 
+	}
+	close(LM);
+    }
+
+    $self->{'languages'} = \@languages;
+    return bless $self, $class;
+}
+
+
+
+# CLASSIFICATION
+#
+# What language is a text string?
+#   Input:  text string
+#   Output: array of language names
+
+sub classify {
+    my ($self, $inputref)=@_;
+    my %results = ();
+    my $maxp = $opt_t;
+
+    # create ngrams for input.
+    my $unknown = $self->create_lm($inputref);
+
+    foreach my $language (@{$self->{'languages'}}) {
+    
+	# compare language model with input ngrams list
+	my ($i,$p)=(0,0);
+	while ($i < scalar (@$unknown)) {
+	    if (defined ($self->{'ngrams'}->{$language}->{$unknown->[$i]})) {
+		$p=$p+abs($self->{'ngrams'}->{$language}->{$unknown->[$i]}-$i);
+	    } else { 
+		$p=$p+$maxp; 
+	    }
+	    ++$i;
+	}
+	$results{$language} = $p;
+    }
+
+    my @results = sort { $results{$a} <=> $results{$b} } keys %results;
+    my $a = $results{$results[0]};
+  
+    my @answers=(shift(@results));
+    while (@results && $results{$results[0]} < ($opt_u *$a)) {
+	@answers=(@answers,shift(@results));
+    }
+
+    return \@answers;
+}
+
+sub create_lm {
+    # $ngram contains reference to the hash we build
+    # then add the ngrams found in each word in the hash
+    my ($self, $textref) = @_;  
+    
+    my $ngram = {};
+
+    foreach my $word (split(/[$non_word_characters]+/, $$textref)) {
+	$word = "_" . $word . "_";
+	my $len = length($word);
+	my $flen=$len;
+	my $i;
+
+	for ($i=0; $i<$flen; $i++) {
+	    $ngram->{substr($word,$i,5)}++ if $len > 4;
+	    $ngram->{substr($word,$i,4)}++ if $len > 3;
+	    $ngram->{substr($word,$i,3)}++ if $len > 2;
+	    $ngram->{substr($word,$i,2)}++ if $len > 1;
+	    $ngram->{substr($word,$i,1)}++;
+	    $len--;
+	}
+    }
+
+    map { if ($ngram->{$_} <= $opt_f) { delete $ngram->{$_}; }
+      } keys %$ngram;
+  
+    # sort the ngrams, and spit out the $opt_t frequent ones.
+    # adding  `or $a cmp $b' in the sort block makes sorting five
+    # times slower..., although it would be somewhat nicer (unique result)
+    my @sorted = sort { $ngram->{$b} <=> $ngram->{$a} } keys %$ngram;
+    splice(@sorted,$opt_t) if (@sorted > $opt_t); 
+    return \@sorted;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/unbuildutil.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/unbuildutil.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/unbuildutil.pm	(revision 21112)
@@ -0,0 +1,48 @@
+###########################################################################
+#
+# unbuildutil.pm -- common methods for the two unbuilding scripts
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package unbuildutil;
+
+sub read_gdbm {
+    my ($filename, $infodb) = @_;
+
+    open (PIPEIN, "db2txt \"$filename\" |") || die "couldn't open pipe from db2txt\n";
+    my $line = ""; my $key = ""; my $value = "";
+    while (defined ($line = <PIPEIN>)) {
+	if ($line =~ /^\[([^\]]+)\]$/) {
+	    $key = $1;
+	} elsif ($line =~ /^-{70}$/) {
+	    $infodb->{$key} = $value;
+	    #print STDERR "adding key $key, value $value\n";
+	    $value = "";
+	    $key = "";
+	} else {
+	    $value .= $line;
+	}
+    }
+    close PIPEIN;
+}
+
+1;
Index: /main/tags/3.01/gsdl/perllib/unicode.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/unicode.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/unicode.pm	(revision 21112)
@@ -0,0 +1,568 @@
+###########################################################################
+#
+# unicode.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2004 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# useful functions for dealing with Unicode
+
+# Unicode strings are stored as arrays of scalars as perl 
+# lacks characters are 8-bit (currently)
+
+package unicode;
+
+eval {require bytes};
+
+use encodings;
+use util;
+
+# ascii2unicode takes an (extended) ascii string (ISO-8859-1)
+# and returns a unicode array.
+sub ascii2unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+	push (@$out, ord(substr ($in, $i, 1)));	
+	$i++;
+    }
+
+    return $out;
+}
+
+# ascii2utf8 takes a reference to an (extended) ascii string and returns a
+# UTF-8 encoded string. This is just a faster version of
+# "&unicode2utf8(&ascii2unicode($str));"
+# "Extended ascii" really means "iso_8859_1"
+sub ascii2utf8 {
+    my ($in) = @_;
+    my $out = "";
+
+    if (!defined($in)|| !defined($$in)) {
+	return $out;
+    }
+
+    my ($c);
+    my $i = 0;
+    my $len = length($$in);
+    while ($i < $len) {
+	$c = ord (substr ($$in, $i, 1));
+	if ($c < 0x80) {
+	    # ascii character
+	    $out .= chr ($c);
+
+	} else {
+	    # extended ascii character
+	    $out .= chr (0xc0 + (($c >> 6) & 0x1f));
+	    $out .= chr (0x80 + ($c & 0x3f));
+	}
+	$i++;
+    }
+
+    return $out;
+}
+
+# unicode2utf8 takes a unicode array as input and encodes it
+# using utf-8
+sub unicode2utf8 {
+    my ($in) = @_;
+    my $out = "";
+    
+    foreach my $num (@$in) {
+	next unless defined $num;
+	if ($num < 0x80) {
+	    $out .= chr ($num);
+
+	} elsif ($num < 0x800) {
+	    $out .= chr (0xc0 + (($num >> 6) & 0x1f));
+	    $out .= chr (0x80 + ($num & 0x3f));
+
+	} elsif ($num < 0xFFFF) {
+	    $out .= chr (0xe0 + (($num >> 12) & 0xf));
+	    $out .= chr (0x80 + (($num >> 6) & 0x3f));
+	    $out .= chr (0x80 + ($num & 0x3f));
+
+	} else {
+	    # error, don't encode anything
+	    die;
+	}
+    }
+    return $out;
+}
+
+# utf82unicode takes a utf-8 string and produces a unicode
+# array
+sub utf82unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my ($c1, $c2, $c3);
+    my $len = length($in);
+    while ($i < $len) {
+	if (($c1 = ord(substr ($in, $i, 1))) < 0x80) {
+	    # normal ascii character
+	    push (@$out, $c1);
+
+	} elsif ($c1 < 0xc0) {
+	    # error, was expecting the first byte of an
+	    # encoded character. Do nothing.
+
+	} elsif ($c1 < 0xe0 && $i+1 < $len) {
+	    # an encoded character with two bytes
+	    $c2 = ord (substr ($in, $i+1, 1));
+	    if ($c2 >= 0x80 && $c2 < 0xc0) {
+		# everything looks ok
+		push (@$out, ((($c1 & 0x1f) << 6) +
+		      ($c2 & 0x3f)));
+		$i++; # gobbled an extra byte
+	    }
+
+	} elsif ($c1 < 0xf0 && $i+2 < $len) {
+	    # an encoded character with three bytes
+	    $c2 = ord (substr ($in, $i+1, 1));
+	    $c3 = ord (substr ($in, $i+2, 1));
+	    if ($c2 >= 0x80 && $c2 < 0xc0 &&
+		$c3 >= 0x80 && $c3 < 0xc0) {
+		# everything looks ok
+		push (@$out, ((($c1 & 0xf) << 12) +
+		      (($c2 & 0x3f) << 6) +
+		      ($c3 & 0x3f)));
+
+		$i += 2; # gobbled an extra two bytes
+	    }
+
+	} else {
+	    # error, only decode Unicode characters not full UCS.
+	    # Do nothing.
+	}
+
+	$i++;
+    }
+
+    return $out;
+}
+
+# unicode2ucs2 takes a unicode array and produces a UCS-2
+# unicode string (every two bytes forms a unicode character)
+sub unicode2ucs2 {
+    my ($in) = @_;
+    my $out = "";
+
+    foreach my $num (@$in) {
+	$out .= chr (($num & 0xff00) >> 8);
+	$out .= chr ($num & 0xff);
+    }
+
+    return $out;
+}
+
+# ucs22unicode takes a UCS-2 string and produces a unicode array
+sub ucs22unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my $len = length ($in);
+    while ($i+1 < $len) {
+	push (@$out, ord (substr($in, $i, 1)) << 8 +
+	      ord (substr($in, $i+1, 1)));
+
+	$i ++;
+    }
+
+    return $out;
+}
+
+# takes a reference to a string and returns a reference to a unicode array
+sub convert2unicode {
+    my ($encoding, $textref) = @_;
+
+    if (!defined $encodings::encodings->{$encoding}) {
+	print STDERR "unicode::convert2unicode: ERROR: Unsupported encoding ($encoding)\n";
+	return [];
+    }
+
+    my $encodename = "$encoding-unicode";
+    my $enc_info = $encodings::encodings->{$encoding};
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "mappings",
+				      "to_uc", $enc_info->{'mapfile'});
+    if (!&loadmapencoding ($encodename, $mapfile)) {
+	print STDERR "unicode: ERROR - could not load encoding $encodename\n";
+	return [];
+    }
+    
+    if (defined $enc_info->{'converter'}) {
+	my $converter = $enc_info->{'converter'};
+	return &$converter ($encodename, $textref);
+    }
+
+    if ($unicode::translations{$encodename}->{'count'} == 1) {
+	return &singlebyte2unicode ($encodename, $textref);
+    } else {
+	return &doublebyte2unicode ($encodename, $textref);
+    }
+}
+
+# singlebyte2unicode converts simple 8 bit encodings where characters below
+# 0x80 are normal ascii characters and the rest are decoded using the
+# appropriate mapping files.
+#
+# Examples of encodings that may be converted using singlebyte2unicode are
+# the iso-8859 and windows-125* series.
+sub singlebyte2unicode {
+    my ($encodename, $textref) = @_;
+
+    my @outtext = ();
+    my $len = length($$textref);
+    my ($c);
+    my $i = 0;
+
+    while ($i < $len) {
+	if (($c = ord(substr($$textref, $i, 1))) < 0x80) {
+	    # normal ascii character
+	    push (@outtext, $c);
+	} else {
+	    $c = &transchar ($encodename, $c);
+	    # put a black square if cannot translate
+	    $c = 0x25A1 if $c == 0;
+	    push (@outtext, $c);
+	}
+	$i ++;
+    }
+    return \@outtext;
+}
+
+# doublebyte2unicode converts simple two byte encodings where characters
+# below code point 0x80 are single-byte characters and the rest are
+# double-byte characters.
+#
+# Examples of encodings that may be converted using doublebyte2unicode are
+# CJK encodings like GB encoded Chinese and UHC Korean.
+#
+# Note that no error checking is performed to make sure that the input text
+# is valid for the given encoding.
+#
+# Also, encodings that may contain characters of more than two bytes are
+# not supported (any EUC encoded text may in theory contain 3-byte
+# characters but in practice only one and two byte characters are used).
+sub doublebyte2unicode {
+    my ($encodename, $textref) = @_;    
+    
+    my @outtext = ();
+    my $len = length($$textref);
+    my ($c1, $c2);
+    my $i = 0;
+
+    while ($i < $len) {
+	if (($c1 = ord(substr($$textref, $i, 1))) >= 0x80) {
+	    if ($i+1 < $len) {
+		# double-byte character
+		$c2 = ord(substr($$textref, $i+1, 1));
+		my $c = &transchar ($encodename, ($c1 << 8) | $c2);
+		# put a black square if cannot translate
+		$c = 0x25A1 if $c == 0;
+		push (@outtext, $c);
+		$i += 2;
+		
+	    } else {
+		# error
+		print STDERR "unicode: ERROR missing second half of double-byte character\n";
+		$i++;
+	    }
+	    
+	} else {
+	    # single-byte character
+	    push (@outtext, $c1);
+	    $i++;
+	}
+    }
+    return \@outtext;
+}
+
+# Shift-JIS to unicode
+# We can't use doublebyte2unicode for Shift-JIS because it uses some
+# single-byte characters above code point 0x80 (i.e. half-width katakana
+# characters in the range 0xA1-0xDF)
+sub shiftjis2unicode {
+    my ($encodename, $textref) = @_;
+    
+    my @outtext = ();
+    my $len = length($$textref);
+    my ($c1, $c2);
+    my $i = 0;
+
+    while ($i < $len) {
+	$c1 = ord(substr($$textref, $i, 1));
+
+	if (($c1 >= 0xA1 && $c1 <= 0xDF) || $c1 == 0x5c || $c1 == 0x7E) {
+	    # Single-byte half-width katakana character or
+	    # JIS Roman yen or overline characters
+	    my $c = &transchar ($encodename, $c1);
+	    # - put a black square if cannot translate
+	    $c = 0x25A1 if $c == 0;
+	    push (@outtext, $c);
+	    $i++;
+
+	} elsif ($c1 < 0x80) {
+	    # ASCII
+	    push (@outtext, $c1);
+	    $i ++;
+
+	} elsif ($c1 < 0xEF) {
+	    if ($i+1 < $len) {
+		$c2 = ord(substr($$textref, $i+1, 1));
+		if (($c2 >= 0x40 && $c2 <= 0x7E) || ($c2 >= 0x80 && $c2 <= 0xFC)) {
+		    # Double-byte shift-jis character
+		    my $c = &transchar ($encodename, ($c1 << 8) | $c2);
+		    # put a black square if cannot translate
+		    $c = 0x25A1 if $c == 0;
+		    push (@outtext, $c);
+		} else {
+		    # error
+		    print STDERR "unicode: ERROR Invalid Shift-JIS character\n";
+		}
+		$i += 2;
+	    } else {
+		# error
+		print STDERR "unicode: ERROR missing second half of Shift-JIS character\n";
+		$i ++;
+	    }
+	} else {
+	    # error
+	    print STDERR "unicode: ERROR Invalid Shift-JIS character\n";
+	    $i ++;
+	}
+    }
+    return \@outtext;
+}
+
+sub transchar {
+    my ($encoding, $from) = @_;
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    return 0 unless defined $unicode::translations{$encoding};
+
+    my $block = $unicode::translations{$encoding}->{'map'};
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	return 0;
+    }
+    return $block->[$high]->[$low];
+}
+
+# %translations is of the form:
+#
+# encodings{encodingname-encodingname}->{'map'}->blocktranslation
+# blocktranslation->[[0-255],[256-511], ..., [65280-65535]]
+#
+# Any of the top translation blocks can point to an undefined
+# value. This data structure aims to allow fast translation and 
+# efficient storage.
+%unicode::translations = ();
+
+# @array256 is used for initialisation, there must be
+# a better way...
+# What about this?: @array256 = (0) x 256;
+@unicode::array256 = (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);
+
+# returns 1 if successful, 0 if unsuccessful
+sub loadmapencoding {
+    my ($encoding, $mapfile) = @_;
+    
+    # check to see if the encoding has already been loaded
+    return 1 if (defined $unicode::translations{$encoding});
+
+    if (! -r $mapfile || -d $mapfile) {
+	return 0;
+    }
+    return 0 unless open (MAPFILE, $mapfile);
+    binmode (MAPFILE);
+
+    $unicode::translations{$encoding} = {'map' => [@unicode::array256], 'count' => 0};
+    my $block = $unicode::translations{$encoding};
+
+    my ($in,$i,$j);
+    while (1) {
+	my $ret=read(MAPFILE, $in, 1);
+	if (!defined($ret)) { # error
+	    print STDERR "unicode.pm: error reading mapfile: $!\n";
+	    last;
+	}
+	if ($ret != 1) { last }
+	$i = unpack ("C", $in);
+	$block->{'map'}->[$i] = [@unicode::array256];
+	for ($j=0; $j<256 && read(MAPFILE, $in, 2)==2; $j++) {
+	    my ($n1, $n2) = unpack ("CC", $in);
+	    $block->{'map'}->[$i]->[$j] = ($n1*256) + $n2;
+	}
+	$block->{'count'} ++;
+    }
+
+    close (MAPFILE);
+}
+
+# unicode2singlebyte converts unicode to simple 8 bit encodings where
+# characters below 0x80 are normal ascii characters and the rest are encoded
+# using the appropriate mapping files.
+#
+# Examples of encodings that may be converted using unicode2singlebyte are
+# the iso-8859 and windows-125* series, KOI8-R (Russian), and the Kazakh encoding.
+sub unicode2singlebyte {
+    my ($uniref, $encoding) = @_;
+
+    my $outtext = "";
+    my $encodename = "unicode-$encoding";
+
+    if (!exists $encodings::encodings->{$encoding}) {
+	print STDERR "unicode.pm: ERROR - unsupported encoding "
+	    . "'$encoding' requested\n";
+	return "";
+    }
+
+    my $enc_info = $encodings::encodings->{$encoding};
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "mappings",
+				      "from_uc", $enc_info->{'mapfile'});
+    if (!&loadmapencoding ($encodename, $mapfile)) {
+	print STDERR "unicode: ERROR - could not load encoding $encodename\n";
+	return "";
+    }
+    
+    foreach my $c (@$uniref) {
+	if ($c < 0x80) {
+	    # normal ascii character
+	    $outtext .= chr($c);
+	} else {
+	    # extended ascii character
+	    $c = &transchar ($encodename, $c);
+
+	    # put a question mark if cannot translate
+	    if ($c == 0) {
+		$outtext .= "?";
+	    } else {
+		$outtext .= chr($c);
+	    }
+	}
+    }
+    return $outtext;
+}
+
+
+# this makes sure that the referenced input string is utf8 encoded, and
+# will change/remove bytes that aren't.
+# returns 0 if the text was already utf8, or 1 if text modified to become utf8
+sub ensure_utf8 {
+    my $stringref=shift;
+
+    if (!defined($stringref) || ref($stringref) ne 'SCALAR') {
+	return $stringref;
+    }
+
+    my $value=$$stringref;
+
+    my $non_utf8_found = 0;
+    $value =~ m/^/g; # to set \G
+    while ($value =~ m!\G.*?([\x80-\xff]+)!sg) {
+	my $highbytes=$1;
+	my $highbyteslength=length($highbytes);
+	# make sure this block of high bytes is utf-8
+	$highbytes =~ /^/g; # set pos()
+	my $byte_replaced = 0;
+	while ($highbytes =~
+		m!\G (?: [\xc0-\xdf][\x80-\xbf]   | # 2 byte utf-8
+			[\xe0-\xef][\x80-\xbf]{2} | # 3 byte
+			[\xf0-\xf7][\x80-\xbf]{3} | # 4 byte
+			[\xf8-\xfb][\x80-\xbf]{4} | # 5 byte
+			[\xfc-\xfd][\x80-\xbf]{5} | # 6 byte
+			)*([\x80-\xff])? !xg
+		) {
+	    # this highbyte is "out-of-place" for valid utf-8
+	    my $badbyte=$1;
+	    if (!defined $badbyte) {next} # hit end of string
+	    my $pos=pos($highbytes);
+	    # replace bad byte. assume iso-8859-1 -> utf-8
+	    # ascii2utf8 does "extended ascii"... ie iso-8859-1
+	    my $replacement=&unicode::ascii2utf8(\$badbyte);
+	    substr($highbytes, $pos-1, 1, $replacement);
+	    # update the position to continue searching (for \G)
+	    pos($highbytes) = $pos+length($replacement)-1;
+	    $byte_replaced = 1;
+	}
+	if ($byte_replaced) {
+	    # replace this block of high bytes in the $value
+	    $non_utf8_found = 1;
+	    my $replength=length($highbytes); # we've changed the length
+	    my $textpos=pos($value); # pos at end of last match
+	    # replace bad bytes with good bytes
+	    substr($value, $textpos-$highbyteslength,
+			    $highbyteslength, $highbytes);
+	    # update the position to continue searching (for \G)
+	    pos($value)=$textpos+($replength-$highbyteslength)+1;
+	}
+    }
+
+    $$stringref = $value;
+    return $non_utf8_found;
+}
+
+
+sub substr
+{
+    my ($utf8_string, $offset, $length) = @_;
+
+    my @unicode_string = @{&utf82unicode($utf8_string)};
+    my $unicode_string_length = scalar(@unicode_string);
+
+    my $substr_start = $offset;
+    if ($substr_start >= $unicode_string_length) {
+	return "";
+    }
+
+    my $substr_end = $offset + $length - 1;
+    if ($substr_end >= $unicode_string_length) {
+	$substr_end = $unicode_string_length - 1;
+    }
+
+    my @unicode_substring = @unicode_string[$substr_start..$substr_end];
+    return &unicode2utf8(\@unicode_substring);
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/util.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/util.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/util.pm	(revision 21112)
@@ -0,0 +1,708 @@
+###########################################################################
+#
+# util.pm -- various useful utilities
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package util;
+
+use File::Copy;
+use File::Basename;
+
+
+# removes files (but not directories)
+sub rm {
+    my (@files) = @_;
+    my @filefiles = ();
+
+    # make sure the files we want to delete exist 
+    # and are regular files
+    foreach my $file (@files) {
+	if (!-e $file) {
+	    print STDERR "util::rm $file does not exist\n";
+	} elsif ((!-f $file) && (!-l $file)) {
+	    print STDERR "util::rm $file is not a regular (or symbolic) file\n";
+	} else {
+	    push (@filefiles, $file);
+	}
+    }
+    
+    # remove the files
+    my $numremoved = unlink @filefiles;
+
+    # check to make sure all of them were removed
+    if ($numremoved != scalar(@filefiles)) {
+	print STDERR "util::rm Not all files were removed\n";
+    }
+}
+
+
+
+# recursive removal
+sub filtered_rm_r {
+    my ($files,$file_accept_re,$file_reject_re) = @_;
+
+    my @files_array = (ref $files eq "ARRAY") ? @$files : ($files);
+
+    # recursively remove the files
+    foreach my $file (@files_array) {
+	$file =~ s/[\/\\]+$//; # remove trailing slashes
+	
+	if (!-e $file) {
+	    print STDERR "util::filtered_rm_r $file does not exist\n";
+
+	} elsif ((-d $file) && (!-l $file)) { # don't recurse down symbolic link
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::filtered_rm_r could not open directory $file\n";
+	    } else {
+		my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+		closedir (INDIR);
+				
+		# remove all the files in this directory
+		map {$_="$file/$_";} @filedir;
+		&filtered_rm_r (\@filedir,$file_accept_re,$file_reject_re);
+
+		if (!defined $file_accept_re && !defined $file_reject_re) {
+		    # remove this directory
+		    if (!rmdir $file) {
+			print STDERR "util::filtered_rm_r couldn't remove directory $file\n";
+		    }
+		}
+	    }
+	} else {
+	    next if (defined $file_reject_re && ($file =~ m/$file_reject_re/));
+
+	    if ((!defined $file_accept_re) || ($file =~ m/$file_accept_re/)) {
+		# remove this file	
+		&rm ($file);
+	    }
+	}
+    }
+}
+
+
+# recursive removal
+sub rm_r {
+    my (@files) = @_;
+    
+    # use the more general (but reterospectively written function
+    # filtered_rm_r function()
+
+    filtered_rm_r(\@files,undef,undef); # no accept or reject expressions
+}
+
+
+
+
+# moves a file or a group of files
+sub mv {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::mv no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::mv if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+
+    # move the files
+    foreach my $file (@srcfiles) {
+	my $tempdest = $dest;
+	if (-d $tempdest) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+	if (!-e $file) {
+	    print STDERR "util::mv $file does not exist\n";
+	} else {
+	    rename ($file, $tempdest);
+	}
+    }
+}
+
+
+# copies a file or a group of files
+sub cp {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::cp if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+
+    # copy the files
+    foreach my $file (@srcfiles) {
+	my $tempdest = $dest;
+	if (-d $tempdest) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+	if (!-e $file) {
+	    print STDERR "util::cp $file does not exist\n";
+	} elsif (!-f $file) {
+	    print STDERR "util::cp $file is not a plain file\n";
+	} else {
+	    &File::Copy::copy ($file, $tempdest);
+	}
+    }
+}
+
+
+
+# recursively copies a file or group of files
+# syntax: cp_r (sourcefiles, destination directory)
+# destination must be a directory - to copy one file to
+# another use cp instead
+sub cp_r {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp_r no destination directory given\n";
+	return;
+    } elsif (-f $dest) {
+	print STDERR "util::cp_r destination must be a directory\n";
+	return;
+    }
+    
+    # create destination directory if it doesn't exist already
+    if (! -d $dest) {
+	my $store_umask = umask(0002);
+	mkdir ($dest, 0777);
+	umask($store_umask);
+    } 
+
+    # copy the files
+    foreach my $file (@srcfiles) {
+
+	if (!-e $file) {
+	    print STDERR "util::cp_r $file does not exist\n";
+
+	} elsif (-d $file) {
+	    # make the new directory
+	    my ($filename) = $file =~ /([^\\\/]*)$/;
+	    $dest = &util::filename_cat ($dest, $filename);
+	    my $store_umask = umask(0002);
+	    mkdir ($dest, 0777);
+	    umask($store_umask);
+
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::cp_r could not open directory $file\n";
+	    } else {
+		my @filedir = readdir (INDIR);
+		closedir (INDIR);
+		foreach my $f (@filedir) {
+		    next if $f =~ /^\.\.?$/;
+		    # copy all the files in this directory
+		    my $ff = &util::filename_cat ($file, $f); 
+		    &cp_r ($ff, $dest);
+		}
+	    }
+
+	} else {
+	    &cp($file, $dest);
+	}
+    }
+}
+
+
+sub mk_dir {
+    my ($dir) = @_;
+
+    my $store_umask = umask(0002);
+    my $mkdir_ok = mkdir ($dir, 0777);
+    umask($store_umask);
+    
+    if (!$mkdir_ok) 
+    {
+	print STDERR "util::mk_dir could not create directory $dir\n";
+	return;
+    }
+}
+
+# in case anyone cares - I did some testing (using perls Benchmark module)
+# on this subroutine against File::Path::mkpath (). mk_all_dir() is apparently
+# slightly faster (surprisingly) - Stefan.
+sub mk_all_dir {
+    my ($dir) = @_;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $dir=~s/[\\\/]+/\//g; 
+    $dir=~s/[\\\/]+$//;
+
+    # make sure the cache directory exists
+    my $dirsofar = "";
+    my $first = 1;
+    foreach my $dirname (split ("/", $dir)) {
+	$dirsofar .= "/" unless $first;
+	$first = 0;
+
+	$dirsofar .= $dirname;
+
+	next if $dirname =~ /^(|[a-z]:)$/i;
+	if (!-e $dirsofar)
+	    {
+		my $store_umask = umask(0002);
+		my $mkdir_ok = mkdir ($dirsofar, 0777);
+		umask($store_umask);
+		if (!$mkdir_ok)
+		{
+		    print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
+		    return;
+		}
+	    }
+    }
+}
+
+# make hard link to file if supported by OS, otherwise copy the file
+sub hard_link {
+    my ($src, $dest) = @_;
+
+    # remove trailing slashes from source and destination files
+    $src =~ s/[\\\/]+$//;
+    $dest =~ s/[\\\/]+$//;
+
+    # a few sanity checks
+    if (-e $dest) {
+	# destination file already exists
+	return;
+    }
+    elsif (!-e $src) {
+	print STDERR "util::hard_link source file $src does not exist\n";
+	return 1;
+    }
+    elsif (-d $src) {
+	print STDERR "util::hard_link source $src is a directory\n";
+	return 1;
+    }
+
+    my $dest_dir = &File::Basename::dirname($dest);
+    mk_all_dir($dest_dir) if (!-e $dest_dir);
+
+    # link not supported on windows 9x
+    if (($ENV{'GSDLOS'} =~ /^windows$/i) && (Win32::FsType() !~ /^ntfs$/i)) {
+	&File::Copy::copy ($src, $dest);
+
+    } elsif (!link($src, $dest)) {
+	print STDERR "util::hard_link: unable to create hard link. ";
+	print STDERR " Attempting to copy file: $src -> $dest\n";
+	&File::Copy::copy ($src, $dest);
+    }
+    return 0;
+}
+
+# make soft link to file if supported by OS, otherwise copy file
+sub soft_link {
+    my ($src, $dest) = @_;
+
+    # remove trailing slashes from source and destination files
+    $src =~ s/[\\\/]+$//;
+    $dest =~ s/[\\\/]+$//;
+
+    # a few sanity checks
+    if (!-e $src) {
+	print STDERR "util::soft_link source file $src does not exist\n";
+	return 0;
+    }
+
+    my $dest_dir = &File::Basename::dirname($dest);
+    mk_all_dir($dest_dir) if (!-e $dest_dir);
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	# symlink not supported on windows
+	&File::Copy::copy ($src, $dest);
+
+    } elsif (!eval {symlink($src, $dest)}) {
+	print STDERR "util::soft_link: unable to create soft link.\n";
+	return 0;
+    }
+
+    return 1;
+}
+
+
+
+
+# updates a copy of a directory in some other part of the filesystem
+# verbosity settings are: 0=low, 1=normal, 2=high
+# both $fromdir and $todir should be absolute paths
+sub cachedir {
+    my ($fromdir, $todir, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $fromdir=~s/[\\\/]+/\//g; 
+    $fromdir=~s/[\\\/]+$//;
+    $todir=~s/[\\\/]+/\//g; 
+    $todir=~s/[\\\/]+$//;
+
+    &mk_all_dir ($todir);
+
+    # get the directories in ascending order
+    if (!opendir (FROMDIR, $fromdir)) {
+	print STDERR "util::cachedir could not read directory $fromdir\n";
+	return;
+    }
+    my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
+    closedir (FROMDIR);
+
+    if (!opendir (TODIR, $todir)) {
+	print STDERR "util::cacedir could not read directory $todir\n";
+	return;
+    }
+    my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
+    closedir (TODIR);
+
+    my $fromi = 0;
+    my $toi = 0;
+		    
+    while ($fromi < scalar(@fromdir) || $toi < scalar(@todir)) {
+#	print "fromi: $fromi toi: $toi\n";
+
+	# see if we should delete a file/directory
+	# this should happen if the file/directory
+	# is not in the from list or if its a different
+	# size, or has an older timestamp
+	if ($toi < scalar(@todir)) {
+	    if (($fromi >= scalar(@fromdir)) ||
+		($todir[$toi] lt $fromdir[$fromi] || 
+		 ($todir[$toi] eq $fromdir[$fromi] && 
+		  &differentfiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]",
+				  $verbosity)))) {
+
+		# the files are different
+		&rm_r("$todir/$todir[$toi]");
+		splice(@todir, $toi, 1); # $toi stays the same
+
+	    } elsif ($todir[$toi] eq $fromdir[$fromi]) {
+		# the files are the same
+		# if it is a directory, check its contents
+		if (-d "$todir/$todir[$toi]") {
+		    &cachedir ("$fromdir/$fromdir[$fromi]",
+			       "$todir/$todir[$toi]", $verbosity);
+		}
+
+		$toi++;
+		$fromi++;
+		next;
+	    }
+	}
+  
+	# see if we should insert a file/directory
+	# we should insert a file/directory if there
+	# is no tofiles left or if the tofile does not exist
+	if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) || 
+					  $todir[$toi] gt $fromdir[$fromi])) {
+	    &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
+	    splice (@todir, $toi, 0, $fromdir[$fromi]);
+
+	    $toi++;
+	    $fromi++;
+	}
+    }
+}
+
+# this function returns -1 if either file is not found
+# assumes that $file1 and $file2 are absolute file names or
+# in the current directory
+# $file2 is allowed to be newer than $file1
+sub differentfiles {
+    my ($file1, $file2, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    $file1 =~ s/\/+$//;
+    $file2 =~ s/\/+$//;
+    
+    my ($file1name) = $file1 =~ /\/([^\/]*)$/;
+    my ($file2name) = $file2 =~ /\/([^\/]*)$/;
+
+    return -1 unless (-e $file1 && -e $file2);
+    if ($file1name ne $file2name) {
+	print STDERR "filenames are not the same\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    my @file1stat = stat ($file1);
+    my @file2stat = stat ($file2);
+
+    if (-d $file1) {
+	if (! -d $file2) {
+	    print STDERR "one file is a directory\n" if ($verbosity >= 2);
+	    return 1;
+	}
+	return 0;
+    }
+
+    # both must be regular files
+    unless (-f $file1 && -f $file2) {
+	print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the size of the files must be the same
+    if ($file1stat[7] != $file2stat[7]) {
+	print STDERR "different sized files\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the second file cannot be older than the first
+    if ($file1stat[9] > $file2stat[9]) {
+	print STDERR "file is older\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    return 0;
+}
+
+
+sub get_tmp_filename {
+    my $tmpdir = filename_cat($ENV{'GSDLHOME'}, "tmp");
+    &mk_all_dir ($tmpdir) unless -e $tmpdir;
+
+    my $count = 1000;
+    my $rand = int(rand $count);
+    while (-e &filename_cat($tmpdir, "F$rand")) {
+	$rand = int(rand $count);
+	$count++;
+    }
+
+    return filename_cat($tmpdir, "F$rand");
+}
+
+
+sub filename_cat {
+    my $first_file = shift(@_);
+    my (@filenames) = @_;
+
+    # Check for empty first filename
+    if ($first_file =~ /\S/) {
+	unshift(@filenames, $first_file);
+    }
+
+    my $filename = join("/", @filenames);
+
+    # remove duplicate slashes and remove the last slash
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$filename =~ s/[\\\/]+/\\/g;
+    } else {
+	$filename =~ s/[\/]+/\//g; 
+	# DB: want a filename abc\de.html to remain like this
+    }
+    $filename =~ s/[\\\/]$//;
+
+    return $filename;
+}
+
+
+sub envvar_prepend {
+    my ($var,$val) = @_;
+
+    my $current_val = $ENV{$var};
+    
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$ENV{$var} .= "$val;$current_val";
+    }
+    else {
+	$ENV{$var} .= "$val:$current_val";
+    }
+}
+
+sub envvar_append {
+    my ($var,$val) = @_;
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$ENV{$var} .= ";$val";
+    }
+    else {
+	$ENV{$var} .= ":$val";
+    }
+}
+
+
+# returns the path of a file without the filename -- ie. the directory the file is in
+sub filename_head {
+    my $filename = shift(@_);
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$filename =~ s/[^\\\\]*$//;
+    }
+    else {
+	$filename =~ s/[^\\\/]*$//;
+    }
+
+    return $filename;
+}
+
+
+# returns 1 if filename1 and filename2 point to the same
+# file or directory
+sub filenames_equal {
+    my ($filename1, $filename2) = @_;
+
+    # use filename_cat to clean up trailing slashes and 
+    # multiple slashes
+    $filename1 = filename_cat ($filename1);
+    $filename2 = filename_cat ($filename2);
+
+    # filenames not case sensitive on windows
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$filename1 =~ tr/[A-Z]/[a-z]/;
+	$filename2 =~ tr/[A-Z]/[a-z]/;
+    }
+    return 1 if $filename1 eq $filename2;
+    return 0;
+}
+
+sub filename_within_collection
+{
+    my ($filename) = @_;
+
+    my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
+    
+    if (defined $collect_dir) {
+	my $dirsep = &util::get_dirsep();
+	if ($collect_dir !~ m/$dirsep$/) {
+	    $collect_dir .= $dirsep;
+	}
+	
+	$collect_dir =~ s/\\/\\\\/g; # escape DOS style file separator
+	
+	if ($filename =~ /^$collect_dir(.*)$/) {
+	    $filename = $1;
+	}
+    }
+    
+    return $filename;
+}
+
+
+sub get_dirsep {
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	return "\\";
+    } else {
+	return "\/";
+    }
+}
+
+sub get_os_dirsep {
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	return "\\\\";
+    } else {
+	return "\\\/";
+    }
+}
+
+sub get_re_dirsep {
+
+    return "\\\\|\\\/";
+}
+
+
+# if this is running on windows we want binaries to end in
+# .exe, otherwise they don't have to end in any extension
+sub get_os_exe {
+    return ".exe" if $ENV{'GSDLOS'} =~ /^windows$/i;
+    return "";
+}
+
+
+# test to see whether this is a big or little endian machine
+sub is_little_endian {
+    return (ord(substr(pack("s",1), 0, 1)) == 1);
+}
+
+
+# will return the collection name if successful, "" otherwise
+sub use_collection {
+    my ($collection, $collectdir) = @_;
+
+    if (!defined $collectdir || $collectdir eq "") {
+	$collectdir = &filename_cat ($ENV{'GSDLHOME'}, "collect");
+    }
+
+    # get and check the collection
+    if (!defined($collection) || $collection eq "") {
+	if (defined $ENV{'GSDLCOLLECTION'}) {
+	    $collection = $ENV{'GSDLCOLLECTION'};
+	} else {
+	    print STDOUT "No collection specified\n";
+	    return "";
+	}
+    }
+    
+    if ($collection eq "modelcol") {
+	print STDOUT "You can't use modelcol.\n";
+	return "";
+    }
+
+    # make sure the environment variables GSDLCOLLECTION and GSDLCOLLECTDIR
+    # are defined
+    $ENV{'GSDLCOLLECTION'} = $collection unless defined $ENV{'GSDLCOLLECTION'};
+    $ENV{'GSDLCOLLECTDIR'} = &filename_cat ($collectdir, $collection);
+
+    # make sure this collection exists
+    if (!-e $ENV{'GSDLCOLLECTDIR'}) {
+	print STDOUT "Invalid collection ($collection).\n";
+	return "";
+    }
+
+    # everything is ready to go
+    return $collection;
+}
+
+sub hyperlink_text
+{
+    my ($text) = @_;
+    
+    $text =~ s/(http:\/\/[^\s]+)/<a href=\"$1\">$1<\/a>/mg;
+    $text =~ s/(^|\s+)(www\.(\w|\.)+)/<a href=\"http:\/\/$2\">$2<\/a>/mg;
+
+    return $text;
+}
+
+
+1;
Index: /main/tags/3.01/gsdl/perllib/webpageutil.pm
===================================================================
--- /main/tags/3.01/gsdl/perllib/webpageutil.pm	(revision 21112)
+++ /main/tags/3.01/gsdl/perllib/webpageutil.pm	(revision 21112)
@@ -0,0 +1,24 @@
+package webpageutil;
+
+use ghtml;
+
+sub error_location
+{
+    my ($args,$text) = @_;
+
+    &ghtml::urlsafe($text);
+    my $mess_url = "$args->{'httpbuild'}&bca=mess&head=_headerror_";
+    print "Location: $mess_url&mess=$text\n\n";
+}
+
+sub status_location
+{
+    my ($args,$text,$tmpname,$bc1finished) = @_;
+
+    &ghtml::urlsafe($text);    
+    my $mess_url = "$args->{'httpbuild'}&bca=buildstatus";
+    $mess_url .= "&bc1tmpfilename=$tmpname&bc1finished=$bc1finished";
+    print "Location: $mess_url&mess=$text\n\n";
+}
+
+1;
Index: /main/tags/3.01/gsdl/setup.bash
===================================================================
--- /main/tags/3.01/gsdl/setup.bash	(revision 21112)
+++ /main/tags/3.01/gsdl/setup.bash	(revision 21112)
@@ -0,0 +1,183 @@
+# if this file is executed, /bin/sh is used, as we don't start with #!
+# this should work under ash, bash, zsh, ksh, sh style shells.
+gsdllang=""
+# encoding inputs and outputs
+gsdltoenc=""
+gsdlfromenc=""
+
+# see if the shell has any language environment variables set
+# see locale(7) manpage for this ordering.
+if test ! -z "$LC_ALL" ; then
+  gsdllang="$LC_ALL"
+elif test ! -z "$LANG"; then
+  gsdllang="$LANG"
+fi
+
+
+# note... all our output strings have the charset hard-coded, but
+# people may be using a different encoding in their terminal. LANG
+# strings look like "en_NZ.UTF-8".
+
+# Determine the requested output encoding
+case $gsdllang in
+  *.*)
+    gsdltoenc=`echo $gsdllang | sed 's/.*\.//'`
+    ;;
+esac
+
+# Our french and spanish strings are currently in iso-8859-1 encoding.
+case $gsdllang in
+  fr*|FR*)
+    gsdllang=fr
+    gsdlfromenc="iso-8859-1"
+  ;;
+  es*|ES*)
+    gsdllang=es
+    gsdlfromenc="iso-8859-1"
+  ;;
+  ru*|RU*)
+    gsdllang=ru
+    gsdlfromenc="koi8r"
+  ;;
+  *) # default
+    gsdllang=en
+    gsdlfromenc="iso-8859-1"
+  ;;
+esac  
+
+# "iconv" is the program for converting text between encodings.
+gsdliconv=`which iconv 2>/dev/null`
+if test $? -ne 0 || test ! -x "$gsdliconv" || test -z "$gsdlfromenc" || test -z "$gsdltoenc"; then
+# we can't convert encodings from some reason
+  gsdliconv="cat"
+else
+# add the encodings
+  gsdliconv="$gsdliconv -f $gsdlfromenc -t $gsdltoenc"
+fi
+
+# make sure we are sourced, and not run
+
+if test "$0" != "`echo $0 | sed s/setup\.bash//`" ; then
+# if $0 contains "setup.bash" we've been run... $0 is shellname if sourced.
+# One exception is zsh has an option to set it temporarily to the script name
+  if test -z "$ZSH_NAME" ; then
+  # we aren't using zsh
+  gsdl_not_sourced=true
+  fi
+fi
+
+if test -n "$gsdl_not_sourced" ; then
+  case "$gsdllang" in
+ "es")
+eval $gsdliconv <<EOF
+      Error: Asegúrese de compilar este guión, no de ejecutarlo. P. ej.:
+         $ source setup.bash
+      o
+         $ . ./setup.bash
+      no
+         $ ./setup.bash
+EOF
+  ;;
+  "fr")
+eval $gsdliconv <<EOF
+      Erreur: Assurez-vous de "sourcer" le script, plutôt que de l'exécuter. Ex:
+         $ source setup.bash
+      ou
+         $ . ./setup.bash
+      pas
+         $ ./setup.bash
+EOF
+  ;;
+  "ru")
+eval $gsdliconv <<EOF
+      ïÛÉÂËÁ: õÄÏÓÔÏ×ÅÒØÔÅÓØ × ÉÓÔÏÞÎÉËÅ ÜÔÏÇÏ ÓËÒÉÐÔÁ. îÅ ÉÓÐÏÌÎÑÊÔÅ ÅÇÏ.
+      îÁÐÒÉÍÅÒ:
+         $ source setup.bash
+      ÉÌÉ
+         $ . ./setup.bash
+      ÎÅÔ
+         $ ./setup.bash
+EOF
+  ;;
+  *)
+eval $gsdliconv <<EOF
+	Error: Make sure you source this script, not execute it. Eg:
+		$ source setup.bash
+	or
+		$ . ./setup.bash
+	not
+		$ ./setup.bash
+EOF
+  ;;
+  esac
+elif test -n "$GSDLHOME" ; then
+  case "$gsdllang" in
+  "es")
+    echo '¡Su ambiente ya está listo para Greenstone!' | eval $gsdliconv
+  ;;
+  "fr")
+    echo 'Votre environnement est déjà préparé pour Greenstone!' | eval $gsdliconv
+ ;;
+  "ru")
+    echo '÷ÁÛÅ ÏËÒÕÖÅÎÉÅ ÕÖÅ ÎÁÓÔÒÏÅÎÏ ÄÌÑ Greenstone!' | eval $gsdliconv
+  ;;
+  *)
+    echo 'Your environment is already set up for Greenstone!'
+  ;;
+  esac
+elif test ! -f setup.bash ; then
+  case "$gsdllang" in
+    "es")
+eval $gsdliconv <<EOF
+Usted debe compilar el guión desde el interior del directorio de inicio
+de Greenstone.
+EOF
+  ;;
+  "fr")
+echo 'Vous devez trouver la source du script dans le répertoire de base de Greenstone' | eval $gsdliconv
+  ;;
+  "ru")
+eval $gsdliconv <<EOF
+÷ÁÍ ÎÅÏÂÈÏÄÉÍ ÉÓÔÏÞÎÉË ÓËÒÉÐÔÁ ÉÚ ÂÁÚÏ×ÏÊ ÄÉÒÅËÔÏÒÉÉ Greenstone
+EOF
+  ;;
+  *)
+    echo 'You must source the script from within the Greenstone home directory'
+  ;;
+  esac
+else
+  GSDLHOME=`pwd`
+  export GSDLHOME
+  GSDLOS=`uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+  # check for running bash under cygwin
+  if test "`echo $GSDLOS | sed 's/cygwin//'`" != "$GSDLOS" ;
+  then
+    GSDLOS=windows
+  fi
+  export GSDLOS
+
+  PATH=$PATH:$GSDLHOME/bin/script:$GSDLHOME/bin/$GSDLOS
+  export PATH
+  MANPATH=$MANPATH:$GSDLHOME/packages/mg/man
+  export MANPATH
+  case "$gsdllang" in
+    "es")
+      echo 'Su ambiente ha sido configurado para correr los programas Greenstone.' | eval $gsdliconv
+    ;;
+    "fr")
+      echo 'Votre environnement a été configuére avec succès pour exécuter Greenstone' | eval $gsdliconv
+    ;;
+    "ru")
+eval $gsdliconv <<EOF
+÷ÁÛÅ ÏËÒÕÖÅÎÉÅ ÂÙÌÏ ÕÓÐÅÛÎÏ ÎÁÓÔÒÏÅÎÏ, ÞÔÏÂÙ ÕÓÔÁÎÏ×ÉÔØ Greenstone
+EOF
+    ;;
+    *)
+      echo 'Your environment has successfully been set up to run Greenstone'
+    ;;
+  esac
+fi
+unset gsdl_not_sourced
+unset gsdliconv
+unset gsdlfromenc
+unset gsdltoenc
Index: /main/tags/3.01/gsdl/src/db2txt/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/db2txt/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/db2txt/Makefile.in	(revision 21112)
@@ -0,0 +1,87 @@
+###########################################################################
+#
+# Makefile -- Makefile for db2txt
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CC = @CC@
+CPP = @CXX@
+CCFLAGS = -Wall @CFLAGS@ 
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib @GDBM_INCLUDE@
+LDFLAGS = @LDFLAGS@
+DPLIBS = 
+LIBS = @GDBM_LIBPATH@ -lgdbm @LIBS@
+GSDLOS = @gsdlos@
+INSTALL = @INSTALL@
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+CPPCOMPILE = $(CPP) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+CPPLINK = $(CPP) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CPPCOMPILE) $< 
+ 
+
+HEADERS = 
+
+SOURCES = db2txt.cpp
+
+OBJECTS = db2txt.o
+
+EXEC    = db2txt
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	$(INSTALL) -d ../../bin/$(GSDLOS)
+	$(INSTALL) -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+distclean:
+	rm -f $(OBJECTS) $(EXEC) Makefile
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+DB2TXT_OBJS = db2txt.o
+
+db2txt: $(DB2TXT_OBJS) $(DPLIBS)
+	$(CPPLINK) $(DB2TXT_OBJS) $(LIBS) 
Index: /main/tags/3.01/gsdl/src/db2txt/db2txt.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/db2txt/db2txt.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/db2txt/db2txt.cpp	(revision 21112)
@@ -0,0 +1,99 @@
+/**********************************************************************
+ *
+ * db2txt.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+#include "gsdlconf.h"
+#include <stdlib.h>
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#include <iostream.h>
+#else
+#include <iostream>
+#endif
+
+// use the standard namespace
+#if !defined (GSDL_NAMESPACE_BROKEN)
+#if defined(GSDL_USE_OBJECTSPACE)
+using namespace ospace::std;
+#else
+using namespace std;
+#endif
+#endif
+
+void print_usage (char *program_name) {
+  cerr << "usage: " << program_name << " database-name\n\n";
+}
+
+
+int main (int argc, char *argv[]) {
+  int block_size = 0, i;
+  GDBM_FILE dbf;
+  datum key, value;
+      
+  // sanity check
+  if (argc != 2) {
+    print_usage (argv[0]);
+    exit (0);
+  }
+  
+  // open the database
+#ifdef __WIN32__
+  dbf = gdbm_open (argv[1], block_size, GDBM_READER, 00664, NULL, 0);
+#else
+  dbf = gdbm_open  (argv[1], block_size, GDBM_READER, 00664, NULL);
+#endif
+  if (dbf == NULL) {
+    cerr << "couldn't create " << argv[1] << "\n";
+    exit (0);
+  }
+
+  key = gdbm_firstkey (dbf);
+  while (key.dptr != NULL) {
+    cout << "[";
+    for (i = 0; i < key.dsize; ++i)
+      cout << key.dptr[i];
+    cout << "]\n";
+    value = gdbm_fetch (dbf, key);
+    for (i = 0; i < value.dsize; ++i)
+      cout << value.dptr[i];
+    cout << "\n----------------------------------------------------------------------\n";
+    key = gdbm_nextkey (dbf, key);
+    delete value.dptr;
+  }
+  
+  gdbm_close (dbf);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/db2txt/win32.mak
===================================================================
--- /main/tags/3.01/gsdl/src/db2txt/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/src/db2txt/win32.mak	(revision 21112)
@@ -0,0 +1,78 @@
+###########################################################################
+#
+# win32 makefile -- db2txt
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..
+USE_STLPORT = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+LDFLAGS =
+
+DEFS = -D__WIN32__ -DHAVE_CONFIG_H
+INCLUDES = -I"$(GSDLHOME)\src\db2txt" -I"$(GSDLHOME)\lib" -I"$(GSDLHOME)\packages\mg\lib" \
+	   -I"$(GSDLHOME)\packages\windows\gdbm\gdbm"
+!IF $(USE_STLPORT)
+DEFS=$(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+
+LIBS = "$(GSDLHOME)\packages\mg\lib\unitool.obj" "$(GSDLHOME)\lib\gsdllib.lib" \
+       "$(GSDLHOME)\packages\windows\gdbm\gdbm\gdbm.lib"
+
+COMPILE = $(CC) -c -GX $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .cpp .obj
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+e = .exe
+
+HEADERS = 
+
+SOURCES = db2txt.cpp
+
+OBJECTS = db2txt$o
+
+EXEC = db2txt$e
+
+all : $(EXEC)
+
+install: $(EXEC)
+	if NOT EXIST "$(GSDLHOME)\bin\windows" mkdir "$(GSDLHOME)\bin\windows"
+	copy $(EXEC) "$(GSDLHOME)\bin\windows"
+
+clean:
+	del *$o
+	del $(EXEC)
+
+db2txt$e: $(OBJECTS) $(LIBS)
+	  $(LINK) $(OBJECTS) $(LIBS)
Index: /main/tags/3.01/gsdl/src/hashfile/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/hashfile/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/hashfile/Makefile.in	(revision 21112)
@@ -0,0 +1,83 @@
+###########################################################################
+#
+# Makefile -- Makefile for hashfile
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CXX = @CXX@
+CXXFLAGS = @CXXFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/gsdllib.a
+GSDLOS = @gsdlos@
+INSTALL = @INSTALL@
+ 
+COMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+LINK = $(CXX) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+ 
+
+HEADERS = 
+
+SOURCES = hashfile.cpp
+
+OBJECTS = hashfile.o
+
+EXEC    = hashfile
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	$(INSTALL) -d ../../bin/$(GSDLOS)
+	$(INSTALL) -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+distclean:
+	rm -f $(OBJECTS) $(EXEC) Makefile
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HASHFILE_OBJS = hashfile.o
+
+hashfile: $(HASHFILE_OBJS) $(LIBS)
+	$(LINK) $(HASHFILE_OBJS) $(LIBS) 
Index: /main/tags/3.01/gsdl/src/hashfile/hashfile.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/hashfile/hashfile.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/hashfile/hashfile.cpp	(revision 21112)
@@ -0,0 +1,263 @@
+/**********************************************************************
+ *
+ * hashfile.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include <stdio.h>
+
+#define MAXNUMLEN  100   // bytes
+
+
+// The number class is used to store very long unsigned integers.
+// The least significant of each number is stored first. Numbers can 
+// be no longer than MAXNUMLEN bytes long. A number with length=0
+// is considered to be equal to 0. There should be no extra zeros
+// at the front of the number.
+
+struct number {
+  unsigned int len;
+  unsigned char num[MAXNUMLEN];
+
+  number () {len = 0;}
+};
+
+
+// evaluates a = a + b
+// a and b can be the same number
+void my_inc (number &a, number &b) {
+  unsigned int carry = 0;
+  unsigned int num = 0;
+  unsigned int i = 0;
+
+  while ((carry || (i < a.len) || (i < b.len)) && (i < MAXNUMLEN)) {
+    num = carry;
+    if (i < a.len) num += a.num[i];
+    if (i < b.len) num += b.num[i];
+
+    if (num >= 256) {
+      num -= 256; // assumes that each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+    a.num[i] = (unsigned char)num;
+
+    ++i;
+  }
+
+  a.len = i;
+}
+
+
+// evaluates a = a - b if a >= b
+// returns true if the subtraction was possible
+int my_ifpos_dec (number &a, number &b) {
+  int carry = 0;
+  int num = 0;
+  int i = 0;
+  unsigned int len = 0; // the last non-zero digit
+
+  // make sure a >= b
+  if (b.len > a.len) return 0;
+  if (b.len == a.len) {
+    i = a.len - 1;
+    while (i >= 0) {
+      if (a.num[i] > b.num[i]) break;
+      if (a.num[i] < b.num[i]) return 0;
+      --i;
+    }
+  }
+
+  unsigned int j = 0;
+  len = 0;
+  while ((j < a.len) || (j < b.len)) {
+    num = -carry;
+    if (j < a.len) num += a.num[j];
+    if (j < b.len) num -= b.num[j];
+
+    if (num < 0) {
+      num += 256; // assumes each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+
+    a.num[j] = (unsigned char) num;
+    
+    ++j;
+    if (num != 0) len = j;
+  }
+
+  a.len = len;
+
+  if (carry) {
+    // panic, this should NOT happen
+    fprintf (stderr, "panic: error in my_ifpos_dec\n");
+  }
+
+  return 1;
+}
+
+
+char *my_convert_num (number &a) {
+  char *result = new char[(a.len*2)+1];
+  int i,len;
+  char convert[16] = {'0','1','2','3','4','5','6','7',
+		      '8','9','a','b','c','d','e','f'};
+
+  // zero might be a special case
+  if (a.len == 0) {
+    result[0] = '0';
+    result[1] = '\0';
+    return result;
+  }
+
+  for (i=a.len-1, len=0; i >= 0; --i) {
+    result[len++] = convert[a.num[i]/16];
+    result[len++] = convert[a.num[i]%16];
+  }
+  result[len] = '\0';
+
+  return result;
+}
+
+
+
+// create a hash string from the contents of a file.
+//
+// The file is treated as a large base 256 number (each char is a digit),
+// and the hash value is the remainder when this number is divided by a
+// very large prime.
+//
+// PROBLEM: This is a flawed hash function because the rightmost (highest)
+// value in remainder is more likely to be a "1" than it is to be any other
+// number.  
+//
+// EVIDENCE: About 50% of the files in any GSDL directory have a hash code
+// that starts with 01* which implies (based on the my_convert_num function)
+// that the rightmost "digit" of the remainder = z such that (z % 16 = 0)
+// and (z / 16 = 1) which means (z = 1).
+//
+// MAJOR REASON: suppose our prime number was 19.  Then, if we have a 
+// reasonably random distribution of numbers for which we are going to 
+// calculate (N % 19), we expect to get a roughly uniform distribution
+// of remainders where the possible values are 0, 1, 2, 3... 18.  The
+// problem is that 9 out of the 19 possible values (10,11... 18) start
+// with the digit 1.  Thus our hascode will start with "01".
+//
+// ANOTHER PROBLEM: Characters in the file are read one at a time; after each 
+// one is read it is prepended to the remainder, then the remainder is
+// recalculated on the string thus far seen.  I am sure the math here 
+// is wrong - if I try calculating (111 mod 7) by the same algorithm, it
+// simply does nor work.
+//
+// ANOTHER POSSIBLE PROLEM: Each character from the file is read into 
+// remainder at the most significant end, and when that character
+// is a zero you get a number like "01" which would be considered 
+// larger than a number like "8" because it is longer (two  digits
+// instead of 1).  
+//
+// These comments added by Gordon Paynter (gwp@cs.waikato.ac.nz) in
+// June 2000.  I didn't write any code, however.
+
+char *hashfile (char *filename) {
+  FILE *infile = (FILE *)NULL;
+  int i;
+
+  number primepow[8];
+  number pow;
+  number remainder;
+  unsigned char c;
+
+  // our prime number (618970019642690137449562111)
+  pow.num[0] = 255;  pow.num[1] = 255;
+  pow.num[2] = 255;  pow.num[3] = 255;
+  pow.num[4] = 255;  pow.num[5] = 255;
+  pow.num[6] = 255;  pow.num[7] = 255;
+  pow.num[8] = 255;  pow.num[9] = 255;
+  pow.num[10] = 255; pow.num[11] = 1;
+  pow.len = 12;
+
+  // calculate 8 multiples of the prime number.
+  // These are used to find the remainder using only subtraction operations
+  for (i=0; i<8; ++i) {
+    primepow[i] = pow;
+    my_inc (pow, pow);
+  }
+
+  // The "remainder" after division by the prime.  Our result.
+  remainder.len = 0;
+
+  // open the file
+  infile = fopen (filename, "rb");
+  if (infile == NULL) {
+    return (char *)NULL;
+  }
+  c = (unsigned char)fgetc(infile);
+  
+  while (!feof (infile)) {
+
+    // make sure the remainder has not grown too large
+    if (remainder.len == MAXNUMLEN-1) {
+      fprintf (stderr, "ERROR - number overflow\n");
+      return (char *)NULL;
+    }
+
+    // remainder = remainder * 256 + c
+    for (i=remainder.len; i>0; --i) {
+      remainder.num[i] = remainder.num[i-1];
+    }
+    remainder.num[0] = c;
+    if (remainder.len > 0 || c != 0) remainder.len = remainder.len+1;
+    
+    // remainder = (remainder % large-prime-number)
+    for (i=7; i>=0; --i) {
+      my_ifpos_dec (remainder, primepow[i]);
+    }
+
+    // read a new character from the file
+    c = (unsigned char)fgetc(infile);
+  }
+
+  fclose (infile);
+
+  return my_convert_num (remainder);
+}
+
+
+int main (int argc, char *argv[]) {
+  int i;
+  char *result;
+
+  for (i = 1; i < argc; ++i) {
+    result = hashfile (argv[i]);
+    if (result == NULL) {
+      printf ("ERROR - could not process %s\n", argv[i]);
+    } else {
+      printf ("%s: %s\n", argv[i], result);
+      delete [] result;
+    }
+  }
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/hashfile/win32.mak
===================================================================
--- /main/tags/3.01/gsdl/src/hashfile/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/src/hashfile/win32.mak	(revision 21112)
@@ -0,0 +1,69 @@
+###########################################################################
+#
+# win32 makefile -- hashfile
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+GSDLHOME = ..\..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+DEFS = -DHAVE_CONFIG_H -D__WIN32__
+INCLUDES = -I"$(GSDLHOME)\src\hashfile"
+LIBS =
+
+COMPILE = $(CC) -c -GX $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .cpp .obj
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+e = .exe
+ 
+HEADERS = 
+
+SOURCES = hashfile.cpp
+
+OBJECTS = hashfile$o
+
+EXEC = hashfile$e
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	if NOT EXIST "$(GSDLHOME)\bin\windows" mkdir "$(GSDLHOME)\bin\windows"
+	copy $(EXEC) "$(GSDLHOME)\bin\windows"
+
+clean:
+	del *$o
+	del $(EXEC)
+
+HASHFILE_OBJS = hashfile$o
+
+hashfile$e: $(HASHFILE_OBJS) $(LIBS)
+	    $(LINK) $(HASHFILE_OBJS) $(LIBS)
Index: /main/tags/3.01/gsdl/src/mgpp/COPYING
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/COPYING	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl/src/mgpp/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/Makefile.in	(revision 21112)
@@ -0,0 +1,113 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CPP = @CXX@
+CFLAGS = @CFLAGS@ -ansi
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib text
+
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall:
+#config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+# toplevel Makefile does our subdirs already (jrm21)
+#distclean: distclean-recursive distclean-local
+#	rm config.status
+distclean:
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive \
+maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile config.cache config.h config.log stamp-h
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in
+
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
+
Index: /main/tags/3.01/gsdl/src/mgpp/acconfig.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/acconfig.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/acconfig.h	(revision 21112)
@@ -0,0 +1,41 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
Index: /main/tags/3.01/gsdl/src/mgpp/aclocal.m4
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/aclocal.m4	(revision 21112)
@@ -0,0 +1,224 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ(regex)
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , 
+   [AC_CHECK_SIZEOF(unsigned char *, unsigned char *)  
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      AC_MSG_WARN(I'm forcing you to use regex because I can't 
+        find a local rx library and the one included with this 
+        distribution doesn't work on 64-bit machines like yours)
+      AC_LIBOBJ(regex)
+    else
+      AC_LIBOBJ(rx)
+    fi]
+  )
+fi
+])
+
+
+## --------------------------------------- ##
+## Check if --with-gnu-readline was given. ##
+## --------------------------------------- ##
+
+AC_DEFUN(fp_WITH_GNU_READLINE,
+[AC_MSG_CHECKING(whether GNU readline requested)
+  AC_ARG_WITH(gnu_readline,
+    [  --with-gnu-readline     compile with GNU readline support],
+    [if test "$withval" = yes; then
+      AC_MSG_RESULT(yes)
+      ac_with_gnu_readline=1
+    else
+      AC_MSG_RESULT(no)
+    fi],
+    AC_MSG_RESULT(no))
+
+  if test -n "$ac_with_gnu_readline"; then
+    AC_CHECK_HEADER(readline/readline.h, ac_mg_readline_header_found=1,
+	AC_MSG_WARN(Can't find GNU readline headers; configuring without \
+GNU readline support))
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      AC_CHECK_LIB(readline, main, 
+        [ac_mg_readline_lib_found=1
+         AC_DEFINE(WITH_GNU_READLINE)
+         LIBS="$LIBS -lreadline"],
+        AC_MSG_WARN(Can't find GNU readline library; configuring without \
+GNU readline support))
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to 
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	AC_CHECK_LIB(curses, main,
+	  LIBS="$LIBS -lcurses")
+      fi
+    fi
+  fi
+])	
+
+
+
+#AC_DEFUN(fp_WITH_GNU_READLINE,
+#[AC_MSG_CHECKING(whether GNU readline requested)
+#  AC_ARG_WITH(gnu-readline,
+#    [  --with-gnu-readline   compile with GNU readline support],
+#    [if test "$withval" = yes; then
+#      AC_MSG_RESULT(yes)
+#      ac_with_gnu_readline=1
+#    else
+#      AC_MSG_RESULT(no)
+#    fi],
+#    AC_MSG_RESULT(no))
+#  if test -n "$ac_with_gnu_readline"; then
+#    AC_CHECK_LIB(readline, main, 
+#      AC_CHECK_HEADER(readline/readline.h,
+#	[AC_DEFINE(WITH_GNU_READLINE)
+## curses required on some systems (DEC OSF/1), but not on others (GNU/Linux,
+##  Solaris 2.6).  Forcing its inclusion is unsophisticated; if it causes 
+##  any actual problems on any UNIX platforms, let the maintainer know.
+#	LIBS="$LIBS -lreadline -lcurses"],
+#        AC_MSG_WARN(Can't find GNU readline headers: configuring \
+#without GNU readline support)
+#      ),
+#      AC_MSG_WARN(Can't find GNU readline library: configuring \
+#without GNU readline support)
+#    )
+#  fi 
+#])
Index: /main/tags/3.01/gsdl/src/mgpp/config.guess
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/config.guess	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/gsdl/src/mgpp/config.h.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/config.h.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/config.h.in	(revision 21112)
@@ -0,0 +1,186 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
+
+/* The number of bytes in a unsigned char *.  */
+#undef SIZEOF_UNSIGNED_CHAR_P
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#undef HAVE_STRERROR
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
Index: /main/tags/3.01/gsdl/src/mgpp/config.sub
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/config.sub	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/gsdl/src/mgpp/configure
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/configure	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/configure	(revision 21112)
@@ -0,0 +1,8488 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="text/MGQuery.cpp"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Features:
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --enable-override-longlong
+                          overide use of long long with GCC
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+  --with-gnu-readline     compile with GNU readline support
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mgpp
+VERSION=1.3
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  echo "$as_me:$LINENO: checking for unsigned char *" >&5
+echo $ECHO_N "checking for unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_type_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((unsigned char * *) 0)
+  return 0;
+if (sizeof (unsigned char *))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_unsigned_char_p=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_unsigned_char_p=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_type_unsigned_char_p" >&6
+
+echo "$as_me:$LINENO: checking size of unsigned char *" >&5
+echo $ECHO_N "checking size of unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_sizeof_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_unsigned_char_p" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_unsigned_char_p=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (unsigned char *)); }
+unsigned long ulongval () { return (long) (sizeof (unsigned char *)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (unsigned char *))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_unsigned_char_p=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_unsigned_char_p=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_sizeof_unsigned_char_p" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_UNSIGNED_CHAR_P $ac_cv_sizeof_unsigned_char_p
+_ACEOF
+
+
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      { echo "$as_me:$LINENO: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&5
+echo "$as_me: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&2;}
+      case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+    else
+      case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+    fi
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking whether GNU readline requested" >&5
+echo $ECHO_N "checking whether GNU readline requested... $ECHO_C" >&6
+
+# Check whether --with-gnu_readline or --without-gnu_readline was given.
+if test "${with_gnu_readline+set}" = set; then
+  withval="$with_gnu_readline"
+  if test "$withval" = yes; then
+      echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+      ac_with_gnu_readline=1
+    else
+      echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+    fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+  if test -n "$ac_with_gnu_readline"; then
+    if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking readline/readline.h usability" >&5
+echo $ECHO_N "checking readline/readline.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <readline/readline.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking readline/readline.h presence" >&5
+echo $ECHO_N "checking readline/readline.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <readline/readline.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: readline/readline.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: readline/readline.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_readline_readline_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+
+fi
+if test $ac_cv_header_readline_readline_h = yes; then
+  ac_mg_readline_header_found=1
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&2;}
+fi
+
+
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      echo "$as_me:$LINENO: checking for main in -lreadline" >&5
+echo $ECHO_N "checking for main in -lreadline... $ECHO_C" >&6
+if test "${ac_cv_lib_readline_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lreadline  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_readline_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_readline_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_readline_main" >&5
+echo "${ECHO_T}$ac_cv_lib_readline_main" >&6
+if test $ac_cv_lib_readline_main = yes; then
+  ac_mg_readline_lib_found=1
+         cat >>confdefs.h <<\_ACEOF
+#define WITH_GNU_READLINE 1
+_ACEOF
+
+         LIBS="$LIBS -lreadline"
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&2;}
+fi
+
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	echo "$as_me:$LINENO: checking for main in -lcurses" >&5
+echo $ECHO_N "checking for main in -lcurses... $ECHO_C" >&6
+if test "${ac_cv_lib_curses_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurses  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_curses_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_curses_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_curses_main" >&5
+echo "${ECHO_T}$ac_cv_lib_curses_main" >&6
+if test $ac_cv_lib_curses_main = yes; then
+  LIBS="$LIBS -lcurses"
+fi
+
+      fi
+    fi
+  fi
+
+
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+# Check whether --enable-override-longlong or --disable-override-longlong was given.
+if test "${enable_override_longlong+set}" = set; then
+  enableval="$enable_override_longlong"
+  cat >>confdefs.h <<\_ACEOF
+#define DISABLE_LONG_LONG 1
+_ACEOF
+
+fi;
+
+# ---------------------------------------------------------------------------
+
+                              ac_config_files="$ac_config_files Makefile text/Makefile lib/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "text/Makefile" ) CONFIG_FILES="$CONFIG_FILES text/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/gsdl/src/mgpp/configure.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/configure.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/configure.in	(revision 21112)
@@ -0,0 +1,108 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(text/MGQuery.cpp)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mgpp
+VERSION=1.3
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+fp_WITH_GNU_READLINE
+
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+AC_ARG_ENABLE(override-longlong,
+[  --enable-override-longlong
+                          overide use of long long with GCC],
+  AC_DEFINE(DISABLE_LONG_LONG))
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile text/Makefile lib/Makefile ], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/gsdl/src/mgpp/install-sh
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/install-sh	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/gsdl/src/mgpp/lib/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,155 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CXX = @CXX@
+ifneq ($(NZDL), 1)
+CFLAGS = @CFLAGS@ -ansi
+else 
+CFLAGS = @CFLAGS@ -ansi -DSILENT
+endif
+CXXFLAGS = @CXXFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+prefix = @prefix@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)  $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) -D__cplusplus $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+# want toplevel config.h, not local one
+INCLUDES = -I../../.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h      \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h
+
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp
+# memlib.cpp removed
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+
+all : libmg.a
+
+libmg.a : $(OBJECTS)
+	rm -f libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmg.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# breaks make distclean (jrm21)
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl/src/mgpp/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,108 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..\..
+USE_STLPORT = 0
+DLL = 0
+DLLDEBUG = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+CC = cl
+
+CPPFLAGS = -GX
+!IF $(DLL)
+CPPFLAGS = $(CPPFLAGS) -MD
+!ELSE
+!IF ($(DLLDEBUG))
+CPPFLAGS = $(CPPFLAGS) -MDd
+!ENDIF
+!ENDIF
+
+DEFS = -DSHORT_SUFFIX -DSILENT -DHAVE_CONFIG_H -D__WIN32__
+INCLUDES = -I"$(GSDLHOME)"
+!IF $(USE_STLPORT)
+DEFS = $(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+
+LDFLAGS =
+
+LIBS = 
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj .cpp
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h     getopt_old.h \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp getopt_old.c
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o  win32in$o getopt_old$o
+
+all : libmg.lib
+
+libmg.lib: $(OBJECTS)
+	if EXIST libmg.lib del libmg.lib
+	lib /out:libmg.lib $(OBJECTS)
+
+clean:
+	del *$o 
+	del libmg.lib
+
+install:
Index: /main/tags/3.01/gsdl/src/mgpp/lib/alloca.c
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.cpp	(revision 21112)
@@ -0,0 +1,134 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::unary_length(val);
+  //  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::binary_length (val, b);
+  return (num);
+}
+
+
+unsigned long BIO_Gamma_Length (unsigned long val) {
+  register unsigned long num;
+  num = bitio_buffer::gamma_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::delta_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  num = bitio_buffer::elias_length(val, b, s);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::bblock_length (val, b);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 ((N - p) / p);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  clogb = bitio_buffer::ceil_log2 (b);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 (b);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::ceil_log2 (b);
+  return logb;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+
+
+
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_abstract.h	(revision 21112)
@@ -0,0 +1,340 @@
+#ifndef H_BITIO_ABSTRACT
+#define H_BITIO_ABSTRACT
+
+class bitio_buffer {
+public:
+  virtual int bit() = 0;
+  virtual void encodeBit(int bit) = 0;
+  virtual void seek(unsigned long topos) = 0;
+  
+  static inline void check_positive(char *type, unsigned long x) {
+    if (x <= 0) {
+      fprintf(stderr, "Error: Cannot %s encode %lu\n", type, x);
+      exit(1);
+    }
+  }
+
+  static inline int floor_log2(unsigned long x) {
+    unsigned long  _B_x = x;
+    int reply;
+    
+    reply = -1;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+  
+  static inline int ceil_log2(unsigned long x) {
+    register int _B_x;
+    unsigned long reply;
+    
+    _B_x = x - 1;
+    reply = 0;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+
+  inline unsigned long unary_decode(unsigned long *count) {
+    unsigned long x;
+    
+    x = 1;
+    while (!this->bit()) ++x;
+    if (count != NULL) {
+      *count += x;
+    }
+    return x;
+  }
+
+  inline void unary_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_x = x;
+    check_positive("unary", _B_x);
+    if (count != NULL) {
+      *count += _B_x;
+    }
+    while(--_B_x) 
+      encodeBit(0);
+    encodeBit(1);
+  }
+
+
+  inline unsigned long binary_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = 0;
+    register unsigned long _B_b = b;
+    register int _B_i, _B_logofb, _B_thresh;
+    register unsigned long reply;
+    
+    if (_B_b != 1) {					
+      _B_logofb = ceil_log2(_B_b);
+      _B_thresh = (1<<_B_logofb) - _B_b;
+      --_B_logofb;
+      
+      if (count != NULL) {
+	*count += _B_logofb;
+      }
+      
+      for (_B_i=0; _B_i < _B_logofb; ++_B_i) {
+	_B_x += _B_x + this->bit();
+      }
+
+      if ((int)_B_x >= _B_thresh) {
+	_B_x += _B_x + this->bit();
+	_B_x -= _B_thresh;
+	if (count != NULL) {
+	  *count = *count + 1;
+	}
+      }
+      reply = _B_x+1;
+      
+    } else reply = 1;
+    
+    return reply;
+  }
+
+  inline void binary_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else {
+      _B_nbits = _B_logofb;
+      _B_x += _B_thresh;
+    }
+    if (count != NULL) {
+      *count += _B_nbits;
+    }
+    while (--_B_nbits>=0)
+      encodeBit((_B_x>>_B_nbits) & 0x1);
+  }
+
+  inline unsigned long bblock_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x1, _B_xx = 0;
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb;
+    
+    _B_xdivb = this->unary_decode(count);
+    --_B_xdivb;				       
+    while (_B_xdivb--) {
+      _B_xx += _B_bb;
+    }
+    _B_x1 = this->binary_decode(_B_bb, count);
+    return (_B_xx + _B_x1);
+  }
+  
+  inline void bblock_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    unary_encode(_B_xdivb+1, count);
+    binary_encode(_B_xx+1, _B_bb, count);
+  }
+
+  inline unsigned long elias_decode(unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k;
+    register double _B_pow=1.0;
+    
+    _B_k = this->unary_decode(count);
+    --_B_k;
+    while (_B_k--) {
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    _B_xx = this->binary_decode(_B_upper-_B_lower+1, count);
+    --_B_xx;
+    check_positive("gamma", _B_xx+_B_lower);
+    return _B_xx+_B_lower;
+  }
+  
+  inline void elias_encode(unsigned long x, unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0;
+    register double _B_pow=1.0;
+    
+    check_positive("elias", _B_xx);
+    
+    while ((int)_B_xx>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    unary_encode(_B_k+1, count);
+    binary_encode(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);
+  }
+  
+  inline unsigned long gamma_decode(unsigned long *count) {
+    register unsigned long _B_xx = 1;
+    register int _B_nbits = 0;
+    
+    while(!this->bit()) ++_B_nbits;
+    if (count != NULL) {
+      *count += _B_nbits*2+1;
+    }
+    while (_B_nbits-- > 0)
+      _B_xx += _B_xx + this->bit();
+    return _B_xx;
+  }
+
+  inline void gamma_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    register int _B_nbits;
+    
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    _B_nbits = _B_logofb+1;
+    
+    if (count != NULL) { 
+      *count += _B_logofb*2+1;
+    }
+    while(_B_logofb--) encodeBit(0);
+    while (--_B_nbits>=0) encodeBit((_B_xx>>_B_nbits) & 0x1);
+  }
+  
+  inline unsigned long delta_decode(unsigned long *count) {
+    register unsigned long _B_xxx;
+    register int _B_logx;
+    _B_logx = gamma_decode(count);
+    --_B_logx;
+    _B_xxx = binary_decode(1<<_B_logx, count); 
+    --_B_xxx;
+    return (_B_xxx + (1<<_B_logx));
+  }
+
+  inline void delta_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx;
+    _B_logx = floor_log2(_B_xxx);
+    gamma_encode(_B_logx+1, count);
+    binary_encode(_B_xxx+1, 1<<_B_logx, count);
+  }
+
+  inline unsigned long huff_decode(unsigned long *mcodes, unsigned long **values,
+				   unsigned long *count = NULL) {
+    register unsigned long *__min_code = mcodes;
+    register unsigned long *__mclen    = mcodes;
+    register unsigned long **__values  = values;
+    register unsigned long __code = 0;
+    do {
+      __code += __code + bit();
+      if (count != NULL) ++(*count);
+    } while (__code < *++__mclen);
+    
+    return __values[__mclen - __min_code][__code - *__mclen];
+  }
+
+  inline void huff_encode(unsigned long x, unsigned long *codes, char *lens,
+			  unsigned long *count = NULL) {
+    register int __i;
+    register int __clen = lens[x];
+    register unsigned long __code = codes[x];
+    for (__i=__clen-1; __i>=0; --__i) {
+      encodeBit((__code >> __i) & 1);
+    }
+    if (count != NULL) {
+      *count += lens[x];
+    }
+  }
+
+  static inline unsigned long unary_length(unsigned long x) {
+    check_positive("unary", x);
+    return x;
+  }
+
+  static inline int binary_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else _B_nbits = _B_logofb;
+    return _B_nbits;
+  }
+
+  static inline unsigned long bblock_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_bcount, count, _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    count = unary_length(_B_xdivb+1);
+    _B_bcount = binary_length(_B_xx+1, _B_bb);
+    count += _B_bcount;
+    return count;
+  }
+  
+  static inline unsigned long elias_length(unsigned long x, unsigned long b, 
+					   double s) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0, _B_ecount;
+    register double _B_pow=1.0;
+    register unsigned long count;
+    check_positive("gamma", _B_xx);
+    
+    while ((int)(_B_xx)>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    count = unary_length(_B_k+1);
+    _B_ecount = binary_length(_B_xx-_B_lower+1, _B_upper-_B_lower+1);
+    count += _B_ecount;
+    return count;
+  }
+  
+  static inline unsigned long gamma_length(unsigned long x) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    return _B_logofb*2+1;
+  }
+
+  static inline unsigned long delta_length(unsigned long x) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx, _B_dcount;
+    register unsigned long count;
+    _B_logx = floor_log2 (_B_xxx);
+    count = gamma_length(_B_logx+1);
+    _B_dcount = binary_length (_B_xxx+1, 1<<_B_logx);
+    count += _B_dcount;
+    return count;
+  }
+};
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
@@ -0,0 +1,128 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mem.h"
+
+mem_bitio_buffer::mem_bitio_buffer() {
+}
+
+mem_bitio_buffer::mem_bitio_buffer(unsigned char *readbuffer, int size) { 
+  pos       = readbuffer;
+  base      = readbuffer;
+  remaining = size;
+  buffer    = 0;
+  btg       = 0;
+}
+
+
+void mem_bitio_buffer::error() {
+  fprintf (stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+void mem_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+
+void mem_bitio_buffer::seek(unsigned long toPos) { 
+  register long _newPos;
+  do {
+    _newPos = toPos;
+    
+    pos    = base + (_newPos >> 3);
+    buffer = *pos++;
+    btg    = 8 - (_newPos & 0x07);
+  } while (0);
+}
+
+long mem_bitio_buffer::addff(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0xff;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+
+  return b;
+}
+
+long mem_bitio_buffer::add00(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0x00;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+  
+  return b;
+}
+
+int mem_bitio_buffer::bit() {
+  if (btg == 0) {
+
+    if (remaining == 0) { 
+      this->error();
+      return 0;
+
+    } else {
+      buffer = *pos++;
+      --remaining;
+      btg = sizeof(buffer) * 8;
+    }
+    
+  }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void mem_bitio_buffer::encodeBit(int b) {
+  --btg;
+  if (b) buffer |= (1 << btg);
+  
+  if (btg == 0) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+    buffer = 0;
+    btg = sizeof(buffer)*8;
+  }
+}
+
+void mem_bitio_buffer::flush() {
+  if (btg != sizeof(buffer)*8) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+  }
+  btg = sizeof(buffer)*8;
+}
+
+void mem_bitio_buffer::done() {
+  // do nothing
+}
+
+int mem_bitio_buffer::encodeLength() {
+  return pos - base;
+}
+
+int mem_bitio_buffer::encodeVerify() {
+  return (remaining != 0);
+}
+
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,104 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+#include "bitio_m_abstract.h"
+
+
+class mem_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *pos;
+  unsigned char *base;
+  int            remaining;
+  unsigned char  buffer;
+  unsigned char  btg;
+  
+public:
+  mem_bitio_buffer ();
+  mem_bitio_buffer (unsigned char *buffer, int size);
+  void seek (unsigned long topos);
+  void error ();
+  long add00 (long b);
+  long addff (long b);
+  int  bit ();
+  void done ();
+
+  void encodeStart ();
+  void encodeBit (int bit);
+  void encodeDone () { flush(); }
+  int  encodeVerify ();
+  int  encodeLength ();
+  void flush ();
+};
+
+
+/*
+#define ENCODE_START(p,r)						\
+  {									\
+    mem_bitio_buffer buffer(p, r);                                     \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush()
+
+#define ENCODE_DONE buffer.flush();}
+
+
+#define DECODE_START(p,r)						\
+  {	                                                                \
+    mem_bitio_buffer buffer(p,r);
+
+#define DECODE_CONTINUE(b)						\
+  {	                                                                \
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
@@ -0,0 +1,89 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mems.h"
+
+/**
+ * Create decoding "stream"/"file"
+ */
+mems_bitio_buffer::mems_bitio_buffer()
+{
+}
+
+mems_bitio_buffer::mems_bitio_buffer(unsigned char *readbuffer, long _pos)
+{ 
+  base = readbuffer;
+  pos  = _pos;
+}
+
+void mems_bitio_buffer::encodeStart()
+{
+  // Do nothing
+}
+
+
+void mems_bitio_buffer::error()
+{
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+
+void mems_bitio_buffer::seek(unsigned long toPos)
+{ 
+  pos = toPos;
+}
+
+unsigned long mems_bitio_buffer::position()
+{
+  return pos;
+}
+
+long mems_bitio_buffer::addff(long b)
+{
+  do
+    {
+      b = b + b + (base[pos>>3] & (0x80 >> (pos & 0x07)) != 0);
+      ++pos;
+    }
+  while(0);
+
+  return b;
+}
+
+long mems_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int mems_bitio_buffer::bit()
+{
+  ++pos;
+  return ((base[(pos-1)>>3] & (0x80 >> ((pos-1) & 0x07))) != 0);
+}
+
+void mems_bitio_buffer::encodeBit(int bit)
+{
+  if (bit)
+    base[pos>>3] |= 0x80 >> (pos & 0x07);
+  else
+    base[pos>>3] &= 0xff7f >> (pos & 0x07);
+  ++pos;
+}
+
+void mems_bitio_buffer::flush()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::done()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::encodeDone()
+{
+  flush();
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+#include "bitio_m_abstract.h"
+
+
+class mems_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *base;
+  unsigned long  pos;
+  
+public:
+  mems_bitio_buffer();
+  mems_bitio_buffer(unsigned char *readbuffer, long _pos);
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  //  int  bitOffset();
+  void done();
+
+  unsigned long position();
+
+  void encodeStart();
+  void encodeBit(int b);
+  void encodeDone();
+  void flush();
+};
+
+
+/*#define ENCODE_START(b,p)						\
+  {									\
+    mems_bitio_buffer buffer(b, p);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+
+#define DECODE_START(b,p)						\
+  {		                                                        \
+    mems_bitio_buffer buffer(b, p);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b);
+
+#define DECODE_ADD_00(b) b = buffer.add00(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "longlong.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_random.h"
+
+random_bitio_buffer::random_bitio_buffer(FILE *f, unsigned long length)
+{
+  buffer = NULL;
+  attachFile (f, length);
+}
+
+random_bitio_buffer::~random_bitio_buffer() {
+  if (buffer != NULL) delete []buffer;
+}
+
+void random_bitio_buffer::attachFile (FILE *f, unsigned long length) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  // reset all the values
+  file   = f;
+  pos    = 0;
+  base   = 0;
+  used   = 0;
+  len    = length - 1;
+  shift  = 0;
+
+  // make sure the length is a power of 2
+  while (len != 0) {
+    ++shift;
+    len = len >> 1;
+  }
+  len = 1 << shift;
+
+  // allocate a new buffer
+  buffer = new u_char[len];
+
+  // fill up the buffer
+  if (file != NULL) {
+    fseek(file, 0, 0);
+    fread(buffer, 1, len, file);
+  }
+}
+
+random_bitio_buffer &random_bitio_buffer::operator= (const random_bitio_buffer &_rbb) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  file = _rbb.file;
+
+  base = _rbb.base;
+  used = _rbb.used;
+  pos = _rbb.pos;
+  len = _rbb.len;
+  shift = _rbb.shift;
+
+  // copy the buffer
+  buffer = new u_char [len];
+  //  memcpy (buffer, _rbb.buffer, len);
+memcpy (buffer, _rbb.buffer, len);
+  
+  return (*this);
+}
+
+void random_bitio_buffer::error() {
+}
+
+void random_bitio_buffer::writeRead()
+{
+  if (used)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+    }
+  //  else
+  //    {
+      base += len;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = 0;
+      used = 0;
+  //    }
+}
+
+void random_bitio_buffer::encodeBit(int bit)
+{
+  do
+    {
+      if (bit != 0)
+	{
+	  buffer[pos >> 3] |= 0x80 >> (pos & 0x07);
+	}
+      else
+	{
+	  buffer[pos >> 3] &= 0xff7f >> (pos & 0x07);
+	}
+      used = 1;
+      ++pos;
+
+      if ((pos >> 3) >= len)
+	{
+	  writeRead();
+	}
+    }
+  while (0);
+}
+
+void random_bitio_buffer::flush()
+{
+  if (used != 0)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+      used = 0;
+    }
+}
+
+void random_bitio_buffer::encodeDone()
+{
+  flush();
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+}
+
+unsigned long random_bitio_buffer::tell()
+{
+  return ((base << 3) + pos);
+}
+
+void random_bitio_buffer::seek(unsigned long toPos)
+{
+  if (((toPos >> 3) >= base) && (((toPos+7) >> 3) < (base + len)))
+    {
+      pos = toPos - (base << 3);
+    }
+  else
+    {
+      flush();
+      base = toPos >> 3;
+      base = (toPos >> (shift + 3)) << shift;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = toPos & 0x07;
+      pos = toPos & ((8 << shift) - 1);
+    }
+}
+
+int random_bitio_buffer::bit()
+{
+  //  fprintf(stderr, "Bit in");
+  ++pos;
+  if (((pos - 1) >> 3) >= len)
+    {
+      writeRead();
+      pos = 1;
+    }
+  return (buffer[(pos - 1) >> 3] & (0x80 >> ((pos - 1) & 0x07))) != 0;
+}
+
+long random_bitio_buffer::addff(long b)
+{
+  return b + b + bit();
+}
+
+long random_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int random_bitio_buffer::bitOffset()
+{
+  return 0;
+}
+
+void random_bitio_buffer::done()
+{
+  encodeDone();
+}
+
+
+#ifdef USE_LONG_LONG
+void random_bitio_buffer::seek_LL(mg_ullong toPos)
+{
+  if (((toPos >> 3) >= (mg_ullong) base) &&
+      (((toPos + 7) >> 3) < (mg_ullong) (base + len)))
+    {
+      pos = (long) (toPos - (mg_ullong) (base << 3));
+    }
+  else
+    {
+      flush();
+      base = (long) ((toPos >> (shift + 3)) << shift);
+
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = (long) (toPos & ((8 << shift) - 1));
+    }
+}
+
+mg_ullong random_bitio_buffer::tell_LL()
+{
+  return (((mg_ullong) base) << 3ull) + pos;
+}
+#endif
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+#include "longlong.h"
+#include "bitio_m_abstract.h"
+
+
+class random_bitio_buffer : public bitio_buffer { 
+public:
+  FILE *file;
+  unsigned char *buffer;
+  unsigned long  base;
+  unsigned long  used;
+  unsigned long  pos;
+  unsigned long  len;
+  unsigned long  shift;
+  void writeRead();
+
+  random_bitio_buffer(FILE *f = 0,
+		      unsigned long length = 8*1024);
+  virtual ~random_bitio_buffer();
+  
+  void attachFile (FILE *f, unsigned long length);
+
+  random_bitio_buffer &operator= (const random_bitio_buffer &_rbb);
+  
+  void encodeDone();
+  void encodeBit(int b);
+  void flush();
+  unsigned long tell();
+
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset();
+  void done();
+
+#ifdef USE_LONG_LONG
+  // extras for longlong.h
+  void seek_LL(mg_ullong pos);
+  mg_ullong tell_LL();
+#endif
+};
+
+
+/*#define ENCODE_START(f,l)						\
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define SEEK fprintf(stderr, "Seek to %d\n",buffer.base)
+#define READ fprintf(stderr, "Read of %d\n",buffer.len)
+#define WRITE fprintf(stderr, "Write of %d\n",buffer.used)
+
+#define WRITE_READ buffer.writeRead()
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+#define ENCODE_DONE buffer.done();}							
+
+#define DECODE_START(f,l) \
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define DECODE_CONTINUE(b) \
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_TELL buffer.tell()
+
+#define DECODE_TELL buffer.tell()
+*/
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
@@ -0,0 +1,125 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_stdio.h"
+
+stdio_bitio_buffer::stdio_bitio_buffer(FILE *f) {
+  attachFile (f);
+}
+
+void stdio_bitio_buffer::attachFile (FILE *f) {
+  file = f;
+  buffer = 0;
+  btg = 0;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+void stdio_bitio_buffer::error() {
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d", __FILE__, __LINE__);
+  exit(1);
+}
+
+void stdio_bitio_buffer::seek (unsigned long byte, unsigned char bit) {
+  if (fseek (file, byte, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = bit;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::seek(unsigned long topos) {
+  if (fseek(file, topos >> 3, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = 8 - (topos & 0x07);
+  bytesWritten = 0;
+}
+
+long stdio_bitio_buffer::addff(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0x0ff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+
+  return b;
+}
+
+long stdio_bitio_buffer::add00(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0xff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+  
+  return b;
+}
+
+void stdio_bitio_buffer::encodeBit(int b)
+{
+  --btg;
+  if (b != 0)
+    buffer |= (1 << btg);
+
+  if (btg == 0)
+    {
+      putc(buffer, file);
+      bytesWritten += sizeof (buffer);
+      buffer = 0;
+      btg = sizeof(buffer)*8;
+    }
+}
+
+int stdio_bitio_buffer::bit()
+{
+  if (btg == 0)
+    {
+      buffer = getc(file);
+      if (feof(file))
+	{
+	  error();
+	  return 0;
+	}
+      else
+	{
+	  btg = sizeof(buffer) * 8;
+	}
+    }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void stdio_bitio_buffer::done() {
+  // no tidying required
+}
+
+void stdio_bitio_buffer::encodeDone() {
+  if (btg != sizeof(buffer) * 8) {
+    putc(buffer, file);
+    bytesWritten += sizeof (buffer);
+  }
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,101 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+#include "bitio_m_abstract.h"
+
+
+class stdio_bitio_buffer : public bitio_buffer {
+protected:
+  FILE *file;
+  unsigned char buffer;
+  unsigned char btg;
+  unsigned long bytesWritten;
+  
+public:
+  stdio_bitio_buffer (FILE *f = NULL);
+
+  void attachFile (FILE *f);
+
+  unsigned char GetBtg () { return btg; }
+  unsigned long GetBytesWritten () { return bytesWritten; }
+  
+  void seek (unsigned long byte, unsigned char bit);
+  void seek (unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset() { return btg; }
+  void done();
+
+  void encodeStart();
+  void encodeBit(int bit);
+  void encodeDone();
+};
+
+
+/*#define ENCODE_START(f)							\
+  {									\
+    stdio_bitio_buffer buffer(f);                                       \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b) buffer.pause(&b);}
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+#define DECODE_START(f)							\
+  {		                                                        \
+    stdio_bitio_buffer buffer(f);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+     buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/ftruncate.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/ftruncate.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/ftruncate.cpp	(revision 21112)
@@ -0,0 +1,99 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.c
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.c	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.c	(revision 21112)
@@ -0,0 +1,748 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+/* jrm21 - GSDL: getopt.h -> getopt_old.h */
+#include "getopt_old.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/getopt_old.h	(revision 21112)
@@ -0,0 +1,134 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef HAVE_DECL_GETOPT
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  int getopt (int, char * const *, const char *);
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/huffman.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/huffman.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/huffman.cpp	(revision 21112)
@@ -0,0 +1,385 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = new huff_data))
+	goto error0;
+    }
+  if (!(heap = new unsigned long[num * 2]))
+    goto error1;
+  if (!(hd->clens = new char[num]))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  memcpy (&heap[num], freqs, num * sizeof (*heap));
+  memset (heap, '\0', num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; ++i)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; --i)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    ++child;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; ++i)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		++child;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	++HNum;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; ++i)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  memset (hd->lencount, '\0', sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; ++i)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      ++hd->lencount[codelen];
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; --i)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  delete []heap;
+  return (hd);
+
+error2:
+  delete []heap;
+error1:
+  if (!data)
+    delete hd;
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  memcpy (mc, data->min_code, sizeof (mc));
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (!(values = new unsigned long *[MAX_HUFFCODE_LEN + 1]))
+    {
+      delete []vals;
+      return (NULL);
+    }
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; ++i)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; ++i)
+    size += counts[i] * hd.clens[i];
+  delete []hd.clens;
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk) {
+  if (fread (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      memset (hd->lencount, '\0', sizeof (hd->lencount));
+      if (fread (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      memset (hd->min_code, '\0', sizeof (hd->min_code));
+
+      if (fread (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = new char[hd->num_codes]))
+	return -1;
+
+
+      if (fread (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/huffman.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/huffman.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+#define MAX_HUFFCODE_LEN 31
+
+struct huff_data {
+  int num_codes;
+  int mincodelen;
+  int maxcodelen;
+  int lencount[MAX_HUFFCODE_LEN + 1];
+  unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+  char *clens;
+};
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.cpp	(revision 21112)
@@ -0,0 +1,411 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = new u_char(src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  memcpy (copy, src_str, src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    ++i;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  memset (buf, '\0', sizeof(buf));
+  for (i = 0; i < len; ++i)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  ++s;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      --s;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; ++i)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+class mgString
+{
+  u_char *data;
+ public:
+  inline int length()
+    {
+      return data[0];
+    }
+  inline u_char *rawdata()
+    {
+      return data;
+    }
+  inline void read(u_char **base)
+    {
+      unsigned int copy, suff;
+      
+      copy  = **base;
+      *base = *base + 1;
+      suff  = **base;
+      *base = *base + 1;
+      memcpy( (data + copy + 1), *base, suff);
+      *data = copy + suff;
+      *base += suff;
+    }
+  inline void read(FILE *dict)
+    {
+      unsigned int copy, suff;
+      copy = fgetc(dict);
+      suff = fgetc(dict);
+      fread(data + copy + 1, sizeof(u_char), suff, dict);
+      *data = copy + suff;
+    }
+  inline static void readString(u_char **base, u_char *prev)
+    {
+      ((mgString *) &prev)->read(base);
+    }
+  inline static void readString(FILE *dict, u_char *prev)
+    { 
+      ((mgString *) &prev)->read(dict);
+    }
+};
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/longlong.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/longlong.h	(revision 21112)
@@ -0,0 +1,85 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+    
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.cpp	(revision 21112)
@@ -0,0 +1,1172 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+/*static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();*/
+
+static int 
+aio (char ch)			/* Rule number 9     */
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (char ch)				/* Rule Number 24    */
+{
+  return (ch != 's');
+}
+
+static int 
+pt (char ch)				/* Rule number 28    */
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (char ch)				/* Rule number 30    */
+{
+  return (ch != 'm');
+}
+
+static int 
+n (char ch)				/* Rule number 32    */
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+/*static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+*/
+static int 
+B (int stem_length, char *end)
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (int stem_length, char *end)
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (int stem_length, char *end)
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (int stem_length, char *end)
+
+     
+{
+  return (*end != 'e');
+}
+
+static int 
+F (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'c');
+}
+
+static int 
+W (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (int stem_length, char *end)
+     
+     
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  memcpy (copy, word, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  memcpy (word, copy, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) (char);	        /* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  ++rule;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) (int length, char *end);		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      ++suffix_start;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,28 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/memlib.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/memlib.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/memlib.cpp	(revision 21112)
@@ -0,0 +1,41 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+char *
+my_strdup(const char *str)
+{
+  char *ret_str = new char[strlen(str)+1];
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_strdup;
Index: /main/tags/3.01/gsdl/src/mgpp/lib/memlib.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/memlib.h	(revision 21112)
@@ -0,0 +1,55 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+/* changed default behaviour - we now assume std memory. - john mcp. */
+/* #ifdef STD_MEM */
+#if !defined(NOT_STD_MEM)
+#include <string.h>
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+
+#if defined (__WIN32__)
+# define Xstrdup _strdup
+#else
+# define Xstrdup strdup
+#endif
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/messages.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/messages.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/messages.cpp	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef SILENT
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/messages.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/messages.h	(revision 21112)
@@ -0,0 +1,49 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+void Message (const char *fmt,...);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
+
+
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.cpp	(revision 21112)
@@ -0,0 +1,148 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; --__i)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; --__i)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	++child;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; --i)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; --i)
+    {
+      SWAP (size, (char *) heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  --(*num);
+  SWAP (size, (char *) heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  ++(*num);
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/gsdl/src/mgpp/lib/netorder.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/netorder.h	(revision 21112)
@@ -0,0 +1,178 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy ( (void *) &d, (const void *) &tmpl, sizeof (double) >> 1);                           \
+	     memcpy ((void *) ((unsigned int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &d, (const void *) &tmpl, sizeof (double) >> 1); \
+	     memcpy ((void *) ((int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &hd, sizeof(double) >> 1);\
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &hd + (sizeof(double) >> 1)), sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy((void *) &nd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) ((unsigned int) &nd + (sizeof(double) >> 1)), (void *) &tmph, sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &nd, (void *) &tmph, sizeof(double) >> 1);      \
+	     memcpy ((void *) &tmpl, (const void *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &hd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) &hd + (sizeof(double) >> 1), (const void *) &tmph, sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define NTOHF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define HTONF2(hf, nf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(hf), sizeof (float));    \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(nf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+#define NTOHF2(nf, hf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(nf), sizeof (float));    \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(hf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/non_ansi.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/non_ansi.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/non_ansi.h	(revision 21112)
@@ -0,0 +1,24 @@
+#if !defined(_NON_ANSI_H)
+#define _NON_ANSI_H 1
+
+#include "config.h" 
+# if !defined(__STDC__)
+
+#include <stdlib.h>  /* for FILE def */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+int fileno(FILE *);
+int ftruncate(int, long);
+int pclose(FILE *);
+FILE *popen(const char *command, const char *type);
+int getpagesize();
+int strcasecmp(const char *, const char *);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* !defined STDC, ... */
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/pathmax.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.cpp	(revision 21112)
@@ -0,0 +1,1074 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+#define STRUCT
+
+#include <time.h>  /* for call to time () in SEED_RANDOM() */
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+
+long irandm (long is[2]);  /* from random.c */
+
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+_delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    _delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    _delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    _delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = (struct Stack *)Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = (struct Stack *) Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  ++StackPos;
+  while (StackPos)
+    {
+      int m, b;
+      --StackPos;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      _delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      _delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      _delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; ++e)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; ++v)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      *n = (int) (num * c + 1);
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; ++i)
+	for (j = 0; j < MAX_L; ++j)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; ++e)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      ++w;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int _xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ _xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; ++e)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; ++i)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  memset (translate, '\0', sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; ++i)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; --len, ++s)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; ++i)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = (int *) Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = (int *)Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = (int *)Xmalloc (sizeof (int) * MAX_M);
+  mk = (char *)Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = (tb_entry **)Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb)
+      if (!(tb[i] = (tb_entry *)Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  memset(translate, '\0', sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  MAX_N = MAX_M + MAX_M / 4;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  memcpy ((void *) g, (const void *) &g[1], n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  phd->g = (int *)Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = (struct tb_entry **)Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb)
+      if (!(phd->tb[i] = (struct tb_entry *)Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      --l;
+      ++s;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/random.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/random.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/random.cpp	(revision 21112)
@@ -0,0 +1,211 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/random.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/random.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/random.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      --last;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') --last;
+      if (last>=0 && wordstart[last]=='r') --last;
+      if (last>=0 && wordstart[last]=='e') --last;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	--last;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) --last;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,26 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/strcasecmp.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/strcasecmp.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/strcasecmp.cpp	(revision 21112)
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(char *s1, char *s2)
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/strstr.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/strstr.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/strstr.cpp	(revision 21112)
@@ -0,0 +1,42 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (char *s1, char *s2)
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, ++i, ++s)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; ++p1, ++p2)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.cpp	(revision 21112)
@@ -0,0 +1,6 @@
+#include <stdlib.h>
+
+void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/sysfuncs.h	(revision 21112)
@@ -0,0 +1,388 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+#include <sys/stat.h>
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+#ifdef __CYGWIN__ 
+/* Cygwin has setbuffer in stdio.h, but can't link against it!! */
+#include <stdio.h>
+#endif
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+
+#ifdef __cplusplus /* compiling for C++ */
+#include <stdlib.h>
+#ifndef __bcopy
+inline void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+#define __bcopy 1
+#endif
+#ifndef __bcmp
+inline int bcmp(void *s1, void *s2, int n)
+{ return memcmp(s1, s2, n);
+}
+#define __bcmp 1
+#endif
+#ifndef __bzero
+inline void bzero(void *m, int n)
+{ memset(m, 0, n);
+}
+#define __bzero 1
+#endif
+#else              /* compiling for C */
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#endif
+
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#ifdef __MSDOS__
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#ifdef __MSDOS__
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#ifndef __WIN32__
+#include <pathmax.h>
+#endif
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+/*#include <getopt.h>*/
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+/*** setlocale is ansi and posix...(jrm21)
+#if !HAVE_SETLOCALE
+#define setlocale(Category, Locale)
+#endif
+***/
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#ifdef HAVE_VPRINTF
+void error __P ((int, int, const char *,...));
+#else
+void error ();
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/gsdl/src/mgpp/lib/unitool.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/unitool.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/unitool.cpp	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; --i) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; ++i) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      ++curpos;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+/*unsigned long *unicode_cat_data;*/
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = (unsigned long *) malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) ++letter_here;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) ++digit_here;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) ++space_here;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	++i;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      ++outhere;
+      ++outlen;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    ++in_i;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/lib/unitool.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/win32in.c
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/win32in.c	(revision 21112)
@@ -0,0 +1,85 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#include "win32in.h"
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/gsdl/src/mgpp/lib/win32in.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/win32in.h	(revision 21112)
@@ -0,0 +1,20 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#ifdef __WIN32__
+#define _LITTLE_ENDIAN
+#endif
+
+#include "longlong.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
+
+#ifdef __cplusplus
+}
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/mkinstalldirs
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/gsdl/src/mgpp/stamp-h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/stamp-h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/stamp-h	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/gsdl/src/mgpp/stamp-h.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
@@ -0,0 +1,129 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.cpp -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FIvfLevelInfo.h"
+
+
+IvfLevelInfo::IvfLevelInfo () {
+  Clear();
+}
+
+void IvfLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  numEntries = 0;
+  approxWeightsDiskPtr = 0;
+  exactWeightsDiskPtr = 0;
+  workInLevel = false;
+}
+
+bool IvfLevelInfo::Read (FILE *f) {
+  workInLevel = false;
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, numEntries) &&
+	  ReadUL (f, approxWeightsDiskPtr) &&
+	  ReadUL (f, exactWeightsDiskPtr));
+}
+
+bool IvfLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, numEntries) &&
+	  WriteUL (f, approxWeightsDiskPtr) &&
+	  WriteUL (f, exactWeightsDiskPtr));
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  numEntries: " << l.numEntries << "\n"
+    << "  approxWeightsDiskPtr: " << l.approxWeightsDiskPtr << "\n"
+    << "  exactWeightsDiskPtr: " << l.exactWeightsDiskPtr << "\n"
+    << "  workInLevel: " << ((l.workInLevel) ? "true" : "false") << "\n\n";
+  
+  return s;
+}
+
+
+
+FIvfLevel::FIvfLevel () {
+  Clear ();
+}
+
+void FIvfLevel::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FIvfLevel::Read (FILE *f) {
+  if (!ReadUCArray (f, docTag) ||
+      !ReadUCArray (f, indexLevel))
+    return false;
+  
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+  
+  IvfLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FIvfLevel::Write (FILE *f) const {
+  if (!WriteUCArray (f, docTag) ||
+      !WriteUCArray (f, indexLevel))
+    return false;
+  
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  IvfLevelInfoMap::const_iterator here = levelInfo.begin();
+  IvfLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l) {
+  s << "docTag: \"" << l.docTag << "\"\n"
+    << "indexLevel: \"" << l.indexLevel << "\"\n";
+  
+  IvfLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  IvfLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FIvfLevelInfo.h	(revision 21112)
@@ -0,0 +1,77 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.h -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FIVFLEVELINFO_H
+#define FIVFLEVELINFO_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+class IvfLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long numEntries; // N for level
+  unsigned long approxWeightsDiskPtr;
+  unsigned long exactWeightsDiskPtr;
+  bool workInLevel;  // this is just a working variable,
+                     // it is not saved or read from the file
+  
+  IvfLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l);
+
+
+typedef map<UCArray, IvfLevelInfo, LTUCArray> IvfLevelInfoMap;
+
+
+class FIvfLevel {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  IvfLevelInfoMap levelInfo;
+  
+  FIvfLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/FText.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FText.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FText.cpp	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * FText.cpp -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FText.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+BitAddr::BitAddr () {
+  Clear ();  
+}
+
+void BitAddr::Clear () {
+  byte = 0;
+  bit = 0;
+}
+
+bool BitAddr::Read (FILE *f) {
+  return (ReadUL (f, byte) && ReadUC (f, bit));
+}
+
+bool BitAddr::Write (FILE *f) const {
+  return (WriteUL (f, byte) && WriteUC (f, bit));
+}
+
+#define BITADDRLEN (sizeof(unsigned long) + sizeof(unsigned char))
+
+  
+TextLevelInfo::TextLevelInfo () {
+  Clear();
+}
+
+void TextLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  textIdxPtr = 0;
+  numEntries = 0;
+}
+
+bool TextLevelInfo::Read (FILE *f) {
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, textIdxPtr) &&
+	  ReadUL (f, numEntries));
+}
+
+bool TextLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, textIdxPtr) &&
+	  WriteUL (f, numEntries));
+}
+
+ostream &operator<<(ostream &s, const TextLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  textIdxPtr: " << l.textIdxPtr << "\n"
+    << "  numEntries: " << l.numEntries << "\n\n";
+  
+  return s;
+}
+
+
+FTextLevel::FTextLevel () {
+  Clear();
+}
+
+void FTextLevel::Clear () {
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FTextLevel::Read (FILE *f) {
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+
+  TextLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FTextLevel::Write (FILE *f) const {
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  TextLevelInfoMap::const_iterator here = levelInfo.begin();
+  TextLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+ostream &operator<<(ostream &s, const FTextLevel &l) {
+  TextLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  TextLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
+
+
+TextIdx::TextIdx () {
+  Clear ();
+}
+
+void TextIdx::Clear () {
+  start.Clear();
+  end.Clear();
+  which = 0;
+}
+
+#define TEXTIDXLEN (BITADDRLEN*2 + sizeof(char))
+
+bool TextIdx::Read (FILE *f, const TextLevelInfo &levelInfo,
+		    unsigned long docNum) {
+  if (!SeekTextIdx (f, levelInfo, docNum)) return false;
+  return Read (f);
+}
+
+bool TextIdx::Read (FILE *f) {
+  return (start.Read (f) && end.Read (f) && ReadUC (f, which));
+}
+
+bool TextIdx::Write (FILE *f) const {
+  return (start.Write (f) && end.Write (f) && WriteUC (f, which));
+}
+
+ostream &operator<<(ostream &s, const TextIdx &t) {
+  s << "start byte: " << t.start.byte << "\n";
+  s << "start bit:  " << (int)t.start.bit << "\n";
+  s << "end byte:   " << t.end.byte << "\n";
+  s << "end bit:    " << (int)t.end.bit << "\n";
+  s << "which:      " << (int)t.which << "\n\n";
+  
+  return s;
+}
+
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum) {
+  if (docNum == 0 || docNum > levelInfo.numEntries) return false;
+  
+  unsigned long seekPos = levelInfo.textIdxPtr + (docNum-1) * TEXTIDXLEN;
+  if (fseek (f, seekPos, SEEK_SET) != 0) return false;
+  
+  return true;
+}
+
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize) {
+  // clear the array
+  a.erase (a.begin(), a.end());
+
+  // read in each element
+  TextIdx idx;
+  while (arrSize > 0) {
+    if (!idx.Read(f)) return false;
+    a.push_back (idx);
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a) {
+  TextIdxArray::const_iterator here = a.begin();
+  TextIdxArray::const_iterator end = a.end();
+  while (here != end) {
+    if (!(*here).Write(f)) return false;
+    ++here;
+  }
+  
+  return true;
+}
+
+
+CompressTextInfo::CompressTextInfo () {
+  Clear();
+}
+
+void CompressTextInfo::Clear () {
+  ResetStart();
+  docPtrs.erase (docPtrs.begin(), docPtrs.end());
+}
+
+void CompressTextInfo::ResetStart () {
+  inDoc = false;
+  start.Clear();
+  which = 0;
+}
+  
+void CompressTextInfo::SetStart (unsigned long startPos,
+				 unsigned char startBit,
+				 unsigned char startWhich) {
+  // place an imaginary end tag if needed
+  // note: the document tag always needs a closing tag
+  if (inDoc) SetEnd (startPos, startBit);
+
+  // remember start of level
+  inDoc = true;
+  start.byte = startPos;
+  start.bit = startBit;
+  which = startWhich;
+}
+
+void CompressTextInfo::SetEnd (unsigned long endPos,
+			       unsigned char endBit) {
+  if (inDoc) {
+    // add completed entry to list of ptrs
+    TextIdx textIdx;
+    textIdx.start = start;
+    textIdx.end.byte = endPos;
+    textIdx.end.bit = endBit;
+    textIdx.which = which;
+    docPtrs.push_back (textIdx);
+    
+    // reset this entry
+    ResetStart();
+  }
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/FText.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FText.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FText.h	(revision 21112)
@@ -0,0 +1,150 @@
+/**************************************************************************
+ *
+ * FText.h -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FTEXT_H
+#define FTEXT_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+// holds a bit address
+class BitAddr {
+public:
+  unsigned long byte;
+  unsigned char bit;
+
+  BitAddr ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+class TextLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long textIdxPtr;
+  unsigned long numEntries;
+
+  TextLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextLevelInfo &l);
+
+
+typedef map<UCArray, TextLevelInfo, LTUCArray> TextLevelInfoMap;
+
+
+class FTextLevel {
+public:
+  TextLevelInfoMap levelInfo;
+  
+  FTextLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FTextLevel &l);
+
+
+// the text.idx file points into the compressed text
+class TextIdx {
+public:
+  BitAddr start;       // the first bit of the compressed text
+  BitAddr end;         // the first bit past the end of the text
+  unsigned char which; // 0 = non-word, 1 = word
+  
+  TextIdx ();
+  void Clear ();
+
+  // does a seek and reads the appropriate record
+  bool Read (FILE *f, const TextLevelInfo &levelInfo,
+	     unsigned long docNum);
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextIdx &t);
+
+
+// note: document numbers start at 1
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum);
+
+
+// used to store the pointers while compressing the text
+typedef vector<TextIdx> TextIdxArray;
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize);
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a);
+
+
+// used to store information about a level while compressing the text
+class CompressTextInfo {
+public:
+  bool inDoc;
+  BitAddr start;
+  unsigned char which;
+  TextIdxArray docPtrs;
+
+  CompressTextInfo ();
+  void Clear ();
+
+  // clears inDoc, start, and which (not the docPtrs)
+  void ResetStart ();
+  
+  // set the start of a level (closing off the last opening
+  // tag if needed)
+  void SetStart (unsigned long startPos,
+		 unsigned char startBit,
+		 unsigned char startWhich);
+
+  // if in a document, it will set the end, add the document to the
+  // list of document ptrs, and then call ResetStart
+  void SetEnd (unsigned long endPos,
+	       unsigned char endBit);
+};
+
+typedef map<UCArray, CompressTextInfo, LTUCArray> CompressTextInfoMap;
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.cpp	(revision 21112)
@@ -0,0 +1,50 @@
+/**************************************************************************
+ *
+ * FileBuf.cpp -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FileBuf.h"
+#include "sysfuncs.h"
+
+#ifdef __WIN32__
+# include <io.h>
+# define read _read
+#endif
+
+bool FileBuf::NextBuf () {
+  if (fd < 0) return false;
+  int numbytes = read (fd, buf, FILEBUFSIZE);
+  if (numbytes < 0) numbytes = 0;
+  
+  here = buf;
+  end = here + numbytes;
+
+  return (numbytes > 0);
+}
+
+FileBuf::FileBuf () {
+  here = end = buf;
+  fd = -1;
+}
+
+void FileBuf::SetFD (int _fd) {
+  here = end = buf;
+  fd = _fd;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FileBuf.h	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * FileBuf.h -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FILEBUF_H
+#define FILEBUF_H
+
+#define FILEBUFSIZE 65536
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+class FileBuf {
+protected:
+  unsigned char buf[FILEBUFSIZE];
+  unsigned char *here;
+  unsigned char *end;
+
+  int fd;
+
+  // gets a new buffer if need be, returns true if
+  // more could be got. if true is returned the caller
+  // can assume that here != end
+  bool NextBuf ();
+  
+public:
+  FileBuf ();
+
+  void SetFD (int _fd);
+
+  inline bool Get (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here++; return true; }
+    return false;
+  }
+  
+  inline bool Peek (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here; return true; }
+    return false;
+  }
+
+  inline bool Next () {
+    if (here != end || NextBuf()) { ++here; return true; }
+    return false;
+  }
+};
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.cpp -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FragLevelConvert.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+
+
+FragLevelConvert::FragLevelConvert () {
+  numFrags = 0;
+  numLevelDocs = 0;
+}
+
+bool FragLevelConvert::Read (FILE *invfFile,
+			     unsigned long levelPtr,
+			     unsigned long _numFrags,
+			     unsigned long _numLevelDocs) {
+  levelStarts.erase (levelStarts.begin(), levelStarts.end());
+  numFrags = _numFrags;
+  numLevelDocs = _numLevelDocs;
+
+  // seek to the appropriate place in the inverted file
+  fseek (invfFile, levelPtr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(invfFile);
+  
+  unsigned long pTag = numLevelDocs*2;
+  unsigned long B = BIO_Bblock_Init (numFrags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  for (i=0; i<numLevelDocs; ++i) {
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    levelStarts.push_back (fragNum);
+
+    // ignore end, assume every fragment is in a level
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+  }
+
+  // fragNum would be the starting fragment of the next document
+  // if there was one
+  levelStarts.push_back (fragNum);
+  
+  if (fragNum != _numFrags) {
+    cerr << "last frag num:"<<fragNum<<" != num frags:"<<_numFrags<<endl;
+  } 
+  buffer.done();
+
+  return true;
+}
+
+bool FragLevelConvert::FragToLevel (unsigned long fragNum,
+				    unsigned long &levelDocNum) const {
+  // do binary search for something containing this fragment number
+  unsigned long low = 1, high = numLevelDocs;
+  unsigned long mid;
+
+  while ((mid = (low+high)/2) >= 1 && low <= high) {
+    if (fragNum > levelStarts[mid]) low = mid+1;
+    else if (fragNum <= levelStarts[mid-1]) high = mid-1;
+    else {
+      levelDocNum = mid;
+      return true;
+    }
+  }
+  return false;
+}
+
+
+bool FragLevelConvert::LevelToLevel (FragLevelConvert sectionLevelConvert,
+		 unsigned long levelNum, unsigned long &DocNum) {
+  
+  if (levelNum==1) {
+    DocNum=1;
+    return true;
+  }
+  unsigned long levelfragnum = sectionLevelConvert.levelStarts[levelNum-1]+1;
+
+  if (FragToLevel(levelfragnum, DocNum)) {
+    return true;
+  } 
+  return false;
+  
+
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/FragLevelConvert.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.h -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FRAGLEVELCONVERT_H
+#define FRAGLEVELCONVERT_H
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+/*Notes about levelStarts (kjm18) - this is my interpretation of
+whats going on, feel free to correct this if its wrong.
+
+say you have 5 docs, consisting of fragments:
+1:  1-55
+2: 56-83
+3: 84-106
+4: 107-152
+5: 153-211
+
+then levelStarts would have the values
+0  1  2   3   4   5 
+0  55 83 106 152 211
+
+*/
+class FragLevelConvert {
+protected:
+  // note fragment start is one before before real start
+  vector<unsigned long> levelStarts;
+  unsigned long numFrags;
+  unsigned long numLevelDocs;
+  
+public:
+  FragLevelConvert ();
+
+  bool Read (FILE *invfFile, unsigned long levelPtr,
+	     unsigned long _numFrags, unsigned long _numLevelDocs);
+
+  bool FragToLevel (unsigned long fragNum, unsigned long &levelDocNum) const;
+
+  // this function converts from one level number to another
+  // the FragLevelConvert thats calling the method should be loaded with
+  // the level that you want to convert to, while the sectionLevelConvert
+  // should be loaded with the level converting from.
+  // level Num has the old level, DocNum will be returned with the new
+  // coresponding level
+  bool LevelToLevel (FragLevelConvert sectionLevelConvert,
+		   unsigned long levelNum, unsigned long &DocNum);
+};
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
@@ -0,0 +1,275 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size() + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //  UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParsePotentialInteger(UCArray::const_iterator &here,
+				  UCArray::const_iterator end,
+				  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen=0;
+  int length=0;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  // check the next character -if it is a letter, then have a term, not an integer
+  if (!is_unicode_letter(c)) {
+    // this was just an integer
+    return (!el.text.empty());
+  }
+  // else its a term
+  el.lexType = TermE;
+  el.num = 0;
+  /* this bit taken from ParseIndexWord in words.h*/
+  while (length+charLen<=MAXSTEMLEN && charLen > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  return (!el.text.empty());
+}
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+  const unsigned char* new_here = ParseIndexWord (&*here, endMinus1, text);
+  here += (new_here - &*here); // advance iterator by number of chars advanced
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+
+  } else if (c =='[') {
+    el.lexType = OpenSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c ==']') {
+    el.lexType = CloseSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '/') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '*') {
+    el.lexType = StarE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c=='&') {
+    el.lexType = AndOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '|') {
+    el.lexType = OrOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '!') {
+    el.lexType = NotOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ) {
+    return  ParseInteger (here, end, el);
+  }
+
+  else if (c >= '0' && c <= '9') {
+    return ParsePotentialInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text))  {
+    // parse term returns false if it hasn't parsed anything that is a term
+    // here should be the same as it was before 
+    el.lexType = UnknownE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  }
+  //return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+  UCArray NEAR; SetCStr (NEAR, "NEAR", 4);
+  if (PrefixLen(el.text, NEAR)==4) {
+    el.lexType = NearOpE;
+    return true;
+  }
+  UCArray WITHIN; SetCStr (WITHIN, "WITHIN", 6);
+  if (PrefixLen(el.text, WITHIN)==6) {
+    el.lexType = WithinOpE;
+    return true;
+  }
+  
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryLex.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.h -- Lexical analyser for a simple query language, for GSDL
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYLEX_H
+#define GSDLQUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, NearOpE, WithinOpE, QuoteE, IntegerE, TermWeightE, 
+	      StemMethodE, RangeE, AtE, TagE, OpenSquareBracketE, 
+	      CloseSquareBracketE, StarE, UnknownE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
@@ -0,0 +1,515 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryParser.h"
+#include "GSDLQueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end, 
+				   int defaultBoolCombine,
+				   int defaultStemMethod);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end, 
+					  int defaultBoolCombine,
+					  int defaultStemMethod) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end, defaultBoolCombine,
+					defaultStemMethod);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+// default is within 20 words
+static void SetRangeValues (TermNode &termNode,
+			    UCArray &nearby,
+			    bool reverse) {
+  UCArray NEARBY; SetCStr(NEARBY, "NEAR", 4);
+  UCArray WITHIN; SetCStr(WITHIN, "WITHIN", 6);
+  
+  if (nearby == NEARBY) { // no modifier
+    termNode.startRange = (NEAR_DEFAULT+1)*-1;
+    termNode.endRange = NEAR_DEFAULT;
+
+  } else if (nearby == WITHIN) { // no modifier
+    if (reverse) {
+      termNode.startRange = (NEAR_DEFAULT+1)*-1;
+      termNode.endRange = -1;
+    } else {
+      termNode.startRange = NEAR_DEFAULT;
+      termNode.endRange = 0;
+    }
+  }
+  else { // extract number
+    UCArray::const_iterator here;
+    bool within = false;
+    if (PrefixLen(nearby, WITHIN)==6) {
+      within=true;
+      here = nearby.begin()+6;
+    } else {
+      here = nearby.begin()+4;
+    }
+    UCArray::const_iterator end = nearby.end();
+    int size=0;
+    while (here != end) {
+      size = size*10 + (*here-'0');
+      ++here;
+    }
+    if (within) {
+      if (reverse) {
+	termNode.startRange = size;
+	termNode.endRange = 0;
+      } else {
+	termNode.startRange = -1 * (size+1);
+	termNode.endRange = -1;
+      }
+    } else {
+      termNode.startRange = -1 * (size+1);
+      termNode.endRange = size;
+    }
+  }
+}
+
+static unsigned long GetStemMethod(LexEl &el, int defaultStemMethod) {
+  // here expect el to contain some of c,s,i,u
+  // stem method 0 = c,u  00
+  // stem method 1 = i,u  01  - default for DL
+  // stem method 2 = c, s  10
+  // stem method 3 = i,s  11
+
+  unsigned long stem = (unsigned long)defaultStemMethod;
+      
+  UCArray::const_iterator here = el.text.begin();
+  UCArray::const_iterator end = el.text.end();
+
+  unsigned char c1 = *here; 
+  if (!(c1 == 'c'|| c1 == 'i' || c1 == 'u' || c1 == 's')) 
+    return 4; // incorrect format
+  
+  ++here;
+  unsigned char c2 = 'a';
+  if (here !=end) {
+    c2 = *here;
+    if (!(c2 == 'c'|| c2 == 'i' || c2 == 'u' || c2 == 's')) 
+      return 4; // incorrect format
+  }
+  
+  if (c1 == 'i'|| c2=='i') stem |= 1; // set bit 0 to 1
+  if (c1 == 'c' || c2 == 'c') stem &=0xe; //set bit 0 to 0
+  if (c1 == 's'|| c2 == 's') stem |= 2; // set bit 1 to 1
+  if (c1 == 'u' || c2 =='u') stem &=0xd; // set bit 1 to 0
+  return stem;
+}
+  
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode,
+				int defaultStemMethod) {
+ 
+  termNode.stemMethod = defaultStemMethod;
+  bool partial_match = false;
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      oldHere = here;
+      LexEl stem;
+      if (ParseLexEl (here, end, stem) && stem.lexType == TermE) {
+	termNode.stemMethod = GetStemMethod(stem, defaultStemMethod);
+	if (termNode.stemMethod == 4) { // error so backtrack
+	  here = oldHere;
+	  termNode.stemMethod = (unsigned long)defaultStemMethod;
+	} 
+      }else here = oldHere; //ignore - wrong syntax
+
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+    } else if (el.lexType == StarE) {
+      partial_match = true;
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    if (partial_match) {
+      termNode.stemMethod |= 4; // set bit 2 to 1
+      termNode.stemMethod &=0xd; // set bit 1 to 0 // we dont have stemming on if doing partial matching.
+    }
+    oldHere = here;
+  }  
+}
+
+static void ParseProxModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				ProxMatchQueryNode *proxNode) {
+  // so far only have one - the tag stuff
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TagE) {
+      oldHere = here; // don't backtrack past here
+      if (ParseLexEl (here, end, el) && el.lexType == TermE) {
+	proxNode->tagNodePtr = new TagNode;
+	proxNode->tagNodePtr->tagName = el.text;
+	
+      }
+      else { // error in tag
+	here = oldHere;
+      }
+    } // TagE
+    // add in other cases here
+    else {
+      // no modifiers
+      here = oldHere;
+      break;
+    }
+    oldHere = here;
+  }//while
+
+
+}
+
+// expects starting brackets to have been parsed
+// sets error to true if something has gone wrong
+static ProxMatchQueryNode *ParseSquareBrackets(UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				/*ProxMatchQueryNode *proxNode,*/
+				int defaultStemMethod,
+				bool & error) {
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+  LexEl el;
+  bool phrase=false;
+  bool first=true;
+  bool prox = false;
+  UCArray near_string;
+  while (ParseLexEl (here, end, el)) {
+    // cant have AND, OR, NOT in square brackets, so assume they are words
+    if (el.lexType == TermE || el.lexType == IntegerE || el.lexType == AndOpE || el.lexType == OrOpE || el.lexType == NotOpE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (phrase) {
+        if (first) first=false;
+        else {
+          termNode.startRange = -2;
+          termNode.endRange = -1;
+        }
+      } else if (prox) {
+	SetRangeValues(termNode, near_string, false);
+	prox = false;
+      }
+      proxNode->terms.push_back(termNode);
+    }
+    else if (el.lexType == CloseSquareBracketE) {
+      break;
+    }
+    else if (el.lexType == QuoteE) {
+      // phrase inside square brackets
+      if (phrase) { // end of phrase
+	phrase=false;
+	first = true;
+      } else {
+	phrase=true; // start of phrase
+      }
+    } else if (el.lexType == NearOpE || el.lexType == WithinOpE) {
+      if (phrase) {
+	// cant have proximity  op in a phrase - just assume its an actual word
+	TermNode termNode;
+	termNode.term = el.text;
+	ParseTermModifiers (here, end, termNode, defaultStemMethod);
+	proxNode->terms.push_back(termNode);
+      } else {
+	// its a NEAR or within op
+	prox = true;
+	near_string = el.text;
+      }
+	  
+    } 
+    else if (el.lexType == UnknownE) {
+      // just ignore it 
+    }
+    else {
+      //error - we set the proxNode to NULL,
+      cerr <<"GSDLQueryParser: bad syntax inside []\n";
+      error = true;
+      return NULL;
+    }
+  } // while
+  return proxNode;
+}
+// expects the starting quote to have been parsed
+// and discarded
+// now phrases use the case and stem preference options
+// ie can search for a phrase ignoring case
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode, 
+			 int defaultStemMethod,
+			 bool &error) {
+  LexEl el;
+  bool first = true;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      //termNode.stemMethod = defaultStemMethod;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (first) {
+	first = false;
+      }
+      else {
+	termNode.startRange = -2;
+	termNode.endRange = -1;
+      }
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it
+    } else {
+      // error 
+      error = true;
+      return;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end,
+			     int defaultBoolCombine,
+			     int defaultStemMethod) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end, defaultBoolCombine,
+				   defaultStemMethod);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode, defaultStemMethod);
+    oldHere = here;  // dont backtrack past here
+    if (ParseLexEl(here, end, el) && (el.lexType == NearOpE || el.lexType == WithinOpE )) {
+	delete proxNode;
+	oldHere = here;
+      // this is calling ParseTerm again, but only a subset of the things accepted by ParseTerm are appropriate here. add in some hacks to avoid segmentation faults - kjdon, 04/2003
+      
+      // if the next element is a '(' have a syntax error, return NULL 
+      LexEl temp_el;
+      if (ParseLexEl(here, end, temp_el) && temp_el.lexType == OpenBracketE) {
+	cerr << "GSDLQueryParser: NEAR/WITHIN cannot be followed by a '('\n";
+	return NULL;
+      }
+      here = oldHere; // else backtrack
+	
+      proxNode = (ProxMatchQueryNode *)ParseTerm(here, end, defaultBoolCombine,
+						 defaultStemMethod);
+      SetRangeValues(termNode, el.text, true);
+      proxNode->terms.push_back (termNode);
+      return proxNode;
+      
+    } else {
+      here = oldHere; // backtrack
+      proxNode->terms.push_back (termNode);
+      ParseProxModifiers(here, end, proxNode);
+      return proxNode;
+    }
+  } else if (el.lexType == QuoteE) {
+    bool error = false;
+    ParsePhrase (here, end, *proxNode, defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    return proxNode;
+  }
+  else if (el.lexType == OpenSquareBracketE) {
+    bool error = false;
+    proxNode = ParseSquareBrackets (here, end, /*proxNode, */defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    ParseProxModifiers (here, end, proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end,
+				   int defaultBoolCombine,
+				   int defaultStemMethod) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) {
+      // parsebracketexpression is waiting for the last bracket, so put it back
+      here = oldHere;
+      break;
+      
+    } else if (el.lexType == OpenSquareBracketE ||
+	       el.lexType == OpenBracketE ||
+	       el.lexType == TermE ||
+	       el.lexType == QuoteE ||
+	       el.lexType == IntegerE ) { 
+      
+      // some type of term, back track and parse it
+      here = oldHere;
+
+      //  parse the term
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine, 
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+
+      // if default==1, AND, else if==0, OR
+      if (defaultBoolCombine) {
+	curTree = AndAdd (curTree, newTerm);
+      }
+      else {
+	curTree = OrAdd (curTree, newTerm); 
+      }
+      
+    } else if (el.lexType == AndOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = AndAdd (curTree, newTerm);
+      
+    } else if (el.lexType == OrOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = OrAdd (curTree, newTerm);
+      
+    } else if (el.lexType == NotOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = NotAdd (curTree, newTerm); 
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it 
+    } else {
+
+      // syntax error, return NUll
+      delete curTree;
+      return NULL;
+    }
+
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end, defaultBoolCombine, defaultStemMethod);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/GSDLQueryParser.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYPARSER_H
+#define GSDLQUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+#define NEAR_DEFAULT 20
+
+// returns NULL if the query could not be parsed
+// defaultBoolCOmbine determines if individual terms are combined using
+// AND or OR in the absence of specified operators.
+// defaultStemMethod used to set stemming and casefolding for terms where
+// its not set explicitly in the query string. THe defaultStemMethod value
+// should be set using the values from the preferences page
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/IndexData.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/IndexData.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/IndexData.cpp	(revision 21112)
@@ -0,0 +1,207 @@
+/**************************************************************************
+ *
+ * IndexData.cpp -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "IndexData.h"
+#include "string.h"
+#include "mg_files.h"
+
+IndexData::IndexData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  dictFile = NULL;
+
+  stem1File = NULL;
+  stem2File = NULL;
+  stem3File = NULL;
+
+  invfFile = NULL;
+
+  approxWeightsFile = NULL;
+  exactWeightsFile = NULL;
+
+  curLevelNum = 0;
+}
+
+IndexData::~IndexData () {
+  UnloadData ();
+}
+
+bool IndexData::LoadData (const char *_basePath, const char *_filename) {
+  if (_filename[0] == '\0') return false;
+  
+  // make sure this data has not already been loaded
+  if (strcmp (_basePath, basePath) == 0 &&
+      strcmp (_filename, filename) == 0) return true;
+  
+  // make sure there is nothing else loaded
+  UnloadData ();
+
+  strcpy (basePath, _basePath);
+  strcpy (filename, _filename);
+
+  set_basepath (basePath);
+
+  // load in the level information
+  FILE *ivfLevelFile;
+  ivfLevelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			    MAGIC_INVF_LEVELS, MG_ABORT);
+  levels.Read (ivfLevelFile);
+  fclose (ivfLevelFile);
+
+  // blocked dictionary
+  dictFile = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "rb",
+			MAGIC_STEM, MG_ABORT);
+  if (!bdh.Read (dictFile)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.wblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biWords)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.tblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biTags)) { UnloadData (); return false; }
+
+  // blocked stem index 1
+  stem1File = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+			 "rb", MAGIC_STEM_1, MG_ABORT);
+  if (!sih1.Read (stem1File)) { UnloadData (); return false; }
+
+  fseek (stem1File, sih1.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem1File, sii1)) { UnloadData (); return false; }
+  
+  // blocked stem index 2
+  stem2File = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+			 "rb", MAGIC_STEM_2, MG_ABORT);
+  if (!sih2.Read (stem2File)) { UnloadData (); return false; }
+
+  fseek (stem2File, sih2.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem2File, sii2)) { UnloadData (); return false; }
+  
+  // blocked stem index 3
+  stem3File = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+			 "rb", MAGIC_STEM_3, MG_ABORT);
+  if (!sih3.Read (stem3File)) { UnloadData (); return false; }
+
+  fseek (stem3File, sih3.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem3File, sii3)) { UnloadData (); return false; }
+
+  // inverted file
+  invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  ifh.Read (invfFile);
+
+  // weights
+  approxWeightsFile = open_file (filename, APPROX_WEIGHTS_SUFFIX, "rb",
+				 MAGIC_WGHT_APPROX, MG_ABORT);
+  exactWeightsFile = open_file (filename, WEIGHTS_SUFFIX, "rb",
+				MAGIC_WGHT, MG_ABORT);
+
+  return true;
+}
+
+bool IndexData::UnloadData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  if (dictFile != NULL) {
+    fclose (dictFile); dictFile = NULL;
+  }
+
+  if (stem1File != NULL) {
+    fclose (stem1File); stem1File = NULL;
+  }
+  if (stem2File != NULL) {
+    fclose (stem2File); stem2File = NULL;
+  }
+  if (stem3File != NULL) {
+    fclose (stem3File); stem3File = NULL;
+  }
+
+  if (invfFile != NULL) {
+    fclose (invfFile); invfFile = NULL;
+  }
+
+  if (approxWeightsFile != NULL) {
+    fclose (approxWeightsFile); approxWeightsFile = NULL;
+  }
+  if (exactWeightsFile != NULL) {
+    fclose (exactWeightsFile); exactWeightsFile = NULL;
+  }
+
+  UnloadLevel();
+  
+  return true;
+}
+
+
+bool IndexData::LoadLevel (const UCArray &level) {
+  // make sure this level is not already loaded
+  if (level == curLevel) return true;
+
+  // unload any levels currently loaded
+  UnloadLevel();
+
+  // make sure the required files are open
+  if (dictFile == NULL || invfFile == NULL ||
+      approxWeightsFile == NULL || exactWeightsFile == NULL)
+    return false;
+  
+  // read in the information from the dictionary
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (dictFile, biTags, bdh.entries_per_tblk,
+			  bdh.tag_dict_size, level, tagEl, tagElNum))
+    return false;
+
+  // read in the level conversion information
+  if (!levelConverter.Read (invfFile, tagEl.invf_ptr,
+			    bdh.num_frags, tagEl.frag_occur))
+    return false;
+
+  // read in the approximate weights
+  if (!weightData.Read (approxWeightsFile,
+			levels.levelInfo[level].approxWeightsDiskPtr,
+			levels.levelInfo[level].numEntries))
+    return false;
+  
+  // find the level number
+  curLevelNum = 0;
+  IvfLevelInfoMap::const_iterator levelHere, levelEnd;
+  for (levelHere=levels.levelInfo.begin(), levelEnd=levels.levelInfo.end();
+       levelHere!=levelEnd && (*levelHere).first != level; ++levelHere)
+    ++curLevelNum;
+
+  // make sure we found the level
+  if (levelHere == levelEnd) return false;
+
+  curLevel = level;
+
+  return true;
+}
+
+bool IndexData::UnloadLevel () {
+  curLevel.erase (curLevel.begin(), curLevel.end());
+  curLevelNum = 0;
+  
+  weightData.Free ();
+    
+  return true;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/IndexData.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/IndexData.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/IndexData.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * IndexData.h -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef INDEXDATA_H
+#define INDEXDATA_H
+
+#include <stdio.h>
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+#include "Weights.h"
+
+
+class IndexData {
+public:
+  // general information
+  char basePath[FILENAME_MAX];
+  char filename[FILENAME_MAX];
+  FIvfLevel levels;
+  
+  // blocked dictionary
+  FILE *dictFile;
+  block_dict_header bdh;
+  block_idx biWords;
+  block_idx biTags;
+
+  // stem indexes
+  FILE *stem1File;
+  FILE *stem2File;
+  FILE *stem3File;
+  stem_idx_header sih1;
+  stem_idx_header sih2;
+  stem_idx_header sih3;
+  block_idx sii1;
+  block_idx sii2;
+  block_idx sii3;
+
+  // inverted file
+  FILE *invfFile;
+  invf_file_header ifh;
+
+  // weights
+  FILE *approxWeightsFile;
+  FILE *exactWeightsFile;
+
+  // level specific information
+  UCArray curLevel; // last level that was loaded
+  unsigned long curLevelNum;
+  FragLevelConvert levelConverter;
+  ApproxWeightsData weightData;
+  
+  IndexData ();
+  ~IndexData ();
+
+  // LoadData will only load the data if it has not been loaded before
+  bool LoadData (const char *_basePath, const char *_filename);
+  bool UnloadData ();
+
+  bool LoadLevel (const UCArray &level);
+  bool UnloadLevel ();
+};
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.cpp	(revision 21112)
@@ -0,0 +1,760 @@
+/**************************************************************************
+ *
+ * MGQuery.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "Terms.h"
+#include "QueryResultsSort.h"
+#include <assert.h>
+
+
+void PrintIndent (ostream &s, int indent) {
+  while (indent-- > 0) s << " ";
+}
+
+
+void PrintIndentText (ostream &s, char *text, int indent) {
+  PrintIndent (s, indent);
+  s << text;
+}
+
+void PrintNode (ostream &s, QueryNode *node, int indent) {
+  if (node == NULL) {
+    PrintIndentText (s, "NULL\n", indent);
+  } else {
+    node->Print (s, indent+2);
+  }
+}
+
+  
+
+QueryNode::QueryNode () {
+}
+
+QueryNode::~QueryNode () {
+}
+
+void QueryNode::Calculate (IndexData &/*indexData*/,
+			   const QueryInfo &/*queryInfo*/,
+			   QueryResult &result) const {
+  result.Clear();
+}
+
+void QueryNode::Free () {
+}
+
+void QueryNode::Print (ostream &/*s*/, int /*indent*/) const {
+}
+
+
+
+AndQueryNode::AndQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+AndQueryNode::~AndQueryNode () {
+  Free ();
+}
+
+void AndQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // an And between nothing and something is nothing...
+  if (leftNode == NULL || rightNode == NULL) return;
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult rightResult;
+  leftNode->Calculate (indexData, queryInfo, result);
+  rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with an And
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long outI = 0;
+  while (leftI < result.docs.size() &&
+	 rightI < rightResult.docs.size()) {
+    if (result.docs[leftI] < rightResult.docs[rightI]) {
+      ++leftI;
+    } else if (result.docs[leftI] > rightResult.docs[rightI]) {
+      ++rightI;
+    } else {
+      // the documents are equal
+      result.docs[outI] = result.docs[leftI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[leftI] + rightResult.ranks[rightI];
+      ++leftI;
+      ++rightI;
+      ++outI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+    
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+}
+
+void AndQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void AndQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "AND\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+OrQueryNode::OrQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+OrQueryNode::~OrQueryNode () {
+  Free ();
+}
+
+void OrQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult leftResult;
+  QueryResult rightResult;
+  if (leftNode != NULL)
+    leftNode->Calculate (indexData, queryInfo, leftResult);
+  if (rightNode != NULL)
+    rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results
+  
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (leftResult.ranks.size() != leftResult.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftSize = leftResult.docs.size();
+  unsigned long rightSize = rightResult.docs.size();
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long leftDocNum = 0;
+  unsigned long rightDocNum = 0;
+  while (leftI < leftSize || rightI < rightSize) {
+    // check leftI
+    if (leftI < leftResult.docs.size())
+      leftDocNum = leftResult.docs[leftI];
+    else leftDocNum = ULONG_MAX;
+    
+    // check rightI
+    if (rightI < rightResult.docs.size())
+      rightDocNum = rightResult.docs[rightI];
+    else rightDocNum = ULONG_MAX;
+    
+    // combine
+    if (leftDocNum < rightDocNum) {
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI]);
+      ++leftI;
+      
+    } else if (leftDocNum > rightDocNum) {
+      result.docs.push_back (rightDocNum);
+      if (haveAccum)
+	result.ranks.push_back (rightResult.ranks[rightI]);
+      ++rightI;
+      
+    } else { // equal
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI] +
+				rightResult.ranks[rightI]);
+      ++leftI;
+      ++rightI;
+    }
+  }
+
+  // combine term frequency information
+  if (queryInfo.needTermFreqs) {
+    result.termFreqs.insert (result.termFreqs.end(),
+			     leftResult.termFreqs.begin(),
+			     leftResult.termFreqs.end());
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+  }
+}
+
+void OrQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void OrQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "OR\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+
+NotQueryNode::NotQueryNode () {
+  queryNode = NULL;
+  notNode = NULL;
+}
+
+NotQueryNode::~NotQueryNode () {
+  Free ();
+}
+
+void NotQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // check for nothing
+  if (queryNode == NULL) return;
+  if (notNode == NULL) {
+    queryNode->Calculate (indexData, queryInfo, result);
+    return;
+  }
+  
+  // calculate the result from the query tree and the result
+  // from the not tree
+  QueryResult notResult;
+  queryNode->Calculate (indexData, queryInfo, result);
+  notNode->Calculate (indexData, queryInfo, notResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with a Not
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    notResult.ranks.size() != notResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long queryI = 0;
+  unsigned long notI = 0;
+  unsigned long outI = 0;
+  while (queryI < result.docs.size() &&
+	 notI < notResult.docs.size()) {
+    if (result.docs[queryI] < notResult.docs[notI]) {
+      // found a document not in the notResults
+      result.docs[outI] = result.docs[queryI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[queryI];
+      ++queryI;
+      ++outI;
+    } else if (result.docs[queryI] > notResult.docs[notI]) {
+      ++notI;
+    } else {
+      // the documents are equal, ignore both
+      ++queryI;
+      ++notI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+  
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     notResult.termFreqs.begin(),
+			     notResult.termFreqs.end());
+}
+
+void NotQueryNode::Free () {
+  if (queryNode != NULL) {
+    delete queryNode;
+    queryNode = NULL;
+  }
+  if (notNode != NULL) {
+    delete notNode;
+    notNode = NULL;
+  }
+}
+
+void NotQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "queryNode:\n", indent);
+  PrintNode (s, queryNode, indent+2);
+  PrintIndentText (s, "NOT\n", indent);
+  PrintIndentText (s, "notNode:\n", indent);
+  PrintNode (s, notNode, indent+2);
+}
+
+
+void TagNode::Clear () {
+  UCArrayClear (tagName);
+}
+
+void TagNode::Calculate (IndexData &indexData,
+			 FragRangeArray &fragRange) const {
+  fragRange.erase (fragRange.begin(), fragRange.end());
+  if (tagName.empty()) return;
+
+  // get information about this tag
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (indexData.dictFile, indexData.biTags,
+			  indexData.bdh.entries_per_tblk,
+			  indexData.bdh.tag_dict_size,
+			  tagName, tagEl, tagElNum))
+    return;
+
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, tagEl.invf_ptr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(indexData.invfFile);
+  
+  unsigned long pTag = tagEl.frag_occur*2;
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  FragRange thisFrag;
+  for (i=0; i<tagEl.frag_occur; ++i) {
+    // get start
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeStart = fragNum;
+
+    // get end
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeEnd = fragNum;
+    fragRange.push_back (thisFrag);
+  }
+
+  buffer.done();
+}
+
+void TagNode::Free () {
+  Clear ();
+}
+
+void TagNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TAG: \"" << tagName << "\"\n";
+}
+
+
+void TermNode::Clear () {
+  UCArrayClear (term);
+  termWeight = 1;
+  stemMethod = 0;
+  startRange = NO_TERM_RANGE_START;
+  endRange = NO_TERM_RANGE_END;
+}
+
+TermNode::TermNode () {
+  Clear ();
+}
+
+void TermNode::Calculate (IndexData &indexData,
+			  bool needFragFreqs,
+			  FragRangeArray *fragLimits,
+			  FragData &fragData,
+			  UCArrayVector &equivTerms) const {
+  fragData.Clear ();
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+
+  // get a list of term numbers
+  vector<unsigned long> equivNums;
+  FindWordNumbers (indexData, term, stemMethod, equivNums);
+
+  // get the information for each word and merge it with
+  // previous results
+  FragData tempFragData1;
+  FragData tempFragData2;
+  UCArray equivWord;
+  vector<unsigned long>::iterator here = equivNums.begin();
+  vector<unsigned long>::iterator end = equivNums.end();
+  while (here != end) {
+    // get the information for this word
+    ReadTermFragData (indexData, needFragFreqs, *here,
+		      tempFragData1, fragLimits, equivWord);
+    equivTerms.push_back(equivWord);
+    // combine with last results
+    tempFragData2 = fragData;
+    CombineFragData (needFragFreqs, tempFragData1, tempFragData2, fragData);
+    
+    ++here;
+  }
+}
+
+void TermNode::Free () {
+  Clear ();
+}
+
+void TermNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TERM: \"" << term << "\"\n";
+  PrintIndent (s, indent+2);
+  s << "termWeight: " << termWeight << "\n";
+  PrintIndent (s, indent+2);
+  s << "stemMethod: " << stemMethod << "\n";
+  PrintIndent (s, indent+2);
+  s << "startRange: " << startRange << "\n";
+  PrintIndent (s, indent+2);
+  s << "endRange: " << endRange << "\n";
+}
+
+
+ProxMatchQueryNode::ProxMatchQueryNode () {
+  tagNodePtr = NULL;
+}
+
+ProxMatchQueryNode::~ProxMatchQueryNode () {
+  Free ();
+}
+
+void ProxMatchQueryNode::Calculate (IndexData &indexData,
+				    const QueryInfo &queryInfo,
+				    QueryResult &result) const {
+  result.Clear ();
+
+  bool needFragFreqs = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  // read in the tag if needed
+  FragRangeArray fragLimits;
+  FragRangeArray *fragLimitsPtr = NULL;
+  if (tagNodePtr == NULL && terms.size() > 1) {
+    // multiple terms must be compared relative to some tag
+    // otherwise phrase matches could span documents
+    TagNode tempTagNode;
+    tempTagNode.tagName = indexData.curLevel;
+    tempTagNode.Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+    
+  } else if (tagNodePtr != NULL) {
+    (*tagNodePtr).Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+  }
+
+  UCArray tagOrLevel = indexData.curLevel;
+  if (tagNodePtr != NULL) tagOrLevel = (*tagNodePtr).tagName;
+
+  // read in the first term
+  FragData termData;
+  UCArrayVector equivTerms;
+  TermNodeArray::const_iterator termHere=terms.begin(), termEnd = terms.end();
+  if (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs, fragLimitsPtr, 
+			   termData, equivTerms);
+
+    // convert initial fragment information
+    FragsToQueryResult (indexData,
+			queryInfo,
+			termData,
+			tagOrLevel,
+			(*termHere).term,
+			(*termHere).stemMethod,
+			(*termHere).termWeight,
+			equivTerms,
+			result);
+  
+    ++termHere;
+
+    if (termHere == termEnd) return; // nothing more to do
+  }
+
+  // read and combine the rest of the terms
+  FragData comTermData;
+  while (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs,
+			   fragLimitsPtr, comTermData, equivTerms);
+
+    AndFragsToQueryResult (indexData,
+			   queryInfo,
+			   comTermData,
+			   tagOrLevel,
+			   (*termHere).term,
+			   (*termHere).stemMethod,
+			   (*termHere).termWeight,
+			   equivTerms,
+			   result);
+    
+    AndCombineFragData (needFragFreqs, termData, comTermData,
+			(*termHere).startRange,
+			(*termHere).endRange,
+			fragLimitsPtr);
+    ++termHere;
+  }
+
+  // remove unwanted document numbers
+  RemoveUnwantedResults (indexData,
+			 queryInfo,
+			 termData,
+			 result);
+}
+
+void ProxMatchQueryNode::Free () {
+  if (tagNodePtr != NULL) {
+    delete tagNodePtr;
+    tagNodePtr = NULL;
+  }
+  terms.erase (terms.begin(), terms.end());
+}
+
+void ProxMatchQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "PROXMATCH\n", indent);
+  if (tagNodePtr != NULL) tagNodePtr->Print (s, indent+2);
+  
+  TermNodeArray::const_iterator here = terms.begin();
+  TermNodeArray::const_iterator end = terms.end();
+  while (here != end) {
+    (*here).Print (s, indent+2);
+    ++here;
+  }
+}
+
+
+void BrowseQueryNode::Clear () {
+  UCArrayClear(term);
+}
+
+void BrowseQueryNode::Calculate (IndexData &indexData, BrowseQueryResult &result) const {
+
+  unsigned long number=0;
+  FindNearestWordNumber(indexData, term, number);
+  if (number + startPosition > 0 ) {
+    number = number+startPosition;
+  }
+  else {
+    number = 1;
+  }
+
+  GetTermList (indexData, number, numTerms, result.termFreqs);
+
+}
+
+
+
+void BrowseQueryNode::Free () {
+  Clear();
+}
+
+
+void BrowseQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText(s, "BROWSEQUERYNODE\n", indent);
+  PrintIndent (s, indent+2);
+  s << "TERM:"<<term<<"\n";
+  PrintIndent (s, indent+2);
+  s << "Start position: "<< startPosition<<", Num terms: "<< numTerms<<"\n";
+
+
+}
+
+
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result) {
+  result.Clear ();
+
+  // make sure level is current
+  if (!indexData.LoadLevel (queryInfo.docLevel)) {
+    return;
+  }
+  // do query
+  if (queryTree == NULL) return;
+  
+  (*queryTree).Calculate (indexData, queryInfo, result);
+
+  // make weights into ranks if needed
+  unsigned long i;
+  if (queryInfo.sortByRank || queryInfo.needRankInfo) {
+    for (i=0; i<result.ranks.size(); ++i) {
+      result.ranks[i] /=
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]);
+    }
+  }
+  
+  unsigned long resultsSize = queryInfo.maxDocs;
+  if (resultsSize == 0 || resultsSize > result.docs.size())
+    resultsSize = result.docs.size();
+
+  result.actualNumDocs = result.docs.size(); // the total number of docs 
+                                   //returned, before pruning based on maxDocs
+
+  // sort results by rank if needed
+  GTRank gtRank;
+  if (queryInfo.sortByRank) {
+    // need in ascending order for SelectAddHeap
+    MakeHeap (result.docs.begin(), result.ranks.begin(),
+	      resultsSize, gtRank);
+    SelectAddHeap (result.docs.begin(), result.ranks.begin(), resultsSize,
+		   result.docs.begin()+resultsSize,
+		   result.ranks.begin()+resultsSize,
+		   (unsigned long) result.ranks.size()-resultsSize,
+		   gtRank);
+
+    // sort into descending order
+    SortHeap (result.docs.begin(), result.ranks.begin(), resultsSize, gtRank);
+  }
+  
+  // get exact weights if needed
+  if (queryInfo.exactWeights &&
+      (queryInfo.sortByRank || queryInfo.needRankInfo)) {
+    unsigned long exactDiskPtr =
+      indexData.levels.levelInfo[indexData.curLevel].exactWeightsDiskPtr;
+    
+    for (i=0; i<resultsSize; ++i) {
+      result.ranks[i] =  result.ranks[i] *
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]) /
+	GetExactDocWeight (indexData.exactWeightsFile, exactDiskPtr,
+			   result.docs[i]);
+    }
+
+    // re-sort top candidates based on exact weights
+    if (queryInfo.sortByRank) {
+      MakeHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+      SortHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+    }
+  }
+
+  // get rid of unwanted ranking results
+  if (result.ranks.empty()) {
+    // do nothing
+  } else if (!queryInfo.needRankInfo) {
+    result.ranks.erase(result.ranks.begin(), result.ranks.end());
+  } else {
+    result.ranks.erase(result.ranks.begin()+resultsSize, result.ranks.end());
+  }
+
+  // remove extra docs that are unwanted
+  result.docs.erase(result.docs.begin()+resultsSize, result.docs.end());
+}
+
+// new MGQuery to retrieve doc and section nums
+// this will return doc nums for the level queried at (set in queryInfo)
+// in QueryResult.docs and if a second level is specified, 
+// it will return corresponding docnums for that level in QueryResult.levels
+// If there is no level specified, or that level is invalid, the query
+// level is used
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &realresult, UCArray &level) {
+  realresult.Clear ();
+  QueryResult result; // temp result
+
+  // do the normal query
+  MGQuery (indexData, queryInfo, queryTree, result);
+
+  // now that have the final result stuff, convert to ExtQueryResult,
+  // add in level nums if needed
+
+  realresult.docs = result.docs;
+  realresult.ranks = result.ranks;
+  realresult.termFreqs = result.termFreqs;
+  realresult.actualNumDocs = result.actualNumDocs;
+
+  if (queryInfo.docLevel == level || level.empty()) {
+    realresult.levels = result.docs;
+    return;
+  }
+
+  // else need to convert from queryInfo.docLevel to level
+
+  // the original level info
+  FragLevelConvert sectionLevelConverter = indexData.levelConverter;
+
+  // the new level info
+  if (!indexData.LoadLevel(level)) {
+    realresult.levels = result.docs;
+    return;
+  }
+  
+  unsigned long DocNum = 0;
+  
+  for (unsigned long i=0; i<realresult.docs.size(); ++i) {
+
+    // do an if ! here????
+    indexData.levelConverter.LevelToLevel(sectionLevelConverter, realresult.docs[i], DocNum);
+    realresult.levels.push_back(DocNum);
+  }
+  
+}
+
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result) {
+
+  indexData.LoadLevel(level);
+  node.Calculate(indexData, result); 
+
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/MGQuery.h	(revision 21112)
@@ -0,0 +1,197 @@
+/**************************************************************************
+ *
+ * MGQuery.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MGQUERY_H
+#define MGQUERY_H
+
+#include "IndexData.h"
+#include <limits.h>
+#include "Terms.h"
+
+// forms base for defining and calculating the boolean
+// part of a query
+class QueryNode {
+public:
+  QueryNode ();
+  virtual ~QueryNode ();
+
+  virtual void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			  QueryResult &result) const;
+  virtual void Free ();
+  virtual void Print (ostream &s, int indent=0) const;
+};
+
+
+
+void PrintIndent (ostream &s, int indent);
+void PrintIndentText (ostream &s, char *text, int indent);
+void PrintNode (ostream &s, QueryNode *node, int indent=0);
+
+
+
+// basic boolean nodes
+
+class AndQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  AndQueryNode ();
+  ~AndQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class OrQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  OrQueryNode ();
+  ~OrQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class NotQueryNode : public QueryNode {
+public:
+  QueryNode *queryNode;
+  QueryNode *notNode;
+  
+  NotQueryNode ();
+  ~NotQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+
+class TagNode {
+public:
+  UCArray tagName;
+
+  void Clear ();
+  TagNode () { Clear (); }
+
+  void Calculate (IndexData &indexData, FragRangeArray &fragRange) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+#define NO_TERM_RANGE_START (LONG_MIN/2)
+#define NO_TERM_RANGE_END   (LONG_MAX/2)
+
+/* NOTE: range stuff - the range values are for the previous term relative
+to the current term. So if searching for the phrase "the cat", 'the' doesn't 
+need range limits, but 'cat' has a range of -2 to -1. ie, if have found 'cat'
+then 'the' has to be at position between -2 and -1 relative to 'cat'.
+"the cat" could also be searched for by 'cat' with no range limits, then 'the' with range 0 to 1.
+range values are relative to the gaps between words:
+   x   y   z   X   a   b   c
+    -3   -2  -1  0   1   2   3
+
+ */
+class TermNode {
+public:
+  UCArray term;
+  unsigned long termWeight;
+  unsigned long stemMethod;
+  signed long startRange; // range relative to last term
+  signed long endRange;
+  
+  void Clear ();
+  TermNode ();
+
+  // fragLimits can be NULL if not neede
+  void Calculate (IndexData &indexData,
+		  bool needFragFreqs,
+		  FragRangeArray *fragLimits,
+		  FragData &fragData,
+		  UCArrayVector &equivTerms) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+typedef vector<TermNode> TermNodeArray;
+
+
+class ProxMatchQueryNode : public QueryNode {
+public:
+  TagNode *tagNodePtr; // optional field
+  TermNodeArray terms;
+  
+  ProxMatchQueryNode ();
+  ~ProxMatchQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class BrowseQueryNode :public QueryNode {
+ public:
+  UCArray term;
+  signed long startPosition;
+  unsigned long numTerms;
+
+  void Clear();
+  BrowseQueryNode () { Clear(); }
+  //  ~BrowseQueryNode ();
+
+  void Calculate (IndexData &indexData, BrowseQueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+
+
+
+
+};
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result);
+
+// this function for retrieving results with both section doc nums
+// and Document docnums
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &result, UCArray &level);
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Makefile.in	(revision 21112)
@@ -0,0 +1,316 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../lib/libmg.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+##########################################
+# George Buchanan - August 1999
+# 
+# compiling with make NZDL=1 to ensure that 
+# the NZDL compile mode is used - this has been changed to the default - kjm18
+#
+##########################################
+GSDLOS = @gsdlos@
+
+bindir = $(top_srcdir)/bin/$(GSDLOS)
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CXXFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) $<
+
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+# want toplevel config.h, not local mg one
+INCLUDES = -I../../.. -I../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgpp_passes mgpp_decompress_text mgpp_perf_hash_build \
+                mgpp_invf_dump mgpp_weights_build mgpp_invf_dict mgpp_stem_idx \
+                mgpp_invf_dict_dump mgpp_compression_dict mgpp_fast_comp_dict \
+                QueryTester Queryer
+
+MAN           =  mgpp_weights_build.1  mgpp_invf_dict.1 \
+                 mgpp_passes.1 mgpp_perf_hash_build.1 \
+                 mgpp_compression_dict.1  mgpp_fast_comp_dict.1 \
+                 mgpp_stem_idx.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CXXFLAGS) $(CFLAGS) -- $(SOURCES)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall
+uninstall: execuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+#mg_errors$o removed from LIB_OBJS to avoid conflict with mg's libtextin.a
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+
+clean: mostlyclean
+	rm -f $(EXEC) libtextin.a
+
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+movclean:
+	for file in $(SOURCES); do \
+          test = $$file; \
+          echo $("test.c"##*.c); \
+        done 
+
+# jrm21
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.cpp	(revision 21112)
@@ -0,0 +1,175 @@
+/**************************************************************************
+ *
+ * QueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  // read in number part
+  el.num = 0;
+  el.lexType = IntegerE;
+  while (c >= '0' && c <= '9') {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  UCArray::const_iterator endMinus1 = end-1;
+  here = ParseIndexWord (here, endMinus1, text);
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+  
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '$') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ||
+	     (c >= '0' && c <= '9')) {
+    return ParseInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text)) return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {  
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryLex.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * QueryLex.h -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYLEX_H
+#define QUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, QuoteE, IntegerE, TermWeightE, StemMethodE,
+	      RangeE, AtE, TagE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.cpp	(revision 21112)
@@ -0,0 +1,219 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryParser.h"
+#include "QueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      termNode.stemMethod = ParseInt (here, end);
+      
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+      
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    oldHere = here;
+  }  
+}
+
+// expects the starting quote to have been parsed
+// and discarded
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode) {
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode);
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else {
+      // error
+      break;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  // check for a tag
+  if (el.lexType == TagE) {
+    oldHere = here; // don't backtrack past here
+    if (!ParseLexEl (here, end, el)) return NULL;
+    if (el.lexType == TermE) {
+      proxNode->tagNodePtr = new TagNode;
+      proxNode->tagNodePtr->tagName = el.text;
+      if (!ParseLexEl (here, end, el)) return NULL;
+    }
+  }
+  
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode);
+    proxNode->terms.push_back (termNode);
+    return proxNode;
+    
+  } else if (el.lexType == QuoteE) {
+    ParsePhrase (here, end, *proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == OpenBracketE ||
+	el.lexType == TermE ||
+	el.lexType == QuoteE ||
+	el.lexType == IntegerE ||
+	el.lexType == TagE) {
+      // some type of term, back track and parse it
+      here = oldHere;
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == AndOpE) {
+      curTree = AndAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == OrOpE) {
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == NotOpE) {
+      curTree = NotAdd (curTree, ParseTerm (here, end));
+      
+    } else break;
+    
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryParser.h	(revision 21112)
@@ -0,0 +1,31 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYPARSER_H
+#define QUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+// returns NULL if the query could not be parsed
+QueryNode *ParseQuery (const UCArray &queryStr);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryResultsSort.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryResultsSort.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryResultsSort.h	(revision 21112)
@@ -0,0 +1,140 @@
+/**************************************************************************
+ *
+ * QueryResultsSort.h -- Functions to sort query results and parallel data
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYRESULTSSORT_H
+#define QUERYRESULTSSORT_H
+
+#include "Terms.h"
+
+
+struct LTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 < f2); }
+};
+
+struct GTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 > f2); }
+};
+
+
+
+// assumes the rank vector is the same size as the document
+// number vector. assumes the value to be pushed is in
+// docNumHeap + heapSize - 1 and rankHeap + heapSize - 1
+// topIndex can be set
+template <class RandomAccessIterator, class Compare>
+void PushHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long heapSize,
+	       const Compare &comp) {
+  unsigned long holeIndex = heapSize-1;
+  unsigned long parent = (holeIndex - 1) / 2;
+  while (holeIndex > 0 && comp (*(parallelHeap+parent),
+				*(parallelHeap+holeIndex))) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+parent));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+parent));
+    holeIndex = parent;
+    parent = (holeIndex - 1) / 2;
+  }
+}
+
+// AdjustHeap is used to add a element to a heap after one
+// element in the heap has been replaced with a new one
+template <class RandomAccessIterator, class Compare>
+void AdjustHeap (DocNumArray::iterator docNumHeap,
+		     RandomAccessIterator parallelHeap,
+		     unsigned long holeIndex,
+		     unsigned long heapSize,
+		     const Compare &comp) {
+  unsigned long secondChild = 2 * holeIndex + 2;
+  while (secondChild < heapSize) {
+    if (comp(*(parallelHeap + secondChild),
+	     *(parallelHeap + (secondChild - 1))))
+      --secondChild;
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+secondChild));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+secondChild));
+    holeIndex = secondChild;
+    secondChild = 2 * secondChild + 2;
+  }
+  if (secondChild == heapSize) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+(secondChild-1)));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+(secondChild-1)));
+    holeIndex = secondChild - 1;
+  }
+  PushHeap (docNumHeap, parallelHeap, holeIndex+1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void PopHeap (DocNumArray::iterator docNumHeap,
+	      RandomAccessIterator parallelHeap,
+	      unsigned long heapSize,
+	      const Compare &comp) {
+  swap (*(parallelHeap), *(parallelHeap+heapSize-1));
+  swap (*(docNumHeap), *(docNumHeap+heapSize-1));
+  AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize-1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void MakeHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+  
+  unsigned long i = 0;
+  for (i=1; i<=size; ++i)
+    PushHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+ 
+template <class RandomAccessIterator, class Compare>
+void SortHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+
+  unsigned long i;
+  for (i=size; i>1; --i)
+    PopHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+// uses the heap to get the top "size" elements
+template <class RandomAccessIterator, class Compare>
+void SelectAddHeap (DocNumArray::iterator docNumHeap,
+		    RandomAccessIterator parallelHeap,
+		    unsigned long heapSize,
+		    DocNumArray::iterator docNumAdd,
+		    RandomAccessIterator parallelAdd,
+		    unsigned long addSize,
+		    const Compare &comp) {
+  unsigned long i;
+  for (i=0; i<addSize; ++i) {
+    if (comp(*(parallelAdd+i), *(parallelHeap))) {
+      swap (*(docNumHeap), *(docNumAdd+i));
+      swap (*(parallelHeap), *(parallelAdd+i));
+      AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize, comp);
+    }
+  }
+}
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/QueryTester.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/QueryTester.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/QueryTester.cpp	(revision 21112)
@@ -0,0 +1,447 @@
+/**************************************************************************
+ *
+ * QueryTester.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+
+
+class SetQueryNode : public QueryNode {
+public:
+  QueryResult queryResult;
+  
+  SetQueryNode () {}
+  ~SetQueryNode () {}
+
+  void Calculate (IndexData &/*indexData*/, const QueryInfo &queryInfo,
+		  QueryResult &result) const {
+    result = queryResult;
+    if (!queryInfo.sortByRank && !queryInfo.needRankInfo)
+      result.ranks.erase (result.ranks.begin(), result.ranks.end());
+    if (!queryInfo.needTermFreqs)
+      result.termFreqs.erase (result.termFreqs.begin(),
+			      result.termFreqs.end());
+  }
+  void Free () { queryResult.Clear(); }
+};
+
+
+bool Test1 () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (15);  rcRankSet.push_back (0.2+0.6);
+  rcDocSet.push_back (19);  rcRankSet.push_back (1.4);
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test1b () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (15); 
+  rcDocSet.push_back (19); 
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2 () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (2);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (11); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (12); rcRankSet.push_back (0.3f);
+  rcDocSet.push_back (13); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (14); rcRankSet.push_back (0.5f);
+  rcDocSet.push_back (15); rcRankSet.push_back (0.2f+0.6f);
+  rcDocSet.push_back (16); rcRankSet.push_back (0.7f);
+  rcDocSet.push_back (17); rcRankSet.push_back (0.8f);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (19); rcRankSet.push_back (0.9f+0.5f);
+  rcDocSet.push_back (20); rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (21); rcRankSet.push_back (0.2f);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2b () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2);
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1);
+  rcDocSet.push_back (2); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (11);
+  rcDocSet.push_back (12);
+  rcDocSet.push_back (13);
+  rcDocSet.push_back (14);
+  rcDocSet.push_back (15);
+  rcDocSet.push_back (16);
+  rcDocSet.push_back (17);
+  rcDocSet.push_back (18);
+  rcDocSet.push_back (19);
+  rcDocSet.push_back (20);
+  rcDocSet.push_back (21);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3 () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3b () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (18);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+
+int main () {
+  if (!Test1 () ||
+      !Test2 () ||
+      !Test2b () ||
+      !Test3 () ||
+      !Test3b ()) return 1;
+  
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/Queryer.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Queryer.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Queryer.cpp	(revision 21112)
@@ -0,0 +1,271 @@
+/**************************************************************************
+ *
+ * Queryer.cpp -- simple interactive query program
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "MGQuery.h"
+#include "TextGet.h"
+
+#include "messages.h"
+#include "mg_files.h"
+
+#include "GSDLQueryParser.h"
+
+void printHelp() {
+
+  cout << "commands available are:\n"
+       << "\t.q\t\tquit\n"
+       << "\t.h\t\tprint the help message\n"
+       << "\t.i\t\tchange the search level (enter the new level at the prompt)\n"
+       << "\t.l\t\tchange the result level ( \"\"   \"\"  )\n"
+       << "\t.b\t\tfull text browse (enter a word or fragment at the prompt)\n"
+       << "\t.r0/.r1\t\tranking off/on\n"
+       << "\t.t0/.t1\t\tquery type some/all\n"
+       << "\t.c0/.c1\t\tcasefolding off/on\n"
+       << "\t.s0/.s1\t\tstemming off/on\n"
+       << "\t.o0/.o1\t\tshort output off/on\n\n"
+       << "\t.p\t\tprint a document (enter the docnum at the prompt)\n"
+       << "otherwise just enter a query\n\n";
+
+}
+
+int main (int argc, char **argv) {
+  int ch;
+  char *textfilename = "";
+  char *indexfilename = "";
+  char *basePath = "";
+  
+  opterr = 0;
+  msg_prefix = argv[0];
+  
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:t:d:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      indexfilename = optarg;
+      break;
+    case 't':
+      textfilename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (basePath);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-d directory] -f indexname -t textname\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  if (textfilename[0] == '\0' || indexfilename[0] == '\0') {
+    FatalError (1, "Index and text file names must be specified with -f and -t \n");
+  }
+  
+  // init the text system
+    TextData textData;
+  if (!textData.LoadData (basePath, textfilename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", textfilename);
+  }
+  
+  // init the query system
+  IndexData indexData;
+  if (!indexData.LoadData (basePath, indexfilename)) {
+    FatalError (1, "Couldn't load index information for \"%s\"", indexfilename);
+  }
+
+  // debug output
+  cerr << "num docs: "<<indexData.bdh.num_docs       
+       << "\nnum frags: "<<indexData.bdh.num_frags
+       << "\nnum words: "<<indexData.bdh.num_words
+       << "\ntotal bytes: "<<indexData.bdh.total_bytes
+       << "\nindex string bytes: "<<indexData.bdh.index_string_bytes
+       << "\nnum levels: "<<indexData.bdh.num_levels<<endl;
+  
+  // do querying
+  QueryInfo queryInfo;
+  SetCStr (queryInfo.docLevel, "Document", 8);
+  queryInfo.maxDocs = 50;
+  queryInfo.sortByRank = true;
+  queryInfo.exactWeights = false;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  ExtQueryResult queryResult;
+  char query[2048];
+  UCArray queryArray;
+  QueryNode *queryTree = NULL;
+
+
+  UCArray docLevel;
+  SetCStr(docLevel, "Document", 8);
+
+  UCArray level;
+  UCArrayClear(level);
+  //SetCStr(level, "");
+   
+  int defaultStemMethod = 0; // uncasefolded, unstemmed
+  int defaultBoolCombine = 0; // OR
+  bool shortOutput = false;
+  BrowseQueryNode browseNode;
+  browseNode.startPosition = -10;
+  browseNode.numTerms = 40;
+
+  BrowseQueryResult browseResult;
+
+  while (true) {
+    cout << "> ";
+    cin.getline(query, 2048, '\n');
+    SetCStr (queryArray, query, strlen(query));
+
+    // check for commands
+    if (queryArray.size() >= 2 && queryArray[0] == '.') {
+      if (queryArray[1] == 'q') break;  // quit
+
+      if (queryArray[1] == 'h') { // help
+	printHelp();
+      } else if (queryArray[1] == 'i') {
+	cout << "current index="<< queryInfo.docLevel << "\nchange to index:";
+	cin >> query;
+	UCArrayClear(queryInfo.docLevel);
+	SetCStr(queryInfo.docLevel, query, strlen(query));
+	cout << "index set to " << queryInfo.docLevel <<"\n";
+	cin.getline(query, 2048, '\n');
+      } else if (queryArray[1] == 'l') {
+	cout << "current level="<< level << "\nchange to level:";
+	cin >> query;
+	UCArrayClear(level);
+	SetCStr(level, query, strlen(query));
+	cout << "level set to " << level <<"\n";
+	cin.getline(query, 2048, '\n');
+      }
+
+
+      else if (queryArray[1] == 'p') {
+	// print
+	UCArray docText;
+	unsigned long docNum = 0;
+	cin >> docNum;
+	cin.getline(query, 2048, '\n'); // eat up return
+	
+	if (!GetDocText (textData, queryInfo.docLevel, docNum, docText)) {
+	  FatalError (1, "Error while trying to get document %u", docNum);
+	}
+	
+	cout << docText << "\n";
+      }
+      else if (queryArray[1] == 't') { // query type - all/some
+	if (queryArray[2] == '1') defaultBoolCombine = 1;
+	else if (queryArray[2] == '0') defaultBoolCombine = 0;
+	else {
+	  cout << "Error: please enter .t0 (some) or .t1 (all)\n";
+	}
+      }
+      else if (queryArray[1] == 'r') { // ranking - on/off
+	if (queryArray[2] == '1') queryInfo.sortByRank = true;
+	else if (queryArray[2] == '0') queryInfo.sortByRank = false;
+	else {
+	  cout << "Error: please enter .r0 (non-ranked) or .r1 (ranked)\n";
+	}
+      }
+      else if (queryArray[1] == 'c') { // casefolding - on/off
+	if (queryArray[2] == '1') defaultStemMethod |= 1;
+	else if (queryArray[2] == '0') defaultStemMethod &= 0xe;
+	else {
+	  cout << "Error: please enter .c0 (case sensitive) or .c1 (casefolded)\n";
+	}
+      }
+      else if (queryArray[1] == 's') { // stemming - on/off
+	if (queryArray[2] == '1') defaultStemMethod |=2;
+	else if (queryArray[2] == '0') defaultStemMethod &=0xd;
+	else {
+	  cout << "Error: please enter .s0 (unstemmed) or .s1 (stemmed)\n";
+	}
+      }      
+      else if (queryArray[1] == 'o') { // output - short/long
+	if (queryArray[2] == '1') shortOutput = true;
+	else if (queryArray[2] == '0') shortOutput = false;
+	else {
+	  cout << "Error: please enter .o0 (long output) or .o1 (short output)\n";
+	}
+      }
+      else if (queryArray[1] == 'b') {
+	// full text browse
+	cout<<"enter a few letters to start browsing from:";
+	cin>>query;
+	UCArrayClear(browseNode.term);
+	SetCStr(browseNode.term, query, strlen(query));
+	cin.getline(query, 2048, '\n'); // get rest of line
+
+	// print the query
+	PrintNode (cout, &browseNode);
+
+	MGBrowseQuery(indexData, docLevel, browseNode, browseResult);
+	cout << browseResult;
+	cout << "\n";
+
+      }
+      else { // bad option
+	cout << "bad command\n\n";
+	printHelp();
+      }
+    } // if a .x query
+    else {
+      // regular query
+      queryTree = ParseQuery (queryArray, defaultBoolCombine, defaultStemMethod);
+      if (queryTree == NULL) {
+	cout << "invalid syntax\n";
+      } else {
+	// print the query
+	PrintNode (cout, queryTree);
+	
+	MGQuery (indexData, queryInfo, queryTree, queryResult, level);
+	if (shortOutput) {
+	  queryResult.printShort(cout);
+	  cout << "\n";
+	} else {
+	  cout << queryResult;
+	  cout << "\n";
+	}
+	// delete the query
+	delete queryTree;
+	queryTree = NULL;
+      }
+    }
+  }
+
+
+  // clean up, everybody clean up
+    textData.UnloadData ();
+  indexData.UnloadData ();
+
+  return (0);
+}
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.cpp	(revision 21112)
@@ -0,0 +1,74 @@
+/**************************************************************************
+ *
+ * TagInfo.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TagInfo.h"
+
+
+void TagInfo::SetDocTag (const char *cStr) {
+  SetCStr (docTag, cStr, strlen(cStr));
+}
+
+void TagInfo::SetIndexLevel (const char *cStr) {
+  SetCStr (indexLevel, cStr, strlen(cStr));
+}
+void TagInfo::AddLevelTag (const char *cStr) {
+  // convert the string
+  UCArray cArr;
+  SetCStr (cArr, cStr, strlen(cStr));
+
+  // insert the tag
+  levelTags.insert (cArr);
+}
+
+
+TagInfo::TagInfo () {
+  Clear ();
+}
+
+void TagInfo::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelTags.erase (levelTags.begin(), levelTags.end());
+}
+
+ostream &operator<<(ostream &s, const TagInfo &t) {
+  // output docTag
+  s << "docTag: \"" << t.docTag << "\"\n";
+
+  // output indexLevel
+  s << "indexLevel: \"" << t.indexLevel << "\"\n";
+
+  // output levelTags
+  s << "levelTags: ";
+
+  UCArraySet::const_iterator here = t.levelTags.begin();
+  UCArraySet::const_iterator end = t.levelTags.end();
+  while (here != end) {
+    s << "\"" << (*here) << "\"";
+    ++here;
+    if (here != end) s << ", ";
+  }
+
+  s << "\n";
+  
+  return s;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TagInfo.h	(revision 21112)
@@ -0,0 +1,46 @@
+/**************************************************************************
+ *
+ * TagInfo.h -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TAGINFO_H
+#define TAGINFO_H
+
+#include "UCArray.h"
+
+
+class TagInfo {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  UCArraySet levelTags;
+
+  void SetDocTag (const char *cStr);
+  void SetIndexLevel (const char *cStr);
+  void AddLevelTag (const char *cStr);
+  
+  TagInfo ();
+  void Clear ();
+};
+
+// stream operator to print TagInfo
+ostream &operator<<(ostream &s, const TagInfo &t);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/Terms.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Terms.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Terms.cpp	(revision 21112)
@@ -0,0 +1,797 @@
+/**************************************************************************
+ *
+ * Terms.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "Terms.h"
+#include "words.h"
+#include "stemmer.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+
+void QueryInfo::Clear () {
+  UCArrayClear (docLevel);
+  maxDocs = 0;
+  sortByRank = true;
+  exactWeights = false;
+  needRankInfo = false;
+  needTermFreqs = false;
+}
+
+
+
+void TermFreqData::Clear () {
+  UCArrayClear (tag);
+  UCArrayClear (term);
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+  stemMethod = 0;
+  matchDocs = 0;
+  termFreq = 0;
+}
+
+ostream &operator<< (ostream &s, const TermFreqData &t) {
+  s << "<" << t.tag << ">\"" << t.term << "\"stem("
+    << t.stemMethod << ")equiv terms(";
+  
+  unsigned long i;
+  for (i=0; i<t.equivTerms.size(); ++i) {
+    s << t.equivTerms[i] << ", ";
+  }
+  s <<")docs(" << t.matchDocs << ")"
+    << "count("<<t.termFreq<<")";
+  return s;
+}
+
+bool operator== (const TermFreqData &t1, const TermFreqData &t2) {
+  return ((t1.tag == t2.tag) &&
+	  (t1.term == t2.term) &&
+	  (t1.stemMethod == t2.stemMethod) &&
+	  (t1.equivTerms == t2.equivTerms) &&
+	  (t1.matchDocs == t2.matchDocs) &&
+	  (t1.termFreq == t2.termFreq));
+}
+
+
+void QueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+QueryResult::QueryResult () {
+  Clear ();
+}
+
+void QueryResult::printShort(ostream &s) {
+
+  s << "termFreqs: ";
+  for (unsigned long i=0; i<termFreqs.size(); ++i)
+    s << termFreqs[i] << ", ";
+  
+  s << "\nactual number of docs found: " << actualNumDocs;
+  s << "\n\n";
+
+}
+
+
+ostream &operator<< (ostream &s, const QueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const QueryResult &r1, const QueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//---------------------------------------------------
+// new ExtQueryResult stuff
+void ExtQueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  levels.erase (levels.begin(), levels.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+ExtQueryResult::ExtQueryResult () {
+  Clear ();
+}
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+
+  s << "\nlevels: ";
+  for (i=0; i<r.levels.size(); ++i)
+    s << r.levels[i] << ", ";
+
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.levels == r2.levels) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//-------------------------------------------------------
+// new BrowseQueryResult stuff
+void BrowseQueryResult::Clear () {
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+}
+
+BrowseQueryResult::BrowseQueryResult () {
+  Clear ();
+}
+
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r) {
+  s << "terms: ";
+  unsigned long i;
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+    s << "\n\n";
+  return s;
+}
+
+
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2) {
+  return ((r1.termFreqs == r2.termFreqs));
+	 
+}
+
+
+
+
+//--------------------------------------
+void FragData::Clear () {
+  matchDocs = 0;
+  fragNums.erase (fragNums.begin(), fragNums.end());
+  fragFreqs.erase (fragFreqs.begin(), fragFreqs.end());
+}
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (stemMethod == 0 || stemMethod==4 || stemMethod==5) {
+    // don't need to stem the word,
+    // find the word number(s) for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (stemMethod ==0) {
+      if (SearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels, term, wordDictEl, wordElNum))
+	equivWords.push_back (wordElNum);
+      
+      return;
+    } else {
+      // partial matching,
+      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod==5?true:false) );
+      return;
+    }
+  }
+		      
+  // need to stem this word and find it in the blocked stem index
+  
+  unsigned char  mgWord[MAXSTEMLEN + 1];
+  UCArray stemTerm;
+  unsigned long stemmerNum = 0;
+  if (stemMethod == 1) stemmerNum = indexData.sih1.stemmer_num;
+  else if (stemMethod == 2) stemmerNum = indexData.sih2.stemmer_num;
+  else if (stemMethod == 3) stemmerNum = indexData.sih3.stemmer_num;
+   
+  // convert the word to an "mg word"
+  mgWord[0] = term.size();
+  memcpy ((char *)&mgWord[1], &(term[0]), term.size());
+  
+  // stem the word
+  stemmer (stemMethod, stemmerNum, mgWord);
+
+  // convert the result back to a UCArray
+  stemTerm.insert (stemTerm.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+  // need to look up this term in the appropriate dictionary
+  stem_block_dict_el stemDictEl;
+  unsigned long stemElNum;
+  bool result = false;
+  if (stemMethod == 1) {
+    result = SearchStemBlockDictEl (indexData.stem1File,
+			   indexData.sii1,
+			   indexData.sih1.entries_per_block,
+			   indexData.sih1.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 2) {
+    result = SearchStemBlockDictEl (indexData.stem2File,
+			   indexData.sii2,
+			   indexData.sih2.entries_per_block,
+			   indexData.sih2.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 3) {
+    result = SearchStemBlockDictEl (indexData.stem3File,
+			   indexData.sii3,
+			   indexData.sih3.entries_per_block,
+			   indexData.sih3.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+  }
+
+  if (result) {
+    equivWords = stemDictEl.equivWords;  
+  }
+}
+
+
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray & termWord) {
+  fragData.Clear();
+
+  // look up the word in the dictionary
+  unsigned long numLevels = indexData.bdh.num_levels;
+  word_block_dict_el wordDictEl;
+  wordDictEl.SetNumLevels (numLevels);
+  if (!SearchWordBlockDictElNum (indexData.dictFile,
+				 indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels,
+				 termNum, wordDictEl))
+    return; // nothing more to do
+
+  fragData.matchDocs = wordDictEl.levelFreqs[indexData.curLevelNum];
+  termWord = wordDictEl.el;
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, wordDictEl.invf_ptr, SEEK_SET);
+  stdio_bitio_buffer buffer (indexData.invfFile);
+    
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags,
+				     wordDictEl.frag_occur);
+  unsigned long fragNum = 0;
+  unsigned long termFreq = 0;
+
+  unsigned long fragLimitI = 0;
+  unsigned long i;
+  for (i=0; i<wordDictEl.frag_occur; ++i) {
+    fragNum += buffer.bblock_decode (B, NULL);
+    if (!indexData.ifh.word_level_index) termFreq = buffer.gamma_decode (NULL);
+    else termFreq = 1;
+
+    // get the right fragment range
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < (*fragLimits).size() &&
+	     fragNum > (*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    // add the entry if it is within the limits
+    if ((fragLimits == NULL) ||
+	(fragLimitI < (*fragLimits).size() &&
+	 fragNum > (*fragLimits)[fragLimitI].rangeStart &&
+	 fragNum <= (*fragLimits)[fragLimitI].rangeEnd)) {
+      fragData.fragNums.push_back (fragNum);
+      if (needFragFreqs)
+	fragData.fragFreqs.push_back (termFreq);
+    }
+  }
+
+  buffer.done();
+}
+
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData) {
+  outFragData.Clear();
+
+  // the new number of matching documents is the maximum
+  // of the two input matching number of documents -- it
+  // is assumed that these are at the same document level
+  outFragData.matchDocs = (f1.matchDocs > f2.matchDocs) ?
+    f1.matchDocs : f2.matchDocs;
+
+  // do or
+  unsigned long f1I = 0, f1Size = f1.fragNums.size();
+  unsigned long f2I = 0, f2Size = f2.fragNums.size();
+  while (f1I < f1Size || f2I < f2Size) {
+    if (f2I < f2Size &&
+	(f1I >= f1Size ||
+	 f1.fragNums[f1I] > f2.fragNums[f2I])) {
+      // output f2I
+      outFragData.fragNums.push_back (f2.fragNums[f2I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f2.fragFreqs[f2I]);
+      ++f2I;
+      
+    } else if (f1I < f1Size &&
+	       (f2I >= f2Size ||
+		f1.fragNums[f1I] < f2.fragNums[f2I])) {
+      // output f1I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]);
+      ++f1I;
+      
+    } else {
+      // must be equal combine f1I and f2I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]+f2.fragFreqs[f2I]);
+      ++f1I;
+      ++f2I;
+    }
+  }
+}
+
+
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits) {
+  // sanity check on range
+  if (startRange > endRange) {
+    signed long temp = endRange;
+    endRange = startRange;
+    startRange = temp;
+  }
+
+  // get min matchdocs
+  if (comFragData.matchDocs < fragData.matchDocs)
+    fragData.matchDocs = comFragData.matchDocs;
+  
+  unsigned long fragDataI = 0;
+  unsigned long fragDataSize = fragData.fragNums.size();
+  unsigned long comFragDataI = 0;
+  unsigned long comFragDataSize = comFragData.fragNums.size();
+  unsigned long fragLimitI = 0;
+  unsigned long fragLimitSize = (fragLimits==NULL) ? 0 : (*fragLimits).size();
+  unsigned long outI = 0;
+
+  while (fragDataI < fragDataSize &&
+	 comFragDataI < comFragDataSize) {
+    signed long fragNum = (signed long)fragData.fragNums[fragDataI];
+    signed long comFragNum = (signed long)comFragData.fragNums[comFragDataI];
+    
+    // go to the right fragment limit (for the com frag)
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < fragLimitSize &&
+	     comFragNum > (signed long)(*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    if (fragNum <= comFragNum+startRange ||
+	(fragLimits!=NULL &&
+	 fragNum<=(signed long)(*fragLimits)[fragLimitI].rangeStart)) {
+      ++fragDataI;
+      
+    } else if (fragNum > comFragNum+endRange ||
+	       (fragLimits!=NULL &&
+		fragNum>(signed long)(*fragLimits)[fragLimitI].rangeEnd)) {
+      ++comFragDataI;
+      
+    } else {
+      // equal and within tag
+      fragData.fragNums[outI] = comFragNum;
+      if (needFragFreqs) {
+	fragData.fragFreqs[outI] =
+	  (fragData.fragFreqs[fragDataI] < comFragData.fragFreqs[comFragDataI]) ?
+	  fragData.fragFreqs[fragDataI] : comFragData.fragFreqs[comFragDataI];
+      }
+      ++fragDataI;
+      ++comFragDataI;
+      ++outI;
+    }
+  }
+
+  // erase unused part of fragData
+  fragData.fragNums.erase (fragData.fragNums.begin()+outI,
+			   fragData.fragNums.end());
+  if (needFragFreqs)
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin()+outI,
+			      fragData.fragFreqs.end());
+  else
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin(),
+			      fragData.fragFreqs.end());
+}
+
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  result.Clear();
+
+  // log (N / ft)
+  unsigned long N = indexData.levels.levelInfo[indexData.curLevel].numEntries;
+  float wordLog = log((double)N / (double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  if (needRanks) {
+	    Wdt = termDocFreq * wordLog;
+	    result.ranks.push_back (Wqt * Wdt);
+	  }
+	  result.docs.push_back (lastLevelDocNum);
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks){
+	termDocFreq += termData.fragFreqs[termDataI];
+	overallwordfreq += termData.fragFreqs[termDataI];
+      }
+    }
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    if (needRanks) {
+      Wdt = termDocFreq * wordLog;
+      result.ranks.push_back (Wqt * Wdt);
+    }
+    result.docs.push_back (lastLevelDocNum);
+  }
+
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq; // will be zero if needRankInfo 
+                                              //not true
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  // log (N / ft)
+  float wordLog =
+    log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries/
+	(double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  Wdt = termDocFreq * wordLog;
+	  
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks)
+	termDocFreq += termData.fragFreqs[termDataI];
+     overallwordfreq += termData.fragFreqs[termDataI]; 
+    }
+    
+    ++termDataI;
+  } // while
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    Wdt = termDocFreq * wordLog;
+
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks)
+	result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+  
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq;
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long lastLevelDocNum = 0;
+
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI];
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+      }
+    }
+    
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks) 
+	result.ranks[resultOutI] = result.ranks[resultI];
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+}
+
+
+
+//--------------------------------------------------------------
+// functions to support full text browse
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number) {
+
+    // find the word number for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (NearestSearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				      indexData.bdh.entries_per_wblk,
+				      indexData.bdh.word_dict_size,
+				      numLevels, term, wordDictEl, wordElNum))
+      number = wordElNum;
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms) {
+
+  word_block_dict_el_array wordBlocks; // = new word_block_dict_el_array();
+  TermFreqData termdata;
+
+  terms.erase(terms.begin(), terms.end());
+
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, wordBlocks);
+
+  word_block_dict_el_array::iterator here = wordBlocks.begin();
+  word_block_dict_el_array::iterator end = wordBlocks.end();
+
+  while (here != end) {
+    termdata.Clear();
+    termdata.term = (*here).el;
+    termdata.termFreq = (*here).freq;
+    terms.push_back(termdata);
+    ++here;
+  }
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms) {
+
+  
+  
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, terms);
+
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/Terms.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Terms.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Terms.h	(revision 21112)
@@ -0,0 +1,215 @@
+/**************************************************************************
+ *
+ * Terms.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TERMS_H
+#define TERMS_H
+
+#include "IndexData.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+class QueryInfo {
+public:
+  // general query information
+  UCArray docLevel;
+  unsigned long maxDocs; // 0 = all
+  bool sortByRank;
+  bool exactWeights;
+
+  // information need to return
+  bool needRankInfo;
+  bool needTermFreqs;
+
+  void Clear ();
+  QueryInfo () { Clear (); }
+};
+
+
+class TermFreqData {
+public:
+  UCArray tag;  // level tag or query tag
+  UCArray term; // unstemmed term
+  int stemMethod;
+  UCArrayVector equivTerms; // the stemmed and casefolded variants of the term
+  unsigned long matchDocs; // tf for level - num levels
+			   // containing this term
+  unsigned long termFreq;  // overall term freq - num words that
+				// are this term
+  void Clear ();
+  TermFreqData () { Clear (); }
+};
+
+ostream &operator<< (ostream &s, const TermFreqData &t);
+bool operator== (const TermFreqData &t1, const TermFreqData &t2);
+
+typedef vector<TermFreqData> TermFreqArray;
+
+
+typedef vector<float> RankArray;
+typedef vector<unsigned long> DocNumArray;
+
+class QueryResult {
+public:
+  DocNumArray docs;
+  RankArray ranks; // used for accumulators during query
+  
+  TermFreqArray termFreqs;
+
+  unsigned long actualNumDocs;
+  void Clear ();
+  QueryResult ();
+  void printShort(ostream &s);
+};
+
+ostream &operator<< (ostream &s, const QueryResult &r);
+bool operator== (const QueryResult &r1, const QueryResult &r2);
+
+typedef vector<unsigned long> FragNumArray;
+typedef vector<unsigned long> FragFreqArray;
+
+class FragData {
+public:
+  unsigned long matchDocs;    // ft for level
+  FragNumArray fragNums;
+  FragFreqArray fragFreqs;
+
+  void Clear ();
+  FragData () { Clear (); }
+};
+
+
+class FragRange {
+public:
+  unsigned long rangeStart;
+  unsigned long rangeEnd;
+
+  void Clear () { rangeStart = rangeEnd = 0; }
+  FragRange () { Clear (); }
+};
+
+typedef vector<FragRange> FragRangeArray;
+
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords);
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray &termWord);
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData);
+
+// output will be in fragData (as this is an and operation)
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits);
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result);
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result);
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result);
+
+//-----------------------------------------------------------------
+// new QueryResult class to handle retrieval of doc and level nums.
+// Use this class with extended version of MGQuery
+
+class ExtQueryResult : public QueryResult {
+public:
+  DocNumArray levels; // used for returning a different granularity, eg
+  // search sections but return Document numbers, or search Documents, 
+  // return Section numbers.
+ 
+  void Clear ();
+  ExtQueryResult ();
+};
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r);
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2);
+
+//------------------------------------------------------------
+// new functions to handle full text browse
+
+class BrowseQueryResult {
+ public:
+  TermFreqArray termFreqs;
+  void Clear();
+  BrowseQueryResult ();
+
+};
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r);
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2);
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number);
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms);
+
+void GetTermList (IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/TextEl.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TextEl.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TextEl.cpp	(revision 21112)
@@ -0,0 +1,211 @@
+/**************************************************************************
+ *
+ * TextEl.cpp -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TextEl.h"
+#include "ctype.h"
+
+#define REC_TERM   '\002'
+#define PARA_TERM  '\003'
+
+
+void TextEl::Clear () {
+  elType = TextE;
+  tagName.erase (tagName.begin(), tagName.end());
+  text.erase (text.begin(), text.end());
+}
+
+
+static bool ReadTextTag (FileBuf &buf, TextEl &el) {
+  // we must have some sort of tag
+  unsigned char c;
+  if (!buf.Peek(c)) return false;
+  
+  // get the '<'
+  if (c != '<') return false;
+  el.text.push_back (c);
+  if (!buf.Next() || !buf.Peek (c)) return false;
+  
+  // see if we have a '/'
+  if (c == '/') {
+    // we have a closing tag
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+    el.elType = CloseTagE;
+  } else {
+    // we have an opening tag
+    el.elType = OpenTagE;
+  }
+
+  // get the tag name
+  while (isalnum(c)) {
+    el.text.push_back (c);
+    el.tagName.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+  
+  // get everything up to the closing '>'
+  while (c != '>') {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+
+  // get the closing '>'
+  el.text.push_back (c);
+  buf.Next();
+
+  // if there is no tag name then make this a text element
+  if (el.tagName.empty()) el.elType = TextE;
+  
+  return true;
+}
+
+static void ToggleParaTag (TextEl &el, bool &compatInPara) {
+  SetCStr (el.tagName, "Paragraph", 9);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (compatInPara) {
+    el.elType = CloseTagE;
+    el.text.push_back (PARA_TERM);
+  } else el.elType = OpenTagE;
+  compatInPara = !compatInPara;
+}
+
+static void SetRecTag (TextEl &el, TextElType elType) {
+  el.elType = elType;
+  SetCStr (el.tagName, "Document", 8);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (elType == CloseTagE)
+    el.text.push_back (REC_TERM);
+}
+
+
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara) {
+  el.Clear();
+
+  unsigned char c;
+  if (!buf.Peek (c)) return false;
+
+  if (compatMode) {
+    if (c == PARA_TERM) {
+      ToggleParaTag (el, compatInPara);
+      if (!compatInPara) buf.Next();
+      return true;
+    }
+    if (c == REC_TERM) {
+      if (compatInPara) {
+	// need to close this paragraph
+	ToggleParaTag (el, compatInPara);
+	return true;
+      }
+      // can close this document
+      buf.Next();
+      SetRecTag (el, CloseTagE);
+      return true;
+    }
+    if (!compatInPara) {
+      // need to open a paragraph
+      ToggleParaTag (el, compatInPara);
+      return true;
+    }
+  }
+  
+  // see if we have some sort of tag
+  if (c == '<') return ReadTextTag (buf, el);
+
+  // read in a text element
+  el.elType = TextE;
+  while (c != '<' && !(compatMode && (c == PARA_TERM || c == REC_TERM))) {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) break;
+  }
+
+  return true;
+}
+
+static void AddTextEl (TextElArray &doc, unsigned long &docLen,
+		       const TextEl &el) {
+  doc.push_back (el);
+  docLen += el.text.size();
+
+//    cout << "elType:  " << el.elType << "\n";
+//    cout << "tagName: " << el.tagName << "\n";
+//    cout << "text:    \"" << el.text << "\"\n\n";
+}
+
+
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode) {
+  bool compatInPara = false;
+  bool foundDocEl = false;
+  TextEl el;
+  
+  doc.erase (doc.begin(), doc.end());
+  docLen = 0;
+
+  if (compatMode) {
+    // add <Document><Paragraph>
+    SetRecTag (el, OpenTagE);
+    AddTextEl (doc, docLen, el);
+    ToggleParaTag (el, compatInPara);
+    AddTextEl (doc, docLen, el);
+    
+  } else {
+    // look for an opening docTag
+    do {
+      if (!ReadTextEl (buf, el, compatMode, compatInPara)) return false;
+    } while (el.elType != OpenTagE || el.tagName != docTag);
+    
+    AddTextEl (doc, docLen, el);
+  }
+
+  
+  // get all elements until the closing docTag
+  while (ReadTextEl (buf, el, compatMode, compatInPara)) {
+    foundDocEl = true;
+    AddTextEl (doc, docLen, el);
+    if (el.elType == CloseTagE && el.tagName == docTag)
+      return true;
+  }
+
+  
+  if (compatMode) {
+    if (!foundDocEl) { // end of text
+      doc.erase (doc.begin(), doc.end());
+      return false;
+    }
+
+    // if we got here then the eof was encountered before
+    // the closing document tag
+    if (compatInPara) {
+      // need to close this paragraph
+      ToggleParaTag (el, compatInPara);
+      AddTextEl (doc, docLen, el);
+    }
+    // close this document
+    SetRecTag (el, CloseTagE);
+    AddTextEl (doc, docLen, el);
+  }
+  
+  return true;
+}
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/TextEl.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TextEl.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TextEl.h	(revision 21112)
@@ -0,0 +1,62 @@
+/**************************************************************************
+ *
+ * TextEl.h -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTEL_H
+#define TEXTEL_H
+
+#include "UCArray.h"
+#include "FileBuf.h"
+
+
+enum TextElType {OpenTagE, CloseTagE, TextE};
+
+class TextEl {
+protected:
+  bool ReadTextTag (FileBuf &buf, TextEl &el);
+public:
+  TextElType elType;
+  UCArray tagName;
+  UCArray text; // complete text, including tag
+
+  TextEl () { Clear (); }
+  void Clear ();
+};
+
+
+typedef vector<TextEl> TextElArray;
+
+
+// compatMode parses documents similar to "old mg" -- ie with
+// C-b between documents and C-c between paragraphs
+
+// returns true if a text element could be read
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara);
+
+// looks for an opening docTag and reads all text elements up
+// to a corresponding closing docTag. note that this does not
+// hierarchically parse the document so you can't nest documents.
+// returns true if a document could be found
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode=false);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/TextGet.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TextGet.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TextGet.cpp	(revision 21112)
@@ -0,0 +1,685 @@
+/**************************************************************************
+ *
+ * TextGet.cpp -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// is important to be first, so we escape the truncation warning on VC++
+#include "TextGet.h"
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "mg_files.h"
+#include "netorder.h"
+#include "mg_errors.h"
+#include "locallib.h"
+#include "words.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+
+typedef enum huff_type {lengths, chars};
+
+
+static auxiliary_dict *LoadAuxDict (compression_dict &cd, FILE *text_aux_dict) {
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = new auxiliary_dict))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  memset (ad, '\0', sizeof (*ad));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = new u_char[ad->afh[i].mem_for_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+      if (!(ad->words[i] = new u_char* [ad->afh[i].num_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < (int)ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd.cdh.novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd.cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+static u_char ***ReadInWords (FILE *dict, compression_dict &cd,
+			      comp_frags_header *cfh, u_char **escape) {
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd.cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i) {
+    ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+    mem_reqd += cfh->huff_words_size[i];
+  }
+
+  if (!(vals = new u_char* [ptrs_reqd]))
+    return (NULL);
+
+  if (!(values = new u_char** [MAX_HUFFCODE_LEN + 1]))
+    return (NULL);
+
+  if (!(next_word[0] = new u_char[mem_reqd])) 
+    return (NULL);
+
+  cd.MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+static int Load_Comp_HuffData(compression_dict &cd, int which, FILE *dict, 
+			      huff_type type) {
+  huff_data * hd;
+  u_long **   vals;
+
+  if (!(hd = new huff_data))
+    return 1;
+  cd.MemForCompDict += sizeof (huff_data);
+  if (Read_Huffman_Data (dict, hd, &cd.MemForCompDict, NULL) == -1)
+    return 2;
+  if (!(vals = Generate_Huffman_Vals (hd, &cd.MemForCompDict)))
+    return 3;
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+  if (type == chars)
+    {
+      cd.chars_huff[which] = hd;
+      cd.chars_vals[which] = vals;
+    }
+  else
+    {
+      cd.lens_huff[which] = hd;
+      cd.lens_vals[which] = vals;
+    }
+
+  return 0;
+}
+
+static int Load_Comp_FragsHeader(compression_dict &cd, int which, int getEscape,
+				 FILE *dict) {
+  if (!(cd.cfh[which] = new comp_frags_header))
+    return 1;
+  cd.MemForCompDict += sizeof (*cd.cfh[which]);
+  if (Read_cfh (dict, cd.cfh[which], &cd.MemForCompDict, NULL) == -1)
+    return 2;
+
+  if (!(cd.values[which] = ReadInWords (dict, cd, cd.cfh[which],
+					getEscape == 0 ? NULL : &cd.escape[which])))
+    return 3;
+  
+  return 0;
+}
+
+static bool LoadSlowCompDict (FILE *dict, FILE *aux_dict, compression_dict &cd) {
+  if (dict == NULL) return false;
+
+  int which;
+
+  memset (&cd, '\0', sizeof (compression_dict));
+
+  cd.MemForCompDict = sizeof (compression_dict);
+
+  if (Read_cdh (dict, &cd.cdh, &cd.MemForCompDict, NULL) == -1)
+    return false;
+
+  for (which = 0; which < 2; ++which)
+    switch (cd.cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Load_Comp_FragsHeader(cd, which, 0, dict) != 0)
+	    return false;
+	  cd.escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+
+	  if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+	    return false;
+
+	  if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+	    return false;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+		return false;
+
+	      if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+		return false;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd.cdh.novel_method == MG_NOVEL_DELTA ||
+      cd.cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  cd.Clear();
+	  return false;
+	}
+
+      if (!(cd.ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  cd.Clear();
+	  return false;
+	}
+    }
+
+  mg_errno = MG_NOERROR;
+
+  cd.fast_loaded = 0;
+  
+  return true;
+}
+
+
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p,cd) & 7))) != 0)
+
+// fast loading really needs to be totally re-writen. "Unloading" the
+// text data will currently cause a crash because memory is being
+// deleted multiple times (and probably a zillion other reasons).
+static bool LoadFastCompDict (FILE *text_fast_comp_dict, compression_dict &_cd) {
+  if (text_fast_comp_dict == NULL) return false;
+  
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  compression_dict *cd;
+  if (!(cd = (compression_dict *)malloc (mem))) {
+    mg_errno = MG_NOMEM;
+    return false;
+  }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = new u_char[fixup_mem]))
+    {
+      mg_errno = MG_NOMEM;
+      return false;
+    }
+
+  fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; ++p)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    for (i = 0; i <= 1; ++i)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; ++j)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  delete []fixup;
+
+  // the whole fast comp dict is a bit of a hack so I don't
+  // feel too bad about the next line :-) -- Rodger.
+  _cd = *cd;
+  
+  return true;
+}
+
+
+static bool LoadCompDict (FILE *compDictFile,
+			  FILE *auxDictFile,
+			  FILE *fastCompDictFile,
+			  compression_dict &cd) {
+  // see if we have a fast loading compression dictionary
+  if (fastCompDictFile != NULL)
+    return LoadFastCompDict (fastCompDictFile, cd);
+  
+  // slow compression dictionary
+  return LoadSlowCompDict (compDictFile, auxDictFile, cd);
+}
+
+
+// try to open the dictionary files and load the dictionary
+static bool OpenLoadCompDict (char *textname, compression_dict &cd) {
+  FILE *compDictFile = NULL;
+  FILE *auxDictFile = NULL;
+  FILE *fastCompDictFile = NULL;
+
+  fastCompDictFile = open_file (textname, TEXT_DICT_FAST_SUFFIX,
+				"rb", MAGIC_FAST_DICT, MG_CONTINUE);
+  
+  if (fastCompDictFile == NULL) {
+    compDictFile = open_file (textname, TEXT_DICT_SUFFIX,
+			      "rb", MAGIC_DICT, MG_MESSAGE);
+    auxDictFile =  open_file (textname, TEXT_DICT_AUX_SUFFIX,
+			      "rb", MAGIC_AUX_DICT, MG_CONTINUE);
+  }
+  
+  bool res = LoadCompDict (compDictFile, auxDictFile, fastCompDictFile, cd);
+
+  if (compDictFile != NULL) fclose (compDictFile);
+  if (auxDictFile != NULL) fclose (auxDictFile);
+  if (fastCompDictFile != NULL) fclose (fastCompDictFile);
+  
+  return res;
+}
+
+static bool LoadLevels (char *textname, FTextLevel &levels) {
+  FILE *levelFile = NULL;
+
+  // open the text level file
+  levelFile = open_file (textname, TEXT_LEVEL_SUFFIX,
+			 "rb", MAGIC_TEXT_LEVELS, MG_CONTINUE);
+  if (levelFile == NULL) return false;
+
+  // seek to the appropriate place and read the level information
+  bool res = ((fseek (levelFile, sizeof (u_long), SEEK_SET) == 0) &&
+	      levels.Read (levelFile));
+
+  // close the file
+  fclose (levelFile);
+  
+  return res;
+}
+
+
+TextData::TextData () {
+  // put file pointers in known state first
+  textFile = NULL;
+  textIdxFile = NULL;
+  Clear ();
+}
+
+void TextData::Clear () {
+  cd.Clear(); 
+  textFile = NULL;
+  textIdxFile = NULL;
+  cth.Clear();
+  levels.Clear();
+}
+
+bool TextData::LoadData (char *basepath, char *textname) {
+  
+  if (textname[0] == '\0') return false;
+  
+  // set the basepath
+  set_basepath(basepath);
+  
+  // load the compression dictionary
+  if (!OpenLoadCompDict (textname, cd)) return false;
+
+  // open the compressed text and text index file
+  textFile = open_file (textname, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_CONTINUE);
+  if (textFile == NULL) return false;
+
+  textIdxFile = open_file (textname, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_CONTINUE);
+  if (textIdxFile == NULL) return false;
+
+  // read in the compressed text header
+  if ((fseek (textFile, sizeof (u_long), SEEK_SET) != 0) || !cth.Read (textFile))
+    return false;
+
+  // read in the level information
+  if (!LoadLevels (textname, levels)) return false;
+  
+  return true;
+}
+
+bool TextData::UnloadData () {
+  // close any open files
+  if (textFile != NULL) {
+    fclose (textFile);
+    textFile = NULL;
+  }
+  if (textIdxFile != NULL) {
+    fclose (textIdxFile);
+    textIdxFile = NULL;
+  }
+
+  // do general clear
+  Clear ();
+  
+  return true;  
+}
+
+
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx) {
+  // make sure the text index file was opened successfully
+  if (td.textIdxFile == NULL) return false;
+
+  // read in the index
+  TextLevelInfo &levelInfo = td.levels.levelInfo[docLevel];
+  if (!docIdx.Read (td.textIdxFile, levelInfo, docNum)) return false;
+  
+  return true;
+}
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        __code += __code + buffer.bit();	                  	\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText) {
+  // erase the current text
+  docText.erase (docText.begin(), docText.end());
+
+  // look up the information about this document
+  TextIdx docIdx;
+  if (!GetDocIdx (td, docLevel, docNum, docIdx)) return false;
+
+  // do seek to appropriate position
+  stdio_bitio_buffer buffer (td.textFile);
+  buffer.seek (docIdx.start.byte, docIdx.start.bit);
+  
+  // decompress the document
+  compression_dict &cd = td.cd;
+  auxiliary_dict *ad = cd.ad;
+  int which = docIdx.which;
+  unsigned long num_bits = (docIdx.end.byte*8+(8-docIdx.end.bit)) -
+    (docIdx.start.byte*8+(8-docIdx.start.bit));
+  unsigned long bits = 0;
+
+  if (docText.capacity() < docText.size() + num_bits + 1) {
+    docText.reserve(docText.size() + num_bits + 1);
+  }
+  // keep decoding bits until enough bits have been decoded
+  while (bits < num_bits) {
+    register unsigned code, len;
+    register int r;
+    register u_char *t, *b = NULL;
+    u_char word[MAXWORDLEN + 1];
+    
+    if (cd.cfh[which]) {
+      MY_HUFF_DECODE (len, code, cd.cfh[which]->hd.min_code);
+      bits += len;
+      
+      r = code & ((1 << cd.cdh.lookback) - 1);
+      t = cd.values[which][len][code >> cd.cdh.lookback];
+      
+      /* step through from base pointer */
+      b = word + 1;
+      while (r--) {
+	register int copy = *t >> 4;
+	memcpy (word + copy + 1, t + 1, *t & 0xf);
+	word[0] = copy + (*t & 0xf);
+	t += ((*t) & 0xf) + 1;
+      }
+    } else t = NULL;
+    
+    if (t == cd.escape[which]) {
+      switch (cd.cdh.novel_method) {
+      case MG_NOVEL_HUFFMAN_CHARS:
+	{
+	  int len, i;
+	  int c;
+	  len = buffer.huff_decode(cd.lens_huff[which]->min_code,
+				   cd.lens_vals[which], &bits);
+	  for (i = 0; i < len; ++i) {
+	    c = buffer.huff_decode(cd.chars_huff[which]->min_code,
+				   cd.chars_vals[which], &bits);
+	    docText.push_back (c);
+	  }
+	}
+	break;
+      case MG_NOVEL_DELTA:
+      case MG_NOVEL_HYBRID:
+	{
+	  int idx = 0, len;
+	  u_char *base;
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_DELTA:
+	      {
+		idx = buffer.delta_decode (&bits);
+		--idx;
+	      }
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      {
+		int k;
+		k = buffer.gamma_decode (&bits);
+		--k;
+		idx = buffer.binary_decode(ad->blk_end[which][k] -
+					   ad->blk_start[which][k] + 1, 
+					   &bits);
+		idx += ad->blk_start[which][k] - 1;
+	      }
+	      break;
+	    }
+	  base = ad->words[which][idx];
+	  len = *base++;
+	  for (; len; --len)
+	    {
+	      docText.push_back (*base++);
+	    }
+	}
+	break;
+      }
+    }
+    else
+      {
+	/* copy over the matching prefix */
+	r = (*t >> 4);
+	while (r--) {
+	  docText.push_back (*b++);
+	}
+	
+	/* and the stored suffix */
+	r = ((*t) & 0xf);
+	while (r--) {
+	  docText.push_back (*++t);
+	}
+      }
+    which = !which;
+  }
+  
+  buffer.done();
+  
+  return true;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/TextGet.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/TextGet.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/TextGet.h	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * TextGet.h -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTGET_H
+#define TEXTGET_H
+
+#include "text.h"
+#include "FText.h"
+
+
+class TextData {
+public:
+  compression_dict cd;
+  FILE *textFile;
+  FILE *textIdxFile;
+  compressed_text_header cth;
+  FTextLevel levels;
+
+  TextData ();
+  void Clear ();
+
+  // loads compression dictionary, the compressed text header,
+  // and all level informaiton
+  bool LoadData (char *basepath, char *textname);
+  bool UnloadData ();
+};
+
+
+// note: document numbers start at 1
+
+// get information about where a document starts and finishes
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx);
+
+
+// decompress a document's text
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/UCArray.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/UCArray.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/UCArray.cpp	(revision 21112)
@@ -0,0 +1,360 @@
+/**************************************************************************
+ *
+ * UCArray.cpp -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+void SetCStr (UCArray &text, const char *cStr) {
+  text.erase(text.begin(), text.end());
+  
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint) {
+  text.erase(text.begin(), text.end());
+
+  // reserve the needed space in advance
+  if (text.capacity() < nSizeHint + 1) {
+    text.reserve(nSizeHint + 1);
+  }
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+char * GetCStr(const UCArray& text) {
+
+  char *cstr = new char[text.size()+1];
+  UCArray::const_iterator here = text.begin();
+  UCArray::const_iterator end = text.end();
+
+  int i = 0;
+  while (here != end) {
+    cstr[i] = text[i];
+    ++here; ++i;
+  }
+  cstr[i]='\0';
+  return cstr;
+}
+
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr)
+{
+  if ((cStr == NULL || *cStr == '\0') && text.empty()) return true;
+  UCArray::const_iterator thisUC = text.begin();
+  UCArray::const_iterator endUC = text.end();
+  while (thisUC != endUC && *cStr != '\0') {
+    if (*thisUC != *cStr) return false;
+    ++cStr; ++thisUC;
+  }
+  if (thisUC == endUC && *cStr == '\0') return true;
+  return false;
+}
+
+ostream &operator<<(ostream &s, const UCArray &a) {
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    s << *here;
+    ++here;
+  }
+
+  return s;
+}
+
+
+bool ReadVarLenUL (FILE *f, unsigned long &n) {
+  register unsigned long temp = 0;
+  register unsigned int bitPos = 0;
+  unsigned char b = 0;
+
+  do {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    temp |= (b & 0x7f) << bitPos;
+    bitPos += 7;
+  } while (b >= 0x80 && bitPos < 32);
+
+  n = temp;
+  
+  return true;
+}
+
+bool WriteVarLenUL (FILE *f, unsigned long n) {
+  register unsigned long temp = n;
+  register unsigned char b = 0;
+  do {
+    b = static_cast<unsigned char> (temp & 0x7f);
+    if (temp >= 0x80) b |= 0x80;
+    fputc (b, f);
+    if (ferror (f) != 0) return false;
+  } while ((temp = temp >> 7) > 0);
+  
+  return true;
+}
+
+
+bool ReadUL (FILE *f, unsigned long &n) {
+  if (fread (&n, sizeof (unsigned long), 1, f) <= 0) return false;
+  NTOHUL (n);
+  return true;
+}
+
+
+bool WriteUL (FILE *f, unsigned long n) {
+  HTONUL (n);
+  return (fwrite (&n, sizeof (unsigned long), 1, f) > 0);
+}
+
+bool ReadF (FILE *f, float &n) {
+  if (fread (&n, sizeof (float), 1, f) <= 0) return false;
+  NTOHF(n);
+  return true;
+}
+
+bool WriteF (FILE *f, float n) {
+  HTONF(n);
+  return (fwrite (&n, sizeof (float), 1, f) > 0);
+}
+
+bool ReadD (FILE *f, double &n) {
+  if (fread (&n, sizeof (double), 1, f) <= 0) return false;
+  NTOHD(n);
+  return true;
+}
+
+bool WriteD (FILE *f, double n) {
+  HTOND(n);
+  return (fwrite (&n, sizeof (double), 1, f) > 0);
+}
+
+bool ReadUCArray (FILE *f, UCArray &a) {
+  // clear the array in preparation
+  a.erase (a.begin(), a.end());
+
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+
+  // reserve the needed space in advance
+  if (a.capacity() < arraySize + 1) {
+    a.reserve(arraySize + 1);
+  }
+
+  // read in the array
+  unsigned char b = 0;
+  while (arraySize > 0) {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    a.push_back (b);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteUCArray (FILE *f, const UCArray &a) {
+  // write out the array size
+  if (!WriteVarLenUL (f, a.size())) return false;
+
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    fputc (*here, f);
+    if (ferror (f) != 0) return false;
+    
+    ++here;
+  }
+  
+  return true;
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+
+int DictCompare (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  unsigned int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((diff = casecharmap[*a1Here] - casecharmap[*a2Here]) != 0)
+      return diff;
+    if (pos == 0 && (diff = *a1Here - *a2Here) != 0)
+      pos = diff;
+
+    ++a1Here;
+    ++a2Here;
+  }
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+}
+
+// does the first string start with the second?
+bool StartsWith (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((*a1Here != *a2Here))
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if (casecharmap[*a1Here] != casecharmap[*a2Here]) 
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2) {
+  unsigned long l = (a1.size() < a2.size()) ? a1.size() : a2.size();
+  unsigned long i = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+
+  while (i < l && *a1Here == *a2Here) {
+    ++i; ++a1Here; ++a2Here;
+  }
+  
+  return i;
+}
+
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a) {
+  unsigned char preLen;
+  unsigned char sufLen;
+  
+  if (prev != NULL) preLen = PrefixLen (*prev, a);
+  else preLen = 0;
+  sufLen = a.size() - preLen;
+
+  // output the prefix length, suffix length, and the suffix
+  fputc (preLen, f);
+  if (ferror(f) != 0) return false;
+  fputc (sufLen, f);
+  if (ferror(f) != 0) return false;
+  char* tmp=GetCStr(a);
+  int ret=(fwrite (tmp+preLen, sizeof (char), sufLen, f) == sufLen);
+  delete []tmp;
+  return (ret != 0);
+}
+
+// a also used for prev
+bool ReadPreSufStr (FILE *f, UCArray &a) {
+  unsigned char preLen = 0;
+  unsigned char sufLen = 0;
+
+  preLen = fgetc(f);
+  sufLen = fgetc(f);
+
+  if (a.size() > preLen) a.erase (a.begin()+preLen, a.end());
+
+  // reserve the needed space in advance
+  if (a.capacity() < a.size() + sufLen + 1) {
+    a.reserve(a.size() + sufLen + 1);
+  }
+
+  while (sufLen > 0) {
+    unsigned char c = fgetc (f);
+    a.push_back (c);
+    --sufLen;
+  }
+
+  return true;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/UCArray.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/UCArray.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/UCArray.h	(revision 21112)
@@ -0,0 +1,131 @@
+/**************************************************************************
+ *
+ * UCArray.h -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef UCARRAY_H
+#define UCARRAY_H
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if !defined (GSDL_NAMESPACE_BROKEN) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+using namespace std;
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\set>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <set.h>
+#else
+#  include <vector>
+#  include <set>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#include <stdio.h>
+
+
+typedef vector<unsigned char> UCArray;
+
+// functions to manipulate UCArrays
+void SetCStr (UCArray &text, const char *cStr);
+// same as SetCStr but first tries to allocate nSizeHint space (only if needed)
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint);
+char * GetCStr(const UCArray& text);
+inline void UCArrayClear (UCArray &a) {
+  a.erase (a.begin(), a.end());
+}
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr);
+inline bool UCArrayCStrEquals(const UCArray &text, const char *cStr) { return UCArrayCStrEquals(text, (const unsigned char *)cStr); }
+
+// stream operator to print UCArray
+ostream &operator<<(ostream &s, const UCArray &a);
+
+// used for reading and writing unsigned characters
+inline bool ReadUC (FILE *f, unsigned char &c) {
+  return (fread (&c, sizeof (unsigned char), 1, f) == 1);
+}
+inline bool WriteUC (FILE *f, unsigned char c) {
+  return (fwrite (&c, sizeof (unsigned char), 1, f) == 1);
+}
+
+// used for reading and writing variable length unsigned longs
+bool ReadVarLenUL (FILE *f, unsigned long &n);
+bool WriteVarLenUL (FILE *f, unsigned long n);
+
+// used for reading and writing unsigned longs
+bool ReadUL (FILE *f, unsigned long &n);
+bool WriteUL (FILE *f, unsigned long n);
+
+// used for reading and writing floats
+bool ReadF (FILE *f, float &n);
+bool WriteF (FILE *f, float n);
+
+// used for reading and writing doubles
+bool ReadD (FILE *f, double &n);
+bool WriteD (FILE *f, double n);
+
+// used for reading and writing arrays to files
+bool ReadUCArray (FILE *f, UCArray &a);
+bool WriteUCArray (FILE *f, const UCArray &a);
+
+
+// compares the two strings in dictionary order
+int DictCompare (const UCArray &a1, const UCArray &a2);
+
+// does the first string start with the second?
+bool StartsWith(const UCArray &a1, const UCArray &a2);
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2);
+
+struct LTUCArray {
+  bool operator()(const UCArray &a1, const UCArray &a2) const
+    { return (a1 < a2); }
+};
+
+struct DictLTUCArray {
+  bool operator() (const UCArray &a1, const UCArray &a2) const
+    { return (DictCompare (a1, a2) < 0); }
+};
+
+
+typedef set<UCArray, LTUCArray> UCArraySet;
+typedef vector<UCArray> UCArrayVector;
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2);
+
+// prev == NULL if no previous string
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a);
+bool ReadPreSufStr (FILE *f, UCArray &a); // a also used for prev
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/WIN32.MAK	(revision 21112)
@@ -0,0 +1,202 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..\..
+USE_STLPORT = 0
+DLL = 0
+DLLDEBUG = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+CC = cl
+
+CPPFLAGS = -GX
+!IF $(DLL)
+CPPFLAGS = $(CPPFLAGS) -MD
+!ELSE
+!IF ($(DLLDEBUG))
+CPPFLAGS = $(CPPFLAGS) -MDd
+!ENDIF
+!ENDIF
+
+DEFS = -DSHORT_SUFFIX -DSILENT -DHAVE_CONFIG_H -D__WIN32__ -D__STDC__ -D__GNU_LIBRARY__ -D__MSDOS__
+INCLUDES = -I"$(GSDLHOME)\src\mgpp\lib" -I"$(GSDLHOME)"
+!IF $(USE_STLPORT)
+DEFS = $(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+
+LDFLAGS =
+
+LIBS = "$(GSDLHOME)\src\mgpp\lib\libmg.lib"
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj .cpp
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h
+
+EXEC          = mgpp_passes$e mgpp_decompress_text$e mgpp_perf_hash_build$e \
+                mgpp_invf_dump$e mgpp_weights_build$e mgpp_invf_dict$e mgpp_stem_idx$e \
+                mgpp_invf_dict_dump$e mgpp_compression_dict$e mgpp_fast_comp_dict$e \
+                QueryTester$e Queryer$e
+
+all:    $(EXEC) libtextin.lib
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer$e: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester$e: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump$e: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+
+PASS_OBJS = \
+	words$o mgpass$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o
+mgpass$e: $(PASS_OBJS)
+	$(LINK) $(PASS_OBJS) $(LIBS)
+
+install: 
+	if NOT EXIST "$(GSDLHOME)\bin\windows" mkdir "$(GSDLHOME)\bin\windows"
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDLHOME)\bin\windows"
+
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libtextin.lib: $(LIB_OBJS)
+	if EXIST libtextin.lib del libtextin.lib
+	lib /out:libtextin.lib $(LIB_OBJS)
+
+clean:
+	del *$o 
+	del $(EXEC)
+	del libtextin.lib
Index: /main/tags/3.01/gsdl/src/mgpp/text/Weights.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Weights.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Weights.cpp	(revision 21112)
@@ -0,0 +1,126 @@
+/**************************************************************************
+ *
+ * Weights.cpp -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "Weights.h"
+#include "sysfuncs.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+
+ApproxWeightsData::ApproxWeightsData () {
+  weightBuf = NULL;
+  table = NULL;
+  Free();
+}
+
+ApproxWeightsData::~ApproxWeightsData () {
+  Free ();
+}
+
+void ApproxWeightsData::Free () {
+  bits = 0;
+  mask = 0;
+  L = 0.0;
+  B = 0.0;
+  numLevelDocs = 0;
+  if (weightBuf != NULL) {
+    delete [] weightBuf;
+    weightBuf = NULL;
+  }
+  if (table != NULL) {
+    delete [] table;
+    table = NULL;
+  }
+}
+
+bool ApproxWeightsData::Read (FILE *approxWeightsFile, unsigned long diskPtr,
+			      unsigned long _numLevelDocs) {
+  Free ();
+  
+  // go to the appropriate place in the weights file
+  fseek (approxWeightsFile, diskPtr, SEEK_SET);
+
+  // read in approx weights parameters
+  ReadUC (approxWeightsFile, bits);
+  ReadD (approxWeightsFile, L);
+  ReadD (approxWeightsFile, B);
+
+  mask = (bits == 32) ? 0xffffffff : (1 << bits) - 1;
+  numLevelDocs = _numLevelDocs;
+
+  // allocate buffer for the weights
+  unsigned long arrSize = (bits*numLevelDocs+sizeof(unsigned long)-1) / 32;
+  weightBuf = new unsigned long[arrSize];
+
+  // read in the weights
+  unsigned long i;
+  for (i=0; i<arrSize; ++i) {
+    ReadUL (approxWeightsFile, weightBuf[i]);
+  }
+
+  // create precomputed table of values if it will be small enough
+  // (if not small enough table == NULL
+  if (bits <= 12) {
+    unsigned long i, tableSize = (1 << bits);
+    table = new float[tableSize];
+    table[0] = L;
+    for (i = 1; i < tableSize; ++i)
+      table[i] = table[i - 1] * B;
+  }
+
+  return true;
+}
+
+float ApproxWeightsData::GetLowerApproxDocWeight (unsigned long levelDocNum) {
+  // sanity check
+  if (bits == 0 || weightBuf == NULL) return 1.0;
+
+  register unsigned long c, pos;
+  register unsigned long *dw;
+
+  // get the compressed version of the weight
+  pos = levelDocNum * bits;
+  dw = &weightBuf[pos / MAXBITS];
+  pos &= (MAXBITS - 1);
+  c = *dw >> pos;
+  if (pos + bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - pos);
+  c &= mask;
+
+  // decompress the weight
+  if (table != NULL) return table[c];
+  return (L * pow (B, (double) c));
+}
+
+
+
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum) {
+  // go to the appropriate place in the weights file
+  fseek (exactWeightsFile, diskPtr+sizeof(float)*levelDocNum, SEEK_SET);
+
+  // read in the exact weight
+  float weight;
+  ReadF (exactWeightsFile, weight);
+  
+  return weight;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/Weights.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/Weights.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/Weights.h	(revision 21112)
@@ -0,0 +1,56 @@
+/**************************************************************************
+ *
+ * Weights.h -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef WEIGHTS_H
+#define WEIGHTS_H
+
+#include <stdio.h>
+
+
+class ApproxWeightsData {
+public:
+  unsigned char bits;
+  unsigned long mask;
+  double L;
+  double B;
+  unsigned long numLevelDocs;
+  unsigned long *weightBuf;
+  float *table;   // precomputed translation table
+  
+  ApproxWeightsData ();
+  ~ApproxWeightsData ();
+
+  void Free (); // frees any allocated memory
+  
+  bool Read (FILE *approxWeightsFile, unsigned long diskPtr,
+	     unsigned long _numLevelDocs);
+
+  float GetLowerApproxDocWeight (unsigned long levelDocNum);
+};
+
+
+// diskPtr points to the start of the exact document weights
+// for this level
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/WordData.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/WordData.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/WordData.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * WordData.h -- 
+ * Copyright (C) 1999  NZDL
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+class WordData
+{
+
+ public:
+  unsigned long docCount;
+  unsigned long wordCount;
+
+  inline void read(FILE *f)
+    {
+      fread((char *) &docCount, sizeof(docCount), 1, f);
+      fread((char *) &wordCount, sizeof(wordCount), 1, f);
+
+      NTOHUL(docCount);
+      NTOHUL(wordCount);
+    }
+
+  inline unsigned long documents()
+    {
+      return docCount;
+    }
+
+  inline int words()
+    {
+      return wordCount;
+    }
+};
Index: /main/tags/3.01/gsdl/src/mgpp/text/build.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/build.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/build.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#include "TextEl.h"
+#include "TagInfo.h"
+
+
+int init_special (const TagInfo &tagInfo, char *file_name);
+int init_text_1 (const TagInfo &tagInfo, char *file_name);
+int init_text_2 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_2 (const TagInfo &tagInfo, char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_2 (const TagInfo &tagInfo, const TextElArray &doc);
+/*
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (const TagInfo &tagInfo, char *filename);
+int done_text_1 (const TagInfo &tagInfo, char *filename);
+int done_text_2 (const TagInfo &tagInfo, char *filename);
+int done_ivf_1 (const TagInfo &tagInfo, char *filename);
+int done_ivf_2 (const TagInfo &tagInfo, char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.cpp	(revision 21112)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * comp_dict.cpp -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = (dict_hash_table *) Xmalloc (sizeof (dict_hash_table) +
+		                            (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = (u_char **) Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  memset (ht->table, '\0', ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = (u_char *) Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; ++i, ++words)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register unsigned long tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+inline int LoadWords(FILE *dict, int which)
+{
+  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+    return 1;
+  
+  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+    return 1;
+
+  return 0;
+}
+
+inline int LoadHuffmanCodes(FILE *dict, int which)
+{
+  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 1;
+  if (!(char_codes[which] =
+	Generate_Huffman_Codes (&char_huff[which], NULL)))
+    return 2;
+  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 3;
+  if (!(lens_codes[which] =
+	Generate_Huffman_Codes (&lens_huff[which], NULL)))
+    return 4;
+
+  return 0;
+}
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (LoadWords(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (LoadHuffmanCodes(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (LoadHuffmanCodes(dict, which) != 0)
+		goto error;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,48 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define HASH_RATIO 2
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/gsdl/src/mgpp/text/hash.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/hash.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HASH
+#define H_HASH
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; --len)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/invf.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/invf.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/invf.cpp	(revision 21112)
@@ -0,0 +1,914 @@
+/**************************************************************************
+ *
+ * invf.cpp -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "invf.h"
+#include "UCArray.h"
+
+
+invf_dict_header::invf_dict_header () {
+  Clear();
+}
+
+invf_dict_header::~invf_dict_header () {
+}
+
+void invf_dict_header::Clear() {
+  lookback = 0;
+  word_dict_start = 0;
+  word_dict_size = 0;
+  tag_dict_start = 0;
+  tag_dict_size = 0;
+  num_docs = 0;
+  num_frags = 0;
+  num_words = 0;
+  total_bytes = 0;
+  index_string_bytes = 0;
+  num_levels = 0;
+}
+
+bool invf_dict_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, word_dict_start) &&
+	  ReadUL (f, word_dict_size) &&
+	  ReadUL (f, tag_dict_start) &&
+	  ReadUL (f, tag_dict_size) &&
+	  ReadUL (f, num_docs) &&
+	  ReadUL (f, num_frags) &&
+	  ReadUL (f, num_words) &&
+	  ReadUL (f, total_bytes) &&
+	  ReadUL (f, index_string_bytes) &&
+	  ReadUL (f, num_levels));
+}
+
+bool invf_dict_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, word_dict_start) &&
+	  WriteUL (f, word_dict_size) &&
+	  WriteUL (f, tag_dict_start) &&
+	  WriteUL (f, tag_dict_size) &&
+	  WriteUL (f, num_docs) &&
+	  WriteUL (f, num_frags) &&
+	  WriteUL (f, num_words) &&
+	  WriteUL (f, total_bytes) &&
+	  WriteUL (f, index_string_bytes) &&
+	  WriteUL (f, num_levels));
+}
+
+
+
+void dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+}
+
+bool dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq));
+}
+
+bool dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq));
+}
+
+
+void word_dict_el::Clear () {
+  dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_dict_el::~word_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+  
+bool word_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!dict_el::Read (f)) return false;
+
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_dict_el::Write (FILE *f, const UCArray *lastEl,
+			  unsigned long numLevels) const {
+  if (!dict_el::Write (f, lastEl)) return false;
+
+  if (levelFreqs == NULL) return false;
+
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+block_dict_header::block_dict_header () {
+  Clear();
+}
+
+void block_dict_header::Clear () {
+  invf_dict_header::Clear();
+
+  entries_per_wblk = 0;
+  num_wblks = 0;
+  max_wblk_size = 0;
+  wblk_start = 0;
+  wblk_idx_start = 0;
+  
+  entries_per_tblk = 0;
+  num_tblks = 0;
+  max_tblk_size = 0;
+  tblk_start = 0;
+  tblk_idx_start = 0;
+}
+
+bool block_dict_header::Read (FILE *f) {
+  return (invf_dict_header::Read (f) &&
+
+	  ReadUL (f, entries_per_wblk) &&
+	  ReadUL (f, num_wblks) &&
+	  ReadUL (f, max_wblk_size) &&
+	  ReadUL (f, wblk_start) &&
+	  ReadUL (f, wblk_idx_start) &&
+  
+	  ReadUL (f, entries_per_tblk) &&
+	  ReadUL (f, num_tblks) &&
+	  ReadUL (f, max_tblk_size) &&
+	  ReadUL (f, tblk_start) &&
+	  ReadUL (f, tblk_idx_start));
+}
+
+bool block_dict_header::Write (FILE *f) const {
+  return (invf_dict_header::Write (f) &&
+
+	  WriteUL (f, entries_per_wblk) &&
+	  WriteUL (f, num_wblks) &&
+	  WriteUL (f, max_wblk_size) &&
+	  WriteUL (f, wblk_start) &&
+	  WriteUL (f, wblk_idx_start) &&
+  
+	  WriteUL (f, entries_per_tblk) &&
+	  WriteUL (f, num_tblks) &&
+	  WriteUL (f, max_tblk_size) &&
+	  WriteUL (f, tblk_start) &&
+	  WriteUL (f, tblk_idx_start));
+}
+
+
+
+
+
+void block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+  invf_ptr = 0;
+}
+
+bool block_dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq) &&
+	  ReadUL (f, invf_ptr));
+}
+
+bool block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq) &&
+	  WriteUL (f, invf_ptr));
+}
+
+
+
+
+
+void word_block_dict_el::Clear () {
+  block_dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_block_dict_el::~word_block_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_block_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+
+bool word_block_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!block_dict_el::Read (f)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_block_dict_el::Write (FILE *f, const UCArray *lastEl,
+				unsigned long numLevels) const {
+  if (!block_dict_el::Write (f, lastEl)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+  
+  return true;
+}
+
+
+
+
+block_idx_info::block_idx_info () {
+  Clear ();
+}
+
+void block_idx_info::Clear () {
+  el.erase (el.begin(), el.end());
+  block_ptr = 0;
+}
+
+bool block_idx_info::Read (FILE *f) {
+  return (ReadUCArray (f, el) &&
+	  ReadUL (f, block_ptr));
+}
+
+bool block_idx_info::Write (FILE *f) const {
+  return (WriteUCArray (f, el) &&
+	  WriteUL (f, block_ptr));
+}
+
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx) {
+  blockIdx.erase (blockIdx.begin(), blockIdx.end());
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  block_idx_info bi;
+  while (arraySize > 0) {
+    if (!bi.Read (f)) return false;
+    blockIdx.push_back (bi);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx) {
+  // write out the array size
+  if (!WriteVarLenUL (f, blockIdx.size())) return false;
+
+  block_idx::const_iterator here = blockIdx.begin();
+  block_idx::const_iterator end = blockIdx.end();
+  while (here != end) {
+    if (!(*here).Write (f)) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+stem_idx_header::stem_idx_header () {
+  Clear ();
+}
+
+void stem_idx_header::Clear () {
+  lookback = 0;
+  dict_size = 0;
+
+  entries_per_block = 0;
+  num_blocks = 0;
+  max_block_size = 0;
+  blocks_start = 0;
+  block_idx_start = 0;
+  
+  stemmer_num = 0;
+  stem_method = 0;
+}
+
+bool stem_idx_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, dict_size) &&
+	  
+	  ReadUL (f, entries_per_block) &&
+	  ReadUL (f, num_blocks) &&
+	  ReadUL (f, max_block_size) &&
+	  ReadUL (f, blocks_start) &&
+	  ReadUL (f, block_idx_start) &&
+	  
+	  ReadUL (f, stemmer_num) &&
+	  ReadUL (f, stem_method));
+}
+
+bool stem_idx_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, dict_size) &&
+	  
+	  WriteUL (f, entries_per_block) &&
+	  WriteUL (f, num_blocks) &&
+	  WriteUL (f, max_block_size) &&
+	  WriteUL (f, blocks_start) &&
+	  WriteUL (f, block_idx_start) &&
+	  
+	  WriteUL (f, stemmer_num) &&
+	  WriteUL (f, stem_method));
+}
+
+
+
+stem_block_dict_el::stem_block_dict_el () {
+  Clear ();
+}
+
+void stem_block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  equivWords.erase (equivWords.begin(), equivWords.end());
+}
+
+bool stem_block_dict_el::Read (FILE *f) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (!ReadPreSufStr (f, el)) return false;
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  unsigned long wordNum;
+  while (arraySize > 0) {
+    if (!ReadUL (f, wordNum)) return false;
+    equivWords.push_back (wordNum);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool stem_block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  if (!WritePreSufStr (f, lastEl, el)) return false;
+
+  // write out the array size
+  if (!WriteVarLenUL (f, equivWords.size())) return false;
+
+  vector<unsigned long>::const_iterator here = equivWords.begin();
+  vector<unsigned long>::const_iterator end = equivWords.end();
+  while (here != end) {
+    if (!WriteUL (f, (*here))) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+invf_file_header::invf_file_header () {
+  Clear ();
+}
+
+void invf_file_header::Clear () {
+  no_of_words = 0;
+  no_of_tags = 0;
+  skip_mode = SKIP_MODE_NO_SKIPS;
+  word_level_index = 0;
+  
+  int i;
+  for (i=0; i<16; ++i) params[i] = 0;
+}
+  
+bool invf_file_header::Read (FILE *f) {
+  if (!ReadUL (f, no_of_words) ||
+      !ReadUL (f, no_of_tags) ||
+      !ReadUL (f, skip_mode) ||
+      !ReadUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!ReadUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+bool invf_file_header::Write (FILE *f) const {
+  if (!WriteUL (f, no_of_words) ||
+      !WriteUL (f, no_of_tags) ||
+      !WriteUL (f, skip_mode) ||
+      !WriteUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!WriteUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+
+
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  // make sure the element number isn't out of range
+  if (elNum > bIdx.size()*entriesPerBlock) return false;
+
+  blockIdxNum = elNum / entriesPerBlock;
+  blockStartElNum = blockIdxNum * entriesPerBlock;
+  
+  return true;
+}
+
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  unsigned long begin = 0;
+  unsigned long bIdxEnd = bIdx.size();
+  unsigned long end = bIdxEnd;
+  unsigned long mid;
+  while (begin < end) {
+    mid = (begin+end)/2;
+    if (DictCompare (el, bIdx[mid].el) < 0) {
+      end = mid;
+    } else if (mid+1 < bIdxEnd &&
+	       DictCompare (el, bIdx[mid+1].el) >= 0) {
+      begin = mid+1;
+    } else {
+      blockIdxNum = mid;
+      blockStartElNum = blockIdxNum * entriesPerBlock;
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+
+
+// use the block dictionary functions for tag entries, and word block dict
+// functions for word entries.
+
+
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+// ------------------------------------------
+// functions for partial term matching
+// ie find all words that start with xxx
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold) {
+
+  UCArrayClear (dictEl.el);
+  elNumList.erase (elNumList.begin(), elNumList.end());
+  unsigned long elNum;
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  // will this work??
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum)) {
+    return false;
+  }
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+  
+  bool still_looking = true;
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // test each element
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    if (StartsWithCasefold(dictEl.el, el)) {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+      }
+      still_looking=false; // we have found one now, so stop the next time we don't have a match
+    } else if (!still_looking) {
+      // we have found a match previously, and now this doesn't match
+      return true;
+    } // else keep looking
+    
+    ++elNum;
+  }
+  // if we get here, we are either still searching for the first
+  // case, or we are collecting terms.
+  if (still_looking) {
+    //we haven't found a match yet, just check the next element,
+    dictEl.Read (dictFile, numLevels);
+    if (!StartsWithCasefold(dictEl.el, el)) {
+      // the first element of the next block is not a match, so there are no matches
+      return false;
+    } else {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+	++elNum;
+      } 
+
+    }
+  }
+  // just keep accumulating matches until there are no more
+  dictEl.Read (dictFile, numLevels);
+  while (StartsWithCasefold(dictEl.el, el)) {
+    if (casefold || StartsWith(dictEl.el, el)) {
+      elNumList.push_back(elNum);
+    }
+    ++elNum;
+    dictEl.Read (dictFile, numLevels);
+  }
+  return true;
+  
+}
+
+
+//----------------------------------------------------------------
+// functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+				   const block_idx &bIdx,
+				   unsigned long entriesPerBlock,
+				   unsigned long dictSize,
+				   unsigned long numLevels,
+				   const UCArray &el,
+				   word_block_dict_el &dictEl,
+				   unsigned long &elNum) {
+
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (el, dictEl.el); // look for the first word that is
+                                     // greater or equal to the el
+    if (res <= 0) {
+      return true; // found one
+   } 
+    
+    ++elNum;
+  }
+  // it must be the last term
+  return true;
+
+
+}
+
+
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms) {
+
+  word_block_dict_el dictEl;  
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  terms.push_back(dictEl.el);
+  while (curElNum <= lastElNum ) {
+    dictEl.Read(dictFile, numLevels);
+    terms.push_back(dictEl.el);
+    ++curElNum;
+  }
+  
+
+  return true;
+}
+
+// NOte: before each addition of dictEl to the array, the level freqs array
+// is deleted, as this was causing problems - generating a seg fault, I think if
+// the vector had to be reallocated or something.
+// setNumLevels has to be called each time before a read, now, to set up the level
+//freqs array. this is necessary.
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms) {
+
+  word_block_dict_el dictEl;
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  block_dict_el elem;
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  dictEl.levelFreqs = NULL;
+  terms.push_back(dictEl);
+
+  while (curElNum <= lastElNum ) { 
+    dictEl.SetNumLevels(numLevels);
+    dictEl.Read(dictFile, numLevels);
+    dictEl.levelFreqs = NULL;
+    terms.push_back(dictEl);
+    ++curElNum;
+  }
+
+  return true;
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/invf.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/invf.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/invf.h	(revision 21112)
@@ -0,0 +1,353 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF
+#define H_INVF
+
+#include <stdio.h>
+
+#include "UCArray.h"
+
+// NOTE: This does not include the magic number
+// header info for .invf.dict file
+struct invf_dict_header {
+  unsigned long lookback;
+  unsigned long word_dict_start;
+  unsigned long word_dict_size;
+  unsigned long tag_dict_start;
+  unsigned long tag_dict_size;
+  unsigned long num_docs;
+  unsigned long num_frags;
+  unsigned long num_words;
+  unsigned long total_bytes;
+  unsigned long index_string_bytes;
+  unsigned long num_levels;
+
+  invf_dict_header ();
+  virtual ~invf_dict_header ();
+  virtual void Clear();
+
+  virtual bool Read (FILE *f);
+  virtual bool Write (FILE *f) const;
+};
+
+
+struct dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur;
+  unsigned long freq;
+
+  virtual void Clear ();
+  dict_el () { Clear (); }
+  virtual ~dict_el () { }
+
+  // Read assumes that the last word is in el
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+struct word_dict_el : public dict_el {
+  unsigned long *levelFreqs;
+
+  void Clear ();
+  word_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+
+// wblk = word block
+// tblk = tag block
+// this version of the blocked dictionary uses a fixed number
+// of entries per block, not a fixed block size
+// info for .invf.dict.blocked file
+// blocked dict has a heap of blocks, some for words, some for tags
+// and an index into each set of blocks. The index has pointers to
+// the first entry in each block. Can do a binary search on the index
+// to find out which block an elemnet is in
+struct block_dict_header : public invf_dict_header {
+  // note: word_dict_start and tag_dict_start are undefined
+  // for blocked dictionaries
+
+  unsigned long entries_per_wblk; // word blocks
+  unsigned long num_wblks;
+  unsigned long max_wblk_size;
+  unsigned long wblk_start;
+  unsigned long wblk_idx_start;
+  
+  unsigned long entries_per_tblk; // tag blocks
+  unsigned long num_tblks;
+  unsigned long max_tblk_size;
+  unsigned long tblk_start;
+  unsigned long tblk_idx_start;
+  
+  block_dict_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct block_dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur; // # entries in invf file - if have a
+  // word level index, this is the same as freq, otherwise, its the number
+  // of fragments containing this word
+  unsigned long freq; // # of times this word occurs
+  unsigned long invf_ptr; // pointer into inverted file
+
+  virtual void Clear ();
+  block_dict_el () { Clear (); }
+  virtual ~block_dict_el () { }
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+struct word_block_dict_el : public block_dict_el {
+  unsigned long *levelFreqs; // freq of the word at each level
+
+  void Clear ();
+  word_block_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_block_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+typedef vector<word_block_dict_el> word_block_dict_el_array;
+
+struct block_idx_info {
+  UCArray el;
+  unsigned long block_ptr;
+
+  block_idx_info ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// used for an index into the word and tag blocks
+typedef vector<block_idx_info> block_idx;
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx);
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx);
+
+
+
+struct stem_idx_header {
+  unsigned long lookback;
+  unsigned long dict_size;
+
+  unsigned long entries_per_block;
+  unsigned long num_blocks;
+  unsigned long max_block_size;
+  unsigned long blocks_start;
+  unsigned long block_idx_start;
+  
+  unsigned long stemmer_num;
+  unsigned long stem_method;
+
+  stem_idx_header ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+struct stem_block_dict_el {
+  UCArray el; // word or tag
+  vector<unsigned long> equivWords;
+  
+  stem_block_dict_el ();
+  void Clear ();
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+
+#define SKIP_MODE_NO_SKIPS 0
+
+// invf file - has a list of frags for each word, but the word is not 
+//  stored in the invf file - the dictionaries store the words, along
+// with num entries, and a pointer into invf file
+struct invf_file_header {
+  unsigned long no_of_words;
+  unsigned long no_of_tags;
+  unsigned long skip_mode;
+  unsigned long word_level_index; // 1 if word level index
+  unsigned long params[16];
+
+  invf_file_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+
+
+
+
+// the search functions returns true if a block that could
+// satisfy the request is found. these functions assume that
+// the block index is sorted by DictCompare (or DictLTUCArray)
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum);
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum);
+
+
+// The next six functions use SearchElNum and SearchEl
+// for a particular type of dictionary (Block, WordBlock, or StemBlock)
+// and then look up the entry
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl);
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum);
+
+// assumes the numLevels has been set for dictEl
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl);
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl);
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+//-----------------------------------------------------
+// New functions for partial matching
+// returns a list of word numbers from the main block index whose
+// words start with el.
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold); 
+
+//----------------------------------------------------------
+
+// new functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+// returns a list of word_block_dict_el, with no levelfreqs
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms);
+				    
+// just returns a list of terms
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms);
+				    
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass1.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass1.cpp	(revision 21112)
@@ -0,0 +1,775 @@
+/**************************************************************************
+ *
+ * ivf.pass1.cpp -- Memory efficient pass 1 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+// a fragment corresponds to the index level (word-level is the
+// minimum index level)
+
+
+// structure to determine level information
+struct LevelWorker {
+  unsigned long lastLevelDocNum;
+  unsigned long count;
+
+  LevelWorker () {
+    lastLevelDocNum = count = 0;
+  }
+};
+
+
+// note: the word is stored in the map
+struct IvfWordInfo {
+  unsigned long wordCount;      // word frequency
+  unsigned long fragCount;      // number of fragments that contain the word
+  unsigned long lastFragNum;    // last fragment to contain the word
+  unsigned long chunkWordCount; // word frequency within this chunk
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the word
+
+  LevelWorker *levels;          // level info for this word
+  
+  IvfWordInfo ();
+  ~IvfWordInfo ();
+  void Clear ();                // will delete levels
+};
+
+typedef map<UCArray, IvfWordInfo, DictLTUCArray> IvfWordInfoMap;
+typedef vector<IvfWordInfoMap::iterator> IvfWordInfoItArray;
+
+// tags don't require as much information
+struct IvfTagInfo {
+  unsigned long tagCount;       // tag frequency
+  unsigned long fragCount;      // number of fragments that contain the tag
+  unsigned long lastFragNum;    // last fragment to contain the tag
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the tag
+
+  IvfTagInfo ();
+  void Clear ();
+};
+
+typedef map<UCArray, IvfTagInfo, DictLTUCArray> IvfTagInfoMap;
+typedef vector<IvfTagInfoMap::iterator> IvfTagInfoItArray;
+
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define CHECK_DIV 1.5
+
+
+static FILE *ic;		// the invf chunk file
+static stdio_bitio_buffer icb;
+
+IvfWordInfoMap ivfWordInfo;
+IvfWordInfoItArray ivfWordInfoOccurOrder;
+IvfTagInfoMap ivfTagInfo;
+IvfTagInfoItArray ivfTagInfoOccurOrder;
+
+static unsigned long chunksWritten;
+static unsigned long maxMemNeeded;
+
+static unsigned long numDocs;
+static unsigned long numChunkDocs;
+
+static unsigned long numFrags;
+static unsigned long numChunkFrags;
+
+static unsigned long numWords;
+
+// the number of document numbers in the inverted file
+static unsigned long numChunkEntries;
+
+// next entry in the inverted file to check memory
+// requirements for the current chunk
+static unsigned long entryCheckPoint;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+
+
+IvfWordInfo::IvfWordInfo () {
+  levels = NULL;
+  Clear();
+}
+
+IvfWordInfo::~IvfWordInfo () {
+  if (levels != NULL) delete [] levels;
+}
+
+void IvfWordInfo::Clear () {
+  wordCount = 0;
+  chunkWordCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+
+  if (levels != NULL) {
+    delete [] levels;
+    levels = NULL;
+  }
+}
+
+
+IvfTagInfo::IvfTagInfo () {
+  Clear();
+}
+
+void IvfTagInfo::Clear () {
+  tagCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+}
+
+
+
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name) {
+  // set up the chunk file
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb",
+			  MAGIC_CHUNK, MG_MESSAGE)))
+    return COMPERROR;
+  fwrite ("    ", sizeof (u_long), 1, ic);	// Space for the maxmem
+  icb.attachFile (ic);
+  icb.encodeStart();
+
+  // reset global variables
+  ivfWordInfo.erase (ivfWordInfo.begin(), ivfWordInfo.end());
+  ivfWordInfoOccurOrder.erase (ivfWordInfoOccurOrder.begin(), ivfWordInfoOccurOrder.end());
+  ivfTagInfo.erase (ivfTagInfo.begin(), ivfTagInfo.end());
+  ivfTagInfoOccurOrder.erase (ivfTagInfoOccurOrder.begin(), ivfTagInfoOccurOrder.end());
+  
+  chunksWritten = 0;
+  maxMemNeeded = 0;
+ 
+  numDocs = 0;
+  numChunkDocs = 0;
+
+  numFrags = 0;
+  numChunkFrags = 0;
+
+  numWords = 0;
+
+  numChunkEntries = 0;
+  entryCheckPoint = (unsigned long) ((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+
+  // init the level information
+  ivfLevel.Clear();
+  ivfLevel.docTag = tagInfo.docTag;
+  ivfLevel.indexLevel = tagInfo.indexLevel;
+  IvfLevelInfo blankLevel;
+  UCArraySet::const_iterator levelHere = tagInfo.levelTags.begin();
+  UCArraySet::const_iterator levelEnd = tagInfo.levelTags.end();
+  while (levelHere != levelEnd) {
+    blankLevel.levelTag = *levelHere;
+    ivfLevel.levelInfo[*levelHere] = blankLevel;
+    ++levelHere;
+  }
+  
+  return COMPALLOK;
+}
+
+static void ProcessOpenTag (const TagInfo &tagInfo, const TextEl &el,
+			    bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for start of next fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    ++numFrags;
+    ++numChunkFrags;
+    inFrag = true;
+  }
+  
+  // update tag stats
+  IvfTagInfo &i = ivfTagInfo[el.tagName];
+  if (i.tagCount == 0) {
+    // new tag, add to list of iterators
+    IvfTagInfoMap::iterator iIt = ivfTagInfo.find (el.tagName);
+    ivfTagInfoOccurOrder.push_back (iIt);
+  }
+  ++i.tagCount;
+
+  // all open tags count as new tags
+  ++numChunkEntries;
+  ++i.fragCount;
+  ++i.chunkFragCount;
+  i.lastFragNum = numFrags;
+  
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    ++levelIt->second.numEntries;
+    levelIt->second.workInLevel = true;
+  }
+}
+
+static void ProcessCloseTag (const TagInfo &tagInfo, const TextEl &el,
+			     bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for end of fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    inFrag = false;
+  }
+
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    (*levelIt).second.workInLevel = false;
+  }
+}
+
+static void ProcessText (const TagInfo &tagInfo, const TextEl &el,
+			 bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // make sure this text is to be indexed
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  UCArray word;
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+  // Each token is then inserted into the set if it does
+  // not exist or has it's frequency count incremented if it does.
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexWord (textHere, textEnd, word);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (!word.empty()) {
+      ++numWords;
+
+      if (wordLevelIndex) {
+	++numFrags;
+	++numChunkFrags;
+      }
+      
+      // update word stats
+
+      IvfWordInfo &i = ivfWordInfo[word];
+      if (i.wordCount == 0) {
+	// new word
+	// add to list of iterators
+	IvfWordInfoMap::iterator iIt = ivfWordInfo.find (word);
+	ivfWordInfoOccurOrder.push_back (iIt);
+
+	// add level information array
+	if (ivfLevel.levelInfo.size() > 0) 
+	  i.levels = new LevelWorker [ivfLevel.levelInfo.size()];
+      }
+
+      ++i.wordCount;
+      ++i.chunkWordCount;
+      if (numFrags > i.lastFragNum) {
+	++numChunkEntries;
+	++i.fragCount;
+	++i.chunkFragCount;
+	i.lastFragNum = numFrags;
+      }
+
+      // update level information for this word
+      if (i.levels != NULL) {
+	IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+	IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+	LevelWorker *levelWorkerPtr = i.levels;
+	while (levelHere != levelEnd) {
+
+	  // check to make sure the level encompases this fragment
+	  if (!(*levelHere).second.workInLevel) {
+	    cerr << "Level tag <" << (*levelHere).first
+		 << "> does not encompass all fragments\n";
+	    exit (1);
+	  }
+	  
+	  if ((*levelHere).second.numEntries > (*levelWorkerPtr).lastLevelDocNum) {
+	    (*levelWorkerPtr).lastLevelDocNum = (*levelHere).second.numEntries;
+	    ++levelWorkerPtr->count;
+	  }
+	  
+	  ++levelHere;
+	  ++levelWorkerPtr;
+	}
+      }
+    }
+  }
+}
+
+
+static unsigned long MemoryRequired (bool wordLevelIndex) {
+  register unsigned long total = 0;
+
+  // add memory required for word entries
+  IvfWordInfoMap::const_iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::const_iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    register const IvfWordInfo &info = (*wordHere).second;
+    if (info.chunkFragCount > 0) {
+      total += BIO_Bblock_Bound (numChunkFrags, info.chunkFragCount);
+      if (!wordLevelIndex) {
+	total += info.chunkWordCount;
+      }
+    }
+    
+    ++wordHere;
+  }
+
+  // add memory required for tag entries
+  IvfTagInfoMap::const_iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::const_iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    register const IvfTagInfo &info = (*tagHere).second;
+    if (info.chunkFragCount > 0) {
+      // two d entries for each frag entry
+      unsigned long pTag = info.chunkFragCount*2;
+      total += BIO_Bblock_Bound (numChunkFrags+pTag, pTag);
+    }
+    
+    ++tagHere;
+  }
+  
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+/*
+static void PrintChunkInfo (unsigned long mem) {
+  cout << "Chunk Number: " << chunksWritten << "\n";
+  cout << "numChunkDocs " << numChunkDocs << "\n";
+  cout << "numChunkFrags " << numChunkFrags << "\n";
+  cout << "mem " << mem << "\n";
+  cout << "numWords " << ivfWordInfo.size() << "\n";
+  cout << "numTags " << ivfTagInfo.size() << "\n\n";
+
+  // output debug tag information in dictionary order
+  IvfTagInfoMap::iterator tagMapHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagMapEnd = ivfTagInfo.end();
+  unsigned long tagNum = 0;
+  while (tagMapHere != tagMapEnd) {
+    cout << (*tagMapHere).first << " " << tagNum
+	 << " " << (*tagMapHere).second.chunkFragCount << "\n";
+    ++tagNum;
+    ++tagMapHere;
+  }
+}
+*/
+
+static void OutputChunkInfo (unsigned long mem, bool /*wordLevelIndex*/) {
+  ++chunksWritten;
+
+  // sanity check
+  if (ivfWordInfo.size() != ivfWordInfoOccurOrder.size()) {
+    Message ("ERROR: Word information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfWordInfo.size(),
+	     (unsigned int)ivfWordInfoOccurOrder.size());
+    exit (1);
+  }
+  if (ivfTagInfo.size() != ivfTagInfoOccurOrder.size()) {
+    Message ("ERROR: Tag information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfTagInfo.size(),
+	     (unsigned int)ivfTagInfoOccurOrder.size());
+    exit (1);
+  }
+
+  icb.gamma_encode (numChunkDocs + 1, NULL);
+  icb.gamma_encode (numChunkFrags + 1, NULL);
+  icb.gamma_encode (mem + 1, NULL);
+  icb.gamma_encode (ivfWordInfo.size() + 1, NULL);
+  icb.gamma_encode (ivfTagInfo.size() + 1, NULL);
+
+  /*  PrintChunkInfo (mem);*/
+
+  // output word information in occurance order
+  IvfWordInfoItArray::iterator wordHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordEnd = ivfWordInfoOccurOrder.end();
+  while (wordHere != wordEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordHere)).second;
+    
+    icb.gamma_encode (ivfWordInfo.chunkWordCount + 1, NULL);
+    if (ivfWordInfo.chunkWordCount >= 2) {
+      icb.gamma_encode (ivfWordInfo.chunkFragCount, NULL);
+    }
+
+    ivfWordInfo.lastFragNum = 0;
+    ivfWordInfo.chunkWordCount = 0;
+    ivfWordInfo.chunkFragCount = 0;
+    
+    ++wordHere;
+  }
+
+  // output tag information in occurance order
+  IvfTagInfoItArray::iterator tagHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagEnd = ivfTagInfoOccurOrder.end();
+  while (tagHere != tagEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagHere)).second;
+
+    icb.gamma_encode (ivfTagInfo.chunkFragCount + 1, NULL);
+
+    ivfTagInfo.lastFragNum = 0;
+    ivfTagInfo.chunkFragCount = 0;
+    
+    ++tagHere;
+  }
+
+  numChunkDocs = 0;
+  numChunkFrags = 0;
+  numChunkEntries = 0;
+}
+
+
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  ++numDocs;
+  ++numChunkDocs;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE)
+      ProcessOpenTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == CloseTagE)
+      ProcessCloseTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == TextE)
+      ProcessText (tagInfo, *here, inFrag);
+    
+    ++here;
+  }
+
+  // check the amount of memory needed for this chunk
+  if (numChunkEntries >= entryCheckPoint) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    if (mem >= invf_buffer_size * CHECK_CLOSE) {
+      if (mem > maxMemNeeded) maxMemNeeded = mem;
+      OutputChunkInfo (mem, wordLevelIndex);
+      entryCheckPoint = (unsigned long)
+	((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+      
+    } else {
+      entryCheckPoint = (unsigned long)
+	(entryCheckPoint * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	 entryCheckPoint);
+      if (entryCheckPoint <= numChunkEntries)
+	entryCheckPoint = numChunkEntries + 1;
+    }
+  }
+  
+  return COMPALLOK;
+}
+
+
+static void CalcInvfDictSize (unsigned long &totalBytes,
+			      unsigned long &indexStringBytes) {
+  totalBytes = 0;	// The sum of the length of all words, including 
+			// the length byte
+  indexStringBytes = 0; // The amount of space required to store the 
+			// words in the diction, this takes into account 
+			// the prefixes
+  const UCArray *lastWord = NULL;
+  
+  // calculate size of word information
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    unsigned long wordSize = (*wordHere).first.size();
+    totalBytes += wordSize + 1;
+    indexStringBytes += wordSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*wordHere).first);
+    lastWord = &((*wordHere).first);
+    
+    ++wordHere;
+  }
+
+  // calculate size of tag information
+  lastWord = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    unsigned long tagSize = (*tagHere).first.size();
+    totalBytes += tagSize + 1;
+    indexStringBytes += tagSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*tagHere).first);
+    lastWord = &((*tagHere).first);
+    
+    ++tagHere;
+  }
+}
+
+
+//      OutputInvfDict ():
+//                writes out the stemmed dictionary file
+//                in the following format 
+//                        lookback value (int)
+//                        totalbytes value (int)
+//                        indexstringbytes (int)
+//                        for each word 
+//                          wordlen (4 bits)
+//                          prefix match (4 bits)
+//                          word (wordlen bytes)
+//                          word frequency (int)
+//                          word count (int)
+//
+//        Accesses outside variables:     
+//
+//        Return value...:                
+static void OutputInvfDict (char *filename) {
+  // create the dictionary header
+  invf_dict_header idh;
+  idh.word_dict_size = ivfWordInfo.size();
+  idh.tag_dict_size = ivfTagInfo.size();
+  idh.num_docs = numDocs;
+  idh.num_frags = numFrags;
+  idh.num_words = numWords;
+  idh.num_levels = ivfLevel.levelInfo.size();
+  CalcInvfDictSize (idh.total_bytes, idh.index_string_bytes);
+
+  // create the inverted dictionary file
+  FILE *sp;
+  if (!(sp = create_file (filename, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))
+    return;
+
+  // write out the dictionary header
+  if (!idh.Write (sp)) { fclose (sp); return; }
+
+  // remember where the word dictionary starts
+  idh.word_dict_start = ftell (sp);
+  
+  // output the word dictionary
+  const UCArray *lastWord = NULL;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisWord = (*wordHere).first;
+    WritePreSufStr (sp, lastWord, thisWord);
+
+    // output the number of fragments the word appeared in and the
+    // number of times the word appeared
+    WriteUL (sp, (*wordHere).second.fragCount);
+    WriteUL (sp, (*wordHere).second.wordCount);
+
+    // output frequency information for each level
+    // note that we are expecting every word to have
+    // level information
+    LevelWorker *lwHere = (*wordHere).second.levels;
+    LevelWorker *lwEnd = lwHere + idh.num_levels;
+    while (lwHere != lwEnd) {
+      WriteUL (sp, (*lwHere).count);
+      ++lwHere;
+    }
+    
+    lastWord = &thisWord;
+    ++wordHere;
+  }
+
+  // remember where the tag dictionary starts
+  idh.tag_dict_start = ftell (sp);
+  
+  // output the tag dictionary
+  const UCArray *lastTag = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisTag = (*tagHere).first;
+    WritePreSufStr (sp, lastTag, thisTag);
+
+    // output the number of fragments the tag appeared in and the
+    // number of times the tag appeared
+    WriteUL (sp, (*tagHere).second.fragCount);
+    WriteUL (sp, (*tagHere).second.tagCount);
+    
+    lastTag = &thisTag;
+    ++tagHere;
+  }
+
+  // write out the updated header
+  fseek (sp, sizeof (u_long), SEEK_SET);
+  if (!idh.Write (sp)) { fclose (sp); return; }
+  
+  fclose (sp);
+}
+
+static void OutputLevelFile (char *filename) {
+  // create the level file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_LEVEL_SUFFIX, "wb", MAGIC_INVF_LEVELS,
+			 MG_MESSAGE)))
+    return;
+
+  // write out the information
+  ivfLevel.Write (f);
+  
+  // close the file
+  fclose (f);
+}
+
+static void OutputTransFile (char *filename) {
+  // save the word number (in lastFragNum :-/)
+  int i = 0;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    (*wordHere).second.lastFragNum = i;
+    ++i; ++wordHere;
+  }
+
+  // save the tag number (in lastFragNum :-/)
+  i = 0;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    (*tagHere).second.lastFragNum = i;
+    ++i; ++tagHere;
+  }
+  
+  // create the translation file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))
+    return;
+
+  stdio_bitio_buffer buffer(f);
+  buffer.encodeStart();
+
+  // write out the word translation table
+  unsigned long wordDictSize = ivfWordInfoOccurOrder.size();
+  IvfWordInfoItArray::iterator wordItHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordItEnd = ivfWordInfoOccurOrder.end();
+  unsigned long oN = 0;
+  while (wordItHere != wordItEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordItHere)).second;
+    buffer.binary_encode (ivfWordInfo.lastFragNum + 1, wordDictSize + 1, NULL);
+    ++oN;
+    ++wordItHere;
+  }
+
+  // write out the tag translation table
+  unsigned long tagDictSize = ivfTagInfoOccurOrder.size();
+  IvfTagInfoItArray::iterator tagItHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagItEnd = ivfTagInfoOccurOrder.end();
+  while (tagItHere != tagItEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagItHere)).second;
+    buffer.binary_encode (ivfTagInfo.lastFragNum + 1, tagDictSize + 1, NULL);
+    ++oN;
+    ++tagItHere;
+  }
+
+  // finish encoding and close the file
+  buffer.encodeDone();
+  fclose (f);
+}
+
+#ifndef SILENT
+static void PrintStats () {
+  Message ("Inverted buffer size:          %8u bytes\n", invf_buffer_size);
+  Message ("Max memory needed for 1 chunk: %8u bytes\n", maxMemNeeded);
+
+  Message ("Number of chunks written:      %8u\n", chunksWritten);
+  Message ("Number of documents:           %8u\n", numDocs);
+  Message ("Number of fragments:           %8u\n", numFrags);
+  Message ("Number of words:               %8u\n", numWords);
+
+  Message ("Size of word dictionary:       %8u\n", ivfWordInfo.size());
+  Message ("Size of tag dictionary:        %8u\n", ivfTagInfo.size());
+}
+#endif
+
+int done_ivf_1 (const TagInfo &tagInfo, char * filename) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+
+  // output the last chunk
+  if (numChunkDocs > 0) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    OutputChunkInfo (mem, wordLevelIndex);
+    if (mem > maxMemNeeded) maxMemNeeded = mem;
+  }
+  
+  // write out and encoded 1 to say there are no more chunks
+  icb.gamma_encode (1, NULL);
+  icb.encodeDone ();
+
+  // write out the maximum memory required and close the file
+  fseek (ic, sizeof (long), 0);
+  WriteUL (ic, maxMemNeeded);
+  fclose (ic);
+
+  // output the inverted dictionary
+  OutputInvfDict (filename);
+
+  // write out the translation file
+  OutputTransFile (filename);
+
+  // output the level information
+  OutputLevelFile (filename);
+  
+  // output statistics
+#ifndef SILENT
+  PrintStats ();
+#endif
+  
+  msg_prefix = temp_str;
+
+  return COMPALLOK;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass2.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/ivf.pass2.cpp	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * ivf.pass2.cpp -- Memory efficient pass 2 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include <stdio.h>
+
+#if defined __WIN32__
+# include <process.h>
+# include <io.h>
+# define getpid _getpid
+# define unlink _unlink
+#else
+# include <unistd.h>
+# include "non_ansi.h"
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_random.h"
+#include "bitio_m_stdio.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+#include "perf_hash.h"
+#include "string.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+#ifdef USE_LONG_LONG
+#define SEEK_X seek_LL
+#define TELL_X tell_LL
+#else
+#define SEEK_X seek
+#define TELL_X tell
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+#endif
+
+
+static unsigned long numDocs = 0;
+static unsigned long numChunkDocs = 0;
+static unsigned long numDocsInChunk = 0;
+
+static unsigned long numFrags = 0;
+static unsigned long numFragsInChunk = 0;
+static unsigned long chunkStartFragNum = 0;
+
+
+
+
+struct BitPtr {
+  unsigned long start;
+  unsigned long here;
+  unsigned long lastFragNum;
+  unsigned long lgB;
+  
+  void Clear () { start = here = lastFragNum = lgB = 0; }
+  BitPtr () { Clear(); }
+};
+
+class WordBitPtrs {
+protected:
+  unsigned long numWords;
+  unsigned long numTags;
+  unsigned long size;
+  BitPtr *wordBitPtrs;
+
+  void CheckBufOverrun (unsigned long num) {
+    if (wordBitPtrs[num].here > wordBitPtrs[num+1].start) {
+      cerr << "numDocs: " << numDocs << "\n";
+      cerr << "numChunkDocs: " << numChunkDocs << "\n";
+      cerr << "numDocsInChunk: " << numDocsInChunk << "\n";
+      cerr << "numFrags: " << numFrags << "\n";
+      cerr << "numFragsInChunk: " << numFragsInChunk << "\n";
+      cerr << "chunkStartFragNum: " << chunkStartFragNum << "\n";
+      cerr << "num: " << num << "\n";
+      cerr << "[num].start: " << wordBitPtrs[num].start << "\n";
+      cerr << "[num].here: " << wordBitPtrs[num].here << "\n";
+      cerr << "[num+1].start: " << wordBitPtrs[num+1].start << "\n";
+      FatalError (1, "Bit buffer overrun");
+    }
+  }
+  
+public:
+  void Clear ();
+  WordBitPtrs () { wordBitPtrs = NULL; Clear(); }
+  ~WordBitPtrs ();
+  void SetSize (unsigned long _numWords,
+		unsigned long _numTags);
+
+  void ResetPtrs () {
+    if (wordBitPtrs == NULL) return;
+    unsigned long i;
+    for (i=0; i<size; ++i) wordBitPtrs[i].Clear();
+  }
+  
+  BitPtr &GetWordBitPtr (unsigned long wordNum)
+    { return wordBitPtrs[wordNum]; }
+  unsigned long &GetWordStart (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].start; }
+  unsigned long &GetWordHere (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].here; }
+  void CheckWordBufOverrun (unsigned long wordNum)
+    { CheckBufOverrun (wordNum); }
+
+  BitPtr &GetTagBitPtr (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords]; }
+  unsigned long &GetTagStart (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].start; }
+  unsigned long &GetTagHere (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].here; }
+  void CheckTagBufOverrun (unsigned long tagNum)
+    { CheckBufOverrun (tagNum + numWords); }
+
+  BitPtr &GetEndBitPtr ()
+    { return wordBitPtrs[size-1]; }
+  unsigned long &GetEndStart ()
+    { return wordBitPtrs[size-1].start; }
+  unsigned long &GetEndHere ()
+    { return wordBitPtrs[size-1].here; }
+};
+
+
+struct IP2TagInfo {
+  bool inTag;
+  unsigned long startFrag;
+  unsigned long tagNum;
+
+  IP2TagInfo () {
+    inTag = false;
+    startFrag = 0;
+    tagNum = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, IP2TagInfo, DictLTUCArray> TagMapDict;
+
+
+// class to handle the translation of occurrence order
+// to dictionary order for words and tags
+class OccurToDictConverter {
+protected:
+  unsigned long pos;
+  unsigned long val;
+  FILE *transFile;
+  random_bitio_buffer rbs;
+
+  unsigned long wordDictSize;
+  unsigned long tagDictSize;
+
+  void SeekStart ();
+  unsigned long TranslateNum (unsigned long num);
+  
+public:
+  OccurToDictConverter ();
+  ~OccurToDictConverter ();
+
+  void Open (char *filename, unsigned long _wordDictSize,
+	     unsigned long _tagDictSize);
+
+  // Close frees all allocated memory
+  void Close ();
+
+  unsigned long TranslateWord (unsigned long occurNum)
+    { return TranslateNum (occurNum); }
+  unsigned long TranslateTag (unsigned long occurNum)
+    { return TranslateNum (occurNum+wordDictSize); }
+};
+
+
+struct InvfStateRec {
+  mg_ullong start;
+  mg_ullong here;
+  unsigned long lastFragNum;
+  unsigned long B;
+
+  void Clear () {
+    start = here = 0;
+    lastFragNum = B = 0;
+  }
+  InvfStateRec () { Clear (); }
+};
+
+
+#define ISR_SIZE 1024
+
+class InvfStateCache {
+protected:
+  InvfStateRec recCache [ISR_SIZE];
+  unsigned long startNum;
+
+  FILE *stateFile;
+
+  void ClearCache () {
+    unsigned int i = 0;
+    for (i=0; i<ISR_SIZE; ++i) recCache[i].Clear();
+  }
+  
+public:
+  InvfStateCache ();
+  ~InvfStateCache ();
+  
+  void Open (char *filename);
+  void Close ();
+
+  // previous references to state records may be
+  // invalidated calling GetRec
+  InvfStateRec &GetRec (unsigned long num);
+};
+
+
+static invf_dict_header idh;
+static WordBitPtrs bitPtrs;
+
+static FILE *chunkFile = NULL;
+static stdio_bitio_buffer chunkBuf;
+
+static unsigned long ivfMemBufSize = 0;
+static char *ivfMemBuf = NULL;
+
+// word and tag dictionaries. a map is used for the tag dictionary
+// as it should never be very big (and the perfect hash function
+// sometimes has trouble with small values).
+static perf_hash_data *wordHashDict = NULL;
+static TagMapDict tagMapDict;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+static OccurToDictConverter occurConvert;
+
+// information about the state of the inverted file
+static InvfStateCache invfState;
+
+static char collectFilename[512];
+
+
+void WordBitPtrs::Clear () {
+  numWords = 0;
+  numTags = 0;
+  size=0;
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+  wordBitPtrs = NULL;
+}
+
+WordBitPtrs::~WordBitPtrs () {
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+}
+
+void WordBitPtrs::SetSize (unsigned long _numWords,
+			   unsigned long _numTags){
+  Clear();
+  numWords = _numWords;
+  numTags = _numTags;
+  size = numWords + numTags + 1;
+  wordBitPtrs = new BitPtr [size];
+}
+
+
+void OccurToDictConverter::SeekStart () {
+  if (transFile == NULL) return;
+  rbs.SEEK_X (sizeof (unsigned long) * 8);
+  pos = 0;
+}
+
+unsigned long OccurToDictConverter::TranslateNum (unsigned long num) {
+  if (num < pos) SeekStart ();
+  while (pos <= num) {
+    if (pos < wordDictSize)
+      val = rbs.binary_decode (wordDictSize + 1, NULL) - 1;
+    else 
+      val = rbs.binary_decode (tagDictSize + 1, NULL) - 1;
+    ++pos;
+  }
+  return val;
+}
+
+OccurToDictConverter::OccurToDictConverter () {
+  pos = 0;
+  val = 0;
+  transFile = NULL;
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+OccurToDictConverter::~OccurToDictConverter () {
+  if (transFile != NULL) Close ();
+}
+
+void OccurToDictConverter::Open (char *filename, unsigned long _wordDictSize,
+				 unsigned long _tagDictSize) {
+  if (transFile != NULL) Close ();
+
+  wordDictSize = _wordDictSize;
+  tagDictSize = _tagDictSize;
+
+  transFile = open_file (filename, INVF_CHUNK_TRANS_SUFFIX, "rb",
+			 MAGIC_CHUNK_TRANS, MG_ABORT);
+  rbs.attachFile (transFile, RND_BUF_SIZE);
+  SeekStart ();
+  val = 0;
+}
+
+void OccurToDictConverter::Close () {
+  if (transFile == NULL) return;
+
+  rbs.done ();
+  fclose (transFile);
+  transFile = NULL;
+  pos = 0;
+  val = 0;
+
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+
+
+
+InvfStateCache::InvfStateCache () {
+  startNum = 0;
+  stateFile = NULL;
+}
+
+InvfStateCache::~InvfStateCache () {
+  if (stateFile != NULL) Close ();
+}
+  
+void InvfStateCache::Open (char *filename) {
+  if (stateFile != NULL) Close();
+
+  // open the state file
+  char path[512];
+  sprintf (path, FILE_NAME_FORMAT ".%ld", get_basepath (), filename,
+           ".invf.state", (long) getpid ());
+  if (!(stateFile = fopen (path, "wb+"))) {
+    Message ("Unable to create \"%s\"", path);
+    exit (1);
+  }
+  unlink (path); // file will be deleted after it is closed
+  // reset the buffer
+  startNum = 0;
+  ClearCache();
+}
+
+void InvfStateCache::Close () {
+  if (stateFile == NULL) return;
+  fclose (stateFile);
+  stateFile = NULL;
+  startNum = 0;
+}
+
+InvfStateRec &InvfStateCache::GetRec (unsigned long num) {
+  // see if cached
+  if ((num >= startNum) && (num < startNum + ISR_SIZE))
+    return recCache[num-startNum];
+
+  // not cached, write out this lot of records and read in
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fwrite ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  // read in the new set of records
+  ClearCache ();
+  startNum = num - (num % ISR_SIZE);
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fread ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  return recCache[num-startNum];
+}
+
+
+
+static void ClearCharBuf (char *buf, unsigned long size) {
+  char *end = buf + size;
+  while (buf != end) *buf++ = 0;
+}
+
+static void ReadWordDict (char *filename) {
+  // read in the perfect hash function for the word dictionary
+  FILE *wordHashFile = open_file (filename, INVF_DICT_HASH_SUFFIX, "rb",
+				  MAGIC_HASH, MG_ABORT);
+  if (!(wordHashDict = read_perf_hash_data (wordHashFile))) {
+    FatalError (1, "Unable to read in hash data for word dictionary");
+  }
+  fclose (wordHashFile);
+}
+
+static void ReadTagDict (char *filename, invf_dict_header &_idh) {
+  // open the file
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+  
+  unsigned long tagNum;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < _idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    tagMapDict[thisEl.el].tagNum = tagNum;
+  }
+  
+  fclose (dictFile);
+}
+
+static void ReadLevelFile (char *filename) {
+  FILE *f;
+  f = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+		 MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Read (f);
+  fclose (f);
+}
+
+void CheckIntOverflow (mg_ullong totalIBits, mg_ullong lastTotalIBits) {
+  if (totalIBits < lastTotalIBits) {
+    fprintf(stderr, "ERROR: The totalIBits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+    if (sizeof (mg_ullong) < 8) {
+      fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+    }
+    fprintf(stderr, "       Build aborted.\n");
+    exit(1);
+  }
+}
+
+// assumes the inverted file state file has been opened
+static void InitInvfState (char *filename,
+			   invf_dict_header &_idh,
+			   InvfStateCache &_invfState,
+			   mg_ullong &totalIBits,
+			   bool wordLevelIndex) {
+  // read in the dictionary, setting inverted state information
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the word dictionary
+  fseek (dictFile, _idh.word_dict_start, SEEK_SET);
+
+  // add the word entries
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (_idh.num_levels);
+  unsigned long dictWordNum, p;
+  mg_ullong lastTotalIBits;
+  unsigned long N = _idh.num_frags;
+  for (dictWordNum=0; dictWordNum<_idh.word_dict_size; ++dictWordNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next word and associated information
+    wordEl.Read (dictFile, _idh.num_levels);
+
+    // update the state record
+    p = wordEl.frag_occur;
+    InvfStateRec &wisr = _invfState.GetRec (dictWordNum);
+    wisr.start = totalIBits;
+    wisr.here = totalIBits;
+    wisr.B = BIO_Bblock_Init (N, p);
+    
+    // add the length of the fragment numbers
+    totalIBits += BIO_Bblock_Bound_b (N, p, wisr.B);
+
+    // if needed, add the length of the fragment frequency information
+    if (!wordLevelIndex)
+      totalIBits += BIO_Gamma_Bound (wordEl.freq, wordEl.frag_occur);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+
+  // add the tag entries
+  dict_el tagEl;
+  unsigned long dictTagNum;
+  N = _idh.num_frags;
+  for (dictTagNum=0; dictTagNum<_idh.tag_dict_size; ++dictTagNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next tag and associated information
+    tagEl.Read (dictFile);
+
+    // update the state record
+    p = tagEl.frag_occur*2;
+    InvfStateRec &tisr = _invfState.GetRec (dictTagNum + _idh.word_dict_size);
+    tisr.start = totalIBits;
+    tisr.here = totalIBits;
+    tisr.B = BIO_Bblock_Init (N+p, p);
+    
+    // add the length of the fragment numbers (two numbers for each
+    // tag, one for start and one for end)
+    totalIBits += BIO_Bblock_Bound_b (N+p, p, tisr.B);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  fclose (dictFile);
+}
+
+/*
+// assumes the chunk tag information has been placed in .first
+static void PrintChunkInfo (unsigned long chunkMem,
+			    unsigned long numChunkWords,
+			    unsigned long numChunkTags) {
+  static unsigned long chunksRead = 0;
+  ++chunksRead;
+  cout << "Chunk Number: " << chunksRead << "\n";
+  cout << "numChunkDocs " << numDocsInChunk << "\n";
+  cout << "numChunkFrags " << numFragsInChunk << "\n";
+  cout << "mem " << chunkMem << "\n";
+  cout << "numWords " << numChunkWords << "\n";
+  cout << "numTags " << numChunkTags << "\n\n";
+
+  TagMapDict::iterator tagMapHere = tagMapDict.begin();
+  TagMapDict::iterator tagMapEnd = tagMapDict.end();
+  while (tagMapHere != tagMapEnd) {
+    unsigned long tagMapNum = (*tagMapHere).second.tagNum;
+    cout << (*tagMapHere).first << " " << tagMapNum << " "
+	 << bitPtrs.GetTagBitPtr(tagMapNum).here << "\n";
+    ++tagMapHere;
+  }
+}
+*/
+
+void ReadChunk (invf_dict_header &_idh, bool wordLevelIndex) {
+  // reset globals
+  numChunkDocs = 0;
+  chunkStartFragNum = numFrags;
+
+  // read in information about this chunk
+  numDocsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  if (numDocsInChunk == 0)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  numFragsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long chunkMem = chunkBuf.gamma_decode (NULL) - 1;
+
+  if (chunkMem > ivfMemBufSize)
+    FatalError (1, "Chunk memory size is greater than maximum");
+  
+  unsigned long numChunkWords = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long numChunkTags = chunkBuf.gamma_decode (NULL) - 1;
+
+  
+  // reset stuff
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+  bitPtrs.ResetPtrs();  
+
+  // read in the entries in occurrence order storing the
+  // "chunkWordCount" in "start" and the "chunkFragCount"
+  // in "here"
+  unsigned long numOccur;
+  unsigned long wordNum;
+  for (numOccur=0; numOccur<numChunkWords; ++numOccur) {
+    wordNum = occurConvert.TranslateWord (numOccur);
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    wordPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    if (wordPtr.start >= 2)
+      wordPtr.here = chunkBuf.gamma_decode (NULL);
+    else wordPtr.here = wordPtr.start;
+  }
+  unsigned long tagNum;
+  for (numOccur=0; numOccur<numChunkTags; ++numOccur) {
+    tagNum = occurConvert.TranslateTag (numOccur);
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    // only chunkFragCount is encoded for tags
+    tagPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    tagPtr.here = tagPtr.start;
+  }
+
+  /*  PrintChunkInfo (chunkMem, numChunkWords, numChunkTags);*/
+  
+  // create the bit ptrs in dictionary order
+  unsigned long totalIBits = 0; // only dealing with memory
+  unsigned long chunkWordCount, chunkFragCount;
+  for (wordNum=0; wordNum<_idh.word_dict_size; ++wordNum) {
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    chunkWordCount = wordPtr.start;
+    chunkFragCount = wordPtr.here;
+    wordPtr.start = totalIBits;
+    wordPtr.here = totalIBits;
+    wordPtr.lastFragNum = chunkStartFragNum;
+    wordPtr.lgB = 0;
+    if (chunkWordCount > 0) {
+      wordPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk,
+						   chunkFragCount));
+      totalIBits += BIO_Bblock_Bound (numFragsInChunk, chunkFragCount);
+      // use unary encoding for memory buffer encoding of fragment freq
+      if (!wordLevelIndex) {
+	totalIBits += chunkWordCount;
+      }
+    }
+  }
+  for (tagNum=0; tagNum<_idh.tag_dict_size; ++tagNum) {
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    chunkFragCount = tagPtr.here;
+    tagPtr.start = totalIBits;
+    tagPtr.here = totalIBits;
+    tagPtr.lastFragNum = chunkStartFragNum;
+    tagPtr.lgB = 0;
+    if (chunkFragCount > 0) {
+      unsigned long pTag = chunkFragCount*2;
+      tagPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk+pTag,
+						  pTag));
+      unsigned long bLen = BIO_Bblock_Bound (numFragsInChunk+pTag,
+					     pTag);
+//        cout << tagNum + _idh.word_dict_size << " ";
+//        cout << "numFrags: " << numFragsInChunk
+//    	   << " chunkFragCount: " << chunkFragCount
+//    	   << " B: " << 1 << tagPtr.lgB
+//    	   << " blen: " << blen << "\n";
+      totalIBits += bLen;
+    }
+  }
+  bitPtrs.GetEndStart() = totalIBits;
+  bitPtrs.GetEndHere() = totalIBits;
+
+  if ((totalIBits + 7ul) >> 3ul > chunkMem) {
+    cerr << "totalIBits: " << totalIBits << "\n";
+    cerr << "bytes: " << ((totalIBits + 7ul) >> 3ul) << "\n";
+    cerr << "chunkMem: " << chunkMem << "\n";
+    FatalError (1, "Pointers exceed buffer size");
+  }
+}
+
+
+
+
+int init_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // read in compressed dictionary header
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  idh.Read (dictFile);
+  fclose (dictFile);
+
+  // set the size of the bit ptrs
+  bitPtrs.SetSize (idh.word_dict_size, idh.tag_dict_size);
+
+  // open the chunk file and read in the maximum memory needed
+  // for the inverted memory buffer
+  chunkFile = open_file (filename, INVF_CHUNK_SUFFIX, "rb",
+			 MAGIC_CHUNK, MG_ABORT);
+  ReadUL (chunkFile, ivfMemBufSize);
+  chunkBuf.attachFile (chunkFile);
+
+  // allocate memory for the inverted buffer
+  ivfMemBuf = new char [ivfMemBufSize];
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+
+  // read in the word dictionary
+  ReadWordDict (filename);
+
+  // read in the tag dictionary
+  ReadTagDict (filename, idh);
+
+  // read in the level information
+  ReadLevelFile (filename);
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  
+  // set up the translation table file
+  occurConvert.Open (filename, idh.word_dict_size, idh.tag_dict_size);
+
+  // reset some globals
+  numDocs = 0;
+  numChunkDocs = 0;
+  numDocsInChunk = 0;
+  numFrags = 0;
+  numFragsInChunk = 0;
+  chunkStartFragNum = 0;
+
+  strcpy (collectFilename, filename);
+  
+  
+  // create the inverted file
+  mg_ullong totalIBits = 0;
+  FILE *invfFile = create_file (filename, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_ABORT);
+  totalIBits += sizeof (unsigned long) * 8; // magic number
+  totalIBits += 8 * 200;                    // 200 byte gap -- why??????
+  fclose (invfFile);
+
+  // init the inverted file state cache
+  invfState.Open (filename);
+  InitInvfState (filename, idh, invfState, totalIBits, wordLevelIndex);
+
+  return COMPALLOK;
+}
+
+static void CloseTextTag (IP2TagInfo &tInfo, const UCArray &/*tagName*/) {
+  if (!tInfo.inTag) return;
+
+    // add this tag to the inverted list
+  BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tInfo.tagNum);
+  unsigned long endFrag = numFrags;
+  int b = 1 << tagBitPtr.lgB;
+
+  /*
+  cout << (tInfo.tagNum+idh.word_dict_size) << " \"<" << tagName << ">\" "
+       << tInfo.startFrag << " " << endFrag << "\n";
+  */
+    
+  mems_bitio_buffer buffer ((u_char *) ivfMemBuf, tagBitPtr.here);
+  buffer.bblock_encode (tInfo.startFrag - tagBitPtr.lastFragNum + 1,
+			b, NULL);
+  buffer.bblock_encode (endFrag - tInfo.startFrag + 1, b, NULL);
+  tagBitPtr.lastFragNum = endFrag;
+  tagBitPtr.here = buffer.position();
+  buffer.encodeDone();
+  
+  // check for buffer overrun
+  bitPtrs.CheckTagBufOverrun (tInfo.tagNum);
+  
+  // reset information about this tag
+  tInfo.inTag = false;
+  tInfo.startFrag = 0;
+}
+
+static void ProcessOpenTag (const TextEl &el, bool &inFrag) {
+  // close tag if open
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  if (tInfo.inTag) CloseTextTag (tInfo, el.tagName);
+
+  // open this tag
+  tInfo.inTag = true;
+  tInfo.startFrag = numFrags;
+
+  // check for start of next fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    ++numFrags;
+    inFrag = true;
+  }
+}
+
+static void ProcessCloseTag (const TextEl &el, bool &inFrag) {
+  // check for end of fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    inFrag = false;
+  }
+
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  CloseTextTag (tInfo, el.tagName);
+}
+
+static void ProcessText (const TextEl &el, bool &inFrag) {
+  // make sure this text is to be indexed
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  unsigned char mgWord[MAXSTEMLEN + 1];
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexMGWord (textHere, textEnd, mgWord);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (mgWord[0] > 0) {
+      if (wordLevelIndex) ++numFrags;
+
+      unsigned long wordNum = perf_hash (wordHashDict, mgWord);
+
+      /*
+      cout << wordNum << " \"";
+      cout.write (mgWord+1, *mgWord);
+      cout << "\" " << numFrags << "\n";
+      */
+      
+      // add this word to the inverted list
+      BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+      unsigned long fragNum = numFrags;
+      int b = 1 << wordBitPtr.lgB;
+  
+      mems_bitio_buffer buffer ((u_char *) ivfMemBuf, wordBitPtr.here);
+
+      // note: this assumes that fragments don't carry over between
+      // chunks (which they don't because all tags are closed at the
+      // end of each document and chunks are based on document
+      // boundaries), i.e. the first fragment number must be greater
+      // than the starting fragment number of the chunk.
+      if (fragNum > wordBitPtr.lastFragNum) {
+	buffer.bblock_encode ((fragNum - wordBitPtr.lastFragNum - 1) + 1,
+			      b, NULL);
+	if (!wordLevelIndex) buffer.encodeBit (1); // freq = 1
+	
+      } else if (!wordLevelIndex) {
+	// add one to the frequency count for this word
+	buffer.seek (buffer.position()-1);
+	buffer.encodeBit (0); // unary encoding -- last = 1
+	buffer.encodeBit (1);
+      }
+      
+      wordBitPtr.lastFragNum = fragNum;
+      wordBitPtr.here = buffer.position();
+      buffer.encodeDone();
+      
+      // check for buffer overrun
+      bitPtrs.CheckWordBufOverrun (wordNum);
+    }
+  }
+}
+
+// combine the in memory inverted buffer with the disk
+// based inverted file
+static void DiskMerge (char *filename) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+
+      // make sure we have something to process
+  if (numChunkDocs <= 0) return;
+    
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb+",
+			      MAGIC_INVF, MG_ABORT);
+  random_bitio_buffer invfOutBuf (invfFile);
+
+  // set up to decode the entries in memory
+  mems_bitio_buffer memInBuf ((u_char *) ivfMemBuf, 0);
+
+  // write out the word information
+  unsigned long wordNum;
+  int b;
+  unsigned long currFragNum;
+  unsigned long delta;
+  unsigned long currFreq;
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &wordDiskState = invfState.GetRec (wordNum);
+    invfOutBuf.SEEK_X (wordDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+    memInBuf.seek(wordBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currFragNum = chunkStartFragNum;
+    while (memInBuf.position() < wordBitPtr.here) {
+      // decode word entry
+      b = 1 << wordBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL);
+      currFragNum += delta;
+      if (!wordLevelIndex) currFreq = memInBuf.unary_decode (NULL);
+      else currFreq = 1;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currFragNum-wordDiskState.lastFragNum,
+				wordDiskState.B, NULL);
+      if (!wordLevelIndex) invfOutBuf.gamma_encode (currFreq, NULL);
+      wordDiskState.lastFragNum = currFragNum;
+    }
+
+    wordDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  // write out the tag information
+  unsigned long tagNum;
+  unsigned long currTagStart;
+  unsigned long currTagEnd;
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &tagDiskState = invfState.GetRec (tagNum+idh.word_dict_size);
+    invfOutBuf.SEEK_X (tagDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tagNum);
+    memInBuf.seek(tagBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currTagEnd = chunkStartFragNum;
+    while (memInBuf.position() < tagBitPtr.here) {
+      // decode tag entry
+      b = 1 << tagBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagStart = currTagEnd + delta;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagEnd = currTagStart + delta;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currTagStart-tagDiskState.lastFragNum+1,
+				tagDiskState.B, NULL);
+      invfOutBuf.bblock_encode (currTagEnd-currTagStart+1,
+				tagDiskState.B, NULL);
+
+      tagDiskState.lastFragNum = currTagEnd;
+    }
+
+    tagDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  memInBuf.done();
+
+  invfOutBuf.encodeDone();
+  fclose (invfFile);
+}
+
+
+int process_ivf_2 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  // get next chunk information if need to. the chunk information
+  // is needed before the first document is processed
+  if (numChunkDocs >= numDocsInChunk) ReadChunk (idh, wordLevelIndex);
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // process this element
+    if ((*here).elType == OpenTagE) ProcessOpenTag (*here, inFrag);
+    else if ((*here).elType == CloseTagE) ProcessCloseTag (*here, inFrag);
+    else ProcessText (*here, inFrag);
+    
+    ++here;
+  }
+
+  // close off any unclosed tags
+  TagMapDict::iterator tdHere = tagMapDict.begin();
+  TagMapDict::iterator tdEnd = tagMapDict.end();
+  while (tdHere != tdEnd) {
+    CloseTextTag ((*tdHere).second, (*tdHere).first);
+    ++tdHere;
+  }
+  
+  // we've processed one more document
+  ++numDocs;
+  ++numChunkDocs;
+
+  // merge the memory based inverted file with the one on
+  // disk if this is the end of this chunk
+  if (numChunkDocs >= numDocsInChunk) DiskMerge (collectFilename);
+  
+  return COMPALLOK;
+}
+
+
+static void CondenseInvfFile (char *filename, unsigned long &bytesOutput) {
+  FILE *inInvfFile = open_file (filename, INVF_SUFFIX, "rb",
+				MAGIC_INVF, MG_ABORT);
+  FILE *outInvfFile = open_file (filename, INVF_SUFFIX, "rb+",
+				 MAGIC_INVF, MG_ABORT);
+
+  // skip the magic number
+  fseek (outInvfFile, sizeof (unsigned long), SEEK_SET);
+  
+  // write the inverted file header -- use defaults for most things
+  invf_file_header ifh;
+  ifh.no_of_words = idh.word_dict_size;
+  ifh.no_of_tags = idh.tag_dict_size;
+  ifh.word_level_index = (ivfLevel.indexLevel.empty()) ? 1 : 0;
+  ifh.Write (outInvfFile);
+
+  bytesOutput = ftell (outInvfFile);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = ifh.no_of_words + ifh.no_of_tags;
+  unsigned long entryNum;
+  mg_ullong lastStart = 0;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // overrun check
+    if (stateRec.start < lastStart)
+      FatalError (1, "Inverted file Buffer overrun");
+    lastStart = stateRec.start;
+    
+    unsigned long oldEntryStart = stateRec.start >> 3;
+    unsigned long oldEntryStartOver = stateRec.start & 7; // should be 0
+    unsigned long oldEntryEnd = (stateRec.here + 7) >> 3;  // byte after end
+    unsigned long oldEntryEndOver = stateRec.here & 7;
+
+    fseek (inInvfFile, oldEntryStart, SEEK_SET);
+
+    stateRec.here -= stateRec.start;
+    stateRec.start = (mg_ullong)bytesOutput * 8 + oldEntryStartOver;
+    stateRec.here += stateRec.start;
+    while (oldEntryStart < oldEntryEnd) {
+      unsigned char c = getc (inInvfFile);
+      if (oldEntryStart == oldEntryEnd - 1) {
+	u_char ands[8] =
+	{0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	c &= ands[oldEntryEndOver];
+      }
+      putc (c, outInvfFile);
+      ++bytesOutput;
+      ++oldEntryStart;
+    }
+  }  
+
+  fclose (inInvfFile);
+
+#ifdef __WIN32__
+  if (_chsize (_fileno (outInvfFile), bytesOutput) != 0)
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (outInvfFile), bytesOutput);
+#endif
+
+  fclose (outInvfFile);
+}
+
+static void OutputInvfIdx (char *filename, unsigned long invfNumBytes) {
+  FILE *invfIdxFile = create_file (filename, INVF_IDX_SUFFIX, "wb",
+				   MAGIC_INVI, MG_ABORT);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = idh.word_dict_size + idh.tag_dict_size;
+  unsigned long entryNum;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // assumes that inverted entries start at beginning of each byte
+    if (!WriteUL (invfIdxFile, (stateRec.start >> 3))) break;
+  }
+
+  WriteUL (invfIdxFile, invfNumBytes);
+
+  fclose (invfIdxFile);
+}
+
+
+int done_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // close most open files
+  if (chunkFile != NULL) {
+    chunkBuf.done();
+    fclose (chunkFile);
+    chunkFile = NULL;
+  }
+  occurConvert.Close();
+  
+  // free allocated memory
+  bitPtrs.Clear();
+  if (ivfMemBuf != NULL) { delete [] ivfMemBuf; ivfMemBuf = NULL; }
+  free_perf_hash (wordHashDict);
+  wordHashDict = NULL;
+  tagMapDict.erase (tagMapDict.begin(), tagMapDict.end());
+
+  // condense the inverted file and truncate it
+  // this function also writes out the inverted header
+  unsigned long invfNumBytes = 0;
+  CondenseInvfFile (filename, invfNumBytes);
+
+  OutputInvfIdx (filename, invfNumBytes);
+
+  // close the rest of the open files
+  invfState.Close ();
+  
+  return COMPALLOK;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/locallib.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/locallib.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/locallib.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+/**************************************************************************
+ *
+ * locallib.cpp -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; ++i)
+    total += A[i];
+  for (i = 0; i < n; ++i)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; --i)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      --n;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  ++N;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	++limit;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * /*mem*/,
+	      u_long * disk) {
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; ++i)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  memset (cfh->huff_words_size, '\0', sizeof (cfh->huff_words_size));
+
+  if ((int)fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; ++i)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/locallib.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/locallib.h	(revision 21112)
@@ -0,0 +1,45 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg.special.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg.special.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg.special.cpp	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * mg.special.cpp -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "mg.h"
+#include "build.h"
+
+int init_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
+
+int process_special (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  // print out the document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE) {
+      cout << "** OpenTag  \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n"; 
+    } else if ((*here).elType == CloseTagE) {
+      cout << "** CloseTag \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n";
+    } else {
+      cout << "** Text     \"" << (*here).text << "\"\n";
+    }
+    
+    ++here;
+  }
+  
+  return COMPALLOK;
+}
+
+int done_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.cpp	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * mg_errors.cpp -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg_files.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg_files.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg_files.cpp	(revision 21112)
@@ -0,0 +1,259 @@
+/**************************************************************************
+ *
+ * mg_files.cpp -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      delete []basepath;
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = new char[strlen (bp) + 2];
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mg_files.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mg_files.h	(revision 21112)
@@ -0,0 +1,343 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXT_LEVELS	GEN_MAGIC('M','G','t','L')
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVF_LEVELS	GEN_MAGIC('M','G','I','L')
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The document levels in the text */
+#ifdef SHORT_SUFFIX
+# define TEXT_LEVEL_SUFFIX	 	".tl"
+#else
+# define TEXT_LEVEL_SUFFIX	 	".text.level"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The level information for the inverted text */
+#ifdef SHORT_SUFFIX
+# define INVF_LEVEL_SUFFIX              ".il"
+#else
+# define INVF_LEVEL_SUFFIX              ".invf.level"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.1	(revision 21112)
@@ -0,0 +1,188 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mgpp_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
@@ -0,0 +1,1021 @@
+/**************************************************************************
+ *
+ * mgpp_compression_dict.cpp -- Routines for creating compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "WordData.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct DictWordData : public WordData
+  {
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+DictWordData;
+
+static DictWordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    DictWordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HDY")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+#ifndef SILENT
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+#endif
+  return (0);
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; ++i)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      DictWordData *wd;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = (DictWordData **) Xmalloc (sizeof (DictWordData *) * Num[i] + 1);
+
+      buf = (char *) Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = (DictWordData *) Xmalloc (sizeof (DictWordData) * Num[i]);
+      unsigned int j;
+      for (j = 0; j < Num[i]; ++j, ++wd)
+	{
+	  int len;
+
+	  // read docCount and wordCount
+	  wd->read(f);
+
+	  // read an mgString
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  unsigned int i;
+  DictWordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (DictWordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; ++i, ++wd)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  unsigned int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; ++i)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; ++i)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->docCount > bb->docCount)
+    return -1;
+  else if (aa->docCount < bb->docCount)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->words() > bb->words())
+    return 1;
+  else if (aa->words() < bb->words())
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  DictWordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = (DictWordData **) Xmalloc (num * sizeof (DictWordData *));
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    wd[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      DictWordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->documents(), word->word, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; ++i)
+    {
+      DictWordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (DictWordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  memset (char_freqs, '\0', sizeof (char_freqs));
+  memset (len_freqs, '\0', sizeof (len_freqs));
+  for (i = 0; i < num; ++i, ++wd)
+    {
+      u_long freq = (*wd)->documents();
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; --len, ++buf)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+inline void CalcBitCostForWordData(DictWordData **word, int num,
+				   double freqs_trans[2], u_long escape[2],
+				   char * char_lens[2], char *len_lens[2], 
+				   double esc[2], int num_trans)
+{ 
+  int j;
+
+  for (j = 0; j < num; ++j, ++word)
+    {
+      float   cbc, wbc;
+      u_char *buf = (*word)->word;
+      int     len = *buf++;
+      u_long  freq = (*word)->documents();
+      int     idx = KIND (*word);
+
+      cbc = len_lens[idx][len];
+      for (; len; --len, ++buf)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*word)->saving = ((*word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*word)->word[0]);
+
+      (*word)->num_trans = num_trans;
+    }
+}
+
+
+
+void 
+CalcBitCost (DictWordData ** discard_word, int discard_num,
+	     DictWordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  CalcBitCostForWordData(discard_word, discard_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  CalcBitCostForWordData(keep_word, keep_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+inline void m3_transferWord(DictWordData **toHeap, int *toNum, heap_comp toSaving,
+			    DictWordData **fromHeap, int *fromNum, heap_comp fromSaving,
+			    int *num_trans, double freqs_trans[], int *mem, double *total,
+			    int *count)
+{
+  /* Transfer the word at the top of the keep heap to the
+     discard heap. */
+  DictWordData *word = fromHeap[0];
+  int idx = KIND (word);
+  heap_deletehead (fromHeap, sizeof (word), fromNum, fromSaving);
+  toHeap[*toNum] = word;
+  heap_additem (toHeap, sizeof (word), toNum, toSaving);
+  freqs_trans[idx] -= word->documents();
+  *mem = (*mem) - sizeof (u_char *) + word->word[0];
+  *num_trans += 1;
+  *total += word->saving;
+  *count += 1;
+#ifdef DEBUG
+  fprintf (stderr,
+	   "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+	   *mem, word->documents(), word->char_bit_cost,
+	   word->saving, word2str (word->word));
+#endif
+}
+
+inline void m3_storeWord(DictWordData **wordHeap, int wordNum, int num_trans, 
+			 double freqs_trans[], u_long escape[], heap_comp hc)
+{
+  DictWordData *word = wordHeap[0];
+  int idx = KIND (word);
+  float wbc;
+#ifdef DEBUG1
+  fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+	   word->saving);
+#endif
+  wbc = -log2 (word->documents() / (freqs_trans[idx] + escape[idx])) *
+    word->documents();
+  word->saving = (word->char_bit_cost - wbc) /
+    (sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+  fprintf (stderr, " %.2f\n", word->saving);
+#endif
+  word->num_trans = num_trans;
+  heap_changedhead (wordHeap, sizeof (word), wordNum, hc);
+}
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  DictWordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+  keep_num = 0;
+  keep_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      DictWordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->documents();
+      ++num_trans;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < (unsigned)num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(keep_heap, keep_num, num_trans, freqs_trans, escape, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(discard_heap, discard_num, num_trans, freqs_trans, escape, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > (unsigned)k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      m3_transferWord(discard_heap, &discard_num, BigSaving,
+			      keep_heap, &keep_num, SmallSaving,
+			      &num_trans, freqs_trans, &mem, &total, &keep_to_discard);
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      m3_transferWord(keep_heap, &keep_num, SmallSaving,
+			      discard_heap, &discard_num, BigSaving,
+			      &num_trans, freqs_trans, &mem, &total, &discard_to_keep);
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  ++recalcs;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; ++i)
+    {
+      DictWordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; ++i)
+    {
+      DictWordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+#ifndef SILENT
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+#endif
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  unsigned int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; ++i)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  unsigned int i, us;
+  for (us = i = 0; i < dd->num_wds; ++i)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = new long [dd->num_wds]))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      freqs[i] = dd->wd[i]->documents();
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  delete []freqs;
+  freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  memset (&huff_words_size, '\0', sizeof (huff_words_size));
+  memset (&lencounts, '\0', sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      ++lencounts[codelen];
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  delete []hd->clens;
+  delete hd;
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[256];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; --len, ++buf)
+	freqs[(u_long) (*buf)] += (*wd)->documents();
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; ++j)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[16];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    freqs[(*wd)->word[0]] += (*wd)->documents();
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; ++j)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  memset (&cdh, '\0', sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; ++i)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < discard[i].num_wds; ++j)
+		esc.docCount += discard[i].wd[j]->documents();
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < all[i].num_wds; ++j)
+		if (all[i].wd[j]->documents() == 1)
+		  ++esc.docCount;
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_decompress_text.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * mgpp_decompress_text.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "TextGet.h"
+#include "messages.h"
+#include "mg_files.h"
+
+int main (int argc, char **argv) {
+  int ch;
+  char *filename = "";
+  char *basePath = "";
+  UCArray level;
+  SetCStr (level, "Document", 8);
+
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:d:K:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      filename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (optarg);
+      break;
+    case 'K':
+      SetCStr (level, optarg, strlen(optarg));
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-K level] [-d directory] -f name\n",
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  // load up the text information
+  TextData td;
+  if (!td.LoadData (basePath, filename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", filename);
+  }
+  
+  // output each document in the level
+//    cout << td.levels << "\n";
+  
+  TextLevelInfo levelInfo = td.levels.levelInfo[level];
+  unsigned long docNum = 1;
+  UCArray docText;
+  while (docNum <= levelInfo.numEntries) {
+//      TextIdx docIdx;
+//      if (GetDocIdx (td, level, docNum, docIdx)) {
+//        cout << "doc: " << docNum << "\n";
+//        cout << docIdx;
+//      }
+    
+    if (!GetDocText (td, level, docNum, docText)) {
+      FatalError (1, "Error while trying to get document %u", docNum);
+    }
+
+    cout << docText << "\n";
+    
+    ++docNum;
+  }
+  
+  return 0;
+}
+
+
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,69 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mgpp_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgppquery (1)
+can load it in very quickly, making the startup time for
+.BR mgppquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
@@ -0,0 +1,707 @@
+/**************************************************************************
+ *
+ * mgpp_fast_comp_dict.cpp -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "huffman.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; ++i)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  memset (buffer, 0, mem);
+  fixup = (u_char *) Xmalloc (fixup_mem);
+  memset (fixup, 0, fixup_mem);
+
+#ifndef SILENT
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+#endif
+
+  load_comp_dict (filename);
+
+#ifndef SILENT
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+#endif
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; ++which)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID)
+      for (which = 0; which <= 1; ++which)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; ++j)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; ++i)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  return (0);
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * /*cdh*/, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; --val)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+static u_long mem_skip_hd(FILE *dict, u_long mem)
+{ huff_data hd;
+
+  mem += sizeof (hd);
+  Read_Huffman_Data (dict, &hd, NULL, NULL);
+  if (hd.clens)
+    delete []hd.clens;
+  mem += hd.num_codes * sizeof (unsigned long);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+  return mem;
+}
+
+static u_long mem_read_cfh(FILE *dict, compression_dict_header *cdh, comp_frags_header *cfh, 
+			   u_long mem)
+{
+  mem += sizeof (comp_frags_header);
+  Read_cfh (dict, cfh, NULL, NULL);
+
+  /* Don't need to count the space for the clens of the huffman data */
+
+  mem += mem_for_words (dict, cdh, cfh);
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+
+  return mem;
+}
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+
+	  mem = mem_skip_hd(dict, mem);
+	  mem = mem_skip_hd(dict, mem);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem = mem_skip_hd(dict, mem);
+	      mem = mem_skip_hd(dict, mem);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((u_long) cur > (u_long)buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = (auxiliary_dict *) getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      unsigned int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = (u_char *) getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = (u_char **) getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = (u_char **) getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = (u_char ***) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = (u_char *) getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = (unsigned long *) getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = (unsigned long **) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; ++i)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+static void load_read_huffman(FILE *dict, compression_dict *cd, int which)
+{
+  huff_data *hd;
+  u_long   **vals;
+
+  hd = (huff_data *) getmem (sizeof (huff_data), sizeof (char *));
+  cd->MemForCompDict += sizeof (huff_data);
+  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+  cd->chars_huff[which] = hd;
+  FIXUP (&cd->chars_huff[which]);
+  cd->chars_vals[which] = vals;
+  FIXUP (&cd->chars_vals[which]);
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+}
+
+static void load_read_words(FILE *dict, compression_dict *cd, int which, int fixescape)
+{
+  cd->cfh[which] = (comp_frags_header *) getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+  FIXUP (&cd->cfh[which]);
+  FIXUP (&cd->values[which]);
+  if (fixescape)
+    { FIXUP(&cd->escape[which]);
+    }
+  else
+    {
+      cd->escape[which] = NULL;
+    }
+}
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = (compression_dict *) getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; ++which)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  load_read_words(dict, cd, which, 0);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+
+	  load_read_huffman(dict, cd, which);
+	  load_read_huffman(dict, cd, which);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      load_read_huffman(dict, cd, which);
+	      load_read_huffman(dict, cd, which);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_invf_dict
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +13n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that can be used during querying.
+Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word. The dictionary is blocked with
+each block containing the same number of entries (not the same size).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to 
+specify the number of entries per block. The default is 16.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict.cpp -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename, unsigned long entriesPerBlock) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // create the blocked dictionary
+  FILE *blockDictFile = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "wb",
+				     MAGIC_STEM, MG_ABORT);
+  block_dict_header bdh;
+  bdh.lookback = idh.lookback;
+  bdh.word_dict_start = idh.word_dict_start;
+  bdh.word_dict_size = idh.word_dict_size;
+  bdh.tag_dict_start = idh.tag_dict_start;
+  bdh.tag_dict_size = idh.tag_dict_size;
+  bdh.num_docs = idh.num_docs;
+  bdh.num_frags = idh.num_frags;
+  bdh.num_words = idh.num_words;
+  bdh.total_bytes = idh.total_bytes;
+  bdh.index_string_bytes = idh.index_string_bytes;
+  bdh.num_levels = idh.num_levels;
+  bdh.Write (blockDictFile);
+
+  
+  // write out the word part of the dictionary
+
+  bdh.entries_per_wblk = entriesPerBlock;
+  bdh.max_wblk_size = 0;
+  bdh.wblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  block_idx wordIdx;
+  word_block_dict_el wordBlockEl;
+  wordBlockEl.SetNumLevels (idh.num_levels);
+  
+  
+  unsigned long wordNum;
+  unsigned long wordInvfPtr;
+  UCArray lastEl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word and inverted file pointer
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordInvfPtr);
+
+    // remember this word (and position) if this is the start
+    // of a new block
+    if (wordNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = wordEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (wordIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(wordIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_wblk_size) bdh.max_wblk_size = blockLen;
+      }
+      
+      wordIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this word
+    wordBlockEl.el = wordEl.el;
+    wordBlockEl.frag_occur = wordEl.frag_occur;
+    wordBlockEl.freq = wordEl.freq;
+    wordBlockEl.invf_ptr = wordInvfPtr;
+    unsigned long tempI;
+    for (tempI=0; tempI<idh.num_levels; ++tempI)
+      wordBlockEl.levelFreqs[tempI] = wordEl.levelFreqs[tempI];
+
+    // write out the word
+    wordBlockEl.Write (blockDictFile, &lastEl, idh.num_levels);
+
+    lastEl = wordBlockEl.el;
+  }
+
+
+  // write out the tag part of the dictionary
+
+  bdh.entries_per_tblk = entriesPerBlock;
+  bdh.max_tblk_size = 0;
+  bdh.tblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+
+  block_idx tagIdx;
+  block_dict_el tagBlockEl;
+  
+  unsigned long tagNum;
+  unsigned long tagInvfPtr;
+  dict_el tagEl;
+  lastEl.erase (lastEl.begin(), lastEl.end());
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // read in the next tag and inverted file pointer
+    tagEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagInvfPtr);
+
+    // remember this tag (and position) if this is the start
+    // of a new block
+    if (tagNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = tagEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (tagIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(tagIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_tblk_size) bdh.max_tblk_size = blockLen;
+      }
+      
+      tagIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this tag
+    tagBlockEl.el = tagEl.el;
+    tagBlockEl.frag_occur = tagEl.frag_occur;
+    tagBlockEl.freq = tagEl.freq;
+    tagBlockEl.invf_ptr = tagInvfPtr;
+
+    // write out the tag
+    tagBlockEl.Write (blockDictFile, &lastEl);
+
+    lastEl = tagBlockEl.el;
+  }
+
+
+  // write out the element indexes
+  bdh.num_wblks = wordIdx.size();
+  bdh.wblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, wordIdx);
+  
+  bdh.num_tblks = tagIdx.size();
+  bdh.tblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, tagIdx);
+
+  // write out the blocked dictionary header
+  fseek (blockDictFile, sizeof(unsigned long), SEEK_SET);
+  bdh.Write (blockDictFile);
+
+
+  // close open files
+  fclose (blockDictFile);
+  fclose (invfIdxFile);
+  fclose (dictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Max word block size = %d\n", bdh.max_wblk_size);
+  Message ("Max tag block size = %d\n", bdh.max_tblk_size);
+  Message ("Number of word blocks written = %d\n", bdh.num_wblks);
+  Message ("Number of tag blocks written = %d\n", bdh.num_tblks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file] "
+	       "[-d data directory] [-b entries-per-block] "
+	       "[-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, entriesPerBlock);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict_dump.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
@@ -0,0 +1,141 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict_dump.cpp -- Program to printthe various inverted dictionaries
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // print the information from the inverted dictionary file header
+  if (printHeader) {
+    cout << "lookback: " << idh.lookback << "\n";
+    cout << "word_dict_start: " << idh.word_dict_start << "\n";
+    cout << "word_dict_size: " << idh.word_dict_size << "\n";
+    cout << "tag_dict_start: " << idh.tag_dict_start << "\n";
+    cout << "tag_dict_size: " << idh.tag_dict_size << "\n";
+    cout << "num_docs: " << idh.num_docs << "\n";
+    cout << "num_frags: " << idh.num_frags << "\n";
+    cout << "num_words: " << idh.num_words << "\n";
+    cout << "total_bytes: " << idh.total_bytes << "\n";
+    cout << "index_string_bytes: " << idh.index_string_bytes << "\n";
+    cout << "num_levels: " << idh.num_levels << "\n";
+
+    cout << "\n";
+  }
+
+
+  if (printWords) {
+    fseek (dictFile, idh.word_dict_start, SEEK_SET);
+    
+    unsigned long wordNum;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      cout << "\"" << wordEl.el << "\"\n";// (" << wordNum << ")\n";
+    }
+    
+    cout << "\n";
+  }
+
+  // write out the tag part of the dictionary
+  if (printTags) {
+    fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+    
+    unsigned long tagNum;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      // read in the next tag and inverted file pointer
+      tagEl.Read (dictFile);
+      
+      cout << "\"" << tagEl.el << "\" (" << tagNum << ")\n";
+    }
+  }
+  
+  // close open files
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+
+  while ((ch = getopt (argc, argv, "f:d:rwth")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-f input_file] "
+	       "[-d data directory]\n(-rwt:print header, words, tags)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, printHeader, printWords, printTags);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dump.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
@@ -0,0 +1,244 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dump.cpp -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+#include "WordData.h"
+
+static void PrintInvfWord (FILE *invfFile,
+			   invf_dict_header &idh,
+			   invf_file_header &ifh,
+			   word_dict_el &wordEl,
+			   unsigned long wordStart,
+			   bool printFrags) {
+  cout << wordEl.frag_occur << " \"" << wordEl.el << "\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL);
+      fragNum += delta;
+      cout << " " << fragNum;
+      
+      if (!ifh.word_level_index ) {
+	unsigned long count = buffer.gamma_decode (NULL);
+	cout << "(" << count << ")";
+      } else {
+	cout << "(1)";
+      }
+    }
+    
+    cout << "\n";
+  
+    buffer.done();
+  }
+}
+
+static void PrintInvfTag (FILE *invfFile,
+			  invf_dict_header &idh,
+			  invf_file_header &/*ifh*/,
+			  dict_el &tagEl,
+			  unsigned long tagStart,
+			  bool printFrags) {
+  cout << tagEl.frag_occur << " \"<" << tagEl.el << ">\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, tagStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long pTag = tagEl.frag_occur*2;
+    unsigned long B = BIO_Bblock_Init (idh.num_frags+pTag, pTag);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<tagEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << " " << fragNum;
+      cout << "-";
+      delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << fragNum;
+    }
+    
+    cout << "\n";
+    
+    buffer.done();
+  }
+}
+
+static void PrintHeaderInfo (invf_dict_header &idh,
+			     invf_file_header &ifh) {
+  cerr << "Lookback:         " << idh.lookback << "\n";
+  cerr << "Word Dict Size:   " << idh.word_dict_size << "\n";
+  cerr << "Tag Dict Size:    " << idh.tag_dict_size << "\n";
+  cerr << "Num Documents:    " << idh.num_docs << "\n";
+  cerr << "Num Fragments:    " << idh.num_frags << "\n";
+  cerr << "Num Words:        " << idh.num_words << "\n";
+
+  cerr << "Skip Mode:        " << ifh.skip_mode << "\n";
+  cerr << "Word Level Index: " << ifh.word_level_index << "\n";
+
+  cerr << "\n";
+}
+
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags,
+			   bool printFrags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  // print out header information
+  if (printHeader) {
+    PrintHeaderInfo (idh, ifh);
+  }
+
+  // open the inverted index
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  // process all the words
+  if (printWords) {
+    unsigned long wordNum;
+    unsigned long wordStart;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      ReadUL (invfIdxFile, wordStart);
+      PrintInvfWord (invfFile, idh, ifh, wordEl, wordStart, printFrags);
+    }
+  }
+
+  // process all the tags
+  if (printTags) {
+    unsigned long tagNum;
+    unsigned long tagStart;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      tagEl.Read (dictFile);
+      ReadUL (invfIdxFile, tagStart);
+      PrintInvfTag (invfFile, idh, ifh, tagEl, tagStart, printFrags);
+    }
+  }
+  // close the open files
+  fclose (invfIdxFile);
+  fclose (invfFile);
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *dir_name, *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+  bool printFrags = false;
+  
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hrwtnf:d:")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath(optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 'n':
+      printFrags = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-n] [-f input_file]"
+	       "[-d data directory]\n(-rwnt:print header, words, tags, fragnums)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+  
+  process_files (filename, printHeader, printWords, printTags, printFrags);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.1	(revision 21112)
@@ -0,0 +1,323 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_passes 1 \*(Dt CITRI
+.SH NAME
+mgpp_passes \- builds mgpp databases
+.SH SYNOPSIS
+.B mgpp_passes
+[
+.BI \-J " doc-tag"
+]
+[
+.BI \-K " level-tag"
+]
+.if n .ti +10n
+[
+.BI \-L " index-level"
+]
+[
+.BI \-m " invf-mem-buffer"
+]
+.if n .ti +10n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+[
+.B \-S
+]
+[
+.B \-C
+]
+.if n .ti +10n
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mgpp_passes
+is the program that does most of the work when building mgpp
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  In general,
+.B mgpp_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+mgpp database.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+mgpp database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpointt\fP'u+2n"
+.BI \-J " doc-tag"
+Specifies the SGML tag that encloses each document.  Text appearing
+outside this tag is ignored.  The document tag defines the highest
+level document that can be queried and printed.  The default document
+tag is 'Document'.
+.TP
+.BI \-K " level-tag"
+Specifies the SGML tag of a sub document level.  A level tag must
+enclose all text enclosed by the document tag.  Levels can be
+queried and printed as if they were separate documents.  Multiple
+document levels can be specified (the document tag is always
+added as a document level).
+.TP
+.BI \-L " index-level"
+Specifies the SGML tag enclosing the smallest indexed element.  The
+index level should be no larger than the smallest document 
+level.  An empty string can be used to specify a word level index 
+(which is the default).
+.TP
+.BI \-m " invf-mem-buffer"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+.IR *.text.level ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mgpp_perf_hash_build (1)
+from the
+.I *.invf.dict
+file.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP 
+.B \-C
+This activates the compatibility parsing mode.  When using this
+mode documents are separated by control-B and paragraphs are separated
+by control-C.  Internally these are converted to documents surrounded
+by 'Document' tags and paragraphs surrounded by 'Paragraph' tags.
+.TP
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an mgpp document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats, *.invf.dict, *.invf.level
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mgpp_passes -T1 -I1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mgpp_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mgpp_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text, *.text.idx, *.text.level
+.I
+#   *.invf and *.invf.idx
+${source} | mgpp_passes -T2 -I2 -f ${text}
+.PP
+.I
+# Create *.text.weight and *.weight.approx
+mgpp_weights_build -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked
+mgpp_invf_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.1
+mgpp_stem_idx -s 1 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.2
+mgpp_stem_idx -s 2 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.3
+mgpp_stem_idx -s 3 -f ${text}
+.PP
+.I
+# Create *.text.dict.fast
+mgpp_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary with index into the dictionary.
+.TP
+.B *.invf.dict.blocked.n
+Transformation dictionary from words stemmed with method 
+.B n
+to unstemmed words.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.level
+Information about the document levels needed for querying.
+.TP
+.B *.text
+Compressed text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.fast
+A fast loading version of the compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.level
+Information about the document levels needed for text decompression.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+The approximate weights file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_passes.cpp	(revision 21112)
@@ -0,0 +1,268 @@
+/**************************************************************************
+ *
+ * mgpp_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#if defined __WIN32__
+# include <io.h>
+# include "getopt_old.h"
+# define close _close
+# define open _open
+#elif defined __CYGWIN__
+#include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "longlong.h"
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-J doc-tag] [-K level-tag] [-L index-level]\n"
+"  %*s [-m invf-memory] [-T1] [-T2] [-I1] [-I2] [-S]\n"
+"  %*s [-C] [-h] [-d directory] -f name\n\n";
+
+
+
+static void Usage (char *err) {
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "");
+  exit (1);
+}
+
+
+
+
+int OpenNextFile (int in_fd) {
+  if (in_fd > 0) close (in_fd);
+  if (num_files == 0) return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  ++files;
+  --num_files;
+  return (in_fd);
+}
+
+
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; ++pass) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      ++numDocs;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+int main (int argc, char **argv) {
+  int ch, in_fd;
+  char *filename = NULL;
+  bool compatMode = false;
+  TagInfo tagInfo;
+  tagInfo.SetDocTag ("Document");
+  
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch=getopt(argc, argv, "J:K:L:f:d:m:I:T:SCh"))!=-1){
+    switch (ch) {
+    case 'J':
+      tagInfo.SetDocTag (optarg);
+      break;
+    case 'K':
+      tagInfo.AddLevelTag (optarg);
+      break;
+    case 'L':
+      tagInfo.SetIndexLevel (optarg);
+      break;
+    case 'f':
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'm':
+      invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+      break;
+    case 'I':
+      if (*optarg == '1')
+	Passes |= IVF_PASS_1;
+      else if (*optarg == '2')
+	Passes |= IVF_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'T':
+      if (*optarg == '1')
+	Passes |= TEXT_PASS_1;
+      else if (*optarg == '2')
+	Passes |= TEXT_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'S':
+      Passes |= SPECIAL;
+      break;
+    case 'C':
+      compatMode = true;
+      break;
+    case 'h':
+    case '?':
+      Usage (NULL);
+    }
+  }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc) {
+    if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+      FatalError (1, "Cannot open %s", argv[optind]);
+    files = &argv[optind + 1];
+    num_files = argc - (optind + 1);
+    
+  } else in_fd = 0;  // stdin
+
+  
+  if (compatMode) tagInfo.SetDocTag ("Document");
+
+  // a document tag is also a level tag
+  tagInfo.levelTags.insert (tagInfo.docTag); 
+  
+  Driver (in_fd, filename, tagInfo, compatMode);
+
+  return (0);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mgpp_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * mgpp_perf_hash_build.cpp -- Program to build a perfect hash function
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define POOL_SIZE 1024*1024
+
+static void ProcessFiles (char *filename, int r) {
+  FILE *dictFile, *hashFile;
+  unsigned long i;
+  invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  unsigned long pool_left;
+  u_char **starts;
+
+  // read in the dictionary
+  dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  if (dictFile==NULL) {
+    FatalError(1, "unable to open file");
+  }
+  idh.Read (dictFile);
+
+  //cerr << idh.lookback<<" "<<idh.word_dict_start<<endl;
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  
+  if (!(pool = (u_char *) Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = (u_char **) Xmalloc (sizeof (u_char *) * idh.word_dict_size)))
+    FatalError (1, "Out of memory");
+  //cerr << "size= "<< idh.word_dict_size<<endl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (i = 0; i < idh.word_dict_size; ++i) {
+    // read the next word and associated information
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // push string onto pool data
+    register unsigned long l;
+    l = wordEl.el.size() + 1;
+    if (pool_left < l) {
+      pool = (u_char *) Xmalloc (POOL_SIZE);
+      pool_left = POOL_SIZE;
+    }
+    starts[i] = pool;
+    
+    *pool++ = wordEl.el.size();
+    memcpy ((char *) pool, &(wordEl.el[0]), wordEl.el.size());
+    //cerr << pool<<"   " <<starts[i]<<endl;
+    pool += wordEl.el.size();
+    pool_left -= l;
+    
+  }
+  fclose (dictFile);
+  //cerr << pool<<"   " <<starts[i-1]<<endl;
+  //cerr<<"starts  "<<starts[113529]<<endl;
+  //cerr << starts[17][1] << " "<<starts[25][4]<<endl;
+  // create perfect hash file
+  hashFile = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+			  MAGIC_HASH, MG_ABORT);
+  if (!(phd = gen_hash_func (idh.word_dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hashFile, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (hashFile);
+}
+
+
+
+int main (int argc, char **argv) {
+  int r = -1;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      r = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  ProcessFiles (filename, r);
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mgpp_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mgpp_stem_idx
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +12n
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +12n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_stem_idx
+generates a stem index file for a collection.
+This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to set
+the number of entries per block. The default is 16.
+.TP
+.BI \-a " stemmer"
+The name of the stemmer to use, the default is the Lovin stemmer.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_stem_idx.cpp -- stem index builder
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+#include "stemmer.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#  include <vector.h>
+#else
+#  include <map>
+#  include <vector>
+#endif
+
+typedef vector<unsigned long> WordNumList;
+typedef map<UCArray, WordNumList, DictLTUCArray> StemMapDict;
+
+void CreateStemDict (char *filename,
+		     StemMapDict &stemDict,
+		     int stemMethod,
+		     int stemmerNum) {
+  stemDict.erase (stemDict.begin(), stemDict.end());
+
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  unsigned long wordNum;
+  u_char mgWord[MAXSTEMLEN + 1];
+  word_dict_el wordEl;
+  UCArray stemEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // convert the word to an "mg word"
+    mgWord[0] = wordEl.el.size();
+    memcpy((char *)&mgWord[1], &(wordEl.el[0]), wordEl.el.size());
+
+    // stem the word
+    stemmer (stemMethod, stemmerNum, mgWord);
+
+    // convert the result back to a UCArray
+    stemEl.erase (stemEl.begin(), stemEl.end());
+    stemEl.insert (stemEl.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+//      cout << "\"" << stemEl << "\" -> \"" << wordEl.el << "\"\n";
+    
+    // add this word number to the list of word numbers for this word
+    stemDict[stemEl].push_back (wordNum);
+  }
+
+  fclose (dictFile);
+}
+
+
+void WriteStemDict (char *filename,
+		    StemMapDict &stemDict,
+		    int stemMethod,
+		    int stemmerNum,
+		    unsigned long entriesPerBlock) {
+  // Create appropriate stem index file
+  FILE *stemDictFile = NULL;
+  if (stemMethod == 1) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+				"wb", MAGIC_STEM_1, MG_ABORT);
+  } else if (stemMethod == 2) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+				"wb", MAGIC_STEM_2, MG_ABORT);
+  } else if (stemMethod == 3) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+				"wb", MAGIC_STEM_3, MG_ABORT);
+  } else {
+    FatalError (1, "Unknown stem method %d", stemMethod);
+  }
+
+  stem_idx_header sih;
+  sih.lookback = 0;
+  sih.dict_size = stemDict.size();
+  sih.entries_per_block = entriesPerBlock;
+  sih.max_block_size = 0;
+  
+  sih.stemmer_num = stemmerNum;
+  sih.stem_method = stemMethod;
+
+  // write out a place-holder version of the header
+  sih.Write (stemDictFile);
+
+  sih.blocks_start = ftell (stemDictFile);
+
+  block_idx stemIdx;
+  unsigned long stemNum = 0;
+  stem_block_dict_el stemEl;
+  UCArray lastEl;
+
+  StemMapDict::const_iterator here = stemDict.begin();
+  StemMapDict::const_iterator end = stemDict.end();
+  while (here != end) {
+    // remember this stem (and position) if this is the start
+    // of a new block
+    if (stemNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = (*here).first;
+      elIdx.block_ptr = ftell (stemDictFile);
+
+      // see if this block is the longest so far
+      if (stemIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(stemIdx.end()-1)).block_ptr;
+	if (blockLen > sih.max_block_size) sih.max_block_size = blockLen;
+      }
+      
+      stemIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this stem
+    stemEl.el = (*here).first;
+    stemEl.equivWords = (*here).second;
+
+    // write out the stem
+    stemEl.Write (stemDictFile, &lastEl);
+    
+    ++here; ++stemNum;
+  }
+
+
+  // write out the element indexes
+  sih.num_blocks = stemIdx.size();
+  sih.block_idx_start = ftell (stemDictFile);
+  WriteBlockIdx (stemDictFile, stemIdx);
+  
+  // write out the stem dictionary header
+  fseek (stemDictFile, sizeof(unsigned long), SEEK_SET);
+  sih.Write (stemDictFile);
+
+
+  // close open files
+  fclose (stemDictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Num word stems = %d\n", sih.dict_size);
+  Message ("Max stem block size = %d\n", sih.max_block_size);
+  Message ("Number of stem blocks written = %d\n", sih.num_blocks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  int stemMethod = 0; // illegal value (no translation)
+  int stemmerNum = 0; // English stemmer
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:s:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 's':
+      stemMethod = atoi (optarg);
+      break;
+    case 'a':
+      stemmerNum = stemmernumber ((unsigned char *) optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-d directory] "
+	       "[-b entries-per-block] [-h] -s 1|2|3 "
+	       "[-a stemmer-method] -f name\n", argv[0]);
+      exit (1);
+    }
+  }
+  
+  if (stemMethod < 1 || stemMethod > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  // read in the dictionary and create the in memory dictionary
+  StemMapDict stemDict;
+  CreateStemDict (filename, stemDict, stemMethod, stemmerNum);
+  
+  // write out the dictionary as a blocked file
+  WriteStemDict (filename, stemDict, stemMethod, stemmerNum, entriesPerBlock);
+  
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.1
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_weights_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mgpp_weights_build
+[
+.B \-h
+]
+[
+.BI \-b " num-bits"
+]
+.if n .ti +17n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_weights_build
+is the program that generates the exact and approximate weights 
+files.  To do this, it uses the inverted file
+and dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num-bits"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1)
Index: /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
@@ -0,0 +1,399 @@
+/**************************************************************************
+ *
+ * mgpp_weights_build.cpp -- Program to build the document weights file
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+struct WBTagPtr {
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  unsigned long fragOccur;
+  
+  WBTagPtr () {
+    tagNum = 0;
+    tagPtr = 0;
+    fragOccur = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, WBTagPtr, DictLTUCArray> WBTagDict;
+
+typedef vector<float> Weights;
+
+
+static void ReadTagDict (const invf_dict_header &idh,
+			 FILE *dictFile,
+			 FILE *invfIdxFile,
+			 WBTagDict &tagDict) {
+  tagDict.erase (tagDict.begin(), tagDict.end());
+
+  // seek to the start of the tag information
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof(unsigned long) +
+	 idh.word_dict_size*sizeof(unsigned long), SEEK_SET);
+  
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagPtr);
+    tagDict[thisEl.el].tagNum = tagNum;
+    tagDict[thisEl.el].tagPtr = tagPtr;
+    tagDict[thisEl.el].fragOccur = thisEl.frag_occur;
+  }
+}
+
+static void AddWeight (Weights &w,
+		       unsigned long levelDocNum,
+		       unsigned long termFreq,
+		       float idf) {
+  double weight = termFreq * idf;
+  w[levelDocNum-1] += weight * weight;
+}
+
+static void GenerateLevelWeights (const invf_dict_header &idh,
+				  const invf_file_header &ifh,
+				  unsigned long numLevelDocs,
+				  unsigned long levelNum,
+				  FILE *dictFile,
+				  FILE *invfFile,
+				  FILE *invfIdxFile,
+				  const FragLevelConvert &fragLevelConvert,
+				  Weights &w) {
+  // pre-allocate the right number of weights
+  w.erase (w.begin(), w.end());
+  w.insert (w.end(), (Weights::size_type)numLevelDocs, (float)0.0);
+
+  double logN = log ((double) numLevelDocs);
+
+  // reset the files
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof (unsigned long), SEEK_SET);
+  
+  // process each word adding its contributions to the document weights
+  unsigned long wordNum;
+  unsigned long wordStart;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+#ifndef SILENT
+    // give a little feedback every 4096 words
+    if ((wordNum & 0xfff) == 0) fprintf (stderr, ".");
+#endif
+    
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordStart);
+
+    float idf = logN - log ((double) wordEl.levelFreqs[levelNum]);
+
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    stdio_bitio_buffer buffer (invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long levelDocNum = 0;
+    unsigned long lastLevelDocNum = 0;
+    unsigned long termFreq = 0;
+    unsigned long checkLevelFreq = 0;
+    
+    unsigned long count, i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      fragNum += buffer.bblock_decode (B, NULL);
+      if (!ifh.word_level_index) count = buffer.gamma_decode (NULL);
+      else count = 1;
+
+      if (fragNum > idh.num_frags)
+	FatalError (1, "fragNum = %d, "
+		    "number of fragments = %d\n"
+		    "wordNum = %d\n"
+		    "i = %d, frag_occur = %d\n",
+		    fragNum, idh.num_frags, wordNum, i, wordEl.frag_occur);
+
+      if (!fragLevelConvert.FragToLevel (fragNum, levelDocNum))
+	FatalError (1, "could not convert fragment number %d in level %d", fragNum, levelNum);
+
+      if (levelDocNum == 0 || levelDocNum > numLevelDocs)
+	FatalError (1, "bad level document number %d in level %d", levelDocNum, levelNum);
+      
+      if (levelDocNum != lastLevelDocNum) {
+	// new level document
+	if (lastLevelDocNum > 0) {
+	  AddWeight (w, lastLevelDocNum, termFreq, idf);
+	  ++checkLevelFreq;
+	}
+	lastLevelDocNum = levelDocNum;
+	termFreq = 0;
+      }
+      termFreq += count;
+    }
+
+    if (lastLevelDocNum > 0) {
+      AddWeight (w, lastLevelDocNum, termFreq, idf);
+      ++checkLevelFreq;
+    }
+
+    if (checkLevelFreq != wordEl.levelFreqs[levelNum]) {
+      cerr << "bad level freq at level " <<levelNum<<" "<< checkLevelFreq << " != "
+	   << wordEl.levelFreqs[levelNum] << ", word \""
+	   << wordEl.el << "\" (" << wordNum << ")\n";
+      exit (1);
+    }
+    
+    buffer.done();
+  }
+
+  if (w.size() != numLevelDocs) 
+    FatalError (1, "wrong number of weights created %d != %d",
+		w.size(), numLevelDocs);
+}
+
+static void WriteExactWeights (FILE *weightsFile,
+			       unsigned long &diskPtr,
+			       const Weights &w) {
+  diskPtr = ftell(weightsFile);
+
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+//      cout << *here << "\n";
+    WriteF (weightsFile, sqrt (*here));
+    ++here;
+  }
+}
+
+static void WriteApproxWeights (FILE *approxWeightsFile,
+				unsigned long &diskPtr,
+				const Weights &w,
+				unsigned char bits) {
+  diskPtr = ftell(approxWeightsFile);
+
+  // calculate L, U and B
+  double L = 1e300;
+  double U = 0;
+  float wgt;
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+    wgt = sqrt (*here);
+    if (wgt > U) U = wgt;
+    if (wgt > 0 && wgt < L) L = wgt;
+    ++here;
+  }
+
+  double B = pow (U / L, pow (2.0, -(double) bits));
+
+#ifndef SILENT
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+#endif
+  
+  WriteUC (approxWeightsFile, bits);
+  WriteD (approxWeightsFile, L);
+  WriteD (approxWeightsFile, B);
+  
+
+  unsigned long max = (bits == 32) ? 0xffffffff : (1 << bits) - 1;;
+  unsigned long i=0, buf=0, pos=0;
+  here = w.begin();
+  end = w.end();
+  while (here != end) {
+    unsigned long fx;
+    wgt = sqrt (*here);
+    if (wgt == 0) {
+      wgt = L;
+#ifndef SILENT
+      Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+    }
+    fx = (unsigned long) floor (log (wgt / L) / log (B));
+
+    if (fx > max) fx = max;
+
+    buf |= (fx << pos);
+    pos += bits;
+
+    if (pos >= MAXBITS) {
+      WriteUL (approxWeightsFile, buf);
+      buf = fx >> (bits - (pos - MAXBITS));
+      pos = pos - MAXBITS;
+    }
+
+    ++here; ++i;
+  }
+
+  // write out the last bits
+  if (pos > 0) WriteUL (approxWeightsFile, buf);
+}
+
+int main (int argc, char **argv) {
+  unsigned char bits = 8;
+  char *filename = "";
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      bits = atoi (optarg);
+      if (bits > 32) {
+	fprintf (stderr, "b may only take values 0-32\n");
+	exit (1);
+      }
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-b bits] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to create weights.");
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+  
+  // read the level information
+  FILE *levelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			       MAGIC_INVF_LEVELS, MG_ABORT);
+  FIvfLevel ivfLevel;
+  ivfLevel.Read (levelFile);
+  fclose (levelFile);
+
+  // read in the tag dictionary and inverted file pointers
+  WBTagDict tagDict;
+  ReadTagDict (idh, dictFile, invfIdxFile, tagDict);
+
+  
+  // create the weights file
+  FILE *weightsFile = create_file (filename, WEIGHTS_SUFFIX, "wb",
+				   MAGIC_WGHT, MG_ABORT);
+
+  // create the approx weights file
+  FILE *approxWeightsFile = create_file (filename, APPROX_WEIGHTS_SUFFIX, "wb",
+					 MAGIC_WGHT_APPROX, MG_ABORT);
+
+  
+  // create weights for each document level
+  FragLevelConvert fragLevelConvert;
+  Weights w;
+  IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+  IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+  unsigned long levelNum = 0;
+  while (levelHere != levelEnd) {
+    const UCArray &levelName = (*levelHere).first;
+    
+    // read the tag information about this level
+    fragLevelConvert.Read(invfFile, tagDict[levelName].tagPtr,
+			  idh.num_frags, tagDict[levelName].fragOccur);
+    
+    // create the weights for this level
+    GenerateLevelWeights (idh, ifh, tagDict[levelName].fragOccur,
+			  levelNum, dictFile, invfFile, invfIdxFile,
+			  fragLevelConvert, w);
+
+    // write out the exact weights
+    WriteExactWeights (weightsFile,
+		       (*levelHere).second.exactWeightsDiskPtr,
+		       w);
+    
+    // write out the approximate weights
+    WriteApproxWeights (approxWeightsFile,
+			(*levelHere).second.approxWeightsDiskPtr,
+			w, bits);
+    
+    ++levelHere; ++levelNum;
+  }
+  
+
+  // close input files
+  fclose (dictFile);
+  fclose (invfFile);
+  fclose (invfIdxFile);
+
+  // update the level information
+  levelFile = create_file (filename, INVF_LEVEL_SUFFIX, "wb",
+			   MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Write (levelFile);
+  fclose (levelFile);
+
+  // close output files
+  fclose (weightsFile);
+  fclose (approxWeightsFile);
+
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/stemmer.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/stemmer.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/stemmer.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * stemmer.cpp -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; ++i) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); ++i)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp ((char *) descript,  "0") == 0) ||
+      (strcmp ((char *) descript, "english") == 0) ||
+      (strcmp ((char *) descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp ((char *) descript, "1") == 1) ||
+      (strcmp ((char *) descript, "french") == 0) ||
+      (strcmp ((char *) descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/stemmer.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/stemmer.h	(revision 21112)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+#ifdef __cplusplus
+extern "C" 
+#endif
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/text.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/text.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/text.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// UCArray currently contains all the basic reading and writing functions
+#include "UCArray.h"
+#include "text.h"
+
+compressed_text_header::compressed_text_header () {
+  Clear();
+}
+
+void compressed_text_header::Clear () {
+  num_of_docs = 0;
+  num_of_words = 0;
+  num_of_bytes = 0.0;
+}
+
+
+bool compressed_text_header::Read (FILE *f) {
+  return (ReadUL (f, num_of_docs) &&
+	  ReadUL (f, num_of_words) &&
+	  ReadD (f, num_of_bytes));
+}
+
+bool compressed_text_header::Write (FILE *f) const {
+  return (WriteUL (f, num_of_docs) &&
+	  WriteUL (f, num_of_words) &&
+	  WriteD (f, num_of_bytes));
+}
+
+
+auxiliary_dict::auxiliary_dict () {
+  word_data[0] = NULL;
+  word_data[1] = NULL;
+  words[0] = NULL;
+  words[1] = NULL;
+}
+
+auxiliary_dict::~auxiliary_dict () {
+  Clear ();
+}
+
+void auxiliary_dict::Clear () {
+  if (word_data[0] != NULL) {
+    delete []word_data[0];
+    word_data[0] = NULL;
+  }
+  if (word_data[1] != NULL) {
+    delete []word_data[1];
+    word_data[1] = NULL;
+  }
+  if (words[0] != NULL) {
+    delete []words[0];
+    words[0] = NULL;
+  }
+  if (words[1] != NULL) {
+    delete []words[1];
+    words[1] = NULL;
+  }
+}
+
+
+compression_dict::compression_dict () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    cfh[which] = NULL;
+    values[which] = NULL;
+    escape[which] = NULL;
+    chars_huff[which] = NULL;
+    chars_vals[which] = NULL;
+    lens_huff[which] = NULL;
+    lens_vals[which] = NULL;
+  }
+
+  MemForCompDict = 0;
+  ad = NULL;
+  fast_loaded = 0;
+}
+
+compression_dict::~compression_dict () {
+  Clear ();
+}
+
+void compression_dict::Clear () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    if (cfh[which] != NULL) {
+      delete cfh[which];
+      cfh[which] = NULL;
+    }
+    if (values[which] != NULL) {
+      delete []values[which][0][0];
+      delete []values[which][0];
+      delete []values[which];
+      values[which] = NULL;
+    }
+    if (escape[which] != NULL) { // don't know if needed
+      //delete escape[which]; // this was causing a seg fault
+      escape[which] = NULL;
+    }
+    if (chars_huff[which] != NULL) {
+      delete chars_huff[which];
+      chars_huff[which] = NULL;
+    }
+    if (chars_vals[which] != NULL) {
+      delete []chars_vals[which][0];
+      delete []chars_vals[which];
+      chars_vals[which] = NULL;
+    }
+    if (lens_huff[which] != NULL) {
+      delete lens_huff[which];
+      lens_huff[which] = NULL;
+    }
+    if (lens_vals[which] != NULL) {
+      delete []lens_vals[which][0];
+      delete []lens_vals[which];
+      lens_vals[which] = NULL;
+    }
+  }
+
+  if (ad != NULL) {
+    delete ad;
+    ad = NULL;
+  }
+
+  MemForCompDict = 0;
+  fast_loaded = 0;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/text.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/text.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/text.h	(revision 21112)
@@ -0,0 +1,199 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT
+#define H_TEXT
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+#include "huffman.h"
+#include <stdio.h>
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+struct compression_dict_header {
+  u_long dict_type;
+  u_long novel_method;
+  u_long params[TEXT_PARAMS];
+  u_long num_words[2];
+  u_long num_word_chars[2];
+  u_long lookback;
+};
+
+
+struct comp_frags_header {
+  huff_data hd;
+  u_long uncompressed_size;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+};
+
+// BOGUSTEXTLEN is used to replace the ratio
+// in the compressed_text_header while preparing
+// for UCArray
+#define BOGUSTEXTLEN 1000000
+
+struct compressed_text_header {
+  u_long num_of_docs;
+  u_long num_of_words; // number of words in collection
+  double num_of_bytes;
+
+  compressed_text_header ();
+  void Clear ();
+
+  // you must seek to the appropriate place before calling
+  // Read or Write
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct compression_stats_header {
+  u_long num_docs;
+  u_long dummy;     // alignment
+  double num_bytes; 
+};
+
+
+struct frags_stats_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct aux_frags_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct auxiliary_dict {
+  aux_frags_header afh[2];
+  u_char *word_data[2];
+  u_char **words[2];
+  int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+					   for the hybrid methods */
+
+  auxiliary_dict ();
+  ~auxiliary_dict ();
+
+  void Clear ();
+};
+
+
+struct compression_dict {
+  compression_dict_header cdh;
+  comp_frags_header *cfh[2];
+  unsigned long MemForCompDict;
+  u_char ***values[2];
+  u_char *escape[2];
+  huff_data *chars_huff[2];
+  u_long **chars_vals[2];
+  huff_data *lens_huff[2];
+  u_long **lens_vals[2];
+  auxiliary_dict *ad;
+  int fast_loaded;
+
+  compression_dict ();
+  ~compression_dict ();
+
+  void Clear ();
+};
+
+
+#endif
Index: /main/tags/3.01/gsdl/src/mgpp/text/text.pass1.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/text.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/text.pass1.cpp	(revision 21112)
@@ -0,0 +1,387 @@
+/**************************************************************************
+ *
+ * text.pass1.cpp -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+#include "TextEl.h"
+
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long longestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh = {0, 0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void ChangeMem (int Change) {
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse;
+}
+
+
+int init_text_1 (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  int which;
+  
+  if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+    Message ("Unable to allocate memory for pool");
+    return (COMPERROR);
+  }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+  
+  for (which = 1; which >= 0; --which) {
+    u_char *word;
+    hash_rec *ent;
+    dict_data *dd = &DictData[which];
+    
+    dd->wordnum = 0;
+    dd->words_read = 0;
+    dd->bytes_diff = 0;
+    dd->HashSize = INITIAL_HASH_SIZE;
+    dd->HashUsed = 0;
+    
+    if (!(dd->HashTable = (hash_rec *) Xmalloc (sizeof (hash_rec) * dd->HashSize))) {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+    ChangeMem (sizeof (hash_rec) * dd->HashSize);
+    memset (dd->HashTable,'\0' , sizeof (hash_rec) * dd->HashSize);
+
+    word = Pool;
+    *Pool++ = '\0';
+    --PoolLeft;
+    {
+      register u_char *wptr;
+      register int hsize = dd->HashSize;
+      register unsigned long hashval, step;
+      
+      HASH (hashval, step, word, hsize);
+      wptr = (dd->HashTable + hashval)->word;
+      while (wptr) {
+	hashval += step;
+	if (hashval >= (unsigned long)hsize)
+	  hashval -= hsize;
+	wptr = (dd->HashTable + hashval)->word;
+      }
+      ent = dd->HashTable + hashval;
+    }
+    ent->wcnt = 1;
+    ent->word = word;
+    dd->HashUsed = 1;
+  }
+  return (COMPALLOK);
+}
+
+
+static int process_text_element (const u_char *s_in, int l_in) {
+  const u_char *end = s_in + l_in - 1;
+  
+   /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  
+  bool which = false; // non-word
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    dict_data *dd = &DictData[which];
+    
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+    
+    ++dd->wordnum;
+    inputbytes += *Word;
+    ++dd->words_read;
+    
+    /* Search the hash table for Word */
+    {
+      register unsigned long hashval, step;
+      register int hsize = dd->HashSize;
+      HASH (hashval, step, Word, hsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	register hash_rec *ent;
+	ent = dd->HashTable + hashval;
+	if (!ent->word) {
+	  int len = *Word + 1;
+	  if (len > PoolLeft) {
+	    if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+	      Message ("Unable to allocate memory for pool");
+	      return (COMPERROR);
+	    }
+	    PoolLeft = POOL_SIZE;
+	    ChangeMem (POOL_SIZE);
+	  }
+	  ent->occurance_num = occurance_num++;
+	  ent->wcnt = 1;
+	  ent->word = Pool;
+	  memcpy (Pool, Word, len);
+	  Pool += len;
+	  PoolLeft -= len;
+	  ++dd->HashUsed;
+	  dd->bytes_diff += Word[0];
+	  break;
+	}
+	
+	/* Compare the words */
+	s1 = Word;
+	s2 = ent->word;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval = (hashval + step);
+	  if (hashval >= (unsigned long)hsize) hashval -= hsize;
+	} else {
+	  ++ent->wcnt;
+	  break;
+	}
+      }
+    }
+
+    
+    if (dd->HashUsed >= dd->HashSize >> 1) {
+      hash_rec *ht;
+      unsigned long size;
+      unsigned long i;
+      size = prime (dd->HashSize * 2);
+      if (!(ht = (hash_rec *) Xmalloc (sizeof (hash_rec) * size))) {
+	Message ("Unable to allocate memory for table");
+	return (COMPERROR);
+      }
+      ChangeMem (sizeof (hash_rec) * size);
+      memset (ht, '\0', sizeof (hash_rec) * size);
+
+      for (i = 0; i < dd->HashSize; ++i)
+	if (dd->HashTable[i].word) {
+	  register u_char *wptr;
+	  register unsigned long hashval, step;
+	  
+	  wptr = dd->HashTable[i].word;
+	  HASH (hashval, step, wptr, size);
+	  wptr = (ht + hashval)->word;
+	  while (wptr) {
+	    hashval += step;
+	    if (hashval >= size) hashval -= size;
+	    wptr = (ht + hashval)->word;
+	  }
+	  ht[hashval] = dd->HashTable[i];
+	}
+      Xfree (dd->HashTable);
+      ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+      dd->HashTable = ht;
+      dd->HashSize = size;
+    }
+  }
+
+  return COMPALLOK;
+}
+
+
+int process_text_1 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  unsigned long textLen = 0;
+  unsigned long docLen = 0;
+  int retValue;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    textLen = (*here).text.size();
+    docLen += textLen;
+
+    retValue = process_text_element (&(here->text[0]), textLen);
+    if (retValue != COMPALLOK) return retValue;
+    
+    ++here;
+  }
+
+  // get max document length
+  if (docLen > longestDoc) longestDoc = docLen;
+
+  // update header information
+  ++csh.num_docs;
+  csh.num_bytes += docLen;
+  
+  return COMPALLOK;
+}
+
+
+static int PackHashTable (dict_data * dd) {
+  int s, d;
+  for (s = d = 0; (unsigned int)s < dd->HashSize; ++s)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  
+  if (!(dd->HashTable = (hash_rec *) Xrealloc (dd->HashTable,
+			   sizeof (hash_rec) * dd->HashUsed))) {
+    Message ("Out of memory");
+    return COMPERROR;
+  }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+static int ent_comp (const void *s1, const void *s2) {
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+static void WriteHashTable (FILE * fp, dict_data * dd) {
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR) return;
+  
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+  
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; ++j)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+  
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+  
+  for (j = 0; j < dd->HashSize; ++j) {
+    curr = dd->HashTable[j].word;
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    HTONUL(dd->HashTable[j].wcnt);
+    HTONUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+    fwrite (&dd->HashTable[j].occurance_num,
+	    sizeof (dd->HashTable[j].occurance_num), 1, fp);
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    NTOHUL(dd->HashTable[j].wcnt);
+    NTOHUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+  }
+}
+
+
+int done_text_1 (const TagInfo &/*tagInfo*/, char *file_name) {
+  char *temp_str;
+  FILE *fp;
+  
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+	      " "
+#endif
+              );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  return COMPALLOK;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/text.pass2.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/text.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/text.pass2.cpp	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * text.pass2.cpp -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#if defined (__WIN32__)
+# include <stdio.h>
+# define unlink _unlink
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+#include "FText.h"
+
+#define POOL_SIZE 1024*256
+
+
+struct char_pool {
+  struct char_pool *next;
+  u_long left;
+  u_char *ptr;
+  u_char pool[POOL_SIZE];
+};
+
+struct novel_hash_rec {
+  u_long ordinal_num;
+  u_char *word;
+};
+
+
+#define INITIAL_HASH_SIZE 7927
+
+struct novel_hash_table {
+  novel_hash_rec *HashTable;
+  u_long HashSize, HashUsed;
+  char_pool *first_pool;
+  char_pool *pool;
+  u_long next_num, binary_start;
+  novel_hash_rec **code_to_nhr;
+};
+
+
+static FILE *text;
+static stdio_bitio_buffer textOutBuf;
+static u_long headLen;
+
+static novel_hash_table nht[2];
+
+int blk_start[2][33], blk_end[2][33];
+
+// 0 = non-word, 1 = word
+unsigned char whichWordType = 0;
+
+CompressTextInfoMap textInfoMap;
+
+
+static char_pool *new_pool (char_pool * pool) {
+  char_pool *p = (char_pool *) Xmalloc (sizeof (char_pool));
+  if (!p) FatalError (1, "Unable to allocate memory for pool");
+  
+  if (pool) pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+int init_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  char path[512];
+
+  // load the compression dictionary
+  if (LoadCompressionDictionary (make_name (fileName, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  // create the compressed text file and attach it to the text buffer
+  if (!(text = create_file (fileName, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))
+    return COMPERROR;
+  textOutBuf.attachFile (text);
+  textOutBuf.encodeStart ();
+  
+  // write out the compressed text header
+  memset (&cth, '\0', sizeof (cth));
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  // the length of the compressed header is the size of the
+  // magic number plus the size of the compressed text header
+  headLen = sizeof (u_long) + sizeof (cth);
+
+  // start off with non-word
+  whichWordType = 0;
+
+  // clear the text pointers
+  textInfoMap.erase (textInfoMap.begin(), textInfoMap.end());
+  
+  // set up the hash table of novel words
+  int i;
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; ++i)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	memset (nht[i].HashTable, '\0',
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		++num;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+static int compress_text (bitio_buffer &buffer, const u_char *s_in,
+			  int l_in, unsigned char &which,
+			  unsigned long &numWords) {
+  const u_char *end = s_in + l_in - 1;
+
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    int res;
+    
+    if (which) ++numWords;
+
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+
+    /* Search the hash table for Word */
+    if (ht[which]) {
+      register unsigned long hashval, step;
+      register int tsize = ht[which]->size;
+      register u_char **wptr;
+      HASH (hashval, step, Word, tsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	wptr = ht[which]->table[hashval];
+	if (wptr == NULL) {
+	  res = COMPERROR;
+	  break;
+	}
+
+	/* Compare the words */
+	s1 = Word;
+	s2 = *wptr;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval += step;
+	  if (hashval >= (unsigned long)tsize) hashval -= tsize;
+	  
+	} else {
+	  res = ht[which]->table[hashval] - ht[which]->words;
+	  break;
+	}
+      }
+    }
+    else
+      res = COMPERROR;
+    /* Check that the word was found in the dictionary */
+    if (res == COMPERROR)
+      {
+	if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	  {
+	    Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	    return (COMPERROR);
+	  }
+	if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	  {
+	    u_long i;
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    buffer.huff_encode (Word[0], lens_codes[which], lens_huff[which].clens, NULL);
+	    for (i = 0; i < Word[0]; ++i)
+	      buffer.huff_encode (Word[i + 1], char_codes[which],
+				  char_huff[which].clens, NULL);
+	  }
+	if (cdh.dict_type == MG_SEED_DICTIONARY)
+	  {
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    switch (cdh.novel_method)
+	      {
+	      case MG_NOVEL_HUFFMAN_CHARS:
+		{
+		  u_long i;
+		  buffer.huff_encode (Word[0], lens_codes[which],
+				      lens_huff[which].clens, NULL);
+		  for (i = 0; i < Word[0]; ++i)
+		    buffer.huff_encode (Word[i + 1], char_codes[which],
+					char_huff[which].clens, NULL);
+		}
+		break;
+	      case MG_NOVEL_DELTA:
+	      case MG_NOVEL_HYBRID:
+		{
+		  register unsigned long hashval, step;
+		  register novel_hash_table *h = &nht[which];
+		  register int hsize = h->HashSize;
+		  register novel_hash_rec *ent;
+		  HASH (hashval, step, Word, hsize);
+		  for (;;)
+		    {
+		      register u_char *s1, *s2;
+		      register int len;
+		      ent = h->HashTable + hashval;
+		      if (!ent->word)
+			{
+			  int len = *Word + 1;
+			  if ((unsigned)len > h->pool->left)
+			    h->pool = new_pool (h->pool);
+			  ent->word = h->pool->ptr;
+			  ent->ordinal_num = h->next_num++;
+			  memcpy (h->pool->ptr, Word, len);
+			  h->pool->ptr += len;
+			  h->pool->left -= len;
+			  ++h->HashUsed;
+			  break;
+			}
+		      /* Compare the words */
+		      s1 = Word;
+		      s2 = ent->word;
+		      len = *s1 + 1;
+		      for (; len; --len)
+			if (*s1++ != *s2++)
+			  break;
+		      
+		      if (!len)
+			break;
+		      
+		      hashval = (hashval + step);
+		      if (hashval >= (unsigned)hsize) hashval -= hsize;
+		    }
+		  
+		  switch (cdh.novel_method) {
+		  case MG_NOVEL_DELTA:
+		    buffer.delta_encode (ent->ordinal_num, NULL);
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    int k = 0;
+		    int j = ent->ordinal_num - 1;
+		    while (j > blk_end[which][k])
+		      ++k;
+		    assert (j - blk_start[which][k] + 1 >= 1 &&
+			    j - blk_start[which][k] + 1 <=
+			    blk_end[which][k] - blk_start[which][k] + 1);
+		    
+		    buffer.gamma_encode (k + 1, NULL);
+		    buffer.binary_encode (j - blk_start[which][k] + 1,
+					  blk_end[which][k] - blk_start[which][k] + 1,
+					  NULL);
+		    break;
+		  }
+		  
+		  if (h->HashUsed >= h->HashSize >> 1)
+		    {
+		      novel_hash_rec *ht;
+		      unsigned long size;
+		      unsigned long i;
+		      size = prime (h->HashSize * 2);
+		      if (!(ht = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * size)))
+			{
+			  Message ("Unable to allocate memory for table");
+			  return (COMPERROR);
+			}
+		      memset (ht, '\0', sizeof (novel_hash_rec) * size);
+		      
+		      for (i = 0; i < h->HashSize; ++i)
+			if (h->HashTable[i].word)
+			  {
+			    register u_char *wptr;
+			    register unsigned long hashval, step;
+			    
+			    wptr = h->HashTable[i].word;
+			    HASH (hashval, step, wptr, size);
+			    wptr = (ht + hashval)->word;
+			    while (wptr)
+			      {
+				hashval += step;
+				if (hashval >= size)
+				  hashval -= size;
+				wptr = (ht + hashval)->word;
+			      }
+			    ht[hashval] = h->HashTable[i];
+			  }
+		      Xfree (h->HashTable);
+		      h->HashTable = ht;
+		      h->HashSize = size;
+		    }
+		}
+		break;
+	      }
+	  }
+      }
+    else
+      {
+	buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+      }
+  }
+  
+  return COMPALLOK;
+}
+
+
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc) {
+  // we need to remember where the last text element ends
+  unsigned char endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+  unsigned long endPos = textOutBuf.GetBytesWritten () + headLen;
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // remember current place in compressed text
+    unsigned char startWhich = whichWordType;
+    unsigned char startBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    unsigned long startPos = textOutBuf.GetBytesWritten () + headLen;
+
+    // compress the text
+    if (compress_text (textOutBuf,
+		       &(here->text[0]),
+		       (*here).text.size(),
+		       whichWordType,
+		       cth.num_of_words) != COMPALLOK)
+      return COMPERROR;
+
+    endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    endPos = textOutBuf.GetBytesWritten () + headLen;
+    
+    // update the pointers if this is a level tag
+    if (tagInfo.levelTags.find ((*here).tagName) != tagInfo.levelTags.end()) {
+      if ((*here).elType == OpenTagE) {
+	// set the start of the document
+	textInfoMap[(*here).tagName].SetStart (startPos, startBit, startWhich);
+	
+      } else if ((*here).elType == CloseTagE) {
+	// set the end of the document
+	textInfoMap[(*here).tagName].SetEnd (endPos, endBit);
+	
+      } else {
+	// TextE, nothing needs to be done
+      }
+    }
+    
+    cth.num_of_bytes += (*here).text.size();
+    ++here;
+  }
+
+  // close off any unclosed tags
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    if ((*tiHere).second.inDoc) (*tiHere).second.SetEnd (endPos, endBit);
+    ++tiHere;
+  }
+  
+  // we've processed one more document
+  ++cth.num_of_docs;
+
+  return COMPALLOK;
+}
+
+
+static int write_aux_dict (char *fileName) {
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (fileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))
+    return COMPERROR;
+  
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+      
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+      
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+      
+      fwrite (&afh, sizeof (afh), 1, aux);
+      
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+static void estimate_compressed_aux_dict (void) {
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      memset (chars, '\0', sizeof (chars));
+      memset (lens, '\0', sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      ++lens[len];
+	      total_uncomp += len + 4;
+	      for (; len; --len)
+		++chars[*buf++];
+	    }
+	}
+      for (j = 0; j < 256; ++j)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; ++j)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+      
+      aux_compressed += (long unsigned int) ((Calculate_Huffman_Size (16, flens, lens) +
+					      Calculate_Huffman_Size (256, fchars, chars)) / 8);
+      
+    }
+}
+
+
+
+static bool WriteTextIdx (char *fileName, FTextLevel &textLevels) {
+  // create the text index
+  FILE *textIdxFile;
+  if (!(textIdxFile = create_file (fileName, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    // remember the level information
+    TextLevelInfo levelInfo;
+    levelInfo.levelTag = (*tiHere).first;
+    levelInfo.textIdxPtr = ftell (textIdxFile);
+    levelInfo.numEntries = (*tiHere).second.docPtrs.size();
+    textLevels.levelInfo[levelInfo.levelTag] = levelInfo;
+
+    // write out the index array for this level
+    if (!WriteTextIdxArray (textIdxFile, (*tiHere).second.docPtrs))
+      return false;
+    
+    ++tiHere;
+  }
+
+  // close the file
+  fclose (textIdxFile);
+
+  return true;
+}
+
+
+bool WriteTextLevels (char *fileName, const FTextLevel &textLevels) {
+  //  cout << textLevels;
+  
+  // create the text index
+  FILE *textLevelFile;
+  if (!(textLevelFile = create_file (fileName, TEXT_LEVEL_SUFFIX, "w+b",
+				     MAGIC_TEXT_LEVELS, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  if (!textLevels.Write (textLevelFile)) return false;
+  
+  // close the file
+  fclose (textLevelFile);
+
+  return true;
+}
+
+
+int done_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  // write out any remaining bits
+  textOutBuf.encodeDone();
+
+  // write out the compressed text header (with updated statistics)
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 || !cth.Write (text))
+    return COMPERROR;
+  fclose (text);
+
+  // write out the text index file and get the level information
+  FTextLevel textLevels;
+  if (!WriteTextIdx (fileName, textLevels)) return COMPERROR;
+  
+  // write out the text level information
+  if (!WriteTextLevels (fileName, textLevels)) return COMPERROR;
+
+  // write out the auxiliary dictionary
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID)) {
+    if (write_aux_dict (fileName) == COMPERROR) return COMPERROR;
+    estimate_compressed_aux_dict ();
+    
+  } else {
+    unlink (make_name (fileName, TEXT_DICT_AUX_SUFFIX, NULL));
+  }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/words.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/words.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/words.cpp	(revision 21112)
@@ -0,0 +1,122 @@
+/**************************************************************************
+ *
+ * words.cpp -- Functions for parsing out words from the source text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* It determines whether a given place in a UTF-8 encoded Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skippend. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+ 
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word) {
+  word.erase (word.begin(), word.end());
+
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      word.push_back (*textHere++); ++length;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
+
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord) {
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      mgWord[++length] = *textHere++;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+
+  mgWord[0] = length;
+  
+  return textHere;
+}
+
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd) {
+  register int charlength = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char(textHere, textEnd, &c);
+  
+  while (charlength > 0 && !is_unicode_letdig(c)) {
+    textHere += charlength;
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
Index: /main/tags/3.01/gsdl/src/mgpp/text/words.h
===================================================================
--- /main/tags/3.01/gsdl/src/mgpp/text/words.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/mgpp/text/words.h	(revision 21112)
@@ -0,0 +1,164 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "unitool.h"
+#include "UCArray.h"
+
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+#define MAXNUMERIC	4
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded Unicode string  is a unicode space. */
+
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with begining unicode spaces skippend. */
+
+#ifdef __cplusplus
+}
+#endif
+
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word);
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord);
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd);
+
+
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
Index: /main/tags/3.01/gsdl/src/phind/generate/.cvsignore
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/.cvsignore	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/.cvsignore	(revision 21112)
@@ -0,0 +1,4 @@
+Makefile
+suffix
+suffix2
+*.o
Index: /main/tags/3.01/gsdl/src/phind/generate/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/Makefile.in	(revision 21112)
@@ -0,0 +1,68 @@
+###########################################################################
+#
+# Makefile -- Makefile for phind
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 Gordon Paynter
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+CXX = @CXX@
+CXXFLAGS = @CXXFLAGS@
+
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = 
+LDFLAGS = @LDFLAGS@
+DPLIBS = 
+LIBS = @LIBS@
+GSDLOS = @gsdlos@
+INSTALL = @INSTALL@
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS) $< 
+
+
+HEADERS = suffix.h phrase.h check.h
+SOURCES = suffix.cpp phrase.cpp
+OBJECTS = phrase.o suffix.o
+EXEC    = suffix
+
+all : $(EXEC)
+
+install : $(EXEC)
+	$(INSTALL) -d ../../../bin/$(GSDLOS)
+	$(INSTALL) -m 775 -s $(EXEC) ../../../bin/$(GSDLOS)
+
+distclean:
+	rm -f $(OBJECTS) $(EXEC) Makefile
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CXXFLAGS) -- $(SOURCES)
+
+suffix : $(OBJECTS)
+	$(CXX) $(LDFLAGS) -o $@ $(OBJECTS)
+
+
Index: /main/tags/3.01/gsdl/src/phind/generate/check.h
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/check.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/check.h	(revision 21112)
@@ -0,0 +1,130 @@
+// check.h -- manipulate the suffixCheck array
+//
+// The suffixCheck array is used to prevent a phrase being added to
+// the results of getExpansions.  It works by declaring an array that
+// has one bit for each entry in the suffix array, which is cleared at
+// the start of getExpansions.  As getExpansions adds each phrase to
+// the result set, we set the bits in the suffixCheck array that
+// correspond to the positions of of the phrase in the suffixArray.
+
+// (The point is that getExpansions can later check whether a phrase
+// or its prefix has been seen before by looking up the bit
+// corresponding to its first occurence in the suffixCheck array: if
+// the bit is set, a subphrase of this phrase is already in the
+// results already, and the current phrase can be ignored.)
+
+#include "suffix.h"
+
+// The suffixCheck array
+#if defined (__WIN32__)
+typedef unsigned char check;
+#else
+#include <inttypes.h>
+typedef uint8_t check;
+#endif
+
+check *suffixCheck;
+
+// The length of the suffixCheck array
+cellcount checkLength;
+
+// Some useful bitmask constants 
+const check ALLBITS        = 255;
+const check LEFTMOSTBIT    = static_cast<check>(ALLBITS << 7);
+const check LEFTMOSTBITS[] = {static_cast<check>(ALLBITS << 8), 
+			      static_cast<check>(ALLBITS << 7), 
+			      static_cast<check>(ALLBITS << 6), 
+			      static_cast<check>(ALLBITS << 5), 
+			      static_cast<check>(ALLBITS << 4),
+			      static_cast<check>(ALLBITS << 3), 
+			      static_cast<check>(ALLBITS << 2), 
+			      static_cast<check>(ALLBITS << 1),  
+			      static_cast<check>(ALLBITS << 0)};
+
+
+// Allocate the initial memory for suffixCheck
+inline void allocateSuffixCheck(cellcount number_of_symbols) 
+{
+  checkLength = number_of_symbols / (sizeof(check) * 8) + 1;
+  suffixCheck = new check[checkLength];
+  if (suffixCheck == NULL) {
+    cerr << "Suffix error: out of memory allocating suffixCheck array" << endl; 
+    exit(2);
+  }
+  memset(suffixCheck, 0, sizeof(check) * checkLength);
+}
+
+
+// Set all bits in suffixCheck to 0
+inline void clearSuffixCheck() 
+{
+  memset(suffixCheck, 0, sizeof(check) * checkLength);
+}
+
+
+// Get the value of a particular bit in suffixCheck
+inline int getSuffixCheck(cellindex position) 
+{
+  cellindex cell = position >> 3;
+  check remainder = position & 0x07; // the last 3 bits
+  if (suffixCheck[cell] & (LEFTMOSTBIT >> remainder)) {
+    return 1;
+  }
+  return 0;
+}                                                                               
+
+// Set the value of a bit in suffixCheck to 1
+inline void setSuffixCheck(cellindex suff) 
+{
+  cellindex cell = suff >> 3;
+  check remainder = suff & 0x07u; // the last 3 bits
+  suffixCheck[cell] |= (LEFTMOSTBIT >> remainder);
+}  
+
+
+// Set the bits in a cell in the suffixArray.
+// Bits are indexed 0-7.
+inline void setCellBits(cellindex cell_offset, unsigned first_bit, unsigned last_bit) 
+{
+  unsigned number_of_bits = last_bit - first_bit + 1;
+  check mask = (LEFTMOSTBITS[number_of_bits] >> first_bit);
+  suffixCheck[cell_offset] |= mask;
+}
+
+
+inline void setSuffixCheck(cellindex first, cellindex last) 
+{
+  // Find the first and last cells in which bits are set
+  cellindex first_cell = first >> 3;
+  cellindex last_cell = last >> 3;
+
+  // If all the values are in the same cell, set them
+  if (first_cell == last_cell) {
+    setCellBits(first_cell, (first & 0x07u), (last & 0x07u));
+    return;
+  }
+
+  // Set the bits in the first and last cells
+  setCellBits(first_cell, (first & 0x07u), 7);
+  setCellBits(last_cell, 0, (last & 0x07u));
+
+  // Set the bits in the intermediate cells
+  ++first_cell;
+  if(last_cell > first_cell)
+    memset(suffixCheck + first_cell, ALLBITS, 
+	   (last_cell - first_cell) * sizeof(check));
+}
+
+
+// Print the suffixCheck array (for debugging)
+void printSuffixCheck() 
+{
+  for (cellcount i = 0; i < checkLength; ++i) {
+    cout << "cell " << i << "  \t";
+    cout << "(" << (i * 8) << "-" << (i * 8 + 7) << ")    \t";
+    for (cellindex j = 0; j < 8; ++j)
+      cout << getSuffixCheck(i * 8 + j);
+    cout << " (" << (unsigned int) suffixCheck[i] << ")\n";
+  }
+}
+
Index: /main/tags/3.01/gsdl/src/phind/generate/phrase.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/phrase.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/phrase.cpp	(revision 21112)
@@ -0,0 +1,853 @@
+/**********************************************************************
+ *
+ * phrase.cpp -- implementation of the phrase object used by suffix.cpp
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include <assert.h>
+#include <stdio.h>
+
+#if defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+#if defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#include "suffix.h"
+#include "phrase.h"
+
+
+// Phrase constructor functions
+
+Phrase::Phrase(symbol *words, cellcount size, int direction) {
+  empty();
+  length = size;
+
+  if (direction == SUFFIX) {
+     forward = words;
+     back = forward + size - 1;
+  } else {
+    back = words;
+    forward = back - size + 1;
+  }
+}
+
+
+Phrase::Phrase() {
+  empty();
+}
+
+
+Phrase::Phrase(const Phrase &p) {
+  forward = p.forward;
+  back = p.back;
+  length = p.length;
+
+  suffixFound = p.suffixFound;
+  prefixFound = p.prefixFound;
+
+  firstSuffix = p.firstSuffix;
+  lastSuffix  = p.lastSuffix;
+  firstSuffixIndex = p.firstSuffixIndex;
+  lastSuffixIndex  = p.lastSuffixIndex;
+  suffixFrequency  = p.suffixFrequency;
+
+  firstPrefix = p.firstPrefix;
+  lastPrefix  = p.lastPrefix;
+  firstPrefixIndex = p.firstPrefixIndex;
+  lastPrefixIndex  = p.lastPrefixIndex;
+  prefixFrequency  = p.prefixFrequency;
+
+  uniqueSuffixExtension = p.uniqueSuffixExtension;
+  uniquePrefixExtension = p.uniquePrefixExtension;
+}
+
+
+// Empty the contents of a phrase
+
+int Phrase::empty() {
+
+  forward = back = NULL;
+  length = 0;
+
+  suffixFound = prefixFound = 0;
+  firstSuffix = firstPrefix = NULL;
+  lastSuffix = lastPrefix = NULL;
+
+  firstSuffixIndex = lastSuffixIndex = suffixFrequency = 0;
+  firstPrefixIndex = lastPrefixIndex = prefixFrequency = 0;
+
+  uniqueSuffixExtension = uniquePrefixExtension = -1;
+
+  return 0;
+}
+
+// Remove the details of the location where a suffix/prefix
+// is found in the array.
+
+int Phrase::clearSuffix() {
+  suffixFound = 0;
+  firstSuffix = lastSuffix = NULL;
+  firstSuffixIndex = lastSuffixIndex = suffixFrequency = 0;
+  uniqueSuffixExtension = -1;
+  return 0;
+}
+
+int Phrase::clearPrefix() {
+  prefixFound = 0;
+  firstPrefix = lastPrefix = NULL;
+  firstPrefixIndex = lastPrefixIndex = prefixFrequency = 0;
+  uniquePrefixExtension = -1;
+  return 0;
+}
+
+// Increase the length of s phrase
+//
+// We increase the length of a string "in place".  When we expand
+// the suffix, however, we invalidate the prefix data, and vice-versa.
+
+int Phrase::increaseSuffixLength(cellcount l) {
+  length = l;
+  clearPrefix();
+  back = forward + l - 1;
+  return 0;
+}
+
+int Phrase::increasePrefixLength(cellcount l) {
+  length = l;
+  clearSuffix();
+  forward = back - l + 1;
+  return 0;
+}
+
+// Shorten a phrase by one symbol
+int Phrase::shortenByOneAtSuffix() {
+  --length;
+  --back;
+  if (phraseMode==STOPWORDS) {
+    while (*back >=firstStopSymbol && *back <= lastStopSymbol) {
+      --length;
+      --back;
+    }
+  }  
+  clearSuffix();
+  clearPrefix();
+  return 0;
+}
+
+int Phrase::shortenByOneAtPrefix() {
+  --length;
+  ++forward;
+  if (phraseMode==STOPWORDS) {
+    while (*forward >=firstStopSymbol && *forward <= lastStopSymbol) {
+      --length;
+      ++forward;
+    }
+  }
+  clearSuffix();
+  clearPrefix();
+  return 0;
+}
+
+
+// Output a phrase to a stream
+ostream &operator<<(ostream &stream, const Phrase &phrase)
+{
+  assert(phrase.forward);
+  symbol *s = phrase.forward;
+
+  stream << "s" << *s++;
+  for (cellcount i = 1; i < phrase.length; ++i)
+    stream << " s" << *s++;
+
+  return stream;
+}
+
+
+// Convert the phrase to a string
+// Note thgat you have to delete the memory yourself.
+char *Phrase::toString() 
+{
+  assert(forward);
+
+  char *str;
+  str = new char[length*20];
+  symbol *s = forward;
+  sprintf(str, "s%d", *s++);
+
+  for (cellcount i = 1; i < length; ++i) {
+    sprintf(str, "%s s%d", str, *s++);
+  }
+
+  return str;
+}
+
+
+// Compare a phrase to an another array of symbols
+//
+// Given an array of words with a specified length, 
+// compare those words to this phrase.
+//
+// Return 0 if the two phrases are the same over length length
+// Return 1 if the phrase is "greater" than tyhe words, or
+// Return -1 if the phrase is less.
+
+int Phrase::compareSuffix(symbol *words, cellcount length) {
+  assert(forward);
+  symbol *p = forward;
+  
+  for (cellcount i = 0; i < length; ++i) {
+    if (*p > *words) {
+      return 1;
+    } else if (*p < *words) {
+      return -1;
+    } else {
+      *p++;
+      *words++;
+    }
+  }
+
+  return 0;
+
+}
+
+int Phrase::comparePrefix(symbol *words, cellcount length) {
+  
+  assert(back);
+  symbol *p = back;
+  
+  for (cellcount i = 0; i < length; ++i) {
+    if (*p > *words) {
+      return -1;
+    } else if (*p < *words) {
+      return 1;
+    } else {
+      *p--;
+      *words--;
+    }
+  }
+
+  return 0;
+
+}
+
+
+// Does the phrase have a unique suffix/prefix extension?
+//
+// For suffix:
+//   If uniqueSuffixExtensions is 1, then phrase has some unique expansion. 
+//   If it is 0 then it does not.  If it is -1 then we don't know, and have
+//   to calculate it from the suffix array.  Same goes for prefixes.
+
+int Phrase::hasUniqueSuffixExtension() {
+
+  // if we don't know, then work it out
+  if (uniqueSuffixExtension == -1) {
+
+    ensureSuffixFound();
+
+    // pointers to the phrase's first and last occurances in te suffixArray
+    symbol *fst = firstSuffix + length;
+    symbol *lst = lastSuffix + length;
+
+    // in ANYPHRASE mode, simply check the next symbol
+    if (phraseMode == ANYPHRASE) {
+      if ((*fst == *lst) && (*fst > LASTDELIMITER)) {
+	uniqueSuffixExtension = 1;
+      } else {
+	uniqueSuffixExtension = 0;
+      }
+    }
+    // in STOPWORDS mode, make sure there is a unique next content symbol
+    else {
+      uniqueSuffixExtension = 0;
+      while ((*fst == *lst) && (*fst > LASTDELIMITER)) {
+	if (*fst >= firstContentSymbol) {
+	  uniqueSuffixExtension = 1;
+	  break;
+	}
+	++fst;
+	++lst;
+      }
+    }
+  }
+
+  return uniqueSuffixExtension;
+}
+
+int Phrase::hasUniquePrefixExtension() {
+
+  // if we don't know, then work it out
+  if (uniquePrefixExtension == -1) {
+
+    ensurePrefixFound();
+
+    // pointers to the phrase's first and last occurances in the prefixArray
+    symbol *fst = firstPrefix - length;
+    symbol *lst = lastPrefix - length;
+
+    // in ANYPHRASE mode, simply check the next symbol
+    if (phraseMode == ANYPHRASE) {
+      if ((*fst == *lst) && (*fst > LASTDELIMITER)) {
+	uniquePrefixExtension = 1;
+      } else {
+	uniquePrefixExtension = 0;
+      }
+    }
+    // in STOPWORDS mode, make sure there is a unique next content symbol
+    else {
+      uniquePrefixExtension = 0;
+      while ((*fst == *lst) && (*fst > LASTDELIMITER)) {
+	if (*fst >= firstContentSymbol) {
+	  uniquePrefixExtension = 1;
+	  break;
+	}
+	--fst;
+	--lst;
+      }
+    }
+  }
+
+  return uniquePrefixExtension;
+}
+
+
+// Expand a phrase with a unique suffix/prefix by 1 symbol
+//
+// Note that in STOPWORDS mode a "unique extension" means a unique
+// extension that ends on a content symbol.  Although we may extend 
+// such a phrase by 2 or more symbols, we will only extend it by a
+// single content symbol.
+
+int Phrase::expandUniqueSuffixExtensionByOne() {
+  assert(suffixFound);
+  assert(uniqueSuffixExtension == 1);
+
+  // in ANYPHRASE mode, expand the phrase by one symbol
+  if (phraseMode == ANYPHRASE) {
+    increaseSuffixLength(length + 1);
+  }
+  // in STOPWORDS mode, expand the phrase up to the next content symbol
+  else {
+    symbol *next = firstSuffix + length;
+    cellcount newlength = length + 1;
+
+    while (*next < firstContentSymbol) {
+      ++next;
+      ++newlength;
+    }
+    increaseSuffixLength(newlength);
+  }
+
+  uniqueSuffixExtension = -1;
+  return 0;
+}
+
+int Phrase::expandUniquePrefixExtensionByOne() {
+  assert(prefixFound);
+  assert(uniquePrefixExtension == 1);
+
+  // in ANYPHRASE mode, expand the phrase by one symbol
+  if (phraseMode == ANYPHRASE) {
+    increasePrefixLength(length + 1);
+  }
+  // in STOPWORDS mode, expand the phrase to the next content symbol
+  else {
+    symbol *next = firstPrefix - length;
+    cellcount newlength = length + 1;
+    while (*next < firstContentSymbol) {
+      --next;
+      ++newlength;
+    }
+    increasePrefixLength(newlength);
+
+  }
+
+  uniquePrefixExtension = -1;
+  return 0;
+}
+
+
+// Find the phrase in a suffixArray
+//
+// Given a suffix array, find the first and last occurances of the phrase.
+// The begin and end parameters identify which cells to search (inclusive). 
+
+int Phrase::findFirstAndLastSuffix() {
+  
+  return findFirstAndLastSuffix(0, inputLength-1);
+}
+
+int Phrase::findFirstAndLastSuffix(cellindex begin, cellindex end) {
+
+  // First find *any* occurance of the phrase
+  assert(begin <= end);
+
+  // if we're only searching one cell, it is very easy
+  if (begin == end) {
+    assert(compareSuffix(suffixArray[begin], length) == 0);
+    firstSuffix = lastSuffix = suffixArray[begin];
+    firstSuffixIndex = lastSuffixIndex = begin;
+    suffixFrequency = 1;
+    suffixFound = 1;
+    return 0;
+  }
+
+  cellindex c;
+  int cmp;
+
+  do {
+    c = (end + begin) / 2;
+    cmp = compareSuffix(suffixArray[c], length);
+    if (cmp == 1) {
+      // target phrase is lower than phrase at suffixArray[c]
+      begin = c + 1;
+    } else if (cmp == -1) {
+      // target phrase is higher than phrase at suffixArray[c]
+      end = c - 1;
+    }
+  } while (cmp != 0);
+    
+  cellindex lastbegin = c;
+  cellindex lastend = end;
+
+  // Next find the first occurance of the phrase.
+  // We know that the first occurance must be between the 
+  // current value of begin and the current value of c.
+  end = c;
+
+  do {
+    if (begin == end) {
+      c = begin;
+      cmp = 0;
+    } else {
+      c = (begin + end) / 2;
+      cmp = compareSuffix(suffixArray[c], length);
+      if (cmp == 1) {
+	// target phrase is lower than phrase at suffixArray[c]
+	begin = c + 1;
+      } else {
+	assert(cmp == 0);
+	// target phrase is the same as the phrase at suffixArray[c].  However,
+	// to find the first occurance, suffixArray[c] must be the same as the 
+	// phrase, but suffixArray[c-1] must be different.
+	if (c>0) {
+	  cmp = compareSuffix(suffixArray[c-1], length);
+	  if (cmp == 0) {
+	    end = c - 1;
+	    assert(end >= begin);
+	    cmp = 1;
+	  } else {
+	    cmp = 0;
+	  }
+	}
+      }
+    }
+  } while (cmp != 0);
+
+  // we have found the first location
+  firstSuffixIndex = c;
+  firstSuffix = suffixArray[c];
+  
+  // Next find the last occurance of the phrase.
+  // We previously stored some bounds for its loccation.
+  begin = lastbegin;
+  end = lastend;
+  
+  do {
+    if (begin == end) {
+      c = begin;
+      cmp = 0;
+    } else {
+      c = (begin + end) / 2;
+      cmp = compareSuffix(suffixArray[c], length);
+      if (cmp == -1) {
+	// target phrase is greater than phrase at suffixArray[c]
+	end = c - 1;
+      } else {
+	assert(cmp == 0);
+	// target phrase is the same as the phrase at suffixArray[c].  However,
+	// to find the last occurance, one of two additional condiditons must be met:
+	// either c must be the very last cell in the array, or
+	// suffixArray[c+1] must be different from phrase
+	if (c < inputLength-1) {
+	  cmp = compareSuffix(suffixArray[c+1], length);
+	  if (cmp == 0) {
+	    begin = c + 1;
+	    cmp = -1;
+	  } else {
+	    cmp = 0;
+	  }
+	} 
+      }
+    }
+  } while (cmp != 0);
+  
+  lastSuffixIndex = c;
+  lastSuffix = suffixArray[c];
+  suffixFrequency = lastSuffixIndex - firstSuffixIndex + 1;
+  suffixFound = 1;
+
+  return 0;
+}
+
+
+// Find the phrase in a prefix array
+//
+// Given a prefix array, find the first and last occurances of the phrase.
+// The begin and end parameters identify which cells to search (inclusive). 
+
+int Phrase::findFirstAndLastPrefix() {
+  return findFirstAndLastPrefix(0, inputLength-1);
+}
+
+int Phrase::findFirstAndLastPrefix(cellindex begin, cellindex end) {
+
+  // First find *any* occurance of the phrase
+  assert(begin <= end);
+
+  // if we're only searching one cell, it is very easy
+  if (begin == end) {
+    assert(comparePrefix(prefixArray[begin], length) == 0);
+    firstPrefix = lastPrefix = prefixArray[begin];
+    firstPrefixIndex = lastPrefixIndex = begin;
+    prefixFrequency = 1;
+    prefixFound = 1;
+    return 0;
+  }
+
+  cellindex c;
+  int cmp;
+
+  do {
+    c = (end + begin) / 2;
+    cmp = comparePrefix(prefixArray[c], length);
+    if (cmp == 1) {
+      begin = c + 1;
+    } else if (cmp == -1) {
+      end = c - 1;
+    }
+  } while (cmp != 0);
+    
+  cellindex lastbegin = c;
+  cellindex lastend = end;
+
+  // Next find the first occurance of the phrase.
+  // We know that the first occurance must be between the 
+  // current value of begin and the cureent value of c.
+  end = c;
+
+  do {
+    if (begin == end) {
+      c = begin;
+      cmp = 0;
+    } else {
+      c = (begin + end) / 2;
+      cmp = comparePrefix(prefixArray[c], length);
+      if (cmp == 1) {
+	// target phrase is lower than phrase at prefixArray[c]
+	begin = c + 1;
+      } else {
+	assert(cmp == 0);
+	// target phrase is the same as the phrase at prefixArray[c].  However, to
+	// find the first occurance, one of two additional condiditons must be met:
+	// either  c == the first cell in prefix array
+	// or      prefixArray[c-1] != phrase.
+	if (c > 0) {
+	  cmp = comparePrefix(prefixArray[c-1], length);
+	  if (cmp == 0) {
+	    end = c - 1;
+	    assert(end >= begin);
+	    cmp = 1;
+	  } else {
+	    cmp = 0;
+	  }
+	}
+      }
+    }
+  } while (cmp != 0);
+
+  // we have found the first location
+  firstPrefixIndex = c;
+  firstPrefix = prefixArray[c];
+
+  // Next find the last occurance of the phrase.
+  // We previously stored some bounds for its loccation.
+  begin = lastbegin;
+  end = lastend;
+  
+  do {
+    if (begin == end) {
+      c = begin;
+      cmp = 0;
+    } else {
+      c = (begin + end) / 2;
+      cmp = comparePrefix(prefixArray[c], length);
+      if (cmp == -1) {
+	// target phrase is greater than phrase at prefixArray[c]
+	end = c - 1;
+      } else {
+	assert(cmp == 0);
+	// target phrase is the same as the phrase at prefixArray[c].  However,
+	// to find the last occurance, prefixArray[c] must be the same as the 
+	// phrase, but prefixArray[c+1] must be different.
+	cmp = comparePrefix(prefixArray[c+1], length);
+	if (cmp == 0) {
+	  begin = c + 1;
+	  cmp = -1;
+	} else {
+	  cmp = 0;
+	}
+      } 
+    }
+  } while (cmp != 0);
+  
+  lastPrefixIndex = c;
+  lastPrefix = prefixArray[c];
+  prefixFrequency = lastPrefixIndex - firstPrefixIndex + 1;
+  prefixFound = 1;
+
+  return 0;
+}
+
+
+// Calculate a set of initial suffix/prefix candidates
+//
+// Expand the phrase to find the initial 
+// set of candidates that may be extensions of that phrase,
+// and add them to the end of the results vector
+
+void Phrase::initialSuffixCandidates(vector<Phrase> &results) {
+
+  ensureSuffixFound();
+  Phrase next;
+  cellindex i = firstSuffixIndex;
+
+  // Find all the expansions of Phrase p
+  while (i <= lastSuffixIndex) {
+
+    // create new phrase exactly one longer than the current one
+    next = newPhraseShortestSuffixExpansion(i);
+
+    // If the expansion occurs more than once and is not delimited, expand it
+    if ((*(next.back) > LASTDELIMITER) && (next.suffixFrequency >= minOccurs)) {
+      next.expandWhileUniqueSuffixExtension();
+      results.push_back(next);
+    }
+    
+    // Move onto the next expansion
+    i = next.lastSuffixIndex + 1;
+  }
+}
+
+
+void Phrase::initialPrefixCandidates(vector<Phrase> &results) {
+
+  ensurePrefixFound();
+  Phrase next;
+  cellindex i = firstPrefixIndex;
+
+  // Find all the expansions of Phrase p
+  while (i <= lastPrefixIndex) {
+
+    // create new phrase exactly one longer than the current one
+    next = newPhraseShortestPrefixExpansion(i);
+
+    // If the expansion occurs more than once and is not delimited, expand it
+    if ((*(next.forward) > LASTDELIMITER) && (next.prefixFrequency >= minOccurs)) {
+      next.expandWhileUniquePrefixExtension();
+      results.push_back(next);
+    }
+    
+    // Move onto the next expansion
+    i = next.lastPrefixIndex + 1;
+  }
+}
+
+
+// Create a new phrase that is longer then the current one, but as short
+// as possible.  In ANYPHRASE mode this means the new phrase is exactly
+// one symbol longer than the current one, but in STOPWORD mode the expansion
+// must introduce a new content symbol
+//
+// Note that we never expand across delimiters; if we encounter a delimiter
+// then this becomes the "end" symbol of the phrase.  The calling function 
+// should check for this condition.
+
+Phrase Phrase::newPhraseShortestSuffixExpansion(cellindex i) {
+  
+  // create a new phrase exactly one symbol longer than the current one
+  Phrase p(suffixArray[i], length + 1, SUFFIX);
+
+  if (phraseMode == STOPWORDS) {
+    // in STOPWORDS mode we must introduce a new content symbol
+    while ((*(p.back) >= firstStopSymbol) &&
+	   (*(p.back) <= lastStopSymbol)) {
+      p.increaseSuffixLength(p.length + 1);
+    }
+  }
+
+  p.findFirstAndLastSuffix(i, lastSuffixIndex);
+  return p;
+}
+
+Phrase Phrase::newPhraseShortestPrefixExpansion(cellindex i) {
+  
+  // create a new phrase exactly one symbol longer than the current one
+  Phrase p(prefixArray[i], length + 1, PREFIX);
+
+  if (phraseMode == STOPWORDS) {
+    // in STOPWORDS mode we must introduce a new content symbol
+    while ((*(p.forward) >= firstStopSymbol) &&
+	   (*(p.forward) <= lastStopSymbol)) {
+      p.increasePrefixLength(p.length + 1);
+    }
+  }
+
+  p.findFirstAndLastPrefix(i, lastPrefixIndex);
+  return p;
+}
+
+
+// Expand a phrase until it no longer has a unique suffix extension
+// 
+// If the phrase only occurs once in the suffix array, do nothing.
+//
+// If we are in stopwords mode then we expand if there is a unique 
+// suffix expansion and this expansion is a content word.  We do this
+// with the content_len variable, which holds the length of the longest
+// phrase ending in a content word. 
+
+int Phrase::expandWhileUniqueSuffixExtension() {
+  ensureSuffixFound();
+
+  // if the phrase occurs only once, do nothing
+  if (suffixFrequency < minOccurs)
+    return 0;
+
+  // if we already know there's no unique suffix extension, do nothing
+  if (uniqueSuffixExtension == 0)
+    return 0;
+
+  // count the length over which the cells are the same
+  // and no delimiter is crossed.
+  symbol *fst = firstSuffix;
+  symbol *lst = lastSuffix;
+  cellindex len = 0;
+  cellindex content_len = 0;
+
+  // Calculate the length over which the phrases match
+  while((*fst == *lst) && (*fst > LASTDELIMITER)) {
+    ++len;
+    if (*fst > lastStopSymbol) content_len = len;
+    ++fst;
+    ++lst;
+  }
+
+  // in ANYPHRASE mode, expand the phrase for as long as possible
+  if (phraseMode == ANYPHRASE) {
+    if (len > length) {
+      increaseSuffixLength(len);
+    }
+  } 
+  // in STOPWORDS mode, expand only as far as the last contentword
+  else {
+    if (content_len > length) {
+      increaseSuffixLength(content_len);
+    }
+  }
+
+  uniqueSuffixExtension = 0;
+  return 0;
+}
+
+int Phrase::expandWhileUniquePrefixExtension() {
+  ensurePrefixFound();
+
+  // if the phrase occurs only once, do nothing
+  if (prefixFrequency < minOccurs)
+    return 0;
+
+  // if we already know there's no unique extension, do nothing
+  if (uniquePrefixExtension == 0)
+    return 0;
+
+  // count the length over which the cells are the same
+  // and no delimiter is crossed.
+  symbol *fst = firstPrefix;
+  symbol *lst = lastPrefix;
+  cellindex len = 0;
+  cellindex content_len = 0;
+
+  // Calculate the length over which the phrases match
+  while((*fst == *lst) && (*fst > LASTDELIMITER)) {
+    ++len;
+    if (*fst > lastStopSymbol) content_len = len;
+    --fst;
+    --lst;
+  }
+
+  // in ANYPHRASE mode, expand the phrase for as long as possible
+  if (phraseMode == ANYPHRASE) {
+    if (len > length) {
+      increasePrefixLength(len);
+    }
+  }
+  // in STOPWORDS mode, expand only as far as the last contentword
+  else {
+    if (content_len > length) {
+      increasePrefixLength(content_len);
+    }
+  }
+
+  uniquePrefixExtension = 0;
+  return 0;
+}
+
+// Compare the length of two phrases
+//
+// Given two phrases, return true if the first is shorter/longer,
+// otherwise return false.  For use in various sort functions.
+
+bool isShorter(Phrase p1, Phrase p2) {
+  if (p1.length < p2.length) {
+    return true;
+  }
+  return false;
+}
+
+bool isLonger(Phrase p1, Phrase p2) {
+  if (p1.length > p2.length) {
+    return true;
+  }
+  return false;
+}
+
+
Index: /main/tags/3.01/gsdl/src/phind/generate/phrase.h
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/phrase.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/phrase.h	(revision 21112)
@@ -0,0 +1,162 @@
+/**********************************************************************
+ *
+ * phrase.h -- definition of the phrase object used by suffix.cpp
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef PHRASE_H
+#define PHRASE_H
+
+// use the standard namespace
+#if !defined (GSDL_NAMESPACE_BROKEN)
+#if defined(GSDL_USE_OBJECTSPACE)
+using namespace ospace::std;
+#else
+using namespace std;
+#endif
+#endif
+
+#if defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+#include "suffix.h"
+
+class Phrase {
+
+public:
+
+  // The phrase itself is stored with two pointers: forward points to
+  // its first cell, back points to its last.  The length is always
+  // stored in length.  If one of these is set, all must be set, and
+  // it must be true that (forward + length - 1) = back.
+  symbol *forward;
+  symbol *back;
+  cellcount length;
+  
+  // Phrase location in the suffix array
+  int suffixFound;
+  symbol *firstSuffix;
+  symbol *lastSuffix;
+  cellindex firstSuffixIndex;
+  cellindex lastSuffixIndex;
+  cellcount suffixFrequency;
+
+  // Phrase location in the prefix array
+  int prefixFound;
+  symbol* firstPrefix;
+  symbol* lastPrefix;
+  cellindex firstPrefixIndex;
+  cellindex lastPrefixIndex;
+  cellcount prefixFrequency;
+
+  // Constructor functions
+  Phrase();
+  Phrase(const Phrase &p);
+
+  // A "partial" constructor: the first argument is an array of words,
+  // second is its length, third is the direction (SUFFIX or PREFIX)
+  // in which the words should be read (defaults to SUFFIX).
+  Phrase(symbol *words, cellcount size, int direction);
+
+  // Represent the phrase as an arracy of characters
+  // You will have to call "delete []" on the array returned. 
+  char *toString();
+
+  // Find an initial set of candidate phrases in the suffix/prefix array
+  void initialSuffixCandidates(vector<Phrase> &results);
+  void initialPrefixCandidates(vector<Phrase> &results);
+  
+  // Does the phrase have a unique extension?
+  int hasUniqueSuffixExtension();
+  int hasUniquePrefixExtension();
+  
+  // Extend a phrase by exactly one symbol
+  int expandUniquePrefixExtensionByOne();
+  int expandUniqueSuffixExtensionByOne();
+
+  // Extend a phrase until it no longer has a unique extanesion
+  int expandWhileUniqueSuffixExtension();
+  int expandWhileUniquePrefixExtension();
+
+  // Shorten a phrase by one symbol
+  int shortenByOneAtSuffix();
+  int shortenByOneAtPrefix();
+
+  // Find the phrase in the suffix/prefix array
+  int findFirstAndLastSuffix();
+  int findFirstAndLastPrefix();
+  int findFirstAndLastSuffix(cellindex begin, cellindex end);
+  int findFirstAndLastPrefix(cellindex begin, cellindex end);
+
+  // Make sure the phrase location in the suffix/prefix array is known
+  inline void Phrase::ensureSuffixFound() {
+    if (!suffixFound)
+      findFirstAndLastSuffix();
+  }
+  inline void Phrase::ensurePrefixFound() {
+    if (!prefixFound)
+      findFirstAndLastPrefix();
+  }
+
+  // Output a phrase to a stream
+  friend ostream &operator<<(ostream &stream, const Phrase &phrase);
+
+  int uniqueSuffixExtension;
+  int uniquePrefixExtension;
+
+private:
+
+  // Does the phrase have a unique suffix/prefix extension?
+  // if yes, then 1; if no then 0; if unknown then -1;
+
+  // reset a phrase
+  int empty();
+
+  // reset phrase information relating to location in suffix/prefix array
+  int clearSuffix();
+  int clearPrefix();
+
+  // increase the length of a phrase "in place"
+  int Phrase::increaseSuffixLength(cellcount l);
+  int Phrase::increasePrefixLength(cellcount l);
+    
+  // Compare the phrase to a given array of symbols
+  int compareSuffix(symbol *words, cellcount length);
+  int comparePrefix(symbol *words, cellcount length);
+
+  // Create a new phrase that is longer than this one, yet as short as possible.
+  Phrase Phrase::newPhraseShortestSuffixExpansion(cellindex i);
+  Phrase Phrase::newPhraseShortestPrefixExpansion(cellindex i);
+
+};
+
+
+bool isShorter(Phrase p1, Phrase p2);
+bool isLonger(Phrase p1, Phrase p2);
+
+#endif
+
Index: /main/tags/3.01/gsdl/src/phind/generate/suffix.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/suffix.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/suffix.cpp	(revision 21112)
@@ -0,0 +1,1094 @@
+/**********************************************************************
+ *
+ * Suffix.cpp -- Extract the repeated phrases in the input with suffix
+ *               and prefix arrays (cgn & gwp's simpler algorithm,
+ *               and kjm's improvements).
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software from the 
+ * New Zealand Digital Library Project at the University of Waikato, 
+ * New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(GSDL_USE_IOS_H)
+#  include <fstream.h>
+#  include <iostream.h>
+#else
+#  include <fstream>
+#  include <iostream>
+#endif
+
+#if defined(GSDL_USE_STL_H)
+#  if defined(GSDL_USE_ALGO_H)
+#    include <algo.h>
+#  else
+#    include <algorithm.h>
+#  endif
+#  include <vector.h>
+#else
+#  include <algorithm>
+#  include <vector>
+#endif
+
+#include "suffix.h"
+#include "phrase.h"
+#include "check.h"
+
+// Global variables declared in suffix.h
+cellcount inputLength;
+
+symbol   *symbols;
+symbol  **suffixArray;
+symbol  **prefixArray;
+
+// How many documents are in this collection?
+cellcount numberOfDocuments;
+symbol  **documentArray;
+
+
+// Do we accept any phrase, or do we eliminate those ending with stopwords ?
+int phraseMode = ANYPHRASE; //STOPWORDS;
+// What is the minimum phrase frequency for a phrase to be included in the hierarchy
+int minOccurs = 2;
+
+// The filestem of the collection's phindex directory
+char collection[FILENAME_MAX];
+
+
+// The ranges of the stopword and content-word symbols for the collection
+symbol firstStopSymbol = 0;
+symbol lastStopSymbol = 0;
+symbol firstContentSymbol = 0;
+symbol lastContentSymbol = 0;
+
+
+// Some useful comparison functions, defined below.
+int suffixCompare(const void *, const void *);
+int prefixCompare(const void *, const void *);
+int pointerCompare(const void *, const void *);
+
+                                           
+// Functions for implementing "phrase memory".  These let us "remember"
+// each phrase that we've expanded without using too much memory.
+void initialisePhraseMemory();
+void rememberThisPhrase(cellindex index, cellcount length);
+bool isPhraseStored(cellindex index, cellcount length);
+void deletePhraseMemory();
+
+
+// how much output do we want?
+int verbosity = 1;
+
+
+// Get a phrase's expansions
+//
+// Get the set of "minimal", "maximal", non-unique expansions of a
+// phrase p, using the simpler algorithm that Craig and Gordon came up
+// with at Google.
+//
+// Returns a vector of Expansions.
+
+void getExpansions(Phrase &p, vector<Phrase> &results) {
+
+  // 1. Get the initial candidates
+  vector<Phrase> candidates;
+  p.initialSuffixCandidates(candidates);
+  int suffcands = candidates.size();
+  p.initialPrefixCandidates(candidates);
+
+  if (candidates.size() == 0)
+    return;
+
+  vector<Phrase>::iterator i;
+  for (i = candidates.begin(); i != candidates.end(), suffcands>0; ++i, --suffcands) {
+    i->expandWhileUniquePrefixExtension();
+    i->ensureSuffixFound();
+  }
+  for (i; i != candidates.end(); ++i) {
+    i->expandWhileUniqueSuffixExtension();
+  }
+
+  // 3. Ensure minimality: ignore phrases whose subphrases are also found
+  results.clear();
+
+  // Initialise the candidates, check array, and various variables.
+  sort(candidates.begin(), candidates.end(), isShorter);
+  unsigned minimum_length = candidates.begin()->length;
+  clearSuffixCheck();
+  
+  // Try to add each candidate to the results set, ignoring the non-minimal
+  for (vector<Phrase>::iterator candidate = candidates.begin(); 
+       candidate != candidates.end(); ++candidate) {
+
+    // Make a copy of candidate to mutilate while performing sub-phrase checks
+    Phrase temp_phrase(*candidate);
+    bool shorter_found = false;
+    
+    // Check for shorter and shorter versions of the temporary phrase
+    while (temp_phrase.length >= minimum_length && !shorter_found) {
+      temp_phrase.ensureSuffixFound();
+      if (getSuffixCheck(temp_phrase.firstSuffixIndex)==0)
+	temp_phrase.shortenByOneAtPrefix();
+      else
+	shorter_found = true;
+
+      // Possible efficiency here: we can finish if the prefix of c
+      // and temp_phrase are the same for candidate->length symbols.
+    }
+      
+    // If no shorter phrase is found, use this one
+    if (!shorter_found) {
+      results.push_back(*candidate);
+      candidate->ensureSuffixFound();
+      setSuffixCheck(candidate->firstSuffixIndex, candidate->lastSuffixIndex);
+    }
+  }
+}
+
+
+// suffixCompare
+//
+// Compare two pointers into a suffix array.  We use this in the 
+// qsort function, so the input are pointers to pointers.  
+//
+// Return -1 if (a < b), otherwise (a > b) so return +1, 
+
+int suffixCompare(const void *cpa, const void *cpb) {
+
+  // Cast then dereference pointers to suffix array elements
+  symbol *pa = (symbol *) cpa;
+  symbol *pb = (symbol *) cpb;
+  pa = (symbol *) *pa;
+  pb = (symbol *) *pb;
+
+  // If the two elements are the same, examine the next one
+  while (*pa == *pb) {
+    *pa++;
+    *pb++;
+  }
+
+  // Make the copmparison and return
+  if ( *pa < *pb) {
+    return -1;
+  } else {
+    return +1;
+  }
+}
+
+
+// prefixCompare
+//
+// Compare two pointers into a prefix array.  We use this in the 
+// qsort function, so the input are pointers to pointers.  
+//
+// Return -1 if (a > b), otherwise (a < b) so return +1, 
+
+int prefixCompare(const void *cpa, const void *cpb) {
+
+  // Cast then dereference pointers to prefix array elements
+  symbol *pa = (symbol *) cpa;
+  symbol *pb = (symbol *) cpb;
+  pa = (symbol *) *pa;
+  pb = (symbol *) *pb;
+
+  // If the two elements are the same, examine the next one
+  while (*pa == *pb) {
+    *pa--;
+    *pb--;
+  }
+
+  // Make the copmparison and return
+  if ( *pa > *pb) {
+    return -1;
+  } else {
+    return +1;
+  }
+}
+
+// simpleCompare
+//
+// Compare two pointers based on the memory location they point to.
+
+int pointerCompare( const void *pa, const void *pb ) {
+
+  symbol **a = (symbol **) pa;
+  symbol **b = (symbol **) pb;
+
+  if (*a < *b) {
+    return  -1;
+  } else if (*a > *b) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+
+// Read the clauses.numbers file into the "symbols" array.
+//
+// Each number in the file is a symbol number; it is essential that 
+// the first symbol (and no others) be COLLECTIONSTART and the last
+// symbol (and no others) be COLLECTIONEND.
+//
+// Return the number of numbers in the array.
+
+int readNumbers() {
+
+  char filename[FILENAME_MAX];
+  sprintf(filename, "%s/clauses.numbers", collection);
+  if (verbosity) {
+    cout << "Reading numbers file: " << filename << endl;
+  }
+
+  // Open the numbers file
+  ifstream inFile1(filename, ios::in);
+  if (!inFile1) {
+    cerr << "File " << filename << " could not be opened\n";
+    exit(1);
+  }
+
+  // Count the number of symbols
+  inputLength = 0;
+  symbol word;
+  while (inFile1 >> word) {
+    ++inputLength;
+  }
+  inFile1.close();
+
+  // Allocate the symbbols array
+  if (verbosity > 1) {
+    cout << "Allocating symbol arrays for " << inputLength << " symbols" << endl;
+  }
+  symbols = new symbol[inputLength];
+  if (symbols == NULL) {
+    cerr << "Suffix error: not enough memory to hold " << inputLength
+	 << " symbols." << endl;
+    exit(2);
+  }
+
+  // Read the numbers file into the numbers array
+  if (verbosity > 2) {
+    cout << "Reading the numbers" << endl;
+  }
+
+  ifstream inFile2(filename, ios::in);
+  if (!inFile2) {
+    cerr << "File " << filename << " could not be opened\n";
+    exit(1);
+  }
+  cellcount next = 0;
+  numberOfDocuments = 0;
+  while (inFile2 >> word) {
+    symbols[next++] = word;
+    if (word == DOCUMENTSTART) {
+      ++numberOfDocuments;
+    }
+  }
+  inFile2.close();
+  
+  // Make sure the numbers file is intact
+  assert(symbols[0] == COLLECTIONSTART);
+  assert(symbols[next-1] == COLLECTIONEND);
+
+  return inputLength;
+}
+
+
+
+// Get Document Occurrance statistics
+//
+// Given a phrase, what documents does it occur in?
+
+cellcount getDocumentOccurrances(const Phrase &p, cellcount *frequency) {
+
+  // cout << "searching for \""<< p << "\" in documents " 
+  //      << 0 << "-" << numberOfDocuments - 1 << endl;
+
+  // The number of documents in which this phrase occurs
+  cellcount df = 0;
+
+  // Initialise the document frequency array
+  //  for (cellindex i = 0; i < numberOfDocuments; ++i) {
+  // frequency[i] = 0;
+  //}
+  memset(frequency, 0, sizeof(cellcount)*numberOfDocuments);
+
+  // variables used to facilitate the search
+  cellindex begin;
+  cellindex end;
+  cellindex d;
+  symbol *target;
+  bool found;
+
+  // search for the document in which each occurence of the phrase is found
+  for (cellcount j = p.firstSuffixIndex; j <= p.lastSuffixIndex; ++j) {
+    
+    // cout << "looking for phrase at suffixArray[" << j << "]\n";
+    
+    target = suffixArray[j];
+    begin = 0;
+    end = numberOfDocuments - 1;
+    found = false;
+
+    // Search for the occurence of a document delimiter that target
+    // occurs immediately after.  
+    // We do this by performing a binary chop search on documentArray.
+    while (!found) {
+
+      // cout << "searching for " << (cellindex) target << " in "
+      //      << begin << " - " << end << endl;
+
+      assert (begin <= end);
+
+      // If the beginning and end of the interval are the same,
+      // then we've found the correct document
+      if (begin == end) {
+	if (frequency[begin] == 0) {
+	  ++df;
+	}
+	++frequency[begin];
+	found = true;
+      }
+
+      // Otherwise, examine a new document midway through the begin-end
+      // interval and see if it is the one.
+      else {
+	d = (begin + end) / 2;
+	if (target > documentArray[d]) {
+	  // If target addrss is greater than this, but thisi sthe last document,
+	  // then this must be the one we want.  Or, if target is greater than
+	  // this one but less then the next, this must be the one we wnat.
+	  if ((d == numberOfDocuments - 1) || (target < documentArray[d+1])) {
+	    if (frequency[d] == 0) {
+	      ++df;
+	    }
+	    ++frequency[d];
+	    found = true;
+	  } else {	
+	    // otherwise we know to search later in the document set
+	    begin = d + 1;
+	  }
+	} else {
+	  // search earlier in the document set
+	  end = d - 1;
+	}
+      }
+    }
+  }
+  return df;
+}
+
+
+
+
+
+
+// phraseExpansionMemory : Which phrases have we expanded?
+//
+// A set of utilities for keeping track of which phrases we have expanded.
+// We don't want to expand a phrase more than once, after all.
+//
+// This REALLY ought to be in its own class, but it works so that's okay.
+//
+// Phrases are identified by their firstSuffixPosition and length.
+//
+// Functions provided are:
+//       void initialisePhraseMemory()
+//       void rememberThisPhrase(index, length)
+//       bool isPhraseStored(index, length)
+//       void deletePhraseMemory()
+//
+// Internally, we will have two separate cases:
+//
+// Phrases of length 1-8: 
+//       unsigned char phraseMemory[inputLength]
+// is an array where each cell "remembers" the corresponding index in the 
+// suffixArray, and each of the 8 bits of the cell correspond to the phrases 
+// of length 1, 2... 8.  
+// Eventually, we will make this disk-based (i.e. store the array in a file).
+//
+// Phrases of length 9+:
+//       file hashTableFile
+//       file listOfEntries
+// The first file is a hash table; each phrase maps to one of its cells, which
+// contains either 0 (empty, no occurence) or a number which is an entry number 
+// in the second file.  This file contains a "list" of entries.  Each consists of 
+// three numbers: the suffixArray index of the phrase, the length of the phrase,
+// and the entry number of the next phrase with the same hash.
+//
+
+
+unsigned char *phraseMemory;
+
+void initialiseLongPhraseMemory();
+void rememberThisLongPhrase(cellindex index, cellcount length);
+bool isLongPhraseStored(cellindex index, cellcount length);
+void deleteLongPhraseMemory();
+
+
+void initialisePhraseMemory() {
+
+  phraseMemory = new unsigned char[inputLength];
+
+  // to begin with, everything is empty
+  //  for (cellcount i = 0; i < inputLength; ++i) {
+  // phraseMemory[i] = 0;
+  //}
+  memset(phraseMemory, 0, sizeof(char)*inputLength);
+  
+  // intialise the hashTable of long phrases
+  initialiseLongPhraseMemory();
+
+}
+
+void rememberThisPhrase(cellindex index, cellcount length) {
+
+  // if the phrase is very long, use the file-based system
+  if (length > 8) {
+    rememberThisLongPhrase(index, length);
+    return;
+  }
+
+  // create a char with just the bit corresponding to length set
+  unsigned char newbit = 1;
+  for (cellcount i = 1; i < length; ++i) {
+    newbit <<= 1;
+  }
+
+  // set that bit in the memory array at position index
+  phraseMemory[index] |= newbit;
+}
+
+
+bool isPhraseStored(cellindex index, cellcount length) {
+
+  // if the phrase is very long, use the file-based system
+  if (length > 8) {
+    return isLongPhraseStored(index, length);
+  }
+
+  // create a char with just the bit corresponding to length set
+  unsigned char newbit = 1;
+  for (cellcount i = 1; i < length; ++i) {
+    newbit <<= 1;
+  }
+
+  // retrurn true if that bit is set in memory arrayat position index
+  return (phraseMemory[index] & newbit);
+}
+
+void deletePhraseMemory() {
+  delete []phraseMemory;
+  deleteLongPhraseMemory();
+}
+
+
+
+// Files etc used to store "long" equavlents of the above
+
+fstream hashTableFile;
+char    hashTableFileName[FILENAME_MAX];
+fstream listOfEntries;
+char    listOfEntriesName[FILENAME_MAX];
+cellindex nextEntryNumber;
+
+const cellcount bigPrime = 7919;
+
+
+void initialiseLongPhraseMemory() {
+
+  cellindex example = 0;
+
+  sprintf(hashTableFileName, "%s/hashTable", collection);
+  sprintf(listOfEntriesName, "%s/hashLists", collection);
+
+
+  // create the new hashtable
+  if (verbosity > 1) {
+    cout << "Initialising hashTable: " << hashTableFileName << endl;
+  }
+  hashTableFile.open(hashTableFileName, ios::in | ios::out);
+  for (cellcount i = 0; i < bigPrime; ++i) {
+    hashTableFile.write((char *) &example, sizeof(example));
+  }
+
+  // create the list of phrases
+  if (verbosity > 1) {
+    cout << "Initialising list of hashtable entries: " << listOfEntriesName << endl;
+  }
+  listOfEntries.open(listOfEntriesName, ios::in | ios::out);
+  listOfEntries.write((char *) &example, sizeof(example));
+  listOfEntries.write((char *) &example, sizeof(example));
+  listOfEntries.write((char *) &example, sizeof(example));
+  nextEntryNumber = 1;
+}
+
+
+void rememberThisLongPhrase(cellindex index, cellcount length) {
+
+  // cout << "rememberThisLongPhrase(" << index << ", " << length << ")\n";
+
+  cellindex hashOffset = ((index + length) % bigPrime) * sizeof(cellindex);
+  cellindex pointer;
+  cellindex zero = 0;
+  cellindex readp = 0;
+  cellindex readi = 0;
+  cellindex readl = 0;
+
+  hashTableFile.seekg(hashOffset);
+  hashTableFile.read((char *) &pointer, sizeof(cellindex));
+
+  if (pointer == 0) {
+    // There is no entry at all in the hash table for this entry
+    // so create one
+
+    pointer = nextEntryNumber++;
+    hashTableFile.seekg(hashOffset);
+    hashTableFile.write((char *) &pointer, sizeof(cellindex));
+        
+    listOfEntries.seekp(pointer * sizeof(cellindex) * 3);
+    listOfEntries.write((char *) &zero, sizeof(cellindex));
+    listOfEntries.write((char *) &index, sizeof(cellindex));
+    listOfEntries.write((char *) &length, sizeof(cellindex));
+
+  } else {
+    // There is a list starting at this hash value, so the phrase may
+    // be already remembered, or it might need to be appended
+    
+    while (pointer != 0) {
+      // Read the entry pointed to by pointer
+      listOfEntries.seekg(pointer * sizeof(cellindex) * 3);
+      listOfEntries.read((char *) &readp, sizeof(cellindex));
+      listOfEntries.read((char *) &readi, sizeof(cellindex));
+      listOfEntries.read((char *) &readl, sizeof(cellindex));
+
+      // cout << "read " << pointer << ", " << readp << ", " << readi << ", " << readl << endl;
+
+      if ((readi == index) && (readl = length)) {
+	// we've found that we've already stored it
+	return;
+      } else if (readp == 0) {
+	// we're reached the end of the list.  Add a new entry.
+	listOfEntries.seekp(pointer * sizeof(cellindex) * 3);
+	listOfEntries.write((char *) &nextEntryNumber, sizeof(cellindex));
+	pointer = nextEntryNumber++;
+
+	listOfEntries.seekp(pointer * sizeof(cellindex) * 3);
+	listOfEntries.write((char *) &zero, sizeof(cellindex));
+	listOfEntries.write((char *) &index, sizeof(cellindex));
+	listOfEntries.write((char *) &length, sizeof(cellindex));
+	return;
+      } else {
+	// go on to the next node
+	pointer = readp;
+      }
+    }
+  }
+
+
+}
+
+
+bool isLongPhraseStored(cellindex index, cellcount length) {
+
+  // cout << "isLongPhraseExpanded(" << index << ", " << length << ")\n";
+
+  cellindex hashOffset = ((index + length) % bigPrime) * sizeof(cellindex);
+  cellindex pointer;
+  cellindex readp = 0;
+  cellindex readi = 0;
+  cellindex readl = 0;
+
+  // Find the phrase in the hashFile
+  hashTableFile.seekg(hashOffset);
+  hashTableFile.read((char *) &pointer, sizeof(cellindex));
+
+  if (pointer == 0) {
+    // There is no entry at all in the hash table for this entry
+    // so nothing is stored
+    return false;
+
+  } else {
+    // There is a list starting at this hash value, so the phrase may
+    // be already remembered in that list
+    while (pointer != 0) {
+      // Read the entry pointed to by pointer
+      listOfEntries.seekg(pointer * sizeof(cellindex) * 3);
+      listOfEntries.read((char *) &readp, sizeof(cellindex));
+      listOfEntries.read((char *) &readi, sizeof(cellindex));
+      listOfEntries.read((char *) &readl, sizeof(cellindex));
+
+      if ((readi == index) && (readl = length)) {
+	// we've found the phrase stored here
+	return true;
+      } else {
+	// go on to the next node
+	pointer = readp;
+      }
+    }
+  }
+  return false;
+}
+
+
+void deleteLongPhraseMemory() {
+  // remove the hash & other files
+
+  hashTableFile.close();
+  listOfEntries.close();
+  remove(hashTableFileName);
+  remove(listOfEntriesName);
+
+}
+
+
+// Read the collection statistics file
+//
+void readStatistics() {
+
+  // open the statistics file
+  char filename[FILENAME_MAX];
+  sprintf(filename, "%s/clauses.stats", collection);
+
+  // Open the file
+  ifstream inFile(filename, ios::in);
+  if (!inFile) {
+    cerr << "File " << filename << " could not be opened\n";
+    exit(1);
+  }
+
+  // Read the numbers file into the numbers array
+  char key[1000];
+  symbol value;
+  while (inFile >> key >> value){
+    if (strcmp(key, "first_stopword") == 0) {
+      firstStopSymbol = value;
+    } else if (strcmp(key, "last_stopword") == 0) {
+      lastStopSymbol = value;
+    } else if (strcmp(key, "first_contentword") == 0) {
+      firstContentSymbol = value;
+    } else if (strcmp(key, "last_contentword") == 0) {
+      lastContentSymbol = value;
+    }
+  }
+  inFile.close();
+
+  // Make sure we have the information we need
+  if (!(firstStopSymbol && lastStopSymbol && firstContentSymbol && lastContentSymbol)) {
+    cerr << "Statistics file incomplete" << endl;
+    exit(1);
+  }
+}
+
+cellcount getContentCount(symbol firstContent) {
+
+  cellcount content=0;
+  for (cellcount i=0; i<inputLength; ++i) {
+    if (symbols[i]>=firstContent) ++content;
+  }
+
+  return content;
+}
+
+
+int main (int argc, char * argv[]) {
+
+  // Command-line arguments
+  // argv[1] is the phindex directory
+  // argv[2] is the mode, where 1 is stopword mode 
+  // argv[3] is the min_occurs, - minimum occurrence frequency for a phrase to be included in the hierarchy
+  // argv[4] is opitonal verbosity
+  if (argc < 4) {
+    cerr << "Usage: " << argv[0] << " phind-directory mode min-phrase-freq [verbosity]" << endl;
+    exit(1);
+  }
+
+  // collection directory
+  strcpy(collection, argv[1]);
+
+  // mode parameter
+  phraseMode = atoi(argv[2]);
+  assert((phraseMode == STOPWORDS) || (phraseMode == ANYPHRASE));
+
+  minOccurs = atoi(argv[3]);
+  assert((minOccurs > 0));
+
+  // optional verbosity parameter
+  if (argc == 5) {
+    verbosity = atoi(argv[4]);
+    assert (verbosity >= 0);
+  }
+
+  if (verbosity) {
+    cout << "suffix: the phrase extraction program" << endl;
+  }
+  if (verbosity > 1) {
+    if (phraseMode == STOPWORDS) {
+      cout << "Stopwords mode: no phrase may begin or end with a stopword" << endl;
+    } else {
+      cout << "AllPhrase mode: extract every phrase that occurs more than once" << endl;
+    }
+  }
+
+  // Read the statistics file 
+  readStatistics();
+  // Read the numbers file
+  readNumbers();
+  
+  if (numberOfDocuments == 0) {
+    cerr << "There are no documents in this collection!" << endl;
+    exit(0); // not necessarily an error...
+  }
+
+  symbol firstContent;
+  if (phraseMode==STOPWORDS) firstContent=firstContentSymbol;
+  else firstContent = firstStopSymbol;
+
+  // Allocate memory for the suffix & prefix arrays
+  cellcount contentLength = 0;
+  contentLength = getContentCount(firstContent);
+  suffixArray = new symbol *[contentLength];
+  prefixArray = new symbol *[contentLength];
+  if (prefixArray == NULL) {
+    cerr << "Suffix: not enough memory to hold " << inputLength << " symbols." << endl;
+    exit(2);
+  }
+  allocateSuffixCheck(contentLength);
+  // Initialise prefix and suffix arrays, only use the needed suffixes
+  for (cellcount j = 0, here = 0; j < inputLength; ++j) {
+    if (symbols[j]>=firstContent) {
+      suffixArray[here] = &symbols[j];
+      prefixArray[here] = &symbols[j];
+      ++here;
+    }    
+  }
+  qsort(suffixArray, contentLength, sizeof(symbol *), suffixCompare);
+  qsort(prefixArray, contentLength, sizeof(symbol *), prefixCompare);
+  // Create the document arrays
+  if (verbosity > 1) {
+    cout << "Allocating document arrays for " << numberOfDocuments << " documents" << endl;
+  }
+
+  // The document frequecy array is used to count the number of times
+  // each phrase occurs in each document.  The number of documents in
+  // which a phrase occurs is stored in df.
+  frequency *documentFrequency = new frequency[numberOfDocuments];
+  frequency df;
+
+  // documentArray will be searched in order to discover which document
+  // each phrase occurs in.
+  documentArray = new symbol *[numberOfDocuments];  
+  if (documentArray == NULL) {
+    cerr << "Suffix: out of memory allocating document arrays." << endl;
+    exit(2);
+  }  
+
+  // just scan through the input text to find the doc starts
+  cellindex d = 0;
+  for (cellindex i=0; i<inputLength; ++i) {
+    if (symbols[i] == DOCUMENTSTART) {
+      documentArray[d] = &symbols[i];
+      ++d;
+    }
+  }
+
+  // the phrases stuff is expecting inputLength to be the length of the
+  // suffix array, so change it.
+  inputLength = contentLength;
+
+  // Extract phrases
+  //
+  // We will make several passesover the data, in each case considering
+  // a set of input phrases and generating a set of output phrases, which
+  // we will expancd in later passes.
+  //
+  // The input phrases in the first pass will be the vocabulary.
+  // In later passes, the input phrases will be the output phrases of the
+  // previous pass.
+  //
+  // In each pass we will consider each input phrase in turn.  If we
+  // have seen it before, we will ignore it.  Otherwise, we will expand
+  // it and add its expansions to the set of output phrases.
+
+  cout <<"\ngenerating the phrase hierarchy\n\n";
+  
+  // Store the phrase data in the phrases file
+  char phraseDataName[FILENAME_MAX];
+  sprintf(phraseDataName, "%s/phrases", collection);
+  ofstream phraseData(phraseDataName, ios::out);
+  if (!phraseData) {
+    cout << "File " << phraseDataName << " could not be opened\n";
+    exit(1);
+  }
+
+  // Count the number of phrases output
+  unsigned long int phraseCounter = 0;
+
+  // Set up the phrase expansion memory.
+  // We need this so that we don't expand a phrase more than once
+  initialisePhraseMemory();
+
+  // The current pass numebr
+  int phrasePass = 1;
+
+
+  // PASS NUMBER 1
+  if (verbosity > 1) {
+    cout << "Starting pass " << phrasePass << endl;
+  }
+
+  ofstream outPhrase;
+  char     outPhraseName[FILENAME_MAX];
+  unsigned long int outPhraseCounter = 0;
+
+  // On the first pass, simply work through the vocabulary
+  sprintf(outPhraseName, "%s/outPhrase.1", collection);
+  outPhrase.open(outPhraseName, ios::out);
+  if (!outPhrase) {
+    cerr << "File " << outPhraseName << " could not be opened\n";
+    exit(1);
+  }
+
+  // Iterate over the different symbols by working through the suffix array
+  vector<Phrase> result;
+  cellindex ij = 0;
+  char *tmpString;
+
+  Phrase p;
+  while (ij < inputLength) {
+
+    // make a new phrase of length 1
+    p = Phrase(suffixArray[ij], 1, SUFFIX);
+    p.findFirstAndLastSuffix(ij, inputLength-1);
+
+    // We ignore this symbol if it occurs only once, if it is a delimiter,
+    // of if we are in stopwords mode and it is a stopword
+    // - in this new version, only need to check freq
+    // We could imagine a new mode/command-line option, which is like 
+    // STOPWORDS but without this restrictrion.  This would let you browse
+    // from "the" to "the AGRIS" for example, but not from "AGRIS" to
+    // "the AGRIS" (where the is a stopword and AGRIS a content word).
+    // The system used to work like this; it is easy to implement, but
+    // it explodes the size of the indexes.  So: would it be useful?  
+    if (p.suffixFrequency >= minOccurs) {
+      // Get minimal expansions of the phrase
+      getExpansions(p, result);
+      
+      if (!result.empty()) {
+	
+	// Remember that we have expanded this phrase
+	rememberThisPhrase(ij, 1);
+
+	// write the phrase text
+	phraseData << ij << "-1:" << p << ":" << p.suffixFrequency << ":"
+		   << result.size() << ":";
+
+	// write the results
+	for (cellcount k = 0; k < result.size(); ++k) {
+	  if (k) {
+	    phraseData << ",";
+	  }
+	  phraseData << result[k].firstSuffixIndex << "-" << result[k].length;
+	  outPhrase << result[k].firstSuffixIndex << " " << result[k].length << endl;
+	  ++outPhraseCounter;
+	}
+	result.clear();
+	
+	// Write the documents in which this phrase occurs
+	df = getDocumentOccurrances(p, documentFrequency);
+	phraseData << ":" << df << ":";
+
+	// write the documents
+	for (cellcount m = 0, first = 1; m < numberOfDocuments; ++m) {
+	  if (documentFrequency[m]) {
+	    if (first) {
+	      first = 0;
+	    } else {
+	      phraseData << ";";
+	    }
+	    // Output the document number.  Note that here we've numbered the 
+	    // N documents from 0 to N-1, but later they'll be 1-N.  Thus we
+	    // add 1 to the document id when we output it.
+	    phraseData << "d" << (m+1);
+	    // Next, output the frequency with which the document occurs, but
+	    // only if it is > 1.
+	    if (documentFrequency[m] > 1) {
+	      phraseData << "," << documentFrequency[m];
+	    }
+	  }
+	}
+
+	phraseData << endl;
+	++phraseCounter;
+
+	// feedback 
+	if (verbosity) {
+	  if (phraseCounter % 1000 == 0) {
+	    cout << "phrase " << phraseCounter << ": "
+		 << "cell " << p.firstSuffixIndex << " - " << p << endl;
+	  }
+	}
+      }
+    }
+   ij = p.lastSuffixIndex + 1;
+  }
+  outPhrase.close();
+
+  // REMAINING PASSES
+  // The previous outPhrase file forms the input to each new pass
+  cellcount start, length;
+  while (outPhraseCounter > 0) {
+
+    // Start a new pass
+    ++phrasePass;
+    if (verbosity) {
+      cout << "Starting pass " << phrasePass << endl;
+    }
+
+    // Open the input file
+    char inPhraseName[FILENAME_MAX];
+    sprintf(inPhraseName, "%s/outPhrase.%d", collection, phrasePass - 1);
+    ifstream inPhrase (inPhraseName, ios::in);
+    if (!inPhrase) {
+      cerr << "File " << inPhraseName << " could not be opened\n";
+      exit(1);
+    }
+
+    // Open the output file
+    sprintf(outPhraseName, "%s/outPhrase.%d", collection, phrasePass);
+    outPhrase.open(outPhraseName, ios::out);
+    if (!outPhrase) {
+      cerr << "File " << outPhraseName << " could not be opened\n";
+      exit(1);
+    }
+    outPhraseCounter = 0;
+
+    // Process each phrase
+    while(inPhrase >> start >> length) {
+
+      // Ignore the phrase if we have expanded it before
+      if (isPhraseStored(start, length))
+	continue;
+
+      // Remember that we have examined this phrase
+      rememberThisPhrase(start, length);
+
+      // Find the phrase in the suffixarray
+      p = Phrase(suffixArray[start], length, SUFFIX);
+      p.findFirstAndLastSuffix(start, inputLength-1);
+
+      // Ignore the phrase if it only occurs once
+      if (p.suffixFrequency < minOccurs)
+	continue;
+
+      // Write the phrase text;
+      phraseData << start << "-" << length << ":" << p << ":" << p.suffixFrequency << ":";
+	
+      // Expand the phrase, if it is fewer than 8 words long
+      if (length <= 8) {
+
+	// Get the minimal expansions for this phrase
+	getExpansions(p, result);
+      
+	// write the results
+	phraseData << result.size() << ":";
+
+	for (cellcount i = 0; i < result.size(); ++i) {
+	  if (i) {
+	    phraseData << ",";
+	  }
+	  phraseData << result[i].firstSuffixIndex << "-" << result[i].length;
+	  outPhrase << result[i].firstSuffixIndex << " " << result[i].length << endl;
+	  ++outPhraseCounter;
+	}
+	result.clear();
+	
+      } else {
+	// phrase is too long to expand further
+	phraseData << "0:";
+      }
+
+      // Write the documents in which this phrase occurs
+      df = getDocumentOccurrances(p, documentFrequency);
+      phraseData << ":" << df << ":";
+
+      // write the documents
+      for (cellcount i = 0, first = 1; i < numberOfDocuments; ++i) {
+	if (documentFrequency[i]) {
+	  if (first) {
+	    first = 0;
+	  } else {
+	    phraseData << ";";
+	  }
+	  // Output the document number.  Note that here we've numbered the 
+	  // N documents from 0 to N-1, but later they'll be 1-N.  Thus we
+	  // add 1 to the document id when we output it.
+	  phraseData << "d" << (i+1);
+	  // Next, output the frequency with which the document occurs, but
+	  // only if it is > 1.
+	  if (documentFrequency[i] > 1) {
+	    phraseData << "," << documentFrequency[i];
+	  }
+	}
+      }
+      
+      phraseData << endl;
+      ++phraseCounter;
+
+      // feedback 
+      if (verbosity) {
+	if (phraseCounter % 1000 == 0) {
+	  cout << "phrase " << phraseCounter << ": "<< "start " << start 
+	       << ", length " << length << " - " << p << endl;
+	}
+      }
+
+    }
+
+    inPhrase.close();
+    outPhrase.close();
+  }
+    
+  phraseData.close();
+  deletePhraseMemory();
+
+  delete [] documentFrequency;
+  delete [] symbols;
+  delete [] suffixArray;
+  delete [] prefixArray;
+  delete [] suffixCheck;
+  delete [] documentArray;
+
+
+  
+  cout << endl << "Done: " << phraseCounter << " phrases in " << phraseDataName << endl;
+  return 0;
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl/src/phind/generate/suffix.h
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/suffix.h	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/suffix.h	(revision 21112)
@@ -0,0 +1,89 @@
+/**********************************************************************
+ *
+ * suffix.h -- definitions used in suffix.cpp
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#ifndef SUFFIX_H
+#define SUFFIX_H
+
+// Types
+
+// The input words stored as an array of type symbol.
+// Required range: 0 - vocabulary size (typically hundreds of thousands)
+typedef unsigned int symbol;
+
+// The cellindex and cellcount types are used when we store indexes
+// to and ranges of cells in large arrays.
+// Required Range: 0 - length of input (typically millions)
+typedef unsigned int cellindex;
+typedef unsigned int cellcount;
+
+// The frequency type is used when we want to store the frequency with 
+// which a phrase (or some other thing) occurs
+// Required range: 0 - frequency of most common symbol (often "the")
+typedef unsigned int frequency;
+
+// Global variables 
+
+// The symbol array holds the input words
+extern cellcount symbol_array_length;
+extern symbol   *symbols;
+
+// The number of words read is storesd in inputLength.
+extern cellcount inputLength;
+
+// Suffix and prefix arrays are used to extract phrases
+extern symbol  **suffixArray;
+extern symbol  **prefixArray;
+
+// Collection-specific information about the first stopword/content symbols
+extern symbol firstStopSymbol;
+extern symbol lastStopSymbol;
+extern symbol firstContentSymbol;
+extern symbol lastContentSymbol;
+
+
+// Are we allowed to terminate a phrase on a stopword?
+extern int phraseMode;
+extern int minOccurs;
+
+#define ANYPHRASE 0
+#define STOPWORDS 1
+
+
+// Direction values
+#define SUFFIX 0
+#define PREFIX 1
+
+
+// Special symbol values
+#define COLLECTIONSTART 1
+#define COLLECTIONEND 2
+#define DOCUMENTSTART 3
+#define PHRASELIMIT 4
+
+#define LASTDELIMITER 4
+
+#endif
Index: /main/tags/3.01/gsdl/src/phind/generate/symbol-names.pl
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/symbol-names.pl	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/symbol-names.pl	(revision 21112)
@@ -0,0 +1,57 @@
+#! /usr/bin/perl -w
+
+###########################################################################
+#
+# symbol-names.pl -- Diagnostic tool that substitutes symbols names for 
+#                    symbol numbers in the output of the suffix program. 
+#
+# Copyright (C) 2000 Gordon Paynter
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+$collection = shift(@ARGV);;
+die "Usage: symbol-names.pl collection [file]\n" unless ($collection);
+
+$vocab = $ENV{'GSDLHOME'} . "/collect/$collection/index/phind1/clauses.vocab";
+if (-e "$vocab") {
+    print "Using vocabulary from: $vocab\n";
+} else {
+    $vocab = $ENV{'GSDLHOME'} . "/collect/$collection/building/phind1/clauses.vocab";
+    if (-e "$vocab") {
+	print "Using vocabulary from: $vocab\n";
+    } else {
+	die "Usage: symbol-names.pl collection [file]\n" unless (-e "$vocab");
+    }
+}
+
+open(V, "<$vocab");
+my $n = 1;
+
+while(<V>) {
+    chomp;
+    $symbol[$n++] = $_;
+}
+
+while(<>) {
+    s/s(\d+)/$symbol[$1]/g;
+    print;
+}
+
Index: /main/tags/3.01/gsdl/src/phind/generate/win32.mak
===================================================================
--- /main/tags/3.01/gsdl/src/phind/generate/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/src/phind/generate/win32.mak	(revision 21112)
@@ -0,0 +1,70 @@
+###########################################################################
+#
+# win32 makefile -- src/phind/generate
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..\..
+USE_STLPORT = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+AR = lib
+CC = cl
+CPPFLAGS = -GX
+LDFLAGS =
+DEFS = -D__WIN32__
+INCLUDES =
+!IF $(USE_STLPORT)
+DEFS = $(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+LIBS=
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .obj
+.cpp.obj:
+	$(COMPILE) $< 
+
+HEADERS = suffix.h phrase.h
+SOURCES = suffix.cpp phrase.cpp
+OBJECTS = phrase.obj suffix.obj
+EXEC    = suffix.exe
+
+all: $(EXEC)
+
+install:
+	copy $(EXEC) "$(GSDLHOME)\bin\windows"
+
+clean:
+	del *.obj
+	del $(EXEC)
+
+suffix.exe: $(OBJECTS)
+	    $(LINK) $(OBJECTS) $(LIBS) 
Index: /main/tags/3.01/gsdl/src/txt2db/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl/src/txt2db/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl/src/txt2db/Makefile.in	(revision 21112)
@@ -0,0 +1,97 @@
+###########################################################################
+#
+# Makefile -- Makefile for txt2db
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CC = @CC@
+CPP = @CXX@
+CCFLAGS = -Wall @CFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib @GDBM_INCLUDE@
+LDFLAGS = @LDFLAGS@ 
+DPLIBS = ../../lib/gsdllib.a ../../packages/mg/lib/unitool.o
+LIBS = ../../lib/gsdllib.a ../../packages/mg/lib/unitool.o @GDBM_LIBPATH@ -lgdbm @LIBS@
+GSDLOS = @gsdlos@
+INSTALL = @INSTALL@
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+CPPCOMPILE = $(CPP) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+MCOMPILE = ../../bin/linux/mico-c++ $(MCFLAGS) $(DEFS) $(MINCLUDES) -c
+LINK = $(CC) $(LDFLAGS) -o $@
+MLINK = ../../bin/linux/mico-ld $(LDFLAGS) -lmico2.2.7 -o $@
+CPPLINK = $(CPP) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CPPCOMPILE) $< 
+ 
+.SUFFIXES: .mpp .o
+.mpp.o:
+	$(MCOMPILE) $<
+
+HEADERS = 
+
+SOURCES = txt2db.cpp
+
+OBJECTS = txt2db.o
+
+EXEC    = txt2db
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	$(INSTALL) -d ../../bin/$(GSDLOS)
+	$(INSTALL) -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+distclean:
+	rm -f $(OBJECTS) $(EXEC) Makefile
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TXT2DB_OBJS = txt2db.o
+
+txt2db: $(TXT2DB_OBJS) $(DPLIBS)
+	$(CPPLINK) $(TXT2DB_OBJS) $(LIBS)
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl/src/txt2db/txt2db.cpp
===================================================================
--- /main/tags/3.01/gsdl/src/txt2db/txt2db.cpp	(revision 21112)
+++ /main/tags/3.01/gsdl/src/txt2db/txt2db.cpp	(revision 21112)
@@ -0,0 +1,171 @@
+/**********************************************************************
+ *
+ * txt2db.cpp -- 
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the 
+ * University of Waikato, New Zealand.
+ *
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **********************************************************************/
+
+
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include <stdlib.h>
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+void print_usage (char *program_name) {
+  cerr << "usage: " << program_name << " [options] database-name\n\n";
+  cerr << "options:\n";
+  cerr << " -append        append to existing database\n\n";
+}
+
+
+int main (int argc, char *argv[]) {
+  int block_size = 0;
+  GDBM_FILE dbf;
+  char c;
+  text_t key;
+  text_t value;
+  text_t tmp;
+  int num_dashes = 0;
+
+  // sanity check
+  if (argc != 2 && argc != 3) {
+    print_usage (argv[0]);
+    exit (0);
+  }
+  
+  char *dbname;
+  int append = 0;
+  if (argc == 3) {
+    if (strcmp (argv[1], "-append") == 0) {
+      append = 1;
+      dbname = argv[2];
+    } else {
+      cerr << argv[1] << " is not a valid option.\n\n";
+      print_usage (argv[0]);
+      exit (0);
+    }
+  } else dbname = argv[1];
+
+  
+  // open the database
+  // note that GDBM_FAST is obsolete on newer versions of gdbm
+  int read_write = GDBM_NEWDB | GDBM_FAST;
+  if (append) read_write = GDBM_WRCREAT | GDBM_FAST;
+  
+#ifdef __WIN32__
+  dbf = gdbm_open (dbname, block_size, read_write, 00664, NULL, 1);
+#else
+  dbf = gdbm_open  (dbname, block_size, read_write, 00664, NULL);
+#endif
+  if (dbf == NULL) {
+    cerr << "couldn't create " << dbname << "\n";
+    exit (0);
+  }
+  
+  cin.get(c);
+  while (!cin.eof()) {
+    num_dashes = 0;
+    key = "";
+    value = "";
+    
+    // look for [key]\n
+    while (!cin.eof() && c != '[') cin.get(c);
+    if (!cin.eof()) cin.get(c); // skip [
+    while (!cin.eof() && c != ']') {
+      key.push_back ((unsigned char)c);
+      cin.get(c);
+    }
+    if (!cin.eof()) cin.get(c); // skip ]
+    while (!cin.eof() && (c == '\n' || c == 'r')) cin.get(c);
+    
+    // look for 70 dashes
+    tmp = "";
+    while (!cin.eof() && (num_dashes < 70)) {
+      if (c == '\n' || c == '\r') {
+	tmp.push_back ((unsigned char)c);
+	num_dashes = 0;
+	
+      } else if (c == '-') {
+	tmp.push_back ((unsigned char)c);
+	++num_dashes;
+	
+      } else {
+	value += tmp;
+	value.push_back ((unsigned char)c);
+	tmp = "";
+	num_dashes = 0;
+      }
+      cin.get(c);
+    }
+    
+    // if the key is not an empty string store this key-value pair
+    if (!key.empty()) {
+      // convert key to a datum datatype
+      datum key_data;
+      key_data.dptr = key.getcstr();
+      if (key_data.dptr == NULL) {
+	cerr << "NULL key_data.dptr\n";
+	exit (0);
+      }
+      key_data.dsize = strlen(key_data.dptr);
+      
+      // convert value to a datum datatype
+      datum value_data;
+      value_data.dptr = value.getcstr();
+      if (value_data.dptr == NULL) {
+	cerr << "NULL value_data.dptr\n";
+	exit (0);
+      }
+      value_data.dsize = strlen(value_data.dptr);
+      
+      // store the value
+      if (gdbm_store (dbf, key_data, value_data, GDBM_REPLACE) < 0) {
+	cerr << "gdbm_store returned an error\n";
+	exit (0);
+      }
+      
+      delete []key_data.dptr;
+      delete []value_data.dptr;
+    }
+  }
+  
+  gdbm_close (dbf);
+  
+  return 0;
+}
Index: /main/tags/3.01/gsdl/src/txt2db/win32.mak
===================================================================
--- /main/tags/3.01/gsdl/src/txt2db/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl/src/txt2db/win32.mak	(revision 21112)
@@ -0,0 +1,78 @@
+###########################################################################
+#
+# win32 makefile -- txt2db
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDLHOME = ..\..
+USE_STLPORT = 0
+STLPATH = "$(GSDLHOME)\packages\windows\stlport\stlport"
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+LDFLAGS =
+
+DEFS = -D__WIN32__ -DHAVE_CONFIG_H
+INCLUDES = -I"$(GSDLHOME)\src\txt2db" -I"$(GSDLHOME)\lib" -I"$(GSDLHOME)\packages\mg\lib" \
+	   -I"$(GSDLHOME)\packages\windows\gdbm\gdbm"
+!IF $(USE_STLPORT)
+DEFS=$(DEFS) -DGSDL_NAMESPACE_BROKEN -DGSDL_USE_IOS_H -D__STL_NO_NEW_IOSTREAMS
+INCLUDES = $(INCLUDES) -I$(STLPATH)
+!ENDIF
+
+LIBS = "$(GSDLHOME)\packages\mg\lib\unitool.obj" "$(GSDLHOME)\lib\gsdllib.lib" \
+       "$(GSDLHOME)\packages\windows\gdbm\gdbm\gdbm.lib"
+
+COMPILE = $(CC) -c -GX $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .cpp .obj
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+e = .exe
+ 
+HEADERS = 
+
+SOURCES = txt2db.cpp
+
+OBJECTS = txt2db$o
+
+EXEC = txt2db$e
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	if NOT EXIST "$(GSDLHOME)\bin\windows" mkdir "$(GSDLHOME)\bin\windows"
+	copy $(EXEC) "$(GSDLHOME)\bin\windows"
+
+clean:
+	del *.obj
+	del $(EXEC)
+
+txt2db$e: $(OBJECTS) $(LIBS)
+	  $(LINK) $(OBJECTS) $(LIBS)
Index: /main/tags/3.01/gsdl3/LICENSE.txt
===================================================================
--- /main/tags/3.01/gsdl3/LICENSE.txt	(revision 21112)
+++ /main/tags/3.01/gsdl3/LICENSE.txt	(revision 21112)
@@ -0,0 +1,350 @@
+Copyright (c) 2003 New Zealand Digital Library Project, University of Waikato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, and distribute copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the GPL and the following conditions (where not in conflict with the GPL):
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software, as will all other notices contained in the copyrights directory.
+
+THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDER (Greenstone Digital Libraries, University of Waikato), OR THEIR STAFF BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of New Zealand Digital Library Project, University of Waikato shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization from the New Zealand Digital Library Project, University of Waikato. Distributed under GPL, as follows: 
+
+
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl3/README-CVS.txt
===================================================================
--- /main/tags/3.01/gsdl3/README-CVS.txt	(revision 21112)
+++ /main/tags/3.01/gsdl3/README-CVS.txt	(revision 21112)
@@ -0,0 +1,88 @@
+Greenstone 3 (GSDL3)
+Copyright (C) 2003 New Zealand Digital Libraries, University Of Waikato
+Greenstone3 comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+This is free software, and you are welcome to redistribute it
+
+These are some extra notes for installing Greenstone from CVS. Please also read the README.txt file for general information as almost all of that applies here too.
+
+Installing Greenstone from a CVS checkout:
+---------------------------------------------------------------------
+
+Make sure the CVS executable is on your PATH.
+
+Checkout the code:
+
+cvs -d :pserver:cvs_anon@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src co greenstone3
+
+Build and install:
+
+In the greenstone3 directory, edit the build.properties file (see 'Configuring your installation' in README.txt), and run 'ant prepare install'. The two targets can be run separately if you like.
+
+The 'prepare' target will download additional code (using CVS and http), so you need to be online to run it. The 'install' target can be run offline.
+
+The prepare/install targets will ask you if you accept the properties before starting.
+To suppress this prompt, use the -Dproperties.accepted=yes flag. E.g.
+ant -Dproperties.accepted=yes prepare install (from CVS), or
+To log the output, run
+ant -Dproperties.accepted=yes -logfile build.log prepare install (from CVS), or
+
+Extra Configuration notes:
+--------------------------------------------------------------
+
+Greenstone 3 uses some parts of Greenstone 2 for collection building using the Librarian Interface. These will be installed during the Greenstone 3 installation process. If you have Greenstone 2 already installed and want to use that version instead (or don't want to use it at all), please set the gsdl2.installed.path property (in build.properties) to be the root of your Greenstone 2 installation, or any non-empty string. Greenstone 3 will then use this installation rather than its own version. (Note, be careful about compatibility between versions.) 
+
+
+Common install/update targets:
+---------------------------------------------------------------------
+1. Install for the first time from CVS:
+cvs co greenstone3
+cd greenstone3
+ant prepare install
+
+2. Install for the first time from CVS, mostly offline:
+[online]
+cvs co greenstone3
+cd greenstone3
+ant prepare
+[offline]
+ant install
+
+3. Install for the first time, with CVS done outside of Ant (if you have problems running CVS commands from Ant):
+cvs co -P greenstone3
+cd greenstone3/src/packages
+cvs co mgpp
+cd ../..
+[if you want greenstone 2 building:
+cvs co -P gli
+cvs co -P gs2build
+]
+[if you are on windows
+cvs co winbin
+]
+ant -Dnocvs.mode=yes prepare install (prepare needs to be online, install can be done offline)
+
+4. Updating your Greenstone installation from CVS (and reconfigure/recompile):
+cd greenstone3
+ant update
+
+5. Updating your Greenstone installation from CVS, offline:
+cd greenstone3
+[online]
+ant cvsupdate
+[offline]
+ant -Dnocvs.mode=yes update
+
+6. Updating your Greenstone installation, with CVS done outside of Ant:
+cd greenstone3
+cvs update -l
+cvs update -dP extensions bin docs lib resources src winutil packages 
+cd web (or the path-to-tomcat/webapps/greenstone3 if have installed greenstone as a webapp in Tomcat, see Using External Tomcat section)
+cvs update -dP 
+cd greenstone3
+[ if you have greenstone 2 building:
+cvs update -dP gli
+cvs checkout -P gs2build
+Note that the gs2build one uses checkout not update.
+]
+ant -Dnocvs.mode=yes update
+
Index: /main/tags/3.01/gsdl3/README.txt
===================================================================
--- /main/tags/3.01/gsdl3/README.txt	(revision 21112)
+++ /main/tags/3.01/gsdl3/README.txt	(revision 21112)
@@ -0,0 +1,245 @@
+Greenstone 3 (GSDL3)
+Copyright (C) 2003 New Zealand Digital Libraries, University Of Waikato
+Greenstone3 comes with ABSOLUTELY NO WARRANTY; for details see LICENSE.txt
+This is free software, and you are welcome to redistribute it
+
+You will need Java and Ant to run Greenstone 3. 
+
+Your Java version should be 1.4 or higher. We recommend Sun Java. You need the SDK (development environment). Set the environment variable JAVA_HOME to be the root of your Java installation.
+
+Ant (Apache's Java based build tool) can be downloaded from http://ant.apache.org/bindownload.cgi. Set the environment variable ANT_HOME to be the root of your Ant installation, and make sure the Ant executables are on your PATH. You may have problems with earlier versions. This has been tested with version 1.6.2
+
+In the greenstone3 directory, you can run 'ant' which will give you a help message.
+Running 'ant -projecthelp' gives a list of the targets that you can run - these
+do various things like compile the source code, start up the server etc.
+
+On GNU/Linux MySQL seems to have trouble with spaces in paths, so it's best to install Greenstone into a path with no spaces.
+
+Installing Greenstone
+---------------------------------------------------------
+
+Download the appropriate zip/tar file (greenstone-3.xx-linux.tar.gz/greenstone-3.xx-win32.zip/greenstone-3.xx-macOSX.tar.gz) from sourceforge.net/projects/greenstone3, and unzip/untar it. In the greenstone3 directory, edit the build.properties file (see 'Configuring your installation' below), and run 'ant install'.
+
+See 'Using External Tomcat' and 'Using External MySQL' sections below if you want to use an existing version.
+
+Configuring your installation:
+--------------------------------------------------------
+
+The file build.properties contains various parameters that can be set by the user. Please check these settings before running the install.
+
+Greenstone 3 comes with MySQL and Tomcat bundled in.  
+
+If you already have Tomcat running, you can set the 'tomcat.installed.path' property (in build.properties) to the base directory of your Tomcat installation, and Greenstone will not use its own Tomcat. (You can delete the packages/tomcat directory if you like.) Please read the section "Using External Tomcat" for details about how to configure Tomcat for Greenstone.
+
+Greenstone 3 uses MySQL for the collection database (with Greenstone 3 native building). If you already have MySQL installed, set the 'mysql.installed.path' property (in build.properties) to the base directory of your MySQL installation, and Greenstone will not use its local MySQL. (You can delete the packages/mysql directory if you like.) Please read the section "Using external MySQL server" for details about how to configure MySQL for Greenstone.
+
+If you are using MySQL that comes with greenstone 3, and you are installing greenstone 3 on an NFS drive, MySQL may not work properly (it has problems removing locks on its data files). After installation, copy the data directory from greenstone3/packages/mysql to a local drive, then set the path to this directory in the mysql.datadir property in build.properties.
+If the version of MySQL that greenstone is using is not suitable for your setup (see notes on versions below), please install an appropriate version and set the mysql.installed.path property.
+
+Mac OS X: You need to have GDBM installed (http://www.gnu.org/software/gdbm/gdbm.html). Please set the gdbm.installed.path property (in build.properties) to the root of your gdbm installation if it is not installed in a default place. If you run GLI or GS2 collection building from the command line, you will need to set the DYLD_LIBRARY_PATH environment variable to include <path-to-gdbm>/lib.
+
+The install target will ask you if you accept the properties before starting.
+To suppress this prompt, use the -Dproperties.accepted=yes flag. E.g.
+ant -Dproperties.accepted=yes install 
+
+To log the output in build.log, run
+ant -Dproperties.accepted=yes -logfile build.log  install 
+
+Running Greenstone:
+---------------------------------------------------------
+
+To startup the local servers (Tomcat adn/or MySQL), run 'ant start'. 
+'ant restart' and 'ant stop' restarts and shuts down the servers. 
+To start or stop just Tomcat or MySQL, use the start-tomcat, start-mysql, stop-tomcat, stop-mysql targets.
+These will only start/stop local servers (ones installed by Greenstone). You will need to manually start/stop external Tomcat/MySQL. if you are using both external Tomcat and external MySQL, then the start, restart, stop targets do nothing.
+
+Once the Tomcat server is running, Greenstone will be available in a browser at "http://localhost:8080/greenstone3".
+You can change the port number by changing the 'tomcat.port' property in build.properties, then running 'ant configure'.
+
+MySQL is only needed if you want to use the GS3MGDemo collection, or if you want to build collections using Greenstone3 native collection building.
+
+
+Building Collections:
+----------------------------------------------------------
+
+You can build collections using either Greenstone 2 style building, or native Greenstone 3 style.
+
+Greenstone 2 style:
+
+You need to have Perl installed and on your PATH. Perl is included as part of the Windows binary distribution.
+run  'ant gli", or cd to greenstone3/gli and run gli4gs3.sh/bat. This is a graphical tool for building collections. Once you have created and built a collection, you can see it by clicking 'Preview collection' on the Build panel.
+
+Note that GLI doesn't start up the Greenstone server (Tomcat/MySQL) so you should run 'ant start' before previewing your collection.
+
+Greenstone 3 style:
+
+In the greenstone3 directory, run: 'source gs3-setup.sh' (GNU/Linux, Mac OS X) or 'gs3-setup' (Windows).
+To create a new collection, run 'gs3-mkcol.sh/bat <sitename> <collname>'
+Put documents in the import directory (greenstone3/web/sites/<sitename>/collect/<collname>/import), edit the collection configuration file (greenstone3/web/sites/<sitename>/collect/<collname>/etc/collectionConfig.xml), and run 'gs3-build.sh <sitename> <collname>'
+Rename the building directory to index (in greenstone3/web/sites/<sitename>/collect/<collname>) and reload the collection in Tomcat (?a=s&sa=a&st=collection&sn=<collname>), or restart Tomcat.
+
+See the manual for more details about both styles of collection building.
+
+Recompiling
+-------------------------------------------------------
+To recompile your Greenstone3 installation, in the top level greenstone3 directory, run:
+
+ant configure
+ant configure-c++
+ant clean
+ant compile
+
+The compile target, under GNU/Linux and Mac OS X, does Java and C/C++ compilation. For Windows, since Visual Studio is not a standard component, only Java compilation is carried out. Pre-compiled binaries are provided for the C/C++ components (src/packages and Greenstone 2 style building). If you have Visual Studio installed (version 6), you can run the compile-windows-c++ targets to compile the code locally. (Don't forget to setup the Visual Studio environment first, by running, e.g. C:/Program Files/Microsoft Visual Studio/VC98/Bin/VCVARS32.BAT or equivalent.)
+
+Any sub targets can be run by themselves. Run 'ant -projecthelp' for a list of public targets, otherwise you can look at the build.xml file to see which targets depend on other ones.
+
+If you run your install using an external Tomcat, the SOAP web service for localsite (used by the gateway servlet) will not be loaded. You need to start up Tomcat, then run 'ant soap-deploy-site' and accept the defaults for sitename and siteuri (both localsite). If you want to use the gateway servlet without restarting Tomcat, you will need to reload the site information. Visit the URL http://localhost:8080/greenstone3/gateway?a=s&sa=c  (substituting your server name and port number if necessary). See the user guide for more information about run time reconfiguration.
+
+
+Using SOAP:
+-------------------------------------------------------
+
+Greenstone comes with Apache Axis installed as part of the Greenstone web application.  A SOAP server on localsite is deployed during installation. You should be able to see all localsite's collections through the gateway servlet. (http://localhost:8080/greenstone3/gateway)
+
+To set up a SOAP server on a new site, run
+ant soap-deploy-site
+This will prompt you for the sitename (the directory name), and the site uri - this should be a unique identifier for the site's web service.
+
+For a non-interactive version, run
+ant -Daxis.sitename=xxx -Daxis.siteuri=yyy soap-deploy-site
+
+The service is accessible at http://localhost:8080/greenstone3/services/<siteuri>
+(or http://<computer-web-address>:<port>/greenstone3/services/<siteuri>)
+
+
+Using External Tomcat:
+---------------------------------------------------
+
+If you want to use an existing Tomcat, set the path to its base directory
+in build.properties (tomcat.installed.path) (on windows need to use double backslashes in paths '\\'). 
+
+ You will need to modify the Tomcat setup slightly.
+
+1. Tell Tomcat about the Greenstone web app. There are two ways to do this.
+
+A. Add in the Greenstone context to Tomcat's server.xml (In the <Host name="localhost"..> element)
+
+<Context path="/greenstone3" docBase="path-to-greenstone3/web" debug="1" 
+reloadable="true"><Resources allowLinking='true'/></Context>
+
+B. Alternatively, you can move (and rename) the greenstone3/web directory to tomcat/webapps/greenstone3 (i.e. the resulting directories will be like
+tomcat/webapps/greenstone3/WEB-INF, no web directory). This should be done after running the initial 'ant install'.
+
+You will need to set the web.home property in the build.properties file
+i.e.
+web.home=${tomcat.installed.path}/webapps/greenstone3
+And then run 'ant configure-web' to reset gsdl3home.
+
+2. Set up the JNI libraries and Java wrappers.
+JNI libraries and their Java wrappers cannot go into the web app. The libraries need to be loaded by the same class loader as their wrappers. The libraries need to be in java.library.path, and I think get loaded by the system class loader.
+The wrappers need to be loaded by this too. 
+
+These JNI bits are located by default in the lib/jni directory. There are two ways to get them into Tomcat:
+A: Keep all the Greenstone stuff inside the greenstone3 directory, and just modify the environment that Tomcat runs in
+
+Set LD_LIBRARY_PATH (GNU/Linux), DYLD_LIBRARY_PATH (Mac OS X) or PATH (windows) to include the  greenstone3/lib/jni directory.
+Add all the jar files in greenstone3/lib/jni directory to the CLASSPATH, then edit tomcats setclasspath.sh/bat to use the system CLASSPATH.
+(in setclasspath.bat, change 
+set CLASSPATH=%JAVA_HOME%\lib\tools.jar
+to 
+set CLASSPATH=%JAVA_HOME%\lib\tools.jar;%CLASSPATH%
+
+in setclasspath.sh, change
+# Set standard CLASSPATH
+if [ "$1" = "debug" -o "$1" = "javac" ] ; then
+  CLASSPATH="$JAVA_HOME"/lib/tools.jar
+fi
+
+to
+# Set standard CLASSPATH
+if [ "$1" = "debug" -o "$1" = "javac" ] ; then
+  CLASSPATH="$JAVA_HOME"/lib/tools.jar:"$CLASSPATH"
+fi
+
+
+B: Copy the files into Tomcat installation:
+Move the greenstone3/lib/jni jar files into tomcat's shared/lib directory.
+Move the greenstone3/lib/jni library files (.so for GNU/Linux, .jnilib for Mac OS X .dll for Windows) into shared/classes, and set LD_LIBARARY_PATH (GNU/Linux), DYLD_LIBRARY_PATH (Mac OS X) or PATH (Windows) to include this directory.
+ This has the advantage that you can use this for other webapps without modifying the Tomcat environment.
+
+Once all these changes have been made, you will need to restart the Tomcat server for them to take effect.
+
+Using External MySQL
+-------------------------------------------------------------
+
+Set the mysql.installed.path property in build.xml to be non-empty (its not actually used) before installation.
+
+You will need to add the two greenstone users: gsdl3reader and gsdl3admin. The reader user is only used for accessing the database, the admin user can be used for modification.
+
+Run MySQL as the root user (or a user with account creation privileges).
+
+Commands to add the two users:
+GRANT SELECT,INSERT,DELETE,UPDATE,DROP,CREATE ON *.* TO gsdl3admin@localhost identified by 'admin-password';
+GRANT SELECT ON *.* TO gsdl3reader@localhost identified by 'reader-password';
+
+You will need to edit the greenstone3/web/WEB-INF/classes/global.properties file and set the two passwords that you used in here:
+mysql.admin.password and mysql.reader.password
+The mysql.tcp.port property specified here should be set to the port that your MySQL is running on. (default 3306).
+
+You should also load up the database for the gs3mgdemo collection:
+create database localsite_gs3mgdemo;
+
+Close MySQL, then run
+mysql localsite_gs3mgdemo < <path-to-greenstone3>/web/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql
+
+(Note that if you have installed the Greenstone web directory into Tomcats webapps dir, then this command will be
+mysql localsite_gs3mgdemo < <path-to-tomcat>/webapps/greenstone3/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql )
+
+You may need to run this using '--user=root -p'
+
+Notes for Mac OS
+------------------------------------------------
+
+Set JAVA_HOME to be /Library/Java/Home
+
+Notes on Versions of Third Party packages
+-----------------------------------------------
+
+Tomcat:
+
+apache-tomcat-5.5.12.zip: latest production quality release as of October, 2005.
+apache-tomcat-5.5.12-compat.zip: Tomcat 5 requires Java 1.5. If using Java 1.4, need to use this compatibility module.
+
+website: http://tomcat.apache.org/
+download: http://tomcat.apache.org/download-55.cgi
+
+MySQL:
+
+Relational database. Binary versions. latest version 4 stable release as at October 2005.
+mysql-noinstall-4.1.15-win32.zip (Windows, without installer)
+mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz (Mac OS X v10.3, without installer)
+mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz (GNU/Linux, x86, glibc-2.2, static (Standard only), gcc)
+
+http://dev.mysql.com
+
+Axis:
+
+Apache Web Services Project, SOAP implementation. Axis is a follow on project to Apache SOAP
+
+axis-bin-1_2_1.zip: latest stable release as of October, 2005
+website: http://ws.apache.org/axis/
+download: http://www.apache.org/dyn/closer.cgi/ws/axis/1_2_1
+
+All available from www.greenstone.org/gs3files if not available at their respective websites.
+
+Other Notes:
+-------------------------------------------------
+
+See greenstone3/docs/manual/manual.pdf for more details about the software and installation etc.
+
+Under GNU/Linux and Mac OS X, Tomcat logs output to logs/catalina.out in the Tomcat directory (greenstone3/packages/tomcat)
+
+To prevent Tomcat showing directory listings, edit conf/web.xml file in the Tomcat directory and set the value of the "listings" servlet parameter to false.
+
+The file web/WEB-INF/classes/global.properties is generated on install and contains some properties for the run time system.
+
Index: /main/tags/3.01/gsdl3/TODO.txt
===================================================================
--- /main/tags/3.01/gsdl3/TODO.txt	(revision 21112)
+++ /main/tags/3.01/gsdl3/TODO.txt	(revision 21112)
@@ -0,0 +1,306 @@
+my specific todo list. in no particular order. smallish  jobs.
+
+* nzdl.org mirror presentation - from ian
+ replace macros
+ horizontal list -> first item should be selected
+ expand text, detach page buttons
+ remove unneccesary blank lines
+ how to in hdl - the entries are double spaced and also vertical lists
+ remove \n in config files
+ the image under the about/search etc titles is funny under macs
+ change font if language english/european??
+ rollover background for buttons and nav bar
+
+* get computer ggraphic design people to give input?? dave N matt J, polly cantlon ??
+
+* self documenting collections?? eg provides links to the source of the  class files , the xslt , the config file etc everything needed to generate teh coll/page 
+ 
+* UPDATE DOCUMENTATION!!!!
+
+* text on nzdl query page to resource bundle
+
+* collection getMetadata service
+
+* rethink how we determine docType - current method doesn't work for phind cos it only gets doc ids - it should come from the document if not specified.
+
+* niupepa display - abstract and commentary links. new action? or just new xslt?
+
+* is the 'to' att compulsory? do we have to have to=""?
+
+* query info - eg num docs found from a search, query term info - 
+is this called metadata for the query and returned in a metadataList? or 
+should it have another name?
+
+<response>
+<metadataList>
+<metadata name="numDocs">55</metadata>
+<content>
+<document name="HASHxxx"/>
+...
+</content>
+</response>
+
+????
+
+* format statements - need to add If and Or. do we test only on metadata? <gsf:if test="Title"> and <gsf:if test="Title='snail farming'> or more sophisticated?
+also need to fix the bit where <a href='[URL]'> gets turned into <a href='<gsf:metadata name='URL'/> - needs to be added as an attribute.
+
+* format statements for doc display
+
+* including named templates - need to replace the ones they overwrite - not allowed two templates with teh same name. eg docHeading, docContent. or should they be match=document mode=heading/content ??
+
+* coll specific xslt/actions?? how get these for remote colls?
+
+* process messages - do we still have pid and handle atts? make them the same, and both atts instead of params??
+
+* equivTermList contains terms
+
+* page link style paging eg 1, 2, 3, .... can have links to all, links to same eg 5 either side of current page. 1,2,3...24,25,26,27,28...231
+
+* multi query - finish.
+ 1. check if coll specified.
+ 2. check if ccs coll specified.
+ 3. get background info (description and format) for current coll - or all colls?? will we format stuff differently for different colls? have templates based on collection??
+ 4. do the query over all colls.
+ 5. get metadata for each result from appropriate coll.
+ 6. merge the results - append initially. sort by rank? this means we need to return the rank.
+
+* metadata retrieve determination - use an action helper? separate class? should look at xslt as well as format statements? 
+
+* what if have two services with teh same name eg two phind applets
+
+* phind applet not working at the moment. why??
+
+* tei building stuff into gsdl3 build code.
+
+* subcollection/language handling for gs2 collections
+
+* datelists formatted nicely with years at top and months down the side.
+
+* on/off format stuff in config file eg query term highlighting, display cover image and toc, offer enriching services to user, buttons, 
+
+* different templates for different positions in the hierarchy??
+X
+ ---
+   ~~~~
+   ~~~~
+ ---
+   ~~~~
+   ~~~~
+X
+ ---
+
+etc can we test for that? gs2 tests for [parent:Title]
+
+* global resource bundles, not class named ones. getGlobalText? getTExt(key, bundle name)?
+
+* fix errors in javadoc make
+
+* send back error messages to user? rather than getting null pointer exceptions. eg if ask for non-existant metadata. also invalid syntax info?
+
+* tidy up phind error messages
+
+* check text strings in code and xslt - put all into resource bundles
+
+* finalise the nzdl interface, and write script to do teh translations from the macro files.
+
+* query results: 7 out of 10 matches
+
+* bibliographic display??
+
+* look at what assumptions OID makes about ids - need it to be general not just working for hash ids.
+
+* specific tool tips for images. alt/title. eg this file is 380Mb.
+
+* nzdl prefs page - where do the defaults come from? do we need a full cgiargs class? with default value? nzdl, we already know what query options there are so dont need to get them from the service, so dont have displayinfo or default.
+
+* proper query term highlighting. phrases. fast.
+
+* system requests shouls also return status id not just a message. like the build stuff. 
+
+* should preferences be carried over between collections? currently they are.
+
+* xslt assumes that something is saving the query args. is this ok?
+
+* nzdl query mode - some/all, boolean/ranked, ranked/natural. result in different query parsing. new queryaction??
+
+* should actions have access to config? currently only applet actionactually uses it. - used for filling in library  - should receptionist do this instead?
+
+* should actions tell recepitonist if they need metadata? should receptionist pass in a list of metadata that the xslt requires?
+
+* polling for collections? delete and start again? check dates and only reload new ones?
+
+* caching. LInkedHashMap. overwrite the removeEldestEntry bits.
+
+* macros in coll description and titles. eg this collection was built _date_ days ago and contains _numdocs_ documents. 
+niupepa titles "_vol_1, _num_ 2" _commentary_
+
+*dynamic query content as well as dynamic query params?
+
+*communicators: 2 modes. 1 relays messages eg to a remote service
+2. site to site - needs to modify addresses and names.
+
+* make sure mgpp colls work with new config file stuff
+
+* presentation prefs - how do we decide on the list of langs? from interface config file? how do we use encoding?
+
+* mg/mgpp errors should not crash the servlet. 
+
+* some stuff for actions can go in config files. metadata determination, query term highlighting, enrich services. result pre or post processing. is there a generic way?
+
+* do we want a 'link to external page' action that displays the page but with teh greenstone header? a=p&sa=external?? also what about displaying whole cached html pages as frames??  gsdl uses a nav frame (a=p&p=nav) and then a second frame which is the html page.
+
+* doc ranking
+
+* generic multi query. a=q&c=demo&ccs=fao,... use a base coll??
+
+* what to do when config file says : {If}{[parent:Title],[parent:Title],[Title]}. maybe we need to do root_or_self type thing?? or have some way of testing.
+
+* stuff from talking to ian - I am sure that I have typed this in already - where is it????
+
+ macros in gdbm db - turn them into real text.
+horizontal list - first item should be selected
+expand text, detach page etc buttons.
+remove extraneous blank lines eg under nav bar
+how to in hdl is double spaced - shouldn't be
+and vertical lists appear to have more space
+remove \n from config files
+on the mac, about image looks weird - the green bar is on teh wrong side
+change font if language is english or european?? san serif font
+rollover background or colour on nav bar and buttons
+
+get meeting with computer design person, dave N, matt J? Polly cantlon?? colours, any quick hints?
+
+self documenting collections: eg links to config file, service code, xslt files,automatically generate but also can provide manually if wish.
+
+cstr browse format statement.
+turn off cover images
+
+Release: present existing gs2 colls
+ skeleton build for eg text and html.
+documentation
+scrap the collector?? ask the group
+release README: how to build a collection - gs2+convert? basic gs3?
+EASY installation
+consolidate and document
+adhoc testing
+translation of resource bundles
+
+help page: type in gsf:format statemenets and it shows you xslt? or html?
+
+test protocol - sample thing that could generate every message?? 
+
+
+
+General todo list. in no particular order. large  jobs.
+
+* gs3 release - basic stuff, translations, installation, documentation. oct 31? gs2 building only, both default and nzdl interfaces? gatherer?
+
+* generic resource retrieval - associated documents, src docs eg pdf/word, niupepa image files etc. if http accessible, put a link otherwise use a general resource retrieve?
+
+* collector collection building - do we want to finish this? needs addDocument and configureCollection services.
+
+The addDocument service (in GS2Construct) has not been implemented. It should take a file name and add the document to the import directory of teh collection.
+there are problems with just transmitting a file name - the service may live remotely and therefore the document is not there. You should probably send it attached to the html - therefore need to work out:
+how to get the document attached to the form (theres something to do with post and encoding = multipart ?? otherwise the browser just sends the filename),
+and then where to get it in the servlet - is it a parameter? or something else?
+
+and then it needs to be added into the xml request to be passed to the service.
+
+If the program is running locally its much simpler just to send a filename - can we somehow check for this?
+
+* configure Collection for building
+
+also to do with building, a little harder, is the ConfigureCollection service. There is no stub for it yet, but easy enough to add one. - need to add this service to the service description xml stuff, and write a processConfigureColleciton() method.  It would be easy enough to display the config file in a big text box, and have the user edit it like the collector does.
+
+the hard bit is that when you click ConfigureCollection, you dont know what collection you are going to be dealing with - all the building services, you select the collection on the service page. with the configure stuff, you need to select the collection, and then the config file needs to be retrieved. so its really a two step process to configure the coll - first select the coll, submit that, then edit the config file, and submit that.
+
+All the services currently are one step - need to think about how this type of service fits into the model.
+
+maybe it needs a hidden arg? - to tell teh service if you're at stage 1 or 2?
+when teh action does the request, it then asks for the service description again to redisplay it for the user. maybe if the service knows that it has done the first half, it sends the second type of description?
+
+also do we use the collect.cfg file or the collectionConfig.xml file to show the user?
+
+* sequence of services
+
+some service clusters  have services that you are supposed to carry out in sequence such as building, but there may be others.
+
+can we do a generic action or xslt or something that sends teh user to teh next service once they've completed the first one? 
+
+Maybe the service cluster/serviceRack class specifies teh sequence of services, and they are all handled individually like present except that some xslt puts a next button on each page with a link to teh next service in the list.
+
+* proper install package
+
+* currently nothing is cached - service decriptions, text strings etc could all be cached. (cgi args are cached by tomcat)
+
+* document structure - new greenstone archive format? TEI, XHTML, OO, GML compatible? parallel document structures?
+  - associated resources/documents - xlinks
+
+* greenstone 3 building - is building going to change for gs3, or will we continue to use the perl scripts? 
+  - import, build, activate
+  - info extraction, augmentation
+  - incremental update?
+  - modular? xml pipeline?
+
+* additional services - music search
+		      - keyphrase stuff
+		      - search history
+
+* all the gs2 admin/security stuff - user management, authentication
+
+* server side threading issues - is it thread safe?
+
+* usage logs
+
+* better error handling
+
+* document version control
+
+* combining requests to MR - results from first one becoming content for second one??
+
+* language translation stuff - when translating, we want to click on a text piece or macro and go to a sample page with that text string in it. can we do this?
+
+can we tell in gs3 where bits will be in pages?
+I think we could have a new type of request, that provides a page containing all teh text strings used by a service/agent. - would it be class specific or service specific?
+eg it would contain the service form, some sample results, and if its a process type of thing, all teh possible error and success messages.
+
+what about the interface bits?
+
+* are message formats fixed? can we create a DTD? or will they evolve too much over time?
+
+* sitewide service agents - may want to specify that only some of the services that could potentially be provided by an agent are actually provided.
+
+* should actions respond to describe requests? and what would they say if they did? are they agents and part of teh system, or somehow outside the system? 
+
+*  what about soap? is there a new version of that?
+
+* combined query services - eg one page with two service forms on there eg text query and music query - need to combine the results. eg teh action could carry out both queries then use xslt to combine the results.
+
+or do the same thing across title and creator indexes in mg to approximate fielded searching?
+
+true cross collection search would probably need a new service??
+
+* more generally, combining services in general, action and service level
+
+* action helpers: query term highlighting, page transformation and metadata determination (remove from receptionist?)
+* if leave an applet page you lose all teh previous info eg for the status display for importing. need a function to retrieve again all the previous messages to redisplay them.
+
+* can the xslt dynamically retrieve the metadata it needs for say query results??
+
+* CSS instead of tables? - use html lists than use css to format them.
+
+* cross coll search - if colls are built the same, just present the form and search both. need to think about ranking.
+if colls are not built the same, and have indexes, subcolls, langs - do we merge the lists? and only search those that have the right ones? what if two indexes have different meanings and display text but the same id? are they treated as same index? which name do you display?
+what about colls with different levels eg oneonly has document, the other doc and section. what do you display? field lists? just combine?
+
+* search history
+
+* use objects instead of applets? java provides a conversion tool. NPX_PLUGIN_PATH=
+
+* danas combined search and browse stuff
+
+* can xslt be used to generate requests as well as transform teh output?
+
+* kea.
+
Index: /main/tags/3.01/gsdl3/bin/script/ConstructCollection
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/ConstructCollection	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/ConstructCollection	(revision 21112)
@@ -0,0 +1,30 @@
+#!/bin/sh
+echo "Constructing collections ..."
+ 
+if [ "x$GSDLHOME" = "x" ] ; then
+  echo "You must have GSDLHOME set for building to work"
+  exit
+fi
+ 
+if [ "x$GSDL3HOME" = "x" ] ; then
+  echo "You must have GSDL3HOME set for building to work"
+  exit
+fi
+ 
+if [ "x$GSDLOS" = "x" ] ; then
+  echo "You must have GSDLOS set for building to work"
+  exit
+fi
+ 
+
+# -Xms32M          To set minimum memory
+# -Xmx32M          To set maximum memory
+# -verbose:gc      To set garbage collection messages
+# -Xincgc          For incremental garbage collection
+# -Xprof           Function call profiling
+# -Xloggc:<file>   Write garbage collection log
+ 
+echo "running java -DGSDLHOME=$GSDLHOME -DGSDLOS=$GSDLOS -DGSDL3HOME=$GSDL3HOME -DPATH=$PATH org.greenstone.gsdl3.build.ConstructCollection  $*"
+java -DGSDLHOME=$GSDLHOME -DGSDLOS=$GSDLOS -DGSDL3HOME=$GSDL3HOME -DPATH=$PATH org.greenstone.gsdl3.build.ConstructCollection  $*
+ 
+echo "Done!"
Index: /main/tags/3.01/gsdl3/bin/script/convert_coll_from_gs2.pl
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/convert_coll_from_gs2.pl	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/convert_coll_from_gs2.pl	(revision 21112)
@@ -0,0 +1,694 @@
+#!/usr/bin/perl -w
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDL3HOME not set\n" unless defined $ENV{'GSDL3HOME'};
+    die "GSDL3SRCHOME not set\n" unless defined $ENV{'GSDL3SRCHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDL3SRCHOME'}/lib/perl/cpan");
+}
+
+use colcfg;
+use util;
+use parsargv;
+use FileHandle;
+use XML::Writer;
+#can't get this to work on windows
+#use GDBM_File;
+
+use strict;
+
+&main();
+sub print_usage() {
+    print STDOUT "Usage: convert_coll_from_gs2.pl [options] coll-name\n";
+    print STDOUT "options:\n";
+    
+    print STDOUT "   -collectdir         Directory where collection lives.\n";
+    print STDOUT "   -verbosity          Controls the amount of output.\n";
+    print STDOUT "   -defaultlang        The language that is considered the default (for display text etc). defaults to 'en'\n\n";
+}
+
+sub main {
+
+    my ($defaultlang, $verbosity, $collectdir);
+    # note that no defaults are passed for most options as they're set
+    # later (after we check the collect.cfg file)
+    if (!&parsargv::parse(\@ARGV,
+			  'verbosity/\d+/', \$verbosity,
+			  'collectdir/.*/', \$collectdir,
+			  'defaultlang/.*/', \$defaultlang)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+    my ($collection) = @ARGV;
+    if (!defined($collection) || $collection eq "") {
+	die "No collection specified\n";
+    }
+    if ($collection eq "gs2model") {
+	die "You cant convert the model collection\n";
+    }
+    
+    if (!defined $collectdir || $collectdir eq "") {
+	$collectdir = &util::filename_cat ($ENV{'GSDLHOME'}, "collect");
+    }
+
+    if (!defined $defaultlang || $defaultlang eq "") {
+	$defaultlang = 'en';
+    }
+    # add on the coll name
+    $collectdir = &util::filename_cat ($collectdir, $collection);
+    
+    my $collconfigfilename = &util::filename_cat ($collectdir, "etc", "collect.cfg");
+    print STDOUT "coll config=$collconfigfilename\n";
+    my $collectcfg;
+    if (-e $collconfigfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($collconfigfilename);
+
+    } else {
+	print STDERR "collect.cfg not found!!";
+	die "\n";
+    }
+    
+
+    my $buildconfigfilename = &util::filename_cat ($collectdir, "index", "build.cfg");
+    my $buildcfg;
+    if (-e $buildconfigfilename) {
+	$buildcfg = &colcfg::read_build_cfg ($buildconfigfilename);
+
+    } else {
+	print STDERR "build.cfg not found!!";
+	die "\n";
+    }
+    
+
+     
+    my $colloutfile = &util::filename_cat ($collectdir, "etc", "collectionConfig.xml");
+    if (-e $colloutfile) {
+	print STDOUT "collectionConfig file already exists! overwriting it!\n";
+	
+    }
+   
+    my $buildoutfile = &util::filename_cat ($collectdir, "index", "buildConfig.xml");
+    if (-e $buildoutfile) {
+	print STDOUT "buildConfig file already exists! overwriting it!\n";
+	
+    }
+
+#    my $db_file = &util::filename_cat ($collectdir, "index", "text", "$collection.ldb");
+    my $database;
+#    if (-e $db_file) {
+#	$database = &open_database($db_file);
+#    } else {
+#	print STDERR "gdbm database file $db_file not found!!";
+#	die "\n";
+#    }
+    
+    my $buildtype;
+    if (defined $buildcfg->{'buildtype'}) {
+	$buildtype = $buildcfg->{'buildtype'};
+    } else {
+	$buildtype = 'mg';
+    }
+    
+    my $indexstem = undef;
+    if (defined $buildcfg->{'indexstem'}) {
+	$indexstem = $buildcfg->{'indexstem'};
+    }
+
+    my $buildoutput = new IO::File(">$buildoutfile");
+    my $buildwriter = new XML::Writer(OUTPUT => $buildoutput, NEWLINES => 1);
+    
+    $buildwriter->startTag('buildConfig', 'xmlns:gsf'=>"http://www.greenstone.org/greenstone3/schema/ConfigFormat");
+    
+    my $colloutput = new IO::File(">$colloutfile");
+    my $collwriter = new XML::Writer(OUTPUT => $colloutput, NEWLINES => 1);
+    
+    $collwriter->startTag('collectionConfig', 'xmlns:gsf'=>"http://www.greenstone.org/greenstone3/schema/ConfigFormat", 'xmlns:xsl'=>'http://www.w3.org/1999/XSL/Transform');
+    
+    #output the collection metadata to the collectionConfig file
+    $collwriter->startTag('metadataList');
+    
+    my $creator = $collectcfg->{'creator'};
+    &output_metadata($collwriter, 'creator', $creator);
+    $collwriter->endTag('metadataList');
+
+    #output the display collectionmeta to collectionConfig.xml
+    
+    my $collectionmeta = $collectcfg->{'collectionmeta'};
+    if (defined $collectionmeta) {
+	my %name_map = ('collectionname', 'name',
+		     'collectionextra', 'description',
+		     'iconcollection', 'icon',
+		     'iconcollectionsmall', 'smallicon');
+	
+	$collwriter->startTag('displayItemList');
+	foreach my $entry ( keys %$collectionmeta) {
+	    if (defined $name_map{$entry}) {
+		my $name= $name_map{$entry};
+		foreach my $lang (keys %{$collectionmeta->{$entry}}) {
+		    my $value = $collectionmeta->{$entry}->{$lang};
+		    if ($entry =~ /^icon/) {
+			$value = format_icon_value($value);
+		    }
+		    &output_display($collwriter, $name, $lang, $value);
+		}
+	    }
+	}
+	$collwriter->endTag('displayItemList');
+    } 
+    
+    # output building metadata to build config file 
+    my $numdocs = $buildcfg->{'numdocs'};
+    $buildwriter->startTag('metadataList');
+    &output_metadata($buildwriter, 'numDocs', $numdocs);
+    &output_metadata($buildwriter, 'buildType', $buildtype);
+    $buildwriter->endTag('metadataList');
+    
+    
+    #indexes
+    my $indexmap = {};
+    my $firstindex = "";
+    my $first = 1;
+    if (defined $buildcfg->{'indexmap'}) {
+	my $indexmap_t = $buildcfg->{'indexmap'};
+	foreach my $i (@$indexmap_t) {
+	    my ($k, $v) = $i =~ /^(.*)\-\>(.*)$/;
+	    $indexmap->{$k} = $v;
+	    if ($first) {
+		$firstindex = $v;
+		$first = 0;
+	    }	
+	}
+    } else {
+	print STDERR "indexmap not defined";
+    }
+    my $defaultindex;
+    if (defined $collectcfg->{'defaultindex'}) {
+	$defaultindex = $collectcfg->{'defaultindex'};
+	$defaultindex = $indexmap->{$defaultindex};
+    } else {
+	# use the first one
+	$defaultindex = $firstindex;
+    }
+    
+    # format stuff
+    my $format = $collectcfg->{'format'};
+
+    #output the search stuff to coll cfg
+    $collwriter->startTag('search');
+    foreach my $i (keys %$indexmap) {
+	my $shortname = $indexmap->{$i};
+	$collwriter->startTag('index', 'name'=>$shortname);
+	#find the coll meta stuff
+	my $indexdisplay = ".$i";
+	foreach my $lang (keys %{$collectionmeta->{$indexdisplay}}) {
+	    my $value = $collectionmeta->{$indexdisplay}->{$lang};
+	    output_display($collwriter, 'name', $lang, $value);
+	}
+	$collwriter->endTag('index');
+	
+    }
+
+    # add in the format stuff
+    if (defined $format->{'SearchVList'}) {
+	
+	$collwriter->startTag('format');
+	write_format($collwriter, $format->{'SearchVList'}, "document");
+	$collwriter->endTag('format');
+    }
+    
+    $collwriter->endTag('search');
+    
+    $buildwriter->startTag('serviceRackList');
+    
+    my @levels = ();
+    my $defaultlevel;
+    
+    #do the retrieve service
+    # assume mgpp or mg 
+    if ($buildtype eq 'mgpp') {
+	#for each level
+	if (defined $buildcfg->{'indexlevels'}) {
+	    push @levels, @{$buildcfg->{'indexlevels'}};
+	    
+	    if (defined $buildcfg->{'textlevel'}) {
+		$defaultlevel = $buildcfg->{'textlevel'};
+	    } else {
+		$defaultlevel = $levels[0];
+	    }
+	} else { #use levels from collect.cfg - must be an old collection
+	    @levels = ('Document');
+	    $defaultlevel = 'Document';
+	    if (defined $collectcfg->{'levels'}) {
+		foreach my $l (@{$collectcfg->{'levels'}}) {
+		    if ($l eq "Section") {
+			$defaultlevel = 'Section';
+		    }
+		    push @levels, $l;
+		}
+	    }
+	}
+	
+	$buildwriter->startTag('serviceRack', 'name'=>'GS2MGPPRetrieve');
+	$buildwriter->emptyTag('defaultLevel', 'name'=>$defaultlevel);
+
+	
+    } else {
+	$buildwriter->startTag('serviceRack', 'name'=>'GS2MGRetrieve');
+	$buildwriter->emptyTag('defaultIndex', 'name'=>$defaultindex);
+	
+    }
+    if (defined $indexstem) {
+	$buildwriter->emptyTag('indexStem', 'name'=>$indexstem);
+    } 
+    # close off the Retrieve service
+    $buildwriter->endTag('serviceRack');
+
+    # add in the classifiers if needed
+
+    my $count = 1;
+    my $phind = 0;
+    my $started_classifiers = 0;
+    if (defined $collectcfg->{'classify'}) { 
+	$collwriter->startTag('browse');
+	# add in default format if necessary
+	if (defined $format->{"VList"} || defined $format->{"HList"}) {
+	    # global formats
+	    $collwriter->startTag('format');
+	    if (defined $format->{"VList"}) {
+		# VLIst applies to both classifier and doc nodes
+		write_format($collwriter, $format->{"VList"}, "document");
+		write_format($collwriter, $format->{"VList"}, "classifier");
+	    }
+	    if (defined $format->{"HList"}) {
+		# hlist is only for classifier nodes
+		write_format($collwriter, $format->{"HList"}, "horizontal");
+	    }
+	    $collwriter->endTag('format');
+	}
+	my $classifiers = $collectcfg->{'classify'};
+	foreach my $cl (@$classifiers) {
+	    my $name = "CL$count";
+	    $count++;
+	    my ($classname) = @$cl[0];
+	    if ($classname =~ /^phind$/i) {
+		$phind=1;
+		#should add it into coll config classifiers
+		next;
+	    }
+	    
+	    my $horizontalAtTop = &isHorizontalClassifier($database, $name);
+	    if (not $started_classifiers) {
+		$buildwriter->startTag('serviceRack', 'name'=>'GS2Browse');
+		if (defined $indexstem) {
+		    $buildwriter->emptyTag('indexStem', 'name'=>$indexstem);
+		} 
+
+		$buildwriter->startTag('classifierList');		
+		$started_classifiers = 1;
+	    }
+	    my $content = ''; #use buttonname first, then metadata
+	    if ($classname eq "DateList") {
+		$content = "Date";
+	    } else {
+		for (my $i=0; $i<scalar(@$cl); $i++) {
+		    my $arg = @$cl[$i];
+		    if ($arg eq "-buttonname"){
+			$content = @$cl[$i+1];
+			last;
+		    } elsif ($arg eq "-metadata") {
+			$content = @$cl[$i+1];
+		    }
+		    
+		}
+	    }
+	    if ($horizontalAtTop) {
+		$buildwriter->emptyTag('classifier', 'name'=>$name, 'content'=>$content, 'horizontalAtTop'=>'true');
+	    } else {
+		$buildwriter->emptyTag('classifier', 'name'=>$name, 'content'=>$content);
+	    } 
+	    
+	    
+	    $collwriter->startTag('classifier', 'name'=>$name);
+	    my $vlist = $name."VList";
+	    my $hlist = $name."HList";
+	    my $dlist = "";
+	    if ($classname eq "DateList") {
+		$dlist = "DateList";
+	    }
+	    # need to work out how to split into classifier and document
+	    if (defined $format->{$vlist} || defined $format->{$hlist} || defined $format->{$dlist}) {
+		$collwriter->startTag('format');
+		if (defined $format->{$vlist}) {
+		    write_format($collwriter, $format->{$vlist}, "document");
+		    write_format($collwriter, $format->{$vlist}, "classifier");
+		} 
+		if (defined $format->{$hlist}) {
+		    write_format($collwriter, $format->{$hlist}, "horizontal");
+		} 
+		
+		if (defined $format->{$dlist}) {
+		    write_format($collwriter, $format->{$dlist}, "document");
+		}
+		$collwriter->endTag('format');
+	    }
+	    $collwriter->endTag('classifier');
+	} #foreach classifier
+	if ($started_classifiers) {
+	    # end the classifiers
+	    $buildwriter->endTag('classifierList');
+	    # close off the Browse service
+	    $buildwriter->endTag('serviceRack');
+	}
+	
+	$collwriter->endTag('browse');
+    }
+    
+    
+    # the phind classifier is a separate service
+    if ($phind) {
+	# if phind classifier
+	$buildwriter->emptyTag('serviceRack', 'name'=>'PhindPhraseBrowse');
+    }
+   
+    # do the search service
+    if ($buildtype eq 'mgpp') {
+		
+	$buildwriter->startTag('serviceRack', 'name'=>'GS2MGPPSearch');
+	$buildwriter->emptyTag('defaultLevel', 'name'=>$defaultlevel);
+	$buildwriter->startTag('levelList');
+	foreach my $level (@levels) {
+	    $buildwriter->emptyTag('level', 'name'=>$level);
+	}
+	$buildwriter->endTag('levelList');
+	
+	#fieldlist
+	my $fieldmap = {};
+	my $fieldlist = ();
+	if (defined $buildcfg->{'indexfields'}) {
+	    push @$fieldlist, @{$buildcfg->{'indexfields'}};
+	    if (defined $buildcfg->{'indexfieldmap'}) {
+		my $fieldmap_t = $buildcfg->{'indexfieldmap'};
+		foreach my $f (@$fieldmap_t) {
+		    my ($k, $v) = $f =~ /^(.*)\-\>(.*)$/;
+		    $fieldmap->{$k} = $v;
+		}
+	    }
+	    $buildwriter->startTag('fieldList');
+	    foreach my $f (@$fieldlist) {
+		my $field = $fieldmap->{$f};
+		$buildwriter->emptyTag('field', 'shortname'=>$field, 'name'=>$f);
+	    }	
+	    $buildwriter->endTag('fieldList');
+	} else {
+	    print STDERR "indexfieldmap not defined";
+	}
+
+	# do the search types if there
+	if (defined $collectcfg->{'searchtype'}) {
+	    $buildwriter->startTag('searchTypeList');
+	    foreach my $st (@{$collectcfg->{'searchtype'}}) {
+		$buildwriter->emptyTag('searchType', 'name'=>$st);
+	    }
+	    $buildwriter->endTag('searchTypeList');
+	}
+    } elsif ($buildtype eq 'mg') {
+	$buildwriter->startTag('serviceRack', 'name'=>'GS2MGSearch');
+    }
+    
+    if (defined $indexstem) {
+	$buildwriter->emptyTag('indexStem', 'name'=>$indexstem);
+    } 
+    
+    $buildwriter->emptyTag('defaultIndex', 'name'=>$defaultindex);
+    $buildwriter->startTag('indexList');
+    #for each index
+    foreach my $i (keys %$indexmap) {
+	my $index = $indexmap->{$i};
+	$buildwriter->emptyTag('index', 'name'=>$index);
+    }	
+    $buildwriter->endTag('indexList');
+    
+    $buildwriter->endTag('serviceRack');
+    
+    $buildwriter->endTag('serviceRackList');
+    $buildwriter->endTag('buildConfig');
+    $collwriter->endTag('collectionConfig');
+    $collwriter->end();
+    $buildwriter->end();
+    $buildoutput->close();
+    $colloutput->close();
+    &close_database($database);
+}
+
+
+sub output_metadata {
+    my ($writer, $metaname,  $metavalue) = @_;
+    $writer->startTag('metadata',  'name'=>$metaname);
+    $writer->characters($metavalue);
+    $writer->endTag('metadata');
+}
+
+sub output_display {
+    my ($writer, $name, $lang, $value) = @_;
+    $lang = 'en' if $lang eq 'default';
+    if ($lang =~ /^\[/) {
+	($lang) = $lang =~ /\[l=(.*)\]/;
+    }
+    $writer->startTag('displayItem', 'name'=>$name, 'lang'=>$lang);
+    $writer->characters($value);
+    $writer->endTag('displayItem');
+}
+sub format_icon_value {
+    my ($value) = @_;
+    if ($value =~ /^_/) {
+	my ($newvalue) = $value =~ /\/([^\/]*)$/;
+	if ($newvalue) {
+	    return $newvalue;
+	}
+    }
+    return $value;
+}
+
+sub write_format {
+    my ($writer, $old_format, $node_type) = @_;
+    # replace \' with '
+    $old_format =~ s/\\\'/\'/g;
+
+    #convert [] to <gsf:...>
+    # assume no nesting {If} or {Or} for now
+    $old_format =~ s/\{If\}\{([^\}]*)\}/&format_if($1, $node_type)/eg;
+    $old_format =~ s/\{Or\}\{([^\}]*)\}/&format_or($1)/eg;
+    $old_format =~ s/\[Text\]/\<gsf:text\/\>/g;
+    $old_format =~ s/\[num\]/\<gsf:num\/\>/g;
+    $old_format =~ s/\[link\]/\<gsf:link type=\'$node_type\'\>/g;
+    $old_format =~ s/\[\/link\]/\<\/gsf:link\>/g;
+    $old_format =~ s/\[srclink\]/\<gsf:link type=\'source\'\>/g;
+    $old_format =~ s/\[\/srclink\]/\<\/gsf:link\>/g;
+    $old_format =~ s/\[icon\]/\<gsf:icon type=\'$node_type\'\/\>/g;
+    $old_format =~ s/\[srcicon\]/\<gsf:icon type=\'source\'\/\>/g;
+		  
+    # what to do with hightlight??
+    $old_format =~ s/\[\/?highlight\]//g;
+
+    #now do the rest of the [] which are assumed to be metadata
+    $old_format =~ s/\[([^\]]*)\]/&format_metadata($1)/eg;
+  
+    # some html tidy
+    #turn <br> into <br />
+    $old_format =~ s/\<br\>/\<br \/\>/g;
+    #turn <p> into <p />
+    $old_format =~ s/\<p\>/\<p \/\>/g;
+    
+    #put quotes around any atts
+    $old_format =~ s/=([a-z]+)([> ])/=\'$1\'$2/g;
+
+    if ($node_type eq "document") {
+	$writer->startTag('gsf:template', 'match'=>'documentNode');
+	$writer->charactersXML($old_format);
+	$writer->endTag('gsf:template');
+    } elsif ($node_type eq "classifier") {
+	$writer->startTag('gsf:template', 'match'=>'classifierNode');
+	$writer->charactersXML($old_format);
+	$writer->endTag('gsf:template');
+    } elsif ($node_type eq "horizontal") {	
+	$writer->startTag('gsf:template', 'match'=>'classifierNode', 'mode'=>'horizontal');
+	$writer->charactersXML($old_format);
+	$writer->endTag('gsf:template');
+
+    }
+}
+
+sub format_metadata {
+    my ($metadata_string) = @_;
+
+    #print STDERR "original meta = $metadata_string\n"; 
+    
+    # what shall we do with cgisafe??
+    my $cgisafe = $metadata_string =~ s/^cgisafe://;
+
+    my ($select) = $metadata_string =~ /^(parent|sibling)/;
+    $metadata_string =~ s/^(parent|sibling)//;
+    my ($scope, $delim);
+    
+    if ($select) {
+	($scope) = $metadata_string =~ /^\((Top|All)/;
+	$metadata_string =~ s/^\((Top|All)\)?//;
+	if ($scope) {
+	    ($delim) = $metadata_string =~ /^\'([^\']*)\'\)/;
+	    $metadata_string =~ s/^\'([^\']*)\'\)//;
+	} 
+    }
+    $metadata_string =~ s/^://;
+    # remove ex.
+    $metadata_string =~ s/^ex\.//;
+    
+    #print STDERR "select=$select, scope=$scope, delim=|$delim|, meta = $metadata_string\n";
+    
+    my $new_format = "<gsf:metadata name='$metadata_string' ";
+    if (defined $select) {
+	if ($select eq "sibling") {
+	    $new_format .= "multiple='true' ";
+	    if (defined $delim) {
+		$new_format .= "separator='$delim' ";
+	    }
+	} elsif ($select eq "parent"){
+	    if (defined $scope) {
+		if ($scope eq "Top") {
+		    $new_format .= "select='root' ";
+		} elsif ($scope eq "All") {
+		    $new_format .= "select='ancestors' ";
+		    if (defined $delim) {
+			$new_format .= "separator='$delim' ";
+		    }
+		}
+	    } else {
+		$new_format .= "select='parent' ";
+	    }	
+	}
+    }
+    $new_format .= "/>";
+    #print STDERR "$new_format\n";
+    return $new_format;
+    
+}
+
+sub format_if {
+
+    my ($if_string, $node_type) = @_;
+    #print STDERR "if string = $if_string\n";
+
+    my @parts = split /,/, $if_string; 
+    my $test = $parts[0];
+    my $true_option = $parts[1];
+    my $false_option;
+    if (scalar (@parts) == 3) {
+	$false_option = $parts[2];
+    }
+    $test =~ s/^\s*//;
+    $test =~ s/\s*$//;
+    my ($test_meta, $test_type, $test_value);
+    if ($test =~ /^(\[.+\])$/) {
+	$test_meta = $1;
+	$test_type = 'exists';
+    } else {
+	my ($lhs, $exp, $rhs) = $test =~ /^(.+)\s+(eq|ne|lt|gt|le|ge|sw|ew)\s+(.+)$/;
+	#print STDERR "lhs, exp, rhs = $lhs, $exp, $rhs\n";
+	if ($exp eq "eq") {
+	    $test_type = "equals";
+	} elsif ($exp eq "sw") {
+	    $test_type = "startsWith";
+	} elsif ($exp eq "ew") {
+	    $test_type = "endsWith";
+	} elsif ($exp eq "ne") {
+	    $test_type = "notEquals";
+	} elsif ($exp eq "lt") {
+	    $test_type = "lessThan";
+	}elsif ($exp eq "gt") {
+	    $test_type = "greaterThan";
+	}elsif ($exp eq "le") {
+	    $test_type = "lessThanOrEquals";
+	}elsif ($exp eq "ge") {
+	    $test_type = "greaterThanOrEquals";
+	}
+	if ($lhs =~ /^\[.+\]$/) {
+	    $test_meta = $lhs;
+	    $test_value = $rhs;
+	} else {
+	    # assume rhs has meta
+	    $test_meta = $rhs;
+	    $test_value = $lhs;
+	}
+	
+	#remove beginning and end quotes
+	$test_value =~ s/^[\'\"]//;
+	$test_value =~ s/[\'\"]$//;
+    }
+    my $test_atts = "test='$test_type' ";
+    if (defined $test_value) {
+	$test_atts .= "test-value='$test_value' ";
+    }
+    #print STDERR "test, true, false = $test, $true_option, $false_option\n";
+    my $new_format = "<gsf:switch>$test_meta";
+    $new_format .= "<gsf:when $test_atts>$true_option</gsf:when>";
+    if (defined $false_option) {
+	$new_format .="<gsf:otherwise>$false_option</gsf:otherwise>";
+    }
+    $new_format .= "</gsf:switch>";
+	
+    #print STDERR "new format = $new_format\n";
+    return $new_format;
+}
+
+sub format_or {
+    my ($or_string) = @_;
+    my @meta_list = split (',', $or_string);
+    return "" unless scalar (@meta_list);
+    my $new_format = "<gsf:choose-metadata>";
+    foreach my $m (@meta_list) {
+	if ($m =~ /^\[(.*)\]$/) {
+	    $new_format .= &format_metadata($1);
+	} else {
+	    # a default value
+	    $new_format .= "<gsf:default>$m</gsf:default>";
+	    last;
+	}
+    }
+    $new_format .= "</gsf:choose-metadata>";
+    return $new_format;
+}
+
+sub open_database {
+    my ($db_file) = @_;
+    
+    my $database = ();
+#    tie (%$database, 'GDBM_File', $db_file, GDBM_READER, 0400) ||
+#	die "Couldn't open database $db_file\n";
+
+    return $database;
+}
+
+sub close_database {
+    my ($database) = @_;
+    untie %$database;
+}
+sub isHorizontalClassifier {
+    my ($database, $name) = @_;
+
+    return 0; # can't get this to work for windows
+    my $record = $database->{$name};
+    my ($childtype) = $record =~ /<childtype>(\w*)/;
+    if ($childtype eq "HList") {
+	return 1;
+    }
+    return 0;
+}
+#$writer->startTag('');
+#$writer->endTag('');
+#$writer->characters();
+#$writer->emptyTag('');
+
+1;
Index: /main/tags/3.01/gsdl3/bin/script/gs2_mkcol.pl
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/gs2_mkcol.pl	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/gs2_mkcol.pl	(revision 21112)
@@ -0,0 +1,297 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# gs2_mkcol.pl -- create the framework for a gs2 compatible collection in 
+# gs3
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will setup a new collection from a model one. It does this by
+# copying the model, moving files to have the correct names, and replacing
+# text within the files to match the parameters.
+
+package gs2_mkcol;
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use cfgread;
+
+sub print_usage {
+    print STDOUT "\n";
+    print STDOUT "gs2_mkcol.pl: Creates the directory structure for a new\n";
+    print STDOUT "          Greenstone2 compatible collection in Greenstone3.\n\n";
+    print STDOUT "  usage: $0 -site site-home -creator email [options] collection-name\n\n";
+    print STDOUT "  options:\n";
+    print STDOUT "   -optionfile file    Get options from file, useful on systems where\n";
+    print STDOUT "                       long command lines may cause problems\n";
+    print STDOUT "   -collectdir         Directory where new collection will be created.\n";
+    print STDOUT "                       Default is " . 
+	&util::filename_cat("site-home", "collect") . "\n";
+    print STDOUT "   -maintainer email   The collection maintainer's email address (if\n";
+    print STDOUT "                       different from the creator)\n";
+    print STDOUT "   -public true|false  If this collection has anonymous access\n";
+    print STDOUT "   -buildtype mg|mgpp  Whether to use mg or mgpp to build the collection\n";
+    print STDOUT "                       Default is mgpp.\n";
+    print STDOUT "   -title text         The title for the collection\n";
+    print STDOUT "   -about text         The about text for the collection\n";
+    print STDOUT "   -plugin text        perl plugin module to use (there may be multiple\n";
+    print STDOUT "                       plugin entries)\n";
+    print STDOUT "   -quiet              Operate quietly\n";
+    print STDOUT "  Note that -creator and -site must be specified. You can make changes to all\n";
+    print STDOUT "  options later by editing the collect.cfg configuration file for your\n";
+    print STDOUT "  new collection (it'll be in the \"etc\" directory).\n\n";
+    print STDOUT "  [Type \"perl -S mkcol.pl | more\" if this help text scrolled off your screen]";
+    print STDOUT "\n" unless $ENV{'GSDLOS'} =~ /^windows$/i;
+}
+
+sub traverse_dir
+{
+    my ($modeldir, $coldir) = @_;
+    my ($newfile, @filetext);
+
+    if (!(-e $coldir)) {
+
+	my $store_umask = umask(0002);
+	my $mkdir_ok = mkdir ($coldir, 0777);
+	umask($store_umask);
+
+	if (!$mkdir_ok) 
+	{
+	    die "$!";
+	}
+    }
+
+    opendir(DIR, $modeldir) || die "Can't read $modeldir";
+    my @files = grep(!/^(\.\.?|CVS)$/, readdir(DIR));
+    closedir(DIR);
+
+    foreach $file (@files)
+    {
+	my $thisfile = &util::filename_cat ($modeldir, $file);
+	if (-d $thisfile) {
+	    my $colfiledir = &util::filename_cat ($coldir, $file);
+	    traverse_dir ($thisfile, $colfiledir);
+
+	} else {
+	    my $destfile = $file;
+	    $destfile =~ s/^modelcol/$collection/;
+	    $destfile =~ s/^MODELCOL/$capcollection/;
+	    print STDOUT "  doing replacements for $destfile\n" unless $quiet;
+	    $destfile = &util::filename_cat ($coldir, $destfile);
+	    
+	    open (INFILE, $thisfile) || 
+		die "ERROR: Can't read file $thisfile";
+	    open (OUTFILE, ">$destfile") ||
+		die "ERROR: Can't create file $destfile";
+
+	    while (defined ($line = <INFILE>)) {
+		$line =~ s/\*\*collection\*\*/$collection/g;
+		$line =~ s/\*\*COLLECTION\*\*/$capcollection/g;
+		$line =~ s/\*\*creator\*\*/$creator/g;
+		$line =~ s/\*\*maintainer\*\*/$maintainer/g;
+		$line =~ s/\*\*public\*\*/$public/g;
+		$line =~ s/\*\*title\*\*/$title/g;
+		$line =~ s/\*\*about\*\*/$about/g;
+		$line =~ s/\*\*plugins\*\*/$pluginstring/g;
+		$line =~ s/\*\*buildtype\*\*/$buildtype/g;
+		$line =~ s/\*\*searchtype\*\*/$searchtype/g;
+		$line =~ s/\*\*indexes\*\*/$indexes/g;
+		$line =~ s/\*\*defaultindex\*\*/$defaultindex/g;
+		$line =~ s/\*\*indexmeta\*\*/$indexmeta/g;
+		$line =~ s/\*\*xmlindexes\*\*/$xmlindexes/g;
+		#$line =~ s/\*\*xmlplugins\*\*/$xmlpluginstring/g;
+		
+		print OUTFILE $line;
+	    }
+	    
+	    close (OUTFILE);
+	    close (INFILE);
+	}
+    }
+}
+
+# get and check options
+sub parse_args {
+    my ($argref) = @_;
+    if (!&parsargv::parse($argref,
+			  'optionfile/.*/', \$optionfile,
+			  'collectdir/.*/', \$collectdir,
+			  'site/.*/', \$sitehome,
+			  'creator/\w+\@[\w\.]+/', \$creator,
+			  'maintainer/\w+\@[\w\.]+/', \$maintainer,
+			  'public/true|false/true', \$public,
+			  'buildtype/mg|mgpp/mgpp', \$buildtype,
+			  'title/.+/', \$title,
+			  'about/.+/', \$about,
+			  'plugin/.+', \@plugin,
+			  'quiet', \$quiet,
+			  )) {
+	&print_usage();
+	die "\n";
+    }
+}
+
+sub main {
+
+    &parse_args (\@ARGV);
+    if ($optionfile =~ /\w/) {
+	open (OPTIONS, $optionfile) || die "Couldn't open $optionfile\n";
+	my $line = [];
+	my $options = [];
+	while (defined ($line = &cfgread::read_cfg_line ('mkcol::OPTIONS'))) {
+	    push (@$options, @$line);
+	}
+	close OPTIONS;
+	&parse_args ($options);
+
+    }
+
+    # load default plugins if none were on command line    
+    if (!scalar(@plugin)) {
+	@plugin = (ZIPPlug,GAPlug,TEXTPlug,HTMLPlug,EMAILPlug,
+		   PDFPlug,RTFPlug,WordPlug,PSPlug,ArcPlug,RecPlug);
+    }
+
+    # get and check the collection name
+    ($collection) = @ARGV;
+    if (!defined($collection)) {
+	print STDOUT "ERROR: no collection name was specified\n";
+	&print_usage();
+	die "\n";
+    }
+
+    if (length($collection) > 8) {
+	print STDOUT "ERROR: The collection name must be less than 8 characters\n";
+	print STDOUT "       so compatibility with earlier filesystems can be\n";
+	print STDOUT "       maintained.\n";
+	die "\n";
+    }
+
+    if ($collection eq "gs2model" || $collection eq "gs3model") {
+	print STDOUT "ERROR: No collection can be named gs2model or gs3model as these are  the\n";
+	print STDOUT "       names of the model collections.\n";
+	die "\n";
+    }
+
+    if ($collection eq "CVS") {
+	print STDOUT "ERROR: No collection can be named CVS as this may interfere\n";
+	print STDOUT "       with directories created by the CVS versioning system\n";
+	die "\n";
+    }
+
+    #check that -site has been specified
+    if (!defined($sitehome) || $sitehome eq "") {
+	print STDOUT "ERROR: The site was not defined. This variable is\n";
+	print STDOUT "       needed to locate the collect directory.\n";
+	die "\n";
+    }
+    #check that its a valid directory
+    if (!-d $sitehome) {
+	print STDOUT "ERROR: $sitehome doesn't exist\n";
+	die "\n";
+    }
+    if (!defined($creator) || $creator eq "") {
+	print STDOUT "ERROR: The creator was not defined. This variable is\n";
+	print STDOUT "       needed to recognise duplicate collection names.\n";
+	die "\n";
+    }
+
+    if (!defined($maintainer) || $maintainer eq "") {
+	$maintainer = $creator;
+    }
+
+    $public = "true" unless defined $public;
+    $buildtype = "mgpp" unless defined $buildtype;
+
+    $searchtype = "";
+    if ($buildtype eq "mgpp") {
+	$searchtype = "searchtype  plain form";
+    }
+    
+    if (!defined($title) || $title eq "") {
+	$title = $collection;
+    }
+
+    # get capitalised version of the collection
+    $capcollection = $collection;
+    $capcollection =~ tr/a-z/A-Z/;
+
+    # get the strings to include.
+    $pluginstring = "";
+    foreach $plugin (@plugin) {
+	if ($plugin eq RecPlug) {
+	    $pluginstring .= "plugin         $plugin -use_metadata_files\n";
+	} else {
+	    $pluginstring .= "plugin         $plugin\n";
+	}
+    }
+
+    $mdir = &util::filename_cat ($sitehome, "collect", "gs2model");
+    
+    if (defined $collectdir && $collectdir =~ /\w/) {
+	if (!-d $collectdir) {
+	    print STDOUT "ERROR: $collectdir doesn't exist\n";
+	    die "\n";
+	}
+	$cdir = &util::filename_cat ($collectdir, $collection);
+    } else {
+	$cdir = &util::filename_cat ($sitehome, "collect", $collection);
+    }
+
+    # make sure the model collection exists
+    die "ERROR: Cannot find the model collection $mdir" unless (-d $mdir);
+
+    # make sure this collection does not already exist
+    if (-e $cdir) {
+	print STDOUT "ERROR: This collection already exists\n";
+	die "\n";
+    }
+
+    # set up the default indexes - this could be a command line option at some stage
+    # the names are added in here for the xml ones, but they should be added after building once the names are known.
+    if ($buildtype eq "mg") {
+	$indexes = "document:text document:Title document:Source";
+	$defaultindex = "defaultindex document:text";
+	$indexmeta = "collectionmeta .document:text    \"text\"\ncollectionmeta .document:Title  \"titles\"\ncollectionmeta .document:Source  \"filenames\"\n";
+    } elsif ($buildtype eq "mgpp") {
+	$indexes = "allfields text metadata";
+	$defaultindex = "";
+	$indexmeta = "collectionmeta .text    \"text\"\ncollectionmeta .allfields \"entire documents\"\n";
+    } else {
+	print STDOUT "Error: buildtype should be mg or mgpp, but its $buildtype\n";
+    }
+    # start creating the collection
+    print STDOUT "\nCreating the collection $collection...\n" unless $quiet;
+    &traverse_dir ($mdir, $cdir);
+    print STDOUT "\nThe new collection was created successfully at\n" unless $quiet;
+    print STDOUT "$cdir\n" unless $quiet;
+}
+
+&main ();
Index: /main/tags/3.01/gsdl3/bin/script/gs3-build.bat
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/gs3-build.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/gs3-build.bat	(revision 21112)
@@ -0,0 +1,23 @@
+@echo off
+
+:: ---- Check that %GSDL3HOME% is set ----
+if not "%GSDL3HOME%" == "" goto start
+    echo "You need to run 'gs3-setup' in the greenstone3 directory before running this script"
+    goto exit
+
+:start
+
+if not "%1" == "" goto start2
+    echo Usage: gs3-build sitename collname
+    goto exit
+
+:start2
+if not "%2" == "" goto run
+    echo Usage: gs3-build sitename collname
+    goto exit
+
+:run
+
+java -cp "%CLASSPATH%" -DGSDL3HOME="%GSDL3HOME%"  -DGSDL3SRCHOME="%GSDL3SRCHOME%" -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser  org.greenstone.gsdl3.gs3build.Build -site %1 -collect %2
+
+:exit
Index: /main/tags/3.01/gsdl3/bin/script/gs3-build.sh
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/gs3-build.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/gs3-build.sh	(revision 21112)
@@ -0,0 +1,20 @@
+#!/bin/sh 
+
+#check that GSDL3HOME is set
+if test -z "$GSDL3HOME" ; then
+  echo "You need to 'source gs3-setup.sh' in the greenstone3 directory before running this script"
+ exit;
+fi
+
+if test $# != 2; then
+  echo "Usage: gs3-build.sh <site name> <collection name>"
+  exit;
+fi
+
+sitename=$1
+collectionname=$2
+
+# we need to stop here if the database isn't running - how can we tell??
+java -DGSDL3HOME=$GSDL3HOME -DGSDL3SRCHOME=$GSDL3SRCHOME -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser org.greenstone.gsdl3.gs3build.Build -site $sitename -collect $collectionname
+
+
Index: /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.bat
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.bat	(revision 21112)
@@ -0,0 +1,23 @@
+@echo off
+
+:: ---- Check that %GSDL3HOME% is set ----
+if not "%GSDL3HOME%" == "" goto start
+    echo "You need to run 'gs3-setup' in the greenstone3 directory before running this script"
+    goto exit
+
+:start
+
+if not "%1" == "" goto start2
+    echo Usage: gs3-mkcol sitename collname
+    goto exit
+
+:start2
+if not "%2" == "" goto run
+    echo Usage: gs3-mkcol sitename collname
+    goto exit
+
+:run
+
+java -cp "%CLASSPATH%" -DGSDL3HOME="%GSDL3HOME%" -DGSDL3SRCHOME="%GSDL3SRCHOME%" -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser  org.greenstone.gsdl3.gs3build.Create -site %1 -collect %2
+
+:exit
Index: /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.sh
===================================================================
--- /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/bin/script/gs3-mkcol.sh	(revision 21112)
@@ -0,0 +1,23 @@
+#!/bin/sh 
+
+#check that GSDL3HOME is set
+if test -z "$GSDL3HOME" ; then
+  echo "You need to 'source gs3-setup.sh' in the greenstone3 directory before running this script"
+ exit;
+fi
+
+if test $# != 2; then
+  echo "Usage: gs3-mkcol.sh <site name> <collection name>"
+  exit;
+fi
+
+sitename=$1
+collectionname=$2
+
+java -DGSDL3HOME=$GSDL3HOME -DGSDL3SRCHOME=$GSDL3SRCHOME -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser org.greenstone.gsdl3.gs3build.Create -site $sitename -collect $collectionname
+
+#/opt/sun-jdk-1.4.2.03/bin/java -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=y -DGSDL3HOME=$GSDL3HOME -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser org.greenstone.gsdl3.gs3build.Build -site $sitename -collect $collectionname
+
+
+
+
Index: /main/tags/3.01/gsdl3/build.properties
===================================================================
--- /main/tags/3.01/gsdl3/build.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/build.properties	(revision 21112)
@@ -0,0 +1,61 @@
+#### properties file for Greenstone3 building using ANT. See the README for more 
+#### information
+
+## java compilation properties - for javac
+#Should compilation include the debug option?
+compile.debug=true
+#Should compilation include the deprecation option?
+compile.deprecation=true
+#Should compilation include the optimize option?
+compile.optimize=true
+
+## windows c++ compilation
+compile.windows.c++.setup=C:/Program Files/Microsoft Visual Studio/VC98/Bin/VCVARS32.BAT
+
+## Tomcat installation
+#Set this to the internet/intranet name of the machine that Tomcat is/will be run on. localhost is fine for accessing the server from the local machine, but not remotely
+tomcat.server=localhost
+# The port number that tomcat is/will be run on
+tomcat.port=8080
+# If tomcat is already running, enter the path to the base directory here
+tomcat.installed.path=
+#uncomment the following if you have moved the greenstone3 web directory to tomcats webapps dir 
+#web.home=${tomcat.installed.path}/webapps/greenstone3
+
+## Greenstone 2 collection building options
+#The path to Greenstone 2 if it is already installed
+#If you do not want to use Greenstone 2 style building at all, set this property to any non-empty string, e.g. xxx
+gsdl2.installed.path=
+
+## Relational database 
+# set this to be non-empty if you are using an external mysql.
+mysql.installed.path=
+#change the following if the defaults are not correct
+mysql.server=localhost
+#port mysql should run on. Change if you already have mysql running on the default port
+mysql.port=3306
+#Should passwords be set for the root and greenstone mysql users?
+mysql.use.passwords=yes
+## If you are installing greenstone (including mysql) in an nfs directory, mysql may not work. The data directory needs to be on a local drive: Copy the greenstone3/packages/mysql/data directory to a local drive and set the location here
+#mysql.datadir=
+## Mysql user passwords: uncomment and set the following if you don't want to be prompted for them during installation
+#The root password for a local mysql installation
+#mysql.root.password=
+# The password for the gsdl3admin user (local and external mysql)
+#mysql.admin.password=
+# The password for the gsdl3reader user (local and external mysql)
+#mysql.reader.password=
+
+## Proxy setup - Greenstone installation will attempt to download extra packages. Set the proxy information here if you are behind a firewall.
+# The proxy server name 
+proxy.host=
+# The proxy server port
+proxy.port=
+# A user name to use when talking to the proxy server (optional, will prompt if not entered here)
+proxy.user=
+# A password to use when talking to the proxy server (optional, will prompt if not entered here)
+proxy.password=
+
+## Mac OS X specific stuff
+# set the path to your GDBM here
+gdbm.installed.path=
Index: /main/tags/3.01/gsdl3/build.xml
===================================================================
--- /main/tags/3.01/gsdl3/build.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/build.xml	(revision 21112)
@@ -0,0 +1,1582 @@
+<?xml version="1.0"?>
+
+<!-- ====================================================================== 
+     March 2005                                                    
+
+     Greenstone3 build and install script
+                   
+     kjdon                                                                
+     ====================================================================== -->
+<project name="greenstone3" default="usage" basedir=".">
+
+ <!-- ============ self defined tasks =================== -->
+
+  <taskdef name="mysetproxy" classname="org.greenstone.anttasks.MySetProxy" classpath="${basedir}/lib/java/anttasks.jar"/>
+  <taskdef name="getuserandpassword" classname="org.greenstone.anttasks.MyGetUserAndPassword" classpath="${basedir}/lib/java/anttasks.jar"/>
+
+
+
+<!-- ===================== Property Definitions =========================== -->
+
+<!--
+
+  Each of the following properties are used in the build script.
+  Values for these properties are set by the first place they are
+  defined, from the following list:
+
+  * Definitions on the "ant" command line (ant -Dfoo=bar compile).
+
+  * Definitions from a "build.properties" file in the top level
+    source directory of this application.
+
+  * Definitions from a "build.properties" file in the user's
+    home directory.
+
+  * Default definitions in this build.xml file.
+
+  You will note below that property values can be composed based on the
+  contents of previously defined properties.  This is a powerful technique
+  that helps you minimize the number of changes required when your development
+  environment is modified.  Note that property composition is allowed within
+  "build.properties" files as well as in the "build.xml" script.
+
+-->
+
+  <property file="build.properties"/>
+  <property file="${user.home}/build.properties"/>
+
+  <!-- get properties from the environment -->
+  <property environment="env"/>
+  <property name="build.home" value="${basedir}/build"/>
+  <property name="src.home" value="${basedir}/src/java"/>
+  <property name="src.packages.home" value="${basedir}/src/packages"/>
+  <property name="packages.home" value="${basedir}/packages"/>
+  <property name="extensions.home" value="${basedir}/extensions"/>
+  <!-- this may be set in build.properties, eg if you move the web dir to 
+  tomcats webapps directory -->
+  <property name="web.home" value="${basedir}/web"/>
+  <!-- jar files needed by applets go here -->
+  <property name="web.applet" value="${web.home}/applet"/>
+  <!-- jar files needed by the servlet (and extra ones) go here -->
+  <property name="web.lib" value="${web.home}/WEB-INF/lib"/>
+  <!-- other files needed by the servlet go here -->
+  <property name="web.classes" value="${web.home}/WEB-INF/classes"/>
+  <!-- jni libraries and java wrappers go here -->
+  <property name="lib.jni" value="${basedir}/lib/jni"/>
+  
+  <property name="javadocs" value="${basedir}/docs/javadoc"/>
+
+  <property name="app.name"      value="greenstone3"/>
+  <property name="app.path"      value="/${app.name}"/>
+  <property name="app.version"   value="3.00alpha"/>
+
+  <property name="cvs.root" value=":pserver:cvs_anon@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src"/>
+  <!-- catalina home is set to tomcat basedir is already installed, otherwise
+  use greenstone's tomcat -->
+  <condition property="catalina.home" value="${tomcat.installed.path}"> 
+    <and>
+      <isset property="tomcat.installed.path"/>
+      <not>
+	<equals arg1="" arg2="${tomcat.installed.path}"/>
+      </not>
+    </and>
+  </condition>
+  <property name="catalina.home" value="${packages.home}/tomcat"/>
+
+  <property name="os.linux" value="Linux"/> 
+  <property name="os.mac" value="Mac OS X"/>
+  <property name="os.windows" value="Windows 95,Windows 98,Windows 2000,Windows XP,Windows NT,Windows ME"/> <!-- check this!!!-->
+
+  <!-- this is true for linux and macs -->
+  <condition property="current.os.isunix">
+    <os family="unix"/>
+  </condition>
+  
+  <condition property="current.os.isunixnotmac">
+    <and>
+      <os family="unix"/>
+      <not>
+	<os family="mac"/>
+      </not>
+    </and>
+  </condition>
+  <condition property="current.os.ismac">
+    <os family="mac"/>
+  </condition>
+  
+  <condition property="current.os.iswindows">
+    <os family="windows"/>
+  </condition>
+
+  <!-- ============= Base dirs for each package and component ============ -->
+  <property name="applet.home" value="${src.home}/org/greenstone/applet"/>
+  <property name="gsdl3.home" value="${src.home}/org/greenstone/gsdl3"/>
+  <property name="anttasks.home" value="${src.home}/org/greenstone/anttasks"/>
+  <property name="gs2build.home" value="${basedir}/gs2build"/>
+  <property name="gli.home" value="${basedir}/gli"/>
+  <property name="vishnu.home" value="${extensions.home}/vishnu"/>
+  <property name="mg.home" value="${src.packages.home}/mg"/>
+  <property name="mgpp.home" value="${src.packages.home}/mgpp"/>
+  <property name="javagdbm.home" value="${src.packages.home}/javagdbm"/>
+  <property name="mysql.home" value="${packages.home}/mysql"/>
+  <property name="gsdl-as.home" value="${extensions.home}/gsdl-as"/>
+
+  <!-- can be set in build.properties if need be -->
+  <property name="mysql.datadir" value="./data"/>
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="true"/>
+  <property name="compile.optimize"    value="true"/>
+
+<!--
+
+  Rather than relying on the CLASSPATH environment variable, Ant includes
+  features that makes it easy to dynamically construct the classpath you
+  need for each compilation.  The example below constructs the compile
+  classpath to include the servlet.jar file, as well as the other components
+  that Tomcat makes available to web applications automatically, plus anything
+  that you explicitly added.
+
+-->
+
+  <path id="compile.classpath">
+    <!-- Include all jar files and libraries in our jni lib directory -->
+    <pathelement location="${lib.jni}"/>
+    <fileset dir="${lib.jni}">
+      <include name="*.jar"/>
+    </fileset>
+    <!-- Include all jar files in our web lib directory -->
+    <pathelement location="${web.lib}"/>
+    <fileset dir="${web.lib}">
+      <include name="*.jar"/>
+    </fileset>
+    
+    <!-- Include the axis jar files -->
+    <!--<fileset dir="${basedir}/comms/soap/axis/lib">
+      <include name="*.jar"/>
+    </fileset>-->
+    
+    <!-- include the jar files from the source packages -->
+    <!-- mg and mgpp get installed into lib/jni but they may not be there yet
+    so we add them in by name -->
+    <pathelement location="${lib.jni}/mg.jar"/>
+    <pathelement location="${lib.jni}/mgpp.jar"/>
+    <pathelement location="${vishnu.home}/build/vishnu.jar"/>
+    
+    <!-- Include all elements that Tomcat exposes to applications -->
+    <pathelement location="${catalina.home}/common/classes"/>
+    <fileset dir="${catalina.home}/common/endorsed">
+      <include name="*.jar"/>
+    </fileset>
+    <fileset dir="${catalina.home}/common/lib">
+      <include name="*.jar"/>
+    </fileset>
+    <pathelement location="${catalina.home}/shared/classes"/>
+    <fileset dir="${catalina.home}/shared/lib">
+      <include name="*.jar"/>
+    </fileset>
+  </path>
+
+  <path id="local.tomcat.classpath">
+    <!-- explicitly include the jni java wrappers in the classpath -->
+    <pathelement location="${lib.jni}"/>
+    <fileset dir="${lib.jni}">
+      <include name="*.jar"/>
+    </fileset>
+  </path>
+  
+  <path id="local.tomcat.path">
+    <pathelement location="${mysql.home}/bin"/>
+    <pathelement location="${basedir}/bin/script"/>
+    <pathelement location="${basedir}/bin"/>
+    <pathelement location="${lib.jni}"/>
+    <pathelement path="${env.PATH}"/>
+    <pathelement path="${env.Path}"/>
+  </path>
+
+  <target name="test-setup">
+    <echo>ant java version=${ant.java.version}</echo>
+    <echo>is linux : ${current.os.isunix}</echo>
+    <echo>is mac : ${current.os.ismac}</echo>
+    <echo>is unixnotmac : ${current.os.isunixnotmac}</echo>
+    <echo>is windows : ${current.os.iswindows}</echo>
+  </target>
+<!-- ==================== Primary and Global Targets ============================= -->
+
+  <!-- add comments about using xxx-core, xxx-packages if only want certain parts?? -->
+
+  <target name="prepare" depends="accept-properties,init,prepare-core,prepare-packages,prepare-extensions,prepare-gs2building,prepare-tomcat,prepare-axis,prepare-mysql,prepare-web"
+    description="Use this when you first checkout the code: 'ant prepare install'. This will do some additional cvs checkouts and downloads, so you need to be online to run this."/>
+
+  <target name="install" depends="init,configure-mysql,configure,configure-c++,compile,deploy-localsite" 
+    description="Install Greenstone 3. Use this when you first checkout the code: 'ant prepare install'."/>
+  
+  <target name="cvsupdate" depends="init,cvsupdate-packages,cvsupdate-extensions,cvsupdate-core,cvsupdate-gs2building,cvsupdate-web"
+    description="Do a cvs update for all sources. Doesn't recompile the code. You need to be online to run this."/>
+
+  <target name="configure-c++" depends="init,configure-packages,configure-core,configure-extensions,configure-gs2building"
+    description="Configure any C/C++ package code and gs2building code if necessary"/>
+  
+  <target name="configure" depends="init,configure-tomcat,configure-web"
+    description="Configure the installation. Includes setting up config files. Should be re-run if you change the build.properties file."/>
+
+  <target name="clean" depends="init,clean-packages,clean-extensions,clean-core,clean-gs2building"
+    description="Remove all old compiled code. Includes core, packages and gs2building if necessary"/>
+  <target name="clean-windows-c++" depends="init,clean-windows-c++-gs2building,clean-windows-c++-packages"
+    description="Remove C++ compiled code for Windows. Requires Microsoft Visual Studio"/>
+  
+  <target name="compile" depends="init,compile-web,compile-packages,compile-core,compile-extensions,compile-gs2building"
+    description="Compile all the source code, includes core, packages and gs2building if necessary. Copy jar files and executables to their correct places."/>
+  
+  <target name="compile-windows-c++" depends="init,compile-windows-c++-gs2building,compile-windows-c++-packages"
+    description="Compile the C/C++ code for Windows. Requires Microsoft Visual Studio"/>
+  
+  <target name="update" depends="init,cvsupdate,clean,configure,configure-c++,compile" 
+    description="Update (thru CVS) all the source (including core, packages and gs2building), then clean, configure and recompile."/>
+
+  <target name="start" depends="init,start-mysql,start-tomcat"
+    description="Startup the (local) Greenstone servers (tomcat,mysql...)" />
+
+  <target name="stop" depends="init,stop-tomcat,stop-mysql" 
+    description="Shutdown the (local) Greenstone servers."/>
+  
+  <target name="restart" description="Shutdown and restart the (local) Greenstone servers" depends="init,stop,start"/>
+
+
+<!-- =========== Help targets ===================================  -->
+  
+  <property name="install-command" value="ant [options] prepare install"/>
+  
+  <target name="usage" description="Print a help message">
+    <echo message="  Execute 'ant -projecthelp' for a list of targets."/>
+    <echo message="  Execute 'ant -help' for Ant help."/>
+    <echo>To install Greenstone3, run '${install-command}'. 
+There are properties defined in build.properties. The install process will ask you if these properties are set correctly. To avoid this prompt, use the '-Dproperties.accepted=yes' option.
+To log the output, use the '-logfile build.log' option.
+The README.txt file has more information about the ant targets and install process.
+    </echo>
+  </target>
+  
+  <target name="help" depends="usage" description="Print a help message"/>
+  
+  <target name="debug" depends="init" description="Display all the currently used properties">
+    <echoproperties/>
+  </target>
+
+<!-- ====== initialization and setup targets ================== -->
+
+  <target name="accept-properties" unless="properties.accepted">
+    <input addproperty="properties.ok" validargs="y,n">The following properties (among others) are being used from a build.properties file found in this directory:
+tomcat.server=${tomcat.server}
+tomcat.port=${tomcat.port} 
+tomcat.installed.path=${tomcat.installed.path} (this is the location of Tomcat's base dir if it is already installed)
+gsdl2.installed.path=${gsdl2.installed.path} (this is the location of Greenstone 2 if you have it)
+proxy.host=${proxy.host}
+proxy.port=${proxy.port}
+mysql.installed.path=${mysql.installed.path} (this is the location of mysql if it is already installed)
+mysql.port=${mysql.port}
+If these are not acceptable, please change them and rerun this target. Continue [y/n]? />
+    </input>
+    <condition property="do.abort"> 
+      <equals arg1="n" arg2="${properties.ok}"/>
+    </condition>
+    <fail if="do.abort">Build aborted by user. Please change your properties settings and re-run the target</fail>
+  </target>
+
+  <target name="check-cvsroot">
+    <condition property="cvsroot.notset">
+      <or>
+	<not>
+	  <isset property="env.CVSROOT"/>
+	</not>
+	<equals arg1="" arg2="${env.CVSROOT}"/>
+      </or>
+    </condition>
+    <fail if="cvsroot.notset" message="You need to set the CVSROOT variable"/>
+  </target>
+
+  <!-- this sets up some initial properties -->
+  <target name="init">
+
+    <condition property="java.too.old">
+      <or>
+	<equals arg1="1.1" arg2="${ant.java.version}"/>
+	<equals arg1="1.2" arg2="${ant.java.version}"/>
+	<equals arg1="1.3" arg2="${ant.java.version}"/>
+      </or>
+    </condition>
+    <fail if="java.too.old" message="You need Java 1.4 or greater to run Greenstone 3"/>
+    
+    <available file="${src.packages.home}/mgpp/text" property="mgpp.present"/>
+    <available file="${packages.home}/mysql/bin" property="mysql.present"/>
+    <available file="${basedir}/gli" property="gli.present"/>
+    <available file="${basedir}/gs2build" property="gs2build.present"/>
+    <condition property="gsdl2.islocal">
+      <or>
+	<not>
+	  <isset property="gsdl2.installed.path"/>
+	</not>
+	<equals arg1="" arg2="${gsdl2.installed.path}"/>
+      </or>
+    </condition>
+    <condition property="mysql.islocal">
+      <or>
+	<not>
+	  <isset property="mysql.installed.path"/>
+	</not>
+	<equals arg1="" arg2="${mysql.installed.path}"/>
+      </or>
+    </condition>
+    <condition property="tomcat.islocal">
+      <or>
+	<not>
+	  <isset property="tomcat.installed.path"/>
+	</not>
+	<equals arg1="" arg2="${tomcat.installed.path}"/>
+      </or>
+    </condition>
+    
+      <echo>tomcat.port:${tomcat.port}, gli.present:${gli.present} gsdlislocal=${gsdl2.islocal} gs2build.present=${gs2build.present} gsdl2.installed.path = ${gsdl2.installed.path}</echo>
+    <condition property="proxy.present">
+      <and>
+	<isset property="proxy.host"/>
+	<not>
+	  <equals arg1="" arg2="${proxy.host}"/>
+	</not>
+      </and>
+    </condition>
+
+    <condition property="need.macos.extra">
+      <and>
+	<isset property="current.os.ismac"/>
+	<isset property="gsdl2.islocal"/>
+      </and>
+    </condition>
+  </target>
+
+  <target name="setup-proxy" depends="init" if="proxy.present">
+    <condition property="ask.user">
+      <or>
+	<equals arg1="" arg2="${proxy.user}"/>
+	<equals arg1="" arg2="${proxy.password}"/>
+      </or>
+    </condition>
+    <getuserandpassword message="Using proxy: ${proxy.host}:${proxy.port}" if="ask.user" username="${proxy.user}" userproperty="proxy.username" pwordproperty="proxy.password"/>
+    <mysetproxy proxyhost="${proxy.host}" proxyport="${proxy.port}" proxyuser="${proxy.username}" proxypassword="${proxy.password}"/>
+  </target>
+     
+ <!-- ========== Web app Targets ================================ -->
+  
+  <target name="prepare-web">
+    <property name="collect.dir" value="${web.home}/sites/localsite/collect"/>
+    <!-- gs3mgdemo -->
+    <unzip src="${collect.dir}/gs3mgdemo/import.zip" 
+      dest="${collect.dir}/gs3mgdemo"/>
+    <unzip src="${collect.dir}/gs3mgdemo/archives.zip" 
+      dest="${collect.dir}/gs3mgdemo"/>
+    <unzip src="${collect.dir}/gs3mgdemo/index/index.zip" 
+      dest="${collect.dir}/gs3mgdemo/index"/>
+    <delete file="${collect.dir}/gs3mgdemo/import.zip"/>
+    <delete file="${collect.dir}/gs3mgdemo/archives.zip"/>
+    <delete file="${collect.dir}/gs3mgdemo/index/index.zip"/>
+    <!-- gs2mgdemo -->
+    <unzip src="${collect.dir}/gs2mgdemo/import.zip" 
+      dest="${collect.dir}/gs2mgdemo"/>
+    <unzip src="${collect.dir}/gs2mgdemo/metadata.zip" 
+      dest="${collect.dir}/gs2mgdemo"/>
+    <unzip src="${collect.dir}/gs2mgdemo/index/index.zip" 
+      dest="${collect.dir}/gs2mgdemo/index"/>
+    <delete file="${collect.dir}/gs2mgdemo/import.zip"/>
+    <delete file="${collect.dir}/gs2mgdemo/metadata.zip"/>
+    <delete file="${collect.dir}/gs2mgdemo/index/index.zip"/>
+    <!-- gs2mgppdemo -->
+    <unzip src="${collect.dir}/gs2mgppdemo/import.zip" 
+      dest="${collect.dir}/gs2mgppdemo"/>
+    <unzip src="${collect.dir}/gs2mgppdemo/metadata.zip" 
+      dest="${collect.dir}/gs2mgppdemo"/>
+    <unzip src="${collect.dir}/gs2mgppdemo/index/index.zip" 
+      dest="${collect.dir}/gs2mgppdemo/index"/>
+    <delete file="${collect.dir}/gs2mgppdemo/import.zip"/>
+    <delete file="${collect.dir}/gs2mgppdemo/metadata.zip"/>
+    <delete file="${collect.dir}/gs2mgppdemo/index/index.zip"/>
+    <!-- gberg -->
+    <unzip src="${collect.dir}/gberg/index/index.zip" 
+      dest="${collect.dir}/gberg/index"/>
+    <delete file="${collect.dir}/gberg/index/index.zip"/>
+    <mkdir dir="${web.home}/applet"/>
+
+    <condition property="need.xml.jars">
+      <equals arg1="1.5" arg2="${ant.java.version}"/>
+    </condition>
+    <antcall target="rename-xml-jars"/>
+  </target>
+  
+  <target name="rename-xml-jars" if="need.xml.jars">
+    <copy file="${web.lib}/xalan.jar.tmp" tofile="${web.lib}/xalan.jar"/>
+    <copy file="${web.lib}/xercesImpl.jar.tmp" tofile="${web.lib}/xercesImpl.jar"/>
+  </target>
+  
+  <target name="configure-web" depends="init,get-mysql-reader-password,get-mysql-admin-password"
+    description="Configure only the web app config files">
+    <!-- we want a unix path in the global.properties file -->
+    <pathconvert targetos="unix" property="gsdl3.home.unix">
+      <path path="${web.home}"/>
+    </pathconvert>
+    <!-- just in case these haven't been defined, we need something to write to
+    the file -->
+    <property name="mysql.admin.password" value=" "/>
+    <property name="mysql.reader.password" value=" "/>
+    <filter token="gsdl3home" value="${gsdl3.home.unix}"/>
+    <filter token="mysql.port" value="${mysql.port}"/>
+    <filter token="mysql.server" value="${mysql.server}"/>
+    <filter token="mysql.admin.pword" value="${mysql.admin.password}"/>
+    <filter token="mysql.reader.pword" value="${mysql.reader.password}"/>
+    <filter token="tomcat.server" value="${tomcat.server}"/>
+    <filter token="tomcat.port" value="${tomcat.port}"/>
+    <copy file="${basedir}/resources/java/global.properties.in" tofile="${web.classes}/global.properties" filtering="true" overwrite="true"/> 
+    <chmod file="${web.classes}/global.properties" perm="600"/>
+  </target>
+
+  <target name="compile-web" depends="init">
+    <javac srcdir="${web.classes}"
+      destdir="${web.classes}"
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath>
+	<path refid="compile.classpath"/>
+      </classpath>
+    </javac>
+  </target>
+
+  <target name="cvsupdate-web" unless="nocvs.mode">
+    <cvs command="update -dP" dest="${web.home}"/>
+  </target>
+
+  <target name="update-web" depends="init,cvsupdate-web,configure-web"
+    description="update only the web stuff (config files)"/>
+
+
+<!-- ======================= Tomcat Targets ========================== -->
+  
+  <!-- this target sets up tomcat for the first time, or resets it any subsequent times -->
+  <target name="prepare-tomcat" depends="init,setup-proxy" if="tomcat.islocal">
+    <get src="http://www.greenstone.org/gs3files/apache-tomcat-5.5.12.zip"
+      dest="${packages.home}/apache-tomcat-5.5.12.zip"
+      usetimestamp="true"/>
+    <delete dir="${packages.home}/tomcat.bak"/>
+    <move todir="${packages.home}/tomcat.bak" failonerror="false">
+      <fileset dir="${packages.home}/tomcat"/>
+    </move>
+    <unzip src="${packages.home}/apache-tomcat-5.5.12.zip" 
+      dest="${packages.home}"/>
+    <condition property="need.tomcat.compat">
+      <equals arg1="1.4" arg2="${ant.java.version}"/>
+    </condition>
+    <antcall target="prepare-tomcat-compat"/>
+    <move todir="${packages.home}/tomcat">
+      <fileset dir="${packages.home}/apache-tomcat-5.5.12"/>
+    </move>
+    <copy file="${packages.home}/tomcat-extra/setclasspath.bat"
+      tofile="${packages.home}/tomcat/bin/setclasspath.bat"
+      overwrite="true"/>
+    <copy file="${packages.home}/tomcat-extra/setclasspath.sh"
+      tofile="${packages.home}/tomcat/bin/setclasspath.sh"
+      overwrite="true"/>
+    <!-- make sure we have execute permission for the .sh files -->
+    <chmod dir="${packages.home}/tomcat/bin" perm="ugo+rx" 
+      includes="*.sh"/>
+  </target>
+
+  
+  <target name="prepare-tomcat-compat" if="need.tomcat.compat">
+    <get src="http://www.greenstone.org/gs3files/apache-tomcat-5.5.12-compat.zip"
+      dest="${packages.home}/apache-tomcat-5.5.12-compat.zip"
+      usetimestamp="true"/>
+    <unzip src="${packages.home}/apache-tomcat-5.5.12-compat.zip" 
+      dest="${packages.home}"/>
+    <copy file="${packages.home}/apache-tomcat-5.5.12/common/endorsed/xml-apis.jar"
+      tofile="${web.applet}/xml-apis.jar"/>
+    <copy file="${packages.home}/apache-tomcat-5.5.12/common/endorsed/xercesImpl.jar"
+      tofile="${web.applet}/xercesImpl.jar"/>
+  </target>
+  
+  <target name="configure-tomcat" depends="init,configure-tomcat-local,configure-tomcat-external"/>
+  
+  <target name="configure-tomcat-local" depends="init" if="tomcat.islocal">
+    <!-- re-setup the server.xml file -->
+    <copy file="${packages.home}/tomcat-extra/server.xml.in" tofile="${packages.home}/tomcat/conf/server.xml" overwrite="true">
+      <filterset>
+	<filter token="port" value="${tomcat.port}"/>
+	<filter token="gsdl3home" value="${basedir}"/>
+      </filterset>
+    </copy>
+  </target>
+
+  <target name="configure-tomcat-external" depends="init" unless="tomcat.islocal">
+    <!-- re-setup the server.xml file -->
+    <!-- need to edit the config file, or do we get the user to do this???-->
+  </target>
+  
+  <target name="reload" description="Reload web application"
+          depends="init,setup-catalina-ant-tasks">
+    <reload  url="http://${tomcat.server}:${tomcat.port}/manager" username="admin" password="admin"
+            path="${app.path}"/>
+  </target>
+  
+  <target name="start-tomcat" description="Startup only Tomcat" depends="init" if="tomcat.islocal">
+   <property name="tomcat.classpath" refid="local.tomcat.classpath"/>
+    <property name="tomcat.path" refid="local.tomcat.path"/>
+    <property name="catalina.opts" value="-Djava.util.prefs.syncInterval=2000000 -DGSDL3HOME=$GSDL3HOME -DGSDLOS=$GSDLOS -DPATH=$PATH -Xmx400M"/>
+    <exec executable="${catalina.home}/bin/startup.sh" os="${os.linux},${os.mac}" dir="${catalina.home}/bin" spawn="false">
+      <!--<env key="GSDLOS" value="linux"/> do we need this?? -->
+      <env key="GSDL3HOME" value="${basedir}"/>
+      <env key="PATH" path="${tomcat.path}"/>
+      <env key="CATALINA_OPTS" value="${catalina.opts}"/>
+      <env key="CATALINA_HOME" value="${catalina.home}"/>
+      <env key="CLASSPATH" path="${tomcat.classpath}"/>
+      <env key="LD_LIBRARY_PATH" path="${env.LD_LIBRARY_PATH}:${lib.jni}"/>
+      <env key="DYLD_LIBRARY_PATH" path="${env.DYLD_LIBRARY_PATH}:${lib.jni}:${gdbm.installed.path}/lib"/> <!-- for mac os -->
+    </exec>
+    <exec executable="${catalina.home}/bin/startup.bat" os="${os.windows}" dir="${catalina.home}/bin" spawn="true">
+      <env key="GSDLOS" value="windows"/>
+      <env key="GSDL3HOME" value="${basedir}"/>
+      <env key="Path" path="${tomcat.path}"/>
+      <env key="PATH" path="${tomcat.path}"/>
+      <env key="CATALINA_OPTS" value="${catalina.opts}"/>
+      <env key="CLASSPATH" path="${tomcat.classpath}"/>
+    </exec>
+    <!-- wait for the server to startup in case other targets need it running -->
+    <waitfor maxwait="5" maxwaitunit="second">
+      <and>
+	<socket server="${tomcat.server}" port="${tomcat.port}"/>
+	<http url="http://${tomcat.server}:${tomcat.port}${app.path}/index.html"/>
+      </and>
+    </waitfor>
+  </target>
+  <!-- windows: do we want to launch a webrowser?? -->
+  <target name="stop-tomcat" description="Shutdown only Tomcat" depends="init" if="tomcat.islocal">
+    <exec executable="${catalina.home}/bin/shutdown.sh" os="${os.linux},${os.mac}" dir="${catalina.home}/bin" spawn="false">
+      <env key="CATALINA_HOME" value="${catalina.home}"/>
+    </exec>
+    <exec executable="${catalina.home}/bin/shutdown.bat" os="${os.windows}" dir="${catalina.home}/bin" spawn="false"/>
+  </target>
+
+  <target name="restart-tomcat" description="Shutdown and restart only Tomcat" depends="init,stop-tomcat,start-tomcat"/>
+
+  <target name="setup-catalina-ant-tasks">
+    <!-- Configure the custom Ant tasks for the Tomcat Manager application -->
+    <taskdef name="deploy"    classname="org.apache.catalina.ant.DeployTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="list"      classname="org.apache.catalina.ant.ListTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="reload"    classname="org.apache.catalina.ant.ReloadTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="resources" classname="org.apache.catalina.ant.ResourcesTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="roles"     classname="org.apache.catalina.ant.RolesTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="start"     classname="org.apache.catalina.ant.StartTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="stop"      classname="org.apache.catalina.ant.StopTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+    <taskdef name="undeploy"  classname="org.apache.catalina.ant.UndeployTask"
+      classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  </target>
+
+
+<!-- ======================= Mysql Targets =========================== -->
+
+  <!-- this is one target that we only want to run once. -->
+  <target name="prepare-mysql" depends="init,init-mysql-vars,install-mysql,start-mysql,setup-mysql,stop-mysql-no-password-force"/>
+
+  <target name="init-mysql-vars">
+    <condition property="mysql.islocal.linux">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.isunixnotmac"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.mac">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.ismac"/>
+      </and>
+    </condition>	   
+    <condition property="mysql.islocal.windows">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="current.os.iswindows"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.usepassword">
+      <and>
+	<isset property="mysql.islocal"/>
+	<isset property="mysql.use.passwords"/>
+	<equals arg1="yes" arg2="${mysql.use.passwords}"/>
+      </and>
+    </condition>
+    <condition property="mysql.islocal.usepassword.windows">
+      <and>
+	<isset property="mysql.islocal.usepassword"/>
+	<isset property="current.os.iswindows"/>
+      </and>
+    </condition>
+     <condition property="mysql.islocal.usepassword.unix">
+      <and>
+	<isset property="mysql.islocal.usepassword"/>
+	<isset property="current.os.isunix"/>
+      </and>
+    </condition>
+   <condition property="mysql.islocal.nopassword">
+      <and>
+	<isset property="mysql.islocal"/>
+	<or>
+	  <not>
+	    <isset property="mysql.use.passwords"/>
+	  </not>
+	  <equals arg1="no" arg2="${mysql.use.passwords}"/>
+	</or>
+      </and>
+    </condition>
+   </target>
+ 
+  <target name="configure-mysql" depends="init,init-mysql-vars,start-mysql,set-mysql-account-passwords,stop-mysql" description="sets up the mysql account passwords"/>
+  
+  <target name="get-mysql-root-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.root.password">
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.root.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify a password for the root mysql user: this is to secure your database."  username="root" pwordproperty="mysql.root.password"/>
+  </target>
+  
+  <target name="get-mysql-root-password-windows" depends="init,init-mysql-vars" if="mysql.islocal.usepassword.windows" unless="mysql.root.password">
+    <echo>(Enter password in the Java popup)</echo>
+    <getuserandpassword message="Please enter the mysql root password"  username="root" pwordproperty="mysql.root.password"/>
+  </target>
+  
+  <target name="get-mysql-reader-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.reader.password" >
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.reader.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify the password for the gsdl3reader mysql user: this is used by greenstone" username="gsdl3reader" pwordproperty="mysql.reader.password"/>
+  </target>
+  
+  <target name="get-mysql-admin-password" depends="init,init-mysql-vars" if="mysql.islocal.usepassword" unless="mysql.admin.password" >
+    <echo>Enter password in the Java popup. (If you do not have a window server running, please set the mysql.admin.password in the build.properties file.)</echo>
+    <getuserandpassword message="Please specify the password for the gsdl3admin mysql user: this is used by greenstone" username="gsdl3admin" pwordproperty="mysql.admin.password"/>
+  </target>
+
+ <target name="install-mysql" depends="init,init-mysql-vars,install-mysql-linux,install-mysql-windows,install-mysql-mac"/>
+  
+  <!-- install the database : linux-->
+  <target name="install-mysql-linux" depends="init,init-mysql-vars" if="mysql.islocal.linux" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      dest="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      usetimestamp="true"/>
+    <untar src="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"
+      dest="${packages.home}"
+      compression="gzip"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686"/>
+    </move>
+    <!-- file permissions are screwed up, so make executables executable -->
+    <chmod file="${mysql.home}/scripts/mysql_install_db" perm="a+x"/>
+    <chmod perm="a+x">
+      <fileset dir="${mysql.home}/bin" />
+    </chmod>
+    <mkdir dir="${mysql.home}/var/log/"/>
+    <chmod file="${mysql.home}/var/" perm="777" type="dir"/>
+    <chmod file="${mysql.home}/var/log/" perm="777" type="dir"/>
+    
+    <exec executable="${mysql.home}/scripts/mysql_install_db" dir="${mysql.home}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <arg value="--user=root"/>
+      <arg value="--force"/>
+    </exec>
+  </target>
+
+  <!-- install the database : mac-->
+  <target name="install-mysql-mac" depends="init,init-mysql-vars" if="mysql.islocal.mac" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      dest="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      usetimestamp="true"/>
+    <untar src="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"
+      dest="${packages.home}"
+      compression="gzip"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc"/>
+    </move>    
+    <!-- file permissions are screwed up, so make executables executable -->
+    <chmod file="${mysql.home}/scripts/mysql_install_db" perm="a+x"/>
+    <chmod perm="a+x">
+      <fileset dir="${mysql.home}/bin" />
+    </chmod>
+    <mkdir dir="${mysql.home}/var/log/"/>
+    <chmod file="${mysql.home}/var/" perm="777" type="dir"/>
+    <chmod file="${mysql.home}/var/log/" perm="777" type="dir"/>
+    <exec executable="${mysql.home}/scripts/mysql_install_db" dir="${mysql.home}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <!--<arg value="- -user=root"/>
+      <arg value="- -force"/>-->
+    </exec>
+  </target>
+
+  <!-- install the database : windows -->
+  <target name="install-mysql-windows" depends="init,init-mysql-vars" if="mysql.islocal.windows" unless="mysql.present">
+    <get src="http://www.greenstone.org/gs3files/mysql-noinstall-4.1.15-win32.zip"
+      dest="${packages.home}/mysql-noinstall-4.1.15-win32.zip"
+      usetimestamp="true"/>
+    <!-- can we run the installer?? -->
+    <unzip src="${packages.home}/mysql-noinstall-4.1.15-win32.zip"
+      dest="${packages.home}"/>
+    <move todir="${packages.home}/mysql">
+      <fileset dir="${packages.home}/mysql-4.1.15-win32"/>
+    </move>
+  </target>
+
+  <target name="setup-mysql" depends="init" if="mysql.islocal">
+    <!-- setup the gsdl3 accounts -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=GRANT SELECT,INSERT,DELETE,UPDATE,DROP,CREATE ON *.* TO gsdl3admin@localhost"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=GRANT SELECT ON *.* TO gsdl3reader@localhost"/>
+    </exec>
+    <!--<exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="- -user=root"/>
+      <arg value="- -execute=GRANT SELECT,INSERT,DELETE,UPDATE,DROP,CREATE ON *.* TO gsdl3admin@'%';"/>
+    </exec>-->
+
+    <!-- remove the anonymous user -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=delete from mysql.user where Host='localhost' and User='';"/>
+    </exec>
+    <!-- load in the demo collection database -->
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=create database localsite_gs3mgdemo;"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}"
+      input="${web.home}/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql">
+      <arg value="--user=root"/>
+      <arg value="localsite_gs3mgdemo"/>
+    </exec>
+  </target>
+
+  <target name="set-mysql-account-passwords" depends="init,init-mysql-vars,get-mysql-root-password,get-mysql-reader-password,get-mysql-admin-password" if="mysql.islocal.usepassword">
+    <exec executable="${mysql.home}/bin/mysql" dir="${mysql.home}">
+      <arg value="--user=root"/>
+      <arg value="--execute=set password for 'root'@'localhost' = PASSWORD('${mysql.root.password}'); set password for 'gsdl3reader'@'localhost' = PASSWORD('${mysql.reader.password}'); set password for 'gsdl3admin'@'localhost' = PASSWORD('${mysql.admin.password}');  flush privileges;"/>
+    </exec>    
+  </target>
+
+  <target name="start-mysql" depends="init,init-mysql-vars" if="mysql.islocal"
+    description="Startup only mysql">
+    <echo>Starting up the mysql server</echo>
+    <exec executable="${mysql.home}/bin/mysqld_safe" dir="${mysql.home}"
+      spawn="true" os="${os.linux},${os.mac}">
+      <arg value="--datadir=${mysql.datadir}"/>
+      <arg value="--basedir=."/>
+      <arg value="--pid_file=gsdl3.pid"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="--port=${mysql.port}"/>
+      <arg value="--err-log=./var/log/mysql.log"/>
+    </exec>
+   <exec executable="${mysql.home}/bin/mysqld" dir="${mysql.home}" spawn="true" os="${os.windows}">
+      <arg value="--port=${mysql.port}"/>   
+    </exec>
+    <sleep seconds="2"/>
+  </target>
+
+  <target name="stop-mysql" depends="init,init-mysql-vars,stop-mysql-use-password-unix,stop-mysql-use-password-windows,stop-mysql-no-password" if="mysql.islocal"
+    description="Shutdown only mysql">
+    <echo>MYSQL database server shutdown successfully</echo>
+  </target>
+  
+  <target name="stop-mysql-no-password" depends="init" if="mysql.islocal.nopassword">
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-no-password-force" depends="init">
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-use-password-unix" depends="init" if="mysql.islocal.usepassword.unix">
+    <echo>At the password prompt, enter the mysql root password.</echo>
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.linux},${os.mac}">
+      <arg value="--user=root"/>
+      <arg value="-p"/>
+      <arg value="--socket=/tmp/mysql.sock"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+  
+  <target name="stop-mysql-use-password-windows" depends="init,get-mysql-root-password-windows" if="mysql.islocal.usepassword.windows">
+    <!-- on windows it doesn't work unless the password is specified here -->
+    <exec executable="${mysql.home}/bin/mysqladmin" dir="${mysql.home}"
+      failonerror="true" os="${os.windows}">
+      <arg value="--user=root"/>
+      <arg value="-p${mysql.root.password}"/>
+      <arg value="shutdown"/>
+    </exec>
+  </target>
+
+  <target name="restart-mysql" description="Shutdown and restart only mysql" depends="init,stop-mysql,start-mysql"/>
+
+
+<!-- ======================= Axis Targets ============================ --> 
+
+  <target name="prepare-axis" depends="init">
+    <get src="http://www.greenstone.org/gs3files/axis-bin-1_2_1.zip"
+      dest="${packages.home}/axis-bin-1_2_1.zip"
+      usetimestamp="true"/>
+    <unzip src="${packages.home}/axis-bin-1_2_1.zip"
+      dest="${packages.home}"/>
+    <move todir="${packages.home}/axis">
+      <fileset dir="${packages.home}/axis-1_2_1"/>
+    </move>
+    <!-- install axis into greenstone web app -->
+    <copy todir="${web.lib}">
+      <fileset dir="${packages.home}/axis/webapps/axis/WEB-INF/lib">
+	<include name="*.jar"/>
+      </fileset>
+    </copy>
+    <copy todir="${web.home}">
+      <fileset dir="${packages.home}/axis/webapps/axis/">
+	<include name="*.jsp"/>
+	<include name="*.jws"/>
+      </fileset>
+    </copy>
+    <copy tofile="${web.home}/axis.html" file="${packages.home}/axis/webapps/axis/index.html"/>
+    <copy todir="${web.classes}">
+      <fileset dir="${packages.home}/axis/webapps/axis/WEB-INF/classes">
+	<include name="*.properties"/>
+      </fileset>
+    </copy>
+  </target>
+
+  <target name="soap-deploy-site" depends="get-sitename,get-siteuri,create-deployment-files"
+    description="Deploy a SOAP web service for a local Greenstone site">
+    <java classname="org.apache.axis.client.AdminClient">
+      <classpath refid="compile.classpath"/>
+      <arg value="-l"/>
+      <arg value="http://${tomcat.server}:${tomcat.port}${app.path}/servlet/AxisServlet"/>
+      <arg file="${basedir}/resources/soap/${axis.sitename}.wsdd"/>
+    </java>    
+  </target>
+  
+  <target name="soap-undeploy-site" depends="get-sitename"
+    description="Undeploy a SOAP web service for a local Greenstone site">
+    <java classname="org.apache.axis.client.AdminClient">
+      <classpath refid="compile.classpath"/>
+      <arg value="-l"/>
+      <arg value="http://${tomcat.server}:${tomcat.port}${app.path}/servlet/AxisServlet"/>
+      <arg file="${basedir}/resources/soap/undeploy-${axis.sitename}.wsdd"/>
+    </java>
+  </target>
+
+  <!-- this target used to deploy the localsite server for the default installation -->
+  <target name="deploy-localsite" depends="init">
+    <antcall target="start-tomcat"/>
+    <antcall target="soap-deploy-site">
+      <param name="axis.sitename" value="localsite"/>
+      <param name="axis.siteuri" value="localsite"/>
+    </antcall>
+    <antcall target="stop-tomcat"/>
+  </target>
+  
+  <target name="get-sitename" unless="axis.sitename">
+    <input addproperty="axis.sitename" message="What site? (press enter for default:localsite)" defaultvalue="localsite"/>
+  </target>
+
+  <target name="get-siteuri" depends="get-sitename" unless="axis.siteuri">
+    <input addproperty="axis.siteuri" message="What name do you want the service to have? (press enter for default:${axis.sitename})" defaultvalue="${axis.sitename}"/>
+    <echo>${axis.sitename}, ${axis.siteuri}</echo>
+  </target>
+  <target name="check-deployment-files" depends="get-sitename">
+    <condition property="deploy.exists">
+      <and>
+	<available file="${basedir}/resources/soap/${axis.sitename}.wsdd"/>
+	<available file="${basedir}/resources/soap/undeploy-${axis.sitename}.wsdd"/>
+	<available file="${web.classes}/org/greenstone/gsdl3/SOAPServer${axis.sitename}.class"/>
+      </and>
+    </condition>
+   </target> 
+
+  <target name="create-deployment-files" depends="get-sitename,get-siteuri,check-deployment-files" 
+    if="axis.sitename" unless="deploy.exists">
+    <filter token="sitename" value="${axis.sitename}"/>
+    <filter token="siteuri" value="${axis.siteuri}"/>
+    <copy file="${basedir}/resources/soap/site.wsdd.template"
+      tofile="${basedir}/resources/soap/${axis.sitename}.wsdd"
+      filtering="true"/>
+     <copy file="${basedir}/resources/soap/undeploy-site.wsdd.template"
+      tofile="${basedir}/resources/soap/undeploy-${axis.sitename}.wsdd"
+      filtering="true"/>
+    <!-- create the java files and compile them -->
+    <copy file="${gsdl3.home}/SOAPServer.java.in"
+      tofile="${gsdl3.home}/SOAPServer${axis.sitename}.java"
+      filtering="true"/>
+    <mkdir dir="${build.home}"/>
+    <javac srcdir="${src.home}"
+      destdir="${build.home}" 
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath refid="compile.classpath"/>
+      <include name="org/greenstone/gsdl3/SOAPServer${axis.sitename}.java" />
+    </javac>
+    <mkdir dir="${web.classes}/org/greenstone/gsdl3"/>
+    <copy file="${build.home}/org/greenstone/gsdl3/SOAPServer${axis.sitename}.class" tofile="${web.classes}/org/greenstone/gsdl3/SOAPServer${axis.sitename}.class" />
+  </target>
+    
+
+<!-- ====================== Core targets ============================== -->
+<!-- core targets refer to the core gsdl3 java src -->
+  
+  <target name="prepare-core" unless="nocvs.mode">
+    <!-- just get rid of empty directories-->
+    <cvs command="update -P"/>
+  </target>
+  
+  <target name="configure-core"/>
+
+  <target name="update-core" depends="init,cvsupdate-core,clean-core,compile-core"
+    description="Update only the Greenstone core" />
+  
+  <target name="cvsupdate-core" unless="nocvs.mode">
+    <cvs command="update -l"/>
+    <cvs command="update -dP bin comms dist-resources docs lib resources src winutil"/>
+  </target>
+
+  <target name="clean-core"
+    description="Clean only the Greenstone core">
+    <delete dir="${build.home}"/>
+  </target>
+
+   <target name="compile-core" 
+    description="Compile only the Greenstone core">
+    <mkdir dir="${build.home}"/>
+    <javac srcdir="${src.home}"
+      destdir="${build.home}"
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath>
+	<path refid="compile.classpath"/>
+      </classpath>
+    </javac>
+    <jar destfile="${build.home}/gsdl3.jar">
+      <fileset dir="${build.home}">
+	<include name="org/greenstone/gsdl3/**"/>
+	<exclude name="**/Test.class"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>
+    <copy file="${build.home}/gsdl3.jar" todir="${web.lib}"/>
+    <jar destfile="${build.home}/phind.jar">
+      <fileset dir="${build.home}">
+	<include name="org/greenstone/applet/phind/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>
+    <copy file="${build.home}/phind.jar" todir="${web.applet}"/>
+    <jar destfile="${build.home}/anttasks.jar">
+      <fileset dir="${build.home}">
+	<include name="org/greenstone/anttasks/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>
+    <copy file="${build.home}/anttasks.jar" todir="${basedir}/lib/java"/>
+    <jar destfile="${build.home}/gsdl3test.jar">
+      <fileset dir="${build.home}">
+	<include name="org/greenstone/gsdl3/**/*Test.class"/>
+	<include name="org/greenstone/testing/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>
+  </target>
+ 
+<!-- ================== Packages targets ================================ -->
+  <!-- these targets refer to the greenstone source packages - these need 
+  updating less often, so are in  separate targets to the core -->
+  <target name="prepare-packages" depends="init,prepare-mgpp,prepare-windows"/>
+  
+  <target name="prepare-windows" depends="init" if="current.os.iswindows">
+    <!-- need to download precompiled binaries for mg and mgpp -->
+  </target>
+
+  <target name="checkout-mgpp" depends="check-cvsroot,init" unless="nocvs.mode">
+    <cvs command="checkout -P" package="mgpp" dest="${src.packages.home}"/>
+  </target>
+  
+  <target name="prepare-mgpp" depends="init" unless="mgpp.present">
+    <antcall target="checkout-mgpp"/>
+  </target>
+
+  <target name="update-packages" depends="init,cvsupdate-packages,configure-packages,clean-packages,compile-packages"
+    description="Update only the source packages"/>
+
+  <target name="cvsupdate-packages" unless="nocvs.mode"> 
+    <cvs command="update -dP" dest="${src.packages.home}"/>
+  </target>
+  
+  <target name="configure-packages" description="Configure only the packages.">
+    <echo>
+      Configuring MG
+    </echo>
+    <exec executable="${mg.home}/configure" os="${os.linux},${os.mac}" 
+      dir="${mg.home}">
+      <arg value="--prefix=${basedir}"/>
+      <arg value="--libdir=${lib.jni}"/>
+    </exec>    
+    <echo>
+      Configuring MGPP
+    </echo>
+    <exec executable="${mgpp.home}/configure" os="${os.linux},${os.mac}"
+      dir="${mgpp.home}">
+      <arg value="--prefix=${basedir}"/>
+      <arg value="--libdir=${lib.jni}"/>
+    </exec>
+    <echo>
+      Configuring JavaGDBM
+    </echo>
+    <exec executable="${javagdbm.home}/configure" os="${os.linux}"
+      dir="${javagdbm.home}">
+      <arg value="--prefix=${basedir}"/>
+      <arg value="--libdir=${lib.jni}"/>
+    </exec>    
+    <exec executable="${javagdbm.home}/configure" os="${os.mac}"
+      dir="${javagdbm.home}">
+      <arg value="--prefix=${basedir}"/>
+      <arg value="--libdir=${lib.jni}"/>
+     <arg value="--with-gdbm=${gdbm.installed.path}"/>
+    </exec>    
+
+  </target>
+
+  <target name="clean-packages" depends="init,clean-packages-c++" description="Clean only the packages"/>
+
+  <target name="clean-packages-c++" depends="init">
+    <!-- mg : just call the make target -->
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${mg.home}">
+      <arg value="clean"/>
+    </exec>
+    <!-- mgpp -->
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${mgpp.home}">
+      <arg value="clean"/>
+    </exec>
+    <!-- javagdbm -->
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${javagdbm.home}">
+      <arg value="clean"/>
+    </exec>
+  </target>
+ 
+  <target name="clean-windows-c++-packages" depends="init" if="current.os.iswindows"
+    description="Clean only the C/C++ packages">
+    <exec executable="${mg.home}/winMake.bat" dir="${mg.home}">
+      <arg value="clean"/>
+    </exec>
+    <exec executable="${mgpp.home}/winMake.bat" dir="${mgpp.home}">
+      <arg value="clean"/>
+    </exec>
+  </target>   
+
+  <target name="compile-packages"
+    description="Compile only the packages">
+    <echo>Compiling MG</echo>
+    <exec executable="make" os="${os.linux},${os.mac}"
+       dir="${mg.home}">
+    </exec>
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${mg.home}">
+      <arg value="install"/>
+    </exec>
+    <!-- windows: just the java stuff. -->
+    <exec executable="${mg.home}/winMake.bat" os="${os.windows}"
+      dir="${mg.home}">
+      <arg value="compile"/>
+      <arg value="javaonly"/>
+    </exec>
+    <!-- install the jar file -->
+    <copy file="${mg.home}/mg.jar" todir="${lib.jni}"/>
+    
+    <!-- mgpp -->
+    <echo>Compiling MGPP</echo>
+    <exec executable="make" os="${os.linux},${os.mac}"
+       dir="${mgpp.home}">
+    </exec>
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${mgpp.home}">
+      <arg value="install"/>
+    </exec>
+    <!-- windows: just the java stuff. -->
+    <exec executable="${mgpp.home}/winMake.bat" os="${os.windows}"
+      dir="${mgpp.home}">
+      <arg value="compile"/>
+      <arg value="javaonly"/>
+    </exec>
+    <!-- install the jar file -->
+    <copy file="${mgpp.home}/mgpp.jar" todir="${lib.jni}"/>
+    
+    <!-- javagdbm -->
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${javagdbm.home}">
+    </exec>
+    <exec executable="make" os="${os.linux},${os.mac}"
+      dir="${javagdbm.home}">
+      <arg value="install"/>
+    </exec>
+    <!-- windows: just the java stuff. -->
+    <exec executable="${javagdbm.home}/winMake.bat" os="${os.windows}"
+      dir="${javagdbm.home}">
+      <arg value="compile"/>
+      <arg value="javaonly"/>
+    </exec>
+    <!-- install the jar file -->
+    <copy file="${javagdbm.home}/javagdbm.jar" todir="${lib.jni}"/>
+
+  </target>
+
+  <target name="compile-windows-c++-packages" depends="init" if="current.os.iswindows"
+    description="Compile only the C/C++ packages">
+    <exec executable="${mg.home}/winMake.bat" dir="${mg.home}"/>
+    <exec executable="${mg.home}/winMake.bat" dir="${mg.home}">
+      <arg value="install"/>
+      <env key="GSDL3HOME" value="${basedir}"/>
+    </exec>
+    <exec executable="${mgpp.home}/winMake.bat" dir="${mgpp.home}"/>
+    <exec executable="${mgpp.home}/winMake.bat" dir="${mgpp.home}">
+      <arg value="install"/>
+      <env key="GSDL3HOME" value="${basedir}"/>
+    </exec>
+    <exec executable="${javagdbm.home}/winMake.bat" dir="${javagdbm.home}"/>
+    <exec executable="${javagdbm.home}/winMake.bat" dir="${javagdbm.home}">
+      <arg value="install"/>
+      <env key="GSDL3HOME" value="${basedir}"/>
+    </exec>
+  </target>   
+  
+  <!-- =============== Extensions targets ==================== -->
+  
+  <target name="prepare-extensions" depends="init"/>
+
+  <target name="cvsupdate-extensions" depends="init">
+    <cvs command="update -dP" dest="${extensions.home}"/>
+  </target>
+
+  <target name="configure-extensions"/>
+  
+  <target name="clean-extensions" depends="init">
+    <!-- vishnu : just call the ant target -->
+    <ant antfile="${vishnu.home}/build.xml" dir="${vishnu.home}"
+      inheritall="false" target="clean">
+      <property name="catalina.home" value="${catalina.home}"/>
+    </ant>
+    <!-- gsdl-as : just call the ant target-->
+    <ant antfile="${gsdl-as.home}/build.xml" dir="${gsdl-as.home}"
+      inheritall="false" target="clean">
+      <property name="catalina.home" value="${catalina.home}"/>
+    </ant>
+  </target>
+
+  <target name="compile-extensions">
+    <!-- vishnu -->
+    <echo>Compiling Vishnu</echo>
+    <ant antfile="${vishnu.home}/build.xml" dir="${vishnu.home}"
+      inheritall="false" target="compile">
+      <property name="catalina.home" value="${catalina.home}"/>
+    </ant>
+
+    <!-- gsdl-as -->
+    <echo>Compiling GSDL-AS</echo>
+    <ant antfile="${gsdl-as.home}/build.xml" dir="${gsdl-as.home}"
+      inheritall="false" target="compile">
+      <property name="catalina.home" value="${catalina.home}"/>
+    </ant>
+  </target>
+
+  <!-- ================== gs2building targets ===============-->
+
+
+  <target name="update-gs2building" if="gsdl2.islocal" 
+    depends="init,cvsupdate-gs2building,rename-gs2build-files,configure-gs2building,clean-gs2building,compile-gs2building"
+    description="Update only the Greenstone 2 building components"/>
+
+  <target name="cvsupdate-gs2building" if="gsdl2.islocal" depends="init" unless="nocvs.mode">
+    <cvs command="update -dP" dest="${gli.home}"/>
+    <!-- Note: can't do a -d update here cos it will get all of gsdl 
+     also, an update doesn't get new files, so we do a checkout instead. -->
+    <cvs command="co gs2build" dest="${basedir}"/>
+  </target>
+
+  <target name="prepare-gs2building" depends="init,prepare-gs2build,prepare-gli" if="gsdl2.islocal">
+  </target>
+
+  <target name="checkout-gs2build" depends="check-cvsroot,init" unless="nocvs.mode">
+    <echo>checking out gs2build</echo>
+    <cvs command="checkout -P" package="gs2build"/>
+  </target>
+
+  <target name="prepare-gs2build" depends="init" if="gsdl2.islocal" unless="gs2build.present">
+    <antcall target="checkout-gs2build"/>
+    <!-- rename the .gs2build files -->
+    <antcall target="rename-gs2build-files"/>
+    <antcall target="unzip-windows-packages"/>
+    <antcall target="get-windows-binaries"/>
+    <antcall target="get-macos-extra"/>
+  </target>
+
+  <target name="checkout-winbin" depends="init" if="current.os.iswindows" 
+    unless="nocvs.mode">
+    <cvs command="checkout -P" package="winbin"/>
+  </target>
+  
+  <target name="get-windows-binaries" depends="init,checkout-winbin" if="current.os.iswindows">
+    <copy todir="${basedir}/bin">
+      <fileset dir="${basedir}/winbin/bin" includes="mg*.exe"/>
+    </copy>
+     <move todir="${gs2build.home}/bin/windows" failonerror="false">
+      <fileset dir="${basedir}/winbin/bin"/>
+    </move>
+    <delete dir="${basedir}/winbin"/>
+   </target>
+  
+  <target name="unzip-windows-packages" depends="init" if="current.os.iswindows">  
+    <unzip src="${gs2build.home}/packages/windows/gdbm/gdbm.zip"
+    dest="${gs2build.home}/packages/windows/gdbm"/>
+   <unzip src="${gs2build.home}/packages/windows/crypt/crypt.zip"
+    dest="${gs2build.home}/packages/windows/crypt"/>
+       <unzip src="${gs2build.home}/packages/windows/expat/expat.zip"
+    dest="${gs2build.home}/packages/windows/expat"/>
+   </target>
+  
+  <!-- downloads a good XML-Parser -->
+  <target name="get-macos-extra" depends="init" if="need.macos.extra">	
+      <get src="http://www.greenstone.org/gs3files/XML-Parser.tar.gz"
+      dest="${gs2build.home}/perllib/cpan/XML-Parser.tar.gz"
+      usetimestamp="true"/>
+  </target>  
+
+  <!-- untars the XML-Parser. need to do this after compiling in gs2build-->
+  <target name="install-macos-extra" depends="init" if="need.macos.extra">
+    <!-- make sure these directories are present, otherwise chmod craps out 
+    this chmod is needed in case we are unpacking for a second time -->
+    <mkdir dir="${gs2build.home}/perllib/cpan/perl-5.8"/>
+    <mkdir dir="${gs2build.home}/perllib/cpan/perl-5.6"/>
+    <chmod dir="${gs2build.home}/perllib/cpan/perl-5.8" perm="ug+w" includes="**"/>
+    <chmod dir="${gs2build.home}/perllib/cpan/perl-5.6" perm="ug+w" includes="**"/>
+    <untar src="${gs2build.home}/perllib/cpan/XML-Parser.tar.gz"
+      dest="${gs2build.home}/perllib/cpan/"
+      compression="gzip"/>
+  </target>
+
+  <target name="rename-gs2build-files" depends="rename-gs2build-files-unix,rename-gs2build-files-windows"/>
+
+  <target name="rename-gs2build-files-windows" if="current.os.iswindows">
+    <!-- we want a windows path in the setup.bat file -->
+    <pathconvert targetos="windows" property="gs2build.home.windows">
+      <path path="${gs2build.home}"/>
+    </pathconvert>
+
+    <property name="gs2build-extra.home" value="${gs2build.home}/gs2build-extra"/>
+   <copy file="${gs2build-extra.home}/lib.win32.mak" tofile="${gs2build.home}/lib/win32.mak"/>
+   <copy file="${gs2build-extra.home}/win32.mak" tofile="${gs2build.home}/win32.mak"/>
+   <copy file="${gs2build-extra.home}/setup.bat" tofile="${gs2build.home}/setup.bat">
+     <filterset>
+       <filter token="gsdlhome" value="${gs2build.home.windows}"/>
+     </filterset>
+    </copy>  
+  </target> 
+  <target name="rename-gs2build-files-unix" if="current.os.isunix">
+    <property name="gs2build-extra.home" value="${gs2build.home}/gs2build-extra"/>
+    <copy file="${gs2build-extra.home}/configure" tofile="${gs2build.home}/configure"/>    
+    <chmod file="${gs2build.home}/configure" perm="a+x"/>
+    <copy file="${gs2build-extra.home}/configure.in" tofile="${gs2build.home}/configure.in"/>
+    <copy file="${gs2build-extra.home}/Makefile.in" tofile="${gs2build.home}/Makefile.in"/>  
+    <copy file="${gs2build-extra.home}/packages.configure" tofile="${gs2build.home}/packages/configure"/>
+    <chmod file="${gs2build.home}/packages/configure" perm="a+x"/>
+    <copy file="${gs2build-extra.home}/packages.Makefile" tofile="${gs2build.home}/packages/Makefile"/>
+    <copy file="${gs2build-extra.home}/lib.Makefile.in" tofile="${gs2build.home}/lib/Makefile.in"/>  
+    
+  </target>
+  <target name="prepare-gli" depends="init" if="gsdl2.islocal" unless="gli.present">
+    <antcall target="checkout-gli"/>
+  </target>
+  
+  <target name="checkout-gli" depends="check-cvsroot,init" if="gsdl2.islocal" unless="nocvs.mode">
+    <echo>checking out gli</echo>
+    <cvs command="checkout -P" package="gli"/>
+  </target>
+    
+  <target name="configure-gs2building" depends="init" if="gsdl2.islocal"
+    description="Configure only the Greenstone 2 building components">
+    <exec executable="${gs2build.home}/configure" os="${os.linux}"
+      dir="${gs2build.home}">
+      <arg value="--prefix=${gs2build.home}"/>
+    </exec>
+    <exec executable="${gs2build.home}/configure" os="${os.mac}"
+      dir="${gs2build.home}">
+      <arg value="--prefix=${gs2build.home}"/>
+      <arg value="--with-gdbm=${gdbm.installed.path}"/>
+    </exec>
+  </target>
+
+  <target name="clean-gs2building" depends="init,clean-gli,clean-gs2build" if="gsdl2.islocal"
+    description="Clean only the Greenstone 2 building components"/>
+  <target name="clean-gli" depends="init">
+    <!-- gli -->
+    <property name="gli.home" value="${basedir}/gli"/>
+    <!-- linux -->
+    <exec executable="clean.sh" os="${os.linux},${os.mac}" dir="${gli.home}"
+      resolveExecutable="true"/>
+    <!-- windows -->
+    <exec executable="clean.bat" os="${os.windows}" dir="${gli.home}"
+      resolveExecutable="true"/>
+  </target>
+  <target name="clean-gs2build" depends="init">
+    <!-- gs2build -->
+    <!--linux:  -->
+    <exec executable="make" os="${os.linux},${os.mac}" dir="${gs2build.home}">
+      <arg value="clean"/>
+    </exec>
+    <!-- windows: -->    
+  </target>
+  
+  <target name="clean-windows-c++-gs2building" depends="init" if="current.os.iswindows"
+    description="remove all object files and executables">
+    <!-- run the setup script -->
+    <exec executable="${compile.windows.c++.setup}"/>
+    <exec executable="nmake" dir="${gs2build.home}">
+     <arg value="/f"/>
+     <arg value="win32.mak"/>
+     <arg value="clean"/>
+    </exec>
+  </target>  
+
+  <target name="compile-gs2building" depends="init" if="gsdl2.islocal"
+    description="Compile only the Greenstone 2 building components">
+    <!-- gli -->
+    <property name="gli.home" value="${basedir}/gli"/>
+    <!-- linux -->
+    <exec executable="makegli.sh" os="${os.linux},${os.mac}" dir="${gli.home}"
+      resolveExecutable="true"/>
+    <!-- windows -->
+    <exec executable="makegli.bat" os="${os.windows}" dir="${gli.home}"
+      resolveExecutable="true"/>
+    <!-- gs2build -->
+    <!--linux: make, make install -->
+    <exec executable="make" os="${os.linux},${os.mac}" dir="${gs2build.home}">
+    </exec>
+    <exec executable="make" os="${os.linux},${os.mac}" dir="${gs2build.home}">
+      <arg value="install"/>
+    </exec>
+    <antcall target="install-macos-extra"/>
+  </target>
+    <!-- windows: -->
+   <target name="compile-windows-c++-gs2building" depends="init" if="current.os.iswindows"
+    description="Use this if you want to compile the C++ code for the gs2build package">
+    <!-- run the setup script -->
+    <exec executable="${compile.windows.c++.setup}"/>
+    <exec executable="nmake" dir="${gs2build.home}">
+     <arg value="/f"/>
+     <arg value="win32.mak"/>
+    </exec>
+  </target>
+  <target name="gli" description="Run the Greenstone Librarian Interface" depends="gli-local,gli-external">
+  </target>
+
+  <target name="gli-local" depends="init" if="gsdl2.islocal">
+    <exec executable="${basedir}/gli/gli4gs3.sh" os="${os.linux}" dir="${basedir}/gli" spawn="true">
+      <env key="gsdl3path" path="${basedir}"/>
+      <env key="gsdlpath" path="${basedir}/gs2build"/>
+    </exec>
+    <exec executable="${basedir}/gli/gli4gs3.sh" os="${os.mac}" dir="${basedir}/gli" spawn="true">
+      <env key="gsdl3path" path="${basedir}"/>
+      <env key="gsdlpath" path="${basedir}/gs2build"/>
+      <env key="DYLD_LIBRARY_PATH" path="${env.DYLD_LIBRARY_PATH}:${gdbm.installed.path}/lib"/>
+    </exec>
+    <exec executable="${basedir}/gli/gli4gs3.bat" os="${os.windows}" dir="${basedir}/gli" spawn="true">
+      <env key="GSDL3PATH" path="${basedir}"/>
+      <env key="GSDLPATH" path="${basedir}/gs2build"/>
+    </exec>
+    <echo>Running GLI from Ant means that you don't get to see any of the terminal output. If you have problems with GLI and want to see the output, please run the script gli4gs3.sh/bat from the greenstone3/gli directory.
+    </echo>
+  </target>
+  
+  <target name="gli-external" depends="init" unless="gsdl2.islocal">
+    <exec executable="${gsdl2.installed.path}/gli/gli4gs3.sh" os="${os.linux},${os.mac}" dir="${gsdl2.installed.path}/gli" spawn="true">
+      <env key="gsdl3path" path="${basedir}"/>
+      <env key="gsdlpath"  path="${gsdl2.installed.path}"/>
+    </exec>
+    <exec executable="${gsdl2.installed.path}/gli/gli4gs3.bat" os="${os.windows}" dir="${gsdl2.installed.path}/gli" spawn="true">
+      <env key="GSDL3PATH" path="${basedir}"/>
+      <env key="GSDLPATH" path="${gsdl2.installed.path}"/>
+    </exec>
+    <echo>Running GLI from Ant means that you don't get to see any of the terminal output. If you have problems with GLI and want to see the output, please run the following in a terminal/command prompt: 
+                          (Linux/Mac OS X     / Windows)
+in greenstone3 directory: source gs3-setup.sh / gs3-setup
+in gsdl directory:        source setup.bash   / setup
+in gli directory:         gli4gs3.sh          / gli4gs3
+    </echo>
+  </target>
+
+
+<!-- ======================== TESTING Targets ========================= -->
+  
+  <target name="test" description="Run the (incomplete) JUnit test suite "
+    depends="init">
+    <mkdir dir="${basedir}/test"/>
+    <junit printsummary="withOutAndErr"
+      errorproperty="test.failed"
+      failureproperty="test.failed"
+      fork="${junit.fork}">
+      <formatter type="plain"/>
+      <classpath>
+	<pathelement location="${build.home}/gsdl3test.jar"/>
+	<path refid="compile.classpath"/>
+      </classpath>
+      <test name="${testcase}" if="testcase"/>
+      <batchtest todir="${basedir}/test" unless="testcase">
+	<fileset dir="${build.home}"
+	  includes="**/*Test.class"
+	  />
+      </batchtest>
+    </junit>
+    <echo>
+      *********************************************
+       Test output can be found in directory 'test'
+      *********************************************
+    </echo>
+  </target>
+
+
+<!-- ====================== DISTRIBUTION Targets ========================= -->
+
+  <target name="prepare-for-dist-1" depends="init">
+    <cvs command="export -D '1 second ago'" package="mgpp" 
+      dest="${src.packages.home}"
+      cvsRoot="${cvs.root}" />
+    <cvs command="export -D '1 second ago'" package="gs2build"
+      cvsRoot="${cvs.root}" />
+    <cvs command="export -D '1 second ago'" package="gli"
+      cvsRoot="${cvs.root}" />
+    <delete file="${gli.home}/gli.bat"/>
+    <delete file="${gli.home}/gli.sh"/>
+    <antcall target="rename-gs2build-files-unix"/>
+    <antcall target="unzip-windows-packages"/>
+    <antcall target="get-windows-binaries"/>
+    <antcall target="get-perl-for-windows"/>
+    <antcall target="get-macos-extra"/>
+    <antcall target="delete-windows-files"/>
+    <antcall target="prepare-tomcat"/>
+    <delete file="${packages.home}/apache-tomcat-5.5.12.zip"/>
+    <delete file="${packages.home}/apache-tomcat-5.5.12-compat.zip"/>
+    <antcall target="configure-tomcat"/>
+    <antcall target="prepare-axis"/>
+    <delete file="${packages.home}/axis-bin-1_2_1.zip"/>
+    <!-- delete old axis dir -->
+    <delete dir="${packages.home}/axis"/>
+    <antcall target="prepare-web"/>
+  </target>
+  
+  <target name="prepare-for-dist-2" depends="init">
+    <antcall target="prepare-mysql"/>
+    <delete file="${packages.home}/mysql-standard-4.1.15-pc-linux-gnu-i686.tar.gz"/>
+    <delete file="${packages.home}/mysql-standard-4.1.15-apple-darwin7.9.0-powerpc.tar.gz"/>
+    <delete file="${packages.home}/mysql-noinstall-4.1.15-win32.zip"/>
+    <delete dir="${packages.home}/mysql/sql-bench"/>
+    <delete dir="${packages.home}/mysql/tests"/>
+    <delete dir="${packages.home}/mysql/mysql-test"/>
+    <delete file="${packages.home}/mysql/var/log/mysql.log"/>
+  </target>
+  
+  <target name="prepare-for-dist-3" depends="init">
+    <antcall target="configure-c++"/>
+  </target>
+  
+  <target name="prepare-for-dist-4" depends="init">
+    <antcall target="compile"/>
+    <antcall target="clean-packages-c++"/>
+    <antcall target="clean-gs2build"/>
+    <antcall target="install-macos-extra"/>
+    <xslt in="build.xml" out="dist-build.xml" style="dist-resources/convert-build-xml.xsl"/>
+    <delete dir="${basedir}/build"/>
+    <delete dir="${basedir}/dist-resources"/>
+  </target>
+  
+  <target name="get-perl-for-windows" if="current.os.iswindows">
+    <get src="http://www.greenstone.org/gs3files/perl-for-windows.zip"
+      dest="${gs2build.home}/bin/windows/perl-for-windows.zip"
+      usetimestamp="true"/>
+    <unzip src="${gs2build.home}/bin/windows/perl-for-windows.zip" 
+      dest="${gs2build.home}/bin/windows"/>
+    <delete file="${gs2build.home}/bin/windows/perl-for-windows.zip"/>
+  </target>
+
+  <target name="delete-windows-files" depends="init" unless="current.os.iswindows">
+    <delete dir="${basedir}/winutil"/>
+    <delete>
+      <fileset dir="${lib.jni}" includes="*.dll"/>
+    </delete>
+  </target>
+
+  <target name="install-for-dist" depends="accept-properties,init,configure-mysql,configure,configure-tomcat">
+    <condition property="need.xml.jars">
+      <equals arg1="1.5" arg2="${ant.java.version}"/>
+    </condition>
+    <antcall target="rename-xml-jars"/>
+    <antcall target="rename-gs2build-files-windows"/>
+    <antcall target="deploy-localsite"/> 
+  </target>
+</project>
+
+
Index: /main/tags/3.01/gsdl3/dist-resources/convert-build-xml.xsl
===================================================================
--- /main/tags/3.01/gsdl3/dist-resources/convert-build-xml.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/dist-resources/convert-build-xml.xsl	(revision 21112)
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+ <!-- we basically want to copy everything except for a few targets that no longer apply -->
+  <xsl:output method="xml"/>
+
+  <!-- for some reason, the default copy-of doesn't copy empty attributes, so have this to get the empty arg1 in equals-->
+  <xsl:template match="project|condition|and|isset|equals|not|or">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <!--<xsl:for-each select="@*">
+	<xsl:attribute name="name()"><xsl:value-of select="."/></xsl:attribute>
+      </xsl:for-each>-->
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+  
+  <!-- rename the 'install-for-dist' target 'install' -->
+  <xsl:template match="target[@name='install-for-dist']">
+    <target name="install">
+      <xsl:copy-of select="@depends|@description|@if|@unless|node()"/>
+    </target>
+  </xsl:template>
+
+  <!-- delete the old install target-->
+  <xsl:template match="target[@name='install']">
+  </xsl:template>
+
+  <!-- delete the unnecessary targets -->
+  <xsl:template match="target[starts-with(@name, 'prepare')]">
+  </xsl:template>
+  <xsl:template match="target[starts-with(@name, 'cvsupdate')]">
+  </xsl:template>
+  <xsl:template match="target[starts-with(@name, 'update')]">
+  </xsl:template>
+
+  <xsl:template match="property[@name='install-command']">
+    <property name="install-command" value="ant [options] install"/>
+  </xsl:template>
+  
+  <xsl:template match="*">
+    <xsl:copy-of select="."/>
+  </xsl:template>
+
+  <xsl:template match="comment()">
+    <xsl:copy-of select="."/>
+    <xsl:text>
+</xsl:text>
+  </xsl:template>
+    
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/dist-resources/tomcat-server.xsl
===================================================================
--- /main/tags/3.01/gsdl3/dist-resources/tomcat-server.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/dist-resources/tomcat-server.xsl	(revision 21112)
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+ <!-- we basically want to copy everything, but add in a few bits for gs3 -->
+  <xsl:output method="xml"/>
+  
+  <xsl:template match="Server">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+    </xsl:copy>
+    <xsl:text>
+    </xsl:text>
+  </xsl:template>
+  
+  <xsl:template match="Service | Engine ">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+  
+  <xsl:template match="Connector[@port='8080']">
+    <xsl:copy>
+    <xsl:for-each select="@*">
+      <xsl:choose>
+	<xsl:when test="name()='port'">
+	  <xsl:attribute name="port">@port@</xsl:attribute>
+	</xsl:when>
+	<xsl:otherwise>
+	  <xsl:copy-of select="."/>
+	</xsl:otherwise>
+      </xsl:choose>
+    </xsl:for-each>
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+	  
+  <xsl:template match="Host">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+      <Context path="/gsdl3" docBase="@gsdl3home@/web" debug="1" 
+	reloadable="true">
+	<Resources allowLinking='true'/>
+      </Context>
+    </xsl:copy>
+  </xsl:template>
+      
+  <xsl:template match="*">
+    <xsl:copy-of select="."/>
+  </xsl:template>
+
+  <xsl:template match="comment()">
+    <xsl:copy-of select="."/>
+    <xsl:text>
+</xsl:text>
+  </xsl:template>
+    
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/docs/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/docs/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/docs/Makefile.in	(revision 21112)
@@ -0,0 +1,42 @@
+###########################################################################
+#
+# Makefile -- Makefile for documentation included with Greenstone
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+GSDL3HOME=..
+
+JAVADOC=$(JAVA_HOME)/bin/javadoc
+JAVADOCDIR=$(GSDL3HOME)/docs/javadoc
+SRCDIR=$(GSDL3HOME)/src/java
+PACKAGEDIR=$(GSDL3HOME)/packages
+
+docs:
+	$(JAVADOC) -d $(JAVADOCDIR) -private -link http://java.sun.com/j2se/1.4.2/docs/api/ `find $(SRCDIR) -name '*.java'` `find $(PACKAGEDIR) -name '*.java'` > $(JAVADOCDIR)/javadoc_log
+
+# what about the CVS dir??
+clean:
+	rm -rf $(JAVADOCDIR)/*
+
+distclean: clean
+	rm -f Makefile
Index: /main/tags/3.01/gsdl3/docs/javadoc/.keepme
===================================================================
--- /main/tags/3.01/gsdl3/docs/javadoc/.keepme	(revision 21112)
+++ /main/tags/3.01/gsdl3/docs/javadoc/.keepme	(revision 21112)
@@ -0,0 +1,1 @@
+a file to ensure cvs keeps the directory
Index: /main/tags/3.01/gsdl3/docs/manual/hints.tex
===================================================================
--- /main/tags/3.01/gsdl3/docs/manual/hints.tex	(revision 21112)
+++ /main/tags/3.01/gsdl3/docs/manual/hints.tex	(revision 21112)
@@ -0,0 +1,228 @@
+\documentclass[a4paper,11pt]{article}
+\usepackage{times,epsfig}
+
+\newenvironment{bulletedlist}%
+{\begin{list}{$\bullet$}{\setlength{\itemsep}{0pt}\setlength{\parsep}{0pt}}}%
+{\end{list}}
+
+
+\begin{document}
+
+\title{Some hints for developers}
+
+\maketitle
+
+\section{Java code}
+
+the java code is found in gsdl3/src/java/org/greenstone/gsdl3
+util has various utility classes, service has the services, action has the actions, build has the building code, etc.
+
+\section{Working with XML}
+
+We use the DOM model for handling XML. This involves Documents, Nodes, Elements etc. Node is the basic thing in the tree, all others inherit from this. A Document represents a whole document, and is a kind of container for all the nodes. Elements and Nodes are not supposed to exist outside of the context of a document, so you have to have a document to create them. The document is not the top level node in the tree, to get this, use Document.getDocumentElement(). If you create nodes etc but dont append them to something already in the document tree, they will be separate - but they still know who their owner document is.
+
+There is a utility class called XMLConverter - this creates new Documents, and converts Strings or files to Documents.
+eg:
+XMLConverter converter = new XMLConverter();
+Document doc = converter.newDOM();
+
+File stylesheet = new File(``query.xsl'');
+Document style = converter.getDOM(stylesheet);
+
+String message = ``<message><request type='cgi'/></message>'';
+Document m = converter.getDOM(message);
+
+To output a document as a String, use converter.getString(doc);
+
+
+To add nodes and stuff to an empty document - create them, then append to the tree
+
+Document doc = converter.newDOM();
+Element e = doc.createElement(``message'');
+doc.appendChild(e);
+
+Note, you can only append one node to a document - this will become the toplevel node. After that, you can append nodes to child nodes as you like, but a document is only allowed one top level node.
+
+DOM006 Hierarchy request error: happens if you have more than one root node in your document
+
+\section{Working with XSLT}
+
+* adding html to an xml doc:
+
+eg I have a text node with html inside it inside a resource element
+to add that to a new XML doc, I use
+{\em <xsl:value-of select='resource'>\/}
+
+if the output mode is xml or html, this will escape any special characters
+ie $<$ and $>$ etc
+
+use
+{\em <xsl:value-of disable-output-escaping="yes" select='resource'>\/}
+instead.
+
+* including an xml doc into a stylesheet:
+
+{\em <xsl:variable name='import' select='document(``newdoc.xml'')'/>\/}
+
+then can use the info:
+
+{\em <xsl:value-of select='\$import/element'/>\/}
+
+* selecting an ancestor:
+
+ the ancestor axis contains the parent of the context node, and its
+ parent and so on. to pick one node among these:
+ ancestor::elem-name. I dont know how this works if there are two
+ nodes with the same name in the axis.
+
+* basic XSLT elements
+\begin{quote}\begin{footnotesize}\begin{verbatim}
+<xsl:template match='xxx' name='yyy'/>
+
+<xsl:apply-templates select='xxx'/>
+<xsl:call-templates name='yyy'/>
+
+<xsl:variable name='doc' select='document("layout.xml")'/>
+
+<xsl:value-of select='$doc/chapter1'/> $
+\end{verbatim}\end{footnotesize}\end{quote}
+
+* using namespaces
+If you are using the same namespace in more than one file, eg in the source xml and in the stylesheet, make sure that the URI for the xmlns:xxx thingy is the same in both cases---otherwise the names dont match. This includes http:// on the front.
+
+\subsection{What can I do to speed up XSL transformations?}
+
+This information taken from the Xalan FAQS page.
+
+\begin{bulletedlist}
+
+\item Use a Templates object (with a different Transformers for each
+transformation) to perform multiple transformations with the same set
+of stylesheet instructions (see Multithreading).
+
+\item Set up your stylesheets to function efficiently.
+
+\item Don't use "//" (descendant axes) patterns near the root of a
+large document.
+
+\item Use xsl:key elements and the key() function as an efficient way
+to retrieve node sets.
+
+\item Where possible, use pattern matching rather than xsl:if or
+xsl:when statements.
+
+\item xsl:for-each is fast because it does not require pattern matching.
+
+\item Keep in mind that xsl:sort prevents incremental processing.
+
+\item When you create variables,
+{\em <xsl:variable name="fooElem" select="foo"/>\/} is usually faster
+than \\
+{\em <xsl:variable name="fooElem"><xsl:value-of-select="foo"/></xsl:variable>\/}.
+
+\item Be careful using the last() function.
+
+\item The use of index predicates within match patterns can be expensive.
+
+\item Decoding and encoding is expensive.
+
+\item For the ultimate in server-side scalability, perform transform
+operations on the client. For examples, see appletXMLtoHTML and
+get-todo-list.
+
+\end{bulletedlist}
+
+\section{Java gdbm}
+
+To talk to gdbm, a jni wrapper called java-gdbm is used. It was
+obtained from:\\ {\tt
+http://aurora.rg.iupui.edu/~schadow/dbm-java/pip/gdbm/}
+
+It uses packing objects to convert to and from an array of bytes (in
+gdbm file) from and to java objects. In my GDBMWrapper class I use
+StringPacking - uese UTF-8 encoding. but some stuff came out funny. so
+I had to changes the from\_bytes method in StringPacking.java to use
+new String(raw, "UTF-8") instead of new String(raw). this seems to
+work.
+
+Note---if we use this gdbm stuff to create the file too, may need to
+alter the to-bytes method.
+
+The makefile in j-gdbm is crap---it tries to get stuff from its
+original CVS tree.  I have created a new Makefile---in my-j-gdbm
+directory.  this stuff needs to go into cvs probably.
+
+* I dont think {\em <xsl:with-param name='xxx' select='true'/>\/} is
+the same as {\em <xsl:with-param name='xxx'>true</xsl:with-param>\/}
+use the second one.
+
+
+\section{Resources}
+
+Contents for 'The Java Native Interface Programmer's Guide and
+Specification' on-line\\ {\tt
+http://java.sun.com/docs/books/jni/html/jniTOC.html}
+
+Java Native Interface Specification\\
+{\tt http://java.sun.com/j2se/1.4/docs/guide/jni/spec/jniTOC.doc.html}
+
+JNI Documentation Contents\\
+{\tt http://java.sun.com/j2se/1.4/docs/guide/jni/index.html}
+
+another JNI page\\
+{\tt http://mindprod.com/jni.html}
+
+Java 1.4 api index\\
+{\tt http://java.sun.com/j2se/1.4/docs/api/index.html}
+
+Java tutorial index\\
+{\tt http://java.sun.com/docs/books/tutorial/index.html}
+
+Safari books online - has java, XML, XSLT, etc books\\
+{\tt http://proquest.safaribooksonline.com/mainhom.asp?home}
+
+Java 1.4 i18n FAQ\\
+{\tt http://www.sun.com/developers/gadc/faq/java/java1.4.html}
+
+Java and XSLT page\\
+{\tt http://www.javaolympus.com/java/Java\%20and\%20XSLT.html}
+
+Xalan-Java overview\\
+{\tt http://xml.apache.org/xalan-j/overview.html}
+
+Tomcat documentation index\\
+{\tt http://jakarta.apache.org/tomcat/tomcat-4.0-doc/index.html}
+
+Servlet and JSP tutorial\\
+{\em http://www.apl.jhu.edu/~hall/java/Servlet-Tutorial/\/}
+
+Core Servlets and JavaServer Pages, book by Marty Hall. download the
+pdf from here (try before you buy link)\\ {\tt
+http://www.coreservlets.com/}
+
+J-gdbm page\\
+{\em http://aurora.rg.iupui.edu/~schadow/dbm-java/pip/gdbm/\/}
+
+Stuarts page of links\\
+{\em http://www.cs.waikato.ac.nz/~nzdl/gsdl3/\/}
+
+a good basic xslt tutorial\\
+{\tt http://www.zvon.org/xxl/XSLTutorial/Books/Output/contents.html}
+
+JAXP (java api for xml processing) package overview\\
+{\tt http://java.sun.com/xml/jaxp/dist/1.1/docs/api/overview-summary.html}
+
+DeveloperWorks, xml zone\\
+{\tt http://www-106.ibm.com/developerworks/xml/}
+
+xslt.com\\
+{\tt http://www.xslt.com/}
+
+jeni tennison's xslt pages\\
+{\tt http://www.jenitennison.com/xslt/}
+
+apaches xml tools\\
+{\tt http://xml.apache.org/}
+
+
+\end{document}
Index: /main/tags/3.01/gsdl3/docs/manual/manual.tex
===================================================================
--- /main/tags/3.01/gsdl3/docs/manual/manual.tex	(revision 21112)
+++ /main/tags/3.01/gsdl3/docs/manual/manual.tex	(revision 21112)
@@ -0,0 +1,2410 @@
+\documentclass[a4paper,11pt]{article}
+\usepackage{isolatin1,times,epsfig}
+\hyphenation{Message-Router Text-Query}
+
+\newenvironment{gsc}% Greenstone text bits
+{\begin{footnotesize}\begin{tt}}%
+{\end{tt}\end{footnotesize}}
+ 
+\newcommand{\gst}[1]{{\footnotesize \tt #1}}
+\newcommand{\gsdlhome}{\$GSDL3HOME}
+
+\newcommand{\gsii}{Greenstone 2}
+\newcommand{\gsiii}{Greenstone 3}
+\newcommand{\gs}{Greenstone}
+
+\begin{document}
+
+\title{\gsiii\ : A modular digital library.}
+
+% if you work on this manual, add your name here
+\author{Katherine Don, George Buchanan and Ian H. Witten \\[1ex]
+        Department of Computer Science \\
+        University of Waikato \\ Hamilton, New Zealand \\ }
+
+\date{}
+
+\maketitle
+
+\newenvironment{bulletedlist}%
+{\begin{list}{$\bullet$}{\setlength{\itemsep}{0pt}\setlength{\parsep}{0pt}}}%
+{\end{list}}
+
+\noindent
+Greenstone Digital Library Version 3 is a complete redesign and
+reimplementation of the \gs\  digital library software.  The current
+version (\gsii) enjoys considerable success and is being widely used.
+\gsiii \  will capitalize on this success, and in addition it will
+\begin{bulletedlist}
+\item improve flexibility, modularity, and extensibility
+\item lower the bar for ``getting into'' the \gs\  code with a view to
+   understanding and extending it
+\item use XML where possible internally to improve the amount of
+   self-documentation
+\item make full use of existing XML-related standards and software
+\item provide improved internationalization, particularly in terms of sort order,
+   information browsing, etc.
+\item include new features that facilitate additional ``content management''
+   operations
+\item operate on a scale ranging from personal desktop to corporate library
+\item easily permit the incorporation of text mining operations
+\item use Java, to encourage multilinguality, X-compatibility, and to permit
+   easier inclusion of existing Java code (such as for text mining).
+\end{bulletedlist}
+Parts of \gs\  will remain in other languages (e.g. MG, MGPP); JNI (Java
+Native Interface) will be used to communicate with these.
+
+A description of the general design and architecture of \gsiii\  is covered by the document {\em The design of Greenstone3: An agent based dynamic digital library} (design-2002.ps, in the docs/manual directory).
+
+This documentation consists of several parts. Section~\ref{sec:install} is for administrators, and covers \gsiii\  installation, how to access the library, and some administration issues. Section~\ref{sec:user} is for users of the software, and looks at using the sample collections, creating new collections, and how to make small customizations to the interface. The remaining sections are aimed towards  the \gs\  developer. Section~\ref{sec:develop-runtime} describes the run-time system, including the structure of the software, and the message format, while Section~\ref{sec:develop-build} describes the collection building process. Section~\ref{sec:new-features} describes how to add new features to \gs, such as how to add new services, new page types, new plugins for different document formats.  Section~\ref{sec:distributed} describes how to make \gs\  run in a distributed fashion, using SOAP as an example communications protocol. Finally, there are several appendices, including how to install \gs\  from CVS, some notes on Tomcat and SOAP, and a comparison of \gsii\  and \gsiii\  format statements.
+\newpage
+\tableofcontents
+\newpage
+\section{\gs\  installation and administration}\label{sec:install}
+
+This section covers where to get \gsiii\  from, how to install it and how to run it. The standard method of running \gsiii\  is as a Java servlet. We provide the Tomcat servlet container to run the servlet. Standard web servers may  be able to be configured to provide servlet support, and thereby remove the need to use Tomcat. Please see your web server documentation for this. This documentation assumes that you are using Tomcat. To access \gsiii, Tomcat must be started up, and then it can be accessed via a web browser. 
+
+Ant (Java's XML based build tool) is used for compilation, installation and running Greenstone. The build.xml file is the configuration file for the Greenstone project, and build.properties contains parameters that can be altered by the user.
+
+\subsection{Get and install \gs\ }
+
+\gsiii\  is available for download from Sourceforge:\\
+ \gst{https://sourceforge.net/projects/greenstone3}. There are Windows, Linux and Mac OS X releases. They consist of a ZIP/TAR file which should be unpacked. Please check and edit (if necessary) the installation properties in build.properties, then run 'ant install' in the greenstone3 directory. Please read the file README.txt for more detailed (and up to date) instructions.
+
+Greenstone 3 can be started by running 'ant start', and will be available at \gst{http://localhost:8080/greenstone3}\\
+(or \gst{http://your-computer-name:your-chosen-port/greenstone3}). \\
+This gets you to a welcome page containing  links to four servlets: the \gst{test} servlet (this allows you to check that Tomcat is running properly); the standard \gst{library} servlet which serves \gst{localsite} site with the \gst{default} interface; the \gst{classic} servlet which serves \gst{localsite} using the \gst{classic} or \gsii-style interface; the \gst{gateway} servlet, which serves \gst{gateway} site with the \gst{default} interface. The \gst{gateway} site uses a SOAP connection to communicate with \gst{localsite}, and demonstrates the library working in a distributed fashion. 
+
+\gsiii\  is also available through CVS (Concurrent Versioning System). This provides the latest development version, and is not guaranteed to be stable. Appendix~\ref{app:cvs} describes how to download and install \gsiii\  from CVS.
+
+\subsection{How the library works}
+
+The standard library program is a Java servlet. We use the Tomcat servlet container to present the servlets over the web. Tomcat takes CGI-style URLs and passes the arguments to the servlet, which processes these and returns a page of HTML. As far as an end-user is concerned, a servlet is a Java version of a CGI program. The interaction is similar: access is via a web browser,  using arguments in a URL.
+
+Other types of interfaces can be used, such as Java GUI programs. See Section~\ref{sec:new-interfaces} for details about how to make these.
+
+\subsubsection{Restarting the library}
+
+The library program (actually Tomcat and MYSQL) can be restarted by running \gst{ant restart} in the greenstone3 directory.
+
+Tomcat must be restarted any time you make changes in the following for those changes to take effect:\\
+\begin{bulletedlist}
+\begin{gsc}
+\item \gsdlhome/web/WEB-INF/web.xml
+\item \gsdlhome/packages/tomcat/conf/server.xml
+\end{gsc}
+\item any classes or jar files used by the servlets
+\end{bulletedlist}
+\noindent Note: stdout and stderr for the servlets (on Linux and Mac OS X) both go to\\
+\gst{\gsdlhome/packages/tomcat/logs/catalina.out}
+
+
+\subsection{Directory structure}
+
+Table~\ref{tab:dirs} shows the file hierarchy for \gsiii\ .
+The first part  shows the common stuff which can be shared between
+\gs\  users---the source, libraries etc. The second part shows the file hierarchy for the greenstone3/web directory, which comprises the greenstone3 context for Tomcat, and is accessible via Tomcat. The main directories are for sites and interfaces: there can be several sites and interfaces per installation, and they are described in the following section.
+
+
+\begin{table}
+\caption{The \gs\  directory structure}
+\label{tab:dirs}
+{\footnotesize
+\begin{tabular}{l p{8cm}}
+\hline
+\bf directory & \bf description \\
+\hline
+greenstone3 
+  & The main installation directory---gsdl3home can be changed to something more standard\\
+greenstone3/src 
+  & Source code lives here \\
+greenstone3/src/java/
+  & main greenstone 3 java source code \\
+greenstone3/src/packages 
+  & Imported source packages from other systems e.g. MG, MGPP \\
+greenstone3/extensions
+  & Extensions to greenstone 3 core functionality, e.g., Vishnu visualizer, Alerting service \\
+greenstone3/lib
+  & Shared library files\\
+greenstone3/lib/java
+  & Java jar files not needed in the greenstone 3 runtime\\
+greenstone3/lib/jni
+  & Jar files and shared library files (.so, .jnilib, .dll) needed for JNI components \\
+greenstone3/resources 
+ & any resources that may be needed\\
+greenstone3/resources/soap
+ & soap service description files \\
+greenstone3/bin
+  & executable stuff lives here\\
+greenstone3/bin/script
+  & some Perl and/or shell building scripts\\
+greenstone3/packages
+  & External packages that may be installed as part of greenstone, e.g. Tomcat, MySQL \\
+greenstone3/docs
+  & Documentation\\
+\hline
+greenstone3/web
+  & This is where the web site is defined. Any static HTML files can go here. This directory is the Tomcat root directory.\\
+greenstone3/web/WEB-INF
+  & The web.xml file lives here (servlet configuration information for Tomcat)\\
+greenstone3/web/WEB-INF/classes
+  & Individual class files needed by the servlet go in here, also properties files for java resource bundles - used to handle all the language specific text. This directory is on the servlet classpath\\
+greenstone3/web/WEB-INF/lib
+  & jar files needed by the servlets go here \\
+greenstone3/web/sites
+  & Contains directories for different sites---a site is a set of collections and services served by a single MessageRouter (MR). The MR may have connections (e.g. soap) to other sites\\
+greenstone3/web/sites/localsite 
+  & An example site - the site configuration file lives here\\
+greenstone3/web/sites/localsite/collect 
+  & The collections directory \\
+greenstone3/web/sites/localsite/images 
+  & Site specific images \\
+greenstone3/web/sites/localsite/transforms 
+  & Site specific transforms \\
+greenstone3/web/interfaces
+  & Contains directories for different interfaces - an interface is defined by its images and XSLT files \\
+greenstone3/web/interfaces/default
+  & The default interface\\
+greenstone3/web/interfaces/default/images
+  & The images for the default interface\\
+greenstone3/web/interfaces/default/transforms
+  & The XSLT files for the default interface\\
+greenstone3/web/applet 
+  & jar files needed by applets can go here \\
+\hline
+\end{tabular}}
+\end{table}
+
+
+\subsection{Sites and interfaces}\label{sec:sites-and-ints}
+
+[local gs stuff (sites and interfaces) vs installed stuff (code)\\
+where they live, whats the difference, what each contains.]\\
+Sites and interfaces contain the content and presentation information, respectively, for the digital library.
+A site is comprised of a set of collections and possibly some site-wide services. An interface (in this web-based servlet context) is a set of images along with a set of XSLT files used for translating xml output from the library into an appropriate form---HTML in general.
+
+One \gsiii\  installation can have many sites and interfaces, and these can be paired in different combinations.  One instantiation of a servlet uses one site and one interface, so every specified pairing results in a new servlet instance.  For example, a single site might be served with two different interfaces. This provides different modes of access to the same content. e.g. HTML vs WML, or perhaps providing a completely different look and feel for different audiences. Alternatively, a standard interface may be used with many different sites---providing a consistent mode of access to a lot of different content.
+
+Collections live in the \gst{collect} directory of a site. Any collections that are found in this directory when the servlet is initialized will be loaded up and presented to the user. Collections require valid configuration files, but apart from this, nothing needs to be done to the site to use new collections. Collections added while Tomcat is running will not be noticed automatically. Either the server needs to be restarted, or a configuration request may be sent to the library, triggering a (re)load of the collection (this is described in Section~\ref{sec:runtime-config}).
+
+There are two  sites that come with the distribution: \gst{localsite}, and \gst{gateway}. \gst{localsite} has several demo  collections, while \gst{gateway} has none. \gst{gateway} specifies that a SOAP connection should be made to \gst{localsite}. Getting this to work involves setting up a soap server for localsite: see Section~\ref{sec:distributed} for details. 
+There are also two interfaces provided in the distribution: \gst{default} and \gst{classic}. The default interface is a generic \gsiii\ interface, while the \gst{classic} interface aims to look like the old \gsii\ interface.
+
+Each site and interface has a configuration file which specifies parameters for the site or interface---these are described in Section~\ref{sec:config}.
+
+\subsection{Configuring Tomcat}\label{sec:tomcat-config}
+
+The file \gst{\gsdlhome/web/WEB-INF/web.xml} contains the configuration information for Tomcat. It tells Tomcat what servlets to load, what initial parameters to pass them, and what web names map to the servlets.
+There are four servlets specified in web.xml (these correspond to the four servlet links in the welcome page for \gsiii): one is a test servlet that just prints ``hello greenstone'' to a web page. This is useful if you are having trouble getting Tomcat set up. The other three are the \gs\  library servlets described in Section~\ref{sec:browser-access}, \gst{library}, \gst{classic} and \gst{gateway}. Each servlet must specify which site and which interface to use. Having multiple servlets provides a way of serving different sites, or the same site with a different style of presentation. Site\_name and interface\_name are just two examples of initialization parameters used by the library servlets. The full list is shown in Table~\ref{tab:serv-init}. 
+
+For more details about Tomcat see Appendix~\ref{app:tomcat}.
+
+\begin{table}
+\caption{\gs\  servlet initialization parameters}
+\label{tab:serv-init}
+{\footnotesize
+\begin{tabular}{llp{5cm}}
+\hline
+\bf name & \bf sample value & \bf description \\
+\hline
+site\_name & localsite & the name of the site to use \\
+interface\_name & default & the name of the interface to use\\
+library\_name & library & the web name of the servlet \\
+default\_lang & en & the default language for the interface\\
+receptionist\_class & NZDLReceptionist & (optional) specifies an alternative Receptionist to use\\
+messagerouter\_class & NewMessageRouter & (optional) specifies an alternative MessageRouter to use\\
+params\_class & NZDLParams & (optional) specifies an alternative GSParams class to use \\
+\hline
+\end{tabular}}
+\end{table}
+
+\subsection{Configuring a \gs\ library}\label{sec:config}
+
+Initial \gsiii\  system configuration is determined by a set of configuration files, all expressed in XML. Each site has a configuration file that binds parameters for the site, \gst{siteConfig.xml}. Each interface has a configuration file, \gst{interfaceConfig.xml}, that specifies Actions for the interface. Collections also have several configuration files; these are discussed in Section~\ref{sec:collconfig}. 
+The configuration files are read in when the system is initialized, and their contents are cached in memory. This means that changes made to these files once the system is running will not take immediate effect. Tomcat needs to be restarted for changes to the interface configuration file to take effect. However, changes to the site configuration file can be incorporated sending a system command to the library.  There are a series of system commands that can be sent to the library to induce reconfiguration of different modules, including reloading the whole site. This removes the need to restart the system to reflect these changes. These commands are described in Section~\ref{sec:runtime-config}.
+
+\subsubsection{Site configuration file}\label{sec:siteconfig}
+
+The file \gst{siteConfig.xml} specifies the URI for the site (\gst{localSiteName}), the HTTP address for site resources (\gst{httpAddress}), any ServiceClusters that the site provides (for example, collection building), any ServiceRacks that do not belong to a cluster or collection, and a list of
+known external sites to connect to.  Collections are not specified in the site
+configuration file, but are determined by the contents of the site's
+collections directory.
+
+The HTTP address is used for retrieving resources from a site outside the XML protocol. Because a site is HTTP accessible through Tomcat, any files (e.g. images) belonging to that site or to its collections can be specified in the HTML of a page by a URL. This avoids having to retrieve these files from a remote site via the XML protocol\footnote{Currently, sites live inside the Tomcat greenstone3 root context, and therefore all their content is accessible over HTTP via the Tomcat address. We need to see if parts can be restricted. Also, if we use a different protocol, then resources from remote sites may need to come through the XML. Also, if we are running locally without using Tomcat, we may want to get them via file:// rather than http://.}.
+ 
+Figure~\ref{fig:siteconfig} shows two example site configuration files. The first example is for a rudimentary site with no site-wide services,
+which does not connect to any external sites. The second example is for a site with one site-wide service cluster - a collection building cluster.  It also connects to the first site using SOAP.
+These two sites happen to be running on the same machine, which is why they can use \gst{localhost} in the address. For site \gst{gsdl1} to talk to site \gst{localsite}, a SOAP server must be run for \gst{localsite}. The address of the SOAP server, in this case, is \gst{http://localhost:8080/greenstone3/services/localsite}.
+
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<siteConfig>
+  <localSiteName value="org.greenstone.localsite"/>
+  <httpAddress value="http://localhost:8080/greenstone3/sites/localsite"/>
+  <serviceClusterList/>
+  <serviceRackList/>
+  <siteList/>
+</siteConfig>
+\end{verbatim}\end{gsc}
+
+\begin{gsc}\begin{verbatim}
+<siteConfig>
+  <localSiteName value="org.greenstone.gsdl1"/>
+  <httpAddress value="http://localhost:8080/greenstone3/sites/gsdl1"/>
+  <serviceClusterList>	
+    <serviceCluster name="build">
+      <metadataList>
+        <metadata name="Title">Collection builder</metadata>
+        <metadata name="Description">Builds collections in a 
+           gsdl2-style manner</metadata>
+      </metadataList>
+      <serviceRackList>
+        <serviceRack name="GS2Construct"/>
+      </serviceRackList>
+    </serviceCluster>
+  </serviceClusterList>
+  <siteList>
+    <site name="org.greenstone.localsite"
+      address="http://localhost:8080/greenstone3/services/localsite"
+      type="soap"/>
+  </siteList>
+</siteConfig>
+\end{verbatim}\end{gsc}
+\caption{Two sample site configuration files}
+\label{fig:siteconfig}
+\end{figure}
+
+\subsubsection{Interface configuration file}\label{sec:interfaceconfig}
+
+The interface configuration file \gst{interfaceConfig.xml} lists all the actions that the interface knows about at the start (other ones can be loaded dynamically). Actions create the web pages for the library: there is generally one Action per type of page. For example, a query action produces the pages for searching, while a document action displays the documents. The configuration file specifies what short name each action maps to (this is used in library URLs for the a (action) parameter) e.g. QueryAction should use a=q. If the interface uses XSLT, it specifies what XSLT file should be used for each action and possibly each subaction. This makes it easy for developers to implement and use different actions and/or XSLT files without recompilation. The server must be restarted, however.
+
+It also lists all the languages that the interface text files have been translated into. These have a \gst{name} attribute, which is the ISO code for the language, and a \gst{displayElement} which gives the language name in that language (note that this file should be encoded in UTF-8). This language list is used on the Preferences page to allow the user to change the interface language. Details on how to add a new language to a \gsiii\  library are shown in Section~\ref{sec:interface-customise}.
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<interfaceConfig>
+  <actionList>
+    <action name='p' class='PageAction'>
+      <subaction name='home' xslt='home.xsl'/>
+      <subaction name='about' xslt='about.xsl'/>
+      <subaction name='help' xslt='help.xsl'/>
+      <subaction name='pref' xslt='pref.xsl'/>
+    </action>
+    <action name='q' class='QueryAction' xslt='basicquery.xsl'/>
+    <action name='b' class='GS2BrowseAction' xslt='classifier.xsl'/>
+    <action name='a' class='AppletAction' xslt='applet.xsl'/>
+    <action name='d' class='DocumentAction' xslt='document.xsl'/>
+    <action name='xd' class='XMLDocumentAction'>
+      <subaction name='toc' xslt='document-toc.xsl'/>
+      <subaction name='text' xslt='document-content.xsl'/>
+    </action>
+    <action name='pr' class='ProcessAction' xslt='process.xsl'/>
+    <action name='s' class='SystemAction' xslt='system.xsl'/>
+  </actionList>
+  <languageList>
+    <language name="en">
+      <displayItem name='name'>English</displayItem>
+    </language>
+    <language name="fr">
+      <displayItem name='name'>Français</displayItem>
+    </language>
+    <language name='es'>
+      <displayItem name='name'>Español</displayItem>
+    </language>
+  </languageList>
+</interfaceConfig>
+\end{verbatim}\end{gsc}
+\caption{Default interface configuration file}
+\label{fig:ifaceconfig}
+\end{figure}
+
+
+\subsection{Run-time re-initialization}\label{sec:runtime-config}
+
+When Tomcat is started up, the site and interface configuration files are read in, and actions/services/collections loaded as necessary. The configuration is then static unless Tomcat is restarted, or re-configuration commands issued. 
+
+There are several commands that can be issued to Tomcat to avoid having to restart the server. These can reload the entire site, or just individual collections. Unfortunately at present there are no commands to reconfigure the interface, so if the interface configuration file has changed, Tomcat must be restarted for those changes to take effect. Similarly, if the Java classes are modified, Tomcat must be restarted then too.
+
+Currently, the runtime configuration commands can only be accessed by typing arguments into the URL; there is no nice web form yet to do this. 
+
+The arguments are entered after the \gst{library?} part of the URL. There are three types of commands: configure, activate, deactivate\footnote{There is no security for these commands yet in \gs, so the deactivate/delete command is disabled}. These are specified by \gst{a=s\&sa=c}, \gst{a=s\&sa=a}, and \gst{a=s\&sa=d}, respectively (\gst{a} is action, \gst{sa} is subaction). By default, the requests are sent to the MessageRouter, but they can be sent to a collection/cluster by the addition of \gst{sc=xxx}, where \gst{xxx} is the name of the collection or cluster. Table~\ref{tab:run-time config} describes the commands and arguments in a bit more detail.
+
+\begin{table}
+\caption{Example run-time configuration arguments.}
+\label{tab:run-time config}
+{\footnotesize
+\begin{tabular}{lp{8cm}}
+\hline
+\gst{a=s\&sa=c} & reconfigures the whole site. Reads in siteConfig.xml, reloads all the collections. Just part of this can be specified with another argument \gst{ss} (system subset). The valid values are \gst{collectionList}, \gst{siteList}, \gst{serviceList}, \gst{clusterList}. \\
+\gst{a=s\&sa=c\&sc=XXX} & reconfigures the XXX collection or cluster. \gst{ss} can also be used here, valid values are \gst{metadataList} and \gst{serviceList}. \\
+\gst{a=s\&sa=a} & (re)activate a specific module. Modules are specified using two arguments, \gst{st} (system module type) and \gst{sn} (system module name). Valid types are \gst{collection}, \gst{cluster} \gst{site}.\\
+\gst{a=s\&sa=d} & deactivate a module. \gst{st} and \gst{sn} can be used here too. Valid types are \gst{collection}, \gst{cluster}, \gst{site}, \gst{service}. Modules are removed from the current configuration, but will reappear if Tomcat is restarted.\\
+\gst{a=s\&sa=d\&sc=XXX} & deactivate a module belonging to the XXX collection or cluster. \gst{st} and \gst{sn} can be used here too. Valid types are \gst{service}. \\
+\hline
+\end{tabular}}
+\end{table}
+\newpage
+\section{Using \gsiii\ }\label{sec:user}
+
+Once \gsiii\ is installed, the sample collections can be accessed. The installation comes with several example collections, and Section~\ref{sec:usecolls} describes these collections and how to use them. Section~\ref{sec:buildcol} describes how to build new collections.
+
+\subsection{Using a collection}\label{sec:usecolls} 
+
+A collection typically consists of a set of documents, which could be text, HTML, word, PDF, images, bibliographic records etc, along with some access methods, or ``services''. Typical access methods include searching or browsing for document identifiers, and retrieval of content or metadata for those identifiers.
+Searching involves entering words or phrases and getting back lists of documents that contain those words. The search terms may be restricted to particular fields of the document. 
+
+Browsing involves navigating pre-defined hierarchies of documents, following links of interest to find documents. The hierarchies may be constructed on different metadata fields, for example, alphabetical lists of Titles, or a hierarchy of Subject classifications. Clicking on a bookshelf icon takes you to a lower level in the hierarchy, while clicking on a book or page icon takes you to a document.
+
+In the standard interface that comes with \gsiii\ \footnote{of course, this is all customizable}, collections in a digital library are presented in the following manner. The 'home' page of the library shows a list of all the public collections in that library. Clicking on a collection link takes you to the home page for the collection, which we call the collection's 'about' page. The standard page banner looks something like that shown in Figure~\ref{fig:page-banner}.
+
+\begin{figure}[h]
+  \centering
+  \includegraphics[width=4in]{pagebanner.ps} %5.8
+  \caption{A sample collection page banner}
+  \label{fig:page-banner}
+\end{figure}
+
+The image at the top left is a link to the collection's home page. The top right has buttons to link to the library home page, help and preferences pages. All the available services are arrayed along a navigation bar, along the bottom of the banner. Clicking on a name accesses that service. 
+
+Search type services generally provide a form to fill in, with parameters including what field or granularity to search, and the query itself. Clicking the search button carries out the search, and a list of matching documents will be displayed. Clicking on the icons in the result list takes you to the document itself. 
+
+Once you are looking at a document, clicking the open book icon at the top of the document, underneath the navigation bar, will take you back to the service page that you accessed the document from.
+
+\subsection{Building a collection}\label{sec:buildcol}
+
+There are three ways to get a new collection into \gsiii. The first is to build it using the \gsiii\ command line building process. The second way is to use the Greenstone Librarian Interface to build a new collection. This creates a collection in a \gsiii\ context, but uses the \gsii\ Perl collection building process. The third way is to import a pre-built \gsii\  collection. 
+
+Collections live in the collect directory of a site. As described in Section~\ref{sec:sites-and-ints}, there can be several sites per \gsiii\  installation. The collect directory is at \gst{\$GSDL3HOME/web/sites/site-name/collect}, where site-name is the name of the site you want your new collection to belong to.
+
+The following three sections describe how to create a collection from scratch, using command line and GLI building, and how to import a \gsii\  collection. Once a collection has been built (and is located in the collect directory), the library server needs to be notified that there is a new collection. This can be accomplished in two ways\footnote{and eventually there will also probably be automatic polling for new collections}. If you are the library administrator, you can restart Tomcat. The library servlet will then be created afresh, and will discover the new collection when it scans the collect directory for the collection list. Alternatively, an activate collection command can be issued to the servlet, using the arguments \gst{a=s\&sa=a\&st=collection\&sn=collname}, where \gst{collname} should be replaced with the collection name---this tells the library program to (re)load the \gst{collname} collection.
+
+
+\subsubsection{Creating a collection from scratch}
+
+To create the director
+Building native \gsiii\  collections is done using the \gst{gs3-build.sh/bat} script, with the \gst{collectionConfig.xml} file controlling how the building is done.  There are a number of considerations in building a collection:  what documents appear in the collection, how they are indexed for searching, which classifications are used for browsing, etc. 
+
+Firstly, the documents that comprise the collection should be placed in the import subdirectory.  At present, only documents in this directory will appear in the collection. Documents can be organized into sub folders inside the import directory.
+[TODO: describe the kinds of documents that can be added, something about METS files?]
+
+Metadata for documents can be added using metadata.xml files.  These files have already been used in \gsii, and the format is the same in \gsiii.  A metadata.xml file has a root element of \gst{<DirectoryMetadata>}.  This encloses a series of \gst{<FileSet>} items.  Neither of these tags has any attributes.  Each \gst{<FileSet>} item includes two parts: firstly, one or more \gst{<FileName>} tags, each of which encloses a regular expression to identify the files which are to be assigned the metadata.  Only files in the same directory as the metadata.xml, or in one of its child directories, will be selected.  The filename tag encloses the regular expression as text, e.g.:
+
+\begin{gsc}\begin{verbatim}
+<FileName>example</FileName>
+\end{verbatim}\end{gsc}
+
+This would match any file containing the text 'example' in its name.  The second part of the \gst{<FileSet>} item is a \gst{<Description>} item.  The \gst{<Description>} tag has no attributes, but encloses one or more \gst{<Metadata>} tags.  Each \gst{<Metadata>} tag contains one metadata item, i.e. a label to describe the metadata and a corresponding value.  The \gst{<Metadata>} tag has one compulsory attribute: ``name''.  This attribute gives the metadata label to add to the document.  Each \gst{<Metadata>} tag also has an optional attribute: ``mode''.  If this attribute is set to ``accumulate'' then the value is added to the document, and any existing values for that metadata item are retained.  If the attribute is set to ``set'' or is omitted, then any existing value of the metadata item will be deleted.
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE DirectoryMetadata SYSTEM "http://greenstone.org/dtd/DirectoryMetadata
+         /1.0/DirectoryMetadata.dtd">
+<DirectoryMetadata>
+  <FileSet>
+    <FileName>ec160e</FileName>
+    <Description>
+      <Metadata name="Title">The Courier - No.160 - Nov - Dec 1996 - 
+         Dossier Habitat - Country reports: Fiji , Tonga (ec160e)</Metadata>
+      <Metadata mode="accumulate" name="Language">English</Metadata>
+      <Metadata mode="accumulate" name="Subject">Settlements and housing: 
+        general works incl. low- cost housing, planning techniques, surveying, 
+        etc.</Metadata>
+      <Metadata mode="accumulate" name="Subject">The Courier ACP 1990 - 1996 
+        Africa-Caribbean-Pacific - European Union</Metadata>
+      <Metadata mode="accumulate" name="Organization">EC Courier</Metadata>
+      <Metadata mode="accumulate" name="AZList">T.1</Metadata>
+    </Description>
+  </FileSet>
+  <FileSet>
+    <FileName>b22bue</FileName>
+    <Description>
+      <Metadata name="Title">Butterfly Farming in Papua New Guinea 
+        (b22bue)</Metadata>
+      <Metadata mode="accumulate" name="Language">English</Metadata>
+      <Metadata mode="accumulate" name="Subject">Other animals (micro-
+        livestock, little known animals, silkworms, reptiles, frogs, 
+        snails, game, etc.)</Metadata>
+      <Metadata mode="accumulate" name="Organization">BOSTID</Metadata>
+      <Metadata mode="accumulate" name="AZList">T.1</Metadata>
+      <Metadata mode="accumulate" name="Keyword">start a butterfly farm
+        </Metadata>
+    </Description>
+  </FileSet>
+</DirectoryMetadata>
+\end{verbatim}\end{gsc}
+\caption{Sample metadata.xml file}
+\label{fig:metadatafile}
+\end{figure}
+
+Figure~\ref{fig:metadatafile} shows an example metadata.xml file.
+Here, only one file pattern is found in each file set.  However, the \gst{Description} tag contains a number of separate metadata items.  Note that the \gst{Title} metadata does not have the \gst{mode=accumulate} attribute.  This means that when the title is assigned to a document, its existing \gst{Title} information will be lost.
+
+The basic means of finding documents in \gs\  is search. Options for building the search indexes include which indexer to use, what granularity to use for the indexes (e.g. whether to index documents as a whole, or sections of documents), what content the index should have (the whole text of the document or one or many metadata fields).  Section-level indexes allow a reader to recall part of a document (for instance, a chapter) rather than the entire document.  However, \gsiii\  must be able to identify the internal structure of the document to achieve this.  The degree to which structure can be found varies from file format to file format.
+
+An alternative means of finding documents is through browsing. Greenstone can create pre-defined browsing hierarchies based on document metadata. Each browsing structure is called a classifier. Options for building classifiers include what type of classifier to use (linear list or multi-level hierarchy), what metadata to build the classifier on, e.g. Title, Author etc. 
+
+The collectionConfig.xml file controls the all of these options for collection building, and the format is described in Section~\ref{sec:collconfig}.
+
+To build a collection, place the source documents and optional metadata.xml file(s) in the import directory, place the \gst{collectionConfig.xml} file in the etc directory, and execute \gst{gs3build.sh/bat sitename collectionname}.  The process will run, placing the new indexes in the \gst{building} subdirectory of the collection's directory. You must have MySQL running before you start building---running \gst{ant start} will start up the MySQL server as well as tomcat.
+
+Once the build process is complete, the building directory should be renamed to index (after deleting or renaming the existing index directory, if any), and Tomcat prompted to reload the collection---either by restarting the server, or by sending an activate collection command to the library servlet.
+
+\subsubsection{Using the Librarian Interface}
+
+The Greenstone Librarian Interface (GLI) can be used to create \gsii\ style collections for \gsiii. It can be started under Windows by selecting Greenstone Librarian Interface from the Greenstone 3 Digital Library menu in the Program Files section of the Start menu. On Linux, run \gst{./gli4gs3.sh} from the \gst{greenstone3/gli} directory.
+
+Currently, the GLI works almost exactly the same as for \gsii\footnote{Eventually the GLI will be modified to use native \gsiii\ configuration files and collection building}. Collection configuration is done in a \gsii\ manner. The main difference is that \gsiii\ has different sites and interfaces and servlets, whereas \gsii\ has a single collect directory, and a single runtime cgi program.
+
+The GLI for \gsiii\ has a couple of new configuration parameters: site and servlet. It operates within a single site---you can edit, delete, create new collections within this site. A servlet is also specified for that site---this is used when previewing a collection. While you are working in one site, you cannot edit collections from another site. However, you can base a collection on one from another site. To change the working site and/or servlet, go to Preferences-$>$Connection in the File menu. By default, the GLI will use site \gst{localsite}, and servlet \gst{library}.
+
+Collection building using the GLI will use the \gsii\ Perl scripts and plugins. At the conclusion of the \gsii\ build process, a conversion script will be run to create the \gsiii\  configuration files. This means that format statements are no longer 'live'---changing these will require changes to the \gsiii\ configuration files. You can either rebuild the collection through the GLI (may take a while), or run the conversion script directly (see following section). 
+ 
+Detailed instructions about using the GLI can be found in Sections 3.1 and 3.2 of the Greenstone 2 User's Guide (\gst{GS2-User-en.pdf}. This can be found in  your \gsii\ installation, or in the greenstone3/docs/manual directory if you have installed \gsiii\ from a distribution.
+
+
+\subsubsection{Importing a \gsii\  collection}
+
+
+Pre-built \gsii\ collections can also be used in \gsiii\footnote{For information about the \gsii\  software, and how to build collections using it, visit \gst{www.greenstone.org}}. The collection folder should be copied to the collect directory of the site it is to appear in (or a symbolic link may be used if possible).
+The \gsiii\  run time system requires different configuration files for a collection, so you need to run a conversion script. All this does is create the new collectionConfig.xml and buildConfig.xml from the old collect.cfg and build.cfg files. It does not change the collection in any way, so it can still be used by \gsii\  software.
+
+The conversion script is \gst{convert\_coll\_from\_gs2.pl}. To run it, make sure you have run \gst{source setup.bash} (or \gst{setup} in Windows) in your top-level gsdl directory of the \gsii\  installation (as well as running the standard \gst{gs3-setup} command). Then you need to specify the path to the collect directory and the collection name as parameters to the conversion script. For example, 
+
+\begin{gsc}
+\begin{verbatim}
+convert_coll_from_gs2.pl -collectdir 
+   $GSDL3HOME/web/sites/localsite/collect demo
+\end{verbatim}
+\end{gsc}
+%$
+The script attempts to create \gsiii\  format statements from the old \gsii\  ones. The conversion may not always work properly, so if the collection looks a bit strange under \gsiii\ , you should check the format statements. Format statements are described in Section~\ref{sec:formatstmt}.
+
+Once again, to have the collection recognized by the library servlet, you can either restart Tomcat, or load it dynamically.
+
+\subsection{Collection configuration files}\label{sec:collconfig}
+
+Each collection has two, or possibly three, configuration files, \gst{collectionConfig.xml} and \gst{buildConfig.xml}, and optionally \gst{collectionInit.xml}, that give metadata, display and other information for the
+collection.\footnote{For collections imported from \gsii, \gst{collectionConfig.xml} and \gst{buildConfig.xml}are generated from \gst{collect.cfg} and \gst{build.cfg}.}  The first includes user-defined presentation metadata for the collection,
+such as its name and the {\em About this collection} text; gives formatting information for the collection display; and also gives
+instructions on how the collection is to be built.  The second is produced by
+the build-time process and includes any metadata that can be determined
+automatically. It also includes configuration information for any ServiceRacks needed by the collection.
+
+All the configuration files should be encoded using UTF-8.
+
+\subsubsection{collectionInit.xml}
+
+This optional file is only used for non-standard, customized collections. It specifies the class name of the non-standard collection class.  The only syntax so far is the class name:
+
+\begin{gsc}\begin{verbatim}
+<collectionInit class="XMLCollection"/>
+\end{verbatim}\end{gsc}
+
+Section~\ref{sec:new-coll-types} describes an example collection where this file is used. Depending on the type of collection that this is used for, one or both of the other configuration files may not be needed.
+
+\subsubsection{collectionConfig.xml}
+
+The collection configuration file is where the collection designer (e.g. a librarian) decides what form the collection should take. This includes the collection metadata such as title and description, and also includes what indexes and browsing structures should be built. The format of \gst{collectionConfig.xml} is still under consideration. However, Figure~\ref{fig:collconfig} shows the parts of it that have been defined so far. 
+
+Display elements for a collection or metadata for a document can be entered in any language---use lang='en' attributes to metadata elements to specify which language they are in.
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/
+  schema/ConfigFormat" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
+  <metadataList>
+    <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Greenstone3 MG demo collection</displayItem>
+    <displayItem name='description' lang='en'>This is a demonstration 
+      collection for the Greenstone3 digital library software.</displayItem>
+    <displayItem name='icon' lang='en'>gs3mgdemo.gif</displayItem>
+    <displayItem name='smallicon' lang='en'>gs3mgdemo_sm.gif</displayItem>
+  </displayItemList>
+  <recognise>
+      <docType name="HTML"/><docType name="Text"/>
+      <docType name="Metadata"/><docType name="JPEG"/>
+  </recognise>		    
+  <search type="mg" name="mgsearch">
+    <index name="sectext">
+      <field>text</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">chapters</displayItem>
+      <displayItem name='name' lang="fr">chapitres</displayItem>
+      <displayItem name='name' lang="es">capÃ­tulos</displayItem>
+    </index>
+    [ ... more indexes ...]
+    <format>
+      <gsf:template match="documentNode"><td valign='top'>
+       <gsf:link><gsf:icon/></gsf:link></td><td><gsf:metadata name='Title'/>
+      </td></gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CLTit" type="AZList" horizontalAtTop='true'>
+      <field>Title</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Titles</displayItem>
+    </classifier>
+    [... more classifiers ...]
+    <classifier name="CLKeyword" type="Hierarchy">
+      <field>Keyword</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>HowTo</displayItem>
+      <file URL="keyword.xml"/>
+      <format>
+        <gsf:template match="documentNode">
+          <br /><gsf:link><gsf:metadata name='Keyword' />
+            </gsf:link></gsf:template>
+      </format>
+    </classifier>
+  </browse>
+  <display>
+    <format>
+      <gsf:option name="coverImages" value="false"/>
+      <!--<gsf:option name="documentTOC" value="false"/>-->
+    </format>
+  </display>
+</collectionConfig>
+\end{verbatim}\end{gsc}
+\caption{Sample collectionConfig.xml file (gs3mgdemo collection)}
+\label{fig:collconfig}
+\end{figure}
+
+The \gst{<metadataList>} element specifies some collection metadata, such as creator. The \gst{<displayItemList>} specifies some language dependent information that is used for collection display, such as collection name and short description. These displayItem elements can be specified in different languages. 
+ 
+The \gst{<search>} element specifies what indexes should be built, and provides some display and formatting information for each one. Search has an attribute, \gst{type}, which specifies which indexer to be used for indexing. Currently, \gst{mg} and \gst{mgpp}[??] are available. If type is not specified, mg is used. Multiple search elements may be specified, if more than one indexer is to be used. (Note, this is not yet recognized by the run-time system.)
+
+Search indexes appear as individual \gst{<index>} elements within the \gst{<search>} element. Some choices for the index are made using attributes of the element itself, and some through child elements.  
+
+Each index must have a unique name, which is used to identify it within \gsiii\   The name is given as an attribute of the \gst{<index>} element.  
+
+The other choices are described using child elements of \gst{<index>}.  The \gst{<level>} tag indicates the index level and the \gst{<field>} tag the text to be used.  The \gst{<level>} tag can contain one of document, section or paragraph, while the \gst{<field>} tag can contain ``text'' or the name of a metadata field.  If the \gst{<level>} tag is omitted, the default setting is to index by document, and if the \gst{<field>} tag is omitted, the default setting is to index the document text.
+
+Example index specifications include:
+
+[NOTE: I think we shouldn't have default level and field and that it must be specified--kjdon]
+
+To index only the title of each separate document in the collection:
+\begin{gsc}\begin{verbatim}
+<index name="dtt">
+  <level>document</level>
+  <field>dc:title</field>
+</index>
+\end{verbatim}\end{gsc}
+...in this case the \gst{<field>} tag refers to the ``title'' metadata item, found in the Dublin Core namespace.  The MG search engine would be used on this index.
+
+Alternatively, to index the full document texts by section:
+\begin{gsc}\begin{verbatim}
+<index name="stx">
+  <level>section</level>
+</index>
+\end{verbatim}\end{gsc}
+...or...
+\begin{gsc}\begin{verbatim}
+<index name="stx">
+  <level>section</level>
+  <field>text</field>
+</index>
+\end{verbatim}\end{gsc}
+...in the first example, the \gst{<field>} tag is not explicitly defined, and would default to 'text', whereas it is explicitly set to 'text' in the second example. As they are of the same name, they should not appear in the same \gst{collectionConfig.xml} file.
+
+Moving onto \gst{<classifier>} items, the format is broadly similar to \gst{<index>} items, but with a couple of different choices.  Firstly, each classifier should have ``name'' and ``type'' attributes.  In the case of \gst{<classifier>} items the ``type'' attribute identifies the type of classifier it is.  At present, this should either be ``Hierarchy'' or ``AZList''.  
+
+The remaining choices for the classifier should follow as child elements of the \gst{<classifier>} element.  The \gst{<file>} element should contain the name of the file that describes the classifier as its ``URL'' attribute.  The format of this file varies from classifier type to classifier type.  The \gst{<field>} element identifies the name of the field to index.  More than one \gst{<field>} element may appear if two or more metadata fields are to be used with the classifier.  Finally, the \gst{<sort>} item identifies another metadata field which the items within one classifier node are to be ordered.  Unlike the \gst{<index>} element, the \gst{<classifier>} element does not have default, assumed values for its children.
+
+Figure~\ref{fig:hierarchyfile} shows the format of the file for a Hierarchy classifier. [TODO add a  description]
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<Hierarchy>
+  <Classification>
+    <Name>ACCU</Name>
+    <Path>1</Path>
+    <Description>ACCU</Description>
+  </Classification>
+  <Classification>
+    <Name>Agenda 21</Name>
+    <Path>2</Path>
+    <Description>Agenda 21</Description>
+  </Classification>
+  <Classification>
+    <Name>FAO</Name>
+    <Path>3</Path>
+    <Description>FAO</Description>
+    <Children>
+      <Classification>
+        <Name>FAO Better Farming series</Name>
+        <Path>3.1</Path>
+        <Description>FAO Better Farming Series</Description>
+      </Classification>
+    </Children>
+  </Classification>
+</Hierarchy>
+\end{verbatim}\end{gsc}
+\caption{Sample Hierarchy classifier file}
+\label{fig:hierarchyfile}
+\end{figure}
+
+Inside the \gst{<search>} and \gst{<browse>} elements, \gst{<displayItem>} elements are used to provide titles for the indexes or classifiers, while \gst{<format>} elements provide formatting instructions, typically for a document or classifier node in a list of results. Placing the \gst{<format>} instructions at the top level in the search or browse element will apply the format to all the indexes or classifiers, while placing it inside an individual index or classifier element will restrict that formatting instruction to that item.
+
+The \gst{<display>} element contains optional formatting information for the display of documents. Templates that can be specified here include \gst{documentHeading}, \gst{DocumentContent}. Other formatting options may also be specified here, such as whether to display a table of contents and/or cover image for the documents.
+
+Format elements are described in Section~\ref{sec:formatstmt}.
+
+An optional \gst{<replaceList>} element can be included at the top level. This contains a list of strings and their replacements. This is particularly useful for Greenstone 2 collections that use macros. 
+
+The format is like the following:
+\begin{gsc}\begin{verbatim}
+<replaceList>
+<replace scope='text' macro="xxx" text="yyy"/>
+<replace scope='metadata' macro="xxx" bundle="yyy" key="zzz"/>
+<replace scope='all' macro='xxx' metadata='yyy'/>
+</replaceList>
+\end{verbatim}\end{gsc}
+
+Scope determines on what text the replacements are carried out: text, metadata, or both (all). An empty scope attribute is equivalent to scope=all. Each replace type can be used with all scope values. Replacing uses Java's 'String.replaceAll' functionality, so macro and replacement text are actually regular expressions. The first example is a straight textual replacement. The second example uses dictionary lookups. xxx will be replaced with the (language-dependent) value for key zzz in resource bundle yyy. The third example uses metadata: xxx will be replaced by the value of the yyy metadata for that document.
+
+Appendix~\ref{app:gs2replace} gives some examples that have been used for Greenstone 2 collections.
+
+\subsubsection{buildConfig.xml}\label{sec:buildconfig}
+
+The file \gst{buildConfig.xml} is produced by the collection building process. Generally it is not necessary to look at this file, but it can be useful in determining what went wrong if the collection doesn't appear quite the way it was planned.
+
+It contains metadata and other information about the collection that can
+be determined automatically, such as the number of
+documents it contains.  It also includes a list of ServiceRack classes that are
+required to provide the services that have been built into the
+collection.  The serviceRack names are Java classes that are loaded
+dynamically at runtime. Any information inside the serviceRack element is
+specific to that service---there is no set format. Figure~\ref{fig:buildconfig} shows an example. This configuration file specifies that the collection should load up 3 ServiceRacks: \gst{GS2Browse}, \gst{GS2MGPPRetrieve} and \gst{GS2MGPPSearch}. The contents of each \gst{<serviceRack>} element are passed to the appropriate ServiceRack objects for configuration. The \gst{collectionConfig.xml} file content is also passed to the ServiceRack objects at configure time---the \gst{format} and \gst{displayItem} information is used directly from the \gst{collectionConfig.xml} file rather than added into \gst{buildConfig.xml} during building. This enables formatting and metadata changes in \gst{collectionConfig.xml} to take effect in the collection without rebuilding being necessary. However, as these files are cached, the collection needs to be reloaded for the changes to appear in the library.
+
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<buildConfig>
+  <metadataList>
+    <metadata name="numDocs">11</metadata>
+    <metadata name="buildType">mgpp</metadata>
+  </metadataList>
+  <serviceRackList>
+    <serviceRack name="GS2Browse">
+      <classifierList>
+        <classifier name="CL1" content="Title"/>
+        <classifier name="CL2" content="Subject" />
+        <classifier name="CL3" content="Organization" />
+        <classifier name="CL4" content="Howto" />
+      </classifierList>
+    </serviceRack>
+    <serviceRack name="GS2MGPPRetrieve">
+      <defaultLevel name="Sec" />
+    </serviceRack>
+    <serviceRack name="GS2MGPPSearch">
+      <defaultLevel name="Sec" />
+      <levelList>
+        <level name="Sec" />
+        <level name="Doc" />
+      </levelList>
+      <fieldList>
+        <field shortname="ZZ" name="allfields" />
+        <field shortname="TX" name="text" />
+        <field shortname="DL" name="dls.Title" />
+        <field shortname="DS" name="dls.Subject" />
+        <field shortname="DO" name="dls.Organization" />
+      </fieldList>
+      <searchTypeList>
+        <searchType name="form" />
+        <searchType name="plain" />
+      </searchTypeList>
+      <defaultIndex name="idx" />
+      <indexList>
+        <index name="idx" />
+      </indexList>
+    </serviceRack>
+  </serviceRackList>
+\end{verbatim}\end{gsc}
+\caption{Sample buildConfig.xml file (gs2mgppdemo collection)}
+\label{fig:buildconfig}
+\end{figure}
+
+\subsection{Formatting the collection}\label{sec:formatstmt}
+
+Part of collection design involves deciding how the collection should look. \gsiii\  has a default 'look' for a collection, so this is optional. However, the default may not suit the purposes of some collections, so many parts to the look of a collection can be determined by the collection designer.
+
+In standard \gsiii\ , the library is served to a web browser by a servlet, and the HTML is generated using XSLT. XSLT templates are used to format all the parts of the pages. These templates can be overridden by including them in the \gst{collectionConfig.xml} file. Some commonly overridden templates are those for formatting lists: search results list, classifier browsing hierarchies, and for parts of the document display. 
+
+Real XSLT templates for formatting search results or classifier lists are quite complicated, and not at all easy for a new user to write. For example, the following is a sample template for formatting a classifier list, to show Keyword metadata as a link to the document.
+ 
+\begin{gsc}\begin{verbatim}
+<xsl:template match="documentNode" priority="2"
+     xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:param name="collName"/>
+    <td><a href="{\$library_name}?a=d&amp;c={\$collName}&amp;
+           d={@nodeID}&amp;dt={@docType}"><xsl:value-of 
+           select="metadataList/metadata[@name='Keyword']"/></a>
+    </td>
+</xsl:template>
+ \end{verbatim}\end{gsc}
+ 
+To write this, the user would need to know that:
+\begin{bulletedlist}
+\item the variable \gst{\$library\_name} exists,
+\item the collection name is passed in as a parameter called \gst{collName}
+\item metadata for a document is found in a \gst{<metadataList>} and that its form is \gst{<metadata name="Keyword">the value</metadata>}
+\item the arguments needed for the link to the document are \gst{a, sa, c, d, a, dt}.
+\end{bulletedlist}
+ 
+Since XSLT is written in XML, we can use XSLT to transform XML into XSLT. \gsiii\  provides a simplified set of formatting commands, written in XML, which will be transformed into proper XSLT. The user specifies a \gst{<gsf:template>} for what they want to format---these typically match \gst{documentNode} or \gst{classifierNode} (for node in a classification hierarchy).
+ 
+The template at the start of this section can be represented as:
+ 
+\begin{gsc}\begin{verbatim}
+<gsf:template match='documentNode'>
+  <td><gsf:link><gsf:metadata name='Keyword'/></gsf:link></td>
+</gsf:template>
+\end{verbatim}\end{gsc}
+
+Table~\ref{tab:gsf-format} shows the set of 'gsf' (Greenstone Format) elements. If you have come from a \gsii\  background, Appendix~\ref{app:gs2format} shows \gsii\  format elements and their equivalents in \gsiii\ .
+ 
+\begin{table}
+\caption{Format elements for GSF format language}
+\label{tab:gsf-format}
+{\footnotesize
+\begin{tabular}{p{6.5cm}p{6.5cm}}
+\hline
+\bf Element       & \bf Description \\
+\hline
+\gst{<gsf:text/>} & The document's text\\
+\hline
+\gst{<gsf:link>...</gsf:link>} & The HTML link to the document itself \\
+\gst{<gsf:link type='document'>...
+</gsf:link>} & Same as above\\
+\gst{<gsf:link type='classifier'>...
+</gsf:link>} & A link to a classification node (use in classifierNode templates)\\
+\gst{<gsf:link type='source'>...
+</gsf:link>} & The HTML link to the original file---set for documents that have been converted from e.g. Word, PDF, PS \\
+\hline
+\gst{<gsf:icon/>}  & An appropriate icon\\
+\gst{<gsf:icon type='document'/>} & same as above\\
+\gst{<gsf:icon type='classifier'/>} & bookshelf icon for classification nodes\\
+\gst{<gsf:icon type='source'/>} & An appropriate icon for the original file e.g. Word, PDF icon\\
+\hline
+\gst{<gsf:metadata name='Title'/>} & The value of a metadata element for the current document or section, in this case, Title\\
+\gst{<gsf:metadata name='Title' select='select-type' [separator='y' multiple='true']/>} & A more extended selection of metadata values. The select field can be one of those shown in Table~\ref{tab:gsf-select-types}. There are two optional attributes: separator gives a String that will be used to separate the fields, default is ``, ``, and if multiple is set to true, looks for multiple values at each section.\\
+\gst{<gsf:metadata name='Date' format='formatDate'/>} & The value of a metadata element for the current document, formatted in some way. Current formatting options available are formatDate: turns '20040201' into '01 February 2004', and formatLanguage: turns 'en' into 'English', both in a language dependent manner. \\
+\hline
+\gst{<gsf:choose-metadata>
+  <gsf:metadata name='metaA'/>
+  <gsf:metadata name='metaB'/>
+  <gsf:metadata name='metaC'/>
+</gsf:choose-metadata>}
+ & A choice of metadata. Will select the first existing one. the metadata elements can have the select, separator and multiple attributes like normal.\\
+\hline
+\gst{<gsf:switch preprocess=
+'preprocess-type'>
+<gsf:metadata name='Title'/> 
+<gsf:when test='test-type'
+test-value='xxx'>...</gsf:when>
+<gsf:when test='test-type' 
+test-value='yyy'>...</gsf:when>
+<gsf:otherwise>...</gsf:otherwise>
+</gsf:switch>} & switch on the value of a particular metadata - the metadata is specified in gsf:metadata, has the same attributes as normal.\\
+\hline
+\end{tabular}}
+\end{table}
+
+The \gst{<gsf:metadata>} elements are used to output metadata values. The simplest case is \gst{<gsf:metadata name='Title'/>}---this outputs the Title metadata for the current document or section. Namespaces are important here: if the Title metadata is in the Dublin Core (dc) namespace, then the element should look like \gst{<gsf:metadata name='dc.Title'/>}. There are three other attributes for this element. The attribute \gst{multiple} is used when there may be more than one value for the selected metadata. 
+For instance, one document may fall into several classification categories, and therefore may have multiple Subject metadata values. Adding \gst{multiple='true'} to the \gst{<gsf:metadata>} element will retrieve all values, not just the first one. Multiple values are separated by commas by default. The \gst{separator} attribute is used to change the separating string. For example, adding \gst{separator=':~'} to the element will separate all values by a colon and a space.
+
+Sometimes you may want to display metadata values for sections other than the current one. For example, in the mgppdemo collection, in a search list we display the Titles of all the enclosing sections, followed by the Title of the current section, all separated by semi-colons. The display ends up looking something like:
+\emph{Farming snails 2; Starting out; Selecting your snails}
+where \emph{Selecting your snails} is the Title of the section in the results list, and \emph{Farming snails 2} and \emph{Starting out} are the Titles of the enclosing sections. The \gst{select} attribute is used to display metadata for sections other than the current one. Table~\ref{tab:gsf-select-types} shows the options available for this attribute. The \gst{separator} attribute is used here also, to specify the separating text.
+
+To get the previous metadata, the format statement would have the following in it:
+
+\begin{gsc}
+\begin{verbatim}
+<gsf:metadata name='Title' select='ancestors' separator='; '/>; 
+    <gsf:metadata name='Title'/>
+\end{verbatim}
+\end{gsc}
+
+\begin{table}
+\caption{Select types for metadata format elements}
+\label{tab:gsf-select-types}
+{\footnotesize
+\begin{tabular}{ll}
+\hline
+\bf Select Type & \bf Description\\
+\hline
+current & The current section \\
+parent & The immediate parent section\\
+ancestors & All the parents back to the root (topmost) section\\
+root & The root or topmost section \\
+siblings & All the sibling sections\\
+children & The immediate child sections of the current section\\
+descendents & All the descendent sections\\
+\hline
+\end{tabular}}
+\end{table}
+
+The \gst{<gsf:choose-metadata} element selects the first available metadata value from the list of options. 
+\begin{gsc}
+\begin{verbatim}
+<gsf:choose-metadata>
+  <gsf:metadata name='dc.Title'/>
+  <gsf:metadata name='dls.Title'/>
+  <gsf:metadata name='Title'/>
+</gsf:choose-metadata>
+\end{verbatim}
+\end{gsc}
+
+This will display the dls.Title metadata if available, otherwise it will use the dc.Title metadata if available, otherwise it will use the Title metadata. If there are no values for any of these metadata elements, then nothing will be displayed.
+
+The \gst{<gsf:switch>} element allows different formatting depending on the value of a specified metadata element. For example, the following switch statement could be used to display a different icon for each document in a list depending on which organization it came from. 
+
+\begin{gsc}
+\begin{verbatim}
+<gsf:switch  preprocess='toLower;stripSpace'>
+  <gsf:metadata name='Organization'/>
+  <gsf:when test='equals' test-value='bostid'>
+     <!-- output BOSTID image --></gsf:when>
+  <gsf:when test='equals' test-value='worldbank'>
+     <!-- output world bank image --></gsf:when>
+  <gsf:otherwise><!-- output default image--></gsf:otherwise>
+</gsf:switch>
+\end{verbatim}
+\end{gsc}
+
+Preprocessing of the metadata value is optional. The preprocess types are \gst{toLower} (make the value lowercase), \gst{toUpper} (make the value uppercase), \gst{stripSpace} (removes any whitespace from the value). These operations are carried out on the value of the selected metadata before the test is carried out. Multiple processing types can be specified, separated by ; and they will be applied in the order specified (from left to right).
+
+Each option specifies a test and a test value. Test values are just text. Tests include \gst{startsWith}, \gst{contains}, \gst{exists}, \gst{equals}, \gst{endsWith}. Exists doesn't need a test value. Having an otherwise option ensures that something will be displayed even when none of the tests match.
+
+If none of the gsf elements meets your needs for formatting, XSLT can be entered directly into the format element, giving the collection designer full flexibility over how the collection appears. 
+
+The collection specific templates are added into the configuration file \gst{collectionConfig.xml}. Any templates found in the XSLT files can be overridden.
+The important part to adding templates into the configuration file is determining where to put them. Formatting templates cannot go just anywhere---there are standard places for them. Figure~\ref{fig:format-places} shows the positions that templates can occur.
+
+\begin{figure}
+\begin{gsc}\begin{verbatim}
+<collectionConfig>
+  <metadataList/>
+  <displayItemList/>
+  <search>
+    <format> <!--Put here templates related to searching and 
+        the query page. The common one is the documentNode 
+        template -->
+      <gsf:template match='documentNode'>...</gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name='xx'>
+      <format><!-- put here templates related to formating a 
+        particular classifier page. Common ones are documentNode 
+        and classifierNode templates-->
+        <gsf:template match='documentNode'>...</gsf:template>
+        <gsf:template match='classifierNode'>...</gsf:template>
+        <gsf:template match='classifierNode' mode='horizontal'>...
+          </gsf:template>
+      </format>
+    </classifier>
+    <classifier>...</classifier>
+    <format><!-- formatting for all the classifiers. these will
+      be overridden by any classifier specific formatting 
+      instructions --></format>
+  </browse>
+  <display>
+    <format><!-- here goes any formatting relating to the display 
+        of the documents. These are generally named templates, 
+        and format options -->
+      <gsf:template name='documentContent'>...</gsf:template>
+      <gsf:option name='TOC' value='true'/>
+    </format>
+  </display>
+</collectionConfig>
+\end{verbatim}\end{gsc}
+\caption{Places for format statements}
+\label{fig:format-places}
+\end{figure} 
+
+ 
+There are also formatting instructions that are not templates but are options. 
+These are described in Table~\ref{tab:format_options}. They are entered into the configuration file like \gst{<gsf:option name='coverImages' value='false'/>}
+
+\begin{table}
+\caption{Formatting options}
+\label{tab:format_options}
+{\footnotesize
+\begin{tabular}{llp{5cm}}
+\hline
+\bf option name & \bf values & \bf description \\
+\hline
+coverImages & true, false & whether or not to display cover images for documents \\
+TOC & true, false & whether or not to display the table of contents for the document\\
+\hline
+\end{tabular}}
+\end{table}
+
+Note, format templates are added into the XSLT files before transforming, while the options are added into the page source, and used in tests in the XSLT.
+\subsubsection{Changing the service text strings}
+
+Each collection has a set of services which are the access points for the information in the collection. Each service has a set of text strings which are used to display it. These include name, description, the text on the submit button, and names and descriptions of all the parameters to the service.
+
+These text strings are found in .properties files, in greenstone3/resources/java. The names of the files are based on class names. Subclasses can defined their own properties, or can use their parent class ones. For example, AbstractSearch defines strings for the TextQuery service, in AbstractSearch.properties. GS2MGSearch just uses these default ones, so doesn't need its own property file.
+
+A particular collection can override the properties for any service. For example, if a collection uses the GS2MGSearch service rack (look in the buildConfig.xml file for a list of service racks used), and the collection builder wants to change the text associated with this service, they can put a GS2MGSearch.properties file in the resources directory of the collection. 
+This will be used in preference to one in the default resources directory.
+Note that while changes in the default properties files seem to require a tomcat restart to take effect, changes in the collection specific properties files take effect immediately.
+
+\subsection{Customizing the interface}\label{sec:interface-customise}
+
+Format statements in the collection configuration files provide a way to change small parts of the collection display. For large scale customizations to a collection, or ones that apply to a site as a whole, a second mechanism is available. The interface is defined by a set of XSLT files that transform the page data into HTML. Any of these files can be overridden to provide specialized display, on a site or collection basis. 
+
+The first section looks at customizing the existing interface, while the second section looks at defining a whole new interface. The last section describes how to add a new language translation of an interface.
+
+\subsubsection{Modifying an existing interface}
+
+Most of an interface is defined by XSLT files, which are stored in \gst{\$GSDL3HOME/\-web/\-interfaces/\-interface-name/\-transform}. These can be changed and the changes will take effect straight away. If changes only apply to certain collections or sites, not everything that uses the interface, you can override some of the files by putting new ones in a different place. XSLT files are looked for in the following  order: collection, site, interface, default interface. (This currently only apples to sites, and therefore collections, that reside in the same \gs\  installation as the interface.) 
+
+Sites and collections can have a transform directory, which is where customized XSLT files should go. Any XSLT files in here will be used in preference to the interface files when using this collection. For example, if you want to have a completely different layout for the about page of a collection, you can put a new \gst{about.xsl} file into the collection's \gst{transform} directory, and this will be used instead. This is what we do for the Gutenberg sample collection.
+
+This also applies to files that are included from other XSLT files. For example the query.xsl for the query pages includes a file called querytools.xsl. To have a particular site show a different query interface either of these files may need to be modified. Creating a new version of either of these and putting it in the site transform directory will work. Either the new query.xsl will include the default querytools, or the default query.xsl will include the new querytools.xsl. The xsl:include directives are preprocessed by the Java code and full paths added based on availability of the files, so that the correct one is used.
+
+Note that you cannot include a file with the same name as the including file. For example query.xsl cannot include query.xsl (it is tempting to want to do this if you just want to change one template for a particular file, and then include the default. but you cant).
+
+\subsubsection{Defining a new interface}
+
+A new interface may be needed if different instantiations of the library require different interfaces, or different developers want their own look and feel. Creating a new interface will allow modifications to be made while leaving the original one intact. 
+
+A new interface needs a directory in \gst{\$GSDL3HOME/web/interfaces}, the name of this directory becomes the interface name. Inside, it needs images and transform directories,  and an interfaceConfig.xml file. Any XSLT may be overridden for a new interface by putting the replacement in the new transform directory. If the appropriate XSLT file is not there, the  one from the default interface will be used - this enables just overriding a few XSLT files as needed.
+
+To use a new interface, the Tomcat web.xml must be edited: either change the interface that a current servlet instance is using, or add another servlet instantiation to the file (see Section~\ref{sec:sites-and-ints} or Appendix~\ref{app:tomcat}). The Tomcat server must be restarted for this to take effect.
+
+\subsubsection{Changing the interface language}
+
+The interface language can be changed by going to the preferences page, and choosing a language from the list, which includes all languages into which the interface has been translated.
+
+It is easy to add a new interface language to \gs\ .  Language specific text strings are separated out from the rest of the system to allow for easy incorporation of new languages. These text strings are contained in Java resource bundle properties files. These are plain text files consisting of key-value pairs, located in \gst{resources/java}. Each interface has one named \gst{interface\_name.properties} (where `name' is the interface name). Each service class has one with the same name as the class (e.g. \gst{GS2Search.properties}). To add another language all of the base .properties  files must be translated. The translated files keep the same names, but with a language extension added. For example, a French version of \gst{interface\_default.properties} would be named \gst{interface\_default\_fr.properties}.
+
+Keys will be looked up in the properties file closest to the specified language. For example, if language \gst{fr\_CA} was specified (French language, country Canada), and the default locale was \gst{en\_GB},  Java would look at properties files in the following order, until it found the key: \gst{XXX\_fr\_CA.properties}, \gst{XXX\_fr.properties},  \gst{XXX\_en\_GB.properties}, then \gst{XXX\_en.properties}, and finally the default \gst{XXX.properties}.
+
+These new files are available straight away---to use the new language, add e.g. \gst{l=fr} to the arguments in the URL. To get \gs\ to add it in to the list of languages on the preferences page, an entry needs to be added into the languages list in the \gst{interfaceConfig.xml} file (see Section~\ref{sec:interfaceconfig}). Modification of this file requires a restart of the Tomcat server for the changes to be recognized.
+
+\newpage 
+\section{Developing \gsiii : Run-time system}\label{sec:develop-runtime}
+
+[TODO: rewrite this!!]
+runtime object structure diagram. describe the modules.\\ 
+class hierarchy,\\
+directory structure and where everything lives\\
+message format.\\
+overall description of message passing sequence.\\
+configuration process - start up and runtime\\
+\\
+page generation\\
+accessing the javadoc\\
+
+\subsection{Overview of modules??}
+
+A \gsiii\  'library' system consists of many components: MessageRouter, Receptionist, Actions, Collections, ServiceRacks etc.  Figure~\ref{fig:local} shows how they fit together in a stand-alone system. The top left part is concerned with displaying the data, while the bottom right part is the collection data serving part. The two sides communicate through the MessageRouter. There is a one-to-one correspondence between modules and Java classes, with the exception of services: for coding and/or run-time efficiency reasons, several Service modules may be grouped together into one ServiceRack class.
+
+\begin{figure}[t]
+  \centering
+  \includegraphics[width=4in]{local} %5.8
+  \caption{A simple stand-alone site.}
+  \label{fig:local}
+\end{figure}
+
+
+{\em MessageRouter}: this is the central module for a site. It controls the site, loading up all the collections, clusters, communicators needed. All messages pass through the MessageRouter. Communication between remote sites is always done between MessageRouters, one for each site.
+
+{\em Collection and ServiceCluster}: these are very similar, and group a set of services into a conceptual group.. They both provide some metadata about the collection/cluster, and a list of services. The services are provided by ServiceRack objects that the collection/cluster loads up. A Collection is a specific type of ServiceCluster. A ServiceCluster groups services that are related conceptually, e.g. all the building services may be part of a cluster. What is part of a cluster is specified by the site configuration file. A Collection's services are grouped by the fact that they all operate on some common data---the documents in the collection.
+Functionally Collection and ServiceCluster are very similar, but conceptually, and to the user, they are quite different.
+
+{\em Service}: these provide the core functionality of the system e.g. searching, retrieving documents, building collections etc. One or more may be grouped into a single Java class (ServiceRack) for code reuse, or to avoid instantiating the same objects several times. For example, MGPP searching services all need to have the index loaded into memory. 
+
+{\em Communicator/Server}: these facilitate communication between remote modules. For example, if you want MR1 to talk to MR2, you need a Communicator-Server pair. The Server sits on top of MR2, and MR1 talks to the Communicator. Each communication type needs a new pair. So far we have only been using SOAP, so we have a SOAPCommunicator and a SOAPServer.
+
+{\em Receptionist}: this is the point of contact for the 'front end'. Its core functionality involves routing requests to the Actions, but it may do more than that. For example, a Receptionist may: modify the request in some way before sending it to the appropriate Action; add some data to the page responses that is common to all pages; transform the response into another form using XSLT. There is a hierarchy of different Receptionist types, which is described in Section~\ref{sec:recepts}.
+
+{\em Actions}: these do the job of creating the 'pages'. There is a different action for each type of page, for example PageAction handles semi-static pages, QueryAction handles queries, DocumentAction displays documents. They know a little bit about specific service types. Based on the 'CGI' arguments passed in to them, they construct requests for the system, and put together the responses into data for the page. This data is returned to the Receptionist, which may transform it to HTML. The various actions are described in  more detail in Section~\ref{sec:pagegen}.
+
+
+\subsection{Start up configuration}\label{sec:startup-config}
+
+We use the Tomcat web server, which operates either stand-alone in a test mode
+or in conjunction with the Apache web server.  The \gs\  LibraryServlet
+class is loaded by Tomcat  and the servlet's \gst{init()} method is called.  Each time a
+\gst{get/put/post} (etc.) is used, a new thread is started and
+\gst{doGet()/doPut()/doPost()} (etc.) is called.
+
+The \gst{init()} method creates a new Receptionist and a new 
+MessageRouter. Default classes (DefaultReceptionist, MessageRouter) are used unless subclasses have been specified  in the servlet initiation parameters (see Section~\ref{sec:sites-and-ints}). The appropriate system variables are set for each object (interface
+name, site name, etc.) and then \gst{configure()} is called on both. The MessageRouter handle 
+is passed to the Receptionist. The servlet then communicates only with
+the Receptionist, not with the MessageRouter.
+
+The Receptionist reads in the \gst{interfaceConfig.xml} file (see Section~\ref{sec:interfaceconfig}), and loads up all the different Action classes. Other Actions may be loaded on the fly as needed. Actions are added to a map, with shortnames for keys. Eg the QueryAction is added with key 'q'. The Actions are passed the MessageRouter reference too.
+If the Receptionist is a TransformingReceptionist, a mapping between shortnames  and XSLT file names is also created. 
+
+The MessageRouter reads in its site configuration file \gst{siteConfig.xml} (see Section~\ref{sec:siteconfig}). It creates a module map that maps names to objects. This is used for routing the messages. It also keeps small chunks of XML---serviceList, collectionList, clusterList and siteList. These are part of what get returned in response to a describe request (see Section~\ref{sec:describe}.).
+
+Each ServiceRack specified in the configuration file is created, then queried for its list of services. Each service name is added to the map, pointing to the ServiceRack object. Each service is also added to the serviceList. After this stage, ServiceRacks are transparent to the system, and each service is treated as a separate module.
+
+ServiceClusters are created and passed the \gst{<serviceCluster>} element for configuration. They are added to the map as is, with the cluster name as a key. A serviceCluster is also added to the serviceClusterList.
+
+For each site specified, the MessageRouter creates an appropriate type of Communicator object. Then it tries to get the site description. If the server for the remote site is up and running, this should  be successful. The site will be added to the mapping with its site name as a key. The site's collections, services and clusters will also be added into the static xml lists. If the server for the remote site is not running, the site will not be included in the siteList or module map. To try again to access the site, either Tomcat must be restarted, or a run-time reconfigure-site command must be sent (see Section~\ref{sec:runtime-config}).
+
+The MessageRouter also looks inside the site's \gst{collect} directory, and  loads up a Collection object for each valid collection found. If a \gst{collectionInit.xml} file is present, a subclass of Collection may be used.
+The Collection object reads its \gst{buildConfig.xml} and \gst{collectionConfig.xml}
+files, determines the metadata, and loads ServiceRack classes based on the 
+names specified in \gst{buildConfig.xml\/}. The \gst{<serviceRack>} XML element is passed to the object to be used in configuration. The \gst{collectionConfig.xml} contents are also passed in to the ServiceRacks. Any format or display information that the services need must be extracted from the collection configuration file.
+Collection objects are added to the module map with their name as a key, and also a collection element is added into the collectionList XML.
+
+\subsection{Message passing}
+
+There are two types of messages used by the system: external and internal messages. All messages have an enclosing \gst{<message>} element, which contains either one or more requests, or one or more responses. In the following descriptions, the message element is not shown, but is assumed to be present.  
+Action in \gsiii\  is originated by a request coming in from the outside. In the standard web-based \gs, this comes from a servlet and is passed into the Receptionist. This ``external'' type request is a request for a page of data, and contains a representation of the CGI style arguments. A page of XML is returned, which can be in HTML format or other depending on the output parameter of the request. 
+
+Messages inside the system (``internal'' messages) all follow the same basic format: message elements contain multiple request elements, or multiple response elements. Messaging is all synchronous. The same number of responses as requests will be returned. Currently all requests are independent, so any requests can be combined into the same message, and they will be answered separately, with their responses being sent back in a single message.
+
+When a page request (external request) comes in to the Receptionist, it looks at the action attribute and passes the request to the appropriate Action module. The Action will fire one or more internal requests to the MessageRouter, based on the arguments. The data is gathered into a  response, which is returned to the Receptionist.  The page that the receptionist returns contains the original request, the response from the action and other info as needed (depends on the type of Receptionist). The data may be transformed in some way --- for the \gs\ servlet  we transform using XSLT to generate HTML pages.
+
+Actions send internal style messages to the MessageRouter. Some can be answered by it, others are passed on to collections, and maybe on to services. Internal requests are for simple actions, such as search, retrieve metadata, retrieve document text
+There are different internal request types: describe, process, system, format, status. Process requests do the actual work of the system, while the other types get auxiliary information. The format of the requests and responses for each internal request type are described in the following sections. External style requests, and their page responses are described in the Section about page generation (Section~\ref{sec:pagegen}).
+
+\subsection{'describe'-type messages}\label{sec:describe}
+
+The most basic of the internal standard requests is ``describe-yourself'', which can be sent to any module in the system. The module responds with a semi-predefined piece of XML, making these requests very efficient. The response is predefined apart from any language-specific text strings, which are put together as each request comes in, based on the language attribute of the request.
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang='en' type='describe' to=''/>
+\end{verbatim}\end{gsc}\end{quote}
+If the \gst{to} field is empty, a request is answered by the MessageRouter. 
+An example response from a MessageRouter might look like this:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<response lang='en' type='describe'>
+  <serviceList/>
+  <siteList>
+    <site name='org.greenstone.gsdl1'
+            address='http://localhost:8080/greenstone3/services/localsite'
+            type='soap' />
+  </siteList>
+  <serviceClusterList>
+    <serviceCluster name="build" />
+  </serviceClusterList>
+  <collectionList>
+    <collection name='org.greenstone.gsdl1/
+  	              org.greenstone.gsdl2/fao' />
+    <collection name='org.greenstone.gsdl1/demo' />
+    <collection name='org.greenstone.gsdl1/fao' />
+    <collection name='myfiles' />
+  </collectionList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+This MessageRouter has no individual site-wide services (an empty \gst{<serviceList>}), but has a service cluster called build (which provides collection importing and building functionality).  It
+communicates with one site, \gst{org.greenstone.gsdl1}.  It is aware of four
+collections.  One of these, \gst{myfiles}, belongs to it; the other three are
+available through the external site.  One of those collections is actually from
+a further external site.
+
+It is possible to ask just for a specific part of the information provided by a
+describe request, rather than the whole thing. For example, these two
+messages get the \gst{collectionList} and the \gst{siteList} respectively: 
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang='en' type='describe' to=''>
+  <paramList>
+    <param name='subset' value='collectionList'/>
+  </paramList>
+</request>
+
+<request lang='en' type='describe' to=''>
+  <paramList>
+    <param name='subset' value='siteList'/>
+  </paramList>
+</request>
+\end{verbatim}\end{gsc}\end{quote}
+
+Subset options for the MessageRouter include \gst{collectionList}, \gst{serviceClusterList}, \gst{serviceList}, \gst{siteList}.
+
+When a collection or service cluster is asked to describe itself, what is returned is a list of metadata, some display elements, and  a list of services.  For example, here is such a message, along with a sample response.
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang='en' type='describe' to='mgppdemo'/>
+
+<response from="mgppdemo" type="describe">
+  <collection name="mgppdemo">
+    <displayItem lang="en" name="name">greenstone mgpp demo
+    </displayItem>
+    <displayItem lang="en" name="description">This is a 
+      demonstration collection for the Greenstone digital 
+      library software. It contains a small subset (11 books) 
+      of the Humanity Development Library. It is built with 
+      mgpp.</displayItem>
+    <displayItem lang="en" name="icon">mgppdemo.gif</displayItem>
+    <serviceList>
+      <service name="DocumentStructureRetrieve" type="retrieve" />
+      <service name="DocumentMetadataRetrieve" type="retrieve" />
+      <service name="DocumentContentRetrieve" type="retrieve" />
+      <service name="ClassifierBrowse" type="browse" />
+      <service name="ClassifierBrowseMetadataRetrieve" 
+            type="retrieve" />
+      <service name="TextQuery" type="query" />
+      <service name="FieldQuery" type="query" />
+      <service name="AdvancedFieldQuery" type="query" />
+      <service name="PhindApplet" type="applet" />
+    </serviceList>
+    <metadataList>
+      <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+      <metadata name="numDocs">11</metadata>  
+      <metadata name="buildType">mgpp</metadata>
+      <metadata name="httpPath">http://kanuka:8090/greenstone3/sites/
+                                localsite/collect/mgppdemo</metadata>
+    </metadataList>
+  </collection>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+Subset options for a collection or serviceCluster include \gst{metadataList}, \gst{serviceList}, and \gst{displayItemList}.
+
+This collection provides many typical services. Notice how this response lists the services available, while the collection configuration file for this collection (Figure~\ref{fig:collconfig}) described serviceRacks. Once the service racks have been configured, they become transparent in the system, and only services are referred to.
+There are three document retrieval services, for structural information, metadata, and content. The Classifier services retrieve classification structure and metadata. These five services were all provided by the GS2MGPPRetrieve ServiceRack. The three query services were provided by GS2MGPPSearch serviceRack, and provide different kinds of query interface. The last service, PhindApplet, is provided by the PhindPhraseBrowse serviceRack and is an applet service.
+
+A \gst{describe} request sent to a service returns a list of parameters that
+the service accepts and some display information, (and in future may describe the content type for the request and response). Subset options for the request include \gst{paramList} and \gst{displayItemList}.
+
+Parameters can be in the following formats:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<param name='xxx' type='integer|boolean|string|invisible' default='yyy'/>
+<param name='xxx' type='enum_single|enum_multi' default='aa'/>
+  <option name='aa'/><option name='bb'/>...
+</param>
+<param name='xxx' type='multi' occurs='4'>
+ <param .../>
+ <param .../>
+</param>
+\end{verbatim}\end{gsc}\end{quote}
+
+If no default is specified, the parameter is assumed to be mandatory.
+Here are some examples of parameters:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<param name='case' type='boolean' default='0'/>
+
+<param name='maxDocs' type='integer' default='50'/>
+
+<param name='index' type='enum' default='dtx'>
+  <option name='dtx'/>
+  <option name='stt'/>
+  <option name='stx'/>
+<param>
+
+<!-- this one is for the text box and field list for the 
+simple field query-->
+<param name='simpleField' type='multi' occurs='4'>
+  <param name='fqv' type='string'/>
+  <param name='fqf' type='enum_single'>
+    <option name='TI'/><option name='AU'/><option name='OR'/>
+  </param>
+</param>
+
+\end{verbatim}\end{gsc}\end{quote}
+The type attribute is used to determine how to display the parameters on a web page or interface. For example, a string parameter may result in a text entry box, a boolean an on/off button, enum\_single/enum\_multi a drop-down menu, where one or many items, respectively, can be selected.
+A multi-type parameter indicates that two or more parameters are associated, and should be displayed appropriately. For example, in a field query, the text box and field list should be associated. The occurs attribute specifies how many times the parameter should be displayed on the page.
+Parameters also come with display information: all the text strings needed to present them to the user. These include the name of the parameter and the display values for any options. These are included in the above parameter descriptions in the form of \gst{<displayItem>} elements.
+
+A service description also contains some display information---this includes the name of the service, and the text  for the submit button. 
+
+Here is a sample describe request to the FieldQuery service of collection mgppdemo, along with its response. The parameters in this example include their display information. Figure~\ref{fig:query-display} shows an example HTML search form that may be generated from this describe response. 
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="mgppdemo/FieldQuery" type="describe" />
+
+<response from="mgppdemo/FieldQuery" type="describe">
+  <service name="FieldQuery" type="query">
+    <displayItem name="name">Form Query</displayItem>
+    <displayItem name="submit">Search</displayItem>
+    <paramList>
+      <param default="Doc" name="level" type="enum_single">
+        <displayItem name="name">Granularity to search at</displayItem>
+        <option name="Doc">
+          <displayItem name="name">Document</displayItem>
+        </option>
+        <option name="Sec">
+          <displayItem name="name">Section</displayItem>
+        </option>
+        <option name="Para">
+          <displayItem name="name">Paragraph</displayItem>
+        </option>
+      </param>
+      <param default="1" name="case" type="boolean">
+        <displayItem name="name">Turn casefolding </displayItem>
+        <option name="0">
+          <displayItem name="name">off</displayItem>
+        </option>
+        <option name="1">
+          <displayItem name="name">on</displayItem>
+        </option>
+      </param>
+      <param default="1" name="stem" type="boolean">
+        <displayItem name="name">Turn stemming </displayItem>
+        <option name="0">
+          <displayItem name="name">off</displayItem>
+        </option>
+        <option name="1">
+          <displayItem name="name">on</displayItem>
+        </option>
+      </param>
+      <param default="10" name="maxDocs" type="integer">
+        <displayItem name="name">Maximum documents to return
+        </displayItem>
+      </param>
+      <param name="simpleField" occurs="4" type="multi">
+        <displayItem name="name"></displayItem>
+        <param name="fqv" type="string">
+          <displayItem name="name">Word or phrase </displayItem>
+        </param>
+        <param default="ZZ" name="fqf" type="enum_single">
+          <displayItem name="name">in field</displayItem>
+          <option name="ZZ">
+            <displayItem name="name">allfields</displayItem>
+          </option>	  
+          <option name="TX">
+            <displayItem name="name">text</displayItem>
+          </option>
+          <option name="TI">
+            <displayItem name="name">Title</displayItem>
+          </option>
+          <option name="SU">
+            <displayItem name="name">Subject</displayItem>
+          </option>
+          <option name="ORG">
+            <displayItem name="name">Organization</displayItem>
+          </option>
+          <option name="SO">
+            <displayItem name="name">Source</displayItem>
+          </option>
+        </param>
+      </param>
+    </paramList>
+  </service>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+\begin{figure}[t]
+  \centering
+  \includegraphics[width=3.5in]{query2.ps}
+  \caption{The previous query service describe response as displayed on the search page.}
+  \label{fig:query-display}
+\end{figure}
+
+A describe request to an applet type service returns the applet HTML element: this will be embedded into a web page to run the applet. 
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request type='describe' to='mgppdemo/PhindApplet'/>
+
+<response type='describe'>
+  <service name='PhindApplet' type='query'>
+    <applet ARCHIVE='phind.jar, xercesImpl.jar, gsdl3.jar, 
+			jaxp.jar, xml-apis.jar'
+            CODE='org.greenstone.applet.phind.Phind.class'
+            CODEBASE='lib/java'
+            HEIGHT='400' WIDTH='500'>
+      <PARAM NAME='library' VALUE=''/>
+      <PARAM NAME='phindcgi' VALUE='?a=a&amp;sa=r&amp;sn=Phind'/>
+      <PARAM NAME='collection' VALUE='mgppdemo' />
+      <PARAM NAME='classifier' VALUE='1' />
+      <PARAM NAME='orientation' VALUE='vertical' />
+      <PARAM NAME='depth' VALUE='2' />
+      <PARAM NAME='resultorder' VALUE='L,l,E,e,D,d' />
+      <PARAM NAME='backdrop' VALUE='interfaces/default/>
+				      images/phindbg1.jpg'/>
+      <PARAM NAME='fontsize' VALUE='10' />
+      <PARAM NAME='blocksize' VALUE='10' />
+      The Phind java applet.
+    </applet>
+    <displayItem name="name">Browse phrase hierarchies</displayItem>
+  </service>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+Note that the library parameter has been left blank. This is because library refers to the current servlet that is running and the name is not necessarily known in advance. So either the applet action or the Receptionist must fill in this parameter before displaying the HTML.
+
+\subsection{'system'-type messages}\label{sec:system}
+
+``System'' requests are used to tell a MessageRouter, Collection or ServiceCluster to update its cached information and activate or deactivate other modules. For example, the MessageRouter has a set of Collection modules that it can talk to. It also holds some XML information about those collections---this is returned when a request for a collection list comes in. If a collection is deleted or modified, or a new one created, this information may need to change, and the list of available modules may also change. Currently these requests are initiated by particular CGI requests (see Section~\ref{sec:runtime-config}).
+
+The basic format of a system request is as follows:
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request type='system' to=''>
+  <system .../>
+</request>
+\end{verbatim}\end{gsc}\end{quote}
+
+One or more actual requests are specified in  system elements. The following are examples:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<system type='configure' subset=''/>
+<system type='configure' subset='collectionList'/>
+<system type='activate' moduleType='collection' moduleName='demo'/>
+<system type='deactivate' moduleType='site' moduleName='site1'/>
+\end{verbatim}\end{gsc}\end{quote}
+
+The first request reconfigures the whole site---the MessageRouter goes through its whole configure process again. The second request just reconfigures the collectionList---the MessageRouter will delete all its collection information, and re-look through the collect directory and reload all the collections again.
+The third request is to activate collection demo. This could be a new collection, or a reactivation of an old one. If a collection module already exists, it will be deleted, and a new one loaded. The final request deactivates the site site1---this removes the site from the siteList and module map, and also removes any of that sites collections/services from the static lists.
+
+A response just contains a status message\footnote{TODO: add in error/status codes}, for example:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<status>MessageRouter reconfigured successfully</status>
+<status>Error on reconfiguring collectionList</status>
+<status>collection:demo activated</status>
+<status>site:site1 deactivated</status>
+\end{verbatim}\end{gsc}\end{quote}
+
+System requests are mainly answered by the MessageRouter. However, Collections and ServiceClusters will respond to a subset of these requests. 
+
+\subsection{'format'-type messages}\label{sec:format}
+
+Collection designers are able to specify how their collection looks to a certain degree. They can specify format statements for display that will apply to the results of a search, the display of a document, entries in a classification hierarchy, for example. This info is generally service specific. All services respond to a format request, where they return any service specific formatting information. A typical request and response looks like this:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="mgppdemo/FieldQuery" type="format" />
+
+<response from="mgppdemo/FieldQuery" type="format">
+  <format>
+    <gsf:template match="documentNode"><td><gsf:link>
+      <gsf:metadata name="Title" />(<gsf:metadata name="Source" />)
+      </gsf:link></td>
+    </gsf:template>
+  </format>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+The actual format statements are described in Section~\ref{sec:formatstmt}. They are templates written directly in XSLT, or in GSF (GreenStone Format) which is a simple XML representation of the more complicated XSLT templates. 
+GSF-style format statements need to be converted to proper XSLT. This is currently done by the Receptionist (but may be moved to an ActionHelper): the format XML is transformed to XSLT using XSLT with the config\_format.xsl stylesheet.
+
+\subsection{'status'-type messages}\label{sec:status}
+
+These are only used with process-type services, which are those where a request is sent to start some type of process (see Section~\ref{sec:process}). An initial 'process' request to a 'process' service generates a response which states whether the process had successfully started, and whether its still continuing. If the process is not finished, status requests can be sent repeatedly to the service to poll the status, using the pid to identify the  process.  Status codes are used to identify the state of a process. The values used at the moment are listed in Table~\ref{tab:status codes}\footnote{A more standard set of codes should probably be used, for example, the HTTP codes}.
+
+\begin{table}
+\caption{Status codes currently used in \gsiii\ }
+\label{tab:status codes}
+{\footnotesize
+\begin{tabular}{llp{8cm}}
+\hline
+\bf code name & \bf code  & \bf meaning \\
+& \bf value & \\
+\hline
+SUCCESS &  1 & the request was accepted, and the process was  completed \\
+ACCEPTED & 2 & the request was accepted, and the process has been started, but it is not completed yet \\
+ERROR & 3 & there was an error and the process was stopped \\
+CONTINUING & 10 & the process is still continuing \\
+COMPLETED & 11 & the process has  finished \\
+HALTED & 12 & the process has stopped  \\
+INFO & 20 & just an info message that doesn't imply anything \\
+\hline
+\end{tabular}}
+\end{table}
+
+ The following shows an example status request, along with two responses, the first a 'OK but continuing' response, and the second a 'successfully completed' response. The content of the status elements in the two responses is the output from the process since the last status update was sent back.
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="build/ImportCollection" type="status">
+  <paramList>
+    <param name="pid" value="2" />
+  </paramList>
+</request>
+
+<response from="build/ImportCollection">
+  <status code="2" pid="2">Collection construction: import collection.
+command = import.pl -collectdir /research/kjdon/home/greenstone3/web/sites/
+    localsite/collect test1
+starting
+  </status>
+</response>
+
+<response from="build/ImportCollection">
+  <status code="11" pid="2">RecPlug: getting directory 
+/research/kjdon/home/greenstone3/web/sites/localsite/collect/test1/import
+WARNING - no plugin could process /.keepme
+ 
+*********************************************
+Import Complete
+*********************************************
+* 1 document was considered for processing
+* 0 were processed and included in the collection
+* 1 was rejected. See /research/kjdon/home/greenstone3/web/sites/
+    localsite/collect/test1/etc/fail.log for a list of rejected documents
+Success
+  </status>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+\subsection{'process'-type messages}
+
+Process requests and responses  provide  the major functionality of the system---these are the ones that do the actual work. The format depends on the service they are for, so I'll describe these by service.
+
+Query type services TextQuery, FieldQuery, AdvancedFieldQuery (GS2MGSearch, GS2MGPPSearch), TextQuery (LuceneSearch)
+The main type of requests in the system are for services. There are different types of services, currently: \gst{query}, \gst{browse}, \gst{retrieve}, \gst{process}, \gst{applet}, \gst{enrich}. Query services do some kind of search and return a list of document identifiers. Retrieve services can return the content of those documents, metadata about the documents, or other resources. Browse is for browsing lists or hierarchies of documents. Process type services are those where the request is for a command to be run. A status code will be returned immediately, and then if the command has not finished, an update of the status can be requested. Applet services are those that run an applet. Enrich services take a document and return the document with some extra markup added.
+
+  Other possibilities include transform, extract, accrete. These types of service generally enhance the functionality of the first set. They may be used during collection formation: 'accrete' documents by adding them to a collection, 'transform' the documents into a different format, 'extract' information or acronyms from the documents, 'enrich' those documents with the information extracted or by adding new information. They may also be used during querying: 'transform' a query before using it to query a collection, or 'transform' the documents you get back into an appropriate form.
+
+The basic structure of a service 'process' request is as follows:
+\begin{quote}\begin{gsc}\begin{verbatim}
+
+<request lang='en'  type='process' to='demo/TextQuery'>
+  <paramList/>
+  other elements...
+</request>
+
+\end{verbatim}\end{gsc}\end{quote}
+
+The parameters are name-value pairs corresponding to parameters that were specified in the service description sent in response to a describe request. 
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<param name='case' value='1'/>
+<param name='maxDocs' value='34'/>
+<param name='index' value='dtx'/>
+\end{verbatim}\end{gsc}\end{quote}
+
+Some requests have other content---for document retrieval, this would be a list of document identifiers to retrieve. For metadata retrieval, the content is the list of documents to retrieve metadata for. 
+
+Responses vary depending on the type of request. The following sections look at the process type requests and responses for each type of service.
+
+\subsubsection{'query'-type services}
+Responses to query requests contain a list of document identifiers, along with some other information, dependent on the query type. For a text query, this includes term frequency information, and some metadata about the result. For instance, a text query on 'snail farming', with the parameter 'maxDocs=10' might return the first 10 documents, and one of the query metadata items would be the total number of documents that matched the query.\footnote{no metadata about the query result is returned yet.}
+
+The following shows an example query request and its response.
+
+Find at most 10 Sections in the mgppdemo collection, containing the word snail (stemmed), returning the results in ranked order:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang='en'  to="mgppdemo/TextQuery" type="process">
+  <paramList>
+    <param name="maxDocs" value="10"/>
+    <param name="queryLevel" value="Section"/>
+    <param name="stem" value="1"/>
+    <param name="matchMode" value="some"/>
+    <param name="sortBy" value="1"/>
+    <param name="index" value="t0"/>
+    <param name="case" value="0"/>
+    <param name="query" value="snail"/>
+  </paramList>
+</request>
+
+<response from="mgppdemo/TextQuery" type="process">
+  <metadataList>
+    <metadata name="numDocsMatched" value="59" />
+  </metadataList>
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2" 
+	docType='hierarchy' nodeType="leaf" />
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.2.12" 
+	docType='hierarchy' nodeType="leaf" />
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.1" 
+	docType='hierarchy' nodeType="interior" />
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.2.2" 
+	docType='hierarchy' nodeType="leaf" />
+    ...
+  </documentNodeList>
+  <termList>
+    <term field="" freq="454" name="snail" numDocsMatch="58" stem="3">
+      <equivTermList>
+        <term freq="" name="Snail" numDocsMatch="" />
+        <term freq="" name="snail" numDocsMatch="" />
+        <term freq="" name="Snails" numDocsMatch="" />
+        <term freq="" name="snails" numDocsMatch="" />
+      </equivTermList>
+    </term>
+  </termList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+The list of document identifiers includes some information about document type and node type. Currently, document types include \gst{simple}, \gst{paged} and \gst{hierarchy}. \gst{simple} is for single section documents, i.e. ones with no sub-structure. \gst{paged} is documents that have a single list of sections, while \gst{hierarchy} type documents have a hierarchy of nested sections. For \gst{paged} and \gst{hierarchy} type documents, the node type identifies whether a section is the root of the document, an internal section, or a leaf.
+
+The term list identifies, for each term in the query, what its frequency in the collection is, how many documents contained that term, and a list of its equivalent terms (if stemming or casefolding was used).
+
+\subsubsection{'browse'-type services}
+
+Browse type services are used for classification browsing. The request consists of a list of classifier identifiers, and some structure parameters listing what structure to retrieve.
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="mgppdemo/ClassifierBrowse" type="process">
+  <paramList>
+    <param name="structure" value="ancestors" />
+    <param name="structure" value="children" />
+  </paramList>
+  <classifierNodeList>
+    <classifierNode nodeID="CL1.2" />
+  </classifierNodeList>
+</request>
+
+<response from="mgppdemo/ClassifierBrowse" type="process">
+  <classifierNodeList>
+    <classifierNode nodeID="CL1">
+      <nodeStructure>
+	<classifierNode nodeID="CL1">
+          <classifierNode nodeID="CL1.2">
+	    <classifierNode nodeID="CL1.2.1" />
+	    <classifierNode nodeID="CL1.2.2" />
+	    <classifierNode nodeID="CL1.2.3" />
+	    <classifierNode nodeID="CL1.2.4" />
+	    <classifierNode nodeID="CL1.2.5" />
+          </classifierNode>	
+	</classifierNode>
+      </nodeStructure>
+    </classifierNode>
+  </classifierNodeList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+Possible values for structure parameters are \gst{ancestors}, \gst{parent}, \gst{siblings}, \gst{children}, \gst{descendents}. The response gives, for each identifier in the request, a \gst{<nodeStructure>} element with all the requested structure put together into a hierarchy. The structure may include classifier and document nodes. 
+
+
+\subsubsection{'retrieve'-type services}
+
+Retrieval services are special in that requests are not explicitly initiated by a user from a form on a web page, but are called from actions in response to other things. This means that their names are hard-coded into the Actions. DocumentContentRetrieve, DocumentStructureRetrieve and DocumentMetadataRetrieve are the standard names for retrieval services for content, structure, and metadata of documents. Requests to each of these include a list of document identifiers. Because these generally refer to parts of documents, the elements are called \gst{<documentNode>}. For the content, that is all that is required. For the metadata retrieval service, the request also needs parameters specifying what metadata is required. For structure retrieval services, requests need parameters specifying what structure or structural info is required.
+
+Some example requests and responses follow.
+
+Give me the Title metadata for these documents:
+\begin{quote}\begin{gsc}\begin{verbatim}
+
+<request lang="en" to="mgppdemo/DocumentMetadataRetrieve" type="process">
+  <paramList>
+    <param name="metadata" value="Title" />
+  </paramList>
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2"/>
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.2.12"/>
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.1"/>
+    ...
+  </documentNodeList>
+</request>
+
+<response from="mgppdemo/DocumentMetadataRetrieve" type="process">
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2">
+      <metadataList>
+        <metadata name="Title">Putting snails in your second pen</metadata>
+      </metadataList>
+    </documentNode>
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.2.12">
+      <metadataList>
+        <metadata name="Title">Now you must decide</metadata>
+      </metadataList>
+    </documentNode>
+    <documentNode nodeID="HASH010f073f22033181e206d3b7.1">
+      <metadataList>
+        <metadata name="Title">Introduction</metadata>
+      </metadataList>
+    </documentNode>
+  </documentNodeList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+One or more parameters specifying metadata may be included in a request. Also, a metadata  value of \gst{all} will retrieve all the metadata for each document.
+
+Any browse-type service must also implement a metadata retrieval service to provide metadata for the nodes in the classification hierarchy. The name of it is the browse service name plus \gst{MetadataRetrieve}. For example, the ClassifierBrowse service described in the previous section should also have a ClassifierBrowseMetadataRetrieve service. The request and response format is exactly the same as for the DocumentMetadataRetrieve service, except that \gst{<documentNode>} elements are replaced by \gst{<classifierNode>} elements (and the corresponding list element is also changed).
+
+Give me the text (content) of this document:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="mgppdemo/DocumentContentRetrieve" type="process">
+  <paramList />
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2" />
+  </documentNodeList>
+</request>
+
+<response from="mgppdemo/DocumentContentRetrieve" type="process">
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2">
+      <nodeContent>&lt;Section&gt; 
+       &lt;/B&gt;&lt;P ALIGN=&quot;JUSTIFY&quot;&gt;&lt;/P&gt;
+       &lt;P ALIGN=&quot;JUSTIFY&quot;&gt;190. When the plants in 
+       your second pen have grown big enough to provide food and 
+       shelter, you can put in the snails.&lt;/P&gt;
+      </nodeContent>
+    </documentNode>
+  </documentNodeList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+The content of a node is returned in a \gst{<nodeContent>} element. In this case it is escaped HTML.
+
+Give me the ancestors and children of the specified node, along with the number of siblings it has:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="mgppdemo/DocumentStructureRetrieve" type="process">
+  <paramList>
+    <param name="structure" value="ancestors" />
+    <param name="structure" value="children" />
+    <param name="info" value="numSiblings" />
+  </paramList>
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2" />
+  </documentNodeList>
+</request>
+
+<response from="mgppdemo/DocumentStructureRetrieve" type="process">
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2">
+      <nodeStructureInfo>
+        <info name="numSiblings" value="2" />
+      </nodeStructureInfo>
+      <nodeStructure>
+        <documentNode nodeID="HASHac0a04dd14571c60d7fbfd" 
+                docType='hierarchy' nodeType="root">
+          <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4" 
+                  docType='hierarchy' nodeType="interior">
+            <documentNode nodeID="HASHac0a04dd14571c60d7fbfd.4.2" 
+                   docType='hierarchy' nodeType="leaf" />
+          </documentNode>
+        </documentNode>
+      </nodeStructure>
+    </documentNode>
+  </documentNodeList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+Structure is returned inside a \gst{<nodeStructure>} element, while structural info is returned in a \gst{<nodeStructureInfo>} element. Possible values for structure parameters are as for browse services: \gst{ancestors}, \gst{parent}, \gst{siblings}, \gst{children}, \gst{descendents}, \gst{entire}. Possible values for info parameters are \gst{numSiblings}, \gst{siblingPosition}, \gst{numChildren}. 
+
+\subsubsection{'process'-type services}\label{sec:process}
+Requests to process-type services are not requests for data---they request some action to be carried out, for example, create a new collection, or import a collection. The response is a status or an error message. The import and build commands may take a long time to complete, so a response is sent back after a successful start to the command. The status may be polled by the requester to see how the process is going. 
+
+Process requests generally contain just a parameter list. Like for any service, the parameters used by a process-type service can be obtained by a describe request to that service.
+
+Here are two example requests for process-services that are part of the build service cluster (hence the addresses all begin with 'build/'), followed by an example response:
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang='en'  type='process' to='build/NewCollection'>
+  <paramList>
+    <param name='creator' value='me@home.com'/>
+    <param name='collName' value='the demo collection'/>
+    <param name='collShortName' value='demo'/>
+  </paramlist>
+</request>
+
+<request lang='en'  type='process' to='build/ImportCollection'>
+  <paramList>
+    <param name='collection' value='demo'/>
+  </paramlist>
+</request>
+
+<response from="build/ImportCollection">
+  <status code="2" pid="2">Starting process...</status>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+The \gst{code} attribute in the response specifies whether the command has been successfully stated, whether its still going, etc (see Table~\ref{tab:status codes} for a list of currently used codes). The pid attribute specifies a process id number that can be used when querying the status of this process. The content of the status element is (currently) just the output from the process so far. Status messages, which were described in Section~\ref{sec:status}, are used to find out how the process is going, and whether it has finished or not.
+
+\subsubsection{'applet'-type services}
+
+Applet-type services are those that process the data for an applet. A request consists only of a list of parameters, and the response contains an \gst{<appletData>} element that contains the XML data to be returned to the applet. The format of this is entirely specific to the applet---there is no set format to the applet data.
+
+Here is an example request and response, used by the Phind applet:
+\begin{quote}\begin{gsc}\begin{verbatim}
+  <request type='query' to='mgppdemo/PhindApplet'>
+    <paramList>
+      <param name='pc' value='1'/>
+      <param name='pptext' value='health'/>
+      <param name='pfe' value='0'/>
+      <param name='ple' value='10'/>
+      <param name='pfd' value='0'/>
+      <param name='pld' value='10'/>
+      <param name='pfl' value='0'/>
+      <param name='pll' value='10'/>
+    </paramList>
+  </request>
+
+  <response type='query' from='mgppdemo/PhindApplet'>
+    <appletData>
+      <phindData df='9' ef='46' id='933' lf='15' tf='296'>
+        <expansionList end='10' length='46' start='0'>
+          <expansion df='4' id='8880' num='0' tf='59'>
+            <suffix> CARE</suffix>
+          </expansion>
+          ...
+        </expansionList>
+        <documentList end='10' length='9' start='0'>
+          <document freq='78' hash='HASH4632a8a51d33c47a75c559' num='0'>
+            <title>The Courier - N??159 - Sept- Oct 1996 Dossier Investing 
+                   in People Country Reports: Mali ; Western Samoa
+            </title>
+          </document>
+          ...
+        </documentList>
+        <thesaurusList end='10' length='15' start='0'>
+          <thesaurus df='7' id='12387' tf='15' type='RT'>
+            <phrase>PUBLIC HEALTH</phrase>
+          </thesaurus>...
+        </thesaurusList>
+      </phindData>
+    </appletData>
+  </response>
+
+\end{verbatim}\end{gsc}\end{quote}
+
+\subsubsection{'enrich'-type services}
+
+Enrich services typically take some text of documents (inside \gst{<nodeContent>} tags) and returns the text marked up in some way. One example of this is the GatePOSTag service: this identifies Dates, Locations, People and Organizations in the text, and annotates the text with the labels. In the following example, the request is for Location and Dates to be identified.
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request lang="en" to="GatePOSTag" type="process">
+  <paramList>
+    <param name="annotationType" value="Date,Location" />
+  </paramList>
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd">
+      <nodeContent>
+        FOOD AND AGRICULTURE ORGANIZATION OF THE UNITED NATIONS
+        Rome 1986
+        P-69
+        ISBN 92-5-102397-2
+        FAO 1986
+      </nodeContent>
+    </documentNode>
+  </documentNodeList>
+</request>
+
+<response from="GatePOSTag" type="process">
+  <documentNodeList>
+    <documentNode nodeID="HASHac0a04dd14571c60d7fbfd">
+      <nodeContent>
+	FOOD AND AGRICULTURE ORGANIZATION OF THE UNITED NATIONS
+	<annotation type="Location">Rome</annotation> 
+        <annotation type="Date">1986</annotation>
+        P-69
+        ISBN 92-5-102397-2
+        FAO <annotation type="Date">1986</annotation>
+      </nodeContent>
+    </documentNode>
+  </documentNodeList>
+</response>
+\end{verbatim}\end{gsc}\end{quote}
+
+\subsection{Page generation}\label{sec:pagegen} 
+
+A 'page' is some XML or HTML (or other?) data returned in response to an 
+external 'page'-type request. These requests originate from outside \gs\ , for example from a servlet, or Java application, and are received by the Receptionist. As described below in Section~\ref{sec:page-requests}, the requests are XML representations of \gs\  URLs. One of the arguments is action (a). This tells the Receptionist which Action module to pass the request to. 
+
+Action modules decode the rest of the arguments to determine what requests need to be made to the system. One or more internal requests may be made to the MessageRouter. A request for format information from the Collection/Service may also be made. The resulting data is gathered together into a single XML response, \gst{<page>}, and returned to the Receptionist. 
+
+The page format is described in Section~\ref{sec:page-format}. The XML may be returned as is, or may be modified by the Receptionist. The various Receptionists are described in Section~\ref{sec:recepts}. The default receptionist used by a servlet transforms the XML into HTML using XSL stylesheets. Section~\ref{sec:collformat} looks at collection specific formatting, in particular for HTML output.
+Sections~\ref{sec:pageaction} to \ref{sec:systemaction} look at the various actions and what kind of data they gather.
+
+\subsubsection{'page'-type requests and their arguments}\label{sec:page-requests}
+
+These are requests for a 'page' of data---for example, the home page for a site; the query page for a collection; the text of a document. They contain, in XML, a list of arguments specifying what type of page is required. If the external context is a servlet, the arguments represent the 'CGI' arguments in a \gs\  URL.  The two main arguments are \gst{a} (action) and \gst{sa} (subaction). All other arguments are encoded as parameters.
+
+Here are some examples of  requests\footnote{In a servlet context, these correspond to the arguments \gst{a=p\&sa=about\&c=demo\&l=fr}, and \gst{a=q\&l=en\&s=TextQuery\&c=demo\&rt=r\&ca=0\&st=1\&m=10\&q=snail}.}:
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request type='page' action='p' subaction='about'
+         lang='fr' output='html'>
+  <paramList>
+    <param name='c' value='demo'/>
+  </paramList>
+</request>
+\end{verbatim}\end{gsc}\end{quote}
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+<request type='page' action='q' lang='en' output='html'>
+  <paramList>
+    <param name='s' value='TextQuery'/>
+    <param name='c' value='demo'/>
+    <param name='rt' value='r'/>
+    <!-- the rest are the service specific params -->
+    <param name='ca' value='0'/> <!-- casefold -->
+    <param name='st' value='1'/> <!-- stem -->
+    <param name='m' value='10'/> <!-- maxdocs -->
+    <param name='q' value='snail'/> <!-- query string -->
+  </paramList>
+</request>
+\end{verbatim}\end{gsc}\end{quote}
+
+There are some standard arguments used in Greenstone, and they are described in Table~\ref{tab:args}. These are used by Receptionists and Actions. The GSParams class specifies all the general basic arguments, and whether they should be saved or not (Some arguments need to be saved during a session, and this needs to be implemented outside \gs\  proper --- currently we do this in the servlet, using servlet session handling). The servlet has an init parameter \gst{params\_class} which specifies which params class to use: GSParams can be subclassed if necessary. The Receptionist and Actions must not have conflicting argument names.
+
+Other arguments are used dynamically and come from the Services. Service arguments must always be saved during a session. Services may be created by different people, and may reside on a different site. There is no guarantee that there is no conflict with argument names between services and actions. Therefore service parameters are namespaced when they are put on the page, whereas interface (receptionist and action) parameters have no namespace. The default namespace is s1 (service1) --- any parameters that are for the service will be prefixed by this. For example, the case parameter for a search will be put in the page as s1.case, and the resulting argument in a search URL will be s1.case. When actions are deciding which parameters need to be sent in a request to a service, they can use the namespace information.
+
+If there are  two or more services combined on a page with a single submit button, they will use namespaces s1, s2, s3 etc as needed. The s  (service) parameter will end up with a list of services. For example, \gst{s=TextQuery,MusicQuery,} and the order of these determines the mapping order of the namespaces, i.e. s1 will map to TextQuery, s2 to MusicQuery.
+
+\begin{table}
+{\footnotesize
+\begin{tabular}{lll}
+\hline
+\bf Argument & \bf Meaning &\bf Typical values \\
+\hline
+a & action & a (applet), q (query), b (browse), p (page), pr (process) \\
+& & s (system)\\
+sa & subaction & home, about (page action)\\
+c & collection or  & demo, build \\
+& service cluster \\
+s & service name & TextQuery, ImportCollection \\
+rt & request type & d (display), r (request), s (status) \\
+ro & response only & 0 or 1 - if set to one, the request is carried out \\
+& & but no processing of the results is done \\
+& & currently only used in process actions \\
+o & output type & XML, HTML, WML \\
+l & language & en, fr, zh ...\\
+d & document id & HASHxxx \\
+r & resource id & ???\\
+pid & process handle & an integer identifying a particular process request \\
+\hline
+\end{tabular}}
+\caption{Generic arguments that can appear in a \gs\  URL}
+\label{tab:args}
+\end{table}
+
+\subsubsection{page format}\label{sec:page-format}
+
+The basic  page format  is:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<page lang='en'>
+  <pageRequest/>
+  <pageResponse/>
+</page>
+\end{verbatim}\end{gsc}\end{quote}
+
+* show configuration and describe whats its used for
+
+There are two main elements in the page: pageRequest, pageResponse. The pageRequest is the original request that came into the Receptionist---this is included so that any parameters  can be preset to their previous values, for example, the query options on the query form. The pageResponse contains all the data that has been gathered from the system by the action. The other two elements contain extra information needed by XSLT. Config contains run-time variables such as the location of the gsdl home directory, the current site name, the name of the executable that is running (e.g. library)---these are needed to allow the XSLT to generate correct HTML URLs. Display contains some of the text strings needed in the interface---these are separate from the XSLT to allow for internationalization.
+
+The following subsections outline, for each action, what data is needed and what requests are generated to send to the system. 
+
+
+Once the XML page has been put together, the page to return to the user is created by transforming the XML using XSLT. The output is HTML at this stage, but it will be possible to generate alternative outputs, such as XML, WML etc. A set of XSLT files defines an 'interface'. Different users can change the look of their web pages by creating new XSLT files for a new 'interface'. Just as we have a sites directory where different sites 'live' (ie where their configuration file and collections are located), we have an interfaces directory where the different interfaces 'live' (ie their transforms and images are located there). The default XSLT files are
+located in interfaces/default/transforms. Collections, sites and other interfaces
+can override these files by having their own copy of the appropriate
+files. New interfaces have their own directory inside interfaces/. Sites and collections can have a transform directory containing XSLT files. The order in which the XSLT files are looked for is collection, site, current
+interface, default interface.\footnote{this currently breaks down for remote sites - need to rethink it a bit.}
+***TODO*** describe a bit more?? currently only can get this locally
+
+\subsubsection{Receptionists}\label{sec:recepts}
+
+The receptionist is the controlling module for the page generation part of \gs\ . It has the job of loading up all the actions, and it knows about the message router it and the actions are supposed to talk to. It routes messages received to the appropriate action (page-type messages) or directly to the message router (all other types). Receptionists also do other things, for example, adding to the page received back from the action any information that is common to all pages.
+
+There are different ways of providing an interface to \gs\ , from web based CGI style (using servlets) to Java GUI applications. These different interfaces require slightly different responses from a receptionist, so we provide several standard types of receptionist.
+
+Receptionist: This is the most basic receptionist. The page it returns consists of the original request, and the response from the action it was sent to. Methods preProcessRequest, and postProcessPage are called on the request and page, respectively, but in this basic receptionist, they don't do anything.
+
+TransformingReceptionist: This extends Receptionist, and overwrites postProcessPage to transform the page using XSLT. An XSLT is listed for each action in the receptionists configuration file, and this is used to transform the page. First, some display information, and configuration information is added to the page. Then it is transformed using the specified XSLT for the action, and returned.
+
+WebReceptionist: The WebReceptionist extends TransformingReceptionist. It doesn't do much else except some argument conversion. To keep the URLs short, parameters from the services are given shortnames, and these are used in the web pages. 
+
+DefaultReceptionist: This extends WebReceptionist, and is the default one for \gsiii\  servlets. Due to the page design, some extra information is needed for each page: some metadata about the current collection. The receptionist sends a describe request to the collection to get this, and appends it to the page before transformation using XSLT.
+
+NZDLReceptionist: (do we want to talk about this?) This is an example of a custom receptionist. For a look-alike nzdl.org system, even more information is needed for each page, namely the list of classifiers available from the ClassifierBrowse service. 
+
+By default, the LibraryServlet uses DefaultReceptionist. However, there is a servlet init-param called \gst{receptionist} which can be set to make the servlet use a different one.
+
+\subsubsection{Collection specific formatting}\label{sec:collformat}
+get format info, transform gsf->xsl. transform xml->html
+
+configuration params are passed in to the transformation
+\subsubsection{CGI arguments}
+
+
+\subsubsection{Page action}\label{sec:pageaction}
+
+PageAction is responsible for displaying kinds of information pages, such as the home page of the library, or the home page of a collection, or the help and preferences pages. These pages are not associated with specific services like the other page types. In general, the data comes from describe requests to various modules.
+The different pages are requested using the subaction argument. For the 'home' page, a 'describe' request is sent to the MessageRouter---this returns a list of all the collections, services, serviceClusters and sites known about. For each collection, its metadata is retrieved via a 'describe' request. This metadata is added into the previous result, which is then added into the page.    For the 'about' page, a \gst{describe} request is sent to the module that the about page is about: this may be a collection or a service cluster.  This returns a list of metadata
+and a list of services.
+
+
+\subsubsection{Query action}\label{sec:queryaction}
+
+The basic URL is \gst{a=q\&s=TextQuery\&c=demo\&rt=d/r}.
+There are three query services which have been implemented: TextQuery, FieldQuery, and AdvancedFieldQuery. These are all handled in the same way by query action.
+For each page, the service description is requested from the  service  of the current collection (via a describe request).  This is currently done every time the query page is
+displayed, but should be cached. The description includes a list of the parameters available for the query, such as case/stem, max num docs to return, etc. If the request type (rt) parameter is set to d for display, the action only needs to display the form, and this is the only request to the service. Otherwise, the submit button has been pressed, and a query request to the TextQuery service is sent. This has  all the parameters from the URL put into the parameter list. A list of document identifiers
+is returned. A followup query is sent to the MetadataRetrieve service of the collection: the content includes the list of
+documents, with a request for some of their metadata. Which metadata to retrieve is determined by looking through the XSLT that will be used to transform the page. The service description and query result are combined into a page of XML, which is returned to the Receptionist.
+
+\subsubsection{Applet action}\label{sec:appletaction}
+
+There are two types of request to the applet action: \gst{a=a \& rt=d\/} and
+\gst{a=a \& rt=r\/}.  The value \gst{rt=d\/} means ``display the applet.'' A
+\gst{describe} request is sent to the service, which returns the \gst{<applet>} HTML element.  The transformation file \gst{applet.xsl} embeds this
+into the page, and the servlet returns the HTML.
+
+The value \gst{rt=r} signals a request from the applet. A process request containing all the parameters is sent to the applet service. The result contains an appletData element, which contains a single  element - this element is returned
+directly to the applet, in XML. No transformation is done. 
+Because the AppletAction doesn't know or care anything about the applet data, it can work with any applet-service pair.
+
+Note that the applet HTML may need to know the name of the \gst{library}
+program.  However, that name is chosen by the person who installed the software
+and will not necessarily be ``library''.  To get around this, the applet can
+put a parameter called ``library'' into the applet data with a null value:
+\begin{quote}\begin{gsc}\begin{verbatim}
+<PARAM NAME='library' VALUE=''/>
+\end{verbatim}\end{gsc}\end{quote}
+When the AppletAction encounters this parameter it inserts the name of the
+current library servlet as its value.
+
+\subsubsection{Document action}\label{sec:documentaction}
+
+DocumentAction is responsible for displaying a document to the user. The display might involve some metadata and/or text for a document or part of a document. For hierarchical documents, a table of contents may be shown, while for paged documents (those with a single linear list of sections), next and previous page buttons may be shown. These different display types require different information about the document. Depending on the arguments, DocumentAction will send requests to several services: DocumentMetadataRetrieve, DocumentStructureRetrieve and DocumentContentRetrieve.
+
+A basic display, for example, Title and text, involves a metadata request to get the Title, and a content request to get the text. Hierarchical table of contents display requires a structure request. If the entire contents is to be displayed, the parameter \gst{structure=entire} would be sent in the request. Otherwise, parameters \gst{structure=ancestors}, \gst{structure=children} and possibly \gst{structure=siblings} may be used, depending in the position of the current node in the document. These return a hierarchical structure of nodes, containing ancestor nodes, child nodes and sibling nodes, respectively.
+For paged display, the structure is not actually needed. A structure request is still sent, but this time it requests some information, rather the structure itself. The information requested includes the number of siblings and the current position of the current node, or the number of children (if the current node is the root of the document).
+
+Metadata may be requested for the current node, or for any nodes in the structure, and content also. The metadata and content are added into the appropriate nodes in the structure hierarchy, and this is returned as the page data.
+
+\subsubsection{XML Document action}\label{sec:xmldocumentaction}
+
+XMLDocumentAction is a little different to the standard DocumentAction. It operates in two modes, \gst{text} and \gst{toc}. In \gst{text} mode, it will retrieve the content of the current document node using a DocumentContentRetrieve request. In \gst{toc} mode, it retrieves the entire table of contents for the document using a DocumentStructureRetrieve request. Either mode may also retrieve metadata for the current section or each section in the table of contents.
+
+\subsubsection{GS2Browse action}\label{sec:browseaction}
+
+GS2BrowseAction is for displaying Greenstone 2 style classifiers. 
+\subsubsection{System action}\label{sec:systemaction}
+
+SystemAction allows for manual reconfiguration of various components at run-time. There is no interactive web-page displaying the options, it merely turns a set of CGI arguments into an XML system request. The response from a system request is a message which is displayed to the user.
+
+\begin{table}
+\caption{Configure CGI arguments}
+\label{tab:system-cgi}
+{\footnotesize
+\begin{tabular}{ll}
+\hline
+\bf arg & \bf description\\
+\hline
+a=s & system action\\
+sa=c$|$a$|$d & type of system request: c (configure), a (add/activate), \\
+& d (delete/deactivate) \\
+c=demo &  the request will go to this collection/servicecluster \\
+& instead of the message router\\
+ss=collectionList & subset for configure: only reconfigure this part.\\
+&  For the MessageRouter,  can be serviceClusterList, serviceList, \\
+& collectionList, siteList.\\
+&  For a collection/cluster, can be metadataList or serviceList.\\
+sn=demo & \\
+st=collection& \\
+\hline
+\end{tabular}}
+\end{table}
+
+
+\subsection{Other code information}
+
+Greenstone has a set of Utility classes, which are briefly described in Table~\ref{tab:utils}.
+
+\begin{table}[h]
+\caption{The utility classes in org.greenstone.gsdl3.util}
+\label{tab:utils}
+{\footnotesize
+\begin{tabular}{lp{3.75in}}
+\hline
+\bf Utility class & \bf Description\\
+\hline
+CollectionClassLoader & ClassLoader that knows about a collection's resource directory \\
+DBInfo & Class to hold info from GDBM database entry \\
+Dictionary & wrapper around a Resource Bundle, providing strings with parameters\\
+GDBMWrapper & Wrapper for GDBM database. Uses JavaGDBM\\
+GSConstants & holds some constants used for servlet arguments and configuration variables\\
+GSEntityResolver & an EntityResolver which can be used to find resources such as DTDs\\
+GSFile & class to create all \gs\  file paths e.g. used to locate configuration files, XSLT files and collection data. \\
+GSHTML & provides convenience methods for dealing with HTML, e.g. making strings HTML safe\\
+GSParams & contains names and default values for interface parameters\\
+NZDLParams & a subclass of GSParams which holds default service parameters too, necessary for the classic style interface.\\
+GSPath & used to create, examine and modify message address paths\\
+GSSQL & contains static strings for all the SQL table/field names\\
+GSStatus & some static codes for status messages\\ 
+GSXML & lots of methods for extracting information out of \gs\  XML, and creating some common types of elements. Also has static Strings for element and attribute names used by \gs\ .\\
+GSXSLT & some manipulation functions for \gs\  XSLT\\
+GlobalProperties & Holds the global properties (from global.properties) \\
+MacroResolver & Used with replace elements in collection configuration files, replaces a macro or string with another string, metadata or text from a dictionary\\
+GS2MacroResolver & MacroResolver for GS2 collections, that uses the GDBM database\\
+Misc & miscellaneous functions\\
+MyNodeList & A simple implementation of an XML NodeList\\
+OID & class to handle \gs\  (2) OIDs\\
+GS3OID & subclass of OID to handle \gsiii\ OIDs\\
+Processing & Runs an external process and prints the output from the process \\
+SQLQuery & contains a connection to a SQL database, along with some methods for accessing the data, such as converting MG numbers to and from Greenstone OIDs.\\
+XMLConverter & provides methods to create new Documents, parse Strings or Files into Documents, and convert Nodes to Strings\\
+XMLTransformer & methods to transform XML using XSLT \\
+XSLTUtil & contains static methods to be called from within XSLT \\
+\hline
+\end{tabular}}
+\end{table}
+
+\newpage
+\section{Collection building architecture}\label{sec:develop-build}
+**** GEORGE ****
+how building actually works\\
+the building structure/architecture\\
+modules API\\
+
+\newpage
+\section{Developing \gsiii\ : Adding new features}\label{sec:new-features}
+
+\subsection{Creating new services}\label{sec:new-services}
+
+*inherit from ServiceRack - abstract base class. this handles the main process method, determines the service name and request type. if request type is describe, and to is empty, it returns a list of services (short\_service\_info) which is initialized in the configure method. a describe request to a particular service results in getServiceDescription being called, which must be supplied by the subclass.
+other request types (process) get sent to processXXX methods, where XXX is the service name.
+
+* what methods are expected
+
+*service type responses expected
+
+*a browse type service must also implement servicenameMetadataRetrieve service.
+
+* should a metadata retrieval service advertise what metadata is available??
+
+standard service type vs new service type - standard needs some xml response syntax.
+
+\subsection{creating new actions/pages}\label{sec:new-pages}
+
+\subsection{new interfaces}\label{sec:new-interfaces}
+
+It is easy to create new interfaces to \gsiii. Here we are talking about interfaces other than those to display in typical browser.
+
+Handheld devices: Use the standard servlet setup, but with a different set of XSLT files to format the pages for small screens, or use WML.
+
+Java GUI Interface: There are couple of alternatives. Depending on what you want to display in the GUI, you could talk to either a Receptionist or a MessageRouter. The library classes can be set up and compiled into the GUI program.
+Talking to a Receptionist will give you access to pages of XML. It is likely that the standard Receptionist class would be used - this doesn't transform the data to HTML. Queries such as ``give me the home page of a collection'' and ``do the following search'' can be issued. All the data needed for the result view is returned. Queries are quite simple, but are limited to what kinds of Actions are available in the library. 
+Talking to a MessageRouter requires a bit more effort on the part of the GUI program, but results in greater flexibility. The kinds of queries that can be issued are individual units of action, such as ``describe yourself'', ``search'', ``retrieve the content for this document''. More than one request may need to be made for a particular feature of the GUI. However you can ask for any combination of data available in the system, you are not relying on Actions. What you will implement though, may be a lot like the Action code in terms of request sequences.
+
+Interfaces in other programming languages: Because the communication is all XML based, other interfaces can talk to the Java library if a communication protocol is set up. This could be done using SOAP for example. Like for Java GUI interfaces, the program could talk to a Receptionist or to a MessageRouter. 
+e.g. Java interface. where you can interface to. MR vs Receptionist. different receptionists. e.g., handheld - using servlet, transforming recpt, but new set of XSLT Java program other program - talk to recpt but just get back XML data for pages. Java gui - just talk to MR, do all processing itself.
+
+Remote interfaces: remote interfaces can be set up in the same way as above, using a communication protocol between the interface, and the library program.
+
+\subsection{Adding new classifiers}\label{sec:new-classifiers}
+*** GEORGE ***
+\subsection{Adding new plugins}\label{sec:new-plugins}
+*** GEORGE ***
+
+\subsection{New types of collections}\label{sec:new-coll-types}
+
+There are two types of standard \gs\  collections: collections built with the \gsiii\  building system, and collections that are imported from \gsii\ . There are many options to collection building but it is conceivable that these options don't meet the needs of all collection builders. \gsiii\  has an ability to use any type of collection you can come up with, assuming  some Java code is provided. 
+
+There are four levels of customization that may be needed with new collections: service, collection, interface XSLT, and action levels. We will use the example collections that come with \gs\  to describe these different levels.
+
+Firstly, new service classes need to be written to provide the functionality to search/browse/whatever the collection. If the services have similar interfaces and functionality to the standard services, this may be all that is needed. For example, the \gsii\  MGPP collections were the first to be served in \gsiii\ . When we came to do \gsii\  MG collections, all we had to do was write some new service classes that interacted with MG instead of MGPP. Because these collections used the same type of services, this was all we had to do. The format of the configuration files was similar, they just specified MG serviceRack classes rather than MGPP ones.
+
+The XML Sample Texts (gberg) collection, however, was done quite differently to the standard collections. New services were provided to search the database (built with Lucene) and to provide the documents and parts of documents (using XSLT to transform the raw XML files). The collectionConfig file had some extra information in it: a list of the documents in the collection along with their Titles. Because the standard collection class has no notion of document lists, a new class was created (org.greenstone.gsdl3.collection.XMLCollection). This class is basically the same as a standard collection class except that it looks for and stores in memory the documentList from the collectionConfig file.
+
+To tell \gs\  to load up a different type of collection class, we use another configuration file: etc/collectionInit.xml. This  specifies the name of the collection class to use.
+Currently, this is all that is specified in that file, but you may want to add parameters for the class etc.
+
+\gst{<collectionInit class="XMLCollection"/>}
+
+The display for the collection is also quite different. The home page for the collection  displays the list of documents. To achieve this, the describe response from the collection had to include the list, and a new XSLT was written for the collection that displayed this. Collection XSLT should be put in the transform directory of the collection\footnote{These are currently only used when running \gs\  in a non-distributed fashion, but it will be added in properly at some stage}.
+
+Document display is  significantly different to standard \gs\ . There are two modes of display: table of contents mode, and content mode. Clicking on a document link from the collection home page takes the user to the table of contents for the collection. Clicking on one of the sections in the table of contents takes them to a display of that section. To facilitate this, not only do we need new XSLT files , we also needed a new action. XMLDocumentAction was created, that used two subactions, toc and text, for the different modes of display.
+
+The Receptionist was told about this new action by the addition of the following element to the interfaceConfig.xml file: 
+
+\begin{gsc}\begin{verbatim}
+<action name='xd' class='XMLDocumentAction'>
+  <subaction name='toc' xslt='document-toc.xsl'/>
+  <subaction name='text' xslt='document-content.xsl'/>
+</action>
+\end{verbatim}\end{gsc}
+
+XSLT files are linked to subactions rather than the action as a whole. The collection supplies the two XSLT files written appropriately for the data it contains.
+
+All links that link to the documents have to be changed to use the xd action rather than the standard d action. These include the links from the home page, and the links from query results.
+
+Querying of the collection is almost the same as usual. The query service provides a list of parameters, does the query and then sends back a list of document identifiers. The standard query action was fine for this collection. The change occurs in the way that the results are displayed---this is accomplished using a format statement supplied in the collectionConfig file inside the search node.
+
+\begin{gsc}\begin{verbatim}
+<search>
+  <format>
+    <gsf:template match="documentNode">
+      <xsl:param name="collName"/>
+      <xsl:param name="serviceName"/>
+      <td>
+        <b><a href="{$library_name}?a=xd&amp;sa=text&amp;c={$collName}&
+            amp;d={@nodeID}&amp;p.a=q&amp;p.s={$serviceName}">
+             <xsl:choose>
+               <xsl:when test="metadataList/metadata[@name='Title']">
+                 <gsf:metadata name="Title"/>
+               </xsl:when>
+               <xsl:otherwise>(section)</xsl:otherwise>
+             </xsl:choose>
+           </a>
+         </b> from <b><a href="{$library_name}?a=xd&amp;sa=toc&amp;
+           c={$collName}&amp;d={@nodeID}.rt&amp;p.a=q&amp;p.s={$serviceName}">
+         <gsf:metadata name="Title" select="root"/></a></b>
+      </td>
+    </gsf:template>
+  </format>
+</search>
+\end{verbatim}\end{gsc}
+
+Instead of displaying an icon and the Title, it displays the Title of the section and the title of the document. Both of these are linked to the document: the section title to the content of that section, the document title to the table of contents for the document. Because these require non-standard arguments to the library, these parts of the template are written in XSLT not \gs\  format language. As is shown here it is perfectly feasible to write a format statement that includes XSLT mixed in with \gs\  format elements.
+
+The document display uses CSS to format the output---these are kept in the collection and specified in the collections XSLT files. The documents also specify DTD files. Due to the way we read in the XML files, Tomcat sometimes has trouble locating the DTDs. One option is to make all the links absolute links to files in the collection folder, the other option is to put them in \gs\ 's DTD folder greenstone3/resources/dtd. 
+
+\subsection{The Classic Interface}
+
+The library seen at \gst{http://www.greenstone.org/greenstone3/nzdl} is like a mirror to \gst{http://www.nzdl.org}---it aims to present the same collections, in the same way but using \gsiii\  instead of \gsii\ . It uses a new site (nzdl) with the classic interface. The web.xml file had a new servlet entry in it to specify the combination of nzdl site and classic interface.
+
+The site was created by making a directory called nzdl in the sites folder. A siteConfig file was created. Because it is running on Linux, we were able to link to all the collections in the old \gs\  installation. The convert\_coll\_from\_gs2.pl script was run over all the collections to produce the new XML configuration files.
+
+The classic interface was created to be used by this site (and is now a standard part of Greenstone).
+In many cases, creating a new interface just requires the new images and XSLT  to be added to the new directory(see Sections~\ref{sec:sites-and-ints} and \ref{sec:interface-customise}). This classic interface required a bit more customization.
+
+The standard \gsiii\  navigation bar lists all the services available for the collection. In \gsii\ , the navigation bar provides the search option, and the different classifiers. This is not service specific, but hard coded to the search and classifiers. The XSLT that produces the navigation bar needed to be altered to produce this. But also, a new Receptionist was needed. 
+The standard receptionist (DefaultReceptionist) gathers a little bit of extra information for each page of XML before transforming it: this is the list of services for the collection and their display information, allowing the services to be listed along the navigation bar. This is information that is needed by every page (except for the library home page) and therefore is obtained by the receptionist instead of by each action. The nzdl interface needed a bit more information than this: for the ClassifierBrowse service, if there was one, the list of classifiers and their display elements must be obtained. So a new Receptionist (NZDLReceptionist) was written that inherited from DefaultReceptionist, and added this new info into the page.
+
+One of the servlet initialization parameters is the receptionist class: this was added to the servlet definition in the web.xml file so that the LibraryServlet would load up the right receptionist class.
+
+
+\newpage
+\section{Distributed \gs\ }\label{sec:distributed}
+
+\gs\  is designed to run in a distributed fashion. One \gs\  installation can talk to several sites on different computers. This requires some sort of communication protocol. Any protocol can be used, currently we have a simple SOAP protocol.
+
+more explanation..
+
+\begin{figure}[h]
+  \centering
+  \includegraphics[width=4in]{remote} %5.8
+  \caption{A distributed digital library configuration running over several servers}
+  \label{fig:remote}
+\end{figure}
+
+We have used Apache Axis SOAP implementation. This is run as a servlet in Tomcat. Axis is setup during installation of Greenstone. For more details about SOAP in Greenstone, see Appendix~\ref{app:soap}. Debugging soap is described in Appendix~\ref{app:soap-debug}.
+
+\subsection{Serving a site using soap}
+
+A webs service for localsite comes predeployed, but if you want to setup a service for another site, run \gst{ant soap-deploy-site}. This will prompt you for the sitename (its directory name), and a siteuri - a unique identifier for the web service. Tomcat needs to be running for this to work.
+
+The ant target deploys the service for the site specified. A resource file (\gst{<sitename>.wsdd}) is created which is used to specify the service. It can be found in \gst{greenstone3/resources/soap}, and is generated from \gst{site.wsdd.template}.
+
+To get siteA to talk to siteB, you need to deploy a SOAP server on siteB, then add a \gst{<site>} element to the \gst{<siteList>} of siteA's \gst{siteConfig.xml} file (in \gst{greenstone3/web/sites/siteA/siteConfig.xml}).
+
+In the \gst{<siteList>} element, add the following (substituting the chosen site uri for siteBuri):
+
+\begin{gsc}\begin{verbatim}
+<site name="siteBuri"
+  address="http://localhost:8080/greenstone3/services/siteBuri"
+  type="soap"/>
+\end{verbatim}\end{gsc}
+
+(Note that localhost and 8080 should be changed to the values you entered when installing \gsiii).
+
+The servlet for siteA needs to be reconfigured \\
+(e.g. \gst{http://localhost:8080/greenstone3/library?a=s\&sa=c}).
+\appendix
+
+\newpage
+\section{Using \gsiii\  from CVS}\label{app:cvs}
+
+[TODO: need to make sure building stuff is in here]
+
+\gsiii\  is also available via CVS. You can download the latest version of the code. This is not guaranteed to be stable, in fact it is likely to be unstable. The advantage of using CVS is that you can update the code and get the latest fixes. 
+
+Note that you will need the Java 2 SDK, version 1.4.0 or higher, and Ant (Apache's Java based build tool, http://ant.apache.org) installed.
+
+To check out the \gs\  code, use:
+
+\begin{quote}\begin{gsc}\begin{verbatim}
+cvs -d :pserver:cvs\_anon@cvs.scms.waikato.ac.nz:2402/usr/local/
+           global-cvs/gsdl-src co -P greenstone3
+\end{verbatim}\end{gsc}\end{quote}
+
+If you need it, the password for anonymous CVS access is \gst{anonymous}. Note that some older versions of CVS have trouble accessing this repository due to the port number being present. We are using version 1.11.1p1. 
+
+Greenstone is built and installed using Ant (Apache's Java based build tool, 
+http://ant.apache.org). You will need a Java Development 
+Environment (1.4 or higher), and Ant installed to use Greenstone. You can download Ant from \\\gst{http://ant.apache.org/bindownload.cgi}.
+
+In the greenstone3 directory, you can run 'ant' which will give you a help message.
+Running 'ant -projecthelp' gives a list of the targets that you can run - these
+do various things like compile the source code, startup the server etc.
+
+The README.txt file has up-to-date instructions for installing from CVS. Briefly, for a first time install, run 'ant prepare install'.
+
+The file build.properties contains various parameters that can be set by the user. Please check these settings before running the installation process. The install process will ask you if you accept the properties before starting.
+For a  non-interactive version of the install, run
+ant -Dproperties.accepted=yes install
+
+To log the output in build.log, run
+ant -Dproperties.accepted=yes -logfile build.log install
+
+Under Linux, Java and C/C++ compilation is carried out. For windows, since Visual Studio is not a standard component, only Java compilation is carried out. Pre-compiled binaries are provided for the C/C++ components (source packages and Greenstone 2 style building). If you have Visual Studio installed (version 6), you can run the compile-windows-c++ targets to compile the code locally. (Don't forget to setup the Visual Studio environment first, by running, e.g. C:/Program Files/Microsoft Visual Studio/VC98/Bin/VCVARS32.BAT or equivalent.)
+
+
+Note: \gst{gs3-setup} sets the environment variables \gst{CLASSPATH, PATH, JAVA\_HOME} and needs to be done in a shell before doing collection building etc. 
+
+To startup or shutdown the library (includes the Tomcat server and MYSQL server), the commands are (run from the greenstone3 directory):
+
+\begin{quote}\begin{gsc}
+ant start \\
+ant stop
+\end{gsc}\end{quote}
+
+If you want to restart only Tomcat, run \gst{ant restart-tomcat}.
+
+\newpage
+\section{Tomcat}\label{app:tomcat}
+
+Tomcat is a servlet container, and Greenstone 3 runs as a servlet inside it. 
+
+The file \gst{\gsdlhome/packages/tomcat/conf/server.xml} is the Tomcat configuration file. The installation process adds a context for \gsiii\  servlets (\gst{\gsdlhome/web})---this tells Tomcat where to find the web.xml file, and what URL (\gst{/greenstone3}) to give it. Anything inside the context directory is accessible via Tomcat\footnote{can we use .htaccess files to restrict access??}. For example, the index.html file that lives in \gst{\gsdlhome/web} can be accessed through the URL \gst{localhost:8080/greenstone3/index.html}. The demo collection's images can be accessed through \\
+\gst{localhost:8080/greenstone3/sites/localsite/collect/demo/images/}.
+
+
+Greenstone sets up Tomcat to run on port 8080 by default. To change this, you can edit the tomcat.port property in build.properties. If you do this before installing Greenstone, then running 'ant install' will use the new port number. If you want to change it later on, shutdown tomcat, run 'ant reconfigure-server-settings', then when you restart tomcat it will use the new port.
+
+Note: Tomcat must be shutdown and restarted any time you make changes in the following for those changes to take effect:
+\begin{bulletedlist}
+\begin{gsc}
+\item \gsdlhome/web/WEB-INF/web.xml
+\item \gsdlhome/packages/tomcat/conf/server.xml
+\end{gsc}
+\item any classes or jar files used by the servlets
+\end{bulletedlist}
+\noindent Note: stdin and stdout for the servlets (on Linux) both go to\\
+\gst{\gsdlhome/packages/tomcat/logs/catalina.out}
+
+On startup, the servlet loads in its collections and services. If the site or collection configuration files are changed, these changes will not take effect until the site/collection is reloaded. This can be done through the reconfiguration messages (see Section~\ref{sec:runtime-config}), or by restarting Tomcat.
+
+We have set up Tomcat to follow symlinks. To disable this feature, remove the \gst{<Resources>} element from the greenstone3 context in \\\gst{\$GSDL3HOME/packages/tomcat/conf/server.xml}:
+
+\begin{quote}\begin{gsc}
+<Context path="/greenstone3" docBase="\$GSDL3HOME/web" debug="1" \\
+reloadable="true">\\
+   <Resources allowLinking='true'/>\\
+</Context>\\
+\end{gsc}\end{quote}
+
+By default, Tomcat allows directory listings. To disable this, change the 'listings' parameter to false in the default servlet definition, in Tomcat's web.xml file (\gst{\$GSDL3HOME/packages/tomcat/conf/web.xml}):
+
+We have set the greenstone context to be reloadable. This means that if a class or resource file in web/WEB-INF/lib or web/WEB-INF/classes changes, the servlet will be reloaded. This is useful for development, but should be turned off for production mode (set the reloadable attribute to false).
+
+Tomcat uses a Manager to handle HTTP session information. This may be stored between restarts if possible. To use a persistent session handling manager, uncomment the \gst{<Manager>} element in \\
+\gst{\$GSDL3HOME/packages/tomcat/conf/server.xml}. For the default manager, session information is stored in the work directory:\\
+\gst{\$GSDL3HOME/packages/tomcat/work/Standalone/localhost/greenstone3/SESSIONS.ser}. Delete this file to clear the cached session info. Note that Tomcat needs to be shutdown to delete this file.
+
+\subsection{Proxying Tomcat with apache}
+
+Instead of incorporating servlet support into your existing web server, an easy alternative is to proxy Tomcat. The \gst{http://www.greenstone.org/greenstone3} site uses apache to proxy Tomcat. ProxyPass and ProxyPassReverse directives need to be added to the Virtualhost description for the www.greenstone.org server.
+
+\begin{quote}\begin{gsc}
+<VirtualHost xx.xx.xx.xx>\\
+ServerName www.greenstone.org\\
+...\\
+ProxyPass /greenstone3 http://puka.cs.waikato.ac.nz:8080/greenstone3\\
+ProxyPassReverse /greenstone3 http://puka.cs.waikato.ac.nz:8080/greenstone3\\
+</VirtualHost>\\
+\end{gsc}\end{quote}
+
+In our example, the \gsiii\  servlet can be accessed at \\
+\gst{http://www.greenstone.org/greenstone3/library}, instead of at \\
+\gst{http://puka.cs.waikato.ac.nz:8080/greenstone3/library}, which is not publically accessible.
+
+\subsection{Running Tomcat behind a proxy}
+
+Almost everything works fine when Tomcat is running behind a proxy. The only time this causes trouble is if the servlet itself needs to make external HTTP connections. We do this in the infomine demo collection for example. One of the service classes sends HTTP requests to the infomine database at riverside. Since this is going through the proxy, a username and password is needed. It is not sufficient to prompt the user for a password because they are unlikely to have a password for the particular proxy that Tomcat is using. What we have done at present is to put a proxy element in the siteConfig.xml file. Here you have to enter a suitable username and password for the proxy server. Unfortunately these are entered in plain text. And the file is viewable via the servlet. So we need a better solution.
+
+\newpage
+\section{SOAP}\label{app:soap}
+
+Greenstone uses the Apache Axis SOAP implementation for distributed communications. Axis runs as a servlet inside Tomcat, and SOAP web services can be deployed by this Axis servlet. The Greenstone installation process sets up Axis for Tomcat, and predeploys the localsite web service.
+
+To deploy a SOAP service for other sites, run \gst{ant soap-deploy-site}
+
+This will prompt you for the sitename (the site's directory name), and a unique URI for the site. It creates a new SOAPServer class for the site \\(\gst{\$GSDL3HOME/src/java/org/greenstone/gsdl3/SOAPServer<sitename>.java}), creates a resource file for deployment (\gst{\$GSDL3HOME/resources/soap/<sitename>.wsdd}), and then tries to deploy the service.
+
+Information about deployed services is maintained between Tomcat sessions---you only need to deploy something once. To undeploy a site, use \gst{ant undeploy-soap-site}.
+
+The axis services can be accessed at \gst{localhost:8080/greenstone3/index.jsp}.
+
+\subsection{Debugging SOAP}\label{app:soap-debug}
+
+If you need to debug the SOAP stuff for some reason, or just want to look at the SOAP messages that are being passed back and forth, you can use the TCP monitor. This intercepts messages coming in to one port, displays them, and passes them to another port.
+To run it, type:
+
+\begin{quote}\gst{java -cp \$GSDL3HOME/web/WEB-INF/lib/axis.jar \\
+org.apache.axis.utils.tcpmon}
+\end{quote}
+
+The listen port is the port that you want the monitor to be listening on. It should 'act as' a Listener, with target hostname 127.0.0.1 (localhost), and target port the port that Tomcat is running on (8080). You need to modify the address used to talk to the SOAP service. For example, if you want to monitor traffic between the gateway site and the localsite SOAP server, you will need to edit gateway's siteConfig.xml file and change the port number (in the site element) to whatever you have chosen as the listen port.
+
+\newpage
+\section{Tidying up the formatting for imported Greenstone 2 collections}\label{app:gs2tidy}
+
+\subsection{Format statements: \gsii\  vs \gsiii\ }\label{app:gs2format}
+The following table shows the \gsii\  format elements, and their equivalents in \gsiii\ 
+\begin{table}[h]
+\caption{\gsiii\  equivalents of \gsii\  format statements}
+{\footnotesize
+\begin{tabular}{ll}
+\hline
+\bf \gsii\         & \bf \gsiii\  \\
+\hline
+\gst{[Text]} & \gst{<gsf:text/>} \\
+\gst{[num]} & \gst{<gsf:metadata name='docnum'/>}\\
+\gst{[link][/link]} & \gst{<gsf:link></gsf:link>} or \\
+& \gst{<gsf:link type='document'></gsf:link>}\\
+\gst{[srclink][/srclink]} & \gst{<gsf:link type='source'></gsf:link>}\\
+\gst{[icon]} & \gst{<gsf:icon/>} or \\
+& \gst{<gsf:icon type='document'/>}\\
+\gst{[srcicon]} & \gst{<gsf:icon type='source'/>}\\
+\gst{[Title]} (metadata) & \gst{<gsf:metadata name='Title'/>} or \\
+& \gst{<gsf:metadata name='Title' select='current'/>}\\
+\gst{[parent:Title]} & \gst{<gsf:metadata name='Title' select='parent' />}\\
+\gst{[parent(All):Title]} & \gst{<gsf:metadata name='Title' select='ancestors'/>}\\
+\gst{[parent(Top):Title]} & \gst{<gsf:metadata name='Title' select='root' />}\\
+\gst{[parent(All': '):Title]} & \gst{<gsf:metadata name='Title' select='ancestors'}\\ 
+& \gst{  separator=': ' />}\\
+\gst{[sibling(All': '):Title]} & \gst{<gsf:metadata name='Title' multiple='true'} \\
+& \gst{  separator=': ' />}\\
+\gst{\{Or\}\{[dc.Title],}  & \gst{<gsf:choose-metadata>}\\
+\gst{  [dls.Title], [Title]\}}& \gst{  <gsf:metadata name='dc.Title'/>}\\
+& \gst{  <gsf:metadata name='dls.Title'/>}\\
+& \gst{  <gsf:metadata name='Title'/>}\\
+& \gst{</gsf:choose-metadata>}\\
+\gst{\{If\}\{[parent:Title],} & \gst{<gsf:choose-metadata>}\\
+\gst{  [parent:Title], [Title]\}}& \gst{  <gsf:metadata name='Title' select='parent'/>}\\
+& \gst{  <gsf:metadata name='Title'/>}\\
+& \gst{</gsf:choose-metadata>}\\
+\gst{\{If\}\{[Subject],} & \gst{<gsf:switch>}\\
+\gst{  <td>[Subject]</td>\}}& \gst{  <gsf:metadata name='Subject'/>}\\
+& \gst{  <gsf:when test='exists'>} \\
+& \gst{     <td><gsf:metadata name='Subject'/></td>}\\
+& \gst{  </gsf:when></gsf:switch>}\\
+\hline
+\end{tabular}}
+\end{table}
+\subsection{Cleaning up macros}\label{app:gs2replace}
+
+Here we show some of the replace items that have been used for Greenstone 2 collections.
+
+Getting rid of silly backslashes:
+\begin{gsc}\begin{verbatim}
+<replace scope='text' macro="\\?\\\(" text="\("/>
+\end{verbatim}\end{gsc}
+
+Macro resolving using resource bundles and metadata:
+\begin{gsc}\begin{verbatim}
+<replace scope='metadata' macro="_magazines_" bundle="NZDLMacros" 
+   key="Magazines"/>
+<replace scope='all' macro='_thisOID_' metadata='archivedir'/>
+<replace macro="_httpcollimg_" 
+   text="sites/localsite/collect/folktale/index/assoc"/>
+\end{verbatim}\end{gsc}
+
+Fixing up broken external links:
+\begin{gsc}\begin{verbatim}
+<replace macro="_httpextlink_&amp;rl=1&amp;href=" 
+   text="?a=d&amp;c=folktale&amp;s0.ext=1&amp;d="/>
+<replace macro="_httpextlink_&amp;rl=0&amp;href=" 
+   text="?a=p&amp;sa=html&amp;c=folktale&amp;url="/>
+\end{verbatim}\end{gsc}
+
+These two examples show how to deal with Greenstone 2's external link macros. The first one is for a 'relative' external link. In this case, the links are like URL's but they actually refer to Greenstone internal documents. So the Greenstone 3 link is to the document, but with parameter s0.ext signifying that the d argument will need translating before retrieving the content.
+The second example is a truly external link. This is translated into a HTML type page action, where the URL is presented as a frame along with the collection header in a separate frame.
+
+Sometimes we need to add in macros to be resolved in a second step:
+\begin{gsc}\begin{verbatim}
+<replace macro="_iconpdf_" scope="metadata" 
+   text="&lt;img title='_texticonpdf_' src='interfaces/default/images/ipdf.gif'/&gt;"/>
+<replace macro="_texticonpdf_" scope="metadata" bundle="interface_classic" 
+   key="texticonpdf"/>
+\end{verbatim}\end{gsc}
+
+\end{document}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/.cvsignore
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/.cvsignore	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/.cvsignore	(revision 21112)
@@ -0,0 +1,3 @@
+build
+dist
+test-classes
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/COPYING
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/COPYING	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/COPYING	(revision 21112)
@@ -0,0 +1,353 @@
+The files in this directory and all its subdirectories belong to the
+Greenstone Alerting Service. These files, with the exemption of the JAR files
+in the lib subdirectory, are Copyright (C) 2004, 2005 by Andrea Schweer
+(schweer@cs.waikato.ac.nz) and are licensed under the GNU General Public
+License Version 2 (see below). For the JAR files in the lib subdirectory, see
+the license information inside each JAR file or in the lib subdirectory.
+
+===============================================================================
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/build.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/build.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/build.xml	(revision 21112)
@@ -0,0 +1,517 @@
+<!--
+     General purpose build script for web applications and web services,
+     including enhanced support for deploying directly to a Tomcat 4
+     based server.
+
+     This build script assumes that the source code of your web application
+     is organized into the following subdirectories underneath the source
+     code directory from which you execute the build script:
+
+        docs                 Static documentation files to be copied to
+                             the "docs" subdirectory of your distribution.
+
+        src                  Java source code (and associated resource files)
+                             to be compiled to the "WEB-INF/classes"
+                             subdirectory of your web applicaiton.
+
+        web                  Static HTML, JSP, and other content (such as
+                             image files), including the WEB-INF subdirectory
+                             and its configuration file contents.
+
+     $Id$
+-->
+
+
+<!-- A "project" describes a set of targets that may be requested
+     when Ant is executed.  The "default" attribute defines the
+     target which is executed if no specific target is requested,
+     and the "basedir" attribute defines the current working directory
+     from which Ant executes the requested task.  This is normally
+     set to the current working directory.
+-->
+
+<project name="GSDL Alerting Service" default="compile" basedir=".">
+
+
+
+<!-- ===================== Property Definitions =========================== -->
+
+
+<!--
+
+  Each of the following properties are used in the build script.
+  Values for these properties are set by the first place they are
+  defined, from the following list:
+
+  * Definitions on the "ant" command line (ant -Dfoo=bar compile).
+
+  * Definitions from a "build.properties" file in the top level
+    source directory of this application.
+
+  * Definitions from a "build.properties" file in the developer's
+    home directory.
+
+  * Default definitions in this build.xml file.
+
+  You will note below that property values can be composed based on the
+  contents of previously defined properties.  This is a powerful technique
+  that helps you minimize the number of changes required when your development
+  environment is modified.  Note that property composition is allowed within
+  "build.properties" files as well as in the "build.xml" script.
+
+-->
+
+  <property file="build.properties"/>
+  <property file="${user.home}/build.properties"/>
+
+
+<!-- ==================== File and Directory Names ======================== -->
+
+
+<!--
+
+  These properties generally define file and directory names (or paths) that
+  affect where the build process stores its outputs.
+
+  app.name             Base name of this application, used to
+                       construct filenames and directories.
+                       Defaults to "myapp".
+
+  app.path             Context path to which this application should be
+                       deployed (defaults to "/" plus the value of the
+                       "app.name" property).
+
+  app.version          Version number of this iteration of the application.
+
+  build.home           The directory into which the "prepare" and
+                       "compile" targets will generate their output.
+                       Defaults to "build".
+
+  catalina.home        The directory in which you have installed
+                       a binary distribution of Tomcat 4.  This will
+                       be used by the "deploy" target.
+
+  dist.home            The name of the base directory in which
+                       distribution files are created.
+                       Defaults to "dist".
+
+  manager.password     The login password of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+  manager.url          The URL of the "/manager" web application on the
+                       Tomcat installation to which we will deploy web
+                       applications and web services.
+
+  manager.username     The login username of a user that is assigned the
+                       "manager" role (so that he or she can execute
+                       commands via the "/manager" web application)
+
+-->
+
+  <property name="app.name"      value="gsdl-as"/>
+  <property name="app.path"      value="/alerting"/>
+  <property name="app.version"   value="0.1-dev"/>
+  <property name="build.home"    value="${basedir}/build"/>
+  <property name="catalina.home" value="../../comms/jakarta/tomcat/"/>
+  <property name="dist.home"     value="${basedir}/dist"/>
+  <property name="docs.home"     value="${basedir}/docs"/>
+  <property name="docs.link"     value="http://java.sun.com/j2se/1.4.2/docs/api/"/>
+  <property name="manager.url"   value="http://localhost:8080/manager"/>
+  <property name="manager.username" value="admin"/>
+  <property name="manager.password" value="admin"/>
+  <property name="src.home"      value="${basedir}/src"/>
+  <property name="web.home"      value="${basedir}/web"/>
+
+
+<!-- ================== Custom Ant Task Definitions ======================= -->
+
+
+<!--
+
+  These properties define custom tasks for the Ant build tool that interact
+  with the "/manager" web application installed with Tomcat 4.  Before they
+  can be successfully utilized, you must perform the following steps:
+
+  - Copy the file "server/lib/catalina-ant.jar" from your Tomcat 4
+    installation into the "lib" directory of your Ant installation.
+
+  - Create a "build.properties" file in your application's top-level
+    source directory (or your user login home directory) that defines
+    appropriate values for the "manager.password", "manager.url", and
+    "manager.username" properties described above.
+
+  For more information about the Manager web application, and the functionality
+  of these tasks, see <http://localhost:8080/tomcat-docs/manager-howto.html>.
+
+-->
+
+  <taskdef name="install" classname="org.apache.catalina.ant.InstallTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="list"    classname="org.apache.catalina.ant.ListTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="reload"  classname="org.apache.catalina.ant.ReloadTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+  <taskdef name="remove"  classname="org.apache.catalina.ant.RemoveTask"
+    classpath="${catalina.home}/server/lib/catalina-ant.jar"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="false"/>
+  <property name="compile.optimize"    value="true"/>
+
+
+
+<!-- ==================== External Dependencies =========================== -->
+
+
+<!--
+
+  Use property values to define the locations of external JAR files on which
+  your application will depend.  In general, these values will be used for
+  two purposes:
+  * Inclusion on the classpath that is passed to the Javac compiler
+  * Being copied into the "/WEB-INF/lib" directory during execution
+    of the "deploy" target.
+
+  Because we will automatically include all of the Java classes that Tomcat 4
+  exposes to web applications, we will not need to explicitly list any of those
+  dependencies.  You only need to worry about external dependencies for JAR
+  files that you are going to include inside your "/WEB-INF/lib" directory.
+
+-->
+
+<!-- Dummy external dependency -->
+<!--
+  <property name="foo.jar"
+           value="/path/to/foo.jar"/>
+-->
+
+
+<!-- ==================== Compilation Classpath =========================== -->
+
+<!--
+
+  Rather than relying on the CLASSPATH environment variable, Ant includes
+  features that makes it easy to dynamically construct the classpath you
+  need for each compilation.  The example below constructs the compile
+  classpath to include the servlet.jar file, as well as the other components
+  that Tomcat makes available to web applications automatically, plus anything
+  that you explicitly added.
+
+-->
+
+  <path id="compile.classpath">
+
+    <!-- Include all JAR files that will be included in /WEB-INF/lib -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <fileset dir="lib">
+    	<include name="*.jar"/>
+    </fileset>
+    <!-- include greenstone files -->
+    <fileset dir="../../web/WEB-INF/lib/">
+      <include name="*.jar"/>
+    </fileset>
+    <!-- Include all elements that Tomcat exposes to applications -->
+    <pathelement location="${catalina.home}/common/classes"/>
+    <fileset dir="${catalina.home}/common/endorsed">
+      <include name="*.jar"/>
+    </fileset>
+    <fileset dir="${catalina.home}/common/lib">
+      <include name="*.jar"/>
+    </fileset>
+    <pathelement location="${catalina.home}/shared/classes"/>
+    <fileset dir="${catalina.home}/shared/lib">
+      <include name="*.jar"/>
+    </fileset>
+
+  </path>
+
+
+
+<!-- ==================== All Target ====================================== -->
+
+<!--
+
+  The "all" target is a shortcut for running the "clean" target followed
+  by the "compile" target, to force a complete recompile.
+
+-->
+
+  <target name="all" depends="clean,compile"
+   description="Clean build and dist directories, then compile"/>
+
+
+
+<!-- ==================== Clean Target ==================================== -->
+
+<!--
+
+  The "clean" target deletes any previous "build" and "dist" directory,
+  so that you can be ensured the application can be built from scratch.
+
+-->
+
+  <target name="clean"
+   description="Delete old build and dist directories">
+    <delete dir="${build.home}"/>
+    <delete dir="${dist.home}"/>
+  </target>
+
+
+
+<!-- ==================== Compile Target ================================== -->
+
+<!--
+
+  The "compile" target transforms source files (from your "src" directory)
+  into object files in the appropriate location in the build directory.
+  This example assumes that you will be including your classes in an
+  unpacked directory hierarchy under "/WEB-INF/classes".
+
+-->
+
+  <target name="compile" depends="prepare"
+   description="Compile Java sources">
+
+    <!-- Compile Java classes as necessary -->
+    <mkdir    dir="${build.home}/WEB-INF/classes"/>
+    <javac srcdir="${src.home}"
+          destdir="${build.home}/WEB-INF/classes"
+            debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+         optimize="${compile.optimize}">
+        <classpath refid="compile.classpath"/>
+    </javac>
+
+    <!-- Copy application resources -->
+    <copy  todir="${build.home}/WEB-INF/classes">
+      <fileset dir="${src.home}" excludes="**/*.java"/>
+    </copy>
+
+  </target>
+
+
+
+<!-- ==================== Dist Target ===================================== -->
+
+
+<!--
+
+  The "dist" target creates a binary distribution of your application
+  in a directory structure ready to be archived in a tar.gz or zip file.
+  Note that this target depends on two others:
+
+  * "compile" so that the entire web application (including external
+    dependencies) will have been assembled
+
+  * "javadoc" so that the application Javadocs will have been created
+
+-->
+
+  <target name="dist" depends="compile,javadoc"
+   description="Create binary distribution">
+
+    <!-- Copy documentation subdirectories -->
+    <mkdir   dir="${dist.home}/docs"/>
+    <copy    todir="${dist.home}/docs">
+      <fileset dir="${docs.home}"/>
+    </copy>
+
+    <!-- Create application JAR file -->
+    <jar jarfile="${dist.home}/${app.name}-${app.version}.war"
+         basedir="${build.home}"/>
+
+    <!-- Copy additional files to ${dist.home} as necessary -->
+
+  </target>
+
+
+
+<!-- ==================== Install Target ================================== -->
+
+<!--
+
+  The "install" target tells the specified Tomcat 4 installation to dynamically
+  install this web application and make it available for execution.  It does
+  *not* cause the existence of this web application to be remembered across
+  Tomcat restarts; if you restart the server, you will need to re-install all
+  this web application.
+
+  If you have already installed this application, and simply want Tomcat to
+  recognize that you have updated Java classes (or the web.xml file), use the
+  "reload" target instead.
+
+  NOTE:  This target will only succeed if it is run from the same server that
+  Tomcat is running on.
+
+  NOTE:  This is the logical opposite of the "remove" target.
+
+-->
+
+  <target name="install" depends="compile"
+   description="Install application to servlet container">
+
+    <install url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"
+            path="${app.path}"
+             war="file://${build.home}"/>
+
+  </target>
+
+
+<!-- ==================== Javadoc Target ================================== -->
+
+<!--
+
+  The "javadoc" target creates Javadoc API documentation for the Java
+  classes included in your application.  Normally, this is only required
+  when preparing a distribution release, but is available as a separate
+  target in case the developer wants to create Javadocs independently.
+
+-->
+
+  <target name="javadoc" depends="compile"
+   description="Create Javadoc API documentation">
+
+    <mkdir          dir="${dist.home}/docs/api"/>
+    <javadoc sourcepath="${src.home}"
+                destdir="${dist.home}/docs/api"
+    			   link="${docs.link}"
+    	        private="true"
+           packagenames="*">
+      <classpath refid="compile.classpath"/>
+    </javadoc>
+
+  </target>
+
+
+
+<!-- ====================== List Target =================================== -->
+
+<!--
+
+  The "list" target asks the specified Tomcat 4 installation to list the
+  currently running web applications, either loaded at startup time or
+  installed dynamically.  It is useful to determine whether or not the
+  application you are currently developing has been installed.
+
+-->
+
+  <target name="list"
+   description="List installed applications on servlet container">
+
+    <list    url="${manager.url}"
+        username="${manager.username}"
+        password="${manager.password}"/>
+
+  </target>
+
+
+<!-- ==================== Prepare Target ================================== -->
+
+<!--
+
+  The "prepare" target is used to create the "build" destination directory,
+  and copy the static contents of your web application to it.  If you need
+  to copy static files from external dependencies, you can customize the
+  contents of this task.
+
+  Normally, this task is executed indirectly when needed.
+
+-->
+
+  <target name="prepare">
+
+    <!-- Create build directories as needed -->
+    <mkdir  dir="${build.home}"/>
+    <mkdir  dir="${build.home}/WEB-INF"/>
+    <mkdir  dir="${build.home}/WEB-INF/classes"/>
+
+
+    <!-- Copy static content of this web application -->
+    <copy todir="${build.home}">
+      <fileset dir="${web.home}"/>
+    </copy>
+
+    <jar destfile="lib/templates.jar" basedir="lib/templates" includes="*.vm"/>
+
+    <!-- Copy external dependencies as required -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+    <mkdir  dir="${build.home}/WEB-INF/lib"/>
+    <copy todir="${build.home}/WEB-INF/lib">
+        <fileset dir="lib" includes="*.jar"/>
+    </copy>
+
+    <!-- Copy static files from external dependencies as needed -->
+    <!-- *** CUSTOMIZE HERE AS REQUIRED BY YOUR APPLICATION *** -->
+
+  </target>
+
+
+<!-- ==================== Reload Target =================================== -->
+
+<!--
+
+  The "reload" signals the specified application Tomcat 4 to shut itself down
+  and reload. This can be useful when the web application context is not
+  reloadable and you have updated classes or property files in the
+  /WEB-INF/classes directory or when you have added or updated jar files in the
+  /WEB-INF/lib directory.
+
+  NOTE: The /WEB-INF/web.xml web application configuration file is not reread
+  on a reload. If you have made changes to your web.xml file you must stop
+  then start the web application. 
+
+-->
+
+  <target name="reload" depends="compile"
+   description="Reload application on servlet container">
+
+    <reload url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+<!-- ==================== Remove Target =================================== -->
+
+<!--
+
+  The "remove" target tells the specified Tomcat 4 installation to dynamically
+  remove this web application from service.
+
+  NOTE:  This is the logical opposite of the "install" target.
+
+-->
+
+  <target name="remove"
+   description="Remove application on servlet container">
+
+    <remove url="${manager.url}"
+       username="${manager.username}"
+       password="${manager.password}"
+           path="${app.path}"/>
+
+  </target>
+
+
+</project>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/create-tables.sql
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/create-tables.sql	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/create-tables.sql	(revision 21112)
@@ -0,0 +1,44 @@
+create table users (
+	username varchar(128) primary key, 
+	password text not null
+);
+
+create table subscriptions (
+	id int primary key auto_increment, 
+	name varchar(255), 
+	email text, 
+	rss bool not null default 0, 
+	page bool not null default 0, 
+	user varchar(128) not null references users.username, 
+	
+	unique(user,name)
+);
+
+create table predicates (
+	id int primary key auto_increment, 
+	field varchar(128) not null, 
+	value varchar(255), 
+	type enum("org.greenstone.gsdlas.profiles.EqualsPredicate", "org.greenstone.gsdlas.profiles.SubstringMatchPredicate", "org.greenstone.gsdlas.profiles.QueryPredicate") not null,
+	
+	unique(field,type,value)
+);
+
+create table subs_to_predicates (
+	subscription int not null references subscriptions.id, 
+	predicate int not null references predicates.id, 
+	
+	unique (subscription,predicate)
+);
+
+create table events (
+	id int primary key auto_increment,
+	timestamp datetime not null,
+	content text not null
+);
+
+create table events_to_subs (
+	event int not null references events.id,
+	subscription int not null references subscriptions.id,
+
+	unique (event,subscription)
+);
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/descriptor.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/descriptor.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/descriptor.xml	(revision 21112)
@@ -0,0 +1,7 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+</isd:service>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/.cvsignore
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/.cvsignore	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/.cvsignore	(revision 21112)
@@ -0,0 +1,1 @@
+templates.jar
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/error.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/error.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/error.vm	(revision 21112)
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Greenstone Digital Library Software</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>$title</p>
+			<p class="error">$message</p>
+			<p class="details">$details</p>
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/events.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/events.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/events.vm	(revision 21112)
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Events for subscription $subscription.name</p>
+<p class="description">$subscription</p>
+<p><a href="/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id">show as feed</a></p>
+#foreach ($event in $list)
+<p>$event</p>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/feed.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/feed.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/feed.vm	(revision 21112)
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<!--<?xml-stylesheet type="text/xml" href="/alerting/rss2html.xml"?>-->
+<rss version="2.0">
+   <channel>
+      <title>Events for Subscription $subscription.name</title>
+      <link>/alerting/service?action=showFeed&amp;subscriptionID=$subscription.id</link>
+      <description>TODO</description>
+      <language>en</language>
+      <lastBuildDate>$lastBuildDate</lastBuildDate>
+      <generator>Alerting Service for Greenstone</generator>
+      <webMaster>$webmaster</webMaster>
+      
+      #foreach($event in $list)
+      <item>
+         <title>$event.document_title</title>
+         <link>$event.document_url</link>
+         <description>some description</description>
+         <pubDate>$event.timestamp</pubDate>
+         <guid>$event.document_url</guid>
+         <category>$event.type</category>
+      </item>
+      #end
+   </channel>
+</rss>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/general.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/general.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/general.vm	(revision 21112)
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="default.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+<p class="message">$message</p>
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/list.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/list.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/list.vm	(revision 21112)
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="list.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>$title</p>
+#foreach($sub in $list)
+	<div class="subscription">
+		<p><span class="sub_id">Subscription $sub.Id</span><br/>
+		<a href="service?action=showEvents&subscriptionID=$sub.Id">show matching events</a> | <a href="service?action=editSubscription&subscriptionID=$sub.Id">edit subscription</a> | <a href="service?action=deleteSubscription&subscriptionID=$sub.Id">delete subscription</a></p>
+<!--		<table class="subscription">
+			<thead>
+				<tr><th>Field</th><th>Predicate</th></tr>
+			</thead>
+			<tfoot><tr><th/><th/></tr></tfoot>
+			<tbody>
+				#foreach($pred in $sub.Predicates)
+					<tr>
+						<td>$pred.Field</td>
+						<td>$pred.Value</td>
+					</tr>
+				#end
+			</tbody>
+		</table> // -->
+		<pre>
+		$sub
+		</pre>
+		<br/>
+	</div>
+#end
+<a href="/alerting">back</a> to the alerting service
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/login.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/login.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/login.vm	(revision 21112)
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Login</p>
+#if ($error) 
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=login" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="login"/>
+</form>
+<div class="description">Don't have an account yet? <a href="/alerting/service?action=showRegistrationForm">Register</a> now!</a></div>
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/register.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/register.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/register.vm	(revision 21112)
@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Greenstone Digital Library Software</title>
+<link href="/alerting/form.css" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+<div id="content">
+<h3>Alerting Service for Greenstone</h3>
+<p>Register</p>
+#if ($error)
+<div class="error">
+An error occured:<br/>
+$errormessage
+</div>
+#end
+<form action="/alerting/service?action=register" method="post"/>
+<table>
+<tr>
+<th><label for="username">username</label></th><td><input type="text" name="username" id="username"/></td>
+</tr>
+<tr>
+<th><label for="password">password</label></th><td><input type="password" name="password" id="password"/></td>
+</tr>
+<tr>
+<th><label for="password2">repeat password</label></th><td><input type="password" name="password2" id="password2"/></td>
+</tr>
+</table>
+<br/>
+<input type="submit" value="create account"/>
+</form>
+</div>
+
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_collection.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_collection.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_collection.vm	(revision 21112)
@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (3/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="current">collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the collection(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which collection(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the collection(s) to monitor: 
+					<ol>
+					<li>You can choose one or more collections from the list below. Only events from the collections(s) you have chosen will be sent to you<br/>
+					<select multiple="multiple" name="hostCollectionID" size="7" style="margin:5px 0px;">
+						<option value="">any collection on any host</option>
+						#foreach($host in $hostnames)
+						<optgroup label="$host">
+							#foreach($collection in $collectionnames.get($host))
+							<option>$host|$collection</option>
+							#end
+							<option selected="selected" value="$host">any collection on host $host</option>
+						</optgroup>
+					#end
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a collection whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>collection name</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="collection_query" style="width:80%;" value="$!prefill.collection_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						<!-- collection description? creator? language? -->
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, events coming from <em>any collection</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the last step, you will be able to configure the the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="collection"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_host.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_host.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_host.vm	(revision 21112)
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (2/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td  class="current">host</td><td>collection</td><td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the host(s) to monitor</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				</div>
+				<div>
+					<h4>Which host(s) to monitor</h4>
+					<div class="explanation">
+					You have two ways of choosing the host(s) to monitor: 
+					<ol>
+					<li>You can choose one or more hosts from the list below. Only events from the host(s) you have chosen will be sent to you.<br/>
+					<select multiple="multiple" name="hostID" size="3" style="margin:5px 0px;">
+					#set($firstoption = true);
+					#foreach($host in $hostnames)
+					<option #if($firstoption) selected="selected" #set($firstoption = false) #end>$host</option>
+					#end
+					<option value="">any host</option>
+					</select></li>
+					<li>You can enter a query in the field below. Only events coming from a host whose name matches this query will be sent to you.<br/>
+					<table style="margin:5px 0px;">
+						<tr class="odd">
+							<th>host</th>
+							<td width="25%">contains <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+							<td>
+								<input type="text" name="host_query" style="width:80%;" value="$!prefill.host_query"/>
+							</td>
+						</tr>
+						</table>
+						If you enter a query, the selection made in the list above will be ignored.
+						</li>
+					</ol>
+					If you just click &quot;continue&quot;, only events coming from the <em>current host</em> will be sent to you.
+					</div>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to choose the collection(s) to monitor, and you will be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="host"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_notification.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_notification.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_notification.vm	(revision 21112)
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Andrea Schweer</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (4/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="completed">type/details</td><td class="completed">host</td><td class="completed">collection</td><td class="current">notification</td>
+			</tr>
+			</table>
+			<p>Configure the way you will be notified of events </p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div><h4>Your subscription so far</h4>
+				<table class="subscription">
+				#set($evenodd = "odd")
+				#foreach($key in $preview.keySet())
+					#if(!($key == ""))
+						<tr class="$evenodd">
+							<th>$key</th>
+							<td>$preview.get($key)</td>
+						</tr>
+					#end
+					#if ($evenodd == "odd") #set($evenodd = "even") #else #set($evenodd = "odd") #end
+				#end
+				</table>
+				<p class="explanation">You can <a href="preview.html" target="_new">preview</a> which documents currently match your criteria (following the link will open a new window).</p>
+				</div>
+				<div>
+					<h4>How to be notified of events matching your subscription</h4>
+					<div class="explanation">
+					You will be able to see the subscription in the list of all your subscriptions after logging in to the alerting service. <br/>
+					You can assign a name to your subscription that makes it easier for you to recognise it: 
+					<input name="subscription_name" type="text" value="$!prefill.subscription_name"/>
+					</div>
+					<br/>
+					<table>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="page" checked="checked" id="chck-page"/></td>
+							<td colspan="2">
+							<label for="chck-page">on my &quot;new since my last login&quot; page</label>
+							</td>
+						</tr>
+						<tr class="even">
+							<td><input name="way" type="checkbox" value="email" id="chck-email"/></td>
+							<td><label for="chck-email">via e-mail to</label>
+							</td>
+							<td>
+								<input type="text" name="email" value="$!prefill.email"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<td><input name="way" type="checkbox" value="rss" id="chck-rss"/></td>
+							<td colspan="2"><label for="chck-rss">with an <acronym title="Really Simple Syndication">RSS</acronym> feed</label> <img class="help" alt="help" src="images/help.png" longdesc="substring" title="substring"/>
+							</td>
+						</tr>
+						</table>
+				</div>
+				<br/>
+				
+				<input type="hidden" name="current_page" value="notification"/>
+				<input type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next" disabled="disabled"/><input type="submit" name="next_page" value="finish"/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_type-details.vm
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_type-details.vm	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/lib/templates/sub_type-details.vm	(revision 21112)
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Create a subscription</title>
+		<link rel="stylesheet" href="/alerting/form.css" type="text/css"/>
+	</head>
+	<body>
+		<div id="content">
+			<h3>Create a subscription (1/4)</h3>
+			<table id="progress">
+			<tr>
+			<td class="current">type/details</td>
+			<td>host</td>
+			<td>collection</td>
+			<td>notification</td>
+			</tr>
+			</table>
+			<p>Choose the event type and how to identify the document(s) to watch</p>
+			<form action="/alerting/service?action=$action" method="post">
+				<div>
+					<h4>Event type</h4>
+					<input name="type" type="checkbox" value="new_document"  checked="checked" id="chck-new-document"/> <label for="chck-new-document">new document(s)</label><br/>
+					<input name="type" type="checkbox" value="document_changed"  id="chck-doc-changed"/> <label for="chck-doc-changed">document(s) changed</label><br/>
+					<input name="type" type="checkbox" value="document_deleted"  id="chck-doc-deleted"/> <label for="chck-doc-deleted">document(s) deleted</label><br/>
+				</div>
+				<div>
+					<h4>How to identify the document(s)</h4>
+					<table>
+						<tr class="even">
+							<th>title</th>
+							<td width="25%">matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_title" style="width:80%;" value="$!prefill.document_title"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>content</th>
+							<td>matches <img class="help" alt="help" src="images/help.png" longdesc="normal Greenstone search" title="normal Greenstone search"/>
+							</td>
+							<td>
+								<input type="text" name="document_content" style="width:80%;" value="$!prefill.document_content"/>
+							</td>
+						</tr>
+						<tr class="even">
+							<th>metadata</th>
+							<td>has field <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists" title="metadata field exists"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_has_field" value="$!prefill.metadata_has_field" style="width:80%;"/>
+							</td>
+						</tr>
+						<tr class="odd">
+							<th>metadata</th>
+							<td>has value <img class="help" alt="help" src="images/help.png" longdesc="metadata field exists and has value" title="metadata field exists and has value"/>
+							</td>
+							<td>
+								<input type="text" name="metadata_field" value="$!prefill.metatada_field" style="width:40%;"/> is
+								<input type="text" name="metadata_value" value="$!prefill.metadata_value" style="width:40%;"/>
+							</td>
+						</tr>
+					</table>
+				</div>
+				<p class="explanation">
+				In the next steps, you will be able to select the hosts and collections that you want to monitor for events. You will also be able to configure the way you will be notified of events matching your subscription.
+				</p>
+				<input type="hidden" name="current_page" value="type-details"/>
+				<input disabled="disabled" type="submit" name="next_page" value="back"/><input type="submit" name="next_page" value="next"/><input disabled="disabled" type="submit" name="next_page" value="finish"/>
+				<br/>
+			</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/AlertingService.java	(revision 21112)
@@ -0,0 +1,547 @@
+/*
+ * Created on Nov 22, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.*;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.*;
+
+import org.apache.velocity.Template;
+import org.apache.velocity.context.Context;
+import org.apache.velocity.servlet.VelocityServlet;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.profiles.Predicate;
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.users.UserManagementException;
+import org.greenstone.gsdlas.users.UserManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingService extends VelocityServlet {
+    
+    /* (non-Javadoc)
+     * @see javax.servlet.Servlet#init(javax.servlet.ServletConfig)
+     */
+    public void init(ServletConfig config) throws ServletException {
+        super.init(config);
+        System.out.println("reloading subs");
+        ProfileStore.getInstance().restoreFromDatabase();
+        try {
+            Set subs = ProfileStore.getInstance().getAllSubscriptionsFor("andrea");
+            System.out.println("reloaded " + subs.size() + " subs for andrea: ");
+            for (Iterator iter = subs.iterator(); iter.hasNext();) {
+                System.out.println(iter.next());
+            }
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+    public static final String[] actions = new String[] {
+            "createSubscription",
+            "deleteSubscription",
+            "editSubscription",
+            "showEvents",
+            "showFeed",
+            "listSubscriptions",
+            "login",
+            "register",
+            "logout",
+            "showLoginForm",
+            "showRegistrationForm"
+    };
+    
+    
+    protected Template handleRequest(HttpServletRequest req,
+            HttpServletResponse res, Context context) {
+        
+        String action = req.getParameter(Constants.ACTION_PARAM);
+
+        Map args = req.getParameterMap();
+        
+        if (action != null && action.equals("receiveEvent")) {
+            receiveEvent(args);
+            return null;
+        }
+        
+        args = normalise(args);
+        
+        if (action == null || !ArrayHelper.contains(actions, action)) {
+            String title = "Unknown action";
+            String message = "I don't know how to " + action;
+            String details = "The only actions I know are " + ArrayHelper.toString(actions);
+            return showError(context, message, details);
+        }
+                
+        String templateString = "";
+        
+        try {
+            Method method = AlertingService.class.getDeclaredMethod(action, new Class[] {Map.class, Context.class});
+            templateString = (String) method.invoke(this, new Object[] {args, context});
+        } catch (Exception e) {
+            String message = "An error has occured, I couldn't do what you told me to do.";
+            String details = e.getMessage() + " (" + e.getClass().getName() + "); "
+            	+ e.getCause() 
+            	+ " ; action is " + action;
+            return showError(context, message, details);
+        }
+        
+        Template template = null;
+        try {
+            template = getTemplate(templateString);
+        } catch (Exception e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+            try {
+                error((HttpServletRequest)context.get(REQUEST), 
+                        (HttpServletResponse)context.get(RESPONSE),
+                        e);
+            } catch (Exception e2) {
+                // TODO Auto-generated catch block
+                e2.printStackTrace();
+            }
+        }
+        return template;
+    }
+    
+    /**
+     * @param arguments
+     * @param context
+     * @return the Velocity template to use
+     * @throws Exception
+     */
+    public String createSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "createSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            arguments.putAll(getPageArgsFromSession(session));
+            arguments.put("username", session.getAttribute("username"));
+            ProfileStore.getInstance().createSubscription(arguments);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, true);
+        }
+    }
+
+    public String deleteSubscription(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "deleteSubscription");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        try {
+            ProfileStore.getInstance().deleteSubscription(subscriptionID);
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String editSubscription(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "editSubscription");
+            return showLoginForm(arguments, context);
+        }
+        if (arguments.containsKey("next_page") && arguments.get("next_page").equals("finish")) {
+            ProfileStore.getInstance().changeSubscription(arguments, session);
+            return listSubscriptions(arguments, context);
+        } else {
+            return showSubscriptionWizardPage(arguments, context, false);
+        }
+    }
+    
+    public String showFeed(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showFeed");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        HttpServletResponse res = (HttpServletResponse) context.get(RESPONSE);
+        res.setContentType("text/xml");
+        return "feed.vm";
+    }
+    
+    public String showEvents(Map arguments, Context context) throws NumberFormatException, DatabaseException, SQLException {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "showEvents");
+            return showLoginForm(arguments, context);
+        }
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        Integer subID = new Integer(subscriptionID);
+        Set events = EventStore.getInstance().getEvents(subID);
+        context.put("list", events);
+        context.put("subscription", ProfileStore.getInstance().getSubscription(subID.intValue()));
+        return "events.vm";
+    }
+    
+    public String showLoginForm(Map arguments, Context context) {
+        return "login.vm";
+    }
+    
+    public String showRegistrationForm(Map arguments, Context context) {
+        return "register.vm";
+    }
+
+    public String listSubscriptions(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (!UserManager.getInstance().isLoggedIn(session)) {
+            session.setAttribute("next_action", "listSubscriptions");
+            return showLoginForm(arguments, context);
+        }
+        String username = (String) session.getAttribute("username");
+        context.put("title", "List of Subscriptions for " + username);
+        try {
+            Collection subscriptions = ProfileStore.getInstance().getAllSubscriptionsFor(username); 
+            context.put("list", subscriptions);
+            return "list.vm";
+        } catch (DatabaseException de) {
+            context.put("message", "couldn't get list of subscriptions for " + username);
+            context.put("details", de.getMessage());
+            return "error.vm";
+        }
+    }
+    
+    public String login(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().loginUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showLoginForm(arguments, context); 
+        }
+        if (session.getAttribute("next_action") != null) {
+            String nextAction = (String) session.getAttribute("next_action");
+            Method method = AlertingService.class.getDeclaredMethod(nextAction, new Class[] {Map.class, Context.class});
+            return (String) method.invoke(this, new Object[] {arguments, context});
+        }
+        return listSubscriptions(arguments, context);
+    }
+    
+    public String register(Map arguments, Context context) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        try {
+            UserManager.getInstance().createUser(arguments, session);
+        } catch (UserManagementException e) {
+            context.put("error", Boolean.TRUE);
+            context.put("errormessage", e.getMessage());
+            return showRegistrationForm(arguments, context);
+        }
+        return login(arguments, context);
+    }
+    
+    public String logout(Map arguments, Context context) {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        Enumeration atts = session.getAttributeNames();
+        while (atts.hasMoreElements()) {
+            session.removeAttribute((String) atts.nextElement());
+        }
+        session.invalidate();
+        return "general.vm";
+    }
+    
+    public void receiveEvent(Map rawEvent) {
+        Map event = new TreeMap();
+        for (Iterator iter = rawEvent.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (key.equals("action")) continue; // we don't want this
+            String[] value = (String[]) rawEvent.get(key);
+            event.put(key, value[0]);
+        }
+        System.out.println("receiving event " + event);
+        
+        GreenstoneCommunicator gsComm = null;
+        try {
+            String hostID = (String) event.get(Constants.HOST_ID_FIELD);
+            gsComm = new GreenstoneCommunicator(new URL(hostID));
+        } catch (Exception e) {
+            System.err.println("Can't communicate to Greenstone: " + e.getMessage());
+            e.printStackTrace();
+        }
+        
+        Set matchedSubscriptions = ProfileStore.getInstance().filter(event, gsComm);
+        
+        try {
+            EventStore.getInstance().add(event, matchedSubscriptions);
+            Notifier.getInstance().sendNotifications(event, matchedSubscriptions);
+        } catch (Exception e) {
+            System.err.println("Couldn't save events: " + e.getMessage());
+            e.printStackTrace();
+        }
+        System.out.println(matchedSubscriptions.size() + " matching subscriptions: " + matchedSubscriptions);
+    }
+
+    /**
+     * @param args
+     */
+    private Map normalise(Map args) {
+        Map result = new TreeMap();
+        for (Iterator iter = args.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (Predicate.isMultiValued(key) || key.equals("way")) {
+                // multi-valued attributes
+                String[] values = ((String[]) args.get(key));
+                result.put(key, Arrays.asList(values));
+            } else {
+                String firstValue = ((String[])args.get(key))[0];
+                result.put(key, firstValue);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * @param context
+     * @param message
+     * @param details
+     * @return
+     */
+    private Template showError(Context context, String message, String details) {
+        context.put("title", "Error");
+        context.put("message", message);
+        context.put("details", details);
+        try {
+            return getTemplate("error.vm");
+        } catch (Exception e) {
+            try {
+                super.error((HttpServletRequest)context.get("req"), 
+                        (HttpServletResponse)context.get("res"), 
+                        e);
+            } catch (Exception e2) {
+                e2.printStackTrace();
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @param arguments
+     * @param context
+     * @param create
+     * @return
+     * @throws Exception
+     */
+    private String showSubscriptionWizardPage(Map arguments, Context context, boolean create) throws Exception {
+        HttpSession session = ((HttpServletRequest)context.get(REQUEST)).getSession(true);
+        if (create) {
+            context.put(Constants.ACTION_PARAM, "createSubscription");
+        } else {
+            // prefill from existing subscription
+            context.put(Constants.ACTION_PARAM, "editSubscription");
+            String subscriptionID = (String) arguments.get("subscriptionID");
+            int subID = Integer.parseInt(subscriptionID);
+            prefillFromSubscription(session, subID);
+        }
+        if (!arguments.containsKey("current_page")) {
+            return "sub_type-details.vm";
+        }
+        
+        String currentPage = (String) arguments.get("current_page");
+        String direction = (String) arguments.get("next_page");
+        
+        if (arguments.containsKey(Constants.HOST_QUERY_FIELD)) {
+            String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+            if (hostQuery != null && hostQuery.length() != 0)
+                arguments.remove(Constants.HOST_ID_FIELD);
+        }
+        if (arguments.containsKey(Constants.COLLECTION_QUERY_FIELD)) {
+            String collQuery = (String) arguments.get(Constants.COLLECTION_QUERY_FIELD);
+            if (collQuery != null && collQuery.length() != 0)
+                arguments.remove(Constants.COLLECTION_ID_FIELD);
+        }
+        
+        // save page arguments
+        savePageArgsToSession(currentPage, arguments, session);
+        
+        String nextPage = getNextPage(currentPage, direction);
+
+        // fill prefill
+        context.put("prefill", getPageArgsFromSession(nextPage, session));
+        
+        // fill preview
+        context.put("preview", getPagePreview(nextPage, session));
+        
+        // get page-specific stuff
+        if (nextPage.equals("host")) {
+            String[] hostNames;
+            try {
+                GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                hostNames = gsComm.getHostNames();
+            } catch (Exception e) {
+                hostNames = new String[] { "localhost" };
+            }
+            context.put("hostnames", hostNames);
+            session.setAttribute("hostnames", hostNames);
+        } else if (nextPage.equals("collection")) {
+            List hostNames = (List) arguments.get(Constants.HOST_ID_FIELD);
+            if (hostNames == null || hostNames.isEmpty()) {
+                hostNames = new Vector();
+                // no host names -> use host query
+                String[] hostsFromSession = (String[]) session.getAttribute("hostnames");
+                if (hostsFromSession == null || hostsFromSession.length == 0) {
+                    try {
+                        GreenstoneCommunicator gsComm = new GreenstoneCommunicator();
+                        hostsFromSession = gsComm.getHostNames();
+                    } catch (Exception e) {
+                        hostsFromSession = new String[] { "localhost" };
+                    }
+                }
+                String hostQuery = (String) arguments.get(Constants.HOST_QUERY_FIELD);
+                for (int i = 0; i < hostsFromSession.length; i++) {
+                    if (hostsFromSession[i] != null && hostsFromSession[i].indexOf(hostQuery) >= 0) {
+                        hostNames.add(hostsFromSession[i]);
+                    }
+                }
+            }
+            
+            Map collNames = new TreeMap();
+            for (Iterator iter = hostNames.iterator(); iter.hasNext();) {
+                String host = (String) iter.next();
+                Set collNamesForHost = new TreeSet();   
+                try {
+                    URL url = new URL("http://" + host + ":8080/soap/servlet/rpcrouter");
+                    GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+                    collNamesForHost.addAll(Arrays.asList(gsComm.getCollectionNames()));
+                } catch (Exception e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }	
+                collNames.put(host, collNamesForHost);
+            }
+            context.put("collectionnames", collNames);
+            context.put("hostnames", hostNames);
+        }
+        
+        return "sub_" + nextPage + ".vm";
+    }
+
+    /**
+     * @param session
+     * @param subID
+     */
+    private void prefillFromSubscription(HttpSession session, int subID) {
+        Subscription sub = ProfileStore.getInstance().getSubscription(subID);
+        Map typeArgs = new HashMap();
+        // TODO really fill stuff
+        savePageArgsToSession("type-details", typeArgs, session);
+        Map hostArgs = new HashMap();
+        savePageArgsToSession("host", hostArgs, session);
+        Map collArgs = new HashMap();
+        savePageArgsToSession("collection", collArgs, session);
+        Map notificationArgs = new HashMap();
+        savePageArgsToSession("notification", notificationArgs, session);
+    }
+
+    /**
+     * @param nextPage
+     * @param session
+     * @return
+     */
+    private Map getPagePreview(String nextPage, HttpSession session) {
+        Map preview = new TreeMap();
+        if (nextPage.equals("type-details")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("type-details", session));
+        if (nextPage.equals("host")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("host", session));
+        if (nextPage.equals("collection")) {
+            return preview;
+        }
+        preview.putAll(getPageArgsFromSession("collection", session));
+        return preview;
+    }
+
+    /**
+     * @param currentPage
+     * @param direction
+     * @return
+     * @throws Exception
+     */
+    private String getNextPage(String currentPage, String direction) throws Exception {
+        String nextPage;
+        if (currentPage.equals("host") && direction.equals("back")) {
+            nextPage = "type-details";
+        } else if (currentPage.equals("type-details") || (currentPage.equals("collection") && direction.equals("back"))) {
+            nextPage = "host";
+        } else if (currentPage.equals("host") || (currentPage.equals("notification") && direction.equals("back"))) {
+            nextPage = "collection";
+        } else if (currentPage.equals("collection")) {
+            nextPage = "notification";
+        } else {
+            throw new Exception("unknown combination of currentPage=" + currentPage + " and nextPage=" + direction);
+        }
+        return nextPage;
+    }
+
+    /**
+     * @param page
+     * @param session
+     * @return 
+     */
+    private Map getPageArgsFromSession(String page, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null || !pageArgs.containsKey(page))
+            return new TreeMap();
+        return (Map) pageArgs.get(page);
+    }
+
+    private Map getPageArgsFromSession(HttpSession session) {
+        Map result = new TreeMap();
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs != null) {
+            for (Iterator iter = pageArgs.values().iterator(); iter.hasNext();) {
+                Map args = (Map) iter.next();
+                result.putAll(args);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @param page
+     * @param arguments
+     * @param session
+     */
+    private void savePageArgsToSession(String page, Map arguments, HttpSession session) {
+        Map pageArgs = (Map) session.getAttribute("page_args");
+        if (pageArgs == null) {
+            pageArgs = new TreeMap();
+        }
+        pageArgs.put(page, arguments);
+        session.setAttribute("page_args", pageArgs);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Constants.java	(revision 21112)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 21, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Constants {
+    public static final String ACTION_PARAM = "action";
+    
+    public static final String HOST_QUERY_FIELD = "host_query";
+    public static final String COLLECTION_QUERY_FIELD = "collection_query";
+    public static final String HOST_ID_FIELD = "hostID";
+    public static final String COLLECTION_ID_FIELD = "collectionID";
+    public static final String HOST_COLLECTION_ID_FIELD = "hostCollectionID";
+    public static final String DOCUMENT_ID_FIELD = "documentID";
+    public static final String DOCUMENT_CONTENT_FIELD = "document_content";
+    public static final String DOCUMENT_TITLE_FIELD = "document_title";
+    public static final String TYPE_FIELD = "type";
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/EventStore.java	(revision 21112)
@@ -0,0 +1,169 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Date;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class EventStore {
+
+    private static EventStore instance;
+    
+    private EventStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static EventStore getInstance() {
+        if (instance == null) instance = new EventStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) throws DatabaseException, SQLException, ParseException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+
+        int eventID = saveEventToDatabase(event);
+        
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            int subID = sub.getId();
+            String sqlString = "INSERT INTO events_to_subs (event,subscription) " +
+            		"VALUES (" + eventID + "," + subID + ");";
+            statement.executeUpdate(sqlString);
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     * @throws ParseException
+     */
+    private int saveEventToDatabase(Map event) throws DatabaseException, SQLException, ParseException {
+        String eventString = mapAsString(event);
+        String timestamp = (String) event.get("timestamp");
+        timestamp = convertToSQLDatetime(timestamp);
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT id FROM events WHERE timestamp = " + timestamp + 
+        	" AND content = '" + eventString + "';";
+        ResultSet results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        
+        String insert = "INSERT INTO events (timestamp,content) " +
+        		"VALUES (" + timestamp + ",'" + eventString + "');"; 
+        statement.executeUpdate(insert);
+        results = statement.executeQuery(query);
+        if (results.next()) {
+            return results.getInt("id");
+        }
+        throw new DatabaseException("couldn't save event to database");
+    }
+
+    /**
+     * converts a timestamp from an event to a format recognised by MySQL
+     * @param timestamp
+     * @return
+     * @throws ParseException
+     */
+    private String convertToSQLDatetime(String timestamp) throws ParseException {
+        Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z").parse(timestamp);
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+
+    /**
+     * @param event
+     */
+    private String encodeSpecialChars(String string) {
+        String encodedString = string.replaceAll("&", "&amp;");
+        encodedString = encodedString.replaceAll("<", "&lt;");
+        encodedString = encodedString.replaceAll(">", "&gt;");
+        encodedString = encodedString.replaceAll("\'", "&apos;");
+        encodedString = encodedString.replaceAll("\"", "&quot;");
+        return encodedString;
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public Set getEvents(Integer subscriptionID) throws DatabaseException, SQLException {
+        Set result = new HashSet(); // can't use TreeSet, because we put Maps in there, and Map isn't instanceof Comparable
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String query = "SELECT content " +
+        		"FROM events e JOIN events_to_subs ets ON (e.id = ets.event) " +
+        		"WHERE ets.subscription = " + subscriptionID.intValue() + ";";
+        ResultSet results = statement.executeQuery(query);
+        while (results.next()) {
+            Map event = stringAsMap(results.getString("content"));
+            result.add(event);
+        }
+        return result;
+    }
+
+    private String mapAsString(Map map) {
+        StringBuffer buffer = new StringBuffer();
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) map.get(key);
+            String encodedValue = encodeSpecialChars(value);
+            buffer.append(key);
+            buffer.append(">"); // > has been encoded to &gt; in the original String
+            buffer.append(encodedValue);
+            if (iter.hasNext()) {
+                buffer.append(">>");
+            }
+        }
+        return buffer.toString();
+    }
+    
+    private Map stringAsMap(String string) {
+        Map map = new HashMap();
+        String[] lines = string.split(">>");
+        for (int i = 0; i < lines.length; i++) {
+            String line = lines[i];
+            String[] columns = line.split(">");
+            String key = columns[0];
+            String value = null;
+            if (columns.length > 1) {
+                value = columns[1];
+            }
+            map.put(key, value);
+        }
+        return map;
+    }
+    
+    public void cleanUp() {
+        // TODO implement method 
+        // delete all events that are too old (?)
+        // delete all events that don't match any subscriptions
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/GreenstoneCommunicator.java	(revision 21112)
@@ -0,0 +1,215 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.*;
+
+import org.apache.soap.Constants;
+import org.apache.soap.Fault;
+import org.apache.soap.rpc.*;
+import org.w3c.dom.*;
+
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicator {
+
+    private URL host;
+    private DocumentBuilder builder;
+
+    public GreenstoneCommunicator(URL host) throws Exception {
+        this.host = host;
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        try {
+            builder = factory.newDocumentBuilder();
+        } catch (ParserConfigurationException e) {
+            throw new Exception(e);
+        }
+
+    }
+
+    /**
+     * 
+     */
+    public GreenstoneCommunicator() {
+        
+        // TODO Auto-generated constructor stub
+    }
+
+    public String[] getHostNames() {
+        // TODO method implementation
+        return new String[] {"localhost"};
+    }
+    
+    public String[] getCollectionNames() throws Exception {
+        Document document = createSubsetDescriptionRequest("", "collectionList");
+
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("collection");
+        String[] result = new String[nodeList.getLength()];
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            Node name = node.getAttributes().getNamedItem("name");
+            result[i] = name.getNodeValue();
+        }
+        
+	System.out.println(ArrayHelper.toString(result));
+        
+        return result;
+    }
+    
+    public boolean supportsTextSearch(String collection) throws Exception {
+        Document document = createSubsetDescriptionRequest(collection, "serviceList");
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList nodeList = response.getElementsByTagName("service");
+        
+        boolean hasTextQuery = false;
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue(); 
+            if (name.equals("TextQuery")) {
+                hasTextQuery = true;
+                break;
+            }
+        }
+        
+        if (!hasTextQuery) return false; // doesn't support text queries at all
+        
+        document = createSubsetDescriptionRequest(collection + "/TextQuery", "paramList");
+        
+        response = sendToGreenstone(document.getDocumentElement());
+        
+        nodeList = response.getElementsByTagName("param");
+        
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String name = node.getAttributes().getNamedItem("name").getNodeValue();
+            if (name == null) break;
+            String value = node.getAttributes().getNamedItem("type").getNodeValue();
+            if (value == null) break;
+            if (name.equals("query") && value.equals("string")) {
+                return true;
+            }
+        }
+        
+        return false;
+    }
+    
+    public Set fullTextSearch(String collection, String query) throws Exception {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "process");
+        requestElement.setAttribute("to", collection + "/TextQuery");
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "query");
+        paramElement.setAttribute("value", query);
+        // BTS 3 what if numReturnedDocs < numMatchedDocs
+        paramListElement.appendChild(paramElement);
+        
+        
+        Set result = new TreeSet();
+        //DOM2Writer.serializeAsXML(root, new OutputStreamWriter(System.out));
+        
+        Element response = sendToGreenstone(document.getDocumentElement());
+        
+        NodeList documentNodes = response.getElementsByTagName("documentNode");
+        
+        for(int i = 0; i < documentNodes.getLength(); i++) {
+            Node node = documentNodes.item(i);
+            String docID = node.getAttributes().getNamedItem("nodeID").getNodeValue();
+            Matcher matcher = Pattern.compile("(\\d{8}:\\d+).*").matcher(docID);
+            if (matcher.matches()) {
+                docID = matcher.group(1);
+            }
+            result.add(docID);
+        }
+        
+        return result;
+    }
+    
+    private Element sendToGreenstone(Element message) throws Exception {
+        Call call = new Call();
+        // TODO get the uri from a configuration file
+        call.setTargetObjectURI("localsite");
+        call.setMethodName("process");
+        // set Encoding Style to use literal XML
+        call.setEncodingStyleURI(Constants.NS_URI_LITERAL_XML);
+        // Set Method Parameters - use literal xml
+        Parameter param = new Parameter("message", 
+                org.w3c.dom.Element.class,
+                message, 
+                Constants.NS_URI_LITERAL_XML);
+
+        Vector paramList = new Vector();
+        paramList.addElement(param);
+        call.setParams(paramList);
+
+        // Invoke the Service
+        Response response = call.invoke(host, "");
+
+        // Check for Faults
+        if (!response.generatedFault()) { // no error
+            // Extract Return value
+            Parameter result = response.getReturnValue();
+            return (Element) result.getValue();
+        } else {
+            //  Extract Fault Code and String
+            Fault f = response.getFault();
+            String faultCode = f.getFaultCode();
+            String faultString = f.getFaultString();
+            throw new Exception(faultCode + ": Fault occured: " + faultString);
+        }
+    }
+
+    /**
+     * @return
+     */
+    private Document createSubsetDescriptionRequest(String object, String subsetType) {
+        Document document = builder.newDocument();
+        Element root = document.createElement("message");
+        document.appendChild(root);
+    
+        Element requestElement = document.createElement("request");
+        requestElement.setAttribute("lang", "en");
+        requestElement.setAttribute("type", "describe");
+        requestElement.setAttribute("to", object);
+        root.appendChild(requestElement);
+    
+        Element paramListElement = document.createElement("paramList");
+        requestElement.appendChild(paramListElement);
+    
+        Element paramElement = document.createElement("param");
+        paramElement.setAttribute("name", "subset");
+        paramElement.setAttribute("value", subsetType);
+        paramListElement.appendChild(paramElement);
+        return document;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/NotificationStore.java	(revision 21112)
@@ -0,0 +1,77 @@
+/*
+ * Created on Nov 23, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class NotificationStore {
+
+    private static NotificationStore instance;
+    private Map eventsForSubscriptionID = new TreeMap();
+    
+    private NotificationStore() {
+    }
+    
+    /**
+     * @return
+     */
+    public static NotificationStore getInstance() {
+        if (instance == null) instance = new NotificationStore();
+        return instance;
+    }
+    
+    public void add(Map event, Set subscriptions) {
+        encodeValues(event);
+        for (Iterator iter = subscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            Integer subscriptionID = new Integer(sub.getId()); 
+            if (!eventsForSubscriptionID.containsKey(subscriptionID)) {
+                eventsForSubscriptionID.put(subscriptionID, new HashSet());
+            }
+            ((Set) eventsForSubscriptionID.get(subscriptionID)).add(event);
+        }
+    }
+
+    /**
+     * @param event
+     */
+    private void encodeValues(Map event) {
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            String encodedValue = value.replaceAll("&", "&amp;");
+            encodedValue = encodedValue.replaceAll("<", "&lt;");
+            encodedValue = encodedValue.replaceAll(">", "&gt;");
+            encodedValue = encodedValue.replaceAll("\'", "&apos;");
+            encodedValue = encodedValue.replaceAll("\"", "&quot;");
+            
+            event.put(key, encodedValue);
+        }
+    }
+
+    /**
+     * @param subscriptionID
+     * @return
+     */
+    public Set getEvents(Integer subscriptionID) {
+        if (eventsForSubscriptionID.containsKey(subscriptionID)) {
+            return Collections.unmodifiableSet((Set) eventsForSubscriptionID.get(subscriptionID));
+        }
+        return new TreeSet();
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/Notifier.java	(revision 21112)
@@ -0,0 +1,96 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.util.*;
+import java.util.Map;
+import java.util.Set;
+
+import javax.mail.MessagingException;
+
+import org.greenstone.gsdlas.profiles.Subscription;
+import org.greenstone.gsdlas.util.Mailer;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ * 
+ * Singleton
+ */
+public class Notifier {
+
+    private static Notifier instance; 
+    
+    private Notifier () {
+        // hide default constructor
+    }
+    
+    /**
+     * @return
+     */
+    public static Notifier getInstance() {
+        if (instance == null) instance = new Notifier();
+        return instance;
+    }
+
+    /**
+     * @param event
+     * @param matchedSubscriptions
+     */
+    public void sendNotifications(Map event, Set matchedSubscriptions) {
+        // TODO read smtp host and sender from configuration file
+        Mailer mailer = new Mailer(System.getProperty("user.name") + "@localhost", "localhost");
+        String eventText = eventToEmail(event);
+        String subject = "Event for your subscription";
+        
+        for (Iterator iter = matchedSubscriptions.iterator(); iter.hasNext();) {
+            Subscription sub = (Subscription) iter.next();
+            if (!sub.wantsEMailNotification()) continue;
+
+            StringBuffer message = new StringBuffer();
+            // TODO formatting of notification email via velocity?
+            message.append("You've created a subscription at the Greenstone Alerting Service running on " +
+            		"localhost:\n\n");
+            message.append(sub.toString());
+            message.append("\n");
+            message.append("An event has occurred that matches this subscription:\n\n");
+            message.append(eventText);
+            message.append("\n");
+            message.append("Go to http://localhost:8080/alerting/service?action=listSubscriptions to " +
+            		"change your subscriptions, or to http://localhost:8080/alerting/service?action=showEvents&subscriptionID=" + 
+            		sub.getId() + " to see all events that match the subscription.\n");
+            
+            String to = sub.getMailAddress();
+            
+            try {
+                mailer.sendMail(to, subject, message.toString());
+            } catch (MessagingException e) {
+                System.err.println("Couldn't notify for subscription " + sub.getId() + ": " + e.getMessage());
+                e.printStackTrace();
+            }
+        }
+    }
+
+    /**
+     * @param event
+     * @return
+     */
+    private String eventToEmail(Map event) {
+        StringBuffer message = new StringBuffer();
+        for (Iterator iter = event.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            String value = (String) event.get(key);
+            message.append(key + ": " + value + "\n");
+        }
+        return message.toString();
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/ProfileStore.java	(revision 21112)
@@ -0,0 +1,416 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.profiles.*;
+
+/**
+ * Storage for all Profiles. Singleton.
+ * 
+ * @author schweer
+ *  
+ */
+public class ProfileStore {
+
+    private static ProfileStore instance;
+
+    private Set noEqualsSubscriptions = new TreeSet();
+    private Map dontCareSubscriptions = new TreeMap();
+    private Map subscriptions = new TreeMap();
+    
+
+    private ProfileStore() {
+        // hide default constructor
+    }
+
+    /**
+     * @return the one existing instance of this class.
+     */
+    public static ProfileStore getInstance() {
+        if (instance == null) {
+            instance = new ProfileStore();
+        }
+        return instance;
+    }
+
+    /**
+     * @param subscription
+     *            the subscription to add to the profile store.
+     */
+    private void addSubscription(Subscription subscription) {
+        List documentIdPredicate = subscription.getPredicateList(Constants.DOCUMENT_ID_FIELD);
+        List collectionIdPredicate = subscription.getPredicateList(Constants.COLLECTION_ID_FIELD);
+        List hostIdPredicate = subscription.getPredicateList(Constants.HOST_ID_FIELD);
+        List typePredicate = subscription.getPredicateList(Constants.TYPE_FIELD);
+        
+        Integer subId = new Integer(subscription.getId());
+        
+        if (documentIdPredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.DOCUMENT_ID_FIELD)) {
+                dontCareSubscriptions.put(Constants.DOCUMENT_ID_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.DOCUMENT_ID_FIELD);
+            list.add(subId);
+        }
+        if (hostIdPredicate.isEmpty()) {
+            if (collectionIdPredicate.isEmpty()) {
+                if (!dontCareSubscriptions.containsKey(Constants.COLLECTION_ID_FIELD)) {
+                    dontCareSubscriptions.put(Constants.COLLECTION_ID_FIELD, new TreeSet());
+                }
+                Set list = (Set) dontCareSubscriptions.get(Constants.COLLECTION_ID_FIELD);
+                list.add(subId);
+            }
+        }
+        if (typePredicate.isEmpty()) {
+            if (!dontCareSubscriptions.containsKey(Constants.TYPE_FIELD)) {
+                dontCareSubscriptions.put(Constants.TYPE_FIELD, new TreeSet());
+            }
+            Set list = (Set) dontCareSubscriptions.get(Constants.TYPE_FIELD);
+            list.add(subId);
+        }
+        
+        if (documentIdPredicate.isEmpty() && collectionIdPredicate.isEmpty() 
+                && hostIdPredicate.isEmpty() && typePredicate.isEmpty()) {
+            noEqualsSubscriptions.add(subId);
+        }
+        subscriptions.put(subId, subscription);
+    }
+
+    /**
+     * @return
+     */
+    public Set getSubscriptionsWithoutEqualsPredicates() {
+        return Collections.unmodifiableSet(noEqualsSubscriptions);
+    }
+
+    /**
+     * Computes the list of equality subscriptions satisfied by the event.
+     * @param event
+     * @return
+     */
+    private Set getPartiallyMatchedSubscriptions(Map event) {
+        Set result = new TreeSet();
+        
+        EqualsPredicate predicate;
+        
+        // all subscriptions with matching type...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.TYPE_FIELD, 
+                (String)event.get(Constants.TYPE_FIELD));
+        if (predicate != null)
+            result.addAll(predicate.getSubscriptionIDs());
+        
+        // ...but only those subscriptions where the doc id matches...
+        Set docIDSubs = new TreeSet();
+        predicate = PredicateFactory.getEqualsPredicate(Constants.DOCUMENT_ID_FIELD, 
+                (String)event.get(Constants.DOCUMENT_ID_FIELD));
+        if (predicate != null)
+            docIDSubs.addAll(predicate.getSubscriptionIDs());
+        docIDSubs.addAll(getDontCareSubscriptions(Constants.DOCUMENT_ID_FIELD));
+        result.retainAll(docIDSubs);
+        
+        // ...and only those subscriptions where the host/collection id matches:
+        Set hostCollIDSubs = new TreeSet();
+        //   i.e.: host|coll is "don't care"...
+        hostCollIDSubs.addAll(getDontCareSubscriptions(Constants.COLLECTION_ID_FIELD));
+        //   ...or host matches (-> coll is "don't care" for this host)...
+        predicate = PredicateFactory.getEqualsPredicate(Constants.HOST_ID_FIELD, 
+                (String)event.get(Constants.HOST_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        //   ...or host|coll matches
+        predicate = PredicateFactory.getEqualsPredicate(Constants.COLLECTION_ID_FIELD, 
+                (String)event.get(Constants.COLLECTION_ID_FIELD));
+        if (predicate != null)
+            hostCollIDSubs.addAll(predicate.getSubscriptionIDs());
+        
+        result.retainAll(hostCollIDSubs);
+        
+        return result;
+    }
+
+    /**
+     * 
+     * @param gsComm 
+     * @return
+     * @throws UnsupportedFieldException
+     */
+    private Set findMatchedQueryPredicates(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedPreds = new TreeSet();
+        
+        // iterate through all document content predicates
+        // execute the query stored in the predicate
+        // predicate is matched iff the docID occurs in the query result
+        
+        for (Iterator iter = PredicateFactory.getQueryPredicates(Constants.DOCUMENT_CONTENT_FIELD).iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            String collection = (String) event.get(Constants.COLLECTION_ID_FIELD);
+            String query = predicate.getValue();
+            Set results;
+            try {
+                results = gsComm.fullTextSearch(collection, query);
+                if (results.contains(event.get(Constants.DOCUMENT_ID_FIELD))) {
+                    matchedPreds.add(predicate);
+                }
+            } catch (Exception e) {
+            }
+        }
+        
+        return matchedPreds;
+    }
+
+    /**
+     * Computes the set of SubstringMatchPredicates satisfied by the event.
+     * @param event
+     * 
+     * @return
+     */
+    private Set findMatchedSubstringMatchPredicates(Map event) {
+        Set result = new TreeSet();
+        
+        for (Iterator iter = PredicateFactory.getAllSubstringMatchPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate.isSatisfied(event)) {
+                result.add(predicate);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Filters the event against the profiles. It uses the equality-preferred
+     * algorithm as described in Fabret, Llirbat, Pereira and Shasha:
+     * <em>Efficient Matching for Content-based Publish/Subscribe 
+     * Systems</em>.
+     * @param event
+     * @param gsComm 
+     */
+    public Set filter(Map event, GreenstoneCommunicator gsComm) {
+        Set matchedSubscriptions = new TreeSet();
+    
+        Set partiallyMatchedSubscriptions 
+        	= getPartiallyMatchedSubscriptions(event);
+    
+        Set matchedPredicates = findMatchedSubstringMatchPredicates(event);
+        matchedPredicates.addAll(findMatchedQueryPredicates(event, gsComm));
+    
+        Map numMatchedPredicates = new TreeMap();
+        for (Iterator iter = matchedPredicates.iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            for (Iterator iterator = pred.getSubscriptionIDs().iterator(); iterator
+                    .hasNext();) {
+                Integer subId = (Integer) iterator.next();
+                Subscription sub = (Subscription) subscriptions.get(subId);
+                Integer count = (Integer) numMatchedPredicates.get(subId);
+                int newCountValue = (count == null ? 1 : count.intValue() + 1);
+                numMatchedPredicates.put(subId,
+                        new Integer(newCountValue));
+            }
+        }
+    
+        Set notUnmatchedSubscriptions = partiallyMatchedSubscriptions;
+        notUnmatchedSubscriptions.addAll(noEqualsSubscriptions);
+        for (Iterator iter = notUnmatchedSubscriptions.iterator(); iter
+                .hasNext();) {
+            Integer subId = (Integer) iter.next();
+            Subscription sub = (Subscription) subscriptions.get(subId);
+            int matchedPredicatesCount = 0;
+            if (numMatchedPredicates.get(subId) != null) {
+                matchedPredicatesCount = ((Integer) numMatchedPredicates.get(subId)).intValue();
+            }
+            if (matchedPredicatesCount == sub.getNumOfNonEqualsPredicates()) {
+                matchedSubscriptions.add(sub);
+            }
+        }
+        return matchedSubscriptions;
+    }
+
+
+    /**
+     * @param arguments
+     * @throws Exception
+     */
+    public void createSubscription(Map arguments) throws Exception {
+        Subscription sub = new Subscription(arguments);
+        addSubscription(sub);
+    }
+
+    
+    /**
+     * @param subscriptionID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void deleteSubscription(String subscriptionID) throws DatabaseException, SQLException {
+        Integer subID = new Integer(subscriptionID);
+        System.out.println("deleting subscription " + subscriptionID);
+        if (!subscriptions.containsKey(subID)) {
+            return;
+        }
+        Subscription sub = (Subscription) subscriptions.get(subID);
+        for (Iterator iter = dontCareSubscriptions.values().iterator(); iter.hasNext();) {
+            Set entry = (Set) iter.next();
+            if (entry.contains(subID))
+                entry.remove(subID);
+        }
+        if (noEqualsSubscriptions.contains(subID))
+            noEqualsSubscriptions.remove(subID);
+        for (Iterator iter = sub.getPredicates().iterator(); iter.hasNext();) {
+            Predicate pred = (Predicate) iter.next();
+            pred.removeSubscription(subID);
+        }
+
+        subscriptions.remove(subID);
+        
+        
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        
+        Statement statement = conn.createStatement();
+        
+        String sqlString = "DELETE FROM subscriptions WHERE id = " + subscriptionID;
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT predicate FROM subs_to_predicates " +
+        		"WHERE subscription = " + subscriptionID;
+        ResultSet predicates = statement.executeQuery(sqlString);
+        while (predicates.next()) {
+            // if there aren't any other subscriptions using this predicate, delete predicate
+            int predicateID = predicates.getInt("predicate");
+            
+        }
+        
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     */
+    public void changeSubscription(Map arguments, HttpSession session) {
+        String subscriptionID = (String) arguments.get("subscriptionID");
+        // BTS 14 really edit the subscription
+        
+    }
+
+    /**
+     * @param username
+     * @return
+     * @throws DatabaseException
+     */
+    public Set getAllSubscriptionsFor(String username) throws DatabaseException {
+        Set result = new TreeSet();
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT id FROM subscriptions " +
+            		"WHERE user like '" + username + "';";
+            ResultSet idList = statement.executeQuery(sqlString);
+            while (idList.next()) {
+                int id = idList.getInt("id");
+                result.add(subscriptions.get(new Integer(id)));
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        
+        return result;
+    }
+
+    /**
+     * @param string
+     * @return
+     */
+    private Set getDontCareSubscriptions(String field) {
+        Set result = new TreeSet();
+        if (dontCareSubscriptions.containsKey(field)) {
+            result = (Set) dontCareSubscriptions.get(field);
+        }
+        return result;
+    }
+
+    /**
+     * 
+     */
+    public void restoreFromDatabase() {
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            String sqlString = "SELECT * FROM subscriptions";
+            ResultSet subs = statement.executeQuery(sqlString);
+            while (subs.next()) {
+                // construct map for all predicates
+                int id = subs.getInt("id");
+                Map map = new TreeMap();
+                map.put("username", subs.getString("user"));
+                map.put("email", subs.getString("email"));
+                map.put("subscription_name", subs.getString("name"));
+                Vector ways = new Vector();
+                if (subs.getInt("rss") != 0) {
+                    ways.add("rss");
+                }
+                if (subs.getInt("page") != 0) {
+                    ways.add("page");
+                }
+                map.put("way", ways);
+                Statement stmnt = conn.createStatement();
+                sqlString = "SELECT p.type, p.field, p.value from subs_to_predicates stp join predicates p on(stp.predicate = p.id) where stp.subscription = " + id + ";";
+                ResultSet predicates = stmnt.executeQuery(sqlString);
+                while (predicates.next()) {
+                    String type = predicates.getString(Constants.TYPE_FIELD);
+                    String field = predicates.getString("field");
+                    if (type.equals(SubstringMatchPredicate.class.getName())) {
+                        if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                            field = Constants.COLLECTION_QUERY_FIELD;
+                        } else if (field.equals(Constants.HOST_ID_FIELD)) {
+                            field = Constants.HOST_QUERY_FIELD;
+                        }
+                    }
+                    String value = predicates.getString("value");
+                    if (type.equals(EqualsPredicate.class.getName())) {
+                        if (!map.containsKey(field)) {
+                            map.put(field, new Vector());
+                        }
+                        ((List) map.get(field)).add(value);
+                    } else {
+                        map.put(field, value);
+                    }
+                }
+                try {
+                    createSubscription(map);
+                } catch (Exception e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+            }
+        } catch (SQLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        } catch (DatabaseException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param i
+     * @return
+     */
+    public Subscription getSubscription(int id) {
+        return (Subscription) subscriptions.get(new Integer(id));
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/UnsupportedFieldException.java	(revision 21112)
@@ -0,0 +1,21 @@
+/*
+ * Created on Nov 8, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UnsupportedFieldException extends Exception {
+    public UnsupportedFieldException(String msg) {
+        super(msg);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseException.java	(revision 21112)
@@ -0,0 +1,29 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseException extends Exception {
+    
+    public DatabaseException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * @param string
+     */
+    public DatabaseException(String message) {
+        super(message);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/database/DatabaseManager.java	(revision 21112)
@@ -0,0 +1,56 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.database;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class DatabaseManager {
+
+    private static DatabaseManager instance;
+    private Connection database;
+
+    private DatabaseManager() {
+        // hide constructor
+    }
+    
+    public static DatabaseManager getInstance() {
+        if (instance == null) {
+            instance = new DatabaseManager();
+        }
+        return instance;
+    }
+    
+    public Connection getDatabaseConnection() throws DatabaseException {
+        if(database == null) {
+            String connectionString = "jdbc:mysql://localhost/alerting?user=gsdl3admin";
+            try {
+                Class.forName("com.mysql.jdbc.Driver").newInstance();
+                database = DriverManager.getConnection(connectionString);
+            } catch (Exception e) {
+                if (e instanceof SQLException) {
+                    SQLException sqle = (SQLException) e;
+                    System.err.println("SQLState: " + sqle.getSQLState());
+                }
+                throw new DatabaseException("Could not establish connection to database (" + connectionString + ")", e);
+            }
+        }
+        return database;
+    }
+    
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/EqualsPredicate.java	(revision 21112)
@@ -0,0 +1,35 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class EqualsPredicate extends Predicate {
+
+    public EqualsPredicate(String field, String id) {
+        super.field = field;
+        super.value = id;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String value = (String) event.get(field);
+        return value != null && value.equals(value);
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/ParseException.java	(revision 21112)
@@ -0,0 +1,26 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+/**
+ * Exception thrown when a Predicate cannot be created due to a syntax error.
+ * @author schweer
+ * 
+ */
+public class ParseException extends Exception {
+
+    /**
+     * @param message the exception message.
+     * @see Exception#Exception(String)
+     */
+    public ParseException(String message) {
+        super(message);
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Predicate.java	(revision 21112)
@@ -0,0 +1,175 @@
+/*
+ * Created on Nov 2, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+import org.greenstone.gsdlas.util.ArrayHelper;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public abstract class Predicate implements Comparable {
+
+    protected Set subscriptions = new TreeSet();
+    protected String field;
+    protected String value;
+    protected int id;
+    
+
+    public static final String[] singleValueFields = new String[] {
+            Constants.DOCUMENT_TITLE_FIELD,
+            Constants.DOCUMENT_CONTENT_FIELD,
+            "metadata_has_field",
+            Constants.HOST_QUERY_FIELD,
+            Constants.COLLECTION_QUERY_FIELD };
+    public static final String[] multiValueFields = new String[] {
+            Constants.TYPE_FIELD,
+            Constants.HOST_ID_FIELD,
+            Constants.COLLECTION_ID_FIELD,
+            Constants.HOST_COLLECTION_ID_FIELD,
+            Constants.DOCUMENT_ID_FIELD };
+    
+    public void addSubscription(int subscriptionID) {
+        subscriptions.add(new Integer(subscriptionID));
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getSubscriptions()
+     */
+    public Set getSubscriptionIDs() {
+        return Collections.unmodifiableSet(subscriptions);
+    }
+
+    public int getID() {
+        return id;
+    }
+    
+    public void setID(int i) {
+        this.id = i;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getKey()
+     */
+    public String getField() {
+        return field;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#getValue()
+     */
+    public String getValue() {
+        return value;
+    }
+
+    public String toString() {
+        return field + "=" + value;
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Predicate)) return false;
+        return id == ((Predicate)other).getID();
+    }
+    
+    public int compareTo(Object other) {
+        Predicate otherPredicate = (Predicate) other;
+        return new Integer(id).compareTo(new Integer(otherPredicate.getID()));
+    }
+    
+    public abstract boolean isSatisfied(Map event);
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isSingleValued(String key) {
+        return ArrayHelper.contains(singleValueFields, key);
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isMultiValued(String key) {
+        return ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @return
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    int saveToDatabase() throws SQLException, DatabaseException {
+        String query = "SELECT id FROM predicates " +
+        		"WHERE field = '" + field + 
+        		"' AND value = '" + value + 
+        		"' AND type = '" + this.getClass().getName() + "';";
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        System.out.println(query);
+        ResultSet result = statement.executeQuery(query);
+        if (result.next()) { // predicate already exists in database
+            int id = result.getInt("id"); 
+            System.out.println("predicate already exists with id " + id);
+            return id;
+        }
+        // predicate has been newly created
+        String insert = "INSERT INTO predicates (field, type, value) " +
+        "VALUES ('" + field + "', '" + this.getClass().getName() + "', '" + value + "');";
+        System.out.println(insert);
+        statement.execute(insert);
+        result = statement.executeQuery(query);
+        if (result.next()) {
+            return result.getInt("id");
+        } else {
+            throw new DatabaseException("could not save predicate");
+        }
+    }
+
+    /**
+     * @param key
+     * @return
+     */
+    public static boolean isFieldName(String key) {
+        return ArrayHelper.contains(singleValueFields, key) || ArrayHelper.contains(multiValueFields, key);
+    }
+
+    /**
+     * @param subID
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public void removeSubscription(Integer subID) throws SQLException, DatabaseException {
+        subscriptions.remove(subID);
+
+        Statement statement = DatabaseManager.getInstance().getDatabaseConnection().createStatement();
+        String sqlString = "DELETE FROM subs_to_predicates " +
+        	"WHERE subscription = " + subID + 
+        	" AND predicate = " + id;
+        statement.executeUpdate(sqlString);
+        
+        if (subscriptions.isEmpty()) {
+            PredicateFactory.deletePredicate(this);
+            sqlString = "DELETE FROM predicates WHERE id = " + id;
+            statement.executeUpdate(sqlString);
+        }
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/PredicateFactory.java	(revision 21112)
@@ -0,0 +1,245 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.SQLException;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public abstract class PredicateFactory {
+
+    private static int id = 0; 
+    
+    private static Map equalsPredicates = new TreeMap();
+    
+    // substring for collection_name -> SubstringMatchPredicates referring to it
+    private static Map collectionNameMatchPredicates = new TreeMap();
+    private static Map hostNameMatchPredicates = new TreeMap();
+    
+    // query -> QueryPredicates referring to it
+    private static Map documentTitleQueryPredicates = new TreeMap();
+    private static Map documentContentQueryPredicates = new TreeMap();
+    
+    
+    /**
+     * @param key
+     * @param value
+     * @return
+     * @throws ParseException
+     * @throws DatabaseException
+     * @throws SQLException
+     */
+    public static Predicate createPredicate(String key, String value)
+            throws SQLException, DatabaseException {
+        System.out.println("creating predicate for key " + key + " and value " + value);
+        if (!Predicate.isSingleValued(key)) {
+            return null;
+        }
+        Predicate result = null;
+        if (value == null || value.length() == 0) {
+            return null;
+        }
+        
+        if (Predicate.isMultiValued(key)) {
+            List list = new ArrayList();
+            list.add(value);
+            return (Predicate) createPredicates(key, list).get(0);
+        } else if (key.startsWith("document")) {
+            result = createQueryPredicate(key, value);
+        } else if (key.endsWith("_query")) {
+            result = createSubstringMatchPredicate(key, value);
+        } else {
+        	return null;
+        }
+        result.setID(result.saveToDatabase());
+        return result;
+    }
+    
+    public static List createPredicates(String key, List values) throws SQLException, DatabaseException {
+        if (!Predicate.isMultiValued(key)) {
+            return null;
+        }
+        List results = createEqualsPredicates(key, values);
+        for (Iterator iter = results.iterator(); iter.hasNext();) {
+            EqualsPredicate pred = (EqualsPredicate) iter.next();
+
+            pred.setID(pred.saveToDatabase());
+        }
+        return results;
+    }
+
+    /**
+     * @param key
+     * @param value
+     * @return
+     */
+    private static SubstringMatchPredicate createSubstringMatchPredicate(String key, String value) {
+        SubstringMatchPredicate predicate = null;
+        if (key.equals(Constants.COLLECTION_QUERY_FIELD)) {
+            if (collectionNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) collectionNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.COLLECTION_ID_FIELD, value);
+                collectionNameMatchPredicates.put(value, predicate);
+            }
+        } else if (key.equals(Constants.HOST_QUERY_FIELD)) {
+            if (hostNameMatchPredicates.containsKey(value)) {
+                predicate = (SubstringMatchPredicate) hostNameMatchPredicates.get(value);
+            } else {
+                predicate = new SubstringMatchPredicate(Constants.HOST_ID_FIELD, value);
+                hostNameMatchPredicates.put(value, predicate);
+            }
+        }
+        // TODO other fields?
+        return predicate;
+    }
+
+    /**
+     * @param key
+     * @param values
+     * @return
+     */
+    private static List createEqualsPredicates(String key, List values) {
+        List result = new Vector();
+        
+        Map keyToPredicates;
+        if (!equalsPredicates.containsKey(key)) {
+            keyToPredicates = new HashMap();
+            equalsPredicates.put(key, keyToPredicates);
+        }
+        keyToPredicates = (Map) equalsPredicates.get(key);
+        
+        for (Iterator iter = values.iterator(); iter.hasNext();) {
+            String value = (String) iter.next();
+            if (value == null || value.length() == 0) {
+                continue;
+            }
+
+            if (key.equals(Constants.HOST_ID_FIELD) && !(value.startsWith("http://"))) {
+                value = "http://" + value + ":8080/soap/servlet/rpcrouter";
+            }
+            
+            EqualsPredicate predicate = null;
+            if (keyToPredicates.containsKey(value)) {
+                predicate = (EqualsPredicate) keyToPredicates.get(value);
+            } else {
+                predicate = new EqualsPredicate(key, value);
+                keyToPredicates.put(value, predicate);
+            }
+            result.add(predicate);
+        }
+        return result;
+    }
+    
+    /**
+     * @param field
+     * @param query
+     * @return
+     */
+    private static QueryPredicate createQueryPredicate(String field,
+            String query) {
+        QueryPredicate predicate = null;
+        if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            if (!documentContentQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentContentQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentContentQueryPredicates.get(query);
+        } else if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            if (!documentTitleQueryPredicates.containsKey(query)) {
+                predicate = new QueryPredicate(field, query);
+                documentTitleQueryPredicates.put(query, predicate);
+            }
+            return (QueryPredicate) documentTitleQueryPredicates.get(query);
+        }
+        return predicate;
+    }
+
+    public static Collection getAllQueryPredicates() {
+        Set result = new TreeSet();
+        result.addAll(documentTitleQueryPredicates.values());
+        result.addAll(documentContentQueryPredicates.values());
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getQueryPredicates(String field) {
+        Set result = new TreeSet();
+        if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+            result.addAll(documentTitleQueryPredicates.values());
+        } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+            result.addAll(documentContentQueryPredicates.values());
+        }
+        return Collections.unmodifiableSet(result);
+    }
+    
+    public static Collection getAllSubstringMatchPredicates() {
+        Collection result = new Vector();
+        result.addAll(hostNameMatchPredicates.values());
+        result.addAll(collectionNameMatchPredicates.values());
+        return Collections.unmodifiableCollection(result);
+    }
+
+    /**
+     * @param string
+     * @param string2
+     * @return
+     */
+    public static EqualsPredicate getEqualsPredicate(String key, String value) {
+        EqualsPredicate result = null; 
+        if (equalsPredicates.containsKey(key)) {
+            result = (EqualsPredicate) ((Map)equalsPredicates.get(key)).get(value);
+        }
+        return result;
+    }
+
+    /**
+     * @param predicate
+     */
+    public static void deletePredicate(Predicate predicate) {
+        String field = predicate.getField();
+        String value = predicate.getValue();
+        
+        if (predicate instanceof EqualsPredicate) {
+            Map map = (Map) equalsPredicates.get(field);
+            if (map.containsKey(value)) {
+                map.remove(value);
+            }
+        } else if (predicate instanceof QueryPredicate) {
+            if (field.equals(Constants.DOCUMENT_TITLE_FIELD)) {
+                documentTitleQueryPredicates.remove(value);
+            } else if (field.equals(Constants.DOCUMENT_CONTENT_FIELD)) {
+                documentContentQueryPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else if (predicate instanceof SubstringMatchPredicate) {
+            if (field.equals(Constants.HOST_ID_FIELD)) {
+                hostNameMatchPredicates.remove(value);
+            } else if (field.equals(Constants.COLLECTION_ID_FIELD)) {
+                collectionNameMatchPredicates.remove(value);
+            } else {
+                System.err.println("problem while removing: unknown field " + field);
+            }
+        } else {
+            System.err.println("problem while removing: unknown class " + predicate.getClass().getName());
+        }
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Profile.java	(revision 21112)
@@ -0,0 +1,33 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.List;
+import java.util.Vector;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Profile {
+    private List subscriptions;
+
+    private String user;
+
+    public Profile(String user) {
+        this.user = user;
+        subscriptions = new Vector();
+    }
+
+    public void add(Subscription sub) {
+        subscriptions.add(sub);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/QueryPredicate.java	(revision 21112)
@@ -0,0 +1,37 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.*;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class QueryPredicate extends Predicate {
+    
+    private Set containsWords = new TreeSet();
+    private Set notContainsWords = new TreeSet();
+    
+    public QueryPredicate (String field, String query) {
+        super.field = field;
+        super.value = query;
+    }
+    public boolean isSatisfied(Map event) {
+        return true;
+    }
+    
+    public String toString() {
+        return field + " matches " + value;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/Subscription.java	(revision 21112)
@@ -0,0 +1,300 @@
+/*
+ * Created on Oct 27, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.sql.*;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.*;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.greenstone.gsdlas.Constants;
+import org.greenstone.gsdlas.database.DatabaseException;
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class Subscription implements Comparable {
+    private Map map;
+    
+    private int id;
+    private String username;
+    private String name;
+    private String email;
+    private boolean rssNotification;
+    private boolean eventsSincePageNotification;
+    
+    private int numOfNonEqualsPredicates;
+    
+    public Subscription(Map valueMap) throws DatabaseException, SQLException {
+        valueMap = cleanHostCollectionIDs(valueMap);
+        map = new TreeMap();
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            
+            if (!Predicate.isFieldName(key)) {
+                continue;
+            }
+            
+            Object value = valueMap.get(key);
+            
+            if (value instanceof String) {
+                Predicate predicate = PredicateFactory.createPredicate(key, (String) value);  
+                map.put(key, predicate);
+
+                if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                    numOfNonEqualsPredicates++;
+                }
+            } else if (value instanceof List) {
+                List values = (List)value;
+                List predicates = PredicateFactory.createPredicates(key, values);
+                map.put(key, predicates);
+                for (Iterator iterator = predicates.iterator(); iterator
+                        .hasNext();) {
+                    Predicate predicate = (Predicate) iterator.next();
+                    
+                    if (predicate != null && !(predicate instanceof EqualsPredicate)) {
+                        numOfNonEqualsPredicates++;
+                    }
+                }
+            } 
+            
+        }
+        
+        System.out.println("finished creating predicates");
+        
+        username = (String) valueMap.get("username");
+        name = (String) valueMap.get("subscription_name");
+        email = (String) valueMap.get("email");
+        rssNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("rss");
+        eventsSincePageNotification = valueMap.containsKey("way") && ((List)valueMap.get("way")).contains("page");
+        id = saveToDatabase();
+        
+        for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+            Predicate predicate = (Predicate) iter.next();
+            if (predicate != null) {
+                predicate.addSubscription(id);
+            }
+        }
+    }
+    
+    /**
+     * @param valueMap
+     * @return
+     */
+    private Map cleanHostCollectionIDs(Map valueMap) {
+        Map result = new TreeMap();
+        Pattern pattern = Pattern.compile("^(.*)\\|(.*)$");
+        for (Iterator iter = valueMap.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            if (!key.equals(Constants.HOST_COLLECTION_ID_FIELD)) {
+                result.put(key, valueMap.get(key));
+                continue;
+            }
+            if (key.equals(Constants.HOST_ID_FIELD)) { continue; }
+            
+            List values = (List) valueMap.get(key);
+            List hostIDs = new Vector();
+            List collIDs = new Vector();
+            for (Iterator iterator = values.iterator(); iterator.hasNext();) {
+                String id = (String) iterator.next();
+                Matcher matcher = pattern.matcher(id);
+                if (matcher.matches()) {
+                    String host = null;
+                    String coll = null;
+                    try {
+                        host = matcher.group(1);
+                        coll = matcher.group(2);
+                    } catch (IndexOutOfBoundsException e) { /*ignore*/ }
+                    if (host != null && host.length() > 0) {
+                        if (coll != null && coll.length() > 0) {
+                            if (!collIDs.contains(id)) collIDs.add(id);
+                        } else {
+                            if (!hostIDs.contains(host)) hostIDs.add(host);
+                        }
+                    }
+                }
+            }
+            result.put(Constants.HOST_ID_FIELD, hostIDs);
+            result.put(Constants.COLLECTION_ID_FIELD, collIDs);
+        }
+        return result;
+    }
+
+    public boolean containsKey(Object key) {
+        return map.containsKey(key);
+    }
+    
+    public boolean containsValue(Object value) {
+        return map.containsValue(value);
+    }
+    
+    public Predicate getPredicate(String field) {
+        if (map.get(field) instanceof Predicate) {
+            return (Predicate) map.get(field);
+        }
+        return null;
+    }
+    
+    public List getPredicateList(String field) {
+        if (map.get(field) instanceof List) {
+            return (List) map.get(field);
+        }
+        return new Vector();
+    }
+    
+    public Set keySet() {
+        return map.keySet();
+    }
+    
+    public Set getPredicates() {
+        Set result = new HashSet();
+        for (Iterator iter = map.values().iterator(); iter.hasNext();) {
+            Object next = iter.next();
+            if (next == null) {
+                continue;
+            } else if (next instanceof Predicate) {
+                result.add(next);
+            } else if (next instanceof List) {
+                result.addAll((Collection) next);
+            }
+        }
+        return result;
+    }
+    
+    /**
+     * @return
+     */
+    public int getId() {
+        return id;
+    }
+    
+    public int getNumOfNonEqualsPredicates() {
+        return numOfNonEqualsPredicates;
+    }
+    
+    public String toString() {
+        StringBuffer result = new StringBuffer("Subscription ");
+        result.append(name != null ? name : "(no name) ");
+        result.append(" (id ");
+        result.append(id);
+        result.append("):\n");
+        for (Iterator iter = map.keySet().iterator(); iter.hasNext();) {
+            String key = (String) iter.next();
+            Object value = map.get(key);
+            if (value instanceof Predicate) {
+                Predicate pred = (Predicate) value;
+                result.append(pred);
+            }
+            if (value instanceof List) {
+                List list = (List) value;
+                for (Iterator iterator = list.iterator(); iterator.hasNext();) {
+                    Predicate pred = (Predicate) iterator.next();
+                    result.append(pred);
+                    if (iterator.hasNext()) {
+                        result.append(" or ");
+                    }
+                }
+            }
+            result.append("\n");
+        }
+        return result.toString();
+    }
+    
+    public boolean equals(Object other) {
+        if (other == null || !(other instanceof Subscription)) return false;
+        return id == ((Subscription)other).id;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Comparable#compareTo(java.lang.Object)
+     */
+    public int compareTo(Object arg0) {
+        Subscription other = (Subscription) arg0;
+        return new Integer(id).compareTo(new Integer(other.id));
+    }
+    
+    private int saveToDatabase() throws DatabaseException, SQLException {
+        Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+        Statement statement = conn.createStatement();
+        String sqlString;
+        boolean initial = true;
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        ResultSet result = statement.executeQuery(sqlString);
+        if (result.next()) {
+            initial = false;
+            this.id = result.getInt("id");
+        }
+        
+        if (initial) {
+            sqlString = "INSERT INTO subscriptions (name, email, rss, page, user) " +
+            		"VALUES ('" + name + "','" + email + "'," + (rssNotification ? 1 : 0) +
+            		"," + (eventsSincePageNotification ? 1: 0)+ ", '" + username + "');";
+        } else {
+            sqlString = "UPDATE subscriptions SET name='" + name + "', email='" +
+            		email + "', rss=" + (rssNotification ? 1 : 0) + ",page=" + 
+            		(eventsSincePageNotification ? 1 : 0)+ " WHERE id=" + this.id + ";";
+            // cannot change user
+        }
+        statement.executeUpdate(sqlString);
+        
+        sqlString = "SELECT id FROM subscriptions WHERE name like '" + name + 
+        			"' AND email like '" + email + "' AND rss=" + (rssNotification ? 1 : 0) 
+        			+ " AND page=" + (eventsSincePageNotification ? 1 : 0) + " AND user like '" +
+        			username + "';";
+        System.out.println(sqlString);
+        result = statement.executeQuery(sqlString);
+        
+        if (result.next()) {
+            id = result.getInt("id");
+        } else {
+            throw new DatabaseException("Couldn't save subscription");
+        }
+        
+        if (initial) {
+           for (Iterator iter = getPredicates().iterator(); iter.hasNext();) {
+               Predicate predicate = (Predicate) iter.next();
+               sqlString = "INSERT INTO subs_to_predicates (subscription, predicate) " +
+               		"VALUES (" + id + "," + predicate.getID() + ");";
+               statement.executeUpdate(sqlString);
+           }
+        }
+        return id;
+    }
+
+    /**
+     * @return
+     */
+    public boolean wantsEMailNotification() {
+        return email != null && email.length() > 1;
+    }
+
+    /**
+     * @return
+     */
+    public String getMailAddress() {
+        return email;
+    }
+    
+    public String getName() {
+        return name;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/profiles/SubstringMatchPredicate.java	(revision 21112)
@@ -0,0 +1,39 @@
+/*
+ * Created on Nov 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.profiles;
+
+import java.util.Map;
+
+
+/**
+ * @author schweer
+ * 
+ * TODO To change the template for this generated type comment go to Window -
+ * Preferences - Java - Code Style - Code Templates
+ */
+public class SubstringMatchPredicate extends Predicate {
+
+    public SubstringMatchPredicate(String field, String substring) {
+        super.field = field;
+        super.value = substring;
+    }
+
+    /* (non-Javadoc)
+     * @see org.greenstone.gsdlas.profiles.Predicate#isSatisfied(java.util.Map)
+     */
+    public boolean isSatisfied(Map event) {
+        String fieldValue = (String) event.get(field);
+        return fieldValue != null && fieldValue.indexOf(value) >= 0;
+    }
+
+    public String toString() {
+        return field + " contains " + value;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/PasswordMismatchException.java	(revision 21112)
@@ -0,0 +1,21 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PasswordMismatchException extends Exception {
+    public PasswordMismatchException(String message) {
+        super(message);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManagementException.java	(revision 21112)
@@ -0,0 +1,31 @@
+/*
+ * Created on Dec 3, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManagementException extends Exception {
+
+    /**
+     * @param e
+     */
+    public UserManagementException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    
+    public UserManagementException(String message) {
+        super(message);
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/users/UserManager.java	(revision 21112)
@@ -0,0 +1,121 @@
+/*
+ * Created on Dec 1, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.users;
+
+import java.sql.*;
+import java.util.Map;
+
+import javax.servlet.http.HttpSession;
+
+import org.greenstone.gsdlas.database.DatabaseManager;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class UserManager {
+    private static UserManager instance;
+    
+    private UserManager() {
+        // hide constructor
+    }
+    
+    static public UserManager getInstance() {
+        if (instance == null) {
+            instance = new UserManager();
+        }
+        return instance;
+    }
+
+    /**
+     * @param session
+     * @return
+     */
+    public boolean isLoggedIn(HttpSession session) {
+        if (session.getCreationTime() - session.getLastAccessedTime() > session.getMaxInactiveInterval())
+            return false;
+        return session.getAttribute("username") != null;
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws PasswordMismatchException
+     * @throws UserManagementException
+     */
+    public void createUser(Map arguments, HttpSession session) throws PasswordMismatchException, UserManagementException {
+        if (!arguments.get("password").equals(arguments.get("password2"))) {
+            throw new PasswordMismatchException("The passwords don't match");
+        }
+        String username = (String) arguments.get("username");
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+//        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+//        } catch (NoSuchAlgorithmException e) {
+//            e.printStackTrace();
+//            throw new UserManagementException("could not create user", e);
+//        }
+        
+        try {
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            statement.executeUpdate("INSERT INTO users (username, password) " +
+            		"VALUES ('" + username + "','" + new String(pwdHash) + "')");
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not create user", e);
+        }
+        
+    }
+
+    /**
+     * @param arguments
+     * @param session
+     * @throws UserManagementException
+     */
+    public void loginUser(Map arguments, HttpSession session) throws UserManagementException {
+        String username = (String) arguments.get("username");
+        
+        if (isLoggedIn(session) && session.getAttribute("username").equals(username)) {
+            return; // already logged in
+        }
+        
+        byte[] password = ((String) arguments.get("password")).getBytes();
+        
+        byte[] pwdHash = password;
+        try {
+//            pwdHash = MessageDigest.getInstance("MD5").digest(password);
+            
+            Connection conn = DatabaseManager.getInstance().getDatabaseConnection();
+            Statement statement = conn.createStatement();
+            ResultSet results = statement.executeQuery("SELECT password " +
+            		"FROM users WHERE username like '" + username + "';");
+            String pwdFromDB = "";
+            if(results.next()) {
+                pwdFromDB = results.getString("password");
+            }
+            if (!pwdFromDB.equals(new String(pwdHash))) {
+                throw new PasswordMismatchException("user " + username 
+                        + " is unknown, or the passwords don't match");
+            }
+            session.setAttribute("username", username);
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new UserManagementException("could not login user", e);
+        }
+
+    }
+    
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/ArrayHelper.java	(revision 21112)
@@ -0,0 +1,40 @@
+/*
+ * Created on Dec 5, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Arrays;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ArrayHelper {
+    static public boolean contains(String[] array, String element) {
+        if (array == null || element == null) {
+            return false;
+        }
+        Arrays.sort(array);
+        int index = Arrays.binarySearch(array, element);
+        return index > -1 && index <= array.length;
+    }
+
+    static public String toString(String [] array) {
+	StringBuffer buffer = new StringBuffer();
+	buffer.append("[");
+	for (int i=0; i<array.length; i++) {
+	    buffer.append((String)array[i]);
+	    buffer.append(",");
+	}
+	buffer.append("]");
+	return buffer.toString();
+
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/Mailer.java	(revision 21112)
@@ -0,0 +1,58 @@
+/*
+ * Created on Jan 1, 2005
+ * Copyright (C) 2004, 2005 Andrea Schweer
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.Properties;
+
+import javax.mail.*;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class Mailer {
+    
+    private String fromAddress;
+    private String mailHost;
+    
+    public Mailer (String from, String host) {
+        fromAddress = from;
+        mailHost = host;
+    }
+    
+    public void sendMail(String to, String subject, String message) throws MessagingException {
+        // Set the host smtp address
+        Properties props = new Properties();
+        props.put("mail.smtp.host", mailHost);
+        
+        // create some properties and get the default Session
+        //Session session = Session.getDefaultInstance(props, null);
+        Session session = Session.getInstance(props);
+        
+        // create a message
+        Message msg = new MimeMessage(session);
+        
+        // set the from and to address
+        InternetAddress addressFrom = new InternetAddress(fromAddress);
+        msg.setFrom(addressFrom);
+        
+        InternetAddress[] addressTo = new InternetAddress[1];
+        addressTo[0] = new InternetAddress(to);
+        msg.setRecipients(Message.RecipientType.TO, addressTo);
+        
+        // Setting the Subject and Content Type
+        msg.setSubject(subject);
+        msg.setContent(message, "text/plain");
+        Transport.send(msg);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/gsdlas/util/PatriciaTree.java	(revision 21112)
@@ -0,0 +1,242 @@
+/*
+ * Created on Nov 11, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ * 
+ */
+package org.greenstone.gsdlas.util;
+
+import java.util.*;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class PatriciaTree {
+	
+	private Node root;
+	
+	public PatriciaTree () {
+	}
+	
+	public void put(CharSequence key, Object value) {
+		root = put(key, value, root, 0);
+	}
+	
+	public Set get(CharSequence key) {
+		return get(key, root, 0);
+	}
+	
+	public String toString() {
+	    return toPreorderString(root).toString();
+	}
+	
+	private StringBuffer toPreorderString(Node root) {
+	    StringBuffer buffer = new StringBuffer("(\n");
+	    buffer.append(root);
+	    buffer.append("\nchildren:");
+	    if (root.hasChildren()) {
+	        for (Iterator iter = root.children.keySet().iterator(); iter.hasNext();) {
+	            Character key = (Character) iter.next();
+	            Node child = root.getChild(key);
+	            buffer.append("\n");
+	            buffer.append(key);
+	            buffer.append("->");
+	            buffer.append(toPreorderString(child));
+	        }
+	    }
+	    
+	    buffer.append("\n)");
+	    return buffer;
+	}
+	
+	/**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return the set of values associated with <code>key</code>
+	 */
+	private Set get(CharSequence key, Node root, int position) {
+		Set result = new HashSet();
+		
+		int i = findFirstMismatch(key, root, position);
+		
+		if (i == root.skipped.length()) {
+			Character nextChar = new Character(key.charAt(position + i));
+			if (key.length() > position + i + 1) { // nextChar is not last character
+				if (root.hasChild(nextChar)) {
+					result.addAll(get(key, root.getChild(nextChar), position + i + 1));
+				}	
+			} else {
+				result.addAll(root.getValues(nextChar));
+			}
+		}
+		
+		return result;
+	}
+
+	/**
+	 * @param key
+	 * @param value
+	 * @param root
+	 * @param position
+	 * @return the new root node
+	 */
+	private Node put(CharSequence key, Object value, Node root, int position) {
+	    if (root == null) {
+	        root = new Node();
+	        root.skipped = key.subSequence(position, key.length() - 1);
+	        root.addValue(new Character(key.charAt(key.length() - 1)), value);
+	        return root;
+	    }
+	    
+	    if (root.skipped == null || root.skipped.length() == 0) {
+	        addAsValueOrChild(key, value, root, position);
+	        return root;
+	    }
+	    
+	    int i = findFirstMismatch(key, root, position);
+	    if (i >= root.skipped.length() && position + i < key.length()) {
+	        addAsValueOrChild(key, value, root, position + i);
+	        return root;
+	    }
+	    
+	    if (i >= root.skipped.length() && isLastCharacter(key, position + i - 1)) {
+	        Node newRoot = new Node();
+	        newRoot.skipped = root.skipped.subSequence(0, i - 1);
+	        newRoot.addChild(new Character(root.skipped.charAt(i - 1)), root);
+	        try {
+		        root.skipped = root.skipped.subSequence(i, root.skipped.length());
+		    } catch (StringIndexOutOfBoundsException e) {
+		        root.skipped = "";
+		    }
+		    addAsValueOrChild(key, value, newRoot, position + i - 1);
+		    return newRoot;
+	    }
+	    
+	    Node newRoot = new Node();
+	    newRoot.skipped = root.skipped.subSequence(0, i);
+	    newRoot.addChild(new Character(root.skipped.charAt(i)), root);
+	    try {
+	        root.skipped = root.skipped.subSequence(i + 1, root.skipped.length());
+	    } catch (StringIndexOutOfBoundsException e) {
+	        root.skipped = "";
+	    }
+	    
+	    addAsValueOrChild(key, value, newRoot, position + i);
+	    
+	    return newRoot;
+	}
+
+    /**
+     * @param key
+     * @param value
+     * @param root
+     * @param position
+     */
+    private void addAsValueOrChild(CharSequence key, Object value, Node root, int position) {
+        Character nextChar = new Character(key.charAt(position)); 
+        if (isLastCharacter(key, position)) {
+            root.addValue(nextChar, value); 
+        } else {
+            Node newChild = put(key, value, root.getChild(nextChar), position + 1);
+            root.addChild(nextChar, newChild);
+        }
+    }
+
+    /**
+     * @param key
+     * @param position
+     * @return
+     */
+    private boolean isLastCharacter(CharSequence key, int position) {
+        return key.length() - position == 1;
+    }
+
+    /**
+	 * @param key
+	 * @param root
+	 * @param position
+	 * @return
+	 */
+	private int findFirstMismatch(CharSequence key, Node root, int position) {
+		int i = 0;
+		while (i < root.skipped.length()) {
+			try {
+				if (root.skipped.charAt(i) == key.charAt(position + i)) {
+					i++;
+				} else {
+					return i;
+				}
+			} catch (StringIndexOutOfBoundsException e) {
+				return i;
+			}
+		}
+		return i;
+	}
+
+
+	private class Node {
+		private CharSequence skipped = "";
+		private Map children;
+		private Map values;
+		
+		/**
+		 * @return
+		 */
+		public Set getValues(Character key) {
+			if (values == null || !values.containsKey(key))
+				return new HashSet();
+			return (Set) values.get(key);
+		}
+
+		boolean hasChild(Character key) {
+			return children != null && children.containsKey(key);
+		}
+		
+		boolean hasChildren() {
+			return children != null && !children.isEmpty();
+		}
+		
+		void addChild(Character key, Node child) {
+			if (children == null ) {
+				children = new TreeMap();
+			}
+			children.put(key, child);
+		}
+		
+		Node getChild(Character key) {
+			if (children == null) return null;
+			return (Node) children.get(key);
+		}
+		
+		void addValue(Character key, Object value) {
+			if (values == null) {
+				values = new TreeMap();
+			}
+			Set valueSet;
+			if (values.containsKey(key)) {
+				valueSet = (Set) values.get(key);
+			} else {
+				valueSet = new HashSet();
+				values.put(key, valueSet);
+			}
+			valueSet.add(value);
+		}
+		
+		public String toString() {
+		    StringBuffer buffer = new StringBuffer();
+		    buffer.append(skipped);
+		    buffer.append("\nvalues:");
+		    if (values != null) {
+		        buffer.append(values);
+		    }
+		    return buffer.toString();
+		}
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractClientResponse.java	(revision 21112)
@@ -0,0 +1,68 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class AbstractClientResponse implements ClientResponse
+{ private boolean success;
+  private Map     values;
+
+  public AbstractClientResponse()
+  { this.success = true;
+    this.values  = new HashMap();
+  }
+  
+  public boolean isSuccessful()
+  { return this.success;
+  }
+  
+  public void setSuccessful(boolean success)
+  { this.success = success;
+  }
+  
+  public void addValue(String label, String value)
+  { if (this.values.get(label) == null)
+    { this.setValue(label, value);
+    }
+    else
+    { Vector set = (Vector) this.values.get(label);
+      set.addElement(value);
+    }
+  }
+  
+  public void setValue(String label, String value)
+  { Vector set = new Vector();
+    set.addElement(value);
+    this.values.put(label, set);
+  }
+  
+  public Vector getValues(String label)
+  { return (Vector) this.values.get(label);
+  }
+
+  public Set getLabels()
+  { return this.values.keySet();
+  }
+
+  public String toString()
+  { Iterator keys = this.values.keySet().iterator();
+
+    StringBuffer buffer = new StringBuffer();
+
+    while (keys.hasNext())
+    { String key = keys.next().toString();
+
+      buffer.append("  " +  key + ":\n");
+
+      Vector vals = (Vector) this.values.get(key);
+      Enumeration valist = vals.elements();
+      while (valist.hasMoreElements())
+	{ String nextVal = valist.nextElement().toString();
+
+	  buffer.append("    ");
+	  buffer.append(nextVal);
+	  buffer.append("\n");
+	}
+    }
+    return buffer.toString();
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/AbstractServerAction.java	(revision 21112)
@@ -0,0 +1,71 @@
+package org.greenstone.p2p;
+
+import java.net.*;
+import java.util.*;
+
+public abstract class AbstractServerAction implements NameServerAction
+{ NameServer host;
+
+  public AbstractServerAction(NameServer host)
+  { this.host = host;
+  }
+
+  public abstract String getActionName();
+  
+  public void propagateAction(URL forwarder, Map parameters)
+  { Enumeration subServers = host.subServers();
+  
+    // poll all child servers
+    while (subServers.hasMoreElements())
+    { URL server = (URL) subServers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.propagateActionViaServer(server, parameters, forwarder);
+      }
+    }
+    
+    // poll upwards to parent
+    if (host.getParent() != null &&
+	!host.getParent().equals(forwarder))
+    { this.propagateActionViaServer(host.getParent(), parameters, forwarder);
+    }
+  }
+
+  public void doAction(URL forwarder, Map parameters)
+  { // start propagating the action outwards
+    this.propagateAction(forwarder, parameters);
+  
+    // now execute it on our children
+    Enumeration servers = host.servers();
+  
+    while (servers.hasMoreElements())
+    { URL server = (URL) servers.nextElement();
+      
+      if (!server.equals(forwarder))
+      { // send server the action
+        String response = this.doActionOnServer(server, parameters, forwarder);
+      }
+    }
+  }
+  
+  String propagateActionViaServer(URL server, Map paramters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to server, in a separate thread
+    // get responses
+    // forward those back
+    return "";
+  }
+  
+  String doActionOnServer(URL server, Map parameters, URL responseTo)
+  { String actionName = this.getActionName();
+  
+    // send action to the destination server (as a URL)
+    
+    // get response (as HTML)
+    
+    // forward it back...
+    return "";
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/Client.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/Client.java	(revision 21112)
@@ -0,0 +1,40 @@
+import java.io.*;
+import java.net.*;
+
+public class Client {
+    public static String read() throws IOException {
+
+        Socket echoSocket = null;
+        PrintWriter out = null;
+        BufferedReader in = null;
+
+        try {
+            echoSocket = new Socket("taranis", 7);
+            out = new PrintWriter(echoSocket.getOutputStream(), true);
+            in = new BufferedReader(new InputStreamReader(
+                                        echoSocket.getInputStream()));
+        } catch (UnknownHostException e) {
+            System.err.println("Don't know about host: taranis.");
+            System.exit(1);
+        } catch (IOException e) {
+            System.err.println("Couldn't get I/O for "
+                               + "the connection to: taranis.");
+            System.exit(1);
+        }
+
+        BufferedReader stdIn = new BufferedReader(
+                                   new InputStreamReader(System.in));
+        String userInput;
+
+        while ((userInput = stdIn.readLine()) != null) {
+            out.println(userInput);
+            System.out.println("echo: " + in.readLine());
+        }
+
+        out.close();
+        in.close();
+        stdIn.close();
+        echoSocket.close();
+	return "";
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ClientResponse.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public interface ClientResponse
+{ boolean isSuccessful(); // obvious
+  Vector  getValues(String label); // get the values associated with a given
+                                   // label; which values are available varies
+                                   // from action to action
+  Set     getLabels();             // get the labels in this response
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/DefaultServerAction.java	(revision 21112)
@@ -0,0 +1,13 @@
+package org.greenstone.p2p;
+
+public class DefaultServerAction extends AbstractServerAction
+{ public static final String DEFAULT_ACTION = "default";
+
+  public DefaultServerAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return DEFAULT_ACTION;
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneClient.java	(revision 21112)
@@ -0,0 +1,31 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public interface GreenstoneClient
+{ /**
+   * This interface needs to be fulfilled by a Greenstone machine wanting to
+   * register and receive messages from the Greenstone Distributed Service.
+   *
+   */
+
+  /**
+   * This method is called by a GDS server when an action (message) has a response
+   * sent in reply.  The Greenstone server that sends the action will have this
+   * method called for each response received from the GDS network.
+   *
+   * @param: <code>NameServerAction</code> - at present, </code>null</code>.
+   * This method *must* be synchronized
+   * @param: <code>ClientResponse</code> - is a simple wrapper for the Map class.
+   * 
+   */
+  public void receiveResponse(NameServerAction action, ClientResponse response);
+
+  /**
+   *
+   */
+  public void receiveMessage(String message, Map parameters);
+  public String getIdentifier();
+  public URL    getURL();
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneInterface.java	(revision 21112)
@@ -0,0 +1,94 @@
+
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+/**
+ * This is the class that provides a set of functions for a Greenstone server to call
+ * in order to send information/messages into a GDS network.
+ */
+
+public class GreenstoneInterface
+{ private GreenstoneClient client;
+  private URL              server;
+
+  // temporary variables which will eventually disapper - ignore!
+  private static GreenstoneClient lastClient;
+  private static NameServer       nameServer = null;
+  // end of temporary variables
+
+  /**
+   * Make a new interface with a client that will receive the responses
+   * to any messages sent.
+   *
+   * @param: the client to receive responses to messages sent through this
+   *         interface.
+   */
+  public GreenstoneInterface(GreenstoneClient client)
+  { this.client = client;
+    lastClient = this.client;
+    //    /*this.*/nameServer = null;
+  }
+
+  /**
+   *  Set up a registration for this machine with a GDS server.
+   *
+   *  The Greenstone machine should take care to store the URL of the
+   *  server for later use - don't continually reregister the same
+   *  server!
+   *
+   *  @param: <code>URL</code> the location of the GDS server to register with
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean registerWithServer(URL server)
+  { // connect to server
+    // 
+    if (/*this.*/nameServer == null) {
+      /*this.*/nameServer = new NameServer(server.toString());
+    }
+    this.server = server;
+    /*this.*/nameServer.registerServer(this.client.getURL());
+
+    return true;
+  }
+
+  /**
+   *  Method to unregister from the current GDS server for this Greenstone
+   *  machine.
+   *
+   *  @return: <code>boolean</code> whether the registration was successful or not.
+   */
+  public boolean unregisterFromServer()
+  { /*this.*/nameServer = null;
+    return true;
+  }
+
+  /**
+   *  Send a message into the GDS system through the current GDS server.
+   *
+   *  Success does not guarantee that you will receive a response.
+   *
+   *  @param: <code>String</code> the identifier for this type of message
+   *  @param: <code>Map</code> a <code>Vector</code> item is associated with
+   *          each key in this map.
+   *
+   *  @return: <code>boolean</code> whether the sending was successful or not.
+   */
+  public boolean sendMessage(String message, Map parameters)
+  { if (/*this.*/nameServer != null)
+      { /*this.*/nameServer.sendMessage(this.client.getURL(), message, parameters);
+    }
+    return false;
+  }
+
+  /**
+   *  Receive a response to a message sent from this Greenstone machine and
+   *  forward it to the current client - won't be needed in the long run, it
+   *  is here at present to provide a route for returning responses.
+   */
+  static public void receiveResponse(URL greenstoneURL, ClientResponse response)
+  { lastClient.receiveResponse(null, response);
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneServer.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+
+/**
+ *  Simple class to hold the location of a Greenstone server.
+ *
+ *  This class should be subclassed and methods overridden for 
+ *  Greenstone 2, 3 or any special installations such as the
+ *  Music DL.
+ */
+
+public class GreenstoneServer
+{ private URL server;
+  
+  public GreenstoneServer(URL url)
+  { this.server = url;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/GreenstoneTest.java	(revision 21112)
@@ -0,0 +1,70 @@
+package org.greenstone.p2p;
+
+import java.net.URL;
+import java.util.*;
+
+public class GreenstoneTest implements GreenstoneClient
+{ String name;
+  URL    url;
+  
+  public GreenstoneTest(String id, String url)
+  { try {
+      this.url = new URL(url);
+    }
+    catch (java.net.MalformedURLException malEx)
+      { this.url = null;
+      }
+    this.name = id;
+  }
+
+  public void receiveResponse(NameServerAction action, ClientResponse response)
+  {
+    System.out.println("\n----\nReceived a response:");
+    System.out.println(response.toString());
+  }
+
+  public void receiveMessage(String message, Map parameters)
+  {
+  }
+
+  public String getIdentifier()
+  { return this.name;
+  }
+
+  public URL getURL()
+  { return this.url;
+  }
+
+  /**
+   *  arg[0] => identifier for this machine
+   *  arg[1] => a url for this machine
+   *  arg[2] => a url for the GDS server
+   *
+   * e.g. run with:
+   * "java org.greenstone.p2p.GreenstoneTest localhost http://localhost http://www.nzdl.org"
+   */
+  public static void main(String args[])
+  { GreenstoneClient client = new GreenstoneTest(args[0], args[1]);
+    GreenstoneClient gsdl   = new GreenstoneTest("gsdl", "http://www.gsdl.org");
+    
+    URL server;
+    try
+      {
+	server = new URL(args[2]);
+      }
+    catch (java.net.MalformedURLException malEx)
+      { server = null;
+      }
+    
+    GreenstoneInterface gsdlIface = new GreenstoneInterface(gsdl);
+    gsdlIface.registerWithServer(server);
+
+    GreenstoneInterface iface = new GreenstoneInterface(client);
+    iface.registerWithServer(server);
+
+    Map parameters = new HashMap();
+    iface.sendMessage("ListServers", parameters);
+
+    iface.sendMessage("DocDelta", parameters);
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ListServersAction.java	(revision 21112)
@@ -0,0 +1,35 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+import java.net.URL;
+
+public class ListServersAction extends AbstractServerAction
+{ public String LISTSERVERS = "ListServers";
+
+  public ListServersAction(NameServer host)
+  { super(host);
+  }
+
+  public String getActionName()
+  { return LISTSERVERS;
+  }
+  
+  public void doAction(URL forwarder, Map parameters)
+  { this.propagateAction(forwarder, parameters);
+  
+    // Build a list of all servers
+    AbstractClientResponse response = new AbstractClientResponse();
+
+    Enumeration servers = this.host.servers();
+    while (servers.hasMoreElements())
+      { URL gsServer = (URL) servers.nextElement();
+	System.out.println("Adding listserver item " + gsServer);
+
+	response.addValue("Server", gsServer.toString());
+      }
+    response.setSuccessful(true);
+
+    // Send it back to the forwarder right now...
+    this.host.receiveResponse(forwarder, response);
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServer.java	(revision 21112)
@@ -0,0 +1,124 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServer
+{ ServerList list;
+  URL        parent;
+  Vector     childServers; // name servers
+  Vector     children;     // greenstone servers
+  String     identity;
+  Map        actionMap;
+ 
+  public NameServer(String identity)
+  { this.list     = new ServerList();
+    this.identity = identity;
+    this.parent   = null;
+    this.children = new Vector();
+    this.childServers = new Vector();
+
+    this.actionMap = new HashMap();
+
+    NameServerAction listServersAction = new ListServersAction(this);
+    this.addAction(listServersAction);
+    NameServerAction defaultAction = new DefaultServerAction(this);
+    this.addAction(defaultAction);
+    
+    // set up registration listener
+    // set up messaging broadcaster
+    // set up messaging listener
+  }
+
+  private void addAction(NameServerAction action)
+  { this.actionMap.put(action.getActionName(), action);
+  }
+
+  private void loadChildren(BufferedReader in)
+  { try 
+      { String child = in.readLine();
+      }
+    catch (IOException ex)
+      {
+      }
+  }
+
+  private void loadServers(BufferedReader in)
+  { try
+      { String child = in.readLine();
+      }
+    catch (IOException io)
+      {
+      }
+  }
+
+  public void sendMessage(URL origin, String actionName, Map parameters)
+  { NameServerAction action = (NameServerAction) this.actionMap.get(actionName);
+    if (action != null)
+      { action.doAction(origin, parameters);
+      }
+    else
+      { System.out.println("Sending action " + actionName + " through <default>");
+
+	// TODO: add action name to the parameter map so that it is not lost...
+	action = (NameServerAction) this.actionMap.get("Default");
+	if (action != null)
+	{ action.doAction(origin, parameters);
+	}
+      }
+    //NameServerAction action = NameServerActionFactory.createAction(action, parameters);
+    //action.doAction();
+  }
+
+  public void receiveResponse(URL responseTo, ClientResponse response)
+  { GreenstoneInterface.receiveResponse(responseTo, response);
+  }
+    
+  private void loadParent()
+  {
+  }
+
+  public String getIdentity()
+  { return this.identity;
+  }
+  
+  public void setParent(URL parent)
+  { this.parent = parent;
+  }
+  
+  public URL getParent()
+  { return this.parent;
+  }
+  
+  public void addSubServer(URL subserver)
+  { this.childServers.add(subserver);
+  }
+  
+  public void registerServer(URL child)
+  { System.out.println("Receiving registration from " + child.toString());
+    this.children.add(child);
+  }
+  
+  public Enumeration subServers()
+  { return this.childServers.elements();
+  }
+  
+  public Enumeration servers()
+  { return this.children.elements();
+  }
+
+  public static boolean pollServer(URL server)
+  { return false;
+  }
+
+  /**
+   *  
+   */
+  public static void main(String args[])
+  { NameServer server = new NameServer(args[0]);
+
+    // Load the action items
+    // Wait for signals
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerAction.java	(revision 21112)
@@ -0,0 +1,10 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+import java.net.URL;
+
+public interface NameServerAction
+{ public String getActionName();
+  public void   doAction(URL forwarder, Map parameters);
+  public void   propagateAction(URL forwarder, Map parameters);
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerActionFactory.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.p2p;
+
+import java.util.Map;
+
+public class NameServerActionFactory
+{ public NameServerAction createAction(String actionName, Map parameters)
+  { return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/NameServerRegistrationThread.java	(revision 21112)
@@ -0,0 +1,16 @@
+package org.greenstone.p2p;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class NameServerRegistrationThread extends Thread
+{ public void run() 
+  { /*try
+    {
+    }
+    catch (IOException ex)
+    {
+    }*/
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/src/org/greenstone/p2p/ServerList.java	(revision 21112)
@@ -0,0 +1,19 @@
+package org.greenstone.p2p;
+
+import java.util.*;
+
+public class ServerList
+{ Map map;
+
+  public ServerList()
+  { this.map = new HashMap();
+  }
+  
+  public void addServer(GreenstoneServer server)
+  { this.map.put(server, server);
+  }
+  
+  Iterator servers()
+  { return ((Collection)this.map.values()).iterator();
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/AlertingServiceTest.java	(revision 21112)
@@ -0,0 +1,163 @@
+/*
+ * Created on Nov 24, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.Velocity;
+import org.apache.velocity.exception.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class AlertingServiceTest extends TestCase {
+
+    private AlertingService as;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(AlertingServiceTest.class);
+    }
+
+    public final void testSubscribe() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.DOCUMENT_CONTENT_FIELD, "albatros");
+            
+            String template = as.createSubscription(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testListSubscriptions() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            
+            String template = as.listSubscriptions(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testReceiveEvent() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put(Constants.COLLECTION_ID_FIELD, "testing");
+            arguments.put(Constants.TYPE_FIELD, "new_document");
+            arguments.put(Constants.DOCUMENT_ID_FIELD, "20041124:1");
+            arguments.put(Constants.DOCUMENT_TITLE_FIELD, "keine albatrosse in sicht");
+            arguments.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            as.receiveEvent(arguments);
+            
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    public final void testShowFeed() {
+        try {
+            Velocity.init("/research/schweer/gsdl3/extensions/gsdl-as/build/velocity.properties");
+            VelocityContext context = new VelocityContext();
+            
+            Map arguments = new TreeMap();
+            arguments.put("subscriptionID", "0");
+            
+            String template = as.showFeed(arguments, context);
+            
+            StringWriter w = new StringWriter();
+            boolean b = Velocity.mergeTemplate(template, 
+                    org.apache.velocity.runtime.RuntimeSingleton.getString(Velocity.INPUT_ENCODING, Velocity.ENCODING_DEFAULT), 
+                    context, 
+                    w);
+            assertTrue(b);
+            System.out.println(w);
+        } catch (ResourceNotFoundException e) {
+            e.printStackTrace();
+            fail("no template");
+        } catch (ParseErrorException e) {
+            e.printStackTrace();
+            fail();
+        } catch (MethodInvocationException e) {
+            e.printStackTrace();
+            fail();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+    protected void setUp() throws Exception {
+        as = new AlertingService();
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/GreenstoneCommunicatorTest.java	(revision 21112)
@@ -0,0 +1,75 @@
+/*
+ * Created on Nov 17, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class GreenstoneCommunicatorTest extends TestCase {
+
+    private GreenstoneCommunicator gsComm;
+    
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(GreenstoneCommunicatorTest.class);
+    }
+    
+    protected void setUp() throws Exception {
+        URL host;
+        try {
+            host = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            gsComm = new GreenstoneCommunicator(host);
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testGetSiteList() {
+        String[] sites;
+        try {
+            sites = gsComm.getCollectionNames();
+            for (int i = 0; i < sites.length; i++) {
+                System.out.println(sites[i]);
+                
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testSupportsFullTextSearch() {
+        try {
+            assertTrue(gsComm.supportsTextSearch("gs2mgppdemo"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+    
+    public final void testFullTextSearch() {
+        try {
+            Set result = gsComm.fullTextSearch("gs2mgppdemo", "Burundi");
+            assertEquals(14, result.size());
+            assertTrue(result.contains("HASH4632a8a51d33c47a75c559.7.1"));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/ProfileStoreTest.java	(revision 21112)
@@ -0,0 +1,87 @@
+/*
+ * Created on Nov 18, 2004
+ * Copyright (C) Andrea Schweer, 2004
+ *
+ * This file is part of the Greenstone Alerting Service.
+ * Refer to the COPYING file in the base directory of this package
+ * for licensing information.
+ */
+package org.greenstone.gsdlas;
+
+import java.net.URL;
+import java.util.*;
+
+import junit.framework.TestCase;
+
+/**
+ * @author schweer
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class ProfileStoreTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(ProfileStoreTest.class);
+    }
+
+    public final void testAddSubscription() {
+        try {
+            addSubscriptions();
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    public final void testFilter() {
+        try {
+            addSubscriptions();
+            
+            URL url = new URL("http://localhost:8080/soap/servlet/rpcrouter");
+            GreenstoneCommunicator gsComm = new GreenstoneCommunicator(url);
+            
+            Map event = new TreeMap();
+            
+            event.put(Constants.DOCUMENT_ID_FIELD, "20041118:5");
+            event.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+            event.put(Constants.HOST_ID_FIELD, "http://localhost:8080/soap/servlet/rpcrouter");
+            
+            Set result = ProfileStore.getInstance().filter(event, gsComm);
+            
+            System.out.println(result.size());
+            
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    private void addSubscriptions() throws Exception {
+        Map valueMap = new TreeMap();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Tomatensalat");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "Water+Buffalo");
+        
+        addSubscription(valueMap);
+        
+        valueMap.clear();
+        
+        valueMap.put(Constants.COLLECTION_ID_FIELD, "gs3mgdemo");
+        valueMap.put(Constants.DOCUMENT_CONTENT_FIELD, "gender+equality");
+        
+        addSubscription(valueMap);
+    }
+    
+    private void addSubscription(Map valueMap) throws Exception {  
+        ProfileStore.getInstance().createSubscription(valueMap);    
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/test-src/org/greenstone/gsdlas/util/MailerTest.java	(revision 21112)
@@ -0,0 +1,35 @@
+/*
+ * Created on Jan 1, 2005
+ *
+ * TODO To change the template for this generated file go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+package org.greenstone.gsdlas.util;
+
+import javax.mail.MessagingException;
+
+import junit.framework.TestCase;
+
+/**
+ * @author andrea
+ *
+ * TODO To change the template for this generated type comment go to
+ * Window - Preferences - Java - Code Style - Code Templates
+ */
+public class MailerTest extends TestCase {
+
+    public static void main(String[] args) {
+        junit.textui.TestRunner.run(MailerTest.class);
+    }
+
+    public final void testSendMail() {
+        Mailer mailer = new Mailer("andrea@localhost", "localhost");
+        try {
+            mailer.sendMail("andrea@localhost", "test", "test message testing...\n");
+        } catch (MessagingException e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/WEB-INF/web.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/WEB-INF/web.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/WEB-INF/web.xml	(revision 21112)
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<web-app>
+
+
+    <!-- General description of your web application -->
+
+    <display-name>Alerting Service for Greenstone</display-name>
+    <description>
+	An alerting service for Greenstone, 
+	written by Andrea Schweer (schweer@cs.waikato.ac.nz).
+    </description>
+
+
+    <!-- Context initialization parameters that define shared
+         String constants used within your application, which
+         can be customized by the system administrator who is
+         installing your application.  The values actually
+         assigned to these parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletContext().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of context initialization
+         parameters, including zero.
+    -->
+
+    <context-param>
+      <param-name>webmaster</param-name>
+      <param-value>schweer@cs.waikato.ac.nz</param-value>
+      <description>
+        The EMAIL address of the administrator to whom questions
+        and comments about this application should be addressed.
+      </description>
+    </context-param>
+
+
+    <!-- Servlet definitions for the servlets that make up
+         your web application, including initialization
+         parameters.  With Tomcat, you can also send requests
+         to servlets not listed here with a request like this:
+
+           http://localhost:8080/{context-path}/servlet/{classname}
+
+         but this usage is not guaranteed to be portable.  It also
+         makes relative references to images and other resources
+         required by your servlet more complicated, so defining
+         all of your servlets (and defining a mapping to them with
+         a servlet-mapping element) is recommended.
+
+         Servlet initialization parameters can be retrieved in a
+         servlet or JSP page by calling:
+
+             String value =
+               getServletConfig().getInitParameter("name");
+
+         where "name" matches the <param-name> element of
+         one of these initialization parameters.
+
+         You can define any number of servlets, including zero.
+    -->
+
+    <servlet>
+      <servlet-name>service</servlet-name>
+      <description>
+        An alerting service for Greenstone 3
+      </description>
+      <servlet-class>org.greenstone.gsdlas.AlertingService</servlet-class>
+      <init-param>
+            <param-name>properties</param-name>
+            <param-value>/velocity.properties</param-value>
+      </init-param>
+    </servlet>
+<!--
+    <servlet>
+      <servlet-name>subscribe</servlet-name>
+      <description>
+        This servlet creates new subscriptions.
+      </description>
+      <servlet-class>org.greenstone.gsdlas.SubscriptionServlet</servlet-class>
+    </servlet> -->
+
+	<!-- Define mappings that are used by the servlet container to
+         translate a particular request URI (context-relative) to a
+         particular servlet.  The examples below correspond to the
+         servlet descriptions above.  Thus, a request URI like:
+
+           http://localhost:8080/{contextpath}/graph
+
+         will be mapped to the "graph" servlet, while a request like:
+
+           http://localhost:8080/{contextpath}/saveCustomer.do
+
+         will be mapped to the "controller" servlet.
+
+         You may define any number of servlet mappings, including zero.
+         It is also legal to define more than one mapping for the same
+         servlet, if you wish to.
+    -->
+
+<!--    <servlet-mapping>
+      <servlet-name>receive</servlet-name>
+      <url-pattern>/receive</url-pattern>
+    </servlet-mapping>
+
+    <servlet-mapping>
+      <servlet-name>subscribe</servlet-name>
+      <url-pattern>/subscribe</url-pattern>
+    </servlet-mapping> -->
+    <servlet-mapping>
+      <servlet-name>service</servlet-name>
+      <url-pattern>/service</url-pattern>
+    </servlet-mapping>
+</web-app>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/about.html
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/about.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/about.html	(revision 21112)
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+      <title>Greenstone Version 3</title>
+			<link rel="stylesheet" type="text/css" href="default.css"/>
+			</head>
+  <body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+    <div id="content"><h3>Alerting Service for Greenstone &ndash; About</h3>
+		<p></p>
+	
+	
+	(description of what an alerting service does)<br/>
+	<a href="index.html">back</a> to the alerting service.<br/>
+	</div>
+  </body></html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/default.css
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/default.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/default.css	(revision 21112)
@@ -0,0 +1,52 @@
+body {
+	color:             black;
+	background-color:  white;
+	/*font-family:       "Trebuchet MS", Arial, helvetica, sans-serif;*/
+	text-align:        center;
+	margin:            0px;
+	background-image:  url("images/chalk.gif");
+	background-repeat: repeat-y;
+}
+
+#content {
+	background-image: url("images/gsdlhead.gif");
+	background-repeat: no-repeat;
+	background-position: center top;
+	padding-top:  88px;
+	width:        537px;
+	position:     relative;
+	text-align:   left;
+	margin-left:  auto;
+	margin-right: auto;
+	margin-top:   10px;
+}
+
+
+.error {
+        color: red;
+        font-weight: bold;
+}
+
+.details {
+        font-size: small;
+}
+
+table, thead {
+        border: 1pt solid black;
+}
+
+thead {
+        background-color: gray;
+}
+
+a:link {
+	color: #006666;
+}
+
+a:active {
+	color: #cc9900;
+}
+
+a:visited {
+	color: #666633;
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/descriptor.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/descriptor.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/descriptor.xml	(revision 21112)
@@ -0,0 +1,8 @@
+<isd:service
+  xmlns:isd="http://xml.apache.org/xml-soap/deployment"
+  id="alerting" checkMustUnderstands="false">
+  <isd:provider type="java" scope="Session" methods="receiveEvent">
+    <isd:java class="org.greenstone.gsdlas.AlertingService" static="false"/>
+  </isd:provider>
+  <isd:faultListener>org.apache.soap.server.DOMFaultListener</isd:faultListener>
+</isd:service>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/feed.css
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/feed.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/feed.css	(revision 21112)
@@ -0,0 +1,45 @@
+@import url("default.css");
+
+#channelhead {
+	margin-bottom: 20px;
+}
+
+#channelhead a:link {
+	text-decoration: none;
+	color:           black;
+}
+
+#channelhead a:visited {
+	text-decoration: none;
+	color:            black;
+}
+
+
+#channelhead a:hover, a:focus, a:active {
+	color:            blue;
+	text-decoration:  underline;
+}
+
+#channelinfo {
+	border-top: 2px solid black;
+	font-size: small;
+	margin-top: 20px;
+}
+
+div.item {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+span.date, span.category {
+	font-style: italic;
+}
+
+div.description {
+	font-size: small;
+}
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/form.css
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/form.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/form.css	(revision 21112)
@@ -0,0 +1,72 @@
+@import url("default.css");
+
+form {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
+td, th {
+	padding: 0px 5px;
+}
+
+thead tr {
+	background-color: #CCCCCC;
+}
+
+tr.even {
+	background-color: #CCCCCC;
+}
+
+.explanation {
+	font-size: small;
+}
+
+#progress {
+	font-size: x-small;
+	border: 0px solid black;
+	width: 80%;
+	margin: 5px auto;
+}
+
+#progress td {
+	width: 25%;
+	vertical-align: top;
+	text-align: center;
+	color: #555;
+	border-top: 10px solid #CCC;
+	font-family: "Trebuchet MS", Arial, helvetica, sans-serif;
+}
+
+#progress td.completed {
+	border-top-color: #9CBE9C;
+	color: #000;
+}
+
+#progress td.current {
+	background-color: #F0E68C;
+	color: #000;
+}
+
+table.subscription {
+	font-size: small;
+}
+
+.rssbutton {
+	background: #ff6600;
+	color: #ffffff;
+	border-left:   1px solid #cc9966;
+	border-top:    1px solid #ccaa99;
+	border-right:  1px solid #993300;
+	border-bottom: 1px solid #331100;
+	padding: 0px 0.5em 0px 0.5em;
+	font-family: helvetica, arial, sans-serif;
+	font-weight: bold;
+	font-size: xx-small;
+	margin-top: 1em;
+}
+
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/index.html
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/index.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/index.html	(revision 21112)
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="default.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>What do you want to do today?</p>
+			
+			<ul>
+				<li>
+					<a href="/alerting/service?action=showLoginForm">Log in</a></li>
+				</li>
+				<li>
+					<a href="subscribe.html">Create</a> a subscription.</li>
+				<li>
+					<a href="/alerting/service?action=listSubscriptions">View and edit</a> your existing subscriptions.</li>
+				<li>
+					<a href="about.html">Read more</a> about the alerting service.</li>
+				<li>
+					<a href="/alerting/service?action=logout">Log out</a></li?
+			</ul>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/list.css
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/list.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/list.css	(revision 21112)
@@ -0,0 +1,12 @@
+@import url("default.css");
+
+div.subscription {
+	margin: 10px 0px;
+	border: 1px dashed gray;
+	border-left: 10px solid #9CBE9C;
+	padding: 2px;
+	padding-left: 8px;
+	width: auto;
+	background-color: #DCDCDC;
+}
+
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/rss2html.xsl
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/rss2html.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/rss2html.xsl	(revision 21112)
@@ -0,0 +1,65 @@
+<?xml version="1.0" ?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+
+	<xsl:output method="xml" 
+  	doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd" 
+	  doctype-public="-//W3C//DTD XHTML 1.0 Transitional//EN" indent="yes"/>
+	<!-- <xsl:output method="html"/> -->
+
+	<xsl:template match="/">
+		<html>
+			<xsl:attribute name="lang">
+				<xsl:value-of select="//channel/language"/>
+			</xsl:attribute>
+			<head>
+				<link rel="stylesheet" type="text/css" href="/alerting/feed.css"/>
+				<title>
+					<xsl:value-of select="//channel/title"/>
+				</title>
+			</head>
+			<body>
+				<div id="content">
+					<xsl:apply-templates/>
+				</div>
+   		</body>
+		</html>
+  </xsl:template>
+	
+	<xsl:template match="channel">
+		<div id="channelhead">
+      <h3>
+				<a>
+      		<xsl:attribute name="href">
+        		<xsl:value-of select="link"/>
+					</xsl:attribute>
+					<xsl:value-of select="title"/>
+				</a>
+			</h3>
+			<div class="description">
+        	<xsl:value-of select="description" disable-output-escaping="yes"/>
+			</div>
+		</div>
+    <xsl:apply-templates/>
+		<div id="channelinfo">
+			Last rebuilt <xsl:value-of select="lastBuildDate"/>.<br/>
+			This file has been automatically generated by <xsl:value-of select="generator"/>; please <a><xsl:attribute name="href">mailto:<xsl:value-of select="webMaster"/></xsl:attribute>contact the webmaster</a> if you have any questions.<br/>
+		</div>
+  </xsl:template>
+
+  <xsl:template match="item">
+		<div class="item">
+			<span class="date"><xsl:value-of select="pubDate"/></span>: 
+			<span class="category"><xsl:value-of select="category"/></span><br/>
+    	<a class="item">
+      	<xsl:attribute name="href">
+        	<xsl:value-of select="link"/>
+				</xsl:attribute>
+				<xsl:value-of select="title"/>
+      </a><br/>
+		</div>
+      <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="text()"/>
+	
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/subscribe.html
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/subscribe.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/subscribe.html	(revision 21112)
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+		<title>Greenstone Version 3</title>
+		<link rel="stylesheet" type="text/css" href="form.css"/>
+	</head>
+	<body>
+    
+    <!-- page banner (_style:pagebanner_) -->
+		<div id="content">
+			<h3>Alerting Service for Greenstone</h3>
+			<p>Create a subscription</p>
+			
+			You can subscribe to be notified of events about these elements:
+			<ul>
+				<li><a href="/alerting/service?action=createSubscription">documents</a> (new, modified or deleted documents)</li>
+				<li><a href="sub_docparts.html">parts of documents</a> (new, modified or deleted parts of documents)</li>
+				<li><a href="sub_metadata.html">document metadata</a> (new, modified or deleted metadata of documents)</li>
+				<li><a href="sub_collection.html">collections</a> (new, rebuilt or deleted collections)</li>
+				<li><a href="sub_classifiers.html">classifiers of collections</a> (new or deleted classifiers) <img alt="help" src="images/help.png" longdesc="this influences the ways you can browse a collection" title="this influences the ways you can browse a collection"/></li>
+				<li><a href="sub_indexes.html">indexes of a collection</a> (new or deleted indexes) <img alt="help" src="images/help.png" longdesc="this influences the ways you can search in a collection" title="this influences the ways you can search in a collection"/></li>
+			</ul>
+			
+			<a href="/alerting">back</a> to the alerting service
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/test.html
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/test.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/test.html	(revision 21112)
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+	<head>
+		<title>Alerting Service for Greenstone</title>
+		<link href="default.css" rel="stylesheet" type="text/css"/>
+		<style type="text/css">
+		<!--
+		table { border:1pt solid black; padding:2px; background:#F0F0F0; }
+		th { text-align:left; background:lightblue; }
+		th,td { border-width:0px; }
+		-->
+		</style>
+	</head>
+	<body>
+		<div id="content">
+		<!-- <h1>Test</h1>
+		<form action="/alerting/service?test" method="post">
+			<input type="text" name="text"/><br/>
+			<input type="submit" value="submit"/>
+		</form>
+		<hr/> -->
+		
+		<h3>Subscribe</h3>
+		<form action="/alerting/service" method="post" name="subscribe">
+			<input type="hidden" name="action" value="subscribe"/>
+			<table border="1">
+				<tr>
+					<th>Event Type</th>
+					<td>
+						<input type="radio" name="type" value="new_document" checked="checked"/> new document
+						<input type="radio" name="type" value="document_deleted"/> document deleted
+						<input type="radio" name="type" value="document_content_changed" /> content changed
+						<input type="radio" name="type" value="document_md_changed"/> metadata changed
+					</td>
+				</tr>
+				<tr>
+					<th>Document ID</th>
+					<td>
+						<input type="text" name="documentID"/> 
+					</td>
+				</tr>
+				<tr>
+					<th>Document Title</th>
+					<td>
+						<input type="text" name="document_title"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Document Content</th>
+					<td>
+						<input type="text" name="document_content"/>
+					</td>
+				</tr>
+				<tr>
+					<th>Collection ID</th>
+					<td>
+					 <input type="text" name="collectionID"/>
+					</td>
+				</tr>
+			</table>
+			<br/>
+			<input type="submit" value="submit"/>
+		</form>
+		</div>
+	</body>
+</html>
Index: /main/tags/3.01/gsdl3/extensions/gsdl-as/web/velocity.properties
===================================================================
--- /main/tags/3.01/gsdl3/extensions/gsdl-as/web/velocity.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/gsdl-as/web/velocity.properties	(revision 21112)
@@ -0,0 +1,155 @@
+#----------------------------------------------------------------------------
+# These are the default properties for the
+# Velocity Runtime. These values are used when
+# Runtime.init() is called, and when Runtime.init(properties)
+# fails to find the specificed properties file.
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+# R U N T I M E  L O G
+#----------------------------------------------------------------------------
+
+#----------------------------------------------------------------------------
+#  default LogSystem to use: default: AvalonLogSystem
+#----------------------------------------------------------------------------
+
+# runtime.log.logsystem.class = org.apache.velocity.runtime.log.AvalonLogSystem,org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+runtime.log.logsystem.class = org.apache.velocity.runtime.log.SimpleLog4JLogSystem
+
+#---------------------------------------------------------------------------
+# This is the location of the Velocity Runtime log.
+#----------------------------------------------------------------------------
+
+runtime.log = ../logs/velocity.log
+
+#----------------------------------------------------------------------------
+# This controls if Runtime.error(), info() and warn() messages include the
+# whole stack trace. The last property controls whether invalid references
+# are logged.
+#----------------------------------------------------------------------------
+
+runtime.log.error.stacktrace = true
+runtime.log.warn.stacktrace = false
+runtime.log.info.stacktrace = false
+runtime.log.invalid.reference = true
+
+#----------------------------------------------------------------------------
+# Configuration for the Log4JLogSystem.
+# You must define the runtime.log.logsystem.class property to be:
+#   org.apache.velocity.runtime.log.Log4JLogSystem
+#
+# You must also include Log4J's .jar files into your classpath. They are
+# included with the Velocity distribution in the build/lib directory.
+#
+# There are several different options that you can configure.
+# Uncomment the ones that you want and also define their settings.
+#----------------------------------------------------------------------------
+#runtime.log.logsystem.log4j.pattern=%d - %m%n
+#runtime.log.logsystem.log4j.file.size=100000
+#runtime.log.logsystem.log4j.file.backups=1
+#runtime.log.logsystem.log4j.syslogd.host=my.syslog.server.com
+#runtime.log.logsystem.log4j.syslogd.facility=LOG_DAEMON
+#runtime.log.logsystem.log4j.remote.host=my.remote.server.com
+#runtime.log.logsystem.log4j.remote.port=1099
+#runtime.log.logsystem.log4j.email.server=localhost
+#runtime.log.logsystem.log4j.email.from=root@localhost
+#runtime.log.logsystem.log4j.email.to=root@localhost
+#runtime.log.logsystem.log4j.email.subject=Velocity Error Report
+#runtime.log.logsystem.log4j.email.buffer.size=512
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  E N C O D I N G
+#----------------------------------------------------------------------------
+
+input.encoding=ISO-8859-1
+output.encoding=ISO-8859-1
+
+#----------------------------------------------------------------------------
+# F O R E A C H  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These properties control how the counter is accessed in the #foreach
+# directive. By default the reference $velocityCount will be available
+# in the body of the #foreach directive. The default starting value
+# for this reference is 1.
+#----------------------------------------------------------------------------
+
+directive.foreach.counter.name = velocityCount
+directive.foreach.counter.initial.value = 1
+
+#----------------------------------------------------------------------------
+# I N C L U D E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+# These are the properties that governed the way #include'd content
+# is governed.
+#----------------------------------------------------------------------------
+
+directive.include.output.errormsg.start = <!-- include error :
+directive.include.output.errormsg.end   =  see error log -->
+
+#----------------------------------------------------------------------------
+# P A R S E  P R O P E R T I E S
+#----------------------------------------------------------------------------
+
+directive.parse.max.depth = 10
+
+#----------------------------------------------------------------------------
+# T E M P L A T E  L O A D E R S
+#----------------------------------------------------------------------------
+#
+#
+#----------------------------------------------------------------------------
+
+# resource.loader = file
+# 
+# file.resource.loader.description = Velocity File Resource Loader
+# file.resource.loader.class = org.apache.velocity.runtime.resource.loader.FileResourceLoader
+# file.resource.loader.path = .
+# file.resource.loader.cache = false
+# file.resource.loader.modificationCheckInterval = 2
+resource.loader = class
+class.resource.loader.description = Velocity Class Resource Loader
+class.resource.loader.class = org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+
+#----------------------------------------------------------------------------
+# VELOCIMACRO PROPERTIES
+#----------------------------------------------------------------------------
+# global : name of default global library.  It is expected to be in the regular
+# template path.  You may remove it (either the file or this property) if
+# you wish with no harm.
+#----------------------------------------------------------------------------
+velocimacro.library = VM_global_library.vm
+
+velocimacro.permissions.allow.inline = true
+velocimacro.permissions.allow.inline.to.replace.global = false
+velocimacro.permissions.allow.inline.local.scope = false
+
+velocimacro.context.localscope = false
+
+#----------------------------------------------------------------------------
+# INTERPOLATION
+#----------------------------------------------------------------------------
+# turn off and on interpolation of references and directives in string
+# literals.  ON by default :)
+#----------------------------------------------------------------------------
+runtime.interpolate.string.literals = true
+
+
+#----------------------------------------------------------------------------
+# RESOURCE MANAGEMENT
+#----------------------------------------------------------------------------
+# Allows alternative ResourceManager and ResourceCache implementations
+# to be plugged in.
+#----------------------------------------------------------------------------
+resource.manager.class = org.apache.velocity.runtime.resource.ResourceManagerImpl
+resource.manager.cache.class = org.apache.velocity.runtime.resource.ResourceCacheImpl
+
+#----------------------------------------------------------------------------
+# PLUGGABLE INTROSPECTOR
+#----------------------------------------------------------------------------
+# Allows alternative introspection and all that can of worms brings
+#----------------------------------------------------------------------------
+
+runtime.introspector.uberspect = org.apache.velocity.util.introspection.UberspectImpl
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/README
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/README	(revision 21112)
@@ -0,0 +1,82 @@
+Vishnu package for ...
+
+If you are running this as part of Greenstone, just installing Greenstone will install Vishnu.
+
+If you want to run it standalone:
+
+You need to have Java 1.4 or higher and Ant installed. The environment variable JAVA_HOME needs to be set. 
+You need to have MG installed - this should be the version from Greenstone, and you need the mg.jar file in your CLASSPATH and the libmgjni.so in your LD_LIBRARY_PATH.
+You need the lucene 1.4 or higher jar file in your CLASSPATH.
+Other jar files needed in CLASSPATH: xercesImpl.jar, servlet-api.jar.
+These jar files all come as part of the Greenstone download. Some in gsdl3/lib/java, and servlet.jar in gsdl3/comms/jakarta/tomcat/lib (when unpacked).
+
+Edit the build.xml file and change the prefix property to be the current directory.
+
+Then run 'ant' (or "ant all').
+
+
+COLLECTION BUILDING:
+
+Make a directory called collections inside the main vishnu directory. Inside this, create a new directory for your collection. The name of this directory is the collection identifier. Inside the collection directory create a new direcotry called import. Inside here put all the documents that you want in your collection. Documents must be plain text, xml or html.
+
+Then you can build the collection:
+
+Command line: 
+java vishnu.builder.CollectionBuilder -collsHome (directory where collections live: path-to-vishnu/collections) -collection (directory name of collection) -name (descriptive name for collection) -description (collection description) -index ("mg" or "luc")
+
+If you want to use views, add the option "-use-views" to the above command. This will create a separate index for each sub-directory in the import folder, and you can search each sub-view independently.
+
+Using a GUI tool:
+java vishnu.builder.IndexerGUI
+
+Fill in the fields as appropriate, and click "Start indexing".
+
+Once the collection is built, you need to rename the building directory to index. Collections are built into the building directory, but served from the index directory. This allows you to serve a built collection while rebuilding.
+
+For collections in the Greenstone 3 context, you will need to create the files etc/collectionConfig.xml and index/buildConfig.xml.
+These should look like the following:
+
+etc/collectionConfig.xml:
+<collectionConfig>
+  <metadataList>
+    <metadata name="creator">creator's email address</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Collection name</displayItem>
+    <displayItem name='description' lang='en'>Collection description
+    </displayItem>
+  </displayItemList>
+</collectionConfig>
+
+index/buildConfig.xml:
+<buildConfig>
+  <metadataList/>
+  <serviceRackList>
+    <serviceRack name="Visualizer">
+      <engineType name="MG"/> <!-- or LUCENE, depending on how you built the collection -->
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
+
+Note that you cannot use views in the Greenstone context yet.
+
+RUNNING THE SERVLET:
+
+
+If you are running this standalone, you need to run a servlet container for VisServlet.
+We have provided an example configuration for use with Tomcat (version 4.1).
+
+The vishnu/web directory is setup to be the context directory for vishnu in Tomcat. 
+You need to edit Tomcat's conf/server.xml file, and add in the context for vishnu. Add the following element after the tomcat root context element:
+
+<Context path="/vishnu" docBase="full-path-to-vishnu/web" debug="1" reloadable="true"><Resources allowLinking='true'/></Context>
+(replacing "full path-to-vishnu" with the path to the vishnu directory.)
+Edit the vishnu/web/WEB-INF/web.xml file and put in the correct path to your collections directory.
+
+The vishnu.jar file needs to be in your CLASSPATH, and make sure that Tomcat is using your CLASSPATH.
+
+RUNNING THE APPLET:
+
+The file vishnu/web/index.html is an introductory page about vishnu, and contains the applet. Make sure that the library parameter has the correct address for your visualiser servlet. The default is http://localhost:8080/vishnu/server. This will be correct if you have installed Tomcat to use port 8080, and have set the vishnu servlet up as described above. But note, if you are using Mozilla, it may not like localhost here, so if you have problems, try putting the full name of the server running Tomcat in place of localhost.
+
+If you are running Tomcat as described above, you can view this index file at http://localhost:8080/vishnu/index.html
Index: /main/tags/3.01/gsdl3/extensions/vishnu/build.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/build.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/build.xml	(revision 21112)
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+
+<!-- ====================================================================== 
+     April 2005                                                    
+
+     Vishnu build script
+                   
+     kjdon                                                                
+     ====================================================================== -->
+<project name="vishnu" default="all" basedir=".">
+
+  <property environment="env"/>
+  <property name="build.dir" value="${basedir}/build"/>
+  <property name="src.dir" value="${basedir}/src"/>
+
+  <!-- if vishnu is installed as a package inside greenstone -->
+  <property name="prefix" value="../.."/>
+  <!-- use if vishnu is standalone -->
+  <property name="prefix" value="${basedir}"/>
+  <property name="catalina.home" value="${env.CATALINA_HOME}"/>
+  <property name="web.lib" value="${prefix}/web/WEB-INF/lib"/>
+  <property name="web.classes" value="${prefix}/web/WEB-INF/classes"/>
+
+
+<!--  ==================== Compilation Control Options ==================== -->
+
+<!--
+
+  These properties control option settings on the Javac compiler when it
+  is invoked using the <javac> task.
+
+  compile.debug        Should compilation include the debug option?
+
+  compile.deprecation  Should compilation include the deprecation option?
+
+  compile.optimize     Should compilation include the optimize option?
+
+-->
+
+  <property name="compile.debug"       value="true"/>
+  <property name="compile.deprecation" value="true"/>
+  <property name="compile.optimize"    value="true"/>
+
+  <!-- classpath : this is for vishnu with Greenstone. For standalone vishnu,
+  make sure all the specified jar files are on the environment CLASSPATH -->
+  <path id="compile.classpath">
+    <pathelement location="${prefix}/lib/jni/mg.jar"/>
+    <pathelement location="${web.lib}/lucene-1.4.1.jar"/>
+    <pathelement location="${catalina.home}/common/lib/servlet-api.jar"/>
+    <pathelement location="${catalina.home}/common/endorsed/xercesImpl.jar"/>
+    <pathelement path="${env.CLASSPATH}"/>
+    <!-- if not usign this with greenstone, don't need this -->
+    <pathelement location="${web.lib}/gsdl3.jar"/>
+  </path>
+
+  <path id="ld.library.path">
+    <pathelement location="${prefix}/lib/jni"/>
+  </path>
+  <target name="all" depends="clean,compile"/>
+
+  <target name="clean" depends=""
+    description="Remove the compiled code">
+    <delete dir="${build.dir}"/>
+  </target>
+  
+  <target name="compile" depends=""
+    description="Compile the code">
+    <mkdir dir="${build.dir}"/>
+    <javac srcdir="${src.dir}"
+      destdir="${build.dir}"
+      debug="${compile.debug}"
+      deprecation="${compile.deprecation}"
+      optimize="${compile.optimize}">
+      <classpath>
+	<path refid="compile.classpath"/>
+      </classpath>
+      <!-- uncomment this if not in a greenstone context -->
+      <!--<exclude name="org/**"/>-->
+    </javac>
+    <jar destfile="${build.dir}/vishnu.jar">
+      <fileset dir="${build.dir}">
+	<include name="vishnu/**"/>
+	<include name="org/**"/>
+      </fileset>
+      <manifest>
+	<attribute name="Built-By" value="${user.name}" />
+      </manifest>
+    </jar>      
+    <copy file="${build.dir}/vishnu.jar" todir="${web.lib}"/>
+  </target>
+</project>
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/org/greenstone/gsdl3/service/Visualizer.java	(revision 21112)
@@ -0,0 +1,295 @@
+/*
+ *    Visualiser.java
+ *    Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Text; 
+import java.util.Vector;
+import java.util.HashMap;
+import java.io.File;
+import java.io.*;
+import vishnu.server.*;
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.Base64;
+
+public class Visualizer
+    extends ServiceRack {
+    
+    // the services on offer
+    private static final String VIS_SERVICE = "VisApplet";
+
+    // other internal strings
+    private static final String ENGINE_TYPE_ELEM = "engineType";
+    private static final String LUCENE_ENGINE = "LUCENE";
+    private static final String MG_ENGINE = "MG";
+
+    private Element applet_description = null;
+    
+    private String engine_type = LUCENE_ENGINE; // lucene is default
+    private String collection_home = null;
+    private SearchInterface engine = null;
+    public Visualizer () {
+
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+	e.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+	short_service_info.appendChild(e);
+
+	applet_description = this.doc.createElement(GSXML.SERVICE_ELEM);
+        applet_description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+        applet_description.setAttribute(GSXML.NAME_ATT, VIS_SERVICE);
+
+
+	String app_info = "<"+GSXML.APPLET_ELEM+" CODEBASE='lib' CODE='vishnu.testvis.visual.VishnuSingle.class' ARCHIVE='vishnu.jar,gsdl3.jar,xercesImpl.jar,xml-apis.jar' WIDTH='1000' HEIGHT='800'>";
+	app_info += "<PARAM NAME='library' VALUE=''/>"; // filled in by receptionist
+	app_info += "<PARAM NAME='viscgi' VALUE='?";
+	app_info += GSParams.ACTION +"=a&amp;"+GSParams.REQUEST_TYPE +"=r&amp;"+GSParams.SERVICE+"="+VIS_SERVICE+"&amp;"+GSParams.OUTPUT+"=xml&amp;"+GSParams.RESPONSE_ONLY+"=1'/>";
+	app_info += "<PARAM NAME='collection'   VALUE='" + this.cluster_name + "'/>";
+	app_info += "<PARAM NAME='engine'   VALUE='GSDLEngine' />";
+	// add view info if appropriate
+	app_info += "The visualization applet.</"+GSXML.APPLET_ELEM+">";
+	   
+	Document dom = converter.getDOM(app_info);
+	if (dom == null) {
+	    System.err.println("Visualizer.configure Error: Couldn't parse applet info");
+	    return false;
+	}
+	Element app_elem = dom.getDocumentElement();
+	applet_description.appendChild(this.doc.importNode(app_elem, true));
+
+	// get engine type from config file
+	Element engine_elem = (Element)GSXML.getChildByTagName(info, ENGINE_TYPE_ELEM);
+	if (engine_elem != null) {
+	    engine_type = engine_elem.getAttribute(GSXML.NAME_ATT);
+	    if (engine_type.equals("")) {
+		engine_type = LUCENE_ENGINE;
+	    }
+	}
+	
+	collection_home = this.site_home + File.separator + "collect"+ File.separator + this.cluster_name + File.separator;
+	if (engine_type.equals(LUCENE_ENGINE)) {
+	    // make the full path the arg to MGSearcher??
+	    engine = new LUCSearcher(collection_home, null);
+	} else if (engine_type.equals(MG_ENGINE)) {
+	    engine = new MGSearcher(collection_home, this.cluster_name, null);
+	} else {
+	    System.err.println("Visualiser: invalid engine type: "+engine_type);
+	    return false;
+	}
+	return true;
+    }	
+    
+    
+    
+    protected Element getServiceDescription(String service, String lang, String subset) {
+	if (!service.equals(VIS_SERVICE)) {
+	    return null;
+	}
+	Element describe = (Element) applet_description.cloneNode(true);
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  
+							    getTextString(VIS_SERVICE+".name", lang)));
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  
+							    getTextString(VIS_SERVICE+".description", lang)));
+	return describe;
+    }
+    
+    protected Element processVisApplet(Element request)
+    {
+
+	Element param_elem = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_elem, false);
+
+	String type = (String)params.get("type");
+
+	// the result element
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, VIS_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// applet result info must be in appletInfo element
+	Element applet_data = this.doc.createElement(GSXML.APPLET_DATA_ELEM);
+	result.appendChild(applet_data);
+	Element vis_data = this.doc.createElement("visData");
+	applet_data.appendChild(vis_data);
+
+	String results = "";
+	if (type.equals("search")) {
+
+	    //Element cluster_results = this.doc.createElement("cluster");
+	    //vis_data.appendChild(cluster_results);
+	    //Element description_results = this.doc.createElement("descriptions");
+	    //vis_data.appendChild(description_results);
+
+	    String query = (String)params.get("q");
+	    System.err.println("the query was "+query);
+	    
+	    Vector doc_nums = new Vector();
+	    Vector descriptions = new Vector();
+	    
+	    engine.search(query);
+	    doc_nums = engine.getDocIdentifiers();
+	    descriptions = engine.getDocDescriptions();
+	    
+	    DataBlock db = generateDataBlock(collection_home, null, doc_nums, descriptions);
+	    System.err.println("got back data, now converting to string");
+	    try {
+		results = Base64.encodeObject(db);
+	    } catch (Exception e) {
+		System.err.println("trying to base64 encode the datablock, but exception happened: "+e);
+	    }
+	    System.err.println("after converting to string");
+// 	    // serialise the data block
+// 	    try {
+// 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+// 		ObjectOutputStream oos = new ObjectOutputStream(baos);
+// 		oos.writeObject(db);
+// 		oos.close();
+		
+// 		//results = baos.toString("UTF-8");
+// 		results = Base64.encode(baos.toByteArray());
+// 	    } catch (Exception e) {
+// 		System.err.println("Visualizer serialise data block error: "+e);
+// 	    }
+	    //System.err.println("num docs = "+doc_nums.size());
+	    //Cluster c = new Cluster();
+	    //results = c.getCluster(collection_home,doc_nums);
+	    //results.trim(); 
+	    //System.err.println("results = "+results);
+
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+
+// 	    for (int i=0; i<descriptions.size(); i++) {
+// 		String d = (String)descriptions.get(i);
+// 		Element de = GSXML.createTextElement(this.doc, "desc", d);
+// 		description_results.appendChild(de);
+// 	    }
+	    System.err.println("end of search");
+	} else if (type.equals("fetch")) {
+	    String docNum = (String)params.get("d");
+	
+	    Vector doc = engine.getDocContent(Integer.parseInt(docNum));
+  	    for (int i=0; i<doc.size(); i++) {
+		results += (String)doc.get(i) +"\n";
+	    }
+	    Text t = this.doc.createTextNode(results);
+	    vis_data.appendChild(t);
+	} else {
+	    System.err.println("invalid type sent to Visualiser process: "+type);
+	    return result;
+	}
+	System.err.println("returning result");
+	return result;
+    }
+
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.err.println("Num descriptions: " + descriptions.size());
+	    System.err.println("Num docs: " + docs.length);
+	    System.err.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    System.err.println("end of try in generatedatablock");
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	System.err.println("returning data");
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CKIndexer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CKIndexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CKIndexer.java	(revision 21112)
@@ -0,0 +1,395 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import java.util.Hashtable;
+import vishnu.util.*;
+
+public class CKIndexer extends Indexer
+{
+
+    static private String delimiter = "()<>.,?!'\" "; 
+
+
+    private static void usage()
+    {
+        System.out.println("java Indexer/CKIndexer \n -i InputDirectory [$COLL_HOME/import/]) \n -o OutputDirectory [$COLL_HOME/index/ck_index/java] \n -r root directory of collection \n -list (name of file listing the docs to be indexed)[docs.lst]\n -d string containing chars to be eliminated from words\n");
+	System.exit(0);
+    }
+    
+    
+    public static void main(String[] args)
+    {
+        String in = null;
+        String out = null;
+        //String COLL_HOME = null;
+	//String docList = null;
+	
+        if( args.length == 0 ){
+            usage();
+            System.exit(0);
+        }
+	
+        int a = 0;
+        while( a < args.length ){
+            if( args[a].equals("-i") )
+                in = args[++a];
+            else if( args[a].equals("-o") )
+                out = args[++a];
+            //else if( args[a].equals("-r") )
+            //    COLL_HOME = args[++a];
+	    //else if( args[a].equals("-list") )
+	    //	docList = args[++a];
+	    else if( args[a].equals("-d") )
+		delimiter = args[++a];
+            a++;
+        }
+	
+	CKIndexer cki = new CKIndexer();
+
+	if (out==null || in==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	//if( COLL_HOME == null ) usage();
+	//cki.setRootDirectory(COLL_HOME);
+	
+	//if( out == null ) out = COLL_HOME + "/index/ck_index/java/";
+	cki.setOutputDirectory(out);
+	
+	//if( in == null ) in = COLL_HOME + "/import/";
+	cki.setInputDirectory(in);
+	
+	//if( docList != null ) cki.setDocList(docList);
+	
+	cki.startIndexing();
+    }
+    
+    
+    public CKIndexer(){
+	super("CKIndexer");
+    }
+    
+    
+//     public void setDocList(String doclist)
+//     {
+// 	docList = doclist;
+//     }
+
+
+    public void startIndexing(){
+	
+	initialize();
+	/**** read in archive ****/
+	
+	Vector docs = getDocList(inputDir); //readDocList(docList);
+	
+	System.out.println(docs.size() + " documents read");
+	
+	/****  hash of ALL words of corpus + their document frequency ****/
+	
+	Hashtable corpus_hash = new Hashtable();
+	
+	
+	/**** readDocuments and populate hash ****/
+
+	System.out.println("First run: getting document frequencies of all words");
+	 
+	getDocFreqs(docs, corpus_hash);
+	
+	
+	/**** prune words to get candidate keywords ****/
+
+	int min = 3;
+	int max = (int)(0.3*docs.size());
+	int minLength = 4;
+	
+	int num_ck = pruneKeywords(corpus_hash, minLength, min, max);
+	
+	
+	/**** sort hashtable by value ****/
+	
+	Enumeration enumer = corpus_hash.keys();
+	String[] words_ck = new String[num_ck];
+	int[] freqs_ck = new int[num_ck];
+	
+	int count = 0;
+	while( enumer.hasMoreElements() ){
+	    words_ck[count] = (String)enumer.nextElement();
+	    freqs_ck[count] = -((Integer)corpus_hash.get(words_ck[count])).intValue();
+	    count++;
+	}
+	
+	Misc.quickSort(freqs_ck,words_ck);
+	
+	
+	/**** normalize document frequencies ****/
+	
+	float[] freqs_f = new float[num_ck];
+	
+	for( int i = 0; i < num_ck; i++ ){
+	    freqs_f[i] = (float)(-1*freqs_ck[i])/docs.size();
+	}
+	
+	
+	/**** save document frequencies and candidate keywords ****/
+	
+	printToFile(freqs_f,outputDir + "doc_freq");
+	printToFile(words_ck,outputDir + "ck_words");	
+	
+	
+	/**** create (ck_word, index hash) ****/
+	
+	Hashtable ck_hash = new Hashtable();
+	
+	for( int i = 0; i < words_ck.length; i++ ){
+	    ck_hash.put(words_ck[i],new Integer(i));
+	}
+	
+	
+	/**** read documents and store for each document the ck words it contains and their frequencies ****/
+
+	System.out.println("Second run: getting term frequencies of candidate keywords");
+	
+	Vector termFreq = getTermFreqs(docs,ck_hash);
+	
+	
+	/**** save term frequency vector ****/ 
+	
+	printToFile(termFreq,outputDir + "term_freq");
+    }
+    
+    
+    private int pruneKeywords(Hashtable corpus_hash, int minLength, int min, int max){
+	Set ck_set = corpus_hash.keySet();
+	Enumeration enumer = corpus_hash.keys();
+	
+	int count = 0;
+	int pcount = 0;
+      	
+	while( enumer.hasMoreElements() ) {
+	    
+	    String word = (String)enumer.nextElement();
+	    int freq = ((Integer)corpus_hash.get(word)).intValue();
+	    
+	    
+	    /**** if word is a number remove ****/
+	    
+	    try{
+		int i = Integer.parseInt(word);
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }catch(NumberFormatException ex){}
+
+
+	    /**** if word too rare or too freq remove ****/
+	    
+	    if( word.length() < minLength || freq < min || freq > max ){
+		corpus_hash.remove(word);
+		ck_set.remove(word);
+		continue;
+	    }
+	}
+	return corpus_hash.size();
+    }
+    
+    
+    private Vector getTermFreqs(Vector docs, Hashtable ck_hash)
+    {
+	Vector termFreqVector = new Vector();
+	
+	for( int i = 0; i < docs.size(); i++ ){
+
+	    Hashtable doc_ck_hash = new Hashtable();
+	    String doc = (String)docs.elementAt(i);
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			Integer index = null;
+			if( (index = (Integer)ck_hash.get(word)) != null ){
+			    Integer count;
+			    
+			    if( (count = (Integer)doc_ck_hash.get(index)) != null ){
+				int newCount = count.intValue() + 1;
+				doc_ck_hash.put(index,new Integer(newCount));
+			    }else{
+				doc_ck_hash.put(index,new Integer(1));				
+			    }
+			}
+		    }
+		}
+		dis.close();
+	    }catch(Exception ex){ex.printStackTrace();}
+	
+	
+	    
+	    termFreqVector.addElement(doc_ck_hash);
+	}
+	
+	return termFreqVector;
+    }
+    
+
+    private void getDocFreqs(Vector docs, Hashtable corpus_hash)
+    {
+	for( int i = 0; i < docs.size(); i++ ){
+	    
+	    String doc = (String)docs.elementAt(i);
+	    
+	    Hashtable doc_hash = new Hashtable();
+	    
+	    try{
+		BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(doc)));
+		String line = "";
+		StringTokenizer strtok;
+		
+		while( (line=dis.readLine())!=null ){
+		    
+		    
+		    /**** read word by word ****/
+		    
+		    strtok = new StringTokenizer(line,delimiter);
+		    
+		    while( strtok.hasMoreTokens() ){
+			String word = strtok.nextToken();
+			
+			
+			if( (Object)doc_hash.get(word) == null ){
+			    
+			    
+			    /**** make a note that there is at least one occurrence ****/
+			    
+			    doc_hash.put(word,word);
+			    
+			    
+			    /**** add it to corpus_hash if necessary or increment the counter ****/
+			    
+			    Integer count;
+			    if( (count = (Integer)corpus_hash.get(word)) == null ){
+				
+				
+				/**** add it to corpus_hash ****/
+				
+				corpus_hash.put(word,new Integer(1));
+			    }
+			    else{
+				
+				/**** increment counter in doc_freq ****/
+				
+				Integer newCount = new Integer(count.intValue()+1);
+				corpus_hash.put(word,newCount);
+			    }
+			}
+		    }
+		}
+		dis.close();
+		
+	    } catch(IOException ex){ex.printStackTrace();}
+	}
+    }
+    
+    private Vector getDocList(String input_dir_name) {
+
+	File input_dir = new File(input_dir_name);
+	if (!input_dir.exists()) {
+	    System.err.println("CKIndexer Error: input directory "+input_dir_name+ " does not exist!");
+	    return null;
+	}
+
+	Vector file_vector = new Vector();
+	processDirectory(input_dir, file_vector);
+	return file_vector;
+    }
+    private void processDirectory(File dir, Vector file_list) {
+	File [] files = dir.listFiles();
+	for (int i=0; i<files.length; i++) {
+	    if (files[i].isDirectory()) {
+		processDirectory(files[i], file_list);
+	    } else {
+		file_list.add(files[i].getPath());
+	    }
+	}
+
+    }
+//     private Vector readDocList(String list)
+//     {
+// 	list = inputDir + "/" + "docs.lst";
+	
+// 	File docList = new File(list);
+	
+// 	if (!docList.exists()){
+// 	    System.out.println("No documents found");
+// 	    System.exit(0);
+// 	}
+	
+// 	Vector docs = new Vector();
+	
+// 	try{
+//             BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+// 	    String line = "";
+//             while( (line=dis.readLine())!=null ){
+// 		docs.addElement(new String(inputDir+"/" + line));
+// 	    }
+//             dis.close();
+// 	} catch(IOException ex){ex.printStackTrace();}
+	
+// 	return docs;
+//     }
+    
+    
+    private void printToFile(Object obj, String name){
+	
+	FileOutputStream fos = null;
+	ObjectOutputStream out = null;
+	
+	try{
+	    fos = new FileOutputStream(name);
+	    out = new ObjectOutputStream(fos);
+	    out.writeObject(obj);
+	    out.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+    }
+}
+
+
+	/*Hgram hg = new Hgram();
+	hg.setBinNumber(50);
+	hg.setLabel("Document frequency");
+	hg.setData(freqs_f,Hgram.RAW_DATA);
+	hg.update();*/
+
+ /*    private Map.Entry[] sort(Hashtable h)
+    {
+	Set set = h.entrySet();
+	Map.Entry [] entries = 
+	    (Map.Entry[])set.toArray(
+				     new Map.Entry[set.size()]);
+	Arrays.sort(entries, new Comparator() {
+		public int compare(Object o1, Object o2) {
+		    int i1 = ((Integer)((Map.Entry)o1).getValue()).intValue();
+		    int i2 = ((Integer)((Map.Entry)o2).getValue()).intValue();
+		    if( i1 < i2 ) return 0;
+		    else return 1;
+		}
+		public boolean equals(Object that) {
+		    return this==that;
+		}
+	    });
+	return entries;
+	}*/
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CollectionBuilder.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/CollectionBuilder.java	(revision 21112)
@@ -0,0 +1,252 @@
+package vishnu.builder;
+
+import java.util.*;
+import java.io.*;
+
+public class CollectionBuilder
+{
+    //String site;
+    String collID = null; // directory name
+    String collHome = null; // full path to collections directory
+    String collName = null; // descriptive name
+    String collDesc = null; // descriptive description :-)
+    String indexer = null; // mg or lucene
+    boolean use_views = false;
+    long numDocs=0;
+    Vector views = null;
+    private static void usage()
+    {
+	System.out.println("java vishnu.builder.CollectionBuilder\n -collsHome (directory where collections live) \n -collection (directory name of collection)\n -name (descriptive name for collection) \n -description (collection description)\n -index (\"mg\" or \"luc\")\n [-use_views ]");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	String collID = null;
+	String collsHome = null;
+	String collName = "";
+	String collDesc = "";
+	String indexer = null;
+	boolean use_views = false;
+	int a = 0;
+	while( a < args.length ){
+	    if (args[a].equals("-collsHome"))
+		collsHome = args[++a];
+	    else if (args[a].equals("-collection"))
+		collID = args[++a];
+	    else if( args[a].equals("-name") )
+		collName = args[++a];
+	    else if( args[a].equals("-description") )
+		collDesc = args[++a];
+	    else if( args[a].equals("-index") )
+		indexer = args[++a];
+	    else if (args[a].equals("-use_views")) 
+		use_views = true;
+	    a++;
+	}
+	
+	if (collsHome==null || collID==null || indexer==null) {
+	    usage();
+	    System.exit(0);
+	}
+	
+	if (collName.equals("")) {
+	    collName = collID;
+	}
+	if( !indexer.equals("mg") && !indexer.equals("luc") )
+	    indexer = "luc";
+	
+	if (!collsHome.endsWith(File.separator)) {
+	    collsHome += File.separator;
+	}
+	String coll_home = collsHome + collID + File.separator;
+	
+	CollectionBuilder cb = new CollectionBuilder(coll_home, collID, collName, collDesc, indexer, use_views);
+    }
+    
+    
+    
+    public CollectionBuilder(String coll_home, String coll_id, String coll_name,String coll_desc,  String ind, boolean use_views)
+    {
+	this.collHome = coll_home;
+	this.collID = coll_id;
+	this.collName = coll_name;
+	this.collDesc = coll_desc;
+	this.indexer = ind;
+	this.use_views = use_views;
+	
+	if (use_views) {
+	    views = new Vector();
+	}
+	if (!setupCollection()) {
+	    System.err.println("Couldn't setup the collection. Quitting build");
+	    return;
+	}
+	
+	/**** get text annotation ****/
+
+	FTIndexing();
+
+
+	/**** extract keywords ****/
+
+	CKIndexing();
+
+	/**** write out the config file ****/
+	printConfig();
+    }
+
+
+
+    private boolean setupCollection()
+    {
+	// check the import directory
+	File import_dir = new File(collHome + "import");
+	if (!import_dir.isDirectory()) {
+	    System.err.println("the import directory "+import_dir.getPath()+" doesn't exist. Quitting build");
+	    return false;
+	}
+	
+	
+	// create the building directory
+	try {
+	    File index_dir = new File(collHome + "building");
+	    if (!index_dir.exists()) {
+		index_dir.mkdir();
+	    } else if (!index_dir.isDirectory()) {
+		// index is present but is not a directory
+		System.err.println("Trying to create building dir but a file already exists at "+index_dir.getPath());
+		return false;
+	    }
+	} catch (Exception e) {
+	    System.err.println("CollectionBuilder.setupCollection Error: "+e);
+	    return false;
+	}
+
+	// create the view list
+	if (use_views) {
+	    // create a new view for each sub directory in import
+	    File [] sub_dirs = import_dir.listFiles();
+	    for (int i=0; i<sub_dirs.length; i++) {
+		if (sub_dirs[i].isDirectory()) {
+		    views.add(sub_dirs[i].getName());
+		}
+	    }
+	    if (views.size()==0) {
+		// we haven't found any subdirectories
+		System.err.println("no sub directories were found in the import dorectory ("+import_dir.getPath()+"), not using views");
+		use_views = false;
+
+	    }
+	}
+	return true;
+    }
+
+    private void printConfig() {
+
+	String index_type="";
+	if (indexer.equals("luc")) {
+	    index_type = "LUCENE";
+	} else if (indexer.equals("mg")) {
+	    index_type = "MG";
+	}
+	
+	try{
+	    File target = new File(collHome + "config.xml");
+	    PrintWriter out = new PrintWriter(new FileWriter(target),true);
+	
+	    StringBuffer buffer = new StringBuffer();
+	    buffer.append("<collection name=\"");
+	    buffer.append(collName);
+	    buffer.append("\" description=\"");
+	    buffer.append(collDesc);
+	    buffer.append("\" engine=\"");
+	    buffer.append(index_type);
+	    buffer.append("\" size=\"");
+	    buffer.append(String.valueOf(numDocs));
+	    buffer.append("\">\n");
+	    
+	    // add in views
+	    if (use_views) {
+		buffer.append("<viewList>\n");
+		for (int i=0; i<views.size(); i++) {
+		    buffer.append("<view name=\"");
+		    buffer.append(views.get(i));
+		    buffer.append("\"/>");
+		}
+		buffer.append("\n</viewList>\n");
+	    } 
+	    buffer.append("</collection>\n");
+		
+	    out.print(buffer.toString());
+	    out.close();
+
+	} catch (IOException e){ System.out.println("Error writing config file: " + e.toString());
+	}
+    }
+    
+
+    private void CKIndexing()
+    {
+	CKIndexer cki = new CKIndexer();
+	cki.setCollectionName(collID);
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator+"ck_index"+File.separator;
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("calculating candidate keywords for view "+view);
+		cki.setOutputDirectory(out+view+File.separator);
+		cki.setInputDirectory(in+view+File.separator);
+		cki.startIndexing();
+	    }
+	} else {
+	    System.out.println("calculating candidate keywords");
+	    // just do the import dir as a whole
+	    cki.setOutputDirectory(out);
+	    cki.setInputDirectory(in);
+	    cki.startIndexing();
+	}
+    }
+    
+
+
+    private void FTIndexing()
+    {
+	String in = collHome + "import"+File.separator;
+	String out = collHome + "building"+File.separator + indexer + "_index"+File.separator;
+	Indexer the_indexer;
+	if (indexer.equals("luc") ) {
+	    the_indexer = new LuceneWrapper();
+	} else {
+	    the_indexer = new MGWrapper();
+	}
+	the_indexer.setCollectionName(collID);
+
+	if (use_views) {
+	    for (int i=0; i<views.size(); i++) {
+		String view = (String)views.get(i);
+		System.out.println("indexing the text for view "+view);
+		the_indexer.setOutputDirectory(out+view+File.separator);
+		the_indexer.setInputDirectory(in+view+File.separator);
+		the_indexer.startIndexing();
+	    }
+
+	} else {
+	    System.out.println("indexing the text");
+	    the_indexer.setOutputDirectory(out);
+ 	    the_indexer.setInputDirectory(in);
+ 	    the_indexer.startIndexing();
+	}
+
+    }
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/FileDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/FileDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/FileDocument.java	(revision 21112)
@@ -0,0 +1,57 @@
+package vishnu.builder;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.FileInputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.DateField;
+
+/** A utility for making Lucene Documents from a File. */
+
+public class FileDocument {
+  /** Makes a document for a File.
+    <p>
+    The document has three fields:
+    <ul>
+    <li><code>path</code>--containing the pathname of the file, as a stored,
+    tokenized field;
+    <li><code>modified</code>--containing the last modified date of the file as
+    a keyword field as encoded by <a
+    href="lucene.document.DateField.html">DateField</a>; and
+    <li><code>contents</code>--containing the full contents of the file, as a
+    Reader field;
+    */
+  public static Document Document(File f, String relative_path)
+       throws java.io.FileNotFoundException {
+	 
+    // make a new, empty document
+    Document doc = new Document();
+
+    // Add the path of the file as a field named "path".  Use a Text field, so
+    // that the index stores the path, and so that the path is searchable
+    //doc.add(Field.Text("path", f.getPath()));
+    doc.add(Field.Text("path", relative_path));
+    // Add the last modified date of the file a field named "modified".  Use a
+    // Keyword field, so that it's searchable, but so that no attempt is made
+    // to tokenize the field into words.
+    doc.add(Field.Keyword("modified",
+			  DateField.timeToString(f.lastModified())));
+
+    // Add the contents of the file a field named "contents".  Use a Text
+    // field, specifying a Reader, so that the text of the file is tokenized.
+    // ?? why doesn't FileReader work here ??
+    FileInputStream is = new FileInputStream(f);
+    Reader reader = new BufferedReader(new InputStreamReader(is));
+    doc.add(Field.Text("contents", reader));
+
+    // return the document
+    return doc;
+  }
+
+  private FileDocument() {}
+}
+    
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/Indexer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/Indexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/Indexer.java	(revision 21112)
@@ -0,0 +1,89 @@
+package vishnu.builder;
+
+import java.io.File;
+abstract public class Indexer
+{ 
+    
+    // views stuff
+//     public static final int DEFAULT = 0;
+//     public static final int DIP = 1;
+//     public static final int REG = 2;
+//     public static final int LEM = 3;
+//     public static final int SYN = 4;
+
+//     public static final String[] types = new String[]{"full","dip","reg","lem","syn"};
+    
+//     boolean use_views = false;
+//     int type = -1;
+    String name = null;
+    //String limiter = null;
+    String outputDir = null;
+    String inputDir = null;
+    String collName = null;
+
+    public Indexer(String n)
+    {
+	name = n;		
+    }
+    
+    
+//     public void setUseViews(boolean use_v) {
+// 	use_views = use_v;
+//     }
+    
+//     public void setIndexingType(int t)
+//     {
+// 	type = t;
+//     }    
+    public void setOutputDirectory(String s)
+    {
+	outputDir = s;
+    }
+    
+//     public void setLimiter(String s)
+//     {
+//         limiter = s;
+//     }
+    
+    public void setInputDirectory(String s)
+    {
+        inputDir = s;
+    }
+    
+     
+    public void setCollectionName(String c)
+    {
+	collName = c;
+    }
+
+    // this should be called at the start of startIndexing()
+    protected void initialize() {
+
+	if( inputDir == null || outputDir == null ){
+	    System.err.println("Input dir or output dir is null");
+	    System.exit(0);
+	}
+	
+	/* check if index dir exists */
+	File in = new File(inputDir);
+	if (!in.isDirectory()) {
+	    System.err.println("input directory ("+inputDir+") not a directory, exiting...");
+	    System.exit(0);
+	}
+
+	/**** check if output directory exists, if not create it ****/	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+	    try {
+		f.mkdirs();
+	    } catch (Exception e) {
+		System.err.println("Couldn't create output directory "+outputDir);
+		System.err.println(e);
+		System.exit(0);
+	    }
+	}
+	
+    }
+    
+    abstract public void startIndexing();
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/IndexerGUI.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/IndexerGUI.java	(revision 21112)
@@ -0,0 +1,126 @@
+package vishnu.builder;
+
+import java.awt.event.*;
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.io.File;
+import vishnu.util.SpringUtilities;
+
+public class IndexerGUI {
+    public final static boolean RIGHT_TO_LEFT = false;
+    public Toolkit toolkit = Toolkit.getDefaultToolkit();
+    private static JTextField[] textFields; 
+    private static JTextArea ja;
+
+    public static void addComponentsToPane(Container pane)
+    {
+	String [] labels = {"Collections directory: ", "Collection id (directory name): ", "Collection name: ","Collection description: ", "Indexer (\"mg\" or \"luc\"): ", "Use views (\"yes\" or \"no\"): "};
+        int numPairs = labels.length;
+        textFields = new JTextField[numPairs];
+	
+        pane.setLayout(new BoxLayout(pane,BoxLayout.Y_AXIS));
+	
+        JPanel p = new JPanel(new SpringLayout());
+	
+        for (int i = 0; i < numPairs; i++) {
+	    JLabel l = new JLabel(labels[i], JLabel.TRAILING);
+	    p.add(l);
+	    textFields[i] = new JTextField(30);
+	    l.setLabelFor(textFields[i]);
+	    p.add(textFields[i]);
+	}
+	
+	SpringUtilities.makeCompactGrid(p,numPairs, 2, 6, 6, //initX, initY
+                                6, 6);       //xPad, yPad
+	
+	pane.add(p);
+	
+	JPanel butPan = new JPanel();
+	butPan.setLayout(new BoxLayout(butPan,BoxLayout.X_AXIS));
+
+	Dimension minSize = new Dimension(10, 50);
+	Dimension prefSize = new Dimension(10, 50);
+	Dimension maxSize = new Dimension(Short.MAX_VALUE, 50);
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+	JButton button = new JButton(" Start indexing ");
+        button.setBorder(BorderFactory.createBevelBorder(0));
+	button.addActionListener(new ActionListener(){
+                public void actionPerformed(ActionEvent e){
+			build();
+                }
+            });
+        butPan.add(button);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+	
+        butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	pane.add(butPan);
+
+	butPan.add(new Box.Filler(minSize, prefSize, maxSize));
+
+	ja = new JTextArea(5,5);
+        JScrollPane scrollingArea = new JScrollPane(ja);
+	scrollingArea.setBorder(BorderFactory.createBevelBorder(1));
+	pane.add(scrollingArea);
+     }
+
+    private static void build()
+    {
+	// get data
+	String colls_home = textFields[0].getText();
+	String coll_id = textFields[1].getText();
+	String coll_name = textFields[2].getText();
+	String coll_desc = textFields[3].getText();
+	String indexer = textFields[4].getText();
+	String use_views_str = textFields[5].getText();
+	String coll_home = colls_home;
+	if (!coll_home.endsWith(File.separator)) {
+	    coll_home += File.separator;
+	}
+	coll_home += coll_id + File.separator;
+
+	if (!indexer.equals("mg") || !indexer.equals("luc")) {
+	    indexer = "mg";
+	}
+	boolean use_views = false;
+	if (use_views_str.equals("yes")) {
+	    use_views = true;
+	}
+	String output = "Launching Collection Builder with \n collections home = "+colls_home + "\n collection id = "+coll_name+"\n indexer = "+indexer;
+	if (use_views) {
+	    output += "\n using views";
+	} else {
+	    output += "\n not using views";
+	}
+	ja.setText(output);
+
+	CollectionBuilder builder = new CollectionBuilder(coll_home, coll_id, coll_name, coll_desc, indexer, use_views); 
+	output += "\n Indexing Finished";
+	ja.setText(output);
+    }
+
+    private static void createAndShowGUI() {
+        JFrame.setDefaultLookAndFeelDecorated(true);
+        JFrame frame = new JFrame("Indexing Tool");
+        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+        addComponentsToPane(frame.getContentPane());
+
+        frame.pack();
+
+	Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
+        int w = frame.getSize().width;
+        int h = frame.getSize().height;
+        int x = (dim.width-w)/2;
+        int y = (dim.height-h)/2;
+        frame.setLocation(x,y);
+
+	frame.setVisible(true);
+    }
+
+    public static void main(String[] args) {
+	createAndShowGUI();
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/LuceneWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/LuceneWrapper.java	(revision 21112)
@@ -0,0 +1,162 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexWriter;
+
+
+public class LuceneWrapper extends Indexer
+{
+    
+    private static void usage()
+    {
+	System.out.println("java LucenceWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+
+	LuceneWrapper luc = new LuceneWrapper();
+	
+	out = out + "luc_index";
+	luc.setCollectionName(coll);
+        luc.setOutputDirectory(out);
+	luc.setInputDirectory(in);
+	luc.startIndexing();
+    }
+    
+    
+    public LuceneWrapper()
+    {
+	super("Lucene Indexer");
+    }
+    
+    public void startIndexing() {
+	initialize();
+	try {
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    // this is a bit of a hack putting import here
+	    indexFiles(writer, new File(inputDir), "import/");
+	    writer.optimize();
+	    writer.close();
+	} catch (Exception e) {
+	    System.err.println("Error creating the Lucene index: "+e);
+	    System.exit(0);
+	}
+
+    }
+
+    private void indexFiles(IndexWriter writer, File input_dir, String current_path) {
+	File [] doc_list = input_dir.listFiles();
+	for (int i=0; i<doc_list.length; i++) {
+	    File f = doc_list[i];
+	    if (f.isDirectory()) {
+		indexFiles(writer, f, current_path+f.getName()+File.separator);
+	    } else {
+		try {
+		    writer.addDocument(FileDocument.Document(f, current_path+f.getName()));	
+		} catch (Exception e) {
+		    System.err.println("Error indexing document "+current_path+f.getName()+": "+e);
+		}
+	    }
+	}
+
+    }
+
+	
+    public void startIndexingOld()
+    {
+	
+	if( inputDir == null || outputDir == null ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	
+	/**** check if output directory exists, if not create it ****/
+	
+	
+        File f = new File(outputDir);
+        if( !f.exists() ){
+            System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/");
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }
+        }	
+	
+	File docList = new File(inputDir + "/docs.lst");
+	
+	if (!docList.exists()){
+	    System.out.println("No documents found");
+	    System.exit(0);
+	}
+	
+	Vector docs = new Vector();
+	
+	try{
+            BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(docList)));
+	    String line = "";
+            while( (line=dis.readLine())!=null ){
+		docs.addElement(new String(inputDir+"/" + line));
+	    }
+            dis.close();
+	} catch(IOException ex){ex.printStackTrace();}
+	
+	try{
+	    
+	    IndexWriter writer = new IndexWriter(outputDir, new StandardAnalyzer(), true);
+	    
+	    for( int i = 0; i < docs.size(); i++ ){
+		File file = new File((String)docs.elementAt(i));		
+		//writer.addDocument(FileDocument.Document(file));
+	    }
+	    writer.optimize();
+	    writer.close();
+        }catch(Exception ex){ex.printStackTrace();}
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/MGWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/MGWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/builder/MGWrapper.java	(revision 21112)
@@ -0,0 +1,123 @@
+package vishnu.builder;
+
+import java.io.*;
+import java.util.*;
+
+public class MGWrapper extends Indexer
+{
+    private static void usage()
+    {
+	System.out.println("java Indexer MGWrapper -i InputDirectory \n -o OutputDirectory \n -c CollectionName");
+    }
+    
+    
+    public static void main(String[] args)
+    {
+	String in = null;
+	String out = null;
+	String coll = null;
+
+	if( args.length == 0 ){
+	    usage();
+	    System.exit(0);
+	}
+	
+	int a = 0;
+	while( a < args.length ){
+	    if( args[a].equals("-i") )
+		in = args[++a];
+	    if( args[a].equals("-o") )
+		out = args[++a];
+	    if( args[a].equals("-c") )
+		coll = args[++a];
+	    a++;
+	}
+	
+	if (out==null || in==null|| coll==null) {
+	    usage();
+	    System.exit(0);
+	}
+	if (!out.endsWith(File.separator)) {
+	    out += File.separator;
+	}
+	if (!in.endsWith(File.separator)) {
+	    in += File.separator;
+	}
+	
+	out = out + "mg_index" + File.separator;
+	
+	MGWrapper mgw = new MGWrapper();
+	mgw.setCollectionName(coll);
+        mgw.setOutputDirectory(out);
+	mgw.setInputDirectory(in);
+	mgw.startIndexing();
+    }
+    
+    
+    public MGWrapper()
+    {
+	super("Managing Gigabytes Indexer");
+    }
+    
+    
+    public void startIndexing()
+    {
+
+	initialize();
+	// new mg requires a .mg_getrc to work
+	// lets put it in the output dir
+	File rc_file = new File(outputDir, ".mg_getrc");
+	if (!rc_file.exists()) {
+	    // if its there, assume it is correct
+	    try {
+		rc_file.createNewFile();
+		BufferedWriter rc_out = new BufferedWriter(new FileWriter(rc_file, true));
+		String data_line = collName+"\tDIR2\t"+inputDir;
+		rc_out.write(data_line, 0, data_line.length());
+		rc_out.newLine();
+		rc_out.flush();
+		rc_out.close();
+	    } catch (Exception ex) {
+		System.err.println(ex);
+		System.err.println("Unable to create file "+rc_file.getPath()+", cancelling build");
+		System.exit(0);
+	    }
+	}
+	/*
+	if( !f.exists() ){
+	    System.out.println("Creating output directory..");
+	    
+            StringTokenizer strt = new StringTokenizer(outputDir,"/"); 
+            String d = "";
+            while(strt.hasMoreTokens()){
+                String str = strt.nextToken();
+                if( str.indexOf(".") == -1 ){
+		    d += "/" + str;
+		    if( !(new File(d)).exists() ){
+			try{
+			    Runtime rt = Runtime.getRuntime();
+			    Process proc = rt.exec("mkdir " + d);
+			    int exitVal = proc.waitFor();
+			}catch(Exception ex){ex.printStackTrace();}
+		    }
+		}
+	    }	
+	}
+	*/
+	try{
+	    //String command = /*GSDL3_HOME +*/ "/packages/VIS/" + "MG/mg/mg-1.2.1/bin/mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    //String command = "mgbuild -d " + outputDir + " -i " + inputDir + " " + collName;
+	    // requires .mg_getrc file
+	    String []env = new String [1];
+	    env[0] = "MG_GETRC="+outputDir+".mg_getrc";
+	    String command = "mgbuild -d " + outputDir + " " + collName;
+	    System.out.println("Executing.. " + command);
+	    Process proc = Runtime.getRuntime().exec(command, env);
+	    proc.waitFor();
+	    System.out.println(proc.exitValue());
+	}
+	catch(IOException ex){ex.printStackTrace();}
+	catch (InterruptedException ex){ System.out.println(ex); ex.printStackTrace();
+	} 
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/AveLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/AveLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/AveLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class AveLink extends LinkFunc
+{
+    float link(float a, float b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Cluster.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Cluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Cluster.java	(revision 21112)
@@ -0,0 +1,76 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int id;
+    public Cluster  parent = null;
+    public Cluster  child_1 = null;
+    public Cluster  child_2 = null;
+    
+    public double children_sim;
+    
+    // size of cluster
+    public int items;
+    
+    public int split;
+    
+    // [] -> centroid for each vector element ?
+    public float[] centroid=null;
+    
+    public Vector iter_items = null;
+    public int iter_items_num = 0;
+    
+    public int x = 0;
+    public int y = 0;
+    
+    public float[] getCentroid(){return centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster c1, Cluster c2, double children_sim, int items)
+    {
+	this.id               = id;
+	this.children_sim     = children_sim;
+	this.items            = items;
+	split                 = 0;
+	this.parent           = parent;
+        iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (c1 != null && c2 != null){
+	    if (c2.items > c1.items){
+		child_1 = c1;
+		child_2 = c2;
+	    }else{
+	        child_1 = c2;
+		child_2 = c1;
+	    }
+	}else{ // if either or both are null
+	    child_1 = c1;
+	    child_2 = c2;
+	}
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (id != -1)
+	    return 0;
+	else
+	    return Math.max(1 + child_1.depth(), 1 + child_2.depth());
+    }
+    
+    
+    void addChildrenToOutputVector(int []coding, Vector output)
+    {
+	if (id != -1){
+    	    Integer ii = new Integer(coding[id]);
+            output.addElement(ii);
+	}else{
+      	    child_1.addChildrenToOutputVector(coding,output);
+      	    child_2.addChildrenToOutputVector(coding,output);
+	}
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/ClusterArray.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/ClusterArray.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(int num_of_clusters)
+    {
+	contents = new Vector(num_of_clusters);
+	
+	for(int i = 0; i < num_of_clusters; i++)
+	    contents.addElement(new Cluster(i, null, null, null, 0, 1));
+    }
+    
+    
+    // counts the number of clusters that are less dissimilar than disim
+    private void sweep_tree (Cluster cluster, double disim, int min_c_items, int[] clusters)
+    {
+	cluster.split = 0;
+	
+	if (cluster.id == -1){
+	    if (cluster.children_sim <= disim && cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }else{
+		sweep_tree (cluster.child_1, disim, min_c_items, clusters);
+		sweep_tree (cluster.child_2, disim, min_c_items, clusters);
+	    }
+    	}else{
+	    if (cluster.items >= min_c_items){
+		cluster.split = 1;
+		contents.addElement(cluster);
+		clusters[0]++;
+	    }
+	}
+    }
+
+
+    double split (Cluster cluster, double[] heights, int n, int target, int min_c_items, int[] clusters)
+    {
+	
+	/*
+	  Try to split 'cluster', made up of 'n' vectors, into 'target'
+	  clusters by incrementing iteratively the mimimum required
+	  similarity between the 'cluster\'s' children. The result is an
+	  array of clusters, 'cluster_ary' which contains 'clusters'
+	  clusters.  Obviously the final number of clusters may not be the
+	  same as the desired number of clusters. The function return the
+	  threshold similarity which resulted in the clusters being split
+	  in 'clusters' clusters.
+	*/
+	
+	int i;
+	
+	if (target < 2) target = 2;
+	
+	if (target > n) target = n;
+	
+	// height store minimum similarity values
+	// this is then the iterative incrementing loop
+	for (i = target - 2; i <= n - 1; i++){
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], min_c_items, clusters);
+	    
+	    if (clusters[0] >= target) break;
+	}
+	
+	// if target - 2 > n - 1, i.e. if the previous loop has not been 
+	// entered at all
+	if (i > n - 1){
+	    /* ignore the min_c_size requirement */
+	    i = target - 1;
+	    clusters[0] = 0;
+	    contents.removeAllElements();
+	    sweep_tree (cluster, heights[i], 1, clusters);
+	}
+	
+	return heights[i];
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Clustering.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Clustering.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Clustering.java	(revision 21112)
@@ -0,0 +1,128 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+    public static final int BUCKSHOT = 0;
+    public static final int COMPLETE = 1;
+    
+    private Vector [] clusters;
+    private double [][] centroids;
+    
+    public Vector [] getClusters()
+    {
+        return clusters;
+    }
+
+    public double[][] getCentroids()
+    {
+        return centroids;
+    }
+    
+    
+    public Clustering (int linkage, float[][] KDMatrix, int min_c_items, int target_clusters, int limit)
+    {
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	
+        int wordCount = KDMatrix[0].length;
+	int hitDocs = KDMatrix.length;
+        
+	Cluster toplevel_cluster;
+        ClusterArray splitclusters = null;
+	int[] n_clusters = new int[]{0};
+	
+        double threshold;
+	
+        Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+
+        int[] coding = new int[hitDocs];
+        
+
+        for (int c = 0; c < hitDocs; c++) coding[c] = c;
+	
+	switch (linkage){
+	case COMPLETE_LINKAGE:
+	    linkage_func = new MaxLink();
+	    break;
+	case AVERAGE_LINKAGE:
+	    linkage_func = new AveLink();
+	    break;
+	case SINGLE_LINKAGE:
+	    linkage_func = new MinLink();
+	    break;
+	default:
+	    System.exit(0);
+	}
+	
+
+        /**** make sure upper bound does not exceed #docs ****/
+        
+	if(limit > hitDocs) limit = hitDocs;
+	limit = (int)hitDocs/2;
+	
+
+	/**** get similarity matrix of the first limit docs with each other ****/
+	
+        sim_matrix = new SimMatrix(KDMatrix, limit, sim_func);
+	
+	heights = new double[hitDocs];
+	
+        Hierarchical hierarchical = new Hierarchical(limit);
+        
+        toplevel_cluster = hierarchical.clustering(sim_matrix, limit, heights, linkage_func);
+	if( toplevel_cluster == null )
+	    return;
+	
+        System.out.println("Clustered items: " + toplevel_cluster.items);
+	Cluster c1 = toplevel_cluster.child_1;
+	Cluster c2 = toplevel_cluster.child_2;
+	System.out.println("Child1: " + c1.items);
+	System.out.println("Child2: " + c2.items);
+	
+	splitclusters = new ClusterArray(0);
+        
+	threshold = splitclusters.split(toplevel_cluster, heights, limit, target_clusters, min_c_items, n_clusters);
+	
+    	System.out.println(n_clusters[0] + " clusters");
+	System.out.println("Threshold: " + threshold);
+	
+        System.out.println("Target_clusters " + target_clusters);
+	System.out.println("Limit " + limit);
+	System.out.println("#docs: " + hitDocs);
+	
+        Iterative iter = new Iterative(wordCount);
+	
+	iter.clustering(splitclusters, n_clusters, KDMatrix, limit, hitDocs, sim_func);
+	
+	System.out.println("Finished iterative clustering");
+
+        clusters = new Vector[n_clusters[0]];
+        centroids = new double[n_clusters[0]][wordCount];
+
+	for (int i = 0; i < n_clusters[0]; i++){
+	    clusters[i]=new Vector();
+	    Cluster clust = (Cluster)splitclusters.contents.elementAt(i);
+	    clust.addChildrenToOutputVector(coding,clusters[i]);
+	    
+	    for (int j = 0; j < clust.iter_items_num; j++){
+		Integer ii = (Integer)clust.iter_items.elementAt(j);
+		Integer jj = new Integer (coding[ii.intValue()]);
+		clusters[i].addElement(jj);
+	    }
+	    
+	    float [] centroid = clust.getCentroid();
+	    
+	    for (int j = 0; j < wordCount; j++){
+		centroids[i][j]=centroid[j];
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Distance.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Distance.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Distance.java	(revision 21112)
@@ -0,0 +1,10 @@
+package vishnu.cluster;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract float get(float[] v1, float[] v2);
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/EuclideanDistance.java	(revision 21112)
@@ -0,0 +1,15 @@
+package vishnu.cluster;
+
+public class EuclideanDistance extends Distance
+{
+    public float get(float[] v1, float[] v2)
+    {
+  	float sum = 0;
+
+	for(int c=0;c<v1.length;c++){
+      	    float x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return (float)Math.sqrt(sum);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Hierarchical.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Hierarchical.java	(revision 21112)
@@ -0,0 +1,234 @@
+package vishnu.cluster;
+
+public class Hierarchical
+{
+    public float[] minrow;
+    public int[] colind;
+    public TriangleIndex triangle;
+    
+    public Hierarchical(int matrix_rows){
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+    }
+    
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+	for (i = index_start + 1; i <= index_end; i++){
+	    if (m.matrix[i] < minrow[row]){
+		minrow[row] = m.matrix[i];
+		colind[row] = i;
+	    }
+	}
+    }
+    
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row){
+	int i;
+	int row;
+	float value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++){
+	    if (minrow[i] < value){
+		row = i;
+		value = minrow[i];
+	    }
+    	}
+	
+	return row;
+    }
+
+    
+    // num_of_rows is "limit" = max docs to be clustered in hierarchical clustering
+    Cluster clustering (SimMatrix sim_matrix, int num_of_rows, double[] heights, LinkFunc linkage_func)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+        int min_row, index, row, col, i;
+	double sim;
+	
+        int last_row_index, sim_matrix_size;
+	
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+	
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	cluster_ary = new ClusterArray (num_of_rows);
+	last_row_index = num_of_rows - 1;
+	
+	// minimum distance for each row of the sim_matrix
+	minrow = new float[num_of_rows];
+	
+	// column index for minimum distance entry
+	colind = new int[num_of_rows];
+	
+	for (i = 1; i < num_of_rows; i++) set_minrow(sim_matrix, i);
+	
+	// do all the hierarchical clustering by running over all rows, updating the matrix 
+	// as a new cluster is formed each time
+	for (i = 0; i < num_of_rows - 1; i++){
+
+	    // find the row with the lowest disimilarity
+	    min_row = get_minrow(last_row_index);
+	    sim = minrow[min_row];
+	    
+	    System.out.println("Min sim: " + sim);
+	    
+	    // the column with the lowest distance in that row
+	    index  = colind[min_row];
+	    
+	    // array of heights, DESCENDING disim, i.e. sim goes last
+	    heights[num_of_rows - 2 - i] = sim; 
+	    
+	    row = triangle.i_ind[index];
+	    col = triangle.j_ind[index];
+	    
+	    // merges rows and cols to create a smaller sim_matrix
+	    merge (row, col, sim_matrix, last_row_index, linkage_func);
+	    System.out.println("Merging: " + row + " and " + col);
+	    
+	    sim_matrix_size -= last_row_index;
+	    
+	    child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+	    child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+	    
+	    // parents get id = -1 to indicate that they are not representing single docs
+	    // all leaves have an id (1 - num_rows) that points to the doc they contain
+	    int sum = child_1.items + child_2.items;
+	    System.out.println("Create new parent with " + child_1.items + " + " + child_2.items + " leaves");
+	    parent = new Cluster (-1, null, child_1, child_2, sim, child_1.items + child_2.items);
+	    
+	    child_1.parent = parent;
+	    child_2.parent = parent;
+	    
+	    cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+	    cluster_ary.contents.setElementAt(parent,col);
+	    
+	    last_row_index--;
+	}
+	
+	return parent;
+    }
+    
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+    {
+	
+	/* i and j are the matrix elements with maximum similarity         */
+	/* matrix is the triangle similarity matrix with last row index n  */
+	/* linkage_func is a pointer to the linkage function               */
+	
+	/* i > j */
+	
+	int c, k, index1, index2;
+	
+        // go over all rows
+	for (k = 0; k <= n; k++){
+	    
+    	    // if k < j the affected elements are on row j
+	    if (k < j){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	    }else if (k > j && k < i){
+		/* if j < k < i then only one element is updated per row */
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if (k > i && k < n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		//printf ("3 - [%d, %d] = link ([%d, %d], [%d, %d]) = %f\n", k, j, k, i, k, j, matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		index1 = triangle.index (k, i);
+		index2 = triangle.index (k, n);
+		m.matrix[index1] = m.matrix[index2];
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+	    }else if(k == n  && i != n){
+		index1 = triangle.index (k, j);
+		index2 = triangle.index (k, i);
+		m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+		
+		if (index1 == colind[k]){
+		    /* the previous minimum of the row has been changed so recaculate minimum of whole row */
+		    set_minrow(m, k);
+		}else{
+		    /* if the changed element is smaller than the current minimum update minrow and colind */
+		    if (m.matrix[index1] < minrow[k]){
+			minrow[k] = m.matrix[index1];
+			colind[k] = index1;
+		    }
+		}
+		
+		for (c = 0; c < i; c++){
+		    index1 = triangle.index (c, i);
+		    index2 = triangle.index (c, n);
+		    m.matrix[index1] = m.matrix[index2];
+		}
+		
+	    }
+    	}
+	set_minrow(m, i);
+	set_minrow(m, j);
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Iterative.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Iterative.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/Iterative.java	(revision 21112)
@@ -0,0 +1,102 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int wordCount;
+    
+    Iterative(int vsize)
+    {
+	wordCount = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+	
+	for (i = 0; i < wordCount; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+
+
+    void cluster_center (Cluster cluster, float [][] vectors, float[] centroid)
+    {
+        int i;
+	
+      	if (cluster.id == -1){
+      	    cluster_center (cluster.child_1, vectors, centroid);
+      	    cluster_center (cluster.child_2, vectors, centroid);
+    	}else{
+            float[] doubleArray = vectors[cluster.id];
+      	    for (i = 0; i < doubleArray.length; i++)
+		centroid[i] += doubleArray[i];
+        }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, float [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+
+	/**** for each hierarchical cluster, compute centroid ****/
+	for (i = 0; i < num_of_clusters; i++){
+	    clust = (Cluster)clusters.contents.elementAt(i);
+	    clust.centroid = new float[wordCount];
+	    cluster_center (clust, vectors, clust.centroid);
+	    
+	    for (ssum = j = 0; j < wordCount; j++)
+		ssum += clust.centroid[j] * clust.centroid[j];
+	    
+	    if(ssum>0)                // normalise
+		for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < wordCount; j++)
+		    clust.centroid[j] *= ssum;
+	}
+    }
+    
+    
+    void place_item (ClusterArray clusters, int num_of_clusters, float [][] vectors, int item_id,  Distance sim_func)
+    {
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	double minDist = sim_func.get(clust.centroid,vectors[item_id]);	
+	int x_minDist = 0;
+	
+	for (int i = 1; i < num_of_clusters; i++){
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    double dist = sim_func.get(clust.centroid,vectors[item_id]);
+	    
+            if (dist < minDist){
+		x_minDist = i;
+		minDist = dist;
+	    }
+    	}
+	
+	//System.out.println("in " + x_minDist); 
+	clust = (Cluster)clusters.contents.elementAt(x_minDist);
+        clust.iter_items.addElement(new Integer(item_id));
+	clust.iter_items_num++;
+    }
+    
+    
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, int[] num_of_clusters, float[][] vectors, int start_index, int end_index,Distance sim_func)
+    {
+        int i;
+
+	System.out.println("Start: " + start_index + " Finish: " + end_index);
+	
+	if (start_index >= end_index)
+	    return;
+	
+	compute_cluster_centers(clusters, num_of_clusters[0], vectors);
+	
+	for (i = start_index; i < end_index; i++){
+	    System.out.println("Placing document " + i);
+	    place_item(clusters, num_of_clusters[0], vectors, i, sim_func);
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/LinkFunc.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/LinkFunc.java	(revision 21112)
@@ -0,0 +1,8 @@
+package vishnu.cluster;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract float link(float a, float b);
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MaxLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MaxLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MaxLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MaxLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a > b? a*99+b: b*99+a)/100;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MinLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MinLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/MinLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.cluster;
+
+public class MinLink extends LinkFunc
+{
+	public float link(float a, float b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/SimMatrix.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/SimMatrix.java	(revision 21112)
@@ -0,0 +1,70 @@
+package vishnu.cluster;
+
+import java.util.*;
+
+public class SimMatrix
+{
+    
+    public float[] matrix;
+    
+
+    public SimMatrix(float[][] KDMatrix, int limit, Distance simfunc){
+	
+	float iarray[];
+	
+	int hitDocs = KDMatrix.length;
+	int wordCount = KDMatrix[0].length;
+	int index = 0;
+	
+	// to exclude diagonal and conjugates
+	// - - - - -
+	//         |
+	// *       |
+	// * *     |
+	// * * *   |    
+	
+	int realSize = (hitDocs * (hitDocs - 1)/2);
+	matrix = new float[realSize];
+	
+	// this gets the sim matrix for all docs rather 
+	// than only the first limit - mistake?
+
+	for(int i = 1; i < hitDocs; i++){
+	    iarray = KDMatrix[i];
+	    for(int j = 0; j < i; j++){
+		matrix[index] = simfunc.get(iarray,KDMatrix[j]);
+	        index++;
+	    }
+	}
+    }
+
+    public void printMatrix()
+    {
+	int countA = 0;
+	int countB = 0;
+
+	for( int i = 0; i < matrix.length; i++ ){
+	    System.out.print(matrix[i] + " ");
+	    if( countA >= countB ){
+		System.out.println();
+		countB++;
+		countA = 0;
+	    }
+	    else countA++;
+	}
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheInt.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheInt.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheInt.java	(revision 21112)
@@ -0,0 +1,11 @@
+package vishnu.cluster;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheVectors.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheVectors.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TheVectors.java	(revision 21112)
@@ -0,0 +1,79 @@
+package vishnu.cluster;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TriangleIndex.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/cluster/TriangleIndex.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.cluster;
+
+class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Collection.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Collection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Collection.java	(revision 21112)
@@ -0,0 +1,23 @@
+package vishnu.datablock;
+
+import java.io.*;
+
+public class Collection implements Serializable
+{
+    public String dir;
+    public String name;
+    public String desc;
+    public int size;
+    public String engine;
+    public String [] views;
+    
+    public Collection(String di, String n, String d, int s, String e, String [] v)
+    {
+	dir = di;
+	name = n; 
+	desc = d; 
+	size = s;
+	engine = e;
+	views = v;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/DataBlock.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/DataBlock.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/DataBlock.java	(revision 21112)
@@ -0,0 +1,35 @@
+package vishnu.datablock;
+
+import java.io.*;
+import java.util.*;
+
+
+
+public class DataBlock implements Serializable
+{
+    public int [] docs = null;
+    public String [] descriptions = null;
+    public String [] words = null;
+    public SparseMatrix matrix = null;
+    public Vector [] clusters = null;
+    public Point2D [] sammon = null;
+
+    public DataBlock() {
+    }
+    public DataBlock(int[] docs,
+		     String [] descriptions,
+		     String [] words,
+		     SparseMatrix matrix,
+		     Vector [] clusters,
+		     Point2D [] sammon)
+    {
+	this.docs = docs;
+	this.descriptions = descriptions;
+        this.words = words;
+	this.matrix = matrix;
+	this.clusters = clusters;
+	this.sammon = sammon;
+    }
+    
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Node.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Node.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Node.java	(revision 21112)
@@ -0,0 +1,37 @@
+package vishnu.datablock;
+
+public class Node
+{
+    Node next;
+    int value;
+    
+    public Node(int i, Node n)
+    {
+        value = i;
+        next = n;
+    }
+    
+    public Node(int i)
+    {
+        value = i;
+        next = null;
+    }
+    
+    public Node push(Node n)
+	{
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+	public Node pop()
+	{
+		return next;
+	}
+	
+	public int getValue()
+	{
+	    return value;
+	}
+  
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Point2D.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Point2D.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Point2D.java	(revision 21112)
@@ -0,0 +1,21 @@
+package vishnu.datablock;
+import java.io.*;
+/** 
+* Point2D.java: Class for points in logical coordinates.
+* From Section 1.5 of
+* @author Ammeraal, L. (1998) Computer Graphics for Java Programmers,
+* Chichester: John Wiley.
+*/       
+public class Point2D  implements Serializable
+{
+   /** x cartesian coordinate */
+   public float x;
+   /** y cartesian coordinate */
+   public float y;
+   /**
+   * constructor
+   * @param cartesian x coordinate
+   * @param cartesian y coordinate
+   */
+   public Point2D(float x, float y){this.x = x; this.y = y;}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Query.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Query.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/Query.java	(revision 21112)
@@ -0,0 +1,69 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+
+package vishnu.datablock;
+import java.io.*;
+
+public class Query implements Serializable
+{
+    public String query = null;
+    public String collection = null;
+    public int docNum = -1;
+    public String action = null;
+
+    public Query(){}
+
+    public Query(String q, String c, String s)
+    {
+	query = q;
+	collection = c;
+    }
+
+    public void setCollection(String c)
+    {
+	collection = c;
+    }
+
+    public void setDocNum(int id)
+    {
+	docNum = id;
+    }
+
+    public void setQuery(String q)
+    {
+	query = q;
+    }
+
+    public void setAction(String a)
+    {
+	action = a;
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/ShortFileException.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/ShortFileException.java	(revision 21112)
@@ -0,0 +1,6 @@
+package vishnu.datablock;
+public class ShortFileException extends Exception
+{
+    public ShortFileException(){super();}
+    public ShortFileException(String s){super(s);}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseMatrix.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseMatrix.java	(revision 21112)
@@ -0,0 +1,81 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseMatrix implements Serializable
+{
+    private SparseNode [] nodes;
+    private int n_cols;
+    private int n_rows;
+    
+    public SparseMatrix(int rows, int cols)
+    {
+        n_rows = rows;
+	n_cols = cols;
+	
+        nodes = new SparseNode[n_rows];
+	
+        for (int c = 0; c < n_rows; c++)
+	    nodes[c] = null;
+    }
+    
+    
+    public void addDataItem(int row, int column, float value)
+    {
+	SparseNode node; 
+	node = new SparseNode(value, column);
+	nodes[row] = node.push(nodes[row]);
+    }
+    
+    
+    public SparseNode getRow(int row)
+    {
+	return nodes[row];
+    }
+
+
+    public int getRowCount()
+    {
+	    return n_rows;
+    }
+
+
+    public int getColumnCount()
+    {
+	    return n_cols;
+    }
+    
+
+    public float[][] getExpanded()
+    {
+        float[][] full = new float[n_rows][n_cols];
+	
+	SparseNode n;
+	
+	for(int r = 0; r < n_rows; r++){
+	    for (int c = 0; c < n_cols; c++)
+		full[r][c]=0;
+	    
+	    n=nodes[r];
+	    
+	    while(n!= null){
+		full[r][n.getColumn()]=n.getValue();
+		n=n.getNext();
+	    }
+	}
+	return full;
+    }
+    
+    
+    public void print()
+    {
+	for( int i = 0; i < nodes.length; i++ ){
+	    SparseNode n = getRow(i);
+	    while(n != null ){
+		System.out.print(n.getColumn()+ " ");
+		n = n.getNext();
+	    }
+	    System.out.println();
+	}
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseNode.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseNode.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/datablock/SparseNode.java	(revision 21112)
@@ -0,0 +1,59 @@
+package vishnu.datablock;
+import java.io.*;
+
+public class SparseNode implements Serializable
+{
+    private float value;
+    
+    private int column;
+    private SparseNode next;
+    
+    
+    SparseNode(float f, int c)
+    {
+	value = f;
+	column = c;
+	next = null;
+    }
+    
+    
+    SparseNode(float f, int c, SparseNode n)
+    {
+	value = f;
+	column = c;
+	next = n;
+    }
+    
+
+    public SparseNode push(SparseNode n)
+    {
+        if (next != null) next.push(n);
+        else next = n;
+        return this;
+    }
+
+
+    public SparseNode pop()
+    {
+	return next;
+    }
+
+    
+    public SparseNode getNext()
+    {
+        return next;
+    }
+
+    
+    public int getColumn()
+    {
+        return column;
+    }
+    
+
+    public float getValue()
+    {
+        return value;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKServer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKServer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKServer.java	(revision 21112)
@@ -0,0 +1,293 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Enumeration;
+import vishnu.util.Misc;
+import vishnu.util.Sammon;
+import vishnu.cluster.*;
+import vishnu.datablock.*;
+
+public class CKServer 
+{
+    private String index_directory = null;
+
+    private final static String FILE_DOC_FREQ = "doc_freq";
+    private final static String FILE_TERM_FREQ = "term_freq";
+    private final static String FILE_CK_WORDS = "ck_words";
+    
+    private float[] docFreq = null;
+    private String[] ck_words = null;
+    private Vector termFreq = null;
+
+
+    private int[] x_keywords;
+    private int[] x_hitdocs;
+    private float[][] KDMatrix;
+    private Vector[] clusters;
+    private double[][] centroids;
+    private Point2D[] sammon;
+    private Vector descriptions;
+    
+    public CKServer(String collection_home, String view) {
+	
+	this.index_directory = collection_home;
+	if (!this.index_directory.endsWith(File.separator)) {
+	    this.index_directory += File.separator;
+	}
+	this.index_directory += "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    this.index_directory += view + File.separator;
+	}
+	    
+	loadIndices();
+    }
+    
+    private void loadIndices(){
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	try {
+	    docFreq = (float[])getObject(this.index_directory  + FILE_DOC_FREQ);
+	    ck_words = (String[])getObject(this.index_directory + FILE_CK_WORDS);
+	    termFreq = (Vector)getObject(this.index_directory + FILE_TERM_FREQ);
+	} catch (Exception e) {
+	    System.err.println("CKServer Error: Couldn't load up ck index data: "+e);
+	}
+    }
+
+    
+    private Object getObject(String str)
+	throws IOException, FileNotFoundException, ClassNotFoundException {
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	fis = new FileInputStream(str);
+	in = new ObjectInputStream(fis);
+	o = in.readObject();
+	in.close();
+	return o;
+    }
+
+
+    public void setDescriptions(Vector desc)
+    {
+	descriptions = desc;
+    }
+    
+
+    public String[] computeKeywords(Vector docNum)
+    {
+	int hits = docNum.size();
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+	
+
+	/**** compute interesting weight for each keyword ****/
+	
+	Enumeration enumer = keyHash.keys();
+	
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+	
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+	    
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+	    
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+	    
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+	
+	
+	/**** store indices of up to 100 top ranked keywords ****/
+	
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	}
+	
+	
+	/**** sort the indices_kw array in increasing order of indices ****/
+	
+	Misc.quickSort(indices_kw);
+
+	x_keywords = indices_kw;
+	
+	String[] keywords = new String[x_keywords.length];
+	for( int i = 0; i < x_keywords.length; i++ ){
+	    keywords[i] = ck_words[x_keywords[i]];
+	}
+
+	return keywords;
+    }
+    
+    
+    public SparseMatrix getSparseMatrix(Vector docNums){
+		
+	int hits = docNums.size();
+	int wordCount = x_keywords.length;
+	int scale = 100; // length of document vector
+	
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+	
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNums.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(x_keywords[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[x_keywords[j]]));
+		    w = term_freq.intValue();
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+	    }
+	    
+	    
+	    /**** sqrt sum to obtain |z| ****/
+	    
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i])/scale; else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix and full matrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+	KDMatrix = new float[hits-emptyRows][wordCount];
+		
+	int ci = 0;
+
+	for( int i = hits-1; i >= 0; i-- ){
+	    if( sum_rows[i] == 0 ){
+		docNums.removeElementAt(i);
+		descriptions.removeElementAt(i);
+	    }
+	}
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 ) continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = m; //(float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    KDMatrix[ci][j] = norm_m; //(float)(norm_m);
+		}
+		else KDMatrix[ci][j] = 0;
+	    }
+	    ci++;
+	}
+	
+	x_hitdocs = new int[docNums.size()];
+	
+	for( int i = 0; i < docNums.size(); i++ ){
+	    x_hitdocs[i] = ((Integer)docNums.elementAt(i)).intValue();
+	}
+	
+	return matrix;
+    }
+
+
+    public Vector[] getClusters(){
+
+	int wordCount = KDMatrix[0].length;
+	int docHits = KDMatrix.length;
+	System.err.println("getClusters: word count = "+wordCount+", docHits = "+docHits);
+        int clusterNum =  docHits>20?10:docHits;
+      
+        int limit =  docHits/2;
+	
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,KDMatrix,1,clusterNum,limit);
+	
+        clusters = clustering.getClusters();
+	centroids= clustering.getCentroids();
+	
+	// what are these for??
+	//Sammon map = new Sammon(centroids);
+	//Point2D [] sammon = map.getMapping();
+	return clusters;
+    }
+    
+    
+    public int[] getHitDocuments()
+    {
+	return x_hitdocs;
+    }
+
+
+    public double[][] getCentroids()
+    {
+	return centroids;
+    }
+
+
+    public String[] getHitDescriptions()
+    {
+	String[] desc = new String[descriptions.size()];
+	for( int i = 0; i < desc.length; i++ ){
+	    desc[i] = (String)descriptions.elementAt(i);
+	}
+
+	return desc;
+    }
+    
+    public Point2D[] getSammonMap(double[][] centroids)
+    {
+	if (centroids == null) {
+	    return null;
+	}
+	Sammon sammon = new Sammon(centroids);
+	Point2D[] p = sammon.getMapping();
+	
+	return p;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKTester.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKTester.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/CKTester.java	(revision 21112)
@@ -0,0 +1,230 @@
+package vishnu.server;
+
+import java.io.*;
+import java.util.*;
+import vishnu.server.Search.*;
+import vishnu.util.*;
+import vishnu.datablock.*;
+
+public class CKTester
+{
+
+    public static void main(String[] args){
+	CKTester ckt = new CKTester(args);
+    }
+
+    
+    public CKTester(String[] args){
+	
+	Vector docNum = new Vector();
+	Vector descriptions = new Vector();
+	
+	String query = args[0];
+	String coll = args[1];
+	String vis_home = args[2];
+	String view = null;
+	if (args.length >= 4) {
+	    view = args[3];
+	}
+
+	/**** read in ck words, doc freq and term freq ****/
+
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	
+	String root = vis_home;
+	if (!root.endsWith(File.separator)) {
+	    root += File.separator;
+	}
+	root += "collections" + File.separator+ coll + File.separator + "index" + File.separator + "ck_index" + File.separator;
+	if (view != null) {
+	    root += view+File.separator;
+	}
+	String fileDocFreq = "doc_freq";
+	String fileTermFreq = "term_freq";
+	String fileCKWords = "ck_words";
+	
+	float[] docFreq = (float[])getObject(root+fileDocFreq);
+	String[] ck_words = (String[])getObject(root + fileCKWords);
+	Vector termFreq = (Vector)getObject(root + fileTermFreq);
+
+
+	/**** get relevant docs ****/
+
+	LUCSearcher lus = new LUCSearcher(coll, view); // coll should be fullpath
+	//lus.setRootDirectory(vis_home);
+	lus.search(query);
+	docNum = lus.getDocIdentifiers();
+	descriptions = lus.getDocDescriptions();
+	
+	int hits = docNum.size();
+	
+
+	/**** pool the candidate keywords and note their frequencies ****/
+	
+	Hashtable keyHash = new Hashtable();
+	
+	int count_ck_all = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    count_ck_all += ht.size();
+	    
+	    Enumeration enumer = ht.keys();
+	    while( enumer.hasMoreElements() ){
+		Integer term_index = (Integer)enumer.nextElement();
+		
+		Integer set_freq;
+		if( (set_freq = (Integer)keyHash.get(term_index)) == null ){
+		    keyHash.put(term_index,new Integer(1));
+		}else{
+		    keyHash.put(term_index,new Integer(set_freq.intValue()+1));
+		}
+	    }
+	}
+
+	System.out.println("all keywords: " + count_ck_all);
+	System.out.println("pooled: " + keyHash.size());
+
+
+	/**** compute interesting weight for each keyword ****/
+
+	Enumeration enumer = keyHash.keys();
+
+	float[] weight_ck = new float[keyHash.size()];
+	Integer[] weight_index = new Integer[keyHash.size()];
+	int count = 0;
+
+	while( enumer.hasMoreElements() ){
+	    Integer index_ck = (Integer)enumer.nextElement();
+	    Integer set_freq = (Integer)keyHash.get(index_ck);
+	    
+	    float set_freq_f = (float)set_freq.intValue()/hits;
+
+	    float f1 = (float)set_freq_f/docFreq[index_ck.intValue()];
+	    float f2 = -(float)Math.log(set_freq_f);
+
+	    weight_index[count] = index_ck;
+	    weight_ck[count] = - f1 * f2;
+
+	    //System.out.println(set_freq_f+" "+docFreq[index_ck.intValue()]+" "+weight_ck[count]);
+	    count++;
+	}
+	
+	Misc.quickSort(weight_ck,weight_index);
+
+
+	/**** store indices of up to 100 top ranked keywords ****/
+
+	int wordCount = Math.min(100,count);
+	int[] indices_kw = new int[wordCount];
+	
+	for( int i = 0; i < wordCount; i++ ){
+	    indices_kw[i] = weight_index[i].intValue();
+	    
+	    float weight = -weight_ck[i];
+	    String ck = ck_words[indices_kw[i]];
+	    System.out.println(indices_kw[i] + " " + ck + " " + weight + " ");
+	}
+
+
+	/**** sort the indices_kw array in increasing order of indices ****/
+
+	Misc.quickSort(indices_kw);
+
+
+	/**** build full matrix: rows indices_kw (wordCount), col docNum (hits) ****/
+
+	float[][] matrix_kd = new float[hits][wordCount];
+	float[] sum_rows = new float[hits];
+	int emptyRows = 0;
+
+	for( int i = 0; i < hits; i++ ){
+	    
+	    int doc_index = ((Integer)docNum.elementAt(i)).intValue();
+	    Hashtable ht = (Hashtable)termFreq.elementAt(doc_index);
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		Integer term_freq;
+		
+		if( (term_freq = (Integer)ht.get(new Integer(indices_kw[j]))) != null ){
+		    float w = -(float)(Math.log(1+term_freq.intValue()) * Math.log(docFreq[indices_kw[j]]));
+		    matrix_kd[i][j] = w;
+		    sum_rows[i] += w*w;
+		}
+		System.out.print(matrix_kd[i][j] + " ");
+	    }
+	    
+	    System.out.println();
+	    
+
+	    /**** sqrt sum to obtain |z| ****/
+
+	    if( sum_rows[i] != 0 ) sum_rows[i] = (float)Math.sqrt(sum_rows[i]); else emptyRows++;
+	}
+	
+	/**** populate sparseMatrix dividing each entry by the row_sum ****/
+	
+	SparseMatrix matrix = new SparseMatrix(hits-emptyRows,wordCount);
+
+	int ci = 0;
+	
+	for( int i = 0; i < hits; i++ ){
+	    
+	    if( sum_rows[i] == 0 )
+		continue;
+	    
+	    for( int j = 0; j < wordCount; j++ ){
+		
+		float m = matrix_kd[i][j];
+		
+		if( m != 0 ){
+		    float norm_m = (float)m/sum_rows[i];
+		    matrix.addDataItem(ci, j, norm_m);
+		    System.out.print(j + " (" + norm_m + ") ");
+		}
+	    }
+
+	    ci++;
+	    System.out.println();
+	}
+    }
+    
+
+
+    private Object getObject(String str){
+	Object o = null;
+	FileInputStream fis = null;
+	ObjectInputStream in = null;
+	try{
+	    fis = new FileInputStream(str);
+	    in = new ObjectInputStream(fis);
+	    o = in.readObject();
+	    in.close();
+	}
+	catch(IOException ex){
+	    ex.printStackTrace();
+	}
+	catch( ClassNotFoundException ex){
+	    ex.printStackTrace();
+	}
+	return o;
+    }
+}
+
+
+class Node{
+    Node next = null;
+    float value = -1;
+    int docIndex = -1;
+    int kwIndex = -1;
+
+    public Node(int d, int k)
+    {
+	docIndex = d;
+	kwIndex = k;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Packaging.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Packaging.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Packaging.java	(revision 21112)
@@ -0,0 +1,376 @@
+package vishnu.server;
+
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+//import org.apache.xpath.XPathAPI;
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+import java.awt.*;
+
+public class Packaging
+{
+    static Document doc = null;
+    static boolean out = false;
+
+    static String matrixString = "";
+
+    public Packaging()
+    {
+    }
+
+
+    public static int[] inInts(StringTokenizer st)
+    {
+	String line;
+	int intCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new int[0];
+	}
+
+	intCount = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(intCount);
+
+	int[] intarray = new int[intCount];
+	for (int c = 0; c < intCount;c++)
+	    {
+		line = st.nextToken();
+		int n = Integer.parseInt(line);
+		intarray[c] = n;
+		if(out) System.out.println(n);
+	    }
+	return intarray;
+    }		
+    
+    
+    public static String [] inStrings(StringTokenizer st)
+    {
+	String line;
+	String [] instrings;
+	int strCount;
+
+	if (!st.hasMoreTokens()) {
+	    return new String[0];
+	}
+	
+	strCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(strCount);
+	   
+	instrings = new String[strCount];
+	for (int c = 0; c < strCount;c++)
+	    {
+		instrings[c] = st.nextToken();
+	    }
+
+	return instrings;
+    }
+
+    public static SparseMatrix inMatrix(StringTokenizer st, int wordCount)
+    {
+	String line;
+	int rowCount,column,value;
+
+	if (!st.hasMoreTokens()) {
+	    return new SparseMatrix(0, wordCount);
+	}
+
+	rowCount = Integer.parseInt(st.nextToken());
+	matrixString += String.valueOf(rowCount) + "\n";
+	if(out) System.out.println(rowCount);
+	
+	SparseMatrix matrix = new SparseMatrix(rowCount,wordCount);
+	
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		matrixString += line + "\n";
+
+		if(out) System.out.println(line);
+		
+		if(line.length() > 1)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					matrix.addDataItem(c, column, value);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+	return matrix;
+    } // inMatrix
+
+    
+    public static Vector [] inCluster(StringTokenizer st, int docCount)
+    {
+	String line;
+	int clusterCount,value;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Vector[0];
+	}
+	clusterCount = Integer.parseInt(st.nextToken());
+	if(out)	System.out.println(clusterCount);
+	Vector [] clusters = new Vector[clusterCount];
+	
+	for (int c = 0; c < clusterCount;c++)
+	    {
+		Vector v = new Vector();
+
+		line = st.nextToken();
+		if(out) System.out.println(line);
+		if (line.length()>0)
+		    {
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				String tokenString = tokens.nextToken();
+				
+				value = Integer.parseInt(tokenString);
+				if(value >= 0 && value <= docCount)
+				    {
+					v.addElement(new Integer(value));
+				    } // if
+			    } // while
+		    } // if
+		
+		clusters[c]=v;
+	    } // for
+	return clusters;
+    }
+ 
+
+    public static Point2D[] inPoint2D(StringTokenizer st)
+    {
+	String line;
+	int count ;
+	float x = 0,y = 0;
+	
+	if (!st.hasMoreTokens()) {
+	    return new Point2D[0];
+	}
+	
+	count = Integer.parseInt(st.nextToken());
+	if(out) System.out.println(count);
+	
+	Point2D [] array = new Point2D[count];
+	
+	for (int c = 0; c < count;c++)
+	    {
+		line = st.nextToken();
+		if(out)	System.out.println(line);
+		StringTokenizer tokens = new StringTokenizer(line,",");
+
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			x = Float.parseFloat(tokenString);
+		    }
+			
+		if(tokens.hasMoreTokens())
+		    {
+			String tokenString = tokens.nextToken();
+			y = Float.parseFloat(tokenString);
+		    } // if
+		    		
+		array[c]= new Point2D(x,y);
+	    } // for
+	return array;
+    }
+
+ 
+ public Element getXMLPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	Element response = doc.createElement("response");
+	response.setAttribute("from","Packaging");
+	
+	Element docList = doc.createElement("docNodeList");
+	response.appendChild(docList);
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		Element node = doc.createElement("docNode");
+		node.setAttribute("docNum", String.valueOf(docs[i]));
+		node.setAttribute("title",(String)desc.elementAt(i));
+		docList.appendChild(node);
+	    }
+
+	Element sammonList = doc.createElement("sammonList");
+	response.appendChild(sammonList);
+	for( int i = 0; i < sammon.length; i++ )
+	    {
+		float x = sammon[i].x;
+		float y = sammon[i].y;		
+		Element node = doc.createElement("sammonNode");
+		node.setAttribute("xpos",String.valueOf(x));
+		node.setAttribute("ypos",String.valueOf(y));
+		sammonList.appendChild(node);
+	    }
+
+	Element wordList = doc.createElement("wordList");
+	response.appendChild(wordList);
+	for( int i = 0; i < words.length; i++ )
+	    {
+		Element node = doc.createElement("wordNode");
+		node.setAttribute("term",words[i]);
+		wordList.appendChild(node);
+	    }
+
+	Element mat = doc.createElement("DocTermSparseMatrix");
+	response.appendChild(mat);
+
+	st = new StringTokenizer(matrixString);
+	String line;
+	int rowCount,column,value;
+
+	rowCount = Integer.parseInt(st.nextToken());
+	mat.setAttribute("rows",String.valueOf(rowCount));
+	mat.setAttribute("cols",String.valueOf(words.length));
+
+	Element rowList = doc.createElement("rowList");
+	mat.appendChild(rowList);
+
+	for (int c = 0; c < rowCount;c++)
+	    {
+		line = st.nextToken();
+		if(line.length() > 1)
+		    {
+			Element colList = doc.createElement("colList");
+			rowList.appendChild(colList);
+			colList.setAttribute("row",String.valueOf(c));
+			
+			StringTokenizer tokens = new StringTokenizer(line,",");
+			
+			while(tokens.hasMoreTokens())
+			    {
+				Element entryElem = doc.createElement("entry");
+				colList.appendChild(entryElem);
+
+				String tokenString = tokens.nextToken();
+				value = Integer.parseInt(tokenString);
+
+				entryElem.setAttribute("docFreq",tokenString);
+				
+				if(tokens.hasMoreTokens())
+				    { 
+					tokenString = tokens.nextToken();
+					column = Integer.parseInt(tokenString);
+					entryElem.setAttribute("term",tokenString);
+				    }
+			    }// while				
+		    } //if
+	    } // for 
+
+	Element clusterList = doc.createElement("ClusterList");
+
+	response.appendChild(clusterList);
+
+	for( int i = 0; i < clusters.length; i++ )
+	    {
+		Element cluster = doc.createElement("Cluster");
+		clusterList.appendChild(cluster);
+
+		Vector vector = clusters[i];
+		for( int c = 0; c < vector.size(); c++ )
+		    {
+			Element entry = doc.createElement("leaf");
+			cluster.appendChild(entry);
+			
+			Integer I = (Integer)vector.elementAt(c);
+			entry.setAttribute("docNum",String.valueOf(I.intValue()));
+		    }
+	    }
+
+	for( int i = 0; i < docs.length; i++ ){
+            descriptions[i] = (String)desc.elementAt(i);
+	}
+
+	return response;
+    }
+
+
+   
+    public DataBlock getDataPackage(String result, Vector desc) 
+    {
+	String [] words;
+	int [] docs;
+	String [] descriptions;
+	SparseMatrix matrix;
+	Vector [] clusters;
+	Point2D [] sammon;
+
+	StringTokenizer st = new StringTokenizer(result);
+
+	docs = inInts(st);
+	words = inStrings(st);
+	matrix = inMatrix(st,words.length);
+	clusters = inCluster(st, docs.length);
+	sammon = inPoint2D(st);	
+	descriptions = new String[docs.length];
+
+	for( int i = 0; i < docs.length; i++ )
+	    {
+		descriptions[i] = (String)desc.elementAt(i);
+		//descriptions[i]	= "doc"+Integer.toString(i);
+	    }
+
+	DataBlock data = new DataBlock(docs,descriptions,words,matrix,clusters,sammon); 
+	return data;
+    }
+
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/LUCSearcher.java	(revision 21112)
@@ -0,0 +1,173 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+
+import java.util.Vector;
+import java.util.Hashtable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+
+public class LUCSearcher implements SearchInterface
+{
+    protected Document doc=null;
+    private String collection_home;
+    private String index_dir;
+    private Vector docIDs;
+    private Vector docDescs;
+    private Hits hits;
+    private Hashtable ht;
+    
+    public LUCSearcher(String coll_home, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	index_dir = collection_home + File.separator + "index" + 
+	    File.separator + "luc_index" + File.separator;
+	if (view != null) {
+	    index_dir += view + File.separator;
+	}
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+    
+    
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+    
+    
+    public Vector getDocContent(int doc_num)
+    {
+	System.err.println("in doc content, doc num="+doc_num);
+	Integer in = null;
+	if (ht==null) {
+	    System.err.println("ht = null");
+	}
+	try {
+	    in = (Integer)ht.get(new Integer(doc_num));
+	} catch (Exception e) {
+	    System.err.println("exception happened: "+e);
+	}
+	System.err.println("*");
+	int id = 1;
+	System.err.println("*");
+	if( in != null ) id = in.intValue();
+	System.err.println("hit id ="+id);
+	Vector v = new Vector();
+
+	try {
+	    Directory directory = FSDirectory.getDirectory(new File(index_dir), false);
+
+	    IndexReader reader = IndexReader.open(directory);
+	    
+	    Document doc = reader.document(doc_num);
+	    String path = doc.get("path");
+	    // make sure the slashes go the right way
+	    path = path.replaceAll("[\\\\\\/]", File.separator);
+	    String full_path = collection_home + path;
+	    File f = new File(full_path);
+	    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+	    String line;
+	    
+	    while( (line=dis.readLine()) != null ){
+		v.addElement(line);
+	    }
+	    dis.close();
+	} catch(IOException ex){v.addElement("Content of this document currently unavailable");}    
+	return v;
+    }
+
+    
+    public void search(String q)
+    {
+	docIDs = new Vector();
+	docDescs = new Vector();
+	ht = new Hashtable();
+	
+	try {
+	    Searcher searcher = new IndexSearcher(index_dir);
+	    Analyzer analyzer = new StandardAnalyzer();
+	    
+	    Query query = QueryParser.parse(q, "contents", analyzer);
+	    System.out.println("Searching for: " + query.toString("contents") + " in " + index_dir);
+	    
+	    hits = searcher.search(query);
+	    System.out.println(hits.length() + " total matching documents");
+	    int max_docs = Math.min(MAX_DOCS,hits.length());
+	    for (int i = 0; i < max_docs; i++) {
+
+		int id = hits.id(i);
+		ht.put(new Integer(id),new Integer(i));
+		docIDs.addElement(new Integer(id));
+		
+		Document doc = hits.doc(i);
+		String path = doc.get("path");
+		// make sure the slashes go the right way
+		path = path.replaceAll("[\\\\\\/]", File.separator);
+		String full_path = collection_home + path;
+		File f = new File(full_path);
+		
+		String line = "";
+		
+		try{
+		    BufferedReader dis = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+		    while( line.length() < MAX_DESCRIPTION_LENGTH ){
+			String tmp = dis.readLine();
+			line += tmp;
+		    }
+		    docDescs.addElement(line.substring(0,MAX_DESCRIPTION_LENGTH));
+		    dis.close();
+		}catch(IOException ex){docDescs.addElement(new String("No description available"));}    
+	    }
+	    searcher.close();
+	    
+	} catch (Exception ex) {ex.printStackTrace();}
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/MGSearcher.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/MGSearcher.java	(revision 21112)
@@ -0,0 +1,101 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+
+import org.greenstone.mg.*;
+
+public class MGSearcher implements SearchInterface
+{
+ 
+    private String collection_home;
+    private String collection_name;
+    private Vector docIDs;
+    private Vector docDescs;
+    private String mg_index_path;
+    private MGWrapper mg_src;
+    
+    public MGSearcher(String coll_home, String coll_name, String view)
+    {
+	collection_home = coll_home;
+	if (!collection_home.endsWith(File.separator)) {
+	    collection_home += File.separator;
+	}
+	collection_name = coll_name;
+	mg_src = new MGWrapper();
+	mg_index_path = "index"+File.separator+"mg_index"+ File.separator;
+	if (view != null) {
+	    mg_index_path += view + File.separator;
+	}
+	mg_index_path += collection_name+File.separator+collection_name;
+	mg_src.setIndex(mg_index_path);
+    }
+
+
+    public Vector getDocIdentifiers()
+    {
+	return docIDs;
+    }
+
+
+    public Vector getDocDescriptions()
+    {
+	return docDescs;
+    }	
+
+    // why do we want document in a vector ?? -- kjdon
+    public Vector getDocContent(int doc_num) {
+
+	Vector document = new Vector();
+	String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	String [] lines = doc_content.split("\n");
+	for (int i=0; i<lines.length; i++) {
+	    document.addElement(lines[i]);
+	}
+	return document;
+
+    }
+
+    public void search(String query) {
+	this.mg_src.runQuery(collection_home, mg_index_path, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	docIDs = new Vector();
+	docDescs = new Vector();
+	Vector doc_results =   mqr.getDocs();
+	int max_docs = Math.min(MAX_DOCS,doc_results.size());
+	for (int i=0; i<max_docs; i++) {
+	    int doc_num = (int)((MGDocInfo)doc_results.get(i)).num_;
+	    docIDs.addElement(new Integer(doc_num));
+	    String doc_content = this.mg_src.getDocument(collection_home, mg_index_path, doc_num);
+	    docDescs.addElement(doc_content.substring(0,MAX_DESCRIPTION_LENGTH));
+	}	
+    }
+    
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/SearchInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/Search/SearchInterface.java	(revision 21112)
@@ -0,0 +1,41 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server.Search;
+
+import vishnu.datablock.*;
+import java.util.*;
+import org.w3c.dom.Element;
+
+public interface SearchInterface
+{
+
+    public static final int MAX_DOCS = 100;
+    public static final int MAX_DESCRIPTION_LENGTH = 200;
+
+    public void search(String query);
+
+    public Vector getDocIdentifiers();
+
+    public Vector getDocDescriptions();
+
+    public Vector getDocContent(int doc_num);
+
+    //public void setRootDirectory(String root);
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/VisServlet.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/VisServlet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/server/VisServlet.java	(revision 21112)
@@ -0,0 +1,328 @@
+/*********************************************************************
+ Copyright (C) 2003 Imperial College London
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.server;
+
+import vishnu.server.Search.*;
+import vishnu.datablock.*;
+import vishnu.util.XMLUtil;
+
+import java.io.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.util.Vector;
+import java.util.Hashtable;
+
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+public class VisServlet extends HttpServlet
+{
+    final String DOCUMENT_FETCH_ACTION = "fetch";
+    final String DOCUMENT_SEARCH_ACTION = "search";
+    final String KEYWORDS_COMPUTE_ACTION = "keywords";
+    final String INITIALIZATION_ACTION = "initialise";
+    
+    protected String COLLS_HOME=null; 
+    
+    Hashtable collH;
+
+    public void init(ServletConfig config) throws ServletException {
+	// always call super.init;
+	super.init(config);
+
+	COLLS_HOME = config.getInitParameter("collectionsHome");
+	if (COLLS_HOME == null || COLLS_HOME.equals("")) {
+	    System.err.println("initialization parameter, collectionsHome, not set");
+	    System.exit(1);
+	}
+	if (!COLLS_HOME.endsWith(File.separator)) {
+	    COLLS_HOME += File.separator;
+	}
+    }
+
+    private Object receiveObject(HttpServletRequest req) throws Exception
+    {
+        ObjectInputStream in = new ObjectInputStream( req.getInputStream() );
+        Object obj = in.readObject();
+        in.close();
+        return obj;
+    }
+    
+    
+    private void sendObject(HttpServletResponse resp, Object obj) throws Exception
+    {
+        ObjectOutputStream out = new ObjectOutputStream( resp.getOutputStream() );
+        out.writeObject(obj);
+        out.close();
+    }
+    
+    public void doGet(HttpServletRequest req, HttpServletResponse resp) 
+	throws IOException {
+
+	resp.setContentType("text/html;charset=UTF-8");
+        PrintWriter out = resp.getWriter();
+	String output = "<html><head><title>Visualiser servlet</title></head>\n<body>The servlet is running, and accepting messages via POST</body></html>";
+	out.println(output);
+
+    }
+    public void doPost(HttpServletRequest req, HttpServletResponse resp)
+    {
+	Object obj;
+	
+        try{
+	    obj = receiveObject(req);
+	    Vector q = (Vector)obj;
+	    Vector response = new Vector();
+	    
+	    String action = (String)q.elementAt(0);
+	    System.out.println("Action.. " + action);
+	    
+ 	    if( action.equals(INITIALIZATION_ACTION) ){
+		Vector colls = activateCollections();
+		sendObject(resp,colls);
+	    }
+	    else if( action.equals(DOCUMENT_SEARCH_ACTION) ){
+		Vector docNums = new Vector();
+		Vector descriptions = new Vector();
+		String engine;
+		
+		String query = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		System.out.println("Requesting..." + collection);
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+		
+		System.out.println("Retrieving with " + engine);
+		
+		String collection_home = COLLS_HOME+collection+File.separator;
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		
+		search_engine.search(query);
+		docNums = search_engine.getDocIdentifiers();
+		descriptions = search_engine.getDocDescriptions();
+		System.out.println(docNums.size() + " documents in VisServlet");
+		DataBlock data = generateDataBlock(collection_home, view, docNums, descriptions);
+		if (data != null) {
+		    response.addElement(data);
+		}
+		sendObject(resp,response);
+		search_engine = null;
+		data = null;
+		docNums = null;
+		descriptions = null;
+		
+	    }
+	    else if( action.equals(DOCUMENT_FETCH_ACTION) ){
+		String docNum = (String)q.elementAt(1);
+		String collection = (String)q.elementAt(2);
+		String view = null;
+		if (q.size() == 4) {
+		    view = (String)q.elementAt(3);
+		}
+		if (view != null && view.equals("")) {
+		    view = null;
+		}
+		String engine;
+		String collection_home = COLLS_HOME+collection+File.separator;
+		
+		try{
+		    engine = (String)collH.get(collection);
+		}catch(Exception ex){engine = "MG";}
+		
+                System.out.println("Retrieving with " + engine);
+		
+		SearchInterface search_engine = null;
+		if( engine.equals("MG") ){
+		    search_engine = new MGSearcher(collection_home, collection, view);
+		} else if( engine.equals("LUCENE") ){
+		    search_engine = new LUCSearcher(collection_home, view);
+		} 
+		System.err.println("Searching for doc "+docNum);
+		Vector doc = search_engine.getDocContent(Integer.parseInt(docNum));
+		response.addElement(doc);			
+		sendObject(resp,response);
+		search_engine = null;
+	    }
+	} catch(Exception e){}
+    }
+    
+
+    public Vector connect(Vector q)
+    {
+	return null;
+    }
+
+
+    private Vector activateCollections()
+    {
+	System.err.println("activating collections");
+	Vector colls = new Vector();
+	collH = new Hashtable();
+	Document doc = null;
+	
+	File dir = new File(COLLS_HOME);
+	if( !dir.exists() ){
+	    System.err.println("collections directory, "+COLLS_HOME+", doesn't exist, can't activate collections");
+	    return null;
+	}
+	
+	File[] list = dir.listFiles();
+	
+	for( int i = 0; i < list.length; i++ ){
+	    File f = list[i];
+	    if( f.isDirectory()){
+		File config_file = new File(f, "config.xml");
+		if (!config_file.exists()) {
+		    System.err.println("Collection "+f.getName()+" has no config file, ignoring");
+		    // ignore this collection
+		    continue;
+		} else {
+		    System.err.println("Trying to init collection "+f.getName());
+		}
+		try {
+		    DocumentBuilderFactory doc_build_fac = DocumentBuilderFactory.newInstance();
+		    DocumentBuilder doc_build = doc_build_fac.newDocumentBuilder();
+		    doc = doc_build.parse(config_file);
+		} catch (Exception e) {
+		    System.err.println("Error parsing config file "+config_file.getPath()+", ignoring this collection");
+		    continue;
+		}
+		Element root = doc.getDocumentElement();
+		String name = root.getAttribute("name");
+		String desc = root.getAttribute("description");
+		String engine = root.getAttribute("engine");
+		int size = Integer.parseInt(root.getAttribute("size"));
+		String coll_id = f.getName();
+		Element view_elem = (Element)XMLUtil.getChildByTagName(root, "viewList");
+		String [] views = null;
+		if (view_elem != null) {
+		    NodeList view_list = view_elem.getElementsByTagName("view");
+		    int num_views = view_list.getLength();
+		    if (num_views > 0) {
+			views = new String[num_views];
+			for (int j=0; j<num_views; j++) {
+			    String v = ((Element)view_list.item(j)).getAttribute("name");
+			    views[j]=v;
+			}
+		    }
+		}
+		Collection c = new Collection(coll_id,name,desc,size,engine,views);
+		System.out.println("Hashing.." + coll_id);
+		colls.addElement(c);
+		collH.put(coll_id,engine);
+	    }			
+	}
+	doc = null;
+	return colls;
+    }
+    
+    private DataBlock generateDataBlock(String collection_home, String view, Vector docNums, Vector descriptions) {
+
+	if (docNums.size()==0) {
+	    return null;
+	}
+	DataBlock data = new DataBlock();
+
+	CKServer ck_server = new CKServer(collection_home, view);
+	
+	/**** set data fields one by one and pass one what ever gets assembled ****/
+		
+	try{
+	    
+	    ck_server.setDescriptions(descriptions);
+	    
+	    /**** get candidate keywords ****/
+	    
+	    String[] keywords = ck_server.computeKeywords(docNums);
+	    
+	    data.words = keywords;
+	    
+	    
+	    /**** get sparse document * keyword matrix ****/
+	    
+	    SparseMatrix matrix = ck_server.getSparseMatrix(docNums);
+	    
+	    data.matrix = matrix;
+	    
+	    
+	    /**** get document indices, this is a subset of the original ****/
+	    /**** those without keywords are excluded                    ****/
+	    
+	    int[] docs = ck_server.getHitDocuments();
+	    
+	    data.docs = docs;
+	    
+	    
+	    String[] desc = ck_server.getHitDescriptions();
+	    
+	    data.descriptions = desc;
+	    
+	    
+	    /**** get 10 or so clusters ****/
+	    
+	    Vector[] clusters = ck_server.getClusters();
+	    
+	    data.clusters = clusters;
+	    
+	    
+	    double[][] centroids = ck_server.getCentroids();
+	    
+	    
+	    /**** send their centroids through sammon mapping ****/
+	    
+	    Point2D[] sammon = ck_server.getSammonMap(centroids);
+	    
+	    System.out.println("Num descriptions: " + descriptions.size());
+	    System.out.println("Num docs: " + docs.length);
+	    System.out.println("Num desc: " + desc.length);
+	    
+	    data.sammon = sammon;
+	    
+	} catch (Exception e) {
+	    System.err.println("VisServlet: computing clustering Error: "+e);
+	    e.printStackTrace();
+	}
+	ck_server = null;
+	return data;
+    }
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/browser/Browser.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/browser/Browser.java	(revision 21112)
@@ -0,0 +1,125 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.browser;
+
+import javax.swing.JFrame;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.applet.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.net.URL;
+import java.io.IOException;
+import java.util.*;
+
+public class Browser extends JFrame
+{
+  JScrollPane scrollPane;
+  JEditorPane editorPane;
+  JPanel panel;
+  public Browser(URL url)
+  {
+    jb_init(url);
+  }
+  
+  public Browser(Vector document, long docNumber)
+  {
+    jb_init(document);
+  }
+
+  private void jb_init(Vector document)
+  {
+     WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = new JEditorPane();
+    editorPane.setEditable(false);               
+    editorPane.setContentType("text/html");   
+    
+    String text = new String("");
+    
+    for( int i = 0; i < document.size(); i++ )
+    {
+        text += (String)document.elementAt(i) + "\n";
+    }
+    
+    editorPane.setText(text);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    
+    /*editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");*/
+    
+    setVisible(true);
+  }
+
+  private void jb_init(URL url)
+  {
+    WindowListener l = new WindowAdapter()
+    {
+      public void windowClosing(WindowEvent e)
+      {
+        setVisible(false);          // hide the Frame
+        dispose();
+      }
+    };
+    panel = new JPanel();
+    setSize(500,400);
+    Container content = this.getContentPane();
+    panel.setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+    panel.setLayout(new BorderLayout(10,10));
+    content.setLayout(new BorderLayout(10,10));
+    editorPane = createEditorPane(url);
+    scrollPane = new JScrollPane();
+    scrollPane.getViewport().add(editorPane);
+    panel.add(scrollPane,"Center");
+    content.add(panel,"Center");
+    setVisible(true);
+  }
+
+  private JEditorPane createEditorPane(URL url)
+  {
+    JEditorPane edPane = new JEditorPane();
+    edPane.setEditable(false);
+    displayURL(url,edPane);
+    return edPane;
+  }
+
+  private void displayURL(URL url, JEditorPane edPane)
+  {
+    try
+    {
+      edPane.setPage(url);
+    }
+    catch(IOException e)
+    {
+      System.out.println("Bad URL: "+ url.toString());
+    }
+  }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/AveLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class AveLink extends LinkFunc
+{
+    double link(double a, double b)
+	{
+  		return (a+b)/2;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Cluster.java	(revision 21112)
@@ -0,0 +1,120 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+public class Cluster
+{
+    public int _id;
+    public Cluster  _parent = null;
+    public Cluster  _child_1 = null;
+    public Cluster  _child_2 = null;
+    public Vector docVect;
+    public Vector docObjVect;
+    public int[] wordFreq;  	
+    
+    public double _children_sim;
+    
+    // probably the size of the cluster
+    public int _items;
+    
+    public int _split;
+    
+    
+    
+    // [] -> centroid for each vector element ?
+    public double[] _centroid=null;
+    
+    public Vector _iter_items = null;
+    public int _iter_items_num = 0;
+    
+    public int _x = 0;
+    public int _y = 0;
+    public double[] getCentroid(){return _centroid;}
+    
+    public Cluster(int id, Cluster parent, Cluster child_1, Cluster child_2, double children_sim, int items)
+    {
+	_id                    = id;
+	_children_sim          = children_sim;
+	_items                 = items;
+	_split                 = 0;
+	_parent                = parent;
+        _iter_items            = new Vector(0);
+        
+        // probably to get a balanced tree
+	if (child_1 != null && child_2 != null)
+	    {
+		if (child_2._items > child_1._items)
+		    {
+		        _child_1 = child_1;
+		        _child_2 = child_2;
+		    }
+    		else
+		    {
+		        _child_1 = child_2;
+		        _child_2 = child_1;
+		    }
+  	    }
+	else // if either or both are null
+	    {
+    		_child_1 = child_1;
+    		_child_2 = child_2;
+	    }
+    }
+    
+    public void setDocVect(Vector docs)
+    {
+	docVect = docs;
+    }
+    
+    public void setDocObjVect(Vector docObj)
+    {
+	docObjVect = docObj;
+    }
+    
+    public Vector getDocObjVect()
+    {
+	return docObjVect;
+    }
+    
+    public Vector getDocVect()
+    {
+	return docVect;
+    }
+    
+    public void setWordFreq(int[] freqs)
+    {
+	wordFreq = freqs;
+    }
+    
+    public int[] getWordFreq()
+    {
+	return wordFreq;
+    }
+    
+    // a nice recursive function to determine the depth of the hierarchy
+    public int depth()
+    {
+	if (_id != -1)
+	    return 0;
+	else
+    		return Math.max(1 + _child_1.depth(), 1 + _child_2.depth());
+	}
+
+    void addChildrenToOutputVector(int []coding, Vector output)
+	{
+  	  if (_id != -1)
+          {
+    	    Integer ii = new Integer(coding[_id]);
+            output.addElement(ii);
+
+          }
+  	  else
+    	  {
+      	    _child_1.addChildrenToOutputVector(coding,output);
+      	    _child_2.addChildrenToOutputVector(coding,output);
+    	  }
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterArray.java	(revision 21112)
@@ -0,0 +1,56 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class ClusterArray
+{
+    Vector contents;
+    
+    ClusterArray(DataManager dm, Vector indices)
+    {
+	int i;
+	
+	contents = new Vector();
+
+	// num_of_clusters = number of docs
+	for (i = 0; i < indices.size(); i++)
+	    {
+		int id = ((Integer)indices.elementAt(i)).intValue();
+		Cluster c = new Cluster(i, null, null, null, 0, 1);
+		Vector docIndexVector = new Vector();
+		docIndexVector.addElement(new Integer(id));
+		c.setDocVect(docIndexVector);
+		
+		Vector docObjVector = new Vector();
+		DocObj docObj = dm.getDocObjAt(id);
+		docObjVector.addElement(docObj);
+		c.setDocObjVect(docObjVector);    
+		
+		DataBlock data = dm.getDataBlock();
+		int[] wordFreqs = new int[data.words.length];
+		for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+		
+		SparseNode row = data.matrix.getRow(id);
+		
+		while( row!= null)
+		    {
+			wordFreqs[row.getColumn()]++;
+			row = row.getNext();
+		    }
+		c.setWordFreq(wordFreqs);
+		
+    		contents.addElement(c);
+		
+		// i.e. id = i, similarity between kids = 0, #items = 1
+	    }
+    }
+    
+    
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/ClusterObj.java	(revision 21112)
@@ -0,0 +1,171 @@
+package vishnu.testvis.dendro;
+
+import java.awt.*;
+import vishnu.testvis.object.*;
+import java.util.Vector;
+
+public class ClusterObj
+
+{
+    Cluster cluster;
+    Point position;
+    int radius;
+    boolean inSubCluster;
+    private boolean orderStatus;
+    private boolean keywordStatus;
+    private String[] freqWords;
+    
+    DocObj[] _docs;		// list of DocObj ref
+    String _label;		// (top 3-5? related words)
+    DataManager _data;
+    
+    public ClusterObj(Cluster c, Point P, int r, DataManager data)
+    {
+        cluster = c;
+        position = P;
+        radius = r;
+        inSubCluster = false;
+        _data = data;
+        orderStatus = false;
+        keywordStatus = false;
+    }
+    
+    
+    public void setPosition(Point p)
+    {
+        position = p;
+    }
+    
+    public void setOrderStatus(boolean b)
+    {
+        orderStatus = b;
+    }
+    
+    public boolean getOrderStatus()
+    {
+        return orderStatus;
+    }
+    
+    public void setFreqWords(String[] str)
+    {
+        freqWords = str;
+        keywordStatus = true;
+    }
+    
+    public String[] getFreqWords()
+    {
+        return freqWords;
+    }
+    
+    public boolean getKeywordStatus()
+    {
+        return keywordStatus;
+    }
+    
+    public void setIsSubset(boolean b)
+    {
+        inSubCluster = b;
+    }
+    
+    public boolean getIsSubset()
+    {
+        return inSubCluster;
+    }
+    
+    public boolean inCircle(int x, int y)
+    {
+        double dist_sqr = (x-position.x)*(x-position.x) + (y-position.y)*(y-position.y);
+        return (dist_sqr <= (4*radius*radius)) ? true : false;
+    }
+    
+    
+    public void setDocObjs(DocObj[] docArray)
+    {
+        _docs = docArray;
+    }
+    
+    
+    public int getX()
+    {
+        return position.x;
+    }
+    
+    public int getY()
+    {
+        return position.y;
+    }
+    
+    public int getRad()
+    {
+        return radius;
+    }
+   
+   
+    public ClusterObj(DocObj[] d, String t, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _data = dm;
+    }
+  
+  
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+  
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    
+    Vector rwVec = new Vector(s);
+    
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+  
+  
+  public int getDocSize()
+  {
+        return cluster._items;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+        
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Clustering.java	(revision 21112)
@@ -0,0 +1,248 @@
+package vishnu.testvis.dendro;
+
+/* Related word hierarchical clustering using a given triangle similarity matrix */
+
+//import cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Clustering
+{
+    public static final int COMPLETE_LINKAGE = 0;
+    public static final int AVERAGE_LINKAGE  = 1;
+    public static final int SINGLE_LINKAGE   = 2;
+
+    private Vector [] theResult;
+    private double [][] theCentroids;
+    private Cluster theRoot;
+    public DataManager dataManager;
+    private int method = 1;
+
+    public Vector [] getTheResults()
+    {
+        return theResult;
+    }
+    public double[][] getTheCentroids()
+    {
+        return theCentroids;
+    }
+    
+    public Cluster getRoot()
+    {
+        return theRoot;
+    }
+
+
+    public Clustering (DataManager dm, int linkage, float [][]m,
+		       int num_of_vectors,
+		       int arraySize, int min_c_items,
+		       int target_clusters, int limit)
+    {
+	
+        SimMatrix sim_matrix;
+	LinkFunc linkage_func=null;
+	dataManager = dm;
+	
+	// number of keywords
+	int VECTOR_SIZE = arraySize;
+        
+	Cluster toplevel_cluster;
+	TheInt clusters = new TheInt(0);
+	
+	Distance sim_func = new EuclideanDistance();
+	
+	double[] heights;
+	double [][] vectors = floatToDoubleMatrix(m);//intToDoubleMatrix(m);
+	//int [] coding = new int[m.length];
+        
+	// populate array with numbers from 0 to #docs-1
+	//for (int c=0;c<m.length;c++) coding[c]=c;
+	
+	int i, j;
+	
+	switch (linkage)
+	    {
+    	    case COMPLETE_LINKAGE:
+      		linkage_func = new MaxLink();
+      		break;
+    	    case AVERAGE_LINKAGE:
+      		linkage_func = new AveLink();
+      		break;
+    	    case SINGLE_LINKAGE:
+      		linkage_func = new MinLink();
+      		break;
+    	    default:
+      		System.out.println("clustering.c: main(): linkage type "+
+	                  linkage +" not defined.");
+      	        System.exit(0);
+    	}
+
+        long startTime = System.currentTimeMillis();
+        long readTime = System.currentTimeMillis();
+        
+        
+        // if fewer documents than upper cluster limit
+        // I think this is an error, should be vectors.length, viz. #documents
+        // which was never spotted as limit is always smaller
+        // than #keywords
+        // if(limit > vectors[0].length) limit = vectors[0].length;
+
+        // contains distance matrix between documents, built using
+        // the double vector[i][] list of keyword frequencies
+
+
+	Vector indices = null;
+	//randomly choose sqrt(k*n) documents
+
+	if( method == 0 )
+	    {
+
+		int sampleSize = 2*(int)Math.sqrt(vectors.length);
+		Vector sampleInt = new Vector();
+		Vector allDocs = new Vector();
+		
+		for( int s=0; s<vectors.length;s++ )
+		    {
+			allDocs.addElement(new Integer(s));
+		    }
+		
+		while( sampleInt.size() < sampleSize )
+		    {
+			int r = (int)(Math.random()*allDocs.size());
+			sampleInt.addElement(new Integer(r));
+			allDocs.removeElementAt(r);
+		    }
+		double[][] sample_matrix = new double[sampleSize][VECTOR_SIZE];
+		for(int s=0; s<sampleSize;s++)
+		    {
+			int docNum = ((Integer)sampleInt.elementAt(s)).intValue();
+			sample_matrix[s] = vectors[docNum];
+		    }
+		
+		indices = sampleInt;
+		sim_matrix = new SimMatrix(sample_matrix,limit,sim_func,VECTOR_SIZE);
+
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+		
+		for( int it = 0; it < allDocs.size(); it++ )
+		    {
+			int docNum = ((Integer)allDocs.elementAt(it)).intValue();
+			
+			double min_dist = 10000;
+			int min_index = -1;
+
+			for( int c = 0; c < content.size(); c++ )
+			    {
+				
+				Cluster clust = (Cluster)content.elementAt(c);
+
+				int docNums = ((Integer)clust.getDocVect().elementAt(0)).intValue();
+
+				double sim = sim_func.get(vectors[docNums],vectors[docNum], VECTOR_SIZE);
+
+				if (sim < min_dist)
+				    {
+					min_dist = sim;
+					min_index = c;
+				    }
+			    }
+			
+			Cluster clust = (Cluster)content.elementAt(min_index);
+			clust._iter_items.addElement(new TheInt(docNum));
+			clust._iter_items_num++;
+			clust.docVect.addElement(new Integer(docNum));
+			long iterativeTime = System.currentTimeMillis();
+			
+		    }
+	    }			 
+	else
+	    {
+   		Vector v = new Vector();
+		for( int l = 0; l < vectors.length; l++ )
+		    {
+			v.addElement(new Integer(l));
+		    }
+
+		indices = v;
+		sim_matrix = new SimMatrix(vectors, limit, sim_func, VECTOR_SIZE);
+	        
+		long simMatixTime = System.currentTimeMillis();
+		
+		// an array of heights sorted by descending disim
+		// as many heights as there are docs
+		heights = new double[m.length];//[limit - 1];
+		
+		//int maxDoc = 0;
+		int maxDoc = indices.size();
+		
+		Hierarchical hierarchical = new Hierarchical(maxDoc);
+		theRoot = hierarchical.clustering(dm, sim_matrix, maxDoc, heights,linkage_func,indices);
+		
+		// collect all clusters
+		Vector content=new Vector();
+		getLeaves(theRoot,content);
+	    }
+	
+    }
+
+    static double[][] intToDoubleMatrix(int [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+    static double[][] floatToDoubleMatrix(float [][] m)
+    {
+        int h=m.length;
+        double [][] res = null;
+        if (h > 0)
+        {
+          int w = m[0].length;
+          res = new double[h][w];
+          for(int c=0;c<h;c++)
+            for(int d = 0; d<w; d++)
+              res[c][d]=(double)m[c][d];
+        }
+        return res;
+   }
+
+
+    private void getLeaves(Cluster node, Vector content)
+    {
+	if( node._child_1 != null )
+	    {
+		getLeaves(node._child_1,content);
+		getLeaves(node._child_2,content);
+	    }
+	else
+	    {
+		content.addElement(node);
+		Vector v = node.docVect;
+		int i = ((Integer)v.elementAt(0)).intValue();
+	    }
+    }
+    
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroKeywordPanel.java	(revision 21112)
@@ -0,0 +1,102 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class DendroKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    DendroPanel parentPane;
+
+    public DendroKeywordPanel(Vishnu currentMain, DendroPanel parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+         _dataModel = new RW_DataModel();
+         _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                 refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+    }
+
+    public void refineEngine()
+    {
+        // the filter words are available from within _dataModel, so nothing
+       // is passed on
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+    }
+        
+
+    public void update(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/DendroPanel.java	(revision 21112)
@@ -0,0 +1,61 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import java.util.*;
+
+/** Panel containing the dendro map interface */
+public class DendroPanel extends JPanel
+{
+
+  public final static int LIST_ON_LEFT = 0;
+  public final static int LIST_ON_RIGHT = 1;
+  
+  public JPanel picturePanel;
+  public TreePanel treePanel;
+  public DendroKeywordPanel keywordPanel;
+  public HLinkPanel hLinkPanel;
+  public Stack dataStack;
+  public DataManager dataMan=null;
+  public Vishnu currentMain;
+  public JPanel topPanel = new JPanel();
+
+  public DendroPanel(Vishnu parent)
+  {
+    currentMain = parent;
+    setUpGui();
+  }
+  
+  public void setUpGui()
+  {
+        setLayout(new BorderLayout(10,10));
+        setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+        treePanel = new TreePanel(this,currentMain);
+        keywordPanel = new DendroKeywordPanel(currentMain,this);	
+	Dimension d = currentMain.getSize();
+        hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+        picturePanel = new JPanel();
+        picturePanel.setLayout(new BorderLayout(0,3));
+              
+        picturePanel.add(treePanel,"Center");
+        add("East",  hLinkPanel.linkPane);
+        add("Center",   picturePanel);
+        add("West",   keywordPanel);
+        setVisible(true);
+  }
+  
+  public void update()
+  {
+    dataMan = currentMain.dataManager;
+    dataStack = new Stack(); 
+    treePanel.clear();
+    treePanel.update();
+    keywordPanel.clear();
+    hLinkPanel.clear();
+  }
+}
+  
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Distance.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Distance.java	(revision 21112)
@@ -0,0 +1,10 @@
+package vishnu.testvis.dendro;
+
+// Abstract class
+
+public abstract class Distance
+{
+	public abstract double get(double[] v1, double[] v2, int size);
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/EuclideanDistance.java	(revision 21112)
@@ -0,0 +1,16 @@
+package vishnu.testvis.dendro;
+
+public class EuclideanDistance extends Distance
+{
+    public double get(double[] v1, double[] v2, int size)
+    {
+  	double sum = 0, x;
+   //System.out.println("Array length "+ v1.length);
+	for(int c=0;c<v1.length;c++)
+	{
+      	    x = v1[c] - v2[c];
+	    sum += x*x;
+	}
+  	return Math.sqrt(sum);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/HLinkPanel.java	(revision 21112)
@@ -0,0 +1,80 @@
+package vishnu.testvis.dendro;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        //linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Hierarchical.java	(revision 21112)
@@ -0,0 +1,356 @@
+package vishnu.testvis.dendro;
+
+import vishnu.cluster.*;
+import java.util.*;
+import vishnu.testvis.object.*;
+
+public class Hierarchical
+{
+    public double[] minrow;
+    public int[]    colind;
+    //public SimMatrix smatrix;
+    public TriangleIndex triangle;
+    public final double weight = 3.2;
+    public DataManager dataManager;
+
+    public Hierarchical(int matrix_rows)
+    {
+	// create Triangle with "limit" arg = maximum #clusters
+	// an object to help with indexing the cells 
+	// in a triangular matrix
+	triangle = new TriangleIndex(matrix_rows);
+	
+    }
+    
+    // set each minrow[i] to the value of the minimum element of that row
+    void set_minrow (SimMatrix m, int row)
+    {
+	// triangular matrix without diagonal, hence zeroth row is all zero
+        if(row == 0) return;
+	
+	int i;
+	int index_start, index_end;
+        
+        // first column
+	index_start = triangle.index (row, 0);
+	// last column (row(!)-1 because as many rows as columns)
+	index_end   = triangle.index (row, row - 1); 
+	
+	minrow[row] = m.matrix[index_start];
+	colind[row] = index_start;
+	
+    	for (i = index_start + 1; i <= index_end; i++)
+	    {
+      		if (m.matrix[i] < minrow[row])
+		    {
+			minrow[row] = m.matrix[i];
+			colind[row] = i;
+		    }
+	    }
+    }
+    
+    // return the ROW in which the minimum disimilarity resides
+    int get_minrow (int last_row)
+    {
+	int i;
+	int row;
+	double value;
+	
+	row   = 1;
+	value = minrow[1];
+	
+	for (i = 2; i <= last_row; i++)
+	    {
+      		if (minrow[i] < value)
+		    {
+			row = i;
+			value = minrow[i];
+		    }
+	    }
+	
+	return row;
+    }
+    
+    // num_of_rows is number of documents, or subsample for Buckshot
+    // indices is a vector of the doc ids (made necessary by Buckshot, which uses a subsample of all docs)
+    Cluster clustering (DataManager dm, SimMatrix sim_matrix, int num_of_rows,
+			double[] heights, LinkFunc linkage_func, Vector indices)
+    {
+	Cluster child_1;
+	Cluster child_2;
+	Cluster parent=null;
+	ClusterArray cluster_ary;
+	
+	
+        int min_row, index, row, col, i;
+	double sim;
+
+        int last_row_index, sim_matrix_size;
+
+        sim_matrix_size = num_of_rows *(num_of_rows - 1) / 2;
+  		
+	// Place each of the documents in a cluster of each own 
+	// there are vector.length docs but only num_of_rows = limit places in the array
+	// may be two-stage clustering process, i.e. cluster subset of docs and then add the rest
+	//	cluster_ary = new ClusterArray (dm, num_of_rows);
+
+	cluster_ary = new ClusterArray(dm,indices);
+	last_row_index = num_of_rows - 1;
+	
+	// a variable that is a double and exists for each row - what can this be?
+	// the lowest dissimilarity score?
+	minrow = new double[num_of_rows];
+	
+	// a variable that is an int and exists for each row - what can that be?
+	// the document index of the document with that lowest score?
+	colind = new int[num_of_rows];
+	
+  	    for (i = 1; i < num_of_rows; i++)
+        {
+            set_minrow(sim_matrix, i);
+        }
+        
+        // do all the hierarchical clustering by running over all rows, updating the matrix 
+        // as a new cluster is formed each time
+  	    for (i = 0; i < num_of_rows - 1; i++)
+        {
+                // find the row with the lowest disimilarity
+      	        min_row = get_minrow(last_row_index);
+      	        sim = minrow[min_row];
+      	        
+      	        // the column with the lowest dis... (sim) in that row
+      	        index  = colind[min_row];
+
+                // array of heights, DESCENDING disim, i.e. sim goes last
+      	        heights[num_of_rows - 2 - i] = sim; 
+
+      	        row = triangle.i_ind[index];
+      	        col = triangle.j_ind[index];
+      	        
+      	        newMerge(row,col,sim_matrix,last_row_index,linkage_func,cluster_ary);
+      	            
+      	        sim_matrix_size -= last_row_index;
+      	        
+      	        child_1 = (Cluster)cluster_ary.contents.elementAt(row);
+      	        child_2 = (Cluster)cluster_ary.contents.elementAt(col);
+
+      	        parent = new Cluster (-1, null, child_1, child_2, sim, child_1._items + child_2._items);
+      	        Vector vDoc = new Vector();
+      	        vDoc.addAll(child_1.getDocVect());
+      	        vDoc.addAll(child_2.getDocVect());
+      	        parent.setDocVect(vDoc);
+      	        
+      	        Vector vDocObj = new Vector();
+      	        vDocObj.addAll(child_1.getDocObjVect());
+      	        vDocObj.addAll(child_2.getDocObjVect());
+      	        parent.setDocObjVect(vDocObj);
+      	        
+      	        int[] wordFreq1 = child_1.getWordFreq();
+      	        int[] wordFreq2 = child_2.getWordFreq();
+      	        for( int z = 0; z < wordFreq1.length; z++ )
+      	        {
+      	            wordFreq1[z] += wordFreq2[z];
+      	        }
+      	        parent.setWordFreq(wordFreq1);
+      	        
+      	        child_1._parent = parent;
+      	        child_2._parent = parent;
+      	        cluster_ary.contents.setElementAt(cluster_ary.contents.elementAt(last_row_index),row);
+      	        cluster_ary.contents.setElementAt(parent,col);
+      	        last_row_index--;
+         	        
+    	    }
+
+  	    return parent;
+
+        }
+        
+    void newMerge(int r, int c, SimMatrix m, int lastRowIndex, LinkFunc linkage_func, ClusterArray clusterArr)
+    {
+        int n = lastRowIndex+1;
+        double newMatrix[][] = new double[n][n];
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( i != j ) newMatrix[i][j] = (double)((int)(m.matrix[triangle.index(i,j)]*100))/100;
+                    else newMatrix[i][j] = 0;
+            }
+        }
+        
+        for( int i = 0; i < n; i++ )
+        {
+            int sizeA =((Cluster)clusterArr.contents.elementAt(c))._items + ((Cluster)clusterArr.contents.elementAt(r))._items;
+            int sizeB = ((Cluster)clusterArr.contents.elementAt(i))._items;
+            // large if cluster differ in size
+            double scale = weight * (double)((sizeA+sizeB)/2*(sizeA+sizeB)/2)/(sizeA*sizeA+sizeB*sizeB);
+            
+            newMatrix[c][i] = scale*linkage_func.link(newMatrix[c][i],newMatrix[r][i]);            
+            
+        }
+        
+        newMatrix[c][c] = 0;
+        newMatrix[c][r] = newMatrix[c][n-1];
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][c] = newMatrix[c][i];
+        }
+        
+        
+        for( int i = 0; i < n; i++ )
+        {
+            if( i != c ) newMatrix[r][i] = newMatrix[n-1][i];
+        }
+        newMatrix[r][r] = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            newMatrix[i][r] = newMatrix[r][i];
+        }
+        
+        int index = 0;
+        for( int i = 0; i < n; i++ )
+        {
+            for( int j = 0; j < n; j++ )
+            {
+                if( j < i )
+                {
+                    m.matrix[index] = newMatrix[i][j];
+                    index++;
+                }
+          
+            }
+          
+       }
+       
+       for (int i = 1; i < n; i++)
+       {
+            set_minrow(m, i);
+       }
+        
+     
+    }
+
+    // form a new cluster from the two most similar documents
+    void merge (int i, int j, SimMatrix m, int n, LinkFunc linkage_func)
+	{
+  /* i and j are the matrix elements with maximum similarity         */
+  /* matrix is the triangle similarity matrix with last row index n  */
+  /* linkage_func is a pointer to the linkage function               */
+
+  /* i > j */
+
+  	    int c, k, index1, index2;
+
+        // go over all rows
+  	    for (k = 0; k <= n; k++)
+    	{
+    	    // if k < j the affected elements are on row j
+      		if (k < j)
+		    {
+		        // index1 is the smaller of the two, as j < i
+	     	    index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+        	}
+      		else if (k > j && k < i)
+		    {
+	            // if j < k < i then only one element is updated per row
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+    	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind
+	      		    if (m.matrix[index1] < minrow[k])
+        			{
+		          	    minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      		else if (k > i && k < n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  	        index2 = triangle.index (k, i);
+	  	        m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  
+          	    if (index1 == colind[k])
+	    	    {
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+	  		            minrow[k] = m.matrix[index1];
+                        colind[k] = index1;
+			        }
+	    	    }
+
+	  	        index1 = triangle.index (k, i);
+	  	        index2 = triangle.index (k, n);
+	  	        m.matrix[index1] = m.matrix[index2];
+	  	  	    if (index1 == colind[k])
+	    	    {
+	            // the previous minimum of the row has been changed so recaculate minimum of whole row 
+	      		    set_minrow(m, k);
+	    	    }
+	  	        else
+	    	    {
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      		    if (m.matrix[index1] < minrow[k])
+			        {
+		  	            minrow[k] = m.matrix[index1];
+		  	            colind[k] = index1;
+			        }
+	    	    }
+		    }
+      	    else if (k == n  && i != n)
+		    {
+	  	        index1 = triangle.index (k, j);
+	  			index2 = triangle.index (k, i);
+	  			m.matrix[index1] = linkage_func.link(m.matrix[index1], m.matrix[index2]);
+	  			if (index1 == colind[k])
+	    		{
+	                // the previous minimum of the row has been changed so recaculate minimum of whole row */
+	      			set_minrow(m, k);
+	    		}
+	  			else
+	    		{
+	                // if the changed element is smaller than the current minimum update minrow and colind */
+	      			if (m.matrix[index1] < minrow[k])
+					{
+		  				minrow[k] = m.matrix[index1];
+		  				colind[k] = index1;
+					}
+	    		}
+
+	   			for (c = 0; c < i; c++)
+	    		{
+	      			index1 = triangle.index (c, i);
+	      			index2 = triangle.index (c, n);
+	      			m.matrix[index1] = m.matrix[index2];
+	      		}
+			}
+						
+    	}
+  		set_minrow(m, i);
+  		set_minrow(m, j);
+	}
+
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/Iterative.java	(revision 21112)
@@ -0,0 +1,111 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+class Iterative
+{
+    Vector vector;
+    int VECTOR_SIZE;
+    
+    Iterative(int vsize)
+    {
+	VECTOR_SIZE = vsize;
+    }
+    
+    public void printVector(double[] a)
+    {
+        int i;
+
+	for (i = 0; i < VECTOR_SIZE; i++)
+    	    System.out.println("v[" + i + "] = " + a[i]);
+    }
+    
+    
+    void cluster_center (Cluster cluster, double [][] vectors, double[] centroid)
+    {
+        int i;
+	
+      	if (cluster._id == -1)
+	    {
+		cluster_center (cluster._child_1, vectors, centroid);
+		cluster_center (cluster._child_2, vectors, centroid);
+	    }
+        else
+	    {
+		double[] doubleArray = vectors[cluster._id];
+		for (i = 0; i < doubleArray.length; i++)
+	            centroid[i] += doubleArray[i];
+	    }
+    }
+    
+    void compute_cluster_centers (ClusterArray clusters, int num_of_clusters, double [][] vectors)
+    {
+	int i, j; double ssum;
+	Cluster clust;
+	
+	for (i = 0; i < num_of_clusters; i++)
+	    {
+		clust = (Cluster)clusters.contents.elementAt(i);
+	        clust._centroid = new double[VECTOR_SIZE];
+		cluster_center (clust, vectors, clust._centroid);
+		for (ssum = j = 0; j < VECTOR_SIZE; j++)
+		    ssum += clust._centroid[j] * clust._centroid[j];
+		if(ssum>0)                // normalise
+		    for(ssum = 1.0/Math.sqrt(ssum), j = 0; j < VECTOR_SIZE; j++)
+			clust._centroid[j] *= ssum;
+	    }
+    }
+    
+
+    void place_item (ClusterArray clusters,
+		     int num_of_clusters,
+		     double [][] vectors,
+		     int item_id,  Distance sim_func)
+    {
+	double max_sim, sim;
+	int max_sim_index;
+	
+	int i;
+	
+	max_sim_index = 0;
+        Cluster clust = (Cluster)clusters.contents.elementAt(0);
+	max_sim = sim_func.get(clust._centroid,
+			       vectors[item_id], VECTOR_SIZE);
+	
+  	    for (i = 1; i < num_of_clusters; i++)
+    	{
+            clust = (Cluster)clusters.contents.elementAt(i);
+      	    sim = sim_func.get(clust._centroid,
+			vectors[item_id], VECTOR_SIZE);
+
+            if (sim < max_sim)
+	        {
+	  	        max_sim_index = i;
+	  	        max_sim = sim;
+	        }
+    	}
+  	    
+  	    clust = (Cluster)clusters.contents.elementAt(max_sim_index);
+        clust._iter_items.addElement(new TheInt(item_id));
+	    clust._iter_items_num++;
+    }
+
+    // add all documents not yet added to the existing cluster hierarchy (which is based on the 
+    // limit first documents. Hence there are vector.length - limit docs to add
+    // consequently index runs from limit (start_index) to vector.length (end_index)
+    void clustering (ClusterArray clusters, TheInt num_of_clusters,
+                      double[][] vectors, int start_index, int end_index,
+                      Distance sim_func)
+    {
+        int i;
+    
+  	    if (start_index >= end_index)
+    	    	return;
+
+  	    compute_cluster_centers(clusters, num_of_clusters.val, vectors);
+
+  	    for (i = start_index; i < end_index; i++)
+      	    	place_item(clusters, num_of_clusters.val, vectors, i, sim_func);
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/LinkFunc.java	(revision 21112)
@@ -0,0 +1,8 @@
+package vishnu.testvis.dendro;
+
+//Abstract linkfunctions class
+
+public abstract class LinkFunc
+{
+	abstract double link(double a, double b);
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MaxLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MaxLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a > b) ? a : b;      //*99+b: b*99+a)/100;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/MinLink.java	(revision 21112)
@@ -0,0 +1,9 @@
+package vishnu.testvis.dendro;
+
+public class MinLink extends LinkFunc
+{
+	public double link(double a, double b)
+	{
+  		return (a < b? a*999+b: b*999+a)/1000;
+	}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/QSortForDocObj.java	(revision 21112)
@@ -0,0 +1,76 @@
+package vishnu.testvis.dendro;
+
+import vishnu.testvis.object.*;
+
+public class QSortForDocObj
+{
+
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+  
+   void QuickSort(DocObj a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+           mid = a[ ( lo0 + hi0 ) / 2 ].getNumber();
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].getNumber() < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].getNumber() > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(DocObj a[], int i, int j)
+   {
+      DocObj D;
+      D = a[i];
+      a[i] = a[j];
+      a[j] = D;
+   }
+
+   public void sort(DocObj a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/SimMatrix.java	(revision 21112)
@@ -0,0 +1,46 @@
+package vishnu.testvis.dendro;
+
+import java.util.*;
+
+// kainof DistanceMatrix
+public class SimMatrix
+{
+  public double[] matrix;
+
+  // "vectors" is the expanded SparseMatrix, i.e. doc x keywords
+  // VECTOR_SIZE = #keywords
+  public SimMatrix(double[][] vectors,
+      int limit, Distance simfunc, int VECTOR_SIZE)
+  {
+    int i,j;
+    double iarray[];
+    
+    // size = #docs
+    int size=vectors.length, index=0;
+    
+    // to exclude diagonal and conjugates
+    // - - - - -
+    //         |
+    // *       |
+    // * *     |
+    // * * *   |    
+    int realSize = (size * (size - 1)/2);
+    matrix=new double[realSize];
+    System.out.println("Similarity matrix size: " + size);
+    
+    for(i = 1; i < size; i++)
+    {
+        iarray=vectors[i];
+        for(j = 0; j < i; j++)
+        {
+            matrix[index] = simfunc.get(iarray,
+                            vectors[j],
+                            VECTOR_SIZE);
+	        index++;
+        }
+    }
+  }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheInt.java	(revision 21112)
@@ -0,0 +1,17 @@
+package vishnu.testvis.dendro;
+
+public class TheInt
+{
+	public int val;
+
+	TheInt(int x)
+	{
+		val = x;
+	}
+	
+	public void setInt(int i)
+	{
+	    val = i;
+	}
+	
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TheVectors.java	(revision 21112)
@@ -0,0 +1,79 @@
+package vishnu.testvis.dendro;
+
+import java.io.*;
+import java.util.*;
+
+public class TheVectors
+{
+    public Vector theData;
+    public int size;
+    public Vector coding;
+    public int zeros;
+
+
+    TheVectors(String file, int VECTOR_SIZE)
+    {
+        theData= new Vector(VECTOR_SIZE);
+	coding= new Vector(VECTOR_SIZE);
+        int index,code =0, zerolist=0;
+	double ssum,value;
+    	try
+	{
+      	    FileReader inFile = new FileReader(file);
+      	    BufferedReader inLines = new BufferedReader(inFile);
+	    size=0;
+            String line = null, tokenString;
+	    do
+	    {
+
+		line = inLines.readLine();
+		if (line!= null)
+		{
+
+		  double[] row= new double[VECTOR_SIZE];
+ 					StringTokenizer tokens = new StringTokenizer(line," ");
+                  ssum=0.0;
+   		  for (index=0; index<VECTOR_SIZE; index++)
+		  {
+                      if(tokens.hasMoreTokens())
+                      {
+                          tokenString = tokens.nextToken();
+           	          value = Double.parseDouble(tokenString);
+		          row[index]= value;
+		          ssum+= value*value;
+                      }
+                      else
+                      {
+                          row[index]=0.0;
+                      }
+	          }
+	          if(index==0) continue;
+                  ssum=Math.sqrt(ssum);
+	          if(ssum!=0.0)
+	          {
+	              for(int i=0;i<index;i++)
+		      row[i]/=ssum;
+		      theData.addElement(row);
+		      coding.addElement(new TheInt(code));
+                      size++;
+		  }
+		  else
+		  {
+		      if(zerolist==0)
+		      {
+			//System.out.print("0 ");
+	                  zerolist=1;
+		      }
+		      System.out.print(code + " ");
+		  }
+	      }
+	      code++;
+	  }while(line != null);
+	}
+	catch(IOException e)
+	{
+	    System.err.println("IO Exception in read_vectors");
+	    e.printStackTrace();
+        }
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TreePanel.java	(revision 21112)
@@ -0,0 +1,822 @@
+package vishnu.testvis.dendro;
+
+import java.awt.image.*;
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.sammon.*;
+import vishnu.datablock.*;
+
+
+public class TreePanel extends JPanel
+{
+    
+    DendroPanel parent;
+    Vishnu currentMain;
+    DataManager dataManager;
+    DataBlock data;
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics offGraphics;
+    Graphics2D offGraphics2D;
+    Vector clusterVect = null;
+    Cluster root = null;
+    Cluster selectedRoot = null;
+    ClusterObj selectedCluster = null;
+    ClusterObj oldCluster = null;
+    
+    int selectedClusterID = -1; 
+    int oldClusterID = -1;
+    JPopupMenu popup;
+    boolean constructed = false;
+    int selectedDepth = -1;
+    int initBranchLength = 85;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    public TreePanel(DendroPanel parent, Vishnu frame)
+    {
+        currentMain = frame;
+        this.parent = parent;
+        dataManager = frame.dataManager;
+             
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+        
+        this.addComponentListener( new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+	    {
+		repaint();
+            }
+        }
+        );
+        
+        this.addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int x = mouseEvent.getX();
+                int y = mouseEvent.getY();
+                
+                if( clusterVect != null )
+                {
+                    
+                    oldClusterID = selectedClusterID;
+                    oldCluster = selectedCluster;
+                    selectedClusterID = -1;
+                    selectedCluster = null;
+                    
+                    boolean notFound = true;
+                    int i = 0;
+                    
+                    while( notFound && (i < clusterVect.size()) )
+                    {
+                       ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                       
+                       if( co.inCircle(x,y) )
+                       {
+                            selectedCluster = co;
+                            selectedClusterID = i;
+                                        
+                            if( selectedClusterID != oldClusterID )
+                            {
+                                 repaint();
+                            }
+                            notFound = false;
+                       }
+                       i++;
+                    }
+                    
+                    if( (oldClusterID != selectedClusterID) && ( oldClusterID != -1 ) )
+                    {
+                        popup.setVisible(false);
+                        repaint();
+			//update();
+                    }
+                    
+                    if( selectedClusterID != -1 )
+                    {
+                        if( selectedClusterID != oldClusterID )
+                        {
+                            popup.removeAll();
+                            int size = selectedCluster.cluster._items;
+                            String str =  size + ((size > 1) ? " documents" : " document");
+                            JMenuItem menuItem = new JMenuItem(str);
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent actionEvent)
+                                {
+                                    getDocObjs();
+                                    String[] drillWords = selectedCluster.getFreqWords();
+                                    String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+
+                                    updateHLinkPanel();                                
+                                    updateKeyWordTable();
+                                }
+                                
+                            });
+                            
+                            popup.add(menuItem);
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            
+                            int mode0 = -1;
+                            
+                            // if the currentRoot is selected, dont show "drilldown"
+                            if( selectedClusterID == 0 )
+                            {
+                                // if we are at the top, dont show "undrill"
+                                if( ((Object)selectedCluster.cluster).equals((Object)root) )
+                                {
+                                    str = "Top of hierarchy";
+                                    mode0 = 2;
+                                }
+                                else
+                                {
+                                    str = "Sequential undrill";
+                                    mode0 = 1;
+                                }
+                            }
+                            // if somewhere else in the tree
+                            else
+                            {
+                                if( selectedCluster.cluster._items > 1 )
+                                {
+                                    str = "Drill down";
+                                    mode0 = 0;
+                                }
+                                else
+                                {
+                                    str = "Outermost twigs";
+                                    mode0 = 3;
+                                }
+                            }
+                            
+                            final int mode = mode0;
+                            
+                            menuItem = new JMenuItem(str);
+                            if( (mode != 0 ) && (mode != 1) )
+                                menuItem.setEnabled(false);
+                                
+                            menuItem.addActionListener(new ActionListener()
+                            {
+                                public void actionPerformed(ActionEvent evt)
+                                {
+                                    if( mode == 0 )
+                                    {
+                                        selectedRoot = selectedCluster.cluster;
+                                        String[] drillWords = selectedCluster.getFreqWords();
+                                        String keyWordStr = new String(drillWords[0]+" "+drillWords[1]+" "+drillWords[2]);
+                                        
+                                        // to establish new clusterobj vector
+                                        constructed = false;   
+                                        repaint();
+					//update();                                  
+                                    }
+                                    if( mode == 1 )
+                                    {
+                                        selectedRoot = selectedRoot._parent;
+                                        constructed = false;
+                                        repaint();
+					//update();
+                                    }
+                                }
+                            });
+                            popup.add(menuItem);
+                            
+                            if( mode == 1 )
+                            {
+                                menuItem = new JMenuItem("Total undrill");
+                                menuItem.addActionListener(new ActionListener()
+                                {
+                                    public void actionPerformed(ActionEvent evt)
+                                    {
+                                        selectedRoot = root;
+                                        constructed = false;
+                                        //update();
+					repaint();
+                                    }
+                                });
+                                popup.add(menuItem);
+                            }
+                            
+                            popup.add(new JPopupMenu.Separator());
+                            int[] wordFreq = selectedCluster.cluster.getWordFreq();
+                            String[] words = getMostFrequentWords(wordFreq);
+                            
+                            selectedCluster.setFreqWords(words);
+                            
+                            menuItem = new JMenuItem(words[0]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[1]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            menuItem = new JMenuItem(words[2]);
+                            menuItem.setEnabled(false);
+                            popup.add(menuItem);
+                            
+                            popup.validate();
+                            popup.setVisible(true);
+                        }
+                        
+                        popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                        mouseEvent.consume();
+                    }     
+                }  // of clustervect != null
+             } // of mouse moved
+        });
+        
+        setVisible(true);
+    }
+    
+    public void clear()
+    {
+        constructed = false;
+    }
+
+        
+    public String[] getMostFrequentWords(int[] wordFreq)
+    {
+        
+        
+        int max1Freq = 0;
+        int max1Index = 0;
+        int max2Freq = 0;
+        int max2Index = 0;
+        int max3Freq = 0;
+        int max3Index = 0;
+        
+        for( int i = 0; i < wordFreq.length; i++ )
+        {
+            if( wordFreq[i] > max3Freq )
+            {
+                if( wordFreq[i] > max2Freq )
+                {
+                    if( wordFreq[i] > max1Freq )
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = max1Freq;
+                        max2Index = max1Index;
+                        max1Freq = wordFreq[i];
+                        max1Index = i;
+                    }
+                    else
+                    {
+                        max3Freq = max2Freq;
+                        max3Index = max2Index;
+                        max2Freq = wordFreq[i];
+                        max2Index = i;
+                    }
+                }
+                else
+                {
+                    max3Freq = wordFreq[i];
+                    max3Index = i;
+                }
+            }
+                
+        }
+        
+        data = dataManager.getDataBlock();
+        String[] words = new String[3];
+        words[0] = data.words[max1Index];
+        words[1] = data.words[max2Index];
+        words[2] = data.words[max3Index];
+        return words;
+    }
+    
+    public void getDocObjs()
+    {
+       
+       Vector docVect = selectedCluster.cluster.getDocVect();
+       DocObj[] docs = new DocObj[docVect.size()];
+       
+       for( int i = 0; i < docVect.size(); i++ )
+       {
+            int index = ((Integer)docVect.elementAt(i)).intValue();
+            docs[i] = dataManager.getDocObjAt(index);
+       }    
+       
+       
+       if( !selectedCluster.getOrderStatus() )
+       {
+            QSortForDocObj qSort = new QSortForDocObj();
+            try
+            {   
+                 qSort.sort(docs);
+            }
+            catch (Exception e)
+            {}
+            
+            selectedCluster.setDocObjs(docs);
+            selectedCluster.setOrderStatus(true);
+       }
+       
+        
+    }
+    
+    public void updateHLinkPanel()
+    {
+       
+        String htmlDescription = selectedCluster.formatDocs();
+        parent.hLinkPanel.setText(htmlDescription);
+    }
+    
+    public void updateKeyWordTable()
+    {
+        parent.keywordPanel.update(selectedCluster);
+    }
+    
+    
+    public void setConstructed(boolean b)
+    {
+        constructed = b;
+    }
+    
+    
+    public void getData()
+    {
+        dataManager = currentMain.dataManager;
+        data = dataManager.getDataBlock();
+        SparseMatrix m = data.matrix;
+        
+        Vector idVect = new Vector();
+        Vector sammCluster =  dataManager.getSammonClusters();
+        
+        for( int i = 0; i < sammCluster.size(); i++ )
+        {
+            SammCluster cluster = (SammCluster)sammCluster.elementAt(i);
+            idVect.addAll(cluster.getIDs());
+        }
+        
+        for( int c = 0; c < idVect.size(); c++ )
+        {
+            Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+        }
+        
+        //int [][] full = m.getExpanded(); 
+        float [][] full = m.getExpanded(); 
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+               
+        int limit =  m.getRowCount();
+        
+        int clusterNum = 10;
+                      
+        Clustering clustering = new Clustering(dataManager,Clustering.COMPLETE_LINKAGE,
+					       full,m.getRowCount(),
+					       m.getColumnCount(),1,
+					       clusterNum,limit);
+          
+        root = clustering.getRoot();
+        selectedRoot = root;
+    }
+    
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	
+	d = getSize();
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics2D = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+		
+	if( !constructed )
+	    {
+		clusterVect = new Vector();
+		selectedCluster = null;
+		selectedClusterID = -1;
+	    }
+	
+	int initRad = 0;  
+	initBranchLength = (int)(Math.min(d.width,d.height)*0.12);
+	
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed ){
+	    ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+	    clusterVect.addElement(rootObj);            
+	}
+	else{
+	    ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+	    c.setPosition(center);
+	    clusterVect.setElementAt(c,0);
+	    c = (ClusterObj)clusterVect.elementAt(0);
+	}
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) )
+                {
+                    selected = true;                    
+                    selectedDepth = 0;
+                    offGraphics2D.setColor(Color.blue);
+                }
+
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+		}
+	    
+	    g2.drawImage(bi, 0, 0, this);
+	constructed = true;
+    }
+
+           
+    public void update()
+    {
+	
+	if( !constructed )
+            {
+                clusterVect = new Vector();
+                selectedCluster = null;
+                selectedClusterID = -1;
+            }
+	
+	Dimension d = getSize();
+	offScreenSize = d;
+	if( d.width <= 0 ) return;
+
+	offScreen = createImage(d.width,d.height);
+	offGraphics = offScreen.getGraphics();
+            
+	Graphics2D offGraphics2D = (Graphics2D)offGraphics;
+	
+	offGraphics2D.setColor(Color.white);
+	offGraphics2D.fillRect(0,0,d.width,d.height);
+	offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	offGraphics2D.setStroke(myStroke);	        
+	
+	int initRad = 0;  
+
+	Cluster cc = selectedRoot;
+	Point center = new Point(d.width/2,d.height/2);
+	int radius = (int)(Math.log(cc._items))* 3;
+	if( radius < 2 ) radius = 2;
+	
+	// all the leaf cluster ids underneath selected node
+	// to be painted bright red
+	Vector allSubNodeIDs = new Vector();
+	
+	if( !constructed )
+            {
+                ClusterObj rootObj = new ClusterObj(cc,center,radius,dataManager);
+                clusterVect.addElement(rootObj);            
+            }
+	else
+            {
+                ClusterObj c = (ClusterObj)clusterVect.elementAt(0);
+                c.setPosition(center);
+                clusterVect.setElementAt(c,0);
+                c = (ClusterObj)clusterVect.elementAt(0);
+            }
+	
+	int depth = 1;
+	TheInt ii = new TheInt(1);
+	
+	boolean selected = false;
+	
+	// used to paint the tree different colour under selected node
+	selectedDepth = -1;
+	
+	if( cc._items > 1 ){
+	    double alpha = (double)Math.PI/Math.pow(2,depth);
+	    double theta = 0;
+	    
+	    if( (selectedClusterID == 0) ){
+		selected = true;                    
+		selectedDepth = 0;
+		offGraphics2D.setColor(Color.blue);
+	    }
+	    
+	    paintPoke(offGraphics2D,cc._child_1,center.x,center.y,center,depth,theta,-alpha,initRad, selected, allSubNodeIDs, ii);
+	    paintPoke(offGraphics2D,cc._child_2,center.x,center.y,center,depth,theta,alpha,initRad, selected, allSubNodeIDs, ii);
+	}
+	
+	for( int i = 0; i < clusterVect.size(); i++ )
+            {
+                
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(i);
+                
+                // if leaf draw reddish
+                if( co.cluster._items == 1 )
+		    {
+			int r = co.getRad();
+                    offGraphics2D.setColor(new Color((float)1, (float)0.7, (float)0.7));
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else  // draw blueish
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(new Color((float)0.7, (float)0.7, (float) 1));
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                               
+            }
+            
+            for( int i = 0; i < allSubNodeIDs.size(); i++ )
+            {
+                int value = ((Integer)allSubNodeIDs.elementAt(i)).intValue();
+                ClusterObj co = (ClusterObj)clusterVect.elementAt(value);    
+                // if leaf
+                if( co.cluster._items == 1 )
+                {
+		    int r = co.getRad();
+                    offGraphics2D.setColor(Color.red);
+                    offGraphics2D.fillArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                    offGraphics2D.setColor(Color.black);
+                    offGraphics2D.drawArc(co.getX()-5,co.getY()-5,2*r,2*r,0,360);    
+                }
+                else
+                {
+                    offGraphics2D.setColor(Color.white);
+                    int r = co.getRad();
+                    offGraphics2D.fillArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                    offGraphics2D.setColor(Color.blue);
+                    offGraphics2D.drawArc(co.getX()-r,co.getY()-r,2*r,2*r,0,360);
+                }
+                
+                
+            }
+            
+            if( (selectedCluster != null)  && (selectedCluster.cluster._items > 1) )
+            {
+                offGraphics2D.setColor(Color.blue);
+                offGraphics2D.fillArc(selectedCluster.getX()-selectedCluster.getRad(),selectedCluster.getY()-selectedCluster.getRad(),2*selectedCluster.getRad(),2*selectedCluster.getRad(),0,360);
+            }
+
+           repaint();
+           constructed = true;
+    }
+    
+    // theta is the angle in the polar representation
+    public void paintPoke(Graphics2D g, Cluster child, int x, int y, Point center, int d, double theta, double alpha,int r, boolean selected, Vector allSubNodeIDs, TheInt ii)
+    {
+	// translate parent cluster position to center
+	int x1 = x - center.x;
+	int y1 = y - center.y;
+	
+	// rotate cluster about origin by alpha to obtain intermediate tree point
+	int x_inter = (int)(x1 * Math.cos(alpha) + y1 * Math.sin(alpha)) + center.x;
+	int y_inter = (int)(x1 * - Math.sin(alpha) + y1 * Math.cos(alpha)) + center.y;
+
+	// rotate cluster about origin by theta
+	int x2 = (int)(x1 * Math.cos(-theta) + y1 * Math.sin(-theta));
+	int y2 = (int)(x1 * - Math.sin(-theta) + y1 * Math.cos(-theta));
+	
+	// extend radius
+	int dRad = (initBranchLength-d*10);
+	if( dRad <= 0 ) dRad = 10;
+	int x3 = x2 + dRad;
+	int y3 = y2;
+
+	// rotate back by (theta +/- alpha)
+	int x4 = (int)(x3 * Math.cos(theta+alpha) + y3 * Math.sin(theta+alpha));
+	int y4 = (int)(x3 * -Math.sin(theta+alpha) + y3 * Math.sin(theta+alpha));
+
+	// finally, add the center back again
+	int x_new = x4 + center.x;
+	int y_new = y4 + center.y;
+		
+	int startAngle =  (int)(180*theta/Math.PI);
+	int rangeAngle = (int)(180*alpha/Math.PI);
+			
+	boolean hitSelectedNode = false;
+	if( selectedClusterID != -1 )
+	{
+	    Object obj1 = (Object)child;
+	    ClusterObj selObj = (ClusterObj)clusterVect.elementAt(selectedClusterID);
+	    Object obj2 = (Object)selObj.cluster;
+	    hitSelectedNode = obj1.equals(obj2);
+	}
+	
+	int currentClusterID = -1;
+    
+	if( selectedClusterID != -1 )
+	    {
+		int w = 0;
+		boolean found = false;
+		while( w < clusterVect.size() && !found )
+		    {
+			Object obj1 = (Object)child;
+			ClusterObj obj = (ClusterObj)clusterVect.elementAt(w);
+			Object obj2 = (Object)obj.cluster;
+			if( obj1.equals(obj2) )
+			    {
+				currentClusterID = w;
+				found = true;
+			    }
+			w++;
+		    }
+	    }
+	
+	if( (d <= selectedDepth) && (!hitSelectedNode) )
+	    {	        
+		g.setColor(new Color( (float) 0.7, (float) 0.7, (float) 1)); 
+		selected = false;
+		
+	    }
+	
+	g.drawArc(center.x-r,center.y-r,2*r,2*r,startAngle,rangeAngle);
+	g.drawLine(x_inter,y_inter, x_new, y_new);
+	
+	if( hitSelectedNode )
+	    {	 
+		selectedDepth = d;
+		selected = true;
+		g.setColor(Color.blue);
+		Vector docVector = child.getDocVect();
+	    }
+	
+	g.setColor(selected ? Color.blue : new Color((float) 0.7, (float) 0.7, (float) 1) );
+	
+	int radius = (int)(Math.log(child._items))* 3;
+	if( radius < 2 ) radius = 2;
+
+	if( child._items == 1 )
+	    {
+		Vector docVector = child.getDocVect();
+		radius = Math.max(5,(int)Math.sqrt(docVector.size())*2);
+	    }
+	
+  	d++;
+	r = (int)(Math.sqrt((x_new-center.x)*(x_new-center.x) + (y_new-center.y)*(y_new-center.y)));
+	
+	Point P = new Point(x_new,y_new);
+	if( !constructed )
+	    {	    
+		ClusterObj cObj = new ClusterObj(child,P,radius,dataManager);
+		clusterVect.addElement(cObj);            
+	    }
+	else
+	    {
+		ClusterObj c = (ClusterObj)clusterVect.elementAt(ii.val);
+		c.setPosition(P);
+		clusterVect.setElementAt(c,ii.val);
+	    }
+	ii.val++;
+	
+   	double new_alpha = (double)alpha/2;
+   	
+   	if( selected )
+   	    allSubNodeIDs.addElement(new Integer(currentClusterID) );
+	
+	if( child._items > 1 && d < 6 ){
+	    paintPoke(g,child._child_1,x_new,y_new,center,d,theta+alpha,new_alpha,r,selected,allSubNodeIDs,ii);
+	    paintPoke(g,child._child_2,x_new,y_new,center,d,theta+alpha,-new_alpha,r,selected, allSubNodeIDs,ii);
+	}
+
+	// we have reached the outermost twigs
+	else{
+	    int[] wordFreq = child.getWordFreq();
+	    String[] words = getMostFrequentWords(wordFreq);
+	    
+	    Font f = new Font("Courier",Font.PLAIN,16);
+	    FontMetrics fm = g.getFontMetrics(f);
+	    int ascent = fm.getAscent();
+	    int height = fm.getHeight();
+	    int wordLength = fm.stringWidth(words[0]);
+	    g.setFont(f);
+	    
+	    if( x_new > center.x ){
+		g.drawString(words[0],x_new + Math.max(radius,5) + 10,y_new + Math.max(radius,5));
+	    }
+	    else{
+		g.drawString(words[0],x_new - Math.max(radius,5) - 10 - wordLength,y_new + Math.max(radius,5));
+	    }
+	}
+    }
+    
+    
+    public void paintTree(Cluster cc, Graphics g, int x, int y, int dist)
+    {
+        int dy = 10;		
+        int rad = 2;
+        int dmt = 4;
+	g.setColor(Color.white);
+	g.fillArc(x-rad,y-rad,dmt,dmt,0,360);
+	g.setColor(Color.blue);
+	g.drawOval(x-rad,y-rad,dmt,dmt);
+	String desc = String.valueOf(cc._items);
+	g.drawString(desc,x-(int)rad/2 - (desc.length()-2)*3,y + (int)rad/2 - 2);
+	
+	if( (cc._items > 1) )
+	    {
+		g.drawLine(x,y+rad,x-dist,y-rad+dy);
+		g.drawLine(x,y+rad,x+dist,y-rad+dy);
+		if( cc._child_1 != null ) paintTree(cc._child_1, g, x - dist, y + dy, (int)dist/2);
+		if( cc._child_2 != null ) paintTree(cc._child_2, g, x +  dist, y + dy, (int)dist/2);
+	    }
+    }
+    
+    public void paint(Graphics g)
+    {
+       update(g);
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/dendro/TriangleIndex.java	(revision 21112)
@@ -0,0 +1,85 @@
+package vishnu.testvis.dendro;
+
+public class TriangleIndex
+{
+	short [] i_ind;    /* from index to i */
+	short [] j_ind;    /* from index to j */
+
+
+
+	TriangleIndex(int matrix_rows)
+	{
+
+  		short i, j;
+        int theIndex;
+
+  		int num_of_elements = matrix_rows * (matrix_rows - 1) / 2;
+
+  		/* Allocate space for lookup tables */
+  		i_ind = new short[num_of_elements];
+  		j_ind = new short [num_of_elements];
+
+	  	for (i = 1; i < matrix_rows; i++)
+    	{
+      		for (j = 0; j < i; j++)
+			{
+	  			theIndex = TriangleIndex.index (i, j);
+				i_ind[theIndex] = i;
+	  			j_ind[theIndex] = j;
+			}
+    	}
+	}
+
+	int row_size (int size)
+	{
+  /* Don't forget that we actually start counting in the second row, since the first one is assumed to be all zeroes */
+  		int i = 1;
+  		int sum = 0;
+
+  		while (sum != size)
+    		sum += i++;
+  		return i;
+	}
+
+	static int index (int i, int j)
+	{
+  /* Assumes lower triangular matrix, i.e. i>j */
+
+  		int temp;
+
+  /* The matrix in this case is symmetric so (i, j) = (j, i) */
+  		if (i < j)
+    	{
+      		temp = i; i = j; j = temp;
+    	}
+
+  		if (i != j)
+    		return (i*i-i)/2 + j;
+    		//(i - 1) * (i - 2) / 2 + i - 1 + j;
+  		else         /* The elements on the diagonal are all 0 */
+    	{
+      		return -1; /* so just return the dummy value -1      */
+    	}
+	}
+	static void print (double[] matrix, int row_size)
+	{
+  		int i, j, theIndex;
+
+  		for (i = 0; i < row_size; i++)
+    	{
+      		for (j = 0; j < row_size; j++)
+			{
+	  			if (i <= j)
+	    			System.out.print(" ");
+	  			else
+	    		{
+	      			theIndex = index (i, j);
+	      			System.out.print(matrix[theIndex] + " ");
+	    		}
+			}
+      		System.out.println();
+    	}
+	}
+
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Logging.java	(revision 21112)
@@ -0,0 +1,107 @@
+
+/**
+ * Title:        Visualisation<p>
+ * Description:  An implementation of some search result visualisations
+ * <p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      <p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.evaluate;
+import vishnu.testvis.object.*;
+import vishnu.testvis.visual.*;
+import java.io.*;
+import java.util.*;
+import java.text.*;
+
+public class Logging
+{
+    private Vishnu displayFrame;
+    private String _id;
+    String filename;
+    PrintWriter outlog;
+    long startmilli;
+
+    public Logging(String id, String path, Vishnu d)
+    {
+        displayFrame = d;
+        DateFormat longTime = DateFormat.getDateTimeInstance(DateFormat.FULL,DateFormat.FULL);
+        DateFormat shortTime = DateFormat.getDateTimeInstance(DateFormat.SHORT,DateFormat.SHORT);
+        Date todayDate = new Date();
+        String node = replace_chars(
+            replace_chars(id + "_" + shortTime.format(new Date()) + ".xml"," ","_"),
+            "/:","-");
+        if (null==path)filename = node;
+        else filename = path + node;
+        System.out.println(filename);
+        //Properties p = d.getProperties();
+        _id = id;
+        long Time;
+        try
+        {
+            /*outlog =
+                new PrintWriter(
+                    new BufferedWriter(
+                        new FileWriter(filename)));*/
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    outlog = new PrintWriter(fw,true);
+		
+		    outlog.println("<LOG>");
+            startmilli=System.currentTimeMillis();
+            log("ID",_id);
+            log("DATE_AND_TIME",longTime.format(todayDate));
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    private String replace_chars(String target,String oldch, String newch)
+    {
+        StringTokenizer tokenizer = new StringTokenizer(target,oldch);
+        StringBuffer buff = new StringBuffer();
+
+        while(tokenizer.hasMoreElements())
+        {
+            if(buff.length()>0)
+                buff.append(newch);
+            buff.append(tokenizer.nextToken());
+        }
+        return buff.toString();
+    }
+    
+    public void log(String tag, String value)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\">" +  value + "</" + tag + ">";
+        outlog.println(out);
+    }
+    
+
+    public void log(String tag)
+    {
+        String out ="<" + tag + " MILLISECS = \"" +
+            Long.toString(System.currentTimeMillis()-startmilli) + "\"/>";
+        outlog.println(out);
+    }
+
+    public void endsection(String tag)
+    {
+        outlog.println("</"+tag+">");
+    }
+
+    public void startsection(String tag)
+    {
+        outlog.println("<"+tag+">");
+    }
+
+    public void logout()
+    {
+        log("LOGOUT",_id);
+        outlog.println("</LOG>");
+        outlog.close();
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/evaluate/Queries.java	(revision 21112)
@@ -0,0 +1,338 @@
+package vishnu.testvis.evaluate;
+import java.io.*; 
+import java.util.*;
+
+public class Queries
+{
+    Vector queryID;
+    Vector queryString;
+    Vector relevance[];
+
+    public Queries()
+    {
+        getQueries();
+        
+        // only of the queries just read
+        getRelevanceData();
+        //printRelevance();
+    }
+    
+    private void getRelevanceData()
+    {
+        
+        relevance = new Vector[queryID.size()];
+        for( int i = 0; i < queryID.size(); i++ )
+        {
+            relevance[i] = new Vector();            
+        }
+        
+        try
+        {
+		    FileReader file = new FileReader("qrels.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    String id = st.nextToken();
+		    //System.out.println(id);
+		    int index = 0;
+		    if( queryID.contains(id) )
+		    {
+		        index = queryID.indexOf(id);
+		        //System.out.println("Contained at position " + index);
+		    }
+		    
+		    String docStr = st.nextToken();
+		    //System.out.println(docStr);
+		    relevance[index].addElement(docStr);
+		    boolean eof = false;
+		    			
+		    while( !eof ) 
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {
+			        st = new StringTokenizer(line);
+	    	    	id = st.nextToken();
+		            if( queryID.contains(id) )
+		            index = queryID.indexOf(id);
+		            docStr = st.nextToken();
+		            relevance[index].addElement(docStr);
+		            st = new StringTokenizer(line);
+		        }
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}    
+    }
+    
+    
+    private void getQueries()
+    {
+        queryID = new Vector();
+        queryString = new Vector();
+        
+        try
+		{
+		    FileReader file = new FileReader("queries.dat");
+		    BufferedReader buff = new BufferedReader(file);
+		    
+		    String line = buff.readLine();
+		    //System.out.println(line);
+		    		    
+		    StringTokenizer st = new StringTokenizer(line);
+		    
+		    String keywords = new String();
+		    
+		    while( st.hasMoreTokens() )
+		    {
+		        String strTemp = st.nextToken();
+		        
+		        // check whether token is last token
+		        // assume last token is the query number
+		        if( !st.hasMoreTokens() )
+		        {
+		            queryID.addElement(strTemp);
+		            keywords = keywords.trim();
+		            queryString.addElement(keywords);
+		        }
+		        else
+		        {
+		            keywords += " " + strTemp;
+		        }
+		    }
+		   	    
+            boolean eof = false;
+			
+		    while( !eof )
+		    {
+		        line = new String();
+			    line = buff.readLine();	
+			    //System.out.println(line);
+			    if( line == null || line == "" )
+			        eof = true;
+			    else
+			    {   
+	    		    keywords = new String();
+		    
+		            st = new StringTokenizer(line);
+		            while( st.hasMoreTokens() )
+		            {
+		                String strTemp = st.nextToken();
+		                
+		                // check whether token is last token
+		                // assume last token is the query number
+		                if( !st.hasMoreTokens() )
+		                {
+		                    queryID.addElement(strTemp);
+		                    queryString.addElement(keywords);
+		                    //System.out.println(keywords);
+		                }
+		                else
+		                {
+		                    keywords += " " + strTemp;
+		                }
+		            }
+        		}
+	        }
+	        buff.close();	
+		}
+		catch( IOException e)
+		{
+			System.out.println("Error -- " + e.toString());
+		}
+	}
+        
+    public Object[] getQueryObjectArray()
+    {
+        Object[] objArray = new Object[queryString.size()];
+	    for( int i = 0; i < queryString.size(); i++ )
+		{
+		    objArray[i] = queryString.elementAt(i);
+		}
+		return objArray;
+    }
+        
+    public Vector getQueryIDs()
+    {
+        return queryID;
+    }
+    
+    
+    public void printRelevance()
+    {
+        for( int i = 0; i < relevance.length; i++ )
+        {
+            for( int j = 0; j < relevance[i].size(); j++ )
+            {
+                System.out.print((String)relevance[i].elementAt(j));
+            }
+            System.out.println();
+            System.out.println();
+        }
+    }
+    
+    // returns vector with first element = total of relevant followed
+    // by precision at 10, 20...
+    public Vector getPerformance(String query, String rankedTrecIDs[])
+    {
+        int num = 0;
+        Vector precisionAt = new Vector();
+        int index = queryString.indexOf(query);
+        //System.out.println("Query number " + index + " selected");
+        if( index == -1 )
+            return precisionAt;  // return empty vector
+                
+        boolean relevant[] = new boolean[rankedTrecIDs.length];
+        //System.out.println("Size of ranked list: " + rankedTrecIDs.length);
+        
+        for( int i = 0; i < rankedTrecIDs.length; i++ )
+        {
+            // determine for each doc in the list whether it is relevant
+            if( relevance[index].contains(rankedTrecIDs[i]) )
+            {
+                relevant[i] = true;
+                num++;
+            }
+            else 
+                relevant[i] = false;
+                
+            // precision at 10,20,30,40,50    
+            if( (i+1) % 10 == 0 && (i+1)/10 < 6)
+            {
+                Integer prec = new Integer(Math.round(((float)num/(i+1)*100)));
+                
+                //System.out.println((double)num/(i+1));
+                precisionAt.addElement(prec);
+                
+            }    
+        }
+        
+        precisionAt.addElement(new Integer(relevance[index].size()));
+        precisionAt.addElement(new Integer(num));
+        //System.out.println(num + " relevant docs from a total of " + relevance[index].size() + " have been found");
+        
+        return precisionAt;
+  }
+    
+}
+
+/*		int totalRel = no_in_cat[categories.indexOf(queryId)]-1; // number of relevant
+		int relSoFar = 0;
+		
+		int pre_vs_rec[] = new int[11];
+			
+				
+		// trec method to compute average precision
+		// compute P for relevant images only
+		// P(r0) = max(P(r >= r0))
+
+		
+		// the following array stores pre/rec for every relevant, the first 
+		// entry is precision for the 0th item = 0.
+		// maximum size of array = items.length
+		// actual size of array = totalRel
+		
+		int recall = 0;
+		int precision = 0;
+		
+		
+		// begin of Rueger method
+		
+		relSoFar = totalRel;
+            
+	   	    // i  = 1 because target img is at zero!!!
+	   	    int maxPrecision = 0;
+	   	    int cutL = 10;
+		    for( int i = no_of_items -1; i > 0; i-- )
+			{
+				String listId = new
+			    String((String)cat_of_item.elementAt((int)list[i]));
+				if( listId.equals(queryId) )
+			    {
+			        recall = (int)(100*relSoFar)/totalRel;
+			        
+			        precision = (int)(100*relSoFar)/i;
+			        
+
+			        double cut = (double)(recall)/10;
+			        
+			        //if( r == 6 ) System.out.print(recall + " " + precision + ":");
+			        			        
+                    // if hit the next cut-off cutL, then set cut-off value to whatever 
+                    // is maximum
+			        if( cut == cutL )
+			        {
+			            //System.out.print("eq ");
+			            if( precision > maxPrecision )
+			            {
+			              //  System.out.print("eq-cm(" + precision + ") ");
+			                maxPrecision = precision;
+			            }
+                        
+                        //System.out.print("eq-ap(" + maxPrecision + ") ");
+			            pre_vs_rec[cutL] = maxPrecision;
+			            cutL--;
+			        }
+			        
+			        // if moved to the left of cut-off, the cut-off value (and those
+			        // lying between it and the last one set) is given the 
+			        // current maxPrecision
+			        else if( cut < cutL )   
+			        {
+			            for( int c = (int)cut+1; c < cutL+1; c++ )
+			            {
+			                pre_vs_rec[c] = maxPrecision;
+			            }
+			            
+			            
+			            if( precision > maxPrecision )
+			            {
+			                maxPrecision = precision;
+			            }
+			            
+			            cutL = (int)cut;
+    
+			        }
+		
+		            
+			        
+			        // just in case cut > cutL, we need still to update maxPrecision
+			        if( precision > maxPrecision )
+			        {
+			            maxPrecision = precision;
+			        }
+			        relSoFar--;
+			    
+			    }
+			    // if not relevant, nothing happens
+		    }
+		    
+		    for( int z = 0; z < cutL+1; z++ )
+		    {
+		        pre_vs_rec[z] = maxPrecision;
+		    }
+		    	    
+            // end of Rueger method
+		
+		// average precision for this query                                                     
+		double area = 0;
+		for(int j = 0; j < 10; j++ )
+		{
+			area += (pre_vs_rec[j]+pre_vs_rec[j+1])/2* 0.1;
+		}
+			
+		Performance p = new Performance(area, pre_vs_rec, queryId);
+		return p;
+  
+	}
+*/
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/CacheManager.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/CacheManager.java	(revision 21112)
@@ -0,0 +1,169 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.*;
+
+
+public class CacheManager
+{
+    
+    int cacheSize;
+    Vector file_names_dat;
+    Vector file_names_doc;
+        
+    public CacheManager()
+    {
+        //getCache();
+    }
+    
+    private void getCache()
+    {
+        Properties p = System.getProperties();
+        String currentDir = p.getProperty("user.dir");
+        File cache_dir = new File(currentDir);
+        file_names_dat = new Vector();
+        file_names_doc = new Vector();
+        
+        String fileArr[] = null;
+        if( cache_dir.exists() )
+        {
+        
+            fileArr = cache_dir.list(new FilenameFilter()
+            {
+                public boolean accept(File dir, String name)
+		        {
+		            return ( name.endsWith(".dat") || name.endsWith(".doc") );
+		        }
+	        });
+	    }
+	    else 
+	    {
+	        System.out.println("Directory does not exist");
+	        System.exit(0);
+	    }
+	    
+	    for( int i = 0; i < fileArr.length; i++ )
+	    {   
+	        if( fileArr[i].endsWith(".dat") )
+	            file_names_dat.addElement(fileArr[i]);
+	        if( fileArr[i].endsWith(".doc") )
+	            file_names_doc.addElement(fileArr[i]);
+	    }	    
+    }
+           
+    public void addDataItem(DataBlock data, String key)
+    {   
+        String fileName = new String(key +".dat");
+        file_names_dat.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(data);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+    
+     public void addDocItem(Vector doc, String key)
+    {   
+        String fileName = new String(key +".doc");
+        file_names_doc.addElement(fileName);
+        
+        try
+        {
+            FileOutputStream ostream = new FileOutputStream(fileName);
+            ObjectOutputStream out = new ObjectOutputStream(ostream); 
+            out.writeObject(doc);
+            ostream.close();                
+        }
+        catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+    }
+   
+        
+    public boolean containsData(String query)
+    {
+        String path = new String(query+".dat");
+        
+        if( file_names_dat.contains(path) )
+            return true;
+            else
+            return false;
+    }
+    
+    public boolean containsDoc(String docName)
+    {
+        String path = new String(docName+".doc");
+        
+        if( file_names_doc.contains(path) )
+            return true;
+            else
+            return false;
+    }
+  
+    
+    public Vector getDoc(String docName)
+    {
+        String filename = new String(docName + ".doc");
+        Vector doc = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            doc = (Vector)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return doc;   
+    }
+    
+    
+ 
+    public DataBlock getData(String query)
+    {
+        String filename = new String(query + ".dat");
+        DataBlock data = null;
+        
+        try
+        {
+            FileInputStream istream = new FileInputStream(filename);
+            ObjectInputStream in = new ObjectInputStream(istream);
+            data = (DataBlock)in.readObject();        
+        }
+    	catch (FileNotFoundException ex) {System.out.println("File not found exception");}
+	    catch (IOException ex) {System.out.println("Error when writing to raw_imagebase");}
+	    catch (Exception ex) {System.out.println(ex);}
+	    
+        return data;   
+    }
+    
+    public int getSize()
+    {
+        return file_names_dat.size();
+    }
+    
+    public Vector listCache()
+    {
+        return file_names_dat;
+    }
+    
+    public void printMe()
+    {
+        for( int i = 0; i < file_names_dat.size(); i++ )
+        {
+            String object = (String)file_names_dat.elementAt(i);
+            System.out.println(object);
+        }
+    }
+  
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterGroupObj.java	(revision 21112)
@@ -0,0 +1,69 @@
+/**
+ * @(#)ClusterGroupObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+
+public class ClusterGroupObj
+{
+    ClusterObj[] clusters;  //-- list of ClusterObj ref
+    String label;  //-- a general description for the group (take the highest related word)
+    private int groupSize;  //-- total number of docs in the group
+    public ClusterGroupObj(ClusterObj[] c, String ls)
+    {
+        //System.out.println("ClusterGroupObject constructor called label " + ls);
+        //System.out.flush();
+        clusters=c; label=ls;
+        groupSize=0;
+        for (int i=0;i<clusters.length;i++)
+      		groupSize+=clusters[i].getDocsSize();
+	}
+
+
+ public String getLabels()
+ {
+   return label;
+ }
+
+
+  public String getLabels_toString()
+  {
+    return label;
+  }
+/*
+  	String tmp="";
+  	for (int i=0;i<label.length;i++) tmp = tmp + " " + label[i];
+  	return tmp;
+  }
+*/
+  /**
+   * @return the total number of documents in the group
+   */
+  public int getGroupSize(){return groupSize;}
+
+  /**
+   * @return the number of clusters in the group
+   */
+  public int getClustersSize(){return clusters.length;}
+
+
+  /**
+   * @return the clusters list
+   */
+  public ClusterObj[] getClusters(){return clusters;}
+
+
+  public String getInfo(){
+  	return "No of clusters: "+ String.valueOf(clusters.length)+ " totally contains: "+
+  		String.valueOf(getGroupSize())+" docs\nGroup Label: "+getLabels_toString()+"\n";
+  }
+
+  public void showInfo(){
+    System.out.print(getInfo());
+    for (int i = 0; i<getClustersSize(); i++)
+      System.out.print( i+")  "+clusters[i].getInfo());
+  }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/ClusterObj.java	(revision 21112)
@@ -0,0 +1,77 @@
+/**
+ * @(#)ClusterObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.Vector;
+
+public class ClusterObj
+{
+    DocObj[] _docs;		//-- list of DocObj ref
+    String _label;		//-- (top 3-5? related words)
+    Point2D [] _sammon;			//-- 2D mapping for the centroids
+    DataManager _data;
+    public ClusterObj(DocObj[] d, String t, Point2D [] sammon, DataManager dm)
+    {
+        _docs=d; _label=t;
+        _sammon = sammon;
+        _data = dm;
+    }
+  /**
+   * @return the number of documents in the cluster
+   */
+  public int getDocsSize(){return _docs.length;}
+
+  public String getLabel(){return _label;}
+
+  public String getLabel1(){return getDocsSize() + " documents found";}
+  public String getLabel2(){return getLabel();}
+
+
+  public String getInfo(){
+    return getDocsSize() + " documents found\n" + getLabel();
+  }
+
+
+  /**
+   * This returns a list of list of documents. The inner list of documents represents
+   * which documents have the corresponding related word
+   * pre: _docs can't be null
+   * @return a vector (same length as the related words) of
+   *         vectors (arbitrary length)
+   *
+   */
+  public Vector getHitDocFreq()
+  {
+    int s = _data.getWordCount();
+    Vector rwVec = new Vector(s);
+    for(int i=0;i<s;i++)
+    {
+      rwVec.addElement(new Vector(0));
+      for(int j=0;j<_docs.length;j++)
+      {
+        if ( _docs[j].getWeights(i) > 0 )
+          ((Vector) rwVec.elementAt(i)).addElement(_docs[j]);
+      }
+    }
+    return rwVec;
+  }
+
+
+  public String formatDocs()
+  {
+    int l = getDocsSize();
+    String s = "<html><font face=Arial size=-1> "+ l +" documents found\n";
+    for(int i=0;i<l;i++)
+    {
+      s += _docs[i].format4Fields();
+    }
+    s += "</font></html>\n";
+    return s;
+  }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DataManager.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DataManager.java	(revision 21112)
@@ -0,0 +1,627 @@
+package vishnu.testvis.object;
+
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.dendro.*;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.io.*;
+import java.net.*;
+
+import vishnu.testvis.sammon.*;
+import vishnu.util.Sammon;
+import vishnu.cluster.Clustering;
+import vishnu.testvis.browser.*;
+
+public class DataManager
+{
+    private Vishnu displayFrame;
+    private DataBlock data = null;
+    private CacheManager cache = null;
+    public Vector collections;
+    public Collection currentCollection = null;
+    public String currentView = null;
+    /** Labels for the clusters needed by tree map,
+     *  generated in this class */
+    private String [] clusterLabels = null;
+    /** vertical array of vectors representing
+     *  the documents that word x appears in */
+    private int [] clustersBigWord;
+    //private Vector [] wordDocuments;
+    /** ClusterObjs as used in Tree map */
+    private DocObj [] docObjs;
+    private ClusterObj [] clusterObjs = null;
+    private ClusterGroupObj [] clusterGroups = null;
+    private int[][] sommClusterWords;
+    private Vector sammonClusters;
+
+    private String engine_type = "Engine";
+    
+    /** Set gsdl state */
+    public void setEngineType(String engine) {
+	engine_type = engine;
+    }
+    /** Generate labels for the clusters needed by tree map,
+     *  pre: data is initialised
+     */
+
+    private void setUpClusterLabels()
+    {
+        int [] wordFreqs = new int [data.words.length];
+        StringBuffer sbuff= new StringBuffer(20);
+        clustersBigWord = new int[data.clusters.length];
+        clusterLabels= new String[data.clusters.length];
+        sommClusterWords= new int [data.clusters.length][4];
+
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            for(int d=0; d < data.words.length; d++) wordFreqs[d]=0;
+
+            for(int d=0; d < data.clusters[c].size(); d++)
+            {
+                Integer n = (Integer) data.clusters[c].elementAt(d);
+                int docNo = n.intValue();
+                SparseNode row = data.matrix.getRow(docNo);
+                while( row!= null)
+                {
+                    wordFreqs[row.getColumn()]++;
+                    row = row.getNext();
+                }
+            }
+
+            int col1 = -1, c1val = 0;
+            int col2 = -1, c2val = 0;
+            int col3 = -1, c3val = 0;
+            int col4 = -1, c4val = 0;
+            for(int d=0; d < data.words.length; d++)
+            {
+                if(c1val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=col1;
+                    c2val=c1val;
+                    col1=d;
+                    c1val=wordFreqs[d];
+                }
+                else if(c2val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=col2;
+                    c3val=c2val;
+                    col2=d;
+                    c2val=wordFreqs[d];
+                }
+                else if(c3val < wordFreqs[d])
+                {
+                    col4=col3;
+                    col3=d;
+                    c3val=wordFreqs[d];
+                }
+                else if(c4val < wordFreqs[d])
+                {
+                    col4=d;
+                    c4val=wordFreqs[d];
+                }
+
+            } // for (each document in the cluster)
+           if(col1 > -1)
+           {
+                sbuff.append(data.words[col1]);
+                if(col2 > -1)
+                {
+                    sbuff.append(" ");
+                    sbuff.append(data.words[col2]);
+                    if(col3 > -1)
+                    {
+                        sbuff.append(" ");
+                        sbuff.append(data.words[col3]);
+                    }  // if
+                } // if
+                clusterLabels[c] = new String(sbuff);
+                sbuff.setLength(0);
+            } //if
+            else
+                clusterLabels[c] = new String("<empty>");
+
+            clustersBigWord[c] = col1;
+            sommClusterWords[c][0] = col1;
+            sommClusterWords[c][1] = col2;
+            sommClusterWords[c][2] = col3;
+            sommClusterWords[c][3] = col4;
+        } // for (each cluster)
+    } // end of setUpClusterLabels()
+    
+
+    private void setUpDocObjs()
+    {
+        docObjs = new DocObj[data.docs.length];
+        for(int c=0;c<data.docs.length;c++)
+            docObjs[c] = new DocObj(c, data.descriptions[c], data.matrix.getRow(c), data.docs[c]);
+    }
+    
+    public DocObj getDocObjAt(int index)
+    {
+        return docObjs[index];
+    }
+
+    public int getClusterGroupsSize()
+    {
+        return clusterGroups.length;
+    }
+
+    public ClusterGroupObj getClusterGroup(int i)
+    {
+        return clusterGroups[i];
+    }
+    private void setUpClusterObjs()
+    {
+        clusterObjs = new ClusterObj[data.clusters.length];
+        for(int c=0;c<data.clusters.length;c++)
+        {
+            DocObj [] dObjArray = new DocObj[data.clusters[c].size()];
+            for(int d=0; d<data.clusters[c].size();d++)
+            {
+                Integer n = (Integer)data.clusters[c].elementAt(d);
+                dObjArray[d] = docObjs[n.intValue()];
+            }
+            clusterObjs[c]= new ClusterObj(dObjArray,clusterLabels[c],data.sammon,this);
+        }
+        //ClusterObj(DocObj[] d, String t, Point2D sammon, DataManager dm)
+        // !!!!remember to comment out this loop when it works
+        //for(int c=0;c<data.clusters.length;c++)
+        //    System.out.println("In cluster " + c  + clusterObjs[c].getDocsSize() + " documents ");
+    }
+    /** This for tree map as it needs clusters of clusters
+     * this is a cheap and probably not very satisfactory solution
+     * as all I am doing is taking clusters with the common most significant
+     * word and putting them in groups.
+     */
+    private void setUpClusterGroupObjs()
+    {
+        int count = data.clusters.length;
+        int num=0, i,j;
+        int [] sizes = new int[count];
+        for(i=0;i < count;i++)
+            sizes[i]=1;
+
+        // contains #supercluster clusterList objects, each a list of clusters
+        Vector groupList = new Vector(count);
+        
+        // contains #supercluster BigWords for each supercluster
+        Vector wordList = new Vector(count);
+        for(i=0;i < count;i++)
+        {
+            if(sizes[i]>0)
+            {
+                Vector clusterList = new Vector(count);
+                clusterList.addElement(clusterObjs[i]);
+                int bigWord = clustersBigWord[i];
+                for (j = i+1; j < count; j++)
+                    if (clustersBigWord[j] == bigWord)
+                    {
+                        sizes[i]+=sizes[j];
+                        if(sizes[j]>0)
+                            clusterList.addElement(clusterObjs[j]);
+                        sizes[j]=0;
+                    }
+                groupList.addElement(clusterList);
+                wordList.addElement(new Integer(bigWord));
+            }
+        }
+
+        clusterGroups = new ClusterGroupObj[groupList.size()];
+
+        for(i=0;i < groupList.size();i++)
+        {
+            Vector clusterList = (Vector)groupList.elementAt(i);
+            if(clusterList == null)
+                System.out.println("Cluster list "+ i + " is null");
+            else
+            {
+                //ClusterObj [] clusterObjsArray = clusterList.toArray();
+                int groupSize = clusterList.size();
+                ClusterObj [] clusterObjsArray = new ClusterObj [groupSize];
+                for (j=0; j < groupSize;j++)
+                    clusterObjsArray[j] = (ClusterObj) clusterList.elementAt(j);
+                //System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                Integer bigWord = (Integer)wordList.elementAt(i);
+                int bw = bigWord.intValue();
+                String word;
+                if (bw <0)
+                {
+                    System.out.println("Group " + i + " is " + clusterObjsArray.length + " long");
+                    System.out.println("bigword is empty");
+                    word = "< Empty >";
+                }
+                else
+                    word = data.words[bigWord.intValue()];
+                clusterGroups[i] = new ClusterGroupObj(clusterObjsArray,word);
+            }
+        }
+    }
+    
+    
+    private void printData(String[] str, String filename)
+    {
+        try
+	    {
+	        filename += ".txt";
+            File target = new File(filename);
+		    FileWriter fw = new FileWriter(target);
+		    PrintWriter out = new PrintWriter(fw,true);
+		    String line = new String();
+		   			
+			    for( int j = 0; j < str.length; j++ )
+			    {
+			        line = str[j] + " ";
+			        out.println(j + " " + line);
+			        System.out.println(j + " " + line);
+		        }
+		    
+		    out.close();
+	    } catch (IOException e){ System.out.println("Error " + e.toString());
+	    } catch (SecurityException se) { System.out.println("Error " + 	se.toString());}
+	}
+	
+	public void setUpDendroClusters()
+	{
+	    displayFrame.dendroPanel.treePanel.getData();
+	}
+    
+       
+    
+    public int query(String queryStr)
+    {
+        if( !displayFrame.refine )
+        {
+            if( displayFrame.firstSearch )
+            {
+                displayFrame.firstSearch = false;
+            }
+        }
+                
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return 0;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine theEngine = new GSDLEngine(displayFrame);
+	System.out.println("Searching for " + queryStr);
+	DataBlock db =  theEngine.getData(queryStr, currentCollection.dir, currentView);
+
+        if( db == null ) return 0;
+
+	data = db;
+	
+        int spaceIndex=0;
+        int count = 0;
+        int wordNum = 0;
+        int clusterNum=0;
+        StringBuffer qbuff = new StringBuffer();
+            
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+        if( !displayFrame.refine ) setUpDendroClusters();
+        
+	count = this.data.docs.length;
+	wordNum = this.data.words.length;
+	clusterNum = this.data.clusters.length;
+	
+        if( !displayFrame.refine )
+	    {
+		displayFrame.searchStarted = true;
+	    }
+        
+	displayFrame.update();
+        return count;
+    }
+
+
+    public void fetch(int index)
+    {
+        if(data==null) return ;
+        if(index < 0 || index >= data.docs.length) return;
+        int docNum = data.docs[index];
+        fetchDocNo(docNum);
+    }
+
+
+    public void fetchDocNo(long docNum)
+    {
+        System.out.println("Fetch document: " + Long.toString(docNum));
+
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+	theEngine.setDisplay(displayFrame);
+	//GSDLEngine engine = new GSDLEngine(displayFrame);
+	Vector document = theEngine.getDocument(Long.toString(docNum),currentCollection.dir, currentView);
+	
+	// Fetch theFetch = new Fetch(displayFrame);
+        // document = theFetch.getDocument(docNum);
+        
+	Browser browser;
+	if (document.size() > 1) {
+	    String url = (String)document.elementAt(1);
+	    
+	    if( url.indexOf("http") != -1 )
+		{
+		    try{
+			displayFrame.getAppletContext().showDocument(new URL(url),"_blank");
+			// http://icarus.umkc.edu/cgi-bin/ptext?doc=perseus:text:1999.01.0154"),"_blank");
+		    }catch(IOException ex){ex.printStackTrace();};
+		}     
+	    else browser = new Browser(document,docNum);
+	} else { 
+	    browser = new Browser(document,docNum);
+	}
+    }
+    
+
+    public String [] getClusterLabels()
+    {
+        return clusterLabels;
+    }
+
+
+    public String [] getDescriptions()
+    {
+        if(data != null)
+            return data.descriptions;
+        else return new String[0];
+    }
+    
+    
+    public String [] getTrecIDs()
+    {
+        if(data != null)
+        {
+            String trecID[] = new String[data.descriptions.length];
+            for( int i = 0; i < data.descriptions.length; i++ )
+            {
+                StringTokenizer st = new StringTokenizer(data.descriptions[i]);
+                trecID[i] = st.nextToken();             
+            }
+            return trecID;
+        }
+        else return new String[0];
+    }
+    
+       
+    public String getDescriptionStr(Vector subset)
+    {
+        String returnStr = new String("");
+        for(int c = 0; c < subset.size();c++)
+        {
+            Integer n = (Integer)(subset.elementAt(c));
+            returnStr = returnStr + data.descriptions[n.intValue()] + "\n";
+        }
+        return returnStr;
+    }
+    public int [] getDocs()
+    {
+        if(data != null)
+            return data.docs;
+        else return new int[0];
+    }
+
+    public String [] getWords()
+    {
+        if(data != null)
+        {
+            if(data.words==null)
+            {
+                System.out.println("No words in data!");
+                return new String[0];
+            }
+            return data.words;
+        }
+        else return new String[0];
+    }
+
+    public SparseMatrix getMatrix()
+    {
+        if(data != null)
+            return data.matrix;
+        else return null;
+    }
+
+    public Vector [] getClusters()
+    {
+        if(data != null)
+            return data.clusters;
+        else return null;
+    }
+    public Point2D [] getSammon()
+    {
+        if(data != null)
+            return data.sammon;
+        else return new Point2D[0];
+    }
+
+    public DataManager(Vishnu d) {
+	this(d, null);
+    }
+    public DataManager(Vishnu d, String engine)
+    {
+        displayFrame = d;
+	if (engine != null && !engine.equals("")) {
+	    engine_type = engine;
+	}
+	Engine theEngine;
+	try {
+	    theEngine = (Engine)Class.forName("vishnu.testvis.object."+engine_type).newInstance();
+	} catch (Exception e) {
+	    System.err.println("Couldn't create an engine "+engine_type+ ", aborting query");
+	    System.err.println(e);
+	    return;
+	}
+
+	theEngine.setDisplay(displayFrame);
+	Vector q = new Vector();
+	q.addElement("initialise");
+	collections = theEngine.getCollections(q); 
+    }
+
+
+    // This constructor creates a smaller data manager from
+    // several clusters from a larger one
+    // it is intended to be used with sammon mapping drilling down
+    // The clustering code is lifted from the C version and could be
+    // improved
+    
+    public DataManager(DataBlock oldData, Vector idVect, Vishnu d)
+    {
+        displayFrame = d;
+        
+        // create new document/keyword matrix with only the 
+        // documents from the selected cluster, but keeping
+        // the same 100 keywords as columns
+        SparseMatrix m = new SparseMatrix(idVect.size(),
+                                          oldData.matrix.getColumnCount());
+
+        for (int c = 0; c< idVect.size();c++)
+        {
+          // the document id of the cth document
+          Integer i = (Integer)idVect.elementAt(c);
+          
+          // ptr to the first document of ith row (linked list)
+          SparseNode row = oldData.matrix.getRow(i.intValue());
+          // add the row to new lean matrix
+          while(row != null)
+          {
+              m.addDataItem(c,row.getColumn(),row.getValue());
+              row=row.getNext();
+          }
+        }
+        
+        float [][] full = m.getExpanded(); 
+        
+        // the number of new clusters generated is either 20
+        // or if there are not many docs half the #docs
+        int clusterNum =  m.getRowCount()>20?10:m.getRowCount()/2;
+        
+        // limit set to 20 if > 20 docs otherwise to 3/4 #docs
+        int limit =  m.getRowCount()>20?20:(m.getRowCount()/2)+
+                      (m.getRowCount()/4);
+                      
+        Clustering clustering = new Clustering(Clustering.COMPLETE_LINKAGE,
+					       full,/*m.getRowCount(),
+						      m.getColumnCount(),*/1,
+					       clusterNum,limit);
+                                                    
+        Vector [] clusters = clustering.getClusters();
+        double [][] centroids= clustering.getCentroids();
+
+        Sammon map = new Sammon(centroids);
+        Point2D [] sammon = map.getMapping();
+        String [] words = oldData.words;
+        int [] newDocs = new int[idVect.size()];
+        String [] newDesc = new String[idVect.size()];
+        for (int c=0;c<idVect.size();c++)
+        {
+          Integer i = (Integer) idVect.elementAt(c);
+          int index = i.intValue();
+          newDocs[c] = oldData.docs[index];
+          newDesc[c] = oldData.descriptions[index];
+        }
+        data = new DataBlock(newDocs,newDesc,words,m,clusters,sammon);
+        
+        setUpClusterLabels();
+        setUpDocObjs();
+        setUpClusterObjs();
+        setUpClusterGroupObjs();
+        setUpSammClusters();
+    }
+
+    public DataBlock getDataBlock()
+    {
+        return data;
+    }
+    
+    public int getWordCount()
+    {
+        if (data!=null)
+            return data.words.length;
+        else return 0;
+    }
+    public int getDocCount()
+    {
+        if (data!=null)
+            return data.docs.length;
+        return 0;
+    }
+
+    public Vector getSammonClusters()
+    {
+        if(data != null)
+            return sammonClusters;
+        else return null;
+    }
+
+    private void setUpSammClusters()
+    {
+        LengthTable [] table = new LengthTable [data.clusters.length];
+        
+        // create as many sammon clusters as there are clusters in the data object
+        // and store it in sammmonClusters
+        sammonClusters = new Vector(data.clusters.length);
+        
+        for (int c = 0; c < data.clusters.length; c++){
+	    table[c]= new LengthTable(c,data.clusters[c].size());
+        }
+        
+        
+        // sort by cluster size, I guess
+        QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+        try
+        {
+            qSort.sort(table);
+        }
+        catch (Exception e)
+        {}
+        
+        
+        for (int c = 0; c < data.clusters.length; c++)
+        {
+            int i = table[c].index;
+             SammCluster cluster = new SammCluster(data.clusters[i],
+                        data.sammon[i], sommClusterWords[i],
+                        clusterObjs[i]);
+            sammonClusters.addElement(cluster);
+        }
+    }
+
+    public void requestCollection()
+    {
+	// should reset the data
+	data = null;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DocObj.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/DocObj.java	(revision 21112)
@@ -0,0 +1,89 @@
+/**
+ * @(#)DocObj.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ * Adapted to use sparce matrix Matthew Carey 7/7/2000
+ */
+
+package vishnu.testvis.object;
+import vishnu.datablock.*;
+import java.util.*;
+public class DocObj
+{
+    private int _number;
+    private int _mgNumber;
+    private String _description;
+//--!!! we don't need to store the whole number, as the actual value is not very important
+//--!!! but we need to keep track the doc contains particular related words or not
+//--!!! therefore, one bit will do
+    private SparseNode _weights, _sparceNode;
+
+
+    public DocObj(int num, String s,SparseNode w, int mgn)
+    {
+        _number = num;
+        _description=s;
+        //_fields=f;
+        _weights=w;
+        _mgNumber = mgn;
+    }
+
+  //public void setWeights(int[] w){_weights=w;}
+    public int getNumber()
+    {
+        return _number;
+    }
+    public String getDescription()
+    {
+        return _description;
+    }
+
+    public SparseNode getWeights()
+    {
+        return _weights;
+    }
+    public float getWeights(int i)
+    {
+        SparseNode test = _weights;
+        while(test != null)
+        {
+            if(test.getColumn() == i) return test.getValue();
+            test = test.getNext();
+        }
+        return 0;
+    }
+    static public String getANumberOfWords(String in, int num)
+    {
+        StringTokenizer st = new StringTokenizer(in);
+        StringBuffer buff = new StringBuffer();
+        for(int c=0;c<num && st.hasMoreTokens() ;c++)
+        {
+            if (c>0) buff.append(" ");
+            buff.append(st.nextToken());
+        }
+        String retString = new String(buff);
+        return retString;
+    }
+    
+    public String getTrecID()
+    {
+        StringTokenizer st = new StringTokenizer(_description);
+        return st.nextToken();             
+    }
+
+    public String format4Fields()
+    {
+        String s = "";
+        s += "<dl>\n";
+        s += "  <dt> <a HREF=\"" + _mgNumber +"\">"+ "Document: " + _mgNumber + "</a>\n";
+        //s += "  <dd> <b>"+ getANumberOfWords(_description,5) +"</b><br> \n";
+	String modDesc = _description.replaceAll("<"," ");
+	modDesc = modDesc.replaceAll(">"," ");
+        s += "    <dd> <font color=gray> " + modDesc + " </font>\n";
+        s += "</dl>\n";
+	System.out.println(s);
+        return s;
+  }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/Engine.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/Engine.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/Engine.java	(revision 21112)
@@ -0,0 +1,118 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+
+public class Engine
+{
+    Vishnu display;
+    String servlet;
+
+    public Engine() {
+    }
+
+    public Engine(Vishnu display)
+    {
+	this.setDisplay(display);
+	
+    }
+    
+    public void setDisplay(Vishnu display) {
+	this.display = display;
+	servlet = display.server_address;
+	System.out.println("servlet= "+servlet);
+
+    }
+    
+    public Vector getCollections(Vector q)
+    {
+	Vector c = null;// = new Vector();
+	try{
+	    URLConnection con = getConnection();
+            System.out.println("Sending request for collections");
+	    sendObject(con,q);
+	    c = (Vector)receiveObject(con);
+	} catch( Exception ex) {System.out.println(ex);}
+	return c;
+    }
+    
+    public DataBlock getData(String query, String collection, String view)
+    {
+        DataBlock data = null;
+        Vector q = new Vector();
+	q.addElement("search");
+        q.addElement(query);
+	q.addElement(collection);
+	q.addElement(view);
+	
+
+        try{
+            URLConnection con = getConnection();
+            sendObject(con,q);
+	    Vector v = (Vector)receiveObject(con);
+	    if( v == null || v.size()==0 ) {
+		System.out.println("Engine.getData: Null or empty result");
+	    } else {
+		data = (DataBlock)v.elementAt(0);
+	    }
+        } catch( Exception ex)  {
+	    System.out.println("Engine.getData: "+ex);
+	}
+	
+	return data;
+    }
+
+    
+    public Vector getDocument(String id, String collection, String view)
+    {
+	Vector q = new Vector();
+	Vector resp = new Vector();
+	q.addElement("fetch");
+	q.addElement(id);
+	q.addElement(collection);
+	q.addElement(view);
+
+	try{
+	    URLConnection con = getConnection();
+	    sendObject(con,q);
+	    resp = (Vector)receiveObject(con);
+	} catch(Exception ex) {ex.printStackTrace();}
+	return (Vector)resp.elementAt(0);
+    }
+
+
+    private URLConnection getConnection() throws IOException
+    {
+        URL u = new URL(servlet);
+        URLConnection con = u.openConnection();
+        con.setDoInput(true);
+        con.setDoOutput(true);
+        con.setUseCaches(false);
+	con.setRequestProperty("Content-type","application/octet-stream");
+        con.setAllowUserInteraction(false);
+        return con;
+    }
+    
+    
+    private void sendObject(URLConnection con, Object obj) throws IOException {
+        ObjectOutputStream out = new  ObjectOutputStream(con.getOutputStream());
+        if( obj != null )
+	    {
+                out.writeObject(obj);
+	    }
+        out.close();
+    }
+    
+    
+    private Object receiveObject(URLConnection con) throws Exception
+    {
+	Object obj;
+	InputStream is = con.getInputStream();
+	ObjectInputStream in = new ObjectInputStream(con.getInputStream() );
+	obj = in.readObject();
+	in.close();
+	return obj;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/GSDLEngine.java	(revision 21112)
@@ -0,0 +1,107 @@
+package vishnu.testvis.object;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.visual.*;
+import vishnu.server.Packaging;
+import vishnu.util.XMLUtil;
+import vishnu.util.Base64;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+
+public class GSDLEngine extends Engine
+{
+    public GSDLEngine() {
+    }
+    
+    public GSDLEngine(Vishnu display)
+    {
+	this.setDisplay(display);
+    }
+        
+    public Vector getCollections(Vector q) {
+	return null;
+    }
+    
+    public DataBlock getData(String query_term, String collection, String view) {
+
+	
+	String query = servlet+"&type=search&q="+query_term;
+	if (view != null) {
+	    query += "&view="+view;
+	}
+	System.out.println("sending query: " + query_term);
+	DataBlock db = null;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    
+	    // unserialize the data block
+	    db = (DataBlock) Base64.decodeToObject(results);
+	    //byte [] bytes = results.getBytes("UTF-8");
+// 	    byte [] bytes = Base64.decode(results);
+// 	    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+// 	    ObjectInputStream ois = new ObjectInputStream(bais);
+// 	    db = (DataBlock)ois.readObject();
+//	    ois.close();
+// 	    Element cluster_results = (Element)XMLUtil.getChildByTagName(data_elem, "cluster");
+// 	    String results = XMLUtil.getNodeText(cluster_results);
+// 	    //System.out.println("recived results:" + results);
+// 	    Element description_results = (Element)XMLUtil.getChildByTagName(data_elem, "descriptions");
+// 	    NodeList descs = description_results.getElementsByTagName("desc");
+// 	    Vector desc_vector = new Vector();
+// 	    for (int i=0; i<descs.getLength(); i++) {
+// 		String d = XMLUtil.getNodeText((Element)descs.item(i));
+// 		desc_vector.add(d);
+// 	    }
+// 	    Packaging p = new Packaging();
+// 	    db = p.getDataPackage(results, desc_vector);
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return db;
+
+    }
+
+    public Vector getDocument(String id, String collection, String view) {
+	String query = servlet+"&type=fetch&d="+id;
+	if (view != null) {
+	    query += "&view="+view;
+	}	
+	System.out.println("sending fetch: " + id);
+	Vector doc;
+	try {
+	    URL cgi = new URL(query);
+	    DataInputStream in = new DataInputStream(cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    String results = XMLUtil.getNodeText(data_elem);
+	    System.out.println("recived results:" + results);
+	    // ugly hack. make this nicer!!
+	    doc = new Vector(Arrays.asList(results.split("\n")));
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to cgi: " + e);
+	    e.printStackTrace();
+	    return null;
+	}
+	return doc;
+		
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/LengthTable.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/object/LengthTable.java	(revision 21112)
@@ -0,0 +1,21 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.object;
+
+public class LengthTable
+{
+    public int index;
+    public int length;
+    public LengthTable(int ind, int len)
+    {
+        index= ind;
+        length = len;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/ClusterPanel.java	(revision 21112)
@@ -0,0 +1,450 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import java.util.Enumeration;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.awt.image.*;
+
+class ClusterPanel extends JPanel
+{
+    Image offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    private SammCluster currentCluster;
+    private final int RADIUS = 30;
+    private SammCluster drawnCluster;
+    private GraphicalPane parentSammon;
+
+    boolean firstTime = true;
+    Graphics2D big;
+    BufferedImage bi;
+    Rectangle area;
+    Dimension d = null;
+    
+    int locationID = -1;
+    JPopupMenu[] popups;
+    JPopupMenu popup;
+    Vishnu _currentMain;
+    DataManager dm; 
+
+    public ClusterPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.parentSammon = parent;
+        _currentMain=currentMain;
+	dm = _currentMain.dataManager;
+        popup = new JPopupMenu();
+        popup.setVisible(false);
+        add(popup);
+
+        setOpaque(true);
+        setPreferredSize(new Dimension(600,600));
+        setBorder(BorderFactory.createCompoundBorder(
+            BorderFactory.createEmptyBorder(20,20,20,20),
+            BorderFactory.createLoweredBevelBorder()));
+            
+        addComponentListener(
+            new ComponentAdapter()
+        {
+            public void componentResized(ComponentEvent evt)
+            {
+                if( parentSammon.dataMan != null && parentSammon.dataMan.getDocCount()>0){
+		    repaint();
+                }
+
+            }
+        }
+        );
+
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent mouseEvent)
+            {
+                int docsInSelectedClusters=0;
+                
+                if( parentSammon.dataMan == null) return;
+                int oldLocationID = locationID;
+                
+                Vector clusters = parentSammon.dataMan.getSammonClusters();
+                if (clusters == null) return;
+                locationID = -1;
+                for (int i = clusters.size()-1 ; i >= 0 ; i--)
+                {
+
+                    SammCluster cluster = (SammCluster) clusters.elementAt(i);
+                    int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+                    int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) +RADIUS;
+
+                    if (inCircle(mouseEvent,x,y,cluster.getDisplayedRadius()))
+                    {
+                        locationID = i;
+                    }
+                    if(cluster.getSelected())
+                      docsInSelectedClusters+=cluster.documentIDs.size();
+                }
+
+                if ((oldLocationID != locationID) && (oldLocationID != -1))
+                    popup.setVisible(false);
+
+
+                if (locationID != -1)
+                {
+		    if (locationID != oldLocationID)
+                    {
+               
+                        popup.removeAll();
+                             
+                        String[] allWords = parentSammon.dataMan.getWords();
+                        
+                        currentCluster = (SammCluster) clusters.elementAt(locationID);
+                        String str =  currentCluster.getSize() + " documents";
+                        JMenuItem menuItem = new JMenuItem(str);
+                        popup.add(menuItem);
+
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent actionEvent)
+                            {
+                                drawnCluster = currentCluster;
+                                updateHLinkPanel();
+                                repaint();
+				//update
+                            }
+                        });
+
+                        popup.add(new JPopupMenu.Separator());
+                        menuItem = new JMenuItem("Drill down");
+
+                        if(!currentCluster.getSelected())
+                          docsInSelectedClusters+=currentCluster.documentIDs.size();
+                                                   
+                        if(docsInSelectedClusters<7)menuItem.setEnabled(false);
+                            else menuItem.setEnabled(true);
+                        
+                        // for drilldown
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                Vector idVector;
+                                SammCluster thisCluster = getCurrentCluster();
+                                                                
+                                int theIndex = thisCluster.getWords()[0];
+                                String drillWords = new String(parentSammon.dataMan.getWords()[theIndex]);
+                                for( int i = 1; i < thisCluster.getWords().length; i++ )
+                                {
+                                    theIndex = thisCluster.getWords()[i];
+                                    drillWords += " " + parentSammon.dataMan.getWords()[theIndex];
+                                }
+				
+				Vector clusterPl = parentSammon.dataMan.getSammonClusters();
+
+                                Vector idVect = (Vector)thisCluster.getIDs().clone();
+                                
+                                thisCluster.setSelected(true);
+                                for (int i = clusterPl.size()-1; i >= 0; i--)
+                                {
+
+                                  SammCluster cluster = (SammCluster) clusterPl.elementAt(i);
+                                  if (!cluster.equals(thisCluster) && cluster.getSelected())
+                                    idVect.addAll(cluster.getIDs());
+                                }
+                                if(idVect.size()>6)
+                                {
+                                
+                                  parentSammon.drillDown(idVect);
+                                  
+                                  currentCluster=null;
+                                  drawnCluster=null;
+				  //update();
+				  repaint();
+                                }
+                                
+                            }
+                        });
+                        popup.add(menuItem);
+                        if(!currentCluster.getSelected())
+                          menuItem = new JMenuItem("Select");
+                        else
+                          menuItem = new JMenuItem("Deselect");
+                        menuItem.addActionListener(new ActionListener()
+                        {
+                            public void actionPerformed(ActionEvent ActionEvent)
+                            {
+                                SammCluster cluster = getCurrentCluster();
+                                cluster.setSelected(!cluster.getSelected());
+                                //update();
+				repaint();
+                                
+                            }
+                        });
+
+                        popup.add(menuItem);
+                        int[] words = currentCluster.getWords();
+                        popup.add(new JPopupMenu.Separator());
+                        for(int c=0;c<4;c++)
+                        {
+                          int index = words[c];
+                          if (index>= 0)menuItem = new JMenuItem(allWords[index]);
+                          popup.add(menuItem);
+                          menuItem.setEnabled(false);
+                        }
+
+                        popup.validate();
+                        popup.setVisible(true);
+			
+                    }
+                    popup.show(mouseEvent.getComponent(), mouseEvent.getX(), mouseEvent.getY());
+                    mouseEvent.consume();
+                }
+		
+            }
+	    });
+	
+        setVisible(true);
+    }
+
+    
+    public SammCluster getCurrentCluster() {return currentCluster;}
+    
+    
+    public void paint(Graphics g)
+    {
+	update(g);
+    }
+    
+
+    public boolean inCircle(MouseEvent mouseEvent, int x, int y, int radius)
+    {
+        int dx2 = (mouseEvent.getX()-x)*(mouseEvent.getX()-x);
+        int dy2 = (mouseEvent.getY()-y)*(mouseEvent.getY()-y);
+
+        return (dx2 + dy2 < radius*radius);
+    }
+    
+    
+    private void drawFlag(Graphics g,int x, int y)
+    {
+	g.drawLine(x,y,x,y-12);
+	int [] xs = new int[3];
+	int [] ys = new int[3];
+	xs[0]=x;xs[1]= x+10; xs[2]=x;
+	ys [0]=y-12; ys[1]=y-7;ys[2]=y-2;
+	g.fillPolygon(xs,ys,3);
+    }
+
+    
+    public void update(Graphics g)
+    {
+	Graphics2D g2 = (Graphics2D)g;
+	Dimension d = getSize();
+	offScreenSize = d;
+	
+	int w = d.width;
+	int h = d.height;
+	
+	area = new Rectangle(d);
+	bi = (BufferedImage)createImage(w, h);
+	offGraphics = bi.createGraphics();
+	area.setLocation(w/2-50, h/2-25);
+	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+	
+	offGraphics.setColor(Color.white);
+	/*offGraphics.clearRect(0, 0, area.width, area.height);
+	offGraphics.draw(area);
+	offGraphics.fill(area);*/
+	offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	if (clusters.size()==0) return;
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+	    myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+	    }
+	
+	g2.drawImage(bi,0,0,this);
+    }
+    
+    
+    public void update()
+    {
+	/*        Dimension d = getSize();
+        offScreenSize = d;
+        offScreen = createImage(d.width,d.height);
+        offGraphics = (Graphics2D)offScreen.getGraphics();
+	
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,d.width,d.height);
+	
+        if( parentSammon.dataMan.getDocCount()<1) return;
+	
+        Vector clusters = parentSammon.dataMan.getSammonClusters();
+	
+	double minNumDocs = ((SammCluster) clusters.elementAt(0)).getSize();
+        double maxNumDocs = ((SammCluster) clusters.elementAt(clusters.size()-1)).getSize();
+        double deltaNumDocs = maxNumDocs - minNumDocs;
+	
+        for (int i = clusters.size()-1; i >= 0; i--) {
+	    
+            SammCluster cluster = (SammCluster) clusters.elementAt(i);
+            int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+            int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+	    
+            // Value between 0 and 1 representing the cluster (0 is the smaller cluster and 1 is the bigger one
+            double clusterWeight = (cluster.getSize() - minNumDocs)/ deltaNumDocs;
+
+            //offGraphics.setColor(new Color((float) (1 - 0.7*clusterWeight),(float) 0, (float) 0));
+            offGraphics.setColor(new Color((float)0.7, (float)0.7, (float)1));
+	    
+            cluster.setDisplayedRadius(RADIUS-(int)(RADIUS*(1-clusterWeight)/2));
+            BasicStroke myStroke = new BasicStroke(3,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+            offGraphics.setStroke(myStroke);	        
+            offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            offGraphics.setColor(new Color((float)0,(float)0, (float)0.7));
+            offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+				 y - cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius(),
+				 2*cluster.getDisplayedRadius());
+	    
+            if (cluster.getSelected())
+		drawFlag(offGraphics,x,y-cluster.getDisplayedRadius());
+	    
+            if (cluster.equals(this.drawnCluster))
+		{
+		    
+		    offGraphics.setColor(new Color((float)0.3, (float)0.3, (float)1));
+		    offGraphics.fillOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		    
+		    offGraphics.setColor(new Color((float)0, (float)0, (float)0.7));  
+		    offGraphics.drawOval(x - cluster.getDisplayedRadius(),
+					 y - cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius(),
+					 2*cluster.getDisplayedRadius());
+		}
+	    
+        }
+	
+        int i = clusters.size();
+        Font normal = new Font("Courier",Font.PLAIN,16);
+        Font bold = new Font("Courier",Font.BOLD,16);
+        for (Enumeration enumer = clusters.elements() ; enumer.hasMoreElements();)
+	    {
+		
+		SammCluster cluster = (SammCluster) enumer.nextElement();
+		int x = (int) (cluster.getX()*(offScreenSize.width-2*RADIUS)) + RADIUS;
+		int y = (int) (cluster.getY()*(offScreenSize.height-2*RADIUS)) + RADIUS;
+		offGraphics.setFont(bold);
+		offGraphics.setColor(new Color((float)0, (float)0,(float)0.4));
+		
+		FontMetrics fm = offGraphics.getFontMetrics();
+		int theIndex = cluster.getWords()[0];
+		
+		String theWord = parentSammon.dataMan.getWords()[theIndex];
+		
+		int width = fm.stringWidth(theWord);
+		offGraphics.drawString(theWord,x-(width/2),y+5);
+		i--;
+		}*/
+        repaint();
+    }
+
+
+    private void updateHLinkPanel()
+    {
+        if( parentSammon.dataMan==null) return;
+        
+        if (locationID != -1)
+	    {
+		
+		Vector clusters = parentSammon.dataMan.getSammonClusters();
+		
+		SammCluster selectedCluster = (SammCluster) clusters.elementAt(locationID);
+		ClusterObj cObj = selectedCluster.getClusterObj();
+		String htmlDescription = cObj.formatDocs();
+		
+		parentSammon.hLinkPanel.setText(htmlDescription);
+		parentSammon.keywordPanel.update(cObj);
+	    }
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/GraphicalPane.java	(revision 21112)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+
+/** Panel containing the sammon map interface */
+public class GraphicalPane extends JPanel
+{
+    public JPanel picturePanel;
+    public ClusterPanel clusterPanel;
+    public SammKeywordPanel keywordPanel;
+    public HLinkPanel hLinkPanel;
+    public Stack dataStack;
+    
+    public DataManager dataMan=null;
+    public Vishnu currentMain;
+    public JLabel pictureLabel = new JLabel("Level 0");
+    public JButton pictureButton = new JButton("Back");
+    public JPanel topPanel = new JPanel();
+    private int largeFont = 16;
+    private int smallFont = 14;
+    private int level = 0;
+    
+    public GraphicalPane(Vishnu currentMain)
+    {
+	this.currentMain = currentMain;
+	dataMan = currentMain.dataManager;
+	
+	dataStack=new Stack();
+	level=0;
+	setName("Sammon Map");
+	setLayout(new BorderLayout(10,10));
+	setBorder(BorderFactory.createEmptyBorder(10,10,10,10));
+	clusterPanel = new ClusterPanel(currentMain,this);
+	keywordPanel = new SammKeywordPanel(currentMain,this);
+	Dimension d = currentMain.getSize();
+	hLinkPanel = new HLinkPanel((int)(d.width)/4,d.height,currentMain);
+	picturePanel = new JPanel();
+	picturePanel.setLayout(new BorderLayout(0,3));
+	topPanel.setLayout(new FlowLayout(FlowLayout.LEFT,10,0));
+	topPanel.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
+	pictureLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	pictureButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	topPanel.add(pictureLabel);
+	topPanel.add(pictureButton);
+	pictureButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    unDrill();
+		}
+	    });
+	
+	pictureButton.setEnabled(false);
+	picturePanel.add(topPanel,"North");
+
+	picturePanel.add(clusterPanel,"Center");
+	add("East", hLinkPanel.linkPane);
+	add("Center", picturePanel);
+	add("West", keywordPanel);
+	setVisible(true);
+    }
+    
+    public void drillDown(Vector idVect)
+    {
+	System.out.println("Drilling");
+
+	// save old data manager and create new one with old data
+	// and vector of document ids of the selected cluster
+	dataStack.push(dataMan);
+	dataMan = new DataManager(dataMan.getDataBlock(), idVect, currentMain);
+	level++;
+	pictureButton.setEnabled(true);
+	keywordPanel.clear();
+	hLinkPanel.clear();
+	pictureLabel.setText("Level "+ level);
+    }
+    
+    public void unDrill()
+    {
+	if(!dataStack.empty()){
+		dataMan = (DataManager)dataStack.pop();
+		hLinkPanel.clear();
+		keywordPanel.clear();
+		level--;
+		clusterPanel.repaint();
+		pictureLabel.setText("Level "+ level);
+	    }
+	else level = 0;
+	if(level==0)
+	    pictureButton.setEnabled(false);
+    }
+  
+    public void update()
+    {
+	dataMan = currentMain.dataManager;
+	dataStack=new Stack(); // dispose of old stack - create new empty one
+	level=0;
+	clusterPanel.repaint();
+	keywordPanel.clear();
+	hLinkPanel.clear();
+    }
+  
+    
+    public void addNotify()
+    {
+        Dimension d = getSize();
+	//Record the size of the window prior to calling parents addNotify.
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    /** Used for addNotify check. */
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/HLinkPanel.java	(revision 21112)
@@ -0,0 +1,80 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+import vishnu.testvis.visual.*;
+
+class HLinkPanel extends JEditorPane {
+
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+
+    public int numDisplayedDocs = 15;
+
+    public HLinkPanel(int width,int length, Vishnu currentMain) {
+
+        linkPane = new JScrollPane(this);
+        this.currentMain=currentMain;
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener{
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            //System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            //System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+            {
+
+                String hpresult = he.getDescription();
+                int index = Integer.parseInt(hpresult);
+                System.out.println("hyperlink result: " + index);
+                if (index > 0)
+                    currentMain.dataManager.fetchDocNo(index);
+            }
+        }
+    }
+
+    /*public void update()
+    {
+        int count = currentMain.dataManager.getDocCount();
+        Vector subSet = new Vector(count);
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+            subSet.addElement(new Integer(i));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDocumentDescription(subSet);
+
+        setText(htmlDescription);
+
+    }*/
+
+    public void clear()
+    {
+        setText("");
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/QSortAlgorithmTableLength.java	(revision 21112)
@@ -0,0 +1,142 @@
+/*
+ * @(#)QSortAlgorithm.java        1.6 96/12/06
+ *
+ * Copyright (c) 1994-1996 Sun Microsystems, Inc. All Rights Reserved.
+ *
+ * Sun grants you ("Licensee") a non-exclusive, royalty free, license to use,
+ * modify and redistribute this software in source and binary code form,
+ * provided that i) this copyright notice and license appear on all copies of
+ * the software; and ii) Licensee does not utilize the software in a manner
+ * which is disparaging to Sun.
+ *
+
+ * This software is provided "AS IS," without a warranty of any kind. ALL
+ * EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, INCLUDING ANY
+ * IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE OR
+ * NON-INFRINGEMENT, ARE HEREBY EXCLUDED. SUN AND ITS LICENSORS SHALL NOT BE
+ * LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING
+ * OR DISTRIBUTING THE SOFTWARE OR ITS DERIVATIVES. IN NO EVENT WILL SUN OR ITS
+ * LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT,
+
+ * INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER
+ * CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF
+ * OR INABILITY TO USE SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ *
+ * This software is not designed or intended for use in on-line control of
+ * aircraft, air traffic, aircraft navigation or aircraft communications; or in
+ * the design, construction, operation or maintenance of any nuclear
+
+ * facility. Licensee represents and warrants that it will not use or
+ * redistribute the Software for such purposes.
+ */
+
+/**
+ * A quick sort demonstration algorithm
+ * SortAlgorithm.java
+ *
+ * @author James Gosling
+ * @author Kevin A. Smith
+ * @version         @(#)QSortAlgorithm.java        1.3, 29 Feb 1996
+ */
+
+package vishnu.testvis.sammon;
+
+import vishnu.testvis.object.*;
+
+public class QSortAlgorithmTableLength
+{
+
+    /**
+     * A version of pause() that makes it easier to ensure that we pause
+     * exactly the right number of times.
+
+     */
+    private boolean pauseTrue(int lo, int hi) throws Exception {
+        return true;
+    }
+
+   /** This is a generic version of C.A.R Hoare's Quick Sort
+    * algorithm.  This will handle arrays that are already
+    * sorted, and arrays with duplicate keys.<BR>
+    *
+    * If you think of a one dimensional array as going from
+    * the lowest index on the left to the highest index on the right
+    * then the parameters to this function are lowest index or
+
+    * left and highest index or right.  The first time you call
+    * this function it will be with the parameters 0, a.length - 1.
+    *
+    * @param a       an integer array
+    * @param lo0     left boundary of array partition
+    * @param hi0     right boundary of array partition
+    */
+   void QuickSort(LengthTable a[], int lo0, int hi0) throws Exception
+   {
+      int lo = lo0;
+      int hi = hi0;
+      int mid;
+
+      if ( hi0 > lo0)
+      {
+
+         /* Arbitrarily establishing partition element as the midpoint of
+
+          * the array.
+          */
+         mid = a[ ( lo0 + hi0 ) / 2 ].length;
+
+         // loop through the array until indices cross
+         while( lo <= hi )
+         {
+            /* find the first element that is greater than or equal to
+             * the partition element starting from the left Index.
+             */
+             while( ( lo < hi0 ) && pauseTrue(lo0, hi0) && ( a[lo].length < mid ))
+                 ++lo;
+
+            /* find an element that is smaller than or equal to
+
+             * the partition element starting from the right Index.
+             */
+             while( ( hi > lo0 ) && pauseTrue(lo0, hi0) && ( a[hi].length > mid ))
+                 --hi;
+
+            // if the indexes have not crossed, swap
+            if( lo <= hi )
+            {
+               swap(a, lo, hi);
+               ++lo;
+               --hi;
+            }
+         }
+
+         /* If the right index has not reached the left side of array
+          * must now sort the left partition.
+
+          */
+         if( lo0 < hi )
+            QuickSort( a, lo0, hi );
+
+         /* If the left index has not reached the right side of array
+          * must now sort the right partition.
+          */
+         if( lo < hi0 )
+            QuickSort( a, lo, hi0 );
+
+      }
+   }
+
+   private void swap(LengthTable a[], int i, int j)
+   {
+      LengthTable T;
+      T = a[i];
+      a[i] = a[j];
+      a[j] = T;
+   }
+
+   public void sort(LengthTable a[]) throws Exception
+   {
+      QuickSort(a, 0, a.length - 1);
+   }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammCluster.java	(revision 21112)
@@ -0,0 +1,86 @@
+package vishnu.testvis.sammon;
+
+import java.util.*;
+import vishnu.datablock.*;
+import vishnu.testvis.object.ClusterObj;
+
+public class SammCluster {
+
+    // The co-ordinates of the centroid in the keyword space
+    //public Vector centroid  = null;
+
+    Point2D sammon;
+
+    // The documents ID contained in this cluster
+    public Vector documentIDs = new Vector();
+    public int[] sigWords;
+    private boolean selected =false;
+    private ClusterObj cObj;
+    public int getSize() { return documentIDs.size();}
+
+    // When looking for a particular cluster in a vector
+    /*public boolean equals(Object obj) {
+
+      System.out.println("Equals " + obj.getClass());
+
+        if (obj instanceof MyInteger)
+        return (documentIDs.size() == ((MyInteger) obj).value);
+
+          return super.equals(obj);
+
+            }
+    */
+    // Co-ordinates of the cluster centroid in the 2 dimension space
+    private double x;
+
+    private double y;
+
+    // Displayed size of the cluster
+    private int displayedRadius;
+
+    public void setDisplayedRadius(int value) {displayedRadius = value;}
+
+    public int getDisplayedRadius() {return displayedRadius;}
+
+
+    //private int numDocs;
+
+    // TEMPORARY: For random generator
+
+    public SammCluster(Vector theDocs, Point2D sammon,
+          int [] sigWords, ClusterObj c)
+    {
+        documentIDs = theDocs;
+        this.sammon = sammon;
+        this.sigWords=sigWords;
+        this.cObj=c;
+    }
+
+    public boolean getSelected() {return selected;}
+    public void setSelected(boolean state){selected=state;}
+
+    double getX() {return sammon.x;}
+
+    double getY() {return sammon.y;}
+
+    public ClusterObj getClusterObj(){ return cObj;}
+
+    //int getNumDocs() {return numDocs;}
+
+    public Vector getIDs() {return documentIDs;}
+
+    void addDocumentID(Integer docID) {
+        documentIDs.addElement(docID);
+    }
+
+   // void setClusterCentroid(Vector centroid) {
+   //     this.centroid = centroid;
+
+    int [] getWords() { return sigWords;}
+
+    void dump()
+    {
+        //System.out.println("Centroid : " + centroid.toString());
+        //System.out.println("Doc IDs: " + documentIDs.toString());
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/sammon/SammKeywordPanel.java	(revision 21112)
@@ -0,0 +1,148 @@
+package vishnu.testvis.sammon;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.*;
+import vishnu.testvis.treemap.table.*;
+
+class SammKeywordPanel extends JPanel
+{
+    RW_DataModel _dataModel;
+    //SammRWTable _dataModel;
+    JTable _table = new JTable();
+    JScrollPane theScrollPane;
+    JCheckBox checkBoxes[];
+    JLabel labels[];
+    Vishnu currentMain;
+    GraphicalPane parentPane;
+
+    public SammKeywordPanel(Vishnu currentMain, GraphicalPane parent)
+    {
+        this.currentMain = currentMain;
+        this.parentPane = parent;
+        
+        theScrollPane = new JScrollPane();
+
+         //setBackground(Color.white);
+        setPreferredSize(new Dimension(200,900));
+        setBorder(BorderFactory.createLoweredBevelBorder());
+        setLayout(new BorderLayout(10,0));
+        Font theFont = new java.awt.Font("Monospaced", 0, 12);
+
+        //JLabel title = new JLabel("Related keywords");
+        //title.setFont(theFont);
+        //add(title,"North");
+
+        _dataModel = new RW_DataModel();
+        //_dataModel = new SammRWTable();
+        _table.setModel(_dataModel);
+
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(100);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(50);
+        _table.setRowSelectionAllowed(false);
+        setupRenderer();
+
+        theScrollPane.getViewport().add(_table, null);
+        add(theScrollPane,"Center");
+
+        setVisible(true);
+
+        JButton refineButton = new JButton("Filter Documents");
+        refineButton.setBorder(BorderFactory.createRaisedBevelBorder());
+        refineButton.setMaximumSize(new Dimension(30,70));
+
+        MouseListener ml = new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+                // what does this do Shalini?
+                // Call the refine engine
+                refineEngine();
+            }
+        };
+
+        refineButton.addMouseListener(ml);
+        refineButton.setFont(theFont);
+        add(refineButton,"South");
+
+
+    }
+    public void refineEngine()
+    {
+       currentMain.log.startsection("FILTERING");
+       
+       parentPane.hLinkPanel.setText(_dataModel.getRefinedDocs());
+       String trec_id[] = _dataModel.getRefinedTrecIDs();
+       String filter_Words[] = _dataModel.getFilterWords();
+       String s = new String("");
+       for( int i = 0; i < filter_Words.length; i++ )
+       {
+            s += filter_Words[i] + " ";
+       }
+      
+       currentMain.log.log("FILTER_WORDS",s);
+       currentMain.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+       Vector performance = currentMain.queries.getPerformance(currentMain.query, trec_id);
+       
+       currentMain.log.startsection("PERFORMANCE");
+
+       if( performance.size() > 0 )
+       {
+            Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+            Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+            currentMain.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+            currentMain.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+            for( int i = 0; i < performance.size()-2; i++ )
+            {
+                Integer perf = (Integer)performance.elementAt(i);
+                String str = "PRECISION_" + Integer.toString((i+1)*10);
+                currentMain.log.log(str,Integer.toString(perf.intValue()));
+            }
+       }
+       else
+       {
+            currentMain.log.log("INFO","No relevance data available");
+       }
+     
+                   
+       currentMain.log.endsection("PERFORMANCE");
+       currentMain.log.endsection("FILTERING");
+    }
+        
+    // have dendroclusterobj/sammonclusterobj etc as subclasses
+    // of some abstract clusterObj superclass with some 
+    // core fields (docs, hitdocFreq etc)
+    public void update(ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+    
+    public void update2(vishnu.testvis.dendro.ClusterObj c)
+    {
+        _dataModel.setData(currentMain.dataManager.getWords(), c);
+    }
+
+    public void clear()
+    {
+        _dataModel.clear();
+    }
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterGroupRect.java	(revision 21112)
@@ -0,0 +1,183 @@
+/**
+ * @(#)ClusterGroupRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.*;
+import java.awt.*;
+import java.awt.geom.AffineTransform;
+
+public class ClusterGroupRect extends ZRect
+{
+  static final private Color __COLOUR=Color.black;
+  private ClusterGroupObj _data;
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,
+    Color c, ClusterGroupObj o)
+  {
+    super(x1,y1,x2,y2,c, o.getGroupSize());
+    _data = o;
+  }
+
+
+  public ClusterGroupRect(float x1, float y1, float x2, float y2,ClusterGroupObj o)
+  {
+      super(x1,y1,x2,y2,__COLOUR, o.getGroupSize());
+      _data = o;
+  }
+
+
+
+  public void paint(Graphics oldg)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    g.draw(getBounds2D());
+  }
+
+
+  public void paint(Graphics oldg, float f)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    centerText2(g,__COLOUR);
+    g.setColor(__COLOUR);
+    paintBorder(oldg, f);
+  }
+
+
+
+
+
+  /**
+   * !!! bug:- should check the boundary of the box and the length of the strings
+   * display at most two labels on the centre of the rectangle
+   * @param oldg is the display area
+   * @param c is the colour
+   */
+  private void centerText(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+
+    if (labels!=null){
+// Font f = new Font("Courier",Font.BOLD,12);
+      Font f = g.getFont();
+      FontMetrics fm = g.getFontMetrics(f);
+      float ascent = (float) fm.getAscent();
+      float height = (float) fm.getHeight();
+      float x=getUpperLeftX(), y=getUpperLeftY(),w=(float)getWidth(), h=(float)getHeight();
+      float width0=0F, width1=0F, x0=0F, x1=0F, y0=0F, y1=0F;
+
+        g.setColor(c);
+        width0 = (float) fm.stringWidth(labels);
+        x0 = x + (w-width0)/2F;
+
+// g.setFont(f);
+//   AffineTransform transform
+//     = AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,100.0,50.0);
+        g.drawString(labels,x0,y0);
+        /*
+        if (labels.length==1){
+          y0 = y + (h-height)/2F + ascent;
+          g.drawString(labels[0],x0,y0);
+        }
+        else
+        {
+          width1 = (float) fm.stringWidth(labels[1]);
+          x1 = x + (w-width1)/2F;
+          y0 = y + (h-(height*2F))/2F + ascent;
+          y1 = y0 + (height*1F);
+          g.drawString(labels[0],x0,y0);
+          g.drawString(labels[1],x1,y1);
+        }
+        */
+    }
+  }  //-- centerText
+
+
+
+AffineTransform transform;
+
+  //-- pre: _data.getLabels() has at least 2 elements
+  //-- post: draw two labels to the graphic
+  private void centerText2(Graphics2D g, Color c)
+  {
+    String labels = _data.getLabels();
+    if (labels!=null)
+    {
+      Font f;
+      FontMetrics fm;
+      int ascent, height;
+      int x=(int) getUpperLeftX(), y=(int)getUpperLeftY(),
+          w=(int) getWidth(),      h=(int)getHeight();
+      int width0=0, width1=0, x0=0, x1=0, y0=0, y1=0;
+
+      g.setColor(c);
+
+      for (int fs=12; fs>=6; fs-=2)
+	  {
+        f = new Font("Dialog",Font.BOLD,fs);    // <--- Font for the labels
+
+        fm = g.getFontMetrics(f);
+        ascent = fm.getAscent();
+        height = fm.getHeight();
+        width0 = fm.stringWidth(labels);
+        /*
+        width0 = fm.stringWidth(labels[0]);
+        width1 = fm.stringWidth(labels[1]);
+        */
+        //-- paint horizontally
+        /*if (Math.max(width0,width1)<w  &&  2*height<h)*/
+        if (width0<w  &&  height<h)
+        {
+          x0 = x + (w-width0)/2;
+          //x1 = x + (w-width1)/2;
+          y0 = y + (h-(height)) + ascent;
+          //y1 = y0 + (height*1);
+          g.setFont(f);
+          g.drawString(labels,x0,y0);
+          //g.drawString(labels[1],x1,y1);
+ //System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    horizontal!");
+
+          break;
+        }
+        //-- paint vertically
+/*
+        else if (Math.max(width0,width1)<h  &&  2*height<w)
+        {
+          g.setFont(f);
+
+          x0 = x + (w-width0)/2;
+          x1 = x + (w-width1)/2;
+          y0 = y + (h-(height*2))/2 + ascent;
+          y1 = y0 + (height*1);
+          g.setFont(f);
+		  transform
+     		= AffineTransform.getRotateInstance(-90.0*Math.PI/180.0,
+                (double)(x+w)/2.0,(double)(y+h)/2.0);
+		 g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         g.drawString(labels[0],x0,y0);
+         g.drawString(labels[1],x1,y1);
+         System.err.println(labels[0] +", "+labels[1]+ " painted in font size: "+fs +"    vertical!");
+         System.err.println("x: "+x+"  y: "+y+"  w: "+w+"  h: "+h );
+
+         transform
+            = AffineTransform.getRotateInstance(360.0*Math.PI/180.0,
+               (double)(x+w)/2.0,(double)(y+h)/2.0);
+         g.setTransform(transform);
+         System.err.println(transform.toString());
+
+         break;
+        }
+*/
+
+
+      }  // for
+
+    }  // if
+  }  // centerText2
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ClusterRect.java	(revision 21112)
@@ -0,0 +1,163 @@
+/**
+ * @(#)ClusterRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import vishnu.testvis.object.ClusterObj;
+import java.awt.*;
+
+
+public class ClusterRect extends ZRect
+{
+    private ClusterObj _data;
+	public ClusterRect(float x1, float y1, float x2, float y2, Color c, ClusterObj o)
+    {
+		super(x1,y1,x2,y2,c,o.getDocsSize());
+        _data = o;
+	}
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill(getBounds2D());
+    g.setColor(Color.white);
+    g.draw(getBounds2D());
+  }
+
+  public ClusterObj getClusterObj(){return _data;}
+
+  public String getInfo(){return _data.getInfo();}
+
+
+  //------------------------------------------------------------------
+  private int _xRef=0, _yRef=0;   // used in paintLabels and createTipBox
+
+  /**
+   * paint the cluster's labels on the graphical pane
+   * @param oldg is the graphics to be pained
+   * @param w is the max width of the graphic
+   * @param h is the max heigth of the graphic
+   */
+  public void paintLabels(Graphics oldg, float w, float h){
+  Color tipBoxColor = new Color(255,255,180);
+  Graphics2D g = (Graphics2D) oldg;
+  Font f = new Font("TimesRoman",Font.PLAIN,12);
+  FontMetrics fm = g.getFontMetrics(f);
+  int ascent = fm.getAscent();
+  int height = fm.getHeight();
+  int boxWidth = 4 + Math.max(fm.stringWidth(_data.getLabel1()),
+                              fm.stringWidth(_data.getLabel2()) );
+  int boxHeight = 0 + height * 2;
+  Polygon poly;
+
+    g.setFont(f);
+
+    //-- setup the position of the tipbox
+    poly = createTipBox(boxWidth, boxHeight, w, h);
+
+    //-- paint the box
+    g.setColor(tipBoxColor);
+    g.fillPolygon(poly);
+    g.setColor(Color.black);
+    g.drawPolygon(poly);
+    //-- paint the text
+    g.drawString(_data.getLabel1(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent);
+    g.drawString(_data.getLabel2(), _xRef+3F,
+                _yRef + (boxHeight-(height*2F))/2F + ascent + height);
+
+  }
+
+
+  private Polygon createTipBox(int boxW, int boxH, float maxW, float maxH){
+  int[] xs = new int[8];
+  int[] ys = new int[8];
+  int x=0, y=0,
+    arrowH=10,
+    arrowW=8,
+    arrowOffset=8,
+    gap=5; //-- gap is the space between the tipbox and the rectangle
+  float position=0.2F;  //-- set the box position
+
+    try{
+
+      //-- try RHS
+      x = (int) getLowerRightX() + gap;
+      y = (int) (getUpperLeftY() + getHeight()*position);
+      if ((x+boxW<=maxW) && (y+boxH<=maxH)){
+        xs[0]=xs[6]=xs[4]=xs[3]=xs[7] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x - arrowH;
+        ys[5]=ys[0]=ys[1]=ys[7] = y;
+        ys[3]=ys[2] = y + boxH;
+        ys[6] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try LHS
+      x = (int) getUpperLeftX() - gap - boxW;
+      if ((x>=0) && (y+boxH<=maxH)){
+        xs[0]=xs[7]=xs[6] = x;
+        xs[1]=xs[2]=xs[4]=xs[5] = x + boxW;
+        xs[3] = x + boxW + arrowH;
+        ys[7]=ys[0]=ys[1]=ys[3] = y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y + arrowOffset;
+        ys[4] = y + arrowOffset + arrowW;
+        throw new Exception();
+      }
+
+      //-- try top
+      x = (int) (getUpperLeftX() + getWidth()*position);
+      y = (int) getUpperLeftY() - boxH - gap;
+      if ((y>=0) && (x+boxW<=maxW)){
+        xs[0]=xs[7]=xs[6]=xs[4] = x;
+        xs[1]=xs[2] = x + boxW;
+        xs[5] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1] = y;
+        ys[6]=ys[5]=ys[3]=ys[2] = y + boxH;
+        ys[4] = y + boxH + arrowH;
+        throw new Exception();
+      }
+
+      //-- try bottom
+      y = (int) getLowerRightY() + gap;
+      if ((y+boxH<=maxH) && (x+boxW<=maxW)){
+        xs[7]=xs[0]=xs[6]=xs[2] = x;
+        xs[4]=xs[5] = x + boxW;
+        xs[1] = x + arrowOffset;
+        xs[3] = x + arrowOffset + arrowW;
+        ys[7]=ys[0]=ys[1]=ys[3]=ys[4]= y;
+        ys[6]=ys[5] = y + boxH;
+        ys[2] = y - arrowH;
+        throw new Exception();
+      }
+
+      //-- try inside the rectangle
+      x = (int) getUpperLeftX() + 5;
+      y = (int) getUpperLeftY() + 5 + 5;
+      xs[7]=xs[0]=xs[6]=xs[2] = x;
+      xs[4]=xs[5] = x + boxW;
+      xs[1] = x + 3;
+      xs[3] = x + 3 + 5;
+      ys[7]= ys[0]=ys[1]=ys[3]=ys[4] = y;
+      ys[6]= ys[5]   = y + boxH;
+      ys[2] = y - 5;
+
+    }
+    catch(Exception e){
+    }
+
+    _xRef = x;
+    _yRef = y;
+    return (new  Polygon(xs,ys,xs.length));
+  }
+
+
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/Rect.java	(revision 21112)
@@ -0,0 +1,119 @@
+/**
+ * @(#)Rect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.geom.Rectangle2D;
+import java.awt.*;
+
+/**
+ * The Rect class contains direction and colour, the direction is calculated
+ * based on the initial points.
+ */
+abstract class Rect extends Rectangle2D.Float{
+//-- the rectangle will move to this direction during zooming
+final static protected int __NORTH=0, __EAST=1, __SOUTH=2, __WEST=3;
+private int _direction;
+//-- any two points to create a rectangle
+private float _left_x, _left_y, _right_x, _right_y;
+//-- color of the rectangle
+private Color _colour;
+
+  public Rect(float x1, float y1, float x2, float y2, Color c){
+    setup(x1, y1, x2, y2);
+    _colour = c;
+  }
+
+
+  public void showInfo(){
+    System.err.println( "x: "+getX() +"  y: "+getY()+"  w: "+getWidth()+"  h: "+getHeight());
+    System.err.println(   "    direction: "+getDirection_toString()+"  colour: "+
+        _colour.toString());
+  }
+
+
+  /**
+   * setup the direction and the corresponding coordination
+   */
+  private void setup(float x1, float y1, float x2, float y2){
+    _left_x=x1; _left_y=y1;
+    _right_x=x2; _right_y=y2;
+    //-- set up the direction
+    if (x1<x2)
+      if (y1<y2) _direction = __WEST;
+      else if (y1>y2) _direction = __SOUTH;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else if (x1>x2)
+      if (y1<y2) _direction = __NORTH;
+      else if (y1>y2) _direction = __EAST;
+      else System.err.println("Error - Rectangle box cannot be zero");
+    else
+      System.err.println("Error - Rectangle box cannot be zero");
+    //-- setup the corresponding coordination of the rectangle obj
+    switch (_direction) {
+      case __NORTH:
+        super.setRect(x2,y1,x1-x2,y2-y1);
+        break;
+      case __EAST:
+        super.setRect(x2,y2,x1-x2,y1-y2);
+        break;
+      case __SOUTH:
+        super.setRect(x1,y2,x2-x1,y1-y2);
+        break;
+      case __WEST:
+        super.setRect(x1,y1,x2-x1,y2-y1);
+        break;
+    }
+  }  //-- setup
+
+
+  public void paint(Graphics oldg){
+    Graphics2D g = (Graphics2D) oldg;
+    g.setColor(getColor());
+    g.fill( getBounds2D() );
+    g.setColor(Color.black);
+    g.draw( getBounds2D() );
+  }
+
+  public void paintBorder(Graphics oldg, float w){
+    Graphics2D g = (Graphics2D) oldg;
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getLowerRightX()-w,getUpperLeftY(),
+       w, (float) getHeight()));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getUpperLeftY(),
+       (float) getWidth(),w));
+    g.fill(new Rectangle2D.Float(getUpperLeftX(),getLowerRightY()-w,
+       (float) getWidth(),w));
+  }
+
+  public boolean isNorth(){return _direction==__NORTH;}
+  public boolean isEast(){return _direction==__EAST;}
+  public boolean isSouth(){return _direction==__SOUTH;}
+  public boolean isWest(){return _direction==__WEST;}
+
+  public Color getColor(){return _colour;}
+  public float getUpperLeftX(){return ((float) getX());}
+  public float getUpperLeftY(){return ((float) getY());}
+  public float getLowerRightX(){return ((float) (getX()+getWidth()));}
+  public float getLowerRightY(){return ((float) (getY()+getHeight()));}
+  public int getDirection(){return _direction;}
+  public int getOppositeDirection(){return (_direction+2)%4;}
+
+  /**
+   * @return the direction of the rectangule as a string
+   */
+  public String getDirection_toString(){
+    switch (_direction){
+      case __NORTH: return "NORTH";
+      case __EAST: return "EAST";
+      case __SOUTH: return "SOUTH";
+      default: return "WEST";
+    }
+  }
+
+
+}  //-- Rect
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/rectangle/ZRect.java	(revision 21112)
@@ -0,0 +1,157 @@
+/**
+ * @(#)ZRect.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.rectangle;
+
+import java.awt.*;
+
+public class ZRect extends Rect{
+//-- no of doc that the rectangle represented
+private int _capacity=0;
+
+//-- the new points that the rectangle will be translated (java coordination)
+protected float _t_left_x=0f, _t_left_y=0f, _t_right_x=0f, _t_right_y=0f;
+//-- if the rectangle has been choosen, set it to true
+private boolean _chosen=false;
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c){
+    super(x1, y1, x2, y2, c);
+  }
+
+
+  public ZRect(float x1, float y1, float x2, float y2, Color c, int s){
+    super(x1, y1, x2, y2, c);
+    _capacity = s;
+  }
+
+  public int getCapacity(){return _capacity;}
+  public void setCapacity(int i){_capacity=i;}
+
+  public boolean isChosen(){return _chosen;}
+  public void setChosen(){_chosen=true;}
+  public void resetChosen(){_chosen=false;}
+
+  /**
+   * set the new coordinate, that the rectangle will move to
+   * pre: the coordinate is based on Java graphic coordination
+   *      x1,y1 is the left upper corner and
+   *      x2,y2 is the right lower corner
+   */
+  public void translateTo(float x1, float y1, float x2, float y2){
+    _t_left_x=x1; _t_left_y=y1;
+    _t_right_x=x2; _t_right_y=y2;
+  }
+
+
+
+  private static boolean almostEqual(float a, float b){
+  float tolerant = 0.00001F;
+    return  (Math.abs(a-b) <= tolerant);
+  }
+
+
+  /**
+   * @param x is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return y, based on x
+   */
+  private static float newY(
+    float x,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//    return (x1==x2)? y2: ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+    if (almostEqual(x1,x2))
+      return ((y2-y1)/(float) totalFrame) * (float) (currentFrame+1) + y1;
+    else
+      return ((x*(y1-y2) + x1*y2 - x2*y1) / (x1-x2));
+  }
+
+  /**
+   * @param y is the input value
+   * @param x1 is the orginal x-coordinate of the rectangle
+   * @param y1 is the orginal y-coordinate of the rectangle
+   * @param x2 is the translated x-coordinate
+   * @param y2 is the translated y-coordinate
+   * @return x, based on y
+   */
+  private static float newX(
+    float y,
+    float x1, float y1, float x2, float y2,
+    int totalFrame, int currentFrame){
+//   return (y1==y2)? x2: ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+    if (almostEqual(y1,y2))
+      return ((x2-x1)/(float) totalFrame) * (float) (currentFrame+1) + x1;
+    else
+      return ((y*(x1-x2) - x1*y2 + x2*y1)/(y1-y2));
+  }
+
+
+  /**
+   * pre: translateTo is called
+   * @param how many animated pictures u want to create
+   * @retrun a list of animation picture (rectangles with diff positions)
+   */
+  public ZRect[] zooming(int noOfFrame){
+  ZRect[] vec = new ZRect[noOfFrame];
+  float dist1, dist2, t1, t2;
+
+    switch (getDirection()){
+      case __NORTH: case __SOUTH:
+        dist1 = (_t_left_y - getUpperLeftY())/(float)noOfFrame;
+        dist2 = (_t_right_y - getLowerRightY())/(float)noOfFrame;
+        t1 = getUpperLeftY();
+        t2 = getLowerRightY();
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            newX(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t1,
+            newX(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            t2,
+            getColor());
+        }
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+        break;
+      default:
+        //-- for WEST and EAST
+        dist1 = (_t_left_x - getUpperLeftX())/(float)noOfFrame;
+        dist2 = (_t_right_x - getLowerRightX())/(float)noOfFrame;
+        t1 = getUpperLeftX();
+        t2 = getLowerRightX();
+
+        //-- create the animated pictures, except the last image
+        for(int i=0;i<noOfFrame-1;i++){
+          t1 += dist1;
+          t2 += dist2;
+          vec[i] = new ZRect(
+            t1,
+            newY(t1,getUpperLeftX(),getUpperLeftY(),_t_left_x,_t_left_y,noOfFrame,i),
+            t2,
+            newY(t2,getLowerRightX(),getLowerRightY(),_t_right_x,_t_right_y,noOfFrame,i),
+            getColor());
+        }
+        /**
+         * create the final picture here, the reason we don't do in the above
+         * for loop is because we avoid the 'tolerant' error from the floating
+         * point computation.
+         */
+        vec[vec.length-1] =
+          new ZRect(_t_left_x, _t_left_y, _t_right_x, _t_right_y, getColor());
+
+
+    }  //-- switch
+
+    return vec;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/BarRenderer.java	(revision 21112)
@@ -0,0 +1,56 @@
+/**
+ * @(#)BarRenderer.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.table.TableCellRenderer;
+
+//!!! the % in getString and getColor may WRONG !!!
+
+public class BarRenderer extends JProgressBar implements TableCellRenderer
+{
+
+  public BarRenderer(){
+    super(JProgressBar.HORIZONTAL);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+
+  public BarRenderer(int min, int max){
+    super(min,max);
+    setBorderPainted(false);
+    this.setFont(new Font("Arial",Font.PLAIN,9));
+  }
+
+  public String getString(){
+    if (getValue()==0) return "";
+    else return String.valueOf(getValue())+"%";
+  }
+
+  public Component getTableCellRendererComponent(JTable table, Object value,
+    boolean isSelected, boolean hasFocus, int row, int column){
+  int n = 0;
+    if (value instanceof Number){
+      n = ((Number) value).intValue();
+    } else{
+      System.err.println("Error - item in bar render is not a number");
+    }
+    setValue(n);
+    setForeground(getColor(n));
+    return this;
+  }
+
+  private Color getColor(int value){
+    if (value < 40) return Color.green;
+    else if (value <80) return Color.yellow;
+    else
+      return Color.red;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/table/RW_DataModel.java	(revision 21112)
@@ -0,0 +1,289 @@
+/**
+ * @(#)RW_DataModel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.table;
+
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DocObj;
+import javax.swing.table.DefaultTableModel;
+import java.util.*;
+import vishnu.testvis.sammon.QSortAlgorithmTableLength;
+import vishnu.testvis.object.LengthTable;
+
+/**
+ * This table contains 3 columns:
+ *   - a tick box
+ *   - a related word
+ *   - and the hit document frequency
+ */
+public class RW_DataModel extends DefaultTableModel{
+static final private String[] __COLNAMES={"","Related Word","Hit Doc Freq"};
+private String[] _related_words;
+private ClusterObj _cObj;
+private Vector _colnames = new Vector(__COLNAMES.length);
+
+private Vector _docs = null;  // this is created inside 'setData'
+
+
+  public RW_DataModel()
+  {
+    //-- setup the column's names
+    for (int i=0;i<__COLNAMES.length;i++)
+      _colnames.addElement(__COLNAMES[i]);
+    clear();
+  }
+
+
+  //-- clear all data variables in this data model
+  public void clear()
+  {
+    _related_words = null;
+    _cObj = null;
+    _docs = null;
+    setDataVector(new Vector(0), _colnames);
+  }
+
+
+  //--------------------------------------------------------------------------
+  /**
+   * Set up the cluster data in the data model
+   * pre: the related-words are new to the data model
+   * @param rw is a list of related words
+   * @param c is the data of the cluster
+   */
+  public void setData(String[] rw, ClusterObj c)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+    _cObj = c;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+  
+  // overloaded, this one for dendro
+  // we pass on freq vector instead of obj
+  public void setData(String[] rw, vishnu.testvis.dendro.ClusterObj _cObj)
+  {
+    Vector dataVec = new Vector(0);
+    Vector hdfVec;
+    Vector docs = new Vector();
+    _related_words = rw;
+
+    hdfVec = _cObj.getHitDocFreq();
+    _docs = new Vector();
+// invarient: _hdfVec.size() == rw.length
+
+    //-- for each related word do
+    for(int i=0;i<hdfVec.size();i++){
+      int freq = ((Vector)hdfVec.elementAt(i)).size();
+      if (freq>0)
+      {
+        Vector  entry;
+        entry = new Vector(__COLNAMES.length+1);
+        entry.addElement(new Boolean(false));
+        entry.addElement(_related_words[i]);
+        entry.addElement(new Integer(freq*100/_cObj.getDocsSize()));
+        dataVec.addElement(entry);
+
+        /**
+         * This extra cell is for internal used only.
+         * each entry consists of a vector of doc, those documents contain the
+         * related-word (with weight>0)
+         *
+         * This is used in the filtering routine
+         * so if I sort the above I must likewise sort this one.
+         */
+         docs.addElement(hdfVec.elementAt(i));
+      }
+    }  // for
+    LengthTable [] table = new LengthTable[dataVec.size()];
+    for (int i=0;i<dataVec.size();i++)
+    {
+      Vector ent =  (Vector)dataVec.elementAt(i);
+      Integer in = (Integer)ent.elementAt(2);
+      table[i]=new LengthTable(i,in.intValue());
+    }
+    QSortAlgorithmTableLength qSort = new QSortAlgorithmTableLength();
+    try
+    {
+      qSort.sort(table);
+    }
+    catch (Exception e)
+    {}
+
+    Vector newDataVec= new Vector(dataVec.size());
+    for (int i=dataVec.size()-1;i>=0;i--)
+    {
+      int index = table[i].index;
+      newDataVec.addElement(dataVec.elementAt(index));
+      _docs.addElement(docs.elementAt(index));
+
+    }
+
+    setDataVector(newDataVec, _colnames);
+  } // setData
+
+
+  public String[] getRefinedTrecIDs()
+  {
+      Vector sVec = new Vector();
+      HashSet set = new HashSet();  //-- this can avoid duplicated items !
+      boolean isNewItem;
+      
+      for(int i=0;i<getDataVector().size();i++)
+      {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                
+                Vector docs = (Vector) _docs.elementAt(i);
+
+                for (int j=0;j<docs.size();j++)
+                {
+                    isNewItem = set.add((DocObj) docs.elementAt(j));
+                    if (isNewItem)
+                    {
+                        sVec.addElement(((DocObj)docs.elementAt(j)).getTrecID());
+                        //System.out.println(((DocObj) docs.elementAt(j)).getTrecID());
+                    }
+                }
+            }
+      }
+      
+      String[] sStr = new String[sVec.size()];
+      for( int i = 0; i < sVec.size(); i++ )
+      {
+          sStr[i] = new String((String)sVec.elementAt(i));
+      }
+      return sStr;
+  }
+  
+  public String[] getFilterWords()
+  {
+        Vector sVect = new Vector();
+        
+        for(int i=0;i<getDataVector().size();i++)
+        {
+            if (((Boolean) getValueAt(i,0)).booleanValue())
+            {
+                sVect.addElement((String)getValueAt(i,1));
+            }   
+        }
+        
+        String sStr[] = new String[sVect.size()];
+        
+        for( int i = 0; i < sVect.size(); i++ )
+        {
+            sStr[i] = (String)sVect.elementAt(i);    
+        }
+        return sStr;
+  }
+
+  public String getRefinedDocs(){
+  String s = "";
+  HashSet set = new HashSet();  //-- this can avoid duplicated items !
+  boolean isNewItem;
+    for(int i=0;i<getDataVector().size();i++){
+
+      if (((Boolean) getValueAt(i,0)).booleanValue()) {
+       Vector docs = (Vector) _docs.elementAt(i);
+
+        for (int j=0;j<docs.size();j++){
+          isNewItem = set.add((DocObj) docs.elementAt(j));
+          if (isNewItem)
+          {
+            s += ((DocObj) docs.elementAt(j)).format4Fields();
+            //System.out.println(((DocObj) docs.elementAt(j)).format4Fields());
+          }
+            
+        }
+
+      }
+
+    }
+    s = "<html><font face=Arial size=+10> " + set.size()  + " documents found\n" + s +
+        "</font></html>\n";
+    return s;
+  }
+
+
+
+
+  //--------------------------------------------------------------------------
+  public Class getColumnClass(int col)
+  {
+    switch (col)
+    {
+      case 0: return Boolean.class;
+      case 1: return String.class;
+      case 2: return Integer.class;
+      default: return Object.class;
+    }
+  }
+
+  public int getColumnCount(){return __COLNAMES.length;}
+
+  public boolean isCellEditable(int row, int col){
+    return ((col==0) ? true : false);
+  }
+  public void setValueAt(Object value, int row, int col){
+    super.setValueAt(value,row,col);
+  }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/NavigatorPanel.java	(revision 21112)
@@ -0,0 +1,189 @@
+/**
+ * @(#)NavigatorPanel.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.visual.Vishnu;
+import vishnu.testvis.object.ClusterObj;
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.table.*;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.util.Vector;
+
+
+public class NavigatorPanel extends JComponent
+{
+    private JTable _table;
+    private JButton _showAllButton, _refineButton;
+    private JEditorPane _display;
+    private TMNavigator _outerFrame;
+    private ClusterObj _cObj;
+//public String _codeBase;
+
+    private RW_DataModel _dataModel;  //-- this is created within this class
+
+
+    public NavigatorPanel(JTable t, /*JButton b1,*/ JButton b2,
+        JEditorPane d, TMNavigator f)
+    {
+        _table = t;
+        /*_showAllButton = b1;
+        _showAllButton.setActionCommand("showAllDoc");
+        _showAllButton.setToolTipText("show ALL documents within the selected cluster");
+        _showAllButton.addActionListener(new ButtonHandler());*/
+
+        _refineButton = b2;
+        _refineButton.setActionCommand("refineDoc");
+        _refineButton.setToolTipText("show documents with any of the selected related-words");
+        _refineButton.addActionListener(new ButtonHandler());
+
+        _display = d;
+        _outerFrame = f;
+
+        _display.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        System.out.println("hyperlink result: " + index);
+                        if (index > 0)
+                            _outerFrame._displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+        _dataModel = new RW_DataModel();
+        _table.setModel(_dataModel);
+
+
+
+        _cObj = null;
+        unableButtons();
+        setupRenderer();
+    }
+
+    private void unableButtons()
+    {
+        //_showAllButton.setEnabled(false);
+        _refineButton.setEnabled(false);
+    }
+    private void enableButtons()
+    {
+        //_showAllButton.setEnabled(true);
+        _refineButton.setEnabled(true);
+    }
+
+    public void clear()
+    {
+        _cObj = null;
+        _dataModel.clear();
+        _display.setText("");
+        unableButtons();
+    }
+
+
+
+    public void setData(String[] rw, ClusterObj c)
+    {
+        _cObj = c;
+        _dataModel.setData(rw, _cObj);
+        enableButtons();
+        setupRenderer();
+    }
+
+
+    private void setupRenderer()
+    {
+        BarRenderer bar = new BarRenderer();
+        //-- Have a number painted in the middle of the bar
+        bar.setStringPainted(true);
+        //-- setup the background color of the bar
+        bar.setBackground(_table.getBackground());
+
+        //-- render the last column as a bar
+        _table.getColumnModel().getColumn(2).setCellRenderer(bar);
+    }
+    
+    public void showDocuments()
+    {
+        if( _cObj != null )
+        {
+            _display.setText(_cObj.formatDocs());
+        }
+    }
+     
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            if (_cObj!=null)
+            {
+                String action = event.getActionCommand();
+                if (action.equals("showAllDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(_cObj.formatDocs());
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+                else if (action.equals("refineDoc"))
+                {
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    _display.setText(   _dataModel.getRefinedDocs()   );
+                    _outerFrame.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                    
+                    _outerFrame._displayFrame.log.startsection("FILTERING");
+       
+                    String trec_id[] = _dataModel.getRefinedTrecIDs();
+                    String filter_Words[] = _dataModel.getFilterWords();
+                    String s = new String("");
+                    for( int i = 0; i < filter_Words.length; i++ )
+                    {
+                            s += filter_Words[i] + " ";
+                    }
+      
+                   _outerFrame._displayFrame.log.log("FILTER_WORDS",s);
+                   _outerFrame._displayFrame.log.log("FILTERED_DOCS",Integer.toString(trec_id.length));
+       
+                   Vector performance = _outerFrame._displayFrame.queries.getPerformance(_outerFrame._displayFrame.query, trec_id);
+                   
+                   _outerFrame._displayFrame.log.startsection("PERFORMANCE");
+                    
+                   if( performance.size() > 0 )
+                   {
+                        Integer relvTotal = (Integer)performance.elementAt(performance.size()-2);
+                        Integer relvRetr = (Integer)performance.elementAt(performance.size()-1);
+                        _outerFrame._displayFrame.log.log("RELEVANT_TOTAL",Integer.toString(relvTotal.intValue()));
+                        _outerFrame._displayFrame.log.log("RELEVANT_RETRIEVED",Integer.toString(relvRetr.intValue()));
+                        for( int i = 0; i < performance.size()-2; i++ )
+                        {
+                            Integer perf = (Integer)performance.elementAt(i);
+                            String str = "PRECISION_" + Integer.toString((i+1)*10);
+                            _outerFrame._displayFrame.log.log(str,Integer.toString(perf.intValue()));
+                        }
+                   }
+                   else
+                   {
+                        _outerFrame._displayFrame.log.log("INFO","No relevance data available");
+                   }
+                
+                    _outerFrame._displayFrame.log.endsection("PERFORMANCE");
+                    _outerFrame._displayFrame.log.endsection("FILTERING");
+                    
+                }
+            }
+        }
+    }  //-- end ButtonHandler
+
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TMNavigator.java	(revision 21112)
@@ -0,0 +1,159 @@
+/**
+ * @(#)Navigator.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.visual.Vishnu;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.border.BevelBorder;
+
+
+public class TMNavigator extends JPanel
+{
+    final private int _TREEMAPWIDTH=300, _TREEMAPHEIGHT=350;
+    public final static int LIST_ON_LEFT = 0;
+    public final static int LIST_ON_RIGHT = 1;
+    private int listPos;
+
+//    JTextField _searchTF = new JTextField();
+//    JButton _searchButton = new JButton("Search");
+    public Vishnu _displayFrame;
+    JScrollPane _displayJSP = new JScrollPane();
+    JEditorPane _displayJEP = new JEditorPane();
+    JTable _table = new JTable();
+    JButton _showAllButton = new JButton("Show All Documents");
+    JButton _refineButton = new JButton("Filter");
+    NavigatorPanel navigator;
+//  = new NavigatorPanel(_table, _showAllButton, _refineButton, _displayJEP, this,_codeBase);
+
+    JButton _backButton = new JButton("<<   Back");
+    TreemapPanel _treemapPanel;
+//  = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _codeBase);
+
+
+
+	public TMNavigator(Vishnu displayFrame, int listPos)
+    {
+   	    _displayFrame = displayFrame;
+   	    this.listPos = listPos;
+   	    
+        navigator = new NavigatorPanel(_table, /*_showAllButton*/ _refineButton, _displayJEP, this);
+        _treemapPanel = new TreemapPanel(_TREEMAPWIDTH,_TREEMAPHEIGHT, _backButton, navigator, _displayFrame);
+        
+        buildGUI();
+        
+	    setSize(_TREEMAPWIDTH*2+40,_TREEMAPHEIGHT+340);
+        //setVisible(false);
+	    //setupEventHandler();
+	}
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        _treemapPanel.setVisible(f);
+    }
+
+    void buildGUI()
+    {
+        JPanel tmpPanel;
+        JScrollPane tmpJSP;
+
+        //this.setLayout(new FlowLayout(FlowLayout.LEFT,10,10));
+        this.setLayout(null);
+//        _searchTF.setFont(new Font("Courier",Font.BOLD,16));
+//        _searchTF.setBounds(10,10,400,30);
+//        add(_searchTF,null);
+
+//        _searchButton.setActionCommand("search");
+//        _searchButton.setBounds(10+400+10,10,100,30);
+//        add(_searchButton,null);
+        _backButton.setBorder(BorderFactory.createEtchedBorder());
+        _backButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _backButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        add(_backButton,null);
+
+        if( listPos == 0 ) // list on left
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 ) // list on right
+        {
+            tmpPanel = newJPanel(_treemapPanel,Color.gray,
+            10+_TREEMAPWIDTH+10,10,_TREEMAPWIDTH+3,_TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+
+
+        _displayJEP.setEditorKit(new HTMLEditorKit());
+        _displayJEP.setEditable(false);
+        Dimension sSize=this.getToolkit().getScreenSize();
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_displayJEP);
+        tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10+_TREEMAPHEIGHT+10+30+10, _TREEMAPWIDTH*2+10,
+            sSize.height>700?200:85);
+        add(tmpPanel,null);
+
+        //-- manually creating the columns
+        _table.setAutoCreateColumnsFromModel(false);
+        _table.getColumnModel().getColumn(0).setPreferredWidth(20);
+        _table.getColumnModel().getColumn(1).setPreferredWidth(_TREEMAPWIDTH*2/3);
+        _table.getColumnModel().getColumn(2).setPreferredWidth(_TREEMAPWIDTH/3);
+        _table.setRowSelectionAllowed(false);
+        tmpJSP = new JScrollPane();
+        tmpJSP.getViewport().add(_table);
+        
+        if( listPos == 0 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10+_TREEMAPWIDTH+10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        if( listPos == 1 )
+        {
+            tmpPanel = newJPanel(tmpJSP, Color.white,
+            10, 10, _TREEMAPWIDTH, _TREEMAPHEIGHT+2);
+            add(tmpPanel,null);
+        }
+        
+        
+        _showAllButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _showAllButton.setBounds(10+_TREEMAPWIDTH+10,10+_TREEMAPHEIGHT+10,200,30);
+        _showAllButton.setBorder(BorderFactory.createEtchedBorder());
+        //add(_showAllButton,null);
+
+        _refineButton.setFont(new java.awt.Font("Monospaced", 0, 12));
+        _refineButton.setBounds(10+_TREEMAPWIDTH+10+200,10+_TREEMAPHEIGHT+10,100,30);
+        _refineButton.setBorder(BorderFactory.createEtchedBorder());
+        add(_refineButton,null);
+
+    }
+
+
+    private JPanel newJPanel(Component c, Color bg,int x, int y, int w, int h)
+    {
+        JPanel p = new JPanel();
+        p.setBorder(new BevelBorder(BevelBorder.LOWERED));
+        p.setLayout(new BorderLayout());
+        p.setBounds(x,y,w,h);
+        p.setBackground(bg);
+        p.add(c,BorderLayout.CENTER);
+        return p;
+  }
+
+    public void update()
+    {
+        _treemapPanel.newSearch();
+    }
+
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/TreemapPanel.java	(revision 21112)
@@ -0,0 +1,539 @@
+/**
+* name      TreemapPanel.java 
+* author    Peter Au
+* modified  Daniel Heesch (29.04.02)
+*/
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.DataManager;
+import vishnu.testvis.treemap.rectangle.ClusterRect;
+import vishnu.testvis.visual.*;
+import vishnu.testvis.object.*;
+import java.util.Stack;
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import java.util.*;
+
+public class TreemapPanel extends Canvas
+{
+
+    private int _width, _height;
+    private JButton _popUpButton;
+    private NavigatorPanel _navigator;
+    private String _newTerms;
+    Vishnu _displayFrame;
+    Image _offScreenImage=null;
+    boolean _init_done=false, _drillDownFlag=false, _newSearchFlag=false, _popUpFlag=false;
+    Stack _visualObjs;
+    
+    JPopupMenu[] popups;
+    PopupMenu popup;
+
+    public TreemapPanel(int w, int h, JButton bb, NavigatorPanel n, Vishnu displayFrame)
+    {
+        _displayFrame = displayFrame;
+        _width=w; _height=h;
+        _popUpButton = bb;
+        _navigator = n;
+        _displayFrame = displayFrame;
+        setSize(_width,_height);
+        _visualObjs = new Stack();
+        addMouseListener(new MouseHandler());
+        _popUpButton.setActionCommand("popUp");
+        _popUpButton.setEnabled(false);
+        _popUpButton.setToolTipText("Go back to the previous level");
+        _popUpButton.addActionListener(new ButtonHandler());
+                   
+        popup = new PopupMenu();
+        
+        add(popup);
+
+    }
+
+    class MouseHandler extends MouseAdapter{}
+
+    class ButtonHandler implements ActionListener
+    {
+        public void actionPerformed(ActionEvent event)
+        {
+            String action = event.getActionCommand();
+            String st;
+            if (action.equals("popUp")){
+                processPopUp();
+            }
+        }
+    }
+    
+    public void update(Graphics oldg)
+    {
+        Graphics2D g = (Graphics2D) oldg;
+        Visualiser currentVObj;
+        
+        if (_newSearchFlag)
+        {
+            _visualObjs.push(new Visualiser(_displayFrame.dataManager, 0F,0F, _width, _height));
+            currentVObj = ((Visualiser) _visualObjs.peek());
+            currentVObj.createTreemapView(createImage(_width, _height));
+            
+            //-- draw the treemap view on the offscreen-buffer
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            
+            _newSearchFlag = false;
+        }
+        else if (_drillDownFlag)
+        {
+            _displayFrame.refine = true;
+            currentVObj = (Visualiser) _visualObjs.peek();
+            
+            //-- create a list of image buffer for the animated pictures
+            Image iBuffer[]=new Image[Visualiser.__FRAME];
+            
+            for(int i=0;i<iBuffer.length;i++)
+                iBuffer[i]=createImage(_width,_height);
+
+            //-- prepare the zooming pictures of the current visaul object
+            currentVObj.prepare_zooming(iBuffer, this);
+
+            int s = _visualObjs.size();
+            float x = Visualiser.__borderwidth*s,
+                y = Visualiser.__borderheight*s;
+         
+            DataManager dm = new DataManager(_displayFrame);
+            dm.query(_newTerms);
+            _visualObjs.push(new Visualiser(dm,x, y, _width-x*2, _height-y*2));
+
+                /**
+                * create a treemap view for the new data
+                * before doing so, we have to copy the last zooming picture to the treemap
+                * this is because the zooming picture contains the borders
+                */
+                
+            Image tmpImg = createImage(_width, _height);
+            Graphics2D tmpGph = (Graphics2D) tmpImg.getGraphics();
+            tmpGph.drawImage(currentVObj.getLastZoomingImage(),0,0,this);
+            
+            //-- now create the treemap view
+            ((Visualiser) _visualObjs.peek()).createTreemapView(tmpImg);
+
+            //-- draw the treemap view (top element of the stack) on the offscreen buffer
+            ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+
+            //-- when everyting is done, we then show the animated pictures
+            //-- we use the 2nd element of the stack, not the top one
+            currentVObj.show_zoomIn(g,this);
+
+            _drillDownFlag = false;
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+        }
+        else if (_popUpFlag)
+        {
+
+            currentVObj = (Visualiser) _visualObjs.peek();
+            //-- redraw the treemap view on the offscreen buffer, as the previous
+            //-- level is gone
+            currentVObj.drawTreemapView(_offScreenImage, this);
+            //-- show the animated pictures
+            currentVObj.show_zoomOut(g,this);
+            _popUpFlag = false;
+        }
+        paint(g);
+    }
+
+
+    public void clearDisplay()
+    {
+        _navigator.clear();
+    }
+
+
+    public void processPopUp()
+    {
+        if (_visualObjs.size() > 1)
+        {
+            _displayFrame.log.log("UNREFINE_QUERY");
+            //-- get rid of the top obj in the stack
+            _visualObjs.pop();
+            //-- reset the necessary instance variable back to the default values
+            ((Visualiser) _visualObjs.peek()).setDefault();
+            //-- clear the table and display
+            clearDisplay();
+            if (_visualObjs.size()==1) _popUpButton.setEnabled(false);
+            _popUpFlag = true;
+            repaint();
+        }
+    }
+
+    public void newSearch()
+    {
+        clearDisplay();
+        _popUpButton.setEnabled(false);
+        _visualObjs = new Stack();
+        _newSearchFlag = true;
+        repaint();
+    }
+
+
+    private void processDrillDown(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+        //-- chose which cluster to drill down
+        if (r!=null){
+            //-- do sth on 'r'??? send it back to the server and do the clustering again
+            _popUpButton.setEnabled(true);
+            clearDisplay();
+            _drillDownFlag = true;
+            _newTerms = r.getClusterObj().getLabel();
+            _navigator.clear();  //-- clear the table and the display
+            repaint();
+
+        }
+        else
+            setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+
+
+
+    private void processInspect(int x, int y){
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+        ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(x,y);
+        if (r!=null){
+            //-- draw a corner on the selected cluster
+            ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                this, x, y, _width, _height);
+            //-- setup the data model in the table
+            _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                r.getClusterObj());
+                
+            // and show all documents!
+            _navigator.showDocuments();            
+                
+            repaint();
+        }
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+    
+    protected void processMouseEvent(MouseEvent e)
+    {
+        if (e.getID()==MouseEvent.MOUSE_CLICKED)
+        {       
+            
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                       
+                    if (r!=null)
+                    {
+                        
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        
+                        String str = r.getClusterObj().getLabel();
+                        _displayFrame.log.log("SELECTED_CLUSTER",str);
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            /*img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            System.out.println("Changed?: " + img_hasChanged);
+            if (img_hasChanged)*/  repaint();
+        }
+        
+        }  // if mouseclick
+    }
+
+
+    //------------------------------------------------------------------
+
+    public void paint(Graphics oldg)
+    {
+        //System.out.println("Tree map paint called");
+        Graphics2D tmpGph, g = (Graphics2D) oldg;
+        Image tmpImg;
+        /**
+        * create off-scrren images for double-buffering
+        * There is a known deficiency bug (Java itself) with createImage in java applications.
+        * Workaround: to create necessary image buffers inside paint!
+        */
+        if (!_init_done){
+            _offScreenImage = createImage(_width, _height);
+            _init_done = true;
+        }
+
+        /*if(this.isVisible())*/ g.drawImage(_offScreenImage,0,0,this);
+
+    }  //-- paint
+
+
+
+}  //-- TreemapPanel
+
+
+        /*
+        addMouseMotionListener( new MouseMotionAdapter()
+        {
+            public void mouseMoved(MouseEvent e)
+            {
+                if (!_visualObjs.empty())
+                {
+                    final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+                    ClusterObj cObj = r.getClusterObj();
+                    int numDocs = cObj.getDocsSize();
+                    System.out.print("Mouse moved to ");           
+                    System.out.println(e.getX() + " " + e.getY());
+                    
+                    boolean img_hasChanged;
+            
+                    popup.removeAll();
+                    MenuItem menuItem = new MenuItem(numDocs + " documents");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Show documents");
+                    
+                            if (r!=null)
+                            {
+                                //-- draw a corner on the selected cluster
+                                //((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                                // this, x, y, _width, _height);
+                                //-- setup the data model in the table
+                                _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                                r.getClusterObj());
+                
+                                // and show all documents!
+                                _navigator.showDocuments();            
+                
+                                 repaint();
+                            }
+         
+                        }
+                    });
+            
+                    popup.add(menuItem);
+                
+                    menuItem = new MenuItem("Drill down");
+                    menuItem.addActionListener(new ActionListener()
+                    {
+                        public void actionPerformed(ActionEvent actionEvent)
+                        {
+                            System.out.println("Drill down clicked");
+                        }
+                    });
+            
+                    popup.add(menuItem);
+            
+                    popup.show(this,e.getX(),e.getY());
+            
+                    img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+                    _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+                    System.out.println("Changed?: " + img_hasChanged);
+                    if (img_hasChanged)  repaint();
+                }
+            }
+        });*/
+        
+        /*  protected void processMouseEvent(MouseEvent e)
+    {
+
+        if (!_visualObjs.empty())
+        {
+            //-- if the mouse points outside the obj, then copy the original treemap view
+            //-- without the tipbox to the screen
+            if (e.getID()==MouseEvent.MOUSE_EXITED){
+                //-- get the most update view of the treemap
+                ((Visualiser) _visualObjs.peek()).drawTreemapView(_offScreenImage, this);
+                repaint();
+            }
+            else if (e.getID()==MouseEvent.MOUSE_CLICKED){
+
+                //-- if the user pressed the first mouse button
+                if ((e.getModifiers() & InputEvent.BUTTON1_MASK) == InputEvent.BUTTON1_MASK)
+                {
+                    System.out.println("Mouse clicked");
+                    processInspect(e.getX(),e.getY());
+                    
+                }
+                //-- if the user pressed the third mouse button
+                else if ((e.getModifiers() & InputEvent.BUTTON3_MASK) == InputEvent.BUTTON3_MASK)
+                    processDrillDown(e.getX(),e.getY());
+
+            }
+        }
+
+    }*/
+
+    //------------------------------------------------------------------
+/*    protected void processMouseMotionEvent(MouseEvent e)
+    {
+        System.out.println("Outer Mouse in motion");
+        if (!_visualObjs.empty())
+        {
+            Font weak = new Font("Arial",Font.PLAIN,14);
+            Font bold = new Font("Arial",Font.BOLD,14);
+            
+            final ClusterRect r = ((Visualiser) _visualObjs.peek()).inspect(e.getX(),e.getY());
+            ClusterObj cObj = r.getClusterObj();
+            int numDocs = cObj.getDocsSize();
+            
+            System.out.println("Inner Mouse in motion");
+            System.out.println(e.getX() + " " + e.getY());
+            //System.out.println("Mouse in motion");
+            boolean img_hasChanged;
+                    
+            popup.removeAll();
+            MenuItem menuItem = new MenuItem(numDocs + " documents");
+            menuItem.setFont(bold);
+            
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    System.out.println("Show documents");
+                    
+                    if (r!=null)
+                    {
+                        //-- draw a corner on the selected cluster
+                //        ((Visualiser) _visualObjs.peek()).drawCorner(r, _offScreenImage,
+                //            this, x, y, _width, _height);
+                        //-- setup the data model in the table
+                        _navigator.setData(((Visualiser) _visualObjs.peek()).getDataManager().getWords(),
+                        r.getClusterObj());
+                
+                        // and show all documents!
+                        _navigator.showDocuments();            
+                
+                        repaint();
+                    }
+         
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+                  
+            menuItem = new MenuItem("Refined Query");
+            menuItem.setFont(bold);
+            menuItem.addActionListener(new ActionListener()
+            {
+                public void actionPerformed(ActionEvent actionEvent)
+                {
+                    setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+                    
+                    // all this does is return the rectangle, on which drill down
+                    // is to be performed
+                    //ClusterRect r = ((Visualiser) _visualObjs.peek()).drillDown(x,y);
+                    //-- chose which cluster to drill down
+                    if (r!=null)
+                    {
+                        //-- do sth on 'r'??? send it back to the server and do the clustering again
+                        _popUpButton.setEnabled(true);
+                        clearDisplay();
+                        _drillDownFlag = true;
+                        _newTerms = r.getClusterObj().getLabel();
+                        _navigator.clear();  //-- clear the table and the display
+                        repaint();
+                    }
+                    else
+                    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+                }
+            });
+            
+            popup.add(menuItem);
+            popup.addSeparator();
+            String label = cObj.getLabel();
+            
+            StringTokenizer st = new StringTokenizer(label);
+                       
+            while( st.hasMoreTokens() )
+            {
+                menuItem = new MenuItem(st.nextToken());
+                menuItem.setFont(bold);
+                menuItem.setEnabled(false);
+                popup.add(menuItem);
+            }
+                        
+            popup.show(this,e.getX(),e.getY());
+            
+            //img_hasChanged = ((Visualiser) _visualObjs.peek()).drawTipBox(
+            //   _offScreenImage, this, e.getX(),e.getY(), _width,_height);
+            //System.out.println("Changed?: " + img_hasChanged);
+            //if (img_hasChanged)
+            repaint();
+        }
+    }*/
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/treemap/visual/Visualiser.java	(revision 21112)
@@ -0,0 +1,671 @@
+/**
+ * @(#)Visualiser.java  12/12/99
+ * @author Peter Au
+ * All rights reserved.
+ */
+
+package vishnu.testvis.treemap.visual;
+
+import vishnu.testvis.object.*;
+import vishnu.testvis.treemap.rectangle.*;
+import java.lang.Exception;
+import java.awt.*;
+import java.awt.image.ImageObserver;
+import java.awt.geom.GeneralPath;
+
+public class Visualiser extends Canvas implements Runnable
+{
+
+  static private final int __FRAMEDELAY = 100;
+
+  static final int __FRAME=10;
+  static float __borderwidth=5F, __borderheight=5F;
+//-- the following variables specify the actual drawing area
+  final private float _upperX, _upperY, _width, _height;
+
+  private DataManager _data;
+  private Image _treemapView = null;  //-- static treemap view
+  private Image[] _zoomingImages;
+
+//-- rectangles represent the cluster-groups and the clusters
+  private ClusterGroupRect[] _gpRects = null;
+  private ClusterRect[][] _cRects = null;
+
+//-- the rectangle the mouse pointed to, used for tipbox
+  private ClusterRect _previousRect = null;
+
+//-- to identify which rectangle has a corner-paint
+  private ClusterRect _cornerRect = null;
+
+
+  /**
+   * Constructor: visualise the data into the specific size of pane
+   * @param d is the data obj
+   * @param x is the left upper x-coordinate of the display pane
+   * @param y is the left upper y-coordinate of the display pane
+   * @param w is the width of the display pane
+   * @param h is the height of the display pane
+   */
+  public Visualiser(DataManager d, float x, float y, float w, float h)
+  {
+    int noOfGroup;
+    _data = d;
+    _upperX=x; _upperY=y; _width=w; _height=h;
+
+    //-- allocate memory for _cRects and _gpRects
+    noOfGroup = _data.getClusterGroupsSize();
+    _gpRects = new ClusterGroupRect[noOfGroup];
+    _cRects = new ClusterRect[noOfGroup][];
+    for (int i=0; i<noOfGroup; i++)
+      _cRects[i] = new ClusterRect [_data.getClusterGroup(i).getClustersSize()];
+    // create rectangles for the corresponding groups and clusters
+    createRects();
+  }
+
+
+  static public void setBorder(float w, float h)
+  {
+    __borderwidth = w;
+    __borderheight = h;
+  }
+
+
+  public DataManager getDataManager(){return _data;}
+
+  //---------------------------------------------------------------------
+  /**
+   * set different variables back to default values
+   */
+  public void setDefault()
+  {
+    _previousRect = null;
+    _cornerRect = null;
+    try
+    {
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].isChosen())
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].isChosen())
+	    {
+              _gpRects[i].resetChosen();
+              _cRects[i][j].resetChosen();
+              throw new Exception();
+            }  //-- if
+    }
+    catch(Exception e)
+    {
+    }  //-- catch
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * @return a snapshot of the tree map
+   */
+  public Image getTreemapView(){
+    return _treemapView;
+  }
+
+
+  /**
+   * create a treemap view of the obj, only call once
+   * @param img
+   */
+  public void createTreemapView(Image img)
+  {
+  	final float GP_BORDERWIDTH = 1F; // <-- set the width of the border of each group
+    _treemapView = img;
+    Graphics2D g = (Graphics2D) _treemapView.getGraphics();
+    for (int i=0;i<_gpRects.length; i++)
+	{
+      for(int j=0;j<_cRects[i].length;j++)
+	  {
+        _cRects[i][j].paint(g);
+      }
+      _gpRects[i].paint(g);
+    } //-- for
+    //-- paint the border of each group
+    for (int i=0;i<_gpRects.length; i++)
+       _gpRects[i].paintBorder(g,GP_BORDERWIDTH);
+    // paint the outer border
+    ZRect fooRect = new ZRect(_upperX,_upperY,_upperX+_width,_upperY+_height,null);
+    g.setColor(Color.black);
+    fooRect.paintBorder(g, GP_BORDERWIDTH);
+
+  }  //-- create_pict
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the treemap view on the image, together with the corner triangle
+   */
+  public void drawTreemapView(Image img, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) img.getGraphics();
+    g.drawImage(_treemapView,0,0,observer);
+    drawCorner(g);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect and make
+   * sure that the tipbox is drawn as well
+   */
+  public void drawCorner(ClusterRect r, Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    _cornerRect = r;
+    _previousRect = null; //-- force the tipbox to be repainted
+    drawTipBox(offScreenImg, observer, x, y, w, h);
+  }
+
+
+  /**
+   * draw a small triangle on the top-right corner of the _cornerRect
+   */
+  private void drawCorner(Graphics2D g)
+  {
+    if (_cornerRect!=null)
+    {
+      GeneralPath p;
+      float x=_cornerRect.getLowerRightX(), y=_cornerRect.getUpperLeftY(), w=8F;
+      p = new GeneralPath(GeneralPath.WIND_NON_ZERO);
+      if (Math.min(_cornerRect.getWidth(),_cornerRect.getHeight())<w)
+        w = (float) Math.min(_cornerRect.getWidth(),_cornerRect.getHeight());
+      p.moveTo(x-0.7F,   y+0.9F);
+      p.lineTo(x-0.7F-w, y+0.9F);
+      p.lineTo(x-0.7F,   y+0.9F+w);
+      p.closePath();
+      g.setColor(Color.black);
+      g.fill(p);
+    }
+  }
+
+
+  /**
+   * signal the caller (by returning ture) to update the screen, a tipbox
+   * with the label of the cluster is drawn on the offscreen-image accordingly
+   * @param offScreenImg is the image where the tip box to be drawn on
+   * @param observer
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param w is the max width of the image to be used
+   * @param h is the max height of the image to be used
+   * @return true if the tipbox is drawn on the offScreenImg, otherwise false
+   */
+  public boolean drawTipBox(Image offScreenImg, ImageObserver observer,
+    int x, int y, float w, float h)
+  {
+    Graphics2D g = (Graphics2D) offScreenImg.getGraphics();
+    ClusterRect tmpRect=null;
+
+    //-- find out which rectangle the mouse points to
+    try{
+      for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+          for(int j=0;j<_cRects[i].length;j++)
+            if (_cRects[i][j].contains(x,y))
+            {
+              tmpRect = _cRects[i][j];
+              throw new Exception();
+            }  //-- if
+
+      //-- the mouse is not pointed to any of the rectangles
+      if (_previousRect==null) return false;
+      else
+      {
+        //-- the previous pict has a tip box => draw the orginal pict
+        _previousRect = null;
+        drawTreemapView(offScreenImg, observer);
+        return true;
+      }
+    }
+    catch(Exception e){
+      //-- the mouse is pointed to a cluster's rectangle
+
+      if (_previousRect==tmpRect)
+      {
+        //-- if the tip box is already on the graphic, we don't need to update
+        return false;
+      }
+      else
+      {
+        _previousRect = tmpRect;
+        drawTreemapView(offScreenImg, observer);
+        if (_previousRect!=null)
+        {
+          _previousRect.paintLabels(g, w, h);
+        }
+        return true;
+      }
+
+    }  //-- catch
+  }  //-- drawTipBox
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * find out which rectangle the mouse point to (for inspect)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect inspect(int x, int y)
+  {
+    return findClusterRect(x, y, false);
+  }
+
+  /**
+   * find out which rectangle the mouse point to (for drill down)
+   * @param x is the x-cordinate of the mouse position
+   * @param y is the y-cordinate of the mouse position
+   * @return a cluster rectangle that the mouse pointed to
+   */
+  public ClusterRect drillDown(int x, int y)
+  {
+    return findClusterRect(x, y, true);
+  }
+
+  /**
+   * @param x is the horizontal x position of the mouse event
+   * @param y is the vertical y position  of the mouse event
+   * @param setChosenFlag
+   * @return the cluster rectangle that the mouse points to, and if the
+   *   'setChosenFlag' is true, then set up the corresponding flag in
+   *   _gpRect and _cRect
+   */
+  private ClusterRect findClusterRect(int x, int y, boolean setChosenFlag)
+  {
+    //-- find out which rectangle the mouse points to
+    for (int i=0;i<_gpRects.length; i++)
+        if (_gpRects[i].contains(x,y))
+            for(int j=0;j<_cRects[i].length;j++)
+                if (_cRects[i][j].contains(x,y))
+                {
+                    if (setChosenFlag)
+                    {
+                        _gpRects[i].setChosen();
+                        _cRects[i][j].setChosen();
+                    }
+                    return _cRects[i][j];
+                }  //-- if
+    return null;
+  }
+
+
+  //---------------------------------------------------------------------
+  /**
+   * !!! BUG - Color is not fixed !!!
+   * initialise _gpRects and _cRects
+   * create_sub_rects is called
+   */
+  public void createRects()
+  {
+    float area = _width*_height;  //-- the area of the display
+    int noOfGroup = _data.getClusterGroupsSize();
+    float totalDocs = (float) _data.getDocCount();
+    float lx=_upperX, ly=_upperY, rx=_upperX+_width, ry=_upperY+_height, tmp=0F;
+    Color baseColor = Color.yellow;
+    ClusterGroupObj gpObj;
+    int gpSize;
+
+    for (int i=0;i<noOfGroup; i++)
+    {
+        gpObj = _data.getClusterGroup(i);
+        gpSize = gpObj.getGroupSize();
+
+    //??? Color is not fixed !!!
+        if (i==0) baseColor=Color.cyan;
+        if (i==1) baseColor=Color.green;
+        if (i==2) baseColor=Color.blue;
+        if (i==3) baseColor=Color.orange;
+        if (i==4) baseColor=Color.pink;
+        if (i==5) baseColor=Color.red;
+
+      //-- West
+        if (rx>lx && ry>ly)
+        {
+            tmp = rx;
+            rx = ((float)gpSize/totalDocs)*area/(ry-ly)+lx;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- North
+        else if (lx>rx && ry>ly)
+        {
+            tmp = ry;
+            ry = ((float)gpSize/totalDocs)*area/(lx-rx)+ly;
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+      //-- East
+        else if (lx>rx && ly>ry)
+        {
+            tmp = rx;
+            rx = lx - ((float)gpSize/totalDocs)*area/(ly-ry);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+                gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            lx = tmp;
+        }
+      //-- South
+        else if (rx>lx && ly>ry)
+        {
+            tmp = ry;
+            ry = ly - ((float)gpSize/totalDocs)*area/(rx-lx);
+
+            create_sub_rects(i, rx, ly, lx, ry,
+            gpObj.getClusters(), gpSize, baseColor);
+
+            _gpRects[i] = new ClusterGroupRect(lx,ly,rx,ry, baseColor, _data.getClusterGroup(i));
+            ly = tmp;
+        }
+    } //-- for
+  }  //-- createRects
+
+
+  /**
+   * !!! BUG: Color hasn't fixed !!!
+   * create the rectangles for the clusters in the specified group
+   * @param gpNo is the index of the Group currently working on
+   * @param lx is the initial left x-coordinate (based on Rect coordination)
+   * @param ly is the initial left y-coordinate (ditto)
+   * @param rx is the initial right x-coordinate (ditto)
+   * @param ry is the initial right y-coordinate (ditto)
+   * @param cs is the list of clusters in the group
+   * @param total is the total number of documents found in the group
+   * @param col is the base colour of the group
+   */
+  private void create_sub_rects(int gpNo, float lx, float ly, float rx, float ry,
+                             ClusterObj[] cs, int total, Color col)
+  {
+
+    ZRect dummy = new ZRect(lx,ly,rx,ry,null);
+    float tmp, area = (float) (dummy.getWidth()*dummy.getHeight());
+
+    for(int i=0;i<cs.length;i++)
+    {
+
+     if (rx>lx && ry>ly)
+     {
+       tmp = rx;
+        rx = ((float)cs[i].getDocsSize()/(float)total)*area/(ry-ly)+lx;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- North
+      else if (lx>rx && ry>ly)
+      {
+        tmp = ry;
+        ry = ((float)cs[i].getDocsSize()/(float)total)*area/(lx-rx)+ly;
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+      //-- East
+      else if (lx>rx && ly>ry)
+      {
+        tmp = rx;
+        rx = lx - ((float)cs[i].getDocsSize()/(float)total)*area/(ly-ry);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        lx = tmp;
+      }
+      //-- South
+      else if (rx>lx && ly>ry)
+      {
+        tmp = ry;
+        ry = ly - ((float)cs[i].getDocsSize()/(float)total)*area/(rx-lx);
+        _cRects[gpNo][i] = new ClusterRect(lx,ly,rx,ry,col,cs[i]);
+        ly = tmp;
+      }
+
+//??? Color hasn't fixed !!!
+      col = darker(col);
+
+    }
+  }  //-- create_sub_rects
+
+
+    /**
+     * @see        java.awt.Color#darker
+     */
+    private Color darker(Color c) {
+    double FACTOR = 0.85;
+      return new Color(Math.max((int)(c.getRed()  *FACTOR), 0),
+       Math.max((int)(c.getGreen()*FACTOR), 0),
+       Math.max((int)(c.getBlue() *FACTOR), 0));
+    }
+
+
+
+  //---------------------------------------------------------------------
+  //-- pre: _zoomImages can't be null
+  public Image getLastZoomingImage()
+  {
+    return _zoomingImages[_zoomingImages.length-1];
+  }
+
+  public Image getZoomingImage(int i){return _zoomingImages[i];}
+
+
+  public void show_zoomIn(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=0;i<_zoomingImages.length;i++)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+  public void show_zoomOut(Graphics oldg, ImageObserver observer)
+  {
+    Graphics2D g = (Graphics2D) oldg;
+    for(int i=_zoomingImages.length-1;i>=0;i--)
+    {
+      //-- time delay for each frame
+      timeDelay();
+      g.drawImage(_zoomingImages[i],0,0,observer);
+    }
+    //-- this is time delay for the last frame and the new treemap view
+    timeDelay();
+  }
+
+
+
+  private void timeDelay()
+  {
+    Thread delayThread = new Thread(this);
+    delayThread.start();
+    try{
+      delayThread.join();
+    }
+    catch (InterruptedException ex)
+    {
+    }
+  }
+
+
+  public void run()
+  {
+    long lastTime=0, time=0;
+    do
+    {
+      time = System.currentTimeMillis();
+      try{
+        Thread.sleep(__FRAMEDELAY);
+      }
+      catch (InterruptedException ex)
+      {
+      }
+      lastTime = time;
+    } while (time - lastTime > __FRAMEDELAY);
+
+  }
+
+
+
+  //---------------------------------------------------------------------
+  /**
+   * draw the animated zooming pictures on the buffer
+   * @param a list of images that is used for animation pictures
+   * @param observer
+   */
+  public void prepare_zooming(Image[] buffer, ImageObserver observer)
+  {
+    _zoomingImages = buffer;
+    prepare_zooming_buffer(observer);
+    create_zooming_picts();
+  }
+
+  /**
+   * draw the border on each buffer
+   * @param a list of buffer
+   */
+  private void prepare_zooming_buffer(ImageObserver observer)
+  {
+    Graphics2D g;
+    if (_zoomingImages==null)
+    {
+      System.err.println("Error - zooming buffer cannot be empty!!!");
+    }
+    else
+    {
+      //-- copy the static view to one buffer
+      g = (Graphics2D) _zoomingImages[0].getGraphics();
+      g.drawImage(_treemapView,0,0,observer);
+      //-- clear the main treemap area and leave the border
+      g.setColor(Color.white);
+      g.fill(new Rectangle.Float(_upperX, _upperY, _width, _height));
+      //-- copy that buffer to other buffers
+      for(int k=1;k<_zoomingImages.length;k++)
+      {
+        g = (Graphics2D) _zoomingImages[k].getGraphics();
+        g.drawImage(_zoomingImages[0],0,0,observer);
+      }
+    }
+  }
+
+
+  private void create_zooming_picts()
+  {
+    int[] edgeSize = {0,0,0,0};
+    float w_y, e_y, n_x, s_x, portion, tmp;
+    ZRect[] tmpRects;
+    Graphics tmpGph;
+    /**
+     * pre-calculate the edgeSize
+     * the edgeSize represents the total number of docs on each border
+     */
+    for(int i=0;i<_gpRects.length;i++)
+      if (!_gpRects[i].isChosen())
+        edgeSize[_gpRects[i].getDirection()] += _gpRects[i].getCapacity();
+      else
+        for(int j=0;j<_cRects[i].length;j++)
+          if(!_cRects[i][j].isChosen())
+            edgeSize[_gpRects[i].getDirection()] += _cRects[i][j].getCapacity();
+
+    //-- initise the starting points on each side
+    w_y=_upperY;                e_y=_upperY+__borderheight;
+    n_x=_upperX+__borderwidth;  s_x=_upperX;
+
+
+    for(int i=0;i<_gpRects.length;i++)
+    {
+      //-- prepare zooming the whole group, if the user has NOT chosen a cluster
+      //-- within the group
+      if (!_gpRects[i].isChosen())
+      {
+        portion = ((float)_gpRects[i].getCapacity()) /
+                  ((float) edgeSize[_gpRects[i].getDirection()]);
+        if (_gpRects[i].isWest())
+        {
+          tmp = w_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+          w_y = tmp;
+        }
+        else if (_gpRects[i].isEast())
+        {
+          tmp = e_y + (portion * (float)(_height-__borderheight));
+          _gpRects[i].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+          e_y = tmp;
+        }
+        else if (_gpRects[i].isNorth())
+        {
+          tmp = n_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+          n_x = tmp;
+
+        }
+        else
+        {
+          tmp = s_x + (portion * (float)(_width-__borderwidth));
+          _gpRects[i].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+          s_x = tmp;
+        }
+
+        //-- generate a list of animation picture
+        tmpRects = _gpRects[i].zooming(_zoomingImages.length);
+
+        //-- draw the rectangles on the offsreen buffer
+        for(int j=0;j<_zoomingImages.length;j++)
+          tmpRects[j].paint(_zoomingImages[j].getGraphics());
+
+      }
+      else
+        //-- prepare zooming each cluster within the group
+        for (int j=0;j<_cRects[i].length;j++)
+          if (!_cRects[i][j].isChosen())
+          {
+            portion = ((float) _cRects[i][j].getCapacity()) /
+                      ((float) edgeSize[_gpRects[i].getDirection()]);
+            if (_gpRects[i].isWest())
+            {
+              tmp = w_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX, w_y, _upperX+__borderwidth, tmp);
+              w_y = tmp;
+            }
+            else if (_gpRects[i].isEast())
+            {
+              tmp = e_y + (portion * (float)(_height-__borderheight));
+              _cRects[i][j].translateTo(_upperX+_width-__borderwidth, e_y, _upperX+_width, tmp);
+              e_y = tmp;
+            }
+            else if (_gpRects[i].isNorth())
+            {
+              tmp = n_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(n_x, _upperY, tmp, _upperY+__borderheight);
+              n_x = tmp;
+            }
+            else
+            {
+              tmp = s_x + (portion * (float)(_width-__borderwidth));
+              _cRects[i][j].translateTo(s_x, _upperY+_height-__borderheight, tmp, _upperY+_height);
+              s_x = tmp;
+            }  //-- if
+
+            //-- generate a list of animation picture
+            tmpRects = _cRects[i][j].zooming(_zoomingImages.length);
+
+            //-- draw the rectangles on the offsreen buffer
+            for(int k=0;k<_zoomingImages.length;k++)
+              tmpRects[k].paint(_zoomingImages[k].getGraphics());
+          }  //-- if
+    }  //-- for
+  }  //-- prepare_zooming
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Cluster.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Cluster.java	(revision 21112)
@@ -0,0 +1,117 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class Cluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    static int data[][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+    static int finalCluster;
+
+    static void buildMatrix ()
+    {
+	float temp;
+	float theDiff;
+	int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	    keyword1 = keywords[word1];
+	    for (int word2=0; word2<MAX;word2++)
+            {
+		keyword2 = keywords[word2];
+                theDiff = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {	
+		    temp= data[doc][keyword1]-data[doc][keyword2];
+                    theDiff+= temp*temp;
+                }
+                temp = (int)Math.sqrt(theDiff);
+                matrix[word1][word2]=matrix[word2][word1]=(int)temp;
+            }
+         }
+	for (int row = 0;row <MAX; row++)
+	   matrix[row][row] = NOVALUE;
+    } 
+
+    static void findSmallest ()
+    {
+	int group = matrix[0].length-1;
+	int smallest = NOVALUE;
+	int smallRow = NOVALUE;
+	int smallCol = NOVALUE;
+	int row,col;
+	for (row =0; row<MAX; row++)
+	    for (col=0; col<MAX; col++)
+	       if (smallest > matrix[row][col])
+	        {
+		   smallest = matrix[row][col];
+		   smallRow = row;	
+		   smallCol = col;
+ 	        }
+	for (col=0; col<MAX; col++)
+	{
+	    if (matrix[smallRow][col] < matrix[smallCol][col])
+		matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = NOVALUE;
+	}
+	addClusters(smallRow, smallCol);
+	for (row =0; row<MAX; row++)
+        {
+	    matrix[row][smallCol] = NOVALUE;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	} 	
+    }
+
+    static void addClusters(int destination, int source)
+    {
+	int i;
+	int temp[] = new int[clusters[destination].length+clusters[source].length];
+	for (i = 0; i<clusters[destination].length;i++)
+	    temp[i] = clusters[destination][i];
+	for (i = 0; i<clusters[source].length;i++)
+	    temp[i+clusters[destination].length] = clusters[source][i];
+	clusters[destination] = temp;
+	clusters[source] = EMPTYARRAY;
+	finalCluster = destination;
+    }
+
+    static void preClusters()
+    {
+	clusters = new int[MAX][1];
+	for (int row =0; row <MAX; row++)
+	{
+	    clusters[row][0] = row;
+	}
+    }
+
+    static int[] maximunDistance (int[][] newData, int[] newKeywords, int documentCount)
+    {
+	data = newData;
+	keywords = newKeywords;
+	maxDocuments = documentCount;
+	MAX = keywords.length;
+	buildMatrix();
+	preClusters();
+	for (int i=1;i<MAX;i++)
+	{
+ 	    findSmallest();
+        }
+	int temp[] = new int[MAX];
+	for (int i=1;i<MAX;i++)
+	    temp[i] = keywords[clusters[finalCluster][i]];
+	return temp;
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/CvManyPoints.java	(revision 21112)
@@ -0,0 +1,1124 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.applet.*;
+import java.awt.event.*;
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.awt.Cursor;
+import vishnu.testvis.object.*;
+import vishnu.datablock.*;
+
+
+public class CvManyPoints extends Canvas
+{
+    protected Vishnu displayFrame;
+    protected RadVizPanel radVizPanel;
+    protected DataManager dataManager;
+    protected int [] docs;
+    
+    /** constant square shape flag */
+    static final int SQUARESHAPE = 0;
+    /** constant round shape flag */
+    static final int ROUNDSHAPE = 1;
+    /** constant diamond shape flag */
+    static final int DIAMONDSHAPE = 2;
+    /** constant maximum keywords allowed */
+    static final int ABSMAXKEYWORD = 1000;
+    /** constant maximum documents allowed */
+    static final int ABSMAXDOC = 1000;
+    /** count of the documents */
+    protected int documentCount = 1000;
+    /** circle array size */
+    protected int maxCircleCount = 40;
+    /** number of circles */
+    protected int circleCount = 40;
+    /** number of spokes */
+    protected int spokeCount = 16;
+    /** radius of the display in fp units */
+    protected float r = 10.0F;
+    /** locked flag */
+    protected boolean isLocked = false;
+    /** projected flag */
+    protected boolean isProjected = true;
+    /** document selected flag */
+    protected boolean atDoc = false;
+    /** grid flag */
+    protected boolean displayGrid = false;
+    /** rim flag */
+    protected boolean displayRim = true;
+    /** doc point shape flag */
+    protected int docsShape = SQUARESHAPE;
+    /** edge node shape flag */
+    protected int nodesShape = ROUNDSHAPE;
+    /** colour of nodes */
+    protected Color defNodesColor = Color.black;
+    /** colour of docs */
+    protected Color defDocsColor = Color.orange;
+    /** colour of highlighted nodes */
+    protected Color defHighlight = Color.blue;
+    /** colour of selected documents */
+    protected Color defSelect = Color.red;
+    /** Radius divided by radians */
+    protected float magic = (float)((double)r / (Math.PI / 2.0) );
+    /** projection factor */
+    protected float factor = 3F;
+    /** image object used for double buffering graphics */
+    protected Image image=null;
+    /** graphics used for double buffering graphics */
+    protected Graphics gImage=null;
+    /** double buffered image size */
+    protected int imgW,imgH;
+    /** dimensions of the canvas */
+    protected Dimension dim;
+    /** the integer location of 0,0 fp axis */
+    protected int centerX, centerY;
+    /** virtual pixel size and canvas fp dimensions */
+    protected float pixelSize, rWidth = 25.0F, rHeight = 25.0F;
+    /** array of edge node locations */
+    protected Point2D[] points= null;
+    /** selected edge node */
+    protected Point2D tp =null;
+    /** selected edge node index */
+    protected int tpIndex = 0;
+    /** angle stored for rotating edge nodes */
+    protected float lastAngle;
+    /** Full 2D linked list of data */
+    
+    protected Pt2DNode matrix= null;
+    /** Array of partial lists for each key word */
+    protected Pt2DNode columns[] = null;
+    /** The currently selected document if any */
+    protected Pt2DNode selectDoc = null;
+    /** A class that stores the keywords and whether they are displayed */
+    protected KeyWordArray keyWordArray;
+    /** Array of linked lists that stores the data to be displayed */
+    protected SparseMatrix theDataMatrix;
+    /** A class that maps displayed keywords to stored keywords */
+    protected MapKeyWords mapping;
+    /** A two dimensional array of integers used for clustering */
+    //protected int [][] fullMatrix;
+    protected float [][] fullMatrix;
+    /** array of data to draw target */
+    protected int circles[][] = new int[maxCircleCount][4];
+    /** array of data to draw rim */
+    protected int rim[] = new int[4];
+    /** array of data to draw spokes */
+    protected int spokes[][] = new int [spokeCount][2];
+    /** font for the mouse 'balloon' */
+    protected Font balloonFont = null;
+    /** metrics of the font for the mouse 'balloon' */
+    protected FontMetrics balloonFontMetrics = null;
+    /** Flag if drawing needs to just pasted from the buffer or redrawn */
+    protected boolean isDirty = true;
+    /** Scale the non-projected output if it goes off the page */
+    protected float scale=1.0F;
+    /** Last scale used when circles calculated (Flat mode only) */
+    protected float lastCircleScale = 1.0F;
+
+    /**
+     *  Rotate a Point2D around the 0.0,0.0 axis by phi angle (in radians)
+     *  @param Point2D around the 0.0,0.0 axis by
+     *  @param phi angle (in radians)
+     */
+    protected void rotatePt(Point2D pt,float phi)
+    {
+        float c = (float)Math.cos(phi), s = (float)Math.sin(phi),
+            r11 = c, r12 = s, r21 = -s, r22 = c;
+        float xNew = pt.x * r11 + pt.y * r21,
+            yNew = pt.x * r12 + pt.y * r22;
+        pt.x = xNew; pt.y = yNew;
+	
+    }
+    /**
+    * This is a public gateway for export of the keyWordsArray class
+    */
+    public KeyWordArray getKeyWords()
+    {
+        return keyWordArray;
+    }
+
+
+    /**
+    *  Fill the integer array circles[circleCount][4] with the correct numbers for
+    *  circleCount concentric circles about the 0.0,0.0 axis
+    */
+
+    protected void setCircles()
+    {
+        float  gap =r/4.0F, t = r/4.0F;
+        rim[0]=iX(-r); rim[1]=iY(r);
+        rim[2]= (int) (2.0F * r / pixelSize);
+        rim[3] = rim[2];
+        for (int count=0;count < circleCount; count++)
+        {
+            Point2D pc = new Point2D(-t,0.0F);
+            projection(pc);
+            circles[count][0]=iX(pc.x);
+            circles[count][2]=(int)((-pc.x) * 2.0F /pixelSize);
+            pc.x=0.0F; pc.y= t;
+            projection(pc);
+            circles[count][1]=iY(pc.y);
+            circles[count][3]=(int)(pc.y * 2.0F /pixelSize);
+            t += gap;
+        }
+        if (!isProjected) lastCircleScale=scale;
+    }
+
+    /** Create an array of linked lists that point only to the tuples affected by
+    *   each particular node, thus allowing faster update of the the locations
+    *   and easy display of only the tuples on that column of data, the membership
+    *   of these lists is subset of the larger list at 'matrix'
+    */
+    protected void setColumns()
+    {
+        int keyWordCount=keyWordArray.getNumWords();
+        columns = new Pt2DNode[keyWordCount];
+        Pt2DNode tails[] = new Pt2DNode[keyWordCount];
+        Pt2DNode k = null;
+        for (int i = 0; i< keyWordCount;i++) tails[i] = columns[i]=null;
+
+        if(keyWordCount==0) return;
+        Pt2DNode m = matrix;
+        SparseNode j=null;
+
+        // This may appear a bit obsure;
+
+        // columns[] is the start of an array of linked lists representing
+        // each of the keywords. tails[] is a parallel linked list representing the
+        // last added node to the columns[] lists;
+
+        // m loops through the rows and j iterates through the columns of each row
+        // when j finds a reference to a particular column[index] row m is at is added to
+        // the linked list for that column[index]
+
+        while (m != null)
+        {
+            j = m.tuple.row;
+            while (j != null)
+            {
+                k = new Pt2DNode(m.tuple);
+                int column = j.getColumn();
+                if(column > keyWordCount)
+                    System.out.println("col "+ column + " of " + keyWordCount);
+                else
+                {
+                    if (tails[column]!=null)
+                        tails[column].setNext(k);
+                    else columns[column] = k;
+                    tails[column]=k;
+                }
+                j=j.getNext();
+            }
+            m=m.getNext();
+        }
+    }
+    
+    /** Build a linked list of Points and tuples down connected to lists of nodes in the
+     * tuples across.
+     * @param dataItems the list
+     * @param refs[] the document titles
+     * @param docCount the number of documents
+     * @param wordCount the number of visible words
+     */
+    protected void setDatalists(SparseMatrix dataItems, String refs[],int docCount, int wordCount)
+    {
+        Pt2DNode m = null, n=null;
+        boolean state;
+        for (int co=0;co<docCount;co++)
+        {
+            SparseNode start = dataItems.getRow(co);
+            Point2D pn = new Point2D(0.0F,0.0F);
+            state = dataItemXY(pn,start);
+
+            n = new Pt2DNode(pn.x,pn.y,co,start,refs[co]);
+            if (state) projection(n.tuple.pnt);
+            n.tuple.state =state;
+            if (state)
+            {
+                n.tuple.iPnt.x=iX(n.tuple.pnt.x);
+                n.tuple.iPnt.y=iY(n.tuple.pnt.y);
+            }
+            if (m != null) m.setNext(n);
+            else matrix = n;
+            m=n;
+        }
+        setColumns();
+    }
+
+    /** Travers the linked lists setting new location values for the document points
+     * @param ma a linked list of document nodes
+     */
+    protected boolean setDocPoints(Pt2DNode ma)
+    {
+        while (ma != null)
+        {
+            ma.tuple.state = dataItemXY(ma.tuple.pnt,ma.tuple.row);
+            if (ma.tuple.state)
+            {
+                //c++;
+                projection(ma.tuple.pnt);
+                ma.tuple.iPnt.x=iX(ma.tuple.pnt.x);
+                ma.tuple.iPnt.y=iY(ma.tuple.pnt.y);
+                if (ma.tuple.iPnt.x<0 || ma.tuple.iPnt.x>dim.width ||
+                    ma.tuple.iPnt.y<0 || ma.tuple.iPnt.y>dim.height)
+                    return  true;
+            }
+            ma=ma.getNext();
+        }
+        return false;
+    }
+
+    /** Fill the integer array spokes[16][2] with the correct numbers for
+    *   16 radial points on the outermost circle, by defining a point on
+    *  the circle and rotating it by PI/8 radians 16 times
+    */
+    protected void setSpokes()
+    {
+        float phi=(float)Math.PI/(float)(spokeCount/2);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < spokeCount;count++)
+        {
+            spokes[count][0]=iX(pt.x);
+            spokes[count][1]=iY(pt.y);
+            rotatePt(pt,phi);
+        }
+    }
+
+    /** set the key word nodes in an even circle round the rim */
+    protected void setKeyNodes()
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        float phi=(float)Math.PI/((float)keyWordCount/2.0F);
+        Point2D pt = new Point2D(0.0F,r);
+        for (int count=0;count < keyWordCount;count++)
+	    {
+		points[count] = new Point2D(pt.x,pt.y);
+		rotatePt(pt,phi);
+	    }
+    }
+    
+    /** find the x y coordinate of a tuple of data
+    *   @param theP point to be set
+    *   @param tuple linked list of data
+    *   @return true if tuple contains a reference to a visible keynode
+    */
+    protected boolean dataItemXY(Point2D theP, SparseNode tuple)
+    {
+        int index;
+        boolean result=false;
+        theP.x=0.0F; theP.y = 0.0F;
+        while (tuple != null)
+        {
+            if (keyWordArray.getState(tuple.getColumn()))
+            {
+                result = true;
+                index = mapping.mapWordsToNodes(tuple.getColumn());
+                theP.x += tuple.getValue()/r * points[index].x;
+                theP.y += tuple.getValue()/r * points[index].y;
+            }
+            tuple=tuple.getNext();
+        }
+        return result;
+    }
+
+    /** set the text balloon label at the correct location, to the correct size
+    *   and with the correct text
+    *   @param theString the text to display
+    *   @param x the x coordinate of the mouse
+    *   @param y the y coordinate of the mouse
+    */
+    void setBalloon(String theString,int x,int y)
+    {
+        Rectangle bounds1 = getBounds();
+	
+        x-=(bounds1.x)-30; y+=(bounds1.y)+20;
+	
+        radVizPanel.docLabel.setText(theString);
+        int width = balloonFontMetrics.stringWidth(theString)+2;
+        int height = balloonFontMetrics.getMaxAscent()+2;
+        radVizPanel.docLabel.setBounds(x,y,width,height);
+    }
+
+    /**  recalculate edge points and document positions in the light of
+    *   of changes to the keywords
+    */
+    public void recalc()
+    {
+        mapping = new MapKeyWords(keyWordArray);
+        tp =null;
+        scale=1.0F;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+        repaint();
+    }
+
+    /**
+    * constructor sets everything up gets creates an instance of DataLink and
+    * gets the data matrix and the keywords calls code to initialise display
+    * @param DisplayFrame the central object of the display
+    * @param width of canvas
+    * @param height of canvas
+    * @throws IOException
+    */
+    public CvManyPoints(Vishnu df) throws IOException
+    {
+        setVisible(false);
+       	this.displayFrame = df;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+	
+       	documentCount=0;
+       	int keyWordCount=0;
+        keyWordArray = new KeyWordArray();
+	
+       	initgr();
+       	setCircles();
+       	setSpokes();
+       	/**
+       	* add an anonymous class to listen for size changed events
+       	*/
+	
+        addComponentListener(new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt)
+		{
+		    initgr();
+		    setCircles();
+		    setSpokes();
+		    while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+		    repaint();
+		}
+	    });
+	
+        // From Java in a Nutshell
+        // listen for mouse button pressed event and if on a node point
+        // record the current angle and set the value of tp
+        /**
+	 * add an anonomous class to listen for mouse pressed events
+	 */
+        addMouseListener(new MouseAdapter()
+	    {
+		public void mousePressed(MouseEvent evt)
+		{
+		    int intX = evt.getX();
+		    int intY = evt.getY();
+		    float x = fx(intX), y = fy(intY);
+		    tp = findPoint(x,y);
+		    if (tp == null){
+			selectDoc = findDoc(intX, intY);
+			if (selectDoc != null){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			}   
+			//else theOuterFrame.setDocSelectMenu("No Document Selected      ",-1);
+			repaint();
+			return;
+		    }
+		    lastAngle=(float)Math.atan2(x,y);
+		    repaint();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int intX = event.getX();
+		    int intY = event.getY();
+		    
+		    selectDoc = findDoc(intX, intY);
+		    if (selectDoc != null){
+			if(event.getClickCount()>1){
+			    StringBuffer buf = new StringBuffer("Doc: ");
+			    buf.append(selectDoc.tuple.ref);
+			    buf.append("  ");
+			    if (buf.length() > 30){
+				buf.setLength(30);
+				buf.append("... ");
+			    }
+			    String theString = new String(buf);
+			    //theOuterFrame.setDocSelectMenu(theString,selectDoc.tuple.index);
+			    dataManager.fetch(selectDoc.tuple.index);
+			}
+		    }
+		}
+		
+		public void mouseReleased(MouseEvent event)
+		{
+		    Cursor releaseCursor = new Cursor(Cursor.DEFAULT_CURSOR);
+		    displayFrame.radVizPanel.setCursor(releaseCursor);
+		}
+	    });
+	
+        // listen for mouse button dragged event and if tp is set
+        // work out a new position for tp and the data points
+        /**
+	 * add an anonomous class to listen for mouse motion events
+	 */
+        addMouseMotionListener(new MouseMotionAdapter(){
+		public void mouseDragged(MouseEvent evt)
+		{
+		    Cursor dragCursor = new Cursor(Cursor.HAND_CURSOR);
+		    displayFrame.radVizPanel.setCursor(dragCursor);
+		    
+		    float x = fx(evt.getX()), y = fy(evt.getY());
+		    if (tp!=null){
+			float thisAngle = (float)Math.atan2(x,y);
+			if(isLocked){
+			    float phi = lastAngle-thisAngle;
+			    rotatePt(tp, phi);
+			}
+			else{
+			    tp.x = x;
+			    tp.y = y;
+			}
+			if (columns!= null){
+			    int index = mapping.mapNodesToWords(tpIndex);
+			    if (columns[index]!= null){
+				if(setDocPoints( columns[index] ) && !isProjected){
+				    do{
+					scale*=0.9;
+				    } while(setDocPoints( matrix ));
+				}
+				repaint();
+			    }
+			}
+			lastAngle=thisAngle;
+		    }
+		}
+		
+		
+		
+		/** listen for mouse movement and try and find a document
+		 * report its presence with a balloon label or ensure
+		 * the balloon label is hidden
+		 * @param evt the mouse event
+		 */
+		public void mouseMoved(MouseEvent evt)
+		{
+		    System.out.println("Moving");
+		    /* int intX = evt.getX();
+		    int intY = evt.getY();
+		    Pt2DNode theNode = findDoc(intX, intY);
+		    
+		    if (theNode != null){
+			if (atDoc) radVizPanel.docLabel.setVisible(false);
+			String theString = new String("Doc: " + theNode.tuple.ref);
+			setBalloon(theNode.tuple.ref,intX,intY);
+			radVizPanel.docLabel.setVisible(true);
+			atDoc = true;
+		    }
+		    else{
+			if (atDoc){
+			    radVizPanel.docLabel.setVisible(false);
+			}
+			atDoc = false;
+			}*/
+		    return;
+		}
+	    });
+    }
+
+    /* the Data loading happens in here now */
+    
+    public void update()
+    {
+        int maxWordsCount;
+        this.radVizPanel = displayFrame.radVizPanel;
+        this.dataManager = displayFrame.dataManager;
+
+        //if(dataManager.getDocCount()==0)return;
+        selectDoc = null;
+        tp =null;
+       	docs = dataManager.getDocs();
+       	documentCount = docs.length;
+        String [] words = dataManager.getWords();
+        maxWordsCount = words.length;
+        //maxWordsCount= in.setKeywordCount(maxWordsCount);
+        int keyWordCount = Math.min(maxWordsCount,12);
+       	keyWordArray = new KeyWordArray(words,words.length);
+        maxWordsCount = keyWordArray.getNumWords();
+
+
+       	mapping = new MapKeyWords(keyWordArray);
+
+        String titles[] = dataManager.getDescriptions();
+        if(titles==null) System.out.println("no descriptions read in");
+        points= new Point2D[keyWordCount];
+        balloonFont = new Font("Dialog", Font.PLAIN, 16);
+        balloonFontMetrics = getFontMetrics(balloonFont);
+        theDataMatrix = dataManager.getMatrix();
+        fullMatrix = theDataMatrix.getExpanded();
+        setKeyNodes();
+        setDatalists(theDataMatrix,titles,documentCount,maxWordsCount);
+        recalc();
+    }
+    /** Cluster the words round the edge
+    * @param type the type of clustering
+    */
+    public void cluster()
+    {
+        if(documentCount ==0) return;
+        int keyWordCount=keyWordArray.getNumEnabled();
+        int keylist[] = new int[keyWordCount];
+
+        for (int c=0;c<keylist.length;c++)
+        {
+            keylist[c]=mapping.mapNodesToWords(c);
+        }
+        if(fullMatrix==null) System.out.println("full matrix is null");
+        else
+        {
+            if(fullMatrix[0]==null) System.out.println("full matrix[0] is null");
+        }
+
+        keylist = HitCluster.maximunDistance (fullMatrix, keylist, documentCount);
+
+        mapping.reMapKeyWords(keylist,keyWordArray);
+
+        tp =null;
+        tpIndex = 0;
+        points= new Point2D[keyWordArray.getNumEnabled()];
+        setKeyNodes();
+        while (setDocPoints(matrix) && !isProjected) scale *=0.9;
+        repaint();
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new dimensions
+    */
+    public void setSize(Dimension d)
+    {
+        super.setSize(d);
+        initgr();
+        setCircles();
+        setSpokes();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9F ;
+    }
+    /**
+    * Overides setSize so that the drawing is correctly scaled on a setSize call
+    * @param new width
+    * @param new height
+    */
+    public void setSize(int x, int y)
+    {
+        Dimension d = new Dimension(x,y);
+        setSize(d);
+    }
+
+
+
+    /**
+    * Tidy up the image before canvas is destroyed a bit like a destructor in C
+    */
+    public void dispose()
+    {
+        gImage.dispose();
+    }
+
+
+    /** Set the shape of the edge word nodes
+    * @param state selects the shape
+    */
+    void setNodeShape(int state)
+    {
+        nodesShape = state;
+        repaint();
+    }
+
+    /** Set the shape of the centre document nodes
+    *  @param state selects the shape
+    */
+    void setDocShape(int state)
+    {
+        docsShape = state;
+        repaint();
+    }
+
+    /** switches the projection on and off
+    * @param state is the switch
+    */
+    public void projection(boolean state)
+    {
+        // to do: code goes here.
+        isProjected = state;
+        circleCount = state?40:4;
+        setCircles();
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Display or not the grid
+    * @param switches on and off
+    */
+
+    public void grid(boolean state)
+    {
+        displayGrid=state;
+        repaint();
+    }
+
+    /**
+    * Display or not the rim
+    * @param switches on and off
+    */
+    public void rim(boolean state)
+    {
+        displayRim=state;
+        repaint();
+    }
+
+    /**
+    * Lock or not the edge nodes so they can only be moved in a circle
+    * @param switches on and off
+    */
+    public void lock(boolean state)
+    {
+        isLocked = state;
+    }
+
+
+    /**
+    * Flattens the projection slightly
+    *
+    */
+    public void flatter()
+    {
+        if(!isProjected)
+            scale/=0.9;
+        else
+        {
+            factor *=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix)&& !isProjected) scale*=0.9;
+        repaint();
+    }
+
+    /**
+    * Heightens the projection slightly
+    *
+    */
+    public void higher()
+    {
+        if(!isProjected)
+            scale*=0.9;
+        else
+        {
+            factor /=0.9;
+            setCircles();
+        }
+        while (setDocPoints(matrix) && !isProjected) scale*=0.9;
+        repaint();
+    }
+
+
+
+
+    /**
+    * Translate to integer raster coordinate X from
+    * floating point coord X based around a notional centre
+    * @param x floating point x coordinate
+    * @return integer X coordinate
+    */
+    int iX(float x){return Math.round(centerX + x/pixelSize);}
+    /**
+    * Translate to integer raster coordinate Y from
+    * floating point coord Y based around a notional centre
+    * @param y floating point y coordinate
+    * @return integer Y coordinate
+    */
+    int iY(float y){return Math.round(centerY - y/pixelSize);}
+    /**
+    * Translate from integer raster coordinate X to
+    * floating point coord X based around a notional centre
+    * @param x integer x coordinate
+    * @return floating point X coordinate
+    */
+    float fx(int X){return (X - centerX) * pixelSize;}
+    /**
+    * Translate from integer raster coordinate Y to
+    * floating point coord Y based around a notional centre
+    * @param y integer y coordinate
+    * @return floating point Y coordinate
+    */
+    float fy(int Y){return (centerY - Y) * pixelSize;}
+
+    /**
+    * find if an edge keyword node is at the x y location
+    * @param x fp coordinate
+    * @param y fp coordinate
+    * @return fp Point at location or null
+    */
+    public Point2D findPoint(float x, float y)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+        for (int i=0;i<keyWordCount;i++)
+        {
+            int intX = iX(x), intY = iY(y),
+                lX = iX(points[i].x)-3, hX = lX+6,
+                lY = iY(points[i].y)-3, hY = lY+6;
+
+            if (intX > lX && intX < hX && intY > lY && intY < hY)
+            {
+
+                tpIndex = i;
+                return points[i];
+            }
+        }
+        return null;
+    }
+
+    /**
+    * find if a Doc node is at the x y location
+    * @param x int coordinate
+    * @param y int coordinate
+    * @return fp Point at location or null
+    */
+    public Pt2DNode findDoc(int intX, int intY)
+    {
+        Pt2DNode thePnt=matrix;
+        while (thePnt!= null)
+        {
+            if (thePnt.tuple.state)
+            {
+                int   lX = thePnt.tuple.iPnt.x-3, hX = lX+6,
+                    lY = thePnt.tuple.iPnt.y-3, hY = lY+6;
+                if (intX > lX && intX < hX && intY > lY && intY < hY) return thePnt;
+            }
+            thePnt = thePnt.getNext();
+        }
+        return null;
+    }
+    /** project the point radially from the centre using the arctan function
+    *  if isProjected is true or scale them it false
+    *  @param point to be projected
+    *  @param ingore scaling flag
+    */
+    protected void projection(Point2D p)
+    {
+        if (!isProjected)
+        {
+            if(scale!=1.0)
+            {
+                p.x*=scale;
+                p.y*=scale;
+            }
+            return;
+        }
+        float newRad;
+        float rad = (float)Math.sqrt(p.x*p.x+p.y*p.y);
+        float sine = p.y/rad;
+        float cosine = p.x/rad;
+        newRad =(float)Math.atan(rad/(factor*Math.PI))*magic;
+        p.x = newRad * cosine;
+        p.y = newRad * sine;
+    }
+
+    /** set up graphic settings such as centre and pixelsize
+    */
+    protected void initgr()
+    {
+        dim = getSize();
+        int maxX = dim.width - 1, maxY = dim.height - 1;
+        pixelSize = Math.max(rWidth/maxX, rHeight/maxY);
+        centerX = maxX/2; centerY = maxY/2;
+    }
+
+
+    /////////Athina has been here!!!!!///////////////////////////////////////
+
+    /** draw a for document node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    protected void drawItemPoint(int x, int y, Graphics g)
+    {
+
+        if (docsShape == SQUARESHAPE)
+            oval(g,x,y);
+        else if (docsShape == ROUNDSHAPE)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+    /**
+    *   draw a for keyword node shape at raster x y
+    *   @param x location
+    *   @param y location
+    *   @param g Graphics object
+    */
+    void drawNodes(int x, int y, Graphics g)
+    {
+        if (nodesShape == 0)
+            oval(g,x,y);
+        else if (nodesShape == 1)
+            rect(g,x,y);
+        else
+            diamond(g,x,y);
+
+    }
+
+
+    /**
+    *  draw the key name near the edgenode
+    *  @param g Graphics class
+    *  @param count index of the name to draw
+    *  @param select if the node is selected.
+    */
+    void drawKeyName(Graphics g, int count, boolean select)
+    {
+        FontMetrics fontMetrics = g.getFontMetrics();
+        Font font = new Font("arial", Font.PLAIN, 16);
+        String word = keyWordArray.getWord(mapping.mapNodesToWords(count));
+        int posX= iX(points[count].x)-2, posY= iY(points[count].y)-2;
+        int width = fontMetrics.stringWidth(word);
+        if (points[count].x < 0.0)
+            posX -= width - (int) (points[count].x/2.00);
+        else
+            posX += (int) (points[count].x/2.00);
+
+        if (points[count].y < 0.0)
+            posY += fontMetrics.getMaxAscent()- (int)(points[count].y/2.00);
+        else
+            posY -= (int) (points[count].y/2.00);
+        g.setFont(font);
+        g.drawString(word,posX,posY);
+        if (select){ g.drawRect(posX-2,posY-15,width+4,20);}
+    }
+
+    /**
+    * draw a box at the edge nodes
+    * @param g Graphics object
+    */
+    protected void drawEdgeNodes(Graphics g)
+    {
+        int keyWordCount=keyWordArray.getNumEnabled();
+
+        for (int count=0;count < keyWordCount; count++)
+        {
+            drawNodes(iX(points[count].x), iY(points[count].y), g);
+            drawKeyName(g,count,false);
+
+        }
+    }
+    /**
+    * Draw the documents that have the selected key word in them
+    * @param g Graphics object
+    */
+    protected void drawSelect(Graphics g)
+    {
+        if (tp!=null)
+        {
+            int x = iX(points[tpIndex].x), y = iY(points[tpIndex].y);
+            g.setColor(defSelect);
+            drawNodes(x, y, g);
+
+            Pt2DNode m = columns[mapping.mapNodesToWords(tpIndex)];
+            while (m != null)
+            {
+                x=m.tuple.iPnt.x;
+                y=m.tuple.iPnt.y;
+
+                drawItemPoint(x, y, g);
+
+                m=m.getNext();
+            }
+            g.setColor(defDocsColor);
+
+        }
+    }
+    /**
+    * draw the key word nodes that feature in the selected document
+    */
+    protected void drawDocSelect(Graphics g)
+    {
+        if (selectDoc==null) return;
+        g.setColor(defHighlight);
+        int x =selectDoc.tuple.iPnt.x, y =selectDoc.tuple.iPnt.y;
+        drawItemPoint(x, y, g);
+
+        SparseNode fn = selectDoc.tuple.row;
+        while (fn != null)
+        {
+            if (keyWordArray.getState(fn.getColumn()))
+            {
+                int index = mapping.mapWordsToNodes(fn.getColumn());
+                drawNodes(iX(points[index].x), iY(points[index].y), g);
+                drawKeyName(g,index,true);
+            }
+            fn=fn.getNext();
+
+        }
+        g.setColor(defNodesColor);
+
+    }
+
+    /**
+    * draw all the document crosses
+    * @param g Graphics object
+    */
+
+    protected void drawDocXs(Graphics g)
+    {
+        Pt2DNode m = matrix;
+        while (m != null)
+        {
+            if (m.tuple.state) drawItemPoint(m.tuple.iPnt.x, m.tuple.iPnt.y, g);
+            m=m.getNext();
+        }
+    }
+
+    /**
+    * draw the target
+    * @param g Graphics object
+    */
+
+    protected void drawTarget(Graphics g)
+    {
+        if (!isProjected && lastCircleScale!=scale)
+            setCircles();
+
+        if (displayRim)
+        {
+            g.setColor(Color.darkGray);
+            g.drawOval(rim[0],rim[1],rim[2],rim[3]);
+        }
+        if (displayGrid)
+        {
+            g.setColor(Color.lightGray);
+            for (int count=0;count < circleCount; count++)
+                g.drawOval(circles[count][0],circles[count][1],circles[count][2],circles[count][3]);
+            for (int count=0;count < spokeCount; count++)
+                g.drawLine(centerX, centerY,spokes[count][0],spokes[count][1]);
+        }
+    }
+
+    /**
+    * Repaint the screen overides canvas.repaint()
+    */
+
+    public void repaint()
+    {
+        isDirty=true;
+        super.repaint();
+    }
+
+    /**
+    * Repaint the screen overides canvas.update()
+    * to stop flickering
+    * @param g Graphics object
+    */
+
+    public void update(Graphics g)
+    {
+        paint(g);
+    }
+
+    /**
+    * draw a filled square with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+
+    public void rect(Graphics g, int x, int y)
+    {
+        g.drawRect(x-2,y-2,4,4);
+        g.fillRect(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled circle with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void oval(Graphics g, int x, int y)
+    {
+        g.drawOval(x-2,y-2,4,4);
+        g.fillOval(x-2,y-2,4,4);
+        repaint();
+    }
+
+    /**
+    * draw a filled diamond with a centre at x y
+    * @param g Graphics object
+    * @param x location
+    * @param y location
+    */
+    public void diamond(Graphics g, int x, int y)
+    {
+        int numpoints = 4;
+        int[] xpoints = new int[numpoints+1];
+        int[] ypoints = new int[numpoints+1];
+        for(int i=0; i < numpoints; i++) {
+            double angle = 2*Math.PI * i / numpoints;
+            xpoints[i] = (int)(x + 3*Math.cos(angle));
+            ypoints[i] = (int)(y - 3*Math.sin(angle));
+        }
+
+        g.fillPolygon(xpoints, ypoints, numpoints);
+        xpoints[numpoints] = xpoints[0];  ypoints[numpoints] = ypoints[0];
+        g.drawPolygon(xpoints, ypoints, numpoints+1);
+        repaint();
+    }
+
+    /**
+    * paint the whole thing
+    * @param g Graphics object
+    */
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Repaint CvManyPoints");
+        if(!isVisible()) return;
+        if (isDirty==true)
+        {
+            if (imgW != dim.width || imgH != dim.height)
+            {
+                if( gImage != null ) gImage.dispose();
+                imgW = dim.width; imgH = dim.height;
+                image = createImage(imgW, imgH);
+                gImage = image.getGraphics();
+            }
+            gImage.setColor(Color.white);
+            gImage.fillRect(0, 0, imgW, imgH);
+
+            drawTarget(gImage);
+            gImage.setColor(defNodesColor);
+            drawEdgeNodes(gImage);
+            gImage.setColor(defDocsColor);
+            drawDocXs(gImage);
+
+            if (tp!=null)
+                drawSelect(gImage);
+            if (selectDoc!=null)
+                drawDocSelect(gImage);
+
+            //gImage.drawRect(1,1,dim.width-1,dim.height-1);
+        }
+        g.drawImage(image, 0, 0, null);
+
+        isDirty=false;
+    }
+}
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/HitCluster.java	(revision 21112)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.util.*;
+import java.net.*;
+import java.io.*;
+import java.lang.*;
+import java.util.zip.*;
+
+class HitCluster
+{
+    static final int NOVALUE = 99999999;
+    static int MAX;
+    static final int EMPTYARRAY[] = {};
+    static int matrix[][];
+    // static int data[][];
+    static float data [][];
+    static int clusters[][];
+    static int keywords[];
+    static int maxDocuments;
+
+    static void buildMatrix ()
+    {
+	    int count;
+	    int keyword1, keyword2;
+    	matrix = new int[MAX][MAX];
+        for (int word1=0;word1<MAX;word1++)
+        {
+	        keyword1 = keywords[word1];
+	        for (int word2=word1+1; word2<MAX;word2++)
+            {
+ 		        keyword2 = keywords[word2];
+                count = 0;
+                for (int doc=0;doc<maxDocuments;doc++)
+                {
+		            if ( (data[doc][keyword1] > 0) && (data[doc][keyword2] > 0))
+			            count++;
+	            }
+                matrix[word1][word2]=matrix[word2][word1]=count;
+            }
+        }
+	    for (int row = 0;row <MAX; row++)
+	    {
+	        matrix[row][row] = NOVALUE;
+	        for (int col = 0; col <row; col++)
+		    matrix[row][col] = matrix[col][row];
+	    }
+    }
+
+    static void findLargest ()
+    {
+	    int group = matrix[0].length-1;
+	    int largest = -1;
+	    int smallRow = NOVALUE;
+	    int smallCol = NOVALUE;
+	    int row,col;
+	    for (row =0; row<MAX; row++)
+	        for (col=row+1; col<MAX; col++)
+	            if (largest < matrix[row][col])
+	            {
+		            largest = matrix[row][col];
+		            smallRow = row;
+		            smallCol = col;
+ 	            }
+	    for (col=0; col<MAX; col++)
+	    {
+	        if (matrix[smallRow][col] > matrix[smallCol][col])
+		        matrix[smallRow][col] = matrix[smallCol][col];
+            matrix[smallCol][col] = -1;
+	    }
+	    clusters[smallRow] = addClusters(smallRow, smallCol);
+	    clusters[smallCol] = EMPTYARRAY;
+	    for (row =0; row<MAX; row++)
+        {
+	        matrix[row][smallCol] = -1;
+            matrix[row][smallRow] = matrix[smallRow][row];
+	    }
+    }
+
+    static int[] addClusters(int c1, int c2)
+    {
+	    int i;
+	    int temp[] = new int[clusters[c1].length+clusters[c2].length];
+	    if (keywords[clusters[c1][0]] < keywords[clusters[c2][0]])
+	    {
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i] = clusters[c1][i];
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i+clusters[c1].length] = clusters[c2][i];
+	    }
+	    else
+	    {
+	        for (i = 0; i<clusters[c2].length;i++)
+	    	    temp[i] = clusters[c2][i];
+	        for (i = 0; i<clusters[c1].length;i++)
+	    	    temp[i+clusters[c2].length] = clusters[c1][i];
+	    }
+	    return temp;
+    }
+
+    static void preClusters()
+    {
+	    clusters = new int[MAX][1];
+	    for (int row =0; row <MAX; row++)
+	    {
+	        clusters[row][0] = row;
+	    }
+    }
+
+    static int[] maximunDistance (float[][] newData, int[] newKeywords, int documentCount)
+    {
+	    data = newData;
+	    keywords = newKeywords;
+	    maxDocuments = documentCount;
+	    if (documentCount > data.length)
+	    {
+	        System.out.println ("WARNING matrix sent is shorter the document Count!");
+	        System.out.println ("I will use the shorter length only.");
+	        documentCount = data.length;
+        }
+	    MAX = keywords.length;
+	    buildMatrix();
+	    preClusters();
+	    for (int i=1;i<MAX;i++)
+	    {
+ 	        findLargest();
+        }
+	    int temp[] = new int[MAX];
+	    for (int i=0;i<MAX;i++)
+	    {
+	        temp[i] = keywords[clusters[0][i]];
+//System.out.println (i+" = "+keywords[i]+" = "+clusters[0][i]);
+    	}
+	    return temp;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Hlink.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Hlink.java	(revision 21112)
@@ -0,0 +1,118 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.JEditorPane;
+import javax.swing.*;
+import javax.swing.text.html.*;
+import javax.swing.event.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.Vector;
+
+public class Hlink extends JEditorPane
+{
+    public JScrollPane linkPane;
+    public Vishnu currentMain;
+    public int index = 0;
+    public int numDisplayedDocs = 15;
+    private int width, length;
+
+    public Hlink(int width, int length , Vishnu currentMain)
+    {
+        this.currentMain=currentMain;
+        this.width=width;
+        this.length=length;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public Hlink()
+    {
+        this.currentMain=null;
+        this.width=500;
+        this.length=300;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public void jbInit() throws Exception
+    {
+        linkPane = new JScrollPane(this);
+        linkPane.setBorder(BorderFactory.createLoweredBevelBorder());
+        linkPane.setPreferredSize(new Dimension(width,length));
+        linkPane.setMinimumSize(new Dimension(width,length));
+        linkPane.setBorder(new javax.swing.border.EtchedBorder());
+        linkPane.getVerticalScrollBar().setUnitIncrement(10);
+
+        setLayout(new GridLayout(17,1));
+
+
+        setEditorKitForContentType("text/html",new HTMLEditorKit());
+        setEditorKit(new HTMLEditorKit());
+        setEditable(false);
+
+        addHyperlinkListener(new MyLinkListener());
+
+        setVisible(true);
+
+    }
+
+    public class MyLinkListener implements HyperlinkListener
+    {
+        public void hyperlinkUpdate(HyperlinkEvent he){
+            System.out.println("Inside hyperlinkevent ... ");
+            HyperlinkEvent.EventType type = he.getEventType();
+            System.out.println("Inside hyperlinkevent ... ");
+            if (type==HyperlinkEvent.EventType.ACTIVATED)
+                System.out.println("Mouse ACTIVATED -> " + he.getURL().toString());
+        }
+    }
+
+    public void update()
+    {
+       int [] docIDs = currentMain.dataManager.getDocs();
+        Vector subSet = new Vector();
+
+        for (int i = index ; i < index + numDisplayedDocs  ; i++)
+        {
+
+            // Check that the element exists
+            if (docIDs.length <= i) break;
+
+            subSet.addElement(new Integer(docIDs[i]));
+        }
+
+        String htmlDescription = currentMain.dataManager.getDescriptionStr(subSet);
+
+        setText(htmlDescription);
+
+    }
+
+    public void clear()
+    {
+        setText("");
+    }
+
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Ipoint2D.java	(revision 21112)
@@ -0,0 +1,17 @@
+package vishnu.testvis.visual;
+/** Integer point class 
+* (minimal)
+* @author Matthew Carey */
+
+public class Ipoint2D
+{
+   /** x value */
+   public int x;
+   /** y value */
+   public int y;
+   /** constructor 
+   * @param horizontal value
+   * @param vertical value
+   */
+   public Ipoint2D(int x, int y){this.x = x; this.y = y;}
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeyWordArray.java	(revision 21112)
@@ -0,0 +1,106 @@
+/** Class to store the keywords and their visible states
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+public class KeyWordArray
+{
+    /** constant default number enabled */
+    public static final int DEFAULT_ENABLED = 12;
+    /** the word array */
+    private String words[];
+    /** the enabled array */
+    private boolean enabled[];
+    /** the count */
+    private int count;
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    * @param number enabled
+    */
+    public KeyWordArray(String words[],int count,int numEnabled)
+    {
+        this.words = words;
+        this.count = count;
+        enabled = new boolean[count];
+        int theNumEnabled = count<numEnabled?count:numEnabled;
+        for (int c = 0; c < numEnabled && c < count; c++)
+            enabled[c]=true;
+        if (count > theNumEnabled)
+            for (int c = theNumEnabled; c < count; c++)
+                enabled[c]=false;
+    }
+    /**
+    * constructor
+    * @param word array
+    * @param count size
+    */
+    public KeyWordArray(String words[],int count)
+    {
+        this(words,count,DEFAULT_ENABLED);
+    }
+
+    public KeyWordArray()
+    {
+        this.count = 0;
+        this.words = null;
+        this.enabled = null;
+    }
+
+    /**
+    * Find the size of the array
+    * @return size of the array
+    */
+    public int getNumWords()
+    {
+        return count;
+    }
+
+    /**
+    * Find the word at index
+    * @param index of word
+    * @return word at index
+    */
+    public String getWord(int index)
+    {
+        if (index >=0 && index < count)
+           return this.words[index];
+        return "Error out of index";
+    }
+
+    /**
+    * Get state of word at index
+    * @param index of word
+    * @return state of word
+    */
+    public boolean getState(int index)
+    {
+        if (index >=0 && index < count)
+           return this.enabled[index];
+        return false;
+    }
+
+    /**
+    * Set state of word at index
+    * @param index of word
+    * @param new state of word
+    */
+    public void setState(int index, boolean state)
+    {
+        if (index >=0 && index < count)
+           this.enabled[index]=state;
+    }
+
+    /**
+    * Get number of enabled words
+    * @return number of enabled words
+    */
+    public int getNumEnabled()
+    {
+        int result = 0;
+        for (int c = 0; c < count; c++)
+            if (enabled[c]) result++;
+        return result;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/KeywordPanel.java	(revision 21112)
@@ -0,0 +1,188 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+import java.awt.List;
+import java.applet.*;
+import javax.swing.*;
+
+/** A panel that is added to the frame containing an on screen list of keywords to select
+*   and deselect
+*   @author Athina Economou
+*/
+public class KeywordPanel extends JPanel
+{
+    /** the base frame */
+    protected RadVizPanel f;
+    protected KeyWordArray keyWordArray;
+    protected int wordCount;
+    protected Vector  selArrayIndex;
+    protected Vector  listArrayIndex;
+    protected List available;
+    protected List selected;
+    protected int largeFont = 16;
+    protected int smallFont = 14;
+
+
+    /** Constructor
+    * @param drawing canvas
+    * @param base frame
+    */
+    public KeywordPanel(final CvManyPoints canvas1, RadVizPanel f)
+    {
+        this.f = f;
+
+        keyWordArray = canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selArrayIndex = new Vector();
+        listArrayIndex = new Vector();
+        available = new List(12, false);
+        selected = new List(12, false);
+        // mc
+        SymAction lSymAction = new SymAction();
+
+//        Panel base = new Panel();
+        setLayout(new BorderLayout(10,0));
+        //setSize(50,400);
+        JPanel labels = new JPanel();
+        Font font1 = new Font("Monospaced", Font.PLAIN, largeFont);
+        labels.setFont(font1);
+        labels.setForeground(Color.black);
+        labels.add(new Label("Available           Displayed"));
+
+        JPanel rightPanel = new JPanel();
+        Font font = new Font("Dialog", Font.PLAIN, smallFont);
+        rightPanel.setFont(font);
+        JPanel leftPanel = new JPanel();
+        leftPanel.setFont(font);
+
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+        available.addActionListener(lSymAction);
+        available.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    if(event==null) return;
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)listArrayIndex.elementAt(index);
+                            selected.add(keywords);
+                            selArrayIndex.addElement(result);
+
+                            available.remove(keywords);
+                            listArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                             keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } //try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    } // catch
+                } //itemstate changed
+            }); // ItemListener
+            //mc
+            selected.addActionListener(lSymAction);
+            selected.addItemListener(new ItemListener()
+            {
+                public void itemStateChanged(ItemEvent event)
+                {
+                    //System.out.println("calling getSeleced index");
+                    try
+                    {
+                        int index =((List)event.getItemSelectable()).getSelectedIndex();
+                    //System.out.println("Index");
+                    //System.out.println(index);
+                        if(index >= 0)
+                        {
+                            String keywords = ((List)event.getItemSelectable()).getSelectedItem();
+                            Integer result = (Integer)selArrayIndex.elementAt(index);
+
+                            available.add(keywords);
+                            listArrayIndex.addElement(result);
+
+                            selected.remove(keywords);
+                            selArrayIndex.removeElementAt(index);
+
+                            for (int c=0; c<keyWordArray.getNumWords(); c++)
+                                keyWordArray.setState(c,false);
+
+                            for (int c=0;c<(int)selArrayIndex.size();c++)
+                            {
+                                Integer result1 = (Integer)selArrayIndex.elementAt(c);
+                                int index1 = result1.intValue();
+                                keyWordArray.setState(index1,true);
+                            } // for
+                            canvas1.recalc();
+                        } // if
+                    } // try
+                    catch (ArrayIndexOutOfBoundsException e)
+                    {
+                    }
+                }
+            });
+
+            leftPanel.add(available);
+            rightPanel.add(selected);
+            add(labels, "North");
+            add(rightPanel, "East");
+            add(leftPanel, "West");
+            //base.add(text, "South");
+            setVisible(true);
+
+    }
+    public void update()
+    {
+        keyWordArray = f.canvas1.getKeyWords();
+        wordCount = keyWordArray.getNumWords();
+        selected.removeAll();
+        available.removeAll();
+        selArrayIndex.removeAllElements();
+        listArrayIndex.removeAllElements();
+        for (int c=0; c< wordCount;c++)
+            if (keyWordArray.getState(c))
+            {
+                selected.add(keyWordArray.getWord(c));
+                selArrayIndex.addElement(new Integer(c));
+            }
+            else
+            {
+                available.add(keyWordArray.getWord(c));
+                listArrayIndex.addElement(new Integer(c));
+            }
+    }
+
+
+    /** A dummy action listener to absorb double mouse clicks */
+    class SymAction implements java.awt.event.ActionListener
+    {
+        public void actionPerformed(java.awt.event.ActionEvent event)
+        {
+            Object object = event.getSource();
+            //System.out.println("Action event");
+        }
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/ListPanel.java	(revision 21112)
@@ -0,0 +1,127 @@
+
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class ListPanel extends JPanel
+{
+    JList documentsList = new JList();
+    BorderLayout borderLayout1 = new BorderLayout();
+    Border border1;
+    JLabel descriptionLabel = new JLabel();
+    Vishnu displayFrame;
+    JPanel southPanel = new JPanel();
+    JButton fetchButton = new JButton();
+
+    public ListPanel()
+    {
+        displayFrame=null;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+    public ListPanel(Vishnu d)
+    {
+        displayFrame=d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border1);
+        descriptionLabel.setFont(new java.awt.Font("Monospaced", 0, 14));
+        descriptionLabel.setText("Document descriptions");
+        documentsList.setFont(new java.awt.Font("Monospaced", 0, 12));
+        documentsList.setToolTipText("");
+        documentsList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+        documentsList.setVisibleRowCount(10);
+        documentsList.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                documentsList_mouseClicked(e);
+            }
+        });
+        fetchButton.setFont(new java.awt.Font("Monospaced", 0, 16));
+        fetchButton.setBorder(BorderFactory.createEtchedBorder());
+        fetchButton.setText("Fetch");
+        fetchButton.addMouseListener(new java.awt.event.MouseAdapter()
+        {
+
+            public void mouseClicked(MouseEvent e)
+            {
+                fetchButton_mouseClicked(e);
+            }
+        });
+        this.add(documentsList, BorderLayout.CENTER);
+        this.add(descriptionLabel, BorderLayout.NORTH);
+        this.add(southPanel, BorderLayout.SOUTH);
+        southPanel.add(fetchButton, null);
+    }
+
+    public void update()
+    {
+       documentsList.setListData(displayFrame.dataManager.getDescriptions());
+       repaint();
+    }
+
+    void documentsList_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseClicked(MouseEvent e)
+    {
+        int index = documentsList.getSelectedIndex();
+        if (index >=0)
+            displayFrame.dataManager.fetch(index);
+    }
+
+    void fetchButton_mouseEntered(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseExited(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mousePressed(MouseEvent e)
+    {
+
+    }
+
+    void fetchButton_mouseReleased(MouseEvent e)
+    {
+
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MapKeyWords.java	(revision 21112)
@@ -0,0 +1,78 @@
+package vishnu.testvis.visual;
+ /** This class is maps the display array of keywords onto the stored.
+  *  array
+  *  @author Matthew Carey</a>
+  */
+public class MapKeyWords
+{
+   /** array map for display to stored */
+   protected int nodesToWord[] = null;
+   /** array map for stored to display */
+   protected int wordsToNodes[] = null;
+   /**
+   * map for display to stored
+   * @param display index
+   * @return stored index
+   */
+   public int mapNodesToWords(int index)
+   {
+        return nodesToWord[index];
+   }
+   /**
+   * map for stored to display
+   * @param stored index
+   * @return the display index
+   */
+   public int mapWordsToNodes(int index)
+   {
+        return wordsToNodes[index];
+   }
+   /**
+   * constructor
+   * @param keywordarray instance containing list of key words and states
+   */
+   public MapKeyWords(KeyWordArray keyWordArray)
+   {
+        int c, n=0;
+        nodesToWord = null;
+        wordsToNodes = null;
+        nodesToWord = new int [keyWordArray.getNumEnabled()];
+        wordsToNodes = new int [keyWordArray.getNumWords()];
+        int totalKeyWord=keyWordArray.getNumWords();
+        for (c=0;c < totalKeyWord; c++)
+        {
+             if (keyWordArray.getState(c))
+             {
+                wordsToNodes[c]=n;
+                nodesToWord[n]=c;
+                n++;
+             }
+             else
+                wordsToNodes[c]=-1;
+        }
+   }
+   /**
+   * re-Map the KeyWords with a new vector
+   * @param array the new vector
+   * @param keywordarray instance containing list of key words and states
+   */
+   public void reMapKeyWords(int array[],KeyWordArray keyWordArray)
+   {
+        if (array.length != keyWordArray.getNumEnabled())
+            return;
+        int c;
+        for (c=0;c<array.length;c++)
+        {
+            if (!keyWordArray.getState(array[c]))
+               return;
+        }
+
+        for (c=0;c<array.length;c++)
+        {
+                wordsToNodes[array[c]]=c;
+                nodesToWord[c]=array[c];
+        }
+   }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/MultiLineLabel.java	(revision 21112)
@@ -0,0 +1,136 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import java.util.*;
+
+/**
+ * A custom component that displays multiple lines of text with specified
+ * margins and alignment.  In Java 1.1, we could extend Component instead
+ * of Canvas, making this a more efficient "Lightweight component"
+ * from Java Examples in a Nutshell,
+ * @author Flanagan, David - O'Rielly 1997 NY
+ */
+public class MultiLineLabel extends Canvas {
+  // User-specified attributes
+  protected String label;             // The label, not broken into lines
+  protected int margin_width;         // Left and right margins
+  protected int margin_height;        // Top and bottom margins
+  protected int alignment;            // The alignment of the text.
+  public static final int LEFT = 0, CENTER = 1, RIGHT = 2; // alignment values
+  // Computed state values
+  protected int num_lines;            // The number of lines
+  protected String[] lines;           // The label, broken into lines
+  protected int[] line_widths;        // How wide each line is
+  protected int max_width;            // The width of the widest line
+  protected int line_height;          // Total height of the font
+  protected int line_ascent;          // Font height above baseline
+  protected boolean measured = false; // Have the lines been measured?
+
+  // Here are five versions of the constructor
+  public MultiLineLabel(String label, int margin_width,
+                        int margin_height, int alignment) {
+    this.label = label;                 // Remember all the properties
+    this.margin_width = margin_width;
+    this.margin_height = margin_height;
+    this.alignment = alignment;
+    newLabel();                         // Break the label up into lines
+  }
+  public MultiLineLabel(String label, int margin_width, int margin_height) {
+    this(label, margin_width, margin_height, LEFT);
+  }
+  public MultiLineLabel(String label, int alignment) {
+    this(label, 10, 10, alignment);
+  }
+  public MultiLineLabel(String label) { this(label, 10, 10, LEFT); }
+  public MultiLineLabel() { this(""); }
+
+  // Methods to set and query the various attributes of the component
+  // Note that some query methods are inherited from the superclass.
+  public void setLabel(String label) {
+    this.label = label;
+    newLabel();               // Break the label into lines
+    measured = false;         // Note that we need to measure lines
+    repaint();                // Request a redraw
+  }
+  public void setFont(Font f) {
+    super.setFont(f);         // tell our superclass about the new font
+    measured = false;         // Note that we need to remeasure lines
+    repaint();                // Request a redraw
+  }
+  public void setForeground(Color c) {
+    super.setForeground(c);   // tell our superclass about the new color
+    repaint();                // Request a redraw (size is unchanged)
+  }
+  public void setAlignment(int a) { alignment = a; repaint(); }
+  public void setMarginWidth(int mw) { margin_width = mw; repaint(); }
+  public void setMarginHeight(int mh) { margin_height = mh; repaint(); }
+  public String getLabel() { return label; }
+  public int getAlignment() { return alignment; }
+  public int getMarginWidth() { return margin_width; }
+  public int getMarginHeight() { return margin_height; }
+
+  /**
+   * This method is called by a layout manager when it wants to
+   * know how big we'd like to be.  In Java 1.1, getPreferredSize() is
+   * the preferred version of this method.  We use this deprecated version
+   * so that this component can interoperate with 1.0 components.
+   */
+  public Dimension getPreferredSize() {
+    if (!measured) measure();
+    return new Dimension(max_width + 2*margin_width,
+                         num_lines * line_height + 2*margin_height);
+  }
+
+  /**
+   * This method is called when the layout manager wants to know
+   * the bare minimum amount of space we need to get by.
+   * For Java 1.1, we'd use getMinimumSize().
+   */
+  public Dimension getMinimumSize() { return getPreferredSize(); }
+
+  /**
+   * This method draws the label (same method that applets use).
+   * Note that it handles the margins and the alignment, but that
+   * it doesn't have to worry about the color or font--the superclass
+   * takes care of setting those in the Graphics object we're passed.
+   */
+  public void paint(Graphics g) {
+    int x, y;
+    Dimension size = this.getSize();  // use getSize() in Java 1.1
+    if (!measured) measure();
+    y = line_ascent + (size.height - num_lines * line_height)/2;
+    for(int i = 0; i < num_lines; i++, y += line_height) {
+      switch(alignment) {
+      default:
+      case LEFT:    x = margin_width; break;
+      case CENTER:  x = (size.width - line_widths[i])/2; break;
+      case RIGHT:   x = size.width - margin_width - line_widths[i]; break;
+      }
+      g.drawString(lines[i], x, y);
+    }
+  }
+
+  /** This internal method breaks a specified label up into an array of lines.
+   *  It uses the StringTokenizer utility class. */
+  protected synchronized void newLabel() {
+    StringTokenizer t = new StringTokenizer(label, "\n");
+    num_lines = t.countTokens();
+    lines = new String[num_lines];
+    line_widths = new int[num_lines];
+    for(int i = 0; i < num_lines; i++) lines[i] = t.nextToken();
+  }
+
+  /** This internal method figures out how the font is, and how wide each
+   *  line of the label is, and how wide the widest line is. */
+  protected synchronized void measure() {
+    FontMetrics fm = this.getFontMetrics(this.getFont());
+    line_height = fm.getHeight();
+    line_ascent = fm.getAscent();
+    max_width = 0;
+    for(int i = 0; i < num_lines; i++) {
+      line_widths[i] = fm.stringWidth(lines[i]);
+      if (line_widths[i] > max_width) max_width = line_widths[i];
+    }
+    measured = true;
+  }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/PlainPanel.java	(revision 21112)
@@ -0,0 +1,153 @@
+/**
+ * Title:        <p>
+ * Description:  <p>
+ * Copyright:    Copyright (c) <p>
+ * Company:      <p>
+ * @author
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+import javax.swing.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Date;
+
+import javax.swing.JPanel;
+import java.awt.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class PlainPanel extends JPanel
+{
+    Vishnu displayFrame;
+    BorderLayout borderLayout1;
+    Border border1;
+    JScrollPane theScrollPane;
+    JEditorPane theEditorPane;
+
+
+  // This method returns an HTML table
+    public String makeHTMLTable()
+    {
+        String [] descriptions = displayFrame.dataManager.getDescriptions();
+        int [] docNums = displayFrame.dataManager.getDocs();
+        // Set up an output stream we can print the table to.
+        // This is easier than concatenating strings all the time.
+        int number;
+        if (docNums==null) number = 0;
+        else number = docNums.length;
+
+        StringWriter sout = new StringWriter();
+        PrintWriter out = new PrintWriter(sout);
+
+        // Print out the table
+	Dimension d = displayFrame.getSize();
+	int totalWidth = d.width-40;
+	int textWidth = totalWidth - 100;
+	
+        out.println("<TABLE BORDER= 2 WIDTH=" + totalWidth + "><TR>");
+
+        out.println("<TH WIDTH= 100>Document Number</TH>");
+        out.println("<TH WIDTH = " + textWidth + ">Sample Text</TH></TR>");
+
+        for(int i=0; i < docNums.length; i++)
+        {
+            out.print("<TR><TD><A HREF=\"");
+            out.print(docNums[i]);
+            out.print("\">");
+            out.print(docNums[i]);
+            out.print("</A></TD><TD>");
+            out.print(descriptions [i]);
+            out.println("</TD></TR>");
+        }
+        out.println("</TABLE>");
+        out.close();
+
+        // Get the string of HTML from the StringWriter and return it.
+        return sout.toString();
+    }
+
+    public PlainPanel()
+    {
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    public PlainPanel(Vishnu d)
+    {
+        displayFrame = d;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+// The sychronized qualifier is placed here because when two plain
+// panels are created in different threads almost simulataniously
+// we seem to get interference happening
+    public synchronized void jbInit() throws Exception
+    {
+        borderLayout1 = new BorderLayout();
+        theEditorPane = new JEditorPane();
+
+        border1 = BorderFactory.createEmptyBorder(10,10,10,10);
+        theEditorPane.setEditable(false);               // we're browsing not editing
+        theEditorPane.setContentType("text/html");      // must specify HTML text
+        theEditorPane.setText("<TABLE BORDER= 2 WIDTH=640><TR>"+
+                                "<TH WIDTH= 100>Document Number</TH>" +
+                                "<TH WIDTH = 520>Sample Text</TH></TR>" +
+                                "</TABLE>");  // specify the text to display
+        theEditorPane.setFont(new java.awt.Font("Monospaced", 0, 12));
+    // Set up the JEditorPane to handle clicks on hyperlinks
+        theEditorPane.addHyperlinkListener(new HyperlinkListener() {
+                public void hyperlinkUpdate(HyperlinkEvent e)
+                {
+	            // Handle clicks; ignore mouseovers and other link-related events
+	                if (e.getEventType() == HyperlinkEvent.EventType.ACTIVATED)
+                    {
+	            // Get the HREF of the link and display it.
+                        String hpresult = e.getDescription();
+                        int index = Integer.parseInt(hpresult);
+                        if (index > 0)
+                            displayFrame.dataManager.fetchDocNo(index);
+	                }
+                }
+            });
+
+        // Put the JEditorPane in a scrolling window and display it.
+
+        this.setLayout(borderLayout1);
+        this.setFont(new java.awt.Font("Monospaced", 0, 12));
+        this.setBorder(border1);
+        theScrollPane = new JScrollPane();
+        //theScrollPane.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_ALWAYS);
+        this.add(theScrollPane, BorderLayout.CENTER);
+        theScrollPane.getViewport().add(theEditorPane, null);
+        setSize(600, 400);
+        setVisible(true);
+    }
+
+    public void update()
+    {
+       theEditorPane.setText(makeHTMLTable());
+    }
+
+    public void paint(Graphics g)
+    {
+	update();
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pnt2DTupleNode.java	(revision 21112)
@@ -0,0 +1,42 @@
+/**
+* The head of the rows of floating point lists contains drawing and
+* title information about a document as well as the document index
+* @author Matthew Carey
+*/
+package vishnu.testvis.visual;
+
+import vishnu.datablock.*;
+
+public class Pnt2DTupleNode
+{
+    /** index of the document */
+    public int index;
+    /** start of linked list */
+    public SparseNode row;
+    /** cartesian floating point location */
+    public Point2D pnt;
+    /** raster integer locatoin */
+    public Ipoint2D iPnt;
+    /** drawn or not */
+    public boolean state;
+    /** document title */
+    public String ref;
+    /**
+    * Constructor
+    * @param cartesian x
+    * @param cartesain y
+    * @param index
+    * @param linked list of data
+    * @param title
+    */
+    Pnt2DTupleNode(float x, float y ,int i, SparseNode fn, String reference)
+    {
+        state = false;
+        index = i;
+        pnt = new Point2D(x,y);
+        iPnt = new Ipoint2D(0,0);
+        ref=reference;
+        if (reference==null) ref = "Not defined";
+        row=fn;
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Pt2DNode.java	(revision 21112)
@@ -0,0 +1,53 @@
+package vishnu.testvis.visual;
+import vishnu.datablock.*;
+/**
+* Downward linked list wrapper class enables several lists to share
+* same row nodes
+* @author Matthew Carey
+*/
+
+public class Pt2DNode
+{
+     /** next node */
+     Pt2DNode next;
+     /** row data */
+     Pnt2DTupleNode tuple;
+     /** constructor
+     * @param another node to link to
+     */
+     Pt2DNode(Pnt2DTupleNode t)
+     {
+        tuple = t;
+        next = null;
+     }
+     /**
+     * constructor
+     * @param cartesian x
+     * @param cartesian y
+     * @param document index
+     * @param data row
+     * @param document title
+     */
+     Pt2DNode(float x, float y ,int i, SparseNode t, String s)
+     {
+        tuple = new Pnt2DTupleNode(x,y,i,t,s);
+        next = null;
+     }
+     /**
+     * Get next node
+     * @return next node
+     */
+     Pt2DNode getNext()
+     {
+        return next;
+     }
+     /**
+     * Set next node
+     * @param another node to be next
+     */
+     public void setNext(Pt2DNode n)
+     {
+        n.next=next;
+        next = n;
+     }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/RadVizPanel.java	(revision 21112)
@@ -0,0 +1,300 @@
+package vishnu.testvis.visual;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.border.*;
+import java.awt.event.*;
+
+public class RadVizPanel extends JPanel
+{
+    protected CvManyPoints canvas1;
+    public java.awt.Label docLabel;
+    protected KeywordPanel keyWordPanel;
+    protected Panel south = new Panel();
+
+    protected Border border1;
+
+    protected JPanel buttonPanel = new JPanel();
+    protected JPanel buttonBasePanel = new JPanel();
+    protected JPanel labelPanel = new JPanel();
+    protected JLabel buttonLabel = new JLabel();
+    protected JButton zoomInButton = new JButton();
+    protected JButton zoomOutButton = new JButton();
+    protected JButton resetButton = new JButton();
+    protected JButton clusterButton = new JButton();
+
+    protected JCheckBox showGridCheckBox = new JCheckBox();
+    protected JCheckBox lockCheckBox = new JCheckBox();
+    protected JCheckBox showRimCheckBox = new JCheckBox();
+    protected JCheckBox projectCheckBox = new JCheckBox();
+
+    protected int smallFont = 14;
+    protected int largeFont = 16;
+    protected Border border3;
+    protected BorderLayout borderLayout1 = new BorderLayout(10,0);
+    protected Vishnu parent;
+
+    public RadVizPanel(Vishnu p)
+    {
+        parent = p;
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+
+
+    private void jbInit() throws Exception
+    {
+        border1 = BorderFactory.createEmptyBorder(5,5,5,5);
+        border3 = BorderFactory.createEmptyBorder(10,10,10,10);
+        this.setLayout(borderLayout1);
+        this.setBorder(border3);
+        this.setPreferredSize(new Dimension(600, 400));
+        canvas1 = new CvManyPoints (parent);
+	
+	// This is used as a balloon by the canvas1
+        docLabel = new java.awt.Label("  ");
+        docLabel.setVisible(false);
+
+	// place it out of view if does become visible
+        docLabel.setBounds(12,-20,10,10); 
+        docLabel.setBackground(new Color(16776895));
+        docLabel.setFont(new Font("Dialog", Font.PLAIN, largeFont));
+	
+        add(docLabel);
+        add(canvas1, "Center");
+	
+        keyWordPanel = new KeywordPanel(canvas1, this);
+	keyWordPanel.setVisible(false);
+	
+        buttonBasePanel.setLayout(new BorderLayout(10,10));
+        buttonPanel.setLayout(new GridLayout(0,2,10,10));
+        buttonLabel.setText("Settings");
+        buttonLabel.setForeground(Color.black);
+        buttonLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+	
+        zoomInButton.setText("Zoom in");
+        zoomInButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        zoomInButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomInButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomInButton_mouseClicked(e);
+		}
+	    });
+        zoomOutButton.setText("Zoom Out");
+        zoomOutButton.setFont(new java.awt.Font("Monospaced", 0,largeFont));
+        zoomOutButton.setBorder(BorderFactory.createEtchedBorder());
+        zoomOutButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    zoomOutButton_mouseClicked(e);
+		}
+	    });
+        resetButton.setText("Reset");
+        resetButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        resetButton.setBorder(BorderFactory.createEtchedBorder());
+        resetButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    resetButton_mouseClicked(e);
+		}
+	    });
+        clusterButton.setText("Cluster");
+        clusterButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        clusterButton.setBorder(BorderFactory.createEtchedBorder());
+        clusterButton.addMouseListener(new java.awt.event.MouseAdapter()
+	    {
+		public void mouseClicked(MouseEvent e)
+		{
+		    clusterButton_mouseClicked(e);
+		}
+	    });
+	
+        showGridCheckBox.setText("Show Grid");
+        showGridCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showGridCheckBox.setSelected(false);
+        showGridCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showGridCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showGridCheckBox_itemStateChanged(e);
+		}
+	    });
+        showRimCheckBox.setText("Show Rim");
+        showRimCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        showRimCheckBox.setSelected(true);
+        showRimCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        showRimCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    showRimCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+        projectCheckBox.setText("Projection");
+        projectCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        projectCheckBox.setSelected(true);
+        projectCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        projectCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    projectCheckBox_itemStateChanged(e);
+		}
+	    });
+        lockCheckBox.setText("Lock Radius");
+        lockCheckBox.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        lockCheckBox.setSelected(false);
+        lockCheckBox.setBorder(BorderFactory.createEtchedBorder());
+        lockCheckBox.addItemListener(new java.awt.event.ItemListener()
+	    {
+		public void itemStateChanged(ItemEvent e)
+		{
+		    lockCheckBox_itemStateChanged(e);
+		}
+	    });
+	
+	
+        buttonPanel.add(zoomInButton);
+        buttonPanel.add(zoomOutButton);
+        buttonPanel.add(resetButton);
+        buttonPanel.add(clusterButton);
+        buttonPanel.add(showGridCheckBox);
+        buttonPanel.add(showRimCheckBox);
+        buttonPanel.add(projectCheckBox);
+        buttonPanel.add(lockCheckBox);
+	
+        labelPanel.add(buttonLabel);
+        buttonBasePanel.add(buttonPanel,"Center");
+        buttonBasePanel.add(labelPanel,"North");
+        south.setLayout(new BorderLayout(10,0));
+	south.add(keyWordPanel,"West");
+        south.add(buttonBasePanel,"East");
+        south.setVisible(false);
+        add(south,"South");
+    }
+    
+    
+    public void setBothVisible(boolean f)
+    {
+        keyWordPanel.setVisible(f);
+        canvas1.setVisible(f);
+        south.setVisible(f);
+    }
+    
+    
+    public void addNotify()
+    {
+	//Record the size of the window prior to calling parents addNotify.
+        Dimension d = getSize();
+
+        super.addNotify();
+	
+        if (fComponentsAdjusted)
+	    return;
+	
+	// Adjust components according to the insets
+        setSize(getInsets().left + getInsets().right + d.width,
+		getInsets().top + getInsets().bottom + d.height);
+        Component components[] = getComponents();
+        for (int i = 0; i < components.length; i++)
+	    {
+		Point p = components[i].getLocation();
+		p.translate(getInsets().left, getInsets().top);
+		components[i].setLocation(p);
+	    }
+        fComponentsAdjusted = true;
+    }
+    
+    
+    void  zoomOutButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.higher();
+    }
+
+
+    void  zoomInButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.flatter();
+    }
+
+
+    void  resetButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.recalc();
+    }
+
+
+    void  clusterButton_mouseClicked(MouseEvent e)
+    {
+        canvas1.cluster();
+    }
+
+
+    void showGridCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.grid(showGridCheckBox.isSelected());
+    }
+
+
+    void showRimCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.rim(showRimCheckBox.isSelected());
+    }
+
+
+    void projectCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.projection(projectCheckBox.isSelected());
+    }
+
+
+    void lockCheckBox_itemStateChanged(ItemEvent e)
+    {
+        canvas1.lock(lockCheckBox.isSelected());
+    }
+
+
+    public void update()
+    {
+        canvas1.update();
+        keyWordPanel.update();
+    }
+
+
+    public void setVisible(boolean f)
+    {
+        super.setVisible(f);
+        setBothVisible(f);
+    }
+
+
+    public void paint(Graphics g)
+    {
+	System.out.println("Updating RadVizPanel");
+	//canvas1.update();
+	keyWordPanel.update();
+    }
+    
+    protected boolean fComponentsAdjusted = false;
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/SimplePanel.java	(revision 21112)
@@ -0,0 +1,205 @@
+
+/**
+ * Title:        VeniVidiVinci<p>
+ * Description:  An applet that displays visualisations of data sets and allows
+ * these visualisations to be manipulated and selections made<p>
+ * Copyright:    Copyright (c) Matthew Carey<p>
+ * Company:      Imperial College<p>
+ * @author Matthew Carey
+ * @version 1.0
+ */
+package vishnu.testvis.visual;
+
+import javax.swing.*;
+import javax.swing.plaf.basic.*;
+import java.awt.*;
+import java.awt.event.*;
+
+
+/** Panel containing the graphical search interface */
+public class SimplePanel extends JPanel {
+
+    JTextField textField;
+    Vishnu currentMain;
+
+
+    public class InfoLabel extends JLabel {
+        void update() {
+            int startIndex = localHLinkPanel.index +1;
+            int endIndex = localHLinkPanel.index + localHLinkPanel.numDisplayedDocs;
+            int numDocs = currentMain.dataManager.getDocCount();
+            if (numDocs < endIndex) endIndex = numDocs;
+
+            setText("Documents "+ startIndex + "-" + endIndex + " out of " + numDocs);
+        }
+
+    }
+
+    InfoLabel infoLabel = new InfoLabel();
+
+    Hlink localHLinkPanel;
+
+    JButton backButton, nextButton;
+    public SimplePanel(Vishnu theMain)
+    {
+        this.currentMain = theMain;
+        localHLinkPanel = new Hlink(600,400,currentMain);
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    public SimplePanel()
+
+    {
+        this.currentMain = null;
+        localHLinkPanel = new Hlink();
+        try
+        {
+            jbInit();
+        }
+        catch(Exception e)
+        {
+            e.printStackTrace();
+        }
+    }
+    private void jbInit() throws Exception
+    {
+        setName("Simple ");
+
+
+        backButton = new JButton("<<");
+        backButton.setBorder(BorderFactory.createEtchedBorder());
+        backButton.setEnabled(false);
+        backButton.addMouseListener( new MouseAdapter() {
+            public void mouseClicked(MouseEvent e) {
+
+                if (!backButton.isEnabled()) return;
+
+                nextButton.setEnabled(true);
+
+                localHLinkPanel.index -= localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index == 0) {
+                    backButton.setEnabled(false);
+                }
+            }
+        });
+        nextButton = new JButton(">>");
+        nextButton.setBorder(BorderFactory.createEtchedBorder());
+        nextButton.setEnabled(false);
+        nextButton.addMouseListener( new MouseAdapter()
+        {
+            public void mouseClicked(MouseEvent e)
+            {
+
+                if (!nextButton.isEnabled()) return;
+
+                backButton.setEnabled(true);
+
+                localHLinkPanel.index += localHLinkPanel.numDisplayedDocs;
+                localHLinkPanel.update();
+                infoLabel.update();
+
+                if (localHLinkPanel.index + localHLinkPanel.numDisplayedDocs >= currentMain.dataManager.getDocCount())
+                {
+                    nextButton.setEnabled(false);
+                }
+            }
+        });
+
+        //
+
+        // Set the new layout
+        GridBagLayout gridBag = new GridBagLayout();
+        setLayout(gridBag);
+
+        GridBagConstraints constraints = new GridBagConstraints();
+
+        constraints.fill = GridBagConstraints.BOTH;
+        constraints.insets = new Insets(10,3,10,3);
+        constraints.ipadx = 0;
+        constraints.ipady = 8;
+
+
+
+        // Link panel
+        constraints.ipady = 4;
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = GridBagConstraints.REMAINDER;
+        constraints.gridheight = 8;
+
+        gridBag.setConstraints(localHLinkPanel.linkPane,constraints);
+        add(localHLinkPanel.linkPane);
+
+        // Info label
+        constraints.gridx = 0;
+        constraints.gridy = GridBagConstraints.RELATIVE;
+        constraints.gridwidth = 6;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        infoLabel.setMinimumSize(new Dimension(500,20));
+        infoLabel.setMaximumSize(new Dimension(500,20));
+        gridBag.setConstraints(infoLabel,constraints);
+        add(infoLabel);
+
+        // Back button
+
+        constraints.fill = GridBagConstraints.NONE;
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+        Dimension buttonDimension = new Dimension(55,30);
+        backButton.setMaximumSize(buttonDimension);
+        backButton.setPreferredSize(buttonDimension);
+        backButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(backButton,constraints);
+        add(backButton);
+
+        // Next button
+        constraints.gridx = GridBagConstraints.RELATIVE;
+        constraints.gridy = 9;
+        constraints.gridwidth = 2;
+        constraints.gridheight = GridBagConstraints.REMAINDER;
+
+
+        nextButton.setMaximumSize(buttonDimension);
+        nextButton.setPreferredSize(buttonDimension);
+        nextButton.setMinimumSize(buttonDimension);
+
+        gridBag.setConstraints(nextButton,constraints);
+        add(nextButton);
+
+        // Global layout
+        setSize(600,400);
+        setPreferredSize(new Dimension(600,400));
+        setMinimumSize(new Dimension(600,400));
+
+        setVisible(true);
+    }
+
+
+    public void update() {
+
+        localHLinkPanel.index = 0;
+
+        infoLabel.update();
+        localHLinkPanel.update();
+
+        if (currentMain.dataManager.getDocCount() > localHLinkPanel.numDisplayedDocs) {
+            nextButton.setEnabled(true);
+        }
+
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/Vishnu.java	(revision 21112)
@@ -0,0 +1,445 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class Vishnu extends JApplet
+{
+    public JTabbedPane theTabbedPane = null;
+    public PlainPanel plainPanel = null;
+    public GraphicalPane sammonPanel = null;
+    public RadVizPanel radVizPanel = null;
+    public DendroPanel dendroPanel = null;
+    public JButton findButton = null;
+    
+    public DataManager dataManager = null;
+    
+    public Logging log = null;
+    public Queries queries = null;
+    public boolean searchStarted = false;
+    public boolean firstSearch = true;
+    public boolean refine = false;
+    
+    public JTextField searchBox = null;
+    public JComboBox collCombo = null;
+    public JComboBox viewCombo = null;
+    protected int largeFont = 16;
+    
+    public String query = null;
+    public String coll = null;
+    public String view = null;
+    
+    public String server_address = null;
+    
+    protected Dimension size = null;
+        
+    public void init()
+    {
+	String library = getParameter("library");
+	if (library == null ) {
+	    System.err.println("Error: library parameter is null!!");
+	    // what should we do?
+	    return;
+	}
+	String viscgi = getParameter("viscgi");
+	
+	query = getParameter("query");
+        coll = getParameter("collection");
+	view = getParameter("view");
+	
+	server_address = library;
+	if (viscgi != null) {
+	    // if viscgi parameter is set (even if empty) we assume that
+	    // the server is running  cgi-like with arguments in the url
+	    server_address += viscgi;
+	    server_address = tidy_URL(server_address, true);
+	} else {
+	    server_address = tidy_URL(server_address, false);
+	}
+	
+	// What kind of engine the dataManager should use
+	String engine = getParameter("engine");
+ 
+	String w = getParameter("width");
+	String h = getParameter("height");
+	
+	if( (w != null) && (h != null) )
+	    size = new Dimension(Integer.parseInt(w),Integer.parseInt(h));
+        else size = new Dimension(1000,800);
+	
+	dataManager = new DataManager(this, engine);
+	
+        this.setSize(size); 
+        
+	theTabbedPane = new JTabbedPane();
+
+	radVizPanel = new RadVizPanel(this);
+        radVizPanel.setName("RadialVis");
+        
+        plainPanel = new PlainPanel(this);
+        plainPanel.setName("PlainVis");
+        
+        sammonPanel = new GraphicalPane(this);
+        sammonPanel.setName("SammonVis");
+        
+        dendroPanel = new DendroPanel(this);
+        dendroPanel.setName("DendroVis");
+	
+	findButton = new JButton();
+        try {
+	    jbInit();
+	} catch(Exception e){e.printStackTrace();}
+	
+	initCollections();
+	
+	if( query != null && !query.equals("null") ){
+	    searchBox.setText(query);
+	    findQuery();
+	}
+    }
+
+    protected void initCollections() {
+
+	Collection current_collection=null;
+	String current_view = null;
+	if( coll == null || coll.equals("") || coll.equals("null") ) {
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	} else {
+	    // try to find the one specified in the parameters
+	    for( int i = 0; i < dataManager.collections.size(); i++ ){
+		Collection c = (Collection)dataManager.collections.elementAt(i);
+		if( c.dir.equals(coll) ){
+		    current_collection = c;
+		    current_view = view; // use the one from the parameters
+		    break;
+		}
+	    }
+	}
+	if (current_collection == null) {
+	    // could still be null if we specified an invalid collection in the parameters
+	    current_collection = ((Collection)dataManager.collections.elementAt(0));
+	    coll = current_collection.dir;
+	}
+	if (current_view == null) {
+	    if (current_collection.views != null) {
+		current_view = (String)current_collection.views[0];
+	    }
+	}
+	dataManager.currentCollection = current_collection;
+	dataManager.currentView = current_view;
+	this.view = current_view;
+	dataManager.requestCollection();
+	System.out.println("Requesting collection " + current_collection.name);
+	collCombo.setSelectedItem(current_collection);
+	if (viewCombo != null && current_view != null) {
+	    viewCombo.setSelectedItem(current_view);
+	    
+	}
+	    
+    }
+
+    protected void jbInit() throws Exception
+    {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+	JPanel collPanel = new JPanel();
+	collPanel.setLayout(new BorderLayout(5,5));
+	collPanel.setBackground(SystemColor.control);
+
+	collCombo = new JComboBox(dataManager.collections);
+	collCombo.setEditable(false);
+	collCombo.addItemListener(new java.awt.event.ItemListener() {
+		public void itemStateChanged(ItemEvent i) {
+		    if (i.getStateChange()==ItemEvent.SELECTED){
+			dataManager.currentCollection = (Collection)i.getItem();
+			collCombo.hidePopup();
+			coll = dataManager.currentCollection.dir;
+			if (dataManager.currentCollection.views != null) {
+			    viewCombo.setModel(new DefaultComboBoxModel(dataManager.currentCollection.views));
+			    viewCombo.setEnabled(true);
+			    view = (String)viewCombo.getItemAt(0);
+			    dataManager.currentView = view;
+
+			} else {
+			    viewCombo.setModel(new DefaultComboBoxModel());
+			    viewCombo.setEnabled(false);
+			    view = null;
+			    dataManager.currentView = view;
+			}
+			dataManager.requestCollection();
+
+			update();
+			
+		    }
+		    
+		}
+		});
+	collCombo.setRenderer(new javax.swing.DefaultListCellRenderer() {
+		public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus)
+		{
+		    setText(((Collection)value).name);
+		    setFont(list.getFont());
+		    return this;
+		}
+	    });
+
+	JLabel collLabel = new JLabel();           
+	collLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        collLabel.setForeground(Color.black); 
+        collLabel.setText("Collection:");
+	
+	collPanel.add(collLabel, BorderLayout.WEST);
+	collPanel.add(collCombo, BorderLayout.CENTER);
+
+	JPanel viewPanel = new JPanel();
+	viewPanel.setLayout(new BorderLayout(5,5));
+	viewPanel.setBackground(SystemColor.control);
+
+	viewCombo = new JComboBox();
+	viewCombo.setEnabled(false);
+
+	JLabel viewLabel = new JLabel();           
+	viewLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        viewLabel.setForeground(Color.black); 
+        viewLabel.setText("View:");
+	
+	viewPanel.add(viewLabel, BorderLayout.WEST);
+	viewPanel.add(viewCombo, BorderLayout.CENTER);
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	JPanel boxesPanel = new JPanel();
+	boxesPanel.setLayout(new BorderLayout(5,5));
+	boxesPanel.setBackground(SystemColor.control);
+	boxesPanel.add(collPanel, BorderLayout.WEST);
+	boxesPanel.add(viewPanel, BorderLayout.EAST);
+	
+	topPanel.add(boxesPanel, BorderLayout.WEST);
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+    public void update()
+    {
+	plainPanel.update();
+        radVizPanel.update();
+        dendroPanel.update();        
+        sammonPanel.update();
+    }
+    
+        
+    void findQuery()
+    {
+	refine = false;
+	
+        int docCount;
+	String text = null;
+	
+	text = (String)searchBox.getText();
+	if(text.trim().length() < 1) return;
+	query = text.trim();
+	
+	if (viewCombo != null) {
+	    String view1 = null;
+	    if (viewCombo.isEnabled()) {
+		view1 = (String)viewCombo.getSelectedItem();	    
+	    }
+	    if (!view1.equals(this.view)) {
+		this.view = view1;
+		dataManager.currentView = view1;
+		dataManager.requestCollection(); // resets previous data
+	    }
+	}
+        setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	
+	int count = 1;
+	String q = query;
+	docCount = dataManager.query(q);
+	if (docCount == 0 ) {
+      	    JOptionPane.showMessageDialog(this,"No documents found related to " + text);
+	    setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    return;
+	}
+
+        setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+    }
+    
+
+    void theTabbedPane_stateChanged(ChangeEvent e)
+    {
+	for( int i = 0; i < theTabbedPane.getTabCount(); i++ )
+        {
+            theTabbedPane.setForegroundAt(i,Color.black);
+        }
+        
+        int tab = theTabbedPane.getSelectedIndex();
+        
+        theTabbedPane.setForegroundAt(tab,Color.blue);
+    }
+
+
+  void findButton_keyPressed(KeyEvent e)
+  {
+      int key = e.getKeyCode();
+      if (key==10)
+	  {
+	      findButton.setSelected(true);
+	      findQuery();
+	      findButton.setSelected(false);
+	  }
+  }
+    
+    void findButton_mouseClicked(MouseEvent e)
+    {
+	findQuery();
+    }
+    
+   
+    // Tidy up URLs
+    //
+    // Ensure a URL address (as string) has a protocol, host, and file.
+    //
+    // If the URL is a CGI script URL, it should be tidied up so that it is
+    // appropriate to tage attrib=value pairs on the end.  This means it
+    // must either end with a "?" or (if it contains a question-mark
+    // internally) end with a "&".
+    // taken from Phind.java, Greenstone digital library.
+    String tidy_URL(String address, boolean isCGI) {
+
+	// System.err.println("tidy URL: " + address);
+	
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+	    
+	}
+	
+	// if the URL is a cgi script, make sure it has a "?" in ti,
+	// and that it ends with a "?" or "&"
+	if (isCGI) {
+	    if (address.indexOf((int) '?') == -1) { 
+		address = address + "?";
+	    } else if (!address.endsWith("?")) {
+		address = address + "&";
+	    }
+	}
+
+	return address;
+    }
+
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/testvis/visual/VishnuSingle.java	(revision 21112)
@@ -0,0 +1,118 @@
+package vishnu.testvis.visual;
+import vishnu.testvis.treemap.visual.TMNavigator;
+import vishnu.testvis.sammon.*;
+import vishnu.testvis.object.*;
+import vishnu.testvis.evaluate.*;
+import vishnu.testvis.dendro.*;
+import java.awt.*;
+import javax.swing.*;
+import java.awt.event.*;
+import java.beans.*;
+import javax.swing.event.*;
+import java.io.*;
+import java.util.Vector;
+import java.util.Properties;
+import java.applet.*;
+import java.net.*;
+import vishnu.datablock.*;
+
+public class VishnuSingle extends Vishnu
+{
+
+    protected void initCollections() {
+	// put in a dummy Collection
+	Collection c = new Collection(coll, coll, coll, 0, coll, null);
+	dataManager.currentCollection = c;
+    }
+
+    // overwrite this to remove the collection stuff - cos we are working within one collection only
+    protected void jbInit() throws Exception {
+	JPanel pane = new JPanel();
+	setContentPane(pane);
+	pane.setBorder(BorderFactory.createLineBorder(Color.black, 2));
+        pane.setLayout(new BorderLayout());
+        
+	JPanel searchPanel = new JPanel();
+	searchPanel.setLayout(new BorderLayout(5,5));
+	searchPanel.setBackground(SystemColor.control);
+        searchBox = new JTextField();
+	searchBox.setPreferredSize(new Dimension((int)(size.width/5), 26));
+        searchBox.setEditable(true);
+        searchBox.addKeyListener(new java.awt.event.KeyAdapter(){
+                public void keyPressed(KeyEvent e)
+                {
+                    findButton_keyPressed(e);
+                }
+            });
+	
+	JLabel searchLabel = new JLabel();
+        searchLabel.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        searchLabel.setForeground(Color.black);
+        searchLabel.setText("Search:");
+
+	searchPanel.add(searchLabel, BorderLayout.WEST);
+	searchPanel.add(searchBox, BorderLayout.CENTER);
+
+
+	JPanel topPanel = new JPanel();
+        topPanel.setLayout(new BorderLayout(10,10));
+	topPanel.setBackground(SystemColor.control);
+        topPanel.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	
+	JPanel backPanel = new JPanel();   
+        backPanel.setPreferredSize(new Dimension(size.width,size.height));
+        backPanel.setBackground(SystemColor.control);
+        backPanel.setLayout(new BorderLayout());
+        
+	theTabbedPane.setBackground(SystemColor.control);
+        theTabbedPane.setFont(new java.awt.Font("Monospaced", Font.BOLD, largeFont));
+        theTabbedPane.addChangeListener(new javax.swing.event.ChangeListener(){
+		public void stateChanged(ChangeEvent e)
+		{
+		    theTabbedPane_stateChanged(e);
+		}
+	    });
+	
+        findButton.setBackground(SystemColor.control);
+        findButton.setFont(new java.awt.Font("Monospaced", 0, largeFont));
+        findButton.setBorder(BorderFactory.createEtchedBorder());
+        findButton.setText("Find");
+        findButton.addKeyListener(new java.awt.event.KeyAdapter(){
+		public void keyPressed(KeyEvent e)
+		{
+		    findButton_keyPressed(e);
+		}
+	    });
+        
+        findButton.addMouseListener(new java.awt.event.MouseAdapter(){
+		public void mouseClicked(MouseEvent e)
+		{
+		    findButton_mouseClicked(e);
+		}
+	    });
+	
+        pane.add(backPanel, BorderLayout.CENTER);
+        backPanel.add(theTabbedPane, BorderLayout.CENTER);
+	
+        theTabbedPane.add(plainPanel, "PlainVis");
+        theTabbedPane.add(radVizPanel, "RadialVis");
+        theTabbedPane.add(sammonPanel,"SammonVis");
+        theTabbedPane.add(dendroPanel,"DendroVis");
+        theTabbedPane.setForegroundAt(0,Color.blue);
+	
+        pane.add(topPanel, BorderLayout.NORTH);
+	
+	topPanel.add(searchPanel, BorderLayout.CENTER);
+	topPanel.add(findButton, BorderLayout.EAST);
+	
+        theTabbedPane.setPreferredSize(size);
+
+    }
+    
+    
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Base64.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Base64.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Base64.java	(revision 21112)
@@ -0,0 +1,1281 @@
+package vishnu.util;
+
+import java.io.*;
+/**
+ * Encodes and decodes to and from Base64 notation.
+ *
+ * <p>
+ * Change Log:
+ * </p>
+ * <ul>
+ *  <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the
+ *   encoded data was a single byte.</li>
+ *  <li>v2.0 - I got rid of methods that used booleans to set options. 
+ *   Now everything is more consolidated and cleaner. The code now detects
+ *   when data that's being decoded is gzip-compressed and will decompress it
+ *   automatically. Generally things are cleaner. You'll probably have to
+ *   change some method calls that you were making to support the new
+ *   options format (<tt>int</tt>s that you "OR" together).</li>
+ *  <li>v1.5.1 - Fixed bug when decompressing and decoding to a             
+ *   byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>.      
+ *   Added the ability to "suspend" encoding in the Output Stream so        
+ *   you can turn on and off the encoding if you need to embed base64       
+ *   data in an otherwise "normal" stream (like an XML file).</li>  
+ *  <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself.
+ *      This helps when using GZIP streams.
+ *      Added the ability to GZip-compress objects before encoding them.</li>
+ *  <li>v1.4 - Added helper methods to read/write files.</li>
+ *  <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li>
+ *  <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream
+ *      where last buffer being read, if not completely full, was not returned.</li>
+ *  <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li>
+ *  <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li>
+ * </ul>
+ *
+ * <p>
+ * I am placing this code in the Public Domain. Do with it as you will.
+ * This software comes with no guarantees or warranties but with
+ * plenty of well-wishing instead!
+ * Please visit <a href="http://iharder.net/xmlizable">http://iharder.net/base64</a>
+ * periodically to check for updates or to contribute improvements.
+ * </p>
+ *
+ * @author Robert Harder
+ * @author rob@iharder.net
+ * @version 2.0
+ */
+public class Base64
+{
+    
+/* ********  P U B L I C   F I E L D S  ******** */   
+    
+    
+    /** No options specified. Value is zero. */
+    public final static int NO_OPTIONS = 0;
+    
+    /** Specify encoding. */
+    public final static int ENCODE = 1;
+    
+    
+    /** Specify decoding. */
+    public final static int DECODE = 0;
+    
+    
+    /** Specify that data should be gzip-compressed. */
+    public final static int GZIP = 2;
+    
+    
+    /** Don't break lines when encoding (violates strict Base64 specification) */
+    public final static int DONT_BREAK_LINES = 8;
+    
+    
+/* ********  P R I V A T E   F I E L D S  ******** */  
+    
+    
+    /** Maximum line length (76) of Base64 output. */
+    private final static int MAX_LINE_LENGTH = 76;
+    
+    
+    /** The equals sign (=) as a byte. */
+    private final static byte EQUALS_SIGN = (byte)'=';
+    
+    
+    /** The new line character (\n) as a byte. */
+    private final static byte NEW_LINE = (byte)'\n';
+    
+    
+    /** The 64 valid Base64 values. */
+    private final static byte[] ALPHABET =
+    {
+        (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G',
+        (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N',
+        (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', 
+        (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z',
+        (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g',
+        (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n',
+        (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', 
+        (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z',
+        (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', 
+        (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/'
+    };
+    
+    /** 
+     * Translates a Base64 value to either its 6-bit reconstruction value
+     * or a negative number indicating some other meaning.
+     **/
+    private final static byte[] DECODABET =
+    {   
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,                 // Decimal  0 -  8
+        -5,-5,                                      // Whitespace: Tab and Linefeed
+        -9,-9,                                      // Decimal 11 - 12
+        -5,                                         // Whitespace: Carriage Return
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 14 - 26
+        -9,-9,-9,-9,-9,                             // Decimal 27 - 31
+        -5,                                         // Whitespace: Space
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,              // Decimal 33 - 42
+        62,                                         // Plus sign at decimal 43
+        -9,-9,-9,                                   // Decimal 44 - 46
+        63,                                         // Slash at decimal 47
+        52,53,54,55,56,57,58,59,60,61,              // Numbers zero through nine
+        -9,-9,-9,                                   // Decimal 58 - 60
+        -1,                                         // Equals sign at decimal 61
+        -9,-9,-9,                                      // Decimal 62 - 64
+        0,1,2,3,4,5,6,7,8,9,10,11,12,13,            // Letters 'A' through 'N'
+        14,15,16,17,18,19,20,21,22,23,24,25,        // Letters 'O' through 'Z'
+        -9,-9,-9,-9,-9,-9,                          // Decimal 91 - 96
+        26,27,28,29,30,31,32,33,34,35,36,37,38,     // Letters 'a' through 'm'
+        39,40,41,42,43,44,45,46,47,48,49,50,51,     // Letters 'n' through 'z'
+        -9,-9,-9,-9                                 // Decimal 123 - 126
+        /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 127 - 139
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 140 - 152
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 153 - 165
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 166 - 178
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 179 - 191
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 192 - 204
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 205 - 217
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 218 - 230
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,     // Decimal 231 - 243
+        -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9         // Decimal 244 - 255 */
+    };
+    
+    private final static byte BAD_ENCODING    = -9; // Indicates error in encoding
+    private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding
+    private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding
+
+    
+    /** Defeats instantiation. */
+    private Base64(){}
+    
+    
+    
+/* ********  E N C O D I N G   M E T H O D S  ******** */    
+    
+    
+    /**
+     * Encodes the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     *
+     * @param threeBytes the array to convert
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes )
+    {   
+        return encode3to4( threeBytes, 3 );
+    }   // end encodeToBytes
+    
+    
+    
+    /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     *
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.3
+     */
+    private static byte[] encode3to4( byte[] threeBytes, int numSigBytes )
+    {  
+        byte[] dest = new byte[4];
+        encode3to4( threeBytes, 0, numSigBytes, dest, 0 );
+        return dest;
+    }
+    
+     /**
+     * Encodes up to the first three bytes of array <var>threeBytes</var>
+     * and returns a four-byte array in Base64 notation.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     * The array <var>threeBytes</var> needs only be as big as
+     * <var>numSigBytes</var>.
+     * Code can reuse a byte array by passing a four-byte array as <var>b4</var>.
+     *
+     * @param b4 A reusable byte array to reduce array instantiation
+     * @param threeBytes the array to convert
+     * @param numSigBytes the number of significant bytes in your array
+     * @return four byte array in Base64 notation.
+     * @since 1.5.1
+     */
+    private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes )
+    {
+        encode3to4( threeBytes, 0, numSigBytes, b4, 0 );
+        return b4;
+    }   // end encode3to4
+
+    
+    /**
+     * Encodes up to three bytes of the array <var>source</var>
+     * and writes the resulting four Base64 bytes to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 3 for
+     * the <var>source</var> array or <var>destOffset</var> + 4 for
+     * the <var>destination</var> array.
+     * The actual number of significant bytes in your array is
+     * given by <var>numSigBytes</var>.
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param numSigBytes the number of significant bytes in your array
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the <var>destination</var> array
+     * @since 1.3
+     */
+    private static byte[] encode3to4( 
+     byte[] source, int srcOffset, int numSigBytes,
+     byte[] destination, int destOffset )
+    {
+        //           1         2         3  
+        // 01234567890123456789012345678901 Bit position
+        // --------000000001111111122222222 Array position from threeBytes
+        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
+        //          >>18  >>12  >> 6  >> 0  Right shift necessary
+        //                0x3f  0x3f  0x3f  Additional AND
+        
+        // Create buffer with zero-padding if there are only one or two
+        // significant bytes passed in the array.
+        // We have to shift left 24 in order to flush out the 1's that appear
+        // when Java treats a value as negative that is cast from a byte to an int.
+        int inBuff =   ( numSigBytes > 0 ? ((source[ srcOffset     ] << 24) >>>  8) : 0 )
+                     | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 )
+                     | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 );
+
+        switch( numSigBytes )
+        {
+            case 3:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = ALPHABET[ (inBuff       ) & 0x3f ];
+                return destination;
+                
+            case 2:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>>  6) & 0x3f ];
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            case 1:
+                destination[ destOffset     ] = ALPHABET[ (inBuff >>> 18)        ];
+                destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ];
+                destination[ destOffset + 2 ] = EQUALS_SIGN;
+                destination[ destOffset + 3 ] = EQUALS_SIGN;
+                return destination;
+                
+            default:
+                return destination;
+        }   // end switch
+    }   // end encode3to4
+    
+    
+    
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * The object is not GZip-compressed before being encoded.
+     *
+     * @param serializableObject The object to encode
+     * @return The Base64-encoded object
+     * @since 1.4
+     */
+    public static String encodeObject( java.io.Serializable serializableObject )
+    {
+        return encodeObject( serializableObject, NO_OPTIONS );
+    }   // end encodeObject
+    
+
+
+    /**
+     * Serializes an object and returns the Base64-encoded
+     * version of that serialized object. If the object
+     * cannot be serialized or there is another error,
+     * the method will return <tt>null</tt>.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     * @param serializableObject The object to encode
+     * @param options Specified options
+     * @return The Base64-encoded object
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeObject( java.io.Serializable serializableObject, int options )
+    {
+        // Streams
+        java.io.ByteArrayOutputStream  baos  = null; 
+        java.io.OutputStream           b64os = null; 
+        java.io.ObjectOutputStream     oos   = null; 
+        java.util.zip.GZIPOutputStream gzos  = null;
+        
+        // Isolate options
+        int gzip           = (options & GZIP);
+        int dontBreakLines = (options & DONT_BREAK_LINES);
+        
+        try
+        {
+            // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream
+            baos  = new java.io.ByteArrayOutputStream();
+            b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+    
+            // GZip?
+            if( gzip == GZIP )
+            {
+                gzos = new java.util.zip.GZIPOutputStream( b64os );
+                oos  = new java.io.ObjectOutputStream( gzos );
+            }   // end if: gzip
+            else
+                oos   = new java.io.ObjectOutputStream( b64os );
+            
+            oos.writeObject( serializableObject );
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            return null;
+        }   // end catch
+        finally
+        {
+            try{ oos.close();   } catch( Exception e ){}
+            try{ gzos.close();  } catch( Exception e ){}
+            try{ b64os.close(); } catch( Exception e ){}
+            try{ baos.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return new String( baos.toByteArray() );
+    }   // end encode
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source )
+    {
+        return encodeBytes( source, 0, source.length, NO_OPTIONS );
+    }   // end encodeBytes
+    
+
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int options )
+    {   
+        return encodeBytes( source, 0, source.length, options );
+    }   // end encodeBytes
+    
+    
+    /**
+     * Encodes a byte array into Base64 notation.
+     * Does not GZip-compress data.
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @since 1.4
+     */
+    public static String encodeBytes( byte[] source, int off, int len )
+    {
+        return encodeBytes( source, off, len, NO_OPTIONS );
+    }   // end encodeBytes
+    
+    
+
+    /**
+     * Encodes a byte array into Base64 notation.
+     * <p>
+     * Valid options:<pre>
+     *   GZIP: gzip-compresses object before encoding it.
+     *   DONT_BREAK_LINES: don't break lines at 76 characters
+     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+     * </pre>
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
+     * <p>
+     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES )</code>
+     *
+     *
+     * @param source The data to convert
+     * @param off Offset in array where conversion should begin
+     * @param len Length of data to convert
+     * @param options Specified options
+     * @see Base64#GZIP
+     * @see Base64#DONT_BREAK_LINES
+     * @since 2.0
+     */
+    public static String encodeBytes( byte[] source, int off, int len, int options )
+    {
+        // Isolate options
+        int dontBreakLines = ( options & DONT_BREAK_LINES );
+        int gzip           = ( options & GZIP   );
+        
+        // Compress?
+        if( gzip == GZIP )
+        {
+            java.io.ByteArrayOutputStream  baos  = null;
+            java.util.zip.GZIPOutputStream gzos  = null;
+            Base64.OutputStream            b64os = null;
+            
+    
+            try
+            {
+                // GZip -> Base64 -> ByteArray
+                baos = new java.io.ByteArrayOutputStream();
+                b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines );
+                gzos  = new java.util.zip.GZIPOutputStream( b64os ); 
+            
+                gzos.write( source, off, len );
+                gzos.close();
+            }   // end try
+            catch( java.io.IOException e )
+            {
+                e.printStackTrace();
+                return null;
+            }   // end catch
+            finally
+            {
+                try{ gzos.close();  } catch( Exception e ){}
+                try{ b64os.close(); } catch( Exception e ){}
+                try{ baos.close();  } catch( Exception e ){}
+            }   // end finally
+
+            return new String( baos.toByteArray() );
+        }   // end if: compress
+        
+        // Else, don't compress. Better not to use streams at all then.
+        else
+        {
+            // Convert option to boolean in way that code likes it.
+            boolean breakLines = dontBreakLines == 0;
+            
+            int    len43   = len * 4 / 3;
+            byte[] outBuff = new byte[   ( len43 )                      // Main 4:3
+                                       + ( (len % 3) > 0 ? 4 : 0 )      // Account for padding
+                                       + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines      
+            int d = 0;
+            int e = 0;
+            int len2 = len - 2;
+            int lineLength = 0;
+            for( ; d < len2; d+=3, e+=4 )
+            {
+                encode3to4( source, d+off, 3, outBuff, e );
+
+                lineLength += 4;
+                if( breakLines && lineLength == MAX_LINE_LENGTH )
+                {   
+                    outBuff[e+4] = NEW_LINE;
+                    e++;
+                    lineLength = 0;
+                }   // end if: end of line
+            }   // en dfor: each piece of array
+
+            if( d < len )
+            {
+                encode3to4( source, d+off, len - d, outBuff, e );
+                e += 4;
+            }   // end if: some padding needed
+
+            return new String( outBuff, 0, e );
+        }   // end else: don't compress
+        
+    }   // end encodeBytes
+    
+
+    
+    
+    
+/* ********  D E C O D I N G   M E T H O D S  ******** */
+    
+    
+    /**
+     * Decodes the first four bytes of array <var>fourBytes</var>
+     * and returns an array up to three bytes long with the
+     * decoded values.
+     *
+     * @param fourBytes the array with Base64 content
+     * @return array with decoded values
+     * @since 1.3
+     */
+    private static byte[] decode4to3( byte[] fourBytes )
+    {
+        byte[] outBuff1 = new byte[3];
+        int    count    = decode4to3( fourBytes, 0, outBuff1, 0 );
+        byte[] outBuff2 = new byte[ count ];
+        
+        for( int i = 0; i < count; i++ )
+            outBuff2[i] = outBuff1[i];
+        
+        return outBuff2;
+    }
+    
+    
+    
+    
+    /**
+     * Decodes four bytes from array <var>source</var>
+     * and writes the resulting bytes (up to three of them)
+     * to <var>destination</var>.
+     * The source and destination arrays can be manipulated
+     * anywhere along their length by specifying 
+     * <var>srcOffset</var> and <var>destOffset</var>.
+     * This method does not check to make sure your arrays
+     * are large enough to accomodate <var>srcOffset</var> + 4 for
+     * the <var>source</var> array or <var>destOffset</var> + 3 for
+     * the <var>destination</var> array.
+     * This method returns the actual number of bytes that 
+     * were converted from the Base64 encoding.
+     * 
+     *
+     * @param source the array to convert
+     * @param srcOffset the index where conversion begins
+     * @param destination the array to hold the conversion
+     * @param destOffset the index where output will be put
+     * @return the number of decoded bytes converted
+     * @since 1.3
+     */
+    private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset )
+    {
+        // Example: Dk==
+        if( source[ srcOffset + 2] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 );
+            
+            destination[ destOffset ] = (byte)( outBuff >>> 16 );
+            return 1;
+        }
+        
+        // Example: DkL=
+        else if( source[ srcOffset + 3 ] == EQUALS_SIGN )
+        {
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6 );
+            
+            destination[ destOffset     ] = (byte)( outBuff >>> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>>  8 );
+            return 2;
+        }
+        
+        // Example: DkLE
+        else
+        {
+            try{
+            // Two ways to do the same thing. Don't know which way I like best.
+            //int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] << 24 ) >>>  6 )
+            //              | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 )
+            //              | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 )
+            //              | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 );
+            int outBuff =   ( ( DECODABET[ source[ srcOffset     ] ] & 0xFF ) << 18 )
+                          | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 )
+                          | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) <<  6)
+                          | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF )      );
+
+            
+            destination[ destOffset     ] = (byte)( outBuff >> 16 );
+            destination[ destOffset + 1 ] = (byte)( outBuff >>  8 );
+            destination[ destOffset + 2 ] = (byte)( outBuff       );
+
+            return 3;
+            }catch( Exception e){
+                System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset     ] ]  ) );
+                System.out.println(""+source[srcOffset+1]+  ": " + ( DECODABET[ source[ srcOffset + 1 ] ]  ) );
+                System.out.println(""+source[srcOffset+2]+  ": " + ( DECODABET[ source[ srcOffset + 2 ] ]  ) );
+                System.out.println(""+source[srcOffset+3]+  ": " + ( DECODABET[ source[ srcOffset + 3 ] ]  ) );
+                return -1;
+            }   //e nd catch
+        }
+    }   // end decodeToBytes
+    
+    
+    
+    
+    /**
+     * Very low-level access to decoding ASCII characters in
+     * the form of a byte array. Does not support automatically
+     * gunzipping or any other "fancy" features.
+     *
+     * @param source The Base64 encoded data
+     * @param off    The offset of where to begin decoding
+     * @param len    The length of characters to decode
+     * @return decoded data
+     * @since 1.3
+     */
+    public static byte[] decode( byte[] source, int off, int len )
+    {
+        int    len34   = len * 3 / 4;
+        byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output
+        int    outBuffPosn = 0;
+        
+        byte[] b4        = new byte[4];
+        int    b4Posn    = 0;
+        int    i         = 0;
+        byte   sbiCrop   = 0;
+        byte   sbiDecode = 0;
+        for( i = off; i < off+len; i++ )
+        {
+            sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits
+            sbiDecode = DECODABET[ sbiCrop ];
+            
+            if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better
+            {
+                if( sbiDecode >= EQUALS_SIGN_ENC )
+                {
+                    b4[ b4Posn++ ] = sbiCrop;
+                    if( b4Posn > 3 )
+                    {
+                        outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn );
+                        b4Posn = 0;
+                        
+                        // If that was the equals sign, break out of 'for' loop
+                        if( sbiCrop == EQUALS_SIGN )
+                            break;
+                    }   // end if: quartet built
+                    
+                }   // end if: equals sign or better
+                
+            }   // end if: white space, equals sign or better
+            else
+            {
+                System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" );
+                return null;
+            }   // end else: 
+        }   // each input character
+                                   
+        byte[] out = new byte[ outBuffPosn ];
+        System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); 
+        return out;
+    }   // end decode
+    
+    
+    
+    
+    /**
+     * Decodes data from Base64 notation, automatically
+     * detecting gzip-compressed data and decompressing it.
+     *
+     * @param s the string to decode
+     * @return the decoded data
+     * @since 1.4
+     */
+    public static byte[] decode( String s )
+    {   
+        byte[] bytes = s.getBytes();
+        bytes = decode( bytes, 0, bytes.length );
+        
+        // Check to see if it's gzip-compressed
+        // GZIP Magic Two-Byte Number: 0x8b1f (35615)
+        if( bytes.length >= 2 )
+        {
+            
+            int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);       
+            if(
+            bytes != null &&      // In case decoding returned null
+            bytes.length >= 4  && // Don't want to get ArrayIndexOutOfBounds exception
+            java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) 
+            {
+                java.io.ByteArrayInputStream  bais = null;
+                java.util.zip.GZIPInputStream gzis = null;
+                java.io.ByteArrayOutputStream baos = null;
+                byte[] buffer = new byte[2048];
+                int    length = 0;
+
+                try
+                {
+                    baos = new java.io.ByteArrayOutputStream();
+                    bais = new java.io.ByteArrayInputStream( bytes );
+                    gzis = new java.util.zip.GZIPInputStream( bais );
+
+                    while( ( length = gzis.read( buffer ) ) >= 0 )
+                    {
+                        baos.write(buffer,0,length);
+                    }   // end while: reading input
+
+                    // No error? Get new bytes.
+                    bytes = baos.toByteArray();
+
+                }   // end try
+                catch( java.io.IOException e )
+                {
+                    // Just return originally-decoded bytes
+                }   // end catch
+                finally
+                {
+                    try{ baos.close(); } catch( Exception e ){}
+                    try{ gzis.close(); } catch( Exception e ){}
+                    try{ bais.close(); } catch( Exception e ){}
+                }   // end finally
+
+            }   // end if: gzipped
+        }   // end if: bytes.length >= 2
+        
+        return bytes;
+    }   // end decode
+
+
+    
+
+    /**
+     * Attempts to decode Base64 data and deserialize a Java
+     * Object within. Returns <tt>null</tt> if there was an error.
+     *
+     * @param encodedObject The Base64 data to decode
+     * @return The decoded and deserialized object
+     * @since 1.5
+     */
+    public static Object decodeToObject( String encodedObject )
+    {
+        // Decode and gunzip if necessary
+        byte[] objBytes = decode( encodedObject );
+        
+        java.io.ByteArrayInputStream  bais = null;
+        java.io.ObjectInputStream     ois  = null;
+        Object obj = null;
+        
+        try
+        {
+            bais = new java.io.ByteArrayInputStream( objBytes );
+            ois  = new java.io.ObjectInputStream( bais );
+        
+            obj = ois.readObject();
+        }   // end try
+        catch( java.io.IOException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        catch( java.lang.ClassNotFoundException e )
+        {
+            e.printStackTrace();
+            obj = null;
+        }   // end catch
+        finally
+        {
+            try{ bais.close(); } catch( Exception e ){}
+            try{ ois.close();  } catch( Exception e ){}
+        }   // end finally
+        
+        return obj;
+    }   // end decodeObject
+    
+    
+    /* ********  I N N E R   C L A S S   I N P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * It will read data from another
+     * {@link java.io.InputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterInputStream
+     * @since 1.3
+     */
+    public static class InputStream extends java.io.FilterInputStream
+    {
+        private int     options;        // Options specified
+        private boolean encode;         // Encoding or decoding
+        private int     position;       // Current position in the buffer
+        private byte[]  buffer;         // Small buffer holding converted data
+        private int     bufferLength;   // Length of buffer (3 or 4)
+        private int     numSigBytes;    // Number of meaningful bytes in the buffer
+        private int     lineLength;
+        private boolean breakLines;     // Break lines at less than 80 characters
+        
+        
+        /**
+         * Constructs a InputStream in DECODE mode.
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @since 1.3
+         */
+        public InputStream( java.io.InputStream in )
+        {   
+            this( in, DECODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a InputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code>
+         *
+         *
+         * @param in the {@link java.io.InputStream} from which to read data.
+         * @param options Specified options
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 2.0
+         */
+        public InputStream( java.io.InputStream in, int options )
+        {   
+            super( in );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.breakLines = breakLines;
+            this.encode = encode;
+            this.bufferLength = encode ? 4 : 3;
+            this.buffer   = new byte[ bufferLength ];
+            this.position = -1;
+            this.lineLength = 0;
+        }   // end constructor
+        
+        /**
+         * Reads enough of the input stream to convert
+         * to/from Base64 and returns the next byte.
+         *
+         * @return next byte
+         * @since 1.3
+         */
+        public int read() throws java.io.IOException 
+        { 
+            // Do we need to get data?
+            if( position < 0 )
+            {
+                if( encode )
+                {
+                    byte[] b3 = new byte[3];
+                    int numBinaryBytes = 0;
+                    for( int i = 0; i < 3; i++ )
+                    {
+                        try
+                        { 
+                            int b = in.read();
+                            
+                            // If end of stream, b is -1.
+                            if( b >= 0 )
+                            {
+                                b3[i] = (byte)b;
+                                numBinaryBytes++;
+                            }   // end if: not end of stream
+                            
+                        }   // end try: read
+                        catch( java.io.IOException e )
+                        {   
+                            // Only a problem if we got no data at all.
+                            if( i == 0 )
+                                throw e;
+                            
+                        }   // end catch
+                    }   // end for: each needed input byte
+                    
+                    if( numBinaryBytes > 0 )
+                    {
+                        encode3to4( b3, 0, numBinaryBytes, buffer, 0 );
+                        position = 0;
+                        numSigBytes = 4;
+                    }   // end if: got data
+                    else
+                    {
+                        return -1;
+                    }   // end else
+                }   // end if: encoding
+                
+                // Else decoding
+                else
+                {
+                    byte[] b4 = new byte[4];
+                    int i = 0;
+                    for( i = 0; i < 4; i++ )
+                    {
+                        // Read four "meaningful" bytes:
+                        int b = 0;
+                        do{ b = in.read(); }
+                        while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC );
+                        
+                        if( b < 0 )
+                            break; // Reads a -1 if end of stream
+                        
+                        b4[i] = (byte)b;
+                    }   // end for: each needed input byte
+                    
+                    if( i == 4 )
+                    {
+                        numSigBytes = decode4to3( b4, 0, buffer, 0 );
+                        position = 0;
+                    }   // end if: got four characters
+                    else if( i == 0 ){
+                        return -1;
+                    }   // end else if: also padded correctly
+                    else
+                    {
+                        // Must have broken out from above.
+                        throw new java.io.IOException( "Improperly padded Base64 input." );
+                    }   // end 
+                    
+                }   // end else: decode
+            }   // end else: get data
+            
+            // Got data?
+            if( position >= 0 )
+            {
+                // End of relevant data?
+                if( /*!encode &&*/ position >= numSigBytes )
+                    return -1;
+                
+                if( encode && breakLines && lineLength >= MAX_LINE_LENGTH )
+                {
+                    lineLength = 0;
+                    return '\n';
+                }   // end if
+                else
+                {
+                    lineLength++;   // This isn't important when decoding
+                                    // but throwing an extra "if" seems
+                                    // just as wasteful.
+                    
+                    int b = buffer[ position++ ];
+
+                    if( position >= bufferLength )
+                        position = -1;
+
+                    return b & 0xFF; // This is how you "cast" a byte that's
+                                     // intended to be unsigned.
+                }   // end else
+            }   // end if: position >= 0
+            
+            // Else error
+            else
+            {   
+                // When JDK1.4 is more accepted, use an assertion here.
+                throw new java.io.IOException( "Error in Base64 code reading stream." );
+            }   // end else
+        }   // end read
+        
+        
+        /**
+         * Calls {@link #read} repeatedly until the end of stream
+         * is reached or <var>len</var> bytes are read.
+         * Returns number of bytes read into array or -1 if
+         * end of stream is encountered.
+         *
+         * @param dest array to hold values
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @return bytes read into array or -1 if end of stream is encountered.
+         * @since 1.3
+         */
+        public int read( byte[] dest, int off, int len ) throws java.io.IOException
+        {
+            int i;
+            int b;
+            for( i = 0; i < len; i++ )
+            {
+                b = read();
+                
+                //if( b < 0 && i == 0 )
+                //    return -1;
+                
+                if( b >= 0 )
+                    dest[off + i] = (byte)b;
+                else if( i == 0 )
+                    return -1;
+                else
+                    break; // Out of 'for' loop
+            }   // end for: each byte read
+            return i;
+        }   // end read
+        
+    }   // end inner class InputStream
+    
+    
+    
+    
+    
+    
+    /* ********  I N N E R   C L A S S   O U T P U T S T R E A M  ******** */
+    
+    
+    
+    /**
+     * A OutputStream will write data to another
+     * {@link java.io.OutputStream}, given in the constructor,
+     * and encode/decode to/from Base64 notation on the fly.
+     *
+     * @see Base64
+     * @see java.io.FilterOutputStream
+     * @since 1.3
+     */
+    public static class OutputStream extends java.io.FilterOutputStream
+    {
+        private int     options;
+        private boolean encode;
+        private int     position;
+        private byte[]  buffer;
+        private int     bufferLength;
+        private int     lineLength;
+        private boolean breakLines;
+        private byte[]  b4; // Scratch used in a few places
+        private boolean suspendEncoding;
+        
+        /**
+         * Constructs a OutputStream in ENCODE mode.
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out )
+        {   
+            this( out, ENCODE );
+        }   // end constructor
+        
+        
+        /**
+         * Constructs a OutputStream in
+         * either ENCODE or DECODE mode.
+         * <p>
+         * Valid options:<pre>
+         *   ENCODE or DECODE: Encode or Decode as data is read.
+         *   DONT_BREAK_LINES: don't break lines at 76 characters
+         *     (only meaningful when encoding)
+         *     <i>Note: Technically, this makes your encoding non-compliant.</i>
+         * </pre>
+         * <p>
+         * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code>
+         *
+         * @param out the {@link java.io.OutputStream} to which data will be written.
+         * @param options Specified options.
+         * @see Base64#ENCODE
+         * @see Base64#DECODE
+         * @see Base64#DONT_BREAK_LINES
+         * @since 1.3
+         */
+        public OutputStream( java.io.OutputStream out, int options )
+        {   
+            super( out );
+            this.options      = options;
+            this.breakLines   = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES;
+            this.encode       = (options & ENCODE) == ENCODE;
+            this.bufferLength = encode ? 3 : 4;
+            this.buffer       = new byte[ bufferLength ];
+            this.position     = 0;
+            this.lineLength   = 0;
+            this.suspendEncoding = false;
+            this.b4           = new byte[4];
+        }   // end constructor
+        
+        
+        /**
+         * Writes the byte to the output stream after
+         * converting to/from Base64 notation.
+         * When encoding, bytes are buffered three
+         * at a time before the output stream actually
+         * gets a write() call.
+         * When decoding, bytes are buffered four
+         * at a time.
+         *
+         * @param theByte the byte to write
+         * @since 1.3
+         */
+        public void write(int theByte) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theByte );
+                return;
+            }   // end if: supsended
+            
+            // Encode?
+            if( encode )
+            {
+                buffer[ position++ ] = (byte)theByte;
+                if( position >= bufferLength )  // Enough to encode.
+                {
+                    out.write( encode3to4( b4, buffer, bufferLength ) );
+
+                    lineLength += 4;
+                    if( breakLines && lineLength >= MAX_LINE_LENGTH )
+                    {
+                        out.write( NEW_LINE );
+                        lineLength = 0;
+                    }   // end if: end of line
+
+                    position = 0;
+                }   // end if: enough to output
+            }   // end if: encoding
+
+            // Else, Decoding
+            else
+            {
+                // Meaningful Base64 character?
+                if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC )
+                {
+                    buffer[ position++ ] = (byte)theByte;
+                    if( position >= bufferLength )  // Enough to output.
+                    {
+                        int len = Base64.decode4to3( buffer, 0, b4, 0 );
+                        out.write( b4, 0, len );
+                        //out.write( Base64.decode4to3( buffer ) );
+                        position = 0;
+                    }   // end if: enough to output
+                }   // end if: meaningful base64 character
+                else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC )
+                {
+                    throw new java.io.IOException( "Invalid character in Base64 data." );
+                }   // end else: not white space either
+            }   // end else: decoding
+        }   // end write
+        
+        
+        
+        /**
+         * Calls {@link #write} repeatedly until <var>len</var> 
+         * bytes are written.
+         *
+         * @param theBytes array from which to read bytes
+         * @param off offset for array
+         * @param len max number of bytes to read into array
+         * @since 1.3
+         */
+        public void write( byte[] theBytes, int off, int len ) throws java.io.IOException
+        {
+            // Encoding suspended?
+            if( suspendEncoding )
+            {
+                super.out.write( theBytes, off, len );
+                return;
+            }   // end if: supsended
+            
+            for( int i = 0; i < len; i++ )
+            {
+                write( theBytes[ off + i ] );
+            }   // end for: each byte written
+            
+        }   // end write
+        
+        
+        
+        /**
+         * Method added by PHIL. [Thanks, PHIL. -Rob]
+         * This pads the buffer without closing the stream.
+         */
+        public void flushBase64() throws java.io.IOException 
+        {
+            if( position > 0 )
+            {
+                if( encode )
+                {
+                    out.write( encode3to4( b4, buffer, position ) );
+                    position = 0;
+                }   // end if: encoding
+                else
+                {
+                    throw new java.io.IOException( "Base64 input not properly padded." );
+                }   // end else: decoding
+            }   // end if: buffer partially full
+
+        }   // end flush
+
+        
+        /** 
+         * Flushes and closes (I think, in the superclass) the stream. 
+         *
+         * @since 1.3
+         */
+        public void close() throws java.io.IOException
+        {
+            // 1. Ensure that pending characters are written
+            flushBase64();
+
+            // 2. Actually close the stream
+            // Base class both flushes and closes.
+            super.close();
+            
+            buffer = null;
+            out    = null;
+        }   // end close
+        
+        
+        
+        /**
+         * Suspends encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void suspendEncoding() throws java.io.IOException 
+        {
+            flushBase64();
+            this.suspendEncoding = true;
+        }   // end suspendEncoding
+        
+        
+        /**
+         * Resumes encoding of the stream.
+         * May be helpful if you need to embed a piece of
+         * base640-encoded data in a stream.
+         *
+         * @since 1.5.1
+         */
+        public void resumeEncoding()
+        {
+            this.suspendEncoding = false;
+        }   // end resumeEncoding
+        
+        
+        
+    }   // end inner class OutputStream
+    
+    
+}   // end class Base64
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Hgram.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Hgram.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Hgram.java	(revision 21112)
@@ -0,0 +1,378 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.imageio.*;
+import javax.imageio.stream.*;
+import java.io.*;
+import java.util.*;
+import java.awt.image.*;
+
+public class Hgram extends JFrame
+{
+    public static int TOP = 0;
+    public static int LARGEST = 1;
+
+    // data is an array with each entry being the frequency of a bin
+    public static int BIN_DATA = 0;
+
+    // data comes in raw format and needs to be allocated to bins
+    public static int RAW_DATA = 1;
+    
+    double[] data;
+    
+    // default size of frame
+    int width = 400;
+    int height = 400;
+
+    // axis limits of histogram
+    double xmin;
+    double xmax;
+
+    // number of bins, bin frequencies and width
+    int noBins;
+    double bins[];
+    double binWidth;
+    
+    BufferedImage offScreen=null;
+    Dimension offScreenSize;
+    Graphics2D offGraphics;
+    
+    String label = null;
+    
+    double median = -1;
+    int med_bin = -1;
+    int mode = -1;
+    
+    
+    public Hgram()
+    {
+	init();
+    }
+
+    
+    public void setData(double[] d, int m){
+	data = d; 
+	mode = m;
+	xmin = getMinimum(data);
+	xmax = getMaximum(data);
+	createBins(data,noBins,xmin, xmax);
+    }
+
+      
+    public void setBins(double[] b){ bins = b; }
+
+
+    public void setBinNumber(int i){ noBins = i; }
+
+    
+    public void setMaximum(double i){ xmax = i; }
+    
+    
+    public void setMinimum(double i){ xmin = i; }
+
+
+    public void setBinWidth(double i){ binWidth = i; }
+    
+    
+    public void setLabel(String l){ label = l; }
+  
+
+    public void init()
+    {
+	this.setDefaultLookAndFeelDecorated(true);
+        this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+	this.setSize(width,height);
+	this.setLocation(0,0);
+	this.setVisible(true);
+	
+	this.addComponentListener( new ComponentAdapter(){
+		public void componentResized(ComponentEvent evt){
+		    update();
+		}
+	    });
+	
+	
+	this.addMouseListener(new MouseAdapter(){
+		public void mousePressed(MouseEvent evt)
+		{
+    		    int intX = evt.getX();
+	    	    int intY = evt.getY();
+		}
+		
+		public void mouseClicked(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    
+		    if( event.getClickCount() == 1 )
+			{
+			}
+		}
+	    });
+	
+	
+	this.addMouseMotionListener(new MouseMotionAdapter()
+	    {
+		public void mouseMoved(MouseEvent event)
+		{
+		    int x = event.getX();
+		    int y = event.getY();
+		    getBin(x,y);
+		}
+	    });
+    }
+
+    
+    public int getBin(int x, int y){ return -1; }
+    
+    
+    private int getMaxBin()
+    {
+	int maxIndex = -1;
+	double max = 0;
+	
+	for( int i = 0; i < bins.length; i++ ){
+	    if( bins[i] > max ){
+		max = bins[i];
+		maxIndex = i;
+	    }
+	}
+	
+	return maxIndex;
+    }
+    
+    
+    public void plot()
+    {
+	update();
+    }
+    
+      
+    public double getMaximum(double[] d)
+    {
+	double max = 0; 
+
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max ) max = d[i];
+	}
+	return max;
+    }
+ 
+
+
+    public double getMinimum(double[] d)
+    {
+	double min = d[0]*d[0]*1000;
+
+	for( int i = 0; i < d.length; i++ )
+	    {
+		if( d[i] < min ) min = d[i];
+	    }
+	return min;
+    }
+
+
+    public double getFirstMaximum(double[] d)
+    {
+	double max = 0;
+	for( int i = 0; i < bins.length; i++ )
+	    {
+		if( bins[i] > max )
+		    {
+			max = bins[i];
+		    }
+	    }
+	return max;
+    }
+
+
+    public double getSecondMaximum(double[] d)
+    {
+	double max_1 = 0;
+	double max_2 = 0;
+	for( int i = 0; i < d.length; i++ ){
+	    if( d[i] > max_1 ){
+		max_2 = max_1;
+		max_1 = d[i];
+	    }
+	    else if( d[i] > max_2 ){
+		max_2 = d[i];
+	    }
+	}
+	return max_2;
+    }
+
+ 
+    public void createBins(double[] data, int noBins, double xmin, double xmax)
+    {
+	bins = new double[noBins];
+	binWidth = (double)(xmax-xmin)/noBins;
+	
+	for( int i = 0; i < data.length; i++ ){
+	    int bin = (int)Math.round((double)(data[i]-xmin)/binWidth);
+	    if( bin >= noBins ) bin = noBins-1;
+	    bins[bin]++;
+	}
+	
+	for( int i = 0; i < noBins; i++ ){
+	    bins[i] = (double)bins[i]/data.length;
+	}
+    }
+    
+    
+    public void update()
+    {
+	if( bins == null ) return;
+	
+	Dimension d = this.getSize();
+	width = d.width;
+	height = d.height;
+        offScreen = new BufferedImage( width, height, BufferedImage.TYPE_INT_ARGB ); 
+	offGraphics = (Graphics2D)offScreen.getGraphics();
+        offGraphics.setColor(Color.white);
+        offGraphics.fillRect(0,0,width,height);
+    	BasicStroke myStroke = new BasicStroke(2,BasicStroke.CAP_BUTT,BasicStroke.JOIN_MITER);
+        offGraphics.setStroke(myStroke);	
+	
+	int xaxis = (int)(0.8 * width); 
+	int xBuffer = (int)(0.1 * width);
+	
+	int yaxis = (int)(0.8 * d.height);
+	int yBuffer = (int)(0.1 * d.height);
+	
+	double maxBinLength = (double)xaxis/bins.length;
+	int effBinLength = (int)(0.8 * maxBinLength);
+	int binBuffer = (int)(0.1 * effBinLength);
+	
+	double max_1 = getFirstMaximum(bins);
+	double max_2 = getSecondMaximum(bins);
+	double max = max_1;
+	boolean cut = false;
+	
+	if( max_1 > 1.5 * max_2 ){
+	    cut = true;
+	    max = max_2;
+	}
+	
+	double pixelPerUnit = (double)yaxis/max; 
+	
+
+	/**** draw the bins ****/
+
+	for( int i = 0; i < bins.length; i++ ){
+	    int level = (int)(bins[i]* pixelPerUnit);
+	    double bound = (double)((int)(100 * (i * (double)(xmax-xmin)/bins.length + xmin)))/100;
+	    offGraphics.setColor(Color.blue);
+	    offGraphics.fillRect(xBuffer + (int)(i*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	}
+
+	if( label != null ){
+	    FontMetrics fm   = offGraphics.getFontMetrics(offGraphics.getFont());
+	    int x = width/2 - fm.stringWidth(label)/2;		
+	    offGraphics.drawString(label,x,yBuffer);
+	}
+
+
+	/**** draw axes and ticks ****/
+
+	offGraphics.setColor(Color.blue);
+	
+	int x = xBuffer + binBuffer;
+	int y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	
+	String tickStr = String.valueOf(xmin);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+	x = xBuffer + binBuffer+(int)(bins.length*maxBinLength)+binBuffer;
+	y = height-yBuffer;
+	offGraphics.drawLine(x, y+5, x, y-5);
+	tickStr = String.valueOf(xmax);
+	offGraphics.drawString(tickStr,x-5,y+20);
+	
+        repaint();
+    }
+    
+    
+    public void paint(Graphics g)
+    {
+        Graphics2D g2 = (Graphics2D)g;
+	
+	if( offScreen!=null )
+	    g2.drawImage(offScreen, 0, 0, null);
+    }
+    
+    
+    public void printBins()
+    {
+	for( int i = 0; i < bins.length; i++ ){
+	    double low = i*binWidth;
+	    double up = (i+1)*binWidth;
+	    System.out.println("Bin.." + i + " " + low + " " + up + " " + bins[i]); 
+	}
+    }
+
+
+    public void save(String name, String type)
+    {
+	
+	try{
+	    Iterator imageWriters =
+		ImageIO.getImageWritersByFormatName(type);
+	    ImageWriter imageWriter = (ImageWriter)imageWriters.next();
+	    File file = new File( name + "." + type );
+	    ImageOutputStream ios =
+		ImageIO.createImageOutputStream( file );
+	    imageWriter.setOutput( ios );
+	    imageWriter.write( offScreen );
+	} catch( IOException ie ) {
+	    ie.printStackTrace();
+	}
+    }
+
+}
+
+ // accepts raw data
+    /*public Hgram(double[] data, int noBins, double min, double max)
+    {
+	super("JHistogram");
+	init();
+	createBins(data,noBins,min,max);
+	update();
+	}*/
+    
+    
+    // accepts both raw and binned data
+    /*public Hgram(double[] data, int noBins, int mode)
+    {
+	super("JHistogram");
+	this.mode = mode;
+	init();
+	createBins(data,noBins,getMinimum(data),getMaximum(data));
+	update();
+	}*/
+    
+
+
+/*
+	int level = (int)(bins[med_bin]* pixelPerUnit);
+	offGraphics.fillRect(xBuffer + (int)(med_bin*maxBinLength)+binBuffer,height-yBuffer-level,effBinLength,level);
+	String med_string = String.valueOf(median);
+	med_string = med_string.substring(0,med_string.indexOf('.')+3);
+	offGraphics.drawString(med_string, xBuffer + (int)(med_bin*maxBinLength)+binBuffer-10,height-yBuffer-level-10);
+	    
+	//saveHistogram(offScreen);
+	
+	repaint();
+	    }
+	    }*/
+
+  /*
+    public Hgram(double[] data, int noBins)
+    {
+	init();
+	createBins(data,noBins,getMinimum(data),getFirstMaximum(data));
+	update();
+	}*/
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Misc.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Misc.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Misc.java	(revision 21112)
@@ -0,0 +1,240 @@
+package vishnu.util;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.util.*;
+
+public class Misc {
+
+    // quickSort
+    private static void qs(int l, int r, double v[][])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[1][(l+r)/2];
+		do
+		{
+
+			while( v[1][i] < pivot )
+			{
+				i++;
+			} 
+			while( v[1][j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[1][i];	
+					double t0 = v[0][i];
+					v[1][i] = v[1][j];
+					v[1][j] = t1;
+					v[0][i] = v[0][j];
+					v[0][j] = t0;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+    
+    
+    // quickSort
+    private static void qs(int l, int r, double v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		double pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					double t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+		
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int v[])
+    {
+	if( l < r )
+	{
+		int i = l;
+		int j = r;
+		int pivot = v[(l+r)/2];
+		do
+		{
+
+			while( v[i] < pivot )
+			{
+				i++;
+			} 
+			while( v[j] > pivot )
+			{
+				j--;
+			}
+
+			if( i <= j )
+			{
+				if( i < j )
+				{
+					int t1 = v[i];	
+					v[i] = v[j];
+					v[j] = t1;
+				}
+				i++;
+				j--;
+			}
+		
+		}
+		while( i <= j );
+		qs(l,j,v);
+		qs(i,r,v);
+	}
+    }
+
+
+ // quickSort
+    private static void qs(int l, int r, int[] v, Object[] objArr)
+    {
+        if( l < r ){
+            int i = l;
+            int j = r;
+
+            int pivot = v[(l+r)/2];
+
+            do{
+                while( v[i] < pivot ){
+                    i++;
+                }
+                while( v[j] > pivot ){
+                    j--;
+                }
+
+                if( i <= j ){
+                    if( i < j ){
+                        int t1 = v[i];
+                        v[i] = v[j];
+                        v[j] = t1;
+
+                        Object tobj = objArr[i];
+                        objArr[i] = objArr[j];
+                        objArr[j] = tobj;
+ }
+                    i++;
+                    j--;
+                }
+
+            }
+            while( i <= j );
+            qs(l,j,v,objArr);
+            qs(i,r,v,objArr);
+        }
+    }
+
+
+    // quickSort
+    private static void qs(int l, int r, float[] v, Object[] objArr)
+    {
+	if( l < r ){
+	    int i = l;
+	    int j = r;
+	    
+	    float pivot = v[(l+r)/2];
+	    
+	    do{
+		while( v[i] < pivot ){
+		    i++;
+		} 
+		while( v[j] > pivot ){
+		    j--;
+		}
+		
+		if( i <= j ){
+		    if( i < j ){
+			float t1 = v[i];	
+			v[i] = v[j];
+			v[j] = t1;
+			
+			Object tobj = objArr[i];
+			objArr[i] = objArr[j];
+			objArr[j] = tobj;
+		    }
+		    i++;
+		    j--;
+		}
+		
+	    }
+	    while( i <= j );
+	    qs(l,j,v,objArr);
+	    qs(i,r,v,objArr);
+	}
+    }
+    
+
+    public static void quickSort(int[] v)
+    {
+	qs(0,v.length-1,v);
+    }
+
+    public static void quickSort(float[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(int[] v, Object[] objArr)
+    {
+	qs(0,v.length-1,v,objArr);
+    }
+
+    public static void quickSort(double v[])
+    {
+	qs(0,v.length-1,v);
+    }
+
+
+    public static void quickSort(double v[][])
+    {
+	qs(0,v[0].length-1,v);
+    }
+}
+    
+    
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Sammon.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Sammon.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/Sammon.java	(revision 21112)
@@ -0,0 +1,247 @@
+package vishnu.util;
+
+import java.io.*;
+import java.util.*;
+import vishnu.datablock.Point2D;
+
+public class Sammon
+{
+
+    public static final double MF =       0.3;		/* Magic Factor          */
+    public static final double _MAX_Err = 0.00005;	/* Max Mapping Error     */
+    public static final int _MAX_M =  1000;		/* Max Iteration Number  */
+    public static final int _MAX_N =  1000;		/* Max Number of Entries */
+    public static final int _MAX_L =  128;		/* Max Dimension for X   */
+
+
+    double[][] Xs;
+    double[][] Ys;
+    int number;
+    int dimension;
+    double total_Xs_dists;
+
+/* SS298: Added function for scaling values between 0 and 1
+* n : number of entries
+*/
+    void scale_values(int n)
+    {
+
+	// get the maximum and the minimum of Ys array
+
+    // get the maximum and the minimum of Ys array
+
+        double maxX = Ys[0][0];
+        double minX = Ys[0][0];
+        double maxY = Ys[0][1];
+        double minY = Ys[0][1];
+
+        int i;
+        for (i = 1 ; i < n ; i++)
+        {
+
+            if (maxX < Ys[i][0]) maxX = Ys[i][0];
+            if (minX > Ys[i][0]) minX = Ys[i][0];
+            if (maxY < Ys[i][1]) maxY = Ys[i][1];
+            if (minY > Ys[i][1]) minY = Ys[i][1];
+        }
+        maxY -= minY;
+        maxX -= minX;
+        for (i = 0 ; i < n ; i++)
+        {
+            if (maxX > 0)
+            {
+                Ys[i][0] -= minX;
+                Ys[i][0] /= maxX;
+                Ys[i][0] *=0.8;
+                Ys[i][0] +=0.1;
+            }
+            else Ys[i][0]=0.5;
+            if (maxY > 0)
+            {
+                Ys[i][1] -= minY;
+                Ys[i][1] /= maxY;
+                Ys[i][1] *= 0.8;
+                Ys[i][1] += 0.1;
+            }
+            else Ys[i][1]=0.5;
+        }
+
+    }
+
+
+
+/* return a Euclidean distance between two 'd'-dimensional vectors 'xs' and 'ys'
+*/
+    double eucl_dist(int d, double[] xs, double[] ys)
+    {
+        double ans=0.0;
+        double tmp;
+        int i;
+        for(i=0;i<d;i++)
+        {
+            tmp = xs[i]-ys[i];
+            ans += tmp * tmp;
+        }
+        return Math.sqrt(ans);
+    }
+
+
+/* n - number of entries
+* d - dimension
+* post:
+*   return an array of eucidean distances for X
+*   and assign its total distance (will be used in the mapping error) to 'total'
+*/
+    double[] compute_Xs_eucl_dists()
+    {
+        double tmp[];
+        int i,j,k;
+        tmp = new double[number*(number-1)/2];
+
+        total_Xs_dists=0.0;
+        k=0;
+        for(i=1;i<number;i++)
+            for(j=0;j<i;j++)
+            {
+                tmp[k] = eucl_dist(dimension, Xs[i], Xs[j]);
+                total_Xs_dists+=tmp[k];
+                k++;
+            }
+            return tmp;
+    }
+    
+    
+    public Point2D [] getMapping()
+    {
+        Point2D [] sam = new Point2D[Ys.length];
+        for (int c=0;c<Ys.length;c++)
+	    {
+            float fx = (float) Ys[c][0];
+            float fy = (float) Ys[c][1];
+            sam[c]=new Point2D(fx,fy);
+        }
+        return sam;
+    }
+
+
+
+/*----------------------------------------------------------------*/
+    
+    public Sammon(double [][] centroids)
+    {
+	
+        int i, j, m, p;
+        double[] Xs_dists;
+
+        double dist_star,dist, alpha, beta, gamma, zeta, err;
+        double uppDer0, uppDer1, lowDer0, lowDer1;
+        double maxX=0.0;
+        double minX=0.0;
+        double maxY=0.0;
+        double minY=0.0;
+
+        number = centroids.length;
+        dimension = centroids[0].length;
+
+        Xs = centroids;
+        Ys = new double[number][2];
+
+        Xs_dists = compute_Xs_eucl_dists();
+
+        p=0;
+        for (i=0;i<number;i++)
+        {
+            Ys[i][0] =  Xs[i][0];
+            double value = Xs_dists.length>0?Xs_dists[(i==(number-1))?i:p]:
+                          0.0;
+            Ys[i][1] =  Xs[i][0]+value;
+            p+=(number-(i+1));
+        }
+
+        m=0;  /* iteration counter */
+        do{
+
+            for(p=0;p<number;p++)
+            {
+                uppDer0=uppDer1=lowDer0=lowDer1=0.0;
+
+
+                /* calcuate partial derivatives */
+
+                for(j=0;j<number;j++)
+                {
+                    if (p==j) continue;
+                    dist_star = Xs_dists[(j>p) ? (j*(j-1)/2+p) : (p*(p-1)/2+j)];
+
+                    dist = eucl_dist(2,Ys[p],Ys[j]);
+                    if (dist == 0) dist = 0.00001;
+                    alpha = dist_star-dist;
+                    beta = dist_star*dist;
+                    if (beta == 0) beta = 0.00000000000001;
+                    gamma = alpha/beta;
+
+                    /* 1st dimension */
+                    zeta = Ys[p][0]-Ys[j][0];
+                    uppDer0 += gamma * zeta;
+                    lowDer0 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+
+                    /* 2nd dimension */
+                    zeta = Ys[p][1]-Ys[j][1];
+                    uppDer1 += gamma * zeta;
+                    lowDer1 += (alpha-(zeta*zeta/dist) * (1+alpha/dist))/beta;
+                }  /* for */
+
+                Ys[p][0] = Ys[p][0]+MF*uppDer0/Math.abs(lowDer0);
+                Ys[p][1] = Ys[p][1]+MF*uppDer1/Math.abs(lowDer1);
+                if(p==0)
+                {
+                    maxX = Ys[0][0];
+                    minX = Ys[0][0];
+                    maxY = Ys[0][1];
+                    minY = Ys[0][1];
+                }
+                else
+                {
+                  if (maxX < Ys[p][0]) maxX = Ys[p][0];
+                  if (minX > Ys[p][0]) minX = Ys[p][0];
+                  if (maxY < Ys[p][1]) maxY = Ys[p][1];
+                  if (minY > Ys[p][1]) minY = Ys[p][1];
+                }
+
+
+            }  /* for */
+
+
+        /* calcuate the mapping error */
+
+            err=0.0;
+            p=0;
+            for(j=1;j<number;j++)
+                for(i=0;i<j;i++)
+                {
+                    dist_star = Xs_dists[p++];
+                    alpha = dist_star - eucl_dist(2,Ys[j],Ys[i]);
+                    err += (alpha*alpha/dist_star);
+                }
+            err=err / total_Xs_dists;
+
+            m++;
+        } while ((err>_MAX_Err) && (m!=_MAX_M));
+
+
+        if(maxY==minY)
+        {
+          System.out.println("Warning: simulated 'y' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][1]=Math.random();
+        }
+        if(maxX==minX)
+        {
+          System.out.println("Warning: simulated 'x' values in mapping");
+          for(p=0;p<number;p++)
+            Ys[p][0]=Math.random();
+        }
+
+        scale_values(number);
+    }
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/SpringUtilities.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/SpringUtilities.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/SpringUtilities.java	(revision 21112)
@@ -0,0 +1,194 @@
+package vishnu.util;
+
+import javax.swing.*;
+import javax.swing.SpringLayout;
+import java.awt.*;
+
+/**
+ * A 1.4 file that provides utility methods for
+ * creating form- or grid-style layouts with SpringLayout.
+ * These utilities are used by several programs, such as
+ * SpringBox and SpringCompactGrid.
+ */
+public class SpringUtilities {
+    /**
+     * A debugging utility that prints to stdout the component's
+     * minimum, preferred, and maximum sizes.
+     */
+    public static void printSizes(Component c) {
+        System.out.println("minimumSize = " + c.getMinimumSize());
+        System.out.println("preferredSize = " + c.getPreferredSize());
+        System.out.println("maximumSize = " + c.getMaximumSize());
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component is as big as the maximum
+     * preferred width and height of the components.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeGrid(Container parent,
+                                int rows, int cols,
+                                int initialX, int initialY,
+                                int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeGrid must use SpringLayout.");
+            return;
+        }
+
+        Spring xPadSpring = Spring.constant(xPad);
+        Spring yPadSpring = Spring.constant(yPad);
+        Spring initialXSpring = Spring.constant(initialX);
+        Spring initialYSpring = Spring.constant(initialY);
+        int max = rows * cols;
+
+        //Calculate Springs that are the max of the width/height so that all
+        //cells have the same size.
+        Spring maxWidthSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        Spring maxHeightSpring = layout.getConstraints(parent.getComponent(0)).
+                                    getWidth();
+        for (int i = 1; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            maxWidthSpring = Spring.max(maxWidthSpring, cons.getWidth());
+            maxHeightSpring = Spring.max(maxHeightSpring, cons.getHeight());
+        }
+
+        //Apply the new width/height Spring. This forces all the
+        //components to have the same size.
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                            parent.getComponent(i));
+
+            cons.setWidth(maxWidthSpring);
+            cons.setHeight(maxHeightSpring);
+        }
+
+        //Then adjust the x/y constraints of all the cells so that they
+        //are aligned in a grid.
+        SpringLayout.Constraints lastCons = null;
+        SpringLayout.Constraints lastRowCons = null;
+        for (int i = 0; i < max; i++) {
+            SpringLayout.Constraints cons = layout.getConstraints(
+                                                 parent.getComponent(i));
+            if (i % cols == 0) { //start of new row
+                lastRowCons = lastCons;
+                cons.setX(initialXSpring);
+            } else { //x position depends on previous component
+                cons.setX(Spring.sum(lastCons.getConstraint(SpringLayout.EAST), xPadSpring));
+            }
+
+            if (i / cols == 0) { //first row
+                cons.setY(initialYSpring);
+            } else { //y position depends on previous row
+                cons.setY(Spring.sum(lastRowCons.getConstraint(SpringLayout.SOUTH),
+                                     yPadSpring));
+            }
+            lastCons = cons;
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH,
+                            Spring.sum(
+                                Spring.constant(yPad),
+                                lastCons.getConstraint(SpringLayout.SOUTH)));
+        pCons.setConstraint(SpringLayout.EAST,
+                            Spring.sum(
+                                Spring.constant(xPad),
+                                lastCons.getConstraint(SpringLayout.EAST)));
+    }
+
+    /* Used by makeCompactGrid. */
+    private static SpringLayout.Constraints getConstraintsForCell(
+                                                int row, int col,
+                                                Container parent,
+                                                int cols) {
+        SpringLayout layout = (SpringLayout) parent.getLayout();
+        Component c = parent.getComponent(row * cols + col);
+        return layout.getConstraints(c);
+    }
+
+    /**
+     * Aligns the first <code>rows</code> * <code>cols</code>
+     * components of <code>parent</code> in
+     * a grid. Each component in a column is as wide as the maximum
+     * preferred width of the components in that column;
+     * height is similarly determined for each row.
+     * The parent is made just big enough to fit them all.
+     *
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param initialX x location to start the grid at
+     * @param initialY y location to start the grid at
+     * @param xPad x padding between cells
+     * @param yPad y padding between cells
+     */
+    public static void makeCompactGrid(Container parent,
+                                       int rows, int cols,
+                                       int initialX, int initialY,
+                                       int xPad, int yPad) {
+        SpringLayout layout;
+        try {
+            layout = (SpringLayout)parent.getLayout();
+        } catch (ClassCastException exc) {
+            System.err.println("The first argument to makeCompactGrid must use SpringLayout.");
+            return;
+        }
+
+        //Align all cells in each column and make them the same width.
+        Spring x = Spring.constant(initialX);
+        for (int c = 0; c < cols; c++) {
+            Spring width = Spring.constant(0);
+            for (int r = 0; r < rows; r++) {
+                width = Spring.max(width,
+                                   getConstraintsForCell(r, c, parent, cols).
+                                       getWidth());
+            }
+            for (int r = 0; r < rows; r++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setX(x);
+                constraints.setWidth(width);
+            }
+            x = Spring.sum(x, Spring.sum(width, Spring.constant(xPad)));
+        }
+
+        //Align all cells in each row and make them the same height.
+        Spring y = Spring.constant(initialY);
+        for (int r = 0; r < rows; r++) {
+            Spring height = Spring.constant(0);
+            for (int c = 0; c < cols; c++) {
+                height = Spring.max(height,
+                                    getConstraintsForCell(r, c, parent, cols).
+                                        getHeight());
+            }
+            for (int c = 0; c < cols; c++) {
+                SpringLayout.Constraints constraints =
+                        getConstraintsForCell(r, c, parent, cols);
+                constraints.setY(y);
+                constraints.setHeight(height);
+            }
+            y = Spring.sum(y, Spring.sum(height, Spring.constant(yPad)));
+        }
+
+        //Set the parent's size.
+        SpringLayout.Constraints pCons = layout.getConstraints(parent);
+        pCons.setConstraint(SpringLayout.SOUTH, y);
+        pCons.setConstraint(SpringLayout.EAST, x);
+    }
+}
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/XMLUtil.java
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/XMLUtil.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/src/vishnu/util/XMLUtil.java	(revision 21112)
@@ -0,0 +1,71 @@
+/*********************************************************************
+ Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+***********************************************************************/
+
+package vishnu.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class XMLUtil {
+    
+    /** returns the (first) child element with the given name */
+    public static Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+    /** takes a list of elements, and returns an array of strings 
+     * of the values of attribute att_name */
+    public static String [] getAttributeValuesFromList(Element list,
+						       String att_name) {
+	
+	NodeList children = list.getChildNodes();
+	
+	int num_nodes = children.getLength();
+	String []ids = new String[num_nodes];
+	for (int i=0; i<num_nodes; i++) {
+	    Element e = (Element)children.item(i);
+	    String id = e.getAttribute(att_name);	    
+	    ids[i] = id;	    
+	}
+	
+	return ids;
+    }
+
+    /** extracts the text out of a node */
+    public static String getNodeText(Element param) {
+	param.normalize();
+	Node n = param.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	if (n == null) {
+	    return "";
+	}
+	return n.getNodeValue();
+    }
+
+}
Index: /main/tags/3.01/gsdl3/extensions/vishnu/web/WEB-INF/web.xml
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/web/WEB-INF/web.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/web/WEB-INF/web.xml	(revision 21112)
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<!-- this is based on a sample web.xml file provided by 
+http://jakarta.apache.org/tomcat/tomcat-4.0-doc/appdev/web.xml.txt
+-->
+<web-app>
+
+  
+  <!-- General description of your web application -->
+  
+  <display-name>VISHNU</display-name>
+  <description>
+    Collection server for Imperial College's new Vishnu Visualisation tool.
+  </description>
+  
+  <context-param>
+    <param-name>webmaster</param-name>
+    <param-value>enter your email address here</param-value>
+    <description>
+      The EMAIL address of the administrator to whom questions
+      and comments about this application should be addressed.
+    </description>
+  </context-param>
+  
+
+  <!-- Servlet definitions for the servlets that make up
+  your web application, including initialization
+  parameters.  
+  -->
+  <servlet>
+    <servlet-name>visualiser</servlet-name>
+    <description>imperial college vishnu servlet</description>
+    <servlet-class>vishnu.server.VisServlet</servlet-class>
+    <init-param>
+      <param-name>collectionsHome</param-name>
+      <!-- edit this to provide the full path to the collections directory-->
+      <!-- make sure the path separators are correct for your platform -->
+      <param-value>path-to-vishnu/collections</param-value>
+    </init-param>
+  </servlet>
+
+  <!-- Define mappings that are used by the servlet container to
+  translate a particular request URI (context-relative) to a
+  particular servlet.  The examples below correspond to the
+  servlet descriptions above.  -->
+  <servlet-mapping>
+    <servlet-name>visualiser</servlet-name>
+    <url-pattern>/server</url-pattern>
+  </servlet-mapping>
+  <!-- Define the default session timeout for your application,
+  in minutes.  From a servlet or JSP page, you can modify
+  the timeout for a particular session dynamically by using
+  HttpSession.getMaxInactiveInterval(). -->
+  
+  <session-config>
+    <session-timeout>30</session-timeout>    <!-- 30 minutes -->
+  </session-config>
+  
+  
+</web-app>
+
+
Index: /main/tags/3.01/gsdl3/extensions/vishnu/web/index.html
===================================================================
--- /main/tags/3.01/gsdl3/extensions/vishnu/web/index.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/extensions/vishnu/web/index.html	(revision 21112)
@@ -0,0 +1,64 @@
+<html>
+  <head>
+    <title>Imperial College Visualizer</title>
+  </head>
+  <body>
+<h1>VISHNU:</h1>
+
+<h2>Overview</h2>
+This is a document search engine with several new visualization front
+ends that aid navigation through the set of documents returned by a
+query. Keywords from the document set are identified and selected on
+the fly. These keywords are then used to compute document clusters. The
+clusters can be displayed and explored using three visualizations:
+Sammon cluster view, Dendro map or radial interactive visualization.<br>      
+
+<h2>Quick start</h2>
+<p>Choose a collection from the colection menu. Type a query into the search box and click the Find button. Each pane will then display the query
+results in a different manner.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight:bold;">PlainVis</span> ---&nbsp; This shows a ranked list of documents
+retrieved. Clicking on the link for a document (in the left most
+column) will bring up a new window containing the document.<br>
+</p>
+
+<p style="margin-left: 40px;"><span style="font-weight: bold;">RadialVis</span> --- This shows the
+key words from the retrieved document set around a circle with the
+documents represented by dots within the circle. The dots are arranged
+closest to the words that contain them. Clicking on a word will
+highlight the documents containing that word. Correspondingly, clicking
+on a dot will show the words that it contains. Words can be dragged
+which will move the dots, thus changing the influence of that particular
+word on the arrangement.<br>
+At the bottom part of the pane key words can be selected and de-selected.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">SammonVis</span> --- This displays
+documents clustered by key words. As you mouse over a cluster a menu is
+displayed. This shows the number of documents in the cluster, and, at
+the bottom of the menu, the top ranked key words used to form the
+cluster. Clicking on the number of documents brings up the key words
+and their percentages in the left hand frame and a summary of the
+documents in the right hand frame. It is possible to drill down two
+levels through this menu. To return to a higher level use the button at
+the top of the frame.<br>
+</p>
+<p style="margin-left: 40px;"><span style="font-weight: bold;">DendroVis</span> ---&nbsp; This
+displays the document clusters arranged in a hierarchical format. As
+you mouse over nodes in the display a menu similar to the SammonVis
+menu is displayed. You can drill down through the hierarchy to examine
+deeper nodes. The key words are displayed at each node. The menu
+indicates if these are from the outer twigs (bottom) of the hierarchy.
+Clicking on the number of documents in the node displays the keywords
+and document summaries.<br>
+</p>
+
+      
+      <p>
+	<center>
+	<applet codebase='lib' code='vishnu.testvis.visual.Vishnu.class' archive='vishnu.jar' width='900' height='700'>
+	  <param name='library' value='http://localhost:8080/vishnu/server'/>
+	</applet>
+      </p>
+    </center>
+  </body>
+</html>
Index: /main/tags/3.01/gsdl3/gs3-setup.bat
===================================================================
--- /main/tags/3.01/gsdl3/gs3-setup.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/gs3-setup.bat	(revision 21112)
@@ -0,0 +1,72 @@
+@echo off
+
+if exist gs3-setup.bat goto start
+  echo This script must be run from within the Greenstone 3 home directory
+  goto exit
+
+:start
+:: ---- Set the GSDL3HOME variable to the current directory ----
+cd | winutil\setvar.exe GSDL3SRCHOME > setgsdl3.bat
+call setgsdl3.bat
+del setgsdl3.bat
+echo GSDL3SRCHOME: %GSDL3SRCHOME%
+set GSDL3HOME=%GSDL3SRCHOME%\web
+echo GSDL3HOME: %GSDL3HOME%
+
+:: change if using external tomcat
+set TOMCAT_HOME=%GSDL3SRCHOME%\packages\tomcat
+
+:: ---- Set other important environment variables ----
+set GSDLOS=windows
+set PATH=%PATH%;%GSDL3SRCHOME%\bin;%GSDL3SRCHOME%\bin\script;%GSDL3SRCHOME%\lib\jni;%GSDL3SRCHOME%\packages\mysql\bin
+
+:: ---- Set the CLASSPATH environment variable ----
+set CLASSPATH=%GSDL3HOME%\WEB-INF\classes;%CLASSPATH%
+if exist setcp.bat del setcp.bat
+for %%j in ("%GSDL3HOME%"\WEB-INF\lib\*.jar) do echo set CLASSPATH=%%CLASSPATH%%;%%j>> setcp.bat
+for %%j in ("%GSDL3SRCHOME%"\lib\jni\*.jar) do echo set CLASSPATH=%%CLASSPATH%%;%%j>> setcp.bat
+for %%j in ("%TOMCAT_HOME%"\common\endorsed\*.jar) do echo set CLASSPATH=%%CLASSPATH%%;%%j>> setcp.bat
+
+call setcp.bat
+del setcp.bat
+
+echo CLASSPATH: %CLASSPATH%
+
+::cd %GSDL3HOME%
+
+:findJava
+:: ---- Check Java SDK exists ----
+set JAVAPATH=
+
+:: Some users may set the above line manually
+if not "%JAVAPATH%" == "" goto testJava
+
+    :: If it is set, use the JAVA_HOME environment variable
+    if not "%JAVA_HOME%" == "" goto exit
+
+    :: Look in the registry for Java SDK installations
+    type nul > jdk.reg
+    regedit /E jdk.reg "HKEY_LOCAL_MACHINE\SOFTWARE\JavaSoft\Java Development Kit"
+    type jdk.reg > jdk.txt
+    del jdk.reg
+
+    winutil\findjava.exe jdk.txt | winutil\setvar.exe JAVAPATH > setjava.bat
+    del jdk.txt
+    call setjava.bat
+    del setjava.bat
+
+    :: If a suitable installation was found in the registry, check it
+    if not "%JAVAPATH%" == "" goto testJava
+
+:noJava
+    echo.
+    echo Failed to locate an appropriate version of Java. You must install a
+    echo Java Development Kit (version 1.4 or greater) before running the
+    echo Greenstone 3 Digital Library Software.
+    pause
+    goto exit
+
+:testJava
+set JAVA_HOME=%JAVAPATH%
+
+:exit
Index: /main/tags/3.01/gsdl3/gs3-setup.sh
===================================================================
--- /main/tags/3.01/gsdl3/gs3-setup.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/gs3-setup.sh	(revision 21112)
@@ -0,0 +1,126 @@
+# if this file is executed, /bin/sh is used, as we don't start with #!
+# this should work under ash, bash, zsh, ksh, sh style shells.
+
+# make sure we are sourced, and not run
+
+if test "$0" != "`echo $0 | sed s/gs3-setup\.sh//`" ; then
+# if $0 contains "gs3-setup.sh" we've been run... $0 is shellname if sourced.
+# One exception is zsh has an option to set it temporarily to the script name
+  if test -z "$ZSH_NAME" ; then
+  # we aren't using zsh
+  gsdl_not_sourced=true
+  fi
+fi
+
+if test -n "$gsdl_not_sourced" ; then
+  echo "	Error: Make sure you source this script, not execute it. Eg:"
+  echo "		$ source gs3-setup.sh"
+  echo "	or"
+  echo "		$ . ./gs3-setup.sh"
+  echo "	not"
+  echo "		$ ./gs3-setup.sh"
+  unset gsdl_not_sourced
+  exit 1
+fi
+
+if test ! -f gs3-setup.sh ; then
+  echo "You must source the script from within the Greenstone home directory"
+  exit 1
+fi
+
+GSDL3SRCHOME=`pwd`
+GSDL3HOME=$GSDL3SRCHOME/web
+export GSDL3HOME
+export GSDL3SRCHOME
+GSDLOS=`uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+# check for running bash under cygwin
+if test "`echo $GSDLOS | sed 's/cygwin//'`" != "$GSDLOS" ; then
+    GSDLOS=windows
+fi
+export GSDLOS
+
+#change this if external tomcat
+TOMCAT_HOME=$GSDL3SRCHOME/packages/tomcat
+
+PATH=$GSDL3SRCHOME/packages/mysql/bin:$GSDL3SRCHOME/bin/script:$GSDL3SRCHOME/bin:$PATH
+export PATH
+MANPATH=$MANPATH:$GSDL3SRCHOME/doc/man
+export MANPATH
+
+CLASSPATH=$GSDL3HOME/WEB-INF/classes:$CLASSPATH
+for JARFILE in $GSDL3SRCHOME/lib/jni/*.jar; do
+    CLASSPATH=$CLASSPATH:$JARFILE
+done
+for JARFILE in $GSDL3HOME/WEB-INF/lib/*.jar; do
+    CLASSPATH=$CLASSPATH:$JARFILE
+done
+for JARFILE in $TOMCAT_HOME/common/endorsed/*.jar; do
+    CLASSPATH=$CLASSPATH:$JARFILE
+done    
+
+export CLASSPATH
+
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GSDL3SRCHOME/lib/jni:$GSDL3SRCHOME/packages/mysql/lib/mysql
+export LD_LIBRARY_PATH
+## for mac
+DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$GSDL3SRCHOME/lib/jni:$GSDL3SRCHOME/packages/mysql/lib/mysql
+export DYLD_LIBRARY_PATH
+
+
+# see if we can find java
+java="java"
+jfound=1
+jok=1
+# if JAVAHOME or JAVA_HOME is set we'll use it to determine where java lives, otherwise
+#  we just hope it's on the search path
+if [ "$JAVA_HOME" != "" ] ; then
+    java="$JAVA_HOME/bin/java"
+elif [ "$JAVAHOME" != "" ] ; then
+    java="$JAVAHOME/bin/java"
+    export JAVA_HOME=$JAVAHOME
+else
+    java=`which java 2> /dev/null`
+    if [ ! -x "$java" ]; then
+	echo "Failed to locate Java. You must install a java runtime environment"
+	echo "(version 1.4 or greater) before installing Greenstone 3."
+	jfound=0
+	jok=0
+    else
+	export JAVA_HOME=`echo $java | sed -n 's/[\\\/]bin[\\\/]java$//p'`
+    fi
+fi
+if [ $jfound -eq 1 ]; then
+    javaversion=`$java -version 2>&1 | sed -n 's/^java version \"\(.*\)\"/\1/p'`
+    jvmajor=`echo $javaversion | sed -n 's/^\([0-9]*\).*$/\1/p'`
+    jvminor=`echo $javaversion | sed -n 's/^[0-9]*\.\([0-9]*\).*$/\1/p'`
+    
+    if [ $jvmajor -lt 1 ] ; then
+	jok=0
+    fi
+    if [ $jvmajor -eq 1 ] && [ $jvminor -lt 4 ] ; then
+	jok=0
+    fi
+fi
+
+if [ $jfound -eq 1 ] && [ $jok -eq 0 ] ; then
+    echo "The version of the java runtime environment you have installed is too"
+    echo "old to run Greenstone 3. Please install a new version of the JRE (version"
+    echo "1.4 or newer) and rerun this installation."
+else
+    echo "Your environment has successfully been set up to run Greenstone3"
+fi
+
+#do we need to check for perl???
+# make sure perl is ok
+#perl=`which perl 2> /dev/null`
+# echo "perl: $perl"
+#if [ ! -x "$perl" ] ; then
+#    echo "Greenstone 3 requires perl in order to operate but this installation"
+#    echo "could not detect perl on your system. Please ensure that perl is installed"
+#    echo "and is on your search path then rerun this installation script."
+#    exit 1
+#fi
+
+
+
+
Index: /main/tags/3.01/gsdl3/lib/perl/cpan/XML/Writer.pm
===================================================================
--- /main/tags/3.01/gsdl3/lib/perl/cpan/XML/Writer.pm	(revision 21112)
+++ /main/tags/3.01/gsdl3/lib/perl/cpan/XML/Writer.pm	(revision 21112)
@@ -0,0 +1,1407 @@
+########################################################################
+# Writer.pm - write an XML document.
+# Copyright (c) 1999 by Megginson Technologies.
+# No warranty.  Commercial and non-commercial use freely permitted.
+#
+# $Id$
+########################################################################
+
+package XML::Writer;
+
+require 5.004;
+
+use strict;
+use vars qw($VERSION);
+use Carp;
+use IO;
+
+$VERSION = "0.4";
+
+
+
+
+########################################################################
+# Constructor.
+########################################################################
+
+#
+# Public constructor.
+#
+# This actually does most of the work of the module: it defines closures
+# for all of the real processing, and selects the appropriate closures
+# to use based on the value of the UNSAFE parameter.  The actual methods
+# are just stubs.
+#
+sub new {
+  my ($class, %params) = (@_);
+
+				# If the user wants namespaces,
+				# intercept the request here; it will
+				# come back to this constructor
+				# from within XML::Writer::Namespaces::new()
+  if ($params{NAMESPACES}) {
+    delete $params{NAMESPACES};
+    return new XML::Writer::Namespaces(%params);
+  }
+
+				# Set up $self and basic parameters
+  my $self;
+  my $output;
+  my $unsafe = $params{UNSAFE};
+  my $newlines = $params{NEWLINES};
+  my $dataMode = $params{DATA_MODE};
+  my $dataIndent = $params{DATA_INDENT};
+
+				# If the NEWLINES parameter is specified,
+				# set the $nl variable appropriately
+  my $nl = '';
+  if ($newlines) {
+    $nl = "\n";
+  }
+
+
+				# Parse variables
+  my @elementStack = ();
+  my $elementLevel = 0;
+  my %seen = ();
+
+  my $hasData = 0;
+  my @hasDataStack = ();
+  my $hasElement = 0;
+  my @hasElementStack = ();
+
+  #
+  # Private method to show attributes.
+  #
+  my $showAttributes = sub {
+    my $atts = $_[0];
+    my $i = 1;
+    while ($atts->[$i]) {
+      my $aname = $atts->[$i++];
+      my $value = _escapeLiteral($atts->[$i++]);
+      $output->print(" $aname=\"$value\"");
+    }
+  };
+
+				# Method implementations: the SAFE_
+				# versions perform error checking
+				# and then call the regular ones.
+  my $end = sub {
+    $output->print("\n");
+  };
+
+  my $SAFE_end = sub {
+    if (!$seen{ELEMENT}) {
+      croak("Document cannot end without a document element");
+    } elsif ($elementLevel > 0) {
+      croak("Document ended with unmatched start tag(s): @elementStack");
+    } else {
+      @elementStack = ();
+      $elementLevel = 0;
+      %seen = ();
+      &{$end};
+    }
+  };
+
+  my $xmlDecl = sub {
+    my ($encoding, $standalone) = (@_);
+    if ($standalone && $standalone ne 'no') {
+      $standalone = 'yes';
+    }
+    $encoding = "UTF-8" unless $encoding;
+    $output->print("<?xml version=\"1.0\"");
+    if ($encoding) {
+      $output->print(" encoding=\"$encoding\"");
+    }
+    if ($standalone) {
+      $output->print(" standalone=\"$standalone\"");
+    }
+    $output->print("?>\n");
+  };
+
+  my $SAFE_xmlDecl = sub {
+    if ($seen{ANYTHING}) {
+      croak("The XML declaration is not the first thing in the document");
+    } else {
+      $seen{ANYTHING} = 1;
+      $seen{XMLDECL} = 1;
+      &{$xmlDecl};
+    }
+  };
+
+  my $pi = sub {
+    my ($target, $data) = (@_);
+    if ($data) {
+      $output->print("<?$target $data?>");
+    } else {
+      $output->print("<?$target?>");
+    }
+    if ($elementLevel == 0) {
+      $output->print("\n");
+    }
+  };
+
+  my $SAFE_pi = sub {
+    my ($name, $data) = (@_);
+    $seen{ANYTHING} = 1;
+    if ($name =~ /xml/i) {
+      carp("Processing instruction target begins with 'xml'");
+    } 
+
+    if ($name =~ /\?\>/ || $data =~ /\?\>/) {
+      croak("Processing instruction may not contain '?>'");
+    } else {
+      &{$pi};
+    }
+  };
+
+  my $comment = sub {
+    my $data = $_[0];
+    $output->print("<!-- $data -->");
+    if ($elementLevel == 0) {
+      $output->print("\n");
+    }
+  };
+
+  my $SAFE_comment = sub {
+    my $data = $_[0];
+    if ($data =~ /--/) {
+      carp("Interoperability problem: \"--\" in comment text");
+    }
+
+    if ($data =~ /-->/) {
+      croak("Comment may not contain '-->'");
+    } else {
+      $seen{ANYTHING} = 1;
+      &{$comment};
+    }
+  };
+
+  my $doctype = sub {
+    my ($name, $publicId, $systemId) = (@_);
+    $output->print("<!DOCTYPE $name");
+    if ($publicId) {
+      $output->print(" PUBLIC \"$publicId\" \"$systemId\"");
+    } elsif ($systemId) {
+      $output->print(" SYSTEM \"$systemId\"");
+    }
+    $output->print(">\n");
+  };
+
+  my $SAFE_doctype = sub {
+    my $name = $_[0];
+    if ($seen{DOCTYPE}) {
+      croak("Attempt to insert second DOCTYPE declaration");
+    } elsif ($seen{ELEMENT}) {
+      croak("The DOCTYPE declaration must come before the first start tag");
+    } else {
+      $seen{ANYTHING} = 1;
+      $seen{DOCTYPE} = $name;
+      &{$doctype};
+    }
+  };
+
+  my $startTag = sub {
+    my $name = $_[0];
+    if ($dataMode) {
+      $output->print("\n");
+      $output->print(" " x ($elementLevel * $dataIndent));
+    }
+    $elementLevel++;
+    push @elementStack, $name;
+    $output->print("<$name");
+    &{$showAttributes}(\@_);
+    $output->print("$nl>");
+    if ($dataMode) {
+      $hasElement = 1;
+      push @hasDataStack, $hasData;
+      $hasData = 0;
+      push @hasElementStack, $hasElement;
+      $hasElement = 0;
+    }
+  };
+
+  my $SAFE_startTag = sub {
+    my $name = $_[0];
+
+    _checkAttributes(\@_);
+
+    if ($seen{ELEMENT} && $elementLevel == 0) {
+      croak("Attempt to insert start tag after close of document element");
+    } elsif ($elementLevel == 0 && $seen{DOCTYPE} && $name ne $seen{DOCTYPE}) {
+      croak("Document element is \"$name\", but DOCTYPE is \""
+	    . $seen{DOCTYPE}
+	    . "\"");
+    } elsif ($dataMode && $hasData) {
+      croak("Mixed content not allowed in data mode: element $name");
+    } else {
+      $seen{ANYTHING} = 1;
+      $seen{ELEMENT} = 1;
+      &{$startTag};
+    }
+  };
+
+  my $emptyTag = sub {
+    my $name = $_[0];
+    if ($dataMode) {
+      $output->print("\n");
+      $output->print(" " x ($elementLevel * $dataIndent));
+    }
+    $output->print("<$name");
+    &{$showAttributes}(\@_);
+    $output->print("$nl />");
+    if ($dataMode) {
+      $hasElement = 1;
+    }
+  };
+
+  my $SAFE_emptyTag = sub {
+    my $name = $_[0];
+
+    _checkAttributes(\@_);
+
+    if ($seen{ELEMENT} && $elementLevel == 0) {
+      croak("Attempt to insert empty tag after close of document element");
+    } elsif ($elementLevel == 0 && $seen{DOCTYPE} && $name ne $seen{DOCTYPE}) {
+      croak("Document element is \"$name\", but DOCTYPE is \""
+	    . $seen{DOCTYPE}
+	    . "\"");
+    } elsif ($dataMode && $hasData) {
+      croak("Mixed content not allowed in data mode: element $name");
+    } else {
+      $seen{ANYTHING} = 1;
+      $seen{ELEMENT} = 1;
+      &{$emptyTag};
+    }
+  };
+
+  my $endTag = sub {
+    my $name = $_[0];
+    my $currentName = pop @elementStack;
+    $name = $currentName unless $name;
+    $elementLevel--;
+    if ($dataMode && $hasElement) {
+      $output->print("\n");
+      $output->print(" " x ($elementLevel * $dataIndent));
+    }
+    $output->print("</$name$nl>");
+    if ($dataMode) {
+      $hasData = pop @hasDataStack;
+      $hasElement = pop @hasElementStack;
+    }
+  };
+
+  my $SAFE_endTag = sub {
+    my $name = $_[0];
+    my $oldName = $elementStack[$#elementStack];
+    if ($elementLevel <= 0) {
+      croak("End tag \"$name\" does not close any open element");
+    } elsif ($name && ($name ne $oldName)) {
+      croak("Attempt to end element \"$oldName\" with \"$name\" tag");
+    } else {
+      &{$endTag};
+    }
+  };
+
+  my $characters = sub {
+    my $data = $_[0];
+    if ($data =~ /[\&\<\>]/) {
+      $data =~ s/\&/\&amp\;/g;
+      $data =~ s/\</\&lt\;/g;
+      $data =~ s/\>/\&gt\;/g;
+    }
+    $output->print($data);
+    $hasData = 1;
+  };
+
+  my $SAFE_characters = sub {
+    if ($elementLevel < 1) {
+      croak("Attempt to insert characters outside of document element");
+    } elsif ($dataMode && $hasElement) {
+      croak("Mixed content not allowed in data mode: characters");
+    } else {
+      &{$characters};
+    }
+  };
+
+   my $charactersXML = sub {
+    my $data = $_[0];
+  #  if ($data =~ /[\&\<\>]/) {
+  #    $data =~ s/\&/\&amp\;/g;
+  #    $data =~ s/\</\&lt\;/g;
+  #    $data =~ s/\>/\&gt\;/g;
+  #  }
+    $output->print($data);
+    $hasData = 1;
+  };
+
+  my $SAFE_charactersXML = sub {
+    if ($elementLevel < 1) {
+      croak("Attempt to insert characters outside of document element");
+    } elsif ($dataMode && $hasElement) {
+      croak("Mixed content not allowed in data mode: characters");
+    } else {
+      &{$charactersXML};
+    }
+  };
+
+  
+				# Assign the correct closures based on
+				# the UNSAFE parameter
+  if ($unsafe) {
+    $self = {'END' => $end,
+	     'XMLDECL' => $xmlDecl,
+	     'PI' => $pi,
+	     'COMMENT' => $comment,
+	     'DOCTYPE' => $doctype,
+	     'STARTTAG' => $startTag,
+	     'EMPTYTAG' => $emptyTag,
+	     'ENDTAG' => $endTag,
+	     'CHARACTERS' => $characters,
+	     'CHARACTERSXML' => $charactersXML};
+  } else {
+    $self = {'END' => $SAFE_end,
+	     'XMLDECL' => $SAFE_xmlDecl,
+	     'PI' => $SAFE_pi,
+	     'COMMENT' => $SAFE_comment,
+	     'DOCTYPE' => $SAFE_doctype,
+	     'STARTTAG' => $SAFE_startTag,
+	     'EMPTYTAG' => $SAFE_emptyTag,
+	     'ENDTAG' => $SAFE_endTag,
+	     'CHARACTERS' => $SAFE_characters,
+	     'CHARACTERSXML' => $SAFE_charactersXML};
+  }
+
+				# Query methods
+  $self->{'IN_ELEMENT'} = sub {
+    my ($ancestor) = (@_);
+    return $elementStack[$#elementStack] eq $ancestor;
+  };
+
+  $self->{'WITHIN_ELEMENT'} = sub {
+    my ($ancestor) = (@_);
+    my $el;
+    foreach $el (@elementStack) {
+      return 1 if $el eq $ancestor;
+    }
+    return 0;
+  };
+
+  $self->{'CURRENT_ELEMENT'} = sub {
+    return $elementStack[$#elementStack];
+  };
+
+  $self->{'ANCESTOR'} = sub {
+    my ($n) = (@_);
+    return $elementStack[$#elementStack-$n];
+  };
+
+				# Set and get the output destination.
+  $self->{'GETOUTPUT'} = sub {
+    return $output;
+  };
+
+  $self->{'SETOUTPUT'} = sub {
+    my $newOutput = $_[0];
+				# If there is no OUTPUT parameter,
+				# use standard output
+    unless ($newOutput) {
+      $newOutput = new IO::Handle();
+      $newOutput->fdopen(fileno(STDOUT), "w") ||
+	croak("Cannot write to standard output");
+    }
+    $output = $newOutput;
+  };
+
+  $self->{'SETDATAMODE'} = sub {
+    $dataMode = $_[0];
+  };
+
+  $self->{'GETDATAMODE'} = sub {
+    return $dataMode;
+  };
+
+  $self->{'SETDATAINDENT'} = sub {
+    $dataIndent = $_[0];
+  };
+
+  $self->{'GETDATAINDENT'} = sub {
+    return $dataIndent;
+  };
+
+				# Set the output.
+  &{$self->{'SETOUTPUT'}}($params{'OUTPUT'});
+
+				# Return the blessed object.
+  return bless $self, $class;
+}
+
+
+
+
+########################################################################
+# Public methods
+########################################################################
+
+#
+# Finish writing the document.
+#
+sub end {
+  my $self = shift;
+  &{$self->{END}};
+}
+
+#
+# Write an XML declaration.
+#
+sub xmlDecl {
+  my $self = shift;
+  &{$self->{XMLDECL}};
+}
+
+#
+# Write a processing instruction.
+#
+sub pi {
+  my $self = shift;
+  &{$self->{PI}};
+}
+
+#
+# Write a comment.
+#
+sub comment {
+  my $self = shift;
+  &{$self->{COMMENT}};
+}
+
+#
+# Write a DOCTYPE declaration.
+#
+sub doctype {
+  my $self = shift;
+  &{$self->{DOCTYPE}};
+}
+
+#
+# Write a start tag.
+#
+sub startTag {
+  my $self = shift;
+  &{$self->{STARTTAG}};
+}
+
+#
+# Write an empty tag.
+#
+sub emptyTag {
+  my $self = shift;
+  &{$self->{EMPTYTAG}};
+}
+
+#
+# Write an end tag.
+#
+sub endTag {
+  my $self = shift;
+  &{$self->{ENDTAG}};
+}
+
+#
+# Write a simple data element.
+#
+sub dataElement {
+  my ($self, $name, $data, %atts) = (@_);
+  $self->startTag($name, %atts);
+  $self->characters($data);
+  $self->endTag($name);
+}
+
+#
+# Write character data.
+#
+sub characters {
+  my $self = shift;
+  &{$self->{CHARACTERS}};
+}
+
+sub charactersXML {
+  my $self = shift;
+  &{$self->{CHARACTERSXML}};
+}
+    
+#
+# Query the current element.
+#
+sub in_element {
+  my $self = shift;
+  return &{$self->{IN_ELEMENT}};
+}
+
+#
+# Query the ancestors.
+#
+sub within_element {
+  my $self = shift;
+  return &{$self->{WITHIN_ELEMENT}};
+}
+
+#
+# Get the name of the current element.
+#
+sub current_element {
+  my $self = shift;
+  return &{$self->{CURRENT_ELEMENT}};
+}
+
+#
+# Get the name of the numbered ancestor (zero-based).
+#
+sub ancestor {
+  my $self = shift;
+  return &{$self->{ANCESTOR}};
+}
+
+#
+# Get the current output destination.
+#
+sub getOutput {
+  my $self = shift;
+  return &{$self->{GETOUTPUT}};
+}
+
+
+#
+# Set the current output destination.
+#
+sub setOutput {
+  my $self = shift;
+  return &{$self->{SETOUTPUT}};
+}
+
+#
+# Set the current data mode (true or false).
+#
+sub setDataMode {
+  my $self = shift;
+  return &{$self->{SETDATAMODE}};
+}
+
+
+#
+# Get the current data mode (true or false).
+#
+sub getDataMode {
+  my $self = shift;
+  return &{$self->{GETDATAMODE}};
+}
+
+
+#
+# Set the current data indent step.
+#
+sub setDataIndent {
+  my $self = shift;
+  return &{$self->{SETDATAINDENT}};
+}
+
+
+#
+# Get the current data indent step.
+#
+sub getDataIndent {
+  my $self = shift;
+  return &{$self->{GETDATAINDENT}};
+}
+
+
+#
+# Empty stub.
+#
+sub addPrefix {
+}
+
+
+#
+# Empty stub.
+#
+sub removePrefix {
+}
+
+
+
+
+########################################################################
+# Private functions.
+########################################################################
+
+#
+# Private: check for duplicate attributes.
+# Note - this starts at $_[1], because $_[0] is assumed to be an
+# element name.
+#
+sub _checkAttributes {
+  my %anames;
+  my $i = 1;
+  while ($_[$i]) {
+    my $name = $_[$i];
+    $i += 2;
+    if ($anames{$name}) {
+      croak("Two attributes named \"$name\"");
+    } else {
+      $anames{$name} = 1;
+    }
+  }
+}
+
+#
+# Private: escape an attribute value literal.
+#
+sub _escapeLiteral {
+  my $data = $_[0];
+  if ($data =~ /[\&\<\>\"]/) {
+    $data =~ s/\&/\&amp\;/g;
+    $data =~ s/\</\&lt\;/g;
+    $data =~ s/\>/\&gt\;/g;
+    $data =~ s/\"/\&quot\;/g;
+  }
+  return $data;
+}
+
+
+
+
+########################################################################
+# XML::Writer::Namespaces - subclass for Namespace processing.
+########################################################################
+
+package XML::Writer::Namespaces;
+use strict;
+use vars qw(@ISA);
+use Carp;
+
+@ISA = qw(XML::Writer);
+
+#
+# Constructor
+#
+sub new {
+  my ($class, %params) = (@_);
+
+  my $unsafe = $params{UNSAFE};
+
+				# Snarf the prefix map, if any, and
+				# note the default prefix.
+  my %prefixMap = ();
+  if ($params{PREFIX_MAP}) {
+    %prefixMap = (%{$params{PREFIX_MAP}});
+    delete $params{PREFIX_MAP};
+  }
+  my $defaultPrefix = $prefixMap{''};
+  delete $prefixMap{''};
+
+				# Generate the reverse map for URIs
+  my %uriMap = ();
+  my $key;
+  foreach $key (keys(%prefixMap)) {
+    $uriMap{$prefixMap{$key}} = $key;
+  }
+
+				# Create an instance of the parent.
+  my $self = new XML::Writer(%params);
+
+				# Snarf the parent's methods that we're
+				# going to override.
+  my $OLD_startTag = $self->{STARTTAG};
+  my $OLD_emptyTag = $self->{EMPTYTAG};
+  my $OLD_endTag = $self->{ENDTAG};
+
+				# State variables
+  my $prefixCounter = 1;
+  my @nsDecls = ();
+  my $nsDecls = {};
+  my @nsDefaultDecl = ();
+  my $nsDefaultDecl = undef;
+  my @nsCopyFlag = ();
+  my $nsCopyFlag = 0;
+
+  #
+  # Push the current declaration state.
+  #
+  my $pushState = sub {
+    push @nsDecls, $nsDecls;
+    push @nsDefaultDecl, $nsDefaultDecl;
+    push @nsCopyFlag, $nsCopyFlag;
+    $nsCopyFlag = 0;
+  };
+
+
+  #
+  # Pop the current declaration state.
+  #
+  my $popState = sub {
+    $nsDecls = pop @nsDecls;
+    $nsDefaultDecl = pop @nsDefaultDecl;
+    $nsCopyFlag = pop @nsCopyFlag;
+  };
+
+  #
+  # Generate a new prefix.
+  #
+  my $genPrefix = sub {
+    my $prefix;
+    do {
+      $prefix = "__NS$prefixCounter";
+      $prefixCounter++;
+    } while ($uriMap{$prefix});
+    return $prefix;
+  };
+
+  #
+  # Perform namespace processing on a single name.
+  #
+  my $processName = sub {
+    my ($nameref, $atts, $attFlag) = (@_);
+    my ($uri, $local) = @{$$nameref};
+    my $prefix = $prefixMap{$uri};
+
+				# Is this an element name that matches
+				# the default NS?
+    if (!$attFlag && ($uri eq $defaultPrefix)) {
+      unless ($nsDefaultDecl) {
+	push @{$atts}, 'xmlns';
+	push @{$atts}, $uri;
+	$nsDefaultDecl = 1;
+      }
+      $$nameref = $local;
+      
+				# Is there a straight-forward prefix?
+    } elsif ($prefix) {
+      unless ($nsDecls->{$uri}) {
+				# Copy on write (FIXME: duplicated)
+	unless ($nsCopyFlag) {
+	  $nsCopyFlag = 1;
+	  my %decls = (%{$nsDecls});
+	  $nsDecls = \%decls;
+	}
+	$nsDecls->{$uri} = $prefix;
+	push @{$atts}, "xmlns:$prefix";
+	push @{$atts}, $uri;
+      }
+      $$nameref = "$prefix:$local";
+
+    } else {
+      $prefix = &{$genPrefix}();
+      $prefixMap{$uri} = $prefix;
+      $uriMap{$prefix} = $uri;
+      unless ($nsCopyFlag) {
+	$nsCopyFlag = 1;
+	my %decls = (%{$nsDecls});
+	$nsDecls = \%decls;
+      }
+      $nsDecls->{$uri} = $prefix;
+      push @{$atts}, "xmlns:$prefix";
+      push @{$atts}, $uri;
+      $$nameref = "$prefix:$local";
+    }
+  };
+
+
+  #
+  # Perform namespace processing on element and attribute names.
+  #
+  my $nsProcess = sub {
+    if (ref($_[0]->[0]) eq 'ARRAY') {
+      &{$processName}(\$_[0]->[0], $_[0], 0);
+    }
+    my $i = 1;
+    while ($_[0]->[$i]) {
+      if (ref($_[0]->[$i]) eq 'ARRAY') {
+	&{$processName}(\$_[0]->[$i], $_[0], 1);
+      }
+      $i += 2;
+    }
+  };
+
+  #
+  # Start tag, with NS processing
+  #
+  $self->{STARTTAG} = sub {
+    my $name = $_[0];
+    unless ($unsafe) {
+      _checkNSNames(\@_);
+    }
+    &{$pushState}();
+    &{$nsProcess}(\@_);
+    &{$OLD_startTag};
+  };
+
+
+  #
+  # Empty tag, with NS processing
+  #
+  $self->{EMPTYTAG} = sub {
+    unless ($unsafe) {
+      _checkNSNames(\@_);
+    }
+    &{$pushState}();
+    &{$nsProcess}(\@_);
+    &{$OLD_emptyTag};
+    &{$popState}();
+  };
+
+
+  #
+  # End tag, with NS processing
+  #
+  $self->{ENDTAG} = sub {
+    my $name = $_[0];
+    &{$nsProcess}(\@_);
+    &{$OLD_endTag};
+    &{$popState}();
+  };
+
+
+  #
+  # Processing instruction, but only if not UNSAFE.
+  #
+  unless ($unsafe) {
+    my $OLD_pi = $self->{PI};
+    $self->{PI} = sub {
+      my $target = $_[0];
+      if ($target =~ /:/) {
+	croak "PI target '$target' contains a colon.";
+      }
+      &{$OLD_pi};
+    }
+  };
+
+
+  #
+  # Add a prefix to the prefix map.
+  #
+  $self->{ADDPREFIX} = sub {
+    my ($uri, $prefix) = (@_);
+    if ($prefix) {
+      $prefixMap{$uri} = $prefix;
+      $uriMap{$prefix} = $uri;
+    } else {
+      $defaultPrefix = $uri;
+    }
+  };
+
+
+  #
+  # Remove a prefix from the prefix map.
+  #
+  $self->{REMOVEPREFIX} = sub {
+    my ($uri) = (@_);
+    if ($defaultPrefix eq $uri) {
+      $defaultPrefix = undef;
+    }
+    delete $prefixMap{$uri};
+  };
+
+
+  #
+  # Bless and return the object.
+  #
+  return bless $self, $class;
+}
+
+
+#
+# Add a preferred prefix for a namespace URI.
+#
+sub addPrefix {
+  my $self = shift;
+  return &{$self->{ADDPREFIX}};
+}
+
+
+#
+# Remove a preferred prefix for a namespace URI.
+#
+sub removePrefix {
+  my $self = shift;
+  return &{$self->{REMOVEPREFIX}};
+}
+
+
+#
+# Check names.
+#
+sub _checkNSNames {
+  my $names = $_[0];
+  my $i = 1;
+  my $name = $names->[0];
+
+				# Check the element name.
+  if (ref($name) eq 'ARRAY') {
+    if ($name->[1] =~ /:/) {
+      croak("Local part of element name '" .
+	    $name->[1] .
+	    "' contains a colon.");
+    }
+  } elsif ($name =~ /:/) {
+    croak("Element name '$name' contains a colon.");
+  }
+
+				# Check the attribute names.
+  while ($names->[$i]) {
+    my $name = $names->[$i];
+    if (ref($name) eq 'ARRAY') {
+      my $local = $name->[1];
+      if ($local =~ /:/) {
+	croak "Local part of attribute name '$local' contains a colon.";
+      }
+    } else {
+      if ($name =~ /^(xmlns|.*:)/) {
+	if ($name =~ /^xmlns/) {
+	  croak "Attribute name '$name' begins with 'xmlns'";
+	} elsif ($name =~ /:/) {
+	  croak "Attribute name '$name' contains ':'";
+	}
+      }
+    }
+    $i += 2;
+  }
+}
+
+
+1;
+__END__
+
+
+########################################################################
+# POD Documentation
+########################################################################
+
+=head1 NAME
+
+XML::Writer - Perl extension for writing XML documents.
+
+=head1 SYNOPSIS
+
+  use XML::Writer;
+  use IO;
+
+  my $output = new IO::File(">output.xml");
+
+  my $writer = new XML::Writer(OUTPUT => $output);
+  $writer->startTag("greeting", 
+                    "class" => "simple");
+  $writer->characters("Hello, world!");
+  $writer->endTag("greeting");
+  $writer->end();
+  $output->close();
+
+
+=head1 DESCRIPTION
+
+XML::Writer is a helper module for Perl programs that write an XML
+document.  The module handles all escaping for attribute values and
+character data and constructs different types of markup, such as tags,
+comments, and processing instructions.
+
+By default, the module performs several well-formedness checks to
+catch errors during output.  This behaviour can be extremely useful
+during development and debugging, but it can be turned off for
+production-grade code.
+
+The module can operate either in regular mode in or Namespace
+processing mode.  In Namespace mode, the module will generate
+Namespace Declarations itself, and will perform additional checks on
+the output.
+
+Additional support is available for a simplified data mode with no
+mixed content: newlines are automatically inserted around elements and
+elements can optionally be indented based as their nesting level.
+
+
+=head1 METHODS
+
+=head2 Writing XML
+
+=over 4
+
+=item new([$params])
+
+Create a new XML::Writer object:
+
+  my $writer = new XML::Writer(OUTPUT => $output, NEWLINES => 1);
+
+Arguments are an anonymous hash array of parameters:
+
+=over 4
+
+=item OUTPUT
+
+An object blessed into IO::Handle or one of its subclasses (such as
+IO::File); if this parameter is not present, the module will write to
+standard output.
+
+=item NAMESPACES
+
+A true (1) or false (0, undef) value; if this parameter is present and
+its value is true, then the module will accept two-member array
+reference in the place of element and attribute names, as in the
+following example:
+
+  my $rdfns = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+  my $writer = new XML::Writer(NAMESPACES => 1);
+  $writer->startTag([$rdfns, "Description"]);
+
+The first member of the array is a namespace URI, and the second part
+is the local part of a qualified name.  The module will automatically
+generate appropriate namespace declarations and will replace the URI
+part with a prefix.
+
+=item PREFIX_MAP
+
+A hash reference; if this parameter is present and the module is
+performing namespace processing (see the NAMESPACES parameter), then
+the module will use this hash to look up preferred prefixes for
+namespace URIs:
+
+
+  my $rdfns = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+  my $writer = new XML::Writer(NAMESPACES => 1,
+                               PREFIX_MAP => {$rdfns => 'rdf'});
+
+The keys in the hash table are namespace URIs, and the values are the
+associated prefixes.  If there is not a preferred prefix for the
+namespace URI in this hash, then the module will automatically
+generate prefixes of the form "__NS1", "__NS2", etc.
+
+To set the default namespace, use '' for the prefix.
+
+=item NEWLINES
+
+A true or false value; if this parameter is present and its value is
+true, then the module will insert an extra newline before the closing
+delimiter of start, end, and empty tags to guarantee that the document
+does not end up as a single, long line.  If the paramter is not
+present, the module will not insert the newlines.
+
+=item UNSAFE
+
+A true or false value; if this parameter is present and its value is
+true, then the module will skip most well-formedness error checking.
+If the parameter is not present, the module will perform the
+well-formedness error checking by default.  Turn off error checking at
+your own risk!
+
+=item DATA_MODE
+
+A true or false value; if this parameter is present and its value is
+true, then the module will enter a special data mode, inserting
+newlines automatically around elements and (unless UNSAFE is also
+specified) reporting an error if any element has both characters and
+elements as content.
+
+=item DATA_INDENT
+
+A numeric value; if this parameter is present, it represents the
+indent step for elements in data mode (it will be ignored when not in
+data mode).
+
+=back
+
+=item end()
+
+Finish creating an XML document.  This method will check that the
+document has exactly one document element, and that all start tags are
+closed:
+
+  $writer->end();
+
+=item xmlDecl([$encoding, $standalone])
+
+Add an XML declaration to the beginning of an XML document.  The
+version will always be "1.0".  If you provide a non-null encoding or
+standalone argument, its value will appear in the declaration (and
+non-null value for standalone except 'no' will automatically be
+converted to 'yes').
+
+  $writer->xmlDecl("UTF-8");
+
+=item doctype($name, [$publicId, $systemId])
+
+Add a DOCTYPE declaration to an XML document.  The declaration must
+appear before the beginning of the root element.  If you provide a
+publicId, you must provide a systemId as well, but you may provide
+just a system ID.
+
+  $writer->doctype("html");
+
+=item comment($text)
+
+Add a comment to an XML document.  If the comment appears outside the
+document element (either before the first start tag or after the last
+end tag), the module will add a carriage return after it to improve
+readability:
+
+  $writer->comment("This is a comment");
+
+=item pi($target [, $data])
+
+Add a processing instruction to an XML document:
+
+  $writer->pi('xml-stylesheet', 'href="style.css" type="text/css"');
+
+If the processing instruction appears outside the document element
+(either before the first start tag or after the last end tag), the
+module will add a carriage return after it to improve readability.
+
+The $target argument must be a single XML name.  If you provide the
+$data argument, the module will insert its contents following the
+$target argument, separated by a single space.
+
+=item startTag($name [, $aname1 => $value1, ...])
+
+Add a start tag to an XML document.  Any arguments after the element
+name are assumed to be name/value pairs for attributes: the module
+will escape all '&', '<', '>', and '"' characters in the attribute
+values using the predefined XML entities:
+
+  $writer->startTag('doc', 'version' => '1.0',
+                           'status' => 'draft',
+                           'topic' => 'AT&T');
+
+All start tags must eventually have matching end tags.
+
+=item emptyTag($name [, $aname1 => $value1, ...])
+
+Add an empty tag to an XML document.  Any arguments after the element
+name are assumed to be name/value pairs for attributes (see startTag()
+for details):
+
+  $writer->emptyTag('img', 'src' => 'portrait.jpg',
+                           'alt' => 'Portrait of Emma.');
+
+=item endTag([$name])
+
+Add an end tag to an XML document.  The end tag must match the closest
+open start tag, and there must be a matching and properly-nested end
+tag for every start tag:
+
+  $writer->endTag('doc');
+
+If the $name argument is omitted, then the module will automatically
+supply the name of the currently open element:
+
+  $writer->startTag('p');
+  $writer->endTag();
+
+=item dataElement($name, $data [, $aname1 => $value1, ...])
+
+Print an entire element containing only character data.  This is
+equivalent to
+
+  $writer->startTag($name [, $aname1 => $value1, ...]);
+  $writer->characters($data);
+  $writer->endTag($name);
+
+=item characters($data)
+
+Add character data to an XML document.  All '<', '>', and '&'
+characters in the $data argument will automatically be escaped using
+the predefined XML entities:
+
+  $writer->characters("Here is the formula: ");
+  $writer->characters("a < 100 && a > 5");
+
+You may invoke this method only within the document element
+(i.e. after the first start tag and before the last end tag).
+
+In data mode, you must not use this method to add whitespace between
+elements.
+
+=item setOutput($output)
+
+Set the current output destination, as in the OUTPUT parameter for the
+constructor.
+
+=item getOutput()
+
+Return the current output destination, as in the OUTPUT parameter for
+the constructor.
+
+=item setDataMode($mode)
+
+Enable or disable data mode, as in the DATA_MODE parameter for the
+constructor.
+
+=item getDataMode()
+
+Return the current data mode, as in the DATA_MODE parameter for the
+constructor.
+
+=item setDataIndent($step)
+
+Set the indent step for data mode, as in the DATA_INDENT parameter for
+the constructor.
+
+=item getDataIndent()
+
+Return the indent step for data mode, as in the DATA_INDENT parameter
+for the constructor.
+
+
+=back
+
+=head2 Querying XML
+
+=over 4
+
+=item in_element($name)
+
+Return a true value if the most recent open element matches $name:
+
+  if ($writer->in_element('dl')) {
+    $writer->startTag('dt');
+  } else {
+    $writer->startTag('li');
+  }
+
+=item within_element($name)
+
+Return a true value if any open elemnet matches $name:
+
+  if ($writer->within_element('body')) {
+    $writer->startTag('h1');
+  } else {
+    $writer->startTag('title');
+  }
+
+=item current_element()
+
+Return the name of the currently open element:
+
+  my $name = $writer->current_element();
+
+This is the equivalent of
+
+  my $name = $writer->ancestor(0);
+
+=item ancestor($n)
+
+Return the name of the nth ancestor, where $n=0 for the current open
+element.
+
+=back
+
+
+=head2 Additional Namespace Support
+
+WARNING: you must not use these methods while you are writing a
+document, or the results will be unpredictable.
+
+=over 4
+
+=item addPrefix($uri, $prefix)
+
+Add a preferred mapping between a Namespace URI and a prefix.  See
+also the PREFIX_MAP constructor parameter.
+
+To set the default namespace, omit the $prefix parameter or set it to
+''.
+
+=item removePrefix($uri)
+
+Remove a preferred mapping between a Namespace URI and a prefix.
+
+To set the default namespace, omit the $prefix parameter or set it to
+''.
+
+=back
+
+
+=head1 ERROR REPORTING
+
+With the default settings, the XML::Writer module can detect several
+basic XML well-formedness errors:
+
+=over 4
+
+=item *
+
+Lack of a (top-level) document element, or multiple document elements.
+
+=item *
+
+Unclosed start tags.
+
+=item *
+
+Misplaced delimiters in the contents of processing instructions or
+comments.
+
+=item *
+
+Misplaced or duplicate XML declaration(s).
+
+=item *
+
+Misplaced or duplicate DOCTYPE declaration(s).
+
+=item *
+
+Mismatch between the document type name in the DOCTYPE declaration and
+the name of the document element.
+
+=item *
+
+Mismatched start and end tags.
+
+=item *
+
+Attempts to insert character data outside the document element.
+
+=item *
+
+Duplicate attributes with the same name.
+
+=back
+
+During Namespace processing, the module can detect the following
+additional errors:
+
+=over 4
+
+=item *
+
+Attempts to use PI targets or element or attribute names containing a
+colon.
+
+=item *
+
+Attempts to use attributes with names beginning "xmlns".
+
+=back
+
+To ensure full error detection, a program must also invoke the end
+method when it has finished writing a document:
+
+  $writer->startTag('greeting');
+  $writer->characters("Hello, world!");
+  $writer->endTag('greeting');
+  $writer->end();
+
+This error reporting can catch many hidden bugs in Perl programs that
+create XML documents; however, if necessary, it can be turned off by
+providing an UNSAFE parameter:
+
+  my $writer = new XML::Writer(OUTPUT => $output, UNSAFE => 1);
+
+
+=head1 AUTHOR
+
+David Megginson, david@megginson.com
+
+=head1 SEE ALSO
+
+XML::Parser
+
+=cut
Index: /main/tags/3.01/gsdl3/packages/tomcat-extra/server.xml.in
===================================================================
--- /main/tags/3.01/gsdl3/packages/tomcat-extra/server.xml.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/packages/tomcat-extra/server.xml.in	(revision 21112)
@@ -0,0 +1,384 @@
+<!-- Example Server Configuration File -->
+<!-- Note that component elements are nested corresponding to their
+     parent-child relationships with each other -->
+
+<!-- A "Server" is a singleton element that represents the entire JVM,
+     which may contain one or more "Service" instances.  The Server
+     listens for a shutdown command on the indicated port.
+
+     Note:  A "Server" is not itself a "Container", so you may not
+     define subcomponents such as "Valves" or "Loggers" at this level.
+ -->
+
+<Server port="8005" shutdown="SHUTDOWN">
+
+  <!-- Comment these entries out to disable JMX MBeans support used for the 
+       administration web application -->
+  <Listener className="org.apache.catalina.core.AprLifecycleListener" />
+  <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener" />
+  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
+  <Listener className="org.apache.catalina.storeconfig.StoreConfigLifecycleListener"/>
+
+  <!-- Global JNDI resources -->
+  <GlobalNamingResources>
+
+    <!-- Test entry for demonstration purposes -->
+    <Environment name="simpleValue" type="java.lang.Integer" value="30"/>
+
+    <!-- Editable user database that can also be used by
+         UserDatabaseRealm to authenticate users -->
+    <Resource name="UserDatabase" auth="Container"
+              type="org.apache.catalina.UserDatabase"
+       description="User database that can be updated and saved"
+           factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
+          pathname="conf/tomcat-users.xml" />
+
+  </GlobalNamingResources>
+
+  <!-- A "Service" is a collection of one or more "Connectors" that share
+       a single "Container" (and therefore the web applications visible
+       within that Container).  Normally, that Container is an "Engine",
+       but this is not required.
+
+       Note:  A "Service" is not itself a "Container", so you may not
+       define subcomponents such as "Valves" or "Loggers" at this level.
+   -->
+
+  <!-- Define the Tomcat Stand-Alone Service -->
+  <Service name="Catalina">
+
+    <!-- A "Connector" represents an endpoint by which requests are received
+         and responses are returned.  Each Connector passes requests on to the
+         associated "Container" (normally an Engine) for processing.
+
+         By default, a non-SSL HTTP/1.1 Connector is established on port 8080.
+         You can also enable an SSL HTTP/1.1 Connector on port 8443 by
+         following the instructions below and uncommenting the second Connector
+         entry.  SSL support requires the following steps (see the SSL Config
+         HOWTO in the Tomcat 5 documentation bundle for more detailed
+         instructions):
+         * If your JDK version 1.3 or prior, download and install JSSE 1.0.2 or
+           later, and put the JAR files into "$JAVA_HOME/jre/lib/ext".
+         * Execute:
+             %JAVA_HOME%\bin\keytool -genkey -alias tomcat -keyalg RSA (Windows)
+             $JAVA_HOME/bin/keytool -genkey -alias tomcat -keyalg RSA  (Unix)
+           with a password value of "changeit" for both the certificate and
+           the keystore itself.
+
+         By default, DNS lookups are enabled when a web application calls
+         request.getRemoteHost().  This can have an adverse impact on
+         performance, so you can disable it by setting the
+         "enableLookups" attribute to "false".  When DNS lookups are disabled,
+         request.getRemoteHost() will return the String version of the
+         IP address of the remote client.
+    -->
+
+    <!-- Define a non-SSL HTTP/1.1 Connector on port 8080 -->
+    <Connector port="@port@" maxHttpHeaderSize="8192"
+               maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
+               enableLookups="false" redirectPort="8443" acceptCount="100"
+               connectionTimeout="20000" disableUploadTimeout="true" />
+    <!-- Note : To disable connection timeouts, set connectionTimeout value
+     to 0 -->
+	
+	<!-- Note : To use gzip compression you could set the following properties :
+	
+			   compression="on" 
+			   compressionMinSize="2048" 
+			   noCompressionUserAgents="gozilla, traviata" 
+			   compressableMimeType="text/html,text/xml"
+	-->
+
+    <!-- Define a SSL HTTP/1.1 Connector on port 8443 -->
+    <!--
+    <Connector port="8443" maxHttpHeaderSize="8192"
+               maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
+               enableLookups="false" disableUploadTimeout="true"
+               acceptCount="100" scheme="https" secure="true"
+               clientAuth="false" sslProtocol="TLS" />
+    -->
+
+    <!-- Define an AJP 1.3 Connector on port 8009 -->
+    <Connector port="8009" 
+               enableLookups="false" redirectPort="8443" protocol="AJP/1.3" />
+
+    <!-- Define a Proxied HTTP/1.1 Connector on port 8082 -->
+    <!-- See proxy documentation for more information about using this. -->
+    <!--
+    <Connector port="8082" 
+               maxThreads="150" minSpareThreads="25" maxSpareThreads="75"
+               enableLookups="false" acceptCount="100" connectionTimeout="20000"
+               proxyPort="80" disableUploadTimeout="true" />
+    -->
+
+    <!-- An Engine represents the entry point (within Catalina) that processes
+         every request.  The Engine implementation for Tomcat stand alone
+         analyzes the HTTP headers included with the request, and passes them
+         on to the appropriate Host (virtual host). -->
+
+    <!-- You should set jvmRoute to support load-balancing via AJP ie :
+    <Engine name="Standalone" defaultHost="localhost" jvmRoute="jvm1">         
+    --> 
+         
+    <!-- Define the top level container in our container hierarchy -->
+    <Engine name="Catalina" defaultHost="localhost">
+
+      <!-- The request dumper valve dumps useful debugging information about
+           the request headers and cookies that were received, and the response
+           headers and cookies that were sent, for all requests received by
+           this instance of Tomcat.  If you care only about requests to a
+           particular virtual host, or a particular application, nest this
+           element inside the corresponding <Host> or <Context> entry instead.
+
+           For a similar mechanism that is portable to all Servlet 2.4
+           containers, check out the "RequestDumperFilter" Filter in the
+           example application (the source for this filter may be found in
+           "$CATALINA_HOME/webapps/examples/WEB-INF/classes/filters").
+
+           Request dumping is disabled by default.  Uncomment the following
+           element to enable it. -->
+      <!--
+      <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
+      -->
+
+      <!-- Because this Realm is here, an instance will be shared globally -->
+
+      <!-- This Realm uses the UserDatabase configured in the global JNDI
+           resources under the key "UserDatabase".  Any edits
+           that are performed against this UserDatabase are immediately
+           available for use by the Realm.  -->
+      <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
+             resourceName="UserDatabase"/>
+
+      <!-- Comment out the old realm but leave here for now in case we
+           need to go back quickly -->
+      <!--
+      <Realm className="org.apache.catalina.realm.MemoryRealm" />
+      -->
+
+      <!-- Replace the above Realm with one of the following to get a Realm
+           stored in a database and accessed via JDBC -->
+
+      <!--
+      <Realm  className="org.apache.catalina.realm.JDBCRealm"
+             driverName="org.gjt.mm.mysql.Driver"
+          connectionURL="jdbc:mysql://localhost/authority"
+         connectionName="test" connectionPassword="test"
+              userTable="users" userNameCol="user_name" userCredCol="user_pass"
+          userRoleTable="user_roles" roleNameCol="role_name" />
+      -->
+
+      <!--
+      <Realm  className="org.apache.catalina.realm.JDBCRealm"
+             driverName="oracle.jdbc.driver.OracleDriver"
+          connectionURL="jdbc:oracle:thin:@ntserver:1521:ORCL"
+         connectionName="scott" connectionPassword="tiger"
+              userTable="users" userNameCol="user_name" userCredCol="user_pass"
+          userRoleTable="user_roles" roleNameCol="role_name" />
+      -->
+
+      <!--
+      <Realm  className="org.apache.catalina.realm.JDBCRealm"
+             driverName="sun.jdbc.odbc.JdbcOdbcDriver"
+          connectionURL="jdbc:odbc:CATALINA"
+              userTable="users" userNameCol="user_name" userCredCol="user_pass"
+          userRoleTable="user_roles" roleNameCol="role_name" />
+      -->
+
+      <!-- Define the default virtual host
+           Note: XML Schema validation will not work with Xerces 2.2.
+       -->
+      <Host name="localhost" appBase="webapps"
+       unpackWARs="true" autoDeploy="true"
+       xmlValidation="false" xmlNamespaceAware="false">
+
+	<!-- Greenstone3 Service -->
+	<Context path="/greenstone3" docBase="@gsdl3home@/web" debug="1" reloadable="true"><Resources allowLinking='true'/></Context>
+	
+        <!-- Defines a cluster for this node,
+             By defining this element, means that every manager will be changed.
+             So when running a cluster, only make sure that you have webapps in there
+             that need to be clustered and remove the other ones.
+             A cluster has the following parameters:
+
+             className = the fully qualified name of the cluster class
+
+             name = a descriptive name for your cluster, can be anything
+
+             mcastAddr = the multicast address, has to be the same for all the nodes
+
+             mcastPort = the multicast port, has to be the same for all the nodes
+             
+             mcastBindAddr = bind the multicast socket to a specific address
+             
+             mcastTTL = the multicast TTL if you want to limit your broadcast
+             
+             mcastSoTimeout = the multicast readtimeout 
+
+             mcastFrequency = the number of milliseconds in between sending a "I'm alive" heartbeat
+
+             mcastDropTime = the number a milliseconds before a node is considered "dead" if no heartbeat is received
+
+             tcpThreadCount = the number of threads to handle incoming replication requests, optimal would be the same amount of threads as nodes 
+
+             tcpListenAddress = the listen address (bind address) for TCP cluster request on this host, 
+                                in case of multiple ethernet cards.
+                                auto means that address becomes
+                                InetAddress.getLocalHost().getHostAddress()
+
+             tcpListenPort = the tcp listen port
+
+             tcpSelectorTimeout = the timeout (ms) for the Selector.select() method in case the OS
+                                  has a wakup bug in java.nio. Set to 0 for no timeout
+
+             printToScreen = true means that managers will also print to std.out
+
+             expireSessionsOnShutdown = true means that 
+
+             useDirtyFlag = true means that we only replicate a session after setAttribute,removeAttribute has been called.
+                            false means to replicate the session after each request.
+                            false means that replication would work for the following piece of code: (only for SimpleTcpReplicationManager)
+                            <%
+                            HashMap map = (HashMap)session.getAttribute("map");
+                            map.put("key","value");
+                            %>
+             replicationMode = can be either 'pooled', 'synchronous' or 'asynchronous'.
+                               * Pooled means that the replication happens using several sockets in a synchronous way. Ie, the data gets replicated, then the request return. This is the same as the 'synchronous' setting except it uses a pool of sockets, hence it is multithreaded. This is the fastest and safest configuration. To use this, also increase the nr of tcp threads that you have dealing with replication.
+                               * Synchronous means that the thread that executes the request, is also the
+                               thread the replicates the data to the other nodes, and will not return until all
+                               nodes have received the information.
+                               * Asynchronous means that there is a specific 'sender' thread for each cluster node,
+                               so the request thread will queue the replication request into a "smart" queue,
+                               and then return to the client.
+                               The "smart" queue is a queue where when a session is added to the queue, and the same session
+                               already exists in the queue from a previous request, that session will be replaced
+                               in the queue instead of replicating two requests. This almost never happens, unless there is a 
+                               large network delay.
+        -->             
+        <!--
+            When configuring for clustering, you also add in a valve to catch all the requests
+            coming in, at the end of the request, the session may or may not be replicated.
+            A session is replicated if and only if all the conditions are met:
+            1. useDirtyFlag is true or setAttribute or removeAttribute has been called AND
+            2. a session exists (has been created)
+            3. the request is not trapped by the "filter" attribute
+
+            The filter attribute is to filter out requests that could not modify the session,
+            hence we don't replicate the session after the end of this request.
+            The filter is negative, ie, anything you put in the filter, you mean to filter out,
+            ie, no replication will be done on requests that match one of the filters.
+            The filter attribute is delimited by ;, so you can't escape out ; even if you wanted to.
+
+            filter=".*\.gif;.*\.js;" means that we will not replicate the session after requests with the URI
+            ending with .gif and .js are intercepted.
+            
+            The deployer element can be used to deploy apps cluster wide.
+            Currently the deployment only deploys/undeploys to working members in the cluster
+            so no WARs are copied upons startup of a broken node.
+            The deployer watches a directory (watchDir) for WAR files when watchEnabled="true"
+            When a new war file is added the war gets deployed to the local instance,
+            and then deployed to the other instances in the cluster.
+            When a war file is deleted from the watchDir the war is undeployed locally 
+            and cluster wide
+        -->
+        
+        <!--
+        <Cluster className="org.apache.catalina.cluster.tcp.SimpleTcpCluster"
+                 managerClassName="org.apache.catalina.cluster.session.DeltaManager"
+                 expireSessionsOnShutdown="false"
+                 useDirtyFlag="true"
+                 notifyListenersOnReplication="true">
+
+            <Membership 
+                className="org.apache.catalina.cluster.mcast.McastService"
+                mcastAddr="228.0.0.4"
+                mcastPort="45564"
+                mcastFrequency="500"
+                mcastDropTime="3000"/>
+
+            <Receiver 
+                className="org.apache.catalina.cluster.tcp.ReplicationListener"
+                tcpListenAddress="auto"
+                tcpListenPort="4001"
+                tcpSelectorTimeout="100"
+                tcpThreadCount="6"/>
+
+            <Sender
+                className="org.apache.catalina.cluster.tcp.ReplicationTransmitter"
+                replicationMode="pooled"
+                ackTimeout="15000"/>
+
+            <Valve className="org.apache.catalina.cluster.tcp.ReplicationValve"
+                   filter=".*\.gif;.*\.js;.*\.jpg;.*\.png;.*\.htm;.*\.html;.*\.css;.*\.txt;"/>
+                   
+            <Deployer className="org.apache.catalina.cluster.deploy.FarmWarDeployer"
+                      tempDir="/tmp/war-temp/"
+                      deployDir="/tmp/war-deploy/"
+                      watchDir="/tmp/war-listen/"
+                      watchEnabled="false"/>
+                      
+            <ClusterListener className="org.apache.catalina.cluster.session.ClusterSessionListener"/>
+        </Cluster>
+        -->        
+
+
+
+        <!-- Normally, users must authenticate themselves to each web app
+             individually.  Uncomment the following entry if you would like
+             a user to be authenticated the first time they encounter a
+             resource protected by a security constraint, and then have that
+             user identity maintained across *all* web applications contained
+             in this virtual host. -->
+        <!--
+        <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
+        -->
+
+        <!-- Access log processes all requests for this virtual host.  By
+             default, log files are created in the "logs" directory relative to
+             $CATALINA_HOME.  If you wish, you can specify a different
+             directory with the "directory" attribute.  Specify either a relative
+             (to $CATALINA_HOME) or absolute path to the desired directory.
+        -->
+        <!--
+        <Valve className="org.apache.catalina.valves.AccessLogValve"
+                 directory="logs"  prefix="localhost_access_log." suffix=".txt"
+                 pattern="common" resolveHosts="false"/>
+        -->
+
+        <!-- Access log processes all requests for this virtual host.  By
+             default, log files are created in the "logs" directory relative to
+             $CATALINA_HOME.  If you wish, you can specify a different
+             directory with the "directory" attribute.  Specify either a relative
+             (to $CATALINA_HOME) or absolute path to the desired directory.
+             This access log implementation is optimized for maximum performance,
+             but is hardcoded to support only the "common" and "combined" patterns.
+        -->
+        <!--
+        <Valve className="org.apache.catalina.valves.FastCommonAccessLogValve"
+                 directory="logs"  prefix="localhost_access_log." suffix=".txt"
+                 pattern="common" resolveHosts="false"/>
+        -->
+        <!-- Access log processes all requests for this virtual host.  By
+             default, log files are created in the "logs" directory relative to
+             $CATALINA_HOME.  If you wish, you can specify a different
+             directory with the "directory" attribute.  Specify either a relative
+             (to $CATALINA_HOME) or absolute path to the desired directory.
+             This access log implementation is optimized for maximum performance,
+             but is hardcoded to support only the "common" and "combined" patterns.
+
+             This valve use NIO direct Byte Buffer to asynchornously store the
+             log.
+        -->
+        <!--
+        <Valve className="org.apache.catalina.valves.ByteBufferAccessLogValve"
+                 directory="logs"  prefix="localhost_access_log." suffix=".txt"
+                 pattern="common" resolveHosts="false"/>
+        -->
+
+      </Host>
+
+    </Engine>
+
+  </Service>
+
+</Server>
Index: /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.bat
===================================================================
--- /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.bat	(revision 21112)
@@ -0,0 +1,57 @@
+rem ---------------------------------------------------------------------------
+rem Set CLASSPATH and Java options
+rem
+rem $Id$
+rem ---------------------------------------------------------------------------
+
+rem Make sure prerequisite environment variables are set
+if not "%JAVA_HOME%" == "" goto gotJavaHome
+echo The JAVA_HOME environment variable is not defined
+echo This environment variable is needed to run this program
+goto exit
+:gotJavaHome
+if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\javaw.exe" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\jdb.exe" goto noJavaHome
+if not exist "%JAVA_HOME%\bin\javac.exe" goto noJavaHome
+goto okJavaHome
+:noJavaHome
+echo The JAVA_HOME environment variable is not defined correctly
+echo This environment variable is needed to run this program
+echo NB: JAVA_HOME should point to a JDK not a JRE
+goto exit
+:okJavaHome
+
+if not "%BASEDIR%" == "" goto gotBasedir
+echo The BASEDIR environment variable is not defined
+echo This environment variable is needed to run this program
+goto exit
+:gotBasedir
+if exist "%BASEDIR%\bin\setclasspath.bat" goto okBasedir
+echo The BASEDIR environment variable is not defined correctly
+echo This environment variable is needed to run this program
+goto exit
+:okBasedir
+
+rem Set the default -Djava.endorsed.dirs argument
+set JAVA_ENDORSED_DIRS=%BASEDIR%\common\endorsed
+
+rem Set standard CLASSPATH
+rem Note that there are no quotes as we do not want to introduce random
+rem quotes into the CLASSPATH
+set CLASSPATH=%JAVA_HOME%\lib\tools.jar;%CLASSPATH%
+
+rem Set standard command for invoking Java.
+rem Note that NT requires a window name argument when using start.
+rem Also note the quoting as JAVA_HOME may contain spaces.
+set _RUNJAVA="%JAVA_HOME%\bin\java"
+set _RUNJAVAW="%JAVA_HOME%\bin\javaw"
+set _RUNJDB="%JAVA_HOME%\bin\jdb"
+set _RUNJAVAC="%JAVA_HOME%\bin\javac"
+
+goto end
+
+:exit
+exit /b 1
+
+:end
Index: /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.sh
===================================================================
--- /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/packages/tomcat-extra/setclasspath.sh	(revision 21112)
@@ -0,0 +1,87 @@
+#!/bin/sh
+# -----------------------------------------------------------------------------
+#  Set CLASSPATH and Java options
+#
+#  $Id$
+# -----------------------------------------------------------------------------
+
+# First clear out the user classpath
+# gs3 - we need to set classpath
+CLASSPATH=$CLASSPATH
+
+# Make sure prerequisite environment variables are set
+if [ -z "$JAVA_HOME" -a -z "$JRE_HOME" ]; then
+  echo "Neither the JAVA_HOME nor the JRE_HOME environment variable is defined"
+  echo "At least one of these environment variable is needed to run this program"
+  exit 1
+fi
+if [ -z "$JAVA_HOME" -a "$1" = "debug" ]; then
+  echo "JAVA_HOME should point to a JDK in order to run in debug mode."
+  exit 1
+fi
+if [ -z "$JRE_HOME" ]; then
+  JRE_HOME="$JAVA_HOME"
+fi
+
+# If we're running under jdb, we need a full jdk.
+if [ "$1" = "debug" -o "$1" = "javac" ] ; then
+  if [ "$os400" = "true" ]; then
+    if [ ! -x "$JAVA_HOME"/bin/java -o ! -x "$JAVA_HOME"/bin/javac ]; then
+      echo "The JAVA_HOME environment variable is not defined correctly"
+      echo "This environment variable is needed to run this program"
+      echo "NB: JAVA_HOME should point to a JDK not a JRE"
+      exit 1
+    fi
+  else
+    if [ ! -x "$JAVA_HOME"/bin/java -o ! -x "$JAVA_HOME"/bin/jdb -o ! -x "$JAVA_HOME"/bin/javac ]; then
+      echo "The JAVA_HOME environment variable is not defined correctly"
+      echo "This environment variable is needed to run this program"
+      echo "NB: JAVA_HOME should point to a JDK not a JRE"
+      exit 1
+    fi
+  fi
+fi
+if [ -z "$BASEDIR" ]; then
+  echo "The BASEDIR environment variable is not defined"
+  echo "This environment variable is needed to run this program"
+  exit 1
+fi
+if [ ! -x "$BASEDIR"/bin/setclasspath.sh ]; then
+  if $os400; then
+    # -x will Only work on the os400 if the files are:
+    # 1. owned by the user
+    # 2. owned by the PRIMARY group of the user
+    # this will not work if the user belongs in secondary groups
+    eval
+  else
+    echo "The BASEDIR environment variable is not defined correctly"
+    echo "This environment variable is needed to run this program"
+    exit 1
+  fi
+fi
+
+# Set the default -Djava.endorsed.dirs argument
+JAVA_ENDORSED_DIRS="$BASEDIR"/common/endorsed
+
+# Set standard CLASSPATH
+if [ "$1" = "debug" -o "$1" = "javac" ] ; then
+  CLASSPATH="$JAVA_HOME"/lib/tools.jar:$CLASSPATH
+fi
+
+# OSX hack to CLASSPATH
+JIKESPATH=
+if [ `uname -s` = "Darwin" ]; then
+  OSXHACK="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Classes"
+  if [ -d "$OSXHACK" ]; then
+    for i in "$OSXHACK"/*.jar; do
+      JIKESPATH="$JIKESPATH":"$i"
+    done
+  fi
+fi
+
+# Set standard commands for invoking Java.
+  _RUNJAVA="$JRE_HOME"/bin/java
+if [ "$os400" != "true" ]; then
+  _RUNJDB="$JAVA_HOME"/bin/jdb
+fi
+_RUNJAVAC="$JAVA_HOME"/bin/javac
Index: /main/tags/3.01/gsdl3/resources/dtd/bookfrag.dtd
===================================================================
--- /main/tags/3.01/gsdl3/resources/dtd/bookfrag.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/dtd/bookfrag.dtd	(revision 21112)
@@ -0,0 +1,529 @@
+<!--This is bookfrag DTD-->
+
+<!--entity declarations-->
+<!--#############################################-->
+
+<!ENTITY % stdatts 
+"ref   IDREF  #IMPLIED
+ id    ID     #IMPLIED
+ type  CDATA  #IMPLIED
+ role  CDATA  #IMPLIED
+ class CDATA  #IMPLIED"
+>
+
+<!ENTITY % inline.class "|quote|emph|ital|reference|date|place|name|graphic|txterr|mkuperr|misc ">
+
+<!ENTITY % block.class "|letter|blockquote|footnote|note|list|deflist|table|blockgraphic">
+
+
+<!-- the top level element-->
+<!--##############################################-->
+
+<!ELEMENT book (acknowledge?,meta*,frontmatter?,bookbody,backmatter?,endmeta*)>
+
+<!--begin top level element declarations-->
+<!--##############################################-->
+
+
+<!--the acknowledge element is for a simple statement 
+designed to be placed at the beginning of the visible document.
+Suggested content:
+  Name of Book
+  Author
+  PG and HWG blurb
+  Original Transcriber + date
+  Marker 
+  e.g.
+  "Origin of the Species by Charles Darwin
+  A project of Project Gutenberg and the HTML Writers Guild.
+  Transcribed by  
+  Markup by Frank Boumphrey. Jan 22 2000"
+  -->
+
+<!ELEMENT acknowledge (#PCDATA %inline.class;)*>
+<!ATTLIST acknowledge
+  %stdatts;
+>
+
+<!--the meta tags are here to provide a means for comment 
+at either the very beginning or end of the document-->
+<!ELEMENT meta EMPTY>
+<!ATTLIST meta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+<!ELEMENT endmeta EMPTY >
+<!ATTLIST endmeta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+
+
+<!--the bookbody element contains the body of the book it can 
+consist of either part or chapter divisions-->
+
+<!ELEMENT bookbody (part*|chapter*)>
+
+<!--
+the backmatter element contains the backmatter of the book -->
+
+<!ELEMENT backmatter ((appendix|index|glossary|biblio|note)*,colophon?)>
+
+<!--The endmeta tag is groupedwith its twin meta-->
+
+
+<!--end top level element declarations-->
+
+<!--begin frontmatter element declarations-->
+<!--##############################################-->
+
+<!ELEMENT frontmatter (htitlepage|copypage|epigraph|titlepage|
+                toc|acksect|dedication|preface|introduction|miscfm)*>
+
+<!ELEMENT htitlepage (#PCDATA|title|subtitle|author|para|poem|song %inline.class;)*>
+<!ATTLIST htitlepage
+  %stdatts;
+>
+
+<!ELEMENT copypage (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST copypage
+  %stdatts;
+>
+
+<!ELEMENT epigraph (#PCDATA|para|poem|song|note|blockquote %inline.class;)*>
+<!ATTLIST epigraph
+  %stdatts;
+>
+
+
+<!ELEMENT titlepage (#PCDATA|partnum|title|subtitle|author|pubinfo|para|poem|song|note|line %inline.class;)*>
+<!ATTLIST titlepage
+  %stdatts;
+>
+
+<!ELEMENT pubinfo (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST pubinfo
+  %stdatts;
+>
+
+
+<!ELEMENT toc (#PCDATA|title|subtitle|subsubtitle|item|list|deflist %inline.class;)*>
+<!--use type or role to describe toctype="other"-->
+<!ATTLIST toc
+  toctype (contents|maps|graphics|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT acksect (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST acksect
+  %stdatts;
+>
+
+<!ELEMENT dedication (#PCDATA|title|para|poem|song|note %inline.class;)*>
+<!ATTLIST dedication
+  %stdatts;
+>
+
+
+<!ELEMENT preface ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?,preauthor?)>
+<!ATTLIST preface  %stdatts;
+>
+
+<!ELEMENT introduction ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST introduction  %stdatts;
+>
+
+<!ELEMENT miscfm (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST miscfm
+  %stdatts;
+>
+
+<!ELEMENT preauthor (#PCDATA|author %inline.class;)*>
+<!ATTLIST preauthor
+  %stdatts;
+>
+
+
+
+<!--end frontmatter element declarations-->
+
+<!--begin bookbody element declarations-->
+<!--##############################################-->
+
+
+<!ELEMENT part (acknowledge?,(titlepage|toc|htitlepage)*,chapter*)>
+<!ATTLIST part
+  %stdatts;
+>
+
+<!ELEMENT chapter ((title|chapheader)?,(para|poem|song|sect1|simplesect|page %block.class;)*,endchap?)>
+<!ATTLIST chapter  %stdatts;
+>
+
+<!--end bookbody element declarations-->
+
+<!--the lowerlevel bookbody elements-->
+<!--##############################################-->
+
+<!--the chapter heading elements-->
+<!--##############################################-->
+
+
+
+
+<!ELEMENT title (#PCDATA %inline.class;)*>
+<!ATTLIST title
+  %stdatts;
+>
+
+<!ELEMENT subtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subtitle
+  %stdatts;
+>
+
+<!ELEMENT subsubtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subsubtitle
+  %stdatts;
+>
+
+
+
+<!--validity constraint. Only one title,chapnum and chapsummary -->
+<!ELEMENT chapheader (title|subtitle|chapnum|chapsummary|blockquote|para|note)*>
+<!ATTLIST chapheader
+  %stdatts;
+>
+
+
+<!ELEMENT chapnum (#PCDATA %inline.class;)*>
+<!ATTLIST chapnum
+  %stdatts;
+>
+
+<!ELEMENT chapsummary (#PCDATA %inline.class;)*>
+<!ATTLIST chapsummary
+  %stdatts;
+>
+
+
+<!--begin backmatter element declarations-->
+<!--##############################################-->
+
+<!--end backmatter element declarations-->
+
+<!ELEMENT index (title|item|list|deflist|note)*>
+<!ATTLIST index
+  indtype (contents|authors|firstlines|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT glossary (title|item|list|deflist|note)*>
+<!ATTLIST glossary  
+  %stdatts;
+>
+
+<!ELEMENT biblio (title|item|list|deflist|note)*>
+<!ATTLIST biblio  
+  %stdatts;
+>
+
+
+<!ELEMENT appendix ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST appendix  %stdatts;
+>
+
+<!ELEMENT colophon (#PCDATA|para|poem|song %inline.class;)*>
+<!ATTLIST colophon
+  %stdatts;
+>
+
+<!--the block elements-->
+<!--##############################################-->
+
+
+<!--validity constraint. Only one title per paragraph-->
+<!ELEMENT para (#PCDATA|title %inline.class;)*>
+<!ATTLIST para
+  %stdatts;
+>
+
+<!ELEMENT simplesect (title,(subtitle|para|poem|song %block.class;)*)>
+<!ATTLIST simplesect
+  %stdatts;
+>
+
+<!ELEMENT sect1 (title,(sect2|simplesect|para|poem|song %block.class;)*)>
+<!ATTLIST sect1
+  %stdatts;
+>
+
+<!ELEMENT sect2 (title,(sect3|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect2
+  %stdatts;
+>
+
+<!ELEMENT sect3 (title,(sect4|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect3
+  %stdatts;
+>
+
+<!ELEMENT sect4 (title,(simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect4
+  %stdatts;
+>
+
+<!ELEMENT page (#PCDATA %inline.class;)*>
+<!ATTLIST page
+  %stdatts;
+>
+
+<!ELEMENT misc (#PCDATA %inline.class;)*>
+<!ATTLIST misc
+  %stdatts;
+>
+
+
+
+<!ELEMENT blockquote (title?,(para|poem|song)*,attrib?)>
+<!ATTLIST blockquote
+  %stdatts;
+>
+
+<!ELEMENT attrib (#PCDATA %inline.class;)*>
+<!ATTLIST attrib
+  %stdatts;
+>
+
+<!ELEMENT caption (#PCDATA %inline.class;)*>
+<!ATTLIST caption
+  %stdatts;
+>
+<!--##############################################-->
+<!--poetry and songs in the text-->
+<!ELEMENT poem (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST poem
+  %stdatts;
+>
+
+<!ELEMENT song (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST song
+  %stdatts;
+>
+
+
+<!ELEMENT verse (title|subtitle|line|note)*>
+<!ATTLIST verse
+  %stdatts;
+>
+
+<!ELEMENT line (#PCDATA %inline.class;)*>
+<!ATTLIST line
+  %stdatts;
+>
+
+
+<!--validity constraint: Only one title per poem or per verse-->
+
+<!--end of poetry in the text-->
+
+<!--letter module-->
+
+<!ELEMENT letter (address|to|from|salut|sig|title|subtitle|para|poem|song|line|note)*>
+<!ATTLIST letter
+  %stdatts;
+>
+
+<!ELEMENT address (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST address
+  %stdatts;
+>
+
+<!ELEMENT to (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST to
+  %stdatts;
+>
+
+
+<!ELEMENT from (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST from
+  %stdatts;
+>
+
+
+<!ELEMENT salut (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST salut
+  %stdatts;
+>
+
+
+<!ELEMENT sig (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST sig
+  %stdatts;
+>
+
+
+<!--end letters module-->
+<!--endchap-->
+<!--##############################################-->
+<!--
+  Endchap should only be used when there is a 
+  considerable ammount of material to be organized.
+  For the occasional note footnote and note will suffice
+-->
+
+<!ELEMENT endchap (para %block.class;)*>
+<!ATTLIST endchap  %stdatts;
+>
+
+<!--lists-->
+<!--##############################################-->
+
+
+<!ELEMENT list (title?,(list|item)*)>
+<!ATTLIST list
+  %stdatts;
+>
+
+<!ELEMENT item (#PCDATA|para|poem|song|simplesect %block.class; %inline.class;)*>
+<!ATTLIST item
+  %stdatts;
+>
+
+<!ELEMENT deflist (title?,(item,desc?,def*)*)>
+<!ATTLIST deflist
+  %stdatts;
+>
+
+<!ELEMENT desc (#PCDATA %inline.class;)*>
+<!ATTLIST desc
+  %stdatts;
+>
+
+<!ELEMENT def (#PCDATA %inline.class;)*>
+<!ATTLIST def
+  %stdatts;
+>
+<!--end lists-->
+<!--table element-->
+<!--##############################################-->
+
+
+<!ELEMENT table (title?,row*,caption?)>
+<!ATTLIST table
+  %stdatts;
+>
+
+<!ELEMENT row (cell)*>
+<!ATTLIST row
+  %stdatts;
+>
+
+<!ELEMENT cell (#PCDATA %block.class; %inline.class;)*>
+<!ATTLIST cell
+  %stdatts;
+>
+
+<!--end table-->
+
+<!--graphics-->
+<!--##############################################-->
+
+<!--
+  Note that the required desc attribute is added 
+  text composed by the marker.
+-->
+
+<!ELEMENT graphic EMPTY >
+<!ATTLIST graphic
+  desc CDATA #REQUIRED
+  href CDATA #REQUIRED
+  %stdatts;
+>
+
+<!ELEMENT blockgraphic (title?,graphic,caption?)>
+<!ATTLIST blockgraphic
+  %stdatts;
+>
+
+
+<!--the inline elements-->
+<!--##############################################-->
+
+
+
+<!ELEMENT footnote (#PCDATA %inline.class;)*>
+<!ATTLIST footnote
+  %stdatts;
+>
+
+<!ELEMENT note (#PCDATA %inline.class;)*>
+<!ATTLIST note
+  %stdatts;
+>
+
+
+<!ELEMENT quote (#PCDATA %inline.class;)*>
+<!ATTLIST quote
+  %stdatts;
+>
+
+<!ELEMENT emph (#PCDATA %inline.class;)*>
+<!ATTLIST emph
+  %stdatts;
+>
+
+<!ELEMENT ital (#PCDATA %inline.class;)*>
+<!ATTLIST ital
+  %stdatts;
+>
+
+<!ELEMENT reference (#PCDATA %inline.class;)*>
+<!ATTLIST reference
+  %stdatts;
+>
+
+<!ELEMENT author (#PCDATA %inline.class;)*>
+<!ATTLIST author
+  %stdatts;
+>
+
+<!ELEMENT date (#PCDATA %inline.class;)*>
+<!ATTLIST date
+  %stdatts;
+>
+
+<!ELEMENT place (#PCDATA %inline.class;)*>
+<!ATTLIST place
+  %stdatts;
+>
+
+<!ELEMENT name (#PCDATA %inline.class;)*>
+<!ATTLIST name
+  %stdatts;
+>
+
+
+<!--use to enclose the text you think is in error
+optional explanation if error is not obvious-->
+<!ELEMENT txterr (#PCDATA)*>
+<!ATTLIST txterr
+  explain  CDATA  #IMPLIED
+  %stdatts;
+>
+
+<!--an explanation is required-->
+<!ELEMENT mkuperr EMPTY>
+<!ATTLIST mkuperr
+  explain  CDATA  #REQUIRED
+  %stdatts;
+>
+
+<!--added as titlepage content  Feb/6/00-->
+<!ELEMENT partnum (#PCDATA %inline.class;)*>
+<!ATTLIST partnum
+  %stdatts;
+>
+
+<!--End of bookfrag dtd-->
Index: /main/tags/3.01/gsdl3/resources/dtd/entities.dtd
===================================================================
--- /main/tags/3.01/gsdl3/resources/dtd/entities.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/dtd/entities.dtd	(revision 21112)
@@ -0,0 +1,50 @@
+<!ENTITY nbsp "&#160;">
+<!ENTITY copy "&#169;">
+<!ENTITY mdash "&#151;">
+<!ENTITY ouml "&#214;">
+<!ENTITY dollar "&#36;" >
+<!ENTITY Eacute "&#201;" >
+<!ENTITY eacute "&#233;" >
+<!ENTITY uacute "&#250;">
+<!ENTITY aacute "&#225;">
+<!ENTITY oacute "&#243;">
+<!ENTITY Eacute "&#201;">
+<!ENTITY iacute "&#237;">
+<!ENTITY agrave "&#224;">
+<!ENTITY egrave "&#232;">
+<!ENTITY igrave "&#236;">
+<!ENTITY ograve "&#242;">
+<!ENTITY ugrave "&#249;">
+<!ENTITY aelig "&#230;">
+<!ENTITY acirc "&#226;">
+<!ENTITY ecirc "&#234;">
+<!ENTITY icirc "&#238;">
+<!ENTITY ocirc "&#244;">
+<!ENTITY ucirc "&#251;">
+<!ENTITY ntilde "&#241;">
+<!ENTITY ccedil "&#231;">
+<!ENTITY shy "&#173;">
+<!ENTITY pound "&#163;">
+<!ENTITY percnt "&#37;">
+<!ENTITY percent "&#37;">
+<!ENTITY iuml "&#239;">
+<!ENTITY Iuml "&#207;">
+<!ENTITY amacr "&#257;">
+<!ENTITY Amacr "&#256;">
+<!ENTITY emacr "&#275;">
+<!ENTITY Emacr "&#274;">
+<!ENTITY omacr "&#333;">
+<!ENTITY imacr "&#299;">
+<!ENTITY Imacr "&#298;">
+<!ENTITY Omacr "&#332;">
+<!ENTITY umacr "&#363;">
+<!ENTITY Umacr "&#362;">
+<!ENTITY oslash "&#248;">
+<!ENTITY euml "&#235;">
+<!ENTITY Euml "&#203;">
+<!ENTITY uuml "&#252;">
+<!ENTITY frac14 "&#188;">
+<!ENTITY frac34 "&#190;">
+<!ENTITY half "&#189;">
+<!ENTITY frac12 "&#189;">
+
Index: /main/tags/3.01/gsdl3/resources/dtd/gutbook1.dtd
===================================================================
--- /main/tags/3.01/gsdl3/resources/dtd/gutbook1.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/dtd/gutbook1.dtd	(revision 21112)
@@ -0,0 +1,40 @@
+<!--
+GUTBOOK DTD V 1.0
+==================
+
+GENERAL NOTES
++++++++++++++
+This is gutbook dtd version 1.0
+file name is gutpoems1.dtd
+This dtd is a first pass at a dtd for marking up 
+books for project gutenberg.
+It was originally developed for marking up the Darwins books.
+
+REVISION HISTORY
+++++++++++++++++
+Original Author
+Frank Boumphrey  1/19/2000
+
+COPYRIGHT NOTICE
+++++++++++++++++
+This DTD is the copyright of the HTML writers guild.
+Permission is given in perpetuity for any one to use 
+this DTD provided that these notes and this  copyright 
+notice is included.
+
+
+-->
+
+<!--Begin gutbook DTD-->
+<!ELEMENT gutbook (gutblurb,markupblurb,book*,endmarkupblurb?,endgutblurb?)>
+
+<!ELEMENT gutblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT markupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endmarkupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endgutblurb (#PCDATA|para|simplesect|title)*>
+
+<!--Include bookfrag.dtd-->
+
+<!ENTITY % bookfrag SYSTEM "bookfrag.dtd">
+%bookfrag;
+<!--end of file-->
Index: /main/tags/3.01/gsdl3/resources/java/global.properties.in
===================================================================
--- /main/tags/3.01/gsdl3/resources/java/global.properties.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/java/global.properties.in	(revision 21112)
@@ -0,0 +1,24 @@
+## greenstone home
+gsdl3.home=@gsdl3home@
+
+# tomcat info
+tomcat.server=@tomcat.server@
+tomcat.port=@tomcat.port@
+tomcat.context=greenstone3
+
+## Proxy setup - set these if you are behind a firewall and you want services that access the internet
+# The proxy server name 
+proxy.host=
+# The proxy server port
+proxy.port=
+# A user name to use when talking to the proxy server 
+proxy.user=
+# A password to use when talking to the proxy server 
+proxy.password=
+
+## mysql server
+mysql.tcp.port=@mysql.port@
+mysql.server=@mysql.server@
+mysql.gsdl3reader.password=@mysql.admin.pword@
+mysql.gsdl3admin.password=@mysql.reader.pword@
+
Index: /main/tags/3.01/gsdl3/resources/soap/localsite.wsdd
===================================================================
--- /main/tags/3.01/gsdl3/resources/soap/localsite.wsdd	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/soap/localsite.wsdd	(revision 21112)
@@ -0,0 +1,10 @@
+<deployment xmlns="http://xml.apache.org/axis/wsdd/"
+    xmlns:java="http://xml.apache.org/axis/wsdd/providers/java"
+    xmlns:xsi="http://www.w3.org/2000/10/XMLSchema-instance">
+  <service name="localsite" style="message">
+    <parameter name="className" value="org.greenstone.gsdl3.SOAPServerlocalsite"/>
+    <parameter name="allowedMethods" value="process"/>
+    <parameter name="scope" value="application"/>
+  </service>
+</deployment>
+
Index: /main/tags/3.01/gsdl3/resources/soap/site.wsdd.template
===================================================================
--- /main/tags/3.01/gsdl3/resources/soap/site.wsdd.template	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/soap/site.wsdd.template	(revision 21112)
@@ -0,0 +1,10 @@
+<deployment xmlns="http://xml.apache.org/axis/wsdd/"
+    xmlns:java="http://xml.apache.org/axis/wsdd/providers/java"
+    xmlns:xsi="http://www.w3.org/2000/10/XMLSchema-instance">
+  <service name="@siteuri@" style="message">
+    <parameter name="className" value="org.greenstone.gsdl3.SOAPServer@sitename@"/>
+    <parameter name="allowedMethods" value="process"/>
+    <parameter name="scope" value="application"/>
+  </service>
+</deployment>
+
Index: /main/tags/3.01/gsdl3/resources/soap/undeploy-localsite.wsdd
===================================================================
--- /main/tags/3.01/gsdl3/resources/soap/undeploy-localsite.wsdd	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/soap/undeploy-localsite.wsdd	(revision 21112)
@@ -0,0 +1,3 @@
+<undeployment xmlns="http://xml.apache.org/axis/wsdd/">
+  <service name="localsite"/>
+</undeployment>
Index: /main/tags/3.01/gsdl3/resources/soap/undeploy-site.wsdd.template
===================================================================
--- /main/tags/3.01/gsdl3/resources/soap/undeploy-site.wsdd.template	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/soap/undeploy-site.wsdd.template	(revision 21112)
@@ -0,0 +1,3 @@
+<undeployment xmlns="http://xml.apache.org/axis/wsdd/">
+  <service name="@siteuri@"/>
+</undeployment>
Index: /main/tags/3.01/gsdl3/resources/xml/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/resources/xml/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/resources/xml/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,67 @@
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">>
+  <metadataList>
+    <!-- email address of collection creator -->
+    <metadata name="creator"></metadata>
+  </metadataList>
+  <displayItemList>
+    <!-- Display name of collection -->
+    <displayItem name='name' lang='en'></displayItem>
+    <!-- Display description of collection -->
+    <displayItem name='description' lang='en'></displayItem>
+    <!-- Icon name for collection home page -->
+    <displayItem name='icon' lang='en'></displayItem>
+    <!-- Icon name for library home page -->
+    <displayItem name='smallicon' lang='en'></displayItem>
+  </displayItemList>
+  <recognise>
+    <docType name="HTML"/>
+    <docType name="Text"/>
+    <docType name="Metadata"/>
+    <docType name="JPEG"/>
+  </recognise>
+  <search type="mg" name="mgsearch">
+    <index name="doctext">
+      <field>text</field>
+      <level>document</level>
+      <displayItem name='name' lang="en">entire documents</displayItem>
+      <displayItem name='name' lang="fr">documents entiers</displayItem>
+      <displayItem name='name' lang="es">documentos enteros</displayItem>
+    </index>
+    <!-- section level text index -->
+    <!-- <index name="sectext">
+    <field>text</field>
+    <level>section</level>
+      <displayItem name='name' lang="en">chapters</displayItem>
+      <displayItem name='name' lang="fr">chapitres</displayItem>
+      <displayItem name='name' lang="es">capÃ­tulos</displayItem>
+    </index>-->
+    <!-- section level title index -->
+    <!--<index name="sectitle">
+      <field>Title</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">section titles</displayItem>
+      <displayItem name='name' lang="fr">titres des sections</displayItem>
+      <displayItem name='name' lang="es">tÃ­tulos de las secciones</displayItem>
+    </index>-->
+    <!-- default format -->
+    <format>
+      <gsf:template match="documentNode"><td valign='top'><gsf:link><gsf:icon/></gsf:link></td><td><gsf:metadata name='Title'/></td></gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CLTit" type="AZList" horizontalAtTop='true'>
+      <field>Title</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Titles</displayItem>
+    </classifier>
+  </browse>
+  <display>
+    <format>
+      <!-- set to true if have cover images -->
+      <gsf:option name="coverImages" value="false"/>
+      <!-- set to true if have sectioned documents -->
+      <gsf:option name="documentTOC" value="false"/>
+    </format>
+  </display>
+</collectionConfig>
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MyGetUserAndPassword.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MyGetUserAndPassword.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MyGetUserAndPassword.java	(revision 21112)
@@ -0,0 +1,235 @@
+/*
+ *    MyGetUserAndPassword.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.anttasks;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+/** A new ant task to prompt the user for username and password.
+ * To use this task, the compiled class must be put in the classpath. add a taskdef line to the build.xml:
+ * <taskdef name="getuserandpassword" classname="org.greenstone.anttasks.MyGetUserAndPassword" />
+ * and call it like the following:
+ *  <getuserandpassword 
+ *       message="a message to display at the top of the dialog" 
+ *       userproperty="the name of the username property to set (optional)" 
+ *       pwordproperty="the name of the password property to set" 
+ *       username="default value for username (optional)" 
+ *       if="property name (optional)" 
+ *       unless="property name (optional)"/>
+ * the if and unless attributes are used to control whether this task runs or not. If the 'if' attribute is set, the task will only run if the specified property is defined. If the unless attribute is set, the task will only run if the property is not defined.
+*/
+
+public class MyGetUserAndPassword
+    extends org.apache.tools.ant.Task {
+    
+    private String message = null;
+    private String user_property = null;
+    private String pword_property = null;
+    private String username = null;
+    private String password = null;
+    private boolean cancelled = false;
+
+    private Dimension SIZE = new Dimension(300, 150);
+    private JButton ok_button = null;
+    private JTextField name_field = null;
+    private JPasswordField pass_field = null;
+    
+    private boolean do_the_task = true;
+    public void setMessage(String m) {
+	message = m;
+    }
+
+    /** specify a property name - the task will only execute if this property is set */
+    public void setIf(String property) {
+	String value = getProject().getProperty(property);
+	if (value== null) {
+	    do_the_task = false;
+	}
+    }
+
+    public void setPwordproperty(String p_prop) {
+	pword_property = p_prop;
+    }
+    
+    /** specify a property name - the task will only execute if this property is not set */
+    public void setUnless(String property) {
+	String value = getProject().getProperty(property);
+	if (value != null) {
+	    do_the_task = false;
+	}
+    }
+    
+    public void setUsername(String user) {
+	username = user;
+    }
+
+    public void setUserproperty(String u_prop) {
+	user_property = u_prop;
+    }
+    public void execute() throws org.apache.tools.ant.BuildException {
+	
+	if (!do_the_task) {
+	    return;
+	}
+	if (pword_property == null) {
+	    throw new org.apache.tools.ant.BuildException("pwordproperty attribute must be specified");
+	}
+	
+	JFrame hiddenFrame = new JFrame("Username and Password Hidden Frame");
+	final JDialog dialog = new JDialog(hiddenFrame, "Username and Password", true);
+	JPanel content_pane = (JPanel)dialog.getContentPane();
+	content_pane.setLayout(new BorderLayout(5,5));
+	content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	dialog.setLocation(200, 300);
+	dialog.setSize(SIZE);
+	dialog.addWindowListener(new WindowAdapter() {
+		public void windowClosing(WindowEvent windowEvent) {
+		    cancelled = true;
+		    dialog.dispose();
+		} 
+	    });
+	
+	JPanel button_pane = new JPanel();
+	button_pane.setLayout(new GridLayout(1,2));
+	JPanel details_pane = new JPanel();
+	details_pane.setLayout(new BorderLayout(5,5));
+	JPanel labels_pane = new JPanel();
+	labels_pane.setLayout(new GridLayout(2,1));
+	JPanel fields_pane = new JPanel();
+	fields_pane.setLayout(new GridLayout(2,1));
+	
+        JTextArea instruction_textarea = new JTextArea();
+	instruction_textarea.setEditable(false);
+	instruction_textarea.setLineWrap(true);
+	instruction_textarea.setRows(2);
+	instruction_textarea.setWrapStyleWord(true);
+	if (message == null) {
+	    message = "Please enter your username and password";
+	}
+	instruction_textarea.setText(message);
+		
+	JLabel name_label = new JLabel("User name:");
+	name_field = new JTextField(16);
+	if (username != null) {
+	    name_field.setText(username);
+	}
+	if (user_property == null ) {
+	    name_field.setEditable(false);
+	} 
+	JLabel pass_label = new JLabel("Password:");
+	pass_field = new JPasswordField(16);
+	
+	FieldListener field_listener = new FieldListener();
+	name_field.addActionListener(field_listener);
+	name_field.getDocument().addDocumentListener(field_listener);
+	pass_field.addActionListener(field_listener);
+	pass_field.getDocument().addDocumentListener(field_listener);
+
+	ok_button = new JButton("Ok");
+	ok_button.setEnabled(false);
+	ok_button.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent actionEvent) {
+		    username = name_field.getText();
+		    password = new String(pass_field.getPassword());
+		    dialog.dispose();
+		}
+	    });
+		
+	JButton cancel_button = new JButton("Cancel");
+	cancel_button.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent actionEvent) {
+		    cancelled = true;
+		    dialog.dispose();
+		}
+	    });
+	
+	button_pane.add(ok_button);
+	button_pane.add(cancel_button);
+		
+	labels_pane.add(name_label);
+	labels_pane.add(pass_label);
+
+	fields_pane.add(name_field);
+	fields_pane.add(pass_field);
+
+	details_pane.add(labels_pane, BorderLayout.WEST);
+	details_pane.add(fields_pane, BorderLayout.CENTER);
+	
+	content_pane.add(instruction_textarea, BorderLayout.NORTH);
+	content_pane.add(details_pane, BorderLayout.CENTER);
+	content_pane.add(button_pane, BorderLayout.SOUTH);
+		
+	dialog.show();
+	hiddenFrame.dispose();
+
+	if (cancelled) {
+	    throw new org.apache.tools.ant.BuildException("Cancelled by user.");
+	}
+	
+	if (user_property != null) {
+	    getProject().setProperty(user_property, username);
+	}
+	getProject().setProperty(pword_property, password);
+	
+	
+	
+    }
+    private class FieldListener
+	implements ActionListener, DocumentListener {
+	
+	public void actionPerformed(ActionEvent actionEvent) {
+	    validateOkButton();
+	}
+	public void changedUpdate(DocumentEvent e) {
+	    
+	    validateOkButton();
+	}
+
+	public void insertUpdate(DocumentEvent e) {
+            
+	    validateOkButton();
+	}
+	
+	public void removeUpdate(DocumentEvent e) {
+            
+	    validateOkButton();
+	}
+	
+	private void validateOkButton() {
+	    if (name_field.getText().length()!=0 && 
+		pass_field.getPassword().length!=0) {		
+		ok_button.setEnabled(true);
+	    } else {
+		ok_button.setEnabled(false);
+	    }
+	}
+    }
+    
+
+    public static void main(String [] args) {
+	MyGetUserAndPassword task = new MyGetUserAndPassword();
+	task.setMessage("hi there kath");
+	task.setUserproperty("username");
+	task.setPwordproperty("password");
+	task.execute();
+
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MySetProxy.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MySetProxy.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/anttasks/MySetProxy.java	(revision 21112)
@@ -0,0 +1,70 @@
+/*
+ *    MyGetUserAndPassword.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.anttasks;
+
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;      
+
+/** A new ant task for as the setproxy task has a bug in it.
+ * To use this task, the compiled class must be put in the classpath. add a taskdef line to the build.xml:
+ * <taskdef name="mysetproxy" classname="MySetProxy" />
+ * and call it like the following:
+ *  <mysetproxy proxyhost="" proxyport="" proxyuser="" proxypassword=""/>
+*/
+public class MySetProxy 
+    extends org.apache.tools.ant.Task {
+
+    String host = null;
+    String port = null;
+    String user = null;
+    String password = null;
+
+    public void setProxyhost(String proxy_host) {
+	host = proxy_host;
+    }
+
+    public void setProxyPort(int proxy_port) {
+	port = Integer.toString(proxy_port);
+    }
+
+    public void setProxyuser(String proxy_user) {
+	user = proxy_user;
+    }
+    
+    public void setProxypassword(String proxy_password) {
+	password = proxy_password;
+    }
+
+    public void execute() {
+
+	try {
+	    System.setProperty("http.proxyType", "4");
+	    System.setProperty("http.proxyHost", host);
+	    System.setProperty("http.proxyPort", port);
+	    System.setProperty("http.proxySet", "true");
+	    Authenticator.setDefault(new Authenticator(){ 
+		    protected PasswordAuthentication getPasswordAuthentication(){ 
+			return new PasswordAuthentication(user, new String(password).toCharArray()); 
+		    } 
+		}); 
+	} catch (Exception e) {
+	    System.err.println("MySetProxy Error: couldn't set up the proxy");    
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/Phind.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/Phind.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/Phind.java	(revision 21112)
@@ -0,0 +1,945 @@
+/**********************************************************************
+ *
+ * Phind.java -- the Phind java applet - modified to work with gsdl3 kjdon
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+/*********************************************************************
+
+To use the applet, you'll need to embed it in a web page like this:
+
+<APPLET CODE="Phind.class" WIDTH=500 HEIGHT=500>
+
+  <PARAM NAME=collection  VALUE="fao.org">
+  <PARAM NAME=classifier  VALUE="1">
+  <PARAM NAME=phindcgi    VALUE="http://kowhai/cgi-bin/phindcgi">
+  <PARAM NAME=library     VALUE="http://kowhai/cgi-bin/library">
+  <PARAM NAME=backdrop    VALUE="http://kowhai/~paynter/transfer/phindtest/green1.jpg">
+  The Phind java applet.
+</APPLET>
+
+There are a bunch of other parameters; these are described in the 
+getParameters method below.  It is all done for you in Greenstone
+in the document.dm macro file (the _phindapplet_ macro).
+
+You may have problems with Java applet security.  Java applet's can only
+open socket connections (including the HTTP connections the applet uses 
+to get data) to the same server the applet was loaded from.  This means
+that your phindcgi, library, and (optional) backdrop URLs must be on the
+same machine that your web page was loaded from.
+
+**********************************************************************
+
+The applet comprises several classes:
+
+1. Phind (this file) is the applet class, loaded by the browser.
+   It also handles network connections.
+2. ResultDisplay is a Panel that sits in the applet and displays 
+   things; displays are connected in a doubly-linked list.
+3. ResultBox holds the results of a query.  Result boxes are shown
+   to the user through ResultDisplays.  Another doubly-linked list.
+4. ResultTitle acts as a caption to a ResultBox describing its contents.
+5. ResultCanvas is what the ResultBox data is drawn on.  It draws the
+   words on the screen, and handles user input from the mouse.
+6. ResultItem represents a single result object (a phrase or document).
+7. PhindTitle is for drawing backdrops in ResultDisplays.
+
+**********************************************************************/
+
+package org.greenstone.applet.phind;
+
+import java.awt.BorderLayout;
+import java.awt.Button;
+import java.awt.Choice;
+import java.awt.Color;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.event.ActionEvent;
+import java.awt.FlowLayout;
+import java.awt.Font;
+import java.awt.GridLayout;
+import java.awt.Image;
+import java.awt.Label;
+import java.awt.Panel;
+import java.awt.TextField;
+import java.awt.event.ActionListener;
+import java.net.URL;
+import java.io.DataInputStream;
+
+import java.net.Socket;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.io.IOException;
+
+import java.util.Vector;
+import java.util.Date;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+//import javax.xml.parsers.*;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+
+
+public class Phind extends java.applet.Applet 
+    implements ActionListener {
+    
+    // What is the collection called?
+    public String collection;
+
+    // Which phind classifier are we using? (There may be more than one.)
+    public String classifier;
+
+    // Internet address of phind resources
+    public String library_address, phindcgi_address;
+
+    // Initial search term
+    public String initialSearch;
+
+    // Number of phrases to retrieve at any one time
+    public int phraseBlockSize;
+
+    // Appearance parameters
+    public boolean vertical;
+    public int depth;
+
+    // Font
+    public int fontSize;
+    public String fontName;
+    public Font plainFont, boldFont;
+
+    // Do we want a background image in the applet?
+    public boolean showImage;
+    public String backdrop_address;
+    public Image backgroundImage;
+    public boolean showBorder;
+
+    // Colours
+    public Color panel_fg, panel_bg,
+	column_1_fg, column_1_bg, 
+	column_2_fg, column_2_bg,
+	highlight_fg, highlight_bg,
+	thesaurus_fg, thesaurus_bg, thesaurus_bar_fg, thesaurus_bar_bg,
+	expansion_fg, expansion_bg, expansion_bar_fg, expansion_bar_bg,
+	document_fg, document_bg, document_bar_fg, document_bar_bg,
+	message_fg, message_bg;
+    
+    // Column dimensions
+    int column_1_width, column_2_width;
+
+    // Where do we open new windows
+    String searchWindowName, documentWindowName;
+
+    // the mode of operation
+    int mode;
+    final int initMode = 0;
+    final int idleMode = 1;
+    final int searchMode = 2;
+
+    // Elements of the control panel
+    boolean showControlPanel;
+    Label titleLabel;
+    TextField wordField;
+    Button searchButton, prevButton, nextButton;
+
+    // Holders for the downloaded data
+    Panel resultPanel;
+    ResultDisplay firstDisplay, lastDisplay;
+
+    // The time at which the last query finished
+    Date lastQueryEndTime;
+
+    // lastQueryEndTime is stored to ensure a 1 second gap between a query
+    // returning and a new one beginning.  It is needed because the FAO
+    // folks in Rome have a huge lag, and frquently click several times
+    // while they wait; these clicks are turned into new queries, which
+    // they await again.  It is no elegant solution, but it seems like the
+    // easiest, given that I don't know threads.
+    // 1. The search button is easy to disable, and is disabled when a
+    //    socket connection is in progress.
+    // 2. ResutCanvas widgets can'r be similarly disabled because the
+    //    browser hides or wipes them, which looks very bad.
+    // 3. I cannot just ignore clicks on canvasses because the browser
+    //    caches the clicks while the socket connection is going on, and then
+    //    sends them through afterwards, when the canvas is accepting clicks
+    //    again.
+    // 4. Current sequence of events is to record the time the last query
+    //    ends, then whenever a click happens make sure a second has past.  if
+    //    you double-click the the first query is sent, returns, end-tie is
+    //    set, and the second (and any others made during query time) is
+    //    *immediately* processed, but since 1 second hasn't past it is
+    //    ignored.
+
+  
+    public String getAppletInfo() {
+	return "Phind by Gordon Paynter (paynter@cs.waikato.ac.nz). Copyright 1997-2000.";
+    }
+
+
+    public void init() {
+
+	mode = initMode;
+	
+	// Read applet parameters
+	getParameters();
+
+	// Initialise the user interface
+	setBackground(panel_bg);
+	lastQueryEndTime = new Date();
+
+	// fonts used to output text
+	plainFont = new Font(fontName, Font.PLAIN, fontSize);
+	boldFont  = new Font(fontName, Font.BOLD, fontSize);
+
+	// The phind applet layout manager
+	setLayout(new BorderLayout());
+
+	// Panel containing the displays is in the center of the display
+	resultPanel = new Panel();
+	if (vertical) {
+	    resultPanel.setLayout(new GridLayout(depth,1,0,2));
+	} else {
+	    System.out.println("horizontal");
+	    resultPanel.setLayout(new GridLayout(1,depth,2,0));
+	}
+	add("Center", resultPanel);
+
+	// Create ResultDisplays and place into the interface
+	ResultDisplay d1, d2 = null;
+	firstDisplay = new ResultDisplay(this, null);
+	resultPanel.add(firstDisplay);
+
+	if (depth == 1) {
+	    lastDisplay = firstDisplay;
+	} else {
+	    d1 = firstDisplay;
+	    for (int i = 2; i <= depth; i++) {
+		d2 = new ResultDisplay(this, d1);
+		resultPanel.add(d2);
+		d1 = d2;
+	    }
+	    lastDisplay = d2;
+	}
+
+	// The control panel
+	initialiseControlPanel();
+
+	// lets get started then
+	setStatus("Welcome to Phind.");
+	mode = idleMode;
+
+	// Perform initial search, if requested
+	if (initialSearch.length() > 0) {
+	    searchForWord(initialSearch);
+	}
+    }
+
+
+    // Display a message in the status bar
+    void setStatus(String status) {
+	showStatus(status);
+    }
+
+    // The user performs an action in the interface
+    /*   public boolean action(Event evt, Object arg) {
+
+	if (evt.target == searchButton) {
+	    System.out.println("evt.target==searchButton");
+	    searchForWord(getSearchTerm());
+	} else if (evt.target == wordField) {
+	    System.out.println("evt.target==wordField");
+	    searchForWord(getSearchTerm());
+	} else if (evt.target == prevButton) {
+	    shiftPrevious();
+	} else if (evt.target == nextButton) {
+	    shiftNext();
+	} else {
+	    System.out.println("unknown action: " + evt.toString() 
+			       + ", object: " + arg.toString());
+	}
+	return true;
+    }
+
+    */
+
+    public void actionPerformed(ActionEvent evt) {
+	
+	Component target = (Component)evt.getSource();
+	if (target==searchButton) {
+	    System.out.println("search button pressed");
+	    searchForWord(getSearchTerm());
+	} else if (target == wordField)  {
+	    System.out.println("word field entered");
+	    searchForWord(getSearchTerm());
+	} else if (target == prevButton) {
+	    System.out.println("prev button pressed");
+	    shiftPrevious();
+	}else if (target == nextButton) {
+	    System.out.println("prev button pressed");
+	    shiftNext();
+	} else {
+	    System.out.println("unknown action: " + evt.toString() );
+	    
+	}
+    }
+
+    // Search for a word 
+    //
+    // Called on two occasions:
+    //   when the "Search" Button is pressed, or
+    //   to perform an "initial search"
+    void searchForWord(String searchWord) {
+
+	System.err.println("in searchforword!!");
+	if (mode == idleMode) {
+	    
+	    setSearchTerm(searchWord);
+
+	    // Convert the String from UTF8 charaters into 
+	    // an encoding that is okay for a URL.
+	    searchWord = URLUTF8Encoder.encode(searchWord);
+
+	    // Look up the word
+	    if (searchWord.length() > 1) {
+		setStatus("searching for \"" + searchWord + "\""); 
+		firstDisplay.emptyContents();
+		ResultBox result = lookupPhraseOnServer(null, false, searchWord, searchWord, 2);
+
+		// if there is an error, return
+		if (result == null) {
+		    setStatus("No results for \"" + searchWord + "\""); 
+		    return;
+		}
+
+		// display the result
+		result.display = firstDisplay.display(result);
+		result.setSize(result.display.getSize());
+		result.paintAll(result.getGraphics());
+	    }
+
+	    enablePreviousAndNext();
+	}
+    }
+  
+
+    // Search for a phrase
+    //
+    // If querymode is 2, the user has clicked on a phrase.
+    // If querymode is 3, the user has requested more phrases.
+    // If querymode is 4, the user has requested more documents.
+    void searchForPhrase(ResultBox source, String key, String phrase, int queryMode) {
+
+	// System.out.println("searchForPhrase: " + key + " " + phrase + " " + queryMode);
+
+	if (mode == idleMode) {
+
+	    // If we are going to replace the first ResultDisplay, then empty it
+	    if (queryMode <= 2) {
+		if (source.display.next != null) source.display.next.emptyContents();
+	    }
+
+	    // look up the word
+	    setStatus("Searching for \"" + phrase + "\""); 
+	    ResultBox result = lookupPhraseOnServer(source, true, key, phrase, queryMode);
+	    if (result == null) {
+		setStatus("No result for \"" + phrase + "\""); 
+		return;
+	    }
+
+	    // If this is not already displayed, display it in the last free spot
+	    if (queryMode <= 2) {
+		result.display = lastDisplay.display(result);
+		result.setSize(result.display.getSize());
+		result.paintAll(result.getGraphics());
+	    }
+
+	    enablePreviousAndNext();
+	}
+    }
+
+
+    // Look up a phrase (or symbol) on the server
+    //
+    // Arguments are the source of the query (a ResultBox, or null if the
+    // query comes from hitting the search button), the key to search for
+    // (the text of a phrase or a symbol number), the phrase as a string, 
+    // and the query mode.
+    // Query modes are: 
+    //   0 = obsolete
+    //   1 = obsolete
+    //   2 = get first N phrases and URLs,
+    //   3 = get another N phrases into same window
+    //   4 = get another N documents into same window
+    //   5 = get another N thesaurus links into same window
+
+    ResultBox lookupPhraseOnServer(ResultBox source, 
+				   boolean keyKnown, String key, String phrase, 
+				   int queryMode) { 
+    	disableSearchButton();
+	mode = searchMode;
+	ResultBox r = null;
+
+	if (queryMode <= 2) {
+	    r = new ResultBox(this, collection, key, phrase, source);
+	} else if ((queryMode == 3) || (queryMode == 4) || (queryMode == 5)) {
+	    r = source;
+	}
+
+	try { 
+	    queryServer(keyKnown, key, queryMode, r);
+	} catch (Exception e) { 
+	    System.out.println("Phind query error: " + e.toString());
+	    setStatus("Query error: " + e.toString());
+	    mode = idleMode;
+	    enableSearchButton();
+	    return null;
+	}
+      
+	// The query is finished
+	setStatus(r.c.numberOfItems + " results for \"" + phrase + "\""); 
+	mode = idleMode;
+	enableSearchButton();
+	lastQueryEndTime = new Date();
+
+	return r;
+    }
+
+
+    // Query the phindcgi program
+    //
+    // Send a query (a word or symbol number) to the server
+    // and pass the response to a ResultBox.
+
+    void queryServer(boolean keyKnown, String word, int queryMode, ResultBox area) 
+	throws IOException {
+
+	// Build the query
+	String query = phindcgi_address + "c=" + collection + "&pc=" + classifier;
+
+	if (keyKnown) {
+	  query = query + "&ppnum=" + word;
+	} else {
+	  query = query + "&pptext=" + word;
+	}
+
+	
+	// Specify the set of results to return 
+	int first_e = 0;
+	int last_e = 0;
+	int first_d = 0;
+	int last_d = 0;
+	int first_l = 0;
+	int last_l = 0;
+
+	// the initial query
+	if (queryMode <= 2) {
+	  last_e = phraseBlockSize;
+	  last_d = phraseBlockSize;
+	  last_l = phraseBlockSize;
+	}
+
+	// add phrases to an existing result set
+	else if (queryMode == 3) {
+	  first_e = area.nextPhraseBlock * phraseBlockSize;
+	  area.nextPhraseBlock++;
+	  last_e = area.nextPhraseBlock * phraseBlockSize;
+	} 
+	
+	// add documents to existing result set
+	else if (queryMode == 4) {
+	  first_d = area.nextDocumentBlock * phraseBlockSize;
+	  area.nextDocumentBlock++;
+	  last_d = area.nextDocumentBlock * phraseBlockSize;
+	}
+
+	// add thesaurus links to existing result set
+	else if (queryMode == 5) {
+	  first_l = area.nextThesaurusLinkBlock * phraseBlockSize;
+	  area.nextThesaurusLinkBlock++;
+	  last_l = area.nextThesaurusLinkBlock * phraseBlockSize;
+	}
+
+	query = query + "&pfe=" + first_e + "&ple=" + last_e
+                      + "&pfd=" + first_d + "&pld=" + last_d
+                      + "&pfl=" + first_l + "&pll=" + last_l;
+
+	// Send the query to the phindcgi program 
+	System.out.println("1:sending query: " + query);
+	try {
+	    URL phindcgi = new URL(query);
+	    DataInputStream in = new DataInputStream(phindcgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    area.parseXML(data_elem);
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to phindcgi: " + e);
+	    e.printStackTrace();
+	}
+	area.repaint();
+    }
+    
+
+    // Tidy up URLs
+    //
+    // Ensure a URL address (as string) has a protocol, host, and file.
+    //
+    // If the URL is a CGI script URL, it should be tidied up so that it is
+    // appropriate to tage attrib=value pairs on the end.  This means it
+    // must either end with a "?" or (if it contains a question-mark
+    // internally) end with a "&".
+    String tidy_URL(String address, boolean isCGI) {
+
+	// System.err.println("tidy URL: " + address);
+	
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+
+	}
+
+	// if the URL is a cgi script, make sure it has a "?" in ti,
+	// and that it ends with a "?" or "&"
+	if (isCGI) {
+	    if (address.indexOf((int) '?') == -1) { 
+		address = address + "?";
+	    } else if (!address.endsWith("?")) {
+		address = address + "&";
+	    }
+	}
+
+	return address;
+    }
+
+
+
+    // Open an arbitrary web page
+    void displayWebPage(String address, String window) {
+	try { 
+	    URL url= new URL(address);
+	    if (window.length() > 0) {
+		getAppletContext().showDocument(url, window);
+	    } else {
+		getAppletContext().showDocument(url);
+	    }
+	} catch (Exception e) { 
+	    System.out.println("Cannot open web page: " + e.toString());
+	}
+    }
+
+
+    // Get the applet parameters
+    void getParameters() {
+
+	// What is this collection called?
+	collection = parameterValue("collection");
+	System.out.println("Phind collection: " + collection);
+
+	// Which of the collection's classifiers are we using?
+	classifier = parameterValue("classifier", "1");
+	System.out.println("Phind classifier: " + classifier);
+
+	// Where is the Greenstone library
+	library_address = parameterValue("library");
+	library_address = tidy_URL(library_address, true);
+	System.out.println("Phind library: " + library_address);
+
+	// Where is the phind CGI script
+	// we assume this is relative to the greenstone library
+	phindcgi_address = parameterValue("library")+parameterValue("phindcgi");
+	phindcgi_address = tidy_URL(phindcgi_address, true);
+	System.out.println("Phind phindcgi: " + phindcgi_address);
+
+	
+	// Is there a default search term?
+	initialSearch = parameterValue("initial_search", "");
+
+	// Should we display the control panel
+	showControlPanel = true;
+	if (parameterValue("control_panel", "show").toLowerCase().equals("hide")) {
+	    showControlPanel = false;
+	}
+	
+	// Should we show a background image?
+	backdrop_address = parameterValue("backdrop", "");
+	if (backdrop_address.length() > 0) {
+	    backdrop_address = tidy_URL(backdrop_address, false);
+	    System.out.println("Phind backdrop URL: " + backdrop_address);
+
+	    try {
+		URL backdrop_url = new URL(backdrop_address);
+		backgroundImage = getImage(backdrop_url);
+		showImage = true;
+	    } catch (Exception e) { 
+		System.out.println("Phind could not load " + backdrop_address);
+		showImage = false;
+	    }
+	}
+
+	// Should we draw a border?
+	showBorder = parameterValue("border", "on").equals("off");
+
+	// Are the windows arranged vertically or horizontally
+	if (parameterValue("orientation", "vertical").toLowerCase().startsWith("hori")) {
+	    vertical = false;
+	} else {
+	    vertical = true;
+	}
+	
+	// How many phind windows are there?
+	depth = parameterValue("depth", 3);
+
+	// Result sort order
+	// Standard is "LlEeDd", expansion-first is "EeLlDd"
+	String order = parameterValue("resultorder", "standard");
+	if (!order.equals("standard")) {
+	    int next = 20;
+	    ResultItem.sortMessage = next;
+	    for (int x = 0; x < order.length(); x++) {
+		if (order.charAt(x) == ',') {
+		    next--;
+		} else if (order.charAt(x) == 'L') {
+		    ResultItem.sortLinkItem = next;
+		} else if (order.charAt(x) == 'l') {
+		    ResultItem.sortMoreLinks = next;
+		} else if (order.charAt(x) == 'E') {
+		    ResultItem.sortPhraseItem = next;
+		} else if (order.charAt(x) == 'e') {
+		    ResultItem.sortMorePhrases = next;
+		} else if (order.charAt(x) == 'D') {
+		    ResultItem.sortDocumentItem = next;
+		} else if (order.charAt(x) == 'd') {
+		    ResultItem.sortMoreDocuments = next;
+		}
+	    }
+	    System.out.println("link: " + ResultItem.sortLinkItem);
+	    System.out.println("exps: " + ResultItem.sortPhraseItem);
+	    System.out.println("docs: " + ResultItem.sortDocumentItem);
+
+	}
+
+	// How many phrases should we fetch at any given time?
+	phraseBlockSize = parameterValue("blocksize", 10);
+
+	// What font should we use?
+	fontSize = parameterValue("fontsize", 10);
+	fontName = parameterValue("fontname", "Helvetica");
+
+	// Column dimensions
+	column_1_width = parameterValue("first_column_width", 6);
+	column_2_width = parameterValue("second_column_width", column_1_width);
+	
+	// Where do we open new windows
+	searchWindowName = parameterValue("search_window", "phindsearch");
+	documentWindowName = parameterValue("document_window", "phinddoc");
+
+	// Colours
+	panel_fg = parameterValue("panel_fg", Color.black);
+	panel_bg = parameterValue("panel_bg", Color.white);
+
+	highlight_bg = parameterValue("highlight_bg", Color.yellow);
+
+	expansion_fg = parameterValue("expansion_fg", Color.black);
+	thesaurus_fg = parameterValue("thesaurus_fg", new Color(0, 100, 0));
+	document_fg = parameterValue("document_fg", Color.blue);
+
+	thesaurus_bar_fg = parameterValue("thesaurus_bar_fg", Color.black);
+	expansion_bar_fg = parameterValue("expansion_bar_fg", Color.black);
+	document_bar_fg = parameterValue("document_bar_fg", Color.black);
+
+	thesaurus_bar_bg = parameterValue("thesaurus_bar_bg", new Color(160, 160, 190));
+	expansion_bar_bg = parameterValue("expansion_bar_bg", new Color(255, 200, 200));
+	document_bar_bg = parameterValue("document_bar_bg", new Color(150, 193, 156));
+
+	column_1_fg = parameterValue("first_column_fg", Color.black);
+	column_1_bg = parameterValue("first_column_bg", new Color(235, 245, 235));
+	column_2_fg = parameterValue("second_column_fg", Color.black);
+	column_2_bg = parameterValue("second_column_bg", new Color(200, 220, 200));
+
+	message_fg = parameterValue("message_fg", Color.black);
+	message_bg = parameterValue("message_bg", Color.white);
+
+	// Colours I don't use, yet
+	// thesaurus_bg = parameterValue("thesaurus_bg", Color.white);
+	// expansion_bg = parameterValue("expansion_bg", Color.white);
+	// document_bg = parameterValue("document_bg", Color.white);
+    }
+
+    // Get the value of a parameter given its name.
+    // There are many types of parameters, hence the variety of functions.
+
+    // Get a REQUIRED string.  Stop the applet if we cannot.
+    String parameterValue(String name) {
+	try { 
+	    return getParameter(name);
+	} catch (Exception e) {
+	    System.err.println("Phind: you must give a parameter for \"" 
+			       + name + "\".  Stopping.");
+	    stop();
+	}
+	return "";
+    }
+
+    // Get an optional parameter.  Return a default if we cannot.
+    String parameterValue(String name, String defaultValue) {
+	String text = getParameter(name);
+	if (text == null) {
+	    return defaultValue;
+	}
+	System.out.println("Phind " + name + ": " + text);
+	return text;
+    }
+
+    int parameterValue(String name, int defaultValue) {
+	int value;
+	try { 
+	    value = Integer.parseInt(getParameter(name));
+	} catch (Exception e) {
+	    return defaultValue;
+	}
+	System.out.println("Phind " + name + ": " + value);
+	return value;
+    }
+
+    Color parameterValue(String name, Color defaultValue) {
+
+	String text = getParameter(name);
+	if (text == null) {
+	    return defaultValue;
+	}
+	text = text.toLowerCase();
+
+	// a number of the form "#ddffee" 
+	if (text.startsWith("#") && (text.length() == 7)) {
+	    text = text.substring(1);
+	    int r, g, b;
+	    try {
+		r = Integer.parseInt(text.substring(0,2), 16);
+		g = Integer.parseInt(text.substring(2,4), 16);
+		b = Integer.parseInt(text.substring(4), 16);
+		return new Color(r, g, b);
+	    }  catch (Exception e) {
+		return defaultValue;
+	    }
+	}
+
+	// a known Java colour string
+	else if (text.equals("black")) { return Color.black; }
+	else if (text.equals("blue")) { return Color.blue; }
+	else if (text.equals("cyan")) { return Color.cyan; }
+	else if (text.equals("darkgray")) { return Color.darkGray; }
+	else if (text.equals("gray")) { return Color.gray; }
+	else if (text.equals("green")) { return Color.green; }
+	else if (text.equals("lightgray")) { return Color.lightGray; }
+	else if (text.equals("magenta")) { return Color.magenta; }
+	else if (text.equals("orange")) { return Color.orange; }
+	else if (text.equals("pink")) { return Color.pink; }
+	else if (text.equals("red")) { return Color.red; }
+	else if (text.equals("white")) { return Color.white; }
+	else if (text.equals("yellow")) { return Color.yellow; }
+
+	return defaultValue;
+    }
+
+
+    // Control panel operations
+
+    // Initialise the control panel
+    void initialiseControlPanel() {
+	
+	if (showControlPanel) {
+	    Panel p1 = new Panel();
+	    add("North", p1);
+	    p1.setLayout(new FlowLayout(FlowLayout.CENTER, 0, 0));
+	    
+	    searchButton = new Button("Search");
+	    searchButton.setFont(boldFont);
+	    //searchButton.setEnabled(true);
+	    searchButton.addActionListener(this);
+	    p1.add(searchButton);
+	    
+	    Label tempLabel = new Label("  for");
+	    tempLabel.setFont(boldFont);
+	    p1.add(tempLabel);
+	    
+	    wordField = new TextField(12);
+	    wordField.setFont(boldFont);
+	    wordField.addActionListener(this);
+	    p1.add(wordField);
+	    
+	    Label temp2 = new Label("          ");
+	    p1.add(temp2);
+	    
+	    prevButton = new Button("Previous");
+	    prevButton.setFont(boldFont);
+	    prevButton.addActionListener(this);
+	    prevButton.setEnabled(false);
+	    
+	    p1.add(prevButton);
+	    
+	    nextButton = new Button("  Next  ");
+	    nextButton.setFont(boldFont);
+	    nextButton.addActionListener(this);
+	    nextButton.setEnabled(false);
+	    p1.add(nextButton);
+	    
+	}
+    }
+    
+    // Button and field functionality
+    
+    // Enable and disable the word field
+    void enableSearchButton() {
+	if (showControlPanel) {
+	    searchButton.setEnabled(true);
+	}
+    }
+    void disableSearchButton() {
+	if (showControlPanel) {
+	    searchButton.setEnabled(false);
+	}
+    }
+
+    // Get and set the search text in the wordField
+    String getSearchTerm() {
+	if (showControlPanel) {
+	    return wordField.getText();
+	} else {
+	    return initialSearch;
+	}
+    }
+    void setSearchTerm(String word) {
+	if (showControlPanel) {
+	    wordField.setText(word);
+	} 
+    }
+
+    // Enable or disable the "Previous" and "Next" buttons
+    void enablePreviousAndNext() {
+	if (showControlPanel) {
+	    Component c = firstDisplay.current;
+	    if (c.getClass().getName().endsWith("ResultBox")) {
+		if (((ResultBox) c).prevBoxExists()) {
+		    prevButton.setEnabled(true);
+		} else {
+		    prevButton.setEnabled(false);
+		}
+	    }
+	    
+	    c = lastDisplay.current;
+	    if (c.getClass().getName().endsWith("ResultBox")) {
+		if (((ResultBox) c).nextBoxExists()) {
+		    nextButton.setEnabled(true);
+		} else {
+		    nextButton.setEnabled(false);
+		}
+	    }
+	}
+    }
+
+    // Shift to previous box
+    //
+    // If the user clicks "Previous" then scroll up.
+    void shiftPrevious() {
+
+	Component c = firstDisplay.current;
+	if (c.getClass().getName().endsWith("ResultBox")) {
+
+	    ResultBox b = (ResultBox) c;
+	    if (b.prevBoxExists()) {
+		b = b.prev;
+
+		// empty all the displays
+		firstDisplay.emptyContents();
+
+		// add as many result boxes as there are displays
+		for (int i = 1 ; ((i <= depth) && (b != null)); i++) {
+		    lastDisplay.display(b);
+		    b.setSize(b.display.getSize());
+		    b.paintAll(b.getGraphics());
+		    b = b.next;
+		}
+	    }
+	}
+	enablePreviousAndNext();
+    }
+
+    // Shift to next box
+    //
+    // If the user clicks "Next" then scroll down if possible
+    void shiftNext() {
+
+	Component c = lastDisplay.current;
+	if (c.getClass().getName().endsWith("ResultBox")) {
+
+	    ResultBox b = (ResultBox) c;
+	    if (b.nextBoxExists()) {
+
+		// find the new "first" displayed box
+		c = firstDisplay.current;
+		b = (ResultBox) c;
+		b = b.next;
+
+		// empty all the displays
+		firstDisplay.emptyContents();
+
+		// add as many result boxes as there are displays
+		for (int i = 1 ; ((i <= depth) && (b != null)); i++) {
+		    lastDisplay.display(b);
+		    b.setSize(b.display.getSize());
+		    b.paintAll(b.getGraphics());
+		    b = b.next;
+		}
+	    }
+	}
+	enablePreviousAndNext();
+    }
+
+
+
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/PhindTitle.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/PhindTitle.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/PhindTitle.java	(revision 21112)
@@ -0,0 +1,84 @@
+/**********************************************************************
+ *
+ * PhindTitle.java -- backdrops to empty ResultDisplay panels.
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+/*********************************************************************
+
+PhindTitle is for drawing backdrops in empty ResultDisplay panels.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.applet.Applet;
+import java.awt.Canvas;
+import java.awt.Color;
+import java.awt.Dimension;
+import java.awt.Graphics;
+import java.awt.Image;
+
+public class PhindTitle extends Canvas {
+
+    Phind phind;
+    static Image backgroundImage;
+    boolean displayInfo;
+    
+    PhindTitle(Phind p) {
+	phind = p;
+    }
+
+    public void update(Graphics g) {
+	paint(g);
+    }
+
+    public void paint(Graphics g) {
+	Dimension canvasSize = getSize();
+
+	Color fore = phind.panel_fg;
+	Color back = phind.panel_bg;
+
+	// set the screen background
+	if (phind.showImage)
+	    try {
+		g.drawImage(phind.backgroundImage, 
+			    0, 0, canvasSize.width, canvasSize.height, back, null);
+	    } catch (Exception e) {
+		System.err.println("PhindTitle paint: "  + e);
+	    }
+	else {
+	    g.setColor(back);
+	    g.fillRect(0,0, canvasSize.width, canvasSize.height);
+	}
+    }
+}
+
+ 
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultBox.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultBox.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultBox.java	(revision 21112)
@@ -0,0 +1,423 @@
+/**********************************************************************
+ *
+ * ResultBox.java -- a list of phrases in the Phind java interface
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A ResultBox holds the results of a query to phindcgi.  They deal mostly
+with the information content of the query, and have methods for parsing the
+input into phrase and document items.  They have little do with display:
+ResultBoxes are shown to the user through "ResultDisplay" panels, and are
+drawn using "ResultCanvas" objects.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Panel;
+import java.awt.BorderLayout;
+import java.awt.Scrollbar;
+
+import java.awt.Label;
+import java.awt.AWTEvent;
+import java.awt.event.AdjustmentListener;
+import java.awt.event.AdjustmentEvent;
+import java.net.*;
+import java.applet.*;
+
+import java.util.Vector;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ResultBox extends Panel 
+    implements AdjustmentListener {
+    
+    // Objects at a "higher" level than this one
+    Phind phind;
+    ResultDisplay display;
+
+    // Surroundig objects
+    ResultBox prev, next;
+
+    // The components contained by this one.
+    ResultCanvas c;
+    ResultTitle t;
+    Scrollbar s;
+    Panel label;
+
+    // The key identifying the phrase displayed, with its text and the
+    // collection from which it is drawn
+    String searchKey, searchPhrase, searchCollection;
+    
+    // The total frequency, expansion frequency, and document frequency
+    // of the phrase
+    int numberOfOccurances;
+    int numberOfExpansions;
+    int numberOfDocuments;
+    int numberOfThesaurusLinks;
+
+    // The number of phrases and documents retrieved, and the number of
+    // times the user has requested more phrases or documents.
+    int expansionsRetrieved;
+    int documentsRetrieved;
+    int thesaurusLinksRetrieved;
+    int nextPhraseBlock;
+    int nextDocumentBlock;
+    int nextThesaurusLinkBlock;
+    
+    int mode;
+    final int initMode = 0;
+    public final int emptyMode = 1;
+    final int loadingMode = 2;
+    final int finishedMode = 3;
+
+    String buffer;
+    boolean finished;
+
+
+    // Create a ResultBox
+    // given details of the search that generated it.
+    ResultBox(Phind p, String collect, String key, String phrase, ResultBox rb) {
+
+	super();
+	mode = initMode;
+
+	phind = p;
+	display = null;
+	next = null;
+	prev = rb;
+	if (prev != null) prev.next = this;
+
+	searchKey = key;
+	searchPhrase = phrase;
+	searchCollection = collect;
+
+	numberOfOccurances = -1;
+	numberOfExpansions = -1;
+	numberOfDocuments = -1;
+	numberOfThesaurusLinks = -1;
+
+	expansionsRetrieved = 0;
+	documentsRetrieved = 0;
+	thesaurusLinksRetrieved = 0;
+
+	nextPhraseBlock = 1;
+	nextDocumentBlock = 1;
+	nextThesaurusLinkBlock = 1;
+
+
+	setLayout(new BorderLayout());
+
+	s = new Scrollbar(Scrollbar.VERTICAL);
+	disableScrollbar();
+	s.addAdjustmentListener(this);
+	add("East", s);
+
+	c = new ResultCanvas(this);
+	add("Center", c);
+
+	t = new ResultTitle(this);
+	add("North", t);
+
+	buffer = "";
+	finished = false;
+	mode = emptyMode;
+    }
+
+    static String describeContents(String phrase, String c) {
+	return( "\"" + phrase + "\" in " + c + ".");
+    }
+
+    String describeContents() {
+	return( describeContents(searchPhrase, searchCollection) );
+    }
+
+
+    // Reset the contents of the box
+    void resetBox( ) {
+	buffer = "";
+	finished = false;
+	c.resetCanvas();
+	disableScrollbar();
+	mode = emptyMode;
+    
+	numberOfExpansions = -1;
+	numberOfDocuments = -1;
+	numberOfThesaurusLinks = -1;
+
+	expansionsRetrieved = 0;
+	documentsRetrieved = 0;
+	thesaurusLinksRetrieved = 0;
+    }
+
+    void setStatus( String status ) {
+	phind.setStatus(status);
+    }
+
+    void disableScrollbar() {
+	if (s.isEnabled()) {
+	    s.setValues(0, 1, 0, 1);
+	    s.setUnitIncrement(1);
+	    s.setBlockIncrement(1);
+	    s.setEnabled(false);
+	}
+    }
+
+    // Are there displays previous to and after this?
+    public boolean prevBoxExists () {
+	return (prev != null);
+    }
+    public boolean nextBoxExists () {
+	return (next != null);
+    }
+
+    
+    // Look up a phrase
+    // Phrase lookups are passed on to the Phind applet itself.
+    void lookupPhrase(String key, String phrase, int queryMode) {
+	buffer = "";
+	finished = false;
+	phind.searchForPhrase(this, key, phrase, queryMode);
+	t.repaint();
+    }
+
+    /*   public void processEvent(AWTEvent event) {
+	//    System.out.println("event: " + event.toString());
+	if ( (event.target == s) && 
+	     ( (event.id == Event.SCROLL_ABSOLUTE) ||
+	       (event.id == Event.SCROLL_LINE_DOWN) ||
+	       (event.id == Event.SCROLL_LINE_UP) ||
+	       (event.id == Event.SCROLL_PAGE_DOWN) ||
+	       (event.id == Event.SCROLL_PAGE_UP) ) ) {
+	    c.repaint();
+	    
+	} else {
+	    super.processEvent(event);
+	}
+	} */  
+
+    public void adjustmentValueChanged(AdjustmentEvent evt) {
+	c.repaint();
+    }
+
+    void parseXML(Element data) {
+
+	//System.out.println("phinddata:"+data.toString());
+
+	//String id="", phrase="";
+	//int df=0, ef=0, lf=0, tf=0;
+	
+	searchKey = data.getAttribute("id");
+	
+	numberOfDocuments = Integer.valueOf(data.getAttribute("df")).intValue();
+	numberOfExpansions = Integer.valueOf(data.getAttribute("ef")).intValue();
+	numberOfThesaurusLinks = Integer.valueOf(data.getAttribute("lf")).intValue();
+	numberOfOccurances = Integer.valueOf(data.getAttribute("tf")).intValue();
+
+	//searchPhrase = "cat"; // for now
+
+	// go through children
+	Node e = data.getFirstChild();
+	while (e!=null) {
+	    String node_name = e.getNodeName();
+	    if (node_name.equals("phrase")) {
+		// the value of the phrase
+		searchPhrase = getNodeText(e);
+	    } else if (node_name.equals("expansionList")) {
+		
+		NodeList expansions = ((Element)e).getElementsByTagName("expansion");
+		for (int i=0; i<expansions.getLength(); i++) {
+		    processExpansionElement((Element)expansions.item(i));
+		}
+	    } else if (node_name.equals("documentList")) {
+			
+		NodeList documents = ((Element)e).getElementsByTagName("document");
+		for (int i=0; i<documents.getLength(); i++) {
+		    processDocumentElement((Element)documents.item(i));
+		}
+	
+	
+	    } else if (node_name.equals("thesaurusList")) {
+				
+		NodeList thesaurai = ((Element)e).getElementsByTagName("thesaurus");
+		for (int i=0; i<thesaurai.getLength(); i++) {
+		    processThesaurusElement((Element)thesaurai.item(i));
+		}
+		
+	    } else {
+		System.out.println("error in phinddata format, have unwanted node, "+node_name);
+	    }
+	    e = e.getNextSibling();
+	}
+	// finished parsing, update the getMoreDocs markers
+	c.updateMarkers();
+    }
+
+
+    /** Add an expansion tag
+     *
+     * Given a string containing an XML expansion element of the form:
+     *   <expansion num="3" id="8421" prefix="PEOPLE and" suffix="" tf="3" df="3"><suffix></suffix><prefix>PEOPLE</prefix></expansion>
+     *
+     * Create a new ResultItemPhrase for display
+     *
+     *  Return true if successful, otherwise false. */
+    boolean processExpansionElement(Element e) {
+
+	String num = "", id = "", tf = "", df = "", 
+	    prefix = "", body = "", suffix = "";
+
+	body = searchPhrase;
+
+	id = e.getAttribute("id");
+	tf = e.getAttribute("tf");
+	df = e.getAttribute("df");
+	num = e.getAttribute("num");
+
+	// get prefix child
+	Node prefix_node = getChildByTagName(e, "prefix");
+	if (prefix_node!=null) {
+	    prefix = getNodeText(prefix_node);
+	}
+	Node suffix_node = getChildByTagName(e, "suffix");
+	if (suffix_node !=null) {
+	    suffix = getNodeText(suffix_node);
+	}
+
+	ResultItemPhrase ri = new ResultItemPhrase(id, tf, df, prefix, body, suffix);
+	
+	if (c.addResultItem(ri)) {
+	    expansionsRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** Add a document tag
+     *
+     * Given an XML Element of the form:
+     *   <document num="2" hash="HASH424e64b811fdad933be69c" freq="1"><title>CONTENTS</title></document>
+     *
+     * Create a new ResultItemDocument for display
+     *
+     * Return true if successful, otherwise false. */
+
+    boolean processDocumentElement(Element d ) {
+	// why do we have num??
+	String num = "", hash = "", freq = "", title = "";
+
+	num = d.getAttribute("num");
+	freq = d.getAttribute("freq");
+	hash = d.getAttribute("hash");
+
+	Node title_node = getChildByTagName(d, "title");
+	if (title_node != null) {
+	    title = getNodeText(title_node);
+	}
+	// Create a new ResultItem and add it to the display
+	ResultItemDocument ri = new ResultItemDocument(hash, title, freq);
+	
+	if (c.addResultItem(ri)) {
+	    documentsRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** Add a thesaurus tag
+     *
+     * Given an  XML element  of the form:
+     *
+     * <thesaurus num="3" id="36506" tf="0" df="0" type="RT"><phrase>ANGLOPHONE AFRICA</phrase></thesaurus>
+     *
+     * Create a new ResultItemLink for display
+     *
+     * Return true if successful, otherwise false. */
+
+    boolean processThesaurusElement(Element t ) {
+	
+	// why do we have num??? - not used anywhere
+	String num = "", id = "", tf = "", df = "", type = "", phrase = "";
+
+	id = t.getAttribute("id");
+	tf = t.getAttribute("tf");
+	df = t.getAttribute("df");
+	type = t.getAttribute("type");
+	num = t.getAttribute("num");
+
+	Node phrase_node = getChildByTagName(t, "phrase");
+	if (phrase_node !=null) {
+	    phrase = getNodeText(phrase_node);
+	}
+	// Create a new ResultItem and add it to the display
+	ResultItemLink ri = new ResultItemLink(id, phrase, type, tf, df);
+
+	if (c.addResultItem(ri)) {
+	    thesaurusLinksRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** extracts the text out of a node */
+    protected String getNodeText(Node elem) {
+	elem.normalize();
+	Node n = elem.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	if (n==null) { // no text node
+	    return "";
+	}
+	return n.getNodeValue();
+    }
+
+    /** returns the (first) child element with the given name */
+    protected Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultCanvas.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultCanvas.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultCanvas.java	(revision 21112)
@@ -0,0 +1,647 @@
+/**********************************************************************
+ *
+ * ResultCanvas.java -- a Canvas onto which a phrase list is drawn
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+The results of a query are displayed on a ResultCanvas object.  Each 
+entry in the result list is stored in a ResultItem, and the ResultCanvas 
+contains a Vector of ResultItems.
+
+Each ResultCanvas is embedded in a ResultBox alongside a Scrollbar.  
+When the ResultCanvas is drawn, it looks at the scrollbar, calculates 
+which ResultItems are visible, then draws them on the screen.
+
+**********************************************************************/
+
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Canvas;
+import java.awt.Scrollbar;
+import java.awt.Graphics;
+import java.awt.Font;
+import java.awt.FontMetrics;
+import java.awt.Dimension;
+import java.awt.Color;
+import java.util.Vector;
+import java.awt.Image;
+//import java.awt.Event;
+import java.awt.event.MouseListener;
+import java.awt.event.MouseEvent;
+import java.awt.event.MouseAdapter;
+import java.net.URL;
+import java.applet.Applet;
+import java.util.Date;
+
+public class ResultCanvas extends Canvas {
+    
+
+    // Other objects associated with this 
+    Phind phind;
+    ResultBox parent;
+    Scrollbar scrollbar;
+    
+    // fonts and font spacings to use
+    Font areaPlain, areaBold;
+    int lineSpacing;
+    
+    // The items to display on this canvas
+    int numberOfItems;
+    int firstItemDisplayed;
+    int itemSelected;
+    Vector items;
+    
+    // the background image
+    public static Image backgroundImage;
+    
+    
+    // Create a ResultCanvas from the ResultBox which is its parent.
+    ResultCanvas(ResultBox p) {
+	
+	parent = p;
+	phind = p.phind;
+	scrollbar = p.s;
+	parent.disableScrollbar();
+
+	areaPlain = phind.plainFont;
+	areaBold  = phind.boldFont;
+
+	lineSpacing = phind.fontSize + 2;
+
+	items = new Vector();
+	numberOfItems = 0;
+	firstItemDisplayed = 0;
+	itemSelected = -1;
+    
+	if (backgroundImage == null) {
+	    backgroundImage = p.phind.backgroundImage;
+	}
+
+	// add a mouse listener for the mouse clicks. we only want to override
+	// one method, so we use the adaptor class
+	addMouseListener(new MouseAdapter() {
+		public void mouseClicked(MouseEvent evt) {
+		    ResultCanvas.this.mouseClicked(evt);
+		}
+	    });
+	
+
+    }
+
+    void resetCanvas( ) {
+	items.removeAllElements();
+	numberOfItems = 0;
+	repaint();
+    }
+
+
+    // Add a new search result.
+    // Return true if successful, otherwise false.
+    boolean addResultItem( ResultItem item ) {
+ 
+	// Add a new result, in sorted order.
+	// First sort key is item kind (lowest first),
+	// second is frequency (highest first).
+	// This is not efficient, but I don't care right now.
+	int index = 0;
+	while ((index < numberOfItems) &&
+	       ((item.sort < ((ResultItem) items.elementAt(index)).sort) ||
+		((item.sort == ((ResultItem) items.elementAt(index)).sort) &&
+		 (item.frequency <= ((ResultItem) items.elementAt(index)).frequency)))) {
+	    index++;
+	}
+	items.insertElementAt(item,index);
+	numberOfItems++;
+	return true;
+    }
+
+
+    // Update more phrases/documents/links markers
+    void updateMarkers() {
+	updateMorePhrasesMarker();
+	updateMoreDocumentsMarker();
+	updateMoreLinksMarker();
+    }
+
+    // Make sure the more Phrases item only appears in the list
+    // if it is meant to.
+    void updateMorePhrasesMarker() {
+    
+	 System.out.println("updateMorePhrasesMarker() ");
+	 System.out.println("expansions retrieved = "+parent.expansionsRetrieved +" num expansions = "+parent.numberOfExpansions);
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (((ResultItem) items.elementAt(index)).isMorePhrases()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.expansionsRetrieved == parent.numberOfExpansions) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.expansionsRetrieved < parent.numberOfExpansions) {
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.morePhrases);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+
+    // Make sure the more documents marker appears when required
+    void updateMoreDocumentsMarker() {
+    
+	// System.out.println("updateMoreDocumentsMarker() ");
+
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (((ResultItem) items.elementAt(index)).isMoreDocuments()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.documentsRetrieved == parent.numberOfDocuments) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.documentsRetrieved < parent.numberOfDocuments) {
+	    // there should be a marker
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.moreDocuments);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+      
+    // Make sure the more links marker appears when required
+    void updateMoreLinksMarker() {
+    
+	// System.out.println("updateMoreLinksMarker() ");
+
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (((ResultItem) items.elementAt(index)).isMoreLinks()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.thesaurusLinksRetrieved == parent.numberOfThesaurusLinks) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.thesaurusLinksRetrieved < parent.numberOfThesaurusLinks) {
+	    // there should be a marker
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.moreLinks);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+      
+
+ 
+
+
+    public void update(Graphics g) {
+	paint(g);
+    }
+
+    public void paint(Graphics g) {
+
+	// calculate the canvas size, margins, and spacing
+	
+	// Note: the size() method is deprecated, but getSize fails in NetScape 4
+	Dimension canvasSize = getSize();
+	
+	g.setFont(areaPlain);
+	int space = g.getFontMetrics().stringWidth(" ");
+	int nought = g.getFontMetrics().stringWidth("0");
+
+	int firstColumnWidth = phind.column_1_width * nought;
+	int secondColumnWidth = phind.column_2_width * nought;
+	int columnWidth = firstColumnWidth + secondColumnWidth;
+
+	int firstColumnLeft = canvasSize.width - columnWidth;
+	int secondColumnLeft = firstColumnLeft + firstColumnWidth;
+
+	int mainLeftMargin = 0;
+	int mainRightMargin = firstColumnLeft;
+
+	// the number of items that will fit on the canvas
+	int visible = canvasSize.height / lineSpacing;
+
+	// calculate the first item to output
+	int scrollValue = scrollbar.getValue();
+	if (numberOfItems <= visible) {
+	    scrollValue = 0;
+	} else if (scrollValue > (numberOfItems - visible)) {
+	    scrollValue = numberOfItems - visible;
+	} 
+	firstItemDisplayed = scrollValue;
+
+	// Draw the phrase area
+	Color fore = phind.panel_fg;
+	Color back = phind.panel_bg;
+
+
+	// draw the background
+	if (phind.showImage) {
+	    try {
+		g.drawImage(backgroundImage, 
+			    mainLeftMargin, 0, mainRightMargin, canvasSize.height, 
+			    back, null);
+	    } catch (Exception e) {
+		phind.showImage = false;
+		System.err.println("ResultCanvas paint error: " + e); 
+		g.setColor(back);
+		g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	    }
+	} else {
+	    g.setColor(back);
+	    g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	}
+    
+	// If there are no phrases, output a brief explanation.
+	if (numberOfItems == 0) {
+	    g.drawString("No phrases match this query.", mainLeftMargin + 10, lineSpacing);
+	}
+
+	// Output each of the visible ResultItems
+	ResultItem result;
+	int tab, i, y = 0;
+	int center = mainLeftMargin 
+	    + ((mainRightMargin - mainLeftMargin 
+		- g.getFontMetrics().stringWidth(parent.searchPhrase)) / 2);
+	int thesColumn = mainLeftMargin + firstColumnWidth 
+	    + g.getFontMetrics().stringWidth("Narrower term");
+
+	String body, prefix, suffix;
+	
+	for (i = scrollValue; 
+	     (i < numberOfItems) && (y + lineSpacing < canvasSize.height); i++) {
+
+	    // Get the resultItem to output
+	    result = (ResultItem) items.elementAt(i);
+	    
+	    // Graphics settings for drawing this line
+	    y += lineSpacing;
+	    g.setFont(areaPlain);
+
+	    // Highlight the selected phrase.
+	    if (i == itemSelected) {
+		g.setColor(phind.highlight_bg);
+		g.fillRect(mainLeftMargin, y-lineSpacing+2, mainRightMargin, lineSpacing);
+	    }
+
+	    // Draw the item
+
+	    // Draw a phrase item
+	    if (result.isPhrase()) {
+		prefix = result.prefixText();
+		body = result.mainText();
+		suffix = result.suffixText();
+		
+		g.setColor(phind.expansion_fg);
+		g.setFont(areaPlain);
+		tab = center - g.getFontMetrics().stringWidth(prefix) - space;
+		g.drawString(prefix, tab, y);
+		
+		g.setFont(areaBold);
+		g.drawString(body, center, y);
+		
+		tab = center + space + g.getFontMetrics().stringWidth(body);
+		g.setFont(areaPlain);
+		g.drawString(suffix, tab, y);
+	
+	    } 
+
+	    // Draw a document item
+	    else if (result.isDocument()){
+		body = result.mainText();
+
+		g.setColor(phind.document_fg);
+		g.setFont(areaPlain);
+		tab = (mainRightMargin - g.getFontMetrics().stringWidth(body)) / 2;
+		g.drawString(body, tab, y);
+      
+	    } 
+	
+	    // Draw a thesaurus link item
+	    else if (result.isLink()){
+		prefix = result.prefixText() + ":";
+		body = result.mainText();
+		tab = thesColumn - g.getFontMetrics().stringWidth(prefix) - space;
+
+		g.setColor(phind.thesaurus_fg);
+		g.setFont(areaPlain);
+		g.drawString(prefix, tab, y);
+		g.setFont(areaBold);
+		g.drawString(body, thesColumn, y);
+	    } 
+	    
+	    // Draw a "more phrases/documents/links" marker
+	    else if (result.isMorePhrases() 
+		     || result.isMoreDocuments() 
+		     || result.isMoreLinks()){
+		body = result.mainText();
+		
+		if (result.isMorePhrases()) {
+		    g.setColor(phind.expansion_bar_bg);
+		} else if (result.isMoreDocuments()) {
+		    g.setColor(phind.document_bar_bg);
+		} else {
+		    g.setColor(phind.thesaurus_bar_bg);		    
+		}
+		g.fillRect(mainLeftMargin, y-lineSpacing+2, mainRightMargin, lineSpacing);
+
+		if (result.isMorePhrases()) {
+		    g.setColor(phind.expansion_bar_fg);
+		} else if (result.isMoreDocuments()) {
+		    g.setColor(phind.document_bar_fg);
+		} else {
+		    g.setColor(phind.thesaurus_bar_fg);		    
+		}
+		g.setFont(areaPlain);
+		tab = (mainRightMargin - g.getFontMetrics().stringWidth(body)) / 2;
+		g.drawString(body, tab, y);
+	    } 
+	}
+
+	// Draw the frequecy columns
+
+	// column backgrounds
+	g.setColor(phind.column_1_bg);
+	g.fillRect(firstColumnLeft, 0, firstColumnWidth, canvasSize.height);
+	g.setColor(phind.column_2_bg);
+	g.fillRect(secondColumnLeft, 0, secondColumnWidth, canvasSize.height);
+
+	// fill in the numbers
+	g.setColor(phind.column_1_fg);
+	g.setFont(areaPlain);
+	y = 0;
+	String docsText, freqText;
+
+	for (i = scrollValue; 
+	     (i < numberOfItems) && (y + lineSpacing < canvasSize.height); i++) {
+	    
+	    // Get the resultItem to output
+	    result = (ResultItem) items.elementAt(i);
+	    docsText = result.docsText();
+	    freqText = result.freqText();
+	    
+	    // Graphics settings for drawing this line
+	    y += lineSpacing;
+
+	    // Write the document frequency
+	    if (docsText.length() > 0) {
+		tab = secondColumnLeft - space - g.getFontMetrics().stringWidth(docsText);
+		g.drawString(docsText, tab, y);
+	    }
+
+	    // Write the term frequency
+	    if (freqText.length() > 0) {
+		tab = secondColumnLeft + secondColumnWidth 
+		    - space - g.getFontMetrics().stringWidth(freqText);
+		g.drawString(freqText, tab, y);
+	    }
+	}
+
+	// Adjust the scrollbar
+	if (visible >= numberOfItems) {
+	    parent.disableScrollbar();
+	} else {
+	    scrollbar.setValues(scrollValue, visible, 0, numberOfItems);
+	    // Deprecated - setPageIncrement() has been replaced by
+	    // setBlockIncrement(), but the latter doesn't work on some
+	    // browsers.  Damn.
+	    scrollbar.setBlockIncrement(visible - 1);
+	    //scrollbar.setPageIncrement(visible - 1);
+	    // Deprecated - more nonsense: enable() has been replaced by
+	    // setEnabled(true), but this doesn't work on some browsers.
+	    scrollbar.setEnabled(true);
+	    //scrollbar.enable();
+	}
+
+	
+	// draw the border
+	if (phind.showBorder) {
+	    g.setColor(phind.panel_fg);
+	    g.drawRect(0,0, canvasSize.width - 1, canvasSize.height - 1);
+	}
+    }
+
+    // handle mouse clicks in the new way. just copy gordons thing for now
+    public void mouseClicked(MouseEvent event) {
+
+	// ignore actions that occur within 1 second of the last
+	Date now = new Date();
+	// System.out.println("Click time: " + now.toString());
+	if (now.getTime() < (phind.lastQueryEndTime.getTime() + 1000)) {
+	    System.out.println("Ignoring click - too close to last query.");
+	    return;
+	}
+
+	// which Item is selected?
+	int rowSelected = event.getY() / lineSpacing;
+	itemSelected = rowSelected + firstItemDisplayed;
+	ResultItem item = (ResultItem) items.elementAt(itemSelected);
+
+	if (itemSelected <= numberOfItems) {
+	    
+	    //User clicks on a phrase
+	    if (item.isPhrase()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+		
+		// If meta key is held down, send query to search engine
+		/*if (event.metaDown() && !phind.library_address.equals("")) {
+		    String address = phind.library_address  
+			+ "a=q&sa=text&c=" + phind.collection
+			+ "&q=%22" + item.toString().replace(' ', '+') + "%22";
+		    phind.displayWebPage(address, phind.searchWindowName);
+		    }*/
+	    } 
+	    
+	    // Click on a thesaurus link
+	    else if (item.isLink()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+		
+		// If meta key is held down, send query to search engine
+		/*if (event.metaDown() && !phind.library_address.equals("")) {
+		    String address = phind.library_address  
+			+ "a=q&sa=text&c=" + phind.collection
+			+ "&q=%22" + item.toString().replace(' ', '+') + "%22";
+		    phind.displayWebPage(address, phind.searchWindowName);
+		    }*/
+	    } 
+	    
+	    // The user clicks on a URL; display it.
+	    else if (item.isDocument()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		
+		String address = phind.library_address 
+		    + "a=d&c=" + phind.collection 
+		    + "&d=" + item.hiddenText()
+		    + "&q=" + parent.searchPhrase.replace(' ', '+');
+		
+		phind.displayWebPage(address, phind.documentWindowName);
+	    }
+	    
+	    // When the user clicks on "get more phrases" or other marker,
+	    // we have to send a new query to the host
+	    else if (item.isMorePhrases()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 3);
+	    } else if (item.isMoreDocuments()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 4);
+	    } else if (item.isMoreLinks()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 5);
+		
+		
+	    }
+	    repaint();
+	}
+    }
+    
+
+    // User interaction
+    //
+    // All interaction with the ResultCanvas is therough mouse clicks, and
+    // is handles in this method.  Note we ignore clicks that follow
+    // another too closely to avoid problems with slow connections.
+    /*  public boolean processEvent(Event event) {
+    
+	if (event.id == Event.MOUSE_UP) {
+
+	    // ignore actions that occur within 1 second of the last
+	    Date now = new Date();
+	    // System.out.println("Click time: " + now.toString());
+	    if (now.getTime() < (phind.lastQueryEndTime.getTime() + 1000)) {
+		System.out.println("Ignoring click - too close to last query.");
+		return true;
+	    }
+
+	    // which Item is selected?
+	    int rowSelected = event.y / lineSpacing;
+	    itemSelected = rowSelected + firstItemDisplayed;
+	    ResultItem item = (ResultItem) items.elementAt(itemSelected);
+
+	    if (itemSelected <= numberOfItems) {
+	
+		//User clicks on a phrase
+		if (item.isPhrase()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+		    parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+
+		    // If meta key is held down, send query to search engine
+		    if (event.metaDown() && !phind.library_address.equals("")) {
+			String address = phind.library_address  
+			    + "a=q&sa=text&c=" + phind.collection
+			    + "&q=%22" + item.toString().replace(' ', '+') + "%22";
+			phind.displayWebPage(address, phind.searchWindowName);
+		    }
+		} 
+
+		// Click on a thesaurus link
+		else if (item.isLink()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+		    parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+
+		    // If meta key is held down, send query to search engine
+		    if (event.metaDown() && !phind.library_address.equals("")) {
+			String address = phind.library_address  
+			    + "a=q&sa=text&c=" + phind.collection
+			    + "&q=%22" + item.toString().replace(' ', '+') + "%22";
+			phind.displayWebPage(address, phind.searchWindowName);
+		    }
+		} 
+
+		// The user clicks on a URL; display it.
+		else if (item.isDocument()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+
+		    String address = phind.library_address 
+			+ "a=r&c=" + phind.collection 
+			+ "&r=" + item.hiddenText()
+			+ "&q=" + parent.searchPhrase.replace(' ', '+');
+
+		    phind.displayWebPage(address, phind.documentWindowName);
+		}
+		 
+		// When the user clicks on "get more phrases" or other marker,
+		// we have to send a new query to the host
+		else if (item.isMorePhrases()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 3);
+		} else if (item.isMoreDocuments()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 4);
+		} else if (item.isMoreLinks()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 5);
+
+
+		}
+		repaint();
+	    }
+	}
+	return true;
+	} */   
+  
+}
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultDisplay.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultDisplay.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultDisplay.java	(revision 21112)
@@ -0,0 +1,126 @@
+/**********************************************************************
+ *
+ * ResultDisplay.java -- Phrase list holder for Phind java applet
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A ResultDisplay is a Panel that can display a single Component, which will
+be a ResultBox or PhindTitle object.
+
+ResultBoxes can be "chained" together with their "prev" and "next" fields,
+allowing us to have an arbitrary number of "displays" in which to show
+phrase lists.
+
+Main methods:
+* emptyContents will remove the component and emptyAll the "next" display
+* display will add an item to the first (ie: most "prev") available box
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Panel;
+import java.awt.BorderLayout;
+import java.awt.Component;
+
+class ResultDisplay extends Panel {
+
+  Phind phind;
+  ResultDisplay prev, next;
+  Component current;
+  boolean empty;
+
+  // Create a new Display
+  ResultDisplay(Phind ph, ResultDisplay pr) {
+    phind = ph;
+    prev = pr;
+    next = null;
+    if (prev != null) prev.next = this;
+    setLayout(new BorderLayout());
+
+    empty = true;
+    current = new PhindTitle(ph);
+    add(current);
+  }
+
+  // Empty the contents of the display,
+  // and all the next displays
+  void emptyContents() {
+    if (empty == false) {
+      empty = true;
+      current = new PhindTitle(phind);
+      removeAll();
+      add(current);
+    }
+    if (next != null) next.emptyContents();
+  }
+
+  // Set the contents of this display
+  Component setContents( Component r ) {
+    removeAll();
+    add("Center", r);
+    current = r;
+    empty = false;
+    if (r.getClass().getName().equals("ResultBox")) {
+      ((ResultBox) r).display = this;
+    }
+    return r;
+  }
+
+
+  
+  // Display a component in the "most previous" display
+  ResultDisplay display( Component r ) {
+
+    if (prev == null) {
+      // if there is no previous box, add the component here
+      setContents(r);
+      return this;
+
+    } else if (prev.empty == true) {
+      // if the previous box is empty, add the component to it instead
+      return prev.display(r);
+
+    } else if (empty == true) {
+      // if this box is empty, add the compnent to it
+      setContents(r);
+      return this;
+
+    } else {
+      // if this box is occupied,
+      // shift it's contents prev-wards and add here.
+      prev.display(current);
+      setContents(r);
+      return this;
+    }
+  }
+
+}
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItem.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItem.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItem.java	(revision 21112)
@@ -0,0 +1,128 @@
+/**********************************************************************
+ *
+ * ResultItemDocument.java -- a result in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A result item holds the data describing a single line returned from a phind
+query.  The complex functionality of phrase and URL items is implemented in
+subclasses; the base class handles simpler cases like the "get more phrases" 
+marker.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItem {
+
+    // A text string describing the item
+    String text = "Generic ResultItem";
+
+    // There are several kinds of ResultItem, identified by their
+    // "kind" field, which has an integer value from the list below.
+    final static int message = 8;
+    final static int linkItem = 7;
+    final static int moreLinks = 6;
+    final static int phraseItem = 5;
+    final static int morePhrases = 4;
+    final static int documentItem = 3;
+    final static int moreDocuments = 2;
+    final static int unknownResultItem = 1;
+
+    // The kind of the item
+    int kind = unknownResultItem;
+
+    // When results are displayed, they are loosely sorted by their
+    // kind.  The user can change the sort order numbers with the 
+    // ResultOrder parameter in Phind.java.
+    static int sortMessage = 8;
+    static int sortLinkItem = 7;
+    static int sortMoreLinks = 6;
+    static int sortPhraseItem = 5;
+    static int sortMorePhrases = 4;
+    static int sortDocumentItem = 3;
+    static int sortMoreDocuments = 2;
+    static int sortUnknownResultItem = 1;
+
+    // the primary sort key
+    int sort = 0;
+    
+    // The frequency of the item (secondary sort key)
+    int frequency = 0;
+
+
+    // Create a blank ResultItem
+    ResultItem() {
+	text = "Unknown result type";
+	kind = unknownResultItem;
+	frequency = 0;
+    }
+
+    // Create a new ResultItem of some given kind
+    ResultItem(int newKind) {
+	kind = newKind;
+	frequency = 0;
+
+	if (kind == moreLinks) {
+	    text = "get more thesaurus links";
+	    sort = sortMoreLinks;
+	} else if (kind == morePhrases) {
+	    text = "get more phrases";
+	    sort = sortMorePhrases;
+	} else if (kind == moreDocuments) {
+	    text = "get more documents";
+	    sort = sortMoreDocuments;
+	} else {
+	    text = "Unknown result type";
+	    sort = sortUnknownResultItem;
+	}
+    }
+
+    // Test the type of a ResultItem
+    public boolean isLink() { return false; }
+    public boolean isMoreLinks() { return (kind == moreLinks); }
+    public boolean isPhrase() { return false; }
+    public boolean isMorePhrases() { return (kind == morePhrases); }
+    public boolean isDocument() { return false; }
+    public boolean isMoreDocuments() { return (kind == moreDocuments); }
+
+    // Return the bare text of the item
+    public String toString() { return(text.trim()); }
+
+    // Return the text of the item components
+    public String mainText() { return text; }
+    public String freqText() { 
+	if (frequency > 0) { return Integer.toString(frequency); } 
+	else { return ""; }
+    }
+    public String docsText() { return ""; }
+    public String prefixText() { return ""; }
+    public String suffixText() { return ""; }
+    public String hiddenText() { return ""; }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemDocument.java	(revision 21112)
@@ -0,0 +1,70 @@
+/**********************************************************************
+ *
+ * ResultItemDocument.java -- a document result in the Phind applet
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a document in which a search term 
+occurs.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItemDocument extends ResultItem {
+
+    // The document's title is stored in the superclasses 
+    // text variable, and the number of times the phrase occurs
+    // in the frequency variable. 
+
+    // The document's unique object identifier (hash value).
+    String hash;
+
+    // Create a ResultItem for a document
+    ResultItemDocument(String newHash, String newTitle, String newFrequency) {
+	
+	kind = documentItem;
+	text = newTitle;
+
+	sort = sortDocumentItem;
+	frequency = Integer.valueOf(newFrequency).intValue();
+	hash = newHash;
+    }
+
+    // Is this item a document?
+    public boolean isDocument() { return true; }
+
+    // Return the bare text of the item
+    public String toString() { return(text); }
+
+    // Return the text of the item components
+    public String hiddenText() { return hash; }
+ 
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemLink.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemLink.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemLink.java	(revision 21112)
@@ -0,0 +1,88 @@
+/**********************************************************************
+ *
+ * ResultItemLink.java -- a thesaurus link in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a thesaurus link.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItemLink extends ResultItem {
+
+    // The phrase number, stored as a string and as a number
+    String id;
+    int symbol;
+
+    // The text of the link is stored in the inherited text field.
+
+    // The type of thesaurus link
+    String linkType;
+
+    // The total frequency is stored in the inherited frequency field.
+
+    // The document frequency of the thesaurus item
+    int documentFrequency;
+
+    // Create a ResultItem for a thesaurus link
+    ResultItemLink(String newId, String newText, String newLinkType, 
+		   String tf, String df) { 
+
+	kind = linkItem;
+	id = newId;
+	symbol = Integer.valueOf(newId).intValue();
+	text = newText.trim();
+	sort = sortLinkItem;
+	linkType = newLinkType;
+	frequency = Integer.valueOf(tf).intValue();
+	documentFrequency = Integer.valueOf(df).intValue();
+    }
+
+    // Is this item a phrase?
+    public boolean isLink() { return true; }
+
+    // Return the text of the item components
+    public String hiddenText() { return id; }
+    public String docsText() { 
+	if (documentFrequency <= 0) { return ""; }
+	else { return Integer.toString(documentFrequency); }
+    }
+    public String prefixText() { 
+	if (linkType.equals("RT")) { return "Related term"; }
+	else if (linkType.equals("BT")) { return "Broader term"; }
+	else if (linkType.equals("NT")) { return "Narrower term"; }
+	else if (linkType.equals("USE")) { return "Use"; }
+	else if (linkType.startsWith("UF")) { return "Used for"; }
+	else { return linkType; }
+    }
+}
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemPhrase.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemPhrase.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultItemPhrase.java	(revision 21112)
@@ -0,0 +1,85 @@
+/**********************************************************************
+ *
+ * ResultItemPhrase.java -- a phrase result in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a phrase.
+
+**********************************************************************/
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+public class ResultItemPhrase extends ResultItem {
+
+    // The phrase number, stored as a string and as a number
+    String id;
+    int symbol;
+
+    // The text of the phrase is split into a prefix, body, and suffix.
+    // The body is the same as the parent phrase (or search term) and 
+    // the prefix and suffix are the surrounding text.
+    String prefix;
+    String body;
+    String suffix;
+
+    // The document frequency of the phrase
+    int documentFrequency;
+
+    // Create a ResultItem for a phrase
+    ResultItemPhrase(String newId, String tf, String df, 
+		     String p, String b, String s) {
+
+	kind = phraseItem;
+
+	id = newId;
+	symbol = Integer.valueOf(newId).intValue();
+
+	sort = sortPhraseItem;
+	frequency = Integer.valueOf(tf).intValue();
+	documentFrequency = Integer.valueOf(df).intValue();
+
+	prefix = p;
+	body = b;
+	suffix = s;
+	text = (prefix + " " + body + " " + suffix);
+
+    }
+
+    // Is this item a phrase?
+    public boolean isPhrase() { return true; }
+
+    // Return the text of the item components
+    public String mainText() { return body; }
+    public String docsText() { return Integer.toString(documentFrequency); }
+    public String prefixText() { return prefix; }
+    public String suffixText() { return suffix; }
+    public String hiddenText() { return id; }
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultTitle.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultTitle.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/ResultTitle.java	(revision 21112)
@@ -0,0 +1,162 @@
+/**********************************************************************
+ *
+ * ResultTitle.java -- describe the contents of a ResultBox
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+ResultTitle is part of the result list displayed to the user.  It lists the
+phrase in the Result box, the number of expansions it occurs in, and the
+number of documents it occurs in.
+
+**********************************************************************/
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+import java.awt.Canvas;
+import java.awt.Graphics;
+import java.awt.Font;
+import java.awt.FontMetrics;
+import java.awt.Dimension;
+import java.awt.Color;
+
+
+public class ResultTitle extends Canvas {
+    
+    // Other objects associated with this 
+    Phind phind;
+    ResultBox parent;
+    ResultCanvas canvas;
+
+    // fonts and font spacings to use
+    Font plain, bold;
+    Graphics g;
+    
+    // Create a ResultTitle from the ResultBox which is its parent.
+    ResultTitle(ResultBox p) {
+	
+	parent = p;
+	phind = p.phind;
+	canvas = p.c;
+	g = getGraphics();
+	
+	plain = phind.plainFont;
+	bold  = phind.boldFont;
+	
+	// Note: the size() and resize() methods are deprecated, but the newer
+	// getSize() and setSize() methods fail in NetScape 4
+	Dimension d = getSize();
+	d.height = phind.fontSize + 10;
+	setSize(d);
+	
+	
+    }
+    
+    public void update(Graphics g) {
+	paint(g);
+    }
+    
+    public void paint(Graphics g) {
+	
+	// calculate the canvas size, margins, and spacing
+	Dimension canvasSize = getSize();
+	
+	g.setFont(plain);
+	int margin = g.getFontMetrics().stringWidth(" 8888 ");
+	int y = phind.fontSize + 5;
+	
+	int rightMargin = canvas.getSize().width;
+	int secondColumn = rightMargin - margin;
+	int firstColumn = secondColumn - margin;
+	
+	g.setColor(Color.white);
+	g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	g.setColor(Color.black);
+    
+    
+	// What is the phrase we are searching for?
+	String phrase = parent.searchPhrase.replace('+', ' ');
+	
+	// Construct the description phrase
+	String links = "";
+	if (parent.numberOfThesaurusLinks <= 0) {
+	    links = "";
+	} else if (parent.numberOfThesaurusLinks == 1) { 
+	    links = "1 link";
+	} else if (parent.thesaurusLinksRetrieved == parent.numberOfThesaurusLinks) {
+	    links = parent.numberOfThesaurusLinks + " links";
+	} else {
+	    links = parent.thesaurusLinksRetrieved + " of " 
+		+ parent.numberOfThesaurusLinks + " links";
+	}
+	
+	String expansions = "";
+	if (parent.numberOfExpansions <= 0) {
+	    expansions = "no phrases";
+	} else if (parent.numberOfExpansions == 1) { 
+	    expansions = "1 phrase";
+	} else if (parent.expansionsRetrieved == parent.numberOfExpansions) {
+	    expansions = parent.numberOfExpansions + " phrases";
+	} else {
+	    expansions = parent.expansionsRetrieved + " of " 
+		+ parent.numberOfExpansions + " phrases";
+	}
+	
+	String documents = "";
+	if (parent.numberOfDocuments <= 0) { 
+	    documents = "no documents";
+	} else if (parent.documentsRetrieved == 1) {
+	    documents = "1 document";
+	} else if (parent.documentsRetrieved == parent.numberOfDocuments) {
+	    documents = parent.numberOfDocuments + " documents";
+	} else {
+	    documents = parent.documentsRetrieved + " of " 
+		+ parent.numberOfDocuments + " documents";
+	}
+	
+	String status = "(";
+	if (parent.numberOfThesaurusLinks > 0) {
+	    status = status + links + ", ";
+	} 
+	status = status + expansions + ", " + documents + ")";
+	
+	
+	// Draw the text 
+	g.setFont(bold);
+	g.drawString(phrase, 0, y);
+	int tab = g.getFontMetrics().stringWidth(phrase + "  ");
+	
+	g.setFont(plain);
+	g.drawString(status, tab, y);
+	tab = tab + g.getFontMetrics().stringWidth(status);
+	
+	if (tab < firstColumn) {
+	    g.drawString("docs", firstColumn, y);
+	    g.drawString("freq", secondColumn, y);
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/URLUTF8Encoder.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/URLUTF8Encoder.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/applet/phind/URLUTF8Encoder.java	(revision 21112)
@@ -0,0 +1,159 @@
+/**********************************************************************
+ *
+ *
+ * URLUTF8Encoder.java -- encode UTF8 String into a URL form
+ *
+ * W3C SOFTWARE NOTICE AND LICENSE
+ *
+ * Copyright (C) 1994-2000 World Wide Web Consortium, 
+ * (Massachusetts Institute of Technology, Institut National de 
+ * Recherche en Informatique et en Automatique, Keio University). 
+ * All Rights Reserved.  http://www.w3.org/Consortium/Legal/
+ * 
+ * This W3C work (including software, documents, or other related items) is
+ * being provided by the copyright holders under the following license. By
+ * obtaining, using and/or copying this work, you (the licensee) agree that
+ * you have read, understood, and will comply with the following terms and
+ * conditions:
+ * 
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation, with or without modification, for any purpose and without
+ * fee or royalty is hereby granted, provided that you include the following
+ * on ALL copies of the software and documentation or portions thereof,
+ * including modifications, that you make:
+ * 
+ *   1. The full text of this NOTICE in a location viewable to users of the
+ *      redistributed or derivative work.
+ * 
+ *   2. Any pre-existing intellectual property disclaimers, notices, or terms
+ *      and conditions. If none exist, a short notice of the following form
+ *      (hypertext is preferred, text is permitted) should be used within the
+ *      body of any redistributed or derivative code: "Copyright (C)
+ *      [$date-of-software] World Wide Web Consortium, (Massachusetts
+ *      Institute of Technology, Institut National de Recherche en
+ *      Informatique et en Automatique, Keio University). All Rights
+ *      Reserved. http://www.w3.org/Consortium/Legal/"
+ * 
+ *   3. Notice of any changes or modifications to the W3C files, including the
+ *      date changes were made. (We recommend you provide URIs to the location
+ *      from which the code is derived.)
+ * 
+ * THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS
+ * MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT
+ * NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR
+ * PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE
+ * ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS.
+ * 
+ * COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR
+ * CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR
+ * DOCUMENTATION.
+ * 
+ * The name and trademarks of copyright holders may NOT be used in advertising
+ * or publicity pertaining to the software without specific, written prior
+ * permission. Title to copyright in this software and any associated
+ * documentation will at all times remain with copyright holders.
+ *
+ *********************************************************************/
+
+/**
+ * Provides a method to encode any string into a URL-safe
+ * form, the so-called "x-www-form-urlencoded" form.
+ * Non-ASCII characters are first encoded as sequences of
+ * two or three bytes, using the UTF-8 algorithm, before being
+ * encoded in "x-www-form-urlencoded".
+ */
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+public class URLUTF8Encoder
+{
+
+  final static String[] hex = {
+    "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07",
+    "%08", "%09", "%0a", "%0b", "%0c", "%0d", "%0e", "%0f",
+    "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17",
+    "%18", "%19", "%1a", "%1b", "%1c", "%1d", "%1e", "%1f",
+    "%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27",
+    "%28", "%29", "%2a", "%2b", "%2c", "%2d", "%2e", "%2f",
+    "%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37",
+    "%38", "%39", "%3a", "%3b", "%3c", "%3d", "%3e", "%3f",
+    "%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47",
+    "%48", "%49", "%4a", "%4b", "%4c", "%4d", "%4e", "%4f",
+    "%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57",
+    "%58", "%59", "%5a", "%5b", "%5c", "%5d", "%5e", "%5f",
+    "%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67",
+    "%68", "%69", "%6a", "%6b", "%6c", "%6d", "%6e", "%6f",
+    "%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77",
+    "%78", "%79", "%7a", "%7b", "%7c", "%7d", "%7e", "%7f",
+    "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87",
+    "%88", "%89", "%8a", "%8b", "%8c", "%8d", "%8e", "%8f",
+    "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97",
+    "%98", "%99", "%9a", "%9b", "%9c", "%9d", "%9e", "%9f",
+    "%a0", "%a1", "%a2", "%a3", "%a4", "%a5", "%a6", "%a7",
+    "%a8", "%a9", "%aa", "%ab", "%ac", "%ad", "%ae", "%af",
+    "%b0", "%b1", "%b2", "%b3", "%b4", "%b5", "%b6", "%b7",
+    "%b8", "%b9", "%ba", "%bb", "%bc", "%bd", "%be", "%bf",
+    "%c0", "%c1", "%c2", "%c3", "%c4", "%c5", "%c6", "%c7",
+    "%c8", "%c9", "%ca", "%cb", "%cc", "%cd", "%ce", "%cf",
+    "%d0", "%d1", "%d2", "%d3", "%d4", "%d5", "%d6", "%d7",
+    "%d8", "%d9", "%da", "%db", "%dc", "%dd", "%de", "%df",
+    "%e0", "%e1", "%e2", "%e3", "%e4", "%e5", "%e6", "%e7",
+    "%e8", "%e9", "%ea", "%eb", "%ec", "%ed", "%ee", "%ef",
+    "%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
+    "%f8", "%f9", "%fa", "%fb", "%fc", "%fd", "%fe", "%ff"
+  };
+
+  /**
+   * Encode a string to the "x-www-form-urlencoded" form, enhanced
+   * with the UTF-8-in-URL proposal. This is what happens:
+   *
+   * <ul>
+   * <li><p>The ASCII characters 'a' through 'z', 'A' through 'Z',
+   *        and '0' through '9' remain the same.
+   *
+   * <li><p>The space character ' ' is converted into a plus sign '+'.
+   *
+   * <li><p>All other ASCII characters are converted into the
+   *        3-character string "%xy", where xy is
+   *        the two-digit hexadecimal representation of the character
+   *        code
+   *
+   * <li><p>All non-ASCII characters are encoded in two steps: first
+   *        to a sequence of 2 or 3 bytes, using the UTF-8 algorithm;
+   *        secondly each of these bytes is encoded as "%xx".
+   * </ul>
+   *
+   * @param s The string to be encoded
+   * @return The encoded string
+   */
+  public static String encode(String s)
+  {
+    StringBuffer sbuf = new StringBuffer();
+    int len = s.length();
+    for (int i = 0; i < len; i++) {
+      int ch = s.charAt(i);
+      if ('A' <= ch && ch <= 'Z') {		// 'A'..'Z'
+	sbuf.append((char)ch);
+      } else if ('a' <= ch && ch <= 'z') {	// 'a'..'z'
+	sbuf.append((char)ch);
+      } else if ('0' <= ch && ch <= '9') {	// '0'..'9'
+	sbuf.append((char)ch);
+      } else if (ch == ' ') {			// space
+	sbuf.append('+');
+      } else if (ch <= 0x007f) {		// other ASCII
+	sbuf.append(hex[ch]);
+      } else if (ch <= 0x07FF) {		// non-ASCII <= 0x7FF
+	sbuf.append(hex[0xc0 | (ch >> 6)]);
+	sbuf.append(hex[0x80 | (ch & 0x3F)]);
+      } else {					// 0x7FF < ch <= 0xFFFF
+	sbuf.append(hex[0xe0 | (ch >> 12)]);
+	sbuf.append(hex[0x80 | ((ch >> 6) & 0x3F)]);
+	sbuf.append(hex[0x80 | (ch & 0x3F)]);
+      }
+    }
+    return sbuf.toString();
+  }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library1.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library1.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library1.java	(revision 21112)
@@ -0,0 +1,69 @@
+/*
+ *    Library1.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.*;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+/** sample command line interface to gsdl3 site
+ *
+ * enter XML queries at the prompt
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+final public class Library1 {
+
+    public static void main(String args[]) {
+
+	if (args.length != 1) {
+	    System.out.println("Usage: Library1 <sitename>");
+	    System.exit(1);
+	}
+	String site_name = args[0];
+
+	MessageRouter mr = new MessageRouter();
+	mr.setSiteName(site_name);
+	mr.configure();
+	
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	String query=null;
+	String result=null;
+	while (true) {
+	    System.out.println("Please enter an XML query all on one line, or 'exit' to quit");
+	    try {
+		query = br.readLine();
+	    } catch (Exception e) {
+		System.err.println("Library1 exception:"+e.getMessage());
+	    }
+	    if (query.startsWith("exit")) {
+		System.exit(1);
+	    }
+	    System.out.println("\nProcessing query");
+	    
+	    result = mr.process(query);
+
+	    System.out.println("Result:\n"+result+"\n");
+
+
+	}
+       
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library2.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library2.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/Library2.java	(revision 21112)
@@ -0,0 +1,183 @@
+/*
+ *    Library2.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.util.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Iterator;
+/**
+ * A program to take cgi args from the command line and return html
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+
+final public class Library2 {
+ 
+    protected XMLConverter converter = null;
+    protected Document doc = null;
+    
+    protected HashMap saved_args = null;
+    protected GSParams params = null;
+    protected DefaultReceptionist recept = null;
+    
+    public Library2 (){
+	this.converter = new XMLConverter();
+	this.doc = converter.newDOM();
+	this.saved_args = new HashMap();
+	this.params = new GSParams();
+	this.recept = new DefaultReceptionist();
+    }
+    
+    public void configure(String site_name, String interface_name) {
+
+	HashMap config_params = new HashMap();
+	//config_params.put(GSConstants.GSDL3_HOME, gsdl_home);
+	config_params.put(GSConstants.SITE_NAME, site_name);
+	config_params.put(GSConstants.INTERFACE_NAME, interface_name);
+	// new message router - create it and pass a handle to recept.
+	// the servlet wont use this directly
+	MessageRouter message_router = new MessageRouter();
+	
+	message_router.setSiteName(site_name);
+	message_router.configure();
+	// new receptionist
+	recept.setConfigParams(config_params);
+	recept.setMessageRouter(message_router);
+	recept.setParams(params);
+	recept.configure();
+    }
+    public static void main(String args[]) {
+
+	if (args.length != 2) {
+	    System.out.println("Usage: Library2 <site name> <interface name>");
+	    System.exit(1);
+	}
+	
+	Library2 library = new Library2();
+	library.configure(args[0], args[1]);
+	
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	String query=null;
+	String result=null;
+	while (true) {
+	    System.out.println("Please enter the  cgi args (all on one line), or 'exit' to quit (default a=p&sa=home)");
+	    try {
+		query = br.readLine();
+	    } catch (Exception e) {
+		System.err.println("Library2 exception:"+e.getMessage());
+	    }
+	    if (query.startsWith("exit")) {
+		System.exit(1);
+	    }
+
+	    result = library.process(query);
+	    
+	    System.out.println(result);
+
+
+	}
+
+    } 
+
+    protected String process(String query) {
+	Element xml_message  = generateRequest(query);
+	System.out.println("*********************");
+	System.out.println(converter.getPrettyString(xml_message));
+	Element xml_result = recept.process(xml_message);
+	return this.converter.getPrettyString(xml_result);
+    }
+    protected Element generateRequest(String cgiargs) {
+
+	Element xml_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element xml_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PAGE, "", "", "");
+	xml_message.appendChild(xml_request);
+	Element xml_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	xml_request.appendChild(xml_param_list);
+
+	// the defaults
+	String action = "p";
+	String subaction = "home";
+	String lang = (String)saved_args.get(GSParams.LANGUAGE);
+	if (lang == null) {
+	    lang = "en";
+	}
+	String output = "html";
+	
+	String args [] = cgiargs.split("&");
+	for (int i=0; i<args.length; i++) {
+	    String arg = args[i];
+	    int pos = arg.indexOf('=');
+	    if (pos == -1) continue;
+	    String name = arg.substring(0, pos);
+	    String value = arg.substring(pos+1);
+	    if (name.equals(GSParams.ACTION)) {
+		action = value;
+	    } else if (name.equals(GSParams.SUBACTION)) {
+		subaction = (value);
+	    } else if (name.equals(GSParams.LANGUAGE)) {
+		lang = value;
+		saved_args.put(name, value);
+	    } else if (name.equals(GSParams.OUTPUT)) {
+		output = value;
+	    } else if (params.shouldSave(name)) {
+		saved_args.put(name, value);
+	    } else {
+		// make a param now
+		Element param = doc.createElement(GSXML.PARAM_ELEM);
+		param.setAttribute(GSXML.NAME_ATT, name);
+		param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(value));
+		xml_param_list.appendChild(param);
+	    }
+	}
+
+	xml_request.setAttribute(GSXML.OUTPUT_ATT, output);
+	xml_request.setAttribute(GSXML.ACTION_ATT, action);
+	xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
+	xml_request.setAttribute(GSXML.LANG_ATT, lang);
+
+
+	// put in all the params from the session cache
+	Set params = saved_args.keySet();
+	Iterator i = params.iterator();
+	while (i.hasNext()) {
+	    String name = (String)i.next();
+	    if (name.equals(GSParams.LANGUAGE)) continue;
+	    Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param.setAttribute(GSXML.NAME_ATT, name);
+	    param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe((String)saved_args.get(name)));
+	    xml_param_list.appendChild(param);
+	}
+
+	return xml_message;
+    }
+}
+
+    
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/LibraryServlet.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/LibraryServlet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/LibraryServlet.java	(revision 21112)
@@ -0,0 +1,293 @@
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.PageAction; // used to get the default action
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import java.io.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.io.File;
+
+/** a servlet to serve the greenstone library - we are using servlets instead
+ * of cgi
+ * the init method is called only once - the first time the servlet classes
+ * are loaded. Each time a request comes in to the servlet, the session()
+ * method is called in a new thread (calls doGet/doPut etc)
+ * takes the a=p&p=home type args and builds a simple request to send to 
+ * its receptionist, which returns a result in html, cos output=html
+ * is set in the request
+ * @see Receptionist
+ */
+public class LibraryServlet extends HttpServlet {
+    
+    /** the receptionist to send messages to */
+    protected Receptionist recept=null;
+    /** the default language - is specified by setting a servlet param,
+     * otherwise DEFAULT_LANG is used*/
+    protected String default_lang= null;
+    /** The default default - used if a default lang is not specified 
+     * in the servlet params */
+    protected final String DEFAULT_LANG = "en";
+    /** container Document to create XML Nodes */
+    protected Document doc=null;    
+    /** a converter class to parse XML and create Docs */
+    protected XMLConverter converter=null;
+    /** the cgi stuff - the Receptionist can add new args to this
+     * its used by the servlet to determine what args to save */
+    //protected GSCGI cgi = null;
+    protected GSParams params = null;
+
+    /** user id - new one per session. This doesn't work if session state is saved between restarts - this requires this value to be saved too. */
+    protected int next_user_id = 0;
+
+    
+    /** initialise the servlet
+     */
+    public void init(ServletConfig config) throws ServletException {
+	// always call super.init;
+	super.init(config);
+	// disable preferences - does this work anyway??
+	//System.setProperty("java.util.prefs.PreferencesFactory", "org.greenstone.gsdl3.util.DisabledPreferencesFactory");
+
+	String library_name = config.getInitParameter(GSConstants.LIBRARY_NAME);
+	String gsdl3_home = config.getInitParameter(GSConstants.GSDL3_HOME);
+	String site_name = config.getInitParameter(GSConstants.SITE_NAME);
+	String interface_name = config.getInitParameter(GSConstants.INTERFACE_NAME);
+	this.default_lang = config.getInitParameter(GSConstants.DEFAULT_LANG);
+	if (library_name == null || site_name ==null ||  interface_name ==null) {
+	    // must have this
+	    System.err.println("initialisation parameters not all set!");
+	    System.err.println(" you must have libraryname, sitename,  interfacename");
+	    System.exit(1);
+	}
+
+	if (this.default_lang == null) {
+	    // choose english
+	    this.default_lang = DEFAULT_LANG;
+	}
+	
+	HashMap config_params = new HashMap();
+	
+	config_params.put(GSConstants.LIBRARY_NAME, library_name);
+	config_params.put(GSConstants.SITE_NAME, site_name);
+	config_params.put(GSConstants.INTERFACE_NAME, interface_name);
+	
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+	
+	// now try and create MR and recpt
+	// new message router - create it and pass reference to recept.
+	// the servlet wont use this directly
+	String mr_name = (String)config.getInitParameter("messagerouter_class");
+	MessageRouter message_router = null;
+	if (mr_name == null) { // just use the normal MR
+	    message_router = new MessageRouter();
+	} else { // try the specified one
+	    try {
+		message_router = (MessageRouter)Class.forName("org.greenstone.gsdl3.core."+mr_name).newInstance();
+	    } catch (Exception e) { // cant use this new one, so use normal one
+		System.err.println("LibraryServlet configure exception when trying to use a new MessageRouter "+mr_name+": "+e.getMessage());
+		e.printStackTrace();
+		message_router = new MessageRouter();
+	    }   
+	}
+	
+	message_router.setSiteName(site_name);
+	message_router.configure();
+	
+	// the receptionist -the servlet will talk to this
+	String recept_name = (String)config.getInitParameter("receptionist_class");
+	if (recept_name == null) {
+	    this.recept = new DefaultReceptionist();
+	} else {
+	    try {
+		this.recept = (Receptionist)Class.forName("org.greenstone.gsdl3.core."+recept_name).newInstance();
+	    } catch (Exception e) { // cant use this new one, so use normal one
+		System.err.println("LibraryServlet configure exception when trying to use a new Receptionist "+recept_name+": "+e.getMessage());
+		e.printStackTrace();
+		this.recept = new DefaultReceptionist();
+	    }
+	}
+	this.recept.setConfigParams(config_params);
+	this.recept.setMessageRouter(message_router);
+	// the params arg thingy
+	
+	String params_name = (String)config.getInitParameter("params_class");
+	if (params_name == null) {
+	    this.params = new GSParams();
+	} else {
+	    try {
+		this.params = (GSParams)Class.forName("org.greenstone.gsdl3.util."+params_name).newInstance();
+	    } catch (Exception e) {
+		System.err.println("LibraryServlet configure exception when trying to use a new params thing "+params_name+": "+e.getMessage());
+		e.printStackTrace();
+		this.params = new GSParams();
+	    }
+	}
+	// pass it to the receptionist
+	this.recept.setParams(this.params);
+	this.recept.configure();
+	
+    }
+    
+    public void doGet(HttpServletRequest request,
+		      HttpServletResponse response)
+	throws ServletException, IOException {
+	
+	HttpSession session = request.getSession(true);
+	String uid = (String)session.getAttribute(GSXML.USER_ID_ATT);
+	if (uid ==null) {
+	    uid = ""+getNextUserId();
+	    session.setAttribute(GSXML.USER_ID_ATT, uid);
+	}
+	request.setCharacterEncoding("UTF-8");
+	response.setContentType("text/html;charset=UTF-8");
+	PrintWriter out = response.getWriter();
+
+	String lang = request.getParameter(GSParams.LANGUAGE);
+	if (lang==null || lang.equals("")) {
+	    // try the session cached lang
+	    lang = (String)session.getAttribute(GSParams.LANGUAGE);
+	    if (lang==null || lang.equals("")) {
+		// still not set, use the default
+		lang = this.default_lang;
+	    }
+	}
+	
+	// set the lang in the session
+	session.setAttribute(GSParams.LANGUAGE, lang);
+
+	String output = request.getParameter(GSParams.OUTPUT);
+	if (output==null || output.equals("")) {
+	    output = "html"; // uses html by default
+	}
+
+	// the request to the receptionist	
+	Element xml_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element xml_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PAGE, "", lang, uid);
+	xml_request.setAttribute(GSXML.OUTPUT_ATT, output);
+	xml_message.appendChild(xml_request);
+	
+	
+	String action = request.getParameter(GSParams.ACTION);
+	String subaction = request.getParameter(GSParams.SUBACTION);
+	if (action==null || action.equals("")) {
+	    // should we do all the following stuff if using default page?
+	    // display the home page  - the default page
+	    action = "p";
+	    subaction = PageAction.HOME_PAGE;
+	    xml_request.setAttribute(GSXML.ACTION_ATT, action);
+	    xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
+	    
+	} else {
+	
+	    xml_request.setAttribute(GSXML.ACTION_ATT, action);
+	    if (subaction != null) {
+		xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
+	    }
+
+	    //  create the param list for the greenstone request - includes
+	    // the params from the current request and any others from the saved session
+	    Element xml_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    xml_request.appendChild(xml_param_list);
+	    
+	    Enumeration params = request.getParameterNames();
+	    while(params.hasMoreElements()) {
+		String name = (String)params.nextElement();
+		if (!name.equals(GSParams.ACTION) && !name.equals(GSParams.SUBACTION) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSParams.OUTPUT)) {// we have already dealt with these 
+		    String value="";
+		    String [] values = request.getParameterValues(name);
+		    value = values[0];
+		    if (values.length > 1) {
+			for (int i=1; i< values.length; i++) {
+			    value += ","+values[i];
+			}
+		    }
+		    // either add it to the param list straight away, or save it to the session and add it later
+		    if (this.params.shouldSave(name)) {			
+			session.setAttribute(name, value);
+		    } else { 
+			Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, name);
+			param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(value));
+			xml_param_list.appendChild(param);
+		    }
+		}
+	    }
+	    
+	    // put in all the params from the session cache
+	    params = session.getAttributeNames();
+	    while(params.hasMoreElements()) {
+		String name = (String)params.nextElement();
+		
+		if ( !name.equals(GSParams.LANGUAGE) && !name.equals(GSXML.USER_ID_ATT)) { // lang and uid are stored but we dont want it in the param list cos its already in the request
+		    
+		    Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+		    param.setAttribute(GSXML.NAME_ATT, name);
+		    String value =  GSXML.xmlSafe((String)session.getAttribute(name));
+		    // ugly hack to undo : escaping
+		    value = value.replaceAll("%3A", "\\:");
+		    param.setAttribute(GSXML.VALUE_ATT,value);
+		    xml_param_list.appendChild(param);
+		}
+	    }
+
+	
+	}
+	
+	if (!output.equals("html")) {
+	    response.setContentType("text/xml"); // for now use text
+	}
+	
+	Element xml_result = this.recept.process(xml_message);
+	encodeURLs(xml_result, response);
+	out.println(this.converter.getPrettyString(xml_result));
+    }
+
+    /** this goes through each URL and adds in a session id if needed--
+     * its needed if the browser doesn't accept cookies
+     * also escapes things if needed
+     */
+    protected void encodeURLs(Element data, HttpServletResponse response) {
+	
+	if (data == null) {
+	    return;
+	}
+	// get all the <a> elements
+	NodeList hrefs = data.getElementsByTagName("a");
+	for (int i=0; i<hrefs.getLength(); i++) {
+	    Element a = (Element)hrefs.item(i);
+	    // ugly hack to get rid of : in the args - interferes with session handling
+	    String href = a.getAttribute("href");
+	    if (!href.equals("")) {
+		if (href.indexOf("?")!=-1) {
+		    String[] parts = href.split("\\?", -1);
+		    parts[1]=parts[1].replaceAll(":", "%3A");
+		    href = parts[0]+"?"+parts[1];
+		}
+		a.setAttribute("href", response.encodeURL(href));
+	    }
+	}
+	
+	// now find any submit bits - get all the <form> elements
+	NodeList forms = data.getElementsByTagName("form");
+	for (int i=0; i<forms.getLength(); i++) {
+	    Element form = (Element)forms.item(i);
+	    form.setAttribute("action", response.encodeURL(form.getAttribute("action")));
+	}
+	// are these the only cases where URLs occur??
+	// we should only do this for greenstone urls?
+	
+    }
+
+    synchronized protected int getNextUserId() {
+	next_user_id++;
+	return next_user_id;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServer.java.in
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServer.java.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServer.java.in	(revision 21112)
@@ -0,0 +1,87 @@
+/*
+ *    SOAPServer.java.in: a template for a new SOAPServer
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.w3c.dom.Element;
+import java.io.File;
+
+/**
+ * The server side of a SOAP connection
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @see <a href="http://www.w3.org/TR/SOAP/">Simple Object Access Protocol (SOAP) 1.1 </a>
+ */
+
+public class SOAPServer@sitename@
+{
+    
+    /** The message router we're talking to */
+    protected MessageRouter mr=null;
+    /** the name of the site we are serving */
+    protected String site_name = "@sitename@";
+  
+    /** The no-args constructor */
+    public SOAPServer@sitename@() {
+	String gsdl3_home = GlobalProperties.getGSDL3Home();
+	if (gsdl3_home == null || gsdl3_home.equals("")) {
+	    System.err.println("Couldn't access GSDL3Home from GlobalProerties.getGSDL3HOME, can't initialize the SOAP Server.");
+	    return;
+	}
+	String site_home = GSFile.siteHome(gsdl3_home, this.site_name);
+	
+	File site_file = new File(site_home);
+	if (!site_file.isDirectory()) {
+	    System.err.println("The site directory "+site_file.getPath()+" doesn't exist. Can't initialize the SOAP Server.");
+	    return;
+	}   	
+	mr = new MessageRouter();
+	mr.setSiteName(this.site_name);
+	mr.configure();
+    }
+    
+    
+    public Element [] process (Element [] xml_in) {
+	Element [] result = new Element[xml_in.length];
+	for (int i=0; i<xml_in.length; i++) {
+	    Element req = xml_in[i];
+	    // get rid of the obligatory namespace that axis needs
+	    String tag_name = req.getTagName();
+	    String namespace="";
+	    if (tag_name.indexOf(':')!= -1) {
+		namespace = tag_name.substring(0, tag_name.indexOf(':'));
+		tag_name = tag_name.substring(tag_name.indexOf(':')+1);	
+	    }
+	    Element new_req = GSXML.duplicateWithNewName(req.getOwnerDocument(), req, tag_name, true);
+	    Element r = mr.process(new_req);
+	    // add the namespace back on
+	    //Element new_res = r;
+	    //if (!namespace.equals("")) {
+	    //	new_res = GSXML.duplicateWithNewName(r.getOwnerDocument(), r, namespace+r.getTagName(), true);
+	    //}
+	    result[i] = r;
+	}
+	return result;
+    }   
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServerlocalsite.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServerlocalsite.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/SOAPServerlocalsite.java	(revision 21112)
@@ -0,0 +1,87 @@
+/*
+ *    SOAPServer.java.in: a template for a new SOAPServer
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.w3c.dom.Element;
+import java.io.File;
+
+/**
+ * The server side of a SOAP connection
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @see <a href="http://www.w3.org/TR/SOAP/">Simple Object Access Protocol (SOAP) 1.1 </a>
+ */
+
+public class SOAPServerlocalsite
+{
+    
+    /** The message router we're talking to */
+    protected MessageRouter mr=null;
+    /** the name of the site we are serving */
+    protected String site_name = "localsite";
+  
+    /** The no-args constructor */
+    public SOAPServerlocalsite() {
+	String gsdl3_home = GlobalProperties.getGSDL3Home();
+	if (gsdl3_home == null || gsdl3_home.equals("")) {
+	    System.err.println("Couldn't access GSDL3Home from GlobalProerties.getGSDL3HOME, can't initialize the SOAP Server.");
+	    return;
+	}
+	String site_home = GSFile.siteHome(gsdl3_home, this.site_name);
+	
+	File site_file = new File(site_home);
+	if (!site_file.isDirectory()) {
+	    System.err.println("The site directory "+site_file.getPath()+" doesn't exist. Can't initialize the SOAP Server.");
+	    return;
+	}   	
+	mr = new MessageRouter();
+	mr.setSiteName(this.site_name);
+	mr.configure();
+    }
+    
+    
+    public Element [] process (Element [] xml_in) {
+	Element [] result = new Element[xml_in.length];
+	for (int i=0; i<xml_in.length; i++) {
+	    Element req = xml_in[i];
+	    // get rid of the obligatory namespace that axis needs
+	    String tag_name = req.getTagName();
+	    String namespace="";
+	    if (tag_name.indexOf(':')!= -1) {
+		namespace = tag_name.substring(0, tag_name.indexOf(':'));
+		tag_name = tag_name.substring(tag_name.indexOf(':')+1);	
+	    }
+	    Element new_req = GSXML.duplicateWithNewName(req.getOwnerDocument(), req, tag_name, true);
+	    Element r = mr.process(new_req);
+	    // add the namespace back on
+	    //Element new_res = r;
+	    //if (!namespace.equals("")) {
+	    //	new_res = GSXML.duplicateWithNewName(r.getOwnerDocument(), r, namespace+r.getTagName(), true);
+	    //}
+	    result[i] = r;
+	}
+	return result;
+    }   
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/Action.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/Action.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/Action.java	(revision 21112)
@@ -0,0 +1,136 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document; 
+
+// other java stuff
+import java.io.File;
+import java.util.Vector;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+
+/** base class for Actions */
+abstract public class Action {
+
+    /** the system set up variables */
+    protected HashMap config_params = null;
+    /** container Document to create XML Nodes */
+    protected Document doc=null; 
+    /** a converter class to parse XML and create Docs */
+    protected XMLConverter converter=null;
+    /** a reference to the message router that it must talk to to
+     * get info. it may be a communicator acting as a proxy, but it 
+     doesn't care about that */
+    protected ModuleInterface mr=null;
+
+    public Action() {
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+    }
+    /** the config variables must be set before configure is called */
+    public void setConfigParams(HashMap params) {
+	this.config_params = params;
+    }
+    /** sets the message router */
+    public void setMessageRouter(ModuleInterface m) {
+       this.mr = m;
+    }
+    public boolean configure() {
+	// does nothing yet
+	return true;
+    }
+    
+    /** process takes an xml representation of cgi args
+     * and returns the page of results - may be in html/xml/other
+     * depending on the output att of the request */
+    public String process(String xml_in) {
+       
+	Document message_doc = this.converter.getDOM(xml_in);
+	if (message_doc == null) {
+	    System.err.println("Action.process(String) Error: Couldn't parse request");
+	    System.err.println(xml_in);
+	    return null;
+	}
+	Element result = process(message_doc.getDocumentElement());
+	return this.converter.getString(result);
+    }
+    
+    /** the main process method - must be implemented in subclass */
+    abstract public Element process(Element xml_in);
+
+    /** tell the param class what its arguments are 
+     * if an action has its own arguments, this should add them to the params
+     * object - particularly important for args that should not be saved */
+    public boolean getActionParameters(GSParams params) {
+	return true;
+    }
+
+    protected void extractMetadataNames(Element format, HashSet meta_names) {
+	//NodeList nodes = format.getElementsByTagNameNS("metadata", "http://www.greenstone.org/configformat");
+	NodeList nodes = format.getElementsByTagName("gsf:metadata");
+        for (int i=0; i<nodes.getLength(); i++) {
+	    Element elem = (Element)nodes.item(i);
+	    StringBuffer metadata = new StringBuffer();
+	    String all = elem.getAttribute("multiple");
+	    String name = elem.getAttribute("name");
+	    String select = elem.getAttribute("select");
+	    String sep = elem.getAttribute("separator");
+	    if (all.equals("true")) {
+		metadata.append("all");
+		metadata.append(GSConstants.META_RELATION_SEP);
+	    }
+	    if (!select.equals("")) {
+		metadata.append(select);
+		metadata.append(GSConstants.META_RELATION_SEP);
+	    }
+	    if (!sep.equals("")) {
+		metadata.append(GSConstants.META_SEPARATOR_SEP);
+		metadata.append(sep);
+		metadata.append(GSConstants.META_SEPARATOR_SEP);
+		metadata.append(GSConstants.META_RELATION_SEP);
+	    }
+
+	    metadata.append(name);
+	    meta_names.add(metadata.toString());
+	}
+		
+    }
+
+    protected Element createMetadataParamList(HashSet metadata_names) {
+	Element param_list  = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	
+	Element param = null;
+	Iterator i = metadata_names.iterator();
+	while (i.hasNext()) {
+	    String name = (String)i.next();
+	    param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param_list.appendChild(param);
+	    param.setAttribute(GSXML.NAME_ATT, "metadata");
+	    param.setAttribute(GSXML.VALUE_ATT, name);
+	
+	}
+	return param_list;
+    }
+
+    protected boolean processErrorElements(Element message, Element page) {
+	NodeList error_nodes = message.getElementsByTagName(GSXML.ERROR_ELEM);
+	if (error_nodes.getLength()==0) {
+	    return false;
+	}
+	Document owner = page.getOwnerDocument();
+	for (int i=0; i<error_nodes.getLength(); i++) {
+	    page.appendChild(owner.importNode(error_nodes.item(i), true));
+	}
+	return true;
+    }
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/AppletAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/AppletAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/AppletAction.java	(revision 21112)
@@ -0,0 +1,121 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.io.File;
+
+/** action class for handling applets */
+public class AppletAction extends Action {
+    
+
+    public Element process (Element message) {
+	
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.appendChild(page_response);
+
+	// get the collection and service params
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+
+	// request_type is display (d) or request (r)
+	String request_type = (String)params.get(GSParams.REQUEST_TYPE);
+	if (!request_type.equals("d")&&!request_type.equals("r")) {
+	    System.err.println("AppletAction Error: the rt arg should be either d or r, instead it was "+request_type+"!");
+	    return result;
+	}
+	
+	String collection = (String)params.get(GSParams.COLLECTION);
+	boolean coll_specified = true;
+	String service_name=(String)params.get(GSParams.SERVICE);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String to=null;
+	if (collection==null||collection.equals("") ) {
+	    coll_specified = false;
+	    to = service_name;
+	} else {
+	    to = GSPath.appendLink(collection, service_name);
+	}
+
+	if (request_type.equals("r")) {
+	    // we are processing stuff for the applet send a message to the service, type="query", and take out the something element, and return that as our result - the applet must take xml
+
+	    Element mr_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    Element mr_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid); 
+	    mr_message.appendChild(mr_request);
+	    // just append all the params for now - should filter out unneeded ones
+	    mr_request.appendChild(this.doc.importNode(cgi_param_list, true));
+       
+	    // process the request
+	    Element mr_response = (Element)this.mr.process(mr_message);
+	    // get the applet data out and pass it back as is. 
+	    String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.APPLET_DATA_ELEM);
+	    Element applet_info = (Element)GSXML.getNodeByPath(mr_response, path).getFirstChild();
+	    return applet_info;
+	    
+	}
+	
+
+	// get the applet description, and the collection info if a collection is specified
+	
+	Element mr_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element applet_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, uid); 
+	mr_message.appendChild(applet_request);
+
+	
+	Element mr_response = (Element)this.mr.process(mr_message);
+
+	// add in the applet info
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.SERVICE_ELEM);
+	//path = GSPath.appendLink(path, GSXML.APPLET_ELEM);
+	Element service_elem = (Element)this.doc.importNode(GSXML.getNodeByPath(mr_response, path), true);
+	Element applet_elem = (Element)GSXML.getChildByTagName(service_elem, GSXML.APPLET_ELEM);
+	// must handle any params that have values that are not 
+	// necessarily known by the service
+	// should this be done here or by web receptionist??
+	// cant really have an applet without web?
+	editLocalParams(applet_elem, (String)config_params.get(GSConstants.LIBRARY_NAME), collection);
+	page_response.appendChild(service_elem);
+	
+	return result;
+	
+    }
+
+    /** this method looks through the PARAMs of the applet description for 
+     * 'library' or 'collection'. If found, the params are set to  
+     * the appropriate values 
+     * should this be done here or in the receptionist? 
+     */
+    protected void editLocalParams(Element description, String library_name,
+				   String full_collection_name) {
+
+	Node child = description.getFirstChild();
+	while (child != null) {
+	    if (child.getNodeType() == Node.ELEMENT_NODE) {
+		String param = child.getNodeName();
+		if (param.equals("PARAM")||param.equals("param")) {
+		    String name = ((Element)child).getAttribute("NAME");
+		    if (name == null || name.equals("")) {
+			// somethings wrong!!
+		    } else if (name.equals("library")) {
+			((Element)child).setAttribute("VALUE", library_name);
+		    } else if (name.equals("collection")) {
+			((Element)child).setAttribute("VALUE", full_collection_name);
+		    }
+		    
+		}
+	    }
+	    child = child.getNextSibling();
+	}
+    }
+	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/BrowseAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/BrowseAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/BrowseAction.java	(revision 21112)
@@ -0,0 +1,293 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.io.File;
+
+//NOTE: this class not used at present!!!!!
+/** action for classifier browsing */
+public class BrowseAction extends Action {
+    
+    public static final String CLASSIFIER_ARG = "cl";
+    public static final String SIBLING_ARG = "sib";
+
+    /** process the request */
+    public Element process (Element message) {
+
+	// get the request - assume only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// the result
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element response = classifierBrowse(request);
+	result.appendChild(response);
+	return result;
+    }
+    
+
+    protected Element classifierBrowse(Element request) {
+
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	// extract the params from the cgi-request, and check that we have a coll specified
+	Element cgi_paramList = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_paramList, false);
+
+	String service_name = (String)params.get(GSParams.SERVICE);
+	String collection = (String)params.get(GSParams.COLLECTION);
+	if (collection == null || collection.equals("")) {
+	    System.err.println("BrowseAction Error:classifierBrowse, need to specify a collection!");
+	    return page_response;
+	   
+	}
+	
+	//whether to retrieve siblings or not
+	boolean get_siblings = false;
+	String sibs = (String) params.get(SIBLING_ARG);
+	if (sibs != null && sibs.equals("1")) {
+	    get_siblings = true;
+	}
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);	
+	String to = GSPath.appendLink(collection, service_name);
+	
+	// the first part of the response is the service description 
+	// for now get this again from the service. 
+	// this should be cached somehow later on. 
+	
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, uid); 
+	info_message.appendChild(info_request);
+	
+	// also get the format stuff now if there is some
+	Element format_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_FORMAT, to, lang, uid);
+	info_message.appendChild(format_request);
+	// process the requests
+
+	Element info_response = (Element) this.mr.process(info_message);
+
+	// the two responses
+	NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	Element service_response = (Element)responses.item(0);
+	Element format_response = (Element)responses.item(1);
+
+	Element service_description = (Element)GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM);
+	page_response.appendChild(this.doc.importNode(service_description, true));
+	
+	// if rt=d, then we are just displaying the service
+	String request_type = (String)params.get(GSParams.REQUEST_TYPE);
+	if (request_type.equals("d")) {
+	    //return the page that we have so far
+	    return page_response;
+	}
+    
+	// get the node that the user has clicked on
+	String classifier_node = (String)params.get(CLASSIFIER_ARG);
+	
+	// if the node is not defined, return the page that we have so far
+	if (classifier_node ==null || classifier_node.equals("")) {
+	    return page_response;
+	}
+	
+	// the id of the classifier is the top id of the selected node
+	String top_id = OID.getTop(classifier_node);
+	
+	HashSet metadata_names = new HashSet();
+
+	// add the format info into the response
+	Element format_elem = (Element)GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+	if (format_elem != null) {
+	    // find the one for the classifier we are in
+	    Element this_format = GSXML.getNamedElement(format_elem, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+	    if (this_format == null) {
+		this_format = (Element)GSXML.getChildByTagName(format_elem, GSXML.DEFAULT_ELEM);
+	    }
+	    if (this_format != null) {
+		Element new_format = GSXML.duplicateWithNewName(this.doc, this_format, GSXML.FORMAT_ELEM, false);
+		// set teh format type
+		new_format.setAttribute(GSXML.TYPE_ATT, "browse");
+
+		page_response.appendChild(new_format);
+		extractMetadataNames(new_format, metadata_names);
+	    }
+	}
+	
+	System.out.println("extracted meta names, "+metadata_names.toString());
+	// get the browse structure for the selected node
+	Element classify_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element classify_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	classify_message.appendChild(classify_request);
+	    
+	//Create a parameter list to specify the required structure information
+	// for now, always get ancestors and children
+	Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	classify_request.appendChild(param_list);
+	Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+	param_list.appendChild(param);
+	param.setAttribute(GSXML.NAME_ATT, "structure");
+	param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+	param = this.doc.createElement(GSXML.PARAM_ELEM);
+	param_list.appendChild(param);
+	param.setAttribute(GSXML.NAME_ATT, "structure");
+	param.setAttribute(GSXML.VALUE_ATT, "children");
+	if (get_siblings) {
+	    param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param_list.appendChild(param);
+	    param.setAttribute(GSXML.NAME_ATT, "structure");
+	    param.setAttribute(GSXML.VALUE_ATT, "siblings");
+	}
+
+	// put the classifier node into a classifier node list
+	Element classifier_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	Element classifier = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	classifier.setAttribute(GSXML.NODE_ID_ATT, classifier_node);
+	classifier_list.appendChild(classifier);
+	classify_request.appendChild(classifier_list);
+	    
+	// process the request
+	Element classify_response = (Element)this.mr.process(classify_message);
+	// get the structure element
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+	path = GSPath.appendLink(path, GSXML.NODE_STRUCTURE_ELEM);
+	// assume that we always get back the top level CL1 node - this becomes the page_classifier node
+	path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+	Element cl_structure = (Element)GSXML.getNodeByPath(classify_response,
+							    path);
+	if (cl_structure ==null) {
+	    System.err.println("BrowseAction: classifier structure request returned no structure");
+	    return page_response;
+	}   
+    
+	// add the classifier node as the page classifier 
+	Element page_classifier = GSXML.duplicateWithNewName(this.doc, cl_structure,  GSXML.CLASSIFIER_ELEM, true);
+	page_response.appendChild(page_classifier);
+	page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+	
+	// get the metadata for each classifier node, 
+	// then for each document node
+
+	Element metadata_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+
+	boolean did_classifier = false;
+	boolean did_documents = false;
+	
+    
+	// if there are classifier nodes 
+	// create a metadata request for the classifier, and add it to 
+	// the the message
+	NodeList cl_nodes = page_classifier.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+	
+	if (cl_nodes.getLength() > 0) {
+	    did_classifier = true;
+	    Element cl_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to+"MetadataRetrieve", lang, uid);
+	    metadata_message.appendChild(cl_meta_request);
+	    
+	    Element new_cl_nodes_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    cl_meta_request.appendChild(new_cl_nodes_list);
+	    
+	    for (int c=0; c<cl_nodes.getLength(); c++) {
+
+		Element cl = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+		cl.setAttribute(GSXML.NODE_ID_ATT, ((Element)cl_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+		new_cl_nodes_list.appendChild(cl);
+	    }
+
+	    // create and add in the param list - for now get all the metadata
+	    // should be based on info sent in from the recept, and the 
+	    // format stuff
+	    Element cl_param_list = null;
+	    if (metadata_names.isEmpty()) {
+		cl_param_list  = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		Element p = this.doc.createElement(GSXML.PARAM_ELEM);
+		cl_param_list.appendChild(p);
+		p.setAttribute(GSXML.NAME_ATT, "metadata");
+		p.setAttribute(GSXML.VALUE_ATT, "Title");
+	    } else {
+		cl_param_list  = createMetadataParamList(metadata_names);
+	    }
+
+	    cl_meta_request.appendChild(cl_param_list);
+	    
+	}
+
+	// if there are document nodes in the classification (happens 
+	// sometimes), create a second request for document metadata and 
+	// append to the message
+	NodeList doc_nodes = page_classifier.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	if (doc_nodes.getLength() > 0) {
+	    did_documents = true;
+	    Element doc_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, GSPath.appendLink(collection, "DocumentMetadataRetrieve"), lang, uid);
+	    metadata_message.appendChild(doc_meta_request);
+	    
+	    Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    doc_meta_request.appendChild(doc_list);
+	    
+	    for (int c=0; c<doc_nodes.getLength(); c++) {
+		
+		Element d = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		d.setAttribute(GSXML.NODE_ID_ATT, ((Element)doc_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+		doc_list.appendChild(d);
+	    }
+
+	    // create and add in the param list - add all for now
+	    Element doc_param_list  = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    Element p = this.doc.createElement(GSXML.PARAM_ELEM);
+	    doc_param_list.appendChild(p);
+	    p.setAttribute(GSXML.NAME_ATT, "metadata");
+	    p.setAttribute(GSXML.VALUE_ATT, "all");
+	    doc_meta_request.appendChild(doc_param_list);
+
+	}
+	
+	// process the metadata requests
+	Element metadata_response = (Element)this.mr.process(metadata_message);
+	if (did_classifier) {
+	    // the classifier one will be the first response
+	    // add the metadata lists for each node back into the 
+	    // page_classifier nodes
+	    path = GSPath.appendLink(GSXML.RESPONSE_ELEM, 
+				   GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    NodeList meta_response_cls = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+	    for (int i = 0; i < cl_nodes.getLength(); i++) {
+		GSXML.mergeMetadataLists(cl_nodes.item(i), meta_response_cls.item(i));
+	    }
+	}
+	if (did_documents) {
+	    NodeList meta_response_docs = null;
+	    if (!did_classifier) {
+		// its the first response
+		path = GSPath.appendLink(GSXML.RESPONSE_ELEM, 
+					 GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+		meta_response_docs = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+	    } else { // its the second response
+		meta_response_docs = GSXML.getChildByTagName(metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1), GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER).getChildNodes();
+	    }
+	    
+	    for (int i = 0; i < doc_nodes.getLength(); i++) {
+		GSXML.mergeMetadataLists(doc_nodes.item(i), meta_response_docs.item(i));
+	    }
+	}
+	    
+
+	///ystem.out.println("(BrowseAction) Page:\n" + this.converter.getPrettyString(page_response));
+	return page_response;
+    }
+
+    protected Element unknownBrowse(Element page, Element request, String browse_type) {
+	System.err.println("BrowseAction Error: unknown browse subtype: "+browse_type);
+	return null;
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/DocumentAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/DocumentAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/DocumentAction.java	(revision 21112)
@@ -0,0 +1,844 @@
+/*
+ *    DocumentAction.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.action;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.io.File;
+
+
+/** Action class for retrieving Documents  via the message router
+ */
+public class DocumentAction extends Action {
+  
+    // this is used to specify that the sibling nodes of a selected one should be obtained
+    public static final String SIBLING_ARG = "sib";
+    public static final String GOTO_PAGE_ARG = "gp";
+    public static final String ENRICH_DOC_ARG = "end";
+    
+    /** if this is set to true, when a document is displayed, any annotation
+     * type services (enrich) will be offered to the user as well */
+    protected boolean provide_annotations = false; 
+    
+    protected boolean highlight_query_terms = false;
+
+    public boolean configure() {
+	super.configure();
+	String highlight = (String)config_params.get("highlightQueryTerms");
+	if (highlight != null && highlight.equals("true")) {
+	    highlight_query_terms = true;
+	}
+	String annotate = (String)config_params.get("displayAnnotationService");
+	if (annotate != null && annotate.equals("true")) {
+	    provide_annotations = true;
+	}
+	return true;
+    }
+    public Element process (Element message)
+    {
+	// for now, no subaction eventually we may want to have subactions such as text assoc or something ?
+	
+	// the response
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.appendChild(page_response);
+
+	// get the request - assume only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	Element cgi_paramList = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_paramList, false);
+
+	// just in case there are some that need to get passed to the services
+	HashMap service_params = (HashMap)params.get("s0"); 
+
+	String collection = (String) params.get(GSParams.COLLECTION);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String document_name = (String) params.get(GSParams.DOCUMENT);
+	if (document_name == null || document_name.equals("")) {
+	    System.err.println("DocumentAction Error: no document specified!");
+	    return result;
+	}
+	String document_type = (String) params.get(GSParams.DOCUMENT_TYPE);
+	if (document_type == null) {
+	    document_type = "simple";
+	}
+	//whether to retrieve siblings or not
+	boolean get_siblings = false;
+	String sibs = (String) params.get(SIBLING_ARG);
+	if (sibs != null && sibs.equals("1")) {
+	    get_siblings = true;
+	}
+	
+	String sibling_num = (String) params.get(GOTO_PAGE_ARG);
+	if (sibling_num != null && !sibling_num.equals("")) {
+	    // we have to modify the doc name
+	    document_name = document_name+"."+sibling_num+".ss";
+	}
+	
+	boolean expand_document = false;
+	String ed_arg = (String) params.get(GSParams.EXPAND_DOCUMENT);
+	if (ed_arg != null && ed_arg.equals("1")) {
+	    expand_document = true;
+	}
+       
+
+	boolean expand_contents = false;
+	if (expand_document) { // we always expand the contents with the text
+	    expand_contents = true;
+	} else {
+	    String ec_arg = (String) params.get(GSParams.EXPAND_CONTENTS);
+	    if (ec_arg != null && ec_arg.equals("1")) {
+		expand_contents = true;
+	    }
+	}
+	// get the additional data needed for the page
+	getBackgroundData(page_response, collection, lang, uid);
+	Element format_elem = (Element)GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);
+	
+	// the_document is where all the doc info - structure and metadata etc
+	// is added into, to be returned in the page
+	Element the_document = this.doc.createElement(GSXML.DOCUMENT_ELEM);
+	page_response.appendChild(the_document);
+
+	// set the doctype from the cgi arg as an attribute
+	the_document.setAttribute(GSXML.DOC_TYPE_ATT, document_type);
+
+	// create a basic doc list containing the current node
+	Element basic_doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	Element current_doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	basic_doc_list.appendChild(current_doc);
+	current_doc.setAttribute(GSXML.NODE_ID_ATT, document_name);
+	
+	// Create a parameter list to specify the required structure information
+	Element ds_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	
+	if (service_params != null) {
+	    GSXML.addParametersToList(this.doc, ds_param_list, service_params);
+	}
+
+	Element ds_param = null; 
+	boolean get_structure = false;
+	boolean get_structure_info = false;
+	if (document_type.equals("paged")) {
+	    get_structure_info = true;
+	    // get teh info needed for paged naviagtion
+	    ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    ds_param_list.appendChild(ds_param);
+	    ds_param.setAttribute(GSXML.NAME_ATT, "info");
+	    ds_param.setAttribute(GSXML.VALUE_ATT, "numSiblings");
+	    ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    ds_param_list.appendChild(ds_param);
+	    ds_param.setAttribute(GSXML.NAME_ATT, "info");
+	    ds_param.setAttribute(GSXML.VALUE_ATT, "numChildren");
+	    ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    ds_param_list.appendChild(ds_param);
+	    ds_param.setAttribute(GSXML.NAME_ATT, "info");
+	    ds_param.setAttribute(GSXML.VALUE_ATT, "siblingPosition");
+	    
+	} else if (document_type.equals("hierarchy")){
+	    get_structure = true;
+	    if (expand_contents) {
+		ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+		ds_param_list.appendChild(ds_param);
+		ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+		ds_param.setAttribute(GSXML.VALUE_ATT, "entire");
+	    } else {
+		// get the info needed for table of contents
+		ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+		ds_param_list.appendChild(ds_param);
+		ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+		ds_param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+		ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+		ds_param_list.appendChild(ds_param);
+		ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+		ds_param.setAttribute(GSXML.VALUE_ATT, "children");
+		if (get_siblings) {
+		    ds_param = this.doc.createElement(GSXML.PARAM_ELEM);
+		    ds_param_list.appendChild(ds_param);
+		    ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+		    ds_param.setAttribute(GSXML.VALUE_ATT, "siblings");
+		}
+	    }
+	} else {
+	    // we dont need any structure
+	}
+
+	boolean has_dummy = false;
+	if (get_structure || get_structure_info) {
+
+	    // Build a request to obtain the document structure
+	    Element ds_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    String to = GSPath.appendLink(collection, "DocumentStructureRetrieve");// Hard-wired?
+	    Element ds_request = GSXML.createBasicRequest(this.doc,GSXML.REQUEST_TYPE_PROCESS, to, lang, uid); 
+	    ds_message.appendChild(ds_request);
+	    ds_request.appendChild(ds_param_list);
+	    
+	    // create a doc_node_list and put in the doc_node that we are interested in
+	    ds_request.appendChild(basic_doc_list);
+	    
+	    // Process the document structure retrieve message
+	    Element ds_response_message = (Element) this.mr.process(ds_message);
+	    if (processErrorElements(ds_response_message, page_response)) {
+		return result;
+	    }
+
+	    // get the info and print out
+	    String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+	    path = GSPath.appendLink(path, "nodeStructureInfo");
+	    Element ds_response_struct_info = (Element) GSXML.getNodeByPath(ds_response_message, path);
+	    // get the doc_node bit 
+	    if (ds_response_struct_info != null) {
+		the_document.appendChild(this.doc.importNode(ds_response_struct_info, true)); 
+	    }
+	    path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+	    path = GSPath.appendLink(path, GSXML.NODE_STRUCTURE_ELEM);
+	    Element ds_response_structure = (Element) GSXML.getNodeByPath(ds_response_message, path);
+	    
+	    if (ds_response_structure != null) {
+		// add the contents of the structure bit into the_document
+		NodeList structs = ds_response_structure.getChildNodes();
+		for (int i=0; i<structs.getLength();i++) {
+		    the_document.appendChild(this.doc.importNode(structs.item(i), true));
+		}
+	    } else {
+		// no structure nodes, so put in a dummy doc node
+		Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		doc_node.setAttribute(GSXML.NODE_ID_ATT, document_name);
+		the_document.appendChild(doc_node);
+		has_dummy = true;
+	    }
+	} else { // a simple type - we dont have a dummy node for simple
+	    // should think about this more
+	    // no structure request, so just put in a dummy doc node
+	    Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, document_name);
+	    the_document.appendChild(doc_node);
+	    has_dummy = true;
+	}
+	
+	// Build a request to obtain some document metadata
+	Element dm_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	String to = GSPath.appendLink(collection, "DocumentMetadataRetrieve");  // Hard-wired?
+	Element dm_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	dm_message.appendChild(dm_request);
+	// Create a parameter list to specify the required metadata information
+	
+	HashSet meta_names = new HashSet();
+	meta_names.add("Title"); // the default
+	if (format_elem != null) {
+	    extractMetadataNames(format_elem, meta_names);
+	}
+	
+	Element dm_param_list = createMetadataParamList(meta_names);
+	if (service_params != null) {
+	    GSXML.addParametersToList(this.doc, dm_param_list, service_params);
+	}
+	
+	dm_request.appendChild(dm_param_list);
+	
+	
+	// create the doc node list for the metadata request
+	Element dm_doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	dm_request.appendChild(dm_doc_list);
+
+	// Add each node from the structure response into the metadata request
+	NodeList doc_nodes = the_document.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	for (int i = 0; i < doc_nodes.getLength(); i++) {
+	    Element doc_node = (Element) doc_nodes.item(i);
+	    String doc_node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+
+	    // Add the documentNode to the list
+	    Element dm_doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    dm_doc_list.appendChild(dm_doc_node);
+	    dm_doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_node_id);
+	    dm_doc_node.setAttribute(GSXML.NODE_TYPE_ATT,
+				     doc_node.getAttribute(GSXML.NODE_TYPE_ATT));
+	}
+
+	// we also want a metadata request to the top level document to get
+	// assocfilepath - this could be cached too
+	Element doc_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	dm_message.appendChild(doc_meta_request);
+	Element doc_meta_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (service_params != null) {
+	    GSXML.addParametersToList(this.doc, doc_meta_param_list, service_params);
+	}
+
+	doc_meta_request.appendChild(doc_meta_param_list);
+	Element doc_param = this.doc.createElement(GSXML.PARAM_ELEM);
+	doc_meta_param_list.appendChild(doc_param);
+	doc_param.setAttribute(GSXML.NAME_ATT, "metadata");
+	doc_param.setAttribute(GSXML.VALUE_ATT, "archivedir");
+
+	// create the doc node list for the metadata request
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	doc_meta_request.appendChild(doc_list);
+
+	Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	// teh node we want is the root document node
+	doc_node.setAttribute(GSXML.NODE_ID_ATT, document_name+".rt");
+	doc_list.appendChild(doc_node);
+	Element dm_response_message = (Element) this.mr.process(dm_message);
+	if (processErrorElements(dm_response_message, page_response)) {
+	    return result;
+	}
+
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	Element dm_response_doc_list = (Element) GSXML.getNodeByPath(dm_response_message, path);
+
+	// Merge the metadata with the structure information
+	NodeList dm_response_docs = dm_response_doc_list.getChildNodes();
+	for (int i = 0; i < doc_nodes.getLength(); i++) {
+	    GSXML.mergeMetadataLists(doc_nodes.item(i), dm_response_docs.item(i));
+	}
+	// get teh top level doc metadata out
+	Element doc_meta_response = (Element)dm_response_message.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1);
+	Element doc_meta_list = (Element)GSXML.getNodeByPath(doc_meta_response, "documentNodeList/documentNode/metadataList");
+	if (doc_meta_list != null) {
+	    the_document.appendChild(this.doc.importNode(doc_meta_list, true));
+	}
+	// Build a request to obtain some document content
+	Element dc_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	to = GSPath.appendLink(collection, "DocumentContentRetrieve");  // Hard-wired?
+	Element dc_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	dc_message.appendChild(dc_request);
+	
+
+	// Create a parameter list to specify the request parameters - empty for now
+	Element dc_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (service_params != null) {
+	    GSXML.addParametersToList(this.doc, dc_param_list, service_params);
+	}
+
+	dc_request.appendChild(dc_param_list);
+
+	// get the content
+	// the doc list for the content request is the same as the one for the structure request unless we want the whole document, in which case its the same as for the metadata request.
+	if (expand_document) {
+	    dc_request.appendChild(dm_doc_list);
+	} else {
+	    dc_request.appendChild(basic_doc_list);
+	}
+	///ystem.err.println("request = "+converter.getString(dc_message));
+	Element dc_response_message = (Element) this.mr.process(dc_message);
+	if (processErrorElements(dc_response_message, page_response)) {
+	    return result;
+	}
+
+	Element dc_response_doc_list = (Element) GSXML.getNodeByPath(dc_response_message, path);
+
+	if (expand_document) {
+	    // Merge the content with the structure information
+	    NodeList dc_response_docs = dc_response_doc_list.getChildNodes();
+	    for (int i = 0; i < doc_nodes.getLength(); i++) {
+		Node content = GSXML.getChildByTagName((Element)dc_response_docs.item(i), "nodeContent");
+		if (content != null) {
+		    doc_nodes.item(i).appendChild(this.doc.importNode(content, true));
+		}
+		//GSXML.mergeMetadataLists(doc_nodes.item(i), dm_response_docs.item(i));
+	    }
+	} else {
+	    
+	    //path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+	    Element dc_response_doc = (Element) GSXML.getChildByTagName(dc_response_doc_list, GSXML.DOC_NODE_ELEM);
+	    Element dc_response_doc_content = (Element) GSXML.getChildByTagName(dc_response_doc, GSXML.NODE_CONTENT_ELEM);
+	    
+	    if (dc_response_doc_content == null) {
+		// no content to add
+		return result;
+	    }
+	    if (highlight_query_terms) {
+		dc_response_doc.removeChild(dc_response_doc_content);
+		
+		dc_response_doc_content = highlightQueryTerms(request, dc_response_doc_content);
+		dc_response_doc.appendChild(dc_response_doc.getOwnerDocument().importNode(dc_response_doc_content, true));
+	    }
+	    
+	
+	if (provide_annotations) {
+	    String service_selected = (String)params.get(ENRICH_DOC_ARG);
+	    if (service_selected != null && service_selected.equals("1")) {
+		// now we can modifiy the response doc if needed
+		String enrich_service = (String)params.get(GSParams.SERVICE);
+		// send a message to the service
+		Element enrich_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+		Element enrich_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, enrich_service, lang, uid);
+		enrich_message.appendChild(enrich_request);
+		// check for parameters
+		HashMap e_service_params = (HashMap)params.get("s1");
+		if (e_service_params != null) { 
+		    Element enrich_pl = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		    GSXML.addParametersToList(this.doc, enrich_pl, e_service_params); 
+		    enrich_request.appendChild(enrich_pl);
+		}
+		Element e_doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+		enrich_request.appendChild(e_doc_list);
+		e_doc_list.appendChild(this.doc.importNode(dc_response_doc, true));
+		
+		Element enrich_response = this.mr.process(enrich_message);
+		
+		String [] links = {GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM, GSXML.NODE_CONTENT_ELEM};
+		path = GSPath.createPath(links);
+		dc_response_doc_content = (Element)GSXML.getNodeByPath(enrich_response, path);
+		
+	    } 
+	} // if provide_annotations
+
+	
+	// use the returned id rather than the sent one cos there may have
+	// been modifiers such as .pr that are removed.
+	String modified_doc_id = dc_response_doc.getAttribute(GSXML.NODE_ID_ATT);
+	the_document.setAttribute("selectedNode", modified_doc_id);
+	if (has_dummy) {
+	    // change the id if necessary and add the content
+	    Element dummy_node = (Element)doc_nodes.item(0);
+	    
+	    dummy_node.setAttribute(GSXML.NODE_ID_ATT, modified_doc_id);
+	    dummy_node.appendChild(this.doc.importNode(dc_response_doc_content, true));
+	    // hack for simple type
+	    if (document_type.equals("simple")) {
+		// we dont want the internal docNode, just want the content and metadata in the document
+		// rethink this!!
+		the_document.removeChild(dummy_node);
+
+		NodeList dummy_children = dummy_node.getChildNodes();
+		//for (int i=0; i<dummy_children.getLength(); i++) {
+		for (int i=dummy_children.getLength()-1; i>=0; i--) {
+		    the_document.appendChild(dummy_children.item(i));
+		    
+		}
+	    }
+	} else {
+	    // Merge the document content with the metadata and structure information
+	    for (int i = 0; i < doc_nodes.getLength(); i++) {
+		Node dn = doc_nodes.item(i);
+		String dn_id = ((Element)dn).getAttribute(GSXML.NODE_ID_ATT);
+		if (dn_id.equals(modified_doc_id)) {
+		    dn.appendChild(this.doc.importNode(dc_response_doc_content, true));
+		    break;
+		}
+	    }
+	}
+	}
+	///ystem.out.println("(DocumentAction) Page:\n" + this.converter.getPrettyString(result));
+	return result;
+    }
+    
+    /** tell the param class what its arguments are 
+     * if an action has its own arguments, this should add them to the params
+     * object - particularly important for args that should not be saved */
+    public boolean getActionParameters(GSParams params) {
+	params.addParameter(GOTO_PAGE_ARG, false);
+	params.addParameter(ENRICH_DOC_ARG, false);
+ 	return true;
+    }
+
+
+    /** this method gets the collection description, the format info, the
+     * list of enrich services, etc - stuff that is needed for the page,
+     * but is the same whatever the query is - should be cached */
+    protected  boolean getBackgroundData(Element page_response, 
+					 String collection, String lang,
+					 String uid) {
+
+	// create a message to process - contains requests for the collection 
+	// description, the format element, the enrich services on offer
+	// these could all be cached
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	String path = GSPath.appendLink(collection, "DocumentContentRetrieve");
+	// the format request - ignore for now, where does this request go to??
+	Element format_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_FORMAT, path, lang, uid);
+	info_message.appendChild(format_request);
+
+	// the enrich_services request - only do this if provide_annotations is true
+
+	if (provide_annotations) {
+	    Element enrich_services_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", lang, uid);
+	    enrich_services_request.setAttribute(GSXML.INFO_ATT, "serviceList");
+	    info_message.appendChild(enrich_services_request);
+	}
+	
+	Element info_response = (Element)this.mr.process(info_message);
+
+	// the collection is the first response
+	NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	Element format_resp = (Element) responses.item(0);
+	
+	Element format_elem = (Element)GSXML.getChildByTagName(format_resp, GSXML.FORMAT_ELEM);
+	if (format_elem != null) {
+	    ///ystem.out.println("doc action found a format statement");
+	    // set teh format type
+	    format_elem.setAttribute(GSXML.TYPE_ATT, "display");  
+	    page_response.appendChild(this.doc.importNode(format_elem, true));
+	}
+
+	if (provide_annotations) {
+	    Element services_resp = (Element)responses.item(1); 
+
+	    // a new message for the mr
+	    Element enrich_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    
+	    NodeList e_services = services_resp.getElementsByTagName(GSXML.SERVICE_ELEM);
+	    boolean service_found = false;
+	    for (int j=0; j<e_services.getLength(); j++) {
+		if (((Element)e_services.item(j)).getAttribute(GSXML.TYPE_ATT).equals("enrich")) {
+		    Element s = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, ((Element)e_services.item(j)).getAttribute(GSXML.NAME_ATT), lang, uid);
+		    enrich_message.appendChild(s);
+		    service_found = true;
+		}
+	    }
+	    if (service_found) {
+		Element enrich_response = this.mr.process(enrich_message);
+
+		NodeList e_responses = enrich_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		Element service_list = this.doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		for (int i=0; i<e_responses.getLength(); i++) {
+		    Element e_resp = (Element)e_responses.item(i);
+		    Element e_service = (Element)this.doc.importNode(GSXML.getChildByTagName(e_resp, GSXML.SERVICE_ELEM), true); 
+		    e_service.setAttribute(GSXML.NAME_ATT, e_resp.getAttribute(GSXML.FROM_ATT));
+		    service_list.appendChild(e_service);
+		}
+		page_response.appendChild(service_list);
+	    }
+	} // if provide_annotations
+	return true;
+	    
+    }
+
+    /** this involves a bit of a hack to get the equivalent query terms - has to requery the query service - uses the last selected service name. (if it ends in query). should this action do the query or should it send a message to the query action? but that will involve lots of extra stuff. also doesn't handle phrases properly - just highlights all the terms found in the text.
+     */
+    protected Element highlightQueryTerms(Element request, Element dc_response_doc_content) {
+
+	// do the query again to get term info 
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+	
+	HashMap previous_params = (HashMap)params.get("p");
+	if (previous_params == null) {
+	    return dc_response_doc_content;
+	}
+       	String service_name = (String)previous_params.get(GSParams.SERVICE);
+	if (service_name == null || !service_name.endsWith("Query")) { // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
+	    System.err.println("DocumentAction: invalid service, not doing highlighting");
+	    return dc_response_doc_content;
+	}
+	String collection = (String)params.get(GSParams.COLLECTION);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String to = GSPath.appendLink(collection, service_name);
+	
+	Element mr_query_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element mr_query_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	mr_query_message.appendChild(mr_query_request);
+	
+	// paramList
+	HashMap service_params = (HashMap)params.get("s1");
+	
+	Element query_param_list = this.doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+	GSXML.addParametersToList(this.doc, query_param_list, service_params);
+	mr_query_request.appendChild(query_param_list);
+
+	// do the query
+       	Element mr_query_response = (Element)this.mr.process(mr_query_message);
+	
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	Element query_term_list_element = (Element) GSXML.getNodeByPath(mr_query_response, path);
+	if (query_term_list_element == null) {
+	    // no term info
+	    System.err.println("DocumentAction: Warning: No query term information.\n");
+	    return dc_response_doc_content;
+	} 
+
+	String content = GSXML.getNodeText(dc_response_doc_content);
+
+	String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
+
+	HashSet query_term_variants = new HashSet();
+	NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
+	for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++) {
+	    Element equivalent_terms_element = (Element) equivalent_terms_nodelist.item(i);
+	    String[] equivalent_terms = GSXML.getAttributeValuesFromList(equivalent_terms_element, GSXML.NAME_ATT);
+	    for (int j = 0; j < equivalent_terms.length; j++) {
+		query_term_variants.add(equivalent_terms[j]);
+	    }
+	}
+
+	ArrayList phrase_query_term_variants_hierarchy = new ArrayList();
+
+	Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
+	String performed_query = GSXML.getNodeText(query_element) + " ";
+
+	ArrayList phrase_query_p_term_variants_list = new ArrayList();
+	int term_start = 0;
+	boolean in_term = false;
+	boolean in_phrase = false;
+	for (int i = 0; i < performed_query.length(); i++) {
+	    char character = performed_query.charAt(i);
+	    boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
+
+	    // Has a query term just started?
+	    if (in_term == false && is_character_letter_or_digit == true) {
+		in_term = true;
+		term_start = i;
+	    }
+
+	    // Or has a term just finished?
+	    else if (in_term == true && is_character_letter_or_digit == false) {
+		in_term = false;
+		String term = performed_query.substring(term_start, i);
+				
+		Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
+		if (term_element != null) {
+		    
+		    HashSet phrase_query_p_term_x_variants = new HashSet();
+		    
+		    NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
+		    for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++) {
+			Element term_equivalent_terms_element = (Element) term_equivalent_terms_nodelist.item(j);
+			String[] term_equivalent_terms = GSXML.getAttributeValuesFromList(term_equivalent_terms_element, GSXML.NAME_ATT);
+			for (int k = 0; k < term_equivalent_terms.length; k++) {
+			    phrase_query_p_term_x_variants.add(term_equivalent_terms[k]);
+			}
+		    }
+		    phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
+		    
+		    if (in_phrase == false) {
+			phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+			phrase_query_p_term_variants_list = new ArrayList();
+		    }
+		}
+	    }
+	    // Watch for phrases (surrounded by quotes)
+	    if (character == '\"') {
+		// Has a phrase just started?
+		if (in_phrase == false) {
+		    in_phrase = true;
+		}
+		// Or has a phrase just finished?
+		else if (in_phrase == true) {
+		    in_phrase = false;
+		    phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+		}
+
+		phrase_query_p_term_variants_list = new ArrayList();
+	    }
+	}
+
+	return highlightQueryTermsInternal(content, query_term_variants, phrase_query_term_variants_hierarchy);
+    }
+
+
+    /**
+     * Highlights query terms in a piece of text.
+     */
+    private Element highlightQueryTermsInternal(String content, HashSet query_term_variants, ArrayList phrase_query_term_variants_hierarchy)
+    {
+	// Convert the content string to an array of characters for speed
+	char[] content_characters = new char[content.length()];
+	content.getChars(0, content.length(), content_characters, 0);
+
+	// Now skim through the content, identifying word matches
+	ArrayList word_matches = new ArrayList();
+	int word_start = 0;
+	boolean in_word = false;
+	boolean preceding_word_matched = false;
+	for (int i = 0; i < content_characters.length; i++) {
+	    boolean is_character_letter_or_digit = Character.isLetterOrDigit(content_characters[i]);
+
+	    // Has a word just started?
+	    if (in_word == false && is_character_letter_or_digit == true) {
+		in_word = true;
+		word_start = i;
+	    }
+
+	    // Or has a word just finished?
+	    else if (in_word == true && is_character_letter_or_digit == false) {
+		in_word = false;
+
+		// Check if the word matches any of the query term equivalents
+		String word = new String(content_characters, word_start, (i - word_start));
+		if (query_term_variants.contains(word)) {
+		    // We have found a matching word, so remember its location
+		    word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
+		    preceding_word_matched = true;
+ 		}
+		else {
+		    preceding_word_matched = false;
+		}
+	    }
+	}
+
+	// Don't forget the last word...
+	if (in_word == true) {
+	    // Check if the word matches any of the query term equivalents
+	    String word = new String(content_characters, word_start, (content_characters.length - word_start));
+	    if (query_term_variants.contains(word)) {
+		// We have found a matching word, so remember its location
+		word_matches.add(new WordMatch(word, word_start, content_characters.length, preceding_word_matched));
+	    }
+	}
+
+	ArrayList highlight_start_positions = new ArrayList();
+	ArrayList highlight_end_positions = new ArrayList();
+
+	// Deal with phrases now
+	ArrayList partial_phrase_matches = new ArrayList();
+	for (int i = 0; i < word_matches.size(); i++) {
+	    WordMatch word_match = (WordMatch) word_matches.get(i);
+
+ 	    // See if any partial phrase matches are extended by this word
+ 	    if (word_match.preceding_word_matched) {
+ 		for (int j = partial_phrase_matches.size() - 1; j >= 0; j--) {
+ 		    PartialPhraseMatch partial_phrase_match = (PartialPhraseMatch) partial_phrase_matches.remove(j);
+		    ArrayList phrase_query_p_term_variants_list = (ArrayList) phrase_query_term_variants_hierarchy.get(partial_phrase_match.query_phrase_number);
+ 		    HashSet phrase_query_p_term_x_variants = (HashSet) phrase_query_p_term_variants_list.get(partial_phrase_match.num_words_matched);
+ 		    if (phrase_query_p_term_x_variants.contains(word_match.word)) {
+ 			partial_phrase_match.num_words_matched++;
+
+ 			// Has a complete phrase match occurred?
+ 			if (partial_phrase_match.num_words_matched == phrase_query_p_term_variants_list.size()) {
+ 			    // Check for overlaps by looking at the previous highlight range
+ 			    if (!highlight_end_positions.isEmpty()) {
+ 				int last_highlight_index = highlight_end_positions.size() - 1;
+ 				int last_highlight_end = ((Integer) highlight_end_positions.get(last_highlight_index)).intValue();
+ 				if (last_highlight_end > partial_phrase_match.start_position) {
+ 				    // There is an overlap, so remove the previous phrase match
+ 				    int last_highlight_start = ((Integer) highlight_start_positions.remove(last_highlight_index)).intValue();
+ 				    highlight_end_positions.remove(last_highlight_index);
+ 				    partial_phrase_match.start_position = last_highlight_start;
+ 				}
+ 			    }
+
+ 			    highlight_start_positions.add(new Integer(partial_phrase_match.start_position));
+ 			    highlight_end_positions.add(new Integer(word_match.end_position));
+ 			}
+ 			// No, but add the partial match back into the list for next time
+ 			else {
+ 			    partial_phrase_matches.add(partial_phrase_match);
+ 			}
+ 		    }
+ 		}
+ 	    }
+ 	    else {
+ 		partial_phrase_matches.clear();
+ 	    }
+
+	    // See if this word is at the start of any of the phrases
+	    for (int p = 0; p < phrase_query_term_variants_hierarchy.size(); p++) {
+		ArrayList phrase_query_p_term_variants_list = (ArrayList) phrase_query_term_variants_hierarchy.get(p);
+		HashSet phrase_query_p_term_1_variants = (HashSet) phrase_query_p_term_variants_list.get(0);
+		if (phrase_query_p_term_1_variants.contains(word_match.word)) {
+		    // If this phrase is just one word long, we have a complete match
+		    if (phrase_query_p_term_variants_list.size() == 1) {
+			highlight_start_positions.add(new Integer(word_match.start_position));
+			highlight_end_positions.add(new Integer(word_match.end_position));
+		    }
+		    // Otherwise we have the start of a potential phrase match
+		    else {
+			partial_phrase_matches.add(new PartialPhraseMatch(word_match.start_position, p));
+		    }
+		}
+	    }
+	}
+
+	// Now add the annotation tags into the document at the correct points
+	Element content_element = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+
+ 	int last_wrote = 0;
+	for (int i = 0; i < highlight_start_positions.size(); i++) {
+	    int highlight_start = ((Integer) highlight_start_positions.get(i)).intValue();
+	    int highlight_end = ((Integer) highlight_end_positions.get(i)).intValue();
+
+	    // Print anything before the highlight range
+	    if (last_wrote < highlight_start) {
+		String preceding_text = new String(content_characters, last_wrote, (highlight_start - last_wrote));
+		// System.err.print(preceding_text);
+		content_element.appendChild(this.doc.createTextNode(preceding_text));
+	    }
+
+	    // Print the highlight text, annotated
+	    if (highlight_end > last_wrote) {
+		String highlight_text = new String(content_characters, highlight_start, (highlight_end - highlight_start));
+		// System.err.print("|" + highlight_text + "|");
+		Element annotation_element = GSXML.createTextElement(this.doc, "annotation", highlight_text);
+		annotation_element.setAttribute("type", "query_term");
+		content_element.appendChild(annotation_element);
+		last_wrote = highlight_end;
+	    }
+	}
+
+	// Finish off any unwritten text
+	if (last_wrote < content_characters.length) {
+	    String remaining_text = new String(content_characters, last_wrote, (content_characters.length - last_wrote));
+	    // System.err.print(remaining_text);
+	    content_element.appendChild(this.doc.createTextNode(remaining_text));
+	}
+
+	return content_element;
+    }
+
+
+    static private class WordMatch
+    {
+	public String word;
+	public int start_position;
+	public int end_position;
+	public boolean preceding_word_matched;
+
+	public WordMatch(String word, int start_position, int end_position, boolean preceding_word_matched)
+	{
+	    this.word = word;
+	    this.start_position = start_position;
+	    this.end_position = end_position;
+	    this.preceding_word_matched = preceding_word_matched;
+	}
+    }
+
+
+    static private class PartialPhraseMatch
+    {
+	public int start_position;
+	public int query_phrase_number;
+	public int num_words_matched;
+
+	public PartialPhraseMatch(int start_position, int query_phrase_number)
+	{
+	    this.start_position = start_position;
+	    this.query_phrase_number = query_phrase_number;
+	    this.num_words_matched = 1;
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GS2BrowseAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GS2BrowseAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GS2BrowseAction.java	(revision 21112)
@@ -0,0 +1,325 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.io.File;
+
+/** action for GS2 style classifier browsing */
+public class GS2BrowseAction extends Action {
+    
+    public static final String CLASSIFIER_ARG = "cl";
+    
+    /** process the request */
+    public Element process (Element message) {
+
+	// get the request - assume only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// the result
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element response = classifierBrowse(request);
+	result.appendChild(response);
+	return result;
+    }
+    
+
+    protected Element classifierBrowse(Element request) {
+
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	// extract the params from the cgi-request, and check that we have a coll specified
+	Element cgi_paramList = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_paramList, false);
+	
+	String service_name = (String)params.get(GSParams.SERVICE);
+	String collection = (String)params.get(GSParams.COLLECTION);
+	if (collection == null || collection.equals("")) {
+	    System.err.println("GS2BrowseAction Error:classifierBrowse, need to specify a collection!");
+	    return page_response;
+	   
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String to = GSPath.appendLink(collection, service_name);
+	
+	// the first part of the response is the service description 
+	// for now get this again from the service. 
+	// this should be cached somehow later on. 
+	
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, uid); 
+	info_message.appendChild(info_request);
+	
+	// also get the format stuff now if there is some
+	Element format_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_FORMAT, to, lang, uid);
+	info_message.appendChild(format_request);
+	// process the requests
+
+	Element info_response = (Element) this.mr.process(info_message);
+
+	// the two responses
+	NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	Element service_response = (Element)responses.item(0);
+	Element format_response = (Element)responses.item(1);
+
+	Element service_description = (Element)GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM);
+	page_response.appendChild(this.doc.importNode(service_description, true));
+	
+	// if rt=d, then we are just displaying the service
+	String request_type = (String)params.get(GSParams.REQUEST_TYPE);
+	if (request_type.equals("d")) {
+	    //return the page that we have so far
+	    return page_response;
+	}
+    
+	// get the node that the user has clicked on
+	String classifier_node = (String)params.get(CLASSIFIER_ARG);
+	
+	// if the node is not defined, return the page that we have so far
+	if (classifier_node ==null || classifier_node.equals("")) {
+	    return page_response;
+	}
+	
+	// the id of the classifier is the top id of the selected node
+	String top_id = OID.getTop(classifier_node);
+	HashSet doc_meta_names = new HashSet();
+	HashSet class_meta_names = new HashSet();
+	// add in the defaults
+	doc_meta_names.add("Title");
+	class_meta_names.add("Title");
+	
+	// add the format info into the response
+	Element format_elem = (Element)GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+	if (format_elem != null) {
+	    
+	    // find the one for the classifier we are in
+	    Element this_format = GSXML.getNamedElement(format_elem, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+	    if (this_format == null) {
+		this_format = (Element)GSXML.getChildByTagName(format_elem, GSXML.DEFAULT_ELEM);
+	    }
+
+	    if (this_format != null) {
+		Element new_format = GSXML.duplicateWithNewName(this.doc, this_format, GSXML.FORMAT_ELEM, false);
+		extractMetadataNames(new_format, doc_meta_names, class_meta_names);
+		// set the format type
+		new_format.setAttribute(GSXML.TYPE_ATT, "browse");
+
+		page_response.appendChild(new_format);
+	    }
+	}
+	
+	// find out if this classifier is horizontal at top
+	Element class_list = (Element)GSXML.getChildByTagName(service_description, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+	Element this_classifier = GSXML.getNamedElement(class_list, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+	boolean horizontal_at_top = false;
+	if (!this_classifier.getAttribute("horizontalAtTop").equals("")) {
+	    horizontal_at_top = true;
+	}
+	if (top_id.equals(classifier_node) && horizontal_at_top) {
+	    // we have asked for a top node - if the first list is horizontal, we will select the first element of that list
+	    // this is a hack. also it craps out if the classifier really isn't horizontalAtTop. - 
+	    classifier_node = classifier_node+".1";
+		    
+	}
+
+	// get the browse structure for the selected node
+	Element classify_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element classify_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	classify_message.appendChild(classify_request);
+	    
+	//Create a parameter list to specify the required structure information
+	// for now, always get ancestors and children
+	Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	classify_request.appendChild(param_list);
+	Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+	param_list.appendChild(param);
+	param.setAttribute(GSXML.NAME_ATT, "structure");
+	param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+	param = this.doc.createElement(GSXML.PARAM_ELEM);
+	param_list.appendChild(param);
+	param.setAttribute(GSXML.NAME_ATT, "structure");
+	param.setAttribute(GSXML.VALUE_ATT, "children");
+
+	// put the classifier node into a classifier node list
+	Element classifier_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	Element classifier = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	classifier.setAttribute(GSXML.NODE_ID_ATT, classifier_node);
+	classifier_list.appendChild(classifier);
+	classify_request.appendChild(classifier_list);
+	    
+	if (horizontal_at_top && !classifier_node.equals(top_id)) {
+	    // also put the top id in, to get the persistant horizontal info
+	    classifier = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	    classifier.setAttribute(GSXML.NODE_ID_ATT, top_id);
+	    classifier_list.appendChild(classifier);
+	}
+	// process the request
+	Element classify_response = (Element)this.mr.process(classify_message);
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	Element class_node_list = (Element)GSXML.getNodeByPath(classify_response, path);
+	
+	path = GSPath.appendLink(GSXML.CLASS_NODE_ELEM, GSXML.NODE_STRUCTURE_ELEM);
+	// assume that we always get back the top level CL1 node - this becomes the page_classifier node
+	path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+	Element cl_structure = (Element)GSXML.getNodeByPath(class_node_list,
+							    path);
+	if (cl_structure ==null) {
+	    System.err.println("GS2BrowseAction: classifier structure request returned no structure");
+	    return page_response;
+	}   
+    
+	Element page_classifier = null;
+	if (horizontal_at_top && !classifier_node.equals(top_id)) {
+	    // get the info for the top node
+	    Element top_node = GSXML.getNamedElement(class_node_list, GSXML.CLASS_NODE_ELEM, GSXML.NODE_ID_ATT, top_id);
+	    if (top_node !=null) {
+		path = GSPath.appendLink(GSXML.NODE_STRUCTURE_ELEM, GSXML.CLASS_NODE_ELEM);
+		Element top_structure = (Element)GSXML.getNodeByPath(top_node, path);	    
+		// add this as the classifier elem
+		page_classifier = GSXML.duplicateWithNewName(this.doc, top_structure,  GSXML.CLASSIFIER_ELEM, true);
+		page_response.appendChild(page_classifier);
+		// now replace the child with the structure from the other request
+		Element new_classifier = (Element)GSXML.getChildByTagName(cl_structure, GSXML.CLASS_NODE_ELEM);
+		String replace_name = new_classifier.getAttribute(GSXML.NODE_ID_ATT);
+		
+		// find the appropriate child
+		Element old_classifier = GSXML.getNamedElement(page_classifier, GSXML.CLASS_NODE_ELEM, GSXML.NODE_ID_ATT, replace_name);
+		page_classifier.replaceChild(this.doc.importNode(new_classifier, true), old_classifier);
+		page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+	    } else {
+		// add the single classifier node as the page classifier 
+		page_classifier = GSXML.duplicateWithNewName(this.doc, cl_structure,  GSXML.CLASSIFIER_ELEM, true);
+		page_response.appendChild(page_classifier);
+		page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+	    }
+	    
+	} else {
+	    // add the single classifier node as the page classifier 
+	    page_classifier = GSXML.duplicateWithNewName(this.doc, cl_structure,  GSXML.CLASSIFIER_ELEM, true);
+	    page_response.appendChild(page_classifier);
+	    page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+	}
+	// get the metadata for each classifier node, 
+	// then for each document node
+
+	Element metadata_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+
+	boolean did_classifier = false;
+	boolean did_documents = false;
+	
+    
+	// if there are classifier nodes 
+	// create a metadata request for the classifier, and add it to 
+	// the the message
+	NodeList cl_nodes = page_classifier.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+	
+	if (cl_nodes.getLength() > 0) {
+	    did_classifier = true;
+	    Element cl_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to+"MetadataRetrieve", lang, uid);
+	    metadata_message.appendChild(cl_meta_request);
+	    
+	    Element new_cl_nodes_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    cl_meta_request.appendChild(new_cl_nodes_list);
+	    
+	    for (int c=0; c<cl_nodes.getLength(); c++) {
+
+		Element cl = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+		cl.setAttribute(GSXML.NODE_ID_ATT, ((Element)cl_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+		new_cl_nodes_list.appendChild(cl);
+	    }
+
+	    // create and add in the param list - for now get all the metadata
+	    // should be based on info sent in from the recept, and the 
+	    // format stuff
+	    Element cl_param_list  = createMetadataParamList(class_meta_names);
+	    cl_meta_request.appendChild(cl_param_list);
+	    
+	}
+
+	// if there are document nodes in the classification (happens 
+	// sometimes), create a second request for document metadata and 
+	// append to the message
+	NodeList doc_nodes = page_classifier.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	if (doc_nodes.getLength() > 0) {
+	    did_documents = true;
+	    Element doc_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, GSPath.appendLink(collection, "DocumentMetadataRetrieve"), lang, uid);
+	    metadata_message.appendChild(doc_meta_request);
+	    
+	    Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    doc_meta_request.appendChild(doc_list);
+	    
+	    for (int c=0; c<doc_nodes.getLength(); c++) {
+		
+		Element d = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		d.setAttribute(GSXML.NODE_ID_ATT, ((Element)doc_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+		doc_list.appendChild(d);
+	    }
+
+	    // create and add in the param list - add all for now
+	    Element doc_param_list  = createMetadataParamList(doc_meta_names);
+	    doc_meta_request.appendChild(doc_param_list);
+
+	}
+	
+	// process the metadata requests
+	Element metadata_response = (Element)this.mr.process(metadata_message);
+	if (did_classifier) {
+	    // the classifier one will be the first response
+	    // add the metadata lists for each node back into the 
+	    // page_classifier nodes
+	    path = GSPath.appendLink(GSXML.RESPONSE_ELEM, 
+				   GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    NodeList meta_response_cls = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+	    for (int i = 0; i < cl_nodes.getLength(); i++) {
+		GSXML.mergeMetadataLists(cl_nodes.item(i), meta_response_cls.item(i));
+	    }
+	}
+	if (did_documents) {
+	    NodeList meta_response_docs = null;
+	    if (!did_classifier) {
+		// its the first response
+		path = GSPath.appendLink(GSXML.RESPONSE_ELEM, 
+					 GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+		meta_response_docs = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+	    } else { // its the second response
+		meta_response_docs = GSXML.getChildByTagName(metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1), GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER).getChildNodes();
+	    }
+	    
+	    for (int i = 0; i < doc_nodes.getLength(); i++) {
+		GSXML.mergeMetadataLists(doc_nodes.item(i), meta_response_docs.item(i));
+	    }
+	}
+	    
+
+	///ystem.out.println("(GS2BrowseAction) Page:\n" + this.converter.getPrettyString(page_response));
+	return page_response;
+    }
+
+   
+    protected void extractMetadataNames(Element new_format, HashSet doc_meta_names, HashSet class_meta_names) {
+
+	NodeList templates = new_format.getElementsByTagName("gsf:template");
+	for (int i=0; i<templates.getLength(); i++) {
+	    Element template = (Element)templates.item(i);
+	    String match = template.getAttribute("match");
+	    if (match.equals("documentNode")) {
+		extractMetadataNames(template, doc_meta_names);
+	    } else if (match.equals("classifierNode")) {
+		extractMetadataNames(template, class_meta_names);
+	    }
+	}
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GeneralAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GeneralAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/GeneralAction.java	(revision 21112)
@@ -0,0 +1,107 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+
+public class GeneralAction extends Action {
+    
+    /** process a request */
+    public Element process (Element message) {
+
+	// the result
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.appendChild(page_response);
+
+	// assume only one request
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// get the param list
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+	String service_name = (String) params.get(GSParams.SERVICE);
+	String cluster_name = (String) params.get(GSParams.CLUSTER);
+	String response_only_p = (String)params.get(GSParams.RESPONSE_ONLY);
+	boolean response_only = false;
+	if (response_only_p!=null) {
+	    response_only = (response_only_p.equals("1")?true:false);
+	}
+	String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	// what is carried out depends on the request_type
+	// if rt=d, then a describe request is done,
+	// is rt=r, a request and then a describe request is done
+	// if rt=s, a status request is done.
+
+	// if ro=1, then this calls for a process only page - we do the request
+	// (rt should be r or s) and just give the response straight back
+	// without any page processing
+
+
+	// where to send requests
+	String to;
+	if (cluster_name != null) {
+	    to = GSPath.appendLink(cluster_name, service_name);
+	} else {
+	    to = service_name;
+	}
+
+	if (request_type.equals("r")) {
+	    //do the request
+	    
+	    Element mr_query_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    Element mr_query_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	    
+	    mr_query_message.appendChild(mr_query_request);
+	    
+	    Element param_list = null;
+	    // add in the service params - except the ones only used by the action
+	    HashMap service_params = (HashMap)params.get("s1");
+	    if (service_params != null) { 
+		param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		GSXML.addParametersToList(this.doc, param_list, service_params);
+		mr_query_request.appendChild(param_list);
+	    }
+	    
+	    Element mr_query_response = (Element)this.mr.process(mr_query_message);
+	    Element result_response = (Element)GSXML.getChildByTagName(mr_query_response, GSXML.RESPONSE_ELEM);
+	    
+	    if (response_only) {
+		// just send the reponse as is
+		return result_response;
+	    }
+	    
+	    // else append the contents of the response to the page 
+	    GSXML.copyAllChildren(page_response, result_response);
+	}
+	
+
+	// another part of the page is the service description
+
+	// request the service info for the selected service - should be cached
+	Element mr_info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element mr_info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, uid);
+	mr_info_message.appendChild(mr_info_request);
+	Element mr_info_response = (Element) this.mr.process(mr_info_message);
+
+	String path = GSXML.RESPONSE_ELEM;
+	path = GSPath.appendLink(path, GSXML.SERVICE_ELEM);
+	Element description = (Element)this.doc.importNode(GSXML.getNodeByPath(mr_info_response, path), true);
+
+	page_response.appendChild(description);
+
+	return result;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/NoCollQueryAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/NoCollQueryAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/NoCollQueryAction.java	(revision 21112)
@@ -0,0 +1,216 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+
+/** action class for queries 
+ * this is used when querying isn't collection specific, but it occurs across all collections in the site. The service description is assumed to be known by the xslt so we dont ask for it. we just pass all the service params to the TextQuery service of all the collections */
+public class NoCollQueryAction extends Action {
+   
+    /** process - processes a request.
+     */ 
+    public Element process (Element message) {
+
+	// get the request - assume there is only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// create the return message
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	// for now we only have one type of query - subaction not used
+	Element response = basicQuery(request);
+	result.appendChild(this.doc.importNode(response, true));
+	return result;
+    }
+    
+    /** a generic query handler 
+     * this gets the service description, does the query (just passes all teh 
+     * params to the service, then gets the titles for any results 
+     */
+    protected Element basicQuery(Element request) {
+	
+	// the result
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	
+	// extract the params from the cgi-request, and check that we have a coll specified
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+	
+	String request_type = (String)params.get(GSParams.REQUEST_TYPE);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	if (request_type.equals("d")) {
+	    // display the query page
+	    // the only info we need to return is the collection list cos the xslt does teh rest
+	    
+	    Element coll_list = getCollectionList(lang, uid);
+	    page_response.appendChild(this.doc.importNode(coll_list, true));
+	    return page_response;
+	    
+	}
+	
+	// else we have a query
+	String service_name = (String)params.get(GSParams.SERVICE);
+	if (service_name == null || service_name.equals("")) {
+	    service_name = "TextQuery";
+	}
+  	String query_coll_list = (String)params.get(GSParams.COLLECTION);
+	
+  	if (query_coll_list == null || query_coll_list.equals("")) {
+  	    System.err.println("NoCollQueryAction Error: no collections were specified!");
+  	    Element coll_list = getCollectionList(lang, uid);
+  	    page_response.appendChild(this.doc.importNode(coll_list, true));
+  	    return page_response; 
+  	}
+
+	// service paramList
+	HashMap service_params = (HashMap)params.get("s1");
+	if (service_params == null) { // no query
+	    return page_response;
+	}
+	Element query_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	GSXML.addParametersToList(this.doc, query_param_list, service_params);
+	
+	// we will do a query for each coll
+	String [] colls = query_coll_list.split(",");
+	
+	Element mr_query_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	for (int i=0; i< colls.length; i++) {
+	    String to = GSPath.appendLink(colls[i], service_name);
+	    Element mr_query_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	    mr_query_message.appendChild(mr_query_request);
+	    mr_query_request.appendChild(query_param_list.cloneNode(true));
+	    
+	}
+	
+	// do the query
+       	Element mr_query_response = (Element)this.mr.process(mr_query_message);
+
+	
+	// get the Title metadata for each node - need Node title and root title
+	Element mr_meta_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	NodeList responses = mr_query_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	for (int j=0; j<responses.getLength(); j++) {
+	    Element document_list = (Element)GSXML.getChildByTagName((Element)responses.item(j), GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    if (document_list != null) {
+		String coll_name = extractCollName(((Element)responses.item(j)).getAttribute(GSXML.FROM_ATT));
+		String path = GSPath.appendLink(coll_name, "DocumentMetadataRetrieve");
+		Element mr_meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, path, lang, uid);
+		mr_meta_message.appendChild(mr_meta_request);
+		mr_meta_request.appendChild(this.doc.importNode(document_list, true));
+		// metadata params 
+		Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		Element param = GSXML.createParameter(this.doc, "metadata", "Title");
+		param_list.appendChild(param);
+		param = GSXML.createParameter(this.doc, "metadata", "root_Title");
+		param_list.appendChild(param);
+		mr_meta_request.appendChild(param_list);
+		
+	    }
+	}
+	
+	// do the request
+	Element mr_meta_response = (Element)this.mr.process(mr_meta_message);
+
+	// the result 
+	Element result_doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	page_response.appendChild(result_doc_list);
+
+	responses = mr_meta_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	for (int j=0; j<responses.getLength(); j++) {
+	    Element document_list = (Element)GSXML.getChildByTagName((Element)responses.item(j), GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    String coll_name = extractCollName(((Element)responses.item(j)).getAttribute(GSXML.FROM_ATT));
+	    
+	    mergeDocLists(result_doc_list, document_list, coll_name );
+	}
+	
+	
+	return page_response;
+    }
+
+    protected String extractCollName(String path) {
+	return GSPath.removeLastLink(path);
+    }
+
+    protected void mergeDocLists(Element result_list, Element from_list, String collection) {
+
+	Document owner = result_list.getOwnerDocument();
+	Node child = from_list.getFirstChild();
+	while (child != null) {
+	    ((Element)child).setAttribute("collection", collection);
+	    result_list.appendChild(owner.importNode(child, true));
+	    child = child.getNextSibling();
+	}
+	
+    }
+
+
+    protected Element getCollectionList(String lang, String uid) {
+	
+	// first, get the message router info
+	Element coll_list_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element coll_list_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", lang, uid);
+	coll_list_message.appendChild(coll_list_request);
+	Element coll_list_response = (Element)this.mr.process(coll_list_message);
+	if (coll_list_response==null) {
+	    System.err.println("NoCollQueryAction Error: couldn't query the message router!");
+	    return null;
+	}
+	
+	// second, get the display info for each collection 
+	NodeList colls = coll_list_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+
+	Element coll_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element param = GSXML.createParameter(this.doc, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER); 
+	coll_param_list.appendChild(param);
+	// we will send all the requests in a single message
+	Element metadata_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	for (int i=0; i<colls.getLength(); i++) {
+	    Element c = (Element)colls.item(i);
+	    String name = c.getAttribute(GSXML.NAME_ATT);	    
+	    
+	    Element metadata_request = GSXML.createBasicRequest(this.doc,  GSXML.REQUEST_TYPE_DESCRIBE, name, lang, uid);
+	    metadata_request.appendChild(coll_param_list.cloneNode(true));
+	    metadata_message.appendChild(metadata_request);
+	}
+	
+	Element metadata_response = (Element)this.mr.process(metadata_message);
+	
+	NodeList coll_responses = metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	// check that have same number of responses as collections
+	if (colls.getLength() != coll_responses.getLength()) {
+	    System.err.println("NoCollQueryAction Error: didn't get a response for each collection - somethings gone wrong!");
+	    // for now, dont use the metadata
+	} else {
+	    for (int i=0; i<colls.getLength(); i++) {
+		Element c1 = (Element)colls.item(i);
+		Element c2 = (Element)coll_responses.item(i);
+		if (c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.FROM_ATT))) {
+		    //add the collection data into the original response
+		    GSXML.mergeElements(c1, (Element)GSXML.getChildByTagName(c2, GSXML.COLLECTION_ELEM));
+		} else {
+		    System.err.println("NoCollQueryAction Error: response does not correspond to request!");
+		} 
+		
+	    }
+	}
+
+	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.COLLECTION_ELEM+GSXML.LIST_MODIFIER);
+	Element response = (Element) GSXML.getNodeByPath(coll_list_response, path);
+	return response;
+
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/PageAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/PageAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/PageAction.java	(revision 21112)
@@ -0,0 +1,207 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.io.File;
+
+public class PageAction extends Action {
+
+    public static final String HOME_PAGE = "home";
+    public static final String ABOUT_PAGE = "about";
+    public static final String PREFS_PAGE = "pref";
+    public Element process (Element message) {
+	
+	Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	// the page name is the subaction
+	String page_name = request.getAttribute(GSXML.SUBACTION_ATT);
+	if (page_name.equals("")) { // if no page specified, assume home page
+	    page_name = HOME_PAGE;
+	}
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element response;
+	if (page_name.equals(HOME_PAGE)) {
+	    response = homePage(request);
+	} else if (page_name.equals(ABOUT_PAGE)) {
+	    response = aboutPage(request);
+//  	} else if (page_name.equals(PREFS_PAGE)) {
+//  	    response = prefsPage(request);
+	} else { // unknown page
+	    
+	    System.err.println("PageAction Error: unknown page specified!");
+	    response =  unknownPage(request);
+	}
+	
+	result.appendChild(this.doc.importNode(response, true));
+	///ystem.out.println("page action result: "+this.converter.getPrettyString(result));
+	return result;
+    }
+
+
+    protected Element homePage(Element request) {
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	// first, get the message router info
+	Element coll_list_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element coll_list_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", lang, uid);
+	coll_list_message.appendChild(coll_list_request);
+	Element coll_list_response = (Element)this.mr.process(coll_list_message);
+	if (coll_list_response==null) {
+	    System.err.println("PageAction Error: couldn't query the message router!");
+	    return null;
+	}
+	
+	// second, get the metadata for each collection - we only want specific
+	// elements but for now, we'll just get it all
+	NodeList colls = coll_list_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+	// we will send all the requests in a single message
+	Element metadata_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	for (int i=0; i<colls.getLength(); i++) {
+	    Element c = (Element)colls.item(i);
+	    String name = c.getAttribute(GSXML.NAME_ATT);	    
+	    
+	    Element metadata_request = GSXML.createBasicRequest(this.doc,  GSXML.REQUEST_TYPE_DESCRIBE, name, lang, uid);
+	    metadata_message.appendChild(metadata_request);
+	}
+
+	Element metadata_response = (Element)this.mr.process(metadata_message);
+
+	NodeList coll_responses = metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	// check that have same number of responses as collections
+	if (colls.getLength() != coll_responses.getLength()) {
+	    System.err.println("PageAction Error: didn't get a response for each collection - somethings gone wrong!");
+	    // for now, dont use the metadata
+	} else {
+	    for (int i=0; i<colls.getLength(); i++) {
+		Element c1 = (Element)colls.item(i);
+		Element c2 = (Element)coll_responses.item(i);
+		if (c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.FROM_ATT))) {
+		    //add the collection data into the original response
+		    GSXML.mergeElements(c1, (Element)GSXML.getChildByTagName(c2, GSXML.COLLECTION_ELEM));
+		} else {
+		    System.err.println("PageAction Error: response does not correspond to request!");
+		} 
+		
+	    }
+	}
+	
+	//now the full response message is in coll_list_response
+	// return the response element
+	Element response = (Element) GSXML.getChildByTagName(coll_list_response, GSXML.RESPONSE_ELEM);
+	return response;
+	
+    } // homePage
+
+    protected Element aboutPage(Element request) {
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	// extract the params from the cgi-request, 
+	Element cgi_paramList = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_paramList, false);
+	
+	String coll_name = (String)params.get(GSParams.COLLECTION);
+	if (coll_name == null || coll_name.equals("")) {
+	    System.err.println("PageAction Error: about page requested with no collection or cluster specified!");
+	    // return an empty response
+	    return this.doc.createElement(GSXML.RESPONSE_ELEM);
+	}
+	
+	// get the collection or cluster description
+	Element coll_about_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	Element coll_about_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE,  coll_name, lang, uid);
+	coll_about_message.appendChild(coll_about_request);
+	
+	Element coll_about_response = (Element)this.mr.process(coll_about_message);
+
+	if (coll_about_response == null) {
+	    return null;
+	}
+	
+	// second, get the info for each service - we only want display items 
+	// but for now, we'll just get it all
+	NodeList services = coll_about_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+	// we will send all the requests in a single message
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	for (int i=0; i<services.getLength(); i++) {
+	    Element c = (Element)services.item(i);
+	    String name = c.getAttribute(GSXML.NAME_ATT);
+	    String address = GSPath.appendLink(coll_name, name);
+	    Element info_request = GSXML.createBasicRequest(this.doc,  GSXML.REQUEST_TYPE_DESCRIBE, address, lang, uid);
+	    info_message.appendChild(info_request);
+	}
+
+	Element info_response = (Element)this.mr.process(info_message);
+
+	NodeList service_responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	// check that have same number of responses as collections
+	if (services.getLength() != service_responses.getLength()) {
+	    System.err.println("PageAction Error: didn't get a response for each service - somethings gone wrong!");
+	    // for now, dont use the metadata
+	} else {
+	    for (int i=0; i<services.getLength(); i++) {
+		Element c1 = (Element)services.item(i);
+		Element c2 = (Element)GSXML.getChildByTagName((Element)service_responses.item(i), GSXML.SERVICE_ELEM);
+		if (c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.NAME_ATT))) {
+		    //add the service data into the original response
+		    GSXML.mergeElements(c1, c2);
+		} else {
+		    System.err.println("PageAction Error: response does not correspond to request!");
+		} 
+		
+	    }
+	}
+	
+	//now the full response message is in coll_list_response
+	// return the response element
+	Element response = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.RESPONSE_ELEM);
+	return response;
+    }
+
+//      protected Element prefsPage(Element request) {
+
+//  	return null;
+//      }
+
+    /** if we dont know the page type, use this method */
+    protected Element unknownPage(Element request) {
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String page_name = request.getAttribute(GSXML.SUBACTION_ATT);
+
+	// extract the params from the cgi-request, 
+	Element cgi_paramList = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_paramList, false);
+	
+	String coll_name = (String)params.get(GSParams.COLLECTION);
+	if (coll_name == null || coll_name.equals("")) {
+	    // just return an empty response
+	    return this.doc.createElement(GSXML.RESPONSE_ELEM);
+	}
+	
+	// else get the coll description - actually this is the same as for the about page - should we merge these two methods??
+	
+	// if there is a service specified should we get the service description instead??
+	// get the collection or cluster description
+	Element coll_about_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	Element coll_about_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE,  coll_name, lang, uid);
+	coll_about_message.appendChild(coll_about_request);
+	
+	Element coll_about_response = (Element)this.mr.process(coll_about_message);
+
+	// add the response to the page get teh response element and return it
+	Element response = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.RESPONSE_ELEM);
+	return response;
+	
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/ProcessAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/ProcessAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/ProcessAction.java	(revision 21112)
@@ -0,0 +1,137 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+
+public class ProcessAction extends Action {
+    
+    /** process a request */
+    public Element process (Element message) {
+
+	// the result
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.appendChild(page_response);
+
+	// assume only one request
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// get the param list
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+	String service_name = (String) params.get(GSParams.SERVICE);
+	String cluster_name = (String) params.get(GSParams.CLUSTER);
+	String response_only_p = (String)params.get(GSParams.RESPONSE_ONLY);
+	boolean response_only = false;
+	if (response_only_p!=null) {
+	    response_only = (response_only_p.equals("1")?true:false);
+	}
+	String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	// what is carried out depends on the request_type
+	// if rt=d, then a describe request is done,
+	// is rt=r, a request and then a describe request is done
+	// if rt=s, a status request is done.
+
+	// if ro=1, then this calls for a process only page - we do the request
+	// (rt should be r or s) and just give the response straight back
+	// without any page processing
+
+
+	// where to send requests
+	String to;
+	if (cluster_name != null) {
+	    
+	    to = GSPath.appendLink(cluster_name, service_name);
+	} else {
+	    to = service_name;
+	}
+
+	if (!request_type.equals("d")) {
+	    // if rt=s or rt=r, do the request
+	    
+	    Element mr_query_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    String request_type_att;
+	    Element param_list = null;
+	    if (request_type.equals("s")) { // status
+		request_type_att = GSXML.REQUEST_TYPE_STATUS;
+		param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+		param.setAttribute(GSXML.NAME_ATT, GSParams.PROCESS_ID);
+		param.setAttribute(GSXML.VALUE_ATT, (String)params.get(GSParams.PROCESS_ID));
+		param_list.appendChild(param);
+	    } else {
+		request_type_att = GSXML.REQUEST_TYPE_PROCESS;
+		// add in the service params - except the ones only used by the action
+		HashMap service_params = (HashMap)params.get("s1");
+		if (service_params != null) { 
+		    param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		    GSXML.addParametersToList(this.doc, param_list, service_params);
+		}
+		
+	    }
+	    Element mr_query_request = GSXML.createBasicRequest(this.doc, request_type_att, to, lang, uid);
+	    if (param_list != null) {
+		mr_query_request.appendChild(param_list);
+	    }
+	    mr_query_message.appendChild(mr_query_request);
+	    
+	    Element mr_query_response = (Element)this.mr.process(mr_query_message);
+	    Element result_response = (Element)GSXML.getChildByTagName(mr_query_response, GSXML.RESPONSE_ELEM);
+	    
+	    if (response_only) {
+		// just send the reponse as is
+		return result_response;
+	    }
+	    
+	    // else append the contents of the response to the page - just the status elem for now
+	    Element status = (Element)GSXML.getChildByTagName(result_response, GSXML.STATUS_ELEM);
+	    page_response.appendChild(this.doc.importNode(status, true));
+	}
+	
+
+	// another part of the page is the service description
+
+	// request the service info for the selected service - should be cached
+	Element mr_info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element mr_info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, uid);
+	mr_info_message.appendChild(mr_info_request);
+	Element mr_info_response = (Element) this.mr.process(mr_info_message);
+
+	String path = GSXML.RESPONSE_ELEM;
+	path = GSPath.appendLink(path, GSXML.SERVICE_ELEM);
+	Element description = (Element)this.doc.importNode(GSXML.getNodeByPath(mr_info_response, path), true);
+
+	page_response.appendChild(description);
+
+	return result;
+    }
+
+    protected Element getServiceParamList(Element cgi_param_list) {
+
+	Element new_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element param;
+	NodeList cgi_params = cgi_param_list.getChildNodes();
+	for (int i=0; i<cgi_params.getLength(); i++) {
+	    Element p = (Element) cgi_params.item(i);
+	    String name = p.getAttribute(GSXML.NAME_ATT);
+	    if (name.equals(GSParams.SERVICE) || name.equals(GSParams.REQUEST_TYPE) || name.equals(GSParams.CLUSTER)) {
+		continue;
+	    }
+	    // else add it in to the list
+	    new_param_list.appendChild(this.doc.importNode(p, true));
+	}
+	return new_param_list;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/QueryAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/QueryAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/QueryAction.java	(revision 21112)
@@ -0,0 +1,257 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+
+/** action class for queries */
+public class QueryAction extends Action {
+    
+    /** process - processes a request.
+     */ 
+    public Element process (Element message) {
+
+	// get the request - assume there is only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// create the return message
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element response = basicQuery(request);
+	result.appendChild(this.doc.importNode(response, true));
+	return result;
+    }
+    
+    /** a generic query handler 
+     * this gets the service description, does the query (just passes all the 
+     * params to the service, then gets the titles for any results 
+     */
+    protected Element basicQuery(Element request) {
+	
+	// the result
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	
+	// extract the params from the cgi-request, and check that we have a coll specified
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+	
+	String request_type = (String)params.get(GSParams.REQUEST_TYPE);
+	String service_name = (String)params.get(GSParams.SERVICE);
+	String collection = (String)params.get(GSParams.COLLECTION);
+	
+	if (collection == null || collection.equals("")) {
+	    System.err.println("QueryAction Error: no collection was specified!");
+	    return page_response; // an empty response 
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String to = GSPath.appendLink(collection, service_name);
+	
+	if (request_type.indexOf("d")!=-1) {
+	    // we have been asked for the service description
+	    Element mr_info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    Element mr_info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE,  to, lang, uid);
+	    mr_info_message.appendChild(mr_info_request);
+	    
+	    // process the message
+	    Element mr_info_response = (Element) this.mr.process(mr_info_message);
+	    // the response
+	    Element service_response = (Element)GSXML.getChildByTagName(mr_info_response, GSXML.RESPONSE_ELEM);
+	    
+	    Element service_description = (Element)this.doc.importNode(GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM), true);
+	    page_response.appendChild(service_description);
+	}
+	
+	if (request_type.indexOf("r") == -1) {
+	    // just a display request, no actual processing to do
+	    return page_response;
+	}
+	
+	// check that we have some service params
+	HashMap service_params = (HashMap)params.get("s1");
+	if (service_params == null) { // no query
+	    return page_response;
+	}
+
+	// create the query request
+	Element mr_query_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element mr_query_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	mr_query_message.appendChild(mr_query_request);
+	
+	Element query_param_list = this.doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+	GSXML.addParametersToList(this.doc, query_param_list, service_params);
+	mr_query_request.appendChild(query_param_list);
+	
+	// also get the format stuff now if there is some
+	Element format_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_FORMAT, to, lang, uid);
+	mr_query_message.appendChild(format_request);
+
+	// do the query
+       	Element mr_query_response = (Element)this.mr.process(mr_query_message);
+
+	// check for errors
+	if (processErrorElements(mr_query_response, page_response)) {
+	    return page_response;
+	}
+	
+	NodeList responses = mr_query_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	Element query_response = (Element) responses.item(0);
+	Element format_response = (Element) responses.item(1);
+	
+	Element query_result_metadata_list = (Element) GSXML.getChildByTagName(query_response, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	if (query_result_metadata_list == null) {
+	    System.err.println("QueryAction: Warning: No query result metadata.\n");
+	} else { // add it into the page response
+	    page_response.appendChild(this.doc.importNode(query_result_metadata_list, true)); 
+	}
+	
+	Element query_term_info_list = (Element) GSXML.getChildByTagName(query_response, GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	if (query_term_info_list == null) {
+	    System.err.println("QueryAction: Warning: No query term information.\n");
+	} else { // add it into the page response
+	    page_response.appendChild(this.doc.importNode(query_term_info_list, true));
+	}
+	
+	// check that there are some documents - for now check the list, but later should use a numdocs metadata elem	
+	Element document_list = (Element)GSXML.getChildByTagName(query_response, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	// documentList not present if no docs found
+	if (document_list == null) {
+	    // add in a dummy doc node list - used by the display. need to think about this
+	    page_response.appendChild(this.doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER));
+	    return page_response;
+	}
+
+	// now we check to see if there is metadata already - some search services return predefined metadata. if there is some, don't do a metadata request
+	NodeList doc_metadata = document_list.getElementsByTagName(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_metadata.getLength()>0) {
+	    System.err.println("have already found metadata!");
+	    // append the doc list to the result
+	    page_response.appendChild(this.doc.importNode(document_list, true));
+	    return page_response;
+	}
+	
+	// get the metadata elements needed from the format statement if any
+	HashSet metadata_names = new HashSet();
+	metadata_names.add("Title");
+	// add in the format info to the stylesheet if there is any
+	Element format_elem = (Element)GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+	if (format_elem != null) {
+	    ///ystem.out.println("QueryAction: found a format element, adding it to the page response");
+	    // set teh format type
+	    format_elem.setAttribute(GSXML.TYPE_ATT, "search");
+	    // for now just add to the response
+	    page_response.appendChild(this.doc.importNode(format_elem, true));
+	    extractMetadataNames(format_elem, metadata_names);
+	} 
+	
+	// paging of the results is done here - we filter the list to remove unwanted entries before retrieving metadata
+	Element filtered_doc_list = filterDocList(params, service_params, document_list);
+	
+	// do the metadata request on the filtered list
+	Element mr_metadata_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	to = GSPath.appendLink(collection, "DocumentMetadataRetrieve"); // Hard-wired?
+	Element mr_metadata_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	mr_metadata_message.appendChild(mr_metadata_request);
+	
+	// just get all for now - the receptionist should perhaps pass in some
+	// metadata that it wants, and QueryAction should look through the format stuff to see if there is any other?
+	
+	Element dm_param_list  = createMetadataParamList(metadata_names);
+	
+	mr_metadata_request.appendChild(dm_param_list);
+	
+	// add in the doc node list too
+	mr_metadata_request.appendChild(filtered_doc_list);
+
+	Element mr_metadata_response = (Element) this.mr.process(mr_metadata_message);	
+	// check for errors
+	processErrorElements(mr_metadata_response, page_response);
+	
+	Element metadata_response = (Element) GSXML.getChildByTagName(mr_metadata_response, GSXML.RESPONSE_ELEM);
+
+	Element query_result_document_list = (Element) GSXML.getChildByTagName(metadata_response, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	
+	if (query_result_document_list != null) {
+	    page_response.appendChild(this.doc.importNode(query_result_document_list, true));
+	}
+	
+	///ystem.out.println("Query page:\n" + this.converter.getPrettyString(page_response));
+	return page_response;
+    }
+
+    /** this filters out some of the doc results for result paging */
+    protected Element filterDocList(HashMap params, HashMap service_params, Element orig_doc_list) {
+	
+	// check the hits_per_page param - is it a service param??
+	String hits_pp = (String) service_params.get("hitsPerPage");
+	if (hits_pp != null) {
+	    // the service is doing the paging, so we want to display all of the returned docs
+	    return (Element)this.doc.importNode(orig_doc_list, true);
+	}
+	
+	hits_pp = (String)params.get("hitsPerPage");
+	int hits = 20;
+	if (hits_pp != null && !hits_pp.equals("")) {
+	    try {
+		hits = Integer.parseInt(hits_pp);
+	    } catch (Exception e) {
+		hits=20;
+	    }
+	}
+
+	if (hits == -1) { // all
+	    return (Element)this.doc.importNode(orig_doc_list, true);
+	}
+	NodeList result_docs = orig_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	
+	int num_docs = result_docs.getLength();
+	if (num_docs <= hits) {
+	    // too few docs to do paging
+	    return (Element)this.doc.importNode(orig_doc_list, true);
+	}
+
+	// now we need our own doc list
+	Element result_list = this.doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+	String start_p = (String)params.get("startPage");
+	int start = 1;
+	if (start_p != null && !start_p.equals("")) {
+	    try {
+		start = Integer.parseInt(start_p);
+	    } catch (Exception e) {
+		start = 1;
+	    }
+	}
+
+	int start_from = (start-1)*hits;
+	int end_at = (start*hits)-1;
+
+	if (start_from > num_docs) {
+	    // something has gone wrong
+	    return result_list;
+	}
+	
+	if (end_at > num_docs) {
+	    end_at = num_docs-1;
+	}
+
+	// now we finally have the docs numbers to use
+	for (int i=start_from; i<=end_at; i++) {
+	    result_list.appendChild(this.doc.importNode(result_docs.item(i), true));
+	}
+
+	return result_list;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/SystemAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/SystemAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/SystemAction.java	(revision 21112)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document; 
+
+// other java stuff
+import java.io.File;
+import java.util.HashMap;
+
+public class SystemAction extends Action {
+    
+    /** process a request */
+    public Element process (Element message) {
+	
+	// assume only one request
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	
+	String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	// get the param list
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	String coll = (String)params.get(GSParams.SYSTEM_CLUSTER);
+
+	String to = "";
+	if (coll!=null && !coll.equals("")) {
+	    to = coll;
+	}
+
+	Element mr_request_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element mr_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_SYSTEM, to, lang, uid);
+	mr_request_message.appendChild(mr_request);
+	
+	Element system = this.doc.createElement(GSXML.SYSTEM_ELEM);
+	mr_request.appendChild(system);
+	
+	// will need to change the following if can do more than one system request at once
+	if (subaction.equals("c")) { // configure
+	    system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_CONFIGURE);
+	    String info = (String)params.get(GSParams.SYSTEM_SUBSET);
+	    system.setAttribute(GSXML.SYSTEM_SUBSET_ATT, info);
+	
+	} else {
+	    String name = (String)params.get(GSParams.SYSTEM_MODULE_NAME);
+	    String type = (String)params.get(GSParams.SYSTEM_MODULE_TYPE);
+
+	    system.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, name);
+	    system.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, type);
+	    
+	    if (subaction.equals("d")) { // delete
+		system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_DEACTIVATE);
+
+	    } else if (subaction.equals("a")) { // add
+		system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ACTIVATE);
+	    } else {
+	    // create the default response
+	    // for now just have an error
+	    System.err.println("SystemAction error: bad subaction type");
+	    Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	    result.appendChild(page_response);
+	
+	    return result;
+	    }
+	}
+
+	Element response_message = this.mr.process(mr_request_message);
+	
+	result.appendChild(GSXML.duplicateWithNewName(this.doc, (Element)GSXML.getChildByTagName(response_message, GSXML.RESPONSE_ELEM), GSXML.RESPONSE_ELEM, true));
+	return result;
+	
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/XMLDocumentAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/XMLDocumentAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/action/XMLDocumentAction.java	(revision 21112)
@@ -0,0 +1,106 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.service.TEIRetrieve;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+
+/** action class for retrieving parts of XML documents */
+public class XMLDocumentAction extends Action {
+    
+   
+    /** process - processes a request.
+     */ 
+    public Element process (Element message) {
+	
+	// get the request - assume there is only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+	// create the return message
+	Element result = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element page_response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.appendChild(page_response);
+	
+	// extract the params from the cgi-request, and check that we have a coll specified
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(cgi_param_list, false);
+
+	String collection = (String)params.get(GSParams.COLLECTION);
+	if (collection == null || collection.equals("")) {
+	    return result;
+	}
+	String doc_name = (String) params.get(GSParams.DOCUMENT);
+	if (doc_name == null || doc_name.equals("")) {
+	    return result;
+	}
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	
+	// subaction used to decide if we are returning content or structure
+	String document_mode = request.getAttribute(GSXML.SUBACTION_ATT);
+	String to = null;
+	if (document_mode.equals("text")) {
+	    to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+	} else if (document_mode.equals("toc")) {
+	    to = GSPath.appendLink(collection, "DocumentStructureRetrieve");
+	} else { // the default is text
+	    to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+	}
+	
+	// make the request to the collection
+	Element mr_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	Element ret_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	mr_message.appendChild(ret_request);
+
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	ret_request.appendChild(doc_list);
+	Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	doc.setAttribute(GSXML.NODE_ID_ATT, doc_name);
+	doc_list.appendChild(doc);
+	
+	// also add in a request for the Title metadata
+	to = GSPath.appendLink(collection, "DocumentMetadataRetrieve");
+	Element meta_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_PROCESS, to, lang, uid);
+	// copy the doc list
+	meta_request.appendChild(doc_list.cloneNode(true));
+	// add in a metadata param
+	Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	meta_request.appendChild(param_list);
+	Element param = GSXML.createParameter(this.doc, "metadata", "root_Title");
+	param_list.appendChild(param);
+	
+	// add the request to the message
+	mr_message.appendChild(meta_request);
+	
+	Element ret_response = this.mr.process(mr_message);
+	String [] links = {GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM};
+	String path = GSPath.createPath(links);
+	Element doc_node = (Element)this.doc.importNode(GSXML.getNodeByPath(ret_response, path), true);
+	page_response.appendChild(doc_node);
+
+	// get the metadata list
+	Element meta_response = (Element)ret_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1);
+	String [] mlinks  = {GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER};
+	path = GSPath.createPath(mlinks);
+	
+	Element meta_list = (Element)GSXML.getNodeByPath(meta_response, path);
+	if (meta_list != null) {
+	    doc_node.appendChild(this.doc.importNode(meta_list, true));
+	}
+	return result;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/CollectionConstructor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/CollectionConstructor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/CollectionConstructor.java	(revision 21112)
@@ -0,0 +1,97 @@
+package org.greenstone.gsdl3.build;
+
+import org.greenstone.gsdl3.util.*;
+//import java.util.Thread
+import javax.swing.event.EventListenerList;
+import org.w3c.dom.Element;
+
+/** base class for collection construction */
+public abstract class CollectionConstructor 
+    extends Thread {
+
+    /** the site in which building is to take place */
+    protected String site_home = null;
+    /** the name of the collection */
+    protected String collection_name = null;
+    /** the stage of construction */
+    protected int process_type = -1;
+    /** other arguments/parameters for the construction process - in a paramList */
+    protected Element process_params = null;
+    /** the list of listeners for the process */
+    protected EventListenerList listeners = null;
+    /** A flag used to determine if this process has been asked to cancel. */
+    protected boolean cancel = false;
+    
+    public CollectionConstructor(String name) {
+	super(name);
+	this.listeners = new EventListenerList();
+    }
+	
+    /** carry out any set up stuff - returns false if couldn't set up properly
+     */
+    public boolean configure() {
+	return true;
+    }
+    public void stopAction() {
+	this.cancel = true;
+    }
+    
+    public void setActionType(int type) {
+	this.process_type = type;
+    }
+    public void setSiteHome(String site_home) {
+	this.site_home = site_home;
+    }
+    public void setCollectionName(String coll_name) {
+	this.collection_name = coll_name;
+    }
+    public void setProcessParams(Element params) {
+	this.process_params = params;
+    }
+    public boolean addListener(ConstructionListener listener) {
+	this.listeners.add(ConstructionListener.class, listener);
+	return true;
+    }
+    public boolean removeListener(ConstructionListener listener) {
+	this.listeners.remove(ConstructionListener.class, listener);
+	return true;
+    }
+
+    protected void sendProcessBegun(ConstructionEvent evt) {
+	Object[] concerned = this.listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i+=2) {
+	    if(concerned[i] == ConstructionListener.class) {
+		((ConstructionListener)concerned[i+1]).processBegun(evt);
+	    }
+	}
+    }
+    protected void sendProcessComplete(ConstructionEvent evt) {
+	Object[] concerned = this.listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i+=2) {
+	    if(concerned[i] == ConstructionListener.class) {
+		((ConstructionListener)concerned[i+1]).processComplete(evt);
+	    }
+	}
+    }
+    
+    protected void sendProcessStatus(ConstructionEvent evt) {
+
+	Object[] concerned = this.listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i+=2) {
+	    if(concerned[i] == ConstructionListener.class) {
+		((ConstructionListener)concerned[i+1]).processStatus(evt);
+	    }
+	}
+    }
+    protected void sendMessage(ConstructionEvent evt) {
+
+	Object[] concerned = this.listeners.getListenerList();
+	for(int i = 0; i < concerned.length ; i+=2) {
+	    if(concerned[i] == ConstructionListener.class) {
+		((ConstructionListener)concerned[i+1]).message(evt);
+	    }
+	}
+    }
+    
+    abstract public void run();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructCollection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructCollection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructCollection.java	(revision 21112)
@@ -0,0 +1,140 @@
+package org.greenstone.gsdl3.build;
+import org.greenstone.gsdl3.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import java.io.File;
+
+/** the class containing the main program for collection building - is essentially a wrapper around a CollectionConstructor
+ * @see CollectionConstructor
+ */
+public class ConstructCollection 
+    implements ConstructionListener {
+    
+    static public void main(String [] args) {
+
+	String site_home = null;
+	String coll_name = null;
+	int process_mode = -1;
+	if (args.length < 4) {
+	    printUsage();
+	    return;
+	}
+
+	// parse the args
+
+	XMLConverter converter = new XMLConverter();
+	Document doc = converter.newDOM();
+	Element option_list = doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element option;
+
+	for (int i=0; i<args.length-1; i++) {
+	    String key = args[i];
+	    if (!key.startsWith("-")) {
+		continue; // ignore options that dont start with '-'
+	    }
+	    key = key.substring(1);
+	    String val = args[i+1];
+	    if (val.startsWith("-")) {
+		val=null;
+	    } else {
+		i++;
+	    }
+	    
+	    if (key.equals("site")) {
+		site_home = val;
+	    } else if (key.equals("mode")) {
+		if (val==null) {
+		    continue;
+		}
+		if (val.equals("new")) {
+		    process_mode = GS2PerlConstructor.NEW;
+		} else if (val.equals("import")) {
+		    process_mode = GS2PerlConstructor.IMPORT;
+		} else if (val.equals("build")) {
+		    process_mode = GS2PerlConstructor.BUILD;
+		} else if (val.equals("activate")) {
+		    process_mode = GS2PerlConstructor.ACTIVATE;
+		}
+
+	    } else { // an option to pass to the builder
+ 		option = doc.createElement(GSXML.PARAM_ELEM);
+		option.setAttribute(GSXML.NAME_ATT, key);
+		if (val!=null) {
+		    option.setAttribute(GSXML.VALUE_ATT, val);
+		}
+		option_list.appendChild(option);
+	    }
+	}
+	String last_arg = args[args.length-1];
+	if (last_arg.startsWith("-")) { // should be coll name
+	    System.out.println("ERROR: the last arg should be the collection name!");
+	    printUsage();
+	    return;
+	}
+	
+	coll_name = last_arg;
+	
+	// check that we have the required fields
+	if (site_home==null || process_mode == -1 || coll_name==null) {
+	    System.out.println("ERROR: you have not specified all the necessary args!");
+	    printUsage();
+	    return;
+	}
+
+	// check that the collection name is valid - ie the directory exists if this is not a new one, or doesn't exist if we are creating a new one
+	File coll_dir = new File(GSFile.collectionBaseDir(site_home, coll_name));
+	if (process_mode != GS2PerlConstructor.NEW && !coll_dir.exists()) {
+	    System.out.println("ERROR: Invalid collection ("+coll_name+").");
+	    printUsage();
+	    return;
+	} else if (process_mode == GS2PerlConstructor.NEW && 
+		   coll_dir.exists()) {
+	    System.out.println("ERROR: there is already a collection named "+coll_name);
+	    printUsage();
+	    return;
+	}
+	
+   
+	ConstructCollection processor = new ConstructCollection();
+	GS2PerlConstructor constructor = new GS2PerlConstructor("perl_build");
+	if (!constructor.configure()) {
+	    System.out.println("couldn't configure the constructor!!");
+	    return;
+	}
+	constructor.setCollectionName(coll_name);
+	constructor.setSiteHome(site_home);
+	constructor.setProcessParams(option_list);
+	constructor.setActionType(process_mode);
+	constructor.addListener(processor);
+	constructor.start();
+    }
+    
+    static protected void printUsage() {
+	System.out.println("Usage: the following arguments need to be supplied:\n -site <site-home> -mode new|import|build|activate [options] <coll-name>");
+	System.out.println("Options available are:");
+	System.out.println("-maxdocs N\t process at most N documents\n"+
+			   "-gs2\t use greenstone 2 style building");
+    }
+
+    /** This event handler used to signify that a task has been started */
+    public void processBegun(ConstructionEvent evt) {
+	System.out.println("begun: "+evt.getMessage());
+    }
+    /** This event handler used to signify that a task has been completed */
+    public void processComplete(ConstructionEvent evt){
+	System.out.println("complete: "+evt.getMessage());
+    }
+    /** This event handler used to send status updates as the task is progressing */
+    public void processStatus(ConstructionEvent evt){
+	System.out.println(evt.getMessage());
+    }
+    /**  This event handler used to send any other messages to the listeners */
+    public void message(ConstructionEvent evt){
+	System.out.println(evt.getMessage());
+    }
+
+
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionEvent.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionEvent.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionEvent.java	(revision 21112)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.build;
+
+import java.awt.AWTEvent;     
+
+public class ConstructionEvent 
+    extends AWTEvent {
+    
+    /** the event id for these events - use a random number thats
+	not reserved*/
+    private static final int EVENT_ID = RESERVED_ID_MAX+77;
+    /** The status associated with the event. */
+    private int status = -1;
+    /** Any message associated with this event. */
+    private String message = null;
+    /* Constructor.
+     * @param source The <strong>CollectionConstructor</strong> that fired this message.
+     * @param status The status code for this event          
+     * @param message A <strong>String</strong> representing any message attatched with this event.
+     */
+    public ConstructionEvent(Object source, int status, String message) {
+	super(source, EVENT_ID);
+	this.message = message;
+	this.status = status;
+    }
+    /** Gets the message associated with this event.
+     * @return The message as a <strong>String</strong> or <i>null</i>. 
+     */
+    public String getMessage() {
+	return this.message;
+    }
+    /** Gets the status associated with this event. This status can then be matched back to the constants in <strong>GShell</strong>.
+     * @return An <strong>int</strong> signifying the process status.
+     */
+    public int getStatus() {
+	return this.status;
+    }
+    /** returns a String representation of the event */
+    public String toString() {
+	return "org.greenstone.gsdl3.build.ConstructionEvent[" + this.message
++ "," + this.status + "]";
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionListener.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionListener.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/ConstructionListener.java	(revision 21112)
@@ -0,0 +1,16 @@
+package org.greenstone.gsdl3.build;
+
+import java.util.EventListener;
+
+public interface ConstructionListener 
+    extends EventListener {
+
+    /** This event handler used to signify that a task has been started */
+    public void processBegun(ConstructionEvent evt);
+    /** This event handler used to signify that a task has been completed */
+    public void processComplete(ConstructionEvent evt);
+    /** This event handler used to send status updates as the task is progressing */
+    public void processStatus(ConstructionEvent evt);
+    /**  This event handler used to send any other messages to the listeners */
+    public void message(ConstructionEvent evt);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlConstructor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlConstructor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlConstructor.java	(revision 21112)
@@ -0,0 +1,297 @@
+package org.greenstone.gsdl3.build;
+
+// greenstome classes
+import org.greenstone.gsdl3.util.*;
+
+
+// xml classes
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+//general java classes
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.util.Vector;
+
+/** CollectionConstructor class for greenstone 2 compatible building
+ * it uses the perl scripts to do the building stuff */
+public class GS2PerlConstructor 
+    extends CollectionConstructor {
+
+    public static final int NEW = 0;
+    public static final int IMPORT = 1;
+    public static final int BUILD = 2;
+    public static final int ACTIVATE = 3;
+
+    /** gsdlhome for greenstone 2 - we use the perl modules and building 
+     * scripts from there */
+    protected String gsdl2home = null; 
+    /** gsdlhome for gsdl3 - shouldn't need this eventually ??*/
+    protected String gsdl3home = null; 
+    /** gsdlos for greenstone 2 */
+    protected String gsdlos = null; 
+    /** the path environment variable */
+    protected String path = null;
+
+    public GS2PerlConstructor(String name) {
+	super(name);
+    }
+
+    /** retrieves the necessary environment variables */
+    public boolean configure() {
+	// try to get the environment variables
+	this.gsdl2home = System.getProperty("GSDLHOME");
+	this.gsdl3home = System.getProperty("GSDL3HOME");
+	this.gsdlos = System.getProperty("GSDLOS");
+	this.path = System.getProperty("PATH");
+	if (this.gsdl2home ==null ) {
+	    System.err.println("You must have greenstone2 installed, and GSDLHOME set for GS2Perl building to work!!");
+	    return false;
+	} 
+	if (this.gsdl3home == null|| this.gsdlos == null) {
+	   System.err.println("You must have GSDL3HOME and GSDLOS set for GS2Perl building to work!!");
+	    return false;
+	}  
+	if (this.path==null) {
+	    System.err.println("You must have the PATH set for GS2Perl building to work!!");
+	    return false;
+	}
+	return true;
+    }
+
+    public void run() {
+	String msg;
+	ConstructionEvent evt;
+	if (this.process_type == -1) {
+	    msg = "Error: you must set the action type";
+	    evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+	    sendMessage(evt);
+	    return;
+	}
+	if (this.site_home==null) { 
+	    msg = "Error: you must set site_home";
+	    evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+	    sendMessage(evt);
+	    return;
+	}
+	if (this.process_type != NEW && this.collection_name ==null) {
+	    msg = "Error: you must set collection_name";
+	    evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+	    sendMessage(evt);
+	    return;
+	}
+	
+	switch (this.process_type) {
+	case NEW:
+	    newCollection();
+	    break;
+	case IMPORT:
+	    importCollection();
+	    break;
+	case BUILD:
+	    buildCollection();
+	    break;
+	case ACTIVATE:
+	    activateCollection();
+	    break;
+	default:
+	    msg = "wrong type of action specified!";
+	    evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+	    sendMessage(evt);
+	    break;
+	}
+
+    }
+
+    protected void newCollection() {
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: new collection."));
+	Vector command = new Vector();
+	command.add("gs2_mkcol.pl");
+	command.add("-site");
+	command.add(this.site_home);
+	command.add("-collectdir");
+	command.add(GSFile.collectDir(this.site_home));
+	command.addAll(extractParameters(this.process_params));
+	command.add(this.collection_name);
+	String [] command_str = {};
+	command_str = (String [])command.toArray(command_str); 
+	if (runPerlCommand(command_str)) {
+	    // success!! - need to send the final completed message
+	    sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+	} // else an error message has already been sent, do nothing
+	
+    }
+
+    protected void importCollection() {
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: import collection."));
+	Vector command = new Vector();
+	command.add("import.pl");
+	command.add("-collectdir");
+	command.add(GSFile.collectDir(this.site_home));
+	command.addAll(extractParameters(this.process_params));
+	command.add(this.collection_name);
+	String [] command_str = {};
+	command_str = (String [])command.toArray(command_str); 
+
+	if (runPerlCommand(command_str)) {
+	    // success!! - need to send the final completed message
+	    sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+	} // else an error message has already been sent, do nothing
+    }
+
+     protected void buildCollection() {
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: build collection."));
+	Vector command = new Vector();
+	command.add("buildcol.pl");
+	command.add("-collectdir");
+	command.add(GSFile.collectDir(this.site_home));
+	command.addAll(extractParameters(this.process_params));
+	command.add(this.collection_name);
+	String [] command_str = {};
+	command_str = (String [])command.toArray(command_str); 
+	
+	if (runPerlCommand(command_str)) {
+	    // success!! - need to send the final completed message
+	    sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+	}// else an error message has already been sent, do nothing
+//  	    
+
+     }
+    protected void activateCollection() {
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: activate collection."));
+
+	// first check that we have a building directory
+	File build_dir = new File(GSFile.collectionBuildDir(this.site_home, this.collection_name));
+	if (!build_dir.exists()) {
+	    sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "build dir doesn't exist!"));
+	    return;
+	}
+
+	// move building to index
+	File index_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
+	if (index_dir.exists()) {
+	    sendMessage(new ConstructionEvent(this, GSStatus.INFO, "deleting index directory"));
+	    index_dir.delete();
+	    if (index_dir.exists()) {
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "index directory still exists!"));
+	    }
+	}
+
+	GSFile.moveDirectory(build_dir, index_dir);
+	if (!index_dir.exists()) {
+	    sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index dir wasn't created!"));
+	}
+
+	// run the convert coll script to convert the gs2 coll to gs3
+	Vector command = new Vector();
+	command.add("convert_coll_from_gs2.pl");
+	command.add("-collectdir");
+	command.add(GSFile.collectDir(this.site_home));
+	command.addAll(extractParameters(this.process_params));
+	command.add(this.collection_name);
+	String [] command_str = {};
+	command_str = (String [])command.toArray(command_str); 
+	
+	if (!runPerlCommand(command_str)) {
+	    // an error has occurred, dont carry on
+	    return;
+	}
+
+	// success!!  - need to send the final completed message
+	sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+    }
+
+    /** extracts all the args from the xml and returns them in a Vector */
+    protected Vector extractParameters(Element param_list) {
+
+	Vector args = new Vector();
+	if (param_list==null) {
+	    return args; // return an empty vector
+	}
+	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	
+	for (int i=0; i<params.getLength(); i++) {
+	    Element p = (Element)params.item(i);
+	    String name = p.getAttribute(GSXML.NAME_ATT);
+	    String value=p.getAttribute(GSXML.VALUE_ATT);
+	    if (!name.equals("")) {
+		args.add("-"+name);
+		if (!value.equals("")) {
+		    args.add(value);
+		}
+	    }
+	}
+	    
+	return args;
+    }
+    /** returns true if completed correctly, false otherwise */
+    protected boolean runPerlCommand(String[] command) {
+	String msg;
+	ConstructionEvent evt;
+	
+	String args[] = {"GSDLHOME="+this.gsdl2home, "GSDL3HOME="+this.gsdl3home, "GSDLOS="+this.gsdlos, "PATH="+this.path};
+	String command_str = "";
+	for(int i = 0; i < command.length; i++) {
+	    command_str = command_str + command[i] + " ";
+	}
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = "+command_str));
+	try {
+	    Runtime rt = Runtime.getRuntime();
+	    sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
+	    Process prcs = rt.exec(command, args);
+
+	    InputStreamReader eisr = new InputStreamReader( prcs.getErrorStream() );
+	    InputStreamReader stdinisr = new InputStreamReader( prcs.getInputStream() );
+	    BufferedReader ebr = new BufferedReader( eisr );
+	    BufferedReader stdinbr = new BufferedReader( stdinisr );
+	    // Captures the std err of a program and pipes it into
+	    // std in of java
+	    String eline = null;
+	    String stdinline = null;
+	    while (((eline = ebr.readLine()) != null || (stdinline = stdinbr.readLine()) != null) && !this.cancel) {
+		if(eline != null) {
+		    sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, eline));
+		}
+		if(stdinline != null) {
+		    sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, stdinline));
+		}
+	    }
+	    if(!this.cancel) {
+		// Now display final message based on exit value
+		prcs.waitFor();
+		
+		if(prcs.exitValue() == 0) {
+		    //status = OK;
+		    sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
+		    
+		} else {
+		    //status = ERROR;
+		    sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));
+		   
+		    return false;
+		    
+		}
+	    }
+	    else {
+		// I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine.
+		sendProcessStatus(new ConstructionEvent(this, GSStatus.HALTED, "killing the process"));
+		prcs.getOutputStream().close();
+		prcs.destroy();
+		//status = ERROR;
+		return false;
+	    }
+	    
+	} catch (Exception e) {
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred "+e.toString()));
+	}
+
+	// we're done, but we dont send a process complete message here cos there ight be stuff to do after this has finished.
+	return true;
+
+    }
+   
+
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlListener.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlListener.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/GS2PerlListener.java	(revision 21112)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.build;
+
+public class GS2PerlListener 
+    implements ConstructionListener {
+
+    /** a buffer holding all the messages */
+    protected StringBuffer log=null;
+    /** a buffer holding only the latest update */
+    protected StringBuffer update = null;
+
+    /** a status code */
+    protected int status = -1;
+    /** whether the process this is listeneing to is finshed */
+    protected boolean finished = false;
+
+    public GS2PerlListener() {
+	this.log = new StringBuffer();
+	this.update = new StringBuffer();
+    }
+
+    public boolean isFinished() {
+	return this.finished;
+    }
+    public String getLog() {
+	return this.log.toString();
+    }
+    public int getStatus() {
+	return this.status;
+    }
+    synchronized public String getUpdate() {
+	this.log.append(this.update);
+	String tmp = this.update.toString();
+	this.update.delete(0, this.update.length());
+	return tmp;
+    }
+
+    // do we need to synchronize the methods below?
+
+    /** This event handler used to signify that a task has been started */
+    synchronized public void processBegun(ConstructionEvent evt) {
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+	
+    }
+    /** This event handler used to signify that a task has been completed */
+    synchronized public void processComplete(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+    /** This event handler used to send status updates as the task is progressing */
+    synchronized public void processStatus(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+    /**  This event handler used to send any other messages to the listeners */
+    synchronized public void message(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/StatusDisplay.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/StatusDisplay.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/build/StatusDisplay.java	(revision 21112)
@@ -0,0 +1,188 @@
+package org.greenstone.gsdl3.build;
+
+import java.applet.Applet;
+import java.awt.*;
+import java.awt.event.*;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+import java.net.URL;
+import java.io.DataInputStream;
+
+
+/** StatusDisplay - a java applet thats sends a message at regular intervals 
+ * to a pre specified address (the library param) and then displays the 
+ * status messages received.
+ * to use, embed in a web page using something like: (note the cgi args may differ - see process.xsl for details )
+ * <applet code="org.greenstone.gsdl3.build.StatusDisplay.class" codebase='lib/java' archive='gsdl3.jar, xercesImpl.jar, jaxp.jar, xml-apis.jar' width='537' height='100'>The status display applet.<param name='library' value='mylibrary?a=pr&rt=s&c=build&o=xml&ro=1&s=ImportCollection&l=en&coll=demo'/></applet>
+ * the cgi params for the library param are dependent on what action/service 
+ *  is using the applet
+ */
+public class StatusDisplay extends Applet 
+    implements Runnable {
+
+    /** the display area for the status messages */
+    protected TextArea display;
+    /** how long to delay between requests */
+    protected int delay=3000; // 3 secs
+    /** the thread used */
+    protected Thread updator_thread;
+    /** the url for the requests */ 
+    protected URL status_cgi = null;
+    /** if true, stops sending requests, and cant be unset */
+    protected boolean completed = false;
+    // init is called if you leave the page and go back - need to somehow retrieve all the previous messages that have been obtained cos we may have left the page and gone back to it during an import or something. dont want to lose all the info.
+    public void init() {
+	//Create the text field and make it uneditable.
+	this.display = new TextArea();
+	this.display.setEditable(false);
+	this.display.setBackground(Color.white);
+	//Set the layout manager so that the text field will be
+	//as wide as possible.
+	setLayout(new java.awt.GridLayout(1,0));
+
+	//Add the text field to the applet.
+	add(this.display);
+	validate();
+
+	// add the initial status text
+	String initial_text = parameterValue("initial_text");
+	addItem(initial_text);
+
+	// check the initial error state - there may be no more messages to come
+	this.completed = GSStatus.isCompleted(Integer.parseInt(parameterValue("initial_code")));
+		
+	// we always do the same query
+	String library = parameterValue("library");
+	library = tidy_URL(library);
+	try {
+	    this.status_cgi = new URL(library);
+	} catch (Exception e) {
+	    System.out.println("Error creating URL for "+library+": "+e.getMessage());
+	}
+    }
+    
+    public void start() {
+	if (this.completed) {
+	    // do nothing
+	} else {
+	    if (this.updator_thread == null) {
+		this.updator_thread = new Thread(this);
+	    }
+	    this.updator_thread.start();
+	}
+    }
+    public void stop() {
+	this.updator_thread = null;
+    }
+    public void run() {
+	
+	if (this.completed) { // do nothing. dont know if we need this here
+	    return;
+	}
+        //Just to be nice, lower this thread's priority
+        //so it can't interfere with other processing going on.
+        Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
+
+        //Remember which thread we are.
+        Thread current_thread = Thread.currentThread();
+
+        //This is the  loop.
+        while (current_thread == this.updator_thread && !this.completed) {
+	    queryServer();
+            this.display.repaint();
+	    
+            //Delay for the specified time
+            try {
+		Thread.sleep(this.delay);
+            } catch (InterruptedException e) {
+                break;
+            }
+        }
+    }
+
+    /** add some more text to the display    */
+    void addItem(String new_text) {
+	String t = this.display.getText()+"\n"+new_text;
+	this.display.setText(t);
+	this.display.setCaretPosition(t.length()); // so that you always see the 
+	// most recent entries - ie so that it scrolls down automatically
+	repaint();
+    }
+    
+    void queryServer() {
+
+	// Send the query 
+	System.out.println("sending query: " + this.status_cgi); // this appears in the java console
+	try {
+	    DataInputStream in = new DataInputStream(this.status_cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    Element status_elem = (Element)GSXML.getChildByTagName(data_elem, GSXML.STATUS_ELEM);
+	    if (status_elem != null) {
+		String status = status_elem.getAttribute(GSXML.STATUS_ERROR_CODE_ATT);
+		System.out.println("received status "+status);
+		this.completed = GSStatus.isCompleted(Integer.parseInt(status));
+		addItem(GSXML.getNodeText(status_elem));
+	    } else {
+		System.out.println("Error - no status receieved");
+		this.completed=true;
+	    }
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query ("+this.status_cgi+"): " + e);
+	}
+
+    }
+  
+    // Get a REQUIRED string.  Stop the applet if we cannot.
+    String parameterValue(String name) {
+	try { 
+	    return getParameter(name);
+	} catch (Exception e) {
+	    System.err.println("StatusDisplay: you must give a parameter for \""+ name + "\".  Stopping.");
+	    stop();
+	}
+	return "";
+    }
+
+    /**Tidy up URLs
+     *
+     * Ensure a URL address (as string) has a protocol, host, and file.
+     */
+    String tidy_URL(String address) {
+
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+
+	}
+
+	return address;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/Collection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/Collection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/Collection.java	(revision 21112)
@@ -0,0 +1,186 @@
+/*
+ *    Collection.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.collection;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.service.*;
+
+
+// java XML classes we're using
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.io.File;
+import java.util.HashMap;
+
+/**
+ * Represents a collection in Greenstone. A collection is an extension of
+ * a ServiceCluster - it has local data that the services use.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ * @see ModuleInterface
+ */
+public class Collection 
+    extends ServiceCluster {
+
+    protected XMLTransformer transformer = null;
+    /** same as setClusterName */
+    public void setCollectionName(String name) {
+	setClusterName(name);
+    }
+
+    public Collection() {
+	super();
+	this.description = this.doc.createElement(GSXML.COLLECTION_ELEM);
+	
+    }
+	
+    /**
+     * Configures the collection.
+     *
+     * gsdlHome and collectionName must be set before configure is called.
+     *
+     * the file buildcfg.xml is located in gsdlHome/collect/collectionName
+     * collection metadata is obtained, and services loaded.
+     *
+     * @return true/false on success/fail
+     */
+    public boolean configure() {
+	
+	if (this.site_home == null || this.cluster_name== null) {
+	    System.err.println("Collection: site_home and collection_name must be set before configure called!");
+	    return false;
+	}
+	
+	Element coll_config_xml = loadCollConfigFile();
+	Element build_config_xml = loadBuildConfigFile();
+	
+	if (coll_config_xml==null||build_config_xml==null) {
+	    return false;
+	}
+	// process the metadata and display items
+	findAndLoadInfo(coll_config_xml, build_config_xml);
+	
+	// now do the services	
+	Element service_list = (Element)GSXML.getChildByTagName(build_config_xml, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+	configureServiceRack(service_list, coll_config_xml);
+	
+	return true;
+	
+    }
+    /** 
+     * load in the collection config file into a DOM Element 
+     */    
+    protected Element loadCollConfigFile() {
+
+	File coll_config_file = new File(GSFile.collectionConfigFile(this.site_home, this.cluster_name));
+	
+	if (!coll_config_file.exists()) {
+	    System.err.println("Collection: couldn't configure collection: "+this.cluster_name+", "+coll_config_file+" does not exist");
+	    return null;
+	}
+	// get the xml for both files
+	Document coll_config_doc = this.converter.getDOM(coll_config_file, CONFIG_ENCODING);
+	Element coll_config_elem = null;
+	if (coll_config_doc != null) {
+	    coll_config_elem = coll_config_doc.getDocumentElement();
+	}
+	return coll_config_elem;
+
+    }
+    
+    /** 
+     * load in the collection build config file into a DOM Element 
+     */        
+    protected Element loadBuildConfigFile() {
+	
+	File build_config_file = new File(GSFile.collectionBuildConfigFile(this.site_home, this.cluster_name));
+	if (!build_config_file.exists()) {
+	    System.err.println("Collection: couldn't configure collection: "+this.cluster_name+", "+build_config_file+" does not exist");
+	    return null;
+	}
+	Document build_config_doc = this.converter.getDOM(build_config_file, CONFIG_ENCODING);
+	Element build_config_elem = null;
+	if (build_config_doc != null) {
+	    build_config_elem = build_config_doc.getDocumentElement();
+	}
+	return build_config_elem;
+    }
+
+    /**
+     * find the metadata and display elems from the two config files and add it to the appropriate lists
+     */
+    protected boolean findAndLoadInfo(Element coll_config_xml, 
+				      Element build_config_xml){
+
+	// metadata
+	Element meta_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+	meta_list = (Element)GSXML.getChildByTagName(build_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+
+	meta_list = this.doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+	GSXML.addMetadata(this.doc, meta_list, "httpPath", this.site_http_address+"/collect/"+this.cluster_name);
+	addMetadata(meta_list);
+
+	// display stuff
+	Element display_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER);
+	if (display_list != null) {
+	    addDisplayItems(display_list);
+	}
+	return true;
+
+    }
+    /** 
+     * do a configure on only part of the collection
+     */
+    protected boolean configureSubset(String subset) {
+
+	// need the coll config files
+	Element coll_config_elem = loadCollConfigFile();
+	Element build_config_elem = loadBuildConfigFile();
+	if (coll_config_elem == null||build_config_elem == null) {
+	    // wont be able to do any of teh requests
+	    return false;
+	}    
+	
+	if (subset.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element service_rack_list = (Element)GSXML.getChildByTagName(build_config_elem, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+	    
+	    return configureServiceRack(service_rack_list, coll_config_elem);
+	} 
+
+	if (subset.equals(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER) || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    return findAndLoadInfo(coll_config_elem, build_config_elem);
+	    
+	}
+	
+	System.err.println("Collection: cant process system request, configure "+subset);
+	return false;
+    } 
+
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/ServiceCluster.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/ServiceCluster.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/ServiceCluster.java	(revision 21112)
@@ -0,0 +1,619 @@
+/*
+ *    ServiceCluster.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+// leave the package name as is for now - should be changed to something better
+// cluster? groups?
+package org.greenstone.gsdl3.collection;
+
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.service.*;
+
+// java XML classes we're using
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Iterator;
+
+/* ServiceCluster - a groups of services that are related in some way
+ * Implements ModuleInterface. Contains a list of services provided by the cluster, along with metadata about the cluster itself.
+ * a collection is a special type of cluster
+ *  @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ *  @version $Revision$
+ *  @see ModuleInterface
+ */
+public class ServiceCluster  
+    implements ModuleInterface {
+
+    protected static final String CONFIG_ENCODING = "utf-8";
+
+    protected static final String DEFAULT_LANG = "en"; // hack for now, should be read from the coll cfg file? or site cfg file for cluster
+    
+    /** base directory for the site that this cluster belongs to*/
+    protected String site_home = null;
+    /** http address of the site that this cluster belongs to */
+    protected String site_http_address = null;
+    /** The name of the cluster - for a collection, this is the collection name*/
+    protected String cluster_name = null;
+
+    /** a reference to the message router */
+    protected MessageRouter router = null;
+    /** The map of services.
+     *
+     * Maps Services to ServiceRack objects
+     * @see ServiceRack
+     *
+     */
+    protected HashMap service_map=null;
+    /** maps pseudo service names to real service names - needed if we have two services with the same name for one collection */
+    protected HashMap service_name_map=null;
+    
+    /** XML converter for String to DOM and vice versa */
+    protected XMLConverter converter=null;
+
+    /** container doc for description elements */
+    protected Document doc = null;
+    /** list of services */
+    protected Element service_list = null;
+    /** list of metadata - all metadata, regardless of language goes in here */
+    protected Element metadata_list = null;
+    /** language specific stuff */
+    //protected Element lang_specific_metadata_list = null;
+    protected Element display_item_list = null;
+    /** the element that will have any descriptions passed back in */
+    protected Element description = null;
+
+    public void setSiteHome(String home) {
+	this.site_home = home;
+    }
+
+    public void setSiteAddress(String address) {
+	this.site_http_address = address;
+    }
+
+    public void cleanUp() {
+	Iterator i = this.service_map.values().iterator();
+	while (i.hasNext()) {
+	    ServiceRack s = (ServiceRack)i.next();
+	    s.cleanUp();
+	}
+    }
+    public void setClusterName(String name) {
+	this.cluster_name = name;	
+	this.description.setAttribute(GSXML.NAME_ATT, name);
+    }  
+
+    public void setMessageRouter(MessageRouter m) {
+	this.router = m;
+    }
+
+    public ServiceCluster() {
+	this.service_map = new HashMap();
+	this.service_name_map = new HashMap();
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+	this.description = this.doc.createElement(GSXML.CLUSTER_ELEM);
+	this.display_item_list = this.doc.createElement(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER);
+	this.metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+    }
+
+    /**
+     * Configures the cluster.
+     *
+     * gsdlHome and clusterName must be set before configure is called.
+     *
+     * reads the site configuration file, and configures itself
+     * this calls configure(Element) with the XML element node from the config
+     * file.
+     * configure(Element) should be used if the config file has already been 
+     * parsed.  This method will work with any subclass.
+     * 
+     * @return true if configure successful, false otherwise.
+     */
+    public boolean configure() {
+
+	if (this.site_home == null || this.cluster_name== null) {
+	    System.err.println("ServiceCluster: site_home and cluster_name must be set before configure called!");
+	    return false;
+	}
+	System.out.println("configuring service cluster");
+	// read the site configuration file
+	File config_file = new File(GSFile.siteConfigFile(this.site_home));
+
+	if (!config_file.exists()) {
+	    System.err.println("ServiceCluster: couldn't configure cluster: "+this.cluster_name +", "+config_file+" does not exist");
+	    return false;
+	}
+
+	Document doc = this.converter.getDOM(config_file, CONFIG_ENCODING);
+	if (doc == null) {
+	    System.err.println("ServiceCluster: couldn't parse config file "+config_file.getPath());
+	    return false;
+	}
+	
+	// get the appropriate service cluster element
+	Element cluster_list = (Element)GSXML.getChildByTagName(doc.getDocumentElement(), GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER);
+	Element sc = GSXML.getNamedElement(cluster_list, GSXML.CLUSTER_ELEM,
+					   GSXML.NAME_ATT, this.cluster_name);
+	
+	return this.configure(sc);
+    }
+    
+    
+    public boolean configure(Element service_cluster_info) {
+	
+	// get the metadata - for now just add it to the list
+	Element meta_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	if (meta_list !=null) {
+	    if (!addMetadata(meta_list)) {
+		
+		System.err.println("ServiceCluster: couldn't configure the metadata");
+	    }
+	}
+	
+	// get the display info
+	Element display_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER);
+	if (display_list !=null) {
+	    if (!addDisplayItems(display_list)) {
+		
+		System.err.println("ServiceCluster: couldn't configure the display items");
+	    }
+	}
+	
+
+	// do the service racks
+	Element service_rack_list = (Element)GSXML.getChildByTagName(service_cluster_info, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+	if (service_rack_list == null) {
+	    // is this an error? could you ever have a service cluster
+	    // without service racks???
+	    System.err.println("ServiceCluster: cluster has no service racks!!");
+	} else {
+	    
+	    if (!configureServiceRack(service_rack_list, null)) {
+		System.err.println("ServiceCluster: couldn't configure the  service racks!!");
+		return false;
+	    }
+	}
+
+	return true;
+    }
+    
+    /** adds metadata from a metadataList into the metadata_list xml 
+     */
+    protected boolean addMetadata(Element metadata_list) {
+	if (metadata_list == null) return false;
+	NodeList metanodes = metadata_list.getElementsByTagName(GSXML.METADATA_ELEM);
+	if (metanodes.getLength()>0) { 	
+	    for(int k=0; k<metanodes.getLength(); k++) {
+		this.metadata_list.appendChild(this.doc.importNode(metanodes.item(k), true));
+	    }
+	}
+	
+	return true;
+    }
+
+    protected boolean addDisplayItems(Element display_list) {
+	
+	if (display_list==null) return false;
+	NodeList displaynodes = display_list.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+	if (displaynodes.getLength()>0) { 	
+	    for(int k=0; k<displaynodes.getLength(); k++) {
+		Element d = (Element) displaynodes.item(k);
+		String lang = d.getAttribute(GSXML.LANG_ATT);
+		if (lang==null||lang.equals("")) {
+		    //set the lang to teh default
+		    d.setAttribute(GSXML.LANG_ATT, DEFAULT_LANG);
+		}
+		String name = d.getAttribute(GSXML.NAME_ATT);
+		Element this_item = GSXML.getNamedElement(this.display_item_list, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, name);
+		if (this_item==null) {
+		    this_item = this.doc.createElement(GSXML.DISPLAY_TEXT_ELEM);
+		    this_item.setAttribute(GSXML.NAME_ATT, name);
+		    this.display_item_list.appendChild(this_item);
+		}
+		
+		this_item.appendChild(this.doc.importNode(d, true));
+		    
+		
+	    }
+	}
+	
+	return true;
+    }
+    
+    /** creates and configures all the services - extra_info is some more xml 
+that is passed to teh service  - eg used for coll config files for Collection
+    */
+    protected boolean configureServiceRack(Element service_rack_list, 
+					   Element extra_info) {
+
+	// empty the service map in case this is a reconfigure
+	service_map.clear();
+	this.service_list = this.doc.createElement(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER);
+
+	// create all the services
+	NodeList nodes = service_rack_list.getElementsByTagName(GSXML.SERVICE_CLASS_ELEM);
+	if (nodes.getLength()==0) {
+	    System.err.println("ServiceCluster configuration error: cluster "+this.cluster_name+" has no service modules!");
+	    return false;
+	}
+
+	for(int i=0; i<nodes.getLength(); i++) {
+	
+	    // the xml request to send to the serviceRack to query what
+	    // services it provides
+	    Element message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    Element request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", "", "");
+	    message.appendChild(request);
+
+	    Element n = (Element)nodes.item(i);
+	    String servicetype = n.getAttribute(GSXML.NAME_ATT);
+	    
+	    try {
+		ServiceRack s = (ServiceRack)Class.forName("org.greenstone.gsdl3.service."+servicetype).newInstance();
+
+		s.setSiteHome(this.site_home);
+		s.setSiteAddress(this.site_http_address);
+		s.setClusterName(this.cluster_name);
+		s.setMessageRouter(this.router);
+		// pass the xml node to the service for configuration
+		if (s.configure(n, extra_info)) {
+		    
+		    // find out the supported service types for this service module
+		    Node types = s.process(message);
+		    NodeList typenodes = ((Element)types).getElementsByTagName(GSXML.SERVICE_ELEM);	   
+		    
+		    for (int j=0; j<typenodes.getLength();j++) {
+			String service = ((Element) typenodes.item(j)).getAttribute(GSXML.NAME_ATT);	   
+			if (service_map.get(service)!=null) {
+			    char extra = '0';
+			    String new_service = service+extra;	   
+
+			    while (service_map.get(new_service)!=null) {
+				extra++;
+				new_service = service+extra;
+			    }
+			    this.service_name_map.put(new_service, service);
+			    service=new_service;
+			    ((Element) typenodes.item(j)).setAttribute(GSXML.NAME_ATT, service);
+			}
+			this.service_map.put(service, s);
+			// also add info to the ServiceInfo XML element
+			this.service_list.appendChild(this.doc.importNode(typenodes.item(j), true));
+		    }
+		}
+	    } catch (Exception e) {
+		System.err.println("ServiceCluster Error:  configure exception: couldn't create service module:org.greenstone.gsdl3.service."+servicetype+"\n"+e.getMessage() + e.getClass() );
+		e.printStackTrace();
+		//return false;
+	    }
+	    
+	}
+	return true;
+	
+	
+    }
+
+
+    /**
+     * Process an XML document - uses Strings 
+     *  just calls process(Node).
+     *
+     * @param in the Document to process - a string
+     * @return the resultant document as a string - contains any error messages
+     * @see String
+     */
+    public String process(String in) {
+
+	Document doc = this.converter.getDOM(in);
+	Element e = doc.getDocumentElement();
+	
+	Element res = process(e);
+	return this.converter.getString(res);
+	
+    }
+    
+    /** process XML as Node
+     *
+     */
+    public Element process(Element message) {
+
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+	Document mess_doc = message.getOwnerDocument();
+	Element mainResult = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	if (requests.getLength()==0) {
+	    System.err.println("ServiceCluster.process: no requests for cluster:"+this.cluster_name);
+	    // no requests
+	    return mainResult; // for now
+	}
+	for (int i=0; i<requests.getLength(); i++) {
+	    Element request = (Element)requests.item(i);
+	    String to = request.getAttribute(GSXML.TO_ATT);
+
+	    // the cluster name should be first, check, then remove
+	    String clustername = GSPath.getFirstLink(to);
+	    if (!clustername.equals(this.cluster_name)){ 
+		System.err.println("ServiceCluster Error: cluster name wrong! was "+clustername+" should have been "+this.cluster_name);
+		continue; // ignore this request
+	    }
+	    to = GSPath.removeFirstLink(to);
+	    request.setAttribute(GSXML.TO_ATT, to);
+	   
+	    if (to.equals("")) { // this command is for me
+		Element response = processMessage(request);
+		mainResult.appendChild(response);
+		
+	    } else { // the request is for one of my services
+		String service = GSPath.getFirstLink(to);
+		
+		if (!this.service_map.containsKey(service)) {
+		    System.err.println("ServiceCluster Error: non-existant service, "+service+", specified!");
+		    continue;
+		}
+		String real_service = service;
+		if (this.service_name_map.containsKey(service)) {
+		    real_service = (String)this.service_name_map.get(service);
+		    // need to change the to att in the request - give the real service name
+		    to = request.getAttribute(GSXML.TO_ATT);
+		    String old_to = to;
+		    to = GSPath.replaceFirstLink(to, real_service);
+		    request.setAttribute(GSXML.TO_ATT, to);
+		}
+		// have to pass the request to the service
+		Element single_message = mess_doc.createElement(GSXML.MESSAGE_ELEM);
+		single_message.appendChild(request);
+		Element response_message = ((ModuleInterface)this.service_map.get(service)).process(single_message);
+		if (response_message != null) {
+		    Element response = (Element) GSXML.getChildByTagName(response_message, GSXML.RESPONSE_ELEM);
+		    String from = response.getAttribute(GSXML.FROM_ATT);
+		    if (!real_service.equals(service)) {
+			// replace the real service name with the pseudo service name
+			from = GSPath.replaceFirstLink(from, service);
+			// also need to do it in the service itself
+			// shoudl this be done here??
+			Element service_elem = (Element) GSXML.getChildByTagName(response, GSXML.SERVICE_ELEM);
+			if (service_elem!= null) {
+			    service_elem.setAttribute(GSXML.NAME_ATT, service);
+			}
+		    }
+		    from = GSPath.prependLink(from, this.cluster_name);
+		    response.setAttribute(GSXML.FROM_ATT, from);
+		    mainResult.appendChild(this.doc.importNode(response, true));
+		}
+		  
+	    } // else
+		
+	   
+	} // for each request
+	return mainResult;
+    }
+
+    /** handles requests made to the ServiceCluster itself 
+     *
+     * @param req - the request Element- <request>
+     * @return the result Element - should be <response>
+     */
+    protected Element processMessage(Element request) {
+
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+	String type = request.getAttribute(GSXML.TYPE_ATT);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	response.setAttribute(GSXML.TYPE_ATT, type);
+	
+	if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE)) {
+	    // create the collection element
+	    Element description = (Element)this.description.cloneNode(false);
+	    response.appendChild(description);
+	    // check the param list
+	    Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    if (param_list == null) {
+		addAllDisplayInfo(description, lang);
+		description.appendChild(this.service_list);
+		description.appendChild(this.metadata_list);
+		return response;
+	    }
+	    
+	    // go through the param list and see what components are wanted
+	    NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	    for (int i=0; i<params.getLength(); i++) {
+		
+		Element param = (Element)params.item(i);
+		// Identify the structure information desired
+		if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM)) {
+		    String info = param.getAttribute(GSXML.VALUE_ATT);
+		    if (info.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(this.service_list);
+		    } else if (info.equals(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(metadata_list);
+		    } else if (info.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+			addAllDisplayInfo(description, lang);
+			
+		    }
+		}
+	    }
+	    return response;
+	}
+	
+	if (type.equals(GSXML.REQUEST_TYPE_SYSTEM)) {
+	    response = processSystemRequest(request);
+	} else { // unknown type
+	    System.err.println("ServiceCluster Error: cant handle request of type "+ type);
+	    
+	}
+	return response;
+    }
+
+    protected Element processSystemRequest(Element request) {
+
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+	response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_SYSTEM);
+
+	// a list of system requests - should put any error messages
+	// or success messages into response
+	NodeList commands = request.getElementsByTagName(GSXML.SYSTEM_ELEM);
+	String message=null;	
+	for (int i=0; i<commands.getLength(); i++) {
+	    // all the commands should be Elements
+	    Element elem = (Element)commands.item(i);
+	    String action = elem.getAttribute(GSXML.TYPE_ATT);
+	    if (action.equals(GSXML.SYSTEM_TYPE_CONFIGURE)) {
+		String subset = elem.getAttribute(GSXML.SYSTEM_SUBSET_ATT);
+		if (subset.equals("")) {
+		    // need to reconfigure the service cluster
+		    
+		    if (this.configure()) {
+			Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, this.cluster_name + " reconfigured");
+			response.appendChild(s);
+			
+		    } else {
+			Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, this.cluster_name + " could not be reconfigured");
+			response.appendChild(s); 
+		    }
+		} else if (this.configureSubset(subset)) {
+		    Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, this.cluster_name + " "+subset+" reconfigured");
+		    response.appendChild(s);
+		} else {
+		    Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, this.cluster_name + " "+subset + " could not be reconfigured");
+		    response.appendChild(s); 
+		}
+		continue;
+	    } // configure action
+	    
+	    String module_name = elem.getAttribute(GSXML.SYSTEM_MODULE_NAME_ATT);
+	    String module_type = elem.getAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT);
+	    if (action.equals(GSXML.SYSTEM_TYPE_ACTIVATE)) {
+		Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, "activate action not yet implemented - does it even make sense in this context??");
+		response.appendChild(s);
+	    } else if (action.equals(GSXML.SYSTEM_TYPE_DEACTIVATE)) {
+		if (module_type.equals(GSXML.SERVICE_ELEM)) {
+		    // deactivate the service
+		    // remove from service_map
+		    this.service_map.remove(module_name);
+		    Element service_elem = GSXML.getNamedElement(this.service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, module_name);
+		    service_list.removeChild(service_elem);		    
+		    message = module_type+": "+module_name+" deactivated";
+		} else {
+		    message = "can't deactivate "+module_type+" type modules!";}
+		Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, message);
+		response.appendChild(s);
+	    } else {
+		System.err.println("ServiceCluster: cant process system request, action "+action);
+		continue;
+	    }
+	} // for each command
+	return response;
+    }
+
+    /** 
+     * do a configure on only part of the collection
+     */
+    protected boolean configureSubset(String subset) {
+	
+	File configFile = new File(GSFile.siteConfigFile(this.site_home));
+	if (!configFile.exists() ) {
+	    System.err.println("ServiceCluster: site config file: "+configFile.getPath()+" not found!");
+	    // wont be able to do any of the requests
+	    return false;
+	    
+	}
+	
+	Document site_config_doc  = this.converter.getDOM(configFile);
+	if (site_config_doc == null) {
+	    System.err.println("ServiceCluster: could not read in site config file: "+configFile.getPath());
+	    return false;
+	}
+	
+	Element site_config_elem = site_config_doc.getDocumentElement();
+	Element cluster_config_elem = GSXML.getNamedElement((Element)GSXML.getChildByTagName(site_config_elem, GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER), GSXML.CLUSTER_ELEM, GSXML.NAME_ATT, this.cluster_name);
+	if (cluster_config_elem == null) {
+	    System.err.println("ServiceCluster: site config file: "+configFile.getPath()+" has no element for cluster "+this.cluster_name);
+	    // wont be able to do any of teh requests
+	    return false;
+	    
+	}
+	if (subset.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element service_rack_list = (Element)GSXML.getChildByTagName(cluster_config_elem, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+	    
+	    return configureServiceRack(service_rack_list, null);
+	} else if (subset.equals(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER)) {
+	    this.metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	    Element metadata_list = (Element)GSXML.getChildByTagName(cluster_config_elem, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	    return addMetadata(metadata_list);
+	} else {
+	    System.err.println("ServiceCluster: cant process system request, configure "+subset);
+	    return false;
+	} 
+
+    } 
+
+
+    protected boolean addAllDisplayInfo(Element description, String lang) {
+	
+	NodeList items = this.display_item_list.getChildNodes();
+	for (int i=0; i<items.getLength(); i++) { // for each key
+	    Element m = (Element) items.item(i);
+	    // find the child with the correct language
+	    Element new_m = GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, lang);
+	    if (new_m==null && lang != DEFAULT_LANG) {
+		// use the default lang
+		new_m = GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, DEFAULT_LANG);
+	    }
+	    if (new_m==null) {
+		// just get the first one
+		new_m = (Element)GSXML.getChildByTagName(m, GSXML.DISPLAY_TEXT_ELEM);
+	    }
+	    description.appendChild(new_m.cloneNode(true));
+	}
+	return true;
+	
+    }
+
+    
+    protected Element getDisplayTextElement(String key, String lang) {
+	
+	Element this_item = GSXML.getNamedElement(this.display_item_list, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, key);
+	if (this_item == null) {
+	    return null;
+	}
+
+	Element this_lang = GSXML.getNamedElement(this_item, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, lang);
+	if (this_lang == null && lang != DEFAULT_LANG) {
+	    // try the default
+	    this_lang = GSXML.getNamedElement(this_item, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, DEFAULT_LANG);
+	}
+	if (this_lang == null) {
+	    // just return the first one
+	    return (Element)this_item.getFirstChild().cloneNode(true);
+	}
+	return (Element)this_lang.cloneNode(true);
+	
+    }
+}
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/XMLCollection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/XMLCollection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/collection/XMLCollection.java	(revision 21112)
@@ -0,0 +1,114 @@
+package org.greenstone.gsdl3.collection;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.service.*;
+
+
+// java XML classes we're using
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.io.File;
+import java.util.HashMap;
+
+/* for a collection that hasn't been built with greenstone build stuff. expects a documentList in the collectionConfig file, and it stores this. this is where doc level metadata comes from */
+public class XMLCollection 
+    extends Collection {
+
+    protected Element document_list = null;
+    /** overwrite this to keep the document list
+     * find the metadata and display elems from the two config files and add it to the appropriate lists
+     */
+    protected boolean findAndLoadInfo(Element coll_config_xml, 
+				      Element build_config_xml){
+
+	// metadata
+	Element meta_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+	meta_list = (Element)GSXML.getChildByTagName(build_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+
+	meta_list = this.doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+	GSXML.addMetadata(this.doc, meta_list, "httpPath", this.site_http_address+"/collect/"+this.cluster_name);
+	addMetadata(meta_list);
+
+	// display stuff
+	Element display_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER);
+	if (display_list != null) {
+	    addDisplayItems(display_list);
+	}
+
+	Element config_doc_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+	if (config_doc_list != null) {
+	    document_list = (Element)this.doc.importNode(config_doc_list, true);
+	} else {
+	    document_list = this.doc.createElement(GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+	}
+	return true;
+
+    }
+
+    /** handles requests made to the ServiceCluster itself 
+     *
+     * @param req - the request Element- <request>
+     * @return the result Element - should be <response>
+     */
+    protected Element processMessage(Element request) {
+
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+	String type = request.getAttribute(GSXML.TYPE_ATT);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	response.setAttribute(GSXML.TYPE_ATT, type);
+	
+	if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE)) {
+	    // create the collection element
+	    Element description = (Element)this.description.cloneNode(false);
+	    response.appendChild(description);
+	    // check the param list
+	    Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    if (param_list == null) {
+		addAllDisplayInfo(description, lang);
+		description.appendChild(this.service_list);
+		description.appendChild(this.metadata_list);
+		description.appendChild(this.document_list);
+		return response;
+	    }
+	    
+	    // go through the param list and see what components are wanted
+	    NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	    for (int i=0; i<params.getLength(); i++) {
+		
+		Element param = (Element)params.item(i);
+		// Identify the structure information desired
+		if (param.getAttribute(GSXML.NAME_ATT) == GSXML.SUBSET_PARAM ) {
+		    String info = param.getAttribute(GSXML.VALUE_ATT);
+		    if (info.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(this.service_list);
+		    } else if (info.equals(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(metadata_list);
+		    } else if (info.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+			addAllDisplayInfo(description, lang);
+			
+		    } else if (info.equals(GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(this.document_list);
+		    }
+		    
+		}
+	    }
+	    return response;
+	}
+	
+	if (type.equals(GSXML.REQUEST_TYPE_SYSTEM)) {
+	    response = processSystemRequest(request);
+	} else { // unknown type
+	    System.err.println("ServiceCluster Error: cant handle request of type "+ type);
+	    
+	}
+	return response;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/Communicator.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/Communicator.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/Communicator.java	(revision 21112)
@@ -0,0 +1,69 @@
+/*
+ *    Communicator.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.comms;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+
+//XML packages
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+/** Communicator - base class for Modules that talk via some protocol to other modules
+ *
+ * can be used by a MessageRouter - in this case set localSiteName
+ * can be used by any other module - in this case it has no local site - dont set localsiteName. the setting of localSiteName affects the handling of teh to and from fields in the message. 
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+abstract public class Communicator
+    implements ModuleInterface {
+
+    /** name of local site */
+    protected String local_site_name_ = null;
+
+    /** name of site to connect to */
+    protected String remote_site_name_=null;
+
+    /** converter for String to DOM and vice versa */
+    protected XMLConverter converter_= null;
+
+    public Communicator() {
+	converter_ = new XMLConverter();
+    }
+    public void cleanUp() {}
+
+    public void setLocalSiteName(String name) {
+	local_site_name_ = name;
+    }
+    /** this should be done as part of configure */
+    public void setRemoteSiteName(String name) {
+	remote_site_name_ = name;
+    }
+    /** configures the Communicator using the <site> element */
+    abstract public boolean configure(Element site_elem); 
+
+    public String process(String xml_in) {
+	Element e = converter_.getDOM(xml_in).getDocumentElement();
+	Node result = process(e);
+	return converter_.getString(result);	
+    }
+
+    abstract public Element process(Element xml_in);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/SOAPCommunicator.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/SOAPCommunicator.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/comms/SOAPCommunicator.java	(revision 21112)
@@ -0,0 +1,270 @@
+/*
+ *    SOAPCommunicator.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.comms;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+
+// classes needed for SOAP stuff
+import java.net.URL;
+import java.net.MalformedURLException;
+import java.util.Vector;
+// import org.apache.soap.SOAPException;
+// import org.apache.soap.Fault;
+// import org.apache.soap.Constants;
+// import org.apache.soap.rpc.Call;
+// import org.apache.soap.rpc.Parameter;
+// import org.apache.soap.rpc.Response;
+import org.apache.axis.client.Call;
+import org.apache.axis.client.Service;
+import org.apache.axis.message.SOAPBodyElement;
+import javax.xml.namespace.QName;
+
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+/*
+ * The Client side of a SOAP Connection
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ * @see <a href="http://www.w3.org/TR/SOAP/">Simple Object Access Protocol (SOAP) 1.1 </a>
+ */
+public class SOAPCommunicator
+    extends Communicator {
+
+    /** address of site to connect to */
+    protected String remote_site_address_=null;
+
+    /** the call object that does the SOAP talking */
+    private Call call_ = null;
+
+  /** The no-args constructor */
+    public SOAPCommunicator() {
+	
+
+	// Set Encoding Style to standard SOAP encoding
+	//call_.setEncodingStyleURI(Constants.NS_URI_SOAP_ENC);
+	// set Encoding Style to use literal XML
+	//call_.setEncodingStyleURI(Constants.NS_URI_LITERAL_XML);
+    }
+        
+    public boolean configure(Element site_elem) {
+
+	
+	String type = site_elem.getAttribute(GSXML.TYPE_ATT);
+	if (!type.equals(GSXML.COMM_TYPE_SOAP_JAVA)) {
+	    System.err.println("SOAPCommunicator: wrong type of site");
+	    return false;
+	}
+	remote_site_name_ = site_elem.getAttribute(GSXML.NAME_ATT);
+	if (remote_site_name_.equals("")) {
+	    System.err.println("SOAPCommunicator: must have name attribute in site element");
+	    return false;
+	}
+	remote_site_address_ = site_elem.getAttribute(GSXML.ADDRESS_ATT);
+	String local_site_name = site_elem.getAttribute(GSXML.LOCAL_SITE_ATT);
+	if (remote_site_address_.equals("") && local_site_name.equals("")) {
+	    System.err.println("SOAPCommunicator: must have address or localSite attributes in site element");
+	    return false;
+	}
+	if (remote_site_address_.equals("")) {
+	    remote_site_address_ = GlobalProperties.getGSDL3WebAddress()+"/services/"+local_site_name;
+	}
+
+	try {
+	    Service service = new Service();
+	    call_ = (Call) service.createCall();
+	    
+	    //call_.setTargetObjectURI(remote_site_name_);
+	    //call_.setMethodName("process");
+	    call_.setTargetEndpointAddress( new java.net.URL(remote_site_address_) );
+	    //	    call_.setOperationStyle(org.apache.axis.enum.Style.MESSAGE_STR);
+	    //call_.setOperationUse(org.apache.axis.enum.Use.LITERAL_STR); 
+	    //call_.setOperationName(new QName("http://soapinterop.org/", "process"));
+	} catch (Exception e) {
+	    System.err.println("SOAPCommunicator.configure() Error: Exception occurred "+e);
+	    return false;
+	}
+	return true;
+    }
+
+    public Element process(Element message) {
+	
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+	if (requests.getLength()==0) {
+	    // no requests
+	    return null;
+	}
+
+	// if the communicator is part of a site, it needs to edit the to and from fields, otherwise it just passes the message on, as is
+	// for now, use local_site_name_ as the flag.
+	if (local_site_name_!=null) { // no local site
+	    
+	    for (int i=0;i<requests.getLength(); i++) {
+		Element e = (Element)requests.item(i);
+		String to = e.getAttribute(GSXML.TO_ATT);
+		to = GSPath.removeFirstLink(to); // remove the server name, should check that it is present
+		e.setAttribute(GSXML.TO_ATT, to);
+		String from = e.getAttribute(GSXML.FROM_ATT);
+		from = GSPath.appendLink(from, local_site_name_);
+		e.setAttribute(GSXML.FROM_ATT, from);
+	    }
+	}
+	// else do nothing to the requests, just pass it on
+
+	// the soap impl needs a namespace for the top level element
+	Element message_to_send = message;
+	if (message.getNamespaceURI() == null) {
+	    message_to_send = GSXML.duplicateWithNewNameNS(message.getOwnerDocument(), message, "gs3:"+message.getTagName(), "urn:foo", true);
+	    //System.err.println("**"+new XMLConverter().getPrettyString(new_message));
+	}
+	
+	// do the soap query
+	Element result = null;
+	try {
+	    SOAPBodyElement[] input = new SOAPBodyElement[1];
+	    input[0] = new SOAPBodyElement(message_to_send);
+	    Vector output = (Vector) call_.invoke( input );
+	    SOAPBodyElement elem = (SOAPBodyElement) output.get(0);
+	    result = elem.getAsDOM();
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+	}
+	
+	if (local_site_name_ != null) {// have to modify the from field
+	    
+	    NodeList responses = result.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	    for (int i=0;i<responses.getLength(); i++) {
+		Element e = (Element)responses.item(i);
+		String from = e.getAttribute(GSXML.FROM_ATT);
+		from = GSPath.prependLink(from, remote_site_name_);
+		e.setAttribute(GSXML.FROM_ATT, from);
+	    }
+	} // else return as is
+	
+	return result;
+    }
+
+    public static void main (String [] args) {
+	SOAPCommunicator comm = new SOAPCommunicator();
+	XMLConverter converter = new XMLConverter();
+	String message = "<site name=\"localsite\" address=\"http://kanuka.cs.waikato.ac.nz:7070/axis/services/localsite\" type=\"soap\"/>";
+	Element site_elem = converter.getDOM(message).getDocumentElement();
+	comm.configure(site_elem);
+	message = "<message><request type=\"describe\" to=\"\" lang=\"en\"/></message>";
+	//message = "<message><request type=\"describe\" to=\"\" lang=\"en\"/></message>";
+	Element request_elem = converter.getDOM(message).getDocumentElement();
+	Element response = comm.process(request_elem);
+	
+	System.err.println("response was "+converter.getPrettyString(response));
+    }
+    /*
+    public Element process(Element message) {
+
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+	if (requests.getLength()==0) {
+	    // no requests
+	    return null;
+	}
+	
+	// if the communicator is part of a site, it needs to edit the to and from fields, otherwise it just passes the message on, as is
+	// for now, use local_site_name_ as the flag.
+	if (local_site_name_!=null) { // no local site
+	    
+	    for (int i=0;i<requests.getLength(); i++) {
+		Element e = (Element)requests.item(i);
+		String to = e.getAttribute(GSXML.TO_ATT);
+		to = GSPath.removeFirstLink(to); // remove the server name, should check that it is present
+		e.setAttribute(GSXML.TO_ATT, to);
+		String from = e.getAttribute(GSXML.FROM_ATT);
+		from = GSPath.appendLink(from, local_site_name_);
+		e.setAttribute(GSXML.FROM_ATT, from);
+	    }
+	}
+	// else do nothing to the message, just pass it on
+	
+	// needs to catch SOAPException, MalformedURLException	
+	try {
+
+	    // Set Method Parameters - use literal xml
+// 	    Parameter param = new Parameter("xml_in", 
+// 					    org.w3c.dom.Element.class,
+// 					    message, 
+// 					    Constants.NS_URI_LITERAL_XML);
+					    
+// 	    Vector param_list = new Vector ();
+// 	    param_list.addElement (param);
+// 	    call_.setParams (param_list);
+	    
+// 	    //  Set the URL for the Web Service
+// 	    URL url = new URL(remote_site_address_);
+	    
+	    // Invoke the Service
+	    Response resp = call_.invoke (url, "");
+	    
+	    // Check for Faults
+	    if (!resp.generatedFault()) { // no error
+		// Extract Return value
+		Parameter result = resp.getReturnValue ();
+		Element result_node = (Element) result.getValue();
+
+		if (local_site_name_ != null) {// have to modify the from field
+
+		    NodeList responses = result_node.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		    for (int i=0;i<responses.getLength(); i++) {
+			Element e = (Element)responses.item(i);
+			String from = e.getAttribute(GSXML.FROM_ATT);
+			from = GSPath.prependLink(from, remote_site_name_);
+			e.setAttribute(GSXML.FROM_ATT, from);
+		    }
+		} // else return as is
+		return result_node;
+	    }
+	    else { // error
+		//  Extract Fault Code and String
+		Fault f = resp.getFault();
+		String faultCode = f.getFaultCode();
+		String faultString = f.getFaultString();
+		System.err.println("Fault Occurred (details follow):");
+		System.err.println("Fault Code:  "+faultCode);
+		System.err.println("Fault String:  "+faultString);
+		return converter_.getDOM("<"+GSXML.MESSAGE_ELEM+">Fault Occurred.  No greeting for you!</"+GSXML.MESSAGE_ELEM+">").getDocumentElement();
+	    }
+	} catch (SOAPException e) {
+	    System.out.println("SOAP error:"+e.getMessage());
+	    return null;
+	} catch (MalformedURLException e) {
+	    System.out.println("URL error:"+e.getMessage());
+	    return null;
+	}
+    }
+    */
+}
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/DefaultReceptionist.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/DefaultReceptionist.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/DefaultReceptionist.java	(revision 21112)
@@ -0,0 +1,159 @@
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+// other java classes
+import java.io.File;
+import java.util.HashMap;
+import java.util.Enumeration;
+
+/** The default greenstone receptionist - needs some extra info for each page
+ */
+public class DefaultReceptionist extends TransformingReceptionist {
+    
+    protected void addExtraInfo(Element page) {
+	// we want to add in the collection description for each page - cos
+	// our default xslt needs this.
+	super.addExtraInfo(page);
+	
+	Element page_request = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+	// if it is a system request, then we don't bother with this.
+	String action = page_request.getAttribute(GSXML.ACTION_ATT);
+	if (action.equals("s")) {
+	    System.err.println("HACK: don't ask for coll info if system action");
+	    return;
+	}
+	///ystem.out.println("add extra info, page request="+this.converter.getString(page_request));
+	// is a collection defined?
+	Element param_list = (Element)GSXML.getChildByTagName(page_request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list==null) { // must be the original home page
+	    ///ystem.out.println("DefaultReceptionist: no param list, assuming home page");
+	    return;
+	}
+	Element coll_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSParams.COLLECTION);
+	if (coll_param == null) {
+	    ///ystem.out.println("DefaultReceptionist: coll param is null, returning");
+	    return;
+	}
+
+	// see if the collection/cluster element is already there
+	String coll_name = coll_param.getAttribute(GSXML.VALUE_ATT);
+	String lang = page_request.getAttribute(GSXML.LANG_ATT);
+	String uid = page_request.getAttribute(GSXML.USER_ID_ATT);
+	
+	boolean get_service_description = false;
+	Element page_response = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+	if (this.language_list != null) {
+	    page_response.appendChild(this.language_list);
+	}
+	Element coll_description = (Element)GSXML.getChildByTagName(page_response, GSXML.COLLECTION_ELEM);
+	if (coll_description == null) {
+	    // try cluster
+	    coll_description = (Element)GSXML.getChildByTagName(page_response, GSXML.CLUSTER_ELEM);
+	}
+	if (coll_description == null) { 
+	    // we dont have one yet - get it
+	    Element coll_about_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	    Element coll_about_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE,  coll_name, lang, uid);
+	    coll_about_message.appendChild(coll_about_request);
+	    
+	    Element coll_about_response_message = this.mr.process(coll_about_message);
+	    Element coll_about_response = (Element)GSXML.getChildByTagName(coll_about_response_message, GSXML.RESPONSE_ELEM);
+	    if (coll_about_response == null) {
+		return;
+	    }
+	    coll_description = (Element)GSXML.getChildByTagName(coll_about_response, GSXML.COLLECTION_ELEM);
+	    if (coll_description==null) { // may be a cluster
+		coll_description = (Element)GSXML.getChildByTagName(coll_about_response, GSXML.CLUSTER_ELEM);
+	    }
+	    
+	    if (coll_description == null) {
+		System.err.println("DefaultReceptionist: no collection description, returning");
+		return;
+	    }
+	    // have found one, append it to the page response
+	    coll_description = (Element)this.doc.importNode(coll_description, true);
+	    page_response.appendChild(coll_description);
+	    get_service_description = true;
+	}
+
+	// have got a coll description
+	
+	// now get the dispay info for the services
+	Element service_list = (Element)GSXML.getChildByTagName(coll_description, GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER);
+	if (service_list == null) {
+	    System.err.println("DefaultReceptionist: no service list, returning");
+	    // something weird has gone wrong
+	    return;
+	}
+
+	NodeList services = service_list.getElementsByTagName(GSXML.SERVICE_ELEM);
+	if (services.getLength()==0) {
+	    System.err.println("DefaultReceoptionist: no services found for colllection/cluster "+ coll_name);
+	    return;
+	}
+	// check one service for display items
+	if (!get_service_description) {
+	    // we dont know yet if we need to get these
+	    int i=1;
+	    Element test_s = (Element)services.item(0);
+	    while (i<services.getLength() && test_s.getAttribute(GSXML.TYPE_ATT).equals(GSXML.SERVICE_TYPE_RETRIEVE)) {
+		test_s = (Element)services.item(i); i++;
+	    }
+	    if (i==services.getLength()) {
+		// we have only found retrieve services, so dont need descripitons anyway
+		return;
+	    }
+	    if (GSXML.getChildByTagName(test_s, GSXML.DISPLAY_TEXT_ELEM) !=null) {
+		// have got descriptions already, 
+		return;
+	    }
+	}
+
+	// if get here, we need to get the service descriptions
+	///ystem.out.println("getting services description");
+	    
+	// we will send all the requests in a single message
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	for (int i=0; i<services.getLength(); i++) {
+	    Element c = (Element)services.item(i);
+	    String name = c.getAttribute(GSXML.NAME_ATT);
+	    String address = GSPath.appendLink(coll_name, name);
+	    Element info_request = GSXML.createBasicRequest(this.doc,  GSXML.REQUEST_TYPE_DESCRIBE, address, lang, uid);
+	    Element req_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    req_param_list.appendChild(GSXML.createParameter(this.doc, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER));
+	    info_request.appendChild(req_param_list);
+	    info_message.appendChild(info_request);
+	    
+	}
+
+	Element info_response = (Element)this.mr.process(info_message);
+
+	NodeList service_responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	// check that have same number of responses as collections
+	if (services.getLength() != service_responses.getLength()) {
+	    System.err.println("DefaultReceptionist Error: didn't get a response for each service - somethings gone wrong!");
+	    // for now, dont use the metadata
+	} else {
+	    for (int i=0; i<services.getLength(); i++) {
+		Element c1 = (Element)services.item(i);
+		Element c2 = (Element)GSXML.getChildByTagName((Element)service_responses.item(i), GSXML.SERVICE_ELEM);
+		if (c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.NAME_ATT))) {
+		    //add the service data into the original response
+		    GSXML.mergeElements(c1, c2);
+		} else {
+		    System.err.println("DefaultReceptionist Error: response does not correspond to request!");
+		} 
+		
+	    }
+	}
+	
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/GSException.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/GSException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/GSException.java	(revision 21112)
@@ -0,0 +1,50 @@
+/*
+ *    GSException.java
+ *    a new exception for Greenstone
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.GSXML;
+
+public class GSException
+    extends Exception {
+
+    private String exception_type = null;
+    // need some better names for these
+    //public static int PERMANENT = 0;
+    //public static int TEMPORARY = 1;
+    //public static int OTHER = 2;
+
+    public GSException(String message) {
+	
+	super(message);
+	this.exception_type = GSXML.ERROR_TYPE_OTHER;
+    }
+    public GSException(String message, String type) {
+
+	super(message);
+	this.exception_type = type;
+    }
+
+    public String getType() {
+	return exception_type;
+    }
+	
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/MessageRouter.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/MessageRouter.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/MessageRouter.java	(revision 21112)
@@ -0,0 +1,928 @@
+/*
+ *    MessageRouter.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.service.*;
+import org.greenstone.gsdl3.comms.*;
+import org.greenstone.gsdl3.collection.*;
+
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+
+// other java classes
+import java.io.File;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;      
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Properties;
+
+/**
+ * The hub of a Greenstone system. 
+ *
+ * Accepts  XML requests (via process method of ModuleInterface) and routes them to the appropriate collection or 
+ * service or external entity.
+ *
+ * contains a map of module objects - may be services, collections, comms 
+ * objects talking to other MessageRouters etc.
+ *
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ * @see ModuleInterface
+ * @see Collection
+ * @see ServiceRack
+ * @see Communicator
+ *
+ */
+public class MessageRouter implements  ModuleInterface {
+
+    /** the (directory) name of the site */
+    protected String site_name = null;
+    /** site home - the home directory for the site */
+    protected String site_home=null;
+    /** the http address for this site */
+    protected String site_http_address=null;
+    
+    /** map of names to Module objects */
+    protected HashMap module_map=null;
+
+    /** container Document to create XML Nodes */
+    protected Document doc=null; 
+    /** the full description of this site */
+
+    // should these things be separated into local and remote??
+
+    /** list of collections that can be reached */
+    protected Element collection_list = null; 
+    /** list of service clusters that can be reached */
+    protected Element cluster_list = null; 
+    /** list of single services that can be reached */
+    protected Element service_list = null; 
+    /** list of sites that can be reached */
+    protected Element site_list = null; 
+    
+   
+    /** a converter class to parse XML and create Docs */
+    protected XMLConverter converter=null;
+
+    
+    //***************************************************************
+    // public methods
+    //***************************************************************
+
+    /** constructor */
+    public MessageRouter() {
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+    }
+
+    public void cleanUp() {
+	if (this.module_map != null) {
+	    Iterator i = this.module_map.values().iterator();
+	    while (i.hasNext()) {
+		((ModuleInterface)i.next()).cleanUp();
+	    }
+	}
+    }
+
+    /** site_name must be set before configure is called */
+    public void setSiteName(String site_name) {
+	this.site_name = site_name;
+    }
+    /**
+     * configures the system
+     *
+     * looks in site_home/collect for collections, reads config file
+     * site_home/siteConfig.xml
+     *
+     */
+    public boolean configure() { 
+
+	System.out.println("MessageRouter:configuring site");
+
+	if (this.site_name==null) {
+	    System.err.println("MessageRouter: You must set site_name before calling configure");
+	    return false;
+	}
+	this.site_home = GSFile.siteHome(GlobalProperties.getGSDL3Home(), this.site_name);
+	this.site_http_address = GlobalProperties.getGSDL3WebAddress()+"/sites/"+this.site_name;
+	
+	// are we behind a firewall?? - is there a better place to set up the proxy?
+	String host = GlobalProperties.getProperty("proxy.host");
+	String port = GlobalProperties.getProperty("proxy.port");
+	final String user = GlobalProperties.getProperty("proxy.user");
+	final String passwd = GlobalProperties.getProperty("proxy.password");
+	  
+	if (host != null && !host.equals("") && port !=null && !port.equals("")) {
+	    System.setProperty("http.proxyType", "4");
+	    System.setProperty("http.proxyHost", host);
+	    System.setProperty("http.proxyPort", port);
+	    System.setProperty("http.proxySet", "true");
+	    // have we got a user/password?
+	    if (user != null && !user.equals("") && passwd != null && !passwd.equals("")) {
+		try {
+		    // set up the authenticator
+		    Authenticator.setDefault(new Authenticator(){ 
+			    protected PasswordAuthentication getPasswordAuthentication(){ 
+				return new PasswordAuthentication(user, new String(passwd).toCharArray()); 
+			    } 
+			}); 
+		    
+		} catch (Exception e) {
+		    System.err.println("MessageRouter Error: couldn't set up an authenticator the proxy");
+		    
+		}
+	    }
+	}
+
+	// read thru own config file - create services and connect to sites
+	File configFile = new File(GSFile.siteConfigFile(this.site_home));
+
+	if (!configFile.exists() ) {
+	    System.err.println("MessageRouter: site config file: "+configFile.getPath()+" not found!");
+	    return false;
+	}
+	
+	Document config_doc = this.converter.getDOM(configFile);
+	if (config_doc == null) {
+System.err.println("MessageRouter: couldn't parse site config file: "+configFile.getPath());
+	    return false;
+	}
+	
+	Element config_elem = config_doc.getDocumentElement();
+	
+	this.module_map = new HashMap();
+	
+	// load up the services
+	Element service_rack_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+	configureServices(service_rack_list);
+	
+	// load up the service clusters
+	Element cluster_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER);	
+	configureClusters(cluster_list);
+
+	// load up the collections
+	configureCollections();
+
+	// load up the external sites - this also adds their services/clusters/collections to the other lists - so must be done last
+	Element site_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.SITE_ELEM+GSXML.LIST_MODIFIER);
+	configureSites(site_list);
+	
+		
+	return true;
+	
+    }
+ 	
+   
+    /**
+     * Process an XML request - as a String
+     *
+     * @param xml_in the request to process
+     * @return the response - contains any error messages
+     * @see String
+     */
+    public String process(String xml_in) {
+	
+	Document doc = this.converter.getDOM(xml_in);
+	
+	Element result = process(doc.getDocumentElement());
+	return this.converter.getString(result);
+    }
+    
+    /**
+     * Process an XML request - as a DOM Element
+     *
+     * @param xml_in the message to process - should be <message>
+     * @return the response - contains any error messages
+     * @see Element
+     */
+    public Element process(Element message) {
+
+	///ystem.out.println("MR received request");
+	///ystem.out.println(this.converter.getString(message));
+	// check that its a correct message tag
+	if (!message.getTagName().equals(GSXML.MESSAGE_ELEM)) {
+	    System.err.println("MessageRouter: Invalid message. GSDL message should start with <"+GSXML.MESSAGE_ELEM+">, instead it starts with:"+message.getTagName()+".");
+	    return null;
+	}
+
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+
+	Element mainResult = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	// empty request
+	if (requests.getLength()==0) {
+	    return mainResult;
+	}
+
+	Document message_doc = message.getOwnerDocument();
+
+	// for now, just process each request one by one, and append the results to mainResult
+	// Note: if you add an element to another node in the same document, it
+	// gets removed from where it was. This changes the node list - you cant iterate over the node list in a normal manner if you are moving elements out of it
+	int num_requests = requests.getLength();
+	for (int i=0; i< num_requests; i++) {
+	    Node result=null;
+	    Element req = (Element)requests.item(i);
+	    if (req == null) {
+		System.err.println("request "+i+" is null");
+		continue;
+	    }
+	    String path = req.getAttribute(GSXML.TO_ATT); // returns "" if no att of this name
+	    if (path.equals("")) {
+		// its a message for the message router
+		if (req.getAttribute(GSXML.TYPE_ATT).equals(GSXML.REQUEST_TYPE_MESSAGING)) {
+		    // its a messaging request - modifies the requests/responses
+		    result = modifyMessages(req, message, mainResult);
+		    mainResult.appendChild(this.doc.importNode(result, true));
+		} else {
+		    // standard request
+		    result = processMessage(req);
+		    mainResult.appendChild(this.doc.importNode(result, true));
+		}
+	    } else {
+		String [] modules = path.split(",");
+		Element mess = this.doc.createElement(GSXML.MESSAGE_ELEM);
+		Element copied_request = (Element)this.doc.importNode(req, true);
+		mess.appendChild(copied_request);
+		
+		for (int j=0; j<modules.length; j++) {
+		    String this_mod = modules[j];
+		    // find the module to pass it on to
+		    // need to put the request into a message element
+		    String obj = GSPath.getFirstLink(this_mod);
+		    
+		    if (this.module_map.containsKey(obj)) {
+			copied_request.setAttribute(GSXML.TO_ATT, this_mod);
+			result = ((ModuleInterface)this.module_map.get(obj)).process(mess);
+			if (result !=null ) {
+			    // append the contents of the message to the mainResult - there will only be one response at this stage
+			    mainResult.appendChild(this.doc.importNode(GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM), true));
+			} else {
+			    // add in a place holder response
+			    Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+			    response.setAttribute(GSXML.FROM_ATT, this_mod);
+			    mainResult.appendChild(response);
+			    System.err.println("MessageRouter Error: request had null result!");
+			}
+			
+		    } else {    
+			System.err.println("MessageRouter Error: request has illegal module name in:\n"+this.converter.getString(req));
+		    }
+		}
+	    }
+	    
+	} // for each request
+	
+	///ystem.out.println("MR returned response");
+	///ystem.out.println(this.converter.getString(mainResult));
+
+	return mainResult;
+	
+    }
+	
+    // ********************************************************************
+    // auxiliary configure methods
+    // *******************************************************************
+
+
+    protected boolean configureServices(Element service_rack_list) {
+
+	this.service_list = this.doc.createElement(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER);
+
+	// load up the individual services
+	System.out.println("loading service modules...");
+	
+	if (service_rack_list == null) {
+	    System.out.println("... none to be loaded");
+	    return true;
+	} 
+	
+	NodeList service_racks = service_rack_list.getElementsByTagName(GSXML.SERVICE_CLASS_ELEM);
+	if (service_racks.getLength()==0) {
+	    System.out.println("... none to be loaded");
+	    return true;
+	} 
+		
+	Element service_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element service_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", "", "");
+	service_message.appendChild(service_request);
+		
+	for(int i=0; i<service_racks.getLength(); i++) {
+	    Element n = (Element)service_racks.item(i);
+	    String service_name = n.getAttribute(GSXML.NAME_ATT);
+	    System.out.println("..."+service_name);
+	    try {
+		ServiceRack s = (ServiceRack)Class.forName("org.greenstone.gsdl3.service."+service_name).newInstance();
+		s.setSiteHome(this.site_home);
+		s.setSiteAddress(this.site_http_address);
+		s.setMessageRouter(this);
+		// pass the XML node to the service for service configuration
+		s.configure(n, null);
+
+		// find out the supported services for this service module
+		Element service_response = (Element) s.process(service_message);
+		NodeList services = service_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+		if (services.getLength()==0) {
+		    System.err.println("MessageRouter configure error: serviceRack "+service_name+" has no services!");
+		} else {
+		    for (int j=0; j<services.getLength();j++) {
+			String service = ((Element)services.item(j)).getAttribute(GSXML.NAME_ATT);
+			this.module_map.put(service, s);
+			
+				// add short info to service_list_ XML
+			this.service_list.appendChild(this.doc.importNode(services.item(j), true));
+		    }
+		}
+	    } catch (Exception e ) {
+		System.err.println("MessageRouter configure exception:  in ServiceRack class specification:  "+ e.getMessage());
+		e.printStackTrace();
+	    } 
+	} // for each service module
+	return true;
+    }
+    
+    protected boolean configureClusters(Element config_cluster_list) {
+
+	this.cluster_list = this.doc.createElement(GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER);
+	// load up the service clusters
+	System.out.println("loading service clusters ...");
+	if (config_cluster_list == null) {
+	    System.out.println("... none to be loaded");
+	    return true;
+	} 
+	NodeList service_clusters = config_cluster_list.getElementsByTagName(GSXML.CLUSTER_ELEM);
+	if (service_clusters.getLength()==0) {
+	    System.out.println("... none to be loaded");
+	    return true;
+	}
+	
+	for (int i=0; i<service_clusters.getLength(); i++) {
+	    Element cluster = (Element)service_clusters.item(i);
+	    String name = cluster.getAttribute(GSXML.NAME_ATT);
+	    System.out.println("..."+name);
+	    ServiceCluster sc = new ServiceCluster();
+	    sc.setSiteHome(this.site_home);
+	    sc.setSiteAddress(this.site_http_address);
+	    sc.setClusterName(name);
+	    sc.setMessageRouter(this);
+	    sc.configure(cluster);
+	    this.module_map.put(name, sc); // this replaces the old one if there was one already present
+	    //add short info to cluster list
+	    Element e = this.doc.createElement(GSXML.CLUSTER_ELEM);
+	    e.setAttribute(GSXML.NAME_ATT, name);
+	    this.cluster_list.appendChild(e);
+	    
+	}
+	return true;
+    }
+    
+    protected boolean configureCollections() {
+
+	this.collection_list = this.doc.createElement(GSXML.COLLECTION_ELEM+GSXML.LIST_MODIFIER);
+
+	// read thru the collect directory and activate all the valid collections
+	File collectDir = new File(GSFile.collectDir(this.site_home));
+	if (collectDir.exists()) {
+	    System.out.println("Reading thru directory "+collectDir.getPath()+" to find collections.");
+	    File[] contents = collectDir.listFiles();
+	    for (int i=0; i<contents.length;i++) {
+		if(contents[i].isDirectory()) {
+		    
+		    String colName = contents[i].getName();
+		    if (!colName.startsWith("CVS")) {
+			activateCollectionByName(colName);
+		    }
+		}
+	    }
+	} // collectDir
+	return true;
+    }
+
+    protected boolean configureSites(Element config_site_list) {
+
+	this.site_list = this.doc.createElement(GSXML.SITE_ELEM+GSXML.LIST_MODIFIER);
+	// load up the sites
+	System.out.println("loading external sites...");
+	if (config_site_list ==null ) {
+	    System.out.println("...none found");
+	    return true;
+	}
+
+	NodeList sites = config_site_list.getElementsByTagName(GSXML.SITE_ELEM);
+	if (sites.getLength()==0) {
+	    System.out.println("...none found");
+	    return true;
+	}
+
+	// this is a name to identify the current site in the Communicator 
+	String local_site_name = config_site_list.getAttribute(GSXML.LOCAL_SITE_NAME_ATT);
+	if (local_site_name.equals("")) {
+	    local_site_name = site_name;
+	}
+
+	for (int i=0; i<sites.getLength(); i++) {
+	    Element s = (Element)sites.item(i);
+	    activateSite(s, local_site_name);	    
+	}
+	return true;
+    }
+    
+    protected boolean activateSite(Element site_elem, String local_site_name) {
+	Communicator comm=null;
+	String type = site_elem.getAttribute(GSXML.TYPE_ATT);
+	String name = site_elem.getAttribute(GSXML.NAME_ATT);
+	if (type.equals(GSXML.COMM_TYPE_SOAP_JAVA)) {
+	    comm = new SOAPCommunicator();
+	    if (comm.configure(site_elem)) {
+		comm.setLocalSiteName(local_site_name);
+		
+		// add to map of modules
+		this.module_map.put(name, comm);
+		this.site_list.appendChild(this.doc.importNode(site_elem, true));
+		// need to get collection list and service 
+		// list from here- if the site isn't up yet, the site will
+		// have to be added later 
+		if (!getRemoteSiteInfo(comm, name)) {
+		    System.err.println("MessageRouter: couldn't get info from site "+name);
+		}
+	    } else {
+		System.err.println("MessageRouter: couldn't configure soap site:"+name);
+		return false;
+	    }
+	    
+	} else {
+	    System.err.print("MessageRouter: cant talk to server of type:"+type + ", so not making a connection to "+name);
+	    return false;
+	}
+	return true;
+    }
+    
+    /** get site info from external site
+     *
+     * @param comm - the communicator object for the external site
+     * @param site_name - the name of the external site
+     * @return true if successful
+     */
+    protected boolean getRemoteSiteInfo(Communicator comm, String site_name) {
+    
+	System.out.println("MessageRouter: getting info from site:"+site_name);
+
+	Element info_request = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element req = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, "", "", "");
+	info_request.appendChild(req);
+
+	// process the message
+	Element info_response = comm.process(info_request);
+	if (info_response == null) {
+	    return false;
+	}
+	// collection info
+	NodeList colls = info_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+	if (colls.getLength()>0) {
+	    for (int i=0; i<colls.getLength(); i++) {
+		Element e = (Element)colls.item(i);
+		String col_name = e.getAttribute(GSXML.NAME_ATT);
+		// add the info to own coll list - may want to keep 
+		// this separate in future - so can distinguish own and 
+		// other collections ??
+		e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(col_name, site_name));
+		this.collection_list.appendChild(this.doc.importNode(e, true));
+	    }
+	}
+
+	// service info
+	NodeList services = info_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+	if (services.getLength()>0) {
+	    for (int i=0; i<services.getLength(); i++) {
+		Element e = (Element)services.item(i);
+		String serv_name = e.getAttribute(GSXML.NAME_ATT);
+		e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(serv_name, site_name));
+		this.service_list.appendChild(this.doc.importNode(e, true));
+	    }
+	}
+	
+	// serviceCluster info
+	NodeList clusters = info_response.getElementsByTagName(GSXML.CLUSTER_ELEM);
+	if (clusters.getLength()>0) {
+	    for (int i=0; i<clusters.getLength(); i++) {
+		Element e = (Element)clusters.item(i);
+		String clus_name = e.getAttribute(GSXML.NAME_ATT);
+		e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(clus_name, site_name));
+		this.cluster_list.appendChild(this.doc.importNode(e, true));
+	    }
+	}
+	return true;
+    }
+
+
+    //*****************************************************************
+    // auxiliary process methods
+    //*****************************************************************
+
+    /** handles requests made to the MessageRouter itself 
+     *
+     * @param req - the request Element- <request>
+     * @return the result Element - should be <response>
+     */
+    protected Element processMessage(Element req) {
+
+	// message for self, should be type=describe/configure at this stage
+	String type = req.getAttribute(GSXML.TYPE_ATT);
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, "");
+	if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE)) {
+	    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_DESCRIBE);
+	    // check the param list
+	    Element param_list = (Element) GSXML.getChildByTagName(req, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    if (param_list == null) {
+		response.appendChild(this.collection_list);
+		response.appendChild(this.cluster_list);
+		response.appendChild(this.site_list);
+		response.appendChild(this.service_list);
+		return response;
+	    }
+	    NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	    
+	    // go through the param list and see what components are wanted
+	    for (int i=0; i<params.getLength(); i++) {
+		
+		Element param = (Element)params.item(i);
+		// Identify the structure information desired
+		if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM)) {
+		    String info = param.getAttribute(GSXML.VALUE_ATT);
+		    if (info.equals(GSXML.COLLECTION_ELEM+GSXML.LIST_MODIFIER)) {
+			response.appendChild(this.collection_list);
+			
+		    } else if (info.equals(GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER)) {
+			response.appendChild(this.cluster_list);
+			
+		    } else if (info.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+			response.appendChild(this.service_list);
+		    } else if (info.equals(GSXML.SITE_ELEM+GSXML.LIST_MODIFIER)) {
+			response.appendChild(this.site_list);
+		    } 
+		}
+	    }
+	    return response;
+	    
+	} 
+	
+	if (type.equals(GSXML.REQUEST_TYPE_SYSTEM)) {
+	    
+	    // a list of system requests - should put any error messages
+	    // or success messages into response
+	    NodeList commands = req.getElementsByTagName(GSXML.SYSTEM_ELEM);
+	    Element site_config_elem = null;
+	    boolean success = false;
+			
+	    for (int i=0; i<commands.getLength(); i++) {
+		// all the commands should be Elements
+		Element elem = (Element)commands.item(i);
+		String action = elem.getAttribute(GSXML.TYPE_ATT);
+		if (action.equals(GSXML.SYSTEM_TYPE_CONFIGURE)) {
+		    String subset = elem.getAttribute(GSXML.SYSTEM_SUBSET_ATT);
+		    if (subset.equals("")) {
+			// need to reconfigure the MR
+			this.configure();
+			Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM,  "MessageRouter reconfigured successfully");
+			response.appendChild(s);
+			
+		    } else {
+			// else it a specific request
+			if (subset.equals(GSXML.COLLECTION_ELEM+GSXML.LIST_MODIFIER)) {
+			    success = configureCollections();
+			} else {
+
+			    // need the site config file
+			    if (site_config_elem==null) {
+				
+				File configFile = new File(GSFile.siteConfigFile(this.site_home));
+				if (!configFile.exists() ) {
+				    System.err.println("MessageRouter: site config file: "+configFile.getPath()+" not found!");
+				    continue;
+				}
+				Document site_config_doc = this.converter.getDOM(configFile);
+				if (site_config_doc == null) {
+				    System.err.println("MessageRouter: couldn't parse site config file: "+configFile.getPath());
+				    continue;
+				}
+				site_config_elem  = site_config_doc.getDocumentElement();
+			    }
+			    if (subset.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+				Element service_rack_list = (Element)GSXML.getChildByTagName(site_config_elem, GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+				
+				success = configureServices(service_rack_list);
+			    } else if (subset.equals(GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER)) {
+				Element cluster_list = (Element)GSXML.getChildByTagName(site_config_elem, GSXML.CLUSTER_ELEM+GSXML.LIST_MODIFIER);	
+				
+				success = configureClusters(cluster_list);
+			    } else if (subset.equals(GSXML.SITE_ELEM+GSXML.LIST_MODIFIER)) {
+				Element site_list = (Element)GSXML.getChildByTagName(site_config_elem, GSXML.SITE_ELEM+GSXML.LIST_MODIFIER);
+				success = configureSites(site_list);
+			    }
+			}
+			String message=null;
+			if (success) {
+			    message = subset + "reconfigured successfully";
+			} else {
+			    message = "Error in reconfiguring "+subset;
+			}
+			Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, message);
+			response.appendChild(s);
+		    }
+		    
+		    
+		} else {
+		    String module_name = elem.getAttribute(GSXML.SYSTEM_MODULE_NAME_ATT);
+		    String module_type = elem.getAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT);
+
+		    if (action.equals(GSXML.SYSTEM_TYPE_DEACTIVATE)) {
+			deactivateModule(module_type, module_name);
+			Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, module_type+": "+module_name+" deactivated");
+			response.appendChild(s);
+		    } else if (action.equals(GSXML.SYSTEM_TYPE_ACTIVATE)) {
+			if (module_type.equals(GSXML.COLLECTION_ELEM)) {
+			    success = activateCollectionByName(module_name);
+			} else if (module_type.equals(GSXML.SITE_ELEM)) {
+			    success = activateSiteByName(module_name);
+			} else if (module_type.equals(GSXML.CLUSTER_ELEM)) {
+			    success = activateServiceClusterByName(module_name);
+			}
+			if (success) {
+			    Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, module_type+": "+module_name+" activated");
+			    response.appendChild(s);
+			} else {
+			    Element s = GSXML.createTextElement(this.doc, GSXML.STATUS_ELEM, module_type+": "+module_name+" could not be activated");
+			    response.appendChild(s);
+			}
+		    }
+		} // else not a configure action
+	    } // for all commands
+	    return response;
+			
+
+	} // system type request
+	
+	// if get here something has gone wrong
+	System.err.println("MessageRouter: cant process request:");
+	System.err.println(this.converter.getString(req));
+	return null;
+       
+    }
+    protected Element modifyMessages(Element request, Element message, Element result) {
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, "");
+	response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_MESSAGING);
+
+	NodeList commands = request.getElementsByTagName("command");
+	if (commands == null) {
+	    System.err.println("no commands, "+converter.getPrettyString(request));
+	    return response;
+	}
+	for (int i=0; i<commands.getLength(); i++) {
+	    Element action = (Element)commands.item(i);
+	    String type = action.getAttribute(GSXML.TYPE_ATT);
+	    if (type.equals("copyNode")) {
+		// copies the from node as a child of to node
+		String from_path = action.getAttribute("from");
+		String to_path = action.getAttribute("to");
+		Element from_node = null;
+		String from_node_root = GSPath.getFirstLink(from_path);
+		if (from_node_root.startsWith(GSXML.REQUEST_ELEM)) {
+		    from_node = message;
+		} else if (from_node_root.startsWith(GSXML.RESPONSE_ELEM)) {
+		    from_node = result;
+		}
+		if (from_node == null) {
+		    continue;
+		}
+		Element to_node = null;
+		String to_node_root = GSPath.getFirstLink(to_path);
+		if (to_node_root.startsWith(GSXML.REQUEST_ELEM)) {
+		    to_node = message;
+		} else if (to_node_root.startsWith(GSXML.RESPONSE_ELEM)) {
+		    to_node = result;
+		}
+		if (to_node == null) {
+		    continue;
+		}
+		// now we know what node to copy where
+		Node orig_node = GSXML.getNodeByPathIndexed(from_node, from_path);
+		if (orig_node == null) {
+		    continue;
+		}
+		Node new_parent = GSXML.getNodeByPathIndexed(to_node, to_path);
+		if (new_parent == null) {
+		    continue;
+		    
+		}
+		new_parent.appendChild(to_node.getOwnerDocument().importNode(orig_node, true));
+	    }
+	    
+	    else if (type.equals("copyChildren")) {
+
+	    }
+	} // for each command
+	return response;
+    }
+    
+    // ****************************************************
+    // other methods
+    // ****************************************************
+
+    /** creates and configures a new collection
+     *
+     *@param col_name the name of the collection
+     *@return true if collection created ok
+     */
+    protected boolean activateCollectionByName(String col_name) {
+
+	System.out.println("MessageRouter:Activating collection: "+col_name+".");
+		
+	// now we need to look for the etc/collectionInit.xml file, and see what sort of Collection to load
+	Collection c = null;
+	File init_file = new File(GSFile.collectionInitFile(this.site_home, col_name));
+	if (init_file.exists()) {
+	    Document init_doc = this.converter.getDOM(init_file);
+	    if (init_doc != null) {
+		Element init_elem = init_doc.getDocumentElement();
+		if (init_elem != null) {
+		    String coll_class_name = init_elem.getAttribute("class");
+		    if (!coll_class_name.equals("")) {
+			try {
+			    c = (Collection)Class.forName("org.greenstone.gsdl3.collection."+coll_class_name).newInstance();
+			} catch (Exception e) {
+			    System.out.println("MessageRouter: couldn't create a new collection, type "+coll_class_name+", defaulting to class Collection");
+			}
+		    }
+		}
+	    }
+	}
+	if (c==null) { // we haven't found anpther classname to use
+	    c = new Collection();
+	}
+	
+	c.setCollectionName(col_name);
+	c.setSiteHome(this.site_home);
+	c.setSiteAddress(this.site_http_address);
+	c.setMessageRouter(this);
+	if (c.configure()) {
+	    // this could be a reactivation, so delete the old version
+	    deactivateModule(GSXML.COLLECTION_ELEM, col_name);
+	    // add to list of collections
+	    this.module_map.put(col_name, c);
+	    //add short description_ to collection_list_
+	    Element e = this.doc.createElement(GSXML.COLLECTION_ELEM);
+	    e.setAttribute(GSXML.NAME_ATT, col_name);
+	    this.collection_list.appendChild(e);
+	    return true;
+	} else {
+	    System.err.println("MessageRouter:Couldn't configure collection: "+
+			       col_name+".");
+	    return false;
+	}
+	
+    }
+    
+    protected boolean activateSiteByName(String site_name) {
+	System.out.println("MessageRouter:Activating site: "+site_name+".");
+	
+	// just in case this is a reactivation, deactivate this site first
+	deactivateModule(GSXML.SITE_ELEM, site_name);
+	File configFile = new File(GSFile.siteConfigFile(this.site_home));
+
+	if (!configFile.exists() ) {
+	    System.err.println("MessageRouter: site config file: "+configFile.getPath()+" not found!");
+	    return false;
+	}
+	Document config_doc = this.converter.getDOM(configFile);
+	if (config_doc == null) {
+	    System.err.println("MessageRouter: couldn't parse site config file: "+configFile.getPath());
+	    return false;
+	}
+	Element config_elem = config_doc.getDocumentElement();
+	
+	Element config_site_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.SITE_ELEM+GSXML.LIST_MODIFIER);
+	if (config_site_list ==null ) {
+	    System.err.println("MessageRouter:activateSite, no sites found");
+	    return false;
+	}
+	// this is a name to identify the current site in the Communicator 
+	String local_site_name = config_site_list.getAttribute("localSiteName");
+	if (local_site_name.equals("")) {
+	    local_site_name = site_name;
+	}
+
+	Element this_site_elem = GSXML.getNamedElement(config_site_list, GSXML.SITE_ELEM, GSXML.NAME_ATT, site_name);
+	if (this_site_elem == null) {
+	    System.err.println("MessageRouter:activateSite, site "+site_name+" not found");
+	    return false;
+	}
+	
+	return activateSite(this_site_elem, local_site_name);
+    }
+    
+    protected boolean activateServiceClusterByName(String cluster_name) {
+	return false;
+
+    }
+
+    protected boolean activateServiceRackByName(String module_name) {
+	return false;
+    }
+
+    protected boolean deactivateModule(String type, String name) {
+	
+	if (this.module_map.containsKey(name)) {
+
+	    System.out.println("MessageRouter: deactivating "+name);
+	    ModuleInterface m = (ModuleInterface)this.module_map.remove(name);
+	    m.cleanUp(); // clean up any open files/connections etc - can cause trouble on windows
+	    // also remove the xml bit from description list
+	    if (type.equals(GSXML.COLLECTION_ELEM)) {
+		Element this_col = GSXML.getNamedElement(this.collection_list, GSXML.COLLECTION_ELEM, GSXML.NAME_ATT, name);
+		if (this_col != null) {
+		    this.collection_list.removeChild(this_col);
+		}
+		return true;
+	    } else if (type.equals(GSXML.SERVICE_ELEM)) {
+		Element this_service = GSXML.getNamedElement(this.service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, name);
+		if (this_service != null) {
+		    this.service_list.removeChild(this_service);
+		}
+		return true;
+	    } else if (type.equals(GSXML.CLUSTER_ELEM)) {
+		Element this_cluster = GSXML.getNamedElement(this.cluster_list, GSXML.CLUSTER_ELEM, GSXML.NAME_ATT, name);
+		if (this_cluster != null) {
+		    this.cluster_list.removeChild(this_cluster);
+		}
+		return true;
+	    } else if (type.equals(GSXML.SITE_ELEM)) {
+		Element this_site = GSXML.getNamedElement(this.site_list, GSXML.SITE_ELEM, GSXML.NAME_ATT, name);
+		if (this_site != null) {
+		    this.site_list.removeChild(this_site);
+		    
+		    // also remove this sites colls, services, clusters etc
+		    removeRemoteSiteInfo(this.collection_list, GSXML.COLLECTION_ELEM, name);
+		    removeRemoteSiteInfo(this.cluster_list, GSXML.CLUSTER_ELEM, name);
+		    removeRemoteSiteInfo(this.service_list, GSXML.SERVICE_ELEM, name);
+		}
+	    } else {
+		System.err.println("MessageRouter: couldn't deactivate coll");
+		// couldn't do it
+		return false;
+	    }
+	}
+	// else not deactivated 
+	return false;
+
+    }
+
+    /**
+     * this looks through a list (module_list) of elements named module_name, 
+     * and removes any whose names start with site_name
+     */
+    protected void removeRemoteSiteInfo(Element module_list, 
+					String module_name, 
+					String site_name) {
+
+	NodeList modules = module_list.getElementsByTagName(module_name);
+	// will this work??
+	for (int i=modules.getLength()-1; i>=0; i--) {
+	    
+	    String name = ((Element)modules.item(i)).getAttribute(GSXML.NAME_ATT);
+	    if (GSPath.getFirstLink(name).equals(site_name)) {
+		module_list.removeChild(modules.item(i));
+	    }
+	}
+    }
+	
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/ModuleInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/ModuleInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/ModuleInterface.java	(revision 21112)
@@ -0,0 +1,62 @@
+/*
+ *    ModuleInterface.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+   
+import org.w3c.dom.Element;                                                
+
+/**
+ * interface for all modules in greenstone.
+ *
+ * all components talk via a process method - in its simplest form,
+ * it takes a String of XML, and returns a String of XML
+ *
+ * the more efficient process method uses DOM Element Nodes instead 
+ * of Strings - this avoids parsing the XML at each module
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+public interface ModuleInterface {
+ 
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xml_in the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+    abstract public String process(String xml_in);
+
+  /**
+   * Process an XML request - as a DOM model Element
+   *
+   * @param xml_in the request to process
+   * @return the response - contains any error messages
+   * @see org.w3c.dom.Element
+   */
+    abstract public Element process(Element xml_in);
+
+    /** 
+     * Do any clean up necessary for deactivating the module, eg
+     * close any open file handles (gdbm in particular) or windows
+     * holds locks on them.
+     */
+    abstract public void cleanUp();
+}    
+                                                                           
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/NZDLReceptionist.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/NZDLReceptionist.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/NZDLReceptionist.java	(revision 21112)
@@ -0,0 +1,95 @@
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+// other java classes
+import java.io.File;
+import java.util.HashMap;
+import java.util.Enumeration;
+
+/** 
+ * This is a new version of the Receptionist for the NZDL interface
+ * the interface requires a lot more info for each page
+ * @see DefaultReceptionist
+ */
+public class NZDLReceptionist extends DefaultReceptionist {
+    
+    protected void addExtraInfo(Element page) {
+	///ystem.out.println("the  page before adding extra info is");
+	///ystem.out.println(this.converter.getPrettyString(page));
+
+	super.addExtraInfo(page);
+
+	// this gets the collection info
+	Element page_response = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+	Element page_request = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+	Element collection_info = (Element)GSXML.getChildByTagName(page_response, GSXML.COLLECTION_ELEM);
+	if (collection_info== null) {
+	    return;
+	}
+
+	String collection = collection_info.getAttribute("name"); // should we take this from the request instead??
+	// look through services and see if classifier one is present - need to
+	// get the list of classifiers
+	Element service_list = (Element)GSXML.getChildByTagName(collection_info, GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER);
+	if (service_list == null) {
+	    // something weird has gone wrong
+	    return;
+	}
+	
+	Element info_message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	String lang = page.getAttribute(GSXML.LANG_ATT);
+	
+	boolean do_classifier = false;
+	boolean do_query = false;
+	Element classifier_service = GSXML.getNamedElement(service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, "ClassifierBrowse");
+	Element query_service = GSXML.getNamedElement(service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, "TextQuery");
+	if (classifier_service != null) {
+	    do_classifier = true;
+	    // find the list of classifiers
+	
+	    String to = GSPath.appendLink(collection, "ClassifierBrowse");
+	    Element info_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, ""); 
+	    info_message.appendChild(info_request);
+	}
+	
+	String action = page_request.getAttribute(GSXML.ACTION_ATT);
+	String subaction = page_request.getAttribute(GSXML.SUBACTION_ATT);
+	
+	if (query_service != null && action.equals("p") && subaction.equals("about")) {
+	    do_query = true;
+	    // get the full description
+	    String to = GSPath.appendLink(collection, "TextQuery");
+	    Element query_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, to, lang, "");
+	    info_message.appendChild(query_request);
+	    
+	}
+	
+	Element info_resp_message = (Element) this.mr.process(info_message);
+	Element info_response = (Element) GSXML.getChildByTagName(info_resp_message, GSXML.RESPONSE_ELEM); 
+	//	String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.SERVICE_ELEM);
+	if (do_classifier) {
+	    Element classifier_service_info = (Element)GSXML.getNamedElement(info_response, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, "CLassifierBrowse");
+	    if (classifier_service_info != null) {
+		service_list.replaceChild(this.doc.importNode(classifier_service_info, true), classifier_service);
+	    }
+	}
+	if (do_query) {
+	    Element query_service_info = (Element)GSXML.getNamedElement(info_response, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, "TextQuery");
+	    if (query_service_info != null) {
+		service_list.replaceChild(this.doc.importNode(query_service_info, true), query_service);
+	    }
+	}
+	///ystem.out.println("the final page before transforming is");
+	///ystem.out.println(this.converter.getPrettyString(page));
+	return;
+    }
+	
+}
+    
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/Receptionist.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/Receptionist.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/Receptionist.java	(revision 21112)
@@ -0,0 +1,277 @@
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+// other java classes
+import java.io.File;
+import java.util.HashMap;
+import java.util.Enumeration;
+import java.util.ArrayList;
+
+/** the most basic Receptionist, used for interface generation. 
+ * Receives requests consisting
+ * of an xml representation of cgi args, and returns the page of data. The requests are processed by the appropriate action class
+ *
+ * @see Action
+ */
+public class Receptionist implements ModuleInterface {
+
+    /** the set up variables */
+    protected HashMap config_params = null;
+    /** container Document to create XML Nodes */
+    protected Document doc=null; 
+   
+    /** a converter class to parse XML and create Docs */
+    protected XMLConverter converter=null;
+
+    /** the message router that the Receptionist and Actions will talk to */
+    protected ModuleInterface mr=null;
+
+    /** the list of actions */
+    protected HashMap action_map=null;
+    
+    /** the list of params */
+    protected GSParams params=null;
+    protected Element language_list = null;
+
+    /** the list of interfaces this is based on */
+    protected ArrayList base_interfaces = null;
+    
+    public Receptionist() {
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+	this.action_map= new HashMap();
+    }
+    
+    public void cleanUp() {}
+    public void setParams(GSParams params) {
+	this.params = params;
+    }
+    public void setConfigParams(HashMap params) {
+	this.config_params = params;
+    }
+    /** sets the message router  - it should already be created and 
+     * configured before  being passed to the receptionist*/
+    public void setMessageRouter(ModuleInterface m) {
+       this.mr = m;
+    }
+
+    /** configures the receptionist */
+    public boolean configure() {
+
+	if (this.config_params==null) {
+	    System.err.println("Receptionist Error: config variables must be set before calling configure");
+	    return false;
+	}
+	if (this.mr==null) {	   
+	    System.err.println("Receptionist Error: message router must be set  before calling configure");
+	    return false;
+	}
+
+	// find the config file containing a list of actions
+	File interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String)this.config_params.get(GSConstants.INTERFACE_NAME))));
+	if (!interface_config_file.exists()) {
+	    System.err.println("Receptionist: interface config file: "+interface_config_file.getPath()+" not found!");
+	    return false;
+	}
+	
+	Document config_doc = this.converter.getDOM(interface_config_file);
+	if (config_doc == null) {
+	    System.err.println("Receptionist: could not parse interface config file: "+interface_config_file.getPath());
+	    return false;
+	}
+	Element config_elem = config_doc.getDocumentElement();
+	String base_interface = config_elem.getAttribute("baseInterface");
+	setUpBaseInterface(base_interface);
+	setUpInterfaceOptions(config_elem);
+
+	// load up the actions
+	Element action_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.ACTION_ELEM+GSXML.LIST_MODIFIER);
+	NodeList actions = action_list.getElementsByTagName(GSXML.ACTION_ELEM);
+
+	for (int i=0; i<actions.getLength(); i++) {
+	    Element action = (Element) actions.item(i);
+	    String class_name = action.getAttribute("class");
+	    String action_name = action.getAttribute("name");
+	    Action ac = null;
+	    try {
+		ac = (Action)Class.forName("org.greenstone.gsdl3.action."+class_name).newInstance();
+	    } catch (Exception e) {
+		System.err.println("Receptionist: couldn't load in action "+class_name);
+		e.printStackTrace();
+		continue;
+	    }
+	    ac.setConfigParams(this.config_params);
+	    ac.setMessageRouter(this.mr);
+	    ac.configure();
+	    ac.getActionParameters(this.params);
+	    this.action_map.put(action_name, ac);
+	}
+
+	this.language_list = (Element)GSXML.getChildByTagName(config_elem, "languageList");
+	if (language_list == null) {
+	    System.err.println("Receptionist: didn't find a language list in the config file!!");
+	}
+
+	return true;
+    }
+    /** process using strings - just calls process using Elements */
+    public String process(String xml_in) {
+
+	Element message = this.converter.getDOM(xml_in).getDocumentElement();
+	Element page = process(message);
+	return this.converter.getString(page);
+    }
+
+    /** process - produce a page of data in response to a request 
+     * if something goes wrong, it returns null - 
+     * TODO:  return a suitable message to the user */
+    public Element process(Element message) {
+
+	// get the request out of the message - assume that there is only one
+	Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	if (request == null) {
+	    System.err.println("Receptionist Error: message had no request!");
+	    return null; 
+	}
+	// check the request type 
+	String type = request.getAttribute(GSXML.TYPE_ATT); // returns "" if no att of this name
+	if (!type.equals(GSXML.REQUEST_TYPE_PAGE)) { 
+	    // now Receptionist forwards non-page requests straight to the MR, and returns the responses
+	    System.err.println("Receptionist: request type is not '"+GSXML.REQUEST_TYPE_PAGE+"', but it is '"+type+"', so forwarding the message to the MR!");
+	    // process the whole message - mr needs <message> tags, and 
+	    // in this case, there may be more than one request in the message
+	    return this.mr.process(message);
+	}	
+	// work out which action to pass to
+	String action = request.getAttribute(GSXML.ACTION_ATT);
+	if (action.equals("")) {
+	    System.err.println("Receptionist Error: no action specified in the request!");
+	    return null;
+	}
+	
+	// find the  appropriate action	
+	Action a = (Action)this.action_map.get(action);
+	String action_name=null;
+	if (a==null) { // not in the map yet
+	    // try to load a new action
+	    try {
+		action_name = action.substring(0,1).toUpperCase()+action.substring(1)+"Action";
+		Action ac = (Action)Class.forName("org.greenstone.gsdl3.action."+action_name).newInstance();
+		ac.setConfigParams(this.config_params);
+		ac.setMessageRouter(this.mr);
+		ac.configure();
+		ac.getActionParameters(this.params);
+		this.action_map.put(action, ac);
+		a = ac;
+	    } catch (Exception e) {
+		
+		System.err.println("Receptionist Error: a new action ("+action_name+") was specified and it couldn't be created. Error message:"+e.getMessage());
+		return null;
+	    }
+	}
+
+	// transform the request in some way
+	preProcessRequest(request);
+	// set up the page
+	Element page = this.doc.createElement(GSXML.PAGE_ELEM);
+	page.setAttribute(GSXML.LANG_ATT, request.getAttribute(GSXML.LANG_ATT));
+	
+	// get the page data from the action
+	Element action_response = a.process(message);
+
+	boolean response_only=false;
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list != null) {
+	    Element param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, "ro");
+	    if (param != null) {
+		String value = param.getAttribute("value");
+		if (value.equals("1")) {
+		    response_only = true;
+		}
+	    }
+	}
+	if (response_only) {
+	    // only the response from the action is sent back
+	    return action_response;
+	}
+	
+	// the request is part of the page
+	page.appendChild(GSXML.duplicateWithNewName(this.doc, request, GSXML.PAGE_REQUEST_ELEM, true));
+	// add the response too
+	Element page_response = GSXML.duplicateWithNewName(this.doc, (Element)GSXML.getChildByTagName(action_response, GSXML.RESPONSE_ELEM), GSXML.PAGE_RESPONSE_ELEM, true);
+	page.appendChild(page_response);
+
+	///ystem.out.println("Receptionist: raw page="+this.converter.getString(page));
+	// transform the result in some way
+	Element resulting_page = postProcessPage(page);
+	return resulting_page;
+	
+    }
+
+    protected boolean setUpBaseInterface(String base_interface) {
+	if (base_interface== null || base_interface.equals("")) {
+	    // there was no base interface, the list remains null
+	    return true;
+	}
+	// foreach base interface
+	while (!base_interface.equals("")) {
+	    // find the base interface config file
+	    File base_interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), base_interface)));
+	    if (!base_interface_config_file.exists()) {
+		System.err.println("Receptionist: base interface config file: "+base_interface_config_file.getPath()+" not found!");
+		return false;
+	    }
+	    // the interface name is valid, add it to the list
+	    if (base_interfaces == null) {
+		base_interfaces = new ArrayList();
+	    }
+	    base_interfaces.add(base_interface);
+	    // now see if this has a base interface
+	    Document config_doc = this.converter.getDOM(base_interface_config_file);
+	    if (config_doc == null) {
+		System.err.println("Receptionist: could not parse base interface config file: "+base_interface_config_file.getPath());
+		return false;
+	    }
+	    Element config_elem = config_doc.getDocumentElement();
+	    base_interface = config_elem.getAttribute("baseInterface");
+	}
+	return true;
+    }
+
+    protected boolean setUpInterfaceOptions(Element config_elem) {
+	Element option_list = (Element)GSXML.getChildByTagName(config_elem, "optionList");
+	if (option_list != null) {
+	    System.err.println("found an option list");
+	    // we set any options in the config params
+	    NodeList options = option_list.getElementsByTagName("option");
+	    for (int i=0; i<options.getLength(); i++) {
+		Element option = (Element)options.item(i);
+		String name = option.getAttribute(GSXML.NAME_ATT);
+		String value = option.getAttribute(GSXML.VALUE_ATT);
+		System.err.println("option: "+name+", "+value);
+		if (!name.equals("") && !value.equals("")) {
+		    this.config_params.put(name, value);
+		}
+	    }
+	}
+
+	return true;
+    }
+
+    protected void preProcessRequest(Element request) {
+	return;
+    }
+
+    protected Element postProcessPage(Element page) {
+	return page;
+    }
+    
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/TransformingReceptionist.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/TransformingReceptionist.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/core/TransformingReceptionist.java	(revision 21112)
@@ -0,0 +1,222 @@
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+// other java classes
+import java.io.File;
+import java.util.HashMap;
+import java.util.Enumeration;
+
+/** A receptionist that uses xslt to transform the page_data before returning it. . Receives requests consisting
+ * of an xml representation of cgi args, and returns the page of data - in 
+ * html by default. The requests are processed by the appropriate action class
+ *
+ * @see Action
+ */
+public class TransformingReceptionist extends Receptionist{
+    
+    /** the list of xslt to use for actions */
+    protected HashMap xslt_map = null;
+    
+    /** a transformer class to transform xml using xslt */
+    protected XMLTransformer transformer=null;
+
+    public TransformingReceptionist() {
+	super();
+	this.xslt_map = new HashMap();
+	this.transformer = new XMLTransformer();
+    }
+    
+    /** configures the receptionist - overwrite this to set up the xslt map*/
+    public boolean configure() {
+	
+	if (this.config_params==null) {
+	    System.err.println("TransformingReceptionist Error: config variables must be set before calling configure");
+	    return false;
+	}
+	if (this.mr==null) {	   
+	    System.err.println("TransformingReceptionist Error: message router must be set  before calling configure");
+	    return false;
+	}
+	
+	// find the config file containing a list of actions
+	File interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String)this.config_params.get(GSConstants.INTERFACE_NAME))));
+	if (!interface_config_file.exists()) {
+	    System.err.println("TransformingReceptionist: interface config file: "+interface_config_file.getPath()+" not found!");
+	    return false;
+	}
+	Document config_doc = this.converter.getDOM(interface_config_file, "utf-8");
+	if (config_doc == null) {
+	    System.err.println("TransformingReceptionist: could not parse interface config file: "+interface_config_file.getPath());
+	    return false;
+	}
+	Element config_elem = config_doc.getDocumentElement();
+	String base_interface = config_elem.getAttribute("baseInterface");
+	setUpBaseInterface(base_interface);
+	setUpInterfaceOptions(config_elem);
+
+	Element action_list = (Element)GSXML.getChildByTagName(config_elem, GSXML.ACTION_ELEM+GSXML.LIST_MODIFIER);
+	NodeList actions = action_list.getElementsByTagName(GSXML.ACTION_ELEM);
+
+	for (int i=0; i<actions.getLength(); i++) {
+	    Element action = (Element) actions.item(i);
+	    String class_name = action.getAttribute("class");
+	    String action_name = action.getAttribute("name");
+	    Action ac = null;
+	    try {
+		ac = (Action)Class.forName("org.greenstone.gsdl3.action."+class_name).newInstance();
+	    } catch (Exception e) {
+		System.err.println("TransformingReceptionist Error: couldn't load in action "+class_name);
+		e.printStackTrace();
+		continue;
+	    }
+	    ac.setConfigParams(this.config_params);
+	    ac.setMessageRouter(this.mr);
+	    ac.configure();
+	    ac.getActionParameters(this.params);
+	    this.action_map.put(action_name, ac);
+
+	    // now do the xslt map
+	    String xslt = action.getAttribute("xslt");
+	    if (!xslt.equals("")) {
+		this.xslt_map.put(action_name, xslt);
+	    }
+	    NodeList subactions = action.getElementsByTagName(GSXML.SUBACTION_ELEM);
+	    for (int j=0; j<subactions.getLength(); j++) {
+		Element subaction = (Element)subactions.item(j);
+		String subname = subaction.getAttribute(GSXML.NAME_ATT);
+		String subxslt = subaction.getAttribute("xslt");
+		
+		String map_key = action_name+":"+subname;
+		///ystem.out.println("adding in to xslt map, "+map_key+"->"+subxslt);
+		this.xslt_map.put(map_key, subxslt);
+	    }
+	    
+	   
+	}
+	Element lang_list = (Element)GSXML.getChildByTagName(config_elem, "languageList");
+	if (lang_list == null) {
+	    System.err.println("TransformingReceptionist: didn't find a language list in the config file!!");
+	} else {
+	    this.language_list = (Element) this.doc.importNode(lang_list, true);
+	}
+
+	return true;
+    }
+
+
+    protected Element postProcessPage(Element page) {
+	
+	// might need to add some data to the page
+	addExtraInfo(page);
+	// transform the page using xslt
+	Element transformed_page = transformPage(page);
+	
+	return transformed_page;
+    }
+    
+    /** overwrite this to add any extra info that might be needed in the page before transformation */
+    protected void addExtraInfo(Element page) {}
+
+    /** transform the page using xslt
+     * we need to get any format element out of the page and add it to the xslt
+     * before transforming */
+    protected Element transformPage(Element page) {
+
+	//ystem.out.println("page before transfomring:");
+	//ystem.out.println(this.converter.getPrettyString(page));
+
+	Element request = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+	String action = request.getAttribute(GSXML.ACTION_ATT);
+	String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+	
+	String output = request.getAttribute(GSXML.OUTPUT_ATT);
+	// we should choose how to transform the data based on output, eg diff
+	// choice for html, and wml??
+	// for now, if output=xml, we don't transform the page, we just return 
+	// the page xml
+	if (output.equals("xml")) {
+	    return page;
+	}
+
+	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	String collection = "";
+	if (cgi_param_list != null) {
+	    HashMap params = GSXML.extractParams(cgi_param_list, false);
+	    collection = (String)params.get(GSParams.COLLECTION);
+	    if (collection == null) collection = "";
+	}
+	
+	String xslt_file = getXSLTFileName(action, subaction, collection);
+	if (xslt_file==null) {
+	    // cant transform the page - return null or the original?
+	    System.err.println("TransformingReceptionist: cant find the xslt file needed, so returning the original page!");
+	    return page;
+	}
+	Document style_doc = this.converter.getDOM(new File(xslt_file), "UTF-8");
+	if (style_doc == null) {
+	    System.err.println("TransformingReceptionist: cant parse the xslt file needed, so returning the original page!");
+	    return page;
+	    
+	}
+
+	// look for the format element in the page response
+	Element page_response = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+	Element format_elem = (Element)GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);
+	if (format_elem != null) {
+	    //page_response.removeChild(format_elem);
+	    ///ystem.err.println("format elem="+this.converter.getPrettyString(format_elem));
+	    // need to transform the format info
+	    String stylesheet_file = GSFile.stylesheetFile(GlobalProperties.getGSDL3Home(), (String)this.config_params.get(GSConstants.SITE_NAME), collection, (String)this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces,   "config_format.xsl");
+	    Document stylesheet_doc = this.converter.getDOM(new File(stylesheet_file));
+	    if (stylesheet_doc != null) {
+		Document format_doc = this.converter.newDOM();
+		format_doc.appendChild(format_doc.importNode(format_elem, true));
+		Element new_format = (Element)this.transformer.transform(stylesheet_doc, format_doc);
+	    ///ystem.err.println("new format elem="+this.converter.getPrettyString(new_format));
+	    
+	    // add it in to the main stylesheet
+		GSXSLT.mergeStylesheets(style_doc, new_format);
+	    } else {
+		System.err.println("TransformingReceptionist: couldn't parse the config_format stylesheet, adding the format info as is");
+		GSXSLT.mergeStylesheets(style_doc, format_elem);
+	    }
+	    ///ystem.out.println("the converted stylesheet is:");
+	    ///ystem.out.println(this.converter.getPrettyString(style_doc.getDocumentElement()));
+	}
+	
+	// there is a thing called a URIResolver which you can set for a transformer or transformer factory. may be able to use this instead of this absoluteIncludepaths hack
+	GSXSLT.absoluteIncludePaths(style_doc, GlobalProperties.getGSDL3Home(), (String)this.config_params.get(GSConstants.SITE_NAME), collection, (String)this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
+	// put the page into a document - this is necessary for xslt to get the paths right if you have paths relative to the document root eg /page.
+	Document doc = this.converter.newDOM();
+	doc.appendChild(doc.importNode(page, true));
+	return (Element)this.transformer.transform(style_doc, doc, config_params);          
+    }
+
+    protected String getXSLTFileName(String action, String subaction,
+				     String collection) {
+
+	String name = null;
+	if (!subaction.equals("")) {
+	    String key = action+":"+subaction;
+	    name = (String) this.xslt_map.get(key);
+	}
+	// try the action by itself
+	if (name==null) {
+	    name = (String) this.xslt_map.get(action);
+	}
+	// now find the absolute path
+	String stylesheet = GSFile.stylesheetFile(GlobalProperties.getGSDL3Home(), (String)this.config_params.get(GSConstants.SITE_NAME), collection, (String)this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, name);
+	if (stylesheet==null) {
+	    System.out.println("TransformingReceptionist: cant find stylesheet for "+name);
+	}
+	return stylesheet;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Build.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Build.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Build.java	(revision 21112)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Build
+{
+  BuildManager manager;
+
+  public Build(List inputRoots, String site, String collection, String outputDir)
+  { this.manager = new BuildManager(inputRoots, site, collection, outputDir);
+  }
+
+  public void run()
+  { this.manager.run();
+  }
+
+  public static void main(String args[])
+  { int a = 0;
+    List inputDirs = new ArrayList();
+    String collection = null;
+    String site = null;
+    String outputDir  = null;
+
+    while (a < args.length)
+    { System.out.println(args[a]);
+      if (args[a].equals("-inputdir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  inputDirs.add(args[a+1]);
+	  a ++;	
+	}
+      }
+      else if (args[a].equals("-collect")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  collection = args[a+1];
+	  a ++;
+	}
+      }
+      else if (args[a].equals("-site")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{
+	  site = args[a+1];
+	  a ++;
+	}
+      }	  
+      else if (args[a].equals("-outputDir")) {
+	if (a < args.length - 1 &&
+	    args[a+1].charAt(0) != '-')
+	{ outputDir = args[a+1];
+	  a ++;
+	}
+      }
+      a ++;
+    }
+
+    if (site == null) {
+	System.err.println("you must specify the site name");
+	System.exit(0);
+    }
+    if (collection == null && inputDirs.size() == 0) {
+	System.err.println("At least one input directory must be given, or a collection name");
+	System.exit(0);
+    }
+    
+    Build build = new Build(inputDirs, site, collection, outputDir);
+    build.run();
+
+    return;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/BuildManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/BuildManager.java	(revision 21112)
@@ -0,0 +1,154 @@
+package org.greenstone.gsdl3.gs3build;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.notifier.NotifierManager;
+import org.greenstone.gsdl3.gs3build.extractor.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+
+import java.io.File;
+import java.util.List;
+
+public class BuildManager 
+{
+    CollectionManager collectionManager;
+    RecogniserManager recogniserManager;
+    IndexerManager    indexerManager;
+    ClassifierManager classifierManager;
+    ExtractorManager  extractorManager;
+    NotifierManager   notifierManager;
+    DocumentList      docList;
+    List              inputRoots;
+    String            outputDir;
+    String            archiveDir;
+    
+    public BuildManager(List inputRoots, String site, String collection, String outputDir) 
+    {
+	// set up the collection manager
+	this.collectionManager = new CollectionManager(site, collection);
+	this.collectionManager.setBuildManager(this);
+	
+	// set up the list of input directories
+	if (inputRoots.size() == 0) {
+	    inputRoots.add(this.collectionManager.getImportDirectory());
+	    System.out.println("Adding collection import directory to input list " + this.collectionManager.getImportDirectory());
+	}
+	this.inputRoots = inputRoots;
+	
+	// set up the output directories (building and archive)
+	this.outputDir  = outputDir != null ? outputDir : this.collectionManager.getBuildDirectory();
+	File outputDirFile = new File(this.outputDir);
+	if (!outputDirFile.isDirectory()) {
+	    System.err.println("Unable to open directory " + this.outputDir + " for writing");
+	    System.exit(1);
+	}
+	System.out.println("Output directory set to " + this.outputDir);
+	
+	// set up the archive directory - this will definitely exist
+	this.archiveDir = this.collectionManager.getArchiveDirectory();
+
+	// the list that documents get added to when they are recognised
+	this.docList = new DocumentList(new DocumentIDFactory(this.collectionManager), 
+					this.collectionManager.getDatabase());
+	
+	// set up the various managers
+	this.recogniserManager = new RecogniserManager(this.docList);
+	this.extractorManager  = new ExtractorManager(this.docList);
+	this.classifierManager = new ClassifierManager(this.docList, collectionManager.getDatabase());
+	this.indexerManager = new IndexerManager(this.docList);
+	this.notifierManager = new NotifierManager();
+	
+	// configure the collection - this will add classifiers, indexers, recognisers to the various managers.
+	this.collectionManager.configureCollection();
+	
+	// add in expanders by hand cos they are not in the config file yet
+	// TODO
+	// add in extractors by hand cos they are not in the config file yet
+	this.extractorManager.addExtractor(new IndexExtractor());
+	this.extractorManager.addExtractor(new MetaXMLExtractor());
+	
+	// add in enrichers by hand cos they are not in the config file yet
+	// TODO
+		
+	// prepare a file crawler on the etc directory, and a crawl observer to respond to
+	// the appropriate files.
+	FileCrawler fileCrawler = new FileCrawler(new File(this.collectionManager.getEtcDirectory()), 
+						  new CollectionCrawlObserver(this));
+
+	fileCrawler.crawl();    
+    }
+  
+    public String getLastBuildSequence()
+    { 
+	return this.collectionManager.getCollectionMetadata("gsdl3", "buildsequence");
+    }
+
+    public ClassifierManager getClassifierManager()
+    { 
+	return this.classifierManager;
+    }
+
+    public IndexerManager getIndexerManager()
+    { 
+	return this.indexerManager;
+    }
+
+    // TODO: validation phase
+    public RecogniserManager getRecogniserManager()
+    {
+	return this.recogniserManager;
+    }
+    
+    public void addIndexer(IndexerInterface iface)
+    { 
+	iface.configure(IndexerManager.outputDir, this.outputDir);
+	this.indexerManager.addIndexer(iface);
+    }
+
+    public void run()
+    { 
+	// Initialise collection manager - note start of build
+	this.collectionManager.startBuild();
+	
+	// TODO: add expansion (e.g. Zip files)
+    
+	// Crawl the file tree - will recognise documents
+	for (int i = 0; i < this.inputRoots.size(); i ++) {
+	    FileCrawler fileCrawler = new FileCrawler(new File((String) this.inputRoots.get(i)), recogniserManager);
+
+	    fileCrawler.crawl();
+	}
+
+	// Extract phase, etc.
+	this.extractorManager.extractDocuments();
+	this.classifierManager.classifyDocuments();
+	this.indexerManager.indexDocuments();
+
+	// Timestamp management - update all timestamps on modified dates...
+	// 
+	// This should only occur at the end of building in case the build is cancelled...
+	docList.updateTimestamps(this.collectionManager.getBuildTimestamp());
+	
+	// TODO: validation phase
+	this.notifierManager.detectEvents(this.collectionManager);
+	
+	if (this.archiveDir != null) {
+	    this.docList.writeDocuments(new File(this.archiveDir));
+	    //this.docList.writeSQLDocuments(this.collectionManager.getDatabase());
+	}
+	
+	this.collectionManager.setCollectionMetadata("gsdl3", "documentCount", Integer.toString(this.docList.getCount()));
+    
+	// TODO: write out collection configuration file (service lists, etc.)
+    
+	this.collectionManager.endBuild();
+    }
+  
+    public void addInputDirectory(String inputDir)
+    {
+	this.inputRoots.add(inputDir);
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/CollectionManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/CollectionManager.java	(revision 21112)
@@ -0,0 +1,594 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.util.Date;
+import java.util.Calendar;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.GregorianCalendar;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+
+import java.net.URL;
+import java.net.URLEncoder;
+
+import java.sql.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.collection.*;
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.indexers.*;
+import org.greenstone.gsdl3.gs3build.doctypes.RecogniserInterface;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+/**
+ *  Store and hold collection-level configuration information for a collection.
+ *  This should be used by BuildManager to work out which classes, etc. to load
+ *  at build time, and as a repository for the collection-level metadata, and
+ *  a means of loading and saving the same to a file or database, as is seen
+ *  fit in the final development of gs3.
+ */
+
+public class CollectionManager
+{
+  GregorianCalendar  lastBuildDate;  // pretty obvious
+  String             adminEmail;     // the email address of the administrator of the
+		                     // collection
+  int                buildDocNo;     // used to generate document identifiers
+  CollectionMetadata metadata;       // collection-level metadata
+  GS3SQLConnection   database;       // the database to store everything in
+  String             collectionHome;
+  String             siteHome;
+  String             collectionName;
+  String             qualifiedCollectionName; // used as the database name 
+  String             notifyHost;
+
+  BuildManager       buildManager;
+   
+  class CollectionClassifier
+  { URL    file;
+    String type;
+    List fields;
+    String sort;
+    String className;
+
+    public CollectionClassifier(URL parentURL, String type, String className, Node node)
+    { this.type = type;
+      this.className = className;
+      this.fields = new ArrayList();
+
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals("file")) {
+	    NamedNodeMap atts = children.item(c).getAttributes();
+	    Node attribute = atts.getNamedItem("URL");
+	    String urlString = attribute.getNodeValue();
+	    if (urlString == null)
+	      continue;
+
+	    System.out.println("Path is" + parentURL.getPath());
+	    System.out.println("Host is" + parentURL.getHost());
+	    System.out.println(urlString);
+
+	    try {
+	      URL url = new URL(parentURL, urlString);
+	      this.file = url;
+	      System.out.println(url);
+	    }
+	    catch (java.net.MalformedURLException malEx) {
+	      System.out.println(malEx);
+	    }
+	  }
+	  else if (name.equals("field")) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	  else if (name.equals("sort")) {
+	    String sortName = DOMUtils.getNodeChildText(children.item(c));
+	    this.sort = sortName;
+	  }
+	}
+      }
+    }
+
+    public ClassifierInterface getClassifier()
+    { ClassifierInterface classifier = null;
+
+      if (this.type == null) {
+	return null;
+      }
+      System.out.println("Creating a classifier of type " + this.type.toLowerCase());
+      
+      if (this.type.toLowerCase().equals("hierarchy")) {
+	System.out.println("  hierarchy file is " + this.file);
+	classifier = new HierarchyClassifier(this.className, this.file, this.fields, this.sort);
+      }
+      else if (this.type.toLowerCase().equals("azlist")) {
+	classifier = new AZListClassifier(this.className, this.fields);
+      }
+
+      return classifier;
+    }
+  }
+   
+  /**
+   *  Create the collection manager for a given collection
+   *
+   * @param site the name of the site
+   *  @param collection <code>String</code> the name of the collection
+   */
+  public CollectionManager(String site, String collection) {
+    
+    String gsdl3Root = System.getProperty("GSDL3HOME");
+    if (gsdl3Root == null) {
+      System.out.println("Error: Unable to locate GSDL3HOME");
+      System.exit(1);
+      //return;
+    }
+    
+    this.siteHome = GSFile.siteHome(gsdl3Root, site);
+    File site_dir = new File(this.siteHome);
+    System.out.println(site_dir);
+    if (!site_dir.exists()) {
+      System.out.println("Error: Non-existant site ("+site+") specified");
+      System.exit(1);
+    }
+    site_dir = null;
+    this.collectionHome = GSFile.collectionBaseDir(this.siteHome, collection);
+    
+    File collect_dir = new File(this.collectionHome);
+    if (!collect_dir.exists()) {
+      System.out.println("Error: Non-existant collection ("+collection+") specified in site "+site);
+      System.exit(1);
+    }
+    collect_dir = null;
+
+    this.collectionName = collection;
+    this.qualifiedCollectionName = site+"_"+collection;
+
+    this.database = GS3SQLConnectionFactory.getGS3SQLConnection(this.qualifiedCollectionName);
+    /*    if (this.database != null) {
+      this.database.clearCollection(collection);
+      this.database = null;
+    }
+    */
+    if (this.database == null) {
+	this.database = GS3SQLConnectionFactory.getGS3SQLConnection("test");
+	if (this.database == null) {
+	    System.err.println("Can't connect to the mysql database. Please make sure your mysql is running, and that the correct passwords (if any) are specified in "+gsdl3Root+"/WEB-INF/classes/global.properties");
+	    System.exit(1);
+	}
+	boolean success = this.database.initCollection(this.qualifiedCollectionName);
+	if (!success) {
+	    System.err.println("couldn't init collection " + this.qualifiedCollectionName);
+	    System.exit(1);
+	}
+    }
+
+    this.metadata = new CollectionMetadata();
+
+    File buildDirectory = new File(getBuildDirectory());
+    if (!buildDirectory.exists()) {
+      buildDirectory.mkdir();
+    }
+    if (!buildDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + buildDirectory + " for writing");
+	System.exit(1);
+    }
+
+    File archiveDirectory = new File(getArchiveDirectory());
+    if (!archiveDirectory.exists()) {
+      archiveDirectory.mkdir();
+    }
+    if (!archiveDirectory.isDirectory()) {
+	System.err.println("Unable to open directory " + archiveDirectory + " for writing");
+	System.exit(1);
+    }
+
+    this.buildDocNo = 1;
+
+    try {
+      GS3SQLSelect select = new GS3SQLSelect("build");
+      select.addField("*");
+      Statement statement = this.database.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	System.out.println("Reading all keys");
+	do {
+	  String key = results.getString("buildKey");
+	  String value = results.getString("buildValue");
+
+	  if (key.equals("NextSeqNo")) {
+	    this.buildDocNo = Integer.parseInt(value);
+	  }
+	  else if (key.equals("lastBuildDate")) {
+	    int year, month, day;
+	    year = Integer.parseInt(value.substring(0, 4));
+	    month = Integer.parseInt(value.substring(4, 6)) - 1; // -1 because Gregorian Calendar perversely treats January as 0, etc.
+	    day = Integer.parseInt(value.substring(6, 8));
+	    
+	    this.lastBuildDate = new GregorianCalendar(year, month, day);
+	  }
+	} while (results.next());
+      }
+      statement.close();
+    }
+    catch (SQLException ex)
+      { System.out.println(ex);
+      }
+  }
+
+  public void setBuildManager(BuildManager build_man) 
+  { this.buildManager = build_man;
+  }
+
+  private void configureBrowsers(Node node, File etcFile)
+  { CollectionClassifier classifier = null;
+    URL etcURL = null;
+
+    etcURL = URLTools.getFileURL(etcFile);
+
+    NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++)
+    { // assume that non-element children are irrelevant
+      if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+      { continue;
+      }
+
+      String name = children.item(c).getNodeName();
+      System.out.println(name);
+
+      if (name.equals(GSXML.CLASSIFIER_ELEM))
+      { NamedNodeMap atts = children.item(c).getAttributes();
+
+        // get the type attribute
+	Node attribute = atts.getNamedItem(GSXML.TYPE_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String type = attribute.getNodeValue();
+
+        // get the type attribute
+	attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	if (attribute == null) {
+	  continue;
+	}
+	String className = attribute.getNodeValue();
+
+	classifier = new CollectionClassifier(etcURL, type, className, children.item(c));
+	
+	System.out.println("Found classifier " + type);
+	
+	// attach the classifier
+	ClassifierInterface classify = classifier.getClassifier();
+	this.buildManager.getClassifierManager().addClassifier(classify);
+      }  
+    }
+  }
+
+  public void configureCollection()
+  { File collectionConfig = new File(GSFile.collectionConfigFile(this.collectionHome));
+
+    // get the File and read it in
+    try
+    {
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      Document document = builder.parse(collectionConfig);
+
+      // TODO: report an error
+      if (document == null)
+      {	
+      }
+
+      // now parse the manager file...
+      Element rootElement = document.getDocumentElement();
+
+      if (rootElement.getTagName() != GSXML.COLLECTION_CONFIG_ELEM)
+      {	// TODO: throw exception
+      }
+
+      System.out.println("Configuring collection");
+
+      NodeList children = rootElement.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++)
+      {	// assume that non-element children are irrelevant
+	if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE)
+	{ continue;
+	}
+
+	String name = children.item(c).getNodeName();
+
+	// the name is a plugin element
+
+	if (name.equals(GSXML.RECOGNISE_ELEM)) {
+	    System.out.println("recognise elem");
+	    
+	    NodeList doc_types = ((Element)children.item(c)).getElementsByTagName(GSXML.DOC_TYPE_ELEM);
+	    for(int i=0; i<doc_types.getLength(); i++) {
+		Element doc_type = (Element)doc_types.item(i);
+		String type = doc_type.getAttribute(GSXML.NAME_ATT);
+		System.err.println("adding recogniser, type "+type);
+		RecogniserInterface ri = this.buildManager.getRecogniserManager().addRecogniser(type);
+		if (ri != null) {
+		    ri.configure(doc_type);
+		}
+	    }
+	} else if (name.equals(GSXML.SEARCH_ELEM)) {
+	  // pick up attributes from the <search> tag now...
+	  NamedNodeMap searchAttributes = children.item(c).getAttributes();
+	  Node searchAttribute = searchAttributes.getNamedItem(GSXML.TYPE_ATT);
+	  String searchType = null;
+	  if (searchAttribute != null) {
+	      searchType = searchAttribute.getNodeValue();
+	  } else {
+	      System.out.println("no "+GSXML.TYPE_ATT+" attribute found for the "+GSXML.SEARCH_ELEM+" element, assuming mg");
+	      searchType = MGIndexer.MG_INDEX_TYPE;
+	  }
+	  
+	  searchAttribute = searchAttributes.getNamedItem(GSXML.NAME_ATT);
+	  String searchName = null;
+	  if (searchAttribute != null) {
+	      searchName = searchAttribute.getNodeValue();
+	  }
+	  if (searchName == null) {
+	      searchName = "idx"; // need to modify this if we have two search elements with no names
+	  }
+	  // create the pertinent indexer...
+	  IndexerInterface indexer = IndexerFactory.makeIndexer(searchType, searchName);
+
+	  if (indexer == null) {
+	    continue;
+	  }
+
+	  // configure the indexer
+	  indexer.configure(children.item(c));
+
+	  // install it into the build manager
+	  this.buildManager.addIndexer(indexer);
+	}
+	else if (name.equals(GSXML.BROWSE_ELEM))
+	{ this.configureBrowsers(children.item(c), collectionConfig);
+	}
+	else if (name.equals(GSXML.NOTIFY_ELEM))
+	{
+	    this.notifyHost = ((Element) children.item(c)).getAttribute(GSXML.NOTIFY_HOST_ATT);
+	}
+	// TODO: other elements - make a factory-method approach here...
+	else
+	{
+	}
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.out.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.out.println(ex);
+    }
+    catch (SAXException ex) {
+      System.out.println(ex);
+    }
+    catch (IOException ex)
+    {
+      System.out.println(ex);
+    }
+
+    System.out.println("<<<Obtaining database>>>>");
+  }
+
+  public String getEtcDirectory()
+  { return GSFile.collectionEtcDir(this.collectionHome);
+  }
+
+  public String getImportDirectory()
+  { return GSFile.collectionImportDir(this.collectionHome);
+  }
+
+  public String getBuildDirectory()
+  { return GSFile.collectionBuildDir(this.collectionHome);
+  }
+
+  public String getArchiveDirectory()
+  { return GSFile.collectionArchiveDir(this.collectionHome);
+  }
+
+  public GS3SQLConnection getDatabase()
+  {
+    return this.database;
+  }
+
+  public long getBuildTimestamp()
+  { return this.lastBuildDate.getTime().getTime();
+  }
+
+  public Date getBuildDate()
+  { return this.lastBuildDate.getTime();
+  }
+
+  public void startBuild()
+  { GregorianCalendar today = new GregorianCalendar();
+  
+    if (this.lastBuildDate != null)
+    { // if the build date is different to the last build date, then reset the build
+      // document number
+      if (today.get(Calendar.YEAR) != this.lastBuildDate.get(Calendar.YEAR) ||
+	  today.get(Calendar.MONTH) != this.lastBuildDate.get(Calendar.MONTH) ||
+	  today.get(Calendar.DAY_OF_MONTH) != this.lastBuildDate.get(Calendar.DAY_OF_MONTH))
+      { this.buildDocNo = 1;
+      }
+      else 
+      { System.out.println("Continuing build sequence from " + this.buildDocNo);
+      }
+    }
+    this.lastBuildDate = today;
+  }
+
+  public void endBuild()
+  {
+    // here we write out the build config file
+    // create the xml for the buildConfig
+    XMLConverter converter = new XMLConverter();
+    Document doc = converter.newDOM();
+    Element build_config = doc.createElement(GSXML.COLLECTION_BUILD_ELEM);
+    Element meta_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    build_config.appendChild(meta_list);
+    Element service_list = doc.createElement(GSXML.SERVICE_CLASS_ELEM+GSXML.LIST_MODIFIER);
+    build_config.appendChild(service_list);
+    
+    // ask the indexers and classifiers to add stuff into the service rack list
+    this.buildManager.getIndexerManager().addServiceDescriptions(service_list);
+    this.buildManager.getClassifierManager().addServiceDescriptions(service_list);
+    // get the String
+    String build_config_string = converter.getString(build_config);
+    // write it to the file
+    try {
+      File build_config_file = new File(GSFile.collectionBuildConfigFileBuilding(this.collectionHome));
+      BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(build_config_file), "UTF-8"));
+      writer.write(build_config_string, 0, build_config_string.length());
+      writer.close();
+    } catch (Exception e) {
+      System.err.println("CollectionManager.endBuild() Error while trying to output the buildConfig.xml file.");
+      System.err.println(e.getMessage());
+    }
+    
+    // Update build date information
+    try {
+      GS3SQLDelete remove = new GS3SQLDelete("build");
+      //    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("buildKey", "=", "NextSeqNo"));
+      //    rem
+      Statement statement = this.database.createStatement();
+      statement.execute(remove.toString());
+      
+      GS3SQLInsert insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "NextSeqNo");
+      insert.addValue("buildValue", Integer.toString(this.buildDocNo));
+      statement.execute(insert.toString());
+    
+      insert = new GS3SQLInsert("build");
+      insert.addValue("buildKey", "lastBuildDate");
+      insert.addValue("buildValue", getDateString(this.lastBuildDate));
+      statement.execute(insert.toString());
+      statement.close();
+    } catch (SQLException e) {
+	System.err.println("CollectionManager.endBuild(): Can't update build information: "+e);
+    }
+    
+    // Do tail of build output
+    Date startDate = this.lastBuildDate.getTime();
+    Date date = new Date();
+
+    long startTime = startDate.getTime();
+    long endTime   = date.getTime();
+
+    long difference = ((endTime - startTime) + 500) / 1000;
+
+    System.out.println("Build completed");
+    System.out.println("---------------");
+    System.out.println("Total Documents: " + this.getCollectionMetadata("gsdl3", "documentCount"));
+    System.out.println("Total Time :     " + (difference / 60) + " min. " + (difference % 60) + " secs.");
+  }
+
+  private static String getDateString(GregorianCalendar date)
+  { StringBuffer dateString = new StringBuffer();
+  
+    int value;
+    dateString.append(date.get(Calendar.YEAR));
+
+    // the use of month is a little odd, hence the following
+    // code.  Calendar.MONTH yields 0 = January, 1 = February,
+    // etc. hence there is a '+1' added to the month to make 
+    // it into January = 1, etc., and the padding is altered
+    // correspondingly.
+    value = date.get(Calendar.MONTH);
+    if (value < 9)
+    { dateString.append("0");
+    }
+    dateString.append(value + 1);
+    value = date.get(Calendar.DAY_OF_MONTH);
+    if (value < 10)
+      dateString.append("0");
+    dateString.append(value);
+
+    return dateString.toString();
+  }
+
+
+  public String getNextDocumentID()
+  { StringBuffer ID = new StringBuffer(getDateString(this.lastBuildDate));
+
+    int value = this.buildDocNo;
+    this.buildDocNo ++;
+    
+    ID.append(":");
+    ID.append(Integer.toString(value));
+    return ID.toString();
+  }
+  
+  public int getDocumentNumber()
+  { this.buildDocNo ++;
+    return this.buildDocNo - 1;
+  }
+
+  /**
+   *  Get the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public String getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getCollectionMetadata(namespace, label).get(0).toString();
+  }
+
+  /**
+   *  Set the collection metadata item in the given namespace
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label
+   *  @param <code>String</code> the value
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setCollectionMetadata(namespace, label, value);
+  }
+  
+  public String getCollectionName() {
+      return collectionName;
+  }
+
+/**
+ * @return
+ */
+public String getNotifyHost() {
+    return notifyHost;
+}
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Create.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Create.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/Create.java	(revision 21112)
@@ -0,0 +1,116 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+import org.greenstone.gsdl3.util.GSFile;
+
+public class Create
+{
+    String gsdl3_home = null;
+    String gsdl3_srchome = null;
+    String site = null;
+    String site_home = null;
+    String collection = null;
+
+    public Create(String gsdl3_home, String gsdl3_srchome, String site, 
+		  String collection)
+    { 
+	this.gsdl3_home = gsdl3_home;
+	this.gsdl3_srchome = gsdl3_srchome;
+	this.site = site;
+	this.site_home = GSFile.siteHome(gsdl3_home, site);
+	this.collection = collection;
+
+	// check the site's collect directory
+	File site_collect = new File(GSFile.collectDir(this.site_home));
+	if (!site_collect.isDirectory()) {
+	    System.err.println("Site's collect directory "+site_collect.getPath()+" doesn't exist. Please create it and try again");
+	    System.exit(1);
+	}
+
+       	
+    }
+    
+    public void makeCollection() {
+	
+	String coll_home = GSFile.collectionBaseDir(this.site_home, this.collection);
+	File coll_dir_file = new File(coll_home);
+	if (coll_dir_file.exists()) {
+	    System.err.println("Collection directory ("+coll_dir_file.getPath()+") already exists, not creating it.");
+	    System.exit(1);
+	}
+	createDirectory(coll_dir_file);
+	createDirectory(new File(GSFile.collectionImportDir(coll_home)));
+	createDirectory(new File(GSFile.collectionEtcDir(coll_home)));
+
+	File config_template = new File(gsdl3_srchome + File.separator+ "resources"+File.separator + "xml"+File.separator+"collectionConfig.xml");
+	if (!config_template.isFile()) {
+	    System.err.println("Couldn't find the config file template "+config_template.getPath());
+	    System.exit(1);
+	}
+
+	File coll_config = new File(GSFile.collectionConfigFile(coll_home));
+	GSFile.copyFile(config_template, coll_config);
+
+	System.err.println("New collection created in "+coll_home);
+	System.err.println("Put source documents into import, set the collection's configuration in etc/collectionConfig.xml, then run \"gs3-build.(sh|bat) "+ this.site+" "+this.collection+"\" to build the collection");
+    }
+
+    protected void createDirectory(File dir) {
+	dir.mkdir();
+	if (!dir.isDirectory()) {
+	    System.err.println("Couldn't create directory "+dir.getPath());
+	    System.exit(1);
+	}
+    }
+
+    public static void main(String args[])
+    { 
+	String gsdl3_home = System.getProperty("GSDL3HOME");
+	String gsdl3_srchome = System.getProperty("GSDL3SRCHOME");
+	if (gsdl3_home == null) {
+	    System.out.println("Error: Unable to locate GSDL3HOME");
+	    System.exit(1);
+	}
+	if (gsdl3_srchome == null) {
+	    System.out.println("Error: Unable to locate GSDL3SRCHOME");
+	    System.exit(1);
+	}
+
+	int a = 0;
+	String collection = null;
+	String site = null;
+	
+	while (a < args.length) {
+	    if (args[a].equals("-collect")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+			collection = args[a+1];
+			a ++;
+		}
+	    } else if (args[a].equals("-site")) {
+		if (a < args.length - 1 &&
+		    args[a+1].charAt(0) != '-') {
+		    
+		    site = args[a+1];
+		    a ++;
+		}
+	    }
+	    a ++;
+	}
+	
+	if (site == null) {
+	    
+	    System.err.println("you must specify the site name");
+	    System.exit(0);
+	}
+	if (collection == null) {
+	    System.err.println("you must specify a collection");
+	    System.exit(0);
+	}
+    
+	Create c = new Create(gsdl3_home, gsdl3_srchome, site, collection);
+	c.makeCollection();
+	return;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawlObserver.java	(revision 21112)
@@ -0,0 +1,8 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public interface FileCrawlObserver
+{
+	public void processFile(File file);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawler.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/FileCrawler.java	(revision 21112)
@@ -0,0 +1,39 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+public class FileCrawler
+{
+  File root;
+  FileCrawlObserver observer;
+
+  public FileCrawler(File file, FileCrawlObserver observer)
+  { this.root = file;
+    this.observer = observer;
+  }
+
+  public void crawl()
+  { this.crawlDirectory(this.root);
+  }
+
+  public boolean crawlDirectory(File file)
+  {
+    File [] children = file.listFiles();
+
+    if (children == null) {
+      return true;
+    }
+
+    for (int c = 0; c < children.length; c ++) {
+      if (children[c].isDirectory()) {
+	crawlDirectory(children[c]);
+      }
+      else
+      {
+	// TODO: process file
+	this.observer.processFile(children[c]);
+      }
+    }
+    return true;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/GSDataDump.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/GSDataDump.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.doctypes.*;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+
+public class GSDataDump
+{ 
+  public static void main(String args[])
+  {
+    String database = args[0];
+
+    GS3SQLConnection connection = GS3SQLConnectionFactory.getGS3SQLConnection(database);
+    DocumentList docList = DocumentList.readSQLDocuments(connection);
+    docList.writeDocuments(new File(database));
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AZListClassifier.java	(revision 21112)
@@ -0,0 +1,356 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class AZListClassifier extends AbstractClassifier
+{
+  GS3SQLConnection database;
+  MultiMap         alphaMap;
+  List             fields;
+  String           name;
+
+  class AZDocumentItem
+  { public DocumentID documentId;
+    public String     title;
+
+    public AZDocumentItem(DocumentID documentId, String title)
+    { this.documentId = documentId;
+      this.title = title;
+    }
+
+    public String getTitle()
+    { return this.title;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.title;
+    }
+  }
+
+  private void sortList(String key)
+  { List list = this.alphaMap.getAll(key);
+    if (list.size() > 1) {
+      int first, last;
+
+      first = 0;
+      last = list.size() - 1;
+      while (first != last)
+      { int at = (first + last) / 2;
+
+        if (list.get(at).toString().compareTo(list.get(list.size() - 1).toString()) > 0) {
+	  last = at;
+        }
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newObject = list.get(list.size()-1);
+      last = list.size() - 1;
+      while (last > first) {
+	list.set(last, list.get(last-1));
+	last --;
+      }
+      list.set(first, newObject);
+    }
+  }
+
+  public AZListClassifier(String name, List fields)
+  { this.fields   = fields;
+    this.alphaMap = new MultiMap();
+    this.name     = name;
+    //  System.out.println("AZList :"+fields.size());
+  }
+
+  public AZListClassifier()
+  { this.fields = new ArrayList();
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  public void setDatabase(GS3SQLConnection connection)
+  { this.database = connection;
+  }
+
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      // ...and send them to the classifier
+      Iterator thisValue = values.iterator();
+      while (thisValue.hasNext()) 
+      { String value = thisValue.next().toString();
+
+	String classText = null;
+
+	int c = 0;
+	while (c < value.length() && classText == null)
+	{ if (Character.isDigit(value.charAt(c))) 
+	  { classText = "0-9";
+	  }
+	  else if (Character.isLetter(value.charAt(c))) 
+	  { classText = value.substring(c, c+1).toLowerCase();
+	  }
+  	  c ++;
+	}
+
+	if (classText != null) {
+	    //	  System.out.println(classText + " " + documentID);
+	  this.alphaMap.put(classText, new AZDocumentItem(documentID, value));
+	  this.sortList(classText);
+	}
+      }
+    }
+
+    return true;    
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void endClassifierPass(int pass)
+  {
+  }
+
+  public void completeClassification()
+  { this.writeSQL(this.database);
+  }
+
+  public int writeSQLClassifyNode(GS3SQLConnection connection, String parentId, int orderRef,
+				  String label, String name, String description, int noOfLeafDocs)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+  
+    int classifyRef;
+
+    // check for existing node
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", label);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    try {
+      Statement statement = connection.createStatement();
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	insert.addValue("ParentID", parentId);
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", label);
+      action.addValue("Name", name);
+      action.addValue("Description", description);
+      action.addValue("ClassifyOrder", Integer.toString(orderRef), GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(noOfLeafDocs), GS3SQLField.INTEGER_TYPE);
+
+      // do the update/insert
+      statement.execute(action.toString());
+
+
+      // get the ClassifyRef if we don't already have it (have done a
+      // insert action above)...
+      if (classifyRef == -1) {
+	results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	  return -1;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      
+      statement.close();
+    }  catch (SQLException sqlEx) {
+	System.err.println("AZListClassifier.writeSQLClassifyNode(): "+sqlEx);
+	return -1;
+    }
+  
+
+    return classifyRef;
+  }
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  { GS3SQLAction action;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+ 
+    Iterator keys;
+ 
+    String prefix = this.name;
+    if (prefix==null || prefix.equals("")) {
+	prefix = "CLAZ"+this.fields.get(0).toString();
+	// TODO: write this name back to collectionConfig.xml
+	this.name = prefix;
+    }
+
+    int leafCount = 0;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+      if (childDocs != null) {
+	leafCount += childDocs.size();
+      }
+    }
+
+    int parentClassify = this.writeSQLClassifyNode(connection, "", 0, prefix, "", "Classifier", leafCount);
+
+    if (parentClassify < 0) 
+    { return false;
+    }
+
+    try {
+	Statement statement = connection.createStatement();
+    
+    List children;
+    
+    // TODO: cope with change rather than create from scratch...
+    int nodeOrder = 1;
+    keys = this.alphaMap.keySet().iterator();
+    while (keys.hasNext()) {
+      Object key = keys.next();
+
+      List childDocs = (List) this.alphaMap.getAll(key);
+
+      if (childDocs != null) {
+	String className = prefix + "." + key.toString();
+
+	int classifyRef = this.writeSQLClassifyNode(connection, prefix, nodeOrder, className, key.toString(), key.toString(), childDocs.size());
+
+	// note the child documents...
+        Iterator iterator = childDocs.iterator();
+	int childOrder = 1;
+	//St
+	while (iterator.hasNext()) {
+	  AZDocumentItem documentItem = (AZDocumentItem) iterator.next();
+	  DocumentID docId = documentItem.getID();
+
+	  insert = new GS3SQLInsert("classdocuments");
+	  insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+	  insert.addValue("DocID", docId.toString());
+	  insert.addValue("DocOrder", Integer.toString(childOrder), GS3SQLField.INTEGER_TYPE);	  
+
+	  statement.execute(insert.toString());
+
+	  childOrder ++;
+	}
+
+	nodeOrder ++;
+      }
+    }
+
+   
+    /*
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+
+      statement.execute(delete.toString());
+    }
+
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	return false;
+      }
+    }
+    */
+    statement.close();
+    } catch (SQLException e) {
+	System.err.println("AZListClassifier.writeSQL(): "+e);
+	return false;
+    }
+
+    return true;
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractClassifier.java	(revision 21112)
@@ -0,0 +1,61 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+
+public abstract class AbstractClassifier implements ClassifierInterface
+{ public AbstractClassifier()
+  {
+  }
+
+  public void configure(URL parentURL, org.w3c.dom.Node node)
+  { NodeList children = node.getChildNodes();
+    for (int c = 0; c < children.getLength(); c ++) {
+      Node child = children.item(c);
+      
+      if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	String name = child.getNodeName();
+	
+	if (name.equals("file")) {
+	  NamedNodeMap atts = children.item(c).getAttributes();
+	  Node attribute = atts.getNamedItem("URL");
+	  String urlString = attribute.getNodeValue();
+	  if (urlString == null)
+	    continue;
+	  
+	  try {
+	    URL url = new URL(parentURL, urlString);
+	    //	    this.file = url;
+	    System.out.println(url);
+	  }
+	  catch (java.net.MalformedURLException malEx) {
+	    System.out.println(malEx);
+	  }
+	}
+	else if (name.equals("field")) {
+	  String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	  System.out.println("Adding field " + fieldName);
+	  //	  this.fields.add(fieldName.toString());
+	}
+	else if (name.equals("sort")) {
+	  String sortName = DOMUtils.getNodeChildText(children.item(c));
+	  //	  this.sort = sortName;
+	}
+      }
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/AbstractHierarchyNode.java	(revision 21112)
@@ -0,0 +1,428 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public abstract class AbstractHierarchyNode
+{
+  String prefix;        // a basic prefix used in this hierarchy
+  String descriptor;    // the textual descriptor used on screen or long-hand
+  String name;          // the index number, letter assignment or other item
+                        //   used to identify the position of the item in the
+                        //   hierarchy
+  String id;            // an identifier used by the GLI for maintenance
+                        //   purposes; this plays no active role in the
+                        //   rebuilding process (at the moment)
+  List childNodes;      // the child classification nodes of this node
+  List childDocs;       // the child documents of this node
+  List matches;         // the other metadata values that may be matched
+                        //   against the classifier
+  AbstractHierarchyNode parent; // the parent of the node
+
+  class AbstractHierarchyDocument
+  { DocumentID documentId;
+    String     sortKey;
+
+    public AbstractHierarchyDocument(DocumentID documentId, String sortKey)
+    { this.documentId = documentId;
+      this.sortKey    = sortKey;
+    }
+
+    public DocumentID getID()
+    { return this.documentId;
+    }
+
+    public String toString()
+    { return this.sortKey;
+    }
+  } 
+
+  /**
+   *  Simple node
+   */
+  public AbstractHierarchyNode()
+  { this.descriptor = "";
+    this.name       = "";
+    this.id         = "";
+    this.prefix     = "";
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+  }
+    
+  public AbstractHierarchyNode(String prefix, String name, String id, String descriptor)
+  { this.descriptor = descriptor;
+    this.name       = name;
+    this.id         = id;
+    this.prefix     = prefix;
+    this.childNodes = new ArrayList();
+    this.childDocs  = new ArrayList();
+    this.parent     = null;
+    this.matches    = new ArrayList();
+    this.matches.add(this.id);
+    this.matches.add(this.name);
+  }
+    
+  public void addChild(AbstractHierarchyNode child)
+  { this.childNodes.add(child);
+    child.setParent(this);
+
+    /**
+    if (this.id == null) {
+      System.out.println(child.id.toString() + " added to root");
+    }
+    else {
+      System.out.println(child.id.toString() + " added to " + this.id);
+    }
+    */
+  }
+
+  public boolean add(AbstractHierarchyNode child)
+  { if (this.id != null && this.id.length() > 0 &&
+        !child.id.startsWith(this.id + "."))
+    { return false;
+    }
+
+    Iterator subNodes = this.childNodes.iterator();
+    while (subNodes.hasNext()) {
+      AbstractHierarchyNode subNode = (AbstractHierarchyNode) subNodes.next();
+      
+      if (subNode.add(child)) {
+	return true;
+      }
+    }
+    
+    this.addChild(child);
+    return true;
+  }
+
+  public void addDocument(DocumentID document, String sortKey)
+  { if (sortKey == null && false) {
+    System.err.println("attempt to classify by an unset key");
+      return;
+    }
+    this.childDocs.add(new AbstractHierarchyDocument(document, sortKey));
+
+    System.out.println("Adding " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    if (sortKey != null) {
+      int first = 0;
+      int last  = this.childDocs.size() - 1;
+
+      while (first != last) {
+	int at = (first + last) / 2;
+	
+	System.out.println(this.name + " " + at + " " + this.childDocs.size() + " " + this.childDocs.get(at) + " " + sortKey);
+	if (this.childDocs.get(at) == null ||
+	    this.childDocs.get(at).toString() == null) {
+	  last = at;
+	  System.out.println("COntinuing");
+	  continue;
+	}
+
+	if (this.childDocs.get(at).toString().compareTo(sortKey) > 0) {
+	  last = at;
+	}
+	else {
+	  first = at + 1;
+	}
+      }
+
+      Object newItem = this.childDocs.get(this.childDocs.size() - 1);
+      last = this.childDocs.size() - 1;
+      while (last > first) {
+	this.childDocs.set(last, this.childDocs.get(last-1));
+	last --;
+      }
+      this.childDocs.set(first, newItem);
+    }
+
+    //  System.out.println("Added " + document.toString() + " to " + this.name + " " + this.childDocs.size());
+    //    for (int a = 0; a < this.childDocs.size(); a ++) {
+    //System.out.println(this.childDocs.get(a));
+    //}
+  }
+
+  public void setParent(AbstractHierarchyNode parent)
+  { this.parent = parent;
+  }
+    
+  public AbstractHierarchyNode getParent()
+  { return this.parent;
+  }
+
+  public String getParentId()
+  { 
+    if (this.id == null) {
+      return "";
+    }
+
+    int dotAt = this.id.lastIndexOf('.');
+    if (dotAt < 0) {
+      return "";
+    }
+    return this.id.substring(0, dotAt);
+  }  
+
+  public void setDescriptor(String descriptor)
+  { this.descriptor = descriptor;
+  }
+
+  public void setID(String id)
+  { this.id = id;
+    this.matches.add(id);
+  }
+
+  public String getID()
+  { return this.id;
+  }
+
+  public int noOfChildDocs()
+  { return this.childDocs.size();
+  }
+
+  public int noOfLeafDocs()
+  { int total = this.childDocs.size();
+
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      total += childNode.noOfLeafDocs();
+    }
+
+    return total;
+  }
+
+  /**
+   *  Set the name for this hierarchy node - i.e. its brief description
+   *
+   *  @param <code>String</code> the new node name.
+   */
+  public void setName(String name)
+  { this.name = name;
+    this.matches.add(name);
+  }
+
+  /**
+   *  Get the name for this hierarchy node - i.e. its brief description
+   *
+   *  @return <code>String</code> the node's name.
+   */
+  public String getName()
+  { return this.name;
+  }
+
+  /**
+   *  Add another string which can be used to describe this hierarchy node - i.e. one which if
+   *  found in a document would indicate that that document belongs to this hierarchy node.
+   *
+   *  @param <code>String</code> the string to match.
+   */
+  public void addMatch(String match)
+  { this.matches.add(match);
+  }
+
+  /**
+   *  Check if a given string matches any of the values given for this hierarchy node...
+   *
+   *  @param <code>String</code> the descriptive string to be compared against the hierarchy
+   *  @return <code>boolean</code> <code>true</code> whether the string matches this hierarchy
+   */
+  public boolean isMatch (String toMatch)
+  { Iterator thisMatch = this.matches.iterator();
+			
+    while (thisMatch.hasNext())
+    { String thisMatchText = thisMatch.next().toString();
+      
+      if (thisMatchText.equals(toMatch))
+      { return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   *  Take a document, and find the classifications that it matches against in
+   *  the current hierarchy.  
+   *
+   *  @param <code>DocumentID</code> the id of the document being classified
+   *  @param <code>List</code> the values against which the classifier should
+   *         test for the document being a match - i.e. the pertinent document
+   *         property values.
+   *  @param <code>ClassifierObserverInterface</code>
+   *         object modifies the document with information about the
+   *         classifications that it fell within.
+   */
+  abstract public void getClassifications(DocumentID documentID, List values, String sortKey,
+					  ClassifierObserverInterface observer);
+
+  public boolean writeSQL(GS3SQLConnection connection)
+  {
+    int classifyRef;
+    GS3SQLAction action = null;
+    GS3SQLSelect select;
+    GS3SQLInsert insert;
+
+    Statement statement;
+    // can we connect to the database?
+    try {
+	statement = connection.createStatement();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	return false;
+    }
+    
+    // Get own full id
+    String fullId = this.id.length() > 0 ? this.prefix+"."+this.id : this.prefix;
+
+    // check for an existing instance of this classifier
+    select = new GS3SQLSelect("classifiers");
+    select.addField("ClassifyRef");
+    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ClassifyID", "=", fullId);
+    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+    select.setWhere(where);
+
+
+    // update or insert the classifier as required
+    try {
+      ResultSet results = statement.executeQuery(select.toString());
+      if (results.first()) {
+	GS3SQLUpdate update = new GS3SQLUpdate("classifiers");
+	update.setWhere(where);
+	action = update;
+ 
+	classifyRef = results.getInt("ClassifyRef");
+      }
+      else {
+	insert = new GS3SQLInsert("classifiers");
+
+	if (this.id == null || this.id.length() == 0) {
+	  insert.addValue("ParentID", "");
+	}
+	else {
+	  String parentId = this.getParentId();
+
+	  if (parentId.length() > 0) {
+	    insert.addValue("ParentID", this.prefix+"."+this.getParentId());
+	  }
+	  else {
+	    insert.addValue("ParentID", this.prefix);
+	  }
+	}
+
+	action = insert;
+	classifyRef = -1;
+      }
+      action.addValue("ClassifyID", fullId);
+      action.addValue("Name", this.name);
+      action.addValue("Description", this.descriptor);
+
+      String tailId = "0";
+      if (this.id.length() > 0) {
+	int lastDot = this.id.lastIndexOf('.');
+	if (lastDot >= 0) {
+	  tailId = this.id.substring(lastDot+1);
+	}
+	else {
+	  tailId = this.id;
+	}
+      }
+      action.addValue("ClassifyOrder", tailId, GS3SQLField.INTEGER_TYPE);
+      action.addValue("NumLeafDocs", Integer.toString(this.noOfLeafDocs()), GS3SQLField.INTEGER_TYPE);
+
+      statement.execute(action.toString());
+    }
+    catch (SQLException sqlEx) {
+      if (action == null) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+      }
+      else {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx + " " + action.toString());
+      }
+      return false;
+    }
+
+    // get the ClassifyRef if we don't already have it (have done a
+    // insert action above)...
+    if (classifyRef == -1) {
+      try {
+        ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) {
+	    statement.close();
+	    return false;
+	}
+	
+	classifyRef = results.getInt("ClassifyRef");
+	
+      }
+      catch (SQLException sqlEx) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+sqlEx);
+	return false;
+      }
+    }
+    else {
+      // TODO: clear 'dead' child classifications
+
+      // delete child documents
+      GS3SQLDelete delete = new GS3SQLDelete("classdocuments");
+      delete.setWhere(where);
+      try {
+	  statement.execute(delete.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+    }
+    
+    // post the child nodes...
+    Iterator iterator = this.childNodes.iterator();
+    while (iterator.hasNext()) {
+      AbstractHierarchyNode childNode = (AbstractHierarchyNode) iterator.next();
+
+      if (!childNode.writeSQL(connection)) {
+	  System.out.println("Failed to write " );
+	  return false;
+      }
+    }
+
+    // note the child documents...
+    iterator = this.childDocs.iterator();
+    int order = 1;
+    while (iterator.hasNext()) {
+      AbstractHierarchyDocument hierarchyDoc = (AbstractHierarchyDocument) iterator.next();
+      DocumentID docId = hierarchyDoc.getID();
+
+      insert = new GS3SQLInsert("classdocuments");
+      insert.addValue("ClassifyRef", Integer.toString(classifyRef), GS3SQLField.INTEGER_TYPE);
+      insert.addValue("DocID", docId.toString());
+      insert.addValue("DocOrder", Integer.toString(order), GS3SQLField.INTEGER_TYPE);
+      try {
+	  statement.execute(insert.toString());
+      } catch (SQLException e) {
+	  System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+	  return false;
+      }
+      order ++;
+    }
+    // close the statment
+    try {
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("AbstractHierarchyNode.writeSQL(): "+e);
+    }
+
+    return true;
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierInterface.java	(revision 21112)
@@ -0,0 +1,44 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface ClassifierInterface
+{
+  /**
+   *  Initialise the classifier as required.  The parameters given are specific
+   *  to the classifier type, so this interface does not guarantee any interpretations
+   *  of any given parameter.
+   *  
+   *  @param <code>List</code> the parameters with which to configure the classifier.
+   */
+  public void configure(List parameters);
+  public void configure(java.net.URL baseURL, org.w3c.dom.Node node);
+
+  public void setDatabase(org.greenstone.gsdl3.gs3build.database.GS3SQLConnection connection);
+
+  /**
+   *  Classify an individual document - the document should be informed of any
+   *  classification nodes which it is assigned to, so that it can obtain knowledge
+   *  of any "parents".  These are assigned into the "classified" metadata item in
+   *  the "gsdl3" namespace...
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document);
+  
+  public void startClassifierPass(int pass);
+  
+  public int getClassifierPasses();
+  
+  public void endClassifierPass(int pass);
+  
+  /**
+   *  Called at the end of the classification sequence - the classifier should
+   *  save its state at this point...
+   */
+  public void completeClassification();
+
+    public boolean addClassifierDescription(Element classifier_list);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierManager.java	(revision 21112)
@@ -0,0 +1,155 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.util.List;
+import java.util.Iterator;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class ClassifierManager
+{
+  DocumentList           documents;
+  ClassifierInterface [] list;
+  int                    size;
+  int                    used;
+  GS3SQLConnection       database;
+
+  /**
+   *  Instantiate a classifier manager that will classify the
+   *  documents in a given <code>DocumentList</code> object.
+   *  The list of documents should not change during the
+   *  <code>classifyDocuments</code> function, and the documents
+   *  themselves may be altered at that time.
+   *
+   *  @param <code>DocumentList</code> a provider of documents for
+   *         classification.
+   */
+  public ClassifierManager(DocumentList documentList, GS3SQLConnection database)
+  { this.list = new ClassifierInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+    this.database  = database;
+  }
+
+  /**
+   *  Add a classifier to the end of the list of classifiers...
+   *
+   *  @param <code>ClassifierInterface</code> the classifier.
+   */
+  public void addClassifier(ClassifierInterface classifier)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = classifier;
+    this.used ++;
+
+    classifier.setDatabase(this.database);
+  }
+
+  /**
+   *  Classify an individual document - the document itself may be written
+   *  to in consequence, in which case the document <strong>must</strong>
+   *  be rewritten to disk.  This function does <strong>not</strong> do the
+   *  writing itself.
+   *
+   *  @param <code>DocumentID</code> the document's identifier.
+   *  @param <code>DocumentInterface</code> the document itself.
+   */
+  public void classifyDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    {	this.list[i].classifyDocument(docId, document);
+    }
+  }
+
+  /**
+   *  Classify all the documents that the document list is able to provide...
+   */
+  public void classifyDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getClassifierPasses(); p ++)
+      {	this.list[i].startClassifierPass(p);
+
+	Iterator iterator = this.documents.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.list[i].classifyDocument(document.getID(), document)) {
+	      System.out.println("Ending document");
+	    }
+	    
+	    if (document.isChanged()) {
+	      //	      System.out.println("Writing document " + document.getID());
+	      this.documents.storeChangedDocument(document);
+	    }
+	  }
+	}
+	this.list[i].endClassifierPass(p);
+      }
+      this.list[i].completeClassification();
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ClassifierInterface newList [] = new ClassifierInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+
+  public static ClassifierInterface loadClassifier(String classifierName)
+  { System.out.println("Loading classifier " + classifierName);
+    try {
+      ClassifierInterface classifier = 
+	(ClassifierInterface) Class.forName("org.greenstone.gsdl3.gs3build.classifier."+classifierName+"Classifier").newInstance();
+      return classifier;
+    }
+    catch (ClassNotFoundException ex) {
+      System.err.println("Attempt to load classifier " + classifierName + " which does not exist");
+    }
+    catch (InstantiationException instEx) {
+      System.err.println("Unable to create classifier " + classifierName);
+    }
+    catch (IllegalAccessException accessEx) {
+      System.err.println("Unable to instantiate classifier " + classifierName + " due to permissions problems " + accessEx.toString());
+    }
+    return null;
+  }
+    
+  public static ClassifierInterface loadClassifier(String classifierName, List params)
+  { ClassifierInterface classifier = loadClassifier(classifierName);
+    if (classifier != null) {
+      classifier.configure(params);
+    }
+    return classifier;
+  }
+
+  public boolean addServiceDescriptions(Element service_class_list) {
+    if (this.used==0) {
+      // no classiifers
+      return true;
+    }
+    Document owner_doc = service_class_list.getOwnerDocument();
+    Element browse_service = owner_doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+    browse_service.setAttribute(GSXML.NAME_ATT, "GS3Browse");
+    service_class_list.appendChild(browse_service);
+    
+    Element classifier_list = owner_doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+    browse_service.appendChild(classifier_list);
+    boolean success = true;
+    for (int i = 0; i < this.used; i ++) {
+	if (!this.list[i].addClassifierDescription(classifier_list)) {
+	    success = false;
+	}
+    }
+    return success;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/ClassifierObserverInterface.java	(revision 21112)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+public interface ClassifierObserverInterface
+{
+  public void recordClassification(String label);
+  public java.util.List getClassifications();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/GS2HierarchyClassifier.java	(revision 21112)
@@ -0,0 +1,309 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+public class GS2HierarchyClassifier implements ClassifierInterface
+{ 
+  class GS2HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public GS2HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+      this.document.setChanged(true);
+      //      System.out.println("Assigned document " + this.document.getID().toString() + " to " + label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class GS2HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public GS2HierarchyNode()
+    { super();
+    }
+    
+    public GS2HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    /**
+     *  Take a document, and find the classifications that it matches against in
+     *  the current hierarchy.  
+     *
+     *  @param <code>DocumentID</code> the id of the document being classified
+     *  @param <code>List</code> the values against which the classifier should
+     *         test for the document being a match - i.e. the pertinent document
+     *         property values.
+     *  @param <code>ClassifierObserverInterface</code>
+     *         object modifies the document with information about the
+     *         classifications that it fell within.
+     */
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { // Go through the list of values, and if this node matches, record
+      // the fact in the classifier observer...
+      Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.id);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((GS2HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class GS2HierarchyHandler extends GS2TextFileHandler
+  { GS2HierarchyNode rootNode = null;
+
+    GS2HierarchyHandler(String hierarchyName, String content)
+    { super(content);
+ 
+      String prefix = "CL"+hierarchyName;
+      this.rootNode = new GS2HierarchyNode(prefix, hierarchyName, "", hierarchyName);
+
+      while (this.hasMoreLines()) {
+	String description;
+	String name, id;
+
+	this.getLine();
+	
+	name = this.getEntry(true);
+	if (name == null || name.length() == 0) {
+	  continue;
+	}
+
+	id = this.getEntry(true);
+	if (id == null || id.length() == 0) {
+	  continue;
+	}
+
+	description = this.getEntry(true);
+	if (description == null || description.length() == 0) {
+	  continue;
+	}
+	
+	// Initialise the tree
+	GS2HierarchyNode node = new GS2HierarchyNode(prefix, name, id, description);
+
+	this.rootNode.add(node);
+      }
+    }
+
+    public GS2HierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+
+  private          GS2HierarchyNode hierarchy;
+  List             fields;
+  String           sortBy;
+  GS3SQLConnection database;
+
+  public GS2HierarchyClassifier()
+  { // a 'do nothing' constructor which takes more information from configure...
+    this.fields = new ArrayList();
+    this.database = null;
+  }
+
+  public GS2HierarchyClassifier(File file)
+  {	
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler("CL", documentText);
+
+    this.hierarchy = handler.getHierarchy();
+    this.fields = new ArrayList();
+    //    System.out.println("loaded hierarchy for "+ file.toString());
+    this.database = null;
+  }
+
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  public void configure(URL url, org.w3c.dom.Node node)
+  {
+  }
+
+  public void configure(List parameters)
+  {
+    File file = null;
+    
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+      else if (param.equals("-hfile")) {
+	// this parameter had better have been expanded previously to a full path,
+	// as we're not going to do that here!!!
+	if (iterator.hasNext()) {
+	  file = new File(iterator.next().toString());
+	}
+      }
+      else if (param.equals("-sort")) {
+	if (iterator.hasNext()) {
+	  String sort = iterator.next().toString();
+	  if (sort != null && sort.length() > 0) {
+	    this.sortBy = sort;
+	  }
+	}
+      }
+    }
+
+    if (file == null) {
+      return;
+    }
+
+    String documentText = 
+      DocumentLoader.getAsString(file);
+	
+    if (documentText == null) {
+      System.err.println("GS2HierarchyClassifier: Unable to load any content for " + file);
+      return;
+    }
+
+    GS2HierarchyHandler handler = new GS2HierarchyHandler(this.fields.get(0).toString(), documentText);
+
+    this.hierarchy = handler.getHierarchy();    
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { if (this.hierarchy == null) {
+      return true;
+    }
+    
+    // the observer records the assignment of documents to classifications
+    GS2HierarchyClassifierObserver classifyObserver = new GS2HierarchyClassifierObserver(document);
+		
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      //      System.out.println("Matching " + values.get(0).toString());
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+
+    // The hierarchy member may be null if the classifier did not initialise correctly...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	//classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier.setAttribute(GSXML.NAME_ATT, "name"); // what is teh name???
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/classifier/HierarchyClassifier.java	(revision 21112)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.classifier;
+
+import java.io.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.util.GSXML;
+
+
+public class HierarchyClassifier extends AbstractClassifier
+{ 
+  class HierarchyClassifierObserver implements ClassifierObserverInterface
+  { List              classifications;
+    DocumentInterface document;
+
+    public HierarchyClassifierObserver(DocumentInterface document)
+    { this.document = document;
+    }
+
+    public void recordClassification(String label)
+    { this.document.addDocumentMetadata("gsdl3", "classified", label);
+    }
+
+    public List getClassifications()
+    { return this.classifications;
+    }
+  }
+
+  class HierarchyNode extends AbstractHierarchyNode
+  { /**
+     *  Simple node
+     */
+    public HierarchyNode()
+    { super();
+    }
+
+    public HierarchyNode(String prefix, String name, String id, String descriptor)
+    { super(prefix, name, id, descriptor);
+    }
+    
+    public void getClassifications(DocumentID documentID, List values, String sortKey,
+				   ClassifierObserverInterface observer)
+    { Iterator valueList = values.iterator();
+
+      while (valueList.hasNext())
+      {	if (this.isMatch(valueList.next().toString()))
+	{ observer.recordClassification(this.name);
+	  this.addDocument(documentID, sortKey);
+	}
+      }
+     
+      // recurse into the child nodes for them to do the same...
+      Iterator childList = this.childNodes.iterator();
+
+      while (childList.hasNext())
+      { ((HierarchyNode) childList.next()).getClassifications(documentID, values, sortKey, observer);
+      }
+    }
+  }
+
+  /**
+   *  An inner class to handle Classifier files
+   */
+  class HierarchyHandler extends DefaultHandler
+  { StringBuffer description;
+    StringBuffer name;
+    StringBuffer path;
+    StringBuffer match;
+    String       prefix;
+    boolean inElement;
+    AbstractHierarchyNode rootNode = null;
+    AbstractHierarchyNode currentNode = null;
+
+    HierarchyHandler(String classifierName)
+    { super();
+ 
+      this.name        = null;
+      this.description = null;
+      this.path        = null;
+      this.match       = null;
+      this.prefix      = classifierName;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("Hierarchy"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(prefix, "", "", "Classifier");
+
+	this.rootNode = node;
+	this.currentNode = node;
+      }
+      else if (localName.equals("Classification"))
+      {	// create a new node in the hierarchy
+	HierarchyNode node = new HierarchyNode(this.prefix, "", "", "");
+
+	// Initialise the tree
+	if (this.currentNode == null)
+	{ this.rootNode = node;
+	}
+        // Add the node
+	else
+	{ this.currentNode.addChild(node);
+	}
+	this.currentNode = node;
+      }
+      else if (localName.equals("Name"))
+      {	this.name = new StringBuffer();
+      }
+      else if (localName.equals("Path"))
+      {	this.path   = new StringBuffer();
+      }
+      else if (localName.equals("Description"))
+      {	this.description = new StringBuffer();
+      }
+      else if (localName.equals("Matches"))
+      {	this.match = new StringBuffer();
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("Classification"))
+      {	if (this.currentNode.getParent() == null) {
+	  // TODO: Error state
+          // this.rootNodes.add(this.currentNode);
+        }
+	this.currentNode = this.currentNode.getParent();
+      }
+      else if (localName.equals("Name"))
+      { this.currentNode.setName(XMLTools.cleanString(this.name.toString()));
+      System.out.println("Name is " + this.name);
+        this.name = null;
+      }
+      else if (localName.equals("Path"))
+      {	this.currentNode.setID(XMLTools.cleanString(this.path.toString()));
+        this.path = null;
+      }
+      else if (localName.equals("Description"))
+      { this.currentNode.setDescriptor(XMLTools.cleanString(this.description.toString()));
+        this.description = null;
+      }
+      else if (localName.equals("Matches"))
+      { this.currentNode.addMatch(XMLTools.cleanString(this.match.toString()));
+        this.match = null;
+      }
+      else if (localName.equals("Children"))
+      {	
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { String string = new String(c, start, length);
+      if (this.path != null)
+      { this.path.append(string);
+      }
+      else if (this.name != null)
+      { this.name.append(string);
+      }
+      else if (this.description != null)
+      { this.description.append(string);
+      }
+      else if (this.match != null)
+      { this.match.append(string);
+      }
+    }
+
+    public AbstractHierarchyNode getHierarchy()
+    { return this.rootNode;
+    }
+  }
+  
+  private GS3SQLConnection database;
+  private AbstractHierarchyNode    hierarchy;
+  private List             fields;
+  private String           sortBy;
+  private String           name;
+
+  public HierarchyClassifier(String name, URL basefile, List fields, String sortBy)
+  {	
+    try 
+    { SAXParser parser = new SAXParser();
+      HierarchyHandler handler = new HierarchyHandler(name);
+      /*
+      XMLReader reader = XMLReaderFactory.createXMLReader();
+      reader.setContentHandler(handler);
+      reader.setErrorHandler(handler);*/
+      parser.setContentHandler(handler);
+
+      // A Classifier consists of one file only - get it.
+      /*
+      FileReader fileReader = new FileReader(basefile);
+      reader.parse(new InputSource(fileReader));
+      */
+      parser.parse(basefile.toString());
+
+      this.hierarchy = handler.getHierarchy();
+      this.fields = fields;
+      this.sortBy = sortBy;
+      this.name = name;
+      System.out.println("Counting " + this.fields.size() + " fields");
+    }
+    catch (SAXException saxException)
+    { // TODO: log error
+	System.out.println("unable to process hierarchy " + saxException.toString());
+    }
+    catch (java.io.FileNotFoundException fileException)
+    {
+	System.out.println("unable to process hierarchy " + fileException.toString());
+    }
+    catch (java.io.IOException ioException)
+    {
+	System.out.println("unable to process hierarchy " + ioException.toString());
+    }
+  }
+
+  public void configure(List parameters)
+  {
+    Iterator iterator = parameters.iterator();
+    while (iterator.hasNext()) {
+      String param = iterator.next().toString();
+      if (param.equals("-metadata")) {
+	if (iterator.hasNext()) {
+	  String field = iterator.next().toString();
+	  if (field != null && field.length() > 0) {
+	    this.fields.add(field);
+	  }
+	}
+      }
+    }
+  }
+
+  /**
+   *  Indicate the database to be used for saving the hierarchy to.
+   *
+   *  @param <code>GS3SQLConnection</code> the database
+   */
+  public void setDatabase(GS3SQLConnection database)
+  { this.database = database;
+  }
+
+  /**
+   *  Classify a particular document - the document id will be used to record its
+   *  membership of any classifier nodes, and the document interface will also be
+   *  used to assign back knowledge of any classifiers to which the document is
+   *  assigned.
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>DocumentInterface</code> the document interface
+   *
+   *  @return <code>boolean</code> a return value - currently always 
+   *          <code>true</code> for success
+   */
+  public boolean classifyDocument(DocumentID documentID, DocumentInterface document)
+  { // the observer records the assignment of documents to classifications
+    HierarchyClassifierObserver classifyObserver = new HierarchyClassifierObserver(document);
+
+    // if we have no fields, there is nothing to do...
+    if (this.fields == null)
+    { return true;
+    }
+
+    // get the metadata item from the document
+    Iterator thisField = this.fields.iterator();
+
+    // for every field listed as being a scannable field...
+    while (thisField.hasNext())
+    { String fieldName = thisField.next().toString();
+			
+      // ...get the values for that field...
+      List values = document.getDocumentMetadataItem(fieldName);
+
+      if (values == null)
+      {	continue;
+      }
+
+      if (documentID == null) {
+	System.out.println("Bad documentID");
+	continue;
+      }
+
+      // get the sort key for the metadata item if possible
+      String sortKey = null;
+
+      if (this.sortBy != null) {
+	List sortKeys = document.getDocumentMetadataItem(this.sortBy);
+	if (sortKeys != null && sortKeys.size() > 0) {
+	  sortKey = sortKeys.get(0).toString();
+	}
+      }
+
+      // ...and send them to the classifier
+      this.hierarchy.getClassifications(documentID, values, sortKey, classifyObserver);
+    }
+    return true;
+  }
+
+  /**
+   *  @see ClassifierInterface:getClassifierPasses
+   */
+  public int getClassifierPasses()
+  { return 1;
+  }
+
+  public void startClassifierPass(int pass)
+  {
+  }
+  
+  public void endClassifierPass(int pass)
+  {
+  }
+  
+  public void completeClassification()
+  { // TODO: store the classifications to file...
+    if (this.hierarchy != null) {
+      this.hierarchy.writeSQL(this.database);
+    }
+  }
+
+    public boolean addClassifierDescription(Element classifier_list) {
+	// TODO check that there are some docs in the classification
+	Element classifier = classifier_list.getOwnerDocument().createElement(GSXML.CLASSIFIER_ELEM);
+	classifier.setAttribute(GSXML.NAME_ATT, this.name);
+	classifier_list.appendChild(classifier);
+	return true;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/BuildIDFactory.java	(revision 21112)
@@ -0,0 +1,31 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.util.GregorianCalendar;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public class BuildIDFactory implements IDFactoryInterface
+{
+	int sequenceNumber;
+
+	public BuildIDFactory(BuildManager manager)
+	{
+		GregorianCalendar today = new GregorianCalendar();
+
+		String oldBuild = manager.getLastBuildSequence();
+		if (oldBuild != null)
+		{ this.sequenceNumber = Integer.parseInt(oldBuild);
+		}
+		else
+		{	this.sequenceNumber = 0;
+		}
+	}
+
+	public DocumentID getDocumentID(DocumentInterface document)
+	{	String value = new String(Integer.toString(this.sequenceNumber));
+		this.sequenceNumber ++;
+		return new DocumentID(value);
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionCrawlObserver.java	(revision 21112)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.gs3build.classifier.*;
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+public class CollectionCrawlObserver implements FileCrawlObserver
+{
+  BuildManager manager;
+
+  public CollectionCrawlObserver(BuildManager manager)
+  { this.manager = manager;
+  }
+
+  public void processFile(File file)
+  { if (file.getName().equals("collect.cfg")) {
+      GS2CollectionCfg config = GS2CollectionCfg.getGS2CollectionCfg(this.manager, file);
+    }
+    else if (file.toString().endsWith(".txt")) {
+      // Commented out: this is now done on the line above...
+      // GS2HierarchyClassifier classifier = new GS2HierarchyClassifier(file);
+      // manager.getClassifierManager().addClassifier(classifier);
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/CollectionMetadata.java	(revision 21112)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+/**
+ *  A class to consolidate the metadata assigned to the documents of a collection
+ *  as per the extension to Greenstone 2 done by George Buchanan.
+ */
+
+public class CollectionMetadata
+{ METSDescriptive metadata;
+	
+  public CollectionMetadata()
+  { this.metadata = new METSDescriptive("CM0", "collection");
+  }
+
+  /**
+   *  Accrue the metadata assigned to a particular document
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the field of metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void addDocumentMetadata(DocumentID documentID, String namespace, String field, String value)
+  { // we don't presently use the document ID in this member function
+    this.metadata.addMetadata(namespace, field, value);
+  }
+
+  /**
+   *  Set the metadata stored on a given metadata item - any other values are
+   *  destroyed.
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   *  @param <code>String</code> the actual value assigned
+   */
+  public void setCollectionMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata(namespace, label, value);
+  }
+
+  /**
+   *  Get the metadata stored on a given metadata item
+   *
+   *  @param <code>String</code> the namespace
+   *  @param <code>String</code> the label of the metadata
+   */
+  public List getCollectionMetadata(String namespace, String label)
+  { return this.metadata.getMetadata(namespace, label);
+  }
+
+  /**
+   *  Write the collection metadata
+   *
+   *  TODO: dummy function - complete
+   */
+  public boolean write(PrintWriter writer)
+  { return false;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/GS2CollectionCfg.java	(revision 21112)
@@ -0,0 +1,159 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import java.io.File;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierInterface;
+import org.greenstone.gsdl3.gs3build.classifier.ClassifierManager;
+
+import org.greenstone.gsdl3.gs3build.indexers.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class GS2CollectionCfg
+{
+  BuildManager buildManager;
+  IndexerInterface indexer;
+
+  class GS2CollectCfgHandler extends GS2TextFileHandler
+  { 
+    
+    GS2CollectCfgHandler(String content, File collectFile, GS2CollectionCfg configurator)
+    { super(content);
+ 
+      while (this.hasMoreLines()) {
+	String lineType;
+
+	this.getLine();
+	
+	lineType = this.getEntry(true);
+	if (lineType == null || lineType.length() == 0) {
+	  continue;
+	}
+
+	if (lineType.equals("classify"))
+	{
+	  String classType = this.getEntry(true);
+	  if (classType == null || classType.length() == 0) {
+	    continue;
+	  }
+	  classType = "GS2" + classType;
+	  
+	  List paramList = new ArrayList();
+	  
+	  // read in the parameters - to allow for better abstraction of other parts,
+	  // the handling of this is a bit 'odd', specifically modifying any file
+	  // references here before passing them on...
+	  String parameter = null;
+	  String lastParam = null;
+	  while (this.hasMore())
+	  { String param = this.getEntry(true);
+	    
+	    if (param != null && param.length() > 0) 
+	    { // expand any hfile references...
+	      if (lastParam != null && lastParam.equals("-hfile")) {
+		param = collectFile.getParent() + File.separator + param;
+	      }
+	      
+	      paramList.add(param);
+	    }
+
+	    lastParam = param;
+	  }
+
+	  ClassifierInterface classifier = ClassifierManager.loadClassifier(classType, paramList);
+	  if (classifier != null) {
+	    configurator.addClassifier(classifier);
+	  }
+	  else {
+	    System.err.println("Classifier not loaded");
+	  }
+	}
+	else if (lineType.equals("buildtype")) {
+	  if (this.hasMore()) {
+	    String type = this.getEntry(true);
+	    if (type != null) {
+	      configurator.setBuildType(type);	      
+	    }
+	  }
+	}
+	else if (lineType.equals("indexes")) {
+	  while (this.hasMore()) {
+	    String index = this.getEntry(true);
+	    if (index != null) {
+	      System.out.println("Adding index " + index);
+	      configurator.addIndex(index);
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  public GS2CollectionCfg(BuildManager manager, File file)
+  {
+    String documentText = 
+      DocumentLoader.getAsString(file);
+
+    if (documentText == null) {
+      return;
+    }
+
+    this.buildManager = manager;
+
+    GS2CollectCfgHandler handler = new GS2CollectCfgHandler(documentText, file, this);
+  }
+  
+  void addClassifier(ClassifierInterface classifier)
+  {
+    ClassifierManager manager = this.buildManager.getClassifierManager();
+
+    if (manager != null) {
+      manager.addClassifier(classifier);
+    }
+    else {
+      System.out.println("Unable to install due to missing manager");
+    }
+  }
+
+  void addIndex(String index)
+  {
+    if (this.indexer == null) {
+      this.setBuildType("mg");
+    }
+    this.indexer.configure(IndexerInterface.GS2_INDEX_LABEL, index);
+  }
+
+  void setBuildType(String type)
+  { if (type.equals("mgpp")) {
+      this.indexer = new MGPPIndexer("mgpp");
+    }
+    else {
+      this.indexer = new MGIndexer("mg");
+    }
+    this.buildManager.addIndexer(this.indexer);
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, File cfgFile)
+  { GS2CollectionCfg configurator = new GS2CollectionCfg(manager, cfgFile);
+    return configurator;
+  }
+
+  public static GS2CollectionCfg getGS2CollectionCfg(BuildManager manager, String etcDir)
+  {
+    File file = new File(etcDir, "collect.cfg");
+    
+    if (!file.exists()) {
+      return null;
+    }
+
+    return getGS2CollectionCfg(manager, file);
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/collection/IDFactoryInterface.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.collection;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface IDFactoryInterface
+{
+	public DocumentID getDocumentID(DocumentInterface document);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/DatabaseAdaptorInterface.java	(revision 21112)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+/**
+ *  Facilitates communication between Greenstone and a database system,
+ *  be it gdbm, an SQL database or other product - what lies underneath
+ *  is down to you...
+ *
+ *  As this is a "write only" interface, the document is only stored as
+ *  required.
+ */
+
+import java.sql.*;
+
+public interface DatabaseAdaptorInterface
+{
+    //  public Connection getConnection();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLAction.java	(revision 21112)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+
+public class GS3SQLAction
+{ String      table;
+  List        values;
+
+  public GS3SQLAction(String table)
+  {
+    this.table  = table;
+    this.values = new ArrayList();
+  }
+
+  public void addValue(String field, String value, String type)
+  { this.values.add(new GS3SQLValue(field, value, type));
+  }
+  
+  public void addValue(String field, String value)
+  { this.values.add(new GS3SQLValue(field, value));
+  }
+
+  public void addDate(String field, java.sql.Timestamp date)
+  { this.values.add(new GS3SQLValue(field, getDateString(date)));
+  }
+
+  public static String getDateString(java.sql.Timestamp date)
+  { StringBuffer reply = new StringBuffer();
+    Calendar calendar = new GregorianCalendar();
+    calendar.setTime(date);
+
+    reply.append(Integer.toString(calendar.get(Calendar.YEAR)));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.MONTH)+1));
+    reply.append("-");
+    reply.append(Integer.toString(calendar.get(Calendar.DAY_OF_MONTH)));
+    reply.append(" ");
+    reply.append(Integer.toString(calendar.get(Calendar.HOUR_OF_DAY)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.MINUTE)));
+    reply.append(":");
+    reply.append(Integer.toString(calendar.get(Calendar.SECOND)));
+
+    return reply.toString();
+  }
+  
+  public void addField(String field)
+  { this.values.add(new GS3SQLValue(field, null));
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnection.java	(revision 21112)
@@ -0,0 +1,245 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+
+public class GS3SQLConnection extends SQLConnection
+{
+
+    public GS3SQLConnection(java.sql.Connection connection, String database)
+    { 
+	super(connection, database);
+    }
+    
+
+  public void clearCollection(String collectionName)
+  {
+      dropDatabase(collectionName);
+  }
+    
+    public GS3SQLConnection cloneConnection() {
+	GS3SQLConnection conn = GS3SQLConnectionFactory.getGS3SQLConnection(this.database);
+	return conn;
+    }
+  /**
+   *  Initialise a collection for use.
+   *
+   *  @return <code>boolean</code> <code>true</code> if the collection successfully initialised.
+   */
+  public boolean initCollection(String collectionName)
+  {	
+    /*
+    // Check if the collection already exists
+    // if so, abort
+
+    // do a sample query to check if the collection exists
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    if (this.execute(select.toString())) {
+      return true;
+    }
+
+    // if there is an error, assume that the database doesn't exist...
+    // and reconnect to a "test" database for now...
+    this.connection = GS3SQLConnectionFactory.getConnection("test");
+    */
+
+      // create the new db
+      if (!createDatabase(collectionName)) {
+          System.err.println("Couldn't create database " + collectionName);
+	  return false;
+      }
+      // now connect to it
+      if (!connectToDatabase(collectionName)) {
+          System.err.println("couldn't connect to the database " + collectionName);
+	  return false;
+      }
+    
+      
+    try {
+      Statement statement = this.connection.createStatement();
+
+      // create build history table
+      GS3SQLCreateTable buildTable = new GS3SQLCreateTable("build");
+      buildTable.addAutoPrimaryKey("buildRef");
+      buildTable.addProperty("buildKey", 64);
+      buildTable.addProperty("buildValue");
+      statement.execute(buildTable.toString());
+
+      // create document table
+      GS3SQLCreateTable docTable = new GS3SQLCreateTable("document");
+      docTable.addProperty("DocID", 64);   // The document identifer, unique in collection
+      docTable.setPrimaryKey("DocID");     // ...which is also the primary key
+      docTable.addProperty("DocType", 64); // The document type - used to regenerate the
+                                           // correct Document object type.
+      docTable.addProperty("AccessionDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("IndexedDate", GS3SQLField.DATETIME_TYPE);
+      docTable.addProperty("ModifiedDate", GS3SQLField.DATETIME_TYPE);
+      statement.execute(docTable.toString());
+      
+      // create structure table
+      GS3SQLCreateTable structureMap = new GS3SQLCreateTable("structure");
+      structureMap.addAutoPrimaryKey("StructureRef");
+      structureMap.addProperty("DocID", 64);          // a 'foreign key' in effect - but not treated
+                                                      // as such for efficiency reasons
+      structureMap.addProperty("StructureID", 64);    // this identifier
+      structureMap.addProperty("StructureType", 64);  // the type of the structure
+      structureMap.addProperty("Label", 128);         // the label of the structure
+      statement.execute(structureMap.toString());
+    
+      // create section table
+      GS3SQLCreateTable sectionMap = new GS3SQLCreateTable("divisions");
+      sectionMap.addAutoPrimaryKey("DivisionRef");
+      sectionMap.addProperty("StructureRef", GS3SQLField.INTEGER_TYPE);
+      sectionMap.addProperty("DocID", 64);        // a 'foreign key' in effect - but not treated
+                                                  // as such for efficiency reasons
+      sectionMap.addProperty("ParentType", 64);   // the type of the parent - document or section
+      sectionMap.addProperty("ParentRef", 64);    // the parent sql reference identifier
+      sectionMap.addProperty("SectionID", 64);    // this identifier
+      sectionMap.addProperty("DivisionType", 64); // the type of the section
+      sectionMap.addProperty("LabelOrder", 64);   // the order of the section
+      sectionMap.addProperty("ShortLabel", 128);  // the short label of the section
+      sectionMap.addProperty("UserLabel", 255);   // the long label of the section
+      statement.execute(sectionMap.toString());
+
+      // a division->metadata reference mapping
+      GS3SQLCreateTable metaRefs = new GS3SQLCreateTable("divisionmetarefs");
+      metaRefs.addProperty("DocID", 64);
+      metaRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      metaRefs.addProperty("DivisionType", 16);   // whether a namespace or meta ref
+      metaRefs.addProperty("MetaID", 64); // the metadata reference itself, as a string
+      statement.execute(metaRefs.toString());
+
+      // a division->file reference mapping
+      GS3SQLCreateTable fileRefs = new GS3SQLCreateTable("divisionfilerefs");
+      fileRefs.addProperty("DocID", 64);
+      fileRefs.addProperty("DivisionRef", GS3SQLField.INTEGER_TYPE);
+      fileRefs.addProperty("DivisionType", 16); // whether section, group or file
+      fileRefs.addProperty("FileID", 32);
+      statement.execute(fileRefs.toString());
+      
+      //   create metadata table section table
+      GS3SQLCreateTable metadataSection = new GS3SQLCreateTable("metadata");
+      metadataSection.addAutoPrimaryKey("MetadataRef");
+      metadataSection.addProperty("DocID", 64);   // this table isn't actually used for much
+      metadataSection.addProperty("MetaID", 64);  // in and of itself...
+      metadataSection.addProperty("GroupID", 64); // the 'name' property
+      statement.execute(metadataSection.toString());
+      
+      //   create namespace table
+      GS3SQLCreateTable namespaces = new GS3SQLCreateTable("namespaces");
+      namespaces.addAutoPrimaryKey("NamespaceRef");
+      namespaces.addProperty("MetadataRef", GS3SQLField.INTEGER_TYPE);
+      namespaces.addProperty("DocID", 64);   // these three fields are the primary key
+      namespaces.addProperty("MetaID", 64);
+      namespaces.addProperty("NamespaceID", 64);
+      namespaces.addProperty("NamespaceType", 64);
+      namespaces.addProperty("FileType", 64);
+      namespaces.addProperty("FileLoc");
+      namespaces.addProperty("Creator", 128);
+      statement.execute(namespaces.toString());
+
+      // create metadata grouping table
+      GS3SQLCreateTable metagroups = new GS3SQLCreateTable("mdgroups");
+      metagroups.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE);
+      metagroups.addAutoPrimaryKey("MdGroupRef");
+      metagroups.addProperty("MdGroupID", 32);
+      statement.execute(metagroups.toString());
+
+      // create metadata qualifiers table
+      GS3SQLCreateTable metaqualifiers = new GS3SQLCreateTable("mdqualifiers");
+      metaqualifiers.addProperty("MdGroupRef", GS3SQLField.INTEGER_TYPE);
+      metaqualifiers.addProperty("QualifierName", 64);
+      metaqualifiers.addProperty("QualifierValue", 64);
+      statement.execute(metaqualifiers.toString());
+      
+      // create metadata values table
+      GS3SQLCreateTable metadata = new GS3SQLCreateTable("mdvalues");
+      metadata.addProperty("NamespaceRef", GS3SQLField.INTEGER_TYPE); // the parent namespace {one candidate primary key}
+      metadata.addProperty("GroupRef", GS3SQLField.INTEGER_TYPE);     // the parent group     {a sub-key or independent primary key}
+      metadata.addProperty("Label", 256);                             // the key of the metadata item
+      metadata.addProperty("Value");                                  // the value of the metadata item
+      statement.execute(metadata.toString());
+    
+      //   create file section table
+      GS3SQLCreateTable filesec = new GS3SQLCreateTable("filesection");
+      filesec.addAutoPrimaryKey("FileSectionRef");
+      filesec.addProperty("DocID", 64);
+      filesec.addProperty("FileSecID", 64);
+      statement.execute(filesec.toString());
+    
+      //   create file groups table
+      GS3SQLCreateTable filegroups = new GS3SQLCreateTable("filegroups");
+      // TODO: some unique identifier
+      filegroups.addAutoPrimaryKey("FileGroupRef");
+      filegroups.addProperty("DocID", 64);
+      filegroups.addProperty("FileGroupID", 64);
+      filegroups.addProperty("ParentRef", GS3SQLField.INTEGER_TYPE);
+      filegroups.addProperty("ParentType", 16);
+      statement.execute(filegroups.toString());
+    
+      //   create file table
+      GS3SQLCreateTable files = new GS3SQLCreateTable("files");
+      // TODO: the unique identifier from file groups table
+      files.addProperty("FileGroupRef", GS3SQLField.INTEGER_TYPE);
+      files.addProperty("FileLocType", 64);
+      files.addProperty("FileLocation");
+      files.addProperty("MIMEType", 64);
+      files.addProperty("FileID", 32);
+      statement.execute(files.toString());
+      
+      //   create admin table...etc.
+
+      // 
+      // END OF METS TABLES
+      //
+
+      //
+      // BEGINNING OF GSDL TABLES
+      //
+      GS3SQLCreateTable classifiers = new GS3SQLCreateTable("classifiers");
+      classifiers.addAutoPrimaryKey("ClassifyRef");
+      classifiers.addProperty("ClassifyID");
+      classifiers.addProperty("ParentID");
+      classifiers.addProperty("ClassifyOrder");
+      classifiers.addProperty("Name");
+      classifiers.addProperty("Description");
+      classifiers.addProperty("NumLeafDocs", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classifiers.toString());
+
+      GS3SQLCreateTable classDocs = new GS3SQLCreateTable("classdocuments");
+      classDocs.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classDocs.addProperty("DocID");
+      classDocs.addProperty("DocOrder", GS3SQLField.INTEGER_TYPE);
+      statement.execute(classDocs.toString());
+
+      GS3SQLCreateTable classData = new GS3SQLCreateTable("classdata");
+      classData.addProperty("ClassifyRef", GS3SQLField.INTEGER_TYPE);
+      classData.addProperty("Label");
+      classData.addProperty("Value");
+      statement.execute(classData.toString());
+
+      statement.close();
+      //
+      // END OF GSDL TABLES
+      //
+
+    }
+    catch (SQLException ex)
+    {
+      System.out.println(ex.toString());
+      return false;
+    }
+    return true;
+  }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLConnectionFactory.java	(revision 21112)
@@ -0,0 +1,76 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.greenstone.gsdl3.util.GlobalProperties;
+
+public class GS3SQLConnectionFactory
+{
+
+    public static Connection getConnection(String database)
+    {
+	try {	    
+	    String server = GlobalProperties.getProperty("mysql.server", "localhost");
+	    String port = GlobalProperties.getProperty("mysql.tcp.port", "3306");
+	    String password = GlobalProperties.getProperty("mysql.gsdl3admin.password");
+	    
+	    Class.forName("com.mysql.jdbc.Driver").newInstance();
+	    String connection_address = "jdbc:mysql://"+server+":"+port+"/"+database+"?user=gsdl3admin";
+	    if (password != null && !password.equals("")) {
+		connection_address += "&password="+password;
+	    }
+	    Connection connection = DriverManager.getConnection(connection_address);
+	    return connection;
+	}
+	catch (Exception ex) {
+	    System.out.println("GS3SQLConnectionFactory.getConnection() Error:");
+	    System.out.println(ex.getMessage());
+	}
+	return null;
+    }
+    
+    public static SQLConnection getSQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new SQLConnection(c, database);
+    }
+
+    
+    public static GS3SQLConnection getGS3SQLConnection(String database)
+    {
+	Connection c = getConnection(database);
+	if (c== null) {
+	    return null;
+	}
+	return new GS3SQLConnection(c, database);
+    }
+
+    /* returns a Connection to the server, */
+    public static Connection getConnection() 
+    {
+	return getConnection("test");
+    }
+
+    /* returns a Connection to the server, */
+    public static SQLConnection getSQLConnection() 
+    {
+	return getSQLConnection("test");
+    }
+
+    public static GS3SQLConnection getGS3SQLConnection()
+    { 
+	return getGS3SQLConnection("test");
+    }
+
+    public static void main(String args[])
+    { 
+	GS3SQLConnection connection = getGS3SQLConnection();
+	connection.initCollection("maya");
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLCreateTable.java	(revision 21112)
@@ -0,0 +1,96 @@
+package org.greenstone.gsdl3.gs3build.database;
+  
+import java.util.List;
+import java.util.ArrayList;
+
+
+public class GS3SQLCreateTable
+{ 
+    String tableName;
+    List   properties;
+    List   primaryKey;
+    List   indexed;
+
+    public GS3SQLCreateTable(String tableName)
+    { 
+	this.tableName = tableName;
+	this.properties = new ArrayList();
+	this.primaryKey = new ArrayList();
+	this.indexed    = new ArrayList();
+    }
+    
+    public void addAutoPrimaryKey(String fieldName)
+    {
+	GS3SQLField field = new GS3SQLField(fieldName, GS3SQLField.AUTOINTEGER_TYPE);
+	this.properties.add(field);
+	this.setPrimaryKey(fieldName);
+    }
+
+    public void addProperty(String fieldname)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname);
+	this.properties.add(field);
+    }
+    
+    public void addProperty(String fieldname, String type)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, type);
+	this.properties.add(field);
+    }
+
+    public void addProperty(String fieldname, int length)
+    { 
+	GS3SQLField field = new GS3SQLField(fieldname, length);
+	this.properties.add(field);
+    }
+
+    public void setPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.clear();
+	this.primaryKey.add(fieldname);
+    }
+
+    public void extendPrimaryKey(String fieldname)
+    { 
+	this.primaryKey.add(fieldname);
+    }
+    
+    public void addIndex(String fieldname)
+    { 
+	this.indexed.add(fieldname);
+    }
+
+    public String toString()
+    {	
+	StringBuffer reply = new StringBuffer();
+	
+	reply.append("CREATE TABLE ");
+	reply.append(this.tableName);
+	reply.append(" (");
+	for (int f = 0; f < this.properties.size(); f ++) {
+	    GS3SQLField field = (GS3SQLField) this.properties.get(f);
+	    if (f != 0) {
+		reply.append(",");
+	    }
+	    reply.append(field.toString());
+	}
+
+	for (int p = 0; p < this.primaryKey.size(); p ++) {
+	    if (p == 0){
+		reply.append(", PRIMARY KEY(");
+	    }
+	    else {
+		reply.append(",");
+	    }
+	    reply.append(this.primaryKey.get(p).toString());
+	    
+	    if (p == this.primaryKey.size() - 1) {
+		reply.append(")");
+	    }
+	}
+	// TODO: add primary key, index etc. notations
+	reply.append(");");
+	
+	return reply.toString();
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLDelete.java	(revision 21112)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLDelete
+{ 
+  GS3SQLWhere where;
+  String      table;
+
+  public GS3SQLDelete(String table)
+  { this.table = table;
+    this.where = null;
+  }
+
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    reply.append("DELETE FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+
+    return reply.toString();
+  }
+}
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLField.java	(revision 21112)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLField
+{ String name;
+  int    length;
+  String type;
+
+    public final static String VARCHAR_TYPE = "VARCHAR";
+    public final static String INTEGER_TYPE = "INTEGER";
+    public final static String AUTOINTEGER_TYPE = "AUTOINTEGER";
+    public final static String DATE_TYPE = "DATE";
+    public final static String DATETIME_TYPE = "DATETIME";
+
+    public GS3SQLField(String name)
+    { this.name = name;
+      this.length = 1024;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = VARCHAR_TYPE;
+    }
+
+    public GS3SQLField(String name, String type)
+    { this.name = name;
+      this.type   = type;
+      this.length = 1024;
+    }
+
+    public GS3SQLField(String name, String type, int length)
+    { this.name = name;
+      this.length = length;
+      this.type   = type;
+    }
+
+    public String getName()
+    { return this.name;
+    }
+
+    public int getLength()
+    { return this.length;
+    }
+
+    public String toString()
+    { StringBuffer reply = new StringBuffer();
+
+      reply.append(this.name);
+      reply.append(" ");
+      if (this.type.equals(VARCHAR_TYPE))
+      {
+	if (length > 255) {
+	  reply.append("TINYTEXT");
+	}
+	else {
+	  reply.append("VARCHAR(");
+	  reply.append(Integer.toString(this.length));
+	  reply.append(")");
+	}
+      }
+      else if (this.type.equals(AUTOINTEGER_TYPE))
+      { reply.append("INTEGER AUTO_INCREMENT");
+      }
+      else
+      {
+	reply.append(this.type);
+      }
+      return reply.toString();
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLInsert.java	(revision 21112)
@@ -0,0 +1,54 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLInsert extends GS3SQLAction
+  { public GS3SQLInsert(String table)
+    { super(table);
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("INSERT INTO ");
+      reply.append(this.table);
+      reply.append(" ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(" VALUES ");
+      if (this.values.size() >= 1)
+	reply.append("(");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+        reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.DATETIME_TYPE) ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.values.size() >= 1)
+	reply.append(")");
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLSelect.java	(revision 21112)
@@ -0,0 +1,48 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLSelect extends GS3SQLAction
+{ GS3SQLWhere where;
+  boolean     distinct;
+
+  public GS3SQLSelect(String table)
+  { super(table);
+    this.where  = null;
+    this.distinct = false;
+  }
+    
+  public void setWhere(GS3SQLWhere where)
+  { this.where = where;
+  }
+
+  public void setDistinct(boolean distinct)
+  { this.distinct = distinct;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+  
+    reply.append("SELECT ");
+
+    if (this.distinct)
+    { reply.append("DISTINCT ");
+    }
+ 
+    for (int i = 0; i < this.values.size(); i ++)
+    { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+      if (i > 0)
+	reply.append(", ");
+      reply.append(value.field);
+    }
+
+    reply.append(" FROM ");
+    reply.append(this.table);
+
+    if (this.where != null)
+    { reply.append(" WHERE ");
+      reply.append(where.toString());
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLUpdate.java	(revision 21112)
@@ -0,0 +1,50 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public   class GS3SQLUpdate extends GS3SQLAction
+  { GS3SQLWhere where;
+
+    public GS3SQLUpdate(String table)
+    { super(table);
+      this.where  = null;
+    }
+
+    public void setWhere(GS3SQLWhere where)
+    { this.where = where;
+    }
+
+    public String toString()
+    {
+      StringBuffer reply = new StringBuffer();
+      reply.append("UPDATE ");
+      reply.append(this.table);
+      reply.append(" SET ");
+      for (int i = 0; i < this.values.size(); i ++)
+      { GS3SQLValue value = (GS3SQLValue) this.values.get(i);
+        if (i > 0)
+	  reply.append(", ");
+        reply.append(value.field);
+	reply.append("=");
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+	reply.append(value.value);
+	if (value.getType() == null ||
+	    value.getType().equals(GS3SQLField.VARCHAR_TYPE))
+	{
+	    reply.append("\"");
+	}
+      }
+      if (this.where != null)
+      { reply.append(" WHERE ");
+	reply.append(where.toString());
+      }
+      reply.append(";");
+      return reply.toString();
+    }
+  }
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLValue.java	(revision 21112)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+public class GS3SQLValue
+{ String field;
+  String value;
+  String type;
+
+  public GS3SQLValue(String field, String value)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = null;
+  }
+
+  public GS3SQLValue(String field, String value, String type)
+  {
+    this.field = field;
+    this.value = value;
+    this.type  = type;
+  }
+
+  public String getType()
+  {
+    return this.type;
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhere.java	(revision 21112)
@@ -0,0 +1,55 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhere implements GS3SQLWhereElement
+{ String condition;
+  List   elements;
+
+  public static final String AND_CONDITION = "AND";
+  public static final String OR_CONDITION = "OR";
+
+  public GS3SQLWhere(GS3SQLWhereElement element)
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();
+    this.elements.add(element);
+  }
+
+  public GS3SQLWhere()
+  { this.condition = AND_CONDITION;
+    this.elements  = new ArrayList();   
+  }
+
+  public GS3SQLWhere(GS3SQLWhereElement element, String condition, GS3SQLWhereElement other)
+  { this.elements = new ArrayList();
+    this.elements.add(element);
+    this.elements.add(other);
+    this.condition = condition;
+  }
+
+  public void add(GS3SQLWhereElement element)
+  { this.elements.add(element);
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    for (int i = 0; i < this.elements.size(); i ++)
+    {
+      if (i != 0) {
+	reply.append(" ");
+	reply.append(this.condition);
+	reply.append(" ");
+      }
+      reply.append(this.elements.get(i).toString());
+    }
+    return reply.toString();
+  }
+
+  public void setCondition(String condition)
+  { this.condition = condition;
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereElement.java	(revision 21112)
@@ -0,0 +1,10 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public interface GS3SQLWhereElement
+{ // empty interface for type safety
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/GS3SQLWhereItem.java	(revision 21112)
@@ -0,0 +1,47 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class GS3SQLWhereItem implements GS3SQLWhereElement
+{ String field;
+  String comparison;
+  String value;
+  String type;
+
+  public GS3SQLWhereItem(String field, String comparison, String value, String type)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = type;
+  }
+
+  public GS3SQLWhereItem(String field, String comparison, String value)
+  { this.field      = field;
+    this.comparison = comparison;
+    this.value      = value;
+    this.type       = GS3SQLField.VARCHAR_TYPE;
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    reply.append(this.field);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    reply.append(this.comparison);
+    if (!this.comparison.equals("=")) {
+      reply.append(" ");
+    }
+    if (type.equals(GS3SQLField.VARCHAR_TYPE)) {
+	reply.append("\"" + this.value + "\"");
+    }
+    else {
+	reply.append(this.value);
+    }
+    return reply.toString();
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/MySQLAdaptor.java	(revision 21112)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.*;
+
+public class MySQLAdaptor implements DatabaseAdaptorInterface
+{
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/SQLConnection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/database/SQLConnection.java	(revision 21112)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+public class SQLConnection
+{
+    protected Connection connection;
+    protected String database;
+    
+    public SQLConnection(java.sql.Connection connection, String database)
+    { 
+	this.connection = connection;
+	this.database = database;
+    }
+    
+    protected void finalize() {
+	if (this.connection!=null) {
+	    try {
+		this.connection.close();		
+	    } catch (Exception e) {}
+	}	
+	this.connection = null;   
+    }
+    
+    public void close() {
+	finalize();
+    }
+    
+    public Statement createStatement() throws SQLException
+    { 
+	return this.connection.createStatement();
+    }
+    
+    public boolean connectToDatabase(String database) {
+	if (this.connection != null) {
+	    try {
+		this.connection.close();
+	    } catch (Exception e) {}
+	}
+	this.connection = GS3SQLConnectionFactory.getConnection(database);
+	if (this.connection == null) {
+	    return false;
+	}
+	this.database = database;
+	return true;
+    }
+    
+    public boolean dropDatabase(String database) {
+	try {
+	    Statement statement = this.connection.createStatement();
+	    statement.execute("DROP DATABASE "+database+";");
+	    statement.close();
+	}
+	catch (SQLException ex){
+	    System.err.println(ex);
+	    return false;
+	}
+	return true;
+	
+    }
+
+    public boolean createDatabase(String database) {
+	try {
+	    String command = "CREATE DATABASE " + database;
+	    Statement statement = this.connection.createStatement();
+	    statement.execute(command);
+	    statement.close();
+	} catch (Exception e) {
+	    System.err.println(e);
+	    return false;
+	}
+	 
+	return true;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractDocument.java	(revision 21112)
@@ -0,0 +1,488 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Date;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.sql.Timestamp;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.NamespaceFactory;
+import org.greenstone.gsdl3.gs3build.metadata.StructureIdentifierFactory;
+import org.greenstone.gsdl3.gs3build.metadata.GSDL3Namespace;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSHeader;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSNamespace;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+/**
+ *  Provide a base-line functionality for the <code>DocumentInterface</code>
+ *  class.
+ */
+
+public abstract class AbstractDocument implements DocumentInterface
+{
+  METSFileSet        fileSet;
+  METSDescriptiveSet metadata;
+  METSStructureSet   structureSet;
+  METSHeader         header;
+  DocumentID         id;
+  boolean            isModified;
+  StructureIdentifierFactory structureIdFactory;
+  java.sql.Timestamp firstDate;
+  java.sql.Timestamp indexDate;
+  java.sql.Timestamp modifiedDate;
+
+  /**
+   *  <p>Create a very vanilla document with a given document identifier.</p>
+   *  <p>Most commonly used in dealing with loading files using DocumentFactory
+   *  or similar.</p>
+   *
+   *  @param <code>DocumentID</code> the document identifier
+   */
+  public AbstractDocument(DocumentID id)
+  { this.fileSet      = new METSFileSet();
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = id;
+    this.structureIdFactory = new StructureIdentifierFactory();
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+    this.modifiedDate = null; // as a signature that the modified date needs finding...
+  }
+  
+  /**
+   *  Create a basic document from a given <code>URL</code.  This is usually the form
+   *  called through the recognisers.
+   *
+   *  @param <code>URL</code> the URL of the first file in the document package
+   */
+  public AbstractDocument(URL url)
+  { this.fileSet      = new METSFileSet();
+    METSFile metsFile = this.fileSet.addFile(url);
+    this.metadata     = new METSDescriptiveSet();
+    this.header       = new METSHeader();
+    this.structureSet = new METSStructureSet();
+    this.id           = null;
+
+    java.util.Date thisDate = new java.util.Date();
+    this.firstDate = new java.sql.Timestamp(thisDate.getTime());
+    this.indexDate = new java.sql.Timestamp(thisDate.getTime());
+
+    METSStructure structure = new METSStructure("All", "All", "Whole Document");
+    METSDivision documentBody = new METSDivision("All", "All", "All", "Whole Document", "Document");
+    structure.addDivision(documentBody);
+    this.structureSet.addStructure(structure);
+    documentBody.addFileReference(metsFile.getID());
+    documentBody.addMetadataReference("DM1");
+  }
+
+  /**
+   *  Set the identified for the document.  Every document should have
+   *  a document number set on its accession, either through metadata
+   *  placed upon it internally or externally, or by assignment through
+   *  a <code>DocumentIDFactory</code>.  Each identifier should be
+   *  unique.
+   *
+   *  @param <code>DocumentID</code> the document identifier - in XML
+   *         terms, the gsdl3:id element.
+   */
+  public void setID(DocumentID id)
+  { this.id = id;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the document identifier - this should be unique to the document,
+   *  but care must be taken in the configuration of the collection to
+   *  ensure that this is the case.
+   *
+   *  @return <code>DocumentID</code> the identifer
+   */
+  public DocumentID getID()
+  { return this.id;
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  { return true;
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getFilesDatestamp()
+  { return this.fileSet.getModifiedDatestamp(); 
+  }
+
+  /**
+   *  Get the date that this file was modified
+   */
+  public long getModifiedDatestamp()
+  { if (this.modifiedDate == null) {
+      this.setModifiedDatestamp();
+    }
+    return this.modifiedDate.getTime();
+  }
+
+  /**
+   *  Update/set the date of the most recent file modification
+   */
+  public void setModifiedDatestamp()
+  { this.modifiedDate = new java.sql.Timestamp(this.fileSet.getModifiedDatestamp()); 
+  }
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate()
+  { return this.firstDate.getTime();
+  }
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate()
+  { return this.indexDate.getTime();
+  }
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long timestamp)
+  { this.indexDate = new java.sql.Timestamp(timestamp);
+  }
+
+  /**
+   *  Check if this document is in the database already.
+   *
+   *  In this simple implementation, the first file in the document's "default"
+   *  filegroup is taken to be the canonical file for this document - any document
+   *  of the same type with the same canonical file is taken to be a match.
+   *
+   *  @return <code>boolean</code> - if a matching document is found in the
+   *                                 database.
+   */
+  public String getDuplicateID(GS3SQLConnection connection)
+  { //String query = "SELECT * FROM document INNER JOIN filegroups ON document.docId=filegroups.docId WHERE DocType=\"" + HTML_DOCUMENT_TYPE + "\"";
+
+    // Query for documents using the same file...
+    String query = "SELECT DocID FROM files INNER JOIN filegroups ON files.FileGroupRef=filegroups.FileGroupRef WHERE (filegroups.FileGroupId=\"default\" AND files.FileLocation=\"" + this.fileSet.getFile(0).getLocation().toString() + "\")";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	List docs = new ArrayList();
+
+	if (results.first()) {
+	    do {
+		String value = results.getString("DocID");		
+		docs.add(value);
+	    } while (results.next());
+	    
+	    Iterator docIterator = docs.iterator();
+	    while (docIterator.hasNext()) {
+		String docId = docIterator.next().toString();
+		String innerQuery = "SELECT * FROM document WHERE DocID=\"" + docId + "\"";
+		results = statement.executeQuery(innerQuery);
+		if (results.first()) {
+		    String docType = results.getString("DocType");
+		    if (docType.equals(this.getDocumentType())) {
+			return docId;
+		    }
+		}
+	    }
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException sqlEx) {
+      System.err.println("AbstractDocument.getDuplicateID(): "+sqlEx);
+    }
+
+    return "";
+  }
+
+  /**
+   *  Obtain the METS header of this document
+   *
+   *  @return <code>METSHeader</code> the header
+   */
+  public METSHeader getHeader()
+  { return this.header;
+  }	
+
+  /**
+   *  Set the METS header for this document.
+   *
+   *  @param <code>METSHeader</code> the header
+   */
+  public void setHeader(METSHeader header)
+  { this.header = header;
+  }
+
+  /**
+   *  A simple implementation of the isDocumentType function that does <b>not</b> consider
+   *  inheritance - it <code>must</code> be extended as required.
+   */
+  public boolean isDocumentType(String type)
+  { return type.equals(this.getDocumentType());
+  }
+
+  public abstract String getDocumentType();
+  
+  public abstract String getDocumentText();
+
+  public abstract String getSectionText(String sectionId);
+
+  public String getMETSType()
+  { return "document";
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.addDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value)
+  { // no need to set isModified, as the following call will do it anyway!
+    this.setDocumentMetadata(label.getNamespace(), label.getLabel(), value);
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.addMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *   @see DocumentInterface:addDocumentMetadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value)
+  { this.metadata.setMetadata("default", namespace, label, value);
+    this.isModified = true;
+  }
+
+  /**
+   *  @see DocumentInterface:removeDocumentMetadata
+   */
+  public void removeDocumentMetadata(String namespace, String label)
+  { this.metadata.removeMetadata("default", namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *
+   */
+  public void removeAllMetadata(String namespace, String label)
+  { this.metadata.removeAllMetadata(namespace, label);
+    this.isModified = true;
+  }
+
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value)
+  {
+    // First get the list of file groups, etc. that this file is associated with...
+    List fileGroups = this.fileSet.findGroups(fileLocation);
+
+    // Next, get the METS divisions associated with each file group...
+    List divisions = this.structureSet.findDivisionsForFiles(fileGroups);
+    
+    // Finally, post the metadata to the metadata group associated with each structure
+    Iterator divisionIter = divisions.iterator();
+    while (divisionIter.hasNext())
+    { METSDivision division = (METSDivision) divisionIter.next();
+
+      // get the open namespace for this division
+      METSNamespace namespaceMetadata = division.findNamespace(namespace, true, this.metadata);
+
+      // then post the metadata to it...
+      namespaceMetadata.addMetadata(label, value);
+    }
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSDescriptive</code> the metadata holder for the document.
+   */
+  public METSDescriptiveSet getDocumentMetadata()
+  { return this.metadata;
+  }
+
+  /**
+   *  Set the metadata structure for this document
+   *
+   *  @param <code>METSDescriptive</code> the new metadata holder for the document.
+   */
+  public void setDocumentMetadata(METSDescriptiveSet metadata)
+  { this.metadata = metadata;
+    this.isModified = true;
+  }
+
+  /**
+   *  Get the metadata structure of the document
+   *
+   *  @return <code>METSStructureSet</code> the metadata holder for the document.
+   */
+  public METSStructureSet getDocumentStructure()
+  { return this.structureSet;
+  }
+
+  public void setDocumentStructure(METSStructureSet structureSet)
+  { this.structureSet = structureSet;
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace to find the values in.
+   *  @param <code>String</code> the label to match to find the values.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespace, String label)
+  { return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  Get the values associated with a particular metadata value.
+   *
+   *  @param <code>String</code> the namespace and label separated by a
+   *         colon.
+   *
+   *  @return <code>List</code> the values.
+   */
+  public List getDocumentMetadataItem(String namespaceLabel)
+  { String namespace, label;
+  
+    int colonAt = namespaceLabel.indexOf(':');
+    if (colonAt < 0)
+    { namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      label     = namespaceLabel;
+    }
+    else
+    { namespace = namespaceLabel.substring(0, colonAt);
+      label     = namespaceLabel.substring(colonAt+1);
+    }
+    return this.metadata.getMetadata("default", namespace, label);
+  }
+
+  /**
+   *  @see DocumentInterface:getDocumentFiles
+   */
+  public METSFileSet getDocumentFiles()
+  { return this.fileSet;
+  }
+
+  public void setDocumentFiles(METSFileSet fileSet)
+  { this.fileSet = fileSet;
+  }
+
+  /**
+   *  This is just a dummy function that does nothing at this level...
+   */
+  public org.w3c.dom.Document getDOMDocument()
+  { return null;
+  }
+  
+  /**
+   *  @see DocumentInterface:isMETSCompatible
+   */
+  public boolean isMETSCompatible()
+  { return true;
+  }
+
+  /**
+   *  Use a default document writer - this may be overridden for subclasses...
+   *
+   *  @see DocumentInterface:writeMETSObject
+   */
+  public DocumentWriter getMETSWriter()
+  { return new DocumentWriter();
+  }
+
+  /**
+   *  Use a default SQL document writer - this may be overridden for subclasses...
+   *
+   */
+  public DocumentSQLWriter getSQLWriter()
+  { return new DocumentSQLWriter();
+  }
+
+  /**
+   *  Obtain a document from the SQL database
+   */
+    public static AbstractDocument readSQL(GS3SQLConnection connection, ResultSet sqlResult)
+  { try {
+      DocumentID id = new DocumentID(sqlResult.getString("DocID"));
+      String     type = sqlResult.getString("docType");
+
+      // Use a factory method to create the correct subtype...
+      AbstractDocument document = DocumentFactory.createDocument(type, id);
+      // Append the document date information
+      document.indexDate    = sqlResult.getTimestamp("IndexedDate");
+      document.firstDate    = sqlResult.getTimestamp("AccessionDate");
+      document.modifiedDate = sqlResult.getTimestamp("ModifiedDate");
+
+      // Get the individual components of the document
+      METSFileSet fileSet = METSFileSet.readSQL(document, connection);
+      document.setDocumentFiles(fileSet);
+      METSDescriptiveSet descriptiveSet = METSDescriptiveSet.readSQL(document, connection);
+      document.setDocumentMetadata(descriptiveSet);
+      METSStructureSet structureSet = METSStructureSet.readSQL(document, connection);
+      document.setDocumentStructure(structureSet);
+
+      // indicate that the document is not currently modified
+      document.setChanged(false);
+      return document;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQL(): Failure to load document: " + sqlEx);
+    }
+    return null;
+  }
+
+  /**
+   *  
+   */
+  public boolean isChanged()
+  { return this.isModified;
+  }
+
+  public void setChanged(boolean isModified)
+  { this.isModified = isModified;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/AbstractRecogniser.java	(revision 21112)
@@ -0,0 +1,95 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+/**
+   A simple base class for Recognisers
+   A new recogniser should set preferred_mime_type, filename_extensions, and document_type in the constructor. 
+   The DocumentFactory class needs to be modified to map the documentType to 
+   the corresponding document
+   And thats all for simple recognisers.
+   More complicated recognisers can just implement from scratch RecogniserInterface
+*/
+abstract public class AbstractRecogniser implements RecogniserInterface
+{
+
+    DocumentList list_repository = null;
+    String preferred_mime_type = "SET THIS IN THE CONCRETE CLASS"; 
+    ArrayList filename_extensions = null;
+    String document_type = "SET THIS IN THE CONCRETE CLASS";
+    
+    /** The constructor should set the variables 
+     * preferredMimeType, filename_extensions and documentType
+     */
+    public AbstractRecogniser() {
+
+    }
+    
+    /** configure by default does nothing */
+    public boolean configure(Element config_elem){
+	return true;
+    }
+    public void setListRepository(DocumentList docList) {
+	this.list_repository = docList;
+    }
+
+    protected boolean isAcceptedFilename(String filename) {
+	if (filename_extensions == null) {
+	    return false; // or should this be true??
+	}
+	for (int i=0; i<filename_extensions.size(); i++) {
+	    if (filename.endsWith((String)filename_extensions.get(i))) {
+		return true;
+	    }
+	}
+	return false;
+    }
+    
+    public boolean parseDocument(METSFile file)
+    {
+	String MIMEType = file.getMIMEType();
+	if (MIMEType == null || 
+	    MIMEType.equals(this.preferred_mime_type)) {
+	    URL location = file.getLocation();
+	    return this.parseDocument(location);
+	}
+	return false;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if ( isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting "+document_type+" Document " + filename);
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	} else {
+	    // Get Mime type remotely, and then proceed if required
+	    String mimeType = HTTPTools.getMIMEType(url);
+	    
+	    if (mimeType == this.preferred_mime_type) {
+		System.out.println("Posting "+document_type+" Document " + url.toString());
+		
+		AbstractDocument doc = DocumentFactory.createDocument(document_type, url);
+		this.list_repository.addDocument(doc);
+		return true;
+	    }
+	}
+	return false;
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentFactory.java	(revision 21112)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class DocumentFactory {
+    
+    public static AbstractDocument createDocument(String type, DocumentID id) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(id);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(id);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(id);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(id);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(id);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(id);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(id);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    public static AbstractDocument createDocument(String type, URL url) {
+	
+	if (type.equals(HTMLDocument.HTML_DOCUMENT_TYPE)) {
+	    return new HTMLDocument(url);
+	} else if (type.equals(TextDocument.TEXT_DOCUMENT_TYPE)) {
+	    return new TextDocument(url);
+	} else if (type.equals(JPEGDocument.JPEG_DOCUMENT_TYPE)) {
+	    return new JPEGDocument(url);
+	} else if (type.equals(IndexDocument.INDEX_DOCUMENT_TYPE)) {
+	    return new IndexDocument(url);
+	} else if (type.equals(MetadataDocument.METADATA_DOCUMENT_TYPE)) {
+	    return new MetadataDocument(url);
+	} else if (type.equals(METSDocument.METS_DOCUMENT_TYPE)) {
+	    return new METSDocument(url);
+	} else if (type.equals(GMLDocument.GML_DOCUMENT_TYPE)) {
+	    return new GMLDocument(url);
+	}
+	
+	/*
+	  else if (type.equals(ExtXMLDocument.EXTXML_DOCUMENT_TYPE))
+	  { return new ExtXMLDocument(id);
+	  }*/
+	return null;
+    }
+    
+  public static DocumentInterface readSQLDocument(GS3SQLConnection connection, DocumentID id)
+  {
+    String query = "SELECT * FROM document WHERE DocID=\""+id.toString()+"\";";
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+	
+	DocumentInterface di = null;
+	if (results.first()) {
+	    di = AbstractDocument.readSQL(connection, results);
+	}
+	statement.close();
+	return di;
+    }
+    catch (SQLException sqlEx) {
+      System.err.println("AbstractDocument.readSQLDocument():"+sqlEx);
+    }
+    return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentID.java	(revision 21112)
@@ -0,0 +1,14 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentID
+{
+	String id;
+
+	public DocumentID(String id)
+	{	this.id = id;
+	}
+
+	public String toString()
+	{	return this.id;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactory.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+
+public class DocumentIDFactory implements DocumentIDFactoryInterface
+{ private CollectionManager collection;
+
+  public DocumentIDFactory(CollectionManager collection)
+  { this.collection = collection;
+  }
+
+  public DocumentID getNewDocumentID(DocumentInterface document)
+  { DocumentID id;
+
+    String idString = this.collection.getNextDocumentID();
+    id = new DocumentID(idString);
+    
+    return id;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentIDFactoryInterface.java	(revision 21112)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public interface DocumentIDFactoryInterface
+{
+	public DocumentID getNewDocumentID(DocumentInterface document);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentInterface.java	(revision 21112)
@@ -0,0 +1,282 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.List;
+import java.util.Map;
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public interface DocumentInterface
+{
+  /**
+   *  Get the immediate document type of the document.  The type may be derived/inherited
+   *  from other DocumentInterface types, but that cannot be checked here.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getDocumentType();
+  
+  /**
+   *  Check if this document is of a particular type, or is derived from a particular
+   *  type - i.e. inheritance is considered as well as the immediate type.
+   *
+   *  @param  <code>String</code> the type to check against.
+   *  @return <code>boolean</code> if the document matches the given type.
+   */
+  public boolean isDocumentType(String documentType);
+  
+  public void setID(DocumentID id);
+  public DocumentID getID();
+
+  /**
+   *  Get the METS type of the document.
+   *
+   *  @return <code>String</code> the document type as a string
+   */
+  public String getMETSType();
+  
+  public void setHeader(METSHeader header);
+  public METSHeader getHeader();
+
+  /**
+   *  Whether the document is indexed or not.
+   *
+   *  @return <code>boolean</code> <code>true</code> by default.
+   */
+  public boolean isIndexed();
+
+  /**
+   *  Get the latest date that a component file was modified...
+   */
+  public long getFilesDatestamp();
+
+  /**
+   *  Get the stored modified date for the files datestamp...
+   */
+  public long getModifiedDatestamp();
+
+  /**
+   *
+   */
+  public void setModifiedDatestamp();
+
+  /**
+   *  Get the date that this document was first indexed
+   */
+  public long getAccessionDate();
+
+  /**
+   *  Get the date that this document was last indexed
+   */
+  public long getLastIndexedDate();
+
+  /**
+   *  Set the last indexed date for this document;
+   */
+  public void setLastIndexedDate(long date);
+
+  /**
+   *  Check if the document matches another in the database
+   */
+  public String getDuplicateID(GS3SQLConnection connection);
+    
+  /**
+   *  The plain text of the document.
+   *
+   *  @return <code>String</code> This value may be <code>null</code>
+   *          for documents which have no textual component - e.g.
+   *          an image file
+   */
+  public String getDocumentText();
+
+  /**
+   *  The document as a dom object
+   */
+  public org.w3c.dom.Document getDOMDocument();
+
+  /**
+   *  Get the text of a section of this document
+   *
+   *  @return <code>String</code> the text as a string - for sections
+   *          that have no textual components, this value may be
+   *          <code>null</code>
+   */
+  public String getSectionText(String sectionId);
+	
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  The returned Map may have the following properties:
+   *  1) It may be <code>null</code> - e.g. for plain text documents
+   *  2) Any value in the map may be a <code>List</code> object containing
+   *     more than one possible value - e.g. they key 'Author' may associated
+   *     with a List of several people.
+   *  3) Any value in the map may itself be a <code>Map</code> where the
+   *     encoding scheme permits groupings of hierarchical metadata items.
+   *  4) Where the namespace for a metadata item is known, its key will
+   *     include the namespace.  Hence Dublin Core 'Author' would be encoded
+   *     as "dc.Author" as the key.
+   *
+   *  @return <code>METSDescriptive</code> the metadata of the document
+   */
+  public METSDescriptiveSet getDocumentMetadata();
+  
+  /**
+   *  The metadata for a particular given upon the document, encoded as a <code>List</code>
+   *
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> the label of the values to obtain
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespace, String label);
+  
+  /**
+   *  The metadata for the document, encoded as a map.
+   *
+   *  @param <code>String</code> the namespace and label of the metadata, separated
+   *                             by a colon.  If no namespace is given, it is 
+   *                             defaulted
+   *
+   *  @return <code>List</code> the metadata values
+   */
+  public List getDocumentMetadataItem(String namespaceLabel);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.  This is a "cheap" form which doesn't have a separate
+   *  namespace element.  Either the data is to be stored in the "open"
+   *  Greenstone metadata namespace, or the namespace is encoded within
+   *  the label.
+   *
+   *  @param <code>MetadataLabel</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures.  The label may commence
+   *                             with a namespace followed by a colon. 
+   *                             <p>e.g. "dc:title" for Dublin Core Title.</p>
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(MetadataLabel label, String value);
+  
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void addDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the decoration of a document with external or extracted
+   *  metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void setDocumentMetadata(String namespace, String label, String value);
+
+  /**
+   *  Facilitate the removal of document metadata.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeDocumentMetadata(String namespace, String label);
+  
+  /**
+   *  Facilitate the removal of all metadata of a given label from the document.
+   *
+   *  @param <code>String</code> namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   */
+  public void removeAllMetadata(String namespace, String label);
+  
+  /**
+   *  Post metadata to a file in this document - the appropriate changes
+   *  should be made...
+   *
+   *  @param <code>URL</code> the location of the file...
+   *  @param <code>String</code> the namespace of the metadata
+   *  @param <code>String</code> label of the metadata, with a '.' to deliminate
+   *                             sub-component structures
+   *  @param <code>String</code> value of the metadata
+   */
+  public void postFileMetadata(URL fileLocation, String namespace, String label, String value);
+
+  /**
+   *  The constituent files for the document: each should be wrapped up
+   *  as a URL
+   *
+   *  @return <code>METSFileSet</code> the files which constitute the document
+   */
+  public METSFileSet getDocumentFiles();
+
+  /**
+   *  Set the constituent files for the document.
+   */
+  public void setDocumentFiles(METSFileSet fileSet);
+
+  /**
+   *  Obtain the structural information on the document
+   *
+   *  @return <code>METSStructureSet</code> the structural information on the
+   *          document.
+   */
+  public METSStructureSet getDocumentStructure();
+  
+  /**
+   *  Indicate whether the document can be stored in a native form in
+   *  a METS wrapper
+   *
+   *  @return <code>boolean</code> <code>true</code> if the document
+   *          is just to be wrapped in a METS shell.
+   */
+  public boolean isMETSCompatible();
+  
+  /**
+   *  Write the document into a METS wrapper - for many document types,
+   *  this will not actually be done by the document itself, but rather
+   *  by the default writer
+   */
+  public DocumentWriter getMETSWriter();
+  
+  /**
+   *  Get the writer to send the document to an SQL database
+   */
+  public DocumentSQLWriter getSQLWriter();
+
+  /**
+   *  Check if the document is changed or not
+   */
+  public boolean isChanged();
+
+  /**
+   *  Set the document modified state
+   */
+  public void setChanged(boolean isModified);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentList.java	(revision 21112)
@@ -0,0 +1,535 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+
+import java.io.PrintWriter;
+import java.io.FileWriter;
+import java.io.File;
+import java.io.IOException;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLSelect;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhereItem;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLWhere;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLField;
+
+public class DocumentList
+{
+  DocumentInterface []       list;       // what is currently cached
+  int                        size;       // the maximum number in the cache
+  int                        used;       // the actual number in the cache
+  int                        count;      // the total number of known documents
+  DocumentIDFactoryInterface idFactory;  // A manufacturer of novel document IDs
+  GS3SQLConnection           connection; // used to query the SQL database
+
+  private static final int maxSize = 3;
+
+  public DocumentList(GS3SQLConnection connection)
+  { this.idFactory = null;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  public DocumentList(DocumentIDFactoryInterface idFactory, GS3SQLConnection connection)
+  { this.idFactory = idFactory;
+    this.list  = new DocumentInterface[3];
+    this.used  = 0;
+    this.size  = 3;
+    this.count = 0;
+    this.connection = connection;
+  }
+
+  /**
+   *  Obtain the list of <code>DocumentID</code> objects representing the unique
+   *  document identifiers of documents that refer to the file given as a parameter.
+   *
+   *  @param <code>URL</code> the location of the file to match
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> reference identifiers.
+   */
+  public List getDocumentIdsWithFile(URL fileLocation)
+  { List reply = new ArrayList();
+
+    GS3SQLSelect select = new GS3SQLSelect("files");
+    select.addField("*");
+    GS3SQLWhere  where  = new GS3SQLWhere(new GS3SQLWhereItem("FileLocation", "=", fileLocation.toString()));
+    select.setWhere(where);
+
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+    
+	select = new GS3SQLSelect("filegroups");
+	select.addField("DocID");
+	select.setDistinct(true);
+      
+	where = new GS3SQLWhere();
+	where.setCondition(GS3SQLWhere.OR_CONDITION);
+	
+	GS3SQLWhereItem whereItem = null;
+
+      	results.first();
+	do {
+	  int fileGroupRef = results.getInt("FileGroupRef");
+	  whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(fileGroupRef), GS3SQLField.INTEGER_TYPE);
+	  where.add(whereItem);
+	}
+	while (results.next());
+	select.setWhere(where);
+	
+	results = statement.executeQuery(select.toString());
+	
+	results.first();
+	do {
+	  String docId = results.getString("DocID");
+	  reply.add(docId);
+	} while (results.next());
+	statement.close();
+    }
+    catch (SQLException sqlEx) {
+	System.err.println("DocumentList.getDocumentIdsWithFile(): "+sqlEx);
+    }
+    
+    return reply;
+  }
+
+  /**
+   *  Get a list of documents that match a given set of patterns,
+   *  within a given URL node.
+   *
+   *  @param <code>List</code> the list of patterns to match
+   *  @param <code>String</code> the partial URL of the root node under which o
+   *         match files.  NB: this is a <code>String</code> as the URL may be
+   *         incomplete and not properly match the strict requirements for <code>URL</code>
+   */
+  public List findDocumentIdsUsingFiles(List fileRefs, String withinNode)
+  {
+    StringBuffer queryBuffer = new StringBuffer("SELECT FileGroupRef FROM files WHERE FileLocation ");
+
+    Iterator files = fileRefs.iterator();
+    while (files.hasNext()) {
+      String file = files.next().toString();
+
+      if (withinNode != null) {
+	queryBuffer.append("REGEXP \"^");
+	queryBuffer.append(withinNode);
+	queryBuffer.append(".*");
+      }
+      else {
+	queryBuffer.append("REGEXP \"");
+      }
+      queryBuffer.append(file);
+      queryBuffer.append("\"");
+
+      if (files.hasNext()) {
+	queryBuffer.append(" OR ");
+      }
+    }
+    queryBuffer.append(";");
+    return this.findDocumentIdsUsingFileQuery(queryBuffer.toString());
+  }
+
+  public List findDocumentIdsUsingFile(String fileRef, String withinNode)
+  { String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"^"+withinNode+".*"+fileRef+"\";";
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  /**
+   *  Return a list of document identifiers against a simple pattern.  No root node is given, so
+   *  any file matching the pattern given will be returned. USE WITH CAUTION!!!
+   *
+   *  @param <code>String</code> a fragment of file pathname to match against.
+   *
+   *  @return <code>List</code> of <code>DocumentID</code> objects.
+   */
+  public List findDocumentIdsUsingFile(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation REGEXP \"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+  public List findDocumentIdsUsingFileExact(String fileRef)
+  { 
+    // Get the simple list of file objects & their file group reference
+    String query = "SELECT FileGroupRef FROM files WHERE FileLocation=\"" + fileRef +"\";";
+
+    return this.findDocumentIdsUsingFileQuery(query);
+  }
+
+    private List findDocumentIdsUsingFileQuery(String query) {
+    
+	try {
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(query);
+
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+      
+      // get a list of group ids first and turn it into a query on filegroups
+      StringBuffer queryBuffer = new StringBuffer("SELECT * FROM filegroups WHERE ");
+      boolean first = true;
+      
+      do {
+	int groupRef = results.getInt("FileGroupRef");
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	queryBuffer.append("FileGroupRef=" + Integer.toString(groupRef));
+      } while (results.next());
+      queryBuffer.append(";");
+      
+      // make a holder for the actual file section identifiers
+      List divisions = new ArrayList();
+      
+      // expand (or, in fact, contract) through the document
+      // structures...recreating new filegroup queries as necessary
+      while (queryBuffer.length() > 0) {
+	results = statement.executeQuery(queryBuffer.toString());
+	
+	if (!results.first()) {
+	    statement.close();
+	    return null;
+	}
+	
+	queryBuffer = new StringBuffer();
+	do {
+	  String type = results.getString("ParentType");
+	  String parentRef = results.getString("ParentRef");
+	  if (type.equals(METSFileGroup.SECTION_PARENT)) {
+	    divisions.add(parentRef);
+	  }
+	  else {
+	    if (queryBuffer.length() > 0) {
+	      queryBuffer.append(" OR ");
+	    }
+	    queryBuffer.append("FileGroupRef=" + parentRef);
+	  }
+	} while (results.next());
+	
+	if (queryBuffer.length() > 0) {
+	  queryBuffer.insert(0, "SELECT * FROM filegroups WHERE ");
+	  queryBuffer.append(";");
+	}
+      }
+      
+      // ok, now find all the sections in which we are interested...
+      queryBuffer.setLength(0);
+      queryBuffer.append("SELECT DISTINCT DocID FROM filesection WHERE ");
+      Iterator iterator = divisions.iterator();
+      first = true;
+      
+      while (iterator.hasNext()) {
+	String ref = iterator.next().toString();
+	
+	if (first) {
+	  first = false;
+	}
+	else {
+	  queryBuffer.append(" OR ");
+	}
+	
+	queryBuffer.append("FileSectionRef="+ref);
+      }
+      queryBuffer.append(";");
+      
+      // execute the division query
+      results = statement.executeQuery(queryBuffer.toString());
+      
+      if (!results.first()) {
+	  statement.close();
+	  return null;
+      }
+
+      List reply = new ArrayList();
+      do {
+	reply.add(results.getString("DocID"));
+      } while (results.next());
+      
+      statement.close();
+      return reply;
+    }
+    catch (SQLException ex) {
+      System.err.println("DocumentList.findDocumentIdsUsingFileQuery()"+ ex);
+    }
+    return null;
+  }
+
+  /**
+   *  Cache a document into the cache, without writing it to the database.
+   *  Used directly by other parts of DocumentList when they know that the
+   *  document is in the database already, or they are going to write it
+   *  themselves...
+   *
+   *  @param <code>DocumentInterface</code> the document to cache
+   */
+  private void cacheDocument(DocumentInterface document)
+  { // increase cache size, etc. as necessary
+    if (this.used == this.size) {
+      if (this.size >= maxSize) {
+	for (int i = 0; i < this.size - 1; i ++) {
+	  this.list[i] = this.list[i+1];
+	}
+	this.used --;
+      }
+      else {
+	this.ensureSize((this.size * 2) > maxSize ? maxSize : (this.size * 2));
+      }
+    }
+    
+    // insert the document itself
+    this.list[this.used] = document;    
+  }
+
+  /**
+   *  Write the document into the document list (cache) and the database.
+   *
+   *  @param <code>DocumentInterface</code> the document itself
+   */
+  public void addDocument(DocumentInterface document)
+  { // initially, test if the document has a duplicate...
+    String duplicateDocID = document.getDuplicateID(this.connection);
+    if (duplicateDocID.length() > 0) {
+      System.out.println("Found duplicate document ");
+      return;
+    }
+
+    // first cache it...
+    this.cacheDocument(document);
+
+    // set the document identifier, if not already set
+    if (document.getID() == null) {
+      System.out.println("Posting new docuument ID");
+      DocumentID id = this.idFactory.getNewDocumentID(document);
+      document.setID(id);
+    }
+
+    // add to the database as well, if it is modified...
+    if (document.isChanged()) {
+      System.out.println("Document was changed");
+      document.getSQLWriter().writeDocument(document, this.connection);
+    }
+
+    // Remember that we've used one more item from the cache.
+    this.used ++;
+
+    // Note additional document
+    this.count ++;
+  }
+
+  /**
+   *  Note that an individual document is modified, and act accordingly
+   *
+   *  @param <code>DocumentInterface</code> the document
+   */
+  public void storeChangedDocument(DocumentInterface document)
+  { document.getSQLWriter().writeDocument(document, this.connection);
+  }
+
+  /**
+   *  Get an iterator across all the documents, not merely those in
+   *  the cache.  Note that this <code>Iterator</code> does <b>not</b>
+   *  support the <code>remove()</code> function, and will raise an
+   *  <code>UnsupportedOperationException</code> if you attempt to do
+   *  so.
+   *
+   *  @return <code>Iterator</code> the iterator across the documents.
+   */
+  public Iterator iterator()
+  { return new DocumentListIterator(connection);
+  }
+
+  /**
+   *  Get the nth member of the <b>cached</b> document list.
+   *
+   *  @deprecated
+   */
+  public DocumentInterface getDocument(int index)
+  { if (index < 0 || index >= this.used)
+    { return null;
+    }
+    return this.list[index];
+  }
+
+  /**
+   *  Simple "obtain a document" function
+   */
+  public DocumentInterface getDocument(DocumentID documentId)
+  {
+    DocumentInterface document = DocumentFactory.readSQLDocument(connection, documentId);
+    if (document != null) {
+      this.cacheDocument(document);
+    }
+    return document;
+  }
+
+  /**
+   *  Update timestamps on an entire document list - done at the beginning of a build cycle
+   *
+   *  @param <code>The date of the new build cycle</code>
+   */
+  public void updateTimestamps(long buildTimeStamp)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      
+      long thisTimeStamp = document.getFilesDatestamp();
+      long lastTimeStamp = document.getModifiedDatestamp();
+
+      if (thisTimeStamp > lastTimeStamp) {
+	System.out.println("Updating timestamps " + thisTimeStamp + " " + lastTimeStamp);
+
+	DocumentSQLWriter.touchDocument(document.getID(), this.connection, buildTimeStamp, thisTimeStamp);
+      }
+    }
+  }
+
+  /**
+   *  A convenience method to map onto the old Vector source code...
+   */
+  protected void ensureSize(int size)
+  { DocumentInterface [] newList = new DocumentInterface[size];
+    System.arraycopy(this.list, 0, newList, 0, this.size);
+    this.list = newList;
+    this.size = size;
+  }
+
+  /**
+   *  Write the documents into a directory as METS/XML
+   */
+  public void writeDocuments(File directory)
+  { Iterator documents = this.iterator();
+    int item = 0;
+
+    while (documents.hasNext())
+    { DocumentInterface document = (DocumentInterface) documents.next();
+      try
+      { item ++;
+	File localFile = new File(directory, "Doc"+Integer.toString(item)+".xml");
+	FileWriter fileWriter = new FileWriter(localFile);
+	PrintWriter writer = new PrintWriter(fileWriter);
+	document.getMETSWriter().writeDocument(document, writer);
+	writer.close();
+	fileWriter.close();
+      }
+      catch (IOException io)
+      {	
+      }
+    }
+  }
+
+  public void writeSQLDocuments(GS3SQLConnection connection)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].getSQLWriter().writeDocument(this.list[i], connection);
+    }
+  }
+
+  public static DocumentList readSQLDocuments(GS3SQLConnection connection)
+  { DocumentList list = new DocumentList(connection);
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+    
+    try {
+	Statement statement = connection.createStatement();
+	ResultSet documents = statement.executeQuery(select.toString());
+	if (documents.first()) { 
+	    do {
+		DocumentInterface document = AbstractDocument.readSQL(connection, documents);
+		list.addDocument(document);
+	    }
+	    while (documents.next());
+	}
+	statement.close();
+    }
+    catch (java.sql.SQLException ex) {
+	System.out.println("DocumentList.writeSQLDocuments(): "+ex);
+	return null;
+    }
+
+    return list;
+  }
+
+  public int getCount()
+  { return this.count;
+  }
+
+  public int size()
+  { return this.used;
+  }
+}
+
+class DocumentListIterator implements Iterator
+{
+  private boolean hasNext;
+  private Statement statement;
+  private ResultSet resultSet;
+  private GS3SQLConnection connection;
+
+  public DocumentListIterator(GS3SQLConnection connection)
+  {
+    this.connection = connection;
+
+    GS3SQLSelect select = new GS3SQLSelect("document");
+    select.addField("*");
+
+    try {
+	this.statement = connection.createStatement();
+      this.resultSet = statement.executeQuery(select.toString());
+      this.hasNext = this.resultSet.first();
+    } catch (SQLException ex) {
+      System.err.println("DocumentListIterator(): "+ex);
+      this.hasNext = false;
+    }
+  }
+
+  public boolean hasNext()
+  { return this.hasNext;
+  }
+
+  public Object next()
+  {
+    // get the 'next' document first
+    DocumentInterface document = AbstractDocument.readSQL(connection, this.resultSet);
+
+    // now actually step forward to the next item, so that we know if we have one!
+    try {
+      this.hasNext = this.resultSet.next();
+
+      if (!this.hasNext) {
+	  this.statement.close(); // be a good citizen & close used statement 
+      }
+    } catch (SQLException ex) {
+	System.err.println("DocumentList.iterator.next(): "+ex);
+      this.hasNext = false;
+    }
+    return document;
+  }
+  
+  public void remove() throws UnsupportedOperationException
+  { throw new UnsupportedOperationException("DocumentList does not support iterator removal of documents");
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentLoader.java	(revision 21112)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.URL;
+
+public class DocumentLoader
+{
+  public static String getAsString(InputStream in)
+  { StringBuffer reply;
+    byte 	data[] = new byte[1024];
+    int		databytes;
+
+    reply	= new StringBuffer();
+    
+    try
+    {
+      do
+      { databytes = in.read(data);
+        if (databytes > 0)
+	{ reply.append(new String(data, 0, databytes));
+	}
+      } while (databytes >= 0);
+    }
+    catch (IOException io)
+    {
+    }
+    
+    return reply.toString();
+  }
+
+  public static String getAsString(File file)
+  { FileInputStream in;
+    String          reply = null;
+
+    try
+    { in	= new FileInputStream(file);
+      if (in == null)
+	{	return null;
+	}
+      reply = getAsString(in);
+      
+      in.close();
+    }
+    catch (IOException io)
+    { return null;
+    }
+    return reply;
+  }
+
+  public static String getAsString(URL url)
+  { if (url.getProtocol().equals("file"))
+    { File file = new File(url.getPath());
+      return getAsString(file);
+    }
+  
+    return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentMetadata.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+public class DocumentMetadata
+{
+	String name;
+	String value;
+
+	public DocumentMetadata(String name, String value)
+	{	this.name = name;
+		this.value = value;
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentSQLWriter.java	(revision 21112)
@@ -0,0 +1,98 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import java.sql.*;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class DocumentSQLWriter
+{
+  public DocumentSQLWriter()
+  {
+  }
+
+  public static boolean touchDocument(DocumentID docID, GS3SQLConnection connection, long touchTime, long modTime)
+  { GS3SQLUpdate update = new GS3SQLUpdate("document");
+    update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", docID.toString())));
+    update.addDate("IndexedDate", new java.sql.Timestamp(touchTime));
+    update.addDate("ModifiedDate", new java.sql.Timestamp(modTime));
+    try {
+	Statement statement = connection.createStatement();
+	statement.execute(update.toString());
+	statement.close();
+    } catch (SQLException e) {
+	System.err.println("DocumentSQLWriter.touchDocument() Error: "+ e);
+	return false;
+    }
+    System.out.println(update.toString());
+    return true;
+  }
+
+  /**
+   * Just write the document to the database, assuming that it is not already there
+   */
+  public boolean writeDocument(DocumentInterface document, GS3SQLConnection connection)
+  {		
+    // TODO: output document in METS format, with Greenstone-specific metadata
+    // placed in a wrapper inside the descriptive metadata block
+
+    // put the document into the database
+    try {
+      if (document.getID() != null)
+      { //tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	GS3SQLSelect select = new GS3SQLSelect("document");
+	select.addField("*");
+	select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	
+	Statement statement = connection.createStatement();
+	ResultSet results = statement.executeQuery(select.toString());
+	if (!results.first()) { // empty result
+	  GS3SQLInsert insert = new GS3SQLInsert("document");
+	  insert.addValue("DocID", document.getID().toString());
+	  insert.addValue("DocType", document.getDocumentType());
+
+	  insert.addDate("AccessionDate", new java.sql.Timestamp(document.getAccessionDate()));
+	  insert.addDate("IndexedDate", new java.sql.Timestamp(document.getLastIndexedDate()));
+	  insert.addDate("ModifiedDate", new java.sql.Timestamp(document.getModifiedDatestamp()));
+
+	  statement.execute(insert.toString());
+	}
+	else {
+	  /* redundant code - not used... */
+	  //	  GS3SQLUpdate update = new GS3SQLUpdate("document");
+	  //	  update.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("DocID", "=", document.getID().toString())));
+	  //	  connection.execute(update.toString());
+	}
+	statement.close();
+      }
+    } catch (SQLException ex) {
+	System.out.println("DocumentSQLWriter.writeDocument() Error:"+ex);
+    }
+    //    output.println(tag);
+    
+    // Write the structural metadata:
+    //   if in "Create METS" mode:
+    //     take all the documents as being in the one group, of the "physical"
+    //     document; particular document plugins may extend this behaviour,
+    //     which would suggest placing it in the AbstractDocument or similar
+    //     class?!
+    if (document.getDocumentMetadata() == null) {
+      System.out.println("Null metadata document");
+    }
+    document.getDocumentMetadata().writeSQL(document, connection);
+
+    document.getDocumentFiles().writeSQL(document, connection);
+
+    document.getDocumentStructure().writeSQL(document, connection);
+
+    // Write the body information
+    // TODO: only output the document if there is some
+    // content/formatting change between the original and
+    // what greenstone actually does; if the original is
+    // good, then place a marker saying "get the document
+    // "from the original file"
+    return true;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/DocumentWriter.java	(revision 21112)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+public class DocumentWriter
+{
+    public DocumentWriter()
+    {
+    }
+
+    public boolean writeDocument(DocumentInterface document, PrintWriter output)
+    {		
+	// TODO: output <xml header>
+	String xml_header ="<?xml ";
+	xml_header = XMLTools.addAttribute(xml_header, "version", "1.0");
+	xml_header = XMLTools.addAttribute(xml_header, "encoding", "UTF-8");
+	xml_header = XMLTools.addAttribute(xml_header, "standalone", "no");
+	xml_header = xml_header+"?>";
+	output.println(xml_header);
+
+	// TODO: output document in METS format, with Greenstone-specific metadata
+	// placed in a wrapper inside the descriptive metadata block
+	String tag = XMLTools.getOpenTag("mets", "mets");
+	if (document.getID() != null){
+	    tag = XMLTools.addAttribute(tag,"xmlns:mets","http://www.loc/gov/METS/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance");
+	    tag = XMLTools.addAttribute(tag,"xmlns:gsdl3","http://www.greenstone.org/gsdlmetadata/1.0/");
+	    tag = XMLTools.addAttribute(tag,"xmlns:xlink","http://www.w3.org/TR/xlink/");
+	    tag = XMLTools.addAttribute(tag,"xsi:schemaLocation","http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/mets.xsd http://www.greenstone.org/gsdlmetadata/1.0/ http://www.greenstone.org/gsdlmetadata/1.0/gsdl_metadata_schema.xsd");
+	    tag = XMLTools.addAttribute(tag, "OBJID", document.getID().toString());
+	}
+	output.println(tag);
+	
+	// Writing the structural metadata:
+	//   if in "Create METS" mode:
+	//     take all the documents as being in the one group, of the "physical"
+	//     document; particular document plugins may extend this behaviour,
+	//     which would suggest placing it in the AbstractDocument or similar
+	//     class?!
+	document.getDocumentMetadata().write(output);
+
+	// TODO: only output the document if there is some
+	// content/formatting change between the original and
+	// what greenstone actually does; if the original is
+	// good, then place a marker saying "get the document
+	// "from the original file"
+	if  (document.isMETSCompatible()){
+	    document.getDocumentFiles().write(output);
+	    document.getDocumentStructure().write(output);
+	}
+	else {
+	    DocumentWriter writer = document.getMETSWriter();
+	    
+	    // ensure that this call isn't recursive - 
+	    // TODO: This is a rather naive check and needs improving
+	    if (writer != this) {
+		writer.writeDocument(document, output);
+	    }
+	}
+	output.write("</mets:mets>\n");
+	return true;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLDocument.java	(revision 21112)
@@ -0,0 +1,51 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class GMLDocument extends AbstractDocument
+{
+  public static final String GML_DOCUMENT_TYPE = "GML";
+
+  public GMLDocument(DocumentID id)
+  { super(id);
+  }
+
+  public GMLDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return GML_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  public METSDescriptiveSet getDocumentMetadata()
+  { return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GMLRecogniser.java	(revision 21112)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class GMLRecogniser extends  AbstractRecogniser
+{
+    
+    public GMLRecogniser()   { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".gml");
+	this.document_type = GMLDocument.GML_DOCUMENT_TYPE;
+    }
+    
+    
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting GML Document " + filename);
+		GMLDocument doc = new GMLDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/GS2METSRecogniser.java	(revision 21112)
@@ -0,0 +1,35 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class GS2METSRecogniser extends AbstractRecogniser
+{
+    public GS2METSRecogniser() 
+    {
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("docmets.xml");
+	this.document_type = METSDocument.METS_DOCUMENT_TYPE;
+    }
+    
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocument.java	(revision 21112)
@@ -0,0 +1,354 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.net.MalformedURLException;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.*;
+
+public class HTMLDocument extends AbstractDocument
+{
+  public static final String HTML_DOCUMENT_TYPE = "HTML";
+  Document domDocument;
+
+  public HTMLDocument(DocumentID id)
+  { super(id);
+    this.domDocument = null;
+  }
+
+  /**
+   *  Create the HTMLDocument from a given URL - the URL may in fact be a reference
+   *  to a local file.
+   *
+   *  @param <code>URL</code> The location from which to load the file
+   */
+  public HTMLDocument(URL url)
+  { super(url);
+
+    this.loadDocument(url);
+
+    this._extractDocumentFiles();
+    this._extractDocumentMetadata();
+    // extract the files before resolving the images
+    resolveImages();
+    HTMLDocumentTools docTools = new HTMLDocumentTools(this.domDocument);
+    docTools.setMetsDocument(this);
+    docTools.setUrl(this.fileSet.getFile(0).getLocation());
+    docTools.findSections();
+    METSStructure sectionStruct = docTools.getStructure();
+
+    if (sectionStruct.size() > 0) {
+      METSStructureSet structureSet = this.getDocumentStructure();
+      structureSet.addStructure(sectionStruct);
+    }
+  }
+
+  private void loadDocument(URL url)
+  {
+    //    HTMLDoc htmlDoc;
+
+    HTMLTidy tidyDoc;
+    if (url.getProtocol().equals("file"))
+    { tidyDoc = new HTMLTidy(new File(url.getPath()));
+    }
+    else
+    { //htmlDoc = new HTMLDoc(url);
+      tidyDoc = new HTMLTidy(url);
+    }
+    
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+
+    this.domDocument = tidyDoc.getDocument(); 
+    Runtime.getRuntime().gc();
+    //    System.out.println(Runtime.getRuntime().freeMemory() + " " + Runtime.getRuntime().totalMemory());
+ }
+
+  private void _extractDocumentMetadata()
+  { 
+    NodeList metadata = this.domDocument.getElementsByTagName("META");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String name = element.getAttribute("name");
+      if (name == null || name.length() == 0) {
+	continue;
+      }
+
+      String value = element.getAttribute("content");
+      if (value == null || value.length() == 0) {
+	value = name;
+      }
+
+      this.addDocumentMetadata(new MetadataLabel(name), value);
+    }
+    
+    NodeList titles = this.domDocument.getElementsByTagName("TITLE");
+    StringBuffer title = new StringBuffer();
+    for (int n = 0; n < titles.getLength(); n ++) {
+      Node node = titles.item(n);
+      Element element = (Element) node;
+
+      NodeList childNodes = node.getChildNodes();
+      for (int c = 0; c < childNodes.getLength(); c ++) {
+	Node child = childNodes.item(c);
+	if (child.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	  title.append(child.getNodeValue());
+	}
+      }
+    }
+
+    if (title.length() > 0)
+    { this.addDocumentMetadata(new MetadataLabel("Title"), title.toString());
+    }
+
+    /* Old HTMLDoc based parsing...
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    boolean inTitle = false;
+    title = new StringBuffer();
+
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      {	HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("meta"))
+	{ // check that the name of the metadata item exists
+	  String name = tag.idValue("name");
+	  if (name == null || name.length() == 0) {
+	    continue;
+	  }
+
+	  // get the value, if it exists
+	  String value = tag.idValue("content");
+
+	  // if value does not exist, default it to being the same
+	  // as the name.
+	  if (value == null || value.length() > 0) {
+	    value = name;
+	  }
+	}
+	else 
+	if (tag.tagName().equals("title"))
+	{ inTitle = true;
+	}
+	else if (tag.tagName().equals("/title"))
+	{ inTitle = false;
+	}
+	// cut off when real body content appears - not a perfect
+        // implementation, just cheap & cheerful
+	else if (tag.tagName().equals("/head"))
+	{ break;
+	}
+	else if (tag.tagName().equals("body"))
+	{ break;
+	}
+      }
+      else if (inTitle == true)
+      { title.append(codedContent.elementAt(e).toString());
+      }
+    }
+      */
+  }
+
+  private void _extractDocumentFiles()
+  { URL homeUrl = this.fileSet.getFile(0).getLocation();
+
+    NodeList metadata = this.domDocument.getElementsByTagName("img");
+    for (int n = 0; n < metadata.getLength(); n ++) {
+      Node node = metadata.item(n);
+      Element element = (Element) node;
+
+      String location = element.getAttribute("src");
+      if (location == null || location.length() == 0) {
+	System.out.println("No name");
+	continue;
+      }
+      try 
+      { // make the url for the image, and then add it to the document list of
+	// 
+	URL imgUrl = new URL(homeUrl, location);
+	METSFile file = this.fileSet.addFile(imgUrl);
+	this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+      }
+      catch (MalformedURLException ex)
+      { // TODO: report exception/failure to resolve...
+      }
+    }
+
+  
+    /**    
+    HTMLBlock codedContent = htmlDoc.getCodedContent();
+    for (int e = 0; e < codedContent.size(); e ++)
+    { if (codedContent.elementAt(e) instanceof HTMLTag)
+      { HTMLTag tag = (HTMLTag) codedContent.elementAt(e);
+
+        if (tag.tagName().equals("img"))
+	{ String location = tag.idValue("src");
+
+	  try 
+	  { // make the url for the image, and then add it to the document list of
+	    // 
+	    URL imgUrl = new URL(homeUrl, location);
+	    METSFile file = this.fileSet.addFile(imgUrl);
+	    this.structureSet.getStructure("All").getDivision("All").addFileReference(file.getID());
+	  }
+	  catch (MalformedURLException ex)
+	  { // TODO: report exception/failure to resolve...
+	  }
+	}
+      }
+    }
+    */
+  }
+
+  public String getDocumentType()
+  { return HTML_DOCUMENT_TYPE;
+  }
+
+    // I think this is used for single section documents, while getSectionText
+    // is used for sectioned documents
+    // we will use the domDocument rather than reading it in again to another HTMLDoc.
+  public String getDocumentText()
+  { 
+      XMLConverter converter = new XMLConverter();
+      return converter.getPrettyString(this.domDocument.getDocumentElement());
+      /*  HTMLDoc htmlDoc;
+    URL     url =(URL) this.fileSet.getFile(0).getLocation();
+
+    this.getSectionText("1");
+
+    if (url.getProtocol().equals("file"))
+    { htmlDoc = new HTMLDoc(url, url.getPath());
+    }
+    else
+    { htmlDoc = new HTMLDoc(url);
+    }
+    return htmlDoc.getContent();
+      */ 
+  }
+
+    private void resolveImages() {
+
+	// find the path of the url relative to the collection
+	URL full_path = this.fileSet.getFile(0).getLocation();
+       
+	String base_url;
+	if (full_path.getProtocol().equals("file")) {
+	    base_url = full_path.getPath();
+	    int import_pos = base_url.indexOf("import");
+	    base_url = base_url.substring(import_pos);
+	    base_url = "_httpcollection_/"+base_url;
+	} else {
+	    base_url = full_path.toString();
+	}
+
+	// need to take off the last part
+	base_url = base_url.substring(0, base_url.lastIndexOf("/")+1);
+
+	NodeList metadata = this.domDocument.getElementsByTagName("img");
+	for (int n = 0; n < metadata.getLength(); n ++) {
+	    Node node = metadata.item(n);
+	    Element element = (Element) node;
+	    
+	    String location = element.getAttribute("src");
+	    if (location != null && location.length() > 0 && isRelative(location)) {
+		// modify the source url
+		element.setAttribute("src", base_url+location);
+	    }
+	}
+    }
+
+    private boolean isRelative(String location) {
+
+	if ( location.startsWith("http:") || location.startsWith("file:")) {
+	    return false;
+	}
+	return true;
+    }
+    
+    public Document getDOMDocument()
+    {    
+      if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+    return this.domDocument;
+  }
+
+  private XPointer getXPointer(METSDivision division)
+  { String fileId = division.getDefaultFileReference();
+
+    if (fileId == null) {
+      System.err.println("Unable to obtain file reference for section " + division.getLabel());
+      return null;
+    }
+
+    METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+    if (fileGroup == null) {
+      System.err.println("Unable to obtain file reference for filegroup " + fileId);
+      return null;
+    }
+
+    METSFile file = fileGroup.getFile(0);
+    if (file == null) {
+      System.err.println("Unable to obtain any files within filegroup " + fileId);
+      return null;
+    }
+
+    URL url = file.getLocation();
+    XPointer xpointer = XPointer.processXPointer(this.domDocument, url);
+
+    return xpointer;
+  }
+
+  private XPointer getXPointer(String sectionId)
+  { if (this.domDocument == null) {
+      URL     url =(URL) this.fileSet.getFile(0).getLocation();
+      this.loadDocument(url);
+      resolveImages();
+    }
+
+    METSDivision division = 
+      this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					      sectionId);
+    if (division == null) {
+      //      System.err.println("Unable to locate section " + sectionId);
+      return null; 
+    }
+
+    return this.getXPointer(division);
+  }
+  
+  public Node getSectionStartNode(METSDivision division)
+  { XPointer xpointer = this.getXPointer(division);
+    return xpointer.getStartNode();
+  }
+
+  public Node getSectionStartNode(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    return xpointer.getStartNode();
+  }
+
+  public String getSectionText(String sectionId)
+  { XPointer xpointer = this.getXPointer(sectionId);
+
+    if (xpointer == null) {
+      return "";
+    }
+
+    // get the XML content of the xpointer...
+    return xpointer.toString();
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLDocumentTools.java	(revision 21112)
@@ -0,0 +1,328 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.w3c.dom.*;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSStructureSet;
+import org.greenstone.gsdl3.gs3build.metadata.AbstractStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSStructure;
+import org.greenstone.gsdl3.gs3build.metadata.METSDivision;
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptive;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileSet;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileGroup;
+import org.greenstone.gsdl3.gs3build.metadata.METSFile;
+import org.greenstone.gsdl3.gs3build.metadata.METSFilePos;
+import org.greenstone.gsdl3.gs3build.metadata.METSFileID;
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+import org.greenstone.gsdl3.gs3build.doctypes.AbstractDocument;
+import org.greenstone.gsdl3.gs3build.util.*;
+
+public class HTMLDocumentTools
+{
+  Document          document;
+  URL               documentUrl;
+  int               sectionLevel;
+  StringBuffer      sectionLabel;
+  List              sectionSeq;
+  HTMLSection       rootSection;
+  HTMLSection       currentSection;
+  int               sectionSeqNo;
+  METSStructure     metsStructure;
+  AbstractStructure currentMets;
+  AbstractDocument  metsDocument;
+
+  /**
+   *
+   */
+  class HTMLSection
+  { String startLocation;
+    String endLocation;
+    String label;
+    List   children;
+    HTMLSection parent;
+    int         seqNo;
+
+    public HTMLSection(String label, int seqNo)
+    { this.label = label;
+      this.seqNo = seqNo;
+      this.parent = null;
+      this.children = new ArrayList();
+    }
+    
+    public void setStartLocation(String location)
+    { this.startLocation = location;
+    }
+    
+    public void setEndLocation(String location)
+    { this.endLocation = location;
+    }
+
+    public void addChild(HTMLSection child)
+    { this.children.add(child);
+      child.parent = this;
+    }
+    
+    public HTMLSection getParent()
+    { return this.parent;
+    }
+
+    public String getLabel()
+    { return this.label;
+    }
+
+    public String getXPointer()
+    { StringBuffer xPointerBuffer = new StringBuffer();
+    
+      xPointerBuffer.append("xpointer(");
+      xPointerBuffer.append(this.startLocation);
+      xPointerBuffer.append("/range-to(");
+      xPointerBuffer.append(this.endLocation);
+      xPointerBuffer.append("))");
+      
+      return xPointerBuffer.toString();
+    }
+  }
+
+  public HTMLDocumentTools(Document document)
+  { this.document = document;
+    this.sectionLevel = 0;
+    this.sectionSeq   = new ArrayList();
+    this.sectionSeq.add(new Integer(0));
+    this.sectionLabel = new StringBuffer();
+    this.rootSection = new HTMLSection("", 0);
+    this.currentSection = this.rootSection;
+    this.metsStructure  = new METSStructure(METSStructureSet.GSDL3_SECTION_STRUCTURE, 
+					    "Section", "Section");
+    this.currentMets    = this.metsStructure;
+    this.metsDocument   = null;
+  }
+
+  public void setMetsDocument(AbstractDocument document)
+  { this.metsDocument = document;
+  }
+
+  public void setUrl(URL url)
+  { this.documentUrl = url;
+  }
+
+  public Object findSections()
+  { this.findSections(document.getDocumentElement(), "/1");
+    return this.rootSection;
+  }
+
+  public void findSections(Node node, String location)
+  { // if at an element, then we aren't going to find section information,
+    // but we do need to recurse into the children to find the section markers
+    // there
+    if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      NodeList children = node.getChildNodes();
+      StringBuffer locationBuffer = new StringBuffer();
+      locationBuffer.append(location);
+      locationBuffer.append("/");
+
+      int elementNo = 1;
+      int commentNo = 1;
+      for (int c = 0; c < children.getLength(); c++)
+      {
+	locationBuffer.setLength(location.length()+1);
+	if (children.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  locationBuffer.append(Integer.toString(elementNo));
+	  elementNo ++;
+	}
+	else if (children.item(c).getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+	  locationBuffer.append("comment()[");
+	  locationBuffer.append(Integer.toString(commentNo));
+	  locationBuffer.append("]");
+	  commentNo ++;
+	}
+	this.findSections(children.item(c), locationBuffer.toString());
+      }
+    }
+    // if at a comment node, then one needs to dissect it to discover if
+    // any section markings are included...
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      String commentText = node.getNodeValue();
+      
+      // parse the node now...checking first for any 'section' items in
+      // it - this is a crude check...
+      if (commentText.indexOf("<Section>") >= 0 ||
+	  commentText.indexOf("</Section>") >= 0) {
+	// parse the section roughly using the primitive HTMLDoc form...      
+	HTMLDoc innerDoc     = HTMLDoc.fromString("dummy", commentText);
+	HTMLBlock innerHtml  = innerDoc.getCodedContent();
+	boolean   inMetadata = false;
+	StringBuffer metaValue = new StringBuffer();
+	String       metaLabel = null;
+
+	// Now just go through the content in sequence...
+	for (int e = 0; e < innerHtml.size(); e ++) {
+	  Object element = innerHtml.elementAt(e);
+
+	  if (element instanceof HTMLTag) {
+	    HTMLTag tag = (HTMLTag) element;
+
+	    if (tag.tagName().equals("section")) {
+	      // get the counters sorted out for the sections...
+	      this.sectionSeq.set(this.sectionLevel, new Integer(((Integer) this.sectionSeq.get(this.sectionLevel)).intValue() + 1));
+	      this.sectionLevel ++;
+	      if (sectionSeq.size() == this.sectionLevel) {
+		this.sectionSeq.add(new Integer(0));
+	      }
+	      else {
+		this.sectionSeq.set(this.sectionLevel, new Integer(0));
+	      }
+
+	      // create the label for the new section...
+	      this.sectionLabel.setLength(0);
+	      for (int s = 0; s < sectionLevel; s ++) {
+		if (s > 0) {
+		  this.sectionLabel.append(".");
+		}
+		this.sectionLabel.append(this.sectionSeq.get(s).toString());
+	      }
+
+	      // note the new section in a new data structure...
+	      HTMLSection newSection = new HTMLSection(this.sectionLabel.toString(), this.sectionSeqNo);
+
+	      newSection.setStartLocation(location);
+	      this.currentSection.addChild(newSection);
+	      this.currentSection = newSection;
+
+	      // do the same for mets...
+	      METSDivision newDivision = new METSDivision("DS" + Integer.toString(this.sectionSeqNo), Integer.toString(this.sectionSeqNo),
+							  this.sectionLabel.toString(), this.sectionLabel.toString(), "Section");
+	      this.currentMets.addDivision(newDivision);
+	      this.currentMets = newDivision;
+
+	      this.sectionSeqNo ++;
+	    }
+	    else if (tag.tagName().equals("/section")) {
+	      // cope with poorly formed metadata...
+	      if (inMetadata) {
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+
+	      // note the end position of the section
+	      this.currentSection.setEndLocation(location);
+
+	      // now post that section to the file list...
+	      this.postFilePtr(this.currentSection);
+
+	      // backtrack up the section tree...
+	      this.currentSection = this.currentSection.getParent();
+	      this.currentMets = this.currentMets.getParent();
+	      this.sectionLevel --;
+
+	      // trim the section label
+	      int cutTo = this.sectionLabel.toString().lastIndexOf('.');
+	      if (cutTo >= 0) {
+		this.sectionLabel.setLength(cutTo);
+	      }
+	      else {
+		this.sectionLabel.setLength(0);
+	      }
+
+	      // we can't be in metadata either...
+	      inMetadata = false;
+	    }
+	    else if (tag.tagName().equals("metadata")) {
+	      if (inMetadata) {
+		// post the metadata
+		this.postMetadata(metaLabel, metaValue.toString());
+	      }
+	      metaValue.setLength(0);
+	      metaLabel = tag.idValue("name");
+	      inMetadata = (metaLabel != null);
+	    }
+	    else if (tag.tagName().equals("/metadata")) {
+	      // post the metadata
+	      this.postMetadata(metaLabel, metaValue.toString());
+	      inMetadata = false;
+	    }
+	  }
+	  else {
+	    if (inMetadata) {
+	      metaValue.append(element.toString());
+	    }
+	  }
+	}
+      }
+    }
+  }
+
+  private String makeXPointer(HTMLSection section)
+  { StringBuffer xPointerBuffer = new StringBuffer();
+
+    if (this.documentUrl != null)
+    { xPointerBuffer.append(this.documentUrl.toString());
+      xPointerBuffer.append("#");
+    }
+    xPointerBuffer.append(section.getXPointer());
+
+    return xPointerBuffer.toString();
+  }
+
+  private void postFilePtr(HTMLSection section)
+  { String xPointer = this.makeXPointer(section);
+
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    try {
+      URL url = new URL(xPointer);
+
+      METSFileSet fileSet = this.metsDocument.getDocumentFiles();
+
+      METSFilePos filePos = new METSFilePos(url);
+      METSFile    file = fileSet.createFile(filePos, "text/html");
+      
+      METSFileGroup fileGroup;
+      String fileRef = ((METSDivision) this.currentMets).getDefaultFileReference();
+
+      if (fileRef == null) {
+	fileGroup = fileSet.createGroup();
+	fileSet.addGroup(fileGroup);
+	((METSDivision) this.currentMets).addFileReference(fileGroup.getId());
+      }
+      else {
+	// simple call - the group should be at the 'root' level, immediately below
+	// the FileSet itself...
+	fileGroup = fileSet.getGroup(fileRef);
+      }
+      fileGroup.addFile(file);
+
+      
+    }
+    catch (java.net.MalformedURLException ex) {
+      System.err.println(ex + " " + xPointer);
+    }
+  }
+
+  private void postMetadata(String label, String value)
+  {
+    if (this.metsDocument == null) {
+      return;
+    }
+
+    String metadataRef = ((METSDivision) this.currentMets).getDefaultMetadataReference();
+    METSDescriptive descriptive;
+    if (metadataRef == null) {
+      descriptive = this.metsDocument.getDocumentMetadata().createDescriptive(this.currentSection.getLabel());
+      ((METSDivision) this.currentMets).addMetadataReference(descriptive.getID());
+    }
+    else {
+      descriptive = this.metsDocument.getDocumentMetadata().getDescriptiveById(metadataRef);
+    }
+    descriptive.addMetadata(MetadataLabel.getDefaultNamespace(), label, value);
+  }
+
+  public METSStructure getStructure()
+  { return this.metsStructure;
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/HTMLRecogniser.java	(revision 21112)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class HTMLRecogniser extends AbstractRecogniser
+{
+    
+    public HTMLRecogniser()
+    { 
+	this.preferred_mime_type = "text/html";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".htm");
+	this.filename_extensions.add(".html");
+	this.document_type = HTMLDocument.HTML_DOCUMENT_TYPE;
+	
+    }
+    
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexDocument.java	(revision 21112)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class IndexDocument extends AbstractDocument
+{
+  public static final String INDEX_DOCUMENT_TYPE = "GSINDEX";
+
+  public IndexDocument(DocumentID id)
+  { super(id);
+  }
+
+  public IndexDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return INDEX_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return true;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/IndexRecogniser.java	(revision 21112)
@@ -0,0 +1,52 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+public class IndexRecogniser extends AbstractRecogniser
+{
+    
+    public IndexRecogniser(DocumentList listRepository)
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("index.txt");
+	this.document_type = IndexDocument.INDEX_DOCUMENT_TYPE;
+    }
+    
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    String leafName;
+	    int leafAt = filename.lastIndexOf(File.separator);
+	    if (leafAt >= 0) {
+		leafName = filename.substring(leafAt+1);
+	    } else {
+		leafName = filename;
+	    }
+
+	    if (leafName.equals("index.txt")) {
+		
+		System.out.println("Posting Index Document " + filename);
+		IndexDocument doc = new IndexDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		//				System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGDocument.java	(revision 21112)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+
+/**
+ *  Another fairly basic class for representing JPEG documents.
+ */
+
+public class JPEGDocument extends AbstractDocument
+{
+  public static final String JPEG_DOCUMENT_TYPE = "JPEG";
+
+  public JPEGDocument(DocumentID id)
+  { super(id);
+  }
+  
+  public JPEGDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return JPEG_DOCUMENT_TYPE;
+  }
+
+  public String getDocumentText()
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { // Just a dummy function for JPEGs
+    return "";
+  }
+
+  public boolean isMETSCompatible()
+  { return true;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/JPEGRecogniser.java	(revision 21112)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.util.ArrayList;
+
+public class JPEGRecogniser extends AbstractRecogniser
+{
+    
+    public JPEGRecogniser()
+    { 
+	this.preferred_mime_type = "image/jpeg";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".jpg");
+	this.filename_extensions.add(".jpeg");
+	this.document_type = JPEGDocument.JPEG_DOCUMENT_TYPE;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/METSDocument.java	(revision 21112)
@@ -0,0 +1,249 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTMLDoc;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+
+public class METSDocument extends AbstractDocument {
+
+    public static final String METS_DOCUMENT_TYPE = "METS";
+    Document domDocument;
+    public METSDocument(DocumentID id) {
+    	super(id);
+	//domDocument = null;
+    }
+
+    public METSDocument(URL url) {
+	super(url);
+	
+	if (url.getProtocol().equals("file")) {
+	    this._parseFile(new File(url.getPath()));
+	} else {
+	    System.err.println("Error: METSDocument must currently be a file");
+	    System.err.println("       URL = " + url.getPath());
+	}
+    }
+    
+    private void _parseFile(File file) {
+	
+	String parseFilePath;
+	try {
+	    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder builder = factory.newDocumentBuilder();
+	    domDocument = builder.parse(file);
+
+	    int filePosition = file.getPath().indexOf("import/")+7;
+	    parseFilePath = file.getPath().substring(0, filePosition);
+	    // TODO: get all the types in the tree
+      
+	    // TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+	    // this shouldn't happen except in a directly defined situation - actually doing the
+	    // parsing in part inside each node would work well provided one checked for a node
+	    // having already been done...
+	    System.out.println("Read");
+      
+	    // Get METSDmdSec metadata sections
+	    NodeList dmdSecs = domDocument.getElementsByTagName("mets:dmdSec");
+	    
+	    this.metadata = METSDescriptiveSet.parseXML(dmdSecs);
+
+	    // Get METSFileSec sections
+	    NodeList fileSecs = domDocument.getElementsByTagName("mets:fileSec");
+	    this.fileSet = METSFileSet.parseXML(fileSecs, fileSet, parseFilePath);
+	    // Get METSStructureMap Sections
+	    NodeList structMapSecs = domDocument.getElementsByTagName("mets:structMap");
+	    this.structureSet = METSStructureSet.parseXML(structMapSecs);
+
+	    //for (int g = 0; g < fileSecs.getLength(); g ++) {
+		//Schema schema = new Schema(schemas.item(s));
+		//this._parseFileSec(fileSecs.item(g));
+	    //}
+      
+	    // Get document metadata sections
+	    //NodeList dmdSecs = document.getElementsByTagName("mets:dmdSec");
+      
+	    //Schema schema = new Schema(schemas.item(s));
+	    //this.metadata = METSDescriptiveSet.parseXML(fileSecs);
+	} catch (FactoryConfigurationError e) {
+	    System.out.println(e);
+	} catch (ParserConfigurationException ex) {
+	    System.out.println(ex);
+	} catch (SAXException ex) {
+	    System.out.println(ex);
+	} catch (IOException ex) {
+	    System.out.println(ex);
+	}
+    }
+    /*
+    private void _parseFileSec(Node fileSec) {
+	// this is in effect a group without a sense of 'self'...
+	this._parseFileGroup((Element) fileSec, null);
+    }
+
+    private void _parseFileGroup(Element groupTag, METSFileGroup group) {
+	NodeList children = groupTag.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+
+	    System.out.println(children.item(c));
+    
+	    Element element = (Element) children.item(c);
+
+	    if (element.getNodeName().equals("mets:File")) {
+		if (group != null) {
+		    METSFile file = METSFile.parseXML(element, group);
+		} else {
+		    // TODO: error
+		}
+	    } else if (element.getNodeName().equals("mets:fileGrp")) {
+		// recurse
+		METSFileGroup childGroup = new METSFileGroup(element.getAttribute("ID"));
+
+		this._parseFileGroup(element, childGroup);
+		if (group != null) {
+		    group.addGroup(childGroup);
+		} else { 
+		    this.fileSet.addGroup(childGroup);
+		}
+	    }
+	}
+	}*/
+	
+    public String getDocumentType() {
+	return METS_DOCUMENT_TYPE;
+    }
+	
+    public String getDocumentText() {
+	// TODO: make this more than a dummy function!
+	HTMLDoc metsDoc;
+	URL url = (URL) this.fileSet.getFile(0).getLocation();
+
+	this.getSectionText("1");
+	
+	if (url.getProtocol().equals("file")) {
+	    metsDoc = new HTMLDoc(url,url.getPath());
+	} else {
+	    metsDoc = new HTMLDoc(url);
+	}
+	return metsDoc.getContent();
+    }
+    public Document getDOMDocument() 
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);
+	    }
+	}
+	return domDocument;
+    }
+
+    private XPointer getXPointer(METSDivision division)
+    {
+	String fileId = division.getDefaultFileReference();
+
+	if (fileId == null) {
+	    System.err.println("Unable to obtain file reference for section " + division.getLabel());
+	    return null;
+	}
+	
+	METSFileGroup fileGroup = this.getDocumentFiles().getGroup(fileId);
+	if (fileGroup == null) {
+	    System.err.println("Unable to obtain file reference for filegroup " + fileId);
+	    return null;
+	}
+	METSFile file = fileGroup.getFile(0);
+	if (file == null) {
+	    System.err.println("Unable to obtain any files within filegroup " + fileId);
+	    return null;
+	}
+	URL url = file.getLocation();
+	XPointer xpointer = XPointer.processXPointer(domDocument, url);
+	
+	return xpointer;
+    }
+    
+    private XPointer getXPointer(String sectionId)
+    {
+	if (domDocument == null) {
+	    URL url = (URL) this.fileSet.getFile(0).getLocation();
+	    try {
+		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		domDocument = builder.parse(new File(url.getPath()));
+	    } catch (FactoryConfigurationError e) {
+		System.out.println(e);
+	    } catch (ParserConfigurationException ex) {
+		System.out.println(ex);
+	    } catch (SAXException ex) {
+		System.out.println(ex);
+	    } catch (IOException ex) {
+		System.out.println(ex);       
+	    }
+	}
+	
+	METSDivision division = 
+	    this.getDocumentStructure().getDivision(METSStructureSet.GSDL3_SECTION_STRUCTURE, sectionId);
+	
+	if(division == null) {
+	    return null;
+	}
+      	return this.getXPointer(division);
+    }
+
+    public Node getSectionStartNode(METSDivision division)
+    {
+	XPointer xpointer = this.getXPointer(division);
+	return xpointer.getStartNode();
+    }
+
+    public Node getSectionStartNode(String sectionId)
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	return xpointer.getStartNode();
+    }
+    
+    public String getSectionText(String sectionId) 
+    {
+	XPointer xpointer = this.getXPointer(sectionId);
+	
+	if (xpointer == null) {
+	    return "";
+	}
+
+	//get the XML content of the xpointer
+	return xpointer.toString();
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataDocument.java	(revision 21112)
@@ -0,0 +1,58 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.greenstone.gsdl3.gs3build.metadata.METSDescriptiveSet;
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+/**
+ *  A metadata-pure document - it only contains metadata
+ *  on other documents, and this will be added to them during
+ *  the "extract" phase.  The decoration of the other documents
+ *  with this extra metadata will only occur later, as at the
+ *  point that this document/file is loaded, we may not in fact
+ *  have come across them yet...
+ */
+
+public class MetadataDocument extends AbstractDocument
+{
+  public static final String METADATA_DOCUMENT_TYPE = "GSMETADATA";
+
+  public MetadataDocument(DocumentID id)
+  { super(id);
+  }
+
+  public MetadataDocument(URL url)
+  { super(url);
+  }
+  
+  public String getDocumentType()
+  { return METADATA_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return "";
+  }
+
+  public String getSectionText(String sectionId)
+  { return "";
+  }
+
+  /**
+   *  Indicate whether this document is indexed.
+   *
+   *  @see: DocumentInterface.isIndexed
+   */
+  public boolean isIndexed()
+  {
+    return false;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/MetadataRecogniser.java	(revision 21112)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+//import java.io.*;
+import java.net.URL;
+import java.util.ArrayList;
+
+
+public class MetadataRecogniser extends AbstractRecogniser
+{
+
+    public MetadataRecogniser()
+    { 
+	this.preferred_mime_type = "text/xml";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add("metadata.xml");
+	this.document_type = MetadataDocument.METADATA_DOCUMENT_TYPE;
+    }
+
+    public boolean parseDocument(URL url)
+    { 
+	String filename = null;
+	
+	if (url.getProtocol().equals("file")) {
+	    filename = url.getPath();
+	}
+	
+	if (filename != null) {
+	    if (isAcceptedFilename(filename)) {
+		
+		System.out.println("Posting Metadata Document " + filename);
+		MetadataDocument doc = new MetadataDocument(url);
+		this.list_repository.addDocument(doc);
+		// TODO: spawn knowledge of children too...
+		// System.out.println(doc.getDocumentText());
+		return true;
+	    }
+	} else {
+	    // TODO: get Mime type remotely, and then proceed if required
+	}
+	return false;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserInterface.java	(revision 21112)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.net.URL;
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+
+/** Interface for Recogniser Objects
+ *
+ * A skeleton class is provided for simple Recognisers (AbstractRecogniser)
+ * 
+*/
+public interface RecogniserInterface
+{
+    public boolean configure(Element config_elem);
+    public void setListRepository(DocumentList docList);
+    public boolean parseDocument(METSFile fileReference);
+    public boolean parseDocument(URL fileReference);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/RecogniserManager.java	(revision 21112)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import org.greenstone.gsdl3.gs3build.FileCrawlObserver;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class RecogniserManager implements FileCrawlObserver
+{
+  RecogniserInterface list[];
+  int                 used;
+  int                 size;
+    DocumentList docList;
+  public RecogniserManager(DocumentList documentList)
+  { this.list = new RecogniserInterface[10];
+    this.used = 0;
+    this.size = 10;
+    this.docList = documentList;
+  }
+
+  /**
+   *  Add a recogniser to this manager
+   */
+    public RecogniserInterface addRecogniser(String documentType)
+    { 
+	ClassLoader loader = ClassLoader.getSystemClassLoader();
+	java.lang.Class thisClass = null;
+	try {
+	    thisClass = loader.loadClass("org.greenstone.gsdl3.gs3build.doctypes." + documentType + "Recogniser");
+	    Object instance = thisClass.newInstance();
+	    this.addRecogniser((RecogniserInterface) instance);
+	    return (RecogniserInterface)instance;
+	}
+	catch (ClassNotFoundException ex) {
+	    System.err.println(ex);
+	    return null;
+	}
+	catch (InstantiationException instEx) {
+	    System.err.println(instEx);
+	}
+	catch (IllegalAccessException accEx) {
+	    System.err.println(accEx);
+	}
+	return null;
+    }
+    
+  /**
+   *
+   */
+    public void addRecogniser(RecogniserInterface recogniser)
+    {  
+	recogniser.setListRepository(this.docList);
+	this.ensureCapacity(this.used + 1);
+  
+	this.list[this.used] = recogniser;
+	this.used ++;
+    }
+
+  public void processFile(URL url)
+  { boolean result;
+
+    for (int r = 0; r < this.used; r ++)
+    { if (list[r].parseDocument(url)) {
+        break;
+      }
+    }
+  }
+ 
+  public void processFile(File file)
+  { 
+      System.err.println("processing file "+file.getPath());
+      URL url = URLTools.getFileURL(file);
+    
+    if (url != null) 
+    { this.processFile(url);
+    }
+  }
+
+  private void ensureCapacity(int size)
+  { while (size >= this.size)
+    { RecogniserInterface newList [] = new RecogniserInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextDocument.java	(revision 21112)
@@ -0,0 +1,45 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *  A pretty basic DocumentInterface class - just able to
+ *  provide unsophisticated access to the document as plain
+ *  text, and no attempt at metadata extraction
+ */
+
+public class TextDocument extends AbstractDocument
+{
+  public static final String TEXT_DOCUMENT_TYPE = "Text";
+
+  public TextDocument(DocumentID id)
+  { super(id);
+  }
+
+  public TextDocument(URL url)
+  { super(url);
+  }
+
+  public String getDocumentType()
+  { return TEXT_DOCUMENT_TYPE;
+  }
+
+  /**
+   *  A pretty minimal and lazy document text extraction process.
+   */
+  public String getDocumentText()
+  { return DocumentLoader.getAsString((URL) this.fileSet.getFile(0).getLocation());
+  }
+
+  /**
+   *  Text documents don't actually have more than one section (at present), so
+   *  this is a dummy function...
+   */
+  public String getSectionText(String sectionId)
+  { return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/doctypes/TextRecogniser.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.doctypes;
+
+import java.io.*;
+import java.net.*;
+import java.util.ArrayList;
+
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+
+public class TextRecogniser extends AbstractRecogniser
+{
+    
+    public TextRecogniser()
+    { 
+	this.preferred_mime_type = "text/plain";
+	this.filename_extensions = new ArrayList();
+	this.filename_extensions.add(".txt");
+	this.filename_extensions.add(".text");
+	this.document_type = TextDocument.TEXT_DOCUMENT_TYPE;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ExpanderInterface.java	(revision 21112)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.File;
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public interface ExpanderInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure(BuildManager manager);
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>URL</code> the name of the file, or its URL
+	 *  @param <code>File</code> the directory to expand files into - the expander
+	 *         should create its own sub-directory to ensure that no later conflicts
+	 *         emerge.
+	 *
+	 *  @return <code>boolean</code> whether the file was expanded.
+	 */
+	public boolean expandFile(URL fileReference, File expansionDirectory);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/expander/ZipExpander.java	(revision 21112)
@@ -0,0 +1,81 @@
+package org.greenstone.gsdl3.gs3build.expander;
+
+import java.io.*;
+import java.net.URL;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.BuildManager;
+
+public class ZipExpander implements ExpanderInterface
+{ BuildManager buildManager;
+
+	public ZipExpander()
+	{
+	}
+
+	public void configure(BuildManager manager)
+	{	this.buildManager = manager;
+	}
+
+	public boolean expandFile(URL url, java.io.File expansionDirectory)
+	{ if (url.toString().startsWith("file://")) 
+		{	String fileName = url.toString().substring(7);
+
+			if (fileName.endsWith(".zip"))
+		  {	System.out.println("Expanding Zip file " + fileName);
+
+				// determine which directory to use
+
+				// get the leaf name
+				String leafName;
+				int leafAt = fileName.lastIndexOf(System.getProperty("file.separator"));
+				
+				if (leafAt >= 0)
+				{ leafName = fileName.substring(leafAt+1);
+				}
+				else
+				{	leafName = fileName;
+				}
+
+			  int extensionAt = leafName.lastIndexOf('.');
+				if (extensionAt >= 0)
+				{	leafName = leafName.substring(0, extensionAt);
+				}
+
+				File targetDir = new File(expansionDirectory, leafName);
+				if (targetDir.exists())
+				{	// find alternative name
+					int  extension = 1;
+					File expandedTarget;
+
+					do
+					{ expandedTarget = new File(targetDir.toString() + Integer.toString(extension));
+					} while (expandedTarget.exists());
+
+					targetDir = expandedTarget;
+				}
+				if (!targetDir.mkdir())
+				{	// TODO: pass/log error - unable to make directory to
+					//       expand file.
+					return false;
+				}
+
+				// Call zip expansion
+				try
+				{
+					Runtime.getRuntime().exec("unzip -d "+targetDir.toString());
+				}
+				catch (IOException io)
+				{ // TODO: handle errors - report to log, etc.
+					return false;
+				}
+
+				// ensure that the build manager knows about the new directory,
+				// and will expand any content in it, etc. in turn
+				this.buildManager.addInputDirectory(targetDir.toString());
+				return true;
+			}
+		}
+		return false;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorInterface.java	(revision 21112)
@@ -0,0 +1,15 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public interface ExtractorInterface
+{
+  public void configure(String outputDir);
+  public void configure(DocumentList list);
+  public void startPass(int passNumber);
+  public void extractDocument(DocumentID documentID, DocumentInterface document);
+  public void endPass(int passNumber);
+  public int getNumberOfPasses();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/ExtractorManager.java	(revision 21112)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.util.Iterator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class ExtractorManager
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  DocumentList          documents;
+  ExtractorInterface [] list;
+  int                   size;
+  int                   used;
+
+  public ExtractorManager(DocumentList documentList)
+  { this.list = new ExtractorInterface[10];
+    this.size = 10;
+    this.used = 0;
+    this.documents = documentList;
+  }
+
+  public void addExtractor(ExtractorInterface extractor)
+  { this.ensureSize(this.used + 1);
+    this.list[this.used] = extractor;
+    this.used ++;
+
+    extractor.configure(this.documents);
+  }
+
+  public void extractDocument(DocumentID docId, DocumentInterface document)
+  { for (int i = 0; i < this.used; i ++)
+    { this.list[i].extractDocument(docId, document);
+    }
+  }
+
+  public void extractDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.list[i].getNumberOfPasses(); p ++)
+      {	this.list[i].startPass(p);
+
+        Iterator iterator = documents.iterator();
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+	  this.list[i].extractDocument(document.getID(), document);
+	}
+	this.list[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureSize(int size)
+  { while (size >= this.size)
+    { ExtractorInterface newList [] = new ExtractorInterface[this.size*2];
+      this.size *= 2;
+      System.arraycopy(this.list, 0, newList, 0, this.size);
+      this.list = newList;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/GMLExtractor.java	(revision 21112)
@@ -0,0 +1,167 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class GMLExtractor implements ExtractorInterface
+{
+  public static final String ACCUMULATE_MODE = "accumulate";
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class GMLHandler extends DefaultHandler
+  { List   files;
+    String label;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+
+    GMLHandler()
+    { super();
+    
+      this.label = null;
+      this.value = null;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+      }
+      else if (localName.equals("Metadata"))
+      { GMLExtractor.postMetadata(this.files, this.label, this.value.toString());
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+  }
+
+  /**
+   *  Construct of extractor
+   */
+  public GMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList documentList)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  {	
+    if (document.getDocumentType().equals(GMLDocument.GML_DOCUMENT_TYPE))
+    { // Extract the content from the GML file
+      try {
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	GMLHandler handler = new GMLHandler();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);
+	  
+	// A GML document consists of one file only - get it from the 'default'
+	// file group
+	FileReader fileReader = new FileReader(document.getDocumentFiles().getFile(0).toString());
+	reader.parse(new InputSource(fileReader));
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+      }
+      catch (java.io.FileNotFoundException fileException)
+      {
+      }
+      catch (java.io.IOException ioException)
+      {
+      }
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(List files, String value, String label)
+  {
+    
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/IndexExtractor.java	(revision 21112)
@@ -0,0 +1,256 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.IndexDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class IndexExtractor implements ExtractorInterface
+{
+  class IndexHandlerException extends Exception
+  { public IndexHandlerException(String value)
+    { super(value);
+    }
+  }
+
+  /**
+   *  An inner class to handle GML files
+   */
+  class IndexHandler extends GS2TextFileHandler
+  { List    labels;
+    URL     base;
+
+    IndexHandler(String content, URL url, DocumentList documentList) throws IndexHandlerException
+    { 
+      super(content);
+
+      this.labels  = new ArrayList();
+      this.base    = url;
+
+      String parentDir;
+      int leaf = this.base.toString().lastIndexOf('/');
+      if (leaf >= 0) {
+	parentDir = this.base.toString().substring(0, leaf+1);
+      }
+      else {
+	parentDir = this.base.toString();
+      }
+
+      // get the first line
+      this.getLine();
+
+      if (!this.hasMore())
+      { throw new IndexHandlerException("No title line");
+      }
+      
+      // get the first totem - it should be "key:"
+      String entry = this.getEntry(true);
+      
+      // now get all the labels
+      while (this.hasMore())
+      { String label = this.getEntry(true);
+        if (label == null || label.length() == 0) {
+	  continue;
+	}
+	
+	this.labels.add(label);
+	System.out.println("Adding label: " + label);
+      }
+
+      while (this.hasMoreLines()) {
+	this.getLine();
+
+	// Get the file pattern itself
+	String filePattern = this.getEntry(true);
+	if (filePattern == null || filePattern.length() == 0) {
+	  continue;
+	}
+
+	// get a list of documents that match the file pattern
+	List documentIds = documentList.findDocumentIdsUsingFile(filePattern);
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+
+	// if no files match this data, then skip this row
+	// TODO: raise a quality error message
+	if (documentIds == null || documentIds.size() == 0) {
+	  continue;
+	}
+
+	// cache up the documents that match for speed improvements...
+	List documents = new ArrayList();
+	Iterator idIterator = documentIds.iterator();
+	while (idIterator.hasNext()) {
+	  String docIdString = idIterator.next().toString();
+	  System.out.println(docIdString);
+	  DocumentID docId   = new DocumentID(docIdString);
+	  DocumentInterface document = documentList.getDocument(docId);
+	  if (document != null) {
+	    documents.add(document);
+	  }
+	}
+
+	// Next, split the row into the separate metadata items
+	int entryNo = 0;
+	while (this.hasMore()) {
+	  String item = this.getEntry(true);
+	  if (item == null || item.length() == 0) {
+	    entryNo ++;
+	    continue;
+	  }
+
+	  String label = null;
+	  if (item.startsWith("<")) {
+	    int labelEnd = item.indexOf('>');
+	    if (labelEnd >= 0) {
+	      label = item.substring(1, labelEnd);
+
+	      item = item.substring(labelEnd+1, item.length());
+
+	      // eliminate any weird whitespace
+	      item.trim();
+
+	      // cope with a solo 'item' label with no following string
+	      if (item.length() == 0) {
+		entryNo ++;
+		continue;
+	      }
+	    }
+	    // starts with a bracketed label
+	  }
+	  else if (entryNo < this.labels.size()) {
+	    label = (String) this.labels.get(entryNo);
+	  }
+
+	  // Actually post the metadata -
+	  // it may be good to have cached all the documents that we're going to change
+	  // in order to minimise rewrites...
+	  if (label != null) {
+	    Iterator docIterator = documents.iterator();
+	    while (docIterator.hasNext()) {
+	      DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	      // Post to document
+	      // TODO: tailor this to posting documents to *sections* as required...
+	      document.addDocumentMetadata(new MetadataLabel(label), item);
+	      System.out.println("Posting " + label + "=" + item + " to " + parentDir + filePattern);
+	    }
+	  }
+	  entryNo ++;
+	}
+	
+	// write out the modified documents
+	// TODO: nicer/more generalised interface for this and related activity in
+	//       extractor manager (actually, enricher manager);
+	Iterator docIterator = documents.iterator();
+	while (docIterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	  System.out.println("Writing modified document " + document.getID());
+	  documentList.storeChangedDocument(document);
+	}
+      }
+    }
+
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public IndexExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a GML document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(IndexDocument.INDEX_DOCUMENT_TYPE))
+    { // Extract the content from the index file
+      
+      // get the file
+      String documentText = 
+	DocumentLoader.getAsString(document.getDocumentFiles().getFile(0).getURL());
+	
+      if (documentText == null) {
+	System.err.println("IndexExtractor: Unable to load any content for " + document.getDocumentFiles().getFile(0).toString());
+	return;
+      }
+
+      try {
+	IndexHandler handler = new IndexHandler(documentText, document.getDocumentFiles().getFile(0).getURL(), this.documentList);
+      }
+      catch (IndexHandlerException ex) {
+      }
+      
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(String file, String value, String label)
+  {
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/extractor/MetaXMLExtractor.java	(revision 21112)
@@ -0,0 +1,268 @@
+package org.greenstone.gsdl3.gs3build.extractor;
+
+import java.io.FileReader;
+
+import java.net.URL;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.greenstone.gsdl3.gs3build.util.GS2TextFileHandler;
+
+import org.apache.xerces.parsers.SAXParser;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.MetadataDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+import org.greenstone.gsdl3.gs3build.metadata.MetadataLabel;
+
+public class MetaXMLExtractor implements ExtractorInterface
+{
+  /**
+   *  An inner class to handle Metadata files
+   */
+  class MetadataHandler extends DefaultHandler
+  { List         files;
+    String       label;
+    StringBuffer value;
+    URL          url;
+    boolean      inElement;
+    boolean      accumulate;
+    DocumentList documentList;
+    List         documentIds;
+    List         documents;
+
+    MetadataHandler(DocumentList documentList)
+    { super();
+    
+      this.label = null;
+      this.value = null;
+      this.documentList = documentList;
+    }
+
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { if (localName.equals("FileName"))
+      {	this.value = new StringBuffer();
+      }
+      else if (localName.equals("FileSet"))
+      { this.files = new ArrayList();
+      }
+      else if (localName.equals("Description"))
+      { this.documentIds = this.documentList.findDocumentIdsUsingFiles(this.files, this.url.toString());	
+
+	if (documentIds != null && documentIds.size() > 0) {
+	  this.documents = new ArrayList();
+	  
+	  Iterator idIterator = documentIds.iterator();
+	  while (idIterator.hasNext()) {
+	    String docIdString = idIterator.next().toString();
+	    DocumentID docId   = new DocumentID(docIdString);
+	    DocumentInterface document = documentList.getDocument(docId);
+	    if (document != null) {
+	      documents.add(document);
+	    }
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { this.label = attributes.getValue("name");
+        this.value = new StringBuffer();
+
+	String mode = attributes.getValue("mode");
+	this.accumulate = mode.equals(ExtractorManager.ACCUMULATE_MODE);	
+      }
+    }
+
+    public void endElement(String URI, String localName, String qName)
+    { if (localName.equals("FileName"))
+      {	String file = this.value.toString();
+        this.value = null;
+	this.files.add(file);
+      }
+      else if (localName.equals("FileSet"))
+      { // post the existing files item...
+      }
+      else if (localName.equals("Description"))
+      {
+	if (this.documents != null && documents.size() > 0) {
+	  // write out the modified documents
+	  // TODO: nicer/more generalised interface for this and related activity in
+	  //       extractor manager (actually, enricher manager);
+	  Iterator docIterator = documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+
+	    //  System.out.println("Writing modified document " + document.getID());
+	    documentList.storeChangedDocument(document);
+	  }
+	}
+      }
+      else if (localName.equals("Metadata"))
+      { MetaXMLExtractor.postMetadata(this.url, this.files, 
+				      this.label, this.value.toString(), 
+				      this.accumulate);
+	/*
+	if (documentIds != null) {
+	  Iterator iterator = documentIds.iterator();
+	  while (iterator.hasNext()) {
+	    System.out.println("Matches file " + iterator.next().toString());
+	  }
+	}
+	*/
+	
+	if (documentIds != null && documentIds.size() > 0) {
+	  Iterator docIterator = this.documents.iterator();
+	  while (docIterator.hasNext()) {
+	    DocumentInterface document = (DocumentInterface) docIterator.next();
+	    
+	    // Post to document
+	    // TODO: tailor this to posting documents to *sections* as required...
+	    if (accumulate) {
+ 	      document.addDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	    else {
+ 	      document.setDocumentMetadata(new MetadataLabel(this.label), this.value.toString());
+	    }
+	  }	
+	}
+
+	// flatten the metadata items again...
+        this.value = null;
+	this.label = null;
+      }
+    }
+
+    public void characters(char c[], int start, int length)
+    { if (this.value != null)
+      { String string = new String(c, start, length);
+        this.value.append(string);
+      }
+    }
+
+    public void setUrl(URL url)
+    { this.url = url;
+    }
+  }
+
+  private DocumentList documentList;
+
+  /**
+   *  Construct of extractor
+   */
+  public MetaXMLExtractor()
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void configure(String outputDir)
+  { // Intentionally left blank
+  }
+
+  public void configure(DocumentList list)
+  { this.documentList = list;
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void startPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  Process the document - for a metadata document, this results in the
+   *  decoration of other files, for other documents, it does nothing.
+   */
+  public void extractDocument(DocumentID docID, DocumentInterface document)
+  { if (document.getDocumentType().equals(MetadataDocument.METADATA_DOCUMENT_TYPE))
+    { // Extract the content from the metadata file
+      URL url;
+
+      try {
+	SAXParser parser = new SAXParser();
+	MetadataHandler handler = new MetadataHandler(this.documentList);
+        /*
+	XMLReader reader = XMLReaderFactory.createXMLReader();
+	reader.setContentHandler(handler);
+	reader.setErrorHandler(handler);*/
+	parser.setContentHandler(handler);
+	  
+	// Get path of file; we cheat here by assuming that the url is a file - this
+	// really ought to be done better [TODO: fix to handle full paths & URLs]
+	url = document.getDocumentFiles().getFile(0).getURL();
+	String filePath = url.getPath();
+	handler.setUrl(new URL(url, "."));
+
+	// A metadata document consists of one file only - get it from the 'default'
+	// file group
+	/*
+	FileReader fileReader = new FileReader(filePath);
+	reader.parse(new InputSource(fileReader));
+	*/
+	parser.parse(filePath);
+      }
+      catch (SAXException saxException)
+      { // TODO: log error
+	System.err.println(saxException);
+      }
+      catch (java.io.FileNotFoundException fileException)
+      { System.err.println(fileException);
+      }
+      catch (java.io.IOException ioException)
+      { System.err.println(ioException);
+      }
+      /*      catch (java.net.MalformedURLException malEx) {
+	System.err.println("Unable to get parent of URL "+url.toString()+" in metadata extraction.");
+	return;
+      }
+      */
+
+      // for each document post it to the corresponding document
+    }
+  }
+
+  protected static void postMetadata(URL url, List files, String label, String value, boolean accumulate)
+  { String file;
+
+    Iterator fileIter = files.iterator();
+    while (fileIter.hasNext()) {
+      file = fileIter.next().toString();
+
+      System.out.println(url.toString() + " " + file + ": " + label + "=" + value);
+    }
+  }
+
+  /**
+   *  This extractor doesn't need to do any preparation/completion work,
+   *  so this member function is empty.
+   */
+  public void endPass(int passNo)
+  { // Intentionally left blank
+  }
+
+  /**
+   *  This extractor is a simple, single-pass extractor
+   *
+   *  @see: org.greenstone.gsdl3.gs3build.extractor.ExtractorInterface:getNumberOfPasses
+   */
+  public int getNumberOfPasses()
+  { return 1;
+  }
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/AbstractIndexer.java	(revision 21112)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+
+public abstract class AbstractIndexer implements IndexerInterface
+{ 
+  class CollectionIndex
+  { String level;
+    String name;
+    List fields;
+
+    public CollectionIndex(String indexName, Node node)
+    { this.fields = new ArrayList();
+      this.name = indexName;
+      this.level = IndexerManager.DEFAULT_LEVEL;
+    
+      NodeList children = node.getChildNodes();
+      for (int c = 0; c < children.getLength(); c ++) {
+	Node child = children.item(c);
+	
+	if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	  String name = child.getNodeName();
+	  
+	  if (name.equals(GSXML.LEVEL_ELEM)) {
+	    String levelName = DOMUtils.getNodeChildText(children.item(c));
+	    this.level = levelName;
+	  }
+	  else if (name.equals(GSXML.FIELD_ELEM)) {
+	    String fieldName = DOMUtils.getNodeChildText(children.item(c));
+	    this.fields.add(fieldName.toString());
+	  }
+	}
+      }
+      if (this.fields.size() == 0) {
+	this.fields.add(IndexerManager.DEFAULT_FIELD);
+      }
+    }
+
+    public void prepareIndexer(IndexerInterface indexer)
+      { //indexer.addIndex(this.name, this.level, this.fields.get(0).toString());
+    }
+  }
+
+
+  public boolean configure(org.w3c.dom.Node searchNode)
+  {
+    NodeList indexChildren = searchNode.getChildNodes();
+
+    // check for <index> tag children
+    for (int i = 0; i < indexChildren.getLength(); i ++) {
+      String childName = indexChildren.item(i).getNodeName();
+
+      if (childName.equals(GSXML.INDEX_ELEM))
+      { NamedNodeMap atts = indexChildren.item(i).getAttributes();
+	
+	Node attribute = atts.getNamedItem(GSXML.NAME_ATT);
+	String indexName = attribute.getNodeValue();
+	      
+	// create the local indexer representation
+	CollectionIndex index = new CollectionIndex(indexName, indexChildren.item(i));
+	     
+	// prepare it...
+	index.prepareIndexer(this);
+      }
+    }
+    return true;
+  }
+
+  public void tidyup()
+  {
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerFactory.java	(revision 21112)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+public class IndexerFactory
+{ public static final IndexerInterface makeIndexer(String type, String name)
+  { if (type.equals(MGIndexer.MG_INDEX_TYPE)) 
+    { return new MGIndexer(name);
+    }
+    else if (type.equals(MGPPIndexer.MGPP_INDEX_TYPE)) {
+      return new MGPPIndexer(name);
+    }
+    return null;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerInterface.java	(revision 21112)
@@ -0,0 +1,29 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+public interface IndexerInterface
+{
+  public static final String DOCUMENT_LEVEL = "document";
+  public static final String SECTION_LEVEL = "section";
+  public static final String PARAGRAPH_LEVEL = "paragraph";
+
+  public static final String GS2_INDEX_LABEL = "Index";  
+
+  public String  getIndexType();
+  public String  getName();
+  public int     getNumberOfPasses();
+  // TODO: add a function call to provide appropriate XML for the service description
+  public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list);
+  public void    tidyup();
+
+  public boolean configure(String label, String value);
+  public boolean configure(org.w3c.dom.Node node);
+    //  public boolean addIndex(String name, String level, String field);
+
+  public boolean startPass(int passNumber);
+  public boolean indexDocument(DocumentID id, DocumentInterface document);
+  public boolean endPass(int passNumber);
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/IndexerManager.java	(revision 21112)
@@ -0,0 +1,100 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+
+public class IndexerManager
+{
+  IndexerInterface indexers[];
+  int size;
+  int used;
+  DocumentList documentList;
+
+  public static final String outputDir = "outputDir";
+  public static final String DEFAULT_LEVEL = "document";
+  public static final String DEFAULT_FIELD = "text";
+
+  public IndexerManager(DocumentList documentList)
+  { this.indexers  = new IndexerInterface[10];
+    this.size      = 10;
+    this.used      = 0;
+    this.documentList = documentList;
+  }
+
+  public void addIndexer(IndexerInterface indexer)
+  { this.ensureCapacity(this.used + 1);
+
+    this.indexers[this.used] = indexer;
+    this.used ++;
+  }
+
+  public void indexDocument(DocumentID docId, DocumentInterface document)
+  { if (!document.isIndexed())
+      return;
+	  
+    for (int i = 0; i < this.used; i ++)
+    { this.indexers[i].indexDocument(docId, document);
+    }
+  }
+
+  public void indexDocuments()
+  { for (int i = 0; i < this.used; i ++)
+    { for (int p = 0; p < this.indexers[i].getNumberOfPasses(); p ++)
+      {	
+	if (!this.indexers[i].startPass(p)) {
+	  continue;
+	}
+
+	Iterator iterator = this.documentList.iterator();
+
+	while (iterator.hasNext()) {
+	  DocumentInterface document = (DocumentInterface) iterator.next();
+
+	  if (document.isIndexed()) {
+	    if (!this.indexers[i].indexDocument(document.getID(), document)) {
+	      System.out.println("Ending document " + document.getID());
+	    }
+	    
+	    // note any changes made to this document...
+	    if (document.isChanged()) {
+	      //System.out.println("Writing document "+document.getID());
+	      this.documentList.storeChangedDocument(document);
+	    }
+	    /**
+	    if (p == 0) {
+	      System.out.println("Writing document "+document.getID());
+	    }
+	    */
+	  }
+	}
+	this.indexers[i].endPass(p);
+      }
+    }
+  }
+
+  public void ensureCapacity(int needSize)
+  { while (needSize >= this.size)
+    { int newSize = this.size * 2;
+      IndexerInterface newIndexers [] = new IndexerInterface[newSize];
+      System.arraycopy(indexers, 0, newIndexers, 0, this.used);
+      this.indexers = newIndexers;
+      this.size = newSize;
+    }
+  }
+
+    public boolean addServiceDescriptions(Element service_class_list) {
+	boolean success = true;
+	for (int i = 0; i < this.used; i ++) {
+	    if (!this.indexers[i].addServiceDescriptions(service_class_list)) {
+		success = false;
+	    }
+	}
+	return success;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGIndexer.java	(revision 21112)
@@ -0,0 +1,740 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.w3c.dom.*;
+
+import org.greenstone.mg.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.util.DOMUtils;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.Processing;
+
+// for debug
+import org.greenstone.gsdl3.util.XMLConverter;
+
+public class MGIndexer //extends AbstractIndexer
+    implements IndexerInterface
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    boolean      firstDocument;
+    String       outputDirectory;
+    StringBuffer indexBuffer;
+    File         textDirectory;
+    File         indexDirectory;
+    String       indexStem;
+    String       textStem;
+    List         indexes;
+    String       overallName; 
+    
+    MGIndex current_index = null;
+    MGPassesWrapper mgPasses;
+    
+    static final char END_OF_DOCUMENT = (char) 2;
+    //static final char END_OF_SECTION  = (char) 3; // actually this is end of para for mg - don't use this yet
+    static final char END_OF_STREAM   = (char) 4;
+
+    public static final String MG_INDEX_TYPE = "mg";
+    public static final String INDEX_FILE_STEM = "index";
+    
+    class MGIndex
+    { 
+	String name=null;
+	String level=null;
+	List fields=null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGIndex(Element index_element) {
+	    
+	    this.fields = new ArrayList();
+	    this.name = index_element.getAttribute(GSXML.NAME_ATT);
+	    if (this.name.equals("")) {
+		// TODO make this dynamic
+		this.name = "xx";
+	    }
+	    NodeList children = index_element.getChildNodes();
+	    for (int c = 0; c < children.getLength(); c ++) {
+		Node child = children.item(c);
+		
+		if (child.getNodeType() == Node.ELEMENT_NODE) {
+		    String name = child.getNodeName();
+	  
+		    if (name.equals(GSXML.LEVEL_ELEM)) {
+			this.level = DOMUtils.getNodeChildText(children.item(c));
+		    }
+		    else if (name.equals(GSXML.FIELD_ELEM)) {
+			String fieldName = DOMUtils.getNodeChildText(children.item(c));
+			this.fields.add(fieldName);
+		    }
+		}
+	    }
+	}
+
+	public MGIndex(String name, String level, List fields)
+	{ 
+	    this.name  = name;
+	    this.level = level;
+	    this.fields = fields;
+	}
+	
+	// old gs2 style config - can we get rid of this??
+	public MGIndex(String indexLabel)
+	{ 
+	    int colonAt = indexLabel.indexOf(':');
+	    
+	    if (colonAt < 0) {
+		System.err.println("MGIndex(): invalid index specification: "+indexLabel);
+		return;
+	    }
+	    String field_string = indexLabel.substring(colonAt+1);
+	    String [] field_list = field_string.split(",");
+	    this.fields = new ArrayList();
+	    for (int i=0; i<field_list.length; i++) {
+		this.fields.add(field_list[i]);
+	    }
+	    this.level = indexLabel.substring(0, colonAt);
+	    //createIndexName
+	    StringBuffer new_name = new StringBuffer();
+	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+	    int c, w;
+	    w = 0;
+	    c = 0;
+	    while (c < field_string.length() && w < 2) {
+		char ch = field_string.charAt(c);
+		
+		ch = Character.toLowerCase(ch);
+		if (Character.isLetter(ch)) {
+		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+			ch != 'o' && ch != 'u') {
+			new_name.append(ch);
+			w++;
+		    }
+		}
+		c ++;
+	    }
+	    this.name = new_name.toString();
+	}
+
+	
+	public String getLevel()
+	{ 
+	    return this.level;
+	}
+
+	public List getFields()
+	{ 
+	    return this.fields;
+	}
+
+	public String getName()
+	{ 
+// 	    if (this.name==null || this.name.equals("")) {
+// 		createIndexName();
+// 	    }
+	    return this.name;
+	}
+
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+// 	private void createIndexName() {
+// 	    StringBuffer new_name = new StringBuffer();
+// 	    new_name.append(Character.toLowerCase((char) this.level.charAt(0)));
+
+// 	    int c, w;
+// 	    w = 0;
+// 	    c = 0;
+// 	    String [] fields_concat = this.fields.toArray
+// 	    while (c < this.field.length() && w < 2) {
+// 		char ch = this.field.charAt(c);
+
+// 		ch = Character.toLowerCase(ch);
+// 		if (Character.isLetter(ch)) {
+// 		    if (ch != 'a' && ch != 'e' && ch != 'i' &&
+// 			ch != 'o' && ch != 'u') {
+// 			new_name.append(ch);
+// 			w++;
+// 		    }
+// 		}
+// 		c ++;
+// 	    }
+// 	    this.name = new_name.toString();
+// 	}
+ 
+
+    } // MGIndex
+
+    public MGIndexer(String name)
+    { 
+	this.indexes = new ArrayList();
+	this.overallName = name; 
+    }
+
+    public String getIndexType()
+    { 
+	return MG_INDEX_TYPE;
+    } 
+
+    public String getName()
+    { 
+	return this.overallName; 
+    }
+    
+    public boolean configure(Node search_node)
+    {
+	NodeList index_children = GSXML.getChildrenByTagName(search_node, GSXML.INDEX_ELEM);
+	
+	// add a text 'index' - we should be able to turn this off in the config file? actually mg needs a text index
+	ArrayList list = new ArrayList();
+	list.add("text");
+	MGIndex index = new MGIndex("text", "section", list);
+	indexes.add(index);
+	for (int i = 0; i < index_children.getLength(); i ++) {
+	    Element index_elem = (Element)index_children.item(i);
+	    index = new MGIndex(index_elem);
+	    if (index.getName() != null && index.getLevel() != null && index.getFields()!= null) {
+		
+		indexes.add(index);
+	    } else {
+		System.err.println("invalid index spec, not including"+new XMLConverter().getPrettyString(index_elem));
+	    }
+	}
+	// TODO make sure all index names are unique
+	return true;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds.
+     *
+     *  @param <code>String</code> the label to configure
+     *  @param <code>String</code> the value...
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    this.pass = 0;
+    
+	    // attempt to ensure that the text subdirectory exists
+	    this.textDirectory = new File(outputDirectory, "text");
+	    if (!textDirectory.exists()) {
+		if (!textDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!textDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+
+	    // Sign to the user which mg directory is being used...
+	    System.out.println("Output MG directory is " + this.textStem);
+	}
+	else if (label.equals(IndexerInterface.GS2_INDEX_LABEL)) {
+	    this.indexes.add(new MGIndex(value));
+	}
+
+	return true;
+    }
+
+
+    private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			    AbstractStructure structure, StringBuffer textBuffer,
+			    StringBuffer extraBuffer, String namespace) 
+	//String name, String namespace, String field)
+    {
+	List fields = current_index.getFields();
+	// send out the ctrl-c...if this is
+	if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	    // try doing this for all index types
+	    // actually we should only need to do this once ????
+	    if (this.pass == 0) {
+	    //if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+		METSDivision division = (METSDivision) structure;
+
+		// get the division metadata block
+		METSDescriptive descriptive;
+		String metadataId = division.getDefaultMetadataReference();
+		if (metadataId == null) {
+		    descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		    division.addMetadataReference(descriptive.getID());
+		}
+		else {
+		    // Get the descriptive item...
+		    descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		}
+	    
+		descriptive.addMetadata("gsdl3", "mgseqno", this.overallName + "." + Integer.toString(this.sectionSeqNo));
+	    
+		metsDoc.setChanged(true);
+		//metsDoc.setModified(true);
+		//	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	    } // section level
+	
+	    // append an 'end of section' marker
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+	
+	    // for document-level indexes, always append an 'end of document' tag at the
+	    // end of the document for each section.  Otherwise, each section is followed
+	    // by an end of document character.  This ensures that all indexes use the
+	    // same document numbering...
+	    if (this.current_index.getLevel().equals(IndexerInterface.DOCUMENT_LEVEL)) {
+		extraBuffer.append(END_OF_DOCUMENT);
+	    }
+	    else {
+		textBuffer.append(END_OF_DOCUMENT);
+		this.documentSeqNo ++;
+	    }
+	    
+	    // produce the body here for metadata output of divisions - in the case of
+	    // text output, that will happen below...
+	    
+	    if (fields.size()>1 || !((String)fields.get(0)).equals("text")) {
+		// if there is only text, don't do this
+		METSDescriptive descriptive;
+	
+		METSDivision division = (METSDivision) structure;
+
+		String metadataId = division.getDefaultMetadataReference();
+		// are there other metadata refs to get??
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+		if (descriptive != null) {
+		    for (int i=0; i<fields.size(); i++) {
+			String field = (String)fields.get(i);
+			if (field.equals("text")) {
+			    continue;
+			}
+			List values = descriptive.getMetadata(namespace, field);
+			if (values != null) {	
+			    Iterator valueIter = values.iterator();
+			    while (valueIter.hasNext()) {
+				String value = valueIter.next().toString();
+				textBuffer.append(value);
+				textBuffer.append(" ");
+			    }
+			}
+		    }
+		}
+	    }
+	}
+	
+	// go through our children as required...
+	Iterator children = structure.getChildIterator();
+	Node startNode;
+	boolean index_text = fields.contains("text");
+	while (children.hasNext()) {
+	    AbstractStructure child = (AbstractStructure) children.next();
+      
+	    // get xpointer for child 
+	    // get start position node
+	    if (metsDoc.getDocumentType() == "METS"){
+		startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+	    } else {
+		startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+	    }
+	    
+	    // while this node isn't the child's start node, produce the 
+	    // HTML node text, if in text field mode...
+	    if (index_text) {
+		while (node != startNode) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+	    
+	    // recurse to child
+	    node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+	} // while next child
+
+	// close a document - the actual closing \B will be done by the main
+	// loop, so only a required \C is printed here...
+	// why have we got STRUCTURE_TYPE here and DIVISION_TYPE above????
+	if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+	    if (index_text) {
+		while (node != null) {
+		    XPointer.printNode(node, textBuffer, false);
+		    node = XPointer.getNextNode(node);
+		}
+	    }
+      
+	    //textBuffer.append(END_OF_SECTION);
+	    this.sectionSeqNo ++;
+      
+	}
+	return node;
+    }
+
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    {
+	//  String name, String namespace, String field)
+	StringBuffer extraBuffer = new StringBuffer();
+	Node node = document.getDocumentElement();
+	StringBuffer textBuffer = new StringBuffer();
+	
+	this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
+	textBuffer.append(extraBuffer.toString());
+	return textBuffer.toString();
+    }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	
+	if (!this.firstDocument) {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    
+	}
+
+	String docText = null;
+	// set the mgseqno if first pass
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	    document.addDocumentMetadata("gsdl3", "mgseqno", this.overallName+"."+Integer.toString(this.sectionSeqNo));
+	}
+
+	this.sectionSeqNo ++;
+	
+	//long start = System.currentTimeMillis();
+	Document domDocument = document.getDOMDocument();
+	if (domDocument != null) {
+	    System.err.println("dom doc is not null");
+	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
+	    if (sections != null) {
+		System.err.println("sections are not null");
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		//	System.out.println(docText);
+	    }
+	}
+	//long finish = System.currentTimeMillis();
+	//System.err.println("dom doc = "+ Long.toString(finish-start));
+	//start = System.currentTimeMillis();
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    StringBuffer doc_text_buffer = new StringBuffer();
+	    List fields = this.current_index.getFields();
+	    for (int i=0; i<fields.size(); i++) {
+		String field = (String)fields.get(i);
+		if (field.equals("text")) {
+		    doc_text_buffer.append(document.getDocumentText());
+		}  else {
+		    // its a metadata - do namespace properly!!
+		    List values = document.getDocumentMetadataItem("gsdl3", field);
+		    if (values != null) {
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    doc_text_buffer.append(value);
+			}
+		    }
+		}
+	    } // for each field
+	    docText = doc_text_buffer.toString();
+	    sectionSeqNo ++;
+	}
+	//finish = System.currentTimeMillis();
+	//System.err.println("whole doc = "+ Long.toString(finish-start));
+	
+	this.indexBuffer.append(docText);
+	// remember that we're not on the first document, 
+	this.firstDocument = false;
+	this.documentSeqNo ++;
+
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber) {
+    	
+	
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo  = 1;
+	
+	this.mgPasses = new MGPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	int indexNo = this.pass/2;
+	this.current_index = null;
+	
+	this.current_index = (MGIndex) this.indexes.get(indexNo);
+	if (this.current_index.hasError()) {
+	    // an error has already occurred for this index, don't continue
+	    System.out.println("pass "+this.pass+": aborted due to errors in the previous pass");
+	    return false;
+	}
+	// attempt to ensure that the text/index subdirectory exists
+	this.indexDirectory = new File(outputDirectory, current_index.getName());
+	if (!indexDirectory.exists()) {
+	    if (!indexDirectory.mkdir()) {
+		return false;
+	    }
+	}
+	else if (!indexDirectory.isDirectory()) {
+	    return false;
+	}
+
+	this.indexStem = this.indexDirectory.getPath() + File.separatorChar + INDEX_FILE_STEM;  // TODO: modify for index
+	if (this.pass == 0) {
+	    // first pass, also set up the textStem
+	    this.textDirectory = this.indexDirectory;
+	    this.textStem = this.indexStem;
+	}
+	mgPasses.setFileName(this.indexStem);
+	if (!Misc.isWindows()) {
+	    mgPasses.setBasePath("/");
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	
+	mgPasses.setBufferSize(100000);
+ 	
+	switch (mgPass) {
+	case 0:
+	    // -b 100000 -T1
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_1);
+	    
+	    
+	    break;
+	    
+	case 1:
+	    // -b 100000 -T2
+	    mgPasses.addPass(MGPassesWrapper.TEXT_PASS_2);
+	    break;
+	    
+	case 2:
+	    // -b 100000 -2 -m 32 -s 0 -G -t 10 -N1
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_1);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.setStemOptions(MGPassesWrapper.STEMMER_ENGLISH, MGPassesWrapper.NO_STEM_OR_CASE);
+	    mgPasses.setInversionMemLimit(32);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	    
+	case 3:
+	    // -b 100000 -2 -c 3 -G -t 10 -N2
+	    mgPasses.addPass(MGPassesWrapper.INDEX_PASS_2);
+	    mgPasses.setInvfLevel(MGPassesWrapper.INVF_LEVEL_2);
+	    mgPasses.ignoreSGMLTags(true);
+	    break;
+	}
+	
+	mgPasses.init();
+	System.out.println("Starting Pass " + this.pass);
+	return true;
+    }
+  
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber) {
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	mgPasses.finish();
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = 0;
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    //assume something has gone wrong, don't continue
+	    current_index.setError(true);
+	    return false;
+	}
+	int mgPass = this.pass < 2 ? this.pass : ((this.pass % 2) + 2);
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+	
+	switch (mgPass) {
+	    
+	case 0:
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mg_compression_dict -f " + this.textStem + osextra + " -S -H -2 -k 5120");
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mg_compression_dict: " + exit_value);
+		current_index.setError(true);
+		
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mg_perf_hash_build -f " + this.indexStem+osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		current_index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mg_weights_build -f " + this.indexStem + " -t " + this.textStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mg_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s1 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s2 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		current_index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mg_stem_idx -b 4096 -s3 -f " + this.indexStem+osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		current_index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+
+	mgPasses = null;
+	return true;
+    }
+    
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	//return 2 + this.indexes.size() * 2;
+	return this.indexes.size()*2;
+    }
+
+    public boolean addServiceDescriptions(org.w3c.dom.Element service_rack_list)
+    {
+	Document doc = service_rack_list.getOwnerDocument();
+
+	// generate the list of indexes
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	boolean found_index = false;
+	String def_index = ""; // the default index will just be the first one created for now.
+	for (int i=1; i<this.indexes.size(); i++) {
+	    // start at 1: 0 will be the text index
+	    MGIndex index = (MGIndex)this.indexes.get(i);
+	    if (!index.hasError()) {
+		Element e = doc.createElement(GSXML.INDEX_ELEM);
+		e.setAttribute(GSXML.NAME_ATT, index.getName());
+		index_list.appendChild(e);
+		if (found_index == false) {
+		    // this is the first index
+		    found_index = true;
+		    def_index = index.getName();
+		}
+	    }
+	}
+	
+	if (!found_index) {
+	    // no indexes were able to be created, so we can't use them or the text
+	    return false;
+	}
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, overallName);
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, INDEX_FILE_STEM);
+	
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	service_rack_list.appendChild(search_service_elem);
+	service_rack_list.appendChild(retrieve_service_elem);
+
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+	
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGRetrieve");
+	retrieve_service_elem.appendChild(default_index.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+    
+}
+    
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/indexers/MGPPIndexer.java	(revision 21112)
@@ -0,0 +1,732 @@
+package org.greenstone.gsdl3.gs3build.indexers;
+
+import org.greenstone.mgpp.MGPPPassesWrapper;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocument;
+import org.greenstone.gsdl3.gs3build.doctypes.METSDocument;
+import org.greenstone.gsdl3.util.Misc;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.Processing;
+import org.greenstone.gsdl3.gs3build.xpointer.XPointer;
+import org.greenstone.gsdl3.gs3build.metadata.*;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.io.File;
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Document;
+
+public class MGPPIndexer extends AbstractIndexer
+{ 
+    int          pass;
+    int          documentSeqNo;
+    int          sectionSeqNo;
+    String       name;
+    boolean      firstDocument;
+    File indexDirectory;
+    File textDirectory;
+    String       indexStem;
+    String       textStem;
+    StringBuffer indexBuffer;
+    String       outputDirectory;
+    //String       outputStem;
+//     String       passExtra;
+//     InputStream	 indexerFeedback;
+//     InputStream  indexerErrors;
+//     OutputStream indexerTextfeed;
+//     Process      mgpp_passes;
+    //String       overallName; 
+    String       currentIndexName;
+    String       currentIndexLevel;
+       String       currentIndexField; 
+    MGPPPassesWrapper mgppPasses;
+    
+
+    static final String documentSeparator = "<Document>";
+    static final String sectionSeparator = "<Section>";
+
+    static final String START_OF_DOCUMENT = "<Document>";
+    static final String END_OF_DOCUMENT = "</Document>";
+    static final String START_OF_SECTION = "<Section>";
+    static final String END_OF_SECTION = "</Section>";
+    
+    
+    public static final String MGPP_INDEX_TYPE = "mgpp";
+    public static final String INDEX_FILE_STEM = "index";
+
+
+    class MGPPIndex
+    {
+	public String name = null;
+	public String doc_level = null; 
+	public ArrayList levels = null;
+	public ArrayList fields = null;
+	boolean error = false;// assume built until we get an error
+	
+	public MGPPIndex(String name) {
+	    this.name = name;
+	    doc_level = "Document";
+	}
+
+	public void setDocLevel(String doc_level) {
+	    this.doc_level = doc_level;
+	}
+
+	public void addLevel(String level) {
+	    this.levels.add(level);
+	}
+
+	// change to allow nested fields
+	public void addField(String field) {
+	    this.fields.add(field);
+	}
+
+	public boolean hasError() {
+	    return this.error;
+	}
+	public void setError(boolean b) {
+	    this.error = b;
+	}
+
+    }
+    
+    
+    public MGPPIndexer(String name)
+    { 
+	this.name = name;
+	//this.passExtra = "";
+    }
+
+    public String getName()
+    { 
+	return this.name;
+    }
+
+    /**
+     *  The output directory should be (collection)/building/text/ for
+     *  normal Greenstone builds
+     */ 
+    public boolean configure(String label, String value)
+    { 
+	if (label.equals(IndexerManager.outputDir)) {
+	    this.outputDirectory = value;
+	    
+	    // attempt to ensure that the text subdirectory exists
+	    this.textDirectory = new File(outputDirectory, "text");
+	    if (!textDirectory.exists()) {
+		if (!textDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!textDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.textStem = this.textDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+	    // attempt to ensure that the index subdir exists
+	    this.indexDirectory = new File(outputDirectory, "idx");
+	    if (!indexDirectory.exists()) {
+		if (!indexDirectory.mkdir()) {
+		    return false;
+		}
+	    }
+	    else if (!indexDirectory.isDirectory()) {
+		return false;
+	    }
+	    this.indexStem = this.indexDirectory.getPath() + File.separator + INDEX_FILE_STEM;
+	    
+	    // Sign to the user which mg directory is being used...
+	    System.out.println("Output MGPP text directory is " + this.textStem);
+	    System.out.println("Output MGPP index directory is " + this.indexStem); 
+	}
+	this.pass = 0;
+	return true;
+    }
+
+    public String getIndexType()
+    { 
+	return MGPP_INDEX_TYPE;
+    }
+
+    public boolean addIndex(String name, String level, String field)
+    { 
+// 	if (level == "doc_level") {
+// 	    passExtra = " -J " + level;
+// 	}
+// 	else {
+// 	    passExtra = " -K " + level;
+// 	}
+	return true;
+    }
+
+    /**
+     *  Index a single document; the document interface can be used to extract individual
+     *  metadata items etc. as required or desired and index those instead or as well as
+     *  the body text of the document.
+     */
+    public boolean indexDocument(DocumentID docID, DocumentInterface document)
+    {
+	if (this.pass == 0) {
+	    document.removeAllMetadata("gsdl3", "mgseqno");
+	}
+
+	// why do this at the start and not at the end???
+	if (!this.firstDocument) {
+	    // Send a '</Document>' at the end of the doc
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgppPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	}
+
+	String docText = null;
+	
+	//int startSeqNo = this.sectionSeqNo;
+	//this.sectionSeqNo ++;
+	int startSeqNo = this.documentSeqNo;
+
+	Document domDocument = document.getDOMDocument();
+	if (domDocument != null) {
+	    System.err.println("dom doc is not null");
+	    METSStructure sections = document.getDocumentStructure().getStructure("Section");
+	    if (sections != null) {
+		System.err.println("sections are not null");
+		docText = this.prepareDOM(document, domDocument, sections, "gsdl3"); //this.name, "gsdl3", this.field);
+		//	System.out.println(docText);
+	    }
+	}
+	if (docText == null) {
+	    System.err.println("dom doc or sections was null - asking for doc text");
+	    //if (this.currentIndexField.equals("text")) {
+		//docText = Character.toString(END_OF_DOCUMENT) + document.getDocumentText(); 
+		docText = document.getDocumentText();
+		//}
+// 	    else {
+// 		StringBuffer textBuffer = new StringBuffer();
+// 		//textBuffer.append(END_OF_DOCUMENT);
+// 		List values = document.getDocumentMetadataItem("gsdl3", this.currentIndexField);
+// 		if (values != null) {
+// 		    Iterator valueIter = values.iterator();
+// 		    while (valueIter.hasNext()) {
+// 			String value = valueIter.next().toString();
+			
+// 			textBuffer.append(value);
+// 			if (valueIter.hasNext()) {
+// 			    //textBuffer.append(END_OF_SECTION);
+// 			    //	      sectionSeqNo ++;
+// 			}
+// 		    }
+// 		}
+// 		else {
+// 		    textBuffer.append("No data");
+// 		}
+// 		docText = textBuffer.toString();
+// 	    }
+	    sectionSeqNo ++;
+	}
+
+	    //try {					
+	    //	this.indexerTextfeed.write(documentSeparator.getBytes(), 0, documentSeparator.getBytes().length);
+	    // }
+// 	    catch (IOException ex) {
+// 		System.out.println("Bad output on end of document" + ex);
+// 		ex.printStackTrace();
+// 		return false;
+// 	    }
+// 	}
+
+	this.indexBuffer.append(START_OF_DOCUMENT);
+	//String docText = document.getDocumentText();
+	this.indexBuffer.append(docText);
+	//int startSeqNo = this.documentSeqNo;
+	
+// 	byte [] bytes = docText.getBytes();
+// 	int pos = 0, end = bytes.length;
+	
+// 	try {
+// 	    while (pos < end) {
+// 		this.indexerTextfeed.write(bytes, pos, (end - pos > 512 ? 512 : end - pos));
+// 		pos = pos + 512;
+	
+// 		try {
+// 		    while (this.indexerFeedback.available() > 0) {
+// 			byte b[] = new byte[this.indexerFeedback.available()];
+// 			System.out.println("Feedback of " + this.indexerFeedback.available());
+// 			this.indexerFeedback.read(b);
+// 			System.out.println(b);
+// 		    }
+// 		}
+// 		catch (IOException ex) {
+		    
+// 		}
+
+
+// 		try {
+// 		    while (this.indexerErrors.available() > 0) {
+// 			byte b[] = new byte[this.indexerErrors.available()];
+// 			System.out.println("Feedback of " + this.indexerErrors.available());
+// 			this.indexerErrors.read(b);
+// 			System.out.println(new String(b));
+// 		    }
+// 		}
+// 		catch (IOException ex){
+		    
+// 		}
+// 	    }
+// 	}
+// 	catch (IOException ex) {
+// 	    System.out.println("Bad output during document write " + ex + " " + pos + " " + end);
+// 	    ex.printStackTrace();
+// 	    return false;
+// 	}
+	this.firstDocument = false;
+
+	if (this.pass == 0) {    
+	    document.addDocumentMetadata("gsdl3", "mgseqno", "dtx."+Integer.toString(startSeqNo));
+	}
+	this.documentSeqNo++;
+	
+// 	try {
+// 	    while (this.indexerErrors.available() > 0) {
+// 		char c = (char) this.indexerErrors.read();
+// 		System.out.println(c);
+// 	    }
+// 	    while (this.indexerFeedback.available() > 0) {
+// 		byte b[] = new byte[this.indexerFeedback.available()];
+// 		System.out.println("Feedback of " + this.indexerFeedback.available());
+// 		this.indexerFeedback.read(b);
+// 	    }
+// 	}
+// 	catch (IOException ex) {
+	    
+// 	}
+	return true;
+    }
+
+    /**
+     *  Initialise the pass: open required files, check status
+     */
+    public boolean startPass(int passNumber)
+    { 
+	this.pass = passNumber;
+	this.firstDocument = true;
+	this.documentSeqNo = 1;
+	this.sectionSeqNo = 1;
+
+	this.mgppPasses = new MGPPPassesWrapper();
+	this.indexBuffer = new StringBuffer();
+	
+	MGPPIndex index = null; // do something with this!!
+	
+	// get the parameters for this execution of mg_passes
+	mgppPasses.setFileName((this.pass < 2 ? this.textStem : this.indexStem ));
+	if (!Misc.isWindows()) {
+	    mgppPasses.setBasePath("/");
+	}
+	
+	mgppPasses.setDocumentTag("Document");
+	//mgppPasses.addLevelTag("Section");
+
+	this.currentIndexLevel = "Document";// index.getLevel();
+	this.currentIndexField = "text";//index.getField();
+	this.currentIndexName = "idx"; //index.getName();
+	    
+
+	switch (this.pass) {
+	case 0:
+	    // -T1
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_1);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem + " -T1");
+	    break;
+	    
+	case 1:
+	    // -T2
+	    mgppPasses.addPass(MGPPPassesWrapper.TEXT_PASS_2);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -T2");
+	    break;
+	    
+	case 2:
+	    // -I1
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_1);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I1");
+	    break;
+	    
+	case 3:
+	    //Process p = Runtime.getRuntime().exec("mgpp_perf_hash_build -f " + this.outputStem);
+	    //p.waitFor();
+	    // -I2
+	    mgppPasses.addPass(MGPPPassesWrapper.INDEX_PASS_2);
+	    //mgpp_passes = Runtime.getRuntime().exec("mgpp_passes " + passExtra + " -f " + this.outputStem +" -I2");
+	    break;
+	}
+	    
+	//this.indexerFeedback = mgpp_passes.getInputStream();
+	//   this.indexerErrors   = mgpp_passes.getErrorStream();
+	//   this.indexerTextfeed = mgpp_passes.getOutputStream();
+	//	}
+	//catch (IOException ex) {
+	//   System.out.println(ex);
+	//   ex.printStackTrace();
+	//   return false;
+	//}/	
+	//catch (InterruptedException ex) {
+	//   System.out.println(ex);
+	//   ex.printStackTrace();
+	//   return false;
+	//}
+	mgppPasses.init();
+	System.out.println("Pass " + this.pass);
+	return true;
+    }
+
+    /**
+     *  Complete a pass - reset file counters, close files, etc.
+     */
+    public boolean endPass(int passNumber)
+    { 
+	// TODO: end pass
+	Process p;
+	MGPPIndex index = null; // do something with this!!
+	try {
+	    this.indexBuffer.append(END_OF_DOCUMENT);
+	    mgppPasses.processDocument(indexBuffer.toString());
+	    this.indexBuffer.delete(0, this.indexBuffer.length());
+	    Thread.sleep(1000); // what for??
+	}
+	catch (InterruptedException ex) {
+	    System.out.println(ex);
+	}
+	mgppPasses.finish();
+	try {
+	    Thread.sleep(1000);
+	} catch (Exception e) {}
+	
+	int exit_value = mgppPasses.exitValue();
+	System.out.println("Pass " + this.pass + " completed with " + exit_value);
+	if (exit_value !=0) {
+	    //assume something has gone wrong, don't continue
+// 	    if (index != null) {
+// 		index.setError(true);
+// 		return false;
+// 	    }
+	}
+	
+	String osextra = "";
+	if (!Misc.isWindows()) {
+	    osextra = " -d / ";
+	}
+
+	switch (this.pass) {
+	case 0:
+	    //System.exit(1);
+	    System.out.println("Compressing dictionary");
+	    exit_value = Processing.runProcess("mgpp_compression_dict -f " + this.textStem + " -S -H -2 -k 5120"+ osextra);
+	    
+	    if (exit_value == 0) {
+		System.out.println("Compressed dictionary successfully written");
+	    } else {
+		System.err.println("Error from mgpp_compression_dict: " + exit_value);
+		//index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 2:
+	    System.out.println("Creating perfect hash");
+	    exit_value = Processing.runProcess("mgpp_perf_hash_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Perfect hashes completed");
+	    } else {
+		System.err.println("Unable to build the perfect hash");
+		//index.setError(true);
+		return false;
+	    }
+	    break;
+	    
+	case 3:
+	    System.out.println("Writing weights file");
+	    exit_value = Processing.runProcess("mgpp_weights_build -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Weights file successfully written");
+	    } else {
+		System.err.println("Unable to create weights file");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating inverted dictionary");
+	    exit_value = Processing.runProcess("mgpp_invf_dict -f " + this.indexStem + osextra);
+	    if (exit_value ==0) {
+		System.out.println("Inverted dictionary file successfully written");
+	    } else {
+		System.out.println("Unable to create inverted dictionary file");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    System.out.println("Creating Stem indexes");
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s1 -f " + this.indexStem +osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 1 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 1");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s2 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 2 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 2");
+		//index.setError(true);
+		return false;
+	    }
+	    exit_value = Processing.runProcess("mgpp_stem_idx -b 4096 -s3 -f " + this.indexStem + osextra);
+	    if (exit_value == 0) {
+		System.out.println("Stemmed index 3 successfully written");
+	    } else {
+		System.out.println("Unable to create stemmed index 3");
+		//index.setError(true);
+		return false;
+	    }
+	    
+	    break;
+	} // switch
+	
+	return true;
+    }
+	
+    /**
+     *  Do any tidying up
+     */
+    public void tidyup()
+    {
+    }
+    
+    /**
+     *  Return the number of passes required for this index.
+     */
+    public int getNumberOfPasses()
+    { 
+	return 4;
+    }
+
+    public boolean addServiceDescriptions(Element service_rack_list) {
+	Document doc = service_rack_list.getOwnerDocument();
+
+	// generate the list of indexes
+	Element index_list = doc.createElement(GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	Element e = doc.createElement(GSXML.INDEX_ELEM);
+	e.setAttribute(GSXML.NAME_ATT, "idx");
+	index_list.appendChild(e);
+	String def_index = "idx";
+	
+// 	boolean found_index = false;
+// 	String def_index = ""; // the default index will just be the first one created for now.
+// 	for (int i=0; i<this.indexes.size(); i++) {
+// 	    MGIndex index = (MGIndex)this.indexes.get(i);
+// 	    if (!index.hasError()) {
+// 		Element e = doc.createElement(GSXML.INDEX_ELEM);
+// 		e.setAttribute(GSXML.NAME_ATT, index.getName());
+// 		index_list.appendChild(e);
+// 		if (found_index == false) {
+// 		    // this is the first index
+// 		    found_index = true;
+// 		    def_index = index.getName();
+// 		}
+// 	    }
+// 	}
+	
+// 	if (!found_index) {
+// 	    // no indexes were able to be created, so we can't use them or the text
+// 	    return false;
+// 	}
+
+	Element f = doc.createElement(GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	
+	Element default_index = doc.createElement("defaultIndex");
+	default_index.setAttribute(GSXML.NAME_ATT, def_index);
+
+	Element base_index_name = doc.createElement("baseIndexPrefix");
+	base_index_name.setAttribute(GSXML.NAME_ATT, "dtx");  //overallName);
+
+	Element index_stem = doc.createElement("indexStem");
+	index_stem.setAttribute(GSXML.NAME_ATT, "index");
+
+	Element search_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element retrieve_service_elem = doc.createElement(GSXML.SERVICE_CLASS_ELEM);
+	Element default_level = doc.createElement("defaultLevel");
+	default_level.setAttribute(GSXML.NAME_ATT, "Document");
+
+	Element level_list = doc.createElement("levelList");
+	Element level = doc.createElement("level");
+	level.setAttribute(GSXML.NAME_ATT, "Document");
+	level_list.appendChild(level);
+
+	Element field_list = doc.createElement("fieldList");
+	Element field = doc.createElement("field");
+	field.setAttribute(GSXML.NAME_ATT, "ZZ");
+	field_list.appendChild(field);
+
+	service_rack_list.appendChild(search_service_elem);
+	service_rack_list.appendChild(retrieve_service_elem);
+
+	search_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPSearch");  
+	search_service_elem.appendChild(index_list);
+	search_service_elem.appendChild(default_index);
+	search_service_elem.appendChild(level_list);
+	search_service_elem.appendChild(default_level);
+search_service_elem.appendChild(field_list); // do we need this??
+	search_service_elem.appendChild(base_index_name);
+	search_service_elem.appendChild(index_stem);
+
+	retrieve_service_elem.setAttribute(GSXML.NAME_ATT, "GS3MGPPRetrieve");
+	retrieve_service_elem.appendChild(default_level.cloneNode(true));
+	retrieve_service_elem.appendChild(base_index_name.cloneNode(true));
+	retrieve_service_elem.appendChild(index_stem.cloneNode(true));
+
+	return true;
+    }
+ 
+ 
+  private Node recurseDOM(DocumentInterface metsDoc, Node node, 
+			  AbstractStructure structure, StringBuffer textBuffer,
+			  StringBuffer extraBuffer, String namespace) 
+			  //String name, String namespace, String field)
+  {
+    // send out the ctrl-c...if this is
+    if (structure.getStructureType().equals(METSDivision.DIVISION_TYPE)) {
+	// try doing this for all index types
+	if ((this.currentIndexName != null)) { // && this.level != null && this.level.equals(IndexerInterface.SECTION_LEVEL)) { //name.startsWith("s")) {
+	    METSDivision division = (METSDivision) structure;
+
+	    // get the division metadata block
+	    METSDescriptive descriptive;
+	    String metadataId = division.getDefaultMetadataReference();
+	    if (metadataId == null) {
+		descriptive = metsDoc.getDocumentMetadata().createDescriptive(division.getLabel());
+		division.addMetadataReference(descriptive.getID());
+	    }
+	    else {
+		// Get the descriptive item...
+		descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	    }
+	    
+	    descriptive.addMetadata("gsdl3", "mgseqno", this.name + "." + Integer.toString(this.sectionSeqNo));
+	    
+	    metsDoc.setChanged(true);
+	    //metsDoc.setModified(true);
+	    //	System.out.println("Assigning " + this.sectionSeqNo + " to " + metsDoc.getID() + " " + division.getLabel());
+	} // section level
+	
+	// append an 'end of section' marker
+	//textBuffer.append(END_OF_SECTION);
+	this.sectionSeqNo ++;
+	
+	// for document-level indexes, always append an 'end of document' tag at the
+	// end of the document for each section.  Otherwise, each section is followed
+	// by an end of document character.  This ensures that all indexes use the
+	// same document numbering...
+	if (this.currentIndexLevel == null ||
+	    this.currentIndexLevel.equals(IndexerInterface.DOCUMENT_LEVEL)) {
+	    extraBuffer.append(END_OF_DOCUMENT);
+	}
+	else {
+	    textBuffer.append(END_OF_DOCUMENT);
+	    this.documentSeqNo ++;
+	}
+	
+	// produce the body here for metadata output of divisions - in the case of
+      // text output, that will happen below...
+      if (!this.currentIndexField.equals("text")) 
+      { METSDescriptive descriptive;
+	
+	METSDivision division = (METSDivision) structure;
+
+	String metadataId = division.getDefaultMetadataReference();
+	    
+	descriptive = metsDoc.getDocumentMetadata().getDescriptiveById(metadataId);
+	if (descriptive != null) {
+	  List values = descriptive.getMetadata(namespace, this.currentIndexField);
+	  
+	  if (values != null) {	
+	    Iterator valueIter = values.iterator();
+	    while (valueIter.hasNext()) {
+	      String value = valueIter.next().toString();
+	      
+	      textBuffer.append(value);
+	      if (valueIter.hasNext()) {
+		  //textBuffer.append(END_OF_SECTION);
+	      }
+	    }
+	  }
+	}
+      }
+    }
+
+    // go through our children as required...
+    Iterator children = structure.getChildIterator();
+    Node startNode;
+    while (children.hasNext()) {
+      AbstractStructure child = (AbstractStructure) children.next();
+      
+      // get xpointer for child 
+      // get start position node
+      if (metsDoc.getDocumentType() == "METS"){
+	  startNode = ((METSDocument) metsDoc).getSectionStartNode((METSDivision) child);	  
+      } else {
+	  startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      }
+	    //Node startNode = ((HTMLDocument) metsDoc).getSectionStartNode((METSDivision) child);
+      
+      // while this node isn't the child's start node, produce the HTML node text, if
+      // in text field mode...
+      if (this.currentIndexField.equals("text")) {
+	while (node != startNode) {
+	  XPointer.printNode(node, textBuffer, false);
+
+	  // print buffer to node
+	  node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+	}
+      }
+      
+      // recurse to child
+      node = this.recurseDOM(metsDoc, node, child, textBuffer, extraBuffer, namespace); // name, namespace, field);
+    } // while next child
+
+    // close a document - the actual closing \B will be done by the main
+    // loop, so only a required \C is printed here...
+    if (structure.getStructureType().equals(METSStructure.STRUCTURE_TYPE)) {
+      while (node != null) {
+	if (this.currentIndexField.equals("text")) {
+	  XPointer.printNode(node, textBuffer, false);
+	}
+	node = XPointer.getNextNode(node, (this.currentIndexField.equals("text") ? textBuffer : null));
+      }
+      
+      //textBuffer.append(END_OF_SECTION);
+      this.sectionSeqNo ++;
+      
+    }
+    return node;
+  }
+
+    private String prepareDOM(DocumentInterface metsDoc, Document document, METSStructure structure, String namespace)
+    //  String name, String namespace, String field)
+  { StringBuffer extraBuffer = new StringBuffer();
+    Node node = document.getDocumentElement();
+    StringBuffer textBuffer = new StringBuffer();
+
+    this.recurseDOM(metsDoc, node, structure, textBuffer, extraBuffer, namespace); //name, namespace, field);
+    textBuffer.append(extraBuffer.toString());
+    return textBuffer.toString();
+  }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractIdentifierFactory.java	(revision 21112)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractIdentifierFactory
+{
+  private String idPrelude;
+  private int    idSeqNo;
+
+  public AbstractIdentifierFactory(String idPrelude)
+  { this.idPrelude = idPrelude;
+    this.idSeqNo   = 1;
+  }
+
+  public String getNextIdentifier()
+  { int id = this.idSeqNo ++;
+
+    return idPrelude + Integer.toString(id);
+  }
+
+  public void noteIdentifier(String identifier)
+  { if (!identifier.startsWith(this.idPrelude)) 
+      return;
+  
+    identifier = identifier.substring(this.idPrelude.length());
+
+    int value = Integer.parseInt(identifier);
+    if (value >= this.idSeqNo) {
+      this.idSeqNo = value + 1;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractMetadataGroup.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class AbstractMetadataGroup
+{ private String ID;
+
+  public AbstractMetadataGroup(String ID)
+  { this.ID = ID;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/AbstractStructure.java	(revision 21112)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+/**
+ *  A helpful 'superclass' of METSStructure and METSDivision which allows for some
+ *  simple generalisation of the two classes.
+ */
+
+public abstract class AbstractStructure
+{ Map               children;
+  AbstractStructure parent;
+  String            ID;     // the unique identifier for this division (unique within the
+                            // document, not necessarily unique within the collection).
+
+  public AbstractStructure(String ID)
+  { this.children = new LinkedHashMap();
+    this.parent   = null;
+    this.ID       = ID;
+  }
+
+  public AbstractStructure getParent()
+  { return this.parent;
+  }
+
+  public void setParent(AbstractStructure parent)
+  { this.parent = parent;
+  }
+
+  /**
+   *  Get the identifer of this structure
+   *
+   *  @param <code>String</code> the identifier.
+   */
+  public String getID()
+  { return this.ID;
+  }
+
+  public abstract String getStructureType();
+
+  /**
+   *  Add a child division, or sub-division to the document.  For example,
+   *  sections within a chapter.
+   *
+   *  @param <code>METSDivision</code> the child division.
+   */
+  public void addDivision(METSDivision child)
+  { this.children.put(child.getLabel(), child);
+    child.setParent(this);
+  }
+
+  public Iterator getChildIterator()
+  { return this.children.values().iterator();
+  }
+    
+  public int size()
+  { return this.children.size();
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/FileIdentifierFactory.java	(revision 21112)
@@ -0,0 +1,18 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String FILE_IDENTIFIER_PRELUDE = "FILE";
+        
+    public FileIdentifierFactory()
+    { 
+	super(FILE_IDENTIFIER_PRELUDE);
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/GSDL3Namespace.java	(revision 21112)
@@ -0,0 +1,113 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class GSDL3Namespace extends SimpleNamespace
+{
+    public static String GSDL3_NAMESPACE_ID = "gsdl3";
+    private static String GSDL3_METADATA_TAG = "Metadata";
+    
+    public GSDL3Namespace()
+    { 
+	super(GSDL3_NAMESPACE_ID);
+    }
+    
+    public GSDL3Namespace(METSLocation location)
+    { 
+	super(GSDL3_NAMESPACE_ID, location);
+    }
+    
+    public GSDL3Namespace(Element mdWrapTag)
+    {	
+	super(GSDL3_NAMESPACE_ID);
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++){
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE){
+		String metadataLabel = childNodes.item(c).getNodeName();
+
+		// cull any leading "GSDL3" tag and its colon...
+		if (metadataLabel.startsWith(GSDL3_NAMESPACE_ID + ":")){
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		// TODO: throw error: bad metadata type
+		if (metadataLabel.equals(GSDL3_METADATA_TAG) == false){
+		    continue;
+		}
+		
+		// Get the label - if the name isn't set, abort.
+		// TODO: throw error: incomplete metadata tag
+		metadataLabel = ((Element) childNodes.item(c)).getAttribute("name");
+		if (metadataLabel == null) {
+		    continue;
+		}
+		
+		// now build the associated value.
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    {	
+	if (this.location == null){
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()){
+		String tag;
+		String value = values.next().toString();
+		
+		tag = XMLTools.getOpenTag(GSDL3_NAMESPACE_ID, GSDL3_METADATA_TAG);
+		tag = XMLTools.addAttribute(tag, "name", label);
+		writer.print(tag);
+		writer.print(value);
+		
+		writer.println(XMLTools.getCloseTag(this.name, GSDL3_METADATA_TAG));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.println(tag);
+	}
+	return true;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAdmin.java	(revision 21112)
@@ -0,0 +1,11 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  METS Administrative metadata
+ *
+ *  May contain zero or more instances of administrative metadata
+ */
+
+public class METSAdmin
+{
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSAgent.java	(revision 21112)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSAgent
+{
+	String name;
+	String note;
+
+	String role;
+	String otherrole;
+	String type;
+	String othertype;
+
+	public METSAgent(String name, String note, String role, String type)
+	{	this.name = name;
+		this.note = note;
+
+		this.setRole(role);
+		this.setType(type);
+	}
+
+	/**
+	 *  Check if the role matches one of the standard offerings
+	 */
+	private boolean validateRole(String role)
+	{	return true;
+	}
+
+	/**
+	 *  Check if the type matches one of the standard offerings
+	 */
+	private boolean validateType(String role)
+	{	if (role.equals("INDIVIDUAL") ||
+		    role.equals("ORGANIZATION")) {
+			return true;
+		}
+		return false;
+	}
+
+	public void setRole(String otherRole)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateRole(role)) {
+			this.role = role;
+			this.otherrole = null;
+		}
+		else {
+			this.role = "OTHER";
+			this.otherrole = role;
+		}
+	}
+
+	public void setType(String otherType)
+	{	// automatically use 'OTHER' if either the role or type of agent don't
+		// validate against the standard list
+		if (this.validateType(type)) {
+			this.type = type;
+			this.othertype = null;
+		}
+		else {
+			this.type = "OTHER";
+			this.othertype = type;
+		}
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDate.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDate.java	(revision 21112)
@@ -0,0 +1,153 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSDate
+{
+	String era;
+	String year;
+	String month;
+	String day;
+	String hour;
+	String minute;
+	String second;
+	String timezone;
+
+	public void METSDate(String string)
+	{	String date;
+
+		int timeAt = string.indexOf('T');
+
+		// if we've got a time signature, get it
+		if (timeAt >= 0) {
+		  // get the date for later use
+			date = string.substring(0, timeAt);
+
+			// extract the time string
+			String time = string.substring(timeAt+1);
+
+			// check for time zone usage - currently '+' and '-' only...
+			int zoneAt = time.indexOf('+');
+			if (zoneAt == -1) {	
+				zoneAt = time.indexOf('-');
+			}
+			if (zoneAt >= 0) {
+				this.timezone = time.substring(zoneAt);
+				time = time.substring(0, zoneAt);
+			}
+
+			// finally, divide into hours, minutes etc.
+			int hourAt = time.indexOf(':');
+			if (hourAt >= 0) {
+				this.hour = time.substring(0, hourAt);
+
+				time = time.substring(hourAt+1);
+				int minuteAt = time.indexOf(':');
+				if (minuteAt >= 0)
+				{	this.minute = time.substring(0, minuteAt);
+					this.second = time.substring(minuteAt+1);
+				}
+				else
+				{ this.minute = time;
+					this.second = null;
+				}
+			}
+		}
+		else 
+		{ this.timezone = null;
+			this.hour     = null;
+			this.minute   = null;
+			this.second   = null;
+
+			date = string;
+		}
+
+		// get the era
+		int prefixEnds = 0;
+		while (!Character.isDigit(date.charAt(prefixEnds)))
+		{ prefixEnds ++;
+		}
+
+		if (prefixEnds != 0) {
+			this.era = date.substring(0, prefixEnds);
+			date = date.substring(prefixEnds);
+		}
+		else {
+			this.era = null;
+		}
+
+		// finally, get the date
+		
+		// now try dividing up by '-'
+		int yearAt = date.indexOf('-');
+		if (yearAt >= 0) {
+			this.year = date.substring(0, yearAt);
+			date = date.substring(yearAt+1);
+
+			int monthAt = date.indexOf('-');
+			if (monthAt >= 0) {
+				this.month = date.substring(0, monthAt);
+				this.day = date.substring(monthAt+1);
+			}
+			else {
+				this.month = date;
+				this.day   = null;
+			}
+		}
+		else {
+			// try dividing by YYYYMMDD - any part of which may be missing...
+			if (date.length() >= 4) {
+				this.year = date.substring(0, 4);
+				if (date.length() >= 6) {
+					this.month = date.substring(4, 6);
+					this.day = date.substring(6);
+				}
+				else {
+					this.month = date.substring(4);
+					this.day   = null;
+				}
+			}
+			// we just have a day in this case...
+			else {
+				this.year = date;
+				this.month = null;
+				this.day = null;
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.year == null) {
+			return "";
+		}
+
+		StringBuffer buffer = new StringBuffer();
+
+		if (this.era != null) {
+			buffer.append(era);
+		}
+		buffer.append(year);
+		if (this.month != null) {
+			buffer.append("-");
+			buffer.append(this.month);
+			if (this.day != null) {
+				buffer.append("-");
+				buffer.append(this.day);
+
+				if (this.hour != null) {
+					buffer.append("T");
+					buffer.append(this.hour);
+					buffer.append(":");
+					buffer.append(this.minute);
+					if (this.second != null) {
+						buffer.append(":");
+						buffer.append(this.second);
+
+						if (this.timezone != null) {
+							buffer.append(this.timezone);
+						}
+					}
+				}
+			}
+		}
+		return buffer.toString();
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptive.java	(revision 21112)
@@ -0,0 +1,494 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Support the descriptive metadata of a given document
+ *
+ *  The descriptive metadata can occur multiple times, so this 'wrapper' is
+ *  in fact a holder for a number of individual metadata sets.  Individual
+ *  metadata groups are found in METSNamespace items.
+ *
+ *  
+ */
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+
+public class METSDescriptive
+{ 
+    MultiMap map;
+    String   ID;
+    String   name;
+    
+    public METSDescriptive(String ID, String name)
+    { 
+	this.map  = new MultiMap();
+	this.ID   = ID;
+	this.name = name;
+    }
+    
+    public String getID()
+    { 
+	return this.ID;
+    }
+    
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    public void addNamespace(METSNamespace namespace)
+    { 
+	this.map.put(namespace.getName(), namespace);
+    }
+    
+    public METSNamespace getNamespace(String namespace)
+    {
+	return (METSNamespace) this.map.get(namespace);
+    }
+    
+    /**
+     *  Get the version of a metadata namespace that can be used for
+     *  adding new values to the document.  Remember, each namespace may
+     *  occur more than once, but only one of the occurrences can be 'open'.
+     *
+     *  @param <code>String</code> the name of the namespace
+     */
+    public METSNamespace getOpenNamespace(String namespace)
+    { 
+	if (this.map.getAll(namespace) == null) {
+	    return null;
+	}
+	
+	Iterator namespaces = ((List) this.map.getAll(namespace)).iterator();
+	
+	while (namespaces.hasNext()) {
+	    METSNamespace namespaceData = (METSNamespace) namespaces.next();
+	    
+	    if (namespaceData.isEditable()){
+		return namespaceData;
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values are preserved.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void addMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.addMetadata(label, value);
+    }
+    
+    /**
+     *  Add a piece of metadata to this document.  Existing values in editable
+     *  parts of the document's metadata are destroyed.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata   
+     *  @param <code>String</code> the value of the metadata
+     */
+    public void setMetadata(String namespace, String label, String value)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData == null ||
+	    namespaceData.isEditable() == false) {
+	    namespaceData = NamespaceFactory.initNamespace(namespace);
+	    this.addNamespace(namespaceData);
+	}
+	namespaceData.setMetadata(label, value);
+    }
+    
+    /**
+     *  Remove the metadata for a particular label.
+     *
+     *  @param <code>String</code> the namespace in which the metadata occurs
+     *  @param <code>String</code> the label of the metadata
+     */
+    public void removeMetadata(String namespace, String label)
+    { 
+	METSNamespace namespaceData = this.getOpenNamespace(namespace);
+	
+	if (namespaceData != null && namespaceData.isEditable() == true) {
+	    namespaceData.removeMetadata(label);
+	}
+    }
+    
+    /**
+     *  Fetch a <code>List</code> of the values matching the given label in
+     *  the requisite namespace.
+     *
+     *  @param <code>String</code> namespace
+     *  @param <code>String</code> label of metadata to match
+     *
+     *  @return <code>List</code> the corresponding values found.
+     */
+    public List getMetadata(String namespace, String label)
+    {
+	// Simple case - no metadata
+	if (!this.map.containsKey(namespace)){
+	    return null;
+	}
+	
+	// if there's just one instance of the requisite namespace,
+	// then again it is a simple case
+	if (this.map.getAll(namespace).size() == 1){
+	    return this.getNamespace(namespace).getMetadata(label);
+	}
+	
+	// otherwise, step through and accumulate
+	List accumulatedValues = new ArrayList();
+	Iterator namespaceIterator = this.map.getAll(namespace).iterator();
+	while (namespaceIterator.hasNext()){
+	    METSNamespace localNamespace = (METSNamespace) namespaceIterator.next();
+	    
+	    List localList = localNamespace.getMetadata(label);
+	    if (localList != null && localList.size() > 0){
+		accumulatedValues.addAll(localList);
+	    }
+	}
+	
+	// if the result of the accumulation is empty, return a null item
+	if (accumulatedValues.size() == 0){
+	    return null;
+	}
+	
+	return accumulatedValues;
+    }
+    
+    protected void parse_xmlData(METSNamespace namespace, Element element) 
+    {
+	if (namespace==null) {
+	    System.err.println("namespace is null!!!!");
+	}
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    // if (childName.equals("gsdl2:Metadata")) {
+	    if (childName.equals("gsdl3:Metadata")) {
+		String metaName = child.getAttribute("name");
+		Text text = (Text) child.getFirstChild();
+		String metaValue = text.getData();
+		
+		addMetadata(namespace.getName(),metaName, metaValue);
+	    }
+	}
+    }
+    
+    public void parse_mdWrap(Element element)
+    {
+	// deal with mets:mdWrap
+	
+	METSNamespace namespace = NamespaceFactory.parseXML(element);
+	
+	NodeList children = element.getChildNodes();
+	
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child  = (Element) children.item(c);
+	    String childName = child.getNodeName();
+
+	    if (childName.equals("mets:xmlData")) {
+		parse_xmlData(namespace, child);
+	    } else {
+		System.err.println("Warning: unrecognised tag " + childName 
+				   + "in mets:mdWrap");
+	    }
+	}
+    }
+
+    /**
+     *  Parse an XML Element as a METS Descriptive Metadata section
+     *
+     *  @param <code>Element</code> the XML element which represents the dmdSec itself
+     */
+    public static METSDescriptive parseXML(Element element) 
+    {
+	// Note: no parsing of attributes required, we just move onto
+	// the metadata namespaces/sections themselves
+
+	// deal with mets:dmdSec
+	String ID = element.getAttribute("ID");
+	String label = element.getAttribute("GROUPID");
+	
+	METSDescriptive thisDescriptive = new METSDescriptive(ID, label);
+
+	//thisDescriptive = new METSDescriptive(ID, label);
+	
+	NodeList children = element.getChildNodes();
+
+	for (int c = 0; c < children.getLength(); c ++) {
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+		continue;
+	    }
+	    
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:mdRef")) {
+		//for External Descriptive Metadata
+		METSNamespace namespace = NamespaceFactory.parseXML(element);
+		System.err.println("*** mets::mdRef does not appear to be implemented yet");
+	    } else if (childName.equals("mets:mdWrap")) {
+		thisDescriptive.parse_mdWrap(child);
+	    } else {
+		// TODO: raise an error!
+		System.out.println("Error: METSDescriptive: unrecognised tag "+childName);
+	    }
+	}
+	return thisDescriptive;
+    }
+    /**
+     *  Write the document metadata to a <code>PrintWriter</code> in METS
+     *  XML format.
+     *  
+     *  @param <code>PrintWriter</code> the writer to use for output
+     */
+    public void write(PrintWriter output)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "dmdSec");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID);
+	tag = XMLTools.addAttribute(tag, "GROUPID", this.name);
+	output.println(tag);
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+	
+		namespace.write(output);
+	    }
+	}
+	
+	tag = XMLTools.getCloseTag("mets", "dmdSec");
+	output.println(tag);
+    }
+    
+    public void copy_metadata(METSDescriptive dst) 
+    {
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()) {
+	    Object nextKey = keys.next();
+	    if (nextKey == null) {
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every copy of the current namespace name - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+      
+	    while (namespaces.hasNext()) {
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		// for each metadata name 
+
+		Iterator mdnIter = namespace.getMetadataNames();
+		while (mdnIter.hasNext()) {
+
+		    String md_name = mdnIter.next().toString();
+		    
+		    List values = getMetadata(key, md_name);
+		    
+		    if (values != null) {	
+			Iterator valueIter = values.iterator();
+			while (valueIter.hasNext()) {
+			    String value = valueIter.next().toString();
+			    dst.addMetadata("gsdl3",md_name,value);
+			}
+		    }
+		}
+	    }
+	}
+    }
+
+
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	// check if this node is in the
+	int sqlId = -1;
+	
+	//    System.out.println("Writing " + connection.toString());
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetaID", "=", this.ID);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    GS3SQLAction action;
+	    
+	    if (resultSet.first()) {
+		// the node already exists - no need to do anything as such
+		GS3SQLUpdate update = new GS3SQLUpdate("metadata");
+		update.setWhere(where);
+		action = update;
+	    }
+	    else { // Set result set to null, just in case first() didn't work above...
+		// It is a new node and needs writing
+		action = new GS3SQLInsert("metadata");
+		
+		action.addValue("DocID", document.getID().toString());
+		action.addValue("MetaID", this.ID);
+	    }
+	    
+	    // always set the group identifier
+	    action.addValue("GroupID", this.name);
+	    
+	    // execute the action as required
+	    statement.execute(action.toString());
+	    
+	    // get the resultSet again
+	    // now execute the select statement again to get the new identifier
+	    // 'MetadataRef'
+	    // System.out.println(select.toString());
+	    resultSet = statement.executeQuery(select.toString());
+	    
+	    if (resultSet.first()) {
+		// get the reference for this item...
+		sqlId = resultSet.getInt("MetadataRef");
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql){
+	    System.err.println("METSDescriptive.writeSQL(): "+sql);
+	}
+	if (sqlId == -1) {
+	    return false;
+	}
+	
+	Iterator keys = this.map.keySet().iterator();
+	
+	// get the keys one by one
+	while (keys.hasNext()){
+	    Object nextKey = keys.next();
+	    if (nextKey == null){
+		continue;
+	    }
+	    String key = nextKey.toString();
+	    
+	    // get every current namespace - namespaces may
+	    // occur more than once, so this is a List
+	    Iterator namespaces = this.map.getAll(key).iterator();
+	    
+	    // namespaces will write themselves...
+	    while (namespaces.hasNext()){
+		METSNamespace namespace = (METSNamespace) namespaces.next();
+		
+		if (!namespace.writeSQL(sqlId, connection)){
+		    return false;
+		}
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDescriptive readSQL(DocumentInterface document, GS3SQLConnection connection,
+					  ResultSet resultSet)
+    {
+	try {
+	    String ID   = resultSet.getString("MetaID");
+	    String name = resultSet.getString("GroupID");
+	    
+	    // create the metadata block object
+	    METSDescriptive descriptive = new METSDescriptive(ID, name);
+	    
+	    // get its metadata reference to retrieve namespaces
+	    int metadataRef = resultSet.getInt("MetadataRef");
+	    
+	    // query the database for matching namespaces for this metadata block
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(metadataRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    ResultSet namespaceSet = statement.executeQuery(select.toString());
+	    
+	    // parse through the namespaces, calling the namespace class to create instances as it will
+	    if (namespaceSet.first()) {
+		do {
+		    METSNamespace namespace = METSNamespace.readSQL(connection, namespaceSet);
+		    if (namespace != null) {
+			descriptive.addNamespace(namespace);
+		    }
+		    else {
+			System.err.println("Null namespace output");
+		    }
+		}
+		while (namespaceSet.next());
+	    }
+	    statement.close();
+	    return descriptive;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSDescriptive.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSection.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public abstract class METSDescriptiveSection
+{ 
+    METSLocation location;
+    String       type;
+    String       otherType;
+    
+    public METSDescriptiveSection(String type)
+    {	
+	this.location  = null;
+	this.type      = type;
+	this.otherType = null;
+    }
+	
+    public METSDescriptiveSection(METSLocation location, String type)
+    {	
+	this.location  = location;
+	this.type      = type;
+	this.otherType = null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDescriptiveSet.java	(revision 21112)
@@ -0,0 +1,266 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDescriptiveSet
+{ 
+    Map children;
+    Map childrenById;
+    DescriptiveIdentifierFactory identifierFactory;
+    
+    class DescriptiveIdentifierFactory extends AbstractIdentifierFactory
+    { 
+	public DescriptiveIdentifierFactory()
+	{ 
+	    super("DM");
+	}
+    }
+    
+    public METSDescriptiveSet()
+    { 
+	this.children = new HashMap();
+	this.childrenById = new HashMap();
+	
+	this.identifierFactory = new DescriptiveIdentifierFactory();
+	
+	this.createDescriptive("default");
+    }
+    
+    /**
+     *  Create a new descriptive child, adding it automatically to this descriptive set.
+     *
+     *  @param <code>String</code> a name to describe the descriptive set.
+     */
+    public METSDescriptive createDescriptive(String name)
+    { 
+	METSDescriptive descriptive = new METSDescriptive(this.identifierFactory.getNextIdentifier(), name);
+	
+	this.children.put(name, descriptive);
+	this.childrenById.put(descriptive.getID(), descriptive);
+	return descriptive;
+    }
+    
+    /**
+     *  Add a descriptive item to this set.
+     *
+     *  @param <code>METSDescriptive</code> the descriptive set
+     */
+    public void addDescriptive(METSDescriptive metadata)
+    { 
+	// insert the item into both maps...
+	this.children.put(metadata.getName(), metadata);
+	this.childrenById.put(metadata.getID(), metadata);
+	
+	// when a descriptive block is added, check that it hasn't been used before...
+	//this.identifierFactory.noteIdentifier(metadata.getID());
+	this.identifierFactory.noteIdentifier(metadata.getName());
+    }
+    
+    /**
+     *  Get a descriptive item by its name(GROUPID in METS file)
+     */
+    public METSDescriptive getDescriptive(String name)
+    { 
+	
+	return (METSDescriptive) this.children.get(name);
+    }
+    
+    /**
+     *  Get a descriptive item by its identifier
+     */
+    public METSDescriptive getDescriptiveById(String id)
+    { 
+	return (METSDescriptive) this.childrenById.get(id);
+    }
+    
+    public void addNamespace(String descriptiveName, METSNamespace namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    descriptive.addNamespace(namespace);
+	}
+    }
+    
+    public METSNamespace getNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	if (descriptive != null){
+	    return descriptive.getNamespace(namespace);
+	}
+	return null;
+    }
+    
+    public METSNamespace getOpenNamespace(String descriptiveName, String namespace)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getOpenNamespace(namespace);
+	}
+	return  null;
+    }
+    
+    public void addMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.addMetadata(namespace, label, value);
+	}
+    }
+    
+    public void setMetadata(String descriptiveName, String namespace, String label, String value)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.setMetadata(namespace, label, value);
+	}
+    }
+    
+    public void removeMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public void removeAllMetadata(String namespace, String label)
+    { 
+	Iterator descriptiveIter = this.children.values().iterator();
+	
+	while (descriptiveIter.hasNext()) {
+	    METSDescriptive descriptive = (METSDescriptive) descriptiveIter.next();
+	    
+	    descriptive.removeMetadata(namespace, label);
+	}
+    }
+    
+    public List getMetadata(String descriptiveName, String namespace, String label)
+    { 
+	METSDescriptive descriptive = this.getDescriptive(descriptiveName);
+	
+	if (descriptive != null){
+	    return descriptive.getMetadata(namespace, label);
+	}
+	return null;
+    }
+    
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDescriptive group = (METSDescriptive) groups.next();
+	    
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSDescriptiveSet parseXML(NodeList dmdSecs) 
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	for (int g = 0; g < dmdSecs.getLength(); g ++) {
+	    METSDescriptive metadata = METSDescriptive.parseXML((Element) dmdSecs.item(g));
+	    
+	    set.addDescriptive(metadata);
+	}
+    
+	System.err.println("******** Away to do transfer!!!!!!");
+	  
+	METSDescriptive top_level_metadata = set.getDescriptive("1");
+	METSDescriptive tl2_metadata = set.getDescriptive("1");
+	METSDescriptive opo_metadata = set.getDescriptive("1.1");
+	METSDescriptive default_metadata = set.getDescriptive("default");
+	
+	if (top_level_metadata != null) {
+	    if (default_metadata != null) {
+		top_level_metadata.copy_metadata(default_metadata);
+	    }
+	    else {
+		System.err.println("Warning: Could not find default metadata");
+	    }
+	}
+	else {
+	    System.err.println("Warning: Could not find top level metadata for groupID=1");
+	}
+	return set;
+    } 
+    
+    public static METSDescriptiveSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    {
+	METSDescriptiveSet set = new METSDescriptiveSet();
+	
+	GS3SQLSelect select = new GS3SQLSelect("metadata");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSDescriptive descriptive = METSDescriptive.readSQL(document, connection, resultSet);
+		    if (descriptive != null) {
+			set.addDescriptive(descriptive);
+		    }
+		    else {
+			System.out.println("Null descriptive");
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSDescriptiveSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	
+	
+	return set;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDivision.java	(revision 21112)
@@ -0,0 +1,681 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSDivision extends AbstractStructure
+{
+    String       order;       // see the METS documentation for the meaning of the different
+    String       orderLabel;  // label types.
+    String       userLabel;
+    String       type;        // e.g. section, chapter, physical or logical, etc.
+    List         metadataRefs;
+    List         adminRefs;
+    List         fileRefs;
+    
+    public static final String DIVISION_TYPE  = "Division";
+    public static final String DIVISION_TAG   = "div";
+    public static final String FPTR_TAG       = "fptr";
+    
+    public METSDivision(String ID, String order, String orderLabel, String userLabel, String type)
+    { 
+	super(ID);
+	
+	this.order = order;
+	this.orderLabel = orderLabel;
+	this.userLabel = userLabel;
+	this.type = type;
+	this.fileRefs = new ArrayList();
+	this.metadataRefs = new ArrayList();
+    }
+    
+    public String getStructureType()
+    { 
+	return DIVISION_TYPE;
+    }
+    
+    public String getLabel()
+    { 
+	return this.orderLabel;
+    }
+    
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching divisions.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { 
+	Iterator iterator = this.fileRefs.iterator();
+	
+	// get the references on this division, and mark if necessary
+	while (iterator.hasNext()) {
+	    String reference = iterator.next().toString();
+	    
+	    Iterator fileIterator = listOfFileIdentifiers.iterator();
+	    while (fileIterator.hasNext()){
+		String fileRef = fileIterator.next().toString();
+		
+		if (fileRef.equals(reference)){
+		    resultList.add(this);
+		}
+	    }
+	}
+	
+	// get the child groups, and check each in turn...
+	Iterator groups = this.children.values().iterator();
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+    
+    /**
+     *  Get the namespace from the given set of descriptive blocks...
+     */
+    public METSNamespace findNamespace(String namespaceName, boolean isOpenNamespace, 
+				       METSDescriptiveSet descriptiveSet)
+    { 
+	METSNamespace namespace = null;
+	
+	// Special case - no existing metadata references
+	if (this.metadataRefs.size() == 0) {
+	    // Create new Descriptive and add it to the descriptive set
+	    METSDescriptive descriptive = new METSDescriptive("DM"+this.ID, this.orderLabel);
+	    descriptiveSet.addDescriptive(descriptive);
+	    
+	    // Create the namespace and add it to the Descriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	    
+	    return namespace;
+	}
+	
+	Iterator descriptiveIterator = this.metadataRefs.iterator();
+	
+	while (descriptiveIterator.hasNext()){
+	    String descriptiveRef = descriptiveIterator.next().toString();
+	    
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(descriptiveRef);
+	    
+	    if (isOpenNamespace) {
+		namespace = descriptive.getOpenNamespace(namespaceName);
+	    }
+	    else {
+		namespace = descriptive.getNamespace(namespaceName);
+	    }
+	    
+	    if (namespace != null)
+		break;
+	}
+	
+	// Special case - no such namespace in any descriptive list...
+	if (namespace == null){
+	    // take the first descriptive area...
+	    METSDescriptive descriptive = descriptiveSet.getDescriptiveById(this.metadataRefs.get(0).toString());
+	    
+	    // create the namespace, and add it to the METSDescriptive
+	    namespace = NamespaceFactory.initNamespace(namespaceName);
+	    if (namespace != null){
+		descriptive.addNamespace(namespace);
+	    }
+	}
+	
+	return namespace;
+    }
+    
+    /**
+     *  Add a new file reference to the structure
+     *
+     *  @param <code>METSFileID</code> the file to be referenced.
+     */
+    public void addFileReference(METSFileID fileRef)
+    { 
+	this.addFileReference(fileRef.toString());
+    }
+    
+    public void addFileReference(String fileRef)
+    { 
+	this.fileRefs.add(fileRef);
+    }
+    
+    /**
+     *  Add a new metadata reference to the structure
+     */
+    public void addMetadataReference(String metadataRef)
+    { 
+	// TODO: correct the type
+	this.metadataRefs.add(metadataRef);
+    }
+
+    /**
+     *  Get all the metadata references
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public List getMetadataReferences()
+    { 
+	return this.metadataRefs;
+    }
+
+    /**
+     *  Get all the default (first) metadata reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    //TODO: check for this first *open* metadata item...
+    public String getDefaultMetadataReference()
+    { 
+	if (this.metadataRefs.size() == 0) {
+	return null;
+    }
+    return this.metadataRefs.get(0).toString();
+    }
+
+    /**
+     *  Get all the default (first) first filegroup reference...
+     *
+     *  @return <code>List</code> containing all the references
+     */
+    public String getDefaultFileReference()
+    { 
+	if (this.fileRefs.size() == 0) {
+	return null;
+    }
+    return this.fileRefs.get(0).toString();
+    }
+
+    /**
+     *  Get a child or descendant division
+     *
+     *  @param <code>String</code> the path to the child division
+     *  @return <code>METSDivision</code> the division found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { 
+	// if the division ID equals this, then process it
+	if (divisionLabel.equals(this.orderLabel)) {
+	    return this;
+	}
+
+	// if we just shouldn't be here, then error...
+	if (!divisionLabel.startsWith(this.orderLabel + ".")) {
+	    return null;
+	}
+
+	// get the next division in the division path...
+	String subdivisionPath = divisionLabel.substring(this.orderLabel.length() + 1);
+	String nextDivision = getTopDivisionName(subdivisionPath);
+	String nextLabel;
+
+	// first try the current division name with an appended '.' and the next child id...
+	METSDivision child = (METSDivision) this.children.get(this.orderLabel + "." + nextDivision);
+	if (child == null){
+	    // if that didn't work, try the next division on its own
+	    child = (METSDivision) this.children.get(nextDivision);
+	    
+	    // if neither worked, then abort...
+	    if (child == null) {
+		return null;
+	    }
+	    else {
+		// set the next identifier
+		nextLabel = METSDivision.getSubDivisionName(divisionLabel);
+	    }
+	} 
+	else {
+	    // set the next identifier...
+	    nextLabel = divisionLabel;
+	}
+	
+	return child.getDivision(nextLabel);
+    }
+    
+    /**
+     *  A convenience method for splitting a division name - this gives the name
+     *  without the leading division label...  To be used when division identifiers
+     *  are given as '.' separated paths and division names do not contain the
+     *  full path of their parents.
+     *
+     *  @return <code>String</code>
+     */
+    public static String getSubDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(at+1);
+	}
+	else {
+		return null;
+	}
+    }
+
+    /**
+     *  Get the parent division name.  Division identifiers
+     *  concatenate the ancestor's names with intervening
+     *  '.' characters.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getParentName(String divisionName)
+    { 
+	int at = divisionName.lastIndexOf('.');
+
+	if (at >= 0) {
+	    return divisionName.substring(0, at);
+	} 
+	return "";
+    }
+    
+    /**
+     *  Get the top division name.
+     *
+     *  @return <code>String</code> the division name
+     */
+    public static String getTopDivisionName(String divisionName)
+    { 
+	int at = divisionName.indexOf('.');
+
+	if (at >= 0){
+	    return divisionName.substring(0, at);
+	}
+	else {
+	    return divisionName;
+	}
+    }
+
+    /*  Parse an XML Element as a METSDivision section
+     *
+     *  @param <code>Element</code> the XML element which represents
+     *  the mets:div itself 
+     */
+    
+    public static METSDivision parseXML(Element element) 
+    {
+ 	METSDivision thisDivision = null;
+ 
+ 	String divId = element.getAttribute("ID");
+ 	String divType = element.getAttribute("TYPE");
+ 	String divOrder = element.getAttribute("ORDER");
+ 	String divOrderLabel = element.getAttribute("ORDERLABEL");
+ 	String divUserLabel = element.getAttribute("LABEL");
+ 	String divMetaRef = element.getAttribute("DMDID");
+ 	
+ 	thisDivision = new METSDivision(divId, divOrder,divOrderLabel, divUserLabel, divType);
+ 		
+ 	thisDivision.addMetadataReference(divMetaRef);
+ 
+ 	NodeList children = element.getChildNodes();
+ 
+ 	for (int c = 0; c < children.getLength(); c++){
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+ 
+ 	    if (childName.equals("mets:fptr")) {
+ 		
+ 		METSDivision.fptr_parseXML(thisDivision, child);
+ 	    } else if (childName.equals("mets:div")) {
+ 		METSDivision division = METSDivision.parseXML(child);
+ 		thisDivision.addDivision(division);
+ 	    } else {
+ 		System.err.println("Error: mets:div expected mets:div or mets:fptr, but got " + childName);
+ 	    } 
+ 	}
+ 	return thisDivision;
+    }
+ 
+ 
+    /**
+     * Parse an XML element as a  METSFptr Section
+     * @param <code>Element</code> the XML element which represents the mets:fptr itself
+     */
+    private static void fptr_parseXML(METSDivision thisDivision, Element element) 
+    {
+ 	String fileRef = element.getAttribute("FILEID");
+ 	thisDivision.addFileReference(fileRef);
+    }
+ 
+    /**
+
+
+    /**
+    *  Write the METS file in an XML to a text-output sink.
+    *
+    *  @param <code>PrintWriter</code> the destination of the output
+    */
+    public boolean write(PrintWriter writer)
+    { 
+	// Get the main tag for the division
+	String tag = XMLTools.getOpenTag("mets", DIVISION_TAG);
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "ORDER", this.order);
+	if (this.orderLabel != null){
+	    tag = XMLTools.addAttribute(tag, "ORDERLABEL", this.orderLabel);
+	}
+	if (this.userLabel != null){ 
+	    tag = XMLTools.addAttribute(tag, "LABEL", this.userLabel);
+	}
+	
+	// Add a metadata reference attribute
+	if (this.metadataRefs.size() > 0){
+	    String metaList = this.writeList(this.metadataRefs);
+	    tag = XMLTools.addAttribute(tag, "DMDID", metaList);
+	}
+	
+	// write the central tag
+	writer.print("   ");  //indentation
+	writer.println(tag);
+
+	// write the list of file pointers for this structural element
+	if (this.fileRefs.size() > 0) {
+	    String fileList = this.writeTagList("mets", "fptr", "FILEID", this.fileRefs);
+	    writer.print("      ");  //indentation
+	    writer.println(fileList);
+	}
+	
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.write(writer)) {
+		return false;
+	    }
+	}
+    
+	// close the div element
+	writer.print("   ");  //indentation
+	writer.println(XMLTools.getCloseTag("mets", DIVISION_TAG));
+
+	return true;
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a series of tags...
+     */
+    private String writeTagList(String namespace, String tagname, 
+				String attribute, List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    String tag = XMLTools.getOpenTag(namespace, tagname);
+	    tag = XMLTools.addAttribute(tag, attribute, item);
+	    tag = XMLTools.makeSingleton(tag);
+	    
+	    listString.append(tag);
+	}
+	return listString.toString();
+    }
+
+    /**
+     *  A convenience method for writing a <code>List</code> as a space-separated <code>String</code>.
+     */
+    private String writeList(List list)
+    { 
+	StringBuffer listString = new StringBuffer();
+	
+	Iterator iterator = list.iterator();
+	while (iterator.hasNext()){
+	    String item = iterator.next().toString();
+	    
+	    if (listString.length() > 0) {
+		listString.append(" ");
+	    }
+	    
+	    listString.append(item);
+	}
+	return listString.toString();
+    }
+    
+    public boolean writeSQL(DocumentID docId, int parentRef, boolean parentIsStructure, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	
+	// set the "where" to find the reference for this division
+	GS3SQLSelect select = new GS3SQLSelect("divisions");
+	select.addField("DivisionRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(parentRef), GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhereItem parentItem = new GS3SQLWhereItem("ParentType", "=", parentIsStructure ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+	GS3SQLWhereItem item = new GS3SQLWhereItem("SectionID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(parentItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	Statement statement = null;
+	ResultSet resultSet = null;
+	
+	// Do the actual writing
+	GS3SQLAction action;
+	
+	try {
+	    statement = connection.createStatement();
+	    resultSet = statement.executeQuery(select.toString());
+	    if (!resultSet.first()) {
+		
+		GS3SQLInsert insert = new GS3SQLInsert("divisions");
+		insert.addValue("DocID", docId.toString());
+		insert.addValue("SectionID", this.ID.toString());
+		insert.addValue("ParentRef", Integer.toString(parentRef));
+		insert.addValue("ParentType", parentIsStructure == true ? METSStructure.STRUCTURE_TYPE : METSDivision.DIVISION_TYPE);
+		
+		action = insert;
+	    }
+	    else {
+		sqlRef = resultSet.getInt("DivisionRef");
+		
+		GS3SQLUpdate update = new GS3SQLUpdate("divisions");
+		GS3SQLWhere  updateWhere = 
+		    new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE));
+		update.setWhere(updateWhere);
+		action = update;
+	    }
+	}
+	catch (SQLException sql) {
+	    System.err.println(sql + " " + select.toString());
+	    return false;
+	}
+	
+	action.addValue("DivisionType", this.type.toString());
+	action.addValue("LabelOrder", this.order);
+	action.addValue("ShortLabel", this.orderLabel);
+	action.addValue("UserLabel", this.userLabel);
+	
+	try {
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSDivision.writeSQL(): "+e);
+	    return false;
+	}
+	
+	// if doing a fresh item, get the new structure reference...
+	if (sqlRef == -1){
+	    // get the new structure reference
+	    try {
+		// read in the structure reference
+		resultSet = statement.executeQuery(select.toString());
+		if (resultSet.first()) {
+		    sqlRef = resultSet.getInt("DivisionRef");
+		}
+	    }
+	    catch (SQLException sqlex) {
+		System.err.println("METSDIVISION.writeSQL(): Unable to retrieve reference for Division " + sqlex);
+		return false;
+	    }
+	}
+	
+	// delete the old file/metadata references
+	try {
+	    GS3SQLWhere referenceWhere = 
+		new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(sqlRef), 
+						    GS3SQLField.INTEGER_TYPE));
+	    
+	    GS3SQLDelete delete = new GS3SQLDelete("divisionfilerefs");
+	    delete.setWhere(referenceWhere);
+	    
+	    statement.execute(delete.toString());
+	    
+	    delete = new GS3SQLDelete("divisionmetarefs");
+	    delete.setWhere(referenceWhere);
+	    statement.execute(delete.toString());
+	
+	    // write the new file references
+	    if (this.fileRefs.size() > 0){
+		Iterator iterator = this.fileRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert fileinsert = new GS3SQLInsert("divisionfilerefs");
+		    fileinsert.addValue("DocID", docId.toString());
+		    fileinsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    fileinsert.addValue("DivisionType", "Group");
+		    fileinsert.addValue("FileID", iterator.next().toString());
+		    statement.execute(fileinsert.toString());
+		}
+	    }
+	    
+	    // write the metadata references
+	    if (this.metadataRefs.size() > 0){
+		Iterator iterator = this.metadataRefs.iterator();
+		
+		while (iterator.hasNext()) {
+		    GS3SQLInsert metainsert = new GS3SQLInsert("divisionmetarefs");
+		    metainsert.addValue("DocID", docId.toString());
+		    metainsert.addValue("DivisionRef", Integer.toString(sqlRef), GS3SQLField.INTEGER_TYPE);
+		    metainsert.addValue("DivisionType", "Group");
+		    metainsert.addValue("MetaID", iterator.next().toString());
+		    statement.execute(metainsert.toString());
+		}
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSDIVISION.writeSQL(): "+e);
+	    return false;
+	}
+	// write out any children in turn
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(docId, sqlRef, false, connection)) {
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    public static METSDivision readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	GS3SQLSelect select = null;
+	try {
+	    // read the basic information
+	    String ID   = resultSet.getString("SectionID");
+	    String type = resultSet.getString("DivisionType");
+	    String order = resultSet.getString("LabelOrder"); 
+	    String orderLabel = resultSet.getString("ShortLabel");
+	    String userLabel  = resultSet.getString("UserLabel");
+      
+	    // construct the division object
+	    METSDivision division = new METSDivision(ID, order, orderLabel, userLabel, type);
+      
+	    // obtain a list of child divisions
+	    int divisionRef = resultSet.getInt("DivisionRef");
+
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(divisionRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSDivision.DIVISION_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet childSet = statement.executeQuery(select.toString());
+      
+	    // circulate through to obtain further children
+	    if (childSet.first())
+		{
+		    do {
+			METSDivision childDivision = METSDivision.readSQL(connection, childSet);
+			division.addDivision(childDivision);
+		    }
+		    while (childSet.next());
+		}
+	    select = new GS3SQLSelect("divisionfilerefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet fileSet = statement.executeQuery(select.toString());
+
+	    if (fileSet.first()){
+		do {
+		    String reference = fileSet.getString("FileID");
+		    division.fileRefs.add(reference);
+		}
+		while (fileSet.next());  
+	    }
+	    
+	    select = new GS3SQLSelect("divisionmetarefs");
+	    select.addField("*");
+	    where = new GS3SQLWhere(new GS3SQLWhereItem("DivisionRef", "=", Integer.toString(divisionRef), 
+							GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    ResultSet metaSet = statement.executeQuery(select.toString());
+
+	    if (metaSet.first()){
+		do {
+		    String reference = metaSet.getString("MetaID");
+		    division.metadataRefs.add(reference);
+		}
+		while (metaSet.next());  
+	    }
+        
+	    statement.close();
+	    return division;
+	}
+	catch (SQLException sqlEx){
+	    System.out.println(sqlEx + " " +select.toString());
+	}    
+	return null;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSDocument.java	(revision 21112)
@@ -0,0 +1,75 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A METS document representation - the METS objects are represented
+ *  by hand-coded classes rather than JAXB products due to the fact
+ *  that we want wrappers to provide higher-level abstractions than
+ *  the basic METS XML mark-up.
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSDocument
+{
+	// TODO: add use of 'ID'
+	String objectID;
+	String label;
+	String type;
+	// TODO: add use of 'PROFILE';
+
+	METSHeader      header;
+	METSDescriptiveSet descriptiveSet;
+	METSAdmin       admin;
+	METSFileSet     fileSet;
+
+	public METSDocument(String identifier)
+	{
+		this.objectID = identifier; // In Greenstone land, this should be a unique identifier within
+		                            // the current collection
+		this.type = "document"; // Types in METS are just strings - we currently use
+		                        // 'Document' as a standard representation.
+                            // Individual options should be defined in a published
+														// METS Profile - in Greenstone we currently 'cheat'
+														// and bodge the system
+		this.header      = new METSHeader();
+		this.descriptiveSet = new METSDescriptiveSet();
+		this.fileSet     = new METSFileSet();
+	}
+
+	/**
+	 *  Set the document identifier for this document
+	 *
+	 *  @param <code>String</code> the identifier
+	 */
+	public void setDocId(String id)
+	{	this.objectID = id;
+	}
+
+	public String getDocId()
+	{	return this.objectID;
+	}
+
+	public METSHeader getHeader()
+	{	return this.header;
+	}	
+
+	public void setHeader(METSHeader header)
+	{	this.header = header;
+	}
+
+	public void addFile(METSFile file)
+	{	this.fileSet.addFile(file);
+	}
+
+	public void addFile(URL url)
+	{	this.fileSet.addFile(url);
+	}
+
+	public void addFile(METSFile file, String toGroup)
+	{	this.fileSet.addFile(file, toGroup);
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFile.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFile.java	(revision 21112)
@@ -0,0 +1,352 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+
+import java.net.URL;
+import java.net.URLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.HTTPTools;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  An individual METS component file for a document.  Many features of METS are not
+ *  currently fully supported/used here, but member variables exist for all of the
+ *  standard members.
+ */
+
+public class METSFile
+{
+    METSFileID    id;
+    METSFilePos   location;
+    String        MIMEType;
+    int           sequenceNo;
+    long          size;
+    METSDate      created;
+    long          checkSum;
+    List          adminRefs;
+    List          describeRefs;
+    METSFileGroup group;
+    
+    /**
+     *  Create a new METSFile object, with the given properties.  The remaining
+     *  fields are filled in with default values.
+     *
+     *  @param <code>String</code> the id of the file
+     *  @param <code>METSFilePos</code> the location of the file
+     *  @param <code>String</code> the MIME type of the file
+     */
+    public METSFile(METSFileID id,  METSFilePos location, String mimeType)
+    {
+	this.id           = id;        // required
+	this.location     = location;  // the location
+	this.MIMEType     = mimeType;  // the MIME type;
+	this.sequenceNo   = -1;       // -1 indicates unset
+	this.size         = -1;        // -1 indicates unset
+	this.checkSum     = -1;        // -1 indicates unset
+	this.created      = null;
+	this.adminRefs    = new ArrayList();
+	this.describeRefs = new ArrayList();
+	this.group        = null;
+    }
+    
+    public boolean equals(URL url)
+    { 
+	return this.location.equals(url);
+    }
+    
+    /**
+     *  Set the parent group
+     *
+     *  @param <code>METSFileGroup</code> the new holding file group
+     */
+    public void setGroup(METSFileGroup group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the parent group of this file
+     *
+     *  @return <code>METSFileGroup</code> the holding file group
+     */
+    public METSFileGroup getGroup()
+    { 
+	return this.group;
+    }
+    
+    /**
+     *  Get the identifier of this file
+     *
+     *  @return <code>METSFileID</code> the file identifier
+     */
+    public METSFileID getID()
+    { 
+	return this.id;
+    }
+    
+    /**
+     *  Set the identifier of this file
+     *
+     *  @param <code>METSFileID</code> the file identifier
+     */
+    public void setID(METSFileID id)
+    { 
+	this.id = id;
+    }
+    
+    /**
+     *  @return <code>String</code> the MIME (content) type of the file
+     */
+    public String getMIMEType()
+    { 
+	return this.MIMEType;
+    }
+    
+    /**
+     *  @return <code>URL</code> the location of the file.  This will often be in
+     *          "file://" form.
+     */
+    public URL getLocation()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Take a url and make a METSFile object out of it - works out
+     *  details such as the Mime type, identifiers and other necessary
+     *  information to call the constructor for METSFile
+     *
+     *  @param <code>URL</code> the url of the object - this may be a 
+     *                          reference to a file on the local filestore
+     *
+     *  @return <code>METSFile</code> the METS file object for the corresponding
+     *                                document component.
+     */
+    public static METSFile makeMETSFile(URL url, String mimeType)
+    {	
+	METSFile reply = new METSFile(new METSFileID(), new METSFilePos(url), mimeType);
+	return reply;
+    }
+    
+    public static METSFile makeMETSFile(URL url)
+    { 
+	String mimeType;
+	
+	if (url.getProtocol().equals("file")){
+	    // TODO: Work out the MIME type
+	    mimeType = URLConnection.getFileNameMap().getContentTypeFor(url.toString().substring(7));
+	    if (mimeType == null) {
+		mimeType = "text/plain";
+	    }
+	} else { 
+	    // TODO: look up the MIME type through the pertinent connection
+	    mimeType = HTTPTools.getMIMEType(url);
+	}
+	// TODO: make the identifier...
+	
+	return makeMETSFile(url, mimeType);
+    }
+    
+    /**
+     *  Write the METS file in an XML to a text-output sink
+     *
+     *  @param <code>PrintWriter</code> the destination of the output
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets", "file");
+	tag = XMLTools.addAttribute(tag, "MIMETYPE", this.MIMEType);
+	tag = XMLTools.addAttribute(tag, "ID", this.id.toString());
+	writer.print("      ");   //indentation
+	writer.println(tag);
+	
+	tag = XMLTools.getOpenTag("mets", "FLocat");
+	tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	
+	tag = XMLTools.makeSingleton(tag);
+	writer.print("         ");  //indentation
+	writer.println(tag);
+	
+	writer.print("      ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets", "file"));
+    }
+    
+    
+    public boolean writeSQL(int groupReference, GS3SQLConnection connection)
+    { 
+	// check if this file is in the table already...
+	GS3SQLAction action;
+	
+	GS3SQLSelect select = new GS3SQLSelect("files");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupReference),
+							GS3SQLField.INTEGER_TYPE);
+	GS3SQLWhere  where  = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("FileID","=", this.id.toString());
+	where.add(whereItem);
+	select.setWhere(where);
+
+	Statement statement = null;
+	
+	// if not, then make an insert action
+	try {
+	    statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(select.toString());
+	    if (!results.first()){
+		GS3SQLInsert insert = new GS3SQLInsert("files");
+		
+		insert.addValue("FileGroupRef", Integer.toString(groupReference), GS3SQLField.INTEGER_TYPE);
+		insert.addValue("FileID", this.id.toString());
+		
+		action = insert;
+	    }
+	    else {
+		GS3SQLUpdate update = new GS3SQLUpdate("files");
+		
+		update.setWhere(where);
+		action = update;
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFile.writeSQL(): "+ex);
+	    return false;
+	}
+	action.addValue("FileLocType", this.location.getType());
+	action.addValue("FileLocation", this.location.getLocation().toString());
+	action.addValue("MIMEType", this.MIMEType);
+	
+	try {
+	    statement.execute(action.toString());
+	    statement.close();
+	    
+	} catch (SQLException e) {
+	    System.err.println("METSFile.writeSQL():"+e);
+	    return false;
+	}
+	return true;
+    }
+    
+    public static METSFile readSQL(GS3SQLConnection connection, ResultSet parentSet)
+    { 
+	try {
+	    String locType = parentSet.getString("FileLocType");
+	    String location = parentSet.getString("FileLocation");
+	    String mimeType = parentSet.getString("MIMEType");
+	    String id = parentSet.getString("FileID");
+	
+	    METSFileID metsID = new METSFileID(id);
+	    METSFilePos metsFilePos = new METSFilePos(locType, location);
+	    
+	    METSFile reply = new METSFile(metsID, metsFilePos, mimeType);
+	    return reply;
+	}
+	catch (SQLException ex){
+	    System.out.println("METSFile.readSQL(): "+ex);
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.out.println("METSFile.readSQL(): "+urlEx);
+	}
+	return null;
+    }
+    
+    public static METSFile parse_file(Element element, METSFileGroup group, String parseFilePath)
+    {
+	NodeList children = element.getChildNodes();
+
+	METSFile thisFile = null;
+	
+	for (int c = 0; c < children.getLength(); c++){
+	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+		continue;
+	    }  
+	    Element child = (Element) children.item(c);
+	    String childName = child.getNodeName();
+	    
+	    if (childName.equals("mets:file")){
+		METSFile.parse_file(child, group, parseFilePath);
+	    } else if (childName.equals("mets:FLocat")) {
+		String mimeType = element.getAttribute("MIMETYPE");
+		String fileId = element.getAttribute("ID");   
+		
+		METSFilePos thisFilePos = METSFile.parse_flocateXML(child, parseFilePath, fileId);
+	   	thisFile = new METSFile(new METSFileID(fileId), thisFilePos, mimeType);    
+		group.addFile(thisFile);
+	    } else {
+		System.err.println("Warning: unrecognised tag " +childName);
+	    }
+	}
+	return thisFile;
+    }
+    
+    public static METSFilePos parse_flocateXML(Element element, String parseFilePath, String fileId)
+    {
+	METSFilePos thisFilePos = null;
+	
+	// get most of the information from the child FLocat node
+	String locationType = element.getAttribute("LOCTYPE");
+	String href = element.getAttribute("xlink:href");
+	//String fileId = element.getAttribute("ID");
+	try {
+	    thisFilePos = new METSFilePos(locationType, href);
+	    
+	    if (fileId.startsWith("default.")) {
+		//retrieve the string after ..import/
+		int importPosition = href.indexOf("import/")+7;
+		href.substring(importPosition);
+		if (fileId.equals("default.1")) {
+		    href = "file:" + parseFilePath+ "doctxt.xml";
+		} else {
+		    href = "file:" + parseFilePath + href.substring(importPosition);
+		}
+	    }
+            thisFilePos = new METSFilePos(locationType, href);
+	} catch (java.net.MalformedURLException ex) {
+	    // TODO: raise error
+	    System.err.println("METSFile.parse_flocateXML(): "+ex);
+	}
+	return thisFilePos;
+    }
+    
+    
+    /**
+     *  Get the location of this file as a URL
+     */
+    public URL getURL()
+    { 
+	return this.location.getLocation();
+    }
+    
+    /**
+     *  Get modified file date
+     */
+    public long getModifiedDatestamp()
+    { 
+	return this.location.getModifiedDatestamp();
+    }
+    
+    /**
+     *  Overridden <code>toString</code> for convenience - returns the location of the file.
+     *
+     *  @return <code>String</code> the location of the file as a string
+     */
+    public String toString()
+    {
+	return this.location.toString();
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileGroup.java	(revision 21112)
@@ -0,0 +1,408 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.net.URL;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.io.PrintWriter;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+/**
+ *  Child groups are indicated by colons.  E.g. Chapter1:Section1 would indicate
+ *  the group "Section1" inside the "Chapter1" group...
+ */
+
+public class METSFileGroup
+{ 
+    List   children;
+    List   childGroups;
+    String id;
+    
+    public static final String SECTION_PARENT = "Section";
+    public static final String GROUP_PARENT = "Group";
+    
+    public METSFileGroup(String id)
+    { 
+	this.children = new ArrayList();
+	this.childGroups = new ArrayList();
+	this.id = id;
+    }
+    
+    /**
+     *  Get the modified datestamp that applies for this file group.  An empty group
+     *  will return <code>0</code>.
+     *
+     *  @return <code>long</code> the date modified as milliseconds from the 1st January 1970
+     */
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0; // empty groups reply '0'
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    long fileStamp = file.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    long fileStamp = fileGroup.getModifiedDatestamp();
+	    if (fileStamp > reply) {
+		reply = fileStamp;
+	    }
+	}
+	
+	return reply;
+    }
+    
+    /**
+     *  Find all the occurrences of a given file, and place them
+     *  in a <code>List</code>.
+     */
+    public void findGroups(URL findFile, List resultList)
+    { 
+	// find in this particular group
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    if (file.equals(findFile))
+		{ resultList.add(this.id);
+		}
+	}
+	
+	// iterate over the child groups
+	childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    fileGroup.findGroups(findFile, resultList);
+	}
+    }
+    
+    /**
+     *  Get a sub group of this METSFileGroup.
+     *
+     *  @param <code>String</code> the name of the subgroup
+     *  @return <code>METSFileGroup</code> the child group, which will
+     *          be <code>null</code> if the group is not known.
+     */
+    public METSFileGroup getSubgroup(String id)
+    { 
+	String childId;
+	
+	int dotAt = id.indexOf(':');
+	if (dotAt == -1){
+	    childId = null;
+	}
+	else{ 
+	    childId = id.substring(dotAt+1);
+	    id = id.substring(0, dotAt);
+	}
+	
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup group = (METSFileGroup) childIter.next();
+	    
+	    if (group.id.equals(id)) {
+		if (childId == null) {
+		    return group;
+		}
+		else {
+		    return group.getSubgroup(childId);
+		}
+	    }
+	}
+	return null;
+    }
+    
+    /**
+     *  Add a filegroup to this group of files.
+     *
+     *	@param <code>METSFileGroup</code> the filegroup.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.childGroups.add(group);
+    }
+    
+    /**
+     *  Add a file to this group of files.
+     *
+     *	@param <code>METSFile</code> the file in a METS File object.
+     */
+    public void addFile(METSFile file)
+    { 
+	this.children.add(file);
+	file.setGroup(this);
+	if (!file.getID().isDefined()){
+	    file.setID(new METSFileID(this.id + "." + Integer.toString(this.children.size())));
+	}
+    }
+    
+    /**
+     *  Get the nth child of the group
+     *
+     *  @param <code>int</code> the index into the group
+     *  @return <code>METSFile</code> the corresponding child.  This may be <code>null</code>
+     *          particularly if the index given falls outside of the valid range of child
+     *          indexes.
+     */
+    public METSFile getFile(int index)
+    { 
+	if (index < 0 || index >= this.children.size()){
+	    return null;
+	}
+	
+	return (METSFile) this.children.get(index);
+    }
+    
+    /**
+     *  Get all the files in the group as a <code>List</code>
+     *
+     *  @return <code>List</code> the file childen of the group.
+     */
+    public List getFiles()
+    { 
+	return this.children;
+    }
+    
+    /**
+     *  Get all the subgroups as a <code>List</code>
+     */
+    public List getSubgroups()
+    { 
+	return this.childGroups;
+    }
+    
+    /**
+     *  Get the number of files in the group
+     *
+     *  @return <code>int</code> the count
+     */
+    public int noOfFiles()
+    { 
+	return this.children.size();
+    }
+    
+    /**
+     *  Get the number of subgroups.
+     */
+    public int noOfSubgroups()
+    { 
+	return this.childGroups.size();
+    }
+    
+    /**
+     *  @return <code>String</code> the name of this group
+     */
+    public String getId()
+    { 
+	return this.id;
+    }
+    
+    public static METSFileGroup parse_groupXML(Element element, METSFileSet set, String parseFilePath, METSFileGroup group)
+    {
+ 	//deal with mets:fileGrp
+ 	NodeList children = element.getChildNodes();
+ 	
+ 	for (int c = 0; c < children.getLength(); c++) {
+ 	    if (children.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+ 	    Element child = (Element) children.item(c);
+ 	    String childName = child.getNodeName();
+	    
+ 	    if (childName.equals("mets:fileGrp")) {
+ 		String groupId = child.getAttribute("ID");
+ 		METSFileGroup childGroup = new METSFileGroup(groupId);
+		
+ 		if (group != null){
+ 		    group.addGroup(childGroup);
+ 		} else {
+ 		    set.addGroup(childGroup);
+ 		}
+ 		METSFile file = METSFile.parse_file(child, childGroup, parseFilePath);
+ 	    }
+ 	}
+ 	return group;
+    }
+
+    /**
+     *  Write the object in an XML METS format.
+     *
+     *  @param <code>PrintWriter</code> the printwriter being used to write the object
+     */
+    public void write(PrintWriter writer)
+    { 
+	String tag = XMLTools.getOpenTag("mets","fileGrp");
+	tag = XMLTools.addAttribute(tag, "ID", this.id);
+	writer.print("   ");   //indentation
+	writer.println(tag);
+	
+	Iterator childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();
+	    file.write(writer);
+	}
+	writer.print("   ");   //indentation
+	writer.println(XMLTools.getCloseTag("mets","fileGrp"));
+    }
+    
+    public boolean writeSQL(DocumentInterface document, String parentRef, boolean parentIsSection, 
+			    GS3SQLConnection connection)
+    { 
+	int sqlId = -1;
+	// check if this node is in the database already
+	GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileGroupID", "=", this.id);
+	GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(whereDoc);
+	select.setWhere(where);
+	
+	ResultSet results = null;
+	Statement statement = null;
+	
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (!results.first()) {
+		GS3SQLInsert insert = new GS3SQLInsert("filegroups");
+		
+		insert.addValue("DocID", document.getID().toString());
+		insert.addValue("FileGroupID", this.id);
+		insert.addValue("ParentRef", parentRef, GS3SQLField.INTEGER_TYPE);
+		insert.addValue("ParentType", parentIsSection ? SECTION_PARENT : GROUP_PARENT);
+		
+		statement.execute(insert.toString());
+	    }
+	    else {
+		// TODO: update the data for this file group...
+	    }
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileGroup.writeSQL(): "+ex);
+	    return false;
+	}
+	
+	// get the filegroup reference now
+	try {
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()) {
+		sqlId = results.getInt("FileGroupRef");
+	    } else {
+		statement.close();
+		return false;
+	    }
+	    statement.close();
+	} catch (SQLException sqlex) {
+	    System.err.println("METSFileGroup.writeSQL(): "+sqlex);
+	    return false;
+	}
+	// iterate over the child groups
+	Iterator childIter = childGroups.iterator();
+	while (childIter.hasNext()){
+	    METSFileGroup fileGroup = (METSFileGroup) childIter.next();
+	    
+	    if (!fileGroup.writeSQL(document, Integer.toString(sqlId), false, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write FileGroup");
+		return false;
+	    }
+	}
+	
+	// iterate over the child files
+	childIter = children.iterator();
+	while (childIter.hasNext()){
+	    METSFile file = (METSFile) childIter.next();	    
+	    if (!file.writeSQL(sqlId, connection)){
+		System.err.println("METSFileGroup.writeSQL(): Couldn't write File");
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileGroup readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	try {
+	    String ID         = resultSet.getString("FileGroupID");
+	    String parentType = resultSet.getString("ParentType");
+	    String parentRef  = resultSet.getString("ParentRef");
+	    
+	    // create the metadata block object
+	    METSFileGroup group = new METSFileGroup(ID);
+	    
+	    // get its metadata reference to retrieve divisions
+	    int groupRef = resultSet.getInt("FileGroupRef");
+	    
+	    // query the database for matching groups which are children of this one
+	    GS3SQLSelect select = new GS3SQLSelect("filegroups");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(groupRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.GROUP_PARENT);
+	    where.add(whereItem);
+	    select.setWhere(where);
+	    
+	    Statement statement = connection.createStatement();
+	    
+	    // parse through the child groups
+	    ResultSet childSet = statement.executeQuery(select.toString());
+	    if (childSet.first()) {
+		do {
+		    METSFileGroup fileGroup = METSFileGroup.readSQL(document, connection, childSet);
+		    if (fileGroup != null) {
+			group.addGroup(fileGroup);
+		    }
+		}
+		while (childSet.next());
+	    }
+	    // now scan for file members
+	    select = new GS3SQLSelect("files");
+	    select.addField("*");
+	    whereItem = new GS3SQLWhereItem("FileGroupRef", "=", Integer.toString(groupRef), GS3SQLField.INTEGER_TYPE);
+	    where = new GS3SQLWhere(whereItem);
+	    select.setWhere(where);
+	    ResultSet childFileSet = statement.executeQuery(select.toString());
+	    if (childFileSet.first()) {
+		do {
+		    METSFile file = METSFile.readSQL(connection, childFileSet);
+		    if (file != null) {
+			group.addFile(file);
+		    }
+		}
+		while (childFileSet.next());
+	    }
+	    statement.close();
+	    return group;
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSFileGroup.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileID.java	(revision 21112)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A type-safe wrapper for METS File identifiers.
+ */
+
+public class METSFileID
+{	
+    String id;
+    
+    public METSFileID()
+    {	
+	this.id = null;
+    }
+    
+    public METSFileID(String id)
+    {	
+	this.id = id;
+    }
+    
+    public String toString()
+    {	
+	return this.id;
+    }
+    
+    public boolean isDefined()
+    {	
+	return this.id != null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePointer.java	(revision 21112)
@@ -0,0 +1,27 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ or AREA 
+ *  file pointer types hence, one can use only a simple file reference
+ */
+
+public class METSFilePointer
+{
+	public static final String SIMPLE = "";
+	public static final String SEQUENCE = "seq";
+	public static final String AREA = "area";
+
+	String type;
+	String ID;
+	String fileId;
+	List   children;
+
+	public METSFilePointer(String ID, String fileId)
+	{	this.ID = ID;
+		this.fileId = fileId;
+		this.type = METSFilePointer.SIMPLE;
+	}	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFilePos.java	(revision 21112)
@@ -0,0 +1,41 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+
+public class METSFilePos
+{
+  METSLocation location;
+
+  public METSFilePos(File file) throws java.net.MalformedURLException
+  { this.location = new METSLocation(file);
+  }
+
+  public METSFilePos(URL url)
+  { this.location = new METSLocation(url);
+  }
+
+  public METSFilePos(String locationType, String location) throws java.net.MalformedURLException
+  { this.location = new METSLocation(locationType, location);
+  }
+  
+  public URL getLocation()
+  { return this.location.getLocation();
+  }
+
+  public String getType()
+  { return this.location.getType();
+  }
+
+  public long getModifiedDatestamp()
+  { return this.location.getModifiedDatestamp();
+  }
+
+  public String toString()
+  { return this.location.getLocation().toString();
+  }
+
+  public boolean equals(URL filePos)
+  { return this.location.equals(filePos);
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileRef.java	(revision 21112)
@@ -0,0 +1,13 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSFileRef
+{	String label;
+
+	public METSFileRef(String label)
+	{	this.label = label;
+	}
+
+	public String toString()
+	{	return this.label;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSFileSet.java	(revision 21112)
@@ -0,0 +1,344 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A class to hold all the alternative file structures
+ *  for a particular METS Document
+ */
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.io.File;
+import java.io.PrintWriter;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+import java.net.URL;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSFileSet
+{ 
+    Map    fileGroups;
+    String reference;
+    FileIdentifierFactory identifierFactory;
+    FileGroupIdentifierFactory groupIdentifierFactory;
+    
+    class FileGroupIdentifierFactory extends AbstractIdentifierFactory
+    {
+	public static final String FILEGROUP_PRELUDE = "FG";
+	
+	public FileGroupIdentifierFactory()
+	{
+	    super("FILEGROUP_PRELUDE");
+	}
+    }
+    
+    public METSFileSet()
+    { 
+	this.fileGroups = new HashMap();
+	this.fileGroups.put("default", new METSFileGroup("default"));
+	this.reference = null;
+	this.identifierFactory = new FileIdentifierFactory();
+	this.groupIdentifierFactory = new FileGroupIdentifierFactory();
+    }
+    
+    public long getModifiedDatestamp()
+    { 
+	long reply = 0;
+	
+	Iterator groupsIter = this.fileGroups.values().iterator();
+	while (groupsIter.hasNext()) {
+		METSFileGroup group = (METSFileGroup) groupsIter.next();
+		long groupStamp = group.getModifiedDatestamp();
+		if (groupStamp > reply){
+		    reply = groupStamp;
+		}
+	    }
+	return reply;
+    }
+    
+    
+    /**
+     *  Get a new FileGroup to lie within this fileset.  The group is not initialised with
+     *  a parent, etc. - all that is done is the allocation of a unique group identifier.
+     *
+     *  @return <code>METSFileGroup</code> the new group.
+     */
+    public METSFileGroup createGroup()
+    { 
+	METSFileGroup group = new METSFileGroup(this.groupIdentifierFactory.getNextIdentifier());
+	return group;
+    }
+    
+    public METSFile createFile(METSFilePos filePos, String type)
+    { 
+	METSFile file = new METSFile(new METSFileID(this.identifierFactory.getNextIdentifier()), filePos, type);
+	return file;
+    }
+    
+    /**
+     *  Get the group that corresponds to the given name...
+     *
+     *  @param <code>String</code> the name of the group.
+     *  @return <code>METSFileGroup</code> the group object - this will be
+     *          <code>null</code> if the group is not found
+     */
+    public METSFileGroup getGroup(String name)
+    { 
+	if (!this.fileGroups.containsKey(name)){
+	    return null;
+	}
+	return (METSFileGroup) this.fileGroups.get(name);
+    }
+    
+    /**
+     *  Get the Nth file from the default group...
+     *
+     *  @param <code>int</code> the index into the default group to use...
+     *  @return <code>METSFile</code> the file.
+     */
+    public METSFile getFile(int index)
+    { 
+	METSFileGroup group = this.getGroup("default");
+
+	if (group == null) {
+	    return null;
+	}
+	return group.getFile(0);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     */
+    public void addFile(METSFile file)
+    { 
+	METSFileGroup defaultGroup = (METSFileGroup) this.fileGroups.get("default");
+	defaultGroup.addFile(file);
+    }
+    
+    /**
+     *  Add a file to the default file group
+     *
+     *  @return <code>METSFile</code> an object that wraps the given url into
+     *          a METS file object
+     */
+    public METSFile addFile(URL url)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file);
+	return file;
+    }
+    
+    public void addFile(METSFile file, String toGroup)
+    { 
+	METSFileGroup destGroup = (METSFileGroup) this.fileGroups.get(toGroup);
+	
+	if (destGroup == null){
+	    destGroup = new METSFileGroup(toGroup);
+	    this.fileGroups.put(destGroup.getId(), destGroup);
+	}
+	
+	destGroup.addFile(file);
+    }
+    
+    public void addFile(URL url, String toGroup)
+    { 
+	METSFile file = METSFile.makeMETSFile(url);
+	this.addFile(file, toGroup);
+    }
+    
+    /**
+     *  Add a group at the top level of the METS File set.
+     *
+     *  @param <code>METSFileGroup</code> the file group to add.
+     */
+    public void addGroup(METSFileGroup group)
+    { 
+	this.fileGroups.put(group.getId(), group);
+    }
+    
+    /**
+     *  Get all the groups that contain a given URL/file.
+     *
+     *  @param <code>URL</code> the location of the file.
+     *  @return <code>List</code> the list of group references that contain the file.
+     */
+    public List findGroups(URL file)
+    { 
+	List resultList = new ArrayList();
+	
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.findGroups(file, resultList);
+	}
+	return resultList;
+    }
+    
+    /*
+     *
+     *
+     */
+    public static METSFileSet parseXML(NodeList fileSecs, METSFileSet fileSet, String parseFilePath) 
+    {
+ 	//METSFileSet set = new METSFileSet();> 
+ 	// this is in effect a group without a sense of 'self'...
+ 	for (int c = 0; c < fileSecs.getLength(); c++) {
+ 	    if (fileSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		continue;
+ 	    }
+	    
+ 	    METSFileGroup groups = METSFileGroup.parse_groupXML((Element)fileSecs.item(c), fileSet, parseFilePath, null);
+ 	}
+ 	return fileSet;
+    }
+    
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	writer.println("<mets:fileSec>");
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:fileSec>");
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator groups = this.fileGroups.values().iterator();
+	
+	// write the set if the reference does not already exist...
+	if (this.reference == null){
+	    // Insert the file section into the database
+	    GS3SQLInsert insert = new GS3SQLInsert("filesection");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("FileSecID", "test"); // TODO: remove magic string
+	    
+	    Statement statement = null;
+	    try {
+		statement = connection.createStatement();
+		statement.execute(insert.toString());
+	    } catch (SQLException e) {
+		System.err.println("METSFileSet.writeSQL(): "+e);
+		return false;
+	    }
+	    
+	    // find the file section number
+	    GS3SQLSelect select = new GS3SQLSelect("filesection");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("FileSecID", "=", "test");
+	    GS3SQLWhereItem whereDoc  = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    where.add(whereDoc);
+	    select.setWhere(where);
+	    
+	    try {
+		ResultSet set = statement.executeQuery(select.toString());
+		if (set.first()) {
+		    int sectionRef = set.getInt("FileSectionRef");
+		    
+		    this.reference = Integer.toString(sectionRef);
+		}
+		statement.close();
+	    }
+	    catch (SQLException ex){
+		System.out.println("METSFileSet.writeSQL(): "+ex);
+		return false;
+	    }
+	}
+	
+	if (this.reference == null) {
+	    return false;
+	}
+	// write out the children
+	while (groups.hasNext()){
+	    METSFileGroup group = (METSFileGroup) groups.next();
+	    
+	    if (!group.writeSQL(document, this.reference, true, connection)){
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSFileSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSFileSet set = new METSFileSet();    
+	
+	// Get file sections from the filesection table (currently redundant)
+	GS3SQLSelect select = new GS3SQLSelect("filesection");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+
+	Statement statement = null;
+	
+	// Get the identifier for this file set, etc.
+	int fileSetRef = -1;
+	try {
+	    statement = connection.createStatement();
+	    ResultSet sections = statement.executeQuery(select.toString());
+	    if (sections.first()) {
+		fileSetRef = sections.getInt("FileSectionRef");
+		set.reference = Integer.toString(fileSetRef);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    
+	}
+	catch (SQLException ex){
+	    System.err.println("METSFileSet.readSQL() "+ex);
+	    return null;
+	}
+	// Get child file groups
+	select = new GS3SQLSelect("filegroups");
+	select.addField("*");
+	whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());    
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(fileSetRef), 
+					GS3SQLField.INTEGER_TYPE);
+	where.add(whereItem);
+	whereItem = new GS3SQLWhereItem("ParentType", "=", METSFileGroup.SECTION_PARENT);
+	where.add(whereItem);
+	select.setWhere(where);
+	// start going through the matching file groups
+	try {
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSFileGroup filegroup = METSFileGroup.readSQL(document, connection, resultSet);
+		    if (filegroup != null) {
+			set.addGroup(filegroup);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.out.println("METSFileSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSHeader.java	(revision 21112)
@@ -0,0 +1,32 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class METSHeader
+{
+    List agents;
+    METSDate createDate;
+    METSDate lastModDate;
+    
+    public METSHeader()
+    {
+	this.agents = new ArrayList();
+    }
+    
+    public METSHeader(METSAgent defaultAgent)
+    {	
+	this();
+	this.agents.add(defaultAgent);
+    }
+    
+    public void setCreateDate(METSDate createDate)
+    {	
+	this.createDate = createDate;
+    }
+    
+    public void setModifiedDate(METSDate modifiedDate)
+    {	
+	this.lastModDate = modifiedDate;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSLocation.java	(revision 21112)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.greenstone.gsdl3.gs3build.util.URLTools;
+
+public class METSLocation
+{
+  String type;
+  URL    location;
+
+  public METSLocation(String type, String location) throws java.net.MalformedURLException
+  { this.type = type;
+    this.location = new URL(location);
+  }
+
+  public METSLocation(URL url)
+  { this.type = "URL";
+    this.location = url;
+  }
+
+  public METSLocation(File file)
+  { this.type = "URL";
+    this.location = URLTools.getFileURL(file);
+  }
+
+  public boolean equals(URL url)
+  { return this.location.equals(url);
+  }
+
+  public URL getLocation()
+  { return this.location;
+  }
+
+  public long getModifiedDatestamp()
+  { if (this.location != null) {
+      if (this.location.getProtocol().equals("file")) {
+	File file = new File(this.location.getPath());
+	return file.lastModified();
+      }
+      else {
+	try {
+	  URLConnection connection = this.location.openConnection();
+	  return connection.getLastModified();
+	}
+	catch (java.io.IOException io)
+	{ // TODO: report error
+	}
+      }
+    }
+    return System.currentTimeMillis();
+  }
+
+  public String getType()
+  { return this.type;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSMetaParams.java	(revision 21112)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class METSMetaParams
+{
+	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSNamespace.java	(revision 21112)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.Iterator;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a single namespace
+ */
+
+public abstract class METSNamespace
+{
+    protected String       type;        // the type of the metadata
+    protected String       otherType;   // if a non-standard type, then...
+    protected String       name;        // e.g. "DC" for Dublin Core
+    protected METSLocation location;    // the location of the corresponding file
+    protected String       group;       // the group of metadata namespaces to which
+    // this namespace belongs
+    protected String       id;          // the ID for this namespace
+
+    public METSNamespace(String name)
+    { 
+	this.name        = name;
+	this.location    = null;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    public METSNamespace(String name, METSLocation location)
+    { 
+	this.location    = location;
+	this.name        = name;
+	this.group       = null;
+	this.id          = null;
+    }
+    
+    /**
+     *  Get the name of the namespace...
+     *
+     *  @return <code>String</code> the name of the namespace
+     */
+    public String getName()
+    { 
+	return this.name;
+    }
+    
+    /**
+     *  Get the group of namespaces to which this namespace belongs.
+     *
+     *  @return <code>String</code> the name of the namespace group
+     */
+    public String getGroup()
+    {
+	return this.group;
+    }
+    
+    /**
+     *  Set the group of namespaces to which this namespace belongs.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setGroup(String group)
+    { 
+	this.group = group;
+    }
+    
+    /**
+     *  Get the ID for this namespace.
+     *
+     *  @return <code>String</code> the id of the namespace
+     */
+    public String getID()
+    {
+	return this.id;
+    }
+    
+    /**
+     *  Set the id of this namespace.
+     *
+     *  @param <code>String</code> the name of the namespace group
+     */
+    public void setID(String id)
+    {
+	this.id = id;
+    }
+
+    /**
+     *  <p>Indicate whether this namespace is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+    
+    public abstract boolean addMetadata(String label, String value);
+    public abstract boolean setMetadata(String label, String value);
+    public abstract boolean removeMetadata(String label);
+    public abstract boolean removeMetadata(String label, String value);
+    public abstract List    getMetadata(String label);
+    public abstract Iterator getMetadataNames();
+    public abstract boolean write(PrintWriter writer);
+    //  public abstract boolean writeSQL(GS3SQLConnection connection);
+
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>int</code> the sql identifier of the parent object (in "metadata") table.
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	GS3SQLAction action;
+
+	// If the namespace is not null, then set it...
+	if (this.id != null) {
+	    // use an update action in this case...
+	    GS3SQLUpdate update = new GS3SQLUpdate("namespaces");
+	    update.addValue("NamespaceID", this.id);
+
+	    // set up the where clause
+	    GS3SQLWhere where = 
+		new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", this.id, 
+						    GS3SQLField.INTEGER_TYPE));
+	    update.setWhere(where);
+	    action = update;
+	}
+	else {
+	    GS3SQLInsert insert = new GS3SQLInsert("namespaces");
+	    action = insert;
+	}
+
+	if (this.location != null) {
+	    action.addValue("FileLoc", this.location.getLocation().toString());
+	    action.addValue("FileType", "URL");
+	}
+	else {
+	    // no location stuff
+	}
+	action.addValue("MetadataRef", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	action.addValue("NamespaceType", this.name);
+
+	// Execute the action
+	Statement statement = null;
+	try {
+	    statement = connection.createStatement();
+	    statement.execute(action.toString());
+	} catch (SQLException e) {
+	    System.err.println("METSNamespace.writeSQL(): "+e);
+	    return false;
+	}
+	// then get the namespace reference number if needsbe...
+	if (this.id == null) {
+	    GS3SQLSelect select = new GS3SQLSelect("namespaces");
+	    select.addField("NamespaceRef");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("NamespaceType", "=", this.name);
+	    where.add(where);
+	    try {
+		ResultSet result  = statement.executeQuery(select.toString());
+      		if (result.last()) {
+		    this.id = Integer.toString(result.getInt("NamespaceRef"));
+		}
+		statement.close();
+	    }
+	    catch (SQLException sqlex){
+		this.id = null;
+		System.err.println("METSNamespace.writeSQL(): "+sqlex);
+		return false;
+	    }
+	}
+	
+	return true;
+    }
+    
+    public static METSNamespace readSQL(GS3SQLConnection connection, ResultSet resultSet)
+    {
+	METSLocation metsLocation = null;
+	
+	try {
+	    String name = resultSet.getString("NamespaceType");
+	    String id = resultSet.getString("NamespaceRef");
+	    String location = resultSet.getString("FileLoc");
+	    String type = resultSet.getString("FileType");
+	    if (location != null && type != null) {
+		metsLocation = new METSLocation(type, location);
+	    }
+	    
+	    METSNamespace namespace = NamespaceFactory.initNamespace(name, metsLocation);
+	    namespace.id = id;
+
+	    int namespaceRef = resultSet.getInt("NamespaceRef");
+      
+	    GS3SQLSelect select = new GS3SQLSelect("mdvalues");
+	    select.addField("*");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", Integer.toString(namespaceRef),
+								    GS3SQLField.INTEGER_TYPE));
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet valuesSet = statement.executeQuery(select.toString());
+	    if (valuesSet.first()) {
+		do {
+		    String label = valuesSet.getString("Label");
+		    String value = valuesSet.getString("Value");
+
+		    namespace.addMetadata(label, value);
+		}
+		while (valuesSet.next());
+	    }
+	    statement.close();
+	    return namespace;
+	}
+	catch (java.net.MalformedURLException urlEx){
+	    System.err.println("METSNamespace.readSQL(): "+urlEx);
+	}
+	catch (SQLException sqlEx){
+	    System.err.println("METSNamespace.readSQL(): "+sqlEx);
+	}    
+	return null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSPointer.java	(revision 21112)
@@ -0,0 +1,19 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ *  Notes: Greenstone doesn't currently use/support the SEQ file pointer type;
+ *         hence, one can use only 'area' or 
+ */
+
+public class METSPointer
+{
+	String ID;
+	METSLocation location; // Or something similar...?
+
+	public METSPointer ()
+	{
+	}	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructure.java	(revision 21112)
@@ -0,0 +1,323 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructure extends AbstractStructure
+{
+    String label;
+    String type;
+    METSDivision divGroup;
+
+    public static final String STRUCTURE_TYPE = "Structure";
+  
+    public METSStructure(String id, String label, String type)
+    { 
+	super(id);
+	
+	this.label    = label;
+	this.type     = type;
+	this.divGroup = null;
+    }
+
+    /**
+     *  Show what sort of <code>AbstractStructure</code> this structure is...
+     *
+     *  @return <code>String The identifying string for a <code>METSStructure</code> type.
+     */
+    public String getStructureType()
+    { return STRUCTURE_TYPE;
+    }
+  
+    /**
+     *  Find all the divisions that match a given list of file group
+     *  identifiers...
+     *
+     *  @param <code>List</code> the list of identifiers to find
+     *  @param <code>List</code> the list into which to place any
+     *         matching identifiers.
+     */
+    public void findDivisionsForFiles(List listOfFileIdentifiers, List resultList)
+    { Iterator groups = this.children.values().iterator();
+    while (groups.hasNext())
+	{ METSDivision group = (METSDivision) groups.next();
+
+	group.findDivisionsForFiles(listOfFileIdentifiers, resultList);
+	}    
+    }
+
+    /**
+     *  Add a division further into this structure.
+     *
+     *  @param <code>String</code> the identifier label for the division
+     *         within which the division is to be added.
+     *  @param <code>METSDivision</code> the division to add.   
+     */
+    public void addSubDivision(String divisionLabel, METSDivision division)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ return;
+	}
+
+    child = child.getDivision(divisionLabel);
+
+    if (child != null) {
+	child.addDivision(division);
+    }
+    }
+
+    /**
+     *  Get a subdivision of a given identifier...
+     * 
+     *  @param <code>String</code> the identifier of the division
+     *  @return <code>METSDivision</code> the division, which will be
+     *          <code>null</code> if it is not found.
+     */
+    public METSDivision getDivision(String divisionLabel)
+    { String topDivision = METSDivision.getTopDivisionName(divisionLabel);
+
+    if (!topDivision.equals("All"))
+	System.out.println("Top division is " + topDivision);
+    METSDivision child = (METSDivision) this.children.get(topDivision);
+    if (child == null)
+	{ System.out.println("No child found");
+	Iterator groups = this.children.keySet().iterator();
+	while (groups.hasNext()) {
+	    System.out.println(groups.next().toString());
+	}
+	return null;
+	}
+
+
+    child = child.getDivision(divisionLabel);
+
+    return child;
+    }
+
+
+    /*
+     *
+     * parse an XML Element as a METSStructure Section
+     *
+     * @param <code>Element</code> the XML element which represents
+     *  the mets:structMap itself 
+     */
+    
+    public static METSStructure parseXML(Element element) 
+    {
+ 	String structId = element.getAttribute("ID");
+ 	String structLabel = element.getAttribute("LABEL");
+	String structType = element.getAttribute("TYPE");
+ 
+ 	METSStructure thisStruct = new METSStructure (structId, structLabel, structType);
+ 	
+ 	NodeList divSecs = element.getChildNodes();
+ 	if (divSecs != null) {
+ 	    for (int c = 0; c < divSecs.getLength(); c++) {
+ 		if (divSecs.item(c).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE){
+ 		    continue;
+ 		}
+ 		Element divElement = (Element) divSecs.item(c);
+ 		String divName = divElement.getNodeName();
+ 		if (divName.equals("mets:div")) {
+ 		    METSDivision division = METSDivision.parseXML(divElement);
+ 		    thisStruct.addDivision(division);
+ 		} else {
+ 		    System.err.println("Error: mets:structMap was expecting mets:div but got "+divName);
+ 		}
+ 	    }
+ 	}
+ 	return thisStruct;
+    }
+ 
+    /**
+     *  Write this structure in XML(METS) format to a <code>PrintWriter</code>
+     */
+    public void write(PrintWriter writer)
+    { 
+	Iterator groups = this.children.values().iterator();
+	
+	String tag = XMLTools.getOpenTag("mets", "structMap");
+	tag = XMLTools.addAttribute(tag, "ID", this.ID.toString());
+	tag = XMLTools.addAttribute(tag, "TYPE", this.type);
+	tag = XMLTools.addAttribute(tag, "LABEL", this.label);
+	
+	writer.println(tag);
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    group.write(writer);
+	}
+	writer.println("</mets:structMap>");
+    }
+    
+    
+    /**
+     *  Write this METSStructure to an SQL database....
+     *
+     *  @param <code>DocumentInterface</code> the enclosing document.
+     *  @param <code>GS3SQLConnection</code> the SQL database connection.
+     */
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	int sqlRef = -1;
+	ResultSet results = null;
+	Statement statement = null;
+	
+	// Prepare query to see if this structure has already been written
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("StructureRef");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	GS3SQLWhereItem item = new GS3SQLWhereItem("StructureID", "=", this.ID.toString());
+	GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	where.add(item);
+	select.setWhere(where);
+	
+	// attempt to execute the query & get the current structure's reference
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(select.toString());
+	    if (results.first()){
+		sqlRef = results.getInt("structureRef");
+	    }
+	}
+	catch (SQLException sqlEx) {
+	    System.err.print("METSStructure.writeSQL(): "+sqlEx);
+	    return false;
+	}
+	
+	// insert a new structure if it wasn't there previously
+	if (sqlRef == -1) {
+	    GS3SQLInsert insert = new GS3SQLInsert("structure");
+	    insert.addValue("DocID", document.getID().toString());
+	    insert.addValue("StructureID", this.ID.toString());
+	    insert.addValue("StructureType", this.type);
+	    insert.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(insert.toString());
+		
+		// get the new structure reference by re-running the original select object
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlRef = results.getInt("StructureRef");
+		} 
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	    
+	}
+	else {
+	    GS3SQLUpdate update = new GS3SQLUpdate("structure");
+	    update.setWhere(where);
+	    
+	    update.addValue("StructureType", this.type);
+	    update.addValue("Label", this.label);
+	    
+	    try {
+		statement.execute(update.toString());
+	    } catch (SQLException sqlEx) {
+		System.err.print("METSStructure.writeSQL(): "+sqlEx);
+		return false;
+	    }
+	}
+	
+	// close the statement
+	try {
+	    statement.close();
+	} catch (SQLException e) {
+	    System.err.println("METSStructure.writeSQL(): "+e);
+	}
+	// write out the child groups (Divisions) now...    
+	Iterator groups = this.children.values().iterator();
+	
+	while (groups.hasNext()){
+	    METSDivision group = (METSDivision) groups.next();
+	    
+	    if (!group.writeSQL(document.getID(), sqlRef, true, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+    
+    /**
+     *  Read a METSStructure from a database - on entry, the current item in the
+     *  <code>ResultSet</code> holds the row in the database with the structure's
+     *  data.
+     *
+     *  @param <code>DocumentInterface</code> the document which owns the structure
+     *  @param <code>GS3SQLConnection</code> the database connection itself, to load
+     *         child objects through, etc.
+     *  @param <code>ResultSet</code> pointing to the current row in the database.
+     */
+    public static METSStructure readSQL(DocumentInterface document, GS3SQLConnection connection,
+					ResultSet resultSet)
+    {
+	GS3SQLSelect select = null;
+	
+	try {
+	    String ID    = resultSet.getString("StructureID");
+	    String type  = resultSet.getString("StructureType");
+	    String label = resultSet.getString("Label");
+
+	    // create the metadata block object
+	    METSStructure structure = new METSStructure(ID, label, type);
+      
+	    // get its metadata reference to retrieve divisions
+	    int structureRef = resultSet.getInt("StructureRef");
+
+	    // query the database for matching division for this structure block
+	    select = new GS3SQLSelect("divisions");
+	    select.addField("*");
+	    GS3SQLWhereItem whereItem = new GS3SQLWhereItem("ParentRef", "=", Integer.toString(structureRef),
+							    GS3SQLField.INTEGER_TYPE);
+	    GS3SQLWhere where = new GS3SQLWhere(whereItem);
+	    whereItem = new GS3SQLWhereItem("ParentType", "=", METSStructure.STRUCTURE_TYPE);
+	    where.add(whereItem);
+	    select.setWhere(where);
+
+	    Statement statement = connection.createStatement();
+	    ResultSet divisionSet = statement.executeQuery(select.toString());
+
+	    // parse through the divisions
+	    if (divisionSet.first()) {
+		do {
+		    METSDivision division = METSDivision.readSQL(connection, divisionSet);
+		    if (division != null) {
+			structure.addDivision(division);
+		    }
+		}
+		while (divisionSet.next());
+	    }
+	    statement.close();
+	    return structure;
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructure.readSQL(): "+ sqlEx + " " + select.toString());
+	    System.exit(1);
+	}
+	return null;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/METSStructureSet.java	(revision 21112)
@@ -0,0 +1,181 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+
+import org.greenstone.gsdl3.gs3build.database.*;
+
+public class METSStructureSet
+{ 
+    Map children;
+
+    public static final String GSDL3_SECTION_STRUCTURE = "Section";
+
+    public METSStructureSet()
+    { 
+	this.children = new HashMap();
+    }
+
+    public void addStructure(METSStructure structure)
+    { 
+	this.children.put(structure.getID(), structure);
+    }
+
+    public METSStructure getStructure(String name)
+    { 
+	return (METSStructure) this.children.get(name);
+    }
+
+    /**
+     *  Find the divisions that contain a list of given files
+     *
+     *  @param <code>List</code> the list of files to find...
+     *  @return <code>List</code> the matching divisions.
+     */
+    public List findDivisionsForFiles(List listOfFileIdentifiers)
+    { 
+	List reply = new ArrayList();
+
+	// iterate across all the children...
+	Iterator structures = this.children.values().iterator();
+	
+	while (structures.hasNext()){
+	    METSStructure structure = (METSStructure) structures.next();
+	    
+	    structure.findDivisionsForFiles(listOfFileIdentifiers, reply);      
+	}    
+	return reply;
+    }
+    
+    /**
+     *  Add a division to an indicated structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addDivision(String structureId, METSDivision division)
+    {
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+	
+	if (structure != null){
+	    structure.addDivision(division);
+	}
+    }
+
+    /**
+     *  Add a division to an indicated division within structure.
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer, within which the division
+     *         is to be placed.
+     *  @param <code>METSDivision</code> the division to be added.
+     */
+    public void addSubDivision(String structureId, String divisionID, METSDivision division)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    structure.addSubDivision(divisionID, division);
+	}
+    }
+
+    /**
+     *  Get a division from within a given structure
+     *
+     *  @param <code>String</code> the structure identifier
+     *  @param <code>String</code> the division identifer
+     *
+     *  @return <code>METSDivision</code> the division
+     */
+    public METSDivision getDivision(String structureId, String divisionId)
+    { 
+	METSStructure structure = (METSStructure) this.children.get(structureId);
+
+	if (structure != null){
+	    return structure.getDivision(divisionId);
+	}
+	return null;
+    }
+    public static METSStructureSet parseXML(NodeList structMapSecs)
+    {
+ 	METSStructureSet set = new METSStructureSet();
+ 
+ 	for (int c = 0; c < structMapSecs.getLength(); c++) {
+ 	    METSStructure structure = METSStructure.parseXML((Element) structMapSecs.item(c));
+ 	    set.addStructure(structure);
+ 	}
+ 	return set;
+    }
+        
+
+    public void write(PrintWriter writer)
+    { 
+	Iterator structures = this.children.values().iterator();
+
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+	    
+	    group.write(writer);
+	}
+    }
+    
+    public boolean writeSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	Iterator structures = this.children.values().iterator();
+    
+	while (structures.hasNext()){
+	    METSStructure group = (METSStructure) structures.next();
+
+	    if (!group.writeSQL(document, connection)){
+		return false;
+	    }
+	}
+	return true;
+    }
+
+    public static METSStructureSet readSQL(DocumentInterface document, GS3SQLConnection connection)
+    { 
+	METSStructureSet set = new METSStructureSet();
+
+	GS3SQLSelect select = new GS3SQLSelect("structure");
+	select.addField("*");
+	GS3SQLWhereItem whereItem = new GS3SQLWhereItem("DocID", "=", document.getID().toString());
+	select.setWhere(new GS3SQLWhere(whereItem));
+	
+	// start going through the matching metadata blocks
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet resultSet = statement.executeQuery(select.toString());
+	    if (resultSet.first()) {
+		do {
+		    METSStructure structure = METSStructure.readSQL(document, connection, resultSet);
+		    if (structure != null) {
+			set.addStructure(structure);
+		    }
+		} while (resultSet.next());
+	    }
+	    statement.close();
+	}
+	catch (SQLException sqlEx) {
+	    System.err.println("METSStructureSet.readSQL(): "+sqlEx);
+	    System.exit(1);
+	}
+	return set;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaElement.java	(revision 21112)
@@ -0,0 +1,66 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.schema.Schema;
+import org.greenstone.gsdl3.gs3build.schema.SchemaNode;
+
+public class MetaElement
+{
+	Map    attributes;
+	List   children;
+	String name;
+
+	public MetaElement(Element element, Schema schema)
+	{	
+		// basic setup
+		this.attributes = new HashMap();
+		this.children   = new ArrayList();
+		this.name = element.getNodeName();
+		StringBuffer contentBuffer = new StringBuffer();
+		
+		// split the name into namespace/workspace
+		String workSpace = SchemaNode.getNamespace(name);
+		name = SchemaNode.getElementName(name);
+
+		// get attributes
+		NamedNodeMap attributes = element.getAttributes();
+		for (int a = 0; a < attributes.getLength(); a ++)
+		{	Node attribute = attributes.item(a);
+
+			this.attributes.put(attribute.getNodeName(), attribute.getNodeValue());
+		}
+
+		// get children
+		NodeList children = element.getChildNodes();
+		for (int c = 0; c < children.getLength(); c++)
+		{	Node child = children.item(c);
+
+			if (child.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE)
+			{	// make a child
+				if (contentBuffer.length() > 0) {
+					this.children.add(contentBuffer.toString());
+					contentBuffer.setLength(0);
+				}
+				Object childElement = new MetaElement((Element) child, schema);
+				this.children.add(childElement);
+			}
+			else
+			{	// append to content if not an element - this may be wasteful & type of
+				// element should be checked
+ 				contentBuffer.append(child.getNodeValue());
+			}
+		}
+		if (contentBuffer.length() > 0) {
+			this.children.add(contentBuffer.toString());
+		}
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetaTools.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+
+/**
+ *  A class which provides metadata-centric tools for handling XML objects
+ */
+public class MetaTools
+{
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/MetadataLabel.java	(revision 21112)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+public class MetadataLabel
+{
+  String namespace;
+  String label;
+
+  public MetadataLabel(String label)
+  { int colonAt = label.indexOf(":");
+    String namespace;
+		
+    if (colonAt > 0) {
+      this.namespace = label.substring(0, colonAt);
+      this.label     = label.substring(colonAt+1);
+    }
+    else {
+      this.namespace = GSDL3Namespace.GSDL3_NAMESPACE_ID;
+      this.label = label;
+    }
+  }
+
+  public String getNamespace()
+  { return this.namespace;
+  }
+
+  public String getLabel()
+  { return this.label;
+  }
+
+  public static String getDefaultNamespace()
+  { return GSDL3Namespace.GSDL3_NAMESPACE_ID;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceFactory.java	(revision 21112)
@@ -0,0 +1,71 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class NamespaceFactory
+{
+    public static METSNamespace initNamespace()
+    {	
+	return new GSDL3Namespace();
+    }
+    
+    public static METSNamespace initNamespace(String name)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)) {
+	    return new GSDL3Namespace();
+	}
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace initNamespace(String name, METSLocation location)
+    {	
+	if (name.equals(GSDL3Namespace.GSDL3_NAMESPACE_ID)){
+	    return new GSDL3Namespace(location);
+	}
+	return new SimpleNamespace(name, location);
+    }
+    
+    /**
+     *  TODO: parse the document in based upon its namespace
+     */
+    public static METSNamespace initNamespace(String name, Element element)
+    {	
+	//return null;
+	return new SimpleNamespace(name);
+    }
+    
+    public static METSNamespace parseXML(Element element)
+    {
+	METSNamespace namespace = null;
+	
+	//String mdType = element.getAttribute("MDType");
+	//String mdType =element.getAttribute("MDTYPE");
+
+	String othermdType = element.getAttribute("OTHERMDTYPE");
+	String mimeType = element.getAttribute("mimeType");
+	
+	if (element.getNodeName().equals("mets:mdWrap")) {
+	    namespace = initNamespace(othermdType, element);
+	}
+	else if (element.getNodeName().equals("mets:mdRef")) {
+	    String mdType = element.getAttribute("MDType");
+	    // just a file wrapper on the metadata...
+	    String location = element.getAttribute("xlink:href");
+	    String locationType = element.getAttribute("LOCTYPE");
+	    
+	    try {
+		METSLocation metsLocation = new METSLocation(locationType, location);
+		namespace = initNamespace(mdType, metsLocation);
+	    }
+	    catch (java.net.MalformedURLException ex) {
+		// TODO: Error, should never happen in the case of Greenstone-exported XML
+	    }
+	}
+	return namespace;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/NamespaceItem.java	(revision 21112)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  A simple metadata item with a label and one value to fit...
+ */
+
+public class NamespaceItem
+{
+	private String label;
+	private String value;
+
+	public NamespaceItem(String label, String value)
+	{	this.label = label;
+		this.value = value;
+	}
+
+	public String getLabel()
+	{	return this.label;
+	}
+
+	public String getValue()
+	{	return this.value;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/OrderedNamespace.java	(revision 21112)
@@ -0,0 +1,259 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+
+/**
+ *  Implement a generic, ordered namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class OrderedNamespace extends METSNamespace
+{
+    List metadataList;
+    
+    public OrderedNamespace(String name)
+    { 
+	super(name);
+	this.metadataList = new ArrayList();
+    }
+    
+    public OrderedNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataList = new ArrayList();
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataList.add(new NamespaceItem(label, value));
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)){
+	    return false;
+	}
+	
+	// TODO:		this.metadataMap.setOnly(label, value);
+	int itemNo = this.findItem(label);
+	if (itemNo >= 0){
+	    this.metadataList.set(itemNo, new NamespaceItem(label, value));
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	while (itemNo >= 0){
+	    this.metadataList.remove(itemNo);
+	    itemNo = this.findItem(label);
+	}
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	int itemNo;
+	
+	itemNo = this.findItem(label);
+	if (itemNo >= 0) {
+	    this.metadataList.remove(itemNo);
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for - must be devoid
+     *                             of namespace prologue (i.e. "title" rather than
+     *                             e.g. "dc:title").
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code> if no values are found for the metadata
+     */
+    public List getMetadata(String label)
+    { 
+	List resultList = new ArrayList();
+	
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    
+	    if (item.getLabel().equals(label)){
+		resultList.add(item.getValue());
+	    }
+	}
+	if (resultList.size() > 0){
+	    return resultList;
+	}
+	return null;
+    }
+    
+    private int findItem(String label)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++) {
+	    if (((NamespaceItem) this.metadataList.get(i)).getLabel().equals(label)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    private int findItem(String label, String value)
+    { 
+	for (int i = 0; i < this.metadataList.size(); i ++){
+	    NamespaceItem item = (NamespaceItem) this.metadataList.get(i);
+	    if (item.getLabel().equals(label) && item.getValue().equals(value)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataList.iterator();
+    }
+    
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    Iterator items = this.metadataList.iterator();
+	    
+	    while (items.hasNext()){
+		NamespaceItem item = (NamespaceItem) items.next();
+		this.writeItem(writer, item);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println("</mets:mdWrap>");
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets", "mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+		tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);
+		//tag = XMLTools.addAttribute (tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name); 
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		tag = XMLTools.makeSingleton(tag);
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, NamespaceItem item)
+    { 
+	// Do some default sillinesses
+	//writer.write("         "); 
+	writer.write(XMLTools.getOpenTag(this.name, item.getLabel()));
+	
+	writer.write(item.getValue());
+	
+	writer.write(XMLTools.getCloseTag(this.name, item.getLabel()));
+	return true;
+    }
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/SimpleNamespace.java	(revision 21112)
@@ -0,0 +1,347 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.ResultSet;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.gs3build.util.MultiMap;
+import org.greenstone.gsdl3.gs3build.util.XMLTools;
+import org.greenstone.gsdl3.gs3build.database.*;
+
+/**
+ *  Implement a simple, unordered, namespace in METS.
+ *
+ *  @see: org.greenstone.gsdl3.gs3build.metadata.METSNamespace
+ */
+
+public class SimpleNamespace extends METSNamespace
+{
+    MultiMap metadataMap;
+    
+    public SimpleNamespace(String name)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, METSLocation location)
+    { 
+	super(name, location);
+	this.metadataMap = new MultiMap();
+    }
+    
+    public SimpleNamespace(String name, Element mdWrapTag)
+    { 
+	super(name);
+	this.metadataMap = new MultiMap();
+	
+	NodeList childNodes = mdWrapTag.getChildNodes();
+	for (int c = 0; c < childNodes.getLength(); c ++) {
+	    // a metadata node
+	    if (childNodes.item(c).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		// get the name of the metadata from the node
+		String metadataLabel = childNodes.item(c).getNodeName();
+		
+		// skip blank labels
+		// TODO: raise an error: metadata item without metadata label set
+		if (metadataLabel == null || metadataLabel.length() == 0) {
+		    continue;
+		}
+		
+		// build the value
+		String metadataValue = "";
+		
+		Element childElement = (Element) childNodes.item(c);
+		for (int i = 0; i < childElement.getChildNodes().getLength(); i ++) {
+		    metadataValue = metadataValue + childElement.getChildNodes().item(i).toString();
+		}
+		
+		// trim any leading namespace identifiers in "namespace:label" metadata
+		if (metadataLabel.startsWith(this.name + ":")) {
+		    metadataLabel = metadataLabel.substring(this.name.length() + 1);
+		}
+		
+		this.addMetadata(metadataLabel, metadataValue);
+	    }
+	}
+    }
+    
+    public boolean validate(String field, String value)
+    { 
+	return true;
+    }
+    
+    /**
+     *  Add a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean addMetadata(String label, String value)
+    {
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.put(label, value);
+	return true;
+    }
+    
+    /**
+     *  Assign a metadata item.  Whether the field and value validate for this
+     *  schema will also be tested.  Any existing metadata for that field will
+     *  be destroyed if the new value validates.
+     *
+     *  @param <code>String</code> the name of the field to be given the value
+     *  @param <code>String</code> the value to be assigned
+     *
+     *  @return <code>boolean</code> whether the field value was added.  This
+     *          would return <code>false</code> if the values did not validate,
+     *          for example.
+     */
+    public boolean setMetadata(String label, String value)
+    { 
+	if (!this.validate(label, value)) {
+	    return false;
+	}
+	
+	this.metadataMap.setOnly(label, value);
+	return true;
+    }
+    
+    /**
+     *  Remove all metadata values for a given field name
+     *
+     *  @param <code>String</code> the field to delete
+     *
+     *  @return <code>boolean</code> whether the field was actually deleted; 
+     *          will return <code>true</code> if the field was already empty.
+     */
+    public boolean removeMetadata(String label)
+    { 
+	this.metadataMap.remove(label);
+	return true;
+    }
+    
+    /**
+     *  Remove a particular incidence of a given metadata field for a document.
+     *  If an exact match for the given value is not found, nothing changes.
+     *  N.B. if a value occurs twice, only the first incidence of it will be
+     *  deleted from the list.
+     *
+     *  @param <code>String</code> the field to have the value removed
+     *  @param <code>String</code> the value to be removed from a given field
+     *
+     *  @return <code>boolean</code> <code>true</code> if an actual metadata text
+     *          is matched against the given value and is thus deleted.
+     */
+    public boolean removeMetadata(String label, String value)
+    { 
+	return this.metadataMap.remove(label, value);
+    }
+    
+    /**
+     *  Get the metadata items for a particular label
+     *
+     *  @param <code>String</code> the label to fetch values for
+     *  
+     *  @return <code>List</code> the list of corresponding values.  May be
+     *          <code>null</code>
+     */
+    public List getMetadata(String label)
+    { 
+	return this.metadataMap.getAll(label);
+    }
+    
+    public Iterator getMetadataNames() 
+    {
+ 	return this.metadataMap.keySet().iterator();
+    }
+    
+    /**
+     *  Write out the metadata to an XML file through a <code>PrintWriter</code>.
+     *
+     *  @param <code>PrintWriter</code> the writer to use.
+     */
+    public boolean write(PrintWriter writer)
+    { 
+	// if this is a non-file block of metadata, write it out in long hand
+	if (this.location == null){
+	    String tag = XMLTools.getOpenTag("mets", "mdWrap");
+	    
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+	    tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+	    if (this.id != null) {
+		this.id = "gsdl"+this.id;
+		tag = XMLTools.addAttribute(tag, "ID", this.id);
+	    }
+	    writer.print("   ");  //indentation
+	    writer.println(tag);
+	    
+	    tag = XMLTools.getOpenTag("mets", "xmlData");
+	    writer.print("      ");  //indentation 
+	    writer.println(tag);
+	    
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		writer.print("         "); //indentation
+		this.writeItem(writer, thisKey);
+	    }
+	    
+	    writer.print("      ");  //indentation 
+	    writer.println(XMLTools.getCloseTag("mets", "xmlData"));
+	    
+	    writer.print("   ");  //indentation
+	    writer.println(XMLTools.getCloseTag("mets", "mdWrap"));
+	}
+	// otherwise, drop the metadata out in a simplified file-reference
+	// form only
+	else
+	    { 
+		String tag = XMLTools.getOpenTag("mets","mdRef");
+		tag = XMLTools.addAttribute(tag, "LOCTYPE", "URL");
+		tag = XMLTools.addAttribute(tag, "xlink:href", location.getLocation().toString());
+		//tag = XMLTools.addAttribute(tag, "MDTYPE", "OTHER");
+		//tag = XMLTools.addAttribute(tag, "OTHERMDTYPE", this.name);
+		tag = XMLTools.addAttribute(tag, "MDTYPE", this.name);	
+		if (this.id != null) {
+		    tag = XMLTools.addAttribute(tag, "ID", this.id);
+		}
+		writer.println(tag);
+		writer.print("   ");  //indentation	
+		writer.println("</mets:mdRef>");
+	    }
+	return true;
+    }
+    
+    /**
+     *  Write out a single element - this may be overloaded to provide for the
+     *  appropriate formatting for this metadata.
+     */
+    protected boolean writeItem(PrintWriter writer, String label)
+    { 
+	if (this.location == null) {
+	    // just place the metadata in a simple wrapper
+	    Iterator values = this.metadataMap.getAll(label).iterator();
+	    
+	    while (values.hasNext()) {
+		String value = values.next().toString();
+		String metaTagName = "Metadata name=" +'"'+ label +'"';
+		//writer.write(XMLTools.getOpenTag(this.name, label));
+		writer.write(XMLTools.getOpenTag(this.name, metaTagName));
+		writer.write(value);
+		
+		//writer.println(XMLTools.getCloseTag(this.name, label));
+		writer.println(XMLTools.getCloseTag(this.name, "Metadata"));
+	    }
+	}
+	else {
+	    String tag = XMLTools.getOpenTag("mets", "mdRef");
+	    tag = XMLTools.addAttribute(tag, "LOCTYPE", this.location.getType());
+	    tag = XMLTools.addAttribute(tag, "xlink:href", this.location.getLocation().toString());
+	    tag = XMLTools.addAttribute(tag, "MDTYPE", this.getName());
+	    tag = XMLTools.makeSingleton(tag);
+	    writer.print("      "); //indentation 
+	    writer.println(tag);
+	}
+	return true;
+    }
+    
+    /**
+     *  Write out the metadata to an SQL database through a <code>GS3SQLConnection</code>.
+     *
+     *  @param <code>GS3SQLConnection</code> the SQL database to use.
+     */
+    public boolean writeSQL(int parentId, GS3SQLConnection connection)
+    { 
+	// write the general stuff
+	String sqlId;
+	
+	if (!super.writeSQL(parentId, connection)) {
+	    return false;
+	}
+	
+	Statement statement = null;
+	ResultSet results = null;
+	try {
+	    statement = connection.createStatement();
+	    if (this.id == null) {
+		GS3SQLSelect select = new GS3SQLSelect("namespaces");
+		select.setWhere(new GS3SQLWhere(new GS3SQLWhereItem("MetadataRef", "=", Integer.toString(parentId), GS3SQLField.INTEGER_TYPE)));
+		select.addField("NamespaceRef");
+		results = statement.executeQuery(select.toString());
+		if (results.first()) {
+		    sqlId = Integer.toString(results.getInt("NamespaceRef"));
+		} else {
+		    statement.close();
+		    return false;
+		}
+	    }
+	    else {
+		sqlId = this.id;
+	    }
+	    
+	    // clear all the existing metadata items for this namespace
+	    GS3SQLDelete delete = new GS3SQLDelete("mdvalues");
+	    GS3SQLWhere where = new GS3SQLWhere(new GS3SQLWhereItem("NamespaceRef", "=", sqlId, GS3SQLField.INTEGER_TYPE));
+	    delete.setWhere(where);
+	    statement.execute(delete.toString());
+	    
+	    // write out the metadata for this namespace
+	    Iterator keys = this.metadataMap.keySet().iterator();
+	    while (keys.hasNext()){
+		String thisKey = keys.next().toString();
+		
+		Iterator values = this.metadataMap.getAll(thisKey).iterator();
+		
+		while (values.hasNext()){
+		    String value = values.next().toString();
+		    
+		    GS3SQLInsert insert = new GS3SQLInsert("mdvalues");
+		    insert.addValue("NamespaceRef", sqlId, GS3SQLField.INTEGER_TYPE);
+		    insert.addValue("Label", thisKey);
+		    insert.addValue("Value", value);
+		    statement.execute(insert.toString());
+		}
+	    }
+	    statement.close();
+	}
+	catch (SQLException sql) {
+	    System.out.println("SimpleNamespace.writeSQL(): "+sql);
+	}
+	return true;
+    }
+    
+    
+    /**
+     *  <p>Indicate whether this metadata is open to being changed or not.</p>
+     *  <p>Metadata which is created from a distinct file cannot be changed,
+     *  only those which have no associated file can be modified.
+     *
+     *	@return <code>boolean</code> whether this namespace can be altered.
+     */
+    public boolean isEditable()
+    { 
+	return (this.location == null);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/metadata/StructureIdentifierFactory.java	(revision 21112)
@@ -0,0 +1,17 @@
+package org.greenstone.gsdl3.gs3build.metadata;
+
+/**
+ *  TODO: complete a class to create file identifiers
+ */
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class StructureIdentifierFactory extends AbstractIdentifierFactory
+{
+    public static final String STRUCTURE_IDENTIFIER_PRELUDE = "DS";
+    
+    public StructureIdentifierFactory()
+    {
+	super(STRUCTURE_IDENTIFIER_PRELUDE);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/notifier/NotifierManager.java	(revision 21112)
@@ -0,0 +1,166 @@
+package org.greenstone.gsdl3.gs3build.notifier;
+
+import java.io.*;
+import java.net.*;
+import java.sql.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.greenstone.gsdl3.gs3build.CollectionManager;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+public class NotifierManager {
+
+    public NotifierManager() {
+    }
+    
+    public void detectEvents(CollectionManager collManager) {
+	String notifyHost = collManager.getNotifyHost();
+	if (notifyHost == null)
+	    return;
+	
+	System.out.println("detecting events");
+	
+	String collectionName = collManager.getCollectionName();
+	String host = "http://localhost:8080/soap/servlet/rpcrouter";
+	
+	String lastBuild = dateToSqlString(collManager.getBuildDate());
+	System.out.println("last build was " + lastBuild);
+        try {
+	    Statement statement = collManager.getDatabase().createStatement();
+            // detect all new documents. A document is new if and only if
+            // AccessionDate >= CollectionLastRebuiltDate
+	    ResultSet results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "new_document");
+	    
+	    // detect modified documents. A document is modified if and only if
+	    // AccessionDate < CollectionLastRebuiltDate (ie, it is not new) and 
+	    // IndexedDate >= CollectionLastRebuiltDate 
+	    results = statement.executeQuery("SELECT DocID FROM document WHERE DocType != 'GSMETADATA' AND AccessionDate < " + lastBuild + " AND IndexedDate >= " + lastBuild);
+	    notify(collectionName, host, notifyHost, results, collManager.getDatabase(), collManager.getBuildDate(), "document_modified");
+	    
+	    statement.close();
+	    // TODO deleted docs?
+        } catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+        }
+    }
+    
+    /**
+     * @param collName
+     * @param hostURL
+     * @param results
+     * @param date
+     * @param eventType TODO
+     * @throws MalformedURLException
+     * @throws IOException
+     * @throws ProtocolException
+     * @throws SQLException
+     */
+    private void notify(String collName, String hostURL, String notifyHost, ResultSet results, GS3SQLConnection database, Date date, String eventType) {
+	try {
+	    System.out.println("notifyHost is " + notifyHost);
+	    URL url = new URL("http://" + notifyHost + "/alerting/service");
+	    System.out.println("trying to send to " + url);
+	    
+	    while(results.next()) {
+		System.out.println(eventType + ": " + results.getString("DocID"));
+	            
+                String documentID = results.getString("DocID");
+                
+		try {
+                    HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // TODO what about proxies?
+                    conn.setUseCaches(false);
+                    conn.setDoInput(true);
+                    conn.setDoOutput(true);
+                    conn.setRequestMethod("POST");
+                    conn.connect();
+                    // Construct data
+                    OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream());
+                    
+                    writer.write("action=receiveEvent&");
+                    writer.write("documentID=");
+                    writer.write(URLEncoder.encode(documentID, "UTF-8"));
+                    writer.write("&type=");
+                    writer.write(URLEncoder.encode(eventType, "UTF-8"));
+                    writer.write("&collectionID=");
+                    writer.write(URLEncoder.encode(collName, "UTF-8"));
+                    writer.write("&hostID=");
+                    writer.write(URLEncoder.encode(hostURL, "UTF-8"));
+                    writer.write("&document_title=");
+                    writer.write(URLEncoder.encode(getDocumentTitle(documentID, database), "UTF-8"));
+                    writer.write("&document_url=");
+                    writer.write(URLEncoder.encode("http://localhost:8080/gsdl3/library?a=d&c=" + collName + "&d=" + documentID, "UTF-8"));
+                    writer.write("&timestamp=");
+                    writer.write(URLEncoder.encode(new SimpleDateFormat("yyyy-MM-dd HH:mm z").format(date), "UTF-8"));
+                    writer.flush();
+                    System.out.println(conn.getResponseCode());
+                    InputStream fromReceiver = conn.getInputStream();
+                    int i;
+                    while ((i = fromReceiver.read()) != -1) {
+                        System.out.write(i);
+                    }
+                    conn.disconnect();
+                } catch (ProtocolException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (UnsupportedEncodingException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                } catch (IOException e1) {
+                    // TODO Auto-generated catch block
+                    e1.printStackTrace();
+                }
+	    }
+	} catch (SQLException e) {
+	    // TODO Auto-generated catch block
+	    e.printStackTrace();
+	} catch (MalformedURLException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * @param documentID
+     * @return
+     * @throws SQLException
+     */
+    private String getDocumentTitle(String documentID, GS3SQLConnection database) {
+        System.out.println("getting title for document " + documentID + " from database");
+
+        try {
+	    Statement statement = database.createStatement();
+            String sqlString = "SELECT mdvalues.value " +
+		"FROM structure s JOIN divisions d ON (s.structureref = d.parentref) " +
+		"                 JOIN divisionmetarefs USING (divisionref) " + 
+		"                 JOIN metadata m USING (metaid) " +
+		"                 JOIN namespaces USING (metadataref) " + 
+		"                 JOIN mdvalues USING (namespaceref) " +
+		"  WHERE mdvalues.label = 'Title' AND s.docid = '" + documentID + "'" + 
+		"                                 AND s.structureType = 'Whole Document' " +
+		"                                 AND d.parenttype = 'Structure' " + 
+		"                                 AND m.docID = s.docid;";
+            ResultSet results = statement.executeQuery(sqlString);
+            if (results.first()) {
+                String title = results.getString("value");
+                System.out.println("title is " + title);
+                return title;
+            }
+	    statement.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return "";
+    }
+
+    /**
+     * @param date
+     * @return
+     */
+    private String dateToSqlString(Date date) {
+        return new SimpleDateFormat("yyyyMMddHHmmss").format(date);
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/ChildWrapper.java	(revision 21112)
@@ -0,0 +1,28 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class ChildWrapper
+{
+    int minOccurs;
+    int maxOccurs;
+
+
+    /**
+     *  The default number of occurrences for an element is 1 (one) for
+     *  both minimum and maximum counts.
+     */
+    public ChildWrapper()
+    {
+	this.minOccurs = 1;
+	this.maxOccurs = 1;
+    }
+
+    public void setMinOccurs(int minOccurs)
+    {
+	this.minOccurs = minOccurs;
+    }
+
+    public void setMaxOccurs(int maxOccurs)
+    {
+	this.maxOccurs = maxOccurs;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/EnumConstraint.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+public class EnumConstraint
+{
+    List options;
+
+    public EnumConstraint()
+    { this.options = new ArrayList();
+    }
+
+  public void addOption(String option)
+  {
+    this.options.add(option);
+  }
+
+  public boolean validate(String value)
+  {
+	  return this.options.contains(value);
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/RangeConstraint.java	(revision 21112)
@@ -0,0 +1,34 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class RangeConstraint
+{
+    public static final int RANGE_MIN_EXCLUSIVE = 0x00;
+    public static final int RANGE_MAX_EXCLUSIVE = 0x01;
+    public static final int RANGE_MIN_INCLUSIVE = 0x10;
+    public static final int RANGE_MAX_INCLUSIVE = 0x11;
+
+    int type;
+    double value;
+
+    public RangeConstraint(int type, double value)
+    {
+	    this.type = type;
+	    this.value = value;
+    }
+
+  public boolean validate(double value)
+  {
+  	switch (this.type) {
+	  case RANGE_MIN_EXCLUSIVE:
+	    return (value > this.value);
+  	case RANGE_MAX_EXCLUSIVE:
+	    return (value < this.value);
+   case RANGE_MIN_INCLUSIVE:
+	    return (value >= this.value);
+   case RANGE_MAX_INCLUSIVE:
+	    return (value <= this.value);
+	}
+	return false;
+}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/Schema.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/Schema.java	(revision 21112)
@@ -0,0 +1,78 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class Schema
+{
+	List children;
+
+	public Schema(Node node)
+	{
+		this.children = new ArrayList();
+
+		NodeList elements = node.getChildNodes();
+
+		System.out.println("Schema has " + elements.getLength() + " children");
+		for (int e = 0; e < elements.getLength(); e++) {
+			System.out.println("Subcomponent " + e);
+			// TODO: actually test what is being parsed here!
+			if (elements.item(e).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+				continue;
+			}
+
+			System.out.println(elements.item(e));
+
+			Element element = (Element) elements.item(e);
+
+			String elementName = element.getLocalName();
+			String fullName = element.getNodeName();
+			String valueName = element.getNodeValue();
+
+			System.out.println(elementName + " " + fullName + " " + valueName);
+
+			if (elementName == null) {
+				elementName = fullName;
+			}
+
+			System.out.println(elementName);
+			elementName = SchemaNode.getElementName(elementName);
+
+			if (elementName.equals("element")) {
+
+				SchemaElement thisElement = new SchemaElement(element);
+/*			
+				NodeList attributes = element.getElementsByTagName("attribute");
+
+				for (int i = 0; i < attributes.getLength(); i++) {
+			    Element attribute = (Element) attributes.item(i);
+					// handle that attribute, create its node and check what you get out
+					try {
+						SchemaAttribute thisAttribute = new SchemaAttribute(attribute);
+						thisElement.addAttribute(thisAttribute);
+					}
+					catch (SchemaException ex) {
+						// do nothing at the moment - though a malformed attribute should be
+						// cause for throwing everything in the air
+					}
+				}*/
+				this.children.add(thisElement);
+			}
+			else {
+				try {
+					Object child = SchemaNodeFactory.makeNode(element);
+					if (child != null) {
+						this.children.add(child);
+					}
+				}
+				catch (SchemaException metaEx)
+				{
+				}
+			}
+		}
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAll.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaAll extends SchemaNode
+{
+	List compulsory;
+
+	public SchemaAll(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	protected List _allocate()
+	{	this.compulsory = new ArrayList();
+		return this.compulsory;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaAttribute.java	(revision 21112)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+public class SchemaAttribute
+{ 
+	public final static String ATTRIBUTE_TYPE_STRING = "String";
+
+  String name;
+  String type;
+  boolean required;
+
+	public SchemaAttribute(Element element) throws SchemaException
+	{
+		// get the name - things are totally unworkable if it doesn't exist!
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			throw new SchemaException("All attributes must have names");
+		}
+
+		// ids are quite important in some metadata schema - copy that too
+		String id = element.getAttribute("id");
+
+		// get the required field - we don't actually cope with "prohibited" at the moment!
+		String use = element.getAttribute("use");
+		this.required = (use != null && use.equals("required"));
+
+		// and the type - we don't reflect any further upon this, actually
+		this.type = element.getAttribute("type");
+		if (this.type == null) {
+			this.type = SchemaAttribute.ATTRIBUTE_TYPE_STRING;
+		}
+
+		// Constraints should now be added, but at present that is not done, as we
+		// are non-validating!
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaChoice.java	(revision 21112)
@@ -0,0 +1,23 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaChoice extends SchemaNode
+{
+	List alternatives;
+
+	public SchemaChoice(Element element) throws SchemaException
+	{
+		super(element);
+	}
+
+	protected List _allocate()
+	{	this.alternatives = new ArrayList();
+		return this.alternatives;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaComplexType.java	(revision 21112)
@@ -0,0 +1,65 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaComplexType implements SchemaType
+{
+	public String name;
+	public List children;
+
+	public SchemaComplexType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		this.children = new ArrayList();
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+				String nodeName = node.getNodeName();
+
+				if (nodeName == null) {
+					continue;
+				}
+
+				nodeName = SchemaNode.getElementName(nodeName);
+				// a simple-based type
+				if (node.getNodeName().equals("simpleContent")) {
+					// we're really just a simple type with attributes - nothing more to do
+				}
+				// a complex-based type
+				else if (node.getNodeName().equals("complexContent")) {
+					// we just contain elements, no text
+				}
+				// the following pretty much speak for themselves
+				else {
+					try {
+						Object child = SchemaNodeFactory.makeNode((Element) node);
+						if (child != null) {
+							this.children.add(child);
+						}
+					}
+					catch (SchemaException ex)
+					{	
+					}
+				}
+			}
+		}
+
+		SchemaTypeMap.theMap.addType(this);
+	}
+	
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaElement.java	(revision 21112)
@@ -0,0 +1,104 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaElement
+{ 
+	// the following may contain duplicates when compared to SchemaAttribute, but this is
+	// deliberate...
+	public static final String ELEMENT_TYPE_STRING = "String";
+
+  String name;
+	String type;
+  List   attributes;
+  List   childElements;
+	int    minOccurs;
+	int    maxOccurs;
+
+	public SchemaElement(Element element)
+	{
+		// get the name of this element
+		this.name = element.getAttribute("name");
+		if (this.name == null) {
+			// TODO: this is a reference object
+			String ref = element.getAttribute("ref");
+
+			if (ref != null) {
+				// TODO: fetch the actual object
+			}
+			else {
+				// TODO: abort
+			}
+		}
+
+		this.type = element.getAttribute("type");
+		if (this.type == null || this.type.length() == 0)
+		{ this.type = SchemaElement.ELEMENT_TYPE_STRING;
+		}
+
+		String temp;
+		temp = element.getAttribute("minOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.minOccurs = Integer.parseInt(temp);
+			this.minOccurs = 1;
+		}
+		temp = element.getAttribute("maxOccurs");
+		if (temp != null && temp.length() > 0) {
+			this.maxOccurs = Integer.parseInt(temp);
+		}
+		else {
+			this.maxOccurs = 1;
+		}
+
+		this.expand(element);
+	}
+
+	private void expand(Element element)
+	{ // TODO
+		// get the first child element
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() != org.w3c.dom.Node.ELEMENT_NODE) {
+					// TODO: raise an error
+					continue;
+				}
+
+				String name = children.item(i).getNodeName();
+				if (name == null) {
+					continue;
+				}
+				
+				try {
+					Object child = SchemaNodeFactory.makeNode((Element) children.item(i));
+					if (child != null) {
+						this.childElements.add(child);
+					}
+				}
+				catch (SchemaException ex)
+				{
+				}
+			}
+		}
+	}
+
+  public SchemaElement(String name)
+  { this.name = name;
+    this.attributes = new ArrayList();
+    this.childElements = new ArrayList();
+  }
+
+  public void addAttribute(SchemaAttribute attribute)
+  {
+	  this.attributes.add(attribute);
+  }
+
+    public void addElement(SchemaElement element)
+    {
+			this.childElements.add(element);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaException.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaException.java	(revision 21112)
@@ -0,0 +1,9 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public class SchemaException extends Exception
+{
+	public SchemaException(String name)
+	{
+		super(name);
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaGroup.java	(revision 21112)
@@ -0,0 +1,21 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaGroup extends SchemaNode
+{
+	List members;
+
+  public SchemaGroup(Element element) throws SchemaException
+  {
+		super(element);
+  }
+
+	public List _allocate()
+	{	this.members = new ArrayList();
+		return this.members;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNode.java	(revision 21112)
@@ -0,0 +1,68 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+
+public abstract class SchemaNode
+{
+	public SchemaNode(Element element) throws SchemaException
+	{	List childList = this._allocate();
+		this.initialise(element, childList);
+		System.out.println(element.getNodeName());
+	}
+	
+	abstract protected List _allocate();
+
+	protected void initialise(Element element, List childList) throws SchemaException
+	{
+		// TODO: check minimum, maximum occurrences etc.
+
+		NodeList children = element.getChildNodes();
+		if (children != null) {
+			System.out.println("  " + children.getLength() + " children");
+			for (int i = 0; i < children.getLength(); i ++)
+			{ if (children.item(i).getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+					Node child = children.item(i);
+
+					String type = child.getNodeName();
+					System.out.println("Child is " + type);
+
+					if (type == null) {
+						throw new SchemaException("Badly defined child - no name");
+					}
+
+					type = SchemaNode.getElementName(type);
+
+					if (type.equals("element")) {
+						SchemaElement elementChild = new SchemaElement((Element) children.item(i));
+						childList.add(elementChild);
+					}
+					else {
+						Object elementChild = SchemaNodeFactory.makeNode((Element) child);
+						if (elementChild != null) {
+							childList.add(elementChild);
+						}
+					}
+					// TODO: do an abstraction between here and SchemaElement
+				}
+			}
+		}
+	}
+
+	public static String getElementName(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(fullname.indexOf(':') + 1);
+		}
+		return fullname;
+	}
+
+	public static String getNamespace(String fullname)
+	{	if (fullname.indexOf(':') >= 0) {
+			return fullname.substring(0, fullname.indexOf(':'));
+		}
+		return null;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaNodeFactory.java	(revision 21112)
@@ -0,0 +1,33 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+public class SchemaNodeFactory
+{
+	public static Object makeNode(Element element) throws SchemaException
+	{
+		String elementName = element.getNodeName();
+
+		System.out.println("Element name " + elementName);
+
+		elementName = SchemaNode.getElementName(elementName);
+
+		if (elementName.equals("choice")) {			
+			return new SchemaChoice(element);
+		}
+		else if (elementName.equals("group")) {
+			return new SchemaGroup(element);
+		}
+		else if (elementName.equals("all")) {
+			return new SchemaAll(element);
+		}
+		else if (elementName.equals("sequence")) {
+			return new SchemaSequence(element);
+		}
+		else if (elementName.equals("element")) {
+			return new SchemaElement(element);
+		}
+		return null;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaParser.java	(revision 21112)
@@ -0,0 +1,73 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.io.*;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+public class SchemaParser
+{
+	public SchemaParser(File file, String namespace)
+	{
+	  try {
+			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+			DocumentBuilder builder = factory.newDocumentBuilder();
+			Document document = builder.parse(file);
+
+			// TODO: get all the types in the tree
+
+			// TODO: do a traverse, and thus cope with elements-within-elements if needsbe, but
+			// this shouldn't happen except in a directly defined situation - actually doing the
+			// parsing in part inside each node would work well provided one checked for a node
+			// having already been done...
+			System.out.println("Read");
+
+			NodeList schemas = document.getElementsByTagName("xs:schema");
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("xsd:schema");
+			}
+			if (schemas.getLength() == 0) {
+				schemas = document.getElementsByTagName("schema");
+			}
+
+			System.out.println(" " + schemas.getLength() + " schemas");
+
+			for (int s = 0; s < schemas.getLength(); s ++) {
+				Schema schema = new Schema(schemas.item(s));
+			}
+		}
+		catch (FactoryConfigurationError e) {
+			System.out.println(e);
+		}
+		catch (ParserConfigurationException ex) {
+			System.out.println(ex);
+		}
+		catch (SAXException ex) {
+			System.out.println(ex);
+		}
+		catch (IOException ex) {
+			System.out.println(ex);
+		}
+	}
+
+	public static void main(String args[])
+	{	try
+		{	File file = new File(args[0]);
+
+			SchemaParser parser = new SchemaParser(file, "");			
+		}
+		catch (Exception ex)
+		{ System.out.println(ex);
+		}
+		return;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaRefMap.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+
+public class SchemaRefMap
+{
+	HashMap map;
+
+	public SchemaRefMap()
+	{
+		this.map = new HashMap();
+	}
+
+	public void addItem(String reference, Object item)
+	{
+		this.map.put(reference, item);
+	}
+
+	public Object getItem(String reference)
+	{	return this.map.get(reference);
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSequence.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class SchemaSequence extends SchemaNode
+{
+	List sequence;
+
+	public SchemaSequence(Element element) throws SchemaException
+	{	super(element);
+	}
+
+	public List _allocate()
+	{	this.sequence = new ArrayList();
+		return this.sequence;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaSimpleType.java	(revision 21112)
@@ -0,0 +1,30 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SchemaSimpleType implements SchemaType
+{
+	String name;
+
+	public void SchemaSimpleType(Element element) throws SchemaException
+	{
+		this.name = element.getAttribute("name");
+		if (name == null)	
+		{ throw new SchemaException("Type defined without name");
+		}
+
+		NodeList childNodes = element.getChildNodes();
+		for (int i = 0; i < childNodes.getLength(); i ++) {
+			Node node = childNodes.item(i);
+
+			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+			}
+		}
+	}
+
+	public String getName()
+	{	return this.name;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaType.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaType.java	(revision 21112)
@@ -0,0 +1,6 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+public interface SchemaType
+{
+	public String getName();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/schema/SchemaTypeMap.java	(revision 21112)
@@ -0,0 +1,26 @@
+package org.greenstone.gsdl3.gs3build.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SchemaTypeMap
+{
+	Map map;
+	public static final SchemaTypeMap theMap;
+
+	static {
+		theMap = new SchemaTypeMap();
+	}
+
+	public SchemaTypeMap()
+	{	this.map = new HashMap();
+	}
+
+	public void addType(SchemaType type)
+	{	this.map.put(type.getName(), type);
+	}
+
+	public SchemaType getType(String name)
+	{	return (SchemaType) this.map.get(name);
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/DOMUtils.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/DOMUtils.java	(revision 21112)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+public class DOMUtils
+{
+  public static String getNodeAttribute(Node node, String attribute)
+  {
+    NamedNodeMap atts = node.getAttributes();
+    if (atts == null) {
+      return null;
+    }
+    Node attributeNode = atts.getNamedItem(attribute);
+    if (attributeNode == null) {
+      return null;
+    }
+    return attributeNode.getNodeValue();
+  }
+    
+  public static String getNodeChildText(Node node)
+  {
+    StringBuffer fieldName = new StringBuffer();
+
+    NodeList fieldChildren = node.getChildNodes();
+    for (int f = 0; f < fieldChildren.getLength(); f ++) {
+      if (fieldChildren.item(f).getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+	fieldName.append(fieldChildren.item(f).getNodeValue());
+      }
+    }
+    return fieldName.toString();
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/FileTrie.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/FileTrie.java	(revision 21112)
@@ -0,0 +1,145 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public class FileTrie
+{	String   stem;
+	List     childTries;
+	List     childDocs;
+	FileTrie parent;
+
+	public FileTrie(String stem)
+	{	this.childTries = new ArrayList();
+		this.childDocs  = new ArrayList();
+		this.stem = stem;
+		this.parent = null;
+	}
+
+	public void setParent(FileTrie trie)
+	{	this.parent = trie;
+	}
+
+	public FileTrie getParent()
+	{	return this.parent;
+	}
+
+	public void addChild(FileTrie trie)
+	{	trie.setParent(this);
+		this.childTries.add(trie);
+	}
+
+	public void replaceChild(FileTrie oldChild, FileTrie newChild)
+	{	for (int i = 0; i < this.childTries.size(); i ++)
+		{	FileTrie thisChild = (FileTrie) this.childTries.get(i);
+			if (thisChild == oldChild)
+			{	this.childTries.set(i, newChild);
+				return;
+			}
+		}
+	}
+
+	public void addDocument(DocumentID documentID)
+	{	this.childDocs.add(documentID);
+	}
+
+	public boolean postDocument(String filename, DocumentID documentID)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					  filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return false;
+		}
+
+		if (common == this.stem.length())
+		{	// post it in this node
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				if (child.postDocument(filename, documentID))
+				{	break;
+				}
+			}
+
+			if (i == this.childTries.size())
+			{ FileTrie newChild = new FileTrie(filename);
+				this.addChild(newChild);
+
+				newChild.addDocument(documentID);
+			}
+		}
+		else
+		{	// divide this node at the 'common' point...
+			FileTrie parentTrie = new FileTrie(this.stem.substring(0, common));
+			this.parent.replaceChild(this, parentTrie);
+
+			// truncate our root
+			this.stem = this.stem.substring(common);
+
+			// add ourselves to our new parent
+			parentTrie.addChild(this);
+
+			// trim back the file name
+			filename = filename.substring(common);
+
+			// recurse into posting the document
+			this.postDocument(filename, documentID);
+		}
+		return true;
+	}
+
+	public boolean postFile(String filename)
+	{	return this.postDocument(filename, null);
+	}
+
+	public List getDocuments(String filename)
+	{	FileTrie leaf = this.getNode(filename);
+		return leaf.childDocs;
+	}
+
+	public FileTrie getNode(String filename)
+	{	int common = 0;
+		int i;
+		
+		while (common < filename.length() &&
+		       common < this.stem.length() &&
+					 filename.charAt(common) == stem.charAt(common))
+		{	common ++;
+		}
+
+		if (common == 0)
+		{ return null;
+		}
+
+		if (common == this.stem.length())
+		{	if (common == filename.length())
+			{	return this;
+			}
+
+			filename = filename.substring(this.stem.length());
+
+			// select which sub-trie to attempt...
+			for (i = 0; i < this.childTries.size(); i ++)
+			{	FileTrie child = (FileTrie) this.childTries.get(i);
+				
+				child = child.getNode(filename);
+
+				if (child != null)
+				{	return child;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/GS2TextFileHandler.java	(revision 21112)
@@ -0,0 +1,111 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class GS2TextFileHandler
+{
+  String  content;
+  String  line;
+
+  public GS2TextFileHandler(String content)
+  { this.content = content;
+  }
+    
+  public boolean hasMore()
+  { return this.line != null;
+  }
+
+  public boolean hasMoreLines()
+  { return this.content != null;
+  }
+
+  public String getEntry()
+  { return this.getEntry(false);
+  }
+
+  public String getEntry(boolean breakSpace)
+  { String reply;
+    int start, tab = 0;
+    boolean quoted = false;
+
+    if (!this.hasMore()) {
+      return null;
+    }
+
+    start = 0;
+    while (start < this.line.length() &&
+	   this.line.charAt(start) == ' ') {
+      start ++;
+    }
+
+    if (start == this.line.length()) {
+      this.line = null;
+      return null;
+    }
+    
+    if (this.line.charAt(start) == '"') {
+      quoted = true;
+      breakSpace = false;
+      start ++;
+    }
+    tab = start;
+
+    while (tab != this.line.length() &&
+	   this.line.charAt(tab) != '\t' &&
+	   !(quoted && this.line.charAt(tab) == '"') &&
+	   !(this.line.charAt(tab) == ' ' && breakSpace))
+    { tab ++;
+    }
+
+    if (start > 0) {
+      this.line = this.line.substring(start);
+      tab -= start;
+    }
+    
+    if (tab == this.line.length()) {
+      reply = this.line;
+      this.line = null;
+    }
+    else {
+      reply = this.line.substring(0, tab);
+      this.line = this.line.substring(tab+1);
+    }
+
+    reply.trim();
+    
+    return reply;
+  }
+
+  public String getLine()
+  { if (this.content == null) {
+      this.line = null;
+      return null;
+    }
+	
+    do {
+      int eol = this.content.indexOf('\n');
+      if (eol < 0) {
+	this.line = this.content;
+	this.content = null;
+      }
+      else {
+	this.line = this.content.substring(0, eol);
+	this.content = this.content.substring(eol+1);
+	while (this.content.length() > 0 &&
+	       this.content.charAt(0) < ' ')
+	{ this.content = this.content.substring(1);
+	}
+      }
+
+      if (this.line != null) {
+	this.line.trim();
+      }
+      if (this.line.length() > 0 &&
+	  this.line.charAt(this.line.length() - 1) == (char) 13) {
+	this.line = this.line.substring(0, this.line.length()-1);
+	this.line.trim();
+      }
+    } while (this.content != null && this.line != null && this.line.length() == 0);
+    
+    return this.line;
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlock.java	(revision 21112)
@@ -0,0 +1,383 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlock
+{
+    Vector content;
+    int    tagpos;
+    int    endpos;
+
+    public HTMLBlock(int docpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+    }
+    
+    public HTMLBlock(int docpos, int endpos)
+    { 
+	this.content = new Vector(1);
+	this.tagpos  = docpos;
+	this.endpos  = endpos;
+    }
+
+    public void addTag(HTMLTag tag)
+    { 
+	if (tag.endPos() > this.endpos){
+	    this.endpos = tag.endPos();
+	}
+	content.addElement(tag);
+    }
+
+    /**
+     *  @deprecccate
+     public void addText(String text)
+     { content.addElement(text);
+     }
+    */
+
+    public void addText(HTMLCText text)
+    { 
+	if (text.endPos() > this.endpos){
+	    this.endpos	= text.endPos();
+	}
+	content.addElement(text);
+    }
+    
+    /**
+     * @return HTML encoded <code>String</code> of the document
+     */
+    public String contentString()
+    { 
+	int member;
+	StringBuffer reply;
+	String type;
+	
+	reply = new StringBuffer();
+	for (member = 1; member < content.size() - 1; member ++){
+	    /*if (content.elementAt(member) instanceof String){
+	      reply.append((String) content.elementAt(member));
+	      }
+	      else 
+	  */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * Returns document position of start of the block
+     */
+    public int startPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * Returns document position of end of the block
+     */
+    public int endPos()
+    { 
+	return this.tagpos;
+    }
+
+    /**
+     * @return head (first) tag of the block
+     */
+    public HTMLTag headTag()
+    { 
+	return ((HTMLTag) content.elementAt(0));
+    }
+
+    /**
+     * @return name (as a <code>String</code>) of the tag at the head of the block
+     */
+    public String headTagName()
+    { 
+	return ((HTMLTag) content.elementAt(0)).tagName();
+    }
+
+    /**
+     * @return name of the tail (last) tag of the block as a <code>String</code>
+     */
+    public String tailTagName()
+    { 
+	return (((HTMLTag) content.elementAt(content.size() - 1)).tagName());
+    }
+	
+	
+    /**
+     *  @return: starting element character position of the item-th item
+     */
+    private int elementStartPos(int item)
+    { 
+	Object element;
+	int	pos = -1;
+		
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	    pos	= ((HTMLTag) element).startPos();
+	}
+	else if (element instanceof HTMLCText){
+	    pos	= ((HTMLCText) element).startPos();
+	}
+	return pos;		
+    }
+    
+    /**
+     *  @return: ending element character position of the item-th item
+     */
+    private int elementEndPos(int item)
+    { 
+	Object	element;
+	int		pos = -1;
+	
+	element = this.content.elementAt(item);
+	if (element instanceof HTMLTag){
+	     pos = ((HTMLTag) element).endPos();
+	}
+	else if (element instanceof HTMLCText){
+	     pos = ((HTMLCText) element).endPos();
+	}
+	return pos;		
+    }
+
+    /**
+     *  @return the HTMLBlock of the indicated subitems
+     */
+    public HTMLBlock subBlock(int startitem, int enditem)
+    { 
+	int       i;
+	Object    element;
+	HTMLBlock reply;
+	reply = new HTMLBlock(this.elementStartPos(startitem),
+			      this.elementEndPos(enditem-1));
+	// copy all the bits
+	for (i = startitem; i < enditem; i++){
+	    element = this.content.elementAt(i);
+	    if (element instanceof HTMLTag){
+		reply.addTag((HTMLTag) element);
+	    }
+	    else if (element instanceof HTMLCText){
+		reply.addText((HTMLCText) element);
+	    }
+	/*
+	  else if (element instanceof String)
+	  {	reply.addText((String) element);
+	  }
+	*/
+	}
+	return reply;
+    }
+
+    /**
+     * Return the position of the given <code>HTMLBlock</code> within this block
+     * @return the tag index of the child block; or < 0 if not a chile
+     */
+    public int find(HTMLBlock subblock)
+    { 
+	int start, end, at, attag;
+	HTMLTag tag;
+	
+	start = 0;
+	end = this.content.size();
+	while (start != end){
+	     /* compare positions */
+	    at = (start + end) / 2;
+	    attag = at;
+	    while (attag < end &&
+		   (content.elementAt(attag) instanceof HTMLTag) == false){
+		attag ++;
+	    }
+	    
+	    if (attag == end){
+		end = at;
+		continue;
+	    }
+	    tag =	(HTMLTag) content.elementAt(attag);
+	    if (tag.startPos() > subblock.startPos()){
+		end = at;				
+	    }
+	    else if (tag.startPos() < subblock.startPos()){
+		start = attag + 1;
+	    }
+	    else { 
+		return attag;
+	    }
+	}
+	return -1;
+    }
+    
+    /**
+     * @return The raw HTML of the block
+     */
+    public String HTMLString()
+    { 
+	int loop;
+	StringBuffer reply;
+
+	reply	= new StringBuffer();
+	for (loop = 0; loop < this.content.size(); loop ++){
+	    /*
+	      if (content.elementAt(loop) instanceof String)
+	      {	reply.append((String) content.elementAt(loop));
+	      }
+	      else 
+	    */
+	    if (content.elementAt(loop) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(loop)).toString());
+	    }
+	    else if (content.elementAt(loop) instanceof HTMLTag){
+		reply.append(((HTMLTag) content.elementAt(loop)).toString());
+	    }
+	}
+	return reply.toString();
+    }
+    
+    /**
+     * @return The text of the block - no HTML tags.  If no text is present, any
+     *         <code>alt</code> information for <code>img</code> tags will be given 
+     *         instead
+     */
+    public String toString()
+    {
+	int member;
+	StringBuffer reply;
+	StringBuffer ireply;
+	
+	reply = new StringBuffer("");
+	ireply = new StringBuffer("");
+	for (member = 0; member < content.size(); member ++){
+	     /*if (content.elementAt(member) instanceof String)
+	       { reply.append((String) content.elementAt(member));
+	       }
+	     */
+	    if (content.elementAt(member) instanceof HTMLCText){
+		reply.append(((HTMLCText) content.elementAt(member)).toString());
+	    }
+	    else if (content.elementAt(member) instanceof HTMLTag){
+		if (((HTMLTag) content.elementAt(member)).tagName().equals("img")){
+		    ireply.append(((HTMLTag) content.elementAt(member)).idValue("alt"));
+		}
+	    }
+	}
+	reply = HTMLCText.cleanString(reply);
+	ireply = HTMLCText.cleanString(ireply);
+	//		reply = reply.trim();
+	if (reply == null || reply.length() == 0){
+	    return ireply.toString();
+	}
+	return reply.toString();
+    }
+    
+    public HTMLObject elementAt(int at)
+    { 
+	return (HTMLObject) this.content.elementAt(at);
+    }
+
+    /**
+     * @return the number of elements in the block
+     */
+    public int size()
+    { 
+	return this.content.size();
+    }
+
+    /**
+     * @return an enumeration of all elements in the block
+     */
+    public Enumeration elements()
+    { 
+	Enumeration enumer;
+  
+	enumer = new HTMLBlockEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * @return an enumeration of all tags in the block
+     */
+    public Enumeration tagElements()
+    { 
+	Enumeration enumer;
+	
+	enumer = new HTMLBlockTagEnumerator(this);
+	return enumer;
+    }
+
+    /**
+     * Finalization method
+     */
+    protected void finalize() throws Throwable
+    { 
+	this.content	= null;
+	super.finalize();
+    }
+}
+
+final class HTMLBlockEnumerator implements Enumeration
+{ 
+    private HTMLBlock block;
+    int			member;
+
+    public HTMLBlockEnumerator(HTMLBlock block)
+    { 
+	this.block	= block;
+	this.member	= 0;
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+
+	element = this.block.content.elementAt(member);
+	member ++;
+	return element;
+    }
+
+    public boolean hasMoreElements()
+    {
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
+final class HTMLBlockTagEnumerator implements Enumeration
+{
+    private HTMLBlock block;
+    int	member;
+    
+    public HTMLBlockTagEnumerator(HTMLBlock block)
+    { 
+	this.block = block;
+	this.member = 0;
+	while ( this.member < this.block.content.size() &&
+		this.block.content.elementAt(this.member) instanceof HTMLTag == false){
+	    this.member ++;
+	}
+    }
+
+    public Object nextElement()
+    { 
+	Object element;
+	
+	element = this.block.content.elementAt(member);
+	do {
+	    member ++;
+	} while (this.block.content.elementAt(member) instanceof HTMLTag == false);
+	return element;
+    }
+    
+    public boolean hasMoreElements()
+    { 
+	if (block == null || this.member == this.block.content.size()){
+	    return false;
+	}
+	return true;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLBlockList.java	(revision 21112)
@@ -0,0 +1,409 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLBlockList
+{ Vector blocks;
+	String title;
+	protected static HTMLTagVector anchorStartTags;
+	protected static HTMLTagVector anchorEndTags;
+	protected static HTMLTagVector anchorOptEndTags;
+	protected static HTMLTagVector headingStartTags;
+	protected static HTMLTagVector headingEndTags;
+
+	static
+	{	anchorStartTags 	= new HTMLTagVector("a");
+		anchorEndTags			= new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		anchorOptEndTags 	= new HTMLTagVector("div,center,form");
+		headingStartTags 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6");
+		headingEndTags	 	= new HTMLTagVector("h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+	}
+
+	// opening   opentag  incontent	result
+	//    T					T					T				*       {if opening, incontent must be false}
+	//    T					T					F				T
+	//    T					F					T				*       {if opening, incontent must be false}
+	//    T					F					F				F
+	//    F					T					T				T
+	//    F					T					F				F				{if getting an opening tag
+	//		F					F					T				T
+	//		F					F					F				T
+	private boolean inTagList(HTMLTag tag, Vector tags, boolean opening)
+	{ int tagref;
+
+		for (tagref = 0; tagref < tags.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) tags.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+	//
+	// Common initialisation; parse the document for text surrounded by
+	// pairs of <starttag>....<endtag/starttag> combos
+	//
+	protected void initialise(HTMLDoc doc, HTMLTagVector starttags, HTMLTagVector endtags, HTMLTagVector optendtags)
+	{ boolean intag;
+		boolean incontent;
+		HTMLCText		text;
+		HTMLTag			tag;
+		HTMLTag 		starttag;
+		HTMLBlock 	block;
+		HTMLBlock		docBlock;
+		Enumeration	elements;
+		Object			element;
+
+		this.blocks = new Vector(1);
+		block				= null;
+		intag				= false;
+		incontent		= false;
+		starttag		= null;
+
+		docBlock		= doc.getCodedContent();
+		elements		= docBlock.elements();
+
+		while (elements.hasMoreElements())
+		{ element	= elements.nextElement();
+
+			// if it's a tag
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+			
+				// end the fragment if getting a closing tag
+				if (intag)
+				{ if (endtags.matches(tag, false) ||
+							(incontent && optendtags.matches(tag, false)))
+					{ block.addTag(starttag.endTag());
+						intag = false;
+						incontent = false;
+					}
+					else
+					{ block.addTag(tag);
+					}
+				}
+
+				// !intag is checked for explicitly, as it may have been
+				// switched off by a non-partner opening tag above: eg:
+				//   <H2>A Heading<H3>A Smaller Heading</H3>
+				//                ^^^^
+				//                here
+
+				// start fragment if the tag is a starter
+				if (!intag)
+				{ if (starttags.matches(tag, true))
+					{ block = new HTMLBlock(tag.startPos(), tag.endPos());
+						this.blocks.addElement((Object) block);
+						block.addTag(tag);
+						starttag = tag;
+						intag = true;
+						incontent = false;
+					}
+				}
+			}	
+			// if the parse returned text
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intag)
+				{ block.addText(text);
+					if (!text.nullString())
+					{ incontent = true;
+					}
+				}
+			}
+		}
+
+		// if a tag wasn't "finished");
+		if (intag)
+		{ block.addTag(starttag.endTag());
+		}
+
+		this.title = doc.urlString();
+	}
+
+	protected HTMLBlockList()
+	{
+	}
+	
+	//
+	// constructor for given vectors of tags: startlist elements always start a new
+	// block [ending an open one if necessary]; endlist elements always end an open 
+	// block; optendlist elements close a block iff a block has been started, and
+	// some non-blank content (ie. some "real" text, not just returns/spaces) has
+	// been read since the beginning of the block
+	//
+	public HTMLBlockList(HTMLDoc doc, Vector starttags, Vector endtags, Vector optendtags)
+	{ HTMLTagVector _starttags;
+		HTMLTagVector _endtags;
+		HTMLTagVector _optendtags;
+
+		_starttags	= new HTMLTagVector(starttags);
+		_endtags		= new HTMLTagVector(starttags);
+		_optendtags	= new HTMLTagVector(starttags);
+		this.initialise(doc, _starttags, _endtags, _optendtags);
+	}
+	
+	//
+	// constructor for explicit lists; a tagVector is created for each list;
+	// see above for list roles
+	//
+	public HTMLBlockList(HTMLDoc doc, String startlist, String endlist, String optendlist)
+	{ HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		starttags		= new HTMLTagVector(startlist);
+		endtags			= new HTMLTagVector(endlist);
+		optendtags	= new HTMLTagVector(optendlist);
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	//
+	// constructor; get all HTMLBlocks for h{level} through to h1;
+	// eg hlevel==3 test h3,h2,h1
+	//
+	public HTMLBlockList(HTMLDoc doc, int hlevel)
+	{ int loop;
+		StringBuffer levelstring;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		levelstring = new StringBuffer("h1");
+		for (loop = 2; loop <= hlevel; loop ++)
+		{ levelstring.append(",h");
+		  levelstring.append(Integer.toString(loop));
+		}
+
+		starttags		= new HTMLTagVector(levelstring.toString());
+		levelstring.append(",table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		endtags			= new HTMLTagVector(levelstring.toString());
+		optendtags	= new HTMLTagVector("div,center,form");
+
+		this.initialise(doc, starttags, endtags, optendtags);
+	}
+
+	/**
+	 * Static method returning list of anchors of a document
+	 */
+	public static synchronized HTMLBlockList anchorlist(HTMLDoc doc)
+	{ HTMLBlockList anchors;
+		HTMLTagVector starttags;
+		HTMLTagVector endtags;
+		HTMLTagVector optendtags;
+
+		anchors = new HTMLBlockList();
+
+		anchors.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+		return anchors;
+	}
+
+	//
+	// return how many tags are in the block list
+	//
+	public int size()
+	{ if (this.blocks == null)
+		{ return 0;
+		}
+
+		return this.blocks.size();
+	}
+
+	/**
+	 * @return number of blocks in the block list
+	 * @deprecated
+	 */
+	public int tagCount()
+	{	return this.size();
+	}
+
+	//
+	// return the position of the tag in the document
+	//
+	public int tagPos(int tag)
+	{ HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return 0;
+		}
+		block = (HTMLBlock)	this.blocks.elementAt(tag);
+		return block.startPos();
+	}
+
+	//
+	// return the nth block in the list
+	//
+	public HTMLBlock tagBlock(int tag)
+	{ HTMLBlock block;
+
+		block = (HTMLBlock) this.blocks.elementAt(tag);
+		return block;
+	}
+
+	//
+	// return the index of the next blocklist of the same level
+	//
+	public int tagNext(int attag)
+	{ HTMLBlock block;
+		String		headTag;
+		int				nextTag;
+
+		block = (HTMLBlock) this.blocks.elementAt(attag);
+		headTag = block.headTagName();
+		nextTag	= attag+1;
+
+		while (nextTag < this.blocks.size())
+		{	block	=	(HTMLBlock) this.blocks.elementAt(nextTag);
+			if (block.headTagName().equals(headTag))
+			{	break;
+			}
+			nextTag ++;
+		}
+		return nextTag;
+	}
+
+	//
+	// return the number of children at the next heading level
+	// 
+	public int tagChildren(int fromtag, int level)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		if (level == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		if (fromtag > 0)
+		{ sibling	= this.tagNext(fromtag-1);
+		}
+		else
+		{	sibling	= this.size();
+		}
+		if (sibling == fromtag)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = fromtag+1; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	level + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+
+	public int tagChildren(int attag)
+	{ int sibling;
+		int descendent;
+		int children;
+		int	atlevel;
+		HTMLTag tag;
+
+		// get current tag level; level 6 headings cannot have
+		// children
+		atlevel		=	this.tagBlock(attag).headTag().tagLevel();
+		if (atlevel == 6)
+		{ return 0;
+		}
+
+		// now, if the next tag at this level is the next tag in
+		// the document, then we can't have any children
+		sibling		= this.tagNext(attag);
+		if (sibling == attag + 1)
+		{ children = 0;
+		}
+		// if the next tag at this level is *not* next, then we have
+		// at least one child (ie the next tag in sequence), even if
+		// it is not at the next heading level down.
+		//
+		// if no further children occur at the next level down, we
+		// have only one child.  If any further children occur at the
+		// next level down, then they are counted, except where it is
+		// the next tag in sequence, which is automatically accounted
+		// for
+		else
+		{	children	= 1;
+			for (descendent = attag + 2; descendent < sibling; descendent ++)
+			{ tag	= this.tagBlock(descendent).headTag();
+				
+				if (tag.tagLevel() ==	atlevel + 1)
+				{	children ++;
+				}
+			}
+		}
+		return children;
+	}
+
+	//
+	// write out a structured output onto System.out
+	//
+  public void sysoutList()
+	{ int item;
+		HTMLBlock block;
+
+		if (this.blocks == null)
+		{ return;
+		}
+
+		for (item = 0; item < this.blocks.size(); item ++)
+		{ block = (HTMLBlock) this.blocks.elementAt(item);
+			System.out.println("  @"+ block.startPos()+":"+block.headTagName() +":"+ block);
+		}
+	}
+
+	//
+	// write out HTMLBlocks as an Enumeration
+	//
+	public Enumeration elements()
+	{ return new HTMLBlockListEnumerator(this);
+	}
+}
+
+final class HTMLBlockListEnumerator implements Enumeration
+{ private HTMLBlockList blocklist;
+	private int						member;
+
+	public HTMLBlockListEnumerator(HTMLBlockList blocklist)
+	{ this.blocklist	= blocklist;
+		this.member			= 0;
+	}
+
+	public Object nextElement()
+	{ Object element;
+
+		element = this.blocklist.blocks.elementAt(member);
+		member ++;
+		return element;
+	}
+
+	public boolean hasMoreElements()
+	{ if (this.blocklist == null || this.member == this.blocklist.blocks.size())
+		{ return false;
+		}
+		return true;
+	}
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLCText.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLCText.java	(revision 21112)
@@ -0,0 +1,105 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLCText implements HTMLObject
+{ String text;
+  int    start;
+  int    end;
+
+  public static final String HTML_TEXT_TYPE = "Text";
+
+  public HTMLCText(String text, int start, int end)
+  { this.text  = text;
+    this.start = start;
+    this.end   = end;
+  }
+
+  public String getHTMLType()
+  { return HTML_TEXT_TYPE;
+  }
+
+  public int startPos()
+  { return this.start;
+  }
+
+  public int endPos()
+  { return this.end;
+  }
+
+  public boolean isEmpty()
+  { return emptyString(this.text);
+  }
+
+  public static boolean emptyString(String s)
+  { int c;
+
+    for (c = 0; c < s.length(); c ++)
+    { if (s.charAt(c) > 32)
+      { return false;
+      }
+    }
+    return true;
+  }
+
+  public static StringBuffer cleanString(StringBuffer buffer)
+  { int c;
+    int w;
+
+    w = 0;	
+    for (c = 0; c < buffer.length(); c ++)
+    { if (buffer.charAt(c) >= 32)
+      { if (buffer.charAt(c) == 32)
+	{ if (w > 0 && buffer.charAt(w-1) > 32)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	    w ++;
+	  }
+	}
+        else
+	{ if (c != w)
+	  { buffer.setCharAt(w, buffer.charAt(c));
+	  }
+	  w ++;
+	}
+      }
+      else
+      { if (w > 0 && buffer.charAt(w-1) > 32)
+	{ buffer.setCharAt(w,' ');
+	  w ++;
+	}
+      }
+    }
+    buffer.setLength(w);
+    return buffer;
+  }
+
+  /**
+   * nullString; used by initialisation for checking for
+   * non-blank content (ie. text other than just spaces/returns)
+   * see constructor (ie HTMLBlockList) methods below for role of
+   * this in the larger plan of things
+   */
+  public boolean nullString()
+  { int ch;
+
+    if (text != null)
+    { for (ch = 0; ch < text.length(); ch ++)
+      { if (text.charAt(ch) > ' ')
+	{ return false;
+	}
+      }
+    }
+    return true;
+  }
+
+  public String toString()
+  { return this.text;
+  }
+
+  public String toString(boolean cleaned)
+  { if (cleaned == false)
+    { return toString();
+    }
+    else
+    { return HTMLEntity.decodeText(this.text);
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLContext.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLContext.java	(revision 21112)
@@ -0,0 +1,201 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLContext
+{ boolean valid;
+	StringBuffer	text;
+	
+	public static	HTMLTagVector breakTags = new HTMLTagVector("a,h1,h2,h3,h4,h5,h6,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+
+	private int offset (String string, int direction)
+	{	int offset;
+
+		if (direction > 0)
+		{	offset = string.indexOf('.');
+		}
+		else
+		{	offset = string.lastIndexOf('.');
+		}
+		return offset;
+	}
+
+	public HTMLContext(HTMLBlock parent, HTMLBlock child, int scope)
+	{ int pos;
+		int start, startoffset;
+		int	end, endoffset;
+		int	length;
+		String	string;
+		HTMLTag	tag;
+
+		this.text	= null;
+
+		// if we can't match it, give up!
+		pos = parent.find(child);
+		if (pos < 0)
+		{ this.valid	= false;
+			return;
+		}
+
+		// get start!
+		if (pos > 0)
+		{ start = pos - 1;
+		}
+		else
+		{	start	= 0;
+		}	
+		startoffset	= -1;
+		length	= 0;
+		while (	start > 0)
+		{	
+		  /*
+		  if (parent.elementAt(start) instanceof String)
+			{	string = (String) parent.elementAt(start);
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+				
+				if (length >= scope)
+				{	break;
+				}
+				}
+				else 
+		  */
+			if (parent.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.elementAt(start)).toString();
+
+				startoffset	= offset(string, -1);
+				if (startoffset > 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.elementAt(start) instanceof HTMLTag)
+			{	// do some checks
+				tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	startoffset	= -1;
+					break;
+				}
+			}
+			start --;
+		}
+
+		// get end!
+		pos = pos + child.size();
+		if (pos < parent.size())
+		{ end	= pos + 1;
+		}
+		else
+		{	end	= parent.size();
+		}	
+		endoffset	= -1;
+		length	= 0;
+		string	= null;
+		while (end < parent.size())
+		{	if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+
+				endoffset	= offset(string, +1);
+				if (endoffset >= 0)
+				{	break;
+				}
+
+				length += string.length();
+
+				if (length >= scope)
+				{	break;
+				}
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLTag)
+			{	tag = (HTMLTag) parent.elementAt(start);
+
+				if (breakTags.matches(tag, false))
+				{	endoffset	= -1;
+					end	= end - 1;
+					break;
+				}
+			}
+			System.out.println("String:"+string);
+			end ++;
+		}
+
+		this.text	= new StringBuffer();
+		if (startoffset >= 0)
+		{	string	= null;
+
+			if (parent.content.elementAt(start) instanceof String)
+			{	string = (String) parent.content.elementAt(start);
+			}
+			else if (parent.content.elementAt(start) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(start)).toString();
+			}
+			
+			if (string != null)
+			{ this.text.append(string.substring(startoffset+1));
+			}
+			start ++;
+		}
+
+		for (pos = start; pos <= end; pos ++)
+		{ if (pos == end && endoffset >= 0)
+			{	break;
+			}
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	this.text.append((String) parent.content.elementAt(pos));
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	this.text.append(((HTMLCText) parent.content.elementAt(pos)).toString());
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLTag)
+			{ // don't add it!
+			}
+		}
+
+		if (pos == end && endoffset > 0)	// this deliberately ignore endoffset == 0
+																		 	// 'cos nothing should happen anyhow!
+		{	string	= null;
+
+			if (parent.content.elementAt(pos) instanceof String)
+			{	string = (String) parent.content.elementAt(pos);
+			}
+			else if (parent.content.elementAt(pos) instanceof HTMLCText)
+			{	string = ((HTMLCText) parent.content.elementAt(pos)).toString();
+			}
+		
+			if (string != null)
+			{	this.text.append(string.substring(0, endoffset));
+			}
+		}
+	}
+
+	public String toString()
+	{	if (this.text == null)
+		{ return "";
+		}
+		return this.text.toString();
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDoc.java	(revision 21112)
@@ -0,0 +1,658 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.io.*;
+import java.applet.*;
+import java.util.*;
+
+/** 
+ *  HTMLDOC: handles an individual HTML document
+ */
+
+public class HTMLDoc
+{ 
+    String        content;
+    String        title;
+    URL           url;
+    int           pos;
+    int           status;
+    HTMLBlockList blocklist;
+    List          urls_out;
+    List          urls_in;
+    Object        note;
+    HTMLLoader    loader;
+    HTMLBlock     coded;
+
+    /**
+     * Constructor which loads a given document; the ref indicates from where!
+     */
+    public HTMLDoc(URL url, int ref)
+    { 
+	URLConnection connect;
+	int length, i, c;
+	byte inbuffer[];
+	InputStream in;
+
+	this.url = url;
+	this.pos = 0;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+	this.note = null;
+	this.status = 0;
+
+	if (ref >= 0){
+	    int attempts;
+	    
+	    attempts = 0;
+	    loader = new HTMLLoader(url);
+	    loader.load();
+	    if (ref == 0){
+		while(this.ready() == false){
+		    try {
+			Thread.sleep(100);
+		    }
+		    catch (InterruptedException ex){
+			
+		    }
+		}
+	    }
+	}
+	else {
+	    this.content = "";
+	}
+    }
+    
+    public HTMLDoc(URL url)
+    {	
+	this(url, 0);
+    }
+    
+    /**
+     * @return <code>true</code> when a document has finished loading.
+     */
+    public synchronized boolean ready()
+    {
+	if (this.loader == null){
+	    return true;
+	}
+	
+	boolean loaded 	= loader.loaded();
+	
+	if (loaded == true){
+	    // reload if necessary
+	    if (this.loader.error == true){
+		if (this.status < 3 && this.loader.notFound == false){
+		    this.status ++;
+		    this.loader = new HTMLLoader(this.url);
+		    loader.load();
+		    return false;
+		}
+		else {
+		    this.status 	= 4;
+		    this.content	= null;
+		    this.loader		= null;
+		    return true;
+		}
+	    }
+	    this.content 	= loader.HTML();
+	    this.loader		= null;
+	}
+	return loaded;
+    }
+    
+    /**
+     * Create a document from a file
+     */
+    public HTMLDoc(URL url, String file)
+    { 
+	FileInputStream _in;
+	byte[] data = new byte[128];
+	int databytes;
+	StringBuffer content;
+	
+	try {
+	    _in = new FileInputStream(file);
+	    if (_in != null){
+		content = new StringBuffer();
+		do{  
+		    databytes = _in.read(data);
+		    if (databytes > 0){
+			content.append(new String(data, 0, databytes));
+		    }		
+		} while (databytes >= 0);
+		this.content		= content.toString();
+	    }
+	}
+	catch (IOException ex){
+	    this.content = null;
+	    System.out.println(ex);
+	}
+	this.url = url;
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.blocklist	= null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+
+    /**
+     * Create a url-only document: the document body is not stored
+     */
+    public HTMLDoc(String url)
+    {
+	if (url != null){
+	    try{
+		this.url = new URL(url);
+	    }
+	    catch (MalformedURLException ex){
+		this.url = null;
+	    }
+	}
+	else {
+	    this.url = null;
+	}
+	this.pos = 0;
+	this.status = 0;
+	this.note = null;
+	this.content = null;
+	this.blocklist = null;
+	this.title = null;
+	this.urls_out = null;
+	this.urls_in = null;
+    }
+    
+    static public HTMLDoc fromString(String url, String string)
+    {	
+	HTMLDoc reply;
+	
+	reply = new HTMLDoc(url);
+	reply.content = string;
+	return reply;
+    }
+    
+    static public boolean validExtension(String url)
+    {	
+	String extension;
+	
+	// get file extension
+	extension = URLString.extension(url);
+	if (extension != null){
+	    extension = extension.toLowerCase();
+	}
+	
+	if (extension == null ||
+	    extension.indexOf("htm") >= 0 || 
+	    extension.indexOf("asp") >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    static boolean validExtension(URL url)
+    {	
+	return validExtension(url.toString());
+    }
+
+    /**
+     * Uncache a single document
+     */
+    public static HTMLDoc uncacheDoc(String filebase, String url)
+    {	
+	String 	pathname;
+	String	filename;
+	HTMLDoc	doc;
+	
+	pathname = URLString.pathName(url);
+	if (pathname == null){
+	    return null;
+	}
+	
+	filename = filebase + pathname.replace('/', File.pathSeparatorChar);
+	doc = new HTMLDoc(URLString.toURL(url), filename);
+	
+	return doc;
+    }
+
+    /**
+     * @return URL (ie. Java URL object) of the document
+     */
+    public URL getUrl()
+    { 
+	return this.url;
+    }
+
+    // Report url to AppletContext
+    public void show(AppletContext ac)
+    { 
+	URL url;
+
+	url = this.getUrl();
+	if (url != null){
+	    ac.showDocument(url);
+	}
+    }
+    
+    // Get a HTMLBlock list of the wanted text in an HTMLdoc
+    public void getSelectContent(int level)
+    { 
+	HTMLBlockList list;
+
+	list = new HTMLBlockList(this, level);
+	this.blocklist = list;
+    }
+    
+    /**
+     * return the selected content
+     */
+    public HTMLBlockList selectContent()
+    { 
+	return this.blocklist;
+    }
+
+    /**
+     * get header blocks as selected content
+     */
+    public HTMLBlockList headerBlocks()
+    {	
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist;
+    }
+    
+    /**
+     * return a tagged-up select with an <A NAME=> at the beginning of each Hx
+     */
+    public String taggedSelectContent()
+    {
+	int heading, offset, pos;
+	String reply;
+	
+	if (this.blocklist == null || 
+	    this.blocklist.size() == 0){
+	    return this.content;
+	}
+	
+	reply = "";
+	offset	= 0;
+	for (heading = 0; heading < this.blocklist.size(); heading ++){
+	    reply = reply + 
+		this.content.substring(offset, this.blocklist.tagPos(heading)) +
+		"<A NAME=\"_L"+heading+"\"></A>";
+	    offset = this.blocklist.tagPos(heading);
+	}
+	reply = reply + this.content.substring(offset, this.content.length());
+	return reply;
+    }
+    
+    /**
+     * Return enumeration of headings in document
+     */
+    public Enumeration headings()
+    {
+	if (this.blocklist == null){
+	    this.getSelectContent(6);
+	}
+	return this.blocklist.elements();
+    }
+    
+    /**
+     *	@return HTMLBlock of the child text underneath a heading
+     */
+    public HTMLBlock headingChildText(int headref)
+    { 
+	HTMLBlock pageblock;
+	HTMLBlock headblock;
+	HTMLBlock childblock;
+	int blockstart, blockend;
+	
+	pageblock = this.getCodedContent(); // get all the coded bits of the page
+	headblock = this.headerBlocks().tagBlock(headref);
+	
+	blockstart = pageblock.find(headblock);
+	blockstart += headblock.size();
+	if (headref == this.headerBlocks().size() - 1) {
+	    blockend = pageblock.size();
+	}
+	else {
+	    blockend = pageblock.find(this.headerBlocks().tagBlock(headref+1));
+	}
+	childblock = pageblock.subBlock(blockstart, blockend);
+	
+	return childblock;
+    }
+    
+    /**
+     * Add incoming url to list of incoming references
+     */
+    public void addUrlIn(String url)
+    {
+	if (this.urls_in == null){
+	    this.urls_in = new ArrayList();
+	}
+	this.urls_in.add(url);
+    }
+    
+    /**
+     * @return a String of all incoming urls
+     */
+    public String urlsInString()
+    { 
+	if (this.urls_in != null){
+	    return this.urls_in.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return an enumeration of all incoming urls as Strings
+     */
+    public Iterator urlsIn()
+    {
+	if (this.urls_in == null){
+	    return null;
+	}
+	return this.urls_in.iterator();
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all incoming urls
+     */
+    public List urlsInList()
+    {	
+	return this.urls_in;
+    }
+
+    /**
+     * @return <code>true</code> if the given url is linked to by this
+     *  document
+     */
+    public boolean linkedTo(URL url)
+    {	
+	if (this.urls_out != null &&
+	    this.urls_out.indexOf(url.toString()) >= 0){
+	    return true;
+	}
+	return false;
+    }
+    
+    /**
+     * See above: returns <code>true</code> if the document is linked to
+     * by this document.
+     */
+    public boolean linkedTo(HTMLDoc doc)
+    {
+	return linkedTo(doc.getUrl());
+    }
+
+    /**
+     * Add outgoing url to list of outgoing references
+     */
+    public void addUrlOut(String url)
+    { 
+	if (this.urls_out == null){
+	    this.urls_out = new ArrayList();
+	}
+	this.urls_out.add(url);
+    }
+    
+    /**
+     * @return a String of all outgoing urls
+     */
+    public String urlsOutString()
+    { 
+	if (this.urls_out != null){
+	    return this.urls_out.toString();
+	}
+	return "";
+    }
+    
+    /**
+     * @return the <code>StringVector</code> of all outgoing urls
+     */
+    public List urlsOutList()
+    {	
+	return this.urls_out;
+    }
+
+    /**
+     * Return an enumeration of outbound urls
+     */
+    public Iterator urlsOut()
+    { 
+	if (this.urls_out == null){
+	    return null;
+	}
+	return this.urls_out.iterator();
+    }
+
+    private static int urlListPos(List urlList, String url)
+    {	
+	for (int i = 0; i < urlList.size(); i ++){
+	    if (((String) urlList.get(i)).equals(url)){
+		return i;
+	    }
+	}
+	return -1;
+    }
+    
+    // Rename internally the content of a list
+    private void __renameUrl(List urlList, String oldUrl, String newUrl)
+    {	
+	if (urlList != null){
+	    int oldPos = urlListPos(urlList, oldUrl);
+	    
+	    if (oldPos >= 0){
+		if (urlListPos(urlList, newUrl) >= 0){
+		    // delete oldUrl: now duplicate
+		    urlList.remove(oldPos);
+		}
+		else {
+		    // rename oldUrl
+		    urlList.set(oldPos, newUrl);
+		}
+	    }
+	}
+    }
+    
+    /**
+     * Rename inbound/outbound url references
+     */
+    public void renameUrl(String oldUrl, String newUrl)
+    {	
+	this.__renameUrl(this.urls_in, oldUrl, newUrl);
+	this.__renameUrl(this.urls_out, oldUrl, newUrl);
+    }
+    
+    /**
+     * Save document
+     */
+    public boolean save(String filename, boolean select)
+    {	
+	FileOutputStream fout;
+	PrintWriter pout;
+	
+	String flattened = URLString.flattenedUrl(filename);
+	
+	try{	
+	    HTMLDocAnchorList anchorList = new HTMLDocAnchorList(this);
+
+	    anchorList.flattenLinks();
+			
+	    fout = new FileOutputStream(flattened);
+	    pout = new PrintWriter(fout);
+	    if (select == true) {
+		pout.print(this.taggedSelectContent());
+	    }
+	    else {
+		pout.print(this.content);
+	    }
+	    pout.close();
+	    fout.close();
+	}
+	catch (IOException ex) {
+	    System.out.println(ex);
+	    return false;
+	}
+	return true;
+    }
+    
+    /**
+     * Save document content
+     */
+    public boolean save(String base)
+    {
+	return this.save(base, false);
+    }
+    
+    /**
+     * Save selected content
+     */
+    public boolean saveSelect(String base)
+    {	
+	return this.save(base, true);
+    }
+
+    /**
+     * Get title of page
+     */
+    public void getTitle()
+    { 
+	HTMLDocTitle doctitle;
+
+	if (this.content != null){
+	    doctitle = new HTMLDocTitle(this);
+	    this.title	= doctitle.Title();
+	}
+    }
+    
+    /**
+     * @return: title of page
+     */
+    public String titleString()
+    {
+	if (this.title == null){
+	    this.getTitle();
+	}
+	return this.title;
+    }
+    
+    /**
+     * @return a <code>String</code> of the url of the page
+     */
+    public String urlString()
+    {
+	return this.url.toString();
+    }
+
+    /**
+     * 	Replace the HTML text of the page with the provided HTML string
+     *	content.  This is an extremely dangerous call to use, as dependent
+     *	objects such as tag objects will be immediately rendered inaccurate
+     *	in terms of pointers held to the original text.
+     */
+    protected void setContent(String newContent)
+    { 
+	this.content = newContent;
+    }
+
+    /**
+     * @return a <code>String</code> of the HTML text of the page
+     */
+    public String getContent()
+    { 
+	return this.content;
+    }
+
+    /**
+     * @return a <code>HTMLBlock</code> of the page content
+     */
+    public HTMLBlock getCodedContent()
+    {	
+	// if we haven't already got it, code the bugger up right now!
+	if (this.coded == null){
+	    HTMLParser parser;
+	    String next;
+	    HTMLBlock coded;
+	    HTMLTag tag;
+	    HTMLCText text;
+
+	    if (this.content == null){
+		return null;
+	    }
+	    
+	    parser = new HTMLParser(this);
+	    coded = new HTMLBlock(0, this.content.length());
+
+	    parser.startParse();
+	    next = parser.fullParse();
+	    while (next != null){
+		if (HTMLTag.isTag(next)){
+		    tag = new HTMLTag(next, parser.lastParse(), parser.atParse());
+		    coded.addTag(tag);
+		}
+		else {
+		     text = new HTMLCText(next, parser.lastParse(), parser.atParse());
+		     coded.addText(text);
+		}
+		next = parser.fullParse();
+	    }
+	    this.coded	= coded;
+	}
+	return this.coded;
+    }
+    
+    /**
+     * @return <code>true</code> if this document is not errored from loading
+     * etc.
+     */
+    public boolean valid()
+    {	
+	return (this.status < 4 && this.content != null && this.content.equals("null") == false);
+    }
+    
+    /**
+     * HTMLString: returns content
+     */
+    public String HTMLString()
+    { 
+	return this.content;
+    }
+    
+    /**
+     * Return text of document as ascii text;
+     */
+    /*
+      public String toString()
+      {	StringBuffer	reply = new StringBuffer();
+      int						parastart = 0;
+
+      return reply.toString();
+      }
+    */
+
+    /**
+     * Set current note on this page
+     * @return old note object
+     */
+    public Object setNote(Object note)
+    {
+	Object oldnote;
+	
+	oldnote	= this.note;
+	this.note = note;
+	return oldnote;
+    }
+    
+    /**
+     * @return current note <code>Object</code> on this page
+     */
+    public Object getNote()
+    {
+	return this.note;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocAnchorList.java	(revision 21112)
@@ -0,0 +1,160 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocAnchorList extends HTMLBlockList
+
+{ HTMLDoc doc;
+
+	public HTMLDocAnchorList(HTMLDoc doc)
+
+	{ super();
+
+		this.doc = doc;	// remember the document reference
+
+
+
+		super.initialise(doc, anchorStartTags, anchorEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	/**
+	 *	Return the HTMLBlock containing the named local anchor and it's
+	 *	child (contained) tags.
+	 *
+	 *	@param <code>String</code> name of the local anchor to find.
+	 *	@return <code>HTMLBlock</code> the corresponding block of HTML
+	 *					contained by the anchor.  If the local anchor does not
+	 *					exist, this value will be <code>null</code>.
+	 */
+	public HTMLBlock localAnchor(String name)
+
+	{	int block;
+
+		String	localname;
+
+
+
+		for (block = 0; block < this.size(); block ++)
+
+		{ localname = this.tagBlock(block).headTag().idValue("name");
+
+			if (localname == null)
+
+			{ continue;
+
+			}
+
+			if (name.equalsIgnoreCase(localname))
+
+			{	return this.tagBlock(block);
+
+			}
+
+		}
+
+		return null;
+
+	}
+
+	private int indexOf(String base, String subString, int from)
+	{	int i, j;
+
+		for (i = from; i < base.length() - subString.length(); i ++)
+		{	if (Character.toLowerCase(base.charAt(i)) == subString.charAt(0))
+			{	j = 0;
+				for (j = 1; j < subString.length(); j ++)
+				{	if (Character.toLowerCase(base.charAt(i+j)) != subString.charAt(j))
+					{	break;
+					}
+				}
+
+				if (j == subString.length())
+				{	return i;
+				}
+			}
+		}
+		return -1;
+	}
+
+	public void flattenLinks()
+	{	int 					block;
+		String				url;
+		char					endsWith;
+		StringBuffer	newContent = new StringBuffer(doc.getContent());
+		boolean				changed = false;
+
+//		System.out.println("Flattening");
+
+		for (block = 0; block < this.size(); block ++)
+		{ HTMLTag tag;
+			int			tagPos;
+			int			valueStart, valueEnd;
+
+			tag = this.tagBlock(block).headTag();	// get the <A> tag itself
+
+			url = tag.idValue("href"); 						// if there is no HREF member, abandon this one
+			if (url == null)
+			{ System.out.println("No href");
+				continue;
+			}
+	
+			tagPos	= tag.startPos();
+			tagPos	=	indexOf(doc.getContent(), "href", tagPos);	// get the HREF itself
+			if (tagPos > tag.endPos() || tagPos < 0)					// if none in the tag, skip it
+			{	System.out.println("Didn't get href");
+				continue;
+			}
+			tagPos += "href".length();												// skip past it
+
+			while (	doc.getContent().charAt(tagPos) <= ' ' ||		// skip pass equals, etc.
+							doc.getContent().charAt(tagPos) == '=')
+			{	tagPos ++;
+			}
+
+			if (doc.getContent().charAt(tagPos) == '"' ||				// get the demarcating whitespace/quotes
+					doc.getContent().charAt(tagPos) == '\'')
+			{	endsWith	= doc.getContent().charAt(tagPos);
+				tagPos ++;
+			}
+			else
+			{	endsWith  = ' ';
+			}
+			valueStart = tagPos;
+
+			//			System.out.println("Flattening " + doc.getContent().length() + " " + tagPos);
+
+			while (tagPos < doc.getContent().length() &&
+			       doc.getContent().charAt(tagPos) != endsWith)	// read it in
+			{	if (doc.content.charAt(tagPos) == '?' ||				// replace special characters			
+						doc.content.charAt(tagPos) == '&')
+				{	newContent.setCharAt(tagPos, '_');						//   and note that we've done so
+					changed	= true;
+				}
+				tagPos ++;
+			}
+			valueEnd = tagPos;
+
+			tagPos = indexOf(doc.getContent(), ".html", valueStart);
+			if (tagPos >= 0 && tagPos < valueEnd)
+			{	// shuffle around the html bit
+				while (tagPos < valueEnd - 5)
+				{	newContent.setCharAt(tagPos, newContent.charAt(tagPos + 5));
+					tagPos ++;
+				}
+				newContent.setCharAt(tagPos ++, '.');
+				newContent.setCharAt(tagPos ++, 'h');
+				newContent.setCharAt(tagPos ++, 't');
+				newContent.setCharAt(tagPos ++, 'm');
+				newContent.setCharAt(tagPos ++, 'l');
+			}
+		}
+
+		if (changed == true)																// replace the document as needsbe
+		{	doc.setContent(newContent.toString());
+		}
+	}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocHeadingList.java	(revision 21112)
@@ -0,0 +1,25 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+public class HTMLDocHeadingList extends HTMLBlockList
+
+{ public HTMLDocHeadingList(HTMLDoc doc)
+
+	{ super();
+
+
+
+		super.initialise(doc, headingStartTags, headingEndTags, anchorOptEndTags);
+
+	}
+
+
+
+	public int headingLevel(int element)
+
+	{	return this.tagBlock(element).headTag().tagLevel();
+
+	}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocLocalHeading.java	(revision 21112)
@@ -0,0 +1,187 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.util.*;
+
+
+
+public class HTMLDocLocalHeading
+
+{	String		heading;
+
+	HTMLBlock	block;
+
+
+
+	public HTMLDocLocalHeading(HTMLDoc doc, String local)
+
+	{	HTMLBlock 				nextheading, lastheading, anchor;
+
+		Enumeration				headings;
+
+		HTMLDocAnchorList	anchors;
+
+		HTMLBlock					docBlock;
+
+		int								_start, _end, _pos;
+
+
+
+		// initialise (to undefined)
+
+		this.heading	= null;
+
+		this.block		= null;
+
+
+
+		// get anchors
+
+		anchors	= new HTMLDocAnchorList(doc);
+
+		anchor	= anchors.localAnchor(local);
+
+		if (anchor == null)
+
+		{	return;
+
+		}
+
+
+
+		// get headings position; lastheading will be previous, nextheading will be next
+
+		headings		= doc.headings();
+
+		lastheading	= null;
+
+		nextheading	= null;
+
+
+
+		while (headings.hasMoreElements())
+
+		{	nextheading	= (HTMLBlock) headings.nextElement();
+
+			if (nextheading.startPos() > anchor.startPos())
+
+			{	break;
+
+			}
+
+			
+
+
+
+			lastheading	= nextheading;
+
+		}
+
+
+
+		// didn't get any actual headings; abort.
+
+		if (nextheading == null)
+
+		{	return;
+
+		}
+
+
+
+		// got to the end
+
+		if (nextheading == lastheading)
+
+		{	this.block = nextheading;
+
+		}
+
+		else
+
+		{	// now see where anchor is; try to find null textual content between anchor
+
+			// and heading
+
+			docBlock	= doc.getCodedContent();
+
+			_start		= docBlock.find(anchor);
+
+			_end			= docBlock.find(nextheading);
+
+			for (_pos = _start; _pos < _end; _pos ++)
+
+			{	// HTML tag
+
+				if (docBlock.elementAt(_pos) instanceof HTMLTag)
+
+				{	
+
+				}
+
+				// HTML text - if non-blank, abort
+
+				else
+
+				{	if (((HTMLCText) docBlock.elementAt(_pos)).nullString() == false)
+
+					{	break;
+
+					}
+
+				}
+
+			}
+
+
+
+			// assign
+
+			if (_pos != _end)
+
+			{	this.block	=	lastheading;
+
+			}
+
+			else
+
+			{	this.block	= nextheading;
+
+			}
+
+		}
+
+		this.heading	= this.block.toString();
+
+	}
+
+
+
+	public String toString()
+
+	{	return this.heading;
+
+	}
+
+
+
+	public HTMLBlock block()
+
+	{	return this.block;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.heading	= null;
+
+		this.block		= null;
+
+		super.finalize();
+
+	}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLDocTitle.java	(revision 21112)
@@ -0,0 +1,89 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLDocTitle
+{ StringBuffer title;
+	private static HTMLTagVector titleStartTags;
+	private static HTMLTagVector titleEndTags;
+	private static HTMLTagVector titleOptEndTags;
+
+	static
+	{	titleStartTags 	= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6");
+		titleEndTags 		= new HTMLTagVector("title,h1,h2,h3,h4,h5,h6,body,head,table,hr,p,ul,ol,dl,dt,dd,li,tr,td,th,map");
+		titleOptEndTags	= new HTMLTagVector("div,center,form");
+	}
+
+	/**
+	 * Constructor: generates a title for the supplied <code>HTMLDoc</code>.
+	 */
+	public HTMLDocTitle(HTMLDoc doc)
+	{ boolean				intitle;
+		boolean				incontent;
+		HTMLCText			text;
+		HTMLTag				tag;
+		HTMLBlock			parser;
+		Enumeration		elements;
+		Object				element;
+		
+		this.title 	= null;
+		intitle			= false;
+		incontent		= false;
+		
+		parser		=	doc.getCodedContent();
+		if (parser == null)
+		{	return;
+		}
+		elements	= parser.elements();
+
+		while (elements.hasMoreElements())
+		{ //System.out.println(read + ":" +  read.charAt(0));
+			element	= elements.nextElement();
+			
+			if (element instanceof HTMLTag)
+			{ tag = (HTMLTag) element;
+
+				if (intitle)
+				{	if (titleEndTags.matches(tag, false) ||
+							(incontent && titleOptEndTags.matches(tag, false))) // if closing
+					{ intitle = false;
+						break;
+					}
+					else // else ignore it
+					{
+					}
+				}
+				else
+				{	if (titleStartTags.matches(tag, true) == true)
+					{ intitle			= true;
+						incontent		= false;
+						this.title	= new StringBuffer();
+					}
+				}
+			}
+			else
+			{ text	= (HTMLCText) element;
+
+				if (intitle == true)
+				{	this.title.append(text.toString());
+					if (!text.nullString())
+					{ incontent	= true;
+					}
+				}
+			}
+		}
+		if (this.title != null)
+		{ this.title = HTMLCText.cleanString(this.title);
+		}
+	}
+
+	/**
+	 * @return the Title as a <code>String</code>.
+	 */
+	public String Title()
+	{	if (this.title == null)
+		{ return null;
+		}
+		return this.title.toString();
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLEntity.java	(revision 21112)
@@ -0,0 +1,346 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.*;
+import java.util.*;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentLoader;
+
+public class HTMLEntity
+{ String  name;
+  int     code;
+  static Vector entities = null;
+  static Vector	nmapping = null;
+
+  public HTMLEntity(String name, int code)
+  { int start, end, at;
+    HTMLEntity compareEntity;
+
+    if (HTMLEntity.entities == null)
+    { HTMLEntity.entities = new Vector();
+    }
+    if (HTMLEntity.nmapping == null)
+    { HTMLEntity.nmapping = new Vector();
+    }
+
+    this.name	= name;
+    this.code	= code;
+
+    entities.addElement(this);
+    start = 0;
+    end   = nmapping.size();
+    while (start != end)
+    { at = (start + end) / 2;
+	
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code > code)
+      { end = at;
+      }
+      else
+      { start = at + 1;
+      }
+    }
+    if (end == nmapping.size())
+    { nmapping.addElement(new Integer(nmapping.size()));
+    }
+    else
+    { nmapping.insertElementAt(new Integer(nmapping.size()), start);
+    }
+  }
+
+  public String toString()
+  { Character c;
+
+    c = new Character((char) this.code);
+    return c.toString();
+  }
+
+  public static String codeString(char c)
+  { StringBuffer 	reply;
+    String        codestring;
+
+    if (c < 128 || c == '&' || c == '<' || c == '>')
+    { return null;
+    }
+    codestring = Integer.toString((int) c);
+    reply      = new StringBuffer("&#");
+    reply.append(codestring);
+    reply.append(";");
+
+    return reply.toString();
+  }
+
+  public static String nameCodeString(char c)
+  { int start, end, at, compare;
+    HTMLEntity compareEntity;
+
+    start = 0;
+    end		= nmapping.size();
+    while (start != end)
+    { at	= (start + end) / 2;
+      
+      compareEntity = 
+	(HTMLEntity) entities.elementAt(((Integer) nmapping.elementAt(at)).intValue());
+      if (compareEntity.code == (int) c)
+      { StringBuffer reply;
+
+        reply = new StringBuffer("&");
+	reply.append(compareEntity.name);
+	reply.append(";");
+	return reply.toString();
+      }
+      else if (compareEntity.code < c)
+      {	end	= at;
+      }
+      else
+      {	start	= at + 1;
+      }
+    }
+    return codeString(c);
+  }
+
+  public static void getEntities(String source)
+  { HTMLEntity 	newentity;
+    int         pos;
+    int					start;
+    String			name;
+    int					code;
+    
+    if (source == null)
+    { return;
+    }
+
+    start = 0;
+    while (start < source.length())
+    { pos = start;
+      while (source.charAt(pos) != ':')
+      { pos ++;
+      }
+      name = source.substring(start, pos);
+
+      // skip colon and reset
+      pos ++;
+      start = pos;
+
+      while(pos < source.length() && source.charAt(pos) > ' ')
+      { pos ++;
+      }
+      code = Integer.parseInt(source.substring(start, pos));
+
+      newentity = new HTMLEntity(name, code);
+      
+      while ( pos < source.length() && source.charAt(pos) < ' ')
+      { pos ++;
+      }
+      start = pos;
+    }
+  }
+
+  public static void getEntities(InputStream in)
+  { getEntities(DocumentLoader.getAsString(in));
+  }
+
+  public static void getEntities(File file)
+  { FileInputStream in;
+
+    try
+    { in = new FileInputStream(file);
+      getEntities(in);
+      in.close();
+    }
+    catch (IOException io)
+    { System.out.println("No entities");
+    }
+  }
+
+  public static void getEntities()
+  { ClassLoader loader;
+    InputStream	in;
+
+    in = ClassLoader.getSystemResourceAsStream("org\\greenstone\\gsdl3\\gs3build\\util\\Entities.map");
+    if (in == null)
+    { System.out.println("Unable to load Entities.map from org.greenstone.gsdl3.gs3build.util");
+    }
+    getEntities(in);
+  }
+
+  public static String encodeText(String source, boolean named)
+  { StringBuffer reply;
+    int          start,at;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply	= new StringBuffer();
+    at 		= 0;
+    start	= 0;
+    while (at < source.length())
+    { if (source.charAt(at) > 128)
+      {	if (at != start)
+	{ reply.append(source.substring(start, at));
+	}
+
+        if (named)
+	{ reply.append(nameCodeString(source.charAt(at)));
+	}
+	else
+	{ reply.append(codeString(source.charAt(at)));
+	}
+	start	= at + 1;
+      }
+      at ++;
+    }
+		
+    return reply.toString();
+  }
+
+  public static String decodeText(String source)
+  { int start, end, loff, roff;
+    int sstart, send, test;
+    int startoff, endoff, baseoff;
+    boolean matched;
+    String  tail;
+    StringBuffer reply;
+
+    if (source == null)
+    { return null;
+    }
+
+    if (HTMLEntity.entities == null)
+    {	getEntities();
+    }
+
+    reply   = new StringBuffer();
+    baseoff = 0;
+
+    tail = source;
+    do
+    { // get next ampersand
+	    startoff = tail.indexOf('&');
+      if (startoff < 0 || startoff == tail.length() - 1)
+      { break;
+      }
+
+      // if it's followed by a hash, evaluate as a number
+      if (tail.charAt(startoff + 1) == '#')
+      { if (startoff < tail.length() - 2 &&
+      			tail.charAt(startoff+2) >= '0' &&
+            tail.charAt(startoff+2) <= '9')
+      	{ int code;
+
+        	loff = startoff + 2;
+        	while (tail.charAt(loff) >= '0' &&
+          				tail.charAt(loff) <= '9')
+        	{ loff ++;
+          }
+
+          code = Integer.parseInt(tail.substring(startoff+2, loff));
+        	reply.append(tail.substring(0, startoff));
+					reply.append(new Character((char) code));
+
+          if (tail.charAt(loff) == ';')
+          { loff ++;
+          }
+          loff = loff - startoff - 1;
+        }
+        else
+        { loff = 0;
+        	reply.append(tail.substring(0, startoff+1));
+        }
+      }
+      else
+      {	start = 0;
+        end   = HTMLEntity.entities.size();
+        loff  = 0;
+
+	while (startoff+1+loff < tail.length() && start != end)
+	{ sstart = start;
+	  send   = end;
+          test	 = (sstart + send) >> 1;
+
+	  matched	= false;
+		      
+	  roff	= startoff + 1 + loff;
+
+	  while (sstart != send)
+	  { test = (sstart + send) >> 1;
+
+	    if (tail.charAt(roff) < ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { send = test;
+	    }
+	    else if (tail.charAt(roff) > ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.charAt(loff))
+	    { sstart	= test + 1;
+	    }
+	    else
+	    { break;
+	    }
+	  }
+
+	  //					System.out.println(sstart+">"+send+":"+loff);
+
+	  if (sstart != send) // found a match
+	  { sstart = test;
+	    while (sstart >= start &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(sstart)).name.charAt(loff))
+	    { sstart --;
+	    }
+	    sstart ++;
+
+	    send = test + 1;
+	    while (send < end &&
+		   tail.charAt(roff) ==
+		   ((HTMLEntity) HTMLEntity.entities.elementAt(send)).name.charAt(loff))
+	    { send ++;
+	    }
+
+//						System.out.println(sstart+"!"+send+":"+loff);
+
+	    start = sstart;
+	    end	= send;
+	  }
+	  else
+	  { loff	= 0;
+	    reply.append(tail.substring(0, startoff+1));
+	    break;
+	  }
+
+//					System.out.println(start+">"+end+":"+loff);
+	  
+	  loff ++;
+	  if (end == start + 1 &&
+	      ((HTMLEntity) HTMLEntity.entities.elementAt(test)).name.length() == loff)
+	  { reply.append(tail.substring(0, startoff));
+	    reply.append(((HTMLEntity) HTMLEntity.entities.elementAt(test)).toString());
+	    if (roff < tail.length() - 1 && tail.charAt(roff+1) == ';')
+	    { loff ++;
+	    }
+	    break;
+	  }
+	}
+      }
+
+      tail = tail.substring(startoff + 1 + loff);
+    } while (tail != null && tail.length() > 0);
+    
+    if (tail != null)
+    { reply.append(tail);
+    }
+    return reply.toString();
+  }
+
+  public static void main(String args[])
+  { String reply;
+
+    getEntities();
+    
+    reply = decodeText(args[0]);
+    System.out.println(reply);
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoadManager.java	(revision 21112)
@@ -0,0 +1,208 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.*;
+import java.util.*;
+
+class HTMLLoadManager implements Runnable
+{	Vector		 	toLoad;
+	Vector			loading;
+	Vector			loaded;
+	boolean			stopFlag;
+	boolean			started;
+	Thread			thread;
+	static int	running;
+
+	static
+	{	running	= 0;
+	}
+
+	/**
+	 *	Vanilla constructor of an object to load HTML documents in the background
+	 */
+	public HTMLLoadManager()
+	{	this.toLoad		= new Vector();
+		this.loading 	= new Vector();
+		this.loaded		= new Vector();
+		this.thread		= null;
+		this.stopFlag	= false;
+		this.started	= false;
+	}
+
+	/**
+	 *	Check if a given document is loading
+	 *
+	 *	@return <code>boolean</code> whether the document is loading or not
+	 */
+	public boolean isLoading(String docURL)
+	{	int i;
+
+		for (i = 0; i < this.toLoad.size(); i ++)
+		{	if (this.toLoad.elementAt(i).toString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loading.size(); i ++)
+		{	if (((HTMLDoc) this.loading.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+
+		for (i = 0; i < this.loaded.size(); i ++)
+		{	if (((HTMLDoc) this.loaded.elementAt(i)).urlString().equals(docURL))
+			{	return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 *	Start this object as a thread
+	 */
+	public synchronized void start()
+	{	if (this.thread == null)
+		{	this.thread	= new Thread(this);
+			this.thread.start();		// start the thread
+			this.stopFlag	= false;	// reset the stop flag as it could have been reset
+			this.started 	= true;		// note that we're running
+			running ++;
+		}
+	} 
+
+	/**
+	 *	The main thread function; it calls a number of private service functions to
+	 *  achieve it's affect.  Once the stop flag is set, it runs until all loading
+	 *  documents are loaded, then stops.
+	 */
+	public void run()
+	{ int i;
+
+		while (this.stopFlag == false || this.loading.size() > 0)
+		{	// slumber while there's nothing to do
+			while (this.toLoad.size() == 0 && this.loading.size() == 0)
+			{	try
+				{
+					thread.sleep(100);
+				}
+				catch (InterruptedException ex)
+				{
+				}
+			}
+
+			// first check for finished documents
+			i = 0;
+			while (i < this.loading.size())
+			{	HTMLDoc doc = (HTMLDoc) this.loading.elementAt(i);
+
+				if (doc.ready())
+				{	this.deschedule(i);
+				}
+				else
+				{	i ++;
+				}
+			}
+
+			// then check for a document to load
+			while (this.stopFlag == false &&
+						 this.toLoad.size() > 0 &&
+						 this.loading.size() < 10)
+			{	this.scheduleNext();
+			}
+		}
+		this.started = false;
+		running --;
+	}
+
+	/**
+	 *	Returns the total number of loader managers running at this time
+	 *
+	 *	@return <code>int</code> the number of managers running.
+	 */
+	public synchronized int running()
+	{	return running;
+	}
+
+	private synchronized void scheduleNext()
+	{	// pop url from list to load
+		URL docURL	= (URL) this.toLoad.elementAt(0);
+		this.toLoad.removeElementAt(0);
+
+		// generate new document and add it to the "loading" vector;
+		// the document content is loaded in another thread
+		HTMLDoc	doc = new HTMLDoc(docURL, 1);
+		this.loading.addElement(doc);
+	}
+
+	private synchronized void deschedule(int i)
+	{	// get the document and place it in the loaded list
+		HTMLDoc	doc = (HTMLDoc) this.loading.elementAt(i);
+		this.loaded.addElement(doc);
+
+		// remove it from the loading list
+		this.loading.removeElementAt(i);
+	}
+
+	/**
+	 *	Get a loaded document, not blocking if nothing is ready yet.
+	 *
+	 *  @return <code>HTMLDoc</code> the document; NB. this may be <code>null</code> as
+	 *                               it is not guaranteed that there is currently a loaded
+	 *                               document.
+	 */
+	public synchronized HTMLDoc nextLoaded()
+	{	HTMLDoc reply;
+
+		// if nothing is loaded, return null
+		if (this.loaded.size() == 0)
+		{	return null;
+		}
+
+		// get the first element from the loaded vector and return it,
+		// removing it from the vector in the meantime
+		reply = (HTMLDoc) this.loaded.elementAt(0);
+		this.loaded.removeElementAt(0);
+		return reply;
+	}
+
+	/**
+	 *	Request the loading of a document; this does not block but schedules
+	 *  the document for loading.
+	 *
+	 *  @param <code>URL</code> the URL of the document to load
+	 */
+	public synchronized void load(URL docURL)
+	{	// check if it's being loaded already; if so do nothing
+		if (this.isLoading(docURL.toString()) == false)
+		{	this.toLoad.addElement(docURL);
+
+			// self-start if the loading thread isn't running
+			if (this.started == false)
+			{	this.start();
+			}
+		}
+	}
+
+	/**
+	 *	Get the total number of documents loading/loaded through this loader
+	 *
+	 *	@param <code>int</code> the number of documents
+	 */
+	public synchronized int loadingDocs()
+	{	return this.toLoad.size() + this.loading.size() + this.loaded.size();
+	}
+
+	/**
+	 *	Instruct the manager to stop loading documents
+	 */
+	public synchronized void stopLoading()
+	{	this.stopFlag = true;
+	}
+
+	protected void finalize() throws Throwable
+	{	this.thread 	= null;
+		this.loading 	= null;
+		this.toLoad		= null;
+		this.loaded		= null;
+		super.finalize();
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLoader.java	(revision 21112)
@@ -0,0 +1,353 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+
+
+import java.net.*;
+
+import java.io.*;
+
+
+
+public class HTMLLoader implements Runnable
+
+{ URL 					from;
+
+	StringBuffer	HTML;
+
+	boolean				loaded;
+
+	boolean				error;
+
+	boolean				notFound;
+
+	Thread				thread;
+
+	static int		running;
+
+
+
+	/**
+
+	 * Constructor to load given URL
+
+	 */
+
+	public HTMLLoader(URL from)
+
+	{ this.from 	= from;
+
+		this.HTML 	= new StringBuffer();
+
+		this.loaded	= false;
+
+		this.error	= false;
+
+		this.thread = null;
+	}
+
+
+
+	/**
+
+	 * @return <code>0</code> if finished running; any other value if
+
+	 *         still executing
+
+	 */
+
+	public static synchronized int running()
+
+	{ return running;
+
+	}
+
+
+
+	/**
+
+	 * Starts the thread; the method <code>load</code> is a synonym for this
+
+	 */
+
+	public synchronized void start()
+
+	{ if (thread == null)
+
+		{ thread = new Thread(this);
+
+			thread.start();
+
+			running	++;
+
+		}
+
+//		System.out.println("Running: "+running);
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading; this will multi-thread with calling task
+
+	 */
+
+	public void run()
+
+	{	URLConnection connect;
+
+		int						length, i, c;
+
+		byte					inbuffer[];
+
+		InputStream		in;
+
+
+
+		inbuffer = new byte[128];
+
+
+
+		this.notFound	= false;
+//		System.out.println("Running loader");
+
+
+
+		// get connection and test length
+
+		try
+
+		{	connect = this.from.openConnection();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.loaded	= true;
+
+			this.error	=	true;
+
+			running --;
+			System.out.println(ioex);
+
+			return;
+
+		}
+
+
+		length = connect.getContentLength();
+
+		if (length == 0)
+
+		{ this.loaded = true;
+
+			running --;
+			System.out.println("Empty");
+
+			return;
+
+		}
+
+
+
+		// get an inputstream to the document body
+
+		try
+
+		{ in = connect.getInputStream();
+
+		}
+
+		catch (IOException ioex)
+
+		{ this.error	= true;
+
+			this.loaded	= true;
+
+			running --;
+			System.out.println("Open failed" + ioex);
+
+			return;
+
+		}
+
+
+
+		if (length < 0)
+
+		{	if (connect.getHeaderField(0) != null &&
+
+					connect.getHeaderField(0).indexOf("404") >= 0)
+
+			{	this.notFound	= true;
+
+				this.error		= true;
+
+				this.loaded		= true;
+
+				running --;
+				System.out.println("Not found");
+
+				return;
+
+			}
+
+		}
+
+//		System.out.println("Ready to read " + length);
+
+
+
+		// read in the document body
+
+		i = 0;
+
+		c = 0;
+
+		while (true || i < length)
+
+		{ // read in 128 characters (abort on IO Exception)
+
+			try
+
+			{ c = in.read(inbuffer);
+
+			}
+
+			catch (IOException ioex)
+
+			{ break;
+
+			}
+
+			// System.out.println("read " + c);
+
+
+
+			// if no characters were read, end, otherwise add it to the body
+
+			if (c != -1)
+
+			{ if (c > 0)
+
+				{ this.HTML.append(new String(inbuffer, 0, c));
+
+					i += c;
+
+				}
+
+			}
+
+			else
+
+			{ break;
+
+			}
+
+			// if an incomplete buffer was read (ie no characters remain), end
+
+/*			if (c < inbuffer.length && i >= length)
+
+			{ break;
+
+			}*/
+
+		}
+
+
+
+		// Close file
+
+		try
+
+		{	in.close();
+
+		}
+
+		catch (IOException ioex)
+
+		{	//System.out.println("Closed except "+ioex);
+
+		}
+
+	
+		this.loaded = true;
+
+  	running --;
+
+	}
+
+
+
+	/**
+
+	 * Do the actual loading (calls a Start on the Thread)
+
+	 */
+
+	public void load()
+
+	{	this.start();
+
+//		this.run();
+
+	}
+
+
+
+	/**
+
+	 * @return HTML text as a <code>String</code>.
+
+	 */
+
+	public synchronized String HTML()
+
+	{ if (this.error)
+
+		{ return "Error";
+
+		}
+
+		return this.HTML.toString();
+
+	}
+
+
+
+	/**
+
+	 * @return  <code>true</code> if the file has successfully loaded
+
+	 */
+
+	public boolean loaded()
+
+	{ return this.loaded;
+
+	}
+
+
+
+	protected void finalize() throws Throwable
+
+	{	this.thread = null;
+
+		this.from		= null;
+
+		this.HTML		= null;
+
+		super.finalize();
+
+	}
+
+
+
+	static
+
+	{ running = 0;
+
+	}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLLocalAnchor.java	(revision 21112)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLLocalAnchor
+{	
+	public static int anchorRef(HTMLBlockList anchors, String name)
+	{ int loop;
+		String	idvalue;
+
+		for (loop = 0; loop < anchors.size(); loop ++)
+		{	idvalue = anchors.tagBlock(loop).headTag().idValue("name");
+			if (idvalue != null && idvalue.equalsIgnoreCase(name))  //TODO: check if this should be case senstitive
+			{ return loop;
+			}
+		}
+		return -1;
+	}
+
+	public static int header(HTMLDoc doc, String local)
+	{	HTMLBlockList anchors;
+		int						heading;
+		HTMLBlock			headingblock;
+		int						anchor;
+		HTMLBlock			anchorblock;
+
+		anchors	= HTMLBlockList.anchorlist(doc);
+		anchor	= anchorRef(anchors, local);
+		if (anchor < 0)
+		{	return -1;
+		}
+
+		anchorblock	= anchors.tagBlock(anchor);
+		for (heading = 0; heading < doc.headerBlocks().size(); heading ++)
+		{	headingblock	= doc.headerBlocks().tagBlock(heading);
+
+			if (headingblock.startPos() > anchorblock.endPos())
+			{	break;
+			}
+		}
+		return heading - 1;
+	}
+	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLObject.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLObject.java	(revision 21112)
@@ -0,0 +1,5 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public interface HTMLObject
+{ public String getHTMLType();
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLParser.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLParser.java	(revision 21112)
@@ -0,0 +1,243 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+
+public class HTMLParser
+{ 
+    HTMLDoc document;
+    int pos;
+    int	lastpos;
+
+    public HTMLParser(HTMLDoc document)
+    { 
+	this.document = document;
+	this.pos = 0;
+    }
+    
+    public void startParse()
+    { 
+	this.pos = 0;
+	this.lastpos = -1;
+	//System.out.println("Starting "+document.urlString());
+    }
+
+    public int atParse()
+    { 
+	return this.pos;
+    }
+
+    public int lastParse()
+    { 
+	return this.lastpos;
+    }
+
+    // do a full text/tag parse
+    public String fullParse()
+    { 
+	String reply = null;
+	int end;
+	int start;
+
+	this.lastpos = this.pos;
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	if (this.document.getContent().charAt(this.pos) == '<'){
+	    start = this.pos;
+	    
+	    // if we're not at the end of the document,
+	    // read the rest of the tag
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    // if the tag is a comment
+	    if (this.pos < this.document.getContent().length() - 3 &&
+		this.document.getContent().substring(this.pos, this.pos+4).equals("<!--")){
+		end = this.document.getContent().substring(this.pos).indexOf("-->") + 3 + this.pos;
+		reply = this.document.getContent().substring(this.pos, end);
+		this.pos = end;
+	    }
+	    else 
+		{ // read up to the end of the tag
+		    end = this.pos + 1;
+		    while (end < this.document.getContent().length() &&
+			   this.document.getContent().charAt(end) != '>'){
+			end ++;
+		    }
+
+		    // get the whole of the tag into 'reply', and
+		    // set the current pos to immediately after the tag
+		    if (end < this.document.getContent().length()){
+			reply = this.document.getContent().substring(this.pos, end + 1);
+			this.pos = end + 1;
+		    }
+		    // patch the trailing > onto the tag string
+		    else {
+			reply = this.document.getContent().substring(this.pos, end) + ">";
+			this.pos = end;
+		    }
+		}
+	}
+	else
+	    { // hunt for the beginning of the next tag
+		start = this.pos;
+		while ((this.pos < this.document.getContent().length()) &&
+		       (this.document.getContent().charAt(this.pos) != '<')){
+		    this.pos ++;
+		}
+		
+		// return everything up to that tag
+	    reply = this.document.getContent().substring(start, this.pos);
+	    }
+	return reply;
+    }
+
+    // Get the next tag to parse
+    public String nextParse()
+    { 
+	String reply = null;
+	int end;
+
+	this.lastpos = this.pos;
+
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	if (this.pos >= this.document.getContent().length()){
+	    return reply;
+	}
+	
+	// hunt for the beginning of the next tag
+	while ((this.pos < this.document.getContent().length()) &&
+	       (this.document.getContent().charAt(this.pos) != '<')){
+	    this.pos ++;
+	}
+	
+	// if we're not at the end of the document,
+	// read the rest of the tag
+	if (this.pos < this.document.getContent().length()){
+	    if (this.pos == this.document.getContent().length() - 1){
+		this.pos = this.document.getContent().length();
+		return reply;
+	    }
+	    
+	    end = this.pos + 1;
+	    while (end < this.document.getContent().length() &&
+			this.document.getContent().charAt(end) != '>'){
+		end ++;
+	    }
+	    
+	    // get the whole of the tag into 'reply', and
+	    // set the current pos to immediately after the tag
+	    if (end < this.document.getContent().length()){
+		reply = this.document.getContent().substring(this.pos, end + 1);
+		this.pos = end + 1;
+	    }
+	    else{ 
+		this.pos = end;
+	    }
+	}
+	return reply;
+    }
+    
+    
+    /* --
+       -- return next HREF value we come across in the document
+       --
+       -- NB: the checking of the quotes etc is rather lazy - and should be tidied
+       --
+    */
+    public String nextHREF()
+    { 
+	String reply;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    
+	    if (tag.tagName().equals("a")) {
+		start = reply.indexOf("href");
+		if (start == -1) {
+		    start = reply.indexOf("HREF");
+		}
+		
+		if (start >= 0) {
+		    start += 4;
+		    quoted = false;
+		    while (reply.charAt(start) == ' ' ||
+			   reply.charAt(start) == '=' ||
+			   reply.charAt(start) == '"') {
+			if (reply.charAt(start) == '"') {
+			    quoted = true;
+			}
+			start ++;
+		    }
+		    
+		    end = -1;
+		    if (quoted) {
+			end = reply.indexOf('"', start+1);
+		    }
+		    if (end == -1) {
+			end = reply.indexOf(' ', start+1);
+			if (end == -1) {
+			    end = reply.length() - 1;
+			}
+		    }
+		    
+		    reply = reply.substring(start, end);
+		    return reply;
+		}
+	    }
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+
+    // Return the next link in the page
+    public String nextLink(AppletContext ac)
+    { 
+	String reply, reply2;
+	int start, end;
+	boolean quoted;
+	HTMLTag tag;
+	
+	if (this.document.getContent() == null){
+	    return null;
+	}
+	
+	reply = this.nextParse();
+	while (this.pos < this.document.getContent().length()){
+	    tag = new HTMLTag(reply);
+	    if (tag.tagName().equals("a")){
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){ // image tag
+		reply = tag.idValue("src");
+	    }
+	    else if (tag.tagName().equals("area")) { // image map area
+		reply = tag.idValue("href");
+	    }
+	    else if (tag.tagName().equals("frame")){
+		reply = tag.idValue("src");
+	    }
+	    else {
+		reply	= null;
+	    }
+	    
+	    if (reply != null){
+		return reply;
+	    }
+ 
+	    reply = this.nextParse();
+	}
+	return null;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTag.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTag.java	(revision 21112)
@@ -0,0 +1,265 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class HTMLTag implements HTMLObject
+{ String tagtext;
+  String name;
+  int	 docpos;
+  int    endpos;
+
+  public static final String HTML_ELEMENT_TYPE = "Element";
+
+  public HTMLTag(String tagtext, int docpos, int endpos)
+  { this.tagtext = tagtext;
+    this.name    = null;
+    this.docpos  = docpos;
+    if (this.docpos < 0)
+    { this.endpos = -1;
+    }
+    else
+    { this.endpos = endpos;
+    }
+  }
+
+  public HTMLTag(String tagtext)
+  { this.tagtext 	= tagtext;
+    this.docpos		= -1;
+    this.name			= null;
+  }
+
+  public String getHTMLType()
+  { return HTML_ELEMENT_TYPE;
+  }
+
+	public HTMLTag endTag()
+	{ HTMLTag end;
+		String	endtext;
+
+		endtext = "</"+this.xtagName()+">";
+
+		end = new HTMLTag(endtext);
+		return end;
+	}
+
+	public boolean tagIsOpening()
+	{ return (this.tagtext.charAt(1) != '/');
+	}
+
+	/**
+	 *  Tells you if a tag is a singleton tag - TODO: complete list
+	 */
+	public boolean tagIsSingleton()
+	{	String tagName = this.tagName();
+		
+		return (tagName.equals("img"));
+	}
+
+	/**
+	 *  @return: the name of a tag including any leading '/'
+	 */
+	public String tagName()
+	{ int offset, start, end;
+		String reply;
+
+		if (this.name != null)
+		{	return this.name;
+		}
+	
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+
+		reply = this.tagtext.substring(start, end);
+		reply = reply.toLowerCase();
+
+		this.name	= reply;
+
+		return reply;
+	}
+
+	// Returns the name of the tag *minus* any leading / for
+	// closing tags
+	public String xtagName()
+	{ String reply;
+
+		reply = this.tagName();
+		if (reply == null || reply.length() == 0)
+		{	return null;
+		}
+
+		if (reply.charAt(0) == '/')
+		{ if (reply.length() > 1)
+			{	reply = reply.substring(1, reply.length());
+			}
+			else
+			{ reply = "";
+			}
+		}
+		return reply;
+	}
+
+	public String endTagName()
+	{	String reply;
+
+		reply = "/" + xtagName();
+		return reply;
+	}
+
+	public boolean isClosing()
+	{ String name;
+
+		name	= this.tagName();
+		if (name.charAt(0) == '/')
+		{	return true;
+		}
+		return false;
+	}
+
+	public int tagLevel()
+	{ String	name;
+		int			level;
+
+		level = 7;
+
+		name = this.xtagName();
+		if (name.charAt(0) == 'h')
+		{ level = Integer.parseInt(name.substring(1, 2));
+		}
+		
+		return level;
+	}
+
+	public int startPos()
+	{	return this.docpos;
+	}
+
+	public int endPos()
+	{	return this.endpos;
+	}
+
+	static public boolean isTag(String text)
+	{ if (text.charAt(0) == '<')
+		{ return true;
+		}
+		return false;
+	}
+
+	public String toString()
+	{	return this.tagtext;
+	}
+
+	//
+	// Returns the value of the identifier given as a string
+	//
+	public String idValue(String idName)
+	{	int offset, start, end;
+		int idstart, idend, valuestart, valueend;
+		boolean isvalue, isquoted, isstopped;
+		char		quotechar;
+		String reply;
+
+		// NB: a starting '<' and an ending '>' are guaranteed
+		offset = 1; // skip <
+		while (this.tagtext.charAt(offset) <= 32 ||
+					 this.tagtext.charAt(offset) == 127)
+		{ offset ++;
+		}
+		start = offset;
+		while (this.tagtext.charAt(offset) > 32 &&
+					 this.tagtext.charAt(offset) != '>' &&
+					 this.tagtext.charAt(offset) != 127)
+		{ offset ++;
+		}
+		end = offset;
+		// End of tag name
+
+		if (this.tagtext.charAt(offset) == '>')
+		{ return null;
+		}
+
+		isstopped = false;
+		while (!isstopped)
+		{	isvalue		= false;
+			isquoted	= false;
+			valuestart = -1;
+			valueend = -1;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+			idstart = offset;
+
+			if (this.tagtext.charAt(offset) == '>')
+			{ break;
+			}
+
+			while (this.tagtext.charAt(offset) > 32 &&
+						 this.tagtext.charAt(offset) != '=' &&
+						 this.tagtext.charAt(offset) != '>' &&
+						 this.tagtext.charAt(offset) != 127)
+			{ offset ++;
+			}
+			idend = offset;
+
+			while (this.tagtext.charAt(offset) <= 32 ||
+						 this.tagtext.charAt(offset) == 127)
+			{ offset ++;
+			}
+
+			if (this.tagtext.charAt(offset) == '=')
+			{ isvalue = true;
+				quotechar = ' ';
+				offset ++;				
+
+				while (this.tagtext.charAt(offset) <= 32 ||
+							 this.tagtext.charAt(offset) == 127)
+				{ offset ++;
+				}
+				if (this.tagtext.charAt(offset) == '"' ||
+						this.tagtext.charAt(offset) == '\'')
+				{ quotechar = this.tagtext.charAt(offset);
+					isquoted = true;
+					offset ++;
+				}
+				valuestart = offset;
+
+				while (offset < this.tagtext.length() &&
+							 (isquoted && this.tagtext.charAt(offset) != quotechar) ||
+							 (!isquoted && this.tagtext.charAt(offset) > 32 &&
+							   this.tagtext.charAt(offset) != '>'))
+				{ offset ++;
+				}
+				valueend = offset;
+			}
+
+			if (offset >= this.tagtext.length() ||
+					this.tagtext.charAt(offset) == '>')
+			{ isstopped = true;
+			}
+			offset ++;
+
+			reply = this.tagtext.substring(idstart, idend);
+			reply = reply.toLowerCase();
+
+			if (reply.equalsIgnoreCase(idName))
+			{ if (isvalue)
+				{ return this.tagtext.substring(valuestart, valueend);
+				}
+				else
+				{ return reply;
+				}
+			}
+		}
+		return null;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTagVector.java	(revision 21112)
@@ -0,0 +1,60 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.*;
+
+public class HTMLTagVector
+{ Vector vector;
+
+	//
+	// private: creates a Vector of tags from a comma separated list
+	//
+	public HTMLTagVector(String list)
+	{ int pos;
+		String tagname;
+
+		this.vector = new Vector(1);
+
+		// get first index
+		pos = list.indexOf(',');
+
+		// add leading tag to index, if there is > 1 tag currently
+		// in the list, and discard the tag
+		while (pos >= 0)
+		{ tagname = list.substring(0, pos);
+			list		= list.substring(pos+1, list.length());
+			this.vector.addElement((Object) tagname);
+
+			pos			= list.indexOf(',');
+		}
+
+		// add the last tag to the list
+		tagname = list;
+		this.vector.addElement((Object) tagname);
+	}
+
+	public HTMLTagVector(Vector vector)
+	{ this.vector = vector;
+	}
+
+	/**
+	 * Check whether a given tag is in the tagVector.  If <CODE>opening</CODE> is false,
+	 * then the method will return true for both opening and closing versions of the tag.
+	 * If the
+	 */
+	public boolean matches(HTMLTag tag, boolean opening)
+	{ int tagref;
+
+		if (tag.xtagName() == null)
+		{	return false;
+		}
+
+		for (tagref = 0; tagref < this.vector.size(); tagref ++)
+		{ if ((tag.xtagName().equals((String) this.vector.elementAt(tagref))) &&
+		      (tag.tagIsOpening() || !opening))
+			{ return true;
+			}
+		}
+		return false;
+	}
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTMLTidy.java	(revision 21112)
@@ -0,0 +1,232 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.File;
+import java.io.FileReader;
+
+import java.net.URL;
+
+import org.w3c.dom.*;
+
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.XMLReaderFactory;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.ext.LexicalHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentList;
+import org.greenstone.gsdl3.gs3build.doctypes.GMLDocument;
+
+public class HTMLTidy extends DefaultHandler
+{
+    String label;
+    File   file;
+    StringBuffer value;
+    boolean inElement;
+    boolean accumulate;
+    Document document;
+    Element  currentElement;
+    Element root;
+
+    class HTMLCommentHandler implements LexicalHandler
+    {
+	HTMLTidy parent;
+
+	HTMLCommentHandler(HTMLTidy tidier)
+	{ 
+	    this.parent = tidier;
+	}
+
+	public void comment(char ch[], int start, int length)
+	{ 
+	    String comment = new String(ch, start, length);
+	    parent.comment(comment);
+	}
+	
+	public void endCDATA()
+	{
+	}
+
+	public void endDTD()
+	{
+	}
+
+	public void endEntity(String name)
+	{
+	}
+
+	public void startCDATA()
+	{
+	}
+
+	public void startDTD(String name, String publicId, String systemId)
+	{
+	}
+
+	public void startEntity(String name)
+	{
+	}
+    }
+
+    public HTMLTidy(URL url)
+    { 
+	// TODO: complete
+    }
+
+    public HTMLTidy(File file)
+    { 
+	this.file = null;
+	this.label = null;
+	this.value = null;
+	
+	this.document = new org.apache.html.dom.HTMLDocumentImpl();
+	
+	this.currentElement = this.document.getDocumentElement();
+	this.root = this.currentElement;
+	
+	try {
+	    XMLReader handler  = new org.ccil.cowan.tagsoup.Parser();
+	    HTMLCommentHandler commentHandler = new HTMLCommentHandler(this);
+	    handler.setContentHandler(this);
+	    handler.setProperty("http://xml.org/sax/properties/lexical-handler", commentHandler);
+	    handler.parse(file.toURI().toString());
+	}
+	catch (java.io.IOException io)
+	    {
+		System.out.println(io);
+	    }
+	catch (org.xml.sax.SAXException saxEx)
+	    {
+		System.out.println(saxEx);
+	    }
+    }
+    
+    public void startElement(String URI, String localName, String qName, Attributes attributes)
+    { 
+	//System.out.println("<"+localName+">");
+	
+	if (localName.equals("html") == false) {
+	    Element childElement = this.document.createElement(localName);
+	    
+	    this.currentElement.appendChild(childElement);
+	    this.currentElement = childElement;
+	    
+	    for (int a = 0; a < attributes.getLength(); a ++){
+		String name = attributes.getLocalName(a);
+		String value = attributes.getValue(a);
+		
+		childElement.setAttribute(name, value);
+	    }
+	}
+    }
+    
+    public void comment(String text)
+    {
+	Comment comment = document.createComment(text);
+	this.currentElement.appendChild(comment);
+	//System.out.println("<!-- "+text+" -->");
+    }
+    
+    public void endElement(String URI, String localName, String qName)
+    { 
+	// System.out.println("Ending " + localName);
+	// System.out.println("</"+localName+">");
+	
+	if (localName.equals("html")) {
+	    return;
+	}
+	
+	// TODO: check is the same name as current node
+	if (!this.currentElement.getTagName().equals(localName.toUpperCase())) {
+	    // TODO: error handling
+	    System.out.println("Bad end at " + localName + " " + this.currentElement.getTagName());
+	    return;
+	}
+	
+	// Optimisation: is this an empty tag?
+	boolean isEmpty = false;
+	
+	if (this.currentElement.getChildNodes().getLength() == 1){
+	    Node child = this.currentElement.getFirstChild();
+	    
+	    if (child.getNodeType() == Node.TEXT_NODE) {
+		String text = child.getNodeValue();
+		
+		int i = 0;
+		while (i < text.length()){
+		    if (text.charAt(i) > 32 && text.charAt(i) != 160) {
+			break;
+		    }
+		    i++;
+		}
+		if (text.length() == i) {
+		    isEmpty = true;
+		}
+	    }
+	}
+	else if (this.currentElement.getChildNodes().getLength() == 0){
+	    isEmpty = true;
+	}
+	
+	boolean isSingleton = false;
+	String tagName = this.currentElement.getTagName().toLowerCase();
+	
+	if (tagName.equals("img") ||
+	    tagName.equals("br") ||
+	    tagName.equals("meta") ||
+	    tagName.equals("input") ||
+	    tagName.equals("area") ||
+	    tagName.equals("link") ||
+	    tagName.equals("base") ||
+	    //tagName.equals("img") ||
+	    tagName.equals("hr")) {
+	    isSingleton = true;
+	}
+	
+	Element parent =  (Element) this.currentElement.getParentNode();
+
+	if (isEmpty == true && !isSingleton) {
+	    // trim the current child
+	    parent.removeChild(this.currentElement);
+	    //      System.out.println("***Removing child***");
+	}
+	
+	this.currentElement = parent;
+    }
+    
+    public void characters(char c[], int start, int length)
+    { 
+	String string = new String(c, start, length);
+	Node text_node = this.document.createTextNode(string);
+	this.currentElement.appendChild(text_node);
+	//    System.out.println(string);
+    }
+    
+    public void recursePrint(Element element, int indent)
+    {
+	for (int i = 0; i < indent; i++) {
+	    System.out.print(" ");
+	}
+	System.out.println(element.toString());
+	for (int c = 0; c < element.getChildNodes().getLength(); c ++){
+	    Node n = element.getChildNodes().item(c);
+	    
+	    if (n.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+		Element e = (Element) n;
+		
+		this.recursePrint(e, indent+2);
+	    }
+	}
+    }
+    
+    public Document getDocument()
+    { 
+	//    this.recursePrint(this.root, 0);
+	//    System.out.println(this.document.toString());
+	//    System.out.println(this.root);
+	return this.document;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTTPTools.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/HTTPTools.java	(revision 21112)
@@ -0,0 +1,22 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.io.IOException;
+import java.net.*;
+
+public class HTTPTools
+{
+	public static String getMIMEType(URL url)
+	{ try
+		{	// initiate the connection
+			URLConnection connection = url.openConnection();
+
+			connection.connect(); // actually make the connection
+
+			return connection.getContentType();
+		}
+		catch (IOException exception)
+		{ // TODO: Log error reading url here...
+		}
+		return null;
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/MultiMap.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/MultiMap.java	(revision 21112)
@@ -0,0 +1,74 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.ArrayList;
+
+public class MultiMap
+{	Map map;
+	
+	public MultiMap()
+	{	this.map = new HashMap();
+	}
+
+	public boolean containsKey(Object key)
+	{	return this.map.containsKey(key);
+	}
+
+	public void setOnly(Object key, Object value)
+	{	List child = new ArrayList();
+		child.add(value);
+		this.map.put(key, child);
+	}
+
+	public void put(Object key, Object value)
+	{	List children;
+
+		children = (List) this.map.get(key);
+		if (children == null)
+		{	children = new ArrayList();
+			this.map.put(key, children);
+		}
+		children.add(value);
+	}
+
+	public Object get(Object key)
+	{	List list = this.getAll(key);
+		if (list == null)
+		{	return null;
+		}
+		return list.get(0);
+	}
+
+	public List getAll(Object key)
+	{	return (List) this.map.get(key);
+	}
+
+	public void remove(Object key)
+	{	this.map.remove(key);
+	}
+
+	public boolean remove(Object key, Object value)
+	{	if (!this.map.containsKey(key))
+		{ return false;
+		}
+
+		List children = (List) this.map.get(key);
+
+		if (children != null)
+		{	for (int i = 0; i < children.size(); i ++)
+			{	if (children.get(i).equals(value))
+				{	children.remove(i);
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+	public Set keySet()
+	{	return this.map.keySet();
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/SQLUtils.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/SQLUtils.java	(revision 21112)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class SQLUtils
+{
+	public void initCollection(String collectionName)
+	{	
+		// Check if the collection already exists
+		// if so, abort
+		// else
+		//   create document table
+		//   create section table
+		//   create metadata table
+		//   create namespace table
+		//   create metadata values table
+		//   create file section table
+		//   create file groups table
+		//   create file table
+		//   create admin table...etc.
+	}
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLString.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLString.java	(revision 21112)
@@ -0,0 +1,619 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.applet.*;
+import java.net.*;
+
+public class URLString
+{ private String url;
+
+  private String urlbase(String url)
+  { int index;
+    String tailString, bodyString;
+    int body;
+
+    index = url.indexOf("//");
+    tailString = url.substring(index+2, url.length());
+    
+    body = tailString.indexOf('/');
+    bodyString = tailString.substring(0, body+1);
+    
+    tailString = url.substring(0, index+2) + bodyString;
+    return tailString;
+  }
+  
+  public URLString(String s)
+  { this.url = s;
+  }
+
+  public URLString(String s, String target)
+  { // reconcile
+    int sref;
+    int cref;
+    int	c;
+
+    // trim fragment identifiers
+    target = noRef(target);
+    if (target == null)
+    { this.url = s;
+      return;
+    }
+
+    // test for absolute
+    cref = target.indexOf(':');
+    if (cref >= 0)
+    { for (c = 0; c < cref; c ++)
+      { if (target.charAt(c) != '+' &&
+	    target.charAt(c) != '-' &&
+	    target.charAt(c) != '.' &&
+	    Character.isLetterOrDigit(target.charAt(c)) == false)
+	{ break;
+	}
+      }
+    }
+    else
+    { c = 0;
+    }
+
+    if (c == cref)
+    { // absolute reference
+      String tmp;
+
+      tmp = target.substring(0, c);
+      tmp = tmp.toLowerCase();
+      if (target.indexOf("//") == cref + 1)
+      { if (target.lastIndexOf('/') == cref + 2)
+	{ target = target + "/";
+	}
+      }
+      /*			if (tmp.equals("http") == false)
+				{ this.url = null;
+				return;
+				}*/
+      this.url = target;
+    }
+    else if (target.indexOf("//") == 0)
+    { // absolute - implicit scheme (ie http).; also check for no trailing '/'
+      if (target.lastIndexOf('/') == 1)
+      { target = target + "/";
+      }
+      target = "http:" + target;
+      this.url = target;
+    }
+    else
+    { // relative - trim silly '/' at end
+      if (target.charAt(0) == '/')	
+      { // if we get a leading '/' then add it to the base of the url
+	s = urlbase(s);
+	target = target.substring(1);
+	this.url = s + target;
+//				System.out.println(this.url+"<"+s+"<"+target);
+	return;
+      }
+
+      while(target.length() >= 2 && target.charAt(0) == '.' && target.charAt(1) == '.')
+      { // move up a level on the left
+	
+	sref = s.lastIndexOf('/');
+	s = s.substring(0, sref);
+
+	target = target.substring(3);
+      }
+      // strip trailing leaf from s
+      sref = s.lastIndexOf('/');
+      s = s.substring(0, sref+1);
+      // append s+target
+      this.url = s + target;
+    }
+    
+    // abandon the url if it is a call to a script
+    if (this.url.indexOf('?') >= 0)
+    { this.url = null;
+    }
+  }
+
+  public static boolean dirParent(String parent, String child)
+  { while (child.length() >= parent.length())
+    { child = child.substring(0, child.lastIndexOf('/'));
+
+      if (parent.equals(child))
+      {	return true;
+      }
+    }
+    return false;
+  }
+
+  public static int dirPos(String from, String to)
+  { String fromdir;
+    String todir;
+
+    fromdir = from.substring(0, from.lastIndexOf('/'));
+    todir   = to.substring(0, to.lastIndexOf('/'));
+
+    if (fromdir.equals(todir))
+    { return 0;
+    }
+    if (fromdir.length() > todir.length())
+    { if (dirParent(todir, fromdir) == true)
+      {	return -1;
+      }
+    }
+    else 
+    { if (dirParent(fromdir, todir) == true)
+      { return 1;
+      }
+    }
+    return -2;
+  }
+
+  public static int commonLevels(String a, String b)
+  { String 	path_a, path_b;
+    int			offset_a, offset_b;
+    String	dir_a, dir_b;
+    int			common;
+
+    path_a	= path(a);
+    path_b	= path(b);
+    common	= 0;
+
+    if (path_a == null || path_b == null)
+    { return 0;
+    }
+
+    do
+    { offset_a = path_a.indexOf('/');
+      offset_b = path_b.indexOf('/');
+
+      if (offset_a > 0)
+      { dir_a		=	path_a.substring(0, offset_a);
+        if (path_a.length() > offset_a + 1)
+	{	path_a	= path_a.substring(offset_a+1);
+	}
+	else
+	{	path_a	= null;
+	}
+      }
+      else
+      {	dir_a		=	path_a;
+        path_a	= null;
+      }
+
+      if (offset_b > 0)
+      {	dir_b		=	path_b.substring(0, offset_b);
+        if (path_b.length() > offset_b + 1)
+	{	path_b	= path_b.substring(offset_b+1);
+	}
+	else
+	{	path_b	= null;
+	}
+      }
+      else
+      {	dir_b		=	path_b;
+        path_b	= null;
+      }
+
+      if (dir_a.length() != dir_b.length() ||
+	  dir_a.equals(dir_b) == false)
+      {	return common;
+      }
+      common ++;
+    } while (path_a != null && path_b != null);
+    
+    return common;
+  }
+
+	public static String dirLevelName(String url, int level)
+	{	String 	lpath;
+		String	dir;
+		int			thislevel = 0;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{ return null;
+		}
+		do
+		{ offset = lpath.indexOf('/');
+
+			if (offset >= 0)
+			{	dir = lpath.substring(0, offset);
+				lpath = lpath.substring(offset+1);
+			}
+			else
+			{	dir = lpath;
+			}
+
+			thislevel ++;
+
+			if (level == thislevel)
+			{ return dir;
+			}
+		} while (offset >= 0);
+
+		return null;
+	}
+
+	public static int dirLevel(String url)
+	{	String 	lpath;
+		int			level;
+		int			offset;
+
+		lpath = path(url);
+		if (lpath == null)
+		{	return 0;
+		}
+		level = 0;
+		do
+		{ offset = lpath.indexOf('/');
+			if (offset >= 0)
+			{	lpath = lpath.substring(offset+1);
+			}
+			level ++;
+		} while (offset >= 0);
+
+		return level;
+	}
+
+	public static String tidyURL(String urlString, boolean permitScript)
+	{	int cref;
+		int c;
+		int sref;
+		String tmp, s;
+
+		s = urlString;
+		cref = s.indexOf(':');
+		if (cref >= 0)
+		{ for (c = 0; c < cref; c ++)
+			{ if (s.charAt(c) != '+' &&
+						s.charAt(c) != '-' &&
+						s.charAt(c) != '.' &&
+						Character.isLetterOrDigit(s.charAt(c)) == false)
+				{ break;
+				}
+			}
+		}
+		else
+		{ c = 0;
+		}
+
+		if (cref == c)
+		{ tmp = s.substring(0, c).toLowerCase();
+			s = tmp + s.substring(c, s.length());
+			if (s.lastIndexOf('/') == cref + 2)
+			{ s = s + "/";
+			}
+		}
+		else
+		{ sref = s.indexOf("//");
+			if (sref == 0)
+			{ s = "http:" + s;
+			}
+			else
+			{ s = "http://" + s;
+			}
+			if (s.lastIndexOf('/') == 6)
+			{ s = s + "/";
+			}
+		}
+		return s;
+	}
+
+	public static String tidyURL(String urlString)
+	{	return tidyURL(urlString, false);
+	}
+
+	public static URL toURL(String urlString)
+	{ URL reply;
+
+	  try
+		{ reply = new URL(URLString.tidyURL(urlString));
+		}
+		catch (MalformedURLException ex)
+		{ return null;
+		}
+		return reply;
+	}
+
+  public void tidy()
+  { this.url = tidyURL(this.url);
+  }
+
+  public boolean isNull()
+  { if (this.url == null)
+    { return true;
+    } 
+    return false;
+  }
+
+  public static boolean isForeign(String home, URL url)
+  { if (url.toString().length() < home.length())
+    { return true;
+    }
+
+    if (home.equals(url.toString().substring(0, home.length())))
+    { return false;
+    }
+    return true;
+  }
+  
+  /**
+   *	Remove any trailing "#localanchor" local reference from a URL string
+   *
+   *  @param  <code>String</code> the original url
+   *  @return <code>String</code> the cleaned url, sans local references
+   */
+  public static String noRef(String target)
+  { if (target == null)
+    { return null;
+    }
+    if (target.indexOf('#') >= 0)
+    { if (target.indexOf('#') == 0)
+      { return null;
+      }
+      target = target.substring(0, target.indexOf('#'));
+    }
+    return target;
+  }
+
+  /**
+   *	Obtain the trailing local anchor reference from a URL, 
+   *  if it exists.
+   *    e.g. for http://www.test.mdx.ac.uk/example/index.html#credits the
+   *         response would be "credits"
+   *
+   *  @param  <code>String</code> the original url.
+   *  @return <code>String</code> the local reference.  <code>null</code> is
+   *                              returned if no local reference exists.
+   */
+  public static String subRef(String target)
+  { if (target.indexOf('#') >= 0)
+    {	return target.substring(target.indexOf('#')+1);
+    }
+    return null;
+  }
+
+  /**
+   *	Clean a URL of the form GET parameters.
+   *
+   *  @param  <code>String</code> the original url, with GET message parameters
+   *  @return <code>String</code> the cleaned url.
+   */
+  public static String noParameters(String url)
+  { int						c;
+
+    c = url.indexOf("?");
+    if (c < 0)
+    { return url;
+    }
+    return url.substring(0, c);
+  }
+
+	/**
+	 *	Obtain the form parameters from a url
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the form "GET" parameters.  The leading '?' is
+	 *                              preserved in the reply.  If no parameters were
+	 *                              given, the result is <code>null</code>.
+	 */
+	public static String formParameters(String url)
+	{	int						c;
+
+		c = url.indexOf("?");
+		if (c < 0)
+		{	return null;
+		}
+		return url.substring(c);
+	}
+
+	/**
+	 *	Flatten a URL, making the '?' and '&' items into underscores
+	 *
+	 *  @param  <code>String</code> the original url, with GET message parameters
+	 *  @return <code>String</code> the "flattened" url
+	 */
+	public static String flattenedUrl(String oldUrl)
+	{	StringBuffer	reply;
+		int						c;
+		boolean				modified = false;
+
+		// replace ? with _
+		// replate & with _
+		reply = new StringBuffer(oldUrl);
+		for (c = 0; c < reply.length(); c ++)
+		{	if (reply.charAt(c) == '?' ||
+					reply.charAt(c) == '&')
+			{	reply.setCharAt(c, '_');
+				modified = true;
+			}
+		}
+
+		if (modified == true)
+		{	c	= oldUrl.indexOf(".html");
+			if (c >= 0)
+			{	while (c < oldUrl.length() - 5)
+				{	reply.setCharAt(c, reply.charAt(c+5));
+					c ++;
+				}
+				reply.setCharAt(c ++, '.');
+				reply.setCharAt(c ++, 'h');
+				reply.setCharAt(c ++, 't');
+				reply.setCharAt(c ++, 'm');
+				reply.setCharAt(c ++, 'l');
+			}
+		}
+
+		return reply.toString();
+	}
+
+	/**
+	 *	Provide the complete pathname in the given url
+	 *    eg. "example/index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String pathName(String url)
+	{	int offset;
+		String remainder;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.indexOf("//");
+		if (offset < 0)
+		{ return null;
+		}
+
+		remainder = url.substring(offset + 2);
+		offset = remainder.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+		return remainder.substring(offset + 1);
+	}
+
+	/**
+	 *	Provide the leaf filename in the given url
+	 *    eg. "index.html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> leaf file name
+	 */
+	public static String leafName(String url)
+	{	int offset;
+		int dot;
+
+		if (url == null)
+		{	return null;
+		}
+		offset = url.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		if (offset == url.length() - 1)
+		{ return null;
+		}
+		dot = url.lastIndexOf('/');
+		if (dot < offset)
+		{	return null;
+		}
+		return url.substring(offset+1);
+	}
+
+	/**
+	 *	Provide the hostname in the given url
+	 *    eg. "www.test.mdx.ac.uk" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> host name
+	 */
+	public static String host(String url)
+	{	String scheme, tail;
+		int		 spos;
+
+		spos = url.indexOf("//");
+		if (spos >= 0)
+		{ scheme = url.substring(0, spos + 2);
+			tail = url.substring(spos + 2, url.length());
+			// do sub-areas
+			spos = tail.indexOf('/');
+			if (spos >= 0)
+			{ tail = tail.substring(0, spos+1);
+			}
+			scheme = scheme + tail;
+			return scheme;
+		}
+		return null;
+	}
+
+	/**
+	 *	Return the logical path of the url on the remote server
+	 *    eg. "example" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *  @param  <code>String</code> url of page
+	 *	@return <code>String</code> logical path
+	 */
+	public static String path(String url)
+	{ String tail;
+		int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset 	= url.indexOf("//");
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= url.substring(offset+2);
+		offset	= tail.indexOf('/');
+		if (offset < 0)
+		{	return null;
+		}
+
+		tail		= tail.substring(offset+1);
+		if (tail == null || tail.length() == 0)
+		{	return null;
+		}
+
+		offset	= tail.lastIndexOf('/');
+		if (offset < 0)
+		{ return null;
+		}
+		else
+		{	tail = tail.substring(0, offset);
+		}
+		return tail;
+	}
+
+	/**
+	 *	Give the 'dot' extension of a url; 
+	 *    eg. "html" for http://www.test.mdx.ac.uk/example/index.html
+	 *
+	 *	@param <code>String</code> url
+	 *  @return <code>String</code> extension.
+	 */
+	public static String extension(String url)
+	{ int offset;
+		String leaf;
+
+		if (url == null)
+		{ return null;
+		}
+
+		offset = url.lastIndexOf('/');
+		if (offset >= 0 && offset < url.length() - 1)
+		{ leaf = url.substring(offset+1, url.length());
+			offset = leaf.lastIndexOf('.');
+
+			// if we've got an extension then use it
+			if (offset >= 0)
+			{ int poffset;
+
+				// get parameter offset
+				poffset = leaf.indexOf('?');
+				if (poffset < 0)
+				{	poffset = leaf.length();
+				}
+
+				// give leaf value
+				return leaf.substring(offset, poffset);
+			}
+			else
+			{ return null;
+			}
+		}
+		return null;
+	}
+
+  public static String extension(URL url)
+  { return extension(url.toString());
+  }
+
+  public String toString()
+  { return this.url;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLTools.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/URLTools.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+import java.net.URL;
+import java.io.File;
+
+public class URLTools
+{
+  public static URL getFileURL(File file)
+  { URL url;
+    String normalisedFileName = file.toString().replace(File.separatorChar, '/');
+
+    try {
+      url = new URL("file", "", normalisedFileName);
+    }
+    catch (java.net.MalformedURLException ex) {
+      return null;
+    }
+    return url;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/XMLTools.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/util/XMLTools.java	(revision 21112)
@@ -0,0 +1,82 @@
+package org.greenstone.gsdl3.gs3build.util;
+
+public class XMLTools
+{
+    public static String getTag(String namespace, String label, boolean open)
+    {	
+	StringBuffer reply = new StringBuffer();
+
+	reply.append("<");
+	if (!open)
+	    {	
+		reply.append("/");
+	    }
+	if (namespace != null && namespace.length() > 0) {
+	    reply.append(namespace);
+	    reply.append(":");
+	}
+	reply.append(label);
+	reply.append(">");
+	return reply.toString();
+    }
+
+    public static String addAttribute(String tag, String attribute, String value)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(reply.length() - 1 , ' ');
+	reply.insert(reply.length() - 1 , attribute);
+	reply.insert(reply.length() - 1 , "=\"");
+	reply.insert(reply.length() - 1 , value);
+	reply.insert(reply.length() - 1 , '\"');
+	return reply.toString();
+    }
+
+    public static String makeSingleton(String tag)
+    {	
+	StringBuffer reply = new StringBuffer(tag);
+	reply.insert(tag.length() - 1, " /");
+	return reply.toString();
+    }
+    
+    public static String getOpenTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, true);
+    }
+    
+    public static String getCloseTag(String namespace, String label)
+    {	
+	return getTag(namespace, label, false);
+    }
+
+    /**
+     *  Clean a string to remove whitespace...
+     *
+     *  @param <code>String</code> the original text
+     *  @return <code>String</code> the 'cleaned' text
+     */
+    public static String cleanString(String original)
+    {	
+	StringBuffer buffer = new StringBuffer(original);
+
+	while (buffer.charAt(0) <= 32) {
+	    buffer.deleteCharAt(0);
+	}
+	
+	while (buffer.charAt(buffer.length() - 1) <= 32){
+	    buffer.deleteCharAt(buffer.length() - 1);
+	}
+	
+	int c = 1;
+	while (c < buffer.length()){
+	    if (buffer.charAt(c) < 32){
+		buffer.setCharAt(c, ' ');
+	    }
+	    if (buffer.charAt(c) == ' ' && buffer.charAt(c-1) == ' '){
+		buffer.deleteCharAt(c);
+		continue;
+	    }
+	    c ++;
+	}
+	return buffer.toString();
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/validator/ValidatorInterface.java	(revision 21112)
@@ -0,0 +1,24 @@
+package org.greenstone.gsdl3.gs3build.validator;
+
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentInterface;
+import org.greenstone.gsdl3.gs3build.doctypes.DocumentID;
+
+public interface ValidatorInterface
+{
+	/**
+	 *  Record the build manager to use for this expander - this is used
+	 *  to pass back any new directories into which expanded files are placed.
+	 */
+	public void configure();
+
+	/**
+	 *  Attempt to expand a file.  The returned result is <code>true</code> if the
+	 *  file was actually expanded.
+	 *
+	 *  @param <code>DocumentID</code> the identity code of the document
+	 *  @param <code>DocumentInterface</code> the document itself
+	 *
+	 *  @return <code>boolean</code> whether the file was validated.
+	 */
+	public boolean validateDocument(DocumentID docID, DocumentInterface document);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/IdentifierXPointer.java	(revision 21112)
@@ -0,0 +1,59 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class IdentifierXPointer extends XPointer
+{ NodeList nodes;
+
+  public IdentifierXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    this.xpointer = XPointer.expandXPath(this.xpointer);
+
+    this.nodes = this.getNodes();
+  }
+
+  private NodeList getNodes()
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(this.document, this.xpointer, 
+					       this.document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  public Node getStartNode()
+  { return this.nodes.item(0);
+  }
+
+  private void recurseStringify(NodeList nodes, StringBuffer buffer)
+  { 
+    for (int n = 0; n < nodes.getLength(); n ++) {
+      Node node = nodes.item(n);
+
+      // append the actual node itself
+      this.printNode(node, buffer, false);
+
+      // then process its children...
+      if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+	NodeList children = node.getChildNodes();
+	this.recurseStringify(children, buffer);
+      }
+
+      // do the close node
+      this.printNode(node, buffer, true);
+    }
+  }
+
+  public String toString()
+  {
+    StringBuffer reply = new StringBuffer();
+    this.recurseStringify(this.nodes, reply);
+    return reply.toString();
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/RangeToXPointer.java	(revision 21112)
@@ -0,0 +1,69 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+public class RangeToXPointer extends XPointer
+{
+  public static final String RANGE_TO = "range-to(";
+  NodeList startNodes;
+  NodeList endNodes;
+
+  public RangeToXPointer(Document document, String xpointer)
+  { super(document, xpointer);
+
+    String fromPointer, toPointer;
+
+    int rangeAt = xpointer.indexOf(RANGE_TO);
+
+    fromPointer = xpointer.substring(0, rangeAt-1);
+    toPointer = xpointer.substring(rangeAt+RANGE_TO.length(), xpointer.length()-1);
+
+    fromPointer = XPointer.expandXPath(fromPointer);
+    toPointer   = XPointer.expandXPath(toPointer);
+    //    System.out.println(fromPointer + " " + toPointer);
+    
+    this.startNodes = this.getNodes(fromPointer);
+    this.endNodes = this.getNodes(toPointer);
+    
+    //    System.out.println(this.startNodes.item(0).toString() + " " + this.endNodes.item(0).toString());
+  }
+
+  public Node getStartNode()
+  { return this.startNodes.item(0);
+  }
+
+  private NodeList getNodes(String pointer)
+  {
+    try {
+      NodeList nodes = XPathAPI.selectNodeList(document, pointer, document.getDocumentElement());
+      return nodes;
+    }
+    catch (Exception ex)
+    { System.out.println(ex.toString());
+    }
+    return null;
+  }
+
+  private void recurseStringify(Node at, Node end, StringBuffer reply)
+  {
+    while (at != end && at != null) {
+      this.printNode(at, reply, false);
+      at = this.getNextNode(at, reply);
+      //      System.out.println(at + " " + end);
+    }
+  }
+
+  public String toString()
+  { StringBuffer reply = new StringBuffer();
+
+    // skip the first node
+    Node firstNode = this.startNodes.item(0);
+    firstNode = this.getNextNode(firstNode, reply);
+    
+    // actually, we don't, as otherwise the indexed copy of 
+    // the document would be incomplete...
+    this.recurseStringify(this.startNodes.item(0), this.endNodes.item(0), reply);
+    return reply.toString();
+  }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/gs3build/xpointer/XPointer.java	(revision 21112)
@@ -0,0 +1,281 @@
+package org.greenstone.gsdl3.gs3build.xpointer;
+
+import java.io.File; 
+
+import java.net.URL;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.*;
+import org.apache.xpath.*;
+
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.greenstone.gsdl3.gs3build.doctypes.HTMLDocumentTools;
+import org.greenstone.gsdl3.gs3build.util.HTMLTidy;
+
+public abstract class XPointer
+{
+  Document document;  
+  String   xpointer;
+
+  public XPointer(org.w3c.dom.Document document, String xpointer)
+  { this.document = document;
+    this.xpointer = xpointer;
+    // now process the type of description
+  }
+
+  public static String expandXPath(String xpath)
+  {
+    int          dashAt;
+    String       pathElement;
+    StringBuffer expandedPtr = new StringBuffer();
+
+    if (xpath.startsWith("//")) {
+      xpath = xpath.substring(2);
+      expandedPtr.append("//");
+    }
+    else if (xpath.startsWith("/")) {
+      xpath = xpath.substring(1);
+      expandedPtr.append("/");
+    }
+
+    while (xpath.length() > 0) {
+      dashAt = xpath.indexOf('/');
+      if (dashAt >= 0) {
+	pathElement = xpath.substring(0, dashAt);
+	xpath = xpath.substring(dashAt+1);
+      }
+      else {
+	pathElement = xpath;
+	xpath = "";
+      }
+
+      int c = 0;
+      while (c < pathElement.length()) {
+	if (!Character.isDigit(pathElement.charAt(c))) 
+	  break;
+	c ++;
+      }
+
+      if (c == pathElement.length()) {
+	expandedPtr.append("*[");
+	expandedPtr.append(pathElement);
+	expandedPtr.append("]");
+      }
+      else {
+	expandedPtr.append(pathElement);
+      }
+
+      if (dashAt >= 0) {
+	expandedPtr.append('/');
+      }
+    }
+    xpath = expandedPtr.toString();
+
+    return xpath;
+  }
+
+  public static XPointer processXPointer(Document document, String xpointer)
+  { // do a "cheap and dirty" detection of range-to
+    if (xpointer.indexOf("range-to") >= 0) {
+      return new RangeToXPointer(document, xpointer);
+    } 
+    return new IdentifierXPointer(document, xpointer);
+  }
+
+  public static XPointer processXPointer(Document document, URL url)
+  { String reference = url.getRef();
+
+    reference.trim();
+    if (reference.startsWith("xpointer(")) {
+      reference = reference.substring(9, reference.length() - 1);
+    }
+    else {
+      return null;
+    }
+    //    System.out.println(reference);
+    
+    return processXPointer(document, reference);
+  }
+
+  public static void printNode(Node node, StringBuffer to, boolean close)
+  {
+    if (node.getNodeType() == org.w3c.dom.Node.TEXT_NODE) {
+      if (!close) {
+	to.append(node.getNodeValue());
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+      if (!close) {
+	to.append("&");
+	to.append(node.getNodeValue());
+	to.append(";");
+	to.append(" ");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.COMMENT_NODE) {
+      if (!close) {
+	to.append("<!-- ");
+	to.append(node.getNodeValue());
+	to.append(" -->");
+      }
+    }
+    else if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
+      boolean hasChildren = (node.getChildNodes().getLength() > 0);
+
+      if (close && !hasChildren) {
+	return;
+      }
+      to.append("<");
+      if (close) {
+	to.append("/");
+      }
+      to.append(node.getNodeName());
+
+      if (!close) {
+	NamedNodeMap attributes = ((Element) node).getAttributes();
+	for (int a = 0; a < attributes.getLength(); a ++) {
+	  Node attributeNode = attributes.item(a);
+	  to.append(" ");
+	  to.append(attributeNode.getNodeName());
+	  
+	  String value = attributeNode.getNodeValue();
+	  if (value != null && value.length() > 0) {
+	    to.append("=\"");
+	    to.append(value);
+	    to.append("\"");
+	  }
+	}
+      }
+
+      if (!hasChildren) {
+	to.append(" /");
+      }
+      to.append(">");
+      if (close && node.getNodeName()=="Section") {
+	  to.append((char) 3);
+      }
+    }
+  }
+
+    public static Node getNextNode(Node currentNode) {
+	
+	// Try to get the first valid child
+	NodeList children = currentNode.getChildNodes();
+	for (int c = 0; c < children.getLength(); c++) {
+	    Node childNode = children.item(c);
+	    
+	    if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+		childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+		return childNode;
+	    }
+	}	
+	
+	// now try a siblings and parents siblings
+	Node siblingNode = null;
+	do {
+	    siblingNode = currentNode.getNextSibling();
+	    if (siblingNode != null) {
+		// should we be checking type??
+		return siblingNode;
+	    }
+	    
+	    currentNode = currentNode.getParentNode();
+	} while (currentNode != null);
+	
+	return null;
+    }
+
+    
+  public static Node getNextNode(Node currentNode, StringBuffer buffer)
+  { Node reply;
+
+    // Try for children only the once...
+    NodeList children = currentNode.getChildNodes();
+    for (int c = 0; c < children.getLength(); c++) {
+      Node childNode = children.item(c);
+
+      if (childNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.COMMENT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.TEXT_NODE ||
+	  childNode.getNodeType() == org.w3c.dom.Node.ENTITY_NODE) {
+	return childNode;
+      }
+    }
+    
+    // Repeat looking for a sibling & going up until done...
+    do {
+      reply = currentNode.getNextSibling();
+      if (reply != null) {
+	return reply;
+      }
+      
+      currentNode = currentNode.getParentNode();
+      if (buffer != null && currentNode != null) {
+	printNode(currentNode, buffer, true);
+      }
+    } while (currentNode != null);
+
+    return reply;
+  }
+
+  public abstract Node getStartNode();
+
+
+  public static void main(String args[])
+  {	
+    try
+    { File file = new File(args[0]);
+
+      Document document;
+      if (args[0].endsWith(".html") || args[0].endsWith(".htm")) {
+	HTMLTidy tidier = new HTMLTidy(file);
+	document = tidier.getDocument();
+
+	HTMLDocumentTools docTools = new HTMLDocumentTools(document);
+	docTools.findSections();
+	System.out.println("Sections found");
+      }
+      else {
+	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	DocumentBuilder builder = factory.newDocumentBuilder();
+	document = builder.parse(file);
+      }
+      
+      XPointer pointer = null;
+      if (args.length < 2) {
+      }
+      else if (args.length == 2) {
+	pointer = XPointer.processXPointer(document, args[1]);
+      }
+      else {
+	pointer = XPointer.processXPointer(document, args[1]+"/range-to("+args[2]+")");
+      }
+      if (pointer != null) {
+	System.out.println(pointer.toString());
+      }
+    }
+    catch (FactoryConfigurationError e) {
+      System.err.println(e);
+    }
+    catch (ParserConfigurationException ex) {
+      System.err.println(ex);
+    }
+    catch (SAXException ex) {
+      System.err.println(ex);
+    }
+    catch (java.io.IOException ex) {
+      System.err.println(ex);
+    }
+  }
+}
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/DocumentStream.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/DocumentStream.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/DocumentStream.java	(revision 21112)
@@ -0,0 +1,55 @@
+/*
+ *    DocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import org.w3c.dom.Document; 
+
+// other java classes
+import java.io.Serializable;
+
+
+/**
+ * Document Stream represents a stream of XML documents
+ *
+ * Based on algorithms in ...
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * 
+ */
+public interface DocumentStream extends Cloneable, Serializable {
+  
+  /**
+   * Get the next document
+   * 
+   * @exception java.io.Exception when something goes wrong
+   * @return the next document
+   */
+  public Document nextDocument() throws Exception ;
+  /**
+   * Is there a next document?
+   * 
+   * @exception java.io.Exception when something goes wrong
+   * @return true if there is a next document
+   */
+  public boolean hasNextDocument() throws Exception ;
+  
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java	(revision 21112)
@@ -0,0 +1,196 @@
+/*
+ *    DocumentStreamFromFiles.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import org.w3c.dom.Document; 
+import org.xml.sax.InputSource;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+
+// other java classes
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Vector;
+
+
+  
+/**
+ * DocumentStreamFromFiles creates a stream of documents from a directory or directories.
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see java.io.File
+ * @see DocumentStream
+ */
+public class FilesDocumentStream 
+  implements DocumentStream {
+  /** the files we've found so far */
+  Vector files = new Vector();
+  /** the directories we've found so far */
+  Vector directories = new Vector();
+
+  /**
+   * Filename constructor
+   * 
+   * @exception java.io.IOException file access do
+   * @param filename the filename of the file or directory
+   */
+  public FilesDocumentStream(String filename)  {
+    File file = new File(filename);
+    if (file.exists() && file.isDirectory())
+      directories.add(file.getAbsolutePath());
+    else
+      files.add(file.getAbsolutePath());
+  }
+  /**
+   * File constructor
+   * 
+   * @exception java.io.IOException file access do
+   * @param file a handle to the file
+   */
+  public FilesDocumentStream(File file) {
+    if (file.exists() && file.isDirectory())
+      directories.add(file.getAbsolutePath());
+    else
+      files.add(file.getAbsolutePath());
+  }
+
+  /** 
+   * A small inner class to filter out those files that don't 
+   * end in ".xml" or ".XML"
+   */
+  class XMLFilenameFilter implements FilenameFilter {
+    /** Tests if a specified file should be included in a file list. */
+    public boolean accept(File dir, String name) {
+      String target = ".xml";
+      if (name.length() < target.length())
+	return false;
+      String extension = name.substring(name.length() - 4, name.length());
+      //System.out.println(name + " ==>> " +  extension);
+      if (extension.equalsIgnoreCase(target)) {
+	return true;
+      } else {
+	return false;
+      }
+    }    
+  }
+
+  /**
+   * Opens a new directory if there are no files are queued
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   */
+  protected void expandIfNecessary() {
+    while (directories.size() > 0) {
+      String dirname = (String) directories.elementAt(0);
+      directories.removeElement(dirname);
+      File dir = new File(dirname);
+      if (!dir.exists()) throw new Error ("error in expand: expecting a directory " + dirname);
+      String[] fileArray = dir.list();
+      for (int i=0;i<fileArray.length;i++){
+	File file = new File(dir, fileArray[i]);
+	if (file.exists() && file.isDirectory()) 
+	  directories.add(file.getPath());
+      }
+      
+      fileArray = dir.list(new XMLFilenameFilter());
+      for (int i=0;i<fileArray.length;i++){
+	File file = new File(dir, fileArray[i]);
+	if (file.exists() && !file.isDirectory())
+	  files.add(file.getPath());
+      }
+      if (files.size() > 0)
+	return;
+    }
+  }
+  
+  /**
+   * Returns the next document
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   * @return the next document
+   */
+  public Document nextDocument()
+    throws Exception {
+    
+    if (!hasNextDocument()) throw new Error("Doesn't have another Document");
+
+    String filename = (String) files.elementAt(files.size() - 1);
+    files.removeElementAt(files.size() - 1);
+    File file = new File(filename);
+    
+    Reader reader = new FileReader(file);
+    InputSource xml_source = new InputSource(reader);
+
+    XMLUtil.getDOMParser().parse(xml_source);
+    Document doc = XMLUtil.getDOMParser().getDocument();
+    
+    return doc;
+  }
+    
+  /**
+   * Is there another document ?
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   * @return the next document
+   */
+
+  public boolean hasNextDocument()  throws Exception {
+    expandIfNecessary();
+    return (files.size() > 0);
+  }
+  /**
+   * Tests...
+   *
+   * 
+   * @exception java.io.IOException when ...
+   * @param args the arguments ...
+   */
+
+  public static void main(String args[]) throws Exception
+  {
+   
+    StreamResult result = new StreamResult(System.out);
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    
+    FilesDocumentStream stream = new FilesDocumentStream(".");
+    while (stream.hasNextDocument()) {
+      Document document = stream.nextDocument();
+      Transformer transformer = transformerFactory.newTransformer();
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+      System.out.println();
+      System.out.println();
+    }
+
+  } 
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java	(revision 21112)
@@ -0,0 +1,148 @@
+/*
+ *    GenerateDocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+
+
+// other java classes
+
+/**
+ * GeneratedDocumentStream generates an unbounded stream of XML documents
+ * Each containing a single string.
+ *
+ * Based as a wrapper around an Increment object
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see DocumentStream
+ * @see Increment
+ * 
+ */
+public class GeneratedDocumentStream implements DocumentStream {
+  
+  /** The next string */
+  protected String next = "";
+  /** The underlying Increment object */
+  protected Increment incrementer = new IncrementRange();
+
+  /** The default constructor. Generates ASCII-only documents */
+  public GeneratedDocumentStream() {
+
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param incrementer the incrementer to use (controls the character set to use)
+   */
+  public GeneratedDocumentStream(Increment incrementer) {
+    this.incrementer = incrementer;
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param next the string for the first document (must suit the incrementor)
+   */
+  public GeneratedDocumentStream(String next) {
+    this.next = next;
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param incrementer the incrementer to use (controls the character set to use)
+   * @param next the string for the first document (must suit the incrementor)
+   */
+  public GeneratedDocumentStream(Increment incrementer, String next) {
+    this.incrementer = incrementer;
+    this.next = next;
+  }
+  /** 
+   * Custom constructor 
+   *
+   * @param first the first letter in the alphabet
+   * @param last the last letter in the alphabet
+   */
+  public GeneratedDocumentStream(char first, char last) {
+    this.incrementer = new IncrementRange(first,last);
+  }
+
+
+  /**
+   * Generates the nest document in the sequence
+   * 
+   * @exception java.io.Exception when something goes wrong in the XML stuff
+   * @return the next Document
+   * @see org.w3c.dom.Document
+   */
+  public Document nextDocument() {
+
+
+   DocumentImpl doc = new DocumentImpl(true);
+    
+   ElementImpl root = new ElementImpl(doc,"document");
+   TextImpl text = new TextImpl(doc,next);
+    
+   doc.appendChild(root);
+   root.appendChild(text);
+ 
+   next = incrementer.incrementString(next);
+   return doc;
+   
+  }
+  /**
+   * Are there more documents ?
+   * 
+   * @return yes
+   */
+  public boolean hasNextDocument(){
+    return true;
+  }
+  /**
+   * Generates an unbounded stream of documents
+   */
+  public static void main(String args[]) throws Exception
+  {
+    
+    StreamResult result = new StreamResult(System.out);
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    
+    GeneratedDocumentStream stream = new GeneratedDocumentStream();
+    while (stream.hasNextDocument()) {
+      Document document = stream.nextDocument();
+      Transformer transformer = transformerFactory.newTransformer();
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+    }
+  }
+  
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/Increment.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/Increment.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/Increment.java	(revision 21112)
@@ -0,0 +1,111 @@
+/*
+ *    Increment.java
+ *    Copyright (C) 2000-2001 Stuart Yeates 
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+// the package we're in
+package org.greenstone.gsdl3.selfContained;
+
+// java standard library classes used
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.Writer;
+
+
+/**
+ * Class Increment is an abstract class for incrementing streams and strings
+ *
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ */
+
+abstract public class Increment implements Cloneable, Serializable
+{
+  /**
+   * Increment a string of characters
+   * 
+   * @exception java.io.Exception if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  abstract String incrementString(String str);
+
+  /**
+   * Increment a stream of characters, reading from the reader and writing the incremented characters to the writer
+   * 
+   * @exception java.io.Exception if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  abstract long incrementStream(Reader rdr, Writer wtr);
+
+  /**
+   * Print to standard out a verbose message about the 
+   * results of incrementing this string
+   * 
+   * @param s the string to increment
+   * @see #incrementString(String)
+   */
+  final void checkStr(String s) {
+    try {
+      System.out.println("incrementing: \"" + s +
+			 "\" to \"" + incrementString(s) +
+			 "\"  " + stringToOctal(s) +
+			 " ===> " + stringToHex(incrementString(s)));
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+  }
+
+  /**
+   * Converts a unicode string to Hex
+   * 
+   * @exception java.io.IOException when ...
+   * @param s the string to convert
+   * @return a string in hex
+   * @see Integer#toHexString(int)
+   */
+  static final String stringToHex(String s) {
+    StringBuffer buffer = new StringBuffer();
+    for (int i=0;i<s.length();i++) {
+      buffer.append(" 0x");
+      buffer.append(Integer.toHexString(s.charAt(i)));
+    } 
+    return buffer.toString();
+  }
+
+  /**
+   * Converts a unicode string to Octal
+   * 
+   * @param s the string
+   * @return the string in octal
+   * @see Integer#toOctalString(int)
+   */
+  static final String stringToOctal(String s) {
+    StringBuffer buffer = new StringBuffer();
+    for (int i=0;i<s.length();i++) {
+      buffer.append(" 0x");
+      buffer.append(Integer.toOctalString(s.charAt(i)));
+    } 
+    return buffer.toString();
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/IncrementRange.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/IncrementRange.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/IncrementRange.java	(revision 21112)
@@ -0,0 +1,240 @@
+/*
+ *    IncrementRange.java
+ *    Copyright (C) 2000-2001 Stuart Yeates 
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+// the package we're in
+package org.greenstone.gsdl3.selfContained;
+
+// java standard library classes used
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.Writer;
+
+/**
+ * Class IncrementRange increments strings and streams over a limited range of Characters.
+ *
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see Increment
+ * 
+ */
+final public class IncrementRange
+  extends Increment 
+  implements Cloneable, Serializable
+{
+  /**
+   * Create an IncrementRange which counts between first and last inclusive
+   * 
+   * @param first the first character in the range
+   * @param last the last character in the range
+   * @see Increment
+   */
+  IncrementRange(char first, char last){
+    this.first = first;
+    this.last = last;
+  }
+  /**
+   * Create a default IncrementRange
+   * 
+   * @see Increment
+   */
+  IncrementRange(){
+  }
+
+  /**
+   * Create a IncrementRange over all Unicode characters
+   * 
+   * @see Increment
+   */
+  IncrementRange(boolean b){
+    if (true) {
+      this.first = Character.MIN_VALUE;
+      this.last = Character.MAX_VALUE;
+    }
+  }  
+  /** the last character in out range */
+  private char first = ' ';
+  /** the last character in out range */
+  private char last = 'z';
+
+  /**
+   * Increment a stream of characters, reading from the reader and writing the incremented characters to the writer
+   * 
+   * @exception java.io.Error if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  long incrementStream(Reader rdr, Writer wtr) {
+    try {
+      long count = 0; // the return value
+    
+      int i = rdr.read();
+      boolean carry = true;
+    
+      while (i != -1) {
+	char c = (char) i;
+      
+	// check the validity of the character we're about to increment
+	if (c < first || c > last) 
+	  throw new Error("Character '" + c + "' is ouside range");
+      
+	if (c == last) {
+	  if (carry == true) {
+	    wtr.write(first);
+	    carry = true;
+	  } else {
+	    wtr.write(c);
+	    carry = false;
+	  }
+	} else {
+	  if (carry == true) {
+	    wtr.write(++c);
+	    carry = false;
+	  } else {
+	    wtr.write(c);
+	    carry = false;
+	  } 
+	}
+	count++;
+
+	// read the next character
+	i = rdr.read();
+      }
+      if (carry == true) {
+	wtr.write(first);
+	count++;
+      }
+      return count;
+    } catch (IOException e) {
+      System.err.println("IOException in incrementStream()");
+      throw new Error(e.toString());
+    }
+  }
+
+  /**
+   * Increment a string of characters
+   * 
+   * @exception java.lang.Error if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  public final String incrementString(String str)  {
+    StringBuffer buffer = new StringBuffer();
+    if (str.equals("")) {
+      buffer.append(first);
+      return buffer.toString();
+    } else {
+      boolean carry = true;
+      for (int i=0;i<str.length();i++) {
+	char c = str.charAt(i);
+	if (c < first || c > last) 
+	  throw new Error("Character '" + c + "' is ouside range");
+
+	if (c == last) {
+	  if (carry == true) {
+	    buffer.append(first);
+	    carry = true;
+	  } else {
+	    buffer.append(c);
+	    carry = false;
+	  }
+	} else {
+	  if (carry == true) {
+	    buffer.append(++c);
+	    carry = false;
+	  } else {
+	    buffer.append(c);
+	    carry = false;
+	  } 
+	}
+      }
+      if (carry == true)
+	buffer.append(first);
+    }
+    return buffer.toString();
+  }
+
+
+  /**
+   * Test the incrementing of strings (never returns)
+   */
+  public static void testStringIncrement() throws IOException
+  {
+    IncrementRange n = new IncrementRange ();
+    try {
+      String tmp = "";
+      System.out.println(stringToHex(tmp));
+      while (true) {
+	//checkStr(tmp);
+	tmp = n.incrementString(tmp);
+	n.checkStr(tmp);
+      }
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+
+  }
+
+  /**
+   * Test the incrementing of streams (never returns)
+   *
+   */
+  public static void testStreamIncrement() 
+  {
+    IncrementRange n = new IncrementRange ();
+    try {
+      long counter = 0;
+      String tmp = "";
+      System.out.println(stringToHex(tmp));
+      while (true) {
+	//checkStr(tmp);
+	String tmp2 = n.incrementString(tmp);
+	StringWriter wtr = new StringWriter();
+	n.incrementStream(new StringReader(tmp),wtr);
+	if (!tmp2.equals(wtr.toString()))
+	  throw new Exception("Error: \"" + tmp2 + 
+			      "\" and \"" + tmp2 + 
+			      "\"");
+	counter++;
+	if (counter % 1000000 == 0)
+	  System.out.println(counter + " " + tmp);
+
+	tmp = tmp2;
+      }
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+  }
+
+  /**
+   * Tests stream incrementing 
+   *
+   * 
+   */
+  public static void main(String args[]) 
+  {
+    testStreamIncrement();
+  } 
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java	(revision 21112)
@@ -0,0 +1,281 @@
+/*
+ *    QueryDocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMResult;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import org.w3c.dom.Document;
+
+// other java classes
+import java.io.StringReader;
+
+/**
+ * QueryDocumentStream takes another document stream and removes 
+ * from the stream all documents which fail to match a XSLT query.
+ * The default query returns only those documents containing the 
+ * string "the".
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see DocumentStream
+ * 
+ */
+public class QueryDocumentStream implements DocumentStream {
+  
+  /** The default XSLT query string. Is true (returns a non-empty document) when the input document contains the string "the". */
+  public static final String DEFAULT_QUERY_STRING =   
+    "<xsl:stylesheet version=\"1.0\" " +
+    "                    xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" +
+    "   <xsl:output omit-xml-declaration=\"yes\"/>" +
+    
+    "   <xsl:template match=\"/*\">" +
+    "      <xsl:if test=\"contains(.,'the')\">" +
+    "         <xsl:copy-of select=\".\"/>" +
+    "      </xsl:if>" + 
+    "   </xsl:template>" +
+    "</xsl:stylesheet>";
+  /** The first part of the default quesy for a string other than "the" */
+  public static final String DEFAULT_QUERY_STRING_1_1 =   
+    "<xsl:stylesheet version=\"1.0\" " +
+    "                    xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" +
+    "   <xsl:output omit-xml-declaration=\"yes\"/>" +
+    
+    "   <xsl:template match=\"/*\">" +
+    "      <xsl:if test=\"contains(.,'";
+
+  /** The second part of the default quesy for a string other than "the" */
+  public static final String DEFAULT_QUERY_STRING_1_2 =   
+    "')\">" +
+    "         <xsl:copy-of select=\".\"/>" +
+    "      </xsl:if>" + 
+    "   </xsl:template>" +
+    "</xsl:stylesheet>";
+
+
+  /** The query */
+  protected StreamSource query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+
+  /** The underlying DocumentStream object */
+  protected DocumentStream stream = null;
+
+  /** A cached Document */
+  protected Document cached = null;
+
+  /** The transformer factory */
+  TransformerFactory transformerFactory  = null;
+
+  /** The transformer */
+  Transformer transformer  = null;
+
+  /** Uses the default query (looking for the) over the default DocumentStream (all ascii strings) */
+  public QueryDocumentStream() {
+    this.query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+    this.stream = new GeneratedDocumentStream();
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+  /** Looks for the given string in the stream of documents  */
+  public QueryDocumentStream(DocumentStream stream, String query) {
+    String fullQuery = DEFAULT_QUERY_STRING_1_1 + query + DEFAULT_QUERY_STRING_1_2;
+    this.query = new StreamSource(new StringReader(fullQuery));
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(this.query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+
+  /** Applies the stylesheet to the stream.  */
+  public QueryDocumentStream(DocumentStream stream, StreamSource source) {
+    this.query = source;
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+
+  /** The default constructor. Searches for documents containing "the" */
+  public QueryDocumentStream(DocumentStream stream) {
+    this.query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  /**
+   * Generates the nest document in the sequence
+   * 
+   * @exception java.io.Exception when something goes wrong in the XML stuff
+   * @return the next Document
+   * @see org.w3c.dom.Document
+   */
+  public Document nextDocument() throws Exception {
+
+    if (hasNextDocument() == false)
+      throw new Error("no more docs");
+    Document result = cached;
+    cached = null;
+    return result;
+  }
+  /**
+   * Are there more documents ?
+   * 
+   * @return true if there are more docs
+   */
+  public boolean hasNextDocument() throws Exception {
+    lookInStream();
+    return (cached != null);
+  }
+  /**
+   *
+   * 
+   * @exception java.io.IOException when ...
+   * @param arg2 ... 
+   * @param arg1 ...
+   * @return ...
+   */
+  protected boolean lookInStream() throws Exception {
+
+    while (stream.hasNextDocument() && cached == null) {
+      
+      Document candidate = stream.nextDocument();
+      
+      DOMSource input = new DOMSource(candidate);
+      DOMResult output = new DOMResult();
+      
+      try {
+	
+	transformer.transform(input, output);
+      
+	// if the transformed document is not empty don't cache it
+	if (output.getNode().getFirstChild() != null)
+	  cached = (Document) output.getNode();
+      
+      } catch (TransformerConfigurationException e) {
+	System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+      } catch (TransformerException e) {
+	System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	cached = null;
+	return false;
+      } catch (Exception e) {
+	System.err.println("Exception: " + e.getMessage());
+	cached = null;
+	return false;
+      }
+      
+      if (false) { // debugging info
+	
+	try {
+	  StreamResult result = new StreamResult(System.out);
+	  Transformer transformer = transformerFactory.newTransformer();
+	  
+	  Source source = new DOMSource(candidate);
+	  transformer.transform(source,result);
+	  
+	  System.out.print(" ==> ");
+	  
+	  if (cached != null) {
+	    Source source2 = new DOMSource(cached);
+	    transformer.transform(source2,result);
+	  } 
+	  System.out.println();
+	  System.out.println("======================================================");
+	  
+	} catch (TransformerConfigurationException e) {
+	  System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+	} catch (TransformerException e) {
+	  System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	  cached = null;
+	  return false;
+	} catch (Exception e) {
+	  System.err.println("Exception: " + e.getMessage());
+	  cached = null;
+	  return false;
+	}
+	
+      }
+    }
+    return cached == null;    
+  }
+
+  /**
+   * Generates an unbounded stream of documents
+   */
+  public static void main(String args[]) throws Exception
+  {
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    Transformer transformer = transformerFactory.newTransformer();
+    
+    GeneratedDocumentStream gStream = new GeneratedDocumentStream();
+    QueryDocumentStream qStream = new QueryDocumentStream(gStream);
+    while (qStream.hasNextDocument()) {
+      Document document = qStream.nextDocument();
+      
+      StreamResult result = new StreamResult(System.out);
+      
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+      
+      System.out.println();
+      System.out.println("======================================================");
+    }
+  }
+  
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/XMLUtil.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/XMLUtil.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/XMLUtil.java	(revision 21112)
@@ -0,0 +1,96 @@
+/*
+ *    XMLUtil.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.xerces.parsers.DOMParser;
+
+// other java classes
+import java.io.IOException;
+
+
+/**
+ * Class XMLUtil holds static global XML configuration objects. 
+ * 
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * 
+ */
+class XMLUtil {
+
+  /** the singleton instance */
+  protected static XMLUtil instance = null;
+
+
+  /** JAXP parser factory */ 
+  protected DocumentBuilderFactory doc_build_fact=null;
+
+  /** JAXP parser factory */ 
+  static public DocumentBuilderFactory getDocumentBuilderFactory() {
+    if (instance == null) 
+      init();
+    return instance.doc_build_fact;
+  }
+
+  /** JAXP parser */ 
+  protected DocumentBuilder doc_builder=null;
+
+  /** JAXP parser */ 
+  static public DocumentBuilder getDocumentBuilder() {
+    if (instance == null) 
+      init();
+    return instance.doc_builder;
+   }
+
+  /** xerces parser */
+  protected DOMParser parser = null;
+
+  /** xerces parser */
+  static public DOMParser getDOMParser() {
+    if (instance == null) 
+      init();
+    return instance.parser;
+  }
+
+  /** initalise the statics */
+  static protected void init() {
+    instance = new XMLUtil();
+    try {
+      instance.doc_build_fact = DocumentBuilderFactory.newInstance();
+      
+      instance.doc_builder = instance.doc_build_fact.newDocumentBuilder(); 
+      instance.parser = new DOMParser();
+    } catch (Exception e) {
+      System.out.println("XMLConverter:exception "+e.getMessage());
+    }
+  }
+
+  /**
+   * Checks that the objects can be instantiated
+   *
+   */
+  public static void main(String args[]) throws IOException
+  {
+    XMLUtil.getDOMParser();
+  }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/go.dtd
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/go.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/selfContained/go.dtd	(revision 21112)
@@ -0,0 +1,103 @@
+<!-- P is the Paragraph of HTML -->
+<!ELEMENT P (#PCDATA)>
+
+<!-- Each Go file consists of one or several GoGame -->
+<!ELEMENT Go (GoGame*)>
+
+<!-- Each GoGame consists of the game Information and the game tree -->
+
+<!ELEMENT GoGame (Information,Nodes)>
+<!ATTLIST GoGame name CDATA #IMPLIED>
+<!ELEMENT Nodes (((Node|Black|White),(Node|Variation|Black|White|Comment)*)?)>
+
+<!ELEMENT Information (Application?|BoardSize?|BlackPlayer?|BlackRank?|
+	WhitePlayer?|WhiteRank?|Date?|Time?|Komi?|Result?|Handicap?|User?|
+	Copyright?|Annotation?|BlackTeam?|WhiteTeam?|Round?|Rules?|Source?)*>
+<!ELEMENT Application (#PCDATA)>
+<!ATTLIST Application format CDATA #IMPLIED>
+<!ELEMENT BoardSize (#PCDATA)>
+<!ATTLIST BoardSize format CDATA #IMPLIED>
+<!ELEMENT BlackPlayer (#PCDATA)>
+<!ATTLIST BlackPlayer format CDATA #IMPLIED>
+<!ELEMENT WhitePlayer (#PCDATA)>
+<!ATTLIST WhitePlayer format CDATA #IMPLIED>
+<!ELEMENT BlackRank (#PCDATA)>
+<!ATTLIST BlackRank format CDATA #IMPLIED>
+<!ELEMENT WhiteRank (#PCDATA)>
+<!ATTLIST WhiteRank format CDATA #IMPLIED>
+<!ELEMENT Date (#PCDATA)>
+<!ATTLIST Date format CDATA #IMPLIED>
+<!ELEMENT Time (#PCDATA)>
+<!ATTLIST Time format CDATA #IMPLIED>
+<!ELEMENT Komi (#PCDATA)>
+<!ATTLIST Komi format CDATA #IMPLIED>
+<!ELEMENT Handicap (#PCDATA)>
+<!ATTLIST Handicap format CDATA #IMPLIED>
+<!ELEMENT Result (#PCDATA)>
+<!ATTLIST Result format CDATA #IMPLIED>
+<!ELEMENT User (#PCDATA)>
+<!ELEMENT Copyright (P+)>
+<!ELEMENT Annotation (#PCDATA)>
+<!ELEMENT BlackTeam (#PCDATA)>
+<!ELEMENT WhiteTeam (#PCDATA)>
+<!ELEMENT Round (#PCDATA)>
+<!ELEMENT Rules (#PCDATA)>
+<!ATTLIST Rules format CDATA #IMPLIED>
+<!ELEMENT Source (#PCDATA)>
+<!ATTLIST Source format CDATA #IMPLIED>
+
+<!-- Game representation in Nodes -->
+
+<!ELEMENT Node ((SGF|Comment|Black|White|AddBlack|AddWhite|Mark|Delete)*)>
+<!ATTLIST Node name CDATA #IMPLIED>
+
+<!-- Possible content of a node (unordered) -->
+
+<!-- Alternative form of specifying the position as tag content. -->
+<!-- The board position is either like D10 or 4,10 (for larger boards). -->
+<!ELEMENT at (#PCDATA)>
+
+<!ELEMENT Comment (P+)>
+<!ELEMENT Black ((at)*)>
+<!ATTLIST Black
+	at CDATA #IMPLIED
+	number CDATA #IMPLIED
+	timeleft CDATA #IMPLIED
+	annotate CDATA #IMPLIED>
+<!ELEMENT White ((at)*)>
+<!ATTLIST White
+	at CDATA #IMPLIED
+	number CDATA #IMPLIED
+	timeleft CDATA #IMPLIED
+	annotate CDATA #IMPLIED>
+<!ELEMENT AddBlack ((at)*)>
+<!ATTLIST AddBlack
+	at CDATA #IMPLIED>
+<!ELEMENT AddWhite ((at)*)>
+<!ATTLIST AddWhite
+	at CDATA #IMPLIED>
+<!ELEMENT Delete ((at)*)>
+<!ATTLIST Delete
+	at CDATA #IMPLIED>
+<!ELEMENT Mark ((at)*)>
+<!ATTLIST Mark 
+	label CDATA #IMPLIED
+	at CDATA #IMPLIED
+	type (square|circle|triangle) #IMPLIED
+	territory (black|white) #IMPLIED>
+<!ELEMENT BlackToPlay (#PCDATA)>
+<!ELEMENT WhiteToPlay (#PCDATA)>
+<!ELEMENT Line (#PCDATA)>
+<!ATTLIST Line 
+	format CDATA #IMPLIED
+	type CDATA #IMPLIED>
+
+<!-- This is to take care of SGF tags, which are not translated -->
+<!ELEMENT SGF (Arg*)>
+<!ATTLIST SGF
+	type CDATA #REQUIRED>
+<!ELEMENT Arg (#PCDATA)>
+
+<!-- Note that the Variation are ALTERNATIVES to the current node -->
+<!-- This is different from SGF and makes the file easier to read by humans -->
+<!ELEMENT Variation ((Node|Variation|White|Black|Comment)*)>
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 21112)
@@ -0,0 +1,732 @@
+/*
+ *    AbstractBrowse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.MacroResolver;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+
+/** Partially implements a generic classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public abstract class AbstractBrowse
+    extends ServiceRack {
+
+    // the browsing services
+    private static final String CLASSIFIER_SERVICE = "ClassifierBrowse";
+    private static final String CLASSIFIER_METADATA_SERVICE = "ClassifierBrowseMetadataRetrieve";
+
+    // do we want to keep info request?
+    
+    protected static final String STRUCT_PARAM = "structure";
+    protected static final String INFO_PARAM = "info";
+    
+    protected static final String STRUCT_ANCESTORS = "ancestors";
+    protected static final String STRUCT_PARENT = "parent";
+    protected static final String STRUCT_SIBS = "siblings";
+    protected static final String STRUCT_CHILDREN = "children";
+    protected static final String STRUCT_DESCENDS = "descendants";
+    protected static final String STRUCT_ENTIRE = "entire";
+
+    protected static final String INFO_NUM_SIBS = "numSiblings";
+    protected static final String INFO_NUM_CHILDREN = "numChildren";
+    protected static final String INFO_SIB_POS = "siblingPosition";
+
+    protected Element config_info = null; // the xml from the config file
+
+    protected MacroResolver macro_resolver = null;
+
+    protected String default_document_type = null;
+    /** constructor */
+    protected AbstractBrowse()
+    {
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractBrowse...");
+	this.config_info = info;
+
+	if (macro_resolver != null) {
+	    // set up the macro resolver
+	    Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
+	    if (replacement_elem != null) {
+		macro_resolver.addMacros(replacement_elem);
+	    }
+
+	}
+
+ 
+        // check that there are classifiers specified
+        Element class_list = (Element)GSXML.getChildByTagName(info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+        if (class_list == null) {
+            // no classifiers specified
+            return false;
+        }
+	
+	// get the display and format elements from the coll config file for
+	// the classifiers
+	extractExtraClassifierInfo(info, extra_info);
+	
+        // short_service_info_ - the browse one
+        Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+        cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+        cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+        this.short_service_info.appendChild(cb_service);
+	
+	// metadata retrieval for the browsing  
+        Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+        cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+        cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+        this.short_service_info.appendChild(cbmr_service);
+
+	// the format info
+	Element cb_format_info = this.doc.createElement(GSXML.FORMAT_ELEM);
+	boolean format_found = false;
+
+	// try the default format first
+	Element def_format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+	if (def_format != null) {
+	    cb_format_info.appendChild(GSXML.duplicateWithNewName(this.doc, def_format, GSXML.DEFAULT_ELEM, true));
+	    format_found = true;
+	}
+	
+	// add in to the description a simplified list of classifiers
+        NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	for(int i=0; i<classifiers.getLength(); i++) {
+	    Element cl = (Element)classifiers.item(i);
+	    Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
+	    
+	    // get the format info out, and put inside a classifier element
+	    Element format_cl = (Element)new_cl.cloneNode(false);
+	    Element format = (Element)GSXML.getChildByTagName(cl, GSXML.FORMAT_ELEM);
+	    if (format != null) {
+		
+		//copy all the children
+		NodeList elems = format.getChildNodes();
+		for (int j=0; j<elems.getLength();j++) {
+		    format_cl.appendChild(this.doc.importNode(elems.item(j), true));
+		}
+		cb_format_info.appendChild(format_cl);
+		format_found = true;
+	    }
+	    	    
+	}
+	    
+	if (format_found) {
+	    this.format_info_map.put(CLASSIFIER_SERVICE, cb_format_info);
+	}
+
+	// look for document display format - is there a default doc type??
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    // check for docType option.
+	    Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+	    if (doc_type_opt != null) {
+		String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+		if (!value.equals("")) {
+		    this.default_document_type = value;
+		}
+	    }
+	}
+
+	return true;
+    }
+
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+
+	if (service_id.equals(CLASSIFIER_SERVICE)) {
+	    
+	    Element class_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+	    if (class_list == null) {
+            // no classifiers specified
+		return null;
+	    }
+	    
+	    Element cb_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+	    cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(CLASSIFIER_SERVICE+".name", lang)));
+	    cb_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(CLASSIFIER_SERVICE+".description", lang)));
+	    
+	    Element cl_list = this.doc.createElement(GSXML.CLASSIFIER_ELEM+GSXML.LIST_MODIFIER);
+	    cb_service.appendChild(cl_list);
+	    NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	    for(int i=0; i<classifiers.getLength(); i++) {
+		Element cl = (Element)classifiers.item(i);
+		Element new_cl = (Element)this.doc.importNode(cl, false); // just import this node, not the children
+		String content = cl.getAttribute(GSXML.CLASSIFIER_CONTENT_ATT);
+		cl_list.appendChild(new_cl);
+		String text = GSXML.getDisplayText(cl, 
+						   GSXML.DISPLAY_TEXT_NAME, 
+						   lang, "en");
+		if (text == null || text.equals("")) {
+		    // no display element was specified, use the metadata name
+		    // for now this looks in the class properties file
+		    // this needs to use a general metadata thing instead
+		    text = getMetadataNameText(content+".buttonname", lang);
+		}
+		if (text == null) {
+		    text = content;
+		}
+		
+		Element cl_name = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, text);
+		new_cl.appendChild(cl_name);
+
+		// description
+		
+		String meta_name = getMetadataNameText(content, lang);
+		if (meta_name==null) {
+		    meta_name = content;
+		}
+		String [] array = {meta_name};
+		String description = getTextString("ClassifierBrowse.classifier_help", lang, array);
+		Element cl_desc = GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, description);
+		new_cl.appendChild(cl_desc);
+		
+	    }
+	    return cb_service;
+	}
+	
+	// these ones are probably never called, but put them here just in case
+	
+	if (service_id.equals(CLASSIFIER_METADATA_SERVICE)) {
+	    
+	    Element cbmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+	    return cbmr_service;
+	}
+
+	return null;
+    }
+
+    /** this looks for any classifier specific display or format info from extra_info and adds it in to the correct place in info */
+    protected boolean extractExtraClassifierInfo(Element info, Element extra_info) {
+	
+	if (extra_info == null) {
+	    return false;
+	}
+	
+	Document owner = info.getOwnerDocument();
+	// so far we have display and format elements that we need for classifiers
+	NodeList classifiers = info.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	Element config_browse = (Element)GSXML.getChildByTagName(extra_info, GSXML.BROWSE_ELEM);
+
+	for (int i=0; i<classifiers.getLength();i++) {
+	    Element cl = (Element)classifiers.item(i);
+	    String name = cl.getAttribute(GSXML.NAME_ATT);
+	    Element node_extra = GSXML.getNamedElement(config_browse,
+						       GSXML.CLASSIFIER_ELEM,
+						       GSXML.NAME_ATT,
+						       name);
+	    if (node_extra == null) {
+		System.err.println("GS2REtrieve: haven't found extra info for classifier named "+name);
+		continue;
+	    }
+	    
+	    // get the display elements if any - displayName
+	    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+	    if (display_names !=null) {
+		Element display = owner.createElement(GSXML.DISPLAY_ELEM);
+		for (int j=0; j<display_names.getLength(); j++) {
+		    Element e = (Element)display_names.item(j);
+		    cl.appendChild(owner.importNode(e, true));
+		    
+		}
+	    }
+	    
+	    // get the format element if any 
+	    Element format = (Element)GSXML.getChildByTagName(node_extra, GSXML.FORMAT_ELEM);
+	    if (format==null) { // try a generic one that applies to all classifiers
+		format = (Element)GSXML.getChildByTagName(extra_info,
+							  GSXML.FORMAT_ELEM);
+	    }
+	    if (format!=null) { // append to index info
+		cl.appendChild(owner.importNode(format, true));
+	    }
+	} // for each classifier
+
+	// now check for default format info
+	Element default_format = (Element)GSXML.getChildByTagName(config_browse, GSXML.FORMAT_ELEM);
+	if (default_format!=null) { // append to  info
+	    info.appendChild(owner.importNode(default_format, true));
+	}
+
+	return true;
+    }
+	    
+    protected Element processClassifierBrowse(Element request) {
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	Element query_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_node_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowse request specified no doc nodes.\n");
+	    return result;
+	}
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowse request had no paramList.");
+	    return result;  // Return the empty result
+	}
+	
+	// the type of info required
+	boolean want_structure = false;
+	boolean want_info = false;
+
+	ArrayList info_types=new ArrayList();
+	// The document structure information desired
+	boolean want_ancestors = false;
+	boolean want_parent = false;
+	boolean want_siblings = false;
+	boolean want_children = false;
+	boolean want_descendants = false;
+
+	boolean want_entire_structure = false;
+	// Process the request parameters
+	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	for (int i=0; i<params.getLength();i++) {
+	    
+	    Element param = (Element)params.item(i);
+	    String p_name = param.getAttribute(GSXML.NAME_ATT);
+	    String p_value = GSXML.getValue(param);
+	    // Identify the structure information desired
+	    if (p_name.equals(STRUCT_PARAM)) {
+		want_structure = true;
+		
+		// This is NOT locale sensitive
+		if (p_value.equals(STRUCT_ANCESTORS))
+		    want_ancestors = true;
+		else if (p_value.equals(STRUCT_PARENT))
+		    want_parent = true;
+		else if (p_value.equals(STRUCT_SIBS))
+		    want_siblings = true;
+		else if (p_value.equals(STRUCT_CHILDREN))
+		    want_children = true;
+		else if (p_value.equals(STRUCT_DESCENDS))
+		    want_descendants = true;
+		else if (p_value.equals(STRUCT_ENTIRE))
+		    want_entire_structure = true;
+		else
+		    System.err.println("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+	    } else if (p_name.equals(INFO_PARAM)) {
+		want_info = true;
+		info_types.add(p_value);
+	    }
+	}
+
+	// Make sure there is no repeated information
+	if (want_ancestors)
+	    want_parent = false;
+	if (want_descendants)
+	    want_children = false;
+	
+	// the node list to hold the results
+	Element node_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the nodes
+	String[] node_ids = GSXML.getAttributeValuesFromList(query_node_list, 
+							     GSXML.NODE_ID_ATT);
+	for (int i = 0; i < node_ids.length; i++) {
+	    // Add the document to the list
+	    Element node = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	    node_list.appendChild(node);
+
+	    String node_id = node_ids[i];
+	    node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	    
+	    if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+		if (node_id == null) {
+		    continue;
+		}
+		node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	    }
+	    
+	    if (want_info) {
+
+		Element node_info_elem = this.doc.createElement("nodeStructureInfo");
+		node.appendChild(node_info_elem);
+		
+		for (int j=0; j<info_types.size(); j++) {
+		    String info_type = (String)info_types.get(j);
+		    String info_value = getStructureInfo(node_id, info_type);
+		    if (info_value != null) {
+			Element info_elem = this.doc.createElement("info");
+			info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+			info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+			node_info_elem.appendChild(info_elem);
+		    }
+		}
+	    }
+	    
+	    if (want_structure) {
+		// all structure info goes into a nodeStructure elem
+		Element structure_elem = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+		node.appendChild(structure_elem);
+		
+		if (want_entire_structure) {
+		    String root_id = getRootId(node_id);
+		    Element root_node = createClassifierNode(root_id); //, true, false);
+		    addDescendants(root_node, root_id, true);
+		    structure_elem.appendChild(root_node);
+		    continue; // with the next document, we dont need to do any more here
+		}
+		
+		// Add the requested structure information
+		Element base_node = createClassifierNode(node_id); //, false, false);
+		
+		//Ancestors: continually add parent nodes until the root is reached
+		Element top_node = base_node; // the top node so far
+		if (want_ancestors) {
+		    String current_id = node_id;
+		    while (true) {
+			String parent_id = getParentId(current_id);
+			//Element parent = getParent(current_id);
+			if (parent_id == null)
+			    break; // no parent
+			Element parent_node = createClassifierNode(parent_id);
+			parent_node.appendChild(top_node);
+			current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+			top_node = parent_node;
+		    }
+		}
+		// Parent: get the parent of the selected node
+		else if (want_parent) {
+		    String parent_id = getParentId(node_id);
+		    if (parent_id != null) {
+			Element parent_node = createClassifierNode(parent_id);
+			parent_node.appendChild(base_node);
+			top_node = parent_node;
+		    }
+		}
+		
+		// now the top node is the root of the structure
+		structure_elem.appendChild(top_node);
+		
+		//Siblings: get the other descendants of the selected node's parent
+		if (want_siblings) {
+		    
+		    Element parent_node = (Element)base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+		    String parent_id = getParentId(node_id);
+		    // add siblings, - returns a pointer to the new current node 
+		    base_node = addSiblings(parent_node, parent_id, node_id);
+		}
+		
+		// Children: get the descendants, but only one level deep
+		if (want_children) {
+		    addDescendants(base_node, node_id, false);
+		} 
+		// Descendants: recursively get every descendant 
+		else if (want_descendants) {
+		    addDescendants(base_node, node_id, true);
+		}
+	    } // if want structure
+	} // for each doc
+	return result;
+    }
+
+
+    protected Element processClassifierBrowseMetadataRetrieve(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_METADATA_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("AbstractBrowse, ClassifierBrowseMetadataRetrieve Error: missing paramList.\n");
+	    return result;  // Return the empty result
+	}
+
+	// The metadata information required
+	ArrayList metadata_names_list = new ArrayList();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		metadata_names_list.add(metadata);
+	    } 
+	    param = (Element) param.getNextSibling();
+	}
+	
+	Element node_list = this.doc.createElement(GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the nodes
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    System.err.println("AbstractBrowse Error: ClassifierBrowseMetadataRetrieve request had no "+GSXML.CLASS_NODE_ELEM+GSXML.LIST_MODIFIER);
+ 	    return result;
+ 	}
+	
+	NodeList request_nodes = request_node_list.getChildNodes();
+	for (int i = 0; i < request_nodes.getLength(); i++) {
+	    Element request_node = (Element) request_nodes.item(i);
+	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+	    
+	    // Add the document to the results list
+	    Element new_node = (Element)this.doc.importNode(request_node, false);
+	    node_list.appendChild(new_node);
+
+	    if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+	    }
+	    if (node_id == null) {
+		continue;
+	    }
+	    
+	    Element metadata_list = getMetadataList(node_id, all_metadata, metadata_names_list);
+	    new_node.appendChild(metadata_list);
+	}
+	
+	return result;
+    }
+
+
+    /** Creates a classifier node */
+    protected Element createClassifierNode(String node_id)
+    {
+	Element node = this.doc.createElement(GSXML.CLASS_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	return node;
+    }
+
+    /** create an element to go into the structure. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+     */
+    protected Element createDocNode(String node_id) {
+	Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+
+	String doc_type = null;
+	if (default_document_type != null) {
+	    doc_type = default_document_type;
+	} else {
+	    doc_type = getDocType(node_id);
+	}
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+   /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    protected String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (getParentId(node_id)==null) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (getChildrenIds(node_id)==null) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+    /** adds all the children of doc_id the the doc element,
+     * and if recursive=true, adds all their children as well*/
+    protected void addDescendants(Element node, String node_id, 
+				  boolean recursive)
+    {
+	ArrayList child_ids = getChildrenIds(node_id);
+	if (child_ids==null) return;
+	for (int i=0; i< child_ids.size(); i++) {
+	    String child_id = (String)child_ids.get(i);
+	    Element child_elem;
+	    if (isDocumentId(child_id)) {
+		child_elem = createDocNode(child_id);
+	    } else {
+		child_elem = createClassifierNode(child_id);
+	    }
+	    node.appendChild(child_elem);
+	    if (recursive) {
+		addDescendants(child_elem, child_id, recursive);
+	    }
+	}
+    }
+
+    /** adds all the siblings of current_id to the parent element. 
+      returns the new current element*/
+    protected Element addSiblings(Element parent_node, String parent_id, 
+				  String current_id) {
+	Element current_node = (Element)parent_node.getFirstChild();
+	if (current_node == null) {
+	    // create a sensible error message
+	    System.err.println("AbstractBrowse Error: there should be a first child.");
+	    return null;
+	}
+	// remove the current child,- will add it in later in its correct place
+	parent_node.removeChild(current_node);
+
+	// add in all the siblings,
+	addDescendants(parent_node, parent_id, false);
+
+	// find the node that is now the current node
+	// this assumes that the new node that was created is the same as 
+	// the old one that was removed - we may want to replace the new one 
+	// with the old one.
+	Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+	return new_current;
+    }
+
+    /** returns true if oid ends in
+	.fc (firstchild), 
+	.lc (lastchild), 
+	.pr (parent), 
+	.ns (next sibling), 
+	.ps (prev sibling), 
+	.rt (root) 
+	.ss (specified sibling), 
+	false otherwise 
+    */
+    protected boolean idNeedsTranslating(String node_id) {
+	String tail = node_id.substring(node_id.length()-3);
+	return (tail.equals(".fc") || tail.equals(".lc") || 
+		tail.equals(".pr") || tail.equals(".ns") || 
+		tail.equals(".ps") || tail.equals(".rt") || 
+		tail.equals(".ss"));
+
+    }
+
+    /** returns the list of sibling ids, including the specified node_id */
+    protected ArrayList getSiblingIds(String node_id) {
+	String parent_id = getParentId(node_id);
+	if (parent_id == null) {
+	    return null;
+	}
+	return getChildrenIds(parent_id);
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    abstract protected String translateId(String node_id);
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    abstract protected String getDocType(String node_id);
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    abstract protected String getRootId(String node_id);
+    /** returns a list of the child ids in order, null if no children */
+    abstract protected ArrayList getChildrenIds(String node_id);
+    /** returns the node id of the parent node, null if no parent */
+    abstract protected String getParentId(String node_id);
+ 
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    abstract protected boolean isDocumentId(String node_id);
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    abstract protected Element getMetadataList(String node_id,
+					       boolean all_metadata,
+					       ArrayList metadata_names);
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    abstract protected String getStructureInfo(String node_id, String info_type);    
+//      /** needs to get info from gdbm database - if the calling code gets it already it may pay to pass it in instead */
+//     protected String resolveTextMacros(String doc_content, String doc_id, String lang)
+//     {
+// 	DBInfo info = null;
+// 	if (doc_content.indexOf("_httpdocimg_")!=-1) {
+// 	    String top_doc_id = OID.getTop(doc_id);
+// 	    info = this.gdbm_src.getInfo(top_doc_id);
+// 	    if (info == null) {
+// 		// perhaps we had per.iods in the ids - just try the current id
+// 		top_doc_id = doc_id;
+// 		info = this.gdbm_src.getInfo(top_doc_id);
+// 	    }
+// 	    if (info != null) {
+// 		String archivedir = info.getInfo("archivedir");
+// 		String image_dir  = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/"+archivedir;
+		
+// 		// Resolve all "_httpdocimg_"s
+// 		doc_content = doc_content.replaceAll("_httpdocimg_", image_dir);
+// 	    }
+// 	}
+// 	// resolve any collection specific macros
+// 	doc_content = macro_resolver.resolve(doc_content, lang, GS2MacroResolver.SCOPE_TEXT, doc_id, info);
+// 	return doc_content;
+//     }
+
+   
+}   
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 21112)
@@ -0,0 +1,687 @@
+/*
+ *    AbstractDocumentRetrieve.java
+ *    a base class for retrieval services
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+//import org.greenstone.gdbm.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.MacroResolver;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Abstract class for Document Retrieval Services
+ *
+ * @author <a href="mailto:greenstone@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public abstract class AbstractDocumentRetrieve
+    extends ServiceRack {
+
+    // the services on offer
+    protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
+    protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
+    
+    protected static final String STRUCT_PARAM = "structure";
+    protected static final String INFO_PARAM = "info";
+    
+    protected static final String STRUCT_ANCESTORS = "ancestors";
+    protected static final String STRUCT_PARENT = "parent";
+    protected static final String STRUCT_SIBS = "siblings";
+    protected static final String STRUCT_CHILDREN = "children";
+    protected static final String STRUCT_DESCENDS = "descendants";
+    protected static final String STRUCT_ENTIRE = "entire";
+
+    protected static final String INFO_NUM_SIBS = "numSiblings";
+    protected static final String INFO_NUM_CHILDREN = "numChildren";
+    protected static final String INFO_SIB_POS = "siblingPosition";
+
+    // means the id is not a greenstone id and needs translating
+    protected static final String EXTID_PARAM = "ext"; 
+    
+    protected Element config_info = null; // the xml from the config file
+
+    protected String default_document_type = null;
+    protected MacroResolver macro_resolver = null;
+
+    /** does this class provide the service?? */
+    protected boolean does_metadata = true;
+    protected boolean does_content = true;
+    protected boolean does_structure = true;
+
+    /** constructor */
+    public AbstractDocumentRetrieve()
+    {
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractDocumentRetrieve...");
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has name and type
+	if (does_structure) {
+	    Element dsr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	    this.short_service_info.appendChild(dsr_service);
+	}
+
+	if (does_metadata) {
+	    Element dmr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	    this.short_service_info.appendChild(dmr_service);
+	}
+
+	if (does_content) {
+	    Element dcr_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	    this.short_service_info.appendChild(dcr_service);
+	}
+	
+	// look for document display format
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.doc.importNode(display_format, true));
+	    // should we keep a copy?
+	    // check for docType option.
+	    Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+	    if (doc_type_opt != null) {
+		String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+		if (!value.equals("")) {
+		    this.default_document_type = value;
+		}
+	    }
+	}
+
+	if (macro_resolver != null) {
+	    // set up the macro resolver
+	    Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
+	    if (replacement_elem != null) {
+		macro_resolver.addMacros(replacement_elem);
+	    }
+	}
+
+	return true;
+    }
+    
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+	
+	// these ones are probably never called, but put them here just in case
+	Element service_elem = this.doc.createElement(GSXML.SERVICE_ELEM);
+	service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	service_elem.setAttribute(GSXML.NAME_ATT, service_id);
+	return service_elem;
+    }
+
+    protected Element processDocumentMetadataRetrieve(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	if (!does_metadata) {
+	    // shouldn't get here
+	    return result;
+	}
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    GSXML.addError(this.doc, result, "DocumentMetadataRetrieve: missing "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+	    return result;  
+	}
+
+	boolean external_id = false;
+	// The metadata information required
+	ArrayList metadata_names_list = new ArrayList();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		metadata_names_list.add(metadata);
+	    } else if (param.getAttribute(GSXML.NAME_ATT).equals(EXTID_PARAM)&& GSXML.getValue(param).equals("1")) {
+		external_id = true;
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+
+	// check that there has been some metadata specified
+	if (!all_metadata && metadata_names_list.size()==0) {
+	    GSXML.addError(this.doc, result, "DocumentMetadataRetrieve: no metadata names found in the "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+	    return result;  
+	}
+
+	// Get the documents
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    GSXML.addError(this.doc, result, "DocumentMetadataRetrieve: missing " +GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+ 	    return result;
+ 	}
+
+	// copy the request doc node list to the response
+	Element response_node_list = (Element) this.doc.importNode(request_node_list, true);
+	result.appendChild(response_node_list);
+	
+	// use the copied list so that we add the metadata into the copy
+	// are we just adding metadata for the top level nodes? or can we accept a hierarchy here???
+	NodeList request_nodes = GSXML.getChildrenByTagName(response_node_list, GSXML.DOC_NODE_ELEM);
+	if (request_nodes.getLength()==0) {
+	    GSXML.addError(this.doc, result, "DocumentMetadataRetrieve: no "+GSXML.DOC_NODE_ELEM +" found in the "+ GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+ 	    return result;
+ 	}
+
+	// Whew, now we have checked (almost) all the syntax of the request, now we can process it.
+	
+	for (int i = 0; i < request_nodes.getLength(); i++) {
+	    Element request_node = (Element) request_nodes.item(i);
+	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+	    if (external_id) {
+		// can we have .pr etc extensions with external ids?
+		node_id = translateExternalId(node_id);
+	    } else if (idNeedsTranslating(node_id)) {
+		node_id = translateId(node_id);
+	    }	    
+	    if (node_id == null) {
+		continue;
+	    }
+	    
+	    try {
+		Element metadata_list = getMetadataList(node_id, all_metadata, metadata_names_list);
+		request_node.appendChild(metadata_list);
+	    } catch (GSException e) {		
+		GSXML.addError(this.doc, result, e.getMessage(), e.getType());
+		if (e.getType().equals(GSXML.ERROR_TYPE_SYSTEM)) {
+		    // there is no point trying any others
+		    return result;
+		}
+	    } 
+	}
+		
+	return result;
+    }
+
+    protected Element processDocumentStructureRetrieve(Element request) {
+    
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	if (!does_structure) {
+	    // shouldn't get here
+	    return result;
+	}
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    GSXML.addError(this.doc, result, "DocumentStructureRetrieve: missing "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+	    return result;  
+	}
+	
+	// get the documents of the request
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    GSXML.addError(this.doc, result, "DocumentStructureRetrieve: missing " +GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+ 	    return result;
+ 	}
+	
+	// copy the doc_list to the response
+	Element response_node_list = (Element) this.doc.importNode(query_doc_list, true);
+	result.appendChild(response_node_list);
+	
+	// check that we have some doc nodes specified
+	NodeList node_list = GSXML.getChildrenByTagName(response_node_list, GSXML.DOC_NODE_ELEM);
+	if (node_list.getLength()==0) {
+	    GSXML.addError(this.doc, result, "DocumentStructureRetrieve: no "+GSXML.DOC_NODE_ELEM +" found in the "+ GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+ 	    return result;
+ 	}
+
+	Element extid_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTID_PARAM);
+	boolean external_id = false;
+	if (extid_param != null && GSXML.getValue(extid_param).equals("1")) {
+	    external_id = true;
+	}
+
+	// the type of info required
+	boolean want_structure = false;
+	boolean want_info = false;
+
+	ArrayList info_types=new ArrayList();
+	// The document structure information desired
+	boolean want_ancestors = false;
+	boolean want_parent = false;
+	boolean want_siblings = false;
+	boolean want_children = false;
+	boolean want_descendants = false;
+
+	boolean want_entire_structure = false;
+	// Process the request parameters
+	NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	for (int i=0; i<params.getLength();i++) {
+	    
+	    Element param = (Element)params.item(i);
+	    String p_name = param.getAttribute(GSXML.NAME_ATT);
+	    String p_value = GSXML.getValue(param);
+	    // Identify the structure information desired
+	    if (p_name.equals(STRUCT_PARAM)) {
+		want_structure = true;
+		
+		// This is NOT locale sensitive
+		if (p_value.equals(STRUCT_ANCESTORS))
+		    want_ancestors = true;
+		else if (p_value.equals(STRUCT_PARENT))
+		    want_parent = true;
+		else if (p_value.equals(STRUCT_SIBS))
+		    want_siblings = true;
+		else if (p_value.equals(STRUCT_CHILDREN))
+		    want_children = true;
+		else if (p_value.equals(STRUCT_DESCENDS))
+		    want_descendants = true;
+		else if (p_value.equals(STRUCT_ENTIRE))
+		    want_entire_structure = true;
+		else
+		    System.err.println("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+	    } else if (p_name.equals(INFO_PARAM)) {
+		want_info = true;
+		info_types.add(p_value);
+	    }
+	}
+
+	// Make sure there is no repeated information
+	if (want_ancestors)
+	    want_parent = false;
+	if (want_descendants)
+	    want_children = false;
+
+	for (int i=0; i < node_list.getLength(); i++) {
+	    Element doc = (Element) node_list.item(i);
+	    String doc_id = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    if (external_id) {
+		doc_id = translateExternalId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    } else if (idNeedsTranslating(doc_id)) {
+		doc_id = translateId(doc_id);	
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    }
+	    if (doc_id == null) {
+		continue;
+	    }
+
+	    if (want_info) {
+
+		Element node_info_elem = this.doc.createElement("nodeStructureInfo");
+		doc.appendChild(node_info_elem);
+		
+		for (int j=0; j<info_types.size(); j++) {
+		    String info_type = (String)info_types.get(j);
+		    String info_value = getStructureInfo(doc_id, info_type);
+		    if (info_value != null) {
+			Element info_elem = this.doc.createElement("info");
+			info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+			info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+			node_info_elem.appendChild(info_elem);
+		    }
+		}
+	    }
+	    
+	    if (want_structure) {
+		// all structure info goes into a nodeStructure elem
+		Element structure_elem = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+		doc.appendChild(structure_elem);
+		
+		if (want_entire_structure) {
+		    String root_id = getRootId(doc_id);
+		    Element root_node = createDocNode(root_id); //, true, false);
+		    addDescendants(root_node, root_id, true);
+		    structure_elem.appendChild(root_node);
+		    continue; // with the next document, we dont need to do any more here
+		}
+		
+		// Add the requested structure information
+		Element base_node = createDocNode(doc_id); //, false, false);
+		
+		//Ancestors: continually add parent nodes until the root is reached
+		Element top_node = base_node; // the top node so far
+		if (want_ancestors) {
+		    String current_id = doc_id;
+		    while (true) {
+			String parent_id = getParentId(current_id);
+			//Element parent = getParent(current_id);
+			if (parent_id == null)
+			    break; // no parent
+			Element parent_node = createDocNode(parent_id);
+			parent_node.appendChild(top_node);
+			current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+			top_node = parent_node;
+		    }
+		}
+		// Parent: get the parent of the selected node
+		else if (want_parent) {
+		    String parent_id = getParentId(doc_id);
+		    if (parent_id != null) {
+			Element parent_node = createDocNode(parent_id);
+			parent_node.appendChild(base_node);
+			top_node = parent_node;
+		    }
+		}
+		
+		// now the top node is the root of the structure
+		structure_elem.appendChild(top_node);
+		
+		//Siblings: get the other descendants of the selected node's parent
+		if (want_siblings) {
+		    
+		    Element parent_node = (Element)base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+		    String parent_id = getParentId(doc_id);
+		    // add siblings, - returns a pointer to the new current node 
+		    base_node = addSiblings(parent_node, parent_id, doc_id);
+		}
+		
+		// Children: get the descendants, but only one level deep
+		if (want_children) {
+		    addDescendants(base_node, doc_id, false);
+		} 
+		// Descendants: recursively get every descendant 
+		else if (want_descendants) {
+		    addDescendants(base_node, doc_id, true);
+		}
+	    } // if want structure
+	} // for each doc
+	return result;
+    }
+
+    /** Retrieve the content of a document */
+    protected Element processDocumentContentRetrieve(Element request)
+    {
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	if (!does_content) {
+	    // shouldn't get here
+	    return result;
+	}
+
+	// Get the parameters of the request
+  	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element extid_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, EXTID_PARAM);
+	boolean external_id = false;
+	if (extid_param != null && GSXML.getValue(extid_param).equals("1")) {
+	    external_id = true;
+	}
+	// Get the request content 
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
+	    return result;
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_list);
+	
+	// set up the retrieval??
+
+	// Get the documents
+	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
+							    GSXML.NODE_ID_ATT);
+	for (int i = 0; i < doc_ids.length; i++) {
+	    String doc_id = doc_ids[i];
+	    // Create the document node
+	    Element doc = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    doc_list.appendChild(doc);
+
+	    if (external_id) {
+		doc_id = translateExternalId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    } else if (idNeedsTranslating(doc_id)) {
+		doc_id = translateId(doc_id);
+		doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    }
+	    if (doc_id == null) {
+		continue;
+	    }
+	    try {
+		Element node_content = getNodeContent(doc_id);
+		doc.appendChild(node_content);
+	    } catch (GSException e) {
+		GSXML.addError(this.doc, result, e.getMessage());
+		return result;
+		
+	    }
+	}
+	return result;
+    }
+    
+    /** create an element to go into the structure. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+     */
+    protected Element createDocNode(String node_id) {
+	Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+
+	String doc_type = null;
+	if (default_document_type != null) {
+	    doc_type = default_document_type;
+	} else {
+	    doc_type = getDocType(node_id);
+	}
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+    /** adds all the children of doc_id the the doc element,
+     * and if recursive=true, adds all their children as well*/
+    protected void addDescendants(Element doc, String doc_id, 
+				  boolean recursive)
+    {
+	ArrayList child_ids = getChildrenIds(doc_id);
+	if (child_ids==null) return;
+	for (int i=0; i< child_ids.size(); i++) {
+	    String child_id = (String)child_ids.get(i);
+	    Element child_elem = createDocNode(child_id);
+	    doc.appendChild(child_elem);
+	    if (recursive && !child_elem.getAttribute(GSXML.NODE_TYPE_ATT).equals(GSXML.NODE_TYPE_LEAF)) {
+		addDescendants(child_elem, child_id, recursive);
+	    }
+	}
+    }
+
+    /** adds all the siblings of current_id to the parent element. 
+      returns the new current element*/
+    protected Element addSiblings(Element parent_node, String parent_id, 
+				  String current_id) {
+	Element current_node = (Element)parent_node.getFirstChild();
+	if (current_node == null) {
+	    // create a sensible error message
+	    System.err.println("AbstractDocumentRetrieve Error: there should be a first child.");
+	    return null;
+	}
+	// remove the current child,- will add it in later in its correct place
+	parent_node.removeChild(current_node);
+
+	// add in all the siblings,
+	addDescendants(parent_node, parent_id, false);
+
+	// find the node that is now the current node
+	// this assumes that the new node that was created is the same as 
+	// the old one that was removed - we may want to replace the new one 
+	// with the old one.
+	Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+	return new_current;
+    }
+
+    /** returns true if oid ends in
+	.fc (firstchild), 
+	.lc (lastchild), 
+	.pr (parent), 
+	.ns (next sibling), 
+	.ps (prev sibling), 
+	.rt (root) 
+	.ss (specified sibling), 
+	false otherwise 
+    */
+    protected boolean idNeedsTranslating(String id) {
+	if (id.length()<5) return false;
+	String tail = id.substring(id.length()-3);
+	return (tail.equals(".fc") || tail.equals(".lc") || 
+		tail.equals(".pr") || tail.equals(".ns") || 
+		tail.equals(".ps") || tail.equals(".rt") || 
+		tail.equals(".ss"));
+
+    }
+
+    /** returns the list of sibling ids, including the specified node_id */
+    protected ArrayList getSiblingIds(String node_id) {
+	String parent_id = getParentId(node_id);
+	if (parent_id == null) {
+	    return null;
+	}
+	return getChildrenIds(parent_id);
+
+    }
+
+   /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    protected String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (getParentId(node_id)==null) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (getChildrenIds(node_id)==null) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id 
+     * default implementation: just remove the suffix */
+    protected String translateId(String id) {
+	return id.substring(0,id.length());
+    }
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+     * it to a greenstone one 
+     * default implementation: return the id */
+    protected String translateExternalId(String id) {
+	return id;
+    }
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+	default implementation: return DOC_TYPE_SIMPLE
+    */
+    protected String getDocType(String node_id) {
+	return GSXML.DOC_TYPE_SIMPLE;
+    }
+	
+
+    /** returns the id of the root node of the document containing 
+     * node node_id. may be the same as node_id 
+     * default implemntation: return node_id
+    */
+    protected String getRootId(String node_id) {
+	return node_id;
+    }
+    /** returns a list of the child ids in order, null if no children 
+     * default implementation: return null */
+    protected ArrayList getChildrenIds(String node_id) {
+	return null;
+    }
+    /** returns the node id of the parent node, null if no parent 
+     * default implementation: return null */
+    protected String getParentId(String node_id) {
+	return null;
+    }
+
+    /** get the metadata for the doc node doc_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    abstract protected Element getMetadataList(String doc_id, 
+					       boolean all_metadata, 
+					       ArrayList metadata_names) throws GSException;
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     * can return 
+     */
+    abstract protected Element getNodeContent(String doc_id) throws GSException;
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    abstract protected String getStructureInfo(String doc_id, String info_type);
+    
+}   
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 21112)
@@ -0,0 +1,474 @@
+/*
+ *    AbstractGS2DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GDBMWrapper;
+import org.greenstone.gsdl3.util.DBInfo;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS2 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS2DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+
+    //    protected static final String EXTLINK_PARAM = "ext"; here or in base??    
+    protected String index_stem = null;
+
+    protected GDBMWrapper gdbm_src = null;
+
+
+    /** constructor */
+    protected AbstractGS2DocumentRetrieve()
+    {
+	this.gdbm_src = new GDBMWrapper();
+	this.macro_resolver = new GS2MacroResolver(gdbm_src);
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.gdbm_src.closeDatabase();
+    }
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractGS2DocumentRetrieve...");
+	//this.config_info = info;
+	
+	// the index stem is either specified in the config file or is  the collection name
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("AbstractGS2DocumentRetrieve.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+
+	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name, this.index_stem);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("AbstractGS2DocumentRetrieve Error: Could not open GDBM database!");
+	    return false;
+	}
+
+	return true;
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	return this.gdbm_src.translateOID(node_id);
+    }
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+	it to a greenstone one*/
+    protected String translateExternalId(String node_id){
+	return this.gdbm_src.externalId2OID(node_id);
+    }
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return OID.getTop(node_id);
+    }
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return null;
+	}
+	ArrayList children = new ArrayList();
+	StringTokenizer st = new StringTokenizer(contains, ";");
+	while (st.hasMoreTokens()) {
+	    String child_id = st.nextToken().replaceAll("\"", node_id);
+	    children.add(child_id);
+	}
+	return children;
+
+    }
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names)  
+	throws GSException {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+	String lang = "en"; // why do we need this??
+	if (all_metadata) {
+	    // return everything out of the database
+	    Set keys = info.getKeys();
+	    Iterator it = keys.iterator();
+	    while(it.hasNext()) {
+		String key = (String)it.next();
+		//String value = info.getInfo(key);
+		Vector values = info.getMultiInfo(key);
+		for(int i=0; i<values.size(); i++) {
+		    GSXML.addMetadata(this.doc, metadata_list, key, this.macro_resolver.resolve((String)values.elementAt(i), lang, MacroResolver.SCOPE_META, node_id));
+		}
+	    }
+	    
+	} else {
+	    for (int i=0; i<metadata_names.size(); i++) {
+		String meta_name = (String) metadata_names.get(i);
+		String value = getMetadata(node_id, info, meta_name, lang);
+		GSXML.addMetadata(this.doc, metadata_list, meta_name, value);
+	    }
+	}
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	    return value;
+	}
+	
+	if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    return String.valueOf(getNumChildren(doc_id));
+	}
+
+	
+	if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		return "-1";
+	    }
+	    
+	    DBInfo info = this.gdbm_src.getInfo(parent_id);
+	    if (info==null) {
+		return "-1";
+	    }
+	    
+	    String contains = info.getInfo("contains");
+	    contains = contains.replaceAll("\"", parent_id);
+	    String [] children = contains.split(";");
+	    for (int i=0;i<children.length;i++) {
+		String child_id = children[i];
+		if (child_id.equals(doc_id)) {
+		    return String.valueOf(i+1); // make it from 1 to length
+			   
+		}
+	    }
+	    
+	    return "-1";
+	} else {
+	    return null;
+	}
+
+    }
+
+    protected int getNumChildren(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return 0;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return 0;
+	}
+	String [] children = contains.split(";");
+	return children.length;
+    }
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id) throws GSException;
+
+    protected String getMetadata(String node_id, DBInfo info, 
+				 String metadata, String lang) {
+	boolean multiple = false;
+	String relation = "";
+	String separator = ", ";
+	int pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	if (pos ==-1) {
+	    // just a plain meta entry eg dc.Title
+	    return macro_resolver.resolve((String)info.getInfo(metadata), lang, MacroResolver.SCOPE_META, node_id);
+	}
+	
+	String temp = metadata.substring(0, pos);
+	metadata = metadata.substring(pos+1);
+	// check for all on the front
+	if (temp.equals("all")) {
+	    multiple=true;	    
+	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	    if (pos ==-1) {
+		temp = "";
+	    } else {
+		temp = metadata.substring(0, pos);
+		metadata = metadata.substring(pos+1);
+	    }
+	}
+	
+	// now check for relational info
+	if (temp.equals("parent") || temp.equals("root") || temp.equals( "ancestors")) { // "current" "siblings" "children" "descendents"
+	    relation = temp;
+	    pos = metadata.indexOf(GSConstants.META_RELATION_SEP);
+	    if (pos == -1) {
+		temp = "";
+	    } else {
+		temp = metadata.substring(0, pos);
+		metadata = metadata.substring(pos+1);
+	    }
+	}
+	
+	// now look for separator info
+	if (temp.startsWith(GSConstants.META_SEPARATOR_SEP) && temp.endsWith(GSConstants.META_SEPARATOR_SEP)) {
+	    separator = temp.substring(1, temp.length()-1);
+	    
+	}
+	
+	String relation_id = node_id;
+	if (relation.equals("parent") || relation.equals("ancestors")) {
+	    relation_id = OID.getParent(node_id);
+	    // parent or ancestor does not include self
+	    if (relation_id.equals(node_id)){
+		return "";
+	    }
+	} else if (relation.equals("root")) {
+	    relation_id = OID.getTop(node_id);
+	}
+	
+	// now we either have a single node, or we have ancestors	
+	DBInfo relation_info;
+	if (relation_id.equals(node_id)) {
+	    relation_info = info;
+	} else {
+	    relation_info = this.gdbm_src.getInfo(relation_id);
+	}
+	if (relation_info == null) {
+	    return "";
+	}
+
+	StringBuffer result = new StringBuffer();
+
+	if (!multiple) {
+	    result.append(this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, MacroResolver.SCOPE_META, relation_id));
+	} else {
+	    // we have multiple meta
+	    Vector values = relation_info.getMultiInfo(metadata);
+	    if (values != null) {
+		boolean first = true;
+		for (int i=0; i<values.size(); i++) {
+		    if (first) {
+			first = false;
+		    } else {
+			result.append(separator);
+		    }
+		    result.append(this.macro_resolver.resolve((String)values.elementAt(i), lang, MacroResolver.SCOPE_META, relation_id));
+		}
+	    }
+	}
+	// if not ancestors, then this is all we do
+	if (!relation.equals("ancestors")) {
+	    return result.toString();
+	}
+	
+	// now do the ancestors 
+	String current_id = relation_id;
+	relation_id = OID.getParent(current_id);
+	while (!relation_id.equals(current_id)) {
+	    relation_info = this.gdbm_src.getInfo(relation_id);
+	    if (relation_info == null) return result.toString();
+	    if (!multiple) {
+		result.insert(0, separator);
+		result.insert(0, this.macro_resolver.resolve(relation_info.getInfo(metadata), lang, MacroResolver.SCOPE_META, relation_id));
+	    } else {
+		Vector values = relation_info.getMultiInfo(metadata);
+		if (values != null) {
+		    for (int i=values.size()-1; i>=0; i--) {
+			result.insert(0, separator);
+			result.insert(0, this.macro_resolver.resolve((String)values.elementAt(i), lang, MacroResolver.SCOPE_META, relation_id));
+		    }
+		}
+		
+	    }
+	    current_id = relation_id;
+	    relation_id = OID.getParent(current_id);
+	} 
+	return result.toString();
+    }
+    
+
+    /** needs to get info from gdbm database - if the calling code gets it already it may pay to pass it in instead */
+    protected String resolveTextMacros(String doc_content, String doc_id, String lang)
+    {
+	DBInfo info = null;
+	if (doc_content.indexOf("_httpdocimg_")!=-1) {
+	    String top_doc_id = OID.getTop(doc_id);
+	    info = this.gdbm_src.getInfo(top_doc_id);
+	    if (info == null) {
+		// perhaps we had per.iods in the ids - just try the current id
+		top_doc_id = doc_id;
+		info = this.gdbm_src.getInfo(top_doc_id);
+	    }
+	    if (info != null) {
+		String archivedir = info.getInfo("archivedir");
+		String image_dir  = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/"+archivedir;
+		
+		// Resolve all "_httpdocimg_"s
+		doc_content = doc_content.replaceAll("_httpdocimg_", image_dir);
+	    }
+	}
+	// resolve any collection specific macros
+	doc_content = macro_resolver.resolve(doc_content, lang, MacroResolver.SCOPE_TEXT, doc_id);
+	return doc_content;
+    }
+
+    protected Element getInfo(String doc_id, String info_type) {
+
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	} else if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    value = String.valueOf(getNumChildren(doc_id));
+	} else if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="-1";
+	    } else {
+		DBInfo info = this.gdbm_src.getInfo(parent_id);
+		if (info==null) {
+		    value ="-1";
+		} else {
+		    String contains = info.getInfo("contains");
+		    contains = contains.replaceAll("\"", parent_id);
+		    String [] children = contains.split(";");
+		    for (int i=0;i<children.length;i++) {
+			String child_id = children[i];
+			if (child_id.equals(doc_id)) {
+			    value = String.valueOf(i+1); // make it from 1 to length
+			    break;
+			}
+		    }
+		}
+	    }
+	} else {
+	    return null;
+	}
+	Element info_elem = this.doc.createElement("info");
+	info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+	info_elem.setAttribute(GSXML.VALUE_ATT, value);
+	return info_elem;
+    }
+
+}   
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractGS3DocumentRetrieve.java	(revision 21112)
@@ -0,0 +1,218 @@
+/*
+ *    AbstractGS3DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+/** Implements the generic retrieval and classifier services for GS3 
+ * collections.
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public abstract class AbstractGS3DocumentRetrieve
+    extends AbstractDocumentRetrieve {
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+
+    protected SQLQuery database = null;
+    protected String index_stem = null;
+
+
+    /** constructor */
+    protected AbstractGS3DocumentRetrieve()
+    {
+	this.database = new SQLQuery();
+	// set up a macro resolver
+    }
+    
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractGS3DocumentRetrieve...");
+
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("GS3MGRetrieve.configure(): indexStem element not found, stem will default to 'index'");
+	    this.index_stem = "index";
+	}
+
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+    
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getTop(node_id);
+    }
+    
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	if (GS3OID.isDocTop(node_id)) { // hack
+	    node_id =  GS3OID.createOID(node_id, "1");
+	}
+	ArrayList documents = database.getDocumentChildren(node_id);
+	if (documents == null || documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+    
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata,
+				      ArrayList metadata_names)   
+	throws GSException {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getDocumentMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null; // not implemented yet
+    }
+
+    
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+    }
+
+    protected String resolveRelativeLinks(String doc_content, String doc_id) {
+
+	String http_path  = this.site_http_address + "/collect/"+this.cluster_name;
+	doc_content = doc_content.replaceAll("_httpcollection_", http_path);
+	return doc_content;
+	
+    }
+
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    abstract protected Element getNodeContent(String doc_id) throws GSException;
+
+}   
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGPPSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGPPSearch.java	(revision 21112)
@@ -0,0 +1,708 @@
+/*
+ *    AbstractMGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.util.Iterator;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.Vector;
+import java.io.File;
+
+
+/** Partially implements a generic MGPP search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGPPSearch
+    extends AbstractSearch
+{
+
+    // extra services offered by mgpp collections
+    private static final String FIELD_QUERY_SERVICE = "FieldQuery";
+    private static final String ADVANCED_FIELD_QUERY_SERVICE = "AdvancedFieldQuery";
+
+    // extra parameters used
+    // private static final String INDEX_FIELD_PARAM = "index";
+    private static final String LEVEL_PARAM = "level";
+    private static final String RANK_PARAM = "sortBy";
+    private static final String RANK_PARAM_RANK = "1";
+    private static final String RANK_PARAM_NONE = "0";
+    private static final String SIMPLE_FIELD_PARAM = "simpleField";
+    private static final String ADVANCED_FIELD_PARAM = "complexField";
+
+    // more params for field query
+    private static final String FIELD_QUERY_PARAM = "fqv";
+    private static final String FIELD_STEM_PARAM = "fqs";
+    private static final String FIELD_CASE_PARAM = "fqc";
+    private static final String FIELD_FIELD_PARAM = "fqf";
+    private static final String FIELD_COMBINE_PARAM = "fqk";
+    private static final String FIELD_COMBINE_PARAM_AND = "0";
+    private static final String FIELD_COMBINE_PARAM_OR = "1";
+    private static final String FIELD_COMBINE_PARAM_NOT = "2";
+
+    // some stuff for config files
+    private static final String SEARCH_TYPE_ELEM = "searchType";
+    private static final String SEARCH_TYPE_PLAIN = "plain";
+    private static final String SEARCH_TYPE_FORM = "form";
+    private static final String SEARCH_TYPE_FORM_SIMPLE = "simple";
+    private static final String SEARCH_TYPE_FORM_ADVANCED = "advanced";
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+    protected static final String LEVEL_ELEM = "level";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    private static final int TEXT_QUERY = 0;
+    private static final int SIMPLE_QUERY = 1;
+    private static final int ADVANCED_QUERY = 2;
+
+    protected static final String FIELD_ATT = "field";
+    private MGPPWrapper mgpp_src=null;
+
+   // the default level for retrieval - and we'll use it for searching too
+    private String default_level=null;
+    // the default field for searching
+    private String default_field = null;
+    // which search services will we offer??
+    private boolean plain_search = false;
+    private boolean simple_form_search = false;
+    private boolean advanced_form_search = false;
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+
+    public AbstractMGPPSearch()
+    {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (super.configure(info, extra_info) == false)
+	    return false;
+	
+	// Do specific configuration
+	System.out.println("Configuring AbstractMGPPSearch...");
+
+	// do we support any of the extended features?
+	does_chunking = true;
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise its "" and will be the first one in the list
+	
+	// the index stem is either specified in the config file or is "index"
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("AbstractMGPPSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+
+	// the generic config has set up the text query service, but we may not want it
+	Element search_type_list = (Element) GSXML.getChildByTagName(info, SEARCH_TYPE_ELEM + GSXML.LIST_MODIFIER);
+	if (search_type_list == null) {
+	    // assume form and plain
+	    this.plain_search = true;
+	    this.simple_form_search = true;
+	    this.advanced_form_search = true;
+	} else {
+	    NodeList types = search_type_list.getElementsByTagName(SEARCH_TYPE_ELEM);
+	    for (int i=0; i<types.getLength(); i++) {
+		Element t = (Element)types.item(i);
+		String type_name = t.getAttribute(GSXML.NAME_ATT);
+		if (type_name.equals(SEARCH_TYPE_PLAIN)) {
+		    this.plain_search = true;
+		} else if (type_name.equals(SEARCH_TYPE_FORM)) {
+		    String type_type = t.getAttribute(GSXML.TYPE_ATT);
+		    if (type_type.equals("")) {
+			this.simple_form_search = true;
+			this.advanced_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_SIMPLE)) {
+			this.simple_form_search = true;
+		    } else if (type_type.equals(SEARCH_TYPE_FORM_ADVANCED)) {
+			this.advanced_form_search = true;
+			
+		    }
+		}
+	    }
+	}
+
+	if (!this.plain_search) {
+	    // need to remove the TextQuery service
+	    Element tq_service = GSXML.getNamedElement(short_service_info, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	    short_service_info.removeChild(tq_service);
+	    
+	}
+
+       
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+
+	// the default level is also the level which gdbm is expecting
+	// this must not be overwritten
+	this.mgpp_src.setReturnLevel(this.default_level);
+	// return term info
+	this.mgpp_src.setReturnTerms(true);
+	// set the default - this may be overwritten by query params
+	this.mgpp_src.setQueryLevel(this.default_level);
+
+	// set up the extra services which are available for this collection
+	// check the config info - if there is no field list, then there is no fielded searching
+	
+	Element field_list = (Element) GSXML.getChildByTagName(info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	if (field_list==null) {
+	    // nothing more to do
+	    return true;
+	}
+
+	// the format info is the same for all services
+	Element format_info = (Element)format_info_map.get(TEXT_QUERY_SERVICE);
+
+	// find the default field - use the first one
+	Element first_field = (Element)GSXML.getChildByTagName(field_list, GSXML.FIELD_ELEM);
+	default_field = first_field.getAttribute(GSXML.SHORTNAME_ATT);
+	// else set up the fielded query services
+	
+	if (this.simple_form_search) {
+	    // set up short_service_info_ - for now just has id and type - name will be added in on teh fly
+	    Element fq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    fq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    fq_service.setAttribute(GSXML.NAME_ATT, FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(fq_service);
+
+	    if (format_info != null) {
+		this.format_info_map.put(FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	
+	if (this.advanced_form_search) {
+	    Element afq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    afq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    afq_service.setAttribute(GSXML.NAME_ATT, ADVANCED_FIELD_QUERY_SERVICE);
+	    this.short_service_info.appendChild(afq_service);
+	    
+	    if (format_info != null) {
+		this.format_info_map.put(ADVANCED_FIELD_QUERY_SERVICE, format_info);
+	    }
+	}
+	return true;
+
+    }
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+	// should we check that the service is actually on offer? presumably we wont get asked for services that we haven't advertised previously.
+	
+	if (!service_id.equals(FIELD_QUERY_SERVICE) && !service_id.equals(ADVANCED_FIELD_QUERY_SERVICE)) {
+	    return super.getServiceDescription(service_id, lang, subset);
+	}
+	
+	
+	Element service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	service.setAttribute(GSXML.NAME_ATT, service_id);
+	if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service_id+".name", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service_id+".submit", lang)));
+	    service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service_id+".description", lang)));
+	
+	}
+	if (subset == null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);	
+	    service.appendChild(param_list);
+	    if (service_id.equals(FIELD_QUERY_SERVICE)) {
+		
+		addCustomQueryParams(param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list, lang);
+		// create a multi param for the fields etc
+		// text box, field
+		Element multiparam = null;
+		Element param=null;
+		multiparam = GSXML.createParameterDescription(this.doc, SIMPLE_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		// the components
+		createParameter(FIELD_QUERY_PARAM, multiparam,  lang);
+		createParameter(FIELD_FIELD_PARAM, multiparam,  lang);
+		
+	    } else { 
+		createParameter(LEVEL_PARAM, param_list,  lang);
+		createParameter(RANK_PARAM, param_list, lang);
+		createParameter(MAXDOCS_PARAM, param_list,  lang);
+		
+		
+		// create a multi param for the fields etc
+		// text box, stem, case, field
+		
+		Element multiparam = null;
+		Element param=null;
+		
+		multiparam = GSXML.createParameterDescription(this.doc, ADVANCED_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+		multiparam.setAttribute("occurs", "4");
+		param_list.appendChild(multiparam);
+		
+		createParameter(FIELD_COMBINE_PARAM, multiparam,  lang);
+		createParameter(FIELD_QUERY_PARAM, multiparam,  lang);
+		createParameter(FIELD_CASE_PARAM, multiparam,  lang);
+		createParameter(FIELD_STEM_PARAM, multiparam,  lang);
+		createParameter(FIELD_FIELD_PARAM, multiparam,  lang);    
+		
+	    } 
+	}
+	return service;
+	
+    }
+
+    /** add in the mgpp specific params to TextQuery */
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(LEVEL_PARAM, param_list, lang);
+	createParameter(CASE_PARAM, param_list, lang);
+	createParameter(STEM_PARAM, param_list, lang);
+	createParameter(MATCH_PARAM, param_list, lang);
+	createParameter(RANK_PARAM, param_list, lang);
+
+    }
+
+    /** create a param and add to the list */
+    protected void createParameter(String name, Element param_list, String lang) 
+    {
+	Element param = null;
+	if (name.equals(LEVEL_PARAM)) {
+	    ArrayList level_ids = new ArrayList();
+	    ArrayList level_names = new ArrayList();
+	    getLevelData(level_ids, level_names, lang);
+	    if (level_ids.size()>1) {
+		// the first one is the default
+		param = GSXML.createParameterDescription2(this.doc, LEVEL_PARAM, getTextString("param."+LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)level_ids.get(0), level_ids, level_names);
+	    }
+	    
+	} else if (name.equals(RANK_PARAM)) {
+	    String [] vals1 = {RANK_PARAM_RANK, RANK_PARAM_NONE };
+	    String [] vals1_texts = { getTextString("param."+RANK_PARAM+"."+RANK_PARAM_RANK, lang), getTextString("param."+RANK_PARAM+"."+RANK_PARAM_NONE, lang)};
+	    
+	    param = GSXML.createParameterDescription(this.doc, RANK_PARAM, getTextString("param."+RANK_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, RANK_PARAM_RANK, vals1, vals1_texts );
+	    
+	} else if (name.equals(FIELD_QUERY_PARAM)) {
+	    param = GSXML.createParameterDescription(this.doc, FIELD_QUERY_PARAM, getTextString("param."+FIELD_QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+	    
+	    
+	} else if (name.equals(FIELD_CASE_PARAM) || name.equals(FIELD_STEM_PARAM)) {
+	    String[] bool_ops = {"0", "1"};
+	    String[] bool_texts = {getTextString("param.boolean.off", lang, "AbstractSearch"),getTextString("param.boolean.on", lang, "AbstractSearch")}; 
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_BOOLEAN, BOOLEAN_PARAM_ON, bool_ops, bool_texts);
+	    
+	} else if (name.equals(FIELD_FIELD_PARAM)) {
+	    ArrayList fields = new ArrayList();
+	    ArrayList field_names = new ArrayList();
+	    getIndexData(fields, field_names, lang);
+	    // the field list -  read from config file
+	    param = GSXML.createParameterDescription2(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)fields.get(0), fields, field_names );
+	    
+	} else if (name.equals(FIELD_COMBINE_PARAM)) {
+	    
+	    String []vals = {FIELD_COMBINE_PARAM_AND, FIELD_COMBINE_PARAM_OR, FIELD_COMBINE_PARAM_NOT};
+	    String []val_texts = {getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_AND, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_OR, lang), getTextString("param."+FIELD_COMBINE_PARAM+"."+FIELD_COMBINE_PARAM_NOT, lang)}; 
+		
+	    
+	    param = GSXML.createParameterDescription(this.doc, FIELD_COMBINE_PARAM, "", GSXML.PARAM_TYPE_ENUM_SINGLE, FIELD_COMBINE_PARAM_AND, vals, val_texts);
+	    param.setAttribute(GSXML.PARAM_IGNORE_POS_ATT, "0");
+	}
+
+	if (param != null) {
+	    param_list.appendChild(param);
+	} else {
+	    super.createParameter(name, param_list, lang);
+	}
+    }
+    
+    // should cache some of this
+    protected void getLevelData(ArrayList level_ids, ArrayList level_names, String lang) 
+    {
+	Element level_list = (Element)GSXML.getChildByTagName(this.config_info, LEVEL_ELEM+GSXML.LIST_MODIFIER);
+	NodeList levels = level_list.getElementsByTagName(LEVEL_ELEM);
+	for (int i=0; i<levels.getLength(); i++) {
+	    level_ids.add(((Element)levels.item(i)).getAttribute(GSXML.NAME_ATT));
+	    level_names.add(getTextString("level."+level_ids.get(i), lang));
+	}
+	
+    }
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang){
+	// we currently only have one index, and use the field data as the index
+		    
+	// the field list -  read from config file
+	Element field_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.FIELD_ELEM+GSXML.LIST_MODIFIER);
+	NodeList fields = field_list.getElementsByTagName(GSXML.FIELD_ELEM);
+	for (int i=0; i< fields.getLength();i++) {
+	    index_ids.add(((Element)fields.item(i)).getAttribute(GSXML.SHORTNAME_ATT));
+	    // should these be changed to a text element based on lang?
+	    // or is the name of a metadata element eg dc:Title its 
+	    // name in all langs
+	    index_names.add(((Element)fields.item(i)).getAttribute(GSXML.NAME_ATT));
+	    
+	}
+    }
+
+    // the following three functions are needed so the base class can 
+    // call the process+SERVICE_NAME methods
+    /** process a text query */
+    protected Element processTextQuery(Element request) {
+	return processAnyQuery(request, TEXT_QUERY);
+    }
+
+    /** process a field query */    
+    protected Element processFieldQuery(Element request) {
+	return processAnyQuery(request, SIMPLE_QUERY);
+    }
+
+    /** process an advanced field query */
+    protected Element processAdvancedFieldQuery(Element request) {
+	return processAnyQuery(request, ADVANCED_QUERY);
+    }
+    
+    /** process a  query */
+    protected Element processAnyQuery(Element request, int query_type)
+    {
+
+	String service_name=null;
+	String empty_query_test_param=null;
+	// set up the type specific bits
+	switch (query_type) {
+	case TEXT_QUERY:
+	    service_name = TEXT_QUERY_SERVICE; 
+	    empty_query_test_param = QUERY_PARAM;
+	    break;
+	case SIMPLE_QUERY:
+	    service_name = FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	case ADVANCED_QUERY:
+	    service_name = ADVANCED_FIELD_QUERY_SERVICE;
+	    empty_query_test_param = FIELD_QUERY_PARAM;
+	    break;
+	default:
+	    // should never get here
+	    System.err.println("AbstractMGPPSearch: wrong query type!!");
+	    return null;
+	}
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, service_name);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("Error: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(empty_query_test_param);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If a field hasn't been specified, use the default - for textQuery
+	String field = (String) params.get(INDEX_PARAM); 
+	if (field == null) {
+	    field = default_field;
+	}
+	
+	// set up mgpp_src
+	String indexdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mgpp_src.loadIndexData(indexdir);
+	setStandardQueryParams(params);
+	
+	// if field search, create the query string
+	switch (query_type) {
+	case TEXT_QUERY:
+	    query = addFieldInfo(query, field);
+	    break;
+	case SIMPLE_QUERY:
+	    query = parseFieldQueryParams(params);
+	    break;
+	case ADVANCED_QUERY:
+	    query = parseAdvancedFieldQueryParams(params);
+	    break;
+	}
+	// run the query
+       	this.mgpp_src.runQuery(query);
+	MGPPQueryResult mqr= this.mgpp_src.getQueryResult();
+
+	// build up the response
+	
+	// Create a metadata list to store information about the query results
+	// should we be using metadataList? or something else?
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long totalDocs = mqr.getTotalDocs();
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+totalDocs);
+
+	// Create a document list to store the matching documents, and add them
+	Vector docs = mqr.getDocs();
+
+	// add a metadata item to specify docs returned
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+docs.size());
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+	if (docs.size()>0) {
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGPPDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGPPDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGPPNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		//doc_node.setAttribute("rank", Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	// Create a term list to store the term information, and add it
+	String query_level = (String)params.get(LEVEL_PARAM); // the current query level
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGPPTermInfo term_info = (MGPPTermInfo) terms.get(t);
+	    
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
+	    term_elem.setAttribute(STEM_ATT, "" + term_info.stem_method_);
+	    term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
+	    term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
+	    field = term_info.tag_;
+	    if (field.equals(query_level)) {
+		// ignore
+		field = "";
+	    }
+	    term_elem.setAttribute(FIELD_ATT, field);
+	    
+	    Vector equiv_terms = term_info.equiv_terms_;
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+	    
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		String equiv_term = (String) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "");
+		equiv_term_elem.setAttribute(FREQ_ATT, "");
+		equiv_term_list.appendChild(equiv_term_elem);
+	    }
+
+	    term_list.appendChild(term_elem);
+	}
+
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params) {
+
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+	   
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setCase(val);		
+	    } else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON)?true:false);
+		this.mgpp_src.setStem(val);
+	    } else if (name.equals(MAXDOCS_PARAM)&& !value.equals("")) {
+		int docs = Integer.parseInt(value);
+		this.mgpp_src.setMaxDocs(docs);
+	    } else if (name.equals(LEVEL_PARAM)) {
+		this.mgpp_src.setQueryLevel(value);
+	    } else if (name.equals(MATCH_PARAM)) {
+		int mode;
+		if (value.equals(MATCH_PARAM_ALL)) mode=1;
+		else mode=0;
+		this.mgpp_src.setMatchMode(mode);
+	    } else if (name.equals(RANK_PARAM)) {
+		if (value.equals(RANK_PARAM_RANK)) {
+		    this.mgpp_src.setSortByRank(true);
+		} else if (value.equals(RANK_PARAM_NONE)) {
+		    this.mgpp_src.setSortByRank(false);
+		}
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    protected String addFieldInfo(String query, String field) {
+	if (field.equals("") || field.equals("ZZ")) {
+	    return query;
+	}
+	return "["+query+"]:"+field;
+    }
+    /** combines all the field params into a single query
+     * - for simple field query */
+    protected String parseFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String combine="&";
+	String match = (String)params.get(MATCH_PARAM);
+	if (match.equals(MATCH_PARAM_SOME)) {
+	    combine = "|";
+	}
+	
+	for (int i=0; i<texts.length; i++) {
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    /** combines all the field params into a single query
+     * - for advanced field query */   
+    protected String parseAdvancedFieldQueryParams(HashMap params) {
+
+	StringBuffer final_query = new StringBuffer(256);
+	String text_line = (String)params.get(FIELD_QUERY_PARAM);
+	String[] texts = text_line.split(",", -1);
+	String field_line = (String)params.get(FIELD_FIELD_PARAM);
+	String[] fields = field_line.split(",", -1);
+	String case_line = (String)params.get(FIELD_CASE_PARAM);
+	String[] cases = case_line.split(",", -1);
+	String stem_line = (String)params.get(FIELD_STEM_PARAM);
+	String[] stems = stem_line.split(",", -1);
+	String  combine_line = (String)params.get(FIELD_COMBINE_PARAM);
+	String [] combines = combine_line.split(",", -1);
+	String combine = "&";
+	for (int i=0; i<texts.length; i++) {
+	    if (i==0) {// assume first one is blank
+		combine = "";
+	    } else {
+		String x = combines[i]; 
+		if (x.equals(FIELD_COMBINE_PARAM_AND)) {
+		    combine = "&";
+		} else if (x.equals(FIELD_COMBINE_PARAM_OR)) {
+		    combine = "|";
+		} else if (x.equals(FIELD_COMBINE_PARAM_NOT)) {
+		    combine = "!";
+		} 
+		
+	    }
+
+	    String q = texts[i].trim();
+	    if (!q.equals("")) {
+		q = addStemAndCase(q, stems[i], cases[i]);
+		addQueryElem(final_query, q, fields[i], combine);
+	    }
+	}
+
+	return final_query.toString();
+    }
+
+    protected void addQueryElem(StringBuffer s, String q, String f, String c) {
+
+	String combine="";
+	if (s.length()>0) {
+	    combine = " "+c+" ";
+	}
+	if (f.equals("")||f.equals("ZZ")) {
+	    s.append(combine+q);
+	} else {
+	    s.append(combine+"["+q+"]:"+f);
+	}
+    }
+    
+    protected String addStemAndCase(String q, String s, String c) {
+	String mods = "#";
+	if (c.equals("1")) {
+	    mods += "i";
+	} else {
+	    mods += "c";
+	}
+	if (s.equals("1")) {
+	    mods += "s";
+	} else {
+	    mods+= "u";
+	}
+	StringBuffer temp = new StringBuffer();
+	String [] terms = q.split(" ");
+	for (int i=0; i<terms.length; i++) {
+	    String t = terms[i].trim();
+	    
+	    if (!t.equals("") && !t.equals("TX")) {
+		temp.append(" "+t+mods);
+	    }
+	}
+	return temp.toString();
+    }
+
+    /** convert MGPP internal id to Greenstone oid */
+    abstract protected String MGPPNum2OID(long docnum);
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractMGSearch.java	(revision 21112)
@@ -0,0 +1,307 @@
+/*
+ *    AbstractMGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.Iterator;
+import java.io.File;
+
+/** Partially implements a generic MG search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+abstract public class AbstractMGSearch
+    extends AbstractSearch
+{
+    protected static final String CASE_PARAM = "case";
+    protected static final String STEM_PARAM = "stem";
+    protected static final String MATCH_PARAM = "matchMode";
+    protected static final String MATCH_PARAM_ALL = "all";
+    protected static final String MATCH_PARAM_SOME = "some";
+
+    protected static final String BOOLEAN_PARAM_ON = "1";
+    protected static final String BOOLEAN_PARAM_OFF = "0";
+
+    protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+    protected static final String STEM_ATT = "stem";
+    protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+    protected static final String FREQ_ATT = "freq";
+
+    // Elements used in the config file that are specific to this class
+    protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    protected static final String INDEX_STEM_ELEM = "indexStem";
+    protected static final String INDEX_ELEM = "index";
+
+    /** the stem used for the index files */
+    protected String index_stem = null;
+    protected MGWrapper mg_src = null;
+
+    
+    public AbstractMGSearch()
+    {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	// do we support any of the extended features?
+	does_chunking = true;
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	} // otherwise will be "", and the first one will be the default
+
+	// the index stem is either the collection name or is specified in the config file
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+	if (index_stem_elem != null) {
+	    this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.index_stem == null || this.index_stem.equals("")) {
+	    System.err.println("AbstractMGSearch.configure(): indexStem element not found, stem will default to collection name");
+	    this.index_stem = this.cluster_name;
+	}
+	
+	// get display info from extra info
+	if (extra_info !=null) {
+	    Document owner = info.getOwnerDocument();
+	    // so far we have index specific display elements, and global format elements 
+	    NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+	    Element config_search = (Element)GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+	    
+	    for (int i=0; i<indexes.getLength();i++) {
+		Element ind = (Element)indexes.item(i);
+		String name = ind.getAttribute(GSXML.NAME_ATT);
+		Element node_extra = GSXML.getNamedElement(config_search,
+							   GSXML.INDEX_ELEM,
+							   GSXML.NAME_ATT,
+							   name);
+		if (node_extra == null) {
+		    System.err.println("GS2Search: haven't found extra info for index named "+name);
+		    continue;
+		}
+		
+		// get the display elements if any - displayName
+		NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (display_names !=null) {
+		    for (int j=0; j<display_names.getLength(); j++) {
+			Element e = (Element)display_names.item(j);
+			ind.appendChild(owner.importNode(e, true));
+		    }
+		}
+	    } // for each index
+	}
+	return true;
+    }
+
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	createParameter(CASE_PARAM, param_list, lang);
+	createParameter(STEM_PARAM, param_list, lang);
+	createParameter(MATCH_PARAM, param_list, lang);	
+    }
+
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang) {
+	// the index info -
+	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+	NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+	int len = indexes.getLength();
+	// now add even if there is only one
+	for (int i=0; i<len; i++) {
+	    Element index = (Element)indexes.item(i);
+	    index_ids.add(index.getAttribute(GSXML.NAME_ATT));
+	    index_names.add(GSXML.getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+	}
+
+    }
+    /** do the actual query */
+    protected Element processTextQuery(Element request) 
+    {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("Error: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If an index hasn't been specified, use the default
+	String index = (String) params.get(INDEX_PARAM); 
+	if (index == null) {
+	    index = this.default_index;
+	}
+
+	// The location of the MG index and text files
+	String basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +  File.separatorChar;  // Needed for MG
+	String textdir = GSFile.collectionTextPath(this.index_stem);
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, index);
+	this.mg_src.setIndex(indexpath);
+	
+	// set the mg query parameters to the values the user has specified
+	setStandardQueryParams(params);
+	this.mg_src.runQuery(basedir, textdir, query);
+	MGQueryResult mqr = this.mg_src.getQueryResult();
+	if (mqr.isClear()) {
+	    // something has gone wrong
+	    GSXML.addError(this.doc, result, "Couldn't query the mg database", GSXML.ERROR_TYPE_SYSTEM);
+	    return result;
+	}
+	long totalDocs = mqr.getTotalDocs();
+
+	// Get the docnums out, and convert to HASH ids
+	Vector docs = mqr.getDocs();
+	if (docs.size() == 0) {
+	    System.err.println("GS2MGSearch: Warning: No results found...\n");
+	}
+
+	// Create a metadata list to store information about the query results
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+
+	// Add a metadata element specifying the number of matching documents
+	// because teh total number is just the number returned, use numDocsReturned, not numDocsMatched
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+totalDocs);
+	// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+	GSXML.addMetadata(this.doc, metadata_list, "query", query);
+
+	if (docs.size() > 0) {
+	    // Create a document list to store the matching documents, and add them
+	    Element document_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	    result.appendChild(document_list);
+	    for (int d = 0; d < docs.size(); d++) {
+		long docnum = ((MGDocInfo) docs.elementAt(d)).num_;
+		float rank = ((MGDocInfo) docs.elementAt(d)).rank_;
+		String doc_id = MGNum2OID(docnum);
+		Element doc_node = createDocNode(doc_id, Float.toString(rank));
+		document_list.appendChild(doc_node);
+	    }
+	}
+	
+	// Create a term list to store the term information, and add it
+	Element term_list = this.doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);
+	Vector terms = mqr.getTerms();
+	for (int t = 0; t < terms.size(); t++) {
+	    MGTermInfo term_info = (MGTermInfo) terms.get(t);
+
+	    String term = term_info.term_;
+	    int stem_method = term_info.stem_method_;
+	    Vector equiv_terms = term_info.equiv_terms_;
+
+	    Element term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+	    term_elem.setAttribute(GSXML.NAME_ATT, term);
+	    term_elem.setAttribute(STEM_ATT, "" + stem_method);
+
+	    Element equiv_term_list = this.doc.createElement(EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+	    term_elem.appendChild(equiv_term_list);
+
+	    long total_term_freq = 0;
+	    for (int et = 0; et < equiv_terms.size(); et++) {
+		MGEquivTermInfo equiv_term_info = (MGEquivTermInfo) equiv_terms.get(et);
+
+		Element equiv_term_elem = this.doc.createElement(GSXML.TERM_ELEM);
+		equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term_info.term_);
+		equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + equiv_term_info.match_docs_);
+		equiv_term_elem.setAttribute(FREQ_ATT, "" + equiv_term_info.term_freq_);
+		equiv_term_list.appendChild(equiv_term_elem);
+
+		total_term_freq += equiv_term_info.term_freq_;
+	    }
+
+	    term_elem.setAttribute(FREQ_ATT, "" + total_term_freq);
+	    term_list.appendChild(term_elem);
+	}
+	return result;
+    }
+
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap params)
+    {
+	// set the default ones
+	this.mg_src.setReturnTerms(true);
+	this.mg_src.setCase(true); // turn casefolding on by default
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+
+	    if (name.equals(CASE_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setCase(val);
+	    }
+	    else if (name.equals(STEM_PARAM)) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setStem(val);
+	    }
+	    else if (name.equals(MATCH_PARAM)) {
+		int mode = (value.equals(MATCH_PARAM_ALL) ? 1 : 0);
+		this.mg_src.setMatchMode(mode);
+	    }
+	    else if (name.equals(MAXDOCS_PARAM)) {
+		int docs = Integer.parseInt(value);
+		this.mg_src.setMaxDocs(docs);
+	    } // ignore any others
+	}
+	return true;
+    }
+
+    /** convert MG internal id to Greenstone oid */
+    abstract protected String MGNum2OID(long docnum);
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/AbstractSearch.java	(revision 21112)
@@ -0,0 +1,327 @@
+/*
+ *    AbstractSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/** Partially implements a generic search service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public abstract class AbstractSearch
+    extends ServiceRack 
+{
+    
+    // the search service
+    protected static final String TEXT_QUERY_SERVICE = "TextQuery";
+
+    // compulsory params
+    protected static final String INDEX_PARAM = "index";
+    protected static final String QUERY_PARAM = "query";
+
+    // optional standard params - some of these have to be implemented
+    protected static final String MAXDOCS_PARAM = "maxDocs";
+    protected static final String HITS_PER_PAGE_PARAM = "hitsPerPage";
+    protected static final String START_PAGE_PARAM = "startPage";
+
+    // some other common params that may be used
+    protected static final String CASE_PARAM = "case";
+    protected static final String STEM_PARAM = "stem";
+    protected static final String BOOLEAN_PARAM_ON = "1";
+    protected static final String BOOLEAN_PARAM_OFF = "0";
+    protected static final String MATCH_PARAM = "matchMode";
+    protected static final String MATCH_PARAM_ALL = "all";
+    protected static final String MATCH_PARAM_SOME = "some";
+
+    /** can more than one index be searched at the smae time? */
+    protected boolean does_multi_index_search = false;
+    /** does this service support paging of results? */
+    protected boolean does_paging = false;
+    /** does this service support asking for a subset of results? */
+    protected boolean does_chunking = false;
+    /** the default document type - use if all documents are the same type
+     */
+    protected String default_document_type = null;
+    /** the default index, or comma separated list if more than one is 
+     * the default (with start and end commas, eg ,TI,SU,). 
+     * Should be set by configure()
+     */
+    protected String default_index = "";
+
+    public AbstractSearch()
+    {
+    }
+
+    /** sets up the short service info for TextQuery. If other services 
+     * will be provided, should be added in the subclass configure
+     * also looks for search format info, and document format info
+     */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring AbstractSearch...");
+	
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has id and type. the name (lang dependent) will be added in if the list is requested.
+	Element tq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	this.short_service_info.appendChild(tq_service);
+
+	// add some format info to service map if there is any - look in extra info
+	// first look in buildConfig
+	Element format = (Element)GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+	
+	if (format==null) {
+	    String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+	    format = (Element) GSXML.getNodeByPath(extra_info, path);
+	}
+	//Element format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+	if (format != null) {
+	    this.format_info_map.put(TEXT_QUERY_SERVICE, this.doc.importNode(format, true));
+	}
+	
+	// look for document display format - for documentType
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    // check for docType option.
+	    Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+	    if (doc_type_opt != null) {
+		String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+		if (!value.equals("")) {
+		    this.default_document_type = value;
+		}
+	    }
+	}
+
+	return true;
+    }
+    
+    /** returns the description of the TextQuery service. If a subclass 
+     * provides other services they need to provides their own descriptions */
+    protected Element getServiceDescription(String service, String lang, String subset) 
+    {
+	if (!service.equals(TEXT_QUERY_SERVICE)) {
+	    return null;
+	}
+
+	Element tq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	if (subset==null || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getServiceName(TEXT_QUERY_SERVICE, lang) ));
+	    tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getServiceSubmit(TEXT_QUERY_SERVICE, lang) ));
+	    tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getServiceDescription(TEXT_QUERY_SERVICE, lang)));
+	}
+	if (subset==null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    addCustomQueryParams(param_list, lang);
+	    addStandardQueryParams(param_list, lang);
+	    tq_service.appendChild(param_list);
+	}
+	return tq_service;
+	
+    }
+
+    // perhaps these should be changed to search down the class hierarchy for 
+    // values - do can just put the info in the resource bundle to use it
+    /** returns the default name for the TextQuery service */
+    protected String getServiceName(String service_id, String lang) {
+	return getTextString(service_id+".name", lang);
+    }
+    
+    /** returns the default description for the TextQuery service */
+    protected String getServiceDescription(String service_id, String lang) {
+	return getTextString(service_id+".description", lang);
+    }
+
+    /** returns the default submit button text for the TextQuery service */
+    protected String getServiceSubmit(String service_id, String lang) {
+	return getTextString(service_id+".submit", lang);
+
+    }
+    /** adds the standard query params into the service description */
+    protected void addStandardQueryParams(Element param_list, String lang) 
+    {
+	createParameter(INDEX_PARAM, param_list, lang);
+	if (does_chunking) {
+	    createParameter(MAXDOCS_PARAM, param_list, lang);
+	}
+	if (does_paging) {
+	    createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
+	    createParameter(START_PAGE_PARAM, param_list, lang);
+	}
+	createParameter(QUERY_PARAM, param_list, lang);
+    }
+
+    /** adds any service specific query params into the service
+     *	default implementation: add nothing. subclasses may need to 
+     * override this to add in their specific parameters
+     */
+    protected void addCustomQueryParams(Element param_list, String lang) 
+    {
+	// default behaviour, do nothing
+    }
+
+    /** default implementations for the standard parameters plus some 
+     * other common ones 
+     * index, maxDocs, hitsPerPage, startPage, query, case, stem, 
+    */
+    protected void createParameter(String name, Element param_list, String lang) {
+	Element param = null;
+	if (name.equals(QUERY_PARAM)) {
+	    param = GSXML.createParameterDescription(this.doc, QUERY_PARAM, getTextString("param."+QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+	    param_list.appendChild(param);
+	} else if (name.equals(INDEX_PARAM)) {
+
+	    // should we make these class fields?
+	    ArrayList index_ids = new ArrayList();
+	    ArrayList index_names = new ArrayList();
+	    getIndexData(index_ids, index_names, lang);
+	    String param_type = GSXML.PARAM_TYPE_ENUM_SINGLE;
+	    if (does_multi_index_search) {
+		param_type = GSXML.PARAM_TYPE_ENUM_MULTI;
+	    }
+	    param = GSXML.createParameterDescription2(this.doc, INDEX_PARAM, getTextString("param."+INDEX_PARAM, lang), param_type, this.default_index, index_ids, index_names);
+	    param_list.appendChild(param);
+	} else if (name.equals(MAXDOCS_PARAM)) {
+	    	param = GSXML.createParameterDescription(this.doc, MAXDOCS_PARAM, getTextString("param."+MAXDOCS_PARAM, lang), GSXML.PARAM_TYPE_INTEGER, "10", null, null);
+	param_list.appendChild(param);
+	} else if (name.equals(CASE_PARAM) || name.equals(STEM_PARAM)) {
+	    String[] bool_ops = {"0", "1"};
+	    String[] bool_texts = {getTextString("param.boolean.off", lang),getTextString("param.boolean.on", lang)}; 
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_BOOLEAN, BOOLEAN_PARAM_ON, bool_ops, bool_texts);
+	    param_list.appendChild(param);
+	} else if (name.equals(MATCH_PARAM)) {
+	    String[] vals = {MATCH_PARAM_SOME, MATCH_PARAM_ALL };
+	    String[] val_texts = {getTextString("param."+MATCH_PARAM+"."+MATCH_PARAM_SOME, lang), getTextString("param."+MATCH_PARAM+"."+MATCH_PARAM_ALL, lang)}; 
+	    param = GSXML.createParameterDescription(this.doc, MATCH_PARAM, getTextString("param."+MATCH_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, MATCH_PARAM_SOME, vals, val_texts);
+	    param_list.appendChild(param);
+	} else if (name.equals(HITS_PER_PAGE_PARAM)) {
+	    // hits per page
+	    String [] hits_options = {"10", "30", "50"};
+	    param = GSXML.createParameterDescription(this.doc, HITS_PER_PAGE_PARAM, getTextString("param."+HITS_PER_PAGE_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, "10", hits_options, hits_options);
+	    param_list.appendChild(param);
+	} else if (name.equals(START_PAGE_PARAM)) {
+	    // start page - set to 1 for the search page
+	    param = GSXML.createParameterDescription(this.doc, START_PAGE_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, "1", null, null);
+	    param_list.appendChild(param);
+	}
+	    
+	    
+    }
+    /** create an element to go into the search results list. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy' rank='0.23'/>
+     */
+    protected Element createDocNode(String node_id, String rank) {
+	Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	node.setAttribute(GSXML.NODE_RANK_ATT, rank);
+	String doc_type = null;
+	if (default_document_type != null) {
+	    doc_type = default_document_type;
+	} else {
+	    doc_type = getDocType(node_id);
+	}
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+    /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    protected String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (!hasParent(node_id)) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (!hasChildren(node_id)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+	default implementation returns GSXML.DOC_TYPE_SIMPLE, over ride 
+	if documents can be hierarchical
+    */
+    protected String getDocType(String node_id) {
+	return GSXML.DOC_TYPE_SIMPLE;
+    }
+    
+    /** returns true if the node has child nodes 
+     * default implementation returns false, over ride if documents can be 
+     * hierarchical 
+     */
+    protected boolean hasChildren(String node_id) {
+	return false;
+    }
+    /** returns true if the node has a parent 
+     * default implementation returns false, over ride if documents can be 
+     * hierarchical*/
+    protected boolean hasParent(String node_id) {
+	return false;
+    }
+
+    /** do the actual query 
+     * must be implemented by subclass */
+    abstract protected Element processTextQuery(Element request); 
+	
+    /** get the details about the indexes available
+     * must be implemented by subclass
+     * there must be at least one index */
+    abstract protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang);
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/FedoraSearch.java.tmp
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/FedoraSearch.java.tmp	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/FedoraSearch.java.tmp	(revision 21112)
@@ -0,0 +1,302 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+//Fedora Web Services classes
+import fedora.client.utility.AutoFinder;
+import fedora.client.APIAStubFactory;
+import fedora.client.Downloader;
+import fedora.server.access.FedoraAPIA;
+import fedora.server.types.gen.FieldSearchQuery;
+import fedora.server.types.gen.FieldSearchResult;
+import fedora.server.types.gen.ListSession;
+import fedora.server.types.gen.ObjectFields;
+import fedora.server.types.gen.Condition;
+import fedora.server.types.gen.ComparisonOperator;
+
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.transform.Templates;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.MalformedURLException;
+import java.lang.Object;
+
+
+/**
+ *
+ * @author <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
+ * @version $Revision$
+ *
+ */
+
+public class FedoraSearch
+    extends AbstractSearch {
+    
+    // the services on offer
+    // these strings must match what is found in the properties file
+    protected static final String FIELD_PARAM = "fields";
+    
+    //Parameters connect to Proxy Server
+    private String proxyUser = null;
+    private String proxyPassword = null;
+    
+    private int maxResult, matchNum;
+    
+    protected String fedora_server_url = null;
+    
+    public FedoraSearch()
+    {
+    }
+    
+    //Configure FedoraSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "fedoraServer");
+	if (server_elem == null) {
+	    System.err.println("FedoraSearch.configure error: no fedoraServer element found");
+	    return false;
+	}
+	fedora_server_url = server_elem.getAttribute("url");
+	System.err.println("Fedora Server:" + fedora_server_url);
+
+	if (fedora_server_url.equals("")) {
+	    System.err.println("FedoraSearch.configure error: no url for the fedoraServer element");
+	    return false;
+	}
+	Element tq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	this.short_service_info.appendChild(tq_service);
+	//does_paging = true;
+	
+	// add some format info to service map if there is any
+	String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+	Element format = (Element) GSXML.getNodeByPath(extra_info, path);
+	if (format != null) {
+	    this.format_info_map.put(TEXT_QUERY_SERVICE, this.doc.importNode(format, true));
+	}
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("FedoraSearch Error:: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+	
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+	//Check Max number of Documents hit
+	String maxDocs = (String) params.get(MAXDOCS_PARAM);
+       	String hits_per_page = (String) params.get(HITS_PER_PAGE_PARAM);
+	
+	//Check the start_page number
+	String start_page = (String) params.get(START_PAGE_PARAM);	
+
+	// tidy whitespace
+	//query = query.replaceAll("\\s+", "+");
+	matchNum = 0;
+	maxResult = 20;
+
+	try {
+	    //Search Fedora Server
+	    String host = "toetoe";
+	    int port = 8200;
+	    String user = "fedoraAdmin";
+	    String pass = "fedoraAdmin";
+	    
+	    AutoFinder finder=new AutoFinder(host, port, user, pass);
+
+	    String[] fieldsArray = new String[] {"pid", "label", "fType", "bDef",
+						 "bMech", "cModel", "state", "ownerId", "cDate", "mDate",
+						 "dcmDate", "title", "creator", "subject", "description",
+						 "publisher", "contributor", "date", "type", "format",
+						 "identifier", "source", "language", "relation", "coverage",
+						 "rights"};
+	    
+	    String match_fields = join(fieldsArray, "|");
+	    
+	    FieldSearchQuery query_fedora=fedoraSearch(query, match_fields);
+	    
+            FieldSearchResult result_fedora = finder.findObjects(fieldsArray, maxResult, query_fedora);
+	    
+	    while (result_fedora != null) {
+		System.err.println("Search Result:" + result_fedora.getResultList().length);
+		for (int i=0; i<result_fedora.getResultList().length; i++) {
+		    ObjectFields o = result_fedora.getResultList()[i];
+		    matchNum++;
+		    String pid = o.getPid();
+		    String fedora_url = fedora_server_url +"/fedora/get/" + pid;
+		    String label = o.getLabel();
+		    String[] creator = o.getCreator();
+		    String[] subject = o.getSubject();
+		    String[] publisher = o.getPublisher();
+		    String[] contributor = o.getContributor();
+		    String[] title = o.getTitle();
+		    String[] description = o.getDescription();
+		    String[] format = o.getFormat();
+		    String[] identifier = o.getIdentifier();
+		    String[] source = o.getSource();
+		    String[] language = o.getLanguage();
+
+		    Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		    doc_node_list.appendChild(doc_node);
+		    
+		    Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		    doc_node.appendChild(metadata_list);
+		    
+		    if (fedora_url != null) {
+			GSXML.addMetadata(this.doc, metadata_list, "URL", fedora_url);
+		    }
+		    if (o.getDescription() != null) {
+			GSXML.addMetadata(this.doc, metadata_list, "Description", description[0]);
+		    }
+		    if (title[0] != null) {
+			GSXML.addMetadata(this.doc, metadata_list, "Title", title[0]);
+		    }
+		}
+		ListSession sess=result_fedora.getListSession();
+		if (sess!=null) {
+		    result_fedora = finder.resumeFindObjects(sess.getToken());
+		} else {
+		    result_fedora = null;
+		}
+	    }
+	} catch (Exception e) {
+	    System.err.println("ERROR: " + e.getClass().getName()
+			       + ((e.getMessage()==null) ? "" : ": " + e.getMessage()));
+	}
+    	
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	//System.err.println("FedoraSearch Number of Documents:" + matchNum);
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+matchNum);
+	
+	return result;
+    }
+
+    protected FieldSearchQuery fedoraSearch(String query_terms, String matchFields) {
+
+	FieldSearchQuery fedora_query=new FieldSearchQuery();
+	try {
+	    // check if query contains any fields name (pid, label, fType....)
+	    if (query_terms.matches("^("+ matchFields+").+")){
+		System.err.println("Implementing Specific Query....");
+
+		List conditions=fedora.server.search.Condition.getConditions(query_terms);
+		
+		Condition[] cond=new Condition[conditions.size()];
+		for (int i=0; i<conditions.size(); i++) {
+		    fedora.server.search.Condition c = (fedora.server.search.Condition) conditions.get(i);
+		    try {
+			String operator = transOperator(c.getOperator().getAbbreviation());
+			cond[i] = new Condition();
+			cond[i].setProperty(c.getProperty());
+			cond[i].setOperator(ComparisonOperator.fromValue(operator));
+			cond[i].setValue(c.getValue());
+		    } catch (Exception e){
+			e.printStackTrace();
+		    }
+		}
+		fedora_query.setConditions(cond);
+	    } else {
+		System.err.println("Implementing Simple Query...");
+		fedora_query.setTerms(query_terms);
+	    }
+	} catch (Exception e){
+	    e.printStackTrace();
+	}
+	return fedora_query;
+    }
+    
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
+       	index_ids.add("kw");
+	index_ids.add("au");	
+	index_ids.add("su");	
+	index_ids.add("ti");
+	index_ids.add("de");
+	index_ids.add("fu");
+	index_names.add(getTextString("param."+FIELD_PARAM+".kw", lang)); 
+	index_names.add(getTextString("param."+FIELD_PARAM+".au", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".su", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".ti", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".de", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".fu", lang));
+    }
+    
+    protected String getDocType (String node_id){
+	return node_id;
+    }
+    protected boolean hasChildren(String node_id){
+	return false;
+    }
+
+    protected boolean hasParent(String node_id){
+	return false;
+    }
+    private String transOperator(String operator){
+	String oper_char ="";
+	if (operator == "~") {
+	    oper_char = "has";
+	} else if (operator == "=") {
+	    oper_char = "eq";
+	} else if (operator == ">"){
+	    oper_char = "gt";
+	} else if (operator == ">="){
+	    oper_char = "ge";
+	} else if (operator == "<"){
+	    oper_char = "lt";
+   	} else if (operator == "<="){
+	    oper_char = "le";
+	}
+	return oper_char;
+    }
+    protected String join( String[] array, String delim ) {
+        String j = "";
+        for ( int i=0; i<array.length; i++ ) {
+            if (i!=0) j += delim;
+            j += array[i];
+        }
+        return j;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GATEServices.java.tmp
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GATEServices.java.tmp	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GATEServices.java.tmp	(revision 21112)
@@ -0,0 +1,291 @@
+/*
+ *    GATEServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// GATE classes
+import gate.*;
+import gate.creole.*;
+import gate.gui.*;
+import gate.util.persistence.PersistenceManager;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+
+/**
+ * <p>Title: GATE Greenstone3 integration</p>
+ * <p>Description: </p>
+ * <p>Copyright: Copyright (c) 2003</p>
+ * <p>Company: University of Waikato</p>
+ * @author unascribed
+ * @version 1.0
+ */
+
+public class GATEServices
+    extends ServiceRack {
+
+    // the services on offer
+    // these strings must match what is found in the properties file
+    protected static final String GATE_TAG_SERVICE = "GateTag";
+
+    protected static final String ANNOTATION_TYPE_PARAM = "annotationType";
+
+    protected Element config_info = null;
+
+    protected CorpusController application;
+
+    protected Corpus corpus;
+
+    protected String[] annotation_types = { "Date", "Location",
+					     "Organization", "Person" };
+    // Address, 
+
+    /** constructor */
+    public GATEServices()
+    {
+    }
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	System.out.println("Configuring GATEServices...");
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has name and type
+	Element tag_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tag_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_ENRICH);
+	tag_service.setAttribute(GSXML.NAME_ATT, GATE_TAG_SERVICE);
+	this.short_service_info.appendChild(tag_service);
+
+
+	// Configure GATE for use
+	try {
+	    Gate.init();
+	    Gate.getUserConfig().put(GateConstants.DOCUMENT_ADD_SPACE_ON_UNPACK_FEATURE_NAME,
+				     new Boolean(false));
+
+	    // MainFrame mainFrame = new MainFrame();
+	    // mainFrame.setSize(new java.awt.Dimension(800, 600));
+	    // mainFrame.setVisible(true);
+
+	    // Load the (pre-created) application
+	    URL applicationFileURL = ClassLoader.getSystemResource("gate.app");
+	    File applicationFile = new File(applicationFileURL.getFile());
+	    this.application = (CorpusController) PersistenceManager.loadObjectFromFile(applicationFile);
+
+	    /* Collection processing_resources = application.getPRs();
+	    Iterator pr_iterator = processing_resources.iterator();
+	    while (pr_iterator.hasNext()) {
+		ProcessingResource pr = (ProcessingResource) pr_iterator.next();
+		String pr_name = pr.getName();
+		System.out.println("PR name: " + pr_name);
+		if (pr_name.startsWith("ANNIE POS Tagger")) {
+		    String as_name = ((POSTagger) pr).getInputASName();
+		    System.out.println("AS name: " + as_name);
+		}
+		} */
+
+	    // Create a new corpus
+	    this.corpus = Factory.newCorpus("GSDL3 Corpus");
+	    this.application.setCorpus(this.corpus);
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	}
+
+	return true;
+    }
+
+    protected Element getServiceDescription(String service, String lang, String subset) {
+
+	if (!service.equals(GATE_TAG_SERVICE)) {
+	    return null;
+	}
+	Element tag_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tag_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_ENRICH);
+	tag_service.setAttribute(GSXML.NAME_ATT, GATE_TAG_SERVICE);
+	if (subset==null || subset.equals(GSXML.DISPLAY_TEXT_ELEM)) {
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service+".name", lang)));
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service+".description", lang)));
+	    tag_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service+".submit", lang)));
+	} 
+	if (subset==null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    createParameter(ANNOTATION_TYPE_PARAM, param_list, lang);
+	    tag_service.appendChild(param_list);
+	}
+	return tag_service;
+    }
+
+
+    /** creates a new param element and adds it to the param list */
+    protected void createParameter(String name, Element param_list, 
+				   String lang)
+    {
+	Element param = null;
+
+	if (name.equals(ANNOTATION_TYPE_PARAM)) {
+	    int len = this.annotation_types.length;
+	    String[] annotation_type_names = new String[len];
+	    for (int i = 0; i < len; i++) {
+		annotation_type_names[i] = getTextString("param." + name + "." + this.annotation_types[i], lang);
+	    }
+	    
+	    param = GSXML.createParameterDescription(this.doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_ENUM_MULTI, this.annotation_types[0], this.annotation_types, annotation_type_names);
+	    param_list.appendChild(param);
+	}
+	
+    }
+    
+    
+    protected Element processGateTag(Element request)
+    {
+	// System.out.println("(GateTag) Request:\n" + converter_.getPrettyString(request));
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, GATE_TAG_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("Error: GateTag request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	Set annTypes = new HashSet();
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the annotation types desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("annotationType")) {
+		String annotation_type = GSXML.getValue(param);
+		String [] types = annotation_type.split(",");
+		for (int i=0; i<types.length; i++) {
+		    annTypes.add(types[i]);
+		}
+	    }
+
+	    param = (Element) param.getNextSibling();
+	}
+
+	// Get the request content 
+      	Element doc_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_node_list == null) {
+	    System.err.println("Error: GateTag request specified no doc nodes.");
+	    return result;  // Return the empty result
+	}
+
+	// Process each document node in the list
+	NodeList doc_nodes = doc_node_list.getChildNodes();
+	for (int i = 0; i < doc_nodes.getLength(); i++) {
+	    Element doc_node = (Element) doc_nodes.item(i);
+	    Element content = (Element) GSXML.getChildByTagName(doc_node, "nodeContent");
+	    Node content_text = (Node) GSXML.getNodeTextNode(content);
+	    String text = content_text.getNodeValue();
+
+	    // GATE needs the text to be a valid HTML file
+	    text = "<html><head></head><body>" + text + "</body></html>";
+	    String annotated_text = processText(text, annTypes);
+
+	    // Remove the surrounding HTML tags
+	    annotated_text = annotated_text.substring(49, annotated_text.length() - 13);
+	    // System.out.println("GATE result:\n" + annotated_text);
+
+	    annotated_text = "<nodeContent>" + annotated_text + "</nodeContent>";
+	    Document annotated_content_doc = this.converter.getDOM(annotated_text);
+	    if (annotated_content_doc != null) {
+		Element annotated_content = annotated_content_doc.getDocumentElement();
+		doc_node.replaceChild(doc_node.getOwnerDocument().importNode(annotated_content, true), content);
+	    } else {
+		System.err.println("GATEServices.processGateTag Error: Couldn't parse annotated text for doc node "+i);
+	    }
+	    
+	}
+
+	result.appendChild(this.doc.importNode(doc_node_list, true));
+	// System.out.println("GateTag result:\n" + converter_.getPrettyString(result));
+	return result;
+    }
+
+
+    public String processText(String text, Set annotationTypesToExport)
+    {
+	try {
+	    // Create a new document containing the text
+	    FeatureMap parameters = Factory.newFeatureMap();
+	    parameters.put("stringContent", text);
+	    parameters.put("markupAware", new Boolean(true));
+	    parameters.put("preserveOriginalContent", new Boolean(true));
+	    parameters.put("collectRepositioningInfo", new Boolean(true));
+	    Document doc = (Document) Factory.createResource("gate.corpora.DocumentImpl",
+							     parameters);
+
+	    // Add it to the corpus
+	    this.corpus.clear();
+	    this.corpus.add(doc);
+
+	    // Process the corpus
+	    this.application.execute();
+
+	    // Extract all the annotations
+	    AnnotationSet annSet = doc.getAnnotations();
+
+	    // Return the desired annotations
+	    AnnotationSet outputAnnotations = annSet.get(annotationTypesToExport);
+	    String result = doc.toXml(outputAnnotations, false);
+	    Factory.deleteResource(doc);
+
+	    // 1. Escape the GATE result
+	    result = GSXML.xmlSafe(result);
+
+	    // 2. Unescape the annotation tags
+	    Iterator setIterator = annotationTypesToExport.iterator();
+	    while (setIterator.hasNext()) {
+		String annotationType = (String) setIterator.next();
+		result = result.replaceAll("&lt;" + annotationType + "&gt;",
+					   "<annotation type=\"" + annotationType + "\">");
+		result = result.replaceAll("&lt;/" + annotationType + "&gt;",
+					   "</annotation>");
+	    }
+
+	    return result;
+	}
+	catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Browse.java	(revision 21112)
@@ -0,0 +1,275 @@
+/*
+ *    GS2Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GDBMWrapper;
+import org.greenstone.gsdl3.util.DBInfo;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.util.Set;
+import java.util.Iterator;
+
+/** Greenstone 2 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public class GS2Browse 
+    extends AbstractBrowse 
+{
+    
+    protected GDBMWrapper gdbm_src = null;
+
+    public GS2Browse() 
+    {
+	this.gdbm_src = new GDBMWrapper();
+	this.macro_resolver = new GS2MacroResolver(this.gdbm_src);
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.gdbm_src.closeDatabase();
+    }
+
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.err.println("Configuring GS2Browse...");
+	// the index stem is either specified in the config file or is  the collection name
+	Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+	String index_stem = null;
+	if (index_stem_elem != null) {
+	    index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+	}
+	if (index_stem == null || index_stem.equals("")) {
+	    index_stem = this.cluster_name;
+	}
+	
+	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name, index_stem);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("GS2Browse Error: Could not open GDBM database!");
+	    return false;
+	}
+	return true;
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	return this.gdbm_src.translateOID(node_id);
+    }
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+
+    }
+
+    /** returns the id of the root node of the document containing node node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return OID.getTop(node_id);
+    }
+    /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+
+	ArrayList children = new ArrayList();
+	
+	String contains = info.getInfo("contains");
+	StringTokenizer st = new StringTokenizer(contains, ";");
+	while (st.hasMoreTokens()) {
+	    String child_id = st.nextToken().replaceAll("\"", node_id);
+	    children.add(child_id);
+	}
+	return children;
+
+    }
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, 
+				      ArrayList metadata_names) {
+	String lang = "en";
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return null;
+	}
+	if (all_metadata) {
+	    // return everything out of the database
+	    Set keys = info.getKeys();
+	    Iterator it = keys.iterator();
+	    while(it.hasNext()) {
+		String key = (String)it.next();
+		String value = info.getInfo(key);
+		GSXML.addMetadata(this.doc, metadata_list, key, this.macro_resolver.resolve(value, lang, GS2MacroResolver.SCOPE_META, node_id));
+	    }
+	    
+	} else {
+	    for (int i=0; i<metadata_names.size(); i++) {
+		String meta_name = (String) metadata_names.get(i);
+		String value = (String)info.getInfo(meta_name);
+		GSXML.addMetadata(this.doc, metadata_list, meta_name, value);
+	    }
+	}
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	String value="";
+	if (info_type.equals(INFO_NUM_SIBS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		value="0";
+	    } else {
+		value = String.valueOf(getNumChildren(parent_id));
+	    }
+	    return value;
+	}
+	
+	if (info_type.equals(INFO_NUM_CHILDREN)) {
+	    return String.valueOf(getNumChildren(doc_id));
+	}
+
+	
+	if (info_type.equals(INFO_SIB_POS)) {
+	    String parent_id = OID.getParent(doc_id);
+	    if (parent_id.equals(doc_id)) {
+		return "-1";
+	    }
+	    
+	    DBInfo info = this.gdbm_src.getInfo(parent_id);
+	    if (info==null) {
+		return "-1";
+	    }
+	    
+	    String contains = info.getInfo("contains");
+	    contains = contains.replaceAll("\"", parent_id);
+	    String [] children = contains.split(";");
+	    for (int i=0;i<children.length;i++) {
+		String child_id = children[i];
+		if (child_id.equals(doc_id)) {
+		    return String.valueOf(i+1); // make it from 1 to length
+			   
+		}
+	    }
+	    
+	    return "-1";
+	} else {
+	    return null;
+	}
+
+    }
+
+    protected int getNumChildren(String node_id) {
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return 0;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return 0;
+	}
+	String [] children = contains.split(";");
+	return children.length;
+    }
+    
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Construct.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Construct.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2Construct.java	(revision 21112)
@@ -0,0 +1,568 @@
+/*
+ *    GS2Construct.java 
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.build.*;
+
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.io.File;
+import java.util.Locale;
+
+/**
+ * A Services class for building collections
+ * provides a wrapper around the old perl scripts
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+public class GS2Construct 
+    extends ServiceRack { 
+    
+    // services offered
+    private static final String NEW_SERVICE = "NewCollection";
+    private static final String ADD_DOC_SERVICE = "AddDocument";
+    private static final String IMPORT_SERVICE = "ImportCollection";
+    private static final String BUILD_SERVICE = "BuildCollection";
+    private static final String ACTIVATE_SERVICE = "ActivateCollection";
+    private static final String DELETE_SERVICE = "DeleteCollection";
+    private static final String RELOAD_SERVICE = "ReloadCollection";
+
+    // params used
+    private static final String COL_PARAM = "collection";
+    private static final String NEW_COL_TITLE_PARAM = "collTitle";
+    private static final String NEW_COL_ABOUT_PARAM = "collAbout";
+    private static final String CREATOR_PARAM = "creator";
+    private static final String NEW_FILE_PARAM = "newfile";
+    private static final String PROCESS_ID_PARAM = GSParams.PROCESS_ID;
+    private static final String BUILDTYPE_PARAM = "buildType";
+    private static final String BUILDTYPE_MG = "mg";
+    private static final String BUILDTYPE_MGPP = "mgpp";
+    
+    // the list of the collections - store between some method calls
+    private String[] collection_list = null;
+
+    // set of listeners for any construction commands
+    protected Map listeners = null;
+    
+    public GS2Construct() {
+	this.listeners = Collections.synchronizedMap(new HashMap());
+	
+    }
+
+    /** returns a specific service description */
+    protected Element getServiceDescription(String service, String lang, String subset) {
+
+	Element description = this.doc.createElement(GSXML.SERVICE_ELEM);
+	description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	description.setAttribute(GSXML.NAME_ATT, service);
+	if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+	    description.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(service+".name", lang)));
+	    description.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(service+".description", lang)));
+	    description.appendChild(GSXML.createDisplayTextElement(this.doc,  GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service+".submit", lang)));
+	}
+	if (subset == null || subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    description.appendChild(param_list);
+	    
+	    if (service.equals(NEW_SERVICE)) {
+		
+		Element param = GSXML.createParameterDescription(this.doc, NEW_COL_TITLE_PARAM, getTextString("param."+NEW_COL_TITLE_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+		param_list.appendChild(param);
+		param = GSXML.createParameterDescription(this.doc, CREATOR_PARAM, getTextString("param."+CREATOR_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+		param_list.appendChild(param);
+		param = GSXML.createParameterDescription(this.doc, NEW_COL_ABOUT_PARAM, getTextString("param."+NEW_COL_ABOUT_PARAM, lang), GSXML.PARAM_TYPE_TEXT, null, null, null);
+		param_list.appendChild(param);
+		String[] types = {BUILDTYPE_MGPP, BUILDTYPE_MG};
+		String[] type_texts = {getTextString("param."+BUILDTYPE_PARAM+"."+BUILDTYPE_MGPP, lang), getTextString("param."+BUILDTYPE_PARAM+"."+BUILDTYPE_MG, lang)};
+		
+		param = GSXML.createParameterDescription(this.doc, BUILDTYPE_PARAM, getTextString("param."+BUILDTYPE_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, BUILDTYPE_MGPP, types, type_texts);
+		param_list.appendChild(param);
+	    }
+	    else if (service.equals(ACTIVATE_SERVICE) || service.equals(IMPORT_SERVICE) || service.equals(BUILD_SERVICE) || service.equals(RELOAD_SERVICE) || service.equals(DELETE_SERVICE)) {
+	    
+		this.collection_list = getCollectionList();
+		Element param = GSXML.createParameterDescription(this.doc, COL_PARAM, getTextString("param."+COL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, null, this.collection_list, this.collection_list);	
+		param_list.appendChild(param);
+	    } else {
+		// invalid service name
+		return null;
+	    }
+	}
+	return description;
+    }
+
+    // each service must have a method "process<New service name>"
+
+    protected Element processNewCollection(Element request) {
+	return runCommand(request, GS2PerlConstructor.NEW);	
+    }
+    
+    /** TODO:implement this */
+    protected Element processAddDocument(Element request) {
+	// decode the file name, add it to the import directory
+	String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, name);
+	Element status = this.doc.createElement(GSXML.STATUS_ELEM);
+	response.appendChild(status);
+	//String lang = request.getAttribute(GSXML.LANG_ATT);
+	//String request_type = request.getAttribute(GSXML.TYPE_ATT);
+	Text t = this.doc.createTextNode("AddDocument: not implemented yet");
+	status.appendChild(t);
+	status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	return response;
+    }
+    protected Element processImportCollection(Element request) {
+	return runCommand(request, GS2PerlConstructor.IMPORT);
+    }
+
+    protected Element processBuildCollection(Element request) {
+	return runCommand(request, GS2PerlConstructor.BUILD);
+    }
+    protected Element processActivateCollection(Element request) {
+	Element response = runCommand(request, GS2PerlConstructor.ACTIVATE);
+	// this activates the collection on disk. but now we need to tell
+	// the MR about it. but we have to wait until the process is finished.
+	Element status = (Element)GSXML.getChildByTagName(response, GSXML.STATUS_ELEM);
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_list, false);
+	String coll_name = (String) params.get(COL_PARAM);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	// check for finished
+	int status_code = Integer.parseInt(status.getAttribute(GSXML.STATUS_ERROR_CODE_ATT));
+	if (GSStatus.isCompleted(status_code) && 
+	    GSStatus.isError(status_code)) {
+	    // we shouldn't carry out the next bit, just return the response
+	    return response;
+	}
+	String id = status.getAttribute(GSXML.STATUS_PROCESS_ID_ATT);
+	GS2PerlListener listener = (GS2PerlListener)this.listeners.get(id);
+	if (listener==null) {
+	    System.err.println("somethings gone wrong, couldn't find the listener");
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    return response;
+	}
+	while (!GSStatus.isCompleted(status_code)) {
+	    // wait for the process, and keep checking the status code
+	    // there is probably a better way to do this.
+	    try {
+		Thread.currentThread().sleep(100);
+	    } catch (Exception e) { // ignore
+	    }
+	    status_code = listener.getStatus();
+	}
+	
+	// add the rest of the messages to the status node
+	Text t = this.doc.createTextNode("\n"+listener.getUpdate());
+	status.appendChild(t);
+	status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus()));
+	if (GSStatus.isError(status_code)) {
+	    return response; // without doing the next bit
+	}
+
+	t = this.doc.createTextNode("\n");
+	status.appendChild(t);
+	// once have got here, we assume
+	// the first bit proceeded successfully, now reload the collection
+	systemRequest("reload", coll_name, status, lang); // this will append more messages to the status, and overwrite the error code att
+	return response;
+	
+    }
+    protected Element processDeleteCollection(Element request) {
+
+	// the response to send back
+	String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, name);
+	Element status = this.doc.createElement(GSXML.STATUS_ELEM);
+	response.appendChild(status);
+	Text t = null; // the text node for the error/success message
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String request_type = request.getAttribute(GSXML.TYPE_ATT);
+	
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	boolean get_status_only = false;
+	if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) {
+	    get_status_only = true;
+	}
+	if (get_status_only) {
+	    // at the moment, delete is synchronous. but it may take ages so should do the command in another thread maybe? in which case we will want to ask for status
+	    System.err.println("had a status request for delete - this shouldn't happen!!");
+	    //t = this.doc.createTextNode("");
+	    //status.appendChild(t);
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    return response;
+	}
+	String coll_name = (String)params.get(COL_PARAM);	
+	String[] args={coll_name};
+	File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, coll_name));
+	// check that the coll is there in the first place
+	if (!coll_dir.exists()) {
+	    t = this.doc.createTextNode(getTextString("delete.exists_error", lang, args));
+	    status.appendChild(t);
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    return response;
+	}
+		
+	// try to delete the directory
+	if (!GSFile.deleteFile(coll_dir)) {
+	    t = this.doc.createTextNode(getTextString("delete.delete_error", lang, args));
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    status.appendChild(t);
+	    return response;
+	}
+	systemRequest("delete", coll_name, status, lang);
+	return response;
+    }
+
+    protected Element processReloadCollection(Element request) {
+
+	// the response to send back
+	String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, name);
+	Element status = this.doc.createElement(GSXML.STATUS_ELEM);
+	response.appendChild(status);
+	Text t = null; // the text node for the error/success message
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String request_type = request.getAttribute(GSXML.TYPE_ATT);
+	
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	boolean get_status_only = false;
+	if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) {
+	    get_status_only = true;
+	}
+	if (get_status_only) {
+	    // reload is synchronous - this makes no sense
+	    System.err.println("had a status request for reload - this shouldn't happen!!");
+	    //t = this.doc.createTextNode("");
+	    //status.appendChild(t);
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    return response;
+	}
+	
+	String coll_name = (String)params.get(COL_PARAM);
+	systemRequest("reload", coll_name, status, lang);
+	return response;	
+	
+    }
+
+    /** send a configure request to the message router 
+     * action name should be "delete" or "reload"
+     * response will be put into the status element
+     */
+    protected void systemRequest(String action_name, String coll_name, Element status, String lang) {
+	
+	// send the request to the MR
+	Element message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	Element request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_SYSTEM, "", lang, "");
+	message.appendChild(request);
+	Element command = this.doc.createElement(GSXML.SYSTEM_ELEM);
+	request.appendChild(command);
+	command.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, GSXML.COLLECTION_ELEM);
+	command.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, coll_name);
+
+	if (action_name.equals("delete")) {
+	    command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_DEACTIVATE);
+	} else if (action_name.equals("reload")) {
+	    command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ACTIVATE);
+	} else {
+	    System.err.println("invalid action name passed to systemRequest:"+action_name);
+	    return;
+	}
+	request.appendChild(command);
+	Element response = this.router.process(message); // at the moment, get no info in response so ignore it
+	Text t;
+	String[] args={coll_name};
+	    
+	if (response == null) {
+	    t = this.doc.createTextNode(getTextString(action_name+".configure_error", lang, args));
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    status.appendChild(t);
+	    return;
+	}
+	    
+	// if we got here, we have succeeded!
+	t = this.doc.createTextNode(getTextString(action_name+".success", lang, args));
+	status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.SUCCESS));
+	status.appendChild(t);
+    }
+
+
+    /** configure the service module
+     * for now, all services have type=build - need to think about this
+     */
+    public boolean configure(Element info, Element extra_info) {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("configuring GS2Construct");
+
+	Element e = null;
+	// hard code in the services for now
+
+	// set up short_service_info_ - for now just has name and type
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, NEW_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, IMPORT_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, BUILD_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, ACTIVATE_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, DELETE_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	e.setAttribute(GSXML.NAME_ATT, RELOAD_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	//e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	//e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+	//e.setAttribute(GSXML.NAME_ATT, ADD_DOC_SERVICE);
+	//this.short_service_info.appendChild(e);
+
+
+	return true;
+    }
+
+    /** returns a response element */
+    protected Element runCommand(Element request, int type ) {
+	
+	// the response to send back
+	String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, name);
+	Element status = this.doc.createElement(GSXML.STATUS_ELEM);
+	response.appendChild(status);
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String request_type = request.getAttribute(GSXML.TYPE_ATT);
+
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	boolean get_status_only = false;
+	if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) {
+	    get_status_only = true;
+	}
+	
+	// just check for status messages if that's all that's required
+	if (get_status_only) {
+	    String id = (String)params.get(PROCESS_ID_PARAM);
+	    status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id);
+	    GS2PerlListener listener = (GS2PerlListener)this.listeners.get(id);
+	    if (listener==null) {
+		Text t = this.doc.createTextNode(getTextString("general.process_id_error", lang));
+		status.appendChild(t);
+		status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    } else {
+		Text t = this.doc.createTextNode(listener.getUpdate());
+		status.appendChild(t);
+		status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus()));
+		// check that we actually should be removing the listener here
+		if (listener.isFinished()) { // remove this listener - its job is done
+		    this.listeners.remove(id); // not working
+		}
+	    }
+	    return response;
+
+	}
+
+	// do teh actual command
+	String coll_name=null;
+	if (type==GS2PerlConstructor.NEW) {
+	    String coll_title = (String)params.get(NEW_COL_TITLE_PARAM);
+	    coll_name = createNewCollName(coll_title);
+	} else {
+	    coll_name = (String)params.get(COL_PARAM);
+	}
+	System.err.println("Coll name = "+coll_name);
+	// makes a paramList of the relevant params
+	Element other_params = extractOtherParams(params, type);
+
+	//create the constructor to do the work
+
+	GS2PerlConstructor constructor = new GS2PerlConstructor("perl_build");
+	if (!constructor.configure()) {
+	    Text t = this.doc.createTextNode(getTextString("general.configure_constructor_error", lang));
+	    status.appendChild(t);
+	    status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+	    return response;
+	}
+		
+	constructor.setSiteHome(this.site_home);
+	constructor.setCollectionName(coll_name);
+	constructor.setActionType(type);
+	constructor.setProcessParams(other_params);
+	// the listener
+	GS2PerlListener listener = new GS2PerlListener();
+	constructor.addListener(listener);
+	constructor.start();
+
+	String id = newID();
+	this.listeners.put(id, listener);
+
+	status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id);
+	status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ACCEPTED));
+	Text t = this.doc.createTextNode(getTextString("general.process_start", lang));
+	status.appendChild(t);
+	return response;
+
+    }
+
+
+    //************************
+    // some helper functions
+    //************************
+
+    /** parse the collect directory and return a list of collection names */
+    protected String[] getCollectionList() {
+	
+	File collectDir = new File(GSFile.collectDir(this.site_home));
+	if (!collectDir.exists()) {
+	    System.err.println("couldn't find collect dir: "+collectDir.toString());
+	    return null;
+	}
+	System.out.println("GS2Construct: reading thru directory "+collectDir.getPath()+" to find collections.");
+	File[] contents = collectDir.listFiles();
+	int num_colls=0;
+	for (int i=0; i<contents.length;i++) {
+	    if(contents[i].isDirectory() && !contents[i].getName().startsWith("CVS")) {
+		num_colls ++;
+	    }
+	}
+	
+	String[] names= new String[num_colls];
+	
+	for (int i=0, j=0; i<contents.length;i++) {
+	    if(contents[i].isDirectory()) {
+		String colName = contents[i].getName();
+		if (!colName.startsWith("CVS")) {
+		    names[j] = colName;
+		    j++;
+		}
+		
+	    } 
+	}
+	
+	return names;
+	
+    }
+   
+    /** ids used for process id */
+    private int current_id = 0;
+    private String newID() {
+	current_id++;
+	return Integer.toString(current_id);
+    }
+
+    /** creates a new short name from the collection title */
+    protected String createNewCollName(String coll_title) {
+
+	String base_name = null;
+	// take the first 6 letters
+	if (coll_title.length()<6) {
+	    base_name = coll_title;
+	} else {
+	    base_name = coll_title.substring(0,6);
+	}
+	File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name));
+	if (!coll_dir.exists()) { // this name is ok - not used yet
+	    return base_name;
+	}
+	
+	// now we have to make a new name until we get a good one
+	// try name1, name2 name3 etc
+	int i=0;
+	while(coll_dir.exists()) {
+	    i++;
+	    coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name+Integer.toString(i)));
+	}
+	return base_name+Integer.toString(i);
+	
+    }
+
+    /** takes the params from the request (in the HashMap) and extracts any
+     * that need to be passed to the constructor and puts them into a 
+     * paramList element */
+    protected Element extractOtherParams(HashMap params, int type) {
+
+	Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (type == GS2PerlConstructor.NEW) {
+	    Element param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param.setAttribute(GSXML.NAME_ATT, "creator");
+	    param.setAttribute(GSXML.VALUE_ATT, (String)params.get(CREATOR_PARAM));
+	    
+	    param_list.appendChild(param);
+	    param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param.setAttribute(GSXML.NAME_ATT, "about");
+	    param.setAttribute(GSXML.VALUE_ATT, (String)params.get(NEW_COL_ABOUT_PARAM));
+	    param_list.appendChild(param);
+	    param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param.setAttribute(GSXML.NAME_ATT, "title");
+	    param.setAttribute(GSXML.VALUE_ATT, (String)params.get(NEW_COL_TITLE_PARAM));
+	    param_list.appendChild(param);
+	    param = this.doc.createElement(GSXML.PARAM_ELEM);
+	    param.setAttribute(GSXML.NAME_ATT, "buildtype");
+	    param.setAttribute(GSXML.VALUE_ATT, (String)params.get(BUILDTYPE_PARAM));
+	    param_list.appendChild(param);
+	    return param_list;
+	}
+
+	// other ones dont have params yet
+	return null;
+    }
+	
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 21112)
@@ -0,0 +1,120 @@
+/*
+ *    GS2MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS2MGPPRetrieve
+    extends AbstractGS2DocumentRetrieve
+{
+     // Parameters used
+    private static final String LEVEL_PARAM = "level";
+
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+    private MGPPWrapper mgpp_src = null;
+    
+    private String default_level = null;
+    private String mgpp_textdir = null;
+	
+    public GS2MGPPRetrieve() {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS2MGPPRetrieve...");
+
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.index_stem);
+
+	// Do generic configuration
+	return true;
+ 
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+	String lang = "en"; //  ********** 
+	long doc_num = this.gdbm_src.OID2Docnum(doc_id);
+	if (doc_num == -1) {
+	    System.err.println("OID "+doc_id +" couldn't be converted to mgpp num");
+	    return null;
+	}
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_num);
+	    if (doc_content != null) {
+		doc_content = resolveTextMacros(doc_content, doc_id, lang);
+	    } 
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_num+"\n";
+
+	}
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGPPSearch.java	(revision 21112)
@@ -0,0 +1,139 @@
+/*
+ *    GS2MGPPSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public class GS2MGPPSearch 
+    extends AbstractMGPPSearch 
+{
+    protected GDBMWrapper gdbm_src = null;
+
+    /** constructor */
+    public GS2MGPPSearch()
+    {
+	this.gdbm_src = new GDBMWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.gdbm_src.closeDatabase();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+    	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name, this.index_stem);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("Error: Could not open GDBM database!");
+	    return false;
+	}
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return false;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return false;
+	}
+	return true;
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MGPP internal id to Greenstone oid */
+    protected String MGPPNum2OID(long docnum)
+    {
+	return this.gdbm_src.docnum2OID(docnum);
+	
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 21112)
@@ -0,0 +1,120 @@
+/*
+ *    GS2MGRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS2MGRetrieve
+    extends AbstractGS2DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+
+    private MGWrapper mg_src = null;
+    private String mg_basedir = null;
+    private String mg_textdir = null;
+    private String default_index = null;
+	
+    public GS2MGRetrieve() {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+	
+	// Do specific configuration
+	System.out.println("Configuring GS2MGRetrieve...");
+	// System.out.println("info:\n" + converter_.getString(info));
+	// System.out.println("extra_info:\n" + converter_.getString(extra_info));
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_index == null || this.default_index.equals("")) {
+	    System.err.println("Error: default index not specified!");
+	    return false;
+	}
+	// System.out.println("Default index: " + this.default_index);
+
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath(this.index_stem);
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mg_src.setIndex(indexpath);
+
+	return true;
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+	String lang = "en"; //  ********** 
+	long doc_num = this.gdbm_src.OID2Docnum(doc_id);
+	if (doc_num == -1) {
+	    System.err.println("OID "+doc_id +" couldn't be converted to mg num");
+	    return null;
+	}
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = this.mg_src.getDocument(this.mg_basedir,
+						     this.mg_textdir, doc_num);
+	
+	if (doc_content!=null) {
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    // replace _httpimg_ with the correct address
+	    doc_content = resolveTextMacros(doc_content, doc_id, lang);
+	    //GSXML.addDocText(this.doc, doc, doc_content);
+	} else {
+	    System.err.println("the doc content was null, not getting that section\n");
+	    doc_content = "couldn't retrieve content for this section\n";
+	}
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS2MGSearch.java	(revision 21112)
@@ -0,0 +1,138 @@
+/*
+ *    GS2MGSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public class GS2MGSearch 
+    extends AbstractMGSearch 
+{
+    protected GDBMWrapper gdbm_src = null;
+
+    /** constructor */
+    public GS2MGSearch()
+    {
+	this.gdbm_src = new GDBMWrapper();
+    }
+    public void cleanUp() {
+	super.cleanUp();
+	this.gdbm_src.closeDatabase();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+    	// Open GDBM database for querying
+	String gdbm_db_file = GSFile.GDBMDatabaseFile(this.site_home, this.cluster_name, this.index_stem);
+	if (!this.gdbm_src.openDatabase(gdbm_db_file, GDBMWrapper.READER)) {
+	    System.err.println("Error: Could not open GDBM database!");
+	    return false;
+	}
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.gdbm_src.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	DBInfo info = this.gdbm_src.getInfo(node_id);
+	if (info == null) {
+	    return false;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return false;
+	}
+	return true;
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MG internal id to Greenstone oid */
+    protected String MGNum2OID(long docnum)
+    {
+	return this.gdbm_src.docnum2OID(docnum);
+	
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3Browse.java	(revision 21112)
@@ -0,0 +1,193 @@
+/*
+ *    GS3Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+import java.io.File;
+
+/** Greenstone 3 collection classifier service
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+public class GS3Browse
+    extends AbstractBrowse
+{
+
+    protected SQLQuery database = null;
+
+    /** constructor */
+    public GS3Browse()
+    {
+	this.database = new SQLQuery();
+
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+    
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.err.println("Configuring GS3Browse...");
+	
+	// open the database for querying
+	// the database name is a combination of site name and collection name
+
+	// check that site_home is set
+	if (this.site_home == null || this.site_home.equals("")) {
+	    System.err.println("GS3Retrieve Error: site_home is not set, so cannot work out the site name and cannot determine the database name");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Retrieve Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Retrieve Error: Could not open SQL database!");
+	    return false;
+	}
+	
+	return true;
+    }
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id */
+    protected String translateId(String node_id) {
+	// should really use a sql query
+	return GS3OID.translateOID(node_id);
+    }
+
+        /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id) {
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+   }
+    
+    /** returns the id of the root node of the document containing node 
+     * node_id. . may be the same as node_id */
+    protected String getRootId(String node_id) {
+	return GS3OID.getDocID(node_id);
+    }
+    
+   /** returns a list of the child ids in order, null if no children */
+    protected ArrayList getChildrenIds(String node_id) {
+	ArrayList documents = database.getClassifierDocChildren(node_id);
+	if (documents == null) {
+	    documents = new ArrayList();
+	}
+	ArrayList children = database.getClassifierChildren(node_id);
+	if (children != null) {
+	    documents.addAll(children);
+	}
+	if (documents.size()==0) {
+	    return null;
+	}
+	return documents;
+    }
+
+    /** returns the node id of the parent node, null if no parent */
+    protected String getParentId(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return null;
+	}
+	return parent;
+    }
+    
+    /** get the metadata for the classifier node node_id
+     * returns a metadataList element:
+     * <metadataList><metadata name="xxx">value</metadata></metadataList>
+     * if all_metadata is true, returns all available metadata, otherwise just
+     * returns requested metadata
+     */
+    // assumes only one value per metadata
+    protected Element getMetadataList(String node_id, boolean all_metadata, ArrayList metadata_names) {
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	if (all_metadata) {
+	    // TODO
+	    
+
+	} else {
+	    for (int m = 0; m < metadata_names.size(); m++) {
+		String metadata = (String) metadata_names.get(m);
+		String value = null;
+		value = database.getClassifierMetadata(node_id, metadata);
+		if (value != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, metadata, value);
+		}
+	    }
+	}
+
+	return metadata_list;
+    }
+
+    /** returns the structural information asked for.
+     * info_type may be one of
+     * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+     */
+    protected String getStructureInfo(String doc_id, String info_type) {
+	return null;
+    }
+
+    /** returns true if the id refers to a document (rather than 
+     * a classifier node)
+     */
+    protected boolean isDocumentId(String node_id){
+	if (node_id.startsWith("CL")) {
+	    return false;
+	}
+	return true;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPRetrieve.java	(revision 21112)
@@ -0,0 +1,121 @@
+/*
+ *    GS3MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGPPRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+    private MGPPWrapper mgpp_src = null;
+    
+    private String default_level = null;
+    private String mgpp_textdir = null;
+	
+    public GS3MGPPRetrieve() {
+	this.mgpp_src = new MGPPWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGPPRetrieve...");
+
+	// Get the default level out of <defaultLevel> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+	if (def != null) {
+	    this.default_level = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_level == null || this.default_level.equals("")) {
+	    System.err.println("Error: default level not specified!");
+	    return false;
+	}
+	// System.out.println("Default level: " + default_level_);
+	
+	// The location of the MGPP text files
+	mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+	    File.separatorChar + GSFile.collectionTextPath(this.index_stem);
+
+	return true;
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mgpp_src.getDocument(this.mgpp_textdir, 
+						    this.default_level,
+						    doc_int);
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+	} catch (Exception e) {
+	    System.out.println("exception happended with mgpp_src.getDocument()" + e);
+	    doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_int+"\n";
+	}
+
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGPPSearch.java	(revision 21112)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGPPSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGPPSearch 
+    extends AbstractMGPPSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGPPSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MGPP internal id to Greenstone oid */
+    protected String MGPPNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGRetrieve.java	(revision 21112)
@@ -0,0 +1,133 @@
+/*
+ *    GS3MGRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+public class GS3MGRetrieve
+    extends AbstractGS3DocumentRetrieve
+{
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+
+    private MGWrapper mg_src = null;
+    private String mg_basedir = null;
+    private String mg_textdir = null;
+    private String default_index = null;
+    
+    public GS3MGRetrieve() {
+	this.mg_src = new MGWrapper();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mg_src.unloadIndexData();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	// Do generic configuration
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+
+	// Do specific configuration
+	System.out.println("Configuring GS3MGRetrieve...");
+	// System.out.println("info:\n" + converter_.getString(info));
+	// System.out.println("extra_info:\n" + converter_.getString(extra_info));
+
+	// Get the default index out of <defaultIndex> (buildConfig.xml)
+	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+	if (def != null) {
+	    this.default_index = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.default_index == null || this.default_index.equals("")) {
+	    System.err.println("Error: default index not specified!");
+	    return false;
+	}
+
+	// System.out.println("Default index: " + this.default_index);
+
+	
+	// The location of the MG index and text files
+	mg_basedir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+	mg_textdir = GSFile.collectionTextPath(this.index_stem);
+	// index is only needed to start up MG, not used so just use the default index
+	String indexpath = GSFile.collectionIndexPath(this.index_stem, this.default_index);
+	this.mg_src.setIndex(indexpath);
+	return true;
+    }
+    
+    /** returns the content of a node.
+     * node_id should already have been translated if necessary
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+    protected Element getNodeContent(String doc_id) throws GSException {
+
+	if (GS3OID.isDocTop(doc_id) && database.isHierarchicalDocument(doc_id)) {
+	    // if we have a whole doc id, and the document is hierarchical,
+	    // we want to change the id to be the top id of the section 
+	    // hierarchy
+	    doc_id = GS3OID.createOID(doc_id, "1");
+	}
+
+	String doc_num = this.database.OID2MGNum(doc_id);	    
+	// doc nums have the index prefixed 
+	doc_num = doc_num.substring(doc_num.indexOf(".")+1);
+	int doc_int = Integer.parseInt(doc_num);
+	
+	String doc_content = "";
+	try {
+	    doc_content = this.mg_src.getDocument(this.mg_basedir, 
+						  this.mg_textdir, 
+						  doc_int);
+	    // remove any ctrl-c or ctrl-b 
+	    doc_content = doc_content.replaceAll("\u0002|\u0003", "");
+	    doc_content = resolveRelativeLinks(doc_content, doc_id);
+
+	} catch (Exception e) {
+	    System.out.println("exception happended with mg_src.getDocument()");
+	    throw new GSException("Couldn't get document content for id: "+ doc_int+"\n"+e.getMessage());
+	    //doc_content = "this is the content for section hash id "+ doc_id+", mg doc num "+doc_int+"\n";
+	}
+	
+	Element content_node = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	
+	Text t = this.doc.createTextNode(doc_content);
+	content_node.appendChild(t);
+	return content_node;
+
+    }
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GS3MGSearch.java	(revision 21112)
@@ -0,0 +1,141 @@
+/*
+ *    GS3MGSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GS3OID;
+import org.greenstone.gsdl3.util.SQLQuery;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java classes
+import java.io.File;
+
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ */
+
+public class GS3MGSearch 
+    extends AbstractMGSearch 
+{
+    protected SQLQuery database = null;
+    /** the base index prefix */
+    protected String base_index_prefix = null;
+
+
+    /** constructor */
+    public GS3MGSearch()
+    {
+	this.database = new SQLQuery();
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.database.closeConnection();
+    }
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// get the base prefix
+	Element def = (Element) GSXML.getChildByTagName(info, "baseIndexPrefix");
+	if (def != null) {
+	    this.base_index_prefix = def.getAttribute(GSXML.NAME_ATT);
+	}
+	if (this.base_index_prefix == null || this.base_index_prefix.equals("")) {
+	    System.err.println("Error: base index prefix not specified!");
+	    return false;
+	}
+	String site_name = this.site_home.substring(this.site_home.lastIndexOf(File.separator)+1);
+	if (site_name.equals("")) {
+	    System.err.println("GS3Search Error: Cannot extract the site name from site home: "+this.site_home);
+	    return false;
+	}
+	if (!database.setDatabase(site_name+"_"+this.cluster_name)) {
+	    System.err.println("GS3Search Error: Could not open SQL database!");
+	    return false;
+	}
+
+	return true;
+    }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    protected String getDocType(String node_id){
+	boolean hierarchical = false;
+	if (!GS3OID.isDocTop(node_id) || database.isHierarchicalDocument(node_id) ) {
+	    hierarchical = true;
+	} 
+	// what about paged???
+	if (!hierarchical) {
+	    // a simple document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	} else {
+	    // a hierarchical doc
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+	// don't have paged yet.
+
+    }
+    
+    /** returns true if the node has child nodes */
+    protected boolean hasChildren(String node_id){
+	return database.documentHasChildren(node_id);
+    }
+    
+    /** returns true if the node has a parent */
+    protected boolean hasParent(String node_id){
+	String parent = GS3OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+    
+   /** convert MG internal id to Greenstone oid */
+    protected String MGNum2OID(long docnum)
+    {
+	String mg_id = this.base_index_prefix+"."+docnum;
+	String doc_id = database.MGNum2OID(mg_id);
+	// hack !!!
+	if (doc_id.endsWith("-All")) {
+	    // get rid of the All bit
+	    doc_id = doc_id.substring(0,doc_id.length()-4);
+	    //doc_id=doc_id.replaceAll("All","1");
+	}
+	return doc_id;
+	
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GoogleSearch.java.tmp
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GoogleSearch.java.tmp	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GoogleSearch.java.tmp	(revision 21112)
@@ -0,0 +1,228 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+//Google Web Services API classes
+//import com.google.soap.search.GoogleSearch;
+import com.google.soap.search.GoogleSearchFault;
+import com.google.soap.search.GoogleSearchResult;
+import com.google.soap.search.GoogleSearchResultElement;
+import com.google.soap.search.GoogleSearchDirectoryCategory;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.MalformedURLException;
+import java.lang.Object;
+
+
+/**
+ *
+ * @author <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
+ * @version $Revision$
+ *
+ */
+
+public class GoogleSearch
+    extends AbstractSearch {
+    
+    //Parameters connect to Proxy Server
+    private boolean using_proxy = false;
+    private String proxy_host = null;
+    private int proxy_port;
+    private String proxy_user = null;
+    private char [] proxy_passwd = null;
+
+    // google key
+    private String client_key = null;
+    
+    public GoogleSearch()
+    {
+    }
+	    
+    //Configure GoogleSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "googleServer");
+	if (server_elem == null) {
+	    System.err.println("GoogleSearch.configure error: no googleServer element found");
+	    return false;
+	}
+	
+	client_key = server_elem.getAttribute("key");
+	if (client_key.equals("")) {
+	    System.err.println("GoogleSearch.configure error: no client_key for the googleServer element");
+	    return false;
+	}
+	
+	does_paging = true;
+	does_chunking = false;
+
+	// are we behind a proxy??
+	// all the details should have been set up by the Message Router
+	proxy_host = System.getProperty("http.proxyHost");
+	if (proxy_host != null && !proxy_host.equals("")) {
+	    using_proxy = true;
+	    try {
+		proxy_port = Integer.parseInt(System.getProperty("http.proxyPort").trim());
+	    } catch (Exception e) {
+		System.err.println("GoogleSearch.configure error: couldn't get proxy port, defaulting to 80");
+		proxy_port = 80;
+	    }
+	    PasswordAuthentication pa = Authenticator.requestPasswordAuthentication(proxy_host, null, proxy_port, "http", "", null);
+	    proxy_user = pa.getUserName();
+	    proxy_passwd = pa.getPassword();
+	}
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+	
+	//Connect to Google Web API service
+	com.google.soap.search.GoogleSearch search = new com.google.soap.search.GoogleSearch();
+
+	// Set mandatory attributes
+	search.setKey(client_key);
+
+	// proxy??
+	if (using_proxy) {
+	    search.setProxyHost(proxy_host);
+	    search.setProxyPort(proxy_port);
+	    search.setProxyUserName(proxy_user);
+	    search.setProxyPassword(new String(proxy_passwd));
+	}
+
+	//set optional attributes
+	search.setSafeSearch(true);
+	
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	Element query_metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(query_metadata_list);
+
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("GoogleSearch.processTextQuery Error:: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    System.err.println("GoogleSearch.processTextQuery Error: TextQuery request had no query string.");
+	    return result;  // Return the empty result
+	}
+       	// tidy whitespace
+	query = query.replaceAll("\\s+", "+");
+	
+	search.setQueryString(query);
+
+	//Check hits_per_page
+	int hits_per_page;
+	try {
+	    hits_per_page = Integer.parseInt(((String)params.get(HITS_PER_PAGE_PARAM)).trim());
+	} catch (Exception e) {
+	    System.err.println("GoogleSearch.processTextQuery error: couldn't get hits per page param, defaulting to 10");
+	    hits_per_page = 10;
+	}
+	//Check the start_page number
+	int start_page;
+	try {
+	    start_page = Integer.parseInt(((String) params.get(START_PAGE_PARAM)).trim());
+	} catch (Exception e) {
+	    System.err.println("GoogleSearch.processTextQuery error: couldn't get start page param, defaulting to 1");
+	    start_page = 1;
+	}
+	
+	//Invoke Actual Search
+	
+	// Google only allows 10 hits per request
+	int loop = hits_per_page/10;
+	int remainder = hits_per_page%10;
+	int google_start_page = (start_page-1)*hits_per_page;
+	int pages_per_loop;
+	for (int j=0; j < loop; j++){
+	    if (j < (loop-1) || remainder == 0) {
+		pages_per_loop = 10;
+	    } else {
+		pages_per_loop = remainder;
+	    }
+	    search.setMaxResults(pages_per_loop); 
+	    search.setStartResult(google_start_page);
+	    google_start_page = google_start_page + pages_per_loop;
+	    GoogleSearchResult google_result;
+	    try{
+		google_result = search.doSearch();
+	    } catch (GoogleSearchFault ex) {
+		System.err.println("GoogleSearch.processTextQuery error: the call to the Google Web APIs failed:" + ex.toString());
+		// add the error to the result
+		return result;
+	    }
+	    if (j==0) {
+		
+		//Total amount of documents Google Search returned
+		// only need to do this on the first loop
+		
+		long numdocs_matched = google_result.getEstimatedTotalResultsCount();	    
+		GSXML.addMetadata(this.doc, query_metadata_list, "numDocsMatched", ""+numdocs_matched);
+		
+	    }
+	    GoogleSearchResultElement[] details = google_result.getResultElements();
+	    for (int i=0; i<details.length; i++){
+		Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		doc_node_list.appendChild(doc_node);
+		
+		Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		doc_node.appendChild(metadata_list);
+		
+		String google_url = details[i].getURL();  
+		String google_title = details[i].getTitle();
+		String google_snippet = details[i].getSnippet();
+		if (google_url !=null) {
+		    GSXML.addMetadata(this.doc, metadata_list, "URL", google_url);
+		}
+		if (google_title != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, "Title", google_title);
+		}
+		if (google_snippet != null) {
+		    GSXML.addMetadata(this.doc, metadata_list, "Snippet", google_snippet);
+		}
+	    }
+	} // for each loop
+	return result;
+    }
+    
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
+       	index_ids.add("idx");
+	index_names.add("Google main index");
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GutBookRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GutBookRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/GutBookRetrieve.java	(revision 21112)
@@ -0,0 +1,41 @@
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+
+public class GutBookRetrieve 
+    extends XMLRetrieve {
+
+    protected Element extractTitleMeta(Element section) {
+	Element meta_elem = this.doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+
+	Element title_elem = (Element)GSXML.getChildByTagName(section, "title");
+	if (title_elem == null) {
+	    title_elem = (Element)GSXML.getNodeByPath(section, "chapheader/title");
+	}
+	if (title_elem == null) {
+	    // find teh first title elem
+	    NodeList titles = section.getElementsByTagName("title");
+	    if (titles != null) {
+		title_elem = (Element)titles.item(0);
+	    }
+	}
+
+	if (title_elem == null) {
+	    return null; // no title found
+	}
+	
+	
+	Text t = (Text) GSXML.getNodeTextNode(title_elem);
+	
+	meta_elem.appendChild(this.doc.importNode(t, false));
+	return meta_elem;
+
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaProxy.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaProxy.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaProxy.java	(revision 21112)
@@ -0,0 +1,504 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.MalformedURLException;
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+
+public class IViaProxy 
+    extends ServiceRack {
+    
+    // the services on offer
+    // these strings must match what is found in the properties file
+    protected static final String TEXT_QUERY_SERVICE = "TextQuery";
+    protected static final String DOC_CONTENT_SERVICE = "DocumentContentRetrieve";
+    protected static final String DOC_META_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String QUERY_PARAM = "query";
+    protected static final String FIELD_PARAM = "fields";
+    // have standard gs param names for hits per page, and start page
+    // these need to be mapped to iVia params
+    protected static final String GS_HITS_PARAM = "hitsPerPage";
+    protected static final String IM_HITS_PARAM = "no_of_records_per_page";
+    protected static final String GS_START_PAGE_PARAM = "startPage";
+    protected static final String IM_START_PAGE_PARAM = "start_page_no";
+
+    protected String ivia_server_url = null;
+    
+    public boolean configure(Element info, Element extra_info) {
+	
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "iViaServer");
+	if (server_elem == null) {
+	    System.err.println("IViaProxy.configure error: no iViaServer element found");
+	    return false;
+	}
+	ivia_server_url = server_elem.getAttribute("url");
+	if (ivia_server_url.equals("")) {
+	    System.err.println("IViaProxy.configure error: no url for the iViaServer element");
+	    return false;
+	}
+	Element tq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	this.short_service_info.appendChild(tq_service);
+
+	Element dc_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	dc_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dc_service.setAttribute(GSXML.NAME_ATT, DOC_CONTENT_SERVICE);
+	this.short_service_info.appendChild(dc_service);
+
+	Element dm_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	dm_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dm_service.setAttribute(GSXML.NAME_ATT, DOC_META_SERVICE);
+	this.short_service_info.appendChild(dm_service);
+	
+	// 
+	// add some format info to service map if there is any
+	String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+	Element format = (Element) GSXML.getNodeByPath(extra_info, path);
+	if (format != null) {
+	    this.format_info_map.put(TEXT_QUERY_SERVICE, this.doc.importNode(format, true));
+	}
+
+	
+	// look for document display format
+	path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    this.format_info_map.put(DOC_CONTENT_SERVICE, this.doc.importNode(display_format, true));
+	    // shoudl we make a copy?
+	}
+
+	return true;
+	
+    }
+    
+    protected Element getServiceDescription(String service, String lang, String subset) {
+	
+	if (service.equals(TEXT_QUERY_SERVICE)) {
+	    Element tq_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+	    if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER)) {
+		tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME, getTextString(TEXT_QUERY_SERVICE+".name", lang)));
+		tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(TEXT_QUERY_SERVICE+".submit", lang)));
+		tq_service.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(TEXT_QUERY_SERVICE+".description", lang)));
+	    } 
+	    if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER)) {
+		Element param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		tq_service.appendChild(param_list);
+		Element param = GSXML.createParameterDescription(this.doc, QUERY_PARAM, getTextString("param."+QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+		param_list.appendChild(param);
+		String [] field_ids = {"kw", "au", "su", "ti", "de", "fu"};
+		String [] field_names = {
+		    getTextString("param."+FIELD_PARAM+".kw", lang), 
+		    getTextString("param."+FIELD_PARAM+".au", lang), 
+		    getTextString("param."+FIELD_PARAM+".su", lang), 
+		    getTextString("param."+FIELD_PARAM+".ti", lang), 
+		    getTextString("param."+FIELD_PARAM+".de", lang), 
+		    getTextString("param."+FIELD_PARAM+".fu", lang) };
+		
+		param = GSXML.createParameterDescription(this.doc, FIELD_PARAM, getTextString("param."+FIELD_PARAM, lang), GSXML.PARAM_TYPE_ENUM_MULTI, "kw,au,su,ti,de,fu", field_ids, field_names);
+		param_list.appendChild(param);
+		
+		
+		String [] hits_options = {"10", "30", "50"};
+		param = GSXML.createParameterDescription(this.doc, GS_HITS_PARAM, getTextString("param."+GS_HITS_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, "10", hits_options, hits_options);
+		param_list.appendChild(param);
+		
+		param = GSXML.createParameterDescription(this.doc, GS_START_PAGE_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, "1", null, null);
+		param_list.appendChild(param);
+	    }
+	    return tq_service;
+	} 
+	if (service.equals(DOC_META_SERVICE)) {
+	    Element dm_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    dm_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    dm_service.setAttribute(GSXML.NAME_ATT, DOC_META_SERVICE);
+	    return dm_service;
+
+	}
+	if (service.equals(DOC_CONTENT_SERVICE)) {
+	    Element dc_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    dc_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    dc_service.setAttribute(GSXML.NAME_ATT, DOC_CONTENT_SERVICE);
+	    return dc_service;
+	    
+	    
+	}
+	return null;
+    }
+   
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+       
+
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("IViaProxy Error:: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+	// tidy whitespace
+	query = query.replaceAll("\\s+", "+");
+	String url_string = ivia_server_url+"/cgi-bin/canned_search?theme=gsdl3&query="+query; 
+
+	// check for fields
+	String fields = (String) params.get(FIELD_PARAM);
+	if (fields != null && !fields.equals("")) {
+	    url_string += "&fields="+fields;
+	}
+
+	//check for hits per page
+	String hits_per_page = (String) params.get(GS_HITS_PARAM);
+	if (hits_per_page != null && !hits_per_page.equals("")) {
+	    url_string += "&"+IM_HITS_PARAM+"="+hits_per_page;
+	}
+
+	// check for start page
+	String start_page = (String) params.get(GS_START_PAGE_PARAM);
+	if (start_page != null && !start_page.equals("")) {
+	    url_string += "&"+IM_START_PAGE_PARAM+"="+start_page;
+	}
+	String results_num = null;
+	String doc_ids = null;
+	try {
+	    ///ystem.err.println("IViaProxy, sending "+url_string);
+	    BufferedReader reader = makeConnection(url_string);
+	    results_num = reader.readLine();
+	    doc_ids = reader.readLine();
+	    
+	} catch (Exception e) {
+	    System.err.println("IViaProxy.TextQuery Error: exception happened during query");
+	    e.printStackTrace();
+	    return result;
+	}
+	    
+	if (results_num.startsWith("Resources: ")) {
+	    results_num = results_num.substring(11);
+	} else {
+	    System.err.println("IViaProxy.TextQuery Error: badly formatted results line: "+results_num);
+	    return result;
+	}
+	if (doc_ids.startsWith("Ids: ")) {
+	    doc_ids = doc_ids.substring(5).trim();
+	} else {
+	    System.err.println("IViaProxy.TextQuery Error: badly formatted docs line: "+doc_ids);
+	    return result;
+	}
+	
+	// get the num docs and add to a metadata list
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long numdocs = Long.parseLong(results_num);
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+numdocs);	
+	String [] ids = doc_ids.split(" ");
+	
+	for (int d=0; d<ids.length; d++) {
+	    Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, ids[d]);
+	    doc_node_list.appendChild(doc_node);
+	}
+	///ystem.err.println("IViaProxy result:");
+	///ystem.err.println(this.converter.getString(result));
+	return result;
+	
+    }
+     
+    protected Element processDocumentMetadataRetrieve(Element request) {
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOC_META_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("IViaProxy.DocumentMetadataRetrieve Error: missing paramList.\n");
+	    return result;  // Return the empty result
+	}
+
+	// The metadata information required
+	StringBuffer field_list = new StringBuffer();
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (isAcceptableMetadata(metadata)) {
+		    field_list.append(metadata);
+		    field_list.append(",");	
+		}
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+	
+	if (field_list.length()==0) {
+	    System.err.println("IViaProxy.DocumentMetadataRetrieve Error: no metadata specified.\n"); 
+	    return result;
+	}
+	    
+	// Get the documents
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    System.err.println("IViaProxy Error: DocumentMetadataRetrieve request had no "+GSXML.DOC_NODE_ELEM+"List.\n");
+ 	    return result;
+ 	}
+	
+	StringBuffer record_id_list = new StringBuffer();
+	
+	NodeList request_nodes = request_node_list.getChildNodes();
+	for (int i = 0; i < request_nodes.getLength(); i++) {
+	    Element request_node = (Element) request_nodes.item(i);
+	    String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+	    record_id_list.append(node_id);
+	    record_id_list.append(",");
+	}
+
+	// do the query to the iVia server
+	String url_string = ivia_server_url+"/cgi-bin/view_record_set?theme=gsdl3&record_id_list="+record_id_list.toString()+"&field_list="+field_list.toString(); 
+
+	Element node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	try {
+	    BufferedReader reader = makeConnection(url_string);
+	    String line;
+	    while ((line = reader.readLine()) != null) {
+		if (!line.startsWith("Record:")) {
+		    continue;
+		}
+		// the first line is the record
+		line=line.substring(8);
+		Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		doc_node.setAttribute(GSXML.NODE_ID_ATT, line);
+		Element meta_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		doc_node.appendChild(meta_list);
+		while  ((line = reader.readLine()) != null) {
+		    //metadata entry
+		    int col_pos = line.indexOf(':');
+		    if (col_pos == -1) {
+			// end of the metadata for this doc
+			break; 
+		    }
+		    String name = line.substring(0,col_pos);
+		    String value = line.substring(col_pos+2); // includes a space
+		    GSXML.addMetadata(this.doc, meta_list, name, value);
+		}
+		node_list.appendChild(doc_node);
+		
+	    }
+	} catch (Exception e) {
+	    System.err.println("IViaProxy Error:exception happened");
+	    e.printStackTrace();
+	}
+	///ystem.out.println("IViaProxy: returning result: ");
+	///ystem.out.println(this.converter.getPrettyString(result));
+	return result;
+ 
+    }
+
+    protected Element processDocumentContentRetrieve(Element request) {
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOC_CONTENT_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	// Get the request doc_list
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    System.err.println("IViaProxy Error: DocumentContentRetrieve request specified no doc nodes.\n");
+	    return result;
+	}
+
+	Element doc_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_list);
+	
+	// Get the documents
+	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
+						     GSXML.NODE_ID_ATT);
+	for (int i = 0; i < doc_ids.length; i++) {
+	    String doc_id = doc_ids[i];
+	    Element doc_node = getDocument(doc_id); 
+	    doc_list.appendChild(doc_node);
+	}
+	return result;
+
+    }
+
+
+    /** gets a document by sending a request to iVia, then processes it and creates a documentNode around the text */
+    protected Element getDocument(String doc_id) {
+
+	String url_string = ivia_server_url+"/cgi-bin/view_record?theme=gsdl3&record_id="+doc_id;
+	StringBuffer buffer = new StringBuffer();
+	try {
+	    BufferedReader reader = makeConnection(url_string);
+    
+	    String line;
+	    while((line = reader.readLine())!= null) {
+		buffer.append(line);
+	    }
+
+	    
+	} catch (Exception e) {
+	    System.err.println("IViaProxy Error:exception happened");
+	    e.printStackTrace();
+	}
+	
+	String node_content = buffer.toString(); 
+	String escaped_content = GSXML.xmlSafe(node_content);
+	
+	StringBuffer processed_content = new StringBuffer(escaped_content.length());
+	processed_content.append("<nodeContent>");
+	int pos = 0;
+	int lastpos = 0;
+	while ((pos = escaped_content.indexOf("&lt;a ", lastpos))!= -1) {
+	    processed_content.append(escaped_content.substring(lastpos, pos));
+	    int endpos = escaped_content.indexOf("&lt;/a&gt;", pos);
+	    if (endpos == -1) {
+		break;
+	    }
+	    String link = escaped_content.substring(pos, endpos+10);
+	    link = convertLink(link);
+	    processed_content.append(link);
+	    lastpos = endpos+10;
+	}
+	processed_content.append(escaped_content.substring(lastpos)); // get the last bit
+	processed_content.append("</nodeContent>");
+	    
+	Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	
+	Document content_doc = this.converter.getDOM(processed_content.toString());
+	if (content_doc != null) {
+	    Element content_element = content_doc.getDocumentElement();
+	    doc_node.appendChild(this.doc.importNode(content_element, true));
+	} else {
+	    System.err.println("IViaProxy.getDocument Error: Couldn't parse the node content");
+	}
+	return doc_node;
+	
+    }
+
+    /** converts a url from an <a> element into a greenstone suitable one */
+    protected String convertLink(String aref) {
+	
+	if (aref.indexOf("href=&quot;http") != -1) {
+	    return aref; // an external link
+	}
+	String type = "other";
+	if (aref.indexOf("/cgi-bin/canned_search")!=-1) {
+	    type="query";
+	} else if (aref.indexOf("/cgi-bin/click_through") != -1) {
+	    type = "external";
+	} else if (aref.indexOf("/cgi-bin/view_record") != -1) {
+	    type="document";
+	} 
+	
+	int href_start = aref.indexOf("href=&quot;")+11;
+	int href_end = aref.indexOf("&gt;", href_start);
+	String href = aref.substring(href_start, href_end);
+	String link_content = aref.substring(href_end+4, aref.length()-10);
+	
+	if (type.equals("external")) {
+	    // the external link is everything after the http at the end.
+	    String address = href.substring(href.lastIndexOf("http"));
+	    address = address.replaceAll("%3[aA]", ":");
+	    address = address.replaceAll("%2[fF]", "/");
+
+	    return "&lt;a href=\""+address+"\"&gt;"+link_content+"&lt;/a&gt;";
+	}
+	if (type.equals("other")) {
+	    return "other type of link ("+link_content+")";
+	}
+	StringBuffer result = new StringBuffer();
+	result.append("<link type='");
+	result.append(type);
+	result.append("'");
+	if (type.equals("query")) {
+	    result.append(" service='TextQuery'");
+	}
+	result.append(">");
+	// add in the parameters
+	href = href.substring(href.indexOf("?")+1);
+	String [] params = href.split("&amp;");
+	for (int i=0; i<params.length; i++) {
+	    String param = params[i];
+	    int eq_pos = param.indexOf("=");
+	    if (eq_pos != -1) {
+		
+		result.append("<param name='"+param.substring(0, eq_pos)+"' value='"+param.substring(eq_pos+1)+"'/>");
+	    }
+	}
+	result.append(link_content);
+	result.append("</link>");
+
+	return result.toString();
+	    
+    }
+
+    // iVia craps out if we ask for a metadata which is not valid. So need
+    // to make sure we only ask for acceptable fields.
+    protected boolean isAcceptableMetadata(String meta) {
+	String valid_metadata = ",title,url,ivia_description,keywords,subjects,";
+	if (valid_metadata.indexOf(","+meta+",")!=-1) {
+	    return true;
+	}
+	return false;
+    }
+    protected BufferedReader makeConnection(String url_string) {
+	BufferedReader reader = null;
+	try {
+	    URL url = new URL(url_string);
+	    HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+	    InputStream input = connection.getInputStream();
+	    reader = new BufferedReader(new InputStreamReader(input));  
+	} catch (java.net.MalformedURLException e) {
+
+	    System.err.println("IViaProxy Error: Malformed URL: "+url_string);
+	} catch (java.io.IOException e) {
+	    System.err.println("IViaProxy Error: An error occurred during IO to url "+url_string);
+	}
+	return reader;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaRetrieve.java	(revision 21112)
@@ -0,0 +1,257 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.MalformedURLException;
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ * Modified by  <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
+ */
+
+public class IViaRetrieve 
+    extends AbstractDocumentRetrieve {
+    
+    protected String ivia_server_url = null;
+    
+    public IViaRetrieve() {
+	does_structure = false;
+    }
+    
+    //Configure IViaRetrieve Service
+    public boolean configure(Element info, Element extra_info) 
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "iViaServer");
+	if (server_elem == null) {
+	    System.err.println("IViaRetrieve.configure error: no iViaServer element found");
+	    return false;
+	}
+	ivia_server_url = server_elem.getAttribute("url");
+	if (ivia_server_url.equals("")) {
+	    System.err.println("IViaRetrieve.configure error: no url for the iViaServer element");
+	    return false;
+	}
+	return true;
+	
+    }
+        
+    /** gets a document by sending a request to iVia, then processes it and creates a documentNode around the text */
+    protected Element getNodeContent(String doc_id) 
+	throws GSException {
+
+	String url_string = ivia_server_url+"/cgi-bin/view_record?theme=gsdl3&record_id="+doc_id;
+
+	StringBuffer buffer = new StringBuffer();
+	try {
+	    BufferedReader reader = Misc.makeHttpConnection(url_string);
+    	    String line;
+	    while((line = reader.readLine())!= null) {
+		buffer.append(line);
+	    }
+   	} catch (java.net.MalformedURLException e) {
+	    throw new GSException("Malformed URL: "+url_string, GSXML.ERROR_TYPE_SYSTEM);
+	} catch (java.io.IOException e) {
+	    throw new GSException("IOException during connection to "+url_string+": "+e.toString(),GSXML.ERROR_TYPE_SYSTEM);
+	}
+
+	String node_content = buffer.toString(); 
+
+	String escaped_content = GSXML.xmlSafe(node_content);
+	
+	StringBuffer processed_content = new StringBuffer(escaped_content.length());
+	processed_content.append("<nodeContent>");
+	int pos = 0;
+	int lastpos = 0;
+	while ((pos = escaped_content.indexOf("&lt;a ", lastpos))!= -1) {
+	    processed_content.append(escaped_content.substring(lastpos, pos));
+	    int endpos = escaped_content.indexOf("&lt;/a&gt;", pos);
+	    if (endpos == -1) {
+		break;
+	    }
+	    String link = escaped_content.substring(pos, endpos+10);
+	    link = convertLink(link);
+	    processed_content.append(link);
+	    lastpos = endpos+10;
+	}
+	processed_content.append(escaped_content.substring(lastpos)); // get the last bit
+	processed_content.append("</nodeContent>");
+	    
+	Document content_doc = this.converter.getDOM(processed_content.toString());
+	if (content_doc == null) {
+	    System.err.println("IViaRetrieve.getNodeContent Error: Couldn't parse node content");
+	    System.err.println(processed_content.toString());
+	    return null;
+	}
+	
+	Element content_element = content_doc.getDocumentElement();
+
+	return (Element)this.doc.importNode(content_element,true);
+    }
+	
+    /** converts a url from an <a> element into a greenstone suitable one */
+    protected String convertLink(String aref) {
+	if (aref.indexOf("href=&quot;http") != -1) {
+	    return aref; // an external link
+	}
+	String type = "other";
+	if (aref.indexOf("/cgi-bin/canned_search")!=-1) {
+	    type="query";
+	} else if (aref.indexOf("/cgi-bin/click_through") != -1) {
+	    type = "external";
+	} else if (aref.indexOf("/cgi-bin/view_record") != -1) {
+	    type="document";
+	} 
+	
+	int href_start = aref.indexOf("href=&quot;")+11;
+	int href_end = aref.indexOf("&gt;", href_start);
+	String href = aref.substring(href_start, href_end);
+	String link_content = aref.substring(href_end+4, aref.length()-10);
+	
+	if (type.equals("external")) {
+	    // the external link is everything after the http at the end.
+	    String address = href.substring(href.lastIndexOf("http"));
+	    address = address.replaceAll("%3[aA]", ":");
+	    address = address.replaceAll("%2[fF]", "/");
+
+	    return "&lt;a href=\""+address+"\"&gt;"+link_content+"&lt;/a&gt;";
+	}
+	if (type.equals("other")) {
+	    return "other type of link ("+link_content+")";
+	}
+	StringBuffer result = new StringBuffer();
+	result.append("<link type='");
+	result.append(type);
+	result.append("'");
+	if (type.equals("query")) {
+	    result.append(" service='TextQuery'");
+	}
+	result.append(">");
+	// add in the parameters
+	href = href.substring(href.indexOf("?")+1);
+	String [] params = href.split("&amp;");
+	for (int i=0; i<params.length; i++) {
+	    String param = params[i];
+	    int eq_pos = param.indexOf("=");
+	    if (eq_pos != -1) {
+		
+		result.append("<param name='"+param.substring(0, eq_pos)+"' value='"+param.substring(eq_pos+1)+"'/>");
+	    }
+	}
+	result.append(link_content);
+	result.append("</link>");
+
+	return result.toString();
+    }
+
+    // iVia craps out if we ask for a metadata which is not valid. So need
+    // to make sure we only ask for acceptable fields.
+    protected boolean isAcceptableMetadata(String meta) {
+	String valid_metadata = ",title,url,ivia_description,keywords,subjects,";
+	if (valid_metadata.indexOf(","+meta+",")!=-1) {
+	    return true;
+	}
+	return false;
+    }
+
+    protected String translateId(String oid){
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    System.out.println("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	String top = oid.substring(0, p);
+	return top;
+    }
+
+    protected String translateExternalId(String id){
+	return id;
+    }
+
+    protected String getDocType(String node_id){
+    	return GSXML.DOC_TYPE_SIMPLE;
+    }
+    protected String getRootId(String node_id){
+   	return node_id; 
+    }
+
+    protected ArrayList getChildrenIds(String node_id){
+	return null; 
+    }
+    
+    protected String getParentId(String node_id){
+	return null;
+    }
+    
+    protected Element getMetadataList (String doc_id,
+				       boolean all_metadata,
+				       ArrayList metadata_names) 
+	throws GSException {
+	
+	Element meta_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	// do the query to the iVia server
+	StringBuffer field_list= new StringBuffer();
+	boolean metadata_found = false;
+
+	for (int i=0; i<metadata_names.size();i++){
+	    if (isAcceptableMetadata((String)metadata_names.get(i))){
+		metadata_found = true;
+		field_list.append((String)metadata_names.get(i));
+		field_list.append(",");
+	    }
+	}
+	if (!metadata_found){
+	    return meta_list;
+	}
+	
+	String url_string = ivia_server_url+"/cgi-bin/view_record_set?theme=gsdl3&record_id_list="+doc_id+"&field_list="+field_list.toString(); 
+	try {
+	    BufferedReader reader = Misc.makeHttpConnection(url_string);
+	    String line;
+	    while  ((line = reader.readLine()) != null) {
+		//metadata entry
+		int col_pos = line.indexOf(':');
+		if (col_pos == -1) {
+		    // end of the metadata for this doc
+		    break; 
+		}
+		String name = line.substring(0,col_pos);
+		String value = line.substring(col_pos+2); // includes a space
+		GSXML.addMetadata(this.doc, meta_list, name, value);
+	    }
+	} catch (java.net.MalformedURLException e) {
+	    throw new GSException("Malformed URL: "+url_string, GSXML.ERROR_TYPE_SYSTEM);
+	} catch (java.io.IOException e) {
+	    throw new GSException("IOException: "+e.toString(), GSXML.ERROR_TYPE_SYSTEM);
+	}
+	return meta_list;
+    }
+    
+    protected String getStructureInfo(String doc_id, String info_type){
+	return "";	
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/IViaSearch.java	(revision 21112)
@@ -0,0 +1,201 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.io.File;
+import java.io.BufferedReader;
+import java.net.Authenticator;
+
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
+ */
+
+public class IViaSearch
+    extends AbstractSearch {
+    
+    // have standard gs param names for hits per page, and start page
+    // these need to be mapped to iVia params
+    protected static final String IM_HITS_PARAM = "no_of_records_per_page";
+    protected static final String IM_START_PAGE_PARAM = "start_page_no";
+
+    protected String ivia_server_url = null;
+    protected ArrayList index_ids = null;
+    public IViaSearch()
+    {
+    }
+    
+    //Configure IViaSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "iViaServer");
+	if (server_elem == null) {
+	    System.err.println("IViaSearch.configure error: no iViaServer element found");
+	    return false;
+	}
+	ivia_server_url = server_elem.getAttribute("url");
+	if (ivia_server_url.equals("")) {
+	    System.err.println("IViaSearch.configure error: no url for the iViaServer element");
+	    return false;
+	}
+	does_paging = true;
+	does_multi_index_search = true;
+	this.default_index = ",kw,au,su,ti,de,fu,"; // all of them
+	index_ids = new ArrayList();
+	index_ids.add("kw");
+	index_ids.add("au");	
+	index_ids.add("su");	
+	index_ids.add("ti");
+	index_ids.add("de");
+	index_ids.add("fu");
+
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("IViaSearch Error:: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// tidy whitespace
+	query = query.replaceAll("\\s+", "+");
+	String url_string = ivia_server_url+"/cgi-bin/canned_search?theme=gsdl3&query="+query; 
+
+	// check for fields
+	String fields = (String) params.get(INDEX_PARAM);
+	fields = checkFieldParam(fields); // removes invalid fields
+	if (!fields.equals("")) {
+	    url_string += "&fields="+fields;
+	}
+	//check for hits per page
+	String hits_per_page = (String) params.get(HITS_PER_PAGE_PARAM);
+	if (hits_per_page != null && !hits_per_page.equals("")) {
+	    url_string += "&"+IM_HITS_PARAM+"="+hits_per_page;
+	}
+
+	// check for start page
+	String start_page = (String) params.get(START_PAGE_PARAM);
+	if (start_page != null && !start_page.equals("")) {
+	    url_string += "&"+IM_START_PAGE_PARAM+"="+start_page;
+	}
+	String results_num = null;
+	String doc_ids = null;
+	BufferedReader reader = null;
+	try {
+	    ///system.err.println("IViaSearch, sending "+url_string);
+	    reader = Misc.makeHttpConnection(url_string);
+	    results_num = reader.readLine();
+	    doc_ids = reader.readLine();
+	} catch (java.net.MalformedURLException e) {
+	    GSXML.addError(this.doc, result, "Malformed URL: "+url_string);
+	    return result;
+	} catch (java.io.IOException e) {
+	    GSXML.addError(this.doc, result, "IOException during connection to "+url_string+": "+e.toString());
+	    return result;
+	}
+	    
+	if (results_num.startsWith("Resources: ") && doc_ids.startsWith("Ids: ")) {
+	    results_num = results_num.substring(11);
+	    doc_ids = doc_ids.substring(5).trim();
+
+	} else {
+	    System.err.println("IViaSearch.TextQuery Error: badly formatted results:");
+	    StringBuffer result_string = new StringBuffer();
+	    result_string.append("Error: badly formatted result from IVia server:\n ");
+	    result_string.append(results_num);
+	    result_string.append(doc_ids);
+	    String line;
+	    try {
+		while((line = reader.readLine()) != null) {
+		    result_string.append(line);
+		}
+	    } catch (Exception e) {
+		result_string.append("Exception: "+e);
+	    }
+	    GSXML.addError(this.doc, result, result_string.toString());
+	    
+	    return result;
+	}
+	
+	// get the num docs and add to a metadata list
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long numdocs = Long.parseLong(results_num);
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+numdocs);	
+	String [] ids = doc_ids.split(" ");
+	
+	for (int d=0; d<ids.length; d++) {
+	    Element doc_node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, ids[d]);
+	    doc_node_list.appendChild(doc_node);
+	}
+	return result;
+    }
+     
+    protected String checkFieldParam(String fields) {
+	
+	if (fields == null) {
+	    // return the default
+	    return "";
+	}
+	StringBuffer new_fields = new StringBuffer();
+	String [] ids = fields.split(",");
+	for (int i=0; i<ids.length; i++) {
+	    if(index_ids.contains(ids[i])) {
+		new_fields.append(ids[i]);
+		new_fields.append(",");
+	    }
+	}
+	if (new_fields.length() == 0) {
+	    return "";
+	}
+	return new_fields.toString();
+    }
+    /**
+       An IVia server has a fixed list of fields to search (I think) so we can hard code them here rather than reading them in from a config file
+    */
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
+	index_ids.addAll(this.index_ids);
+	index_names.add(getTextString("param."+INDEX_PARAM+".kw", lang)); 
+	index_names.add(getTextString("param."+INDEX_PARAM+".au", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".su", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".ti", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".de", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".fu", lang));
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/LuceneSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/LuceneSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/LuceneSearch.java	(revision 21112)
@@ -0,0 +1,120 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.util.ArrayList;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.*; //Document;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.index.Term;
+
+import java.io.File;
+/**
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+
+public class LuceneSearch 
+    extends AbstractSearch {
+
+    protected static final String INDEX_ELEM = "index";
+    
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang) 
+    {
+	// the index info - read from config file - cache it??
+	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+	if (index_list != null) {
+	    NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+	    int len = indexes.getLength();
+	    // now add even if there is only one
+	    for (int i=0; i<len; i++) {
+		Element index = (Element)indexes.item(i);
+		index_ids.add(index.getAttribute(GSXML.NAME_ATT));
+		index_names.add(GSXML.getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+	    }
+	} else {
+	    // there is only one index, so we assume the default
+	    index_ids.add("idx");
+	    index_names.add("Default index");
+	}
+
+    }
+
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+
+	// Create a new (empty) result message
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(metadata_list);
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("LuceneSearch Error: TextQuery request had no paramList.");
+	    GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", "0");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query_string = (String) params.get(QUERY_PARAM);
+	if (query_string == null || query_string.equals("")) {
+	    System.err.println("LuceneSearch Error: TextQuery request had no query string.");
+	    GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", "0");
+	    return result;  // Return the empty result
+	}
+
+	// Get the index
+	String index = (String) params.get(INDEX_PARAM);
+        try {
+	    String index_dir = GSFile.collectionIndexDir(this.site_home, this.cluster_name);
+	    index_dir += File.separator+index;
+	    Searcher searcher = new IndexSearcher(index_dir);
+	    Analyzer analyzer = new StandardAnalyzer();
+	    
+	    Term term = new Term("content", query_string);
+        
+	    Query query = new TermQuery(term);
+
+	    Hits hits = searcher.search(query);
+	    GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", ""+hits.length());
+	    
+	    for (int i=0; i<hits.length(); i++) {
+		org.apache.lucene.document.Document luc_doc = hits.doc(i);
+		String node_id = luc_doc.get("nodeID");
+		Element node = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+		doc_node_list.appendChild(node);
+	    }
+	} catch (Exception e) {
+	    e.printStackTrace();
+	}
+	
+	return result;
+	    
+    }
+
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapRetrieve.java	(revision 21112)
@@ -0,0 +1,839 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.io.*;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.lang.reflect.Array;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import javax.swing.tree.DefaultMutableTreeNode;
+
+/** 
+ */
+
+public class MapRetrieve
+    extends ServiceRack {
+
+    // the services on offer
+    protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
+    protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
+
+    protected static final int DOCUMENT=1;
+
+    protected ArrayList index_name_list = null;
+    protected ArrayList index_display_list = null;
+    protected String files_home_dir = null; 
+    protected String temp_files_dir = null;
+    //protected String temp_image_file = null;
+    //protected String temp_image_file2 = null;
+    protected final String jpg_ext = ".jpg";
+    protected String http_image_dir = null;
+    protected String http_temp_image_dir = null;
+    private LinkedList namesInList = new LinkedList();
+
+    private DefaultMutableTreeNode tree;
+
+    /** constructor */
+    public MapRetrieve() {}
+
+
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("Configuring MapRetrieve...");
+	this.config_info = info;
+
+	// set up short_service_info_ - for now just has name and type
+	Element dmr_service = doc.createElement(GSXML.SERVICE_ELEM);
+	dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	short_service_info.appendChild(dmr_service);
+
+	Element dcr_service = doc.createElement(GSXML.SERVICE_ELEM);
+	dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	short_service_info.appendChild(dcr_service);
+
+	Element dsr_service = doc.createElement(GSXML.SERVICE_ELEM);
+	dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	short_service_info.appendChild(dsr_service);
+
+	// set the files_home variable for this collection
+	this.files_home_dir = GSFile.collectionIndexDir(this.site_home, this.cluster_name)+ File.separator+"assoc"+File.separator; 
+	this.temp_files_dir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separator + "temp"+File.separator; 
+
+	    //this.files_home_dir + "maps" + File.separator + "temp" + File.separator;
+	//this.temp_image_file = this.temp_files_dir+"temp_";
+	//this.temp_image_file2 = this.temp_files_dir+"temp2_";
+	
+	
+	this.http_image_dir = this.site_http_address + "/collect/"+this.cluster_name+"/index/assoc/maps/";
+	this.http_temp_image_dir = this.site_http_address + "/collect/"+this.cluster_name+"/temp/";
+
+	this.index_name_list = new ArrayList();
+	this.index_display_list = new ArrayList();
+	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, GSXML.INDEX_ELEM+GSXML.LIST_MODIFIER);
+	Element display_index_list = (Element)GSXML.getChildByTagName(extra_info, "search");
+	if (index_list != null && display_index_list != null) {
+	    NodeList indexes = index_list.getElementsByTagName(GSXML.INDEX_ELEM);
+	    for (int i=0; i<indexes.getLength(); i++) {
+		String name = ((Element)indexes.item(i)).getAttribute(GSXML.NAME_ATT);
+		// add the index name
+		this.index_name_list.add(name);
+		// now look for the display name
+		Element this_index = GSXML.getNamedElement(display_index_list, "index", "name", name);
+		if (this_index != null) {
+		    Element disp = GSXML.getNamedElement(this_index, "displayItem", "name", "name");
+		    if (disp != null) {
+			String display = GSXML.getNodeText(disp);
+			if (!display.equals("")) {
+			    this.index_display_list.add(display);
+			    continue;
+			}
+			
+		    }
+		}
+		// add the id as default text
+		this.index_display_list.add(name);
+
+	    }
+	}
+
+	// look for document display format
+	String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.doc.importNode(display_format, true));
+	    // shoudl we make a copy?
+	}
+
+	// load the 2-dimensional tree for searching
+	try{
+	    ObjectInputStream objectin = new ObjectInputStream(new FileInputStream(this.files_home_dir+"nametree.dat"));
+	    tree = (DefaultMutableTreeNode)objectin.readObject();
+	    objectin.close();
+	}catch(IOException ioe){
+	    ioe.printStackTrace();
+	}catch(ClassNotFoundException cnfe){
+	    cnfe.printStackTrace();
+	}
+    
+
+	return true;
+    }
+
+    /** */
+    protected Element getServiceDescription(String service, String lang, String subset) 
+    {
+	if (service.equals(DOCUMENT_STRUCTURE_RETRIEVE_SERVICE)) {
+	    
+	    Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+	    tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+	    return tq_service;
+	}
+	else if (service.equals(DOCUMENT_METADATA_RETRIEVE_SERVICE)) {
+	    
+	    Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+	    tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	    return tq_service;
+	}
+	else if (service.equals(DOCUMENT_CONTENT_RETRIEVE_SERVICE)) {
+	    
+	    Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+	    tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	    tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	    return tq_service;
+	}
+
+	return null;
+	
+    }
+
+    /** Retrieve the structure of a document */
+    protected Element processDocumentStructureRetrieve(Element request)
+    {
+	// Create a new (empty) result message
+	Element result = doc.createElement(GSXML.RESPONSE_ELEM);
+	return result;
+    }
+
+    protected Element processDocumentMetadataRetrieve(Element request) {
+	// Create a new (empty) result message
+	try{
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	if (uid.equals("")) {
+	    System.out.println("in metadata retrieve, uid = ''\n"+converter.getPrettyString(request));
+	}
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("MapRetrieve Error: missing paramList.\n");
+	    return result;  // Return the empty result
+	}
+
+	// The metadata information required
+	Vector metadata_list = new Vector();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		metadata_list.add(metadata);
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+
+	Element node_list = this.doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(node_list);
+	
+	// Get the documents
+	Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (request_node_list == null) {
+	    System.err.println("Error: DocumentMetadataRetrieve request had no "+GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+ 	    return result;
+ 	}
+
+	NodeList request_nodes = request_node_list.getChildNodes();
+
+	try{
+	    //used just to ensure outfile is initialised
+	    BufferedWriter outfile = new BufferedWriter(new FileWriter(this.temp_files_dir+"emptynothingness"));
+	    
+	    for (int i = 0; i < request_nodes.getLength(); i++) {
+		Element request_node = (Element) request_nodes.item(i);
+		String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+		String place_data = "";
+		String year = "";
+		String coOrdinates = "";
+		String thumb = "";
+		String link = "";
+		int mapFreq = 0;
+		LinkedList place_data_list = new LinkedList();
+		LinkedList coOrdinates_list = new LinkedList();
+
+		//if this is a first displaying of the map
+		if(node_id.indexOf("```") != -1 && node_id.indexOf("imageChooser") == -1){
+		    //get the number of query terms on this map
+		    mapFreq = Integer.parseInt(node_id.substring(node_id.lastIndexOf('`',node_id.indexOf("```")-1)+1,node_id.indexOf("```")));
+		    //get the place names on this map
+		    place_data = node_id.substring(node_id.indexOf("```")+3,node_id.length());
+		    //get the map metadata
+		    node_id = node_id.substring(0,node_id.indexOf('`',node_id.indexOf("```")));
+		    
+		    try{
+			// loop for each query term on the map
+			for(int r=0; r<mapFreq; r++){
+			    // get title, type, location, and the string for creating the hyperlink for this query term
+			    String title = place_data.substring(0,place_data.indexOf('`'));
+			    String type = place_data.substring(place_data.indexOf('`')+1,place_data.indexOf('`',place_data.indexOf('`')+1));
+			    String location = place_data.substring(place_data.indexOf('`',place_data.indexOf('`')+1)+1, place_data.indexOf('`',place_data.indexOf('`',place_data.indexOf('`')+1)+1));
+			    if(place_data.indexOf("```") != -1)
+				link=place_data.substring(0,place_data.indexOf("```"));
+			    else
+				link=place_data;
+			    // resolve the type and location
+			    type = getType(type);
+			    location = getLocation(location);
+			    // remove this query term from the string
+			    if(place_data.indexOf("```") != -1)
+				place_data = place_data.substring(place_data.indexOf("```")+3,place_data.length());
+			    
+			    //add the co-ordinates of this query term to the co-ordinates list
+			    coOrdinates_list.add("`"+link.substring(link.lastIndexOf('`')+1,link.length())+"`"+link.substring(link.lastIndexOf('`',link.lastIndexOf('`')-1)+2,link.lastIndexOf('`')));
+			    
+			    // add the title, type and location to the places list
+			    place_data_list.add(title + ", " + type + ", " + location + ";");
+			}
+		    }catch(StringIndexOutOfBoundsException sioobe){sioobe.printStackTrace();}
+		}
+		
+		// Add the document to the list
+		Element new_node = (Element)this.doc.importNode(request_node, false);
+		node_list.appendChild(new_node);
+		
+		// Add the requested metadata information
+		Element node_meta_list = this.doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		new_node.appendChild(node_meta_list);
+		
+		// create the navigation thumbnails.  This doesn't seem to work most of the time ???????
+		for (int m = 0; m < metadata_list.size(); m++) {
+		    String metadata = (String) metadata_list.get(m);
+		    thumb = "";
+		    String value = "";
+		    if(node_id.indexOf('.') != -1) 
+			thumb = node_id.substring(0,node_id.indexOf('.'));
+		    if(node_id.indexOf('`') != -1) 
+			value = node_id.substring(node_id.lastIndexOf('`',node_id.lastIndexOf('`')-1)+1,node_id.lastIndexOf('`',node_id.lastIndexOf('`')-1)+5);
+		    year = value;
+		    
+		    place_data="";
+		    if(place_data_list.size() != 0)
+			for(int q=0; q<mapFreq; q++){
+			    link = (String)place_data_list.get(q);
+			    if(q!=0){
+				place_data = place_data + "<br>" + link;
+			    }
+			    else{
+				place_data = link;
+				coOrdinates = "``";
+			    }
+			    coOrdinates = coOrdinates+(String)coOrdinates_list.get(q);
+			}
+		    
+		    link = "<a href=\"?a=d&c="+this.cluster_name+"&d="+node_id+coOrdinates+"&dt=map\">";
+		    thumb = "<img src=\""+this.http_image_dir + thumb + "thumb.jpg\" border=0>";
+		    value = "<table><tr><td>"+link+"<p>"+place_data+"<br>"+value+"</a></td><td>"+link+thumb+"</a></td></tr></table>";
+		    if(metadata.equals("Title"))
+			if(!(place_data.equals("")) && place_data.indexOf(", , ;")==-1 && node_id.indexOf("```")==-1)
+			    GSXML.addMetadata(this.doc, node_meta_list, "Title", value);//metadata, value);
+			else
+			    GSXML.addMetadata(this.doc, node_meta_list, metadata, "");
+		    
+		    if(place_data.indexOf(", , ;")==-1){
+			if(i==0)
+			    if(!(mapFreq==0)){
+				outfile = new BufferedWriter(new FileWriter(this.temp_files_dir+"links"+uid));
+				outfile.write("<table align=\"center\"><tr>");
+			    }
+			
+			if(!(mapFreq==0)){
+			    if(i%7 == 0)
+				outfile.write("</tr><tr>");
+			    outfile.write("<td align=\"center\">"+link+thumb+"</a><br>"+link+year+"</a></td>");
+			}
+			
+			if(i == request_nodes.getLength()-1){
+			    outfile.write("</tr></table><p>");
+			    outfile.flush();
+			    outfile.close();
+			}
+		    }
+		} 
+		
+	    }
+	}catch(IOException ioe){ioe.printStackTrace();}
+	
+	return result;
+	}catch(Exception excep){excep.printStackTrace();}
+	return null;
+
+    }
+
+    protected Element processDocumentContentRetrieve(Element request){
+	// Create a new (empty) result message
+	Element result = doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	String uid = request.getAttribute(GSXML.USER_ID_ATT);
+	String temp_image_file = this.temp_files_dir+"temp_"+uid+".jpg";
+	String temp_image_file2 = this.temp_files_dir+"temp_"+uid+"_2.jpg";
+
+	Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (query_doc_list == null) {
+	    System.err.println("Error: DocumentContentRetrieve request specified no doc nodes.\n");
+	    return result;
+	}
+	
+	String legend_file = this.temp_files_dir+"legend_"+uid+".jpg";
+	String blank_file = this.files_home_dir + "blank.jpg";
+	Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_list);
+	
+	// Get the documents
+	String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, 
+							     GSXML.NODE_ID_ATT);
+
+	for (int i = 0; i < doc_ids.length; i++) {
+	    String doc_id = doc_ids[i];
+	    // img_num is the name of the map file eg. 072.jpg
+	    String img_num = doc_id.substring(0,doc_id.indexOf('.'));
+	    // strings for inserting image in html
+	    String img_left = "<img border=0 src=\""+this.http_temp_image_dir;
+	    String doc_content = "";
+	    String co_ordinates = "";
+	    String img_size = "";
+	    int height = 0;
+	    int width = 0;
+	    // the number for the legend image if adding locations to the map
+	    int leg_num = 0;
+	    double ratio = 0;
+
+	    // if user has clicked on the map.  This does not resolve maps that are not North oriented.
+	    if(doc_id.indexOf("imageChooser")==0){
+		try{
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.length());
+		    img_num = doc_id.substring(0,doc_id.indexOf('.'));
+
+		    // get the map size
+		    String get_size[] = {"identify","-size","10000", temp_image_file};
+		    Process proc;
+		    proc = Runtime.getRuntime().exec(get_size);
+		    BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+		    img_size = br.readLine();
+		    proc.waitFor();
+		    img_size = img_size.substring(img_size.indexOf("JPEG")+5,img_size.indexOf(" ",img_size.indexOf("JPEG")+5));
+		    width = Integer.parseInt(img_size.substring(0,img_size.indexOf("x")));
+		    height = Integer.parseInt(img_size.substring(img_size.indexOf("x")+1,img_size.length()));
+
+		    // scale image size according to the ratio
+		    ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`',doc_id.indexOf("```")-1)+1,doc_id.indexOf("```")));
+		    width = (int)(width*ratio);
+		    height = (int)(height*ratio);
+
+		    // get the position of the mouse click on the image
+		    int xclick = Integer.parseInt(doc_id.substring(doc_id.indexOf("```")+3,doc_id.lastIndexOf('`')));
+		    int yclick = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`')+1,doc_id.length()));
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.indexOf("```"));
+
+		    // convert click position to percentage distance accross and down the image.
+		    double xpercent = (xclick*1.0)/width;
+		    double ypercent = (yclick*1.0)/height;
+
+		    // get the top left and bottom right co-ordinates of the map
+		    double ytop = Double.parseDouble(doc_id.substring(0,doc_id.indexOf('`')))*-1;
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.length());
+		    double xleft = Double.parseDouble(doc_id.substring(0,doc_id.indexOf('`')));
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.length());
+		    double ybot = Double.parseDouble(doc_id.substring(0,doc_id.indexOf('`')))*-1;
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.length());
+		    double xright = Double.parseDouble(doc_id.substring(0,doc_id.indexOf('`')));
+		    doc_id = doc_id.substring(doc_id.indexOf('`')+1,doc_id.length());
+
+		    // calculate the map co-ordinates of the mouse click
+		    xpercent = ((xright-xleft)*xpercent)+xleft;
+		    ypercent = ((ybot-ytop)*ypercent)+ytop;
+
+		    // search the tree for nearby place names
+		    namesInList.clear();
+		    findName(xpercent, ypercent, 0.1, 0.1, tree, true);
+
+		    int namesInArraySize = namesInList.size();
+		    String returnNames[]=new String[namesInArraySize];
+
+		    //put the names in an array
+		    for(int ri=0; namesInList.size()>0; ri++){
+			returnNames[ri]=(String)namesInList.getFirst();
+			returnNames[ri] = returnNames[ri].substring(0,returnNames[ri].indexOf('`'));
+			namesInList.removeFirst();
+		    }
+
+		    //sort the names
+		    Arrays.sort(returnNames);
+		    doc_content = "\n<script>\nfunction openIt(loc){\n\topener.location=loc;\n\tself.close();\n}\n</script>";
+		    for(int nameIndex=0; nameIndex < namesInArraySize; nameIndex++){
+			String tempName = returnNames[nameIndex];
+			//convert names with spaces for hyperlinks
+			if(returnNames[nameIndex].indexOf(' ') != -1){
+			    returnNames[nameIndex] = returnNames[nameIndex].replaceAll(" ","+");
+			    returnNames[nameIndex] = "%22"+returnNames[nameIndex]+"%22";
+			}
+			// add the place name to the html
+			doc_content = doc_content+"<a href=\"\" onClick=openIt('?a=q&sa=&rt=r&s=MapQuery&c="+this.cluster_name+"&startPage=1&s1.index=none&s1.maxDocs=10&s1.query="+returnNames[nameIndex]+"')>"+tempName+"</a><br>";
+		    }
+		    
+		}catch(Exception ioexception){ioexception.printStackTrace();}
+	    }
+	    else{
+		
+		try{
+		    //file for converting image    
+		    BufferedWriter bw = new BufferedWriter(new FileWriter(this.temp_files_dir + "add_x_"+uid));;
+		    Process proc;
+		    
+		    // if a new search
+		    if(doc_id.indexOf("```") != -1){
+			// copy requested map to temp.jpg
+			proc = Runtime.getRuntime().exec("cp "+this.files_home_dir+"maps"+File.separator+img_num+".jpg "+temp_image_file);
+			proc.waitFor();
+		    }
+
+		    //get the image size
+		    String get_size[] = {"identify","-size","10000", temp_image_file};
+		    proc = Runtime.getRuntime().exec(get_size);
+		    BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+		    img_size = br.readLine();
+		    proc.waitFor();
+		    img_size = img_size.substring(img_size.indexOf("JPEG")+5,img_size.indexOf(" ",img_size.indexOf("JPEG")+5));
+		    if (img_size.indexOf("+")!=-1) {
+			img_size = img_size.substring(0, img_size.indexOf("+"));
+		    }
+		    width = Integer.parseInt(img_size.substring(0,img_size.indexOf("x")));
+		    height = Integer.parseInt(img_size.substring(img_size.indexOf("x")+1,img_size.length()));
+		    
+		    // if a new search
+		    if(doc_id.indexOf("```") != -1){
+			co_ordinates = doc_id.substring(doc_id.indexOf("```")+2, doc_id.length());
+			// get the number of places to mark on the map
+			int x_number = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`',doc_id.indexOf("```")-1)+1,doc_id.indexOf("```")));
+
+			//write the convert command
+			bw.write("convert -font helvetica -fill red -pointsize 48 ");
+
+			for(int xi=0; xi<x_number; xi++){
+
+			    // get the co-ordinates of the place name
+			    double xco = Double.parseDouble(co_ordinates.substring(co_ordinates.lastIndexOf('`',co_ordinates.lastIndexOf('`')-1)+1,co_ordinates.lastIndexOf('`',co_ordinates.lastIndexOf('`')-1)+7));
+			    double yco = Double.parseDouble(co_ordinates.substring(co_ordinates.lastIndexOf('`')+1,co_ordinates.length()));
+			    if(xi != x_number-1)
+				co_ordinates = co_ordinates.substring(0,co_ordinates.lastIndexOf('`',co_ordinates.lastIndexOf('`')-1));
+			
+			    int index = 0;
+			    index = doc_id.indexOf('`')+2;
+			    
+			    // get the zoom ratio and the top left and bottom right co-ordinates of the map
+			    ratio = 0.4;
+			    double tly = Double.parseDouble(doc_id.substring(index,index+5));
+			    index = doc_id.indexOf('`',index)+1;
+			    double tlx = Double.parseDouble(doc_id.substring(index,index+6));
+			    index = doc_id.indexOf('`',index)+2;
+			    double bry = Double.parseDouble(doc_id.substring(index,index+5));
+			    index = doc_id.indexOf('`',index)+1;
+			    double brx = Double.parseDouble(doc_id.substring(index,index+6));
+			    index = doc_id.indexOf('`',index)+1;
+			    double orient = Double.parseDouble(doc_id.substring(index,doc_id.indexOf('`',index)));
+
+			    // find the centre of the map
+			    double xcent = ((brx-tlx)/2)+tlx;
+			    double ycent = ((bry-tly)/2)+tly;
+
+			    // get the orientation of the map
+			    orient = Math.toRadians(orient);
+
+			    // rotate the co-ordinates around the centre of the map 
+			    xco=xco-xcent;
+			    yco=yco-ycent;
+			    double oldx=xco;
+			    xco=xco*Math.cos(orient)-yco*Math.sin(orient);
+			    yco=oldx*Math.sin(orient)+yco*Math.cos(orient);
+			    xco=xco+xcent;
+			    yco=yco+ycent;
+			    xco = (xco-tlx)/(brx-tlx);
+			    yco = (yco-tly)/(bry-tly);
+
+			    // calculate the pixels for placing the mark
+			    width = (int)(xco*width)-5;
+			    height = (int)(yco*height)-5;
+			
+			    // write the options for convert
+			    bw.write("-draw 'text "+width+","+height+" \"x\"' ");
+			    bw.flush();
+			
+			    // reset the width and height variables for the image
+			    width = Integer.parseInt(img_size.substring(0,img_size.indexOf("x")));
+			    height = Integer.parseInt(img_size.substring(img_size.indexOf("x")+1,img_size.length()));
+			}
+			doc_id = doc_id.substring(0,doc_id.lastIndexOf('`',doc_id.indexOf("```")-1));
+
+			// write the end of the convert command, then command for zooming image to the right level, then command to copy the blank image over legend.jpg
+			bw.write(temp_image_file+" "+temp_image_file+";convert -scale "+((int)(ratio*width))+"x"+((int)(ratio*height))+" "+temp_image_file+" "+temp_image_file2+";cp "+blank_file +" "+legend_file);
+
+			// reset legend number
+			leg_num = 0;
+		    }
+
+		    else
+			// if changing zoom level
+			if(doc_id.substring(doc_id.lastIndexOf('`'),doc_id.length()).indexOf('.')!=-1){
+			    // get the ratio
+			    ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`')+1,doc_id.length()));
+			    // write the command for scaling image
+			    bw.write("convert -scale "+((int)(ratio*width))+"x"+((int)(ratio*height))+" "+temp_image_file+" "+temp_image_file2);
+			    doc_id = doc_id.substring(0,doc_id.lastIndexOf('`'));
+			}
+
+		        // if adding locations to map
+			else{
+
+			    // get the location type to add
+			    String restricter= doc_id.substring(doc_id.lastIndexOf('`')+1,doc_id.length());
+			    doc_id = doc_id.substring(0,doc_id.lastIndexOf('`'));
+
+			    // get the number for the legend image
+			    leg_num = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`')+1,doc_id.length()));
+			    doc_id = doc_id.substring(0,doc_id.lastIndexOf('`'));
+
+			    //open file for location type
+			    BufferedReader inType = new BufferedReader(new FileReader(this.files_home_dir + "place_types"+File.separator + restricter + ".txt"));
+
+			    //check through the file and add any places that are in the bounds of this map.
+
+			    // check value. set to true if a place to add is found
+			    boolean add_place_type = false;
+
+			    int index = 0;
+			    index = doc_id.indexOf('`')+2;
+			    // get zoom ratio
+			    ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`')+1,doc_id.length()));
+			    // get top left and bottom right co-ordinates of the map
+			    double tly = Double.parseDouble(doc_id.substring(index,index+5));
+			    index = doc_id.indexOf('`',index)+1;
+			    double tlx = Double.parseDouble(doc_id.substring(index,index+6));
+			    index = doc_id.indexOf('`',index)+2;
+			    double bry = Double.parseDouble(doc_id.substring(index,index+5));
+			    index = doc_id.indexOf('`',index)+1;
+			    double brx = Double.parseDouble(doc_id.substring(index,index+6));
+			    index = doc_id.indexOf('`',index)+1;
+			    // get orientation of the map
+			    double orient = Double.parseDouble(doc_id.substring(index,doc_id.indexOf('`',index)));
+			    // calculate centre of map
+			    double xcent = ((brx-tlx)/2)+tlx;
+			    double ycent = ((bry-tly)/2)+tly;
+			    orient = Math.toRadians(orient);
+
+			    String type_point = "";
+			    double xco = 0.0;
+			    double yco = 0.0;
+
+			    // read the file
+			    while(inType.ready()){
+				//read a line
+				type_point = inType.readLine();
+
+				// get the co-ordinates of the point
+				xco = Double.parseDouble(type_point.substring(type_point.lastIndexOf('`')+1,type_point.length()));
+				yco = Double.parseDouble(type_point.substring(type_point.lastIndexOf('`',type_point.lastIndexOf('`')-1)+2,type_point.lastIndexOf('`')));
+
+				// if it is within the co-ordinates of the map
+				if(xco >= tlx && xco < brx && yco >= tly && yco < bry){
+				    // rotate the point around the centre according to map orientation
+				    xco=xco-xcent;
+				    yco=yco-ycent;
+				    double oldx=xco;
+				    xco=xco*Math.cos(orient)-yco*Math.sin(orient);
+				    yco=oldx*Math.sin(orient)+yco*Math.cos(orient);
+				    xco=xco+xcent;
+				    yco=yco+ycent;
+				    // get the pixels for where to place the mark
+				    xco = (xco-tlx)/(brx-tlx);
+				    yco = (yco-tly)/(bry-tly);
+				    width = (int)(xco*width)-5;
+				    height = (int)(yco*height)-5;
+
+				    //if this is the first point to be added
+				    if(!add_place_type){
+					// write the initial convert command
+					bw.write("convert -font helvetica -fill red -pointsize 36 ");
+					// toggle check value
+					add_place_type = true;
+				    }
+
+				    // write the options for the convert command
+				    bw.write("-draw 'text "+width+","+height+" \""+leg_num+"\"' ");
+				    bw.flush();
+			
+				    // reset width and height variables for the image
+				    width = Integer.parseInt(img_size.substring(0,img_size.indexOf("x")));
+				    height = Integer.parseInt(img_size.substring(img_size.indexOf("x")+1,img_size.length()));
+				}
+			    }
+
+			    //if there are places to mark on the map
+			    if(add_place_type){
+				// finish the convert command and write command for scaling image
+				bw.write(temp_image_file+" "+temp_image_file+";convert -scale "+((int)(ratio*width))+"x"+((int)(ratio*height))+" "+temp_image_file+" "+temp_image_file2);
+
+				// open file for converting the legend command
+				BufferedWriter buf = new BufferedWriter(new FileWriter(this.temp_files_dir+"add_l_"+uid));
+				if(leg_num == 1)
+				    buf.write("cp "+blank_file +" "+legend_file+";");
+				// write the command for adding to the legend
+				buf.write("convert -font helvetica -fill red -pointsize 12 -draw 'text 15,"+(leg_num*15+20)+" \""+leg_num+" "+getType(restricter)+"\"' "+legend_file+" "+legend_file);
+				buf.flush();
+				buf.close();
+				// execute the command for the legend image
+				proc = Runtime.getRuntime().exec("sh "+this.temp_files_dir+"add_l_"+uid);
+				proc.waitFor();
+			    }
+			    inType.close();
+			}
+		    bw.flush();
+		    bw.close();
+		
+		    // execute the convert commands etc.
+		    proc = Runtime.getRuntime().exec("sh "+this.temp_files_dir+"add_x_"+uid);
+		    proc.waitFor();
+		
+		}catch(Exception ioe){ioe.printStackTrace();}
+
+		//write the html for the document
+
+		String doc_content_head = "?a=d&c="+this.cluster_name+"&dt=map&d=";
+		doc_content = "<td valign=\"top\"><script>document.write('"+img_left+"legend_"+uid+".jpg"+"?'+new Date().getTime())</script>\"></td>";
+		doc_content = doc_content+"<td><script>document.write('"+img_left+"temp_"+uid+"_2.jpg?'+new Date().getTime())</script>\" onclick=\"imgClickHandler(event, this, '"+img_num+"')\"></td>";
+
+		// replace _httpimg_ with the correct address
+		doc_content = "<table><tr>"+doc_content+"</tr></table>";
+
+		String javascript = getJavascript(ratio, doc_id, leg_num);
+
+		doc_content = "<center>You are currently viewing the map at "+(int)(ratio*100)+"%<br>Select a radio button for a new zoom level.</center>"+javascript+doc_content;
+		doc_content = doc_content.replaceAll("zoomhead", doc_content_head);
+		doc_content = doc_content.replaceAll("placeChooserImage", doc_content_head+"imageChooser`"+doc_id+"`"+ratio);
+
+		// read the file for the navigation thumbnails
+		try{
+		    BufferedReader infile = new BufferedReader(new FileReader(this.temp_files_dir+"links"+uid));
+		    doc_content = doc_content + infile.readLine();
+		    infile.close();
+		}catch(Exception ioexc){ioexc.printStackTrace();}
+	    }
+
+	    // put the html in a text node
+	    Element text_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
+	    text_doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    GSXML.addDocText(doc, text_doc, doc_content);
+	    doc_list.appendChild(text_doc);
+	    
+	}
+        return result;
+    }
+
+    // resolves location codes for places from LINZ database
+    private String getLocation(String location){
+	String read;
+	try{
+	    BufferedReader in = new BufferedReader(new FileReader(this.files_home_dir+"files"+File.separator+"landdist.txt"));
+	    in.readLine();
+	    while(in.ready()){
+		read = in.readLine();
+		if(read.substring(0,2).equals(location)){
+		    in.close();
+		    return read.substring(3, read.length());
+		}
+	    }
+	}catch(Exception e){}
+	return "";
+    }
+
+    // resolves type codes for the LINZ database
+    private String getType(String type){
+	String read;
+	try{
+	    BufferedReader in = new BufferedReader(new FileReader(this.files_home_dir+"files"+File.separator+"pointdes.txt"));
+	    in.readLine();
+	    while(in.ready()){
+		read = in.readLine();
+		if(read.substring(0,read.indexOf('`')).toLowerCase().equals(type.toLowerCase())){
+		    in.close();
+		    return read.substring(read.indexOf('`')+1, read.indexOf(':'));
+		}
+	    }
+	}catch(Exception e){}
+	return "";
+    }
+
+    //recursive pre-order traversal of the 2-dimensional tree
+    // x & y are co-ordinates of mouse click, xoff & yoff are the distance in degrees away from x & y to search,
+    // d is the current node of the tree, xcomp controls whether searching is by x co-ordinate or y co-ordinate.
+    private void findName(double x, double y, double xoff, double yoff, DefaultMutableTreeNode d, boolean xcomp){
+	//if a leaf node return.  All leaf nodes are empty
+	if(d.isLeaf())
+	    return;
+	//get coordinates of the current node
+	double xco = Double.parseDouble(((String)d.getUserObject()).substring(((String)d.getUserObject()).length()-6,((String)d.getUserObject()).length()));
+	double yco = Double.parseDouble(((String)d.getUserObject()).substring(((String)d.getUserObject()).length()-12,((String)d.getUserObject()).length()-7));
+	//if in range
+	if((x-xoff)<xco && (x+xoff)>xco && (y-yoff)<yco && (y+yoff)>yco){
+	    //add to list
+	    namesInList.addFirst((String)d.getUserObject());
+	}
+	//if comparing the x axis
+	if(xcomp){
+	    //if need to search left subtree
+	    if((x-xoff)<xco)
+		findName(x,y,xoff,yoff,(DefaultMutableTreeNode)d.getChildAt(0),!xcomp);
+	    //if need to search right subtree
+	    if((x+xoff)>xco)
+		findName(x,y,xoff,yoff,(DefaultMutableTreeNode)d.getChildAt(1),!xcomp);
+	}
+	else{
+	    //if need to search left subtree
+	    if((y-yoff)<yco)
+		findName(x,y,xoff,yoff,(DefaultMutableTreeNode)d.getChildAt(0),!xcomp);
+	    //if need to search right subtree
+	    if((y+yoff)>yco)
+		findName(x,y,xoff,yoff,(DefaultMutableTreeNode)d.getChildAt(1),!xcomp);
+	}
+    }
+
+    // a whole lot of stuff for the html that is hidden at the bottom here because it clutters more than I already have further up!
+    private String getJavascript(double ratio, String doc_id, int leg_num)
+    {
+	String javascript = "\n<script>\nfunction imgClickHandler (evt, img, num) {\n\tif (window.event){\n\t\tvar chooserString = 'placeChooserImage' + '```' + window.event.offsetX + '`' + window.event.offsetY\n\t\tmywindow = window.open(chooserString,'Choose','width=600, height=500, scrollbars=yes, resizable=yes, toolbar=no, location=no, status=yes, menubar=no');\n\t\t//alert(window.event.offsetX + ':' + window.event.offsetY + ':' + num);\n\t}\n\telse if (evt.target) {\n\t\tvar coords = {x: 0, y: 0 };\n\t\tvar el = evt.target;\n\t\tdo {\n\t\t\tcoords.x += el.offsetLeft;\n\t\t\tcoords.y += el.offsetTop;\n\t\t}\n\t\twhile ((el = el.offsetParent));\n\t\tvar offsetX = evt.clientX - coords.x;\n\t\tvar offsetY = evt.clientY - coords.y;\n\t\talert(offsetX + ':' + offsetY + ':' + num);\n\t}\n}\nfunction leapTo (link)\n{\n\tvar new_url=link;\n\tif (  (new_url != \"\")  &&  (new_url != null)  ) \n\t\twindow.location=new_url;\n\telse \n\t\talert(\"You must make a selection.\");\n}\nfunction jumpTo (front,link,back)\n{\n\tvar new_url=front+link+back;\n\tif (  (new_url != \"\")  &&  (new_url != null)  ) \n\t\twindow.location=new_url;\n\telse \n\t\talert(\"You must make a selection.\");\n}// Deactivate Cloaking -->\n</script>";
+
+	String radioButtons = "\n<center><FORM>";
+	if(ratio != 0.4)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.4')\">40%";
+	if(ratio != 0.5)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.5')\">50%";
+	if(ratio != 0.6)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.6')\">60%";
+	if(ratio != 0.7)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.7')\">70%";
+	if(ratio != 0.8)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.8')\">80%";
+	if(ratio != 0.9)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`0.9')\">90%";
+	if(ratio != 1.0)
+	    radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead"+doc_id+"`1.0')\">100%";
+	radioButtons = radioButtons + "\n</FORM></center>";
+
+	leg_num = leg_num+1;
+
+	String doc_link = "?a=d&c="+this.cluster_name+"&dt=map&d="+doc_id+"`"+ratio+"`"+leg_num+"`";
+
+	StringBuffer dropDownBox = new StringBuffer();
+
+	dropDownBox.append("Add Locations Of Type: <form name=\"myForm\"><select name=\"s1index\" onChange=\"jumpTo('");
+	dropDownBox.append(doc_link);
+	dropDownBox.append("',document.myForm.s1index.options[document.myForm.s1index.selectedIndex].value,'')\">\n");
+
+	for (int i=0; i<this.index_name_list.size(); i++) {
+	    String name = (String)this.index_name_list.get(i);
+	    dropDownBox.append("<option value=\"");
+	    dropDownBox.append(name);
+	    dropDownBox.append("\">");
+	    String display = (String)this.index_display_list.get(i);
+	    dropDownBox.append(display);
+	    dropDownBox.append("\n");
+	}
+	// for each option, Malcolm had onClick=\"leapTo('"+doc_link+name+"')\" - dont seem to need this
+	dropDownBox.append("</select></form>\n");
+
+	return javascript + radioButtons + dropDownBox;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapSearch.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapSearch.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/MapSearch.java	(revision 21112)
@@ -0,0 +1,389 @@
+/*
+ *    MapSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+// General java classes
+import java.util.Hashtable;
+import java.util.HashMap;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.lang.reflect.Array;
+import java.util.Comparator;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.FileInputStream;
+import java.lang.ClassNotFoundException;
+
+/**
+ *
+ * 
+ */
+public class MapSearch 
+    extends AbstractSearch {
+
+    protected String files_home_dir = null;
+
+    protected static final String FREQ_ATT = "freq";
+
+    // use the following if we add back in index searching
+    //protected String default_index = null;
+    //protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    //protected static final String INDEX_ELEM = "index";  
+    
+    /** Hashtable containing the place names database, indexed on the place name in lower case. */
+    public static Hashtable hashNames;
+
+    /** Constructor */
+    public MapSearch() {}
+
+    /** */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	System.out.println("Configuring MapSearch...");
+
+	// set the files_home_dir variable for this collection
+	this.files_home_dir = GSFile.collectionIndexDir(site_home, cluster_name)+File.separator+"assoc"+File.separator;
+
+	//load the hashtable
+	hashNames = new Hashtable();
+	try{
+	    ObjectInputStream in = new ObjectInputStream(new FileInputStream(this.files_home_dir+"hashnames.dat"));
+	    hashNames = (Hashtable)in.readObject();
+	    in.close();
+	}catch(IOException ioe){
+	    ioe.printStackTrace();
+	    return false;
+	}catch(ClassNotFoundException cnf){
+	    cnf.printStackTrace();
+	    return false;
+	}
+	 
+	// just in case we get the index stuff going, here is the code
+	
+	// get the display stuff for the indexes
+	
+// 	NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+// 	if (extra_info != null) {
+// 	    Document owner = info.getOwnerDocument();
+// 	    Element config_search = (Element)GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+// 	    if (config_search != null) {
+// 		for (int i=0; i<indexes.getLength();i++) {
+// 		    Element ind = (Element)indexes.item(i);
+// 		    String name = ind.getAttribute(GSXML.NAME_ATT);
+// 		    Element node_extra = GSXML.getNamedElement(config_search,
+// 							       GSXML.INDEX_ELEM,
+// 							       GSXML.NAME_ATT,
+// 							       name);
+// 		    if (node_extra == null) {
+// 			// no index specification for this index
+// 			continue;
+// 		    }
+		    
+// 		    // get the display elements if any - displayName
+// 		    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+// 		    if (display_names !=null) {
+// 			for (int j=0; j<display_names.getLength(); j++) {
+// 			    Element e = (Element)display_names.item(j);
+// 			    ind.appendChild(owner.importNode(e, true));
+// 			}
+// 		    }
+// 		} // for each index
+// 	    }
+// 	}
+// 	// Get the default index out of <defaultIndex> (buildConfig.xml)
+// 	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+// 	if (def != null) {
+// 	    default_index = def.getAttribute(GSXML.NAME_ATT);
+// 	}
+// 	if (default_index == null || default_index.equals("")) {
+// 	    // use the first one from the index list
+// 	    default_index = ((Element)indexes.item(0)).getAttribute(GSXML.NAME_ATT);
+// 	}
+
+	return true;
+    }
+
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names, String lang) 
+    {
+	// for now, we just have one dummy index
+	index_ids.add("idx");
+	index_names.add("maps"); // get from properties
+
+	// use the following if we ever get the index search working
+// 	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+// 	NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+// 	for (int i=0; i<indexes.getLength(); i++) {
+// 	    Element index = (Element)indexes.item(i);
+// 	    index_ids.add( index.getAttribute(GSXML.NAME_ATT));
+// 	    index_names.add(GSXML.getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+// 	}
+    }
+    
+
+    /** Process a map query */
+    protected Element processTextQuery(Element request) 
+    {
+	// Create a new (empty) result message
+	Element result = doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("Error: TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String searchTerm = (String) params.get(QUERY_PARAM);
+	if (searchTerm == null || searchTerm.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If an index hasn't been specified, use the default
+// 	String index = (String) params.get(INDEX_PARAM); 
+// 	if (index == null) {
+// 	    index = default_index;
+// 	}
+
+	//convert the query string from the form '&quotTe Awamutu&quot Hamilton' to 'Te Awamutu+Hamilton'
+
+	//replace ' ' with '+'
+	searchTerm = searchTerm.replace(' ','+');
+
+	//replace '&quot' with '"'
+	int place = -1;
+	while(searchTerm.indexOf("&quot;",place+1) != -1){
+	    place = searchTerm.indexOf("&quot;",place+1);
+	    searchTerm = (searchTerm.substring(0,place) + "\"" + searchTerm.substring(place+6,searchTerm.length()));
+	}
+
+	//replace spaces in string enclosed in quotes
+	place = -1;
+	while(searchTerm.indexOf('"',place+1) != -1){
+	    place = searchTerm.indexOf('"',place+1);
+	    if(searchTerm.indexOf('"',place+1) != -1)
+		searchTerm = (searchTerm.substring(0,place) + searchTerm.substring(place, searchTerm.indexOf('"',place+1)).replace('+',' ') + searchTerm.substring(searchTerm.indexOf('"',place+1), searchTerm.length()));
+	    place = searchTerm.indexOf('"',place);
+	}
+
+	//remove speech marks
+	place = 0;
+	while(place != -1){
+	    place = searchTerm.indexOf('"', place);
+	    if(place != -1){
+		searchTerm = searchTerm.substring(0, place) + searchTerm.substring(place+1, searchTerm.length());
+		place=0;
+	    }
+	}
+
+	//find the number of search terms (number of '+' plus 1)
+	int words = 1;
+	place = 0;
+	while (place != -1){
+	    place = searchTerm.indexOf('+', place+1);
+	    if(place != -1) words++;
+	}
+	place = 0;
+
+	//store each search term in a string array
+	String terms[] = new String[words];
+	String terms_freq[] = new String[words];
+	for (int i=0; i<(words-1); i++){
+	    terms[i]=searchTerm.substring(place, searchTerm.indexOf('+', place));
+	    place = searchTerm.indexOf('+', place)+1;
+	}
+	terms[words-1] = searchTerm.substring(place, searchTerm.length());
+
+	Object nameArray[] = new Object[1];
+	int nameArraySize;
+	String nameData;
+	double xco, yco;
+	LinkedList mapList = new LinkedList();
+	LinkedList placeList = new LinkedList();
+	String readString = "";
+
+	//capitalise the search terms
+	for(int i=0;i<words;i++)
+	    terms[i] = terms[i].toUpperCase();
+
+	for(int k=0; k<words; k++){
+	    nameArraySize=0;
+
+
+	    if((nameArray = (Object[])hashNames.get(terms[k].toLowerCase())) != null){
+		//System.out.println(hashNames.get(terms[k].toLowerCase()));
+		
+		nameArraySize = Arrays.asList(nameArray).size();
+	    }
+	    
+	    
+	    int h;
+	    for(h=0;h<nameArraySize;h++){
+
+		nameData = (String)nameArray[h];
+
+		// get the co-ordinates of the current place name
+		xco = Double.parseDouble(nameData.substring(nameData.lastIndexOf('`')+1,nameData.length()));
+		yco = Double.parseDouble(nameData.substring(nameData.lastIndexOf('`',nameData.lastIndexOf('`')-1)+2,nameData.lastIndexOf('`')));
+
+		//open the text file containing map metadata (first line is a description of the file)
+		try{
+		    BufferedReader inmap = new BufferedReader(new FileReader(this.files_home_dir + "files"+File.separator+"maps"+File.separator+"mapdata.txt"));
+		    inmap.readLine();
+
+		    // for each map
+		    while(inmap.ready()){
+			readString = inmap.readLine();
+			int indexer = 0;
+			indexer = readString.indexOf('`')+2;
+			// get the co-ordinates of the top left and bottom right corners of the map.
+			double ytop = Double.parseDouble(readString.substring(indexer,indexer+5));
+			indexer = readString.indexOf('`',indexer)+1;
+			double xtop = Double.parseDouble(readString.substring(indexer,indexer+6));
+			indexer = readString.indexOf('`',indexer)+2;
+			double ybot = Double.parseDouble(readString.substring(indexer,indexer+5));
+			indexer = readString.indexOf('`',indexer)+1;
+			double xbot = Double.parseDouble(readString.substring(indexer,indexer+6));
+
+			//if the place is within the map, put the map metadata and the placename metadata in linked list
+			if(xco >= xtop && xco < xbot && yco >= ytop && yco < ybot)
+			    mapList.add(readString+"```"+nameData);
+		    }
+		}catch(Exception e){e.printStackTrace();}
+	    }
+	}
+
+	// use an array object to sort the linked list by map number
+	Object[] mapListArray = mapList.toArray();
+	Arrays.sort(mapListArray);
+	mapList.clear();
+	for(int mla=0; mla<Array.getLength(mapListArray); mla++)
+	    mapList.addFirst(mapListArray[mla]);
+
+	//for each map, create a list of the query matches on that map
+	LinkedList tempList = new LinkedList();
+	String mapNumber = "";
+	String currentMap[] = {"",""};
+	int mapFreq = 0;
+	while(mapList.size()>0){
+	    readString = (String)mapList.removeFirst();
+	    if(mapNumber.equals(readString.substring(0,readString.indexOf('.')))){
+		currentMap[1] = currentMap[1]+readString.substring(readString.indexOf("```"),readString.length());
+		mapFreq++;
+	    }
+	    else{
+		if(!currentMap[0].equals(""))
+		    tempList.add(currentMap[0]+"`"+mapFreq+currentMap[1]);
+		currentMap[0] = readString.substring(0,readString.indexOf("```"));
+		currentMap[1] = readString.substring(readString.indexOf("```"),readString.length());
+		mapNumber = readString.substring(0,readString.indexOf('.'));
+		mapFreq=1;
+	    }
+	}
+	if(!currentMap[0].equals(""))
+	    tempList.add(currentMap[0]+"`"+mapFreq+currentMap[1]);
+
+	int totalDocs = tempList.size();
+
+	// use an array object to sort the linked list by number of matches on each map
+	Object[] tempListArray = tempList.toArray();
+	Arrays.sort(tempListArray, new MapResultSorter());
+	tempList.clear();
+	for(int tla=0; tla<Array.getLength(tempListArray); tla++)
+	    tempList.add(tempListArray[tla]);
+
+	// Create a metadata list to store information about the query results
+	Element metadata_list = doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+			
+	// Add a metadata element specifying the number of matching documents
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsMatched", "" + totalDocs);
+	GSXML.addMetadata(this.doc, metadata_list, "numDocsReturned", ""+totalDocs);
+       	// Create a document list to store the matching documents, and add them
+	Element document_list = doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(document_list);
+	for (int d = 0; d < totalDocs; d++) {
+	    String doc_id = (String)tempList.get(d);
+
+	    Element doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    doc_node.setAttribute(GSXML.NODE_TYPE_ATT, "thumbnail");
+	    doc_node.setAttribute(GSXML.DOC_TYPE_ATT, "map");
+	    document_list.appendChild(doc_node);
+	}
+			
+	// Create a term list to store the term information, and add it
+	Element term_list = doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);	
+	
+
+	for (int t=0; t<words; t++){
+	    String term = terms[t];
+	
+	    Element term_elem = doc.createElement(GSXML.TERM_ELEM);
+  	    term_elem.setAttribute(GSXML.NAME_ATT, term);
+  	    term_elem.setAttribute(FREQ_ATT, "" + terms_freq[t]);
+
+	    term_list.appendChild(term_elem);
+	}
+	return result;
+    }
+ 
+
+    public class MapResultSorter implements Comparator {
+	
+	public int compare(Object o1, Object o2) {
+	    String first = (String)o1;
+	    String second = (String) o2;
+	    int firstInt;
+	    int secondInt;
+	    first = first.substring(first.lastIndexOf('`',first.indexOf("```")-1)+1,first.indexOf("```"));
+	    second = second.substring(second.lastIndexOf('`',second.indexOf("```")-1)+1,second.indexOf("```"));
+	    firstInt = Integer.parseInt(first);
+	    secondInt = Integer.parseInt(second);
+	    if(firstInt > secondInt)
+		return -1;
+	    else if(firstInt == secondInt)
+		return 0;
+	    else
+		return 1;
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ModuleWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ModuleWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ModuleWrapper.java	(revision 21112)
@@ -0,0 +1,235 @@
+/*
+ *    ModuleWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+import org.greenstone.gsdl3.core.ModuleInterface;
+
+// XML classes
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.SAXParser;
+import org.apache.xerces.dom.TextImpl;
+import org.apache.xerces.parsers.DOMParser;
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element;
+import org.w3c.dom.Node; 
+import org.w3c.dom.Node;                                                
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+// java classes
+import java.io.StringReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.security.SecureRandom;
+
+/**
+ * A classes for logging and checking all traffic to and from a ModuleInterface
+ * 
+ * if logCounter hasn't been initialised, initialise it to a random 
+ * value. This has to use secure random rather than a normal random
+ * or every time the application is run the same numbers will be
+ * generated and the search for a free range will become a linear
+ * search rather than a hash.
+ *
+ *
+ * @author <a href="mailto:s.yeates@cs.waikato.ac.nz">Stuart Yeates</a>
+ * @version $Revision$
+ */
+public class ModuleWrapper 
+  implements ModuleInterface 
+{
+  /** the module we're wrapping */
+  protected ModuleInterface inner = null;
+  /** the module we're wrapping */
+  public ModuleInterface getInner(){return inner;};
+  /** the module we're wrapping */
+  public void setInner(ModuleInterface i){inner=i;};
+
+  /** The schema of the IN XML */
+  protected String fromSchema = null;
+  /** The schema of the IN XML */
+  public String getFromSchema(){return fromSchema;};
+  /** The schema of the IN XML */
+  public void setFromSchema(String s){fromSchema=s;};
+
+  /** The schema of the OUT XML */
+  protected String toSchema = null;
+  /** The schema of the OUT XML */
+  public String getToSchema(){return toSchema;};
+  /** The schema of the OUT XML */
+  public void setToSchema(String s){toSchema=s;};
+
+  /** Are we logging the XML ? */
+  protected boolean logging = true;
+  /** Are we logging the XML ? */
+  public boolean getLogging(){return logging;};
+  /** Are we logging the XML ? */
+  public void setLogging(boolean b){logging=b;};
+
+  /** Where we are logging the XML */
+  protected String logDirectory = "/tmp/";
+  /** Where we are logging the XML */
+  public String getLogDirectory(){return logDirectory;};
+  /** Where we are logging the XML */
+  public void setLogDirectory(String s){logDirectory=s;};
+
+  /** The number of the previous log file */
+  protected static long logCounter = 0;
+
+  /** The no-args constructor */
+  public ModuleWrapper(){
+
+  }
+  
+  public void cleanUp() {}
+  
+  /** The all-args constructor */
+  public ModuleWrapper(String in,String out, ModuleInterface inner){
+    this.setFromSchema(in);
+    this.setToSchema(out);
+    this.setInner(inner);
+  }
+
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xmlIn the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+  public String process(String xmlIn) {
+    
+    long logNumber = 0;
+    String xmlOut = "";
+    if (getLogging()) {
+      if (logCounter == 0)
+	logCounter = new SecureRandom().nextLong();
+      try {
+	logNumber = logCounter++;
+	String filename = getLogDirectory() + File.separator + logNumber + ".in";
+	File file = new File(filename);
+	
+	while (file.exists()){
+	  logCounter = new SecureRandom().nextLong();
+	  logNumber = logCounter++;
+	  filename = getLogDirectory() + File.separator + logNumber + ".in";
+	  file = new File(filename);
+	}
+	
+	FileWriter writer = new FileWriter(file);
+	writer.write(xmlIn);
+      } catch (IOException e) {
+	System.err.println("ModuleWrapper: caught exception: " +e );
+      }
+    }
+    xmlOut = processInner(xmlIn);
+
+    try {
+      String filename = getLogDirectory() + File.separator + logNumber + ".out";
+      File file = new File(filename);
+      FileWriter writer = new FileWriter(file);
+      writer.write(xmlOut);
+    } catch (IOException e) {
+      System.err.println("ModuleWrapper: caught exception: " +e );
+    }
+    return xmlOut;
+  }
+
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xmlIn the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+  protected String processInner(String xmlIn) 
+  {
+    try {
+      DocumentBuilderFactory factory=DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder= factory.newDocumentBuilder();
+      DOMParser parser = new DOMParser();
+      try {
+	
+	parser.setFeature("http://xml.org/sax/features/validation",true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema",true);
+	parser.setFeature("http://apache.org/xml/features/validation/dynamic", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema-full-checking", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema/normalized-value", true);
+	parser.setFeature("http://apache.org/xml/features/validation/warn-on-duplicate-attdef", true);
+	parser.setFeature("http://apache.org/xml/features/validation/warn-on-undeclared-elemdef", true);
+	parser.setFeature("http://apache.org/xml/features/warn-on-duplicate-entitydef", true);
+      } catch (Exception a) {
+	System.err.println("unable to set feature:" +a);
+	a.printStackTrace();
+      }
+
+      // do the pre-call checking
+      InputSource xml_source = new InputSource(new StringReader(xmlIn));  
+      try {
+	parser.parse(xml_source);
+      } catch (Exception e){  
+	System.err.println("parsing error:" +e);
+	e.printStackTrace();
+	return "<response> <error class=\"ModuleWrapper\" code=1> Error: supplied string contained the parse error: " + e + " </error> </response>";
+      } 
+
+      String xmlOut = inner.process(xmlIn);
+
+      // do the post-call checking
+      InputSource xmlResult = new InputSource(new StringReader(xmlIn));  
+      try {
+	parser.parse(xmlResult);
+      } catch (Exception e){  
+	System.err.println("parsing error:" +e);
+	e.printStackTrace();
+	return "<response> <error class=\"ModuleWrapper\" code=2> Error: returned string contained the parse error: " + e + "</error></response>";
+      } 
+
+      return xmlOut;
+
+    } catch (Exception e){  
+      System.err.println("other error:" +e);
+      e.printStackTrace();
+      return "<response> <error class=\"ModuleWrapper\" code=3>Error: Unknown error or warning: " + e + " </error></response>";
+    } 
+  }
+
+  /**
+   * Process an XML request - as a DOM Element
+   *
+   * @param in the request to process
+   * @return the response - contains any error messages
+   * @see org.w3c.dom.Element
+   */
+  public Element process(Element xmlIn) {
+    throw new Error("Not implmented yet. Should be faked by stringizing the node.");
+  }
+}    
+                                                                           
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/PhindPhraseBrowse.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/PhindPhraseBrowse.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/PhindPhraseBrowse.java	(revision 21112)
@@ -0,0 +1,459 @@
+/*
+ *    PhindServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+
+import org.greenstone.mgpp.*;
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Text; 
+
+import java.util.Vector;
+import java.util.HashMap;
+import java.io.File;
+/** 
+ * PhindServices - the phind phrase browsing service
+ * 
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+public class PhindPhraseBrowse
+    extends ServiceRack {
+    
+    // the services on offer
+    private static final String PHIND_SERVICE = "PhindApplet";
+
+    private MGPPWrapper mgpp_src=null;
+    private String basepath = null;
+
+    private Element applet_description = null;
+    
+    public PhindPhraseBrowse() {
+	this.mgpp_src = new MGPPWrapper();
+	// set up the default params
+	this.mgpp_src.setQueryLevel("Document");
+	this.mgpp_src.setReturnLevel("Document");
+	this.mgpp_src.setMaxDocs(5);
+	this.mgpp_src.setStem(false);
+	this.mgpp_src.setCase(true);
+    }
+
+    public void cleanUp() {
+	super.cleanUp();
+	this.mgpp_src.unloadIndexData();
+    }
+
+    /** configure the service module
+     *
+     * @param info a DOM Element containing any config info for the service
+     * @return true if configured
+     */
+    public boolean configure(Element info, Element extra_info) {
+
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	System.out.println("configuring PhindPhraseBrowse");
+
+	// set up short_service_info_ - for now just has name and type
+	Element e = this.doc.createElement(GSXML.SERVICE_ELEM);
+	e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+	e.setAttribute(GSXML.NAME_ATT, PHIND_SERVICE);
+	this.short_service_info.appendChild(e);
+
+	// set up the static applet description
+
+	applet_description = this.doc.createElement(GSXML.SERVICE_ELEM);
+	applet_description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+	applet_description.setAttribute(GSXML.NAME_ATT, PHIND_SERVICE);
+
+	// add in the applet info for the phind applet
+	// need to make this dynamic - library names etc
+	// change the applet params - have a single param with the library name
+	// this is left blank at this end, and must be filled in by applet action - if the library name is not needed, this param is left out
+	// phindcgi param now is not complete - library must be prepended to it.
+	String app_info = "<"+GSXML.APPLET_ELEM+" CODEBASE='applet' CODE='org.greenstone.applet.phind.Phind.class' ARCHIVE='phind.jar, xercesImpl.jar, xml-apis.jar' WIDTH='500' HEIGHT='400'><PARAM NAME='library' VALUE=''/> <PARAM NAME='phindcgi' VALUE='?";
+	app_info += GSParams.ACTION +"=a&amp;"+GSParams.REQUEST_TYPE +"=r&amp;"+GSParams.SERVICE+"="+PHIND_SERVICE+"&amp;"+GSParams.OUTPUT+"=xml&amp;"+GSParams.RESPONSE_ONLY+"=1'/>";
+	app_info +="<PARAM NAME='collection'   VALUE='";
+	app_info += this.cluster_name;
+	app_info += "'/> <PARAM NAME='classifier' VALUE='1'/>  <PARAM NAME='orientation'  VALUE='vertical'/> <PARAM NAME='depth' VALUE='2'/> <PARAM NAME='resultorder' VALUE='L,l,E,e,D,d'/> <PARAM NAME='backdrop' VALUE='interfaces/default/images/phindbg1.jpg'/><PARAM NAME='fontsize' VALUE='10'/> <PARAM NAME='blocksize'    VALUE='10'/>The Phind java applet.</"+GSXML.APPLET_ELEM+">";
+	
+	Document dom = this.converter.getDOM(app_info);
+	if (dom==null) {
+	    System.err.println("PhindPhraseBrowse.configure Error: Couldn't parse applet info");
+	    return false;
+	}
+	Element app_elem = dom.getDocumentElement();
+	applet_description.appendChild(this.doc.importNode(app_elem, true));
+
+	return true;
+    }
+
+    protected Element getServiceDescription(String service, String lang, String subset) {
+	if (!service.equals(PHIND_SERVICE)) {
+	    return null;
+	}
+	Element describe = (Element) applet_description.cloneNode(true);
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(PHIND_SERVICE+".name", lang)));
+	describe.appendChild(GSXML.createDisplayTextElement(this.doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(PHIND_SERVICE+".description", lang)));
+	return describe;
+    }
+
+    protected Element processPhindApplet(Element request) {
+	
+	Element param_elem = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params = GSXML.extractParams(param_elem, false);
+	
+	long first_e = Long.parseLong((String)params.get("pfe"));
+	long last_e = Long.parseLong((String)params.get("ple"));
+	long first_l = Long.parseLong((String)params.get("pfl"));
+	long last_l = Long.parseLong((String)params.get("pll"));
+	long first_d = Long.parseLong((String)params.get("pfd"));
+	long last_d = Long.parseLong((String)params.get("pld"));
+	
+	long phrase;
+	String phrase_str = (String)params.get("ppnum");
+	if (phrase_str == null || phrase_str.equals("")) {
+	    phrase=0;
+	} else {
+	    phrase = Long.parseLong(phrase_str);
+	}
+	String word = (String)params.get("pptext");
+	String phind_index = (String)params.get("pc");
+	// the location of the mgpp database files
+	this.basepath = GSFile.phindBaseDir(this.site_home, this.cluster_name, phind_index);
+
+	// the result element
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, PHIND_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// applet result info must be in appletInfo element
+	Element applet_data = this.doc.createElement(GSXML.APPLET_DATA_ELEM);
+	result.appendChild(applet_data);
+	Element phind_data = this.doc.createElement("phindData");
+	applet_data.appendChild(phind_data);
+
+
+	// if we dont know the phrase number, look it up
+	if (phrase == 0) {
+	    if (word==null || word.equals("")) {
+		Element error = phindError("no word or phrase");
+		phind_data.appendChild(error);
+		return result;
+	    }
+	    phrase = findPhraseNumberFromWord( word);
+	}
+	if (phrase==0) {
+	    // the word is not in the collection
+	    // return a phind error string
+	    Element error = phindError("the term "+word+" is not in the collection");
+	    phind_data.appendChild(error);
+	    return result;
+	}
+	
+	// get the phrase data into the phind_data node
+	getPhraseData(phind_data, phrase, first_l, last_l,
+		      first_e, last_e,  first_d, last_d);
+	return result;
+	
+	
+    }// processPhindApplet
+    
+    protected long findPhraseNumberFromWord(String word) {
+
+	// set the mgpp index data - we are looking up pword
+	this.mgpp_src.loadIndexData(this.basepath+File.separatorChar+"pword");
+
+	this.mgpp_src.runQuery(word);
+
+	MGPPQueryResult res = this.mgpp_src.getQueryResult();
+	Vector docs = res.getDocs();
+	if (docs.size()==0) {
+	    // phrase not found
+	    return 0;
+	}
+	MGPPDocInfo doc = (MGPPDocInfo)docs.firstElement();
+	return doc.num_;
+    }
+
+    protected boolean getPhraseData(Element phind_data,
+				    long phrase, long first_l, long last_l,
+				    long first_e, long last_e, long first_d, 
+				    long last_d) {
+
+	String record = this.mgpp_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+					      phrase);
+	if (record.equals("")) {
+	    Element error = phindError("somethings gone wrong - we haven't got a record for phrase number "+phrase);
+	    phind_data.appendChild(error);
+	    return false;
+	}
+	
+	// parse the record - its in gordons cryptic form
+	// ":word:tf:ef:df:el:dl:lf:ll"
+	// el: e,e,e
+	// dl: d;f,d;f,
+	// lf and ll may be null
+	// l: type,dest, dest; type,dest,dest
+
+	// ignore everything up to and including first colon (has
+	// <Document>3505: at the start)
+	record = record.substring(record.indexOf(':')+1);
+
+	// split on ':'
+	String [] fields = record.split(":");
+	String word = fields[0];
+	String tf = fields[1];
+	String ef = fields[2];
+	String df = fields[3];
+
+	
+	String expansions = fields[4];
+	String documents = fields[5];
+	String lf = "0";
+	String linklist = "";
+	if (fields.length > 7) {// have thesaurus stuff
+	    lf =fields[6];
+	    linklist = fields[7];
+	}
+	
+	// the phindData attributes and phrase
+	phind_data.setAttribute("id", Long.toString(phrase));
+	phind_data.setAttribute("df", df);
+	phind_data.setAttribute("ef", ef);
+	phind_data.setAttribute("lf", lf);
+	phind_data.setAttribute("tf", tf);
+	GSXML.createTextElement(this.doc, "phrase", word);
+
+	addExpansionList(phind_data, expansions, word, ef, first_e, last_e);
+	addDocumentList(phind_data, documents, word, df, first_d, last_d);
+	if (!lf.equals("0")) {
+	    addThesaurusList(phind_data, linklist, word, lf, first_l, last_l);
+	}
+	return true;
+    }
+
+    protected boolean addExpansionList( Element phind_data, String record, 
+				       String word,
+				       String freq,
+				       long first, long last) {
+
+	Element expansion_list = this.doc.createElement("expansionList");
+	phind_data.appendChild(expansion_list);
+	expansion_list.setAttribute("length", freq);
+	expansion_list.setAttribute("start", Long.toString(first));
+	expansion_list.setAttribute("end", Long.toString(last));
+
+	// get the list of strings
+	String [] expansions = record.split(",");
+	int length = expansions.length;
+	if (length < last) last = length;
+	for (long i = first; i < last; i++) {
+	    long num  = Long.parseLong(expansions[(int)i]);
+	    Element expansion = getExpansion( num, word);
+	    expansion.setAttribute("num", Long.toString(i));
+	    expansion_list.appendChild(expansion);
+	}
+	return true;
+    }
+    
+    protected Element getExpansion(long phrase_num, 
+				   String orig_phrase) {
+	
+	// look up the phrase in the pdata thingy
+	String record = this.mgpp_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+					      phrase_num);
+
+	if (record ==null || record.equals("")) return null;
+
+	// ignore everything up to and including first colon 
+	record = record.substring(record.indexOf(':')+1);
+
+	String [] fields = record.split(":");
+	String phrase = fields[0];
+	String tf = fields[1];
+	//String ef = fields[2]; dont use this
+	String df = fields[3];
+
+	Element expansion = this.doc.createElement("expansion");
+	expansion.setAttribute("tf", tf);
+	expansion.setAttribute("df", df);
+	expansion.setAttribute("id", Long.toString(phrase_num));
+
+	// get teh suffix and prefix
+	String [] ends = splitPhraseOnWord(phrase, orig_phrase);
+	if (!ends[0].equals("")) {
+	    expansion.appendChild(GSXML.createTextElement(this.doc, "prefix", ends[0]));
+	}
+	if (!ends[1].equals("")) {
+	    expansion.appendChild(GSXML.createTextElement(this.doc, "suffix", ends[1]));
+	}
+
+	return expansion;
+
+    }
+
+    protected boolean addDocumentList(Element phind_data, String record, 
+				      String word,
+				      String freq,
+				      long first, long last) {
+
+	Element document_list = this.doc.createElement("documentList");
+	phind_data.appendChild(document_list);
+	document_list.setAttribute("length", freq);
+	document_list.setAttribute("start", Long.toString(first));
+	document_list.setAttribute("end", Long.toString(last));
+
+	// get the list of doc,freq
+	String [] doc_freqs = record.split(";");
+	int length = doc_freqs.length;
+	if (length<last) last=length;
+
+	for (long i = first; i < last; i++) {
+	    String doc_elem = doc_freqs[(int)i];
+	    int p = doc_elem.indexOf(',');
+	    long doc_num;
+	    String doc_freq;
+	    if (p == -1) { // there is no freq in the record
+		doc_num =Long.parseLong(doc_elem);
+		doc_freq = "1";
+	    } else {
+		doc_num = Long.parseLong(doc_elem.substring(0,p));
+		doc_freq = doc_elem.substring(p+1);
+	    }
+	    Element document = getDocument( doc_num);
+	    document.setAttribute("freq", doc_freq);
+	    document.setAttribute("num", Long.toString(i));
+	    document_list.appendChild(document);
+	}
+
+	
+	return true;
+    }
+
+
+    protected Element getDocument(long doc_num) {
+	
+	// look up the phrase in the docs thingy
+	String record = this.mgpp_src.getDocument(this.basepath+File.separatorChar+"docs", "Document",
+					      doc_num);
+	
+	if (record ==null || record.equals("")) return null;
+	
+	// ignore everything up to and including first \t
+	record = record.substring(record.indexOf('\t')+1);
+
+	String [] fields = record.split("\t");
+	String hash = fields[0];
+	String title = fields[1];
+
+	Element d = this.doc.createElement("document");
+	d.setAttribute("hash", hash);
+	d.appendChild(GSXML.createTextElement(this.doc, "title", title));
+	
+	return d;
+
+    }
+    protected boolean addThesaurusList(Element phind_data, String record,
+				       String word,
+				       String freq,
+				       long first, long last) {
+
+
+	Element thesaurus_list = this.doc.createElement("thesaurusList");
+	phind_data.appendChild(thesaurus_list);
+	thesaurus_list.setAttribute("length", freq);
+	thesaurus_list.setAttribute("start", Long.toString(first));
+	thesaurus_list.setAttribute("end", Long.toString(last));
+	
+	// get the list of type,dest,dest
+	String [] links = record.split(";");
+	int length = links.length;
+	long index = 0;
+	for (int i = 0; i < length; i++) { // go through the entries
+	    String link_info = links[(int)i];
+	    String [] items = link_info.split(",");
+	    // the first entry is teh type
+	    String type = items[0];
+	    for (int j = 1; j<items.length; j++, index++) {
+		if (index >= first && index < last) { // only output the ones we want
+		    long phrase = Long.parseLong(items[j]);
+		    Element t = getThesaurus(phrase);
+		    t.setAttribute("type", type);
+		    thesaurus_list.appendChild(t);
+		}
+	    }
+	}
+
+	return true;
+    }
+
+    protected Element getThesaurus(long phrase_num) {
+
+	// look up the phrase in the pdata thingy
+	String record = this.mgpp_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+					      phrase_num);
+
+	if (record ==null || record.equals("")) return null;
+
+	// ignore everything up to and including first colon 
+	record = record.substring(record.indexOf(':')+1);
+
+	String [] fields = record.split(":");
+	String phrase = fields[0];
+	String tf = fields[1];
+	//String ef = fields[2]; dont use this
+	String df = fields[3];
+
+	Element thesaurus = this.doc.createElement("thesaurus");
+	thesaurus.setAttribute("tf", tf);
+	thesaurus.setAttribute("df", df);
+	thesaurus.setAttribute("id", Long.toString(phrase_num));
+	thesaurus.appendChild(GSXML.createTextElement(this.doc, "phrase", phrase));
+	return thesaurus;
+
+    }
+
+    /** returns an array of two elements - the prefix and the suffix*/
+    protected String [] splitPhraseOnWord(String phrase, String word) {
+	
+	if (word.equals("")) {
+	    
+	    String [] res =  {phrase, ""};
+	    return res;
+	}
+	// use 2 so that we only split on the first occurrance. trailing empty strings should be included 
+	String [] result = phrase.split(word, 2);
+	return result;
+	
+    }
+
+    protected Element phindError(String message) {
+	Element e = this.doc.createElement("phindError");
+	Text t = this.doc.createTextNode(message);
+	e.appendChild(t);
+	return e;
+    }
+    
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ServiceRack.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ServiceRack.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/ServiceRack.java	(revision 21112)
@@ -0,0 +1,410 @@
+/*
+ *    ServiceRack.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+
+// xml classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+import org.apache.xpath.XPathAPI;
+
+// general java classes
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.File;
+import java.util.HashMap;
+import java.util.ResourceBundle;
+import java.util.Locale;
+import java.lang.reflect.Method;
+
+/**
+ * ServiceRack - abstract base class for services
+ *
+ * A ServiceRack provides one or more Services. 
+ * This base class implements the process method.
+ *Each service is invoked
+ * by a method called process<service name> which takes one parameter - the xml request Element, and returns an XML response Element. 
+ * for example, the TextQuery service would be invoked by 
+ * processTextQuery(Element request)
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+public abstract class ServiceRack
+    implements ModuleInterface
+{
+
+    /** the absolute address of the site home */
+    protected String site_home =null;
+    /** the http address of the site home */
+    protected String site_http_address =null;
+
+    /** the name of the cluster (or collection) that this service 
+	belongs to - if any */ 
+    protected String cluster_name = null;
+
+    /** some services can talk back to the message router */
+    protected ModuleInterface router = null;
+
+    /** a converter class to create Documents etc */
+    protected XMLConverter converter = null;
+
+    /** the original config info - if need to store it */
+    protected Element config_info = null;
+
+    /** XML element for describe requests - the container doc */
+    protected Document doc = null;
+
+    /** XML element for describe requests - list of supported services
+	- this is static     */
+    protected Element short_service_info = null;
+
+    /** XML element for stylesheet requests - map of service name to format 
+	elem */
+    protected HashMap format_info_map = null;
+
+    /** A class loader that knows about the collection resources directory 
+     *  can put properties files, dtds etc  in here */
+    CollectionClassLoader class_loader = null;
+    
+    /** sets the cluster name */
+    public void setClusterName(String cluster_name) {
+	this.cluster_name = cluster_name;
+    }
+    /** sets the collect name */
+    public void setCollectionName(String coll_name) {
+	setClusterName(coll_name);
+    }
+
+    public void cleanUp() {}
+    
+    /** sets the site home */
+    public void setSiteHome(String site_home) {
+	this.site_home = site_home;
+    }
+    /** sets the site http address */
+    public void setSiteAddress(String site_address) {
+	this.site_http_address = site_address;
+    }
+
+    /** sets the message router */
+    public void setMessageRouter(ModuleInterface m) {
+       this.router = m;
+    }
+
+    /** the no-args constructor */
+    public ServiceRack() {
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+	this.short_service_info = this.doc.createElement(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER);
+	this.format_info_map = new HashMap();
+    }
+    
+
+    /** configure the service module
+     *
+     * @param info the XML node <serviceRack name="XXX"/> with name equal
+     * to the class name (of the subclass)
+     *
+     * must configure short_service_info_ and service_info_map_
+     * @return true if configured ok
+     * must be implemented in subclasses
+     */
+    public boolean configure(Element info) {
+	return configure(info, null);
+    }
+    
+    public boolean configure(Element info, Element extra_info) {
+	// set up the class loader
+	this.class_loader = new CollectionClassLoader(this.getClass().getClassLoader(), this.site_home, this.cluster_name);
+	return true;
+    }
+
+    /**
+     * Process an XML document - convenience method that uses Strings rather than Elements. just calls process(Element).
+     *
+     * @param xml_in the Document to process - a string
+     * @return the resultant document as a string - contains any error messages
+     * @see String
+     */
+    public String process(String xml_in) {
+	
+	Document doc = this.converter.getDOM(xml_in);
+	if (doc == null) {
+	    System.err.println("ServiceRack.process(String) Error: Couldn't parse request");
+	    System.err.println(xml_in);
+	    return null;
+	}
+	Element res = process(doc.getDocumentElement());
+	return this.converter.getString(res);
+	
+    }
+
+    /** process an XML request in DOM form
+     *
+     * @param xml_in the Element node containing the request
+     * should be <message> 
+     * @return an Element with the result XML
+     * @see Element
+     */
+    public Element process(Element message) {
+
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+	Document mess_doc = message.getOwnerDocument();
+	Element mainResult = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	if (requests.getLength()==0) {
+	    // no requests
+	    return mainResult; // empty message for now
+	}
+
+	for (int i=0; i<requests.getLength(); i++) {
+	    Element request = (Element)requests.item(i);
+	
+	    String type = request.getAttribute(GSXML.TYPE_ATT);
+	    if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE)) {
+		Element response = processDescribe(request);
+		if (response !=null) {
+		    mainResult.appendChild(this.doc.importNode(response, true));
+		}
+		
+	    } else if (type.equals(GSXML.REQUEST_TYPE_FORMAT)) {
+		Element response = processFormat(request);
+		    mainResult.appendChild(this.doc.importNode(response, true));
+		
+	       
+	    } else {
+		// other type of request, must be processed by the subclass - 
+		// send to the service method
+		StringBuffer error_string = new StringBuffer();
+		String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));		
+		Element response = null;
+		try {
+		    Class c = this.getClass();
+		    Class []params = {Class.forName("org.w3c.dom.Element")};
+		    
+		    String method_name = "process"+to;
+		    Method m = null;
+		    while (c != null) {
+			
+			try {
+			    m = c.getDeclaredMethod(method_name, params);
+			    // if this has worked, break
+			    break;
+			} catch (NoSuchMethodException e) {
+			    c = c.getSuperclass();
+			} catch (SecurityException e) {
+			    System.err.println("ServiceRack.process: security exception for finding method "+method_name);
+			    error_string.append("ServiceRack.process: security exception for finding method "+method_name);
+			}
+		    } // while
+		    if (m != null) {
+			Object []args = {request};
+			try {
+			    response = (Element)m.invoke(this, args);
+			    
+			} catch (Exception e) {
+			    System.err.println("ServiceRack.process: Trying to call a processService type method (process"+to+") on a subclass("+this.getClass().getName()+"), but an exception happened:"+e.toString());
+			    error_string.append("ServiceRack.process: Trying to call a processService type method (process"+to+") on a subclass("+this.getClass().getName()+"), but an exception happened:"+e.toString());
+			}
+		    } else {
+			System.err.println("ServiceRack.process: method "+method_name+" not found for class "+this.getClass().getName());
+			error_string.append("ServiceRack.process: method "+method_name+" not found for class "+this.getClass().getName());
+		    }
+		    
+		} catch (ClassNotFoundException e) {
+		    System.err.println("ServiceRack error: Element class not found");
+		    error_string.append("ServiceRack error: Element class not found");
+		}
+		if (response !=null) {
+		    mainResult.appendChild(this.doc.importNode(response, true));
+		} else {
+		    // add in a dummy response
+		    System.err.println("adding in an error element\n");
+		    response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+		    GSXML.addError(this.doc, response, error_string.toString());
+		    mainResult.appendChild(response);
+		    
+		}
+ 
+	    } // else process request
+	} // for each request
+	
+	return mainResult;
+		
+    }
+
+ 
+
+    /** process method for describe requests 
+     */
+    protected Element processDescribe(Element request) {
+	
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_DESCRIBE);
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	if (to.equals("")) { // return the service list
+	    response.appendChild(getServiceList(lang));
+	    return response;
+	} 
+	response.setAttribute(GSXML.FROM_ATT, to);
+	Element param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element description = null;
+	if (param_list == null) {
+	    description = getServiceDescription(to, lang, null);
+	} else {
+	    NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	    for (int i=0; i<params.getLength(); i++) {
+		
+		Element param = (Element)params.item(i);
+		// Identify the structure information desired
+		if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM )) {
+		    String info = param.getAttribute(GSXML.VALUE_ATT);
+		    if (description == null) {
+			 description = getServiceDescription(to, lang, info);
+		    } else {
+			Element temp = getServiceDescription(to, lang, info);
+			GSXML.mergeElements(description, temp);
+		    }
+		}
+	    }
+	}
+	if (description != null) { // may be null if non-existant service
+	    response.appendChild(description);
+	}
+	return response;
+    
+    }
+
+    /** process method for stylesheet requests
+     */
+    protected Element processFormat(Element request) {
+	Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_FORMAT);
+	
+	String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+	
+	if (to.equals("")) { // serviceRack query - is this appropriate??
+	    return response;
+	} 
+	
+	
+	// describe a particular service	
+	if (this.format_info_map.containsKey(to)) {
+	    response.appendChild(getServiceFormat(to));
+	    response.setAttribute(GSXML.FROM_ATT, to);
+	    return response;
+	}
+	// else no format info
+	System.err.println("ServiceRack describe request: no format info for "+to+".");
+	return response;
+    }	
+    
+    /** returns the service list for the subclass */
+    protected Element getServiceList(String lang) {
+	// for now, it is static and has no language stuff
+	return (Element) this.short_service_info.cloneNode(true);
+    }
+    
+    /** returns a specific service description */
+    abstract protected Element getServiceDescription(String service, String lang, String subset);
+
+    protected Element getServiceFormat(String service) {
+	Element format = (Element)((Element)this.format_info_map.get(service)).cloneNode(true);
+	return format;
+    }
+
+    /** overloaded version for no args case */
+    protected String getTextString(String key, String lang) {
+	return getTextString(key, lang, null, null);
+    }
+
+    protected String getTextString(String key, String lang, String dictionary) {
+	return getTextString(key, lang, dictionary, null);
+    }
+    protected String getTextString(String key, String lang, String [] args) {
+	return getTextString(key, lang, null, args);
+    }
+	
+    /** getTextString - retrieves a language specific text string for the given
+key and locale, from the specified resource_bundle (dictionary)
+    */
+    protected String getTextString(String key, String lang, String dictionary, String[] args) {
+
+	// we want to use the collection class loader in case there are coll specific files
+	if (dictionary != null) {
+	    // just try the one specified dictionary
+	    Dictionary dict = new Dictionary(dictionary, lang, this.class_loader);
+	    String result = dict.get(key, args);
+	    if (result == null) { // not found
+		return "_"+key+"_";
+	    }
+	    return result;
+	}
+
+	// now we try class names for dictionary names
+	String class_name = this.getClass().getName();
+	class_name = class_name.substring(class_name.lastIndexOf('.')+1);
+	Dictionary dict = new Dictionary(class_name, lang, this.class_loader);
+	String result = dict.get(key, args);
+	if (result != null) {
+	    return result;
+	}
+
+	// we have to try super classes
+	Class c = this.getClass().getSuperclass();
+	while (result == null && c != null) {
+	    class_name = c.getName();
+	    class_name = class_name.substring(class_name.lastIndexOf('.')+1);
+	    if (class_name.equals("ServiceRack")) {
+		// this is as far as we go
+		break;
+	    }
+	    dict = new Dictionary(class_name, lang, this.class_loader);
+	    result = dict.get(key, args);
+	    c = c.getSuperclass();
+	}
+	if (result == null) {
+	    return "_"+key+"_";
+	}
+	return result;
+	
+    }
+
+    protected String getMetadataNameText(String key, String lang) {
+
+	String properties_name = "metadata_names";
+	Dictionary dict = new Dictionary(properties_name, lang);
+	
+	String result = dict.get(key);
+	if (result == null) { // not found
+	    return null;
+	} 
+	return result;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/TEIRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/TEIRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/TEIRetrieve.java	(revision 21112)
@@ -0,0 +1,52 @@
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+
+// General Java classes
+
+// just needs to override a few methods that are TEI specific
+public class TEIRetrieve extends XMLRetrieve {
+
+   protected Element extractTitleMeta(Element section) {
+	
+	Element meta_elem = this.doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+	
+	Element section_head = (Element)GSXML.getChildByTagName(section, "head");
+	if (section_head == null || !section_head.hasChildNodes()) {
+	    // there is no head element, so take the type attribute and make that the title
+	    String title = "("+section.getAttribute("type")+")";
+	    Text t = this.doc.createTextNode(title);
+	    meta_elem.appendChild(t);
+	} else {
+	    // add the head element as the metadata content
+	    meta_elem.appendChild(this.doc.importNode(section_head, true));
+	}
+	return meta_elem;
+    }
+
+    // this assumes that the scope refers to a top level node - this may be overwritten if the scope bit in the id is a shorthand of some sort
+    protected String translateScope(String scope) {
+	if (scope.equals("front")) {
+	    return "text/front";
+	}
+	if (scope.equals("body")) {
+	    return "text/body";
+	}
+	if (scope.equals("back")) {
+	    return "text/back";
+	}
+	if (scope.equals("full")) { // is this ever used???
+	    return "";
+	}
+	return scope;
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XMLRetrieve.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XMLRetrieve.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XMLRetrieve.java	(revision 21112)
@@ -0,0 +1,446 @@
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Attr;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+
+// General Java classes
+import java.io.File;
+import java.util.Vector;
+import java.util.HashMap;
+
+
+public class XMLRetrieve extends ServiceRack {
+
+    protected static final String CONTENT_SERVICE = "DocumentContentRetrieve";
+    protected static final String METADATA_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String STRUCTURE_SERVICE = "DocumentStructureRetrieve";
+    
+    protected String toc_xsl_name = "";
+    protected String document_encoding = "";
+    protected String document_root_tag = "";
+    
+    protected Element collection_doc_list = null;
+    
+    protected boolean provide_content = true;
+    protected boolean provide_structure = true;
+    protected boolean provide_metadata = true;
+    
+
+    public boolean configure(Element info, Element extra_info) {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+	System.out.println("configuring XMLRetrieve...");
+	// look for the parameters
+	Element param_list = (Element)GSXML.getChildByTagName(info, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap params;
+	String services_to_provide = "";
+	if (param_list != null) {
+	    params = GSXML.extractParams(param_list, false);
+	    this.toc_xsl_name = (String)params.get("tocXSLT");
+	    this.document_encoding = (String)params.get("documentEncoding");
+	    this.document_root_tag = (String)params.get("documentRootTag");
+	    services_to_provide = (String)params.get("provideServices");
+	}
+	if (this.toc_xsl_name == null || this.toc_xsl_name.equals("")) {
+	    this.toc_xsl_name = "default_toc";
+	}
+	this.toc_xsl_name  = this.toc_xsl_name+".xsl";
+	
+	if (this.document_encoding == null || this.document_encoding.equals("")) {
+	    this.document_encoding = "UTF-8";
+	}
+	
+	if (services_to_provide != null && !services_to_provide.equals("")) {
+	    if (services_to_provide.indexOf("content")==-1) {
+		provide_content = false;
+	    }
+	    if (services_to_provide.indexOf("metadata")==-1) {
+		provide_metadata = false;
+	    }
+	    if (services_to_provide.indexOf("structure")==-1) {
+		provide_structure = false;
+	    }
+	    
+	}
+	
+	// set up short_service_info_ - for now just has name and type
+	Element retrieve_service;
+	if (provide_content) {
+	    retrieve_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, CONTENT_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	if (provide_metadata) {
+	    retrieve_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, METADATA_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	if (provide_structure) {
+	    retrieve_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, STRUCTURE_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	// find the doc list from the extra_info and keep it - should this be in collect.cfg or build.cfg??
+	collection_doc_list = (Element)GSXML.getChildByTagName(extra_info, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+
+	GSEntityResolver resolver = new GSEntityResolver();
+	resolver.setClassLoader(this.class_loader);
+	this.converter.setEntityResolver(resolver);
+	return true;
+    }
+
+    // this may get called but is not useful in the case of retrieve services
+    protected Element getServiceDescription(String service_id, String lang, String subset) {
+
+	Element retrieve_service = this.doc.createElement(GSXML.SERVICE_ELEM);
+	retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	retrieve_service.setAttribute(GSXML.NAME_ATT, service_id);
+	return retrieve_service;
+    }
+    
+    protected Element processDocumentContentRetrieve(Element request) {
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, CONTENT_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    return result;
+	}
+	Element result_doc_list = (Element)this.doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	for (int i=0; i<docs.getLength(); i++) {
+
+	    Element doc = (Element)docs.item(i);
+	    Element content = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	    doc.appendChild(content);
+
+	    String node_id = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    String doc_name = getWorkName(node_id);
+	    
+	    Element doc_elem = loadDocument(doc_name); // should perhaps cache the read in docs??
+	    if (doc_elem == null) {
+		continue;
+	    } 
+
+	    
+	    // if we have asked for the whole doc, just append it
+	    if (doc_name.equals(node_id)) {
+		content.appendChild(this.doc.importNode(doc_elem, true));
+		continue;
+	    }
+		   
+	    // else we only want a sub section
+	    
+	    Element section = getSection(doc_elem, node_id);
+	    if (section != null) {
+		content.appendChild(this.doc.importNode(section, true));
+	    }
+	    
+	} // for each doc
+
+	return result;
+	
+    }
+    
+    protected Element processDocumentStructureRetrieve(Element request) {
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, STRUCTURE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    System.err.println("XMLRetrieve.DocumentStructureRetrieve: no documents specified in the request. ");
+	    return result;
+	}
+
+	Element result_doc_list = (Element)this.doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+	// first look for the stylesheet in the collection
+	File stylesheet = new File(GSFile.collStylesheetFile(this.site_home, this.cluster_name, this.toc_xsl_name));
+	if (!stylesheet.exists()) {
+	    // now try in the site
+	    stylesheet = new File(GSFile.siteStylesheetFile(this.site_home, this.toc_xsl_name));
+	}
+	if (!stylesheet.exists()) {
+	    System.err.println("XMLRetrieve.DocumentStructureRetrieve: couldn't find the stylesheet file to produce the table of contents:"+stylesheet.getPath());
+	    return result;
+	}
+
+	// for now, we dont have any params, and we always return the structure of the whole document
+	
+	XMLTransformer transformer = new XMLTransformer();
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+
+	for (int i=0; i<docs.getLength(); i++) {
+
+	    Element doc = (Element)docs.item(i);
+	    
+	    Element structure = this.doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+	    doc.appendChild(structure);
+	    String doc_name = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    // make sure we are at the top level
+	    doc_name = getWorkName(doc_name);
+	    
+	    File doc_file = new File(GSFile.collectionIndexDir(this.site_home, this.cluster_name)+File.separator+"text"+File.separatorChar+doc_name+".xml");
+	    
+	    if (!doc_file.exists()) {
+		System.err.println("XMLRetrieve.DocumentStructureRetrieve: couldn't find file in coll "+this.cluster_name +", file "+doc_name+".xml");
+	    } else {
+		try {
+		    Node toc = transformer.transform(stylesheet, doc_file);
+		    structure.appendChild(this.doc.importNode(toc, true));
+		} catch (Exception e) {
+		    System.err.println("XMLRetrieve.DocumentStructureRetrieve: couldn't transform the document to get the toc");
+		}
+	    }
+	    
+	}
+	
+	return result;
+	
+    }
+    
+    // this just extracts a bit of text from the section to use as the Title
+    // this should be overwritten for any format that  has something more suitable
+    protected Element processDocumentMetadataRetrieve(Element request) {
+	Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, METADATA_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    System.err.println("XMLRetrieve.DocumentMetadataRetrieve: no documents in the request");
+	    return result;
+	}
+
+	Element result_doc_list = (Element)this.doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    System.err.println("XMLRetrieve.DocumentMetadataRetrieve: no metadata in the request");
+	    return result;
+	}
+	
+	Vector meta_name_list = new Vector();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = (Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		meta_name_list.add(metadata);
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+		
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	for (int i=0; i<docs.getLength(); i++) {
+	    Element doc = (Element)docs.item(i);
+	    String node_id = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    String doc_name = getWorkName(node_id);
+	    
+	    Element metadata_list = getMetadata(node_id, all_metadata, meta_name_list); 
+	    doc.appendChild(metadata_list);
+	}
+	
+	return result;
+    }
+    
+    protected Element loadDocument(String doc_name) {
+	// try to find the document
+	File doc_file = new File(GSFile.collectionIndexDir(this.site_home, this.cluster_name)+File.separator+"text"+File.separatorChar+doc_name+".xml");
+	    
+	if (!doc_file.exists()) {
+	    System.out.println("XMLRetrieve.loadDocument: couldn't find file in coll "+this.cluster_name +", file "+doc_name+".xml");
+	    return null;
+	}
+	
+	Document the_doc = null;
+	try {
+	    the_doc = this.converter.getDOM(doc_file, this.document_encoding);
+	} catch (Exception e) {
+	    System.err.println("XMLRetrieve.loadDocument: couldn't create a DOM from file "+doc_file.getPath());
+	    return null;
+	}
+	
+	return the_doc.getDocumentElement();
+
+    }
+
+    
+    protected Element getSection(Element doc_elem, String node_id) {
+	String [] bits = node_id.split("\\.");
+	if (bits.length > 4) {
+	    System.err.println("XMLRetrieve.getSection: badly formatted node id ("+node_id +"), cant retrieve the section");
+	    return null;
+	}
+	    
+	String id="";
+	String tagname = "";
+	String scope = "";
+	if (bits.length==2) {
+	    tagname = bits[1];
+	} else { 
+	    scope = bits[1];
+	    tagname = bits[2];
+	    
+	    if (bits.length == 4) {
+		id = bits[3];
+	    }
+	}
+	scope = translateScope(scope);
+	Element top=null;
+	if (!scope.equals("")) {
+	    top = (Element)GSXML.getNodeByPath(doc_elem, scope);
+	    if (top == null) {
+		// something gone wrong
+		return null;
+	    }
+	} else {
+	    top = doc_elem;
+	}
+	
+	NodeList elements = top.getElementsByTagName(tagname);
+	if (elements.getLength() == 0) {
+	    return null;
+	}
+	// no id, just return the first one
+	if (id.equals("")) {
+	    return (Element)elements.item(0);
+	}
+	// have an id, need to check and find the right one.
+	for (int i=0; i<elements.getLength();i++) {
+	    Element e = (Element)elements.item(i);
+	    if (e.getAttribute("gs3:id").equals(id)) {
+		return e;
+	    }
+	}
+	return null;
+	
+    }
+
+    protected Element getMetadata(String node_id, boolean all, Vector meta_name_list) {
+
+	// our default strategy here is to only return Title and root:Title
+	// ignore all others 
+	// the title of a section is just a little bit of the text inside it.
+	// the root_Title is the title from the doc info in the config file
+	Element metadata_list = this.doc.createElement(GSXML.METADATA_ELEM+ GSXML.LIST_MODIFIER);
+	String doc_name = getWorkName(node_id);
+	boolean node_is_root = false;
+	if (doc_name.equals(node_id)) {
+	    node_is_root = true;
+	}
+	
+	Element this_doc = GSXML.getNamedElement(this.collection_doc_list, GSXML.DOCUMENT_ELEM, GSXML.NAME_ATT, doc_name);
+	Element doc_meta_list  = (Element) GSXML.getChildByTagName(this_doc, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	
+	boolean get_section_title = false;
+	
+	if (all) {
+	    if (node_is_root) {
+	    return (Element)this.doc.importNode(doc_meta_list, true);
+	    } else {
+		get_section_title = true;
+	    }
+	    
+	} else {
+	    // have to process metadata one by one
+	    for (int i=0; i<meta_name_list.size(); i++) {
+		String meta_name = (String) meta_name_list.elementAt(i);
+		String actual_meta_name = meta_name;
+		if (meta_name.startsWith("root_")) {
+		    actual_meta_name = meta_name.substring(5);
+		} else {
+		    // its a section level one - check to see if doc is root
+		    if (!node_is_root) {
+			if (meta_name.equals("Title")) {
+			    get_section_title = true;
+			}
+			continue; // move on to teh next metadata
+		    }
+		}
+		
+		// here, we look for the specific meta elem in doc_meta_list
+		Element meta_item = GSXML.getNamedElement(doc_meta_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, actual_meta_name);
+		if (meta_item != null) {
+		    meta_item = (Element)this.doc.importNode(meta_item, true);
+		    meta_item.setAttribute(GSXML.NAME_ATT, meta_name);
+		    metadata_list.appendChild(meta_item);
+		}
+	    } // for each metadata
+	}
+
+	// now we have processed all teh doc metadata, just have section one to go, if needed
+	if (get_section_title) {
+	    
+	    Element doc_elem = loadDocument(doc_name);
+	    if (doc_elem != null) {
+		Element section = getSection(doc_elem, node_id);
+		if (section != null) {
+		    Element title_meta = extractTitleMeta(section);
+		    if (title_meta != null) {
+			metadata_list.appendChild(title_meta);
+		    }
+		}
+	    }
+		
+	}
+	return metadata_list;
+    }
+    
+    protected Element extractTitleMeta(Element section) {
+	Element meta_elem = this.doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+
+	String title = "dummy title";
+	Text t = this.doc.createTextNode(title);
+	meta_elem.appendChild(t);
+	return meta_elem;
+
+    }
+    // some methods for handling nodeIDs - they may be different for different colls, so they can be overwritten
+
+    // the full default nodeID looks like work.scope.tag.id
+    // the shorter versions are work, work.tag, work.scope.tag
+    protected String getWorkName(String node_id) {
+	int pos = node_id.indexOf('.');
+	if (pos == -1) {
+	    return node_id;
+	}
+	return node_id.substring(0, pos);
+    }
+    
+    // this assumes that the scope refers to a top level node - this may be overwritten if the scope bit in the id is a shorthand of some sort
+    protected String translateScope(String scope) {
+	if (this.document_root_tag != null) {
+	    return GSPath.appendLink(this.document_root_tag, scope);
+	}
+	return scope;
+    }
+    
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XSLTServices.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XSLTServices.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/service/XSLTServices.java	(revision 21112)
@@ -0,0 +1,264 @@
+/*
+ *    XSLTServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.selfContained.*;
+
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Map;
+import java.util.Iterator;
+import java.util.Locale;
+import java.io.*;
+import java.net.URLEncoder;
+import java.net.URLDecoder;
+
+/**
+ * A ServiceRack class for XSLT query. 
+ *
+ * Document ids are formed by encoding the document using standard URLEncoding
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @author <a href="mailto:s.yeates@cs.waikato.ac.nz">Stuart Yeates</a>
+ * @version $Revision$
+ * @see ServiceRack
+ * @see java.net.URLEncoder
+ * @see java.net.URLDecoder
+ */
+
+public class XSLTServices 
+  extends ServiceRack {
+  
+  // these strings must match what is found in the properties file
+  // the services on offer
+  private static final String XSLT_QUERY_SERVICE        = "XSLTQuery";
+  private static final String RESOURCE_RETRIEVE_SERVICE = "ResourceRetrieve";
+  // params used
+  private static final String QUERY_PARAM        = "query";
+  private static final String ALPHABET_PARAM     = "alphabet";
+  
+  private static final int RESULT_SET_SIZE        = 10;
+
+  // alphabets
+  private static final String MINIMUM_ALPHABET   = "minimum";
+  private static final String ASCII_ALPHABET     = "ascii";
+  private static final String UNICODE_ALPHABET   = "unicode";
+
+  private static final String DEFAULT_QUERY_STRING   =
+    "<xsl:stylesheet version=\"1.0\""+
+    "                xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">"+
+    "    <xsl:template match=\"*|/\">"+
+    "         <xsl:apply-templates/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"*|/\" mode=\"m\">"+
+    "         <xsl:apply-templates mode=\"m\"/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"text()|@*\">"+
+    "         <xsl:value-of select=\".\"/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"processing-instruction()|comment()\"/>"+
+    "</xsl:stylesheet>";
+  
+
+  private String alphabet = MINIMUM_ALPHABET;
+  private Element config_info_ = null;
+  
+
+  public XSLTServices() {
+  }
+  
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info) {
+      	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+      System.out.println("configuring XSLTServices");
+      this.config_info = info;
+    try {
+      System.out.println("XSLTService::configure() called with:");
+      TransformerFactory transformerFactory = TransformerFactory.newInstance();
+      Element doc = (Element)info.getFirstChild();
+      Transformer transformer = transformerFactory.newTransformer();
+      StreamResult result = new StreamResult(System.out);
+      Source source = new DOMSource(doc);
+      transformer.transform(source,result);
+    } catch (Throwable t) {
+      System.err.println("Error printing XML in XSLTService::configure()");
+    }
+    return true;    
+  }
+
+    protected Element getServiceDescription(String service, String lang, String subset) {
+
+	return null;
+    }
+  
+  /** process method for specific services - must be implemented by all
+   * subclasses
+   * should implement all services supported by the servicesImpl except
+   * for describe, which is implemented by this base class
+   *
+   */
+  protected Element processService(String service, Element request){
+    if (service.equals(XSLT_QUERY_SERVICE)) {
+      return processXSLTQuery(request);
+    } else if (service.equals(RESOURCE_RETRIEVE_SERVICE)) {
+      return processResourceRetrieve(request);
+    }
+    // create the response so we can report the error
+    Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, RESOURCE_RETRIEVE_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+    
+    System.err.println("XSLTServices:should never get here. service type wrong:"+service);
+    GSXML.addError(this.doc, response,"XSLTServices:should never get here. service type wrong:"+service);
+    return response;
+  }
+
+  /** process a document resquest query */
+  protected Element processResourceRetrieve(Element request) {
+    
+    // create the result and set the path so we know where we are
+    Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, RESOURCE_RETRIEVE_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+    // get param list 
+    Element param_elem=null;
+    Node n = request.getFirstChild();
+    while (n!=null) {
+	String node_name = n.getNodeName();
+	if (node_name.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    param_elem = (Element)n;
+	    break;
+	}
+	n = n.getNextSibling();
+    }
+
+    if (param_elem==null) { 
+	System.err.println("bad query request");
+	GSXML.addError(this.doc, response,"bad query request in XSLTServices");
+	return response; 
+    }
+    
+    // Documents are just the ids decoding using standard URL decoding
+    
+    Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+    String []ids = GSXML.getAttributeValuesFromList(doc_list, GSXML.NAME_ATT);
+    for (int j=0; j<ids.length; j++) {
+      String document = null;
+      try {
+	document = URLDecoder.decode(ids[j],"UTF-8");
+      } catch (UnsupportedEncodingException e) {
+	throw new Error(e.toString());
+      }
+      // something funny with the doc - 
+      Element new_doc = this.doc.createElement(GSXML.DOCUMENT_ELEM);
+      new_doc.setAttribute(GSXML.NAME_ATT, ids[j]); //GSXML.createDocumentElement(this.doc, ids[j]);
+      GSXML.addDocText(this.doc, new_doc, document);
+      response.appendChild(new_doc);
+    }
+    
+    return response;
+  }
+  
+  /** process a XSLT query */
+  protected Element processXSLTQuery(Element request) {
+    
+    // create the result and set the path so we know where we are
+    Element response = this.doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, XSLT_QUERY_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+    // get the parameter list
+    Element param_list = 
+      (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+
+    // extract the only parameter we care about
+    HashMap params = GSXML.extractParams(param_list, false);
+    String query = (String)params.get(QUERY_PARAM);
+    
+    if (query == null || query.equals(""))
+      query = DEFAULT_QUERY_STRING;
+    
+    
+    DocumentStream stream = null;
+    if (alphabet.equals(MINIMUM_ALPHABET)){
+      stream = new GeneratedDocumentStream('a','c');
+    } else if(alphabet.equals(ASCII_ALPHABET)){
+      stream = new GeneratedDocumentStream();
+    } else if(alphabet.equals(UNICODE_ALPHABET)){
+      stream = new GeneratedDocumentStream(Character.MIN_VALUE,
+					   Character.MAX_VALUE);
+    } else {
+      System.err.println("XSLTServices: bad alphabet name : "+ alphabet);
+      GSXML.addError(this.doc, response,"XSLTServices: bad alphabet name : "+ alphabet);
+      stream = new GeneratedDocumentStream();
+    }
+
+    Element resource_list = this.doc.createElement(GSXML.RESOURCE_ELEM+GSXML.LIST_MODIFIER);
+    response.appendChild(resource_list);
+
+    // Framework to stringise the document
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+
+    // add each resource
+    for (int d=0; d<RESULT_SET_SIZE; d++) {
+      try {
+	Document document = stream.nextDocument();
+	Element doc = (Element)document.getFirstChild();
+	Transformer transformer = transformerFactory.newTransformer();
+	StringWriter writer = new StringWriter();
+	StreamResult result = new StreamResult(writer);
+	Source source = new DOMSource(doc);
+	transformer.transform(source,result);
+	String id = writer.toString();
+	Element e = this.doc.createElement(GSXML.DOCUMENT_ELEM);
+	e.setAttribute(GSXML.NAME_ATT, id); 
+	//Node no = GSXML.createDocumentElement(this.doc, id);
+	resource_list.appendChild(e);
+      } catch (Throwable t) {
+	GSXML.addError(this.doc, response, "Error in XSLTServices finding results:" + t.toString());
+      }
+    }
+    return response; 
+  }
+}  
+ 
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/CollectionClassLoader.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/CollectionClassLoader.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/CollectionClassLoader.java	(revision 21112)
@@ -0,0 +1,45 @@
+/*
+ *    CollectionClassLoader.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.net.URL;
+import java.io.File;
+
+/** Looks for classes/resources in the collection resources directory 
+ */
+public class CollectionClassLoader
+    extends ClassLoader {
+
+    String base_dir = null;
+    public CollectionClassLoader(ClassLoader parent, String site_home, String collection_name) {
+	super(parent);
+	this.base_dir = GSFile.collectionResourceDir(site_home, collection_name);
+    } 
+
+    public URL findResource(String name) {
+	File resource_path = new File(this.base_dir, name);
+	try {
+	    if (resource_path.exists()) {
+		return new URL("file://"+resource_path.getAbsolutePath());
+	    }
+	} catch (Exception e) {};
+
+	return super.findResource(name);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/DBInfo.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/DBInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/DBInfo.java	(revision 21112)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.util;
+
+import java.util.Vector;
+import java.util.HashMap;
+import java.util.Set;
+/** class to hold info from a gdbm (or equiv) query
+ * maps a String key to a list of Strings (values)
+ * at the moment, the user must know if something has a single value or not
+ */
+public class DBInfo {
+
+    protected HashMap info_map_;
+
+    public DBInfo() {
+	info_map_ = new HashMap();
+    }
+
+    // methods for keys that can have a single value
+
+    /** set the value for a key - replaces any existing value */
+    public void setInfo(String key, String value) {
+	Vector v = new Vector();
+	v.add(value);
+	info_map_.put(key, v);
+    }
+
+    /** get the value - used for keys that have one value, otherwise just
+     * returns the first one */
+    public String getInfo(String key) {
+	Vector items = (Vector)info_map_.get(key);
+	if (items==null) {
+	    return "";
+	}
+	return (String)items.firstElement();
+    }
+
+    // methods for keys that can have multiple values
+    
+    /** add a value to a key - for keys that can have multiple values */
+    public void addInfo(String key, String value) {
+	Vector v = (Vector)info_map_.get(key);
+	if (v==null) {
+	    v = new Vector();
+	}
+	v.add(value);
+	info_map_.put(key, v);
+    }
+    /** return a vector of all values associated with a key 
+     * @return Vector of Strings
+     */
+    public Vector getMultiInfo(String key) {
+	return (Vector)info_map_.get(key);
+    }
+
+    /** returns a list of all the keys in the info */
+    public Set getKeys() {
+	return info_map_.keySet();
+    }
+    /** returns the Info as a string */
+    public String toString() {
+	return info_map_.toString();
+
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Dictionary.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Dictionary.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Dictionary.java	(revision 21112)
@@ -0,0 +1,136 @@
+/*
+ *    Dictionary.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ResourceBundle;
+import java.util.Locale;
+import java.util.Enumeration;
+
+public class Dictionary {
+
+    /** The locale of this dictionary. */
+    protected Locale locale = null;
+    
+    /** The resource that has been loaded */
+    protected String resource = null;
+
+    /** The ResourceBundle which contains the raw key-value mappings. Loaded from a file named "resource_locale.properties*/
+    private ResourceBundle raw = null;
+
+    /**  Constructs the Dictionary class by first creating a locale from the specified language, (or using the default locale if this didn't work), then loading a resource bundle based on this locale. 
+     */
+    public Dictionary(String resource, String lang) {
+	// Initialize.
+	
+	this.locale = new Locale(lang);
+	this.resource = resource;
+	if (this.locale == null) {
+	    this.locale = Locale.getDefault();
+	}
+	try {
+	    this.raw = ResourceBundle.getBundle(this.resource, this.locale);
+	} catch (Exception e) {
+	    //System.err.println("Dictionary: couldn't locate a resource bundle for "+resource);
+	}
+    }
+
+    /**  Constructs the Dictionary class by first creating a locale from the specified language, (or using the default locale if this didn't work), then loading a resource bundle based on this locale. A classloader is specified which can be used to find the resource.
+     */
+    public Dictionary(String resource, String lang, ClassLoader loader) {
+	// Initialize.
+	
+	this.locale = new Locale(lang);
+	this.resource = resource;
+	if (this.locale == null) {
+	    this.locale = Locale.getDefault();
+	}
+	// try the specified class loader
+	try {
+	    this.raw = ResourceBundle.getBundle(this.resource, this.locale, loader);
+	    return;
+	} catch (Exception e) {};
+	
+	try {
+	    this.raw = ResourceBundle.getBundle(this.resource, this.locale);
+	} catch (Exception ex) {
+	    //System.err.println("Dictionary: couldn't locate a resource bundle for "+resource);
+	}
+	
+    }
+
+    
+    public Enumeration getKeys() {
+	if (this.raw != null) {
+	    return this.raw.getKeys();
+	}
+	return null;
+    }
+    /** Overloaded to call get with both a key and an empty argument array.
+     * @param key A <strong>String</strong> which is mapped to a initial String within the ResourceBundle.
+     * @return A <strong>String</strong> which has been referenced by the key String and that  contains no argument fields.
+     */
+    public String get(String key) {
+	return get(key, null);
+    }
+    
+    /** Used to retrieve a property value from the Locale specific ResourceBundle, based upon the key and arguments supplied. If the key cannot be found or if some other part of the call fails a default (English) error message is returned. <BR>
+     * Here the get recieves a second argument which is an array of Strings used to populate argument fields, denoted {<I>n</I>}, within the value String returned.
+     * @param key A <strong>String</strong> which is mapped to a initial String within the ResourceBundle.
+     * @param args A <strong>String[]</strong> used to populate argument fields within the complete String.
+     * @return A <strong>String</strong> which has been referenced by the key String and that either contains no argument fields, or has had the argument fields  populated  with argument Strings provided in the get call.
+     */
+    public String get(String key, String args[]) {
+	if (this.raw == null) {
+	    return null;
+	}
+	try {
+	    String initial_raw = this.raw.getString(key);
+	    // convert to unicode, copied from gatherer dictionary
+	    String initial = new String(initial_raw.getBytes("ISO-8859-1"), "UTF-8");
+	    // If the string contains arguments we have to insert them.
+	    String complete = "";
+	    // While we still have initial string left.
+	    while(initial.length() > 0 && initial.indexOf('{') != -1 && initial.indexOf('}') != -1) {
+		// Remove preamble
+		int opening = initial.indexOf('{');
+		int closing = initial.indexOf('}');
+		complete = complete + initial.substring(0, opening);
+		// Parse arg_num
+		String arg_str = initial.substring(opening + 1, closing);
+		int arg_num = Integer.parseInt(arg_str);
+		if(closing + 1 < initial.length()) {
+		    initial = initial.substring(closing + 1);
+		}
+		else {
+		    initial = "";
+		}
+		// Insert argument
+		if(args != null && 0 <= arg_num && arg_num < args.length) {
+		    complete = complete + args[arg_num];
+		}
+	    }
+	    return complete + initial;
+	}
+	catch (Exception e) {
+	    //System.err.println("Dictionary Error: couldn't find string for key:" + key +" in resource "+this.resource);
+	    return null;
+	}
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GDBMWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GDBMWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GDBMWrapper.java	(revision 21112)
@@ -0,0 +1,239 @@
+package org.greenstone.gsdl3.util;
+
+import au.com.pharos.gdbm.GdbmFile;
+import au.com.pharos.packing.*;
+import au.com.pharos.gdbm.GdbmException;
+
+/** java wrapper class for gdbm - uses Java-GDBM written by Martin Pool
+ * replaces gdbmclass in the old version
+ */
+
+public class GDBMWrapper {
+
+    // Values must match those from gdbm.h - uses the definitions from GdbmFile !
+    /** Indicates that the caller will just read the database.  Many
+     * readers may share a database. */
+    public final static int READER = GdbmFile.READER;
+ 
+    /** The caller wants read/write access to an existing database and
+     * requires exclusive access. */
+    public final static int WRITER = GdbmFile.WRITER;
+ 
+    /** The caller wants exclusive read/write access, and the database
+     * should be created if it does not already exist. */
+    public final static int WRCREAT = GdbmFile.WRCREAT;
+ 
+    /** The caller wants exclusive read/write access, and the database
+     * should be replaced if it already exists. */
+    public final static int NEWDB = GdbmFile.NEWDB;
+
+    protected GdbmFile db_=null;
+
+    /** open the database filename, with mode mode - uses the constants
+	above, eg GdbmFile.WRITER */
+    public boolean openDatabase(String filename, int mode){
+	try {
+	    if (db_!=null) {
+		db_.close();
+	    }
+	    db_ = new GdbmFile(filename, mode);
+	} catch ( GdbmException e) { // the database wasn't opened or created
+	    System.err.println("GDBMWrapper: couldn't open database "+filename);
+	    return false;
+	} 
+	db_.setKeyPacking(new StringPacking());
+	db_.setValuePacking(new StringPacking());
+	return true;
+    }
+
+    /** cloase the database associated with this wrapper */
+    public void closeDatabase() {
+	try {
+	    if (db_ != null) {
+		db_.close();
+		db_ = null;
+	    }
+	} catch (GdbmException e) {
+	    // should never get here - close never actually throws an exception
+	    System.err.println("GDBMWrapper: error on close()");
+	}
+    }
+
+    /** returns a DBInfo object containing all the name-value pairs for 
+     * main_key 
+     * @see DBInfo
+     */
+    public DBInfo getInfo(String main_key) {
+	if (db_==null) {
+	    return null;
+	}
+	String s_info;
+	try {
+	    s_info = (String)db_.fetch(main_key);
+	} catch (GdbmException e) {
+	    System.err.println("GDBMWrapper: couldn't get record");
+	    return null;
+	}
+	if (s_info==null) {
+	    // record not present
+	    System.err.println("GDBMWrapper: key "+main_key+" not present in db");
+	    return null;
+	}
+	DBInfo info = new DBInfo();
+
+	String [] lines = s_info.split("\n");
+	String key;
+	String value;
+	for (int i=0; i<lines.length; i++) {
+	    //System.out.println("line:"+lines[i]);
+	    int a = lines[i].indexOf('<');
+	    int b= lines[i].indexOf('>');
+	    if (a==-1 || b==-1) {
+		System.err.println("error: bad format in db");
+	    }
+	    else {
+		key=lines[i].substring(a+1, b);
+		value=lines[i].substring(b+1);
+		//System.out.println("key="+key+", val="+value);
+		info.addInfo(key, value);
+
+	    }
+	}
+	return info;
+    }
+
+    /** sets all the name-value pairs in info as the content for key key 
+     * TODO - not implemented yet */
+    public boolean setInfo(String key, DBInfo info) {
+	if (db_==null) {
+	    return false;
+	}
+	return true;
+    }
+    /** sets the key value as info 
+     * TODO - not implemented yet */
+    public boolean setInfo (String key, String info) {
+	if (db_==null) {
+	    return false;
+	}
+	return true;
+    }
+    /** deletes the entry for key 
+     * TODO - not implemented yet */
+    public boolean deleteKey(String key) {
+	if (db_==null) {
+	    return false;
+	}
+	return true;
+    }
+
+    // greenstone convenience methods - should these go into a separate class?
+    // yes I think so.
+    /** converts a greenstone OID to internal docnum */
+    public long OID2Docnum(String OID) {
+	DBInfo info = getInfo(OID);
+	if (info != null) {
+	    long real_num = Long.parseLong(info.getInfo("docnum"));
+	    return real_num;
+	}
+	return -1;
+    }
+    /** converts a docnum to greenstone OID */
+    public String docnum2OID(long docnum) {
+	DBInfo info = getInfo(Long.toString(docnum));
+	String oid = info.getInfo("section");
+	return oid;
+    }
+
+    /** converts an external id to greenstone OID */
+    public String externalId2OID(String extid) {
+	DBInfo info = getInfo(extid);
+	if (info != null) {
+	    String oid = info.getInfo("section");
+	    return oid;
+	}
+	return null;
+    }
+    /** translates relative oids into proper oids:
+     * .pr (parent), .rt (root) .fc (first child), .lc (last child),
+     * .ns (next sibling), .ps (previous sibling) 
+     * a suffix is expected to be present so test before using 
+     * TODO: fc, lc, ns, ps, .rt .ss*/
+    public String translateOID(String oid) {
+
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    System.out.println("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	
+	String top = oid.substring(0, p);
+	String suff = oid.substring(p+1);
+	if (suff.equals("pr")) {
+	    return OID.getParent(top);
+	} else if (suff.equals("rt")) {
+	    return OID.getTop(top);
+	} else {
+	    int sibling_num = 0;
+	    if (suff.equals("ss")) {
+		// we have to remove the sib num before we get top
+		p = top.lastIndexOf('.');
+		sibling_num = Integer.parseInt(top.substring(p+1));
+		top = top.substring(0, p);
+	    }
+	    
+	    // need to get info out of gdbm db -
+	    String doc_id = top;
+	    if (suff.endsWith("s")) {
+		doc_id = OID.getParent(top);
+	    }
+	    DBInfo info = getInfo(doc_id);
+	    if (info==null) {
+		System.out.println("info is null!!");
+		return top;
+	    }
+
+
+	    /** ATTACK OF THE RANDOM BLARG!!!! **/
+	    /** All your code are belong to me! **/
+
+	    String contains = info.getInfo("contains");
+	    if (contains.equals("")) {
+		// something is wrong
+		return top;
+	    }
+	    contains = contains.replaceAll("\"", doc_id);
+	    String [] children = contains.split(";");
+	    if (suff.equals("fc")) {
+		return children[0];
+	    } else if (suff.equals("lc")) {
+		return children[children.length-1];
+	    } else {
+		if (suff.equals("ss")) {
+		    return children[sibling_num-1];
+		}
+		// find the position that we are at.
+		int i=0;
+		while (i<children.length) {
+		    if (children[i].equals(top)) {
+			break;
+		    }
+		    i++;
+		}
+		
+		if (suff.equals("ns")) {
+		    if (i==children.length-1) {
+			return children[i];
+		    }
+		    return children[i+1];
+		} else if (suff.equals("ps")) {
+		    if (i==0) {
+			return children[i];
+		    }
+		    return children[i-1];
+		}
+	    }
+	}
+	return top;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS2MacroResolver.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS2MacroResolver.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS2MacroResolver.java	(revision 21112)
@@ -0,0 +1,102 @@
+/*
+ *    GS2MacroResolver.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+public class GS2MacroResolver 
+    extends MacroResolver 
+{
+
+    protected GDBMWrapper gdbm_src = null;
+    
+    // need to make it not add macros if they are already present
+    public GS2MacroResolver(GDBMWrapper gdbm) {
+	gdbm_src = gdbm;
+    }
+
+    
+     public String resolve(String text, String lang, String scope, 
+			  String doc_oid) {
+	//System.err.println("resolving macros for "+text);
+	if (scope.equals(SCOPE_TEXT) && text_macros.size()==0) return text;
+	if (scope.equals(SCOPE_META) && metadata_macros.size() ==0) return text;
+
+	DBInfo node_info = null;
+	DBInfo root_info = null;
+	boolean new_lang = false;
+	if (this.lang == null || !this.lang.equals(lang) ) {
+	    new_lang = true;
+	    this.lang = lang;
+	} 
+
+	ArrayList macros;
+	if (scope.equals(SCOPE_TEXT)) {
+	    macros = text_macros;
+	} else {
+	    macros = metadata_macros;
+	}
+	
+	for (int i=0; i<macros.size(); i++) {
+	    Macro m = (Macro)macros.get(i);
+	    switch (m.type) {
+	    case TYPE_DICT:
+		if (m.text==null || new_lang) {
+		    Dictionary dict = new Dictionary(m.bundle, lang);
+		    m.text = dict.get(m.key, null);
+		}
+		// now drop through to text case
+	    case TYPE_TEXT:
+		text = text.replaceAll(m.macro, m.text);
+		break;
+	    case TYPE_META:
+		if (text.indexOf(m.macro) != -1) {
+		    if (node_info == null) {
+			node_info = gdbm_src.getInfo(doc_oid);
+			if (node_info == null) {
+			    break;
+			}
+		    }
+		    
+		    String value = node_info.getInfo(m.text);
+		    if (value != null) {
+			text = text.replaceAll(m.macro, value);
+		    } else {
+			// try the root node
+			if (root_info == null && !OID.isTop(doc_oid)) {
+			    root_info = gdbm_src.getInfo(OID.getTop(doc_oid));
+			}
+			if (root_info == null) break;
+			value = root_info.getInfo(m.text);
+			if (value != null) {
+			    text = text.replaceAll(m.macro, value);
+			}
+		    }
+		}
+		
+		break;
+	    } // switch
+	   
+	}
+	
+	return text;
+	
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS3OID.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS3OID.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GS3OID.java	(revision 21112)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.util;
+
+public class GS3OID extends OID {
+
+    /**  public static boolean isTop(String oid) {
+	if (oid.equals("")) {
+	    return false;
+	}
+	int pos = oid.indexOf('.');
+	if (pos == -1) { // its the top document, not the top structure
+	    return false;
+	}
+	pos = pod.indexOf('.', pos+1);
+	if (pos == -1) {
+	    return true; // there is only one dot
+	}
+	return false; // there is more than one dot
+		
+	}*/
+
+    public static boolean isDocTop(String oid) {
+	return oid.indexOf('-')==-1;
+    }
+
+    public static String createOID(String doc_id, String section_label) {
+	return doc_id + "-"+section_label;
+    }
+
+    public static String getDocID(String oid) {
+	if (isDocTop(oid)) {
+	    return oid;
+	}
+	return oid.substring(0, oid.indexOf('-'));
+    }
+    public static String getSectionLabel(String oid) {
+
+	if (isDocTop(oid)) {
+	    return "";
+	}
+	return oid.substring(oid.indexOf('-')+1);
+
+    }
+
+    // put this here for now - may need to go with SQL stuff later
+    public static String translateOID(String oid) {
+	
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    System.out.println("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	
+	String top = oid.substring(0, p);
+	String suff = oid.substring(p+1);
+	if (suff.equals("pr")) {
+	    return getParent(top);
+	} else if (suff.equals("rt")) {
+	    return getTop(top);
+	} 
+	return oid;
+
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSConstants.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSConstants.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSConstants.java	(revision 21112)
@@ -0,0 +1,15 @@
+package org.greenstone.gsdl3.util;
+
+/** these constants are used for servlet args, and config variables */
+public class GSConstants {
+
+    public static final String GSDL3_HOME = "gsdl3_home";
+    public static final String INTERFACE_NAME = "interface_name";
+    public static final String SITE_NAME = "site_name";
+    public static final String LIBRARY_NAME = "library_name";
+    public static final String DEFAULT_LANG = "default_lang";
+
+
+    public static final String META_RELATION_SEP = "_";
+    public static final String META_SEPARATOR_SEP = "*";
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSEntityResolver.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSEntityResolver.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSEntityResolver.java	(revision 21112)
@@ -0,0 +1,46 @@
+package org.greenstone.gsdl3.util;
+
+
+import org.xml.sax.InputSource;
+import org.xml.sax.EntityResolver;
+import java.io.File;
+import java.net.URL;
+
+// uses a class loader to find entities
+// The class loader to use can be set by setClassLoader(), otherwise it will use the class loader that loaded itself. For the Tomcat webapp, this will be the webapp class loader, not the system class loader. the webapp classloader knows about the classes in the WEB-INF/classes dir, the system classloader knows about the ones on the classpath. The system class loader is a parent to web app classloader, so this will be used as well.
+
+public class GSEntityResolver implements EntityResolver {
+
+    ClassLoader class_loader = null;
+    
+    public void setClassLoader(ClassLoader loader) {
+	this.class_loader = loader;
+    }
+    
+    public InputSource resolveEntity (String public_id, String system_id) {
+	
+	//System.out.println("entity resolver for "+system_id);
+	String temp_id = system_id;
+	if (temp_id.startsWith("file://")) {
+	    File f = new File(system_id);
+	    if (f.exists()) {
+		return new InputSource(system_id);
+	    } else {
+		temp_id = temp_id.substring(temp_id.lastIndexOf("/")+1);
+	    }
+	} else {
+	    if (temp_id.indexOf("/")!= -1) {
+		temp_id = temp_id.substring(temp_id.lastIndexOf("/")+1);
+	    }
+	}
+	// try using a class loader
+	if (this.class_loader==null) {
+	    this.class_loader = this.getClass().getClassLoader();
+	}
+	URL url = class_loader.getResource(temp_id);
+	if (url == null) {
+	    return null;
+	}
+	return new InputSource("file://"+url.getFile());
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFile.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFile.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFile.java	(revision 21112)
@@ -0,0 +1,439 @@
+/*
+ *    GSFile.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.File; 
+//import org.apache.soap.encoding.soapenc.Base64;
+import org.apache.axis.encoding.Base64;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.lang.ClassLoader; // to find files on the class path
+import java.net.URL;
+import java.util.ArrayList;
+
+/**
+ * GSFile - utility class for Greenstone.
+ *
+ * all file paths are created here
+ * also has file utility methods
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ * @see File
+ */
+
+public class GSFile {
+
+    /**  site config file path */
+    static public String siteConfigFile(String site_home) {
+	return site_home + File.separatorChar+"siteConfig.xml";
+
+    }
+    /**  interface config file path */
+    static public String interfaceConfigFile(String interface_home) {
+	return interface_home + File.separatorChar+"interfaceConfig.xml";
+
+    }
+
+    /** collection directory path */
+    static public String collectDir(String site_home) {
+	return site_home+File.separatorChar+"collect";
+    }
+    
+    
+    /** collection config file path*/
+    static public String collectionConfigFile(String site_home, 
+					      String collection_name) {
+	return collectionConfigFile(collectionBaseDir(site_home, collection_name));
+    }
+    static public String collectionConfigFile(String collection_home) {
+	return collectionEtcDir(collection_home) +
+			File.separatorChar+"collectionConfig.xml";
+
+    }
+    /** collection init file path*/
+    static public String collectionInitFile(String site_home, 
+					    String collection_name) {
+	return site_home+File.separatorChar+"collect"+
+			File.separatorChar+collection_name+
+			File.separatorChar+"etc"+
+			File.separatorChar+"collectionInit.xml";
+
+    }
+
+    /** collection build config file path*/
+    static public String collectionBuildConfigFile(String site_home, 
+						   String collection_name ) {
+	return site_home+File.separatorChar+"collect"+
+			File.separatorChar+collection_name+
+			File.separatorChar+"index"+
+			File.separatorChar+"buildConfig.xml";
+    }
+    /** collection build config file path*/
+    static public String collectionBuildConfigFileBuilding(String site_home, 
+						   String collection_name ) {
+	return collectionBuildConfigFileBuilding(collectionBaseDir(site_home, collection_name));
+    }
+    static public String collectionBuildConfigFileBuilding(String collection_home ) {
+	return collection_home+
+	    File.separatorChar+"building"+
+	    File.separatorChar+"buildConfig.xml";
+    }
+
+    /** XML Transform directory path */
+    static public String xmlTransformDir(String interface_home) {
+	return interface_home+File.separatorChar+"transform";
+    }
+
+    /** collection base directory path */	 
+    static public String collectionBaseDir(String site_home,
+					   String collection_name) {
+	return site_home+File.separatorChar+"collect"+
+	    File.separatorChar+collection_name;
+    }
+    /** collection archive directory path */
+    static public String collectionArchiveDir(String site_home,
+					   String collection_name) {
+	return collectionArchiveDir(collectionBaseDir(site_home, collection_name)); 
+    }
+    static public String collectionArchiveDir(String collection_home) {
+	return collection_home+File.separatorChar+"archives";
+    }
+    
+    /** collection building directory path */	 
+    static public String collectionBuildDir(String site_home,
+					   String collection_name) {
+	return collectionBuildDir(collectionBaseDir(site_home, collection_name));
+    }
+    static public String collectionBuildDir(String collection_home) {
+	return collection_home+File.separator+"building";
+    }
+    /** collection building directory path */	 
+    static public String collectionEtcDir(String site_home,
+					   String collection_name) {
+	return collectionEtcDir(collectionBaseDir(site_home, collection_name));
+    }
+    static public String collectionEtcDir(String collection_home) {
+	return collection_home+File.separator+"etc";
+    }
+
+    /** collection building directory path */	 
+    static public String collectionImportDir(String site_home,
+					   String collection_name) {
+	return collectionImportDir(collectionBaseDir(site_home, collection_name));
+	    
+    }
+    static public String collectionImportDir(String collection_home) {
+	return collection_home + File.separatorChar+"import";
+    }
+    /** collection building directory path */	 
+    static public String collectionIndexDir(String site_home,
+					   String collection_name) {
+	return collectionIndexDir(collectionBaseDir(site_home, collection_name));
+	    
+    }
+    static public String collectionIndexDir(String collection_home) {
+	return collection_home + File.separatorChar+"index";
+    }
+    
+    /** text path (for doc retrieval) relative to collectionBaseDir */
+    static public String collectionTextPath(String index_stem) {
+	return "index"+File.separatorChar+"text"+File.separatorChar+
+	    index_stem;
+    }
+
+    /** index path (for querying) relative to collectionBaseDir */
+    static public String collectionIndexPath(String index_stem,
+					     String index_name) {
+	return "index"+File.separatorChar+index_name+File.separatorChar+
+	    index_stem;
+    }
+    
+    /** collection resources directory path */	 
+    static public String collectionResourceDir(String site_home,
+					       String collection_name) {
+	return collectionResourceDir(collectionBaseDir(site_home, collection_name));
+	    
+    }
+    static public String collectionResourceDir(String collection_home) {
+	return collection_home + File.separatorChar+"resources";
+    }
+
+    /** absolute path for an associated file */
+    static public String assocFileAbsolutePath(String site_home,
+					       String collection_name,
+					       String assoc_file_path,
+					       String filename) {
+	return collectionBaseDir(site_home, collection_name)+
+	    File.separatorChar+"index"+File.separatorChar+
+	    "assoc"+File.separatorChar+assoc_file_path+
+	    File.separatorChar+filename;
+    }
+
+    static public String siteHome(String gsdl3_home, String site_name) {
+	return gsdl3_home + File.separatorChar + "sites" + 
+	    File.separatorChar +site_name;
+    }
+
+    static public String interfaceHome(String gsdl3_home, 
+				       String interface_name) {
+	return gsdl3_home + File.separatorChar + "interfaces" + 
+	    File.separatorChar + interface_name;
+    }
+    
+    static public String siteStylesheetFile(String site_home,
+					    String filename) {
+	return site_home+File.separatorChar + "transform" +
+	    File.separatorChar + filename;
+    }
+    static public String collStylesheetFile(String site_home,
+					    String coll_name,
+					    String filename) {
+	return collectionBaseDir(site_home, coll_name) + File.separatorChar+
+	    "transform" + File.separatorChar + filename;
+    }
+	
+	
+    /** returns the absolute path to a stylesheet 
+     * stylesheets are looked for in the following places, in the 
+     * following order:
+     * current collection, current site, current interface, base interfaces
+     * returns null  if the file cannot be found 
+     *
+     * this is not so good because sites may be on a different computer */
+    static public String stylesheetFile(String gsdl3_home,
+					String site_name,
+					String collection,
+					String interface_name,
+					ArrayList base_interfaces,
+					String filename) {
+
+	String site_home = siteHome(gsdl3_home, site_name);
+	// try collection first
+	File stylesheet = null;
+	if (!collection.equals("")) {
+	    
+	    String coll_home = collectionBaseDir(site_home, collection);
+	    stylesheet = new File(coll_home +File.separatorChar+
+				  "transform"+File.separatorChar+filename);
+	    if (stylesheet.exists()) {
+		return stylesheet.getPath();
+	    }
+	}
+	
+	// try site one next	
+	stylesheet = new File(site_home +File.separatorChar+
+			      "transform"+File.separatorChar+filename);
+	if (stylesheet.exists()) {
+	    return stylesheet.getPath();
+	}
+	
+	// try current interface
+	String interface_home = interfaceHome(gsdl3_home, 
+					      interface_name);
+	stylesheet = new File(interface_home+File.separatorChar+
+			      "transform"+File.separatorChar+filename);
+	if (stylesheet.exists()) {
+	    return stylesheet.getPath();
+	}
+	// try base interface
+	if (base_interfaces==null || base_interfaces.size()==0) {
+	    return null; // no base interfaces to look for
+	}
+	for (int i=0; i<base_interfaces.size(); i++) {
+	    interface_home = interfaceHome(gsdl3_home, (String)base_interfaces.get(i));
+	    stylesheet = new File(interface_home+File.separatorChar+
+				  "transform"+File.separatorChar+filename);
+	    if (stylesheet.exists()) {
+		return stylesheet.getPath();
+	    }
+	}
+	    
+	// still can't find it and we have looked everywhere
+	return null;
+    }
+
+
+    /** base directory for phind data */
+    public static String phindBaseDir(String site_home, String coll_name,
+				      String phind_index) {
+	return site_home + File.separatorChar +
+	    "collect" + File.separatorChar +
+	    coll_name + File.separatorChar +
+	    "index" + File.separatorChar +
+	    "phind"+phind_index;
+    }
+
+    /** the gdbm database file - 
+     * note, need to change extension depending on OS */
+    static public String GDBMDatabaseFile(String site_home, 
+					  String collection_name,
+					  String index_stem) {
+
+	String db_ext = ".ldb";
+	if (Misc.isMac()) {
+	    db_ext = ".bdb";
+	}
+	return site_home + File.separatorChar +
+	    "collect" + File.separatorChar +
+	    collection_name + File.separatorChar +
+	    "index" + File.separatorChar + "text" + File.separatorChar +
+	    index_stem + db_ext;
+
+    }
+
+    // some file utility methods
+
+    /** read in a file and encode it using base64
+     * encoded data returned as a String */
+    static public String base64EncodeFromFile(String in_filename) {
+	byte [] data=null;
+	try {
+	    data = readFile(in_filename);
+	} catch (Exception e) {
+	    System.err.println("GSFile.base64EncodeFromFile: couldn't read the file");
+	}
+	String encodedString = Base64.encode(data);
+	return encodedString;
+
+    }
+
+    /** decode some base64 data, and write it to the specified file */
+    static public boolean base64DecodeToFile(String data, String out_filename) {
+	try {
+	    byte[] buffer=Base64.decode(data);
+	    writeFile(buffer, out_filename);
+	   
+	} catch (Exception e) {
+	    System.err.println("GSFile.base64DecodeToFile: file opening/closing errors"+e.getMessage());
+	    return false;
+	}
+	return true;
+	
+    }
+
+    /** read in a file to a byte array */
+    public static byte[] readFile(String filename) throws IOException {
+	File file = new File(filename);
+	BufferedInputStream bis = new BufferedInputStream(new
+	    FileInputStream(file));
+	int bytes = (int) file.length();
+	byte[] buffer = new byte[bytes];
+	int readBytes = bis.read(buffer);
+	bis.close();
+	return buffer;
+    }
+
+    /** write a byte array to a file */
+    public static void writeFile(byte [] buffer, String filename) throws IOException {
+	File file = new File(filename);
+	BufferedOutputStream bos = new BufferedOutputStream(new
+	    FileOutputStream(file));
+	bos.write(buffer);
+	bos.close();
+    }
+
+    public static boolean deleteFile(File f) {
+
+	if (f.isDirectory()) {
+	    File[] files = f.listFiles();
+	    for (int i=0; files!=null && i<files.length; i++) {
+		deleteFile(files[i]);
+	    }
+
+	}
+	// delete the file or directory
+	return f.delete();
+    }
+    
+    public static boolean copyFile(File source, File destination) {
+	if (!source.isFile()) {
+	    System.err.println("GSFile.copyFile(): "+source.getPath()+" is not a file!");
+	    return false;
+	}
+	try {
+	    destination.getParentFile().mkdirs();
+	    FileInputStream in = new FileInputStream(source);
+	    FileOutputStream out = new FileOutputStream(destination);
+	    int value = 0;
+	    while((value = in.read()) != -1) {
+		out.write(value);
+	    }
+	    in.close();
+	    out.close();
+	} catch (Exception e) {
+	    System.err.println("GSFile.copyFile(): something went wrong copying "+source.getPath()+" to "+destination.getPath());
+	    System.err.println("Exception: "+e.getMessage());
+	    return false;
+	}
+
+	return true;
+    }
+    /** Recursively moves the contents of source file to destination,
+     * maintaining paths.
+     * @param source A File representing the directory whose contents you
+     * wish to move.
+     * @param destination A File representing the directory you wish to move
+     * files to.
+     * @return true if successful
+     */
+    public static boolean moveDirectory(File source, File destination) {
+
+	// first try rename
+	if (source.renameTo(destination)) {
+	    return true;
+	}
+
+	// john T. said that this sometimes doesn't work, so you have to copy files manually
+
+	// copied from gatherer Utility class
+	File input[] = source.listFiles();
+	for(int i = 0; i < input.length; i++) {
+	    File output = new File(destination, input[i].getName());
+	    if (input[i].isDirectory()) {
+		moveDirectory(input[i], output);
+	    } else {
+		// Copy the file
+		try {
+		    output.getParentFile().mkdirs();
+		    FileInputStream in = new FileInputStream(input[i]);
+		    FileOutputStream out = new FileOutputStream(output);
+		    int value = 0;
+		    while((value = in.read()) != -1) {
+			out.write(value);
+		    }
+		    in.close();
+		    out.close();
+		    // Delete file
+		    input[i].delete();
+		} catch (Exception e) {
+		    System.err.println("GSFile.moveDirectory: exception: "+e.getMessage());
+		    return false;
+		}
+		
+	    }
+	}
+	return true;
+    }
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFileTest.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFileTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSFileTest.java	(revision 21112)
@@ -0,0 +1,49 @@
+package org.greenstone.gsdl3.util;
+
+import junit.framework.*; 
+import java.io.File;
+
+public class GSFileTest extends TestCase {
+
+    /** what type of test - used by the testing framework */
+    public static final String TEST_ALL_TEST_TYPE="UNIT";
+
+    public GSFileTest(String name) {
+	super(name);
+    }
+
+    /** test suite that dynamically runs all the tests */
+    public static Test suite() {
+	return new TestSuite(GSFileTest.class);
+    }
+
+    public void testBase64Conversion() {
+	
+	// start off with reverse conversion - decode then encode
+	String data = "sdkfjho9w4385khrf9ow834rfia8ufd38rhf9812";
+	File outfile;
+	try {
+	    outfile = File.createTempFile("b64convert_test", "");
+	} catch (java.io.IOException e) {
+	    fail("couldn't create a temporary file in default temp dir");
+	    return;
+	}
+	String filename = outfile.getAbsolutePath();
+	
+	String newdata;
+	if (GSFile.base64DecodeToFile(data, filename)) {
+	    newdata = GSFile.base64EncodeFromFile(filename);
+
+	    assertEquals("decoded/encoded string does not match the origianl",
+			 data, newdata);
+	    // clean up
+	    outfile.delete();
+	} else {
+	    fail("couldn't decode the data string");
+	}
+
+	//try with a real file???
+	
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSHTML.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSHTML.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSHTML.java	(revision 21112)
@@ -0,0 +1,63 @@
+package org.greenstone.gsdl3.util;
+
+/** GSHTML - provides some convenience methods for dealing
+ * with html
+ */
+public class GSHTML {
+
+    /** make a string html safe */
+    public static String htmlSafe(String input) {
+
+	StringBuffer filtered = new StringBuffer(input.length());
+	char c;
+	for (int i=0; i<input.length(); i++) {
+	    c = input.charAt(i);
+	    if (c == '<') {
+		filtered.append("&lt;");
+	    } else if (c == '>') {
+		filtered.append("&gt;");
+	    } else if (c == '"') {
+		filtered.append("&quot;");
+	    } else if (c == '&') {
+		filtered.append("&amp;");
+	    } else {
+		filtered.append(c);
+	    }
+	}
+	return(filtered.toString());
+    }
+    /** undo the html safe action */
+    public static String htmlUnsafe(String input) {
+	StringBuffer filtered = new StringBuffer(input.length());
+	char c;
+	for (int i=0; i<input.length(); i++) {
+	    c = input.charAt(i);
+	    if (c =='&') {
+		int j=input.indexOf(';', i);
+		String entity = input.substring(i, j);
+		i=j;
+		if (entity.equals("amp")) {
+		    filtered.append('&');
+		} else if (entity.equals("lt")){
+		    filtered.append('<');
+		} else if (entity.equals("gt")){
+		    filtered.append('>');
+		} else if (entity.equals("quot")) {
+		    filtered.append('"');
+		} // else just ignore it.
+	    } else {
+		filtered.append(c);
+	    }
+	}
+	return(filtered.toString());
+    }
+    
+    /** produce a default error page */
+    public static String  errorPage(String error) {
+	String page = "<html><head><Title>GSDL3 Error!</Title></head>\n"+
+	    "<body><h1>Greenstone Error!</h1>"+
+	    "<p/>"+error+"</body></html>";
+	return page;
+    }
+	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSParams.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSParams.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSParams.java	(revision 21112)
@@ -0,0 +1,101 @@
+package org.greenstone.gsdl3.util;
+
+import java.util.HashMap;
+
+/** keeps track of the interface parameters, and their defaults */
+public class GSParams {
+
+    public static final String ACTION = "a"; // the major type of action- eg query or browse or process
+    public static final String SUBACTION = "sa"; // subtype of action if we want different processing than the default
+    public static final String REQUEST_TYPE = "rt"; // whether the request is just to display the service form, or to actually do a request to teh service
+    public static final String RESPONSE_ONLY = "ro"; // if == 1 do the request and pass back the response xml - no page formatting
+    public static final String OUTPUT = "o"; // if processing is to be done, what type of output - html/xml/other??
+    public static final String SERVICE = "s"; // the name of the service
+    public static final String CLUSTER = "c";   // these two are the same
+    public static final String COLLECTION = "c"; 
+    public static final String LANGUAGE = "l";
+    public static final String DOCUMENT = "d";
+    public static final String DOCUMENT_TYPE = "dt";
+    public static final String RESOURCE = "r";
+    public static final String PROCESS_ID = "pid"; // if a request wasn't completed, this identifies the request - used when asking for a status update
+    
+    public static final String SIBLING = "sib"; // this should not be in here
+    // internal configure args
+    public static final String SYSTEM_SUBSET = "ss";
+    public static final String SYSTEM_CLUSTER = "sc";
+    public static final String SYSTEM_MODULE_NAME = "sn";
+    public static final String SYSTEM_MODULE_TYPE = "st";
+
+    public static final String EXPAND_DOCUMENT = "ed";
+    public static final String EXPAND_CONTENTS = "ec";
+    protected HashMap param_map = null;
+    
+    public GSParams() {
+	this.param_map = new HashMap(30);
+
+	// add in all the standard params
+	addParameter(ACTION, false); 
+	addParameter(SUBACTION,  false); 
+	addParameter(REQUEST_TYPE,  false);
+	addParameter(RESPONSE_ONLY,  false);
+	addParameter(CLUSTER,  true); 
+	//addParameter(COLLECTION); 
+	addParameter(LANGUAGE,  true); 
+	addParameter(DOCUMENT,  true); 
+	addParameter(RESOURCE,  true); 
+	addParameter(OUTPUT, false); 
+	addParameter(SERVICE, false); 
+	addParameter(PROCESS_ID, true); 
+	addParameter(SYSTEM_SUBSET, false);
+	addParameter(SYSTEM_CLUSTER, false);
+	addParameter(SYSTEM_MODULE_NAME, false);
+	addParameter(SYSTEM_MODULE_TYPE, false);
+	addParameter(SIBLING, false);
+	addParameter(DOCUMENT_TYPE, true);
+	addParameter(EXPAND_DOCUMENT, false);
+	addParameter(EXPAND_CONTENTS, false);
+ 	//addParameter();
+	// ugly hack so we don't save the extlink param
+	addParameter("s0.ext", false);
+    }
+    
+    public boolean addParameter(String name, boolean save) {
+	return addParameter(name, "", save);
+    }
+    
+    public boolean addParameter(String name, String default_value, boolean save) {
+	if (this.param_map.containsKey(name)) {
+	    // already there so could not add
+	    return false;
+	}
+
+	this.param_map.put(name, new Param(default_value, save));
+	return true;
+    }
+	
+    public boolean setParamDefault(String name, String default_value) {
+	Param p = (Param)this.param_map.get(name);
+	if (p==null) return false;
+	p.default_value = default_value;
+	return true;
+    }
+    
+    public boolean shouldSave(String name) {
+	Param p = (Param)this.param_map.get(name);
+	if (p== null) return true; // if things are not in here, always save.
+	return p.save;
+    }
+    
+    private class Param {
+
+	public String default_value = null;
+	public boolean save = true;
+
+	public Param(String default_value, boolean save) {
+	    this.default_value = default_value;
+	    this.save = save;
+	}
+    }
+}
+
+    
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSPath.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSPath.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSPath.java	(revision 21112)
@@ -0,0 +1,150 @@
+/*
+ *    GSPath.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+/**
+ * GSPath - utility class for greenstone
+ *
+ * modifies and examines message address paths and names
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ *
+ */
+
+public class GSPath {
+
+    /** adds new_link to the end of path
+     *
+     * @return the new path
+     */
+    static public String appendLink(String path, String new_link) {
+	if (path.equals("")) {
+	    return new_link;
+	}
+	return path +"/"+new_link;
+    }
+
+    /** adds new_link to the front of path
+     *
+     * @return the new path
+     */
+    static public String prependLink(String path, String new_link) {
+	if (path.equals("")) {
+	    return new_link;
+	}
+	return  new_link+"/"+path;
+    }
+
+    /** replaces the first link in the path with the specified name
+     */
+    static public String replaceFirstLink(String path, String new_first_link) {
+	path = removeFirstLink(path);
+	path = prependLink(path, new_first_link);
+	return path;
+
+    }
+    /** returns the first name in the path
+     *
+     */
+    static public String getFirstLink(String path) {
+
+	int i = path.indexOf("/");
+	if (i==0) { // have a '/' on the front
+	    path = path.substring(1);
+	    i= path.indexOf("/");
+	}
+	if (i==-1) { // '/' not found
+	    return path;
+	}
+	
+	return path.substring(0,i);
+    }
+
+    /** removes the first name in the path
+     *
+     * @return the modified path
+     */
+    static public String removeFirstLink(String path) {
+	int i = path.indexOf("/");
+	if (i==0) { // have a '/' on the front
+	    path = path.substring(1);
+	    i= path.indexOf("/");
+	}
+	if (i==-1) { // '/' not found - remove the whole path
+	    return "";
+	}
+	return path.substring(i+1); // remove the '/' char
+    }
+
+    /** removes the last name in the path
+     *
+     * @return the modified path
+     */
+    static public String removeLastLink(String path) {
+	int i = path.lastIndexOf("/");
+	if (i==-1) { // '/' not found - remove the whole path
+	    return "";
+	}
+	return path.substring(0, i); // remove the '/' char
+    }
+    
+    static public String getLastLink(String path) {
+	int i = path.lastIndexOf("/");
+	if (i==-1) { // '/' not found - remove the whole path
+	    return path;
+	}
+	return path.substring(i+1); // remove the last link
+    }
+
+    
+    static public String createPath(String [] links) {
+	String path = links[0];
+	for (int i=1; i<links.length; i++) {
+	    path = appendLink(path, links[i]);
+	}
+	return path;
+    }
+
+    static public int getIndex(String link) {
+	int i = link.indexOf("[");
+	if (i==-1) {
+	    return -1;
+	}
+	int j = link.indexOf("]");
+	String num = link.substring(i+1,j);
+	try {
+	    int index = Integer.parseInt(num);
+	    return index;
+	} catch (Exception e) {
+	    return -1;
+	}
+    }
+
+    static public String removeIndex(String link) {
+	int i = link.indexOf("[");
+	if (i==-1) {
+	    return link;
+	}
+	return link.substring(0, i);
+    }
+
+}
+
+    
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSSQL.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSSQL.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSSQL.java	(revision 21112)
@@ -0,0 +1,80 @@
+package org.greenstone.gsdl3.util;
+
+public class GSSQL {
+
+    // table names
+    public static final String DOCUMENT_TABLE = "document";
+    public static final String STRUCTURE_TABLE = "structure";
+    public static final String DIVISION_TABLE = "divisions";
+    public static final String DIVISION_FILE_TABLE = "divisionfilerefs";
+    public static final String DIVISION_METADATA_TABLE = "divisionmetarefs";
+    public static final String FILE_SECTION_TABLE = "filesection";
+    public static final String FILE_GROUP_TABLE = "filegroups";
+    public static final String FILE_TABLE = "files";
+    public static final String METADATA_TABLE = "metadata"; 
+    public static final String NAMESPACE_TABLE = "namespaces";
+    public static final String METADATA_VALUE_TABLE = "mdvalues";
+    public static final String CLASSIFIER_TABLE = "classifiers";
+    public static final String CLASS_DATA_TABLE = "classdata";
+    public static final String CLASS_DOCUMENT_TABLE = "classdocuments";
+
+    // unique key names - references to the tables
+    public static final String METADATA_REF = "MetadataRef";
+    public static final String NAMESPACE_REF = "NamespaceRef";
+    public static final String STRUCTURE_REF = "StructureRef";
+    public static final String DIVISION_REF = "DivisionRef";
+    public static final String FILE_SECTION_REF = "FileSectionRef";
+    public static final String FILE_GROUP_REF = "FileGroupRef";
+    public static final String CLASSIFIER_REF = "ClassifyRef";
+    
+    // combined refs
+    public static final String PARENT_REF = "ParentRef";
+    public static final String PARENT_TYPE = "ParentType";
+    
+    // id fields - these are unique ids per document - ie combined with docid
+    public static final String DOCUMENT_ID = "DocID";
+    public static final String METADATA_ID = "MetaID";
+    public static final String FILE_ID = "FileID";
+    public static final String STRUCTURE_ID = "StructureID";
+    public static final String SECTION_ID = "SectionID";
+    public static final String FILE_GROUP_ID = "FileID";
+    public static final String FILE_SECTION_ID = "FileID";
+    public static final String CLASSIFIER_ID = "ClassifyID";
+    public static final String PARENT_ID = "ParentID";
+    
+    // type fields
+    public static final String DOC_TYPE = "DocType";
+    public static final String NAMESPACE_TYPE = "NamespaceType";
+    public static final String STRUCTURE_TYPE = "StructureType";
+    public static final String DIVISION_TYPE = "DivisionType";
+    public static final String FILE_LOCATION_TYPE = "FileLocType";
+    public static final String MIME_TYPE = "MIMEType";
+
+    // other stuff
+    public static final String LABEL = "Label";
+    public static final String VALUE = "Value";
+    public static final String LABEL_ORDER = "LabelOrder";
+    public static final String SHORT_LABEL = "ShortLabel";
+    public static final String USER_LABEL = "UserLabel";
+    public static final String NAME = "Name";
+    public static final String DESCRIPTION = "Description";
+    public static final String FILE_LOCATION = "FileLocation";
+    public static final String CREATOR = "Creator";
+    public static final String DOCUMENT_ORDER = "DocOrder";
+    public static final String NUM_LEAF_DOCUMENTS = "NumLeafDocs";
+
+    public static final String COMMA = ",";
+    public static final String EQUALS = "=";
+    public static final String QUOTE = "'";
+    public static final String EQUALS_QUOTE = "='";
+    public static final String DOT = ".";
+    public static final String END = ";";
+    public static final String NOT="!";
+
+    // some values in the fields
+    public static final String STRUCTURE = "Structure";
+    public static final String DIVISION = "Division";
+    public static final String ALL = "All";
+    public static final String MG_NUM = "mgseqno";
+    
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSStatus.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSStatus.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSStatus.java	(revision 21112)
@@ -0,0 +1,36 @@
+package org.greenstone.gsdl3.util;
+
+public class GSStatus {
+
+    // responses for initial request
+    public static final int SUCCESS = 1; // request succeeded
+    public static final int ACCEPTED = 2; // request accepted but not completed
+    public static final int ERROR = 3; // error and process stopped
+
+    // responses for status requests
+    public static final int CONTINUING = 10; // request still continuing
+    public static final int COMPLETED = 11; // request finished
+    public static final int HALTED = 12; // process stopped
+	    
+    // other
+    public static final int INFO = 20; // just an info message that doesnt mean anything
+
+    /** returns true if teh code indicates that a process is no longer running 
+     */
+    public static boolean isCompleted(int code) {
+	if (code == SUCCESS || code == ERROR || code == COMPLETED || code == HALTED) {
+	    return true;
+	} 
+	return false;
+    }
+
+    // there may be other types of error codes
+    public static boolean isError(int code) {
+	if (code == ERROR) {
+	    return true;
+	}
+	return false;
+    }
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXML.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXML.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXML.java	(revision 21112)
@@ -0,0 +1,841 @@
+package org.greenstone.gsdl3.util;
+
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Text;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.HashMap;
+import java.util.Vector;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+//import java.util.Locale;
+
+/** various functions for extracting info out of GS XML */
+public class GSXML {
+
+    // greenstone xml elements
+    public static final String MESSAGE_ELEM = "message";
+    public static final String REQUEST_ELEM = "request";
+    public static final String RESPONSE_ELEM = "response";
+    public static final String COLLECTION_ELEM = "collection";
+    public static final String SERVICE_ELEM = "service";
+    public static final String CLUSTER_ELEM = "serviceCluster";
+    public static final String SITE_ELEM = "site";
+    public static final String PARAM_ELEM = "param";
+    public static final String PARAM_OPTION_ELEM = "option";
+    public static final String CONTENT_ELEM = "content";
+    public static final String RESOURCE_ELEM = "resource";
+    public static final String DOCUMENT_ELEM = "document";
+    public static final String METADATA_ELEM = "metadata";
+    public static final String SERVICE_CLASS_ELEM = "serviceRack";
+    public static final String CLASSIFIER_ELEM = "classifier";
+    public static final String APPLET_ELEM = "applet";
+    public static final String APPLET_DATA_ELEM = "appletData";
+    public static final String CONFIGURE_ELEM = "configure";
+    public static final String STATUS_ELEM = "status";
+    public static final String ERROR_ELEM = "error";
+    public static final String DEFAULT_ELEM = "default";
+    public static final String STYLESHEET_ELEM = "format";//"stylesheet"; // any additional stylesheet stuff is carried in the message inside this elem
+    public static final String FORMAT_ELEM = "format"; // config files use format - should we use this instead of stylesheet??
+    public static final String TERM_ELEM = "term";
+    public static final String SYSTEM_ELEM = "system";
+
+    //config file elems
+    public static final String COLLECTION_CONFIG_ELEM = "collectionConfig";
+    public static final String COLLECTION_BUILD_ELEM = "buildConfig";
+    public static final String COLLECTION_INIT_ELEM = "collectionInit";
+    public static final String RECOGNISE_ELEM = "recognise";
+    public static final String DOC_TYPE_ELEM = "docType";
+    public static final String SEARCH_ELEM = "search";
+    public static final String INDEX_ELEM = "index";
+    public static final String INDEX_STEM_ELEM = "indexStem";
+    public static final String BROWSE_ELEM = "browse";
+    public static final String DISPLAY_ELEM = "display";
+    public static final String FIELD_ELEM = "field";
+    public static final String LEVEL_ELEM = "level";
+    public static final String SHORTNAME_ATT = "shortname";
+    public static final String NOTIFY_ELEM = "notify";
+    public static final String NOTIFY_HOST_ATT = "host";
+
+    // elems for the pages to be processed by xslt
+    public final static String PAGE_ELEM = "page";
+    public final static String CONFIGURATION_ELEM = "config";
+    public final static String PAGE_REQUEST_ELEM = "pageRequest";
+    public final static String PAGE_RESPONSE_ELEM = "pageResponse";
+    public final static String PAGE_EXTRA_ELEM = "pageExtra";
+
+    //public final static String DESCRIPTION_ELEM = "description";
+
+    public static final String  ACTION_ELEM = "action";
+    public static final String  SUBACTION_ELEM = "subaction";
+    
+    // add on to another elem type to get a list of that type
+    public static final String LIST_MODIFIER = "List";
+
+    // greenstone xml attributes
+    public static final String NAME_ATT = "name";
+    public static final String TO_ATT = "to";
+    public static final String USER_ID_ATT = "uid";
+    public static final String FROM_ATT = "from";
+    public static final String LANG_ATT = "lang";
+    public static final String TYPE_ATT = "type";
+    public static final String VALUE_ATT = "value";
+    public static final String DEFAULT_ATT = "default";
+    public static final String INFO_ATT = "info";
+    public static final String ACTION_ATT = "action";
+    public static final String SUBACTION_ATT = "subaction";
+    public static final String OUTPUT_ATT = "output";
+    public static final String ADDRESS_ATT = "address";
+    public static final String LOCAL_SITE_ATT = "localSite";
+    public static final String LOCAL_SITE_NAME_ATT = "localSiteName";
+    public static final String STATUS_ERROR_CODE_ATT = "code";
+    public static final String STATUS_PROCESS_ID_ATT = "pid";
+    public static final String PARAM_SHORTNAME_ATT = "shortname";
+    public static final String PARAM_IGNORE_POS_ATT = "ignore";
+    public static final String CLASSIFIER_CONTENT_ATT = "content";
+    public static final String ERROR_TYPE_ATT = "type";
+
+    // document stuff
+    public static final String DOC_TYPE_ATT = "docType";
+    public static final String DOC_NODE_ELEM = "documentNode";
+    public static final String NODE_CONTENT_ELEM = "nodeContent";
+    public static final String NODE_STRUCTURE_ELEM = "nodeStructure";
+    public static final String NODE_ID_ATT = "nodeID";
+    public static final String NODE_NAME_ATT = "nodeName";
+    public static final String NODE_TYPE_ATT = "nodeType";
+    public static final String NODE_RANK_ATT = "rank";
+    public static final String NODE_TYPE_ROOT = "root";
+    public static final String NODE_TYPE_INTERNAL = "internal";
+    public static final String NODE_TYPE_LEAF = "leaf";
+
+    public static final String DOC_TYPE_SIMPLE = "simple";
+    public static final String DOC_TYPE_PAGED = "paged";
+    public static final String DOC_TYPE_HIERARCHY = "hierarchy";
+
+    // classifier stuff
+    public static final String CLASS_NODE_ELEM = "classifierNode";
+    public static final String CLASS_NODE_ORIENTATION_ATT = "orientation";
+    
+    // parameter types
+    public static final String PARAM_TYPE_INTEGER = "integer";
+    public static final String PARAM_TYPE_BOOLEAN = "boolean";
+    public static final String PARAM_TYPE_ENUM_START = "enum";
+    public static final String PARAM_TYPE_ENUM_SINGLE = "enum_single";
+    public static final String PARAM_TYPE_ENUM_MULTI = "enum_multi";
+    public static final String PARAM_TYPE_STRING = "string";
+    public static final String PARAM_TYPE_TEXT = "text";
+    public static final String PARAM_TYPE_MULTI = "multi";
+    public static final String PARAM_TYPE_FILE = "file";
+    public static final String PARAM_TYPE_INVISIBLE = "invisible";
+    // stuff for text strings 
+    public static final String DISPLAY_TEXT_ELEM = "displayItem";
+    // the following are used for the name attributes
+    public static final String DISPLAY_TEXT_NAME = "name";
+    public static final String DISPLAY_TEXT_SUBMIT = "submit";
+    public static final String DISPLAY_TEXT_DESCRIPTION = "description";
+
+    // request types 
+    // get the module description
+    public static final String REQUEST_TYPE_DESCRIBE = "describe";
+    // startup a process 
+    public static final String REQUEST_TYPE_PROCESS = "process";
+    // get the status of an ongoing process
+    public static final String REQUEST_TYPE_STATUS = "status";
+    // system type request - eg reload a collection
+    public static final String REQUEST_TYPE_SYSTEM = "system"; 
+    // page requests to the Receptionist/Actions
+    public static final String REQUEST_TYPE_PAGE = "page"; // used to be cgi
+    // get any format info for a service
+    public static final String REQUEST_TYPE_FORMAT = "format"; 
+    // modify the requests
+    public static final String REQUEST_TYPE_MESSAGING = "messaging";
+
+    // service types
+    public static final String SERVICE_TYPE_QUERY = "query";
+    public static final String SERVICE_TYPE_RETRIEVE = "retrieve";
+    public static final String SERVICE_TYPE_BROWSE = "browse";
+    public static final String SERVICE_TYPE_APPLET = "applet";
+    public static final String SERVICE_TYPE_PROCESS = "process";
+    public static final String SERVICE_TYPE_ENRICH = "enrich";
+
+    // system command types and attributes
+    public static final String SYSTEM_TYPE_CONFIGURE = "configure";
+    public static final String SYSTEM_TYPE_ACTIVATE = "activate";
+    public static final String SYSTEM_TYPE_DEACTIVATE = "deactivate";
+
+    public static final String SYSTEM_SUBSET_ATT = "subset";
+    public static final String SYSTEM_MODULE_TYPE_ATT = "moduleType";
+    public static final String SYSTEM_MODULE_NAME_ATT = "moduleName";
+    
+    // communicator types
+    public static final String COMM_TYPE_SOAP_JAVA = "soap";
+    
+    // error types
+    public static final String ERROR_TYPE_SYNTAX = "syntax";
+    public static final String ERROR_TYPE_SYSTEM = "system";
+    public static final String ERROR_TYPE_INVALID_ID = "invalid_id";
+    public static final String ERROR_TYPE_OTHER = "other";
+
+    // some system wide param names
+    public static final String SUBSET_PARAM = "subset";
+
+    /** takes a list of elements, and returns an array of strings 
+     * of the values of attribute att_name */
+    public static String [] getAttributeValuesFromList(Element list,
+						       String att_name) {
+	
+	NodeList children = list.getChildNodes();
+	
+	int num_nodes = children.getLength();
+	String []ids = new String[num_nodes];
+	for (int i=0; i<num_nodes; i++) {
+	    Element e = (Element)children.item(i);
+	    String id = e.getAttribute(att_name);	    
+	    ids[i] = id;	    
+	}
+	
+	return ids;
+    }
+
+    /** takes a paramList element, and gets a HashMap of name-value pairs 
+     * if deep=true, extracts embedded params, otherwise just top level 
+     * params*/
+    public static HashMap extractParams(Element xml, boolean deep) {
+
+	if (!xml.getNodeName().equals(PARAM_ELEM+LIST_MODIFIER)) {
+	    System.err.println("GSXML:paramList element should have been passed to extractParams, instead it was "+xml.getNodeName());
+	    return null;
+	}
+
+	NodeList params = null;
+	if (deep) { // get all the nested ones
+	    params = xml.getElementsByTagName(PARAM_ELEM);
+	} else { // just get the top  level ones
+	    params = xml.getChildNodes();
+	}
+	HashMap param_map = new HashMap();
+	for (int i=0; i<params.getLength(); i++) {
+	    if (params.item(i).getNodeName().equals(PARAM_ELEM)) {
+		Element param = (Element)params.item(i);
+		String name=param.getAttribute(NAME_ATT);
+		String value=getValue(param); //att or content
+		int pos = name.indexOf('.');
+		if (pos == -1) { // a base param
+		    param_map.put(name, value);
+		} else { // a namespaced param
+		    
+		    String namespace = name.substring(0, pos);
+		    name = name.substring(pos+1);
+		    HashMap map = (HashMap)param_map.get(namespace);
+		    if (map == null) {
+			map = new HashMap();
+			param_map.put(namespace, map);
+		    }
+		    map.put(name, value);
+		}
+	    }
+	}
+	return param_map;
+    }
+
+    /** gets the value att or the text content */
+    public static String getValue(Element e) {
+	String val = e.getAttribute(VALUE_ATT);
+	if (val ==null || val.equals("")) {
+	    // have to get it out of the text
+	    val=getNodeText(e); 
+
+	} else {
+	    // unescape the xml stuff
+	    val = unXmlSafe(val);
+	}
+	return val;
+    }
+
+    /** extracts the text out of a node */
+    public static Node getNodeTextNode(Element param) {
+	param.normalize();
+	Node n = param.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	return n;
+    }
+    
+    /** extracts the text out of a node */
+    public static String getNodeText(Element param) {
+	Node text_node = getNodeTextNode(param);
+	if (text_node == null) {
+	    return "";
+	}
+	return text_node.getNodeValue();
+    }
+
+	
+    /** add text to a document/subsection  element */
+    public static boolean addDocText(Document owner, Element doc, String text) {
+
+	Element content = owner.createElement(NODE_CONTENT_ELEM);
+	Text t = owner.createTextNode(text);
+	content.appendChild(t);
+	doc.appendChild(content);
+	return true;
+    }
+  
+    /** add an error message, unknown error type */
+    public static boolean addError(Document owner, Element doc, String text) {
+	return addError(owner, doc, text, ERROR_TYPE_OTHER);
+    }
+    /** add an error message */
+    public static boolean addError(Document owner, Element doc, String text,
+				   String error_type) {
+
+	Element content = owner.createElement(ERROR_ELEM);
+	content.setAttribute(ERROR_TYPE_ATT, error_type);
+	Text t = owner.createTextNode(text);
+	content.appendChild(t);
+	doc.appendChild(content);
+	return true;
+    }
+
+    /** add an error message */
+    public static boolean addError(Document owner, Element doc, Throwable error) {
+      return addError(owner, doc, error, ERROR_TYPE_OTHER);
+    }
+    
+    /** add an error message */
+    public static boolean addError(Document owner, Element doc, 
+				   Throwable error, String error_type) {
+	error.printStackTrace();
+	return addError(owner, doc, error.toString(), error_type);
+    }
+
+    public static Element createMetadataParamList(Document owner, Vector meta_values) {
+
+	Element meta_param_list = owner.createElement(PARAM_ELEM+LIST_MODIFIER);
+	Iterator i = meta_values.iterator();
+	while(i.hasNext()) {
+	    String next = (String)i.next();
+	    Element meta_param = owner.createElement(PARAM_ELEM);
+	    meta_param_list.appendChild(meta_param);
+	    meta_param.setAttribute(NAME_ATT, "metadata");
+	    meta_param.setAttribute(VALUE_ATT, next);
+	}
+	return meta_param_list;
+    }
+	
+    /** adds a metadata elem to a list */
+    public static boolean addMetadata(Document owner, Element list, 
+				      String meta_name, String meta_value) {
+	if (meta_value==null || meta_value.equals("")) {
+	    return false;
+	}
+	Element data = owner.createElement(METADATA_ELEM);
+	data.setAttribute(NAME_ATT, meta_name);
+	Text t = owner.createTextNode(meta_value);
+	data.appendChild(t);
+	list.appendChild(data);
+	return true;
+
+    }
+
+    /** copies the metadata out of teh metadataList of 'from' into
+     * the metadataList of 'to' */
+    public static boolean mergeMetadataLists(Node to, Node from) {
+	Node to_meta = getChildByTagName(to, METADATA_ELEM+LIST_MODIFIER);
+	Node from_meta = getChildByTagName(from, METADATA_ELEM+LIST_MODIFIER);
+	
+	if  (from_meta == null) { // nothing to copy
+	    return true;
+	}
+	Document to_owner = to.getOwnerDocument();
+	Node new_from = to_owner.importNode(from_meta, true);
+
+	if (to_meta == null) { // just copy the whole list
+	    to.appendChild(new_from);
+	    return true;
+	}
+	
+	// copy individual elements
+	Node child = new_from.getFirstChild();
+	while ( child != null) {
+	    to_meta.appendChild(child);
+	    child = child.getNextSibling();
+	}
+	return true;
+    }
+	
+    /** copies all the children from from to to */
+    public static boolean mergeElements(Element to, Element from) {
+
+	Document owner = to.getOwnerDocument();
+	Node child = from.getFirstChild();
+	while (child != null) {
+	    to.appendChild(owner.importNode(child, true));
+	    child = child.getNextSibling();
+	}
+	return true;
+    }
+    /** returns the (first) child element with the given name */
+    public static Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+    
+    /** returns the (nth) child element with the given name 
+     * index numbers start at 0 */
+    public static Node getChildByTagNameIndexed(Node n, String name, int index) {
+	if (index == -1) {
+	    return getChildByTagName(n, name);
+	}
+	int count = 0;
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		if (count == index) {
+		    return child;
+		} else {
+		    count++;
+		}
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+    /** takes an xpath type expression of the form name/name/... 
+	and returns the first node that matches, or null if not found */
+    public static Node getNodeByPath(Node n, String path) {
+	
+	String link = GSPath.getFirstLink(path);
+	path = GSPath.removeFirstLink(path);
+	while (!link.equals("")) {
+	    n = getChildByTagName(n, link);
+	    if (n==null) {
+		return null;
+	    }
+	    link = GSPath.getFirstLink(path);
+	    path = GSPath.removeFirstLink(path);
+	}
+	return n;
+    }
+
+    /** takes an xpath type expression of the form name/name/... 
+     * and returns the first node that matches, or null if not found 
+     * can include [i] indices. index numbers start at 0 */
+    public static Node getNodeByPathIndexed(Node n, String path) {
+	
+	String link = GSPath.getFirstLink(path);
+	int index = GSPath.getIndex(link);
+	if (index != -1) {
+	    link = GSPath.removeIndex(link);
+	}
+	path = GSPath.removeFirstLink(path);
+	while (!link.equals("")) {
+	    n = getChildByTagNameIndexed(n, link, index);
+	    if (n==null) {
+		return null;
+	    }
+	    link = GSPath.getFirstLink(path);
+	    index = GSPath.getIndex(link);
+	    if (index != -1) {
+		link = GSPath.removeIndex(link);
+	    }
+	    path = GSPath.removeFirstLink(path);
+	}
+	return n;
+    }
+
+    public static HashMap getChildrenMap(Node n) {
+	
+	HashMap map= new HashMap();
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    String name = child.getNodeName();
+	    map.put(name, child);
+	    child = child.getNextSibling();
+	}
+	return map;
+    }
+       
+    public static NodeList getChildrenByTagName(Node n, String name) {
+	MyNodeList node_list = new MyNodeList();
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		node_list.addNode(child);
+	    }
+	    child = child.getNextSibling();
+	}
+	return node_list;
+    }
+	
+
+    /** Duplicates an element, but gives it a new name */
+    public static Element duplicateWithNewName(Document owner, Element element,
+					       String element_name, boolean with_attributes)
+    {
+	return duplicateWithNewNameNS(owner, element, element_name, null, with_attributes);
+    }
+
+    /** Duplicates an element, but gives it a new name */
+    public static Element duplicateWithNewNameNS(Document owner, 
+						 Element element,
+						 String element_name, 
+						 String namespace_uri,
+						 boolean with_attributes)
+    {
+	Element duplicate;
+	if (namespace_uri == null) {
+	    duplicate = owner.createElement(element_name);
+	} else {
+	    duplicate = owner.createElementNS(namespace_uri, element_name);
+	}
+	// Copy element attributes
+	if (with_attributes) {
+	    NamedNodeMap attributes = element.getAttributes();
+	    for (int i = 0; i < attributes.getLength(); i++) {
+		Node attribute = attributes.item(i);
+		duplicate.setAttribute(attribute.getNodeName(), attribute.getNodeValue());
+	    }
+	}
+
+	// Copy element children
+	NodeList children = element.getChildNodes();
+	for (int i = 0; i < children.getLength(); i++) {
+	    Node child = children.item(i);
+	    duplicate.appendChild(owner.importNode(child, true));
+	}
+
+	return duplicate;
+    }
+
+    public static void copyAllChildren(Element to, Element from) {
+
+	Document to_doc = to.getOwnerDocument();
+	Node child = from.getFirstChild();
+	while (child != null) {
+	    to.appendChild(to_doc.importNode(child, true));
+	    child = child.getNextSibling();
+	}
+    }
+    /** returns a basic request message */
+    public static  Element createBasicRequest(Document owner, 
+					      String request_type, String to, 
+					      String lang,
+					      String uid) {
+	Element request = owner.createElement(REQUEST_ELEM);
+	request.setAttribute(TYPE_ATT, request_type);
+	request.setAttribute(LANG_ATT, lang);
+	request.setAttribute(TO_ATT, to);
+	request.setAttribute(USER_ID_ATT, uid);
+	return request;
+    }
+    
+    public static Element createTextElement(Document owner, String elem_name,
+					    String text) {
+	Element e = owner.createElement(elem_name);
+	Text t = owner.createTextNode(text);
+	e.appendChild(t);
+	return e;
+
+    }
+
+    public static Element createDisplayTextElement(Document owner, 
+						   String text_name,
+						   String text) {
+	Element e = owner.createElement(DISPLAY_TEXT_ELEM);
+	e.setAttribute(NAME_ATT, text_name);
+	Text t = owner.createTextNode(text);
+	e.appendChild(t);
+	return e;
+
+    }
+
+
+    public static Element createParameter(Document owner, String name, 
+					  String value) {
+	Element param = owner.createElement(PARAM_ELEM);
+	param.setAttribute(NAME_ATT, name);
+	param.setAttribute(VALUE_ATT, value);
+	return param;
+    }
+
+    public static void addParametersToList(Document owner, Element param_list, 
+					   HashMap params) {
+	Set items = params.entrySet();
+	Iterator i = items.iterator();
+	while(i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    param_list.appendChild(createParameter(owner, (String)m.getKey(), (String)m.getValue()));
+	}
+	
+    }
+
+    public static Element createParameterDescription(Document owner, 
+						     String id,
+						     String display_name,
+						     String type, 
+						     String default_value,
+						     String []option_ids,
+						     String []option_names) {
+
+	
+	Element p = owner.createElement(PARAM_ELEM);
+	p.setAttribute(NAME_ATT, id);
+	p.setAttribute(TYPE_ATT, type);
+	p.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, display_name));
+	if (default_value != null) {
+	    p.setAttribute(DEFAULT_ATT, default_value);
+	}
+	if (option_ids!=null && option_names!=null) {
+	    for (int i=0; i<option_ids.length; i++) {
+		Element e = owner.createElement(PARAM_OPTION_ELEM);
+		e.setAttribute(NAME_ATT, option_ids[i]);
+		e.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, option_names[i]));
+		p.appendChild(e);
+	    }
+	}
+	return p;
+    }
+    public static Element createParameterDescription2(Document owner, 
+						     String id,
+						     String display_name,
+						     String type, 
+						     String default_value,
+						     ArrayList option_ids,
+						     ArrayList option_names) {
+
+	
+	Element p = owner.createElement(PARAM_ELEM);
+	p.setAttribute(NAME_ATT, id);
+	p.setAttribute(TYPE_ATT, type);
+	p.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, display_name));
+	if (default_value != null) {
+	    p.setAttribute(DEFAULT_ATT, default_value);
+	}
+	if (option_ids!=null && option_names!=null) {
+	    for (int i=0; i<option_ids.size(); i++) {
+		Element e = owner.createElement(PARAM_OPTION_ELEM);
+		e.setAttribute(NAME_ATT, (String)option_ids.get(i));
+		e.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, (String)option_names.get(i)));
+		p.appendChild(e);
+	    }
+	}
+	return p;
+    }
+
+
+    /** returns the element parent/node_name[@attribute_name='attribute_value']
+     */
+    public static Element getNamedElement(Element parent, String node_name,
+					  String attribute_name, 
+					  String attribute_value) {
+
+	NodeList children = parent.getChildNodes();
+	for (int i=0; i<children.getLength(); i++) {
+	    Node child = children.item(i);
+	    ///ystem.out.println("getnamed elem, node nmae="+child.getNodeName());
+	    if (child.getNodeName().equals(node_name)) {
+		if (((Element)child).getAttribute(attribute_name).equals(attribute_value))
+		    return (Element)child;
+	    }
+	}
+	// not found
+	return null;
+    }
+
+    public static int SORT_TYPE_STRING = 0;
+    public static int SORT_TYPE_INT = 1;
+    public static int SORT_TYPE_FLOAT = 2;
+    
+    // sort type: 
+    public static Element insertIntoOrderedList(Element parent_node,
+						String node_name,
+						Element start_from_elem, 
+						Element new_elem, String sort_att,
+						boolean descending)
+    {
+	if (new_elem == null) return null;
+	System.err.println("+");
+	Element cloned_elem = (Element)parent_node.getOwnerDocument().importNode(new_elem, true);
+	if (start_from_elem == null) {
+	    parent_node.appendChild(cloned_elem);
+	    return cloned_elem;
+	}
+	System.err.println("++");
+	
+	Node current_node = start_from_elem;
+	String insert_att = cloned_elem.getAttribute(sort_att);
+	String list_att = start_from_elem.getAttribute(sort_att);
+	while ((!descending && list_att.compareTo(insert_att)<0) || (descending && list_att.compareTo(insert_att)>0)) {
+	    System.err.println("+++");
+	    current_node = current_node.getNextSibling();
+	    if (current_node == null) break; // end of the list
+	    if (!current_node.getNodeName().equals(node_name)) {
+		continue; // not a valid node
+	    }
+	    list_att = ((Element)current_node).getAttribute(sort_att);
+	}
+	System.err.println("++++");
+
+	parent_node.insertBefore(cloned_elem, current_node);
+	return cloned_elem;
+    }
+
+
+    /** Returns the appropriate language element from a display elem,
+     display is the containing element, name is the name of the element to 
+     look for, lang is the preferred language, lang_default is the fall back 
+     lang if neither lang is found, will return the first one it finds*/
+    public static String getDisplayText(Element display, String name,
+					String lang, String lang_default) {
+	
+	String def = null;
+	String first = null;
+	NodeList elems = display.getElementsByTagName(DISPLAY_TEXT_ELEM);
+	if (elems.getLength() == 0) return "";
+	for (int i=0; i<elems.getLength(); i++) {
+	    Element e = (Element)elems.item(i);
+	    String n = e.getAttribute(NAME_ATT);
+	    if (name.equals(n)) {
+		String l = e.getAttribute(LANG_ATT);
+		if (lang.equals(l)) {
+		    return getNodeText(e);
+		} else if (lang_default.equals(l)) {
+			def = getNodeText(e);
+		} else if (first == null) {
+		    first = getNodeText(e);
+		}
+	    } else {
+		continue;
+	    }
+	}
+
+	if (def != null) {
+	    return def;
+	}
+	if (first != null) {
+	    return first;
+	}
+	return "";
+    }
+    
+    // replaces < > " ' & in the original with their entities
+    public static String xmlSafe(String original) {
+
+	StringBuffer filtered = new StringBuffer(original.length());
+	char c;
+	for (int i=0; i<original.length(); i++) {
+	    c = original.charAt(i);
+	    if (c == '>') {
+		filtered.append("&gt;");
+	    } else if (c == '<') {
+		filtered.append("&lt;");
+	    } else if (c == '"') {
+		filtered.append("&quot;");
+	    } else if (c == '&') {
+		filtered.append("&amp;");
+	    } else if (c == '\'') {
+		filtered.append("&apos;");
+	    } else {
+		filtered.append(c);
+	    }
+	}
+	return filtered.toString();
+    }
+
+    
+    // replaces < > " ' & entities with their originals
+    public static String unXmlSafe(String original) {
+	
+ 	StringBuffer filtered = new StringBuffer(original.length());
+ 	char c;
+ 	for (int i=0; i<original.length(); i++) {
+ 	    c = original.charAt(i);
+ 	    if (c == '&') {
+ 		int pos = original.indexOf(";", i);
+		String entity = original.substring(i+1, pos);
+		if (entity.equals("gt")) {
+		    filtered.append(">");
+		} else if (entity.equals("lt")) {
+		    filtered.append("<");
+		} else if (entity.equals("apos")) {
+		    filtered.append("'");
+		} else if (entity.equals("amp")) {
+		    filtered.append("&");
+		} else if (entity.equals("quot")) {
+		    filtered.append("\"");
+		} else {
+		    filtered.append("&"+entity+";");
+		}
+		i = pos;
+	    }
+	    else {
+ 		filtered.append(c);
+ 	    }
+ 	}
+ 	return filtered.toString();
+     }
+
+    public static void printXMLNode(Node e) {
+	printXMLNode(e, 0) ;
+    }
+   
+    public static void printXMLNode(Node e, int depth) { //recursive method call using DOM API...
+	
+	for (int i=0 ; i<depth ; i++) 
+	    System.out.print(' ') ;
+
+	if (e.getNodeType() == Node.TEXT_NODE){
+	    System.out.println("text") ;
+	    //System.out.println(e.getNodeValue()) ;
+	    return ;
+	}
+	
+	System.out.print('<');
+	System.out.print(e.getNodeName());
+	NamedNodeMap attrs = e.getAttributes();
+	for (int i = 0; i < attrs.getLength(); i++) {
+	    Node attr = attrs.item(i);
+	    System.out.print(' ');  
+	    System.out.print(attr.getNodeName());
+	    System.out.print("=\"");
+	    System.out.print(attr.getNodeValue());   
+	    System.out.print('"');
+	}
+
+	NodeList children = e.getChildNodes();
+
+	if (children == null || children.getLength() == 0)
+	    System.out.println("/>") ;
+	else {
+	    
+	    System.out.println('>') ;
+	    
+	    int len = children.getLength();
+	    for (int i = 0; i < len; i++) {
+		printXMLNode(children.item(i), depth + 1);
+	    }
+	    
+	    for (int i=0 ; i<depth ; i++) 
+		System.out.print(' ') ;
+	    
+	    System.out.println("</" + e.getNodeName() + ">");
+	}	
+
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXSLT.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXSLT.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GSXSLT.java	(revision 21112)
@@ -0,0 +1,99 @@
+package org.greenstone.gsdl3.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Text;
+
+import java.util.HashMap;
+import java.util.Vector;
+import java.util.ArrayList;
+
+/** various functions for manipulating Greenstone xslt  */
+public class GSXSLT {
+
+    /** takes a stylesheet Document, and adds in any child nodes from extra_xsl
+     * named templates overwrite any existing one, while match templates are
+just added to the end of teh stylesheet
+     */
+    public static void mergeStylesheets(Document main_xsl, Element extra_xsl) {
+	
+	Element main = main_xsl.getDocumentElement();
+	
+	NodeList children = extra_xsl.getElementsByTagNameNS("http://www.w3.org/1999/XSL/Transform", "template");
+	for (int i=0; i<children.getLength(); i++) {
+	    String name = ((Element)children.item(i)).getAttribute("name");
+	    if (!name.equals("")) {
+		Element old_template = GSXML.getNamedElement(main, "xsl:template", "name", name);
+		if (old_template != null) {
+		    main.removeChild(old_template);
+		}
+	    }
+		
+	    main.appendChild(main_xsl.importNode(children.item(i), true));
+	}
+
+    }
+
+    /** takes any import or include nodes, and creates absolute path names
+     * for the files
+     */
+    public static void absoluteIncludePaths(Document stylesheet, 
+					    String gsdl3_home,
+					    String site_name,
+					    String collection,
+					    String interface_name,
+					    ArrayList base_interfaces) {
+	
+	
+	Element base_node = stylesheet.getDocumentElement();
+	Node child = base_node.getFirstChild();
+	while(child != null) {
+	    String name = child.getNodeName();
+	    if (name.equals("xsl:import") || name.equals("xsl:include")) {
+		((Element)child).setAttribute("href", GSFile.stylesheetFile(gsdl3_home, site_name, collection, interface_name, base_interfaces, ((Element)child).getAttribute("href")));
+	    }
+	    child = child.getNextSibling();
+	}
+	
+    }
+
+    /** 
+     * looks through a stylesheet for <xxx:template match='template_name'>
+     * inside this template it looks for any <xxx:value-of select='metadataList/metadata[@name=yyy]> elements, and extracts the metadata names into a Vector
+     */
+    public static Vector extractWantedMetadata(Document stylesheet, String template_name) {
+	
+	Vector metadata = new Vector();
+	Element base_node = stylesheet.getDocumentElement();
+	NodeList templates = base_node.getElementsByTagNameNS("*","template");
+	for (int i=0; i<templates.getLength(); i++) {
+	    Element template = (Element)templates.item(i);
+	    String match_name = template.getAttribute("match");
+	    if (!match_name.equals(template_name)) {
+		continue; // we're only looking for specific templates
+	    }
+	    String mode = template.getAttribute("mode");
+	    if (!mode.equals("")) {   
+		continue; // we only want ones without modes - these are processing ones, not display ones
+	    }
+	    // we have one that we want to look through
+	    NodeList values = template.getElementsByTagNameNS("*", "value-of");
+	    for (int v=0; v<values.getLength(); v++) {
+		String select = ((Element)values.item(v)).getAttribute("select");
+		if (select.startsWith("metadataList/metadata[@name=")) {
+		    String []bits = select.split("'|\"");
+		    // there should be two quotes in teh string, therefore 3 items, and the second one is teh one we want
+		    String name = bits[1];
+		    metadata.add(name);
+		}
+	    }
+	}
+	return metadata;	
+    }
+
+    
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GlobalProperties.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GlobalProperties.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/GlobalProperties.java	(revision 21112)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.util;
+
+import java.util.Properties;
+import java.io.File;
+import java.io.InputStream;
+
+/** holds some global properties for the application. Read from a properties file */
+public class GlobalProperties {
+
+    private static Properties properties = null;
+    private static String properties_filename = "global.properties";
+    private static String gsdl3_home = null;
+    private static String gsdl3_web_address = null;
+    
+    // Note, that if the servlet is reloadable, then it is reloaded each time the file is changed.
+    static {
+	//load in the properties
+	properties = new Properties();
+	reload();	
+    }
+    
+    /** get the value of the property 'key'. returns null if not found */
+    public static String getProperty(String key) {
+	return properties.getProperty(key);
+    }
+
+    /** get the value of the property 'key'. 
+     * returns default_value if not found */
+    public static String getProperty(String key, String default_value) {
+	return properties.getProperty(key, default_value);
+    }
+
+    /** some special ones */
+    public static String getGSDL3Home() {
+	return gsdl3_home;
+    }
+
+    public static String getGSDL3WebAddress() {
+	return gsdl3_web_address;
+    }
+    
+    public static void reload() {
+	try {
+	    InputStream in = Class.forName("org.greenstone.gsdl3.util.GlobalProperties").getClassLoader().getResourceAsStream(properties_filename);
+	    if (in != null) {
+		System.err.println("loading global properties");
+		properties.load(in);
+		in.close();
+	    } else {
+		System.err.println("couldn't load gloabl properties!");
+	    }
+	    gsdl3_home = properties.getProperty("gsdl3.home");
+	    // make sure the path separators are correct
+	    File gs3_file = new File(gsdl3_home);
+	    gsdl3_home = gs3_file.getPath();
+	    gsdl3_web_address = "http://"+ properties.getProperty("tomcat.server")+":"+ properties.getProperty("tomcat.port")+"/"+properties.getProperty("tomcat.context");
+	} catch (Exception e) {
+	    System.err.println("Exception trying to reload global.properties: "+e);
+	}
+	
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MacroResolver.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MacroResolver.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MacroResolver.java	(revision 21112)
@@ -0,0 +1,119 @@
+/*
+ *    MacroResolver.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+abstract public class MacroResolver {
+
+    public static final String SCOPE_TEXT = "text";
+    public static final String SCOPE_META = "metadata";
+    public static final String SCOPE_ALL = "all";
+
+    public static final int TYPE_TEXT = 0;
+    public static final int TYPE_META = 1;
+    public static final int TYPE_DICT = 2;
+    
+    protected ArrayList text_macros = null;
+    protected ArrayList metadata_macros = null;
+
+    String lang = null;
+
+    public MacroResolver() {
+    	this.text_macros = new ArrayList();
+	this.metadata_macros = new ArrayList();
+    }
+    
+    public void addMacro(int type, String macro, String text_or_metadata, String scope) {
+	Macro m = new Macro();
+	m.type = type;
+	m.macro = macro;
+	m.text = text_or_metadata;
+	addMacro(m, scope);
+    }
+
+    public void addMacro(int type, String macro, String bundle, String key, String scope) {
+	Macro m = new Macro();
+	m.type = type;
+	m.macro = macro;
+	m.bundle = bundle;
+	m.key = key;
+	addMacro(m, scope);
+    }
+    
+    public void addMacros(Element replace_list_elem) {
+	NodeList replaces = replace_list_elem.getElementsByTagName("replace");
+	for (int i=0; i<replaces.getLength(); i++) {
+	    Element e = (Element)replaces.item(i);
+	    String scope = e.getAttribute("scope");
+	    if (scope.equals("")) {
+		scope = SCOPE_ALL;
+	    }
+	    String from = e.getAttribute("macro");
+	    String to = e.getAttribute("text");
+	    if (!to.equals("")) {
+		addMacro(TYPE_TEXT, from, to, scope);
+	    } else {
+		String meta = e.getAttribute("metadata");
+		if (!meta.equals("")) {
+		    addMacro(TYPE_META, from, meta, scope);
+		} else {
+		    String key = e.getAttribute("key");
+		    String bundle = e.getAttribute("bundle");
+		    addMacro(TYPE_DICT, from, bundle, key, scope);
+		}
+	    } 
+	}
+	
+    }
+
+    protected void addMacro(Macro m, String scope) {
+
+	if (scope.equals(SCOPE_TEXT)) {
+	    this.text_macros.add(m);
+	}
+	else if (scope.equals(SCOPE_META)) {
+	    this.metadata_macros.add(m);
+	}
+	else if (scope.equals(SCOPE_ALL)) {
+	    this.text_macros.add(m);
+	    this.metadata_macros.add(m);
+	    
+	}
+
+    }
+    
+    abstract public String resolve(String text, String lang, String scope, 
+				   String doc_oid);
+    
+    protected class Macro {
+	
+	public int type;
+	public String macro = null;
+	public String text = null;
+	public String key = null;
+	public String bundle = null;
+	
+    }
+	
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Misc.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Misc.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Misc.java	(revision 21112)
@@ -0,0 +1,84 @@
+ /*
+ *    Misc.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.util;
+
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Map;
+import java.util.Iterator;
+import java.util.Properties;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+/** contains miscellaneous functions */
+public class Misc {
+    
+    public static void printHash(HashMap map) {
+	Set entries = map.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+	}  
+    }
+    
+    /** Method to determine if the host system is Windows based.
+     * @return a boolean which is true if the platform is Windows, 
+     * false otherwise
+     */
+    public static boolean isWindows() {
+	Properties props = System.getProperties();
+	String os_name = props.getProperty("os.name","");
+	if(os_name.startsWith("Windows")) {
+	    return true;
+	}
+	return false;
+    }
+    
+    /** Method to determine if the host system is MacOS based.
+     * @return a boolean which is true if the platform is MacOS, false otherwise
+     */
+    public static boolean isMac() {
+        Properties props = System.getProperties();
+        String os_name = props.getProperty("os.name","");
+        if(os_name.startsWith("Mac OS")) {
+            return true;
+        }
+        return false;
+    }
+
+    public static BufferedReader makeHttpConnection(String url_string)
+	throws java.net.MalformedURLException, java.io.IOException {
+	BufferedReader reader = null;
+	URL url = new URL(url_string);
+	HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+	InputStream input = connection.getInputStream();
+	reader = new BufferedReader(new InputStreamReader(input));  
+	return reader;
+    }
+
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MyNodeList.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MyNodeList.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/MyNodeList.java	(revision 21112)
@@ -0,0 +1,46 @@
+/*
+ *    MyNodeList.java
+ *   
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+
+public class MyNodeList implements NodeList {
+    private ArrayList node_list = null;
+    
+    public MyNodeList() {
+	node_list = new ArrayList();
+    }
+    public int getLength() {
+	return node_list.size();
+    }
+    
+    public Node item(int i) {
+	if (i<0 || i >= node_list.size()) return null;
+	return (Node)node_list.get(i);
+    }
+    
+    public void addNode(Node n) {
+	node_list.add(n);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/NZDLParams.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/NZDLParams.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/NZDLParams.java	(revision 21112)
@@ -0,0 +1,25 @@
+package org.greenstone.gsdl3.util;
+
+public class NZDLParams extends GSParams {
+
+    public NZDLParams() {
+	super();
+
+	// now add in all the service ones we will be using
+	addParameter("stem",  true);	
+	addParameter("case",  true);
+	addParameter("index", true);
+	addParameter("matchMode",  true);
+	addParameter("maxDocs",  true);
+	addParameter("query", true);
+	addParameter("level",  true);
+	addParameter("qb",  true); // queryBox
+	addParameter("qm",  true); // queryMode
+	addParameter("qt",  true); // query type
+	addParameter("qfm",  true); // query form mode
+	addParameter("qfn",  true); // query form number of boxes
+	addParameter("ct",  true); // collection type - mg or mgpp
+	
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OID.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OID.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OID.java	(revision 21112)
@@ -0,0 +1,81 @@
+package org.greenstone.gsdl3.util;
+
+/** utility class to handle greenstone OIDs
+ *
+ * based around OIDtools.h from gsdl
+ */
+public class OID {
+    
+    /** returns everything up to the first dot 
+	if no dot, returns oid */
+    public static String getTop(String oid) {
+	int pos = oid.indexOf('.');
+	if (pos == -1) {
+	    return oid;
+	}
+	return oid.substring(0, pos);
+	
+    }
+    /** returns true is oid is top level (ie has no dots) 
+     returns false for an empty oid */
+    public static boolean isTop(String oid) {
+	if (oid.equals("")) {
+	    return false;
+	}
+	return (oid.indexOf('.')==-1);
+    }
+	
+    /** returns the parent of oid (everything up to last dot) 
+     returns oid if oid has no parent */
+    public static String getParent(String oid) {
+	int pos = oid.lastIndexOf('.');
+	if (pos == -1) {
+	    return oid;
+	}
+	return oid.substring(0, pos);
+    }
+
+    /** returns the full name - replaces all " with parent */
+    public static String translateParent(String oid, String parent) {
+	return oid.replaceAll("\"", parent); 
+    }
+    /** does the opposite to translate_parent */
+    public static String shrinkParent(String oid) {
+	int pos = oid.lastIndexOf('.');
+	if (pos==-1) return oid;
+	return "\""+oid.substring(pos);
+    }
+    /** returns true if oid uses .fc, .lc, .pr, .ns, .ps .rt (root) .ss (specified sibling)*/
+    public static boolean needsTranslating(String oid) {
+	String tail = oid.substring(oid.length()-3);
+	return (tail.equals(".fc") || tail.equals(".lc") || 
+		tail.equals(".pr") || tail.equals(".ns") || 
+		tail.equals(".ps") || tail.equals(".rt") || 
+		tail.equals(".ss"));
+    }
+    /** strips suffix from end */
+    public static String stripSuffix(String oid) {
+	String tail = oid.substring(oid.length()-3);
+	while (tail.equals(".fc") || tail.equals(".lc") || tail.equals(".pr")
+	       || tail.equals(".ns") || tail.equals(".ps") || tail.equals(".rt") || tail.equals(".ss")) {
+	    if (tail.equals(".ss")) { // have doc.sibnum.ss
+		oid = oid.substring(0, oid.length()-3);
+		int pos = oid.lastIndexOf('.');
+		//strip that too
+		oid = oid.substring(0, pos);
+	    } 
+	    oid = oid.substring(0, oid.length()-3);
+	    tail = oid.substring(oid.length()-3);
+	}
+	
+	return oid;
+    }
+    /** returns true if child is a child of parent 
+     an oid is not a child of itself */
+    public static boolean isChildOf(String parent, String child) {
+	if (parent.equals(child)) {
+	    return false;
+	}
+	return child.startsWith(parent);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OIDTest.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OIDTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/OIDTest.java	(revision 21112)
@@ -0,0 +1,94 @@
+package org.greenstone.gsdl3.util;
+
+import junit.framework.*; 
+
+public class OIDTest extends TestCase {
+
+    /** what type of test - used by the testing framework */
+    public static final String TEST_ALL_TEST_TYPE="UNIT";
+
+    private static final String empty = "";
+    private static final String oid1="HASH123";
+    private static final String oid2="HASH123.1";
+    private static final String oid3="HASH123.1.2";
+    private static final String oidfc = oid1+".fc";
+    private static final String oidlc = oid1+".lc";
+    private static final String oidpr = oid1+".pr";
+    private static final String oidns = oid1+".ns";
+    private static final String oidps = oid1+".ps";
+
+    public OIDTest(String name) {
+	super(name);
+    }
+
+    /** test suite that dynamically runs all the tests */
+    public static Test suite() {
+	return new TestSuite(OIDTest.class);
+    }
+
+    public void testGetTop() {
+	
+	assertEquals(oid1, OID.getTop(oid1));
+	assertEquals(oid1, OID.getTop(oid2));
+	assertEquals(empty, OID.getTop(empty));
+    }
+  
+    public void testIsTop() {
+	assertTrue(!OID.isTop(empty));
+	assertTrue(OID.isTop(oid1));
+	assertTrue(!OID.isTop(oid2));
+    }
+
+    public void testGetParent() {
+
+	assertEquals(empty, OID.getParent(oid1));
+	assertEquals(oid1, OID.getParent(oid2));
+	assertEquals(empty, OID.getParent(empty));
+	
+    }
+
+    public void testTranslateParent() {
+	String short_id = "\".1";
+	assertEquals(oid2, OID.translateParent(short_id, oid1));
+	// should return the original oid if its not a shortened form
+	assertEquals(oid2, OID.translateParent(oid2, oid1));
+	
+    }
+
+    public void testShrinkParent() {
+	String short_oid2 = "\".1";
+	String short_oid3 = "\".2";
+
+	assertEquals(oid1, OID.shrinkParent(oid1));
+	assertEquals(short_oid2, OID.shrinkParent(oid2));
+	assertEquals(short_oid3, OID.shrinkParent(oid3));
+    }
+
+    public void testNeedsTranslating() {
+	assertTrue(!OID.needsTranslating(oid1));
+	assertTrue(!OID.needsTranslating(oid2));
+	assertTrue(OID.needsTranslating(oidfc));
+	assertTrue(OID.needsTranslating(oidlc));
+	assertTrue(OID.needsTranslating(oidpr));
+	assertTrue(OID.needsTranslating(oidns));
+	assertTrue(OID.needsTranslating(oidps));
+
+    }
+
+    public void testStripSuffix() {
+	assertEquals(oid1, OID.stripSuffix(oidfc));
+	assertEquals(oid1, OID.stripSuffix(oidlc));
+	assertEquals(oid1, OID.stripSuffix(oidpr));
+	assertEquals(oid1, OID.stripSuffix(oidns));
+	assertEquals(oid1, OID.stripSuffix(oidps));
+    }
+
+    public void testIsChildOf() {
+	assertTrue(OID.isChildOf(oid1, oid2));
+	assertTrue(OID.isChildOf(oid1, oid3));
+	assertTrue(OID.isChildOf(oid2, oid3));
+	assertTrue(!OID.isChildOf(oid3, oid1));
+	assertTrue(!OID.isChildOf(oid1, oid1));
+    }
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Processing.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Processing.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/Processing.java	(revision 21112)
@@ -0,0 +1,98 @@
+package org.greenstone.gsdl3.util;
+
+import java.io.InputStreamReader;
+
+public class Processing {
+
+      /** Determine if the given process is still executing. It does this by attempting to throw an exception - not the most efficient way, but the only one as far as I know
+     * @param process the Process to test
+     * @return true if it is still executing, false otherwise
+     */
+    static public boolean processRunning(Process process) {
+	boolean process_running = false;
+
+	try {
+	    process.exitValue(); // This will throw an exception if the process hasn't ended yet.
+	}
+	catch(IllegalThreadStateException itse) {
+	    process_running = true;
+	}
+	catch(Exception exception) {
+	    exception.printStackTrace();	    
+	}
+	return process_running;
+    }
+    
+    static public int runProcess(String  command) {
+	
+	System.err.println("executing command "+command);
+	int exit_value = -1;
+	try {
+	    Process prcs = Runtime.getRuntime().exec(command);
+	    
+	    InputStreamReader error_stream = new InputStreamReader( prcs.getErrorStream(), "UTF-8" );
+	    InputStreamReader output_stream = new InputStreamReader( prcs.getInputStream(), "UTF-8"  );
+	    
+	    StringBuffer error_buffer = new StringBuffer();
+	    StringBuffer output_buffer = new StringBuffer();
+	    
+	    while(processRunning(prcs)) {
+		// Hopefully this doesn't block if the process is trying to write to STDOUT.
+		if((error_stream!=null)) {// && error_stream.ready()) { 
+		    while (error_stream.ready()) {
+			error_buffer.append((char)error_stream.read());
+		    }
+		    //error_buffer = get_stream_char(error_stream,error_buffer,bos);
+		}
+		//Hopefully this won't block if the process is trying to write to STDERR
+		/*else if*/ while (output_stream.ready()) {
+		    output_buffer.append((char)output_stream.read());
+		    //output_buffer = get_stream_char(output_stream,output_buffer,bos);
+		}
+		//else {
+		try {
+		    Thread.sleep(100);
+		}
+		catch(Exception exception) {
+		}
+		// }
+	    }
+	    
+	    // Of course, just because the process is finished doesn't
+	    // mean the incoming streams are empty. Unfortunately I've
+	    // got no chance of preserving order, so I'll process the
+	    // error stream first, then the out stream
+	    while(error_stream.ready()) {
+		error_buffer.append((char)error_stream.read());
+		//error_buffer = get_stream_char(error_stream,error_buffer,bos);
+	    }
+	    
+	    while(output_stream.ready()) {
+		output_buffer.append((char)output_stream.read());
+		//output_buffer = get_stream_char(output_stream,output_buffer,bos);
+	    }
+	    
+	    // do something with the messages
+	    System.err.println("err>"+error_buffer.toString());
+	    System.err.println("out>"+output_buffer.toString());
+	    // Ensure that any messages still remaining in the string buffers are fired off.
+	    // Now display final message based on exit value
+	    
+	    prcs.waitFor(); 
+	    error_stream.close();
+	    output_stream.close();
+	    error_buffer = null;
+	    output_buffer = null;
+	    
+	    exit_value =  prcs.exitValue();
+	} catch (Exception e) {
+	    System.err.println("runProcess ERROR: "+e);
+	    return 1;
+	}
+	return exit_value;
+	
+	
+    }
+   
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/SQLQuery.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/SQLQuery.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/SQLQuery.java	(revision 21112)
@@ -0,0 +1,416 @@
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnectionFactory;
+import org.greenstone.gsdl3.gs3build.database.GS3SQLConnection;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.util.ArrayList;
+
+public class SQLQuery {
+    GS3SQLConnection connection = null;
+    
+    public SQLQuery() {
+
+    }
+
+    public boolean setDatabase(String db_name) {
+	connection = GS3SQLConnectionFactory.getGS3SQLConnection(db_name);
+	try {
+
+	    // test the connection
+	    String query = "select * from "+GSSQL.DOCUMENT_TABLE+GSSQL.END;
+	    Statement s = connection.createStatement();
+	    s.execute(query);
+	    s.close();
+	    return true;
+	} catch (Exception e) {
+	    System.err.println("SQLQuery.setDatabase():"+e);
+	}
+	closeConnection();
+	return false;
+    }
+
+    public void closeConnection() {
+	if (connection != null) {
+	    connection.close();
+	    connection=null;
+	}
+    }
+
+    public String MGNum2OID(String mg_num) {
+	// get doc id and meta id
+	String query = "select "+ 
+	    GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+
+	    GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+
+	    " from "+GSSQL.METADATA_VALUE_TABLE+GSSQL.COMMA+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+
+	    " where "+
+	    GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.LABEL+GSSQL.EQUALS_QUOTE+GSSQL.MG_NUM+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.VALUE+GSSQL.EQUALS_QUOTE+mg_num+GSSQL.QUOTE+
+	    " and "+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String doc_id = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		doc_id = results.getString(GSSQL.DOCUMENT_ID);
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    System.err.println("SQLQuery.MGNum2OID(): "+e);
+	    return null;
+	}
+
+	// now get division label
+	query = "select "+
+	    GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+
+	    " from "+GSSQL.DIVISION_TABLE+GSSQL.COMMA+GSSQL.DIVISION_METADATA_TABLE+
+	    " where "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    GSSQL.END;
+
+	
+	String short_label = null;
+	try {
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		short_label = results.getString(GSSQL.SHORT_LABEL);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	    statement.close();
+	} catch (SQLException e) {
+
+	    System.err.println("SQLQuery.MGNum2OID() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return GS3OID.createOID(doc_id, short_label);
+    }
+
+    public String OID2MGNum(String oid) {
+	String id = getDocumentMetadata(oid, "gsdl3.mgseqno");
+	return id;
+
+    }
+
+    // does the document have sections, or is it just a single page document?
+    public boolean isHierarchicalDocument(String oid) {
+	// first just check the OID - it may already have section info in it
+	if (!GS3OID.isDocTop(oid)) {
+	    return true;
+	}
+
+	// now we need to check the database
+	String query = "select * "+ 
+	    " from "+GSSQL.STRUCTURE_TABLE +
+	    " where "+ GSSQL.DOCUMENT_ID +GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and "+ GSSQL.STRUCTURE_ID +GSSQL.EQUALS_QUOTE + "Section" + GSSQL.QUOTE +
+	    GSSQL.END;
+	boolean is_hierarchical = false;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    is_hierarchical = results.first();
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    return false;
+	}
+	
+	return is_hierarchical;
+    }
+    
+    // classifier metadata is not actually metadata
+    public String getClassifierMetadata(String oid, String full_meta_name) {
+
+	if (!full_meta_name.equals("Title") && !full_meta_name.equals("numleafdocs")) {
+	    return null;
+	}
+	// get the description
+	String field_name="";
+	if (full_meta_name.equals("Title")) {
+	    field_name = GSSQL.DESCRIPTION;
+	} else {
+	    field_name = GSSQL.NUM_LEAF_DOCUMENTS;
+	}
+	
+	String query = "select "+ field_name+
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    GSSQL.END;
+	
+	String value = null;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    if (results.first()) {
+		value = results.getString(field_name);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	return value;
+	
+    }
+    
+    public ArrayList getDocumentChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+	// get the division ref for this section
+	String query = "select "+GSSQL.DIVISION_REF +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    GSSQL.END;
+
+	ResultSet results = null;
+	Statement statement = null;
+	String div_ref = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		div_ref = results.getString(GSSQL.DIVISION_REF);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error1: "+e.getMessage());
+	    return null;
+	}
+
+	// now get the list of children
+	query = "select "+GSSQL.SHORT_LABEL +
+	    " from "+GSSQL.DIVISION_TABLE+
+	    " where "+ GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_TYPE+GSSQL.EQUALS_QUOTE+GSSQL.DIVISION+GSSQL.QUOTE+
+	    " and "+GSSQL.PARENT_REF+GSSQL.EQUALS_QUOTE+div_ref+GSSQL.QUOTE+
+	    GSSQL.END;
+	try {
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.SHORT_LABEL);
+		children.add(GS3OID.createOID(doc_id, id));
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getDocumentChildren() Error2: "+e.getMessage());
+	    return null;
+	}
+
+	return children;
+	
+    }
+    
+    public boolean documentHasChildren(String oid) {
+	// for now, use getChildren, but can we do this a better way??
+	ArrayList children = getDocumentChildren(oid);
+	if (children==null || children.size()==0) {
+	    return false;
+	}
+	return true;
+	
+    }
+    public ArrayList getClassifierChildren(String oid) {
+
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASSIFIER_ID +
+	    " from " + GSSQL.CLASSIFIER_TABLE +
+	    " where " + GSSQL.PARENT_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.NUM_LEAF_DOCUMENTS + GSSQL.NOT+GSSQL.EQUALS_QUOTE+ "0"+GSSQL.QUOTE +
+	    GSSQL.END; 
+	
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.CLASSIFIER_ID);
+		children.add(id);
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+	    
+	    System.err.println("SQLQuery.getClassifierChildren Error: "+e.getMessage());
+	    return null;
+	}
+	return children;
+    }
+
+    public ArrayList getClassifierDocChildren(String oid) {
+	ArrayList children = new ArrayList();
+
+	String query = "select "+ GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.DOCUMENT_ID+
+	    " from " + GSSQL.CLASSIFIER_TABLE + GSSQL.COMMA+GSSQL.CLASS_DOCUMENT_TABLE+
+	    " where " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_ID + GSSQL.EQUALS_QUOTE + oid + GSSQL.QUOTE +
+	    " and " + GSSQL.CLASSIFIER_TABLE +GSSQL.DOT+ GSSQL.CLASSIFIER_REF+GSSQL.EQUALS+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+ GSSQL.CLASSIFIER_REF+
+	    " order by "+GSSQL.CLASS_DOCUMENT_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ORDER+
+	    GSSQL.END;
+	try {
+	    Statement statement = connection.createStatement();
+	    ResultSet results = statement.executeQuery(query);
+	    while (results.next()) {
+		String id = results.getString(GSSQL.DOCUMENT_ID);
+		children.add(id);
+	    }
+	} catch (java.sql.SQLException e) {
+		
+	    System.err.println("SQLQuery.getClassifierDocChildren() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return children;
+    }
+    
+    public String getDocumentMetadata(String oid, String full_meta_name) {
+
+	String doc_id = oid;
+	String section_id = "All"; // if we are just passed a doc id
+	if (!GS3OID.isDocTop(oid)) {
+	    doc_id = GS3OID.getDocID(oid);
+	    section_id = GS3OID.getSectionLabel(oid);
+	}
+//  	int sep_index = oid.indexOf(".");
+//  	if (sep_index != -1) {
+//  	    doc_id = oid.substring(0, sep_index);
+//  	    section_id = oid.substring(sep_index+1);
+//  	}
+
+
+	// get the metadata block id
+	String query = "select "+ 
+	    GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID +
+	    " from " + GSSQL.DIVISION_METADATA_TABLE+GSSQL.COMMA+GSSQL.DIVISION_TABLE+
+	    " where " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+GSSQL.EQUALS+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DIVISION_REF+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.SHORT_LABEL+GSSQL.EQUALS_QUOTE+section_id+GSSQL.QUOTE+
+	    " and " + GSSQL.DIVISION_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and "+GSSQL.DIVISION_METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+ 
+	    GSSQL.END;
+	
+	Statement statement = null;
+	ResultSet results = null;
+	String meta_id = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    if (results.first()) {
+		meta_id = results.getString(GSSQL.METADATA_ID);
+	    } else {
+		statement.close();
+		return null;
+	    }
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	// now get the list of namespace refs for the right namespace
+	int sep_index = full_meta_name.indexOf('.');
+	String meta_ns = "gsdl3";
+	String meta_name = full_meta_name;
+	if (sep_index != -1) {
+
+	    meta_ns = full_meta_name.substring(0, sep_index);
+	    meta_name = full_meta_name.substring(sep_index+1);
+	}
+
+	query = "select * "+ 
+	    " from "+GSSQL.NAMESPACE_TABLE+GSSQL.COMMA+GSSQL.METADATA_TABLE+GSSQL.COMMA+GSSQL.METADATA_VALUE_TABLE+
+	    " where "+GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_ID+GSSQL.EQUALS_QUOTE+meta_id+GSSQL.QUOTE+
+	    " and "+ GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.DOCUMENT_ID+GSSQL.EQUALS_QUOTE+doc_id+GSSQL.QUOTE+
+	    " and " + GSSQL.METADATA_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+GSSQL.EQUALS+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.METADATA_REF+
+	    " and "+ GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_TYPE+GSSQL.EQUALS_QUOTE+meta_ns+GSSQL.QUOTE+
+	    " and "+GSSQL.NAMESPACE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+GSSQL.EQUALS+GSSQL.METADATA_VALUE_TABLE+GSSQL.DOT+GSSQL.NAMESPACE_REF+
+	    GSSQL.END;
+	
+	String meta_value = null;
+	try {
+	    statement = connection.createStatement();
+	    results = statement.executeQuery(query);
+	    while (results.next()) {
+		String m_name = results.getString(GSSQL.LABEL);
+		if (meta_name.equals(m_name)) {
+		    meta_value = results.getString(GSSQL.VALUE);
+		    break;
+		}
+	    }
+	    statement.close();
+	} catch (java.sql.SQLException e) {
+
+	    System.err.println("SQLQuery.getDocumentMetadata() Error: "+e.getMessage());
+	    return null;
+	}
+	
+	return meta_value;
+    }
+    public static void main (String [] args) {
+
+	SQLQuery self = new SQLQuery();
+	self.setDatabase("gs3test");
+	System.out.println("getting oid for mg num 3");
+	String oid = self.MGNum2OID("stx.3");
+	System.out.println("oid was "+oid);
+	
+	System.out.println("title metadata for mg num 3:");
+	String meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 3:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	String id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+
+	System.out.println("getting oid for mg num 8");
+	oid = self.MGNum2OID("stx.8");
+	System.out.println("oid was "+oid);
+
+	System.out.println("title metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.title");
+	if (meta != null) {System.out.println(meta);}
+	System.out.println("subject metadata for mg num 8:");
+	meta = self.getDocumentMetadata(oid, "gsdl3.Subject");
+	if (meta != null) {System.out.println(meta);}
+
+	System.out.println("converting back to mg, oid "+oid);
+	id = self.OID2MGNum(oid);
+	System.out.println(""+id);
+    }
+
+
+
+
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLConverter.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLConverter.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLConverter.java	(revision 21112)
@@ -0,0 +1,337 @@
+/*
+ *    XMLConverter.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+// XML classes
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+import org.xml.sax.InputSource;
+import org.xml.sax.EntityResolver;
+import org.apache.xerces.parsers.DOMParser;
+import org.apache.xerces.dom.DocumentImpl; // for new Documents
+
+// other java classes
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+
+/** XMLConverter - utility class for greenstone
+ *
+ * parses XML Strings into Documents, converts Nodes to Strings
+ * different parsers have different behaviour - can experiment in here
+ * now we only use xerces
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ *
+ */
+public class XMLConverter {
+
+    /** xerces parser */
+    protected DOMParser parser = null;
+
+    private boolean outputEscaping = true;
+
+
+     /** the no-args constructor */
+    public XMLConverter() {
+	try {
+	    this.parser = new DOMParser();
+	    this.parser.setFeature("http://xml.org/sax/features/validation", false);
+
+	    // a performance test showed that having this on lead to increased 
+	    // memory use for small-medium docs, and not much gain for large 
+	    // docs.
+	    // http://www.sosnoski.com/opensrc/xmlbench/conclusions.html
+	    this.parser.setFeature("http://apache.org/xml/features/dom/defer-node-expansion", false);
+	} catch (Exception e) {
+	    System.err.println("XMLConverter:exception "+e.getMessage());
+	}
+    }
+
+    /** sets the entity resolver. pass in null to unset it */
+    public void setEntityResolver(EntityResolver er) {
+	this.parser.setEntityResolver(er);
+    }
+	
+    /** returns a DOM Document */
+    public Document getDOM(String in) {
+	    
+	try {
+	    Reader reader = new StringReader(in);
+	    InputSource xml_source = new InputSource(reader);
+	    
+	    this.parser.parse(xml_source);
+	    Document doc = this.parser.getDocument();
+	    
+	    return doc;
+	    
+	} catch (Exception e) {
+	    System.err.println("XMLConverter.getDOM(String): exception "+e.getMessage());
+	}
+	return null;
+    }
+  
+    /** returns a DOM Document */
+    public Document getDOM(File in) {
+	try {
+	    
+	    FileReader reader = new FileReader(in);
+	    InputSource xml_source = new InputSource(reader);
+		    
+	    this.parser.parse(xml_source);
+	    Document doc = this.parser.getDocument();
+	    
+	    return doc;
+
+	} catch (Exception e) {
+	    System.err.println("XMLConverter.getDOM(File): exception "+e.getMessage());
+	    e.printStackTrace();
+	}
+	return null;
+    }
+
+    /** returns a DOM document */
+    public Document getDOM(File in, String encoding) {
+	try {
+	    
+	    InputStreamReader isr = new InputStreamReader(new FileInputStream(in), encoding);
+	    InputSource xml_source = new InputSource(isr);
+	    
+	    this.parser.parse(xml_source);
+	    Document doc = this.parser.getDocument();
+	    
+	    return doc;
+
+	} catch (Exception e) {
+	    System.err.println("XMLConverter.getDOM(File): exception "+e.getMessage());
+	}
+	return null;
+    }
+    
+
+    /** creates a new empty DOM Document */
+    public Document newDOM() {
+	Document doc = new DocumentImpl(); 
+	return doc;
+    }
+
+    /** returns the Node as a String */
+    public String getString(Node xmlNode)
+    {
+	outputEscaping = true;
+	StringBuffer xmlRepresentation = new StringBuffer();
+	getString(xmlNode, xmlRepresentation, 0, false);
+	return xmlRepresentation.toString();
+    }
+
+    /** returns the node as a nicely formatted String - this introduces extra
+     * text nodes if the String is read back in as a DOM, so should only be 
+     * used for printing */
+    public String getPrettyString(Node xmlNode) {
+	
+	outputEscaping = true;
+	StringBuffer xmlRepresentation = new StringBuffer();
+	getString(xmlNode, xmlRepresentation, 0, true);
+	return xmlRepresentation.toString();
+    }
+
+    private void getString(Node xmlNode, StringBuffer xmlRepresentation, 
+			     int depth, boolean pretty)
+    {
+	
+	if (xmlNode == null) {
+	    xmlRepresentation.append("<null>");
+	    return;
+	}
+
+	short nodeType = xmlNode.getNodeType();
+	String nodeName = xmlNode.getNodeName();
+
+	if (nodeType == Node.DOCUMENT_NODE) {
+	    getString(((Document)xmlNode).getDocumentElement(), xmlRepresentation, depth, pretty);
+	    return;
+	}
+	// Handle Element nodes
+	if (nodeType == Node.ELEMENT_NODE) {
+	    if (pretty) {
+		xmlRepresentation.append("\n");
+		for (int i = 0; i < depth; i++) {
+		    xmlRepresentation.append("  ");
+		}
+	    }
+
+	    // Write opening tag
+	    xmlRepresentation.append("<");
+	    xmlRepresentation.append(nodeName);
+	    
+	    // Write the node attributes
+	    NamedNodeMap nodeAttributes = xmlNode.getAttributes();
+	    for (int i = 0; i < nodeAttributes.getLength(); i++) {
+		Node attribute = nodeAttributes.item(i);
+		xmlRepresentation.append(" ");
+		xmlRepresentation.append(attribute.getNodeName());
+		xmlRepresentation.append("=\"");
+		xmlRepresentation.append(attribute.getNodeValue());
+		xmlRepresentation.append("\"");
+	    }
+
+	    // If the node has no children, close the opening tag and return
+	    if (xmlNode.hasChildNodes() == false) {
+		// This produces somewhat ugly output, but it is necessary to compensate
+		// for display bugs in Netscape. Firstly, the space is needed before the
+		// closing bracket otherwise Netscape will ignore some tags (<br/>, for
+		// example). Also, a newline character would be expected after the tag,
+		// but this causes problems with the display of links (the link text
+		// will contain a newline character, which is displayed badly).
+		xmlRepresentation.append(" />");
+		return;
+	    }
+	    
+	    // Close the opening tag
+	    xmlRepresentation.append(">");
+	    
+	    // Apply recursively to the children of this node
+	    // hack for nodes next to text nodes - dont make them pretty
+	    // this is needed for text inside a <pre> element - any new lines 
+	    // or spaces around the span elements show up in the text
+	    NodeList children = xmlNode.getChildNodes();
+	    boolean do_pretty = pretty;
+	    for (int i = 0; i < children.getLength(); i++) {
+		if (children.item(i).getNodeType()==Node.TEXT_NODE) {
+		    do_pretty=false; // if there is a text node amongst the children, do teh following nodes in non-pretty mode - hope this doesn't stuff up something else
+		}
+		getString(children.item(i), xmlRepresentation, depth + 1, do_pretty);
+	    }
+	    
+	    // Write closing tag
+	    if (pretty) {
+		if (xmlRepresentation.charAt(xmlRepresentation.length()-1) == '\n') {
+		    for (int i = 0; i < depth; i++)
+			xmlRepresentation.append("  ");
+		}
+	    }
+	    xmlRepresentation.append("</");
+	    xmlRepresentation.append(nodeName);
+	    xmlRepresentation.append(">");
+	    if (pretty) {
+		xmlRepresentation.append("\n");
+	    }
+	}
+	
+	// Handle Text nodes
+	else if (nodeType == Node.TEXT_NODE) {
+	    String text = xmlNode.getNodeValue();
+
+	    // Perform output escaping, if required
+	    if (outputEscaping) {
+		text = text.replaceAll("&", "&amp;");  // Must be done first!!
+		text = text.replaceAll("<", "&lt;");
+		text = text.replaceAll(">", "&gt;");
+		text = text.replaceAll("\"", "&quot;");
+		text = text.replaceAll("\'", "&apos;");
+	    }
+
+	    // Remove any control-C characters
+	    text = text.replaceAll("" + (char) 3, "");
+	    xmlRepresentation.append(text);
+	}
+
+	// Handle Processing Instruction nodes
+	else if (nodeType == Node.PROCESSING_INSTRUCTION_NODE) {
+	    if (nodeName.equals("javax.xml.transform.disable-output-escaping")) {
+		outputEscaping = false;
+	    }
+	    else if (nodeName.equals("javax.xml.transform.enable-output-escaping")) {
+		outputEscaping = true;
+	    }
+	    else {
+		System.err.println("XMLConverter.getString(): Warning: Unhandled processing instruction " + nodeName);
+	    }
+	}
+	
+	else if (nodeType == Node.COMMENT_NODE) {
+	    String text = xmlNode.getNodeValue();
+	    xmlRepresentation.append("<!-- ");
+	    xmlRepresentation.append(text);
+	    xmlRepresentation.append(" -->");
+	}
+	    
+	    
+	
+	// A type of node that is not handled yet
+	else {
+	    System.err.println("XMLConverter.getString(): Warning: Unknown node type: " + nodeType+" "+getNodeTypeString(nodeType));
+	}
+
+	return;
+    }
+
+    protected static String getNodeTypeString(short node_type) {
+
+	String type = "";
+	switch(node_type) {
+	case Node.ATTRIBUTE_NODE: 
+	    type="ATTRIBUTE_NODE";
+	    break;
+	case Node.CDATA_SECTION_NODE:
+	    type="CDATA_SECTION_NODE";
+	    break;
+	case Node.COMMENT_NODE:
+	    type="COMMENT_NODE";
+	    break;
+	case Node.DOCUMENT_FRAGMENT_NODE:
+	    type="DOCUMENT_FRAGMENT_NODE";
+	    break;
+	case Node.DOCUMENT_NODE:
+	    type="DOCUMENT_NODE";
+	    break;
+	case Node.DOCUMENT_TYPE_NODE:
+	    type="DOCUMENT_TYPE_NODE";
+	    break;
+	case Node.ELEMENT_NODE:
+	    type="ELEMENT_NODE";
+	    break;
+	case Node.ENTITY_NODE:
+	    type="ENTITY_NODE";
+	    break;
+	case Node.ENTITY_REFERENCE_NODE:
+	    type="ENTITY_REFERENCE_NODE";
+	    break;
+	case Node.NOTATION_NODE:
+	    type="NOTATION_NODE";
+	    break;
+	case Node.PROCESSING_INSTRUCTION_NODE:
+	    type="PROCESSING_INSTRUCTION_NODE";
+	    break;
+	case Node.TEXT_NODE:
+	    type="TEXT_NODE";
+	    break;
+	default:
+	    type="UNKNOWN";
+	}
+
+	return type;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLTransformer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLTransformer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XMLTransformer.java	(revision 21112)
@@ -0,0 +1,192 @@
+/*
+ *    XMLTransformer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+// XML classes
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.dom.DOMResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+// other java classes
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Map;
+import java.util.Iterator;
+/** XMLTransformer - utility class for greenstone
+ *
+ * transforms xml using xslt
+ *
+ * @author <a href="mailto:kjdon@cs.waikato.ac.nz">Katherine Don</a>
+ * @version $Revision$
+ */
+public class XMLTransformer {
+
+  /** The transformer factory we're using */
+    TransformerFactory t_factory=null;
+  
+  /**
+   * The no-arguments constructor. 
+   *
+   * Any exceptions thrown are caught internally
+   * 
+   * @see javax.xml.transform.TransformerFactory
+   */
+    public XMLTransformer() {
+
+	// make sure we are using the xalan transformer
+	System.setProperty("javax.xml.transform.TransformerFactory", "org.apache.xalan.processor.TransformerFactoryImpl");
+	try {
+	    this.t_factory = org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+
+	} catch (Exception e) {
+	    System.err.println("XMLTransformer() exception "+e.getMessage());
+	}
+    }
+
+ 
+	
+  /**
+   * Transform an XML document using a XSLT stylesheet
+   * 
+   * @param stylesheet a filename for an XSLT stylesheet
+   * @param xml_in the XML to be transformed
+   * @return the transformed XML
+   */
+    public String transform(String stylesheet, String xml_in) {
+	
+	try {
+	    // Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+	    Transformer transformer = this.t_factory.newTransformer(new StreamSource(stylesheet));
+
+	    // Use the Transformer to transform an XML Source and send the output to a Result object.
+	    StringWriter output = new StringWriter();
+
+	    transformer.transform(new StreamSource(new StringReader(xml_in)), new StreamResult(output));
+	    return output.toString();
+	} catch (TransformerConfigurationException e) {
+	    System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessageAndLocation());
+	    System.err.println(e.getLocationAsString());  
+	    return "";
+	} catch (TransformerException e) {
+	    System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	    return "";
+	}	
+    }
+
+    public String transformToString(Document stylesheet, Document source) {
+	return transformToString(stylesheet, source, null);
+    }
+    
+    public String transformToString(Document stylesheet, Document source, HashMap parameters) {
+	
+	try {
+	    // Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+	    Transformer transformer = this.t_factory.newTransformer(new DOMSource(stylesheet));
+	    if (parameters != null) {
+		Set params = parameters.entrySet();
+		Iterator i = params.iterator();
+		while (i.hasNext()) {
+		    Map.Entry m = (Map.Entry)i.next();
+		    transformer.setParameter((String)m.getKey(), m.getValue());
+		}
+	    }
+	    //transformer.setParameter("page_lang", source.getDocumentElement().getAttribute(GSXML.LANG_ATT));
+
+	    
+	    // Use the Transformer to transform an XML Source and send the output to a Result object.
+	    StringWriter output = new StringWriter();
+	    
+	    transformer.transform(new DOMSource(source), new StreamResult(output));
+	    return output.toString();
+	} catch (TransformerConfigurationException e) {
+	    System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessageAndLocation());
+	    System.err.println(e.getLocationAsString());  
+	    return "";
+	} catch (TransformerException e) {
+	    System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	    return "";
+	}	
+      }
+
+    public Node transform(Document stylesheet, Document source) {
+	return transform(stylesheet, source, null);
+    }
+
+    public Node transform(Document stylesheet, Document source, HashMap parameters) {
+
+	try {
+	    // Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+	    Transformer transformer = this.t_factory.newTransformer(new DOMSource(stylesheet));
+	    if (parameters != null) {
+		Set params = parameters.entrySet();
+		Iterator i = params.iterator();
+		while (i.hasNext()) {
+		    Map.Entry m = (Map.Entry)i.next();
+		    transformer.setParameter((String)m.getKey(), m.getValue());
+		}
+	    }
+
+	    DOMResult result = new DOMResult();
+	    transformer.transform(new DOMSource(source), result);
+	    return result.getNode().getFirstChild();
+	} catch (TransformerConfigurationException e) {
+	    System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessageAndLocation());
+	    System.err.println(e.getLocationAsString());  
+	    return null;
+	} catch (TransformerException e) {
+	    System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	    return null;
+	}	
+	
+
+    }
+
+    public Node transform(File stylesheet, File source) {
+
+	try {
+	    Transformer transformer = this.t_factory.newTransformer(new StreamSource(stylesheet));
+	    
+	    DOMResult result = new DOMResult();
+	    transformer.transform(new StreamSource(source), result);
+	    return result.getNode().getFirstChild();
+	} catch (TransformerConfigurationException e) {
+	    System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessageAndLocation());
+	    System.err.println(e.getLocationAsString());  
+	    return null;
+	} catch (TransformerException e) {
+	    System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	    return null;
+	}	
+    }
+}
+
+ 
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XSLTUtil.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XSLTUtil.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/gsdl3/util/XSLTUtil.java	(revision 21112)
@@ -0,0 +1,144 @@
+package org.greenstone.gsdl3.util;
+
+import java.util.Date;
+import java.util.Locale;
+import java.text.SimpleDateFormat;
+
+/** a class to contain various static methods that are used by the xslt
+ * stylesheets
+ */
+public class XSLTUtil {
+
+    /* some tests */
+    public static boolean equals(String s1, String s2) {
+	return s1.equals(s2);
+    }
+    public static boolean notEquals(String s1, String s2) {
+	return !s1.equals(s2);
+    }
+    public static boolean exists(String s1, String s2) {
+	return !s1.equals("");
+    }
+    public static boolean contains(String s1, String s2) {
+	return (s1.indexOf(s2) != -1);
+    }
+    public static boolean startsWith(String s1, String s2) {
+	return s1.startsWith(s2);
+    }
+    public static boolean endsWith(String s1, String s2) {
+	return s1.endsWith(s2);
+    }
+    public static boolean lessThan(String s1, String s2) {
+	return (s1.compareTo(s2) < 0);
+    }
+    public static boolean lessThanOrEquals(String s1, String s2) {
+	return (s1.compareTo(s2) <= 0);
+    }
+    public static boolean greaterThan(String s1, String s2) {
+	return (s1.compareTo(s2) > 0);
+    }
+    public static boolean greaterThanOrEquals(String s1, String s2) {
+	return (s1.compareTo(s2) >= 0);
+    }
+
+    /* some preprocessing functions */
+    public static String toLower(String orig) {
+	return orig.toLowerCase();
+    }
+    public static String toUpper(String orig) {
+	return orig.toUpperCase();
+    }
+
+    public static byte[] toUTF8(String orig) {
+	
+	try {
+	    byte[] utf8 = orig.getBytes("UTF-8");
+	    return utf8;
+	}
+	catch (Exception e){
+	    System.err.println("XSLTUtil.toUTF8: unsupported encoding");
+	    return orig.getBytes();
+	}
+    }
+
+    public static String getNumberedItem(String list, int number) {
+	String [] items = list.split(",", -1);	
+	if (items.length > number) {
+	    return items[number];
+	}
+	return ""; // index out of bounds
+    }
+
+    
+    public static String tidyWhitespace(String original) {
+	
+	if (original==null || original.equals("")) {
+	    return original;
+	}
+	String new_s = original.replaceAll("\\s+", " ");
+	return new_s;
+    }
+
+
+    public static String getInterfaceText(String interface_name, String lang, String key) {
+	return getInterfaceText(interface_name, lang, key, null);
+    }
+    public static String getInterfaceText(String interface_name, String lang, String key, String args_str) {
+	String [] args = null;
+	if (args_str!=null && !args_str.equals("")) {
+	    args = args_str.split(";");
+	}
+	Dictionary dict = new Dictionary("interface_"+interface_name, lang);
+        String result = dict.get(key, args);
+
+	if (result == null && !interface_name.equals("default")) { // not found, try the default interface
+	    dict = new Dictionary("interface_default", lang);
+	    result = dict.get(key, args);
+	}
+	    
+        if (result == null) { // not found
+            return "_"+key+"_";
+        }
+        return result;
+    }
+
+    public static boolean isImage(String mimetype) {
+	if (mimetype.startsWith("image/")) {
+	    return true;
+	}
+	return false;
+    }
+
+    public static String formatDate(String date, String lang) {
+	
+	String in_pattern = "yyyyMMdd";
+	String out_pattern = "dd MMMM yyyy";
+	if (date.length()==6) {
+	    in_pattern = "yyyyMM";
+	}
+	
+	SimpleDateFormat formatter = new SimpleDateFormat(in_pattern, new Locale(lang));
+	try {
+	    Date d = formatter.parse(date);
+	    formatter.applyPattern(out_pattern);
+	    String new_date = formatter.format(d);
+	    return new_date;
+	} catch (Exception e) {
+	    return date;
+	}
+	
+    }
+
+    public static String formatLanguage(String display_lang, String lang) {
+
+	return new Locale(display_lang).getDisplayLanguage(new Locale(lang));
+    }
+
+    public static String cgiSafe(String original, String lang) {
+
+	original = original.replace('&', ' ');
+	original = original.replaceAll(" ", "%20");
+	return original;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/ClassFinder.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/ClassFinder.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/ClassFinder.java	(revision 21112)
@@ -0,0 +1,83 @@
+/** this file originally downloaded from 
+ *http://www.javaworld.com/javaworld/jw-12-2000/junit/jw-1221-junit.zip
+ *
+ */
+
+package org.greenstone.testing;
+
+import java.util.*;
+import java.io.*;
+/*
+ * This used to use JCF. However, JCF seems to have dissapeared from the web.
+ * import lti.java.jcf.*;
+ */
+
+
+/**
+ * This class is responsible for searching a directory for class files. It builds
+ * a list of fully qualified class names from the class files in the directory tree.
+ */
+public class ClassFinder {
+    final private Vector classNameList = new Vector ();
+    final private int startPackageName;
+
+    /**
+     * Construct the class finder and locate all the classes in the directory structured
+     * pointed to by <code>classPathRoot</code>. Only classes in the package <code>packageRoot</code>
+     * are considered.
+     */
+    public ClassFinder(final File classPathRoot, final String packageRoot) throws IOException {
+        startPackageName = classPathRoot.getAbsolutePath().length() + 1;
+        String directoryOffset = packageRoot.replace ('.', File.separatorChar);
+        findAndStoreTestClasses (new File (classPathRoot, directoryOffset));
+    }
+    
+    /**
+     * Given a file name, guess the fully qualified class name.
+     */
+    private String computeClassName (final File file) {
+        String absPath = file.getAbsolutePath();
+        String packageBase = absPath.substring (startPackageName, absPath.length () - 6);
+        String className;
+        className = packageBase.replace(File.separatorChar, '.');
+        return className;
+    }
+
+    /**
+     * This method does all the work. It runs down the directory structure looking
+     * for java classes.
+     */
+    private void findAndStoreTestClasses (final File currentDirectory) throws IOException {
+        String files[] = currentDirectory.list();
+        for(int i = 0;i < files.length;i++) {
+            File file = new File(currentDirectory, files[i]);
+            String fileBase = file.getName ();
+            int idx = fileBase.indexOf(".class");
+            final int CLASS_EXTENSION_LENGTH = 6;
+            
+            if(idx != -1 && (fileBase.length() - idx) == CLASS_EXTENSION_LENGTH) {
+/* 
+ * This used to use JCF. However, JCF seems to have dissapeared from the web so we fallback
+ * to a less elegant method. We compute the class name from the file name :-(
+ *               JcfClassInputStream inputStream = new JcfClassInputStream(new FileInputStream (file));
+ *               JcfClassFile classFile = new JcfClassFile (inputStream);
+ *               System.out.println ("Processing: " + classFile.getFullName ().replace ('/','.'));
+ *               classNameList.add (classFile.getFullName ().replace ('/','.'));
+ */
+                String className = computeClassName (file);
+                classNameList.add (className);
+            } else {
+                if(file.isDirectory()) {
+                    findAndStoreTestClasses (file);
+                }
+            }
+        }
+    }
+
+    /**
+     * Return the found classes.
+     */
+    public Iterator getClasses () {
+        return classNameList.iterator ();
+    }
+}
Index: /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/TestCaseLoader.java
===================================================================
--- /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/TestCaseLoader.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/java/org/greenstone/testing/TestCaseLoader.java	(revision 21112)
@@ -0,0 +1,83 @@
+package org.greenstone.testing;
+import java.util.*;
+import java.lang.reflect.*;
+import junit.framework.*;
+/**
+ * Responsible for loading classes representing valid test cases.
+ */
+public class TestCaseLoader {
+    final private Vector classList = new Vector ();
+    final private String requiredType;
+    
+    /** 
+     * Adds <code>testCaseClass</code> to the list of classdes
+     * if the class is a test case we wish to load. Calls
+     * <code>shouldLoadTestCase ()</code> to determine that.
+     */
+    private void addClassIfTestCase (final Class testCaseClass) {
+        if (shouldAddTestCase (testCaseClass)) {
+            classList.add (testCaseClass);
+        }
+    }
+
+    /**
+     * Determine if we should load this test case. 
+     */
+    private boolean shouldAddTestCase (final Class testCaseClass) {
+        boolean isOfTheCorrectType = false;
+        if (TestCase.class.isAssignableFrom(testCaseClass)) {
+            try {
+                Field testAllIgnoreThisField = testCaseClass.getDeclaredField("TEST_ALL_TEST_TYPE");
+                final int EXPECTED_MODIFIERS = Modifier.STATIC | Modifier.PUBLIC | Modifier.FINAL;
+                if (((testAllIgnoreThisField.getModifiers() & EXPECTED_MODIFIERS) != EXPECTED_MODIFIERS) ||
+                    (testAllIgnoreThisField.getType() != String.class)) {
+                    throw new IllegalArgumentException ("TEST_ALL_TEST_TYPE should be static public final String");
+                }
+                String testType = (String)testAllIgnoreThisField.get(testCaseClass);
+                isOfTheCorrectType = requiredType.equals (testType);
+            } catch (NoSuchFieldException e) {
+            } catch (IllegalAccessException e) {
+                throw new IllegalArgumentException ("The field " + testCaseClass.getName () + ".TEST_ALL_TEST_TYPE is not accessible.");
+            }
+        }
+        return isOfTheCorrectType;
+    }
+    
+       
+    
+    /**
+     * Load the classes that represent test cases we are interested. 
+     * @param classNamesIterator An iterator over a collection of fully qualified class names
+     */
+    public void loadTestCases (final Iterator classNamesIterator) {
+        while (classNamesIterator.hasNext ()) {
+            String className = (String)classNamesIterator.next ();
+	    System.out.println("trying class "+className);
+            try {
+                Class candidateClass = Class.forName (className);
+                addClassIfTestCase (candidateClass);
+            } catch (ClassNotFoundException e) {
+                System.err.println ("Cannot load class: " + className + " " + e.getMessage ());
+            } catch (NoClassDefFoundError e) {
+                System.err.println ("Cannot load class that " + className + " is dependant on");
+            }
+        }
+    }
+        
+    /**
+     * Construct this instance. Load all the test cases possible that derive
+     * from <code>baseClass</code> and cannot be ignored. 
+     * @param classNamesIterator An iterator over a collection of fully qualified class names
+     */
+    public TestCaseLoader(final String requiredType) {
+        if (requiredType == null) throw new IllegalArgumentException ("requiredType is null");
+        this.requiredType = requiredType;
+    }
+    
+    /**
+     * Obtain an iterator over the collection of test case classes loaded by <code>loadTestCases</code>
+     */
+    public Iterator getClasses () {
+        return classList.iterator ();
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/CHANGES
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/CHANGES	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/CHANGES	(revision 21112)
@@ -0,0 +1,26 @@
+Release 0006 1998-03-06
+ - Added FAQ.html
+
+Release 0005 1997-10-30
+ - Deserializing objects from databases is implemented properly
+ - Added the Closeable interface
+ - Fixed link to home page in documentation
+
+Release 0004 1997-10-24
+ - Fixed bug in autoconfig script
+
+Release 0003 1997-10-24
+ - Release-management and CVS restructurings
+
+Release 0002 1997-10-23
+ - Much more developer documentation in javadoc format
+ - Support for Serialization and QuotedVector packing
+ - Packing objects now have consistent names
+ - New distribution page, separate from mbp's home page
+ - Moved all the code into packages which use the standard Java naming
+   conventions: au.com.pharos.gdbm
+
+Release 0001 1997-10-03
+ - Functional JNI-based interface
+ - Scanty documentation
+ - GPL release disclaimer from jdg
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/FAQ.html
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/FAQ.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/FAQ.html	(revision 21112)
@@ -0,0 +1,173 @@
+<head><title>FAQ for JavaGDBM</title>
+</head>
+
+<h1>FAQ for JavaGDBM</h1>
+<p><pre>$Id$</pre>
+<p><a href="http://www.pharos.com.au/gdbm/"><b>Home page</b></a> |
+<a href="mailto:mbp@pharos.com.au"><b>mbp@pharos.com.au</b></a>
+
+<p><b>This is not an official publication of Pharos.  There is no
+warranty of any kind on this software or document.</b>
+
+<h3>Is this a JDBC driver?</h3>
+
+<p>(Alain Magloire)
+
+<p>No. JDBC seems mainly intended for interfaces to relational
+databases, and GDBM is not a relational database. 
+
+<p>However, JDBC does permit use of a subset to interface to
+non-relational databases.  Implementing this might be interesting: let
+me know if it's useful to you.
+
+<h3>Does JavaGDBM understand SQL?</h3>
+
+<p>(Alain Magloire)
+
+<p>No, JavaGDBM is intended to store on-disk hash tables.  They solve
+a different class of problem to that appropriate for SQL and
+relational databases.
+
+<h3>What type of database should I use?</h3>
+
+<p>Broadly, options for data storage in Java include
+
+<ul>
+<li>A hash file through JavaGDBM.
+<li>A relational database through JDBC
+<li>An object-oriented database
+<li>Serialization to a file
+</ul>
+
+<p>Make sure you choose the right tools: there is no single `best' answer.
+
+<h3>Are you maintaining GDBM?</h3>
+
+<p>No.  I understand that GDBM is still maintained by the FSF.
+
+<h3>Does it work on NT?</h3>
+
+<p>It should, but I haven't tried it.  You'll need a C compiler to
+build the native library.  Let me know if you try.
+
+<h3>Why does the test harness report exceptions</h3>
+
+<p>The test harness may report messages such as
+
+<pre>
+caught: au.com.pharos.gdbm.GdbmException: File open error: No such file or
+directory? (gdbmjava.c:101)
+</pre>
+
+<p>These are expected exceptions: the test harness is making sure that
+exceptions are thrown when they should be.
+
+<h3>I was trying to compile the Java GDBM distribution and it says gdbm.h not found</h3>
+
+<p>(Sandeep Kumar)
+
+<ol>
+<li>Make sure you have the GDBM library installed on your system.  
+If the library is not installed, you should be able to find it at 
+<a href="http://www.gnu.org/">www.gnu.org</a>
+
+<li>Make sure your compiler is configured to find the libraries and
+include files.
+
+</ol>
+
+<h3><tt>GdbmFile.h</tt> is missing</h3>
+
+<p>(Paul Matthews)
+
+<p><tt>GdbmFile.h</tt> is build by <tt>javah</tt> when the library is
+built.
+
+<h3>What is the best source to learn Java?</h3>
+
+<p>(Celil Germeyan)
+
+<p>Personally, I recommend the excellent Java series of books from
+<a href="http://www.ora.com/info/java/bibliography.html/">O'Reilly and
+Associates</a> and the
+<a href="http://java.sun.com/docs/books/index.html">Javasoft/Addison
+Wesley</a> books.
+
+<h3>I get a `not ok 40' message running the test</h3>
+
+<p>(Steve Waldman)
+
+<p>If you find a bug, either have a look at the code yourself, or
+send e-mail to 
+<a href="mailto:mbp@pharos.com.au"><tt>mbp@pharos.com.au</tt></a>.  
+There's no warranty on JavaGDBM, but I will try to respond to your
+mail and problem.
+
+<h3>I get linker errors on Solaris</h3>
+
+<p>(Alex Soto)
+
+<pre>
+Hi,
+
+    I'm hoping to use your JavaGDBM api for my application, but I'm
+having some problems compiling the libgdbmjava.so.  Here is a partial
+listing of the errors:
+
+gcc -o libgdbmjava.so -shared gdbmjava.o -lgdbm
+Text relocation remains                       referenced
+    against symbol                  offset      in file
+free                                0x258
+/usr/local/lib/libgdbm.a(gdbmreorg.o)
+free                                0xd0
+/usr/local/lib/libgdbm.a(gdbmclose.o)
+free                                0xb8
+/usr/local/lib/libgdbm.a(gdbmclose.o)
+free                                0x218
+[snipped]
+
+I have used the libgdbm.a lib before, and don't believe there is a
+problem with it.  I'm compiling to Solaris 2.6, using JDK 1.1.4.
+</pre>
+
+<p>The solution turned out to be that the gdbm native library
+must be compiled with the <tt>-fpic</tt> flag to cause the compiler to
+generate position-independant code.  
+
+<h3>What happens if a reader tries to open the database while a writer is doing something?</h3>
+
+<p>(Rich Neill)
+
+<p>Access will be denied until the writer releases its lock.  
+
+<p>JavaGDBM uses a shared-read/exclusive-write protocol at the OS
+level.  However, multiple Java threads can access a file concurrently.
+
+<h3>What platforms does JavaGDBM run on?</H3>
+
+<p>It should run on any platform that supports JDK1.1 or higher, JNI
+(Java Native Interface), and GDBM.  JavaGDBM is known to run on 
+
+<ul>
+<li>Linux/i386
+<li>Solaris/SPARC and Solaris/i86
+<li>AIX 4
+</ul>
+
+<p>If you have any information on other platforms, please let me know.
+
+<h3>You don't anticipate going Pure Java soon, do you?</h3>
+
+<p>(Brian Thorstad)
+
+<p>It's on the `long-term' TODO list.  I'd like to do it, but don't have
+any date in mind.
+
+<h3>Acknowledgements</h3>
+
+<p>Everybody who's sent in questions or comments.
+
+<p>Redhat and Javasoft, and the authors of GDBM, Linux, gcc, autoconf,
+and other software used in building JavaGDBM.
+
+</html>
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/LICENSE
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/LICENSE	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/LICENSE	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/Makefile.in	(revision 21112)
@@ -0,0 +1,19 @@
+# Main Makefile for Java GDBM
+# Katherine Don
+
+SUBDIRS = java jni
+MDEFINES = 
+
+all install clean:
+	for subdir in $(SUBDIRS); do \
+	echo making $@ in $$subdir; \
+	(cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+distclean: 
+	rm -f Makefile config.cache config.log config.status ;\
+	for subdir in $(SUBDIRS); do \
+	echo making $@ in $$subdir; \
+	(cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done	
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/README.txt
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/README.txt	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/README.txt	(revision 21112)
@@ -0,0 +1,31 @@
+A Java Interface to GDBM
+
+*****
+This is a most of the Java gdbm download. I have left out some unnecessary bits for the greenstone version. The original home page for this is no longer available, but http://aurora.regenstrief.org/~schadow/dbm-java/ has links to the original tar file.
+I have also updated the Makefiles so that it works on Windows and MacOSX.
+[kjdon, 2005]
+*****
+
+Martin Pool <m.pool@pharos.com.au>  1998-03-12
+
+ This is a Java interface to the GDBM database library written by Philip A Nelson. Like GDBM itself, this interface is free software, and comes with no warranty of any kind.
+
+GDBM is a library of database functions for implementing a hash on disk. JavaGDBM is an API for the Java programmer, not an applet or database package for end users.
+
+In Java terms, a GdbmFile is very similar to a java.util.Dictionary object, except that objects are reliably written to and retrieved from disk. An adaptor class, GdbmDictionary, presents that interface to maintain compatibility with existing code.
+
+GDBM enforces file and object locking to allow databases to be safely shared between multiple threads or processes. A database may be open read-write by one process, or read-only by any number of processes. Within each process, any number of threads may share the database. Within the limits of the operating system and hardware, GDBM will ensure that changes are atomically committed to disk, making the database safe even if the program terminates abnormally.
+
+JavaGDBM stores key/data pairs in a GdbmFile object, which represents a single disk file. Each key must be unique, and is paired with a single data item. The keys are not ordered: if you want to retrieve the objects in sorted order you can build an external index. The basic operations on
+
+As for a Dictionary, the basic operations that may be performed on a GdbmFile are to fetch, test for existence, store and delete records.
+
+Since Java objects are dynamically typed and GDBM data are typeless, JavaGDBM uses Strategy objects to allow objects to be encoded and decoded in various ways. Classes derived from the abstract Packing class may pack ojbects for storage as byte arrays, Unicode or ASCII strings, as serialized Java objects, or in a user-defined fashion. The packing strategy is defined at runtime for both the key and value of the records.
+
+Some example code is available in the test harness GdbmTest.java.
+
+The current implementation of JavaGDBM uses native glue code to link to the GDBM library via JNI, the Java Native Interface. This means that you must have a C compiler, the GDBM library, and related tools to compile the native portions JavaGDBM also requires a JDK1.1-compatible Java compiler, libraries, and runtime.
+
+If the GDBM libraries didn't come with your system, then you can download the GDBM source and install it.
+
+GDBM is known to work on Linux, NetBSD, Solaris, and AIX. 
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/TODO
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/TODO	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/TODO	(revision 21112)
@@ -0,0 +1,21 @@
+TODO for JavaGDBM
+$Id$
+
+Short-term
+
+ - Fix reported bugs
+
+Medium-term
+
+ - Modify the Makefile and configuration code to make sure JavaGDBM
+   builds on more different systems.
+
+ - Add support for non-default block sizes and other options.
+
+ - Offer more developer documentation.
+
+Long-term
+
+ - Rewrite the native code in Java to make the system 100% Pure.
+ - Offer precompiled binaries(?)
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/configure
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/configure	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/configure	(revision 21112)
@@ -0,0 +1,5326 @@
+#! /bin/sh
+# From configure.in Revision: 1.2 .
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="jni/gdbmjava.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S CPP EGREP JAVA_HOME javagdbm_cv_path_javah javagdbm_cv_path_java GDBM_LIBPATH GDBM_INCLUDE PKG JNIINC JNISUFFIX JNIFLAGS LIBOBJS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-gdbm=dir         Use local gdbm library and headers
+
+Some influential environment variables:
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# Check whether --with-gdbm or --without-gdbm was given.
+if test "${with_gdbm+set}" = set; then
+  withval="$with_gdbm"
+  gdbmdir=$withval
+fi;
+
+# TODO: allow the user to specify whether they want a debugging
+# library or not?
+
+# check for compilers and other tools
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C compiler default output file name" >&5
+echo $ECHO_N "checking for C compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C compiler works" >&5
+echo $ECHO_N "checking whether the C compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+echo "$as_me:$LINENO: checking for void *" >&5
+echo $ECHO_N "checking for void *... $ECHO_C" >&6
+if test "${ac_cv_type_void_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((void * *) 0)
+  return 0;
+if (sizeof (void *))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_void_p=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_void_p=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_void_p" >&5
+echo "${ECHO_T}$ac_cv_type_void_p" >&6
+
+echo "$as_me:$LINENO: checking size of void *" >&5
+echo $ECHO_N "checking size of void *... $ECHO_C" >&6
+if test "${ac_cv_sizeof_void_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_void_p" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (void *))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (void *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (void *))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (void *))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (void *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_void_p=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (void *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (void *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (void *)); }
+unsigned long ulongval () { return (long) (sizeof (void *)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (void *))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (void *))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (void *))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_void_p=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (void *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (void *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_void_p=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_void_p" >&5
+echo "${ECHO_T}$ac_cv_sizeof_void_p" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_VOID_P $ac_cv_sizeof_void_p
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking for int" >&5
+echo $ECHO_N "checking for int... $ECHO_C" >&6
+if test "${ac_cv_type_int+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((int *) 0)
+  return 0;
+if (sizeof (int))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_int=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_int=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_int" >&5
+echo "${ECHO_T}$ac_cv_type_int" >&6
+
+echo "$as_me:$LINENO: checking size of int" >&5
+echo $ECHO_N "checking size of int... $ECHO_C" >&6
+if test "${ac_cv_sizeof_int+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_int" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (int))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (int))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (int))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (int))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (int))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_int=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (int), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (int), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (int)); }
+unsigned long ulongval () { return (long) (sizeof (int)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (int))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (int))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (int))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_int=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (int), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (int), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_int=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_int" >&5
+echo "${ECHO_T}$ac_cv_sizeof_int" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_INT $ac_cv_sizeof_int
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking for long" >&5
+echo $ECHO_N "checking for long... $ECHO_C" >&6
+if test "${ac_cv_type_long+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((long *) 0)
+  return 0;
+if (sizeof (long))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_long=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_long=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_long" >&5
+echo "${ECHO_T}$ac_cv_type_long" >&6
+
+echo "$as_me:$LINENO: checking size of long" >&5
+echo $ECHO_N "checking size of long... $ECHO_C" >&6
+if test "${ac_cv_sizeof_long+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_long" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (long))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (long))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (long))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (long))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (long))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_long=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (long), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (long), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (long)); }
+unsigned long ulongval () { return (long) (sizeof (long)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (long))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (long))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (long))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_long=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (long), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (long), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_long=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_long" >&5
+echo "${ECHO_T}$ac_cv_sizeof_long" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_LONG $ac_cv_sizeof_long
+_ACEOF
+
+
+
+######## check for the JDK and tools within the JDK
+######## look in the JAVA_HOME environment variable at configure-time
+echo "$as_me:$LINENO: checking for the JDK" >&5
+echo $ECHO_N "checking for the JDK... $ECHO_C" >&6
+if test "z$JAVA_HOME" != z &&
+   test -d $JAVA_HOME &&
+   test -d $JAVA_HOME/include
+then
+
+	echo "$as_me:$LINENO: result: found in $JAVA_HOME" >&5
+echo "${ECHO_T}found in $JAVA_HOME" >&6
+else
+	{ { echo "$as_me:$LINENO: error: not found in \$JAVA_HOME" >&5
+echo "$as_me: error: not found in \$JAVA_HOME" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+# TODO: Check for specific header files in the JDK, or
+# functions/definitions within those headers?
+
+# TODO: guess which include directory to use under the JDK
+
+# Extract the first word of "javah", so it can be a program name with args.
+set dummy javah; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_path_javagdbm_cv_path_javah+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  case $javagdbm_cv_path_javah in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_javagdbm_cv_path_javah="$javagdbm_cv_path_javah" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_javagdbm_cv_path_javah="$as_dir/$ac_word$ac_exec_ext"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_path_javagdbm_cv_path_javah" && ac_cv_path_javagdbm_cv_path_javah="missing_javah"
+  ;;
+esac
+fi
+javagdbm_cv_path_javah=$ac_cv_path_javagdbm_cv_path_javah
+
+if test -n "$javagdbm_cv_path_javah"; then
+  echo "$as_me:$LINENO: result: $javagdbm_cv_path_javah" >&5
+echo "${ECHO_T}$javagdbm_cv_path_javah" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+if test "$javagdbm_cv_path_javah" = missing_javah; then
+	{ { echo "$as_me:$LINENO: error: javah not found." >&5
+echo "$as_me: error: javah not found." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+# Extract the first word of "java", so it can be a program name with args.
+set dummy java; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_path_javagdbm_cv_path_java+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  case $javagdbm_cv_path_java in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_javagdbm_cv_path_java="$javagdbm_cv_path_java" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_javagdbm_cv_path_java="$as_dir/$ac_word$ac_exec_ext"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_path_javagdbm_cv_path_java" && ac_cv_path_javagdbm_cv_path_java="missing"
+  ;;
+esac
+fi
+javagdbm_cv_path_java=$ac_cv_path_javagdbm_cv_path_java
+
+if test -n "$javagdbm_cv_path_java"; then
+  echo "$as_me:$LINENO: result: $javagdbm_cv_path_java" >&5
+echo "${ECHO_T}$javagdbm_cv_path_java" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+if test "$javagdbm_cv_path_java" = missing; then
+	{ { echo "$as_me:$LINENO: error: java not found." >&5
+echo "$as_me: error: java not found." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+# check for gdbm library
+#AC_CHECK_LIB(gdbm, gdbm_open, [javagdbm_cv_lib_gdbm="-lgdbm"],
+#	[javagdbm_cv_lib_gdbm=no])
+#if test "$javagdbm_cv_lib_gdbm" = "no"; then
+#	AC_MSG_ERROR([no native gdbm library found.])
+#fi
+#LIBS="$javagdbm_cv_lib_gdbm"
+
+# gdbm stuff copied from gsdl [kjdon]
+
+
+echo "$as_me:$LINENO: checking that the GDBM library is available" >&5
+echo $ECHO_N "checking that the GDBM library is available... $ECHO_C" >&6
+if test ! -z "$gdbmdir" ; then
+  # look in given directory for header file
+  if test -r $gdbmdir/include/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir/include" ;
+  elif test -r $gdbmdir/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir" ;
+  fi
+
+  # look in given directory for library
+  if test -r $gdbmdir/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir" ;
+  elif test -r $gdbmdir/lib/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/lib" ;
+  elif test -r $gdbmdir/.libs/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/.libs" ;
+  fi
+
+  if test ! -z "$GDBM_LIBPATH" -a ! -z "$GDBM_INCLUDE" ; then
+    echo "$as_me:$LINENO: result: $gdbmdir" >&5
+echo "${ECHO_T}$gdbmdir" >&6 ;
+  else
+    echo "$as_me:$LINENO: result: can't find gdbm.h and libgdbm.a in $gdbmdir" >&5
+echo "${ECHO_T}can't find gdbm.h and libgdbm.a in $gdbmdir" >&6;
+    echo "$as_me:$LINENO: checking looking for GDBM library in default locations" >&5
+echo $ECHO_N "checking looking for GDBM library in default locations... $ECHO_C" >&6
+  fi
+fi
+
+# look in default path and in /usr/local
+if test -z "$GDBM_INCLUDE" ; then
+  # try in system directory
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <gdbm.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+success=no
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+  if test $success = "no"; then
+# last chance - look in /usr/local (eg FreeBSD)
+    GDBM_LIBPATH=-L/usr/local/lib
+    GDBM_INCLUDE=-I/usr/local/include
+    ac_save_CXXFLAGS="$CXXFLAGS"
+    ac_save_LIBS="$LIBS"
+      CXXFLAGS="$CXXFLAGS -I/usr/local/include"
+      LIBS="$LIBS -L/usr/local/lib -lgdbm"
+      cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <gdbm.h>
+#include <stdio.h>
+
+int
+main ()
+{
+printf("%s",gdbm_version);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  success=/usr/local
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+success=no
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+      CXXFLAGS=$ac_save_CXXFLAGS
+      LIBS=$ac_save_LIBS
+  fi
+
+  echo "$as_me:$LINENO: result: $success" >&5
+echo "${ECHO_T}$success" >&6
+  if test $success = "no"; then
+  { { echo "$as_me:$LINENO: error: GDBM Library not available - cannot install" >&5
+echo "$as_me: error: GDBM Library not available - cannot install" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+fi
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_LIBGDBM  1
+_ACEOF
+
+
+
+
+
+########## set compiler options
+# TODO: is there a more appropriate way to guess which switches to use
+# with which systems/compilers?
+CFLAGS="-Wall -O2 -fpic"
+PKG="au.com.pharos.gdbm"
+
+
+
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+
+
+
+
+# write the output
+                              ac_config_files="$ac_config_files Makefile jni/Makefile java/Makefile"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+# Transform confdefs.h into DEFS.
+# Protect against shell expansion while executing Makefile rules.
+# Protect against Makefile macro expansion.
+#
+# If the first sed substitution is executed (which looks for macros that
+# take arguments), then we branch to the quote section.  Otherwise,
+# look for a macro that doesn't take arguments.
+cat >confdef2opt.sed <<\_ACEOF
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*([^)]*)\)[	 ]*\(.*\),-D\1=\2,g
+t quote
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\),-D\1=\2,g
+t quote
+d
+: quote
+s,[	 `~#$^&*(){}\\|;'"<>?],\\&,g
+s,\[,\\&,g
+s,\],\\&,g
+s,\$,$$,g
+p
+_ACEOF
+# We use echo to avoid assuming a particular line-breaking character.
+# The extra dot is to prevent the shell from consuming trailing
+# line-breaks from the sub-command output.  A line-break within
+# single-quotes doesn't work because, if this script is created in a
+# platform that uses two characters for line-breaks (e.g., DOS), tr
+# would break.
+ac_LF_and_DOT=`echo; echo .`
+DEFS=`sed -n -f confdef2opt.sed confdefs.h | tr "$ac_LF_and_DOT" ' .'`
+rm -f confdef2opt.sed
+
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+
+Configuration files:
+$config_files
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "jni/Makefile" ) CONFIG_FILES="$CONFIG_FILES jni/Makefile" ;;
+  "java/Makefile" ) CONFIG_FILES="$CONFIG_FILES java/Makefile" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@JAVA_HOME@,$JAVA_HOME,;t t
+s,@javagdbm_cv_path_javah@,$javagdbm_cv_path_javah,;t t
+s,@javagdbm_cv_path_java@,$javagdbm_cv_path_java,;t t
+s,@GDBM_LIBPATH@,$GDBM_LIBPATH,;t t
+s,@GDBM_INCLUDE@,$GDBM_INCLUDE,;t t
+s,@PKG@,$PKG,;t t
+s,@JNIINC@,$JNIINC,;t t
+s,@JNISUFFIX@,$JNISUFFIX,;t t
+s,@JNIFLAGS@,$JNIFLAGS,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/configure.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/configure.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/configure.in	(revision 21112)
@@ -0,0 +1,183 @@
+dnl Process this file with autoconf to produce a configure script.
+
+dnl autoconf configuration script for the JNI part of the 
+dnl au.com.pharos.gdbm Java package
+dnl $Id$
+
+AC_REVISION($Revision$)
+
+# Copyright (C) 1997 by Pharos IP Pty Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+AC_INIT(jni/gdbmjava.c)
+
+dnl
+dnl look for libgdbm if not in standard location
+dnl
+AC_ARG_WITH(gdbm, [  --with-gdbm=dir         Use local gdbm library and headers], gdbmdir=$withval)
+
+# TODO: allow the user to specify whether they want a debugging
+# library or not?
+
+# check for compilers and other tools
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+
+AC_CHECK_SIZEOF(void *)
+AC_CHECK_SIZEOF(int)
+AC_CHECK_SIZEOF(long)
+
+######## check for the JDK and tools within the JDK
+######## look in the JAVA_HOME environment variable at configure-time
+AC_MSG_CHECKING([for the JDK])
+if test "z$JAVA_HOME" != z &&
+   test -d $JAVA_HOME && 
+   test -d $JAVA_HOME/include
+then
+	AC_SUBST(JAVA_HOME)
+	AC_MSG_RESULT([found in $JAVA_HOME])
+else
+	AC_MSG_ERROR([not found in \$JAVA_HOME])
+fi
+
+# TODO: Check for specific header files in the JDK, or
+# functions/definitions within those headers?
+
+# TODO: guess which include directory to use under the JDK
+
+AC_PATH_PROG(javagdbm_cv_path_javah, javah, missing_javah)
+if test "$javagdbm_cv_path_javah" = missing_javah; then
+	AC_MSG_ERROR([javah not found.])
+fi
+
+AC_PATH_PROG(javagdbm_cv_path_java, java, missing)
+if test "$javagdbm_cv_path_java" = missing; then
+	AC_MSG_ERROR([java not found.])
+fi
+
+# check for gdbm library
+#AC_CHECK_LIB(gdbm, gdbm_open, [javagdbm_cv_lib_gdbm="-lgdbm"],
+#	[javagdbm_cv_lib_gdbm=no])
+#if test "$javagdbm_cv_lib_gdbm" = "no"; then
+#	AC_MSG_ERROR([no native gdbm library found.])
+#fi
+#LIBS="$javagdbm_cv_lib_gdbm"
+
+# gdbm stuff copied from gsdl [kjdon]
+
+dnl
+dnl Check that the GDBM library is available
+dnl
+dnl check libgdbm manually, in case it's not in the standard location.
+
+AC_MSG_CHECKING(that the GDBM library is available)
+if test ! -z "$gdbmdir" ; then
+  # look in given directory for header file
+  if test -r $gdbmdir/include/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir/include" ;
+  elif test -r $gdbmdir/gdbm.h ; then
+    GDBM_INCLUDE="-I$gdbmdir" ;
+  fi
+
+  # look in given directory for library
+  if test -r $gdbmdir/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir" ;
+  elif test -r $gdbmdir/lib/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/lib" ;
+  elif test -r $gdbmdir/.libs/libgdbm.a ; then
+    GDBM_LIBPATH="-L$gdbmdir/.libs" ;
+  fi
+
+  if test ! -z "$GDBM_LIBPATH" -a ! -z "$GDBM_INCLUDE" ; then
+    AC_MSG_RESULT($gdbmdir) ;
+  else
+    AC_MSG_RESULT(can't find gdbm.h and libgdbm.a in $gdbmdir);
+    AC_MSG_CHECKING(   looking for GDBM library in default locations)
+  fi
+fi
+
+# look in default path and in /usr/local
+if test -z "$GDBM_INCLUDE" ; then
+  # try in system directory 
+  AC_TRY_COMPILE(
+  [#include <gdbm.h>
+],
+  [],
+  success=yes,
+  success=no
+  )
+
+  if test $success = "no"; then
+# last chance - look in /usr/local (eg FreeBSD)
+    GDBM_LIBPATH=-L/usr/local/lib
+    GDBM_INCLUDE=-I/usr/local/include
+    ac_save_CXXFLAGS="$CXXFLAGS"
+    ac_save_LIBS="$LIBS"
+      CXXFLAGS="$CXXFLAGS -I/usr/local/include"
+      LIBS="$LIBS -L/usr/local/lib -lgdbm"
+      AC_TRY_LINK(
+  	[#include <gdbm.h>
+#include <stdio.h>
+	],
+  	[printf("%s",gdbm_version);],
+  	success=/usr/local,
+  	success=no
+      )
+      CXXFLAGS=$ac_save_CXXFLAGS
+      LIBS=$ac_save_LIBS
+  fi
+
+  AC_MSG_RESULT($success)
+  if test $success = "no"; then
+  AC_MSG_ERROR(GDBM Library not available - cannot install)
+  fi
+fi
+dnl I don't think anything actually checks this define, but...
+AC_DEFINE(HAVE_LIBGDBM , 1)
+
+AC_SUBST(GDBM_LIBPATH)
+AC_SUBST(GDBM_INCLUDE)
+
+########## set compiler options
+# TODO: is there a more appropriate way to guess which switches to use
+# with which systems/compilers?
+CFLAGS="-Wall -O2 -fpic"
+PKG="au.com.pharos.gdbm"
+AC_SUBST(CFLAGS)
+AC_SUBST(PKG)
+AC_SUBST(LIBS)
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  AC_MSG_RESULT(no)
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+AC_SUBST(JNIINC)
+AC_SUBST(JNISUFFIX)
+AC_SUBST(JNIFLAGS)
+
+# write the output
+AC_OUTPUT([Makefile jni/Makefile java/Makefile])
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/install-sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/install-sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/Makefile.in	(revision 21112)
@@ -0,0 +1,31 @@
+# Makefile for Java part of JavaGDBM
+# Katherine Don, 2005
+
+JAVAC = $(JAVA_HOME)/bin/javac
+JAVAH = $(JAVA_HOME)/bin/javah
+JAR = $(JAVA_HOME)/bin/jar
+
+JAVAGDBMHOME=../
+JAVACLASSDIR=./classes
+JAVASRCDIR=.
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) au/com/pharos/*/*.java
+	$(JAVAH) -o $(JAVAGDBMHOME)/jni/GdbmFile.h -classpath $(JAVACLASSPATH) au.com.pharos.gdbm.GdbmFile
+	$(JAR) cf $(JAVAGDBMHOME)/javagdbm.jar -C $(JAVACLASSDIR) au
+
+clean:
+	rm -rf $(JAVACLASSDIR) $(JAVAGDBMHOME)/javagdbm.jar $(JAVAGDBMHOME)/jni/GdbmFile.h 
+
+install:
+
+distclean: clean
+	rm -f Makefile
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFDbInterface.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFDbInterface.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFDbInterface.java	(revision 21112)
@@ -0,0 +1,37 @@
+/*
+ * (C) Pharos IP Pty Ltd 1997
+ */
+
+package au.com.pharos.dbinterface.vf;
+
+import java.util.Date;
+
+import au.com.pharos.gdbm.GdbmFile;
+
+import au.com.pharos.packing;
+
+import au.com.pharos.dbinterface.vf;
+
+public class VFDbInterface {
+
+  /* The Order database */
+  private String orderDB = "/fn/data/Orders.gdbm";
+
+  /* The Stock database */
+  private String stockDB = "/fn/data/Stock.gdbm";
+
+  /* Initialisation.  Nothing needs to happen yet. */
+  public VFDbInterface()
+  {
+    ;
+  }
+
+  /* Submit an order to the database */
+  public void submitOrder(VFOrder order)
+  {
+    GdbmFile db = new GdbmFile(orderDB);
+    db.setKeyPacking(new StringPacking());
+    db.setValuePacking(new QuotedVectorPacking());
+
+    Vector details = order.toVectorString();
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFItem.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFItem.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFItem.java	(revision 21112)
@@ -0,0 +1,106 @@
+/*
+ * (C) Pharos IP Pty Ltd 1997
+ */
+package au.com.pharos.dbinterface.vf;
+
+import java.util.Date;
+
+public class VFItem {
+
+  /* The title of the item */
+  public String title;
+
+  /* The name fo the item's distributor */
+  public String distributor;
+
+  /* The category of the item */
+  // TODO: Create a more appropriate type for this?
+  public String category;
+
+  /* The rating of the item */
+  //TODO: Again, a more appropriate type?
+  public String rating;
+
+  /* The list price */
+  public float listPrice;
+
+  /* The supplier's price */
+  public float ourPrice;
+
+  /* The release date for the item */
+  public Date releaseDate;
+
+  /* The date upon which the video is accessible to PayTV */
+  public Date payTVWindow;
+
+  /* The date upon which the video is accessible to free-to-air TV */
+  public Date tvWindow;
+
+  /* The sell-through beginning date */
+  public Date sellThruWindow;
+
+  /* Whether or not the item has an internal magazine advertisement */
+  public boolean magAdvert;
+
+  /* The recommended number of items for Budget 1 stores */
+  public short budget1;
+
+  /* For budget 2 stores */
+  public short budget2;
+
+  /* Budget 3 stores */
+  public short budget3;
+
+  /* Budget 4 stores */
+  public short budget4;
+
+  /* Will the item be advertised by the VideoIndustry Professional Association */
+  boolean vipa;
+
+  /* Initialise all the attributes */
+  public VFItem(String newTitle,
+		String newDistrib,
+		String newCat,
+		String newRating,
+		float newList,
+		float newPrice,
+		Date newRelease,
+		Date newPaytv,
+		Date newTv,
+		boolean isMagAdvert,
+		short newBudget1,
+		short newBudget2,
+		short newBudget3,
+		short newBudget4
+		boolean newvipa)
+  {
+    title = newTitle;
+    distributor = newDistrib;
+    category = newCat;
+    rating = newRating;
+    listPrice = newList;
+    ourPrice = newPrice;
+    releaseDate = newRelease;
+    payTVWindow = newPaytv;
+    tvWindow = newTv;
+    magAdvert = isMagAdvert;
+    budget1 = newBudget1;
+    budget2 = newBudget2;
+    budget3 = newBudget3;
+    budget4 = newBudget4;
+    vipa = newvipa;
+  }
+
+  public boolean equals (Object o)
+  {
+    if (!(o instanceof VFItem)) {
+      return false;
+    }
+    VFItem other = (VFItem) o;
+    if (title.equals(other.title) && distributor.equals(other.distributor) && category.equals(other.category)) {
+      return true;
+    }
+    return false;
+  }
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFOrder.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFOrder.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/dbinterface/vf/VFOrder.java	(revision 21112)
@@ -0,0 +1,116 @@
+/*
+ * (C) Pharos IP PTy Ltd 1997
+ */
+
+package au.com.pharos.dbinterface.vf;
+
+import java.util.Date;
+import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Enumeration;
+
+import java.text.SimpleDateFormat;
+import java.text.ParsePosition;
+
+import java.text.SimpleDateFormat;
+
+public class VFOrder {
+
+  /* The final date for submission */
+  public Date finalSubmit;
+
+  /* The date the order was created */
+  public Date orderDate;
+
+  /* A hash mapping titles requested to numbers of each title*/
+  public Hashtable orderNumbers;
+  
+  /* The name of the ordering store */
+  public String name;
+
+  public VFOrder(Date newFinal,
+		 String newName,
+		 Hashtable newOrder)
+  {
+    finalSubmit = newFinal;
+    orderNumbers = newOrder;
+    name = newName;
+    orderDate = new Date(System.currentTimeMillis());
+    if (orderDate.after(finalSubmit)) {
+      throw new IllegalArgumentException("Final submission date is earlier than present date.");
+    }
+  }
+
+  public VFOrder(Date newFinal,
+		 Date newDate,
+		 String newName,
+		 Hashtable newOrder)
+  {
+    finalSubmit = newFinal;
+    orderDate = newDate;
+    orderNumbers = newOrder;
+    name = newName;
+    orderDate = new Date(System.currentTimeMillis());
+    if (orderDate.after(finalSubmit)) {
+      throw new IllegalArgumentException("Final submission date is earlier than present date.");
+    }
+  }
+
+  public Vector toVectorString()
+  {
+    Vector result = new Vector();
+    result.addElement(new Long(finalSubmit.getTime()));
+    result.addElement(new Long(orderDate.getTime()));
+    result.addElement(orderNumbers.toString());
+    result.addElement(name);
+    return result;
+  }
+
+  public VFOrder fromVectorString(Vector input)
+       throws java.text.ParseException
+  {
+    Hashtable data = new Hashtable();
+    SimpleDateFormat form = new SimpleDateFormat();
+    for (Enumeration e = input.elements(); e.hasMoreElements(); ) {
+      System.out.println(e.nextElement());
+    }
+    try {
+      finalSubmit = new Date(((Long) input.elementAt(0)).longValue());
+    } catch (NullPointerException e) {
+      System.err.println("Waaaah!");
+      finalSubmit = new Date();
+    }
+    System.err.println("No 2:" + input.elementAt(1).toString());
+    orderDate = new Date(((Long) input.elementAt(1)).longValue());
+    name = (String) input.elementAt(3);
+    String hashString = (String) input.elementAt(2);
+    StringBuffer key = new StringBuffer();;
+    StringBuffer value = new StringBuffer();
+    String type = new String("key");
+    for (int count = 0; count < hashString.length(); count++) {
+      char i = hashString.charAt(count);
+      if (i == '{' || i == ' ') { ; }
+      else if (i == '}') { data.put(key, value); }
+      else if (i == ',') {data.put(key, value); }
+      else if (i == '=') {
+	if (type.equals("key")) {
+	  type = "value";
+	} else {
+	  type = "key";
+	}
+      } else {
+	if (type.equals("key")) {
+	  key.append(i);
+	} else {
+	  value.append(i);
+	}
+      }
+    }
+    return new VFOrder(finalSubmit, orderDate, name, data);
+  }
+	
+      
+
+
+}
+  
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/Closeable.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/Closeable.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/Closeable.java	(revision 21112)
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * Confidential.  All rights reserved.
+ * $Id$
+ */
+
+package au.com.pharos.gdbm;
+
+import java.io.IOException;
+
+/** A Closeable object is one having a connection to an underlying
+ * data store that may be flushed or closed.
+ *
+ * <P>This interface is intended to complement the Dictionary and
+ * Collection interfaces by reflecting the fact that many such objects
+ * may be closed.
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ **/
+public interface Closeable {
+    /** Write outstanding changes to the underlying database, and
+     * break the connection, freeing any associated resources or
+     * locks.  This generally does not imply that the data will be
+     * lost, merely that it will be released.  If this is not
+     * meaningful for a particular Closeable object, then it should be
+     * a no-op.
+     *
+     * @exception java.io.IOException passed up from the underlying
+     * database.
+     **/
+    public void close() throws IOException;
+
+    /** Write outstanding changes to the underlying database, but
+     * leave the database open.  If this is not meaningful for a
+     * particular Closeable object, then it should be a no-op.
+     *
+     * @exception java.io.IOException passed up from the underlying
+     * database.
+     **/
+    public void sync() throws IOException;
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmDictionary.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmDictionary.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmDictionary.java	(revision 21112)
@@ -0,0 +1,157 @@
+/*
+ * module: pip/java/gdbm -- A Java interface to the GDBM library
+ * class:  GdbmDictionary -- Wrap up a GDBM File for use as a Dictionary
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.gdbm;
+
+import java.util.Dictionary;
+import java.util.Enumeration;
+
+/** A GDBM database, presented as a Java dictionary.
+ *
+ * <P>This is a separate class which defers most operations to
+ * GdbmFile because GdbmFile can throw IO exceptions, and because the
+ * interface assumptions for Dictionary are not the same as those of
+ * the IO library.  Perhaps they should be the same.
+ *
+ * <P>The Dictionary class will be partially replaced by the much
+ * more elegant 
+ * <A HREF="http://www.javasoft.com/products/jdk/preview/docs/guide/collections/overview.html">Collections API</A>
+ * in JDK1.2, at which time a GDBM Collections interface will probably
+ * be available.
+ *
+ * @see java.util.Dictionary
+ * @see GdbmFile
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ **/
+// TODO: I wonder if this should be an inner class of GdbmFile?  It
+// would bloat that source file a bit, but would (arguably) more
+// accurately capture the intended design. -- mbp
+public class GdbmDictionary extends Dictionary implements Closeable
+{
+    // XXX: This should probably become a private field, but on the other
+    // hand it's probably useful to be able to apply GDBM-specific methods to
+    // the underlying DB.  -- mbp
+    GdbmFile db;
+
+    public GdbmDictionary(GdbmFile db) {
+	this.db = db;
+    }
+
+    private Exception caughtException;
+    
+    public Exception checkError() {
+	return caughtException;
+    }	
+
+    public int size() {
+	try {
+	    return db.size();
+	} catch (Exception e) {
+	    caughtException = e;
+	    return 0;
+	}
+    }
+
+    public boolean isEmpty() {
+	try {
+	    return db.isEmpty();
+	} catch (Exception e) {
+	    caughtException = e;
+	    return false;
+	}
+    }
+
+    public Enumeration keys() {
+	try {
+	    return db.keys();
+	} catch (Exception e) {
+	    caughtException = e;
+	    return null;
+	}
+    }
+
+    public Enumeration elements() {
+	// TODO: write an iterator class to do this
+	throw new NoSuchMethodError();
+    }
+
+    public Object get(Object key) {
+	try {
+	    return db.fetch(key);
+	} catch (Exception e) {
+	    caughtException = e;
+	    return null;
+	}
+    }
+
+    public Object put(Object key, Object value)  {
+	try {
+	    Object oldValue = db.fetch(key);
+	    db.store(key, value);
+	    return oldValue;
+	} catch (Exception e) {
+	    caughtException = e;
+	    return null;
+	}
+    }
+
+    public Object remove(Object key) {
+	try {
+	    Object oldValue = db.fetch(key);
+	    db.delete(key);
+	    return oldValue;
+	} catch (Exception e) {
+	    caughtException = e;
+	    return null;
+	}
+    }
+
+    /** Write outstanding changes to the underlying database, and
+     * close the database.
+     *
+     * @see au.com.pharos.gdbm.Closeable
+     **/
+    public void close() {
+	try {
+	    if (db != null && db.isOpen())
+		db.close();
+	    db = null;
+	} catch (GdbmException e) {
+	    ;
+	}
+    }
+
+    /** Write outstanding changes to the underlying database, but
+     * leave the database open.
+     *
+     * @see au.com.pharos.gdbm.Closeable
+     **/
+    public void sync() {
+	try {
+	    db.sync();
+	} catch (GdbmException e) {
+	    ;
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmException.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmException.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmException.java	(revision 21112)
@@ -0,0 +1,45 @@
+/*
+ * module: pip/java/gdbm -- A Java interface to the GDBM library
+ * class:  GdbmException -- An exception originating in the GDBM library
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.gdbm;
+
+import java.io.IOException;
+
+/** An exception originating in the GDBM library.
+ *
+ * @author Martin Pool
+ * @version $Revision$ 
+ * @see au.com.pharos.gdbm.GdbmFile
+ */
+
+public class GdbmException extends IOException
+{
+    // TODO: Perhaps provide more information about which particular
+    // exception caused the problem?
+    /** Constructs a GdbmException with the specified reason.
+    **/
+    public GdbmException(String reason)
+    {
+	super(reason);
+    }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmFile.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmFile.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmFile.java	(revision 21112)
@@ -0,0 +1,563 @@
+/*
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.gdbm;
+
+import java.util.Enumeration;
+import java.util.NoSuchElementException;
+
+import au.com.pharos.packing.Packing;
+import au.com.pharos.packing.RawPacking;
+
+/** Java interface to a GDBM database table.
+ *
+ * <P>This database is a simple on-disk hash table.
+ *
+ * <P>Both the hash keys and values
+ * are binary strings of any length.  They are converted to and
+ * from Java objects using customizable packing strategies.
+ *
+ * <P>The implementation of this class consists of two levels: a
+ * Java-level interface, and a set of private native functions
+ * implemented in C.  As much functionality as possible is implemented
+ * in Java: the C functions generally just marshal the information for
+ * presentation to the native GDBM library.
+ *
+ * <P>The native library <CODE>gdbmjava</CODE> must be available
+ * for dynamic loading in a system-dependant manner.
+ *
+ * <p>See the GDBM documentation file, and the
+ * <A HREF="http://www.pharos.com.au/gdbm/">JavaGDBM home page</A>
+ * for more information.
+ *
+ * @see au.com.pharos.packing.Packing
+ * @see au.com.pharos.gdbm.GdbmTest
+ * @see au.com.pharos.gdbm.GdbmDictionary
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ **/
+
+public class GdbmFile implements Closeable {
+    /** The GDBM handle for the database file, or 0 if the database has
+     * been closed.  Java doesn't understand it, but it stores it and
+     * passes it to the C routines. */
+    private transient volatile long dbf;
+
+    // Remember the parameters used to open the database
+    private int openFlags;
+    private String dbFilename;
+
+    // -----------------------------------------------------------------
+    // Constructors
+
+    // Values must match those from gdbm.h !
+    /** Indicates that the caller will just read the database.  Many
+     * readers may share a database. */
+    public final static int READER = 0;
+
+    /** The caller wants read/write access to an existing database and
+     * requires exclusive access. */
+    public final static int WRITER = 1;
+
+    /** The caller wants exclusive read/write access, and the database
+     * should be created if it does not already exist. */
+    public final static int WRCREAT = 2;
+
+    /** The caller wants exclusive read/write access, and the database
+     * should be replaced if it already exists. */
+    public final static int NEWDB = 3;
+
+    /** Flag indicating GDBM should write to the database without disc
+     * synchronization.  This allows faster writes, but may produce an
+     * inconsistent database in the event of abnormal termination of
+     * the writer. */
+    public final static int FAST = 16;
+
+    // TODO: Allow FAST mode to be toggled
+
+    /** Creates a new GdbmFile object representing an disk database.  The
+     * database is opened or created on disk.
+     *
+     * <P>Both key and value strategies default to
+     * RawPacking, meaning that the database will use raw byte arrays
+     * for both key and value.
+     *
+     * <P>TODO: Allow the caller to specify the block size and/or
+     * cache size.
+     *
+     * @param fileName the disk filename in which the data will be stored.
+     *
+     * @param flags any of READER, WRITER, WRCREAT, and NEWDB, optionally
+     * ORed with FAST
+     *
+     * @exception GdbmException if the database cannot be opened or
+     * created.
+     *
+     * @see GdbmFile#READER
+     * @see GdbmFile#WRITER
+     * @see GdbmFile#WRCREAT
+     * @see GdbmFile#NEWDB
+     * @see GdbmFile#FAST
+     */
+    public GdbmFile(String fileName, int flags)
+	 throws GdbmException
+    {
+	openFlags = flags;
+	dbFilename = fileName;
+	keyPacking = new RawPacking();
+	valuePacking = new RawPacking();
+	dbf = gdbm_open(fileName, flags);
+    }
+
+    /** Close the database file if it is still open.
+     *
+     * <P><B>Note</B> that the disk file is locked against access from
+     * other processes while it is open.  To prevent contention, it
+     * may be useful to explicitly close the file rather than waiting
+     * for the garbage-collector.
+     *
+     * @see GdbmFile#isOpen()
+     **/
+    public synchronized void close() throws GdbmException
+    {
+	if (dbf != 0)
+	    gdbm_close(dbf);
+	dbf = 0;		// No longer connected
+    }
+
+    /** Close the database when the GdbmFile is garbage-collected.
+     *
+     * @see GdbmFile#close()
+     */
+    public void finalize() throws GdbmException
+    {
+	close();
+    }
+
+    /* The Gdbm file is unlocked when the process terminates, but
+     * nevertheless it would be good to close it in finalization, in
+     * case the OS process continues after the JVM terminates.
+     *
+     * XXX: Unfortunately, anybody else can turn this off again: it
+     * would be nice if there was a way to avoid this. */
+    static {
+	System.runFinalizersOnExit(true);
+    }
+
+
+
+    // -----------------------------------------------------------------
+    // Packing strategies
+    private Packing keyPacking, valuePacking;
+
+    /** Set the object to be used as the packing strategy for
+     * converting key objects to and from the byte arrays stored in
+     * the database.
+     *
+     * <P>Depending on the class of Java object used as the key of your
+     * database, it may be appropriate to use a packing strategy from
+     * the
+     * <A HREF="Package-au.com.pharos.packing.html"><CODE>au.com.pharos.packing</CODE></A>
+     * package, or to define a new subclass of one of those strategies.
+     *
+     * @see au.com.pharos.packing.Packing
+     * @see GdbmFile#setValuePacking(au.com.pharos.packing.Packing)
+     **/
+    public void setKeyPacking(Packing newPacking) {
+	keyPacking = newPacking;
+    }
+
+    /** Set the object to be used as the packing strategy for
+     * converting value objects to and from the byte arrays stored
+     *  in the database.
+     *
+     * <P>Depending on the class of Java object used as the key of your
+     * database, it may be appropriate to use a packing strategy from
+     * the
+     * <A HREF="Package-au.com.pharos.packing.html"><CODE>au.com.pharos.packing</CODE></A>
+     * package, or to define a new subclass of one of those strategies.
+     *
+     * @see au.com.pharos.packing.Packing
+     * @see GdbmFile#setKeyPacking(au.com.pharos.packing.Packing )
+     **/
+    public void setValuePacking(Packing newPacking) {
+	valuePacking = newPacking;
+    }
+
+    // -----------------------------------------------------------------
+
+    /** Returns a string indicating the version of the underlying GDBM
+     * library.
+     *
+     * @return the version of the native GDBM library.
+     **/
+    public static String getLibraryVersion()
+    {
+	return gdbm_getversion();
+    }
+
+    /** Return a string indicating the version of the JavaGDBM library
+     * wrapper.
+     *
+     * <P>The most current release is available from the
+     * <A HREF="http://www.pharos.com.au/javagdbm/">JavaGDBM home page</A>.
+     *
+     * @return the release number of the JavaGDBM library
+     **/
+    public static String getWrapperVersion()
+    {
+	return gdbm_wrapperVersion();
+    }
+
+    /** Indicate whether the database is writable.
+     *
+     * <P>Databases are opened in either read-write or read-only mode,
+     * and remain in that mode until they are closed.
+     *
+     * @return true if the database may be written; otherwise false.
+     *
+     * @see GdbmFile#GdbmFile(java.lang.String, int)
+     **/
+    public boolean isWritable()
+    {
+	return (openFlags & 0x03) != READER;
+    }
+
+    /** Indicate whether the database is open or not.
+     *
+     * <P>A database is open from the point of creation until close()
+     * is called, if ever, after which it is closed.
+     *
+     * @return false if the database has been closed; otherwise true.
+     *
+     * @see GdbmFile#close()
+     **/
+    public boolean isOpen()
+    {
+	return dbf != 0;
+    }
+
+
+    /** Compact the database file.
+     *
+     * <P>If you have had a lot of deletions and would like to shrink the
+     * space used by the GDBM file, this function will reorganize the
+     * database.  GDBM will not shorten the length of a GDBM file
+     * (deleted file space will be reused) except by using this
+     * reorganization, though it will reuse the vacant space.
+     *
+     * <P>The database must be writable.
+     */
+    public void reorganize()
+	 throws GdbmException
+    {
+	gdbm_reorganize(dbf);
+    }
+    
+
+    /** Flush changes to disk.
+     *
+     * <P>This function is only required when the database is
+     * opened with the FAST flag set.  By default, changes are
+     * flushed to disk after every update.
+     *
+     * <P>The database must be writable.
+     *
+     * @see GdbmFile#FAST
+     **/
+    public void sync()
+	 throws GdbmException
+    {
+	gdbm_sync(dbf);
+    }
+
+
+
+    /** Retrieve the value corresponding to a particular key.
+     *
+     * @param key the key of the record to be retrieved
+     *
+     * @return the value of the record with the specified key; or
+     * null if the key is not present in the database.
+     */
+    public Object fetch(Object key) throws GdbmException
+    {
+	byte[] keyBytes = keyPacking.toBytes(key);
+	return valuePacking.fromBytes(gdbm_fetch(dbf, keyBytes));
+    }
+
+
+    /** Indicate whether the specified key is in the database,
+     * without returning the value.
+     *
+     * @param key the key of the record to be retrieved
+     *
+     * @return true if a record with the specified key is present;
+     * otherwise false.
+     */
+    public boolean exists(Object key) throws GdbmException
+    {
+	byte[] keyBytes = keyPacking.toBytes(key);
+	return gdbm_exists(dbf, keyBytes);
+    }
+
+
+
+    /** Store a value in the database, replacing any existing value
+     * with the same key.
+     *
+     * @param key key under which to store the value
+     *
+     * @param value value to be stored
+     *
+     * @exception GdbmException if the object is a reader; or
+     * if an IO error occurs
+     **/
+    public void store(Object key, Object value)
+	 throws GdbmException
+    {
+	byte[] keyBytes = keyPacking.toBytes(key);
+	byte[] valueBytes = valuePacking.toBytes(value);
+	gdbm_store(dbf, keyBytes, valueBytes, true);
+    }
+
+
+    
+    /** Store a value in the database, unless a record with the same
+     * key already exists.
+     *
+     * @param key key under which to store the value.
+     *
+     * @param value value to be stored.
+     *
+     * @exception GdbmException if a record with the specified key
+     * already exists; or if the object is a reader; or
+     * if an IO error occurs
+     */
+    public void storeNoReplace(Object key, Object value)
+	 throws GdbmException
+    {
+	byte[] keyBytes = keyPacking.toBytes(key);
+	byte[] valueBytes = valuePacking.toBytes(value);
+ 	gdbm_store(dbf, keyBytes, valueBytes, false);
+    }
+
+
+    
+    /** Remove a record from the database.
+     *
+     * @param key key of the record to be removed
+     *
+     * @exception GdbmException if there is no record with the
+     * specified key; or if the object is a reader; or if an IO error
+     * occurs.
+     **/
+    public void delete(Object key)
+	 throws GdbmException
+    {
+	byte[] keyBytes = keyPacking.toBytes(key);
+	gdbm_delete(dbf, keyBytes);
+    }
+
+
+
+
+
+    // -----------------------------------------------------------------
+    // Iterator support
+
+    // TODO: Use a flag/timestamp approach to throw an exception if
+    // somebody modifies the database while they're trying to iterate
+    // over it?  Approach suggested by Doug Lea's Collections API. --
+    // mbp
+
+    /** Return an enumeration which will return all of the keys for the
+     * database of the file in (apparently) random order.
+     *
+     * <P><B>Caution:</B> If the database is modified while
+     * an enumeration is in progress,
+     * then changes in the hash table may cause the enumeration to
+     * miss some records.
+     *
+     * @see java.util.Enumeration
+     **/
+    public Enumeration keys() throws GdbmException
+    {
+	// Return an inner class which will do the iteration in the
+	// context of this GdbmFile object.
+	return new KeyEnumeration();
+    }
+
+    // Synchronization is performed in the GdbmFile's methods
+    class KeyEnumeration implements Enumeration {
+	private byte[] currKey, nextKey;
+
+	KeyEnumeration() throws GdbmException {
+	    currKey = null;
+	    nextKey = getFirstKeyRaw();
+	}
+
+	public boolean hasMoreElements() {
+	    return nextKey != null;
+	}
+
+	public Object nextElement() throws NoSuchElementException {
+	    try {
+		currKey = nextKey;
+		if (currKey == null)
+		    throw new NoSuchElementException();
+		else
+		    nextKey = getNextKeyRaw(currKey);
+		return keyPacking.fromBytes(currKey);
+	    } catch ( GdbmException e ) {
+		// Can't propagate this through java.util.Enumeration, dammit
+		throw new NoSuchElementException();
+	    }
+	}
+    }
+
+    /** Start a visit to all keys in the hashtable, using raw data values.
+     *
+     * @return the first key in the hash table as a byte array; or null if
+     * the database is empty. */
+    byte[] getFirstKeyRaw() throws GdbmException
+    {
+	return gdbm_firstkey(dbf);
+    }
+
+    /** Return the first record in the hashtable.
+     *
+     * <P>Note that the database
+     * is not ordered, so the key returned is simply the first in the
+     * hashtable and effectively randomly selected.
+     *
+     * @return the first key in the hash table; or null if
+     * the database is empty.
+     *
+     * @see GdbmFile#getNextKey(java.lang.Object)
+     * @see GdbmFile#keys()
+     **/
+    Object getFirstKey() throws GdbmException
+    {
+	return keyPacking.fromBytes(getFirstKeyRaw());
+    }
+
+
+    /** Find and read the next key in the hashtable.
+     *
+     * @return the next key; or null if <EM>keyBytes</EM> is the last key
+     * in the hashtable.
+     **/
+    byte[] getNextKeyRaw(byte[] keyBytes) throws GdbmException
+    {
+	return gdbm_nextkey(dbf, keyBytes);
+    }
+
+
+    /** Check whether the database is empty.
+     *
+     * @return true if the database contains no records; otherwise
+     * false.
+     **/
+    public boolean isEmpty() throws GdbmException
+    {
+	return getFirstKeyRaw() == null;
+    }
+
+
+    /** Count the records in the database.
+     *
+     * <P>This is implemented by iterating over the database, and so is
+     * a fairly expensive operation.
+     *
+     * <P>This method locks the database to make sure the count is
+     * accurate.
+     *
+     * @return the number of records in the database
+     * @see GdbmFile#isEmpty()
+     **/
+    public synchronized int size() throws GdbmException
+    {
+	int s = 0;
+	byte[] key = getFirstKeyRaw();
+	while (key != null) {
+	    s++;
+	    key = getNextKeyRaw(key);
+	}
+	return s;
+    }
+
+
+    static {
+	String libraryFile = System.getProperty
+	    ("au.com.pharos.gdbm.libraryFile");
+
+	if (libraryFile != null) {
+	    System.load(libraryFile);
+	} else {
+	    System.loadLibrary("gdbmjava");
+	}
+    }
+
+    private synchronized native long
+    gdbm_open(String fileName, int flags)
+	 throws GdbmException;
+
+    private synchronized native void
+    gdbm_close(long dbf);
+
+    private synchronized native void
+    gdbm_store(long dbf,
+	       byte[] key,
+	       byte[] content,
+	       boolean replace);
+
+    private synchronized native byte[]
+    gdbm_fetch(long dbf,
+	       byte[] key);
+
+    private synchronized native boolean
+    gdbm_exists(long dbf,
+		byte[] key);
+
+    private synchronized native void
+    gdbm_delete(long dbf,
+		byte[] key);
+
+    private synchronized native byte[]
+    gdbm_firstkey(long dbf)
+	 throws GdbmException;
+
+    private synchronized native byte[]
+    gdbm_nextkey(long dbf, byte[] key)
+	 throws GdbmException;
+
+    private synchronized static native String
+    gdbm_getversion();
+
+    private synchronized static native String
+    gdbm_wrapperVersion();
+
+    private synchronized native void
+    gdbm_reorganize(long dbf);
+
+    private synchronized native void
+    gdbm_sync(long dbf);
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmTest.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/gdbm/GdbmTest.java	(revision 21112)
@@ -0,0 +1,296 @@
+/*
+ * module: pip/java/gdbm -- A Java interface to the GDBM library
+ * class:  GdbmTest -- Test suite for this package
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.gdbm;
+
+import java.io.PrintStream;
+
+import java.util.Enumeration;
+
+import au.com.pharos.test.Test;
+import au.com.pharos.packing.Packing;
+import au.com.pharos.packing.StringPacking;
+import au.com.pharos.packing.RawPacking;
+
+/** Test harness for the Gdbm library.
+ *
+ * <P>The source of this class forms a useful demonstration of the
+ * JavaGDBM library and is commended to the intending programmer.
+ *
+ * <P>More test cases would be very welcome.
+ *
+ * @see au.com.pharos.gdbm.GdbmFile
+ * @see au.com.pharos.packing.Packing
+ * @author Martin Pool
+ * @version $Revision$
+ */
+public class GdbmTest {
+    // Don't instantiate
+    private GdbmTest() { ; }
+    
+    private static void testLongData(GdbmFile db)
+	 throws Exception
+    {
+	StringBuffer lgBuffer = new StringBuffer();
+	for (int i = 0; i < 100; i++)
+	    lgBuffer.append("ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890" +
+			    "ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890" +
+			    "ABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890");
+
+	String lgString = lgBuffer.toString();
+	lgBuffer = null;
+	String smallKey = "largeDatum";
+
+	db.store(smallKey, lgString);
+	String retr = (String) db.fetch(smallKey);
+
+	Test.ok(9, retr.hashCode() == lgString.hashCode());
+	Test.ok(10, retr.equals(lgString));
+
+	db.delete(smallKey);
+	Test.ok(11, true);
+
+	db.reorganize();
+	Test.ok(12, true);
+    }
+
+
+    static final int MANY_RECORDS = 200;
+    
+    private static void testBigFile(GdbmFile db)
+	 throws Exception
+    {
+	String key;
+
+	// Test a lot of data
+	byte[] bigData = new byte[10000];
+	for (int i = 0; i < bigData.length; i++)
+	    bigData[i] = (byte)((i & 63) + 32);
+	db.setValuePacking(new RawPacking());
+	Test.ok(150, true);
+
+	// Store a series of records in the file
+	for (int i = 0; i < MANY_RECORDS; i++) {
+	    if ((i % 100) == 0)
+		System.out.print(Integer.toString(i)+ " ");
+	    db.store(Integer.toString(i, 16), bigData);
+	}
+	System.out.println();
+	Test.ok(151, true);
+
+	// Enumerate keys and values out of the file
+	int count = 0;
+	Enumeration keys = db.keys();
+	while ( keys.hasMoreElements() ) {
+	    key = (String) keys.nextElement();
+	    count++;
+	    if ((count % 100) == 0)
+		System.out.print(Integer.toString(count)+ " ");
+	}
+	System.out.println();
+	Test.ok(152, count==MANY_RECORDS);
+
+	// Delete all the records from the file
+	count = 0;
+	while ( (key = (String) db.getFirstKey()) != null ) {
+	    db.delete(key);
+	    count++;
+	    if ((count % 100) == 0)
+		System.out.print(Integer.toString(count)+ " ");
+	}
+	System.out.println();
+	Test.ok(153, count==MANY_RECORDS);
+
+	db.reorganize();
+	Test.ok(154, true);
+    }
+
+    public static void main(String[] argv) {
+	try {
+	    final String dbFile = "/tmp/java.gdbm";
+	    String key, value;
+	    boolean found;
+
+	    // Test cases, inspired by db-hash.t in the Perl5.003_93
+	    // distribution
+	    System.out.print("GDBM Database tests:\n\t" +
+			     GdbmFile.getLibraryVersion() + "\n\t" +
+			     GdbmFile.getWrapperVersion() + "\n");
+
+	    // Create a simple database
+	    GdbmFile db = new GdbmFile(dbFile, GdbmFile.NEWDB);
+	    Test.ok(1, db != null);
+
+	    // Set the packing strategy
+	    db.setKeyPacking(new StringPacking());
+	    db.setValuePacking(new StringPacking());
+
+	    // Store some data in the file
+	    db.store("mountain", "Kosciusko");
+	    Test.ok(2, true);
+
+	    // Store and read back data
+	    db.storeNoReplace("abc", "ABC");
+	    Test.ok(3, ((String) db.fetch("abc")).equals("ABC"));
+
+	    // Check whether keys exist or not
+	    Test.ok(4, db.exists("abc"));
+	    Test.ok(5, !db.exists("jimmy"));
+	    Test.ok(6, !db.exists("abc\0"));
+
+	    // Delete those records
+	    db.delete("abc");
+	    db.delete("mountain");
+
+	    Test.ok(10, !db.exists("abc"));
+	    Test.ok(11, !db.exists("mountain"));
+
+	    // Store a series of records in the file
+	    for (int i = 100; i < 200; i++)
+		db.storeNoReplace(Integer.toString(i, Character.MAX_RADIX),
+				  Integer.toString(i, Character.MIN_RADIX));
+
+	    // Enumerate keys and values out of the file
+	    int count = 0;
+	    Enumeration keys = db.keys();
+	    while ( keys.hasMoreElements() ) {
+		key = (String) keys.nextElement();
+		// System.out.print(key + "=");
+		// value = (String) db.fetch(key);
+		// System.out.println(value);
+		count++;
+	    }
+	    Test.ok(20, count==100);
+
+	    // Delete all the records from the file
+	    count = 0;
+	    while ( (key = (String) db.getFirstKey()) != null ) {
+		db.delete(key);
+		count++;
+	    }
+	    Test.ok(30, count==100);
+
+	    // Try to fetch a non-existent key
+	    boolean caught = false;
+	    try {
+		db.fetch("larry!");
+	    } catch ( GdbmException e ) {
+		// TODO: Check the exception was the one we expected?
+		caught = true;
+	    }
+	    Test.ok(40, caught);
+
+	    // Store some very large data
+	    testLongData(db);
+
+	    // Store strings containing non-ASCII characters
+	    final String euroString = "Some funny characters: «¡ æ ø â ß !»";
+	    db.store("high ASCII", euroString);
+	    Test.ok(50, db.fetch("high ASCII").equals(euroString));
+
+	    // Close the database
+	    db.close();
+	    Test.ok(60, true);
+
+	    // Try to read from a database that is closed
+	    caught = false;
+	    try {
+		db.fetch("larry!");
+	    } catch ( Exception e ) {
+		System.out.println("caught: " + e.toString());
+		caught = true;
+	    }
+	    Test.ok(70, caught);
+
+	    // Try to open a non-existent database
+	    caught = false;
+	    try {
+		db = new GdbmFile("/tmp/somenonexistentdatabase.gdbm",
+				  GdbmFile.READER);
+	    } catch ( GdbmException e ) {
+		System.out.println("caught: " + e.toString());
+		caught = true;
+	    }
+	    Test.ok(80, caught);
+
+	    // Open a database read-only
+	    db = new GdbmFile(dbFile, GdbmFile.READER);
+	    db.setKeyPacking(new StringPacking());
+	    db.setValuePacking(new StringPacking());
+	    Test.ok(90);
+
+	    // Try to insert
+	    caught = false;
+	    try {
+		db.store("apple", "crumble");
+	    } catch ( GdbmException e ) {
+		System.out.println("caught: " + e.toString());
+		caught = true;
+	    }
+	    Test.ok(91, caught);
+
+	    // Replace an existing database
+	    db.close();
+	    db = new GdbmFile(dbFile, GdbmFile.NEWDB);
+	    db.setKeyPacking(new StringPacking());
+	    db.setValuePacking(new StringPacking());
+	    Test.ok(100, !db.exists("camel"));
+
+	    // Sync to disk
+	    db.sync();
+	    Test.ok(105, true);
+
+	    // Juggle three databases at once
+	    final String dbFile2 = "/tmp/java2.gdbm";
+	    GdbmFile db2 = new GdbmFile(dbFile2, GdbmFile.NEWDB);
+	    final String dbFile3 = "/tmp/java3.gdbm";
+	    GdbmFile db3 = new GdbmFile(dbFile3, GdbmFile.NEWDB);
+	    Test.ok(110, true);
+
+	    keys = db.keys();
+	    while (keys.hasMoreElements()) {
+		key = (String) keys.nextElement();
+		value = (String) db.fetch(key);
+		db2.store(key.getBytes(), value.getBytes());
+		db3.store(key.getBytes(), value.getBytes());
+	    }
+	    Test.ok(120, true);
+
+	    keys = db2.keys();
+	    found = true;
+	    while (keys.hasMoreElements())
+		found &= db3.exists(keys.nextElement());
+	    Test.ok(130, found);
+
+	    db2.close();
+	    db3.close();
+
+	    Test.ok(140, true);
+
+	    testBigFile(db);
+
+	    db.close();
+	    Test.ok(160, true);
+	} catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/io/IOUtils.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/io/IOUtils.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/io/IOUtils.java	(revision 21112)
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 1997 by Pharos IP Pty Ltd
+ * All rights reserved.
+ * $Id$
+ */
+
+package au.com.pharos.io;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+
+/**
+ * Plumbing utilities for manipulating Java IO classes.
+ *
+ * <P>These utilities really ought to be part of the
+ * <CODE>java.io</CODE> package, but they're not.  Since they're
+ * pretty useful, here they are.
+ *
+ * @version $Revision$
+ * @author Martin Pool
+ **/
+public class IOUtils {
+    /** Size of blocks to read during IO. **/
+    static public final int BLOCK_SIZE = 4<<10;
+
+    static public void pumpStream(InputStream inStream,
+				  OutputStream outStream,
+				  int contentLength)
+	 throws IOException
+    {
+	int readBytes = 0;
+	byte[] block = new byte[BLOCK_SIZE];
+	int blockBytes;
+	while (readBytes < contentLength
+	       && (blockBytes = inStream.read(block)) > 0) {
+	    readBytes += blockBytes;
+	    outStream.write(block, 0, blockBytes);
+	}
+    }
+
+    static public int pumpStream(InputStream inStream,
+				 OutputStream outStream)
+	 throws IOException
+    {
+	byte[] block = new byte[BLOCK_SIZE];
+	int blockBytes, readBytes = 0;
+	while ( (blockBytes = inStream.read(block)) > 0 ) {
+	    outStream.write(block, 0, blockBytes);
+	    readBytes += blockBytes;
+	}
+	return readBytes;
+    }
+
+    /** Reads the entire contents of the stream into a
+     * newly-constructed byte array.
+     **/
+    static public byte[] readAll(InputStream inStream)
+	 throws IOException
+    {
+	ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+	pumpStream(inStream, byteStream);
+	byte[] buf = byteStream.toByteArray();
+	byteStream.close();
+	return buf;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/SerializationDumper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/SerializationDumper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/SerializationDumper.java	(revision 21112)
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ * Confidential. All rights reserved.
+ */
+
+package au.com.pharos.meta;
+
+import java.lang.reflect.Field;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InvalidClassException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectStreamException;
+import java.io.OptionalDataException;
+import java.io.PrintWriter;
+
+/*
+ * TODO: Allow people to specify an input file?
+ * TODO: Allow people to customize the display format?
+ *
+ * FIXME: We should use a custom classloader to load the untrusted
+ * classes under examination and to allow for flexible handling or
+ * tracing as classes are loaded.
+ */
+
+
+/** Java utility to display in a debugging format the contents of a
+ * serialized object stream.
+ *
+ * <P>Usage:
+ * <PRE>
+ *   java au.com.pharos.meta.SerializationDumper <SERFILE
+ * </PRE>
+ *
+ * where SERFILE is the name of the serialization file.
+ *
+ * <P><B>Limitations:</B>
+ *
+ * <OL>
+ *   <P><LI>Values are displayed only for public fields, due to Java
+ *    security restrictions.
+ *   <P><LI>Only top-level objects are dumped.
+ * </OL>
+ *
+ * <P>These are fairly serious restrictions, but SerializationDumper
+ * at least checks that the stream is well-formed, and gives some
+ * indication of what it contains.
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ **/
+public class SerializationDumper {
+    public static void main(String argv[])
+    {
+	try {
+	    ObjectInput in = new ObjectInputStream(System.in);
+	    PrintWriter writer = new PrintWriter(System.out);
+	    SerializationDumper dumper = new SerializationDumper();
+	    dumper.dump(in, writer);
+	    writer.flush();
+	} catch (IOException e) {
+	    e.printStackTrace();
+	}
+    }
+
+
+    /** Dump all objects read from <EM>in</EM> to <EM>out</EM>
+     * in a human-readable debugging form.
+     **/
+    public void dump(ObjectInput in, PrintWriter out)
+	 throws IOException
+    {
+	while (true) {		// exception on eof
+	    Object obj = null;
+	    try {
+		obj = in.readObject();
+		displayObject(obj, out);
+	    }
+	    catch (ClassNotFoundException e) {
+		warning(e, out);
+	    }
+	    catch (OptionalDataException e) {
+		error(e, out);
+		return;
+	    }
+	    catch (ObjectStreamException e) {
+		warning(e, out);
+	    }
+	    catch (EOFException e) {
+		return;
+	    }
+	}
+    }
+
+
+    /** Display an object in human-readable form. **/
+    private void displayObject(Object obj, PrintWriter out)
+	 throws IOException
+    {
+	if (obj == null) {
+	    out.println("(null)");
+	    return;
+	}
+
+	Class cls = obj.getClass();
+
+	// Print the header: the name, hashcode, and string
+	// representation
+	out.println(cls.getName() + "@0x" +
+		    Integer.toHexString(obj.hashCode()) + "{");
+	out.println("\t\"" + obj.toString() + "\"");
+
+	// TODO: Recursive descent of the graph, allowing for circular
+	// structures.
+
+	// Print each field of the object
+	Field[] fields = cls.getDeclaredFields();
+	for (int i = 0; i < fields.length; i++)
+	    displayField(obj, fields[i], out);
+
+	out.println("}");
+    }
+
+
+    /** Display a particular field from an object in human-readable
+      * form.  Fields which are not accessible for security reasons
+      * are noted as such, and the name and type of the field is
+      * still printed. **/
+    private void displayField(Object obj, Field fld, PrintWriter out)
+	 throws IOException
+    {
+	int flags = fld.getModifiers();
+	out.print("\t");
+	out.print(fld.toString() + ": ");
+	try {
+	    Object value = fld.get(obj);
+	    if (value == null) {
+		out.println("(null)");
+	    }
+	    else {
+		out.println(value.getClass().getName() + "@0x" +
+			    Integer.toHexString(value.hashCode()) + ":");
+		out.println("\t\t\"" + value.toString() + "\"");
+	    }
+	} catch (IllegalAccessException e) {
+	    out.println("(not accessible)");
+	}
+    }
+
+
+    /** Display a warning that an exception occurred while dumping
+     * information. **/
+    private void warning(Exception e, PrintWriter out)
+	 throws IOException
+    {
+	out.println("warning: " + e);
+    }
+
+    /** Display an exception that occurred while dumping
+     * information. **/
+    private void error(Exception e, PrintWriter out)
+	 throws IOException
+    {
+	out.println("error: " + e.toString());
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/ShowClass.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/ShowClass.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/meta/ShowClass.java	(revision 21112)
@@ -0,0 +1,118 @@
+// This example is from the book "Java in a Nutshell, Second Edition".
+// Written by David Flanagan.  Copyright (c) 1997 O'Reilly & Associates.
+// You may distribute this source code for non-commercial purposes only.
+// You may study, modify, and use this example for any purpose, as long as
+// this notice is retained.  Note that this example is provided "as is",
+// WITHOUT WARRANTY of any kind either expressed or implied.
+
+import java.lang.reflect.*;
+
+/** A program that displays a class synopsis for the named class */
+public class ShowClass {
+  /** The main method.  Print info about the named class */
+  public static void main(String[] args) throws ClassNotFoundException {
+    Class c = Class.forName(args[0]);
+    print_class(c);
+  }
+
+  /** Display the modifiers, name, superclass and interfaces of a class
+   *  or interface. Then go and list all constructors, fields, and methods. */
+  public static void print_class(Class c)
+  {
+    // Print modifiers, type (class or interface), name and superclass.
+    if (c.isInterface()) {
+      // The modifiers will include the "interface" keyword here...
+      System.out.print(Modifier.toString(c.getModifiers()) +  c.getName());
+    }
+    else
+      System.out.print(Modifier.toString(c.getModifiers()) + " class " +
+                       c.getName() +
+                       " extends " + c.getSuperclass().getName());
+
+    // Print interfaces or super-interfaces of the class or interface.
+    Class[] interfaces = c.getInterfaces();
+    if ((interfaces != null) && (interfaces.length > 0)) {
+      if (c.isInterface()) System.out.println(" extends ");
+      else System.out.print(" implements ");
+      for(int i = 0; i < interfaces.length; i++) {
+        if (i > 0) System.out.print(", ");
+        System.out.print(interfaces[i].getName());
+      }
+    }
+
+    System.out.println(" {");            // Begin class member listing.
+
+    // Now look up and display the members of the class.
+    System.out.println(" // Constructors");
+    Constructor[] constructors = c.getDeclaredConstructors();
+    for(int i = 0; i < constructors.length; i++)      // Display constructors.
+      print_method_or_constructor(constructors[i]);
+
+    System.out.println(" // Fields");
+    Field[] fields = c.getDeclaredFields();           // Look up fields.
+    for(int i = 0; i < fields.length; i++)            // Display them.
+      print_field(fields[i]);
+
+    System.out.println(" // Methods");
+    Method[] methods = c.getDeclaredMethods();        // Look up methods.
+    for(int i = 0; i < methods.length; i++)           // Display them.
+      print_method_or_constructor(methods[i]);
+
+    System.out.println("}");             // End class member listing.
+  }
+
+  /** Return the name of an interface or primitive type, handling arrays. */
+  public static String typename(Class t) {
+    String brackets = "";
+    while(t.isArray()) {
+      brackets += "[]";
+      t = t.getComponentType();
+    }
+    return t.getName() + brackets;
+  }
+
+  /** Return a string version of modifiers, handling spaces nicely. */
+  public static String modifiers(int m) {
+    if (m == 0) return "";
+    else return Modifier.toString(m) + " ";
+  }
+
+  /** Print the modifiers, type, and name of a field */
+  public static void print_field(Field f) {
+    System.out.println("  " +
+                       modifiers(f.getModifiers()) +
+                       typename(f.getType()) + " " + f.getName() + ";");
+  }
+
+  /** Print the modifiers, return type, name, parameter types and exception
+   *  type of a method or constructor.  Note the use of the Member interface
+   *  to allow this method to work with both Method and Constructor objects */
+  public static void print_method_or_constructor(Member member) {
+    Class returntype=null, parameters[], exceptions[];
+    if (member instanceof Method) {
+      Method m = (Method) member;
+      returntype = m.getReturnType();
+      parameters = m.getParameterTypes();
+      exceptions = m.getExceptionTypes();
+    } else {
+      Constructor c = (Constructor) member;
+      parameters = c.getParameterTypes();
+      exceptions = c.getExceptionTypes();
+    }
+
+    System.out.print("  " + modifiers(member.getModifiers()) +
+                     ((returntype!=null)? typename(returntype)+" " : "") +
+                     member.getName() + "(");
+    for(int i = 0; i < parameters.length; i++) {
+      if (i > 0) System.out.print(", ");
+      System.out.print(typename(parameters[i]));
+    }
+    System.out.print(")");
+    if (exceptions.length > 0) System.out.print(" throws ");
+    for(int i = 0; i < exceptions.length; i++) {
+      if (i > 0) System.out.print(", ");
+      System.out.print(typename(exceptions[i]));
+    }
+    System.out.println(";");
+  }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/NativeStringPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/NativeStringPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/NativeStringPacking.java	(revision 21112)
@@ -0,0 +1,82 @@
+/*
+ * module: pip/java/packing -- Strategy objects for converting
+ * Java objects to and from stored data.
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+/** Convert an array of bytes in a Java string without using String
+ * encodings, just using expanding 8-bit bytes to 16-bit chars.
+ *
+ * <P>This will only work for ASCII characters, of course, but for
+ * moving bulk data around it is considerably faster than using
+ * UTF-8 encoding strategies.  Naively compressing Unicode into
+ * 8-bit characters in this manner is deprecated in JDK1.1 to
+ * help ensure that programs are Unicode-safe.
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ *
+ * @see au.com.pharos.packing.Packing
+ * @see au.com.pharos.packing.StringPacking
+ */
+public class NativeStringPacking extends Packing
+{
+    /** Convert an object to an ASCII String representation.
+     *
+     * @return null if obj is null; obj is obj is already a
+     * byte[]; otherwise an native 8-bite encoded version of
+     * obj.toString().
+     **/
+    public byte[] toBytes(final Object obj)
+    {
+	String str;
+	if (obj == null)
+	    return null;
+	else if (obj instanceof byte[])
+	    return (byte[]) obj;
+	else if (obj instanceof String)
+	    str = (String) obj;
+	else
+	    str = obj.toString();
+
+	int len = str.length();
+	byte[] rawArray = new byte[len];
+	str.getBytes(0, len, rawArray, 0);
+	return rawArray;		     
+    }
+
+    /** Convert an array of bytes into a String
+     *
+     * @param raw the raw data to be decoded.
+     *
+     * @return null if <em>raw</em> is null; otherwise a string with the low
+     * 8 bits of each character taken from the corresponding byte in
+     * <em>raw</em>.
+     **/
+    public Object fromBytes(final byte[] raw)
+    {
+	if (raw == null)
+	    return null;
+	else
+	    return new String(raw, 0);
+    }
+}
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/Packing.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/Packing.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/Packing.java	(revision 21112)
@@ -0,0 +1,77 @@
+/*
+ * module: pip/java/packing -- Packing strategy objects for converting
+ *           Java objects to and from stored data.
+ * class: Packing -- Abstract strategy class for packing/unpacking data
+ *
+ * $Id$
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+/** A packing Packing defines a method of converting an object to
+ * and from an array of bytes for storage in a database.
+ *
+ * <P>Generally, each packing Packing should be reflexive: that is,
+ * converting any object to and from an array of bytes should yield
+ * the same object, or an object appropriately equivalent.
+ *
+ * <P>Some Packing subclasses may not be able to pack all
+ * objects, or may not be able to unpack an object from some byte
+ * arrays.  For example, only Strings can safely be stored as string
+ * representations, and not all arrays are valid Java serialization
+ * forms.  In this case, the Packing object should throw an
+ * <code>IllegalArgumentException</code> from <code>toBytes()</code>
+ * or <code>toBytes()</code>.
+ *
+ * <P>In both cases, null references should be passed through
+ * unchanged.
+ *
+ * <P>Example:
+ * <PRE>
+ *    Packing strategy = new StringPacking();
+ *    byte[] raw = strategy.toBytes("Hello world");
+ *    ... store and retrieve raw[]
+ *    String result = (String) strategy.fromBytes(raw)
+ * </PRE>
+ *
+ *
+ * @see au.com.pharos.packing.StringPacking
+ * @see au.com.pharos.packing.NativeStringPacking
+ * @see au.com.pharos.packing.RawPacking
+ * @see au.com.pharos.packing.SerializationPacking
+ *
+ * @version $Revision$
+ * @author Martin Pool
+ **/
+public abstract class Packing {
+    /** Convert an object to an array of bytes. **/
+    public abstract byte[] toBytes(Object obj) throws IllegalArgumentException;
+
+    /** Reconstitute an array of bytes into an object. **/
+    public abstract Object fromBytes(byte[] raw) throws IllegalArgumentException;
+
+    /** Helper function that generates an IllegalArgumentException */
+    protected IllegalArgumentException cantConvert(Object obj) {
+	return new IllegalArgumentException
+	    ("Can't pack object \"" + obj.toString() +
+	     "\" (" + obj.getClass().toString() + ")" + 
+	     "\n\tusing strategy " + this.toString());
+    }
+}
+
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/QuotedVectorPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/QuotedVectorPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/QuotedVectorPacking.java	(revision 21112)
@@ -0,0 +1,187 @@
+/*
+ * module: pip/java/packing -- Strategy objects for converting
+ *         Java objects to and from stored data.
+ * class:  QuotedVectorPacking -- Vectors of Strings packed in a Perl-
+ *         style format
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+import java.util.Vector;
+
+// FIXME: Escape bad characters! -- mbp
+// TODO: Write a more formal definition of the encoding system
+// TODO: More documentation
+
+// XXX: Perhaps the strategy for encoding characters should be in a
+// separate class used by this class?  They would have to
+// interact to make sure that separator chatacters were decoded
+// properly.  Probably not worth worrying about, as this class
+// is only a compatibility measure in any case.
+
+/** Stores Vectors of strings, packed into a single string, as used by
+ * some Perl code.  The encoding format is similar to that used in
+ * Unix databases like
+ * <A HREF="file:/etc/passwd/"><CODE>/etc/passwd</CODE></A>, except that
+ * it uses escape sequences to represent characters that would
+ * normally not be allowed.
+ *
+ * <P>The strings within the vector are separated by separator characters,
+ * which default to colons.
+ *
+ * <P>The resulting packed string is UTF encoded after escaping
+ * control-characters, so non-ASCII characters should be passed through
+ * unchanged.
+ *
+ * <P>Certain characters are escaped as quoted-printable strings.
+ * These characters include colon, equals, and escape characters.  The
+ * encoding accepts and interprets any hex escape when reading data.
+ *
+ * <P>If any elements of the vector are null, they are stored as empty
+ * strings.  On unpacking, empty strings are returned as such.
+ *
+ * <P>An example of a vector encoded in this manner is:
+ * <BLOCKQUOTE>
+ * <PRE>oC7okz059wGnQ:vf:Peter Barnes%2c Pharos Business Solutions</PRE>
+ * </BLOCKQUOTE>
+ *
+ * @author Martin Pool
+ * @version $Revision$ $Date$
+ *
+ * @see java.util.Vector
+ **/
+public class QuotedVectorPacking extends Packing
+implements java.io.Serializable
+{
+    private char hexEscape;
+    private char separator;
+    
+    /** Constructs a QuotedVectorPacking using the default separator
+     * character ':' and escape character '%'.
+     **/
+    public QuotedVectorPacking() {
+	this('%', ':');
+    }
+
+
+    /** Constructs a QuotedVectorPacking with specified separator and
+     * hex-escape characters.
+     *
+     * @param hexEscape the character used to begin a three-character
+     * hex escape
+     *
+     * @param separator the character used to separate elements in the
+     * vector.
+     **/
+    public QuotedVectorPacking(char hexEscape, char separator) {
+	if (hexEscape == separator)
+	    throw new IllegalArgumentException
+		("hexEscape and separator characters are the same");
+	this.hexEscape = hexEscape;
+	this.separator = separator;
+    }
+    
+    
+    /** Convert a Vector of Strings to a QuotedVector represented as an
+     * array of bytes.
+     *
+     * @param obj the Vector to encode; or null.
+     * 
+     * @returns an array of bytes containing a packed representation of
+     * <EM>obj</EM>; or null if <EM>obj</EM> is null.
+     *
+     * @exception java.lang.ClassCastException if obj is not a Vector
+     * or null; or if any element of <EM>obj</EM> is not a String or
+     * null.
+     **/
+    public byte[] toBytes(Object obj)
+    {
+	if (obj == null)
+	    return null;
+	Vector vec = (Vector) obj;
+	StringBuffer sb = new StringBuffer();
+	Object elem;
+	for (int i = 0; i < vec.size(); i++) {
+	    if (i > 0)
+		sb.append(separator);
+	    elem = vec.elementAt(i);
+	    sb.append((elem == null) ? "" : encodeString((String) elem));
+	}
+	return sb.toString().getBytes();
+    }
+
+
+
+    /** Perform hex-escapes on a string.
+     * @returns <EM>from</EM> encoded as a quoted-printable string
+     **/
+    private String encodeString(String from) {
+	StringBuffer buf = new StringBuffer(from.length());
+	char[] chars = from.toCharArray();
+	for (int i = 0; i < chars.length; i++) {
+	    char ch = chars[i];
+	    if (ch == separator
+		|| ch == hexEscape
+		|| Character.isISOControl(ch)
+		|| (ch != ' ' && ch != '\t' && Character.isWhitespace(ch)))
+		{
+		    buf.append(hexEscape);
+		    // XXX: This will truncate high Unicode characters, but I
+		    // don't know what else we can do with them. -- mbp
+		    buf.append(Character.forDigit((ch>>4) & 0xf, 16));
+		    buf.append(Character.forDigit(ch      & 0xf, 16));
+		} else {
+		    buf.append(ch);
+		}
+	}
+	return buf.toString();
+    }
+    
+    
+
+    /** Interprets <em>raw</em> as a packed quoted Vector, and
+     * returns the unpacked Vector of strings. */
+    public Object fromBytes(byte[] raw)
+    {
+	if (raw == null)
+	    return null;
+	
+	Vector result = new Vector();
+	StringBuffer sb = new StringBuffer();
+	for (int i = 0; i < raw.length; i++) {
+	    char ch = (char) raw[i];
+	    if (ch == separator) {
+		result.addElement(sb.toString());
+		sb.setLength(0);
+	    } else if (ch == hexEscape && i < (raw.length - 2)) {
+		byte[] hexBytes = new byte[2];
+		hexBytes[0] = raw[++i];
+		hexBytes[1] = raw[++i];
+		String hex = new String(hexBytes);
+		ch = (char)Integer.parseInt(hex, 16);
+		sb.append(ch);
+	    } else {
+		sb.append(ch);
+	    }
+	}
+	result.addElement(sb.toString());
+	return result;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/RawPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/RawPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/RawPacking.java	(revision 21112)
@@ -0,0 +1,60 @@
+/*
+ * module: pip/java/packing -- Strategy objects for converting
+ * Java objects to and from stored data.
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+/** A packing strategy which stores only raw arrays of bytes.  Other
+ * objects cannot be stored using this strategy.
+ *
+ * @author Martin Pool
+ * @version $Revision$
+ **/
+public class RawPacking extends Packing implements java.io.Serializable
+{
+    /** Perform the trivial packing case, casting an array of bytes
+     * into the same.
+     *
+     * @throw IllegalArgumentException if <em>obj</em> is neith null nor
+     * an array of bytes.
+     */
+    public byte[] toBytes(Object obj) throws IllegalArgumentException
+    {
+	if (obj == null)
+	    return null;
+	else if (obj instanceof byte[])
+	    return (byte[]) obj;
+	else
+	    throw cantConvert(obj);
+    }
+
+    /** Performs the trivial unpacking case of returning <em>raw</em>
+     * untouched.
+     *
+     * @return either null or <em>raw</em>; always assignable to
+     * <code>byte[]</code>
+     **/
+    public Object fromBytes(byte[] raw)
+    {
+	return raw;
+    }
+}
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/SerializationPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/SerializationPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/SerializationPacking.java	(revision 21112)
@@ -0,0 +1,100 @@
+/*
+ * class: SerializationPacking -- Store an object as serialized data
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ObjectInputStream;
+import java.io.IOException;
+
+/** A packing strategy which stores and retrieves objects using
+ * <a href="http://www.javasoft.com/products/jdk/1.1/docs/guide/serialization/index.html">Java serialization</a>.
+ *
+ * @version $Revision$
+ * @author Martin Pool
+ * @see au.com.pharos.packing.Packing
+ **/
+public class SerializationPacking extends Packing
+implements java.io.Serializable
+{
+    /** Convert an object to a Java serialization format.
+     *
+     * @returns a buffer containing a a serialized representation of <em>obj</em>;
+     * or null if <em>obj</em> if null
+     * @exception IllegalArgumentException if <em>obj</em> (or any object
+     * referred to by <em>obj</em>) cannot be serialized
+     **/
+    public byte[] toBytes(Object obj) throws IllegalArgumentException
+    {
+	if (obj == null)
+	    return null;
+
+	// Serialize via a stream into a buffer
+	ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+	try {
+	    ObjectOutputStream objStream = new ObjectOutputStream(byteStream);
+	    objStream.writeObject(obj);
+	    // Don't close the stream, or you will lose the byte array
+	    objStream.flush();
+	} catch (IOException e) {
+	    throw new IllegalArgumentException("IOException during serialization: " +
+					       e.toString());
+	}
+
+	return byteStream.toByteArray();
+    }
+
+    /** Interprets <em>raw</em> as a Java serialization stream, and
+     * returns <B>the first</B> object read from that stream.
+     *
+     * <P><B>Note:</B>There is currently no means by which any later
+     * objects in the can be retrieved: it is assumed that the object
+     * written to the buffer contains references to all the
+     * information required.  This is consistent with the fact that
+     * toBytes() stores only a single object, so this packing strategy
+     * is completely reversable.
+     **/
+    // FIXME: Is the correct word 'reversable', 'reflexive', or
+    // something else?  -- mbp@pharos.com.au
+    public Object fromBytes(byte[] raw)
+    {
+	if (raw == null)
+	    return null;
+
+	Object obj;
+	try {
+	    ByteArrayInputStream byteStream = new ByteArrayInputStream(raw);
+	    ObjectInputStream objStream = new ObjectInputStream(byteStream);
+	    obj = objStream.readObject();
+	    objStream.close();
+	} catch (IOException e) {
+	    throw new IllegalArgumentException("Exception during deserialization: " +
+					       e.toString());
+	} catch (ClassNotFoundException e) {
+	    throw new IllegalArgumentException("Exception during deserialization: " +
+					       e.toString());
+	}	    
+	return obj;
+    }
+}
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/StringPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/StringPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/StringPacking.java	(revision 21112)
@@ -0,0 +1,85 @@
+/*
+ * module: pip/java/packing -- Strategy objects for converting
+ * Java objects to and from stored data.
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+/** StringPacking converts Java Strings to and from byte arrays
+ * using the default UTF-8 encoding.
+ *
+ * <P>For strings containing only ASCII-7 characters, this is
+ * equivalent to <code>NativeStringPacking</code> but probably
+ * somewhat slower.  However, it will safely handle non-ASCII
+ * characters, or unusual locales.
+ *
+ * @see au.com.pharos.packing.Packing
+ * @see au.com.pharos.packing.NativeStringPacking
+ **/
+public class StringPacking extends Packing
+implements java.io.Serializable
+{
+    /** Convert a String to an array of bytes using Java's default
+     * encoding.
+     *
+     * <P>If <em>obj</em> is not a String, array of bytes, or null
+     * then it's <code>toString()</code> method is called first to
+     * convert it to an array of bytes.  This will lose information in
+     * many cases.
+     *
+     * @param obj The object to convert.
+     *
+     * @return <em>obj</em> converted to an array of bytes; or null if
+     * <em>obj</em> is null.
+     **/
+    public byte[] toBytes(Object obj)
+    {
+	if (obj == null)
+	    return null;
+	else if (obj instanceof byte[])
+	    return (byte[]) obj;
+	else if (obj instanceof String)
+	    return ((String) obj).getBytes();
+	else
+	    return obj.toString().getBytes();
+    }
+
+    /** Decode an array of bytes using the default String encoding.
+     *
+     * @param raw An array of bytes to decode.
+     *
+     * @return A String representation of <em>raw</em>; or null if
+     * <em>raw</em> is null.
+     **/     
+    public Object fromBytes(byte[] raw)
+    {
+	if (raw == null)
+	    return null;
+	else {
+	    try{
+	    return new String(raw, "UTF-8");
+	    } catch (Exception e) {
+		System.err.println("String Packing:encoding not supported");
+		return new String(raw);
+	    }
+	}
+    }
+}
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/TestQuotedVectorPacking.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/TestQuotedVectorPacking.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/packing/TestQuotedVectorPacking.java	(revision 21112)
@@ -0,0 +1,84 @@
+/*
+ * module: pip/java/packing -- Strategy objects for converting
+ *         Java objects to and from stored data.
+ * class:  TestQuotedVectorPacking -- Test harness for QuotedVectorPacking
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.packing;
+
+import java.util.Vector;
+
+// TODO: Perhaps make this into a more general-purpose test harness
+// for testing any Packing strategy? -- mbp 19971021
+
+/** Test harness for the QuotedVectorPacking strategy class.
+ *
+ * <P><B>Usage:</B>
+ * <UL><P><CODE>java TestQuotedVectorPacking -[de] ARGS</CODE><UL>
+ *
+ * <P><B>Options</B>
+ * <DL>
+ *   <P><DT><CODE>-d</CODE><DD>Print the results of decoding
+ *      the each of the command-
+ *      line arguments as QuotedVectorPacking encoding strings
+ *
+ *   <P><DT><CODE>-e</CODE><DD>Print the results of encoding
+ *      a vector containing
+ *      all of the command-line arguments 
+ * </DL>
+ *
+ * @author Martin Pool
+ * @version $Revision$ $Date$
+ **/
+public class TestQuotedVectorPacking {
+    private TestQuotedVectorPacking() {}
+    
+    public static void main(String[] argv) {
+	Packing strat = new QuotedVectorPacking();
+	final String usage = "usage: TestQuotedVectorPacking -[ed] ARGS";
+
+	if (argv.length == 0) {
+	    System.err.println(usage);
+	    return;
+	}
+	
+	if (argv[0].equals("-e")) {
+	    Vector encode = new Vector(argv.length - 1);
+	    for (int i = 1; i < argv.length; i++) {
+		System.out.println(argv[i]);
+		encode.addElement(argv[i]);
+	    }
+	    byte[] packed = strat.toBytes(encode);
+	    System.out.print("\t");
+	    System.out.println(new String(packed));
+	} else if (argv[0].equals("-d")) {
+	    for (int i = 1; i < argv.length; i++) {
+		Vector decode = (Vector) strat.fromBytes(argv[i].getBytes());
+		System.out.println(argv[i]);
+		for (int j = 0; j < decode.size(); j++) {
+		    System.out.print("\t");
+		    System.out.println(decode.elementAt(j));
+		}
+	    }
+	} else {
+	    System.err.println(usage);
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/test/Test.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/test/Test.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/test/Test.java	(revision 21112)
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ * $Id$
+ * $Source$
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package au.com.pharos.test;
+
+/** Utilities for automated testing.  This class contains only static
+ * functions and should not be instantiated.
+ *
+ * @author Martin Pool
+ * @version $Revision$ $Date$
+ **/
+// TODO: Somehow automatically generate sequence numbers, or perhaps use
+// strings to report progress
+public class Test {
+    // Don't instantiate
+    private Test() {}
+
+    static int nPassed, nFailed;
+
+    /** Register that a particular test executed successfully, and
+     * output a success or failure message.
+     *
+     * @param number test sequence number
+     **/
+    static public void ok(int no) {
+	ok(no, true);
+    }
+
+    /** Register the result of a test, and output a success or failure
+     * message.
+     *
+     * @param number test sequence number
+     * @param result whether the test was successful or not
+     */
+    static public void ok(int number, boolean passed) {
+	if (passed) {
+	    nPassed++;
+	}
+	else {
+	    System.out.print("not ");
+	    nFailed++;
+	}
+	System.out.print("ok " + number + "\n");
+    }
+
+    /**
+     * Print a summary of test results.
+     **/
+    static public void summary() {
+	System.out.print("test summary: " + nPassed + " passed, " +
+			 nFailed + " failed");
+    }
+}
+    
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/StringOps.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/StringOps.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/StringOps.java	(revision 21112)
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 1997 by Pharos IP Pty Ltd
+ * Confidential.  All rights reserved.
+ * $Id$
+ * $Source$
+ *
+ * ``No strings attached.''
+ */
+
+package au.com.pharos.util;
+
+/**
+ * Mixed bag of string manipulation functions.  It would be nice if
+ * some of these were in <CODE>java.lang.String</CODE>, but since
+ * they're not, they're implemented as static functions within this
+ * class.
+ *
+ * Typical use:
+ * <PRE>
+ * String foo = "some string"
+ * int cnt = StringOps.countOccurrences(foo, 's');
+ * // --> cnt == 2
+ * </PRE>
+ *
+ * @author Martin Pool
+ * @version $Revision$, $Date$
+ **/
+public class StringOps
+{
+    /** This class contains only static public functions. **/
+    private StringOps() { ; }
+
+    /** Count the number of occurences of a character in a
+     * string.
+     *
+     * @param needle the character to search
+     * @param haystack string to be searched
+     *
+     * @return 0 if <EM>needle</EM> does not occur in <EM>haystack</EM>;
+     * or a positive number indicating the number of occurrences
+     **/
+    static public int countOccurrences(String haystack, char needle)
+    {
+	byte[] hay = haystack.getBytes();
+	int count = 0;
+	for (int i = 0; i < hay.length; i++)
+	    if (hay[i] == needle)
+		count++;
+	return count;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/test/StringOpsTest.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/test/StringOpsTest.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/au/com/pharos/util/test/StringOpsTest.java	(revision 21112)
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 1997 by Pharos IP Pty Ltd
+ * Confidential.  All rights reserved.
+ * $Id$
+ * $Source$
+ */
+
+package au.com.pharos.util.test;
+
+import au.com.pharos.test.Test;
+
+import au.com.pharos.util.StringOps;
+
+/**
+ * Test harness for <CODE>au.com.pharos.util.StringOps</CODE>
+ *
+ * @author Martin Pool
+ * @version $Revision$ $Date$
+ **/
+public class StringOpsTest
+{
+    static public void main(String[] args)
+    {
+	new StringOpsTest().run();
+    }
+
+    public void run()
+    {
+	try {
+	    testCountOccurrences();
+	}
+	finally {
+	    Test.summary();
+	}
+    }
+
+    public void testCountOccurrences()
+    {
+	// Simple tests of countOccurences(String, char)
+	Test.ok(100, ( StringOps.countOccurrences("hello world", 'z') == 0) );
+	Test.ok(110, ( StringOps.countOccurrences("zhello world", 'z') == 1) );
+	Test.ok(120, ( StringOps.countOccurrences("hello worldz", 'z') == 1) );
+	Test.ok(130, ( StringOps.countOccurrences("z", 'z') == 1) );
+	Test.ok(140, ( StringOps.countOccurrences("zzz", 'z') == 3) );
+	Test.ok(150, ( StringOps.countOccurrences("0z0z0z", 'z') == 3) );
+	Test.ok(160, ( StringOps.countOccurrences("\0z\0z\0z", '\0') == 3) );
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/java/winMake.bat
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/java/winMake.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/java/winMake.bat	(revision 21112)
@@ -0,0 +1,45 @@
+@echo off
+
+set JAVAGDBMHOME=..
+
+rem ---- Where to put class files ----
+set JAVACLASSDIR=classes
+
+rem ---- Name and location of java programs ----
+set JAVAC="%JAVA_HOME%\bin\javac"
+set JAVAH="%JAVA_HOME%\bin\javah"
+set JAVA="%JAVA_HOME%\bin\java"
+set JAVADOC="%JAVA_HOME%\bin\javadoc"
+set JAR="%JAVA_HOME%\bin\jar"
+
+set JAVACOPTIONS= -deprecation -g -O
+
+
+if ""%1"" == """" goto compile
+if ""%1"" == ""compile"" goto compile
+if ""%1"" == ""install"" goto install
+if ""%1"" == ""clean"" goto clean
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:compile
+	if not exist %JAVACLASSDIR% mkdir %JAVACLASSDIR%
+	echo Compiling...
+        %JAVAC% -d %JAVACLASSDIR% %JAVACOPTIONS%  au\com\pharos\gdbm\*.java au\com\pharos\io\*.java au\com\pharos\meta\*.java au\com\pharos\packing\*.java au\com\pharos\test\*.java au\com\pharos\util\*.java
+        %JAVAH% -classpath %JAVACLASSDIR% -o %JAVAGDBMHOME%\jni\GdbmFile.h au.com.pharos.gdbm.GdbmFile
+        %JAR% cf %JAVAGDBMHOME%\javagdbm.jar -C %JAVACLASSDIR% au
+	goto done
+
+:install
+	goto done
+
+:clean
+	echo Cleaning up...
+	if exist %JAVACLASSDIR% rmdir /S /Q %JAVACLASSDIR%
+        if exist %JAVAGDBMHOME%\javagdbm.jar del %JAVAGDBMHOME%\javagdbm.jar 
+        if exist %JAVAGDBMHOME%\jni\GdbmFile.h del %JAVAGDBMHOME%\jni\GdbmFile.h 
+	goto done
+
+:done
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/GdbmFile.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/GdbmFile.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/GdbmFile.h	(revision 21112)
@@ -0,0 +1,119 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class au_com_pharos_gdbm_GdbmFile */
+
+#ifndef _Included_au_com_pharos_gdbm_GdbmFile
+#define _Included_au_com_pharos_gdbm_GdbmFile
+#ifdef __cplusplus
+extern "C" {
+#endif
+#undef au_com_pharos_gdbm_GdbmFile_READER
+#define au_com_pharos_gdbm_GdbmFile_READER 0L
+#undef au_com_pharos_gdbm_GdbmFile_WRITER
+#define au_com_pharos_gdbm_GdbmFile_WRITER 1L
+#undef au_com_pharos_gdbm_GdbmFile_WRCREAT
+#define au_com_pharos_gdbm_GdbmFile_WRCREAT 2L
+#undef au_com_pharos_gdbm_GdbmFile_NEWDB
+#define au_com_pharos_gdbm_GdbmFile_NEWDB 3L
+#undef au_com_pharos_gdbm_GdbmFile_FAST
+#define au_com_pharos_gdbm_GdbmFile_FAST 16L
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_open
+ * Signature: (Ljava/lang/String;I)J
+ */
+JNIEXPORT jlong JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1open
+  (JNIEnv *, jobject, jstring, jint);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_close
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1close
+  (JNIEnv *, jobject, jlong);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_store
+ * Signature: (J[B[BZ)V
+ */
+JNIEXPORT void JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1store
+  (JNIEnv *, jobject, jlong, jbyteArray, jbyteArray, jboolean);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_fetch
+ * Signature: (J[B)[B
+ */
+JNIEXPORT jbyteArray JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1fetch
+  (JNIEnv *, jobject, jlong, jbyteArray);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_exists
+ * Signature: (J[B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1exists
+  (JNIEnv *, jobject, jlong, jbyteArray);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_delete
+ * Signature: (J[B)V
+ */
+JNIEXPORT void JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1delete
+  (JNIEnv *, jobject, jlong, jbyteArray);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_firstkey
+ * Signature: (J)[B
+ */
+JNIEXPORT jbyteArray JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1firstkey
+  (JNIEnv *, jobject, jlong);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_nextkey
+ * Signature: (J[B)[B
+ */
+JNIEXPORT jbyteArray JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1nextkey
+  (JNIEnv *, jobject, jlong, jbyteArray);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_getversion
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1getversion
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_wrapperVersion
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1wrapperVersion
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_reorganize
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1reorganize
+  (JNIEnv *, jobject, jlong);
+
+/*
+ * Class:     au_com_pharos_gdbm_GdbmFile
+ * Method:    gdbm_sync
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_au_com_pharos_gdbm_GdbmFile_gdbm_1sync
+  (JNIEnv *, jobject, jlong);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,63 @@
+# Makefile for the native parts of the au.com.pharos.gdbm
+# Java package.
+# Modified by kjdon 2005
+
+# Copyright (C) 1997 by Pharos IP Pty Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+#srcdir=@srcdir@
+libdir=@libdir@
+
+CC=@CC@
+JDK=@JAVA_HOME@
+INC=-I. -I$(JDK)/include -I$(JDK)/include/linux @GDBM_INCLUDE@ 
+CFLAGS=-c @CFLAGS@ $(INC) -DWRAPPER_VERSION=\"$(WRAPPER_VERSION)\" 
+#PKG=@PKG@
+GDBM_LIBS=@GDBM_LIBPATH@ -lgdbm
+LIBS= $(GDBM_LIBS) @LIBS@
+LDFLAGS = @LDFLAGS@
+JNIFLAGS=@JNIFLAGS@
+JNISUFFIX=@JNISUFFIX@
+JNIINC=@JNIINC@
+
+# TODO: Build versioned shared libraries?
+
+# TODO: Build shared libraries in a way that will work on different
+# platforms -- mbp
+
+WRAPPER_VERSION=0005
+NATIVE_LIB=libgdbmjava.$(JNISUFFIX)
+
+all: $(NATIVE_LIB)
+
+$(NATIVE_LIB): GdbmFile.h gdbmjava.o Makefile
+	$(CC) $(LDFLAGS) $(JNIFLAGS) -o $(NATIVE_LIB) gdbmjava.o $(LIBS)
+
+gdbmjava.o: gdbmjava.c	
+	$(CC) $(INC) $(CFLAGS) $?
+
+# no easy way to tell if a class file has changed or not
+#GdbmFile.h: 
+#	$(javah) -o GdbmFile.h -jni $(PKG).GdbmFile 
+
+clean:	
+	rm -f  *.o $(NATIVE_LIB) 
+
+distclean: clean
+	rm -f Makefile
+
+install: 
+	cp $(NATIVE_LIB) $(libdir)/
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/gdbmjava.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/gdbmjava.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/gdbmjava.c	(revision 21112)
@@ -0,0 +1,476 @@
+/*
+ * module: pip/jni/gdbm -- A Java interface to the GDBM library
+ * file:   gdbmjava.c -- Native parts of the au.com.pharos.gdbm.GdbmFile
+ *             Java class
+ *
+ * Copyright (C) 1997 Pharos IP Pty Ltd
+ *
+ * $Id$ 
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/* NOTE ON MEMORY MANAGEMENT: GDBM always mallocs memory and expects
+ * the caller to free it.  Java requires native functions to interact
+ * with it to lock and unlock buffers. */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined (__WIN32__)
+#include <gdbmdefs.h>
+#include <gdbmerrno.h>
+#define GDBM_FILE gdbm_file_info *
+extern char* gdbm_version;
+#else
+#include <gdbm.h>
+#endif
+
+#include <jni.h>
+
+#include <GdbmFile.h>
+
+#ifdef DEBUG
+#define ASSERT(x) if (!(x)) { \
+  fprintf(stderr, "%s:%d: assertion failed\n", __FILE__, __LINE__); \
+  abort(); }
+#else /* !DEBUG */
+#define ASSERT(x) 
+#endif /* !DEBUG */
+
+/* The Java class within which the native methods are declared */
+#define JAVA_CLASS "au.com.pharos.gdbm.GdbmFile"
+
+#define NULL_PTR_EXCEPTION(env) nullPtrException(env, __FILE__, __LINE__)
+#define GDBM_EXCEPTION(env) gdbmException(env, __FILE__, __LINE__)
+
+#define CHECK_NOT_NULL(ptr, env) if (!ptr) { NULL_PTR_EXCEPTION(env); return 0; }
+#define CHECK_NOT_NULL_VOID(ptr, env) if (!ptr) { NULL_PTR_EXCEPTION(env); return; }
+
+void gdbmException(JNIEnv *env, const char *file, int line);
+void nullPtrException(JNIEnv *env, const char *file, int line);
+
+void releaseArray(JNIEnv *env, jbyteArray array, datum *fromDatum);
+void releaseArrayAbort(JNIEnv *env, jbyteArray array, datum *fromDatum);
+int arrayToDatum(JNIEnv *env, jbyteArray fromArray, datum *toDatum);
+jbyteArray datumToArray(JNIEnv *env, datum *fromDatum);
+
+/* Convert between a jlong and a void ptr using a well-defined cast.
+ * (Casting between a pointer and an integer of different sizes spooks
+ * both gcc and mbp. */
+#if (SIZEOF_VOID_P == SIZEOF_LONG)
+#  define DBF_TO_JLONG(x) ((jlong)((long) x))
+#  define JLONG_TO_DBF(x) ((GDBM_FILE)((long) x))
+#elif (SIZEOF_VOID_P == SIZEOF_INT)
+#  define DBF_TO_JLONG(x) ((jlong)((int) (x)))
+#  define JLONG_TO_DBF(x) ((GDBM_FILE)((int) (x)))
+#else
+#  define DBF_TO_JLONG(x) ((jlong)(x))
+#  define JLONG_TO_DBF(x) ((GDBM_FILE)(x))
+#endif
+
+
+JNIEXPORT jlong JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1open
+    (JNIEnv *env, jobject obj, jstring fileName, jint flags)
+{
+    GDBM_FILE dbf;
+    const char *utfFileName;
+   
+    utfFileName = (*env)->GetStringUTFChars(env, fileName, 0);
+    if (!utfFileName) 
+	return 0;
+    
+    setbuf(stderr, 0);
+   
+    /* XXX: Should we let the caller specify the file mode?  I think
+     * not -- Java is above the level of octal file modes. [mbp] */
+    /* Couldn't get it to work properly on Windows without the 0 here - 
+       wouldn't allow mulitple READs on a single file [kjdon] */
+#if defined (__WIN32__) 
+    dbf = gdbm_open((char *) utfFileName, 0, flags, 0660, NULL, 0);
+#else
+    dbf = gdbm_open((char *) utfFileName, 0, flags, 0660, NULL);
+#endif
+    if (utfFileName)
+	(*env)->ReleaseStringUTFChars(env, fileName, utfFileName);
+
+    if (!dbf) {
+	GDBM_EXCEPTION(env);
+	return 0;
+    }	
+   
+    return DBF_TO_JLONG(dbf);
+}
+
+
+
+
+
+JNIEXPORT jbyteArray JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1firstkey
+(JNIEnv *env, jobject obj, jlong dbf)
+{
+    datum	keyDatum;
+    jbyteArray 	keyArray;
+
+    CHECK_NOT_NULL(dbf, env);
+
+    keyDatum = gdbm_firstkey(JLONG_TO_DBF(dbf));
+    if ( gdbm_errno != GDBM_NO_ERROR ) {
+	GDBM_EXCEPTION(env);
+	return 0;
+    }
+    if ( !keyDatum.dptr )
+	return 0;
+
+    keyArray = datumToArray(env, &keyDatum);
+    free(keyDatum.dptr); 
+
+    return keyArray;
+}
+
+
+
+
+
+JNIEXPORT jbyteArray JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1nextkey
+(JNIEnv *env, jobject this, jlong dbf, jbyteArray keyArray)
+{
+    datum	keyDatum;
+    datum	nextDatum;
+    jbyteArray	nextArray;
+
+    CHECK_NOT_NULL(dbf, env);
+
+    if (!arrayToDatum(env, keyArray, &keyDatum)) {
+	NULL_PTR_EXCEPTION(env);
+	return 0;
+    }
+    
+    nextDatum = gdbm_nextkey(JLONG_TO_DBF(dbf), keyDatum);
+    releaseArrayAbort(env, keyArray, &keyDatum);
+
+    if ( gdbm_errno != GDBM_NO_ERROR ) {
+	GDBM_EXCEPTION(env);
+	return 0;
+    }
+    if ( !nextDatum.dptr )
+	return 0;
+
+    nextArray = datumToArray(env, &nextDatum);
+    free(nextDatum.dptr);
+    
+    return nextArray;
+}
+
+
+
+
+
+
+JNIEXPORT jbyteArray JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1fetch
+  (JNIEnv *env, jobject this, jlong dbf, jbyteArray keyArray)
+{
+    datum 	keyDatum;
+    datum 	valueDatum;
+    jbyteArray	valueArray;
+
+    CHECK_NOT_NULL(dbf, env);
+
+    if (!arrayToDatum(env, keyArray, &keyDatum)) {
+	NULL_PTR_EXCEPTION(env);
+	return 0;
+    }
+
+    valueDatum = gdbm_fetch(JLONG_TO_DBF(dbf), keyDatum);
+    releaseArrayAbort(env, keyArray, &keyDatum);
+
+    if ( !valueDatum.dptr )
+	return 0;
+    if ( gdbm_errno != GDBM_NO_ERROR ) {
+	GDBM_EXCEPTION(env);
+	return 0;
+    }
+
+    valueArray = datumToArray(env, &valueDatum);
+    free(valueDatum.dptr);
+
+    return valueArray;
+}
+
+
+
+JNIEXPORT jboolean JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1exists
+  (JNIEnv *env, jobject obj, jlong dbf, jbyteArray keyArray)
+{
+    datum 	keyDatum;
+    int		result;
+
+    CHECK_NOT_NULL(dbf, env);
+    if (!arrayToDatum(env, keyArray, &keyDatum)) {
+	NULL_PTR_EXCEPTION(env);
+	return JNI_FALSE;
+    }
+    result = gdbm_exists(JLONG_TO_DBF(dbf), keyDatum);
+    if ( gdbm_errno != GDBM_NO_ERROR ) {
+	GDBM_EXCEPTION(env);
+	return 0;
+    }
+    releaseArrayAbort(env, keyArray, &keyDatum);
+    return result ? JNI_TRUE : JNI_FALSE;
+}
+
+
+
+
+
+JNIEXPORT void JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1store
+  (JNIEnv *env, jobject obj, jlong dbf, 
+   jbyteArray keyArray, jbyteArray valueArray, jboolean replace)
+{
+    datum	keyDatum;
+    datum	valueDatum;
+    
+    CHECK_NOT_NULL_VOID(dbf, env);
+    
+    if ( !arrayToDatum(env, keyArray, &keyDatum)
+	 || !arrayToDatum(env, valueArray, &valueDatum) ) {
+	NULL_PTR_EXCEPTION(env);
+	return;
+    }
+
+    gdbm_store(JLONG_TO_DBF(dbf), keyDatum, valueDatum, 
+	       replace ? GDBM_REPLACE : 0);
+
+    releaseArrayAbort(env, keyArray, &keyDatum);
+    releaseArrayAbort(env, valueArray, &valueDatum);
+
+    if ( gdbm_errno != GDBM_NO_ERROR )
+	GDBM_EXCEPTION(env);
+}
+
+
+
+JNIEXPORT void JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1delete
+   (JNIEnv *env, jobject obj, jlong dbf, jbyteArray keyArray)
+{
+    datum 	keyDatum;
+
+    CHECK_NOT_NULL_VOID(dbf, env);
+
+    if (!arrayToDatum(env, keyArray, &keyDatum)) {
+	NULL_PTR_EXCEPTION(env);
+	return;
+    }
+
+    gdbm_delete(JLONG_TO_DBF(dbf), keyDatum);
+
+    releaseArrayAbort(env, keyArray, &keyDatum);
+    
+    if ( gdbm_errno != GDBM_NO_ERROR )
+	GDBM_EXCEPTION(env);
+}
+
+
+
+JNIEXPORT jstring JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1getversion
+  (JNIEnv *env, jclass cls)
+{
+    return (*env)->NewStringUTF(env, gdbm_version);
+}
+
+
+JNIEXPORT jstring JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1wrapperVersion
+  (JNIEnv *env, jclass cls)
+{
+    return (*env)->NewStringUTF(env, "JavaGDBM release 0005 built " __DATE__);
+}
+				
+
+
+
+JNIEXPORT void JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1reorganize
+  (JNIEnv *env, jobject obj, jlong dbf)
+{
+    CHECK_NOT_NULL_VOID(dbf, env);
+
+    gdbm_reorganize(JLONG_TO_DBF(dbf));
+
+    if ( gdbm_errno != GDBM_NO_ERROR )
+	GDBM_EXCEPTION(env);
+}
+
+
+
+JNIEXPORT void JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1close
+  (JNIEnv *env, jobject obj, jlong dbf)
+{
+    CHECK_NOT_NULL_VOID(dbf, env);
+
+    gdbm_close(JLONG_TO_DBF(dbf));
+
+    if ( gdbm_errno != GDBM_NO_ERROR )
+	GDBM_EXCEPTION(env);
+}
+
+
+
+JNIEXPORT void JNICALL 
+Java_au_com_pharos_gdbm_GdbmFile_gdbm_1sync
+  (JNIEnv *env, jobject obj, jlong dbf)
+{
+    CHECK_NOT_NULL_VOID(dbf, env);
+
+    gdbm_sync(JLONG_TO_DBF(dbf));
+
+    if ( gdbm_errno != GDBM_NO_ERROR )
+	GDBM_EXCEPTION(env);
+}
+
+
+
+
+/**********************************************************************
+ *
+ * Following are support functions which aid in interfacing C to 
+ * Java. */
+
+/** Create a new Java byte array from a GDBM datum, and return a
+ * pointer thereto.  */
+
+jbyteArray datumToArray(JNIEnv *env, datum *fromDatum) 
+{
+    jbyteArray toArray;
+
+    if (!fromDatum || !fromDatum->dptr)
+	return 0;
+
+    toArray = (*env)->NewByteArray(env, fromDatum->dsize);
+    ASSERT(toArray);
+    (*env)->SetByteArrayRegion(env, toArray, 
+			       0, fromDatum->dsize, fromDatum->dptr);
+    
+    return toArray;
+}
+
+/** Convert a Java byte array to a GDBM datum.
+ *
+ * The Java array is pinned or copied for use in the datum, and must
+ * be released after use by releaseBytes. 
+ *
+ * Returns true if the array is non-null and could be pinned.  Otherwise,
+ * returns false. 
+ */
+int arrayToDatum(JNIEnv *env, jbyteArray fromArray, datum *toDatum)
+{
+    if (fromArray) {
+	toDatum->dptr = (*env)->GetByteArrayElements(env, fromArray, 0);
+	toDatum->dsize = (*env)->GetArrayLength(env, fromArray);
+	return (int) toDatum->dptr;
+    }
+    else 
+	return 0;
+}
+
+/** Release a byte array pinned or copied for use in a datum. */
+void releaseArray(JNIEnv *env, jbyteArray array, datum *fromDatum) 
+{
+    ASSERT(fromDatum->dptr);
+    (*env)->ReleaseByteArrayElements(env, array, fromDatum->dptr, 0);
+    fromDatum->dptr = 0;	/* no longer valid */
+}
+
+/** Release a byte array pinned or copied for use in a datum, aborting 
+ * any changes.  This potentially saves the runtime from having to 
+ * copy back an unchanged array. */
+void releaseArrayAbort(JNIEnv *env, jbyteArray array, datum *fromDatum) 
+{
+    ASSERT(fromDatum->dptr);
+    (*env)->ReleaseByteArrayElements(env, array, fromDatum->dptr, 
+				     JNI_ABORT);
+    fromDatum->dptr = 0;	/* no longer valid */
+}
+
+
+/** Throw a null pointer exception. */
+void nullPtrException(JNIEnv *env, const char *file, int line) 
+{
+    jclass exClass;
+    char reason[1024];
+    sprintf(reason, "Null pointer exception in GDBM wrapper (%s:%d)",
+	    file, line);
+    
+    exClass = (*env)->FindClass(env, "java/lang/NullPointerException");
+    ASSERT(exClass);
+
+    (*env)->ThrowNew(env, exClass, reason);
+}
+
+/** Translate the GDBM error into throw a Java exception, and throw
+ * same. 
+ *
+ * TODO: Throw different classes of exceptions depending on what the 
+ * underlying error is. 
+ */
+void gdbmException(JNIEnv *env, const char *file, int line) {
+    jclass exClass;
+    static char reason[1500];
+    static char srcLocation[500];
+
+    exClass = (*env)->FindClass(env, "au/com/pharos/gdbm/GdbmException");
+    ASSERT(exClass);
+
+    strncpy(reason, gdbm_strerror(gdbm_errno), 500);
+    sprintf(srcLocation, " (%s:%d)", file, line);
+
+    /* If the error code suggests that an OS or stdio error may have occurred,
+     * include supplementary information from errno. */
+    switch (gdbm_errno) {
+    case GDBM_FILE_OPEN_ERROR:
+    case GDBM_FILE_WRITE_ERROR:
+    case GDBM_FILE_SEEK_ERROR:
+    case GDBM_FILE_READ_ERROR:
+    case GDBM_MALLOC_ERROR:
+    case GDBM_REORGANIZE_FAILED:
+	strcat(reason, ": ");
+	strncat(reason, strerror(errno), 490);
+	strcat(reason, "?");
+    default:
+	/* errno is probably not useful */
+	;
+    }
+    
+    strncat(reason, srcLocation, 495);
+    gdbm_errno = GDBM_NO_ERROR;	/* this one has been handled */
+    (*env)->ThrowNew(env, exClass, reason);    
+}
+
+/* 
+ * Local variables:
+ * c-basic-offset: 4
+ * End:
+ */
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/win32.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/jni/win32.mak	(revision 21112)
@@ -0,0 +1,75 @@
+###########################################################################
+#
+# win32 makefile -- javagdbm\jni
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+JAVAGDBMHOME = ..
+ 
+GSDL3HOME = ..\..\..
+GSDLHOME= $(GSDL3HOME)\gs2build
+GDBMHOME = $(GSDLHOME)\packages\windows\gdbm\gdbm
+# set javahome here
+#JAVA_HOME =
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -D__WIN32__ -DHAVE_STRING_H
+
+INCLUDES = -I"$(GDBMHOME)" -I. \
+           -I"$(JAVA_HOME)\include" -I"$(JAVA_HOME)\include\win32"           
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    GdbmFile.h
+
+SOURCES = \
+    gdbmjava.c
+
+OBJECTS = \
+    gdbmjava$o   $(GDBMHOME)\gdbm.lib
+ 
+all : compile link
+
+compile:
+        $(COMPILE) gdbmjava.c
+
+link:
+        $(CC) -MD -LD $(OBJECTS) -Fegdbmjava.dll
+
+install: 
+        copy gdbmjava.dll $(GSDL3HOME)\lib\jni
+
+clean:
+	del *$o 
Index: /main/tags/3.01/gsdl3/src/packages/javagdbm/winMake.bat
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/javagdbm/winMake.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/javagdbm/winMake.bat	(revision 21112)
@@ -0,0 +1,29 @@
+@echo off
+
+set MAKE=nmake
+set MAKE_OPTIONS=/f
+
+if ""%1"" == """" goto all
+if ""%1"" == ""compile"" goto go
+if ""%1"" == ""install"" goto go
+if ""%1"" == ""clean"" goto go
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:go
+	if ""%2"" == ""javaonly"" goto java
+
+:all
+	cd jni
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+:java
+	cd java
+	call winMake.bat %1
+	cd ..
+	goto done
+
+:done
Index: /main/tags/3.01/gsdl3/src/packages/mg/COPYING
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/COPYING	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/gsdl3/src/packages/mg/INSTALL
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/INSTALL	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/INSTALL	(revision 21112)
@@ -0,0 +1,176 @@
+Basic Installation
+==================
+
+   These are generic installation instructions.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, a file
+`config.cache' that saves the results of its tests to speed up
+reconfiguring, and a file `config.log' containing compiler output
+(useful mainly for debugging `configure').
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If at some point `config.cache'
+contains results you don't want to keep, you may remove or edit it.
+
+   The file `configure.in' is used to create `configure' by a program
+called `autoconf'.  You only need `configure.in' if you want to change
+it or regenerate `configure' using a newer version of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.  If you're
+     using `csh' on an old version of System V, you might need to type
+     `sh ./configure' instead to prevent `csh' from trying to execute
+     `configure' itself.
+
+     Running `configure' takes awhile.  While running, it prints some
+     messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+   Some systems require unusual options for compilation or linking that
+the `configure' script does not know about.  You can give `configure'
+initial values for variables by setting them in the environment.  Using
+a Bourne-compatible shell, you can do that on the command line like
+this:
+     CC=c89 CFLAGS=-O2 LIBS=-lposix ./configure
+
+Or on systems that have the `env' program, you can do it like this:
+     env CPPFLAGS=-I/usr/local/include LDFLAGS=-s ./configure
+
+Compiling For Multiple Architectures
+====================================
+
+   You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you must use a version of `make' that
+supports the `VPATH' variable, such as GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   If you have to use a `make' that does not supports the `VPATH'
+variable, you have to compile the package for one architecture at a time
+in the source code directory.  After you have installed the package for
+one architecture, use `make distclean' before reconfiguring for another
+architecture.
+
+Installation Names
+==================
+
+   By default, `make install' will install the package's files in
+`/usr/local/bin', `/usr/local/man', etc.  You can specify an
+installation prefix other than `/usr/local' by giving `configure' the
+option `--prefix=PATH'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+give `configure' the option `--exec-prefix=PATH', the package will use
+PATH as the prefix for installing programs and libraries.
+Documentation and other data files will still use the regular prefix.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+   Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+   There may be some features `configure' can not figure out
+automatically, but needs to determine by the type of host the package
+will run on.  Usually `configure' can figure that out, but if it prints
+a message saying it can not guess the host type, give it the
+`--host=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name with three fields:
+     CPU-COMPANY-SYSTEM
+
+See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the host type.
+
+   If you are building compiler tools for cross-compiling, you can also
+use the `--target=TYPE' option to select the type of system they will
+produce code for and the `--build=TYPE' option to select the type of
+system on which you are compiling the package.
+
+Sharing Defaults
+================
+
+   If you want to set default values for `configure' scripts to share,
+you can create a site shell script called `config.site' that gives
+default values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Operation Controls
+==================
+
+   `configure' recognizes the following options to control how it
+operates.
+
+`--cache-file=FILE'
+     Use and save the results of the tests in FILE instead of
+     `./config.cache'.  Set FILE to `/dev/null' to disable caching, for
+     debugging `configure'.
+
+`--help'
+     Print a summary of the options to `configure', and exit.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`--version'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`configure' also accepts some other, not widely useful, options.
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/INSTALL.mg
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/INSTALL.mg	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/INSTALL.mg	(revision 21112)
@@ -0,0 +1,38 @@
+                     BUILDING AND INSTALLATION OF MG
+                     ===============================
+ 
+This file explains how to build the mg system.
+ 
+(1) Follow the general GNU install instructions in the INSTALL file.
+    NOTE: this procedure has changed from version mg-1.1 which did
+    not use a GNU configure script.
+    e.g.
+      cd ~/mg-1.2
+      ./configure --prefix=`pwd`
+      make
+      make check
+      make install
+    Report portability problems to tes@kbs.citri.edu.au .
+ 
+(2) You will need to set the environment variable MGDATA which 
+    indicates where the mg data files will be created. You will also
+    need to create the directory you specify.
+ 
+    e.g.
+        setenv MGDATA ~/mgdata
+	mkdir ~/mgdata
+
+(4) You may also need to set the environment variable MGSAMPLE which
+    is used by mg_get and indicates where the sample data included
+    with this package is located.
+    If this is not specified it defaults to ``./SampleData''
+
+(4) Included with mg is an X11 interface to mgquery called xmg. 
+    Xmg is a ``wish'' script which uses the Tcl/Tk packages available
+    by anonymous ftp from allspice.berkeley.edu [128.32.150.27].
+ 
+(5) Another environment variable that you may wish to set is
+    MGIMAGEVIEWER, this variable sets the image viewer to be used
+    to display images. The image viewer must take the image from stdin.
+    If this is not specified it defaults to ``xv -''.
+ 
Index: /main/tags/3.01/gsdl3/src/packages/mg/MODIFICATIONS
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,1023 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Similarity variants
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  Can only use one type of similarity measure - the
+  standard cosine measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+  The standard measure can be broken up into 7 components.
+  The 7 components are  
+  Each one of these components has a number of alternatives.
+  The overall measure, S_qd, can also be altered.
+  Thus the particular similarity measure used can be specified
+  by an 8 dimensional vector.
+  What is desired is to be able to specify to mgquery an option
+  and a 8-digit string representing this vector (assuming that
+  any one alternative can have at most 9 (not using zero) variants).
+SOLUTION
+  The programs which had to be modified were:
+  (i) mgquery,
+  (ii) mg_weights_build.
+  The other mg programs in existence store the text, indexing info,
+  and the basic statistics such as N, n, ft, fdt.  
+  Other programs which had to be created were:
+  (i) mg_fmd_build,
+  (ii) mg_wt_build.
+  Mg_fmd_build will create the file to store the f_md statistic, 
+  where f_md is the largest (maximum) f_dt of any term in document, d.
+  Mg_wt_build will create the file to store the w_t primitive.
+  It only creates this for the w_t variants 6-9 which would require
+  extra passes of invf at query time if they were not stored here.
+  For details on similarity changes for mgquery and mg_weights_build,
+  please see the other modification entries.  
+FILES
+ *   mg_fmd_build.c
+ *   mg_wt_build.c
+ *   build_lib.c, build_lib.h
+*************************************************************
+TITLE
+  Similarity variants for mgquery
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mgquery" needs to be altered to allow modification of
+  the similarity measure. 
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+
+  Most of the similarity measures, Sqd, are of the 
+  form:   Aqd
+         -----
+          Bqd
+  where Bqd is an expression involving Wd and possibly Wq,
+  where Aqd is a sum over the common document/query terms
+  of w_qt and w_dt.
+  Building of Aqd
+  ===============
+  The calculation of Aqd is done in the file build_Aqd.c .
+  The functions for doing this used to be in the file invf_get.c .
+  Build_Aqd.c contains 4 different functions for building Aqd, each
+  of them building a different data structure:
+    (i) Array, (ii) Splay Tree, (iii) Hash Table, (iv) List Table.
+  Each of these routines seems to have been construction by modifications
+  to duplicated code. This is often the easiest way to construct variants
+  but is quite difficult to maintain consistency. 
+  As the aim of the exercise was to try out different sim. measures for
+  retrieval effectiveness, I only modified the code that constructed
+  an array. This routine was called "CosineDecode"; I changed it to
+  "build_Aqd_Array." This change reflects the fact that we are only 
+  calculating Aqd and this need not be used for the Cosine measure.
+  The other routines: "CosineDecodeSplay," "CosineDecodeHash," and
+  "CosineDecodeList" have been left unaltered - they need to be updated
+  in the future which would be best be done by abstracting out common code.
+  By the stage of building Aqd, the query terms have been looked up in
+  the inverted file dictionary and put into a list.
+  This list of common terms is traversed to lookup the corresponding
+  invf entries. Before the invf entry is processed, all query and term
+  relevant statistics and primitive quantities are calculated.
+  For example, fqt, ft, wt, rqt, wqt, Wq-partial-sum.
+  To save unnecessary calculations, there is a test for each value
+  to see whether it is needed e.g. "if (sim->variant_needs & NEEDS_wt) ... ".  
+  Aside: Variant Needs
+  --------------------
+  The idea behind the "variant_needs" field is to be able to have all
+  the code in the one place for each possible variant and this code would
+  get the information at the correct time/place only when it is needed.
+  The overhead is a "bit-and" and "test" for each component.
+  The important concern though, is that the "variant_needs" must be
+  accurate i.e. it should be carefully maintained.
+  Each possible need is stored as a bit position in a constant/macro
+  of the form "NEEDS_component" e.g. NEEDS_wt, NEEDS_nt 
+  More recently (Jan/95), I have found that the _need_ing of a component
+  may be relevant to a particular purpose, that is, it may be needed in
+  one situation and not another. This was the case for Wd. 
+  For rdt#6, Wd was needed, however, it might not be needed for Bqd, the
+  denominator of Sqd. So I changed from NEEDS_Wd to NEEDS_Wd_for_Sqd and
+  NEEDS_Wd_for_rdt. 
+  --------------------
+  To handle the different calculations of the variants, I wrote macros
+  based around a switch statement. All these macros are stored in the file
+  "similarity.h" . The point of doing this is to centralise most of the changes
+  and cut down on the number of files which have to be altered if a new
+  way of calculating a primitive is to be added.  
+  This is achieved by having a data record called, "Similarity_variant" whose
+  fields includes all the statistics and the similarity primitives.
+  So the standard procedure is to see if something is needed and if so, then
+  extract it from an mg file or calculate it using a macro - most of the
+  input and output to the macro is done via the "Similarity_variant" structure.
+  As well as calculating Aqd, it is also a convenient place to calculate Wq.
+  Previously, for Cosine measure, Wq was not needed because each Aqd was directly
+  divided by Wq and thus would not change the ordering. However, there are
+  some Sqd's which divide by a sum involving Wq and thus need Wq. 
+  Calculating Sqd and Ordering Documents
+  ====================================== 
+  The file query.ranked.c contains the code for calculating Sqd using the
+  approximate Wd in order to do the ranking of the documents.
+  The mg-1 file was cleaned up and modified slightly.
+  All the heap data structures and routines were taken out and placed in 
+  their own file, heap_weights.c/.h .
+  The major components/steps in the ranking process were abstracted into
+  macros and functions:
+    calc_MaxParas
+    insert_heap_entry
+    insert_greater_heap_entry
+    approx_guided_insert
+    fill_initial_heap
+    add_heap_remainders
+    change_heap_to_exact
+    add_remainder_exact_weights
+    Make_Exact_Root
+    build_doc_list
+  Aside: Zeroing of an Aqd Element
+  -------------------------
+  One interesting change concerned the use of zeroing out an Aqd element
+  so as to mark it as being used in the heap.
+  In the heap, the approx. Sqd is stored based on Aqd/Wd-approx.
+  Later when Aqd is required, it can be extracted from the approx. Sqd
+  by muliplication of Wd-approx i.e. Aqd = Sqd-approx * Wd-approx.
+  This, however, is not always possible for the various Sqd variants.
+  So, instead of zeroing out Aqd, I decided just to make it negative.
+  -------------------------
+  In some Sqd cases, Wd and Wd-approx. are not required. In which case,
+  step 3 which calls on "change_heap_to_exact", "Heap_Build" and
+  "add_remainder_exact_weights" is not required.
+ 
+FILES
+ *    build_Aqd.c
+ *    query.ranked.c
+ *    similarity.c/.h (in libmg)
+ *    heap_weights.c/.h
+ *    backend.c/.h
+*************************************************************
+TITLE
+  Similarity variants for mg_weights_build
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mg_weights_build" needs to be altered to allow modification of
+  the similarity measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+  The weight files which are generated for a particular similarity
+  measure have their names extended by a suffix.
+  In the case of Wd#1 no weights are generated.
+  In the case of the standard cosine weights, Wd#2, a 3 letter suffix
+  is used to represent .
+  In the case of the other Wd variants, a one letter suffix is used
+  to represent which Wd variant it is.
+  In each case, the variant input (e.g. -q 22222222) should be the whole
+  similarity variant string and the relevant fields will be extracted out.
+  This is done for consistency in code and interface. 
+  The code is fairly similar to the original.
+  A dependency check has been added so that the dates of files and the
+  type of needed files is verified before building.
+  The dependencies include, invf dictionary, invf index, invf, fmd and wt files.
+  The major change here, is the possible use of fmd and/or wt files.
+  Later when I was having to write mg_fmd_build and mg_wt_build,
+  I decided to abstract out some macros, namely:
+    Get_ft, Get_ft_Ft, loop_invf_entry,
+  which were put into "build_lib.c" .
+  "loop_invf_entry" takes a function/macro name as a parameter and applies
+  it to the sim record (with field fdt set), current doc number and modifies
+  the return value. 
+FILES
+ *   mg_weights_build.c
+ *   build_lib.c
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  Boolean tree optimisations
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  (i) tes@kbs.citri.edu.au - 28/Sep/94
+  (ii) tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  (i) tes@kbs.citri.edu.au - 18/Oct/94
+  (ii) tes@kbs.citri.edu.au - 21/Dec/94
+CLAIM
+  The initial prompt for investigating the optimisation of
+  boolean queries is noted in the mg-1 mod14.txt. 
+  The code for optimising seemed to have a number of faults.
+PROBLEM
+  Boolean optimisation was unreliable.
+SOLUTION
+  Initially (in case (i) above, see mg-1/mod14.txt), I rewrote 
+  all the boolean tree and optimising code. I converted the boolean
+  expression into DNF. I did this after reading some notes about 
+  the steps involved in optimisation and they suggested standardising
+  in some normal form. I thought that DNF would be appropriate so that
+  all the terms are converted to be part of "and" expressions and be
+  evaluated quickly using skipping.
+  This, however, can suffer quite badly if the distributive law is
+  applied to often and the query expands in size. If there was
+  some sort of cacheing of invf entries, then it might not be so
+  bad otherwise there is quite an overhead on reading the same
+  invf entry more than once.
+
+  As it happens, CNF queries are reasonably common, where the user
+  queries with a conjunction of disjunctions of similes:
+    e.g. (car | automobile | vehicle) & (fast | quick | speedy)
+  This sort of CNF query expands a hell of a lot !
+  So after speaking with Justin who wanted to benchmark Atlas with Mg on
+  these sort of queries, I looked up the MG book for other ideas.
+
+  The method that I implemented was the following:
+
+  -----------------------------------------------------------------
+  Steps of tree modifications:
+    Gets literals by pushing the nots down, detecting T/F at leaves
+    and collapsing the tree by detecting  'and' of 'and's and 'or' of
+    'or's.
+    Next it looks at the or nodes and if all the children are terms
+    then it marks the or-node as such.
+    Finally, the or-term-nodes are sorted by using the sum of their
+    ft's for comparison.
+  Steps at query evaluation:
+    If it comes across an 'and' of 'or-terms' then the evaluation is
+    done noting the distributive law.
+    I.e. a & (b | c | d) = (a & b) | (a & c) | (a & d)
+    Assuming 'a' is the c-set of documents.
+    All of 'a' is tested against 'b' and matching ones are marked.
+    Next, all the unmarked members of 'a' are tested against 'c'.
+    Likewise for 'd'.
+    Now all the marked members of our c-set are kept.
+    When we do the testing, we can use the skipping in the invf entries.
+  -----------------------------------------------------------------
+
+
+  After doing this, I added the choice of which type of optimisation
+  the user wanted by adding query-environment-variable, "optimise_type".
+  Type 0 = no parse tree modification. 
+  Type 1 = Or-term recognition and CNF query evaluation optimisation. 
+  Type 2 = Put into DNF form. [generally not recommended]
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+ *   query_env.c
+ *   invf_get.c [GetDocsOp]
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  TREC DocNo file   
+APPLICATION
+  mg-2
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 26/May/95
+FIX
+  tes@kbs.citri.edu.au - 26/May/95
+CLAIM
+  MG has problems dealing with trec docnos for trec disk 3.
+PROBLEM
+  Trec DocNos file didn't have a wide enough field
+  to handle disk 3. 
+SOLUTION
+  Allow different width fields for file.
+  It is still fixed width but a number in the header
+  says how wide the field is.
+FILES
+ *   passes/mg.special.c
+ *   query/mgquery.c
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+   Autoconfiguring mg-1  
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  many people - 94/95
+FIX
+  tes@kbs.citri.edu.au - Aug/95
+CLAIM
+  Portability is limited by setting up c-macros just for particular
+  machines and operating systems.
+  People had to make changes for HP, Next, Linux, Dec Alpha, ...
+PROBLEM
+  Porting was only targetting at the machines that the author had
+  access to.
+SOLUTION
+  Use GNU's autoconfigure program.
+  This allows checking of the systems features/characteristics.
+  It also allows some checking for specific machines/OS - although
+  I have not utilised this option.
+  I used GNU's tar-1.11.8 as an example to base my changes on.
+  I also used autoscan to generate the initial "configure.in".
+  The "Makefile.in"s were done very similarly to GNU tar's.
+  The "config.h" and "sysfuncs.h" files were scrapped and 
+  rewritten. The new "config.h" file is generated by the configure
+  script - it contains all the #define's for the system features.
+  The "sysfuncs.h" file wraps up a number of system headers. 
+  For example, some systems use , while some use ;
+  which one is included is decided in "sysfuncs.h".
+  I have also used GNU tar's use of ansi2knr in its Makefiles.
+  This should hopefully allow the package to work on a system with
+  only a K&R C compiler.
+  However, there are probably problems with what I have done.
+  I am concerned about <stdarg.h> for example.
+  I also noticed that "ansi2knr" require function definitions as
+  the GNU coding style recommends ie. with function name the first
+  string on the line. This prompted me to run all the package's code
+  thru GNU's indent.
+  Setting up the configure changes is difficult. It really seems
+  necessary to try the package out on many target machines so one
+  can know what is necessary.
+  A simple check target for the main Makefile has been written.
+  It is used to see if the installation is working - it does
+  not test much of the functionality of mg.
+  It does cmp's on data files and diff's on query/result files.
+FILES
+  Most of the files in the distribution.
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/gsdl3/src/packages/mg/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,157 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib src/text src/images src/scripts java/org/greenstone/mg jni
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall: config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+check: all
+	echo making $@ in ./test directory
+	(cd ./test && $(MAKE) all) || exit 1
+	./test/testmg `pwd`
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+distclean: distclean-recursive distclean-local
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm -f config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile stamp-h config.status config.h config.log
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in sysfuncs.h
+
+dist: $(DISTFILES)
+#-----------------------------------------------------------------
+	#create distribution directory
+	-rm -rf $(distdir)
+	mkdir $(distdir)
+#
+	#link in local distribution files
+	ln $(DISTFILES) $(distdir)
+#
+	#link in SampleData
+	mkdir $(distdir)/SampleData
+	(cd SampleData; $(MAKE) "distdir=../$(distdir)/SampleData" dist)
+	# link in davinci stuff
+	mkdir $(distdir)/SampleData/davinci
+	(cd SampleData/davinci; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/davinci" dist)
+	# link in bible stuff
+	mkdir $(distdir)/SampleData/bible
+	(cd SampleData/bible; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/bible" dist)
+#
+	#link in tests
+	mkdir $(distdir)/test
+	(cd test; $(MAKE) "distdir=../$(distdir)/test" dist)
+#
+	#link in docs directory
+	mkdir $(distdir)/docs
+	(cd docs; $(MAKE) "distdir=../$(distdir)/docs" dist)
+#
+	#link in lib directory
+	mkdir $(distdir)/lib
+	(cd lib; $(MAKE) "distdir=../$(distdir)/lib" dist)
+#
+	#link in src distribution files
+	mkdir $(distdir)/src
+	for dir in src/text src/images src/scripts; do \
+	  echo makeing $@ in $$dir ; \
+	  mkdir $(distdir)/$$dir; \
+	  (cd $$dir; $(MAKE) "distdir=../../$(distdir)/$$dir" $@) ; \
+	done
+#
+	# archive and compress
+	tar -chf - $(distdir) | gzip -9 > $(distname).tar.gz
+	tar -chf - $(distdir) | compress > $(distname).tar.Z
+#
+	# cleanup
+	rm -rf $(distdir)
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl3/src/packages/mg/README
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/README	(revision 21112)
@@ -0,0 +1,156 @@
+                 MG INFORMATION RETRIEVAL SYSTEM
+                 ===============================
+
+The MG system is a suite of programs for compressing and indexing text
+and images. Most of the functionality implemented in the suite is as
+described in the book ``Managing Gigabytes:  Compressing and Indexing
+Documents and Images'', I.H. Witten, A.  Moffat, and T.C. Bell; Van
+Nostrand Reinhold, New York, 1994, ISBN 0-442-01863-0; US $54.95; call
+1 (800) 544-0550 to order.
+
+These features include:
+
+-- text compression using a Huffman-coded semi-static word-based scheme
+-- two-level context-based compression of bi-level images
+-- FELICS lossless compression of gray-scale images
+-- combined lossy/lossless compression for textual images
+-- indexing algorithms for large volumes of text in limited main memory
+-- index compression
+-- a retrieval system that processes Boolean and ranked queries
+-- an X windows interface to the retrieval system
+
+As one example, a collection of 2 Gb of text (1,700,000 documents) can
+be indexed (on a SPARC 10 Model 512) in about four hours and compressed
+in a further four hours to make a database that in total occupies less
+than 800 Mb, or 40% of the original size. This includes a full index to
+every word and number in the original text. Boolean queries such as
+``managing AND gigabytes'' run in a few seconds, and ranked queries of
+30--50 terms are evaluated in 10--30 seconds.
+
+Details of these methods and further performance results appear in the
+MG book.
+
+The MG system comes with ABSOLUTELY NO WARRANTY; for details see the
+file COPYING.
+
+Instructions on how to build and install mg are in the file INSTALL.
+
+
+** CHANGES FROM BOOK
+
+For copyright reasons the stemmer used in this distribution of MG is
+not the same as the one illustrated in Figure 3.8 on page 108 of the MG
+book.  This means that the numbers generated by the command ``mgstat
+alice'' will not match those numbers in Figure A.1 on page 394.
+Another stemmer was initially written as a simple stopgap for version
+1.0.  That stemmer has been replaced by a stemmer based on the Lovin's
+stemming algorithm for mg-1.1.
+
+The output format of ``mgstat'' has changed since Figure A.1 (page 394)
+was prepared. The same information is displayed but formatted
+differently.
+
+** MG VERSIONS
+
+The current version is mg-1.2, September 1995. The changes from earlier 
+versions are listed in the file MODIFICATIONS. This can be accessed
+with mg by building a database using ``mgbuild mods'' and can also be
+accessed from the mg web page (see below). 
+
+The mg-1.2 extensions include:
+
+-- Source modifications for use of GNU's autoconf.
+
+The mg-1.1 extensions include:
+
+-- A new highlighting mode.
+   The output mode ``hilite'' will highlight the query terms in the
+   retrieved text documents. The variable ``hilite_style'' can be set
+   to ``bold'' or ``underline''. It works best with the pager
+   ``less''.  A .mgrc to use would include:
+	.set pager less
+	.set mode hilite
+	.set hilite_style bold
+
+-- A web site containing manual pages, documentation, and a
+   mgquery demo page (utilising cgi scripts). 
+   See: http://www.kbs.citri.edu.au/mg
+   One of these pages ``about_mg.html'' is included in this
+   distribution.
+
+-- A revised mg_get script which uses a .mg_getrc file to map
+   specific collection names to filter types.  (Modifications by Bruce
+   McKenzie). See mg_get.1 for more details.
+
+-- Code to perform merging of existing databases. This code
+   was created by Shane Hudson and is documented in the mgmerge.README
+   file found in the docs subdirectory.  This code is maintained by
+   Shane Hudson (shane@cosc.canterbury.ac.nz).
+
+-- Revised man pages, including some new entries (thanks to Nelson
+   Beebe). See mg.1, mgintro.1, mgintro++.1.
+
+-- A real (rather than toy) stemmer.
+
+** PORTABILITY
+
+Please refer to "README.port".
+
+** CREDITS
+ 
+The MG development is largely the result of research collaboration
+between:
+
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Ian Witten             <ihw@cs.waikato.ac.nz>
+        Alistair Moffat        <alistair@cs.mu.oz.au>
+        Justin Zobel           <jz@cs.rmit.oz.au>
+
+The bulk of the programming work has been carried out by:
+
+        Stuart Inglis          <singlis@cs.waikato.ac.nz>
+        Craig Nevill-Manning   <cgn@cs.waikato.ac.nz>
+        Neil Sharman           <neil@kbs.citri.edu.au>
+        Tim Shimmin            <tes@kbs.citri.edu.au>
+
+In addition to these, the following people have contributed to the
+development of the MG software:
+
+        Lachlan Andrew         <lha@kbs.citri.edu.au>
+        Gary Eddy              <gary@cs.mu.oz.au>
+        Hugh Emberson          <hugh@cosc.canterbury.ac.nz>
+        Kerry Guise            <kerry@cs.waikato.ac.nz>
+	Shane Hudson           <shane@cosc.canterbury.ac.nz>
+        Linh Huynh             <linh@kbs.citri.edu.au>
+        Bohdan S. Majewski     <bohdan@cs.uq.oz.au>
+        Bruce McKenzie         <bruce@cosc.canterbury.ac.nz>
+
+In addition to these, the following people have submitted bug reports
+and suggestions/fixes:
+
+        Rex Barzee             <rex@python.cs.byu.edu>
+        Tim A.H. Bell          <bhat@cs.mu.oz.au>
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Nelson Beebe           <beebe@math.utah.edu>
+	Rodney Brown           <RBrown@cmutual.com.au>
+        Rok Sosic              <R.Sosic@cit.gu.edu.au>
+        Carl Staelin           <staelin@cello.hpl.hp.com>
+
+Development of the MG system was supported by the Australian Research
+Council; the Universities of Melbourne, Waikato, Canterbury, and
+Calgary; RMIT; and the Collaborative Information Technology Research
+Institute (Melbourne).
+
+** BUG REPORTS
+
+Send bug reports to <alistair@cs.mu.oz.au> and <tes@kbs.citri.edu.au>.
+Back-traces from gdb are always welcome but not mandatory :-)
+
+** FURTHER READING
+
+A bibliography of MG related research work appears in the files
+MG.Bibliography.ps and MG.Bibliography.bib on the ftp site and
+is accessible through the Web page.
+
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/README.port
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/README.port	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/README.port	(revision 21112)
@@ -0,0 +1,125 @@
+P O R T A B I L I T Y
+=====================
+
+The main development of MG is done under Solaris on a Sun sparc.
+However, we would also like MG to be reasonably portable.
+Therefore, MG-1.2 now uses a configure stript generated by GNU's autoconf.
+Below, is a list of the machines that the s/ware has been ported to. 
+Most only need the configure script and some require manual modifications 
+mentioned in the "Problems" field.
+
+------------------------------------------------------------------------- 
+** Systems which have successfully been built.
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2 8 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2_DB 14 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.3_U1 1 sun4c
+Compiler:	gcc version 2.5.7
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.3 Generic_Patch sun4m sparc
+Compiler:	gcc version 2.4.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Silicon Graphics
+System:		IRIX 5.2 02282014 IP7
+Compiler:	gcc version 2.6.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.4 Generic_101945-27 sun4c sparc
+Compiler:	cc: SC3.0.1 13 Jul 1994
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:
+System:		SCO 3.2v4.1
+Compiler:	gcc 2.6.1
+Problems:	requires RANLIB=true, use of tcsh
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	HP
+System:		HP-UX A.09.04 D 9000/837 415303291 64-user license
+Compiler:	HP92453-01 A.09.73 HP C Compiler
+Problems:
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Data General
+System:		dgux 5.4R3.00 generic AViiON mc88110 
+Compiler:	gcc 2.4.5 (dg supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Pyramid MIPS R3000 box
+System:		
+Compiler:	cc (vendor supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	IBM RS6000
+System:		AIX 3.2
+Compiler:	AIX XL C Compiler/6000
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Linux 1.2.9
+System:		
+Compiler:	gcc 2.6.3
+Problems:	none
+
+-----------------------------------------------------------------------
+** Systems which have NOT successfully been built.
+
+Tester:		RBrown@cmutual.com.au, 
+		Lloyd.Parkes@vuw.ac.nz
+Date:		Oct, Nov, 1995
+Machine:	Dec Alpha OSF/1
+System:
+Compiler:	cc
+Problems: 	Many - see below.
+
+* Problems with Dec Alpha
+  - 64 bit length longs as opposed to 32 bit longs.
+  - Lloyd.Parkes@vuw.ac.nz says that there is some bit encoding problems 
+    but the decoding seems to work.
+  - RBrown@cmutual.com.au says that it will compile but will core dump
+    when  building the test database.
+  - tes.kbs.citri.edu.au says that he needs to look into the repercussions
+    of 64 bit longs, however, he does not have access to such a machine.
+		
+-----------------------------------------------------------------------
+** Other systems. 
+
+Systems other than the above may have installation problems.
+If this is the case, please send me (tes@kbs.citri.edu.au) a log of the output
+and I will modify the configuration files in an effort to fix the problems.
+
+-----------------------------------------------------------------------
Index: /main/tags/3.01/gsdl3/src/packages/mg/SampleData/MODIFICATIONS
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/SampleData/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/SampleData/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,664 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/gsdl3/src/packages/mg/SampleData/Makefile
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/SampleData/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/SampleData/Makefile	(revision 21112)
@@ -0,0 +1,4 @@
+DISTFILES = Makefile alice13a.txt.Z MODIFICATIONS 
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/gsdl3/src/packages/mg/SampleData/alice13a.txt.Z
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/SampleData/alice13a.txt.Z	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/SampleData/alice13a.txt.Z	(revision 21112)
@@ -0,0 +1,2293 @@
+ 
+
+EÊ%EPq€
+"NrD'D@©âdÈERL)DÊ$ Š )Yä
+K
+`I³fMcRÄÈ#FÌ
+ Èy£ŠÌ:
+ëÐ)ãFL9g@ 	3D0dÊ#GN7gÊŽaJ§ê4tÐh¬ãfÌWeæá:F­7r@(H«L9tä€£4Í7 ÞYÛö­žsåÔQÑÀÓÁd°W9y@,ÅCÇE@'o¶9\&ô_sÞ°Ñ¶
+bB
+%z9éÒŠOÏ¶È
+2ÃÐ1ìŠêÔtÞè	TW1\Ã0v(¶ÜáÊ¥«(O
+y8dÀ
+2µ
+0iÎža"	6Ä_
+ Žà
+|òhŒöÚSÉQÆ
+\Á_
+oÔaU¡Áñ`  ¢tqÔVnäqOYžUUl
+U[ÇPføo F[uteÛpzÁDUVH¡üñegaÜbÌår
+XUsÔ1Öm¹¡Ñ
+ÜR#ªöOUµ%X^\þUb[Zý( !¡rÌU
+×qv
+FÍ¥eXvÔPnh&Z€Í1
+£e[JØ'ø}	[Yå5Æpä!Øh UŠ^7öØ_iäõãRgFG
+üñ%á]¹§ÆQG­¶Ú}sTÅW"v«tðwšnÚUkhU×WaÝVæyœqb§ÂF\Ã]¡
+$Uæ<Þâb_mkgÝV%lnÁ_l Üáí[§
+yâ³i^¶¡š²}Ÿ1V»Z¡ªVÂU\Á_¹rn(§Ã.F®¢çj:š_ZXzŠõÚUn°)ÇpIší
+²qÛlÝp_
+cX^5ó9GcmóS/n%ÆvžšF
+ þ8udUã
+iŽšyŠ\Ò¥QeTiuÕÙ+š³ÐñÆÕYoE6Ëß v€¡XøÝt
+!
+@
+[ÇvŽA
+b 'L:Îä_.ù.
+¶iš NyífiH|±UÔG(ÅSPÝ>èw!öØx©5Õ
+
+ä9ažq§N}s¿Ÿ§VUDµ
+dïú|ÅýøÆ_ŠýôS6ÕI
+QÄ	SXDQQEHõ'áWxF&Âd:kgÂ	œf\ÊÇÈŸ
+H(à&@Ä	IšB@`*0aR!e*$A 
+f°>
+ÒCÐ	Ob°
+Ç@<	Ž¡XÃ!¶ï$AÂC€ÐÔ°FÔ¹Ãì-sthd£
+B1TÀO01ŒFN9Ã"
+)ß
+Ír$tE
+o9m
+ì`c'ªé%&bÌR±ã5ðç+u\ÌT
+×F
+Ù¬eBãcÆ9æ,ŠrÀ¡
+eøämn7­\í
+k
+ÕÈªQš*ò
+X¬¥!
+Â=©:).¬Þ÷v·4¡
+?`@dÁŠâ#gñkZYvNiÈè%-bRÈš­á,Ø¬¥ºòKAê0ÃÔa~`M@d
+©Ñä«R=²êGQÑÓ
+§hÝ42ØÇh$ÏÕAhüQ°¶'
+
+CÛ
+ÍJãÄ6	eh	:š
+)àÏÊà«kò	pë§Å-|Ö
+øC
+Zð-=í `z^ÒÂPÌÔ¡JZQÀ¢ÞÇšxE·ç\ZEþe]s)Ó¥¿1
+
+`Ð€)\QDàTº
+Éš<Ó+è\pØ¥}òÔ+
+÷Èš
+RÈ$t9¬aoi3w ŒŒøå6Ü¬ä^DVÝu"zmÃäÔ!€,ªcjXÄzU
+X
+DË€ŽÈ
+
+f@æÔ®  «=Nd^ÛDMJÉ8NyÉB`M `€TåS8µÉ4Ë~o+fX
+DšÚ21cCèH±=ŠE9°áS6[UÝ0ôM@
+Xn]9÷ZånÙòÛÂ¢2Vmn§ I)UÀµëÙ¯¶·
+
+ð%XA ÂI@ð0Øý¡Bý\)(ø		6PP0%
+
+ØÞÊda÷HÝ
+bÐ0ÝuÕq?Äé&lâŽq¥Feµ\f+€Tò$¯>T=MÄ×EJAqÉŠãê·
+ÀÒ\Ž¢
+¯¬Ús]IµC!;J9-ðÐºµãAqY9!mVÙñ¡X£Ö>É0áãUvµÖ­
+¹Y
+Uèº\åÚ2Ä·/K¶¬¬*[ë×K±)?×CpÐ
+rZŠ
+Ž
+e¯þmÆK*ªUEdË[Ä|¥GF¶YÌÔ2Å+±TN.ÕPÜdªQ2šõLÃ=Ûš1{5M§r tAçZŠ7&ŠaÒV}A>Y¢æbÁf
+3ü7KD£²hØÂ&ã,46QËlBÉÄø¢MØ²âGoÆ«*œ
+äWÉÕVÃ6=2{k3`P5ÖšÀîŽqJ÷ì2R©|ZBoëÒ{×è¢Y	å¶CmÊŒÎÔ|šqæ
+rPÂapiãE?vÑ9ÄèXåÏ¹ùc¬Í2$
+Íe)MÐÏÚãe*­Öæ=²ûÆá-Õ2±;×ç0u³a7SnG©¶°Vp¬€8
+ðž2¹ÍZ®»ËŠÑ©
+èÐÒGHa6êW)ÀÒùª+aÛMq¥wŒ®r°yY£Ñzqd7ÅÅÌí
+Æ`ÚenSgéÂáj\in©Ùy²!ksÈ/ìšäÕSüiöº
+
+¶$âqú2pRaQ=OÁRïäyt®vš¡³f
+ŽX3Ÿ*
+õU+(³äBÂF
+S,0<A
+9Pòó»{u©«¹ fY÷Ð­
+."ðBT,pØã
+zXº>)
+EICiUt|y±PÕY²TåÓF'kWCzF08me·,µ€#$Ì×xÞâQ '¢WtA5sÆÁ,xWr`à.ÅÁ[{²]ÛTF'³YmqÈº6U#fà^4b^üza[sptÅXQqY^R%
+ÇOjvŽµEö"^Ñ7|'ð1ÍSJQ]3Žuh gBxSBã' dxHÈtžc
+PY#-!&ø<QEÑi
+ØPÐä(@¡,÷¡n¶rPeà)P$?ASÄ÷fSÔ.ACyZÑãEW5eòLAŽB^!uÅ&5úÂ)ýq}ã@ÇV{òË1ìôzøA^16Ã¡ªh\¹ET.
+nâzlÒYUdF$²R{È'ÃÑ@H1pRJÓohSáô4"f6²xMtp-ÐÖÀPÂ2eükRásp#gJFO!%¹)Urà#+F5Ù±(ÇuAbl0yÃ¥æõhþ§
+x¥WÏñÄp;Åh;g..¥D5ôb.äL-Á|€5­DE4(³
+s}ÎAbIå×SIeò4®a\O`Ðfå\¥
+'°%WSKUŽ*Ü3wbFíx-w­èJÇâKb#TyÁ2a£I@Zñ
+"ÓŸe"­5uH!ËAóP`¡b£e°£~x­±m)+Ê4"7CÈ!(!A"VPiôÑn0
+Åu»vÙå^¶žWùÁ]
+AQGUth
+æ¢)ävYh
+S-e2wm×^oRw<÷IšXà(%©TgŽ%z÷Æ\Ž
+)A"ip6AhR?²íøqKÊ±Gg*è).Q?NÂ|6ùøÔ+
+R{±^/ù\¥J!"]îU«wxR °-p-BHÆÑsVTJ
+óvI_Qó§ÂáÎ%,+yÃ)å"M'5ü¡ñ	­c/Z0 "bÇè³œP"VPáb×ÒUr±Qé4
+Ò-íØmC°?VYà?A#crÇqCQ¬ÂPø!ŽEØB&]ø¥Ehøihñ8 A bªož2ÿGtXjl{¶ÿè
+hfs°T`A[šQ¢t §
+Ðk
+
+ú§óIr0³W[Sñ1Vºò£iÌ8RâíÄ>x.(ªŽÕsåHoØ(kIo00©a±%O%T€.õÓDð#azj65%Yiö-)ÚŽ§18µ§.è[œZ2ø|â¥ŠÅš1dyzÊWb<z±GO39Ÿ¶èp®2-°È«Ÿªm­Å¥q¡
+iÇO®WgbxªR!¹d6Ô['-8èÂŠ~
+ ,fC\Ø:Ó€
+ÉLQ	n5²Ðt!Ù²¡Ã(µ
+}¢
+Uá~ž¡²ÛIr kz®g àl
+à%es.§g9kYbWwqW¢®v
+2¯°±Sãcz^­+]ØQ?À«<ª²bµ*Û
+%kîê\R{cW[µHAµœbJÑw­(2Ç('
+`,q(J(d ·Æ*¯Êah+R ÒÃ!Ø(ÛØoái&zpÐyžiÈXgÀ8sáWPögžJlÊ€O©=Óú«GŽ
+Ãl97 šqü±ºhÐ+b
+€8vJÓÂÂyÁ²wµœÈ_úvUz^¯%œöñZ
+WPn5
+%ÚÁY~šq¡'bszÛî5WXó%g©pž!",V­Ë'[cÉrT»ERi"pGûÇ'×¿/qîS}÷Ÿ:ÕPsõj%#m4}žŸZ5_Zs5Ÿ=÷üq¿ Ñ-ò¯4âH+kñR&¹¥VtÀ8ÍYÛvk2ørþys
+¹
+CieÂY=6@["r{lB^u£3ª%¯¡ŽÇyñÙ-àÓÂ·	*\
+«Ú[_ó!ŠT5Jy7UGÅš	Å
+
+¡%ì¹äÁaA©[SæÜ§c5Z<b³4bö2©l U;W³G&Uv*ö!Sm|ÃP9;	qõ.1
+x±%s¥­1·e,ry1é^12f¶_IÇtésN±uýråqÌgO¬wGUEBå+œòÂmŒôJb:IqÂ'Zd`oüTEfó7P0ëR¯¹ÈëFÜfºjªîóœ·œZÑœI{Éuz2h !eÐP.¢X.â^W!6ò³qåÉ%GÃU¡ª')-HA)f5G¡ñ
+B³uÝ¢gÍ#bž¶±9>Ùo"2±Üµ}2B1
+XAåÊàÓ|t[u(d¯E(§»ŒfÃ§[Pá"²XZÑÑ'œwsp·Ç[Ñ^)jhVèãjS}ÐÏ2
+RŒCENc­(&Ë3ÏPSÏŒW#7ÇAR"z²d>gÃgF9eçqü\<§ù¬iÖ
+>º¶u^Ã"E¶,dÇm§²Ao  Ô¥%Ö>ýcF°Û~ØVÂ]"#tj¯õÎöÕeÇ'2bwaq(7"¥N  ìþžD
+È{âTmÑØôâž¥bÙwv&È
+Ûy²
+#Y3{³¡·uÇ{\°7RXx2rÇÎÖf÷Ù\JÄD5	WKÍ3/žVN!zB
+ÿ©)7×&)Mku*²Qq$çÅÞÚ{ÕeÂál{ô/¥ÜÅÚÖYL"fà÷¢L«àá³ys¥¬ÂÖtF3ØžLVý¬¹uÐ
+
+MÙNHÁŽŽm2
+«%Æö¯€BX¡ÊÆbi
+«
+éRŽ%FžãX¢Šyž¡
+e/:¬¹ÓÖ[F"«!nKÎSY- íÊ"-ÄuÊ>©TÅºÕhÑ[yÁS³FØoEQö²Xø$Åâ,Çg,êheã!Í2Qr«ü^bqÇ¹å$,nSøQTžXEø
+X2
+«^³
+cfpÂ%Ð-,%*ËÈ'*]Øû©ª*Žbf&Õ¡Á³âFMÔ²žsJéÆRlMÄA5äsPäsð'ê®÷ÚûÙŠÔ#M3;ßÔ ¡fT éhó{K1)qìá¢%lþf!šLhÍPl²éBÓž¡ìõ×÷ášºµ}"¥= U!µRà?Öó£N
+aåh7r+ÉÂìiâi>MRÊ|ïyG#îuQ*Wi%µAVcÁí\iÄk5éA%¿5ìPV÷ºf¢Ê+±zòœÓ¢9š³ø1§#
+vÖTPOó°ÊQ ŽL$µGfì&å," 
+)äH'_;GË·Kïõ=þÈ>­âÑAvRÙfuq6+rcž
+¬ç~[ÉÇõLeb°F5l¡ëT!§x9[Fáï­zËÔ`L6
+®m ®ÎèEëÀÔµ¢)d0Eæs%ÂàÙû·Tü¹W¶mH1€|6ž$!,ý¥Î9ÙÏ5ÔLXìVy¥/,Ò
++#h`A5)[¡díÚ£
+ôbØ.""\SÕ\¬£5ÝIÊcp©IÆäb
+8ÝiAÊ
+îtX}µÍÄHôÁí:Ë«"Ó.×8
+í÷.Ú3&°Êå·d
+¿©üÕß¯q5Ù?O(ú
+±ýÝçýû¡æÅª!VnO²Ø
+Ðí±È0dI>#²Kêñ`¿°ÒÒÔâ4­€XJÀÉâÇx
+Q.C]x
+ ¡]}þ ]Ôâc¥Åq¶WÞÛ»&A¢£õ
+J,C9Êå±=
+JÍ!1`ä«("PRÉ* &DA0gB
+W
+JÁT¶©`Ž_
+rÁÉ`Ô^0
+Áw¶*vèQkhO\ÐÍC,2(Â	!É"tG%û qãë§Je3«iù5áO@	hlóË^899ØÚb
+«	ò(²d"LUú
+,ªõgE\ 1{h·É?ŠúT^yÄi»D7ª×)ùm²í}9(!Â02`gÇ°Í
+]43s0öüß!*Õb\
+V}6ýÀ+H°£±Íöìž
+°"ë±7óáJJ«o§(ËU¶zÁíîà·[?µ8ÄÀŽ¢áùh#ì
+D7Rx]àÝì]5
+$ÕUË-°1
+üNÆc£Fj
+\ÊSEDàÂ)+`×s¡Fd#V
+*ÁO*ÎîYŸD7øR:[$ËPË÷"ÌA5¢pÑÀÈT*Dk#«ÁS 9Š`ÄN_0
+ç#
+ž{O [á
+
+»üŠrX(
+âEÉ ×Â_Õ[r(e¢©X3£èßhÓc~ÊnëH:Zšá6
+Ó)Æyx¡é
+T)	h@
+Ç
+b§nèâq·Á	:²Q%^Ý\
+h¡±FD÷\ä
+@)Õ.4,gøÄB
+)G=Ä=â@Äo
+/S%6AÓ
+H|ÈÓA5?6ÔlÎurTyàøy±­C×:É
+I'ã<Œ#ï€ÙOZ!+æWEËûá'ÊDÚ=64d
+pÑ­=á'òKñXIÆþ¢wHZr£<€jä*B Ø8 V(
+kEX$ôM @ê'nÀwž °ÆÂÎ~Q¢oxÇÅpQ:y!
+ÃLÀDq.ÊßiÈZÜ
+â`.Z)Ql­ %eNÅF3ñðÞ¹ð=¢ô
+p1Ú»ZC\£1X<à
+*ZW£=ì1"
+Ll#'Ðd`HÓÈ¢DÔ8w}b(è-%s
+ÈUÒŠ9`»
+yÉÑá5BsäÄ]pQáFS#OÐ0eà8	D0`²ðNtŒ1,
+~8RÈ]ã·C&eBKþcÁ.É²ºn
+¡â+ÊÀÆÝlñ7>¢eH.í0P€ùæÀ'9
+
+VŽÃ0Ù<É{ÄDÎÁáœºáe
+ìb$¬âRJ
+ÞJ@¥±©69Æû¢1á5ç¶¥¬nÖôBáJ3)Þ
+(°ô5éskïßŒmôF	º.!øè
+ô»ÖÇ`YÇ€æmGn5n)})_š+ÆWd	eÇœ
+?+Zk@ØòÅiK7ñœhj·å)l§ðà'\^0
+EŠÁÓòkKÃÙÜÂ§xÝ`2YE$c
+¥,¹íGüa
+s Rþ5f2Ü[Ä"­k;~<mf1Z6h@¢ÉY²Õ*°1&4ÌÕe/Š§ ñà(D¬(ËÄpA­Eè
+Üå{ÊEÒ1J 1ðšDô'¢RtñÐŽÚ€ÌZã#­ð
+²¡É$Þ£}ìGÅ¡­<ÌD^èóq?ôD[ª7§áòõS
+Ó	
+C)
+%€ûQáZx90ÇÆÜhABi(ÂØ6àK²Ç!àeF_òQôâl (nI
+Ì6ÞÄáz+25Q0ð!I1G0K(Á £[è³A0:Kç4iP}ŽÓ©Å`ëDªÓuÁ×:g§ítbÄ2œ30RýaUÞyª@Hã
+HwòÀÀž3áPœ(	Mž5ñ:Ê
+¡ÈEvÐß!ú`>3(À#Zp
+±÷Ò'áÎ#é
+"lÀ/ª^Ì¢åæNR-Áì[V·Bž1Då
+:.¯¶
+ T ÌÎÑ1h¡ÈÅÌVbnÚY2OÇGÄå7édnãÏ²Ô;ÂÂ,bzK
+oÝ
+TÍ@
+M5Î*tÁF
+ØñJâÅ_jìñ1àÀUa+#Æ
+ÆLÓÂ
+Ê®0²Öþ7â
+$/päÖ±áÈŠñAsÖx?,JY€Ú9ÃâÊóG6©$û2ãXÇHjX Ò^h}_°l¥Z²ã(BTtòYp;7âÔ pT
+&]Ã 0³â9ÿdä°Ôð
+h6²
+Ýš
+0i
+Ôb×MØû°øÚ)i{Ô
+ÐðI®¥øTc€!1ËÒ{ZÔ<òåÊgUŠØË0b[ÄÑçé®UY:©æ	6I"2yE/gè&-Ôü`,H&Ö°±ø
+¥-ðy$ÈvÅõ8
+fôTrÚ§Mxñc~Ô{ 0 2L(9
+H)	®	6%â	JÉÓ F" )0P§@D-%ÀSùA?8ÃÀ/TÅz%U°
+h--¬ŠÍê ,
+J ésbÆä@8IÒ3Z³þ$c
+zbãÈ¥?¢>{
+l€q	°Ñá&¡¥DéìÜKÀØ¢F£lJí¶ç
+Ã¢e,f/ã©»É{x[ÂMX>
+9Š,è«zÖ± ³DWXVØëôRFÉP âÖ©3_Ó¬PEÊ©³¬,+>µ¡|NáLLÑ÷hlÄäžÁÍõHKuÜ
+IRelaÒŠÜšTÕô
+-iéí
+i2ñ¢,+Ö
+ãà±Øx"uõmâ%!Á@³QA2­6EîDºòm}è=0±qÑªLb3¡åjŽóYkFñož(Á ÅeábüÁBÑ>éÀz[pD[È8»~žÞÈg'°*L#îãTéð
+I¿p¢£4K&2X1â
+å¬!Sµ'ò-Caôj3R
+Ù
+w^+s
+#wÒø!B}
+)H*ö3
+%Ð9,ö&pÏ§)-9M,
+&Ý`>03Åtg y{rqSR4èT®bn)Ž¡Ç$Ki»xuì
+
+I#f¬ð>µ¡|ðê\ªÈUlÁ$Ùð¢0
+
+¿sv`Ÿ¶Ô7º
+EÛ#]ŠšD+{Ž
+aVpý«BýK>fR%l |¡¡Ã&öåYå/U$Ââ±§ùŒ€87ëvsÑ+Ê·Ýô
+1%÷äBB!rà;O=ëk
+Ç2
+@§NéÀÁpfMjÞ.Ø.ÆPTëZSŒ±ŽÙÊ[í
+«÷H¹©"ã-®Ús_Clø€f&s<)OïË¹ útjÚ0cÇ/ëcÖËÌ[
+
+ø:i)ITgÀI­È`fK@ènzkÀÆº¢%E
+)¶5 $ÂŠ€\Ð-wH³t48J#à3V8.ûÆ¶Î·š}ÆÒ€heÕ*è%
+^ÝqÌ$ç4&Bb×rÄ 8gšyd!tO5ð øÄU
+m€Ó°År
+=ZPèTv\3
+¹ë: @o/á\ÈPjÈ©ó.Ùhhè]&/ÁTÔ§øEF
+Ðz öDÈ«
+;8'éž3o1%»
+Å)nR®òË?ô4ŸÐ§i
+ŽIVWúR"¡Y$ý]2¶PºÆéj©LT"R¡
+
+
+0¥
+6\läËÜRÎÄÎÄhÇÜ¬e(
+çÈÔÂž ·©0Œ$eKŽâ"ÛŽøB\Ê­Ü¢à^BPÎœ€¶×xòõKIvÙÖDS\þ£-B .,
+ç¢äé<·?¬ÑÞká¶QÎè"]û£­qø
+GNO]š}»oÓÔ}·¿§@:I²ÂÁÍ€ãµ£	©)Ò@ßÐE¥Zº)£}ÍÚXH°ÜÚ$[åw
+[O±ÍÞû{]Û¢Æaù9°hkÄGzÁ.°Î6úhÀAY(E3šÄ@"%r`ŒUN¶<À8ïÆZŸG L:&8P`"ê×ª1%à
+òàðÆöGM
+8Ç#ÇB;âüÝúÑŸ Åo|¢å4iQ>xKŒKPfj\
+ÃŒÐyEò-Br£@ÐÝOYù¯
+é`M8Êµ
+Í7þ]Ç'o÷š ¯Rì
+Oáº¢BBjèt¯9]!!Þ'ÞQ?{ÑfŸm Æ¿~10(a ·³aÀ¡t]*£HG
+Ñ$-D
+ïÎöª0¬
+ù/Hj6t6Iöa(èÇ× I~SI
+
+U$Bñ~ÙÂ+(ª»ÓöñC*}ªŽÑ-dÑ1h*ñm:OàeMUèÚdÊ?"/îö4-lÐgî5|0pI%×Ð%šçf±MžÚÃ>:ãŠv6\Êñ©Æ3 È]hAwßYL`ÑÚ€Vz×·£,€É1S6¥îË}8Œíìm±²xñýÊ£1êZ¡	'(
+dhjY9Pì
+-(
+bfÃ÷kØeH%¢ë¬
+mA»\4P
+'
+òdi/-@
+ß»ÓŽÛÁè
+'õH»ÕYå·e×1­#C^Øš-±>*÷ò*dgÉJD5k«ç«ê©EX\s2º6°Å§È°xE7êH¶Ð
+¹yMÝÖµ=Iå^&a\]ªq
+ÛÜ­ÀsUU#xa·cZS~«Èc9
+'%á€£­áÔÅàHò9füdNÛ{QÚbl@.=Zfê¯õ
+ÙSÅ;"pb3 â¹2QE[òÎB»B}ÏdßsO÷FB
+Exûnc"±`7Ö8H	Û"Ò±E «îð"fV9
+
+ÂMaR4ï¥Y³÷(©üÐÑØ
+ûË)T0bä.ëí-ôÑRŠH`l+ºGøB$^ÇLzbŠ¢©"d/jÚŒ
+(UYFÖ$í)²LªðK@&#
+oüºÊç7k¹Wp4$š¬æ,GyªóõÉtàKÚPLXT?ç¡ÒphQÃBâRŒÂGÁ*	ÅÝ(*[`q¥tÁ€[
+Ìdû^ž/
+ xSOÒ3J¬f01
+hýâÆ¶[I}Øóä Š'=ÕÑnË:F³âÁD[#üã57("
++o¬J®}ê§?§£)böãQ§9Rô<áLEþ\(³º£A×Û}ê
+ÀºÈéŠ^&i§E[ªPœê¡
+l'ÌÇ¹)°ß
+u`ÃÅA©Y2Ù*o"qûh»ª)á(Ï%XiïVy¹&+ëIl[¡,Ë
+"ß	'²ÒZÝ¿
+V581B(+LcOQZ°³mYio"ßú}·ã
+k¥]áP\£ÖIk§K«ÁX
+w>dÔ$¯%r53Øµõ\ìÓÐ C¹xµN
+Æ]b6¡+ù5­±MãåâŽªÝ(
+XËÛo€]ÜØžoê0á°ØºQŠNóYç$3
+PÙ|2jñ.e"ÛÅù?ÅrùD¯sQQx
+²QÁÑê9.Â	AÂ'ègyÊWãþ
+€Ù<nÇ:W-Ü^
+r-xñ\UQ#V[]¿pÔHÁ¹
+Mi áhã}8³À1i¹Ëñ#R9É¹ð5€É0ªLâbjQT&@ÿþ¬ÍSÑ-8ÐÈÕ²b	L4gNÚžm,ÚI?¢éTS
+é+È-ì,ºg]ÐU3
+dØ@!åŸMcÈÃµã3þ­>4RpNsè*êðÆYÐ°Õª»Tszi}3.6¢t©Ú!ÜJ$W	Ù/ÀÄÛ× »Í-º=ï,¥ÊTÚžIäëŒîyËè[¿¡î*X®k8Ä¡$ÿÊžgs55©AQ¶6åv4$­1ÓçBa7ÈHrpP4Ðí!JŽe[Ç+jŸ±Ð©Ë9ZwÕŠ%Ò6î×~Ì,k¥ª°_
+×úà
+BÊØàžj"âÄá©²îjÕïM(÷9ìâ{
+8ÑÓqûH1R
+_Î
+¹~Ä<ÎêñYÕœTâº K[U9yoWGÕ\D<±Zà\ßxSä|c¶qx
+=1r¢··{¢Ü¶ÅeË±KNØæñA·ÊÝ!lÇE<&
+zpØ¹v|_€ò[T!wÞÏð}"éÚFÀ5)P ÛËo=ËU
+Gµž¥ãìRd%ŒØTÆn,œ€'Wh}«={/F¬ãíš¶phÒx%ocŒG+êÔ¢g\51gíË&_Â÷õ­«ÂCqÈ
+^%ÂŽtÃ9ÔÔcÙpSA§HíÓÀ,ò×Ûà²íç 
+ßžFj
+±Z^0ÂRò
+%  Ïp® Ò
+ÝÞ
+;£· +wÍê{Ù#ª%Ug«ôÌÃvåkDº=^s6
+*E[\¢7D'îfAVÐ€ÄšnW¡O0¥ÐŒfe×á[.Šp°TØ
+Áæ
+]-ÖÃP ÎÎ
+KÈŠ<?Ïü$æ¡èÎþ(Xèè€|ºjäó`(ÐBiD/¬û (åõ|Á
+À
+ä³
+¡3AkÆM~
+²!%D
+H[}â±',îÂôXT
+x³0dÛ§Z.pC¶R^Ás¬&
+Omú0©ôð|àÏ*
+4Hµ*ÜŸòŒY3ËžNÌ³*¬Zú
+ºuCn£Pâ	3sß\±Q|²
+S·"Æ¬ 
+¿œq¯Z³ÂŠ,€À»|Ã~QJ\hK
+ô$"Åèë¯/[,F³liÜ
+PXc€&Ö«{,X]ávòž~:FÖÄYjiÐ6¯qëÉ¡-;ãtÛûúõèrÚThäé&&LTDdÁÊÞ#÷¡ÇEªp^©FÒJ- 9ÜÎÇ¥^ÓB=5ÄGÓÉâÆê|Ü-ŒàÅ|JÅzC«}¹nÁ¢ÂP=F
+3'
+{ÔÎ
+
+vLO°õhFX)ñ5Ÿ÷é2+íù#ß~ÔÏ
+?
+(^.:vùVag0îtŠ÷±œŠ7š#ëÍÅ7BÓvÔuTð.âë4ŸÂÔê7»§š*]8DÓ`ŸnÅ-Ø%ÄÀî ŽÅ¡sBt$¥Df©
+'¯dªÅ8J`Ð37?ØÂ
+kb"rëðbRUÒY¢2IÒ7&FjB*Jì
+4Çz¶[²ò^ûÚÕgA¥Ê_ùñ!LÚÄvòÁ"CtòH¯!®$°äÜLü
+¹Ô$Œ)ï?6ì+Ð-³=eLÿ·s!¡hu|è8"~ï°·CÆGäúì³9},"_5ì(9op
+8Ò7mÒL€ž@ÈT
+nšÍLYî«wdQ¡G-¢D·Ô¯cz¢Ð\yDùzÜg©jB3F©y1As×5ògèÝ)š±Ìß°žÂäÎ	ÔÆ6PßådI;°Ð±pÍ
+XöÙid[rß²\ÆjcA&Î{uaÅ¶($ ð¥R`@ÃW%®
+HèïœÌj]±'­T¥ï0VlBÑÔµ
+q2Ï
+þŸE+v^Mù	;
+<W1ëw±lè©"b1>MØRÚà|c5 æ
+â·Ç¥%¡k¯xé3fÈ»fÇ ò»5-ËÜ5øžšSäÖgÞ}ýeŒèÛ/¡_Wçóz
+hEÔ¢§gqNÌT;)&úÅžã(ÊýïÜìn"<x6
+ºüŒë?ÀyËoÑ
+rœ Û£Å äébCÞ.ÇŸµÀ®.Œ]G&Tîßã|4FÕŽË0ûV1uáìD=<QJ
+ÊeŸLR¹
+Üh¢¯+ZÿJS+DÕ"
+i
+×$ò¬:)Ôéàï.qŸ­w|"sD&)Za5àûÜëÛÃugÉ/+FDKQøÔQhsxc9Ô%
+zÊ§zY_Çp°G³MŽÌ2Ë¶AÔtËlÖ±5+ÌäP	ŸÓ<	O_Ä1ÉA×Ðx»ÇŽ4
+X`Á<*ž
+·çX1²?svž 
+á
+aÏlÃ²
+a_w²ÂÕ/Û_8DåIä~<
+À@Õ
+|ÌÉT·\C>áRŽ1­J)Ñ@
+`ožÃñgdm÷R[`1(¡#sŽk2×Ô·(T(Ø+aãš3ïG$	;IÏaÅÔØÆ!*{
+ÁGÌÐsÑ
+@£a_(¥^RÕ}µiÆÝµåRô èál€djRó"}jØYŽ¡`6
+ËÇ`ù><Ã	HáyËô'nüsP0kZü§gyØá*~~ÑA±x4ÉUCÛq€Ùl6Ù 
+oûboPSðc%=Ø{$¿\5¡
+d/mÆrµIBO²{ak3)âžrÓað<bÐÁïbKl	ºÍùòpkeuÅ/²Í
+ôã5q`1ŠèÊB2>ø;Ø®°Qž7 
+ÄõíFÈÄ "š#eÏP@
+íIÄI%_LÀðôo«]W''0ÛÅ(Ç÷^
+rTý>÷
+µ³üL€ 7
+1NáL@ôÿ
+1ÌÖÅF`_
+S8ähä PÌb,=Q -WÍW#`
+4JD~ÿál/Y
+â© ?×ÞØ,;'-Ø;
+°ä<
+·³P0*Sµá ÌhÒÏõ^õÂâÒá	
+ZýD,ÙB£eä"8V>(|B9wÎa9à~£÷è?=8ôK	Qiy
+Vå
+¥åjÍ\Ÿ"U€ð¥`mÆSl
+LÅÄÛ-Pt *	Á
+`ÉQõEPË¥¥dÀAÈ·(4Oô 
+(š6ŽaÑpaqWÀ* n)Èì
+
+Â÷à·xŠÑàíiCáßç2çÂÏt-H
+Â%ã!)qœÊm0ÈqœTÖõž!C!
+úhgËw1\&MW÷°(
+4!Ž#BH¬`F
+	dagø&G¡¡[pÍÌ&Z@JCÑZ/ôŸ&[
+kK&DùÊ
+[åÃVs£
+=€@
+d1Altr@¡g;p
+ýM;B²AWÅq<îYSÇ'š*H¬kRyžVÔaÔ
+ÿa
+7@éQ¶pðéOm=Zsã©cXámÒ#þš7r
+!#Ul
+xÒ$äÐn§þ)qÖÜÕ$£@hÏÜgIpxApÒñv{
+öÙJGÿQE<ùÊÌÔUp
+qØ
+bÅ]£ÏðnßÕž0.µ#Ü-KY
+!>âÓœVß\gØæ|Á
+' ímfáry)<sR6xN§[»ÎA
+Kõ@LíEaŸÍ \©`2 \Ê
+Ä¢p¢Ž8ÃŒ7u7ÁÆ$!ÅŽÔ+œHÒwàH²&ú-
+ÞÕ÷wx>@aPÓ^¬,õ 7ÄäXcAòð¬x
+!s}XÐ¹È	_ÈäÅ
+åFÃ9Zä¡²¬bAœ¡gOEKä§Ad Ü'^È
+:F®Bš$ñàå	\ÄP1aš®`0ÎðFH
+à_HZlÐ ÇQõSŸ
+VEýGRXn]óuÑS
+>€q WîŽW9E&FGe¬ùTÛIÆž}¬
+}ÃÝ1Ü-cX1öKŒàqèr
+N³x
+è
+7 }È¶Šý
+{ID^
+wèÀU
+gÆQ<ŒB¹ØþÝ\Z
+r\ 1ß²d¶hÙUXŽ8rOátQ`
+Ô%`G`FP6jhcžuË¯ÝmÀÛ @#üf.b
+IÅ[H^€a©WyLOŸ#<D^!Tdp
+
+¿y¬'€«I
+^Èåµ!
+¢!Qc·ÙLpR×"9©AÛ*è	áOçBþqgÜÌRH,õ
+¯çØü
+Å¹pŽ
+oÙ
+8Í!<â 
+
+íÓèdo_"ÇT&ï
+0ŸeK~EÁmZ	è-v
+ÐX+èskÅsP
+ÔÂiÒ­<`sà2ž°jzà/BUjÄŽ%
+$°á5ÑVKà.ëau*þã$xYã^4À'ü
+x°A5šB×ä
+T 
+
+iÖÒ!J(	ÏÙld¥Ã7];]LA1@9]ßÇgtBãÕ4HÅ÷àHØ^An3úÞGŒ?M
+N<áÚçd
+eÚŸèb³Íã@i#äÁ"(||Ú,³Q|
+CÆ
+`ý;(Æ}`<2dò3InÇµ<ä[pâOÀ àeâ
+ø7Jh¡^sP· Óã ÷¥=-	äñËOtÈHìÜrìZíhIa3ÓÖkUA¡à vá 1Ð
+^E',Ø[Ã¥$ô
+ïSÕtÙ47œD9A3
+xfÊT7aDUl
+fÄÿ·šJy!¡d±86÷YP¹{qY
+yA+Tü±AÂ§PR¯\sKÆœÈÓ'Ñáét>òIÂ
+.zFí58@øc=ö¬î_s+CL!TFC "a£Fpô9l¡&æD~®R?ä¯5 IHŽT× C
+(BA KŸñä<É¬MNÂâÁePD#ÃY­Éàž
+gÉ>A5däáCõc,Ú4*Ýmiã\ `Åo4\èL÷wF"Cq
+=H[·=@
+_U0ðÈÃÈ¡3WG£Á|ÄêQ@eÄ2}
+"\åHŠ
+Ø¥¡šI¢#Cä© O3$¡$Ê#ê(ýQâ!&kãDÑ2ßEÊæQ>¢
+BIRô"ÕÖhÔ+
+$®ATªMZÎ¢ñ4J*Híp-!%~ 5
+^ ¹DRÕ9iqì\!Ê3!'ÐäÄEŽd%
+IxZ6ÃY90heQ³
+®
+a(TgxÎÊ%È)%P
+&üEhbpæúm#hQt¡fî0FGàZ2ÃMjäÖE±7šiÍÆè2ÊM'A08&ò"úÚ0Ì
+¢[08²Q}bvqK"MtEQB
+öAv× akHRÅ@EBaxä*É`éjUQ]%
+Tà:Î£D)ìHc#ïÈ#üA
+2ÁNr
+Ì±@58 è1w@Fyå1/þÄ C±(@ÛØ!ÂÜgSBíòKn3Ïä35	}¡ŒeçÕE'äögpRW9v°KDÈi$ILÔ£¡{
+@C6ŽÅ-^wMVB¬ ¬Ü1³_Ô°2Ðs ²TºC
+1òËø~¯ñh2Ž#C@Aè]TUpí£2Üi
+4ß3yb(&N{DQí
+ÒämåÖ r
+;k%I¥'ŠVYz>v£¬&
+ÖwžÙñ`r$*Ö'xx3¹¥	Hq
+ó§ñ¬Ÿ
+^yXâ`MûERòàQ=	B0Dº%f	>Ôð«Xvs,Vfb$h¿Pa±~Ïsü!$þÃ[Ô_ HbÞ
+AB©°ÕàQÇå`Š'fDI 1Dtd.B¬)©|"Dà?jÊvc Â
+°I©Š+È±­	FèuUÀp@dÎ`ŸŠ
+!û­vNB¶iCÐÓaâyíCNiÁQD"%0<¹©ù
+?I¶Iq¹8¯	75®
+`ôdS(m*u\¡ÖlÒ=
+ LÞS!äÀÌÀÙ>ìvúà5ãn.àéP9	[0q
+ÂÖjôFÄØøo*¯€Cnvî&òêç@ fQáqê6ÙÆ)ä'¯°o>A¿jOÒé<š«	L6§ËYs>Ë3Ãs_øt/Gt.ÉÑÓé)Š-çað1ÇæØZ`#;'c
+WÀQÚ
+&)'R
+²
+òòTmÒB]
+cI$>YåphgŸCR`ê©ÐE&»\šNAÀÐ[u!z§AK8üOZ÷bW0
+aN²
+ž×)cÝ$·`yi%éåq°T(­vn4ñ!7ÛåYr
+Í]Ó!ÿ
+(1Ê"çE
+,BE1
+ä=0ò
+¬q_9;ÅßyzlQÙî
+:RBe¢ªâ6çiw¥]±|T\ |ú-Ë\ Ñž rõwÉ ¢p¥°aç%Ù°Ü|EÑW/øf€)guK8&©ùGÂÜ¥}ëß ³QWmeMØ#@€kÛ€vá+À2ÆŸØ_Š	I#B¶dÚÉÏ1¬È
+r-Øydm§1,
+8£Íè
+dÁíò('&Æ.'_ð2;CÜUÖ#ïÈ-jÊG°
+n}šhÐ62WNSÜÚ* Hoê3XbÔ#MnH">
+4mØ,SH>TðsõXô·Åøv'J5Òá,ßòcv¬Ô&±ð/Ù²E+c·¹(aÀÄÈs	mÄº7h@ÄQùä+X
+hcJ)ZA¥Qž£'r
+Ëeb`)ß§ižEg·F¯qÛ	¬œÙ!DXç2è4TGâô
+fpPp¢	ã2ãñoII!#íP¯ïõÄET "
+C2ÕapÆ'¬e%±T&
+©ÞSp»°ÁzŸÁ¡qTþ
+(ŒK7õ]a+ÿ}ÇzåwYZaGÍçRxbœG^×^b@àI¢íšg·ó¬r`dJÔ
+Í
+4°zòÒ
+
+?š÷#(UÍÞA(¥ådPu¶ K_)ur Ô`?ßä0O\Dt?Zd€ZT'L€Qú(g@LYàœà
+ÖplºÑQÏUÂØíb[€Ù0±qAÅ­ØP
+ÍÀõ€wùN2Ô0(I€3!rQZGš
+gú
+Â¥.'ŽesÈDWFešIiÚÁºÝ+ÆZòg~q
+?C.ÂX0
+ldY
+¢
+$'u
+ò	jœ/Äq§L<Ý%žeL^¬ÁoHnÂÄ60Éïbí]tcq#Òâš¡Y°	-O)Zö&ÇÒ¹p-î5Ra^á4£ÕQ£
+Õ
+Y
+Ðµ+îséœb+}%¶i 0
+ÚÍÎ#,8>UÌ!9oyÁæ§Ðý&j hÒüeVÇÂ@h"ÆÍQzš
+º{šãÜ3lxfãCE@øYLX}õÜW
+c×JD*
+	VÒÚ
+4²q&5£à)XrýÆ¥Kü[ÍÂ¡7ôÅàRj€Åæ€<b(µ1ñ ðXŒØAèYSÀ)ÒiA&&šS#`V1VaE3ÊÎ2²ønßBÁ+ ìÙ1ÕÐÆÕ×Õ'é%ÊÃ
+oG0_
+})Í­³<Š
+nDD®LVëŠ<Y
+Æ[¬ŠEx÷æýš$ÉwõËü`ÌçLÌD)Ÿ4E1bsÐö,
+úæõ}ÂyÖDŒV'H
+D°©Ì2ÄÌ¡yd#³[%
+S&FÜ `ùÔki5Í5>lg lQouìâ%+ýmÝÍa,Æ9#ødûâ#h[AEœàz2Q;¢»œ!{`cznÂ§
+×ÐS!@ÕÝ±:VöÔd^X$GXX`SéP¢Yæj"ºSrX²Ë¬Â8éYvâ9ÐU§^¡
+étWÏ`ÊÂP¶Å­P5zé:ò!°ŠÀmÊKjÈÿŽ$NÀ¹!w¢Ð­¹6²šŒÂ^w6D
+;F
+m
+
+§ŸC·ÚÙÄøü'bIwXIš{$vtù95õòGÞš,0{m#Â©8xÉî
+Egµ3ÈÒ5ØH¯¯97
+¹°éFZñKu7ÌYø$-eï
+óaÛÛe
+µ°õ>?V@S4Péké©šG&çú
+~ŠIxÐ°ãgógâÖçuJÍX7ÄkÖJbÑQ³(ÜJÕÊ|BS+`$¡	aÉ5¬ Šø!-D;Áð
+
+³³žt6Åë
+òFÔ##UUwGñ1i
+GÕÍã¬){[UF5«¡Á³¬¶q¬ fSr~
+ÅK
+ÑÍª«
+ÏXn9"a_1U
+ CLp¶±~G®Õ)
+q#ÍÒG"C×Z EnÎ¯¬ïÛ	E*Ìùõ4M×ñ®6]¶J%õDRÓRzIeß[£7)q"À	hàŽ
+WÆ
+BK9*4L%ó
+
++#=\}zŽ%-GïÐ$
+c! \ádbOQ²ó<%JÀi*LáRÀr2då&êöu!À]59k\¥ãùÊyŒË135$Œf
+ÁÍaÊêå:§ªO@1¥Ìàê"ÔÖY4(YÅ-ô^U£ÄD-XBN€ÔÔrÆJ&ÑøA·1(ã¡Ýr*?#
+
+ÍÇ0Í#VÎ2j56ºhÔžÒ©nÊ
+
+byj{êxi
+SO#MöIñ$Æ#ØécËÇ`Ï`§
+þØ@.ÃG`ŽÏI:!ê a|J'\ñ
+jÂ{4GbÈèaïà;DÀíèµ¢Ò(*$x`99HRtwÌr²T£ŸA8)ŽM0Ÿ
+m¬W8uh
+³Öw	Å³C
+BËò2\<åK
+Ð¢
+ciŽ)ñŠKé
+sÊÒ&*X±ÁÊjÇ-Mmª/
+5:©£{N[Ü|ã:Øg4FÓ
+ÕŸÕ
+;
+u©7î ¶*.\EmÓe`ø0+Ý8°æŸ7Hš;.qZNŸL6
+I68 íÌð6gcnkæ`ú¯3SQrnîåN*AônnX"×-*)AM`AeÇÍ3ÿ,ëæªq@iPIU8¢hz8Êe×Ôb0íÈþ0+î(óÙPF(2?²
+ŠÑÜ²ÌÁ0pLŸÁšñLÇZüCËÁlÙQð1Y[øçÙIŽÝìm9­ßlqÆ®U¥ÒcA~*
+cd,Ch7xØÑŠ[«Ñ}Zâ"Y#%×ÐÉpj
+
+É(°WœÍ2ŽcšúÌ8
+rŽ íé 
+,.<	oÒUf	(í ]nÍ
+8@xæ@I
+
+ËLÇP°ÛÓVÝüSEu
+lÂóª±2pžËm¢y<N_Ù5
+Ç
+Þ€,Í WÀëætsè
+¶Ê¬1/UwšR©ªömH
+ìY€?áà4µu,Õ~Ø
+NÂ3ÓŸi&FrQÈ±eR]böeLR©HiçdY@¡L«$/Ç
+O*DÂb>íÚç$DÄ©qÃ 9 gótüÑŒM£2š7·sÁç'æÈ²UAÅ+!·
+_Xk]Ñ	7ÊmÓî'I/D]0oú
+ŸÐ¥kYoPoãŠó¡^}oí­p#à)]¿'&í5§ÊQ
+ú×V¶-ÕæûhQ-lZy© YAG',zAßû9bwÓÓÔ
+3Ešé\à!­WyR4M!·üHžãmâz¬¬T_GX,
+¬J¬UVŠC¹tÜ
+1ã¹\í©ëÄ£]Ä5µy;_°Ù
+"GÅ©7(
+Ëç.œ|]àLÉ6(b
+¡B5P)Ól
+¹çÞ¶@0n|A.bI"KH)n`Àl*"¢³Ñål4NÒ[°XoæÂ[Š±pQâÆÜ¡h5gÄL¬€3UJ%Y
+_lh[º4ºð51çæF{lAàÅ88E·Àµ&'Cý
+	ÌŒf¬¢^³#e
+åÙ/
+êùb éŠ	«µ¬n[n¬(u¹
+Ò @mŠžzApë-êEÎÃZÒ¡Ó=y£
+qñVÉŠi2íZ2³3Â~
+¬6}Q/Ùèr`.7y1iŸÂ^Uhš7wàÀÓ/ª¬¥
+Øu»2Óºô!ñFÒ1§iPiuRénñî³jjvhb«s
+Ó¹ÒV
+Ûàú[OÈ@Öø°¡\>|2ÖÅÑæYþÅ(RŽÄ<~×5£Þ
+¥_Ù¡Né­Öå&juEMó4AR!SëMá»Á}Ñ©¬µb¡úa.ñRÿ
+ìÆÔy;ÚîVMi&ž£EÏè°wÑªÊn\
+Hóñ;UQÕÐŽ
+i&É\`!eÙÔb$Öì(©²]ÃÑoöAç#µ .ØÀ?ŸC×iý±Z	5*#CDÆ~z
+1uøýZù«¡Àµ
+`§-
+7Ýrj&¶
+YLÅÇËsDD:J¹
+ÙèYJ
+ŸYøq­ÍÏÇ¢Óc{æ{ç.
+bÄ8m)gë\>®Cê³\Z²ŽT£øB±Å[·. )ìSSEIäÝø În
+Î¡|ÈmjÁÓ
+Öß}(1¡Ã^P¢¶H%TÛHÒEý&(>rFMÚÀbr\×NQrŽ"ÙÐëkQv
+
+Vª³ÎŒ×YTÒ'g<1¥úc£©b|
+X©âZ×ŒDÆþ¢t{%eðJÑ§Ò	ð§p!ŒL	€çõ*þï(cfÀh/{ïZ×¢êÇ'!h|Êë3Õ(d[ûÇNÑ
+
+O
+I`uû
+8ó8Öå.)&ðâ"¿&ãæú0Nxoä	 U5çTµðé3(µ¡(Çg4@
+ú	ý("$]à©D"lÐÂÖªC
+÷)¬eé
+ì±ÖB.ò$¶þîIÏ
+ Q
+ÌhQ§ú0
+z
+üû\
+ü'jqÈÆ²çºt¢
+¹,HÃ%Ö
+}ùjÑº
+ë0Ä»±ÚÕÔ°¢*Èè~rt
+Êq&î¥CÌlGV:1hbßèO°ßŸ4VÝ¿UüúÕ-p(Y·h
+?ð!á-ÂÆWŽR¶T
+HÎð!šC§tÒ
+í'ä
+dó@cS)A¹PjkÇækÓ®/Úôté
+µÂ
+Á/šJLBÝë[5×À¶ïP°ÎÚ°\TéÙlõ£:¿LÝùÅ[8!
+²ævAÄÉÞ2z
+ÊÂ&àÔè·,^|ÛŸµ¯þ&eô(j6ø=D@òÅhÞWVF¬ËL5ÌSà­1d¢¡}c¬Cç%ŒnÈ:Áìpyß
+€ýº1à}E
+ÉÀª
+®4ñž%°H÷+cô'Ù§²À=ÿF¬æJ¶¥þÊÀÑD>?@;2ëÒ·*Þ ðÊMu^ð gL<ÖÃíÈRÌŒÚÞ,êšH;b»¬phýyYÊOA;âò,ZÁWìÌ}MÚ
+}¿ÕÜB.9L
+±0
+RÒßrIÖÒÚ ?KXšÞdA¢¢
+²áVÌHÅ¯
+L8jÜfÉ"àEŠ
+MµöÄ,íÈ)b ¯ëÔZas
+|qÄW~q±Ù·EÁDü
+Za
+ãh)Àø»
+9aÓ{ªTŒDD®ÍuQR7zu×­ÁmìŒÃ'~òZP%éÚ0lc'Ž!@T
+úüÆÙœ¡wŠâ+èÓšÉ
+PÚ0®Â¥=Ï^+fe^ëé&YTÜSNp,Ñáµ­Á§CkÉ=R¬2lêÐæ$&œ	ÅÐOª¥R[Ã^IŠ«0åz{lR
+DÛ?YP,ÊzÙG"
+óc PiîRíF5Y¹PòÇ:Æ_»Tl:o>ó³6B$cÞ1vU%o=Äñt\jB±YC©oG_|ªvþÈö·(ÈHŽ),ß
+ûJA~±ŒÝ(MrH
+r9wSz0ñ%ßÐS²øcKÙš_€SèÆ«J·Ö6+geCŸ 7µŽÉj]WÞ4^tYqŽúþ5Z¢?RÊ¡ó
+$>qx«ú
+º»ã/±íí§ ° Ý­W¶AX`Ó1pÅ>jkùßap
+b]7ïùbpT(ÏITVT£õB;B
+nÃÌZ6hÅFb:8&H
+Ì .ÅÐþ^Ú 
+äPŸXBþŒÝPóÚÎzË²~Ÿ>k¬É&
+I3â(%uðv(9Yœm;¿
+Y¡W=Q0«¡*(.ëõÓZcÔÈf4
+®p
+ÐèEâSŽN¿8¬!yÍyA 4~-À°ÂtxäqT@\ lÞ·ÀiÊL1ð2€gÜ
+Ði
+êS+ß0¹æxr%ú0llV.FX
+2V")è=ÈÎ
+€­q)Zþ À3è)rÁ6Ò¿bÀ³ŠR&Â­
+_ÄI>7üŠyèÇj
+ØÃÈdê0EÄ÷$¶!`k»Þ+0ÖD^žxÝ§Ç«v
+Q[SAE\`v¯hBù4@PµÊ[°0*;Ã
+4CÌuJñFÀÎ¥ÂïDæÂËç1ÑäcŽqXgî$
+Ž|/ß¢0"&œÈ9lFx»xCÁÜ@b°cìãeÀ;k ÌÃE_ 
+,2ôî,Ë
+òÜœùpë4Â4ÐÂVq±5$ß/uÅsneU¥`«æÊÀå/
+ô(hN³yçýJ]¬ð)Ò¿°aa3­0¬â»Ú
+1¹ê7Ç@QÍ×
+QÎ@EP3°Wg&]5¯KN/ß*ÐY1mTë{¬>²D
+·KeâGŠ$ËÁ4na¿dFËc¬µcÚj×ÇjjãdÁHš±Ð¿
+ÑP4JÔ5£RÔ`þYGºZÓÍÜ7/Fu¹ÏB¬lÚXZeÂÏÊã®oëK¢©Uþ*Ü¬ÙªÛ	#+Í!y±EBI¡sÕÓKõEè©&G¢µ[ý)žØTçý
+
+)È 1àCJ2RŸjqÔè¡Çb
+úÐÂ*Îüq³¬Ã FaII
+
+ÏH8B
+°äI¹œNÞÃeþsDßq/ìBæ8ØÐ,7OÇÃT!T0
+ÏACë«¹
+^.9æ4ÍB[OâôÚ
+Ófâ±AÔILn¯§ðwêÇö¬qòæP&<ïx$$Î16µ	h¯N}C6ŽBª¬/CBeÍî&ü¬LËªåU
+%Ö.T0-X8¢NµqŠcçÓ¢
+Ö4eÚbx!þéÒ_AÐOþ¡­J¹È©niøl]VÁ€d¥_=
+ÆâÃÖèÔÕ5Nñ"Þv
+wIûO0ÌZ"Y ð®Qì0{zLlÙEêÂV3,
+µ,}ãÍe£
+ÿf)UG
+ Wt¹0É-fQo\
+0o«XaPuF*;îðrš2a,o&€ß¢l­®µá]4SÏ*
+;m·,qUÃ[Ëá?dqL¯KÖÔ>D=¬U&.¶KVÚL©Ÿâ²@ótÚDœú|Í'Ýâ(äªµ©¥NTšRvÔ²LGBèAïÊ.ÈÇÄ²ì5U1Ugï
+»R·ïLí `ËõBy¯ŸdR
+ùÈûè=Üð®É	_3(BQNv¡M
+q*eÀW/Ôóÿ!ŸÒTF)¥¯&}äÞ¶¯y»Ïë{.
+ M	%AZ ¯*z9ŸŽ«œ°Ø°2ò#Ž
+^ 'Óïïæ
+|9E_£
+šæE>Wu
+"vÎVÑ.€
+#Á4 stÁQj
+wt§æ+GšP§
+/êäµ(ä|,5±Ì6
+7g|9ÎXè5¥/¡¡í¥eÛ
+Ñª€z¹uÃ¢dßÜô9dGÙ°^.E*Â
+ ]ö/ú»HRð}Æžs]Ú	"`;]77rõJŽÈX7ZJtÔœìÌ<ÖÓyK»Put
+Q
+So×f
+Çs³3ÕE{Mysµ1Q
+ÈrLI>:
+Þ@o€`ÃiLÈÛô&
+ª»¡.':"÷(sF»mÏ&§¥º9ûÊÉê ÄàØ I)
+µ+þ:XZ~oLN!éDyô]mÆÛÍ[
+¥ÈÞÌ7_Ðåòº·527MK©qšO I®éÌ§xPÙ	0mv°ÛÑWùµOÒ*nÚa/<µ3¡*kÒŽ;ZÔkmÚÃKÝ°GÁÎÞé]î tì%« òI±2Y09®Êõb
+ª"€jB\ÒÂ%ÊbºZºÚX±I²Õ¥Eââ,Ò7¯«ÊZR
+7èŽ#¯HÛ8Å<œÔ¢é¢®nLAãv-Þ"ý£æ2(ÀXÞ.ÑÅ²
+£4ÉyR'j$Ž.Ð?µsùà%>×cºµÓ¡ü
+ÒLc?a@â`FyŽèëX
+êËåB?äÆœ£hŠ$kXP²£ŸeçI{¶16Y*K7³ÔÊòAà^)ãÝÚzæ$óÖre0õMeÜÓ²ŒfâKÅýôŸ³-õ5(F]LwÌ*Õ
+6óè<%+ÕA#¹UÏÊ¬11Cž*hóHþË€
+s
+¥ËQð
+³ÉÇÉF:\CÂýjÀÉË>æÅùl&UÒ¬*q*ë¯)êBq0J'ÑÈLF¡ËqÆËp!%òI±¬Hép
+¬K~£,ûÎÇ7Ý¹5
+«èkXñ43Uú§ºSß=þ§êJð;1*à-Dà]R'²ûjòSPu£
+}·*
+!èÓY@Â²^n)
+WAIqnPÄFNU)
+1(YE4Þ·Õ-È,
+64*ñ«®.'ckÙûÉß a	ÚL²¹à<>Fáw2èE°Pø1Cz?Þ¬è}¢pŠh_mIHÿ€Ç×.1Á
+@ŒÅ2Ö
+ž1G
+p»š"þìµ
+ | Ädd?o¡a€YnåyBöpM:¬|ÌêÇiGVè¿YÖ7Žª_Œø#ð×5KLtpH,âg1¥
+ß§7å|§
+Ê
+ºøK	ëêU¯šÅ÷Ðª ÉL °0,;oZi¡ÔÈÄbûê8+CF
+Ló(qæzÂûZévjÇ·å
+²A¹Wyh^
+SWÒú\ª]¿ÊÚwO+ÈµÁ÷M«eD
+Óîw'ø{38q75ÝÍaÅÈÎ*B0¡PresJFŒ >>_^³Â`Ç`ýøò9ySÝ>øçØ,Ia'Š ]rË XFÿ·]PÍ
+(š·-daÑÆlÁJt Á¹ö0%?ÃÇö¡[êÐh.ø·G>**-â@P0v»èzÏû%(¡Ø
+Én"Þ×þ
+v×á4-íàª^Ë w8"¥¯
+×unsâNLÒyÐ²ä^ÄËZBßáâEïV&è;g.ù÷àªwÅ°Ñ
+œuIè
+LÂ·§@|3k(ÓÔêªiZ@Ö±¥kÄñÉsTßº5ÄYhuPÝz	£
+ËÀñv?{·OÃÝ€X ÿÑ"óµG
+óô©rÙú}ëßShGdqYÉ0ÇÎ
+%à*S¹8'ß]5"L{lkfÒåRXÛýó;ŽÙ03Ü;8ÕxiÂBÎhyÚ ËYV9FYôÛÎmŒ¶Ugói
+SžÂõ¢ŒÖE2.GãRé·WÆgn	úH!µó%
+©	'fÁö)ÎaÛê"ã ïäÓÓÓDš5UÜµØKq(þ>£B9
+®zûmßñU|R	 1šg}vdÕ0ñ,gC²Ð 
+*ùP6$£úISFÜAÍÇfYÒ­ñ©w¯|c4©LØâ&î^C¥6H®Ñ##éÄŽ$ÄD
+5pÍ&VçS\ªDF*rrÈÞ|Ø]o¿œ Y_wUÛd
+ŒÛMYžµº
+ÈÂg®}yZs/àÙ(!¢6ÀåÄ­âßáÌ7$
+L	âÁx1Ðâ{n^ÕÀO_þ-^ÙØK7Ôä
+M
+ôÐž7ÌÄ4!Þ{,Õð7/e>;Þ³žwãdB­<
+!OÀ ð?(¡AÀTàA¯UÀ WHj¬Ñs-ÂLäÓdMg©d
+b4á9V±Ò(Žvéïm<u
+fp2Ç/Áð÷ðF@A0H3ü4,mÇâjù_­ã·àäÉaI
+ªYCÐÎÙÁ2Ùè5^^)sÏ
+Zª{H
+¢£2Š­[¢ÝTà¡d
+±"»à©ùÅ@°n	0€ºœ²kµAx)ß^w
+CºMëø	ED"=tGUœ5aéàÜYHRæ>>ÞfaÂÁPD1r·Ò±Œšhãd^Õú#TéAdLg]lãHåšPÎëú¥È9
+ÅÍq{bº{ùÙÄ£k>:BQ.	3v2QìAdb×pËPD8un.¹
+ÀÚŸ	'¡ršhE&ìähé!k`»Öçæ/Y3šæL ŒÜÛg	ã
+
+ÀŠI)©
+§Ãâ³ë®JÅ8KÇÒ~$"
+4<í±%ÇŽFb@UPs8ÐRg¢;ÂhÕÈkpµRdAïEbè$
+Å
+ëVº18ç¯ÃmjPè:Ïú§v&¢:,.­Gw+Ã9Ö)D¬ÛüEð|ywÅà³òIq«¹º¢zZQ0a}
+e¬©5Htpøº@p¬··ñ-ôR®£-°·Ñ:þw(=4cå°
+[²©€W'äE(ÈBÚKµúÃSŽÒŸÏÅ3\ty×Ü`Á­Tf2(m|Pc;{í#(@Màpë ³6}öè+EÁþ¢ìi$Œ
+
+lCÐ±®të?ÅR|HÈËÅ±;ÊRÂxú43
+a¬ä-
++ ì[0vá¿Í#
+Ë7,}
+ÈËË.±
+£
+ua¥ì<Ö6Ò¬,-ôKø4ËkP³ÃNÆèIéÐÖeÕdzp
+ŠüµaqE7ñ0ëK@&M{é!0ÇËc
+È©ÄÝ×2Ì­Ånäí,[6;ô	Éái9|ŸÇ]×aPŽÄñþD	áÀ8¶X
+Å:3ÒÚãN/iãÅ
+ÞL
+>X-ãBgŸíxöå1z'¬%E*è,ãÅ¢_Å³^Ž<æsñÔ_tÝWø9®cè?f95õËâáATþt1xç*íw3/áÿÛ#Ùjßù`Ù· ¬Š[4À§Ö×wÉæÂ
+Ÿ;'ÀB°ì±ïÊ»~<«Àbmx\:ÓBnjAõ ùGþ
+No^³¿YÐ%#HŠYþM¡?kÔ
+îyòz»ðc]µ
+Ã¶c 
+èi;0Ô
+vxŠG
+J­ÄMîÏMxUn={\ÆuuSY)45ï]Šî¥Åñº÷ûåuçÙ
+Â×us<ÄEúŸ
+ÃR:ch|©Æ'-Y.ûÓÅ`ºZåg²PÝò«;em%U¹ÃÓµL4v·Ø5Ž_û!IñE»J¶ðg[jvçL*&
+>BZÆÃÂOñ\ºŽtÅ­÷_!
+³œmnÞa¹kS§\aù;Ãc; O
+a®
+\ºtu|¥-iÂT ÉÅœ»Ä
+k|óñ«·ÚPžÞ·ù=,{ó)F
+@©9¿F8I§FRÕö¢27bS$
+ÈLUEt¿uI,äÑ$ç¥ù)ZWœÆÎÁa?
+Ûwé×üâžçŒŸ3ÎÍŽNI9ñÆ»€=ËD¯N)J)ÛÔŠ«k]-&eì~3X<mk[J8ÿœ1Hk)4gHD%ºpk°È;ÞÂ_Ýç7áH·]Ûø¢Kr¬ALšE(Ù MæóÓ448ÔIM
+N27}
+ìZ8ã®G]ŽhoYd%#Ô_2`Öî|âÇ<ê£áòE°Í`Ë	:6M&Î\ììÆB¶{ŠoYs/Êž7Ø
+KžJ¢ÍJ'4b¬È³²ô
+Ö°96AèN€,Ž«7Ì€fsÇ E÷@­îcú%GÊD€P9· P&ö|B¢±kß
+©o£UÄ;üâÐÑò;~iqtÑ[c*5cÊËXàTàKµé¬rt­ùûÎ@Ä°v.Ï«=µó(¿çºVë6hq®õžy.AìXV<
+bi
+i
+œGKv7B9A&êJÅF1°,ËsÐXR`Ünïè·w×÷)ž1ZL
+A÷0N;® í¯U¿²)œ±ÆðB8³6¡_šš[lÉQ¯]iLÐÎ%ÁbÒuî]v*t;Yø²SákAwí€á{ø>oý§¬Ó
+/âN§oÏ`#Òž°¬yÃÌc>IKûŽ$ÖË
+ïûÜk¥Ú ÄõTÃú$³Ók[¬{k© =§t5m[yo;r­%Ã¹-wyªl0GaÒÐñ­<Å¢PÎR2ÑÎl[&¬þPê|¡ó\0>€?®.¢-*mÏ2OgÁÒžÈK|Wyx©Å
+vÁæÏ£yÄáØBçèðÎŠž
+_?$¥¶OG>¢ÎPê"á\Œ"Âf·gá ¶i ÷Þ*Ù<Rq£TbšÝfà`}ùclæ1
+ÁtžÞÛi·{sfï
+ñÂ7 
+ñ
+LÂœäŠðoé¶Ì'€àHúDïÅV©àcG®ÐÏñ¬HjÐôªm4Fqmµf^PÎVNUæ~âœòVC¯ø«õ¥Á\4€ù!YúR¢ìä°
+ÂæÇÒ7D]7x46Ÿ|ËÀDµÊÙORŽøY<-%<FmÑÅö
+Z¯/Ã#ÂªïÍÓÀŸ€T.,€L$I
+?9§Éñ	Ð#QP
+;àÒ¡>Šµ÷f]
+ùAÄÒ&7Çøw±I~Ï~¿cræaV
+Ênœãï¡$šå>`ñ!üß6RQ
+¹æÈøÓOŽ
+@¯hr}ÕlBìxCôJ²;ïfdóóÙÎœJÃ?Å§5Òó~õçÎ©ÉXRÓ"È|VÈCd^¥ýÁVHé³ð}òá*
+Ó~Ïm
+ylNÉ&Q(V&çm¿Äöš?Òç!á­LI9#¢£ Žkn<ý^ÏxG5sRÑºâãP6ßäüe+æïÊ²ÞÕ$sK0®
+cGŸyšïÙ7>¿ÛŒUá%GìþbŠ8øü?ò
+Z¢É¢
+V¹}gT`ÁÏÞ#õ¿Ï(m+,
+`þÒ×Ib€áñåÔçÆÔ¯4Äz
+üfnù£Ö²&·à\ÆôcGdvíÞãÂ
+ÏpðTGPWðŠËxÿ·Ä¿&ŒfÆ
+èãa4Q
+#}7va/°K:F6&¡§7 uÚ?Ö
+(ÁíÐá-CZZ­ÉŸSäiè(3²Ãå€T3oÐFu©éZGhÈ-$v
+
+E^orýcØFcÙ­UÁËÆÈÓ2 ú8^AŽÛÐçwôÓó¯~ü±÷ïÒûRÌÙ}åzè3rGÐâZé¯ÇÀ9÷¢ÓýöNáKßÏ`Õ.r@rm' ÜðúA
+¹SL2Xpà_â ÕÓî1ôÂÖÞÍÄ©"«=hà
+BI¿áww	ö
+B0jõ
+¯\ät±9
+mg6ùãWð:È]ú
+.
+K,
+×
+­tcê!î
+R_:C Aš3°ñž(²ïMÆF[D
+,ú&€?ÖZ­K:rIP×ð-À¶
+ÏuMø]¬£.0~C
+Ð
+µC<ÿÎß§À¡¯DÐJž
+
+Al[èhÆÇ¥	J­
+$áÅËGhSýû8U<Ä¶c >ËT5x
+É'økG¿«+ïõ
+2l^,£#úúó4Í$iœË8Çû@Øã7c&#"Z`yØÌo(Å.¡æ+H×ÆjÖVh!!ÍÉóî,š
+ðëÇ8	²-FOPÔÉHAAÒ,(ýÉÞkItP#.!7Úä#ò
+Â+QR²j})øÑÊxô³1X®?©q/²Sc¡ÔRI?nàfç3eÃ=ENÐ?Uz ±&¶F,ÀÂJN,2Fz5và
+wšcó@`#£K ÍHVIû&ùâ«ÄôQj{Ô.Œš+ÅO±"ë>(þìrÚ  -ã
+
+L%|RdpAÉ"åü£B`è*j&lAäÃ%%r+É
+
+i&¡n±¡ÓV$ôI¯eë=SbJÄ€BIÑÍ§ePÂ|5š71³ÊZßÁ@"2
+K"ÏS°]#ZfŒ¿ÐKÒ.äwkÔÄ}!ÁXÀ[]\÷e€÷g±æÉ¿tKê>Gþ)D}ãk!
+`$UBOÇø3¡"07
+p¡)JT
+÷gë¹·yçø$®q£¢%©}Bè2Ÿ¯-ï¡5Ù9ð@K,Ðo*&¶
+Ô[º÷uW
+=Ø"øè{wóaþôm¹MÈþr3ÛýÆ¡§t»W¬\ª`6EîE2åÐ}2AÒBð
+wC€Ñ¢6
+R;ªš
+Ô8«{äfs
+­ŽÕb. ,é€Lpm~­úA{êÁÔ¯ÈáPJ2
+à
+ÑÊ¹`Ži
+
+O,TVÃH,¬GX¬£ö¢JŽ³¡IþüRl.é
+
+0¡OòÌ±¹È=ùtÊâ7öµ50ïÂ
+
+Dîµ
+DHm­{ØSÐVúSc »:=Ý oM
+£MH±xsæP
+ÃeÈ¢ÈE^NÃ×Y$ÁqÓô+Ze 
+xåb%ô»u*ÂhêŽõK1
+sŽxTÄ?M GõU¬<SÈ%B±Žz	ÂŠÄjwX×aÈ×Cg|;8ÎLêGb`Ûšý!ií!Y­¥9°x°ß\
+]ÔV
+KÊøÑè	ŽG§Å4£5Tõ;Ò$äÑ²KaŽõ
+r4+OŸ"6Þao=Lza
+TëšQg8QÐ
+^*R/êEBlk{¹Âgá+'Çtý¡ø;X~ËH(_FÊ8 HMÉÂ€ABÁ
+«út$Ì¥Üà¥ßpÕFL	nF qÅiìÜW]è>H<¢ô c ðžAnÔK,í8é[kF'iÝ!À£G+Tè	|gCÐî
+_âÖ«jui.	K 4ô!y-61þÁÖCÅ!RòæšfkfÆ ¥ia ,Á|,<'ËdA!
+1àqØ `]%ñÀù
+C"u3÷â
+YRu¬ß€;.²V¹Âó¯õÃ€èJÐjÈ¶á8@6ô
+Ô~_C9ãGhàq×ÐÒÀAÐ®ŽqKw
+zú»ÌâCÚóhü7ê'.¶¯©*ØÉ÷
+Cb1ÝfÌÇ¹ ÔÜBì<4œÈÝª4â	ä!ñä7â
+0P$#š7ü¶ª0Ã~ÖÌPYèõp¹Ô@Û¬ÉáÕ-±8
+
+k!l!È2pòÝ°Z`¯ ŠÞ©çzwëâ³ÉÐR
++	^Æ"m¬³Êâ^Æ¹¿æKÛê7R.
+â
+¢;1<ÍdY'$
+86LÌÂµBîEŽá#4b:­
+(ØDJÓèò
+,`ç
+§<0æTEµµ\óa1@s4PÂÆY³é	Vî¢TÝ«Ùd\º^
+¯BNð^Z+höL
+N4®gåžðHÊG ÉEJ§EBtŸP,&&E
+4dÀFÄoëp­
+Iý§
+Ôì.Œ"f3HAÂvè(ª2Ð>Q³4ÁÊ |juæhoSÏ5ÃÂÎX;9@à!$ÅÁõô²`dh$~\á@»IU4Y
+ãi%þ)@ý )ªqq/9ÏŽ
+DÂcmôNAë!_
+{xWê)PŠÔÜ
+ñ"43iŒïp>HÂ¶÷ål~bo@æh-T8lïé7Žz5íG"
+'ÁÂ¢uûè³š
+¹n$ZËV+  \áOL5`
+Ä¢ÚG#r7×»ž`eN­¶
+sm±êHrñ¡rër>ÀDyÝãÄAôÈ7e
+YÖ.qV@ÔXÌqÃpp,ÛÐ
+¬U	häá*
+œþ^
+kÕNÂaQÈP@Õ²ÄGä) Ä÷
+Z!;TGÎØ3"ÊžAZãøØÚŽiÊ*ÞµæBÓ8,ïDi
+°HÿÔ»B
+ðÔ
+áèÞDÐÙ
+$WÎa¢qD¶øS,xkÑO2{C*ÄÃÄ·(÷E$
+&Úv<1(Oîû)ð5Ë-ç9L#I9ì 
+ÃÙ'œú8
+Ð
+
+VV
+ñÉXùXÖeýAÎHÃlø¢HŸ%yáÆbù'È%Mœz
+3OQDõÕÒÜz@6(ÔGÈÆÊ
+%žÌ¯Ôn.e8ð%*Ú"2ÍÄ5ç0J*Û¹=*zt
+©E±xæª%¹ADf·
+ú©È\j#
+Â
+íÃ·¡ÁFŽz_xÏÓ-°aÀÔÓ=gG
+É`Õ³,H1i=×_îv»Ò.Ð
+uf6Å¬aW±V
+â­
+õE\*k©À×VYtæ
+5©§
+2¯h@ôùT¢]Ÿ!ãPFÃGX
+®|ÐxþSáføp|¢DŸWàìv 7ªÜž°p[Óð
+T5_TwB.#¿ÐR ãs+^Qv
+
+@Q>€'l5
+Ù
+Ä'T(X;è%t«ŠÎLïÏVÓÍüÑç 
+1'ªîEÉJC
+Þ¶êú
+p:m
+é"l@ÂXð.Pæ:Y
+ãlŒšgðcü
+$¹6
+KâÆR[-2ØÜDcÍ}À<T<¹&](
+	ââ	€ÕTT2E
+_œZ
+*!XµbÁ/a+t|ÊCHÉÁùµ\<n)D94âŒTçSÇx(Ø>8TpN€Ëûÿ
+4nY@IóG~ÃAªheY%?g 3]lH
+ÛŒþÝéÙqÿ3èù.
+Ù"ÆÎ
+g¬aŒ)6lENoºÝI2H|:0æüÀ,aÙ%bgÀ¬É×l¯~×oØüNö¢ø hÁØäË
+¬©6qªPT|oGAGÙèµX.9	KB©B
+a°}25­5Î5åÕùøèOú
+ôž¢ñ`p¥rtõLØ=
+=Áy	XNÑ
+jD%}jÆRå»S0<ÜEãÖ°YP§ÑÅ·Æj1Î
+š0rŸD%îÌ
+ïÐ"»hEá<¹8éÏ|A,d©zµ]_èGØæÅ-ÈA9ÚF&K·âÃ
+ÀEHE:O±TLø­ÏÞ#zÅJI£XfG€ÜT#ÌÀï¬w5¡èYü=
+tÜŸqÑúÏDn:héf]Öè~ÄzCHDp
+ôÅü¢ÿÂÑ!û
+ÖÞ
+ã=ËIÁg2Í áP^­ÁwÅ"B!b<·±eCÛ
+	A¿
+R6XCîk
+
+ošÜÆÅñwT4D=	
+r"dÐÂsXN;ÑŸ§UHß fDg?££«#Qlžðàð¢-mNýaE
+t_P#GIapsøHq@ ÖŸ
+hÑxâAà#^
+ÙpÎSP­Jc(xtUæ·éE,t3:V=¶{VE
+C¡®üÉU@Bà
+P <ž16KŽ~"ÉÑÊ*W\
+ {
+ QŠi\?€c
+ Î	ò
+Z
+P3
+3ÇŠI)¥e@žf)$,"FwÈÏcBt4ŒGÈCa/5pøY#õ Z	u®V¿] 6:.^5~°W$dÂ
+y:§
+DÎ5+J¡Ö&Ù
+úLB_Õ
+)åéâ9=Ñðæ
+JxZÁÉú%2Åj'#än \Ì_»lÁsÆñ9°H`ÏÞAAQGdpØí,Rb-ÇdÞ£÷I@ Ñ¿
+£q@ñ_ì¶è<²<ö=b¡'µxÞi¯DãåÄ|'À	
+	ªcáŠf[2/.'Ðô%Z­ á
+|"r!7f[°Æ:IënÓ4\Á5ª²8TþËL¡ücÀm'FdKÂÂ  ÒwIhd
+,.
+£±Nî	ïCl¢è;PiÎòc1²7Ø	6wU"'<*=[æcµÏ
+ÃÅ66ð]-ÀV7íùgq³Ëò'ýZ6GnH]'æg
+û";ßcã±S ê=(#Ô¢à°ÿ·³³ÒÀ"þÓÄh«	ÁÌ21ÁÓÞþÏ£¡@œ<€1«ÜN.0KºÒNß³SªÉ AU,FãÃ=Ä685é6N+2Xåb¶ôÍ"K;ªmã@\à³ñeq nè
+ÄfÙ®C¢Šµcá¶ç	È/* *7	qÜR
+$èÕD%š8ï"öÖ¢w#+<#€EdZ\
+Xí ÎT-i²2"/v1E¬Ø.8ø"É:1`tw"#
+ù-r&L!L
+@d$3h0TÅ]¶b¯DaŸÃT[ìíb<Œ 
+
+ÊTâR~
+J ÆàÚñÈ7Ø{= öNå@ØfÌÙ œ2B l62árÈêÓðdgC·ÃšxÂß«LxŸ0g'âtàU	-^l¬;CXa4±íŒð*	/RHögíh
+Ä6¢(G+vC¹Ž
+! ÉWñõCr®Å!:a7eÂid¡
+ð
+4¯îÈ%FŸÌ"(
+ÁôŸ
+c"7Ihoõ8aà 
+I¹
+¿glÅeºÛ€%EreBHšš'ŒçF(ìÄ7á-Å4i6NýD%KÑa?qV'³zÍÎ*~
+æ÷D0ø¬nP>d_ê»ô
+ÌÉüoN
+10cLD&~
+ÇEóÒ%ÁNð\šªÕ6Î
+ÂÌ¥l;'rO
+c06X:äòúK%
+-
+ÒI-ÉPö|äÙ  Æž¿§Íp=\vË#ÛgµsÚ$iLsÚ2£àìÈâcÃhøF \^ÒÂÝrTžÄ-&,cybpë;~=â>Í€ãÒœk3|¢DëÈ¥\®ÀB¡
+( à_çX0Sae
+7
+ï#ï³PTïg
+ÅN?1P ÊY$¬>kV!Gwê
+¶,Â#J PWuÊ±=nÜi#ªSi!òCB¢ADà5à±×ÍëpèX0eÈ4T#%J&W7ÈZM.ÑŒ*°²
+$:)á\õ!ÿ¬,ö1bŽèOTè8»¡&ÜÛµGH8Ÿô>gPÓòfq[è$v>
+;à)€
+Y)6pe%d¬ÚHÍÛew§
+¢Pj¬¥óÏBÂD¹~UDðžÍÃHä"Îq¡8èF'ï3pèBàüµ(¥ÌÇD©;øV\ýB}E`8 £ì
+Œª>%µL
+¹
+RßPeùIÊ1t
+â	Š¹çGìJ
+5g€¹ÆÎL{H#Pb ïn)«3²BVñžCÖkµÙÚ29ÂöG³¬	^ž&Ñ­Àé¹­<¬}ãÈLËí:cåã'ÔmVüªåÙìf
+sDxb^J­å© 6÷Qx{n*6ÁÝ¢÷WXhé$|4šüõ'29é>åº D,®ôÚ ø~®!£Ã²+6XÙòÔ+K
+õ 9PB@ÝÈÐI
+9©Ë24PvsÆ\iñØøVGäj'YŠŒ$ÍâÜä¿q³Å€z£òÄ©êÌj`°
+Ä[[sí£.øÁ ä2F$K
+ï8.lk¶RºÃuÈ6VTCYH<ò²%(G©B­+Ók-³ÄË<ž3ùM
+zFJ»Í_£Ecm	Š
+ì©ŽÐ¡Y£«.JS,
+Jô#nžºó¶?ŒÇÞbåòþteäsÐ1~9öÁªI9!Ó0øèdÃÄþ!Êæ°º}07Š1â
+É
+7ÕX
+ô<d2ãßdØÖÆù¢ð9Ž·	sdré)p
+B]#[pïfTAU¢ìÊ³q€g€2añµå®g@£¥,"1SùJtªX(2|
+dJDù
+
+¥KÁ!ÑÙd)§/ Èäª¡
+!3rÅ>`ŸIáq$Ç£È}b]z¯4]
+óYc
+÷²MŸO¡ewšEàÚ@8rãcQäy(
+­Ô@à#ít3i)£÷?æû.¡è«Z÷³Qd²þ@<i	Ât
+äÓµxüµìì;Àeh¬äHÉûŽ4øYÇuðÒh&	0IvÀMñ§ðcCG<;Yòõu§üDö(K'	,h	Y<- F
+h$¯}?Œ²$
+ÈÀÓ§Ë¡Z
+wj €FPjU;ø±xo®%cÌóÊÈ¢xú8zà
+ªÝ²dY#hVMŒj+ÀSa¢±±Œ,BØ@
+ÄdÇÐn«%Ø_i0%)|
+{£ &Ð0zÓÄÁ#
+åü·Ã
+äôeJ
+?°¶Z°T
+Ã`DáÌe)$öÀÐd1Ð7ËÓð€N
+CJ^ ïE$óÞ%bVILC¯ñ­~ÃKŒE
+afÙ@\YSRvÀ9
+§òà÷Óeêœ¶pi+L3ç¡é9šKÎgg'ÇDŒ}å7n<c¯
+
+î©Ñn°ü¥1ó00@
+îÌHë¹èHõ8dz#ú¢bK
+ì+­9ôÆá
+LÈÅØWæ^<	
+_È *ÀŽk¬
+èA8°h Å«©»Ì2Ç@€KCáaÃÀ?_nºæ	!µÕ÷ËÀÔ]él($
++
+
+CÄ¹
+CÐÎLôùCD¶ºJ³ÇGyÜœã7Cif§GËãMì<ar%X¢
+òÐªZ(ÕqŠÇsn>/iè@
+Á Ç7RÆáMî,ÎÜ32<hÄ[æÍ"24QLÂa
+é[xf 4°µóV3o¬ãÀúå<Üön- )*ÊrÊ¢!ZdíüSÐñìÆÚFÙòYŽÜLšJË"ã%JÚöq
+&-^þÌÂoÄ!µiR5§:,Ó.«)Í
+Þ#Å¬ëÉ£R¡©sM.è¶ØK;FŒ°5çÝ&!ÚeÖw4
+?
+ "Ó€`ü?d
+r@_Ï!(hÁÐkŽÎ§Ä
+eBF¹ª¹¥xæICd
+ïK*@*a-×@ÂE
+*T!ÖQXNëRŒC
+
+8ZzCfßfé"$
+µ57#äþHÿqÁTe}ñ¬°&Ç 6gsÉÏPADÚ
+ÉŠ%!Gp0lÎÁÚîÉŠ5¡	7#rvir£œ€y×1ªŽPªØÝðDûYíd	'j¿A°Ë`a°ïP0ÊçBUs8Qü°$À[JF(	 9=\Wt¥€ŒïC¥T[L gÄMß*4£Ž€D]
+S'ßEò¥o Â£*èÿJS¿ÏÚÅ~I\Ä#Ð
+ÔF"[¡Ñd\>4"C¶¡(L)Q%Má
+²Þëhª5Èg€oCL'îÐom*8¬ñÍM²c
+l>^Ñ+ÙÂXb[7x,2 S*É°¬3\Cä
+ŠÔ@Ì
+áÎ`/;"`ûåÃsæg÷;Ñ"ÂbÑ/0ö2F·'[LáæŠ)E`h44{äãÂÔ ò9RB!Ñûf~ùîÑ³<
+ bÂéyüòiæÞ@d
+ÈÃž|S/«+g¡PÇ44Ð)é«M S°$6
+O§INâ6 <
+­`ÄªÛEc"µÎš%àKNd$€@N(Ž:Û
+ËÍs[¹ÓžÔ
+Ô|S&_¡V"(hÝò$ø%Hçp¡
+°ÿDÛq
+ŠÃS +Ÿ$ÂZ 
+Öwµøh4µ
+Æ©€9. f
+ZµÅÊ,)H%ÎO§Ã¹-Dú{o
+
+Ú®%¹ŸzqøÀEÔA^dqÉba
+øyè&R^IÚG
+Lïo8Ä:5ë²É=[D?$ZÊrhGÒ
+n«Àà è¬ìýRÆàOS
+âwiKãŒ¯"_»ÑttbÁ2Öé*ŽÎMàÊŒåá¹JÜ€<l1Î7¯@u·Ú¬$|ÛhJmœjÄ¶tû»7<ÂpÞð0Õšåèà9ÒDµÑõSä
+rzOþÑS÷h[äs6oõb
+wh8
+~4&`=9UÄñOaòû,
+?sñ}?aó÷
+»l|
+"å¢Ü'Ó ³Ìrø1ÒPŠª×gÒn4 °æ¬V°&5RîWÊPJÙà;0Š|Ïâ£0DOžuŒ'ÊE
+ÏÔ!Q<(ÝL$ÆÑhtÒ/R¥¡Î Fä2=ìÍ®]ÓêÛðµ
+IŸíî<*ŽÚØgôOV±
+¡öŸ°ÈDÃsQhDd |ügbà¶JÜÇH<Pz	Tæ/àír/@§A=â£:ªã9\]3òÝMufYØÀÜkÄÙHfyMqW=&Uëlm
+ÿsŒ4)¥À5YëüµdŠÌUóÊµFÄ]dI¢V[Î[&Ûy
+ü¬nb»º1¯åo¢w%ºÏ±.£èBiòéóèj¥¥éË?M>¹râÊo(Ýö²à©õðtœÇ|0 ßPuÅïMÕOÐåhÑô;
+Ä3×uš¢žŒ©_ ýIZü§ jeWßš'^
+ywìÂË1¢!$¶=LNKÈ¥*0
+
+­ÔB§
+"=ÏÙ©šÆT­Ðº0P®äæ
+¿B
+ÌBQµÐ7Â-Ôú4!·Ðf¡ÎZhž!
+ÿd
+¡Ú ç24øe<ªj¡<öÅ6NLL@ùxæ(e¡@,œ²¶Àþ>¡Ë^èýçÐZýÊÜP7S@GØpÑ
+&9y
+ŠX\þÓrd
+ÙõØ0(AÑ|VÊàH¬^%,°PIÌàxsD/y¹ŠGzNàùZZRCÐsK\À[§TÓnŠ&
+â%(RÏH9Q9ÁO
+Å3myþhçyð*}\%JãWIÎ@Mê&ÈXÎ"k­ÃBŠ3(BUqºuïµ
+ÿóÅàöý£ÞnÇTÊd±Y€YÜhÕ
+×@d©lª/çYÛIwÈJûÐ{}Iä;
+d5áiQ
+î¢bF
+A'oÁÊ7L1pb@
+ÄËúãÿÔhJ¶þL@O
+Ào3¡ª"H,
+{QûIy=B0Åæ	*ÀÚ4"bHFìòh0ZþT 5Ê³<q@}Púß#
+pfeÄBÁhô>(
+(Ðíæ2`³sÅX,S·6°Å>ÅZ$<@ß
+<Îšþ)gÛ£w¬v
+Û²	Å)3?_ÐG[éz
+,@Ã7Gå§­Dæ%Ÿ	,ö¢ëÀdfo1äêrš×H1I>Wö`yf
+I\	·aÑÖ§l1åçðÏÄ£ŸªÒá¥œU°E³6ÅàÐÆØt?ô<=PæÂpUÌŠ@^¢	J0üI!X5}¿}NøÃVT2(üa
+3êFe$ÔIIê)q6
+ù¬+Ë+Ò±`¡Úö9çÈálHCKÀ.^J
+HŽF{ÀAEäôì¢2qÃsÀZÊþSÔügT1öÙcó;dØÉÆðUûÉ¯Lš.À&ÁtvÉXáø[¯°ë&ÎFànlºÕÙÂK³Ï
+ÐSCö"à6xð/hc.ÔÃzEÔ|O,]ànûI DCY
+ØºTÂÍs`XDùÌÒ28ÍPAúöéT-J€Ö#£Q	ã@XPbÔB«E¬ÃMJÂ°ìÔZFÙ#§%
+ÄÎ.€1«Â"
+
+ä±÷ìí5'.K.ÉÜóÉÔŸñtZ
+3PlÜ÷PžFzjàŽ
+ôÇ²$pÐí§¬.ÊyäÐÂ9_f}ã\Ó~>>vÓ2ãÓ2U<l%((¯ %aC@€-§ šmºf
+nžÄ
+@th0²Â$iËÒkTçœ¶qJNô&*XÌhðü!CÅc!Ž&Ï»`JñGÑ>LR*+œ;>Îæ4.ÿP£=]ãÁ¥ bùAáõ RôR1`gT HErð (åþ)Fô"-jj¢Œw€ßYªÚê:Èhòf}17HtA?ÑöàÜ
+I1€°¶Oå
+weq-€à2\Öb)ÂníÂ²A
+ÃObVqSx;MEKÓÑöÍFü4ä[H%Éxj`°ô
+*bÂiTQÊº¿ýL€ËôÇr!hê
+ç6mZ
+MÍÝñœžP
+RŠd¿qÇç1Ð07mddhºÄ(šC+Y9R
+²oÆÙ±#bÂlAÑhé^Ë4ùÂ/`äC8YÉ NÇ8TOÛ²èòg%úÞîÑ4ÊS;ÑDa çÍ£©
+S
+s-}\5Hk#€
+µ@9U|6Ôv¡b
+Ò
+®-·UdõRzÅ"Ù§èb:2o`Lœ¢©^ej³#õs`FÏgÄ± ð»ÈKÞ@ôís¥3o)h¬F(€#kê,PóúŒ
+íçú@PÈûr¿Èèî3;Êi8>Å1ú9tò6¢SN-g
+@ö ¿ŒMÝ!XBaÊÄòÎNÛÌÞÕ#ãèLÊ]§m×PX .¹ý.N
+2>Å
+¹s&hÄÀM(f×ž@TÈ+Z²z€ÎEcw»ð6uÆ}pv
+WÀ ÷=ñN0š
+Èã4*àùé¶b¡€C©g¬{
+âìq7I|È(á@JÀ¢rP5šÔ^ÁU:GðÌÂª¥NHïàEê{svüf¬bõ³¡1Ïl8Úl]'¡á§¥N6L¢BBC­ilÈÌ«ÅÌ`j
+šéE
+9ýIÌ[Œ«kÂz!Ëã
+á¢ ¯WŠÒÝÕrq÷4È²HÊ¬	Õš`ð§tj74UaÐ
+ª²	-ÊQ¹¬­E¥éq€¬4ý0ü)ÚÑ/¡= 7ýiXpS«àâ]bèu
+©z¡ ã3ÔÓ	Îªðq
+ÈíÂ
+§OAWìòõç«N¯Íh@Ðã#<%\ÖzÄ
+uÌØØ
+
+
+Ù>I9Ìòç0­Îù4%îctÇäa
+J4}íD.°
+;­¡îA#u±v*ML
+C
+-s(Ž*y
+
+Æb	
+±51
+Be;±»È¹IÀ1úT$>)0
+ç0knv|ãeàº5)ùøSÄâÇú€bYíºPV÷+§Å'ÀCEÌu¡ßEUÀWæ¿Úhºás!Ü%ÓÜó2;óvU-T&V èî>'Ò"(Gº¹Äšu J«
+#äÄS:fâÙPþš¯ª&Ègæ#ÖózŠÙorR¶*FáÇ¹­Wµ@Ù{'è²èã9XLôX>
+Û
+rX°ë(qÒ3ÆTÊ¥ñ_ð»0È'(¥3\Æ8ÀT]8
+`ÚOÉJJ,d& ?Á
+¬?ŽÔÐiÊ"£4q;~b
+š0ó&È
+ôx82+kÎzæÌU©$$	wŠšJ}
+«ÕGcx×ª'
+@mª¶ò8>eŽVÀ`IelÐkKHÈ~ôÒòÌ.³ñYî	%¡wÐ(
+içûl B»«°«]&PøFOÍa9C+­] r7>ž+~,(,®Zê41¡%~Lã¹[u©óQÎ°zñ®©ð
+ðœõKÕ3À`fõ)YDÜ€"¡ÐG,ÄÔ2ÿØ 9v:š>Ç¢qNáž}>Ô£eol³§F
+]X>i"¡¶ðXdš@N5^/G47|Áœe/mjæ:ó0 Äq,~û8èÕï*ö#ŒªS9PW;±Îp²Tš$:uR
+$®œlþ§áŠ[4;œÒéc&Qr)YÛ\7"«ÀÕê.L	ª	­€(g@$0š¶šÆ,+Uàc"gm|4 Ë`NïøhLº
+?E
+3Š1· z i. %
+èörxŠBg
+Ök``zÊÐŽ0õ°÷?Idº:ëJ1{OŸC7MQ"ñD÷ÄÊ
+
+ôèv²)pB®ÞÐØ[vE
+ÉÅÆÚŠ
+;²PŸè%ý²ãCq²)ÃªŠ
+qReãÛ\°r:ª!®Uäá¡°(
+ýküWõ9cß
+f5îÉåzöÔ°¹ÂHµ [2Þ.>PUèÁStzê¹¥^jwÔ0jÀé{RRyúGYfíš¯ÀÌ[üÌrŒ¢Œ¥n1WÕ=Ž
+DÁ¡D¡gÒ*1%ÁÃe7Ê	O
+iE9ñ(0*ÃrMéÚËõ°«y«Y#5àÔ÷Ôü=¯qekÆ9rXZ?*N<Ç®¡puçA\Å>Î{P3#OQäÑñ»f«xi¢óªÒ£^uŒ|Rà
+å|¿H.	°Ââ­Í_YÔ%
+)fpDÜ)¥ãaÐŽ
+d%Árª±UÇ¢ä|qÍ ³F66nV
+£>PÔu­u.	X
+ÊhÛ5²y»þ2d?³!³STæášE¬·l š=d.F'2QòšÎâûÞM5Î#äÇOØê¢¹c2)ÿöùÉq/U£ØYÚ¿²gWµÀrórG[ÕÚÀgïÄÖ )²
+œZIKØË~ØðóÊ4:8¬ÑKÔLjfà1-²53FgÒA»Àõ &â{tüÈ4×ÖÀõÃvv
+<_;€Ÿ·ÖA¹m}<äKV0[*¡yŸF_Õ6
+ÎÃe ûÊT]^
+a
+LžbIøÞZÍ)¥Qàäw5Šm#
+G<4Øñ"JºIÕôL€{ÄDã°Xz€ú]#
+íGµq­øxJ¿€qí€aWÏD°CåUç­
+Woe4\úØ£ORpªàdóäÚ,iuíûŒ:Ô
+^y!à©ÞW{]?
+ÌA"ÒRÚü3ê)pHOò	5ã¹žs4m}Ã<€ÊçŒÊ\mÒ)UÆ
+yõÄch2
+m®ŒÐ#ŠZí|&â%¿+%
+K&Ç°Ÿ(
+.ŸÏQAZÄ!Z!{ZÉ®,x]Z#©V¢AÄtöëCQCa6$\£Gg5#|Âv×T,º=NÅ[ÅÉÄÌ
+D«UÒ9B,pûÙM^-
+*óo!X4K¬B"P€²{è8,
+§@kšÔªãw_àb6°Ç@aQ'Šx³Kªj
+Ùoc
+÷CÕOæýºSØ>ÙGÚ/
+ÏØYœ@¯Ô`uÆJ4VF Ëxd+ÐØ{	ëVEÁæÁ§	næ¥ÊbØ<mµ;IG<¹Nú%°YêÐFß€»Ís°ø,d8£y#ÔÄóŒf\·bÞ&pY€<+BžÇOtÃ
+Á5hÀ2 ÉW%¹.åví ¬bÑ¿ LOtÏQ°ÎfVÜ9š\G">c¬ù­Å_Ï,ë
+Ò7iÎØ`äuÎ1SðÞÒßüEgýH AññaÅèW$Ñ(éRâB]î#Íç@OžüÑÇRî$\!:EGYŒ!p
+±ºÓ&T£ìå¯m&²b0ÙœÕ9Fæ0Hã4$çÃìqÔüÔ?+Sú°RVÉàÓpcÑ'Ä ®³ýî-}ãXj$ní^ðúÓ¥¶6ú
+³ë¡]
+
+)0vúv :Ç<á
+œTÕ+e
+)Q!¡NL	ÆŠAEQ¶ÈóæVèPv*ngoìyÌ4Âqnx^*Nv4k¯TñQ}
+1ÓÏböÐôÔ7ÈmÅ
+TÅß ÅM
+|7uç-Rð&rwŸ;ñ*JªLdc±z
+IZ
+WI*ØÃbÃô38à,T>
+/Ö
+öåiè
+xWïlÏDþ(oy2LjðDh8h±óXÌâ7Ÿ7(â£dbšúô­À­9È\Ñ(JŽÈö&]S.9º-i²Î]ßa°úÒÉjÚf`ŽÄÒ¡Í Dd
+Ì>õ­!òTfËðùŠ
+ÆY0BÊ
+%ÎP·#
+ÚÂÌ¬p$šêÂÈ¢Íiáòü)Š2j1:ŸíÖáâvîñ\6eQÅJBOÍ>ÜML	
+ØUçíá 
+m02uYr·àÑZnÔfÒYÚòÅny
+ž[º:oÈ:Èñàñßæ2x"óAÈK0n
+è2Ði¥n|þÔÏŸZr0@ìÖ1ÖQ§>ÚFa(=X*±£îÈ~,s¯²~ÙVÐÎH
+
+¥V%Ç
+³wjDÐJð9Àž­B"fÍ ÖŸq€sáÜ?k5/ù©t°Óp&W	:
+Þ$qi²sô£{á
+9¢
+Yb
+Òr
+8ÒAkè¶f%
+É]ÅéÌPÇ§?uJ(§tf9ÌKðdPüË1cCnšÉú
+:;+&ÁB5MÊÛÔ³§maTëšŸmH($õð%Q®®|ÞÌ²6îhÙXk[ô@6&
+©,&1lkZ0 1O
+dpÇ¹Ž!Á³L]3Ò L ¬HÒnB8ÍJªÞa4lz%A# ïsSî4×žm
+wršl&¢Y°E2çé€Œ@
+e©#
+[_v À6ñÔØšM
+µ0HÉÿó
+TW)ØUµUNSGµÌGÎm<ûìL£
+Ë¥Üm§`äÇA«xÊH~n>€Å¥`
+ñN-
+±)ÍÏ³6¡ÚµðmX)«9ÍÃýC Ühðò¡ÈÍ×à§PÖ°2Šg^-,M'Qµ-$$ìÖv7ôAÎ 764è¥<F Ì
+.ø[H h1ðAP*ZÃ¯ž¿­59
+ù(OÁ/À|
+wT³â<vÃ×Lòº*RÍ4
+ÿßv|Vš
+žóšUlNPJ>Ûë`
+Î@lzEÓQ+jüVw 4hÙK+óE
+Ñ/r
+ '
+çmpŒä)P%Ø¯ÿk5Ô2'JèÅ·
+
+KS!§b
+kß!O¡S~À«I¡Ú*¿°KfÃ:±
+ÒÈ
+ätF³M"hUHC·5akRÝ"]äÆÈÞžk<[N:ÞJc§e	P2àr. š>
+{ý/ªß>eé	§yËD0`ÉL7²i¢s žtàC£q!8"êÌ fC!îzùç@K@pX(@
+
+d@Â@®[îHtÀÞ1<Òv .$!©Ç
+ZÅDDž¥ÏŸ%
+·t«vÉ¡@; È.
+±ÛÌ)ñ!
+Q±pÑ)ÇÎÈì
+¡:mêØu4š
+ÙWÍ5Ÿ'.XAôÆñÿP
+iØÕ2_[
+ÑÌÁZÌ·MHß	3ËT!
+ã¢)Û:
+°Éj
+%
+.i
+;èÈ
+œ@áátVéŽtÐÑÃêÙGò,Cm3á/Àé¡¹Y>áj|®Èö{ø)N€KÃÌHÏ(àòâ?}f§PÈFNsòŒØ(8NPh1 	w/P=s*nŒ²^%l\4p\ÁC.;M-ìÓä5[øV!¢À¥éó$@áØ
+¢FØ*\E/ûE2G#\¢ûÀŠ#äz¶žWH®) ³ñ²Ú¢}
+
+On$@SÜåJú³
+F^4ëQy
+H%mùÚL%fÆÏ ,°-öõ	€£
+­²uóÚvB,w©
+[¥À$³mü\VÉ£ýCòš|õ¡Îœ
+ànÚ î¥Éï!F#èæîZ·ÈçÂ 
+¹é:I\XöG¡BËnIšm¬" ñ²¢+ÛÔ2$²ÓÌHÐ$
+éÌXã$ðÓÂ±NCþ3ÒJkÅtpQž	~aH·ÐÀaPºSq€ScŸÌcS8$Ý¢Q{`>óQA±íJ
+Hå	g2N¥
+2é&Q¯åÌp fÝ2-â%
+îQu;B*S&<±šZÄÚfç­Ë
+B(ç(3®@ªÓÖzæÉ%Ú
+Â¶j®
+òshiŸeL^MUUîw²1cH0.»¢]w[DU
+Gº
+~sú;
+DæÇñóçºuš!OONæYžÕÓà7š¶ÎP£c/;W­å Ý ?·*ÁÿÓånGx¢8ðÑr€h ¬r«
+¬³îçÂÓ·ýÚù2MYo	ué3ÍD
+¢+ï"ÞY)ê¿[Ébm
+}O
+	fÆ±­7/
+×\ÑN6woñ:õvÈk§§áÚ
+"$
+i1\ZoW)ç°FŒHe?×eâ¡¶Ë€áD€øèÈ@ÁUÏÂPJ©d¬Ú¡UŠ)µÈ%&lè!ÎÌÌdŒÛÎ4·"äImb(°
+lºÔÖë§;@©rKsJO×»[EÙ·²i)nZ*õ!«-÷¥²³e±öMÒ$UØg5ò`ºø
+xYÔM1%
+ë6[í$Ý
+Z¢j}šlJFú
+{Á
+jžªlÍg]`º ³;Óå2%¬"Þ§àÊ	ýXåw¿+ûÝ3LË;Ó`E©Ò&Ê<­±æ²ªófle"íËU×Ló(YÓÜ*)	œK>>BŠå9ZoûriÈ£Ö¶ñ(³hñÀ¥»AêEÌ
+Úž@£÷ÐZUiö&SÄRJrŠíŸC$Pjg·
+­z[u±;<5's3¿$µÇ:iüØy¯3><Iÿ¡@>R$>åpG*Æ$
+4Wè
+Æ¶!ŠbÁ±ªóìïêª±cn«EiCÝ¢ïÞYÅK¥š¿yJÚZûÏ7	Ji3ÑZ2$ÜV²tXMìbXÛÅ]í:
+àÕìoÃù
+)SruKN©
+îÖÓŸD£óöÑxg¡î#6fÑ0e<8çjmX
+êzeiÕööŽ§+°
+øi
+Ò|¬^2:°_W'oW@Ks
+*Àµy²ÆpwcE&€emZžY`ßž}è
+[ä×±Ž Úø;zjúO$K
+#Þk1aÄ'
+ú'OM59ö{jüºê
+b š\ïYz]tºøÇ&·zëÎvueY¥ÅkðÓÅA«õ62Ð]SI°
+«_K­
+Øšd
+p_
+fÜ %s>å6ÊB·+ÊA
+AÍ=bfù
+¹ êË«EaøHÔì¬è§M^³üÈ¥
+QYMÌu¶yÇŒØñ¬©ÆçÇé$z
+qôm×_°èíòÇûlÁcÜ¶ÇS¡àÉÀ'²Pb-üeðv(¥ T¿ïgô€c`_ÞEÔB
+y°*ºÜz@ÄppI}HàFgqËŽkæ
+©äQužWü
+HhxNè m@ÿäÒå}	âÃ¡eBÁÏåÚzHfîúm®õì£ËL®]ôL	
+kA,ky4ú}€Q»à±6Ö#LC\bèUj!Ò³ïU³È%
+{+Iä€=Xö-N&±éd×G2÷zê-
+ªj+¿NÈnèôû$u&¬4,ØèÌH®4[ÊÍHÿ'W_$ã·üâ
+â8Ë
+Xoªôc\bà9ÎŽ¯õ^âý)Ql	Bé÷ìCXµL 0ØÉlf!Q#{9³hØÎDHÛßKý4
+
+j¿·V¶²E Ýž€qÐÆi +°ÈîTm	~
+4¥ŠÂÅ{YUK)S}RÑ_kïüqÁ()
+~«µ¬BE3ôêØTö¶'^1å_Ã@×SÚÉŠ¹' ñJæ>ÛCÙiUãy?Â×ôÍLÙ
+ »	?õºçtÊµñøqÌ@ÀŸ_¢^«æð_»îù}Ñ¿IkHÇD8Äë
+ÄÜ7/)`Ð€²àRð>x­·ßSÄ ÀŒÖÕÌQ³°|L<Z4îbêýŠÆ(8ÕÖ3BŠ 5©Ð]6A^VëfgL9Õ°f
+{Þwp)2Ô%ÊÉpSÑ!ìTÌÉ]pãkËž9Æ
+C
+Ï;ç2IÉXKKí
+ý`t1{J 2g[ðÊÁu&¢îÕºº
+€¥àsçlkŠÝºŒŒ#8£ÉöI@bøzÝ0*]$Ð çbP?Qk
+
+ZaA!y&pøê6ÇS
+ÆZÆ²|±	jvX(Á»Õà${÷1[¬¥B0/ÅÁÓYÈ]Y~ª>M^¶€üÌ)K1]*å@Ç¯u
+Ùb!Š(k6CY@ËtŸ4
+|Rëqf^È 5#±óJÃÃ(¬@®æU­ûÝfü	ÚÅ#P2d8¢2Ï$ÍDJ
+åÅÏ68pTJÜ$ÈÝ}oR
+Ï`fWÅ
+nØŽ
+œ0(êhèqŠíßÃÞb%wåZ(`1]KŒqYæ§ZÓÐm4	Qç
+2&R€1Ýs=îíoù·aÔ+s|ÉÀ¬³Ê©^â}rÃ	îÒÜ°¥È9è_õkÁ+Ô'ºž:TU9Lõ),ØÌ²J
+ÆLõ­ÛJXÈN7²ÍÃÄ9_Wð
+îBÔ÷xPèÙ(¢
+c<Ü¡(2
+ý ®DäV3¢dâYàßVåSo.áŽ_Š¬äË+»Î×à-_žpÖ@é6å¯p`Õœk¢;š0ÕéW!žöÆ£4Ï²Ž!B±­!r]ú"Ù¹øã³âìû}ŸŽÓX¡D1Ë,;=.aædBBI°ÚœØÌè)ø³ë°pŸd!ºféÂÑûjfTâÈ>¬>ÒOŽHÐÕ[Ø?Ÿ¿ä78Ù=.¡Vxcô5ú!KH°ÀþaCòÕÊg¶#¢j-XïÉ@øU[€`Cª:EWU(õócf9.pÑjx)ªGJ4Åä®còhÌýônÆŠ¢ãŒ²ÍZt©ò|œR+Sñ¢BŒ
+at5_ªŸîGÂöFx$Ì¶)ÞQóØ0.sz]%4óÏBqìž.ZkI3ÜPTnœ¢/Óç
+Z¶À:<<yÞJ¹HOL
+.[_
+¬ÙmŽB&öñŸC$n<ÐI!°4Õq]¹qCºV"QRÎéÒPa,Ïš	ù·K&u®F|j,:ë$|ŒšØ¥ÐrÐë
+ª³ÆÜ¶ÂËíeÖ	ø·æ¶2![öqMÚr
+ Î
+k·ª4.öY^v
+šž
+ÙHùÌ~
+:Áš`w¹m|x«Áû©xbR¶°žù£p$/€¡ßÇ *ÖOýø"
+|&Ä®žŒfHO!76	 ©Ð¥
+2#C­òì¥ÆM'øB0ùJ#Ú=0íx:k®
+§h}xŠ¿ÏïçÈSÈü°³Ör\éÕu\qXâ
+	tIºŠàeÅc
+ Ââ%
+äÑH~œä+U~WÑO1êU»æ,Žs0Ä`èÌòCâ~b L
+³£Æ íÊF¢'H=0
+ÑTŒQ{VJ`åQ#ÂùÆÙ}ìÖT	~cZåÃ×w,Š	Ó2-+P
+$èß©ô^.
+1À²)E|KbÓjÛh/ñj`üÅH;M4"K¥îØÍz0?
+0{ÔñëÁ;[ù^lÀ+	®§ìÎŸ,â8%#Q&¯ùLÛBÃ
+0eäEa#Óz
+tÏ%æ8Îô&õþŽÓFÓsC³8>AÙ\÷ÑöBøÁ&íŸ
+ÿÃ^MÉNšËÿj×0`Ì7)ªÙ0®4Ý%eŒ
+P³PšÙG+"GìA»ÐŒÓø@§*r
+
+ÅâUÇÆQL¹jJÙBVìÛüÒÄÊj"Ý,$K
+µ¶u³5|+±­Û?µlhÄ>Só¡Tîv]Ü~Iå\E>1±Á%ãTå>¯K$Þ`yJ
+Ìÿ¥v8>ñª9Æcq^3Y$±£ÄF
+Qáì%ûž«X?]ëS¬È6È+±­F¢Ö®Ö"ËZ
+W\Õ	ÕÈÕh`#uë0Ù/BŒä1F!»ÁBAm7€ZÌ}€4>¶îX¿çPªFý&$ë°ãÈ·Ù<q}³ÎEµ¬7ÄéUÖûd_
+J
+<19€ mDìèd`ÈÝ7òæ"Œmûz
+ME%è?€!Â×@òJ4
+i©lNQÌ_Ê}
+©rdQPŒ@|bÐà*TÖKÉ'Cq|Â±­ÐÖ+
+$ó{
+
+S±5 Oå+PÙ=Q`©€ý
+ïæëµXšÆ<,Zd+îÏ;M6F
+[GÕØQrå(ïx-²4à-
+ éâåoêFlÈ£Nt`
+ÆÕµ&((ZYÈbZÐn	Jõ1Ÿ³r5q¥Žf²ÑFv$»Ü#AWCËGŒ€(ûØL:=3"OªGG+ðyûÀüÜ±C^"W0¡]gjÏ
+òEHôÈºHÂ"íq#ž;µ$M9	A
+è©?uaGÞŠNìÎ$:kAš?m:šq
+ËÀÆ·P	ìâ¬aÍÅe¥ŒMQjÐŽ
+PÙ·U)Ôîáµ`Üjo®ÓtÛTDSë
+j¶DštÒ¬³æºÖð7ëgåóŒÀzjrë
+6&ÄµdÏ|9Zf»P(ÁÝMÎúÓV	ØJWê<5k¢úpÁ7(Ñ	` 
+`4ªI¡»#
+£B'»šB~~©%)6Ik¢0Ñ)M5*»7¬žhXÌ
+B)Û£BLðniS[¡pZëMY¡
+Z®sm#ŸL]GC!°ûúÿØ»0°*n0ŠQPÄ©Æ¶Ó£
+ 
+ïÙ
+î0+¶å
+Lâò2
+ŽÀZÇgÄÑº€_ðéÃ"p_a­þ­œe`ŠÐ%hóŒwÕ~¢§ËžÑ1Ž+xÎ©Î©î#ÇáIxË-u
+dÊa_ôøŠ
+Tvè;O¶,Â+Ò¢D
+]Ymë^&Ëö,õìúfb*-³»ÍÒjîZËHoËY&ÛC}dDv3;ìÁg}òœ¶à[â¬í2ÌŠ§ b>Ù4ëQ²L,c$z.uÅIã.,ÓbN€|€0±z>	
+®Ø'±éåK°U]
+`ÓEÑÄ4PWxÁ"RèYfÉ±Í/]d)È2Ê> iÀcTßûÎ³
+8ó<KºÛlŒ
+òÔºcaŸ8d_Â2ÒÖPÁ<Wž}`)²w²ÑZ÷@IægdF
+°Áá:(`ÊW÷åLEÀè#~
+L.-PkËŒž§k*LÔ)ßè
+ÕiYaÆ/
+0Ìæ]îÁšb[7fgÐòX
+ÝØ4kæ6¯eXNú ÕT)<R!¡Ä9·Aß6;Q
+E°x
+XøÅæøEœüÄ±©}TªÃ
+,|R=/Ÿ?ÔØÆÆB&(&W£Ä]HNnbøššmØ¢.*¯JT¬R.-DwÍrJÌsHÄ
+tFs|Ž'YR25²|ÍxÌb`%ø
+5`>x>ºldHEæéK=Pôä¬
+Ìàó]0ô­Ý&Â]
+Wß(pF®5ò
+ÆÖRÐ
+ ÌË`j\!
+~¿rF!\qÅûtÅ`78ffåÔe y$iîÉpð+ñž*ÂL£Ï÷åãp³Ü²¡Æž8äñÙjÚsf³8ždv¯Á}¿h°
+Ó©Áÿ_Ìþº>ï4õp¡îéŒ^Èa,Gâ.å^Ç#ÞÜ7í» ÏU¿,àcë\
+£ ¿fÆAVô ]bD7g*€GZqè900*
+37dhfõª €%ÿ ðã
+øtFŠ2·aI8åd
+Ó`»(`Í@:Ê
+Â§B#|kr
+$Èü ÓMiKuVyf²ÐÆf[EIEúÕè%Nxz3	¿ÇÇàÆbA< ZU'@Y rŽ#utõÎÍuý9×,I>
+{dÚg@Bw€o*éx6.Xjšà€X
+Ô"¥ÏGfÖŒGäŒý%{5$
+.YÊL-yx,©·M³	ïé3{®3$D©ÊÇ(ïÌÕØYynwÐc1¶(ÚB¶*2TxÖç€]tZ6Ï«ÑtÇ#IB8|ÔÂnšqËkº4
+PX·9ò
+`kj¬oŒZH
+€i!|I
+þÙ
+ uJ
+eV#ÃÀ²¢
+TÛI1¡!ù:l'£®,Ro®sèû¶e¡DY*T
+[{Ãt`#·ÐêèëaþdëÆ<ÂÔÓ
+üfjL#	M5§g%úC>J#{]+Ò©!œñj(Ì¢Þ|è^'wPÄûnds©êà1côöÁÐ ¡ÌÒHË
+HOGŽSJ¿ÕW°QLÎÙ_p1ìLrr$à-æ,3Ñ0P"å êGJÆHŽò«€Ü-¥ïß«a&Ÿª%ð±j¡0íûñP&ÿ Ý	ê å[t<§îªD	BÂ¶`ê\	âªUºr
+p€EGSæq+©2Ž¬¹TNâÑ,»Y-pŒUs
+&¥/Ît
+gö5c
++/
+o7áQt :ODÒÈ2SGñÃx'Q8.®w,'D.ÔÝp
+ç×*šyà3LNdg
+	<Ÿ
+Má×Òñzú².ÕªÎðl<ØÅBÊ Èün¿9ÝÑÕØMæqº¡BQ¥Ê«ÇT²	6jµè°¹amî®@&)bŽ×º5Ü
+
+œ@Ÿy×Ò¢J+=%þMâèÅ'ãÁ[7;ë÷Ÿy
++1D8Ð±¡aæÈÖçÀÂÝªÅ,
+}>Âup&ÝýŒ*ÇÖ/t
+Ñ¬1ðh~Û¯Û ºKRhÉ[HUÈ/"'·¹9ÃŒ)ôÆêG&×»œ@T_*±ÔqCSÆYg#-Žxi[Šj{Xµ%âÇVUÀ
+î%K%4:Aã ± LŽœRÄp|F£ârkmÖ±Î ©¶tÍx8%VèÙ[-lÿ[ÆùdÛ|¡óÇNÃqÎâckùý`M(ò£íY(ÕºüPQV:AìÓz]F#5îÇ9+ 
+Ô`4Gä9!
+Î
+äŠÈ`:P°íëÁkltZÑÛÓ5ì
+
+Ûa©®[
+ï|5A]Ý#	âäÁ°$ÐãZ-pGDNwå4k
+ÃMó¹jŒŽÖ4m$
+9(á:äîÏÄ%
+x­Î5%
+jaWßÂñe=-
+ÊCÞ|
+ãÄêF,¡ÜZ6Ñ[oÎïàY9Zô2ç rPc`é%I£¹x
+š5óQÊP$D"â»"¹¥CJ*šµ#°LÃÍ`óeýÈéuw3¥_j
+u&wwúÔ:RuUÌWoÔÆEÍ.&:
+"2³âÑnÚöcº'5by)X[¬
+63Þí3¬{Æm¢®«Ü¥'¹Í,6:@ÄðuÖ3ÞóÛŽ÷ªš40å'Se
+e%Í~'¯)³ô8[ñ©
+°¶(PÏhÆZóVf¥?M~«<Š Ð}
+v<·Ù.;	ažûÈ-6ÔÜ@õ ZZà\pCÈ	41âÚÝ¹'rbå"î?y¢0ªbÅ>&@ËuzBÍÙ GÊÔ×8ùÌ>uÜyhâgvêI
+SOMØ,á/YPy)íRúÂ¥àÓ*%%Õ€04š
+è±°ótÑ
+üúAß4€Oá§Ï4qrPþD3¢5/ÜÁ»0GQä4·ÄÅ6x	Y=ŽÀ.D :¥K¶áëSšºÂ[
+_
+aÕà
+Î)÷n4e>ò¢cWÃ@FÊ	Âú«
+ÀcD;l,lð-AÈé¬ÿÂe×%îb|œ=T€3šÆšÇLz4  ŒJð°AÌyà£QÚÍÙÓ=JYÀÍ+¹ÎŽ9ðU5qbO­&/4'ØòÓ\wI) ^BÁW39téú;r>?gcZF­ªVeLšo³h¶ŽÖÅÓ® ížgí8yÎø`av9kÉŒrQëÍ5C{vÙçÐ¬à©œÒÇÀÖvPØkþ	Xªí¬àÄ7r)¯\KgP+NË'³_:§2
+Ö"
+õÜÂÔ%!*6õ+ýÃî
+NWÎXg
+7Öž£h*©ŸµÀ^õ7([õ«÷!VJÀeÄ
+Œ
+µh-
+¯ýôYRY#!übæq4£úáÕPÑa#]©j±
+Lª	qïæÆåž1ÁªÉ"ÈòEd¿8ËÕAIÔNëu#cŒÂ2yk @(f¢:až{
+¥2FyF³Žž@RoÚØ
+©e&Ä
+?ÈNcB\x>ìwhü?ZèYÀ$çLÑÃµ9A°ô?ÐOA?xÛcô]çŠdÌæ]d~c%Iö³[ŒlÄø^ÑžME&h}š2QLÔùÜhŽFW,©»Ær5NžïàÎÌZØc«L{Ð·À×¡}
+rÊ5À¡+oíìV×ÒVž
+Ú4^ÀY°ø*óG¡]èÐª¶*ËžÇ`ßäQ
+#[A&@ÇØ ÿ ç\ógö_%LÛ
+POtHÔÿjÝSð®<MÝyô»	=ûž\€A.€¹ä9J,gþt¿ŽÉçûiÏu+ç€6ÀÀMÚâMP	t°VâÈVTÅÀà
+iI°ÝEM`LäJ!\Ü0	vôÎ	u¶qË
+i"|U;¢
+ubB4hVTH6|é~=O[°Lª/±sª
+NXE\»e²
+l
+Æ+²ÓÁáà>	Ø­§µµ*Öb= 0²9ªqÙDÕkÜb íN€eñq KºåNP©)
+å¬Yk€Êa{"¢êôv>D
+ó£q#öÌGh,,i€6Kpí:kZó°,DÁÆ 1«¬2#õ«2#ï£
+,·v7ìØ)ÕàeLÇÙTñ<P¢ÌŒ{nSõÛž1FííÖcÖø*
+û,Â,ÔöÇC°8äõ9»­úCÉ
+|f¶Vž"[ûE
+Fß]E>#º^w(BiqlÄ\Z}õw¶÷(3 /Õíá?{ H¥ [àºmC_
+©¶ñÓþuðŽw ñL3cchÀ> šr)313e 
+ŒÆJÛ
+s¶,»²1RVCžXp­ÎUÍÒÑæ¬
+èK€CtŸ	Á
+]cš#ÿEº]÷¹±ô;¹
+ŠØ
+€`µF`ËöÒµ!¯lsÇ9ÛqjhS#ñ,×øŒv¥S
+)Ð`<¬ÖÁVqöçµ
+>Ó1@=#`P	8)$UVíAváùšü`=7uÞc¥öâÆ
+/æ9H±­l
+Öö	=PP()]Væ¶[;°qö[a°
+,7yeÚæcsyË&à\nß^9çX@]hõªc
+ðÀì@WúN×h[\ž×[°ýŽÆàé¹¢ý*_
+]­ Æ[šÛÊÁ1$&0ø$ÁzÝ!/À6t8QÀè	DíÔ×wâ2Þ,
+š/èÁŠEx"
+šT¹£:ÑmPÔH)ž>XZ2é5ŠãŸë0éª¬eQpgÀŽDÚçõÂ
+0­ÒfÉÈHéIí€Ÿj'PsÀÙ©<X,êÉä +ÏÚúÿV¥adn»Å
+'ŸåOpM-¶ÌŠF¢Q«š,À¡PŒfÚÕ
+û[r(â8:I]Ó¢¢Å±€¯¢·P
+RLZkÇF£Íª
+c+s¥Jk\Á&#g°Yñô{Ñ\ìcèH«­ŒÖŸ±@ŠÔ/£ŒÁÊsyÖ¬ìº"Î­\]WïlçŠ¡=HP
+æ>&2œ:­IÅ'|¿4a`Kw!(zD}hLÅF
+§G&Pp
+ž
+ 5ã~ù .ÈABîGnÅúR&ÌéÊµ³æÈÙ UL
+En]áµ|î©¡f?tu€R±»¥Mi|¬~ŠZ#(n|^ñy€m»&
+q
+
+­®©Tp£â¹ËÂâ
+Ö6_¶ñíS£ÀßÇÚ¿mÓ€MûZûåp
+üí¬?Û³;Ým·Ÿ¶#g?xªšž×ÎKEÊ
+Ä6
+Ôàþ°Ò üMsíZPM
+ûâ±
+Èh+]èãb[9VKÅÇ×¶y³øRMö/a8g
+¹
+
+WÉî7:è¡n fèQÂÄTùÜGk£RtPâØÌîé@RèÌpíyÎF#ÇmnÈÇ0A}ŠÉR)Ú5T¢²n5:çå$]Cîj7­bVü"í3'`c2Î®»%@8&@ÇóíÀBP|Ö'6=ÉRHp%X°ÇÏSÚ|³`.XVÏ$µe(° 
+-Ñè¹i<ØPHhð4Kzªçãçœª~kSœ¢öR¬
+Å
+Ì8y#¢Ò ìpk5=µeMÆHÌš·Qp-Ñ¶ž}EÒ²KâØïg,ß'MpP²1¥uH€í0g';å:(×ÖUXžÔItIæV^ÃSvkœV¶ÕžÎ­©`Î0îæÑö¯ª,®_SLç
+GýÐ?>
+JzB«nÝ3ÞÀ
+^#yC©­Âu&,¥^ŽTpw@àÑL³$
+é
+a}xÅIQí<p^¹5$(LêŒÞ²»ð
+Øh^I0Ž^Ìœ*5F91"éò#ÞÚïT÷€ÏM«Qv
+WHŠ¿*¯r|5ªïÀM
+æ+øA
+Iwå¿0ðžp¶o
+é$ÛwYL/`K
+1ŸÁ"ÓbD<XcŽ\
+N ù_G!wðTjW
+¿'¥lVe
+žÕ|m¶RÛÊU
+­}c¹oÐLÏãòèe]šÝ»YÒNxÁ=±^xß®÷¥v
+°àÇWÛÐ
+AýÀ}Ø>nÍHûhw):w»?Æ¢h²Lšð->]Øb4]£F)Ÿ
+ùå{º(5iLæ8P°C\w%ž#`Ï ÞîÎ7Øõo-õP(nSãG§ñ<®[1"8	©OðDØHsÚãñŽ\¬SÏX
+
+üÞ-lè5gÂó<k
+åä=Lš3ÂuÛù³
+nU^¹te&|Ã
+à¯èHN9
+U×Ûmþ·K×Løö|ÒÄ
+falfcŸ÷œMÂ+áì©
+³cÓadtr{ÿqÚj(
+­!àÀu§ª5¯PH»»ÖlÃù
+dQ'§Î5Ùú>_êdÎ_BN/ïjïfMŸ2,
+Œb÷G	µýXŒd\ÀÑÁ1PŒüyh³ŠÕ
+ÍúÞš
+@L5¬@3mÊfvÓk=¿'@ž
+æ|æ­×!°`
+ý£ ÷°òãÉÖjlGú£Ò²­ÍìVA€yVðïXÇ}Ivm:³Aùð
+á»ž(Ná"ø?
+/<FÖ#¹MDfYáŽ>ócV¹rRüè
+i~tÚ€5ŽÎuá·nXM
+q%t]%âWI!E+:¥výWÏìMtuîÝ!ÛxUb
+víß§ñÆ`©VO}>}MŒ
+7¬Î@
+5ŽŽ&PrdÊBê	æXž
+w}T.}JÛ¢Â¬dÀIÜÂôiPx¢· JÌŠEž²>XÂÖi1	X4Ó	ªøuÛÖ
+;Ò
+Û\AU·|FÙó'8m\Q5îúð*à¡èO¿
+À{C&vOÕîN&qëõÄõKïè×IÇUqÒñèž×»
+w" ÄÍ&Ê.h:>
+×§âäãqìøv\9^EhÇµ`Ì@õ58ü0ós0%);x 6áåeàLèâ2+Föi²õ0ä£)%mãOñp8s|êÄ ®WJ1ªÄ(Ò Ù» î
+æÇ
+\`&Ž,
+Zß
+r@b<%áÜc$|Yí
+Ûí§fÜéT"îx{
+ð
+ï§wnù!Ã¡ÜïÈÂH=Aa ÐTOÙÈñ:
+9\­$×çÈä1råø\ P$¿«>i~Ÿ!fêŠýââÜYX#>.ã
+)]
+I¢¬u>&]P¥ÏöÂ
+ëÇE0aU4w;z'Pœ@#-®±Op}9ç6±µú%5Í!k³¬`ÿCåVbñpÆN7pŽ2AFÎÚ0AÄã×qôxvÜ:Ä&OÊ+ÙFy!Öô
+¯ÝMÁÌãÒ1ê8³)=.)¯«ÇäN¹sŒâ\
+FbÆ>²&6yó¡à4&
+9ÉþNµRÁjØ·Š~
+ [%Ü^3°ŠäaÑÚDÛ
+ŽM
+øÝk«CjÙöÃ*ò¢B`[N Œe@âð
+²X`
+wqµçFx&®øóS¡¥B2§¿ ^Ú'?µs¢ÖÄŽ
+Qx(ÅÏÙDMw«2º	'Tt9¹åR[fùÛ1áuA{WaÖóiÌ:51ûN=Tf7æ)cÌ÷pWWC×gŽ\ÇÔñÈÊwEgåâ[ŸÖék[ü#sŽ±åî
+iHPE&e  5c(11+æ¬µwÓÏ¹Œ*
+uÞy&äÇÃ×VË.mV>mª; b^ë/ìJxUg
+mÓcÃß¡Ð@aÚHL±&a±šá§9ÝØòÉà«Äá)Û~lu:5¡v{Íù
+å+yâÀf%iž,>*æisÔÌl»Hæª²föpëì³D
+o"@cå"'6ŸemŒ|j?V^!ûTGS¶ÝXI1³?+Ì;¢µ\t«cÐ ¢VÝDØ¡¿/Ž	]ãrúGM
+­È}D'Hà "ŸÂ§Ôfž*ÙùGró]	/¡Éâ,q6ŒŠ!Éàx
+].k
+éhjSë
+e%ek¹Y¿FÊ9ÈÒRD&vàO!ÊUÆl÷: )nÅWA¢ë9nB+9aE@·
+ÊAV-N)¶ ÖÞÑ
+¬Å¹s¿lÕ<
+*Aãçg"Dh7çÈ<_õfxC0y©
+ÊgŸ`¡Ç9ûmg8s,0­Z;ñØ^xóKþoFLP¯ÆWØðYDè$Æèi&ñÎ
+è&ÜàKÍÓŽËÔû×ÑVÝÍÁâ¥
+À
+ø[Yn©43ö
+0LïZ-ñBÂ0NŽ2}ñ^+ddÆrÝÉæ±XHáR
+[
+Äcu€ìhK°eBh¹œw4+RÐiqÿM7¬¶ÂÕ1Ã­t ÇáJªþ®enäX4Në.;ÿ
+®s*aûášFªp`æ/Ó]ï;qyn1þa6³¢ goE¹ÃñE
+àOsµç3µ)æèl¡¯`'ìU¶©;©+°{¬	-$EÇÄ¡dôÁ6
+®jËÛQà{§Ý
+Þ9lÕ'7éSÂÒ{ðçÈÌë6È,FÔ+»ÄÀ
+8ØJ9u^.ûÛÎ}8LjÕK'¡©
+ÆåbqHCÚÒœ
+ö@
+àÌÞ&G%ø`ò»`àV6Åœ|BÝ,eOÎ#Ät­mPØjB³2
+¥Á\ô
+c4L
+ä¬§À0kœüŠ|ÿŸÐzc.ÖÚÕ
+éüqÊŽv°žo]Ÿ^N_·Ör¿ÅFÎo¢¢
+Â ^¯ì
+*BÚÌý
+óŠîr{Ü2ªÄD÷ð¢»ÔîÉÞ*8µSÏD:GÐà3×5%µD7&ÖÔuÖ®|5¬MAÄaÞìžiÝKQDwdÒ
+èüs(ºB`E7H¹Žä1C&öÊ
+ó 
+§ySéOëáwÛ<Am ­I"iÓÊrýOz7Ÿòª"Ë
+yá{ù·Ûs
+Ç6:èÀéd8ä ýðk5
+Eõ%Šíf]ÿDí0?£Ó7)CÁ+³-hõ×œçw&7j^OÇ>š­ŽBÙÞ
+Úg¹/^à@Þ
+çö·bØU)CÕ+Û)Œ¶Ñ
+×
+ð§àp_žS&lâP82¹è: õI5G{c$Qåá
+øk-ep/Í@y­¿aøŠ2/!ù³r³~P
+íÁì(åJž,0FÃêca}/r-`öÅ«
+?üö-ü~pT52ñ}¹ 
+iÃÕ·?C¯d\]RèiOž0ôÒ
+àÝ±^¿~©74+a
+ÚÀRG:;d°Û
+RÄÞÚó¹þ;
+Ð×ÎI ðäq³œTÀš³BBHxð6¥J'T.èñßSÇúsA¿ì­1fØÊY¶ñä
+_ ÐC
+ÅÈT}Õá"÷žÂÀÜ*-ô
+|×@àCÒdŒÔ°1êOóvcêKøä
+ì`1I¿2FQ·¯
+DV€Z4Pc`Ç`ÛÖpÃÉÙëí8z
+Q³
+ 
+087ÖÕÉpÝZ(Á[d:ûkmÖXkŸzo>wíŸËZXÖMÝ6³[)Øk8/U%ÍV2HÅTìKä5âëé'[×ŒCB
+:ö
+
+çq_
+÷á(%1äd9ÐæÊï­ÚÌ`*ÖXïYa
+ýTŽSËo~
+I
+	.Jè·v»ËE
+«ŽöÀÎs!n€jT¥M[/©ÞÍìÆÃ±¹g{Žñï¡1
+vÝlÝ
+2ê,ÉþÄ¯Ù®
+B­\zð4}ßAŒ]€"úâ'ü®VB1¹·{pÀ&Sy³AíL¥d±œQ@TG--,t¶ee
+Nê°m%û,­ðlšô?L#	*A},f]ì0nŽc°ÕÚËD
+KF@œicçXÄzBÇ*uïtp¯jë¶¯iân(¯Â`'Û{ŽOšêC"³re ÖR0&c§Œ1
+vì
+»A!ÝwD±ûüBÖ¥Î
+eÎUhT!ôZ9Ä@`­ÉP¹_²#âs"8Ç'Hpî)|QäsoÜnc¬HMÖð	¬îõ	nêN#€<|¡;paŠÉ2ì
+6Íœ€ÐŒr
+xWo£ŸA`Ár¢°fÒTÆ$B.ÁÀ/O
+!á:Û ³/
+Ö6^lä¿
+BHÅêó%ÃËèáÞ+ØUiò[ìÚGXD
+AÖâb¬	Qz¢ßÆ5\OÞ$ç
+üC3P¹ íÌÐP¡ÄÝÑÓ'Õ¯Ù\y		ÃžwD+
+£ÌqùRÅ
+3^?@»p(ž{Ý2Ðòí×ÓêGKÜÞºŠW\
+DªeKÿ`ÄBõáê;æÝdI0€¬±&tahEå»;¡ g_6s,×VzÃ`Y¡(Šáþ,F¢€ºC^PÖôý"Û8nœngŠmâ÷ÁöïBûÐkZþŸñ/â 8LÚ±ýöÇð²ßp6@Â¿>!ž
+BÜµ¥cÒvÌsº·kWàxaú{ù%0Ú)S°
+Jt2àT°NDWä
+T§
+;sLíÕ!CøOÏoZ#~ð~GÐãQ ÂÏb/þÞKHObÂS/RMPxwÙc
+¯Âoá¯ð5K
+#à)Ã#ái6,<û ÀgÎÍváð:5<Þ80ßm-pÁÐðäÐÌÍ-
+
+J
+>Ä
+ŒVÁý­#ïG}%S¥Ö€0ÌŠL|(
+d%%ü¯_Ä7âSxt.üQ?gÂÃñ,Œ~oÄ¹qÀðixá(Ÿ/ÿÂ;á[ñ§x2Œ€j	ËøTäð¯'N|;¢ð@
+U z
+ú0	 ø#°<þbé(tÆ`ý,¢PÕr.Æ|ãuÅ/¶2>žQ©Ø
+aô|äm,FîW9kš%kâÐãÇ(¯Õ/pÜÌ$¡`¿H#SÁ*ø§ ªöC>=MßŽ¶éA¹åÛ~nÑ€ñUnf3»w>y
+\rÆÔŠn/­×}!e}pì
+!Á}TN-Jí²ZÚî:Š"^è,!²3qs?øÍâg&2<-
+ÿ
+Â³á%€ø!üIEö
+F`wbàÄµ
+Q¹ÜñQ<Ç+0\²uQÁm{P:£çÔßî6ÿ
+Zªîø,ØÐ
+X5:¥ŒËl[ò	Kr&fhDjéw-öRòhµº€Ëèiçú0B»
+\QÓNr>qSQGt­àît"ab©ï
+¯àÎ]Eò<
+Ê¯É£ÙÞ
+ÕMsá4œš{­µÖñ*³®ªo|@Ô©
+ÐRå0LC¡X¡.}qA«1·Õ©'
+€kò!6"šnr[ôÖ<µ[¢)ý7FràQM#êü#.[§ŒÓÌÔìóG¬gíá
+:-
+úÔ|Œ5	Uäà\¶%Ëá})}Åy e(®=c£ú!q«ÕDY0~1 r3lx¿
+
+Mw|¹1^¥ž §GÈüÇî vuÜiŠäfmãhZ¶öáÅŠsŸOBõ¶fN¯O5õëgõç(Ù.g6
+?ïjAÑf4Ý
+N,c@¯LÌ¬Õ?kŸ"2ðl2
+kbò¿.ð­4ÍþKuji
+Ôÿ%ûù1\§ŠÇÌ
+xo,è
+æsN3~pL/\Þú¶9Ÿ¿
+ñ·o!rA%°l@¹9úèDWý{@
+6µEè YdÆsnÈvx€š,îÔØ7ª¿Œ28ŠÜg±CiÛçµ°v
+ð}Úß7Ð¥ÓÅ
+ZcÚ®ì
+J$7ß*Ð|B¯0:ÇU²lH8
+>
+ôàïŠøéŸ¢!òa	0`}ÒàW"ìèq\¡
+FñÒxk¥id=Wºr±ìB?ëR¢*Ä?%éS7ÝLO0SzŸŸyÀ»ÈVüÚè¯9ÿxq?ÌZÝÆÖæ±ñš±8Ç¶µ5@ó<Ïå«kè_pÝÚmŠå(	+ßÃ§Ib|ŽêßŒx1Æg¯átæÂ­dÝ<!çJG5Ö(¹2ýàðèÞ¯6ã.oþTó¯IdµŸ3KüÛ0·$Þ'uT`Æ\¬pgÖ%ØYfºÖËþ²h¬?œHNv\5W8š]÷(ÆÎ{
+ø]F}Š×Èéð8Ž¹šŠÄ|$)Q0ØŠWÂ¶Ô­CYTb¡œŒt?7[©xÂ`Å8ð°?ÉÊ£{ÐòêÛcœJg?f%'8c6E7Æò¬«JâÁpêŠ z
+èíÃ1aÎáÉÒ 8:ÓÐE²Lö)ys×ŸÞd(ù u«¿5(t¥|Mõ_ÉXlV¢jÖªT4¢$ûqÈ<ÇEËÑ×ž5Fÿá÷â ²ß?I4¶HÆiP¢tSéÆŽCò€ÐC$ÂG^Ñ[haxÚ?0ê
+®\HmÝi€G_ò£0ì³[
+6l>áE6
+é¢;Ð-8Nêb§¬²L
+©àòT-0ÿ0þ±Qlßl&k¢õ-Áó¢ì
+ŽoýE]ÏTÈºž°ôç_yÙC€0LÓöÛŽÀÆ<Àµ6dxjWŽDwË*I^  Ž­€å[¹dÿ
+3Úä°sJök^éD N€všÐûk
+^40×ª ÞvŽ9hÁÓ'³oK
+Vß
+^ N]¿D¯Òo4IÒ²šš
+C³#Ô³žËœ®íÿègŒf£F:îºx®úì®yî
+~þÕ
+
+GÆ`9ž>Š_=º?¡Žl?7Ïw/Ð35œVMí:á«å²[''ìÁf¢vÑý'5«+ä8À¯ILÃ§C[
+«|ö¢TÈüÞ)šó
+&d90iþ.·#øÏ\(Tjÿ}¡%ô*Ð&z¯Rr»~è	cÖ,è?Ú×ç&ïõžø
+KB{;ä»@ü
+öŸYðDK-Œ>
+6Ý
+x2ÖeªZý2FWÌ9#äwÕwane"]ŸÅ~Yó«
+ñ))"Ãï²æ-­©ÖPõA€<þŽn.­ã|ÚÑÀJQ°[¿gÔku	@9§Ù
+ªçòPc©ã­y0
+ü î~ñq³ß:>ùær'®õ
+,å%%Š$ÄRK£¢œuqºGnLêt%¿
++U×DkNÏëðI:ÖÙÉ¿à'ùå5£÷öYXŽL8åhcÙáÙF¿	8
+Œ°eãoyúéã×QÓ­V2hê±Ÿ!,,Kp2{ZÐá\ð2»~Ltàk×š]±A³@Jô9úÁ9ô°µ
+T­
+yh­rZðÇ	Ú~oÏ%­
+è€^6,.AÿÝŸd.:÷ÛÜþKO9^--üÃ¬×{Ý3ú
+82sÇW-
+µ
+\üÿ×ò
+IdþYUÑc
+çàÏý'=IÐÆýŽZY
+¿DÃw^]]ø*j%ÍäaœôÜðõl%dôÎŠ$E×ÄåÃ`H©æI8×À]CAs°ímÐØš*Éš ÑæŒ{Ñûný^Z©óºo©m
+œúqTdØ#Ì­·kh¡gŠ<Ñ/
+¢o:	÷u,e{Ît}l«^ÿÕ»L€GåqPáRÚé`äzÝ¡î.
+
+q±ÊrSG²ÕÊ+Zð0EEœ:/psRÝŽì
+!ÏQ@ßËü{Úž
+_.Wg±Ïc4ù
+¯Qv€
+Ý6Ù ekbýÃ{ÞÑŸ¥²þáª®TóA^D{£·FOŽ-åV[ô0%¹>Š<ËénÕý[ÿ\WoíòLOhD?·­K?mÀ:ëÇ5JÓvYdØø:í6OVöùoÍ@l5AÄlär®@îs©¹	ÃOÇ^ÓŠ
+`ùþBðï·'PÃlžß.ãt	öðHíf
+Íí6¹Ã:OµŠøãÒaÈÒ0
+°fq#A_ŽÐ§7(ôé÷Põ 9òà,¥èÇä'QIÁïg¿
+
+n<OÕëä}
+
+ôî
+N£3Áü!Í*E.(&­}«Ž¢É®HV
+ššîgXDúåß:°(%úàe
+ŸXSüÌèº.Ûy¶š
+TsÜÉ ÈÀô@NµüÜ6ŸŸÞ<É9Gœ*PíÛ
+bå¿ð ï °°
+*Àl/
+1¡#Ð rÖ2äS'p¢àÛÅñó?ÿ`ã}æ	¹Iª)ÊÛ,°Ÿ
+F3Ÿ!_tÔùæru©Ÿ|3Sà(1çÇ']áú7±ÌçÈ
+s
+À:@bØº¯¿Ke1Ý(p¯|m¡ïRÈŠ»¶÷/Â/š 
+b±ð C5û($»Í_Oá>Ô°k$U:çN_ ürÙ®ëX4¯ö^õªF^
+vÞ
+e×ÙÐè¹Ð ñúJÌ×f
+²ØQ|·cV¬Ë¿"øy'é"?yH`€1Y
+*P(÷nãy@ìÙðy/²V	Ò©ÇÀKqi=¢Aã?FZ#
+:rm;]$+=LX¹ÀM²Í÷C6\÷óíÐ{·)I£UK{á|_ _Dºí*£®VÏiD7aŠA¡ÀÜ6l<»8`
+)ŠÆyPŒ,Ô)JÏ;ÞWC@éPíVùÔ &ã§;eù²Uü,:h1=g
+ÌQ$|i\(âÓîâøwÑ$
+ºÌY=±b&4[¡N]Åè¥rR®\
+ºí£_áqyüÆéÑ¥ÚAš°ÙD=áä\8W÷`¯üÃœlò
+`b$n%ÿ
+ÛÉ_!p±Ü÷ñ-ÿžÈ¿ü°ÂkëÁâ§Ç
+:%+Cïbx4±ªó>"D÷sh'Ý/þúÕebBRÚ*·ÙYÝ+ýî[È³ÞLÕT;®zÄ
+@ç¿ÝÐ]
+ÀšÛý	MÇßí1Á.3hÿ<RÇô-»×Užhì
+œet}è9Yìk
+=ðÆgFáÏOÎ¹ Í;Ã?(þu'=ÄÆúÁú^%«?_¯wþý÷I(v€ü
+ú»ÑHj·îWÈœüIª
+õhtÑ`n	Vµ
+*ÍÔ«ðÌŸÞp×eAR¬'ßSê.ZiT|cè*ø}Kh¬~lûjõ
+;{<Z5=%fO/ÌÏb&|Z
+}¡Å |ÎXäQÓE"&
+fy[Çxõe B3YbKõi	F¶$~|dGÇzÐòAt÷#Å.Í
+öe}y­
+
+/F1DBÚu~~ß
+~0[|c~Iq8å"UI]×ÁQü^Ygè%|Û^èPUjo·	ùZñOô.
+×?W Å
+EÁgiB!Í€~ÌDÛ	rÈ|PjR
+5Ñ":!yn
+`<e8¯9ql-€È!6
+ZQ6v1fXÀAl×c9>=9ã»]gnn
+á  y61s^+S{e!Îa°8WÖFMWÅ}3ì?¡è.\{fe®^Ñ8ýýW	?Î0Û
+é'
+aqß~WÓ	¥]bXE(4]·4"Ž??Ú
+_€Tcn45I*Ÿ<÷;â9;tÚÛyÀ`Xô%?U<bXŽVãz	"ÏiŸ`	uZLdÀo:$p
+Ù:>
+e${jž
+é
+<~
+VXä*mð@P¿+Mìi&#}oWfÅZ³tO`
+
+±a¢j²X
+`I)
+;{
+åi^\&­=0nz
+žeL
+~$662YgeáVN2^=ÄTyzÿ&
+W
+¬aqu©_VJid3h;
+,E
+óxskò*P\!Yï3Sa4=2h\&ä
+]M
+*$Z
+C}M/è%¹@~ºoÝÌ2ïy[+£.B
+r6yÏY4y72
+#
+	}l(I¥pGS6hÅz)X?
+
+ç5³þ
+,t°:gY7÷Aø;|*Â]Js
+4,oÐñ"0z1ß
+v
+=²hiKø
+
+_wö{H!r<2¢@Ñ;4s>ë
+zp>±nb;|gnyÀ^xc]ºpÕOktœ/âoÑ€^ÆAœc³¯WFJ~~
+GB
+E
+âx!n#
+t|t@C0»8Ä]$?ëC>9:`ocEó
+4`]Z
+©ô0ema0!!p³æ3tç.Ñ,6Ypdo{\ÂódÆ,cñ<$]1M
+ì¹;!VW
+#Ss×xj\µvï+xv<N%"i>~ÎAÏ9'I;Mâf\U%z_ëm!|ÿvÕ[Å&EøËg"D[/·gT*DîÁkTn¥kD{31øeÃ>­dn ð,UK4Ýx
+Ñ	í\Q4&5K9s~L
+@MñdÜ+yÓ"4V|f9ä@9^y|#mWç>(4`
+%g3v,Ïv#øYe8ë[9Û6e,VÒ
+>M}|¯)«SÎ;Ð;å@;éC<{úxsZ93yFj°(cc|¡d,d
+
+uálŠš5®Wœ{O¥|²|H{!.ha{10uXñpësi%fä=N8~>zU[!íh¿-
+SUe{zÎ=îði
+$y{gièPÅy?VMQß
+ÜiŒ?
+XßjX]>i	Q.JÖIè]1	
+kÃTIs÷åfMuTk¯&¹rBbT"ÑbÖ!?ë9Švœí } ÷x·ä
+`
+:q+XÀRvGN9ó{*dvÌ|d}k4]ªZå8${ï0-qzŽBv
+F|o«Q`ÝÈwfú]A|g
+ëf~	O{
+UA%erÉKe
+Ü?
+i7Xö8&A]¢áB DvZ*
+â!sÍ$M7H?|ÿ{
+E_ÕL:#	_ötŠgVÿ$í"	M9ãcý2!)	Ñ[ŽcPF¶m
+="%uY3
+|wU
+Ç-@Tºb?`I!'KQ{8Ý*Ú=©
+û
+dE7b{€|zvÒOØ"@A
+ŠPï3ìkš}ö,ä ºt4êzh|ìQèkPQzñ#z
+i'
+ý
+Øb)
+ÈPI70;3
+yHA»/a(gÈÚ*ì
+»í
+@ÿI3P#Nõ~V/E:ÿòNþI±	ª
+c
+
+G>/\ÒF;\à"(kÀGqŒL€te	ýUÄD^9b~{9Mâ<=#
+
+ãPG_^W@TE^û<¥~·D&büyb=èAVhÜÄmëdC-WÏ,b>Z!q9E·  \\÷ojUÏÄKK{@ð+Å}ªuÞ{L
++rù
+Cs
+^J¿vt']ÅBÁ!ñ{œŒ||-FRjú~
+
+#i»Oµ.7 ¬z€U
+B|Lu$
+4/TGËU;ö?ºgU
+Xù
+ò	òS[
+Y
+Ao
+QBý\w^í@ÅR}Ušù{ÊuPFùrz ðu)5.-
+T
+{
+Ïiöd:æí
+ñÔf3N}uæ&
+Vâ"RÈzdp,.F!<"k:>À|géQÝG¢\ö	þ*Å7Û~þC}u+eL=<cµ
+å"F#PI8mi`¢E?&]U
+{DR]ên7^5:
+"ÉoÈbvZP P.âQ+Ô/ï\lŒ=G±A@TœtZ(zmáîB
+{z]~kULýg	6pg1ì0ð}²{ï×FÀRÎ
+²^#Êr¹mŸÔ	|ž%õ65`ÜdIS×? ð8
+K-rh¿euPz(­&
+à3 JY&_^F^@|
+ biy€ËLÕµGjsÿ3c/Wgt|0@
+ÐU;(jè%§78GnmúÅO#
+Þ#")Ó|ÜeR~¯
+g'è~
+ Š^ÌFmoãd©
+û=@p
+ÉÌ\3+!;êåtœE·?5ZjÔÈo÷[éŠ&óDü;àœxt;C¡T-&";Q^ÆøõÚ:Çú<j¡}uP`ÀŠO-^	smœVêhj%nÞ
+
+lz
+²XctöúzÔ	Üô|
+)nðzz,ê/IFYG fü]AQWPè
+xw3è%ÂhËÛ9Q
+
+r°ó	rÿ'
+§ôj
+ûEC[Èzaí[x¡±Ë{4D$t':tÅ*Õ{eÖ	¬z#Â%f%Ü{5õTKâ^ßB	úRŸ&7%5S\J@ IB0
+_Šk¶m)Gkm
+I
+Ÿa&njg1JM
+4 /Ï
+>²yYJ*^
+3("'a»2
+>ã,Ÿ+Gpsã,ÇB_3Ÿ&éLç&r<	
+I¿q3fŽVLÇBÙLE
+7²õW
+\&±øHû01kÿX¹-Â%1'If@²ÞUAT[ÐK!ŠH»{¹{ãÐO8G>/@-w4U_A
+'\
+ôbLÖ
+
+SüOñe
+±ñs){
+PtšSî3	¶oðzÐF^9OÒM
+hv]Wê
+46_Y t"ÏFÃdS
+3['YCò)qØy©&
+Ð&ü!!DÏZ;
+±oDymÂl» }oöy±:>0zh]Û.TÆ`VNK?ðjxlnÈáWÅ'ör
+@m
+WcGá:€DamY °:ÞA,q@qWIø;'/l¶OŠPn
+õ\±7'XWñoPzq/¹$gk³|RThs®©GÊ°mA"o¹m¢K9|I\
+C{ù ôSùkòß»gÆ@7ËZR«I1OÖ×é|ÔZyrwfètv!ø~0
+³@Ù{z
+iTŒb
+?°&§)1hhÈpA\Mm*
+¬yUfùwgB
+
+ðr@%]bÞå/6U8
+
+.-^i7ÆzJ-&PéaVJ
+K9
+ic*ÛNß"d!í8P:PBÓvàGS-M
+"b
+¥zU@ö 8
+â\¬`E
+hyF]iy~ÞSH`k%×R^
+Bîg2T
+ã ËdÝ8×
+qmýD?7ÙN	Ö	\ey"!j
+5N9>óKb~zRmlRë4"8
+
+Ä+WK'ú9-åQª
+)Ev
+Ï|%tà+Z³PŒu>ðd€@9zäd#Ë0OáZ¬"q
+{î6f~æ{9TÔ~M/P;ÿ"v)PŽVF#Ò
+;-:
+pv>ÿ1zìuVöV·
+úSE
+W~ y07êg}-äyu£q
+Zoh{/k4sA 	ÿgE÷/Eµï]iBF-#S/i|Š\(¥6W79°Psµ^?{°8TÂ`Ö8$ìYo
+
+ŒÁV%~4>I%
+Ô	NB6ÄR1**æ}î(QKn¥|¯]{v$õ	^do%ÆIÝ&ÛÞ.|-PŒ8€abUf:X7Ø9
+Ès94;2`Uÿ
+Wª 
+Â9Ð	x.|/iR»?D5@
+ùn=#
+MÕ93^0QG2I|ü¬teVÁ/ÀTÉÞfA`?T+DÖOØG?\'
+8mIs^©:=ÀUi"Ò'·¬@¡j«uMc2o1-¡J9S/Íz³;
+
+%r]Þ'GA(×RÔ	â3®"{qnÎŽ
+z
+o(Ò!¢
++6c&/:Ë]Õ3 t=lLW
+©gÆm]­Ä^J(y'HTjP@šÀG{WtbµAvF\@o\â^ÉE¬Š,qc:Ä9÷&hNdyc í\-ÛDÔdÞ$ÞYDÜp}ò%O?BL.m
+Yd§zÍ0<?ê)c_-v #@J»MÑo
+,
+Š]E5eYº
+j&5 »!Ì>qrŽ5
+yíbÎ2?ü5Ñ'>!QJm9+`?à=@jVô4¡yK\Ž|'
+iU
+
+HE³
+,Ê³.Ip6jpLë1
+
+]V­ ù
+©_LR«?>tûxQp<«ÅUýzxKÊÏ
+Ô9Ò1íA_7ñtvzø
+šd<ç&û)t
+a#3f3®*ñ'J*	[6]Ÿ9
+R5aXK
+a×
+®SKhú
+$
+ÙLäs°_õ8>)€{ìc+Hu-){4gAVhbJÅ4WY^-r%Ip0
+/YÏ?[Ác__OpalQ6
+@ÃæIXb^ôfxd1Ã>/ÁZoÊ&/jP#49lI
+Q(Eû	uÒw9/k
+uMùjC
+HY¹
+v Î}OdJCÔE?\K
+Ë{s[Àkh]·Ät	C3"§n &Î
+¶,[ÀÓhz$qqy~ÀHT+è)e-X5€F
+²YSl0z²
+U.mx
+Ì2,7¢œëHD¢dóö_9Q
+90$ý[sÔvÜ-/ÊQ%M	ýabl­;ñ'(c­;A cyW4Dz__L¿5hC%uÄ+
+ØGÛfK
+â%Ì
+ë6°Çpb÷=53Üö^enõ*dä
+û~Y14<K¬.{, 	õ{ôH#¢=èr;° (¢SÏ;d\wŽ^{
+Å
+áf
+'$­Nvi=kTW$T5n8^=)Ý[ç=}Ìò:Ìg$?Z,ÁHIJFt*Uë$ZGEÜŽ/©Mdp\lÈz¯°0lÎK{`~Hœjç@ñat"
+	H\Ÿ
+
+È
+¿
+Rp
+K!f)!¢HŽ>ù	å:°ö~üp5qàkC+Ž»!dXïX  #s§.
+9
+9-
+fwóXíyWIt\	I/ýrúcÚKù3:)#	n0.°+V2://ç6nª	0¢Æ
+>jü
+Ín<
+u
+L@}ª$
+.oa_~ö&1pÛKz *~iÙNÐrø.ñ/\,
+fZüü?æaI1DÇw,FA 0lÉÅ
+x{œ%Dül1!KYX
+ ŒXd
+¢
+D}\$	OKa:.eNÆ]šaØ
+
+?cßxhrVv!
+Ëxõ85l7lêÂ2RÒ$;ueÅ#€O5Wt
+#Y^4
+9í
+q¹nbNÓO.ySj0!Ï
+ÈSBY5Œ5×	Tf¬k
+ë5 *>TFB#(«QÏ·	I,at:è-û
+
+l<]«
+þ9IL#!)î
+P-1©3%¢O4weúJI¥
+¹¥»T-
+
+¯Yº/rŽ28î
+ÒTÈl­/kX ?+;%))Ü}UIN	ø
+wHüBþ
+Ÿ"
+È-òý"š'¹
+ /\
+z
Index: /main/tags/3.01/gsdl3/src/packages/mg/WIN32cfg.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/WIN32cfg.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/WIN32cfg.h	(revision 21112)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /main/tags/3.01/gsdl3/src/packages/mg/about_mg.html
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/about_mg.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/about_mg.html	(revision 21112)
@@ -0,0 +1,75 @@
+<HTML>
+<HEAD>
+<TITLE>About MG</TITLE>
+</HEAD>
+<BODY>
+<H1>About MG</H1>
+ 
+<HR SIZE=4>
+
+MG is a public domain
+indexing and retrieval system for text, images, and textual images.
+MG is covered by a 
+<a href="http://deimos.kbs.citri.edu.au/mg/intro/copying.html">GNU 
+public licence</a>.
+<P>
+MG is available for ftp <a href="ftp://munnari.oz.au/pub/mg">here</a>
+
+<H2>Developers</H2>
+
+The MG system is an embodiment of ideas developed primarily by:
+<ul>
+<li> <a href="http://www.cosc.canterbury.ac.nz/~tim">Tim C. Bell</a>,
+	University of Canterbury;
+<li> <a href="http://www.cs.mu.oz.au/~alistair">Alistair Moffat</a>,
+	University of Melbourne;
+<li> <a href="http://www.cs.waikato.ac.nz/~ihw">Ian Witten</a>,
+	University of Waikato; and
+<li> <a href="http://www.cs.rmit.edu.au/~jz">Justin Zobel</a>,
+	RMIT.
+</ul>
+
+<H2>Programmers</H2>
+
+The bulk of the programming work has been undertaken by:
+<ul>
+<li><A HREF="http://www.cs.waikato.ac.nz/~singlis/">Stuart Inglis</A>,
+       University of Waikato;
+<li><A HREF="http://www.cs.waikato.ac.nz/~cgn/">Craig Nevill-Manning</A>,
+       University of Waikato;
+<li> Neil Sharman, University of Melbourne; and
+<li> Tim Shimmin, Collaborative Information Technology Research Institute (CITRI).
+</ul>
+A full list of contributors appears
+<A HREF="http://deimos.kbs.citri.edu.au/mg/intro/mgintro.html#xtocid2063010">
+here</a>.
+
+<H2>Institutional Support</H2>
+
+The MG project has been supported by the following organisations:
+<ul>
+<li>	The Australian Research Council;
+<li>	<a HREF="http://www.citri.edu.au/">CITRI</a>, Melbourne;
+<li>	The <a HREF="http://www.cpsc.ucalgary.ca/">University
+		of Calgary</a>, Canada;
+<li>	The <a HREF="http://www.cosc.canterbury.ac.nz/">University
+		of Canterbury</a>, New Zealand;
+<li>	The <a HREF="http://www.cs.mu.oz.au/">University
+		of Melbourne</a>, Australia;
+<li>	<a HREF="http://www.cs.rmit.edu.au/">RMIT</a>, Australia; and
+<li>	The <a HREF="http://www.cs.waikato.ac.nz/">University
+		of Waikato</a>, New Zealand.
+</ul>
+
+<H2>Backpointer</H2>
+
+If you didn't just come from there, the MG home page is accessible
+<a href="http://www.kbs.citri.edu.au/mg/">here</a>.
+
+<HR SIZE=4>
+<ADDRESS>alistair@cs.mu.oz.au</ADDRESS><BR>
+ 
+</BODY>
+</HTML>
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/acconfig.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/acconfig.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/acconfig.h	(revision 21112)
@@ -0,0 +1,35 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
Index: /main/tags/3.01/gsdl3/src/packages/mg/aclocal.m4
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/aclocal.m4	(revision 21112)
@@ -0,0 +1,140 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ([regex])
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , [AC_LIBOBJ([rx])])
+fi
+])
Index: /main/tags/3.01/gsdl3/src/packages/mg/build_options.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/build_options.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/build_options.in	(revision 21112)
@@ -0,0 +1,2 @@
+set stem_method = 3
+set do_indexes = 1
Index: /main/tags/3.01/gsdl3/src/packages/mg/config.guess
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/config.guess	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/gsdl3/src/packages/mg/config.h.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/config.h.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/config.h.in	(revision 21112)
@@ -0,0 +1,177 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#define HAVE_STRERROR 1
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
Index: /main/tags/3.01/gsdl3/src/packages/mg/config.sub
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/config.sub	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/gsdl3/src/packages/mg/configure
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/configure	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/configure	(revision 21112)
@@ -0,0 +1,7753 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="src/text/mgquery.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS JNIINC JNISUFFIX JNIFLAGS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mg
+VERSION=1.2
+
+LDFLAGS=
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -single_module -framework JavaVM"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+
+
+
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+fi
+
+fi
+
+
+# text for endianness
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+
+                                                                      ac_config_files="$ac_config_files Makefile src/text/Makefile src/images/Makefile src/scripts/Makefile lib/Makefile jni/Makefile java/org/greenstone/mg/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "src/text/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/text/Makefile" ;;
+  "src/images/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/images/Makefile" ;;
+  "src/scripts/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/scripts/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "jni/Makefile" ) CONFIG_FILES="$CONFIG_FILES jni/Makefile" ;;
+  "java/org/greenstone/mg/Makefile" ) CONFIG_FILES="$CONFIG_FILES java/org/greenstone/mg/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@JNIINC@,$JNIINC,;t t
+s,@JNISUFFIX@,$JNISUFFIX,;t t
+s,@JNIFLAGS@,$JNIFLAGS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/configure.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/configure.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/configure.in	(revision 21112)
@@ -0,0 +1,123 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(src/text/mgquery.c)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mg
+VERSION=1.2
+
+LDFLAGS=
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -single_module -framework JavaVM"
+else
+  AC_MSG_RESULT(no)
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+AC_SUBST(JNIINC)
+AC_SUBST(JNISUFFIX)
+AC_SUBST(JNIFLAGS)
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# text for endianness
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile
+	  jni/Makefile java/org/greenstone/mg/Makefile ], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/Makefile
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/Makefile	(revision 21112)
@@ -0,0 +1,5 @@
+DISTFILES = Makefile mgmerge.README mg_invf_dump.changes \
+  README.man mg.1 mgintro.1 mgintro++.1
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/README.man
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/README.man	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/README.man	(revision 21112)
@@ -0,0 +1,1 @@
+The rest of the man files are in the respective src directories.
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/mg.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/mg.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/mg.1	(revision 21112)
@@ -0,0 +1,462 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg 1 \*(Dt CITRI
+.\"=====================================================================
+.\" Author:
+.\"     Nelson H. F. Beebe
+.\"     Center for Scientific Computing
+.\"     Department of Mathematics
+.\"     University of Utah
+.\"     Salt Lake City, UT 84112
+.\"     USA
+.\"     Email: beebe@math.utah.edu (Internet)
+.\"=====================================================================
+.if t .ds Bi B\s-2IB\s+2T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Bi BibTeX
+.if t .ds Te T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Te TeX
+.\"=====================================================================
+.SH NAME
+mg \- full-text inverted index support
+.\"=====================================================================
+.SH DESCRIPTION
+.B mg
+is a suite of programs that can be used to create
+and query full-text inverted indexes for a
+collection of documents.
+.PP
+An inverted index is essentially a list of
+pointers to occurrences of every word in a
+collection of documents, so that, for example, in
+a collection of Shakespeare's works, one can pose
+questions like
+.I "How often is a fool mentioned in the plays?"
+and
+.I "Are Romeo and Juliet ever mentioned in the same sentence?"
+While search utilities like the UNIX
+.BR grep (1)
+family can be used for simple queries, they suffer
+from several limitations:
+.TP \w'\(bu'u+2n
+\(bu
+Each invocation requires a separate pass over the
+documents, which becomes impossibly slow if the
+document collection is very large.
+.TP
+\(bu
+It is difficult to compose Boolean queries like
+.IR "Caesar AND Brutus AND NOT Antony" .
+.TP
+\(bu
+They provide no mechanism for ranking matches
+according to importance, which is extremely
+important when queries are complex and numerous
+matches are found.
+.TP
+\(bu
+They provide no mechanism for displaying
+surrounding context (although the Free Software
+Foundation
+implementations for the GNU Project remedy this
+with their
+.BI \-A " num"
+(after)
+and
+.BI \-B " num"
+(before)
+switches).
+.TP
+\(bu
+They provide no mechanism for searching for
+occurrences in the same paragraph, unless
+preprocessing is done on the files to wrap
+paragraphs into long lines.  Even that will often
+fail, because input buffer sizes may be limited to
+a few hundred characters.
+.TP
+\(bu
+They provide no easy way to deal with grammatical
+word-ending variants, except by explicit
+enumeration, such as searching for
+.IR compress ,
+.IR compressed ,
+.IR compresses ,
+.IR compressing ,
+and
+.IR compression .
+.PP
+Most computer users have been faced with the
+problem of finding information from a large
+collection of files, such as electronic mail,
+on-line documentation, or source code.  Inverted
+indices provide a highly-effective solution to
+this problem.
+.PP
+The
+.B mg
+software is described in Appendix A of the book
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994
+.fi
+.RE
+.PP
+Many of the algorithms implemented in
+.B mg
+represent significant advances over previous work,
+both in speed, and in storage requirements.  On a
+fast workstation, in tens of minutes, or a few
+hours,
+.BR mgbuild (1)
+can create an index to
+.I all
+of the words in hundreds of thousands of documents
+occupying hundreds of megabytes, or even more than
+a gigabyte, of disk space.
+.BR mgquery (1)
+can then be used to answer complex queries, with
+responses often returned in a second or less.
+.B mg
+also contains algorithms to deal with images, so
+that with a small amount of descriptive text for
+each image, it is possible to do searches in
+collections of images, and to have retrievals
+display the images using a viewer like
+.BR xv (1).
+.PP
+.B mg
+can deal with compressed text and image files and
+surprisingly, it usually runs faster than it would
+if the files were not compressed!  Thus, the
+considerable disk space savings possible from
+file compression are not lost because of the need
+for fast document search and retrieval.
+.PP
+The Free Software Foundation GNU Project
+compression utilities
+.BR gzip (1)
+and
+.BR gunzip (1)
+are recommended for general use over older
+alternatives, like
+.BR compress (1),
+because of their speed and high compression
+ratios.
+.\"=====================================================================
+.SH AVAILABILITY
+The
+.B mg
+software can be obtained via anonymous ftp to the
+Australian archive host
+.I "munnari.oz.au [128.250.1.21]"
+from the directory
+.IR "/pub/mg" .
+.\"=====================================================================
+.SH "TYPICAL USE OF mg"
+Although
+.B mg
+consists of more than 20 separate programs, many
+of which have complicated command-line options,
+take heart: most users require only two or three
+of these programs, and nothing more than a
+document name on the command line.
+.PP
+A
+.I document
+for
+.B mg
+is a fragment of text suitable for retrieval as a
+unit when it is found to contain a requested word,
+or words.  In a collection of poetry, a document
+might be a stanza, while in a novel, it could be a
+paragraph.  In an index of first lines of poems, a
+document would likely be just a single line.
+.PP
+Just what constitutes a document is decided by a
+user-modifiable UNIX shell script,
+.BR mg_get (1).
+The default script provided with the
+.B mg
+source distribution knows about these named
+document collections:
+.TP \w'mailfiles'u+2n
+.I alice
+Lewis Carroll's
+.I "Alice in Wonderland"
+book.
+.TP
+.I allfiles
+all mail files in the directory tree
+.IR $HOME/Mail ,
+including all of its nested subdirectories.
+.TP
+.I mailfiles
+individual mail messages in
+.IR $HOME/mbox
+and
+.IR $HOME/.sentmail.
+.TP
+.I davinci
+A small collection of text and images from the
+work of Leonardo da Vinci.
+.PP
+A document collection name is used by
+.B mg
+as a
+.BR csh (1)
+.I case
+statement selector, and as a subdirectory name in the
+.B $MGDATA
+directory, or the current directory, if
+.B MGDATA
+is not defined.
+.\"=====================================================================
+.SH "EXTENDING mg_get"
+This section describes how to extend
+.BR mg_get (1)
+to handle a new document collection.
+.PP
+Let us take two examples: all \*(Bi
+.I .bib
+files, and all \*(Te files, contained in
+subdirectories under the login directory.
+.PP
+For \*(Bi, each bibliographic entry will be
+considered a separate document.  In order to
+facilitate easy identification of entries, we
+shall require them to begin at the start of a
+line; the
+.BR bibclean (1)
+utility can be used to standardize the format of
+.I .bib
+files, and to validate their string values, so
+that this requirement is met.
+.PP
+For \*(Te, each paragraph will be a separate
+document, and we assume that paragraphs are
+separated by blank lines.  We assume that files
+with extensions
+.IR .atx ,
+.IR .ltx ,
+.IR .stx ,
+and
+.I .tex
+contain input to common \*(Te macro package
+variants.
+.PP
+Make a personal copy of the
+.I mg_get
+script, using the one in the
+.B mg
+source distribution
+.RI ( mg-1.0/mg/mg_get.sh ),
+or the one in the local binary program directory,
+at many sites called
+.IR /usr/local/bin/mg_get .
+.PP
+Examination of the
+.I mg_get
+script shows that each document collection name is
+used in a
+.BR csh (1)
+.I case
+statement selector, and that most of work is done
+by very simple
+.BR awk (1)
+programs that extract documents from files.  In
+your private copy of the
+.I mg_get
+file, after the line
+.PP
+.nf
+  breaksw #davinci
+.fi
+.PP
+and before the line
+.PP
+.nf
+  default:
+.fi
+.PP
+insert this new code:
+.PP
+.nf
+  case bibfiles:
+# Takes a list of files that contain BibTeX entries, and splits them up
+#   by putting ^B after each entry. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*.bib' -print | \e
+        sort | \e
+        xargs -l100 awk \e
+          '/^@/&&NR!=1{print "^B"} {print $0} END{print "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bibfiles
+
+  case texfiles:
+# Takes a list of TeX files and split them up
+#   by putting ^B after each paragraph. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*x' -print | \e
+        egrep '[.]tex$|[.]ltx$|[.]atx$|[.]stx$' | \e
+        sort | \e
+        xargs -l100 nawk '   /^ *$/ {if (b!=1) printf "^B";b=1} \e
+            \e!/^ *$/ {print;b=0} \e
+            END {printf "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #texfiles
+.fi
+.PP
+The ^B characters here are Control-B characters,
+.I not
+caret-B pairs.
+.PP
+If you have a large number of \*(Bi or \*(Te
+files, it is likely that a list of them would be
+too long for the UNIX shell to hold in a single
+variable, or on a single command line.  Thus,
+instead of storing the output of
+.BR find (1)
+in a variable, we proceed more cautiously, and
+employ it to produce a list of the required files,
+then pipe them to
+.BR xargs (1),
+which in turn passes up to 100 filenames at a time
+to
+.BR nawk (1)
+for document selection.
+.PP
+Install this modified
+.I mg_get
+script in your private directory for executable
+programs (e.g.
+.IR $HOME/bin ),
+create a directory
+.I $HOME/mgdata
+to hold the index, issue a
+.B rehash
+command if you are using
+.BR csh (1)
+or
+.BR tcsh (1),
+ensure that
+.I mg_get
+occurs in your search path
+.I before
+any system-wide one (the command
+.BI which " mg_get"
+will tell you which version will be selected),
+then create the inverted indexes by
+.BI mgbuild " bibfiles"
+and
+.B mgbuild
+.IR texfiles .
+These commands may take several minutes to run if
+you have a lot of \*(Bi or \*(Te files, or a large
+home directory tree.  Once they are complete, you
+can then query the index with the commands
+.BI mgquery " bibfiles"
+and
+.B mgquery
+.IR texfiles .
+These should respond very rapidly.
+.PP
+In order to keep your index up-to-date, you should
+arrange for it to be recreated automatically and
+regularly, probably every night.  You can do this
+with
+.BR cron (1).
+Use the command
+.B "crontab \-e"
+to edit your
+.I crontab
+file and add two lines like this:
+.PP
+.nf
+00 04 * * * mgbuild bibfiles >$HOME/mgdata/bibfiles.log 2>&1
+15 04 * * * mgbuild texfiles >$HOME/mgdata/texfiles.log 2>&1
+.fi
+.PP
+Save the file and exit the editor.  Now, every
+night at 4am and 4:15am,
+.BR mgbuild (1)
+will reconstruct your inverted indexes, and the
+results of the builds will be saved in log files
+in your
+.I $HOME/mgdata
+directory.
+.\"=====================================================================
+.SH "SEE ALSO"
+.na
+.BR awk (1),
+.BR bibclean (1),
+.BR bibtex (1),
+.BR compress (1),
+.BR csh (1),
+.BR grep (1),
+.BR gunzip (1),
+.BR gzip (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR nawk (1),
+.BR tcsh (1),
+.BR tex (1),
+.BR xargs (1),
+.BR xmg (1),
+.BR xv (1).
+.\"=====================================================================
+.\" This is for GNU Emacs file-specific customization:
+.\" Local Variables:
+.\" fill-column: 50
+.\" End:
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro++.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro++.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro++.1	(revision 21112)
@@ -0,0 +1,212 @@
+.\"------------------------------------------------------------  
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4 
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------  
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO++ 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro++ \- extended introduction to the MG system
+.\"-------------------------------------------------------------
+.SH DESCRIPTION
+This manual assumes the reader has already read
+.BR mgintro (1).
+.\"-------------------------------------------------------------
+.SS Creating Different Databases
+If a user wants to build databases other than for some
+predefined ones, such as "alice", "davinci", "mailfiles", "allfiles",
+then the user has a couple of choices.
+Ultimately (s)he must produce a text file with control-Bs 
+terminating the documents. 
+To do this one can produce one or more such files, or write a
+"get" command (typically in the form of a script or c program).
+.\"-------------------------------------------------------------
+.SS Using Input Files for mgbuild
+If you don't want to write a "get" script and just want to use
+one or more text files as input, then you must first generate
+the file with control-Bs. For a simple example, you could take
+any text file(s) such as "test1.txt" and "test2.txt", and use 
+.BR vi (1)
+to insert control-Bs by typing "control-V b".
+Next you should create a file with "set" statements 
+in the following form:
+.PP
+.IP
+\fBset pipe = 0 # do not use pipe - use file instead 
+.br
+\fBset input_files = 'test1.txt test2.txt'
+.LP
+Let's call this file, "build_options".
+Now issue the command:
+.IP
+.B mgbuild -s build_options test
+.LP
+This should build a database called "test" in the $MGDATA directory,
+based on the source data of "test1.txt" and "test2.txt".
+The build_options file is simply sourced by 
+.BR mgbuild (1)
+after it has set up its variables. 
+Therefore, any settings one makes in the
+build_options file will override the standard settings.
+See 
+.BR mgbuild (1) 
+for more information.
+.\"-------------------------------------------------------------
+.SS Writing A Get Program
+Instead of using files as input, it is often more convenient to 
+write a "get" program. This program is called by 
+.BR mgbuild (1)
+to get the text data with control-Bs as document terminators.
+It should take three options:
+.br
+(i) -init; (ii) -text; (iii) -cleanup.
+.br
+Get will be called with "init" first and with "cleanup" at the end.
+It will call get with "text" when it wants the text and it should
+write the text to stdout.
+.br
+See 
+.BR mg_get (1)
+for an example.
+.\"-------------------------------------------------------------
+.SS Regular Builds
+The MG system provides a static database; 
+there are no update commands.
+So if one wants to keep one's database reasonably up-to-date
+then one can have this done automatically on a regular basis by 
+.BR cron (1).
+A crontab file can be created using:
+crontab -e
+A crontab file contains lines of the form:
+.nf
+.IP
+\fBminute hour day-of-month month day-of-week shell-command.
+.LP
+.fi
+See
+.BR crontab (1)
+for more information.
+.nf
+An example crontab entry is:
+.IP
+\fB15 02 * * * mgbuild allfiles >$MGDATA/allfiles/allfiles.log 2>&1
+.LP
+.fi
+This will build up the mg database for "allfiles", your mail in
+the folders, every morning at 2:15am. 
+.\"
+.\"-------------------------------------------------------------
+.SS Command Structure
+There are 22 commands that make up the mg system. However,
+a user may only need to be aware of a few: 
+.BR mgbuild (1),
+.BR mgquery (1), 
+and perhaps
+.BR mg_get (1). 
+Many of the commands are called by 
+.BR mgbuild(1).
+The commands can be broken up into a hierarchy.
+.PP
+--------------------------------------
+.br
+MG--+--image compression
+    |  |
+    |  +--mgbilevel
+    |  |
+    |  +--mgfelics
+    |  |
+    |  +--mgtic
+    |  |
+    |  +--mgticbuild
+    |  |
+    |  +--mgticdump
+    |  |
+    |  +--mgticprune
+    |  |
+    |  +--mgticstat
+    |
+    +--text 
+       |
+       +--compression
+       |  |
+       |  +--mg_passes -T1
+       |  |
+       |  +--mg_passes -T2
+       |  |
+       |  +--mg_compression_dict
+       |  |
+       |  +--mg_fast_comp_dict
+       |
+       +--indexing
+       |  |
+       |  +--mg_passes -N1
+       |  |
+       |  +--mg_passes -N2
+       |  |
+       |  +--mg_perf_hash_build
+       |  |
+       |  +--mg_invf_dict
+       |  |
+       |  +--mg_invf_rebuild
+       |  
+       +--weights
+       |  |
+       |  +--mg_weights_build
+       |
+       +--query
+       |  |
+       |  +--mgquery
+       |
+       +--tools
+          |
+          +--mg_invf_dump
+          |
+          +--mg_text_estimate
+          |
+          +--mgdictlist
+          |
+          +--mgstat
+.br
+--------------------------------------
+.PP
+.nf
+.BR mgbuild (1)
+calls the following commands:
+.RS
+.BR mg_passes (1), mg_compression_dict (1)
+.BR mg_perf_hash_build (1), mg_invf_dict (1), mg_invf_rebuild (1)
+.BR mg_weights_build (1)
+.RE
+.fi
+.\"--------------------------------------------
+.SH SEE ALSO
+.BR mgintro (1),
+.BR mgbuild (1),
+.BR mg_get (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95 
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/mgintro.1	(revision 21112)
@@ -0,0 +1,155 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro \- introduction to the MG system
+.\"--------------------------------------------------------------
+.SH DESCRIPTION
+The MG (Managing Gigabytes) system is a collection of
+programs which comprise a full-text retrieval system.
+A full-text retrieval system allows one to create a
+database out of some given documents and then do queries
+upon it to retrieve any relevant documents.
+It is "full-text" in the sense that every word in the
+text is indexed and the query operates only on this index 
+to do the searching.
+.PP
+For example, one could have a database on the book,
+"Alice in Wonderland." A document could be represented by
+each paragraph in the book.
+Having built up the "Alice" database, one could do queries
+such as "cat alice grin" and retrieve any paragraphs which
+match the query. The matching could either be boolean, that
+is the retrieved paragraphs contain a boolean expression of
+the query terms e.g. "cat alice grin"; or the matching
+could be ranked i.e. the most relevant documents to the query
+in relevance order, using some standard heuristic measure.
+.\"--------------------------------------------------------------
+.SS Motivation
+If one wants to find some particular information which
+is stored in a computer text file then one has a few alternative
+courses of action. One can operate directly on the text files
+with utilities such as grep or can process the text files into
+some form of database. Grep is generally limited to identifying
+lines by matching on regular expressions. If the collection 
+of files which grep operates on becomes large, then continual
+passes over the entire text on each query becomes expensive.
+However, its usage is simple as no auxiliary files must be created.
+.PP
+A database consists of some data and indexes into that data. By having
+indexes one can query a large database quickly. Standard databases
+divide the data up into records of fields. This means that the granularity
+of search is a field. In a full-text system, such as \*r,
+there are no fields 
+(or there is an arbitrary sized list of word fields per document) 
+and instead every word is indexed. 
+Using this method, we can except free-form information and yet be fast on searches.
+The next question is what is the overhead of this database.
+In \*r most files which are produced are in a compressed form. The
+two notable compressed files being the given data and the index, called
+an "inverted file". By compressing the files it is possible to have the
+size of the database smaller than the size of the source data.
+.\"--------------------------------------------------------------
+.SS Typical Usage
+The most common use for \*r
+has been as a search database on unix mail files.
+However, any set of text data can be used, one just needs to determine
+what constitutes a document (see 
+.BR mgintro++ (1)
+).
+\*r has also been used on large collections such
+as Comact (Commonwealth Acts of Australia) which is around 132 megabytes 
+and also on sizes up to around 2 gigabytes for TREC 
+(a mixture of collections such as the Wall Street Journal 
+and Associated Press).
+.\"--------------------------------------------------------------
+.SS Getting Started with \*r
+The first thing to do is install the package; 
+please follow the INSTALL instructions.
+Having done this, it is necessary to set a couple of environment variables.
+MGDATA should be set to a directory which is to hold subdirectories for
+each database that you build. For example: 
+.IP
+.B mkdir ~/mgdata; setenv MGDATA ~/mgdata.
+.LP
+If you want to try out building some sample databases then there is
+some sample data such as the "Alice In Wonderland" book. To make sure
+this is accessible you should set the environment variable MGSAMPLE.
+For example: 
+.IP
+.B setenv MGSAMPLE ~/mg/SampleData
+.LP
+Here, "~/mg/SampleData" should contain alice.z .
+.PP
+To build the Alice database (to be contained in $MGDATA/alice subdirectory), 
+type the command 
+.IP
+.B mgbuild alice 
+.LP
+Assuming all went well and some status messages 
+were printed indicating the build was completed, then type 
+.IP
+.B mgquery alice 
+.LP
+to query the database. 
+You can type a few words at the prompt, hit return and
+some relevant documents, Alice paragraphs, should be retrieved.
+Type ".set query ranked" to do ranking queries. Please refer to the 
+.BR mgquery (1)
+man-page for more information on the commands and options of 
+.BR mgquery (1).
+.PP
+The next thing to do is to use \*r
+on a more personal database. If you have
+your mail stored in subdirectories of ~/Mail, such as is done if you use the
+typical set up of 
+.BR elm (1), 
+then type 
+.IP
+.B mgbuild allfiles 
+.LP
+If, however, you keep all your mail in ~/mbox or ~/sentmail, then type 
+.IP
+.B mgbuild mailfiles
+.LP
+.\"--------------------------------------------------------------
+.SH AVAILABILITY
+The \*r software for SunOS 4, Solaris, HPUX, and MIPS,
+can be ftped from: munnari.oz.au [128.250.1.21] in the directory
+/pub/mg. 
+.\"--------------------------------------------------------------
+.SH SEE ALSO
+.na
+.BR mgintro++ (1),
+.BR mgbuild (1),
+.BR mgquery (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
Index: /main/tags/3.01/gsdl3/src/packages/mg/docs/mgmerge.README
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/docs/mgmerge.README	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/docs/mgmerge.README	(revision 21112)
@@ -0,0 +1,230 @@
+A CRASH COURSE GUIDE TO MGMERGE
+===============================
+
+Shane Hudson
+15 November 1994
+
+
+This document is intended as a note to the maintainers of mgmerge
+outlining the mgmerge utility, the changes
+that had to be made to the existing mg code to make mgmerge
+work, and the new source code for mgmerge.
+
+
+****  NOTE:    ****
+All the NEW/CHANGED files are in the file "new_mg.tar" which
+you should have received with this; files that were
+not changed are NOT in the tar file  so it will have to be
+extracted "on top of" the existing code. You'll figure it out :)
+
+
+****  DESCRIPTION:   ****
+mgmerge adds documents to an existing mg database
+without the need to rebuild the database from scratch.
+It works by building a temporary new database from the
+new documents, then merging the "old" and "new" databases
+to give one merged database.
+
+
+The source consists of:
+    mgmerge.sh		The main script, a lot like mgbuild.sh
+    mg_get_merge.sh	Script to retrieve new documents, a lot like
+			mg_get.sh
+    mg_merge.h		header file used by mg_invf_merge.c
+			and mg_text_merge.c
+    mg_text_merge.c	Program to append one compressed text file
+			to another
+    mg_invf_merge.c	Program to merge two inverted files and
+			stemmed dictionaries
+
+See the man pages for mgmerge, mg_get_merge, mg_text_merge and
+mg_invf_merge for a brief overview.
+
+mg_get_merge.sh should really be put into mg_get.sh, with a "-merge"
+option if it is being called by the mgmerge utility, but I was
+too lazy to change it to that.
+
+
+The files updated by mgmerge are:
+
+*.text			with mg_text_merge
+*.text.idx		with mg_text_merge
+*.text.idx.wgt		with mg_weights_build (after merging)
+*.invf			with mg_invf_merge
+*.invf.idx		with      "
+*.invf.dict		with      "
+*.weight		with mg_invf_merge or mg_weights_build
+*.invf.dict.blocked	with mg_invf_dict (after merging)
+
+The *.text.stats file and *.text.dict file remain the same.
+Any other database files (ie, *.invf.dict.hash)
+will be out of date after a merge and will need to be recomputed.
+
+--------------------------------------------------------------------------
+
+MG_TEXT_MERGE
+=============
+
+The two "merging" utilities are mg_text_merge and mg_invf_merge
+
+mg_text_merge simply appends the compressed text for the new
+documents to the old compressed text file.
+For this to succeed, the two files should have been created
+with the same parameters to mg_passes and the same compression
+model must be used. So *.text.dict from the old (static) database
+is used as the model for compressing the new documents.
+The default "-C" option to mg_compression_dict in mgbuild was
+changed to "-S" so novel words in the new documents can be coded.
+The option must be the same in mgmerge -- it is "-S" there too.
+
+---------------
+
+EFFECT on FILE COMPRESSION PERFORMANCE
+
+Since the compression model is not being updated,compression
+performance for the text will slowly degrade. The only solution
+is a periodic rebuild from scratch.
+
+---------------------------------------------------------------------------
+
+MG_INVF_MERGE
+=============
+
+mg_invf_merge updates the stemmed dictionary and
+inverted file.
+For reasons of simplicity, the stemmed dictionaries merged
+are in the ".invf.dict" format, NOT the ".invf.dict.blocked"
+format. 
+So if a database is to be merged the .invf.dict file should
+not be deleted after the .invf.dict.blocked file used by mgquery
+has been created.
+Also for simplicity, level 3 inverted files are not supported, nor
+are skipped format files.
+
+
+
+Merging the inverted files requires every entry in each file
+to be decoded and merged if an entry for the same term appears
+in the other inverted file.
+This can be a slow process if the old inverted file is very large.
+Especially since each entry is decoded bit-by-bit with
+the bit-level I/O rouutines.
+As was noted on page 94 of the book "Managing Gigabytes" (hereafter 
+called "MG"), switching from bernoulli coding (called "Bblock" in
+the mg source code) to gamma or delta coding for the
+inter-document gaps would be an advantage since they require no
+parameters.
+With Bblock coding, N (the number of documents) is a parameter and
+as N changes whenever documents are added, every entry must
+be decoded and recoded.
+My approach was to keep Bblock coding, but keep the N parameter
+constant by recording the artificial N  used to code document
+gaps (called "Nstatic") as well as the real number of documents.
+
+Then, any entry in the old inverted file (called "IFold") that
+is not merged with an entry in the new inverted file (called "IFnew")
+can be copied directly to the merged inverted file (called "IFmerge").
+This can give a significant increase in speed, since usually the
+size of IFnew is very small if only a few documents are being added.
+
+To store Nstatic, a field "static_num_of_docs" was added to
+the invf_dict_header and stem_dict_header structs defined
+in invf.h
+Any program that decodes inverted file entries also had to be changed.
+(The existing mg source code that I changed has been returned with
+this document.)
+Most changes were simply altering a line that
+called BIO_Bblock_Init() so it used the static_num_of_docs
+value rather than num_of_docs.
+
+So since the file format has changed, any existing collection will
+have to be rebuilt first even using mgmerge with it is not
+intended.
+
+-------------
+
+EFFECT on INVERTED FILE COMPRESSION PERFORMANCE
+
+As more merges are accumulatively performed on a database, the
+compression performance will decline.
+The solution is an option in mg_invf_merge to do a "slow"
+merge where every inverted file entry is decoded and recoded using
+the real value of N.
+This is not a lot slower than a fast merge when compared to the
+cost of completely rebuilding the collection from scratch, and 
+the option can be used periodically (when, say, 40% of the
+inverted file has been added since the previous slow merge).
+
+-----------------
+
+THE EXACT WEIGHTS FILE
+
+Recomputing exact document weights requires a complete scan over the
+merged inverted file. This is a waste since the weights for old
+documents will (hopefully) not change much anyway, and the weights
+for new documents can be computed exactly when merging the inverted files,
+at no extra time cost.
+So mg_invf_merge computes the weights for new documents and
+updates the .weight file.
+The weights for old documents are left untouched.
+mgmerge has an option to recompute the weights file.
+This is not much more expensive in terms of time, and is only
+periodically needed.
+
+The effect of leaving the old weights unchanged is difficult to
+asess since the "correct" ranking of documents for a query is
+subjective.
+The frequency with which a rebuild of the weights file with the
+"-w" option should be done is hard to guess.
+But if most merges involves only a few documents, it makes sense
+to not bother recomputing it since the change in old weights values
+is very small and the weights only stray from their "true"
+values slowly over accumulated merging.
+
+-------------------------------------------------------------------------
+
+EXPERIMENTAL RESULTS
+====================
+
+With some small test collections (only a few Mb of text)
+and a slow merge, and a weights file rebuild,
+mgmerge typically took under 20% of the time to completely
+rebuild. Using the default options (fast merge, dont rebuild weights)
+took around 10% or more.
+For larger collections, the savings were greater, especially if the
+added text is very small.
+For example, when the last 6Kb of text of the GNUBib collection was
+added to the rest of it (14Mb) using mgmerge, the total mgmerge time
+was around 20 seconds whereas a complete mgbuild took 280 seconds.
+Only one larger collection was tested (resource contraints prevented
+testing on any really large collections): two short (one-line)
+documents were added to the Gutenberg collection, which comprised
+of nearly 74Mb of source text and had an inverted file size of 9Mb.
+mgmerge took 42 seconds (or 142 seconds with a slow merge and rebuilding the
+weights file), and I'd estimate that an mgbuild on the same machine would 
+have taken at least 20 minutes.
+
+To summarise, mgmerge is not as fast as some sophisticated method
+such as the fixed-length blocks described in section 5.7 of "MG",
+but its huge advantage is that it required almost no changes to
+the existing mg code and is still far better than using mgbuild
+every time a document needs to be added to a collection.
+
+----------------------------------------------------------------------------
+
+OTHER NOTES
+===========
+
+Since it works like mgbuild, mgmerge needs mg_get_merge
+to return the SAME text each time "mg_get_merge -text .." is called.
+
+----
+
+One other thing discovered (but not fixed) was a bug in "invf.pass2.c":
+It crashes on parsed words longer than 128 characters.
+So if any text with words >128 chars is piped to mg_passes,
+the "-N1" and "-N2" options (the memory-efficient inversion method)
+had better be used and not the "-I1"/"-I2" option (which would only
+work on small collections anyway, being the memory-inefficient
+method).
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/install-sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/install-sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGDocInfo.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
@@ -0,0 +1,42 @@
+/*
+ *    MGDocInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** DocInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @See MGQueryResult
+ */
+public class MGDocInfo
+{
+    public long num_ = 0;
+    public float rank_ = 0;
+
+    public MGDocInfo(long doc, float rank)
+    {
+	num_ = doc;
+	rank_ = rank;
+    }
+
+    public String toString()
+    {
+	return "" + num_ + " (" + rank_ + ")";
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGEquivTermInfo.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGEquivTermInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGEquivTermInfo.java	(revision 21112)
@@ -0,0 +1,53 @@
+/*
+ *    MGEquivTermInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** EquivTermInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @see MGQueryResult
+ */
+public class MGEquivTermInfo
+{
+    /** the term itself */
+    public String term_ = null;
+
+    /** the number of documents containing this term */
+    public long match_docs_ = 0;
+
+    /** overall term freq - word level */
+    public long term_freq_ = 0;
+
+
+    public MGEquivTermInfo(String term, long match, long freq)
+    {
+	term_ = term;
+	match_docs_ = match;
+	term_freq_ = freq;
+    }
+
+
+    public String toString()
+    {
+	String result = "\"" + term_ + "\"";
+	result += " (" + match_docs_ + "/" + term_freq_ + ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGPassesWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGPassesWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGPassesWrapper.java	(revision 21112)
@@ -0,0 +1,154 @@
+/*
+ *    MGPassesWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** java wrapper class for access to gs3_mg_passes in C
+ *
+ * the native side implemented in MGPassesWrapperImpl.c
+ */
+
+public class MGPassesWrapper
+{
+    static {
+	System.loadLibrary("mgpassjni");
+	initIDs();
+    }
+    
+    static final public char INVF_LEVEL_1 = '1';
+    static final public char INVF_LEVEL_2 = '2';
+    static final public char INVF_LEVEL_3 = '3';
+    
+    static final public int TEXT_PASS_1 = 0;
+    static final public int TEXT_PASS_2 = 1;
+    static final public int INDEX_PASS_1 = 2;
+    static final public int INDEX_PASS_2 = 3;
+    static final public int SPECIAL_PASS = 4;
+
+    static final public int NO_STEM_OR_CASE = 0;
+    static final public int CASE_ONLY = 1;
+    static final public int STEM_ONLY = 2;
+    static final public int STEM_AND_CASE = 3;
+
+    static final public String STEMMER_ENGLISH = "english";
+    static final public String STEMMER_FRENCH = "french";
+    static final public String STEMMER_LOVIN = "lovin";
+    static final public String STEMMER_SIMPLE_FRENCH = "simple-french";
+
+    static final private char END_OF_DOCUMENT = (char) 2;
+
+    public MGPassesWrapper() {
+	initCSide();
+    }
+
+    /** initialise the pass through the documents */
+    public native boolean init();
+    
+    /** add a pass declaration */
+    public void addPass(int pass) {
+	switch (pass) {
+	case TEXT_PASS_1:
+	    addPass('T','1');
+	    break;
+	case TEXT_PASS_2:
+	    addPass('T','2');	    
+	    break;
+	case INDEX_PASS_1:
+	    addPass('I','1');
+	    break;
+	case INDEX_PASS_2:
+	    addPass('I','2');
+	    break;
+	case SPECIAL_PASS:
+	    addPass('S','1');
+	    break;
+	}
+    }
+    /** set the base path */
+    public native void setBasePath(String basepath);
+   /** set the file name */
+    public native void setFileName(String filename);
+
+    public native void setStemOptions(String stemmer_type, int stem_method);
+
+    public native void setInvfLevel(char level);
+
+    /** Specify  the size of the document buffer in kilobytes.
+	If any document is larger than  bufsize,  the  program
+	will  abort with an error message. 
+     */
+    public native void setBufferSize(long bufsize);
+    
+    /** Maximum amount of memory to use for  the index  pass-2  file
+	inversion  in  megabytes.
+    */
+    public native void setInversionMemLimit(int limit);
+
+    /** If true, treat   SGML  tags  as  non-words  when  building  the
+	inverted file.
+    */
+    public native void ignoreSGMLTags(boolean ignore);
+    
+    /** if mg_passes fails, the document that caused teh failure will be
+	output to teh trace file or STDERR. 
+    */
+    public native void dumpFailedDocument(boolean dump);
+    
+    /** output statistics on the compression performance to  a  file  
+	called  *.compression.stats.    frequency  specifies  the  interval
+	(in kilobytes of source text) between outputting each line of 
+	statistics. 
+    */
+    public native void outputCompStats(int frequency);
+
+    /** activate tracing, a line will be output every tracepos input bytes */
+    public native void enableTracing(int tracepos);
+    /** process a Greenstone document, which may consist of many MG documents (seeparated by ^B */
+    public boolean processDocument(String docs_text) {
+	// need -1 in the following to keep empty strings at the end
+	String [] docs = docs_text.split(String.valueOf(END_OF_DOCUMENT), -1);
+	System.err.println("GS document split into "+docs.length+" mg documents");
+	for (int i=0; i<docs.length; i++) {
+	    try {
+		processMGDocument(docs[i].getBytes("UTF-8"));
+	    } catch (Exception e) {
+		e.printStackTrace();
+	    }
+	}
+	return true;
+    }
+   
+    /** finalise the pass through the documents */
+    public native boolean finish();
+
+    /** get the exit value once finished */
+    public native int exitValue();
+ 
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises any C side stuff */
+    private native boolean initCSide();
+
+    private native void addPass(char pass_type, char pass_num);
+
+    /** process a MG document */
+    private native boolean processMGDocument(byte[] text);
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGQueryResult.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
@@ -0,0 +1,155 @@
+/*
+ *    MGQueryResult.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** a java version of QueryResult for mg
+ *
+ * contains get methods for the java side, and simple
+ * set methods to be called from c side
+ *
+ */
+public class MGQueryResult
+{
+    /** the list of MGDocInfo */
+    protected Vector docs_ = null;
+    /** the list of MGTermInfo */
+    protected Vector terms_ = null;
+    /** the total number of docs found - not likely to be the size of docs_ */
+    protected long total_num_docs_ = 0;
+
+
+    MGQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+    }
+
+
+    /** clear the info from the last query
+	should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_ = 0;
+	docs_.clear();
+	terms_.clear();
+    }
+
+    public boolean isClear() {
+	return (total_num_docs_ == 0 && docs_.isEmpty() && terms_.isEmpty());
+    }
+
+    /** returns the result as a String - useful for printing out results */
+    public String toString()
+    {
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i = 0; i < docs_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGDocInfo) docs_.elementAt(i)).toString();
+	}
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = " + total_num_docs_;
+	return result;
+    }
+
+
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+
+
+    // set methods used by c++ code
+
+    public void setTotalDocs(long num) {
+	total_num_docs_=num;
+    }
+
+
+    public void addDoc(long doc, float rank) {
+	MGDocInfo doc_info = new MGDocInfo(doc, rank);
+	///ystem.out.println("(Java) Added doc " + doc_info);
+
+	docs_.add(doc_info);
+    }
+
+
+    public void addTerm(String term, int stem)
+    {
+	MGTermInfo ti = new MGTermInfo();
+	ti.term_ = term;
+	ti.stem_method_ = stem;
+	terms_.add(ti);
+	///ystem.out.println("(Java) Added term " + ti);
+    }
+
+
+    public void addEquivTerm(String term, String equivTerm,
+			     long match, long freq)
+    {
+	// Find the term to add the equivalent to
+	MGTermInfo ti = null;
+	for (int i = (terms_.size() - 1); i >= 0; i--) {
+	    ti = (MGTermInfo) terms_.elementAt(i);
+	    // Found
+	    if (ti.term_ == term) {
+		break;
+	    }
+	}
+
+	if (ti == null) {
+	    System.err.println("Internal error: No term exists to add to.\n");
+	}
+	else {
+	    ti.addEquivTerm(equivTerm, match, freq);
+	    ///ystem.out.println("(Java) Added equivalent term " + equivTerm + ", match: " + match + ", freq: " + freq);
+	}
+    }
+
+
+    // Get methods for the java side - GS2MGSearch.java, GS2MGRetrieve.java
+    public Vector getDocs() {
+	return docs_;
+    }
+
+
+    public Vector getTerms() {
+	return terms_;
+    }
+
+
+    public long getTotalDocs() {
+	return total_num_docs_;
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGTermInfo.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
@@ -0,0 +1,72 @@
+/*
+ *    MGTermInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** TermInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @see MGQueryResult
+ */
+public class MGTermInfo
+{
+    /** the term itself */
+    public String term_ = null;
+
+    /** the stem and casefold method used
+	0 = none
+	1 = casefold only
+	2 = stem only
+	3 = casefold and stem */
+    public int stem_method_ = 0;
+
+    /** list of stemmed and casefolded equivalent terms - if stem_method_ is non-zero 
+      - Vector of Strings */
+    public Vector equiv_terms_ = null;
+
+
+    public MGTermInfo()
+    {
+	equiv_terms_ = new Vector();
+    }
+
+
+    public void addEquivTerm(String equivTerm, long match, long freq)
+    {
+	equiv_terms_.add(new MGEquivTermInfo(equivTerm, match, freq));
+    }
+
+
+    /** output the class as a string */
+    public String toString()
+    {
+	String result = "\"" + term_ + "\"";
+	result += " stem(" + stem_method_ + ")";
+	result += " equiv terms(";
+	for (int i = 0; i < equiv_terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += equiv_terms_.elementAt(i);
+	}
+	result += ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGWrapper.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
@@ -0,0 +1,99 @@
+/*
+ *    MGWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** java wrapper class for access to mg in C
+ *
+ * the native side implemented in MGWrapperImpl.c
+ * uses MGQueryResult to hold the result of a query.
+ * the result of a getDocument is a String
+ * uses the jni
+ *
+ *@see MGQueryResult
+ */
+public class MGWrapper
+{
+    /** the query result, filled in by runQuery */
+    protected MGQueryResult mg_query_result_ = null;
+
+    /** pointer to c MGWrapperData class - cached indexData and queryInfo */
+    protected long mg_data_ptr_ = 0;
+
+    static {
+	System.loadLibrary("mgjni");
+	initIDs();
+    }
+
+    public MGWrapper() {
+	mg_query_result_ = new MGQueryResult();
+	initCSide();
+    }
+
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises the mg_data_ptr_ */
+    private native boolean initCSide();
+
+    /** returns a document: number docnum at level level 
+     * the base_dir and text_path paths should join together to provide
+     * the absolute location of the mg text files eg ..../index/text/demo
+     * returns the doc in utf-8
+     */
+    public native String getDocument(String base_dir, String text_dir, long docnum);
+
+    /** unloads the data */
+    public native boolean unloadIndexData();
+
+    // query param methods
+
+    /** if on=true, sets default casefolding on - it's off by default */
+    public native void setCase(boolean on);
+    /** if on=true, sets default stemming on - it's off by default */
+    public native void setStem(boolean on);
+    /** default is 50 */
+    public native void setMaxDocs(int num);
+    /** if on=true, a query returns term freq info - default is on */
+    public native void setReturnTerms(boolean on);
+    /** sets the index to search - default is 'dtx' */
+    public native void setIndex(String index);
+    /** sets the default boolean operator - AND(=1)/OR(=0) */
+    public native void setMatchMode(int mode);
+
+    /** returns a string with all the current query param settings */
+    public native String getQueryParams();
+
+    /** actually carry out the query.
+	Use the set methods to set query results.
+	Writes the result to query_result.
+     * - maintains state between requests as can be slow  
+     * base_dir and index_path should join together to provide
+     * the absolute location of the mg index files eg ..../index/dtx/demo 
+     * base_dir must end with a file separator (OS dependant)
+     */
+    public native void runQuery(String base_dir, String text_dir, String query_string);
+
+
+    /** get the result out of the wrapper */
+    public MGQueryResult getQueryResult()
+    {
+	return mg_query_result_;
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,62 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = java/org/greenstone/mg
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+JAVAC = $(JAVA_HOME)/bin/javac
+JAVAH = $(JAVA_HOME)/bin/javah
+JAR = $(JAVA_HOME)/bin/jar
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS =  -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+MGHOME=../../../..
+JAVACLASSDIR=./classes
+JAVASRCDIR=../../..
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) *.java
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGHOME)/jni org.greenstone.mg.MGWrapper
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGHOME)/jni org.greenstone.mg.MGPassesWrapper
+	$(JAR) cf $(MGHOME)/mg.jar -C $(JAVACLASSDIR) org
+
+clean:
+	rm -rf $(JAVACLASSDIR) $(MGHOME)/mg.jar $(MGHOME)/jni/org_greenstone_mg_MGWrapper.h $(MGHOME)/jni/org_greenstone_mg_MGPassesWrapper.h
+
+install:
+
+distclean: clean
+	rm -f Makefile
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Queryer.java
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
@@ -0,0 +1,175 @@
+/*
+ *    Queryer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+/** Queryer - java port of c++ Queryer 
+ * - uses MGWrapper to access mg
+ * only has get document and search - full text browse not implemented
+ *
+ * run as:
+ * java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>
+ *@see MGWrapper
+ */
+public class Queryer {
+  
+    /** outputs to std out a help message describing the commands used by 
+     * Queryer 
+     */
+    public void printHelp()
+    {
+	System.out.println( "commands available are:\n"+
+			    "\t.q\t\tquit\n"+
+			    "\t.h\t\tprint this help message\n"+
+			    "\t.d\t\tprint the current query parameter settings\n" +
+			    "\t.i<index>\tsearch index <index>\n"+
+			    "\t.t0/.t1\t\tquery type some/all\n"+
+			    "\t.c0/.c1\t\tcasefolding off/on\n"+
+			    "\t.s0/.s1\t\tstemming off/on\n"+
+			    "\t.o0/.o1\t\tshort output off/on\n"+
+			    "\t.m<num>\t\tset max docs to return to num\n\n"+
+			    "\t.p<docnum>\tprint document docnum\n"+
+			    "\t<query string>\tdo a query\n");
+    }
+
+
+    public static void main(String[] args)
+    {
+	if (args.length != 3) {
+	    System.out.println("Usage: java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>");
+	    return;
+	}
+
+	Queryer self = new Queryer();
+
+	String base_dir = args[0];
+	String index_path = args[1];
+	String text_path = args[2];
+
+	// the jni class to access mg stuff
+	MGWrapper wrapper = new MGWrapper();
+	wrapper.setIndex(index_path);
+
+	System.out.println("Welcome to Java Queryer :-)");
+	self.printHelp();
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	boolean shortOutput = false;
+
+	String command; // user input
+	char x; // the command letter
+	String data = null; // any auxiliary input to a command
+	while (true) {
+	    System.out.print(">"); // the prompt
+            try {
+                command = br.readLine();
+                command = command.trim();
+		if (command.startsWith(".")) {
+		    // a system command
+		    x = command.charAt(1);
+		    if (command.length() > 2) {
+			data = command.substring(2);
+			data = data.trim();
+		    }
+
+		    switch (x) {
+		    case 'q': // clean up and exit
+			wrapper.unloadIndexData();
+			return;
+		    case 'h': // print help message
+			self.printHelp();
+			break;
+		    case 'd': // print query param settings
+			String info = wrapper.getQueryParams();
+			System.out.println(info);
+			break;
+		    case 'p': // print doc
+			int docnum = Integer.parseInt(data);
+			String doc = wrapper.getDocument(base_dir, text_path, docnum);
+			System.out.println(doc);
+			break;
+		    case 'm': //match docs
+			int match = Integer.parseInt(data);
+			wrapper.setMaxDocs(match);
+			break;
+		    case 's': // set stem on/off
+			int stem = Integer.parseInt(data);
+			if (stem==0 ){
+			    wrapper.setStem(false);
+			} else if(stem==1) {
+			    wrapper.setStem(true);
+			} else {
+			    System.err.println("Error: stem should be 0 or 1");
+			}
+			break;
+		    case 'c': // set case on/off
+			int casef = Integer.parseInt(data);
+			if (casef==0) {
+			    wrapper.setCase(false);
+			} else if (casef==1) {
+			    wrapper.setCase(true);
+			} else {
+			    System.err.println("Error: case should be 0 or 1");
+			}
+			break;
+		    case 'i': // set index
+			wrapper.setIndex(data);
+			break;
+		    case 't': // set query type some/all
+			int type = Integer.parseInt(data);
+			if (type==0 || type==1) {
+			    wrapper.setMatchMode(type);
+			} else {
+			    System.err.println("Error: type should be 0 (some) or 1 (all)");
+			}
+			break;
+		    case 'o': // set output short/long
+			int output = Integer.parseInt(data);
+			if (output==0) {
+			    shortOutput = false;
+			} else if (output==1) {
+			    shortOutput = true;
+			} else {
+			    System.err.println("Error: output should be 0 or 1");
+			}
+			break;
+		    }
+		}
+		else {
+		    // a query
+		    wrapper.runQuery(base_dir, text_path, command);
+		    MGQueryResult res = wrapper.getQueryResult();
+		    System.out.println("(Java) Matching documents: " + res.getTotalDocs());
+		    if (shortOutput) {
+			System.out.println(res.toShortString());
+		    } else {
+			System.out.println(res.toString());
+		    }
+		}
+
+	    } catch (Exception e) {
+		System.out.println("Queryer error: "+e.getClass() + " "+e.getMessage());
+		e.printStackTrace();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/winMake.bat
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/winMake.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/java/org/greenstone/mg/winMake.bat	(revision 21112)
@@ -0,0 +1,47 @@
+@echo off
+
+set MGHOME=..\..\..\..
+
+rem ---- Where to put class files ----
+set JAVACLASSDIR=classes
+
+rem ---- Name and location of java programs ----
+set JAVAC="%JAVA_HOME%\bin\javac"
+set JAVAH="%JAVA_HOME%\bin\javah"
+set JAVA="%JAVA_HOME%\bin\java"
+set JAVADOC="%JAVA_HOME%\bin\javadoc"
+set JAR="%JAVA_HOME%\bin\jar"
+
+set JAVACOPTIONS= -deprecation -g -O
+
+
+if ""%1"" == """" goto compile
+if ""%1"" == ""compile"" goto compile
+if ""%1"" == ""install"" goto install
+if ""%1"" == ""clean"" goto clean
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:compile
+	if not exist %JAVACLASSDIR% mkdir %JAVACLASSDIR%
+	echo Compiling...
+        %JAVAC% -d %JAVACLASSDIR% %JAVACOPTIONS% *.java
+	%JAVAH% -classpath %JAVACLASSDIR% -d %MGHOME%\jni org.greenstone.mg.MGWrapper
+	%JAVAH% -classpath %JAVACLASSDIR% -d %MGHOME%\jni org.greenstone.mg.MGPassesWrapper
+	%JAR% cf %MGHOME%\mg.jar -C %JAVACLASSDIR% org
+	goto done
+
+:install
+	goto done
+
+:clean
+	echo Cleaning up...
+	if exist %JAVACLASSDIR% rmdir /S /Q %JAVACLASSDIR%
+        if exist %MGHOME%\mg.jar del %MGHOME%\mg.jar
+        if exist %MGHOME%\jni\org_greenstone_mg_MGWrapper.h del %MGHOME%\jni\org_greenstone_mg_MGWrapper.h
+        if exist %MGHOME%\jni\org_greenstone_mg_MGPassesWrapper.h del %MGHOME%\jni\org_greenstone_mg_MGPassesWrapper.h
+	goto done
+
+:done
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/MGPassesWrapperImpl.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/MGPassesWrapperImpl.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/MGPassesWrapperImpl.c	(revision 21112)
@@ -0,0 +1,243 @@
+/*
+ *    MGPassesWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <jni.h>
+#include <assert.h>
+#include "sysfuncs.h"
+#include "org_greenstone_mg_MGPassesWrapper.h"
+
+#include "mg_passes_4jni.h"
+#include "mg_files.h"
+
+/* if we need to use java objects, we should initialise their field ids here*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  return;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  clear_variables();
+  return 1;  /* true - no errors */
+}  
+
+/* add a pass type T1, T2, I1, I2, S */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_addPass(JNIEnv *j_env, 
+					       jobject j_obj, 
+					       jchar j_pass_type,
+					       jchar j_pass_num)
+{
+  /* get the level as a c char */
+  const char pass_type = j_pass_type;
+  const char pass_num = j_pass_num;
+  add_pass(pass_type, pass_num);
+  
+}
+
+/* Set the filename */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setFileName(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_filename)
+{
+  /* Get the filename as a C string */
+  const char* filename = (*j_env)->GetStringUTFChars(j_env, j_filename, NULL);
+ 
+  assert(filename != NULL);
+  set_filename(filename);
+  
+  /* Release the string */
+ (*j_env)->ReleaseStringUTFChars(j_env, j_filename, filename);
+
+}
+
+/* Set the base path */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setBasePath(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_basepath)
+{
+  /* Get the base_path as a C string */
+  const char* basepath = (*j_env)->GetStringUTFChars(j_env, j_basepath, NULL);
+  assert(basepath != NULL);
+ 
+  set_basepath(basepath);
+  
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_basepath, basepath);
+
+}
+
+/* set the level for the inverted file */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setInvfLevel(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jchar j_level)
+{
+  /* get the level as a c char */
+  const char level = j_level;
+  set_invf_level(level);
+  
+}
+
+/* set the stemmer and stem method */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_setStemOptions(JNIEnv *j_env, 
+						      jobject j_obj,
+						      jstring j_stemmer, 
+						      jint j_method)
+{
+
+  const char* stemmer = (*j_env)->GetStringUTFChars(j_env, j_stemmer, NULL);
+  int method = j_method;
+
+  assert(stemmer != NULL);
+  set_stem_options(stemmer, method);
+  
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_stemmer, stemmer);
+ }
+
+/** Specify  the size of the document buffer in kilobytes.
+    If any document is larger than  bufsize,  the  program
+    will  abort with an error message. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setBufferSize(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jlong j_bufsize){
+  long buffer = j_bufsize;
+  set_buffer_size(buffer);
+}
+    
+/** Maximum amount of memory to use for  the index  pass-2  file
+    inversion  in  megabytes.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setInversionMemLimit(JNIEnv *j_env, 
+							    jobject j_obj, 
+							    jint j_limit) {
+  int limit = j_limit;
+  set_inversion_limit(limit);
+}
+
+/** If true, treat   SGML  tags  as  non-words  when  building  the
+    inverted file.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_ignoreSGMLTags(JNIEnv *j_env, 
+						      jobject j_obj, 
+						      jboolean j_ignore){
+  int ignore = j_ignore;
+  ignore_sgml_tags(ignore);
+}
+
+/** if mg_passes fails, the document that caused the failure will be
+    output to the trace file or STDERR. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_dumpFailedDocument(JNIEnv *j_env, 
+							  jobject j_obj, 
+							  jboolean j_dump) {
+  int dump = j_dump;
+  dump_failed_document(dump);
+}
+
+/** output statistics on the compression performance to  a  file  
+    called  *.compression.stats.    frequency  specifies  the  interval
+    (in kilobytes of source text) between outputting each line of 
+    statistics. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_outputCompStats(JNIEnv *j_env, 
+						       jobject j_obj, 
+						       jint j_frequency){
+  int comp_stat_point = j_frequency;
+  set_comp_stat_point(comp_stat_point);
+  
+}
+/** activate tracing, a line will be output every tracepos input bytes */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_enableTracing(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jint j_tracepos){
+  int tracepos = j_tracepos;
+  set_trace_point(tracepos);
+}
+
+/** specify the name of the trace file */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setTraceFile(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jstring j_tracefile){
+
+  const char* tracefile = (*j_env)->GetStringUTFChars(j_env, j_tracefile, NULL);
+  assert(tracefile != NULL);
+  set_trace_file(tracefile);
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_tracefile, tracefile);
+}
+
+/* initialise the pass through the documents. must be called after all
+   the set methods 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_init(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  init_driver();
+  return 1;
+}
+
+
+/* process one document */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_processMGDocument(JNIEnv *j_env, 
+						       jobject j_obj,
+						       jbyteArray j_doc_text) {
+  /* Get the text as a C string */
+  int length = (*j_env)->GetArrayLength(j_env, j_doc_text);
+  u_char * text_buffer = (u_char *)(*j_env)->GetByteArrayElements(j_env, j_doc_text, NULL);
+  process_document(text_buffer, length);
+  /* Release the string */
+  (*j_env)->ReleaseByteArrayElements(j_env, j_doc_text, text_buffer,0);
+  return 1;
+}
+
+/* finalise the pass through the documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_finish(JNIEnv *j_env, 
+					    jobject j_obj) {
+  
+  finalise_driver();
+  return 1;
+}
+
+/** get the exit value once finished */
+JNIEXPORT jint JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_exitValue(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  return get_exit_value();
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.c	(revision 21112)
@@ -0,0 +1,725 @@
+/*
+ *    MGWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "MGWrapperImpl.h"
+#include <jni.h>
+#include "org_greenstone_mg_MGWrapper.h"
+
+#include "backend.h"
+#include "environment.h"
+#include "text_get.h"
+#include "stemmer.h"
+
+/*************************************************************************
+   NOTES
+
+ - Features supported by MGPP but not by MG (AFAIK)
+
+   - Sorting results by rank.
+     Done here as a post-processing operation. Could be more efficient in
+     some cases: the current solution is not very good when the number of
+     matching documents is large and the number of desired matches is
+     small. In this case it would be better to iterate through the array
+     picking out the best documents rather than sorting them all.
+
+   - Asking for query term frequencies to be returned.
+     This cannot be turned off in MG. If the term frequencies are not
+     required, they are simply not passed back to the Java side.
+
+   - Choosing the index for queries.
+     It is possible for MG to build Section and Document indexes (for
+     example), but these are two separate indexes, and must be loaded
+     separately. This module can load more than one index at a time, thus
+     processing queries to different indexes more quickly.
+     NOTE: This replaces TWO options in the MGPP version: returnLevel and
+           queryLevel.
+
+ *************************************************************************/
+
+
+#define MAX_INDEXES_CACHED 3
+
+/* copied from mgquery, needed to convert paragraph numbers to document numbers
+   for greenstone */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+static int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+        low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+        high = mid-1;
+      else
+        return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+  return 0;
+}
+#endif
+
+
+/*********************************************
+   initialisation stuff
+ *********************************************/
+
+/* cached ids for java stuff */
+jfieldID FID_mg_data = NULL; /* MGWrapperData */
+jfieldID FID_query_result = NULL; /* MGQueryResult */
+jmethodID MID_addDoc = NULL; /* MGQueryResult.addDoc() */
+jmethodID MID_addTerm = NULL; /* MGQueryResult.addTerm() */
+jmethodID MID_addEquivTerm = NULL; /* MGQueryResult.addEquivTerm() */
+jmethodID MID_setTotalDocs = NULL; /* MGQueryResult.setTotalDocs() */
+jmethodID MID_clearResult = NULL; /* MGQueryResult.clear() */
+
+
+/* to access objects and methods on java side, need their field/method ids - 
+ this initialises them at the start to avoid recalculating them each time they
+ are needed
+Note: the descriptors need to be exactly right, otherwise you get an error 
+saying "no such field" but no reference to the fact that it has the right 
+name but the wrong type.
+Note: apparently the jclass is a local ref and should only work 
+in the method that created it. It seems to work ok, but I'll make it
+global cos the book said I should, and it may avoid future hassles.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  jclass JC_MGQueryResult;
+
+  /* a long-"J" */
+  FID_mg_data = (*j_env)->GetFieldID(j_env, j_cls, "mg_data_ptr_", "J");
+  assert(FID_mg_data != NULL);
+
+  /* an object -"L<class name>;" */
+  FID_query_result = (*j_env)->GetFieldID(j_env, j_cls, "mg_query_result_",
+					  "Lorg/greenstone/mg/MGQueryResult;");
+  assert(FID_query_result != NULL);
+
+  /* the methods we want to use */
+  JC_MGQueryResult = (*j_env)->FindClass(j_env, "org/greenstone/mg/MGQueryResult");
+
+  /* addDoc(long doc, float rank) */
+  MID_addDoc = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addDoc", "(JF)V");
+  assert(MID_addDoc != NULL);
+
+  /* addTerm(String term, int stem) */
+  MID_addTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addTerm", "(Ljava/lang/String;I)V");
+  assert(MID_addTerm != NULL);
+
+  /* addEquivTerm(String term, String equivTerm, long match, long freq) */
+  MID_addEquivTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addEquivTerm", "(Ljava/lang/String;Ljava/lang/String;JJ)V");
+  assert(MID_addEquivTerm != NULL);
+
+  /* setTotalDocs(long) */
+  MID_setTotalDocs = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "setTotalDocs", "(J)V");
+  assert(MID_setTotalDocs != NULL);
+
+  /* clear(void) */
+  MID_clearResult = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "clear", "()V");
+  assert(MID_clearResult != NULL);
+}
+
+
+/* the java side MGWrapper has a pointer to a C object - MGWrapperData
+   initialise this and set the pointer 
+*/
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  /* Allocate a MGWrapperData object to store query parameters */
+  MGWrapperData* data = (MGWrapperData*) malloc(sizeof(MGWrapperData));
+  assert(data != NULL);
+
+  /* Set default values - no stemming, no case-folding, boolean OR queries */
+  data->defaultStemMethod = 0;
+  data->defaultBoolCombine = 0;
+
+  /* Allocate a QueryInfo object to store more query parameters */
+  data->queryInfo = (QueryInfo*) malloc(sizeof(QueryInfo));
+  assert(data->queryInfo != NULL);
+
+  /* Set default values - 50 documents max, return term freqs, sort by rank */
+  data->queryInfo->index = NULL;
+  data->queryInfo->maxDocs = 50;
+  data->queryInfo->needTermFreqs = 1;
+
+  /* Save the object on the Java side */
+  (*j_env)->SetIntField(j_env, j_obj, FID_mg_data, (long) data);
+
+  /* Initialise MG environment variables */
+  InitEnv();
+  SetEnv("expert", "true", NULL);
+  SetEnv("mode", "docnums", NULL);
+  return 1;  /* true - no errors */
+}
+
+
+/*******************************************
+   Index caching
+ *******************************************/
+
+query_data* cached_indexes[MAX_INDEXES_CACHED] = { NULL };
+
+
+/* Get the index data necessary to perform a query or document retrieval */
+query_data*
+loadIndexData(char* base_dir, char* index_path, char* text_path)
+{
+  char* index_path_name;
+  char* text_path_name;
+  query_data* qd;
+  int i = 0;
+
+  /* Form the path name of the desired indexes */
+  index_path_name = (char*) malloc(strlen(base_dir) + strlen(index_path) + 1);
+  assert(index_path_name != NULL);
+  strcpy(index_path_name, base_dir);
+  strcat(index_path_name, index_path);
+  printf("Index pathname: %s\n", index_path_name);
+
+  text_path_name = (char*) malloc(strlen(base_dir) + strlen(text_path) + 1);
+  assert(text_path_name != NULL);
+  strcpy(text_path_name, base_dir);
+  strcat(text_path_name, text_path);
+  printf("Text pathname: %s\n", text_path_name);
+
+  /* Search through the cached indexes for the desired one */
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    printf("(Cached) Pathname: %s\n", cached_indexes[i]->pathname);
+    printf("(Cached) Textpathname: %s\n", cached_indexes[i]->textpathname);
+
+    /* Check if the index has already been loaded */
+    if ((strcmp(index_path_name, cached_indexes[i]->pathname) == 0) &&
+	(strcmp(text_path_name, cached_indexes[i]->textpathname) == 0)) {
+      /* Index has already been loaded and cached, so return it */
+      printf("Found index!\n");
+      free(index_path_name);
+      free(text_path_name);
+      return cached_indexes[i];
+    }
+
+    i++;
+  }
+
+  /* Text strings no longer needed */
+  free(index_path_name);
+  free(text_path_name);
+
+  /* The index is not cached, so load it now */
+  qd = InitQuerySystem(base_dir, index_path, text_path, NULL);
+  if (!qd) {
+    printf("Error: Could not InitQuerySystem()...\n");
+    return NULL;
+  }
+
+  /* The index loaded OK, so cache it */
+  /* This could be a little more sophisticated, eg. replace least frequently used index */
+  if (i >= MAX_INDEXES_CACHED)
+    i = MAX_INDEXES_CACHED - 1;
+
+  /* Free the index being replaced */
+  if (cached_indexes[i] != NULL)
+    FinishQuerySystem(cached_indexes[i]);
+
+  /* Cache the loaded index, and return it */
+  cached_indexes[i] = qd;
+  return cached_indexes[i];
+}
+
+
+/* Clean up by unloading all cached indexes */
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_unloadIndexData(JNIEnv* j_env, jobject j_obj)
+{
+  /* Free all the loaded indexes */
+  int i = 0;
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    FinishQuerySystem(cached_indexes[i]);
+    cached_indexes[i] = NULL;
+    i++;
+  }
+
+  return 1;  /* true - no errors */
+}
+
+
+/****************************************************
+   retrieve a document
+ ****************************************************/
+
+/* Returns a document from mg as a string */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getDocument(JNIEnv *j_env, jobject j_obj,
+					     jstring j_base_dir, jstring j_text_path,
+					     jlong j_docnum)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  u_long pos, len;
+  u_char* c_buffer = NULL;
+  u_char* uc_buffer = NULL;
+  int ULen;
+
+  jstring result;
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return NULL;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return NULL;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  printf("Document retrieval, index path: %s\n", index_path);
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  if (qd==NULL) {
+    return NULL;
+  }
+  /*assert(qd != NULL);*/
+
+  /* Get the document position and length in the text file */
+  printf("Fetching document number %ld...\n", (unsigned long) j_docnum);
+  FetchDocStart(qd, (unsigned long) j_docnum, &pos, &len);
+  printf("Fetched document start. Pos: %ld, Len: %ld\n", pos, len);
+
+  /* Allocate memory for the document text (from mg/src/text/mgquery.c:RawDocOutput()) */
+  c_buffer = (u_char*) malloc(len);
+  assert(c_buffer != NULL);
+  uc_buffer = (u_char*) malloc((int) (qd->td->cth.ratio * 1.01 * len) + 100);
+  assert(uc_buffer != NULL);
+
+  /* Seek to the correct position in the file and read the document text */
+  Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+
+  /* Decompress the document text into another buffer, and terminate it */
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  uc_buffer[ULen] = '\0';
+
+  /* Load the document text into a Java string */
+  result = (*j_env)->NewStringUTF(j_env, uc_buffer);
+  assert(result != NULL);
+
+  /* Free C buffers */
+  free(c_buffer);
+  free(uc_buffer);
+
+  /* Return the document text */
+  return result;
+}
+
+
+/*******************************************
+   do a query
+ *******************************************/
+
+/* do the actual query - the results are written to query_result held on the Java side */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_runQuery(JNIEnv *j_env, jobject j_obj,
+					  jstring j_base_dir, jstring j_text_path,
+					  jstring j_query)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  jobject result_ptr;
+  char* query;
+  int i, j;
+
+  jthrowable exc;
+
+  /* First of all, clear the previous result */
+  /* The result to write to */
+  result_ptr = (*j_env)->GetObjectField(j_env, j_obj, FID_query_result);
+  assert(result_ptr != NULL);
+
+  /* Clear any previous query results */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_clearResult);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  if (qd == NULL) {
+    return;
+  }
+  /*assert(qd != NULL);*/
+
+  /* Remove anything hanging around from last time */
+  FreeQueryDocs(qd);
+
+  /* Obtain a C version of the query string */
+  query = (char*) (*j_env)->GetStringUTFChars(j_env, j_query, NULL);
+  assert(query != NULL);
+  printf("Searching for query \"%s\"...\n", query);
+
+  /* Make sure the query isn't empty */
+  if (strlen(query) == 0) {
+    printf("Warning: Empty query.\n");
+    return;
+  }
+
+  /* "Some" queries are done as ranked queries */
+  if (data->defaultBoolCombine == 0) {
+    RankedQueryInfo rqi;
+    rqi.QueryFreqs = 1;  /* Use the frequency of each query term in the query - OK? */
+    rqi.Exact = 1;  /* Perform exact ranking */
+    rqi.MaxDocsToRetrieve = data->queryInfo->maxDocs;  /* Get only the desired number */
+    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;  /* OK? */
+    /* we may need to get more paragraphs to get enough documents. I copied the following from mgquery. it seems to work, not sure why - kjdon */
+    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas")) {
+      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+    }
+    
+    rqi.Sort = 1;  /* Sort the query terms by frequency before ranking */
+    rqi.AccumMethod = 'L';  /* Use a list when accumulating (has bugs though...) */
+    /* rqi.MaxAccums = -1; */ /* Use as many accumulators as necessary - CRASHES with list */
+    rqi.MaxAccums = 100000;
+    rqi.MaxTerms = -1;  /* Use all the query terms */
+    /* rqi.StopAtMaxAccum = 0;*/  /* Don't care (using as many accumulators as necessary) */
+    rqi.StopAtMaxAccum = 1;
+    rqi.HashTblSize = 1000;  /* Don't care (not using a hash table) */
+    rqi.skip_dump = NULL;  /* Don't dump skip information */
+
+    /* RankedQuery() reads 'casefold' and 'stem' parameters from the environment */
+    SetEnv("casefold", ((data->defaultStemMethod & 1) ? "on" : "off"), NULL);
+    SetEnv("stem", ((data->defaultStemMethod & 2) ? "on" : "off"), NULL);
+
+    RankedQuery(qd, query, &rqi);
+  }
+  /* "All" queries are done as boolean queries */
+  else {
+    BooleanQueryInfo bqi;
+    bqi.MaxDocsToRetrieve = data->queryInfo->maxDocs;
+
+    /* Had to add "words$o" to LIB_OBJS in mg/src/text/Makefile and recompile mg for this */
+    BooleanQuery(qd, query, &bqi, data->defaultStemMethod);
+  }
+
+  /* Finished with the C query string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_query, query);
+
+  /* Check the query was processed successfully */
+  assert(qd->DL != NULL);
+  assert(qd->QTL != NULL);
+  assert(qd->TL != NULL);
+
+  /* Record the total number of matching documents */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_setTotalDocs, (jlong) qd->DL->num);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Record the matching documents, but only the number requested */
+  printf("Number of matching documents: %d\n", qd->DL->num);
+  
+  for (i = 0; (i < qd->DL->num && i < data->queryInfo->maxDocs); i++) {
+    int doc_num = qd->DL->DE[i].DocNum;
+    float doc_weight = qd->DL->DE[i].Weight;
+
+#if defined(PARADOCNUM) || defined(NZDL)
+    if (qd->id->ifh.InvfLevel == 3) {
+      /* pararaph level, need to convert to doc level*/
+      doc_num = GetDocNumFromParaNum(qd, doc_num);
+    }
+#endif
+    
+    
+    /* Call the addDoc function (Java side) to record a matching document */
+    (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addDoc,
+			     (jlong) doc_num, (jfloat) doc_weight);
+    exc = (*j_env)->ExceptionOccurred(j_env);
+    if (exc) {
+      (*j_env)->ExceptionDescribe(j_env);
+      return;
+    }
+  }
+
+  /* Record the term information, if desired */
+  if (data->queryInfo->needTermFreqs) {
+    /* The following code is a lot more complicated than it could be, but it is necessary
+       to compensate for an oddity in MG. */
+    unsigned char** stemmed_terms = malloc(sizeof(unsigned char*) * qd->TL->num);
+
+    printf("Number of terms: %d\n", qd->TL->num);
+    printf("Number of query terms: %d\n", qd->QTL->num);
+
+    /* Generate the stemmed form of each of the relevant terms */
+    for (i = 0; i < qd->TL->num; i++) {
+      u_char* raw_term = qd->TL->TE[i].Word;
+      unsigned int term_length = raw_term[0];
+
+      u_char* raw_stemmed_term = malloc(term_length + 1);
+      unsigned int stemmed_term_length;
+
+      /* Copy the term, and stem it */
+      for (j = 0; j <= term_length; j++)
+	raw_stemmed_term[j] = raw_term[j];
+      stemmer(data->defaultStemMethod, qd->sd->sdh.stemmer_num, raw_stemmed_term);
+
+      /* Allocate memory to store the stemmed term, and fill it */
+      stemmed_term_length = raw_stemmed_term[0];
+      stemmed_terms[i] = malloc(stemmed_term_length + 1);
+      assert(stemmed_terms[i] != NULL);
+      strncpy(stemmed_terms[i], &(raw_stemmed_term[1]), stemmed_term_length);
+      stemmed_terms[i][stemmed_term_length] = '\0';
+    }
+
+    /* Record every query term, along with their equivalent terms */
+    for (i = 0; i < qd->QTL->num; i++) {
+      u_char* raw_query_term = qd->QTL->QTE[i].Term;
+      unsigned int query_term_length = raw_query_term[0];
+      unsigned char* query_term;
+      jstring j_query_term;
+
+      u_char* raw_stemmed_query_term = malloc(query_term_length + 1);
+      unsigned int stemmed_query_term_length;
+      unsigned char* stemmed_query_term;
+
+      /* Allocate memory to store the query term, and fill it */
+      query_term = malloc(query_term_length + 1);
+      assert(query_term != NULL);
+      strncpy(query_term, &(raw_query_term[1]), query_term_length);
+      query_term[query_term_length] = '\0';
+
+      /* Allocate a new jstring for the query term */
+      j_query_term = (*j_env)->NewStringUTF(j_env, query_term);
+      assert(j_query_term != NULL);
+
+      /* Call the addTerm function (Java side) to record the query term */
+      (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addTerm,
+			       j_query_term, (jint) data->defaultStemMethod);
+      exc = (*j_env)->ExceptionOccurred(j_env);
+      if (exc) {
+	(*j_env)->ExceptionDescribe(j_env);
+	return;
+      }
+
+      /* Copy the query term, and stem it */
+      for (j = 0; j <= query_term_length; j++)
+	raw_stemmed_query_term[j] = raw_query_term[j];
+      stemmer(data->defaultStemMethod, qd->sd->sdh.stemmer_num, raw_stemmed_query_term);
+
+      /* Allocate memory to store the stemmed query term, and fill it */
+      stemmed_query_term_length = raw_stemmed_query_term[0];
+      stemmed_query_term = malloc(stemmed_query_term_length + 1);
+      assert(stemmed_query_term != NULL);
+      strncpy(stemmed_query_term, &(raw_stemmed_query_term[1]), stemmed_query_term_length);
+      stemmed_query_term[stemmed_query_term_length] = '\0';
+
+      /* Find all the terms equivalent to the query term */
+      for (j = 0; j < qd->TL->num; j++) {
+	/* Check if the stemmed query term matches the stemmed term */
+	if (strcmp(stemmed_query_term, stemmed_terms[j]) == 0) {
+	  u_char* raw_term = qd->TL->TE[j].Word;
+	  unsigned int term_length = raw_term[0];
+	  unsigned char* term;
+	  jstring j_term;
+
+	  /* Allocate memory to store the query term, and fill it */
+	  term = malloc(term_length + 1);
+	  assert(term != NULL);
+	  strncpy(term, &(raw_term[1]), term_length);
+	  term[term_length] = '\0';
+
+	  /* Allocate a new jstring for the query term */
+	  j_term = (*j_env)->NewStringUTF(j_env, term);
+	  assert(j_term != NULL);
+
+	  /* Call the addEquivTerm function (Java side) to record the equivalent term */
+	  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addEquivTerm,
+				   j_query_term, j_term,
+				   (jlong) qd->TL->TE[j].WE.doc_count,
+				   (jlong) qd->TL->TE[j].WE.count);
+	  exc = (*j_env)->ExceptionOccurred(j_env);
+	  if (exc) {
+	    (*j_env)->ExceptionDescribe(j_env);
+	    return;
+	  }
+	}
+      }
+    }
+  }
+}
+
+
+/*******************************************
+   set query options
+ *******************************************/
+
+/* Turn casefolding on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setCase(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 1;
+  } else {
+    data->defaultStemMethod &= 0xe;
+  }
+}
+
+
+/* Turn stemming on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setStem(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 2;
+  } else {
+    data->defaultStemMethod &= 0xd;
+  }
+}
+
+
+/* Set the maximum number of documents to return from a query */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setMaxDocs(JNIEnv *j_env, jobject j_obj, 
+					    jint j_max)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->maxDocs = j_max;
+}
+
+
+/* Turn term frequency recording on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setReturnTerms(JNIEnv *j_env, jobject j_obj, 
+						jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->needTermFreqs = j_on;
+}
+
+
+/* Choose MG index to search */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setIndex(JNIEnv *j_env, jobject j_obj,
+					  jstring j_index)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  /* Get the index name as a C string */
+  const char* index = (*j_env)->GetStringUTFChars(j_env, j_index, NULL);
+  assert(index != NULL);
+  printf("Choosing index %s...\n", index);
+
+  /* Free the previous index name */
+  if (data->queryInfo->index)
+    free(data->queryInfo->index);
+
+  /* Allocate memory for the index name, and fill it */
+  data->queryInfo->index = (char*) malloc(strlen(index) + 1);
+  assert(data->queryInfo->index != NULL);
+  strcpy(data->queryInfo->index, index);
+
+  /* Release the index string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_index, index);
+}
+
+
+/* Choose boolean AND or boolean OR queries */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mg_MGWrapper_setMatchMode(JNIEnv *j_env, jobject j_obj,
+					      jint j_mode)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->defaultBoolCombine = j_mode;
+}
+
+
+/* Get a text representation of the current parameter values */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getQueryParams(JNIEnv *j_env, jobject j_obj)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  char result[512];  /* Assume this is big enough */
+
+  /* Print the data to a character array */
+  sprintf(result, "Query params:\nindex\t\t%s\ncasefold\t%d\nstem\t\t%d\nquery type\t%s\nmax docs\t%ld\n",
+	  (data->queryInfo->index == NULL ? "<none loaded>" : data->queryInfo->index),
+	  (data->defaultStemMethod & 1),
+	  (data->defaultStemMethod & 2),
+	  (data->defaultBoolCombine == 1 ? "all" : "some"),
+	  (data->queryInfo->maxDocs));
+
+  /* Convert to a jstring, and return */
+  return (*j_env)->NewStringUTF(j_env, result);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/MGWrapperImpl.h	(revision 21112)
@@ -0,0 +1,52 @@
+/*
+ *    MGWrapperImpl.h
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "backend.h"
+
+
+/*************************************************************************
+   NOTES
+
+ - These data structures are based on the MGPP ones but are modified
+   to reflect the different capabilities of the MG system.
+
+ *************************************************************************/
+
+ 
+typedef struct QueryInfo
+{
+  /* Name of index to use */
+  char* index;
+  /* Maximum number of documents to retrieve */
+  unsigned long maxDocs;
+  /* Whether term frequency information is desired (boolean value) */
+  int needTermFreqs;
+} QueryInfo;
+
+
+typedef struct MGWrapperData
+{
+  /* Information about a query, see above */
+  QueryInfo* queryInfo;
+  /* Whether to perform stemming and case-folding */
+  int defaultStemMethod;
+  /* Whether to perform boolean AND (1) or boolean OR (0) queries */
+  int defaultBoolCombine;
+} MGWrapperData;
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,67 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = jni
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX -DNZDL -DQUIET -DPARADOCNUM
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#NOTE: libtextin.a needs to come before libmg.a !!!
+QUERY_LIBS = ../src/text/libtextin.a ../lib/libmg.a @LIBS@
+PASSES_LIBS = ../src/text/libmgpass.a ../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+JNIFLAGS=@JNIFLAGS@
+JNISUFFIX=@JNISUFFIX@
+JNIINC=@JNIINC@
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+CCOMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CFLAGS)
+
+.SUFFIXES:
+
+.SUFFIXES: .c .o
+.c.o:
+	$(CCOMPILE) $<
+
+# can we get rid of JAVA_HOME from here??
+INCLUDES = -I.. -I../lib -I../src/text -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JNIINC)
+
+all: compile link
+
+compile:
+	$(CCOMPILE) MGWrapperImpl.c MGPassesWrapperImpl.c
+
+link:
+	$(CC) $(LDFLAGS) $(JNIFLAGS) -o libmgjni.$(JNISUFFIX) MGWrapperImpl.o $(QUERY_LIBS)
+	$(CC) $(LDFLAGS)  $(JNIFLAGS) -o libmgpassjni.$(JNISUFFIX)  MGPassesWrapperImpl.o  $(PASSES_LIBS)
+
+clean:
+	rm -rf *.o libmgjni.$(JNISUFFIX) libmgpassjni.$(JNISUFFIX) 
+
+distclean: clean
+	rm -f Makefile
+
+install:
+	../mkinstalldirs $(libdir); \
+	cp libmgjni.$(JNISUFFIX) libmgpassjni.$(JNISUFFIX)  $(libdir)/;
+
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGPassesWrapper.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGPassesWrapper.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGPassesWrapper.h	(revision 21112)
@@ -0,0 +1,175 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mg_MGPassesWrapper */
+
+#ifndef _Included_org_greenstone_mg_MGPassesWrapper
+#define _Included_org_greenstone_mg_MGPassesWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_1
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_1 49L
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_2
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_2 50L
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_3
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_3 51L
+#undef org_greenstone_mg_MGPassesWrapper_TEXT_PASS_1
+#define org_greenstone_mg_MGPassesWrapper_TEXT_PASS_1 0L
+#undef org_greenstone_mg_MGPassesWrapper_TEXT_PASS_2
+#define org_greenstone_mg_MGPassesWrapper_TEXT_PASS_2 1L
+#undef org_greenstone_mg_MGPassesWrapper_INDEX_PASS_1
+#define org_greenstone_mg_MGPassesWrapper_INDEX_PASS_1 2L
+#undef org_greenstone_mg_MGPassesWrapper_INDEX_PASS_2
+#define org_greenstone_mg_MGPassesWrapper_INDEX_PASS_2 3L
+#undef org_greenstone_mg_MGPassesWrapper_SPECIAL_PASS
+#define org_greenstone_mg_MGPassesWrapper_SPECIAL_PASS 4L
+#undef org_greenstone_mg_MGPassesWrapper_NO_STEM_OR_CASE
+#define org_greenstone_mg_MGPassesWrapper_NO_STEM_OR_CASE 0L
+#undef org_greenstone_mg_MGPassesWrapper_CASE_ONLY
+#define org_greenstone_mg_MGPassesWrapper_CASE_ONLY 1L
+#undef org_greenstone_mg_MGPassesWrapper_STEM_ONLY
+#define org_greenstone_mg_MGPassesWrapper_STEM_ONLY 2L
+#undef org_greenstone_mg_MGPassesWrapper_STEM_AND_CASE
+#define org_greenstone_mg_MGPassesWrapper_STEM_AND_CASE 3L
+#undef org_greenstone_mg_MGPassesWrapper_END_OF_DOCUMENT
+#define org_greenstone_mg_MGPassesWrapper_END_OF_DOCUMENT 2L
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    init
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_init
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setBasePath
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setBasePath
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setFileName
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setFileName
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setStemOptions
+ * Signature: (Ljava/lang/String;I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setStemOptions
+  (JNIEnv *, jobject, jstring, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setInvfLevel
+ * Signature: (C)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setInvfLevel
+  (JNIEnv *, jobject, jchar);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setBufferSize
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setBufferSize
+  (JNIEnv *, jobject, jlong);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setInversionMemLimit
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setInversionMemLimit
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    ignoreSGMLTags
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_ignoreSGMLTags
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    dumpFailedDocument
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_dumpFailedDocument
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    outputCompStats
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_outputCompStats
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    enableTracing
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_enableTracing
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    finish
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_finish
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    exitValue
+ * Signature: ()I
+ */
+JNIEXPORT jint JNICALL Java_org_greenstone_mg_MGPassesWrapper_exitValue
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    addPass
+ * Signature: (CC)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_addPass
+  (JNIEnv *, jobject, jchar, jchar);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    processMGDocument
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_processMGDocument
+  (JNIEnv *, jobject, jbyteArray);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGWrapper.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
@@ -0,0 +1,109 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mg_MGWrapper */
+
+#ifndef _Included_org_greenstone_mg_MGWrapper
+#define _Included_org_greenstone_mg_MGWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getDocument
+ * Signature: (Ljava/lang/String;Ljava/lang/String;J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getDocument
+  (JNIEnv *, jobject, jstring, jstring, jlong);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    unloadIndexData
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_unloadIndexData
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setCase
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setCase
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setStem
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setStem
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMaxDocs
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMaxDocs
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setReturnTerms
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setReturnTerms
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setIndex
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setIndex
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMatchMode
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMatchMode
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getQueryParams
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getQueryParams
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    runQuery
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_runQuery
+  (JNIEnv *, jobject, jstring, jstring, jstring);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/jni/win32.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/jni/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/jni/win32.mak	(revision 21112)
@@ -0,0 +1,81 @@
+###########################################################################
+#
+# win32 makefile -- mg\jni
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN -DPARADOCNUM \
+       -DSHORT_SUFFIX
+INCLUDES = -I"$(MGHOME)" -I"$(MGHOME)\lib" -I"$(MGHOME)\src\text" \
+           -I"C:\j2sdk1~1.1_0\include" -I"C:\j2sdk1~1.1_0\include\win32"           
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    MGWrapperImpl.h   org_greenstone_mg_MGWrapper.h  \
+    org_greenstone_mg_MGPassesWrapper.h
+
+
+SOURCES = \
+    MGWrapperImpl.c MGPassesWrapperImpl.c
+
+OBJECTS = \
+    MGWrapperImpl$o   $(MGHOME)\src\text\libmgtextin.lib \
+    $(MGHOME)\lib\libmg.lib
+PASSES_OBJECTS = \
+    MGPassesWrapperImpl$o   $(MGHOME)\src\text\libmgpass.lib \
+    $(MGHOME)\lib\libmg.lib
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : compile link
+
+compile:
+        $(COMPILE) MGWrapperImpl.c MGPassesWrapperImpl.c
+
+link:
+        $(CC) -MD -LD $(OBJECTS) -Femgjni.dll
+        $(CC) -MD -LD $(PASSES_OBJECTS) -Femgpassjni.dll
+
+
+install: 
+       copy mgjni.dll "$(GSDL3HOME)\lib\jni"
+       copy mgpassjni.dll "$(GSDL3HOME)\lib\jni"
+clean:
+        if exist *$o del *$o 
+	if exist mgjni.dll del mgjni.dll
+	if exist mgpassjni.dll del mgpassjni.dll
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/GCC.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/GCC.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/GCC.mak	(revision 21112)
@@ -0,0 +1,84 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+
+CPPFLAGS = -g
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I".." -I"."
+
+o = .o
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+
+
+HEADERS = \
+    bitio_m_random.h filestats.h local_strings.h lovinstem.h \
+    bitio_m_stdio.h getopt.h memlib.h bitio_gen.h bitio_mem.h \
+    mgheap.h messages.h bitio_m.h bitio_mems.h huffman.h \
+    timing.h bitio_m_mem.h bitio_random.h huffman_mem.h \
+    perf_hash.h bitio_m_mems.h bitio_stdio.h huffman_stdio.h \
+    sptree.h rx.h pathmax.h getpagesize.h random.h \
+    win32in.h simplefrenchstem.h unitool.h
+
+SOURCES = \
+    bitio_random.c mgheap.c memlib.c bitio_stdio.c huffman.c \
+    messages.c bitio_gen.c filestats.c huffman_mem.c \
+    perf_hash.c bitio_mem.c getopt.c huffman_stdio.c \
+    sptree.c bitio_mems.c getopt1.c local_strings.c \
+    lovinstem.c timing.c rx.c alloca.c error.c xmalloc.c \
+    strstr.c ftruncate.c strcasecmp.c random.c win32in.c \
+    simplefrenchstem.c unitool.h
+
+OBJECTS = \
+    rx$o bitio_random$o mgheap$o memlib$o bitio_stdio$o \
+    huffman$o messages$o bitio_gen$o filestats$o huffman_mem$o \
+    perf_hash$o bitio_mem$o getopt$o huffman_stdio$o sptree$o \
+    bitio_mems$o getopt1$o local_strings$o lovinstem$o \
+    timing$o error$o xmalloc$o random$o win32in$o \
+    simplefrenchstem$o unitool$o
+ 
+all : libmg.a
+
+libmg.a : $(OBJECTS)
+	$(DELETE) libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+install: 
+ 
+clean: 
+	$(DELETE) *$o
+	$(DELETE) libmg.a
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,174 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DQUIET
+##########################################
+
+ 
+prefix = @prefix@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+ 
+INCLUDES = -I../../.. -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+    pathmax.h          getpagesize.h \
+    random.h           simplefrenchstem.h unitool.h  longlong.h
+# regex.h            rx.h               
+
+
+SOURCES = \
+    bitio_random.c    mgheap.c            memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c  \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    gmalloc.c         ftruncate.c       strcasecmp.c      random.c \
+    simplefrenchstem.c                  unitool.c
+# regex.c           rx.c              
+
+OBJECTS = @ALLOCA@ @LIBOBJS@ \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+#   gmalloc$o regex$o		rx$o 
+
+LIB_OBJECTS =   \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o          \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.a gsdllibmg.a
+
+libmg.a : $(OBJECTS)
+	rm -f libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+$(OBJECTS): ../config.h
+
+gsdllibmg.a: $(LIB_OBJECTS)
+	rm -f gsdllibmg.a
+	$(AR) cru gsdllibmg.a $(LIB_OBJECTS)
+	$(RANLIB) gsdllibmg.a
+ 
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmg.a gsdllibmg.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,91 @@
+###########################################################################
+#
+# win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN -DPARADOCNUM \
+       -DSHORT_SUFFIX
+
+INCLUDES = -I"." -I"$(MGHOME)"
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+ 
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+o = .obj
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+                       rx.h               pathmax.h          getpagesize.h \
+    random.h           win32in.h          simplefrenchstem.h unitool.h \
+    longlong.h	
+
+SOURCES = \
+    bitio_random.c    mgheap.c          memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c                            rx.c              \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    ftruncate.c       strcasecmp.c      random.c          win32in.c \
+    simplefrenchstem.c                  unitool.h
+
+OBJECTS =   rx$o \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o           \
+    error$o           xmalloc$o \
+    random$o          win32in$o         simplefrenchstem$o \
+    unitool$o
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.lib
+
+libmg.lib : $(OBJECTS)
+        if exist libmg.lib del libmg.lib
+	$(AR) /out:libmg.lib $(OBJECTS)
+
+install: 
+
+clean:
+        if exist *$o del *$o 
+        if exist libmg.lib del libmg.lib
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/alloca.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+
+
+
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BINARY_LENGTH (val, b, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Gamma_Length (unsigned long val)
+{
+  register unsigned long num;
+  GAMMA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  DELTA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  ELIAS_LENGTH (val, b, s, num);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BBLOCK_LENGTH (val, b, num);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  FLOORLOG_2 ((N - p) / p, logb);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  CEILLOG_2 (b, clogb);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+/* copied from mg-1.3f, 17-09-01 kjm18 */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  /* return ((int) (p * (2 * log2 ((double) N / p) + 1))); */
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  FLOORLOG_2 (b, logb);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  CEILLOG_2 (b, logb);
+  return logb;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,63 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m.h	(revision 21112)
@@ -0,0 +1,543 @@
+/**************************************************************************
+ *
+ * bitio_m.h -- Macros for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_M
+#define H_BITIO_M
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#ifndef BIO_ENCODE_PROLOGUE
+#define BIO_ENCODE_PROLOGUE
+#endif
+
+#ifndef BIO_DECODE_PROLOGUE
+#define BIO_DECODE_PROLOGUE
+#endif
+
+#ifndef BIO_ENCODE_EPILOGUE
+#define BIO_ENCODE_EPILOGUE
+#endif
+
+#ifndef BIO_DECODE_EPILOGUE
+#define BIO_DECODE_EPILOGUE
+#endif
+
+#ifndef DECODE_ADD
+#define DECODE_ADD(b) (b) += (b) + DECODE_BIT
+#endif
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#define POSITIVE(f, x)							\
+  if ((x)<=0) 								\
+    fprintf(stderr,"Error: Cannot "#f" encode %lu\n",(unsigned long)x),exit(1);
+
+
+#define CEILLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x) - 1;						\
+  (v) = 0;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+#define FLOORLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x);						\
+  (v) = -1;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+/****************************************************************************/
+
+
+
+#define UNARY_ENCODE(x)							\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  (count) += _B_x;							\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE(x)							\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE_L(x, count)					\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  (count) += (x);							\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+
+#define UNARY_LENGTH(x, count)						\
+do {									\
+  POSITIVE(unary, x);							\
+  (count) = (x);							\
+} while(0)
+
+
+/****************************************************************************/
+
+
+#define BINARY_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  (count) += _B_nbits;							\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+
+#define BINARY_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      (count) += _B_logofb;					       	\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+          (count)++;							\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    _B_nbits = _B_logofb;						\
+  (count) = _B_nbits;							\
+} while(0)
+
+/****************************************************************************/
+
+
+
+
+#define GAMMA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  (count) += _B_logofb*2+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  (count) += _B_nbits*2+1;						\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  (count) = _B_logofb*2+1;						\
+} while (0)
+
+
+
+/****************************************************************************/
+
+
+#define DELTA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE(_B_logx+1);						\
+  BINARY_ENCODE(_B_xxx+1, 1<<_B_logx);					\
+} while (0)
+
+#define DELTA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE_L(_B_logx+1, count);					\
+  BINARY_ENCODE_L(_B_xxx+1, 1<<_B_logx, count);				\
+} while (0)
+
+
+#define DELTA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE(_B_logx); _B_logx--;					\
+  BINARY_DECODE(_B_xxx, 1<<_B_logx); _B_xxx--;				\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE_L(_B_logx, count); _B_logx--;				\
+  BINARY_DECODE_L(_B_xxx, 1<<_B_logx, count); _B_xxx--;			\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx, _B_dcount;					\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_LENGTH(_B_logx+1, count);					\
+  BINARY_LENGTH(_B_xxx+1, 1<<_B_logx, _B_dcount);			\
+  (count) += _B_dcount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+
+
+#define ELIAS_ENCODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE(_B_k+1);							\
+  BINARY_ENCODE(_B_xx-_B_lower+1, _B_upper-_B_lower+1);			\
+} while (0)
+
+#define ELIAS_ENCODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE_L(_B_k+1, count);					\
+  BINARY_ENCODE_L(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);	\
+} while (0)
+
+#define ELIAS_DECODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE(_B_k); _B_k--;						\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE(_B_xx, _B_upper-_B_lower+1); _B_xx--;			\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_DECODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE_L(_B_k, count); _B_k--;					\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE_L(_B_xx, _B_upper-_B_lower+1, count); _B_xx--;		\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_LENGTH(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0, _B_ecount;					\
+  register double _B_pow=1.0;						\
+  POSITIVE(gamma, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_LENGTH(_B_k+1, count);						\
+  BINARY_LENGTH(_B_xx-_B_lower+1, _B_upper-_B_lower+1, _B_ecount);	\
+  (count) += _B_ecount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+#define BBLOCK_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE(_B_xdivb+1);						\
+  BINARY_ENCODE(_B_xx+1, _B_bb);					\
+} while (0)
+
+#define BBLOCK_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE_L(_B_xdivb+1, count);					\
+  BINARY_ENCODE_L(_B_xx+1, _B_bb, count);				\
+} while (0)
+
+#define BBLOCK_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE(_B_xdivb); _B_xdivb--;					\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE(_B_x1, _B_bb);						\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE_L(_B_xdivb, count); _B_xdivb--;				\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE_L(_B_x1, _B_bb, count);					\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_bcount, _B_xx = (x);			\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_LENGTH(_B_xdivb+1, count);					\
+  BINARY_LENGTH(_B_xx+1, _B_bb, _B_bcount);				\
+  (count) += _B_bcount;  						\
+} while (0)
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,191 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+	    *__pos++ = __buff;						\
+	    __remaining--;						\
+	  }  								\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_FLUSH							\
+  do {									\
+    if (__btg != sizeof(__buff)*8)					\
+      if (__remaining)							\
+	{								\
+          *__pos++ = __buff;						\
+          __remaining--;						\
+	}  								\
+    __btg = sizeof(__buff)*8;						\
+  } while (0)
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0xff;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0x00;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (!__remaining ? 							\
+    (DECODE_ERROR, 0) :							\
+    (__buff = *__pos++, __remaining--, __btg = sizeof(__buff)*8, 	\
+     (__buff >> --__btg) & 1)))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    __pos = __base + (__new_pos >> 3);					\
+    __buff = *__pos++;							\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+#define ENCODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __base[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __base[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __pos++;								\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define ENCODE_FLUSH
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    (b) += (b) + (__base[__pos>>3] & (0x80 >> (__pos&7)) != 0);		\
+    __pos++;								\
+  } while(0)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_BIT 							\
+    (__pos++, ((__base[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define DECODE_SEEK(pos) __pos = (pos)
+
+#define ENCODE_SEEK(pos) __pos = (pos)
+
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,168 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+#define ENCODE_START(f,l)						\
+  {									\
+    register FILE *__file = f;						\
+    register unsigned char *__buf;					\
+    register unsigned long __pos = 0;					\
+    register unsigned long __base = 0;					\
+    register unsigned long __used = 0;					\
+    register unsigned long __len = (l)-1;				\
+    register unsigned long __sft = 0;					\
+    while (__len) { __sft++; __len >>=1; }				\
+    __len = 1<<__sft;							\
+    __buf = Xmalloc(__len);						\
+    fseek(__file, 0, 0);						\
+    fread(__buf, 1, __len, __file);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register FILE *__file = (b).File;					\
+    register unsigned char *__buf = (b).Buf;				\
+    register unsigned long __pos = (b).pos;				\
+    register unsigned long __base = (b).Base;				\
+    register unsigned long __used = (b).Used;				\
+    register unsigned long __len = (b).len;				\
+    register unsigned long __sft = (b).sft;
+
+#define SEEK fprintf(stderr, "Seek to %d\n",__base)
+#define READ fprintf(stderr, "Read of %d\n",__len)
+#define WRITE fprintf(stderr, "Write of %d\n",__used)
+
+#define WRITE_READ 							\
+	(__used ? (fseek(__file, __base, 0),				\
+                   fwrite(__buf, 1, __len, __file)) : 0,		\
+        __base += __len,						\
+	fseek(__file, __base, 0),					\
+        fread(__buf, 1, __len, __file),					\
+	__pos = 0, __used = 0)
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __buf[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __buf[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __used = 1;								\
+    __pos++;								\
+    if ((__pos>>3) >= __len)						\
+      (void)WRITE_READ;							\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __file;							\
+    (b).Buf = __buf;							\
+    (b).pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Used = __used;							\
+    (b).len = __len;							\
+    (b).sft = __sft;							\
+  }
+
+#define ENCODE_FLUSH							\
+  if (__used)								\
+    {									\
+      fseek(__file, __base, 0);						\
+      fwrite(__buf, 1, __len, __file);					\
+      __used = 0;							\
+    }
+
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+    free(__buf);							\
+  }
+
+
+#define DECODE_START(f,l) ENCODE_START(f,l)
+
+#define DECODE_CONTINUE(b) ENCODE_CONTINUE(b)
+
+#define DECODE_BIT 							\
+    (__pos++, (((__pos-1)>>3) >= __len ? WRITE_READ, __pos=1 : 0),	\
+     ((__buf[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_ADD_FF(b) ((b) += (b) + DECODE_BIT)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_DONE ENCODE_DONE
+
+#define DECODE_PAUSE(b)	ENCODE_PAUSE(b)
+
+#define ENCODE_SEEK(pos) 						\
+  do {									\
+    if ((((pos)>>3) >= __base) && ((((pos)+7)>>3) < (__base+__len)))   	\
+      __pos = (pos)-(__base << 3); 					\
+    else								\
+      {									\
+        ENCODE_FLUSH;							\
+        __base = (pos) >> 3;						\
+        __base = ((pos)  >> (__sft+3)) << __sft;			\
+        fseek(__file, __base, 0);					\
+        fread(__buf, 1, __len, __file);					\
+        __pos = (pos)&7; 						\
+        __pos = (pos)&((8 << __sft)-1);					\
+      }									\
+  } while(0)
+
+
+#define DECODE_SEEK(pos) ENCODE_SEEK(pos)
+
+#define ENCODE_TELL ((__base << 3) + __pos)
+
+#define DECODE_TELL ENCODE_TELL
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,147 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(f)							\
+  {									\
+    FILE *__outfile = f;						\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    FILE *__outfile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	putc(__buff, __outfile);					\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __outfile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_DONE							\
+    if (__btg != sizeof(__buff)*8)					\
+      putc(__buff,__outfile);						\
+  }
+
+
+#define DECODE_START(f)							\
+  {									\
+    FILE *__infile = f;							\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    FILE *__infile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (__buff = getc(__infile), (feof(__infile) ?				\
+    (DECODE_ERROR, 0) :							\
+    (__btg = sizeof(__buff)*8, (__buff >> --__btg) & 1))))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).File = __infile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    fseek(__infile, __new_pos >> 3, 0);					\
+    __buff = getc(__infile);						\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.c	(revision 21112)
@@ -0,0 +1,259 @@
+/**************************************************************************
+ *
+ * bitio_mem.c -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  ENCODE_START (buf, rem)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mem_Encode_Done (mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  DECODE_START (buf, rem)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Elias_Decode (unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mem.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * bitio_mem.h -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are faster than the ones 
+ *  in "mems" files. But with these routines you cannot mix reads and writes 
+ *  on the array of chars, or multiple write, at the same time and guarantee
+ *  them to work, also you cannot seek to a point and do a write. The decode
+ *  routine can detect when you run off the end of the array and will produce
+ *  an approate error message, and the encode routine will stop when it gets
+ *  to the end of the character array. 
+ *
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_MEM
+#define H_BITIO_MEM
+
+
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs);
+void BIO_Mem_Encode_Done (mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+				    unsigned long *bits);
+
+
+
+void BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+			   mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Elias_Decode (unsigned long b, double s,
+				 mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+void BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.c	(revision 21112)
@@ -0,0 +1,268 @@
+/**************************************************************************
+ *
+ * bitio_mems.c -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mems.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mems_Encode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  ENCODE_START (buf, pos)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Done (mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mems_Decode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  DECODE_START (buf, pos)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Elias_Decode (unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_mems.h	(revision 21112)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * bitio_mems.h -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_MEMS
+#define H_BITIO_MEMS
+
+
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mems_Encode_Start (char *buf, unsigned long pos, mems_bitio_state * bs);
+void BIO_Mems_Encode_Done (mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Decode_Start (void *buf, unsigned long pos, mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+			    mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Elias_Decode (unsigned long b, double s,
+				mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.c	(revision 21112)
@@ -0,0 +1,331 @@
+/*************************************************************************
+ *
+ * bitio_random.c -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:45:03  kjm18
+   updated mg to be in line with version mg-1.3f. Now uses long long for
+   some variables to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:16:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:50  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:40  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "longlong.h"
+
+#include "bitio_m_random.h"
+#include "bitio_m.h"
+
+
+void 
+BIO_Random_Start (FILE * f, unsigned long len,
+		  random_bitio_state * bs)
+{
+  ENCODE_START (f, len)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Random_Done (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+void 
+BIO_Random_Encode_Bit (int bit, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+int 
+BIO_Random_Decode_Bit (random_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+void 
+BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Unary_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Binary_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Gamma_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Delta_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			 random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Elias_Decode (unsigned long b, double s,
+			 random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Bblock_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Seek (unsigned long pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
+void 
+BIO_Random_Flush (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_FLUSH;
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Random_Tell (random_bitio_state * bs)
+{
+  register unsigned long t;
+  ENCODE_CONTINUE (*bs)
+    t = ENCODE_TELL;
+  ENCODE_PAUSE (*bs)
+    return (t);
+}
+
+#ifdef USE_LONG_LONG
+
+void 
+BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE(*bs)
+
+    if ((((pos) >> 3) >= (mg_ullong)__base) &&
+	((((pos)+7) >> 3) < (mg_ullong)(__base + __len)))
+      {
+	__pos = (long)((pos) - (mg_ullong)(__base << 3));
+      }
+    else
+      {
+        ENCODE_FLUSH;
+	__base = (long)(((pos) >> (__sft+3)) << __sft);
+	
+	fseek(__file,__base,0);
+	fread(__buf,1,__len,__file);
+	__pos = (long)((pos) & ((8 << __sft)-1));
+      }
+  
+  ENCODE_PAUSE(*bs)
+}
+
+mg_ullong 
+BIO_Random_Tell_LL (random_bitio_state * bs)
+{
+  mg_ullong t;
+  ENCODE_CONTINUE(*bs)
+    t = (((mg_ullong)__base) << 3ull) + __pos;
+  ENCODE_PAUSE(*bs)
+  return(t);  
+}
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_random.h -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ *  Modified:
+ *   - long long seek and tell ops
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ **************************************************************************/
+
+#ifndef H_BITIO_RANDOM
+#define H_BITIO_RANDOM
+
+#include "longlong.h"
+
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Random_Start (FILE * f, unsigned long len,
+		       random_bitio_state * bs);
+void BIO_Random_Done (random_bitio_state * bs);
+
+
+
+void BIO_Random_Decode_Start (void *buf, unsigned long pos,
+			      random_bitio_state * bs);
+
+void BIO_Random_Encode_Bit (int bit, random_bitio_state * bs);
+
+int BIO_Random_Decode_Bit (random_bitio_state * bs);
+
+
+void BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Unary_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Binary_Decode (unsigned long b, random_bitio_state * bs,
+					unsigned long *bits);
+
+
+
+void BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Gamma_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Delta_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			      random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Elias_Decode (unsigned long b, double s,
+				       random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Bblock_Decode (unsigned long b,
+					random_bitio_state * bs,
+					unsigned long *bits);
+
+
+void BIO_Random_Seek (unsigned long pos, random_bitio_state * bs);
+
+void BIO_Random_Flush (random_bitio_state * bs);
+
+unsigned long BIO_Random_Tell (random_bitio_state * bs);
+
+#ifdef USE_LONG_LONG
+
+void BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs);
+mg_ullong BIO_Random_Tell_LL (random_bitio_state * bs);
+
+#endif /* USE_LONG_LONG */
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.c	(revision 21112)
@@ -0,0 +1,284 @@
+/**************************************************************************
+ *
+ * bitio_stdio.c -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:41  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_stdio.h"
+#include "bitio_m.h"
+
+
+
+void 
+BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  ENCODE_START (f)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Done (stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  DECODE_START (f)
+    DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+
+int 
+BIO_Stdio_Decode_Bit (stdio_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Elias_Decode (unsigned long b, double s,
+			stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+void 
+BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/bitio_stdio.h	(revision 21112)
@@ -0,0 +1,106 @@
+/**************************************************************************
+ *
+ * bitio_stdio.h -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of number on a FILE*. With these function you cannot mix reads and writes
+ *  on the FILE, or multiple writes, at the same time and guarantee them to 
+ *  work, also you  cannot seek to a point and do a write. The decode function
+ *  can detect when you run off the end of the file and will produce an 
+ *  approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_STDIO
+#define H_BITIO_STDIO
+
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs);
+void BIO_Stdio_Encode_Done (stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs);
+int BIO_Stdio_Decode_Bit (stdio_bitio_state * bs);
+
+
+void BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			     stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Elias_Decode (unsigned long b, double s,
+				      stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/error.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/error.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/error.c	(revision 21112)
@@ -0,0 +1,127 @@
+/* error.c -- error handler for noninteractive utilities
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by David MacKenzie <djm@gnu.ai.mit.edu>.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <stdio.h>
+
+#if HAVE_VPRINTF || HAVE_DOPRNT
+# if __STDC__
+#  include <stdarg.h>
+#  define VA_START(args, lastarg) va_start(args, lastarg)
+# else
+#  include <varargs.h>
+#  define VA_START(args, lastarg) va_start(args)
+# endif
+#else
+# define va_alist a1, a2, a3, a4, a5, a6, a7, a8
+# define va_dcl char *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8;
+#endif
+
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <string.h>
+#else
+void exit ();
+#endif
+
+/* If NULL, error will flush stdout, then print on stderr the program
+   name, a colon and a space.  Otherwise, error will call this
+   function without parameters instead.  */
+void (*error_print_progname) () = NULL;
+
+/* The calling program should define program_name and set it to the
+   name of the executing program.  */
+#if 0 /* [RB/TS:10/95] do not use program_name */
+extern char *program_name;
+#endif
+
+#if HAVE_STRERROR
+char *strerror (int errnum);
+#else
+static char *
+private_strerror (errnum)
+     int errnum;
+{
+  extern char *sys_errlist[];
+  extern int sys_nerr;
+
+  if (errnum > 0 && errnum <= sys_nerr)
+    return sys_errlist[errnum];
+  return "Unknown system error";
+}
+#define strerror private_strerror
+#endif
+
+/* Print the program name and error message MESSAGE, which is a printf-style
+   format string with optional args.
+   If ERRNUM is nonzero, print its corresponding system error message.
+   Exit with status STATUS if it is nonzero.  */
+/* VARARGS */
+
+void
+#if defined(VA_START) && __STDC__
+error (int status, int errnum, const char *message, ...)
+#else
+error (status, errnum, message, va_alist)
+     int status;
+     int errnum;
+     char *message;
+     va_dcl
+#endif
+{
+#ifdef VA_START
+  va_list args;
+#endif
+
+  if (error_print_progname)
+    (*error_print_progname) ();
+  else
+    {
+      fflush (stdout);
+#if 0 /* [RB/TS:10/95] do not use program_name */
+      fprintf (stderr, "%s: ", program_name);
+#endif
+    }
+
+#ifdef VA_START
+  VA_START (args, message);
+# if HAVE_VPRINTF
+  vfprintf (stderr, message, args);
+# else
+  _doprnt (message, args, stderr);
+# endif
+  va_end (args);
+#else
+  fprintf (stderr, message, a1, a2, a3, a4, a5, a6, a7, a8);
+#endif
+
+  if (errnum)
+    fprintf (stderr, ": %s", strerror (errnum));
+  putc ('\n', stderr);
+  fflush (stderr);
+  if (status)
+    exit (status);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.c	(revision 21112)
@@ -0,0 +1,136 @@
+/**************************************************************************
+ *
+ * filestats.c -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:54  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:42  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+File *
+Fopen (char *name, char *mode, unsigned long magic)
+{
+  FILE *f;
+  File *F;
+  unsigned long m;
+  if (!(f = fopen (name, mode)))
+    return (NULL);
+  if (magic)
+    switch (*mode)
+      {
+      case 'r':
+	fread ((char *) &m, sizeof (m), 1, f);
+	NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (m != magic)
+	  {
+	    fclose (f);
+	    return (NULL);
+	  }
+	break;
+      case 'w':
+	HTONUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+	fwrite ((char *) &magic, sizeof (magic), 1, f);
+      }
+  if (!(F = Xmalloc (sizeof (File))))
+    {
+      fclose (f);
+      return (NULL);
+    }
+  F->pathname = Xstrdup (name);
+  F->name = strrchr (F->pathname, '/');
+  F->name = F->name ? F->name + 1 : F->pathname;
+  F->f = f;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+  F->Cumulative = F->Current;
+  return (F);
+}
+
+
+int 
+Fclose (File * F)
+{
+  int num;
+  if (!F)
+    return (0);
+  num = fclose (F->f);
+  if (F->pathname)
+    Xfree (F->pathname);
+  Xfree (F);
+  return (num);
+}
+
+size_t 
+Fread (void *ptr, size_t size, size_t nitems, File * F)
+{
+  int num;
+  num = fread ((char *) ptr, size, nitems, F->f);
+  F->Current.NumReads++;
+  F->Current.NumBytes += num * size;
+  return (num);
+}
+
+int 
+Fseek (File * F, long offset, int ptrname)
+{
+  int num;
+  num = fseek (F->f, offset, ptrname);
+  F->Current.NumSeeks++;
+  return (num);
+}
+
+void 
+Rewind (File * F)
+{
+  rewind (F->f);
+  F->Current.NumSeeks++;
+}
+
+
+void 
+ZeroFileStats (File * F)
+{
+  F->Cumulative.NumSeeks += F->Current.NumSeeks;
+  F->Cumulative.NumReads += F->Current.NumReads;
+  F->Cumulative.NumBytes += F->Current.NumBytes;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/filestats.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filestats.h -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_FILESTATS
+#define H_FILESTATS
+
+#include "sysfuncs.h"
+
+typedef struct FileStats
+  {
+    unsigned long NumSeeks;
+    unsigned long NumReads;
+    unsigned long NumBytes;
+  }
+FileStats;
+
+
+
+typedef struct
+  {
+    FILE *f;
+    char *pathname;
+    char *name;
+    FileStats Current;
+    FileStats Cumulative;
+  }
+File;
+
+/* if magic is 0 no magic number is read or written */
+File *Fopen (char *name, char *mode, unsigned long magic);
+
+size_t Fread (void *ptr, size_t size, size_t nitems, File * F);
+
+int Fseek (File * F, long offset, int ptrname);
+
+void Rewind (File * F);
+
+int Fclose (File * F);
+
+#define Getc(F) (F->Current.NumReads++, F->Current.NumBytes++, getc(F->f))
+
+/* This adds the Current file stats to the Cumulative stats and then zeros
+   the Current stats */
+void ZeroFileStats (File * F);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/ftruncate.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/ftruncate.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/ftruncate.c	(revision 21112)
@@ -0,0 +1,107 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.c	(revision 21112)
@@ -0,0 +1,750 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "getopt.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt.h	(revision 21112)
@@ -0,0 +1,130 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  extern int getopt ();
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt1.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt1.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/getopt1.c	(revision 21112)
@@ -0,0 +1,178 @@
+/* getopt_long and getopt_long_only entry points for GNU getopt.
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#include "getopt.h"
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef __GNU_LIBRARY__
+#include <stdlib.h>
+#else
+char *getenv ();
+#endif
+
+#ifndef	NULL
+#define NULL 0
+#endif
+
+int
+getopt_long (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 0);
+}
+
+/* Like getopt_long, but '-' as well as '--' can indicate a long option.
+   If an option that starts with '-' (not '--') doesn't match a long option,
+   but does match a short option, it is parsed as a short option
+   instead.  */
+
+int
+getopt_long_only (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 1);
+}
+
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+#include <stdio.h>
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+      int option_index = 0;
+      static struct option long_options[] =
+      {
+	{"add", 1, 0, 0},
+	{"append", 0, 0, 0},
+	{"delete", 1, 0, 0},
+	{"verbose", 0, 0, 0},
+	{"create", 0, 0, 0},
+	{"file", 1, 0, 0},
+	{0, 0, 0, 0}
+      };
+
+      c = getopt_long (argc, argv, "abc:d:0123456789",
+		       long_options, &option_index);
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case 0:
+	  printf ("option %s", long_options[option_index].name);
+	  if (optarg)
+	    printf (" with arg %s", optarg);
+	  printf ("\n");
+	  break;
+
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case 'd':
+	  printf ("option d with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.c	(revision 21112)
@@ -0,0 +1,426 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:44  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = Xmalloc (sizeof (huff_data))))
+	goto error0;
+    }
+  if (!(heap = Xmalloc (num * 2 * sizeof (*heap))))
+    goto error1;
+  if (!(hd->clens = Xmalloc (num * sizeof (char))))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  bcopy ((char *) freqs, (char *) &heap[num], num * sizeof (*heap));
+  bzero ((char *) heap, num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; i++)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; i--)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    child++;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; i++)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		child++;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	HNum++;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; i++)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  bzero ((char *) hd->lencount, sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; i++)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      hd->lencount[codelen]++;
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; i--)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  free (heap);
+  return (hd);
+
+error2:
+  free (heap);
+error1:
+  if (!data)
+    free (hd);
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = Xmalloc (data->num_codes * sizeof (*codes))))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  bcopy ((char *) data->min_code, (char *) mc, sizeof (mc));
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = Xmalloc (data->num_codes * sizeof (*vals))))
+    return (NULL);
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *))))
+    {
+      free (vals);
+      return (NULL);
+    }
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; i++)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; i++)
+    size += counts[i] * hd.clens[i];
+  Xfree (hd.clens);
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+
+static int 
+General_Read_Huffman_Data (size_t (*rd) (), void *f,
+			   huff_data * hd, u_long * mem, u_long * disk)
+{
+  if (rd (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      bzero ((char *) (hd->lencount), sizeof (hd->lencount));
+      if (rd (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      bzero ((char *) (hd->min_code), sizeof (hd->min_code));
+
+      if (rd (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = Xmalloc (sizeof (*hd->clens) * hd->num_codes)))
+	return -1;
+
+
+      if (rd (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
+
+
+
+int 
+Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (fread, f, hd, mem, disk);
+}
+
+
+
+
+int 
+F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (Fread, f, hd, mem, disk);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+
+
+#define MAX_HUFFCODE_LEN 31
+
+typedef struct huff_data
+  {
+    int num_codes;
+    int mincodelen;
+    int maxcodelen;
+    int lencount[MAX_HUFFCODE_LEN + 1];
+    unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+    char *clens;
+  }
+huff_data;
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+int F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+
+#define HUFF_ENCODE(x, codes, lens)					\
+  do {									\
+    register int __i;							\
+    register int __clen = (lens)[x];					\
+    register unsigned long __code = (codes)[x];				\
+    for (__i=__clen-1; __i>=0; __i--)					\
+      ENCODE_BIT((__code >> __i) & 1);					\
+  } while(0)
+
+#define HUFF_ENCODE_L(x, codes, lens, count)				\
+  do {									\
+    HUFF_ENCODE(x, codes, lens);		   			\
+    (count) += (lens)[x];						\
+  } while(0)
+
+
+#define HUFF_DECODE(x, mcodes, values)					\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+#define HUFF_DECODE_L(x, mcodes, values, count)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+	(count)++;							\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.c	(revision 21112)
@@ -0,0 +1,69 @@
+/**************************************************************************
+ *
+ * huffman_mem.c -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:45  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+#include "bitio_m_mem.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+#endif
+
+void 
+BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+		     char *clens, mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Mem_Huff_Decode (unsigned long *mincodes,
+		     unsigned long **values, mem_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_mem.h	(revision 21112)
@@ -0,0 +1,36 @@
+/**************************************************************************
+ *
+ * huffman_mem.h -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_MEM
+#define H_HUFFMAN_MEM
+
+
+
+void BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+			  char *clens, mem_bitio_state * bs);
+
+unsigned long BIO_Mem_Huff_Decode (unsigned long *mincodes,
+			      unsigned long **values, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.c	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * huffman_stdio.c -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:52  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:07  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:46  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+int _filbuf (FILE *);
+int _flsbuf (unsigned, FILE *);
+#endif
+
+void 
+BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+		       char *clens, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+		       unsigned long **values,
+		       stdio_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/huffman_stdio.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * huffman_stdio.h -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_STDIO
+#define H_HUFFMAN_STDIO
+
+
+
+void BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+			    char *clens, stdio_bitio_state * bs);
+
+
+unsigned long BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+				     unsigned long **values,
+				     stdio_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.c	(revision 21112)
@@ -0,0 +1,415 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = Xmalloc (src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  bcopy ((char *) src_str, (char *) (copy), src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    i++;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  bzero (buf, sizeof (buf));
+  for (i = 0; i < len; i++)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  s++;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      s--;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; i++)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/longlong.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/longlong.h	(revision 21112)
@@ -0,0 +1,81 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifndef __WIN32__
+#include "config.h"
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+	mgstat.c
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.c	(revision 21112)
@@ -0,0 +1,1187 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();
+
+static int 
+aio (ch)			/* Rule number 9     */
+     char ch;
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (ch)				/* Rule Number 24    */
+     char ch;
+{
+  return (ch != 's');
+}
+
+static int 
+pt (ch)				/* Rule number 28    */
+     char ch;
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (ch)				/* Rule number 30    */
+     char ch;
+{
+  return (ch != 'm');
+}
+
+static int 
+n (ch)				/* Rule number 32    */
+     char ch;
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+
+static int 
+B (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'e');
+}
+
+static int 
+F (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'c');
+}
+
+static int 
+W (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  bcopy ((char *) word, (char *) copy, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  bcopy ((char *) copy, (char *) word, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) ();		/* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  rule++;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) ();		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      suffix_start++;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,56 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:36  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:32:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:20:10  tes
+   * Changes made for 1.1
+   *
+   *
+ */
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.c	(revision 21112)
@@ -0,0 +1,66 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/11/29 03:15:13  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:47  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+/* change the name so we do not have problems with other libs */
+char *
+my_mg_strdup(const char *str)
+{
+  char *ret_str = malloc(strlen(str)+1);
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_mg_strdup;
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/memlib.h	(revision 21112)
@@ -0,0 +1,51 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+#ifdef STD_MEM
+
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+#define Xstrdup strdup
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.c	(revision 21112)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/23 03:38:06  rjmcnab
+
+   combined unix and windows versions of mg
+
+   Revision 1.1  1998/11/19 03:00:32  sjboddie
+   Added windows gdbm and mg versions
+
+   * Revision 1.1  1994/08/22  00:24:48  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef QUIET
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/messages.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+
+void Message (const char *fmt,...);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.c	(revision 21112)
@@ -0,0 +1,173 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1999/01/15 03:05:50  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.1  1998/11/17 09:32:01  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:43  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; __i--)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; __i--)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	child++;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; i--)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; i--)
+    {
+      SWAP (size, heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)--;
+  SWAP (size, heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)++;
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,84 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/netorder.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/netorder.h	(revision 21112)
@@ -0,0 +1,178 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &hd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &hd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &nd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &nd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &nd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &hd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &hd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define NTOHF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define HTONF2(hf, nf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(hf), (char *) &tmp, sizeof (float));                \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(nf), sizeof (float));                \
+	}while(0)
+#define NTOHF2(nf, hf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(nf), (char *) &tmp, sizeof (float));                \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(hf), sizeof (float));                \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/pathmax.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.c	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/02/22 01:35:03  jrm21
+   Fixed bug that caused perf_hash to fail for lexicon of < 4 words.
+   Added a #define to increase size of n (for vertices in the graph) as we
+   need to allocate more memory somewhere else before calling tree_builder()
+
+   Revision 1.1  1999/08/10 21:16:57  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:17  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/08/22  00:26:30  tes
+   * Made perf_hash take the ceiling of 1.23 * num
+   * in lieu of the floor.
+   * This was done so that perf_hash works ok on
+   * a small number of keys.
+   *
+   * Revision 1.1  1994/08/22  00:24:49  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define STRUCT
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+/* An arbitrary amount to add to both MAX_N and n for building tree/acyclic
+   graph -- John McPherson - jrm21 (22-02-2001) */
+#define MAX_N_HEADROOM  100
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  StackPos++;
+  while (StackPos)
+    {
+      int m, b;
+      StackPos--;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; e++)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; v++)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      /* Added variable - need to increase MAX_N on allocate as well */
+      *n = (int) (num * c + MAX_N_HEADROOM); /* john mcpherson 22-02-2001 */
+
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; i++)
+	for (j = 0; j < MAX_L; j++)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; e++)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      w++;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; e++)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; i++)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  bzero ((char *) translate, sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; i++)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, s++)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; i++)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; i++)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = Xmalloc (sizeof (int) * MAX_M);
+  mk = Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    if (tb)
+      if (!(tb[i] = Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = Xmalloc (sizeof (u_char) * 256);
+  bzero ((char *) translate, sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  /* For extremely small lexicons (<4 words), this isn't good enough! 
+   -- John McPherson jrm21 22-02-2001 */
+  /* MAX_N = MAX_M + MAX_M / 4; */
+  MAX_N = (int)(ceil (1.25 * MAX_M)); 
+  /* Add some space for "experimentation" of n in tree_builder() */
+  MAX_N += MAX_N_HEADROOM;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  bcopy ((char *) &g[1], (char *) g, n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = Xmalloc (sizeof (u_char) * 256);
+  phd->g = Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb)
+      if (!(phd->tb[i] = Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      l--;
+      s++;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+
+
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/random.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/random.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/random.c	(revision 21112)
@@ -0,0 +1,228 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:14:16  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:16:57  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:20  rjmcnab
+*** empty log message ***
+
+*/
+
+static char *RCSID = "$Id$";
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/random.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/random.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/random.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:14:16  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:16:58  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:21  rjmcnab
+*** empty log message ***
+
+*/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.c	(revision 21112)
@@ -0,0 +1,5426 @@
+/* Extended regular expression matching and search library,
+   version 0.12.
+   (Implements POSIX draft P10003.2/D11.2, except for
+   internationalization features.)
+
+   Copyright (C) 1993, 1994, 1995 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* AIX requires this to be the first thing in the file. */
+#if defined (_AIX) && !defined (REGEX_MALLOC)
+  #pragma alloca
+#endif
+
+#define _GNU_SOURCE
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+/* We need this for `regex.h', and perhaps for the Emacs include files.  */
+#include <sys/types.h>
+
+/* This is for other GNU distributions with internationalized messages.  */
+#if HAVE_LIBINTL_H || defined (_LIBC)
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+/* The `emacs' switch turns on certain matching commands
+   that make sense only in Emacs. */
+#ifdef emacs
+
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+
+#else  /* not emacs */
+
+/* If we are not linking with Emacs proper,
+   we can't use the relocating allocator
+   even if sysfuncs.h says that we can.  */
+#undef REL_ALLOC
+
+#if defined (STDC_HEADERS) || defined (_LIBC)
+#include <stdlib.h>
+#else
+char *malloc ();
+char *realloc ();
+#endif
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+   `BSTRING', as far as I know, and neither of them use this code.  */
+#ifndef INHIBIT_STRING_HEADER
+#if HAVE_STRING_H || STDC_HEADERS || defined (_LIBC)
+#include <string.h>
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+#else
+#include <strings.h>
+#endif
+#endif
+
+/* Define the syntax stuff for \<, \>, etc.  */
+
+/* This must be nonzero for the wordchar and notwordchar pattern
+   commands in re_match_2.  */
+#ifndef Sword 
+#define Sword 1
+#endif
+
+#ifdef SWITCH_ENUM_BUG
+#define SWITCH_ENUM_CAST(x) ((int)(x))
+#else
+#define SWITCH_ENUM_CAST(x) (x)
+#endif
+
+#ifdef SYNTAX_TABLE
+
+extern char *re_syntax_table;
+
+#else /* not SYNTAX_TABLE */
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE 256
+
+static char re_syntax_table[CHAR_SET_SIZE];
+
+static void
+init_syntax_once ()
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+
+#endif /* not SYNTAX_TABLE */
+
+#define SYNTAX(c) re_syntax_table[c]
+
+#endif /* not emacs */
+
+
+/* Get the interface, including the syntax bits.  */
+#include "regex.h"
+
+/* isalpha etc. are used for the character classes.  */
+#include <ctype.h>
+
+/* Jim Meyering writes:
+
+   "... Some ctype macros are valid only for character codes that
+   isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when
+   using /bin/cc or gcc but without giving an ansi option).  So, all
+   ctype uses should be through macros like ISPRINT...  If
+   STDC_HEADERS is defined, then autoconf has verified that the ctype
+   macros don't need to be guarded with references to isascii. ...
+   Defining isascii to 1 should let any compiler worth its salt
+   eliminate the && through constant folding."  */
+
+#ifndef ISASCII
+#  if defined (STDC_HEADERS) || (!defined (isascii) && !defined (HAVE_ISASCII))
+#    define ISASCII(c) 1
+#  else
+#    define ISASCII(c) isascii(c)
+#  endif
+#endif
+
+#ifdef isblank
+#define ISBLANK(c) (ISASCII (c) && isblank (c))
+#else
+#define ISBLANK(c) ((c) == ' ' || (c) == '\t')
+#endif
+#ifdef isgraph
+#define ISGRAPH(c) (ISASCII (c) && isgraph (c))
+#else
+#define ISGRAPH(c) (ISASCII (c) && isprint (c) && !isspace (c))
+#endif
+
+#define ISPRINT(c) (ISASCII (c) && isprint (c))
+#define ISDIGIT(c) (ISASCII (c) && isdigit (c))
+#define ISALNUM(c) (ISASCII (c) && isalnum (c))
+#define ISALPHA(c) (ISASCII (c) && isalpha (c))
+#define ISCNTRL(c) (ISASCII (c) && iscntrl (c))
+#define ISLOWER(c) (ISASCII (c) && islower (c))
+#define ISPUNCT(c) (ISASCII (c) && ispunct (c))
+#define ISSPACE(c) (ISASCII (c) && isspace (c))
+#define ISUPPER(c) (ISASCII (c) && isupper (c))
+#define ISXDIGIT(c) (ISASCII (c) && isxdigit (c))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* We remove any previous definition of `SIGN_EXTEND_CHAR',
+   since ours (we hope) works properly with all combinations of
+   machines, compilers, `char' and `unsigned char' argument types.
+   (Per Bothner suggested the basic approach.)  */
+#undef SIGN_EXTEND_CHAR
+#if __STDC__
+#define SIGN_EXTEND_CHAR(c) ((signed char) (c))
+#else  /* not __STDC__ */
+/* As in Harbison and Steele.  */
+#define SIGN_EXTEND_CHAR(c) ((((unsigned char) (c)) ^ 128) - 128)
+#endif
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+   use `alloca' instead of `malloc'.  This is because using malloc in
+   re_search* or re_match* could cause memory leaks when C-g is used in
+   Emacs; also, malloc is slower and causes storage fragmentation.  On
+   the other hand, malloc is more portable, and easier to debug.  
+   
+   Because we sometimes use alloca, some routines have to be macros,
+   not functions -- `alloca'-allocated space disappears at the end of the
+   function it is called in.  */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE malloc
+#define REGEX_REALLOCATE(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE free
+
+#else /* not REGEX_MALLOC  */
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+#define REGEX_ALLOCATE alloca
+
+/* Assumes a `char *destination' variable.  */
+#define REGEX_REALLOCATE(source, osize, nsize)				\
+  (destination = (char *) alloca (nsize),				\
+   bcopy (source, destination, osize),					\
+   destination)
+
+/* No need to do anything to free, after alloca.  */
+#define REGEX_FREE(arg) ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+
+#endif /* not REGEX_MALLOC */
+
+/* Define how to allocate the failure stack.  */
+
+#ifdef REL_ALLOC
+#define REGEX_ALLOCATE_STACK(size)				\
+  r_alloc (&failure_stack_ptr, (size))
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)		\
+  r_re_alloc (&failure_stack_ptr, (nsize))
+#define REGEX_FREE_STACK(ptr)					\
+  r_alloc_free (&failure_stack_ptr)
+
+#else /* not REL_ALLOC */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE_STACK malloc
+#define REGEX_REALLOCATE_STACK(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE_STACK free
+
+#else /* not REGEX_MALLOC */
+
+#define REGEX_ALLOCATE_STACK alloca
+
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)			\
+   REGEX_REALLOCATE (source, osize, nsize)
+/* No need to explicitly free anything.  */
+#define REGEX_FREE_STACK(arg)
+
+#endif /* not REGEX_MALLOC */
+#endif /* not REL_ALLOC */
+
+
+/* True if `size1' is non-NULL and PTR is pointing anywhere inside
+   `string1' or just past its end.  This works if PTR is NULL, which is
+   a good thing.  */
+#define FIRST_STRING_P(ptr) 					\
+  (size1 && string1 <= (ptr) && (ptr) <= string1 + size1)
+
+/* (Re)Allocate N items of type T using malloc, or fail.  */
+#define TALLOC(n, t) ((t *) malloc ((n) * sizeof (t)))
+#define RETALLOC(addr, n, t) ((addr) = (t *) realloc (addr, (n) * sizeof (t)))
+#define RETALLOC_IF(addr, n, t) \
+  if (addr) RETALLOC((addr), (n), t); else (addr) = TALLOC ((n), t)
+#define REGEX_TALLOC(n, t) ((t *) REGEX_ALLOCATE ((n) * sizeof (t)))
+
+#define BYTEWIDTH 8 /* In bits.  */
+
+#define STREQ(s1, s2) ((strcmp (s1, s2) == 0))
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+static int re_match_2_internal ();
+
+
+/* These are the command codes that appear in compiled regular
+   expressions.  Some opcodes are followed by argument bytes.  A
+   command code can specify any interpretation whatsoever for its
+   arguments.  Zero bytes may appear in the compiled regular expression.  */
+
+typedef enum
+{
+  no_op = 0,
+
+  /* Succeed right away--no more backtracking.  */
+  succeed,
+
+        /* Followed by one byte giving n, then by n literal bytes.  */
+  exactn,
+
+        /* Matches any (more or less) character.  */
+  anychar,
+
+        /* Matches any one char belonging to specified set.  First
+           following byte is number of bitmap bytes.  Then come bytes
+           for a bitmap saying which chars are in.  Bits in each byte
+           are ordered low-bit-first.  A character is in the set if its
+           bit is 1.  A character too large to have a bit in the map is
+           automatically not in the set.  */
+  charset,
+
+        /* Same parameters as charset, but match any character that is
+           not one of those specified.  */
+  charset_not,
+
+        /* Start remembering the text that is matched, for storing in a
+           register.  Followed by one byte with the register number, in
+           the range 0 to one less than the pattern buffer's re_nsub
+           field.  Then followed by one byte with the number of groups
+           inner to this one.  (This last has to be part of the
+           start_memory only because we need it in the on_failure_jump
+           of re_match_2.)  */
+  start_memory,
+
+        /* Stop remembering the text that is matched and store it in a
+           memory register.  Followed by one byte with the register
+           number, in the range 0 to one less than `re_nsub' in the
+           pattern buffer, and one byte with the number of inner groups,
+           just like `start_memory'.  (We need the number of inner
+           groups here because we don't have any easy way of finding the
+           corresponding start_memory when we're at a stop_memory.)  */
+  stop_memory,
+
+        /* Match a duplicate of something remembered. Followed by one
+           byte containing the register number.  */
+  duplicate,
+
+        /* Fail unless at beginning of line.  */
+  begline,
+
+        /* Fail unless at end of line.  */
+  endline,
+
+        /* Succeeds if at beginning of buffer (if emacs) or at beginning
+           of string to be matched (if not).  */
+  begbuf,
+
+        /* Analogously, for end of buffer/string.  */
+  endbuf,
+ 
+        /* Followed by two byte relative address to which to jump.  */
+  jump, 
+
+	/* Same as jump, but marks the end of an alternative.  */
+  jump_past_alt,
+
+        /* Followed by two-byte relative address of place to resume at
+           in case of failure.  */
+  on_failure_jump,
+	
+        /* Like on_failure_jump, but pushes a placeholder instead of the
+           current string position when executed.  */
+  on_failure_keep_string_jump,
+  
+        /* Throw away latest failure point and then jump to following
+           two-byte relative address.  */
+  pop_failure_jump,
+
+        /* Change to pop_failure_jump if know won't have to backtrack to
+           match; otherwise change to jump.  This is used to jump
+           back to the beginning of a repeat.  If what follows this jump
+           clearly won't match what the repeat does, such that we can be
+           sure that there is no use backtracking out of repetitions
+           already matched, then we change it to a pop_failure_jump.
+           Followed by two-byte address.  */
+  maybe_pop_jump,
+
+        /* Jump to following two-byte address, and push a dummy failure
+           point. This failure point will be thrown away if an attempt
+           is made to use it for a failure.  A `+' construct makes this
+           before the first repeat.  Also used as an intermediary kind
+           of jump when compiling an alternative.  */
+  dummy_failure_jump,
+
+	/* Push a dummy failure point and continue.  Used at the end of
+	   alternatives.  */
+  push_dummy_failure,
+
+        /* Followed by two-byte relative address and two-byte number n.
+           After matching N times, jump to the address upon failure.  */
+  succeed_n,
+
+        /* Followed by two-byte relative address, and two-byte number n.
+           Jump to the address N times, then fail.  */
+  jump_n,
+
+        /* Set the following two-byte relative address to the
+           subsequent two-byte number.  The address *includes* the two
+           bytes of number.  */
+  set_number_at,
+
+  wordchar,	/* Matches any word-constituent character.  */
+  notwordchar,	/* Matches any char that is not a word-constituent.  */
+
+  wordbeg,	/* Succeeds if at word beginning.  */
+  wordend,	/* Succeeds if at word end.  */
+
+  wordbound,	/* Succeeds if at a word boundary.  */
+  notwordbound	/* Succeeds if not at a word boundary.  */
+
+#ifdef emacs
+  ,before_dot,	/* Succeeds if before point.  */
+  at_dot,	/* Succeeds if at point.  */
+  after_dot,	/* Succeeds if after point.  */
+
+	/* Matches any character whose syntax is specified.  Followed by
+           a byte which contains a syntax code, e.g., Sword.  */
+  syntaxspec,
+
+	/* Matches any character whose syntax is not that specified.  */
+  notsyntaxspec
+#endif /* emacs */
+} re_opcode_t;
+
+
+/* Common operations on the compiled pattern.  */
+
+/* Store NUMBER in two contiguous bytes starting at DESTINATION.  */
+
+#define STORE_NUMBER(destination, number)				\
+  do {									\
+    (destination)[0] = (number) & 0377;					\
+    (destination)[1] = (number) >> 8;					\
+  } while (0)
+
+/* Same as STORE_NUMBER, except increment DESTINATION to
+   the byte after where the number is stored.  Therefore, DESTINATION
+   must be an lvalue.  */
+
+#define STORE_NUMBER_AND_INCR(destination, number)			\
+  do {									\
+    STORE_NUMBER (destination, number);					\
+    (destination) += 2;							\
+  } while (0)
+
+/* Put into DESTINATION a number stored in two contiguous bytes starting
+   at SOURCE.  */
+
+#define EXTRACT_NUMBER(destination, source)				\
+  do {									\
+    (destination) = *(source) & 0377;					\
+    (destination) += SIGN_EXTEND_CHAR (*((source) + 1)) << 8;		\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number (dest, source)
+    int *dest;
+    unsigned char *source;
+{
+  int temp = SIGN_EXTEND_CHAR (*(source + 1)); 
+  *dest = *source & 0377;
+  *dest += temp << 8;
+}
+
+#ifndef EXTRACT_MACROS /* To debug the macros.  */
+#undef EXTRACT_NUMBER
+#define EXTRACT_NUMBER(dest, src) extract_number (&dest, src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+/* Same as EXTRACT_NUMBER, except increment SOURCE to after the number.
+   SOURCE must be an lvalue.  */
+
+#define EXTRACT_NUMBER_AND_INCR(destination, source)			\
+  do {									\
+    EXTRACT_NUMBER (destination, source);				\
+    (source) += 2; 							\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number_and_incr (destination, source)
+    int *destination;
+    unsigned char **source;
+{ 
+  extract_number (destination, *source);
+  *source += 2;
+}
+
+#ifndef EXTRACT_MACROS
+#undef EXTRACT_NUMBER_AND_INCR
+#define EXTRACT_NUMBER_AND_INCR(dest, src) \
+  extract_number_and_incr (&dest, &src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+
+/* If DEBUG is defined, Regex prints many voluminous messages about what
+   it is doing (if the variable `debug' is nonzero).  If linked with the
+   main program in `iregex.c', you can enter patterns and strings
+   interactively.  And if linked with the main program in `main.c' and
+   the other test files, you can run the already-written tests.  */
+
+#ifdef DEBUG
+
+/* We use standard I/O for debugging.  */
+#include <stdio.h>
+
+/* It is useful to test things that ``must'' be true when debugging.  */
+#include <assert.h>
+
+static int debug = 0;
+
+#define DEBUG_STATEMENT(e) e
+#define DEBUG_PRINT1(x) if (debug) printf (x)
+#define DEBUG_PRINT2(x1, x2) if (debug) printf (x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3) if (debug) printf (x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4) if (debug) printf (x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e) 				\
+  if (debug) print_partial_compiled_pattern (s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)			\
+  if (debug) print_double_string (w, s1, sz1, s2, sz2)
+
+
+/* Print the fastmap in human-readable form.  */
+
+void
+print_fastmap (fastmap)
+    char *fastmap;
+{
+  unsigned was_a_range = 0;
+  unsigned i = 0;  
+  
+  while (i < (1 << BYTEWIDTH))
+    {
+      if (fastmap[i++])
+	{
+	  was_a_range = 0;
+          putchar (i - 1);
+          while (i < (1 << BYTEWIDTH)  &&  fastmap[i])
+            {
+              was_a_range = 1;
+              i++;
+            }
+	  if (was_a_range)
+            {
+              printf ("-");
+              putchar (i - 1);
+            }
+        }
+    }
+  putchar ('\n'); 
+}
+
+
+/* Print a compiled pattern string in human-readable form, starting at
+   the START pointer into it and ending just before the pointer END.  */
+
+void
+print_partial_compiled_pattern (start, end)
+    unsigned char *start;
+    unsigned char *end;
+{
+  int mcnt, mcnt2;
+  unsigned char *p = start;
+  unsigned char *pend = end;
+
+  if (start == NULL)
+    {
+      printf ("(null)\n");
+      return;
+    }
+    
+  /* Loop over pattern commands.  */
+  while (p < pend)
+    {
+      printf ("%d:\t", p - start);
+
+      switch ((re_opcode_t) *p++)
+	{
+        case no_op:
+          printf ("/no_op");
+          break;
+
+	case exactn:
+	  mcnt = *p++;
+          printf ("/exactn/%d", mcnt);
+          do
+	    {
+              putchar ('/');
+	      putchar (*p++);
+            }
+          while (--mcnt);
+          break;
+
+	case start_memory:
+          mcnt = *p++;
+          printf ("/start_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case stop_memory:
+          mcnt = *p++;
+	  printf ("/stop_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case duplicate:
+	  printf ("/duplicate/%d", *p++);
+	  break;
+
+	case anychar:
+	  printf ("/anychar");
+	  break;
+
+	case charset:
+        case charset_not:
+          {
+            register int c, last = -100;
+	    register int in_range = 0;
+
+	    printf ("/charset [%s",
+	            (re_opcode_t) *(p - 1) == charset_not ? "^" : "");
+            
+            assert (p + *p < pend);
+
+            for (c = 0; c < 256; c++)
+	      if (c / 8 < *p
+		  && (p[1 + (c/8)] & (1 << (c % 8))))
+		{
+		  /* Are we starting a range?  */
+		  if (last + 1 == c && ! in_range)
+		    {
+		      putchar ('-');
+		      in_range = 1;
+		    }
+		  /* Have we broken a range?  */
+		  else if (last + 1 != c && in_range)
+              {
+		      putchar (last);
+		      in_range = 0;
+		    }
+                
+		  if (! in_range)
+		    putchar (c);
+
+		  last = c;
+              }
+
+	    if (in_range)
+	      putchar (last);
+
+	    putchar (']');
+
+	    p += 1 + *p;
+	  }
+	  break;
+
+	case begline:
+	  printf ("/begline");
+          break;
+
+	case endline:
+          printf ("/endline");
+          break;
+
+	case on_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case on_failure_keep_string_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_keep_string_jump to %d", p + mcnt - start);
+          break;
+
+	case dummy_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/dummy_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case push_dummy_failure:
+          printf ("/push_dummy_failure");
+          break;
+          
+        case maybe_pop_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/maybe_pop_jump to %d", p + mcnt - start);
+	  break;
+
+        case pop_failure_jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/pop_failure_jump to %d", p + mcnt - start);
+	  break;          
+          
+        case jump_past_alt:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump_past_alt to %d", p + mcnt - start);
+	  break;          
+          
+        case jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump to %d", p + mcnt - start);
+	  break;
+
+        case succeed_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/succeed_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case jump_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/jump_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case set_number_at: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/set_number_at location %d to %d", p + mcnt - start, mcnt2);
+          break;
+        
+        case wordbound:
+	  printf ("/wordbound");
+	  break;
+
+	case notwordbound:
+	  printf ("/notwordbound");
+          break;
+
+	case wordbeg:
+	  printf ("/wordbeg");
+	  break;
+          
+	case wordend:
+	  printf ("/wordend");
+          
+#ifdef emacs
+	case before_dot:
+	  printf ("/before_dot");
+          break;
+
+	case at_dot:
+	  printf ("/at_dot");
+          break;
+
+	case after_dot:
+	  printf ("/after_dot");
+          break;
+
+	case syntaxspec:
+          printf ("/syntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+          break;
+	  
+	case notsyntaxspec:
+          printf ("/notsyntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+	  break;
+#endif /* emacs */
+
+	case wordchar:
+	  printf ("/wordchar");
+          break;
+	  
+	case notwordchar:
+	  printf ("/notwordchar");
+          break;
+
+	case begbuf:
+	  printf ("/begbuf");
+          break;
+
+	case endbuf:
+	  printf ("/endbuf");
+          break;
+
+        default:
+          printf ("?%d", *(p-1));
+	}
+
+      putchar ('\n');
+    }
+
+  printf ("%d:\tend of pattern.\n", p - start);
+}
+
+
+void
+print_compiled_pattern (bufp)
+    struct re_pattern_buffer *bufp;
+{
+  unsigned char *buffer = bufp->buffer;
+
+  print_partial_compiled_pattern (buffer, buffer + bufp->used);
+  printf ("%d bytes used/%d bytes allocated.\n", bufp->used, bufp->allocated);
+
+  if (bufp->fastmap_accurate && bufp->fastmap)
+    {
+      printf ("fastmap: ");
+      print_fastmap (bufp->fastmap);
+    }
+
+  printf ("re_nsub: %d\t", bufp->re_nsub);
+  printf ("regs_alloc: %d\t", bufp->regs_allocated);
+  printf ("can_be_null: %d\t", bufp->can_be_null);
+  printf ("newline_anchor: %d\n", bufp->newline_anchor);
+  printf ("no_sub: %d\t", bufp->no_sub);
+  printf ("not_bol: %d\t", bufp->not_bol);
+  printf ("not_eol: %d\t", bufp->not_eol);
+  printf ("syntax: %d\n", bufp->syntax);
+  /* Perhaps we should print the translate table?  */
+}
+
+
+void
+print_double_string (where, string1, size1, string2, size2)
+    const char *where;
+    const char *string1;
+    const char *string2;
+    int size1;
+    int size2;
+{
+  unsigned this_char;
+  
+  if (where == NULL)
+    printf ("(null)");
+  else
+    {
+      if (FIRST_STRING_P (where))
+        {
+          for (this_char = where - string1; this_char < size1; this_char++)
+            putchar (string1[this_char]);
+
+          where = string2;    
+        }
+
+      for (this_char = where - string2; this_char < size2; this_char++)
+        putchar (string2[this_char]);
+    }
+}
+
+#else /* not DEBUG */
+
+#undef assert
+#define assert(e)
+
+#define DEBUG_STATEMENT(e)
+#define DEBUG_PRINT1(x)
+#define DEBUG_PRINT2(x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)
+
+#endif /* not DEBUG */
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+/* This has no initializer because initialized variables in Emacs
+   become read-only after dumping.  */
+reg_syntax_t re_syntax_options;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+{
+  reg_syntax_t ret = re_syntax_options;
+  
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.
+   POSIX doesn't require that we do anything for REG_NOERROR,
+   but why not be nice?  */
+
+static const char *re_error_msgid[] =
+  { "Success",					/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+  };
+
+
+/* Avoiding alloca during matching, to placate r_alloc.  */
+
+/* Define MATCH_MAY_ALLOCATE unless we need to make sure that the
+   searching and matching functions should not call alloca.  On some
+   systems, alloca is implemented in terms of malloc, and if we're
+   using the relocating allocator routines, then malloc could cause a
+   relocation, which might (if the strings being searched are in the
+   ralloc heap) shift the data out from underneath the regexp
+   routines.
+
+   Here's another reason to avoid allocation: Emacs 
+   processes input from X in a signal handler; processing X input may
+   call malloc; if input arrives while a matching routine is calling
+   malloc, then we're scrod.  But Emacs can't just block input while
+   calling matching routines; then we don't notice interrupts when
+   they come in.  So, Emacs blocks input around all regexp calls
+   except the matching calls, which it leaves unprotected, in the
+   faith that they will not malloc.  */
+
+/* Normally, this is fine.  */
+#define MATCH_MAY_ALLOCATE
+
+/* When using GNU C, we are not REALLY using the C alloca, no matter
+   what sysfuncs.h may say.  So don't take precautions for it.  */
+#ifdef __GNUC__
+#undef C_ALLOCA
+#endif
+
+/* The match routines may not allocate if (1) they would do it with malloc
+   and (2) it's not safe for them to use malloc.
+   Note that if REL_ALLOC is defined, matching would not use malloc for the
+   failure stack, but we would still use it for the register vectors;
+   so REL_ALLOC should not affect this.  */
+#if (defined (C_ALLOCA) || defined (REGEX_MALLOC)) && defined (emacs)
+#undef MATCH_MAY_ALLOCATE
+#endif
+
+
+
+/* Failure stack declarations and macros; both re_compile_fastmap and
+   re_match_2 use a failure stack.  These have to be macros because of
+   REGEX_ALLOCATE_STACK.  */
+   
+
+/* Number of failure points for which to initially allocate space
+   when matching.  If this number is exceeded, we allocate more
+   space, so it is not a hard limit.  */
+#ifndef INIT_FAILURE_ALLOC
+#define INIT_FAILURE_ALLOC 5
+#endif
+
+/* Roughly the maximum number of failure points on the stack.  Would be
+   exactly that if always used MAX_FAILURE_SPACE each time we failed.
+   This is a variable only so users of regex can assign to it; we never
+   change it ourselves.  */
+#if defined (MATCH_MAY_ALLOCATE)
+int re_max_failures = 200000;
+#else
+int re_max_failures = 2000;
+#endif
+
+union fail_stack_elt
+{
+  unsigned char *pointer;
+  int integer;
+};
+
+typedef union fail_stack_elt fail_stack_elt_t;
+
+typedef struct
+{
+  fail_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} fail_stack_type;
+
+#define FAIL_STACK_EMPTY()     (fail_stack.avail == 0)
+#define FAIL_STACK_PTR_EMPTY() (fail_stack_ptr->avail == 0)
+#define FAIL_STACK_FULL()      (fail_stack.avail == fail_stack.size)
+
+
+/* Define macros to initialize and free the failure stack.
+   Do `return -2' if the alloc fails.  */
+
+#ifdef MATCH_MAY_ALLOCATE
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.stack = (fail_stack_elt_t *)				\
+      REGEX_ALLOCATE_STACK (INIT_FAILURE_ALLOC * sizeof (fail_stack_elt_t));	\
+									\
+    if (fail_stack.stack == NULL)					\
+      return -2;							\
+									\
+    fail_stack.size = INIT_FAILURE_ALLOC;				\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()  REGEX_FREE_STACK (fail_stack.stack)
+#else
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()
+#endif
+
+
+/* Double the size of FAIL_STACK, up to approximately `re_max_failures' items.
+
+   Return 1 if succeeds, and 0 if either ran out of memory
+   allocating space for it or it was already too large.  
+   
+   REGEX_REALLOCATE_STACK requires `destination' be declared.   */
+
+#define DOUBLE_FAIL_STACK(fail_stack)					\
+  ((fail_stack).size > re_max_failures * MAX_FAILURE_ITEMS		\
+   ? 0									\
+   : ((fail_stack).stack = (fail_stack_elt_t *)				\
+        REGEX_REALLOCATE_STACK ((fail_stack).stack, 			\
+          (fail_stack).size * sizeof (fail_stack_elt_t),		\
+          ((fail_stack).size << 1) * sizeof (fail_stack_elt_t)),	\
+									\
+      (fail_stack).stack == NULL					\
+      ? 0								\
+      : ((fail_stack).size <<= 1, 					\
+         1)))
+
+
+/* Push pointer POINTER on FAIL_STACK. 
+   Return 1 if was able to do so and 0 if ran out of memory allocating
+   space to do so.  */
+#define PUSH_PATTERN_OP(POINTER, FAIL_STACK)				\
+  ((FAIL_STACK_FULL ()							\
+    && !DOUBLE_FAIL_STACK (FAIL_STACK))					\
+   ? 0									\
+   : ((FAIL_STACK).stack[(FAIL_STACK).avail++].pointer = POINTER,	\
+      1))
+
+/* Push a pointer value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_POINTER(item)					\
+  fail_stack.stack[fail_stack.avail++].pointer = (unsigned char *) (item)
+
+/* This pushes an integer-valued item onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_INT(item)					\
+  fail_stack.stack[fail_stack.avail++].integer = (item)
+
+/* Push a fail_stack_elt_t value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_ELT(item)					\
+  fail_stack.stack[fail_stack.avail++] =  (item)
+
+/* These three POP... operations complement the three PUSH... operations.
+   All assume that `fail_stack' is nonempty.  */
+#define POP_FAILURE_POINTER() fail_stack.stack[--fail_stack.avail].pointer
+#define POP_FAILURE_INT() fail_stack.stack[--fail_stack.avail].integer
+#define POP_FAILURE_ELT() fail_stack.stack[--fail_stack.avail]
+
+/* Used to omit pushing failure point id's when we're not debugging.  */
+#ifdef DEBUG
+#define DEBUG_PUSH PUSH_FAILURE_INT
+#define DEBUG_POP(item_addr) *(item_addr) = POP_FAILURE_INT ()
+#else
+#define DEBUG_PUSH(item)
+#define DEBUG_POP(item_addr)
+#endif
+
+
+/* Push the information about the state we will need
+   if we ever fail back to it.  
+   
+   Requires variables fail_stack, regstart, regend, reg_info, and
+   num_regs be declared.  DOUBLE_FAIL_STACK requires `destination' be
+   declared.
+   
+   Does `return FAILURE_CODE' if runs out of memory.  */
+
+#define PUSH_FAILURE_POINT(pattern_place, string_place, failure_code)	\
+  do {									\
+    char *destination;							\
+    /* Must be int, so when we don't save any registers, the arithmetic	\
+       of 0 + -1 isn't done as unsigned.  */				\
+    int this_reg;							\
+    									\
+    DEBUG_STATEMENT (failure_id++);					\
+    DEBUG_STATEMENT (nfailure_points_pushed++);				\
+    DEBUG_PRINT2 ("\nPUSH_FAILURE_POINT #%u:\n", failure_id);		\
+    DEBUG_PRINT2 ("  Before push, next avail: %d\n", (fail_stack).avail);\
+    DEBUG_PRINT2 ("                     size: %d\n", (fail_stack).size);\
+									\
+    DEBUG_PRINT2 ("  slots needed: %d\n", NUM_FAILURE_ITEMS);		\
+    DEBUG_PRINT2 ("     available: %d\n", REMAINING_AVAIL_SLOTS);	\
+									\
+    /* Ensure we have enough space allocated for what we will push.  */	\
+    while (REMAINING_AVAIL_SLOTS < NUM_FAILURE_ITEMS)			\
+      {									\
+        if (!DOUBLE_FAIL_STACK (fail_stack))				\
+          return failure_code;						\
+									\
+        DEBUG_PRINT2 ("\n  Doubled stack; size now: %d\n",		\
+		       (fail_stack).size);				\
+        DEBUG_PRINT2 ("  slots available: %d\n", REMAINING_AVAIL_SLOTS);\
+      }									\
+									\
+    /* Push the info, starting with the registers.  */			\
+    DEBUG_PRINT1 ("\n");						\
+									\
+    for (this_reg = lowest_active_reg; this_reg <= highest_active_reg;	\
+         this_reg++)							\
+      {									\
+	DEBUG_PRINT2 ("  Pushing reg: %d\n", this_reg);			\
+        DEBUG_STATEMENT (num_regs_pushed++);				\
+									\
+	DEBUG_PRINT2 ("    start: 0x%x\n", regstart[this_reg]);		\
+        PUSH_FAILURE_POINTER (regstart[this_reg]);			\
+                                                                        \
+	DEBUG_PRINT2 ("    end: 0x%x\n", regend[this_reg]);		\
+        PUSH_FAILURE_POINTER (regend[this_reg]);			\
+									\
+	DEBUG_PRINT2 ("    info: 0x%x\n      ", reg_info[this_reg]);	\
+        DEBUG_PRINT2 (" match_null=%d",					\
+                      REG_MATCH_NULL_STRING_P (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" active=%d", IS_ACTIVE (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" matched_something=%d",				\
+                      MATCHED_SOMETHING (reg_info[this_reg]));		\
+        DEBUG_PRINT2 (" ever_matched=%d",				\
+                      EVER_MATCHED_SOMETHING (reg_info[this_reg]));	\
+	DEBUG_PRINT1 ("\n");						\
+        PUSH_FAILURE_ELT (reg_info[this_reg].word);			\
+      }									\
+									\
+    DEBUG_PRINT2 ("  Pushing  low active reg: %d\n", lowest_active_reg);\
+    PUSH_FAILURE_INT (lowest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing high active reg: %d\n", highest_active_reg);\
+    PUSH_FAILURE_INT (highest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing pattern 0x%x: ", pattern_place);		\
+    DEBUG_PRINT_COMPILED_PATTERN (bufp, pattern_place, pend);		\
+    PUSH_FAILURE_POINTER (pattern_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing string 0x%x: `", string_place);		\
+    DEBUG_PRINT_DOUBLE_STRING (string_place, string1, size1, string2,   \
+				 size2);				\
+    DEBUG_PRINT1 ("'\n");						\
+    PUSH_FAILURE_POINTER (string_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing failure id: %u\n", failure_id);		\
+    DEBUG_PUSH (failure_id);						\
+  } while (0)
+
+/* This is the number of items that are pushed and popped on the stack
+   for each register.  */
+#define NUM_REG_ITEMS  3
+
+/* Individual items aside from the registers.  */
+#ifdef DEBUG
+#define NUM_NONREG_ITEMS 5 /* Includes failure point id.  */
+#else
+#define NUM_NONREG_ITEMS 4
+#endif
+
+/* We push at most this many items on the stack.  */
+#define MAX_FAILURE_ITEMS ((num_regs - 1) * NUM_REG_ITEMS + NUM_NONREG_ITEMS)
+
+/* We actually push this many items.  */
+#define NUM_FAILURE_ITEMS						\
+  ((highest_active_reg - lowest_active_reg + 1) * NUM_REG_ITEMS 	\
+    + NUM_NONREG_ITEMS)
+
+/* How many items can still be added to the stack without overflowing it.  */
+#define REMAINING_AVAIL_SLOTS ((fail_stack).size - (fail_stack).avail)
+
+
+/* Pops what PUSH_FAIL_STACK pushes.
+
+   We restore into the parameters, all of which should be lvalues:
+     STR -- the saved data position.
+     PAT -- the saved pattern position.
+     LOW_REG, HIGH_REG -- the highest and lowest active registers.
+     REGSTART, REGEND -- arrays of string positions.
+     REG_INFO -- array of information about each subexpression.
+   
+   Also assumes the variables `fail_stack' and (if debugging), `bufp',
+   `pend', `string1', `size1', `string2', and `size2'.  */
+
+#define POP_FAILURE_POINT(str, pat, low_reg, high_reg, regstart, regend, reg_info)\
+{									\
+  DEBUG_STATEMENT (fail_stack_elt_t failure_id;)			\
+  int this_reg;								\
+  const unsigned char *string_temp;					\
+									\
+  assert (!FAIL_STACK_EMPTY ());					\
+									\
+  /* Remove failure points and point to how many regs pushed.  */	\
+  DEBUG_PRINT1 ("POP_FAILURE_POINT:\n");				\
+  DEBUG_PRINT2 ("  Before pop, next avail: %d\n", fail_stack.avail);	\
+  DEBUG_PRINT2 ("                    size: %d\n", fail_stack.size);	\
+									\
+  assert (fail_stack.avail >= NUM_NONREG_ITEMS);			\
+									\
+  DEBUG_POP (&failure_id);						\
+  DEBUG_PRINT2 ("  Popping failure id: %u\n", failure_id);		\
+									\
+  /* If the saved string location is NULL, it came from an		\
+     on_failure_keep_string_jump opcode, and we want to throw away the	\
+     saved NULL, thus retaining our current position in the string.  */	\
+  string_temp = POP_FAILURE_POINTER ();					\
+  if (string_temp != NULL)						\
+    str = (const char *) string_temp;					\
+									\
+  DEBUG_PRINT2 ("  Popping string 0x%x: `", str);			\
+  DEBUG_PRINT_DOUBLE_STRING (str, string1, size1, string2, size2);	\
+  DEBUG_PRINT1 ("'\n");							\
+									\
+  pat = (unsigned char *) POP_FAILURE_POINTER ();			\
+  DEBUG_PRINT2 ("  Popping pattern 0x%x: ", pat);			\
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, pat, pend);			\
+									\
+  /* Restore register info.  */						\
+  high_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping high active reg: %d\n", high_reg);		\
+									\
+  low_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping  low active reg: %d\n", low_reg);		\
+									\
+  for (this_reg = high_reg; this_reg >= low_reg; this_reg--)		\
+    {									\
+      DEBUG_PRINT2 ("    Popping reg: %d\n", this_reg);			\
+									\
+      reg_info[this_reg].word = POP_FAILURE_ELT ();			\
+      DEBUG_PRINT2 ("      info: 0x%x\n", reg_info[this_reg]);		\
+									\
+      regend[this_reg] = (const char *) POP_FAILURE_POINTER ();		\
+      DEBUG_PRINT2 ("      end: 0x%x\n", regend[this_reg]);		\
+									\
+      regstart[this_reg] = (const char *) POP_FAILURE_POINTER ();	\
+      DEBUG_PRINT2 ("      start: 0x%x\n", regstart[this_reg]);		\
+    }									\
+									\
+  set_regs_matched_done = 0;						\
+  DEBUG_STATEMENT (nfailure_points_popped++);				\
+} /* POP_FAILURE_POINT */
+
+
+
+
+/* Structure for per-register (a.k.a. per-group) information.
+   Other register information, such as the
+   starting and ending positions (which are addresses), and the list of
+   inner groups (which is a bits list) are maintained in separate
+   variables.  
+   
+   We are making a (strictly speaking) nonportable assumption here: that
+   the compiler will pack our bit fields into something that fits into
+   the type of `word', i.e., is something that fits into one item on the
+   failure stack.  */
+
+typedef union
+{
+  fail_stack_elt_t word;
+  struct
+  {
+      /* This field is one if this group can match the empty string,
+         zero if not.  If not yet determined,  `MATCH_NULL_UNSET_VALUE'.  */
+#define MATCH_NULL_UNSET_VALUE 3
+    unsigned match_null_string_p : 2;
+    unsigned is_active : 1;
+    unsigned matched_something : 1;
+    unsigned ever_matched_something : 1;
+  } bits;
+} register_info_type;
+
+#define REG_MATCH_NULL_STRING_P(R)  ((R).bits.match_null_string_p)
+#define IS_ACTIVE(R)  ((R).bits.is_active)
+#define MATCHED_SOMETHING(R)  ((R).bits.matched_something)
+#define EVER_MATCHED_SOMETHING(R)  ((R).bits.ever_matched_something)
+
+
+/* Call this when have matched a real character; it sets `matched' flags
+   for the subexpressions which we are currently inside.  Also records
+   that those subexprs have matched.  */
+#define SET_REGS_MATCHED()						\
+  do									\
+    {									\
+      if (!set_regs_matched_done)					\
+	{								\
+	  unsigned r;							\
+	  set_regs_matched_done = 1;					\
+	  for (r = lowest_active_reg; r <= highest_active_reg; r++)	\
+	    {								\
+	      MATCHED_SOMETHING (reg_info[r])				\
+		= EVER_MATCHED_SOMETHING (reg_info[r])			\
+		= 1;							\
+	    }								\
+	}								\
+    }									\
+  while (0)
+
+/* Registers are set to a sentinel when they haven't yet matched.  */
+static char reg_unset_dummy;
+#define REG_UNSET_VALUE (&reg_unset_dummy)
+#define REG_UNSET(e) ((e) == REG_UNSET_VALUE)
+
+
+/* Subroutine declarations and macros for regex_compile.  */
+
+static void store_op1 (), store_op2 ();
+static void insert_op1 (), insert_op2 ();
+static boolean at_begline_loc_p (), at_endline_loc_p ();
+static boolean group_in_compile_stack ();
+static reg_errcode_t compile_range ();
+
+/* Fetch the next character in the uncompiled pattern---translating it 
+   if necessary.  Also cast from a signed character in the constant
+   string passed to us by the user to an unsigned char that we can use
+   as an array index (in, e.g., `translate').  */
+#define PATFETCH(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    if (translate) c = translate[c]; 					\
+  } while (0)
+
+/* Fetch the next character in the uncompiled pattern, with no
+   translation.  */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+/* If `translate' is non-null, return translate[D], else just D.  We
+   cast the subscript to translate because some data is declared as
+   `char *', to avoid warnings when a string constant is passed.  But
+   when we use a character as a subscript we must make it unsigned.  */
+#define TRANSLATE(d) (translate ? translate[(unsigned char) (d)] : (d))
+
+
+/* Macros for outputting the compiled pattern into `buffer'.  */
+
+/* If the buffer isn't allocated when it comes in, use this.  */
+#define INIT_BUF_SIZE  32
+
+/* Make sure we have at least N more bytes of space in buffer.  */
+#define GET_BUFFER_SPACE(n)						\
+    while (b - bufp->buffer + (n) > bufp->allocated)			\
+      EXTEND_BUFFER ()
+
+/* Make sure we have one more byte of buffer space and then add C to it.  */
+#define BUF_PUSH(c)							\
+  do {									\
+    GET_BUFFER_SPACE (1);						\
+    *b++ = (unsigned char) (c);						\
+  } while (0)
+
+
+/* Ensure we have two more bytes of buffer space and then append C1 and C2.  */
+#define BUF_PUSH_2(c1, c2)						\
+  do {									\
+    GET_BUFFER_SPACE (2);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+  } while (0)
+
+
+/* As with BUF_PUSH_2, except for three bytes.  */
+#define BUF_PUSH_3(c1, c2, c3)						\
+  do {									\
+    GET_BUFFER_SPACE (3);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+    *b++ = (unsigned char) (c3);					\
+  } while (0)
+
+
+/* Store a jump with opcode OP at LOC to location TO.  We store a
+   relative address offset by the three bytes the jump itself occupies.  */
+#define STORE_JUMP(op, loc, to) \
+  store_op1 (op, loc, (to) - (loc) - 3)
+
+/* Likewise, for a two-argument jump.  */
+#define STORE_JUMP2(op, loc, to, arg) \
+  store_op2 (op, loc, (to) - (loc) - 3, arg)
+
+/* Like `STORE_JUMP', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP(op, loc, to) \
+  insert_op1 (op, loc, (to) - (loc) - 3, b)
+
+/* Like `STORE_JUMP2', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP2(op, loc, to, arg) \
+  insert_op2 (op, loc, (to) - (loc) - 3, arg, b)
+
+
+/* This is not an arbitrary limit: the arguments which represent offsets
+   into the pattern are two bytes long.  So if 2^16 bytes turns out to
+   be too small, many things would have to change.  */
+#define MAX_BUF_SIZE (1L << 16)
+
+
+/* Extend the buffer by twice its current size via realloc and
+   reset the pointers that pointed into the old block to point to the
+   correct places in the new one.  If extending the buffer results in it
+   being larger than MAX_BUF_SIZE, then flag memory exhausted.  */
+#define EXTEND_BUFFER()							\
+  do { 									\
+    unsigned char *old_buffer = bufp->buffer;				\
+    if (bufp->allocated == MAX_BUF_SIZE) 				\
+      return REG_ESIZE;							\
+    bufp->allocated <<= 1;						\
+    if (bufp->allocated > MAX_BUF_SIZE)					\
+      bufp->allocated = MAX_BUF_SIZE; 					\
+    bufp->buffer = (unsigned char *) realloc (bufp->buffer, bufp->allocated);\
+    if (bufp->buffer == NULL)						\
+      return REG_ESPACE;						\
+    /* If the buffer moved, move all the pointers into it.  */		\
+    if (old_buffer != bufp->buffer)					\
+      {									\
+        b = (b - old_buffer) + bufp->buffer;				\
+        begalt = (begalt - old_buffer) + bufp->buffer;			\
+        if (fixup_alt_jump)						\
+          fixup_alt_jump = (fixup_alt_jump - old_buffer) + bufp->buffer;\
+        if (laststart)							\
+          laststart = (laststart - old_buffer) + bufp->buffer;		\
+        if (pending_exact)						\
+          pending_exact = (pending_exact - old_buffer) + bufp->buffer;	\
+      }									\
+  } while (0)
+
+
+/* Since we have one byte reserved for the register number argument to
+   {start,stop}_memory, the maximum number of groups we can report
+   things about is what fits in that byte.  */
+#define MAX_REGNUM 255
+
+/* But patterns can have more than `MAX_REGNUM' registers.  We just
+   ignore the excess.  */
+typedef unsigned regnum_t;
+
+
+/* Macros for the compile stack.  */
+
+/* Since offsets can go either forwards or backwards, this type needs to
+   be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.  */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  pattern_offset_t begalt_offset;
+  pattern_offset_t fixup_alt_jump;
+  pattern_offset_t inner_group_offset;
+  pattern_offset_t laststart_offset;  
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / BYTEWIDTH]               \
+   |= 1 << (((unsigned char) c) % BYTEWIDTH))
+
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (ISDIGIT (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (STREQ (string, "alpha") || STREQ (string, "upper")			\
+    || STREQ (string, "lower") || STREQ (string, "digit")		\
+    || STREQ (string, "alnum") || STREQ (string, "xdigit")		\
+    || STREQ (string, "space") || STREQ (string, "print")		\
+    || STREQ (string, "punct") || STREQ (string, "graph")		\
+    || STREQ (string, "cntrl") || STREQ (string, "blank"))
+
+
+#ifndef MATCH_MAY_ALLOCATE
+
+/* If we cannot allocate large objects within re_match_2_internal,
+   we make the fail stack and register vectors global.
+   The fail stack, we grow to the maximum size when a regexp
+   is compiled.
+   The register vectors, we adjust in size each time we
+   compile a regexp, according to the number of registers it needs.  */
+
+static fail_stack_type fail_stack;
+
+/* Size with which the following vectors are currently allocated.
+   That is so we can make them bigger as needed,
+   but never make them smaller.  */
+static int regs_allocated_size;
+
+static const char **     regstart, **     regend;
+static const char ** old_regstart, ** old_regend;
+static const char **best_regstart, **best_regend;
+static register_info_type *reg_info; 
+static const char **reg_dummy;
+static register_info_type *reg_info_dummy;
+
+/* Make the register vectors big enough for NUM_REGS registers,
+   but don't make them smaller.  */
+
+static
+regex_grow_registers (num_regs)
+     int num_regs;
+{
+  if (num_regs > regs_allocated_size)
+    {
+      RETALLOC_IF (regstart,	 num_regs, const char *);
+      RETALLOC_IF (regend,	 num_regs, const char *);
+      RETALLOC_IF (old_regstart, num_regs, const char *);
+      RETALLOC_IF (old_regend,	 num_regs, const char *);
+      RETALLOC_IF (best_regstart, num_regs, const char *);
+      RETALLOC_IF (best_regend,	 num_regs, const char *);
+      RETALLOC_IF (reg_info,	 num_regs, register_info_type);
+      RETALLOC_IF (reg_dummy,	 num_regs, const char *);
+      RETALLOC_IF (reg_info_dummy, num_regs, register_info_type);
+
+      regs_allocated_size = num_regs;
+    }
+}
+
+#endif /* not MATCH_MAY_ALLOCATE */
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is zero;
+     `re_nsub' is the number of subexpressions in PATTERN;
+     `not_bol' and `not_eol' are zero;
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+/* Return, freeing storage we allocated.  */
+#define FREE_STACK_RETURN(value)		\
+  return (free (compile_stack.stack), value)
+
+static reg_errcode_t
+regex_compile (pattern, size, syntax, bufp)
+     const char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer *bufp;
+{
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random temporary spot in PATTERN.  */
+  const char *p1;
+
+  /* Points to the end of the buffer, where we should append.  */
+  register unsigned char *b;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  const char *p = pattern;
+  const char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  char *translate = bufp->translate;
+
+  /* Address of the count-byte of the most recently inserted `exactn'
+     command.  This makes it possible to tell if a new exact-match
+     character can be added to that command or if the character requires
+     a new `exactn' command.  */
+  unsigned char *pending_exact = 0;
+
+  /* Address of start of the most recently finished expression.
+     This tells, e.g., postfix * where to find the start of its
+     operand.  Reset at the beginning of groups and alternatives.  */
+  unsigned char *laststart = 0;
+
+  /* Address of beginning of regexp, or inside of last group.  */
+  unsigned char *begalt;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+     which to go back if the interval is invalid.  */
+  const char *beg_interval;
+                
+  /* Address of the place where a forward jump should go to the end of
+     the containing expression.  Each alternative of an `or' -- except the
+     last -- ends with a forward jump of this sort.  */
+  unsigned char *fixup_alt_jump = 0;
+
+  /* Counts open-groups as they are encountered.  Remembered for the
+     matching close-group on the compile stack, so the same register
+     number is put in the stop_memory as the start_memory.  */
+  regnum_t regnum = 0;
+
+#ifdef DEBUG
+  DEBUG_PRINT1 ("\nCompiling pattern: ");
+  if (debug)
+    {
+      unsigned debug_count;
+      
+      for (debug_count = 0; debug_count < size; debug_count++)
+        putchar (pattern[debug_count]);
+      putchar ('\n');
+    }
+#endif /* DEBUG */
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack = TALLOC (INIT_COMPILE_STACK_SIZE, compile_stack_elt_t);
+  if (compile_stack.stack == NULL)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  bufp->syntax = syntax;
+  bufp->fastmap_accurate = 0;
+  bufp->not_bol = bufp->not_eol = 0;
+
+  /* Set `used' to zero, so that if we return an error, the pattern
+     printer (for debugging) will think there's no pattern.  We reset it
+     at the end.  */
+  bufp->used = 0;
+  
+  /* Always count groups, whether or not bufp->no_sub is set.  */
+  bufp->re_nsub = 0;				
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  if (bufp->allocated == 0)
+    {
+      if (bufp->buffer)
+	{ /* If zero allocated, but buffer is non-null, try to realloc
+             enough space.  This loses if buffer's address is bogus, but
+             that is the user's responsibility.  */
+          RETALLOC (bufp->buffer, INIT_BUF_SIZE, unsigned char);
+        }
+      else
+        { /* Caller did not allocate a buffer.  Do it for them.  */
+          bufp->buffer = TALLOC (INIT_BUF_SIZE, unsigned char);
+        }
+      if (!bufp->buffer) FREE_STACK_RETURN (REG_ESPACE);
+
+      bufp->allocated = INIT_BUF_SIZE;
+    }
+
+  begalt = b = bufp->buffer;
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+              BUF_PUSH (begline);
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+               BUF_PUSH (endline);
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (!laststart)
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                FREE_STACK_RETURN (REG_BADRPT);
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* Are we optimizing this jump?  */
+            boolean keep_string_p = false;
+            
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!laststart)  
+              break;
+
+            /* Now we know whether or not zero matches is allowed
+               and also whether or not two or more matches is allowed.  */
+            if (many_times_ok)
+              { /* More than one repetition is allowed, so put in at the
+                   end a backward relative jump from `b' to before the next
+                   jump we're going to put in below (which jumps from
+                   laststart to after this jump).  
+
+                   But if we are at the `*' in the exact sequence `.*\n',
+                   insert an unconditional jump backwards to the .,
+                   instead of the beginning of the loop.  This way we only
+                   push a failure point once, instead of every time
+                   through the loop.  */
+                assert (p - 1 > pattern);
+
+                /* Allocate the space for the jump.  */
+                GET_BUFFER_SPACE (3);
+
+                /* We know we are not at the first character of the pattern,
+                   because laststart was nonzero.  And we've already
+                   incremented `p', by the way, to be the character after
+                   the `*'.  Do we have to do something analogous here
+                   for null bytes, because of RE_DOT_NOT_NULL?  */
+                if (TRANSLATE (*(p - 2)) == TRANSLATE ('.')
+		    && zero_times_ok
+                    && p < pend && TRANSLATE (*p) == TRANSLATE ('\n')
+                    && !(syntax & RE_DOT_NEWLINE))
+                  { /* We have .*\n.  */
+                    STORE_JUMP (jump, b, laststart);
+                    keep_string_p = true;
+                  }
+                else
+                  /* Anything else.  */
+                  STORE_JUMP (maybe_pop_jump, b, laststart - 3);
+
+                /* We've added more stuff to the buffer.  */
+                b += 3;
+              }
+
+            /* On failure, jump from laststart to b + 3, which will be the
+               end of the buffer after this jump is inserted.  */
+            GET_BUFFER_SPACE (3);
+            INSERT_JUMP (keep_string_p ? on_failure_keep_string_jump
+                                       : on_failure_jump,
+                         laststart, b + 3);
+            pending_exact = 0;
+            b += 3;
+
+            if (!zero_times_ok)
+              {
+                /* At least one repetition is required, so insert a
+                   `dummy_failure_jump' before the initial
+                   `on_failure_jump' instruction of the loop. This
+                   effects a skip over that instruction the first time
+                   we hit that loop.  */
+                GET_BUFFER_SPACE (3);
+                INSERT_JUMP (dummy_failure_jump, laststart, laststart + 6);
+                b += 3;
+              }
+            }
+	  break;
+
+
+	case '.':
+          laststart = b;
+          BUF_PUSH (anychar);
+          break;
+
+
+        case '[':
+          {
+            boolean had_char_class = false;
+
+            if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+            /* Ensure that we have enough space to push a charset: the
+               opcode, the length count, and the bitset; 34 bytes in all.  */
+	    GET_BUFFER_SPACE (34);
+
+            laststart = b;
+
+            /* We test `*p == '^' twice, instead of using an if
+               statement, so we only need one BUF_PUSH.  */
+            BUF_PUSH (*p == '^' ? charset_not : charset); 
+            if (*p == '^')
+              p++;
+
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+
+            /* Push the number of bytes in the bitmap.  */
+            BUF_PUSH ((1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* Clear the whole map.  */
+            bzero (b, (1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* charset_not matches newline according to a syntax bit.  */
+            if ((re_opcode_t) b[-2] == charset_not
+                && (syntax & RE_HAT_LISTS_NOT_NEWLINE))
+              SET_LIST_BIT ('\n');
+
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                PATFETCH (c);
+
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    SET_LIST_BIT (c1);
+                    continue;
+                  }
+
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  break;
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  FREE_STACK_RETURN (REG_ERANGE);
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                /* See if we're at the beginning of a possible character
+                   class.  */
+
+                else if (syntax & RE_CHAR_CLASSES && c == '[' && *p == ':')
+                  { /* Leave room for the null.  */
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+
+                    PATFETCH (c);
+                    c1 = 0;
+
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+                          break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = STREQ (str, "alnum");
+                        boolean is_alpha = STREQ (str, "alpha");
+                        boolean is_blank = STREQ (str, "blank");
+                        boolean is_cntrl = STREQ (str, "cntrl");
+                        boolean is_digit = STREQ (str, "digit");
+                        boolean is_graph = STREQ (str, "graph");
+                        boolean is_lower = STREQ (str, "lower");
+                        boolean is_print = STREQ (str, "print");
+                        boolean is_punct = STREQ (str, "punct");
+                        boolean is_space = STREQ (str, "space");
+                        boolean is_upper = STREQ (str, "upper");
+                        boolean is_xdigit = STREQ (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str))
+			  FREE_STACK_RETURN (REG_ECTYPE);
+
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+
+                        if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                        for (ch = 0; ch < 1 << BYTEWIDTH; ch++)
+                          {
+			    /* This was split into 3 if's to
+			       avoid an arbitrary limit in some compiler.  */
+                            if (   (is_alnum  && ISALNUM (ch))
+                                || (is_alpha  && ISALPHA (ch))
+                                || (is_blank  && ISBLANK (ch))
+                                || (is_cntrl  && ISCNTRL (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_digit  && ISDIGIT (ch))
+                                || (is_graph  && ISGRAPH (ch))
+                                || (is_lower  && ISLOWER (ch))
+                                || (is_print  && ISPRINT (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_punct  && ISPUNCT (ch))
+                                || (is_space  && ISSPACE (ch))
+                                || (is_upper  && ISUPPER (ch))
+                                || (is_xdigit && ISXDIGIT (ch)))
+			      SET_LIST_BIT (ch);
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+                        SET_LIST_BIT ('[');
+                        SET_LIST_BIT (':');
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+                    SET_LIST_BIT (c);
+                  }
+              }
+
+            /* Discard any (non)matching list bytes that are all 0 at the
+               end of the map.  Decrease the map-length byte too.  */
+            while ((int) b[-1] > 0 && b[b[-1] - 1] == 0) 
+              b[-1]--; 
+            b += b[-1];
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+           if (syntax & RE_INTERVALS && syntax & RE_NO_BK_BRACES)
+             goto handle_interval;
+           else
+             goto normal_char;
+
+
+        case '\\':
+          if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              bufp->re_nsub++;
+              regnum++;
+
+              if (COMPILE_STACK_FULL)
+                { 
+                  RETALLOC (compile_stack.stack, compile_stack.size << 1,
+                            compile_stack_elt_t);
+                  if (compile_stack.stack == NULL) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+              /* These are the values to restore when we hit end of this
+                 group.  They are all relative offsets, so that if the
+                 whole pattern moves because of realloc, they will still
+                 be valid.  */
+              COMPILE_STACK_TOP.begalt_offset = begalt - bufp->buffer;
+              COMPILE_STACK_TOP.fixup_alt_jump 
+                = fixup_alt_jump ? fixup_alt_jump - bufp->buffer + 1 : 0;
+              COMPILE_STACK_TOP.laststart_offset = b - bufp->buffer;
+              COMPILE_STACK_TOP.regnum = regnum;
+
+              /* We will eventually replace the 0 with the number of
+                 groups inner to this one.  But do not push a
+                 start_memory for groups beyond the last one we can
+                 represent in the compiled pattern.  */
+              if (regnum <= MAX_REGNUM)
+                {
+                  COMPILE_STACK_TOP.inner_group_offset = b - bufp->buffer + 2;
+                  BUF_PUSH_3 (start_memory, regnum, 0);
+                }
+                
+              compile_stack.avail++;
+
+              fixup_alt_jump = 0;
+              laststart = 0;
+              begalt = b;
+	      /* If we've reached MAX_REGNUM groups, then this open
+		 won't actually generate any code, so we'll have to
+		 clear pending_exact explicitly.  */
+	      pending_exact = 0;
+              break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_backslash;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+            handle_close:
+              if (fixup_alt_jump)
+                { /* Push a dummy failure point at the end of the
+                     alternative for a possible future
+                     `pop_failure_jump' to pop.  See comments at
+                     `push_dummy_failure' in `re_match_2'.  */
+                  BUF_PUSH (push_dummy_failure);
+                  
+                  /* We allocated space for this jump when we assigned
+                     to `fixup_alt_jump', in the `handle_alt' case below.  */
+                  STORE_JUMP (jump_past_alt, fixup_alt_jump, b - 1);
+                }
+
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+              assert (compile_stack.avail != 0);
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+
+                compile_stack.avail--;		
+                begalt = bufp->buffer + COMPILE_STACK_TOP.begalt_offset;
+                fixup_alt_jump
+                  = COMPILE_STACK_TOP.fixup_alt_jump
+                    ? bufp->buffer + COMPILE_STACK_TOP.fixup_alt_jump - 1 
+                    : 0;
+                laststart = bufp->buffer + COMPILE_STACK_TOP.laststart_offset;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		/* If we've reached MAX_REGNUM groups, then this open
+		   won't actually generate any code, so we'll have to
+		   clear pending_exact explicitly.  */
+		pending_exact = 0;
+
+                /* We're at the end of the group, so now we know how many
+                   groups were inside this one.  */
+                if (this_group_regnum <= MAX_REGNUM)
+                  {
+                    unsigned char *inner_group_loc
+                      = bufp->buffer + COMPILE_STACK_TOP.inner_group_offset;
+                    
+                    *inner_group_loc = regnum - this_group_regnum;
+                    BUF_PUSH_3 (stop_memory, this_group_regnum,
+                                regnum - this_group_regnum);
+                  }
+              }
+              break;
+
+
+            case '|':					/* `\|'.  */
+              if (syntax & RE_LIMITED_OPS || syntax & RE_NO_BK_VBAR)
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+              /* Insert before the previous alternative a jump which
+                 jumps to this alternative if the former fails.  */
+              GET_BUFFER_SPACE (3);
+              INSERT_JUMP (on_failure_jump, begalt, b + 6);
+              pending_exact = 0;
+              b += 3;
+
+              /* The alternative before this one has a jump after it
+                 which gets executed if it gets matched.  Adjust that
+                 jump so it will jump to this alternative's analogous
+                 jump (put in below, which in turn will jump to the next
+                 (if any) alternative's such jump, etc.).  The last such
+                 jump jumps to the correct final destination.  A picture:
+                          _____ _____ 
+                          |   | |   |   
+                          |   v |   v 
+                         a | b   | c   
+
+                 If we are at `b', then fixup_alt_jump right now points to a
+                 three-byte space after `a'.  We'll put in the jump, set
+                 fixup_alt_jump to right after `b', and leave behind three
+                 bytes which we'll fill in when we get to after `c'.  */
+
+              if (fixup_alt_jump)
+                STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+              /* Mark and leave space for a jump after this alternative,
+                 to be filled in later either by next alternative or
+                 when know we're at the end of a series of alternatives.  */
+              fixup_alt_jump = b;
+              GET_BUFFER_SPACE (3);
+              b += 3;
+
+              laststart = 0;
+              begalt = b;
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      FREE_STACK_RETURN (REG_EBRACE);
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') FREE_STACK_RETURN (REG_EBRACE);
+
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (!laststart)
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      FREE_STACK_RETURN (REG_BADRPT);
+                    else if (syntax & RE_CONTEXT_INDEP_OPS)
+                      laststart = b;
+                    else
+                      goto unfetch_interval;
+                  }
+
+                /* If the upper bound is zero, don't want to succeed at
+                   all; jump from `laststart' to `b + 3', which will be
+                   the end of the buffer after we insert the jump.  */
+                 if (upper_bound == 0)
+                   {
+                     GET_BUFFER_SPACE (3);
+                     INSERT_JUMP (jump, laststart, b + 3);
+                     b += 3;
+                   }
+
+                 /* Otherwise, we have a nontrivial interval.  When
+                    we're all done, the pattern will look like:
+                      set_number_at <jump count> <upper bound>
+                      set_number_at <succeed_n count> <lower bound>
+                      succeed_n <after jump addr> <succeed_n count>
+                      <body of loop>
+                      jump_n <succeed_n addr> <jump count>
+                    (The upper bound and `jump_n' are omitted if
+                    `upper_bound' is 1, though.)  */
+                 else 
+                   { /* If the upper bound is > 1, we need to insert
+                        more at the end of the loop.  */
+                     unsigned nbytes = 10 + (upper_bound > 1) * 10;
+
+                     GET_BUFFER_SPACE (nbytes);
+
+                     /* Initialize lower bound of the `succeed_n', even
+                        though it will be set during matching by its
+                        attendant `set_number_at' (inserted next),
+                        because `re_compile_fastmap' needs to know.
+                        Jump to the `jump_n' we might insert below.  */
+                     INSERT_JUMP2 (succeed_n, laststart,
+                                   b + 5 + (upper_bound > 1) * 5,
+                                   lower_bound);
+                     b += 5;
+
+                     /* Code to initialize the lower bound.  Insert 
+                        before the `succeed_n'.  The `5' is the last two
+                        bytes of this `set_number_at', plus 3 bytes of
+                        the following `succeed_n'.  */
+                     insert_op2 (set_number_at, laststart, 5, lower_bound, b);
+                     b += 5;
+
+                     if (upper_bound > 1)
+                       { /* More than one repetition is allowed, so
+                            append a backward jump to the `succeed_n'
+                            that starts this interval.
+                            
+                            When we've reached this during matching,
+                            we'll have matched the interval once, so
+                            jump back only `upper_bound - 1' times.  */
+                         STORE_JUMP2 (jump_n, b, laststart + 5,
+                                      upper_bound - 1);
+                         b += 5;
+
+                         /* The location we want to set is the second
+                            parameter of the `jump_n'; that is `b-2' as
+                            an absolute address.  `laststart' will be
+                            the `set_number_at' we're about to insert;
+                            `laststart+3' the number to set, the source
+                            for the relative address.  But we are
+                            inserting into the middle of the pattern --
+                            so everything is getting moved up by 5.
+                            Conclusion: (b - 2) - (laststart + 3) + 5,
+                            i.e., b - laststart.
+                            
+                            We insert this at the beginning of the loop
+                            so that if we fail during matching, we'll
+                            reinitialize the bounds.  */
+                         insert_op2 (set_number_at, laststart, b - laststart,
+                                     upper_bound - 1, b);
+                         b += 5;
+                       }
+                   }
+                pending_exact = 0;
+                beg_interval = NULL;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               assert (beg_interval);
+               p = beg_interval;
+               beg_interval = NULL;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+              BUF_PUSH (at_dot);
+              break;
+
+            case 's':	
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (syntaxspec, syntax_spec_code[c]);
+              break;
+
+            case 'S':
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (notsyntaxspec, syntax_spec_code[c]);
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+              laststart = b;
+              BUF_PUSH (wordchar);
+              break;
+
+
+            case 'W':
+              laststart = b;
+              BUF_PUSH (notwordchar);
+              break;
+
+
+            case '<':
+              BUF_PUSH (wordbeg);
+              break;
+
+            case '>':
+              BUF_PUSH (wordend);
+              break;
+
+            case 'b':
+              BUF_PUSH (wordbound);
+              break;
+
+            case 'B':
+              BUF_PUSH (notwordbound);
+              break;
+
+            case '`':
+              BUF_PUSH (begbuf);
+              break;
+
+            case '\'':
+              BUF_PUSH (endbuf);
+              break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                FREE_STACK_RETURN (REG_ESUBREG);
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+                goto normal_char;
+
+              laststart = b;
+              BUF_PUSH_2 (duplicate, c1);
+              break;
+
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	      /* If no exactn currently being built.  */
+          if (!pending_exact 
+
+              /* If last exactn not at current position.  */
+              || pending_exact + *pending_exact + 1 != b
+              
+              /* We have only one byte following the exactn for the count.  */
+	      || *pending_exact == (1 << BYTEWIDTH) - 1
+
+              /* If followed by a repetition operator.  */
+              || *p == '*' || *p == '^'
+	      || ((syntax & RE_BK_PLUS_QM)
+		  ? *p == '\\' && (p[1] == '+' || p[1] == '?')
+		  : (*p == '+' || *p == '?'))
+	      || ((syntax & RE_INTERVALS)
+                  && ((syntax & RE_NO_BK_BRACES)
+		      ? *p == '{'
+                      : (p[0] == '\\' && p[1] == '{'))))
+	    {
+	      /* Start building a new exactn.  */
+              
+              laststart = b;
+
+	      BUF_PUSH_2 (exactn, 0);
+	      pending_exact = b - 1;
+            }
+            
+	  BUF_PUSH (c);
+          (*pending_exact)++;
+	  break;
+        } /* switch (c) */
+    } /* while p != pend */
+
+  
+  /* Through the pattern now.  */
+  
+  if (fixup_alt_jump)
+    STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+  if (!COMPILE_STACK_EMPTY) 
+    FREE_STACK_RETURN (REG_EPAREN);
+
+  /* If we don't want backtracking, force success
+     the first time we reach the end of the compiled pattern.  */
+  if (syntax & RE_NO_POSIX_BACKTRACKING)
+    BUF_PUSH (succeed);
+
+  free (compile_stack.stack);
+
+  /* We have succeeded; set the length of the buffer.  */
+  bufp->used = b - bufp->buffer;
+
+#ifdef DEBUG
+  if (debug)
+    {
+      DEBUG_PRINT1 ("\nCompiled pattern: \n");
+      print_compiled_pattern (bufp);
+    }
+#endif /* DEBUG */
+
+#ifndef MATCH_MAY_ALLOCATE
+  /* Initialize the failure stack to the largest possible stack.  This
+     isn't necessary unless we're trying to avoid calling alloca in
+     the search and match routines.  */
+  {
+    int num_regs = bufp->re_nsub + 1;
+
+    /* Since DOUBLE_FAIL_STACK refuses to double only if the current size
+       is strictly greater than re_max_failures, the largest possible stack
+       is 2 * re_max_failures failure points.  */
+    if (fail_stack.size < (2 * re_max_failures * MAX_FAILURE_ITEMS))
+      {
+	fail_stack.size = (2 * re_max_failures * MAX_FAILURE_ITEMS);
+
+#ifdef emacs
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xmalloc (fail_stack.size 
+					    * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xrealloc (fail_stack.stack,
+					     (fail_stack.size
+					      * sizeof (fail_stack_elt_t)));
+#else /* not emacs */
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) malloc (fail_stack.size 
+					   * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) realloc (fail_stack.stack,
+					    (fail_stack.size
+					     * sizeof (fail_stack_elt_t)));
+#endif /* not emacs */
+      }
+
+    regex_grow_registers (num_regs);
+  }
+#endif /* not MATCH_MAY_ALLOCATE */
+
+  return REG_NOERROR;
+} /* regex_compile */
+
+
+/* Subroutines for `regex_compile'.  */
+
+/* Store OP at LOC followed by two-byte integer parameter ARG.  */
+
+static void
+store_op1 (op, loc, arg)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg);
+}
+
+
+/* Like `store_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+store_op2 (op, loc, arg1, arg2)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg1);
+  STORE_NUMBER (loc + 3, arg2);
+}
+
+
+/* Copy the bytes from LOC to END to open up three bytes of space at LOC
+   for OP followed by two-byte integer parameter ARG.  */
+
+static void
+insert_op1 (op, loc, arg, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 3;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op1 (op, loc, arg);
+}
+
+
+/* Like `insert_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+insert_op2 (op, loc, arg1, arg2, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 5;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op2 (op, loc, arg1, arg2);
+}
+
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+   after an alternative or a begin-subexpression.  We assume there is at
+   least one character before the ^.  */
+
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+    const char *pattern, *p;
+    reg_syntax_t syntax;
+{
+  const char *prev = p - 2;
+  boolean prev_prev_backslash = prev > pattern && prev[-1] == '\\';
+  
+  return
+       /* After a subexpression?  */
+       (*prev == '(' && (syntax & RE_NO_BK_PARENS || prev_prev_backslash))
+       /* After an alternative?  */
+    || (*prev == '|' && (syntax & RE_NO_BK_VBAR || prev_prev_backslash));
+}
+
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+   at least one character after the $, i.e., `P < PEND'.  */
+
+static boolean
+at_endline_loc_p (p, pend, syntax)
+    const char *p, *pend;
+    int syntax;
+{
+  const char *next = p;
+  boolean next_backslash = *next == '\\';
+  const char *next_next = p + 1 < pend ? p + 1 : NULL;
+  
+  return
+       /* Before a subexpression?  */
+       (syntax & RE_NO_BK_PARENS ? *next == ')'
+        : next_backslash && next_next && *next_next == ')')
+       /* Before an alternative?  */
+    || (syntax & RE_NO_BK_VBAR ? *next == '|'
+        : next_backslash && next_next && *next_next == '|');
+}
+
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/* Read the ending character of a range (in a bracket expression) from the
+   uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+   starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+   Then we set the translation of all bits between the starting and
+   ending characters (inclusive) in the compiled pattern B.
+   
+   Return an error code.
+   
+   We use these short variable names so we can use the same macros as
+   `regex_compile' itself.  */
+
+static reg_errcode_t
+compile_range (p_ptr, pend, translate, syntax, b)
+    const char **p_ptr, *pend;
+    char *translate;
+    reg_syntax_t syntax;
+    unsigned char *b;
+{
+  unsigned this_char;
+
+  const char *p = *p_ptr;
+  int range_start, range_end;
+  
+  if (p == pend)
+    return REG_ERANGE;
+
+  /* Even though the pattern is a signed `char *', we need to fetch
+     with unsigned char *'s; if the high bit of the pattern character
+     is set, the range endpoints will be negative if we fetch using a
+     signed char *.
+
+     We also want to fetch the endpoints without translating them; the 
+     appropriate translation is done in the bit-setting loop below.  */
+  /* The SVR4 compiler on the 3B2 had trouble with unsigned const char *.  */
+  range_start = ((const unsigned char *) p)[-2];
+  range_end   = ((const unsigned char *) p)[0];
+
+  /* Have to increment the pointer into the pattern string, so the
+     caller isn't still at the ending character.  */
+  (*p_ptr)++;
+
+  /* If the start is after the end, the range is empty.  */
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  /* Here we see why `this_char' has to be larger than an `unsigned
+     char' -- the range is inclusive, so if `range_end' == 0xff
+     (assuming 8-bit characters), we would otherwise go into an infinite
+     loop, since all characters <= 0xff.  */
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      SET_LIST_BIT (TRANSLATE (this_char));
+    }
+  
+  return REG_NOERROR;
+}
+
+
+/* re_compile_fastmap computes a ``fastmap'' for the compiled pattern in
+   BUFP.  A fastmap records which of the (1 << BYTEWIDTH) possible
+   characters can start a string that matches the pattern.  This fastmap
+   is used by re_search to skip quickly over impossible starting points.
+
+   The caller must supply the address of a (1 << BYTEWIDTH)-byte data
+   area as BUFP->fastmap.
+   
+   We set the `fastmap', `fastmap_accurate', and `can_be_null' fields in
+   the pattern buffer.
+
+   Returns 0 if we succeed, -2 if an internal error.   */
+
+int
+re_compile_fastmap (bufp)
+     struct re_pattern_buffer *bufp;
+{
+  int j, k;
+#ifdef MATCH_MAY_ALLOCATE
+  fail_stack_type fail_stack;
+#endif
+#ifndef REGEX_MALLOC
+  char *destination;
+#endif
+  /* We don't push any register information onto the failure stack.  */
+  unsigned num_regs = 0;
+  
+  register char *fastmap = bufp->fastmap;
+  unsigned char *pattern = bufp->buffer;
+  unsigned long size = bufp->used;
+  unsigned char *p = pattern;
+  register unsigned char *pend = pattern + size;
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* Assume that each path through the pattern can be null until
+     proven otherwise.  We set this false at the bottom of switch
+     statement, to which we get only if a particular path doesn't
+     match the empty string.  */
+  boolean path_can_be_null = true;
+
+  /* We aren't doing a `succeed_n' to begin with.  */
+  boolean succeed_n_p = false;
+
+  assert (fastmap != NULL && p != NULL);
+  
+  INIT_FAIL_STACK ();
+  bzero (fastmap, 1 << BYTEWIDTH);  /* Assume nothing's valid.  */
+  bufp->fastmap_accurate = 1;	    /* It will be when we're done.  */
+  bufp->can_be_null = 0;
+      
+  while (1)
+    {
+      if (p == pend || *p == succeed)
+	{
+	  /* We have reached the (effective) end of pattern.  */
+	  if (!FAIL_STACK_EMPTY ())
+	    {
+	      bufp->can_be_null |= path_can_be_null;
+
+	      /* Reset for next path.  */
+	      path_can_be_null = true;
+
+	      p = fail_stack.stack[--fail_stack.avail].pointer;
+
+	      continue;
+	    }
+	  else
+	    break;
+	}
+
+      /* We should never be about to go beyond the end of the pattern.  */
+      assert (p < pend);
+      
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+
+        /* I guess the idea here is to simply not bother with a fastmap
+           if a backreference is used, since it's too hard to figure out
+           the fastmap for the corresponding group.  Setting
+           `can_be_null' stops `re_search_2' from using the fastmap, so
+           that is all we do.  */
+	case duplicate:
+	  bufp->can_be_null = 1;
+          goto done;
+
+
+      /* Following are the cases which match a character.  These end
+         with `break'.  */
+
+	case exactn:
+          fastmap[p[1]] = 1;
+	  break;
+
+
+        case charset:
+          for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH)))
+              fastmap[j] = 1;
+	  break;
+
+
+	case charset_not:
+	  /* Chars beyond end of map must be allowed.  */
+	  for (j = *p * BYTEWIDTH; j < (1 << BYTEWIDTH); j++)
+            fastmap[j] = 1;
+
+	  for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (!(p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH))))
+              fastmap[j] = 1;
+          break;
+
+
+	case wordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notwordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+        case anychar:
+	  {
+	    int fastmap_newline = fastmap['\n'];
+
+	    /* `.' matches anything ...  */
+	    for (j = 0; j < (1 << BYTEWIDTH); j++)
+	      fastmap[j] = 1;
+
+	    /* ... except perhaps newline.  */
+	    if (!(bufp->syntax & RE_DOT_NEWLINE))
+	      fastmap['\n'] = fastmap_newline;
+
+	    /* Return if we have already set `can_be_null'; if we have,
+	       then the fastmap is irrelevant.  Something's wrong here.  */
+	    else if (bufp->can_be_null)
+	      goto done;
+
+	    /* Otherwise, have to check alternative paths.  */
+	    break;
+	  }
+
+#ifdef emacs
+        case syntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notsyntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+      /* All cases after this match the empty string.  These end with
+         `continue'.  */
+
+
+	case before_dot:
+	case at_dot:
+	case after_dot:
+          continue;
+#endif /* not emacs */
+
+
+        case no_op:
+        case begline:
+        case endline:
+	case begbuf:
+	case endbuf:
+	case wordbound:
+	case notwordbound:
+	case wordbeg:
+	case wordend:
+        case push_dummy_failure:
+          continue;
+
+
+	case jump_n:
+        case pop_failure_jump:
+	case maybe_pop_jump:
+	case jump:
+        case jump_past_alt:
+	case dummy_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+	  p += j;	
+	  if (j > 0)
+	    continue;
+            
+          /* Jump backward implies we just went through the body of a
+             loop and matched nothing.  Opcode jumped to should be
+             `on_failure_jump' or `succeed_n'.  Just treat it like an
+             ordinary jump.  For a * loop, it has pushed its failure
+             point already; if so, discard that as redundant.  */
+          if ((re_opcode_t) *p != on_failure_jump
+	      && (re_opcode_t) *p != succeed_n)
+	    continue;
+
+          p++;
+          EXTRACT_NUMBER_AND_INCR (j, p);
+          p += j;		
+	  
+          /* If what's on the stack is where we are now, pop it.  */
+          if (!FAIL_STACK_EMPTY () 
+	      && fail_stack.stack[fail_stack.avail - 1].pointer == p)
+            fail_stack.avail--;
+
+          continue;
+
+
+        case on_failure_jump:
+        case on_failure_keep_string_jump:
+	handle_on_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+
+          /* For some patterns, e.g., `(a?)?', `p+j' here points to the
+             end of the pattern.  We don't want to push such a point,
+             since when we restore it above, entering the switch will
+             increment `p' past the end of the pattern.  We don't need
+             to push such a point since we obviously won't find any more
+             fastmap entries beyond `pend'.  Such a pattern can match
+             the null string, though.  */
+          if (p + j < pend)
+            {
+              if (!PUSH_PATTERN_OP (p + j, fail_stack))
+		{
+		  RESET_FAIL_STACK ();
+		  return -2;
+		}
+            }
+          else
+            bufp->can_be_null = 1;
+
+          if (succeed_n_p)
+            {
+              EXTRACT_NUMBER_AND_INCR (k, p);	/* Skip the n.  */
+              succeed_n_p = false;
+	    }
+
+          continue;
+
+
+	case succeed_n:
+          /* Get to the number of times to succeed.  */
+          p += 2;		
+
+          /* Increment p past the n for when k != 0.  */
+          EXTRACT_NUMBER_AND_INCR (k, p);
+          if (k == 0)
+	    {
+              p -= 4;
+  	      succeed_n_p = true;  /* Spaghetti code alert.  */
+              goto handle_on_failure_jump;
+            }
+          continue;
+
+
+	case set_number_at:
+          p += 4;
+          continue;
+
+
+	case start_memory:
+        case stop_memory:
+	  p += 2;
+	  continue;
+
+
+	default:
+          abort (); /* We have listed all the cases.  */
+        } /* switch *p++ */
+
+      /* Getting here means we have found the possible starting
+         characters for one path of the pattern -- and that the empty
+         string does not match.  We need not follow this path further.
+         Instead, look at the next alternative (remembered on the
+         stack), or quit if no more.  The test at the top of the loop
+         does these things.  */
+      path_can_be_null = false;
+      p = pend;
+    } /* while p */
+
+  /* Set `can_be_null' for the last path (also the first path, if the
+     pattern is empty).  */
+  bufp->can_be_null |= path_can_be_null;
+
+ done:
+  RESET_FAIL_STACK ();
+  return 0;
+} /* re_compile_fastmap */
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+    struct re_pattern_buffer *bufp;
+    struct re_registers *regs;
+    unsigned num_regs;
+    regoff_t *starts, *ends;
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t *) 0;
+    }
+}
+
+
+/* Searching routines.  */
+
+/* Like re_search_2, below, but only one string is specified, and
+   doesn't let you say where to stop matching. */
+
+int
+re_search (bufp, string, size, startpos, range, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, startpos, range;
+     struct re_registers *regs;
+{
+  return re_search_2 (bufp, NULL, 0, string, size, startpos, range, 
+		      regs, size);
+}
+
+
+/* Using the compiled pattern in BUFP->buffer, first tries to match the
+   virtual concatenation of STRING1 and STRING2, starting first at index
+   STARTPOS, then at STARTPOS + 1, and so on.
+   
+   STRING1 and STRING2 have length SIZE1 and SIZE2, respectively.
+   
+   RANGE is how far to scan while trying to match.  RANGE = 0 means try
+   only at STARTPOS; in general, the last start tried is STARTPOS +
+   RANGE.
+   
+   In REGS, return the indices of the virtual concatenation of STRING1
+   and STRING2 that matched the entire BUFP->buffer and its contained
+   subexpressions.
+   
+   Do not consider matching one past the index STOP in the virtual
+   concatenation of STRING1 and STRING2.
+
+   We return either the position in the strings at which the match was
+   found, -1 if no match, or -2 if error (such as failure
+   stack overflow).  */
+
+int
+re_search_2 (bufp, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+{
+  int val;
+  register char *fastmap = bufp->fastmap;
+  register char *translate = bufp->translate;
+  int total_size = size1 + size2;
+  int endpos = startpos + range;
+
+  /* Check for out-of-range STARTPOS.  */
+  if (startpos < 0 || startpos > total_size)
+    return -1;
+    
+  /* Fix up RANGE if it might eventually take us outside
+     the virtual concatenation of STRING1 and STRING2.  */
+  if (endpos < -1)
+    range = -1 - startpos;
+  else if (endpos > total_size)
+    range = total_size - startpos;
+
+  /* If the search isn't to be a backwards one, don't waste time in a
+     search for a pattern that must be anchored.  */
+  if (bufp->used > 0 && (re_opcode_t) bufp->buffer[0] == begbuf && range > 0)
+    {
+      if (startpos > 0)
+	return -1;
+      else
+	range = 1;
+    }
+
+  /* Update the fastmap now if not correct already.  */
+  if (fastmap && !bufp->fastmap_accurate)
+    if (re_compile_fastmap (bufp) == -2)
+      return -2;
+  
+  /* Loop through the string, looking for a place to start matching.  */
+  for (;;)
+    { 
+      /* If a fastmap is supplied, skip quickly over characters that
+         cannot be the start of a match.  If the pattern can match the
+         null string, however, we don't need to skip characters; we want
+         the first null string.  */
+      if (fastmap && startpos < total_size && !bufp->can_be_null)
+	{
+	  if (range > 0)	/* Searching forwards.  */
+	    {
+	      register const char *d;
+	      register int lim = 0;
+	      int irange = range;
+
+              if (startpos < size1 && startpos + range >= size1)
+                lim = range - (size1 - startpos);
+
+	      d = (startpos >= size1 ? string2 - size1 : string1) + startpos;
+   
+              /* Written out as an if-else to avoid testing `translate'
+                 inside the loop.  */
+	      if (translate)
+                while (range > lim
+                       && !fastmap[(unsigned char)
+				   translate[(unsigned char) *d++]])
+                  range--;
+	      else
+                while (range > lim && !fastmap[(unsigned char) *d++])
+                  range--;
+
+	      startpos += irange - range;
+	    }
+	  else				/* Searching backwards.  */
+	    {
+	      register char c = (size1 == 0 || startpos >= size1
+                                 ? string2[startpos - size1] 
+                                 : string1[startpos]);
+
+	      if (!fastmap[(unsigned char) TRANSLATE (c)])
+		goto advance;
+	    }
+	}
+
+      /* If can't match the null string, and that's all we have left, fail.  */
+      if (range >= 0 && startpos == total_size && fastmap
+          && !bufp->can_be_null)
+	return -1;
+
+      val = re_match_2_internal (bufp, string1, size1, string2, size2,
+				 startpos, regs, stop);
+#ifndef REGEX_MALLOC
+#ifdef C_ALLOCA
+      alloca (0);
+#endif
+#endif
+
+      if (val >= 0)
+	return startpos;
+        
+      if (val == -2)
+	return -2;
+
+    advance:
+      if (!range) 
+        break;
+      else if (range > 0) 
+        {
+          range--; 
+          startpos++;
+        }
+      else
+        {
+          range++; 
+          startpos--;
+        }
+    }
+  return -1;
+} /* re_search_2 */
+
+
+/* Declarations and macros for re_match_2.  */
+
+static int bcmp_translate ();
+static boolean alt_match_null_string_p (),
+               common_op_match_null_string_p (),
+               group_match_null_string_p ();
+
+/* This converts PTR, a pointer into one of the search strings `string1'
+   and `string2' into an offset from the beginning of that string.  */
+#define POINTER_TO_OFFSET(ptr)			\
+  (FIRST_STRING_P (ptr)				\
+   ? ((regoff_t) ((ptr) - string1))		\
+   : ((regoff_t) ((ptr) - string2 + size1)))
+
+/* Macros for dealing with the split strings in re_match_2.  */
+
+#define MATCHING_IN_FIRST_STRING  (dend == end_match_1)
+
+/* Call before fetching a character with *d.  This switches over to
+   string2 if necessary.  */
+#define PREFETCH()							\
+  while (d == dend)						    	\
+    {									\
+      /* End of string2 => fail.  */					\
+      if (dend == end_match_2) 						\
+        goto fail;							\
+      /* End of string1 => advance to string2.  */ 			\
+      d = string2;						        \
+      dend = end_match_2;						\
+    }
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+   of `string1' and `string2'.  If only one string, it's `string2'.  */
+#define AT_STRINGS_BEG(d) ((d) == (size1 ? string1 : string2) || !size2)
+#define AT_STRINGS_END(d) ((d) == end2)	
+
+
+/* Test if D points to a character which is word-constituent.  We have
+   two special cases to check for: if past the end of string1, look at
+   the first character in string2; and if before the beginning of
+   string2, look at the last character in string1.  */
+#define WORDCHAR_P(d)							\
+  (SYNTAX ((d) == end1 ? *string2					\
+           : (d) == string2 - 1 ? *(end1 - 1) : *(d))			\
+   == Sword)
+
+/* Test if the character before D and the one at D differ with respect
+   to being word-constituent.  */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG (d) || AT_STRINGS_END (d)				\
+   || WORDCHAR_P (d - 1) != WORDCHAR_P (d))
+
+
+/* Free everything we malloc.  */
+#ifdef MATCH_MAY_ALLOCATE
+#define FREE_VAR(var) if (var) REGEX_FREE (var); var = NULL
+#define FREE_VARIABLES()						\
+  do {									\
+    REGEX_FREE_STACK (fail_stack.stack);				\
+    FREE_VAR (regstart);						\
+    FREE_VAR (regend);							\
+    FREE_VAR (old_regstart);						\
+    FREE_VAR (old_regend);						\
+    FREE_VAR (best_regstart);						\
+    FREE_VAR (best_regend);						\
+    FREE_VAR (reg_info);						\
+    FREE_VAR (reg_dummy);						\
+    FREE_VAR (reg_info_dummy);						\
+  } while (0)
+#else
+#define FREE_VARIABLES() ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+#endif /* not MATCH_MAY_ALLOCATE */
+
+/* These values must meet several constraints.  They must not be valid
+   register values; since we have a limit of 255 registers (because
+   we use only one byte in the pattern for the register number), we can
+   use numbers larger than 255.  They must differ by 1, because of
+   NUM_FAILURE_ITEMS above.  And the value for the lowest register must
+   be larger than the value for the highest register, so we do not try
+   to actually save any registers when none are active.  */
+#define NO_HIGHEST_ACTIVE_REG (1 << BYTEWIDTH)
+#define NO_LOWEST_ACTIVE_REG (NO_HIGHEST_ACTIVE_REG + 1)
+
+
+/* Matching routines.  */
+
+#ifndef emacs   /* Emacs never uses this.  */
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+int
+re_match (bufp, string, size, pos, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, pos;
+     struct re_registers *regs;
+{
+  int result = re_match_2_internal (bufp, NULL, 0, string, size,
+				    pos, regs, size);
+  alloca (0);
+  return result;
+}
+#endif /* not emacs */
+
+
+/* re_match_2 matches the compiled pattern in BUFP against the
+   the (virtual) concatenation of STRING1 and STRING2 (of length SIZE1
+   and SIZE2, respectively).  We start matching at POS, and stop
+   matching at STOP.
+   
+   If REGS is non-null and the `no_sub' field of BUFP is nonzero, we
+   store offsets for the substring each group matched in REGS.  See the
+   documentation for exactly how many groups we fill.
+
+   We return -1 if no match, -2 if an internal error (such as the
+   failure stack overflowing).  Otherwise, we return the length of the
+   matched substring.  */
+
+int
+re_match_2 (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  int result = re_match_2_internal (bufp, string1, size1, string2, size2,
+				    pos, regs, stop);
+  alloca (0);
+  return result;
+}
+
+/* This is a separate function so that we can force an alloca cleanup
+   afterwards.  */
+static int
+re_match_2_internal (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  /* General temporaries.  */
+  int mcnt;
+  unsigned char *p1;
+
+  /* Just past the end of the corresponding string.  */
+  const char *end1, *end2;
+
+  /* Pointers into string1 and string2, just past the last characters in
+     each to consider matching.  */
+  const char *end_match_1, *end_match_2;
+
+  /* Where we are in the data, and the end of the current string.  */
+  const char *d, *dend;
+  
+  /* Where we are in the pattern, and the end of the pattern.  */
+  unsigned char *p = bufp->buffer;
+  register unsigned char *pend = p + bufp->used;
+
+  /* Mark the opcode just after a start_memory, so we can test for an
+     empty subpattern when we get to the stop_memory.  */
+  unsigned char *just_past_start_mem = 0;
+
+  /* We use this to map every character in the string.  */
+  char *translate = bufp->translate;
+
+  /* Failure point stack.  Each place that can handle a failure further
+     down the line pushes a failure point on this stack.  It consists of
+     restart, regend, and reg_info for all registers corresponding to
+     the subexpressions we're currently inside, plus the number of such
+     registers, and, finally, two char *'s.  The first char * is where
+     to resume scanning the pattern; the second one is where to resume
+     scanning the strings.  If the latter is zero, the failure point is
+     a ``dummy''; if a failure happens and the failure point is a dummy,
+     it gets discarded and the next next one is tried.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  fail_stack_type fail_stack;
+#endif
+#ifdef DEBUG
+  static unsigned failure_id = 0;
+  unsigned nfailure_points_pushed = 0, nfailure_points_popped = 0;
+#endif
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* We fill all the registers internally, independent of what we
+     return, for use in backreferences.  The number here includes
+     an element for register zero.  */
+  unsigned num_regs = bufp->re_nsub + 1;
+  
+  /* The currently active registers.  */
+  unsigned lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+  unsigned highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+
+  /* Information on the contents of registers. These are pointers into
+     the input strings; they record just what was matched (on this
+     attempt) by a subexpression part of the pattern, that is, the
+     regnum-th regstart pointer points to where in the pattern we began
+     matching and the regnum-th regend points to right after where we
+     stopped matching the regnum-th subexpression.  (The zeroth register
+     keeps track of what the whole pattern matches.)  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **regstart, **regend;
+#endif
+
+  /* If a group that's operated upon by a repetition operator fails to
+     match anything, then the register for its start will need to be
+     restored because it will have been set to wherever in the string we
+     are when we last see its open-group operator.  Similarly for a
+     register's end.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **old_regstart, **old_regend;
+#endif
+
+  /* The is_active field of reg_info helps us keep track of which (possibly
+     nested) subexpressions we are currently in. The matched_something
+     field of reg_info[reg_num] helps us tell whether or not we have
+     matched any of the pattern so far this time through the reg_num-th
+     subexpression.  These two fields get reset each time through any
+     loop their register is in.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  register_info_type *reg_info; 
+#endif
+
+  /* The following record the register info as found in the above
+     variables when we find a match better than any we've seen before. 
+     This happens as we backtrack through the failure points, which in
+     turn happens only if we have not yet matched the entire string. */
+  unsigned best_regs_set = false;
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **best_regstart, **best_regend;
+#endif
+  
+  /* Logically, this is `best_regend[0]'.  But we don't want to have to
+     allocate space for that if we're not allocating space for anything
+     else (see below).  Also, we never need info about register 0 for
+     any of the other register vectors, and it seems rather a kludge to
+     treat `best_regend' differently than the rest.  So we keep track of
+     the end of the best match so far in a separate variable.  We
+     initialize this to NULL so that when we backtrack the first time
+     and need to test it, it's not garbage.  */
+  const char *match_end = NULL;
+
+  /* This helps SET_REGS_MATCHED avoid doing redundant work.  */
+  int set_regs_matched_done = 0;
+
+  /* Used when we pop values we don't care about.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **reg_dummy;
+  register_info_type *reg_info_dummy;
+#endif
+
+#ifdef DEBUG
+  /* Counts the total number of registers pushed.  */
+  unsigned num_regs_pushed = 0; 	
+#endif
+
+  DEBUG_PRINT1 ("\n\nEntering re_match_2.\n");
+  
+  INIT_FAIL_STACK ();
+  
+#ifdef MATCH_MAY_ALLOCATE
+  /* Do not bother to initialize all the register variables if there are
+     no groups in the pattern, as it takes a fair amount of time.  If
+     there are groups, we include space for register 0 (the whole
+     pattern), even though we never use it, since it simplifies the
+     array indexing.  We should fix this.  */
+  if (bufp->re_nsub)
+    {
+      regstart = REGEX_TALLOC (num_regs, const char *);
+      regend = REGEX_TALLOC (num_regs, const char *);
+      old_regstart = REGEX_TALLOC (num_regs, const char *);
+      old_regend = REGEX_TALLOC (num_regs, const char *);
+      best_regstart = REGEX_TALLOC (num_regs, const char *);
+      best_regend = REGEX_TALLOC (num_regs, const char *);
+      reg_info = REGEX_TALLOC (num_regs, register_info_type);
+      reg_dummy = REGEX_TALLOC (num_regs, const char *);
+      reg_info_dummy = REGEX_TALLOC (num_regs, register_info_type);
+
+      if (!(regstart && regend && old_regstart && old_regend && reg_info 
+            && best_regstart && best_regend && reg_dummy && reg_info_dummy)) 
+        {
+          FREE_VARIABLES ();
+          return -2;
+        }
+    }
+  else
+    {
+      /* We must initialize all our variables to NULL, so that
+         `FREE_VARIABLES' doesn't try to free them.  */
+      regstart = regend = old_regstart = old_regend = best_regstart
+        = best_regend = reg_dummy = NULL;
+      reg_info = reg_info_dummy = (register_info_type *) NULL;
+    }
+#endif /* MATCH_MAY_ALLOCATE */
+
+  /* The starting position is bogus.  */
+  if (pos < 0 || pos > size1 + size2)
+    {
+      FREE_VARIABLES ();
+      return -1;
+    }
+    
+  /* Initialize subexpression text positions to -1 to mark ones that no
+     start_memory/stop_memory has been seen for. Also initialize the
+     register information struct.  */
+  for (mcnt = 1; mcnt < num_regs; mcnt++)
+    {
+      regstart[mcnt] = regend[mcnt] 
+        = old_regstart[mcnt] = old_regend[mcnt] = REG_UNSET_VALUE;
+        
+      REG_MATCH_NULL_STRING_P (reg_info[mcnt]) = MATCH_NULL_UNSET_VALUE;
+      IS_ACTIVE (reg_info[mcnt]) = 0;
+      MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+      EVER_MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+    }
+  
+  /* We move `string1' into `string2' if the latter's empty -- but not if
+     `string1' is null.  */
+  if (size2 == 0 && string1 != NULL)
+    {
+      string2 = string1;
+      size2 = size1;
+      string1 = 0;
+      size1 = 0;
+    }
+  end1 = string1 + size1;
+  end2 = string2 + size2;
+
+  /* Compute where to stop matching, within the two strings.  */
+  if (stop <= size1)
+    {
+      end_match_1 = string1 + stop;
+      end_match_2 = string2;
+    }
+  else
+    {
+      end_match_1 = end1;
+      end_match_2 = string2 + stop - size1;
+    }
+
+  /* `p' scans through the pattern as `d' scans through the data. 
+     `dend' is the end of the input string that `d' points within.  `d'
+     is advanced into the following input string whenever necessary, but
+     this happens before fetching; therefore, at the beginning of the
+     loop, `d' can be pointing at the end of a string, but it cannot
+     equal `string2'.  */
+  if (size1 > 0 && pos <= size1)
+    {
+      d = string1 + pos;
+      dend = end_match_1;
+    }
+  else
+    {
+      d = string2 + pos - size1;
+      dend = end_match_2;
+    }
+
+  DEBUG_PRINT1 ("The compiled pattern is: ");
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, p, pend);
+  DEBUG_PRINT1 ("The string to match is: `");
+  DEBUG_PRINT_DOUBLE_STRING (d, string1, size1, string2, size2);
+  DEBUG_PRINT1 ("'\n");
+  
+  /* This loops over pattern commands.  It exits by returning from the
+     function if the match is complete, or it drops through if the match
+     fails at this starting point in the input data.  */
+  for (;;)
+    {
+      DEBUG_PRINT2 ("\n0x%x: ", p);
+
+      if (p == pend)
+	{ /* End of pattern means we might have succeeded.  */
+          DEBUG_PRINT1 ("end of pattern ... ");
+          
+	  /* If we haven't matched the entire string, and we want the
+             longest match, try backtracking.  */
+          if (d != end_match_2)
+	    {
+	      /* 1 if this match ends in the same string (string1 or string2)
+		 as the best previous match.  */
+	      boolean same_str_p = (FIRST_STRING_P (match_end) 
+				    == MATCHING_IN_FIRST_STRING);
+	      /* 1 if this match is the best seen so far.  */
+	      boolean best_match_p;
+
+	      /* AIX compiler got confused when this was combined
+		 with the previous declaration.  */
+	      if (same_str_p)
+		best_match_p = d > match_end;
+	      else
+		best_match_p = !MATCHING_IN_FIRST_STRING;
+
+              DEBUG_PRINT1 ("backtracking.\n");
+              
+              if (!FAIL_STACK_EMPTY ())
+                { /* More failure points to try.  */
+
+                  /* If exceeds best match so far, save it.  */
+                  if (!best_regs_set || best_match_p)
+                    {
+                      best_regs_set = true;
+                      match_end = d;
+                      
+                      DEBUG_PRINT1 ("\nSAVING match as best so far.\n");
+                      
+                      for (mcnt = 1; mcnt < num_regs; mcnt++)
+                        {
+                          best_regstart[mcnt] = regstart[mcnt];
+                          best_regend[mcnt] = regend[mcnt];
+                        }
+                    }
+                  goto fail;	       
+                }
+
+              /* If no failure points, don't restore garbage.  And if
+                 last match is real best match, don't restore second
+                 best one. */
+              else if (best_regs_set && !best_match_p)
+                {
+  	        restore_best_regs:
+                  /* Restore best match.  It may happen that `dend ==
+                     end_match_1' while the restored d is in string2.
+                     For example, the pattern `x.*y.*z' against the
+                     strings `x-' and `y-z-', if the two strings are
+                     not consecutive in memory.  */
+                  DEBUG_PRINT1 ("Restoring best registers.\n");
+                  
+                  d = match_end;
+                  dend = ((d >= string1 && d <= end1)
+		           ? end_match_1 : end_match_2);
+
+		  for (mcnt = 1; mcnt < num_regs; mcnt++)
+		    {
+		      regstart[mcnt] = best_regstart[mcnt];
+		      regend[mcnt] = best_regend[mcnt];
+		    }
+                }
+            } /* d != end_match_2 */
+
+	succeed_label:
+          DEBUG_PRINT1 ("Accepting match.\n");
+
+          /* If caller wants register contents data back, do it.  */
+          if (regs && !bufp->no_sub)
+	    {
+              /* Have the register data arrays been allocated?  */
+              if (bufp->regs_allocated == REGS_UNALLOCATED)
+                { /* No.  So allocate them with malloc.  We need one
+                     extra element beyond `num_regs' for the `-1' marker
+                     GNU code uses.  */
+                  regs->num_regs = MAX (RE_NREGS, num_regs + 1);
+                  regs->start = TALLOC (regs->num_regs, regoff_t);
+                  regs->end = TALLOC (regs->num_regs, regoff_t);
+                  if (regs->start == NULL || regs->end == NULL)
+		    {
+		      FREE_VARIABLES ();
+		      return -2;
+		    }
+                  bufp->regs_allocated = REGS_REALLOCATE;
+                }
+              else if (bufp->regs_allocated == REGS_REALLOCATE)
+                { /* Yes.  If we need more elements than were already
+                     allocated, reallocate them.  If we need fewer, just
+                     leave it alone.  */
+                  if (regs->num_regs < num_regs + 1)
+                    {
+                      regs->num_regs = num_regs + 1;
+                      RETALLOC (regs->start, regs->num_regs, regoff_t);
+                      RETALLOC (regs->end, regs->num_regs, regoff_t);
+                      if (regs->start == NULL || regs->end == NULL)
+			{
+			  FREE_VARIABLES ();
+			  return -2;
+			}
+                    }
+                }
+              else
+		{
+		  /* These braces fend off a "empty body in an else-statement"
+		     warning under GCC when assert expands to nothing.  */
+		  assert (bufp->regs_allocated == REGS_FIXED);
+		}
+
+              /* Convert the pointer data in `regstart' and `regend' to
+                 indices.  Register zero has to be set differently,
+                 since we haven't kept track of any info for it.  */
+              if (regs->num_regs > 0)
+                {
+                  regs->start[0] = pos;
+                  regs->end[0] = (MATCHING_IN_FIRST_STRING
+				  ? ((regoff_t) (d - string1))
+			          : ((regoff_t) (d - string2 + size1)));
+                }
+              
+              /* Go through the first `min (num_regs, regs->num_regs)'
+                 registers, since that is all we initialized.  */
+	      for (mcnt = 1; mcnt < MIN (num_regs, regs->num_regs); mcnt++)
+		{
+                  if (REG_UNSET (regstart[mcnt]) || REG_UNSET (regend[mcnt]))
+                    regs->start[mcnt] = regs->end[mcnt] = -1;
+                  else
+                    {
+		      regs->start[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regstart[mcnt]);
+                      regs->end[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regend[mcnt]);
+                    }
+		}
+              
+              /* If the regs structure we return has more elements than
+                 were in the pattern, set the extra elements to -1.  If
+                 we (re)allocated the registers, this is the case,
+                 because we always allocate enough to have at least one
+                 -1 at the end.  */
+              for (mcnt = num_regs; mcnt < regs->num_regs; mcnt++)
+                regs->start[mcnt] = regs->end[mcnt] = -1;
+	    } /* regs && !bufp->no_sub */
+
+          DEBUG_PRINT4 ("%u failure points pushed, %u popped (%u remain).\n",
+                        nfailure_points_pushed, nfailure_points_popped,
+                        nfailure_points_pushed - nfailure_points_popped);
+          DEBUG_PRINT2 ("%u registers pushed.\n", num_regs_pushed);
+
+          mcnt = d - pos - (MATCHING_IN_FIRST_STRING 
+			    ? string1 
+			    : string2 - size1);
+
+          DEBUG_PRINT2 ("Returning %d from re_match_2.\n", mcnt);
+
+          FREE_VARIABLES ();
+          return mcnt;
+        }
+
+      /* Otherwise match next pattern command.  */
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+        /* Ignore these.  Used to ignore the n of succeed_n's which
+           currently have n == 0.  */
+        case no_op:
+          DEBUG_PRINT1 ("EXECUTING no_op.\n");
+          break;
+
+	case succeed:
+          DEBUG_PRINT1 ("EXECUTING succeed.\n");
+	  goto succeed_label;
+
+        /* Match the next n pattern characters exactly.  The following
+           byte in the pattern defines n, and the n bytes after that
+           are the characters to match.  */
+	case exactn:
+	  mcnt = *p++;
+          DEBUG_PRINT2 ("EXECUTING exactn %d.\n", mcnt);
+
+          /* This is written out as an if-else so we don't waste time
+             testing `translate' inside the loop.  */
+          if (translate)
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (translate[(unsigned char) *d++] != (char) *p++)
+                    goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  else
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (*d++ != (char) *p++) goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  SET_REGS_MATCHED ();
+          break;
+
+
+        /* Match any character except possibly a newline or a null.  */
+	case anychar:
+          DEBUG_PRINT1 ("EXECUTING anychar.\n");
+
+          PREFETCH ();
+
+          if ((!(bufp->syntax & RE_DOT_NEWLINE) && TRANSLATE (*d) == '\n')
+              || (bufp->syntax & RE_DOT_NOT_NULL && TRANSLATE (*d) == '\000'))
+	    goto fail;
+
+          SET_REGS_MATCHED ();
+          DEBUG_PRINT2 ("  Matched `%d'.\n", *d);
+          d++;
+	  break;
+
+
+	case charset:
+	case charset_not:
+	  {
+	    register unsigned char c;
+	    boolean not = (re_opcode_t) *(p - 1) == charset_not;
+
+            DEBUG_PRINT2 ("EXECUTING charset%s.\n", not ? "_not" : "");
+
+	    PREFETCH ();
+	    c = TRANSLATE (*d); /* The character to match.  */
+
+            /* Cast to `unsigned' instead of `unsigned char' in case the
+               bit list is a full 32 bytes long.  */
+	    if (c < (unsigned) (*p * BYTEWIDTH)
+		&& p[1 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+	      not = !not;
+
+	    p += 1 + *p;
+
+	    if (!not) goto fail;
+            
+	    SET_REGS_MATCHED ();
+            d++;
+	    break;
+	  }
+
+
+        /* The beginning of a group is represented by start_memory.
+           The arguments are the register number in the next byte, and the
+           number of groups inner to this one in the next.  The text
+           matched within the group is recorded (in the internal
+           registers data structure) under the register number.  */
+        case start_memory:
+	  DEBUG_PRINT3 ("EXECUTING start_memory %d (%d):\n", *p, p[1]);
+
+          /* Find out if this group can match the empty string.  */
+	  p1 = p;		/* To send to group_match_null_string_p.  */
+          
+          if (REG_MATCH_NULL_STRING_P (reg_info[*p]) == MATCH_NULL_UNSET_VALUE)
+            REG_MATCH_NULL_STRING_P (reg_info[*p]) 
+              = group_match_null_string_p (&p1, pend, reg_info);
+
+          /* Save the position in the string where we were the last time
+             we were at this open-group operator in case the group is
+             operated upon by a repetition operator, e.g., with `(a*)*b'
+             against `ab'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regstart[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                             ? REG_UNSET (regstart[*p]) ? d : regstart[*p]
+                             : regstart[*p];
+	  DEBUG_PRINT2 ("  old_regstart: %d\n", 
+			 POINTER_TO_OFFSET (old_regstart[*p]));
+
+          regstart[*p] = d;
+	  DEBUG_PRINT2 ("  regstart: %d\n", POINTER_TO_OFFSET (regstart[*p]));
+
+          IS_ACTIVE (reg_info[*p]) = 1;
+          MATCHED_SOMETHING (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+          
+          /* This is the new highest active register.  */
+          highest_active_reg = *p;
+          
+          /* If nothing was active before, this is the new lowest active
+             register.  */
+          if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+            lowest_active_reg = *p;
+
+          /* Move past the register number and inner group count.  */
+          p += 2;
+	  just_past_start_mem = p;
+
+          break;
+
+
+        /* The stop_memory opcode represents the end of a group.  Its
+           arguments are the same as start_memory's: the register
+           number, and the number of inner groups.  */
+	case stop_memory:
+	  DEBUG_PRINT3 ("EXECUTING stop_memory %d (%d):\n", *p, p[1]);
+             
+          /* We need to save the string position the last time we were at
+             this close-group operator in case the group is operated
+             upon by a repetition operator, e.g., with `((a*)*(b*)*)*'
+             against `aba'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regend[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                           ? REG_UNSET (regend[*p]) ? d : regend[*p]
+			   : regend[*p];
+	  DEBUG_PRINT2 ("      old_regend: %d\n", 
+			 POINTER_TO_OFFSET (old_regend[*p]));
+
+          regend[*p] = d;
+	  DEBUG_PRINT2 ("      regend: %d\n", POINTER_TO_OFFSET (regend[*p]));
+
+          /* This register isn't active anymore.  */
+          IS_ACTIVE (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+
+          /* If this was the only register active, nothing is active
+             anymore.  */
+          if (lowest_active_reg == highest_active_reg)
+            {
+              lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+              highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+            }
+          else
+            { /* We must scan for the new highest active register, since
+                 it isn't necessarily one less than now: consider
+                 (a(b)c(d(e)f)g).  When group 3 ends, after the f), the
+                 new highest active register is 1.  */
+              unsigned char r = *p - 1;
+              while (r > 0 && !IS_ACTIVE (reg_info[r]))
+                r--;
+              
+              /* If we end up at register zero, that means that we saved
+                 the registers as the result of an `on_failure_jump', not
+                 a `start_memory', and we jumped to past the innermost
+                 `stop_memory'.  For example, in ((.)*) we save
+                 registers 1 and 2 as a result of the *, but when we pop
+                 back to the second ), we are at the stop_memory 1.
+                 Thus, nothing is active.  */
+	      if (r == 0)
+                {
+                  lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+                  highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+                }
+              else
+                highest_active_reg = r;
+            }
+          
+          /* If just failed to match something this time around with a
+             group that's operated on by a repetition operator, try to
+             force exit from the ``loop'', and restore the register
+             information for this group that we had before trying this
+             last match.  */
+          if ((!MATCHED_SOMETHING (reg_info[*p])
+               || just_past_start_mem == p - 1)
+	      && (p + 2) < pend)              
+            {
+              boolean is_a_jump_n = false;
+              
+              p1 = p + 2;
+              mcnt = 0;
+              switch ((re_opcode_t) *p1++)
+                {
+                  case jump_n:
+		    is_a_jump_n = true;
+                  case pop_failure_jump:
+		  case maybe_pop_jump:
+		  case jump:
+		  case dummy_failure_jump:
+                    EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+		    if (is_a_jump_n)
+		      p1 += 2;
+                    break;
+                  
+                  default:
+                    /* do nothing */ ;
+                }
+	      p1 += mcnt;
+        
+              /* If the next operation is a jump backwards in the pattern
+	         to an on_failure_jump right before the start_memory
+                 corresponding to this stop_memory, exit from the loop
+                 by forcing a failure after pushing on the stack the
+                 on_failure_jump's jump in the pattern, and d.  */
+              if (mcnt < 0 && (re_opcode_t) *p1 == on_failure_jump
+                  && (re_opcode_t) p1[3] == start_memory && p1[4] == *p)
+		{
+                  /* If this group ever matched anything, then restore
+                     what its registers were before trying this last
+                     failed match, e.g., with `(a*)*b' against `ab' for
+                     regstart[1], and, e.g., with `((a*)*(b*)*)*'
+                     against `aba' for regend[3].
+                     
+                     Also restore the registers for inner groups for,
+                     e.g., `((a*)(b*))*' against `aba' (register 3 would
+                     otherwise get trashed).  */
+                     
+                  if (EVER_MATCHED_SOMETHING (reg_info[*p]))
+		    {
+		      unsigned r; 
+        
+                      EVER_MATCHED_SOMETHING (reg_info[*p]) = 0;
+                      
+		      /* Restore this and inner groups' (if any) registers.  */
+                      for (r = *p; r < *p + *(p + 1); r++)
+                        {
+                          regstart[r] = old_regstart[r];
+
+                          /* xx why this test?  */
+                          if (old_regend[r] >= regstart[r])
+                            regend[r] = old_regend[r];
+                        }     
+                    }
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  PUSH_FAILURE_POINT (p1 + mcnt, d, -2);
+
+                  goto fail;
+                }
+            }
+          
+          /* Move past the register number and the inner group count.  */
+          p += 2;
+          break;
+
+
+	/* \<digit> has been turned into a `duplicate' command which is
+           followed by the numeric value of <digit> as the register number.  */
+        case duplicate:
+	  {
+	    register const char *d2, *dend2;
+	    int regno = *p++;   /* Get which register to match against.  */
+	    DEBUG_PRINT2 ("EXECUTING duplicate %d.\n", regno);
+
+	    /* Can't back reference a group which we've never matched.  */
+            if (REG_UNSET (regstart[regno]) || REG_UNSET (regend[regno]))
+              goto fail;
+              
+            /* Where in input to try to start matching.  */
+            d2 = regstart[regno];
+            
+            /* Where to stop matching; if both the place to start and
+               the place to stop matching are in the same string, then
+               set to the place to stop, otherwise, for now have to use
+               the end of the first string.  */
+
+            dend2 = ((FIRST_STRING_P (regstart[regno]) 
+		      == FIRST_STRING_P (regend[regno]))
+		     ? regend[regno] : end_match_1);
+	    for (;;)
+	      {
+		/* If necessary, advance to next segment in register
+                   contents.  */
+		while (d2 == dend2)
+		  {
+		    if (dend2 == end_match_2) break;
+		    if (dend2 == regend[regno]) break;
+
+                    /* End of string1 => advance to string2. */
+                    d2 = string2;
+                    dend2 = regend[regno];
+		  }
+		/* At end of register contents => success */
+		if (d2 == dend2) break;
+
+		/* If necessary, advance to next segment in data.  */
+		PREFETCH ();
+
+		/* How many characters left in this segment to match.  */
+		mcnt = dend - d;
+                
+		/* Want how many consecutive characters we can match in
+                   one shot, so, if necessary, adjust the count.  */
+                if (mcnt > dend2 - d2)
+		  mcnt = dend2 - d2;
+                  
+		/* Compare that many; failure if mismatch, else move
+                   past them.  */
+		if (translate 
+                    ? bcmp_translate (d, d2, mcnt, translate) 
+                    : bcmp (d, d2, mcnt))
+		  goto fail;
+		d += mcnt, d2 += mcnt;
+
+		/* Do this because we've match some characters.  */
+		SET_REGS_MATCHED ();
+	      }
+	  }
+	  break;
+
+
+        /* begline matches the empty string at the beginning of the string
+           (unless `not_bol' is set in `bufp'), and, if
+           `newline_anchor' is set, after newlines.  */
+	case begline:
+          DEBUG_PRINT1 ("EXECUTING begline.\n");
+          
+          if (AT_STRINGS_BEG (d))
+            {
+              if (!bufp->not_bol) break;
+            }
+          else if (d[-1] == '\n' && bufp->newline_anchor)
+            {
+              break;
+            }
+          /* In all other cases, we fail.  */
+          goto fail;
+
+
+        /* endline is the dual of begline.  */
+	case endline:
+          DEBUG_PRINT1 ("EXECUTING endline.\n");
+
+          if (AT_STRINGS_END (d))
+            {
+              if (!bufp->not_eol) break;
+            }
+          
+          /* We have to ``prefetch'' the next character.  */
+          else if ((d == end1 ? *string2 : *d) == '\n'
+                   && bufp->newline_anchor)
+            {
+              break;
+            }
+          goto fail;
+
+
+	/* Match at the very beginning of the data.  */
+        case begbuf:
+          DEBUG_PRINT1 ("EXECUTING begbuf.\n");
+          if (AT_STRINGS_BEG (d))
+            break;
+          goto fail;
+
+
+	/* Match at the very end of the data.  */
+        case endbuf:
+          DEBUG_PRINT1 ("EXECUTING endbuf.\n");
+	  if (AT_STRINGS_END (d))
+	    break;
+          goto fail;
+
+
+        /* on_failure_keep_string_jump is used to optimize `.*\n'.  It
+           pushes NULL as the value for the string on the stack.  Then
+           `pop_failure_point' will keep the current value for the
+           string, instead of restoring it.  To see why, consider
+           matching `foo\nbar' against `.*\n'.  The .* matches the foo;
+           then the . fails against the \n.  But the next thing we want
+           to do is match the \n against the \n; if we restored the
+           string value, we would be back at the foo.
+           
+           Because this is used only in specific cases, we don't need to
+           check all the things that `on_failure_jump' does, to make
+           sure the right things get saved on the stack.  Hence we don't
+           share its code.  The only reason to push anything on the
+           stack at all is that otherwise we would have to change
+           `anychar's code to do something besides goto fail in this
+           case; that seems worse than this.  */
+        case on_failure_keep_string_jump:
+          DEBUG_PRINT1 ("EXECUTING on_failure_keep_string_jump");
+          
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x):\n", mcnt, p + mcnt);
+
+          PUSH_FAILURE_POINT (p + mcnt, NULL, -2);
+          break;
+
+
+	/* Uses of on_failure_jump:
+        
+           Each alternative starts with an on_failure_jump that points
+           to the beginning of the next alternative.  Each alternative
+           except the last ends with a jump that in effect jumps past
+           the rest of the alternatives.  (They really jump to the
+           ending jump of the following alternative, because tensioning
+           these jumps is a hassle.)
+
+           Repeats start with an on_failure_jump that points past both
+           the repetition text and either the following jump or
+           pop_failure_jump back to this on_failure_jump.  */
+	case on_failure_jump:
+        on_failure:
+          DEBUG_PRINT1 ("EXECUTING on_failure_jump");
+
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x)", mcnt, p + mcnt);
+
+          /* If this on_failure_jump comes right before a group (i.e.,
+             the original * applied to a group), save the information
+             for that group and all inner ones, so that if we fail back
+             to this point, the group's information will be correct.
+             For example, in \(a*\)*\1, we need the preceding group,
+             and in \(\(a*\)b*\)\2, we need the inner group.  */
+
+          /* We can't use `p' to check ahead because we push
+             a failure point to `p + mcnt' after we do this.  */
+          p1 = p;
+
+          /* We need to skip no_op's before we look for the
+             start_memory in case this on_failure_jump is happening as
+             the result of a completed succeed_n, as in \(a\)\{1,3\}b\1
+             against aba.  */
+          while (p1 < pend && (re_opcode_t) *p1 == no_op)
+            p1++;
+
+          if (p1 < pend && (re_opcode_t) *p1 == start_memory)
+            {
+              /* We have a new highest active register now.  This will
+                 get reset at the start_memory we are about to get to,
+                 but we will have saved all the registers relevant to
+                 this repetition op, as described above.  */
+              highest_active_reg = *(p1 + 1) + *(p1 + 2);
+              if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+                lowest_active_reg = *(p1 + 1);
+            }
+
+          DEBUG_PRINT1 (":\n");
+          PUSH_FAILURE_POINT (p + mcnt, d, -2);
+          break;
+
+
+        /* A smart repeat ends with `maybe_pop_jump'.
+	   We change it to either `pop_failure_jump' or `jump'.  */
+        case maybe_pop_jump:
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT2 ("EXECUTING maybe_pop_jump %d.\n", mcnt);
+          {
+	    register unsigned char *p2 = p;
+
+            /* Compare the beginning of the repeat with what in the
+               pattern follows its end. If we can establish that there
+               is nothing that they would both match, i.e., that we
+               would have to backtrack because of (as in, e.g., `a*a')
+               then we can change to pop_failure_jump, because we'll
+               never have to backtrack.
+               
+               This is not true in the case of alternatives: in
+               `(a|ab)*' we do need to backtrack to the `ab' alternative
+               (e.g., if the string was `ab').  But instead of trying to
+               detect that here, the alternative has put on a dummy
+               failure point which is what we will end up popping.  */
+
+	    /* Skip over open/close-group commands.
+	       If what follows this loop is a ...+ construct,
+	       look at what begins its body, since we will have to
+	       match at least one of that.  */
+	    while (1)
+	      {
+		if (p2 + 2 < pend
+		    && ((re_opcode_t) *p2 == stop_memory
+			|| (re_opcode_t) *p2 == start_memory))
+		  p2 += 3;
+		else if (p2 + 6 < pend
+			 && (re_opcode_t) *p2 == dummy_failure_jump)
+		  p2 += 6;
+		else
+		  break;
+	      }
+
+	    p1 = p + mcnt;
+	    /* p1[0] ... p1[2] are the `on_failure_jump' corresponding
+	       to the `maybe_finalize_jump' of this case.  Examine what 
+	       follows.  */
+
+            /* If we're at the end of the pattern, we can change.  */
+            if (p2 == pend)
+	      {
+		/* Consider what happens when matching ":\(.*\)"
+		   against ":/".  I don't really understand this code
+		   yet.  */
+  	        p[-3] = (unsigned char) pop_failure_jump;
+                DEBUG_PRINT1
+                  ("  End of pattern: change to `pop_failure_jump'.\n");
+              }
+
+            else if ((re_opcode_t) *p2 == exactn
+		     || (bufp->newline_anchor && (re_opcode_t) *p2 == endline))
+	      {
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+
+                if ((re_opcode_t) p1[3] == exactn && p1[5] != c)
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset
+			 || (re_opcode_t) p1[3] == charset_not)
+		  {
+		    int not = (re_opcode_t) p1[3] == charset_not;
+                    
+		    if (c < (unsigned char) (p1[4] * BYTEWIDTH)
+			&& p1[5 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+		      not = !not;
+
+                    /* `not' is equal to 1 if c would match, which means
+                        that we can't change to pop_failure_jump.  */
+		    if (!not)
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+            else if ((re_opcode_t) *p2 == charset)
+	      {
+#ifdef DEBUG
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+#endif
+
+                if ((re_opcode_t) p1[3] == exactn
+		    && ! ((int) p2[1] * BYTEWIDTH > (int) p1[4]
+			  && (p2[1 + p1[4] / BYTEWIDTH]
+			      & (1 << (p1[4] % BYTEWIDTH)))))
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset_not)
+		  {
+		    int idx;
+		    /* We win if the charset_not inside the loop
+		       lists every character listed in the charset after.  */
+		    for (idx = 0; idx < (int) p2[1]; idx++)
+		      if (! (p2[2 + idx] == 0
+			     || (idx < (int) p1[4]
+				 && ((p2[2 + idx] & ~ p1[5 + idx]) == 0))))
+			break;
+
+		    if (idx == p2[1])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+		else if ((re_opcode_t) p1[3] == charset)
+		  {
+		    int idx;
+		    /* We win if the charset inside the loop
+		       has no overlap with the one after the loop.  */
+		    for (idx = 0;
+			 idx < (int) p2[1] && idx < (int) p1[4];
+			 idx++)
+		      if ((p2[2 + idx] & p1[5 + idx]) != 0)
+			break;
+
+		    if (idx == p2[1] || idx == p1[4])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+	  }
+	  p -= 2;		/* Point at relative address again.  */
+	  if ((re_opcode_t) p[-1] != pop_failure_jump)
+	    {
+	      p[-1] = (unsigned char) jump;
+              DEBUG_PRINT1 ("  Match => jump.\n");
+	      goto unconditional_jump;
+	    }
+        /* Note fall through.  */
+
+
+	/* The end of a simple repeat has a pop_failure_jump back to
+           its matching on_failure_jump, where the latter will push a
+           failure point.  The pop_failure_jump takes off failure
+           points put on by this pop_failure_jump's matching
+           on_failure_jump; we got through the pattern to here from the
+           matching on_failure_jump, so didn't fail.  */
+        case pop_failure_jump:
+          {
+            /* We need to pass separate storage for the lowest and
+               highest registers, even though we don't care about the
+               actual values.  Otherwise, we will restore only one
+               register from the stack, since lowest will == highest in
+               `pop_failure_point'.  */
+            unsigned dummy_low_reg, dummy_high_reg;
+            unsigned char *pdummy;
+            const char *sdummy;
+
+            DEBUG_PRINT1 ("EXECUTING pop_failure_jump.\n");
+            POP_FAILURE_POINT (sdummy, pdummy,
+                               dummy_low_reg, dummy_high_reg,
+                               reg_dummy, reg_dummy, reg_info_dummy);
+          }
+          /* Note fall through.  */
+
+          
+        /* Unconditionally jump (without popping any failure points).  */
+        case jump:
+	unconditional_jump:
+	  EXTRACT_NUMBER_AND_INCR (mcnt, p);	/* Get the amount to jump.  */
+          DEBUG_PRINT2 ("EXECUTING jump %d ", mcnt);
+	  p += mcnt;				/* Do the jump.  */
+          DEBUG_PRINT2 ("(to 0x%x).\n", p);
+	  break;
+
+	
+        /* We need this opcode so we can detect where alternatives end
+           in `group_match_null_string_p' et al.  */
+        case jump_past_alt:
+          DEBUG_PRINT1 ("EXECUTING jump_past_alt.\n");
+          goto unconditional_jump;
+
+
+        /* Normally, the on_failure_jump pushes a failure point, which
+           then gets popped at pop_failure_jump.  We will end up at
+           pop_failure_jump, also, and with a pattern of, say, `a+', we
+           are skipping over the on_failure_jump, so we have to push
+           something meaningless for pop_failure_jump to pop.  */
+        case dummy_failure_jump:
+          DEBUG_PRINT1 ("EXECUTING dummy_failure_jump.\n");
+          /* It doesn't matter what we push for the string here.  What
+             the code at `fail' tests is the value for the pattern.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          goto unconditional_jump;
+
+
+        /* At the end of an alternative, we need to push a dummy failure
+           point in case we are followed by a `pop_failure_jump', because
+           we don't want the failure point for the alternative to be
+           popped.  For example, matching `(a|ab)*' against `aab'
+           requires that we match the `ab' alternative.  */
+        case push_dummy_failure:
+          DEBUG_PRINT1 ("EXECUTING push_dummy_failure.\n");
+          /* See comments just above at `dummy_failure_jump' about the
+             two zeroes.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          break;
+
+        /* Have to succeed matching what follows at least n times.
+           After that, handle like `on_failure_jump'.  */
+        case succeed_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING succeed_n %d.\n", mcnt);
+
+          assert (mcnt >= 0);
+          /* Originally, this is how many times we HAVE to succeed.  */
+          if (mcnt > 0)
+            {
+               mcnt--;
+	       p += 2;
+               STORE_NUMBER_AND_INCR (p, mcnt);
+               DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p, mcnt);
+            }
+	  else if (mcnt == 0)
+            {
+              DEBUG_PRINT2 ("  Setting two bytes from 0x%x to no_op.\n", p+2);
+	      p[2] = (unsigned char) no_op;
+              p[3] = (unsigned char) no_op;
+              goto on_failure;
+            }
+          break;
+        
+        case jump_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING jump_n %d.\n", mcnt);
+
+          /* Originally, this is how many times we CAN jump.  */
+          if (mcnt)
+            {
+               mcnt--;
+               STORE_NUMBER (p + 2, mcnt);
+	       goto unconditional_jump;	     
+            }
+          /* If don't have to jump any more, skip over the rest of command.  */
+	  else      
+	    p += 4;		     
+          break;
+        
+	case set_number_at:
+	  {
+            DEBUG_PRINT1 ("EXECUTING set_number_at.\n");
+
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            p1 = p + mcnt;
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p1, mcnt);
+	    STORE_NUMBER (p1, mcnt);
+            break;
+          }
+
+        case wordbound:
+          DEBUG_PRINT1 ("EXECUTING wordbound.\n");
+          if (AT_WORD_BOUNDARY (d))
+	    break;
+          goto fail;
+
+	case notwordbound:
+          DEBUG_PRINT1 ("EXECUTING notwordbound.\n");
+	  if (AT_WORD_BOUNDARY (d))
+	    goto fail;
+          break;
+
+	case wordbeg:
+          DEBUG_PRINT1 ("EXECUTING wordbeg.\n");
+	  if (WORDCHAR_P (d) && (AT_STRINGS_BEG (d) || !WORDCHAR_P (d - 1)))
+	    break;
+          goto fail;
+
+	case wordend:
+          DEBUG_PRINT1 ("EXECUTING wordend.\n");
+	  if (!AT_STRINGS_BEG (d) && WORDCHAR_P (d - 1)
+              && (!WORDCHAR_P (d) || AT_STRINGS_END (d)))
+	    break;
+          goto fail;
+
+#ifdef emacs
+  	case before_dot:
+          DEBUG_PRINT1 ("EXECUTING before_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) >= point)
+  	    goto fail;
+  	  break;
+  
+  	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) != point)
+  	    goto fail;
+  	  break;
+  
+  	case after_dot:
+          DEBUG_PRINT1 ("EXECUTING after_dot.\n");
+          if (PTR_CHAR_POS ((unsigned char *) d) <= point)
+  	    goto fail;
+  	  break;
+#if 0 /* not emacs19 */
+	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+	  if (PTR_CHAR_POS ((unsigned char *) d) + 1 != point)
+	    goto fail;
+	  break;
+#endif /* not emacs19 */
+
+	case syntaxspec:
+          DEBUG_PRINT2 ("EXECUTING syntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchsyntax;
+
+        case wordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs wordchar.\n");
+	  mcnt = (int) Sword;
+        matchsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) != (enum syntaxcode) mcnt)
+	    goto fail;
+          SET_REGS_MATCHED ();
+	  break;
+
+	case notsyntaxspec:
+          DEBUG_PRINT2 ("EXECUTING notsyntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchnotsyntax;
+
+        case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs notwordchar.\n");
+	  mcnt = (int) Sword;
+        matchnotsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) == (enum syntaxcode) mcnt)
+	    goto fail;
+	  SET_REGS_MATCHED ();
+          break;
+
+#else /* not emacs */
+	case wordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs wordchar.\n");
+	  PREFETCH ();
+          if (!WORDCHAR_P (d))
+            goto fail;
+	  SET_REGS_MATCHED ();
+          d++;
+	  break;
+	  
+	case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs notwordchar.\n");
+	  PREFETCH ();
+	  if (WORDCHAR_P (d))
+            goto fail;
+          SET_REGS_MATCHED ();
+          d++;
+	  break;
+#endif /* not emacs */
+          
+        default:
+          abort ();
+	}
+      continue;  /* Successfully executed one pattern command; keep going.  */
+
+
+    /* We goto here if a matching operation fails. */
+    fail:
+      if (!FAIL_STACK_EMPTY ())
+	{ /* A restart point is known.  Restore to that state.  */
+          DEBUG_PRINT1 ("\nFAIL:\n");
+          POP_FAILURE_POINT (d, p,
+                             lowest_active_reg, highest_active_reg,
+                             regstart, regend, reg_info);
+
+          /* If this failure point is a dummy, try the next one.  */
+          if (!p)
+	    goto fail;
+
+          /* If we failed to the end of the pattern, don't examine *p.  */
+	  assert (p <= pend);
+          if (p < pend)
+            {
+              boolean is_a_jump_n = false;
+              
+              /* If failed to a backwards jump that's part of a repetition
+                 loop, need to pop this failure point and use the next one.  */
+              switch ((re_opcode_t) *p)
+                {
+                case jump_n:
+                  is_a_jump_n = true;
+                case maybe_pop_jump:
+                case pop_failure_jump:
+                case jump:
+                  p1 = p + 1;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  p1 += mcnt;	
+
+                  if ((is_a_jump_n && (re_opcode_t) *p1 == succeed_n)
+                      || (!is_a_jump_n
+                          && (re_opcode_t) *p1 == on_failure_jump))
+                    goto fail;
+                  break;
+                default:
+                  /* do nothing */ ;
+                }
+            }
+
+          if (d >= string1 && d <= end1)
+	    dend = end_match_1;
+        }
+      else
+        break;   /* Matching at this starting point really fails.  */
+    } /* for (;;) */
+
+  if (best_regs_set)
+    goto restore_best_regs;
+
+  FREE_VARIABLES ();
+
+  return -1;         			/* Failure to match.  */
+} /* re_match_2 */
+
+
+/* Subroutine definitions for re_match_2.  */
+
+
+/* We are passed P pointing to a register number after a start_memory.
+   
+   Return true if the pattern up to the corresponding stop_memory can
+   match the empty string, and false otherwise.
+   
+   If we find the matching stop_memory, sets P to point to one past its number.
+   Otherwise, sets P to an undefined byte less than or equal to END.
+
+   We don't handle duplicates properly (yet).  */
+
+static boolean
+group_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  /* Point to after the args to the start_memory.  */
+  unsigned char *p1 = *p + 2;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and return true or
+	 false, as appropriate, when we get to one that can't, or to the
+         matching stop_memory.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+        /* Could be either a loop or a series of alternatives.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          
+          /* If the next operation is not a jump backwards in the
+	     pattern.  */
+
+	  if (mcnt >= 0)
+	    {
+              /* Go through the on_failure_jumps of the alternatives,
+                 seeing if any of the alternatives cannot match nothing.
+                 The last alternative starts with only a jump,
+                 whereas the rest start with on_failure_jump and end
+                 with a jump, e.g., here is the pattern for `a|b|c':
+
+                 /on_failure_jump/0/6/exactn/1/a/jump_past_alt/0/6
+                 /on_failure_jump/0/6/exactn/1/b/jump_past_alt/0/3
+                 /exactn/1/c						
+
+                 So, we have to first go through the first (n-1)
+                 alternatives and then deal with the last one separately.  */
+
+
+              /* Deal with the first (n-1) alternatives, which start
+                 with an on_failure_jump (see above) that jumps to right
+                 past a jump_past_alt.  */
+
+              while ((re_opcode_t) p1[mcnt-3] == jump_past_alt)
+                {
+                  /* `mcnt' holds how many bytes long the alternative
+                     is, including the ending `jump_past_alt' and
+                     its number.  */
+
+                  if (!alt_match_null_string_p (p1, p1 + mcnt - 3, 
+				                      reg_info))
+                    return false;
+
+                  /* Move to right after this alternative, including the
+		     jump_past_alt.  */
+                  p1 += mcnt;	
+
+                  /* Break if it's the beginning of an n-th alternative
+                     that doesn't begin with an on_failure_jump.  */
+                  if ((re_opcode_t) *p1 != on_failure_jump)
+                    break;
+		
+		  /* Still have to check that it's not an n-th
+		     alternative that starts with an on_failure_jump.  */
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  if ((re_opcode_t) p1[mcnt-3] != jump_past_alt)
+                    {
+		      /* Get to the beginning of the n-th alternative.  */
+                      p1 -= 3;
+                      break;
+                    }
+                }
+
+              /* Deal with the last alternative: go back and get number
+                 of the `jump_past_alt' just before it.  `mcnt' contains
+                 the length of the alternative.  */
+              EXTRACT_NUMBER (mcnt, p1 - 2);
+
+              if (!alt_match_null_string_p (p1, p1 + mcnt, reg_info))
+                return false;
+
+              p1 += mcnt;	/* Get past the n-th alternative.  */
+            } /* if mcnt > 0 */
+          break;
+
+          
+        case stop_memory:
+	  assert (p1[1] == **p);
+          *p = p1 + 2;
+          return true;
+
+        
+        default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    } /* while p1 < end */
+
+  return false;
+} /* group_match_null_string_p */
+
+
+/* Similar to group_match_null_string_p, but doesn't deal with alternatives:
+   It expects P to be the first byte of a single alternative and END one
+   byte past the last. The alternative can contain groups.  */
+   
+static boolean
+alt_match_null_string_p (p, end, reg_info)
+    unsigned char *p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  unsigned char *p1 = p;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and break when we get 
+         to one that can't.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+	/* It's a loop.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+          break;
+          
+	default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    }  /* while p1 < end */
+
+  return true;
+} /* alt_match_null_string_p */
+
+
+/* Deals with the ops common to group_match_null_string_p and
+   alt_match_null_string_p.  
+   
+   Sets P to one after the op and its arguments, if any.  */
+
+static boolean
+common_op_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  boolean ret;
+  int reg_no;
+  unsigned char *p1 = *p;
+
+  switch ((re_opcode_t) *p1++)
+    {
+    case no_op:
+    case begline:
+    case endline:
+    case begbuf:
+    case endbuf:
+    case wordbeg:
+    case wordend:
+    case wordbound:
+    case notwordbound:
+#ifdef emacs
+    case before_dot:
+    case at_dot:
+    case after_dot:
+#endif
+      break;
+
+    case start_memory:
+      reg_no = *p1;
+      assert (reg_no > 0 && reg_no <= MAX_REGNUM);
+      ret = group_match_null_string_p (&p1, end, reg_info);
+      
+      /* Have to set this here in case we're checking a group which
+         contains a group and a back reference to it.  */
+
+      if (REG_MATCH_NULL_STRING_P (reg_info[reg_no]) == MATCH_NULL_UNSET_VALUE)
+        REG_MATCH_NULL_STRING_P (reg_info[reg_no]) = ret;
+
+      if (!ret)
+        return false;
+      break;
+          
+    /* If this is an optimized succeed_n for zero times, make the jump.  */
+    case jump:
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+      if (mcnt >= 0)
+        p1 += mcnt;
+      else
+        return false;
+      break;
+
+    case succeed_n:
+      /* Get to the number of times to succeed.  */
+      p1 += 2;		
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+
+      if (mcnt == 0)
+        {
+          p1 -= 4;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+        }
+      else
+        return false;
+      break;
+
+    case duplicate: 
+      if (!REG_MATCH_NULL_STRING_P (reg_info[*p1]))
+        return false;
+      break;
+
+    case set_number_at:
+      p1 += 4;
+
+    default:
+      /* All other opcodes mean we cannot match the empty string.  */
+      return false;
+  }
+
+  *p = p1;
+  return true;
+} /* common_op_match_null_string_p */
+
+
+/* Return zero if TRANSLATE[S1] and TRANSLATE[S2] are identical for LEN
+   bytes; nonzero otherwise.  */
+   
+static int
+bcmp_translate (s1, s2, len, translate)
+     unsigned char *s1, *s2;
+     register int len;
+     char *translate;
+{
+  register unsigned char *p1 = s1, *p2 = s2;
+  while (len)
+    {
+      if (translate[*p1++] != translate[*p2++]) return 1;
+      len--;
+    }
+  return 0;
+}
+
+
+/* Entry points for GNU code.  */
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in BUFP.
+   Returns 0 if the pattern was valid, otherwise an error string.
+   
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in BUFP on entry.
+   
+   We call regex_compile to do the actual compilation.  */
+
+const char *
+re_compile_pattern (pattern, length, bufp)
+     const char *pattern;
+     int length;
+     struct re_pattern_buffer *bufp;
+{
+  reg_errcode_t ret;
+  
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  bufp->regs_allocated = REGS_UNALLOCATED;
+  
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  bufp->no_sub = 0;
+  
+  /* Match anchors at newline.  */
+  bufp->newline_anchor = 1;
+  
+  ret = regex_compile (pattern, length, re_syntax_options, bufp);
+
+  if (!ret)
+    return NULL;
+  return gettext (re_error_msgid[(int) ret]);
+}     
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them unless specifically requested.  */
+
+#ifdef _REGEX_RE_COMP
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer re_comp_buf;
+
+char *
+re_comp (s)
+    const char *s;
+{
+  reg_errcode_t ret;
+  
+  if (!s)
+    {
+      if (!re_comp_buf.buffer)
+	return gettext ("No previous regular expression");
+      return 0;
+    }
+
+  if (!re_comp_buf.buffer)
+    {
+      re_comp_buf.buffer = (unsigned char *) malloc (200);
+      if (re_comp_buf.buffer == NULL)
+        return gettext (re_error_msgid[(int) REG_ESPACE]);
+      re_comp_buf.allocated = 200;
+
+      re_comp_buf.fastmap = (char *) malloc (1 << BYTEWIDTH);
+      if (re_comp_buf.fastmap == NULL)
+	return gettext (re_error_msgid[(int) REG_ESPACE]);
+    }
+
+  /* Since `re_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  re_comp_buf.newline_anchor = 1;
+
+  ret = regex_compile (s, strlen (s), re_syntax_options, &re_comp_buf);
+  
+  if (!ret)
+    return NULL;
+
+  /* Yes, we're discarding `const' here if !HAVE_LIBINTL.  */
+  return (char *) gettext (re_error_msgid[(int) ret]);
+}
+
+
+int
+re_exec (s)
+    const char *s;
+{
+  const int len = strlen (s);
+  return
+    0 <= re_search (&re_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* _REGEX_RE_COMP */
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#ifndef emacs
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+int
+regcomp (preg, pattern, cflags)
+    regex_t *preg;
+    const char *pattern; 
+    int cflags;
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = (cflags & REG_EXTENDED) ?
+      RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->used = 0;
+  
+  /* Don't bother to use a fastmap when searching.  This simplifies the
+     REG_NEWLINE case: if we used a fastmap, we'd have to put all the
+     characters after newlines into the fastmap.  This way, we just try
+     every character.  */
+  preg->fastmap = 0;
+  
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+      
+      preg->translate = (char *) malloc (CHAR_SET_SIZE);
+      if (preg->translate == NULL)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = ISUPPER (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = NULL;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we 
+     can use strlen here in compiling the pattern.  */
+  ret = regex_compile (pattern, strlen (pattern), syntax, preg);
+  
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+  
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+   
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+   
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+   
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    const regex_t *preg;
+    const char *string; 
+    size_t nmatch; 
+    regmatch_t pmatch[]; 
+    int eflags;
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+  
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+  
+  /* The user has told us exactly how many registers to return
+     information about, via `nmatch'.  We have to pass that on to the
+     matching routines.  */
+  private_preg.regs_allocated = REGS_FIXED;
+  
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start = TALLOC (nmatch, regoff_t);
+      regs.end = TALLOC (nmatch, regoff_t);
+      if (regs.start == NULL || regs.end == NULL)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg, string, len,
+                   /* start: */ 0, /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+  
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   We don't use PREG here.  */
+
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    const regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+{
+  const char *msg;
+  size_t msg_size;
+
+  if (errcode < 0
+      || errcode >= (sizeof (re_error_msgid) / sizeof (re_error_msgid[0])))
+    /* Only error codes returned by the rest of the code should be passed 
+       to this routine.  If we are given anything else, or if other regex
+       code generates an invalid error code, then the program has a bug.
+       Dump core so we can fix it.  */
+    abort ();
+
+  msg = gettext (re_error_msgid[errcode]);
+
+  msg_size = strlen (msg) + 1; /* Includes the null.  */
+  
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+void
+regfree (preg)
+    regex_t *preg;
+{
+  if (preg->buffer != NULL)
+    free (preg->buffer);
+  preg->buffer = NULL;
+  
+  preg->allocated = 0;
+  preg->used = 0;
+
+  if (preg->fastmap != NULL)
+    free (preg->fastmap);
+  preg->fastmap = NULL;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != NULL)
+    free (preg->translate);
+  preg->translate = NULL;
+}
+
+#endif /* not emacs  */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/regex.h	(revision 21112)
@@ -0,0 +1,497 @@
+/* Definitions for data structures and routines for the regular
+   expression library, version 0.12.
+
+   Copyright (C) 1985, 89, 90, 91, 92, 93, 95 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef __REGEXP_LIBRARY_H__
+#define __REGEXP_LIBRARY_H__
+
+/* POSIX says that <sys/types.h> must be included (by the caller) before
+   <regex.h>.  */
+
+#if !defined (_POSIX_C_SOURCE) && !defined (_POSIX_SOURCE) && defined (VMS)
+/* VMS doesn't have `size_t' in <sys/types.h>, even though POSIX says it
+   should be there.  */
+#include <stddef.h>
+#endif
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+typedef unsigned reg_syntax_t;
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* If this bit is set, succeed as soon as we match the whole pattern,
+   without further backtracking.  */
+#define RE_NO_POSIX_BACKTRACKING (RE_UNMATCHED_RIGHT_PAREN_ORD << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+/* P1003.2/D11.2, section 4.20.7.1, lines 5078ff.  */
+#define RE_SYNTAX_ED RE_SYNTAX_POSIX_BASIC
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+
+/* This data structure represents a compiled pattern.  Before calling
+   the pattern compiler, the fields `buffer', `allocated', `fastmap',
+   `translate', and `no_sub' can be set.  After the pattern has been
+   compiled, the `re_nsub' field is available.  All other fields are
+   private to the regex routines.  */
+
+struct re_pattern_buffer
+{
+/* [[[begin pattern_buffer]]] */
+	/* Space that holds the compiled pattern.  It is declared as
+          `unsigned char *' because its elements are
+           sometimes used as array indexes.  */
+  unsigned char *buffer;
+
+	/* Number of bytes to which `buffer' points.  */
+  unsigned long allocated;
+
+	/* Number of bytes actually used in `buffer'.  */
+  unsigned long used;	
+
+        /* Syntax setting with which the pattern was compiled.  */
+  reg_syntax_t syntax;
+
+        /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+           the fastmap, if there is one, to skip over impossible
+           starting points for matches.  */
+  char *fastmap;
+
+        /* Either a translate table to apply to all characters before
+           comparing them, or zero for no translation.  The translation
+           is applied to a pattern when it is compiled and to a string
+           when it is matched.  */
+  char *translate;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+        /* Zero if this pattern cannot match the empty string, one else.
+           Well, in truth it's used only in `re_search_2', to see
+           whether or not we should use the fastmap, so we don't set
+           this absolutely perfectly; see `re_compile_fastmap' (the
+           `duplicate' case).  */
+  unsigned can_be_null : 1;
+
+        /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+             for `max (RE_NREGS, re_nsub + 1)' groups.
+           If REGS_REALLOCATE, reallocate space if necessary.
+           If REGS_FIXED, use what's there.  */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned regs_allocated : 2;
+
+        /* Set to zero when `regex_compile' compiles a pattern; set to one
+           by `re_compile_fastmap' if it updates the fastmap.  */
+  unsigned fastmap_accurate : 1;
+
+        /* If set, `re_match_2' does not return information about
+           subexpressions.  */
+  unsigned no_sub : 1;
+
+        /* If set, a beginning-of-line anchor doesn't match at the
+           beginning of the string.  */ 
+  unsigned not_bol : 1;
+
+        /* Similarly for an end-of-line anchor.  */
+  unsigned not_eol : 1;
+
+        /* If true, an anchor at a newline matches.  */
+  unsigned newline_anchor : 1;
+
+/* [[[end pattern_buffer]]] */
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+   `re_match_2' returns information about at least this many registers
+   the first time a `regs' structure is passed.  */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+/* Declarations for routines.  */
+
+/* To avoid duplicating every routine declaration -- once with a
+   prototype (if we are ANSI), and once without (if we aren't) -- we
+   use the following macro to declare argument types.  This
+   unfortunately clutters up the declarations a bit, but I think it's
+   worth it.  */
+
+#if __STDC__
+
+#define _RE_ARGS(args) args
+
+#else /* not __STDC__ */
+
+#define _RE_ARGS(args) ()
+
+#endif /* not __STDC__ */
+
+/* Sets the current default syntax to SYNTAX, and return the old syntax.
+   You can also simply assign to the `re_syntax_options' variable.  */
+extern reg_syntax_t re_set_syntax _RE_ARGS ((reg_syntax_t syntax));
+
+/* Compile the regular expression PATTERN, with length LENGTH
+   and syntax given by the global `re_syntax_options', into the buffer
+   BUFFER.  Return NULL if successful, and an error string if not.  */
+extern const char *re_compile_pattern
+  _RE_ARGS ((const char *pattern, int length,
+             struct re_pattern_buffer *buffer));
+
+
+/* Compile a fastmap for the compiled pattern in BUFFER; used to
+   accelerate searches.  Return 0 if successful and -2 if was an
+   internal error.  */
+extern int re_compile_fastmap _RE_ARGS ((struct re_pattern_buffer *buffer));
+
+
+/* Search in the string STRING (with length LENGTH) for the pattern
+   compiled into BUFFER.  Start searching at position START, for RANGE
+   characters.  Return the starting position of the match, -1 for no
+   match, or -2 for an internal error.  Also return register
+   information in REGS (if REGS and BUFFER->no_sub are nonzero).  */
+extern int re_search
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+            int length, int start, int range, struct re_registers *regs));
+
+
+/* Like `re_search', but search in the concatenation of STRING1 and
+   STRING2.  Also, stop searching at index START + STOP.  */
+extern int re_search_2
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, int range, struct re_registers *regs, int stop));
+
+
+/* Like `re_search', but return how many characters in STRING the regexp
+   in BUFFER matched, starting at position START.  */
+extern int re_match
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+             int length, int start, struct re_registers *regs));
+
+
+/* Relates to `re_match' as `re_search_2' relates to `re_search'.  */
+extern int re_match_2 
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, struct re_registers *regs, int stop));
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using BUFFER and REGS will use this memory
+   for recording register information.  STARTS and ENDS must be
+   allocated with malloc, and must each be at least `NUM_REGS * sizeof
+   (regoff_t)' bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+extern void re_set_registers
+  _RE_ARGS ((struct re_pattern_buffer *buffer, struct re_registers *regs,
+             unsigned num_regs, regoff_t *starts, regoff_t *ends));
+
+#ifdef _REGEX_RE_COMP
+/* 4.2 bsd compatibility.  */
+extern char *re_comp _RE_ARGS ((const char *));
+extern int re_exec _RE_ARGS ((const char *));
+#endif
+
+/* POSIX compatibility.  */
+extern int regcomp _RE_ARGS ((regex_t *preg, const char *pattern, int cflags));
+extern int regexec
+  _RE_ARGS ((const regex_t *preg, const char *string, size_t nmatch,
+             regmatch_t pmatch[], int eflags));
+extern size_t regerror
+  _RE_ARGS ((int errcode, const regex_t *preg, char *errbuf,
+             size_t errbuf_size));
+extern void regfree _RE_ARGS ((regex_t *preg));
+
+#endif /* not __REGEXP_LIBRARY_H__ */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.c	(revision 21112)
@@ -0,0 +1,7271 @@
+/*	Copyright (C) 1992, 1993, 1994, 1995 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+
+/* NOTE!!!  AIX is so losing it requires this to be the first thing in the 
+ * file. 
+ * Do not put ANYTHING before it!  
+ */
+#if !defined (__GNUC__) && defined (_AIX)
+ #pragma alloca
+#endif
+
+/* To make linux happy? */
+#ifndef	_GNU_SOURCE
+#define	_GNU_SOURCE
+#endif
+
+#if HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+#  include <malloc.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+
+
+char rx_version_string[] = "GNU Rx version 0.07.1";
+
+			/* ``Too hard!''
+			 *	    -- anon.
+			 */
+
+
+
+#include <stdio.h>
+#include <ctype.h>
+#ifndef isgraph
+#define isgraph(c) (isprint (c) && !isspace (c))
+#endif
+#ifndef isblank
+#define isblank(c) ((c) == ' ' || (c) == '\t')
+#endif
+
+#include <sys/types.h>
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+#ifndef __GCC__
+#undef __inline__
+#define __inline__
+#endif
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+/* Memory management and stuff for emacs. */
+
+#define CHARBITS 8
+#define remalloc(M, S) (M ? realloc (M, S) : malloc (S))
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+ * use `alloca' instead of `malloc' for the backtracking stack.
+ *
+ * Emacs will die miserably if we don't do this.
+ */
+
+#ifdef REGEX_MALLOC
+#define REGEX_ALLOCATE malloc
+#else /* not REGEX_MALLOC  */
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#define REGEX_ALLOCATE _alloca
+#else
+#define REGEX_ALLOCATE alloca
+#endif /* __WIN32__ */
+#endif /* not REGEX_MALLOC */
+
+
+#ifdef RX_WANT_RX_DEFS
+#define RX_DECL extern
+#define RX_DEF_QUAL 
+#else
+#define RX_WANT_RX_DEFS
+#define RX_DECL static
+#define RX_DEF_QUAL static
+#endif
+#include "rx.h"
+#undef RX_DECL
+#define RX_DECL RX_DEF_QUAL
+
+
+#ifndef emacs
+
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else /* not SYNTAX_TABLE */
+
+#ifdef __STDC__
+static void
+init_syntax_once (void)
+#else
+static void
+init_syntax_once ()
+#endif
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+#endif /* not SYNTAX_TABLE */
+#endif /* not emacs */
+
+
+/* Compile with `-DRX_DEBUG' and use the following flags.
+ *
+ * Debugging flags:
+ *   	rx_debug - print information as a regexp is compiled
+ * 	rx_debug_trace - print information as a regexp is executed
+ */
+
+#ifdef RX_DEBUG
+
+int rx_debug_compile = 0;
+int rx_debug_trace = 0;
+static struct re_pattern_buffer * dbug_rxb = 0;
+
+#ifdef __STDC__
+typedef void (*side_effect_printer) (struct rx *, void *, FILE *);
+#else
+typedef void (*side_effect_printer) ();
+#endif
+
+#ifdef __STDC__
+static void print_cset (struct rx *rx, rx_Bitset cset, FILE * fp);
+#else
+static void print_cset ();
+#endif
+
+#ifdef __STDC__
+static void
+print_rexp (struct rx *rx,
+	    struct rexp_node *node, int depth,
+	    side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_rexp (rx, node, depth, seprint, fp)
+     struct rx *rx;
+     struct rexp_node *node;
+     int depth;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  if (!node)
+    return;
+  else
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  {
+	    fprintf (fp, "%*s", depth, "");
+	    print_cset (rx, node->params.cset, fp);
+	    fputc ('\n', fp);
+	    break;
+	  }
+
+ 	case r_opt:
+	case r_star:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_opt ? "opt" : "star");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+	case r_2phase_star:
+	  fprintf (fp, "%*s2phase star\n", depth, "");
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+
+	case r_alternate:
+	case r_concat:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_alternate ? "alt" : "concat");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  break;
+	case r_side_effect:
+	  fprintf (fp, "%*sSide effect: ", depth, "");
+	  seprint (rx, node->params.side_effect, fp);
+	  fputc ('\n', fp);
+	}
+    }
+}
+
+#ifdef __STDC__
+static void
+print_nfa (struct rx * rx,
+	   struct rx_nfa_state * n,
+	   side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_nfa (rx, n, seprint, fp)
+     struct rx * rx;
+     struct rx_nfa_state * n;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      fprintf (fp, "node %d %s\n", n->id,
+	       n->is_final ? "final" : (n->is_start ? "start" : ""));
+      while (e)
+	{
+	  fprintf (fp, "   edge to %d, ", e->dest->id);
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	      fprintf (fp, "epsilon\n");
+	      break;
+	    case ne_side_effect:
+	      fprintf (fp, "side effect ");
+	      seprint (rx, e->params.side_effect, fp);
+	      fputc ('\n', fp);
+	      break;
+	    case ne_cset:
+	      fprintf (fp, "cset ");
+	      print_cset (rx, e->params.cset, fp);
+	      fputc ('\n', fp);
+	      break;
+	    }
+	  e = e->next;
+	}
+
+      while (ec)
+	{
+	  int x;
+	  struct rx_nfa_state_set * s;
+	  struct rx_se_list * l;
+	  fprintf (fp, "   eclosure to {");
+	  for (s = ec->destset; s; s = s->cdr)
+	    fprintf (fp, "%d ", s->car->id);
+	  fprintf (fp, "} (");
+	  for (l = ec->effects; l; l = l->cdr)
+	    {
+	      seprint (rx, l->car, fp);
+	      fputc (' ', fp);
+	    }
+	  fprintf (fp, ")\n");
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+}
+
+static char * efnames [] =
+{
+  "bogon",
+  "re_se_try",
+  "re_se_pushback",
+  "re_se_push0",
+  "re_se_pushpos",
+  "re_se_chkpos",
+  "re_se_poppos",
+  "re_se_at_dot",
+  "re_se_syntax",
+  "re_se_not_syntax",
+  "re_se_begbuf",
+  "re_se_hat",
+  "re_se_wordbeg",
+  "re_se_wordbound",
+  "re_se_notwordbound",
+  "re_se_wordend",
+  "re_se_endbuf",
+  "re_se_dollar",
+  "re_se_fail",
+};
+
+static char * efnames2[] =
+{
+  "re_se_win",
+  "re_se_lparen",
+  "re_se_rparen",
+  "re_se_backref",
+  "re_se_iter",
+  "re_se_end_iter",
+  "re_se_tv"
+};
+
+static char * inx_names[] = 
+{
+  "rx_backtrack_point",
+  "rx_do_side_effects",
+  "rx_cache_miss",
+  "rx_next_char",
+  "rx_backtrack",
+  "rx_error_inx",
+  "rx_num_instructions"
+};
+
+
+#ifdef __STDC__
+static void
+re_seprint (struct rx * rx, void * effect, FILE * fp)
+#else
+static void
+re_seprint (rx, effect, fp)
+     struct rx * rx;
+     void * effect;
+     FILE * fp;
+#endif
+{
+  if ((int)effect < 0)
+    fputs (efnames[-(int)effect], fp);
+  else if (dbug_rxb)
+    {
+      struct re_se_params * p = &dbug_rxb->se_params[(int)effect];
+      fprintf (fp, "%s(%d,%d)", efnames2[p->se], p->op1, p->op2);
+    }
+  else
+    fprintf (fp, "[complex op # %d]", (int)effect);
+}
+
+
+/* These are so the regex.c regression tests will compile. */
+void
+print_compiled_pattern (rxb)
+     struct re_pattern_buffer * rxb;
+{
+}
+
+void
+print_fastmap (fm)
+     char * fm;
+{
+}
+
+#endif /* RX_DEBUG */
+
+
+
+
+/* This page: Bitsets.  Completely unintersting. */
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_equal (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  RX_subset s = b[0];
+  b[0] = ~a[0];
+
+  for (x = rx_bitset_numb_subsets(size) - 1; a[x] == b[x]; --x)
+    ;
+
+  b[0] = s;
+  return !x && s == a[0];
+}
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_subset (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets(size) - 1;
+  while (x-- && (a[x] & b[x]) == a[x]);
+  return x == -1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_empty (int size, rx_Bitset set)
+#else
+RX_DECL int
+rx_bitset_empty (size, set)
+     int size;
+     rx_Bitset set;
+#endif
+{
+  int x;
+  RX_subset s = set[0];
+  set[0] = 1;
+  for (x = rx_bitset_numb_subsets(size) - 1; !set[x]; --x)
+    ;
+  set[0] = s;
+  return !s;
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_null (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_null (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  bzero (b, rx_sizeof_bitset(size));
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_universe (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_universe (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    *b++ = ~(RX_subset)0;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_complement (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_complement (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    {
+      *b = ~*b;
+      ++b;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_assign (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_union (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_union (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] |= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_intersection (int size,
+			rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_intersection (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_difference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &=  ~ b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_revdifference (int size,
+			 rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_revdifference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = ~a[x] & b[x];
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_xor (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] ^= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL unsigned long
+rx_bitset_hash (int size, rx_Bitset b)
+#else
+RX_DECL unsigned long
+rx_bitset_hash (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  unsigned long hash = (unsigned long)rx_bitset_hash;
+
+  for (x = rx_bitset_numb_subsets(size) - 1; x >= 0; --x)
+    hash ^= rx_bitset_subset_val(b, x);
+
+  return hash;
+}
+
+
+RX_DECL RX_subset rx_subset_singletons [RX_subset_bits] = 
+{
+  0x1,
+  0x2,
+  0x4,
+  0x8,
+  0x10,
+  0x20,
+  0x40,
+  0x80,
+  0x100,
+  0x200,
+  0x400,
+  0x800,
+  0x1000,
+  0x2000,
+  0x4000,
+  0x8000,
+  0x10000,
+  0x20000,
+  0x40000,
+  0x80000,
+  0x100000,
+  0x200000,
+  0x400000,
+  0x800000,
+  0x1000000,
+  0x2000000,
+  0x4000000,
+  0x8000000,
+  0x10000000,
+  0x20000000,
+  0x40000000,
+  0x80000000
+};
+
+#ifdef RX_DEBUG
+
+#ifdef __STDC__
+static void
+print_cset (struct rx *rx, rx_Bitset cset, FILE * fp)
+#else
+static void
+print_cset (rx, cset, fp)
+     struct rx *rx;
+     rx_Bitset cset;
+     FILE * fp;
+#endif
+{
+  int x;
+  fputc ('[', fp);
+  for (x = 0; x < rx->local_cset_size; ++x)
+    if (RX_bitset_member (cset, x))
+      {
+	if (isprint(x))
+	  fputc (x, fp);
+	else
+	  fprintf (fp, "\\0%o ", x);
+      }
+  fputc (']', fp);
+}
+
+#endif /*  RX_DEBUG */
+
+
+
+
+static unsigned long rx_hash_masks[4] =
+{
+  0x12488421,
+  0x96699669,
+  0xbe7dd7eb,
+  0xffffffff
+};
+
+
+/* Hash tables */
+#ifdef __STDC__
+RX_DECL struct rx_hash_item * 
+rx_hash_find (struct rx_hash * table,
+	      unsigned long hash,
+	      void * value,
+	      struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item * 
+rx_hash_find (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks [0];
+  int bucket = (hash & mask) % 13;
+
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+    }
+
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+
+  return 0;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_hash_item *
+rx_hash_store (struct rx_hash * table,
+	       unsigned long hash,
+	       void * value,
+	       struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item *
+rx_hash_store (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks[0];
+  int bucket = (hash & mask) % 13;
+  int depth = 0;
+  
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+      ++depth;
+    }
+  
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+  
+  {
+    if (   (depth < 3)
+	&& (table->bucket_size [bucket] >= 4))
+      {
+	struct rx_hash * newtab = ((struct rx_hash *)
+				   rules->hash_alloc (rules));
+	if (!newtab)
+	  goto add_to_bucket;
+	bzero (newtab, sizeof (*newtab));
+	newtab->parent = table;
+	{
+	  struct rx_hash_item * them = table->buckets[bucket];
+	  unsigned long newmask = rx_hash_masks[maskc + 1];
+	  while (them)
+	    {
+	      struct rx_hash_item * save = them->next_same_hash;
+	      int new_buck = (them->hash & newmask) % 13;
+	      them->next_same_hash = newtab->buckets[new_buck];
+	      newtab->buckets[new_buck] = them;
+	      them->table = newtab;
+	      them = save;
+	      ++newtab->bucket_size[new_buck];
+	      ++newtab->refs;
+	    }
+	  table->refs = (table->refs - table->bucket_size[bucket] + 1);
+	  table->bucket_size[bucket] = 0;
+	  table->buckets[bucket] = 0;
+	  table->children[bucket] = newtab;
+	  table = newtab;
+	  bucket = (hash & newmask) % 13;
+	}
+      }
+  }
+ add_to_bucket:
+  {
+    struct rx_hash_item  * it = ((struct rx_hash_item *)
+				 rules->hash_item_alloc (rules, value));
+    if (!it)
+      return 0;
+    it->hash = hash;
+    it->table = table;
+    /* DATA and BINDING are to be set in hash_item_alloc */
+    it->next_same_hash = table->buckets [bucket];
+    table->buckets[bucket] = it;
+    ++table->bucket_size [bucket];
+    ++table->refs;
+    return it;
+  }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_hash_free (it, rules)
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+  if (it)
+    {
+      struct rx_hash * table = it->table;
+      unsigned long hash = it->hash;
+      int depth = (table->parent
+		   ? (table->parent->parent
+		      ? (table->parent->parent->parent
+			 ? 3
+			 : 2)
+		      : 1)
+		   : 0);
+      int bucket = (hash & rx_hash_masks [depth]) % 13;
+      struct rx_hash_item ** pos = &table->buckets [bucket];
+      
+      while (*pos != it)
+	pos = &(*pos)->next_same_hash;
+      *pos = it->next_same_hash;
+      rules->free_hash_item (it, rules);
+      --table->bucket_size[bucket];
+      --table->refs;
+      while (!table->refs && depth)
+	{
+	  struct rx_hash * save = table;
+	  table = table->parent;
+	  --depth;
+	  bucket = (hash & rx_hash_masks [depth]) % 13;
+	  --table->refs;
+	  table->children[bucket] = 0;
+	  rules->free_hash (save, rules);
+	}
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+		    struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_free_hash_table (tab, freefn, rules)
+     struct rx_hash * tab;
+     rx_hash_freefn freefn;
+     struct rx_hash_rules * rules;
+#endif
+{
+  int x;
+
+  for (x = 0; x < 13; ++x)
+    if (tab->children[x])
+      {
+	rx_free_hash_table (tab->children[x], freefn, rules);
+	rules->free_hash (tab->children[x], rules);
+      }
+    else
+      {
+	struct rx_hash_item * them = tab->buckets[x];
+	while (them)
+	  {
+	    struct rx_hash_item * that = them;
+	    them = that->next_same_hash;
+	    freefn (that);
+	    rules->free_hash_item (that, rules);
+	  }
+      }
+}
+
+
+
+
+/* Utilities for manipulating bitset represntations of characters sets. */
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_cset (struct rx *rx)
+#else
+RX_DECL rx_Bitset
+rx_cset (rx)
+     struct rx *rx;
+#endif
+{
+  rx_Bitset b = (rx_Bitset) malloc (rx_sizeof_bitset (rx->local_cset_size));
+  if (b)
+    rx_bitset_null (rx->local_cset_size, b);
+  return b;
+}
+
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_copy_cset (struct rx *rx, rx_Bitset a)
+#else
+RX_DECL rx_Bitset
+rx_copy_cset (rx, a)
+     struct rx *rx;
+     rx_Bitset a;
+#endif
+{
+  rx_Bitset cs = rx_cset (rx);
+
+  if (cs)
+    rx_bitset_union (rx->local_cset_size, cs, a);
+
+  return cs;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_cset (struct rx * rx, rx_Bitset c)
+#else
+RX_DECL void
+rx_free_cset (rx, c)
+     struct rx * rx;
+     rx_Bitset c;
+#endif
+{
+  if (c)
+    free ((char *)c);
+}
+
+
+
+/* Hash table memory allocation policy for the regexp compiler */
+
+#ifdef __STDC__
+static struct rx_hash *
+compiler_hash_alloc (struct rx_hash_rules * rules)
+#else
+static struct rx_hash *
+compiler_hash_alloc (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  return (struct rx_hash *)malloc (sizeof (struct rx_hash));
+}
+
+
+#ifdef __STDC__
+static struct rx_hash_item *
+compiler_hash_item_alloc (struct rx_hash_rules * rules, void * value)
+#else
+static struct rx_hash_item *
+compiler_hash_item_alloc (rules, value)
+     struct rx_hash_rules * rules;
+     void * value;
+#endif
+{
+  struct rx_hash_item * it;
+  it = (struct rx_hash_item *)malloc (sizeof (*it));
+  if (it)
+    {
+      it->data = value;
+      it->binding = 0;
+    }
+  return it;
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash (struct rx_hash * tab,
+		    struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash (tab, rules)
+     struct rx_hash * tab;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)tab);
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash_item (struct rx_hash_item * item,
+			 struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash_item (item, rules)
+     struct rx_hash_item * item;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)item);
+}
+
+
+
+/* This page: REXP_NODE (expression tree) structures. */
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rexp_node (struct rx *rx,
+	   enum rexp_node_type type)
+#else
+RX_DECL struct rexp_node *
+rexp_node (rx, type)
+     struct rx *rx;
+     enum rexp_node_type type;
+#endif
+{
+  struct rexp_node *n;
+
+  n = (struct rexp_node *)malloc (sizeof (*n));
+  bzero (n, sizeof (*n));
+  if (n)
+    n->type = type;
+  return n;
+}
+
+
+/* free_rexp_node assumes that the bitset passed to rx_mk_r_cset
+ * can be freed using rx_free_cset.
+ */
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_cset (struct rx * rx,
+	      rx_Bitset b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_cset (rx, b)
+     struct rx * rx;
+     rx_Bitset b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_cset);
+  if (n)
+    n->params.cset = b;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_concat (struct rx * rx,
+		struct rexp_node * a,
+		struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_concat (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_concat);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (struct rx * rx,
+		   struct rexp_node * a,
+		   struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_alternate);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_opt (struct rx * rx,
+	     struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_opt (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_opt);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_star (struct rx * rx,
+	      struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_star (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (struct rx * rx,
+		     struct rexp_node * a,
+		     struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_2phase_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (struct rx * rx,
+		     rx_side_effect a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (rx, a)
+     struct rx * rx;
+     rx_side_effect a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_side_effect);
+  if (n)
+    {
+      n->params.side_effect = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_data  (struct rx * rx,
+	       void * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_data  (rx, a)
+     struct rx * rx;
+     void * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_data);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_rexp (struct rx * rx, struct rexp_node * node)
+#else
+RX_DECL void
+rx_free_rexp (rx, node)
+     struct rx * rx;
+     struct rexp_node * node;
+#endif
+{
+  if (node)
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  if (node->params.cset)
+	    rx_free_cset (rx, node->params.cset);
+
+	case r_side_effect:
+	  break;
+	  
+	case r_concat:
+	case r_alternate:
+	case r_2phase_star:
+	case r_opt:
+	case r_star:
+	  rx_free_rexp (rx, node->params.pair.left);
+	  rx_free_rexp (rx, node->params.pair.right);
+	  break;
+
+	case r_data:
+	  /* This shouldn't occur. */
+	  break;
+	}
+      free ((char *)node);
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node * 
+rx_copy_rexp (struct rx *rx,
+	   struct rexp_node *node)
+#else
+RX_DECL struct rexp_node * 
+rx_copy_rexp (rx, node)
+     struct rx *rx;
+     struct rexp_node *node;
+#endif
+{
+  if (!node)
+    return 0;
+  else
+    {
+      struct rexp_node *n = rexp_node (rx, node->type);
+      if (!n)
+	return 0;
+      switch (node->type)
+	{
+	case r_cset:
+	  n->params.cset = rx_copy_cset (rx, node->params.cset);
+	  if (!n->params.cset)
+	    {
+	      rx_free_rexp (rx, n);
+	      return 0;
+	    }
+	  break;
+
+	case r_side_effect:
+	  n->params.side_effect = node->params.side_effect;
+	  break;
+
+	case r_concat:
+	case r_alternate:
+	case r_opt:
+	case r_2phase_star:
+	case r_star:
+	  n->params.pair.left =
+	    rx_copy_rexp (rx, node->params.pair.left);
+	  n->params.pair.right =
+	    rx_copy_rexp (rx, node->params.pair.right);
+	  if (   (node->params.pair.left && !n->params.pair.left)
+	      || (node->params.pair.right && !n->params.pair.right))
+	    {
+	      rx_free_rexp  (rx, n);
+	      return 0;
+	    }
+	  break;
+	case r_data:
+	  /* shouldn't happen */
+	  break;
+	}
+      return n;
+    }
+}
+
+
+
+
+/* This page: functions to build and destroy graphs that describe nfa's */
+
+/* Constructs a new nfa node. */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (struct rx *rx)
+#else
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state * n = (struct rx_nfa_state *)malloc (sizeof (*n));
+  if (!n)
+    return 0;
+  bzero (n, sizeof (*n));
+  n->next = rx->nfa_states;
+  rx->nfa_states = n;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_state (struct rx_nfa_state * n)
+#else
+RX_DECL void
+rx_free_nfa_state (n)
+  struct rx_nfa_state * n;
+#endif
+{
+  free ((char *)n);
+}
+
+
+/* This looks up an nfa node, given a numeric id.  Numeric id's are
+ * assigned after the nfa has been built.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (struct rx * rx,
+		    int id)
+#else
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (rx, id)
+     struct rx * rx;
+     int id;
+#endif
+{
+  struct rx_nfa_state * n;
+  for (n = rx->nfa_states; n; n = n->next)
+    if (n->id == id)
+      return n;
+  return 0;
+}
+
+
+/* This adds an edge between two nodes, but doesn't initialize the 
+ * edge label.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (struct rx *rx,
+	     enum rx_nfa_etype type,
+	     struct rx_nfa_state *start,
+	     struct rx_nfa_state *dest)
+#else
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (rx, type, start, dest)
+     struct rx *rx;
+     enum rx_nfa_etype type;
+     struct rx_nfa_state *start;
+     struct rx_nfa_state *dest;
+#endif
+{
+  struct rx_nfa_edge *e;
+  e = (struct rx_nfa_edge *)malloc (sizeof (*e));
+  if (!e)
+    return 0;
+  e->next = start->edges;
+  start->edges = e;
+  e->type = type;
+  e->dest = dest;
+  return e;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_edge (struct rx_nfa_edge * e)
+#else
+RX_DECL void
+rx_free_nfa_edge (e)
+     struct rx_nfa_edge * e;
+#endif
+{
+  free ((char *)e);
+}
+
+
+/* This constructs a POSSIBLE_FUTURE, which is a kind epsilon-closure
+ * of an NFA.  These are added to an nfa automaticly by eclose_nfa.
+ */  
+
+#ifdef __STDC__
+static struct rx_possible_future * 
+rx_possible_future (struct rx * rx,
+		 struct rx_se_list * effects)
+#else
+static struct rx_possible_future * 
+rx_possible_future (rx, effects)
+     struct rx * rx;
+     struct rx_se_list * effects;
+#endif
+{
+  struct rx_possible_future *ec;
+  ec = (struct rx_possible_future *) malloc (sizeof (*ec));
+  if (!ec)
+    return 0;
+  ec->destset = 0;
+  ec->next = 0;
+  ec->effects = effects;
+  return ec;
+}
+
+
+#ifdef __STDC__
+static void
+rx_free_possible_future (struct rx_possible_future * pf)
+#else
+static void
+rx_free_possible_future (pf)
+     struct rx_possible_future * pf;
+#endif
+{
+  free ((char *)pf);
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa (struct rx *rx)
+#else
+RX_DECL void
+rx_free_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  while (rx->nfa_states)
+    {
+      while (rx->nfa_states->edges)
+	{
+	  switch (rx->nfa_states->edges->type)
+	    {
+	    case ne_cset:
+	      rx_free_cset (rx, rx->nfa_states->edges->params.cset);
+	      break;
+	    default:
+	      break;
+	    }
+	  {
+	    struct rx_nfa_edge * e;
+	    e = rx->nfa_states->edges;
+	    rx->nfa_states->edges = rx->nfa_states->edges->next;
+	    rx_free_nfa_edge (e);
+	  }
+	} /* while (rx->nfa_states->edges) */
+      {
+	/* Iterate over the partial epsilon closures of rx->nfa_states */
+	struct rx_possible_future * pf = rx->nfa_states->futures;
+	while (pf)
+	  {
+	    struct rx_possible_future * pft = pf;
+	    pf = pf->next;
+	    rx_free_possible_future (pft);
+	  }
+      }
+      {
+	struct rx_nfa_state *n;
+	n = rx->nfa_states;
+	rx->nfa_states = rx->nfa_states->next;
+	rx_free_nfa_state (n);
+      }
+    }
+}
+
+
+
+
+/* This page: translating a pattern expression into an nfa and doing the 
+ * static part of the nfa->super-nfa translation.
+ */
+
+/* This is the thompson regexp->nfa algorithm. 
+ * It is modified to allow for `side-effect epsilons.'  Those are
+ * edges that are taken whenever a similar epsilon edge would be,
+ * but which imply that some side effect occurs when the edge 
+ * is taken.
+ *
+ * Side effects are used to model parts of the pattern langauge 
+ * that are not regular (in the formal sense).
+ */
+
+#ifdef __STDC__
+RX_DECL int
+rx_build_nfa (struct rx *rx,
+	      struct rexp_node *rexp,
+	      struct rx_nfa_state **start,
+	      struct rx_nfa_state **end)
+#else
+RX_DECL int
+rx_build_nfa (rx, rexp, start, end)
+     struct rx *rx;
+     struct rexp_node *rexp;
+     struct rx_nfa_state **start;
+     struct rx_nfa_state **end;
+#endif
+{
+  struct rx_nfa_edge *edge;
+
+  /* Start & end nodes may have been allocated by the caller. */
+  *start = *start ? *start : rx_nfa_state (rx);
+
+  if (!*start)
+    return 0;
+
+  if (!rexp)
+    {
+      *end = *start;
+      return 1;
+    }
+
+  *end = *end ? *end : rx_nfa_state (rx);
+
+  if (!*end)
+    {
+      rx_free_nfa_state (*start);
+      return 0;
+    }
+
+  switch (rexp->type)
+    {
+    case r_data:
+      return 0;
+
+    case r_cset:
+      edge = rx_nfa_edge (rx, ne_cset, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.cset = rx_copy_cset (rx, rexp->params.cset);
+      if (!edge->params.cset)
+	{
+	  rx_free_nfa_edge (edge);
+	  return 0;
+	}
+      return 1;
+ 
+    case r_opt:
+      return (rx_build_nfa (rx, rexp->params.pair.left, start, end)
+	      && rx_nfa_edge (rx, ne_epsilon, *start, *end));
+
+    case r_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& star_start
+		&& star_end
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, star_end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, star_start));
+      }
+
+    case r_2phase_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	struct rx_nfa_state * loop_exp_start = 0;
+	struct rx_nfa_state * loop_exp_end = 0;
+
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& rx_build_nfa (rx, rexp->params.pair.right,
+				 &loop_exp_start, &loop_exp_end)
+		&& star_start
+		&& star_end
+		&& loop_exp_end
+		&& loop_exp_start
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, loop_exp_start)
+		&& rx_nfa_edge (rx, ne_epsilon, loop_exp_end, star_start));
+      }
+
+
+    case r_concat:
+      {
+	struct rx_nfa_state *shared = 0;
+	return
+	  (rx_build_nfa (rx, rexp->params.pair.left, start, &shared)
+	   && rx_build_nfa (rx, rexp->params.pair.right, &shared, end));
+      }
+
+    case r_alternate:
+      {
+	struct rx_nfa_state *ls = 0;
+	struct rx_nfa_state *le = 0;
+	struct rx_nfa_state *rs = 0;
+	struct rx_nfa_state *re = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left, &ls, &le)
+		&& rx_build_nfa (rx, rexp->params.pair.right, &rs, &re)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, ls)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, rs)
+		&& rx_nfa_edge (rx, ne_epsilon, le, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, re, *end));
+      }
+
+    case r_side_effect:
+      edge = rx_nfa_edge (rx, ne_side_effect, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.side_effect = rexp->params.side_effect;
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+/* RX_NAME_NFA_STATES identifies all nodes with outgoing non-epsilon
+ * transitions.  Only these nodes can occur in super-states.  
+ * All nodes are given an integer id. 
+ * The id is non-negative if the node has non-epsilon out-transitions, negative
+ * otherwise (this is because we want the non-negative ids to be used as 
+ * array indexes in a few places).
+ */
+
+#ifdef __STDC__
+RX_DECL void
+rx_name_nfa_states (struct rx *rx)
+#else
+RX_DECL void
+rx_name_nfa_states (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+
+  rx->nodec = 0;
+  rx->epsnodec = -1;
+
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+
+      if (n->is_start)
+	n->eclosure_needed = 1;
+
+      while (e)
+	{
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      break;
+
+	    case ne_cset:
+	      n->id = rx->nodec++;
+	      {
+		struct rx_nfa_edge *from_n = n->edges;
+		while (from_n)
+		  {
+		    from_n->dest->eclosure_needed = 1;
+		    from_n = from_n->next;
+		  }
+	      }
+	      goto cont;
+	    }
+	  e = e->next;
+	}
+      n->id = rx->epsnodec--;
+    cont:
+      n = n->next;
+    }
+  rx->epsnodec = -rx->epsnodec;
+}
+
+
+
+/* This page: data structures for the static part of the nfa->supernfa
+ * translation.
+ *
+ * There are side effect lists -- lists of side effects occuring
+ * along an uninterrupted, acyclic path of side-effect epsilon edges.
+ * Such paths are collapsed to single edges in the course of computing
+ * epsilon closures.  Such single edges are labled with a list of all
+ * the side effects entailed in crossing them.  Like lists of side
+ * effects are made == by the constructors below.
+ *
+ * There are also nfa state sets.  These are used to hold a list of all
+ * states reachable from a starting state for a given type of transition
+ * and side effect list.   These are also hash-consed.
+ */
+
+/* The next several functions compare, construct, etc. lists of side
+ * effects.  See ECLOSE_NFA (below) for details.
+ */
+
+/* Ordering of rx_se_list
+ * (-1, 0, 1 return value convention).
+ */
+
+#ifdef __STDC__
+static int 
+se_list_cmp (void * va, void * vb)
+#else
+static int 
+se_list_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_se_list * a = (struct rx_se_list *)va;
+  struct rx_se_list * b = (struct rx_se_list *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: ((long)a->car < (long)b->car
+		   ? 1
+		   : ((long)a->car > (long)b->car
+		      ? -1
+		      : se_list_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+
+#ifdef __STDC__
+static int 
+se_list_equal (void * va, void * vb)
+#else
+static int 
+se_list_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !(se_list_cmp (va, vb));
+}
+
+static struct rx_hash_rules se_list_hash_rules =
+{
+  se_list_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_se_list * 
+side_effect_cons (struct rx * rx,
+		  void * se, struct rx_se_list * list)
+#else
+static struct rx_se_list * 
+side_effect_cons (rx, se, list)
+     struct rx * rx;
+     void * se;
+     struct rx_se_list * list;
+#endif
+{
+  struct rx_se_list * l;
+  l = ((struct rx_se_list *) malloc (sizeof (*l)));
+  if (!l)
+    return 0;
+  l->car = se;
+  l->cdr = list;
+  return l;
+}
+
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_cons_se_prog (struct rx * rx,
+		   struct rx_hash * memo,
+		   void * car, struct rx_se_list * cdr)
+#else
+static struct rx_se_list *
+hash_cons_se_prog (rx, memo, car, cdr)
+     struct rx * rx;
+     struct rx_hash * memo;
+     void * car;
+     struct rx_se_list * cdr;
+#endif
+{
+  long hash = (long)car ^ (long)cdr;
+  struct rx_se_list template;
+
+  template.car = car;
+  template.cdr = cdr;
+  {
+    struct rx_hash_item * it = rx_hash_store (memo, hash,
+					      (void *)&template,
+					      &se_list_hash_rules);
+    if (!it)
+      return 0;
+    if (it->data == (void *)&template)
+      {
+	struct rx_se_list * consed;
+	consed = (struct rx_se_list *) malloc (sizeof (*consed));
+	*consed = template;
+	it->data = (void *)consed;
+      }
+    return (struct rx_se_list *)it->data;
+  }
+}
+     
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_se_prog (struct rx * rx, struct rx_hash * memo, struct rx_se_list * prog)
+#else
+static struct rx_se_list *
+hash_se_prog (rx, memo, prog)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_se_list * prog;
+#endif
+{
+  struct rx_se_list * answer = 0;
+  while (prog)
+    {
+      answer = hash_cons_se_prog (rx, memo, prog->car, answer);
+      if (!answer)
+	return 0;
+      prog = prog->cdr;
+    }
+  return answer;
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_cmp (void * va, void * vb)
+#else
+static int 
+nfa_set_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state_set * a = (struct rx_nfa_state_set *)va;
+  struct rx_nfa_state_set * b = (struct rx_nfa_state_set *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: (a->car->id < b->car->id
+		   ? 1
+		   : (a->car->id > b->car->id
+		      ? -1
+		      : nfa_set_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_equal (void * va, void * vb)
+#else
+static int 
+nfa_set_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !nfa_set_cmp (va, vb);
+}
+
+static struct rx_hash_rules nfa_set_hash_rules =
+{
+  nfa_set_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_cons (struct rx * rx,
+	      struct rx_hash * memo, struct rx_nfa_state * state,
+	      struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_cons (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  struct rx_nfa_state_set template;
+  struct rx_hash_item * node;
+  template.car = state;
+  template.cdr = set;
+  node = rx_hash_store (memo,
+			(((long)state) >> 8) ^ (long)set,
+			&template, &nfa_set_hash_rules);
+  if (!node)
+    return 0;
+  if (node->data == &template)
+    {
+      struct rx_nfa_state_set * l;
+      l = (struct rx_nfa_state_set *) malloc (sizeof (*l));
+      node->data = (void *) l;
+      if (!l)
+	return 0;
+      *l = template;
+    }
+  return (struct rx_nfa_state_set *)node->data;
+}
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (struct rx * rx,
+		struct rx_hash * memo, struct rx_nfa_state * state,
+		struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  if (!set || state->id < set->car->id)
+    return nfa_set_cons (rx, memo, state, set);
+  if (state->id == set->car->id)
+    return set;
+  else
+    {
+      struct rx_nfa_state_set * newcdr
+	= nfa_set_enjoin (rx, memo, state, set->cdr);
+      if (newcdr != set->cdr)
+	set = nfa_set_cons (rx, memo, set->car, newcdr);
+      return set;
+    }
+}
+
+
+
+
+/* This page: computing epsilon closures.  The closures aren't total.
+ * Each node's closures are partitioned according to the side effects entailed
+ * along the epsilon edges.  Return true on success.
+ */ 
+
+struct eclose_frame
+{
+  struct rx_se_list *prog_backwards;
+};
+
+
+#ifdef __STDC__
+static int 
+eclose_node (struct rx *rx, struct rx_nfa_state *outnode,
+	     struct rx_nfa_state *node, struct eclose_frame *frame)
+#else
+static int 
+eclose_node (rx, outnode, node, frame)
+     struct rx *rx;
+     struct rx_nfa_state *outnode;
+     struct rx_nfa_state *node;
+     struct eclose_frame *frame;
+#endif
+{
+  struct rx_nfa_edge *e = node->edges;
+
+  /* For each node, we follow all epsilon paths to build the closure.
+   * The closure omits nodes that have only epsilon edges.
+   * The closure is split into partial closures -- all the states in
+   * a partial closure are reached by crossing the same list of
+   * of side effects (though not necessarily the same path).
+   */
+  if (node->mark)
+    return 1;
+  node->mark = 1;
+
+  if (node->id >= 0 || node->is_final)
+    {
+      struct rx_possible_future **ec;
+      struct rx_se_list * prog_in_order
+	= ((struct rx_se_list *)hash_se_prog (rx,
+					      &rx->se_list_memo,
+					      frame->prog_backwards));
+      int cmp;
+
+      ec = &outnode->futures;
+
+      while (*ec)
+	{
+	  cmp = se_list_cmp ((void *)(*ec)->effects, (void *)prog_in_order);
+	  if (cmp <= 0)
+	    break;
+	  ec = &(*ec)->next;
+	}
+      if (!*ec || (cmp < 0))
+	{
+	  struct rx_possible_future * saved = *ec;
+	  *ec = rx_possible_future (rx, prog_in_order);
+	  (*ec)->next = saved;
+	  if (!*ec)
+	    return 0;
+	}
+      if (node->id >= 0)
+	{
+	  (*ec)->destset = nfa_set_enjoin (rx, &rx->set_list_memo,
+					   node, (*ec)->destset);
+	  if (!(*ec)->destset)
+	    return 0;
+	}
+    }
+
+  while (e)
+    {
+      switch (e->type)
+	{
+	case ne_epsilon:
+	  if (!eclose_node (rx, outnode, e->dest, frame))
+	    return 0;
+	  break;
+	case ne_side_effect:
+	  {
+	    frame->prog_backwards = side_effect_cons (rx, 
+						      e->params.side_effect,
+						      frame->prog_backwards);
+	    if (!frame->prog_backwards)
+	      return 0;
+	    if (!eclose_node (rx, outnode, e->dest, frame))
+	      return 0;
+	    {
+	      struct rx_se_list * dying = frame->prog_backwards;
+	      frame->prog_backwards = frame->prog_backwards->cdr;
+	      free ((char *)dying);
+	    }
+	    break;
+	  }
+	default:
+	  break;
+	}
+      e = e->next;
+    }
+  node->mark = 0;
+  return 1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int 
+rx_eclose_nfa (struct rx *rx)
+#else
+RX_DECL int 
+rx_eclose_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct eclose_frame frame;
+  static int rx_id = 0;
+  
+  frame.prog_backwards = 0;
+  rx->rx_id = rx_id++;
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+  while (n)
+    {
+      n->futures = 0;
+      if (n->eclosure_needed && !eclose_node (rx, n, n, &frame))
+	return 0;
+      /* clear_marks (rx); */
+      n = n->next;
+    }
+  return 1;
+}
+
+
+/* This deletes epsilon edges from an NFA.  After running eclose_node,
+ * we have no more need for these edges.  They are removed to simplify
+ * further operations on the NFA.
+ */
+
+#ifdef __STDC__
+RX_DECL void 
+rx_delete_epsilon_transitions (struct rx *rx)
+#else
+RX_DECL void 
+rx_delete_epsilon_transitions (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct rx_nfa_edge **e;
+
+  while (n)
+    {
+      e = &n->edges;
+      while (*e)
+	{
+	  struct rx_nfa_edge *t;
+	  switch ((*e)->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      t = *e;
+	      *e = t->next;
+	      rx_free_nfa_edge (t);
+	      break;
+
+	    default:
+	      e = &(*e)->next;
+	      break;
+	    }
+	}
+      n = n->next;
+    }
+}
+
+
+
+/* This page: storing the nfa in a contiguous region of memory for
+ * subsequent conversion to a super-nfa.
+ */
+
+/* This is for qsort on an array of nfa_states. The order
+ * is based on state ids and goes 
+ *		[0...MAX][MIN..-1] where (MAX>=0) and (MIN<0)
+ * This way, positive ids double as array indices.
+ */
+
+#ifdef __STDC__
+static int 
+nfacmp (void * va, void * vb)
+#else
+static int 
+nfacmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state **a = (struct rx_nfa_state **)va;
+  struct rx_nfa_state **b = (struct rx_nfa_state **)vb;
+  return (*a == *b		/* &&&& 3.18 */
+	  ? 0
+	  : (((*a)->id < 0) == ((*b)->id < 0)
+	     ? (((*a)->id  < (*b)->id) ? -1 : 1)
+	     : (((*a)->id < 0)
+		? 1 : -1)));
+}
+
+#ifdef __STDC__
+static int 
+count_hash_nodes (struct rx_hash * st)
+#else
+static int 
+count_hash_nodes (st)
+     struct rx_hash * st;
+#endif
+{
+  int x;
+  int count = 0;
+  for (x = 0; x < 13; ++x)
+    count += ((st->children[x])
+	      ? count_hash_nodes (st->children[x])
+	      : st->bucket_size[x]);
+  
+  return count;
+}
+
+
+#ifdef __STDC__
+static void 
+se_memo_freer (struct rx_hash_item * node)
+#else
+static void 
+se_memo_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+#ifdef __STDC__
+static void 
+nfa_set_freer (struct rx_hash_item * node)
+#else
+static void 
+nfa_set_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+/* This copies an entire NFA into a single malloced block of memory.
+ * Mostly this is for compatability with regex.c, though it is convenient
+ * to have the nfa nodes in an array.
+ */
+
+#ifdef __STDC__
+RX_DECL int 
+rx_compactify_nfa (struct rx *rx,
+		   void **mem, unsigned long *size)
+#else
+RX_DECL int 
+rx_compactify_nfa (rx, mem, size)
+     struct rx *rx;
+     void **mem;
+     unsigned long *size;
+#endif
+{
+  int total_nodec;
+  struct rx_nfa_state *n;
+  int edgec = 0;
+  int eclosec = 0;
+  int se_list_consc = count_hash_nodes (&rx->se_list_memo);
+  int nfa_setc = count_hash_nodes (&rx->set_list_memo);
+  unsigned long total_size;
+
+  /* This takes place in two stages.   First, the total size of the
+   * nfa is computed, then structures are copied.  
+   */   
+  n = rx->nfa_states;
+  total_nodec = 0;
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      ++total_nodec;
+      while (e)
+	{
+	  ++edgec;
+	  e = e->next;
+	}
+      while (ec)
+	{
+	  ++eclosec;
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+
+  total_size = (total_nodec * sizeof (struct rx_nfa_state)
+		+ edgec * rx_sizeof_bitset (rx->local_cset_size)
+		+ edgec * sizeof (struct rx_nfa_edge)
+		+ nfa_setc * sizeof (struct rx_nfa_state_set)
+		+ eclosec * sizeof (struct rx_possible_future)
+		+ se_list_consc * sizeof (struct rx_se_list)
+		+ rx->reserved);
+
+  if (total_size > *size)
+    {
+      *mem = remalloc (*mem, total_size);
+      if (*mem)
+	*size = total_size;
+      else
+	return 0;
+    }
+  /* Now we've allocated the memory; this copies the NFA. */
+  {
+    static struct rx_nfa_state **scratch = 0;
+    static int scratch_alloc = 0;
+    struct rx_nfa_state *state_base = (struct rx_nfa_state *) * mem;
+    struct rx_nfa_state *new_state = state_base;
+    struct rx_nfa_edge *new_edge =
+      (struct rx_nfa_edge *)
+	((char *) state_base + total_nodec * sizeof (struct rx_nfa_state));
+    struct rx_se_list * new_se_list =
+      (struct rx_se_list *)
+	((char *)new_edge + edgec * sizeof (struct rx_nfa_edge));
+    struct rx_possible_future *new_close =
+      ((struct rx_possible_future *)
+       ((char *) new_se_list
+	+ se_list_consc * sizeof (struct rx_se_list)));
+    struct rx_nfa_state_set * new_nfa_set =
+      ((struct rx_nfa_state_set *)
+       ((char *)new_close + eclosec * sizeof (struct rx_possible_future)));
+    char *new_bitset =
+      ((char *) new_nfa_set + nfa_setc * sizeof (struct rx_nfa_state_set));
+    int x;
+    struct rx_nfa_state *n;
+
+    if (scratch_alloc < total_nodec)
+      {
+	scratch = ((struct rx_nfa_state **)
+		   remalloc (scratch, total_nodec * sizeof (*scratch)));
+	if (scratch)
+	  scratch_alloc = total_nodec;
+	else
+	  {
+	    scratch_alloc = 0;
+	    return 0;
+	  }
+      }
+
+    for (x = 0, n = rx->nfa_states; n; n = n->next)
+      scratch[x++] = n;
+
+    qsort (scratch, total_nodec,
+	   sizeof (struct rx_nfa_state *), (int (*)())nfacmp);
+
+    for (x = 0; x < total_nodec; ++x)
+      {
+	struct rx_possible_future *eclose = scratch[x]->futures;
+	struct rx_nfa_edge *edge = scratch[x]->edges;
+	struct rx_nfa_state *cn = new_state++;
+	cn->futures = 0;
+	cn->edges = 0;
+	cn->next = (x == total_nodec - 1) ? 0 : (cn + 1);
+	cn->id = scratch[x]->id;
+	cn->is_final = scratch[x]->is_final;
+	cn->is_start = scratch[x]->is_start;
+	cn->mark = 0;
+	while (edge)
+	  {
+	    int indx = (edge->dest->id < 0
+			 ? (total_nodec + edge->dest->id)
+			 : edge->dest->id);
+	    struct rx_nfa_edge *e = new_edge++;
+	    rx_Bitset cset = (rx_Bitset) new_bitset;
+	    new_bitset += rx_sizeof_bitset (rx->local_cset_size);
+	    rx_bitset_null (rx->local_cset_size, cset);
+	    rx_bitset_union (rx->local_cset_size, cset, edge->params.cset);
+	    e->next = cn->edges;
+	    cn->edges = e;
+	    e->type = edge->type;
+	    e->dest = state_base + indx;
+	    e->params.cset = cset;
+	    edge = edge->next;
+	  }
+	while (eclose)
+	  {
+	    struct rx_possible_future *ec = new_close++;
+	    struct rx_hash_item * sp;
+	    struct rx_se_list ** sepos;
+	    struct rx_se_list * sesrc;
+	    struct rx_nfa_state_set * destlst;
+	    struct rx_nfa_state_set ** destpos;
+	    ec->next = cn->futures;
+	    cn->futures = ec;
+	    for (sepos = &ec->effects, sesrc = eclose->effects;
+		 sesrc;
+		 sesrc = sesrc->cdr, sepos = &(*sepos)->cdr)
+	      {
+		sp = rx_hash_find (&rx->se_list_memo,
+				   (long)sesrc->car ^ (long)sesrc->cdr,
+				   sesrc, &se_list_hash_rules);
+		if (sp->binding)
+		  {
+		    sesrc = (struct rx_se_list *)sp->binding;
+		    break;
+		  }
+		*new_se_list = *sesrc;
+		sp->binding = (void *)new_se_list;
+		*sepos = new_se_list;
+		++new_se_list;
+	      }
+	    *sepos = sesrc;
+	    for (destpos = &ec->destset, destlst = eclose->destset;
+		 destlst;
+		 destpos = &(*destpos)->cdr, destlst = destlst->cdr)
+	      {
+		sp = rx_hash_find (&rx->set_list_memo,
+				   ((((long)destlst->car) >> 8)
+				    ^ (long)destlst->cdr),
+				   destlst, &nfa_set_hash_rules);
+		if (sp->binding)
+		  {
+		    destlst = (struct rx_nfa_state_set *)sp->binding;
+		    break;
+		  }
+		*new_nfa_set = *destlst;
+		new_nfa_set->car = state_base + destlst->car->id;
+		sp->binding = (void *)new_nfa_set;
+		*destpos = new_nfa_set;
+		++new_nfa_set;
+	      }
+	    *destpos = destlst;
+	    eclose = eclose->next;
+	  }
+      }
+  }
+  rx_free_hash_table (&rx->se_list_memo, se_memo_freer, &se_list_hash_rules);
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  rx_free_hash_table (&rx->set_list_memo, nfa_set_freer, &nfa_set_hash_rules);
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+
+  rx_free_nfa (rx);
+  rx->nfa_states = (struct rx_nfa_state *)*mem;
+  return 1;
+}
+
+
+
+/* The functions in the next several pages define the lazy-NFA-conversion used
+ * by matchers.  The input to this construction is an NFA such as 
+ * is built by compactify_nfa (rx.c).  The output is the superNFA.
+ */
+
+/* Match engines can use arbitrary values for opcodes.  So, the parse tree 
+ * is built using instructions names (enum rx_opcode), but the superstate
+ * nfa is populated with mystery opcodes (void *).
+ *
+ * For convenience, here is an id table.  The opcodes are == to their inxs
+ *
+ * The lables in re_search_2 would make good values for instructions.
+ */
+
+void * rx_id_instruction_table[rx_num_instructions] =
+{
+  (void *) rx_backtrack_point,
+  (void *) rx_do_side_effects,
+  (void *) rx_cache_miss,
+  (void *) rx_next_char,
+  (void *) rx_backtrack,
+  (void *) rx_error_inx
+};
+
+
+
+
+/* Memory mgt. for superstate graphs. */
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc (struct rx_cache * cache, int bytes)
+#else
+static char *
+rx_cache_malloc (cache, bytes)
+     struct rx_cache * cache;
+     int bytes;
+#endif
+{
+  while (cache->bytes_left < bytes)
+    {
+      if (cache->memory_pos)
+	cache->memory_pos = cache->memory_pos->next;
+      if (!cache->memory_pos)
+	{
+	  cache->morecore (cache);
+	  if (!cache->memory_pos)
+	    return 0;
+	}
+      cache->bytes_left = cache->memory_pos->bytes;
+      cache->memory_addr = ((char *)cache->memory_pos
+			    + sizeof (struct rx_blocklist));
+    }
+  cache->bytes_left -= bytes;
+  {
+    char * addr = cache->memory_addr;
+    cache->memory_addr += bytes;
+    return addr;
+  }
+}
+
+#ifdef __STDC__
+static void
+rx_cache_free (struct rx_cache * cache,
+	       struct rx_freelist ** freelist, char * mem)
+#else
+static void
+rx_cache_free (cache, freelist, mem)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     char * mem;
+#endif
+{
+  struct rx_freelist * it = (struct rx_freelist *)mem;
+  it->next = *freelist;
+  *freelist = it;
+}
+
+
+/* The partially instantiated superstate graph has a transition 
+ * table at every node.  There is one entry for every character.
+ * This fills in the transition for a set.
+ */
+#ifdef __STDC__
+static void 
+install_transition (struct rx_superstate *super,
+		    struct rx_inx *answer, rx_Bitset trcset) 
+#else
+static void 
+install_transition (super, answer, trcset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     rx_Bitset trcset;
+#endif
+{
+  struct rx_inx * transitions = super->transitions;
+  int chr;
+  for (chr = 0; chr < 256; )
+    if (!*trcset)
+      {
+	++trcset;
+	chr += 32;
+      }
+    else
+      {
+	RX_subset sub = *trcset;
+	RX_subset mask = 1;
+	int bound = chr + 32;
+	while (chr < bound)
+	  {
+	    if (sub & mask)
+	      transitions [chr] = *answer;
+	    ++chr;
+	    mask <<= 1;
+	  }
+	++trcset;
+      }
+}
+
+
+#ifdef __STDC__
+static int
+qlen (struct rx_superstate * q)
+#else
+static int
+qlen (q)
+     struct rx_superstate * q;
+#endif
+{
+  int count = 1;
+  struct rx_superstate * it;
+  if (!q)
+    return 0;
+  for (it = q->next_recyclable; it != q; it = it->next_recyclable)
+    ++count;
+  return count;
+}
+
+#ifdef __STDC__
+static void
+check_cache (struct rx_cache * cache)
+#else
+static void
+check_cache (cache)
+     struct rx_cache * cache;
+#endif
+{
+  struct rx_cache * you_fucked_up = 0;
+  int total = cache->superstates;
+  int semi = cache->semifree_superstates;
+  if (semi != qlen (cache->semifree_superstate))
+    check_cache (you_fucked_up);
+  if ((total - semi) != qlen (cache->lru_superstate))
+    check_cache (you_fucked_up);
+}
+
+/* When a superstate is old and neglected, it can enter a 
+ * semi-free state.  A semi-free state is slated to die.
+ * Incoming transitions to a semi-free state are re-written
+ * to cause an (interpreted) fault when they are taken.
+ * The fault handler revives the semi-free state, patches
+ * incoming transitions back to normal, and continues.
+ *
+ * The idea is basicly to free in two stages, aborting 
+ * between the two if the state turns out to be useful again.
+ * When a free is aborted, the rescued superstate is placed
+ * in the most-favored slot to maximize the time until it
+ * is next semi-freed.
+ */
+
+#ifdef __STDC__
+static void
+semifree_superstate (struct rx_cache * cache)
+#else
+static void
+semifree_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int disqualified = cache->semifree_superstates;
+  if (disqualified == cache->superstates)
+    return;
+  while (cache->lru_superstate->locks)
+    {
+      cache->lru_superstate = cache->lru_superstate->next_recyclable;
+      ++disqualified;
+      if (disqualified == cache->superstates)
+	return;
+    }
+  {
+    struct rx_superstate * it = cache->lru_superstate;
+    it->next_recyclable->prev_recyclable = it->prev_recyclable;
+    it->prev_recyclable->next_recyclable = it->next_recyclable;
+    cache->lru_superstate = (it == it->next_recyclable
+			     ? 0
+			     : it->next_recyclable);
+    if (!cache->semifree_superstate)
+      {
+	cache->semifree_superstate = it;
+	it->next_recyclable = it;
+	it->prev_recyclable = it;
+      }
+    else
+      {
+	it->prev_recyclable = cache->semifree_superstate->prev_recyclable;
+	it->next_recyclable = cache->semifree_superstate;
+	it->prev_recyclable->next_recyclable = it;
+	it->next_recyclable->prev_recyclable = it;
+      }
+    {
+      struct rx_distinct_future *df;
+      it->is_semifree = 1;
+      ++cache->semifree_superstates;
+      df = it->transition_refs;
+      if (df)
+	{
+	  df->prev_same_dest->next_same_dest = 0;
+	  for (df = it->transition_refs; df; df = df->next_same_dest)
+	    {
+	      df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	      df->future_frame.data = 0;
+	      df->future_frame.data_2 = (void *) df;
+	      /* If there are any NEXT-CHAR instruction frames that
+	       * refer to this state, we convert them to CACHE-MISS frames.
+	       */
+	      if (!df->effects
+		  && (df->edge->options->next_same_super_edge[0]
+		      == df->edge->options))
+		install_transition (df->present, &df->future_frame,
+				    df->edge->cset);
+	    }
+	  df = it->transition_refs;
+	  df->prev_same_dest->next_same_dest = df;
+	}
+    }
+  }
+}
+
+
+#ifdef __STDC__
+static void 
+refresh_semifree_superstate (struct rx_cache * cache,
+			     struct rx_superstate * super)
+#else
+static void 
+refresh_semifree_superstate (cache, super)
+     struct rx_cache * cache;
+     struct rx_superstate * super;
+#endif
+{
+  struct rx_distinct_future *df;
+
+  if (super->transition_refs)
+    {
+      super->transition_refs->prev_same_dest->next_same_dest = 0; 
+      for (df = super->transition_refs; df; df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_next_char];
+	  df->future_frame.data = (void *) super->transitions;
+	  /* CACHE-MISS instruction frames that refer to this state,
+	   * must be converted to NEXT-CHAR frames.
+	   */
+	  if (!df->effects
+	      && (df->edge->options->next_same_super_edge[0]
+		  == df->edge->options))
+	    install_transition (df->present, &df->future_frame,
+				df->edge->cset);
+	}
+      super->transition_refs->prev_same_dest->next_same_dest
+	= super->transition_refs;
+    }
+  if (cache->semifree_superstate == super)
+    cache->semifree_superstate = (super->prev_recyclable == super
+				  ? 0
+				  : super->prev_recyclable);
+  super->next_recyclable->prev_recyclable = super->prev_recyclable;
+  super->prev_recyclable->next_recyclable = super->next_recyclable;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = super->next_recyclable
+     = super->prev_recyclable
+     = super);
+  else
+    {
+      super->next_recyclable = cache->lru_superstate;
+      super->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      super->next_recyclable->prev_recyclable = super;
+      super->prev_recyclable->next_recyclable = super;
+    }
+  super->is_semifree = 0;
+  --cache->semifree_superstates;
+}
+
+#ifdef __STDC__
+static void
+rx_refresh_this_superstate (struct rx_cache * cache, struct rx_superstate * superstate)
+#else
+static void
+rx_refresh_this_superstate (cache, superstate)
+     struct rx_cache * cache;
+     struct rx_superstate * superstate;
+#endif
+{
+  if (superstate->is_semifree)
+    refresh_semifree_superstate (cache, superstate);
+  else if (cache->lru_superstate == superstate)
+    cache->lru_superstate = superstate->next_recyclable;
+  else if (superstate != cache->lru_superstate->prev_recyclable)
+    {
+      superstate->next_recyclable->prev_recyclable
+	= superstate->prev_recyclable;
+      superstate->prev_recyclable->next_recyclable
+	= superstate->next_recyclable;
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      superstate->next_recyclable->prev_recyclable = superstate;
+      superstate->prev_recyclable->next_recyclable = superstate;
+    }
+}
+
+#ifdef __STDC__
+static void 
+release_superset_low (struct rx_cache * cache,
+		     struct rx_superset *set)
+#else
+static void 
+release_superset_low (cache, set)
+     struct rx_cache * cache;
+     struct rx_superset *set;
+#endif
+{
+  if (!--set->refs)
+    {
+      if (set->cdr)
+	release_superset_low (cache, set->cdr);
+
+      set->starts_for = 0;
+
+      rx_hash_free
+	(rx_hash_find
+	 (&cache->superset_table,
+	  (unsigned long)set->car ^ set->id ^ (unsigned long)set->cdr,
+	  (void *)set,
+	  &cache->superset_hash_rules),
+	 &cache->superset_hash_rules);
+      rx_cache_free (cache, &cache->free_supersets, (char *)set);
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void 
+rx_release_superset (struct rx *rx,
+		     struct rx_superset *set)
+#else
+RX_DECL void 
+rx_release_superset (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  release_superset_low (rx->cache, set);
+}
+
+/* This tries to add a new superstate to the superstate freelist.
+ * It might, as a result, free some edge pieces or hash tables.
+ * If nothing can be freed because too many locks are being held, fail.
+ */
+
+#ifdef __STDC__
+static int
+rx_really_free_superstate (struct rx_cache * cache)
+#else
+static int
+rx_really_free_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int locked_superstates = 0;
+  struct rx_superstate * it;
+
+  if (!cache->superstates)
+    return 0;
+
+  {
+    /* This is a total guess.  The idea is that we should expect as
+     * many misses as we've recently experienced.  I.e., cache->misses
+     * should be the same as cache->semifree_superstates.
+     */
+    while ((cache->hits + cache->misses) > cache->superstates_allowed)
+      {
+	cache->hits >>= 1;
+	cache->misses >>= 1;
+      }
+    if (  ((cache->hits + cache->misses) * cache->semifree_superstates)
+	< (cache->superstates		 * cache->misses))
+      {
+	semifree_superstate (cache);
+	semifree_superstate (cache);
+      }
+  }
+
+  while (cache->semifree_superstate && cache->semifree_superstate->locks)
+    {
+      refresh_semifree_superstate (cache, cache->semifree_superstate);
+      ++locked_superstates;
+      if (locked_superstates == cache->superstates)
+	return 0;
+    }
+
+  if (cache->semifree_superstate)
+    {
+      it = cache->semifree_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->semifree_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+      --cache->semifree_superstates;
+    }
+  else
+    {
+      while (cache->lru_superstate->locks)
+	{
+	  cache->lru_superstate = cache->lru_superstate->next_recyclable;
+	  ++locked_superstates;
+	  if (locked_superstates == cache->superstates)
+	    return 0;
+	}
+      it = cache->lru_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->lru_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+    }
+
+  if (it->transition_refs)
+    {
+      struct rx_distinct_future *df;
+      for (df = it->transition_refs,
+	   df->prev_same_dest->next_same_dest = 0;
+	   df;
+	   df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	  df->future_frame.data = 0;
+	  df->future_frame.data_2 = (void *) df;
+	  df->future = 0;
+	}
+      it->transition_refs->prev_same_dest->next_same_dest =
+	it->transition_refs;
+    }
+  {
+    struct rx_super_edge *tc = it->edges;
+    while (tc)
+      {
+	struct rx_distinct_future * df;
+	struct rx_super_edge *tct = tc->next;
+	df = tc->options;
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+	while (df)
+	  {
+	    struct rx_distinct_future *dft = df;
+	    df = df->next_same_super_edge[0];
+	    
+	    
+	    if (dft->future && dft->future->transition_refs == dft)
+	      {
+		dft->future->transition_refs = dft->next_same_dest;
+		if (dft->future->transition_refs == dft)
+		  dft->future->transition_refs = 0;
+	      }
+	    dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+	    dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+	    rx_cache_free (cache, &cache->free_discernable_futures,
+			   (char *)dft);
+	  }
+	rx_cache_free (cache, &cache->free_transition_classes, (char *)tc);
+	tc = tct;
+      }
+  }
+  
+  if (it->contents->superstate == it)
+    it->contents->superstate = 0;
+  release_superset_low (cache, it->contents);
+  rx_cache_free (cache, &cache->free_superstates, (char *)it);
+  --cache->superstates;
+  return 1;
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get (struct rx_cache * cache,
+	      struct rx_freelist ** freelist)
+#else
+static char *
+rx_cache_get (cache, freelist)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+#endif
+{
+  while (!*freelist && rx_really_free_superstate (cache))
+    ;
+  if (!*freelist)
+    return 0;
+  {
+    struct rx_freelist * it = *freelist;
+    *freelist = it->next;
+    return (char *)it;
+  }
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc_or_get (struct rx_cache * cache,
+			struct rx_freelist ** freelist, int bytes)
+#else
+static char *
+rx_cache_malloc_or_get (cache, freelist, bytes)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     int bytes;
+#endif
+{
+  if (!*freelist)
+    {
+      char * answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	return answer;
+    }
+
+  return rx_cache_get (cache, freelist);
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get_superstate (struct rx_cache * cache)
+#else
+static char *
+rx_cache_get_superstate (cache)
+	  struct rx_cache * cache;
+#endif
+{
+  char * answer;
+  int bytes = (   sizeof (struct rx_superstate)
+	       +  cache->local_cset_size * sizeof (struct rx_inx));
+  if (!cache->free_superstates
+      && (cache->superstates < cache->superstates_allowed))
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	{
+	  ++cache->superstates;
+	  return answer;
+	}
+    }
+  answer = rx_cache_get (cache, &cache->free_superstates);
+  if (!answer)
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	++cache->superstates_allowed;
+    }
+  ++cache->superstates;
+  return answer;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+supersetcmp (void * va, void * vb)
+#else
+static int
+supersetcmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_superset * a = (struct rx_superset *)va;
+  struct rx_superset * b = (struct rx_superset *)vb;
+  return (   (a == b)
+	  || (a && b && (a->car == b->car) && (a->cdr == b->cdr)));
+}
+
+#ifdef __STDC__
+static struct rx_hash_item *
+superset_allocator (struct rx_hash_rules * rules, void * val)
+#else
+static struct rx_hash_item *
+superset_allocator (rules, val)
+     struct rx_hash_rules * rules;
+     void * val;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  struct rx_superset * template = (struct rx_superset *)val;
+  struct rx_superset * newset
+    = ((struct rx_superset *)
+       rx_cache_malloc_or_get (cache,
+			       &cache->free_supersets,
+			       sizeof (*template)));
+  if (!newset)
+    return 0;
+  newset->refs = 0;
+  newset->car = template->car;
+  newset->id = template->car->id;
+  newset->cdr = template->cdr;
+  newset->superstate = 0;
+  rx_protect_superset (rx, template->cdr);
+  newset->hash_item.data = (void *)newset;
+  newset->hash_item.binding = 0;
+  return &newset->hash_item;
+}
+
+#ifdef __STDC__
+static struct rx_hash * 
+super_hash_allocator (struct rx_hash_rules * rules)
+#else
+static struct rx_hash * 
+super_hash_allocator (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  return ((struct rx_hash *)
+	  rx_cache_malloc_or_get (cache,
+				  &cache->free_hash, sizeof (struct rx_hash)));
+}
+
+
+#ifdef __STDC__
+static void
+super_hash_liberator (struct rx_hash * hash, struct rx_hash_rules * rules)
+#else
+static void
+super_hash_liberator (hash, rules)
+     struct rx_hash * hash;
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       (char *)rules - (long)(&((struct rx_cache *)0)->superset_hash_rules));
+  rx_cache_free (cache, &cache->free_hash, (char *)hash);
+}
+
+#ifdef __STDC__
+static void
+superset_hash_item_liberator (struct rx_hash_item * it,
+			      struct rx_hash_rules * rules)
+#else
+static void
+superset_hash_item_liberator (it, rules) /* Well, it does ya know. */
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+}
+
+int rx_cache_bound = 128;
+static int rx_default_cache_got = 0;
+
+#ifdef __STDC__
+static int
+bytes_for_cache_size (int supers, int cset_size)
+#else
+static int
+bytes_for_cache_size (supers, cset_size)
+     int supers;
+     int cset_size;
+#endif
+{
+  /* What the hell is this? !!!*/
+  return (int)
+    ((float)supers *
+     (  (1.03 * (float) (  rx_sizeof_bitset (cset_size)
+			 + sizeof (struct rx_super_edge)))
+      + (1.80 * (float) sizeof (struct rx_possible_future))
+      + (float) (  sizeof (struct rx_superstate)
+		 + cset_size * sizeof (struct rx_inx))));
+}
+
+#ifdef __STDC__
+static void
+rx_morecore (struct rx_cache * cache)
+#else
+static void
+rx_morecore (cache)
+     struct rx_cache * cache;
+#endif
+{
+  if (rx_default_cache_got >= rx_cache_bound)
+    return;
+
+  rx_default_cache_got += 16;
+  cache->superstates_allowed = rx_cache_bound;
+  {
+    struct rx_blocklist ** pos = &cache->memory;
+    int size = bytes_for_cache_size (16, cache->local_cset_size);
+    while (*pos)
+      pos = &(*pos)->next;
+    *pos = ((struct rx_blocklist *)
+	    malloc (size + sizeof (struct rx_blocklist))); 
+    if (!*pos)
+      return;
+
+    (*pos)->next = 0;
+    (*pos)->bytes = size;
+    cache->memory_pos = *pos;
+    cache->memory_addr = (char *)*pos + sizeof (**pos);
+    cache->bytes_left = size;
+  }
+}
+
+static struct rx_cache default_cache = 
+{
+  {
+    supersetcmp,
+    super_hash_allocator,
+    super_hash_liberator,
+    superset_allocator,
+    superset_hash_item_liberator,
+  },
+  0,
+  0,
+  0,
+  0,
+  rx_morecore,
+
+  0,
+  0,
+  0,
+  0,
+  0,
+
+  0,
+  0,
+
+  0,
+
+  0,
+  0,
+  0,
+  0,
+  128,
+
+  256,
+  rx_id_instruction_table,
+
+  {
+    0,
+    0,
+    {0},
+    {0},
+    {0}
+  }
+};
+
+/* This adds an element to a superstate set.  These sets are lists, such
+ * that lists with == elements are ==.  The empty set is returned by
+ * superset_cons (rx, 0, 0) and is NOT equivelent to 
+ * (struct rx_superset)0.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superset_cons (struct rx * rx,
+		  struct rx_nfa_state *car, struct rx_superset *cdr)
+#else
+RX_DECL struct rx_superset *
+rx_superset_cons (rx, car, cdr)
+     struct rx * rx;
+     struct rx_nfa_state *car;
+     struct rx_superset *cdr;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  if (!car && !cdr)
+    {
+      if (!cache->empty_superset)
+	{
+	  cache->empty_superset
+	    = ((struct rx_superset *)
+	       rx_cache_malloc_or_get (cache, &cache->free_supersets,
+				       sizeof (struct rx_superset)));
+	  if (!cache->empty_superset)
+	    return 0;
+	  bzero (cache->empty_superset, sizeof (struct rx_superset));
+	  cache->empty_superset->refs = 1000;
+	}
+      return cache->empty_superset;
+    }
+  {
+    struct rx_superset template;
+    struct rx_hash_item * hit;
+    template.car = car;
+    template.cdr = cdr;
+    template.id = car->id;
+    hit = rx_hash_store (&cache->superset_table,
+			 (unsigned long)car ^ car->id ^ (unsigned long)cdr,
+			 (void *)&template,
+			 &cache->superset_hash_rules);
+    return (hit
+	    ?  (struct rx_superset *)hit->data
+	    : 0);
+  }
+}
+
+/* This computes a union of two NFA state sets.  The sets do not have the
+ * same representation though.  One is a RX_SUPERSET structure (part
+ * of the superstate NFA) and the other is an NFA_STATE_SET (part of the NFA).
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl) 
+#else
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union (rx, set, ecl)
+     struct rx * rx;
+     struct rx_superset *set;
+     struct rx_nfa_state_set *ecl;
+#endif
+{
+  if (!ecl)
+    return set;
+
+  if (!set->car)
+    return rx_superset_cons (rx, ecl->car,
+			     rx_superstate_eclosure_union (rx, set, ecl->cdr));
+  if (set->car == ecl->car)
+    return rx_superstate_eclosure_union (rx, set, ecl->cdr);
+
+  {
+    struct rx_superset * tail;
+    struct rx_nfa_state * first;
+
+    if (set->car > ecl->car)
+      {
+	tail = rx_superstate_eclosure_union (rx, set->cdr, ecl);
+	first = set->car;
+      }
+    else
+      {
+	tail = rx_superstate_eclosure_union (rx, set, ecl->cdr);
+	first = ecl->car;
+      }
+    if (!tail)
+      return 0;
+    else
+      {
+	struct rx_superset * answer;
+	answer = rx_superset_cons (rx, first, tail);
+	if (!answer)
+	  {
+	    rx_protect_superset (rx, tail);
+	    rx_release_superset (rx, tail);
+	    return 0;
+	  }
+	else
+	  return answer;
+      }
+  }
+}
+
+
+
+
+
+/*
+ * This makes sure that a list of rx_distinct_futures contains
+ * a future for each possible set of side effects in the eclosure
+ * of a given state.  This is some of the work of filling in a
+ * superstate transition. 
+ */
+
+#ifdef __STDC__
+static struct rx_distinct_future *
+include_futures (struct rx *rx,
+		 struct rx_distinct_future *df, struct rx_nfa_state
+		 *state, struct rx_superstate *superstate) 
+#else
+static struct rx_distinct_future *
+include_futures (rx, df, state, superstate)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+     struct rx_nfa_state *state;
+     struct rx_superstate *superstate;
+#endif
+{
+  struct rx_possible_future *future;
+  struct rx_cache * cache = rx->cache;
+  for (future = state->futures; future; future = future->next)
+    {
+      struct rx_distinct_future *dfp;
+      struct rx_distinct_future *insert_before = 0;
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      for (dfp = df; dfp; dfp = dfp->next_same_super_edge[0])
+	if (dfp->effects == future->effects)
+	  break;
+	else
+	  {
+	    int order = rx->se_list_cmp (rx, dfp->effects, future->effects);
+	    if (order > 0)
+	      {
+		insert_before = dfp;
+		dfp = 0;
+		break;
+	      }
+	  }
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+      if (!dfp)
+	{
+	  dfp
+	    = ((struct rx_distinct_future *)
+	       rx_cache_malloc_or_get (cache, &cache->free_discernable_futures,
+				       sizeof (struct rx_distinct_future)));
+	  if (!dfp)
+	    return 0;
+	  if (!df)
+	    {
+	      df = insert_before = dfp;
+	      df->next_same_super_edge[0] = df->next_same_super_edge[1] = df;
+	    }
+	  else if (!insert_before)
+	    insert_before = df;
+	  else if (insert_before == df)
+	    df = dfp;
+
+	  dfp->next_same_super_edge[0] = insert_before;
+	  dfp->next_same_super_edge[1]
+	    = insert_before->next_same_super_edge[1];
+	  dfp->next_same_super_edge[1]->next_same_super_edge[0] = dfp;
+	  dfp->next_same_super_edge[0]->next_same_super_edge[1] = dfp;
+	  dfp->next_same_dest = dfp->prev_same_dest = dfp;
+	  dfp->future = 0;
+	  dfp->present = superstate;
+	  dfp->future_frame.inx = rx->instruction_table[rx_cache_miss];
+	  dfp->future_frame.data = 0;
+	  dfp->future_frame.data_2 = (void *) dfp;
+	  dfp->side_effects_frame.inx
+	    = rx->instruction_table[rx_do_side_effects];
+	  dfp->side_effects_frame.data = 0;
+	  dfp->side_effects_frame.data_2 = (void *) dfp;
+	  dfp->effects = future->effects;
+	}
+    }
+  return df;
+}
+
+
+
+
+/* This constructs a new superstate from its state set.  The only 
+ * complexity here is memory management.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_superstate *
+rx_superstate (struct rx *rx,
+	       struct rx_superset *set)
+#else
+RX_DECL struct rx_superstate *
+rx_superstate (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  struct rx_superstate * superstate = 0;
+
+  /* Does the superstate already exist in the cache? */
+  if (set->superstate)
+    {
+      if (set->superstate->rx_id != rx->rx_id)
+	{
+	  /* Aha.  It is in the cache, but belongs to a superstate
+	   * that refers to an NFA that no longer exists.
+	   * (We know it no longer exists because it was evidently
+	   *  stored in the same region of memory as the current nfa
+	   *  yet it has a different id.)
+	   */
+	  superstate = set->superstate;
+	  if (!superstate->is_semifree)
+	    {
+	      if (cache->lru_superstate == superstate)
+		{
+		  cache->lru_superstate = superstate->next_recyclable;
+		  if (cache->lru_superstate == superstate)
+		    cache->lru_superstate = 0;
+		}
+	      {
+		superstate->next_recyclable->prev_recyclable
+		  = superstate->prev_recyclable;
+		superstate->prev_recyclable->next_recyclable
+		  = superstate->next_recyclable;
+		if (!cache->semifree_superstate)
+		  {
+		    (cache->semifree_superstate
+		     = superstate->next_recyclable
+		     = superstate->prev_recyclable
+		     = superstate);
+		  }
+		else
+		  {
+		    superstate->next_recyclable = cache->semifree_superstate;
+		    superstate->prev_recyclable
+		      = cache->semifree_superstate->prev_recyclable;
+		    superstate->next_recyclable->prev_recyclable
+		      = superstate;
+		    superstate->prev_recyclable->next_recyclable
+		      = superstate;
+		    cache->semifree_superstate = superstate;
+		  }
+		++cache->semifree_superstates;
+	      }
+	    }
+	  set->superstate = 0;
+	  goto handle_cache_miss;
+	}
+      ++cache->hits;
+      superstate = set->superstate;
+
+      rx_refresh_this_superstate (cache, superstate);
+      return superstate;
+    }
+
+ handle_cache_miss:
+
+  /* This point reached only for cache misses. */
+  ++cache->misses;
+#if RX_DEBUG
+  if (rx_debug_trace > 1)
+    {
+      struct rx_superset * setp = set;
+      fprintf (stderr, "Building a superstet %d(%d): ", rx->rx_id, set);
+      while (setp)
+	{
+	  fprintf (stderr, "%d ", setp->id);
+	  setp = setp->cdr;
+	}
+      fprintf (stderr, "(%d)\n", set);
+    }
+#endif
+  superstate = (struct rx_superstate *)rx_cache_get_superstate (cache);
+  if (!superstate)
+    return 0;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = superstate->next_recyclable
+     = superstate->prev_recyclable
+     = superstate);
+  else
+    {
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      (  superstate->prev_recyclable->next_recyclable
+       = superstate->next_recyclable->prev_recyclable
+       = superstate);
+    }
+  superstate->rx_id = rx->rx_id;
+  superstate->transition_refs = 0;
+  superstate->locks = 0;
+  superstate->is_semifree = 0;
+  set->superstate = superstate;
+  superstate->contents = set;
+  rx_protect_superset (rx, set);
+  superstate->edges = 0;
+  {
+    int x;
+    /* None of the transitions from this superstate are known yet. */
+    for (x = 0; x < rx->local_cset_size; ++x) /* &&&&& 3.8 % */
+      {
+	struct rx_inx * ifr = &superstate->transitions[x];
+	ifr->inx = rx->instruction_table [rx_cache_miss];
+	ifr->data = ifr->data_2 = 0;
+      }
+  }
+  return superstate;
+}
+
+
+
+/* This computes the destination set of one edge of the superstate NFA.
+ * Note that a RX_DISTINCT_FUTURE is a superstate edge.
+ * Returns 0 on an allocation failure.
+ */
+
+#ifdef __STDC__
+static int 
+solve_destination (struct rx *rx, struct rx_distinct_future *df)
+#else
+static int 
+solve_destination (rx, df)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc = df->edge;
+  struct rx_superset *nfa_state;
+  struct rx_superset *nil_set = rx_superset_cons (rx, 0, 0);
+  struct rx_superset *solution = nil_set;
+  struct rx_superstate *dest;
+
+  rx_protect_superset (rx, solution);
+  /* Iterate over all NFA states in the state set of this superstate. */
+  for (nfa_state = df->present->contents;
+       nfa_state->car;
+       nfa_state = nfa_state->cdr)
+    {
+      struct rx_nfa_edge *e;
+      /* Iterate over all edges of each NFA state. */
+      for (e = nfa_state->car->edges; e; e = e->next)
+        /* If we find an edge that is labeled with 
+	 * the characters we are solving for.....
+	 */
+	if (rx_bitset_is_subset (rx->local_cset_size,
+				 tc->cset, e->params.cset))
+	  {
+	    struct rx_nfa_state *n = e->dest;
+	    struct rx_possible_future *pf;
+	    /* ....search the partial epsilon closures of the destination
+	     * of that edge for a path that involves the same set of
+	     * side effects we are solving for.
+	     * If we find such a RX_POSSIBLE_FUTURE, we add members to the
+	     * stateset we are computing.
+	     */
+	    for (pf = n->futures; pf; pf = pf->next)
+	      if (pf->effects == df->effects)
+		{
+		  struct rx_superset * old_sol;
+		  old_sol = solution;
+		  solution = rx_superstate_eclosure_union (rx, solution,
+							   pf->destset);
+		  if (!solution)
+		    return 0;
+		  rx_protect_superset (rx, solution);
+		  rx_release_superset (rx, old_sol);
+		}
+	  }
+    }
+  /* It is possible that the RX_DISTINCT_FUTURE we are working on has 
+   * the empty set of NFA states as its definition.  In that case, this
+   * is a failure point.
+   */
+  if (solution == nil_set)
+    {
+      df->future_frame.inx = (void *) rx_backtrack;
+      df->future_frame.data = 0;
+      df->future_frame.data_2 = 0;
+      return 1;
+    }
+  dest = rx_superstate (rx, solution);
+  rx_release_superset (rx, solution);
+  if (!dest)
+    return 0;
+
+  {
+    struct rx_distinct_future *dft;
+    dft = df;
+    df->prev_same_dest->next_same_dest = 0;
+    while (dft)
+      {
+	dft->future = dest;
+	dft->future_frame.inx = rx->instruction_table[rx_next_char];
+	dft->future_frame.data = (void *) dest->transitions;
+	dft = dft->next_same_dest;
+      }
+    df->prev_same_dest->next_same_dest = df;
+  }
+  if (!dest->transition_refs)
+    dest->transition_refs = df;
+  else
+    {
+      struct rx_distinct_future *dft = dest->transition_refs->next_same_dest;
+      dest->transition_refs->next_same_dest = df->next_same_dest;
+      df->next_same_dest->prev_same_dest = dest->transition_refs;
+      df->next_same_dest = dft;
+      dft->prev_same_dest = df;
+    }
+  return 1;
+}
+
+
+/* This takes a superstate and a character, and computes some edges
+ * from the superstate NFA.  In particular, this computes all edges
+ * that lead from SUPERSTATE given CHR.   This function also 
+ * computes the set of characters that share this edge set.
+ * This returns 0 on allocation error.
+ * The character set and list of edges are returned through 
+ * the paramters CSETOUT and DFOUT.
+} */
+
+#ifdef __STDC__
+static int 
+compute_super_edge (struct rx *rx, struct rx_distinct_future **dfout,
+			  rx_Bitset csetout, struct rx_superstate *superstate,
+			  unsigned char chr)  
+#else
+static int 
+compute_super_edge (rx, dfout, csetout, superstate, chr)
+     struct rx *rx;
+     struct rx_distinct_future **dfout;
+     rx_Bitset csetout;
+     struct rx_superstate *superstate;
+     unsigned char chr;
+#endif
+{
+  struct rx_superset *stateset = superstate->contents;
+
+  /* To compute the set of characters that share edges with CHR, 
+   * we start with the full character set, and subtract.
+   */
+  rx_bitset_universe (rx->local_cset_size, csetout);
+  *dfout = 0;
+
+  /* Iterate over the NFA states in the superstate state-set. */
+  while (stateset->car)
+    {
+      struct rx_nfa_edge *e;
+      for (e = stateset->car->edges; e; e = e->next)
+	if (RX_bitset_member (e->params.cset, chr))
+	  {
+	    /* If we find an NFA edge that applies, we make sure there
+	     * are corresponding edges in the superstate NFA.
+	     */
+	    {
+	      struct rx_distinct_future * saved;
+	      saved = *dfout;
+	      *dfout = include_futures (rx, *dfout, e->dest, superstate);
+	      if (!*dfout)
+		{
+		  struct rx_distinct_future * df;
+		  df = saved;
+		  df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+		  while (df)
+		    {
+		      struct rx_distinct_future *dft;
+		      dft = df;
+		      df = df->next_same_super_edge[0];
+
+		      if (dft->future && dft->future->transition_refs == dft)
+			{
+			  dft->future->transition_refs = dft->next_same_dest;
+			  if (dft->future->transition_refs == dft)
+			    dft->future->transition_refs = 0;
+			}
+		      dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+		      dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+		      rx_cache_free (rx->cache,
+				     &rx->cache->free_discernable_futures,
+				     (char *)dft);
+		    }
+		  return 0;
+		}
+	    }
+	    /* We also trim the character set a bit. */
+	    rx_bitset_intersection (rx->local_cset_size,
+				    csetout, e->params.cset);
+	  }
+	else
+	  /* An edge that doesn't apply at least tells us some characters
+	   * that don't share the same edge set as CHR.
+	   */
+	  rx_bitset_difference (rx->local_cset_size, csetout, e->params.cset);
+      stateset = stateset->cdr;
+    }
+  return 1;
+}
+
+
+
+/* This is a constructor for RX_SUPER_EDGE structures.  These are
+ * wrappers for lists of superstate NFA edges that share character sets labels.
+ * If a transition class contains more than one rx_distinct_future (superstate
+ * edge), then it represents a non-determinism in the superstate NFA.
+ */
+
+#ifdef __STDC__
+static struct rx_super_edge *
+rx_super_edge (struct rx *rx,
+	       struct rx_superstate *super, rx_Bitset cset,
+	       struct rx_distinct_future *df) 
+#else
+static struct rx_super_edge *
+rx_super_edge (rx, super, cset, df)
+     struct rx *rx;
+     struct rx_superstate *super;
+     rx_Bitset cset;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc =
+    (struct rx_super_edge *)rx_cache_malloc_or_get
+      (rx->cache, &rx->cache->free_transition_classes,
+       sizeof (struct rx_super_edge) + rx_sizeof_bitset (rx->local_cset_size));
+
+  if (!tc)
+    return 0;
+  tc->next = super->edges;
+  super->edges = tc;
+  tc->rx_backtrack_frame.inx = rx->instruction_table[rx_backtrack_point];
+  tc->rx_backtrack_frame.data = 0;
+  tc->rx_backtrack_frame.data_2 = (void *) tc;
+  tc->options = df;
+  tc->cset = (rx_Bitset) ((char *) tc + sizeof (*tc));
+  rx_bitset_assign (rx->local_cset_size, tc->cset, cset);
+  if (df)
+    {
+      struct rx_distinct_future * dfp = df;
+      df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      while (dfp)
+	{
+	  dfp->edge = tc;
+	  dfp = dfp->next_same_super_edge[0];
+	}
+      df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+    }
+  return tc;
+}
+
+
+/* There are three kinds of cache miss.  The first occurs when a
+ * transition is taken that has never been computed during the
+ * lifetime of the source superstate.  That cache miss is handled by
+ * calling COMPUTE_SUPER_EDGE.  The second kind of cache miss
+ * occurs when the destination superstate of a transition doesn't
+ * exist.  SOLVE_DESTINATION is used to construct the destination superstate.
+ * Finally, the third kind of cache miss occurs when the destination
+ * superstate of a transition is in a `semi-free state'.  That case is
+ * handled by UNFREE_SUPERSTATE.
+ *
+ * The function of HANDLE_CACHE_MISS is to figure out which of these
+ * cases applies.
+ */
+
+#ifdef __STDC__
+static void
+install_partial_transition  (struct rx_superstate *super,
+			     struct rx_inx *answer,
+			     RX_subset set, int offset)
+#else
+static void
+install_partial_transition  (super, answer, set, offset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     RX_subset set;
+     int offset;
+#endif
+{
+  int start = offset;
+  int end = start + 32;
+  RX_subset pos = 1;
+  struct rx_inx * transitions = super->transitions;
+  
+  while (start < end)
+    {
+      if (set & pos)
+	transitions[start] = *answer;
+      pos <<= 1;
+      ++start;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_inx *
+rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data) 
+#else
+RX_DECL struct rx_inx *
+rx_handle_cache_miss (rx, super, chr, data)
+     struct rx *rx;
+     struct rx_superstate *super;
+     unsigned char chr;
+     void *data;
+#endif
+{
+  int offset = chr / RX_subset_bits;
+  struct rx_distinct_future *df = data;
+
+  if (!df)			/* must be the shared_cache_miss_frame */
+    {
+      /* Perhaps this is just a transition waiting to be filled. */
+      struct rx_super_edge *tc;
+      RX_subset mask = rx_subset_singletons [chr % RX_subset_bits];
+
+      for (tc = super->edges; tc; tc = tc->next)
+	if (tc->cset[offset] & mask)
+	  {
+	    struct rx_inx * answer;
+	    df = tc->options;
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	    install_partial_transition (super, answer,
+					tc->cset [offset], offset * 32);
+	    return answer;
+	  }
+      /* Otherwise, it's a flushed or  newly encountered edge. */
+      {
+	char cset_space[1024];	/* this limit is far from unreasonable */
+	rx_Bitset trcset;
+	struct rx_inx *answer;
+
+	if (rx_sizeof_bitset (rx->local_cset_size) > sizeof (cset_space))
+	  return 0;		/* If the arbitrary limit is hit, always fail */
+				/* cleanly. */
+	trcset = (rx_Bitset)cset_space;
+	rx_lock_superstate (rx, super);
+	if (!compute_super_edge (rx, &df, trcset, super, chr))
+	  {
+	    rx_unlock_superstate (rx, super);
+	    return 0;
+	  }
+	if (!df)		/* We just computed the fail transition. */
+	  {
+	    static struct rx_inx
+	      shared_fail_frame = { 0, 0, (void *)rx_backtrack, 0 };
+	    answer = &shared_fail_frame;
+	  }
+	else
+	  {
+	    tc = rx_super_edge (rx, super, trcset, df);
+	    if (!tc)
+	      {
+		rx_unlock_superstate (rx, super);
+		return 0;
+	      }
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	  }
+	install_partial_transition (super, answer,
+				    trcset[offset], offset * 32);
+	rx_unlock_superstate (rx, super);
+	return answer;
+      }
+    }
+  else if (df->future) /* A cache miss on an edge with a future? Must be
+			* a semi-free destination. */
+    {				
+      if (df->future->is_semifree)
+	refresh_semifree_superstate (rx->cache, df->future);
+      return &df->future_frame;
+    }
+  else
+    /* no future superstate on an existing edge */
+    {
+      rx_lock_superstate (rx, super);
+      if (!solve_destination (rx, df))
+	{
+	  rx_unlock_superstate (rx, super);
+	  return 0;
+	}
+      if (!df->effects
+	  && (df->edge->options->next_same_super_edge[0] == df->edge->options))
+	install_partial_transition (super, &df->future_frame,
+				    df->edge->cset[offset], offset * 32);
+      rx_unlock_superstate (rx, super);
+      return &df->future_frame;
+    }
+}
+
+
+
+
+
+/* The rest of the code provides a regex.c compatable interface. */
+
+
+__const__ char *re_error_msg[] =
+{
+  0,						/* REG_NOUT */
+  "No match",					/* REG_NOMATCH */
+  "Invalid regular expression",			/* REG_BADPAT */
+  "Invalid collation character",		/* REG_ECOLLATE */
+  "Invalid character class name",		/* REG_ECTYPE */
+  "Trailing backslash",				/* REG_EESCAPE */
+  "Invalid back reference",			/* REG_ESUBREG */
+  "Unmatched [ or [^",				/* REG_EBRACK */
+  "Unmatched ( or \\(",				/* REG_EPAREN */
+  "Unmatched \\{",				/* REG_EBRACE */
+  "Invalid content of \\{\\}",			/* REG_BADBR */
+  "Invalid range end",				/* REG_ERANGE */
+  "Memory exhausted",				/* REG_ESPACE */
+  "Invalid preceding regular expression",	/* REG_BADRPT */
+  "Premature end of regular expression",	/* REG_EEND */
+  "Regular expression too big",			/* REG_ESIZE */
+  "Unmatched ) or \\)",				/* REG_ERPAREN */
+};
+
+
+
+
+/* 
+ * Macros used while compiling patterns.
+ *
+ * By convention, PEND points just past the end of the uncompiled pattern,
+ * P points to the read position in the pattern.  `translate' is the name
+ * of the translation table (`TRANSLATE' is the name of a macro that looks
+ * things up in `translate').
+ */
+
+
+/*
+ * Fetch the next character in the uncompiled pattern---translating it 
+ * if necessary. *Also cast from a signed character in the constant
+ * string passed to us by the user to an unsigned char that we can use
+ * as an array index (in, e.g., `translate').
+ */
+#define PATFETCH(c)							\
+ do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    c = translate[c];		 					\
+ } while (0)
+
+/* 
+ * Fetch the next character in the uncompiled pattern, with no
+ * translation.
+ */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+#define TRANSLATE(d) translate[(unsigned char) (d)]
+
+typedef unsigned regnum_t;
+
+/* Since offsets can go either forwards or backwards, this type needs to
+ * be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.
+ */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  struct rexp_node ** top_expression; /* was begalt */
+  struct rexp_node ** last_expression; /* was laststart */
+  pattern_offset_t inner_group_offset;
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / CHARBITS]               \
+   |= 1 << (((unsigned char) c) % CHARBITS))
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (isdigit (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (!strcmp (string, "alpha") || !strcmp (string, "upper")		\
+    || !strcmp (string, "lower") || !strcmp (string, "digit")		\
+    || !strcmp (string, "alnum") || !strcmp (string, "xdigit")		\
+    || !strcmp (string, "space") || !strcmp (string, "print")		\
+    || !strcmp (string, "punct") || !strcmp (string, "graph")		\
+    || !strcmp (string, "cntrl") || !strcmp (string, "blank"))
+
+
+
+/* These predicates are used in regex_compile. */
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+ * after an alternative or a begin-subexpression.  We assume there is at
+ * least one character before the ^.  
+ */
+
+#ifdef __STDC__
+static boolean
+at_begline_loc_p (__const__ char *pattern, __const__ char * p, reg_syntax_t syntax)
+#else
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+     __const__ char *pattern;
+     __const__ char * p;
+     reg_syntax_t syntax;
+#endif
+{
+  __const__ char *prev = p - 2;
+  boolean prev_prev_backslash = ((prev > pattern) && (prev[-1] == '\\'));
+  
+    return
+      
+      (/* After a subexpression?  */
+       ((*prev == '(') && ((syntax & RE_NO_BK_PARENS) || prev_prev_backslash))
+       ||
+       /* After an alternative?  */
+       ((*prev == '|') && ((syntax & RE_NO_BK_VBAR) || prev_prev_backslash))
+       );
+}
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+ * at least one character after the $, i.e., `P < PEND'.
+ */
+
+#ifdef __STDC__
+static boolean
+at_endline_loc_p (__const__ char *p, __const__ char *pend, int syntax)
+#else
+static boolean
+at_endline_loc_p (p, pend, syntax)
+     __const__ char *p;
+     __const__ char *pend;
+     int syntax;
+#endif
+{
+  __const__ char *next = p;
+  boolean next_backslash = (*next == '\\');
+  __const__ char *next_next = (p + 1 < pend) ? (p + 1) : 0;
+  
+  return
+    (
+     /* Before a subexpression?  */
+     ((syntax & RE_NO_BK_PARENS)
+      ? (*next == ')')
+      : (next_backslash && next_next && (*next_next == ')')))
+    ||
+     /* Before an alternative?  */
+     ((syntax & RE_NO_BK_VBAR)
+      ? (*next == '|')
+      : (next_backslash && next_next && (*next_next == '|')))
+     );
+}
+
+
+
+unsigned char rx_id_translation[256] =
+{
+  0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+ 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+ 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+ 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+ 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+ 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
+ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
+ 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
+
+ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
+ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249,
+ 250, 251, 252, 253, 254, 255
+};
+
+/* The compiler keeps an inverted translation table.
+ * This looks up/inititalize elements.
+ * VALID is an array of booleans that validate CACHE.
+ */
+
+#ifdef __STDC__
+static rx_Bitset
+inverse_translation (struct re_pattern_buffer * rxb,
+		     char * valid, rx_Bitset cache,
+		     unsigned char * translate, int c)
+#else
+static rx_Bitset
+inverse_translation (rxb, valid, cache, translate, c)
+     struct re_pattern_buffer * rxb;
+     char * valid;
+     rx_Bitset cache;
+     unsigned char * translate;
+     int c;
+#endif
+{
+  rx_Bitset cs
+    = cache + c * rx_bitset_numb_subsets (rxb->rx.local_cset_size); 
+
+  if (!valid[c])
+    {
+      int x;
+      int c_tr = TRANSLATE(c);
+      rx_bitset_null (rxb->rx.local_cset_size, cs);
+      for (x = 0; x < 256; ++x)	/* &&&& 13.37 */
+	if (TRANSLATE(x) == c_tr)
+	  RX_bitset_enjoin (cs, x);
+      valid[c] = 1;
+    }
+  return cs;
+}
+
+
+
+
+
+/* More subroutine declarations and macros for regex_compile.  */
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+#ifdef __STDC__
+static boolean
+group_in_compile_stack (compile_stack_type compile_stack, regnum_t regnum)
+#else
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+#endif
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/*
+ * Read the ending character of a range (in a bracket expression) from the
+ * uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+ * starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+ * Then we set the translation of all bits between the starting and
+ * ending characters (inclusive) in the compiled pattern B.
+ * 
+ * Return an error code.
+ * 
+ * We use these short variable names so we can use the same macros as
+ * `regex_compile' itself.  
+ */
+
+#ifdef __STDC__
+static reg_errcode_t
+compile_range (struct re_pattern_buffer * rxb, rx_Bitset cs,
+	       __const__ char ** p_ptr, __const__ char * pend,
+	       unsigned char * translate, reg_syntax_t syntax,
+	       rx_Bitset inv_tr,  char * valid_inv_tr)
+#else
+static reg_errcode_t
+compile_range (rxb, cs, p_ptr, pend, translate, syntax, inv_tr, valid_inv_tr)
+     struct re_pattern_buffer * rxb;
+     rx_Bitset cs;
+     __const__ char ** p_ptr;
+     __const__ char * pend;
+     unsigned char * translate;
+     reg_syntax_t syntax;
+     rx_Bitset inv_tr;
+     char * valid_inv_tr;
+#endif
+{
+  unsigned this_char;
+
+  __const__ char *p = *p_ptr;
+
+  unsigned char range_end;
+  unsigned char range_start = TRANSLATE(p[-2]);
+
+  if (p == pend)
+    return REG_ERANGE;
+
+  PATFETCH (range_end);
+
+  (*p_ptr)++;
+
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      rx_Bitset it =
+	inverse_translation (rxb, valid_inv_tr, inv_tr, translate, this_char);
+      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+    }
+  
+  return REG_NOERROR;
+}
+
+
+
+/* This searches a regexp for backreference side effects.
+ * It fills in the array OUT with 1 at the index of every register pair
+ * referenced by a backreference.
+ *
+ * This is used to help optimize patterns for searching.  The information is
+ * useful because, if the caller doesn't want register values, backreferenced
+ * registers are the only registers for which we need rx_backtrack.
+ */
+
+#ifdef __STDC__
+static void
+find_backrefs (char * out, struct rexp_node * rexp,
+	       struct re_se_params * params)
+#else
+static void
+find_backrefs (out, rexp, params)
+     char * out;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (rexp)
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return;
+      case r_alternate:
+      case r_concat:
+      case r_opt:
+      case r_star:
+      case r_2phase_star:
+	find_backrefs (out, rexp->params.pair.left, params);
+	find_backrefs (out, rexp->params.pair.right, params);
+	return;
+      case r_side_effect:
+	if (   ((long)rexp->params.side_effect >= 0)
+	    && (params [(long)rexp->params.side_effect].se == re_se_backref))
+	  out[ params [(long)rexp->params.side_effect].op1] = 1;
+	return;
+      }
+}
+
+
+
+
+/* Returns 0 unless the pattern can match the empty string. */
+
+#ifdef __STDC__
+static int
+compute_fastset (struct re_pattern_buffer * rxb, struct rexp_node * rexp)
+#else
+static int
+compute_fastset (rxb, rexp)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 1;
+  switch (rexp->type)
+    {
+    case r_data:
+      return 1;
+    case r_cset:
+      {
+	rx_bitset_union (rxb->rx.local_cset_size,
+			 rxb->fastset, rexp->params.cset);
+      }
+      return 0;
+    case r_concat:
+      return (compute_fastset (rxb, rexp->params.pair.left)
+	      && compute_fastset (rxb, rexp->params.pair.right));
+    case r_2phase_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      /* compute_fastset (rxb, rexp->params.pair.right);  nope... */
+      return 1;
+    case r_alternate:
+      return !!(compute_fastset (rxb, rexp->params.pair.left)
+		+ compute_fastset (rxb, rexp->params.pair.right));
+    case r_opt:
+    case r_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      return 1;
+    case r_side_effect:
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* returns
+ *  1 -- yes, definately anchored by the given side effect.
+ *  2 -- maybe anchored, maybe the empty string.
+ *  0 -- definately not anchored
+ *  There is simply no other possibility.
+ */
+
+#ifdef __STDC__
+static int
+is_anchored (struct rexp_node * rexp, rx_side_effect se)
+#else
+static int
+is_anchored (rexp, se)
+     struct rexp_node * rexp;
+     rx_side_effect se;
+#endif
+{
+  if (!rexp)
+    return 2;
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+    case r_concat:
+    case r_2phase_star:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	return (l == 2 ? is_anchored (rexp->params.pair.right, se) : l);
+      }
+    case r_alternate:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	int r = l ? is_anchored (rexp->params.pair.right, se) : 0;
+
+	if (l == r)
+	  return l;
+	else if ((l == 0) || (r == 0))
+	  return 0;
+	else
+	  return 2;
+      }
+    case r_opt:
+    case r_star:
+      return is_anchored (rexp->params.pair.left, se) ? 2 : 0;
+      
+    case r_side_effect:
+      return ((rexp->params.side_effect == se)
+	      ? 1 : 2);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This removes register assignments that aren't required by backreferencing.
+ * This can speed up explore_future, especially if it eliminates
+ * non-determinism in the superstate NFA.
+ * 
+ * NEEDED is an array of characters, presumably filled in by FIND_BACKREFS.
+ * The non-zero elements of the array indicate which register assignments
+ * can NOT be removed from the expression.
+ */
+
+#ifdef __STDC__
+static struct rexp_node *
+remove_unecessary_side_effects (struct rx * rx, char * needed,
+				struct rexp_node * rexp,
+				struct re_se_params * params)
+#else
+static struct rexp_node *
+remove_unecessary_side_effects (rx, needed, rexp, params)
+     struct rx * rx;
+     char * needed;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  struct rexp_node * l;
+  struct rexp_node * r;
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return rexp;
+      case r_alternate:
+      case r_concat:
+      case r_2phase_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	r = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.right, params);
+	if ((l && r) || (rexp->type != r_concat))
+	  {
+	    rexp->params.pair.left = l;
+	    rexp->params.pair.right = r;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = rexp->params.pair.right = 0;
+	    rx_free_rexp (rx, rexp);
+	    return l ? l : r;
+	  }
+      case r_opt:
+      case r_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	if (l)
+	  {
+	    rexp->params.pair.left = l;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = 0;
+	    rx_free_rexp (rx, rexp);
+	    return 0;
+	  }
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   (se >= 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen))
+	      && (params [se].op1 > 0)
+	      && (!needed [params [se].op1]))
+	    {
+	      rx_free_rexp (rx, rexp);
+	      return 0;
+	    }
+	  else
+	    return rexp;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+pointless_if_repeated (struct rexp_node * node, struct re_se_params * params)
+#else
+static int
+pointless_if_repeated (node, params)
+     struct rexp_node * node;
+     struct re_se_params * params;
+#endif
+{
+  if (!node)
+    return 1;
+  switch (node->type)
+    {
+    case r_cset:
+      return 0;
+    case r_alternate:
+    case r_concat:
+    case r_2phase_star:
+      return (pointless_if_repeated (node->params.pair.left, params)
+	      && pointless_if_repeated (node->params.pair.right, params));
+    case r_opt:
+    case r_star:
+      return pointless_if_repeated (node->params.pair.left, params);
+    case r_side_effect:
+      switch ((enum re_side_effects) (((long)node->params.side_effect < 0)
+	      ? (enum re_side_effects) (long) node->params.side_effect
+	      : (enum re_side_effects)params[(long)node->params.side_effect].se))
+	{
+	case re_se_try:
+	case re_se_at_dot:
+	case re_se_begbuf:
+	case re_se_hat:
+	case re_se_wordbeg:
+	case re_se_wordbound:
+	case re_se_notwordbound:
+	case re_se_wordend:
+	case re_se_endbuf:
+	case re_se_dollar:
+	case re_se_fail:
+	case re_se_win:
+	  return 1;
+	case re_se_lparen:
+	case re_se_rparen:
+	case re_se_iter:
+	case re_se_end_iter:
+	case re_se_syntax:
+	case re_se_not_syntax:
+	case re_se_backref:
+	  return 0;
+	}
+    case r_data:
+    default:
+      return 0;
+    }
+}
+
+
+
+
+#ifdef __STDC__
+static int
+registers_on_stack (struct re_pattern_buffer * rxb,
+		    struct rexp_node * rexp, int in_danger,
+		    struct re_se_params * params)
+#else
+static int
+registers_on_stack (rxb, rexp, in_danger, params)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+     int in_danger;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return 0;
+      case r_alternate:
+      case r_concat:
+	return (   registers_on_stack (rxb, rexp->params.pair.left,
+				       in_danger, params)
+		|| (registers_on_stack
+		    (rxb, rexp->params.pair.right,
+		     in_danger, params)));
+      case r_opt:
+	return registers_on_stack (rxb, rexp->params.pair.left, 0, params);
+      case r_star:
+	return registers_on_stack (rxb, rexp->params.pair.left, 1, params);
+      case r_2phase_star:
+	return
+	  (   registers_on_stack (rxb, rexp->params.pair.left, 1, params)
+	   || registers_on_stack (rxb, rexp->params.pair.right, 1, params));
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   in_danger
+	      && (se >= 0)
+	      && (params [se].op1 > 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen)))
+	    return 1;
+	  else
+	    return 0;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+static char idempotent_complex_se[] =
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     IDEM,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  23
+};
+
+static char idempotent_se[] =
+{
+  13,
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      IDEM,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  42
+};
+
+
+
+
+
+#ifdef __STDC__
+static int
+has_any_se (struct rx * rx,
+	    struct rexp_node * rexp)
+#else
+static int
+has_any_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return 1;
+      
+    case r_2phase_star:
+    case r_concat:
+    case r_alternate:
+      return
+	(   has_any_se (rx, rexp->params.pair.left)
+	 || has_any_se (rx, rexp->params.pair.right));
+
+    case r_opt:
+    case r_star:
+      return has_any_se (rx, rexp->params.pair.left);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This must be called AFTER `convert_hard_loops' for a given REXP. */
+#ifdef __STDC__
+static int
+has_non_idempotent_epsilon_path (struct rx * rx,
+				 struct rexp_node * rexp,
+				 struct re_se_params * params)
+#else
+static int
+has_non_idempotent_epsilon_path (rx, rexp, params)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_star:
+      return 0;
+
+    case r_side_effect:
+      return
+	!((long)rexp->params.side_effect > 0
+	  ? idempotent_complex_se [ params [(long)rexp->params.side_effect].se ]
+	  : idempotent_se [-(long)rexp->params.side_effect]);
+      
+    case r_alternate:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 || has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_2phase_star:
+    case r_concat:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 && has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_opt:
+      return has_non_idempotent_epsilon_path (rx,
+					      rexp->params.pair.left, params);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This computes rougly what it's name suggests.   It can (and does) go wrong 
+ * in the direction of returning spurious 0 without causing disasters.
+ */
+#ifdef __STDC__
+static int
+begins_with_complex_se (struct rx * rx, struct rexp_node * rexp)
+#else
+static int
+begins_with_complex_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return ((long)rexp->params.side_effect >= 0);
+      
+    case r_alternate:
+      return
+	(   begins_with_complex_se (rx, rexp->params.pair.left)
+	 && begins_with_complex_se (rx, rexp->params.pair.right));
+
+
+    case r_concat:
+      return has_any_se (rx, rexp->params.pair.left);
+    case r_opt:
+    case r_star:
+    case r_2phase_star:
+      return 0;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This destructively removes some of the re_se_tv side effects from 
+ * a rexp tree.  In particular, during parsing re_se_tv was inserted on the
+ * right half of every | to guarantee that posix path preference could be 
+ * honored.  This function removes some which it can be determined aren't 
+ * needed.  
+ */
+
+#ifdef __STDC__
+static void
+speed_up_alt (struct rx * rx,
+	      struct rexp_node * rexp,
+	      int unposix)
+#else
+static void
+speed_up_alt (rx, rexp, unposix)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     int unposix;
+#endif
+{
+  if (!rexp)
+    return;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_side_effect:
+      return;
+
+    case r_opt:
+    case r_star:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      return;
+
+    case r_2phase_star:
+    case r_concat:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right, unposix);
+      return;
+
+    case r_alternate:
+      /* the right child is guaranteed to be (concat re_se_tv <subexp>) */
+
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right->params.pair.right, unposix);
+      
+      if (   unposix
+	  || (begins_with_complex_se
+	      (rx, rexp->params.pair.right->params.pair.right))
+	  || !(   has_any_se (rx, rexp->params.pair.right->params.pair.right)
+	       || has_any_se (rx, rexp->params.pair.left)))
+	{
+	  struct rexp_node * conc = rexp->params.pair.right;
+	  rexp->params.pair.right = conc->params.pair.right;
+	  conc->params.pair.right = 0;
+	  rx_free_rexp (rx, conc);
+	}
+    }
+}
+
+
+
+
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is set to zero;
+     `re_nsub' is set to the number of groups in PATTERN;
+     `not_bol' and `not_eol' are set to zero.
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+
+
+#ifdef __STDC__
+RX_DECL reg_errcode_t
+rx_compile (__const__ char *pattern, int size,
+	    reg_syntax_t syntax,
+	    struct re_pattern_buffer * rxb) 
+#else
+RX_DECL reg_errcode_t
+rx_compile (pattern, size, syntax, rxb)
+     __const__ char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  RX_subset
+    inverse_translate [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+  char
+    validate_inv_tr [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random tempory spot in PATTERN.  */
+  __const__ char *p1;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  __const__ char *p = pattern;
+  __const__ char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  unsigned char *translate = (rxb->translate
+			      ? rxb->translate
+			      : rx_id_translation);
+
+  /* When parsing is done, this will hold the expression tree. */
+  struct rexp_node * rexp = 0;
+
+  /* In the midst of compilation, this holds onto the regexp 
+   * first parst while rexp goes on to aquire additional constructs.
+   */
+  struct rexp_node * orig_rexp = 0;
+  struct rexp_node * fewer_side_effects = 0;
+
+  /* This and top_expression are saved on the compile stack. */
+  struct rexp_node ** top_expression = &rexp;
+  struct rexp_node ** last_expression = top_expression;
+  
+  /* Parameter to `goto append_node' */
+  struct rexp_node * append;
+
+  /* Counts open-groups as they are encountered.  This is the index of the
+   * innermost group being compiled.
+   */
+  regnum_t regnum = 0;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+   * which to go back if the interval is invalid.  
+   */
+  __const__ char *beg_interval;
+
+  struct re_se_params * params = 0;
+  int paramc = 0;		/* How many complex side effects so far? */
+
+  rx_side_effect side;		/* param to `goto add_side_effect' */
+
+  bzero (validate_inv_tr, sizeof (validate_inv_tr));
+
+  rxb->rx.instruction_table = rx_id_instruction_table;
+
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack =  (( compile_stack_elt_t *) malloc ((INIT_COMPILE_STACK_SIZE) * sizeof ( compile_stack_elt_t)));
+  if (compile_stack.stack == 0)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  rxb->rx.cache = &default_cache;
+  rxb->syntax = syntax;
+  rxb->fastmap_accurate = 0;
+  rxb->not_bol = rxb->not_eol = 0;
+  rxb->least_subs = 0;
+  
+  /* Always count groups, whether or not rxb->no_sub is set.  
+   * The whole pattern is implicitly group 0, so counting begins
+   * with 1.
+   */
+  rxb->re_nsub = 0;
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_hat);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_dollar);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (pointless_if_repeated (*last_expression, params))
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                return REG_BADRPT;
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!last_expression)
+              break;
+
+	    /* Now we know whether or not zero matches is allowed
+	     * and also whether or not two or more matches is allowed.
+	     */
+
+	    {
+	      struct rexp_node * inner_exp = *last_expression;
+	      int need_sync = 0;
+
+	      if (many_times_ok
+		  && has_non_idempotent_epsilon_path (&rxb->rx,
+						      inner_exp, params))
+		{
+		  struct rexp_node * pusher
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushpos);
+		  struct rexp_node * checker
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_chkpos);
+		  struct rexp_node * pushback
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushback);
+		  rx_Bitset cs = rx_cset (&rxb->rx);
+		  struct rexp_node * lit_t = rx_mk_r_cset (&rxb->rx, cs);
+		  struct rexp_node * fake_state
+		    = rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		  struct rexp_node * phase2
+		    = rx_mk_r_concat (&rxb->rx, checker, fake_state);
+		  struct rexp_node * popper
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_poppos);
+		  struct rexp_node * star
+		    = rx_mk_r_2phase_star (&rxb->rx, inner_exp, phase2);
+		  struct rexp_node * a
+		    = rx_mk_r_concat (&rxb->rx, pusher, star);
+		  struct rexp_node * whole_thing
+		    = rx_mk_r_concat (&rxb->rx, a, popper);
+		  if (!(pusher && star && pushback && lit_t && fake_state
+			&& lit_t && phase2 && checker && popper
+			&& a && whole_thing))
+		    return REG_ESPACE;
+		  RX_bitset_enjoin (cs, 't');
+		  *last_expression = whole_thing;
+		}
+	      else
+		{
+		  struct rexp_node * star =
+		    (many_times_ok ? rx_mk_r_star : rx_mk_r_opt)
+		      (&rxb->rx, *last_expression);
+		  if (!star)
+		    return REG_ESPACE;
+		  *last_expression = star;
+		  need_sync = has_any_se (&rxb->rx, *last_expression);
+		}
+	      if (!zero_times_ok)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, inner_exp,
+				      rx_copy_rexp (&rxb->rx,
+						    *last_expression));
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		}
+	      if (need_sync)
+		{
+		  int sync_se = paramc;
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					sizeof (*params) * (1 + paramc)))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (*params))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+		  params [sync_se].se = re_se_tv;
+		  side = (rx_side_effect)sync_se;
+		  goto add_side_effect;
+		}
+	    }
+	    /* The old regex.c used to optimize `.*\n'.  
+	     * Maybe rx should too?
+	     */
+	  }
+	  break;
+
+
+	case '.':
+	  {
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * n = rx_mk_r_cset (&rxb->rx, cs);
+	    if (!(cs && n))
+	      return REG_ESPACE;
+
+	    rx_bitset_universe (rxb->rx.local_cset_size, cs);
+	    if (!(rxb->syntax & RE_DOT_NEWLINE))
+	      RX_bitset_remove (cs, '\n');
+	    if (!(rxb->syntax & RE_DOT_NOT_NULL))
+	      RX_bitset_remove (cs, 0);
+
+	    append = n;
+	    goto append_node;
+	    break;
+	  }
+
+
+        case '[':
+	  if (p == pend) return REG_EBRACK;
+          {
+            boolean had_char_class = false;
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * node = rx_mk_r_cset (&rxb->rx, cs);
+	    int is_inverted = *p == '^';
+	    
+	    if (!(node && cs))
+	      return REG_ESPACE;
+	    
+	    /* This branch of the switch is normally exited with
+	     *`goto append_node'
+	     */
+	    append = node;
+	    
+            if (is_inverted)
+	      p++;
+	    
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+	    
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) return REG_EBRACK;
+		
+                PATFETCH (c);
+		
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+		    
+                    PATFETCH (c1);
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c1);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                    continue;
+                  }
+		
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  goto finalize_class_and_append;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  return REG_ERANGE;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (rxb, cs, &p, pend, translate, syntax,
+				       inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+		    
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (rxb, cs, &p, pend, translate, syntax,
+					 inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                /* See if we're at the beginning of a possible character
+                   class.  */
+		
+		else if ((syntax & RE_CHAR_CLASSES)
+			 && (c == '[') && (*p == ':'))
+                  {
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+		    
+                    PATFETCH (c);
+                    c1 = 0;
+		    
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) return REG_EBRACK;
+		    
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+			  break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+		    
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = !strcmp (str, "alnum");
+                        boolean is_alpha = !strcmp (str, "alpha");
+                        boolean is_blank = !strcmp (str, "blank");
+                        boolean is_cntrl = !strcmp (str, "cntrl");
+                        boolean is_digit = !strcmp (str, "digit");
+                        boolean is_graph = !strcmp (str, "graph");
+                        boolean is_lower = !strcmp (str, "lower");
+                        boolean is_print = !strcmp (str, "print");
+                        boolean is_punct = !strcmp (str, "punct");
+                        boolean is_space = !strcmp (str, "space");
+                        boolean is_upper = !strcmp (str, "upper");
+                        boolean is_xdigit = !strcmp (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str)) return REG_ECTYPE;
+			
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+			
+                        if (p == pend) return REG_EBRACK;
+			
+                        for (ch = 0; ch < 1 << CHARBITS; ch++)
+                          {
+                            if (   (is_alnum  && isalnum (ch))
+                                || (is_alpha  && isalpha (ch))
+                                || (is_blank  && isblank (ch))
+                                || (is_cntrl  && iscntrl (ch))
+                                || (is_digit  && isdigit (ch))
+                                || (is_graph  && isgraph (ch))
+                                || (is_lower  && islower (ch))
+                                || (is_print  && isprint (ch))
+                                || (is_punct  && ispunct (ch))
+                                || (is_space  && isspace (ch))
+                                || (is_upper  && isupper (ch))
+                                || (is_xdigit && isxdigit (ch)))
+			      {
+				rx_Bitset it =
+				  inverse_translation (rxb, 
+						       validate_inv_tr,
+						       inverse_translate,
+						       translate,
+						       ch);
+				rx_bitset_union (rxb->rx.local_cset_size,
+						 cs, it);
+			      }
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 '[');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 ':');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                  }
+              }
+
+	  finalize_class_and_append:
+	    if (is_inverted)
+	      {
+		rx_bitset_complement (rxb->rx.local_cset_size, cs);
+		if (syntax & RE_HAT_LISTS_NOT_NEWLINE)
+		  RX_bitset_remove (cs, '\n');
+	      }
+	    goto append_node;
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+	  if ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+	    goto handle_interval;
+	  else
+	    goto normal_char;
+
+
+        case '\\':
+          if (p == pend) return REG_EESCAPE;
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              rxb->re_nsub++;
+              regnum++;
+              if (COMPILE_STACK_FULL)
+                { 
+                  ((compile_stack.stack) =
+		   (compile_stack_elt_t *) realloc (compile_stack.stack, ( compile_stack.size << 1) * sizeof (
+													      compile_stack_elt_t)));
+                  if (compile_stack.stack == 0) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, 0);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+
+              /*
+	       * These are the values to restore when we hit end of this
+               * group.  
+	       */
+	      COMPILE_STACK_TOP.top_expression = top_expression;
+	      COMPILE_STACK_TOP.last_expression = last_expression;
+              COMPILE_STACK_TOP.regnum = regnum;
+	      
+              compile_stack.avail++;
+	      
+	      top_expression = last_expression;
+	      break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+            handle_close:
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  return REG_ERPAREN;
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+		struct rexp_node ** inner = top_expression;
+
+                compile_stack.avail--;
+		top_expression = COMPILE_STACK_TOP.top_expression;
+		last_expression = COMPILE_STACK_TOP.last_expression;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		{
+		  int left_se = paramc;
+		  int right_se = paramc + 1;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 2) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (2 * sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  paramc += 2;
+
+		  params[left_se].se = re_se_lparen;
+		  params[left_se].op1 = this_group_regnum;
+		  params[right_se].se = re_se_rparen;
+		  params[right_se].op1 = this_group_regnum;
+		  {
+		    struct rexp_node * left
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)left_se);
+		    struct rexp_node * right
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)right_se);
+		    struct rexp_node * c1
+		      = (*inner
+			 ? rx_mk_r_concat (&rxb->rx, left, *inner) : left);
+		    struct rexp_node * c2
+		      = rx_mk_r_concat (&rxb->rx, c1, right);
+		    if (!(left && right && c1 && c2))
+		      return REG_ESPACE;
+		    *inner = c2;
+		  }
+		}
+		break;
+	      }
+
+            case '|':					/* `\|'.  */
+              if ((syntax & RE_LIMITED_OPS) || (syntax & RE_NO_BK_VBAR))
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+	      {
+		struct rexp_node * alt
+		  = rx_mk_r_alternate (&rxb->rx, *top_expression, 0);
+		if (!alt)
+		  return REG_ESPACE;
+		*top_expression = alt;
+		last_expression = &alt->params.pair.right;
+		{
+		  int sync_se = paramc;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 1) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+
+		  params[sync_se].se = re_se_tv;
+		  {
+		    struct rexp_node * sync
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)sync_se);
+		    struct rexp_node * conc
+		      = rx_mk_r_concat (&rxb->rx, sync, 0);
+
+		    if (!sync || !conc)
+		      return REG_ESPACE;
+
+		    *last_expression = conc;
+		    last_expression = &conc->params.pair.right;
+		  }
+		}
+	      }
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      return REG_EBRACE;
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') return REG_EBRACE;
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (pointless_if_repeated (*last_expression, params))
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      return REG_BADRPT;
+                    else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                      goto unfetch_interval;
+		    /* was: else laststart = b; */
+                  }
+
+                /* If the upper bound is zero, don't want to iterate
+                 * at all.
+		 */
+                 if (upper_bound == 0)
+		   {
+		     if (*last_expression)
+		       {
+			 rx_free_rexp (&rxb->rx, *last_expression);
+			 *last_expression = 0;
+		       }
+		   }
+		else
+		  /* Otherwise, we have a nontrivial interval. */
+		  {
+		    int iter_se = paramc;
+		    int end_se = paramc + 1;
+		    params = (params
+			      ? ((struct re_se_params *)
+				 realloc (params,
+					  sizeof (*params) * (2 + paramc)))
+			      : ((struct re_se_params *)
+				 malloc (2 * sizeof (*params))));
+		    if (!params)
+		      return REG_ESPACE;
+		    paramc += 2;
+		    params [iter_se].se = re_se_iter;
+		    params [iter_se].op1 = lower_bound;
+		    params[iter_se].op2 = upper_bound;
+
+		    params[end_se].se = re_se_end_iter;
+		    params[end_se].op1 = lower_bound;
+		    params[end_se].op2 = upper_bound;
+		    {
+		      struct rexp_node * push0
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_push0);
+		      struct rexp_node * start_one_iter
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)iter_se);
+		      struct rexp_node * phase1
+			= rx_mk_r_concat (&rxb->rx, start_one_iter,
+					  *last_expression);
+		      struct rexp_node * pushback
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_pushback);
+		      rx_Bitset cs = rx_cset (&rxb->rx);
+		      struct rexp_node * lit_t
+			= rx_mk_r_cset (&rxb->rx, cs);
+		      struct rexp_node * phase2
+			= rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		      struct rexp_node * loop
+			= rx_mk_r_2phase_star (&rxb->rx, phase1, phase2);
+		      struct rexp_node * push_n_loop
+			= rx_mk_r_concat (&rxb->rx, push0, loop);
+		      struct rexp_node * final_test
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)end_se);
+		      struct rexp_node * full_exp
+			= rx_mk_r_concat (&rxb->rx, push_n_loop, final_test);
+
+		      if (!(push0 && start_one_iter && phase1
+			    && pushback && lit_t && phase2
+			    && loop && push_n_loop && final_test && full_exp))
+			return REG_ESPACE;
+
+		      RX_bitset_enjoin(cs, 't');
+
+		      *last_expression = full_exp;
+		    }
+		  }
+                beg_interval = 0;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               p = beg_interval;
+               beg_interval = 0;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+	      side = (rx_side_effect)rx_se_at_dot;
+	      goto add_side_effect;
+              break;
+
+            case 's':
+	    case 'S':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * set = rx_mk_r_cset (&rxb->rx, cs);
+		if (!(cs && set))
+		  return REG_ESPACE;
+		if (c == 'S')
+		  rx_bitset_universe (rxb->rx.local_cset_size, cs);
+
+		PATFETCH (c);
+		{
+		  int x;
+		  enum syntaxcode code = syntax_spec_code [c];
+		  for (x = 0; x < 256; ++x)
+		    {
+		      
+		      if (SYNTAX (x) == code)
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, validate_inv_tr,
+						 inverse_translate,
+						 translate, x);
+			  rx_bitset_xor (rxb->rx.local_cset_size, cs, it);
+			}
+		    }
+		}
+		append = set;
+		goto append_node;
+	      }
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+            case 'W':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * n = (cs ? rx_mk_r_cset (&rxb->rx, cs) : 0);
+		if (!(cs && n))
+		  return REG_ESPACE;
+		if (c == 'W')
+		  rx_bitset_universe (rxb->rx.local_cset_size ,cs);
+		{
+		  int x;
+		  for (x = rxb->rx.local_cset_size - 1; x > 0; --x)
+		    if (SYNTAX(x) & Sword)
+		      RX_bitset_toggle (cs, x);
+		}
+		append = n;
+		goto append_node;
+	      }
+              break;
+
+/* With a little extra work, some of these side effects could be optimized
+ * away (basicly by looking at what we already know about the surrounding
+ * chars).  
+ */
+            case '<':
+	      side = (rx_side_effect)re_se_wordbeg;
+	      goto add_side_effect;
+              break;
+
+            case '>':
+              side = (rx_side_effect)re_se_wordend;
+	      goto add_side_effect;
+              break;
+
+            case 'b':
+              side = (rx_side_effect)re_se_wordbound;
+	      goto add_side_effect;
+              break;
+
+            case 'B':
+              side = (rx_side_effect)re_se_notwordbound;
+	      goto add_side_effect;
+              break;
+
+            case '`':
+	      side = (rx_side_effect)re_se_begbuf;
+	      goto add_side_effect;
+	      break;
+	      
+            case '\'':
+	      side = (rx_side_effect)re_se_endbuf;
+	      goto add_side_effect;
+              break;
+
+	    add_side_effect:
+	      {
+		struct rexp_node * se
+		  = rx_mk_r_side_effect (&rxb->rx, side);
+		if (!se)
+		  return REG_ESPACE;
+		append = se;
+		goto append_node;
+	      }
+	      break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                return REG_ESUBREG;
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+		return REG_ESUBREG;
+
+	      {
+		int backref_se = paramc;
+		params = (params
+			  ? ((struct re_se_params *)
+			     realloc (params,
+				      sizeof (*params) * (1 + paramc)))
+			  : ((struct re_se_params *)
+			     malloc (sizeof (*params))));
+		if (!params)
+		  return REG_ESPACE;
+		++paramc;
+		params[backref_se].se = re_se_backref;
+		params[backref_se].op1 = c1;
+		side = (rx_side_effect)backref_se;
+		goto add_side_effect;
+	      }
+              break;
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	    {
+	      rx_Bitset cs = rx_cset(&rxb->rx);
+	      struct rexp_node * match = rx_mk_r_cset (&rxb->rx, cs);
+	      rx_Bitset it;
+	      if (!(cs && match))
+		return REG_ESPACE;
+	      it = inverse_translation (rxb, validate_inv_tr,
+					inverse_translate, translate, c);
+	      rx_bitset_union (CHAR_SET_SIZE, cs, it);
+	      append = match;
+
+	    append_node:
+	      /* This genericly appends the rexp APPEND to *LAST_EXPRESSION
+	       * and then parses the next character normally.
+	       */
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, append);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+	      else
+		*last_expression = append;
+	    }
+	} /* switch (c) */
+    } /* while p != pend */
+
+  
+  {
+    int win_se = paramc;
+    params = (params
+	      ? ((struct re_se_params *)
+		 realloc (params,
+			  sizeof (*params) * (1 + paramc)))
+	      : ((struct re_se_params *)
+		 malloc (sizeof (*params))));
+    if (!params)
+      return REG_ESPACE;
+    ++paramc;
+    params[win_se].se = re_se_win;
+    {
+      struct rexp_node * se
+	= rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)win_se);
+      struct rexp_node * concat
+	= rx_mk_r_concat (&rxb->rx, rexp, se);
+      if (!(se && concat))
+	return REG_ESPACE;
+      rexp = concat;
+    }
+  }
+
+
+  /* Through the pattern now.  */
+
+  if (!COMPILE_STACK_EMPTY) 
+    return REG_EPAREN;
+
+      free (compile_stack.stack);
+
+  orig_rexp = rexp;
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      dbug_rxb = rxb;
+      fputs ("\n\nCompiling ", stdout);
+      fwrite (pattern, 1, size, stdout);
+      fputs (":\n", stdout);
+      rxb->se_params = params;
+      print_rexp (&rxb->rx, orig_rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    rx_Bitset cs = rx_cset(&rxb->rx);
+    rx_Bitset cs2 = rx_cset(&rxb->rx);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+    char * se_map = (char *) _alloca (paramc);
+#else
+    char * se_map = (char *) alloca (paramc);
+#endif
+    struct rexp_node * new_rexp = 0;
+
+
+    bzero (se_map, paramc);
+    find_backrefs (se_map, rexp, params);
+    fewer_side_effects =
+      remove_unecessary_side_effects (&rxb->rx, se_map,
+				      rx_copy_rexp (&rxb->rx, rexp), params);
+
+    speed_up_alt (&rxb->rx, rexp, 0);
+    speed_up_alt (&rxb->rx, fewer_side_effects, 1);
+
+    {
+      char * syntax_parens = rxb->syntax_parens;
+      if (syntax_parens == (char *)0x1)
+	rexp = remove_unecessary_side_effects
+	  (&rxb->rx, se_map, rexp, params);
+      else if (syntax_parens)
+	{
+	  int x;
+	  for (x = 0; x < paramc; ++x)
+	    if ((   (params[x].se == re_se_lparen)
+		 || (params[x].se == re_se_rparen))
+		&& (!syntax_parens [params[x].op1]))
+	      se_map [x] = 1;
+	  rexp = remove_unecessary_side_effects
+	    (&rxb->rx, se_map, rexp, params);
+	}
+    }
+
+    /* At least one more optimization would be nice to have here but i ran out 
+     * of time.  The idea would be to delay side effects.  
+     * For examle, `(abc)' is the same thing as `abc()' except that the
+     * left paren is offset by 3 (which we know at compile time).
+     * (In this comment, write that second pattern `abc(:3:)' 
+     * where `(:3:' is a syntactic unit.)
+     *
+     * Trickier:  `(abc|defg)'  is the same as `(abc(:3:|defg(:4:))'
+     * (The paren nesting may be hard to follow -- that's an alternation
+     *	of `abc(:3:' and `defg(:4:' inside (purely syntactic) parens
+     *  followed by the closing paren from the original expression.)
+     *
+     * Neither the expression tree representation nor the the nfa make
+     * this very easy to write. :(
+     */
+
+  /* What we compile is different than what the parser returns.
+   * Suppose the parser returns expression R.
+   * Let R' be R with unnecessary register assignments removed 
+   * (see REMOVE_UNECESSARY_SIDE_EFFECTS, above).
+   *
+   * What we will compile is the expression:
+   *
+   *    m{try}R{win}\|s{try}R'{win}
+   *
+   * {try} and {win} denote side effect epsilons (see EXPLORE_FUTURE).
+   * 
+   * When trying a match, we insert an `m' at the beginning of the 
+   * string if the user wants registers to be filled, `s' if not.
+   */
+    new_rexp =
+      rx_mk_r_alternate
+	(&rxb->rx,
+	 rx_mk_r_concat (&rxb->rx, rx_mk_r_cset (&rxb->rx, cs2), rexp),
+	 rx_mk_r_concat (&rxb->rx,
+			 rx_mk_r_cset (&rxb->rx, cs), fewer_side_effects));
+
+    if (!(new_rexp && cs && cs2))
+      return REG_ESPACE;
+    RX_bitset_enjoin (cs2, '\0'); /* prefixed to the rexp used for matching. */
+    RX_bitset_enjoin (cs, '\1'); /* prefixed to the rexp used for searching. */
+    rexp = new_rexp;
+  }
+
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      fputs ("\n...which is compiled as:\n", stdout);
+      print_rexp (&rxb->rx, rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    struct rx_nfa_state *start = 0;
+    struct rx_nfa_state *end = 0;
+
+    if (!rx_build_nfa (&rxb->rx, rexp, &start, &end))
+      return REG_ESPACE;	/*  */
+    else
+      {
+	void * mem = (void *)rxb->buffer;
+	unsigned long size = rxb->allocated;
+	int start_id;
+	char * perm_mem;
+	int iterator_size = paramc * sizeof (params[0]);
+
+	end->is_final = 1;
+	start->is_start = 1;
+	rx_name_nfa_states (&rxb->rx);
+	start_id = start->id;
+#ifdef RX_DEBUG
+	if (rx_debug_compile)
+	  {
+	    fputs ("...giving the NFA: \n", stdout);
+	    dbug_rxb = rxb;
+	    print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	  }
+#endif
+	if (!rx_eclose_nfa (&rxb->rx))
+	  return REG_ESPACE;
+	else
+	  {
+	    rx_delete_epsilon_transitions (&rxb->rx);
+	    
+	    /* For compatability reasons, we need to shove the
+	     * compiled nfa into one chunk of malloced memory.
+	     */
+	    rxb->rx.reserved = (   sizeof (params[0]) * paramc
+				+  rx_sizeof_bitset (rxb->rx.local_cset_size));
+#ifdef RX_DEBUG
+	    if (rx_debug_compile)
+	      {
+		dbug_rxb = rxb;
+		fputs ("...which cooks down (uncompactified) to: \n", stdout);
+		print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	      }
+#endif
+	    if (!rx_compactify_nfa (&rxb->rx, &mem, &size))
+	      return REG_ESPACE;
+	    rxb->buffer = mem;
+	    rxb->allocated = size;
+	    rxb->rx.buffer = mem;
+	    rxb->rx.allocated = size;
+	    perm_mem = ((char *)rxb->rx.buffer
+			+ rxb->rx.allocated - rxb->rx.reserved);
+	    rxb->se_params = ((struct re_se_params *)perm_mem);
+	    bcopy (params, rxb->se_params, iterator_size);
+	    perm_mem += iterator_size;
+	    rxb->fastset = (rx_Bitset) perm_mem;
+	    rxb->start = rx_id_to_nfa_state (&rxb->rx, start_id);
+	  }
+	rx_bitset_null (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->can_match_empty = compute_fastset (rxb, orig_rexp);
+	rxb->match_regs_on_stack =
+	  registers_on_stack (rxb, orig_rexp, 0, params); 
+	rxb->search_regs_on_stack =
+	  registers_on_stack (rxb, fewer_side_effects, 0, params);
+	if (rxb->can_match_empty)
+	  rx_bitset_universe (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->is_anchored = is_anchored (orig_rexp, (rx_side_effect) re_se_hat);
+	rxb->begbuf_only = is_anchored (orig_rexp,
+					(rx_side_effect) re_se_begbuf);
+      }
+    rx_free_rexp (&rxb->rx, rexp);
+    if (params)
+      free (params);
+#ifdef RX_DEBUG
+    if (rx_debug_compile)
+      {
+	dbug_rxb = rxb;
+	fputs ("...which cooks down to: \n", stdout);
+	print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+      }
+#endif
+  }
+  return REG_NOERROR;
+}
+
+
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.  */
+
+__const__ char * rx_error_msg[] =
+{ 0,						/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+};
+
+
+
+
+
+char rx_slowmap [256] =
+{
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+};
+
+#ifdef __STDC__
+RX_DECL void
+rx_blow_up_fastmap (struct re_pattern_buffer * rxb)
+#else
+RX_DECL void
+rx_blow_up_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  int x;
+  for (x = 0; x < 256; ++x)	/* &&&& 3.6 % */
+    rxb->fastmap [x] = !!RX_bitset_member (rxb->fastset, x);
+  rxb->fastmap_accurate = 1;
+}
+
+
+
+
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#define RE_SEARCH_2_FN	inner_re_search_2
+#define RE_S2_QUAL static
+#else
+#define RE_SEARCH_2_FN	re_search_2
+#define RE_S2_QUAL 
+#endif
+
+struct re_search_2_closure
+{
+  __const__ char * string1;
+  int size1;
+  __const__ char * string2;
+  int size2;
+};
+
+
+static __inline__ enum rx_get_burst_return
+re_search_2_get_burst (pos, vclosure, stop)
+     struct rx_string_position * pos;
+     void * vclosure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)vclosure;
+  if (!closure->string2)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      if ((inset < -1) || (inset > closure->size1))
+	return rx_get_burst_no_more;
+      else
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+  else if (!closure->string1)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      pos->pos = (__const__ unsigned char *) closure->string2 + inset;
+      pos->string = (__const__ unsigned char *) closure->string2;
+      pos->size = closure->size2;
+      pos->end = ((__const__ unsigned char *)
+		  MIN(closure->string2 + closure->size2,
+		      closure->string2 + stop));
+      pos->offset = 0;
+      return ((pos->pos < pos->end)
+	      ? rx_get_burst_ok
+	      :  rx_get_burst_no_more);
+    }
+  else
+    {
+      int inset;
+
+      inset = pos->pos - pos->string + pos->offset;
+      if (inset < closure->size1)
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return rx_get_burst_ok;
+	}
+      else
+	{
+	  pos->pos = ((__const__ unsigned char *)
+		      closure->string2 + inset - closure->size1);
+	  pos->string = (__const__ unsigned char *) closure->string2;
+	  pos->size = closure->size2;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string2 + closure->size2,
+			  closure->string2 + stop - closure->size1));
+	  pos->offset = closure->size1;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+}
+
+
+static __inline__ enum rx_back_check_return
+re_search_2_back_check (pos, lparen, rparen, translate, vclosure, stop)
+     struct rx_string_position * pos;
+     int lparen;
+     int rparen;
+     unsigned char * translate;
+     void * vclosure;
+     int stop;
+{
+  struct rx_string_position there;
+  struct rx_string_position past;
+
+  there = *pos;
+  there.pos = there.string + lparen - there.offset;
+  re_search_2_get_burst (&there, vclosure, stop);
+
+  past = *pos;
+  past.pos = past.string + rparen - there.offset;
+  re_search_2_get_burst (&past, vclosure, stop);
+
+  ++pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+
+  while (   (there.pos != past.pos)
+	 && (pos->pos != pos->end))
+    if (TRANSLATE(*there.pos) != TRANSLATE(*pos->pos))
+      return rx_back_check_fail;
+    else
+      {
+	++there.pos;
+	++pos->pos;
+	if (there.pos == there.end)
+	  re_search_2_get_burst (&there, vclosure, stop);
+	if (pos->pos == pos->end)
+	  re_search_2_get_burst (pos, vclosure, stop);
+      }
+
+  if (there.pos != past.pos)
+    return rx_back_check_fail;
+  --pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+  return rx_back_check_pass;
+}
+
+static __inline__ int
+re_search_2_fetch_char (pos, offset, app_closure, stop)
+     struct rx_string_position * pos;
+     int offset;
+     void * app_closure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)app_closure;
+  if (offset == 0)
+    {
+      if (pos->pos >= pos->string)
+	return *pos->pos;
+      else
+	{
+	  if (   (pos->string == (__const__ unsigned char *) closure->string2)
+	      && (closure->string1)
+	      && (closure->size1))
+	    return closure->string1[closure->size1 - 1];
+	  else
+	    return 0;		/* sure, why not. */
+	}
+    }
+  if (pos->pos == pos->end)
+    return *closure->string2;
+  else
+    return pos->pos[1];
+}
+     
+
+#ifdef __STDC__
+RE_S2_QUAL int
+RE_SEARCH_2_FN (struct re_pattern_buffer *rxb,
+		__const__ char * string1, int size1,
+		__const__ char * string2, int size2,
+		int startpos, int range,
+		struct re_registers *regs,
+		int stop)
+#else
+RE_S2_QUAL int
+RE_SEARCH_2_FN (rxb,
+		string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int answer;
+  struct re_search_2_closure closure;
+  closure.string1 = string1;
+  closure.size1 = size1;
+  closure.string2 = string2;
+  closure.size2 = size2;
+  answer = rx_search (rxb, startpos, range, stop, size1 + size2,
+		      re_search_2_get_burst,
+		      re_search_2_back_check,
+		      re_search_2_fetch_char,
+		      (void *)&closure,
+		      regs,
+		      0,
+		      0);
+  switch (answer)
+    {
+    case rx_search_continuation:
+      abort ();
+    case rx_search_error:
+      return -2;
+    case rx_search_soft_fail:
+    case rx_search_fail:
+      return -1;
+    default:
+      return answer;
+    }
+}
+
+/* Export rx_search to callers outside this file.  */
+
+int
+re_rx_search (rxb, startpos, range, stop, total_size,
+	      get_burst, back_check, fetch_char,
+	      app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+{
+  return rx_search (rxb, startpos, range, stop, total_size,
+		    get_burst, back_check, fetch_char, app_closure,
+		    regs, resume_state, save_state);
+}
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#ifdef __STDC__
+int
+re_search_2 (struct re_pattern_buffer *rxb,
+	     __const__ char * string1, int size1,
+	     __const__ char * string2, int size2,
+	     int startpos, int range,
+	     struct re_registers *regs,
+	     int stop)
+#else
+int
+re_search_2 (rxb, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int ret;
+  ret = inner_re_search_2 (rxb, string1, size1, string2, size2, startpos,
+			   range, regs, stop);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return ret;
+}
+#endif
+
+
+/* Like re_search_2, above, but only one string is specified, and
+ * doesn't let you say where to stop matching.
+ */
+
+#ifdef __STDC__
+int
+re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+	   int size, int startpos, int range,
+	   struct re_registers *regs)
+#else
+int
+re_search (rxb, string, size, startpos, range, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string;
+     int size;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+#endif
+{
+  return re_search_2 (rxb, 0, 0, string, size, startpos, range, regs, size);
+}
+
+#ifdef __STDC__
+int
+re_match_2 (struct re_pattern_buffer * rxb,
+	    __const__ char * string1, int size1,
+	    __const__ char * string2, int size2,
+	    int pos, struct re_registers *regs, int stop)
+#else
+int
+re_match_2 (rxb, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  struct re_registers some_regs;
+  regoff_t start;
+  regoff_t end;
+  int srch;
+  int save = rxb->regs_allocated;
+  struct re_registers * regs_to_pass = regs;
+
+  if (!regs)
+    {
+      some_regs.start = &start;
+      some_regs.end = &end;
+      some_regs.num_regs = 1;
+      regs_to_pass = &some_regs;
+      rxb->regs_allocated = REGS_FIXED;
+    }
+
+  srch = re_search_2 (rxb, string1, size1, string2, size2,
+		      pos, 1, regs_to_pass, stop);
+  if (regs_to_pass != regs)
+    rxb->regs_allocated = save;
+  if (srch < 0)
+    return srch;
+  return regs_to_pass->end[0] - regs_to_pass->start[0];
+}
+
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+#ifdef __STDC__
+int
+re_match (struct re_pattern_buffer * rxb,
+	  __const__ char * string,
+	  int size, int pos,
+	  struct re_registers *regs)
+#else
+int
+re_match (rxb, string, size, pos, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char *string;
+     int size;
+     int pos;
+     struct re_registers *regs;
+#endif
+{
+  return re_match_2 (rxb, string, size, 0, 0, pos, regs, size);
+}
+
+
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+reg_syntax_t re_syntax_options = RE_SYNTAX_EMACS;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+#ifdef __STDC__
+reg_syntax_t
+re_set_syntax (reg_syntax_t syntax)
+#else
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+#endif
+{
+  reg_syntax_t ret = re_syntax_options;
+
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+#ifdef __STDC__
+void
+re_set_registers (struct re_pattern_buffer *bufp,
+		  struct re_registers *regs,
+		  unsigned num_regs,
+		  regoff_t * starts, regoff_t * ends)
+#else
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+     struct re_pattern_buffer *bufp;
+     struct re_registers *regs;
+     unsigned num_regs;
+     regoff_t * starts;
+     regoff_t * ends;
+#endif
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t) 0;
+    }
+}
+
+
+
+
+
+#ifdef __STDC__
+static int 
+cplx_se_sublist_len (struct rx_se_list * list)
+#else
+static int 
+cplx_se_sublist_len (list)
+     struct rx_se_list * list;
+#endif
+{
+  int x = 0;
+  while (list)
+    {
+      if ((long)list->car >= 0)
+	++x;
+      list = list->cdr;
+    }
+  return x;
+}
+
+
+/* For rx->se_list_cmp */
+
+#ifdef __STDC__
+static int 
+posix_se_list_order (struct rx * rx,
+		     struct rx_se_list * a, struct rx_se_list * b)
+#else
+static int 
+posix_se_list_order (rx, a, b)
+     struct rx * rx;
+     struct rx_se_list * a;
+     struct rx_se_list * b;
+#endif
+{
+  int al = cplx_se_sublist_len (a);
+  int bl = cplx_se_sublist_len (b);
+
+  if (!al && !bl)
+    return ((a == b)
+	    ? 0
+	    : ((a < b) ? -1 : 1));
+  
+  else if (!al)
+    return -1;
+
+  else if (!bl)
+    return 1;
+
+  else
+    {
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+      rx_side_effect * av = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (bl + 1)));
+#else
+      rx_side_effect * av = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (bl + 1)));
+#endif
+      struct rx_se_list * ap = a;
+      struct rx_se_list * bp = b;
+      int ai, bi;
+      
+      for (ai = al - 1; ai >= 0; --ai)
+	{
+	  while ((long)ap->car < 0)
+	    ap = ap->cdr;
+	  av[ai] = ap->car;
+	  ap = ap->cdr;
+	}
+      av[al] = (rx_side_effect)-2;
+      for (bi = bl - 1; bi >= 0; --bi)
+	{
+	  while ((long)bp->car < 0)
+	    bp = bp->cdr;
+	  bv[bi] = bp->car;
+	  bp = bp->cdr;
+	}
+      bv[bl] = (rx_side_effect)-1;
+
+      {
+	int ret;
+	int x = 0;
+	while (av[x] == bv[x])
+	  ++x;
+ 	ret = (((unsigned *)(av[x]) < (unsigned *)(bv[x])) ? -1 : 1);
+	return ret;
+      }
+    }
+}
+
+
+
+
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in RXB.
+   Returns 0 if the pattern was valid, otherwise an error string.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in RXB on entry.
+
+   We call rx_compile to do the actual compilation.  */
+
+#ifdef __STDC__
+__const__ char *
+re_compile_pattern (__const__ char *pattern,
+		    int length,
+		    struct re_pattern_buffer * rxb)
+#else
+__const__ char *
+re_compile_pattern (pattern, length, rxb)
+     __const__ char *pattern;
+     int length;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  reg_errcode_t ret;
+
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  rxb->regs_allocated = REGS_UNALLOCATED;
+
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  rxb->no_sub = 0;
+
+  rxb->rx.local_cset_size = 256;
+
+  /* Match anchors at newline.  */
+  rxb->newline_anchor = 1;
+ 
+  rxb->re_nsub = 0;
+  rxb->start = 0;
+  rxb->se_params = 0;
+  rxb->rx.nodec = 0;
+  rxb->rx.epsnodec = 0;
+  rxb->rx.instruction_table = 0;
+  rxb->rx.nfa_states = 0;
+  rxb->rx.se_list_cmp = posix_se_list_order;
+  rxb->rx.start_set = 0;
+
+  ret = rx_compile (pattern, length, re_syntax_options, rxb);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return rx_error_msg[(int) ret];
+}
+
+
+
+#ifdef __STDC__
+int
+re_compile_fastmap (struct re_pattern_buffer * rxb)
+#else
+int
+re_compile_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  rx_blow_up_fastmap (rxb);
+  return 0;
+}
+
+
+
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them if this is an Emacs or POSIX compilation.  */
+
+#if (!defined (emacs) && !defined (_POSIX_SOURCE)) || defined(USE_BSD_REGEX)
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer rx_comp_buf;
+
+#ifdef __STDC__
+char *
+re_comp (__const__ char *s)
+#else
+char *
+re_comp (s)
+    __const__ char *s;
+#endif
+{
+  reg_errcode_t ret;
+
+  if (!s || (*s == '\0'))
+    {
+      if (!rx_comp_buf.buffer)
+	return "No previous regular expression";
+      return 0;
+    }
+
+  if (!rx_comp_buf.fastmap)
+    {
+      rx_comp_buf.fastmap = (char *) malloc (1 << CHARBITS);
+      if (!rx_comp_buf.fastmap)
+	return "Memory exhausted";
+    }
+
+  /* Since `rx_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  rx_comp_buf.newline_anchor = 1;
+
+  rx_comp_buf.re_nsub = 0;
+  rx_comp_buf.start = 0;
+  rx_comp_buf.se_params = 0;
+  rx_comp_buf.rx.nodec = 0;
+  rx_comp_buf.rx.epsnodec = 0;
+  rx_comp_buf.rx.instruction_table = 0;
+  rx_comp_buf.rx.nfa_states = 0;
+  rx_comp_buf.rx.start = 0;
+  rx_comp_buf.rx.se_list_cmp = posix_se_list_order;
+  rx_comp_buf.rx.local_cset_size = 256;
+
+  ret = rx_compile (s, strlen (s), re_syntax_options, &rx_comp_buf);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* Yes, we're discarding `__const__' here.  */
+  return (char *) rx_error_msg[(int) ret];
+}
+
+
+#ifdef __STDC__
+int
+re_exec (__const__ char *s)
+#else
+int
+re_exec (s)
+    __const__ char *s;
+#endif
+{
+  __const__ int len = strlen (s);
+  return
+    0 <= re_search (&rx_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* not emacs and not _POSIX_SOURCE */
+
+
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#if !defined(emacs)
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+
+#ifdef __STDC__
+int
+regcomp (regex_t * preg, __const__ char * pattern, int cflags)
+#else
+int
+regcomp (preg, pattern, cflags)
+    regex_t * preg;
+    __const__ char * pattern;
+    int cflags;
+#endif
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = cflags & REG_EXTENDED ? RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->fastmap = malloc (256);
+  if (!preg->fastmap)
+    return REG_ESPACE;
+  preg->fastmap_accurate = 0;
+
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+
+      preg->translate = (unsigned char *) malloc (256);
+      if (!preg->translate)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = isupper (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = 0;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we
+     can use strlen here in compiling the pattern.  */
+  preg->re_nsub = 0;
+  preg->start = 0;
+  preg->se_params = 0;
+  preg->syntax_parens = 0;
+  preg->rx.nodec = 0;
+  preg->rx.epsnodec = 0;
+  preg->rx.instruction_table = 0;
+  preg->rx.nfa_states = 0;
+  preg->rx.local_cset_size = 256;
+  preg->rx.start = 0;
+  preg->rx.se_list_cmp = posix_se_list_order;
+  preg->rx.start_set = 0;
+  ret = rx_compile (pattern, strlen (pattern), syntax, preg);
+
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+#ifdef __STDC__
+int
+regexec (__const__ regex_t *preg, __const__ char *string,
+	 size_t nmatch, regmatch_t pmatch[],
+	 int eflags)
+#else
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    __const__ regex_t *preg;
+    __const__ char *string;
+    size_t nmatch;
+    regmatch_t pmatch[];
+    int eflags;
+#endif
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+
+  /* The user has told us exactly how many registers to return
+   * information about, via `nmatch'.  We have to pass that on to the
+   * matching routines.
+   */
+  private_preg.regs_allocated = REGS_FIXED;
+
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      regs.end =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      if (regs.start == 0 || regs.end == 0)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg,
+		   string, len,
+                   /* start: */ 0,
+		   /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   */
+
+#ifdef __STDC__
+size_t
+regerror (int errcode, __const__ regex_t *preg,
+	  char *errbuf, size_t errbuf_size)
+#else
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    __const__ regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+#endif
+{
+  __const__ char *msg
+    = rx_error_msg[errcode] == 0 ? "Success" : rx_error_msg[errcode];
+  size_t msg_size = strlen (msg) + 1; /* Includes the 0.  */
+
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+#ifdef __STDC__
+void
+regfree (regex_t *preg)
+#else
+void
+regfree (preg)
+    regex_t *preg;
+#endif
+{
+  if (preg->buffer != 0)
+    free (preg->buffer);
+  preg->buffer = 0;
+  preg->allocated = 0;
+
+  if (preg->fastmap != 0)
+    free (preg->fastmap);
+  preg->fastmap = 0;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != 0)
+    free (preg->translate);
+  preg->translate = 0;
+}
+
+#endif /* not emacs  */
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/rx.h	(revision 21112)
@@ -0,0 +1,3755 @@
+#if !defined(RXH) || defined(RX_WANT_SE_DEFS)
+#define RXH
+
+/*	Copyright (C) 1992, 1993 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+/*  t. lord	Wed Sep 23 18:20:57 1992	*/
+
+
+
+
+
+
+
+
+
+#ifndef RX_WANT_SE_DEFS
+
+/* This page: Bitsets */
+
+#ifndef RX_subset
+typedef unsigned int RX_subset;
+#define RX_subset_bits	(32)
+#define RX_subset_mask	(RX_subset_bits - 1)
+#endif
+
+typedef RX_subset * rx_Bitset;
+
+#ifdef __STDC__
+typedef void (*rx_bitset_iterator) (void *, int member_index);
+#else
+typedef void (*rx_bitset_iterator) ();
+#endif
+
+#define rx_bitset_subset(N)  ((N) / RX_subset_bits)
+#define rx_bitset_subset_val(B,N)  ((B)[rx_bitset_subset(N)])
+#define RX_bitset_access(B,N,OP) \
+  ((B)[rx_bitset_subset(N)] OP rx_subset_singletons[(N) & RX_subset_mask])
+#define RX_bitset_member(B,N)   RX_bitset_access(B, N, &)
+#define RX_bitset_enjoin(B,N)   RX_bitset_access(B, N, |=)
+#define RX_bitset_remove(B,N)   RX_bitset_access(B, N, &= ~)
+#define RX_bitset_toggle(B,N)   RX_bitset_access(B, N, ^= )
+#define rx_bitset_numb_subsets(N) (((N) + RX_subset_bits - 1) / RX_subset_bits)
+#define rx_sizeof_bitset(N)	(rx_bitset_numb_subsets(N) * sizeof(RX_subset))
+
+
+
+
+/* This page: Splay trees. */
+
+#ifdef __STDC__
+typedef int (*rx_sp_comparer) (void * a, void * b);
+#else
+typedef int (*rx_sp_comparer) ();
+#endif
+
+struct rx_sp_node 
+{
+  void * key;
+  void * data;
+  struct rx_sp_node * kids[2];
+};
+
+#ifdef __STDC__
+typedef void (*rx_sp_key_data_freer) (struct rx_sp_node *);
+#else
+typedef void (*rx_sp_key_data_freer) ();
+#endif
+
+
+
+/* giant inflatable hash trees */
+
+struct rx_hash_item
+{
+  struct rx_hash_item * next_same_hash;
+  struct rx_hash * table;
+  unsigned long hash;
+  void * data;
+  void * binding;
+};
+
+struct rx_hash
+{
+  struct rx_hash * parent;
+  int refs;
+  struct rx_hash * children[13];
+  struct rx_hash_item * buckets [13];
+  int bucket_size [13];
+};
+
+struct rx_hash_rules;
+
+#ifdef __STDC__
+/* should return like == */
+typedef int (*rx_hash_eq)(void *, void *);
+typedef struct rx_hash * (*rx_alloc_hash)(struct rx_hash_rules *);
+typedef void (*rx_free_hash)(struct rx_hash *,
+			    struct rx_hash_rules *);
+typedef struct rx_hash_item * (*rx_alloc_hash_item)(struct rx_hash_rules *,
+						    void *);
+typedef void (*rx_free_hash_item)(struct rx_hash_item *,
+				 struct rx_hash_rules *);
+#else
+typedef int (*rx_hash_eq)();
+typedef struct rx_hash * (*rx_alloc_hash)();
+typedef void (*rx_free_hash)();
+typedef struct rx_hash_item * (*rx_alloc_hash_item)();
+typedef void (*rx_free_hash_item)();
+#endif
+
+struct rx_hash_rules
+{
+  rx_hash_eq eq;
+  rx_alloc_hash hash_alloc;
+  rx_free_hash free_hash;
+  rx_alloc_hash_item hash_item_alloc;
+  rx_free_hash_item free_hash_item;
+};
+
+
+
+/* Forward declarations */
+
+struct rx_cache;
+struct rx_superset;
+struct rx;
+struct rx_se_list;
+
+
+
+
+/* 
+ * GLOSSARY
+ *
+ * regexp
+ * regular expression
+ * expression
+ * pattern - a `regular' expression.  The expression
+ *       need not be formally regular -- it can contain
+ *       constructs that don't correspond to purely regular
+ *       expressions.
+ *
+ * buffer
+ * string - the string (or strings) being searched or matched.
+ *
+ * pattern buffer - a structure of type `struct re_pattern_buffer'
+ *       This in turn contains a `struct rx', which holds the
+ *       NFA compiled from a pattern, as well as some of the state
+ *       of a matcher using the pattern.
+ *
+ * NFA - nondeterministic finite automata.  Some people
+ *       use this term to a member of the class of 
+ *       regular automata (those corresponding to a regular
+ *       language).  However, in this code, the meaning is
+ *       more general.  The automata used by Rx are comperable
+ *       in power to what are usually called `push down automata'.
+ *
+ *       Two NFA are built by rx for every pattern.  One is built
+ *       by the compiler.  The other is built from the first, on
+ *       the fly, by the matcher.  The latter is called the `superstate
+ *       NFA' because its states correspond to sets of states from
+ *       the first NFA.  (Joe Keane gets credit for the name
+ *       `superstate NFA').
+ *
+ * NFA edges
+ * epsilon edges
+ * side-effect edges - The NFA compiled from a pattern can have three
+ *       kinds of edges.  Epsilon edges can be taken freely anytime
+ *       their source state is reached.  Character set edges can be
+ *       taken when their source state is reached and when the next 
+ *       character in the buffer is a member of the set.  Side effect
+ *       edges imply a transition that can only be taken after the
+ *       indicated side effect has been successfully accomplished.
+ *       Some examples of side effects are:
+ *
+ *		Storing the current match position to record the
+ *              location of a parentesized subexpression.
+ *
+ *              Advancing the matcher over N characters if they
+ *              match the N characters previously matched by a 
+ *              parentesized subexpression.
+ *
+ *       Both of those kinds of edges occur in the NFA generated
+ *       by the pattern:  \(.\)\1
+ *
+ *       Epsilon and side effect edges are similar.  Unfortunately,
+ *       some of the code uses the name `epsilon edge' to mean
+ *       both epsilon and side effect edges.  For example,  the
+ *       function has_non_idempotent_epsilon_path computes the existance
+ *       of a non-trivial path containing only a mix of epsilon and
+ *       side effect edges.  In that case `nonidempotent epsilon' is being
+ *       used to mean `side effect'.
+ */
+
+
+
+
+
+
+/* LOW LEVEL PATTERN BUFFERS */
+
+/* Suppose that from some NFA state, more than one path through
+ * side-effect edges is possible.  In what order should the paths
+ * be tried?  A function of type rx_se_list_order answers that
+ * question.  It compares two lists of side effects, and says
+ * which list comes first.
+ */
+ 
+#ifdef __STDC__
+typedef int (*rx_se_list_order) (struct rx *,
+				 struct rx_se_list *, 
+				 struct rx_se_list *);
+#else
+typedef int (*rx_se_list_order) ();
+#endif
+
+
+
+/* Struct RX holds a compiled regular expression - that is, an nfa
+ * ready to be converted on demand to a more efficient superstate nfa.
+ * This is for the low level interface.  The high-level interfaces enclose
+ * this in a `struct re_pattern_buffer'.  
+ */
+struct rx
+{
+  /* The compiler assigns a unique id to every pattern.
+   * Like sequence numbers in X, there is a subtle bug here
+   * if you use Rx in a system that runs for a long time.
+   * But, because of the way the caches work out, it is almost
+   * impossible to trigger the Rx version of this bug.
+   *
+   * The id is used to validate superstates found in a cache
+   * of superstates.  It isn't sufficient to let a superstate
+   * point back to the rx for which it was compiled -- the caller
+   * may be re-using a `struct rx' in which case the superstate
+   * is not really valid.  So instead, superstates are validated
+   * by checking the sequence number of the pattern for which
+   * they were built.
+   */
+  int rx_id;
+
+  /* This is memory mgt. state for superstates.  This may be 
+   * shared by more than one struct rx.
+   */
+  struct rx_cache * cache;
+
+  /* Every regex defines the size of its own character set. 
+   * A superstate has an array of this size, with each element
+   * a `struct rx_inx'.  So, don't make this number too large.
+   * In particular, don't make it 2^16.
+   */
+  int local_cset_size;
+
+  /* After the NFA is built, it is copied into a contiguous region
+   * of memory (mostly for compatability with GNU regex).
+   * Here is that region, and it's size:
+   */
+  void * buffer;
+  unsigned long allocated;
+
+  /* Clients of RX can ask for some extra storage in the space pointed
+   * to by BUFFER.  The field RESERVED is an input parameter to the
+   * compiler.  After compilation, this much space will be available 
+   * at (buffer + allocated - reserved)
+   */
+  unsigned long reserved;
+
+  /* --------- The remaining fields are for internal use only. --------- */
+  /* --------- But! they must be initialized to 0.	       --------- */
+
+  /* NODEC is the number of nodes in the NFA with non-epsilon
+   * transitions. 
+   */
+  int nodec;
+
+  /* EPSNODEC is the number of nodes with only epsilon transitions. */
+  int epsnodec;
+
+  /* The sum (NODEC + EPSNODEC) is the total number of states in the
+   * compiled NFA.
+   */
+
+  /* Lists of side effects as stored in the NFA are `hash consed'..meaning
+   * that lists with the same elements are ==.  During compilation, 
+   * this table facilitates hash-consing.
+   */
+  struct rx_hash se_list_memo;
+
+  /* Lists of NFA states are also hashed. 
+   */
+  struct rx_hash set_list_memo;
+
+
+
+
+  /* The compiler and matcher must build a number of instruction frames.
+   * The format of these frames is fixed (c.f. struct rx_inx).  The values
+   * of the instructions is not fixed.
+   *
+   * An enumerated type (enum rx_opcode) defines the set of instructions
+   * that the compiler or matcher might generate.  When filling an instruction
+   * frame, the INX field is found by indexing this instruction table
+   * with an opcode:
+   */
+  void ** instruction_table;
+
+  /* The list of all states in an NFA.
+   * During compilation, the NEXT field of NFA states links this list.
+   * After compilation, all the states are compacted into an array,
+   * ordered by state id numbers.  At that time, this points to the base 
+   * of that array.
+   */
+  struct rx_nfa_state *nfa_states;
+
+  /* Every nfa begins with one distinguished starting state:
+   */
+  struct rx_nfa_state *start;
+
+  /* This orders the search through super-nfa paths.
+   * See the comment near the typedef of rx_se_list_order.
+   */
+  rx_se_list_order se_list_cmp;
+
+  struct rx_superset * start_set;
+};
+
+
+
+
+
+/* SYNTAX TREES */
+
+/* Compilation is in stages.  
+ *
+ * In the first stage, a pattern specified by a string is 
+ * translated into a syntax tree.  Later stages will convert
+ * the syntax tree into an NFA optimized for conversion to a
+ * superstate-NFA.
+ *
+ * This page is about syntax trees.
+ */
+
+enum rexp_node_type
+{
+  r_cset,			/* Match from a character set. `a' or `[a-z]'*/
+  r_concat,			/* Concat two subexpressions.   `ab' */
+  r_alternate,			/* Choose one of two subexpressions. `a\|b' */
+  r_opt,			/* Optional subexpression. `a?' */
+  r_star,			/* Repeated subexpression. `a*' */
+
+
+  /* A 2phase-star is a variation on a repeated subexpression.
+   * In this case, there are two subexpressions.  The first, if matched,
+   * begins a repitition (otherwise, the whole expression is matches the
+   * empth string).  
+   * 
+   * After matching the first subexpression, a 2phase star either finishes,
+   * or matches the second subexpression.  If the second subexpression is
+   * matched, then the whole construct repeats.
+   *
+   * 2phase stars are used in two circumstances.  First, they
+   * are used as part of the implementation of POSIX intervals (counted
+   * repititions).  Second, they are used to implement proper star
+   * semantics when the repeated subexpression contains paths of
+   * only side effects.  See rx_compile for more information.
+   */
+  r_2phase_star,
+
+
+  /* c.f. "typedef void * rx_side_effect" */
+  r_side_effect,
+
+  /* This is an extension type:  It is for transient use in source->source
+   * transformations (implemented over syntax trees).
+   */
+  r_data
+};
+
+/* A side effect is a matcher-specific action associated with
+ * transitions in the NFA.  The details of side effects are up
+ * to the matcher.  To the compiler and superstate constructors
+ * side effects are opaque:
+ */
+
+typedef void * rx_side_effect;
+
+/* Nodes in a syntax tree are of this type:
+ */
+struct rexp_node
+{
+  enum rexp_node_type type;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+    struct
+      {
+	struct rexp_node *left;
+	struct rexp_node *right;
+      } pair;
+    void * data;
+  } params;
+};
+
+
+
+
+/* NFA
+ *
+ * A syntax tree is compiled into an NFA.  This page defines the structure
+ * of that NFA.
+ */
+
+struct rx_nfa_state
+{
+  /* These are kept in a list as the NFA is being built. */
+  struct rx_nfa_state *next;
+
+  /* After the NFA is built, states are given integer id's.
+   * States whose outgoing transitions are all either epsilon or 
+   * side effect edges are given ids less than 0.  Other states
+   * are given successive non-negative ids starting from 0.
+   */
+  int id;
+
+  /* The list of NFA edges that go from this state to some other. */
+  struct rx_nfa_edge *edges;
+
+  /* If you land in this state, then you implicitly land
+   * in all other states reachable by only epsilon translations.
+   * Call the set of maximal paths to such states the epsilon closure
+   * of this state.
+   *
+   * There may be other states that are reachable by a mixture of
+   * epsilon and side effect edges.  Consider the set of maximal paths
+   * of that sort from this state.  Call it the epsilon-side-effect
+   * closure of the state.
+   * 
+   * The epsilon closure of the state is a subset of the epsilon-side-
+   * effect closure.  It consists of all the paths that contain 
+   * no side effects -- only epsilon edges.
+   * 
+   * The paths in the epsilon-side-effect closure  can be partitioned
+   * into equivalance sets. Two paths are equivalant if they have the
+   * same set of side effects, in the same order.  The epsilon-closure
+   * is one of these equivalance sets.  Let's call these equivalance
+   * sets: observably equivalant path sets.  That name is chosen
+   * because equivalance of two paths means they cause the same side
+   * effects -- so they lead to the same subsequent observations other
+   * than that they may wind up in different target states.
+   *
+   * The superstate nfa, which is derived from this nfa, is based on
+   * the observation that all of the paths in an observably equivalant
+   * path set can be explored at the same time, provided that the
+   * matcher keeps track not of a single nfa state, but of a set of
+   * states.   In particular, after following all the paths in an
+   * observably equivalant set, you wind up at a set of target states.
+   * That set of target states corresponds to one state in the
+   * superstate NFA.
+   *
+   * Staticly, before matching begins, it is convenient to analyze the
+   * nfa.  Each state is labeled with a list of the observably
+   * equivalant path sets who's union covers all the
+   * epsilon-side-effect paths beginning in this state.  This list is
+   * called the possible futures of the state.
+   *
+   * A trivial example is this NFA:
+   *             s1
+   *         A  --->  B
+   *
+   *             s2  
+   *            --->  C
+   *
+   *             epsilon           s1
+   *            --------->  D   ------> E
+   * 
+   * 
+   * In this example, A has two possible futures.
+   * One invokes the side effect `s1' and contains two paths,
+   * one ending in state B, the other in state E.
+   * The other invokes the side effect `s2' and contains only
+   * one path, landing in state C.
+   */
+  struct rx_possible_future *futures;
+
+
+  /* There are exactly two distinguished states in every NFA: */
+  unsigned int is_final:1;
+  unsigned int is_start:1;
+
+  /* These are used during NFA construction... */
+  unsigned int eclosure_needed:1;
+  unsigned int mark:1;
+};
+
+
+/* An edge in an NFA is typed: */
+enum rx_nfa_etype
+{
+  /* A cset edge is labled with a set of characters one of which
+   * must be matched for the edge to be taken.
+   */
+  ne_cset,
+
+  /* An epsilon edge is taken whenever its starting state is
+   * reached. 
+   */
+  ne_epsilon,
+
+  /* A side effect edge is taken whenever its starting state is
+   * reached.  Side effects may cause the match to fail or the
+   * position of the matcher to advance.
+   */
+  ne_side_effect		/* A special kind of epsilon. */
+};
+
+struct rx_nfa_edge
+{
+  struct rx_nfa_edge *next;
+  enum rx_nfa_etype type;
+  struct rx_nfa_state *dest;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+  } params;
+};
+
+
+
+/* A possible future consists of a list of side effects
+ * and a set of destination states.  Below are their
+ * representations.  These structures are hash-consed which
+ * means that lists with the same elements share a representation
+ * (their addresses are ==).
+ */
+
+struct rx_nfa_state_set
+{
+  struct rx_nfa_state * car;
+  struct rx_nfa_state_set * cdr;
+};
+
+struct rx_se_list
+{
+  rx_side_effect car;
+  struct rx_se_list * cdr;
+};
+
+struct rx_possible_future
+{
+  struct rx_possible_future *next;
+  struct rx_se_list * effects;
+  struct rx_nfa_state_set * destset;
+};
+
+
+
+
+/* This begins the description of the superstate NFA.
+ *
+ * The superstate NFA corresponds to the NFA in these ways:
+ *
+ * Every superstate NFA states SUPER correspond to sets of NFA states,
+ * nfa_states(SUPER).
+ *
+ * Superstate edges correspond to NFA paths.
+ *
+ * The superstate has no epsilon transitions;
+ * every edge has a character label, and a (possibly empty) side
+ * effect label.   The side effect label corresponds to a list of
+ * side effects that occur in the NFA.  These parts are referred
+ * to as:   superedge_character(EDGE) and superedge_sides(EDGE).
+ *
+ * For a superstate edge EDGE starting in some superstate SUPER,
+ * the following is true (in pseudo-notation :-):
+ *
+ *       exists DEST in nfa_states s.t. 
+ *         exists nfaEDGE in nfa_edges s.t.
+ *                 origin (nfaEDGE) == DEST
+ *              && origin (nfaEDGE) is a member of nfa_states(SUPER)
+ *              && exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *
+ * also:
+ *
+ *      let SUPER2 := superedge_destination(EDGE)
+ *          nfa_states(SUPER2)
+ *           == union of all nfa state sets S s.t.
+ *                          exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                       sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *                          && S == dests_of_possible_future (PF) }
+ *
+ * Or in english, every superstate is a set of nfa states.  A given
+ * character and a superstate implies many transitions in the NFA --
+ * those that begin with an edge labeled with that character from a
+ * state in the set corresponding to the superstate.
+ * 
+ * The destinations of those transitions each have a set of possible
+ * futures.  A possible future is a list of side effects and a set of
+ * destination NFA states.  Two sets of possible futures can be
+ * `merged' by combining all pairs of possible futures that have the
+ * same side effects.  A pair is combined by creating a new future
+ * with the same side effect but the union of the two destination sets.
+ * In this way, all the possible futures suggested by a superstate
+ * and a character can be merged into a set of possible futures where
+ * no two elements of the set have the same set of side effects.
+ *
+ * The destination of a possible future, being a set of NFA states, 
+ * corresponds to a supernfa state.  So, the merged set of possible
+ * futures we just created can serve as a set of edges in the
+ * supernfa.
+ *
+ * The representation of the superstate nfa and the nfa is critical.
+ * The nfa has to be compact, but has to facilitate the rapid
+ * computation of missing superstates.  The superstate nfa has to 
+ * be fast to interpret, lazilly constructed, and bounded in space.
+ *
+ * To facilitate interpretation, the superstate data structures are 
+ * peppered with `instruction frames'.  There is an instruction set
+ * defined below which matchers using the supernfa must be able to
+ * interpret.
+ *
+ * We'd like to make it possible but not mandatory to use code
+ * addresses to represent instructions (c.f. gcc's computed goto).
+ * Therefore, we define an enumerated type of opcodes, and when
+ * writing one of these instructions into a data structure, use
+ * the opcode as an index into a table of instruction values.
+ * 
+ * Here are the opcodes that occur in the superstate nfa:
+ */
+ 
+
+/* Every superstate contains a table of instruction frames indexed 
+ * by characters.  A normal `move' in a matcher is to fetch the next
+ * character and use it as an index into a superstates transition
+ * table.
+ *
+ * In the fasted case, only one edge follows from that character.
+ * In other cases there is more work to do.
+ * 
+ * The descriptions of the opcodes refer to data structures that are
+ * described further below. 
+ */
+
+enum rx_opcode
+{
+  /* 
+   * BACKTRACK_POINT is invoked when a character transition in 
+   * a superstate leads to more than one edge.  In that case,
+   * the edges have to be explored independently using a backtracking
+   * strategy.
+   *
+   * A BACKTRACK_POINT instruction is stored in a superstate's 
+   * transition table for some character when it is known that that
+   * character crosses more than one edge.  On encountering this
+   * instruction, the matcher saves enough state to backtrack to this
+   * point in the match later.
+   */
+  rx_backtrack_point = 0,	/* data is (struct transition_class *) */
+
+  /* 
+   * RX_DO_SIDE_EFFECTS evaluates the side effects of an epsilon path.
+   * There is one occurence of this instruction per rx_distinct_future.
+   * This instruction is skipped if a rx_distinct_future has no side effects.
+   */
+  rx_do_side_effects = rx_backtrack_point + 1,
+
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_CACHE_MISS instructions are stored in rx_distinct_futures whose
+   * destination superstate has been reclaimed (or was never built).
+   * It recomputes the destination superstate.
+   * RX_CACHE_MISS is also stored in a superstate transition table before
+   * any of its edges have been built.
+   */
+  rx_cache_miss = rx_do_side_effects + 1,
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_NEXT_CHAR is called to consume the next character and take the
+   * corresponding transition.  This is the only instruction that uses 
+   * the DATA field of the instruction frame instead of DATA_2.
+   * (see EXPLORE_FUTURE in regex.c).
+   */
+  rx_next_char = rx_cache_miss + 1, /* data is (struct superstate *) */
+
+  /* RX_BACKTRACK indicates that a transition fails.
+   */
+  rx_backtrack = rx_next_char + 1, /* no data */
+
+  /* 
+   * RX_ERROR_INX is stored only in places that should never be executed.
+   */
+  rx_error_inx = rx_backtrack + 1, /* Not supposed to occur. */
+
+  rx_num_instructions = rx_error_inx + 1
+};
+
+/* An id_instruction_table holds the values stored in instruction
+ * frames.  The table is indexed by the enums declared above.
+ */
+extern void * rx_id_instruction_table[rx_num_instructions];
+
+/* The heart of the matcher is a `word-code-interpreter' 
+ * (like a byte-code interpreter, except that instructions
+ * are a full word wide).
+ *
+ * Instructions are not stored in a vector of code, instead,
+ * they are scattered throughout the data structures built
+ * by the regexp compiler and the matcher.  One word-code instruction,
+ * together with the arguments to that instruction, constitute
+ * an instruction frame (struct rx_inx).
+ *
+ * This structure type is padded by hand to a power of 2 because
+ * in one of the dominant cases, we dispatch by indexing a table
+ * of instruction frames.  If that indexing can be accomplished
+ * by just a shift of the index, we're happy.
+ *
+ * Instructions take at most one argument, but there are two
+ * slots in an instruction frame that might hold that argument.
+ * These are called data and data_2.  The data slot is only
+ * used for one instruction (RX_NEXT_CHAR).  For all other 
+ * instructions, data should be set to 0.
+ *
+ * RX_NEXT_CHAR is the most important instruction by far.
+ * By reserving the data field for its exclusive use, 
+ * instruction dispatch is sped up in that case.  There is
+ * no need to fetch both the instruction and the data,
+ * only the data is needed.  In other words, a `cycle' begins
+ * by fetching the field data.  If that is non-0, then it must
+ * be the destination state of a next_char transition, so
+ * make that value the current state, advance the match position
+ * by one character, and start a new cycle.  On the other hand,
+ * if data is 0, fetch the instruction and do a more complicated
+ * dispatch on that.
+ */
+
+struct rx_inx 
+{
+  void * data;
+  void * data_2;
+  void * inx;
+  void * fnord;
+};
+
+#ifndef RX_TAIL_ARRAY
+#define RX_TAIL_ARRAY  1
+#endif
+
+/* A superstate corresponds to a set of nfa states.  Those sets are
+ * represented by STRUCT RX_SUPERSET.  The constructors
+ * guarantee that only one (shared) structure is created for a given set.
+ */
+struct rx_superset
+{
+  int refs;			/* This is a reference counted structure. */
+
+  /* We keep these sets in a cache because (in an unpredictable way),
+   * the same set is often created again and again.  But that is also
+   * problematic -- compatibility with POSIX and GNU regex requires
+   * that we not be able to tell when a program discards a particular
+   * NFA (thus invalidating the supersets created from it).
+   *
+   * But when a cache hit appears to occur, we will have in hand the
+   * nfa for which it may have happened.  That is why every nfa is given
+   * its own sequence number.  On a cache hit, the cache is validated
+   * by comparing the nfa sequence number to this field:
+   */
+  int id;
+
+  struct rx_nfa_state * car;	/* May or may not be a valid addr. */
+  struct rx_superset * cdr;
+
+  /* If the corresponding superstate exists: */
+  struct rx_superstate * superstate;
+
+
+  /* There is another bookkeeping problem.  It is expensive to 
+   * compute the starting nfa state set for an nfa.  So, once computed,
+   * it is cached in the `struct rx'.
+   *
+   * But, the state set can be flushed from the superstate cache.
+   * When that happens, we can't know if the corresponding `struct rx'
+   * is still alive or if it has been freed or re-used by the program.
+   * So, the cached pointer to this set in a struct rx might be invalid
+   * and we need a way to validate it.
+   *
+   * Fortunately, even if this set is flushed from the cache, it is
+   * not freed.  It just goes on the free-list of supersets.
+   * So we can still examine it.  
+   *
+   * So to validate a starting set memo, check to see if the
+   * starts_for field still points back to the struct rx in question,
+   * and if the ID matches the rx sequence number.
+   */
+  struct rx * starts_for;
+
+  /* This is used to link into a hash bucket so these objects can
+   * be `hash-consed'.
+   */
+  struct rx_hash_item hash_item;
+};
+
+#define rx_protect_superset(RX,CON) (++(CON)->refs)
+
+/* The terminology may be confusing (rename this structure?).
+ * Every character occurs in at most one rx_super_edge per super-state.
+ * But, that structure might have more than one option, indicating a point
+ * of non-determinism. 
+ *
+ * In other words, this structure holds a list of superstate edges
+ * sharing a common starting state and character label.  The edges
+ * are in the field OPTIONS.  All superstate edges sharing the same
+ * starting state and character are in this list.
+ */
+struct rx_super_edge
+{
+  struct rx_super_edge *next;
+  struct rx_inx rx_backtrack_frame;
+  int cset_size;
+  rx_Bitset cset;
+  struct rx_distinct_future *options;
+};
+
+/* A superstate is a set of nfa states (RX_SUPERSET) along
+ * with a transition table.  Superstates are built on demand and reclaimed
+ * without warning.  To protect a superstate from this ghastly fate,
+ * use LOCK_SUPERSTATE. 
+ */
+struct rx_superstate
+{
+  int rx_id;			/* c.f. the id field of rx_superset */
+  int locks;			/* protection from reclamation */
+
+  /* Within a superstate cache, all the superstates are kept in a big
+   * queue.  The tail of the queue is the state most likely to be
+   * reclaimed.  The *recyclable fields hold the queue position of 
+   * this state.
+   */
+  struct rx_superstate * next_recyclable;
+  struct rx_superstate * prev_recyclable;
+
+  /* The supernfa edges that exist in the cache and that have
+   * this state as their destination are kept in this list:
+   */
+  struct rx_distinct_future * transition_refs;
+
+  /* The list of nfa states corresponding to this superstate: */
+  struct rx_superset * contents;
+
+  /* The list of edges in the cache beginning from this state. */
+  struct rx_super_edge * edges;
+
+  /* A tail of the recyclable queue is marked as semifree.  A semifree
+   * state has no incoming next_char transitions -- any transition
+   * into a semifree state causes a complex dispatch with the side
+   * effect of rescuing the state from its semifree state.
+   *
+   * An alternative to this might be to make next_char more expensive,
+   * and to move a state to the head of the recyclable queue whenever
+   * it is entered.  That way, popular states would never be recycled.
+   *
+   * But unilaterally making next_char more expensive actually loses.
+   * So, incoming transitions are only made expensive for states near
+   * the tail of the recyclable queue.  The more cache contention
+   * there is, the more frequently a state will have to prove itself
+   * and be moved back to the front of the queue.  If there is less 
+   * contention, then popular states just aggregate in the front of 
+   * the queue and stay there.
+   */
+  int is_semifree;
+
+
+  /* This keeps track of the size of the transition table for this
+   * state.  There is a half-hearted attempt to support variable sized
+   * superstates.
+   */
+  int trans_size;
+
+  /* Indexed by characters... */
+  struct rx_inx transitions[RX_TAIL_ARRAY];
+};
+
+
+/* A list of distinct futures define the edges that leave from a 
+ * given superstate on a given character.  c.f. rx_super_edge.
+ */
+
+struct rx_distinct_future
+{
+  struct rx_distinct_future * next_same_super_edge[2];
+  struct rx_distinct_future * next_same_dest;
+  struct rx_distinct_future * prev_same_dest;
+  struct rx_superstate * present;	/* source state */
+  struct rx_superstate * future;	/* destination state */
+  struct rx_super_edge * edge;
+
+
+  /* The future_frame holds the instruction that should be executed
+   * after all the side effects are done, when it is time to complete
+   * the transition to the next state.
+   *
+   * Normally this is a next_char instruction, but it may be a
+   * cache_miss instruction as well, depending on whether or not
+   * the superstate is in the cache and semifree.
+   * 
+   * If this is the only future for a given superstate/char, and
+   * if there are no side effects to be performed, this frame is
+   * not used (directly) at all.  Instead, its contents are copied
+   * into the transition table of the starting state of this dist. future.
+   */
+  struct rx_inx future_frame;
+
+  struct rx_inx side_effects_frame;
+  struct rx_se_list * effects;
+};
+
+#define rx_lock_superstate(R,S)  ((S)->locks++)
+#define rx_unlock_superstate(R,S) (--(S)->locks)
+
+
+
+/* This page destined for rx.h */
+
+struct rx_blocklist
+{
+  struct rx_blocklist * next;
+  int bytes;
+};
+
+struct rx_freelist
+{
+  struct rx_freelist * next;
+};
+
+struct rx_cache;
+
+#ifdef __STDC__
+typedef void (*rx_morecore_fn)(struct rx_cache *);
+#else
+typedef void (*rx_morecore_fn)();
+#endif
+
+/* You use this to control the allocation of superstate data 
+ * during matching.  Most of it should be initialized to 0.
+ *
+ * A MORECORE function is necessary.  It should allocate
+ * a new block of memory or return 0.
+ * A default that uses malloc is called `rx_morecore'.
+ *
+ * The number of SUPERSTATES_ALLOWED indirectly limits how much memory
+ * the system will try to allocate.  The default is 128.  Batch style
+ * applications that are very regexp intensive should use as high a number
+ * as possible without thrashing.
+ * 
+ * The LOCAL_CSET_SIZE is the number of characters in a character set.
+ * It is therefore the number of entries in a superstate transition table.
+ * Generally, it should be 256.  If your character set has 16 bits, 
+ * it is better to translate your regexps into equivalent 8 bit patterns.
+ */
+
+struct rx_cache
+{
+  struct rx_hash_rules superset_hash_rules;
+
+  /* Objects are allocated by incrementing a pointer that 
+   * scans across rx_blocklists.
+   */
+  struct rx_blocklist * memory;
+  struct rx_blocklist * memory_pos;
+  int bytes_left;
+  char * memory_addr;
+  rx_morecore_fn morecore;
+
+  /* Freelists. */
+  struct rx_freelist * free_superstates;
+  struct rx_freelist * free_transition_classes;
+  struct rx_freelist * free_discernable_futures;
+  struct rx_freelist * free_supersets;
+  struct rx_freelist * free_hash;
+
+  /* Two sets of superstates -- those that are semifreed, and those
+   * that are being used.
+   */
+  struct rx_superstate * lru_superstate;
+  struct rx_superstate * semifree_superstate;
+
+  struct rx_superset * empty_superset;
+
+  int superstates;
+  int semifree_superstates;
+  int hits;
+  int misses;
+  int superstates_allowed;
+
+  int local_cset_size;
+  void ** instruction_table;
+
+  struct rx_hash superset_table;
+};
+
+
+
+
+/* The lowest-level search function supports arbitrarily fragmented
+ * strings and (optionally) suspendable/resumable searches.
+ *
+ * Callers have to provide a few hooks.
+ */
+
+#ifndef __GNUC__
+#ifdef __STDC__
+#define __const__ const
+#else
+#define __const__
+#endif
+#endif
+
+/* This holds a matcher position */
+struct rx_string_position
+{
+  __const__ unsigned char * pos;	/* The current pos. */
+  __const__ unsigned char * string; /* The current string burst. */
+  __const__ unsigned char * end;	/* First invalid position >= POS. */
+  int offset;			/* Integer address of the current burst. */
+  int size;			/* Current string's size. */
+  int search_direction;		/* 1 or -1 */
+  int search_end;		/* First position to not try. */
+};
+
+
+enum rx_get_burst_return
+{
+  rx_get_burst_continuation,
+  rx_get_burst_error,
+  rx_get_burst_ok,
+  rx_get_burst_no_more
+};
+
+
+/* A call to get burst should make POS valid.  It might be invalid
+ * if the STRING field doesn't point to a burst that actually
+ * contains POS.
+ *
+ * GET_BURST should take a clue from SEARCH_DIRECTION (1 or -1) as to
+ * whether or not to pad to the left.  Padding to the right is always
+ * appropriate, but need not go past the point indicated by STOP.
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the get_burst.
+ */
+
+#ifdef __STDC__
+typedef enum rx_get_burst_return
+  (*rx_get_burst_fn) (struct rx_string_position * pos,
+		      void * app_closure,
+		      int stop);
+					       
+#else
+typedef enum rx_get_burst_return (*rx_get_burst_fn) ();
+#endif
+
+
+enum rx_back_check_return
+{
+  rx_back_check_continuation,
+  rx_back_check_error,
+  rx_back_check_pass,
+  rx_back_check_fail
+};
+
+/* Back_check should advance the position it is passed 
+ * over rparen - lparen characters and return pass iff
+ * the characters starting at POS match those indexed
+ * by [LPAREN..RPAREN].
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the back_check.
+ */
+
+#ifdef __STDC__
+typedef enum rx_back_check_return
+  (*rx_back_check_fn) (struct rx_string_position * pos,
+		       int lparen,
+		       int rparen,
+		       unsigned char * translate,
+		       void * app_closure,
+		       int stop);
+					       
+#else
+typedef enum rx_back_check_return (*rx_back_check_fn) ();
+#endif
+
+
+
+
+/* A call to fetch_char should return the character at POS or POS + 1.
+ * Returning continuations here isn't supported.  OFFSET is either 0 or 1
+ * and indicates which characters is desired.
+ */
+
+#ifdef __STDC__
+typedef int (*rx_fetch_char_fn) (struct rx_string_position * pos,
+				 int offset,
+				 void * app_closure,
+				 int stop);
+#else
+typedef int (*rx_fetch_char_fn) ();
+#endif
+
+
+enum rx_search_return
+{
+  rx_search_continuation = -4,
+  rx_search_error = -3,
+  rx_search_soft_fail = -2,	/* failed by running out of string */
+  rx_search_fail = -1		/* failed only by reaching failure states */
+  /* return values >= 0 indicate the position of a successful match */
+};
+
+
+
+
+
+
+
+/* regex.h
+ * 
+ * The remaining declarations replace regex.h.
+ */
+
+/* This is an array of error messages corresponding to the error codes.
+ */
+extern __const__ char *re_error_msg[];
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+/* The regex.c support, as a client of rx, defines a set of possible
+ * side effects that can be added to the edge lables of nfa edges.
+ * Here is the list of sidef effects in use.
+ */
+
+enum re_side_effects
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      NAME VALUE,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     NAME VALUE,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+   re_floogle_flap = 65533
+};
+
+/* These hold paramaters for the kinds of side effects that are possible
+ * in the supported pattern languages.  These include things like the 
+ * numeric bounds of {} operators and the index of paren registers for 
+ * subexpression measurement or backreferencing.
+ */
+struct re_se_params
+{
+  enum re_side_effects se;
+  int op1;
+  int op2;
+};
+
+typedef unsigned reg_syntax_t;
+
+struct re_pattern_buffer
+{
+  struct rx rx;
+  reg_syntax_t syntax;		/* See below for syntax bit definitions. */
+
+  unsigned int no_sub:1;	/* If set, don't  return register offsets. */
+  unsigned int not_bol:1;	/* If set, the anchors ('^' and '$') don't */
+  unsigned int not_eol:1;	/*     match at the ends of the string.  */  
+  unsigned int newline_anchor:1;/* If true, an anchor at a newline matches.*/
+  unsigned int least_subs:1;	/* If set, and returning registers, return
+				 * as few values as possible.  Only 
+				 * backreferenced groups and group 0 (the whole
+				 * match) will be returned.
+				 */
+
+  /* If true, this says that the matcher should keep registers on its
+   * backtracking stack.  For many patterns, we can easily determine that
+   * this isn't necessary.
+   */
+  unsigned int match_regs_on_stack:1;
+  unsigned int search_regs_on_stack:1;
+
+  /* is_anchored and begbuf_only are filled in by rx_compile. */
+  unsigned int is_anchored:1;	/* Anchorded by ^? */
+  unsigned int begbuf_only:1;	/* Anchored to char position 0? */
+
+  
+  /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+   * for `max (RE_NREGS, re_nsub + 1)' groups.
+   * If REGS_REALLOCATE, reallocate space if necessary.
+   * If REGS_FIXED, use what's there.  
+   */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned int regs_allocated:2;
+
+  
+  /* Either a translate table to apply to all characters before
+   * comparing them, or zero for no translation.  The translation
+   * is applied to a pattern when it is compiled and to a string
+   * when it is matched.
+   */
+  unsigned char * translate;
+
+  /* If this is a valid pointer, it tells rx not to store the extents of 
+   * certain subexpressions (those corresponding to non-zero entries).
+   * Passing 0x1 is the same as passing an array of all ones.  Passing 0x0
+   * is the same as passing an array of all zeros.
+   * The array should contain as many entries as their are subexps in the 
+   * regexp.
+   *
+   * For POSIX compatability, when using regcomp and regexec this field
+   * is zeroed and ignored.
+   */
+  char * syntax_parens;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+  void * buffer;		/* Malloced memory for the nfa. */
+  unsigned long allocated;	/* Size of that memory. */
+
+  /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+   * the fastmap, if there is one, to skip over impossible
+   * starting points for matches.  */
+  char *fastmap;
+
+  unsigned int fastmap_accurate:1; /* These three are internal. */
+  unsigned int can_match_empty:1;  
+  struct rx_nfa_state * start;	/* The nfa starting state. */
+
+  /* This is the list of iterator bounds for {lo,hi} constructs.
+   * The memory pointed to is part of the rx->buffer.
+   */
+  struct re_se_params *se_params;
+
+  /* This is a bitset representation of the fastmap.
+   * This is a true fastmap that already takes the translate
+   * table into account.
+   */
+  rx_Bitset fastset;
+};
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+ * `re_match_2' returns information about at least this many registers
+ * the first time a `regs' structure is passed. 
+ *
+ * Also, this is the greatest number of backreferenced subexpressions
+ * allowed in a pattern being matched without caller-supplied registers.
+ */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+extern int rx_cache_bound;
+extern char rx_version_string[];
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+/* This is decls to the interesting subsystems and lower layers
+ * of rx.  Everything which doesn't have a public counterpart in 
+ * regex.c is declared here.
+ */
+
+
+#ifdef __STDC__
+typedef void (*rx_hash_freefn) (struct rx_hash_item * it);
+#else /* ndef __STDC__ */
+typedef void (*rx_hash_freefn) ();
+#endif /* ndef __STDC__ */
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL int rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_empty (int size, rx_Bitset set);
+RX_DECL void rx_bitset_null (int size, rx_Bitset b);
+RX_DECL void rx_bitset_universe (int size, rx_Bitset b);
+RX_DECL void rx_bitset_complement (int size, rx_Bitset b);
+RX_DECL void rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_union (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_intersection (int size,
+				     rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_revdifference (int size,
+				      rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL unsigned long rx_bitset_hash (int size, rx_Bitset b);
+RX_DECL struct rx_hash_item * rx_hash_find (struct rx_hash * table,
+					    unsigned long hash,
+					    void * value,
+					    struct rx_hash_rules * rules);
+RX_DECL struct rx_hash_item * rx_hash_store (struct rx_hash * table,
+					     unsigned long hash,
+					     void * value,
+					     struct rx_hash_rules * rules);
+RX_DECL void rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules);
+RX_DECL void rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+				 struct rx_hash_rules * rules);
+RX_DECL rx_Bitset rx_cset (struct rx *rx);
+RX_DECL rx_Bitset rx_copy_cset (struct rx *rx, rx_Bitset a);
+RX_DECL void rx_free_cset (struct rx * rx, rx_Bitset c);
+RX_DECL struct rexp_node * rexp_node (struct rx *rx,
+				      enum rexp_node_type type);
+RX_DECL struct rexp_node * rx_mk_r_cset (struct rx * rx,
+					 rx_Bitset b);
+RX_DECL struct rexp_node * rx_mk_r_concat (struct rx * rx,
+					   struct rexp_node * a,
+					   struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_alternate (struct rx * rx,
+					      struct rexp_node * a,
+					      struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_opt (struct rx * rx,
+					struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_star (struct rx * rx,
+					 struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_2phase_star (struct rx * rx,
+						struct rexp_node * a,
+						struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_side_effect (struct rx * rx,
+						rx_side_effect a);
+RX_DECL struct rexp_node * rx_mk_r_data  (struct rx * rx,
+					  void * a);
+RX_DECL void rx_free_rexp (struct rx * rx, struct rexp_node * node);
+RX_DECL struct rexp_node * rx_copy_rexp (struct rx *rx,
+					 struct rexp_node *node);
+RX_DECL struct rx_nfa_state * rx_nfa_state (struct rx *rx);
+RX_DECL void rx_free_nfa_state (struct rx_nfa_state * n);
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state (struct rx * rx,
+						  int id);
+RX_DECL struct rx_nfa_edge * rx_nfa_edge (struct rx *rx,
+					  enum rx_nfa_etype type,
+					  struct rx_nfa_state *start,
+					  struct rx_nfa_state *dest);
+RX_DECL void rx_free_nfa_edge (struct rx_nfa_edge * e);
+RX_DECL void rx_free_nfa (struct rx *rx);
+RX_DECL int rx_build_nfa (struct rx *rx,
+			  struct rexp_node *rexp,
+			  struct rx_nfa_state **start,
+			  struct rx_nfa_state **end);
+RX_DECL void rx_name_nfa_states (struct rx *rx);
+RX_DECL int rx_eclose_nfa (struct rx *rx);
+RX_DECL void rx_delete_epsilon_transitions (struct rx *rx);
+RX_DECL int rx_compactify_nfa (struct rx *rx,
+			       void **mem, unsigned long *size);
+RX_DECL void rx_release_superset (struct rx *rx,
+				  struct rx_superset *set);
+RX_DECL struct rx_superset * rx_superset_cons (struct rx * rx,
+					       struct rx_nfa_state *car, struct rx_superset *cdr);
+RX_DECL struct rx_superset * rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl);
+RX_DECL struct rx_superstate * rx_superstate (struct rx *rx,
+					      struct rx_superset *set);
+RX_DECL struct rx_inx * rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data);
+RX_DECL reg_errcode_t rx_compile (__const__ char *pattern, int size,
+				  reg_syntax_t syntax,
+				  struct re_pattern_buffer * rxb);
+RX_DECL void rx_blow_up_fastmap (struct re_pattern_buffer * rxb);
+#else /* STDC */
+RX_DECL int rx_bitset_is_equal ();
+RX_DECL int rx_bitset_is_subset ();
+RX_DECL int rx_bitset_empty ();
+RX_DECL void rx_bitset_null ();
+RX_DECL void rx_bitset_universe ();
+RX_DECL void rx_bitset_complement ();
+RX_DECL void rx_bitset_assign ();
+RX_DECL void rx_bitset_union ();
+RX_DECL void rx_bitset_intersection ();
+RX_DECL void rx_bitset_difference ();
+RX_DECL void rx_bitset_revdifference ();
+RX_DECL void rx_bitset_xor ();
+RX_DECL unsigned long rx_bitset_hash ();
+RX_DECL struct rx_hash_item * rx_hash_find ();
+RX_DECL struct rx_hash_item * rx_hash_store ();
+RX_DECL void rx_hash_free ();
+RX_DECL void rx_free_hash_table ();
+RX_DECL rx_Bitset rx_cset ();
+RX_DECL rx_Bitset rx_copy_cset ();
+RX_DECL void rx_free_cset ();
+RX_DECL struct rexp_node * rexp_node ();
+RX_DECL struct rexp_node * rx_mk_r_cset ();
+RX_DECL struct rexp_node * rx_mk_r_concat ();
+RX_DECL struct rexp_node * rx_mk_r_alternate ();
+RX_DECL struct rexp_node * rx_mk_r_opt ();
+RX_DECL struct rexp_node * rx_mk_r_star ();
+RX_DECL struct rexp_node * rx_mk_r_2phase_star ();
+RX_DECL struct rexp_node * rx_mk_r_side_effect ();
+RX_DECL struct rexp_node * rx_mk_r_data  ();
+RX_DECL void rx_free_rexp ();
+RX_DECL struct rexp_node * rx_copy_rexp ();
+RX_DECL struct rx_nfa_state * rx_nfa_state ();
+RX_DECL void rx_free_nfa_state ();
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state ();
+RX_DECL struct rx_nfa_edge * rx_nfa_edge ();
+RX_DECL void rx_free_nfa_edge ();
+RX_DECL void rx_free_nfa ();
+RX_DECL int rx_build_nfa ();
+RX_DECL void rx_name_nfa_states ();
+RX_DECL int rx_eclose_nfa ();
+RX_DECL void rx_delete_epsilon_transitions ();
+RX_DECL int rx_compactify_nfa ();
+RX_DECL void rx_release_superset ();
+RX_DECL struct rx_superset * rx_superset_cons ();
+RX_DECL struct rx_superset * rx_superstate_eclosure_union ();
+RX_DECL struct rx_superstate * rx_superstate ();
+RX_DECL struct rx_inx * rx_handle_cache_miss ();
+RX_DECL reg_errcode_t rx_compile ();
+RX_DECL void rx_blow_up_fastmap ();
+#endif /* STDC */
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+
+#ifdef __STDC__
+extern int re_search_2 (struct re_pattern_buffer *rxb,
+			__const__ char * string1, int size1,
+			__const__ char * string2, int size2,
+			int startpos, int range,
+			struct re_registers *regs,
+			int stop);
+extern int re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+		      int size, int startpos, int range,
+		      struct re_registers *regs);
+extern int re_match_2 (struct re_pattern_buffer * rxb,
+		       __const__ char * string1, int size1,
+		       __const__ char * string2, int size2,
+		       int pos, struct re_registers *regs, int stop);
+extern int re_match (struct re_pattern_buffer * rxb,
+		     __const__ char * string,
+		     int size, int pos,
+		     struct re_registers *regs);
+extern reg_syntax_t re_set_syntax (reg_syntax_t syntax);
+extern void re_set_registers (struct re_pattern_buffer *bufp,
+			      struct re_registers *regs,
+			      unsigned num_regs,
+			      regoff_t * starts, regoff_t * ends);
+extern __const__ char * re_compile_pattern (__const__ char *pattern,
+					int length,
+					struct re_pattern_buffer * rxb);
+extern int re_compile_fastmap (struct re_pattern_buffer * rxb);
+extern char * re_comp (__const__ char *s);
+extern int re_exec (__const__ char *s);
+extern int regcomp (regex_t * preg, __const__ char * pattern, int cflags);
+extern int regexec (__const__ regex_t *preg, __const__ char *string,
+		    size_t nmatch, regmatch_t pmatch[],
+		    int eflags);
+extern size_t regerror (int errcode, __const__ regex_t *preg,
+			char *errbuf, size_t errbuf_size);
+extern void regfree (regex_t *preg);
+
+#else /* STDC */
+extern int re_search_2 ();
+extern int re_search ();
+extern int re_match_2 ();
+extern int re_match ();
+extern reg_syntax_t re_set_syntax ();
+extern void re_set_registers ();
+extern __const__ char * re_compile_pattern ();
+extern int re_compile_fastmap ();
+extern char * re_comp ();
+extern int re_exec ();
+extern int regcomp ();
+extern int regexec ();
+extern size_t regerror ();
+extern void regfree ();
+
+#endif /* STDC */
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+struct rx_counter_frame
+{
+  int tag;
+  int val;
+  struct rx_counter_frame * inherited_from; /* If this is a copy. */
+  struct rx_counter_frame * cdr;
+};
+
+struct rx_backtrack_frame
+{
+  char * counter_stack_sp;
+
+  /* A frame is used to save the matchers state when it crosses a 
+   * backtracking point.  The `stk_' fields correspond to variables
+   * in re_search_2 (just strip off thes `stk_').  They are documented
+   * tere.
+   */
+  struct rx_superstate * stk_super;
+  unsigned int stk_c;
+  struct rx_string_position stk_test_pos;
+  int stk_last_l;
+  int stk_last_r;
+  int stk_test_ret;
+
+  /* This is the list of options left to explore at the backtrack
+   * point for which this frame was created. 
+   */
+  struct rx_distinct_future * df;
+  struct rx_distinct_future * first_df;
+
+#ifdef RX_DEBUG
+   int stk_line_no;
+#endif
+};
+
+struct rx_stack_chunk
+{
+  struct rx_stack_chunk * next_chunk;
+  int bytes_left;
+  char * sp;
+};
+
+enum rx_outer_entry
+{
+  rx_outer_start,
+  rx_outer_fastmap,
+  rx_outer_test,
+  rx_outer_restore_pos
+};
+
+enum rx_fastmap_return
+{
+  rx_fastmap_continuation,
+  rx_fastmap_error,
+  rx_fastmap_ok,
+  rx_fastmap_fail
+};
+
+enum rx_fastmap_entry
+{
+  rx_fastmap_start,
+  rx_fastmap_string_break
+};
+
+enum rx_test_return
+{
+  rx_test_continuation,
+  rx_test_error,
+  rx_test_fail,
+  rx_test_ok
+};
+
+enum rx_test_internal_return
+{
+  rx_test_internal_error,
+  rx_test_found_first,
+  rx_test_line_finished
+};
+
+enum rx_test_match_entry
+{
+  rx_test_start,
+  rx_test_cache_hit_loop,
+  rx_test_backreference_check,
+  rx_test_backtrack_return
+};
+
+struct rx_search_state
+{
+  /* Two groups of registers are kept.  The group with the register state
+   * of the current test match, and the group that holds the state at the end
+   * of the best known match, if any.
+   *
+   * For some patterns, there may also be registers saved on the stack.
+   */
+  unsigned num_regs;		/* Includes an element for register zero. */
+  regoff_t * lparen;		/* scratch space for register returns */
+  regoff_t * rparen;
+  regoff_t * best_lpspace;	/* in case the user doesn't want these */
+  regoff_t * best_rpspace;	/* values, we still need space to store
+				 * them.  Normally, this memoryis unused
+				 * and the space pointed to by REGS is 
+				 * used instead.
+				 */
+  
+  int last_l;			/* Highest index of a valid lparen. */
+  int last_r;			/* It's dual. */
+  
+  int * best_lparen;		/* This contains the best known register */
+  int * best_rparen;		/* assignments. 
+				 * This may point to the same mem as
+				 * best_lpspace, or it might point to memory
+				 * passed by the caller.
+				 */
+  int best_last_l;		/* best_last_l:best_lparen::last_l:lparen */
+  int best_last_r;
+
+
+  unsigned char * translate;  
+
+  struct rx_string_position outer_pos;
+
+  struct rx_superstate * start_super;
+  int nfa_choice;
+  int first_found;		/* If true, return after finding any match. */
+  int ret_val;
+
+  /* For continuations... */
+  enum rx_outer_entry outer_search_resume_pt;
+  struct re_pattern_buffer * saved_rxb;
+  int saved_startpos;
+  int saved_range;
+  int saved_stop;
+  int saved_total_size;
+  rx_get_burst_fn saved_get_burst;
+  rx_back_check_fn saved_back_check;
+  struct re_registers * saved_regs;
+  
+  /**
+   ** state for fastmap
+   **/
+  char * fastmap;
+  int fastmap_chr;
+  int fastmap_val;
+
+  /* for continuations in the fastmap procedure: */
+  enum rx_fastmap_entry fastmap_resume_pt;
+
+  /**
+   ** state for test_match 
+   **/
+
+  /* The current superNFA position of the matcher. */
+  struct rx_superstate * super;
+  
+  /* The matcher interprets a series of instruction frames.
+   * This is the `instruction counter' for the interpretation.
+   */
+  struct rx_inx * ifr;
+  
+  /* We insert a ghost character in the string to prime
+   * the nfa.  test_pos.pos, test_pos.str_half, and test_pos.end_half
+   * keep track of the test-match position and string-half.
+   */
+  unsigned char c;
+  
+  /* Position within the string. */
+  struct rx_string_position test_pos;
+
+  struct rx_stack_chunk * counter_stack;
+  struct rx_stack_chunk * backtrack_stack;
+  int backtrack_frame_bytes;
+  int chunk_bytes;
+  struct rx_stack_chunk * free_chunks;
+
+  /* To return from this function, set test_ret and 
+   * `goto test_do_return'.
+   *
+   * Possible return values are:
+   *     1   --- end of string while the superNFA is still going
+   *     0   --- internal error (out of memory)
+   *	-1   --- search completed by reaching the superNFA fail state
+   *    -2   --- a match was found, maybe not the longest.
+   *
+   * When the search is complete (-1), best_last_r indicates whether
+   * a match was found.
+   *
+   * -2 is return only if search_state.first_found is non-zero.
+   *
+   * if search_state.first_found is non-zero, a return of -1 indicates no match,
+   * otherwise, best_last_r has to be checked.
+   */
+  int test_ret;
+
+  int could_have_continued;
+  
+#ifdef RX_DEBUG
+  int backtrack_depth;
+  /* There is a search tree with every node as set of deterministic
+   * transitions in the super nfa.  For every branch of a 
+   * backtrack point is an edge in the tree.
+   * This counts up a pre-order of nodes in that tree.
+   * It's saved on the search stack and printed when debugging. 
+   */
+  int line_no;
+  int lines_found;
+#endif
+
+
+  /* For continuations within the match tester */
+  enum rx_test_match_entry test_match_resume_pt;
+  struct rx_inx * saved_next_tr_table;
+  struct rx_inx * saved_this_tr_table;
+  int saved_reg;
+  struct rx_backtrack_frame * saved_bf;
+  
+};
+
+
+
+extern char rx_slowmap[];
+extern unsigned char rx_id_translation[];
+
+static __inline__ void
+init_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  search_state->fastmap = (rxb->fastmap
+			   ? (char *)rxb->fastmap
+			   : (char *)rx_slowmap);
+  /* Update the fastmap now if not correct already. 
+   * When the regexp was compiled, the fastmap was computed
+   * and stored in a bitset.  This expands the bitset into a
+   * character array containing 1s and 0s.
+   */
+  if ((search_state->fastmap == rxb->fastmap) && !rxb->fastmap_accurate)
+    rx_blow_up_fastmap (rxb);
+  search_state->fastmap_chr = -1;
+  search_state->fastmap_val = 0;
+  search_state->fastmap_resume_pt = rx_fastmap_start;
+}
+
+static __inline__ void
+uninit_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  /* Unset the fastmap sentinel */
+  if (search_state->fastmap_chr >= 0)
+    search_state->fastmap[search_state->fastmap_chr]
+      = search_state->fastmap_val;
+}
+
+static __inline__ int
+fastmap_search (rxb, stop, get_burst, app_closure, search_state)
+     struct re_pattern_buffer * rxb;
+     int stop;
+     rx_get_burst_fn get_burst;
+     void * app_closure;
+     struct rx_search_state * search_state;
+{
+  enum rx_fastmap_entry pc;
+
+  if (0)
+    {
+    return_continuation:
+      search_state->fastmap_resume_pt = pc;
+      return rx_fastmap_continuation;
+    }
+
+  pc = search_state->fastmap_resume_pt;
+
+  switch (pc)
+    {
+    default:
+      return rx_fastmap_error;
+    case rx_fastmap_start:
+    init_fastmap_sentinal:
+      /* For the sake of fast fastmapping, set a sentinal in the fastmap.
+       * This sentinal will trap the fastmap loop when it reaches the last
+       * valid character in a string half.
+       *
+       * This must be reset when the fastmap/search loop crosses a string 
+       * boundry, and before returning to the caller.  So sometimes,
+       * the fastmap loop is restarted with `continue', othertimes by
+       * `goto init_fastmap_sentinal'.
+       */
+      if (search_state->outer_pos.size)
+	{
+	  search_state->fastmap_chr = ((search_state->outer_pos.search_direction == 1)
+				       ? *(search_state->outer_pos.end - 1)
+				       : *search_state->outer_pos.string);
+	  search_state->fastmap_val
+	    = search_state->fastmap[search_state->fastmap_chr];
+	  search_state->fastmap[search_state->fastmap_chr] = 1;
+	}
+      else
+	{
+	  search_state->fastmap_chr = -1;
+	  search_state->fastmap_val = 0;
+	}
+      
+      if (search_state->outer_pos.pos >= search_state->outer_pos.end)
+	goto fastmap_hit_bound;
+      else
+	{
+	  if (search_state->outer_pos.search_direction == 1)
+	    {
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      if (*search_state->outer_pos.pos != search_state->fastmap_chr)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  ++search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == search_state->outer_pos.end)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	  else
+	    {
+	      __const__ unsigned char * bound;
+	      bound = search_state->outer_pos.string - 1;
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      if ((*search_state->outer_pos.pos != search_state->fastmap_chr) || search_state->fastmap_val)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  --search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == bound)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	}
+      
+    case rx_fastmap_string_break:
+    fastmap_hit_bound:
+      {
+	/* If we hit a bound, it may be time to fetch another burst
+	 * of string, or it may be time to return a continuation to 
+ 	 * the caller, or it might be time to fail.
+	 */
+
+	int burst_state;
+	burst_state = get_burst (&search_state->outer_pos, app_closure, stop);
+	switch (burst_state)
+	  {
+	  default:
+	  case rx_get_burst_error:
+	    return rx_fastmap_error;
+	  case rx_get_burst_continuation:
+	    {
+	      pc = rx_fastmap_string_break;
+	      goto return_continuation;
+	    }
+	  case rx_get_burst_ok:
+	    goto init_fastmap_sentinal;
+	  case rx_get_burst_no_more:
+	    /* ...not a string split, simply no more string. 
+	     *
+	     * When searching backward, running out of string
+	     * is reason to quit.
+	     *
+	     * When searching forward, we allow the possibility
+	     * of an (empty) match after the last character in the
+	     * virtual string.  So, fall through to the matcher
+	     */
+	    return (  (search_state->outer_pos.search_direction == 1)
+		    ? rx_fastmap_ok
+		    : rx_fastmap_fail);
+	  }
+      }
+    }
+
+}
+
+
+
+
+#ifdef emacs
+/* The `emacs' switch turns on certain matching commands
+ * that make sense only in Emacs. 
+ */
+#include "sysfuncs.h"
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+#endif /* emacs */
+
+/* Setting RX_MEMDBUG is useful if you have dbmalloc.  Maybe with similar
+ * packages too.
+ */
+#ifdef RX_MEMDBUG
+#include <malloc.h>
+#endif /* RX_RX_MEMDBUG */
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+ * `BSTRING', as far as I know, and neither of them use this code.  
+ */
+#if HAVE_STRING_H || STDC_HEADERS
+#include <string.h>
+
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+
+#else /*  HAVE_STRING_H || STDC_HEADERS */
+#include <strings.h>
+#endif   /* not (HAVE_STRING_H || STDC_HEADERS) */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else /* not STDC_HEADERS */
+char *malloc ();
+char *realloc ();
+#endif /* not STDC_HEADERS */
+
+
+
+
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE (1 << CHARBITS)
+
+#ifndef emacs
+/* Define the syntax basics for \<, \>, etc.
+ * This must be nonzero for the wordchar and notwordchar pattern
+ * commands in re_match_2.
+ */
+#ifndef Sword 
+#define Sword 1
+#endif
+#define SYNTAX(c) re_syntax_table[c]
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else
+RX_DECL char re_syntax_table[CHAR_SET_SIZE];
+#endif
+#endif /* not emacs */
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+ *  of `string1' and `string2'.  If only one string, it's `string2'.  
+ */
+
+#define AT_STRINGS_BEG() \
+  (   -1		 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+#define AT_STRINGS_END() \
+  (   (total_size - 1)	 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+
+/* Test if POS + 1 points to a character which is word-constituent.  We have
+ * two special cases to check for: if past the end of string1, look at
+ * the first character in string2; and if before the beginning of
+ * string2, look at the last character in string1.
+ *
+ * Assumes `string1' exists, so use in conjunction with AT_STRINGS_BEG ().  
+ */
+#define LETTER_P(POS,OFF)						\
+  (   SYNTAX (fetch_char(POS, OFF, app_closure, stop))			\
+   == Sword)
+
+/* Test if the character at D and the one after D differ with respect
+ * to being word-constituent.  
+ */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG () || AT_STRINGS_END () || LETTER_P (d,0) != LETTER_P (d, 1))
+
+
+#ifdef RX_SUPPORT_CONTINUATIONS
+#define RX_STACK_ALLOC(BYTES) malloc(BYTES)
+#define RX_STACK_FREE(MEM) free(MEM)
+#else
+#define RX_STACK_ALLOC(BYTES) alloca(BYTES)
+#define RX_STACK_FREE(MEM) \
+      ((struct rx_stack_chunk *)MEM)->next_chunk = search_state.free_chunks; \
+      search_state.free_chunks = ((struct rx_stack_chunk *)MEM);
+
+#endif
+
+#define PUSH(CHUNK_VAR,BYTES)   \
+  if (!CHUNK_VAR || (CHUNK_VAR->bytes_left < (BYTES)))  \
+    {					\
+      struct rx_stack_chunk * new_chunk;	\
+      if (search_state.free_chunks)			\
+	{				\
+	  new_chunk = search_state.free_chunks;	\
+	  search_state.free_chunks = search_state.free_chunks->next_chunk; \
+	}				\
+      else				\
+	{				\
+	  new_chunk = (struct rx_stack_chunk *)RX_STACK_ALLOC(search_state.chunk_bytes); \
+	  if (!new_chunk)		\
+	    {				\
+	      search_state.ret_val = 0;		\
+	      goto test_do_return;	\
+	    }				\
+	}				\
+      new_chunk->sp = (char *)new_chunk + sizeof (struct rx_stack_chunk); \
+      new_chunk->bytes_left = (search_state.chunk_bytes \
+			       - (BYTES) \
+			       - sizeof (struct rx_stack_chunk)); \
+      new_chunk->next_chunk = CHUNK_VAR; \
+      CHUNK_VAR = new_chunk;		\
+    } \
+  else \
+    (CHUNK_VAR->sp += (BYTES)), (CHUNK_VAR->bytes_left -= (BYTES))
+
+#define POP(CHUNK_VAR,BYTES) \
+  if (CHUNK_VAR->sp == ((char *)CHUNK_VAR + sizeof(*CHUNK_VAR))) \
+    { \
+      struct rx_stack_chunk * new_chunk = CHUNK_VAR->next_chunk; \
+      RX_STACK_FREE(CHUNK_VAR); \
+      CHUNK_VAR = new_chunk; \
+    } \
+  else \
+    (CHUNK_VAR->sp -= BYTES), (CHUNK_VAR->bytes_left += BYTES)
+
+
+
+#define SRCH_TRANSLATE(C)  search_state.translate[(unsigned char) (C)]
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL __inline__ int
+rx_search  (struct re_pattern_buffer * rxb,
+	    int startpos,
+	    int range,
+	    int stop,
+	    int total_size,
+	    rx_get_burst_fn get_burst,
+	    rx_back_check_fn back_check,
+	    rx_fetch_char_fn fetch_char,
+	    void * app_closure,
+	    struct re_registers * regs,
+	    struct rx_search_state * resume_state,
+	    struct rx_search_state * save_state)
+#else
+RX_DECL __inline__ int
+rx_search  (rxb, startpos, range, stop, total_size,
+	    get_burst, back_check, fetch_char,
+	    app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+#endif
+{
+  int pc;
+  int test_state;
+  struct rx_search_state search_state;
+
+  search_state.free_chunks = 0;
+  if (!resume_state)
+    pc = rx_outer_start;
+  else
+    {
+      search_state = *resume_state;
+      regs = search_state.saved_regs;
+      rxb = search_state.saved_rxb;
+      startpos = search_state.saved_startpos;
+      range = search_state.saved_range;
+      stop = search_state.saved_stop;
+      total_size = search_state.saved_total_size;
+      get_burst = search_state.saved_get_burst;
+      back_check = search_state.saved_back_check;
+      pc = search_state.outer_search_resume_pt;
+      if (0)
+	{
+	return_continuation:
+	  if (save_state)
+	    {
+	      *save_state = search_state;
+	      save_state->saved_regs = regs;
+	      save_state->saved_rxb = rxb;
+	      save_state->saved_startpos = startpos;
+	      save_state->saved_range = range;
+	      save_state->saved_stop = stop;
+	      save_state->saved_total_size = total_size;
+	      save_state->saved_get_burst = get_burst;
+	      save_state->saved_back_check = back_check;
+	      save_state->outer_search_resume_pt = pc;
+	    }
+	  return rx_search_continuation;
+	}
+    }
+
+  switch (pc)
+    {
+    case rx_outer_start:
+      search_state.ret_val = rx_search_fail;
+      (  search_state.lparen
+       = search_state.rparen
+       = search_state.best_lpspace
+       = search_state.best_rpspace
+       = 0);
+      
+      /* figure the number of registers we may need for use in backreferences.
+       * the number here includes an element for register zero.  
+       */
+      search_state.num_regs = rxb->re_nsub + 1;
+      
+      
+      /* check for out-of-range startpos.  */
+      if ((startpos < 0) || (startpos > total_size))
+	return rx_search_fail;
+      
+      /* fix up range if it might eventually take us outside the string. */
+      {
+	int endpos;
+	endpos = startpos + range;
+	if (endpos < -1)
+	  range = (-1 - startpos);
+	else if (endpos > (total_size + 1))
+	  range = total_size - startpos;
+      }
+      
+      /* if the search isn't to be a backwards one, don't waste time in a
+       * long search for a pattern that says it is anchored.
+       */
+      if (rxb->begbuf_only && (range > 0))
+	{
+	  if (startpos > 0)
+	    return rx_search_fail;
+	  else
+	    range = 1;
+	}
+      
+      /* decide whether to use internal or user-provided reg buffers. */
+      if (!regs || rxb->no_sub)
+	{
+	  search_state.best_lpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_rpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_lparen = search_state.best_lpspace;
+	  search_state.best_rparen = search_state.best_rpspace;
+	}
+      else
+	{	
+	  /* have the register data arrays been allocated?  */
+	  if (rxb->regs_allocated == REGS_UNALLOCATED)
+	    { /* no.  so allocate them with malloc.  we need one
+		 extra element beyond `search_state.num_regs' for the `-1' marker
+		 gnu code uses.  */
+	      regs->num_regs = MAX (RE_NREGS, rxb->re_nsub + 1);
+	      regs->start = ((regoff_t *)
+			     malloc (regs->num_regs * sizeof ( regoff_t)));
+	      regs->end = ((regoff_t *)
+			   malloc (regs->num_regs * sizeof ( regoff_t)));
+	      if (regs->start == 0 || regs->end == 0)
+		return rx_search_error;
+	      rxb->regs_allocated = REGS_REALLOCATE;
+	    }
+	  else if (rxb->regs_allocated == REGS_REALLOCATE)
+	    { /* yes.  if we need more elements than were already
+		 allocated, reallocate them.  if we need fewer, just
+		 leave it alone.  */
+	      if (regs->num_regs < search_state.num_regs + 1)
+		{
+		  regs->num_regs = search_state.num_regs + 1;
+		  regs->start = ((regoff_t *)
+				 realloc (regs->start,
+					  regs->num_regs * sizeof (regoff_t)));
+		  regs->end = ((regoff_t *)
+			       realloc (regs->end,
+					regs->num_regs * sizeof ( regoff_t)));
+		  if (regs->start == 0 || regs->end == 0)
+		    return rx_search_error;
+		}
+	    }
+	  else if (rxb->regs_allocated != REGS_FIXED)
+	    return rx_search_error;
+	  
+	  if (regs->num_regs < search_state.num_regs + 1)
+	    {
+	      search_state.best_lpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_rpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_lparen = search_state.best_lpspace;
+	      search_state.best_rparen = search_state.best_rpspace;
+	    }
+	  else
+	    {
+	      search_state.best_lparen = regs->start;
+	      search_state.best_rparen = regs->end;
+	    }
+	}
+      
+      search_state.lparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+      search_state.rparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)); 
+      
+      if (! (   search_state.best_rparen
+	     && search_state.best_lparen
+	     && search_state.lparen && search_state.rparen))
+	return rx_search_error;
+      
+      search_state.best_last_l = search_state.best_last_r = -1;
+      
+      search_state.translate = (rxb->translate
+				? rxb->translate
+				: rx_id_translation);
+      
+      
+      
+      /*
+       * two nfa's were compiled.  
+       * `0' is complete.
+       * `1' faster but gets registers wrong and ends too soon.
+       */
+      search_state.nfa_choice = (regs && !rxb->least_subs) ? '\0' : '\1';
+      
+      /* we have the option to look for the best match or the first
+       * one we can find.  if the user isn't asking for register information,
+       * we don't need to find the best match.
+       */
+      search_state.first_found = !regs;
+      
+      if (range >= 0)
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = 1;
+	}
+      else
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = -1;
+	}
+      
+      /* the vacuous search always turns up nothing. */
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos > search_state.outer_pos.search_end)
+	  : (startpos < search_state.outer_pos.search_end))
+	return rx_search_fail;
+      
+      /* now we build the starting state of the supernfa. */
+      {
+	struct rx_superset * start_contents;
+	struct rx_nfa_state_set * start_nfa_set;
+	
+	/* we presume here that the nfa start state has only one
+	 * possible future with no side effects.  
+	 */
+	start_nfa_set = rxb->start->futures->destset;
+	if (   rxb->rx.start_set
+	    && (rxb->rx.start_set->starts_for == &rxb->rx))
+	  start_contents = rxb->rx.start_set;
+	else
+	  {
+	    start_contents =
+	      rx_superstate_eclosure_union (&rxb->rx,
+					    rx_superset_cons (&rxb->rx, 0, 0),
+					    start_nfa_set);
+	    
+	    if (!start_contents)
+	      return rx_search_fail;
+	    
+	    start_contents->starts_for = &rxb->rx;
+	    rxb->rx.start_set = start_contents;
+	  }
+	if (   start_contents->superstate
+	    && (start_contents->superstate->rx_id == rxb->rx.rx_id))
+	  {
+	    search_state.start_super = start_contents->superstate;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	  }
+	else
+	  {
+	    rx_protect_superset (&rxb->rx, start_contents);
+	    
+	    search_state.start_super = rx_superstate (&rxb->rx, start_contents);
+	    if (!search_state.start_super)
+	      return rx_search_fail;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	    rx_release_superset (&rxb->rx, start_contents);
+	  }
+      }
+      
+      
+      /* The outer_pos tracks the position within the strings
+       * as seen by loop that calls fastmap_search.
+       *
+       * The caller supplied get_burst function actually 
+       * gives us pointers to chars.
+       * 
+       * Communication with the get_burst function is through an
+       * rx_string_position structure.  Here, the structure for
+       * outer_pos is initialized.   It is set to point to the
+       * NULL string, at an offset of STARTPOS.  STARTPOS is out
+       * of range of the NULL string, so the first call to 
+       * getburst will patch up the rx_string_position to point
+       * to valid characters.
+       */
+
+      (  search_state.outer_pos.string
+       = search_state.outer_pos.end
+       = 0);
+
+      search_state.outer_pos.offset = 0;
+      search_state.outer_pos.size = 0;
+      search_state.outer_pos.pos = (unsigned char *)startpos;
+      init_fastmap (rxb, &search_state);
+
+      search_state.fastmap_resume_pt = rx_fastmap_start;
+    case rx_outer_fastmap:
+      /* do { */
+    pseudo_do:
+      {
+	{
+	  int fastmap_state;
+	  fastmap_state = fastmap_search (rxb, stop, get_burst, app_closure,
+					  &search_state);
+	  switch (fastmap_state)
+	    {
+	    case rx_fastmap_continuation:
+	      pc = rx_outer_fastmap;
+	      goto return_continuation;
+	    case rx_fastmap_fail:
+	      goto finish;
+	    case rx_fastmap_ok:
+	      break;
+	    }
+	}
+	
+	/* now the fastmap loop has brought us to a plausible 
+	 * starting point for a match.  so, it's time to run the
+	 * nfa and see if a match occured.
+	 */
+	startpos = (  search_state.outer_pos.pos
+		    - search_state.outer_pos.string
+		    + search_state.outer_pos.offset);
+/*|*/	if ((range > 0) && (startpos == search_state.outer_pos.search_end))
+/*|*/	  goto finish;
+      }
+
+      search_state.test_match_resume_pt = rx_test_start;
+      /* do interrupted for entry point... */
+    case rx_outer_test:
+      /* ...do continued */
+      {
+	goto test_match;
+      test_returns_to_search:
+	switch (test_state)
+	  {
+	  case rx_test_continuation:
+	    pc = rx_outer_test;
+	    goto return_continuation;
+	  case rx_test_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_test_fail:
+	    break;
+	  case rx_test_ok:
+	    goto finish;
+	  }
+	search_state.outer_pos.pos += search_state.outer_pos.search_direction;
+	startpos += search_state.outer_pos.search_direction;
+#if 0
+/*|*/	if (search_state.test_pos.pos < search_state.test_pos.end)
+/*|*/	  break;
+#endif
+      }
+      /* do interrupted for entry point... */
+    case rx_outer_restore_pos:
+      {
+	int x;
+	x = get_burst (&search_state.outer_pos, app_closure, stop);
+	switch (x)
+	  {
+	  case rx_get_burst_continuation:
+	    pc = rx_outer_restore_pos;
+	    goto return_continuation;
+	  case rx_get_burst_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_get_burst_no_more:
+	    if (rxb->can_match_empty)
+	      break;
+	    goto finish;
+	  case rx_get_burst_ok:
+	    break;
+	  }
+      } /* } while (...see below...) */
+
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos < search_state.outer_pos.search_end)
+	  : (startpos > search_state.outer_pos.search_end))
+	goto pseudo_do;
+
+	
+    finish:
+      uninit_fastmap (rxb, &search_state);
+      if (search_state.start_super)
+	rx_unlock_superstate (&rxb->rx, search_state.start_super);
+      
+#ifdef regex_malloc
+      if (search_state.lparen) free (search_state.lparen);
+      if (search_state.rparen) free (search_state.rparen);
+      if (search_state.best_lpspace) free (search_state.best_lpspace);
+      if (search_state.best_rpspace) free (search_state.best_rpspace);
+#endif
+      return search_state.ret_val;
+    }
+
+
+ test_match:
+  {
+    enum rx_test_match_entry test_pc;
+    int inx;
+    test_pc = search_state.test_match_resume_pt;
+    if (test_pc == rx_test_start)
+      {
+#ifdef RX_DEBUG
+	search_state.backtrack_depth = 0;
+#endif
+	search_state.last_l = search_state.last_r = 0;
+	search_state.lparen[0] = startpos;
+	search_state.super = search_state.start_super;
+	search_state.c = search_state.nfa_choice;
+	search_state.test_pos.pos = search_state.outer_pos.pos - 1;    
+	search_state.test_pos.string = search_state.outer_pos.string;
+	search_state.test_pos.end = search_state.outer_pos.end;
+	search_state.test_pos.offset = search_state.outer_pos.offset;
+	search_state.test_pos.size = search_state.outer_pos.size;
+	search_state.test_pos.search_direction = 1;
+	search_state.counter_stack = 0;
+	search_state.backtrack_stack = 0;
+	search_state.backtrack_frame_bytes =
+	  (sizeof (struct rx_backtrack_frame)
+	   + (rxb->match_regs_on_stack
+	      ? sizeof (regoff_t) * (search_state.num_regs + 1) * 2
+	      : 0));
+	search_state.chunk_bytes = search_state.backtrack_frame_bytes * 64;
+	search_state.test_ret = rx_test_line_finished;
+	search_state.could_have_continued = 0;
+      }  
+    /* This is while (1)...except that the body of the loop is interrupted 
+     * by some alternative entry points.
+     */
+  pseudo_while_1:
+    switch (test_pc)
+      {
+      case rx_test_cache_hit_loop:
+	goto resume_continuation_1;
+      case rx_test_backreference_check:
+	goto resume_continuation_2;
+      case rx_test_backtrack_return:
+	goto resume_continuation_3;
+      case rx_test_start:
+#ifdef RX_DEBUG
+	/* There is a search tree with every node as set of deterministic
+	 * transitions in the super nfa.  For every branch of a 
+	 * backtrack point is an edge in the tree.
+	 * This counts up a pre-order of nodes in that tree.
+	 * It's saved on the search stack and printed when debugging. 
+	 */
+	search_state.line_no = 0;
+	search_state.lines_found = 0;
+#endif
+	
+      top_of_cycle:
+	/* A superstate is basicly a transition table, indexed by 
+	 * characters from the string being tested, and containing 
+	 * RX_INX (`instruction frame') structures.
+	 */
+	search_state.ifr = &search_state.super->transitions [search_state.c];
+	
+      recurse_test_match:
+	/* This is the point to which control is sent when the
+	 * test matcher `recurses'.  Before jumping here, some variables
+	 * need to be saved on the stack and the next instruction frame
+	 * has to be computed.
+	 */
+	
+      restart:
+	/* Some instructions don't advance the matcher, but just
+	 * carry out some side effects and fetch a new instruction.
+	 * To dispatch that new instruction, `goto restart'.
+	 */
+	
+	{
+	  struct rx_inx * next_tr_table;
+	  struct rx_inx * this_tr_table;
+	  /* The fastest route through the loop is when the instruction 
+	   * is RX_NEXT_CHAR.  This case is detected when SEARCH_STATE.IFR->DATA
+	   * is non-zero.  In that case, it points to the next
+	   * superstate. 
+	   *
+	   * This allows us to not bother fetching the bytecode.
+	   */
+	  next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	  this_tr_table = search_state.super->transitions;
+	  while (next_tr_table)
+	    {
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		{
+		  struct rx_superset * setp;
+		  
+		  fprintf (stderr, "%d %d>> re_next_char @ %d (%d)",
+			   search_state.line_no,
+			   search_state.backtrack_depth,
+			   (search_state.test_pos.pos - search_state.test_pos.string
+			    + search_state.test_pos.offset), search_state.c);
+		  
+		  search_state.super =
+		    ((struct rx_superstate *)
+		     ((char *)this_tr_table
+		      - ((unsigned long)
+			 ((struct rx_superstate *)0)->transitions)));
+		  
+		  setp = search_state.super->contents;
+		  fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+			   rxb->rx.rx_id, setp);
+		  while (setp)
+		    {
+		      fprintf (stderr, "%d ", setp->id);
+		      setp = setp->cdr;
+		    }
+		  fprintf (stderr, "\n");
+		}
+#endif
+	      this_tr_table = next_tr_table;
+	      ++search_state.test_pos.pos;
+	      if (search_state.test_pos.pos == search_state.test_pos.end)
+		{
+		  int burst_state;
+		try_burst_1:
+		  burst_state = get_burst (&search_state.test_pos,
+					   app_closure, stop);
+		  switch (burst_state)
+		    {
+		    case rx_get_burst_continuation:
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      test_pc = rx_test_cache_hit_loop;
+		      goto test_return_continuation;
+		      
+		    resume_continuation_1:
+		      /* Continuation one jumps here to do its work: */
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      goto try_burst_1;
+		      
+		    case rx_get_burst_ok:
+		      /* get_burst succeeded...keep going */
+		      break;
+		      
+		    case rx_get_burst_no_more:
+		      search_state.test_ret = rx_test_line_finished;
+		      search_state.could_have_continued = 1;
+		      goto test_do_return;
+		      
+		    case rx_get_burst_error:
+		      /* An error... */
+		      search_state.test_ret = rx_test_internal_error;
+		      goto test_do_return;
+		    }
+		}
+	      search_state.c = *search_state.test_pos.pos;
+	      search_state.ifr = this_tr_table + search_state.c;
+	      next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	    } /* Fast loop through cached transition tables */
+	  
+	  /* Here when we ran out of cached next-char transitions. 
+	   * So, it will be necessary to do a more expensive
+	   * dispatch on the current instruction.  The superstate
+	   * pointer is allowed to become invalid during next-char
+	   * transitions -- now we must bring it up to date.
+	   */
+	  search_state.super =
+	    ((struct rx_superstate *)
+	     ((char *)this_tr_table
+	      - ((unsigned long)
+		 ((struct rx_superstate *)0)->transitions)));
+	}
+	
+	/* We've encountered an instruction other than next-char.
+	 * Dispatch that instruction:
+	 */
+	inx = (int)search_state.ifr->inx;
+#ifdef RX_DEBUG_0
+	if (rx_debug_trace)
+	  {
+	    struct rx_superset * setp = search_state.super->contents;
+	    
+	    fprintf (stderr, "%d %d>> %s @ %d (%d)", search_state.line_no,
+		     search_state.backtrack_depth,
+		     inx_names[inx],
+		     (search_state.test_pos.pos - search_state.test_pos.string
+		      + (test_pos.half == 0 ? 0 : size1)), search_state.c);
+	    
+	    fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+		     rxb->rx.rx_id, setp);
+	    while (setp)
+	      {
+		fprintf (stderr, "%d ", setp->id);
+		setp = setp->cdr;
+	      }
+	    fprintf (stderr, "\n");
+	  }
+#endif
+	switch ((enum rx_opcode)inx)
+	  {
+	  case rx_do_side_effects:
+	    
+	    /*  RX_DO_SIDE_EFFECTS occurs when we cross epsilon 
+	     *  edges associated with parentheses, backreferencing, etc.
+	     */
+	    {
+	      struct rx_distinct_future * df =
+		(struct rx_distinct_future *)search_state.ifr->data_2;
+	      struct rx_se_list * el = df->effects;
+	      /* Side effects come in lists.  This walks down
+	       * a list, dispatching.
+	       */
+	      while (el)
+		{
+		  long effect;
+		  effect = (long)el->car;
+		  if (effect < 0)
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			{
+			  struct rx_superset * setp = search_state.super->contents;
+			  
+			  fprintf (stderr, "....%d %d>> %s\n", search_state.line_no,
+				   search_state.backtrack_depth,
+				   efnames[-effect]);
+			}
+#endif
+		      switch ((enum re_side_effects) effect)
+
+			{
+			case re_se_pushback:
+			  search_state.ifr = &df->future_frame;
+			  if (!search_state.ifr->data)
+			    {
+			      struct rx_superstate * sup;
+			      sup = search_state.super;
+			      rx_lock_superstate (rx, sup);
+			      if (!rx_handle_cache_miss (&rxb->rx,
+							 search_state.super,
+							 search_state.c,
+							 (search_state.ifr
+							  ->data_2)))
+				{
+				  rx_unlock_superstate (rx, sup);
+				  search_state.test_ret = rx_test_internal_error;
+				  goto test_do_return;
+				}
+			      rx_unlock_superstate (rx, sup);
+			    }
+			  /* --search_state.test_pos.pos; */
+			  search_state.c = 't';
+			  search_state.super
+			    = ((struct rx_superstate *)
+			       ((char *)search_state.ifr->data
+				- (long)(((struct rx_superstate *)0)
+					 ->transitions)));
+			  goto top_of_cycle;
+			  break;
+			case re_se_push0:
+			  {
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH (search_state.counter_stack,
+				  sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_iter;
+			    cf->val = 0;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			case re_se_fail:
+			  goto test_do_return;
+			case re_se_begbuf:
+			  if (!AT_STRINGS_BEG ())
+			    goto test_do_return;
+			  break;
+			case re_se_endbuf:
+			  if (!AT_STRINGS_END ())
+			    goto test_do_return;
+			  break;
+			case re_se_wordbeg:
+			  if (   LETTER_P (&search_state.test_pos, 1)
+			      && (   AT_STRINGS_BEG()
+				  || !LETTER_P (&search_state.test_pos, 0)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordend:
+			  if (   !AT_STRINGS_BEG ()
+			      && LETTER_P (&search_state.test_pos, 0)
+			      && (AT_STRINGS_END ()
+				  || !LETTER_P (&search_state.test_pos, 1)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordbound:
+			  if (AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_notwordbound:
+			  if (!AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_hat:
+			  if (AT_STRINGS_BEG ())
+			    {
+			      if (rxb->not_bol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      char pos_c = *search_state.test_pos.pos;
+			      if (   (SRCH_TRANSLATE (pos_c)
+				      == SRCH_TRANSLATE('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			case re_se_dollar:
+			  if (AT_STRINGS_END ())
+			    {
+			      if (rxb->not_eol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      if (   (   SRCH_TRANSLATE (fetch_char
+						    (&search_state.test_pos, 1,
+						     app_closure, stop))
+				      == SRCH_TRANSLATE ('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			  
+			case re_se_try:
+			  /* This is the first side effect in every
+			   * expression.
+			   *
+			   *  FOR NO GOOD REASON...get rid of it...
+			   */
+			  break;
+			  
+			case re_se_pushpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH(search_state.counter_stack,
+				 sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_pushpos;
+			    cf->val = urhere;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			  
+			case re_se_chkpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * cf
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (cf->val == urhere)
+			      goto test_do_return;
+			    cf->val = urhere;
+			    break;
+			  }
+			  break;
+			  
+			case re_se_poppos:
+			  POP(search_state.counter_stack,
+			      sizeof (struct rx_counter_frame));
+			  break;
+			  
+			  
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+#ifdef emacs
+			  /* 
+			   * this release lacks emacs support
+			   */
+#endif
+			  break;
+			case re_se_win:
+			case re_se_lparen:
+			case re_se_rparen:
+			case re_se_backref:
+			case re_se_iter:
+			case re_se_end_iter:
+			case re_se_tv:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  else
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			fprintf (stderr, "....%d %d>> %s %d %d\n", search_state.line_no,
+				 search_state.backtrack_depth,
+				 efnames2[rxb->se_params [effect].se],
+				 rxb->se_params [effect].op1,
+				 rxb->se_params [effect].op2);
+#endif
+		      switch (rxb->se_params [effect].se)
+			{
+			case re_se_win:
+			  /* This side effect indicates that we've 
+			   * found a match, though not necessarily the 
+			   * best match.  This is a fancy assignment to 
+			   * register 0 unless the caller didn't 
+			   * care about registers.  In which case,
+			   * this stops the match.
+			   */
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    if (   (search_state.best_last_r < 0)
+				|| (urhere + 1 > search_state.best_rparen[0]))
+			      {
+				/* Record the best known and keep
+				 * looking.
+				 */
+				int x;
+				for (x = 0; x <= search_state.last_l; ++x)
+				  search_state.best_lparen[x] = search_state.lparen[x];
+				search_state.best_last_l = search_state.last_l;
+				for (x = 0; x <= search_state.last_r; ++x)
+				  search_state.best_rparen[x] = search_state.rparen[x];
+				search_state.best_rparen[0] = urhere + 1;
+				search_state.best_last_r = search_state.last_r;
+			      }
+			    /* If we're not reporting the match-length 
+			     * or other register info, we need look no
+			     * further.
+			     */
+			    if (search_state.first_found)
+			      {
+				search_state.test_ret = rx_test_found_first;
+				goto test_do_return;
+			      }
+			  }
+			  break;
+			case re_se_lparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    int reg = rxb->se_params [effect].op1;
+#if 0
+			    if (reg > search_state.last_l)
+#endif
+			      {
+				search_state.lparen[reg] = urhere + 1;
+				/* In addition to making this assignment,
+				 * we now know that lower numbered regs
+				 * that haven't already been assigned,
+				 * won't be.  We make sure they're
+				 * filled with -1, so they can be
+				 * recognized as unassigned.
+				 */
+				if (search_state.last_l < reg)
+				  while (++search_state.last_l < reg)
+				    search_state.lparen[search_state.last_l] = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_rparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    int reg = rxb->se_params [effect].op1;
+			    search_state.rparen[reg] = urhere + 1;
+			    if (search_state.last_r < reg)
+			      {
+				while (++search_state.last_r < reg)
+				  search_state.rparen[search_state.last_r]
+				    = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_backref:
+			  {
+			    int reg = rxb->se_params [effect].op1;
+			    if (   reg > search_state.last_r
+				|| search_state.rparen[reg] < 0)
+			      goto test_do_return;
+			    
+			    {
+			      int backref_status;
+			    check_backreference:
+			      backref_status
+				= back_check (&search_state.test_pos,
+					      search_state.lparen[reg],
+					      search_state.rparen[reg],
+					      search_state.translate,
+					      app_closure,
+					      stop);
+			      switch (backref_status)
+				{
+				case rx_back_check_continuation:
+				  search_state.saved_reg = reg;
+				  test_pc = rx_test_backreference_check;
+				  goto test_return_continuation;
+				resume_continuation_2:
+				  reg = search_state.saved_reg;
+				  goto check_backreference;
+				case rx_back_check_fail:
+				  /* Fail */
+				  goto test_do_return;
+				case rx_back_check_pass:
+				  /* pass --
+				   * test_pos now advanced to last
+				   * char matched by backref
+				   */
+				  break;
+				}
+			    }
+			    break;
+			  }
+			case re_se_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val == rxb->se_params[effect].op2)
+			      goto test_do_return;
+			    else
+			      ++csp->val;
+			    break;
+			  }
+			case re_se_end_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val < rxb->se_params[effect].op1)
+			      goto test_do_return;
+			    else
+			      {
+				struct rx_counter_frame * source = csp;
+				while (source->inherited_from)
+				  source = source->inherited_from;
+				if (!source || !source->cdr)
+				  {
+				    POP(search_state.counter_stack,
+					sizeof(struct rx_counter_frame));
+				  }
+				else
+				  {
+				    source = source->cdr;
+				    csp->val = source->val;
+				    csp->tag = source->tag;
+				    csp->cdr = 0;
+				    csp->inherited_from = source;
+				  }
+			      }
+			    break;
+			  }
+			case re_se_tv:
+			  /* is a noop */
+			  break;
+			case re_se_try:
+			case re_se_pushback:
+			case re_se_push0:
+			case re_se_pushpos:
+			case re_se_chkpos:
+			case re_se_poppos:
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+			case re_se_begbuf:
+			case re_se_hat:
+			case re_se_wordbeg:
+			case re_se_wordbound:
+			case re_se_notwordbound:
+			case re_se_wordend:
+			case re_se_endbuf:
+			case re_se_dollar:
+			case re_se_fail:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  el = el->cdr;
+		}
+	      /* Now the side effects are done,
+	       * so get the next instruction.
+	       * and move on.
+	       */
+	      search_state.ifr = &df->future_frame;
+	      goto restart;
+	    }
+	    
+	  case rx_backtrack_point:
+	    {
+	      /* A backtrack point indicates that we've reached a
+	       * non-determinism in the superstate NFA.  This is a
+	       * loop that exhaustively searches the possibilities.
+	       *
+	       * A backtracking strategy is used.  We keep track of what
+	       * registers are valid so we can erase side effects.
+	       *
+	       * First, make sure there is some stack space to hold 
+	       * our state.
+	       */
+	      
+	      struct rx_backtrack_frame * bf;
+	      
+	      PUSH(search_state.backtrack_stack,
+		   search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG_0
+	      ++search_state.backtrack_depth;
+#endif
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+	      {
+		bf->stk_super = search_state.super;
+		/* We prevent the current superstate from being
+		 * deleted from the superstate cache.
+		 */
+		rx_lock_superstate (&rxb->rx, search_state.super);
+#ifdef RX_DEBUG_0
+		bf->stk_search_state.line_no = search_state.line_no;
+#endif
+		bf->stk_c = search_state.c;
+		bf->stk_test_pos = search_state.test_pos;
+		bf->stk_last_l = search_state.last_l;
+		bf->stk_last_r = search_state.last_r;
+		bf->df = ((struct rx_super_edge *)
+			  search_state.ifr->data_2)->options;
+		bf->first_df = bf->df;
+		bf->counter_stack_sp = (search_state.counter_stack
+					? search_state.counter_stack->sp
+					: 0);
+		bf->stk_test_ret = search_state.test_ret;
+		if (rxb->match_regs_on_stack)
+		  {
+		    int x;
+		    regoff_t * stk =
+		      (regoff_t *)((char *)bf + sizeof (*bf));
+		    for (x = 0; x <= search_state.last_l; ++x)
+		      stk[x] = search_state.lparen[x];
+		    stk += x;
+		    for (x = 0; x <= search_state.last_r; ++x)
+		      stk[x] = search_state.rparen[x];
+		  }
+	      }
+	      
+	      /* Here is a while loop whose body is mainly a function
+	       * call and some code to handle a return from that
+	       * function.
+	       *
+	       * From here on for the rest of `case backtrack_point' it
+	       * is unsafe to assume that the search_state copies of 
+	       * variables saved on the backtracking stack are valid
+	       * -- so read their values from the backtracking stack.
+	       *
+	       * This lets us use one generation fewer stack saves in
+	       * the call-graph of a search.
+	       */
+	      
+	    while_non_det_options:
+#ifdef RX_DEBUG_0
+	      ++search_state.lines_found;
+	      if (rx_debug_trace)
+		fprintf (stderr, "@@@ %d calls %d @@@\n",
+			 search_state.line_no, search_state.lines_found);
+	      
+	      search_state.line_no = search_state.lines_found;
+#endif
+	      
+	      if (bf->df->next_same_super_edge[0] == bf->first_df)
+		{
+		  /* This is a tail-call optimization -- we don't recurse
+		   * for the last of the possible futures.
+		   */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  
+		  rx_unlock_superstate (&rxb->rx, search_state.super);
+		  POP(search_state.backtrack_stack,
+		      search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG
+		  --search_state.backtrack_depth;
+#endif
+		  goto restart;
+		}
+	      else
+		{
+		  if (search_state.counter_stack)
+		    {
+		      struct rx_counter_frame * old_cf
+			= ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      struct rx_counter_frame * cf;
+		      PUSH(search_state.counter_stack, sizeof (struct rx_counter_frame));
+		      cf = ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      cf->tag = old_cf->tag;
+		      cf->val = old_cf->val;
+		      cf->inherited_from = old_cf;
+		      cf->cdr = 0;
+		    }			
+		  /* `Call' this test-match block */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  goto recurse_test_match;
+		}
+	      
+	      /* Returns in this block are accomplished by
+	       * goto test_do_return.  There are two cases.
+	       * If there is some search-stack left,
+	       * then it is a return from a `recursive' call.
+	       * If there is no search-stack left, then
+	       * we should return to the fastmap/search loop.
+	       */
+	      
+	    test_do_return:
+	      
+	      if (!search_state.backtrack_stack)
+		{
+#ifdef RX_DEBUG_0
+		  if (rx_debug_trace)
+		    fprintf (stderr, "!!! %d bails returning %d !!!\n",
+			     search_state.line_no, search_state.test_ret);
+#endif
+		  
+		  /* No more search-stack -- this test is done. */
+		  if (search_state.test_ret)
+		    goto return_from_test_match;
+		  else
+		    goto error_in_testing_match;
+		}
+	      
+	      /* Returning from a recursive call to 
+	       * the test match block:
+	       */
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		fprintf (stderr, "+++ %d returns %d (to %d)+++\n",
+			 search_state.line_no,
+			 search_state.test_ret,
+			 bf->stk_search_state.line_no);
+#endif
+	      
+	      while (search_state.counter_stack
+		     && (!bf->counter_stack_sp
+			 || (bf->counter_stack_sp
+			     != search_state.counter_stack->sp)))
+		{
+		  POP(search_state.counter_stack,
+		      sizeof (struct rx_counter_frame));
+		}
+	      
+	      if (search_state.test_ret == rx_test_error)
+		{
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      /* If a non-longest match was found and that is good 
+	       * enough, return immediately.
+	       */
+	      if (   (search_state.test_ret == rx_test_found_first)
+		  && search_state.first_found)
+		{
+		  rx_unlock_superstate (&rxb->rx, bf->stk_super);
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      search_state.test_ret = bf->stk_test_ret;
+	      search_state.last_l = bf->stk_last_l;
+	      search_state.last_r = bf->stk_last_r;
+	      bf->df = bf->df->next_same_super_edge[0];
+	      search_state.super = bf->stk_super;
+	      search_state.c = bf->stk_c;
+#ifdef RX_DEBUG_0
+	      search_state.line_no = bf->stk_search_state.line_no;
+#endif
+	      
+	      if (rxb->match_regs_on_stack)
+		{
+		  int x;
+		  regoff_t * stk =
+		    (regoff_t *)((char *)bf + sizeof (*bf));
+		  for (x = 0; x <= search_state.last_l; ++x)
+		    search_state.lparen[x] = stk[x];
+		  stk += x;
+		  for (x = 0; x <= search_state.last_r; ++x)
+		    search_state.rparen[x] = stk[x];
+		}
+	      
+	      {
+		int x;
+	      try_burst_2:
+		x = get_burst (&bf->stk_test_pos, app_closure, stop);
+		switch (x)
+		  {
+		  case rx_get_burst_continuation:
+		    search_state.saved_bf = bf;
+		    test_pc = rx_test_backtrack_return;
+		    goto test_return_continuation;
+		  resume_continuation_3:
+		    bf = search_state.saved_bf;
+		    goto try_burst_2;
+		  case rx_get_burst_no_more:
+		    /* Since we've been here before, it is some kind of
+		     * error that we can't return.
+		     */
+		  case rx_get_burst_error:
+		    search_state.test_ret = rx_test_internal_error;
+		    goto test_do_return;
+		  case rx_get_burst_ok:
+		    break;
+		  }
+	      }
+	      search_state.test_pos = bf->stk_test_pos;
+	      goto while_non_det_options;
+	    }
+	    
+	    
+	  case rx_cache_miss:
+	    /* Because the superstate NFA is lazily constructed,
+	     * and in fact may erode from underneath us, we sometimes
+	     * have to construct the next instruction from the hard way.
+	     * This invokes one step in the lazy-conversion.
+	     */
+	    search_state.ifr = rx_handle_cache_miss (&rxb->rx,
+						     search_state.super,
+						     search_state.c,
+						     search_state.ifr->data_2);
+	    if (!search_state.ifr)
+	      {
+		search_state.test_ret = rx_test_internal_error;
+		goto test_do_return;
+	      }
+	    goto restart;
+	    
+	  case rx_backtrack:
+	    /* RX_BACKTRACK means that we've reached the empty
+	     * superstate, indicating that match can't succeed
+	     * from this point.
+	     */
+	    goto test_do_return;
+	    
+	  case rx_next_char:
+	  case rx_error_inx:
+	  case rx_num_instructions:
+	    search_state.ret_val = 0;
+	    goto test_do_return;
+	  }
+	goto pseudo_while_1;
+      }
+    
+    /* Healthy exits from the test-match loop do a 
+     * `goto return_from_test_match'   On the other hand, 
+     * we might end up here.
+     */
+  error_in_testing_match:
+    test_state = rx_test_error;
+    goto test_returns_to_search;
+    
+    /***** fastmap/search loop body
+     *	      considering the results testing for a match
+     */
+    
+  return_from_test_match:
+    
+    if (search_state.best_last_l >= 0)
+      {
+	if (regs && (regs->start != search_state.best_lparen))
+	  {
+	    bcopy (search_state.best_lparen, regs->start,
+		   regs->num_regs * sizeof (int));
+	    bcopy (search_state.best_rparen, regs->end,
+		   regs->num_regs * sizeof (int));
+	  }
+	if (regs && !rxb->no_sub)
+	  {
+	    int q;
+	    int bound = (regs->num_regs > search_state.num_regs
+			 ? regs->num_regs
+			 : search_state.num_regs);
+	    regoff_t * s = regs->start;
+	    regoff_t * e = regs->end;
+	    for (q = search_state.best_last_l + 1;  q < bound; ++q)
+	      s[q] = e[q] = -1;
+	  }
+	search_state.ret_val = search_state.best_lparen[0];
+	test_state = rx_test_ok;
+	goto test_returns_to_search;
+      }
+    else
+      {
+	test_state = rx_test_fail;
+	goto test_returns_to_search;
+      }
+    
+  test_return_continuation:
+    search_state.test_match_resume_pt = test_pc;
+    test_state = rx_test_continuation;
+    goto test_returns_to_search;
+  }
+}
+
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+#else /* RX_WANT_SE_DEFS */
+  /* Integers are used to represent side effects.
+   *
+   * Simple side effects are given negative integer names by these enums.
+   * 
+   * Non-negative names are reserved for complex effects.
+   *
+   * Complex effects are those that take arguments.  For example, 
+   * a register assignment associated with a group is complex because
+   * it requires an argument to tell which group is being matched.
+   * 
+   * The integer name of a complex effect is an index into rxb->se_params.
+   */
+ 
+  RX_DEF_SE(1, re_se_try, = -1)		/* Epsilon from start state */
+
+  RX_DEF_SE(0, re_se_pushback, = re_se_try - 1)
+  RX_DEF_SE(0, re_se_push0, = re_se_pushback -1)
+  RX_DEF_SE(0, re_se_pushpos, = re_se_push0 - 1)
+  RX_DEF_SE(0, re_se_chkpos, = re_se_pushpos -1)
+  RX_DEF_SE(0, re_se_poppos, = re_se_chkpos - 1)
+
+  RX_DEF_SE(1, re_se_at_dot, = re_se_poppos - 1)	/* Emacs only */
+  RX_DEF_SE(0, re_se_syntax, = re_se_at_dot - 1) /* Emacs only */
+  RX_DEF_SE(0, re_se_not_syntax, = re_se_syntax - 1) /* Emacs only */
+
+  RX_DEF_SE(1, re_se_begbuf, = re_se_not_syntax - 1) /* match beginning of buffer */
+  RX_DEF_SE(1, re_se_hat, = re_se_begbuf - 1) /* match beginning of line */
+
+  RX_DEF_SE(1, re_se_wordbeg, = re_se_hat - 1) 
+  RX_DEF_SE(1, re_se_wordbound, = re_se_wordbeg - 1)
+  RX_DEF_SE(1, re_se_notwordbound, = re_se_wordbound - 1)
+
+  RX_DEF_SE(1, re_se_wordend, = re_se_notwordbound - 1)
+  RX_DEF_SE(1, re_se_endbuf, = re_se_wordend - 1)
+
+  /* This fails except at the end of a line. 
+   * It deserves to go here since it is typicly one of the last steps 
+   * in a match.
+   */
+  RX_DEF_SE(1, re_se_dollar, = re_se_endbuf - 1)
+
+  /* Simple effects: */
+  RX_DEF_SE(1, re_se_fail, = re_se_dollar - 1)
+
+  /* Complex effects.  These are used in the 'se' field of 
+   * a struct re_se_params.  Indexes into the se array
+   * are stored as instructions on nfa edges.
+   */
+  RX_DEF_CPLX_SE(1, re_se_win, = 0)
+  RX_DEF_CPLX_SE(1, re_se_lparen, = re_se_win + 1)
+  RX_DEF_CPLX_SE(1, re_se_rparen, = re_se_lparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_backref, = re_se_rparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_iter, = re_se_backref + 1) 
+  RX_DEF_CPLX_SE(0, re_se_end_iter, = re_se_iter + 1)
+  RX_DEF_CPLX_SE(0, re_se_tv, = re_se_end_iter + 1)
+
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.c	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      last--;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') last--;
+      if (last>=0 && wordstart[last]=='r') last--;
+      if (last>=0 && wordstart[last]=='e') last--;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	last--;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) last--;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:37  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/24 06:37:53  rjmcnab
+
+   Incorporated the french stemmer better.
+
+
+ */
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.c	(revision 21112)
@@ -0,0 +1,394 @@
+/**************************************************************************
+ *
+ * sptree.c -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include	"sysfuncs.h"
+#include	"sptree.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:17  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:50  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define TRACE(w)
+
+#define POOL_SIZE 1024
+
+
+typedef struct node_rec node;
+
+struct node_rec
+  {
+    node *left, *rght;
+    node *par;
+    char *datsun;
+  };
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    node nodes[POOL_SIZE];
+  };
+
+static node *SP_splay ();
+
+#define	CNULL	((char *) NULL)
+#define	NNULL	((node *) NULL)
+#define	SNULL	((Splay_Tree *) NULL)
+
+
+/*
+ *    createset():
+ *              allocates memory for and initialises a set.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+Splay_Tree *
+SP_createset (int (*cmp) (void *, void *))
+{
+  Splay_Tree *S;
+
+  if (!(S = (Splay_Tree *) malloc (sizeof (Splay_Tree))))
+    return (SNULL);
+  S->cmp = cmp;
+  S->root = CNULL;
+  S->no_of_items = 0;
+  S->mem_in_use = sizeof (Splay_Tree);
+  S->pool = NULL;
+  S->pool_chunks = 1024;
+  return S;
+}				/* createset() */
+
+
+
+static node *
+SP_get_node (Splay_Tree * S)
+{
+  struct pool *p = S->pool;
+  if (!p || p->pos == p->length)
+    {
+      p = malloc (sizeof (struct pool));
+      if (!p)
+	return NULL;
+      S->mem_in_use += sizeof (struct pool);
+      p->length = POOL_SIZE;
+      p->pos = 0;
+      p->next = S->pool;
+      S->pool = p;
+    }
+  return &(p->nodes[p->pos++]);
+}
+
+
+
+/*
+ *    SP_freeset():
+ *              free the memory allocated to a splay tree. Note this Does not
+ *              free the data in the tree you must do this yourself.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+void 
+SP_freeset (Splay_Tree * S)
+{
+  while (S->pool)
+    {
+      struct pool *p = S->pool;
+      S->pool = p->next;
+      free (p);
+    }
+  free (S);
+}
+
+
+/*
+ *    insert():
+ *              inserts the item given into the set given in the order
+ *              specified by the comparison function in the set 
+ *              definition.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+
+void *
+SP_insert (void *item, Splay_Tree * S)
+{
+  int dir = 0;
+  register node *curr, *prev;
+  node *tmp;
+
+  prev = NNULL;
+  curr = (node *) S->root;
+  for (; curr;)
+    {
+      prev = curr;
+      dir = S->cmp (item, curr->datsun);
+      if (dir < 0)
+	curr = curr->left;
+      else
+	curr = curr->rght;
+    }
+  if (!(tmp = SP_get_node (S)))
+    return NULL;
+  tmp->datsun = item;
+
+  if (prev == NNULL)
+    S->root = (char *) tmp;
+  else if (dir < 0)
+    prev->left = tmp;
+  else
+    prev->rght = tmp;
+
+  tmp->par = prev;
+  tmp->left = NNULL;
+  tmp->rght = NNULL;
+  S->no_of_items++;
+  S->root = (char *) SP_splay (tmp);
+  return (tmp->datsun);
+}				/* insert() */
+
+/*
+ *    get_first():
+ *              returns a pointer to the first item in the set. If the set 
+ *              is empty a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_first (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  for (curr = (node *) S->root, prev = NNULL; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_first() */
+
+/*
+ *    get_next():
+ *              returns the in-order successor of the currently fingered
+ *              item of the set. If no item is fingered or if there is 
+ *              no in-order successor a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_next (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  curr = (node *) S->root;
+  if (!curr || !curr->rght)
+    return (CNULL);
+
+  for (prev = curr, curr = curr->rght; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_next() */
+
+/*
+ *    deletemin():
+ *              removes the first element in the set and returns a
+ *              pointer to it. If the set is empty a NULL pointer 
+ *              is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_deletemin (Splay_Tree * S)
+{
+  node *tmp;
+  char *dat;
+
+  dat = SP_get_first (S);
+  if (dat)
+    {
+      tmp = (node *) S->root;
+      S->root = (char *) tmp->rght;
+      if (S->root)
+	((node *) S->root)->par = NULL;
+    }
+  S->no_of_items--;
+  return dat;
+}				/* deletemin() */
+
+
+/*
+ *    member():
+ *              searches the given set for an item which matches (has
+ *              the same key) and returns a pointer to a copy of that item.
+ *              If no matching item is found a NULL pointer is 
+ *              returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_member (void *dat, Splay_Tree * S)
+{
+  register node *curr;
+  register int dir;
+  static int calls = 0;
+
+  TRACE ("in member");
+  for (curr = (node *) S->root; curr;)
+    {
+      TRACE ("call to cmp");
+      dir = S->cmp (dat, curr->datsun);
+      TRACE ("out of cmp");
+      if (dir < 0)
+	curr = curr->left;
+      else if (dir > 0)
+	curr = curr->rght;
+      else
+	{
+	  /* splay occasionaly to speed up execution */
+	  if ((calls++ % 4) == 0 && curr)
+	    S->root = (char *) SP_splay (curr);
+	  TRACE ("out of member");
+	  return (curr->datsun);
+	}
+    }
+  TRACE ("out of member");
+  return (CNULL);
+}				/* member() */
+
+#define	ROTATEL(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->rght = p->left;			\
+		if(q->rght)				\
+			q->rght->par = q;		\
+		p->left = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+#define	ROTATER(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->left = p->rght;			\
+		if(q->left)				\
+			q->left->par = q;		\
+		p->rght = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+
+/*
+ *    splay():
+ *              i
+ *
+ *      Accesses outside variables:     i
+ *
+ *      Return value...:                i
+ */
+
+static node *
+SP_splay (p)
+     register node *p;
+{
+  register node *q, *g;
+
+  q = p->par;
+  /* splay the tree about p */
+  while (q != NNULL)
+    {
+      g = q->par;
+      if (p == q->left)
+	{
+	  if (g == NNULL)
+	    ROTATER (p, q);
+	  else if (q == g->left)
+	    {
+	      ROTATER (q, g);
+	      ROTATER (p, q);
+	    }
+	  else
+	    {
+	      ROTATER (p, q);
+	      ROTATEL (p, g);
+	    }
+	}
+      else
+	{
+	  if (g == NNULL)
+	    ROTATEL (p, q);
+	  else if (q == g->rght)
+	    {
+	      ROTATEL (q, g);
+	      ROTATEL (p, q);
+	    }
+	  else
+	    {
+	      ROTATEL (p, q);
+	      ROTATER (p, g);
+	    }
+	}
+      q = p->par;
+    }
+  return (p);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/sptree.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * sptree.h -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef SPTREE_H
+#define SPTREE_H
+
+typedef struct set_rec Splay_Tree;
+
+struct set_rec
+  {
+    int no_of_items;
+    unsigned long mem_in_use;
+    int (*cmp) (void *, void *);
+    void *root;
+    void *pool;
+    int pool_chunks;
+  };
+
+Splay_Tree *SP_createset (int (*cmp) (void *, void *));
+void *SP_insert (void *item, Splay_Tree * S);
+void *SP_get_first (Splay_Tree * S);
+void *SP_get_next (Splay_Tree * S);
+void *SP_deletemin (Splay_Tree * S);
+void *SP_member (void *dat, Splay_Tree * S);
+void SP_freeset (Splay_Tree * S);
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/strcasecmp.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/strcasecmp.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/strcasecmp.c	(revision 21112)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(s1, s2)
+    char *s1, *s2;
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/strstr.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/strstr.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/strstr.c	(revision 21112)
@@ -0,0 +1,44 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (s1, s2)
+     char *s1;
+     char *s2;
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, i++, s++)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; p1++, p2++)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.c	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * timing.c -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:17  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:05  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:53  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "timing.h"
+
+
+/*
+ * Return the time in seconds since 00:00:00 GMT, Jan. 1, 1970 to the
+ * best precision possible
+ *
+ */
+double 
+RealTime (void)
+{
+  return ((double) time (NULL));
+}
+
+
+
+/*
+ * Return the amount of system and user CPU used by the process to date to 
+ * the best precision possible. If user is non-null then it is initialised to 
+ * the user time. If sys is non-null then it is initialised to the system time.
+ *
+ */
+double 
+CPUTime (double *user, double *sys)
+{
+
+#if defined(HAVE_TIMES)
+
+  struct tms buffer;
+  static double clk_tck = 0;
+  double u, s;
+
+  times (&buffer);
+
+  if (clk_tck == 0)
+    clk_tck = CLK_TCK;
+
+  u = (double) buffer.tms_utime / clk_tck;
+  s = (double) buffer.tms_stime / clk_tck;
+  if (user)
+    *user = u;
+  if (sys)
+    *sys = s;
+  return u + s;
+
+#elif defined(HAVE_GETRUSAGE)
+
+  struct rusage ruse;
+  getrusage (RUSAGE_SELF, &ruse);
+  if (user)
+    *user = (double) ruse.ru_utime.tv_sec +
+      (double) ruse.ru_utime.tv_usec / 1000000;
+  if (sys)
+    *sys = (double) ruse.ru_stime.tv_sec +
+      (double) ruse.ru_stime.tv_usec / 1000000;
+  return ((double) ruse.ru_utime.tv_sec + ruse.ru_stime.tv_sec +
+	((double) ruse.ru_utime.tv_usec + ruse.ru_stime.tv_usec) / 1000000);
+
+#else
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (user) *user = 0.0;
+  if (sys) *sys = 0.0;
+  return 0.0;
+#else
+  -->Can NOT find a system time routine to use ! !!
+#endif  /* __WIN32__ */
+
+#endif
+
+}
+
+
+
+
+/*
+ * Get the Real and CPU time and store them in a ProgTime structure
+ */
+void 
+GetTime (ProgTime * StartTime)
+{
+  StartTime->RealTime = RealTime ();
+  StartTime->CPUTime = CPUTime (NULL, NULL);
+}
+
+
+
+
+/*
+ * Display the Real and CPU time elapsed since the StartTime anf FinishTime 
+ * structures were initialised. If FinishTime is NULL then FinishTime is
+ * Now.
+ */
+char *
+ElapsedTime (ProgTime * StartTime,
+	     ProgTime * FinishTime)
+{
+  static char buf[50];
+  double Real, CPU, hour, min, sec;
+  if (!FinishTime)
+    {
+      Real = RealTime () - StartTime->RealTime;
+      CPU = CPUTime (NULL, NULL) - StartTime->CPUTime;
+    }
+  else
+    {
+      Real = FinishTime->RealTime - StartTime->RealTime;
+      CPU = FinishTime->CPUTime - StartTime->CPUTime;
+    }
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f cpu, %02d:%02d:%02d elapsed.",
+	   hour, min, sec,
+	   ((int) ceil (Real)) / 3600,
+	   (((int) ceil (Real)) % 3600) / 60, ((int) ceil (Real)) % 60);
+  return (buf);
+}
+
+
+#define MILLION 1000000
+
+/* ===============================================================================
+ * Function: cputime_string
+ * Description:
+ *      Prints out given cpu time into a string buffer.
+ * Input: cpu-time in clock_t or timeval format
+ * Output: pointer to internal buffer 
+ * =============================================================================== */
+
+#ifdef HAVE_TIMES
+char *
+cputime_string (clock_t clk)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) clk / (double) CLK_TCK;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#else
+char *
+cputime_string (struct timeval *t)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) t->tv_sec + (double) t->tv_usec / MILLION;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#endif
+
+
+
+/* ===============================================================================
+ * Function: time_normalise
+ * Description:
+ *      If the micro-second component is negative or over a million,
+ *      then must take the over/under flow and add it to the second component.
+ * Usage:
+ *      Use this routine when you have just been doing straight addition and
+ *      subtraction on the micro-second component. For this could lead to a
+ *      negative number or over a million.
+ *
+ * =============================================================================== */
+
+#ifndef HAVE_TIMES
+
+#ifndef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+static 
+#endif
+void
+time_normalise (struct timeval *t)
+{
+  while (t->tv_usec < 0)
+    {
+      t->tv_usec += MILLION;
+      t->tv_sec--;
+    }
+  while (t->tv_usec > MILLION)
+    {
+      t->tv_usec -= MILLION;
+      t->tv_sec++;
+    }
+}
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/timing.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * timing.h -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TIMING
+#define H_TIMING
+
+
+#ifdef HAVE_TIMES
+# if TIME_WITH_SYS_TIME
+#  include <sys/times.h>
+#  include <time.h>
+#  ifndef CLK_TCK
+#   define CLK_TCK sysconf(_SC_CLK_TCK)
+#  endif
+# else
+#  if HAVE_SYS_TIME_H
+#   include <sys/time.h>
+#  else
+#   include <time.h>
+#  endif
+# endif
+#else
+/* [RPAP - Feb 97: WIN32 Port] */
+# ifdef __WIN32__
+   struct timeval { long tv_sec, tv_usec; };
+# else
+#  include <sys/time.h>
+#  include <sys/resource.h>
+# endif
+#endif
+
+
+typedef struct
+  {
+    double RealTime, CPUTime;
+  }
+ProgTime;
+
+double RealTime (void);
+
+double CPUTime (double *user, double *sys);
+
+void GetTime (ProgTime * StartTime);
+
+/* FinishTime may be NULL */
+char *ElapsedTime (ProgTime * StartTime, ProgTime * FinishTime);
+
+#ifdef HAVE_TIMES
+char *cputime_string (clock_t clk);
+#else
+void time_normalise (struct timeval *t);
+char *cputime_string (struct timeval *t);
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.c	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; i--) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; i++) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      curpos++;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data;
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) letter_here++;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) digit_here++;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) space_here++;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	i++;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      outhere++;
+      outlen++;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    in_i++;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.c	(revision 21112)
@@ -0,0 +1,86 @@
+#include "longlong.h"
+
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/win32in.h	(revision 21112)
@@ -0,0 +1,10 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+#include "longlong.h"
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
Index: /main/tags/3.01/gsdl3/src/packages/mg/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif;
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_passes.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_weights_build.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbilevel.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild_4.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgdictlist.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgfelics.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgmerge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgquery.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgquery.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstat.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgtic.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgtic.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticbuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticdump.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticprune.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticstat.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/man/man1/xmg.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/man/man1/xmg.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/man/man1/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/mkinstalldirs
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/Makefile.in	(revision 21112)
@@ -0,0 +1,217 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/images
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+GSDLOS = @gsdlos@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics mgbilevel mgtic mgticbuild mgticprune mgticstat \
+                mgticdump
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/WIN32.MAK
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/WIN32.MAK	(revision 21112)
@@ -0,0 +1,187 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+MGHOME = ..\..
+
+CC = cl
+CPPFLAGS = 
+DEFS = -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I"." -I"$(MGHOME)" -I"$(MGHOME)\lib"
+LDFLAGS =
+LIBS = $(MGHOME)\lib\libmg.lib 
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+ 
+o = .obj
+e = .exe
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics$e mgbilevel$e mgtic$e mgticbuild$e mgticprune$e mgticstat$e \
+                mgticdump$e
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics$e: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel$e: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic$e: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild$e: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune$e: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat$e: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump$e: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#ansi2knr: ansi2knr.o
+#	$(LINK) ansi2knr.o $(LIBS)
+#
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+#execinstall: $(EXEC)
+#	$(top_srcdir)/mkinstalldirs $(bindir); \
+#	for name in $(EXEC); do \
+#	  $(INSTALL_PROGRAM) $$name \
+#	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+#execuninstall:
+#	for name in $(EXEC); do \
+#	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	if exist *$o del *$o
+	if exist  _*.c del  _*.c
+	if exist _*.o  del  _*.o 
+	if exist *._c  del *._c
+	if exist *._o  del *._o
+	if exist core  del  core
+	if exist core.* del  core.*
+ 
+clean: mostlyclean
+	if exist *$e del *$e
+ 
+distclean: clean
+	del ansi2knr
+	 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#	for file in $(DISTFILES); do \
+#	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#	  || cp -p $(srcdir)/$$file $(distdir); \
+#	done
+ 
+#Makefile: Makefile.in ../../config.status
+#	cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.c	(revision 21112)
@@ -0,0 +1,471 @@
+/**************************************************************************
+ *
+ * arithcode.c -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *	Arithmetic encoding. Taken largely from Witten, Neal, Cleary,
+ *		CACM, 1987, p520.
+ *	Includes bitfiddling routines.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "arithcode.h"
+
+
+FILE *arith_in, *arith_out;
+
+/*
+   $log$
+ */
+
+static char *RCSID = "$Id$";
+
+#undef BECAREFUL
+#define INTEGRATED
+
+
+codevalue S_low = 0, S_high = 0, S_value = 0;
+long S_bitstofollow = 0;
+int S_buffer = 0, S_bitstogo = 0;
+
+long cmpbytes = 0, rawbytes = 0;
+
+#ifdef INTEGRATED
+long CountOfBitsOut;
+#endif
+
+
+/*==================================*/
+
+#define BITPLUSFOLLOW(b)		\
+{	OUTPUTBIT((b));			\
+	while (bitstofollow)		\
+	{	OUTPUTBIT(!(b));	\
+		bitstofollow -= 1;	\
+	}				\
+}
+
+#ifdef INTEGRATED
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+	CountOfBitsOut++;		\
+}
+#else
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+}
+#endif
+
+#define ADDNEXTINPUTBIT(v)		\
+{	if (bitstogo==0)		\
+	{	buffer = getc(arith_in);	\
+		bitstogo = 8;		\
+	}				\
+	v = (v<<1)+(buffer&1);		\
+	buffer >>= 1;			\
+	bitstogo -=1;			\
+}
+
+/*==================================*/
+
+void
+arithmetic_encode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue H;
+    register long bitstofollow;
+    register int buffer, bitstogo;
+
+    bitstofollow = S_bitstofollow;
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+
+    for (;;)
+      {
+	if (high < H)
+	  {
+	    BITPLUSFOLLOW (0);
+	  }
+	else if (low >= H)
+	  {
+	    BITPLUSFOLLOW (1);
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    bitstofollow++;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high++;
+      }
+    S_bitstofollow = bitstofollow;
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+  }
+}
+
+/*==================================*/
+
+/* Made into a macro 
+
+   codevalue
+   arithmetic_decode_target(totl)
+   register int totl;
+   {    return (((S_value-S_low+1)*totl-1) / (S_high -S_low+1));
+   } 
+ */
+
+/*==================================*/
+
+void
+arithmetic_decode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue value, H;
+    register int buffer, bitstogo;
+
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+    value = S_value;
+
+    for (;;)
+      {
+	if (high < H)
+	  {			/* nothing */
+	  }
+	else if (low >= H)
+	  {
+	    value -= H;
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    value -= firstqtr;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high += 1;
+	ADDNEXTINPUTBIT (value);
+      }
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+    S_value = value;
+  }
+}
+
+/*==================================*/
+
+
+static void
+startoutputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 8;
+}
+
+static void
+startinputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 0;
+}
+
+static void
+doneoutputingbits ()
+{				/* write another byte if necessary */
+  if (S_bitstogo < 8)
+    {
+      putc (S_buffer >> S_bitstogo, arith_out);
+      cmpbytes += 1;
+    }
+}
+
+/*==================================*/
+
+static void
+startencoding ()
+{
+  S_low = 0;
+  S_high = topvalue;
+  S_bitstofollow = 0;
+}
+
+static void
+startdecoding ()
+{
+  register int i;
+  register int buffer, bitstogo;
+  S_low = 0;
+  S_high = topvalue;
+  S_value = 0;
+  bitstogo = S_bitstogo;
+  buffer = S_buffer;
+  for (i = 0; i < codevaluebits; i++)
+    ADDNEXTINPUTBIT (S_value);
+  S_bitstogo = bitstogo;
+  S_buffer = buffer;
+}
+
+static void
+doneencoding ()
+{
+  register long bitstofollow;
+  register int buffer, bitstogo;
+  bitstofollow = S_bitstofollow;
+  buffer = S_buffer;
+  bitstogo = S_bitstogo;
+  bitstofollow += 1;
+  if (S_low < firstqtr)
+    {
+      BITPLUSFOLLOW (0);
+    }
+  else
+    {
+      BITPLUSFOLLOW (1);
+    }
+  S_bitstofollow = bitstofollow;
+  S_buffer = buffer;
+  S_bitstogo = bitstogo;
+}
+
+
+
+void 
+EncodeGammaDist (int Off)
+/* Encode Off with probability distribution derived from the Elias gamma distribution. */
+{
+  int Len, i;
+
+  Off++;
+  for (Len = 31; Len >= 0 && !(Off & (1 << Len)); Len--);
+
+  for (i = Len; i >= 0; i--)
+    arithmetic_encode (0, 1, 2);
+  arithmetic_encode (1, 2, 2);
+
+  for (i = Len - 1; i >= 0; i--)
+    if (Off & (1 << i))
+      arithmetic_encode (1, 2, 2);
+    else
+      arithmetic_encode (0, 1, 2);
+}
+
+
+
+int 
+DecodeGammaDist ()
+/* Decode next Off using probability distribution derived from the Elias gamma distribution. Inverse of EncodeGammaDist. */
+{
+  int i = 1, Len = 0, bit;
+
+  do
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	{
+	  arithmetic_decode (0, 1, 2);
+	  Len++;
+	}
+    }
+  while (!bit);
+
+  Len--;
+  while (Len-- > 0)
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	arithmetic_decode (0, 1, 2);
+      i = (i << 1) | bit;
+    }
+  return (i - 1);
+}
+
+
+
+
+
+void 
+EncodeGammaSigned (int snum, int *pos, int *neg)
+{
+  if (snum >= 0)
+    arithmetic_encode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_encode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++;
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+  EncodeGammaDist (abs (snum));
+}
+
+
+int 
+DecodeGammaSigned (int *pos, int *neg)
+{
+  int signedpos = 1;
+  if (arithmetic_decode_target (*pos + (*neg)) < *pos)
+    arithmetic_decode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_decode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++, signedpos = 0;
+
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+
+  if (signedpos)
+    return DecodeGammaDist ();
+  else
+    return (-DecodeGammaDist ());
+}
+
+
+void 
+InitArithEncoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startoutputingbits ();
+  startencoding ();
+}
+
+void 
+InitArithDecoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startinputingbits ();
+  startdecoding ();
+}
+
+void 
+CloseDownArithEncoding (void)
+{
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void 
+CloseDownArithDecoding (void)
+{
+}
+
+
+
+
+
+/* encode that famous number, 42 uses 12 bits */
+void 
+EncodeChecksum ()
+{
+  EncodeGammaDist (42);
+}
+
+void 
+DecodeChecksum (char str[])
+{
+  if (DecodeGammaDist () != 42)
+    {
+      fprintf (stderr, "%s: checksum error, file corrupt.\n", str);
+      exit (1);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/arithcode.h	(revision 21112)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * arithcode.h -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef _arithcode_h
+#define _arithcode_h
+
+extern FILE *arith_in, *arith_out;
+
+#define codevaluebits 16
+#if (codevaluebits<16)
+typedef unsigned short codevalue;
+#else
+typedef unsigned long codevalue;
+#endif
+
+#define topvalue ((codevalue)((1<<codevaluebits)-1))
+#define maxfrequency ((topvalue+1)/4 + 1)
+
+#define firstqtr (topvalue/4+1)
+#define half     (2*firstqtr)
+#define thirdqtr (3*firstqtr)
+
+#define	escape_event	U->totalcnt-U->notfound, U->totalcnt, U->totalcnt
+
+void arithmetic_encode ();
+codevalue arithmetic_decode_target ();
+
+#define	arithmetic_decode_target(totl)	\
+	(  ((S_value-S_low+1)*(totl)-1) / (S_high-S_low+1)  )
+
+void arithmetic_decode ();
+
+
+extern codevalue S_low, S_high, S_value;
+extern long S_bitstofollow;
+extern int S_buffer, S_bitstogo;
+
+extern long cmpbytes, rawbytes;
+
+extern long CountOfBitsOut;
+
+void EncodeGammaDist (int Off);
+int DecodeGammaDist ();
+
+void EncodeGammaSigned (int snum, int *pos, int *neg);
+int DecodeGammaSigned (int *pos, int *neg);
+
+
+void InitArithEncoding (void);
+void InitArithDecoding (void);
+
+void CloseDownArithEncoding (void);
+void CloseDownArithDecoding (void);
+
+void EncodeChecksum ();
+void DecodeChecksum (char str[]);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/basictypes.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/basictypes.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/basictypes.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * basictypes.h -- base data types
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file contains information relating to the base data types
+ *
+ **************************************************************************/
+
+#ifndef __basictypes_h
+#define __basictypes_h
+
+typedef unsigned long Pixel;
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.c	(revision 21112)
@@ -0,0 +1,747 @@
+/**************************************************************************
+ *
+ * bilevel.c -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  
+ *  
+ *  This is the code used to compress a bitmap.  The template is given in
+ *  the following format. 
+ *   '.'  = pixel not used
+ *   'p'  = pixel used in 1st order context (the superset)
+ *   '2'  = pixel used in 2nd order context (the subset)
+ *   ';'  = marker used to denote the end of a row in the template
+ *   '*'  = the current context pixel.
+ *  for instance, the common 22/10 template would be shown as:
+ *     ".ppppp.;"
+ *     "pp222pp;"
+ *     "p22222p;"
+ *     "p22*...;"
+ * 
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "bilevel.h"
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+
+
+#define INITVALUE 2
+#define INCREMENT 4
+
+#define MAX_T_W 32
+
+/* note memory usage is 2*TEMPLATESIZE*sizeof(prob), which is currently 4.9 Mb */
+#define TEMPLATESIZE 612317
+
+
+#define HASH(x) ((x)%TEMPLATESIZE)
+
+
+typedef struct
+  {
+    unsigned short s, n;
+  }
+prob;				/* set and notset */
+typedef struct
+  {
+    short int x, y;
+  }
+coordinate;
+
+coordinate MM[MAX_T_W][MAX_T_W];	/* the actual (x,y) positions */
+unsigned int MMcount[MAX_T_W];	/* number of bits in this row of the template */
+unsigned int MMmask[MAX_T_W];	/* the "current" mask value */
+unsigned int MMbits[MAX_T_W];	/* the bitmask to mask out higher bits */
+unsigned int MMrows;		/* total number of rows */
+unsigned int MM2level[MAX_T_W];	/* bitmask to calculate the 2nd level mask */
+int twolevel, curr_row, surr;
+
+prob *P = NULL;			/* probability array, up to N bits */
+prob *P2 = NULL;
+
+void 
+bl_clearmodel ()
+{
+  int i;
+
+  P = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  P2 = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P2)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  for (i = 0; i < TEMPLATESIZE; i++)
+    {
+      P[i].s = P[i].n = INITVALUE;
+      P2[i].s = P2[i].n = INITVALUE;
+    }
+}
+
+
+void 
+bl_freemodel ()
+{
+  free (P);
+  free (P2);
+}
+
+
+
+
+#define update_counts(p, pixel)                        \
+    pixel ? (p->s+=INCREMENT) : (p->n+=INCREMENT);     \
+                                                       \
+    if(p->s+p->n >= (unsigned)(maxfrequency)) {        \
+       p->s=(p->s+1)/(unsigned)2;                      \
+       p->n=(p->n+1)/(unsigned)2;                      \
+    }
+
+
+
+
+/* mask is the bigger one, mask2l is smaller one */
+static void 
+bl_encode_in_context (unsigned long mask, unsigned long mask2l, unsigned long pixel)
+{
+  register int twol, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+  twol = twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT));
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_encode (0, p->s, t);
+      else
+	arithmetic_encode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_encode (0, p2->s, t);
+      else
+	arithmetic_encode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+}
+
+
+static void 
+bl_decode_in_context (unsigned long mask, unsigned long mask2l, unsigned long *val)
+{
+  register int twol, pixel, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+
+  if (twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT)))
+    {
+      pixel = arithmetic_decode_target (p2->s + p2->n) < p2->s, twol = 1;
+    }
+  else
+    {
+      pixel = arithmetic_decode_target (p->s + p->n) < p->s, twol = 0;
+    }
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_decode (0, p->s, t);
+      else
+	arithmetic_decode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_decode (0, p2->s, t);
+      else
+	arithmetic_decode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+
+  *val = pixel;
+}
+
+static void 
+bl_string_to_array (char inputstr[], char parsestr[], int *w, int *h)
+{
+  int r, c;
+  int i, j, lastc, p, l;
+  int xc, yc;
+  int foundstatus;
+
+  surr = twolevel = 0;
+  for (i = 0; i < MAX_T_W; i++)
+    MMcount[i] = MM2level[i] = 0;
+  MMrows = 0;
+
+  for (i = 0; i < (int) strlen (inputstr); i++)
+    {
+      p = inputstr[i];
+      if ((p != '.') && (p != '2') && (p != '*') && (p != 'p') && (p != ';'))
+	{
+	  fprintf (stderr, "Template string can only contain [.p2*;], not \"%c\".\n", inputstr[i]);
+	  exit (1);
+	}
+    }
+
+  /* get the width and height, test rectangularity... */
+  parsestr[l = 0] = 0;
+  for (c = 0, r = 0, lastc = -1, p = 0; p < (int) strlen (inputstr); p++)
+    {
+      if ((inputstr[p] == ';'))
+	{
+	  r++;
+	  if (lastc != -1)
+	    {			/* this is the 2nd row at least */
+	      if (lastc != c)
+		{
+		  fprintf (stderr, "The rows have to be the same width. "
+			   "Row %d has width %d (previous row has width %d)\n", r, c, lastc);
+		  exit (1);
+		}
+	    }
+	  lastc = c;
+	  c = 0;
+	}
+      else
+	{
+	  c++;
+	  parsestr[l] = inputstr[p];
+	  parsestr[++l] = 0;
+	}
+    }
+
+  if (c != 0)
+    lastc = c;
+  *w = lastc;
+  *h = r;
+  if ((*w) <= 0)
+    {
+      fprintf (stderr, "Must have at least one column!\n");
+      exit (1);
+    }
+  if ((*h) <= 0)
+    {
+      fprintf (stderr, "Must have at least one row! Terminate each row with a ';'\n");
+      exit (1);
+    }
+  if ((*w > MAX_T_W) || (*h > MAX_T_W))
+    {
+      fprintf (stderr, "template size bigger than internal threshold of %d\n", MAX_T_W);
+      exit (1);
+    }
+
+  /* get the (x,y) position of the current (*) pixel */
+  for (xc = -1, yc = 0, i = 0; i < *w; i++)
+    for (j = 0; j < *h; j++)
+      if (parsestr[j * (*w) + i] == '*')
+	{
+	  if (xc >= 0)
+	    {
+	      fprintf (stderr, "There can only be ONE current ('*') pixel.\n");
+	      exit (1);
+	    }
+	  xc = i;
+	  yc = j;
+	  curr_row = j + 1;
+	}
+  if (xc < 0)
+    error_msg ("bl_string_to_array", "Error in mask, no current ('*') pixel.", "");
+  /* found the position (xc,yc) */
+
+
+  /* convert parsestr[] into a MM[][] structure */
+  for (j = 0; j < *h; j++)
+    {
+      foundstatus = 0;
+      for (i = 0; i < *w; i++)
+	{
+	  p = j * (*w) + i;
+	  if ((parsestr[p] == 'p') || (parsestr[p] == '2'))
+	    {
+	      if (!foundstatus)
+		foundstatus = 1;
+	      MM2level[j] = MM2level[j] * 2 | (parsestr[p] == '2');
+	      if (parsestr[p] == '2')
+		twolevel = 1;
+
+	      MM[j][MMcount[j]].x = i - xc;
+	      MM[j][MMcount[j]++].y = j - yc;
+	      if (((i - xc) >= 0) && ((j - yc) >= 0))
+		surr = 1;
+	    }
+	  else
+	    {
+	      if (foundstatus == 1)
+		foundstatus = 2;
+	    }
+
+	}
+      if (foundstatus)
+	MMrows++;
+      MMbits[j] = (1 << MMcount[j]) - 1;
+    }
+
+  /* reverse matrix, so [][0] is the closest to the right */
+  {
+    coordinate temp;
+    for (i = 0; i < MMrows; i++)
+      for (j = 0; j < MMcount[i] / (unsigned) 2; j++)
+	{
+	  temp = MM[i][MMcount[i] - 1 - j];
+	  MM[i][MMcount[i] - 1 - j] = MM[i][j];
+	  MM[i][j] = temp;
+	}
+  }
+}
+
+
+
+
+
+
+
+
+void 
+bl_writetemplate (char inputstr[])
+{
+  register int i, j;
+  int w, h;
+  char ind[255];
+  char *str;
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+
+  bl_string_to_array (inputstr, str, &w, &h);
+
+  EncodeGammaDist (surr);
+  EncodeGammaDist (w);
+  EncodeGammaDist (h);
+  ind['.'] = 0;
+  ind['p'] = 1;
+  ind['2'] = 2;
+  ind['*'] = 3;
+  for (j = 0; j < h; j++)
+    for (i = 0; i < w; i++)
+      EncodeGammaDist (ind[(int) str[j * w + i]]);
+
+  free (str);
+}
+
+
+void 
+bl_readtemplate ()
+{
+  register int i, j;
+  int w, h, p;
+  char ind[4];
+  char *inputstr, *str;
+
+  surr = DecodeGammaDist (surr);
+  if (surr)
+    {
+      fprintf (stderr, "It's fine compressing an image using a 'surrounding' template, but I can't "
+	       "decompress it!\n");
+      exit (1);
+    }
+  w = DecodeGammaDist ();
+  h = DecodeGammaDist ();
+
+  inputstr = (char *) malloc ((w + 2) * h * sizeof (char) + 2);
+  ind[0] = '.';
+  ind[1] = 'p';
+  ind[2] = '2';
+  ind[3] = '*';
+  for (p = 0, j = 0; j < h; j++)
+    {
+      for (i = 0; i < w; i++)
+	{
+	  inputstr[p++] = ind[DecodeGammaDist ()];
+	}
+      inputstr[p++] = ';';
+      inputstr[p] = 0;
+    }
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+  bl_string_to_array (inputstr, str, &w, &h);
+}
+
+
+
+
+
+/* Currently if there are more than 32 bits, the topmost 32 will be discarded, if the 
+   topbits are to be used in compression, define _MORETHAN32 */
+#undef _MORETHAN32
+
+void 
+bl_compress_mark (marktype d)
+{
+  register int r, c;
+  register int i, j;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d.w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  EncodeGammaDist (d.w);
+  EncodeGammaDist (d.h);
+
+  for (r = 0; r < min (curr_row, d.h); r++)
+    for (c = 0; c < d.w; c++)
+      array[r % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r);
+
+  for (r = 0; r < d.h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+
+      if (r + curr_row < d.h)
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r + curr_row);
+      else
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = 0;
+
+      for (c = 0; c < d.w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (d.bitmap, c, r));
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_decompress_mark (marktype * d)
+{
+  register int r, c;
+  register int i, j;
+  unsigned long pix;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  d->w = DecodeGammaDist ();
+  d->h = DecodeGammaDist ();
+
+  d->bitmap = pbm_allocarray (d->w, d->h);
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d->w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  for (r = 0; r < d->h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+      for (c = 0; c < d->w; c++)
+	array[r % MAX_T_W][c] = pbm_getpixel (d->bitmap, c, r);
+      for (c = 0; c < d->w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)	/* get all pixels */
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_decode_in_context (mask, mask2level, &pix);
+	  if (pix)
+	    {
+	      pbm_putpixel (d->bitmap, c, r, pix);
+	      array[r % MAX_T_W][c] = pix;
+	    }
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_compress (marktype d, char inputstr[])
+{
+  bl_clearmodel ();
+  bl_writetemplate (inputstr);
+  bl_compress_mark (d);
+  bl_freemodel ();
+}
+
+
+void 
+bl_decompress (marktype * d)
+{
+  bl_clearmodel ();
+  bl_readtemplate ();
+  bl_decompress_mark (d);
+  bl_freemodel ();
+}
+
+
+
+
+
+
+/* 
+   The bitmap in d2, can be examined clairvoyantly.  The ordering of the
+   params is therefore non-clairvoyant, and clairvoyant in that order.
+
+ */
+/*
+   coordinate M1[6]={        { 0,-2},
+   {-1,-1},{ 0,-1}, {1,-1},
+   {-2,0},  {-1, 0}};
+   };
+   coordinate M2[13]={       { 0,-2},
+   {-1,-1}, { 0,-1},   {1, -1},
+   {-2,0}, {-1, 0}, { 0, 0},   { 1, 0},  {2,0},
+   {-1, 1}, { 0, 1},   { 1, 1},
+   { 0, 2}
+ */
+
+coordinate M1[6] =
+{
+  {0, -2},
+  {-2, 0},
+  {1, -1},
+  {-1, -1},
+  {0, -1},
+  {-1, 0}};
+coordinate M2[13] =
+{
+  {0, -2},
+  {1, -1},
+  {-2, 0},
+  {2, 0},
+  {-1, 1},
+  {1, 1},
+  {0, 2},
+  {-1, -1},
+  {0, -1},
+  {1, 0},
+  {-1, 0},
+  {0, 1},
+  {0, 0}
+};
+#define M1S 6
+#define M1S_2level 3
+#define M2S 13
+#define M2S_2level 6
+
+
+void 
+bl_clair_compress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  EncodeGammaDist (cols);
+  EncodeGammaDist (rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (b1, c, r));
+
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
+
+
+void 
+bl_clair_decompress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  unsigned long p;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  i = DecodeGammaDist ();
+  j = DecodeGammaDist ();
+  if ((i != cols) || (j != rows))
+    error_msg ("bl_clair_decompress", "incorrect size of residue image", "");
+
+  if (V)
+    fprintf (stderr, "cols=%d rows=%d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+
+	  bl_decode_in_context (mask, mask2level, &p);
+	  if (p)
+	    pbm_putpixel (b1, c, r, 1);
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/bilevel.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * bilevel.h -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __BILEVEL_H
+#define __BILEVEL_H
+
+#include "marklist.h"
+
+/* must be called before the start of de/compression */
+void bl_clearmodel ();
+void bl_freemodel ();
+
+/* writes the template to the stream, and sets up the de/compression params */
+void bl_writetemplate (char inputstr[]);
+void bl_readtemplate ();
+
+/* actually performs the de/compression */
+void bl_compress (marktype d, char str[]);
+void bl_compress_mark (marktype d);
+void bl_decompress (marktype * d);
+void bl_decompress_mark (marktype * d);
+
+/* clairvoyantly de/compress d1 with respect to clairvoyantly viewable d2 */
+void bl_clair_compress (marktype d1, marktype d2);
+void bl_clair_decompress (marktype d1, marktype d2);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.c	(revision 21112)
@@ -0,0 +1,257 @@
+/**************************************************************************
+ *
+ * codeoffsets.c -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+#define TOP_INC 2
+#define MID_INC 1
+#define BOT_INC 1
+
+
+void 
+EncodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int found;
+  int dox = 0, leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;	/* all previous offsets */
+  int p, f, t, xoff, yoff, symnum, count;
+  elemlist *xcond, *ycond;	/* previous offsets conditioned on symbol num */
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (t = 0; t < nsyms; t++)
+    {
+      xcond[t].head = ycond[t].head = NULL;
+      xcond[t].tot = ycond[t].tot = 0;
+    }
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      xoff = step->data.xoffset;
+      yoff = step->data.yoffset;
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* encode the offsets, x first then y */
+
+	  if (dox)
+	    found = elem_find (&xcond[symnum], xoff, &p, &f, &t);
+	  else
+	    found = elem_find (&ycond[symnum], yoff, &p, &f, &t);
+
+	  if (found)
+	    {
+	      arithmetic_encode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+	      arithmetic_encode (p, f, t);
+	    }
+	  else
+	    {
+	      if (dox)
+		found = elem_find (&xlist, xoff, &p, &f, &t);
+	      else
+		found = elem_find (&ylist, yoff, &p, &f, &t);
+
+	      if (found)
+		{
+		  arithmetic_encode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+		  levelmid += MID_INC;
+		  arithmetic_encode (p, f, t);
+		}
+	      else
+		{
+		  arithmetic_encode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+		  levelbot += BOT_INC;
+		  if (dox)
+		    EncodeGammaSigned (xoff, &xpos, &xneg);
+		  else
+		    EncodeGammaSigned (yoff, &ypos, &yneg);
+		}
+	    }
+	  if (dox)
+	    {
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+  if (V)
+    fprintf (stderr, "offset models: top %d, mid %d, bot %d\n", (leveltop - 1) / TOP_INC, (levelmid - 1) / MID_INC, (levelbot - 1) / BOT_INC);
+}
+
+
+
+
+
+void 
+DecodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int i, target, dox;
+  int leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;
+  int p, f, t, xoff = 0, yoff = 0, symnum, count = 0;
+  elemlist *xcond, *ycond;
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (i = 0; i < nsyms; i++)
+    {
+      xcond[i].head = ycond[i].head = NULL;
+      xcond[i].tot = ycond[i].tot = 0;
+    }
+
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* decode x offset first, then y */
+	  target = arithmetic_decode_target (leveltop + levelmid + levelbot);
+	  if (target < leveltop)
+	    {
+	      arithmetic_decode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xcond[symnum].tot);
+	      else
+		i = arithmetic_decode_target (ycond[symnum].tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xcond[symnum], i, &xoff);
+		  elem_find (&xcond[symnum], xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ycond[symnum], i, &yoff);
+		  elem_find (&ycond[symnum], yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else if (target < leveltop + levelmid)
+	    {
+	      arithmetic_decode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+	      levelmid += MID_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xlist.tot);
+	      else
+		i = arithmetic_decode_target (ylist.tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xlist, i, &xoff);
+		  elem_find (&xlist, xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ylist, i, &yoff);
+		  elem_find (&ylist, yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else
+	    {
+	      arithmetic_decode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+	      levelbot += BOT_INC;
+	      if (dox)
+		xoff = DecodeGammaSigned (&xpos, &xneg);
+	      else
+		yoff = DecodeGammaSigned (&ypos, &yneg);
+	    }
+
+	  if (dox)
+	    {
+	      step->data.xoffset = xoff;
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      step->data.yoffset = yoff;
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/codeoffsets.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * codeoffsets.h -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __CODEOFFSETS_H
+#define __CODEOFFSETS_H
+
+#include "marklist.h"
+
+void EncodeOffsets (marklistptr MarkList, int num_symbols);
+void DecodeOffsets (marklistptr MarkList, int num_symbols);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.c	(revision 21112)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * codesyms.c -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Hugh Emberson
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+#include "codesyms.h"
+
+int charsetsize = 256;
+
+void 
+EncodeSymbols (marklistptr MarkList, int NSyms)
+{
+  marklistptr step;
+  unsigned short Events[PPM_CHUNK];
+  int j = 0;
+
+  charsetsize = max (NSyms, 256);
+
+  step = MarkList;
+  while (step)
+    {
+      Events[j] = step->data.symnum;
+
+      assert (Events[j] > 0 || Events[j] < (unsigned) NSyms);
+
+      j++;
+
+      if (j == PPM_CHUNK)
+	{
+	  encodestring (Events, PPM_CHUNK);
+	  j = 0;
+	}
+      step = step->next;
+    };
+
+  if (j != 0)
+    encodestring (Events, j);
+
+  encodestring (Events, 0);
+}
+
+
+marklistptr 
+DecodeSymbols (int NSyms)
+{
+  marklistptr MarkList = NULL;
+  unsigned short Events[PPM_CHUNK];
+  int j, n;
+  marktype d;
+
+  charsetsize = max (NSyms, 256);
+
+  do
+    {
+      decodestring (Events, &n);
+
+      for (j = 0; j < n; j++)
+	{
+	  d.symnum = Events[j];
+	  marklist_add (&MarkList, d);
+	}
+    }
+  while (n != 0);
+
+  return MarkList;
+}
+
+void 
+InitPPM (void)
+{
+  kbytes = 896;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/codesyms.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * codesyms.h -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef _CodeSymbols_h_
+#define _CodeSymbols_h_
+
+#include "marklist.h"
+#include "arithcode.h"
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+#define PPM_CHUNK	1024
+
+extern int kbytes;
+extern void decodestring ();
+extern void encodestring ();
+
+void EncodeSymbols (marklistptr MarkList, int NSyms);
+marklistptr DecodeSymbols (int NSyms);
+
+void InitPPM (void);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.c	(revision 21112)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * extractor.c -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "utils.h"
+
+
+/* 
+   'splitjoined' defines whether the extractor modules tries to break
+   "seemingly" merged/joined marks.
+ */
+int splitjoined = 1;
+
+#define MAX_FILL_STACK_SIZE 10000
+
+/* RunRegionFill uses line conherence to implement a better fill
+   routine.  The stack has currently got 10000 elements, although the
+   biggest I have encountered used a maximum stack size of 1294.  usage:
+   RunRegionFill(bitmap,mark,i,j,xpos,ypos); xpos and ypos are the top
+   left corners of the mark.  */
+
+int maxst = 0, st = 0;
+struct
+  {
+    int x, y;
+  }
+a[MAX_FILL_STACK_SIZE];
+
+#define push(i,j) a[st].x=i;a[st].y=j;st++;if(st>MAX_FILL_STACK_SIZE){error_msg ("extractor/flood_fill","woooo! Fill stack overflow!","");}
+#define pop(i,j) st--;i=a[st].x;j=a[st].y
+
+void 
+RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows)
+{
+  int l, r, t, start_up, start_down;
+
+  push (i, j);
+  while (st > 0)
+    {
+      if (st > maxst)
+	maxst = st;
+
+      pop (l, j);
+      r = l;
+
+      while (pbm_getpixel (bitmap, l - 1, j))
+	l--;
+      while (pbm_getpixel (bitmap, r + 1, j))
+	r++;
+      start_up = start_down = 1;
+      for (t = min (r + 1, cols - 1); t >= max (l - 1, 0); t--)
+	{
+	  if ((t >= l) && (t <= r))
+	    {
+	      pbm_putpixel (bitmap, t, j, 0);
+
+	      if (mark)
+		pbm_putpixel (mark->bitmap, t - xl, j - yt, 1);
+	    }
+
+	  if ((j - 1 >= 0) && (pbm_getpixel (bitmap, t, j - 1)))
+	    {
+	      if (start_up)
+		{
+		  push (t, j - 1);
+		  start_up = 0;
+		}
+	    }
+	  else
+	    start_up = 1;
+	  if ((j + 1 < rows) && (pbm_getpixel (bitmap, t, j + 1)))
+	    {
+	      if (start_down)
+		{
+		  push (t, j + 1);
+		  start_down = 0;
+		}
+	    }
+	  else
+	    start_down = 1;
+	}
+    }
+}
+
+
+
+
+/*
+
+   ExtractNextMark starts at (tx,ty) and looks for a set (==1) pixel.
+   The mark is then boundary traced to give an outline {The process is a
+   4-pixel exterior check -- giving 8 way connectivity}, and then
+   RunRegionFill is used to remove the mark and set the pixels in the
+   mark.
+
+   usage: ExtractNextMark(bitmap,&marklist,&startx,&starty,int cols,int rows);
+   Defines:
+   gp - gets a pixel, but returns 0 for outside regions.
+   boundary_{x,y}comp - returns the {x,y} component of the direction.
+   dir 0=right, 1=down, 2=left, 3=up.
+ */
+
+#define gp(i,j) ( (((i)<0)||((j)<0)||((i)>=cols)||((j)>=rows))? 0 : pbm_getpixel(bitmap,(i),(j)))
+
+#define boundary_xcomp(x) (x==0?1:(x==2?-1 : 0))
+#define boundary_ycomp(x) (x==1?1:(x==3?-1 : 0))
+
+int 
+ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows)
+{
+  int i, j, xl, xr, yt, yb;
+  register int dir, tempdir;
+  marktype d;
+
+  xl = INT_MAX;
+  xr = (-1);
+  yt = INT_MAX;
+  yb = (-1);
+
+  i = *tx;
+  j = *ty;
+  if ((i < 0) || (j < 0))
+    error_msg ("ExtractNextMark()", "Stupid values for x,y in findnextmark()", "");
+  while (!pbm_getpixel (bitmap, i, j))
+    {
+      i++;
+      if (i >= cols)
+	{
+	  i = 0;
+	  j++;
+	  if (j >= rows)
+	    return 0;
+	}
+    }
+
+  *tx = i;
+  *ty = j;
+  dir = 0;
+  j = j - 1;			/* start above */
+  do
+    {
+      if (i > xr)
+	xr = i - 1;
+      if (i < xl)
+	xl = i + 1;
+      if (j < yt)
+	yt = j + 1;
+      if (j > yb)
+	yb = j - 1;
+      tempdir = (dir + 1) % 4;
+      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	dir = tempdir;
+      else
+	{
+	  tempdir = dir;
+	  if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	    dir = tempdir;
+	  else
+	    {
+	      tempdir = (dir + 3) % 4;
+	      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+		dir = tempdir;
+	      else
+		dir = (dir + 2) % 4;
+	    }
+	}
+      i += boundary_xcomp (dir);
+      j += boundary_ycomp (dir);
+    }
+  while ((i != (*tx)) || (j != (*ty - 1)));
+
+  d.w = (xr - xl + 1);
+  d.h = (yb - yt + 1);
+  d.xpos = xl;
+  d.ypos = yt;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+  RunRegionFill (bitmap, &d, *tx, *ty, xl, yt, cols, rows);
+
+  marktype_calc_features (&d, "?");
+  marklist_add (marklist, d);
+
+  *tx = *tx + 1;
+  return 1;
+}
+
+
+
+
+static void 
+normalise (float *p, int w)
+{
+  int x;
+  float mx;
+
+  mx = p[0];
+  for (x = 0; x < w; x++)
+    if (p[x] > mx)
+      mx = p[x];
+  if (mx != 0)
+    {
+      for (x = 0; x < w; x++)
+	p[x] = p[x] / mx;
+    }
+}
+
+
+
+#define MAXPROFILE 1000
+float profile[MAXPROFILE], p2[MAXPROFILE];
+
+
+static int 
+possibly_joined (marktype * d)
+{
+  float aspect;
+  int i, n, x, y;
+  int leftable, l2;
+
+
+  aspect = d->w / (float) d->h;
+
+  if ((aspect >= 1.1) || (d->w < MAXPROFILE))
+    {
+      /* examine it more closely */
+
+
+      /* set up profile[] with a vertical profile */
+      for (x = 0; x < d->w; x++)
+	{
+	  profile[x] = 0;
+	  for (y = 0; y < d->h; y++)
+	    if (pbm_getpixel (d->bitmap, x, y))
+	      profile[x]++;
+	}
+
+      /* average the profile from (up to) 3 either way */
+      for (x = 0; x < d->w; x++)
+	{
+	  n = 0;
+	  p2[x] = 0;
+	  for (i = x - 3; i <= x + 3; i++)
+	    if ((i >= 0) && (i < d->w))
+	      {
+		n++;
+		p2[x] += profile[x];
+	      }
+	  p2[x] = p2[x] / (float) n;
+	}
+      for (x = 0; x < d->w; x++)
+	profile[x] = p2[x];
+
+      /* normalise profile[] */
+      normalise (profile, d->w);
+
+      /* find a little local minimum, with profile <0.25, make p2[] either 0 or 1 */
+      for (x = 0; x < d->w; x++)
+	p2[x] = 0;
+      for (x = 3; x < d->w - 3; x++)
+	{
+	  if (((profile[x - 2] - profile[x]) > 0.00) &&
+	      ((profile[x + 2] - profile[x]) > 0.00) &&
+	      (profile[x] < .25))
+	    p2[x] = 1;
+	}
+
+      /* travel rightwards, checking aspect ratio, looking at p2 now */
+      leftable = l2 = 0;
+      for (x = 0; x < d->w; x++)
+	if (p2[x] == 1)
+	  {
+	    if ((fabs ((abs (x - leftable) / (float) d->h) - 0.7) > 0.3) &&
+		(fabs ((abs (x - l2) / (float) d->h) - 0.7) > 0.3))
+	      p2[x] = 0;
+	    if (p2[x] == 1)
+	      {
+		leftable = l2;
+		l2 = x;
+	      }
+	  }
+
+      /* travel inwards, checking widths of chars, looking at p2 */
+      /* if a char has an aspect ratio of <= ? kill it */
+      l2 = 0;
+      leftable = d->w - 1;
+      x = 0;
+      y = d->w - 1;
+      while (x <= y)
+	{
+	  if (p2[x])
+	    {
+	      if (abs (x - l2) < max (d->h * 0.6, 5))
+		p2[x] = 0;
+	      if (p2[x])
+		l2 = x;
+	    }
+	  if (p2[y])
+	    {
+	      if (abs (y - leftable) < max (d->h * 0.6, 5))
+		p2[y] = 0;
+	      if (p2[y])
+		leftable = y;
+	    }
+	  x++;
+	  y--;
+	}
+
+
+      for (x = 0; x < d->w; x++)
+	if (p2[x])
+	  return 1;
+    }
+  return 0;
+}
+
+
+static void 
+split_mark (marklistptr * list, marktype * joinedmark)
+{
+  marktype new, old;
+  int x, found = 0, left = 0;
+  int i, j;
+  int oldw, oldh;
+  int overwrite = 1;
+
+  old = marktype_copy (*joinedmark);
+  oldw = joinedmark->w;
+  oldh = joinedmark->h;
+  for (x = 0; x < old.w; x++)
+    if (p2[x])
+      {
+	found = x;
+	break;
+      }
+  do
+    {
+      /* split at (found) position */
+      new.w = found + 1 - left;
+      new.h = old.h;
+      new.xpos = old.xpos + left;
+      new.ypos = old.ypos;
+      new.bitmap = pbm_allocarray (new.w, new.h);
+
+      for (i = 0; i < new.w; i++)
+	for (j = 0; j < new.h; j++)
+	  pbm_putpixel (new.bitmap, i, j, pbm_getpixel (old.bitmap, left + i, j));
+
+      marktype_adj_bound (&new);
+
+      if (overwrite)
+	{
+	  overwrite = 0;
+	  pbm_freearray (&(joinedmark->bitmap), oldh);
+	  *joinedmark = new;
+	}
+      else
+	marklist_add (list, new);
+
+
+      p2[found] = 0;
+      left = found;
+      found = 0;
+      for (x = left; x < old.w; x++)
+	if (p2[x])
+	  {
+	    found = x;
+	    break;
+	  }
+      if (left == old.w - 1)
+	break;			/* if we've got to the right */
+      if (found == 0)
+	found = old.w - 1;
+    }
+  while (found);
+  pbm_freearray (&(old.bitmap), oldh);
+}
+
+
+
+
+
+void 
+ExtractAllMarks (Pixel ** bitmap, marklistptr * list, int cols, int rows)
+{
+  int startx, starty, count;
+  marklistptr step;
+
+  startx = starty = 0;
+  while (ExtractNextMark (bitmap, list, &startx, &starty, cols, rows))
+    ;
+
+  if (splitjoined)
+    for (step = (*list), count = 0; step; step = step->next, count++)
+      if (possibly_joined (&(step->data)))
+	split_mark (list, &(step->data));
+
+  for (step = (*list), count = 0; step; step = step->next, count++)
+    step->data.symnum = count;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/extractor.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * extractor.h -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _EXTRACTOR_H_
+#define _EXTRACTOR_H_
+
+#include "marklist.h"
+#include "pbmtools.h"
+
+extern int splitjoined;
+
+void RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows);
+
+int ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows);
+void ExtractAllMarks (Pixel ** bitmap, marklistptr * marklist, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/felics.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/felics.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/felics.c	(revision 21112)
@@ -0,0 +1,340 @@
+/**************************************************************************
+ *
+ * felics.c -- The main guts of the felics algorithm
+ * Copyright (C) 1994  Neil Sharman and Kerry Guise
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+
+
+#undef putw
+#define putw(x)	{					\
+	short z;					\
+	byteptr = (unsigned char *)(x);			\
+	for (z = 0; z < sizeof(int); z++)		\
+		(void) putc((int) byteptr[z], fp_out);	\
+}
+#define INTSIZE sizeof(int) * 8
+#define BOOLEAN short
+#define TRUE 1
+#define FALSE 0
+
+#define IN_RANGE 0
+#define OUT_OF_RANGE 1
+#define BELOW_RANGE 0
+#define ABOVE_RANGE 1
+
+
+void 
+encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out, int (*get) (FILE *))
+{
+  int x, y;
+  unsigned long P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  ENCODE_START (fp_out)
+
+    for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		line[x] = get (fp_in);	/* +------- */
+		BINARY_ENCODE (line[x] + 1, maxval + 1);	/* |P****** */
+		continue;	/* |******* */
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	line[x] = P = get (fp_in);
+
+	if ((L <= P) && (P <= H))
+	  {
+	    /* IN-RANGE */
+	    unsigned long range, logofrange, thresh, numlong;
+	    int nbits;
+
+	    ENCODE_BIT (IN_RANGE);
+
+	    P -= L;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P -= (numlong >> 1);
+		if ((long) P < 0)
+		  P += range;
+
+		if (P < thresh)
+		  nbits = logofrange - 1;
+		else
+		  {
+		    nbits = logofrange;
+		    P += thresh;
+		  }
+		while (--nbits >= 0)
+		  ENCODE_BIT ((P >> nbits) & 1);
+	      }
+	  }
+	else
+	  {
+	    /* OUT_OF_RANGE */
+	    unsigned long diff, parm, i, min;
+
+	    ENCODE_BIT (OUT_OF_RANGE);
+
+	    if (P < L)
+	      {
+		ENCODE_BIT (BELOW_RANGE);
+		diff = L - P - 1;
+	      }
+	    else
+	      {
+		ENCODE_BIT (ABOVE_RANGE);
+		diff = P - H - 1;
+	      }
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_ENCODE ((diff >> parm) + 1);
+	    BINARY_ENCODE ((diff & ((1 << parm) - 1)) + 1, 1 << parm);
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+	  }
+
+      }
+
+  ENCODE_DONE
+
+}
+
+
+
+
+
+
+void 
+decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out)
+{
+  int x, y;
+  unsigned int P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  DECODE_START (fp_in);
+
+  for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		BINARY_DECODE (line[x], maxval + 1);
+		line[x]--;
+		if (maxval < 256)
+		  putc (line[x], fp_out);	/* +------- */
+		else		/* |P****** */
+		  fprintf (fp_out, "%u ", line[x]);	/* |******* */
+		continue;
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	if (DECODE_BIT == IN_RANGE)
+	  {
+	    unsigned long i, range, logofrange, thresh, numlong;
+	    unsigned xx, l;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+		for (P = i = 0; i < logofrange - 1; i++)
+		  DECODE_ADD (P);
+		xx = P;
+		l = logofrange - 1;
+		if (P >= thresh)
+		  {
+		    DECODE_ADD (P);
+		    xx = P;
+		    l++;
+		    P -= thresh;
+		  }
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P += (numlong >> 1);
+		if ((long) P >= range)
+		  P -= range;
+
+		P += L;
+	      }
+	    else
+	      P = L;
+	  }
+	else
+	  {
+	    int out_of_range = DECODE_BIT;
+	    unsigned long diff, unary, binary, i, min, parm;
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_DECODE (unary);
+	    diff = (unary - 1) << parm;
+	    BINARY_DECODE (binary, 1 << parm);
+	    diff |= binary - 1;
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+
+	    if (out_of_range == BELOW_RANGE)
+	      {
+		P = L - diff - 1;
+	      }
+	    else
+	      {
+		P = diff + H + 1;
+	      }
+	  }
+
+	line[x] = P;
+
+	if (maxval < 256)
+	  (void) putc (P, fp_out);
+	else
+	  fprintf (fp_out, "%u ", P);
+      }
+
+  DECODE_DONE
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/lstevent.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/lstevent.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/lstevent.c	(revision 21112)
@@ -0,0 +1,424 @@
+/**************************************************************************
+ *
+ * lstevent.c -- Routines to implement "events" using MTF lists.
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:16:56  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:40  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+void
+write_method ()
+{
+  fprintf (stderr, "C");
+}
+
+
+/*===========================================*/
+
+/*
+ * [Rodney Brown - Sep/95]
+ * On IBM RS/6000 AIX version 3.2 the compiler defines NULL as (void *)0
+ * when in ANSI mode. Hence the (event) and (point) casts
+ */
+
+
+#define GETNEWNODE(e,node)		\
+do 					\
+{	node = p32(numnodes) ;		\
+	node->eventnum = (event)e ;	\
+	node->count = increment ;	\
+	node->foll.totalcnt = 0 ;	\
+	node->foll.notfound = 0 ;	\
+	node->foll.list = (point)NULL ;	\
+	numnodes += 1 ;			\
+	nfreenodes -= 1 ;		\
+}					\
+while (false)
+
+/*===========================================*/
+
+#define INSERTNODE(U,node)		\
+do					\
+{	node->next = U->list ;		\
+	U->list = p16(node) ;		\
+	U->totalcnt += increment;	\
+}					\
+while (false)
+
+/*===========================================*/
+
+int
+halvelist (p)
+     register eventnode *p;
+{
+  register int sum = 0;
+  while (p != ENULL)
+    {
+      sum += p->count = (p->count + 1) / 2;
+      p = p32 (p->next);
+    }
+  return (sum);
+}
+
+/*===========================================*/
+
+void
+halveallcounts (U)
+     register eventset *U;
+{
+  U->notfound = (U->notfound + 1) / 2;
+  U->totalcnt = U->notfound;
+  U->totalcnt += halvelist (p32 (U->list));
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event_noex (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (e == node->eventnum)
+	break;
+      lbnd += node->count;
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  totl = U->totalcnt;
+
+  if (node == ENULL)
+    {				/* event not found in list */
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event_noex (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = U->totalcnt;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (tget < (lbnd += node->count))
+	    break;
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node, *p;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	{
+	  if (e == node->eventnum)
+	    break;
+	  lbnd += node->count;
+	}
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  if (node == ENULL)
+    {				/* event not found in list, and whole list has been scanned */
+      totl = lbnd + U->notfound;
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      /* first, total up rest of list */
+      totl = lbnd + node->count;
+      p = p32 (node->next);
+      while (p != ENULL)
+	{
+	  if (!excluded[p->eventnum])
+	    totl += p->count;
+	  p = p32 (p->next);
+	}
+      totl += U->notfound;
+
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = 0;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	totl += node->count;
+      node = p32 (node->next);
+    }
+  totl += U->notfound;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (!excluded[node->eventnum])
+	    {
+	      lbnd += node->count;
+	      if (tget < lbnd)
+		break;
+	    }
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+newnode (e)
+     event e;
+{
+  register eventnode *node;
+  GETNEWNODE (e, node);
+  node->next = (point)NULL;
+  node->prev = (point)NULL;
+  return (node);
+}
+
+/*===========================================*/
+
+/* not required in this version */
+
+#if 0
+eventnode *
+addevent (U, e)
+     register eventset *U;
+     event e;
+{
+  register eventnode *node;
+  /* get an event node */
+  GETNEWNODE (e, node);
+  INSERTNODE (U, node);
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.c	(revision 21112)
@@ -0,0 +1,827 @@
+/**************************************************************************
+ *
+ * marklist.c -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+void 
+marktype_dump (marktype d)
+{
+  fprintf (stderr, "symnum: %d\n", d.symnum);
+  fprintf (stderr, "xpos: %d, ypos: %d\n", d.xpos, d.ypos);
+  fprintf (stderr, "w: %d, h: %d\n", d.w, d.h);
+  fprintf (stderr, "xcen: %d, ycen: %d\n", d.xcen, d.ycen);
+  write_library_mark (stderr, d);
+}
+
+
+/* copy, including make a copy of the bitmap */
+marktype 
+marktype_copy (marktype d)
+{
+  marktype d2;
+  int i, j;
+
+  d2 = d;
+  d2.bitmap = pbm_allocarray (d2.w, d2.h);
+  for (i = 0; i < d2.w; i++)
+    for (j = 0; j < d2.h; j++)
+      pbm_putpixel (d2.bitmap, i, j, pbm_getpixel (d.bitmap, i, j));
+  return d2;
+}
+
+
+float 
+marktype_density (marktype d)
+{
+  return d.set / (float) (d.w * d.h);
+}
+
+
+
+void 
+marktype_calc_centroid (marktype * b)
+{
+  int r, c;
+  int count = 0;
+  int xtot = 0, ytot = 0;
+
+  for (r = 0; r < b->h; r++)
+    for (c = 0; c < b->w; c++)
+      if (pbm_getpixel (b->bitmap, c, r))
+	{
+	  count++;
+	  xtot += c;
+	  ytot += r;
+	}
+  if (count)
+    {
+      xtot /= count;
+      ytot /= count;
+    }
+  b->xcen = xtot;
+  b->ycen = ytot;
+}
+
+
+
+static void 
+marktype_area (marktype * d)
+{
+  int x, y;
+  int set = 0;
+
+  for (y = 0; y < d->h; y++)
+    for (x = 0; x < d->w; x++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	set++;
+  d->set = set;
+}
+
+
+static void 
+marktype_h_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+  for (y = 0; y < d->h; y++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (x = 0; x < d->w; x++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->h_runs = num;
+}
+
+static void 
+marktype_v_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+
+  for (x = 0; x < d->w; x++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (y = 0; y < d->h; y++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->v_runs = num;
+}
+
+
+/* assume we already have the centroid! */
+static void 
+marktype_prog_cent (marktype * d)
+{
+  int x, y;
+  int xtot = 0, ytot = 0;
+  int num = 0;
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen1 = -1;
+      d->ycen1 = -1;
+    }
+  else
+    {
+      d->xcen1 = xtot / num;
+      d->ycen1 = ytot / num;
+    }
+
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen2 = -1;
+      d->ycen2 = -1;
+    }
+  else
+    {
+      d->xcen2 = xtot / num;
+      d->ycen2 = ytot / num;
+    }
+
+
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+
+
+  if (num == 0)
+    {
+      d->xcen3 = -1;
+      d->ycen3 = -1;
+    }
+  else
+    {
+      d->xcen3 = xtot / num;
+      d->ycen3 = ytot / num;
+    }
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen4 = -1;
+      d->ycen4 = -1;
+    }
+  else
+    {
+      d->xcen4 = xtot / num;
+      d->ycen4 = ytot / num;
+    }
+}
+
+
+
+/* call this to get all features */
+void 
+marktype_calc_features (marktype * d, char str[])
+{
+  if (strlen (str))
+    strcpy (d->name, str);
+  marktype_calc_centroid (d);
+  marktype_area (d);
+  marktype_prog_cent (d);
+  marktype_h_runs (d);
+  marktype_v_runs (d);
+}
+
+
+
+
+
+void 
+marktype_adj_bound (marktype * m)
+{
+  int x, y, w, h;
+  int xpos, ypos;
+  int top = 0, left = 0, right = 0, bot = 0;
+  marktype temp;
+
+  w = m->w;
+  h = m->h;
+  xpos = m->xpos;
+  ypos = m->ypos;
+
+  for (y = 0; y < h; y++)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  top = y;
+	  x = w;
+	  y = h;
+	}
+
+  for (y = h - 1; y >= 0; y--)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  bot = y;
+	  x = w;
+	  y = -1;
+	}
+
+  for (x = 0; x < w; x++)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  left = x;
+	  x = w;
+	  y = h;
+	}
+
+  for (x = w - 1; x >= 0; x--)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  right = x;
+	  x = -1;
+	  y = h;
+	}
+
+  temp.w = right - left + 1;
+  temp.h = bot - top + 1;
+  temp.xpos = xpos + left;
+  temp.ypos = ypos + top;
+  temp.bitmap = pbm_allocarray (temp.w, temp.h);
+
+  for (x = 0; x < temp.w; x++)
+    for (y = 0; y < temp.h; y++)
+      pbm_putpixel (temp.bitmap, x, y, pbm_getpixel (m->bitmap, x + left, y + top));
+
+  pbm_freearray (&(m->bitmap), h);
+
+  marktype_calc_features (&temp, "?");
+  *m = temp;
+}
+
+
+
+
+
+
+/* returns a newly created mark reprenting the average of the marklist */
+void 
+marklist_average (marklistptr list, marktype * avgmark)
+{
+  int w = 0, h = 0;
+  int t = INT_MAX, b = 0, l = INT_MAX, r = 0, matched = 0;
+  marktype d, temp;
+  marklistptr step;
+  static int a[100][100];
+  int x, y;
+
+  if (list == NULL)
+    error_msg ("marklist_average", "Can't average 'nothing'!", "");
+
+  for (step = list; step; step = step->next)
+    {
+      matched++;
+      temp = step->data;
+      w = max (w, temp.w);
+      h = max (h, temp.h);
+      r = max (r, temp.xcen);
+      l = min (l, temp.xcen);
+      b = max (b, temp.ycen);
+      t = min (t, temp.ycen);
+    }
+  w = w + (r - l);
+  h = h + (b - t);
+  if ((w >= 100) || (h >= 100))
+    matched = 0;
+
+  if (matched > 1)
+    {
+      int x_off, y_off;
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  a[x][y] = 0;
+
+      for (step = list; step; step = step->next)
+	{
+	  x_off = r - step->data.xcen;
+	  y_off = b - step->data.ycen;
+	  for (x = 0; x < step->data.w; x++)
+	    for (y = 0; y < step->data.h; y++)
+	      if (pbm_getpixel (step->data.bitmap, x, y))
+		a[x + x_off][y + y_off]++;
+	}
+
+      d.w = w;
+      d.h = h;
+      d.bitmap = pbm_allocarray (w, h);
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  if (a[x][y] > max (1, (matched / 2)))		/* > gives better CR than >= */
+	    pbm_putpixel (d.bitmap, x, y, 1);
+      marktype_adj_bound (&d);
+      strcpy (d.name, "?");
+      marktype_calc_centroid (&d);
+    }
+  else
+    d = marktype_copy (list->data);
+
+  *avgmark = d;
+}
+
+
+
+
+
+
+void 
+marklist_setnull (marklistptr * list)
+{
+  *list = NULL;
+}
+
+
+marklistptr 
+marklist_add (marklistptr * list, marktype m)
+{
+  marklistptr p;
+
+  if (*list == NULL)
+    {
+      *list = (marklistptr) malloc (sizeof (marklisttype));
+      if ((*list) == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = *list;
+    }
+  else
+    {
+      p = *list;
+      while (p->next != NULL)
+	p = p->next;
+      p->next = (marklistptr) malloc (sizeof (marklisttype));
+      if (p->next == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = p->next;
+    }
+  p->data = m;
+  p->next = NULL;
+  return p;
+
+}
+
+
+
+marklistptr 
+marklist_addcopy (marklistptr * list, marktype m)
+{
+  marktype d;
+
+  d = marktype_copy (m);
+  return marklist_add (list, d);
+}
+
+
+
+int 
+marklist_length (marklistptr list)
+{
+  int count = 0;
+
+  while (list)
+    {
+      count++;
+      list = list->next;
+    }
+  return count;
+}
+
+
+
+
+
+void 
+marklist_dump (marklistptr list)
+{
+  int count = 0;
+  marktype d;
+
+  while (list)
+    {
+      d = list->data;
+      marktype_dump (d);
+
+      count++;
+      list = list->next;
+    }
+}
+
+
+
+
+/* pos starts from 0, to up marklist_length-1 */
+marklistptr 
+marklist_getat (marklistptr list, int pos, marktype * d)
+{
+  int count = 0;
+
+  while (list)
+    {
+      if (count == pos)
+	{
+	  *d = list->data;
+	  return list;
+	}
+      count++;
+      list = list->next;
+    }
+  fprintf (stderr, "marklist_getat(): access off ends of list: pos %d\n", pos);
+  return NULL;
+}
+
+
+
+void 
+marklist_free (marklistptr * list)
+{
+  marklistptr n;
+
+  while ((*list) != NULL)
+    {
+      n = (*list)->next;
+      pbm_freearray (&((*list)->data.bitmap), (*list)->data.h);
+      free ((*list));
+      (*list) = n;
+    }
+}
+
+
+
+int 
+marklist_removeat (marklistptr * list, int pos)
+{
+  marklistptr c, n, p = NULL;
+  int count = 0;
+
+  if ((pos == 0) && (*list))
+    {				/* if removing the 1st element */
+      *list = (*list)->next;
+      return 1;
+    }
+  else
+    {
+      c = *list;
+      while (c)
+	{
+	  if (count == pos)
+	    {			/* two cases, no previous, >=1 previous */
+	      if (p == NULL)	/* no previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  c = n;
+		  return 1;
+		}
+	      else
+		/* >=1 previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  p->next = n;
+		  return 1;
+		}
+	    }
+	  count++;
+	  p = c;
+	  c = c->next;
+	}
+    }
+  fprintf (stderr, "marklist_removeat(): access off ends of list: pos %d\n", pos);
+  return 0;			/* nothing was removed */
+}
+
+
+
+marklistptr 
+marklist_next (marklistptr list)
+{
+  return list->next;
+}
+
+
+/* only writes out ascii form */
+void 
+write_library_mark (FILE * fp, marktype d)
+{
+  int r, c, rows, cols;
+  int xtot = d.xcen, ytot = d.ycen;
+
+  rows = d.h;
+  cols = d.w;
+
+  fprintf (fp, "Mark: %d\n", d.symnum);
+  fprintf (fp, "Char: %s\n", d.name);
+  fprintf (fp, "Cols: %d Rows: %d\n", d.w, d.h);
+  for (r = 0; r < d.h; r++)
+    {
+      for (c = 0; c < d.w; c++)
+	if ((r == ytot) && (c == xtot))
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'C' : '!', fp);
+	else
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'X' : '.', fp);
+      putc ('\n', fp);
+    }
+}
+
+
+
+/* only reads in ascii library */
+int 
+read_library_mark (FILE * fp, marktype * d)
+{
+  int r, c, rows, cols, numread, ch1 = 'C';
+  char str1[100], str2[100];
+
+  numread = fscanf (fp, "%s %d\n", str1, &(d->symnum));
+  if ((numread != 2) || (strcmp (str1, "Mark:") != 0))
+    return 0;
+  fscanf (fp, "%s %s\n", str1, (d->name));
+  fscanf (fp, "%s %d %s %d\n", str1, &(d->w), str2, &(d->h));
+  cols = d->w;
+  rows = d->h;
+  d->bitmap = pbm_allocarray (cols, rows);
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c++)
+	{
+	  ch1 = getc (fp);
+	  if ((ch1 == 'X') || (ch1 == 'C'))
+	    pbm_putpixel (d->bitmap, c, r, 1);
+	  else
+	    pbm_putpixel (d->bitmap, c, r, 0);
+	}
+      ch1 = getc (fp);
+    }
+  marktype_calc_features (d, "?");
+
+  return 1;
+}
+
+
+
+
+
+
+
+int 
+read_library (char libraryname[], marklistptr * library)
+{
+  int err, count;
+  FILE *lib;
+  marktype d;
+  marklistptr step;
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg ("read_library()", "Trouble opening library file", "");
+  *library = NULL;
+  count = 0;
+  while (!isEOF (lib))
+    {
+      err = read_library_mark (lib, &d);
+      if (!err)
+	error_msg ("read_library()", "unknown format of the library file.", "");
+      d.symnum = count;
+      count++;
+      if ((*library) == NULL)
+	step = marklist_add (library, d);
+      else
+	step = marklist_add (&step, d);
+    }
+  count = marklist_length (*library);
+  fclose (lib);
+
+  return count;
+}
+
+
+
+
+
+
+/**********************************************************************
+
+'elem' routines for coding offsets
+
+the elem datatype contains the 3 params for arithmetic coding,
+stored as a no-duplicates order linked list.
+
+************************************************************************/
+
+
+
+
+void 
+elem_add (elemlist * list, int num)
+{
+  elemptr temp, nn;
+  int p, f;
+
+  list->tot++;
+  if (list->head == NULL)
+    {
+      list->head = (elemptr) malloc (sizeof (elem));
+      if ((list->head) == NULL)
+	error_msg ("", "ekk", "");
+      temp = list->head;
+      temp->next = NULL;
+      temp->num = num;
+      temp->pos = 0;
+      temp->freq = 1;
+    }
+  else
+    {				/* add to list */
+      temp = list->head;
+      p = 0;
+      f = 0;
+      while ((num > temp->num) && (temp->next != NULL))
+	{
+	  p = temp->pos;
+	  f = temp->freq;
+	  temp = temp->next;
+	}
+
+      if (temp->num == num)
+	temp->freq++;
+      else
+	{			/* insert it */
+	  nn = (elemptr) malloc (sizeof (elem));
+	  if (nn == NULL)
+	    error_msg ("", "ekk", "");
+
+	  if (num < temp->num)
+	    {
+	      nn->num = temp->num;
+	      nn->pos = temp->pos;
+	      nn->freq = temp->freq;
+	      nn->next = temp->next;
+	      temp->num = num;
+	      temp->pos = p + f;
+	      temp->freq = 1;
+	      temp->next = nn;
+	    }
+	  else
+	    {
+	      nn->next = temp->next;
+	      temp->next = nn;
+	      nn->pos = temp->pos + temp->freq;
+	      nn->num = num;
+	      nn->freq = 1;
+	      temp = nn;
+	    }
+	}
+    }
+  /* update frequencies */
+  temp = temp->next;
+  if (temp)
+    do
+      {
+	temp->pos++;
+	temp = temp->next;
+      }
+    while (temp);
+}
+
+
+void 
+elem_free (elemlist * list)
+{
+  elemptr n, p;
+
+  p = list->head;
+  while (p != NULL)
+    {
+      n = p->next;
+      free (p);
+      p = n;
+    }
+  list->head = NULL;
+}
+
+int 
+elem_find (elemlist * list, int n, int *p, int *f, int *t)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if (temp->num == n)
+	{
+	  *p = temp->pos;
+	  *f = temp->freq + temp->pos;
+	  *t = list->tot;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
+
+
+int 
+elem_getrange (elemlist * list, int range, int *num)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if ((range >= temp->pos) && (range < (temp->pos + temp->freq)))
+	{
+	  *num = temp->num;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/marklist.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * marklist.h -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MARKLIST_H
+#define __MARKLIST_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+typedef struct
+  {
+    int symnum;
+    int xpos, ypos;
+    int w, h;
+    int xcen, ycen;
+    int xoffset, yoffset;
+    char name[10];
+    Pixel **bitmap;
+
+    int xcen1, ycen1;
+    int xcen2, ycen2;
+    int xcen3, ycen3;
+    int xcen4, ycen4;
+    int set;
+    int h_runs;
+    int v_runs;
+  }
+marktype;
+
+
+typedef struct marklisttype *marklistptr;
+typedef struct marklisttype
+  {
+    marktype data;
+    marklistptr next;
+  }
+marklisttype;
+
+
+void marktype_dump (marktype d);
+marktype marktype_copy (marktype d);
+
+float marktype_density (marktype d);
+void marktype_calc_centroid (marktype * b);
+void marktype_calc_features (marktype * d, char str[]);
+void marktype_adj_bound (marktype * m);
+
+void marklist_average (marklistptr list, marktype * avgmark);
+void marklist_setnull (marklistptr * list);
+marklistptr marklist_add (marklistptr * list, marktype m);
+marklistptr marklist_addcopy (marklistptr * list, marktype m);
+int marklist_length (marklistptr list);
+void marklist_dump (marklistptr list);
+marklistptr marklist_getat (marklistptr list, int pos, marktype * d);
+void marklist_free (marklistptr * list);
+int marklist_removeat (marklistptr * list, int pos);
+marklistptr marklist_next (marklistptr list);
+
+void write_library_mark (FILE * fp, marktype d);
+int read_library_mark (FILE * fp, marktype * d);
+int read_library (char libraryname[], marklistptr * library);
+
+
+
+
+
+
+/* 
+   The 'elem' structure is an ordered linked list, with fields corresponding
+   directly to the arithmetic coding routines.  
+ */
+typedef struct elem *elemptr;
+typedef struct elem
+  {
+    int num;
+    int pos;
+    int freq;
+    elemptr next;
+  }
+elem;
+
+typedef struct elemlist
+  {
+    elemptr head;
+    int tot;
+  }
+elemlist;
+
+void elem_add (elemlist * list, int num);
+void elem_free (elemlist * list);
+int elem_find (elemlist * list, int n, int *p, int *f, int *t);
+int elem_getrange (elemlist * list, int range, int *num);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.c	(revision 21112)
@@ -0,0 +1,417 @@
+/**************************************************************************
+ *
+ * match.c -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/* error mask goes from 0..8, single pixel errors are worth nothing */
+int 
+WXOR_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+  marktype d;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  d.w = maxc - minc + 1;
+  d.h = maxr - minr + 1;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 = back ground colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;		/* background colour */
+	pbm_putpixel (d.bitmap, c - minc, r - minr, p1 != p2);
+      }
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (pbm_getpixel (d.bitmap, c - minc, r - minr))
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= 0) && (c + i >= 0) && (r + j < d.h) && (c + i < d.w) && ((i != 0) || (j != 0)) && (pbm_getpixel (d.bitmap, c + i - minc, r + j - minr)))
+	      count++;
+  pbm_freearray (&d.bitmap, d.h);
+  return count;
+}
+
+
+
+/* define an array for most matches, if it is a massive mark, then WXOR is called */
+#define S_SIZE 500
+
+static unsigned char scratch[S_SIZE][S_SIZE], sc2[S_SIZE][S_SIZE];
+
+
+
+
+/* Weighted AND-NOT */
+
+int 
+WAN_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (b1, b2);
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	scratch[r - minr][c - minc] = sc2[r - minr][c - minc] = 0;
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 == background colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;
+
+	if ((p1 == 0) && (p2 == 1))	/* p1 not set, p2 = set */
+	  scratch[r - minr][c - minc] = 1;
+	if ((p1 == 1) && (p2 == 0))	/* p1 set, p2 not set */
+	  sc2[r - minr][c - minc] = 1;
+      }
+
+  count = 0;
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && scratch[r - minr + j][c - minc + i])
+	      count++;
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (sc2[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && sc2[r - minr + j][c - minc + i])
+	      count++;
+
+  return count;
+}
+
+/* oc 8 must be in a clock wise order, with oc[0] a corner */
+struct
+{
+  int x, y;
+}
+oc[8] =
+{
+  {
+    -1, -1
+  }
+  ,
+  {
+    0, -1
+  }
+  ,
+  {
+    1, -1
+  }
+  ,
+  {
+    1, 0
+  }
+  ,
+  {
+    1, 1
+  }
+  ,
+  {
+    0, 1
+  }
+  ,
+  {
+    -1, 1
+  }
+  ,
+  {
+    -1, 0
+  }
+};
+
+
+
+static void 
+xormatchscratch (marktype * b1, marktype * b2, int minr, int maxr, int minc, int maxc, int offx, int offy)
+{
+  int r, c, rmm, roo, coo;
+  int p1, p2;
+  Pixel **bm1 = b1->bitmap, **bm2 = b2->bitmap;
+  int w1 = b1->w, h1 = b1->h, w2 = b2->w, h2 = b2->h;
+
+  for (r = minr; r < maxr; r++)
+    {
+      rmm = r - minr;
+      roo = r - offy;
+      p1 = 0;
+      for (c = minc; c < 0; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+      for (c = 0; c < maxc; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((r >= 0) && (c < w1) && (r < h1))
+	    p1 = pbm_getpixel (bm1, c, r);
+	  else
+	    p1 = 0;		/* background */
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+    }
+}
+
+
+
+int 
+CSIS_match (marktype * b1, marktype * b2)
+{
+  int r, c, i;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy, fepcount;
+  int totscratch = 0;
+
+
+  offx = b1->xcen - b2->xcen;
+  offy = b1->ycen - b2->ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1->w, b2->w + offx);
+  maxr = max (b1->h, b2->h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (*b1, *b2);
+
+  /* set the scratch array to 1 if there is an XOR match */
+
+  xormatchscratch (b1, b2, minr, maxr, minc, maxc, offx, offy);
+  /* We how have the XOR map. Scratch contains the number of neighbours, 1=1 neighbour set...etc */
+
+  /* CSIS rule 1, if FEP (four error pels) then exit */
+  fepcount = 0;
+  for (r = minr; r < maxr - 1; r++)
+    for (c = minc; c < maxc - 1; c++)
+      if ((scratch[r - minr][c - minc]) && (scratch[r - minr + 1][c - minc]) &&
+	  (scratch[r - minr][c - minc + 1]) && (scratch[r - minr + 1][c - minc + 1]))
+	{
+	  fepcount++;
+	  if (fepcount == 1)
+	    return 10001;
+	}
+
+  /* foreach set pixel, update scratch with the number of pels around in five dir's. 
+     FPN (five-pel neighbour hood) */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	{
+	  sc2[r - minr][c - minc] = 1;	/* itself */
+	  for (i = 1; i < 8; i += 3)
+	    if (((r + oc[i].y) >= minr) && ((c + oc[i].x) >= minc) && ((r + oc[i].y) < maxr) && ((c + oc[i].x) < maxc))
+	      if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])
+		sc2[r - minr][c - minc]++;
+	}
+  /* now scratch has the value of the FPN - 1=single error, 5=4 neighbouring errors etc... */
+
+  /* set scratch and sc2 to be the same */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      totscratch += scratch[r - minr][c - minc] = sc2[r - minr][c - minc];
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	/* CSIS rule 2i: an error pel has 2 or more neighbours---this num is the same as PMS */
+	if (scratch[r - minr][c - minc] >= 3)
+	  if ((r - minr >= 1) && (c - minc >= 1) && (r - minr < (maxr - 1)) && (c - minc < (maxc - 1)))
+	    {
+	      int i, numsep = 0;
+	      /* find how many neighbours a set pixel has, only check the 4 edges-FPN */
+	      for (i = 1; i < 8; i += 2)
+		if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])	/* if a scratch pixel is set */
+		  {
+		    if ((scratch[r + oc[(i + 2) % 8].y - minr][c + oc[(i + 2) % 8].x - minc] == 0) &&
+			(scratch[r + oc[(i + 6) % 8].y - minr][c + oc[(i + 6) % 8].x - minc] == 0)
+		      )
+		      numsep++;
+		  }
+
+	      if ((b1->w >= 12) || (b2->w >= 12) || (b1->h >= 12) || (b2->h >= 12))
+		{
+
+		  /* PMS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, j, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i >= 0) && (r + j >= 0) && (c + i < b1->w) && (r + j < b1->h))
+				  p2 = pbm_getpixel (b1->bitmap, c + i, r + j);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10002;
+			    }
+			}	/* (*b1) in range */
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i - offx >= 0) && (r + j - offy >= 0) && (c + i - offx < b2->w) && (r + j - offy < b2->h))
+				  p2 = pbm_getpixel (b2->bitmap, c + i - offx, r + j - offy);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	      else
+		{		/* do the CSIS rule */
+		  /* CSIS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x >= 0) && (r + oc[i].y >= 0) && (c + oc[i].x < b1->w) && (r + oc[i].y < b1->h))
+			      p2 = pbm_getpixel (b1->bitmap, c + oc[i].x, r + oc[i].y);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10002;
+			    }
+			}
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x - offx >= 0) && (r + oc[i].y - offy >= 0) && (c + oc[i].x - offx < b2->w) && (r + oc[i].y - offy < b2->h))
+			      p2 = pbm_getpixel (b2->bitmap, c + oc[i].x - offx, r + oc[i].y - offy);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	    }
+      }
+
+  return totscratch;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/match.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * match.h -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MATCH_H
+#define __MATCH_H
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/*
+   #define NOT_SCREENED(b1,b2) ((abs(b1.w-b2.w)<=4) && (abs(b1.h-b2.h)<=4))
+ */
+
+#define NOT_SCREENED(b1,b2) ((abs(b1.xcen-b2.xcen)+abs(b1.ycen-b2.ycen))<=3)
+
+#define MATCH_VAL 1000
+#define MATCH(b1,b2) (CSIS_match(b1,b2))
+
+int XOR_match (marktype b1, marktype b2);
+int WXOR_match (marktype b1, marktype b2);
+int WAN_match (marktype b1, marktype b2);
+int PMS_match (marktype b1, marktype b2);
+int CSIS_match (marktype * b1, marktype * b2);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgbilevel.c	(revision 21112)
@@ -0,0 +1,149 @@
+/**************************************************************************
+ *
+ * mgbilevel.c -- Program to compress/decompress bilevel images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "utils.h"
+#include "pbmtools.h"
+#include "arithcode.h"
+#include "bilevel.h"
+
+
+
+static char default_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+Encode_bitmap (marktype b, char template[])
+{
+  InitArithEncoding ();
+
+  bl_compress (b, template);
+
+  CloseDownArithEncoding ();
+}
+
+void 
+Decode_bitmap (marktype * b)
+{
+  InitArithDecoding ();
+
+  bl_decompress (b);
+
+  CloseDownArithDecoding ();
+}
+
+
+void 
+usage ()
+{
+  fprintf (stderr,
+	   "usage:\n"
+	   "       mgbilevel -e  [-t \"string\"]  infile      >compressed\n"
+	   "or\n"
+	   "       mgbilevel -d                 compressed  >outfile\n");
+  exit (1);
+}
+
+#define BUF_SIZE 65536
+
+void 
+main (int argc, char *args[])
+{
+  marktype d;
+  char *filename = NULL, *template = NULL;
+  Pixel **bitmap;
+  int encode = 0, decode = 0, i;
+  char buffer[BUF_SIZE];
+
+  if (argc < 2)
+    usage ();
+  template = default_template;
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp (args[i], "-e") == 0)
+	encode = 1;
+      else if (strcmp (args[i], "-d") == 0)
+	decode = 1;
+      else if (strcmp (args[i], "-h") == 0)
+	usage ();
+      else if (strcmp (args[i], "-t") == 0)
+	{
+	  template = args[i + 1];
+	  i++;
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!filename)
+	filename = args[i];
+      else
+	error_msg (args[0], "only specify 1 filename", "");
+    }
+  if (encode == decode)
+    {
+      fprintf (stderr, "You must specify either encode XOR decode.\n");
+      exit (1);
+    }
+
+  if (!filename)
+    {
+      fprintf (stderr, "You must specify a filename.\n");
+      exit (1);
+    }
+
+  if (encode)
+    {
+      bitmap = pbm_readnamedfile (filename, &d.w, &d.h);
+      if (bitmap == NULL)
+	error_msg (args[0], "problem reading bitmap", "");
+
+      arith_out = stdout;
+      setbuffer (stdout, buffer, BUF_SIZE);
+      magic_write (stdout, MAGIC_BILEVEL);
+      if (bitmap)
+	{
+	  d.bitmap = bitmap;
+	  Encode_bitmap (d, template);
+	}
+    }
+  else if (decode)
+    {
+      FILE *tpin;
+      tpin = fopen (filename, "rb");
+      if (!tpin)
+	error_msg ("mgbilevel", "Can't open file:", filename);
+      arith_in = tpin;
+      setbuffer (tpin, buffer, BUF_SIZE);
+      magic_check (tpin, MAGIC_BILEVEL);
+      Decode_bitmap (&d);
+      fclose (tpin);
+
+      setbuffer (stdout, buffer, BUF_SIZE);
+      pbm_writefile (stdout, d.bitmap, d.w, d.h);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgfelics.c	(revision 21112)
@@ -0,0 +1,155 @@
+/**************************************************************************
+ *
+ * mgfelics.c -- Program to compress/decompress grey scale images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "utils.h"
+
+
+#define MAXKVALUE 32
+
+
+
+void encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out, int (*get) (FILE *));
+
+void decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out);
+
+
+void 
+usage (char *prog)
+{
+  fprintf (stderr, "usage:\n");
+  fprintf (stderr, "\t%s -e [-k n]   infile    [>compressed]\n"
+	   "or\n"
+	   "\t%s -d  compressed-infile   [>outfile]\n", prog, prog);
+  exit (1);
+}
+
+
+
+
+int 
+main (int argc, char **argv)
+{
+  FILE *fp_in = (stdin), *fp_out = (stdout);
+  int width, height, maxval;
+  u_long magic;
+  char *filename = NULL;
+  unsigned int *line;
+  int maxk = 0;
+  int i;
+  int encode_flag = 0, decode_flag = 0;
+
+  if (argc < 2)
+    usage (argv[0]);
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (argv[i], "-h"))
+	usage (argv[0]);
+      else if (!strcmp (argv[i], "-e"))
+	encode_flag = 1;
+      else if (!strcmp (argv[i], "-d"))
+	decode_flag = 1;
+      else if (!strcmp (argv[i], "-k"))
+	{
+	  maxk = atoi (argv[++i]);
+	}
+      else if (argv[i][0] == '-')
+	error_msg (argv[0], "unknown switch:", argv[i]);
+      else if (!filename)
+	filename = argv[i];
+      else
+	error_msg (argv[0], "too many filenames specified.", "");
+    }
+
+  if (encode_flag == decode_flag)
+    error_msg (argv[0], "please specify either encode XOR decode.", "");
+
+  if (!filename)
+    error_msg (argv[0], "please specify a filename.", "");
+
+  if ((encode_flag) && (maxk >= MAXKVALUE))
+    {				/* to fit into 32 bit integers */
+      fprintf (stderr, "%s: k value of %d is too large; it must be less than %d\n",
+	       argv[0], maxk, MAXKVALUE);
+      exit (1);
+    }
+
+
+  if ((fp_in = fopen (filename, "rb")) == NULL)
+    error_msg (argv[0], "unable to open file:", filename);
+
+  /* read file header to determine the type and dimensions */
+
+
+  magic = magic_read (fp_in);
+  if (magic != MAGIC_FELICS)
+    {
+      rewind (fp_in);
+      magic = getmagicno_short_pop (fp_in);
+    }
+
+  if ((decode_flag && (magic != MAGIC_FELICS)) ||
+      (encode_flag && ((magic != MAGIC_P2) && (magic != MAGIC_P5))))
+    {
+      fclose (fp_in);
+      error_msg (argv[0], encode_flag ? "only PGM files can be used." : "only felics compressed files can be used.", "");
+    }
+
+  width = gethint (fp_in);
+  height = gethint (fp_in);
+  maxval = gethint (fp_in);
+
+  if ((line = (unsigned int *) malloc (width * sizeof (int *))) == NULL)
+    {
+      fclose (fp_in);
+      error_msg (argv[0], "unable to allocate memory,", "");
+    }
+
+
+  if (encode_flag)
+    {
+      magic_write (fp_out, MAGIC_FELICS);
+      fprintf (fp_out, "%d\n%d\n%d\n%d\n", width, height, maxval, maxk);
+      encode (line, width, height, maxval, maxk, fp_in, fp_out,
+	      magic == MAGIC_P2 ? getint : fgetc);
+    }
+  else if (decode_flag)
+    {
+      if (maxk)
+	warn (argv[0], "k value not required for decompression, ignored.", "");
+
+      maxk = gethint (fp_in);
+      fprintf (fp_out, "P%d\n%d %d\n%d\n", maxval < 256 ? 5 : 2,
+	       width, height, maxval);
+      decode (line, width, height, maxval, maxk, fp_in, fp_out);
+    }
+
+
+  fclose (fp_in);
+  fclose (fp_out);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgtic.c	(revision 21112)
@@ -0,0 +1,596 @@
+/**************************************************************************
+ *
+ * mgtic.c -- Program to compress/decompress textual images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file is the basis of mgtic, which compresses the components of an
+ * image. 
+ * The compressed file contains the following sections:
+ *         mgtic magic_no
+ *         mgtic version
+ *         lossy/lossless flag 
+ *         internal/external library flag 
+ *         cols, rows, number of marks, library size
+ *         [library sequence]
+ *         symbol sequence
+ *         offset sequence
+ *         [residue bitmap]
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+#include "sortmarks.h"
+#include "codesyms.h"
+#include "arithcode.h"
+#include "codeoffsets.h"
+#include "bilevel.h"
+
+static char library_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgtic -e [-L] [-Q] [-X]  libraryfile   infile    [>compressed]\n"
+	   "or\n"
+	   "\tmgtic -d [-L] [-X libraryfile]  compressed  [>outfile]\n");
+  exit (1);
+}
+
+
+/* encode the list of marks 'list', w.r.t. the library 'library' */
+void 
+match_sequence (marklistptr list, marklistptr library, marklistptr * symbol_list, Pixel ** recon, int quick)
+{
+  int carryx, carryy, gap;
+  int lastx = 0, lasty = 0;
+  int i, j;
+  marktype d, d2;
+  marklistptr liststep, librarystep;
+  int score, found;
+  int r, c;
+  int p;
+
+  carryx = carryy = 0;
+  gap = max (1, marklist_length (list) / 10);
+
+  for (liststep = list, i = 0; liststep; liststep = liststep->next, i++)
+    {
+      if (V)
+	if (!(i % gap))
+	  fprintf (stderr, ".");
+      d = liststep->data;
+
+      found = -1;
+      score = INT_MAX;
+      for (librarystep = library, j = 0; librarystep; librarystep = librarystep->next, j++)
+	{
+	  d2 = librarystep->data;
+
+	  if (NOT_SCREENED (d2, d))
+	    if ((p = MATCH (&d, &d2)) < MATCH_VAL)
+	      {
+		if (p < score)
+		  {
+		    score = p;
+		    found = j;
+		    if (quick)
+		      break;
+		  }
+	      }
+	}
+      /*
+         if(found<0){
+         for(librarystep=library,j=0; librarystep; librarystep=librarystep->next,j++){
+         d2=librarystep->data;
+
+         if((p=MATCH(&d,&d2))<MATCH_VAL){
+         if(p<score){
+         score=p;found=j; if(quick) break;
+         }
+         }
+         }
+         }
+       */
+      if (found >= 0)
+	{
+	  liststep->data.symnum = found;
+	  marklist_getat (library, found, &d2);
+	  d2.symnum = found;
+
+	  d2.xoffset = d.xpos - lastx + carryx;		/* propagate changes to next mark */
+	  d2.yoffset = d.ypos - lasty + carryy;
+	  carryx = (d2.xcen - d.xcen);	/* offet centroids because thats where we match them! */
+	  carryy = (d2.ycen - d.ycen);
+	  d2.xoffset -= carryx;	/* make the changes to this mark */
+	  d2.yoffset -= carryy;
+	  lastx = d.xpos + d2.w;
+	  lasty = d.ypos;
+
+	  marklist_add (symbol_list, d2);
+
+	  /* add to the reconstructed image--this is the lossy version */
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, d.xpos + c - carryx, d.ypos + r - carryy, 1, d2.w, d2.h);
+	}
+      else
+	{
+	  /* didn't match at all..hopefully noise */
+	  if ((liststep->data.w >= 10) || (liststep->data.w >= 10))
+	    {
+	      fprintf (stderr, "mgtic: warning, unable to match a char.\n");
+	      write_library_mark (stderr, liststep->data);
+	    }
+	  if (V)
+	    fprintf (stderr, "x");
+	  liststep->data.symnum = -1;
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+}
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i, n, r, c, ch;
+  FILE *inf;
+  marklistptr library = NULL, list = NULL, step = NULL;
+  marklistptr symbol_list = NULL;
+  marktype d, d2;
+  Pixel **bitmap, **recon, **bitmapcopy;
+  char *splitfilename = NULL;
+  int cols, rows, count, librarysize, lossy = 1;
+  int quick = 0;
+  int ticencode = 0, ticdecode = 0, external = 0;
+  char *libraryname = NULL, *infile = NULL;
+  char *s1 = NULL, *s2 = NULL;
+  char bufferin[BUFSIZ], bufferout[BUFSIZ];
+
+  if (argc < 2)
+    usage ();
+
+  while ((ch = getopt (argc, args, "edlLQhvXR:")) != -1)
+    switch (ch)
+      {
+      case 'e':
+	ticencode = 1;
+	break;
+      case 'd':
+	ticdecode = 1;
+	break;
+      case 'l':
+	lossy = 1;
+	break;
+      case 'L':
+	lossy = 0;
+	break;
+      case 'Q':
+	quick = 1;
+	break;
+      case 'v':
+	V = 1;
+	break;
+      case 'X':
+	external = 1;
+	break;
+      case 'R':
+	splitfilename = optarg;
+	break;
+      case 'h':
+      case '?':
+	usage ();
+      }
+
+  for (i = 1; i < argc; i++)
+    if (args[i][0] != '-')
+      if (strcmp (args[i - 1], "-R"))
+	{			/* ignore arg following -R, ie. -R <filename> */
+	  if (!s1)
+	    s1 = args[i];
+	  else if (!s2)
+	    s2 = args[i];
+	  else
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+  if (ticencode == ticdecode)
+    error_msg (args[0], "please specify either encode XOR decode", "");
+
+  if (ticencode)
+    {
+	  /*****************
+	    ENCODING STAGE
+	  *****************/
+      long PrevBits = 0;
+      long BITS_header = 0, BITS_symbols = 0, BITS_offsets = 0, BITS_residue = 0,
+        BITS_footer = 0, BITS_library = 0;
+
+      libraryname = s1;
+      if (!libraryname)
+	error_msg (args[0], "please specify a library file", "");
+      infile = s2;
+      if (!infile)
+	error_msg (args[0], "please specify a file name", "");
+
+      inf = fopen (infile, "rb");
+      if (inf == NULL)
+	error_msg (args[0], "Trouble opening file:", infile);
+      setbuf (inf, bufferin);
+
+      count = librarysize = read_library (libraryname, &library);
+
+      if (V)
+	fprintf (stderr, "%s: processing...\n", args[0]);
+
+      if (pbm_isapbmfile (inf))
+	{
+	  if (V)
+	    fprintf (stderr, "reading file %s...\n", infile);
+	  bitmap = pbm_readfile (inf, &cols, &rows);
+	  fclose (inf);
+
+	  bitmapcopy = pbm_copy (bitmap, cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "extracting...");
+	  ExtractAllMarks (bitmap, &list, cols, rows);
+	  if (V)
+	    fprintf (stderr, "(%d marks)\n", marklist_length (list));
+
+	  /* sort into reading order */
+	  if (V)
+	    fprintf (stderr, "sorting...\n");
+	  list = sortmarks (list);
+
+	  pbm_freearray (&bitmap, rows);	/* clear old version */
+	  bitmap = bitmapcopy;	/* point to the copy */
+	  recon = pbm_allocarray (cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "matching...");
+
+	  match_sequence (list, library, &symbol_list, recon, quick);
+
+
+	  PrevBits = 0;
+	  setbuf (stdout, bufferout);
+	  /* start output */
+	  magic_write (stdout, MAGIC_TIC);
+	  InitArithEncoding ();
+
+	  EncodeGammaDist (1);	/* version 1 of the program */
+	  EncodeGammaDist (lossy);	/* lossy=1 = no residue */
+	  EncodeGammaDist (external);	/* 1== external file */
+
+	  count = marklist_length (symbol_list);
+	  if (V)
+	    fprintf (stderr, "encoding cols, rows, and number of symbols=%d\n", count);
+
+	  EncodeGammaDist (cols);
+	  EncodeGammaDist (rows);
+	  EncodeGammaDist (count);
+	  EncodeGammaDist (librarysize);
+
+	  BITS_header = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the library sequence */
+	  if (external == 0)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding library\n");
+
+	      bl_clearmodel ();
+	      bl_writetemplate (library_template);
+	      for (step = library; step; step = step->next)
+		bl_compress_mark (step->data);
+	      bl_freemodel ();
+	      BITS_library = CountOfBitsOut - PrevBits;
+	      PrevBits = CountOfBitsOut;
+	    }
+
+	  /* output the symbol sequence */
+	  if (V)
+	    fprintf (stderr, "encoding symbol sequence\n");
+	  InitPPM ();
+	  EncodeSymbols (symbol_list, count);
+	  BITS_symbols = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the offset sequence */
+	  if (V)
+	    fprintf (stderr, "encoding offset sequence\n");
+	  EncodeOffsets (symbol_list, count);
+	  BITS_offsets = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+	  EncodeChecksum ();	/* code lossy checksum */
+
+	  /* calculate the residue...and compress it---if need be! */
+	  if (!lossy)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding residue...\n");
+
+	      d.bitmap = bitmap;
+	      d.h = rows;
+	      d.w = cols;
+	      d2.bitmap = recon;
+	      d2.h = rows;
+	      d2.w = cols;
+
+	      if (splitfilename)
+		{
+		  FILE *temp;
+
+		  CloseDownArithEncoding ();
+		  fclose (stdout);
+		  /* no residue result */
+		  BITS_footer = CountOfBitsOut - PrevBits;
+
+		  temp = fopen (splitfilename, "wb");
+		  if (temp == NULL)
+		    error_msg (args[0], "Trouble creating file:", splitfilename);
+
+		  arith_out = temp;
+		  InitArithEncoding ();
+		  bl_clair_compress (d, d2);
+		  CloseDownArithEncoding ();
+		  BITS_residue = CountOfBitsOut;
+		  fclose (temp);
+		}
+	      else
+		{
+		  bl_clair_compress (d, d2);
+		  BITS_residue = CountOfBitsOut - PrevBits;
+		  PrevBits = CountOfBitsOut;
+		  EncodeChecksum ();	/* code lossless checksum */
+		  CloseDownArithEncoding ();
+		  BITS_footer = CountOfBitsOut - PrevBits;
+		}
+	    }
+	  else
+	    {
+	      if (V)
+		fprintf (stderr, "not encoding residue..lossy mode\n");
+	      CloseDownArithEncoding ();
+	      BITS_footer = CountOfBitsOut - PrevBits;
+	    }
+
+	  /* because we edit the values above */
+	  CountOfBitsOut = BITS_header + BITS_library + BITS_symbols + BITS_offsets + BITS_residue + BITS_footer;
+
+	  fprintf (stderr, "bits: header=%ld, library=%ld, "
+		   "symbols=%ld, offsets=%ld, residue=%ld, footer=%ld\n",
+		   BITS_header, BITS_library, BITS_symbols,
+		   BITS_offsets, BITS_residue, BITS_footer);
+	  fprintf (stderr, "total bits: %ld, ", CountOfBitsOut);
+	  fprintf (stderr, "Lossy CR: %4.2f", (cols * rows) / (float) (CountOfBitsOut - BITS_residue));
+	  if (external)
+	    fprintf (stderr, " (excluding external lib)");
+	  fprintf (stderr, ", Lossless CR: %4.2f\n", (!lossy) * (cols * rows) / (float) (CountOfBitsOut));
+	}
+      else
+	error_msg (args[0], "unknown format of bitmap--expecting PBM.", "");
+    }
+  else
+    {
+	  /*****************
+	    DECODING STAGE
+	  *****************/
+      int lastx, lasty;
+      librarysize = 0;
+
+      if (external)
+	{
+	  libraryname = s1;
+	  infile = s2;
+	  count = librarysize = read_library (libraryname, &library);
+	}
+      else
+	{
+	  infile = s1;
+	  if (infile && s2)
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+      if (!freopen (infile, "rb", stdin))
+	error_msg (args[0], "Trouble opening file:", infile);
+
+
+      if (V)
+	fprintf (stderr, "decompressing...\n");
+
+      setbuf (stdin, bufferin);
+      magic_check (stdin, MAGIC_TIC);
+
+      InitArithDecoding ();
+
+      {
+	int version = DecodeGammaDist ();
+	if (version != 1)
+	  error_msg (args[0], "Need later version of decompressor.", "");
+      }
+
+      {
+	int templossy = DecodeGammaDist ();
+	if (!lossy)
+	  lossy = templossy;	/* can only choose if encoded file is lossless */
+	if (V)
+	  {
+	    if (lossy)
+	      fprintf (stderr, "lossy mode\n");
+	    else
+	      fprintf (stderr, "lossless mode\n");
+	  }
+      }
+
+      if (DecodeGammaDist ())
+	{			/* if compressed file doesn't contain library */
+	  if (!external)
+	    error_msg (args[0], "compressed file doesn't contain library, specify externally", "");
+	  external = 1;
+	}
+      else
+	{			/* if compressed file contains library */
+	  if (external)
+	    fprintf (stderr, "ignoring external library file\n");
+	  external = 0;
+	}
+
+      cols = DecodeGammaDist ();
+      rows = DecodeGammaDist ();
+      count = DecodeGammaDist ();
+
+      i = DecodeGammaDist ();	/* librarysize */
+      if (external)
+	{
+	  if (i > librarysize)
+	    error_msg (args[0], "external library file is too small!", "");
+	  else if (i < librarysize)
+	    fprintf (stderr, "%s: warning, expecting a different (smaller) library.\n", args[0]);
+	}
+      librarysize = i;
+
+      if (V)
+	fprintf (stderr, "cols %d, rows %d, num syms %d, library size %d\n", cols, rows, count, librarysize);
+
+      /* decode library */
+      if (external == 0)
+	{
+	  if (V)
+	    fprintf (stderr, "reading library\n");
+	  bl_clearmodel ();
+	  bl_readtemplate ();
+	  for (n = 0; n < librarysize; n++)
+	    {
+	      bl_decompress_mark (&d);
+	      d.symnum = n;
+	      if (library == NULL)
+		step = marklist_add (&library, d);
+	      else
+		step = marklist_add (&step, d);
+	    }
+	  bl_freemodel ();
+	  if (V)
+	    fprintf (stderr, "read %d marks from library\n", marklist_length (library));
+	}
+
+      recon = pbm_allocarray (cols, rows);
+
+      /* decode symbols */
+      InitPPM ();
+      if (V)
+	fprintf (stderr, "decompressing %d symbols\n", count);
+      symbol_list = DecodeSymbols (count);
+
+      /* decode offsets */
+      if (V)
+	fprintf (stderr, "reading offsets...\n");
+      DecodeOffsets (symbol_list, count);
+      lastx = lasty = 0;
+      for (step = symbol_list; step; step = step->next)
+	{
+	  lastx = lastx + step->data.xoffset;
+	  lasty = lasty + step->data.yoffset;
+
+	  marklist_getat (library, step->data.symnum, &d2);
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, lastx + c, lasty + r, 1, cols, rows);	/* we don't care, already warned them! */
+	  lastx += d2.w;
+	}
+
+      DecodeChecksum (args[0]);
+
+      /* decode the residue */
+      if (!lossy)
+	{
+	  if (V)
+	    fprintf (stderr, "decoding residue...\n");
+
+	  bitmap = pbm_allocarray (cols, rows);
+	  d.bitmap = bitmap;
+	  d.w = d2.w = cols;
+	  d.h = d2.h = rows;
+
+	  d2.bitmap = recon;
+
+	  /* NOTE: the 2nd argument is clairvoyantly compressed */
+	  if (splitfilename)
+	    {
+	      FILE *temp;
+
+	      CloseDownArithDecoding ();
+	      fclose (stdin);
+
+	      temp = fopen (splitfilename, "rb");
+	      if (temp == NULL)
+		error_msg (args[0], "Trouble opening file:", splitfilename);
+
+	      arith_in = temp;
+	      InitArithDecoding ();
+	      bl_clair_decompress (d, d2);
+	      CloseDownArithDecoding ();
+	      fclose (temp);
+	    }
+	  else
+	    {
+	      bl_clair_decompress (d, d2);
+	      DecodeChecksum (args[0]);
+	      CloseDownArithDecoding ();
+	    }
+	  pbm_freearray (&recon, rows);
+	  recon = bitmap;	/* point to the bitmap */
+	}
+      else
+	{
+	  CloseDownArithDecoding ();
+	}
+
+      if (V)
+	fprintf (stderr, "writing pbm file...\n");
+      setbuf (stdout, bufferout);
+      pbm_writefile (stdout, recon, cols, rows);
+      pbm_freearray (&recon, rows);
+    }				/* end decoding */
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticbuild.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * mgticbuild.c -- Program to build the textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgticbuild   libraryfile   infile.pbm  [infile2 infile3 ...]\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib = NULL, *inf = NULL;
+  marklistptr list = NULL, step;
+  Pixel **bitmap;
+  int cols, rows;
+  int count, pos = 0;
+  char *libraryname = NULL;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	pos = i;
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+  if (pos < 2)
+    usage ();
+
+  count = 0;
+  for (i = pos; i < argc; i++)
+    if (args[i][0] != '-')
+      {
+	inf = fopen (args[i], "rb");
+	count = 0;
+	if (inf == NULL)
+	  fprintf (stderr, "Trouble opening file: %s\n", args[i]);
+	else
+	  {
+	    if (pbm_isapbmfile (inf))
+	      {
+		if (V)
+		  fprintf (stderr, "%s: processing...\n", args[0]);
+		if (V)
+		  fprintf (stderr, "reading file %s...", args[i]);
+		bitmap = pbm_readfile (inf, &cols, &rows);
+		if (V && (count == 0))
+		  fprintf (stderr, "building...");
+
+		list = NULL;
+
+		ExtractAllMarks (bitmap, &list, cols, rows);
+
+		if (V)
+		  fprintf (stderr, "writing...");
+
+		if ((!lib) && (marklist_length (list)))
+		  {
+		    lib = fopen (libraryname, "r");
+		    if (lib)
+		      {
+			fclose (lib);
+			lib = fopen (libraryname, "ab");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble appending library file", "");
+			fprintf (stderr, "\nwarning: appending library file: %s\n", libraryname);
+		      }
+		    else
+		      {
+			lib = fopen (libraryname, "wb");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble creating library file", "");
+		      }
+		  }		/* if !lib */
+
+		step = list;
+		while (step)
+		  {
+		    step->data.symnum = count;
+		    write_library_mark (lib, step->data);
+		    count++;
+		    step = step->next;
+		  }
+		pbm_freearray (&bitmap, rows);
+	      }
+	    else
+	      fprintf (stderr, "warning: %s is not a PBM file.\n", args[i]);
+	    fclose (inf);
+	  }
+	if (count)
+	  {
+	    if (V)
+	      fprintf (stderr, "%d marks written.\n", count);
+	  }
+	else
+	  fprintf (stderr, "No marks written!\n");
+      }
+
+  if (lib)
+    fclose (lib);
+  marklist_free (&list);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticdump.c	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mgticdump.c -- Program to dump out a library in a human readable form
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticdump   libraryfile\n");
+  exit (1);
+}
+
+
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib;
+  int count, i;
+  marktype d;
+  char *libraryname = NULL;
+  FILE *outf = (stdout);
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble opening library file:", libraryname);
+
+  count = 0;
+  while (!isEOF (lib))
+    {
+      read_library_mark (lib, &d);
+
+      write_library_mark (outf, d);
+      count++;
+
+      pbm_freearray (&d.bitmap, d.h);
+    }
+  fprintf (stderr, "count=%d", count);
+  fclose (lib);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticprune.c	(revision 21112)
@@ -0,0 +1,393 @@
+/**************************************************************************
+ *
+ * mgticprune.c -- Program to prune a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+
+marklistptr *avglib = NULL;
+int avglibsize;
+int libsize = 0;
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+    "\tmgticbuild [-a n]  [-s n]  [-Q]   libraryfile   [pruned-library]\n");
+  exit (1);
+}
+
+
+
+
+/* foreach list in the avglib[] array, an average mark is returned */
+marklistptr 
+average_library ()
+{
+  marklistptr librarylist;
+  int i;
+  marktype avgmark;
+
+  librarylist = NULL;
+  for (i = 0; i < libsize; i++)
+    {
+      if (avglib[i])
+	{
+	  marklist_average (avglib[i], &avgmark);
+	  avgmark.symnum = i;
+	  marklist_addcopy (&librarylist, avgmark);
+	}
+    }
+  return librarylist;
+}
+
+
+void 
+average_space_enlarge ()
+{
+  int j;
+
+  if (libsize >= avglibsize)
+    {
+      avglib = (marklistptr *) realloc (avglib, (avglibsize + 100) * sizeof (marklistptr));
+      if (!avglib)
+	error_msg ("mgticprune", "out of memory in reallocating..", "");
+      for (j = avglibsize; j < avglibsize + 100; j++)
+	avglib[j] = NULL;
+      avglibsize += 100;
+    }
+}
+
+
+/* for each mark in 'origlist', if it can't be matched with one in 
+   'librarylist', then add to the librarylist, and update the relevant
+   avglib[] position */
+void 
+greedy_add (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, step_i;
+  int i, j, notfound;
+  marktype d_i;
+
+  for (i = 0, step_i = origlist; step_i; i++, step_i = step_i->next)
+    {
+      d_i = step_i->data;
+
+      notfound = 1;
+      for (j = 0, step = *librarylist; step; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d_i, step->data))
+	    if (MATCH (&d_i, &step->data) < MATCH_VAL)
+	      {
+		d_i.symnum = j;
+		marklist_addcopy (&avglib[j], d_i);
+		notfound = 0;
+		break;
+	      }
+	}
+
+      if (notfound)
+	{
+	  d_i.symnum = libsize;
+	  marklist_addcopy (librarylist, d_i);
+	  marklist_addcopy (&avglib[libsize], d_i);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+/* for each mark in 'library', if another matches with it, remove
+   one copy from the library, and erase the relevant avglib[] */
+void 
+greedy_prune (marklistptr * library)
+{
+  int count;
+  int i, j;
+  marklistptr step;
+  marktype d;
+
+  count = marklist_length (*library);
+  /* an invariant, all marks <i have been kept in the pruned library */
+  for (i = 0; i < count; i++)
+    {
+      marklist_getat (*library, i, &d);
+
+      for (j = 0, step = *library; j < i; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d, step->data))
+	    if (MATCH (&d, &step->data) < MATCH_VAL)
+	      {
+		/* [i,d] has matched with library element [j,d2]. */
+		marklist_removeat (library, i);		/* because we kill it here */
+		marklist_free (&avglib[d.symnum]);
+		avglib[d.symnum] = NULL;
+		i--, count--;
+		break;
+	      }
+	}
+    }
+}
+
+
+void 
+prune_on_size (marklistptr * list, int sizethres)
+{
+  int count, i;
+  marktype d;
+
+  if (sizethres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if ((d.w <= sizethres) && (d.h <= sizethres))
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+
+void 
+prune_on_area (marklistptr * list, int areathres)
+{
+  int count, i;
+  marktype d;
+
+  if (areathres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if (d.set <= areathres)
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+void 
+averagedout_addagain (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, stepi;
+  int notfound;
+  int i, j;
+
+  for (i = 0, step = origlist; step; i++, step = step->next)
+    {
+      notfound = 1;
+
+      for (j = 0, stepi = *librarylist; stepi; j++, stepi = stepi->next)
+	if (NOT_SCREENED (step->data, stepi->data))
+	  if (MATCH (&step->data, &stepi->data) < MATCH_VAL)
+	    {
+	      notfound = 0;
+	      break;
+	    }
+
+      if (notfound)
+	{
+	  marklist_addcopy (librarylist, step->data);
+	  marklist_addcopy (&avglib[libsize], step->data);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib;
+
+  marklistptr origlist = NULL, librarylist = NULL;
+  marklistptr step;
+  char tempn[1000], sysstr[1000];
+  int count;
+  char *libraryname = NULL, *prunename = NULL;
+  int sizethres = 3, areathres = 0;
+  int pass;
+  int num_passes = 3;
+  int overwrite = 0;
+
+  V = 0;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (!strcmp (args[i], "-Q"))
+	{
+	  num_passes = 0;
+	}
+      else if (!strcmp (args[i], "-s"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    sizethres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -s option with a numeric arg >=0", "");
+	}
+      else if (!strcmp (args[i], "-a"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    areathres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -a option with a numeric arg >=0", "");
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else if (!prunename)
+	prunename = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+  if (V)
+    fprintf (stderr, "%s: processing...\n", args[0]);
+
+  overwrite = 0;
+  if (!prunename)
+    {
+      prunename = libraryname;
+      if (V)
+	fprintf (stderr, "overwriting previous library file.\n");
+      overwrite = 1;
+    }
+
+  if (overwrite)
+    {
+      tmpnam (tempn);
+    }
+  else
+    strcpy (tempn, prunename);
+
+
+  avglibsize = count = read_library (libraryname, &origlist);
+  if (V)
+    fprintf (stderr, "%d marks...", count);
+
+  avglib = (marklistptr *) malloc (count * sizeof (marklistptr));
+  if (!avglib)
+    error_msg (args[0], "out of memory", "");
+  for (i = 0; i < avglibsize; i++)
+    avglib[i] = NULL;
+
+
+  prune_on_size (&origlist, sizethres);
+
+  prune_on_area (&origlist, areathres);
+  if (V)
+    fprintf (stderr, "after threshold pruning, %d marks\n", count = marklist_length (origlist));
+
+
+  if (V)
+    fprintf (stderr, "starting greedy library creation...\n");
+
+  greedy_add (origlist, &librarylist);
+
+
+  for (pass = 0; pass < num_passes; pass++)
+    {
+      marklist_free (&librarylist);
+      if (V)
+	fprintf (stderr, "averaging...\n");
+      librarylist = average_library ();
+      averagedout_addagain (origlist, &librarylist);
+      greedy_prune (&librarylist);
+      if (V)
+	fprintf (stderr, "after pass %d greedy prune %d marks\n", 1 + pass, marklist_length (librarylist));
+    }
+
+
+  lib = fopen (tempn, "wb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble creating prune-library file.", "");
+
+
+  if (V)
+    fprintf (stderr, "writing to file: %s...", prunename);
+  for (step = librarylist, count = 0; step; step = step->next, count++)
+    {
+      step->data.symnum = count;
+      write_library_mark (lib, step->data);
+    }
+
+  if (V)
+    fprintf (stderr, "%d marks.\n", count);
+  fclose (lib);
+
+  if (overwrite)
+    {
+      sprintf (sysstr, "mv %s %s", tempn, prunename);
+      system (sysstr);		/* move temp to proper name */
+    }
+
+  for (i = 0; i < avglibsize; i++)
+    {
+      marklist_free (&avglib[i]);
+      free (avglib[i]);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/mgticstat.c	(revision 21112)
@@ -0,0 +1,130 @@
+/**************************************************************************
+ *
+ * mgticstat.c -- Program to print out stats about a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticstat   libraryfile\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib, *outf = (stdout);
+  int count;
+  int small1, small2, small4, small9, small16, small25, small36, small49,
+    small64, small81, small100;
+  marktype d;
+  int avx, avy;
+  int hhh, www, i;
+  int avruns_h = 0, avruns_v = 0;
+  int av_area = 0;
+  char *libraryname = NULL;
+
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "r");
+  if (lib == NULL)
+    error_msg (args[0], "Trouble opening library file", "");
+
+  count = 0;
+  small1 = small2 = small4 = small9 = small16 = small25 = small36 = small49 = small64 = small81 = small100 = 0;
+  avx = avy = 0;
+  www = hhh = 0;
+  while (!isEOF (lib))
+    {
+      if (!read_library_mark (lib, &d))
+	error_msg (args[0], "error in library file", "");
+
+      av_area += d.set;
+      if (d.set <= 1)
+	small1++;
+      if (d.set <= 2)
+	small2++;
+      if (d.set <= 4)
+	small4++;
+      if (d.set <= 9)
+	small9++;
+      if (d.set <= 16)
+	small16++;
+      if (d.set <= 25)
+	small25++;
+      if (d.set <= 36)
+	small36++;
+      if (d.set <= 49)
+	small49++;
+      if (d.set <= 64)
+	small64++;
+      if (d.set <= 81)
+	small81++;
+      if (d.set <= 100)
+	small100++;
+      avx += d.xcen;
+      avy += d.ycen;
+      www += d.w;
+      hhh += d.h;
+      avruns_h += d.h_runs;
+      avruns_v += d.v_runs;
+      pbm_freearray (&d.bitmap, d.h);
+      count++;
+      if (count % 5000 == 0)
+	fprintf (outf, "%dk ", count / 1000);
+    }
+  fprintf (outf, "number of marks=%d\n", count);
+  fprintf (outf, "average area=%d\n", av_area / count);
+  fprintf (outf, "area <=1:%d, <=2:%d, <=4:%d, <=9:%d, <=16:%d, <=25:%d, <=36:%d, <=49:%d, <=64:%d, <=81:%d, <=100:%d\n", small1, small2, small4, small9, small16, small25, small36, small49, small64, small81, small100);
+
+  fprintf (outf, "average num runs, horizontal=%d, vertical=%d\n", avruns_h / count, avruns_v / count);
+  fprintf (outf, "average centroid position=(%d,%d)\n", avx / count, avy / count);
+  fprintf (outf, "average width=%d, average height=%d\n", www / count, hhh / count);
+  fclose (lib);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/model.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/model.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/model.h	(revision 21112)
@@ -0,0 +1,92 @@
+/**************************************************************************
+ *
+ * model.h -- Part pf PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#define defaultkbytes	896
+#define minkbytes	56
+#define maxkbytes	((65536/1024)*sizeof(eventnode))
+
+#ifndef increment
+#define	increment	8
+#endif
+#ifndef halveat
+#define halveat		64
+#endif
+#define maxtotalcnt 	(increment*halveat)
+
+typedef int boolean;
+#define false           0
+#define true            1
+
+typedef short event;
+typedef unsigned short point;
+
+typedef struct
+  {
+    short totalcnt;
+    short notfound;
+    point list;			/* list storing the event records */
+  }
+eventset;
+
+typedef struct
+  {
+    event eventnum;
+    short count;
+    point next, prev;
+    eventset foll;
+  }
+eventnode, *eventptr;
+
+#define memsize		(kbytes*1024)
+#define maxnodes	(memsize/sizeof(eventnode))
+/* #define charsetsize     256 -- MAHE */
+#define nchars          (charsetsize+1)
+
+extern int charsetsize;		/* -- MAHE */
+
+extern eventptr E;
+extern long kbytes;
+extern point numnodes, nfreenodes;
+extern boolean excluded[];	/* -- MAHE */
+
+#define ENULL		(E)
+
+typedef unsigned u;
+#define p32(p)	(E+p)
+#define p16(p)	(p-E)
+
+/* in ppm.c */
+void write_model ();
+void encodestring ();
+void decodestring ();
+void encodefile ();
+void decodefile ();
+
+/* in lstevent.c */
+void write_method ();
+eventnode *encode_event_noex ();
+eventnode *decode_event_noex ();
+eventnode *encode_event ();
+eventnode *decode_event ();
+/* eventnode *addevent(); */
+eventnode *newnode ();
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.c	(revision 21112)
@@ -0,0 +1,288 @@
+/**************************************************************************
+ *
+ * pbmtools.c -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+
+void 
+pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    pbm_putpixel (bitmap, c, r, val);
+  else
+    fprintf (stderr, "putpitxel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+}
+
+int 
+pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    return pbm_getpixel (bitmap, c, r);
+  else
+    {
+      fprintf (stderr, "getpixel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+      return 0;
+    }
+}
+
+
+
+
+
+/*
+   Returns 1 if *fp points to a PBM (magic is P1 or P4) file.
+   leaves the fp in it's original position.
+   usage: if(pbm_isapbmfile(fp)) process(fp);
+ */
+int 
+pbm_isapbmfile (FILE * fp)
+{
+  u_long magic;
+
+  magic = getmagicno_short (fp);
+  if ((magic == MAGIC_P4) || (magic == MAGIC_P1))
+    return 1;
+  else
+    return 0;
+}
+
+
+/*
+   Frees the memory used in the bitmap structure.
+   usage: pbm_freearray(&bitmap);
+   Sets bitmap to be NULL on return.
+ */
+void 
+pbm_freearray (Pixel *** bitmap, int rows)
+{
+  int r;
+  if (rows == 0)
+    rows++;
+  for (r = 0; r < rows; r++)
+    free ((*bitmap)[r]);
+  free (*bitmap);
+  (*bitmap) = NULL;
+}
+
+
+/*
+   This function allocates the bitmap.
+   usage: bitmap=pbm_allocarray(cols,rows);
+ */
+Pixel **
+pbm_allocarray (int cols, int rows)
+{
+  Pixel **p;
+  int i, newc = (cols + PBITS) / PBITS;
+
+  if (cols == 0)
+    cols++;
+  if (rows == 0)
+    rows++;
+  p = (Pixel **) malloc (rows * sizeof (Pixel *));
+  if (p == NULL)
+    error_msg ("pbm_allocarray", "out of memory", "");
+  for (i = 0; i < rows; i++)
+    {
+      p[i] = (Pixel *) calloc (newc, sizeof (Pixel));
+      if (p[i] == NULL)
+	error_msg ("pbm_allocarray", "out of memory", "");
+    }
+  return p;
+}
+
+
+
+Pixel **
+pbm_copy (Pixel ** bitmap, int cols, int rows)
+{
+  Pixel **p;
+  int newc = (cols + PBITS) / PBITS;
+  int x, y;
+
+  p = pbm_allocarray (cols, rows);
+  for (x = 0; x < newc; x++)
+    for (y = 0; y < rows; y++)
+      p[y][x] = bitmap[y][x];
+  return p;
+}
+
+
+
+
+/*
+   Reads in a pbm file and returns a pointer to the allocated bitmap, 
+   in the P4 format.
+   Assumes fp was opened and keeps it that way.
+   usage: bitmap=pbm_readfile(fp,&cols,&rows);
+ */
+Pixel **
+pbm_readfile (FILE * fp, int *cols, int *rows)
+{
+  Pixel **p = NULL;
+  char s[100];
+  unsigned char *buf;
+  int r, c, ii = 0;
+  int p1 = 0;
+  int arg;
+
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  if (strcmp (s, "P1") == 0)
+    p1 = 1;
+  else if (strcmp (s, "P4") == 0)
+    p1 = 0;
+  else
+    error_msg ("pbm_readfile", "only pbm files are supported", "");
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  arg = sscanf (s, "%d %d", cols, rows);
+  if (arg == 1)
+    {
+      readline (s, fp);
+      arg = sscanf (s, "%d", rows);
+      if (arg != 1)
+	error_msg ("pbm_readfile", "bizzare pbm format", "");
+    }
+  p = pbm_allocarray (*cols, *rows);
+  buf = (unsigned char *) malloc ((*cols + 2) * sizeof (unsigned char));
+  if (!buf)
+    error_msg ("pbm_readfile", "out of memory", "");
+
+  if (p1)
+    {
+      for (r = 0; r < *rows; r++)
+	for (c = 0; c < *cols; c++)
+	  {
+	    fscanf (fp, "%d", &ii);
+	    pbm_putpixel (p, c, r, ii);
+	  }
+    }
+  else
+    {
+      for (r = 0; r < *rows; r++)
+	{
+	  c = 0;
+	  fread (buf, 1, (*cols + 7) / 8, fp);
+	  for (c = 0; c < *cols; c++)
+	    {
+	      if (!(c % 8))
+		ii = buf[c / 8];
+	      if (ii < 0)
+		error_msg ("pbm_readfile", "file reading error", "");
+	      pbm_putpixel (p, c, r, (ii & 128) ? 1 : 0);
+	      ii <<= 1;
+	    }
+	}
+    }				/* p4 type */
+  free (buf);
+  return p;
+}
+
+
+/*
+   Writes a pbm file given a file pointer and bitmap. Assumes that
+   fp is already open, and keeps in that way.
+   usage: pbm_writefile(fp,bitmap,cols,rows);
+ */
+void 
+pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows)
+{
+  int r, c, ii = 0, j;
+  unsigned char *buf;
+
+  buf = (unsigned char *) malloc (((cols + 14) / 8) * sizeof (unsigned char));
+
+  fprintf (fp, "%s\n", "P4");
+  fprintf (fp, "%d %d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c += 8)
+	{
+	  ii = 0;
+	  for (j = 0; j < 8; j++)
+	    if (c + j < cols)
+	      {
+		if (pbm_getpixel (bitmap, c + j, r))
+		  ii |= (128 >> j);
+	      }
+	  buf[c / 8] = ii;
+	}
+      fwrite (buf, 1, (cols + 7) / 8, fp);
+    }
+  free (buf);
+/*      putc('\n',fp);   if this line is included programs don't say things like 'short image' &c, but then again,
+   in most cases a 'cmp' between the original will fail because of the last byte...sigh, "standards". */
+}
+
+#define BUF_SIZE 131072
+char buffer[BUF_SIZE];
+
+void 
+pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows)
+{
+  FILE *tempfp;
+
+  tempfp = fopen (fn, "wb");
+  if (tempfp == NULL)
+    error_msg ("pbm_writenamedfile", "can't open file:", fn);
+  setbuffer (tempfp, buffer, BUF_SIZE);
+  pbm_writefile (tempfp, bitmap, cols, rows);
+  fclose (tempfp);
+}
+
+
+
+Pixel **
+pbm_readnamedfile (char fn[], int *cols, int *rows)
+{
+  FILE *tempfp;
+  Pixel **temp;
+
+  tempfp = fopen (fn, "rb");
+  if (tempfp == NULL)
+    return NULL;
+  setbuffer (tempfp, buffer, BUF_SIZE);
+
+  temp = pbm_readfile (tempfp, cols, rows);
+
+  fclose (tempfp);
+  return temp;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/pbmtools.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * pbmtools.h -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __PBMTOOLS_H
+#define __PBMTOOLS_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+#include "utils.h"
+#include "marklist.h"
+
+#define PBITS 32
+#define PSH 5
+
+
+/* The next two are ordered pixels col0=left, cols31=right */
+
+/*
+   #define pbm_getpixel(bitmap,c,r)  \
+   ((bitmap[r][(c)>>PSH] >> ((PBITS-1)-((c)&(PBITS-1))))&1)
+   #define pbm_putpixel(bitmap,c,r,val) \
+   if((val))  \
+   bitmap[r][(c)>>PSH] |=  (((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))); \
+   else \
+   bitmap[r][(c)>>PSH] &= ~(((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))) 
+ */
+
+
+
+
+/* The next two defines are ordered pixel cols0=right, cols31=left */
+
+#define pbm_getpixel(bitmap,c,r)                            \
+         ((bitmap[r][(c)>>PSH] >> ((c)&(PBITS-1)))&1)
+#define pbm_putpixel(bitmap,c,r,val)                        \
+         if((val))                                          \
+              bitmap[r][(c)>>PSH] |= (1<<((c)&(PBITS-1)));  \
+         else                                               \
+              bitmap[r][(c)>>PSH] &= ~(1<<((c)&(PBITS-1)))
+
+
+#define pbm_putpixel_trunc(bitmap,c,r,val,cols,rows) \
+       if(((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) \
+             pbm_putpixel(bitmap,c,r,val)
+
+#define pbm_getpixel_trunc(bitmap,c,r,cols,rows)       \
+       ((((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) ?  \
+             pbm_getpixel(bitmap,c,r) : 0)
+
+
+
+
+int pbm_isapbmfile (FILE * fp);
+void pbm_freearray (Pixel *** bitmap, int rows);
+Pixel **pbm_copy (Pixel ** bitmap, int cols, int rows);
+Pixel **pbm_allocarray (int cols, int rows);
+Pixel **pbm_readfile (FILE * fp, int *cols, int *rows);
+Pixel **pbm_readnamedfile (char fn[], int *cols, int *rows);
+void pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows);
+void pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows);
+
+void pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows);
+int pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/ppm.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/ppm.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/ppm.c	(revision 21112)
@@ -0,0 +1,588 @@
+/**************************************************************************
+ *
+ * ppm.c -- PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:16:56  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:33  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:55  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:02  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* FIXEDORDER sets the maximum context length that will be used.
+   Making FIXEDORDER large increases the compression if there is enough
+   space to store the model, but when space limit is reached increasing
+   FIXEDORDER will decrease compression because of model thrashing.
+   Third order (FIXEDORDER=3) is suitable for a few hundred Kb of model
+   space, FIXEDORDER=1 is suitable for a few tens of kB.
+   This version requires that FIXEDORDER be a preprocessor rather
+   than runtime variable to allow loops to be unrolled and make for faster
+   execution. Maximum in this implementation is 3 */
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+/* global variables */
+eventptr E = NULL;		/* array of eventnodes */
+point numnodes = 0, nfreenodes = 0;
+long kbytes = defaultkbytes;
+/*boolean               excluded[nchars]={false};  - MAHE */
+boolean excluded[16384] =
+{false};			/* excluded characters */
+
+/* local variables */
+#define defaultchunksize        4096	/* compression chunks */
+
+#define EOSchar         (charsetsize)
+
+unsigned chunksize = defaultchunksize;
+eventptr s[FIXEDORDER + 1] =
+{NULL},				/* current contexts */
+  t[FIXEDORDER + 1] =
+{NULL};				/* new contexts */
+
+#if (FIXEDORDER==3)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1] ; s[3] = t[2]
+#else
+#if (FIXEDORDER==2)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1]
+#else
+#if (FIXEDORDER==1)
+#define SHIFT_ST	s[1] = t[0]
+#else
+#define SHIFT_ST
+#endif
+#endif
+#endif
+
+/* follow a character list and set the exclusions */
+#define SET_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = true;	\
+}						\
+while (false)
+
+/* clear the exclusions */
+#define CLEAR_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = false;	\
+}						\
+while (false)
+
+/*==================================*/
+
+void
+write_model ()
+{
+  fprintf (stderr, "ppm");
+  write_method ();
+#ifdef NOEXCLUSIONS
+  fprintf (stderr, "nx");
+#endif
+  fprintf (stderr, "-o%1d", FIXEDORDER);
+  fprintf (stderr, "-k%ld", kbytes);
+}
+
+/*==================================*/
+
+void
+encode_in_order (n, e)
+     int n;
+     event e;
+/* encode character e in order n */
+{
+  boolean encoded;
+  int i, c;
+  t[n] = encode_event_noex (&s[n]->foll, e, &encoded);
+  if (encoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = encode_event_noex (&s[c]->foll, e, &encoded);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = encode_event (&s[c]->foll, e, &encoded);
+#endif
+	if (encoded)
+	  break;
+      }
+  if (!encoded)
+    arithmetic_encode (e, e + 1, nchars);
+  for (i = c + 1; i <= n; i++)
+    if (i > 0)
+      t[i]->prev = p16 (t[i - 1]);
+    else
+      t[0]->prev = p16 (s[0]);
+  for (i = c - 1; i >= 0; i--)
+    t[i] = p32 (t[i + 1]->prev);
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+/*==================================*/
+
+/* not required in this version */
+
+#if 0
+
+void
+decode_in_order (n, e)
+     int n;
+     event *e;
+{
+  boolean decoded;
+  int i, c;
+  t[n] = decode_event_noex (&s[n]->foll, &decoded, e);
+  if (decoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = decode_event_noex (&s[c]->foll, &decoded, e);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = decode_event (&s[c]->foll, &decoded, e);
+#endif
+	if (decoded)
+	  break;
+      }
+  if (!decoded)
+    {
+      *e = arithmetic_decode_target (nchars);
+      arithmetic_decode (*e, *e + 1, nchars);
+    }
+  for (i = c + 1; i <= n; i++)
+    {
+      t[i]->eventnum = *e;
+      if (i > 0)
+	t[i]->prev = p16 (t[i - 1]);
+      else
+	t[0]->prev = p16 (s[0]);
+    }
+  {
+    register int i;
+    for (i = c - 1; i >= 0; i--)
+      t[i] = p32 (t[i + 1]->prev);
+  }
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+#endif
+
+/*==================================*/
+
+void
+start_model ()
+{
+  int i;
+  /* is this the first call? */
+  if (E == NULL)
+    E = (eventptr) malloc ((unsigned) kbytes * 1024);
+  /* start off with no information */
+  numnodes = 0;
+  nfreenodes = maxnodes;
+  for (i = 0; i <= FIXEDORDER; i++)
+    {
+      s[i] = newnode (NULL);
+      if (i > 0)
+	s[i]->prev = p16 (s[i - 1]);
+    }
+}
+
+/*==================================*/
+
+void
+encodestring (str, n)
+     unsigned short *str;
+     int n;
+/* encode all n characters of the supplied string str */
+{
+  event e;
+  boolean encoded;
+  register eventptr S, T;
+  int i;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  if (E == NULL)
+    start_model ();
+  /* set the initial context node */
+  S = s[FIXEDORDER];
+
+  for (i = 0; i < n; i++)
+    {
+      /* encode one character */
+      e = *str++;
+      /* this next monstrosity is logically equivalent to 
+         encode_in_order(FIXEDORDER,e)
+         the loop has been unrolled to make it all go faster */
+      for (;;)
+	{
+	  T = encode_event_noex (&S->foll, e, &encoded);
+	  if (encoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = encode_event_noex (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = encode_event_noex (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = encode_event_noex (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    break;
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = encode_event (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = encode_event (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = encode_event (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  arithmetic_encode (e, e + 1, nchars);
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      /* phew! Update the current context to the latest node */
+      S = p32 (T->prev);
+
+      /* is there a danger of running out of space? */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+    }
+
+  /* save the current contexts for the various order submodels */
+  s[FIXEDORDER] = S;
+  for (i = FIXEDORDER; i > 0; i--)
+    s[i - 1] = p32 (s[i]->prev);
+  /* and send the terminator character */
+  encode_in_order (FIXEDORDER, EOSchar);
+  /* and check the space again */
+  if (nfreenodes <= FIXEDORDER)
+    start_model ();
+}
+
+/*==================================*/
+
+void
+decodestring (str, n)
+     unsigned short *str;
+     int *n;
+{
+  event e;
+  boolean decoded;
+  register eventptr S, T;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  /* just mimic the corresponding encoding actions */
+  if (E == NULL)
+    start_model ();
+  S = s[FIXEDORDER];
+
+  *n = 0;
+  for (;;)
+    {				/* logically equivalent to "decode_in_order(FIXEDORDER,&e)" */
+
+      for (;;)
+	{
+	  T = decode_event_noex (&S->foll, &decoded, &e);
+	  if (decoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      break;
+	    }
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = decode_event (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = decode_event (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = decode_event (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  e = arithmetic_decode_target (nchars);
+	  arithmetic_decode (e, e + 1, nchars);
+#if (FIXEDORDER>2)
+	  t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+	    t[2]->eventnum =
+#endif
+#if (FIXEDORDER>0)
+	    t[1]->eventnum =
+#endif
+	    t[0]->eventnum = e;
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      S = p32 (T->prev);
+
+      /* check the space requirements */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+      /* are we done?  or do we save the char */
+      if (e == EOSchar)
+	break;
+      *str++ = e;
+      (*n)++;
+    }
+  s[FIXEDORDER] = S;
+}
+
+
+#if 0				/* Writing my own version of this - MAHE */
+void
+encodefile ()
+/* encodes a file by breaking into a sequence of fixed length chunks */
+{
+  unsigned char *str;
+  int c, n = 0;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startoutputingbits ();
+  startencoding ();
+
+  /* to encode the file, break it into chunks and encode the
+     chunks one by one */
+  while ((c = getchar ()) != EOF)
+    {
+      str[n++] = c;
+      if (n == chunksize)
+	{
+	  encodestring (str, n);
+	  rawbytes += n;
+	  n = 0;
+	}
+    }
+  /* send final partial chunk */
+  if (n > 0)
+    {
+      encodestring (str, n);
+      rawbytes += n;
+    }
+  /* send EOF as an empty string */
+  encodestring (str, 0);
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void
+decodefile ()
+{
+  unsigned char *str;
+  int n, i;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startinputingbits ();
+  startdecoding ();
+  for (;;)
+    {
+      decodestring (str, &n);
+      if (n == 0)
+	break;
+      for (i = 0; i < n; i++)
+	putchar ((int) str[i]);
+      rawbytes += n;
+    }
+}
+
+#endif /* #if 0 */
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.c	(revision 21112)
@@ -0,0 +1,93 @@
+/**************************************************************************
+ *
+ * sortmarks.c -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "sortmarks.h"
+
+#define GAP 4
+
+
+
+static int 
+CmpOnY (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->ycen * 0 + ((marktype *) e1)->ypos) -
+    (((marktype *) e2)->ycen * 0 + ((marktype *) e2)->ypos);
+}
+
+static int 
+CmpOnX (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->xcen * 0 + ((marktype *) e1)->xpos) -
+    (((marktype *) e2)->xcen * 0 + ((marktype *) e2)->xpos);
+}
+
+
+
+
+
+
+
+
+marklistptr 
+sortmarks (marklistptr listofmarks)
+{
+  marktype *table;
+  int count = 0;
+  int i, s, e;
+
+
+  table = calloc ((unsigned int) marklist_length (listofmarks), sizeof (marklisttype));
+
+  while (listofmarks)
+    {
+      table[count] = listofmarks->data;
+      marklist_removeat (&listofmarks, 0);
+      count++;
+    }
+
+  /* sort first on the y position of their centroids */
+  qsort ((void *) table, (unsigned int) (count), sizeof (*table), CmpOnY);
+
+  for (s = 0; s < count;)
+    {
+      int c;
+
+      for (e = s + 1, c = 0; (e < count) &&
+	   ((table[e].ypos + table[e].ycen * 0) - (table[e - 1].ypos + table[e - 1].ycen * 0) <= GAP); c++, e++);
+
+      if (e < count && c >= 1)
+	qsort (&table[s], (unsigned int) (e - s), sizeof (*table), CmpOnX);
+      s = e;
+    }
+
+  for (i = 0; i < count; i++)
+    {
+      table[i].symnum = i;
+      marklist_add (&listofmarks, table[i]);
+    }
+
+  return listofmarks;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/sortmarks.h	(revision 21112)
@@ -0,0 +1,30 @@
+/**************************************************************************
+ *
+ * sortmarks.h -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _SortMarks_h_
+#define _SortMarks_h_
+
+marklistptr sortmarks (marklistptr listofmarks);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.c	(revision 21112)
@@ -0,0 +1,270 @@
+/**************************************************************************
+ *
+ * utils.c -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "utils.h"
+
+
+int V = 0;
+
+
+int 
+isEOF (FILE * fp)
+{
+  int ch;
+
+  ch = getc (fp);
+  ungetc (ch, fp);
+  if (ch == EOF)
+    return 1;
+  return 0;
+}
+
+
+int 
+getmagicno_short (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 8) | ch2);
+}
+
+int 
+getmagicno_short_pop (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    return 0;
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    return 0;
+
+  return ((ch1 << 8) | ch2);  
+}
+
+int 
+getmagicno_byte (FILE * fp)
+{
+  int ch1;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ungetc (ch1, fp);
+  return (ch1);
+}
+
+
+int 
+getmagicno_long (FILE * fp)
+{
+  int ch1, ch2, ch3, ch4;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ch3 = getc (fp);
+  if (ch3 == EOF)
+    {
+      ungetc (ch3, fp);
+      return 0;
+    }
+  ch4 = getc (fp);
+  if (ch4 == EOF)
+    {
+      ungetc (ch4, fp);
+      return 0;
+    }
+  ungetc (ch4, fp);
+  ungetc (ch3, fp);
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 24) | (ch2 << 16) | (ch3 << 8) | (ch4));
+}
+
+
+
+
+void 
+error_msg (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+  exit (1);
+}
+
+void 
+warn (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+}
+
+
+
+void 
+readline (char str[], FILE * fp)
+{
+  int i = 0, ch;
+
+  while (((ch = fgetc (fp)) != '\n') && (!feof (fp)))
+    str[i++] = ch;
+  str[i] = '\0';
+}
+
+
+
+
+
+
+
+int 
+getint (FILE * fp)
+{
+  register char ch;
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+/* get_header_int i.e. get an integer from a PNM header ; basically  
+   the same as
+   above except for comment checking ; above kept because of lower  
+   overhead */
+
+unsigned int 
+gethint (FILE * fp)
+{
+  register char ch, prev = '\0';
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if ((ch == '#') && (prev == '\n'))
+	while (((ch = getc (fp)) != '\n') && !(feof (fp)));
+      prev = ch;
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+
+
+
+void 
+magic_write (FILE * fp, u_long magic_num)
+{
+  HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fwrite (&magic_num, sizeof (magic_num), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't write magic number.", "");
+}
+
+
+void 
+magic_check (FILE * fp, u_long magic_num)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1 || NTOHUL(magic) != magic_num)  /* [RPAP - Jan 97: Endian Ordering] */
+    error_msg ("magic num", "Incorrect magic number.", "");
+}
+
+
+u_long 
+magic_read (FILE * fp)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't read magic number.", "");
+  return NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+isinteger (char s[])
+{
+  int i = 0;
+
+  for (i = 0; s[i] != '\0'; i++)
+    if (!isdigit (s[i]))
+      return 0;
+  return 1;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/images/utils.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * utils.h -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __UTILS_H
+#define __UTILS_H
+
+#include "sysfuncs.h"
+
+extern int V;
+
+#define DEBUG
+
+#define min(a,b) ((a)<(b) ? (a) : (b))
+#define max(a,b) ((a)>(b) ? (a) : (b))
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+                                            ((c)<<8) + (d)))
+ 
+/* --- MG magic file codes [TS:Aug/95] --- */
+#define MAGIC_FELICS            GEN_MAGIC('M','G','F','X')
+#define MAGIC_BILEVEL           GEN_MAGIC('M','G','B','I')
+#define MAGIC_TIC               GEN_MAGIC('M','G','T','C')
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+/* --------------------------------------- */
+
+#define MAGIC_P1      0x5031	/* P1   - pbm */
+#define MAGIC_P2      0x5032	/* P2   - pgm */
+#define MAGIC_P3      0x5033	/* P3   - ppm */
+#define MAGIC_P4      0x5034	/* P4   - rawbits pbm */
+#define MAGIC_P5      0x5035	/* P5   - rawbits pgm */
+#define MAGIC_P6      0x5036	/* P6   - rawbits ppm */
+
+int isEOF (FILE * fp);
+int getmagicno_byte (FILE * fp);
+int getmagicno_short (FILE * fp);
+int getmagicno_short_pop (FILE * fp);
+int getmagicno_long (FILE * fp);
+
+
+
+void error_msg (char *prog, char *message, char *extra);
+void warn (char *prog, char *message, char *extra);
+
+void readline (char str[], FILE * fp);
+
+void magic_write (FILE * fp, u_long magic_num);
+void magic_check (FILE * fp, u_long magic_num);
+u_long magic_read (FILE * fp);
+/* kerry's code */
+
+int getint (FILE * fp);
+unsigned int gethint (FILE * fp);
+int isinteger (char s[]);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/Makefile.in	(revision 21112)
@@ -0,0 +1,152 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/scripts
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+GSDLOS = @gsdlos@
+ 
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+SHFILES       =	mgbuild.sh mgbuild_4.sh mg_get.sh xmg.sh  mg_get_merge.sh mgmerge.sh
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SHFILES) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgbuild mg_get xmg mg_get_merge mgmerge mgbuild_4
+
+MAN           = mgbuild.1 mgbuild_4.1 mg_get.1 xmg.1 mg_get_merge.1 mgmerge.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+mgbuild : mgbuild.sh
+	@rm -f $@
+	cat mgbuild.sh >$@
+	chmod +x $@
+
+mgbuild_4 : mgbuild_4.sh
+	@rm -f $@
+	cat mgbuild_4.sh >$@
+	chmod +x $@
+
+mg_get : mg_get.sh
+	@rm -f $@
+	cat mg_get.sh >$@
+	chmod +x $@
+
+mgmerge : mgmerge.sh
+	@rm -f $@
+	cat mgmerge.sh >$@
+	chmod +x $@
+
+mg_get_merge : mg_get_merge.sh
+	@rm -f $@
+	cat mg_get_merge.sh >$@
+	chmod +x $@
+
+xmg : xmg.sh
+	@rm -f $@
+	cat xmg.sh >$@
+	chmod +x $@
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+mostlyclean:
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get.sh	(revision 21112)
@@ -0,0 +1,447 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell
+# Changed to allow it to be driven from a file ~/.mg_getrc
+# by Bruce McKenzie, Oct 1994
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mg_get.sh	1.11 25 Mar 1994
+#
+###########################################################################
+#
+# "mg_get" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+# The file ~/.mg_get contains lines of the form:
+#     name	TYPE	files and directories
+# where name is the name supplied to mg_get, TYPE is one of
+# PARA, MAIL, DIR, DIR2, BIB or TXTIMG and says how mg_get should process the
+# named files and directories
+#
+###########################################################################
+
+if ($?MG_GETRC) then
+  set mg_getrc = $MG_GETRC
+else
+  set mg_getrc = ~/.mg_getrc
+endif
+
+# Added by [TS:May/95]
+# if we don't have an mg_getrc file in existence then create a default one
+if !(-e $mg_getrc) then 
+  echo "Can not find a .mg_getrc, so creating a default one: $mg_getrc"
+
+  if ($?MGSAMPLE) then
+    set SampleData = $MGSAMPLE
+  else
+    set SampleData = ./SampleData
+  endif
+
+  cat > $mg_getrc << END
+alice	PARA	$SampleData/alice13a.txt.Z
+davinci	TXTIMG	$SampleData/davinci
+mailfiles	MAIL	~/mbox ~/.sentmail
+allfiles	DIR	~/Mail
+mods	LINESEP	$SampleData/MODIFICATIONS
+bible	PLAIN	$SampleData/bible/genesis.txt.Z $SampleData/bible/revelation.txt.Z
+END
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: mg_get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set COLLECTION = $1
+set TYPE  = `grep \^$1 $mg_getrc | cut -f2`
+set FILES = `grep \^$1 $mg_getrc | cut -f3`
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($TYPE)
+  case PLAIN:
+# Just outputs the given text, assumes ^Bs already inserted 
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case LINESEP:
+# Assumes that the documents are separated by a line of =====
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f |  sed -e '/\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/s/.*//' < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case PARA:
+# Splits the file at blank lines, which corresponds to paragraphs.
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '   /^ *$/ {if (b!=1) printf "";b=1} \
+		\!/^ *$/ {print;b=0} \
+		 END {printf ""}'
+       end #foreach
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case BIB:
+# Takes a list of files that contain bibliographies, and splits them up 
+#   by putting ^B after each reference. Assumes that each reference
+#   begins with a line '^@'.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '/^@/&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #BIB
+
+  case MAIL:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f| awk '/xbtoa Begin/,/xbtoa End/ {next} /^From /&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #MAIL
+
+  case DIR:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz' -exec echo '<' {} '>' \;		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' -exec echo '<' {} '>' \;		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)' -exec echo '<' {} \
+#			'>' \; -exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec echo '<' {} '>' \; \
+			-exec cat {} \; -exec echo -n '' \;
+
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR
+
+
+
+  case DIR2:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory. Does not include filename
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz'  		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' 	 		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)'	\
+#			-exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR2
+
+
+
+
+  case TXTIMG:
+# compress and index the collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# Files that are related have the same prefix. For example,
+#  monaLisa.pgm might be a gray-level image, and
+#  monaLisa.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt	ascii text
+#    .ptm	scanned text stored as a bilevel image
+#    .pbm	a black and white image (typically a line drawing)
+#    .pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $FILES
+  set targetDir = $MGDATA/$COLLECTION
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   # create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null     # in case it already existed
+    
+      # take care if no match to foreach statements
+      set nonomatch
+
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+        if ($f == "$sourceDir/*.ptm") then
+                break
+        endif
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+        if ($f == "$sourceDir/*.pgm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+        if ($f == "$sourceDir/*.pbm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+      unset nonomatch
+    breaksw #-init
+
+    case '-text'
+      # take care if no match to foreach statements
+      set nonomatch
+
+      #output each text file and tmp.txt, with names of other associated files
+
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+        if ($f == "$sourceDir/*.txt") then
+           continue
+        endif
+        if ($f == "$sourceDir/*.tmp") then
+           continue
+        endif
+	set r = $f:r	#root name of file
+	set base = $r:t
+        set first_image = 1 # set so that I print out image header once
+                            # also signifies if header printed out at all
+
+	foreach d ($targetDir/$base.*)
+          if ($d == "$targetDir/$base.*") then
+                break
+          endif
+
+          if ($first_image) then
+            set first_image = 0
+            echo '::::::::::'  #separate the image information from the text
+            echo 'Image(s) available:'
+          endif
+
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+
+	end #foreach d (associated file)
+
+        if (! $first_image) then
+	  echo '::::::::::'  #separate the image information from the text
+        endif
+
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+      unset nonomatch
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #IMGTXT
+
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mg_get_merge.sh	(revision 21112)
@@ -0,0 +1,313 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get_merge.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# "mg_get_merge" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+###########################################################################
+
+
+
+# The directory containing the data files is stored in the environment
+#   variable MGSAMPLE
+
+if ($?MGSAMPLE) then
+  set SampleData = $MGSAMPLE
+else
+  set SampleData = ./SampleData
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($1)
+  case alice:
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+	echo "This is a test document 1 2 3 4 5 6 7 8 9 "
+	echo ""
+	echo "A B C"
+     breaksw
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case bible:
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+       # use uncompressed file if it exists
+       # revelation.txt already contains '^B' characters between verses
+       if (-e $SampleData/bible/revelation.txt) then
+          cat $SampleData/bible/revelation.txt
+       else
+          uncompress <$SampleData/bible/revelation.txt.Z
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bible
+
+
+
+  case mailfiles:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+# If messages are already in individual files, use "get allfiles"
+#   rather than this one.
+# The EXTRA documents added are in files in the directory mailextra
+#### The variable 'mailfiles' contains a list of files to be split up
+  set mailfiles = '~/mbox ~/.sentmail'
+  set mailextra = '~/extra/*'
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      foreach f ($mailextra)
+	awk '/^From /&&NR!=1{print ""} {print $0} END{print ""}' $f
+      end # foreach
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #mailfiles
+
+
+
+  case allfiles:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+# The variable 'directory' on the next line contains the directory to be used
+  set directory = ~/Mail
+
+# The date of the file ".mgbuildtime" in 'directory' is the last
+# build or merge. At the start of the merge ("-init" argument) the
+# file ".mgmergetime" is touched. Only files newer than ".mgbuildtime"
+# but older than ".mgmergetime" are output by mg_get_merge
+
+  switch ($flag)
+    case '-init':
+        touch $directory/.mgmergetime
+    breaksw
+
+    case '-text'
+      find $directory -type f -newer $directory/.mgbuildtime \! -newer $directory/.mgmergetime \! -name '.*' -exec echo '<' {} '>' \; -exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+        touch $directory/.mgmergetime
+        touch $directory/.mgbuildtime
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #allfiles
+
+
+
+
+  case davinci:
+# compress and index the DaVinci collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# It assumes that all files are in one directory ($sourceDir).
+# Files that are related have the same prefix. For example,
+#  monaLisa.img.pgm might be a gray-level image, and
+#  monaLisa.txt.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt.txt	ascii text
+#    .txt.pbm	scanned text stored as a bilevel image
+#    .img.pbm	a black and white image (typically a line drawing)
+#    .img.pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $SampleData/davinci
+  set targetDir = $MGDATA/davinci
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   #create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null #in case it already existed
+    
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+    breaksw #-init
+
+    case '-text'
+      #output each text file and tmp.txt, with names of other associated files
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+	set r = $f:r	#root name of file
+	set base = $r:t
+	set n = `/bin/ls $sourceDir/$base.* | wc -l`
+	if ($n != 1) then # more files than just the text one
+	 echo '::::::::::'  #separate the image information from the text
+	 echo 'Image(s) available:'
+	 foreach d ($targetDir/$base.*)
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+	 end #foreach d (associated file)
+	echo '::::::::::'  #separate the image information from the text
+	endif # if more than one file
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #davinci
+
+
+#---------------------------
+  case mods:
+# Splits the modifications file at the mod separator "--------"
+  switch ($flag)
+     case '-init':
+     breaksw
+ 
+     case '-text':
+       sed -e 's/^---.*//g' < ${SampleData}/MODIFICATIONS
+     breaksw
+ 
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw #mods
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild.sh	(revision 21112)
@@ -0,0 +1,339 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+###########################################################################
+#  mg.build
+# Creates a Full Text Retrieval database
+#
+#  @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+
+set complex = ""
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-d") then
+    shift
+    if ($#argv >= 1) then
+      set mgdata = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-d mgdata dir] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get
+endif
+
+if ($?mgdata) then
+  setenv MGDATA $mgdata
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# If do_text == 1 then do the text component of building a mg database. 
+# Otherwise don't do the text component.
+set do_text = 1
+
+
+# If do_invf == 1 then do the inversion component of building a mg database. 
+# Otherwise don't do the inversion component.
+set do_invf = 1
+
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+set do_pass2 = 1
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+#### Note: Set -S so novel words can be encoded, as this will happen if
+#### mgmerge is used.
+set mcd = -S
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+
+#build up the command lines for pass 1 and 2
+set pass1 = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${bname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+if ($do_text) then
+  set pass1 = (${pass1} -T1)
+  set pass2 = (${pass2} -T2)
+endif
+
+if ($do_invf) then
+  set pass1 = (${pass1} -I1)
+  set pass2 = (${pass2} -I2)
+endif
+  
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1}"
+      $get $text $complex| $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1} ${input_files}
+    $bindir/mg_passes ${pass1} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2}"
+      $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2}"
+      $get $text $complex | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2} ${input_files}
+    $bindir/mg_passes ${pass2} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+echo "mg_weights_build -f ${bname} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+echo "mg_invf_dict -f ${bname} -b 4096"
+$bindir/mg_invf_dict -f ${bname} -b 4096
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+# [RPAP - Jan 97: Stem Index Change]
+if ("$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+echo "-- The fast-loading compression dictionary has not been built."
+echo "-- If you wish to build it, execute the following command:"
+echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+echo "-----------------------------------"
+
+echo ""
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgbuild_4.sh	(revision 21112)
@@ -0,0 +1,435 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild_4.sh -- Script used to build mg text collection in 4 passes.
+# Copyright (C) 1994  Neil Sharman; Modified by tes@kbs 1995
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# Does an mgbuild in 4 passes instead of 2 passes.
+# This is useful if one does not have as much RAM and a large collection.
+#
+
+set complex = ""
+
+# ************ parse the command line arguments ************
+
+while ($#argv >= 1)
+  if ("$1" == "-s" || "$1" == "--source") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g" || "$1" == "--get") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c" || "$1" == "--complex") then
+    set complex = "-text"
+    shift
+  else
+    if (! $?text ) then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if (! $?text ) then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = `which $0` # find which one we are executing
+set bindir = $bindir:h # use it as the bindir path
+
+# ************ set up shell variables ************
+# Override these by providing a script to source using the -s option
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+set get_options = ""
+
+
+if (! $?get ) then
+  set complex = "-text"
+  set get = $bindir/mg_get $get_options
+endif
+
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# whether to build weights or not
+set do_weights = 1
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+# Text 1st pass
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+# Text 2nd pass
+set do_pass2 = 1
+
+# If do_pass3 == 1 then do pass3 of building a mg database. 
+# Otherwise don't do pass3.
+# Invf 1st pass
+set do_pass3 = 1
+
+# If do_pass4 == 1 then do pass4 of building a mg database. 
+# Otherwise don't do pass4.
+# Invf 2nd pass
+set do_pass4 = 1
+
+# Buffer size used by mg_passes
+# Should be big enough to hold largest document
+# In Kilobytes
+set buf_size = 3072
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 5
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -C
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+# ************ set up pass parameters ************
+
+if ($do_pass1) then
+  set pass1_text = (-f ${bname} -T1)
+endif
+
+if ($do_pass2) then
+  set pass2_text = (-f ${bname} -T2)
+endif
+
+if ($do_pass3) then
+  set pass3_invf = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+  set pass3_invf = (${pass3_invf} -N1)
+endif
+
+if ($do_pass4) then
+  set pass4_invf = (-f ${bname} -${invf_level} -N2)
+endif
+
+if ($strip_sgml) then
+  set pass1_text = (${pass1_text} -G)
+  set pass2_text = (${pass2_text} -G)
+  set pass3_invf = (${pass3_invf} -G)
+  set pass4_invf = (${pass4_invf} -G)
+endif
+
+if ($?trace) then
+  set pass1_text = (${pass1_text} -t ${trace})
+  set pass2_text = (${pass2_text} -t ${trace})
+  set pass3_invf = (${pass3_invf} -t ${trace})
+  set pass4_invf = (${pass4_invf} -t ${trace})
+endif
+
+if ($?trace_name) then
+  set pass1_text = (${pass1_text} -n ${trace_name})
+  set pass2_text = (${pass2_text} -n ${trace_name})
+  set pass3_invf = (${pass3_invf} -n ${trace_name})
+  set pass4_invf = (${pass4_invf} -n ${trace_name})
+endif
+
+if ($buf_size) then
+  set pass1_text = (${pass1_text} -b ${buf_size})
+  set pass2_text = (${pass2_text} -b ${buf_size})
+  set pass3_invf = (${pass3_invf} -b ${buf_size})
+  set pass4_invf = (${pass4_invf} -b ${buf_size})
+endif
+  
+if ($?comp_stats) then
+  set pass2_text = (${pass2_text} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+
+# ************ init get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+# ************************************* T E X T **********************************
+
+# ************ pass 1_text ************
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1_text}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1_text}"
+      $get $text $complex| $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1_text} ${input_files}
+    $bindir/mg_passes ${pass1_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ compression dict  ************
+if ($do_pass2) then 
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ************ pass 2_text ************
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2_text}"
+      $get $text $complex| $pass2filter | $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2_text}"
+      $get $text $complex| $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2_text} ${input_files}
+    $bindir/mg_passes ${pass2_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************************************* I N V F **********************************
+
+# ************ pass 3_invf ************
+if (${do_pass3}) then
+  if ($pipe) then
+    if ($?pass3filter) then
+      echo "$get $text $complex | $pass3filter | mg_passes ${pass3_invf}"
+      $get $text $complex| $pass3filter | $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass3_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass3_invf} ${input_files}
+    $bindir/mg_passes ${pass3_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ perfect hash  ************
+if ($do_pass4) then
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ pass 4_invf ************
+if (${do_pass4}) then
+  if ($pipe) then
+    if ($?pass4filter) then
+      echo "$get $text $complex | $pass4filter | mg_passes ${pass4_invf}"
+      $get $text $complex| $pass4filter | $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass4_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass4_invf} ${input_files}
+    $bindir/mg_passes ${pass4_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+
+# ************ build invf dictionary ************
+
+if ($do_pass4) then
+  echo "mg_invf_dict -f ${bname} -b 4096"
+  $bindir/mg_invf_dict -f ${bname} -b 4096
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ******** build indexes to stem dict *********
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_pass4 && "$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ build weights ************
+
+if ($do_pass4 && $do_weights) then
+  echo "mg_weights_build -f ${bname} -b ${weight_bits} "
+  $bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# *******************************************************************************
+
+
+# ************ print out statistics ************
+if ($do_pass1 && $do_pass2 && $do_pass3 && $do_pass4) then
+  echo "mgstat -f ${bname} -E"
+  $bindir/mgstat -f ${bname} -E
+  if ("$status" != "0") exit 1 
+endif
+
+
+# ************ cleanup get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+if ($do_pass2) then
+  echo "-- The fast-loading compression dictionary has not been built."
+  echo "-- If you wish to build it, execute the following command:"
+  echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+  echo "-----------------------------------"
+endif
+
+echo ""
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/mgmerge.sh	(revision 21112)
@@ -0,0 +1,417 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgmerge.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+
+set complex = ""
+set slow_merge = ""
+set guess_weights = "-w "
+# [RPAP - Feb 97: Level 3 Merge]
+set text_pass = 1
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else if ("$1" == "-S") then
+# set slow merge option in mg_invf_merge
+    set slow_merge = "-s "
+    shift
+  else if ("$1" == "-w") then
+# turn weights guessing off (rebuild weights file from scratch)
+    set guess_weights = ""
+    shift
+
+# [RPAP - Feb 97: Level 3 Merge]
+  else if ("$1" == "-T") then
+# turn text pass off
+    set text_pass = 0
+    shift
+
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get_merge
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the inversion method this may be either "I" or "N"
+# N is memory-efficient, I is faster for small collections
+set invf_method = N
+
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+# This only has an effect if $invf_method is "N".
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+# This only has an effect if $invf_method is "N".
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+# Note: The value *MUST* be the same as was used to build the collection
+# that is being merged.
+# In the current mg_invf_merge implementation the maximum level is 2. 
+# Paragraph-level inversion (invf_level = 3) is NOT supported.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -S
+
+# $merge_name is the name of the subdirectory under ${MGDATA} where
+# the mergeing steps will be performed. There should NOT be an mg
+# collection with this name! The default is "MERGE"
+set merge = "MERGE"
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ($do_indexes) then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+
+# Generate the directory where merging will take place
+if (-e $MGDATA/${merge}) then
+else
+  mkdir $MGDATA/${merge}
+endif
+
+# Generate the base name for the collection.
+# [RPAP - Feb 97: Level 3 Merge]
+set bname = ${coll_name}
+
+# generate the old, new and merge names in the $merge directory
+set oldname = ${merge}/${coll_name}.old
+set newname = ${merge}/${coll_name}.new
+set mergename = ${merge}/${coll_name}
+
+# build up the command lines for pass 1 and 2 to build a
+# dummy collection for the new documents
+set pass1 = (-f ${newname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${newname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+# Note that a -T1 pass isnt done since the old .text.stats file is used
+# as the compression model for compressing the new documents.
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Do not perform -T2 pass if text_pass == 0
+if ($text_pass) then
+  set pass2 = (${pass2} -T2)
+endif
+
+set pass1 = (${pass1} -${invf_method}1)
+set pass2 = (${pass2} -${invf_method}2)
+
+
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+########################################################################
+# Here is where mgmerge goes to work.
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "MGMERGE collection: ${bname}"
+echo "--------------------------------------------------------------"
+echo "FIRST PHASE: build new collection from new documents"
+echo "--------------------------------------------------------------"
+
+# move and rename the text dictionary file
+# [RPAP - Feb 97: Level 3 Merge] - only if performing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text.dict $MGDATA/${newname}.text.dict
+endif
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo ">> $get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if ($pipe) then
+  if ($?pass1filter) then
+    echo ">> $get $text $complex | $pass1filter | mg_passes ${pass1}"
+    $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass1}"
+    $get $text $complex| $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass1} ${input_files}"
+  $bindir/mg_passes ${pass1} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+echo "mg_perf_hash_build -f ${newname}"
+$bindir/mg_perf_hash_build -f ${newname}
+if ("$status" != "0") exit 1 
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ($?pass2filter) then
+    echo ">> $get $text $complex | $pass2filter | mg_passes ${pass2}"
+    $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass2}"
+    $get $text $complex | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass2} ${input_files}"
+  $bindir/mg_passes ${pass2} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo ">> $get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "---------------------------------------"
+date
+echo "--------------------------------------------------------------"
+echo "SECOND PHASE: merge the two collections"
+echo "--------------------------------------------------------------"
+
+# move the appropriate files from the base directory to the merge dir
+echo "(moving files....)"
+# [RPAP - Feb 97: Level 3 Merge]
+# Move files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text       $MGDATA/${oldname}.text
+  mv $MGDATA/${bname}.text.idx   $MGDATA/${oldname}.text.idx
+else
+# Need this file if building weights
+  cp -d $MGDATA/${bname}.text.idx   $MGDATA/${mergename}.text.idx
+endif
+mv $MGDATA/${bname}.invf       $MGDATA/${oldname}.invf
+mv $MGDATA/${bname}.invf.idx   $MGDATA/${oldname}.invf.idx
+mv $MGDATA/${bname}.invf.dict  $MGDATA/${oldname}.invf.dict
+# the old weight file may be used for the new file
+mv $MGDATA/${bname}.weight     $MGDATA/${mergename}.weight
+# EOC: the old invf.paragraph file may be used for the new file for Level 3 Invf
+if (-e $MGDATA/${bname}.invf.paragraph) then
+  mv $MGDATA/${bname}.invf.paragraph  $MGDATA/${mergename}.invf.paragraph
+endif
+echo "---------------------------------------"
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Only merge text if doing text pass
+if ($text_pass) then
+  echo ">> mg_text_merge -f ${mergename}"
+  $bindir/mg_text_merge -f $mergename
+  echo "---------------------------------------"
+endif
+
+echo ">> mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}"
+$bindir/mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}
+echo "---------------------------------------"
+
+#remove .weight file, so a new one will be created by mg_weights_build
+if ("$guess_weights" == "") then
+  echo "rm ${mergename}.weight"
+  rm $MGDATA/${mergename}.weight
+endif
+
+# mv $MGDATA/${mergename}.weight $MGDATA/${mergename}.w2
+
+echo ">> mg_weights_build -f ${mergename} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${mergename} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "---------------------------------------"
+
+echo ">> mg_invf_dict -f ${mergename} -b 4096"
+$bindir/mg_invf_dict -f ${mergename} -b 4096
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  echo "-----------------------------------"
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+endif
+
+echo "--------------------------------------------------------------"
+
+# move files back to base dir
+# [RPAP - Feb 97: Level 3 Merge]
+# mv the text files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${oldname}.text         $MGDATA/${bname}.text
+  mv $MGDATA/${mergename}.text.idx   $MGDATA/${bname}.text.idx
+  mv $MGDATA/${newname}.text.dict    $MGDATA/${bname}.text.dict
+endif
+mv $MGDATA/${mergename}.invf       $MGDATA/${bname}.invf
+mv $MGDATA/${mergename}.invf.idx   $MGDATA/${bname}.invf.idx
+mv $MGDATA/${mergename}.invf.dict  $MGDATA/${bname}.invf.dict
+mv $MGDATA/${mergename}.invf.dict.blocked  $MGDATA/${bname}.invf.dict.blocked
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  mv $MGDATA/${mergename}.invf.dict.blocked.1 $MGDATA/${bname}.invf.dict.blocked.1
+  mv $MGDATA/${mergename}.invf.dict.blocked.2 $MGDATA/${bname}.invf.dict.blocked.2
+  mv $MGDATA/${mergename}.invf.dict.blocked.3 $MGDATA/${bname}.invf.dict.blocked.3
+endif
+
+mv $MGDATA/${mergename}.weight     $MGDATA/${bname}.weight
+mv $MGDATA/${mergename}.weight.approx $MGDATA/${bname}.weight.approx
+mv $MGDATA/${mergename}.text.idx.wgt $MGDATA/${bname}.text.idx.wgt
+
+# EOC: mv .invf.paragraph file if exists
+if (-e $MGDATA/${mergename}.invf.paragraph) then
+  mv $MGDATA/${mergename}.invf.paragraph   $MGDATA/${bname}.invf.paragraph
+endif
+
+# remove old files in the MERGE directory
+rm -f $MGDATA/${merge}/*
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "--------------------------------------------------------------"
+
+echo ""
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.sh
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.sh	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/scripts/xmg.sh	(revision 21112)
@@ -0,0 +1,1376 @@
+#!/usr/local/bin/wish -f
+# Edit the line above to point to your version of wish
+#
+###############################################################################
+#
+# xmg -- X interface to the mg information retrieval system
+# Copyright (C) 1994  Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)xmg.sh	1.4 25 Mar 1994
+#
+###############################################################################
+#
+# "xmg" is an X interface to the mg information retrieval system.
+#
+# Type a query in the "Query" box.
+# The first line or so of each matching document is listed.
+# Click on any line to show the complete document.
+# Shift-click to show the document in a new window.
+# Press Next and Previous to cycle through the list of documents (Shift works
+#   here too). 
+# If you change the query, existing windows continue to refer to the original
+#   query. 
+# 
+# You can select either Boolean or Ranked queries.
+# Boolean queries: use "&" for AND, "|" for OR, "!" for NOT, and parentheses
+# for grouping 
+# Ranked queries:  - documents are listed in order of relevance; a weight is
+# 		     shown alongside each 
+#                  - weights are normalized to 100 for the most relevant
+#		     document 
+#                  - choose Approximate Ranking for greater retrieval speed. 
+# 
+# In document windows (other than the main window): 
+#		Typing 'n' = next document
+#		Typing 'N' = next document in new window
+#		Typing 'p' = previous document
+#		Typing 'P' = previous document in new window
+#		Typing 'q' = close window
+#
+###############################################################################
+
+# Date: 13 March 1994
+# Authors: Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+
+###############################################################################
+# Customizing xmg
+###############################################################################
+
+# To override the default font options, use RESOURCE_MANAGER properties like
+# the following in your .Xdefaults file
+#
+#	xmg.textfont: fixed
+#	xmg.buttonfont: fixed
+#
+# To override the default query type of ranked, use
+#
+#	xmg.querytype: boolean
+#
+# To override the default collection name (the first one listed by /bin/ls from
+# your MGDATA environment variable), use
+#
+#	xmg.collection: mail93
+
+###############################################################################
+# Global variables
+###############################################################################
+
+#   genuine constants:
+#
+#	env		the environment variables
+#	Terminator	terminator for mgquery
+#	MaxWindowHeight	the maximum height permitted for a document window
+#	ListBoxLength	the number of items in the document list
+#	HeadLength	the length document head displayed in the list box
+#	ImageString	the string at the start of a document associated
+#			with an image
+#
+#   things that rarely get changed:
+#
+#	querytype	the current query type (boolean, ranked, approx-ranked)
+#	booleanquery	says whether the query is boolean or ranked
+#	approxquery	says whether ranking is to be approximate or not
+#	collection	the name of the document collection being used
+#	fileId		file identifer for a pipe to the mgquery process
+#       errorOccurred	flag to say that there has been an I/O error on the
+#			current collection
+#
+#   things that reflect items in the main panel
+#
+#	query		the current query
+#	hits		number of hits for the current query
+#	display		shell command to display the image associated with
+#			the last document retrieved
+#	scrolledto	document prefixes have been obtained for all documents
+#			up to this position in the document list
+#
+#   static variable needed in just one procedure
+#
+#	top_rank	weight associated with the first document retrieved
+#	windowname	unique id for each document window
+
+###############################################################################
+# High-level procedures
+###############################################################################
+
+proc SaveDocument {w} {
+  set file [FSBox "Save document to file:"]
+
+  if {$file != ""} {
+    if [file exists $file] {
+      toplevel .r
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text "The file already exists"
+      button .r.replace -text "Replace" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"w\" $w" 
+      button .r.append -text "Append" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"a\" $w"
+      button .r.cancel -text "Cancel" -command {destroy .r} -width 10 -height 2
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.replace .r.append .r.cancel -side left -pady 10 -padx 20
+
+    } else {
+       WriteDocument $file "w" $w
+    }
+  }
+}
+
+proc Dialog {text command} {
+      if {![winfo exists .r] } { toplevel .r }
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text $text
+      button .r.ok -text "OK" -width 10 -height 2 \
+          -command "destroy .r $command"
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.ok -side bottom -pady 10 -padx 20
+}
+
+proc WriteDocument {file mode w} {
+    if {[catch {set f [open $file $mode]}] == 0} {
+        if [winfo exists $w] {
+            puts $f [$w get 0.0 end]
+        } else {
+            Dialog "Unable to save text" ""
+        }
+        flush $f
+        close $f
+    } else {
+        Dialog "Unable to open file for writing" "; SaveDocument $w"
+    }
+}
+
+# Send a query to mg and enter the response in the document list
+#
+proc sendQuery {window} {
+    global query hits querytype errorOccurred scrolledto
+
+    .m.t.q.retrieved.label configure -foreground Gray
+    .m.t.q.retrieved.hits configure -foreground Gray
+
+    sendto ".set query \"$querytype\""
+    sendto ".set mode \"docnums\""
+    foreach f "sdocnums sdocs srank" { $window.$f delete 0 end }
+    disableButton .m.d.buttons.previous
+    disableButton .m.d.buttons.next
+    if {[string index $query 0] == "." || [string length $query] > 255} {
+        set doclist ""
+    } else { set doclist [sendandget $query] }
+
+    if {$errorOccurred} {
+        enterText .m.d.text [mgqueryFailure]
+        .m.t.q.retrieved.label configure -foreground Black
+        .m.t.q.retrieved.hits configure -foreground Black 
+        update
+        return
+    }
+    sendto ".set query \"DocNums\""
+    sendto ".set mode  \"text\""
+    .m.t.q.retrieved.label configure -foreground Black
+    .m.t.q.retrieved.hits configure -foreground Black 
+    set hits [llength $doclist]
+    set i 0
+    global top_rank
+
+    scan [lindex $doclist 0] "%d %f %d" d rank size
+    set top_rank $rank
+    if {$top_rank == 0} {set top_rank 1}
+
+    while {$i < $hits} {
+	scan [lindex $doclist $i] "%d %f %d" d rank size
+        if {$rank == "Inf"} {set rank 1}
+	if {$querytype == "boolean"} {
+	    $window.srank insert end ""
+	} else {
+	    $window.srank insert end \
+		[format "%3.0f" [expr $rank / $top_rank * 99 + 1]]
+	}
+	$window.sdocnums insert end [format " %d" $d]
+	incr i
+    }
+
+    set scrolledto 0
+    scroll $window 0
+    bind $window.sdocs <ButtonRelease-1> \
+	"showDocument $window.sdocs %y .m.d \"$doclist\" \"$query\""
+    bind $window.sdocs <Shift-ButtonRelease-1> \
+	"showDocument $window.sdocs %y New \"$doclist\" \"$query\""
+    enterText .m.d.text ""
+}
+
+# Scrolling routine for the document list
+#
+proc scroll {window n} {
+    global hits scrolledto ListBoxLength Terminator fileId ImageString
+
+    $window.sdocnums yview $n
+    $window.sdocs yview $n
+    $window.srank yview $n
+    set last $n
+    if {$last <= 0} {set last 0}
+    incr last $ListBoxLength
+    if {$last > $hits} {set last $hits}
+
+#    .m config -cursor {watch}
+#    update
+    sendto ".set mode \"heads\""
+    sendto ".set terminator \"\""
+    while {$scrolledto < $last} {
+	if {[scan [$window.sdocnums get $scrolledto] "%d" d] == 1} {
+            sendto $d
+            gets $fileId line
+            set i [string first " " $line]
+            if {$i != -1} {set line [string range $line [expr $i + 1] end]}
+            if {[string compare $ImageString [string range $line 0 9]] == 0} {
+                set line [getImageDocumentHead $d]
+            }
+	    $window.sdocs insert $scrolledto $line
+	}
+	incr scrolledto
+    }
+    sendto ".set terminator \"$Terminator\\n\""
+    sendto ".set mode text"
+#   .m config -cursor {}
+}
+
+proc getImageDocumentHead {d} {
+    global HeadLength ImageString fileId
+
+    sendto ".set heads_length 1000"
+    sendto $d
+    gets $fileId line
+
+    set line [string range $line 11 end]
+
+    set s [string first $ImageString $line]
+    if {$s != -1} {
+        incr s 11
+        set line [string range $line $s [expr $s + $HeadLength]]
+    }
+  
+    sendto ".set heads_length $HeadLength"
+
+    return "* $line"
+}
+
+# Show document from the document list
+#
+proc showDocument {window y newWindow doclist query} {
+    showThisDocument [$window nearest $y] $doclist $newWindow $query
+}
+
+proc adjacentDocument {index doclist window q button} {
+    global ListBoxLength
+
+    showThisDocument $index $doclist $window $q
+
+    if {$window == ".m.d"} {
+      set first [.m.t.q.r.sdocs nearest 0]
+      if {$index < $first || $index >= $first + $ListBoxLength} {
+          scroll .m.t.q.r [expr $index - $ListBoxLength / 2]
+      }
+      .m.t.q.r.sdocs select from $index
+    } 
+}
+
+
+# Show document with this index in the document list
+#
+proc showThisDocument {index doclist window q} {
+    global collection MaxWindowHeight display windowname
+
+    set document ""
+    scan [lindex $doclist $index] "%d %f" document rank
+    if {$document == ""} { return }
+    set textlist [sendandget $document]
+	
+    if {[string length $display] != 0} { exec csh -c $display >& /dev/null & }
+	
+# throw away the first line
+    set text [join [lrange $textlist 1 end] "\n"]
+
+# if necessary, create a new window for this document
+    if {$window == "New"} {
+        .m.t.q.r.sdocs select clear
+	set window .m.document$windowname
+        incr windowname
+        set existing ""
+        set raised 0
+        foreach w [lrange [winfo children .m] 2 end] {
+           if {[lindex [$w.docnum configure -text] 4] == $index && \
+               [lindex [$w.query configure -text] 4] == $q} {
+              if {$raised} {
+                 closeDocumentWindow $w
+              } else {
+                  raise $w
+		  getFocus $w
+                  set window $w
+                  set raised 1
+              }
+           }
+        }
+  
+        if {!$raised} {
+   	    toplevel $window
+            $window configure -borderwidth 2 -relief raised
+            wm protocol $window WM_TAKE_FOCUS "getFocus $window"
+	    getFocus $window
+	    makeDocumentPanel $window
+            bind $window q "$window.buttons.close invoke"
+            label $window.docnum -text $index
+            label $window.query -text $q
+       }
+    }
+
+# configure the window for this document
+    set rows [expr [llength $textlist] - 1]
+    if {$window != ".m.d"} {
+	wm title $window "$collection \"$q\""
+	wm iconname $window "$document"
+	$window.text configure -height $rows
+	if {$rows > $MaxWindowHeight} {
+	    $window.text configure -height $MaxWindowHeight
+	}
+    }
+
+# show the document text in the window
+    enterText $window.text $text
+
+# configure the window's Previous and Next buttons
+    if {$index == 0} {
+        disableButton $window.buttons.previous
+    } else {
+	set newindex [expr $index-1]
+	bindBoth $window previous $newindex $doclist $q
+        bind $window p [bind $window.buttons.previous <1>]
+        bind $window P [bind $window.buttons.previous <Shift-1>]
+    }
+    if {$index == [expr [llength $doclist]-1]} {
+	disableButton $window.buttons.next 
+    } else {
+	set newindex [expr $index+1]
+	bindBoth $window next $newindex $doclist $q
+        bind $window n [bind $window.buttons.next <1>]
+        bind $window N [bind $window.buttons.next <Shift-1>]
+    }
+
+    if {$window != ".m.d"} { $window.docnum configure -text $index }
+
+    if {[string length $display] != 0} { 
+        set on "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground Black
+                 update\}
+        }"
+        set off "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground [lindex [.m.d configure -background] 4]
+                 update\}
+        }"
+        set t 0
+        while {$t < 5} {
+            after [expr $t * 1000] eval $on
+            after [expr $t * 1000 + 500] eval $off
+            incr t
+        }   
+    }
+}
+
+# Bind both Button-1 and Shift-Button-1 commands
+#
+proc bindBoth {window button index doclist query} {
+    $window.buttons.$button configure -state normal
+    bind $window.buttons.$button <Button-1> \
+	"adjacentDocument $index \"$doclist\" $window \"$query\" $button"
+    bind $window.buttons.$button <Shift-Button-1> \
+	"adjacentDocument $index \"$doclist\" New \"$query\" $button"
+}
+
+# Change the type of query
+#
+proc changeQuerytype {name element op} {
+    global querytype booleanquery approxquery
+
+    if {$booleanquery} {
+	.m.t.b.type.approx configure -state disabled
+	.m.t.q.retrieved.weight configure \
+             -foreground [lindex [.m configure -background] 4]
+	set querytype boolean
+    } else {
+	.m.t.b.type.approx configure -state normal
+	.m.t.q.retrieved.weight configure -foreground Black
+	if {$approxquery} {
+	    set querytype "approx-ranked"
+	} else {
+	    set querytype "ranked"
+	}
+    }
+}
+
+proc enterText {window text} {
+    $window configure -state normal
+    $window delete 0.0 end
+    $window insert 0.0 $text
+    $window configure -state disabled
+}
+
+###############################################################################
+# IO and initialization
+###############################################################################
+
+# Send a message to mg and return the reply
+#
+proc sendandget {message} {
+    global fileId Terminator display errorOccurred ImageString
+    set display ""
+
+    .m config -cursor {watch}
+    update idletasks
+    set reply ""
+    sendto "$message\n"
+    if {$message != "" && !$errorOccurred} {
+        gets $fileId line
+    	while {[string compare $Terminator $line] != 0} {
+            lappend reply $line;
+            gets $fileId line
+        }
+
+        if {[string compare $ImageString [lindex $reply 1]] == 0 } {
+          if {[string compare "MGDATA" [string range [lindex $reply 3] 0 5]] == 0} {
+             set display [string range [lindex $reply 3] 7 end];
+          }
+          set reply [lrange $reply 4 end]
+        }
+    }
+
+    .m config -cursor {}
+    return $reply
+}
+    
+# Send a message to mg
+#
+proc sendto {message} {
+    global fileId errorOccurred
+
+    puts $fileId $message
+    set errorOccurred [catch {flush $fileId}]
+}
+
+# Reinitialise mgquery to work with a new document collection
+#
+proc reInitMG {c} {
+    global fileId
+    catch {close $fileId}
+    initMG $c
+}
+
+# Initialize mgquery to work with a specified text collection
+#
+proc initMG {c} {
+    global query hits fileId collection Terminator env errorOccurred
+    global HeadLength
+
+    set collection $c
+    set query ""
+    set hits 0
+    set errorOccurred 0
+
+    foreach f "sdocs srank" { .m.t.q.r.$f delete 0 end }
+    enterText .m.d.text ""
+    .m.t.b.collect.m configure -text $collection
+    foreach w [winfo children .m] {
+	if {[string first ".m.document" $w] == 0} { closeDocumentWindow $w }
+    }
+
+    disableButton .m.d.buttons.previous 
+    disableButton .m.d.buttons.next
+
+    if {$c != ""} {
+        set f $env(MGDATA)/$c
+
+        if {[file exists $f] && [file readable $f]} {
+    	    set fileId [open "| mgquery $collection" RDWR]
+	    sendto ".set expert \"true\""
+	    sendto ".set terminator \"$Terminator\\n\""
+	    sendto ".set pager \"cat\""
+            sendto ".set heads_length $HeadLength"
+	    return
+        }
+    }
+    bind .m.t.q.e.entry <Return> {null}
+    enterText .m.d.text [initializationFailure]
+    update
+}
+
+# Routine to obtain values from RESOURCE_MANAGER
+#
+proc query_resource {option class default} {
+	set val [option get . $option $class]
+	if {[string length $val] == 0} { return $default }
+	return $val
+}
+
+
+proc disableButton {b} {
+    bind $b <Button-1> {null}
+    bind $b <Shift-Button-1> {null}
+    $b configure -state disabled
+}
+
+proc null {} {}
+
+###############################################################################
+# Setting up windows
+###############################################################################
+
+proc removeFromWindowList {window} {
+    global stacking
+
+    set i 0
+    while {$i < [llength $stacking]} {
+      if {$window == [lindex $stacking $i]} {
+         set stacking [lreplace $stacking $i $i]
+      }
+      incr i
+    }
+}
+
+proc getFocus {window} {
+    global stacking
+
+    focus $window
+    removeFromWindowList $window
+    set stacking [linsert $stacking 0 $window]
+}
+
+proc closeDocumentWindow {w} {
+   global stacking
+   removeFromWindowList $w
+   set t [lindex $stacking 0]
+
+   if {$t != ""} {
+     focus $t
+     raise $t
+  }
+  destroy $w
+}
+
+# Make a "document" panel, including Previous, Next, and Close buttons
+#
+proc makeDocumentPanel {window} {
+    frame $window.buttons
+    button $window.buttons.previous -text "Previous" -width 10 -state disabled
+    button $window.buttons.next -text "Next" -width 10 -state disabled
+    button $window.buttons.close -text "Close" -width 10 \
+	-command "closeDocumentWindow $window"
+    button $window.buttons.save -text "Save" -width 10 \
+        -command "SaveDocument $window.text"
+
+    label $window.buttons.image -text "Fetching image..." \
+            -foreground [lindex [$window configure -background] 4]
+    pack $window.buttons.previous $window.buttons.next -side left
+    pack $window.buttons.image -side left -padx 10
+    pack $window.buttons.close $window.buttons.save -side right 
+    pack $window.buttons -side top -fill x 
+
+    text $window.text -borderwidth 2 -relief raised \
+             -setgrid true -width 80 -state disabled
+
+    $window.text configure -yscrollcommand "$window.sy set" 
+    scrollbar $window.sy -relief flat -command "$window.text yview"
+    pack $window.sy -side right -fill y 
+    pack $window.text -side top -expand yes -fill both
+}
+
+# Make the "buttons" panel, with the query type and collection name
+#
+proc mkButtons {window} {
+    global collection booleanquery approxquery env
+    frame $window.type
+    label $window.type.label -text "Query type"
+
+    radiobutton $window.type.boolean -text "Boolean" -variable booleanquery \
+	  -value 1 -relief flat
+
+    radiobutton $window.type.ranked -text "Ranked" -variable booleanquery \
+	  -value 0 -relief flat
+
+    checkbutton $window.type.approx -text "Approximate" -variable approxquery \
+	  -relief flat
+
+    trace variable booleanquery w changeQuerytype
+    trace variable approxquery w changeQuerytype
+
+    pack $window.type.label   -side top -anchor w
+    pack $window.type.boolean -side top -anchor w -padx 12
+    pack $window.type.ranked  -side top -anchor w -padx 12
+    pack $window.type.approx  -side top -anchor w -padx 36
+
+    pack $window.type -side top -anchor w -pady 12
+
+    frame $window.collect
+    label $window.collect.label -text "Collection: "
+    pack $window.collect.label -side left -anchor w
+
+    getViewer
+
+    set dirs [getMGDATA]
+    set collection ""
+    if {$dirs != ""} { set collection [lindex $dirs 0] }
+   
+    menubutton $window.collect.m -menu $window.collect.m.menu \
+	-relief sunken
+    menu $window.collect.m.menu
+    foreach c $dirs {
+        if {[file readable $env(MGDATA)/$c] && 
+            [file isdirectory $env(MGDATA)/$c]} {
+    	    $window.collect.m.menu add command -label $c \
+	        -command "reInitMG $c"
+        }
+    }
+    pack $window.collect.m -side left
+    pack $window.collect -side bottom -anchor w
+
+
+}
+
+# Check that the MGDATA environment variable is set, and is non-null
+#
+proc getMGDATA {} {
+    global env
+
+    set MGDATA 0
+    foreach i [array names env] {
+	if {$i == "MGDATA"} {
+	    set dirs [exec /bin/ls $env(MGDATA)]
+	    if {[info exists dirs]} { return $dirs }
+	}
+    }
+
+    return ""
+}
+
+proc getViewer {} {
+    global env
+
+    set viewer 0
+    foreach i [array names env] {
+	if {$i == "MGIMAGEVIEWER"} {set viewer $env(MGIMAGEVIEWER)}
+    }
+
+    if {$viewer == 0} {set env(MGIMAGEVIEWER) "xloadimage -quiet -fit stdin"}
+}
+
+# Make the "query" panel, with the query and list of retrieved documents
+#
+proc mkQuery {window} {
+    global hits ListBoxLength
+
+# at the top of the "query" panel is the "entry" frame
+    frame $window.e
+    label $window.e.label -text "Query"
+    entry $window.e.entry -relief sunk -width 47 -textvariable query
+    pack $window.e.label -side left
+    pack $window.e.entry -side right -expand yes -fill x
+    focus default $window.e.entry
+    focus $window.e.entry
+
+    bind $window.e.entry <Control-Key> {null}
+    bind $window.e.entry <Return> "sendQuery $window.r"
+    bind $window.e.entry <Escape> { set query "" }
+    bind $window.e.entry <2> {
+	%W insert insert [selection get STRING]
+	tk_entrySeeCaret %W
+    }
+
+# next is the "retrieved" frame
+    frame $window.retrieved
+    label $window.retrieved.weight -text "Weight"
+    label $window.retrieved.label -text "   Documents retrieved: "
+    label $window.retrieved.hits -width 4 -relief sunk -textvariable hits
+    set documentsRetrieved 0
+    pack $window.retrieved.weight $window.retrieved.label \
+        $window.retrieved.hits -side left
+
+# next is the "results" frame (note that the docnums listbox doesn't appear on
+# the screen)
+    frame $window.r
+    scrollbar $window.r.sscroll -relief sunken -command "scroll $window.r"
+#    bind $window.r.sscroll <ButtonRelease-1> "updateSdocs $window.r.sdocs $window.r.sdocnums"
+    listbox $window.r.sdocnums
+    listbox $window.r.sdocs -relief sunken \
+            -setgrid 1 -geometry 50x$ListBoxLength
+    listbox $window.r.srank -yscroll "$window.r.sscroll set" \
+            -setgrid 1 -geometry 4x$ListBoxLength
+  
+    bind $window.r.srank <Any-1> {null}
+    bind $window.r.srank <Any-2> {null}
+    bind $window.r.srank <Any-3> {null}
+
+    bind $window.r.srank <Any-B1-Motion> {null}
+    bind $window.r.srank <Any-B2-Motion> {null}
+    bind $window.r.srank <Any-B3-Motion> {null}
+
+    bind $window.r.sdocs <Any-2> {null}
+    bind $window.r.sdocs <Any-B2-Motion> {null}
+
+    pack $window.r.sscroll -side right -fill y
+    pack $window.r.srank -side left 
+    pack $window.r.sdocs -side left -fill x -expand yes
+
+# pack them up
+    pack $window.e -side top -anchor e -pady 10 -fill x -expand yes
+    pack $window.retrieved -anchor w
+    pack $window.r -side bottom -anchor e -pady 10 -fill x -expand yes
+}
+
+# Make a "help" window
+#
+proc mkHelp {} {
+    if [winfo exists .help] { destroy .help }
+    toplevel .help
+    wm title .help "X interface to the mg information retrieval system"
+    wm iconname .help "About xmg"
+    frame .help.buttons
+    button .help.buttons.close -text Close -command "destroy .help" -width 10
+    button .help.buttons.help -text Help -width 10
+    button .help.buttons.conditions -text Conditions -width 10
+    button .help.buttons.warranty -text Warranty -width 10
+    button .help.buttons.bugs -text Bugs -width 10
+
+    frame .help.t
+    text .help.t.text -relief raised -bd 2 -setgrid true -height 24 -width 95
+    .help.t.text configure -yscrollcommand ".help.t.sy set" 
+    scrollbar .help.t.sy -relief flat -command ".help.t.text yview"
+    
+    pack .help.buttons.close .help.buttons.help -side right -fill x
+    pack .help.buttons.conditions .help.buttons.warranty \
+         .help.buttons.bugs -side left
+    pack .help.buttons -side bottom -fill x
+
+    pack .help.t.text -side left -expand yes -fill both
+    pack .help.t.sy -side right -fill y 
+    pack .help.t -side top -fill both -expand yes
+    enterText .help.t.text [HelpText]
+    grayButton help
+}
+
+proc grayButton {button} {
+    bind .help.buttons.help <1> "enterText .help.t.text \[HelpText\]
+         grayButton help" 
+    .help.buttons.help configure -state normal
+
+    bind .help.buttons.bugs <1> "enterText .help.t.text \[Bugs\]
+         grayButton bugs" 
+    .help.buttons.bugs configure -state normal
+
+    bind .help.buttons.warranty <1> "enterText .help.t.text \[Warranty\]
+         grayButton warranty"
+    .help.buttons.warranty configure -state normal
+
+    bind .help.buttons.conditions <1> "enterText .help.t.text \[Conditions\]
+         grayButton conditions" 
+    .help.buttons.conditions configure -state normal
+    disableButton .help.buttons.$button
+}
+
+# The text for errors in the MG environment variable
+#
+
+
+proc bookDetails {} {
+  return { \
+
+ This X interface was written by Bruce McKenzie, Ian Witten and Craig Nevill-Manning.  
+ For information on mg, see
+             Witten, I.H.,  Moffat, A. and Bell, T.C. (1994) "Managing gigabytes: compressing
+             and indexing documents and images." Van Nostrand Reinhold, New York.
+             ISBN 0-442-01863-0, US $54.95; call 1 (800) 544-0550 to order.
+}
+}
+
+proc initializationFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ Your MGDATA variable is unset or does not specify a directory with files in; therefore XMG
+ cannot proceed.
+} [bookDetails]]
+}
+
+proc mgqueryFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ There is a problem with the collection that you have chosen. Please re-index the collection before
+ you use it again.
+} [bookDetails]]
+
+}
+
+# The text for the help message
+#
+proc HelpText {} {
+    return [format "%s\n%s" {\
+This is an X interface to the mg information retrieval system.
+
+ Type a query in the "Query" box.
+ The first line or so of each matching document is listed. 
+ Click on any line to show the complete document.
+ Shift-click to show the document in its own window
+ Press Next and Previous to cycle through the list of documents (Shift works here too).
+ In document windows (other than the main window), typing n or p shows the next and previous 
+ documents; N or P creates a new window for them. q closes the most recently created window.
+ If you change the query, existing windows continue to refer to the original query.
+ 
+ You can select either Boolean or Ranked queries.
+ Boolean queries: use & for AND, | for OR, ! for NOT, and parentheses for grouping
+ Ranked queries:  documents are listed in order of relevance; a weight is shown alongside each
+                                  weights are normalized to 100 for the most relevant document
+                                  choose Approximate Ranking for greater retrieval speed. 
+ } [bookDetails]]
+}
+
+proc reformat {text} {
+    regsub -all "\n *\n *" $text "BLEQUEWFI" text
+    regsub -all "\[ \t\n\r\]+" $text " " text
+    regsub -all "BLEQUEWFI" $text "\n\n" text
+    return $text
+}
+
+proc Conditions {} {
+    return [reformat [join [sendandget ".conditions\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Warranty {} {
+    return [reformat [join [sendandget ".warranty\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Bugs {} {
+    return [reformat {
+Known shortcomings of xmg include:
+
+    Slow picture viewing.  Pictures are decompressed in their entirety before a window is created 
+        for the picture.  This often results in a long delay with no feedback. Moreover, when 
+        an image's document is displayed repeatedly, multiple identical images are shown.
+
+    Sequential viewing.  There is no way of going to the next document in the collection to see the
+        context of documents that have been retrieved.  This would be trivial to implement but
+        conflicts with the interpretation of the "previous" and "next" buttons.
+    }]
+}
+
+###############################################################################
+# The main program
+###############################################################################
+
+scan [info tclversion] "%d.%d" major minor
+
+if {$major < 7} {
+    puts "xmg requires version 7.0 of the tcl interpreter. You are using version $major.$minor"
+    exit
+}
+
+# Default font settings
+set defaulttextfont   -adobe-times-medium-r-normal--14-140-75-75-p-74-iso8859-1
+set defaultbuttonfont -adobe-times-bold-r-normal--14-140-75-75-p-77-iso8859-1
+set textfont   [query_resource textfont Textfont $defaulttextfont]
+set buttonfont [query_resource buttonfont Buttonfont $defaultbuttonfont]
+
+set MaxWindowHeight 10
+set HeadLength 100
+set scrolledto 0
+set ListBoxLength 12
+set windowname 0
+set Terminator "RECORD_END"
+set fileId ""
+set query ""
+set ImageString "::::::::::"
+set stacking ".m.t"
+
+tk_listboxSingleSelect Listbox
+option add *Text.wrap word
+
+option add *Text.font $textfont
+option add *Listbox.font $textfont
+option add *Entry.font $textfont
+
+option add *Button.font $buttonfont
+option add *Menubutton.font $buttonfont
+option add *Radiobutton.font $buttonfont
+option add *Checkbutton.font $buttonfont
+option add *Label.font $buttonfont
+option add *Menu.font $buttonfont
+
+wm title . "Managing Gigabytes"
+wm iconname . "xmg"
+frame .m -borderwidth 5 -relief raised
+
+# top part of frame
+frame .m.t
+
+# on the left side of the top part is the "buttons" panel
+frame .m.t.b
+mkButtons .m.t.b
+
+# on the right side is the "query" panel
+frame .m.t.q
+mkQuery .m.t.q
+
+set q [query_resource querytype Querytype approx-ranked]
+set booleanquery [expr \"$q\" == \"boolean\"]
+set approxquery [expr \"$q\" == \"approx-ranked\"]
+# the above lines will trigger changeQuerytype
+
+pack .m.t.b -side left -anchor nw
+pack .m.t.q -side right -anchor ne -fill both -expand yes
+
+# at the bottom is the "document" panel
+frame .m.d
+makeDocumentPanel .m.d
+
+.m.d.buttons.close configure -text "Exit" \
+    -command "destroy ."
+button .m.d.buttons.help -text "Help" -width 10 \
+	-command "mkHelp"
+
+pack .m.d.buttons.help -side right
+pack .m.t -side top -fill x -expand yes
+pack .m.d -side bottom -fill both -expand yes
+
+pack .m -fill both -expand yes
+wm geometry . +300+85
+
+set collection [query_resource collection Collection $collection]
+if {$argc>0} { set collection $argv }
+
+enterText .m.d.text [HelpText]
+
+initMG $collection
+
+###############################################################################
+###############################################################################
+# The following file selection box procedure was written by Andrew Donkin
+# of the University of Waikato. 
+###############################################################################
+
+proc FSBox { {fsBoxMessage "Select file:"}} {
+# xf ignore me 5
+  global fsBox
+
+  set fsBox(name) ""
+  set fsBox(internalPath) [pwd]
+  set fsBox(pattern) "*"
+  set fsBox(all) 0
+
+  # build widget structure
+
+  # start build of toplevel
+  if {"[info commands XFDestroy]" != ""} {
+    catch {XFDestroy .fsBox}
+  } {
+    catch {destroy .fsBox}
+  }
+  toplevel .fsBox -borderwidth 0
+  wm geometry .fsBox 350x300 
+  wm title .fsBox {File select box}
+  wm maxsize .fsBox 1000 1000
+  wm minsize .fsBox 100 100
+  # end build of toplevel
+
+  label .fsBox.message1 -anchor c -text "$fsBoxMessage"
+  frame .fsBox.frame1  -borderwidth 0  -relief raised
+
+  button .fsBox.frame1.ok  -text "OK"  -command "
+      global fsBox
+      set fsBox(name) \[.fsBox.file.file get\]
+      set fsBox(path) \[.fsBox.path.path get\]
+
+      set fsBox(internalPath) \[.fsBox.path.path get\]
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  button .fsBox.frame1.rescan -text "Rescan" -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+  }
+
+  button .fsBox.frame1.cancel -text "Cancel" -command "
+      global fsBox
+      set fsBox(name) {}
+      set fsBox(path) {}
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  frame .fsBox.path -borderwidth 0 -relief raised
+  frame .fsBox.path.paths 
+  label .fsBox.path.paths.paths -borderwidth 0 -text "Pathname:"
+
+  entry .fsBox.path.path -relief sunken
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+  frame .fsBox.pattern -borderwidth 0 -relief raised
+  frame .fsBox.pattern.patterns 
+
+  label .fsBox.pattern.patterns.patterns -borderwidth 0 -text "Selection pattern:"
+  entry .fsBox.pattern.pattern -relief sunken
+  .fsBox.pattern.pattern insert 0 $fsBox(pattern)
+  
+  frame .fsBox.files -borderwidth 0 -relief raised
+  scrollbar .fsBox.files.vscroll -relief raised -command ".fsBox.files.files yview"
+  scrollbar .fsBox.files.hscroll -orient horiz -relief raised -command ".fsBox.files.files xview" 
+  listbox .fsBox.files.files -exportselection false -relief raised -xscrollcommand \
+        ".fsBox.files.hscroll set" -yscrollcommand ".fsBox.files.vscroll set"
+  frame .fsBox.file -borderwidth 0 -relief raised
+  label .fsBox.file.labelfile -text "Filename:"
+  entry .fsBox.file.file -relief sunken
+  focus .fsBox.file.file
+
+  .fsBox.file.file delete 0 end
+  .fsBox.file.file insert 0 $fsBox(name)
+  
+  checkbutton .fsBox.pattern.all -offvalue 0 -onvalue 1 -text "Show all files" -variable fsBox(all) -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    }
+
+  FSBoxFSShow $fsBox(internalPath) $fsBox(pattern) $fsBox(all)
+
+  # bindings
+  bind .fsBox.files.files <Double-Button-1> "FSBoxFSFileSelectDouble %W %y"
+  bind .fsBox.files.files <ButtonPress-1> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-ButtonPress-1> "FSBoxFSFileSelect %W %y"
+
+  bind .fsBox.path.path <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.path.path <Up> {}"
+  bind .fsBox.path.path <Down> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+
+  bind .fsBox.file.file <Return> "
+    global fsBox
+    set fsBox(name) \[.fsBox.file.file get\]
+    set fsBox(path) \[.fsBox.path.path get\]
+    set fsBox(internalPath) \[.fsBox.path.path get\]
+
+    if {\"\[info commands XFDestroy\]\" != \"\"} {
+      catch {XFDestroy .fsBox}
+    } {
+      catch {destroy .fsBox}
+    }"
+  bind .fsBox.file.file <Up> {.fsBox.path.path icursor end
+    focus .fsBox.path.path}
+  bind .fsBox.file.file <Down> {.fsBox.pattern.pattern icursor end
+    focus .fsBox.pattern.pattern}
+
+  bind .fsBox.pattern.pattern <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)}
+  bind .fsBox.pattern.pattern <Up> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.pattern.pattern <Down> {}"
+
+  # packing
+  pack .fsBox.files.vscroll -side left -fill y 
+  pack .fsBox.files.hscroll -side bottom -fill x
+  pack .fsBox.files.files -side left -fill both -expand yes
+
+  pack .fsBox.file.labelfile -side left
+  pack .fsBox.file.file -side left -fill  both -expand yes
+
+  pack .fsBox.frame1.ok -side left -fill both -expand yes
+  pack .fsBox.frame1.rescan -side left -fill both -expand yes
+  pack .fsBox.frame1.cancel -side left -fill both -expand yes
+
+  pack .fsBox.path.paths.paths -side left
+
+  pack .fsBox.pattern.patterns.patterns -side left
+
+  pack .fsBox.path.paths -side left
+  pack .fsBox.path.path -side left -fill both -expand yes
+
+  pack .fsBox.pattern.patterns -side left
+  pack .fsBox.pattern.all -side right -fill both
+  pack .fsBox.pattern.pattern -side left -fill both -expand yes
+
+  pack .fsBox.message1 -side top -fill both
+  pack .fsBox.frame1 -side bottom -fill both
+  pack .fsBox.pattern -side bottom -fill both
+  pack .fsBox.file -side bottom -fill both
+  pack .fsBox.path -side bottom -fill both
+  pack .fsBox.files -side left -fill both -expand yes
+
+  update idletask
+  grab .fsBox
+  tkwait window .fsBox
+
+  if {"[string trim $fsBox(path)]" != "" ||
+      "[string trim $fsBox(name)]" != ""} {
+    if {"[string trimleft [string trim $fsBox(name)] /]" == ""} {
+      return [string trimright [string trim $fsBox(path)] /]
+    } {
+      return [string trimright [string trim $fsBox(path)] /]/[string trimleft [string trim $fsBox(name)] /]
+    }
+  }
+}
+
+proc FSBoxBindSelectOne { fsBoxW fsBoxY} {
+# xf ignore me 6
+
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    $fsBoxW select from $fsBoxNearest
+    $fsBoxW select to $fsBoxNearest
+  }
+}
+
+proc FSBoxFSFileSelect { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+        "[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+      set fsBoxFileName [string range $fsBoxTmpEntry 0             [expr [string length $fsBoxTmpEntry]-2]]
+      if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+          ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    } {
+      if {"[string index $fsBoxTmpEntry             [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0           [expr [string length $fsBoxTmpEntry]-2]]
+        if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    }
+    if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+      set fsBox(name) $fsBoxFileName
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 $fsBox(name)
+    }
+  }
+}
+
+proc FSBoxFSFileSelectDouble { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"$fsBoxTmpEntry" == "../"} {
+      set fsBoxTmpEntry [string trimright [string trim $fsBox(internalPath)] "@/"]
+      if {"$fsBoxTmpEntry" == ""} {
+        return
+      }
+      FSBoxFSShow [file dirname $fsBoxTmpEntry]         [.fsBox.pattern.pattern get] $fsBox(all)
+      .fsBox.path.path delete 0 end
+      .fsBox.path.path insert 0 $fsBox(internalPath)
+    } {
+      if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+          "[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0               [expr [string length $fsBoxTmpEntry]-2]]
+        if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+            ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+          set fsBoxFileName [string range $fsBoxTmpEntry 0                 [expr [string length $fsBoxTmpEntry]-2]]
+          if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+            set fsBoxFileName $fsBoxTmpEntry
+          }
+        } {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      }
+      if {[IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBox(internalPath) "[string trimright $fsBox(internalPath) {/@}]/$fsBoxFileName"
+        FSBoxFSShow $fsBox(internalPath) [.fsBox.pattern.pattern get] $fsBox(all)
+        .fsBox.path.path delete 0 end
+        .fsBox.path.path insert 0 $fsBox(internalPath)
+      } {
+        set fsBox(name) $fsBoxFileName
+        set fsBox(path) $fsBox(internalPath)
+        if {"[info commands XFDestroy]" != ""} {
+          catch {XFDestroy .fsBox}
+        } {
+          catch {destroy .fsBox}
+        }
+      }
+    }
+  }
+}
+
+proc FSBoxFSShow { fsBoxPath fsBoxPattern fsBoxAll} {
+# xf ignore me 6
+  global fsBox
+
+  set fsBox(pattern) $fsBoxPattern
+
+  if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+      [IsADir $fsBoxPath]} {
+    set fsBox(internalPath) $fsBoxPath
+  } {
+    if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+        [IsAFile $fsBoxPath]} {
+      set fsBox(internalPath) [file dirname $fsBoxPath]
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 [file tail $fsBoxPath]
+      set fsBoxPath $fsBox(internalPath)
+    } {
+      while {"$fsBoxPath" != "" && "$fsBoxPath" != "/" &&
+             ![file isdirectory $fsBoxPath]} {
+        set fsBox(internalPath) [file dirname $fsBoxPath]
+         set fsBoxPath $fsBox(internalPath)
+      }
+    }
+  }
+  if {"$fsBoxPath" == ""} {
+    set fsBoxPath "/"
+    set fsBox(internalPath) "/"
+  }
+  .fsBox.path.path delete 0 end
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+
+  if {[.fsBox.files.files size] > 0} {
+    .fsBox.files.files delete 0 end
+  }
+  if {$fsBoxAll} {
+    if {[catch "exec ls -F -a $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  } {
+    if {[catch "exec ls -F $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  }
+  set fsBoxElementList [lsort $fsBoxResult]
+
+  foreach fsBoxCounter [winfo children .fsBox.pattern.patterns.patterns] {
+    if {[string length [info commands XFDestroy]] > 0} {
+      catch {XFDestroy $fsBoxCounter}
+    } {
+      catch {destroy $fsBoxCounter}
+    }
+  }
+  menu .fsBox.pattern.patterns.patterns.menu
+
+  if {"$fsBoxPath" != "/"} {
+    .fsBox.files.files insert end "../"
+  }
+  foreach fsBoxCounter $fsBoxElementList {
+    if {[string match $fsBoxPattern $fsBoxCounter] ||
+        [IsADir [string trimright $fsBoxPath/$fsBoxCounter "/@"]]} {
+      if {"$fsBoxCounter" != "../" &&
+          "$fsBoxCounter" != "./"} {
+        .fsBox.files.files insert end $fsBoxCounter
+      }
+    }
+  }
+}
+
+proc IsADir { pathName} {
+# xf ignore me 5
+  if {[file isdirectory $pathName]} {
+    return 1
+  } {
+    catch "file type $pathName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $pathName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isdirectory $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsAFile { fileName} {
+# xf ignore me 5
+  if {[file isfile $fileName]} {
+    return 1
+  } {
+    catch "file type $fileName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $fileName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isfile $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsASymlink { fileName} {
+# xf ignore me 5
+  catch "file type $fileName" fileType
+  if {"$fileType" == "link"} {
+    return 1
+  }
+  return 0
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/GCC.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/GCC.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/GCC.mak	(revision 21112)
@@ -0,0 +1,205 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+INSTALL = copy
+
+o = .o
+e = .exe
+/ = \\
+
+CPPFLAGS = -g
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D_LITTLE_ENDIAN
+LIBS = ..$/..$/lib$/libmg.a
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+ 
+INCLUDES = -I"..$/.." -I"..$/..$/lib"
+
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+EXEC = \
+  mgquery$e mg_weights_build$e mgstat$e mg_invf_dump$e \
+  mg_invf_dict$e mg_invf_rebuild$e mgdictlist$e \
+  mg_passes$e mg_perf_hash_build$e mg_compression_dict$e \
+  mg_text_estimate$e mg_fast_comp_dict$e mg_hilite_words$e \
+  mg_invf_merge$e mg_text_merge$e bool_tester$e \
+  mg_stem_idx$e  mgstemidxlist$e mg_decompress_text$e
+
+all:    $(EXEC) libtextin.a
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(QUERY_OBJS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+install: 
+	$(INSTALL) *$e ..$/..$/..$/..$/bin$/windows
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	$(DELETE) libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+clean:
+	$(DELETE) *$o 
+	$(DELETE) *$e
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/MGPASS.C
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/MGPASS.C	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/MGPASS.C	(revision 21112)
@@ -0,0 +1,510 @@
+/**************************************************************************
+ *
+ * mgpass.cpp -- Driver for the various passes -
+
+	V1 -  removed all the pipe processing and replaced with
+	      code to directly explore a directory of files.
+	V2 -  rebuilt to extract non text files from web browser
+	      'catch' file.  Also to display progress count
+			   GH/WJR
+
+ * Copyright (C) 1994  Neil Sharman, ..
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <malloc.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <io.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include "memlib.h"
+#include "messages.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/20 02:32:00  cs025
+   a few fixes for compiling under windows with new long long stuff in mg
+
+   Revision 1.1  1999/08/10 21:17:39  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.5  1998/12/02 01:32:40  sjboddie
+   fixed 1 missing semi-colon..
+
+   Revision 1.4  1998/12/02 00:57:10  rjmcnab
+
+   Fixed to compile with the changes to the stemming.
+
+   Revision 1.3  1998/11/24 03:39:23  rjmcnab
+
+   Fixed a small error in the windows compilation
+
+   Revision 1.2  1998/11/24 01:29:39  rjmcnab
+
+   Fixed a few problems with the windows build
+
+   Revision 1.1  1998/11/17 09:34:12  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define path_length 256
+
+unsigned long buf_size = 3 * 1024 * 1024;       /* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;       /* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+int num_docs = 0;
+mg_ullong bytes_received = 0;
+int stem_method = 0;
+int stemmer_num = 0; /* default to the Lovin stemmer */
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+static char dirname[path_length], wildname[path_length];
+static int by_para = 0, recurse = 0, html_catch = 0;
+static char *buffer;
+char *line_start, *data_end, *base, *scan;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+  }
+pass_data;
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method] -f doc-collection-name\n"
+"  %*s [source directory\\] [source file]\n";
+
+static void usage (char *err)
+{
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+    strlen (msg_prefix), "", strlen (msg_prefix), "");
+  exit (1);
+}
+
+void do_process(char *buffer, int num)
+{
+  int pass;
+  if (num == 0) Message ("Warning : Processing zero length document");
+  num_docs++;
+  bytes_processed += num;
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+	if (PassData[pass].process((u_char *)buffer, num) == COMPERROR)
+	   FatalError(1, "Error during file processing");
+}
+
+int refill(int in_fd)
+{
+  int num, bitleft;
+  bitleft = data_end - base;
+  memmove(buffer, base, bitleft);
+  if (buf_size - bitleft < MIN_BUF)
+    FatalError(1, "Paragraph too big for buffer");
+  num = read(in_fd, &buffer[bitleft], buf_size - bitleft);
+  line_start -= (base - buffer);
+  scan -= (base - buffer);
+  base = buffer;
+  data_end = buffer + bitleft + num;
+  if (num > 0) return 1; else return 0;
+}
+
+char *scanpara(int in_fd)
+{
+  int num, blank, in_blank, at_end;
+  num = read(in_fd,buffer,buf_size);
+  in_blank = 1;  base = buffer;  line_start = buffer;  scan = buffer;
+  data_end = buffer + num;  at_end = 0;
+  for (;;) {
+    blank = 1;  line_start = scan;                   // Get a line
+    while (scan < data_end  &&  *scan != '\n')
+      if (*scan++ > ' ') blank = 0;
+    if (scan >= data_end) {
+      if (refill(in_fd)) {
+	while (scan < data_end  &&  *scan != '\n')
+	  if (*scan++ > ' ') blank = 0;
+	if (scan < data_end) scan++;
+	}
+      else at_end = 1;
+      }
+    else scan++;
+    if (line_start < scan) {                         // If we have a line
+      if (in_blank) {
+	if (!blank) { in_blank = 0; base = line_start; }
+	}
+      else {
+	if (blank) {
+	  do_process(base, line_start - base);
+	  in_blank = 1;  base = line_start;
+	  }
+	}
+      }
+    if (at_end) break;
+    }
+  if (in_blank) base = scan;
+  if (scan + 2 <= buffer + buf_size) {
+    *scan++ = 26;  *scan++ = 10;
+    }
+  if (base < scan) do_process(base, scan - base);
+  return NULL;
+}
+
+char *scanfile(int in_fd)
+{
+  int num = read(in_fd,buffer,buf_size);  /*expect to read the whole file*/
+  if (num < 0) return "file locked";
+  if (num >= buf_size-1) return "file too long";
+  do_process(buffer, num);
+  return NULL;
+}
+
+void search(char *dname, char *fname)
+{
+  long dirtag;  struct _finddata_t dirinfo;  int in_fd;  char *res;
+  char search_name[path_length], found_name[path_length];
+
+  sprintf(search_name, "%s%s", dname, fname);    /*Scan files*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag >= 0) {
+    do {
+      if ((dirinfo.attrib & (_A_SUBDIR | _A_HIDDEN | _A_SYSTEM)) == 0) {
+	sprintf(found_name,"%s%s",dname,dirinfo.name);
+	in_fd = open(found_name,O_RDONLY|O_BINARY);
+	if (in_fd >= 0) {
+	  if (by_para)
+	    res = scanpara(in_fd);
+	  else
+	    res = scanfile(in_fd);
+	  if (res != NULL) {
+	    Message("Error %s in processing file %s\n", res, found_name);
+	    exit(1);
+	    }
+	  close(in_fd);
+	  }
+	}
+      } while (_findnext(dirtag, &dirinfo) == 0);
+    _findclose(dirtag);
+    }
+
+  if (recurse == 0) return;
+
+  sprintf(search_name, "%s*.*", dname);          /*Look for subdirs*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag < 0) return;
+  do {
+    if ( ((dirinfo.attrib & (_A_HIDDEN | _A_SYSTEM)) == 0)  &&
+	 ((dirinfo.attrib & _A_SUBDIR) != 0)  &&
+	 strcmp(dirinfo.name,".") != 0 &&
+	 strcmp(dirinfo.name,"..") != 0) {
+      sprintf(found_name,"%s%s",dname,dirinfo.name);
+      strcat(found_name,"\\");
+      search(found_name,fname);
+      }
+    } while (_findnext(dirtag, &dirinfo) == 0);
+  _findclose(dirtag);
+}
+
+static int toobig(int n)
+{
+  if (n > path_length) {
+    printf("Cannot handle urls > %d characters in length\n", path_length);
+    exit(1);
+    }
+  return 0;
+}
+
+void scan_catch(char *dname, char *fname)
+{
+  int in_fd, urllen, conlen, filesize;  char catch_name[path_length];
+  int filecount = 0;
+  enum { filekind_redirected, filekind_text, filekind_other } filekind;
+  sprintf(catch_name, "%s%s", dname, fname);
+  in_fd = open(catch_name,O_RDONLY|O_BINARY);
+  if (in_fd < 0)
+    FatalError(1, "Couldn't open catch file \"%s\"", catch_name);
+  for (;;) {
+    filecount++;  if (filecount%100 == 0) { printf("%d\r", filecount);  fflush(stdout); }
+    if (read(in_fd, &urllen, sizeof(int)) != sizeof(int) || toobig(urllen) ||
+	read(in_fd, catch_name, urllen) != urllen ||
+	read(in_fd, &conlen, sizeof(int)) != sizeof(int) || toobig(conlen) ||
+	read(in_fd, catch_name, conlen) != conlen) break;
+    if (conlen >= 1 && *catch_name == '@')
+      filekind = filekind_redirected;
+    else {
+      if (conlen >= 4 && strncmp(catch_name, "text", 4) == 0)
+	filekind = filekind_text;
+      else
+	filekind = filekind_other;
+      if (read(in_fd, &filesize, sizeof(int)) != sizeof(int))
+	FatalError(1, "File read failed for size field");
+      if (filesize > buf_size)
+	FatalError(1, "File too large (%d > %d)", filesize, buf_size);
+      if (read(in_fd, buffer, filesize) != filesize)
+	FatalError(1, "Failed to read file data");
+      }
+    if (filekind == filekind_text) do_process(buffer, filesize);
+    }
+  close(in_fd);
+}
+
+static void driver (FILE * Trace, char *file_name)
+{
+  int pass;
+
+  buffer = (char *)Xmalloc (buf_size);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"",PassData[pass].name);
+      }
+
+  if (html_catch == 0) search(dirname, wildname);
+		  else scan_catch(dirname, wildname);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+
+
+  free (buffer);
+}
+
+void main (int argc, char **argv)
+{
+  int ch;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WHGpSD123f:d:b:T:I:t:m:N:c:n:s:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'H':
+	  html_catch = 1;
+	  break;
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'p':
+	  by_para = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (argc - optind == 1) {
+    strcpy(dirname,"");
+    strcpy(wildname,argv[optind]);
+    }
+  else if (argc - optind == 2) {
+    strcpy(dirname,argv[optind]);
+    strcpy(wildname,argv[optind+1]);
+    }
+  else FatalError(1, "Finder code requires directory and filespec.");
+
+  if (strrchr(wildname,'*') != NULL  ||  strrchr(wildname,'?') != NULL)
+    recurse = 1;
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  exit (0);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/Makefile.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/Makefile.in	(revision 21112)
@@ -0,0 +1,388 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+YACC = @YACC@
+GSDLOS = @gsdlos@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c                 mg_passes_4jni.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+
+MISC          = Makefile.in help.mg.src bool_parser.y warranty.src conditions.src
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgquery mg_weights_build mgstat mg_invf_dump mg_invf_dict \
+                mg_invf_rebuild mgdictlist mg_passes mg_perf_hash_build \
+                mg_compression_dict mg_text_estimate mg_fast_comp_dict \
+                mg_hilite_words mg_invf_merge mg_text_merge bool_tester \
+                mg_stem_idx mgstemidxlist mg_decompress_text 
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+                mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+                mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+                mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+                mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a libmgpass.a
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+help.mg.h : help.mg.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+warranty.h : warranty.src
+	sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+conditions.h : conditions.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+
+mg_hilite_words: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+bool_parser.c: bool_parser.y
+	$(YACC) $?
+	mv y.tab.c $@
+
+BOOL_OBJS = bool_tester$o bool_tree$o bool_parser$o bool_optimiser$o  \
+            term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o environment$o
+
+bool_tester: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+
+mgquery: $(QUERY_OBJS)	
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat:	$(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+
+mg_invf_rebuild: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+
+mgdictlist: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = \
+                mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+
+mgstemidxlist: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text.o query_term_list.o mg_files.o mg_errors.o \
+                text_get.o locallib.o backend.o stem_search.o \
+                term_lists.o stemmer.o weights.o invf_get.o lists.o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+
+mg_invf_merge: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+JNI_LIB_OBJS = \
+	mg_passes_4jni$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o  words$o environment$o\
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o 
+
+libmgpass.a: $(JNI_LIB_OBJS)
+	rm -f libmgpass.a
+	$(AR) cru libmgpass.a $(JNI_LIB_OBJS)
+	$(RANLIB) libmgpass.a
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC) libtextin.a libmgpass.a
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+  
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.c	(revision 21112)
@@ -0,0 +1,838 @@
+/**************************************************************************
+ *
+ * backend.c -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "filestats.h"
+#include "sptree.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+
+static File *
+OpenFile (char *base, char *suffix, unsigned long magic, int *ok)
+{
+  char FileName[512];
+  File *F;
+  sprintf (FileName, "%s%s", base, suffix);
+  if (!(F = Fopen (FileName, "rb", 0)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      mg_errno = MG_NOFILE;
+      MgErrorData (FileName);
+      if (ok)
+	*ok = 0;
+      return (NULL);
+    }
+  if (magic)
+    {
+      unsigned long m;
+      if (fread ((char *) &m, sizeof (m), 1, F->f) == 0)
+	{
+	  mg_errno = MG_READERR;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+      NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (m != magic)
+	{
+	  mg_errno = MG_BADMAGIC;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+    }
+  return (F);
+}
+
+
+static int 
+open_all_files (query_data * qd)
+{
+  int ok = 1;
+
+  qd->File_text = OpenFile (qd->textpathname, TEXT_SUFFIX, /* [RJM 06/97: text filename] */
+			    MAGIC_TEXT, &ok);
+  qd->File_fast_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				      TEXT_DICT_FAST_SUFFIX, MAGIC_FAST_DICT, NULL);
+  if (!qd->File_fast_comp_dict)
+    {
+      qd->File_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				     TEXT_DICT_SUFFIX, MAGIC_DICT, &ok);
+      qd->File_aux_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				    TEXT_DICT_AUX_SUFFIX, MAGIC_AUX_DICT, NULL);
+    }
+  else
+    qd->File_comp_dict = qd->File_aux_dict = NULL;
+
+  qd->File_stem = OpenFile (qd->pathname, INVF_DICT_BLOCKED_SUFFIX,
+			    MAGIC_STEM, &ok);
+
+  /* [RPAP - Jan 97: Stem Index Change]
+     These will fail if collection not built with stem indexes */
+  qd->File_stem1 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_1_SUFFIX,
+			     MAGIC_STEM_1, NULL);
+  qd->File_stem2 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_2_SUFFIX,
+			     MAGIC_STEM_2, NULL);
+  qd->File_stem3 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_3_SUFFIX,
+			     MAGIC_STEM_3, NULL);
+
+  qd->File_invf = OpenFile (qd->pathname, INVF_SUFFIX,
+			    MAGIC_INVF, &ok);
+
+  /* These will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_text_idx_wgt = OpenFile (qd->pathname, TEXT_IDX_WGT_SUFFIX,
+				    MAGIC_TEXI_WGT, NULL);
+  qd->File_weight_approx = OpenFile (qd->pathname, APPROX_WEIGHTS_SUFFIX,
+				     MAGIC_WGHT_APPROX, NULL);
+  if (qd->File_text_idx_wgt == NULL && qd->File_weight_approx == NULL)
+    qd->File_text_idx = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				  TEXT_IDX_SUFFIX, MAGIC_TEXI, NULL);
+  else
+    qd->File_text_idx = NULL;
+
+
+  if (!ok)
+    {
+      Fclose (qd->File_text);
+      if (qd->File_fast_comp_dict)
+	Fclose (qd->File_fast_comp_dict);
+      if (qd->File_comp_dict)
+	Fclose (qd->File_comp_dict);
+      Fclose (qd->File_stem);
+
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+
+      Fclose (qd->File_invf);
+      if (qd->File_text_idx_wgt)
+	Fclose (qd->File_text_idx_wgt);
+      if (qd->File_weight_approx)
+	Fclose (qd->File_weight_approx);
+      if (qd->File_text_idx)
+	Fclose (qd->File_text_idx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+static void 
+close_all_files (query_data * qd)
+{
+  Fclose (qd->File_text);
+  if (qd->File_fast_comp_dict)
+    Fclose (qd->File_fast_comp_dict);
+  if (qd->File_aux_dict)
+    Fclose (qd->File_aux_dict);
+  if (qd->File_comp_dict)
+    Fclose (qd->File_comp_dict);
+  Fclose (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    Fclose (qd->File_stem1);
+  if (qd->File_stem2)
+    Fclose (qd->File_stem2);
+  if (qd->File_stem3)
+    Fclose (qd->File_stem3);
+
+  Fclose (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    Fclose (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    Fclose (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    Fclose (qd->File_text_idx);
+}
+
+/* If textname is equal to null then name will be used instead */
+/* [RJM 06/97: text filename] */
+query_data *
+InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{
+  query_data *qd;
+  char *s;
+
+  if (textname == NULL) textname = name; /* [RJM 06/97: text filename] */
+
+  if (!(qd = Xmalloc (sizeof (query_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) qd, sizeof (*qd));
+
+  qd->mem_in_use = qd->max_mem_in_use = 0;
+
+  qd->doc_pos = qd->buf_in_use = 0;
+  qd->TextBufferLen = 0;
+  qd->DL = NULL;
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  qd->TL = NULL;
+  qd->QTL = NULL;
+
+  qd->TextBuffer = NULL;
+
+  qd->tot_hops_taken = 0;
+  qd->tot_num_of_ptrs = 0;
+  qd->tot_num_of_accum = 0;
+  qd->tot_num_of_terms = 0;
+  qd->tot_num_of_ans = 0;
+  qd->tot_text_idx_lookups = 0;
+
+  qd->hops_taken = 0;
+  qd->num_of_ptrs = 0;
+  qd->num_of_accum = 0;
+  qd->num_of_terms = 0;
+  qd->num_of_ans = 0;
+  qd->text_idx_lookups = 0;
+
+  qd->pathname = NULL; /* RJM 06/97: text filename] */
+  qd->textpathname = NULL; /* RJM 06/97: text filename] */
+
+  s = strrchr (dir, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 1)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 1)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      sprintf (qd->pathname, "%s%s", dir, name);
+      sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+    }
+
+  else
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 2)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 2)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      if (dir == NULL || dir[0] == '\0') {
+	sprintf (qd->pathname, "%s", name);
+	sprintf (qd->textpathname, "%s", textname); /* [RJM 06/97: text filename] */
+      } else {
+	sprintf (qd->pathname, "%s%s", dir, name);
+	sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+      }
+#else
+      sprintf (qd->pathname, "%s/%s", dir, name);
+      sprintf (qd->textpathname, "%s/%s", dir, textname); /* [RJM 06/97: text filename] */
+#endif
+    }
+
+  if (open_all_files (qd) == -1)
+    {
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->Start);
+
+  /* Initialise the stemmed dictionary system */
+  if (!(qd->sd = ReadStemDictBlk (qd->File_stem)))
+    {
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if ((qd->sd->sdh.indexed & 7) && qd->File_stem1 && qd->File_stem2 && qd->File_stem3)
+    {
+      if (!(qd->sd->stem1 = ReadStemIdxBlk (qd->File_stem1)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem2 = ReadStemIdxBlk (qd->File_stem2)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem3 = ReadStemIdxBlk (qd->File_stem3)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      }
+  else if (qd->sd->sdh.indexed != 0)
+    {
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  else
+    {
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+      qd->File_stem1 = NULL;
+      qd->File_stem2 = NULL;
+      qd->File_stem3 = NULL;
+      qd->sd->stem1 = NULL;
+      qd->sd->stem2 = NULL;
+      qd->sd->stem3 = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->StemDict);
+  if (qd->File_weight_approx)
+    {
+      if (!(qd->awd = LoadDocWeights (qd->File_weight_approx,
+				      qd->sd->sdh.num_of_docs)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+    }
+  else
+    qd->awd = NULL;
+
+
+  if (iqt)
+    GetTime (&iqt->ApproxWeights);
+
+  if (!(qd->cd = LoadCompDict (qd->File_comp_dict, qd->File_aux_dict,
+			       qd->File_fast_comp_dict)))
+    {
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->CompDict);
+
+  if (!(qd->id = InitInvfFile (qd->File_invf, qd->sd)))
+    {
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  if ((qd->File_text_idx_wgt == NULL || qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      mg_errno = MG_INVERSION;
+      return (NULL);
+    }
+  if (iqt)
+    GetTime (&iqt->Invf);
+
+  if (!(qd->td = LoadTextData (qd->File_text, qd->File_text_idx_wgt,
+			       qd->File_text_idx)))
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+
+/*
+
+This code is based on the TREC_MODE code below to read the .paragraph
+file to determine what document numbers correspond to what paragraphs.
+This code is more space efficient, reading in the .paragraph file
+into memory as an accumulate docnum array.  Eg.  the .paragraph may contain
+
+	[5 3 6 4 7 9 4]
+
+indicating the first document has 5 paragraphs, the next 3, etc.
+This will be stored in memory as
+
+	[0 5 8 14 18 25 34 38]
+
+so a binary search can be performed.  The first 0 is for convenience;
+it prevents testing boundary conditions.
+
+
+The TREC_MODE code does this differently; it stores the array
+
+	[1 1 1 1 1 2 2 2 3 3 3 3 3 3 ....]
+
+allowing directy paragraph to docnum conversion, at the expense
+of memory.
+
+*/ 
+  qd->paragraph = NULL;
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      unsigned long magic;
+      FILE *paragraph;
+      int i;
+      char paraFile[512];
+
+      sprintf(paraFile, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+      paragraph = fopen(paraFile, "rb");
+      if (!paragraph)
+	FatalError(1, "Unable to open 'paraFile'.", paraFile);
+
+      fread((void *)&magic, sizeof(magic), 1, paragraph);
+      qd->paragraph = Xmalloc((qd->td->cth.num_of_docs+1)*sizeof(int));
+      qd->paragraph[0] = 0;
+      for (i = 1; i <= qd->td->cth.num_of_docs; i++)
+	{
+	  int count;
+
+	  if (fread((void *)&count, sizeof(count), 1, paragraph) != 1)
+	    FatalError(1, "Unexpected EOF while reading '%s'.", paraFile);
+	  NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  qd->paragraph[i] = qd->paragraph[i-1]+count;
+	}
+
+      fclose (paragraph); /* [RJM 07/98: Memory Leak] */
+    }
+  
+  
+#endif
+
+#ifdef TREC_MODE
+  {
+    extern char *trec_ids;
+    extern long *trec_paras;
+    int size;
+    char FileName[512];
+    FILE *f;
+    if (!strstr (qd->pathname, "trec"))
+      goto error;
+    sprintf (FileName, "%s%s", qd->pathname, ".DOCIDS");
+    if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+      {
+	Message ("Unable to open \"%s\"", FileName);
+	goto error;
+      }
+    fseek (f, 0, 2);
+    size = ftell (f);
+    fseek (f, 0, 0);
+    trec_ids = Xmalloc (size);
+    if (!trec_ids)
+      {
+	fclose (f);
+	goto error;
+      }
+    fread (trec_ids, 1, size, f);
+    fclose (f);
+    if (qd->id->ifh.InvfLevel == 3)
+      {
+	int i, d;
+	unsigned long magic;
+	trec_paras = Xmalloc (qd->sd->sdh.num_of_docs * sizeof (long));
+	if (!trec_paras)
+	  {
+	    Xfree (trec_ids);
+	    trec_ids = NULL;
+	    goto error;
+	  }
+	sprintf (FileName, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+	if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	  {
+	    Message ("Unable to open \"%s\"", FileName);
+	    goto error;
+	  }
+	if (fread ((char *) &magic, sizeof (magic), 1, f) != 1 ||
+	    NTOHUL(magic) != MAGIC_PARAGRAPH)  /* [RPAP - Jan 97: Endian Ordering] */
+	  {
+	    fclose (f);
+	    Message ("Bad magic number in \"%s\"", FileName);
+	    goto error;
+	  }
+
+	for (d = i = 0; i < qd->td->cth.num_of_docs; i++)
+	  {
+	    int count;
+	    if (fread ((char *) &count, sizeof (count), 1, f) != 1)
+	      {
+		fclose (f);
+		goto error;
+	      }
+	    NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	    while (count--)
+	      trec_paras[d++] = i;
+	  }
+	fclose (f);
+      }
+    goto ok;
+  error:
+    if (trec_ids)
+      Xfree (trec_ids);
+    if (trec_paras)
+      Xfree (trec_paras);
+    trec_ids = NULL;
+    trec_paras = NULL;
+  ok:
+    ;
+  }
+#endif
+
+  if (iqt)
+    GetTime (&iqt->Text);
+
+  return (qd);
+}
+
+
+
+
+
+
+/*
+ * Change the amount of memory currently in use
+ *
+ */
+void 
+ChangeMemInUse (query_data * qd, long delta)
+{
+  qd->mem_in_use += delta;
+  if (qd->mem_in_use > qd->max_mem_in_use)
+    qd->max_mem_in_use = qd->mem_in_use;
+}
+
+
+void 
+FinishQuerySystem (query_data * qd)
+{
+/* [RJM 07/98: Memory Leak] */
+#if defined(PARADOCNUM) || defined(NZDL)
+  if (qd->paragraph != NULL) {
+    Xfree (qd->paragraph);
+    qd->paragraph = NULL;
+  }
+#endif
+
+  FreeTextData (qd->td);
+  FreeInvfData (qd->id);
+  FreeCompDict (qd->cd);
+  if (qd->awd)
+    FreeWeights (qd->awd);
+  FreeStemDict (qd->sd);
+  close_all_files (qd);
+  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+  Xfree (qd->pathname);
+  FreeQueryDocs (qd);
+  if (qd->TL != NULL) FreeTermList(&qd->TL);         /* [RJM 07/98: Memory Leak] */
+  if (qd->QTL != NULL) FreeQueryTermList(&qd->QTL);  /* [RJM 07/98: Memory Leak] */
+  Xfree (qd);
+
+  /* other global stuff hanging around */
+  MgErrorDeinit ();
+}
+
+
+void 
+ResetFileStats (query_data * qd)
+{
+  ZeroFileStats (qd->File_text);
+  if (qd->File_comp_dict)
+    ZeroFileStats (qd->File_comp_dict);
+  if (qd->File_fast_comp_dict)
+    ZeroFileStats (qd->File_fast_comp_dict);
+  ZeroFileStats (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    ZeroFileStats (qd->File_stem1);
+  if (qd->File_stem2)
+    ZeroFileStats (qd->File_stem2);
+  if (qd->File_stem3)
+    ZeroFileStats (qd->File_stem3);
+
+  ZeroFileStats (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    ZeroFileStats (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    ZeroFileStats (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    ZeroFileStats (qd->File_text_idx);
+}
+
+
+void 
+TransFileStats (query_data * qd)
+{
+  qd->File_text->Current = qd->File_text->Cumulative;
+  if (qd->File_comp_dict)
+    qd->File_comp_dict->Current = qd->File_comp_dict->Cumulative;
+  if (qd->File_fast_comp_dict)
+    qd->File_fast_comp_dict->Current = qd->File_fast_comp_dict->Cumulative;
+  qd->File_stem->Current = qd->File_stem->Cumulative;
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    qd->File_stem1->Current = qd->File_stem1->Cumulative;
+  if (qd->File_stem2)
+    qd->File_stem2->Current = qd->File_stem2->Cumulative;
+  if (qd->File_stem3)
+    qd->File_stem3->Current = qd->File_stem3->Cumulative;
+
+  qd->File_invf->Current = qd->File_invf->Cumulative;
+  if (qd->File_text_idx_wgt)
+    qd->File_text_idx_wgt->Current = qd->File_text_idx_wgt->Cumulative;
+  if (qd->File_weight_approx)
+    qd->File_weight_approx->Current = qd->File_weight_approx->Cumulative;
+  if (qd->File_text_idx)
+    qd->File_text_idx->Current = qd->File_text_idx->Cumulative;
+}
+
+
+void 
+FreeTextBuffer (query_data * qd)
+{
+  if (qd->TextBuffer)
+    {
+      Xfree (qd->TextBuffer);
+      ChangeMemInUse (qd, -qd->TextBufferLen);
+    }
+  qd->TextBuffer = NULL;
+  qd->TextBufferLen = 0;
+}
+
+void 
+FreeQueryDocs (query_data * qd)
+{
+  qd->doc_pos = 0;
+  qd->buf_in_use = 0;
+  if (qd->DL)
+    {
+      int i;
+      for (i = 0; i < qd->DL->num; i++)
+	if (qd->DL->DE[i].CompTextBuffer)
+	  {
+	    Xfree (qd->DL->DE[i].CompTextBuffer);
+	    qd->DL->DE[i].CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -qd->DL->DE[i].Len);
+	  }
+      Xfree (qd->DL);
+    }
+  qd->DL = NULL;
+  FreeTextBuffer (qd);
+}
+
+int 
+LoadCompressedText (query_data * qd, int max_mem)
+{
+  DocEntry *DE;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+  if (!DE->CompTextBuffer)
+    {
+      int i;
+      DocEntry *de;
+      for (i = 0, de = qd->DL->DE; i < qd->DL->num; i++, de++)
+	if (de->CompTextBuffer)
+	  {
+	    Xfree (de->CompTextBuffer);
+	    de->CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -de->Len);
+	  }
+      if (LoadBuffers (qd, &qd->DL->DE[qd->doc_pos], max_mem,
+		       qd->DL->num - qd->doc_pos) == -1)
+	return -1;
+    }
+  return 0;
+}
+
+int 
+GetDocNum (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].DocNum;
+}
+
+DocEntry *
+GetDocChain (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+  return &(qd->DL->DE[qd->doc_pos]);
+}
+
+float 
+GetDocWeight (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Weight;
+}
+
+long 
+GetDocCompLength (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Len;
+}
+
+
+u_char *
+GetDocText (query_data * qd, unsigned long *len)
+{
+  DocEntry *DE;
+  int ULen;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+
+  if (!DE->CompTextBuffer)
+    {
+      fprintf (stderr, "The compressed text buffer is NULL\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  FreeTextBuffer (qd);
+
+  qd->TextBufferLen = (int) (qd->td->cth.ratio * 1.01 *
+			     DE->Len) + 100;
+  if (!(qd->TextBuffer = Xmalloc (qd->TextBufferLen)))
+    {
+      fprintf (stderr, "No memory for TextBuffer\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  DecodeText (qd->cd, (u_char *) (DE->CompTextBuffer), DE->Len,
+	      (u_char *) (qd->TextBuffer), &ULen);
+  qd->TextBuffer[ULen] = '\0';
+
+  if (ULen >= qd->TextBufferLen)
+    {
+      fprintf (stderr, "%d >= %d\n", ULen, qd->TextBufferLen);
+      mg_errno = MG_BUFTOOSMALL;
+      return (NULL);
+    }
+
+  if (len)
+    *len = ULen;
+
+  return qd->TextBuffer;
+}
+
+int 
+NextDoc (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return 0;
+  qd->doc_pos++;
+  return qd->doc_pos < qd->DL->num;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/backend.h	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * backend.h -- Underlying routines and datastructures for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef BACKEND_H
+#define BACKEND_H
+
+#include "sysfuncs.h"
+
+#include "timing.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	/* N parameter for decoding inverted file entries */
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+     *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    u_char *word_data[2];
+    u_char **words[2];
+    int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+						   for the hybrid methods */
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    u_char ***values[2];
+    u_char *escape[2];
+    huff_data *chars_huff[2];
+    u_long **chars_vals[2];
+    huff_data *lens_huff[2];
+    u_long **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx   /* [RPAP - Jan 97: Stem Index Change] */
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemDict;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+
+typedef struct RankedQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;           /* use exact weights for ranking or not */
+    long MaxDocsToRetrieve;	/* may be -1 for all */
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		/* 'A' = array,  'S' = splay tree,  'H' = hash_table */
+    long MaxAccums;		/* may be -1 for all */
+    long MaxTerms;		/* may be -1 for all */
+    int StopAtMaxAccum; /* Stop at maximum accumulator or not */
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo
+  {
+    long MaxDocsToRetrieve;
+  }
+BooleanQueryInfo;
+
+
+/* [TS:24/Aug/94] - maximum number of characters in term string */
+#define MAXTERMSTRLEN 1023
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+#if defined(PARADOCNUM) || defined(NZDL)
+    int *paragraph;
+#endif
+    char *pathname;
+    char *textpathname; /* [RJM 06/97: text filename] */
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		/* [TS:Oct/94] - so term list for query can easily be accessed */
+    u_char *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;    /* [RPAP - Feb 97: Term Frequency] */
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+/* [RJM 06/97: text filename] */
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void FreeTextBuffer (query_data * qd);
+
+void FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+u_char *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.c	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994,95  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:20  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1995/07/27  04:54:55  tes
+   * Committing optimiser before I add the code
+   * to find diff-nodes in any kind of tree.
+   *
+   * Revision 1.3  1994/10/20  00:02:55  tes
+   * Fixed bug with bool_optimiser:
+   *   - it couldn't handle "false | false"
+   *   - it couldn't handle "true & true"
+   * These would collapse into an empty list and cause an error.
+   *
+   * Revision 1.2  1994/10/18  06:11:02  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:28  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bool_tree.h"
+#include "term_lists.h"
+#include "messages.h"
+
+typedef int (*CompFunc) (const void *, const void *);
+typedef void (*TreeFunc) (bool_tree_node *);
+
+/* --- routines --- */
+static void PermeateNots (bool_tree_node * tree);
+static void DoubleNeg (bool_tree_node * not_tree);
+static void AndDeMorgan (bool_tree_node * not_tree);
+static void OrDeMorgan (bool_tree_node * not_tree);
+static int AndDistribute (bool_tree_node * tree);
+static void DNF_Distribute (bool_tree_node * tree);
+static void ReplaceWithChildren (bool_tree_node * tree);
+static void CollapseTree (bool_tree_node * tree);
+static void AndTrueFalse (bool_tree_node * and_tree);
+static void OrTrueFalse (bool_tree_node * or_tree);
+static void TF_Idempotent (bool_tree_node * tree);
+static void AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func);
+static void DiffDNF (bool_tree_node * or_node);
+static void DiffTree (bool_tree_node * tree);
+static void AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp);
+static void AndOrSort (bool_tree_node * tree, TermList * tl);
+static void AndSortDNF (bool_tree_node * or_node, TermList * tl);
+static void MarkOrTerms (bool_tree_node * tree, TermList * tl);
+static int calc_sum_ft (bool_tree_node * term_tree, TermList * tl);
+static int AndOrSort_comp (const void *A, const void *B);
+static int AndSort_comp (const void *A, const void *B);
+
+/* =========================================================================
+ * Function: OptimiseBoolTree
+ * Description: 
+ *      For case 2:
+ *        Do three major steps:
+ *        (i) put into standard form
+ *            -> put into DNF (disjunctive normal form - or of ands)
+ *            Use DeMorgan's, Double-negative, Distributive rules
+ *        (ii) simplify
+ *             apply idempotency rules
+ *        (iii) ameliorate
+ *              convert &! to diff nodes, order terms by frequency,...
+ *     Could also do the matching idempotency laws i.e. ...
+ *     (A | A), (A | !A), (A & !A), (A & A), (A & (A | B)), (A | (A & B)) 
+ *     Job for future.... ;-) 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+OptimiseBoolTree (bool_tree_node * parse_tree, TermList * tl, int type)
+{
+  switch (type)
+    {
+    case 1:
+      /* --- sorts for and of or-terms --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      CollapseTree (parse_tree);
+      DiffTree (parse_tree);
+      MarkOrTerms (parse_tree, tl);
+      AndOrSort (parse_tree, tl);
+      break;
+    case 2:
+      /* --- put into DNF --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      DNF_Distribute (parse_tree);
+      CollapseTree (parse_tree);
+      AndSortDNF (parse_tree, tl);
+      DiffDNF (parse_tree);
+      break;
+    default:
+      /* --- do nothing --- */
+      break;
+    }				/*switch */
+}				/*OptimiseBoolTree */
+
+/* =========================================================================
+ * Function: DoubleNeg
+ * Description: 
+ *      !(!(a) = a
+ *      Assumes binary tree.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+DoubleNeg (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *sibling = BOOL_SIBLING (not_tree);
+
+  /* copy a into not tree */
+  bcopy (a, not_tree, sizeof (bool_tree_node));
+
+  BOOL_SIBLING (not_tree) = sibling;
+
+  Xfree (child);
+  Xfree (a);
+
+}
+
+/* =========================================================================
+ * Function: AndDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'and'  i.e. !(a & b) <=> (!a) | (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+AndDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'or' */
+  BOOL_TAG (not_tree) = N_or;
+
+  /* change 'and' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: OrDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'or' i.e. !(a | b) <=> (!a) & (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+OrDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'and' */
+  BOOL_TAG (not_tree) = N_and;
+
+  /* change 'or' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: PermeateNots
+ * Description: 
+ *      Use DeMorgan's and Double-negative
+ *      Assumes tree in binary form (i.e. No ands/ors collapsed)
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PermeateNots (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if (BOOL_TAG (tree) == N_not)
+    {
+      switch (BOOL_TAG (BOOL_CHILD (tree)))
+	{
+	case N_not:
+	  DoubleNeg (tree);
+/** NOTE: I do not go on to do the children **/
+	  /* I permeate the nots on the tree here & leave it at that */
+	  /* In this case I do not know what type 'tree' will end up */
+	  PermeateNots (tree);
+	  return;
+	case N_and:
+	  AndDeMorgan (tree);
+	  break;
+	case N_or:
+	  OrDeMorgan (tree);
+	  break;
+	default:
+	  /* leave alone */
+	  break;
+	}
+    }
+
+  /* Permeate Nots for children */
+  if (BOOL_HAS_CHILD (tree))
+    {
+      PermeateNots (BOOL_CHILD (tree));
+      PermeateNots (BOOL_SIBLING (BOOL_CHILD (tree)));
+    }
+
+}
+
+/* =========================================================================
+ * Function: DNF_Distribute
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+#define ITER_LIMIT 500		/* if reach limit then probably in trouble */
+
+static void
+DNF_Distribute (bool_tree_node * tree)
+{
+  int i = 0;
+
+  /* multiple passes until no change */
+  /* not very smart :-( */
+  while (1)
+    {
+      if (AndDistribute (tree) == 0)
+	return;
+      if (i == ITER_LIMIT)
+	FatalError (1, "DNF infinite loop");
+      i++;
+    }				/*while */
+
+}
+
+/* =========================================================================
+ * Function: AndDistribute
+ * Description: 
+ *      (a | b) & A <=> (a & A) | (b & A)
+ * Input: 
+ *      binary tree of "AND" , "OR"s.
+ * Output: 
+ *      return 1 if changed the tree
+ *      return 0 if there was NO change (no distributive rule to apply)
+ * ========================================================================= */
+
+static int
+AndDistribute (bool_tree_node * tree)
+{
+  if (!tree)
+    return 0;
+
+  if (BOOL_TAG (tree) != N_and)
+    {
+      /* try the children if got some */
+      if (BOOL_TAG (tree) == N_or)
+	{
+	  bool_tree_node *left_child = BOOL_CHILD (tree);
+	  int left = AndDistribute (left_child);
+	  int right = AndDistribute (BOOL_SIBLING (left_child));
+	  return (left || right);
+	}
+
+      /* no children, then no change */
+      return 0;
+    }
+
+  /* AND case */
+  {
+    /* we have to check the children to see if can distribute */
+    bool_tree_node *left_child = BOOL_CHILD (tree);
+    bool_tree_node *right_child = BOOL_SIBLING (left_child);
+    int swap = 0;
+
+    /* If "or" on the right side then do a swap */
+    /* This is done for code simplicity */
+    if (BOOL_TAG (right_child) == N_or)
+      {
+	/* do a swap */
+	bool_tree_node *temp;
+	temp = right_child;
+	right_child = left_child;
+	left_child = temp;
+
+	BOOL_SIBLING (left_child) = right_child;
+	BOOL_SIBLING (right_child) = NULL;
+	BOOL_CHILD (tree) = left_child;
+
+	swap = 1;
+      }
+
+    /* do the actual distribution */
+    if (swap || BOOL_TAG (left_child) == N_or)
+      {
+	bool_tree_node *A = right_child;
+	bool_tree_node *A_copy = CopyBoolTree (A);
+	bool_tree_node *a = BOOL_CHILD (left_child);
+	bool_tree_node *b = BOOL_SIBLING (a);
+
+	/* then can distribute away ... */
+	BOOL_TAG (tree) = N_or;
+	BOOL_TAG (left_child) = N_and;
+	BOOL_SIBLING (left_child) = CreateBoolTreeNode (N_and, b, A);
+	BOOL_SIBLING (a) = A_copy;
+
+	AndDistribute (tree);
+	return 1;
+      }
+
+    /* if "and" of something other than "or" s then try the children */
+    {
+      int left = AndDistribute (left_child);
+      int right = AndDistribute (right_child);
+      return (left || right);
+    }
+  }				/* AND case */
+
+}				/*AndDistribute */
+
+/* =========================================================================
+ * Function: CollapseTree
+ * Description: 
+ *      Collapse "and" and "or"s.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+CollapseTree (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if ((BOOL_TAG (tree) == N_and) || (BOOL_TAG (tree) == N_or))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  while (BOOL_TAG (child) == BOOL_TAG (tree))
+	    {
+	      ReplaceWithChildren (child);
+	    }
+	  CollapseTree (child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+  /* assume "not"s permeated */
+  return;
+}
+
+/* =========================================================================
+ * Function: ReplaceWithChildren
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+ReplaceWithChildren (bool_tree_node * tree)
+{
+  bool_tree_node *next = BOOL_SIBLING (tree);
+  bool_tree_node *child = BOOL_CHILD (tree);
+  bool_tree_node *last = LastBoolSibling (child);
+
+  bcopy ((char *) child, (char *) tree, sizeof (bool_tree_node));
+  Xfree (child);
+  BOOL_SIBLING (last) = next;
+
+}
+
+/* =========================================================================
+ * Function: TF_Idempotent
+ * Description: 
+ *      Applies Idempotency laws which refer to True or False (hence "TF")
+ *      uses the true or false idempotent rules
+ * Input: 
+ *      operates on n-ary trees (not restricted to binary)
+ * Output: 
+ * ========================================================================= */
+
+static void
+TF_Idempotent (bool_tree_node * tree)
+{
+  bool_tree_node *child = NULL;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	{
+	  child = BOOL_CHILD (tree);
+	  TF_Idempotent (child);
+	}
+
+      /* --- process node --- */
+      switch (BOOL_TAG (tree))
+	{
+	case N_not:
+	  child = BOOL_CHILD (tree);
+
+	  /* if true then falsify */
+	  if (BOOL_TAG (child) == N_all)
+	    {
+	      BOOL_TAG (tree) = N_none;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  /* if false then trueify */
+	  if (BOOL_TAG (child) == N_none)
+	    {
+	      BOOL_TAG (tree) = N_all;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  break;
+	case N_and:
+	  AndTrueFalse (tree);
+	  break;
+	case N_or:
+	  OrTrueFalse (tree);
+	  break;
+
+	default:
+	  /* do nothing */
+	  break;
+	}			/*switch */
+
+      tree = BOOL_SIBLING (tree);
+    }				/*while */
+}				/*TF_Idempotent */
+
+/* =========================================================================
+ * Function: AndTrueFalse
+ * Description: 
+ *      If any "false" then falsify whole tree 
+ *      If any "true" then delete them
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndTrueFalse (bool_tree_node * and_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_none)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (and_tree)));
+	  BOOL_TAG (and_tree) = N_none;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_all)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+
+	  /* --- link over child --- */
+	  if (!prev)
+	    {
+	      BOOL_CHILD (and_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+
+	  Xfree (child);
+	  child = sibling;
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (and_tree);
+
+  /* if no children ... */
+  /* => all have been true */
+  if (!child)
+    {
+      BOOL_TAG (and_tree) = N_all;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (and_tree);
+      bcopy ((char *) child, (char *) and_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (and_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: OrTrueFalse
+ * Description: 
+ *      If any "true" then trueify whole tree 
+ *      If any "false" then delete them 
+ *      ***** NOTE: This is a reflection of AndTrueFalse *****
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+OrTrueFalse (bool_tree_node * or_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (or_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_all)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (or_tree)));
+	  BOOL_TAG (or_tree) = N_all;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_none)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+	  if (!prev)
+	    {
+	      BOOL_CHILD (or_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+	  Xfree (child);
+	  child = sibling;
+
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (or_tree);
+
+  /* if no children ... */
+  /* => all have been false */
+  if (!child)
+    {
+      BOOL_TAG (or_tree) = N_none;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (or_tree);
+      bcopy ((char *) child, (char *) or_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (or_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: AndNotToDiff
+ * Description: 
+ *      Convert an "And" tree with possibly some negated terms into
+ *      a "And - neg-terms". 
+ * Input: 
+ *      Expects "And" of only literals - as tree should be in DNF form.
+ *      This means that no recursion is necessary.
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func)
+{
+
+  /* -------------------------------------------------- */
+  /* --- Divide the children up into positive and negative literals --- */
+  /* Preserves ordering of children */
+  bool_tree_node *pos = NULL;
+  bool_tree_node *neg = NULL;
+  bool_tree_node *last_pos = NULL;
+  bool_tree_node *last_neg = NULL;
+
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+  while (child)
+    {
+      bool_tree_node *next = BOOL_SIBLING (child);
+      if (BOOL_TAG (child) == N_not)
+	{
+	  /* add to end of neg list */
+	  if (!neg)
+	    neg = child;
+	  else
+	    BOOL_SIBLING (last_neg) = child;
+	  last_neg = child;
+	}
+      else
+	{
+	  /* add to end of pos list */
+	  if (!pos)
+	    pos = child;
+	  else
+	    BOOL_SIBLING (last_pos) = child;
+	  last_pos = child;
+	}
+      child = next;
+    }				/*while */
+  if (last_pos)
+    BOOL_SIBLING (last_pos) = NULL;
+  if (last_neg)
+    BOOL_SIBLING (last_neg) = NULL;
+  /* -------------------------------------------------- */
+
+  if (!neg)			/* no negtives => no subtraction */
+    {
+      /* fix up parent's child link */
+      BOOL_CHILD (and_tree) = pos;
+      return;
+    }
+
+  if (!pos)			/* no positives = all negatives */
+    {
+      /* Choose one of the negs for the left-hand-side of the subtraction */
+      /* In particular, choose the first */
+      pos = neg;
+      neg = BOOL_SIBLING (neg);
+      BOOL_SIBLING (pos) = NULL;
+    }
+
+  /* Get rid of all '!' from not nodes */
+  /* Make a list out of the children */
+  {
+    bool_tree_node *n = neg;
+    bool_tree_node *next = NULL;
+
+    neg = BOOL_CHILD (neg);
+    while (n)
+      {
+	bool_tree_node *c = BOOL_CHILD (n);
+	next = BOOL_SIBLING (n);
+	Xfree (n);
+	if (next)
+	  BOOL_SIBLING (c) = BOOL_CHILD (next);
+	n = next;
+      }
+  }
+
+  /* --- restructure the tree with "and" and "diff" --- */
+  BOOL_TAG (and_tree) = N_diff;
+  if (!BOOL_SIBLING (pos))	/* i.e. only one pos */
+    {
+      /* (A & !B & !C & !D & ...) ==> A - (B,C,D,...) */
+
+      BOOL_CHILD (and_tree) = pos;
+      BOOL_SIBLING (pos) = neg;
+    }
+  else
+    {
+      /* (A & !C & B & !D & ...) ==> (A & B & ....) - (C, D, ...) */
+
+      bool_tree_node *n = CreateBoolNode (N_and);
+      BOOL_CHILD (and_tree) = n;
+      BOOL_CHILD (n) = pos;
+      BOOL_SIBLING (n) = neg;
+    }				/*if */
+
+  if (recurs_func)
+    {
+      recurs_func (BOOL_CHILD (and_tree));
+    }
+
+}				/*AndNotToDiff */
+
+/* =========================================================================
+ * Function: DiffTree
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ *      Unlike DiffDNF, this routine will search throughout the tree for
+ *      "and"-nodes. It will call AndNotToNeg with this as its recursive 
+ *      routine.
+ * Date: 27/July/95
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+DiffTree (bool_tree_node * tree)
+{
+  bool_tree_node *t = tree;
+
+  while (t)
+    {
+      if (BOOL_TAG (t) == N_and)
+	{
+	  AndNotToDiff (t, DiffTree);
+	}
+      t = BOOL_SIBLING (t);
+    }				/*while */
+
+}				/*DiffTree */
+
+/* =========================================================================
+ * Function: DiffDNF
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ * Input: 
+ *      A DNF tree
+ * Output: 
+ *      A tree possibly with some diffs in it.
+ * ========================================================================= */
+
+static void
+DiffDNF (bool_tree_node * DNF_tree)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndNotToDiff (child, NULL);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndNotToDiff (DNF_tree, NULL);
+    }
+}
+
+/* =========================================================================
+ * Function: AndSort_comp 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+/* Module variable, "term_list",  used as parameter for comparison function */
+/* The comparison function only allows certain fixed parameters */
+static TermList *term_list;
+
+static int
+AndSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) != N_term)
+    return -1;
+
+  if (BOOL_TAG (*a) != N_term && BOOL_TAG (*b) == N_term)
+    return 1;
+
+  return 0;
+}				/*AndSort_comp */
+
+/* =========================================================================
+ * Function: AndSort
+ * Description: 
+ *      Sort the list of nodes by increasing doc_count 
+ *      Using some Neil Sharman code - pretty straight forward.
+ *      Note: not-terms are sent to the end of the list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp)
+{
+  bool_tree_node **list = NULL;
+  bool_tree_node *child = NULL;
+  int count = 0;
+  int i = 0;
+
+  if (BOOL_TAG (and_tree) != N_and)
+    return;
+
+  /* --- Convert linked list into an array for sorting --- */
+  child = BOOL_CHILD (and_tree);
+  count = 0;
+  while (child)
+    {
+      count++;
+      child = BOOL_SIBLING (child);
+    }
+
+  if (!(list = Xmalloc (sizeof (bool_tree_node *) * count)))
+    return;
+
+  child = BOOL_CHILD (and_tree);
+  for (i = 0; i < count; i++)
+    {
+      list[i] = child;
+      child = BOOL_SIBLING (child);
+    }
+  /* ----------------------------------------------------- */
+
+  term_list = tl;
+  qsort (list, count, sizeof (bool_tree_node *), comp);
+
+  /* --- Convert array back into linked list --- */
+  for (i = 0; i < count - 1; i++)
+    BOOL_SIBLING (list[i]) = list[i + 1];
+  BOOL_SIBLING (list[count - 1]) = NULL;
+  BOOL_CHILD (and_tree) = list[0];
+  Xfree (list);
+  /* ----------------------------------------------------- */
+
+}				/*AndSort */
+
+/* =========================================================================
+ * Function: AndSortDNF
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+static void
+AndSortDNF (bool_tree_node * DNF_tree, TermList * tl)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndSort (child, tl, AndSort_comp);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndSort (DNF_tree, tl, AndSort_comp);
+    }
+}
+
+/* =========================================================================
+ * Function: MarkOrTerms
+ * Description: 
+ *      Mark or-nodes whose children are all terms.
+ *      Also store the sum of the ft's of these nodes.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+MarkOrTerms (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_TAG (tree) == N_or)
+    {
+      int found_non_term = 0;
+      bool_tree_node *child = BOOL_CHILD (tree);
+
+      /* --- check to see if got all terms --- */
+      while (child)
+	{
+	  if (BOOL_TAG (child) != N_term)
+	    {
+	      found_non_term = 1;
+	    }
+	  if (BOOL_HAS_CHILD (tree))
+	    {
+	      MarkOrTerms (child, tl);
+	    }
+	  child = BOOL_SIBLING (child);
+	}			/*while */
+
+      if (!found_non_term)
+	{
+	  /* Must have all terms for the OR node */
+	  /* So change its tag and calculate its sum */
+	  BOOL_TAG (tree) = N_or_terms;
+	  BOOL_SUM_FT (tree) = calc_sum_ft (tree, tl);
+	}			/*if */
+      /* ------------------------------------- */
+
+    }				/*if OR node */
+
+  else if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  MarkOrTerms (child, tl);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+}				/*MarkOrTerms */
+
+/* =========================================================================
+ * Function: calc_sum_ft
+ * Description: 
+ *      Sums up the ft's of all the terms 
+ * Input: 
+ *      Assumes that the tree only contains terms in it 
+ * Output: 
+ *      the ft sum
+ * ========================================================================= */
+static int
+calc_sum_ft (bool_tree_node * term_tree, TermList * tl)
+{
+  int sum_ft = 0;
+  bool_tree_node *child = BOOL_CHILD (term_tree);
+  while (child)
+    {
+      sum_ft += GetNthWE (tl, BOOL_TERM (child))->doc_count;
+      child = BOOL_SIBLING (child);
+    }
+  return sum_ft;
+}
+
+/* =========================================================================
+ * Function: AndOrSort
+ * Description: 
+ *      This is like an AndSort of just terms. 
+ *      The difference is that we are now also 
+ *      considering sorting with <or of terms> by using the sum of f_t's.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+AndOrSort (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+
+      if (BOOL_TAG (tree) == N_and)
+	{
+	  AndSort (tree, tl, AndOrSort_comp);
+	}
+
+      /* Check out any children down the tree */
+      /* Probably a waste of time */
+      {
+	bool_tree_node *child = BOOL_CHILD (tree);
+	while (child)
+	  {
+	    AndOrSort (child, tl);
+	    child = BOOL_SIBLING (child);
+	  }
+      }				/*do the tree */
+
+    }				/*if has children */
+}				/*AndOrSort */
+
+/* =========================================================================
+ * Function: AndOrSort_comp
+ * Description: 
+ *      Comparison function used to order "term"s and "or_terms"s.
+ *      It's purpose is to be used by a sort routine.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int
+AndOrSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if ((BOOL_TAG (*a) == N_term || BOOL_TAG (*a) == N_or_terms) &&
+      (BOOL_TAG (*b) != N_term && BOOL_TAG (*b) != N_or_terms))
+    return -1;
+
+  if ((BOOL_TAG (*a) != N_term && BOOL_TAG (*a) != N_or_terms) &&
+      (BOOL_TAG (*b) == N_term || BOOL_TAG (*b) == N_or_terms))
+    return 1;
+
+  return 0;
+}				/*AndOrSort_comp */
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_optimiser.h	(revision 21112)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:23  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:21  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.1  1994/10/20  03:56:32  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/10/18  06:11:03  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:29  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_OPTIMISER_H
+#define BOOL_OPTIMISER_H
+
+#include "bool_tree.h"
+#include "term_lists.h"
+
+void OptimiseBoolTree (bool_tree_node * parse_tree, TermList * term_list, int type);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.c	(revision 21112)
@@ -0,0 +1,1196 @@
+
+/*  A Bison parser, made from bool_parser.y
+ by  GNU Bison version 1.27
+  */
+
+#define YYBISON 1  /* Identify Bison output.  */
+
+#define	TERM	257
+
+#line 24 "bool_parser.y"
+
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+
+#line 66 "bool_parser.y"
+typedef union {
+  char *text;
+  bool_tree_node *node;
+} YYSTYPE;
+#include <stdio.h>
+
+#ifndef __cplusplus
+#ifndef __STDC__
+#define const
+#endif
+#endif
+
+
+
+#define	YYFINAL		20
+#define	YYFLAG		-32768
+#define	YYNTBASE	11
+
+#define YYTRANSLATE(x) ((unsigned)(x) <= 257 ? yytranslate[x] : 16)
+
+static const char yytranslate[] = {     0,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     8,     2,     2,     2,     2,     9,     2,     4,
+     5,     6,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     7,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,    10,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     1,     3
+};
+
+#if YYDEBUG != 0
+static const short yyprhs[] = {     0,
+     0,     2,     4,     8,    10,    12,    14,    17,    21,    24,
+    26,    30
+};
+
+static const short yyrhs[] = {    15,
+     0,     3,     0,     4,    15,     5,     0,     6,     0,     7,
+     0,    12,     0,     8,    13,     0,    14,     9,    13,     0,
+    14,    13,     0,    13,     0,    15,    10,    14,     0,    14,
+     0
+};
+
+#endif
+
+#if YYDEBUG != 0
+static const short yyrline[] = { 0,
+    76,    80,    81,    82,    83,    86,    87,    90,    91,    92,
+    95,    96
+};
+#endif
+
+
+#if YYDEBUG != 0 || defined (YYERROR_VERBOSE)
+
+static const char * const yytname[] = {   "$","error","$undefined.","TERM","'('",
+"')'","'*'","'_'","'!'","'&'","'|'","query","term","not","and","or", NULL
+};
+#endif
+
+static const short yyr1[] = {     0,
+    11,    12,    12,    12,    12,    13,    13,    14,    14,    14,
+    15,    15
+};
+
+static const short yyr2[] = {     0,
+     1,     1,     3,     1,     1,     1,     2,     3,     2,     1,
+     3,     1
+};
+
+static const short yydefact[] = {     0,
+     2,     0,     4,     5,     0,     6,    10,    12,     1,     0,
+     7,     0,     9,     0,     3,     8,    11,     0,     0,     0
+};
+
+static const short yydefgoto[] = {    18,
+     6,     7,     8,     9
+};
+
+static const short yypact[] = {    10,
+-32768,    10,-32768,-32768,    10,-32768,-32768,     2,    -9,    14,
+-32768,    10,-32768,    10,-32768,-32768,     2,     4,    15,-32768
+};
+
+static const short yypgoto[] = {-32768,
+-32768,    -5,   -12,    18
+};
+
+
+#define	YYLAST		24
+
+
+static const short yytable[] = {    11,
+    14,    17,    13,    19,     1,     2,    16,     3,     4,     5,
+    12,    13,     1,     2,    20,     3,     4,     5,    15,    10,
+     0,     0,     0,    14
+};
+
+static const short yycheck[] = {     5,
+    10,    14,     8,     0,     3,     4,    12,     6,     7,     8,
+     9,    17,     3,     4,     0,     6,     7,     8,     5,     2,
+    -1,    -1,    -1,    10
+};
+/* -*-C-*-  Note some compilers choke on comments on `#line' lines.  */
+#line 3 "/usr/lib/bison.simple"
+/* This file comes from bison-1.27.  */
+
+/* Skeleton output parser for bison,
+   Copyright (C) 1984, 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place - Suite 330,
+   Boston, MA 02111-1307, USA.  */
+
+/* As a special exception, when this file is copied by Bison into a
+   Bison output file, you may use that output file without restriction.
+   This special exception was added by the Free Software Foundation
+   in version 1.24 of Bison.  */
+
+/* This is the parser code that is written into each bison parser
+  when the %semantic_parser declaration is not specified in the grammar.
+  It was written by Richard Stallman by simplifying the hairy parser
+  used when %semantic_parser is specified.  */
+
+#ifndef YYSTACK_USE_ALLOCA
+#ifdef alloca
+#define YYSTACK_USE_ALLOCA
+#else /* alloca not defined */
+#ifdef __GNUC__
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#else /* not GNU C.  */
+#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386))
+#define YYSTACK_USE_ALLOCA
+#include <alloca.h>
+#else /* not sparc */
+/* We think this test detects Watcom and Microsoft C.  */
+/* This used to test MSDOS, but that is a bad idea
+   since that symbol is in the user namespace.  */
+#if (defined (_MSDOS) || defined (_MSDOS_)) && !defined (__TURBOC__)
+#if 0 /* No need for malloc.h, which pollutes the namespace;
+	 instead, just don't use alloca.  */
+#include <malloc.h>
+#endif
+#else /* not MSDOS, or __TURBOC__ */
+#if defined(_AIX)
+/* I don't know what this was needed for, but it pollutes the namespace.
+   So I turned it off.   rms, 2 May 1997.  */
+/* #include <malloc.h>  */
+ #pragma alloca
+#define YYSTACK_USE_ALLOCA
+#else /* not MSDOS, or __TURBOC__, or _AIX */
+#if 0
+#ifdef __hpux /* haible@ilog.fr says this works for HPUX 9.05 and up,
+		 and on HPUX 10.  Eventually we can turn this on.  */
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#endif /* __hpux */
+#endif
+#endif /* not _AIX */
+#endif /* not MSDOS, or __TURBOC__ */
+#endif /* not sparc */
+#endif /* not GNU C */
+#endif /* alloca not defined */
+#endif /* YYSTACK_USE_ALLOCA not defined */
+
+#ifdef YYSTACK_USE_ALLOCA
+#define YYSTACK_ALLOC alloca
+#else
+#define YYSTACK_ALLOC malloc
+#endif
+
+/* Note: there must be only one dollar sign in this file.
+   It is replaced by the list of actions, each action
+   as one case of the switch.  */
+
+#define yyerrok		(yyerrstatus = 0)
+#define yyclearin	(yychar = YYEMPTY)
+#define YYEMPTY		-2
+#define YYEOF		0
+#define YYACCEPT	goto yyacceptlab
+#define YYABORT 	goto yyabortlab
+#define YYERROR		goto yyerrlab1
+/* Like YYERROR except do call yyerror.
+   This remains here temporarily to ease the
+   transition to the new meaning of YYERROR, for GCC.
+   Once GCC version 2 has supplanted version 1, this can go.  */
+#define YYFAIL		goto yyerrlab
+#define YYRECOVERING()  (!!yyerrstatus)
+#define YYBACKUP(token, value) \
+do								\
+  if (yychar == YYEMPTY && yylen == 1)				\
+    { yychar = (token), yylval = (value);			\
+      yychar1 = YYTRANSLATE (yychar);				\
+      YYPOPSTACK;						\
+      goto yybackup;						\
+    }								\
+  else								\
+    { yyerror ("syntax error: cannot back up"); YYERROR; }	\
+while (0)
+
+#define YYTERROR	1
+#define YYERRCODE	256
+
+#ifndef YYPURE
+#define YYLEX		yylex()
+#endif
+
+#ifdef YYPURE
+#ifdef YYLSP_NEEDED
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, &yylloc, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval, &yylloc)
+#endif
+#else /* not YYLSP_NEEDED */
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval)
+#endif
+#endif /* not YYLSP_NEEDED */
+#endif
+
+/* If nonreentrant, generate the variables here */
+
+#ifndef YYPURE
+
+int	yychar;			/*  the lookahead symbol		*/
+YYSTYPE	yylval;			/*  the semantic value of the		*/
+				/*  lookahead symbol			*/
+
+#ifdef YYLSP_NEEDED
+YYLTYPE yylloc;			/*  location data for the lookahead	*/
+				/*  symbol				*/
+#endif
+
+int yynerrs;			/*  number of parse errors so far       */
+#endif  /* not YYPURE */
+
+#if YYDEBUG != 0
+int yydebug;			/*  nonzero means print parse trace	*/
+/* Since this is uninitialized, it does not stop multiple parsers
+   from coexisting.  */
+#endif
+
+/*  YYINITDEPTH indicates the initial size of the parser's stacks	*/
+
+#ifndef	YYINITDEPTH
+#define YYINITDEPTH 200
+#endif
+
+/*  YYMAXDEPTH is the maximum size the stacks can grow to
+    (effective only if the built-in stack extension method is used).  */
+
+#if YYMAXDEPTH == 0
+#undef YYMAXDEPTH
+#endif
+
+#ifndef YYMAXDEPTH
+#define YYMAXDEPTH 10000
+#endif
+
+
+/* Define __yy_memcpy.  Note that the size argument
+   should be passed with type unsigned int, because that is what the non-GCC
+   definitions require.  With GCC, __builtin_memcpy takes an arg
+   of type size_t, but it can handle unsigned int.  */
+
+#if __GNUC__ > 1		/* GNU C and GNU C++ define this.  */
+#define __yy_memcpy(TO,FROM,COUNT)	__builtin_memcpy(TO,FROM,COUNT)
+#else				/* not GNU C or C++ */
+#ifndef __cplusplus
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (to, from, count)
+     char *to;
+     char *from;
+     unsigned int count;
+{
+  register char *f = from;
+  register char *t = to;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#else /* __cplusplus */
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (char *to, char *from, unsigned int count)
+{
+  register char *t = to;
+  register char *f = from;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#endif
+#endif
+
+
+#line 216 "/usr/lib/bison.simple"
+
+/* The user can define YYPARSE_PARAM as the name of an argument to be passed
+   into yyparse.  The argument should have type void *.
+   It should actually point to an object.
+   Grammar actions can access the variable by casting it
+   to the proper pointer type.  */
+
+#ifdef YYPARSE_PARAM
+#ifdef __cplusplus
+#define YYPARSE_PARAM_ARG void *YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL
+#else /* not __cplusplus */
+#define YYPARSE_PARAM_ARG YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL void *YYPARSE_PARAM;
+#endif /* not __cplusplus */
+#else /* not YYPARSE_PARAM */
+#define YYPARSE_PARAM_ARG
+#define YYPARSE_PARAM_DECL
+#endif /* not YYPARSE_PARAM */
+
+/* Prevent warning if -Wstrict-prototypes.  */
+#ifdef __GNUC__
+#ifdef YYPARSE_PARAM
+int yyparse (void *);
+#else
+int yyparse (void);
+#endif
+#endif
+
+int
+yyparse(YYPARSE_PARAM_ARG)
+     YYPARSE_PARAM_DECL
+{
+  register int yystate;
+  register int yyn;
+  register short *yyssp;
+  register YYSTYPE *yyvsp;
+  int yyerrstatus;	/*  number of tokens to shift before error messages enabled */
+  int yychar1 = 0;		/*  lookahead token as an internal (translated) token number */
+
+  short	yyssa[YYINITDEPTH];	/*  the state stack			*/
+  YYSTYPE yyvsa[YYINITDEPTH];	/*  the semantic value stack		*/
+
+  short *yyss = yyssa;		/*  refer to the stacks thru separate pointers */
+  YYSTYPE *yyvs = yyvsa;	/*  to allow yyoverflow to reallocate them elsewhere */
+
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylsa[YYINITDEPTH];	/*  the location stack			*/
+  YYLTYPE *yyls = yylsa;
+  YYLTYPE *yylsp;
+
+#define YYPOPSTACK   (yyvsp--, yyssp--, yylsp--)
+#else
+#define YYPOPSTACK   (yyvsp--, yyssp--)
+#endif
+
+  int yystacksize = YYINITDEPTH;
+  int yyfree_stacks = 0;
+
+#ifdef YYPURE
+  int yychar;
+  YYSTYPE yylval;
+  int yynerrs;
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylloc;
+#endif
+#endif
+
+  YYSTYPE yyval;		/*  the variable used to return		*/
+				/*  semantic values from the action	*/
+				/*  routines				*/
+
+  int yylen;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Starting parse\n");
+#endif
+
+  yystate = 0;
+  yyerrstatus = 0;
+  yynerrs = 0;
+  yychar = YYEMPTY;		/* Cause a token to be read.  */
+
+  /* Initialize stack pointers.
+     Waste one element of value and location stack
+     so that they stay on the same level as the state stack.
+     The wasted elements are never initialized.  */
+
+  yyssp = yyss - 1;
+  yyvsp = yyvs;
+#ifdef YYLSP_NEEDED
+  yylsp = yyls;
+#endif
+
+/* Push a new state, which is found in  yystate  .  */
+/* In all cases, when you get here, the value and location stacks
+   have just been pushed. so pushing a state here evens the stacks.  */
+yynewstate:
+
+  *++yyssp = yystate;
+
+  if (yyssp >= yyss + yystacksize - 1)
+    {
+      /* Give user a chance to reallocate the stack */
+      /* Use copies of these so that the &'s don't force the real ones into memory. */
+      YYSTYPE *yyvs1 = yyvs;
+      short *yyss1 = yyss;
+#ifdef YYLSP_NEEDED
+      YYLTYPE *yyls1 = yyls;
+#endif
+
+      /* Get the current used size of the three stacks, in elements.  */
+      int size = yyssp - yyss + 1;
+
+#ifdef yyoverflow
+      /* Each stack pointer address is followed by the size of
+	 the data in use in that stack, in bytes.  */
+#ifdef YYLSP_NEEDED
+      /* This used to be a conditional around just the two extra args,
+	 but that might be undefined if yyoverflow is a macro.  */
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yyls1, size * sizeof (*yylsp),
+		 &yystacksize);
+#else
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yystacksize);
+#endif
+
+      yyss = yyss1; yyvs = yyvs1;
+#ifdef YYLSP_NEEDED
+      yyls = yyls1;
+#endif
+#else /* no yyoverflow */
+      /* Extend the stack our own way.  */
+      if (yystacksize >= YYMAXDEPTH)
+	{
+	  yyerror("parser stack overflow");
+	  if (yyfree_stacks)
+	    {
+	      free (yyss);
+	      free (yyvs);
+#ifdef YYLSP_NEEDED
+	      free (yyls);
+#endif
+	    }
+	  return 2;
+	}
+      yystacksize *= 2;
+      if (yystacksize > YYMAXDEPTH)
+	yystacksize = YYMAXDEPTH;
+#ifndef YYSTACK_USE_ALLOCA
+      yyfree_stacks = 1;
+#endif
+      yyss = (short *) YYSTACK_ALLOC (yystacksize * sizeof (*yyssp));
+      __yy_memcpy ((char *)yyss, (char *)yyss1,
+		   size * (unsigned int) sizeof (*yyssp));
+      yyvs = (YYSTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yyvsp));
+      __yy_memcpy ((char *)yyvs, (char *)yyvs1,
+		   size * (unsigned int) sizeof (*yyvsp));
+#ifdef YYLSP_NEEDED
+      yyls = (YYLTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yylsp));
+      __yy_memcpy ((char *)yyls, (char *)yyls1,
+		   size * (unsigned int) sizeof (*yylsp));
+#endif
+#endif /* no yyoverflow */
+
+      yyssp = yyss + size - 1;
+      yyvsp = yyvs + size - 1;
+#ifdef YYLSP_NEEDED
+      yylsp = yyls + size - 1;
+#endif
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Stack size increased to %d\n", yystacksize);
+#endif
+
+      if (yyssp >= yyss + yystacksize - 1)
+	YYABORT;
+    }
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Entering state %d\n", yystate);
+#endif
+
+  goto yybackup;
+ yybackup:
+
+/* Do appropriate processing given the current state.  */
+/* Read a lookahead token if we need one and don't already have one.  */
+/* yyresume: */
+
+  /* First try to decide what to do without reference to lookahead token.  */
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yydefault;
+
+  /* Not known => get a lookahead token if don't already have one.  */
+
+  /* yychar is either YYEMPTY or YYEOF
+     or a valid token in external form.  */
+
+  if (yychar == YYEMPTY)
+    {
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Reading a token: ");
+#endif
+      yychar = YYLEX;
+    }
+
+  /* Convert token to internal form (in yychar1) for indexing tables with */
+
+  if (yychar <= 0)		/* This means end of input. */
+    {
+      yychar1 = 0;
+      yychar = YYEOF;		/* Don't call YYLEX any more */
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Now at end of input.\n");
+#endif
+    }
+  else
+    {
+      yychar1 = YYTRANSLATE(yychar);
+
+#if YYDEBUG != 0
+      if (yydebug)
+	{
+	  fprintf (stderr, "Next token is %d (%s", yychar, yytname[yychar1]);
+	  /* Give the individual parser a way to print the precise meaning
+	     of a token, for further debugging info.  */
+#ifdef YYPRINT
+	  YYPRINT (stderr, yychar, yylval);
+#endif
+	  fprintf (stderr, ")\n");
+	}
+#endif
+    }
+
+  yyn += yychar1;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != yychar1)
+    goto yydefault;
+
+  yyn = yytable[yyn];
+
+  /* yyn is what to do for this token type in this state.
+     Negative => reduce, -yyn is rule number.
+     Positive => shift, yyn is new state.
+       New state is final state => don't bother to shift,
+       just return success.
+     0, or most negative number => error.  */
+
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrlab;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrlab;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+  /* Shift the lookahead token.  */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting token %d (%s), ", yychar, yytname[yychar1]);
+#endif
+
+  /* Discard the token being shifted unless it is eof.  */
+  if (yychar != YYEOF)
+    yychar = YYEMPTY;
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  /* count tokens shifted since error; after three, turn off error status.  */
+  if (yyerrstatus) yyerrstatus--;
+
+  yystate = yyn;
+  goto yynewstate;
+
+/* Do the default action for the current state.  */
+yydefault:
+
+  yyn = yydefact[yystate];
+  if (yyn == 0)
+    goto yyerrlab;
+
+/* Do a reduction.  yyn is the number of a rule to reduce with.  */
+yyreduce:
+  yylen = yyr2[yyn];
+  if (yylen > 0)
+    yyval = yyvsp[1-yylen]; /* implement default value of the action */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      int i;
+
+      fprintf (stderr, "Reducing via rule %d (line %d), ",
+	       yyn, yyrline[yyn]);
+
+      /* Print the symbols being reduced, and their result.  */
+      for (i = yyprhs[yyn]; yyrhs[i] > 0; i++)
+	fprintf (stderr, "%s ", yytname[yyrhs[i]]);
+      fprintf (stderr, " -> %s\n", yytname[yyr1[yyn]]);
+    }
+#endif
+
+
+  switch (yyn) {
+
+case 1:
+#line 76 "bool_parser.y"
+{ tree_base = yyvsp[0].node;;
+    break;}
+case 2:
+#line 80 "bool_parser.y"
+{ yyval.node = CreateBoolTermNode(term_list, yyvsp[0].text, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); ;
+    break;}
+case 3:
+#line 81 "bool_parser.y"
+{ yyval.node = yyvsp[-1].node; ;
+    break;}
+case 4:
+#line 82 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_all, NULL, NULL); ;
+    break;}
+case 5:
+#line 83 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_none, NULL, NULL); ;
+    break;}
+case 7:
+#line 87 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_not, yyvsp[0].node, NULL); ;
+    break;}
+case 8:
+#line 90 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+case 9:
+#line 91 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-1].node, yyvsp[0].node); ;
+    break;}
+case 11:
+#line 95 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_or, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+}
+   /* the action file gets copied in in place of this dollarsign */
+#line 542 "/usr/lib/bison.simple"
+
+
+  yyvsp -= yylen;
+  yyssp -= yylen;
+#ifdef YYLSP_NEEDED
+  yylsp -= yylen;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+  *++yyvsp = yyval;
+
+#ifdef YYLSP_NEEDED
+  yylsp++;
+  if (yylen == 0)
+    {
+      yylsp->first_line = yylloc.first_line;
+      yylsp->first_column = yylloc.first_column;
+      yylsp->last_line = (yylsp-1)->last_line;
+      yylsp->last_column = (yylsp-1)->last_column;
+      yylsp->text = 0;
+    }
+  else
+    {
+      yylsp->last_line = (yylsp+yylen-1)->last_line;
+      yylsp->last_column = (yylsp+yylen-1)->last_column;
+    }
+#endif
+
+  /* Now "shift" the result of the reduction.
+     Determine what state that goes to,
+     based on the state we popped back to
+     and the rule number reduced by.  */
+
+  yyn = yyr1[yyn];
+
+  yystate = yypgoto[yyn - YYNTBASE] + *yyssp;
+  if (yystate >= 0 && yystate <= YYLAST && yycheck[yystate] == *yyssp)
+    yystate = yytable[yystate];
+  else
+    yystate = yydefgoto[yyn - YYNTBASE];
+
+  goto yynewstate;
+
+yyerrlab:   /* here on detecting error */
+
+  if (! yyerrstatus)
+    /* If not already recovering from an error, report this error.  */
+    {
+      ++yynerrs;
+
+#ifdef YYERROR_VERBOSE
+      yyn = yypact[yystate];
+
+      if (yyn > YYFLAG && yyn < YYLAST)
+	{
+	  int size = 0;
+	  char *msg;
+	  int x, count;
+
+	  count = 0;
+	  /* Start X at -yyn if nec to avoid negative indexes in yycheck.  */
+	  for (x = (yyn < 0 ? -yyn : 0);
+	       x < (sizeof(yytname) / sizeof(char *)); x++)
+	    if (yycheck[x + yyn] == x)
+	      size += strlen(yytname[x]) + 15, count++;
+	  msg = (char *) malloc(size + 15);
+	  if (msg != 0)
+	    {
+	      strcpy(msg, "parse error");
+
+	      if (count < 5)
+		{
+		  count = 0;
+		  for (x = (yyn < 0 ? -yyn : 0);
+		       x < (sizeof(yytname) / sizeof(char *)); x++)
+		    if (yycheck[x + yyn] == x)
+		      {
+			strcat(msg, count == 0 ? ", expecting `" : " or `");
+			strcat(msg, yytname[x]);
+			strcat(msg, "'");
+			count++;
+		      }
+		}
+	      yyerror(msg);
+	      free(msg);
+	    }
+	  else
+	    yyerror ("parse error; also virtual memory exceeded");
+	}
+      else
+#endif /* YYERROR_VERBOSE */
+	yyerror("parse error");
+    }
+
+  goto yyerrlab1;
+yyerrlab1:   /* here on error raised explicitly by an action */
+
+  if (yyerrstatus == 3)
+    {
+      /* if just tried and failed to reuse lookahead token after an error, discard it.  */
+
+      /* return failure if at end of input */
+      if (yychar == YYEOF)
+	YYABORT;
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Discarding token %d (%s).\n", yychar, yytname[yychar1]);
+#endif
+
+      yychar = YYEMPTY;
+    }
+
+  /* Else will try to reuse lookahead token
+     after shifting the error token.  */
+
+  yyerrstatus = 3;		/* Each real token shifted decrements this */
+
+  goto yyerrhandle;
+
+yyerrdefault:  /* current state does not do anything special for the error token. */
+
+#if 0
+  /* This is wrong; only states that explicitly want error tokens
+     should shift them.  */
+  yyn = yydefact[yystate];  /* If its default is to accept any token, ok.  Otherwise pop it.*/
+  if (yyn) goto yydefault;
+#endif
+
+yyerrpop:   /* pop the current state because it cannot handle the error token */
+
+  if (yyssp == yyss) YYABORT;
+  yyvsp--;
+  yystate = *--yyssp;
+#ifdef YYLSP_NEEDED
+  yylsp--;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "Error: state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+yyerrhandle:
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yyerrdefault;
+
+  yyn += YYTERROR;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != YYTERROR)
+    goto yyerrdefault;
+
+  yyn = yytable[yyn];
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrpop;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrpop;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting error token, ");
+#endif
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  yystate = yyn;
+  goto yynewstate;
+
+ yyacceptlab:
+  /* YYACCEPT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 0;
+
+ yyabortlab:
+  /* YYABORT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 1;
+}
+#line 99 "bool_parser.y"
+
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+ 
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:43  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:40  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:26  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:33  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:31  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_PARSER_H
+#define BOOL_PARSER_H
+
+#include "backend.h"
+#include "term_lists.h"
+#include "invf.h"
+
+bool_tree_node *ParseBool (char *query_line, int query_len,
+			   TermList ** the_term_list, int the_stemmer_num, 
+			   int the_stem_method, int *res,
+			   stemmed_dict * the_sd, int is_indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			   QueryTermList ** the_query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.y
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.y	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_parser.y	(revision 21112)
@@ -0,0 +1,316 @@
+/**************************************************************************
+ *
+ * bool_parser - boolean query parser
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+ 
+/**************************************************************************/
+%{
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+%}
+ 
+ 
+%union {
+  char *text;
+  bool_tree_node *node;
+}
+ 
+%token <text> TERM
+%type <node> query term not and or
+ 
+%%
+ 
+query: or  { tree_base = $1;}
+;
+ 
+ 
+term:     TERM  { $$ = CreateBoolTermNode(term_list, $1, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+        | '(' or ')' { $$ = $2; }
+        | '*' { $$ = CreateBoolTreeNode(N_all, NULL, NULL); }
+        | '_' { $$ = CreateBoolTreeNode(N_none, NULL, NULL); }
+;
+ 
+not:      term
+        | '!' not { $$ = CreateBoolTreeNode(N_not, $2, NULL); }
+;
+ 
+and:      and '&' not { $$ = CreateBoolTreeNode(N_and, $1, $3); }
+        | and not { $$ = CreateBoolTreeNode(N_and, $1, $2); }
+        | not
+;
+ 
+or:       or '|' and { $$ = CreateBoolTreeNode(N_or, $1, $3); }
+        | and
+;
+ 
+%%
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.c	(revision 21112)
@@ -0,0 +1,353 @@
+/**************************************************************************
+ *
+ * bool_query.c -- Boolean query evaluation
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "sptree.h"
+
+
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "backend.h"
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "read_line.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "environment.h"
+
+
+/**********************
+  [RPAP - Feb 97: Term Frequency]
+
+  Shifted to bool_parser.y
+
+  *********************/
+
+/* =========================================================================
+ * Function: ReadInTermInfo
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+/**************************
+static void 
+ReadInTermInfo (query_data * qd)
+{
+  int i;
+  TermList *tl = qd->TL;
+
+  for (i = 0; i < tl->num; i++)
+    {
+      TermEntry *te = &(tl->TE[i]);
+      WordEntry *we = &(te->WE);
+
+       we->word_num = FindWord (qd->sd, te->Word,
+			       &we->count, &we->doc_count,
+			       &we->invf_ptr, &we->invf_len);
+      if (we->word_num == -1)
+	we->count = we->doc_count = 0;
+    }
+}
+
+***********************/
+
+
+
+/* =========================================================================
+ * Function: FindNoneTerms 
+ * Description: 
+ *      Set any terms which are not in dictionary to N_none (i.e. false)
+ *      Assumes a prior call to ReadInTermInfo
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FindNoneTerms (query_data * qd, bool_tree_node * tree)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  FindNoneTerms (qd, child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (tree) == N_term)
+    {
+      int doc_count = qd->TL->TE[BOOL_TERM (tree)].WE.doc_count;
+      if (!doc_count)
+	BOOL_TAG (tree) = N_none;
+    }
+
+}				/*FindNoneTerms */
+
+/* =========================================================================
+ * Function: ReduceList
+ * Description: 
+ *      Go thru doc-set and get rid of all docs which are unmarked
+ *      The motivation for this routine is in the case of CNF queries.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ReduceList (DocList * cset)
+{
+  DocEntry *src;
+  DocEntry *dest;
+
+  src = dest = cset->DE;
+  for (src = dest = cset->DE; src < cset->DE + cset->num; src++)
+    {
+      if (src->or_included)
+	{
+	  src->or_included = 0;
+	  *dest++ = *src;
+	}
+    }
+  cset->num = dest - cset->DE;
+
+}				/*ReduceList */
+
+/* =========================================================================
+ * Function: BooleanGet
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static DocList *
+BooleanGet (query_data * qd, bool_tree_node * n,
+	    BooleanQueryInfo * bqi)
+{
+  if (n)
+    {
+      DocList *res = NULL;
+      switch (BOOL_TAG (n))
+	{
+	case N_all:
+	  Message ("Unexpected \"all\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_not:
+	  Message ("Unexpected \"not\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_none:
+	  {
+	    res = MakeDocList (0);
+	    break;
+	  }
+	case N_diff:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		if (BOOL_TAG (n) == N_term)
+		  {
+		    WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		    res = GetDocsOp (qd, we, op_diff_terms, res);
+		  }
+		else
+		  res = DiffLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_and:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);		/* initial fill of c-set */
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		switch (BOOL_TAG (n))
+		  {
+		  case N_term:
+		    {
+		      WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		      res = GetDocsOp (qd, we, op_and_terms, res);
+		    }
+		    break;
+		  case N_or_terms:
+		    {
+		      bool_tree_node *child = BOOL_CHILD (n);
+		      /* note: all children are terms - predetermined */
+		      while (child)
+			{
+			  WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (child));
+			  res = GetDocsOp (qd, we, op_and_or_terms, res);
+			  child = BOOL_SIBLING (child);
+			}
+		      ReduceList (res);
+		    }
+		    break;
+		  default:
+		    res = IntersectLists (res, BooleanGet (qd, n, bqi));
+		  }		/*switch */
+		n = next;
+	      }
+	    break;
+	  }
+	case N_or:
+	case N_or_terms:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		res = MergeLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_term:
+	  {
+	    res = GetDocsOp (qd, GetNthWE (qd->TL, BOOL_TERM (n)), op_term, NULL);
+	    break;
+	  }
+	}
+      return (res ? res : MakeDocList (0));
+    }
+  else
+    return MakeDocList (0);
+}
+
+/* =========================================================================
+ * Function: AdjustParaDocs
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+static void
+AdjustParaDocs (query_data * qd, DocList * Docs)
+{
+  DocEntry *DE = Docs->DE;
+  int s, d;
+  Splay_Tree *SP;
+
+
+  /* Make the doc-list contain only references to unique
+   * real documents. So can get rid of cases where there
+   * are multiple paragraphs for a real document.
+   */
+
+  /* Note: [TS: 29/Sep/94] */
+  /* reads in seek-pos for docnums so can compare docs */
+  /* seek-pos used as real doc identifier */
+  /* whereas doc-num entries refer to paragraphs -- I think */
+  for (s = 0; s < Docs->num; s++)
+    FetchDocStart (qd, DE[s].DocNum, &DE[s].SeekPos, &DE[s].Len);
+
+  Message ("Original Number = %d\n", Docs->num);
+  SP = SP_createset (DE_comp);
+  d = 0;
+  for (s = 0; s < Docs->num; s++)
+    if (!SP_member (&DE[s], SP))
+      {
+	DE[d] = DE[s];
+	SP_insert (&DE[d], SP);
+	d++;
+      }
+  SP_freeset (SP);
+  Docs->num = d;
+  Message ("Final Number = %d\n", Docs->num);
+}
+
+/* =========================================================================
+ * Function: BooleanQuery 
+ * Description: 
+ *      The main control function for the boolean query.
+ * Input: 
+ * Output: 
+ *      qd->DL
+ * ========================================================================= */
+
+void 
+BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+	      int stem_method)
+{
+  bool_tree_node *tree;
+  DocList *Docs;
+  int res = 0;
+
+  tree = ParseBool (Query, MAXLINEBUFFERLEN, &(qd->TL), 
+		    qd->sd->sdh.stemmer_num, stem_method, &res,
+		    qd->sd, qd->sd->sdh.indexed,         /* [RPAP - Jan 97: Stem Index Change] */
+		    &(qd->QTL));                         /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [TS:Aug/94] no longer tries to continue after parse failure */
+  if (res != 0)			/* failed the parse */
+    {
+      qd->DL = NULL;		/* have empty doc-list */
+      return;
+    }
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  /*  ReadInTermInfo (qd); */
+
+  FindNoneTerms (qd, tree);
+
+  OptimiseBoolTree (tree, qd->TL, atoi (GetEnv ("optimise_type")));
+
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_terms = 0;
+
+  Docs = BooleanGet (qd, tree, bqi);
+
+/* [RPAP - Feb 97: NZDL Specifics] */
+#if !defined(PARABOOLEANMATCHES) && !defined(NZDL)
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      AdjustParaDocs (qd, Docs);
+    }
+#endif
+
+  if (bqi->MaxDocsToRetrieve != -1 && Docs->num > bqi->MaxDocsToRetrieve)
+    Docs->num = bqi->MaxDocsToRetrieve;
+
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_query.h	(revision 21112)
@@ -0,0 +1,55 @@
+/**************************************************************************
+ *
+ * bool_query.h -- boolean query processing
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:28  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:36  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_QUERY_H
+#define BOOL_QUERY_H
+
+#include "backend.h"
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tester.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tester.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tester.c	(revision 21112)
@@ -0,0 +1,185 @@
+/**************************************************************************
+ *
+ * bool_tester -- used to test bool_tree, bool_parser, bool_optimiser
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2000/03/05 22:58:24  sjboddie
+   compiler complained about using stdin and stdout to do global
+   initialization - variables were never changed so they were
+   replaced with #defines
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:41  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:23  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:05  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#define MAX_LINE_LEN  255
+#define STEMMER_NUM 0 /* Lovin's stemmer */
+#define STEM_METHOD 3
+
+/* Newer compilers complain when you use stdin and stdout to do global 
+initialization. Since they're static we'll just use #defines instead
+static FILE *file_in = stdin;
+static FILE *file_out = stdout;
+*/
+#define FILE_IN stdin
+#define FILE_OUT stdout
+
+static char line[MAX_LINE_LEN + 1];
+
+/* --- prototypes --- */
+static char *prompt (char *str);
+
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int main (int argc, char *argv[])
+{
+  bool_tree_node *tree = NULL;
+  TermList *term_list = NULL;
+  QueryTermList *query_term_list = NULL;  /* [RPAP - Feb 97: Term Frequency] */
+  int opt_type = 0;
+
+  while (1)
+    {
+      int res = 0;
+      int len = 0;
+
+      /* get a line */
+      if (!prompt ("Please enter boolean expression..."))
+	break;
+
+      len = strlen (line) - 1;	/* -1 => ignore the \n */
+
+      tree = ParseBool (line, len, &term_list, STEMMER_NUM, STEM_METHOD, &res,
+			NULL, 0,             /* [RPAP - Jan 97: Stem Index Change] */
+			&query_term_list);   /* [RPAP - Feb 97: Term Frequency] */
+
+      {
+	int done = 0;
+	while (!done)
+	  {
+	    if (!prompt ("Which type of optimisation 1 or 2 ?"))
+	      break;
+	    opt_type = atoi (line);
+	    done = (opt_type == 1 || opt_type == 2);
+	  }
+      }
+
+      if (!prompt ("Do you want to assign doc_counts to terms ?"))
+	break;
+
+      if (toupper (line[0]) == 'Y')
+	{
+	  /* cycle thru terms asking for doc_counts */
+	  int i;
+	  for (i = 0; i < term_list->num; i++)
+	    {
+	      char str[80];
+	      TermEntry *te = &(term_list->TE[i]);
+
+	      sprintf (str, "Please enter doc count for term '%s'",
+		       str255_to_string (te->Word, NULL));
+	      prompt (str);
+	      te->WE.doc_count = atoi (line);
+	      printf ("doc_count = %ld\n", te->WE.doc_count);
+	    }			/*for */
+	}			/*if */
+
+
+      if (res == 0)
+	{
+	  fprintf (FILE_OUT, "\n***Parsed Expression***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	  OptimiseBoolTree (tree, term_list, opt_type);
+	  fprintf (FILE_OUT, "\n***Optimised Expression ***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	}
+
+    }
+  return 0;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static char *
+prompt (char *str)
+{
+  fprintf (FILE_OUT, "\n%s\n", str);
+  return fgets (line, MAX_LINE_LEN, FILE_IN);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.c	(revision 21112)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * bool_tree.c -- Boolean parse tree ADT
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *       @(#)query.bool.y	1.9 16 Mar 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bool_tree.h"
+
+static char *RCSID = "$Id$";
+
+
+/* =========================================================================
+ * Function: CreateBoolNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolNode (N_Tag tag)
+{
+  bool_tree_node *n;
+  if (!(n = Xmalloc (sizeof (bool_tree_node))))
+    FatalError (1, "Can not create bool_tree_node for boolean query");
+  n->tag = tag;
+  BOOL_SIBLING (n) = NULL;
+  return n;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+		    u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len,  /* [RPAP - Feb 97: Term Frequency] */
+		    int stemmer_num) 
+{
+  bool_tree_node *n = NULL;
+
+  /* allocate bool_tree_node */
+  n = CreateBoolNode (N_term);
+
+  BOOL_TERM (n) = AddTerm (tl, (u_char *) text, Count, word_num, count, doc_count, 
+			   invf_ptr, invf_len, stemmer_num);  /* [RPAP - Feb 97: Term Frequency] */
+
+  return n;
+}
+
+
+/* =========================================================================
+ * Function: CreateTreeNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CreateBoolTreeNode (N_Tag tag, bool_tree_node * left, bool_tree_node * right)
+{
+  bool_tree_node *n = CreateBoolNode (tag);
+
+  BOOL_CHILD (n) = left;
+  if (left)
+    BOOL_SIBLING (left) = right;
+  if (right)
+    BOOL_SIBLING (right) = NULL;
+
+  return (n);
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+EqualBoolNodes (bool_tree_node * a, bool_tree_node * b)
+{
+  if (BOOL_TAG (a) == N_term && BOOL_TAG (a) == N_term)
+    return BOOL_TERM (a) == BOOL_TERM (b);
+  return 0;
+}
+
+/* =========================================================================
+ * Function: FreeBoolTree
+ * Description: 
+ *      Notes: does not fix up any parent or sibling links around given tree
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FreeTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	FreeTree (BOOL_CHILD (tree));
+      sibling = BOOL_SIBLING (tree);
+      Xfree (tree);
+      tree = sibling;
+    }
+}
+
+void 
+FreeBoolTree (bool_tree_node ** the_tree)
+{
+  bool_tree_node *tree = *the_tree;
+
+  if (!tree)
+    return;
+
+  FreeTree (tree);
+  *the_tree = NULL;
+/*********************************/
+  /* THIS IS POTENTIALLY DANGEROUS */
+  /* FIX UP LATER */
+  /* Need to correct the links for parent/siblings */
+/*********************************/
+
+}
+
+/* =========================================================================
+ * Function: PrintBoolTree
+ * Description: 
+ *      Print out a text version of the tree to the file
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PrintBinaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+  assert (BOOL_HAS_CHILD (tree));
+
+  child = BOOL_CHILD (tree);
+  while (child)
+    {
+      PrintBoolTree (child, file);
+      child = BOOL_SIBLING (child);
+      if (child)
+	fprintf (file, " %c ", op);
+    }
+}
+
+static void
+PrintUnaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+
+  fprintf (file, " %c ", op);
+  assert (BOOL_HAS_CHILD (tree));
+  child = BOOL_CHILD (tree);
+  PrintBoolTree (child, file);
+}
+
+void
+PrintBoolTree (bool_tree_node * tree, FILE * file)
+{
+  if (!tree)
+    return;
+
+  fprintf (file, "(");
+  switch (BOOL_TAG (tree))
+    {
+    case N_term:
+      fprintf (file, "term %d", BOOL_TERM (tree));
+      break;
+    case N_and:
+      PrintBinaryOp (tree, file, '&');
+      break;
+    case N_or:
+      PrintBinaryOp (tree, file, '|');
+      break;
+    case N_or_terms:
+      PrintBinaryOp (tree, file, '+');
+      break;
+    case N_diff:
+      PrintBinaryOp (tree, file, '-');
+      break;
+    case N_not:
+      PrintUnaryOp (tree, file, '!');
+      break;
+    case N_all:
+      fprintf (file, " TRUE ");
+      break;
+    case N_none:
+      fprintf (file, " FALSE ");
+      break;
+    default:
+      break;
+    }
+
+  fprintf (file, ")");
+}
+
+
+/* =========================================================================
+ * Function: CopyBoolTree
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CopyBoolTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling = NULL;
+  bool_tree_node *tree_copy = NULL;
+  bool_tree_node *prev = NULL;
+
+  if (!tree)
+    return NULL;
+
+  sibling = tree;
+  prev = NULL;
+  while (sibling)
+    {
+      /* copy sibling */
+      bool_tree_node *sibling_copy = (bool_tree_node *) malloc (sizeof (bool_tree_node));
+      if (!sibling_copy)
+	FatalError (1, "No memory for bool tree copy");
+      bcopy ((char *) sibling, (char *) sibling_copy, sizeof (bool_tree_node));
+
+      if (BOOL_HAS_CHILD (sibling))
+	{
+	  BOOL_CHILD (sibling_copy) = CopyBoolTree (BOOL_CHILD (sibling));
+	}
+
+      /* do the link up of copies */
+      if (prev)
+	BOOL_SIBLING (prev) = sibling_copy;
+      else
+	tree_copy = sibling_copy;
+      prev = sibling_copy;
+
+      sibling = BOOL_SIBLING (sibling);
+    }
+  return tree_copy;
+}
+
+
+/* =========================================================================
+ * Function: LastBoolSibling
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+LastBoolSibling (bool_tree_node * tree)
+{
+
+  if (!tree)
+    return NULL;
+
+  do
+    {
+      tree = BOOL_SIBLING (tree);
+    }
+  while (BOOL_SIBLING (tree));
+
+  return tree;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/bool_tree.h	(revision 21112)
@@ -0,0 +1,113 @@
+/**************************************************************************
+ *
+ * bool_tree.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:42  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:25  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:07  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:35  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_TREE_H
+#define BOOL_TREE_H
+
+#include "sysfuncs.h"
+#include "term_lists.h"
+
+typedef enum
+  {
+    N_term, N_and, N_or, N_diff, N_none, N_all,
+    N_not, N_or_terms
+  }
+N_Tag;
+
+typedef struct bool_tree_node
+  {
+    N_Tag tag;
+    struct bool_tree_node *sibling;
+    union
+      {
+	struct bool_tree_node *child;
+	int term_id;
+      }
+    fields;
+    int sum_ft;			/* for or_terms nodes */
+  }
+bool_tree_node;
+
+
+#define BOOL_HAS_CHILD(n) (\
+			(n)->tag==N_and || \
+			(n)->tag==N_or || (n)->tag==N_or_terms || \
+			(n)->tag==N_not || (n)->tag==N_diff)
+#define BOOL_TAG(n) ((n)->tag)
+#define BOOL_SIBLING(n) ((n)->sibling)
+#define BOOL_TERM(n) ((n)->fields.term_id)
+#define BOOL_CHILD(n) ((n)->fields.child)
+#define BOOL_SUM_FT(n) ((n)->sum_ft)
+
+bool_tree_node *CreateBoolNode (N_Tag tag);
+bool_tree_node *CopyBoolTree (bool_tree_node * tree);
+bool_tree_node *CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+				    u_long count, u_long doc_count, u_long invf_ptr, 
+				    u_long invf_len, int stemmer_num);
+bool_tree_node *CreateBoolTreeNode (N_Tag tag,
+			     bool_tree_node * left, bool_tree_node * right);
+void FreeBoolTree (bool_tree_node ** tree);
+void PrintBoolTree (bool_tree_node * tree, FILE * file);
+bool_tree_node *LastBoolSibling (bool_tree_node * tree);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/build.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/build.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/build.h	(revision 21112)
@@ -0,0 +1,171 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#define	TERMPARAGRAPH	'\003'
+
+
+int init_special (char *file_name);
+int init_text_1 (char *file_name);
+int init_text_2 (char *file_name);
+int init_invf_1 (char *file_name);
+int init_invf_2 (char *file_name);
+int init_ivf_1 (char *file_name);
+int init_ivf_2 (char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (u_char * s_in, int l_in);
+int process_text_1 (u_char * s_in, int l_in);
+int process_text_2 (u_char * s_in, int l_in);
+int process_invf_1 (u_char * s_in, int l_in);
+int process_invf_2 (u_char * s_in, int l_in);
+int process_ivf_1 (u_char * s_in, int l_in);
+int process_ivf_2 (u_char * s_in, int l_in);
+/*
+ * s_in               IN      The binary string to be compressed
+ * l_in         IN      The number of characters in s_in
+ *
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (char *filename);
+int done_text_1 (char *filename);
+int done_text_2 (char *filename);
+int done_invf_1 (char *filename);
+int done_invf_2 (char *filename);
+int done_ivf_1 (char *filename);
+int done_ivf_2 (char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char InvfLevel;
+/*
+ * This will determine the level of the inverted file it can take on the
+ * values 1, 2, or 3.
+ *
+ * Level 1: The inverted file contains only document numbers making it possible
+ *          to do only boolean queries.
+ * 
+ * Level 2: The inverted file also contains word counts per document making it
+ *          possible to do cosine ranked queries.
+ *
+ * Level 3: The inverted file contains word positions.
+ */
+
+
+extern unsigned long buf_size;
+/*
+ * The size of the document input buffer.
+ */
+
+extern unsigned long ChunkLimit;
+/*
+ * The maximum number of chunks that can be written to disk.
+ */
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+extern char SkipSGML;
+/*
+ * 1 if SGML tags are to be considered non-words when building the 
+ * inverted file.
+ */
+
+extern char MakeWeights;
+/*
+ * 1 if the weights file should be generated.
+ */
+
+extern FILE *Comp_Stats;
+/*
+ * Contains a file pointer to the file where compression stats should be sent
+ */
+
+extern int comp_stat_point;
+/*
+ * Generate a compression stat entry every comp_stat_point bytes 
+ */
+
+extern mg_ullong bytes_processed;
+/*
+ * The number of bytes processed. NOTE: This excludes document separators.
+ */
+
+extern mg_ullong bytes_received;
+/*
+ * The number of bytes processed. NOTE: This includes document separators.
+ */
+
+
+extern int stemmer_num;
+/*
+ * The stemmer to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+
+extern int stem_method;
+/*
+ * The method to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.c	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * commands.c -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:32  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:22  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "environment.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"		/* for qd */
+#include "term_lists.h"		/* for MAXTERMSTRLEN */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define OPENPIPE        _popen
+# define CLOSEPIPE       _pclose
+#else
+# define OPENPIPE        popen
+# define CLOSEPIPE       pclose
+#endif
+
+extern char *PathName;
+extern FILE *OutFile, *InFile;
+extern int OutPipe, InPipe;
+extern int Quitting;
+
+char *CommandsErrorStr = NULL;
+
+/* [TS: June/95] */
+/* current qd for use by any commands */
+static query_data *the_qd = NULL;
+
+
+#include "help.mg.h"
+#include "warranty.h"
+#include "conditions.h"
+
+static void 
+CmdQuit (void)
+{
+  Quitting = 1;
+}
+
+static void 
+CmdPush (void)
+{
+  PushEnv ();
+}
+
+
+static void 
+CmdPop (void)
+{
+  PopEnv ();
+  if (EnvStackHeight () == 1)
+    PushEnv ();
+}
+
+static void 
+CmdHelp (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (help_str) / sizeof (help_str[0]); i++)
+    fputs (help_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdWarranty (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (warranty_str) / sizeof (warranty_str[0]); i++)
+    fputs (warranty_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdConditions (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (cond_str) / sizeof (cond_str[0]); i++)
+    fputs (cond_str[i], more);
+
+  CLOSEPIPE (more);
+}
+
+static void 
+CmdReset (void)
+{
+  while (EnvStackHeight () > 1)
+    PopEnv ();
+  PushEnv ();
+}
+
+static void 
+CmdDisplay (void)
+{
+  int i;
+  size_t l = 0;
+  char *name;
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    if (strlen (name) > l)
+      l = strlen (name);
+
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+}
+
+static void 
+CmdQueryTerms (void)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+
+  if (the_qd && the_qd->TL)
+    {
+      ConvertTermsToString (the_qd->TL, terms_str);
+      printf ("%s\n", terms_str);
+      fflush (stdout);
+    }
+
+}
+
+
+static void 
+CmdSet (char *name, char *value)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  switch (SetEnv (name, value, NULL))
+    {
+    case -1:
+      CommandsErrorStr = "Out of memory\n";
+      break;
+    case -2:
+      CommandsErrorStr = ConstraintErrorStr;
+      break;
+    }
+}
+
+static void 
+CmdUnset (char *name)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  if (UnsetEnv (name, 0) == -1)
+    CommandsErrorStr = "Specified name does not exist";
+}
+
+
+
+
+
+
+/*
+ * Do the .input command 
+ */
+static void 
+CmdInput (char *param1, char *param2)
+{
+  enum
+  {
+    READ, APPEND, PIPEFROM
+  }
+  mode;
+  FILE *NewInput;
+  char *buf, *s;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '<')
+    {
+      s++;
+      mode = READ;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPEFROM;
+      s++;
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == READ)
+	NewInput = fopen (s, "r");
+      else
+	NewInput = OPENPIPE (s, "r");  /* [RPAP - Feb 97: WIN32 Port] */
+
+      if (NewInput)
+	{
+	  if (InPipe)
+	    CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (InFile != stdin)
+	    fclose (InFile);
+	  InPipe = (mode == PIPEFROM);
+	  InFile = NewInput;
+	}
+      else
+	CommandsErrorStr = "Bad input file/pipe";
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+    }
+  Xfree (buf);
+}
+
+
+
+static void 
+CmdOutput (char *param1, char *param2)
+{
+  enum
+  {
+    WRITE, APPEND, PIPETO
+  }
+  mode;
+  char *buf, *s;
+  FILE *NewOutput;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '>')
+    {
+      s++;
+      mode = APPEND;
+      if (*s == '>')
+	s++;
+      else
+	mode = WRITE;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPETO;
+      s++;
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == PIPETO)
+	NewOutput = OPENPIPE (s, "w");  /* [RPAP - Feb 97: WIN32 Port] */
+      else
+	NewOutput = fopen (s, mode == WRITE ? "w" : "a");
+      if (NewOutput)
+	{
+	  if (OutPipe)
+	    CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (OutFile != stdout)
+	    fclose (OutFile);
+	  OutPipe = (mode == PIPETO);
+	  OutFile = NewOutput;
+	}
+      else
+	CommandsErrorStr = "Bad output file/pipe";
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+    }
+  Xfree (buf);
+}
+
+
+
+static char *
+ParseArg (char *line, char *buffer)
+{
+  if (*line == '\"')
+    {
+      line++;
+      while (*line != '\"' && *line != '\0')
+	{
+	  if (*line == '\\')
+	    *buffer++ = *line++;
+	  if (*line != '\0')
+	    *buffer++ = *line++;
+	}
+      if (*line == '\"')
+	line++;
+    }
+  else
+    while (*line != ' ' && *line != '\t' && *line != '\0')
+      {
+	if (*line == '\\')
+	  line++;
+	if (*line != '\0')
+	  *buffer++ = *line++;
+      }
+  *buffer = '\0';
+  while (*line == ' ' || *line == '\t')
+    line++;
+  return (line);
+}
+
+
+
+char *
+ProcessCommands (char *line, query_data * qd)
+{
+  struct
+  {
+    char *Command;
+    void (*action) (void);
+    void (*action1) (char *);
+    void (*action2) (char *, char *);
+  }
+   *dc, dot_command[] =
+  {
+    {
+      "set", NULL, NULL, CmdSet
+    }
+    ,
+    {
+      "input", NULL, NULL, CmdInput
+    }
+    ,
+    {
+      "output", NULL, NULL, CmdOutput
+    }
+    ,
+    {
+      "unset", NULL, CmdUnset, NULL
+    }
+    ,
+    {
+      "help", CmdHelp, NULL, NULL
+    }
+    ,
+    {
+      "reset", CmdReset, NULL, NULL
+    }
+    ,
+    {
+      "display", CmdDisplay, NULL, NULL
+    }
+    ,
+    {
+      "push", CmdPush, NULL, NULL
+    }
+    ,
+    {
+      "quit", CmdQuit, NULL, NULL
+    }
+    ,
+    {
+      "pop", CmdPop, NULL, NULL
+    }
+    ,
+    {
+      "warranty", CmdWarranty, NULL, NULL
+    }
+    ,
+    {
+      "conditions", CmdConditions, NULL, NULL
+    }
+    ,
+    {
+      "queryterms", CmdQueryTerms, NULL, NULL
+    }
+  };
+
+
+  char command[512];
+  char param1[512], param2[512];
+  CommandsErrorStr = NULL;
+  the_qd = qd;			/* set globally to be accessed by command functions if needed */
+  while (*line == ' ' || *line == '\t')
+    line++;
+  while (*line == '.')
+    {
+      int i;
+      line++;
+      for (i = 0; isalpha (*line) && i < 512; i++, line++)
+	command[i] = tolower (*line);
+      while (*line == ' ' || *line == '\t')
+	line++;
+      if (i != 512)
+	{
+	  command[i] = '\0';
+	  for (i = 0, dc = dot_command; i < NUMOF (dot_command); i++, dc++)
+	    if (!strcmp (dc->Command, command))
+	      {
+		if (dc->action)
+		  dc->action ();
+		else if (dc->action1)
+		  {
+		    line = ParseArg (line, param1);
+		    dc->action1 (param1);
+		    break;
+		  }
+		else if (dc->action2)
+		  {
+		    line = ParseArg (line, param1);
+		    line = ParseArg (line, param2);
+		    dc->action2 (param1, param2);
+		    break;
+		  }
+		if (CommandsErrorStr)
+		  *line = '\0';
+		break;
+	      }
+	  if (i == NUMOF (dot_command))
+	    i = 100;
+	}
+      if (i == 100)
+	{
+	  CommandsErrorStr = "Illegal command";
+	  *line = '\0';
+	}
+      while (*line == ' ' || *line == '\t')
+	line++;
+    }
+  return (line);
+}
+
+
+
+
+
+void 
+read_mgrc_file (void)
+{
+  char line[1000];
+  char FileName[256];
+  int LineNum = 0;
+  FILE *mgrc;
+
+  strcpy (FileName, ".mgrc");
+  mgrc = fopen (FileName, "r");
+
+  if (!mgrc)
+    {
+      char *home_path = getenv ("HOME");
+      if (home_path)
+	{
+	  sprintf (FileName, "%s/.mgrc", home_path);
+	  mgrc = fopen (FileName, "r");
+	}
+    }
+
+  if (!mgrc)
+    return;
+
+  while (fgets (line, sizeof (line), mgrc))
+    {
+      char *s, linebuf[1000];
+      LineNum++;
+      if ((s = strchr (line, '\n')) != NULL)
+	*s = '\0';
+      strcpy (linebuf, line);
+      if ((s = strchr (line, '#')) != NULL)
+	*s = '\0';
+      if (!*line)
+	continue;
+      ProcessCommands (line, NULL);
+      if (CommandsErrorStr)
+	Message ("%s\n%s : ERROR in line %d of %s\n%s : %s\n",
+		 linebuf,
+		 msg_prefix, LineNum, FileName,
+		 msg_prefix, CommandsErrorStr);
+    }
+  fclose (mgrc);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/commands.h	(revision 21112)
@@ -0,0 +1,33 @@
+/**************************************************************************
+ *
+ * commands.h -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef COMMANDS_H
+#define COMMANDS_H
+
+#include "backend.h"
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.c	(revision 21112)
@@ -0,0 +1,258 @@
+/**************************************************************************
+ *
+ * comp_dict.c -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:41  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:24  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = Xmalloc (sizeof (dict_hash_table) +
+		      (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  bzero ((char *) ht->table, ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; i++, words++)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register int tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+	    goto error;
+
+	  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+	    goto error;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(char_codes[which] =
+		Generate_Huffman_Codes (&char_huff[which], NULL)))
+	    goto error;
+	  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(lens_codes[which] =
+		Generate_Huffman_Codes (&lens_huff[which], NULL)))
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(char_codes[which] =
+		    Generate_Huffman_Codes (&char_huff[which], NULL)))
+		goto error;
+	      if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(lens_codes[which] =
+		    Generate_Huffman_Codes (&lens_huff[which], NULL)))
+		goto error;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+
+#define HASH_RATIO 2
+
+
+
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+
+
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.h	(revision 21112)
@@ -0,0 +1,226 @@
+static char *cond_str[] = {
+"		    GNU GENERAL PUBLIC LICENSE\n",
+"   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n",
+"\n",
+"  0. This License applies to any program or other work which contains\n",
+"a notice placed by the copyright holder saying it may be distributed\n",
+"under the terms of this General Public License.  The \"Program\", below,\n",
+"refers to any such program or work, and a \"work based on the Program\"\n",
+"means either the Program or any derivative work under copyright law:\n",
+"that is to say, a work containing the Program or a portion of it,\n",
+"either verbatim or with modifications and/or translated into another\n",
+"language.  (Hereinafter, translation is included without limitation in\n",
+"the term \"modification\".)  Each licensee is addressed as \"you\".\n",
+"\n",
+"Activities other than copying, distribution and modification are not\n",
+"covered by this License; they are outside its scope.  The act of\n",
+"running the Program is not restricted, and the output from the Program\n",
+"is covered only if its contents constitute a work based on the\n",
+"Program (independent of having been made by running the Program).\n",
+"Whether that is true depends on what the Program does.\n",
+"\n",
+"  1. You may copy and distribute verbatim copies of the Program's\n",
+"source code as you receive it, in any medium, provided that you\n",
+"conspicuously and appropriately publish on each copy an appropriate\n",
+"copyright notice and disclaimer of warranty; keep intact all the\n",
+"notices that refer to this License and to the absence of any warranty;\n",
+"and give any other recipients of the Program a copy of this License\n",
+"along with the Program.\n",
+"\n",
+"You may charge a fee for the physical act of transferring a copy, and\n",
+"you may at your option offer warranty protection in exchange for a fee.\n",
+"\n",
+"  2. You may modify your copy or copies of the Program or any portion\n",
+"of it, thus forming a work based on the Program, and copy and\n",
+"distribute such modifications or work under the terms of Section 1\n",
+"above, provided that you also meet all of these conditions:\n",
+"\n",
+"    a) You must cause the modified files to carry prominent notices\n",
+"    stating that you changed the files and the date of any change.\n",
+"\n",
+"    b) You must cause any work that you distribute or publish, that in\n",
+"    whole or in part contains or is derived from the Program or any\n",
+"    part thereof, to be licensed as a whole at no charge to all third\n",
+"    parties under the terms of this License.\n",
+"\n",
+"    c) If the modified program normally reads commands interactively\n",
+"    when run, you must cause it, when started running for such\n",
+"    interactive use in the most ordinary way, to print or display an\n",
+"    announcement including an appropriate copyright notice and a\n",
+"    notice that there is no warranty (or else, saying that you provide\n",
+"    a warranty) and that users may redistribute the program under\n",
+"    these conditions, and telling the user how to view a copy of this\n",
+"    License.  (Exception: if the Program itself is interactive but\n",
+"    does not normally print such an announcement, your work based on\n",
+"    the Program is not required to print an announcement.)\n",
+"
+\n",
+"These requirements apply to the modified work as a whole.  If\n",
+"identifiable sections of that work are not derived from the Program,\n",
+"and can be reasonably considered independent and separate works in\n",
+"themselves, then this License, and its terms, do not apply to those\n",
+"sections when you distribute them as separate works.  But when you\n",
+"distribute the same sections as part of a whole which is a work based\n",
+"on the Program, the distribution of the whole must be on the terms of\n",
+"this License, whose permissions for other licensees extend to the\n",
+"entire whole, and thus to each and every part regardless of who wrote it.\n",
+"\n",
+"Thus, it is not the intent of this section to claim rights or contest\n",
+"your rights to work written entirely by you; rather, the intent is to\n",
+"exercise the right to control the distribution of derivative or\n",
+"collective works based on the Program.\n",
+"\n",
+"In addition, mere aggregation of another work not based on the Program\n",
+"with the Program (or with a work based on the Program) on a volume of\n",
+"a storage or distribution medium does not bring the other work under\n",
+"the scope of this License.\n",
+"\n",
+"  3. You may copy and distribute the Program (or a work based on it,\n",
+"under Section 2) in object code or executable form under the terms of\n",
+"Sections 1 and 2 above provided that you also do one of the following:\n",
+"\n",
+"    a) Accompany it with the complete corresponding machine-readable\n",
+"    source code, which must be distributed under the terms of Sections\n",
+"    1 and 2 above on a medium customarily used for software interchange; or,\n",
+"\n",
+"    b) Accompany it with a written offer, valid for at least three\n",
+"    years, to give any third party, for a charge no more than your\n",
+"    cost of physically performing source distribution, a complete\n",
+"    machine-readable copy of the corresponding source code, to be\n",
+"    distributed under the terms of Sections 1 and 2 above on a medium\n",
+"    customarily used for software interchange; or,\n",
+"\n",
+"    c) Accompany it with the information you received as to the offer\n",
+"    to distribute corresponding source code.  (This alternative is\n",
+"    allowed only for noncommercial distribution and only if you\n",
+"    received the program in object code or executable form with such\n",
+"    an offer, in accord with Subsection b above.)\n",
+"\n",
+"The source code for a work means the preferred form of the work for\n",
+"making modifications to it.  For an executable work, complete source\n",
+"code means all the source code for all modules it contains, plus any\n",
+"associated interface definition files, plus the scripts used to\n",
+"control compilation and installation of the executable.  However, as a\n",
+"special exception, the source code distributed need not include\n",
+"anything that is normally distributed (in either source or binary\n",
+"form) with the major components (compiler, kernel, and so on) of the\n",
+"operating system on which the executable runs, unless that component\n",
+"itself accompanies the executable.\n",
+"\n",
+"If distribution of executable or object code is made by offering\n",
+"access to copy from a designated place, then offering equivalent\n",
+"access to copy the source code from the same place counts as\n",
+"distribution of the source code, even though third parties are not\n",
+"compelled to copy the source along with the object code.\n",
+"
+\n",
+"  4. You may not copy, modify, sublicense, or distribute the Program\n",
+"except as expressly provided under this License.  Any attempt\n",
+"otherwise to copy, modify, sublicense or distribute the Program is\n",
+"void, and will automatically terminate your rights under this License.\n",
+"However, parties who have received copies, or rights, from you under\n",
+"this License will not have their licenses terminated so long as such\n",
+"parties remain in full compliance.\n",
+"\n",
+"  5. You are not required to accept this License, since you have not\n",
+"signed it.  However, nothing else grants you permission to modify or\n",
+"distribute the Program or its derivative works.  These actions are\n",
+"prohibited by law if you do not accept this License.  Therefore, by\n",
+"modifying or distributing the Program (or any work based on the\n",
+"Program), you indicate your acceptance of this License to do so, and\n",
+"all its terms and conditions for copying, distributing or modifying\n",
+"the Program or works based on it.\n",
+"\n",
+"  6. Each time you redistribute the Program (or any work based on the\n",
+"Program), the recipient automatically receives a license from the\n",
+"original licensor to copy, distribute or modify the Program subject to\n",
+"these terms and conditions.  You may not impose any further\n",
+"restrictions on the recipients' exercise of the rights granted herein.\n",
+"You are not responsible for enforcing compliance by third parties to\n",
+"this License.\n",
+"\n",
+"  7. If, as a consequence of a court judgment or allegation of patent\n",
+"infringement or for any other reason (not limited to patent issues),\n",
+"conditions are imposed on you (whether by court order, agreement or\n",
+"otherwise) that contradict the conditions of this License, they do not\n",
+"excuse you from the conditions of this License.  If you cannot\n",
+"distribute so as to satisfy simultaneously your obligations under this\n",
+"License and any other pertinent obligations, then as a consequence you\n",
+"may not distribute the Program at all.  For example, if a patent\n",
+"license would not permit royalty-free redistribution of the Program by\n",
+"all those who receive copies directly or indirectly through you, then\n",
+"the only way you could satisfy both it and this License would be to\n",
+"refrain entirely from distribution of the Program.\n",
+"\n",
+"If any portion of this section is held invalid or unenforceable under\n",
+"any particular circumstance, the balance of the section is intended to\n",
+"apply and the section as a whole is intended to apply in other\n",
+"circumstances.\n",
+"\n",
+"It is not the purpose of this section to induce you to infringe any\n",
+"patents or other property right claims or to contest validity of any\n",
+"such claims; this section has the sole purpose of protecting the\n",
+"integrity of the free software distribution system, which is\n",
+"implemented by public license practices.  Many people have made\n",
+"generous contributions to the wide range of software distributed\n",
+"through that system in reliance on consistent application of that\n",
+"system; it is up to the author/donor to decide if he or she is willing\n",
+"to distribute software through any other system and a licensee cannot\n",
+"impose that choice.\n",
+"\n",
+"This section is intended to make thoroughly clear what is believed to\n",
+"be a consequence of the rest of this License.\n",
+"
+\n",
+"  8. If the distribution and/or use of the Program is restricted in\n",
+"certain countries either by patents or by copyrighted interfaces, the\n",
+"original copyright holder who places the Program under this License\n",
+"may add an explicit geographical distribution limitation excluding\n",
+"those countries, so that distribution is permitted only in or among\n",
+"countries not thus excluded.  In such case, this License incorporates\n",
+"the limitation as if written in the body of this License.\n",
+"\n",
+"  9. The Free Software Foundation may publish revised and/or new versions\n",
+"of the General Public License from time to time.  Such new versions will\n",
+"be similar in spirit to the present version, but may differ in detail to\n",
+"address new problems or concerns.\n",
+"\n",
+"Each version is given a distinguishing version number.  If the Program\n",
+"specifies a version number of this License which applies to it and \"any\n",
+"later version\", you have the option of following the terms and conditions\n",
+"either of that version or of any later version published by the Free\n",
+"Software Foundation.  If the Program does not specify a version number of\n",
+"this License, you may choose any version ever published by the Free Software\n",
+"Foundation.\n",
+"\n",
+"  10. If you wish to incorporate parts of the Program into other free\n",
+"programs whose distribution conditions are different, write to the author\n",
+"to ask for permission.  For software which is copyrighted by the Free\n",
+"Software Foundation, write to the Free Software Foundation; we sometimes\n",
+"make exceptions for this.  Our decision will be guided by the two goals\n",
+"of preserving the free status of all derivatives of our free software and\n",
+"of promoting the sharing and reuse of software generally.\n",
+"\n",
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.src
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.src	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/conditions.src	(revision 21112)
@@ -0,0 +1,225 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.c	(revision 21112)
@@ -0,0 +1,691 @@
+/**************************************************************************
+ *
+ * environment.c -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/04/25 23:01:18  kjdon
+   added a new -M option to mg_passes, allowing maxnumeric to be altered - made this change to keep gsdl3 mg inline with gsdl2 mg.
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:50  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.5  1995/03/14  05:15:26  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.4  1994/11/25  03:47:43  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:56:43  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:25  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+
+#include "environment.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define STRCASECMP      stricmp
+#else
+# define STRCASECMP      strcasecmp
+#endif
+
+typedef struct
+  {
+    char *Name, *Data;
+    char *(*Constraint) (char *Old, char *New);
+  }
+EEntry;
+
+typedef struct Env
+  {
+    EEntry *Environment;
+    int NumEnv;
+    struct Env *Next;
+  }
+Env;
+
+
+char *ConstraintErrorStr;
+
+static Env Base =
+{NULL, 0, NULL};
+
+
+/*
+ * Add name and data to the environment (data may be NULL)
+ * All names are converted to lower case
+ * returns 0 on success, -1 on memory failure or -2 on constraint failure
+ */
+int 
+SetEnv (char *name, char *data, char *(*Constraint) (char *, char *))
+{
+  int i;
+  EEntry *newenv, *env = Base.Environment;
+
+  if (data && *data == '\0')
+    data = NULL;
+
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* if not found the increase the environment size */
+  if (i >= Base.NumEnv)
+    {
+      if (Constraint)
+	{
+	  data = Constraint (NULL, data);
+	  if (!data)
+	    return (-2);
+	}
+      /* allocate memory for the new environment */
+      newenv = Base.Environment ?
+	Xrealloc (Base.Environment, sizeof (EEntry) * (Base.NumEnv + 1)) :
+	Xmalloc (sizeof (EEntry) * (Base.NumEnv + 1));
+      if (!newenv)
+	return (-1);
+      Base.Environment = newenv;
+      env = &newenv[Base.NumEnv];
+      env->Data = NULL;
+      env->Name = NULL;
+      env->Constraint = Constraint;
+      env->Name = Xstrdup (name);
+      if (!env->Name)
+	return (-1);
+      Base.NumEnv++;
+      for (i = 0; env->Name[i]; i++)
+	env->Name[i] = tolower (env->Name[i]);
+    }
+  if (env->Constraint)
+    {
+      data = env->Constraint (env->Data, data);
+      if (!data)
+	return (-2);
+    }
+  if (env->Data)
+    Xfree (env->Data);
+  env->Data = NULL;
+  if (data && *data != '\0')
+    {
+      env->Data = Xstrdup (data);
+      if (!env->Data)
+	return (-1);
+    }
+  return (0);
+}
+
+
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns NULL.
+ */
+char *
+GetEnv (char *name)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv)
+    return (NULL);
+
+  return (env->Data ? env->Data : "");
+}
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns default.
+ */
+char *
+GetDefEnv (char *name, char *def)
+{
+  char *data = GetEnv (name);
+  return (data ? data : def);
+}
+
+
+
+/*
+ * This function pushes the environment on to a stack and duplicated it in 
+ * the current stack.
+ *
+ */
+int 
+PushEnv (void)
+{
+  Env *env = Xmalloc (sizeof (Env));
+  int i;
+  *env = Base;
+  Base.Next = env;
+  Base.Environment = NULL;
+  Base.NumEnv = 0;
+  for (i = 0; i < env->NumEnv; i++)
+    if (SetEnv (env->Environment[i].Name, env->Environment[i].Data,
+		env->Environment[i].Constraint))
+      return (-1);
+  return (0);
+}
+
+int 
+PopEnv (void)
+{
+  Env *env;
+  int i;
+  if (!Base.Next)
+    return (-1);
+  for (i = 0; i < Base.NumEnv; i++)
+    {
+      Xfree (Base.Environment[i].Name);
+      if (Base.Environment[i].Data)
+	Xfree (Base.Environment[i].Data);
+    }
+  if (Base.Environment)
+    Xfree (Base.Environment);
+  env = Base.Next;
+  Base = *env;
+  Xfree (env);
+  return (0);
+}
+
+
+/*
+ * Count the number of environments on the stack, including the current one.
+ *
+ */
+int 
+EnvStackHeight (void)
+{
+  int i = 1;
+  Env *env = &Base;
+  while (env->Next)
+    {
+      i++;
+      env = env->Next;
+    }
+  return (i);
+}
+
+/*
+ * Delete environment variable name. If name does not exist it returns -1
+ * otherwise it returns 0
+ */
+int 
+UnsetEnv (char *name, int Force)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv ||
+      (Base.Environment[i].Constraint && !Force))
+    return (-1);
+  Xfree (Base.Environment[i].Name);
+  if (Base.Environment[i].Data)
+    Xfree (Base.Environment[i].Data);
+  for (i++; i < Base.NumEnv; i++)
+    Base.Environment[i - 1] = Base.Environment[i];
+  Base.NumEnv--;
+  if (Base.NumEnv == 0)
+    {
+      Xfree (Base.Environment);
+      Base.Environment = NULL;
+    }
+  return (0);
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns the name of environment variable number i or NULL if i is greater 
+ * than or equal to the number of environment variables. Names start from
+ * zero. 
+ */
+char *
+GetEnvName (int i)
+{
+  return (i < Base.NumEnv ? Base.Environment[i].Name : NULL);
+}
+
+static char *BooleanStrs[] =
+{"false", "true", "no", "yes", "off", "on"};
+
+static char *
+BooleanCons (char *Old, char *New)
+{
+  int i;
+  int old = -1, new = -1;
+  if (Old)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (Old, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  old = i;
+	  break;
+	}
+  if (New)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  if (!New)
+    return (BooleanStrs[Old ? old ^ 1 : 0]);
+  ConstraintErrorStr = "Invalid argument [true|false|yes|no|on|off] required";
+  return (NULL);
+}
+
+/*
+ * returns 0 or 1  for a boolean string
+ * or default on a error
+ */
+int 
+BooleanEnv (char *data, int def)
+{
+  int i;
+  if (!data)
+    return (def);
+  for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+    if (!STRCASECMP (data, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (i & 1);
+  return (def);
+}
+
+/*
+ * returns the value for a integer string
+ * or default on a error
+ */
+long 
+IntEnv (char *data, long def)
+{
+  long val;
+  char *ptr;
+  if (!data)
+    return (def);
+  val = strtol (data, &ptr, 10);
+  return *ptr ? def : val;
+}
+
+static char *
+NumberCmp (char *num, int min, int max)
+{
+  char *err;
+  int val;
+  static char Err[100];
+  ConstraintErrorStr = "Not a valid number";
+  if (!num)
+    return (NULL);
+  val = strtol (num, &err, 10);
+  if (*err)
+    return (NULL);
+  sprintf (Err, "Not in legal range [%d <= num <= %d]", min, max);
+  ConstraintErrorStr = Err;
+  if (val < min || val > max)
+    return (NULL);
+  return (num);
+}
+
+/*ARGSUSED */
+static char *
+MaxDocsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/*ARGSUSED */
+static char *
+MaxTermsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/* ARGSUSED */
+static char *
+BufferCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 16 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxNodesCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHashCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxParasCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHeadsCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 1000));
+}
+
+/* ARGSUSED */
+char *
+OptimiseCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 2));
+}
+
+/* ARGSUSED */
+char *
+MaxNumericCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 4, 512));
+}
+
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+static char *
+QueryCons (char *Old, char *New)
+{
+  static char *QueryStrs[] =
+  {
+    "boolean", "ranked",
+    "docnums", "approx-ranked"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (QueryStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, QueryStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [boolean|ranked|docnums|approx-ranked] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid accum type
+ *
+ */
+/* ARGSUSED */
+static char *
+AccumCons (char *Old, char *New)
+{
+  static char *AccumStrs[] =
+  {
+    "array", "splay_tree", "hash_table", "list"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (AccumStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, AccumStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument "
+    "[array|splay_tree|hash_table|list] required";
+  return (NULL);
+}
+
+
+
+/*
+ * Makes sure that New is a valid output type type
+ *
+ */
+/* ARGSUSED */
+static char *
+OutputTypeCons (char *Old, char *New)
+{
+  static char *OutputTypeStrs[] =
+  {"text", "silent", "docnums",
+   "count", "heads", "hilite"
+#ifdef TREC_MODE
+   ,"extras_for_trec"
+#endif
+  };
+
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (OutputTypeStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, OutputTypeStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [text|silent|docnums|heads|count|hilite"
+#ifdef TREC_MODE
+    "|extras_for_trec"
+#endif
+    "] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+char *
+HiliteStyleCons (char *Old, char *New)
+{
+  static char *StyleStrs[] =
+  {"bold", "underline"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (StyleStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, StyleStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [bold|underline] required";
+  return (NULL);
+}
+
+/* =========================================================================
+ * Function: get_output_type
+ * Description:
+ *      Map a mode string onto a mode char e.g. OUTPUT_????
+ * Input:
+ * Output:
+ *      The mode char which is a subrange value.
+ * ========================================================================= */
+
+char
+get_output_type (void)
+{
+  char *env_str = GetDefEnv ("mode", "text");
+  char ch1 = toupper (env_str[0]);
+  char ch2 = toupper (env_str[1]);
+
+  switch (ch1)
+    {
+    case 'T':
+      return OUTPUT_TEXT;
+    case 'S':
+      return OUTPUT_SILENT;
+    case 'E':
+      return OUTPUT_EXTRAS;
+    case 'C':
+      return OUTPUT_COUNT;
+    case 'D':
+      return OUTPUT_DOCNUMS;
+    case 'H':
+      if (ch2 == 'E')
+	return OUTPUT_HEADERS;
+      else
+	return OUTPUT_HILITE;
+    default:
+      FatalError (1, "Problem in output type switch");
+    }
+
+  return '\0';			/*shouldn't reach here -- keep compiler happy */
+}
+
+/* =========================================================================
+ * Function: get_query_type
+ * Description:
+ *      Map a query type string onto a char.
+ * Input:
+ * Output:
+ *      Query type char.
+ * ========================================================================= */
+
+char
+get_query_type (void)
+{
+  char *env_str = GetDefEnv ("query", "boolean");
+  char ch1 = toupper (env_str[0]);
+
+  switch (ch1)
+    {
+    case 'R':
+      return QUERY_RANKED;
+    case 'A':
+      return QUERY_APPROX;
+    case 'B':
+      return QUERY_BOOLEAN;
+    case 'D':
+      return QUERY_DOCNUMS;
+    default:
+      FatalError (1, "Problem in query type switch");
+    }
+
+  return '\0';			/*shouldn't reach here */
+}
+
+
+
+/*
+ * This initialises certain environment variables
+ *
+ */
+void 
+InitEnv (void)
+{
+  SetEnv ("hilite_style", "underline", HiliteStyleCons);	/*[TS:Sep/94] */
+  SetEnv ("briefstats", "off", BooleanCons);
+  SetEnv ("diskstats", "off", BooleanCons);
+  SetEnv ("expert", "false", BooleanCons);
+  SetEnv ("mgdir", getenv ("MGDATA") ? getenv ("MGDATA") : ".", NULL);
+  SetEnv ("mgname", "", NULL);
+  SetEnv ("maxdocs", "all", MaxDocsCons);
+  SetEnv ("memstats", "off", BooleanCons);
+  SetEnv ("mode", "text", OutputTypeCons);
+  SetEnv ("pager", getenv ("PAGER") ? getenv ("PAGER") : "more", NULL);
+  SetEnv ("qfreq", "true", BooleanCons);
+  SetEnv ("query", "boolean", QueryCons);
+  SetEnv ("sizestats", "off", BooleanCons);
+  SetEnv ("timestats", "off", BooleanCons);
+  SetEnv ("verbatim", "off", BooleanCons);
+  SetEnv ("sorted_terms", "on", BooleanCons);
+  SetEnv ("accumulator_method", "array", AccumCons);
+  SetEnv ("stop_at_max_accum", "off", BooleanCons);
+  SetEnv ("buffer", "1048576", BufferCons);
+  SetEnv ("max_accumulators", "50000", MaxNodesCons);
+  SetEnv ("max_terms", "all", MaxTermsCons);
+  SetEnv ("maxparas", "1000", MaxParasCons);
+  SetEnv ("hash_tbl_size", "1000", MaxHashCons);
+  SetEnv ("skip_dump", "skips.%d", NULL);
+  SetEnv ("ranked_doc_sepstr", "---------------------------------- %n %w\\n", NULL);
+  SetEnv ("doc_sepstr", "---------------------------------- %n\\n", NULL);
+  SetEnv ("para_sepstr", "\\n######## PARAGRAPH %n ########\\n", NULL);
+  SetEnv ("para_start", "***** Weight = %w *****\\n", NULL);
+  SetEnv ("terminator", "", NULL);
+  SetEnv ("heads_length", "50", MaxHeadsCons);
+  SetEnv ("optimise_type", "1", OptimiseCons);	/*[TS:Mar/95] */
+  SetEnv ("casefold", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("stem", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("term_freq", "off", BooleanCons);  /* [RPAP - Feb 97: Term Frequency] */
+  SetEnv ("maxnumeric", "4", MaxNumericCons);  /* [sjboddie - Jun 2002: Max Numeric word length] */
+}
+
+
+void 
+UninitEnv (void)
+{
+  char *name;
+  while (PopEnv () == 0);
+  while ((name = GetEnvName (0)) != 0)
+    UnsetEnv (name, 1);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/environment.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * environment.h -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef ENVIRONMENT_H
+#define ENVIRONMENT_H
+
+#define OUTPUT_TEXT     'T'
+#define OUTPUT_HEADERS  'H'
+#define OUTPUT_SILENT   'S'
+#define OUTPUT_EXTRAS   'E'
+#define OUTPUT_HILITE   'I'
+#define OUTPUT_COUNT    'C'
+#define OUTPUT_DOCNUMS  'D'
+
+#define QUERY_RANKED    'R'
+#define QUERY_APPROX    'A'
+#define QUERY_BOOLEAN   'B'
+#define QUERY_DOCNUMS   'D'
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/globals.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/globals.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/globals.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * globals.h -- Some global stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_GLOBALS
+#define H_GLOBALS
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/hash.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/hash.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/hash.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_HASH
+#define H_HASH
+
+
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; len--)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.h	(revision 21112)
@@ -0,0 +1,320 @@
+static char *help_str[] = {
+"\n",
+"			  HELP for mgquery\n",
+"		   ================================\n",
+"\n",
+"The text is a summary of the information in the \"mgquery\" manual pages.\n",
+"\n",
+"The input to 'mgquery' consists of a series of input lines. The backslash\n",
+"character (\"\\\") is used at the end of lines to indicate that input \n",
+"continues on the next line.\n",
+"\n",
+"Input lines on which the first character is a dot (\".\") are commands to\n",
+"the mgquery program. Input lines that do not start with a dot are queries.\n",
+"\n",
+"A query consists of two parts. One part is a boolean or ranked query that\n",
+"identifies documents. The second part is a post-processing pattern matching\n",
+"operation. Any text between the first speech mark (\") and the last speech\n",
+"mark is considered to the a post-processing pattern. \n",
+"\n",
+"\n",
+"The following command are available :-\n",
+"\n",
+"    .help	     - displays this text.\n",
+"    .quit            - quits the program.\n",
+"    .set name value  - sets parameter \"name\" to \"value\" . If the parameter\n",
+"                       is a boolean parameter and value is omitted the\n",
+"	    	       parameter will be inverted (i.e. if it is true it\n",
+"		       will change to false, if it is false it will change\n",
+"		       to true).\n",
+"    .unset name      - deletes parameter \"name\"\n",
+"    .reset           - sets all the parameters to their initial state.\n",
+"    .display         - displays the values of all the current parameters.\n",
+"    .push            - pushes the current parameters on to a stack.\n",
+"    .pop             - destroys the current parameters and pops a new set\n",
+"                       of parameters off the stack.\n",
+"    .output arg      - This is used to specify where to send the documents.\n",
+"                       Arg may one be of the following:\n",
+"		       > filename  : Send output to the specified file.\n",
+"		       >> filename : Append output to the specified file.\n",
+"                       | command   : The output is piped into command, \n",
+"                                     which is executed by sh.\n",
+"    .input arg       - This is used to specify where input comes from.\n",
+"                       Arg may one be of the following:\n",
+"		       < filename  : Get the input from the specified\n",
+"                                     file.\n",
+"                       | command   : The input comes from the standard\n",
+"                                     output of command, which is executed\n",
+"                                     by sh.\n",
+"\n",
+"On startup the mgquery program reads from the file .mgrc a sequence \n",
+"of commands (NOTE: The .mgrc file may not contain any queries). mgquery \n",
+"first looks for .mgrc in the current directory and then in the users home \n",
+"directory. Lines starting with a '#' in the .mgrc file are considered to\n",
+"be comments and are ignored.\n",
+"\n",
+"The following parameters (used in the .set and .unset commands) are \n",
+"predefined and have special significance :-\n",
+"\n",
+"accumulator_method = `array'\n",
+"	This parameter is used during ranking, and specifies how the\n",
+"	weight for each document should be accumulated. The following\n",
+"	methods are available `array', `splay_tree', `hash_table', and\n",
+"	`list'.\n",
+"\n",
+"briefstats = `off'\n",
+"	This is a boolean parameter that determines whether the the\n",
+"	totals for disk, memory and time usage statistics will be \n",
+"	displayed. at the end of each query. \n",
+"	NOTE: this takes precedence over the parameters \"diskstats\",\n",
+"	\"memstats\" and \"timestats\". This parameter may take the values \n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"buffer = `1048576'\n",
+"	When the documents are being read in they are read into a \n",
+"	buffer of this size and then displayed from this buffer. If \n",
+"        the documents are larger than this buffer the buffer is\n",
+"	expanded automatically. Having a large buffer gives a very\n",
+"	slight performance improvement because it allows the order of \n",
+"	disk operations to be optimised. The buffer size is measured\n",
+"	in bytes.\n",
+"\n",
+"diskstats = `off'\n",
+"	This is a boolean parameter that determines whether the disk\n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"doc_sepstr = `---------------------------------- %n\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `boolean' or `docnums'\n",
+"	queries. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the MG document number use\n",
+"	the sequence `%n'.\n",
+"\n",
+"expert = `false'\n",
+"	If this is true then a lot of the waffle that the program \n",
+"	spits out is suppressed. This parameter may take the values\n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"hash_tbl_size = `1000'\n",
+"	One of the options during ranking queries is to use a hash \n",
+"	table to accumulate the weights for each document. The hash \n",
+"	table is a simple chained type. This parameter specifies the \n",
+"	size of the hash table and may take any value between 8 and\n",
+"	268435456. \n",
+"\n",
+"heads_length = `50'\n",
+"	When the mode is `heads' this specifies the number of \n",
+"	characters that	will be output for each document.\n",
+"\n",
+"maxdocs = `all'\n",
+"	The maximum number of documents to display in response to a\n",
+"	query. This parameter may take on a numeric value between 1 \n",
+"	and 429467295 or the word `all'.\n",
+"\n",
+"maxparas = `1000'\n",
+"	The maximum number of paragraphs to identify during a ranked\n",
+"	query with paragraph indexing. After the paragraphs have been\n",
+"	identified the paragraphs are converted into documents, and \n",
+"	because some of the paragraphs may refer to the same documents \n",
+"	the final number of answers may be less that maxparas. The \n",
+"	maxdocs parameter will then be applied. This parameter may \n",
+"	take on a numeric value between 1 and 429467295.\n",
+"\n",
+"max_accumulators = `50000'\n",
+"	This parameter limits the number of different paragraph/\n",
+"	document numbers to be accumulated during ranked queries when \n",
+"	the parameter `accumulator_method` is set to `splay_tree',\n",
+"	`hash_table', or `list'.  This parameter may take any value \n",
+"	between 8 and 268435456.\n",
+"\n",
+"max_terms = `all'\n",
+"	This parameter limits the number of terms that will actually\n",
+"	be used during a ranked query. If more terms than the number\n",
+"	specified by max_terms are entered, then the extra terms will\n",
+"	be discarded. If `sorted_terms' is on then the limiting will \n",
+"	be done after the terms have been sorted. This parameter may\n",
+"	take any value between 1 and 429467295 or the word `all'.\n",
+"\n",
+"memstats = `off'\n",
+"	This is a boolean parameter that determines whether the memory \n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes', \n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"mgdir = `.'\n",
+"	This specifies the directory where the MG files may be found.\n",
+"	If the environment variable `MGDATA' is set then `mgdir' is\n",
+"	initialised to the value in `MGDATA'.\n",
+"\n",
+"mgname = `'\n",
+"	This specifies the name of the MG database to process.\n",
+"\n",
+"mode = `text'\n",
+"	This specifies how documents should be displayed when they\n",
+"	are retrieved it may take four different values `text', \n",
+"	`docnums', `silent', `heads' or `count'. `text' displays \n",
+"	the contents of the document. `docnums' displays only the\n",
+"	document numbers. `Silent' retrieves all the documents but\n",
+"	displays nothing except how many documents were retrieved.\n",
+"	This mode is intended to be used in timing experiments. \n",
+"	`Heads` is used to print out the head of each document.\n",
+"	`Count' does the minimum amount of work required to determine\n",
+"	how many documents would be retrieved, but does not retrieve\n",
+"	them.\n",
+"\n",
+"pager = `more'\n",
+"	This is the name of the program that will be used to display\n",
+"	the help and the retrieved documents. If the environment \n",
+"	variable \"PAGER\" is defined then `pager' takes on that value.\n",
+"\n",
+"para_sepstr = `\\n######## PARAGRAPH %n ########\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	paragraphs. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the paragraph number within\n",
+"	the document use the sequence `%n'.\n",
+"\n",
+"para_start = `***** Weight = %w *****\\n'\n",
+"	This specifies the string that will be used at the head of \n",
+"	paragraphs for a paraghaph level index following a ranked query.\n",
+"	The standard C escape character sequences (see the man page)\n",
+"	may be used to place special characters in the string. For \n",
+"	example, a newline would the `\\n'. To include a `%' use the\n",
+"	sequence `%%'. To include the paragraph weight use the \n",
+"	sequence `%w'.\n",
+"\n",
+"qfreq = `true'\n",
+"	This determine whether the ranked queries will take into \n",
+"	account the number of times each query term is specified.\n",
+"	When this is `true' the number of times a term appears in\n",
+"	the query is used in the ranking. When this is `false' all \n",
+"	query term are assumed to occur only once. This parameter\n",
+"	may take the values `yes', `no', `true', `false', `on' or\n",
+"	`off'.\n",
+"\n",
+"query = `boolean'\n",
+"	This specifies the type of queries that are to be specified.\n",
+"	It can take four different values `boolean', `ranked', \n",
+"	`docnums' or `approx-ranked'. \n",
+"\n",
+"	`boolean' is for boolean queries. \n",
+"		The yacc grammar for boolean queries is as follows :-\n",
+"\n",
+"		query   : or;\n",
+" \n",
+"		or      : or '|' and\n",
+"		        | and ;	\n",
+" \n",
+"		and     : and '&' not\n",
+"        		| and not\n",
+"        		| not ;\n",
+"\n",
+"		not     : term\n",
+"		        | '!' not ;\n",
+" \n",
+"		term    : TERM\n",
+"		        | '(' or ')' ;\n",
+" \n",
+"\n",
+"\n",
+"\n",
+"	`ranked' and `approx-ranked' are for queries ranked by the\n",
+"		cosine measure. `approx-ranked' uses only the low\n",
+"		precision document lengths, and therefore only\n",
+"		produces an approximation to full cosine ranking.\n",
+"\n",
+"		query   : TERM\n",
+"      			| query TERM ;\n",
+"	\n",
+"	`docnums' allows the entry of document numbers. Multiple \n",
+"		numbers separated by spaces may be specified\n",
+"		or ranges separated by hyphens.\n",
+"\n",
+"\n",
+"		query   : range\n",
+"			| query range ;\n",
+"\n",
+"\n",
+"		range   : num\n",
+"			| num '-' num ;\n",
+"\n",
+"\n",
+"ranked_doc_sepstr = `---------------------------------- %n %w\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `ranked' or \n",
+"	`approx-ranked'	queries. The standard C escape character \n",
+"	sequences (see the man page) may be used to place special\n",
+"	characters in the string. For example, a newline would the\n",
+"	`\\n'. To include a `%' use the sequence `%%'. To include the\n",
+"	MG document number use the sequence `%n'. To include the\n",
+"	document weight use the sequence `%w'.\n",
+"\n",
+"sizestats = `false'\n",
+"	If this is true then various numbers are output at the end\n",
+"	of each query indicating what went on during the query. \n",
+"	This parameter may take the values `yes', `no', `true', \n",
+"	`false', `on' or `off'.\n",
+"\n",
+"skip_dump = `skips.%d'\n",
+"	If this parameter is set then during ranked queries on skipped\n",
+"	inverted files when `accumulator_method' is set to `splay_tree',\n",
+"	`hash_table', or `list` a file will be produced in the current\n",
+"	directory. The name of the file is the value of this parameter,\n",
+"	a `%d' in the file name will be replaced with the process id of\n",
+"	mgquery. This file will contain information about the usage of\n",
+"	skips during the query processing. This option is expensive; \n",
+"	use `.unset skip_dump' to obtain optimal performance.\n",
+"\n",
+"sorted_terms = `on'\n",
+"	This specifies whether of not the terms should be sorted into\n",
+"	decreasing occurrence in documents so that the least often\n",
+"	occurring terms are processed first when ranked queries are\n",
+"	being done. When this is true the terms are sorted. When this\n",
+"	is false the terms are not sorted and are instead processed in\n",
+"	order of occurrence. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"\n",
+"stop_at_max_accum = `on'\n",
+"	This specifies what should happen when the maximum number of\n",
+"	accumulators set by `max_accumulators' is reached. When this\n",
+"	is true the the processing of terms is stopped at the completion\n",
+"	of the current term. When this is false processing continues but\n",
+"	no new accumulators are created. This parameter may take the \n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"terminator = `'\n",
+"	This specifies the string that will be output after the	last\n",
+"	document from the previous query has been output. The standard\n",
+"	C escape character sequences (see the man page) may be used to\n",
+"	place special characters in the string. For example, a newline\n",
+"	would the `\\n'. To include a `%' use the sequence `%%'.\n",
+"	\n",
+"\n",
+"timestats = `false'\n",
+"	If this is true then the time to process a query is displayed\n",
+"	in both real time and CPU time. This parameter may take the\n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"verbatim = `off'\n",
+"	This is a boolean parameter that determines whether the program\n",
+"	should attempt to do a regular expression match on the retrieved\n",
+"	text. If verbatim is `on' and a post-processing strng is specified\n",
+"	with the query then the post-processing string will be searched for\n",
+"	in the documents just before they are displayed. If the string is\n",
+"	found the document will be displayed, if not the document will not\n",
+"	be displayed. If verbatim is `off' the post-processing string will\n",
+"	be considered a regular expression like in `vi' or `egrep'.\n",
+"	E.G. If verbatim is `on', \"and.*the\" will look for the 8 character\n",
+"	     sequence \"and.*the\". If verbatim is `off', \"and.*the\" will\n",
+"	     look for the sequence \"and\" followed somewhere later in the\n",
+"	     document by the sequence \"the\".\n",
+"	This parameter may take the values `yes', `no', `true', `false',\n",
+"	`on' or `off'.\n",
+"\n",
+"\n"};
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.src
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.src	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/help.mg.src	(revision 21112)
@@ -0,0 +1,347 @@
+###########################################################################
+#
+# help.mg.src -- Source for the help command
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)help.mg.src	1.8 21 Mar 1994
+#
+###########################################################################
+#
+# The help file for mgquery.  
+# Lines starting with '#' are treated as comments and are discarded when
+# the 'help.mg.h' file is produced
+#
+###########################################################################
+
+			  HELP for mgquery
+		   ================================
+
+The text is a summary of the information in the "mgquery" manual pages.
+
+The input to 'mgquery' consists of a series of input lines. The backslash
+character ("\") is used at the end of lines to indicate that input 
+continues on the next line.
+
+Input lines on which the first character is a dot (".") are commands to
+the mgquery program. Input lines that do not start with a dot are queries.
+
+A query consists of two parts. One part is a boolean or ranked query that
+identifies documents. The second part is a post-processing pattern matching
+operation. Any text between the first speech mark (") and the last speech
+mark is considered to the a post-processing pattern. 
+
+
+The following command are available :-
+
+    .help	     - displays this text.
+    .quit            - quits the program.
+    .set name value  - sets parameter "name" to "value" . If the parameter
+                       is a boolean parameter and value is omitted the
+	    	       parameter will be inverted (i.e. if it is true it
+		       will change to false, if it is false it will change
+		       to true).
+    .unset name      - deletes parameter "name"
+    .reset           - sets all the parameters to their initial state.
+    .display         - displays the values of all the current parameters.
+    .push            - pushes the current parameters on to a stack.
+    .pop             - destroys the current parameters and pops a new set
+                       of parameters off the stack.
+    .output arg      - This is used to specify where to send the documents.
+                       Arg may one be of the following:
+		       > filename  : Send output to the specified file.
+		       >> filename : Append output to the specified file.
+                       | command   : The output is piped into command, 
+                                     which is executed by sh.
+    .input arg       - This is used to specify where input comes from.
+                       Arg may one be of the following:
+		       < filename  : Get the input from the specified
+                                     file.
+                       | command   : The input comes from the standard
+                                     output of command, which is executed
+                                     by sh.
+
+On startup the mgquery program reads from the file .mgrc a sequence 
+of commands (NOTE: The .mgrc file may not contain any queries). mgquery 
+first looks for .mgrc in the current directory and then in the users home 
+directory. Lines starting with a '#' in the .mgrc file are considered to
+be comments and are ignored.
+
+The following parameters (used in the .set and .unset commands) are 
+predefined and have special significance :-
+
+accumulator_method = `array'
+	This parameter is used during ranking, and specifies how the
+	weight for each document should be accumulated. The following
+	methods are available `array', `splay_tree', `hash_table', and
+	`list'.
+
+briefstats = `off'
+	This is a boolean parameter that determines whether the the
+	totals for disk, memory and time usage statistics will be 
+	displayed. at the end of each query. 
+	NOTE: this takes precedence over the parameters "diskstats",
+	"memstats" and "timestats". This parameter may take the values 
+	`yes', `no', `true', `false', `on' or `off'.
+
+buffer = `1048576'
+	When the documents are being read in they are read into a 
+	buffer of this size and then displayed from this buffer. If 
+        the documents are larger than this buffer the buffer is
+	expanded automatically. Having a large buffer gives a very
+	slight performance improvement because it allows the order of 
+	disk operations to be optimised. The buffer size is measured
+	in bytes.
+
+diskstats = `off'
+	This is a boolean parameter that determines whether the disk
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+doc_sepstr = `---------------------------------- %n\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `boolean' or `docnums'
+	queries. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the MG document number use
+	the sequence `%n'.
+
+expert = `false'
+	If this is true then a lot of the waffle that the program 
+	spits out is suppressed. This parameter may take the values
+	`yes', `no', `true', `false', `on' or `off'.
+
+hash_tbl_size = `1000'
+	One of the options during ranking queries is to use a hash 
+	table to accumulate the weights for each document. The hash 
+	table is a simple chained type. This parameter specifies the 
+	size of the hash table and may take any value between 8 and
+	268435456. 
+
+heads_length = `50'
+	When the mode is `heads' this specifies the number of 
+	characters that	will be output for each document.
+
+maxdocs = `all'
+	The maximum number of documents to display in response to a
+	query. This parameter may take on a numeric value between 1 
+	and 429467295 or the word `all'.
+
+maxparas = `1000'
+	The maximum number of paragraphs to identify during a ranked
+	query with paragraph indexing. After the paragraphs have been
+	identified the paragraphs are converted into documents, and 
+	because some of the paragraphs may refer to the same documents 
+	the final number of answers may be less that maxparas. The 
+	maxdocs parameter will then be applied. This parameter may 
+	take on a numeric value between 1 and 429467295.
+
+max_accumulators = `50000'
+	This parameter limits the number of different paragraph/
+	document numbers to be accumulated during ranked queries when 
+	the parameter `accumulator_method` is set to `splay_tree',
+	`hash_table', or `list'.  This parameter may take any value 
+	between 8 and 268435456.
+
+max_terms = `all'
+	This parameter limits the number of terms that will actually
+	be used during a ranked query. If more terms than the number
+	specified by max_terms are entered, then the extra terms will
+	be discarded. If `sorted_terms' is on then the limiting will 
+	be done after the terms have been sorted. This parameter may
+	take any value between 1 and 429467295 or the word `all'.
+
+memstats = `off'
+	This is a boolean parameter that determines whether the memory 
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes', 
+	`no', `true', `false', `on' or `off'.
+
+mgdir = `.'
+	This specifies the directory where the MG files may be found.
+	If the environment variable `MGDATA' is set then `mgdir' is
+	initialised to the value in `MGDATA'.
+
+mgname = `'
+	This specifies the name of the MG database to process.
+
+mode = `text'
+	This specifies how documents should be displayed when they
+	are retrieved it may take four different values `text', 
+	`docnums', `silent', `heads' or `count'. `text' displays 
+	the contents of the document. `docnums' displays only the
+	document numbers. `Silent' retrieves all the documents but
+	displays nothing except how many documents were retrieved.
+	This mode is intended to be used in timing experiments. 
+	`Heads` is used to print out the head of each document.
+	`Count' does the minimum amount of work required to determine
+	how many documents would be retrieved, but does not retrieve
+	them.
+
+pager = `more'
+	This is the name of the program that will be used to display
+	the help and the retrieved documents. If the environment 
+	variable "PAGER" is defined then `pager' takes on that value.
+
+para_sepstr = `\n######## PARAGRAPH %n ########\n'
+	This specifies the string that will be used to separate 
+	paragraphs. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the paragraph number within
+	the document use the sequence `%n'.
+
+para_start = `***** Weight = %w *****\n'
+	This specifies the string that will be used at the head of 
+	paragraphs for a paraghaph level index following a ranked query.
+	The standard C escape character sequences (see the man page)
+	may be used to place special characters in the string. For 
+	example, a newline would the `\n'. To include a `%' use the
+	sequence `%%'. To include the paragraph weight use the 
+	sequence `%w'.
+
+qfreq = `true'
+	This determine whether the ranked queries will take into 
+	account the number of times each query term is specified.
+	When this is `true' the number of times a term appears in
+	the query is used in the ranking. When this is `false' all 
+	query term are assumed to occur only once. This parameter
+	may take the values `yes', `no', `true', `false', `on' or
+	`off'.
+
+query = `boolean'
+	This specifies the type of queries that are to be specified.
+	It can take four different values `boolean', `ranked', 
+	`docnums' or `approx-ranked'. 
+
+	`boolean' is for boolean queries. 
+		The yacc grammar for boolean queries is as follows :-
+
+		query   : or;
+ 
+		or      : or '|' and
+		        | and ;	
+ 
+		and     : and '&' not
+        		| and not
+        		| not ;
+
+		not     : term
+		        | '!' not ;
+ 
+		term    : TERM
+		        | '(' or ')' ;
+ 
+
+
+
+	`ranked' and `approx-ranked' are for queries ranked by the
+		cosine measure. `approx-ranked' uses only the low
+		precision document lengths, and therefore only
+		produces an approximation to full cosine ranking.
+
+		query   : TERM
+      			| query TERM ;
+	
+	`docnums' allows the entry of document numbers. Multiple 
+		numbers separated by spaces may be specified
+		or ranges separated by hyphens.
+
+
+		query   : range
+			| query range ;
+
+
+		range   : num
+			| num '-' num ;
+
+
+ranked_doc_sepstr = `---------------------------------- %n %w\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `ranked' or 
+	`approx-ranked'	queries. The standard C escape character 
+	sequences (see the man page) may be used to place special
+	characters in the string. For example, a newline would the
+	`\n'. To include a `%' use the sequence `%%'. To include the
+	MG document number use the sequence `%n'. To include the
+	document weight use the sequence `%w'.
+
+sizestats = `false'
+	If this is true then various numbers are output at the end
+	of each query indicating what went on during the query. 
+	This parameter may take the values `yes', `no', `true', 
+	`false', `on' or `off'.
+
+skip_dump = `skips.%d'
+	If this parameter is set then during ranked queries on skipped
+	inverted files when `accumulator_method' is set to `splay_tree',
+	`hash_table', or `list` a file will be produced in the current
+	directory. The name of the file is the value of this parameter,
+	a `%d' in the file name will be replaced with the process id of
+	mgquery. This file will contain information about the usage of
+	skips during the query processing. This option is expensive; 
+	use `.unset skip_dump' to obtain optimal performance.
+
+sorted_terms = `on'
+	This specifies whether of not the terms should be sorted into
+	decreasing occurrence in documents so that the least often
+	occurring terms are processed first when ranked queries are
+	being done. When this is true the terms are sorted. When this
+	is false the terms are not sorted and are instead processed in
+	order of occurrence. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+
+stop_at_max_accum = `on'
+	This specifies what should happen when the maximum number of
+	accumulators set by `max_accumulators' is reached. When this
+	is true the the processing of terms is stopped at the completion
+	of the current term. When this is false processing continues but
+	no new accumulators are created. This parameter may take the 
+	values `yes', `no', `true', `false', `on' or `off'.
+
+terminator = `'
+	This specifies the string that will be output after the	last
+	document from the previous query has been output. The standard
+	C escape character sequences (see the man page) may be used to
+	place special characters in the string. For example, a newline
+	would the `\n'. To include a `%' use the sequence `%%'.
+	
+
+timestats = `false'
+	If this is true then the time to process a query is displayed
+	in both real time and CPU time. This parameter may take the
+	values `yes', `no', `true', `false', `on' or `off'.
+
+verbatim = `off'
+	This is a boolean parameter that determines whether the program
+	should attempt to do a regular expression match on the retrieved
+	text. If verbatim is `on' and a post-processing strng is specified
+	with the query then the post-processing string will be searched for
+	in the documents just before they are displayed. If the string is
+	found the document will be displayed, if not the document will not
+	be displayed. If verbatim is `off' the post-processing string will
+	be considered a regular expression like in `vi' or `egrep'.
+	E.G. If verbatim is `on', "and.*the" will look for the 8 character
+	     sequence "and.*the". If verbatim is `off', "and.*the" will
+	     look for the sequence "and" followed somewhere later in the
+	     document by the sequence "the".
+	This parameter may take the values `yes', `no', `true', `false',
+	`on' or `off'.
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_INVF
+#define H_INVF
+
+/* NOTE: This does not include the magic number */
+struct invf_dict_header
+  {
+    unsigned long lookback;
+    unsigned long dict_size;
+    unsigned long total_bytes;
+    unsigned long index_string_bytes;
+    double input_bytes; /* [RJM 07/97: 4G limit] */
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+  };
+
+struct stem_dict_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+    unsigned long indexed;       /* [RPAP - Jan 97: Stem Index Change] */
+  };
+
+struct invf_file_header
+  {
+    unsigned long no_of_words;
+    unsigned long no_of_ptrs;
+    unsigned long skip_mode;
+    unsigned long params[16];
+    unsigned long InvfLevel;
+  };
+
+/* [RPAP - Jan 97: Stem Index Change] */
+struct stem_idx_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_words;
+  };
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.c	(revision 21112)
@@ -0,0 +1,1666 @@
+/**************************************************************************
+ *
+ * invf_get.c -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "bitio_gen.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "sptree.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "invf_get.h"
+#include "mg_errors.h"
+
+#define SKIPPING
+#define SHORT_CUT
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    Invf_Doc_Entry ide[1];
+  };
+
+static int skip_header = 0;
+
+static char skip_header_text0[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of docnum/word_count entries decoded.\n"
+"# 6\tNumber of hits.\n"
+"# 7\tNumber of entries in the splay tree or hash table.\n"
+"# 8\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char skip_header_text1[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of skips taken in evaluating the inverted file entry.\n"
+"# 6\tNumber of docnum/word_count entries decoded.\n"
+"# 7\tNumber of hits.\n"
+"# 8\tNumber of entries in the splay tree or hash table.\n"
+"# 9\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+
+static char skip_header_text2[] =
+"#######################################################################\n"
+"#\n"
+"#  1\tWord number.\n"
+"#  2\tNumber of time the word occurs in the collection.\n"
+"#  3\tNumber of documents the word occurs in.\n"
+"#  4\tThe word.\n"
+"#  5\tSkip size.\n"
+"#  6\tNumber of skips taken in evaluating the inverted file entry.\n"
+"#  7\tNumber of docnum/word_count entries decoded.\n"
+"#  8\tNumber of hits.\n"
+"#  9\tNumber of entries in the splay tree or hash table.\n"
+"# 10\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char *skip_header_text[] =
+{skip_header_text0,
+ skip_header_text1,
+ skip_header_text2};
+
+
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static Hash_Table *
+HT_create (query_data * qd, unsigned max_entries,
+	   unsigned max_size)
+{
+  unsigned es, hts;
+  Hash_Table *HT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * (max_entries - 1);
+  hts = sizeof (Invf_Doc_EntryH *) * max_size;
+
+  HT = Xmalloc (es);
+  if (!HT)
+    return NULL;
+
+  HT->HashTable = Xmalloc (hts);
+  if (!(HT->HashTable))
+    {
+      Xfree (HT);
+      return NULL;
+    }
+
+  ChangeMemInUse (qd, es + hts);
+  bzero ((char *) (HT->HashTable), hts);
+  HT->max = max_entries;
+  HT->num = 0;
+  HT->Suplimentary_Entries = NULL;
+  HT->Suplimentary_Size = 0;
+  HT->Suplimentary_Num = 0;
+  HT->Head = NULL;
+  HT->size = max_size;
+  return HT;
+}
+
+
+static void 
+HT_Sup_create (query_data * qd, Hash_Table * HT, unsigned max_size)
+{
+  int es = sizeof (Invf_Doc_EntryH) * max_size;
+  HT->Suplimentary_Entries = Xmalloc (es);
+  if (!HT->Suplimentary_Entries)
+    return;
+  HT->Suplimentary_Size = max_size;
+  HT->Suplimentary_Num = 0;
+  ChangeMemInUse (qd, es);
+}
+
+static void 
+HT_Sup_Add (Hash_Table * HT, unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH *IDEH;
+  IDEH = &HT->Suplimentary_Entries[HT->Suplimentary_Num++];
+  IDEH->next = NULL;
+  IDEH->IDE.DocNum = DN;
+  IDEH->IDE.Sum = Sum;
+}
+
+Invf_Doc_Entry *
+HT_find (Hash_Table * HT, unsigned long DN)
+{
+  Invf_Doc_EntryH *IDEH;
+  if (!HT->HashTable)
+    FatalError (1, "Accessing the hash_table through HT_find after HT_sort");
+  IDEH = HT->HashTable[DN % HT->size];
+  while (IDEH)
+    {
+      if (IDEH->IDE.DocNum == DN)
+	return &(IDEH->IDE);
+      IDEH = IDEH->next;
+    }
+  return NULL;
+}
+
+
+static int 
+HT_insert (Hash_Table * HT,
+	   unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH **IDEH;
+  if (HT->num == HT->max)
+    return 0;
+  IDEH = &(HT->HashTable[DN % HT->size]);
+  while (*IDEH)
+    IDEH = &((*IDEH)->next);
+  *IDEH = &(HT->IDEH[HT->num++]);
+  (*IDEH)->next = NULL;
+  (*IDEH)->IDE.DocNum = DN;
+  (*IDEH)->IDE.Sum = Sum;
+  return 1;
+}
+
+
+void 
+HT_free (query_data * qd, Hash_Table * HT)
+{
+  unsigned es;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * HT->max +
+    sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  if (HT->Suplimentary_Entries)
+    {
+      Xfree (HT->Suplimentary_Entries);
+      es += sizeof (Invf_Doc_EntryH) * HT->Suplimentary_Size;
+    }
+  Xfree (HT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDEH_comp (const void *a, const void *b)
+{
+  const Invf_Doc_EntryH *A = a;
+  const Invf_Doc_EntryH *B = b;
+  return A->IDE.DocNum - B->IDE.DocNum;
+}
+
+static void 
+HT_sort (query_data * qd, Hash_Table * HT)
+{
+  int i;
+  Invf_Doc_EntryH *l, *r, **b;
+  unsigned es = sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  HT->size = 0;
+  HT->HashTable = NULL;
+  ChangeMemInUse (qd, -es);
+
+  qsort (HT->IDEH, HT->num, sizeof (Invf_Doc_EntryH), IDEH_comp);
+
+  for (i = 0; i < HT->num - 1; i++)
+    HT->IDEH[i].next = &HT->IDEH[i + 1];
+  HT->IDEH[i].next = NULL;
+  l = HT->IDEH;
+
+  if (HT->Suplimentary_Entries)
+    {
+      for (i = 0; i < HT->Suplimentary_Num - 1; i++)
+	HT->Suplimentary_Entries[i].next = &HT->Suplimentary_Entries[i + 1];
+      HT->Suplimentary_Entries[i].next = NULL;
+    }
+  r = HT->Suplimentary_Entries;
+
+  b = &HT->Head;
+
+  while (l || r)
+    {
+      if (r && (!l || (l->IDE.DocNum > r->IDE.DocNum)))
+	{
+	  *b = r;
+	  b = &r->next;
+	  r = *b;
+	}
+      else
+	{
+	  *b = l;
+	  b = &l->next;
+	  l = *b;
+	}
+    }
+  *b = NULL;
+}
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static List_Table *
+LT_create (query_data * qd, unsigned max_entries)
+{
+  unsigned es;
+  List_Table *LT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * (max_entries - 1);
+
+  LT = Xmalloc (es);
+  if (!LT)
+    return NULL;
+
+  ChangeMemInUse (qd, es);
+  LT->max = max_entries;
+  LT->num = 0;
+  return LT;
+}
+
+
+
+Invf_Doc_Entry *
+LT_find (List_Table * LT, unsigned long DN)
+{
+  int l = 0;
+  int r = LT->num - 1;
+  int m, c;
+  while (l <= r)
+    {
+      m = (l + r) / 2;
+      c = DN - LT->IDE[m].DocNum;
+      if (c < 0)
+	r = m - 1;
+      else if (c > 0)
+	l = m + 1;
+      else
+	return &LT->IDE[m];
+    }
+  return NULL;
+}
+
+static List_Table *
+LT_add (query_data * qd, List_Table * LT,
+	unsigned long DN, float Sum)
+{
+  Invf_Doc_Entry *ide;
+
+  /* increase the size of the list if there is not enough */
+  /* room for this entry */
+  if (LT->num == LT->max)
+    {
+      int old, new, max_entries;
+      old = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+      max_entries = LT->max + (LT->max >> 1);
+      new = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * max_entries;
+
+      LT = Xrealloc (LT, new);
+      if (!LT)
+	return NULL;
+
+      ChangeMemInUse (qd, new - old);
+      LT->max = max_entries;
+    }
+
+  /* add the entry to the end of the list */
+  ide = &LT->IDE[LT->num];
+  ide->DocNum = DN;
+  ide->Sum = Sum;
+  LT->num++;
+
+  return LT;
+}
+
+
+void 
+LT_free (query_data * qd, List_Table * LT)
+{
+  unsigned es;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+  Xfree (LT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDE_comp (const void *a, const void *b)
+{
+  const Invf_Doc_Entry *A = a;
+  const Invf_Doc_Entry *B = b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+LT_sort (List_Table * LT)
+{
+  qsort (LT->IDE, LT->num, sizeof (Invf_Doc_Entry), IDE_comp);
+}
+
+
+/* [RJM 07/97: Ranked Required Terms] */
+/* accumulators with a sum less than or equal to 0.0 are to be deleted */
+static void 
+LT_pack (List_Table * LT)
+{
+  int s, d = 0;
+  for (s = 0; s < LT->num; s++) {
+      if (LT->IDE[s].Sum <= 0.0) {
+	/* ignore this accumulator */
+
+      } else if (d > 0 && LT->IDE[s].DocNum == LT->IDE[d-1].DocNum) {
+	LT->IDE[d-1].Sum += LT->IDE[s].Sum;
+
+      } else {
+	if (d != s) LT->IDE[d] = LT->IDE[s];
+	d++;
+      }
+  }
+
+  if (LT->num) LT->num = d;
+}
+
+
+/**************************************************************************/
+
+
+
+invf_data *
+InitInvfFile (File * InvfFile, stemmed_dict * sd)
+{
+  invf_data *id;
+  if (!(id = Xmalloc (sizeof (invf_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return NULL;
+    }
+
+  id->InvfFile = InvfFile;
+
+  Fread (&id->ifh, sizeof (id->ifh), 1, InvfFile);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  {
+    int i;
+    
+    NTOHUL(id->ifh.no_of_words);
+    NTOHUL(id->ifh.no_of_ptrs);
+    NTOHUL(id->ifh.skip_mode);
+    for (i = 0; i < 16; i++)
+      NTOHUL(id->ifh.params[i]);
+    NTOHUL(id->ifh.InvfLevel);
+  }
+
+  id->N = sd->sdh.num_of_docs;
+  id->Nstatic = sd->sdh.static_num_of_docs;
+
+  return (id);
+}
+
+
+void 
+FreeInvfData (invf_data * id)
+{
+  Xfree (id);
+}
+
+/****************************************************************************/
+
+
+static void 
+CalcBlks (query_data * qd, WordEntry * we, int *blk,
+	  int *dn_blk, int *len_blk, int *k, int *p)
+{
+  *p = we->doc_count;
+
+  *blk = BIO_Bblock_Init (qd->id->Nstatic, *p);
+  switch (qd->id->ifh.skip_mode)
+    {
+    case 1:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[0])
+	  *k = 0;
+	else
+	  {
+	    *k = qd->id->ifh.params[0];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs, (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    case 2:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[1])
+	  *k = 0;
+	else
+	  {
+	    *k = (int) (2 * sqrt (((double) (*p)) / qd->id->ifh.params[0]));
+	    if (*k <= qd->id->ifh.params[1])
+	      *k = qd->id->ifh.params[1];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs,
+				       (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    default:
+      *k = 0;
+    }
+}
+
+
+/* =========================================================================
+ * Function: GetDocsOp
+ * Description: 
+ *      if op == op_term then 
+ *        look up term in invf and decode all its entries/documents
+ *      if op == op_and_terms then 
+ *        go thru L, testing if it exists in invf using skips  
+ *        if it does exist then overwrite the entry nearer the start of the list
+ *      if op == op_and_or_terms then
+ *        go thru L, testing if it exists in invf using skips  
+ *        BUT only mark ones which are found - keep list intact
+ *      if op == op_diff_terms then     >>>> NOT tested <<<<
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+DocList *
+GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L)
+{
+  u_char *buffer;
+  DocEntry *src = NULL, *dest = NULL;
+  int i;			/* index counter into inverted list */
+  unsigned long pos;
+  int next_start, next_doc_num, next_i;
+  int block_counter;		/* within counter in block */
+  int dn_blk = 0, len_blk = 0;
+  int skip_size, have_skipping;
+  int blk;			/* bblock control parameter */
+  int ft;			/* The number of documents the word occurs in */
+  unsigned long CurrDoc = 0;	/* NOTE: Document numbers start at 1 */
+  float Weight;
+
+  /* ... AND empty */
+  if (op == op_and_terms && (!L || !L->num))
+    {
+      if (L)
+	Xfree (L);
+      return MakeDocList (0);
+    }
+
+  if (op == op_term)
+    L = MakeDocList (we->doc_count);
+
+  CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &skip_size, &ft);
+  have_skipping = (skip_size != 0);
+  if (!(buffer = Xmalloc (we->invf_len)))
+    return MakeDocList (0);
+
+  ChangeMemInUse (qd, we->invf_len);
+
+  qd->num_of_terms++;
+
+  /* --- Read an inverted file entry for the given word --- */
+  Fseek (qd->File_invf, we->invf_ptr, 0);
+  Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+  /* Read from the buffer */
+  DECODE_START (buffer, we->invf_len);
+
+  Weight = qd->sd->sdh.num_of_words / (100 * log2 ((float) we->count + 1));
+
+  if (op != op_term)
+    dest = src = L->DE;
+  i = pos = 0;
+  next_i = next_start = next_doc_num = 0;
+  block_counter = skip_size - 1;
+
+  /* Go thru the elements of an inverted list for a term */
+  /* May not go sequentially if skipping is done */
+  while (i < ft && ((op == op_term) || src - L->DE < L->num))
+    {
+      unsigned long diff, Temp;
+
+      if (have_skipping)
+	{
+
+	  /* --- Skip to next block --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    {
+	    ShortCut:
+	      i = next_i;
+	      DECODE_SEEK (next_start);
+	      pos = next_start;
+	      CurrDoc = next_doc_num;
+	      qd->hops_taken++;
+	      block_counter = skip_size - 1;
+	    }
+
+	  /* --- Decode the skip block header --- */
+	  if (block_counter == skip_size - 1)
+	    {
+	      if (i + skip_size < ft)
+		{
+		  /* --- Decode the skip block header --- */
+		  int doc_diff, pos_diff;
+		  BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		  next_doc_num += doc_diff;
+		  next_i += skip_size;
+		  BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		  next_start = pos + pos_diff;
+		}
+	      else
+		next_doc_num = -1;
+	    }
+
+	  /* --- Do a skip if c-set doc is higher than block header doc --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    goto ShortCut;
+
+
+	}			/*if skipping */
+
+      /* --- increment CurrDoc --- */
+      if (have_skipping && block_counter == 0 && i != ft - 1)
+	CurrDoc = next_doc_num;
+      else
+	{
+	  BBLOCK_DECODE_L (diff, blk, pos);
+	  CurrDoc += diff;
+	}
+
+      /* --- If have f_dt's then eat them up --- */
+      if (qd->id->ifh.InvfLevel >= 2)
+	GAMMA_DECODE_L (Temp, pos);	/* discard word count */
+
+      /* --- Increment counters --- */
+      qd->num_of_ptrs++;
+      i++;
+      block_counter--;
+      if (block_counter < 0)	/* ??? Why would this happen ? */
+	block_counter = skip_size - 1;
+
+      switch (op)
+	{
+	case op_term:
+	  /* --- Add the doc from invf onto the c-set list --- */
+	  L->DE[L->num].DocNum = CurrDoc;
+	  L->DE[L->num].Weight = Weight;
+	  L->DE[L->num].SeekPos = 0;
+	  L->DE[L->num].Len = 0;
+	  L->DE[L->num].CompTextBuffer = NULL;
+	  L->DE[L->num].Next = NULL;
+	  L->DE[L->num].or_included = 0;
+	  L->num++;
+	  break;
+
+	case op_and_terms:
+	case op_and_or_terms:
+	  /* --- see if have a match and save it --- */
+
+	  while (src < (L->DE + L->num) && CurrDoc > src->DocNum)
+	    {
+	      src++;
+	    }
+	  /* Unless at end of list, assert(CurrDoc <= src->DocNum) */
+
+	  /* --- Accept the doc as a candidate --- */
+	  if (src < (L->DE + L->num) && CurrDoc == src->DocNum)
+	    {
+	      if (op == op_and_terms)
+		{
+		  *dest++ = *src;
+		}
+	      else
+		{
+		  src->or_included = 1;
+		}
+	      src++;		/* Move onto next one */
+	    }
+
+	  /* If the next one to try has already been tried then move on again */
+	  if (op == op_and_or_terms)
+	    {
+	      while (src < (L->DE + L->num) && src->or_included)
+		src++;
+	    }
+
+	  break;
+
+	case op_diff_terms:
+	  while (src < (L->DE + L->num) && src->DocNum < CurrDoc)
+	    *dest++ = *src++;
+	  if (src < (L->DE + L->num) && src->DocNum == CurrDoc)
+	    src++;
+	  break;
+
+	default:
+	  break;
+	}			/*switch */
+
+    }				/*while traversing invf */
+
+  if (op == op_and_terms || op == op_diff_terms)
+    {
+      if (op == op_diff_terms)
+	while (src - L->DE < L->num)
+	  *dest++ = *src++;
+
+      /* update size of c-set */
+      L->num = dest - L->DE;
+    }
+  DECODE_DONE
+
+    Xfree (buffer);
+  ChangeMemInUse (qd, -we->invf_len);
+
+  return (L);
+}
+
+
+
+
+
+
+
+float *
+CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi)
+{
+  float *AccumulatedWeights;
+  int j, num_docs, num_terms;
+  int seen_must_match = 0; /* [RJM 07/97: Ranked Required Terms] says whether to 'and' queries or not */
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  if (!(AccumulatedWeights = Xmalloc (num_docs * sizeof (float))))
+      return (NULL);
+
+  bzero ((char *) AccumulatedWeights, sizeof (float) * num_docs);
+  ChangeMemInUse (qd, sizeof (float) * num_docs);
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      u_char *buffer;
+      int i, kd;
+      unsigned long next_mjr_dn = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long TempDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Terms] */
+      WordEntry *we = &Terms->TE[j].WE;
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      kd = 0;
+      for (i = 0; i < p; i++, kd++)
+	{
+	  int Count;		/* The number of times the word occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  if (kd == k)
+	    {
+	      kd = 0;
+	    }
+	  if (k && kd == 0)
+	    if (i + k < p)
+	      {
+		int temp;
+		BBLOCK_DECODE (next_mjr_dn, dn_blk);
+		next_mjr_dn += CurrDocNum;
+		BBLOCK_DECODE (temp, len_blk);
+	      }
+	  if (k && kd == k - 1 && i != p - 1)
+	    CurrDocNum = next_mjr_dn;
+	  else
+	    {
+	      BBLOCK_DECODE (diff, blk);
+	      CurrDocNum += diff;
+	    }
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE (Count);
+	  else
+	    Count = (double) we->count / p;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* zero the weights of intermediate documents if this term is required to match */
+	  if (require_match == 1) {
+	    for (TempDocNum = LastDocNum+1; TempDocNum < CurrDocNum; TempDocNum++)
+	      AccumulatedWeights[TempDocNum - 1] = 0.0;
+	  }
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if a term has been required to match before, the weight we are */
+	  /* adding to must be non-zero */
+	  if (!seen_must_match || (AccumulatedWeights[CurrDocNum - 1] > 0.0)) {
+	      AccumulatedWeights[CurrDocNum - 1] += Wqt * Wdt;
+	  }
+
+	  qd->num_of_ptrs++;
+	  LastDocNum = CurrDocNum; /* [RJM 07/97: Ranked Required Terms] */
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* zero the remaining weights if this term is required to match */
+      if (require_match == 1) {
+	for (TempDocNum = LastDocNum+1; TempDocNum <= num_docs; TempDocNum++)
+	  AccumulatedWeights[TempDocNum - 1] = 0.0;
+	seen_must_match = 1;
+      }
+    }
+  return AccumulatedWeights;
+}
+
+
+
+static int 
+doc_comp (void *a, void *b)
+{
+  return ((Invf_Doc_Entry *) a)->DocNum - ((Invf_Doc_Entry *) b)->DocNum;
+}
+
+
+static Invf_Doc_Entry *
+get_ide (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  struct pool *p = pool->pool;
+  if (!p || p->pos == p->length)
+    {
+      register unsigned size = sizeof (struct pool) +
+      (pool->pool_chunks - 1) * sizeof (Invf_Doc_Entry);
+      p = malloc (size);
+      if (!p)
+	return NULL;
+      ChangeMemInUse (qd, size);
+      p->length = pool->pool_chunks;
+      p->pos = 0;
+      p->next = pool->pool;
+      pool->pool = p;
+    }
+  return &(p->ide[p->pos++]);
+}
+
+
+void 
+free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  while (pool->pool)
+    {
+      struct pool *p = pool->pool;
+      pool->pool = p->next;
+      ChangeMemInUse (qd, -(sizeof (struct pool) + (p->length - 1) *
+			    sizeof (Invf_Doc_Entry)));
+      free (p);
+    }
+  pool->pool = NULL;
+}
+
+
+FILE *
+open_skip (char *sd)
+{
+  char buf[256];
+  if (sd == NULL || *sd == '\0')
+    return NULL;
+  sprintf (buf, sd, getpid ());
+  return fopen (buf, "a");
+}
+
+
+
+Splay_Tree *
+CosineDecodeSplay (query_data * qd, TermList * Terms,
+		   RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool)
+{
+  Splay_Tree *Doc_Set;
+  int j, num_docs, num_terms, MoreAccum;
+  int Anding = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  Doc_Set = SP_createset (doc_comp);
+
+  pool->pool = NULL;
+  pool->pool_chunks = 1024;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos, i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && Doc_Set->no_of_items >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  next = SP_get_first (Doc_Set);
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry ent, *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->DocNum < CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      if (next && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->DocNum <= CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      goto NextDocNum;
+	    }
+	  ent.DocNum = CurrDocNum - 1;
+	  if ((mem = SP_member (&ent, Doc_Set)) == NULL)
+	    {
+	      hits++;
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      if ((mem = get_ide (qd, pool)) == NULL)
+		return NULL;
+	      ChangeMemInUse (qd, sizeof (*mem));
+	      mem->DocNum = CurrDocNum - 1;
+	      mem->Sum = 0;
+	      ChangeMemInUse (qd, -Doc_Set->mem_in_use);
+	      if (SP_insert (mem, Doc_Set) == NULL)
+		{
+		  Message ("Unable to allocate memory for a splay node");
+		  Abort = 1;
+		  goto FastAbort;
+		}
+	      ChangeMemInUse (qd, Doc_Set->mem_in_use);
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+
+	  mem->Sum += Wqt * Wdt;
+	  total_added += Wqt * Wdt;
+
+	NextDocNum:
+	  ;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, Doc_Set->no_of_items, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  return Doc_Set;
+}
+
+
+
+
+Hash_Table *
+CosineDecodeHash (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  Hash_Table *HT;
+  int Anding = 0;
+  int Sorted = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  HT = HT_create (qd, rqi->MaxAccums, rqi->HashTblSize);
+  if (!HT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_EntryH *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  if (!Sorted)
+	    {
+	      HT_sort (qd, HT);
+	      Sorted = 1;
+	    }
+	  next = HT->Head;
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+#ifdef SHORT_CUT
+		ShortCut:
+#endif
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		{
+		  if (i + k < p)
+		    {
+		      int doc_diff, pos_diff;
+		      BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		      next_doc_num += doc_diff;
+		      next_i += k;
+		      BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		      next_start = pos + pos_diff;
+		    }
+		  else
+		    next_doc_num = -1;
+		}
+#ifdef SHORT_CUT
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		goto ShortCut;
+#else
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+#endif
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    {
+	      CurrDocNum = next_doc_num;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->IDE.DocNum < CurrDocNum - 1)
+		next = next->next;
+	      if (next && next->IDE.DocNum == CurrDocNum - 1)
+		{
+		  Wdt = Count * WordLog;
+		  next->IDE.Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->IDE.DocNum <= CurrDocNum - 1)
+		next = next->next;
+	      goto NextDocNum;
+	    }
+	  if ((mem = HT_find (HT, CurrDocNum - 1)) == NULL)
+	    {
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      Wdt = Count * WordLog;
+	      if (!HT_insert (HT, CurrDocNum - 1, Wqt * Wdt))
+		{
+		  if (!HT->Suplimentary_Entries)
+		    HT_Sup_create (qd, HT, we->doc_count - i + 1);
+		  HT_Sup_Add (HT, CurrDocNum - 1, Wqt * Wdt);
+		}
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+	  else
+	    {
+	      Wdt = Count * WordLog;
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+	  ;
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+	       ptrs_dec, hits, HT->num + HT->Suplimentary_Num, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  if (!HT->Head)
+    HT_sort (qd, HT);
+  return HT;
+}
+
+
+
+List_Table *
+CosineDecodeList (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  List_Table *LT;
+  int Anding = 0;
+  int Sorted = 0;
+  int has_grown = 0; /* [RJM 07/97: Ranked Required Match] whether elements have been added lately */
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+
+  /* Create the list for the documents */
+  LT = LT_create (qd, rqi->MaxAccums);
+  if (!LT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i, n = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0;	/* [RJM 07/97: Ranked Required Match] */
+      int this_item = 0; /* [RJM 07/97: Ranked Required Match] points to the next possible LT to delete */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Match] */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums)
+	MoreAccum = 0;
+
+      if ((rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums) || Anding)
+	{
+	  Anding = 1;
+	  if (!Sorted || has_grown)
+	    {
+	      LT_sort (LT);
+	      LT_pack (LT);
+	      Sorted = 1;
+	      has_grown = 0;
+	    }
+	  next = LT->IDE;
+	  n = 0;
+	}
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* make sure that the document list is set up correctly for */
+      /* requiring a term to match */
+      if (require_match == 1) {
+	if (!Sorted || has_grown) {
+	  LT_sort (LT);
+	  LT_pack (LT);
+	  Sorted = 1;
+	  has_grown = 0;
+	}
+	/* this_item is already set to zero */
+      }
+
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || n < LT->num))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+
+	  /* deal with skipping */
+	  if (k)
+	    {
+	      /* see if we should take a skip */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+
+	      /* should we take another skip? */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if this is a required match we must remove all accumulators */
+	  /* of intermediate documents */
+	  if (require_match == 1) {
+	    while (this_item < LT->num && LT->IDE[this_item].DocNum < CurrDocNum - 1) {
+	      if (LastDocNum==0 ||(LT->IDE[this_item].DocNum > LastDocNum - 1))
+		LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	      this_item++;
+	    }
+	  }
+
+	  if (Anding)
+	    {
+	      while (n < LT->num && next->DocNum < CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      if (n < LT->num && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (n < LT->num && next->DocNum <= CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      goto NextDocNum;
+	    }
+	  if (!Anding)
+	    {
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      LT = LT_add (qd, LT, CurrDocNum - 1, Wqt * Wdt);
+	      if (!LT)
+		{
+		  Abort = 1;
+		  goto FastAbort;
+		}
+
+	      /* the list of accumulators just got bigger */
+	      has_grown = 1;
+
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+
+
+	    }
+	  else if ((mem = LT_find (LT, CurrDocNum - 1)))
+	    {
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+
+	  LastDocNum = CurrDocNum;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, LT->num, total_added);
+	}
+      if (Abort)
+	break;
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* remove the remaining accumulators if this term is required to match */
+      if (require_match == 1) {
+	while (this_item < LT->num) {
+	  if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	    LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	  this_item++;
+	}
+	Anding = 1; /* no more documents should be added */
+
+	/* Get rid of any documents which have been deleted */
+	LT_pack (LT);
+      }
+    }
+
+  if (sk)
+    fclose (sk);
+  return LT;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/invf_get.h	(revision 21112)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * invf_get.h -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF_GET
+#define H_INVF_GET
+
+typedef struct Invf_Doc_Entry
+  {
+    int DocNum;
+    float Sum;
+  }
+Invf_Doc_Entry;
+
+typedef struct Invf_Doc_Entry_Pool
+  {
+    void *pool;
+    unsigned pool_chunks;
+  }
+Invf_Doc_Entry_Pool;
+
+typedef struct Invf_Doc_EntryH
+  {
+    Invf_Doc_Entry IDE;
+    struct Invf_Doc_EntryH *next;
+  }
+Invf_Doc_EntryH;
+
+typedef struct Hash_Table
+  {
+    Invf_Doc_EntryH *Suplimentary_Entries;
+    int Suplimentary_Size;
+    int Suplimentary_Num;
+    unsigned max;
+    unsigned num;
+    unsigned size;
+    Invf_Doc_EntryH **HashTable;
+    Invf_Doc_EntryH *Head;
+    Invf_Doc_EntryH IDEH[1];
+  }
+Hash_Table;
+
+typedef struct List_Table
+  {
+    unsigned max;
+    unsigned num;
+    Invf_Doc_Entry IDE[1];
+  }
+List_Table;
+
+typedef enum
+  {
+    op_term, op_and_terms, op_and_or_terms, op_diff_terms
+  }
+Op_types;
+
+/*Invf_Doc_Entry *HT_find(Hash_Table *HT, unsigned long DN); */
+void HT_free (query_data * qd, Hash_Table * HT);
+void LT_free (query_data * qd, List_Table * LT);
+
+invf_data *InitInvfFile (File * InvfFile, stemmed_dict * sd);
+void FreeInvfData (invf_data * id);
+
+DocList *GetDocs (query_data * qd, WordEntry * we, BooleanQueryInfo * bqi);
+
+float *CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi);
+
+void free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool);
+
+Splay_Tree *CosineDecodeSplay (query_data * qd, TermList * Terms,
+			 RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool);
+
+Hash_Table *CosineDecodeHash (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+List_Table *CosineDecodeList (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+
+
+DocList *GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass1.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass1.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass1.c	(revision 21112)
@@ -0,0 +1,866 @@
+/**************************************************************************
+ *
+ * ivf.pass1.c -- Memory efficient pass 1 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:50  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.6  1995/01/16  03:57:05  tes
+   * Fixed bug for index_string_bytes which was calculated
+   * by adding the suffix lengths except for every lookback (block)
+   * number of words. This was incorrect as invf.dict is fully
+   * front coded - perhaps it was partially coded in the past.
+   * This would only affect a statistic - hence the reason why the
+   * bug could exist ;-)
+   *
+   * Revision 1.5  1994/11/29  00:31:59  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:48  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:34  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long bit counts for inverted file
+ *     (1999-08-05 Tim Bell <bhat@cs.mu.oz.au>)
+ */
+static char *RCSID = "$Id$";
+
+#define LOGLOOKBACK	2
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define PARA_DIV 1.5
+#define NORM_DIV 1.0
+
+
+typedef struct hash_rec
+  {
+    unsigned long fcnt;		/* fragment frequency */
+    unsigned long lfcnt;	/* local fragment frequency */
+    unsigned long fnum;		/* last fragment to use stem */
+    unsigned long wcnt;		/* stem frequency */
+    unsigned long lwcnt;	/* local stem frequency */
+    u_char *word;
+  }
+hash_rec;
+
+static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0;
+static unsigned long outputbytes = 0;
+static unsigned long inputbytes = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static unsigned long ChunksWritten = 0;
+
+
+static FILE *ic;		/* The invf block file */
+static stdio_bitio_state sbs;
+
+static hash_rec **HashTable;
+static unsigned long HashSize=0;
+static unsigned long HashUsed=0;
+static u_char *Pool;
+static int PoolLeft=0;
+
+static hash_rec *hr_pool;
+static int hr_PoolLeft=0;
+
+static hash_rec **first_occr;
+static int max_first_occr=0;
+
+static mg_ullong L1_bits = 0;
+static mg_ullong L2_bits = 0;
+static mg_ullong L3_bits = 0;
+static unsigned long L1_ohead = 0;
+static unsigned long L2_ohead = 0;
+static unsigned long L3_ohead = 0;
+static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0;
+static unsigned long ptrcnt = 0;
+static unsigned long checknum=0;
+static long max_mem = 0;
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+void
+ResetStaticI1Vars () 
+{
+  words_read = 0;
+  words_diff = 0; 
+  bytes_diff = 0;
+  outputbytes = 0;
+  inputbytes = 0;
+  MaxMemInUse = 0;
+  MemInUse = 0;
+  ChunksWritten = 0;
+  
+  HashSize=0;
+  HashUsed=0;
+  PoolLeft=0;
+  
+  hr_PoolLeft=0;
+  
+  max_first_occr=0;
+  
+  L1_bits = 0;
+  L2_bits = 0;
+  L3_bits = 0;
+  L1_ohead = 0;
+  L2_ohead = 0;
+  L3_ohead = 0;
+  callnum = 0;
+  lcallnum = 0;
+  wordnum = 0;
+  lwordnum = 0;
+  ptrcnt = 0;
+  checknum=0;
+  max_mem = 0;
+  
+}
+int 
+init_ivf_1 (char *file_name)
+{
+  ResetStaticI1Vars();
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fwrite ("    ", sizeof (u_long), 1, ic);	/* Space for the maxmem */
+  ENCODE_START (ic)
+    ENCODE_PAUSE (sbs)
+
+    if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (hash_rec))))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+
+  HashSize = INITIAL_HASH_SIZE;
+  HashUsed = 0;
+  if (!(HashTable = Xmalloc (sizeof (hash_rec *) * HashSize)))
+    {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * HashSize);
+  bzero ((char *) HashTable, sizeof (hash_rec *) * HashSize);
+  max_first_occr = 8192;
+  if (!(first_occr = Xmalloc (sizeof (hash_rec *) * max_first_occr)))
+    {
+      Message ("Unable to allocate memory for first_occur");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * max_first_occr);
+
+  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+
+  return (COMPALLOK);
+}
+
+
+
+static unsigned long 
+mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? lwordnum : lcallnum; */
+  register unsigned long N = lcallnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->lwcnt : ent->lfcnt; */
+      register unsigned long p = ent->lfcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->lwcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+static unsigned long 
+max_mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? wordnum : callnum; */
+  register unsigned long N = callnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->wcnt : ent->fcnt; */
+      register unsigned long p = ent->fcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->wcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+static void 
+dump_dict (unsigned long mem)
+{
+  int i;
+
+  ChunksWritten++;
+
+  ENCODE_CONTINUE (sbs)
+
+    GAMMA_ENCODE (lcallnum + 1);
+  if (mem > max_mem)
+    max_mem = mem;
+  GAMMA_ENCODE (mem + 1);
+  lwordnum = lcallnum = 0;
+  GAMMA_ENCODE (HashUsed + 1);
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *ent = first_occr[i];
+      GAMMA_ENCODE (ent->lwcnt + 1);
+      if (ent->lwcnt >= 2)
+	GAMMA_ENCODE (ent->lfcnt);
+      ent->lwcnt = ent->lfcnt = 0;
+    }
+  ptrcnt = 0;
+  ENCODE_PAUSE (sbs)
+}
+
+
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  u_char *end = s_in + l_in - 1;
+
+  callnum++;
+  lcallnum++;
+  inputbytes += l_in;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      
+      stemmer (stem_method, stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      words_read++;
+      wordnum++;
+      lwordnum++;
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = HashTable[hashval];
+	    if (!ent)
+	      {
+		int len = *Word + 1;
+		if (!hr_PoolLeft)
+		  {
+		    if (!(hr_pool = Xmalloc (HASH_POOL_SIZE *
+					     sizeof (hash_rec))))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    hr_PoolLeft = HASH_POOL_SIZE;
+		    ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+		  }
+		ent = hr_pool++;
+		hr_PoolLeft--;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->wcnt = 1;
+		ent->fcnt = 1;
+		ent->lwcnt = 1;
+		ent->lfcnt = 1;
+		ent->fnum = callnum;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		if (HashUsed == max_first_occr - 1)
+		  {
+		    ChangeMem (-sizeof (hash_rec *) * max_first_occr);
+		    max_first_occr *= 2;
+		    if (!(first_occr = Xrealloc (first_occr, sizeof (hash_rec *) *
+						 max_first_occr)))
+		      {
+			Message ("Unable to allocate memory for first_occr");
+			return (COMPERROR);
+		      }
+		    ChangeMem (sizeof (hash_rec *) * max_first_occr);
+		  }
+		first_occr[HashUsed] = ent;
+		HashUsed++;
+		HashTable[hashval] = ent;
+		bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		ent->lwcnt++;
+		if (callnum > ent->fnum)
+		  {
+		    ptrcnt++;
+		    ent->fcnt++;
+		    ent->lfcnt++;
+		    ent->fnum = callnum;
+		  }
+		break;
+	      }
+	  }
+      }
+
+      if (HashUsed >= HashSize >> 1)
+	{
+	  hash_rec **ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec *) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  bzero ((char *) ht, sizeof (hash_rec *) * size);
+	  ChangeMem (sizeof (hash_rec *) * size);
+
+	  for (i = 0; i < HashSize; i++)
+	    if (HashTable[i])
+	      {
+		register u_char *wptr;
+		hash_rec *ent;
+		register unsigned long hashval, step;
+
+		wptr = HashTable[i]->word;
+		HASH (hashval, step, wptr, size);
+		ent = ht[hashval];
+		while (ent)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    ent = ht[hashval];
+		  }
+		ht[hashval] = HashTable[i];
+	      }
+	  Xfree (HashTable);
+	  ChangeMem (-sizeof (hash_rec *) * HashSize);
+	  HashTable = ht;
+	  HashSize = size;
+	}
+
+    }
+
+  if (ptrcnt >= checknum)
+    {
+      unsigned long mem;
+      /*fprintf(stderr, "Checking at %u . . . ", ptrcnt); */
+      mem = mem_reqd ();
+      if (mem >= invf_buffer_size * CHECK_CLOSE)
+	{
+	  /*fprintf(stderr, "Got a match\n"); */
+	  dump_dict (mem);
+	  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+	    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+	}
+      else
+	{
+	  checknum = checknum * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	    checknum;
+	  if (checknum <= ptrcnt)
+	    checknum = ptrcnt + 1;
+	  /*fprintf(stderr, "Next check at %u\n", checknum); */
+	}
+    }
+  return (COMPALLOK);
+
+}				/* encode */
+
+
+int 
+process_ivf_1 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	return process_doc (start, pos + s_in - start);
+    }
+  return COMPALLOK;
+}
+
+static int 
+PackHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < HashSize; s++)
+    if (HashTable[s])
+      HashTable[d++] = HashTable[s];
+  ChangeMem (-sizeof (hash_rec *) * HashSize);
+  ChangeMem (sizeof (hash_rec *) * HashUsed);
+  if (!(HashTable = Xrealloc (HashTable, sizeof (hash_rec *) * HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  HashSize = HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((hash_rec **) A))->word;
+  u_char *s2 = (*((hash_rec **) B))->word;
+
+  return (casecompare (s1, s2)); /* [RPAP - Jan 97: Stem Index Change] */
+}				/* stem_comp */
+
+
+
+
+
+/*
+ * void count_text()
+ *
+ * The maths used in this function is described in the paper "Coding for 
+ * Compression in Full-Text Retrieval Systems"
+ *
+ */
+static void 
+count_text ()
+{
+  int i;
+  mg_ullong oldL12_bits = 0;
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *wrd = HashTable[i];
+      /* estimate size of a level 1 inverted file */
+      L1_bits += (int) (0.99 + wrd->fcnt * (1.6 + log2 (1.0 * callnum / wrd->fcnt)));
+      L1_ohead += BIO_Gamma_Length (wrd->fcnt);
+
+      L2_bits += BIO_Unary_Length (wrd->wcnt);
+      L2_ohead += BIO_Gamma_Length (wrd->wcnt - wrd->fcnt + 1);
+
+      L3_bits += (int) (0.99 + wrd->wcnt *
+		  (1.6 + log2 (1.0 * words_read / (wrd->wcnt + callnum))));
+      L3_ohead += 0;
+
+      /* check for overflow */
+      if (L1_bits + L2_bits < oldL12_bits) {
+	fprintf(stderr, "ERROR: Inverted file size will probably overflow %d byte unsigned integer\n", sizeof (mg_ullong));
+	fprintf(stderr, "       counter in pass 2.\n");
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+      oldL12_bits = L1_bits + L2_bits;
+    }
+  L3_bits = (L3_bits + L2_bits + L1_bits + 7) / 8;
+  L3_ohead = (L3_ohead + L2_ohead + L1_ohead + 7) / 8;
+  L2_bits = (L2_bits + L1_bits + 7) / 8;
+  L2_ohead = (L2_ohead + L1_ohead + 7) / 8;
+  L1_bits = (L1_bits + 7) / 8;
+  L1_ohead = (L1_ohead + 7) / 8;
+}				/* count_text */
+
+
+
+/*
+ *    write_stem_file():
+ *              writes out the stemmed dictionary file
+ *              in the following format 
+ *                      lookback value (int)
+ *                      totalbytes value (int)
+ *                      indexstringbytes (int)
+ *                      for each word 
+ *                        wordlen (4 bits)
+ *                        prefix match (4 bits)
+ *                        word (wordlen bytes)
+ *                        word frequency (int)
+ *                        word count (int)
+ *
+ *      Accesses outside variables:     
+ *
+ *      Return value...:                
+ */
+
+static void 
+write_stem_file (char *file_name)
+{
+  long j;
+  struct invf_dict_header idh;
+  long lookback = (1 << LOGLOOKBACK);	/* ???? */
+  long totalbytes = 0;		/* The sum of the length of all words, including 
+				   the length byte */
+  long indexstringbytes = 0;	/* The amount of space required to store the 
+				   words in the diction, this takes into account 
+				   the prefixes */
+  u_char *lastword = NULL;	/* A pointer to the last word processed */
+  FILE *sp;
+
+
+  /* Calculate the size of certain things */
+  for (j = 0; j < HashSize; j++)
+    {
+      u_char *word = HashTable[j]->word;
+      indexstringbytes += word[0] + 2;
+      totalbytes += word[0] + 1;
+      if (lastword)
+	indexstringbytes -= prefixlen (lastword, word);
+      lastword = word;
+    }
+
+  lastword = NULL;
+
+  if (!(sp = create_file (file_name, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(lookback, idh.lookback);
+  HTONUL2(HashSize, idh.dict_size);
+  HTONUL2(totalbytes, idh.total_bytes);
+  HTONUL2(indexstringbytes, idh.index_string_bytes);
+  HTOND2(inputbytes, idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL2(callnum, idh.num_of_docs);
+  HTONUL2(callnum, idh.static_num_of_docs);
+  HTONUL2(words_read, idh.num_of_words);
+  HTONUL2(stemmer_num, idh.stemmer_num);
+  HTONUL2(stem_method, idh.stem_method);
+
+  fwrite ((char *) &idh, sizeof (idh), 1, sp);
+  outputbytes += sizeof (idh);
+
+  for (j = 0; j < HashSize; j++)
+    {
+      int i;
+      unsigned long wcnt, fcnt;  /* [RPAP - Jan 97: Endian Ordering] */
+      hash_rec *ent = HashTable[j];
+      if (lastword != NULL)
+	/* look for prefix match with prev string */
+	i = prefixlen (lastword, ent->word);
+      else
+	i = 0;
+      fputc (i, sp);
+      fputc (ent->word[0] - i, sp);
+      fwrite ((char *) ent->word + i + 1, sizeof (u_char), ent->word[0] - i, sp);
+      outputbytes += ent->word[0] - i + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->fcnt, fcnt);
+      fwrite ((char *) &(fcnt), sizeof (fcnt), 1, sp);
+
+      outputbytes += sizeof (ent->fcnt);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->wcnt, wcnt);
+      fwrite ((char *) &(wcnt), sizeof (wcnt), 1, sp);
+
+      outputbytes += sizeof (ent->wcnt);
+      lastword = ent->word;
+    }
+
+  fclose (sp);
+}				/* write_stem_file() */
+
+
+
+/*
+ *    write_codes():
+ *              calls functions to assign and write out codes and
+ *              then prints out stats about execution results.
+ *
+ *      Accesses outside variables:     z
+ *
+ *      Return value...:                z
+ */
+
+static void 
+write_codes (char *file_name)
+{
+  unsigned long dicts = 0;
+  outputbytes = 0;
+  write_stem_file (file_name);
+  dicts = outputbytes;
+
+#ifndef SILENT
+  Message ("Chunks written      : %d\n",
+	   ChunksWritten);
+  if (InvfLevel == 3)
+    Message ("Paragraphs          : %8d\n", callnum);
+  Message ("Peak memory usage   : %10.1f Mb\n",
+	   (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("Stems size          : %10.1f kB  %5.2f%%\n",
+	   (double) dicts / 1024, (double) 100 * dicts / inputbytes);
+  Message ("Lev 1 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L1_bits + L1_ohead) / 1024,
+	   (double) 100 * (L1_bits + L1_ohead) / inputbytes);
+  Message ("Lev 2 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L2_bits + L2_ohead) / 1024,
+	   (double) 100 * (L2_bits + L2_ohead) / inputbytes);
+  Message ("Lev 3 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L3_bits + L3_ohead) / 1024,
+	   (double) 100 * (L3_bits + L3_ohead) / inputbytes);
+  Message ("Record addresses    : %10.1f kB  %5.2f%%\n",
+	   (double) words_diff * 4 / 1024, (double) 100 * words_diff * 4 / inputbytes);
+#endif
+}				/* write_codes() */
+
+
+
+void 
+write_num_file (char *file_name)
+{
+
+  int i;
+  FILE *f;
+
+  for (i = 0; i < HashSize; i++)
+    HashTable[i]->fnum = i;
+
+  if (!(f = create_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+#if 1
+  ENCODE_START (f)
+
+    for (i = 0; i < HashSize; i++)
+    BINARY_ENCODE (first_occr[i]->fnum + 1, HashSize + 1);
+
+  ENCODE_DONE;
+#else
+  for (i = 0; i < HashSize; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      unsigned long fnum;
+      HTONUL2(first_occur[i]->fnum, fnum);
+      fwrite ((char *) &fnum, sizeof (unsigned long), 1, f);
+    }
+#endif
+
+  fclose (f);
+}
+
+int 
+done_ivf_1 (char *FileName)
+{
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+#ifndef SILENT
+  Message ("Mem reqd for 1 chunk: %8u bytes\n",
+	   max_mem_reqd ());
+#endif
+  if (lcallnum)
+    dump_dict (mem_reqd ());
+
+  if (PackHashTable () == COMPERROR)
+    return COMPERROR;
+
+  qsort (HashTable, HashUsed, sizeof (hash_rec *), ent_comp);
+
+  count_text ();
+  write_codes (FileName);
+
+  ENCODE_CONTINUE (sbs)
+    GAMMA_ENCODE (1);
+  ENCODE_DONE
+    fseek (ic, sizeof (long), 0);
+  
+  HTONSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&max_mem, sizeof (max_mem), 1, ic);
+  NTOHSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (ic);
+
+  write_num_file (FileName);
+
+  msg_prefix = temp_str;
+
+  /* free the memory */
+  Xfree(HashTable);
+  if (Pool) {
+    Pool -= (POOL_SIZE-PoolLeft);
+    Xfree(Pool);
+  }
+  if (hr_pool) {
+    hr_pool -= (HASH_POOL_SIZE-hr_PoolLeft);
+    Xfree(hr_pool);
+  }
+  if (first_occr) {
+    Xfree(first_occr);
+  }
+
+  return (COMPALLOK);
+}				/* done_encode */
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass2.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass2.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/ivf.pass2.c	(revision 21112)
@@ -0,0 +1,1191 @@
+/**************************************************************************
+ *
+ * ivf.pass2.c -- Memory efficient pass 2 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  2004/06/10 05:07:43  kjdon
+   have to declare vars before calling functions!!
+
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+ * Revision 1.2  1997/08/02  05:01:57  wew
+ * changed literal values of 32 for the bit size of magic numbers of
+ * files to sizeof (unsigned long) * 8, increased the gap at the start
+ * of the invf during processing to 200 bytes
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:51  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:49  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:35  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long disk pointers and bit counts for inverted file
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ *     Code provided by Owen de Kretser <oldk@cs.mu.oz.au>
+ */
+
+static char *RCSID = "$Id$";
+
+#include "local_strings.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "stemmer.h"
+#include "perf_hash.h"
+#include "bitio_m.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include "bitio_random.h"
+#include "bitio_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "mg.h"
+#include "build.h"
+#include "words.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+#ifdef USE_LONG_LONG
+#define BIO_Random_Seek_X BIO_Random_Seek_LL
+#define BIO_Random_Tell_X BIO_Random_Tell_LL
+#else
+#define BIO_Random_Seek_X BIO_Random_Seek
+#define BIO_Random_Tell_X BIO_Random_Tell
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include <io.h>
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+/*#define RND_BUF_SIZE 128 */
+#endif
+
+#define print_fsize(file)\
+do\
+  {\
+    struct stat file_state;\
+    fstat(fileno(invf_out), &file_state);\
+    Message("len(invf) = %ld", file_state.st_size);\
+  }while(0)
+
+typedef struct word_rec
+  {
+    unsigned long ptr;
+    unsigned long last;
+  }
+word_rec;
+
+typedef struct invf_state_rec
+  {
+    mg_ullong Disk_Ptr;
+    mg_ullong Disk_Last;
+    unsigned long Disk_B;
+  }
+invf_state_rec;
+
+typedef struct chunk
+  {
+    unsigned long start_doc;
+    unsigned long params_pos;
+    unsigned long disk_pos;
+    unsigned long N;
+  }
+chunk;
+
+
+static FILE *dict;		/* Stemmed dictionary file */
+static FILE *hash;		/* Stemmed dictionary hash file */
+static FILE *invf;		/* Inverted file */
+static FILE *invf_in;		/* Inverted file */
+static FILE *invf_out;		/* Inverted file */
+static FILE *invf_idx;		/* Inverted index file */
+static FILE *count;		/* Count file */
+static FILE *count_trans;	/* Count translation file */
+static FILE *invf_state;	/* Inverted file State */
+static FILE *chunk_state;	/* Chunk state */
+static FILE *chunks;		/* Chunk state */
+static FILE *invf_para = NULL;	/* Paragraph counts file */
+static FILE *weights = NULL;	/* Weights file */
+
+static stdio_bitio_state sbs;
+static random_bitio_state crbs;
+static chunk *chunk_data = NULL;
+static random_bitio_state rbs, rbsp;
+
+static int docs_left = 0, next_docs_left = 0;
+static unsigned long N;
+
+static word_rec *WordRecs;
+static u_char *lg_bs;
+static float *idf = NULL;
+
+static char *MemoryBuffer = NULL;
+static unsigned long MemBufSize;
+static unsigned long BufToUse;
+static struct invf_dict_header idh;
+
+static perf_hash_data *phd;
+
+static unsigned long *word_list = NULL;
+static unsigned long wl_size = 0;
+
+static unsigned long dict_size;
+static unsigned long no_of_ptrs = 0;
+static unsigned long chunks_read = 0;
+static unsigned long Disk_pos = 0;
+static unsigned long callnum = 0;
+static unsigned long wordnum = 0;
+
+static unsigned long totalIbytes = 0;
+static unsigned long totalDbytes = 0;
+static unsigned long totalHbytes = 0;
+
+static unsigned long MemInUse = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long max_buffer_len;
+
+void 
+ChangeMemInUse (int mem)
+{
+  MemInUse += mem;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+void 
+ResetStaticI2Vars() 
+{
+  docs_left = 0;
+  next_docs_left = 0;
+  N = 0;
+  MemBufSize=0;
+  BufToUse=0;
+  memset(&idh, 0, sizeof(idh));
+  wl_size = 0;
+  
+  dict_size = 0;
+  no_of_ptrs = 0;
+  chunks_read = 0;
+  Disk_pos = 0;
+  callnum = 0;
+  wordnum = 0;
+  
+  totalIbytes = 0;
+  totalDbytes = 0;
+  totalHbytes = 0;
+  
+  MemInUse = 0;
+  MaxMemInUse = 0;
+  max_buffer_len = 0;
+
+}
+
+
+static int 
+open_files (char *file_name)
+{
+  char FName[200];
+
+  if (!(dict = open_file (file_name, INVF_DICT_SUFFIX, "rb",
+			  MAGIC_STEM_BUILD, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(hash = open_file (file_name, INVF_DICT_HASH_SUFFIX, "rb",
+			  MAGIC_HASH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(count = open_file (file_name, INVF_CHUNK_SUFFIX, "rb",
+			   MAGIC_CHUNK, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fread (&max_buffer_len, sizeof (max_buffer_len), 1, count);
+  NTOHUL(max_buffer_len);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  BIO_Stdio_Decode_Start (count, &sbs);
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  if (!(count_trans = open_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "rb",
+				 MAGIC_CHUNK_TRANS, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(invf = create_file (file_name, INVF_SUFFIX, "w+b",
+			    MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fflush (invf);
+  if (!(invf_in = open_file (file_name, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  if (!(invf_out = create_file (file_name, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbs);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbsp);
+  ChangeMemInUse (RND_BUF_SIZE * 2);
+
+  if (!(invf_idx = create_file (file_name, INVF_IDX_SUFFIX, "wb",
+				MAGIC_INVI, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (InvfLevel == 3)
+    if (!(invf_para = create_file (file_name, INVF_PARAGRAPH_SUFFIX, "wb",
+				   MAGIC_PARAGRAPH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (COMPERROR);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".invf.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(invf_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunk.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunk_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+  BIO_Random_Start (chunk_state, RND_BUF_SIZE, &crbs);
+  ChangeMemInUse (RND_BUF_SIZE);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunks", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunks = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  return (COMPALLOK);
+}
+
+
+
+
+
+#define ISR_CACHE 1024
+#define ISR_ENTRY_SIZE (sizeof(mg_ullong)*2 + sizeof(unsigned long))
+
+invf_state_rec *
+in_cache (int pos)
+{
+  static char isr_data[ISR_CACHE * ISR_ENTRY_SIZE];
+  static invf_state_rec isr;
+  static int isr_base = 0, isr_num = -1, isr_pos = -1;
+  if (isr_pos >= 0)
+    bcopy ((char *) &isr, &isr_data[isr_pos * ISR_ENTRY_SIZE], ISR_ENTRY_SIZE);
+  if (pos < isr_base || pos >= isr_base + isr_num)
+    {
+      if (isr_num >= 0)
+	{
+	  fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+	  fwrite (isr_data, 1, ISR_ENTRY_SIZE * isr_num, invf_state);
+	}
+      isr_base = pos;
+      fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+      fread (isr_data, 1, ISR_ENTRY_SIZE * ISR_CACHE, invf_state);
+      isr_num = ISR_CACHE;
+    }
+  isr_pos = pos - isr_base;
+  bcopy (&isr_data[isr_pos * ISR_ENTRY_SIZE], (char *) &isr, ISR_ENTRY_SIZE);
+  return &isr;
+}
+
+
+
+
+
+unsigned long 
+occur_to_lexical (long occ, int clear_state)
+{
+  static long pos = -1;
+  static random_bitio_state rbs;
+  static int val = 0;
+  if (clear_state) {
+    pos = -1;
+    val = 0;
+    return 0;
+  }
+  if (pos == -1)
+    {
+      BIO_Random_Start (count_trans, RND_BUF_SIZE, &rbs);
+      pos = 0x7fffffff;
+    }
+  if (occ < pos)
+    {
+      if (occ == -1)
+	{
+	  BIO_Random_Done (&rbs);
+	  return 0;
+	}
+      BIO_Random_Seek_X (sizeof (unsigned long) * 8, &rbs);
+      pos = 0;
+    }
+  while (pos <= occ)
+    {
+      val = BIO_Random_Binary_Decode (dict_size + 1, &rbs, NULL) - 1;
+      pos++;
+    }
+  return (val);
+}
+
+
+void 
+add_chunk_state (unsigned long pos, unsigned long start_doc,
+		 unsigned long N)
+{
+  chunk_data[chunks_read].params_pos = pos;
+  chunk_data[chunks_read].start_doc = start_doc;
+  chunk_data[chunks_read].N = N;
+  chunks_read++;
+}
+
+
+int 
+init_ivf_2 (char *file_name)
+{
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+  mg_ullong totalIbits;
+  mg_ullong lasttotalIbits;
+  double logN = 0.0;
+
+  ResetStaticI2Vars(); /* clear the global statics */
+  occur_to_lexical(0, 1); /* clear the statics in here*/
+
+  if (open_files (file_name) == COMPERROR)
+    return COMPERROR;
+
+
+  /* Read in the stemmed dictionary file header */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  dict_size = idh.dict_size;
+
+  N = idh.num_of_docs;
+
+  if (!(phd = read_perf_hash_data (hash)))
+    {
+      Message ("Unable to read in hash data");
+      return COMPERROR;
+    }
+  totalHbytes = sizeof (perf_hash_data) + sizeof (u_char) * 256 +
+    sizeof (int) * (phd->MAX_N + 1) + sizeof (int *) * 3 * phd->MAX_CH +
+    sizeof (long) * phd->MAX_CH * phd->MAX_L;
+
+  if (!(WordRecs = Xmalloc (sizeof (word_rec) * idh.dict_size)))
+    {
+      Message ("No memory for word entries");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (word_rec) * idh.dict_size;
+  /* separate storage for the log(b) values, one byte each */
+  if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size)))
+    {
+      Message ("No memory for lg b's");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (u_char) * idh.dict_size;
+
+  if (MakeWeights)
+    {
+      /* separate storage for the idf values, one single each */
+      if (!(idf = Xmalloc (sizeof (float) * idh.dict_size)))
+	{
+	  Message ("No memory for idf's");
+	  return COMPERROR;
+	}
+      totalDbytes += sizeof (float) * idh.dict_size;
+
+      if (!(weights = create_file (file_name, WEIGHTS_SUFFIX, "wb",
+				 MAGIC_WGHT, MG_CONTINUE))) { /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Couldn't open weights file for writing");
+	return (COMPERROR);
+      }
+    }
+  else
+    {
+      unlink (make_name (file_name, WEIGHTS_SUFFIX, NULL));
+    }
+
+  chunk_data = Xmalloc (sizeof (chunk) * (ChunkLimit + 2));
+  totalDbytes += sizeof (chunk) * (ChunkLimit + 2);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  if (MakeWeights)
+    {
+      wl_size = 1024;
+      if (!(word_list = Xmalloc (sizeof (*word_list) * wl_size)))
+	{
+	  Message ("No memory for word_list");
+	  return COMPERROR;
+	}
+
+      logN = log ((double) N);
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      invf_state_rec *isr;
+      register unsigned long copy, suff, p;
+      unsigned long fcnt, wcnt;
+
+      lasttotalIbits = totalIbits;
+      
+      copy = fgetc (dict);
+      suff = fgetc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      WordRecs[i].last = 0;
+      WordRecs[i].ptr = 0;
+
+      p = fcnt;
+
+      if (MakeWeights)
+	idf[i] = logN - log ((double) fcnt);
+
+
+      isr = in_cache (i);
+
+      isr->Disk_Last = 0;
+      isr->Disk_Ptr = totalIbits;
+
+      isr->Disk_B = BIO_Bblock_Init (N, p);
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+      if (totalIbits < lasttotalIbits) {
+	fprintf(stderr, "ERROR: The totalIbits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+    }
+
+
+  /* now convert to bytes, and actually get the space */
+#ifdef USE_LONG_LONG
+  totalIbytes = (totalIbits + 7ull) >> 3ull;
+#else
+  totalIbytes = (totalIbits + 7ul) >> 3ul;
+#endif
+  return (COMPALLOK);
+
+}
+
+
+
+
+
+static void 
+LoadCounts (void)
+{
+  unsigned long numwords, i, last_total;
+  static unsigned long local_N = 0;
+  unsigned long totalIbits, crbs_pos;
+  word_rec *wr;
+  unsigned long *counts;
+
+  if (MemoryBuffer == NULL)
+    {
+      MemBufSize = sizeof (unsigned long) * dict_size;
+      if (max_buffer_len > MemBufSize) 
+	MemBufSize = max_buffer_len;
+      if (!(MemoryBuffer = Xmalloc (MemBufSize)))
+	FatalError (1, "Unable to allocate memory for buffer");
+      ChangeMemInUse (MemBufSize);
+    }
+  counts = (unsigned long *) MemoryBuffer;
+/*   bzero ((char *) counts, sizeof (unsigned long) * dict_size); */
+  bzero ((char *) counts, MemBufSize);
+  docs_left = next_docs_left; 
+  if (!docs_left)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  BufToUse = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  numwords = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  local_N = docs_left;
+
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    wr->ptr = 0; 
+
+  bzero ((char *) lg_bs, dict_size);
+
+  for (i = 0; i < numwords; i++)
+    {
+      unsigned long word_num, wcnt, fcnt, p;
+      word_num = occur_to_lexical (i,0);
+      
+      wr = &WordRecs[word_num];
+      
+      wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+      if (wcnt >= 2)
+	fcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL);
+      else
+	fcnt = wcnt;
+
+
+      p = fcnt;
+      if (wcnt)
+	{
+	  register unsigned long length;
+	  counts[word_num] = p;
+	  length = BIO_Bblock_Bound (local_N, p);
+	  if (InvfLevel >= 2)
+	    length += wcnt;
+	  wr->ptr = length;
+	  lg_bs[word_num] = floorlog_2 (BIO_Bblock_Init_W (local_N, p));
+	}
+
+
+  }  
+  crbs_pos = BIO_Random_Tell (&crbs);
+  totalIbits = 0;
+  last_total = 0;
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    {
+      register unsigned long length;
+      length = wr->ptr;
+      wr->last = callnum;
+      BIO_Random_Gamma_Encode (counts[i] + 1, &crbs, NULL);
+      if (counts[i])
+	{
+	  if (i)
+	    BIO_Random_Delta_Encode (totalIbits - last_total + 1, &crbs, NULL);
+	  else
+	    BIO_Random_Delta_Encode (1, &crbs, NULL);
+
+	  last_total = totalIbits;
+	}
+      wr->ptr = totalIbits;
+      totalIbits += length;
+    }
+  add_chunk_state (crbs_pos, callnum, local_N);
+
+  if ((totalIbits + 7ul) >> 3ul > BufToUse)
+    FatalError (1, "Pointers exceed buffer size");
+
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+}
+
+
+
+
+static void 
+DumpChunk (void)
+{
+  chunk_data[chunks_read - 1].disk_pos = Disk_pos << 3;
+  fseek (chunks, Disk_pos, 0);
+  fwrite (MemoryBuffer, sizeof (char), BufToUse, chunks);
+  Disk_pos += BufToUse;
+}
+
+
+
+
+static void 
+DiskMerge (void)
+{
+  random_bitio_state *rbsi;
+  random_bitio_state *chks = NULL;
+  unsigned long *chunk_ptrs;
+  int i;
+
+  BIO_Random_Flush (&crbs);
+
+  chunk_ptrs = Xmalloc (chunks_read * sizeof (unsigned long));
+  ChangeMemInUse (chunks_read * sizeof (unsigned long));
+  bzero ((char *) chunk_ptrs, chunks_read * sizeof (unsigned long));
+
+  rbsi = Xmalloc (chunks_read * sizeof (random_bitio_state));
+  ChangeMemInUse (chunks_read * sizeof (random_bitio_state));
+  for (i = 0; i < chunks_read; i++)
+    {
+      rbsi[i] = crbs;
+      rbsi[i].Buf = Xmalloc (rbsi[i].len);
+      ChangeMemInUse (rbsi[i].len);
+      bcopy ((char *) (crbs.Buf), (char *) (rbsi[i].Buf), rbsi[i].len);
+      BIO_Random_Seek (chunk_data[i].params_pos, &rbsi[i]);
+    }
+
+  if (chunks_read > 1)
+    {
+      int j;
+      chks = Xmalloc ((chunks_read - 1) * sizeof (random_bitio_state));
+      ChangeMemInUse ((chunks_read - 1) * sizeof (random_bitio_state));
+      BIO_Random_Start (chunks, RND_BUF_SIZE, &chks[0]);
+      ChangeMemInUse (RND_BUF_SIZE);
+      for (j = 1; j < chunks_read - 1; j++)
+	{
+	  chks[j] = chks[0];
+	  chks[j].Buf = Xmalloc (chks[0].len);
+	  ChangeMemInUse (chks[0].len);
+	  bcopy ((char *) (chks[0].Buf), (char *) (chks[j].Buf), chks[0].len);
+	}
+    }
+  for (i = 0; i < dict_size; i++)
+    {
+      int j;
+      invf_state_rec *isr = in_cache (i);
+      register int B;
+
+      BIO_Random_Seek_X (isr->Disk_Ptr, &rbs);	/* Position in invf file */
+
+      B = isr->Disk_B;
+
+      for (j = 0; j < chunks_read; j++)
+	{
+	  int p;
+	  p = BIO_Random_Gamma_Decode (&rbsi[j], NULL) - 1;
+
+	  if (p)
+	    {
+	      int ptr, b;
+	      chunk_ptrs[j] += BIO_Random_Delta_Decode (&rbsi[j], NULL) - 1;
+	      ptr = chunk_ptrs[j];
+	      b = 1 << floorlog_2 (BIO_Bblock_Init_W (chunk_data[j].N, p));
+
+	      if (j == chunks_read - 1)
+		{
+		  int k, CurrDoc;
+		  DECODE_START ((u_char *) MemoryBuffer, ptr)
+		    CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      BBLOCK_DECODE (x, b);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  UNARY_DECODE (tf);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  DECODE_DONE
+		    isr->Disk_Last = CurrDoc;
+		}
+	      else
+		{
+		  int k, CurrDoc;
+		  random_bitio_state *Chks = chks + j;
+		  BIO_Random_Seek (chunk_data[j].disk_pos + ptr, Chks);
+		  CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      x = BIO_Random_Bblock_Decode (b, Chks, NULL);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  tf = BIO_Random_Unary_Decode (Chks, NULL);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  isr->Disk_Last = CurrDoc;
+		}
+	    }
+	}
+
+      isr->Disk_Ptr = BIO_Random_Tell_X (&rbs);
+
+    }
+  if (chunks_read > 1)
+    {
+      int j;
+      for (j = 0; j < chunks_read - 1; j++)
+	{
+	  Xfree (chks[j].Buf);
+	  ChangeMemInUse (-chks[j].len);
+	}
+      Xfree (chks);
+      ChangeMemInUse (-(chunks_read - 1) * sizeof (random_bitio_state));
+    }
+
+  for (i = 0; i < chunks_read; i++)
+    {
+      Xfree (rbsi[i].Buf);
+      ChangeMemInUse (-rbsi[i].len);
+    }
+  Xfree (rbsi);
+  ChangeMemInUse (-chunks_read * sizeof (random_bitio_state));
+/*   chunks_read = 0; */
+  Xfree (chunk_ptrs);
+  ChangeMemInUse (-chunks_read * sizeof (unsigned long));
+  chunks_read = 0;
+  Disk_pos = 0;
+  BIO_Random_Seek (0, &crbs);
+}
+
+static void 
+MergeIn (void)
+{
+  static int disk_chunks = 0;
+  static header = 0;
+  if (!header)
+    {
+      fprintf (stderr, "ivf.pass2 : ");
+      header = 1;
+    }
+  if (disk_chunks == ChunkLimit || next_docs_left == 0)
+    {
+      fprintf (stderr, "M");
+      DiskMerge ();
+      disk_chunks = 0;
+    }
+  else
+    {
+      fprintf (stderr, "-");
+      DumpChunk ();
+      disk_chunks++;
+    }
+  if (next_docs_left == 0)
+    fprintf (stderr, "\n");
+}
+
+
+static int 
+wl_comp (const void *a, const void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  int res;
+  u_char *end = s_in + l_in - 1;
+  unsigned long tocode;
+  unsigned long wl_pos = 0;
+
+  if (!docs_left)
+    LoadCounts ();
+
+  callnum++;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      stemmer (idh.stem_method, idh.stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      if (*Word == 0)
+	continue;
+
+      res = perf_hash (phd, Word);
+
+      {
+	word_rec *arr = &WordRecs[res];
+	int b = 1 << lg_bs[res];
+	wordnum++;
+
+	tocode = callnum;
+
+	ENCODE_START ((u_char *) MemoryBuffer, arr->ptr)
+
+	  if (tocode > arr->last)
+	  {
+	    register int x;
+	    x = tocode - arr->last - 1;
+	    BBLOCK_ENCODE (x + 1, b);
+	    if (InvfLevel >= 2)
+	      ENCODE_BIT (1);
+	    no_of_ptrs++;
+	    arr->last = tocode;
+	  }
+	else if (InvfLevel >= 2)
+	  {
+	    __pos--;
+	    ENCODE_BIT (0);
+	    ENCODE_BIT (1);
+	  }
+	arr->ptr = __pos;
+	ENCODE_DONE
+      }
+
+      if (MakeWeights)
+	{
+	  if (wl_pos >= wl_size)
+	    {
+	      wl_size += (wl_size >> 1);
+	      word_list = Xrealloc (word_list, sizeof (*word_list) * wl_size);
+	    }
+	  word_list[wl_pos++] = res;
+	}
+    }
+  if (MakeWeights)
+    {
+      float doc_weight = 0.0;
+      if (wl_pos)
+	{
+	  unsigned long *wl = word_list;
+	  unsigned long i, count, val;
+	  qsort (wl, wl_pos, sizeof (*wl), wl_comp);
+	  count = 1;
+	  val = *wl++;
+	  for (i = 1; i <= wl_pos; i++, wl++)
+	    if (i == wl_pos || val != *wl)
+	      {
+		double weight = count * idf[val];
+		doc_weight += weight * weight;
+		count = 1;
+		val = *wl;
+	      }
+	    else
+	      count++;
+	}
+      HTONF(doc_weight);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &doc_weight, sizeof (doc_weight), 1, weights);
+    }
+  docs_left--;
+  if (!docs_left)
+    MergeIn ();
+
+  return COMPALLOK;
+}
+
+int 
+process_ivf_2 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int count = 0;
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	      count++;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	{
+	  if (process_doc (start, pos + s_in - start) != COMPALLOK)
+	    return (COMPERROR);
+	  count++;
+	}
+      HTONSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &count, sizeof (count), 1, invf_para);
+    }
+  return COMPALLOK;
+}
+
+
+
+
+
+static void 
+stats (unsigned long len)
+{
+#ifndef SILENT
+  fseek (count, 0, 2);
+  fseek (count_trans, 0, 2);
+  fseek (invf_state, 0, 2);
+  fseek (invf, 0, 0);
+  fseek (invf, 0, 2);
+  fseek (chunks, 0, 2);
+  fseek (chunk_state, 0, 2);
+  Message ("File sizes\n");
+  Message ("  Chunk desc    : %10u bytes\n", ftell (count));
+  Message ("  Chunk trans   : %10u bytes\n", ftell (count_trans));
+  Message ("  Chunks        : %10u bytes\n", ftell (chunks));
+  Message ("  Chunk state   : %10u bytes\n", ftell (chunk_state));
+  Message ("  Invf state    : %10u bytes\n", ftell (invf_state));
+  Message ("  Peak invf     : %10u bytes\n", len);
+  Message ("  Final invf    : %10u bytes\n", ftell (invf));
+  Message ("Peak disk usage : %10.2f %%\n",
+	   (double) (ftell (count) + ftell (count_trans) +
+		     ftell (invf_state) + ftell (chunks) +
+		     ftell (chunk_state) + len) / ftell (invf) * 100.0);
+#endif
+}
+
+
+/* ARGSUSED */
+int 
+done_ivf_2 (char *FileName)
+{
+  long i;
+  mg_ullong totalIbits;
+  unsigned long invf_len;
+  unsigned long bytes_output;
+  struct invf_file_header ifh;
+
+  if (weights)
+    fclose (weights);
+  if (invf_para)
+    fclose (invf_para);
+
+  free_perf_hash (phd);
+  phd = NULL;
+
+  Xfree (MemoryBuffer);
+  MemoryBuffer = NULL;
+  ChangeMemInUse (-MemBufSize);
+
+  BIO_Random_Done (&rbs);
+  BIO_Random_Done (&rbsp);
+  fflush (invf);
+
+  fseek (invf, 0, 2);
+  invf_len = ftell (invf);
+
+  fseek (invf_out, sizeof (long), 0);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(dict_size, ifh.no_of_words);
+  HTONUL2(no_of_ptrs, ifh.no_of_ptrs);
+  ifh.skip_mode = 0;
+  bzero ((char *) ifh.params, sizeof (ifh.params));
+  HTONUL2(InvfLevel, ifh.InvfLevel);
+  fwrite ((char *) &ifh, sizeof (ifh), 1, invf_out);
+
+  bytes_output = ftell (invf_out);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  /* find the right place in the file to start reading p values */
+  fseek (dict, sizeof (unsigned long) + sizeof (struct invf_dict_header), 0);
+  for (i = 0; i < dict_size; i++)
+    {
+      invf_state_rec *isr;
+      unsigned long fcnt, wcnt, s, e;
+      register unsigned long p;
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+
+      /* output location to the invf_idx  */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      isr = in_cache (i);
+
+      e = (isr->Disk_Ptr + 7ul) >> 3ul;
+      s = totalIbits >> 3;
+
+      fseek (invf_in, s, 0);
+      while (s < e)
+	{
+	  u_char c = getc (invf_in);
+	  if (s == e - 1)
+	    {
+	      u_char ands[8] =
+	      {0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	      c &= ands[isr->Disk_Ptr & 7ul];
+	    }
+	  putc (c, invf_out);
+	  bytes_output++;
+	  s++;
+	}
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+    }
+
+  fclose (invf_in);
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (!(_chsize (_fileno (invf_out), bytes_output)))
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (invf_out), bytes_output);
+#endif
+
+  fclose (invf_out);
+
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (invf_idx);
+
+#ifndef SILENT
+  {
+    char *temp_str = msg_prefix;
+    unsigned long total;
+    msg_prefix = "ivf.pass2";
+    stats (invf_len);
+    Message ("Pass two data structures      : %6.3f Mbyte\n",
+	     (double) totalDbytes / 1024 / 1024);
+    total = totalDbytes;
+    Message ("Pass two hash structure(s)    : %6.3f Mbyte\n",
+	     (double) totalHbytes / 1024 / 1024);
+    total += totalHbytes;
+    Message ("Peak extra memory in use      : %6.3f Mbyte\n",
+	     (double) MaxMemInUse / 1024 / 1024);
+    total += MaxMemInUse;
+    Message ("Peak total memory in use      : %6.3f Mbyte\n",
+	     (double) total / 1024 / 1024);
+    msg_prefix = temp_str;
+  }
+#endif
+
+  Xfree(chunk_data);
+  chunk_data = NULL;
+  Xfree (WordRecs);
+  WordRecs = NULL;
+  Xfree (lg_bs);
+  lg_bs = NULL;
+  Xfree (idf);
+  idf = NULL;
+  Xfree (word_list);
+  word_list = NULL;
+  /* Free the memory allocated for the BIO_Random */
+  occur_to_lexical (-1,1);
+
+  BIO_Random_Done (&crbs);
+
+  fclose (invf);
+  fclose (dict);
+  fclose (hash);
+  fclose (count);
+  fclose (count_trans);
+  fclose (chunk_state);
+  fclose (chunks);
+  fclose (invf_state);
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.c	(revision 21112)
@@ -0,0 +1,217 @@
+/**************************************************************************
+ *
+ * lists.c -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+
+#include "lists.h"
+#include "local_strings.h"
+#include "locallib.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:01  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1999/07/01 09:30:48  rjmcnab
+   Changes for better match reporting.
+
+   Revision 1.1  1998/11/17 09:34:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:50  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:37  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/*
+ * Make a DocList large enough to hold num entries 
+ */
+DocList *
+MakeDocList (int num)
+{
+  DocList *d;
+  if (!(d = Xmalloc (sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  d->num = 0;
+  d->total_retrieved = 0;
+  d->is_approx = 0;
+  return (d);
+}
+
+
+/*
+ * Change the size of a DocList so that it can hold num entries 
+ */
+DocList *
+ResizeDocList (DocList * d, int num)
+{
+  if (!(d = Xrealloc (d, sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  return (d);
+}
+
+
+/* IntersectLists () */
+/* Returns the intersection of List1 and List2 by deleting from List1 all */
+/* entries not in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the intersection of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+IntersectLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List2 || !List1)
+    {
+      if (List1)
+	Xfree (List1);
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num && s2 - List2->DE < List2->num)
+    {
+      if (s1->DocNum == s2->DocNum)
+	*d1++ = *s1++;
+      if (s1 - List1->DE < List1->num)
+	while (s2 - List2->DE < List2->num && s2->DocNum < s1->DocNum)
+	  s2++;
+      if (s2 - List2->DE < List2->num)
+	while (s1 - List1->DE < List1->num && s1->DocNum < s2->DocNum)
+	  s1++;
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+
+/* DiffLists () */
+/* Returns the difference of List1 and List2 by deleting from List1 all */
+/* entries in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the differrence of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DiffLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num)
+    {
+      if (s2 - List2->DE == List2->num)
+	*d1++ = *s1++;
+      else
+	{
+	  if (s1->DocNum == s2->DocNum)
+	    {
+	      s1++;
+	      s2++;
+	    }
+	  else if (s1->DocNum < s2->DocNum)
+	    *d1++ = *s1++;
+	  else
+	    s2++;
+	}
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+/* MergeLists () */
+/* Finds the union of two lists and returns the head of the new list formed. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+MergeLists (DocList * List1, DocList * List2)
+{
+  DocList *NewList;
+  DocEntry *d, *s1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	return (List2);
+      else
+	return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  if (!(NewList = MakeDocList (List1->num + List2->num)))
+    {
+      Xfree (List1);
+      Xfree (List2);
+      return (NULL);
+    }
+
+  s1 = List1->DE;
+  s2 = List2->DE;
+  d = NewList->DE;
+  while (s1 - List1->DE < List1->num || s2 - List2->DE < List2->num)
+    {
+      if (s1 - List1->DE >= List1->num)
+	*d++ = *s2++;
+      else if (s2 - List2->DE >= List2->num)
+	*d++ = *s1++;
+      else if (s1->DocNum == s2->DocNum)
+	{
+	  *d = *s1++;
+	  (d++)->Weight += (s2++)->Weight;
+	}
+      else if (s1->DocNum < s2->DocNum)
+	*d++ = *s1++;
+      else
+	*d++ = *s2++;
+    }
+  NewList->num = d - NewList->DE;
+  Xfree (List1);
+  Xfree (List2);
+  return (NewList);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/lists.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * lists.h -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_LISTS
+#define H_LISTS
+
+
+
+typedef struct DocEntry
+  {
+    float Weight;
+    int DocNum;
+    unsigned long SeekPos;	/* position in the text file in bytes */
+    unsigned long Len;		/* length of the document in bytes */
+    char *CompTextBuffer;
+    struct DocEntry *Next;
+    short or_included;		/*[TS:Mar/95] whether included in an AND of ORed-terms */
+  }
+DocEntry;
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved; /* valid if > 0 */
+    int is_approx;       /* whether total_retrieved is approximate */
+    DocEntry DE[1];
+  }
+DocList;
+
+DocList *MakeDocList (int num);
+DocList *ResizeDocList (DocList * d, int num);
+DocList *IntersectLists (DocList * List1, DocList * List2);
+DocList *DiffLists (DocList * List1, DocList * List2);
+DocList *MergeLists (DocList * List1, DocList * List2);
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.c	(revision 21112)
@@ -0,0 +1,307 @@
+/**************************************************************************
+ *
+ * locallib.c -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:02  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:51  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:38  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; i++)
+    total += A[i];
+  for (i = 0; i < n; i++)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; i--)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      n--;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  N++;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	limit++;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int 
+Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem,
+	  u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+	    u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (Fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
+
+
+int 
+F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (F_Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (Fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (Fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/locallib.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+int F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+		u_long * disk);
+
+int F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+
+
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.special.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.special.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg.special.c	(revision 21112)
@@ -0,0 +1,118 @@
+/**************************************************************************
+ *
+ * mg.special.c -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg.h"
+#include "build.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:03  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:53  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:44  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+FILE *fp;
+
+
+int 
+init_special (char *FileName)
+{
+  char FName[200];
+  sprintf (FName, "%s%s", FileName, ".docnums");
+  if (!(fp = fopen (FName, "w")))
+    return (COMPERROR);
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_special (u_char * s_in, int l_in)
+{
+  static int un = 1;
+  char *s_done = (char *) (s_in + l_in);
+  for (; (char *) s_in != s_done; s_in++)
+    if (*s_in == '<')
+      {
+	if (strncmp ((char *) s_in, "<DOCNO>", 7) == 0)
+	  {
+	    char *s_end = (char *) (s_in + 1);
+	    for (; s_end != s_done; s_end++)
+	      if (*s_end == '<')
+		if (strncmp (s_end, "</DOCNO>", 8) == 0)
+		  break;
+	    if (s_end == s_done)
+	      {
+		fprintf (fp, "NO DOC ID %d\n", un++);
+		return COMPALLOK;
+	      }
+	    s_in += 7;
+	    while (s_end != (char *) s_in &&
+		   (*(s_end - 1) == ' ' || *(s_end - 1) == '\n'))
+	      s_end--;
+	    while ((char *) s_in != s_end &&
+		   (*s_in == ' ' || *s_in == '\n'))
+	      s_in++;
+	    for (; (char *) s_in != s_end; s_in++)
+	      putc (*s_in, fp);
+	    putc ('\n', fp);
+	    return (COMPALLOK);
+	  }
+      }
+  fprintf (fp, "NO DOC ID %d\n", un++);
+  return (COMPALLOK);
+}
+
+
+
+
+
+
+
+
+int 
+done_special (char *FileName)
+{
+  fclose (fp);
+  return COMPALLOK;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_compression_dict.c	(revision 21112)
@@ -0,0 +1,1087 @@
+/**************************************************************************
+ *
+ * mg_compression_dict.c -- Routines for reading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  1999/12/10 15:04:25  cs025
+   Incorrect use of NTOHUL on a double value (num_bytes).
+
+   Revision 1.1  1999/08/10 21:18:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:52  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:54  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:45  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* #define DEBUG1 */
+
+/* #define DEBUG */
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct WordData
+  {
+    u_long freq, occur_num;
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+WordData;
+
+static WordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    WordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HBDYM")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'B':
+	novel_method = MG_NOVEL_BINARY;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'M':
+	novel_method = MG_NOVEL_HYBRID_MTF;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; i++)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      WordData *wd;
+      int j;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = Xmalloc (sizeof (WordData *) * Num[i] + 1);
+
+      buf = Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = Xmalloc (sizeof (WordData) * Num[i]);
+      for (j = 0; j < Num[i]; j++, wd++)
+	{
+	  int len;
+	  fread (&wd->freq, sizeof (wd->freq), 1, f);
+	  NTOHUL(wd->freq);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fread (&wd->occur_num, sizeof (wd->occur_num), 1, f);
+	  NTOHUL(wd->occur_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  int i;
+  WordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (WordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; i++, wd++)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; i++)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; i++)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->freq > bb->freq)
+    return -1;
+  else if (aa->freq < bb->freq)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->occur_num > bb->occur_num)
+    return 1;
+  else if (aa->occur_num < bb->occur_num)
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  WordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = Xmalloc (num * sizeof (WordData *));
+  for (i = 0; i < Num[0]; i++)
+    wd[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      WordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->freq, word->occur_num, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; i++)
+    {
+      WordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (WordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  bzero ((char *) char_freqs, sizeof (char_freqs));
+  bzero ((char *) len_freqs, sizeof (len_freqs));
+  for (i = 0; i < num; i++, wd++)
+    {
+      u_long freq = (*wd)->freq;
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; len--, buf++)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+
+
+
+
+void 
+CalcBitCost (WordData ** discard_word, int discard_num,
+	     WordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  int j;
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  for (j = 0; j < discard_num; j++, discard_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*discard_word)->word;
+      int len = *buf++;
+      u_long freq = (*discard_word)->freq;
+      int idx = KIND (*discard_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*discard_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*discard_word)->saving = ((*discard_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*discard_word)->word[0]);
+
+      (*discard_word)->num_trans = num_trans;
+    }
+  for (j = 0; j < keep_num; j++, keep_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*keep_word)->word;
+      int len = *buf++;
+      u_long freq = (*keep_word)->freq;
+      int idx = KIND (*keep_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*keep_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*keep_word)->saving = ((*keep_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*keep_word)->word[0]);
+
+      (*keep_word)->num_trans = num_trans;
+    }
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  WordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+  keep_num = 0;
+  keep_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+
+  for (i = 0; i < Num[0]; i++)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      WordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->freq;
+      num_trans++;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (keep_heap, sizeof (word), keep_num, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "DISCARD \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (discard_heap, sizeof (word),
+				discard_num, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      discard_heap[discard_num] = word;
+	      heap_additem (discard_heap, sizeof (word), &discard_num,
+			    BigSaving);
+	      freqs_trans[idx] -= word->freq;
+	      mem -= sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      keep_to_discard++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (discard_heap, sizeof (word),
+			       &discard_num, BigSaving);
+	      keep_heap[keep_num] = word;
+	      heap_additem (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      freqs_trans[idx] += word->freq;
+	      mem += sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      discard_to_keep++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "DISCARD -> KEEP %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  recalcs++;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; i++)
+    {
+      WordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; i++)
+    {
+      WordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  int i, us;
+  for (us = i = 0; i < dd->num_wds; i++)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = Xmalloc ((dd->num_wds) * sizeof (*freqs))))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      freqs[i] = dd->wd[i]->freq;
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  Xfree (freqs), freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  bzero ((char *) &huff_words_size, sizeof (huff_words_size));
+  bzero ((char *) &lencounts, sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      lencounts[codelen]++;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  Xfree (hd->clens);
+  Xfree (hd);
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  int j;
+  long freqs[256];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; len--, buf++)
+	freqs[(u_long) (*buf)] += (*wd)->freq;
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; j++)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  int j;
+  long freqs[16];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    freqs[(*wd)->word[0]] += (*wd)->freq;
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; j++)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  bzero((char *) &cdh, sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; i++)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < discard[i].num_wds; j++)
+		esc.freq += discard[i].wd[j]->freq;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < all[i].num_wds; j++)
+		if (all[i].wd[j]->freq == 1)
+		  esc.freq++;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_decompress_text.c	(revision 21112)
@@ -0,0 +1,167 @@
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "netorder.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "backend.h"  /* compressed_dict struct */
+#include "text_get.h"
+#include "words.h"
+
+#define EOD         '\002'
+#define EOP         '\003'
+
+/* These are defined in mgquery, and they need to be here for backend.o */
+/* Messy I know but this was meant to be a quick hack :-) */
+#if defined(PARADOCNUM) || defined(NZDL)
+int *Paragraph = NULL;
+int Documents = 0;
+#endif
+
+int
+main (int argc, char **argv)
+{
+  FILE *text, *text_idx, *para;
+  File *text_comp_dict = NULL, *text_aux_dict = NULL, *text_fast_comp_dict = NULL;
+
+  int ch;
+  unsigned long start, finish, len;
+  u_long total_chars = 0;
+  u_char *c_buffer = 0;
+  int buf_len = -1;
+  u_char *uc_buffer = 0;
+  int ULen;
+  compression_dict *cd = NULL;
+  compressed_text_header cth;
+  u_long doc;
+  int count;
+  char *filename = "";
+  char path[512];
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-d directory] -f name\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  /* Open files */
+  text = open_file (filename, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text);
+
+  text_idx = open_file (filename, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text_idx);
+
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  para = open_file (filename, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH, MG_CONTINUE);
+
+  sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_FAST_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(text_fast_comp_dict = Fopen (path, "rb", MAGIC_FAST_DICT)))
+    {
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_comp_dict = Fopen (path, "rb", MAGIC_DICT);
+      if (!text_comp_dict)
+	FatalError (1, "Could not open compressed dictionary");
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_AUX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_aux_dict = Fopen (path, "rb", MAGIC_AUX_DICT);
+    }
+
+  cd = LoadCompDict (text_comp_dict, text_aux_dict, text_fast_comp_dict);
+
+
+  /* Uncompress text */
+
+  doc = 1;
+  fread ((char *)&start, sizeof (start), 1, text_idx);
+  if (para)
+    fread ((char *)&count, sizeof (count), 1, para);
+  else
+    count = 1;
+
+  for (;;)
+    {
+      fread ((char *) &finish, sizeof (finish), 1, text_idx);
+      len = finish - start;
+      
+      if ((int) len > buf_len)
+	{
+	  if (c_buffer)
+	    {
+	      Xfree (c_buffer);
+	      Xfree (uc_buffer);
+	    }
+	  if (!(c_buffer = Xmalloc (len)))
+	    FatalError (1, "Cannot allocate memory for compressed buffer");
+	  if (!(uc_buffer = Xmalloc ((int) (cth.ratio * 1.01 * len) + 100)))
+	    FatalError (1, "Cannot allocate memory for uncompressed buffer");
+	  buf_len = len;
+	}
+
+      fread (c_buffer, 1, len, text);
+      DecodeText (cd, c_buffer, len, uc_buffer, &ULen);
+      fwrite (uc_buffer, ULen, sizeof (u_char), stdout);
+      fflush (stdout);
+
+      total_chars += ULen;
+
+      if (++doc > cth.num_of_docs)
+	{
+	  if (para)
+	    fputc (EOP, stdout);
+	  fputc (EOD, stdout);
+	  fflush (stdout);
+	  break;
+	}
+
+      if (!--count)
+	{
+	  /* End of document */
+	  fputc (EOD, stdout);
+	  if (para)
+	    fread ((char *)&count, sizeof (count), 1, para);
+	  else
+	    count = 1;
+	}
+      else
+	fputc (EOP, stdout);
+      fflush (stdout);
+      start = finish;
+    }
+
+  /* Close files */
+
+  fclose (text);
+  fclose (text_idx);
+  if (para)
+    fclose (para);
+
+#if 0
+  printf ("\n#Total chars output = %lu\n", total_chars);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.c	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * mg_errors.c -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
@@ -0,0 +1,801 @@
+/**************************************************************************
+ *
+ * mg_fast_comp_dict.c -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:06  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:57  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:55  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:47  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "huffman.h"
+#include "timing.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "backend.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; i++)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  bzero (buffer, mem);
+  fixup = Xmalloc (fixup_mem);
+  bzero (fixup, fixup_mem);
+
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+
+  load_comp_dict (filename);
+
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; which++)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+	cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+      for (which = 0; which <= 1; which++)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; j++)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; i++)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = buffer; (u_long) p < (u_long) cur; p++)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * cdh, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  u_long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; val--)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem += sizeof (comp_frags_header);
+	  Read_cfh (dict, &cfh, NULL, NULL);
+
+	  /* Don't need to count the space for the clens of the huffman data */
+
+	  mem += mem_for_words (dict, &cdh, &cfh);
+	  if (cfh.hd.clens)
+	    Xfree (cfh.hd.clens);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY ||
+      cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID ||
+      cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((long) cur > (long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID ||
+	  cdh->novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; i++)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+	  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+	  FIXUP (&cd->cfh[which]);
+	  FIXUP (&cd->values[which]);
+	  cd->escape[which] = NULL;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->chars_huff[which] = hd;
+	  FIXUP (&cd->chars_huff[which]);
+	  cd->chars_vals[which] = vals;
+	  FIXUP (&cd->chars_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->lens_huff[which] = hd;
+	  FIXUP (&cd->lens_huff[which]);
+	  cd->lens_vals[which] = vals;
+	  FIXUP (&cd->lens_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->chars_huff[which] = hd;
+	      FIXUP (&cd->chars_huff[which]);
+	      cd->chars_vals[which] = vals;
+	      FIXUP (&cd->chars_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->lens_huff[which] = hd;
+	      FIXUP (&cd->lens_huff[which]);
+	      cd->lens_vals[which] = vals;
+	      FIXUP (&cd->lens_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = buffer; (u_long) p < (u_long) cur; p++)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.c	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * mg_files.c -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      Xfree (basepath);
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = Xmalloc (strlen (bp) + 2);
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_files.h	(revision 21112)
@@ -0,0 +1,329 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_hilite_words.c	(revision 21112)
@@ -0,0 +1,693 @@
+/**************************************************************************
+ *
+ * mg_hilite_words -- display text and highlight particular words which 
+ *                    it contains
+ * Copyright (C) 1994 Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "getopt.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "stemmer.h"
+#include "words.h"
+
+
+/*
+ * Description
+ * -----------
+ *   Hilite_words reads text from stdin and outputs it into a pager such as
+ *   less. It's command arguments include a list of stemmed words 
+ *   which if extracted from the text will be highlighted on the output.
+ *
+ * Implementation
+ * --------------
+ *   Extracting of words - using PARSE_STEM_WORD & stemmer
+ *   
+ *   Highlighting of words - for standard pager:
+ *                             using back-space character code
+ *                             e.g. bolded a = "a\ba", underlined a = "_\ba"
+ *                         - for pager==html:
+ *                             use some of the standard html character 
+ *                             formatting tags
+ *
+ *   Storage of words - using hashtable (set) of size equalling a constant
+ *                      times the number of words
+ * 
+ * Usage
+ * -----
+ *   mg_hilite_words --stem_method [0-3] 
+ *                   --style [bold|underline|italic|emphasis|strong]
+ *                   --pager [less|more|html|???]
+ *                   --terminator [terminator-string]
+ *                   list-of-words-to-highlight
+ */
+
+/*
+ * Modifications:
+ *
+ * 21/Apr/95: To handle outputting html tags to stdout
+ *
+ */
+
+/* --- constants --- */
+
+/* highlighting styles */
+#define HILITE_MAX      5	/* the number of styles */
+#define BOLD 		0
+#define UNDERLINE	1
+#define ITALIC     	2
+#define EMPHASIS     	3
+#define STRONG     	4
+
+/* maximum length of line buffer */
+#define MAX_LINE_BUFFER 200
+
+/* set pager to this to get html hilited text to stdout */
+#define HTML_OUT "html"
+
+/* output types */
+#define PAGER 0
+#define HTML  1
+
+/* --- types --- */
+
+typedef u_char *Word;
+
+/* --- globals --- */
+
+/* keep in synch with constants */
+static char *hilite_names[] =
+{"bold", "underline", "italic", "emphasis", "strong"};
+static char *hilite_tags[] =
+{"B", "", "I", "EM", "STRONG"};
+static short hilite_style = BOLD;
+static char *pager = "less";
+static int stemmer_num = 0;	/* Lovin's stemmer */
+static int stem_method = 3;	/* fold & stem */
+static char **word_list;
+static int num_of_words = 0;
+static int output_type = PAGER;
+static char *terminator = NULL;
+
+/* --- prototypes --- */
+
+static Word copy_c_word (char *w);
+static void process_words (char **words, int num_of_words);
+static int get_line (u_char * line, int n, FILE * stream);
+static void process_text (FILE * input_file, FILE * output_file);
+static void copy_word (u_char * w1, u_char * w2);
+static void process_buffer (u_char * s_in, int len, FILE * output_file);
+static void output_word (u_char * s_start, u_char * s_finish, FILE * output_file);
+static void output_hilite_word (u_char * s_start, u_char * s_finish,
+				FILE * output_file);
+static void process_args (int argc, char *argv[]);
+static void print_line (u_char * line, int n, FILE * stream);
+
+
+/******************************** word set ************************************/
+
+#include "hash.h"
+#include "locallib.h"
+
+#define SIZE_FACTOR 2
+
+typedef Word *WordSet;
+static WordSet set_of_words = NULL;
+static int hash_size = 0;
+
+/* prototypes - set routines */
+static void set_create (int num_of_words);
+static void set_add (Word word);
+static int set_member (Word word);
+static void set_print (void);
+
+/* =========================================================================
+ * Function: set_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_print (void)
+{
+  int i = 0;
+
+  for (i = 0; i < hash_size; i++)
+    {
+      Word word = set_of_words[i];
+
+      if (word)
+	{
+	  int len = *word++;
+
+	  fprintf (stderr, "[%d] = ", i);
+	  while (len--)
+	    fputc (*word++, stderr);
+	  fputc ('\n', stderr);
+	}
+    }
+
+}
+
+
+/* =========================================================================
+ * Function: set_create
+ * Description: 
+ *      Allocate memory for the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_create (int num_of_words)
+{
+  WordSet set;
+
+  hash_size = prime (num_of_words * SIZE_FACTOR);
+  set = (WordSet) malloc (hash_size * sizeof (Word));
+  if (!set)
+    FatalError (1, "Runout of memory for word hashtable");
+  bzero ((char *) set, hash_size * sizeof (Word));  /* [RPAP - Feb 97: WIN32 Port] */
+
+  set_of_words = set;
+
+}
+
+/* =========================================================================
+ * Function: set_add
+ * Description: 
+ *      Add a string element to the set.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_add (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	{
+	  set_of_words[hash_val] = word;
+	  break;
+	}
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	break;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+
+/* =========================================================================
+ * Function: set_member
+ * Description: 
+ *      Tests whether a string is a member of the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+set_member (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	return 0;
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	return 1;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+/******************************** end of word set ******************************/
+
+
+
+/* =========================================================================
+ * Function: copy_c_word
+ * Description: 
+ *      Allocate enough memory and copy word over
+ * Input: 
+ *      w = null terminated string (c_word)
+ * Output: 
+ *      word with length in 1st byte
+ * ========================================================================= */
+
+static Word
+copy_c_word (char *w)
+{
+  int len = strlen (w);
+  Word w_copy = (Word) malloc (len + 1);
+  Word w_ptr = NULL;
+  int j = 0;
+
+  if (!w_copy)
+    FatalError (1, "Not enough memory to copy a word");
+
+  w_copy[0] = len;
+  w_ptr = w_copy + 1;
+  for (j = 0; j < len; j++)
+    *w_ptr++ = *w++;
+
+  return w_copy;
+}
+
+/* =========================================================================
+ * Function: process_words
+ * Description: 
+ *      Go through the stemmed words and add to word set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_words (char **words, int num_of_words)
+{
+  int i = 0;
+
+  set_create (num_of_words);
+
+  for (i = 0; i < num_of_words; i++)
+    {
+      Word word = copy_c_word (words[i]);
+      set_add (word);
+    }
+
+}
+
+/* =========================================================================
+ * Function: get_line
+ * Description: 
+ *      Equivalent of fgets for u_char*.
+ *      But returns length of read-in line.
+ *      Expects to see a '\n' before an EOF
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+get_line (u_char * line, int n, FILE * stream)
+{
+  int i = 0;
+  int ch = '\0';
+
+  while (1)
+    {
+      if (i == n)
+	return i;
+
+      ch = fgetc (stream);
+
+      if (ch == EOF)
+	{
+	  if (!feof (stream))
+	    FatalError (1, "Error on reading a line from stdin");
+	  return EOF;
+	}
+
+      if (ch == '\n')
+	return i;
+
+      *line++ = ch;
+      i++;
+    }
+
+}
+
+/* =========================================================================
+ * Function: print_line
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+print_line (u_char * line, int n, FILE * stream)
+{
+
+  while (n--)
+    {
+      fputc (*line++, stream);
+    }
+  fputc ('\n', stream);
+
+}
+
+/* =========================================================================
+ * Function: process_text
+ * Description: 
+ *      Go through the text from input_file and highlight to output_file 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_text (FILE * input_file, FILE * output_file)
+{
+  static u_char line_buffer[MAX_LINE_BUFFER];
+
+
+  while (1)
+    {
+      int len = get_line (line_buffer, MAX_LINE_BUFFER, input_file);
+
+      if (len == EOF)
+	break;
+      process_buffer (line_buffer, len, output_file);
+
+    }
+}
+
+/* =========================================================================
+ * Function: copy_word
+ * Description: 
+ *      Copies w2 into w1. Assumes both have storage allocated.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+copy_word (u_char * w1, u_char * w2)
+{
+  int i;
+  int len = w2[0];
+
+  for (i = 0; i <= len; i++)
+    *w1++ = *w2++;
+
+}
+
+/* =========================================================================
+ * Function: process_buffer
+ * Description: 
+ *      Parse & stem words of line buffer
+ *      Based on the usage of PARSEing in other mg files.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_buffer (u_char * s_in, int len, FILE * output_file)
+{
+  u_char *end = s_in + len - 1;
+  u_char *s_start = NULL;
+
+  if (!inaword (s_in, end))
+    {
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+    }
+
+  while (s_in <= end)
+    {
+      u_char word[MAXSTEMLEN + 1];
+
+      s_start = s_in;
+      PARSE_STEM_WORD (word, s_in, end);
+
+      stemmer (stem_method, stemmer_num, word);
+
+      if (set_member (word))	/* output with highlighting */
+	{
+	  output_hilite_word (s_start, s_start + word[0] - 1, output_file);
+	  s_start += word[0];	/* step over hilited output */
+	}
+      output_word (s_start, s_in - 1, output_file);
+
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+
+    }				/*while */
+
+  fputc ('\n', output_file);
+  fflush (output_file);
+
+}				/*process_buffer */
+
+/* =========================================================================
+ * Function: output_word
+ * Description: 
+ *      Output a word which lies from s_start to s_finish in buffer
+ * Input: 
+ *      s_start = ptr to 1st char
+ *      s_finish = ptr to last char
+ * Output: 
+ * ========================================================================= */
+
+static void
+output_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+  while (s_start <= s_finish)
+    {
+      fputc (*s_start++, output_file);
+    }
+}
+
+
+/* =========================================================================
+ * Function: output_hilite_word
+ * Description: 
+ *      Highlight a word (with length in 1st byte)
+ *      Pager highlighting:
+ *        Highlighting is either by bolding or underlining using
+ *        the method used by UNIX utilities More(1) and Less(1)
+ *      HTML highlighting:
+ *        use the appropriate start and end tags around the word
+ * ========================================================================= */
+
+static void
+output_hilite_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+
+  if (output_type == HTML)
+    {
+      char *hilite_tag = hilite_tags[hilite_style];
+
+      /* print start tag */
+      fprintf (output_file, "<%s>", hilite_tag);
+
+      output_word (s_start, s_finish, output_file);
+
+      /* print end tag */
+      fprintf (output_file, "</%s>", hilite_tag);
+    }
+
+  else
+    /* PAGER */
+    {
+      /* use backspaces around each letter */
+      while (s_start <= s_finish)
+	{
+	  switch (hilite_style)
+	    {
+	    case BOLD:
+	      fputc (*s_start, output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    case UNDERLINE:
+	      fputc ('_', output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    default:
+	      fputc (*s_start, output_file);
+	    }
+	  s_start++;
+	}			/*while */
+    }
+}
+
+/* =========================================================================
+ * Function: process_args
+ * Description: 
+ *      sets the global variables:
+ *              hilite_style, pager, num_of_words, word_list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+struct option long_opts[] =
+{
+  {"style", required_argument, 0, 's'},
+  {"terminator", required_argument, 0, 't'},
+  {"pager", required_argument, 0, 'p'},
+  {"stem_method", required_argument, 0, 'm'},
+  {"stemmer", required_argument, 0, 'a'},
+  {0, 0, 0, 0}
+};
+
+static void
+process_args (int argc, char *argv[])
+{
+  int ch;
+
+
+  opterr = 0;
+  while ((ch = getopt_long (argc, argv, "s:p:t:m:a:", long_opts, (int *) 0)) != -1)
+    {
+      switch (ch)
+	{
+	case 's':
+	  {
+	    int i;
+	    for (i = 0; i < HILITE_MAX; i++)
+	      if (strcmp (optarg, hilite_names[i]) == 0)
+		break;
+
+	    if (i < HILITE_MAX)
+	      hilite_style = i;
+	  }
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 't':
+	  terminator = optarg;
+	  break;
+	case 'm':
+	  stem_method = atoi (optarg);
+	  break;
+	case 'p':
+	  pager = optarg;
+	  break;
+	default:
+	  FatalError (1, "Usage: \n"
+		      "mg_hilite_words --stem_method [0-3]\n"
+	  "                --stemmer [english|lovin|french|simplefrench]\n"
+	  "                --style [bold|underline|italic|emphasis|strong]\n"
+		      "                --pager [less|more|html|???]\n");
+	}
+    }
+
+  num_of_words = argc - optind;
+
+  word_list = &argv[optind];
+
+  /* fix up output type */
+  if (strcmp (pager, HTML_OUT) == 0)
+    output_type = HTML;
+  else
+    output_type = PAGER;
+
+}
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+main (int argc, char *argv[])
+{
+  FILE *output = NULL;
+
+  process_args (argc, argv);
+
+  /* set output file */
+  if (output_type == PAGER)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    output = _popen (pager, "w");
+#else
+    output = popen (pager, "w");
+#endif
+  else
+    output = stdout;
+
+  if (!output)
+    FatalError (1, "Unable to run \"%s\"\n", pager);
+
+  if (num_of_words < 1)
+    {
+      int ch;
+
+      /* just echo the input */
+      /* better not to call this program at all ;-) */
+      while ((ch = fgetc (stdin)) != EOF)
+	{
+	  fputc (ch, output);
+	}
+    }
+  else
+    {
+      /* set up hash table for words */
+      process_words (word_list, num_of_words);
+
+
+      /* Go thru lines of text from stdin and 
+       * output words with hilite info if
+       * words parse into existence in the hash table 
+       */
+      process_text (stdin, output);
+    }
+
+  if (terminator)
+    fprintf (output, "%s\n", terminator);
+
+  if (output != stdout)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (output);
+#else
+    pclose (output);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dict.c	(revision 21112)
@@ -0,0 +1,347 @@
+/**************************************************************************
+ *
+ * mg_invf_dict.c -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:08  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:45  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:00  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:49  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static void process_files (char *filename);
+
+int main (int argc, char **argv)
+{
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:b:Fh")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b num] [-F] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  process_files (file_name);
+  return 0;
+}
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *id, *idb, *tmp, *ii;
+  unsigned long i, pos, num, First_word, invf_ptr, invf_len;
+  unsigned long last_ptr = 0;
+  char *FName;
+  struct invf_dict_header idh;
+  struct stem_dict_header sdh;
+  u_char prev[MAXSTEMLEN + 1];
+  u_char *buffer;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+
+  id = open_file (filename, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ii = open_file (filename, INVF_IDX_SUFFIX, "rb", MAGIC_INVI, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  idb = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "w+b", MAGIC_STEM,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  fread (&idh, sizeof (idh), 1, id);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  sdh.lookback = idh.lookback;
+  sdh.block_size = block_size;
+  sdh.num_blocks = 0;
+  sdh.index_chars = 0;
+  sdh.blocks_start = 0;
+  sdh.num_of_docs = idh.num_of_docs;
+  sdh.static_num_of_docs = idh.static_num_of_docs;
+  sdh.num_of_words = idh.num_of_words;
+  sdh.stemmer_num = idh.stemmer_num;
+  sdh.stem_method = idh.stem_method;
+  sdh.indexed = 0;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long extra, copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (id);
+      suff = getc (id);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, id);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, id);
+      fread (&wcnt, sizeof (wcnt), 1, id);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      /* read in the inverted file position */
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (word_num % idh.lookback == 0)
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + suff + 1 + sizeof (fcnt) + sizeof (wcnt) +
+	  sizeof (First_word) + sizeof (invf_ptr) + sizeof (invf_len) > block_size)
+	{
+	  int chunk;
+	  invf_len = invf_ptr - last_ptr;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUL(First_word);
+	  HTONUL(invf_len);
+	  HTONUS(word_num);
+
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sdh.num_blocks++;
+	}
+
+      if (word_num % idh.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (prev + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUL(fcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &fcnt, (char *) (buffer + buf_in_use), sizeof (fcnt));
+      buf_in_use += sizeof (fcnt);
+      HTONUL(wcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &wcnt, (char *) (buffer + buf_in_use), sizeof (wcnt));
+      buf_in_use += sizeof (wcnt);
+      last_ptr = invf_ptr;
+      HTONUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &invf_ptr, (char *) (buffer + buf_in_use), sizeof (invf_ptr));
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (invf_ptr);
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 34876234\n");
+      if (word_num == 0)
+	{
+	  fwrite (prev, sizeof (u_char), *prev + 1, idb);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idb);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sdh.index_chars += *prev + 1;
+	  First_word = i;
+	}
+      word_num++;
+    }
+  if (buf_in_use)
+    {
+      int chunk;
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      invf_len = invf_ptr - last_ptr;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUL(invf_len);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sdh.num_blocks++;
+    }
+  fclose (id);
+  fclose (ii);
+
+  rewind (tmp);
+  sdh.blocks_start = sdh.index_chars + sizeof (u_long) + sizeof (sdh) +
+    sdh.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sdh.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idb);
+	  sdh.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idb);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idb, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sdh.lookback);
+  HTONUL(sdh.block_size);
+  HTONUL(sdh.num_blocks);
+  HTONUL(sdh.blocks_start);
+  HTONUL(sdh.index_chars);
+  HTONUL(sdh.num_of_docs);
+  HTONUL(sdh.static_num_of_docs);
+  HTONUL(sdh.num_of_words);
+  HTONUL(sdh.stemmer_num);
+  HTONUL(sdh.stem_method);
+  HTONUL(sdh.indexed);
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+
+
+  Message ("Block size = %d\n", block_size);
+  Message ("Number of blocks written = %d\n", NTOHUL(sdh.num_blocks));
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_dump.c	(revision 21112)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * mg_invf_dump.c -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+
+extern unsigned long S_btg;
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/06/12 01:41:39  kjm18
+   changed the help message
+
+   Revision 1.1  1999/08/10 21:18:09  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/11/29  00:32:01  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.2  1994/09/20  04:41:50  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+static void process_files (char *filename);
+
+int binary = 0;
+int word_counts = 0;
+int term_dump = 0;
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hbwtf:d:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+        set_basepath(optarg);
+	break;
+      case 'b':
+	binary = 1;
+	break;
+      case 'w':
+	word_counts = 1;
+	break;
+      case 't':
+	term_dump = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-b] [-w] [-t] [-f input_file]"
+		 "[-d data directory]\n"
+		 "(b - binary mode, w - wordcount, t - term dump)\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+static void 
+process_files (char *name)
+{
+  unsigned long N, Nstatic, k;
+  FILE *dict;
+  FILE *invf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (!(invf = open_file (name, INVF_SUFFIX ".ORG", "rb", MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    invf = open_file (name, INVF_SUFFIX, "rb", MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  if (ifh.skip_mode != 0)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  DECODE_START (invf)
+    N = idh.num_of_docs;
+  Nstatic = idh.static_num_of_docs;
+  if (binary)
+    {
+      fwrite ((char *) &N, sizeof (N), 1, stdout);
+      fwrite ((char *) &ifh.no_of_words, sizeof (ifh.no_of_words), 1, stdout);
+    }
+  else
+    printf ("%ld %ld\n", N, ifh.no_of_words);
+  for (k = 0; k < ifh.no_of_words; k++)
+    {
+      int i, blk, doc;
+      register unsigned long suff, prefix;
+      unsigned long fcnt, wcnt;
+      char term[MAXSTEMLEN + 1];
+
+      prefix = fgetc (dict);
+      suff = fgetc (dict);
+      fread (&term[prefix], sizeof (char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+      term[prefix + suff] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (binary)
+	fwrite ((char *) &fcnt, sizeof (fcnt), 1, stdout);
+      else
+	{
+	  if (term_dump)
+	    printf ("%ld    \"%s\"\n", fcnt, term);
+	  else
+	    printf ("%ld\n", fcnt);
+	}
+      blk = BIO_Bblock_Init (Nstatic, fcnt);
+      for (doc = i = 0; i < fcnt; i++)
+	{
+	  int num;
+	  BBLOCK_DECODE (num, blk);
+	  doc += num;
+	  if (binary)
+	    fwrite ((char *) &doc, sizeof (doc), 1, stdout);
+	  else
+	    printf (" %d", doc);
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      int count;
+	      GAMMA_DECODE (count);
+	      if (word_counts)
+		if (binary)
+		  fwrite ((char *) &count, sizeof (count), 1, stdout);
+		else
+		  printf (" %d", count);
+	    }
+	  if (!binary)
+	    putchar ('\n');
+	}
+      while (__btg)
+	DECODE_BIT;
+    }
+  DECODE_DONE
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_merge.c	(revision 21112)
@@ -0,0 +1,852 @@
+/**************************************************************************
+ *
+ * mg_invf_merge.c -- description
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Last edited:  31 March, 1995 
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "bitio_m.h"
+#include "bitio_gen.h"
+#include "bitio_stdio.h"
+
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+
+typedef char FileName[256];
+
+
+/*
+ *  The merge_info struct contains all the data for an inverted file and
+ *  lexicon. We declare 3 merge_info variables, m[OLD] m[NEW] and m[MERGE].
+ */
+typedef struct merge_info_type
+  {
+    FileName fname;
+    FILE *invf;
+    FILE *dict;
+    FILE *idx;
+    FILE *para;  /* [RPAP - Feb 97: Level 3 Merge] */
+    struct invf_dict_header idh;
+    struct invf_file_header ifh;
+    u_long nDocs;
+
+    /* Lexicon processing variables */
+    u_long fcnt, wcnt;
+    int suff, pref;
+    unsigned char term[MAXSTEMLEN + 1];
+    u_long term_count;
+    int done;
+
+    /* Variables used for merging */
+    struct stdio_bitio_state sbs;	/* for bit-based I/O on inverted files */
+    u_long nTerms;		/* = number of invf entries */
+    int *fcntlist;		/* store fcnt values read in from the lexicons */
+
+  }
+merge_info;
+
+
+/***************************************************
+ **** GLOBALS ****
+ ***************************************************/
+
+merge_info m[3];		/*  m[OLD], m[NEW] and m[MERGE] */
+u_long magicsize;		/* how big is magic number? currently always 4 */
+u_long Nstatic;			/* static # of documents in file */
+
+float *DocWeightBuffer;
+int weightOption = 0;		/* true if weights file is to be appended */
+
+/*** Globals for processing the lexicons -- see readTerm() and writeTerm() 
+ ***/
+ProgTime start;			/* for printing elapsed time */
+unsigned char prevTerm[MAXSTEMLEN + 1];
+
+/*** Globals for merging -- see processEntry() 
+ ***/
+char *mergedata;		/* stores value OLD/NEW/MERGE for each term */
+int fastMerge = 1;		/* True if fast merge selected. Default = true */
+
+u_long oldIdxValue;		/* stores last read value from m[OLD].idx
+				   Used to calculate entry lengths */
+u_long bytes_output;		/* for writing .invf.idx file */
+u_long oldOnlyBits = 0;		/* entries in old only */
+u_long oldMergeBits = 0;	/* entries in old that are merged with 
+				   a new entry */
+
+/**************************************************************************/
+
+
+/*=======================================================================
+ * init_merge_invf(): open files, initialise globals, etc
+ *=======================================================================*/
+int 
+init_merge_invf ()
+{
+  prevTerm[0] = prevTerm[1] = 0;
+
+    /***
+     *  open .dict files
+     ***/
+  m[OLD].dict = open_file (m[OLD].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (m[OLD].dict);
+  fread (&(m[OLD].idh), sizeof (m[OLD].idh), 1, m[OLD].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].idh.lookback);
+  NTOHUL(m[OLD].idh.dict_size);
+  NTOHUL(m[OLD].idh.total_bytes);
+  NTOHUL(m[OLD].idh.index_string_bytes);
+  NTOHD(m[OLD].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[OLD].idh.num_of_docs);
+  NTOHUL(m[OLD].idh.static_num_of_docs);
+  NTOHUL(m[OLD].idh.num_of_words);
+  NTOHUL(m[OLD].idh.stemmer_num);
+  NTOHUL(m[OLD].idh.stem_method);
+
+  m[NEW].dict = open_file (m[NEW].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[NEW].idh), sizeof (m[NEW].idh), 1, m[NEW].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].idh.lookback);
+  NTOHUL(m[NEW].idh.dict_size);
+  NTOHUL(m[NEW].idh.total_bytes);
+  NTOHUL(m[NEW].idh.index_string_bytes);
+  NTOHD(m[NEW].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[NEW].idh.num_of_docs);
+  NTOHUL(m[NEW].idh.static_num_of_docs);
+  NTOHUL(m[NEW].idh.num_of_words);
+  NTOHUL(m[NEW].idh.stemmer_num);
+  NTOHUL(m[NEW].idh.stem_method);
+
+  m[MERGE].dict = create_file (m[MERGE].fname, INVF_DICT_SUFFIX, "wb",
+			       MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header */
+  fwrite ((char *) &(m[OLD].idh), sizeof (m[OLD].idh), 1, m[MERGE].dict);
+
+  m[OLD].nDocs = m[OLD].idh.num_of_docs;
+  m[NEW].nDocs = m[NEW].idh.num_of_docs;
+  Nstatic = m[OLD].idh.static_num_of_docs;
+  m[OLD].nTerms = m[OLD].idh.dict_size;
+  m[NEW].nTerms = m[NEW].idh.dict_size;
+  m[MERGE].nDocs = m[OLD].nDocs + m[NEW].nDocs;
+  m[MERGE].nTerms = 0;
+
+  /* Set up weight buffer for weights of new documents */
+  if (!(DocWeightBuffer = Xmalloc (m[NEW].nDocs * sizeof (*DocWeightBuffer))))
+    {
+      Message ("Insufficient memory\n");
+      return COMPERROR;
+    }
+  bzero ((char *) DocWeightBuffer, m[NEW].nDocs * sizeof (*DocWeightBuffer));
+
+    /***
+     *  open .invf files
+     *  Try ".ORG" extension first for m[OLD].invf since an inverted file
+     *  with skips may have been created.
+     ****/
+  if (!(m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX ".ORG", "rb",
+				 MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[OLD].ifh), sizeof (m[OLD].ifh), 1, m[OLD].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].ifh.no_of_words);
+  NTOHUL(m[OLD].ifh.no_of_ptrs);
+  NTOHUL(m[OLD].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[OLD].ifh.params[i]);
+  }
+  NTOHUL(m[OLD].ifh.InvfLevel);
+
+  m[NEW].invf = open_file (m[NEW].fname, INVF_SUFFIX, "rb",
+			   MAGIC_INVF, MG_ABORT);
+  fread (&(m[NEW].ifh), sizeof (m[NEW].ifh), 1, m[NEW].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].ifh.no_of_words);
+  NTOHUL(m[NEW].ifh.no_of_ptrs);
+  NTOHUL(m[NEW].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[NEW].ifh.params[i]);
+  }
+  NTOHUL(m[NEW].ifh.InvfLevel);
+
+  if (m[OLD].ifh.skip_mode != 0)
+    FatalError (1, "The old invf file contains skips. Unable to merge.");
+
+  if (m[OLD].ifh.InvfLevel != m[NEW].ifh.InvfLevel)
+    FatalError (1, "The two invf files have different inversion levels!");
+
+  /********** [RPAP - Feb 97: Level 3 Merge]
+
+  if ((m[OLD].ifh.InvfLevel > 2) || (m[NEW].ifh.InvfLevel > 2))
+    FatalError (1, "mgmerge cannot handle level 3 inverted files!");
+
+    *************/
+
+  m[MERGE].invf = create_file (m[MERGE].fname, INVF_SUFFIX, "wb",
+			       MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header in inverted file */
+  m[MERGE].ifh.no_of_words = m[MERGE].ifh.no_of_ptrs = 0;
+  m[MERGE].ifh.skip_mode = 0;
+  m[MERGE].ifh.InvfLevel = 2;
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = 0;
+  }
+  fwrite ((char *) &(m[MERGE].ifh), sizeof (m[MERGE].ifh), 1, m[MERGE].invf);
+
+    /***
+     *  open .invf.idx files
+     ***/
+  m[OLD].idx = open_file (m[OLD].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+  fread (&oldIdxValue, sizeof (u_long), 1, m[OLD].idx);
+  NTOHUL(oldIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  m[NEW].idx = open_file (m[NEW].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+
+  m[MERGE].idx = create_file (m[MERGE].fname, INVF_IDX_SUFFIX, "wb",
+			      MAGIC_INVI, MG_ABORT);
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+
+    /***
+     *  open .invf.parargraph files
+     ***/
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = open_file (m[NEW].fname, INVF_PARAGRAPH_SUFFIX, "rb",
+			       MAGIC_PARAGRAPH, MG_ABORT);
+      m[MERGE].para = open_file (m[MERGE].fname, INVF_PARAGRAPH_SUFFIX, "rb+",
+				 MAGIC_PARAGRAPH, MG_ABORT);
+    }
+  else
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = NULL;
+      m[MERGE].para = NULL;
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * procEntry_slow(): merge an entry, slow method
+ *=======================================================================*/
+void 
+procEntry_slow (int tnum, int type, u_long oldN, u_long newN, u_long mergeN)
+/* 
+ *  Process an invf entry. 
+ *  Slow Merge. (Always decode/re-encode).
+ *
+ *  tnum is the entry number, type = OLD or NEW or MERGE.
+ *  oldN, newN and mergeN are the N parameters for Bblock coding.
+ *  They are passed so other procEntry_fast can call this one
+ *  when a decode/recode is needed to save repeating code.
+ */
+{
+  int doc = 0;
+  int fcntOLD = m[OLD].fcntlist[tnum];
+  int fcntNEW = m[NEW].fcntlist[tnum];
+  int inblk, outblk;
+  u_long inbits = 0, outbits = 0;
+
+  /* write .invf.idx pointer */
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  outblk = BIO_Bblock_Init (mergeN, (fcntOLD + fcntNEW));
+
+/*** OLD entry -> merge entry ***/
+  if (type != NEW)
+    {
+      int i;
+      inblk = BIO_Bblock_Init (oldN, fcntOLD);
+      for (i = 0; i < fcntOLD; i++)
+	{
+	  int num;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[OLD].sbs), &inbits);
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[OLD].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	    }
+	}
+      if (type == OLD)
+	oldOnlyBits += (inbits + m[OLD].sbs.Btg);
+      else
+	oldMergeBits += (inbits + m[OLD].sbs.Btg);
+    }
+
+/*** NEW entry -> merge entry ***/
+  if (type != OLD)
+    {
+      int i;
+      int offset;
+      double logN = log ((double) m[MERGE].nDocs);
+      double idf = logN - log ((double) (fcntOLD + fcntNEW));
+
+      inblk = BIO_Bblock_Init (newN, fcntNEW);
+      offset = (m[OLD].nDocs - doc);	/* amount to be added to 1ST pointer */
+      for (i = 0; i < fcntNEW; i++)
+	{
+	  register int num;
+	  register double weight;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[NEW].sbs), &inbits);
+	  if (i == 0)
+	    num += offset;
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[NEW].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	      weight = num * idf;
+	      DocWeightBuffer[doc - m[OLD].nDocs - 1] += weight * weight;
+	    }
+	}
+    }
+/*** Now the padding bits ***/
+  while (m[OLD].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[OLD].sbs));
+  while (m[NEW].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[NEW].sbs));
+  while (m[MERGE].sbs.Btg != 8)
+    {
+      BIO_Stdio_Encode_Bit (0, &(m[MERGE].sbs));
+      outbits++;
+    }
+  bytes_output += (outbits >> 3);
+  return;
+}
+
+
+/*=======================================================================
+ * procEntry_fast(): merge an entry, WITHOUT decoding if it's in the
+ *                   old inverted file only
+ *=======================================================================*/
+void 
+procEntry_fast (int tnum, int type)
+/*
+ *  Merge an invf entry. 
+ *  Faster Method : Copy (don't re-code) entries for terms only in IFold.
+ *  The N parameter is Nstatic, from m[OLD].idh.static_num_of_docs.
+ *  If entry type isnt OLD, just call procEntry_slow()
+ */
+{
+  u_long newIdxValue, len = 0;
+
+  /* Calculate IFold entry length in bytes if not NEW type */
+  if (type != NEW)		/* read in an index number from m[OLD].idx */
+    {
+      fread (&newIdxValue, sizeof (u_long), 1, m[OLD].idx);
+      NTOHUL(newIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+      len = newIdxValue - oldIdxValue;
+      oldIdxValue = newIdxValue;
+    }
+
+  if (type != OLD)		/* must decode/recode */
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, Nstatic);
+  else
+    {
+      /* write .invf.idx pointer */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* copy entry */
+      {
+	int i;
+	char c;
+	for (i = 0; i < len; i++)
+	  {
+	    c = fgetc (m[OLD].invf);
+	    fputc (c, m[MERGE].invf);
+	  }
+      }
+      bytes_output += len;
+      oldOnlyBits += (len * 8);
+    }
+  return;
+}
+
+
+/*=======================================================================
+ * processEntry(): merge an entry. Call either procEntry_slow() or
+ *                 procEntry_fast.
+ *=======================================================================*/
+void 
+processEntry (int tnum, int type)
+/* 
+ *  Read an entry from IFold and/or IFnew, and write to IFmerge.
+ *  tnum is the term number, type is OLD or NEW or MERGE
+ */
+{
+  if (fastMerge)
+    procEntry_fast (tnum, type);
+  else
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, m[MERGE].nDocs);
+  return;
+}
+
+
+/*=======================================================================
+ * readTerm()
+ *=======================================================================*/
+void 
+readTerm (int x)
+/*
+ *  Read the next term from the appropriate ".invf.dict" file.
+ *  x should be either OLD or NEW.
+ *  Sets m[x].done to true when that lexicon has been completely read.
+ */
+{
+  int i;
+
+  if (m[x].term_count >= m[x].nTerms)
+    {
+      m[x].term[0] = 0;
+      m[x].done = 1;
+      return;
+    }
+  m[x].pref = fgetc (m[x].dict);
+  m[x].suff = fgetc (m[x].dict);
+  m[x].term[0] = m[x].pref + m[x].suff;
+
+  for (i = 0; i < m[x].suff; i++)
+    {
+      m[x].term[m[x].pref + i + 1] = fgetc (m[x].dict);
+    }
+  m[x].term_count = m[x].term_count + 1;
+
+  fread ((char *) &(m[x].fcnt), sizeof (u_long), 1, m[x].dict);
+  fread ((char *) &(m[x].wcnt), sizeof (u_long), 1, m[x].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[x].fcnt);
+  NTOHUL(m[x].wcnt);
+
+  return;
+}
+
+
+/*=======================================================================
+ * writeTerm()
+ *=======================================================================*/
+void 
+writeTerm (int x)
+/*
+ *  Write the current term to m[MERGE].dict file.
+ *  x is OLD, NEW or MERGE.
+ */
+{
+  unsigned char i, prefix, suffix;
+  if (x == MERGE)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt + m[NEW].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt + m[NEW].wcnt;
+      m[MERGE].pref = m[OLD].pref;
+      m[MERGE].suff = m[OLD].suff;
+      memcpy (m[MERGE].term, m[OLD].term, m[OLD].term[0] + 1);
+    }
+  if (x == OLD)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt;
+    }
+  if (x == NEW)
+    {
+      m[MERGE].fcnt = m[NEW].fcnt;
+      m[MERGE].wcnt = m[NEW].wcnt;
+    }
+  prefix = prefixlen (prevTerm, m[x].term);
+  suffix = m[x].term[0] - prefix;
+  fputc (prefix, m[MERGE].dict);	/* prefix length */
+  fputc (suffix, m[MERGE].dict);	/* suffix length */
+  for (i = 0; i < suffix; i++)
+    fputc (m[x].term[prefix + i + 1], m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].fcnt);
+  HTONUL(m[MERGE].wcnt);
+
+  fwrite ((char *) &(m[MERGE].fcnt), sizeof (m[MERGE].fcnt), 1, m[MERGE].dict);
+  fwrite ((char *) &(m[MERGE].wcnt), sizeof (m[MERGE].wcnt), 1, m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[MERGE].fcnt);
+  NTOHUL(m[MERGE].wcnt);
+
+  memcpy (prevTerm, m[x].term, prefix + suffix + 1);
+  m[MERGE].idh.total_bytes += m[x].term[0] + 1;
+  m[MERGE].idh.index_string_bytes += m[x].term[0] + 2 - prefix;
+  fflush (m[MERGE].dict);
+}
+
+
+/*=======================================================================
+ * process_merge_invf(): The main loop
+ *=======================================================================*/
+int 
+process_merge_invf (void)
+/*
+ *  The main function to merge the lexicons and inverted files.
+ */
+{
+/*** Some extra Initialisation stuff first ***/
+
+  m[OLD].term[0] = m[NEW].term[0] = '\0';
+  m[OLD].term_count = m[NEW].term_count = 0;
+  m[OLD].done = m[NEW].done = 0;
+  m[MERGE].idh.total_bytes = m[MERGE].idh.index_string_bytes = 0;
+
+/*** malloc arrays: mergedata, m[OLD].fcntlist, m[NEW].fcntlist ***/
+  mergedata = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (char));
+  if (mergedata == 0)
+    {
+      fprintf (stderr, "MALLOC error!\n");
+      exit (1);
+    }
+  {
+    int i;
+    for (i = 0; i < 2; i++)
+      {
+	m[i].fcntlist = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (int));
+	if (m[i].fcntlist == 0)
+	  {
+	    fprintf (stderr, "MALLOC error!\n");
+	    exit (1);
+	  }
+      }
+  }
+
+/*========================*/
+/***    LEXICON PASS    ***/
+/*========================*/
+
+/*** read first terms ***/
+  readTerm (OLD);
+  readTerm (NEW);
+
+  m[MERGE].nTerms = 0;
+  while (m[OLD].done == 0 || m[NEW].done == 0)
+    {
+      int i;
+      if (m[OLD].done)
+	i = 1; /* NEW will always be greater */
+      else if (m[NEW].done)
+	i = -1; /* OLD will always be greater */
+      else
+	i = casecompare (m[OLD].term, m[NEW].term);
+
+      if (i < 0)
+	{			/* term in OLD only */
+	  mergedata[m[MERGE].nTerms] = (char) OLD;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (OLD);
+	  readTerm (OLD);
+	}
+      if (i == 0)
+	{			/* term in both lexions */
+	  mergedata[m[MERGE].nTerms] = (char) MERGE;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[MERGE].nTerms++;
+	  writeTerm (MERGE);
+	  readTerm (OLD);
+	  readTerm (NEW);
+	}
+      if (i > 0)
+	{			/* term in NEW only */
+	  mergedata[m[MERGE].nTerms] = (char) NEW;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (NEW);
+	  readTerm (NEW);
+	}
+    }/*while*/
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  /* print some results about terms */
+  {
+    fprintf (stderr, "   Terms: OLD = %ld,  NEW = %ld,  MERGE = %ld\n",
+	     m[OLD].nTerms, m[NEW].nTerms, m[MERGE].nTerms);
+    fprintf (stderr, "   In OLD only: %ld,  In NEW only: %ld,  BOTH: %ld\n",
+	     (m[MERGE].nTerms - m[NEW].nTerms),
+	     (m[MERGE].nTerms - m[OLD].nTerms),
+	     (m[OLD].nTerms + m[NEW].nTerms - m[MERGE].nTerms));
+  }
+
+
+/*========================*/
+/***      INVF PASS     ***/
+/*========================*/
+
+  bytes_output = ftell (m[MERGE].invf);
+  BIO_Stdio_Decode_Start (m[OLD].invf, &m[OLD].sbs);
+  BIO_Stdio_Decode_Start (m[NEW].invf, &m[NEW].sbs);
+  BIO_Stdio_Encode_Start (m[MERGE].invf, &m[MERGE].sbs);
+  {
+    int i;
+    for (i = 0; i < m[MERGE].nTerms; i++)
+      {
+	processEntry (i, mergedata[i]);
+      }
+    /* write final .invf.idx pointer */
+    HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+    fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+    NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+    fprintf (stderr, "Old invf: old only bits: %ld, merged bits: %ld\n",
+	     oldOnlyBits, oldMergeBits);
+  }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * done_merge_invf(): write headers, close files, append weights to
+ *                     weights file
+ *=======================================================================*/
+int 
+done_merge_invf (void)
+{
+/*** write new lexicon header ***/
+  m[MERGE].idh.dict_size = m[MERGE].nTerms;
+  m[MERGE].idh.lookback = m[OLD].idh.lookback;
+  m[MERGE].idh.input_bytes = m[OLD].idh.input_bytes + m[NEW].idh.input_bytes;
+  m[MERGE].idh.total_bytes = m[OLD].idh.total_bytes + m[NEW].idh.total_bytes;
+  m[MERGE].idh.num_of_docs = m[MERGE].nDocs;
+  m[MERGE].idh.num_of_words = m[OLD].idh.num_of_words + m[NEW].idh.num_of_words;
+  m[MERGE].idh.stemmer_num = m[OLD].idh.stemmer_num;
+  m[MERGE].idh.stem_method = m[OLD].idh.stem_method;
+
+  /* if fastMerge, static num of docs stays the same! */
+  if (fastMerge)
+    m[MERGE].idh.static_num_of_docs = m[OLD].idh.static_num_of_docs;
+  else
+    m[MERGE].idh.static_num_of_docs = m[MERGE].nDocs;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].idh.lookback);
+  HTONUL(m[MERGE].idh.dict_size);
+  HTONUL(m[MERGE].idh.total_bytes);
+  HTONUL(m[MERGE].idh.index_string_bytes);
+  HTOND(m[MERGE].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(m[MERGE].idh.num_of_docs);
+  HTONUL(m[MERGE].idh.static_num_of_docs);
+  HTONUL(m[MERGE].idh.num_of_words);
+  HTONUL(m[MERGE].idh.stemmer_num);
+  HTONUL(m[MERGE].idh.stem_method);
+
+  fseek (m[MERGE].dict, magicsize, 0);
+  fwrite (&(m[MERGE].idh), sizeof (struct invf_dict_header), 1, m[MERGE].dict);
+
+/*** write new inverted file header ***/
+  m[MERGE].ifh.no_of_words = m[MERGE].nTerms;
+  m[MERGE].ifh.no_of_ptrs = m[OLD].ifh.no_of_ptrs + m[NEW].ifh.no_of_ptrs;
+  m[MERGE].ifh.skip_mode = m[OLD].ifh.skip_mode;
+  m[MERGE].ifh.InvfLevel = m[OLD].ifh.InvfLevel;
+
+  /* ifh.params[16] -- I have NO IDEA what these do!!!!! */
+  /* only set if a inverted file is in skipped format, I think */
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = m[OLD].ifh.params[i];
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].ifh.no_of_words);
+  HTONUL(m[MERGE].ifh.no_of_ptrs);
+  HTONUL(m[MERGE].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      HTONUL(m[MERGE].ifh.params[i]);
+  }
+  HTONUL(m[MERGE].ifh.InvfLevel);
+
+  fseek (m[MERGE].invf, magicsize, 0);	/* go to start of header */
+  fwrite (&(m[MERGE].ifh), sizeof (struct invf_file_header), 1, m[MERGE].invf);
+
+/*** Write new document weights --- leave old weights alone ***/
+  if ((weightOption) && (m[OLD].ifh.InvfLevel >= 2))
+    {
+      FILE *weightfile = open_file (m[MERGE].fname, WEIGHTS_SUFFIX, "r+b",
+				    MAGIC_WGHT, MG_CONTINUE);  /* [RPAP - Feb 97: WIN32 Port] */
+      if (weightfile)
+	{
+	  u_long i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+	  fprintf (stderr, "writing new weights: Nnew = %ld\n", m[NEW].nDocs);
+	  fseek (weightfile, 0, 2);	/* end of file */
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (i = 0; i < m[NEW].nDocs; i++)
+	    HTONF(DocWeightBuffer[i]);
+
+	  fwrite ((char *) DocWeightBuffer, sizeof (float), m[NEW].nDocs,
+		  weightfile);
+	}
+      fclose (weightfile);
+    }
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      /* Update paragraph file */
+      int buffer_size = 1000;
+      u_char *buffer;
+      int num;
+
+      if (!(buffer = (u_char *) Xmalloc (buffer_size)))
+	FatalError (1, "Could not allocate memory for paragraph buffer");
+
+      fseek (m[MERGE].para, 0, SEEK_END);
+      while ((num = fread ((char *) buffer, sizeof (*buffer), buffer_size, m[NEW].para)) != 0)
+	fwrite ((char *) buffer, sizeof (*buffer), num, m[MERGE].para);
+
+      Xfree (buffer);
+    }
+
+/*** close files ***/
+  fclose (m[MERGE].invf);
+  fclose (m[OLD].invf);
+  fclose (m[NEW].invf);
+  fclose (m[MERGE].dict);
+  fclose (m[OLD].dict);
+  fclose (m[NEW].dict);
+  fclose (m[MERGE].idx);
+  fclose (m[OLD].idx);
+  fclose (m[NEW].idx);
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      fclose (m[NEW].para);
+      fclose (m[MERGE].para);
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s [-s] [-w] [-d directory] -f mg_collection\n",
+	   progname);
+  fprintf (stderr, "  example: %s -s -w -f MERGE/mailfiles\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int 
+main (int argc, char *argv[])
+{
+  int ch;
+  char *progname;
+  progname = argv[0];
+  m[MERGE].fname[0] = '\0';
+
+  /* message and timing information */
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:swh")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (m[MERGE].fname, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	fastMerge = 0;
+	break;
+      case 'w':
+	weightOption = 1;
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (m[MERGE].fname[0] == '\0')
+    usage (progname);
+  strcpy (m[OLD].fname, m[MERGE].fname);
+  strcat (m[OLD].fname, ".old");
+  strcpy (m[NEW].fname, m[MERGE].fname);
+  strcat (m[NEW].fname, ".new");
+
+  GetTime (&start);
+  init_merge_invf ();
+  process_merge_invf ();
+  done_merge_invf ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  exit (0);
+}
+
+/*************************************************************************/
+/*   EOF: mg_invf_merge_new.c                                            */
+/*************************************************************************/
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_invf_rebuild.c	(revision 21112)
@@ -0,0 +1,567 @@
+/**************************************************************************
+ *
+ * mg_invf_rebuild.c -- Program to rebuild an inverted file with skipping
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define LEN 40
+#define WRD 1
+
+typedef struct invf_info
+  {
+    unsigned long doc_num, count, bits_so_far, count_bits;
+  }
+invf_info;
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:11  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:03  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:51  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+static char pathname[256];
+
+static void process_files (char *filename);
+
+static int k = -1;
+static int mode = 0;
+static int max_nodes = -1;
+static int mins = -1;
+
+void 
+usage (char *pgname)
+{
+  fprintf (stderr, "usage: %s [-f input_file]"
+       "[-d data directory] -0|-1|-2 [-k num] [-m num] [-s num]\n", pgname);
+  printf ("Mode 0: (the default)\n");
+  printf ("\tk, m, and s have no meaning and if specified produce an error\n");
+  printf ("Mode 1:\n");
+  printf ("\tm, and s have no meaning and if specified produce an error\n");
+  printf ("\tk The size of skips\n");
+  printf ("Mode 2:\n");
+  printf ("\tk has no meaning and if specified will produce an error\n");
+  printf ("\tm is the the number of accumulators that will be used for the\n");
+  printf ("\t  ranking. The program builds an inverted file that is \n");
+  printf ("\t  \"optimal\" for that number of accumulators.\n");
+  printf ("\ts is the minimum size for skips.\n");
+  exit (1);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  strcpy (pathname, dir_name ? dir_name : ".");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "012hf:d:k:b:s:m:")) != -1)
+    switch (ch)
+      {
+      case '0':
+	mode = 0;
+	break;
+      case '1':
+	mode = 1;
+	break;
+      case '2':
+	mode = 2;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	strcpy (pathname, optarg);
+	break;
+      case 'k':
+	k = atoi (optarg);
+	break;
+      case 'm':
+	max_nodes = atoi (optarg);
+	break;
+      case 's':
+	mins = atoi (optarg);
+	if (mins < 1)
+	  FatalError (1, "The number for the -m option must be greater"
+		      " than or equal to 1");
+	break;
+      case 'h':
+      case '?':
+	usage (argv[0]);
+      }
+
+  if ((mode == 0 && (k != -1 || max_nodes != -1 || mins != -1)) ||
+      (mode == 1 && (max_nodes != -1 || mins != -1)) ||
+      (mode == 2 && (k != -1)))
+    {
+      Message ("Illegal parameters for mode");
+      usage (argv[0]);
+    }
+
+  if (mode == 1 && k == -1)
+    {
+      k = 8;
+      Message ("k is required for mode 1: defaulting k to %d", k);
+    }
+  if (mode == 2 && max_nodes == -1)
+    {
+      max_nodes = 1024;
+      Message ("m is required for mode 2: defaulting m to %d", max_nodes);
+    }
+  if (mode == 2 && mins == -1)
+    {
+      mins = 1;
+      Message ("s is required for mode 2: defaulting s to %d", mins);
+    }
+
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  Message ("**** Don\'t forget to rebuild the stemmed dictionary with mg_invf_dict. ****\n");
+  Message ("**** If the collection was built with stem indexes don\'t forget to     ****\n");
+  Message ("**** rebuild them with mg_stem_idx.                                     ****\n");
+
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *in, *out, *idx, *odx, *dict;
+  unsigned long magic, outmode, N, in_k, out_k;
+  unsigned long bits_out, bytes_out, i, j;
+  stdio_bitio_state out_buf, in_buf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh_in, ifh_out;
+  char FName[256];
+
+  outmode = mode;
+
+  /* open .invf.ORG, rename .invf if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      rename (fname, FName);
+      if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.ORG */
+  if (fread (&magic, sizeof (magic), 1, in) != 1 || NTOHUL(magic) != MAGIC_INVF)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+
+  /* open .invf.idx.ORG, rename .invf.idx if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      rename (fname, FName);
+      if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.idx.ORG */
+  if (fread (&magic, sizeof (magic), 1, idx) != 1 || NTOHUL(magic) != MAGIC_INVI)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(out = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(odx = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Opening \"%s\"\n", FName);
+  if (!(dict = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  if (fread (&magic, sizeof (magic), 1, dict) != 1 || NTOHUL(magic) != MAGIC_STEM_BUILD)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  HTONUL2(MAGIC_INVF, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, out);
+
+  fread ((char *) &ifh_in, sizeof (ifh_in), 1, in);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_in.no_of_words);
+  NTOHUL(ifh_in.no_of_ptrs);
+  NTOHUL(ifh_in.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_in.params[i]);
+  NTOHUL(ifh_in.InvfLevel);
+
+  ifh_out = ifh_in;
+  ifh_out.skip_mode = outmode;
+  bzero ((char *) ifh_out.params, sizeof (ifh_out.params));
+  switch (outmode)
+    {
+    case 0:
+      break;
+    case 1:
+      ifh_out.params[0] = k;
+      break;
+    case 2:
+      ifh_out.params[0] = max_nodes;
+      ifh_out.params[1] = mins;
+      break;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(ifh_out.no_of_words);
+  HTONUL(ifh_out.no_of_ptrs);
+  HTONUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    HTONUL(ifh_out.params[i]);
+  HTONUL(ifh_out.InvfLevel);
+
+  fwrite ((char *) &ifh_out, sizeof (ifh_out), 1, out);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_out.no_of_words);
+  NTOHUL(ifh_out.no_of_ptrs);
+  NTOHUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_out.params[i]);
+  NTOHUL(ifh_out.InvfLevel);
+
+  Message ("The file is a level %d inverted file.\n", ifh_in.InvfLevel);
+
+  bits_out = ftell (out) * 8;
+
+
+  HTONUL2(MAGIC_INVI, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, odx);
+
+  DECODE_START (in)
+    DECODE_PAUSE (in_buf)
+
+    ENCODE_START (out)
+    ENCODE_PAUSE (out_buf)
+
+    N = idh.num_of_docs;
+
+  for (i = 0; i < ifh_in.no_of_words; i++)
+    {
+      unsigned long blk, p;
+      unsigned long odn_blk = 0, olen_blk = 0;
+      unsigned long idn_blk = 0, ilen_blk = 0;
+      register unsigned long suff;
+      unsigned long fcnt, wcnt, doc_num, bits_so_far, last_major;
+      unsigned long next_mjr_dn, kd;
+      char dummy2[MAXSTEMLEN + 1];
+      invf_info *ii;
+
+      fgetc (dict);
+      suff = fgetc (dict);
+      fread (dummy2, sizeof (u_char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+      NTOHUL(bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      p = fcnt;
+      blk = BIO_Bblock_Init (idh.static_num_of_docs, p);
+      switch (outmode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_out.params[0])
+	      out_k = 0;
+	    else
+	      {
+		out_k = ifh_out.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= mins)
+	      out_k = 0;
+	    else
+	      {
+		out_k = (int) (2 * sqrt ((double) p / max_nodes));
+		if (out_k <= mins)
+		  out_k = mins;
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	default:
+	  out_k = 0;
+	}
+
+      switch (ifh_in.skip_mode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[0])
+	      in_k = 0;
+	    else
+	      {
+		in_k = ifh_in.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[1])
+	      {
+		in_k = 0;
+	      }
+	    else
+	      {
+		in_k = (int) (2 * sqrt ((double) p / ifh_in.params[0]));
+		if (in_k <= ifh_in.params[1])
+		  in_k = ifh_in.params[1];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	default:
+	  in_k = 0;
+	}
+
+      if (!(ii = Xmalloc (sizeof (invf_info) * p)))
+	FatalError (1, "Unable to allocate memory for \"ii\"\n");
+
+      doc_num = bits_so_far = 0;
+      next_mjr_dn = 0;
+      kd = 0;
+      DECODE_CONTINUE (in_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  unsigned long doc_diff, count = 0;
+	  if (kd == in_k)
+	    kd = 0;
+	  if (in_k && kd == 0 && j + in_k < p)
+	    {
+	      int temp;
+	      BBLOCK_DECODE (next_mjr_dn, idn_blk);
+	      next_mjr_dn += doc_num;
+	      BBLOCK_DECODE (temp, ilen_blk);
+	    }
+	  ii[j].bits_so_far = bits_so_far;
+	  if (in_k && kd == in_k - 1 && j != p - 1)
+	    {
+	      int count;
+	      BBLOCK_LENGTH (next_mjr_dn - doc_num, blk, count);
+	      bits_so_far += count;
+	      doc_num = next_mjr_dn;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (doc_diff, blk, bits_so_far);
+	      doc_num += doc_diff;
+	    }
+	  ii[j].doc_num = doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    {
+	      int count_bits = 0;
+	      GAMMA_DECODE_L (count, count_bits);
+	      ii[j].count_bits = count_bits;
+	      bits_so_far += count_bits;
+	      ii[j].count = count;
+	    }
+	}
+
+      /* read till a byte boundary */
+      while (__btg)
+	{
+	  DECODE_BIT;
+	  bits_so_far++;
+	}
+
+      DECODE_PAUSE (in_buf)
+
+	doc_num = bits_so_far = 0;
+      last_major = 0;
+      kd = 0;
+      ENCODE_CONTINUE (out_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  if (kd == out_k)
+	    kd = 0;
+	  if (out_k && kd == 0)
+	    {
+	      if (j + out_k < p)
+		{
+		  int num = ii[j + out_k - 1].doc_num - last_major;
+		  BBLOCK_ENCODE_L (num, odn_blk, bits_out);
+		  last_major = ii[j + out_k - 1].doc_num;
+
+		  num = ii[j + out_k - 1].bits_so_far + ii[j + out_k - 1].count_bits -
+		    bits_so_far;
+		  BBLOCK_ENCODE_L (num, olen_blk, bits_out);
+		  bits_so_far = ii[j + out_k].bits_so_far;
+		}
+	    }
+	  if (!(out_k && kd == out_k - 1 && j != p - 1))
+	    BBLOCK_ENCODE_L (ii[j].doc_num - doc_num, blk, bits_out);
+	  doc_num = ii[j].doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    GAMMA_ENCODE_L (ii[j].count, bits_out);
+	}
+
+      /* write till a byte boundary */
+      while (__btg != 8)
+	{
+	  ENCODE_BIT (0);
+	  bits_out++;
+	}
+      ENCODE_PAUSE (out_buf)
+
+	Xfree (ii);
+
+    }
+  ENCODE_CONTINUE (out_buf)
+    ENCODE_DONE
+
+    HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+  NTOHUL(bytes_out);
+
+  fclose (idx);
+  fclose (odx);
+  fclose (in);
+  fclose (out);
+  fclose (dict);
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_merge.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_merge.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_merge.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mgmerge.h --- mgmerge #defines, etc
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edtited: November 12 1994
+ *
+ **************************************************************************/
+
+#ifndef MGMERGE_H
+#define MGMERGE_H
+
+typedef unsigned char byte;
+
+#define OLD 0
+#define NEW 1
+#define MERGE 2
+
+#define OK 0
+#define ERROR_MGMERGE 1
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes.c	(revision 21112)
@@ -0,0 +1,641 @@
+/**************************************************************************
+ *
+ * mg_passes.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+#include <stdlib.h>
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.3  2004/11/29 03:15:13  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.2  2004/04/25 23:01:18  kjdon
+   added a new -M option to mg_passes, allowing maxnumeric to be altered - made this change to keep gsdl3 mg inline with gsdl2 mg.
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.3  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.2  2001/06/12 23:23:42  jrm21
+   fixed a bug where mg_passes segfaults when trying to print the usage message.
+
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
+"  %*s [-a stemmer] [-M max-numeric] -f doc-collection-name\n";
+
+
+static void 
+usage (char *err)
+{
+  if (err)
+    Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "",strlen (msg_prefix), "",
+	   strlen (msg_prefix),"");
+  exit (1);
+}
+
+
+
+
+#if 0
+static char *
+str_comma (unsigned long u)
+{
+  static char buf[20];
+  unsigned long a, b, c, d;
+  a = u / 1000000000;
+  u -= a * 1000000000;
+  b = u / 1000000;
+  u -= b * 1000000;
+  c = u / 1000;
+  u -= c * 1000;
+  d = u;
+
+  if (a)
+    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
+  else if (b)
+    sprintf (buf, "%u,%03u,%03u", b, c, d);
+  else if (c)
+    sprintf (buf, "%u,%03u", c, d);
+  else
+    sprintf (buf, "%u", d);
+  return (buf);
+}
+#endif
+
+
+
+
+int 
+open_next_file (int in_fd)
+{
+  if (in_fd > 0)
+    close (in_fd);
+  if (num_files == 0)
+    return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  files++;
+  num_files--;
+  return (in_fd);
+}
+
+
+static void 
+driver (int in_fd, FILE * Trace, char *file_name)
+{
+  int pass, num = 1;
+
+  char *buffer = Xmalloc (buf_size);
+  unsigned long num_docs = 0;
+  unsigned long block_bytes = 0;
+  register int buf_left = buf_size;
+  register char *look_pos = buffer;
+  register char *end_pos = buffer;
+
+  ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+  GetTime (&StartTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->init_time);
+#endif
+      }
+
+  GetTime (&InitTime);
+  for (;;)
+    {
+      int len = 0;
+      char *base = look_pos;
+
+      while (look_pos != end_pos && *look_pos != TERMRECORD)
+	look_pos++;
+
+      while (look_pos == end_pos)
+	{
+	  if (buf_left < MIN_BUF)
+	    {
+	      bcopy (base, buffer, end_pos - base);
+	      look_pos = buffer + (end_pos - base);
+	      buf_left = buf_size - (end_pos - base);
+	      end_pos = look_pos;
+	      base = buffer;
+	    }
+	  if (buf_left)
+	    {
+    	      num = read (in_fd, end_pos, buf_left);
+	      if (num < 0) num = 0; /* RJM - quick hack :-) */
+	      if (num == 0)
+		if ((in_fd = open_next_file (in_fd)) != -1)
+		  num = read (in_fd, end_pos, buf_left);
+	      bytes_received += num;
+	      buf_left -= num;
+	      end_pos += num;
+	    }
+	  while (look_pos < end_pos && *look_pos != TERMRECORD)
+	    look_pos++;
+	  if (buf_left == 0 && base == buffer && look_pos == end_pos)
+	    {
+	      Message ("Unable to find document terminator (i.e ^B)"
+		       " in the document");
+	      FatalError (1, "The document is in excess of %d chars long",
+			  look_pos - base);
+	    }
+	  if (!num)
+	    break;
+	}
+      len = look_pos++ - base;
+
+      if (!num && base == end_pos)
+	break;
+
+      bytes_processed += len;
+
+#ifndef QUIET
+      if (!len)
+	Message ("Warning : Processing zero length document");
+#endif
+
+      for (pass = 0; pass < MAX_PASSES; pass++)
+	if (Passes & (1 << pass))
+	  {
+	    register pass_data *pd = &PassData[pass];
+
+#ifdef HAVE_TIMES
+	    struct tms tims;
+	    times (&tims);
+	    pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    struct rusage ru;
+	    register struct timeval *tv = &pd->process_time;
+
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+
+	    if (pd->process ((u_char *) base, len) == COMPERROR)
+	      {
+		Message ("Error during processing of \"%s\"", pd->name);
+		if (Dump || Trace)
+		  {
+		    int i;
+		    FILE *f = Trace ? Trace : stderr;
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		    for (i = 0; i < len; i++)
+		      {
+			char ch = base[i];
+			if (ch == '\1' || ch == '\2')
+			  ch = '\n';
+			putc (ch, f);
+		      }
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		  }
+		if (Trace)
+		  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+			   bytes_processed, num_docs,
+			   ElapsedTime (&StartTime, NULL));
+		exit (1);
+	      }
+
+#ifdef HAVE_TIMES
+	    times (&tims);
+	    pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	  }
+      num_docs++;
+      if (Trace)
+	{
+	  block_bytes += (look_pos - base);
+	  if (block_bytes >= trace)
+	    {
+#ifdef HAVE_MALLINFO
+	      struct mallinfo mi;
+	      mi = mallinfo ();
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		       ElapsedTime (&StartTime, NULL));
+#else
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+#endif
+	    }
+	}
+      if (!num && look_pos - 1 == end_pos)
+	break;
+    }
+
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  free (buffer);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  int ch, in_fd;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:M:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'M':
+	  SetEnv ("maxnumeric", optarg, NULL);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc)
+    {
+      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+	FatalError (1, "Cannot open %s", argv[optind]);
+      files = &argv[optind + 1];
+      num_files = argc - (optind + 1);
+    }
+  else
+    in_fd = 0;			/* stdin */
+
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (in_fd, Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.c	(revision 21112)
@@ -0,0 +1,546 @@
+/**************************************************************************
+ *
+ * mg_passes_4jni.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+/* this needs to come first */
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include "mg_passes_4jni.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+#include "stemmer.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+
+#include "words.h"
+#include "environment.h"
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+FILE * Trace;
+char * filename;
+unsigned long num_docs = 0;
+unsigned long block_bytes = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+int mg_passes_exit_value = 0;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+
+/* clear all the settings from one mg_passes run to the next */
+void clear_variables() {
+
+  buf_size = 3 * 1024 * 1024;	/* 3Mb */
+  invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+  ChunkLimit = 0;
+  InvfLevel = 2;
+  SkipSGML = 0;
+  MakeWeights = 0;
+  Comp_Stats = NULL;
+  comp_stat_point = 0;
+  bytes_processed = 0;
+  bytes_received = 0;
+  stemmer_num = 0; /* default to the lovin stemmer */
+  stem_method = 0;
+  Trace = NULL;
+  filename = NULL;
+  num_docs = 0;
+  block_bytes = 0;
+  
+  Passes = 0;
+  trace = 0;
+  Dump = 0;
+  files = NULL;
+  num_files = 0;
+  trace_name = NULL;
+  
+  mg_passes_exit_value = 0;
+}
+
+/* ################################################## */
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num) {
+
+  switch(pass_type) {
+  case 'S':
+    Passes |= SPECIAL;
+    break;
+  case 'I':
+  case 'N':
+    if (pass_num == '1')
+      Passes |= IVF_PASS_1;
+    else if (pass_num == '2')
+      Passes |= IVF_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  case 'T':
+    if (pass_num == '1')
+      Passes |= TEXT_PASS_1;
+    else if (pass_num == '2')
+      Passes |= TEXT_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  }
+}
+
+/* -D arg to mg_passes */
+void dump_failed_document(int dump) {
+  Dump = dump;
+}
+
+/* -G arg to mg_passes */
+void ignore_sgml_tags(int ignore) {
+  if (ignore) {
+    SkipSGML = 1;
+  } else {
+    SkipSGML = 0;
+  }
+}
+
+/* -b arg to mg_passes */
+void set_buffer_size(long size) {
+  buf_size = size * 1024;
+  if (buf_size <  MIN_BUF) {
+    buf_size = MIN_BUF;
+  }
+}
+
+/* -c arg to mg_passes */
+void set_chunk_limit(long chunk_limit) {
+  ChunkLimit = chunk_limit;
+}
+
+/* -C arg to mg_passes */
+void set_comp_stat_point(int stat_point) {
+  comp_stat_point = stat_point * 1024;
+}
+
+/* -f arg to mg_passes */
+void set_filename(const char * filen) {
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+  filename = Xstrdup (filen);  
+}
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit) {
+  invf_buffer_size = limit * 1024 * 1024;
+}
+
+/* -1, -2, -3 args to mg_passes */
+void set_invf_level(char level) {
+  switch (level) {
+  case '1':
+    InvfLevel = 1;
+    break;
+  case '2':
+    InvfLevel = 2;
+    break;
+  case '3':
+    InvfLevel = 3;
+    break;
+  }
+}
+
+/* -W arg to mg_passes */
+void set_make_weights(int make_w) {
+  MakeWeights = make_w;
+}
+
+/* -M arg to mg_passes */
+void set_max_numeric(int max_numeric) {
+  char data[99];
+  sprintf(data, "%d", max_numeric);
+  SetEnv ("maxnumeric", data, NULL);
+}
+
+/* -a, -s args to mg_passes */
+void set_stem_options(const char * stemmer, int method) {
+  stemmer_num = stemmernumber (stemmer);
+  stem_method = method & STEMMER_MASK;
+}
+
+/* -t arg to mg_passes */
+void set_trace_point(int tracepos) {
+  trace = (unsigned long) (tracepos * 1024 * 1024);
+}
+
+/* -n arg to mg_passes */
+void set_trace_file(const char * filen) {
+  if (trace_name) {
+    Xfree (trace_name);
+    trace_name = NULL;
+  }
+  trace_name = Xstrdup (filen);
+}
+
+/* ############################################### */
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+
+ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+void 
+init_driver ()
+{
+  int pass;
+  if (!filename || *filename == '\0') {
+    mg_passes_exit_value = 1;
+    FatalError (1, "A document collection name must be specified.");
+  }
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2)) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  }
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2)) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+  }
+  if (!Passes) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+  }
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+  if (Trace)
+    {
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      /* print out the args to mg_passes */
+      fprintf (Trace, "\n\n");
+    }
+
+
+  GetTime (&StartTime);
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+      struct tms tims;
+      times (&tims);
+      pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      struct rusage ru;
+      
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      if (pd->init (filename) == COMPERROR) {
+	mg_passes_exit_value = 1;
+	FatalError (1, "Error during init of \"%s\"", pd->name);
+      }
+#ifdef HAVE_TIMES
+      times (&tims);
+      pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+      time_normalise (&pd->init_time);
+#endif
+    }
+  }
+  GetTime (&InitTime);
+}
+
+
+void process_document(const u_char *buffer, int len) {
+  int pass;
+  bytes_processed += len;
+
+#ifndef QUIET
+  if (!len)
+    Message ("Warning : Processing zero length document");
+#endif
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass))
+      {
+	register pass_data *pd = &PassData[pass];
+	
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	register struct timeval *tv = &pd->process_time;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->process ((u_char *) buffer, len) == COMPERROR)
+	  {
+	    Message ("Error during processing of \"%s\"", pd->name);
+	    if (Dump || Trace)
+	      {
+		int i;
+		FILE *f = Trace ? Trace : stderr;
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		for (i = 0; i < len; i++)
+		  {
+		    char ch = buffer[i];
+		    if (ch == '\1' || ch == '\2')
+		      ch = '\n';
+		    putc (ch, f);
+		  }
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+	      }
+	    if (Trace)
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+	    exit (1);
+	  }
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      }
+  }
+  num_docs++;
+  if (Trace)
+    {
+      block_bytes += len;
+      if (block_bytes >= trace)
+	{
+#ifdef HAVE_MALLINFO
+	  struct mallinfo mi;
+	  mi = mallinfo ();
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		   bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		   ElapsedTime (&StartTime, NULL));
+#else
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		   bytes_processed, num_docs,
+		   ElapsedTime (&StartTime, NULL));
+#endif
+	}
+    }
+}
+
+void finalise_driver() {
+  int pass;
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (filename) == COMPERROR) {
+	  mg_passes_exit_value = 1;
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+	}
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  //free (buffer);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+}
+
+int get_exit_value() {
+  return mg_passes_exit_value;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_passes_4jni.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mg_passes_4jni.h -- modified mg_passes for use from jni
+ * Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef H_MG_PASSES_4JNI
+#define H_MG_PASSES_4JNI
+
+/* clear all the settings from one mg_passes run to the next */
+void clear_variables();
+
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass(char pass_type, char pass_num);
+
+/* -D arg to mg_passes */
+void dump_failed_document(int dump);
+
+/* -G arg to mg_passes */
+void ignore_sgml_tags(int ignore);
+
+/* -b arg to mg_passes */
+void set_buffer_size(long size);
+
+/* -c arg to mg_passes */
+void set_chunk_limit(long chunk_limit);
+
+/* -C arg to mg_passes */
+void set_comp_stat_point(int stat_point);
+
+/* -f arg to mg_passes */
+void set_filename(const char * filen);
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit);
+
+/* -1, -2, -3 args to mg_passes */
+void set_invf_level(char level);
+
+/* -W arg to mg_passes */
+void set_make_weights(int make_w);
+
+/* -M arg to mg_passes */
+void set_max_numeric(int max_numeric);
+
+/* -a, -s args to mg_passes */
+void set_stem_options(const char * stemmer, int method);
+
+/* -t arg to mg_passes */
+void set_trace_point(int tracepos);
+
+/* -n arg to mg_passes */
+void set_trace_file(const char * filen);
+
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver ();
+
+void process_document(const u_char *buffer, int len);
+
+void finalise_driver();
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_perf_hash_build.c	(revision 21112)
@@ -0,0 +1,186 @@
+/**************************************************************************
+ *
+ * mg_perf_hash_build.c -- Program to build a perfect hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:13  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:15  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:58  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:53  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#define POOL_SIZE 1024*1024
+
+static void process_files (char *filename);
+
+int r = -1;
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'r':
+	r = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *dict, *hash;
+  unsigned long i;
+  u_char prev[MAXSTEMLEN + 1];
+  struct invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  int pool_left;
+  u_char **starts;
+
+
+  dict = open_file (filename, INVF_DICT_SUFFIX, "rb",
+		    MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  hash = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+		      MAGIC_HASH, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (!(pool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = Xmalloc (sizeof (u_char *) * idh.dict_size)))
+    FatalError (1, "Out of memory");
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff, l;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (dict);
+      suff = getc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, dict);
+      fread (&wcnt, sizeof (wcnt), 1, dict);
+
+      l = *prev + 1;
+      if (pool_left < l)
+	{
+	  pool = Xmalloc (POOL_SIZE);
+	  pool_left = POOL_SIZE;
+	}
+      starts[i] = pool;
+      bcopy ((char *) prev, (char *) pool, l);
+      pool += l;
+      pool_left -= l;
+    }
+  if (!(phd = gen_hash_func (idh.dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hash, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (dict);
+  fclose (hash);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_stem_idx.c	(revision 21112)
@@ -0,0 +1,892 @@
+/**************************************************************************
+ *
+ * mg_stem_idx.c -- Memory efficient stem index builder
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"     /* For struct stemmed_dict */
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+#include "local_strings.h"
+
+typedef struct PosEntry
+{
+  unsigned int num_cases;
+  unsigned int blk;
+  unsigned short blk_index;
+  unsigned short offset;
+}
+PosEntry;
+
+
+typedef struct PosList
+{
+  unsigned int list_size;
+  unsigned int num_entries;
+  PosEntry PE[1];
+}
+PosList;
+
+
+typedef struct idx_hash_rec
+{
+  u_char *word;
+  PosList *PL;
+}
+idx_hash_rec;
+
+
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+
+static idx_hash_rec **IdxHashTable;
+static unsigned long IdxHashSize;
+static unsigned long IdxHashUsed;
+static u_char *IdxPool;
+static int IdxPoolLeft;
+
+static idx_hash_rec *idx_hr_pool;
+static int idx_hr_PoolLeft;
+
+static idx_hash_rec **idx_first_occr;
+static int idx_max_first_occr;
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static long lookback = 4;
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+/* =========================================================================
+ * Function: MakePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+PosList *
+MakePosList (int n)
+{
+  PosList *pl;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  pl = Xmalloc (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+  if (!pl)
+    FatalError (1, "Unable to allocate term list");
+  ChangeMem (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+
+  pl->num_entries = n;
+  pl->list_size = list_size;
+
+  return pl;
+}
+
+/* =========================================================================
+ * Function: ResizePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizePosList (PosList ** pos_list)
+{
+  PosList *pl = *pos_list;
+
+  ChangeMem (-(sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)));
+  if (pl->num_entries > pl->list_size)
+    {
+      if (pl->list_size)
+	pl->list_size *= GROWTH_FACTOR;
+      else
+	pl->list_size = MIN_SIZE;
+    }
+  pl = Xrealloc (pl, sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  if (!pl)
+    FatalError (1, "Unable to resize pos list");
+  ChangeMem (sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  *pos_list = pl;
+}
+
+
+/* =========================================================================
+ * Function: FreePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreePosList (PosList ** the_pl)
+{
+  PosList *pl = *the_pl;
+
+  ChangeMem (-(sizeof(PosList) + sizeof (PosEntry) * (pl->list_size - 1)));
+  Xfree (pl);
+
+  *the_pl = NULL;
+}
+
+
+/* =========================================================================
+ * Function: ResetPosList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetPosList (PosList ** pl)
+{
+  if (*pl)
+    FreePosList (pl);
+  *pl = MakePosList (0);
+}
+
+/* =========================================================================
+ * Function: AddPosEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddPosEntry (PosList ** pos_list, PosEntry * pe)
+{
+  PosList *pl = *pos_list;
+
+  pl->num_entries++;
+  ResizePosList (pos_list);
+  pl = *pos_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) pe, (char *) &(pl->PE[pl->num_entries - 1]), sizeof (PosEntry));
+
+  return pl->num_entries - 1;
+}
+
+
+/* Modified from stem_search.c */
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    FatalError (1, "Could not allocate memory for stemmed dict");
+
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stemmer_num);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  return sd;
+}
+
+
+void
+mg_init_process ()
+{
+  /* Allocate memory for idx hash table */
+
+  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  IdxPoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec))))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  idx_hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+
+  IdxHashSize = INITIAL_HASH_SIZE;
+  IdxHashUsed = 0;
+  if (!(IdxHashTable = Xmalloc (sizeof (idx_hash_rec *) * IdxHashSize)))
+    FatalError (1, "Unable to allocate memory for idx table");
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashSize);
+  bzero ((char *) IdxHashTable, sizeof (idx_hash_rec *) * IdxHashSize);
+  idx_max_first_occr = 8192;
+  if (!(idx_first_occr = Xmalloc (sizeof (idx_hash_rec *) * idx_max_first_occr)))
+    FatalError (1, "Unable to allocate memory for idx_first_occur");
+  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+}
+
+
+void
+PackIdxHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < IdxHashSize; s++)
+    if (IdxHashTable[s])
+      IdxHashTable[d++] = IdxHashTable[s];
+  ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashUsed);
+  if (!(IdxHashTable = Xrealloc (IdxHashTable, sizeof (idx_hash_rec *) * IdxHashUsed)))
+    FatalError (1, "Out of memory");
+  IdxHashSize = IdxHashUsed;
+}
+
+
+void
+process_stem_dict (stemmed_dict * sd, int stem_method)
+{
+  int block;
+  short blk_index = -1;
+  int wordnum = -1;
+  PosEntry *prevPE = NULL;
+  idx_hash_rec *prevIdx = NULL;
+  u_char word[MAXSTEMLEN + 1];
+  u_char prev[MAXSTEMLEN + 1];
+
+  /* For each block in stem dict... */
+  for (block = 0; block < sd->sdh.num_blocks; block++)
+    {
+      register unsigned int res;
+      int num_indexes;
+      unsigned long *first_word, *last_invf_len;
+      unsigned short *num_words;
+      u_char *base;
+      unsigned short *index;
+
+      /* Read block into buffer */
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* Move through block header */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      {
+	/* [RPAP - Jan 97: Endian Ordering] */
+	int i;
+	for (i = 0; i < num_indexes; i++)
+	  NTOHUS(index[i]);
+      }
+
+      base = (u_char *) (index + num_indexes);
+      blk_index = -1;
+
+      /* For each word in block... */
+      for (res = 0; res < *num_words; res++)
+	{
+	  unsigned copy, suff;
+	  register idx_hash_rec *idx_ent = 0;
+
+	  /* Update blk_index */
+	  if (!(res % sd->sdh.lookback))
+	    blk_index++;
+
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+
+	  /* Skip irrelevant word info */
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+
+	  /* Stem word */
+	  bcopy ((char *) prev, (char *) word, *prev + 1);
+	  stemmer (stem_method, sd->sdh.stemmer_num, word);
+
+	  /* Check if word follows straight on from previous word */
+	  if (prevIdx && !compare (word, prevIdx->word))
+	    prevPE->num_cases++;
+
+	  else
+	    {
+	      /* Search the idx hash table for word */
+	      register unsigned long hashval, step;
+	      register int hsize = IdxHashSize;
+	      HASH (hashval, step, word, hsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  idx_ent = IdxHashTable[hashval];
+		  if (!idx_ent)
+		    {
+		      /* Create a next entry in the hash table */
+		      int len = *word + 1;
+		      if (!idx_hr_PoolLeft)
+			{
+			  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE *
+						       sizeof (idx_hash_rec))))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  idx_hr_PoolLeft = HASH_POOL_SIZE;
+			  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+			}
+		      idx_ent = idx_hr_pool++;
+		      idx_hr_PoolLeft--;
+		      if (len > IdxPoolLeft)
+			{
+			  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  IdxPoolLeft = POOL_SIZE;
+			  ChangeMem (POOL_SIZE);
+			}
+		      wordnum++;
+
+		      idx_ent->word = IdxPool;
+		      idx_ent->PL = MakePosList (0);
+		      {
+			PosEntry PE;
+			PE.num_cases = 1;
+			PE.blk = block;
+			PE.blk_index = (unsigned short) blk_index;
+			PE.offset = res % sd->sdh.lookback;
+			AddPosEntry (&(idx_ent->PL), &PE);
+		      }
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+
+		      bcopy ((char *) word, (char *) IdxPool, len);
+		      IdxPool += len;
+		      IdxPoolLeft -= len;
+		      if (IdxHashUsed == idx_max_first_occr - 1)
+			{
+			  ChangeMem (-sizeof (idx_hash_rec *) * idx_max_first_occr);
+			  idx_max_first_occr *= 2;
+			  if (!(idx_first_occr = Xrealloc (idx_first_occr, sizeof (idx_hash_rec *) *
+							   idx_max_first_occr)))
+			    FatalError (1, "Unable to allocate memory for idx_first_occr");
+			  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+			}
+		      idx_first_occr[IdxHashUsed] = idx_ent;
+		      IdxHashUsed++;
+		      IdxHashTable[hashval] = idx_ent;
+		      break;
+		    }
+		  
+		  /* Compare the words */
+		  s1 = word;
+		  s2 = idx_ent->word;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+		  
+		  if (len)
+		    {
+		      /* Entry is not the right one - move to next hash index */
+		      hashval = (hashval + step);
+		      if (hashval >= hsize)
+			hashval -= hsize;
+		    }
+		  else
+		    {
+		      /* Entry is correct - added PosEntry to word */
+		      PosEntry PE;
+		      PE.num_cases = 1;
+		      PE.blk = block;
+		      PE.blk_index = (unsigned short) blk_index;
+		      PE.offset = res % sd->sdh.lookback;
+		      AddPosEntry (&(idx_ent->PL), &PE);
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+		      break;
+		    }
+		}      
+	    }
+
+	  if (IdxHashUsed >= IdxHashSize >> 1)
+	    {
+	      idx_hash_rec **ht;
+	      unsigned long size;
+	      unsigned long i;
+	      size = prime (IdxHashSize * 2);
+	      if (!(ht = Xmalloc (sizeof (idx_hash_rec *) * size)))
+		FatalError (1, "Unable to allocate memory for idx table");
+	      bzero ((char *) ht, sizeof (idx_hash_rec *) * size);
+	      ChangeMem (sizeof (idx_hash_rec *) * size);
+
+	      for (i = 0; i < IdxHashSize; i++)
+		if (IdxHashTable[i])
+		  {
+		    register u_char *wptr;
+		    idx_hash_rec *ent;
+		    register unsigned long hashval, step;
+		    
+		    wptr = IdxHashTable[i]->word;
+		    HASH (hashval, step, wptr, size);
+		    ent = ht[hashval];
+		    while (ent)
+		      {
+			hashval += step;
+			if (hashval >= size)
+			  hashval -= size;
+			ent = ht[hashval];
+		      }
+		    ht[hashval] = IdxHashTable[i];
+		  }
+	      Xfree (IdxHashTable);
+	      ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+	      IdxHashTable = ht;
+	      IdxHashSize = size;
+	    }
+	  
+	} /* end for each word */
+
+    } /* end for each block */
+
+}
+ 
+static int 
+idx_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((idx_hash_rec **) A))->word;
+  u_char *s2 = (*((idx_hash_rec **) B))->word;
+  return (casecompare (s1, s2));
+}
+
+
+void
+save_idx (char * filename, int stem_method)
+{
+  char *FName;
+  unsigned long i, j, pos, First_word, num;
+  struct stem_idx_header sih;
+  u_char *buffer, *last_word = NULL;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+  FILE *idbi = NULL, *tmp = NULL;
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  /* Create appropriate stem index file */
+  switch (stem_method)
+    {
+    case (1):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "wb", MAGIC_STEM_1,
+			    MG_ABORT);
+	break;
+      }
+
+    case (2):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "wb", MAGIC_STEM_2,
+			    MG_ABORT);
+	break;
+      }
+    case (3):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "wb", MAGIC_STEM_3,
+			    MG_ABORT);
+	break;
+      }
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT create .invf.blocked.%d file", stem_method);
+
+  PackIdxHashTable();
+  qsort (IdxHashTable, IdxHashUsed, sizeof (idx_hash_rec *), idx_comp);
+
+  sih.lookback = lookback;
+  sih.block_size = block_size;
+  sih.num_blocks = 0;
+  sih.blocks_start = 0;
+  sih.index_chars = 0;
+  sih.num_of_words = IdxHashUsed;
+
+  fwrite ((char *) &sih, sizeof (sih), 1, idbi);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+
+  /* For each word in the hashtable... */
+  for (i = 0; i < IdxHashUsed; i++)
+    {
+      register unsigned long extra, copy, suff;
+      register struct idx_hash_rec *ent = IdxHashTable[i];
+
+      /* build a new word on top of prev */
+      if (last_word != NULL)
+	copy = prefixlen (last_word, ent->word);
+      else
+	copy = 0;
+      suff = *(ent->word) - copy;
+      last_word = ent->word;
+
+      if (word_num % sih.lookback == 0)
+	/* Will need copy chars to add + a pointer in index */
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + 
+	  ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	{
+	  /* Dump buffer to tmp file */
+	  int chunk;
+	  HTONUL(First_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUS(word_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sih.num_blocks++;
+
+	  /* Check that entry will fit into new block */
+	  if (sizeof (*pointers) + sizeof (ptrs_in_use) + extra + sizeof (First_word) +
+	      suff + 1 + sizeof (ent->PL->num_entries) + 
+	      ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	    FatalError (1, "Block size to small");
+
+	}
+
+      if (word_num % sih.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+
+      /* Output Word information */
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (ent->word + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &(ent->PL->num_entries), (char *) (buffer + buf_in_use), sizeof (ent->PL->num_entries));
+      NTOHUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (ent->PL->num_entries);
+
+      for (j = 0; j < ent->PL->num_entries; j++)
+	{
+	  register PosEntry *pe = &(ent->PL->PE[j]);
+	  HTONUI(pe->num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->num_cases), (char *) (buffer + buf_in_use), sizeof (pe->num_cases));
+	  buf_in_use += sizeof (pe->num_cases);
+	  HTONUI(pe->blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk), (char *) (buffer + buf_in_use), sizeof (pe->blk));
+	  buf_in_use += sizeof (pe->blk);
+	  HTONUS(pe->blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk_index), (char *) (buffer + buf_in_use), sizeof (pe->blk_index));
+	  buf_in_use += sizeof (pe->blk_index);
+	  HTONUS(pe->offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->offset), (char *) (buffer + buf_in_use), sizeof (pe->offset));
+	  buf_in_use += sizeof (pe->offset);
+	}
+
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 64209258\n");
+
+      if (word_num == 0)
+	{
+	  /* Write word to main index */
+	  fwrite (ent->word, sizeof (u_char), *(ent->word) + 1, idbi);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idbi);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sih.index_chars += *(ent->word) + 1;
+	  First_word = i;
+	}
+      word_num++;
+    } /* end for each word */
+
+  if (buf_in_use)
+    {
+      /* Write last buffer to tmp file */
+      int chunk;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sih.num_blocks++;
+    }
+
+  rewind (tmp);
+  sih.blocks_start = sih.index_chars + sizeof (u_long) + sizeof (sih) +
+    sih.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sih.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idbi);
+	  sih.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idbi);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idbi, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sih.lookback);
+  HTONUL(sih.block_size);
+  HTONUL(sih.num_blocks);
+  HTONUL(sih.blocks_start);
+  HTONUL(sih.index_chars);
+  HTONUL(sih.num_of_words);
+
+  fwrite (&sih, sizeof (sih), 1, idbi);
+  fclose (idbi);
+
+#ifndef SILENT
+  Message ("Stem %d:\n", stem_method);
+  Message ("     Block size   : %10d\n", block_size);
+  Message ("     Num_blocks   : %10d\n", NTOHUL(sih.num_blocks));  /* [RPAP - Jan 97: Endian Ordering] */
+  Message ("     Max mem used : %10.1f Mb\n", (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("     Num_of_words : %10d\n", NTOHUL(sih.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+#endif
+}
+
+
+void
+UpdateStemDict (char * filename, int stem_method)
+{
+  FILE *idb;
+  struct stem_dict_header sdh;
+
+  if (!(idb = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "r+b",
+			 MAGIC_STEM, MG_CONTINUE)))
+    FatalError (1, "Could not update stemmed dict");
+
+  fread ((char *) &sdh, sizeof (sdh), 1, idb);
+  NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+  sdh.indexed |= 1 << (stem_method - 1);
+  HTONUL(sdh.indexed);
+  fseek (idb, sizeof (u_long), 0);
+  fwrite ((char *) &sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+}
+
+
+
+/* Main */
+int main (int argc, char **argv)
+{
+  File *idb;    /* File to .invf.dict.blocked */
+  char *filename = "";
+  stemmed_dict *sd;   /* Stemmed dictionary */
+  int ch;
+  char path[512];
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:b:hFs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d directory] "
+		 "[-b num] [-F] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  if (stem_method < 1 || stem_method > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  /* Open required stem dict file */
+  sprintf (path, FILE_NAME_FORMAT, get_basepath (), filename, INVF_DICT_BLOCKED_SUFFIX);
+  if (!(idb = Fopen (path, "rb", MAGIC_STEM)))
+    FatalError (1, "Unable to open \"%s\"", path);
+
+  /* Read in idb header and index to blocks */
+  if (!(sd = ReadStemDictBlk (idb)))
+    FatalError (1, "Could not read stemmed dictionary");
+
+  /* Process stemmed dictionary */
+  mg_init_process ();
+  process_stem_dict (sd, stem_method);
+  save_idx (filename, stem_method);
+ 
+  /* Close stemmed dict */
+  Fclose (idb);
+
+  /* Update stemmed dict */
+  UpdateStemDict (filename, stem_method);
+
+  return 0;
+}
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_estimate.c	(revision 21112)
@@ -0,0 +1,584 @@
+/**************************************************************************
+ *
+ * mg_text_estimate.c -- Program to estimate the compressed text size
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "comp_dict.h"
+#include "hash.h"
+
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:14  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:48  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:19  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:59  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:54  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+typedef struct stats_t
+  {
+    u_long freq, occur;
+    u_long len;
+  }
+stats_t;
+
+static u_long huff_count = 0;
+static u_long escape_count = 0;
+
+
+char mode = 'H';
+
+/* [RJM 07/97: 4G limit] */
+static double ReadInWordsStandard (char *name, double *num_bytes);
+static double ReadInWordsSpecial (char *name, double *num_bytes,
+				  double *num_extra_bytes,
+				  u_long *aux_compressed);
+
+
+
+
+int main (int argc, char **argv)
+{
+  char *stats_dict = NULL, *comp_dict = NULL;
+  ProgTime StartTime;
+  double num_bytes, num_extra_bytes = 0; /* [RJM 07/97: 4G limit] */
+  u_long aux_compressed;
+  int ch;
+  double bytes;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "BMDYHh")) != -1)
+    switch (ch)
+      {
+      case 'B':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = ch;
+	break;
+      case 'M':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = 'Y';
+	break;
+      case 'D':
+      case 'Y':
+      case 'H':
+	mode = ch;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    stats_dict = argv[optind++];
+  else
+    stats_dict = NULL;
+  if (optind < argc)
+    comp_dict = argv[optind++];
+  else
+    {
+      fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+	       argv[0]);
+      exit (1);
+    }
+
+
+
+  GetTime (&StartTime);
+
+  if (LoadCompressionDictionary (comp_dict) == COMPERROR)
+    exit (1);
+
+  if (mode == 'H')
+    bytes = ReadInWordsStandard (stats_dict, &num_bytes);
+  else
+    bytes = ReadInWordsSpecial (stats_dict, &num_bytes, &num_extra_bytes,
+				&aux_compressed);
+
+
+  Message ("Compressed Text %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   bytes / 1024.0 / 1024, bytes, bytes * 100 / num_bytes);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / num_bytes);
+  if (mode == 'H')
+    Message ("Huffman info for chars in dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	     Chars_disk / 1024.0 / 1024, Chars_disk * 1.0,
+	     (Chars_disk * 100.0) / num_bytes);
+  else
+    {
+      Message ("Aux dictionary (Uncompressed) %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       num_extra_bytes / 1024.0 / 1024, num_extra_bytes * 1.0,
+	       (num_extra_bytes * 100.0) / num_bytes);
+      Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	       (aux_compressed * 100.0) / num_bytes);
+    }
+  Message ("Words and non-words in the text %u", huff_count + escape_count);
+  Message ("Words and non-words coded by escapes %u ( %0.3f %%)", escape_count,
+	   escape_count * 100.0 / huff_count);
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+
+static double 
+ReadInWordsStandard (char *name, double *num_bytes)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long which;
+  double bytes, bits = 0;
+
+  f = open_named_file (name, "rb", MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      frags_stats_header fsh;
+      int j;
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, freq, res, occur_num;
+	  u_char Word[16];
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(freq);
+	  NTOHSI(occur_num);
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      escape_count += freq;
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	      if (cdh.dict_type == MG_SEED_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * freq;
+	      huff_count += freq;
+	    }
+	}
+    }
+  fclose (f);
+
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
+
+
+static int 
+stats_comp (const void *a, const void *b)
+{
+  return ((stats_t *) a)->occur - ((stats_t *) b)->occur;
+}
+
+
+
+
+
+static double 
+ReadInWordsSpecial (char *name, double *num_bytes,
+		    double *num_extra_bytes,
+		    u_long *aux_compressed)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long magic, which;
+  double bytes, bits = 0;
+  double size[2];
+
+  if (!(f = fopen (name, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open file \"%s\"\n", name);
+
+  if (fread (&magic, sizeof (magic), 1, f) != 1)
+    FatalError (1, "Unable to read magic number");
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (magic != MAGIC_STATS_DICT)
+    FatalError (1, "Bad magic number");
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      stats_t *stats;
+      frags_stats_header fsh;
+      int j, num = 0;
+      long lens[16], chars[256];
+
+      bzero ((char *) lens, sizeof (lens));
+      bzero ((char *) chars, sizeof (chars));
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      stats = Xmalloc (fsh.num_frags * sizeof (*stats));
+      if (!stats)
+	FatalError (1, "Unable to allocate memory for stats");
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, res;
+	  u_char Word[16];
+	  fread (&stats[num].freq, sizeof (stats[num].freq), 1, f);
+	  fread (&stats[num].occur, sizeof (stats[num].occur), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(stats[num].freq);
+	  NTOHSI(stats[num].occur);
+
+	  stats[num].len = len + 1;
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  assert (stats[num].freq > 0);
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if ((cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		   cdh.dict_type == MG_SEED_DICTIONARY) &&
+		  ht[which])
+		{
+		  int k;
+		  res = ht[which]->hd->num_codes - 1;
+		  bits += ht[which]->hd->clens[res] * stats[num].freq;
+		  lens[Word[0]]++;
+		  for (k = 0; k < (int) Word[0]; k++)
+		    chars[Word[k + 1]]++;
+		}
+	      num++;
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * stats[num].freq;
+	    }
+	}
+
+      {
+	int j, num_buckets = 0;
+	int start[10], end[10];
+	long flens[16], fchars[256];
+
+	for (j = 0; j < 256; j++)
+	  if (!chars[j] && PESINAWORD (j) == which)
+	    fchars[j] = 1;
+	  else
+	    fchars[j] = chars[j];
+	for (j = 0; j < 16; j++)
+	  if (!lens[j])
+	    flens[j] = 1;
+	  else
+	    flens[j] = lens[j];
+
+	size[which] = Calculate_Huffman_Size (16, flens, lens) +
+	  Calculate_Huffman_Size (256, fchars, chars);
+
+
+	qsort (stats, num, sizeof (*stats), stats_comp);
+
+
+	if (mode == 'Y')
+	  {
+	    int k;
+	    num_buckets = 1;
+	    start[0] = 0;
+	    end[0] = cdh.num_words[which] - 1;
+	    Message ("There are %d novel %swords", num, which ? "" : "non-");
+	    while (end[num_buckets - 1] < num)
+	      {
+		start[num_buckets] = end[num_buckets - 1] + 1;
+		end[num_buckets] = start[num_buckets] +
+		  (end[num_buckets - 1] - start[num_buckets - 1]) * 2;
+		num_buckets++;
+	      }
+	    for (k = 0; k < num_buckets; k++)
+	      Message ("Bucket %d   %d <= x <= %d\n", k + 1, start[k], end[k]);
+	  }
+
+	for (j = 0; j < num; j++)
+	  {
+	    (*num_extra_bytes) += stats[j].len;
+	    switch (mode)
+	      {
+	      case 'D':
+		bits += BIO_Delta_Length (j + 1) * stats[j].freq;
+		break;
+	      case 'B':
+		bits += BIO_Binary_Length (j + 1, num) * stats[j].freq;
+		break;
+	      case 'Y':
+		{
+		  int k = 0;
+		  while (j > end[k])
+		    k++;
+		  if (!(j - start[k] + 1 >= 1 &&
+			j - start[k] + 1 <= end[k] - start[k] + 1))
+		    {
+		      Message ("k = %d, j = %d", k, j);
+		      assert (j - start[k] + 1 >= 1 &&
+			      j - start[k] + 1 <= end[k] - start[k] + 1);
+		    }
+		  else
+		    {
+		      bits += (BIO_Gamma_Length (k + 1) +
+			       BIO_Binary_Length (j - start[k] + 1,
+						  end[k] - start[k] + 1)) *
+			stats[j].freq;
+		    }
+		}
+	      }
+	  }
+	if (mode == 'B' && num)
+	  bits += BIO_Delta_Length (num) * csh.num_docs;
+      }
+      Xfree (stats);
+    }
+  fclose (f);
+
+  *aux_compressed = (size[0] + size[1]) / 8;
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_text_merge.c	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * mg_text_merge.c --- merge  *.text, *.text.idx files
+ *                     part of the mgmerge utility
+ * Copyright (C) 1995 Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edited:  November 11 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "text.h"
+
+
+/**** GLOBALS ****/
+FILE *text[3], *idx[3];
+
+typedef char FileName[256];
+FileName old_name, new_name, merge_name;
+
+long magicsize;			/* == where the header in a file begins */
+
+compressed_text_header cth[3];
+
+/*=======================================================================
+ * init_merge_text(): open files, set up global variables, etc
+ *=======================================================================*/
+int 
+init_merge_text ()
+{
+
+  /* open .text files */
+  text[OLD] = open_file (old_name, TEXT_SUFFIX, "r+b",
+			 MAGIC_TEXT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (text[OLD]);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, text[OLD]);
+
+  text[NEW] = open_file (new_name, TEXT_SUFFIX, "rb+",
+			 MAGIC_TEXT, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, text[NEW]);
+
+  /* open .text.idx files */
+  idx[OLD] = open_file (old_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, idx[OLD]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[OLD].num_of_docs);
+  NTOHD(cth[OLD].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[OLD].num_of_words);
+  NTOHUL(cth[OLD].length_of_longest_doc);
+  NTOHD(cth[OLD].ratio);
+
+  idx[NEW] = open_file (new_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, idx[NEW]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[NEW].num_of_docs);
+  NTOHD(cth[NEW].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[NEW].num_of_words);
+  NTOHUL(cth[NEW].length_of_longest_doc);
+  NTOHD(cth[NEW].ratio);
+
+  idx[MERGE] = create_file (merge_name, TEXT_IDX_SUFFIX, "wb",
+			    MAGIC_TEXI, MG_ABORT);
+  return OK;
+}
+
+
+/*=======================================================================
+ * process_merge_text(): merge the files
+ *=======================================================================*/
+int 
+process_merge_text (void)
+{
+  int i;
+  u_long data, offset;
+  byte c;
+
+  /* update and write merged header to .text and .text.idx files */
+  /* they have the exact same header */
+  cth[MERGE].num_of_docs = cth[OLD].num_of_docs
+    + cth[NEW].num_of_docs;
+  cth[MERGE].num_of_bytes = cth[OLD].num_of_bytes
+    + cth[NEW].num_of_bytes;
+  cth[MERGE].num_of_words = cth[OLD].num_of_words;
+  cth[MERGE].length_of_longest_doc =
+    (cth[OLD].length_of_longest_doc > cth[NEW].length_of_longest_doc
+     ? cth[OLD].length_of_longest_doc
+     : cth[NEW].length_of_longest_doc);
+  cth[MERGE].ratio = ((cth[OLD].num_of_bytes * cth[OLD].ratio) +
+		      (cth[NEW].num_of_bytes * cth[NEW].ratio))
+    / cth[MERGE].num_of_bytes;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth[MERGE].num_of_docs);
+  HTOND(cth[MERGE].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth[MERGE].num_of_words);
+  HTONUL(cth[MERGE].length_of_longest_doc);
+  HTOND(cth[MERGE].ratio);
+
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, idx[MERGE]);
+  fseek (text[OLD], magicsize, 0);
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, text[OLD]);
+
+  /*
+   *  Update *.text.idx:  need to know where each new doc starts
+   *  in the appended .text file
+   */
+  for (i = 0; i < cth[OLD].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[OLD]);
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+
+  /* offset is the amount to add to each entry from idx[NEW] */
+  fread (&offset, sizeof (u_long), 1, idx[OLD]);
+  NTOHUL(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  offset -= (4 + sizeof (cth[OLD]));	/* 4 for the magic number */
+
+  for (i = 0; i < cth[NEW].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[NEW]);
+      NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      data += offset;
+      HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+  /* write last u_long in idx[MERGE] (= length of file) */
+  fread (&data, sizeof (u_long), 1, idx[NEW]);
+  NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  data += offset;
+  HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+
+/******* update .text *******/
+  /* simply cat's the files together, except for the headers 
+   * and magic numbers, of course
+   */
+  fseek (text[OLD], 0L, 2);
+  while (!feof (text[NEW]))
+    {
+      fread (&c, sizeof (c), 1, text[NEW]);
+      if (!feof (text[NEW]))
+	fwrite (&c, sizeof (c), 1, text[OLD]);
+    }
+
+  return OK;
+}
+
+
+
+/*=======================================================================
+ * done_merge_text(): close files.
+ *=======================================================================*/
+int 
+done_merge_text (void)
+{
+  fclose (idx[MERGE]);
+  fclose (text[OLD]);
+  fclose (idx[OLD]);
+  fclose (text[NEW]);
+  fclose (idx[NEW]);
+
+  fprintf (stderr, "mg_text_merge: %ld documents added to %s\n",
+	   cth[NEW].num_of_docs, merge_name);
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s -f collection_name\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int main (int argc, char *argv[])
+{
+  char *progname;
+  ProgTime start;
+  int ch;			/* for command line processing */
+
+  progname = argv[0];
+  msg_prefix = argv[0];
+  merge_name[0] = '\0';
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (merge_name, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (merge_name[0] == '\0')
+    usage (progname);
+  strcpy (old_name, merge_name);
+  strcat (old_name, ".old");
+  strcpy (new_name, merge_name);
+  strcat (new_name, ".new");
+
+  GetTime (&start);
+  init_merge_text ();
+  process_merge_text ();
+  done_merge_text ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mg_weights_build.c	(revision 21112)
@@ -0,0 +1,514 @@
+/**************************************************************************
+ *
+ * mg_weights_build.c -- Program to build the document weights file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.2  2004/05/24 21:12:18  kjdon
+   changed a message
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:16  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:22  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:05  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:00  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:55  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+unsigned char bits = 8;
+static char *file_name = "";
+static char *text_file_name = "";
+static unsigned long NumPara = 0;
+static unsigned long StaticNumOfDocs = 0;
+
+unsigned long get_NumPara (void);
+unsigned long get_StaticNumOfDocs (void);
+void GenerateWeights (void);
+void Make_weight_approx (void);
+void Make_text_idx_wgt (void);
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:t:d:b:sh")) != -1) /* [RJM 10/98 - Text Filename] */
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	if (strlen(text_file_name) == 0) text_file_name = optarg;
+	break;
+      /* [RJM 10/98 - Text Filename] */
+      case 't':		/* text input file */
+	text_file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	bits = atoi (optarg);
+	if (bits > 32)
+	  {
+	    fprintf (stderr, "b may only take values 0-32\n");
+	    exit (1);
+	  }
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b bits] [-s] [-h]\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  GenerateWeights ();
+
+  Make_weight_approx ();
+
+  Make_text_idx_wgt ();
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+
+  return 0;
+}
+
+
+
+
+unsigned long 
+get_NumPara (void)
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (NumPara)
+    return (NumPara);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.num_of_docs, NumPara);  /* [RPAP - Jan 97: Endian Ordering] */
+  return NumPara;
+}
+
+
+
+unsigned long 
+get_StaticNumOfDocs (void)
+/* the static number of documents is the N parameter used to
+ * decode document gaps in the inverted file encoded using
+ * the Bblock method.
+ */
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (StaticNumOfDocs)
+    return (StaticNumOfDocs);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.static_num_of_docs, StaticNumOfDocs);  /* [RPAP - Jan 97: Endian Ordering] */
+  return StaticNumOfDocs;
+}
+
+
+
+void GenerateWeights (void) {
+  FILE *dict, *invf, *f, *idx;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;
+  double logN;
+  float *DocWeights;
+
+  /* make sure the globals NumPara and StaticNumOfDocs are loaded */
+  get_NumPara ();
+  get_StaticNumOfDocs ();
+
+  /* check to see if the weights file has already been built */
+  if ((f = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		      MG_CONTINUE)) != NULL) {
+      fclose (f);
+      return;
+  }
+  Message ("The file \"%s.weight\" does not exist.", file_name);
+  Message ("Building the weight data from the file \"%s.invf\".", file_name);
+
+  logN = log ((double) NumPara);
+
+  /* allocate memory for the weights */
+  if (!(DocWeights = Xmalloc (sizeof (float) * (NumPara + 1))))
+      FatalError (1, "No memory for doc weights");
+  bzero ((char *) DocWeights, sizeof (float) * (NumPara + 1));
+
+  /* open the .invf.dict file and read in its header */
+  dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+		    MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  /* open .invf.idx */
+  idx = open_file (file_name, INVF_IDX_SUFFIX, "rb", MAGIC_INVI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* open .invf and read in its header */
+  invf = open_file (file_name, INVF_SUFFIX, "rb", MAGIC_INVF,
+		    MG_ABORT);
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i <= 15; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  /* make sure the inverted file does not contain skips and is not level 1 */
+  if (ifh.skip_mode != 0)
+    FatalError (0, "Can\'t make weights file from a skipped inverted file.");
+  if (ifh.InvfLevel == 1)
+    FatalError (0, "Can\'t make weights file from level 1 inverted file.");
+
+  DECODE_START (invf)
+
+    /* process each word adding its contributions to the document weights */
+    for (i = 0; i < ifh.no_of_words; i++)
+    {
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+      unsigned long fcnt, wcnt, blk, CurrDoc, p, j;
+      float idf;
+
+      /* give a little feedback every 4096 words */
+      if ((i & 0xfff) == 0)
+	fprintf (stderr, ".");
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      dummy2[dummy1] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      idf = logN - log ((double) fcnt);
+      blk = BIO_Bblock_Init (StaticNumOfDocs, p);
+      CurrDoc = 0;
+
+      /* check the inverted file index entry for this word */
+      {
+	unsigned long loc;
+	fread ((char *) &loc, sizeof (loc), 1, idx);
+	NTOHUL(loc);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (ftell (invf) != loc)
+	  {
+	    FatalError (1, "Word %d  %d != %d", i, ftell (invf), loc);
+	  }
+      }
+
+      for (j = 0; j < p; j++)
+	{
+	  unsigned long x, tf;
+	  BBLOCK_DECODE (x, blk);
+	  CurrDoc += x;
+
+	  if (CurrDoc > idh.num_of_docs) {
+	    FatalError (1, "CurrDoc = %d, number of documents = %d", 
+			CurrDoc, idh.num_of_docs);
+	  }	  
+
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      double weight;
+	      GAMMA_DECODE (tf);
+	      weight = tf * idf;
+	      DocWeights[CurrDoc - 1] += weight * weight;
+	    }
+	}
+      
+      while (__btg)
+	DECODE_BIT;
+    }
+
+  DECODE_DONE
+
+  fclose (dict);
+  fclose (invf);
+  fprintf (stderr, "\n");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < NumPara; i++)
+    HTONF(DocWeights[i]);
+
+  f = create_file (file_name, WEIGHTS_SUFFIX, "wb", MAGIC_WGHT,
+		   MG_ABORT);
+
+  fwrite ((char *) DocWeights, sizeof (float), NumPara, f);
+  fclose (f);
+  Xfree (DocWeights);
+}
+
+
+
+
+
+
+
+
+
+
+
+void 
+Make_weight_approx (void)
+{
+  int i, pos, max;
+  unsigned long buf;
+  double U, L, B;
+  FILE *approx, *exact;
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* calculate U and L */
+  L = 1e300;
+  U = 0;
+  for (i = 0; i < NumPara; i++)
+    {
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt > U)
+	U = wgt;
+      if (wgt > 0 && wgt < L)
+	L = wgt;
+
+    }
+  fseek (exact, sizeof (u_long), SEEK_SET);
+
+  B = pow (U / L, pow (2.0, -(double) bits));
+
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+
+
+
+  approx = create_file (file_name, APPROX_WEIGHTS_SUFFIX, "wb",
+			MAGIC_WGHT_APPROX, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fwrite ((char *) &bits, sizeof (bits), 1, approx);
+  HTOND(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  HTOND(B);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &L, sizeof (L), 1, approx);
+  fwrite ((char *) &B, sizeof (B), 1, approx);
+  NTOHD(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHD(B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  max = bits == 32 ? 0xffffffff : (1 << bits) - 1;
+  for (buf = pos = i = 0; i < NumPara; i++)
+    {
+      unsigned long fx;
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt == 0)
+	{
+	  wgt = L;
+#ifndef QUIET
+	  Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+	}
+      fx = (int) floor (log (wgt / L) / log (B));
+
+      if (fx > max)
+	fx = max;
+
+      buf |= (fx << pos);
+      pos += bits;
+
+      if (pos >= MAXBITS)
+	{
+	  HTONUL(buf);
+	  fwrite ((char *) &buf, sizeof (buf), 1, approx);
+	  buf = fx >> (bits - (pos - MAXBITS));
+	  pos = pos - MAXBITS;
+	}
+    }
+  if (pos > 0)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(buf);
+      fwrite ((char *) &buf, sizeof (buf), 1, approx);
+    }
+
+  fclose (approx);
+  fclose (exact);
+}
+
+
+
+
+
+void 
+Make_text_idx_wgt (void)
+{
+  compressed_text_header cth;
+  int i;
+  FILE *idx_wgt, *idx, *para, *exact;
+
+  idx_wgt = create_file (file_name, TEXT_IDX_WGT_SUFFIX, "wb", MAGIC_TEXI_WGT,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* [RJM 10/98 - Text Filename] */
+  idx = open_file (text_file_name, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (fread (&cth, sizeof (cth), 1, idx) != 1)
+    FatalError (1, "Unable to read header of index file");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  get_NumPara ();
+  if (cth.num_of_docs != NumPara)
+    {
+      Message ("The number of documents %d does not equal "
+	       "the number of paragraphs %d.", cth.num_of_docs, NumPara);
+      Message ("Using the \"%s.invf.paragraph\" file\n", file_name);
+      para = open_file (file_name, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH,
+			MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+    }
+  else
+    para = NULL;
+
+  {
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+    data;
+    for (i = 0; i < cth.num_of_docs; i++)
+      {
+	int count;
+	fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+	if (para && i < cth.num_of_docs)
+	  {
+	    /* [RPAP - Jan 97: Endian Ordering] */
+	    fread ((char *) &count, sizeof (count), 1, para);
+	    NTOHSI(count);
+	  }
+	else
+	  count = 1;
+	while (count--)
+	  {
+	    fread ((char *) &data.Weight, sizeof (float), 1, exact);
+	    NTOHF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    data.Weight = sqrt (data.Weight);
+	    HTONF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    fwrite ((char *) &data, sizeof (data), 1, idx_wgt);
+	  }
+      }
+    /* Write out the extra entry for the idx file */
+    fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+    data.Weight = 0;
+    fwrite((char*)&data, sizeof(data), 1, idx_wgt);
+  }
+
+  fclose (idx_wgt);
+  fclose (idx);
+  fclose (exact);
+  if (para)
+    fclose (para);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgdictlist.c	(revision 21112)
@@ -0,0 +1,408 @@
+/**************************************************************************
+ *
+ * mgdictlist.c -- Program to list a dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "text.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:17  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:24  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:07  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:01  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:56  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int quick = 0;
+int no_of_words[2];
+u_long maxcodelen[2];
+
+char *dictname = "";
+
+
+
+
+void 
+DumpStemDict (FILE * f)
+{
+  struct invf_dict_header idh;
+  int i;
+  u_char prev[MAXSTEMLEN + 1];
+
+  fread (&idh, sizeof (idh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (quick)
+    printf ("%ld\n", idh.dict_size);
+  else
+    {
+      printf ("# lookback           = %lu\n", idh.lookback);
+      printf ("# dict size          = %lu\n", idh.dict_size);
+      printf ("# total bytes        = %lu\n", idh.total_bytes);
+      printf ("# index string bytes = %lu\n", idh.index_string_bytes);
+      printf ("# input bytes        = %.0f\n", idh.input_bytes); /* [RJM 07/97: 4G limit] */
+      printf ("# num of docs        = %lu\n", idh.num_of_docs);
+      printf ("# static num of docs = %lu\n", idh.static_num_of_docs);
+      printf ("# num of words       = %lu\n", idh.num_of_words);
+      printf ("#\n");
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (f);
+      suff = getc (f);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, f);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, f);
+      fread (&wcnt, sizeof (wcnt), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (!quick)
+	{
+	  printf ("%d: %8ld ", i, wcnt);
+	  printf ("/ %5ld ", fcnt);
+	  printf ("%2d %2ld\t\"", *prev, copy);
+	}
+      printf ("%s", word2str (prev));
+      if (quick)
+	printf (" %ld %ld\n", wcnt, fcnt);
+      else
+	{
+	  putchar ('"');
+	  putchar ('\n');
+	}
+    }
+}
+
+
+
+
+void 
+ReadInWords (FILE * f)
+{
+  comp_frags_header cfh;
+  u_long *codes;
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+
+  if (Read_cfh (f, &cfh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read in the dictionary");
+
+  printf ("#\n");
+  printf ("#   max code len       = %u\n", cfh.hd.maxcodelen);
+  printf ("#   total bytes        = %lu\n", cfh.uncompressed_size);
+  printf ("#\n");
+
+  if (!(codes = Generate_Huffman_Codes (&cfh.hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  for (i = 0; i < cfh.hd.num_codes; i++)
+    {
+      register int val, copy, j, k;
+      char code[33];
+      val = fgetc (f);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (prev + copy + 1, sizeof (u_char), val, f);
+      *prev = val + copy;
+
+      for (k = 0, j = cfh.hd.clens[i] - 1; j >= 0; j--, k++)
+	code[k] = '0' + ((codes[i] >> j) & 1);
+      code[k] = '\0';
+
+      printf ("%d: %2d : %*s : \"%s\"\n", i, cfh.hd.clens[i],
+	      cfh.hd.maxcodelen, code, word2str (prev));
+    }
+  Xfree (codes);
+  Xfree (cfh.hd.clens);
+}
+
+
+void 
+ReadCharHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : \"%s\"\n", hd.clens[i],
+		hd.maxcodelen, code, char2str (i));
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+void 
+ReadLenHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : %d\n", hd.clens[i],
+		hd.maxcodelen, code, i);
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+
+
+
+void 
+DumpTextDict (FILE * f)
+{
+  struct compression_dict_header cdh;
+  int which;
+
+  if (Read_cdh (f, &cdh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read dictionary header");
+  switch (cdh.dict_type)
+    {
+    case MG_COMPLETE_DICTIONARY:
+      printf ("# COMPLETE DICTIONARY\n");
+      break;
+    case MG_PARTIAL_DICTIONARY:
+      printf ("# PARTIAL DICTIONARY\n");
+      break;
+    case MG_SEED_DICTIONARY:
+      printf ("# SEED DICTIONARY\n");
+      break;
+    }
+  printf ("# num words          = %lu\n", cdh.num_words[1]);
+  printf ("# num word chars     = %lu\n", cdh.num_word_chars[1]);
+  printf ("# num non-words      = %lu\n", cdh.num_words[0]);
+  printf ("# num non-word chars = %lu\n", cdh.num_word_chars[0]);
+  printf ("# lookback           = %lu\n", cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  ReadInWords (f);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      }
+}
+
+
+
+
+void 
+DumpStatsDict (FILE * f)
+{
+  int i;
+  compression_stats_header csh;
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  for (i = 0; i < 2; i++)
+    {
+      int j;
+      frags_stats_header fsh;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      if (!quick)
+	printf ("#\n# num %9s      = %lu\n#\n", i ? "words" : "non-words",
+		fsh.num_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  u_char Word[16];
+	  u_long freq, occur_num;
+
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHUL(freq);
+	  NTOHUL(occur_num);
+
+	  Word[0] = fgetc (f);
+	  fread (Word + 1, Word[0], 1, f);
+	  printf ("%d: %7ld : %7ld : \"%s\"\n", j, freq,
+		  occur_num, word2str (Word));
+	}
+    }
+}
+
+
+int main (int argc, char **argv)
+{
+  FILE *fp;
+  unsigned long magic = 0;
+
+  if (argc < 2)
+    FatalError (1, "A file name must be specified");
+  dictname = argv[1];
+  if (strcmp (dictname, "-q") == 0)
+    {
+      quick = 1;
+      if (argc < 3)
+	FatalError (1, "A file name must be specified");
+      dictname = argv[2];
+    }
+  if (!(fp = fopen (dictname, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\"", dictname);
+
+  fread (&magic, sizeof (magic), 1, fp);
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  switch (magic)
+    {
+    case MAGIC_STEM_BUILD:
+      if (!quick)
+	printf ("# Contents of STEM file \"%s\"\n#\n", dictname);
+      DumpStemDict (fp);
+      break;
+    case MAGIC_DICT:
+      if (!quick)
+	printf ("# Contents of DICT file \"%s\"\n#\n", dictname);
+      DumpTextDict (fp);
+      break;
+    case MAGIC_STATS_DICT:
+      if (!quick)
+	printf ("# Contents of STATS file \"%s\"\n#\n", dictname);
+      DumpStatsDict (fp);
+      break;
+    default:
+      FatalError (1, "Bad magic number. \"%s\" cannot be dumped", dictname);
+    }
+  fclose (fp);
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgquery.c	(revision 21112)
@@ -0,0 +1,1502 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/06/22 04:17:51  kjdon
+   fixed a couple of compiler warnings
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+ 
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+
+
+#include <stdarg.h>
+#include <signal.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F) 
+    {
+      if ((int) F != -1)
+	{
+	  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	    Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		       (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		       F->Current.NumReads);
+	  NumBytes += F->Current.NumBytes;
+	  NumSeeks += F->Current.NumSeeks;
+	  NumReads += F->Current.NumReads;
+	}
+      else
+	{
+	  Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		     (double) NumBytes / 1024, NumSeeks, NumReads);
+	  NumSeeks = NumReads = NumBytes = 0;
+	}
+    }
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+      if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+	{
+	  char *s;
+	  s = re_comp (post_proc);
+	  if (!s)
+	    {
+	      Xfree (post_proc);
+	      post_proc = NULL;
+	    }
+	}
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  if (verbatim) {
+    return (strstr (UDoc, post_proc) != NULL);
+  }
+  return re_exec ((char *) UDoc);
+}
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static u_long last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{
+	  /* load the compressed text */
+	  if (LoadCompressedText (qd, max_buf))
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+
+	  /* uncompress the loaded text */
+	  UDoc = GetDocText (qd, &ULen);
+	  if (UDoc == NULL)
+	    FatalError (1, "UDoc is unexpectedly NULL");
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->Weight);
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%7d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->Weight);
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->Weight);
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->num;
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      ConvertTermsToString (qd->TL, terms_str);
+      pager = Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      ConvertTermsToString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->num > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->num;
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->num,
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->num == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+  start_up_stats (qd, iqt);
+
+
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					     (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	    else
+	      BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFREG) != 0)
+	{
+	  /* The name is a directory */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    {
+      query ();
+    }
+  else
+    {
+      int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstat.c	(revision 21112)
@@ -0,0 +1,406 @@
+/**************************************************************************
+ *
+ * mgstat.c -- Program to generate statistics on a text collection
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "mg.h"
+#include "words.h"
+#include "invf.h"
+#include "text.h"
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:18:19  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:59  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+char *search_for_collection (char *name);
+int process_file (char *, char *, int);
+void ProcessDict (char *);
+void ProcessStem (char *);
+long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */
+
+static mg_ullong inputbytes = 0;
+static mg_ullong total = 0;
+
+
+
+
+
+int main (int argc, char **argv)
+{
+  mg_ullong sub_total;
+  int fast;
+  char *file_name = "";
+  int ch;
+  int exact = 0;
+  long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
+    switch (ch)
+      {
+      case 'E':
+	exact = 1;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    file_name = search_for_collection (argv[optind]);
+
+  ProcessDict (file_name);
+  ProcessStem (file_name);
+  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */
+
+  {
+    char Name[256];
+    char *s;
+    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
+    s = strrchr (Name, '/');
+    if (s)
+      *s = '\0';
+    printf ("\nThe collection is in \"%s\"\n", Name);
+  }
+
+
+  printf ("\n\t\tFiles required by mgquery\n");
+  process_file (file_name, TEXT_SUFFIX, exact);
+  process_file (file_name, INVF_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
+  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (indexed)
+    {
+      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
+    }
+
+  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
+  if (!fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+
+
+  process_file (NULL, "SUB TOTAL", exact);
+  sub_total = total;
+  total = 0;
+
+  printf ("\n\t\tFiles NOT required by mgquery\n");
+  if (fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+  process_file (file_name, INVF_DICT_SUFFIX, exact);
+  process_file (file_name, INVF_IDX_SUFFIX, exact);
+  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_SUFFIX, exact);
+  process_file (file_name, WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
+  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
+  process_file (NULL, "SUB TOTAL", exact);
+  total += sub_total;
+  printf ("\n");
+  process_file (NULL, "TOTAL", exact);
+  return 0;
+
+}
+
+
+
+char *
+search_for_collection (char *name)
+{
+  char *dir = get_basepath ();
+  static char buffer[512];
+  struct stat stat_buf;
+
+  /* Look in the current directory first */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  set_basepath (".");
+	  return buffer;
+	}
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  set_basepath (".");
+	  return name;
+	}
+    }
+  sprintf (buffer, "%s/%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  return buffer;
+	}
+    }
+  return name;
+}
+
+
+
+
+
+
+void 
+ProcessDict (char *name)
+{
+  FILE *f;
+  compression_dict_header cdh;
+  int have_cdh = 0;
+  compressed_text_header cth;
+  int have_cth = 0;
+
+  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Read_cdh (f, &cdh, NULL, NULL);
+      fclose (f);
+      have_cdh = 1;
+    }
+
+  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fread ((char *) &cth, sizeof (cth), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(cth.num_of_docs);
+      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+      NTOHUL(cth.num_of_words);
+      NTOHUL(cth.length_of_longest_doc);
+      NTOHD(cth.ratio);
+
+      fclose (f);
+      have_cth = 1;
+    }
+
+  if (have_cth)
+    {
+      inputbytes = cth.num_of_bytes;
+      printf ("Input bytes                        : %10" ULL_FS ", %8.2f Mbyte\n",
+	      cth.num_of_bytes, (double) cth.num_of_bytes / 1024 / 1024);
+      printf ("Documents                          : %10lu\n", cth.num_of_docs);
+      printf ("Words in collection [dict]         : %10lu\n", cth.num_of_words);
+      printf ("Longest doc in collection [dict]   : %10lu characters\n",
+	      cth.length_of_longest_doc);
+      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
+    }
+
+  if (have_cdh)
+    {
+      printf ("Words in dict                      : %10lu\n", cdh.num_words[1]);
+      printf ("Non-words in dict                  : %10lu\n", cdh.num_words[0]);
+      printf ("Total chars of distinct words      : %10lu\n", cdh.num_word_chars[1]);
+      printf ("Total chars of distinct non-words  : %10lu\n", cdh.num_word_chars[0]);
+    }
+
+}
+
+
+
+
+
+
+
+
+void 
+ProcessStem (char *name)
+{
+  FILE *f;
+  struct invf_dict_header idh;
+
+  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
+		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+  fread ((char *) &idh, sizeof (idh), 1, f);
+  printf ("Words in collection [stem]         : %10ld\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Words in stem                      : %10ld\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Indexed fragments                  : %10ld\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Total chars of stem words          : %10ld\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
+  fclose (f);
+}
+
+
+
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+long 
+ProcessStemBlk (char *name)
+{
+  FILE *f;
+  struct stem_dict_header sdh;
+
+  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
+		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return 0;
+  fread ((char *) &sdh, sizeof (sdh), 1, f);
+  fclose (f);
+  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+process_file (char *name, char *ext, int exact)
+{
+  static double scale = 0;
+  static char *units;
+  struct stat buf;
+  if (scale == 0)
+    {
+      /* So can get output in Mb or Kb, */
+      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
+      /* Note: if inputbytes==0, then use 1024 as default */
+      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
+      units = scale == 1024 ? "Kb" : "Mb";
+    }
+  if (name)
+    {
+      char Name[256];
+      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
+      if (!stat (Name, &buf))
+	{
+	  char fname[256];
+	  char *nam = strrchr (name, '/');
+	  nam = nam ? nam + 1 : name;
+	  sprintf (fname, "%s%s", nam, ext);
+
+	  if (inputbytes == 0.0)
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size,
+			100.0 * buf.st_size / inputbytes);
+	      else
+		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units,
+			100.0 * buf.st_size / inputbytes);
+	    }
+	  else
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size);
+	      else
+		printf ("%s%*s: %8.2f %s\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units);
+	    }
+	  total += buf.st_size;
+	  return 1;
+	}
+      else
+	return 0;
+    }
+  else
+    {
+      if (inputbytes == 0.0)
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes   %7.3f%%\n", ext,
+		    total,
+		    100.0 * total / inputbytes);
+	  else
+	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
+		    total / scale, units,
+		    100.0 * total / inputbytes);
+	}
+      else
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes\n", ext, total);
+	  else
+	    printf ("%-34s : %8.2f %s\n", ext,
+		    total / scale, units);
+	}
+    }
+  return 1;
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/mgstemidxlist.c	(revision 21112)
@@ -0,0 +1,245 @@
+/**************************************************************************
+ *
+ * mgstemidxlist.c -- Text dumper for the stem indexes
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+
+void
+read_3_in_4 (FILE * idbi)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+  int block = 0;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      return;
+    }
+
+  si->MemForStemIdx = 0;
+
+  fread (&(si->sih), sizeof (si->sih), 1, idbi);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = fgetc (idbi);
+      *buffer++ = len;
+      fread (buffer, sizeof (u_char), len, idbi);
+      buffer += len;
+      fread (&si->pos[i], sizeof (*si->pos), 1, idbi);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  printf ("# lookback     = %lu\n", si->sih.lookback);
+  printf ("# block_size   = %lu\n", si->sih.block_size);
+  printf ("# num_blocks   = %lu\n", si->sih.num_blocks);
+  printf ("# blocks_start = %lu\n", si->sih.blocks_start);
+  printf ("# index_chars  = %lu\n", si->sih.index_chars);
+  printf ("# num_of_words = %lu\n", si->sih.num_of_words);
+
+  block = 0;
+  while (block < si->sih.num_blocks)
+    {
+      unsigned long *first_word;
+      unsigned short *num_words;
+      unsigned short *index;
+      long res;
+      u_char *base;
+      int num_indexes;
+
+      /* Read in next block */
+      fseek (idbi, si->pos[block] + si->sih.blocks_start, 0);
+      fread (si->buffer, si->sih.block_size, sizeof (u_char), idbi);
+      si->active = si->pos[block];
+
+      first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (first_word + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+
+      base = (u_char *) (index + num_indexes);
+      base += index[0];
+
+      printf ("\n# block      = %d\n", block);
+      printf ("# first_word = %lu\n", *first_word);
+      printf ("# num_words  = %u\n", *num_words);
+
+      res = 0;
+      while (res < *num_words)
+	{
+	  unsigned copy, suff;
+	  u_char prev[MAXSTEMLEN + 1];
+	  unsigned int num_entries, num_cases, blk;
+	  unsigned short blk_index, offset;
+
+	  /* Read word entry */
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  *prev = copy + suff;
+	  base += suff;
+	  bcopy ((char *) base, (char *) &num_entries, sizeof (num_entries));
+	  base += sizeof (num_entries);
+	  NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+	  printf ("%u \"%s\"\n", num_entries, word2str (prev));
+
+	  /* For all the PosEntries for the word... */
+	  for (i = 0; i < num_entries; i++)
+	    {
+	      bcopy ((char *) base, (char *) &num_cases, sizeof (num_cases));
+	      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (num_cases);
+	      bcopy ((char *) base, (char *) &blk, sizeof (blk));
+	      NTOHUI(blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk);
+	      bcopy ((char *) base, (char *) &blk_index, sizeof (blk_index));
+	      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk_index);
+	      bcopy ((char *) base, (char *) &offset, sizeof (offset));
+	      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (offset);
+
+	      printf ("    -> %4u %4u %4u %4u\n", num_cases, blk, blk_index, offset);
+	    }
+	  res++;
+	}
+      block++;
+    }
+  fclose (idbi);
+}
+
+int
+main (int argc, char **argv)
+{
+  FILE *idbi;
+  char *filename = "";
+  int ch;
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:hs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d data directory] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  /* Open required files */
+  switch (stem_method)
+    {
+    case (1):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "rb", MAGIC_STEM_1,
+			MG_ABORT);
+      break;
+    case (2):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "rb", MAGIC_STEM_2,
+			MG_ABORT);
+      break;
+    case (3):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "rb", MAGIC_STEM_3,
+			MG_ABORT);
+      break;
+    default:
+      FatalError (1, "Stem method must be 1, 2 or 3\n");
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT open file");
+
+  read_3_in_4 (idbi);
+
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.docnums.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.docnums.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.docnums.c	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * query.docnums.c -- Docnums query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "filestats.h"
+#include "timing.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+
+
+static DocList *
+AddToDocList (DocList * Docs, int res)
+{
+  int j;
+  /* Search the list for the word */
+  for (j = 0; j < Docs->num; j++)
+    if (Docs->DE[j].DocNum == res)
+      break;
+
+  /* Increment the weight if the word is in the list */
+  if (j < Docs->num)
+    Docs->DE[j].Weight++;
+  else
+    {
+      /* Create a new entry in the list for the new document */
+      Docs = ResizeDocList (Docs, Docs->num + 1);
+      Docs->DE[Docs->num].DocNum = res;
+      Docs->DE[Docs->num].Weight = 1;
+      Docs->DE[Docs->num].SeekPos = 0;
+      Docs->DE[Docs->num].Len = 0;
+      Docs->DE[Docs->num].CompTextBuffer = NULL;
+      Docs->DE[Docs->num].Next = NULL;
+      Docs->num++;
+    }
+  return (Docs);
+}
+
+void 
+DocnumsQuery (query_data * qd, char *QueryLine)
+{
+  DocList *Docs = MakeDocList (0);
+  unsigned long start = 0, finish;
+  int range = 0; /* is there a range or not ? */
+
+  /* find the start of the first word */
+  while (*QueryLine)
+    {
+      char *end;
+      finish = strtol (QueryLine, &end, 10);
+      if (end != (char *) QueryLine && finish >= 1 &&
+	  finish <= qd->sd->sdh.num_of_docs)
+	{
+	  unsigned long i;
+	  if (!range)
+	    start = finish;
+	  else
+	    range = 0;
+	  for (i = start; i <= finish; i++)
+	    Docs = AddToDocList (Docs, i);
+	  while (*end == ' ' || *end == '\t' || *end == '\n')
+	    end++;
+	  if (*end == '-')
+	    {
+	      start = finish;
+	      range = 1;
+	      end++;
+	    }
+	}
+      else
+	break;
+      QueryLine = end;
+    }
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.ranked.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.ranked.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/query.ranked.c	(revision 21112)
@@ -0,0 +1,994 @@
+/**************************************************************************
+ *
+ * query.ranked.c -- Ranked query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "messages.h"
+#include "timing.h"
+#include "sptree.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "weights.h"
+#include "text_get.h"
+#include "invf_get.h"
+#include "words.h"
+#include "stemmer.h"
+#include "locallib.h"
+#include "environment.h"
+#include "term_lists.h"
+#include "local_strings.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:20  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:46  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:03  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:04  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/*************************************************************************/
+
+typedef struct HeapEntry
+  {
+    float Weight;
+    float *OrgWeight;
+    int DocNum;
+    u_long SeekPos;		/* position in the text file in bytes */
+    u_long Len;			/* length of the document in bytes */
+  }
+HeapEntry;
+
+typedef int (*HeapComp) (HeapEntry *, HeapEntry *);
+
+typedef struct Heap
+  {
+    int NumItems;
+    int MaxSize;
+    HeapComp HC;
+    HeapEntry HE[1];
+  }
+Heap;
+
+
+Heap *
+Heap_Make (int size, HeapComp hc)
+{
+  Heap *H;
+  H = Xmalloc (sizeof (Heap) + (size - 1) * sizeof (HeapEntry));
+  if (!H)
+    return NULL;
+  H->NumItems = 0;
+  H->MaxSize = size;
+  H->HC = hc;
+  return H;
+}
+
+int 
+Heap_Size (Heap * H)
+{
+  return sizeof (Heap) + H->MaxSize * sizeof (HeapEntry);
+}
+
+HeapEntry *
+Heap_GetHead (Heap * H)
+{
+  if (H && H->NumItems)
+    return &H->HE[0];
+  else
+    return NULL;
+}
+
+
+
+void 
+Heap_Heapify (Heap * H, int i)
+{
+  register int curr, child;
+  curr = i;
+  child = curr * 2;
+  while (child <= H->NumItems)
+    {
+      if (child < H->NumItems && H->HC (&H->HE[child], &H->HE[child - 1]) > 0)
+	child++;
+      if (H->HC (&H->HE[curr - 1], &H->HE[child - 1]) < 0)
+	{
+	  HeapEntry temp = H->HE[child - 1];
+	  H->HE[child - 1] = H->HE[curr - 1];
+	  H->HE[curr - 1] = temp;
+	  curr = child;
+	  child = 2 * child;
+	}
+      else
+	break;
+    }
+}
+
+
+void 
+Heap_Build (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems / 2; i > 0; i--)
+    Heap_Heapify (H, i);
+}
+
+
+void 
+Heap_Sort (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems; i > 1; i--)
+    {
+      HeapEntry temp = H->HE[0];
+      H->HE[0] = H->HE[i - 1];
+      H->HE[i - 1] = temp;
+      H->NumItems--;
+      Heap_Heapify (H, 1);
+    }
+}
+
+
+void 
+Heap_DeleteHead (Heap * H)
+{
+  H->HE[0] = H->HE[--H->NumItems];
+  Heap_Heapify (H, 1);
+}
+
+int 
+Heap_Lesser (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? -1 :
+	  (a->Weight == b->Weight ? 0 : 1));
+}
+
+
+int 
+Heap_Greater (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? 1 :
+	  (a->Weight == b->Weight ? 0 : -1));
+}
+
+int 
+Make_Exact_Root (query_data * qd, Heap * H)
+{
+  int num = 0;
+  HeapEntry *he = H->HE;
+  while (he->SeekPos == 0)
+    {
+      he->Weight = he->Weight *
+	GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+      Heap_Heapify (H, 1);
+      num++;
+    }
+  return num;
+}
+
+void 
+Heap_Dump (Heap * H, int num)
+{
+  int i, l, lines, r;
+  if (num > H->NumItems)
+    num = H->NumItems;
+  lines = (num + 3) / 4;
+  for (l = 0; l < lines; l++)
+    for (r = 0; r < 4; r++)
+      {
+	i = lines * r + l;
+	if (i < num)
+	  fprintf (stderr, "[%2d] %7.4f", i, H->HE[i].Weight);
+	fprintf (stderr, r == 3 ? "\n" : "        ");
+      }
+  fprintf (stderr, "\n");
+}
+
+/*************************************************************************/
+
+
+
+
+
+
+int 
+doc_count_comp (const void *A, const void *B)
+{
+  const TermEntry *a = A;
+  const TermEntry *b = B;
+  return (a->WE.doc_count - b->WE.doc_count);
+}
+
+/* =========================================================================
+ * Function: ParseRankedQuery
+ * Description:
+ *      Takes a string query line and extracts the terms in the query
+ *      which exist in the stemmed dictionary.
+ *      Optionally sorts the terms into ascending order by doc. number.
+ * Input:
+ *      stemmed dictionary, query line, sort-flag
+ * Output:
+ *      A list of terms.
+ * ========================================================================= */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+static TermList *
+ParseRankedQuery (stemmed_dict * sd, char *QueryLine, int Sort, int indexed,
+		  QueryTermList **query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  u_char Word[MAXSTEMLEN + 1];
+  u_char sWord[MAXSTEMLEN + 1];
+  u_char *end, *s_in;
+  int default_stem_method = 0;
+  TermList *Terms = MakeTermList(0);
+
+  s_in = (u_char *) QueryLine;
+  end = s_in + strlen ((char *) s_in) - 1;
+  *query_term_list = MakeQueryTermList(0);  /* [RPAP - Feb 97: Term Frequency] */
+
+  if (indexed)
+    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+  else
+    default_stem_method = sd->sdh.stem_method;
+
+  while (s_in <= end)
+    {
+      int j;
+      long num_entries, word_num;
+      unsigned long count, doc_count, invf_ptr, invf_len;
+      int weight_to_apply, stem_to_apply;
+      int method_using = -1;
+
+      /* 0=optional, 1=mustmatch */
+      int require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+      /* Skip over the non word separator taking note of any parameters */
+      PARSE_RANKED_NON_STEM_WORD (require_match, s_in, end); /* [RJM 07/97: Ranked Required Terms] */
+      if (s_in > end) break;
+
+      /* Get a word and stem it */
+      PARSE_STEM_WORD (Word, s_in, end);
+
+      /* Extract any parameters */
+      weight_to_apply = 1;
+      stem_to_apply = default_stem_method;
+      while (s_in <= end)
+	{
+	  int stem_param, weight_param, param_type;
+	  char param[MAXPARAMLEN + 1];
+
+	  param_type = 0;
+	  PARSE_OPT_TERM_PARAM (param, param_type, s_in, end);
+	  if (!param_type)
+	    break;
+
+	  switch (param_type)
+	    {
+	    case (WEIGHTPARAM):
+	      weight_param = atoi (param);
+	      if (errno != ERANGE && weight_param > 0)
+		weight_to_apply = weight_param;
+	      break;
+
+	    case (STEMPARAM):
+	      stem_param = atoi (param);
+	      if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		method_using = stem_to_apply = stem_param;
+	      break;
+	    }
+	}
+
+      bcopy ((char *) Word, (char *) sWord, *Word + 1);
+      stemmer (stem_to_apply, sd->sdh.stemmer_num, sWord);
+
+      if (!indexed || stem_to_apply == 0)
+	{
+	  /* Look for the word in the already identified terms */
+	  for (j = 0; j < Terms->num; j++)
+	    if (compare (Terms->TE[j].Word, Word) == 0)
+	      break;
+
+	  /* Increment the weight if the word is in the list */
+	  /* Update the require match attribute */
+	  if (j < Terms->num)
+	    {
+	      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+				    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+	      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+	      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+	    }
+	  else
+	    {
+	      /* Look for it in the stemmed dictionary */
+	      if ((word_num = FindWord (sd, sWord, &count, &doc_count,
+					&invf_ptr, &invf_len)) != -1)
+		{
+		  /* Search the list for the word */
+		  for (j = 0; j < Terms->num; j++)
+		    if (Terms->TE[j].WE.word_num == word_num)
+		      break;
+
+		  /* Increment the weight if the word is in the list */
+		  if (j < Terms->num)
+		    {
+		      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+					    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+		      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+		      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+		    }
+		  else
+		    {
+		      /* Create a new entry in the list for the new word */
+		      TermEntry te;
+
+		      te.WE.word_num = word_num;
+		      te.WE.count = count;
+		      te.WE.doc_count = doc_count;
+		      te.WE.max_doc_count = doc_count;
+		      te.WE.invf_ptr = invf_ptr;
+		      te.WE.invf_len = invf_len;
+		      te.Count = weight_to_apply;
+		      te.Word = copy_string (Word);
+		      if (!te.Word)
+			FatalError (1, "Could NOT create memory to add term");
+		      te.Stem = NULL;
+		      te.require_match = require_match;
+
+		      AddTermEntry (&Terms, &te);
+
+		      /* [RPAP - Feb 97: Term Frequency] */
+		      AddQueryTerm (query_term_list, Word, count, method_using);
+		    }
+		}
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      else
+		AddQueryTerm (query_term_list, Word, 0, method_using);
+	    }
+	}
+      else
+	{
+	  int total_count = 0;  /* [RPAP - Feb 97: Term Frequency] */
+	  TermList *tempList = MakeTermList (0);
+	  if ((num_entries = FindWords (sd, sWord, stem_to_apply, &tempList)) > 0)
+	    {
+	      int i;
+	      u_long max_doc_count = 0;
+
+	      /* get the maximum doc count */
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  if (tempList->TE[i].WE.doc_count > max_doc_count)
+		    max_doc_count = tempList->TE[i].WE.doc_count;
+		  total_count += tempList->TE[i].WE.count;  /* [RPAP - Feb 97: Term Frequency] */
+		}
+
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  /* Look for the word(s) in the already identified terms */
+		  for (j = 0; j < Terms->num; j++) 
+		    {
+		      if (compare (Terms->TE[j].Word, tempList->TE[i].Word) == 0)
+			{
+			  /* found the word */
+			  /* Modify weight */
+			  Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+						INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+			  if (Terms->TE[j].WE.max_doc_count < max_doc_count)
+			    Terms->TE[j].WE.max_doc_count = max_doc_count;
+			  break;
+			}
+		    }
+
+		  if (j == Terms->num)
+		    {
+		      /* word was not found */
+		      tempList->TE[i].WE.max_doc_count = max_doc_count;
+		      tempList->TE[i].Count = weight_to_apply;
+
+		      /* We cannot require a term to match if it is expanded */
+		      /* into multiple terms :-( */
+		      tempList->TE[i].require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+		      AddTermEntry (&Terms, &tempList->TE[i]);
+		    }
+		}
+	    }
+	  /* [RPAP - Feb 97: Term Frequency] */
+	  AddQueryTerm (query_term_list, Word, total_count, method_using);
+
+	  if (tempList != NULL) Xfree(tempList); /* [RJM 07/98: Memory Leak] */
+	} /* end indexed */
+    } /* end while */
+
+  if (Sort)
+    /* Sort the terms in ascending order by doc_count */
+    qsort (Terms->TE, Terms->num, sizeof (TermEntry), doc_count_comp);
+  return (Terms);
+}
+
+
+
+
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+
+
+
+
+/*
+ * This function is given a list of term numbers and it returns a list
+ * of document numbers based on the cosine document weighting system.
+ * This puts the entries in an array.
+ * inverted file.
+ * If MaxDocs == -1 then it means all 
+ */
+static DocList *
+CosineGet (query_data * qd, TermList * Terms, RankedQueryInfo * rqi) {
+  DocList *Docs;
+  float *AccumulatedWeights = NULL;
+  Splay_Tree *ST = NULL;
+  Splay_Tree *Para_ST = NULL;
+  Hash_Table *HT = NULL;
+  List_Table *LT = NULL;
+  Heap *H;
+  HeapEntry *he;
+  register float *fptr = NULL;
+  register Invf_Doc_Entry *ide = NULL;
+  register Invf_Doc_EntryH *ideh = NULL;
+  int BackEnd, NumExact, MaxExact, NumParas;
+  int MaxDocs = 0, MaxParas = 0;
+  int i;
+  Invf_Doc_Entry_Pool ide_pool;
+  ide_pool.pool = NULL;
+
+  qd->hops_taken = qd->num_of_ptrs = qd->num_of_accum = 0;
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      ST = CosineDecodeSplay (qd, Terms, rqi, &ide_pool);
+      if (!ST)
+	return NULL;
+      break;
+    case 'A':
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      if (!AccumulatedWeights)
+	return NULL;
+      break;
+    case 'H':
+      HT = CosineDecodeHash (qd, Terms, rqi);
+      if (!HT)
+	return NULL;
+      break;
+    case 'L':
+      LT = CosineDecodeList (qd, Terms, rqi);
+      if (!LT)
+	return NULL;
+      break;
+    }
+
+#if 0
+  if (rqi->UseSplayTree)
+    {
+
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      fptr = AccumulatedWeights;
+      ide = SP_get_first (ST);
+      for (i = 0; i < qd->sd->sdh.num_of_docs; i++)
+	{
+	  if (AccumulatedWeights[i] != 0)
+	    {
+	      if (i != ide->DocNum)
+		fprintf (stderr, "Sum mismatch for %d %f %d %f\n", i + 1,
+			 AccumulatedWeights[i], ide->DocNum + 1, ide->Sum);
+	      ide = SP_get_next (ST);
+	    }
+	}
+    }
+#endif
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      MaxParas = ST->no_of_items;
+      break;
+    case 'A':
+      {				/* count the number of non-zero document weights */
+	register int i = qd->sd->sdh.num_of_docs;
+	register float *d;
+	MaxParas = 0;
+	for (d = AccumulatedWeights; i; i--, d++)
+	  if (*d)
+	    MaxParas++;
+      }
+      break;
+    case 'H':
+      MaxParas = HT->num + HT->Suplimentary_Num;
+      break;
+    case 'L':
+      MaxParas = LT->num;
+      break;
+    }
+
+  if (rqi->MaxParasToRetrieve != -1 && MaxParas > rqi->MaxParasToRetrieve)
+    MaxParas = rqi->MaxParasToRetrieve;
+  MaxDocs = MaxParas;
+
+  /* Allocate memory for the heap */
+  Docs = MakeDocList (MaxDocs);
+  ChangeMemInUse (qd, sizeof (DocEntry) * MaxDocs);
+
+  H = Heap_Make (MaxDocs, Heap_Lesser);
+
+
+  /* Get the sums from the array divide the sums by the 
+     document weights which we retrieve from the ".idx.wgt" file and put
+     the resulting data into a heap */
+
+
+  he = H->HE;
+  H->NumItems = MaxDocs;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	ide = SP_get_first (ST);
+	for (i = 0; i < H->NumItems; i++, ide = SP_get_next (ST), he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'A':
+      {
+	fptr = AccumulatedWeights;
+	for (i = 0; i < H->NumItems; i++, fptr++, he++)
+	  {
+	    he->DocNum = i + 1;
+	    he->OrgWeight = fptr;
+	    if (*fptr)
+	      qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'H':
+      {
+	ideh = HT->Head;
+	for (i = 0; i < H->NumItems; i++, ideh = ideh->next, he++)
+	  {
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'L':
+      {
+	ide = LT->IDE;
+	for (i = 0; i < H->NumItems; i++, ide++, he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    }
+
+  he = H->HE;
+  for (i = 0; i < H->NumItems; i++, he++)
+    {
+      *he->OrgWeight /= GetLowerApproxDocWeight (qd->awd, he->DocNum - 1);
+      he->Weight = *he->OrgWeight;
+      *he->OrgWeight = 0;
+      he->SeekPos = he->Len = 0;
+    }
+
+  Heap_Build (H);
+
+  he = H->HE;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	for (i = MaxDocs; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	  {
+	    ide->Sum /= GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'A':
+      {
+	for (i = MaxDocs; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	  {
+	    if (!*fptr)
+	      continue;
+	    qd->num_of_accum++;
+	    *fptr /= GetLowerApproxDocWeight (qd->awd, i);
+	    if (*fptr <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = i + 1;
+	    he->Weight = *fptr;
+	    he->OrgWeight = fptr;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'H':
+      {
+	for (; ideh; ideh = ideh->next)
+	  {
+	    qd->num_of_accum++;
+	    ideh->IDE.Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum);
+	    if (ideh->IDE.Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->Weight = ideh->IDE.Sum;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'L':
+      {
+	for (i = MaxDocs; i < LT->num; i++, ide++)
+	  {
+	    qd->num_of_accum++;
+	    ide->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    }
+
+
+  if (rqi->Exact && qd->id->ifh.InvfLevel != 3)
+    {
+      HeapEntry *he = H->HE;
+
+      for (i = 0; i < H->NumItems; i++, he++)
+	{
+	  he->Weight = he->Weight *
+	    GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	    FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+	}
+
+      Heap_Build (H);
+
+      he = H->HE;
+
+      switch (rqi->AccumMethod)
+	{
+	case 'S':
+	  {
+	    ide = SP_get_first (ST);
+	    for (i = 0; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+
+	  /* up to here */
+
+	case 'A':
+	  {
+	    fptr = AccumulatedWeights;
+	    for (i = 0; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!*fptr)
+		  continue;
+		if (*fptr <= he->Weight)
+		  continue;
+		Weight = *fptr *
+		  GetLowerApproxDocWeight (qd->awd, i) /
+		  FetchDocStart (qd, i + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = i + 1;
+		he->OrgWeight = fptr;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		*fptr = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'H':
+	  {
+	    ideh = HT->Head;
+	    for (ideh = HT->Head; ideh; ideh = ideh->next)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ideh->IDE.Sum)
+		  continue;
+		if (ideh->IDE.Sum <= he->Weight)
+		  continue;
+		Weight = ideh->IDE.Sum *
+		  GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum) /
+		  FetchDocStart (qd, ideh->IDE.DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ideh->IDE.DocNum + 1;
+		he->OrgWeight = &ideh->IDE.Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ideh->IDE.Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'L':
+	  {
+	    ide = LT->IDE;
+	    for (i = 0; i < LT->num; i++, ide++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	}
+    }
+
+
+
+  H->HC = Heap_Greater;
+  Heap_Build (H);
+
+
+  MaxDocs = H->NumItems;
+  if (rqi->MaxDocsToRetrieve != -1 && MaxDocs > rqi->MaxDocsToRetrieve)
+    MaxDocs = rqi->MaxDocsToRetrieve;
+
+  /* Alarm */
+
+  he = H->HE;
+  BackEnd = H->NumItems - 1;
+  NumExact = 0;
+  MaxExact = H->NumItems;
+  NumParas = 0;
+  Para_ST = SP_createset (DE_comp);
+  while (H->NumItems && Docs->num < MaxDocs)
+    {
+      DocEntry DocEnt;
+      DocEntry *mem;
+
+      if (rqi->Exact)
+	{
+	  if (H->HE[0].SeekPos == 0)
+	    NumExact += Make_Exact_Root (qd, H);
+	}
+      else
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+
+      NumParas++;
+
+      DocEnt.DocNum = he->DocNum;
+      DocEnt.Weight = he->Weight;
+      DocEnt.Len = he->Len;
+      DocEnt.SeekPos = he->SeekPos;
+      DocEnt.CompTextBuffer = NULL;
+      DocEnt.Next = NULL;
+
+      Heap_DeleteHead (H);
+
+      if (!(mem = SP_member (&DocEnt, Para_ST)))
+	{
+	  Docs->DE[Docs->num] = DocEnt;
+	  SP_insert (&Docs->DE[Docs->num], Para_ST);
+	  Docs->num++;
+	}
+      else
+	{
+	  DocEnt.Next = mem->Next;
+	  Docs->DE[BackEnd] = DocEnt;
+	  mem->Next = &Docs->DE[BackEnd--];
+	}
+    }
+  SP_freeset (Para_ST);
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      Message ("%d Paragraphs were required to get %d documents",
+	       NumParas, Docs->num);
+      if (NumExact == MaxExact)
+	{
+	  Message ("The exact weights of all %d paragraphs had to be calculated", MaxExact);
+	  Message ("to obtain %d paragraphs. This may mean that the the documents", NumParas);
+	  Message ("returned do not necessarly represent an exact cosine ranking.");
+	  Message ("This problem may be corrected by increasing \'maxparas\'.");
+	}
+    }
+#if 0
+  {
+    int i;
+    FILE *f = fopen ("top.paras", "w");
+    fprintf (f, "=========================\nTop Paragraphs\n");
+    for (i = 0; i < Docs->num; i++)
+      {
+	DocEntry *d;
+	fprintf (f, "<%d(%f)>  ", Heap[i].DocNum, Heap[i].Weight);
+	for (d = Heap[i].Next; d; d = d->Next)
+	  fprintf (f, "%d(%f)  ", d->DocNum, d->Weight);
+	fprintf (f, "\n");
+      }
+    fprintf (f, "=========================\n");
+    fclose (f);
+  }
+#endif
+
+  if (AccumulatedWeights)
+    {
+      Xfree (AccumulatedWeights);
+      ChangeMemInUse (qd, -sizeof (float) * qd->sd->sdh.num_of_docs);
+    }
+  if (ST)
+    {
+      int mem = ST->mem_in_use;
+      SP_freeset (ST);
+      ChangeMemInUse (qd, -mem);
+      free_ide_pool (qd, &ide_pool);
+    }
+  if (HT)
+    HT_free (qd, HT);
+
+  if (LT)
+    LT_free (qd, LT);
+
+  if (H) Xfree (H); /* [RJM 07/98: Memory Leak] */
+
+  return (Docs);
+}
+
+
+
+
+/* if MaxDocs == -1 it means all */
+void 
+RankedQuery (query_data *qd, char *Query, RankedQueryInfo *rqi)
+{
+  DocList *dl;
+
+  if (qd->TL)
+    FreeTermList (&(qd->TL));
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  if (qd->QTL)
+    FreeQueryTermList (&(qd->QTL));
+
+  qd->TL = ParseRankedQuery (qd->sd, Query, rqi->Sort, qd->sd->sdh.indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			     &(qd->QTL));  /* [RPAP - Feb 97: Term Frequency] */
+
+  /*  PrintTermList (qd->TL, stderr); */
+
+  dl = CosineGet (qd, qd->TL, rqi);
+
+  if (!dl)
+    FatalError (1, "Out of memory\n");
+
+  FreeQueryDocs (qd);
+
+  qd->DL = dl;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.c	(revision 21112)
@@ -0,0 +1,228 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:36  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "query_term_list.h"
+#include "term_lists.h"  /* for MAXTERMSTRLEN */
+#include "messages.h"
+
+
+/* =========================================================================
+ * Function: MakeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+QueryTermList *
+MakeQueryTermList (int n)
+{
+  QueryTermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (QueryTermList) + (list_size - 1) * sizeof (QueryTermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate query term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeQueryTermList (QueryTermList ** query_list)
+{
+  QueryTermList *qtl = *query_list;
+
+  if (qtl->num > qtl->list_size)
+    {
+      if (qtl->list_size)
+	qtl->list_size *= GROWTH_FACTOR;
+      else
+	qtl->list_size = MIN_SIZE;
+    }
+  qtl = Xrealloc (qtl, sizeof (QueryTermList) + (qtl->list_size - 1) * sizeof (QueryTermEntry));
+
+  if (!qtl)
+    FatalError (1, "Unable to resize query term list");
+
+  *query_list = qtl;
+}
+
+/* =========================================================================
+ * Function: ConvertQueryTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = query list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertQueryTermsToString (QueryTermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->QTE[i].Term;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (query_term_list->QTE[i].stem_method >= 0)
+	total_len += 2;         /* '#' and the stem number */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (query_term_list->QTE[i].stem_method >= 0)
+	{
+	  *str++ = '#';
+	  *str++ = '0' + query_term_list->QTE[i].stem_method;
+	}
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetQueryTermList (QueryTermList ** qtl)
+{
+  if (*qtl)
+    FreeQueryTermList (qtl);
+  *qtl = MakeQueryTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddQueryTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTermEntry (QueryTermList ** query_term_list, QueryTermEntry * qte)
+{
+  QueryTermList *qtl = *query_term_list;
+
+  qtl->num++;
+  ResizeQueryTermList (query_term_list);
+  qtl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) qte, (char *) &(qtl->QTE[qtl->num - 1]), sizeof (QueryTermEntry));
+
+  return qtl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddQueryTerm
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method)
+{
+  /* Create a new entry in the list for the new word */
+  QueryTermEntry qte;
+
+  qte.Count = Count;
+  qte.stem_method = stem_method;
+  qte.Term = copy_string (Term);
+  if (!qte.Term)
+    FatalError (1, "Could NOT create memory to add term");
+
+  return AddQueryTermEntry (query_term_list, &qte);
+}
+
+/* =========================================================================
+ * Function: FreeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeQueryTermList (QueryTermList ** the_qtl)
+{
+  int j;
+  QueryTermList *qtl = *the_qtl;
+
+  for (j = 0; j < qtl->num; j++)
+    if (qtl->QTE[j].Term)
+      Xfree (qtl->QTE[j].Term);
+  Xfree (qtl);
+
+  *the_qtl = NULL;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/query_term_list.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef QUERY_TERM_LIST_H
+#define QUERY_TERM_LIST_H
+
+#include "sysfuncs.h"
+
+typedef struct QueryTermEntry
+  {
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Term;		/* The query term */
+    int stem_method;            /* The steming method to apply to the term : -1 is default */
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+/* --- prototypes --- */
+void ConvertQueryTermsToString (QueryTermList * query_term_list, char *str);
+int AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method);
+void ResetQueryTermList (QueryTermList ** tl);
+void FreeQueryTermList (QueryTermList ** the_qtl);
+QueryTermList *MakeQueryTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.c	(revision 21112)
@@ -0,0 +1,214 @@
+/**************************************************************************
+ *
+ * read_line.c -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#ifdef GNU_READLINE
+#include "readline.h"
+#include "chardefs.h"
+#include "history.h"
+#endif
+
+
+#include "globals.h"
+#include "environment.h"
+#include "read_line.h"
+
+
+
+#ifdef GNU_READLINE
+int rl_bind_key (int, int (*)());
+
+
+void 
+Init_ReadLine (void)
+{
+  /* Make TAB just insert itself rather than do a file name completion */
+  rl_bind_key (TAB, rl_insert);
+}
+#else
+
+static FILE *rl_instream = NULL;
+static FILE *rl_outstream = NULL;
+
+void 
+Init_ReadLine (void)
+{
+  rl_instream = stdin;
+  rl_outstream = stdout;
+}
+
+static char *
+readline (char *pmt)
+{
+  static char buf[MAXLINEBUFFERLEN + 1];
+  char *s;
+
+  fprintf (rl_outstream, "%s", pmt);
+  s = fgets (buf, sizeof (buf), rl_instream);
+  if (s)
+    {
+      char *s1 = strrchr (s, '\n');
+      if (s1 && *(s1 + 1) == '\0')
+	*s1 = '\0';
+    }
+  return s ? Xstrdup (s) : NULL;
+}
+#endif
+
+/* WritePrompt() */
+/* Write out a prompt if user is a TTY */
+void 
+WritePrompt (void)
+{
+  if (isatty (fileno (InFile)))
+    {
+      if (!BooleanEnv (GetEnv ("expert"), 0))
+	fprintf (stderr, "Enter a command or query (.quit to terminate, .help for assistance).\n");
+    }
+}
+
+#ifdef GNU_READLINE
+static void memory_error_and_abort ();
+
+char *
+xmalloc (bytes)
+     int bytes;
+{
+  char *temp = (char *) Xmalloc (bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+  return (temp);
+}
+
+char *
+xrealloc (pointer, bytes)
+     char *pointer;
+     int bytes;
+{
+  char *temp;
+
+  if (!pointer)
+    temp = (char *) xmalloc (bytes);
+  else
+    temp = (char *) Xrealloc (pointer, bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+
+  return (temp);
+}
+
+static void
+memory_error_and_abort ()
+{
+  fprintf (stderr, "history: Out of virtual memory!\n");
+  abort ();
+}
+#endif
+
+
+/*
+ * This routine returns a pointer to the users entered line 
+ *
+ */
+static char *
+GetLine (char *pmt)
+{
+  static char *the_line = NULL;
+  if (the_line)
+    Xfree (the_line);
+  the_line = NULL;
+  rl_instream = InFile;
+  if (isatty (fileno (InFile)))
+    {
+      fputc ('\r', stderr);
+      if (!isatty (fileno (OutFile)))
+	rl_outstream = stderr;
+      else
+	rl_outstream = OutFile;
+      the_line = readline (pmt);
+    }
+  else
+    {
+      if (isatty (fileno (OutFile)))
+	{
+	  the_line = readline (pmt);
+	  fprintf (stderr, "%s\n", the_line ? the_line : "");
+	}
+      else
+	{
+	  the_line = readline ("");
+	  if (the_line)
+	    fprintf (stderr, "%s%s\n", pmt, the_line);
+	}
+    }
+#ifdef GNU_READLINE
+  if (the_line && *the_line)
+    add_history (the_line);
+#endif
+  return (the_line);
+}
+
+
+
+
+
+
+char *
+GetMultiLine (void)
+{
+  static char *line = NULL;
+  char *s;
+  if (line)
+    Xfree (line);
+  line = NULL;
+  if (!(s = GetLine ("> ")))
+    return (NULL);
+  if (!(line = Xstrdup (s)))
+    {
+      fprintf (stderr, "Unable to allocate memory for the line\n");
+      abort ();
+    }
+
+  while ((s = strrchr (line, '\\')) && *(s + 1) == '\0')
+    {
+      char *new;
+      *strrchr (line, '\\') = '\0';
+      if (!(s = GetLine ("? ")))
+	return (NULL);
+      if (!(new = Xmalloc (strlen (line) + strlen (s) + 2)))
+	{
+	  fprintf (stderr, "Unable to allocate memory for the line\n");
+	  abort ();
+	}
+      strcpy (new, line);
+      strcat (new, "\n");
+      strcat (new, s);
+      Xfree (line);
+      line = new;
+    }
+  return (line);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/read_line.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * read_line.h -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef READ_LINE_H
+#define READ_LINE_H
+
+#define MAXLINEBUFFERLEN  65535
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.c	(revision 21112)
@@ -0,0 +1,775 @@
+/**************************************************************************
+ *
+ * stem_search.c -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "words.h"
+#include "locallib.h"
+#include "stem_search.h"
+#include "mg_errors.h"
+#include "term_lists.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:22  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/07/02 00:18:55  rjmcnab
+   Changed so FindWords could be used in new ways.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:39  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:04  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:08  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem dict = %i\n", sd->MemForStemDict); */
+
+  return sd;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_idx *
+ReadStemIdxBlk (File * stem_idx_file)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  si->stem_idx_file = stem_idx_file;
+  si->MemForStemIdx = 0;
+
+  Fread (&si->sih, sizeof (si->sih), 1, stem_idx_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = Getc (stem_idx_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_idx_file);
+      buffer += len;
+      Fread (&si->pos[i], sizeof (*si->pos), 1, stem_idx_file);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem idx = %i\n", si->MemForStemIdx); */
+
+  return si;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* word should be appropriately stemed */
+static int
+GetIdxBlock (stemmed_idx * si, u_char * word)
+{
+  register int lo = 0, hi = si->sih.num_blocks - 1;
+  register int mid = 0, c = 0;
+
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, si->index[mid]);
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+static int 
+GetBlock (stemmed_dict * sd, u_char * Word)
+{
+  register int lo = 0, hi = sd->sdh.num_blocks - 1;
+  register int mid = 0, c = 0;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, sd->index[mid]);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+/*
+ * This function looks up a word in the stemmed dictionary, it returns -1
+ * if the word cound not be found, and 0 if it successfully finds the word.
+ * If count is non-null the ulong it is pointing to is set to the number of 
+ * occurances of the stemmed word in the collection. i.e wcnt.
+ * If doc_count is non-null the ulong it is pointing to is set to the number
+ * of documents that the word occurs in. i.e fcnt
+ * If invf_ptr is non-null the ulong it is pointing to is set to the position
+ * of the inverted file where the entry for this word start.
+ */
+int 
+FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	  unsigned long *doc_count, unsigned long *invf_ptr,
+	  unsigned long *invf_len)
+{
+  register int lo, hi, mid, c;
+  register unsigned int res;
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  block = GetBlock (sd, Word);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (sd->active != sd->pos[block])
+    {
+      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+    }
+  else
+    {
+      first_word = (unsigned long *) (sd->buffer);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+    }
+  base = (u_char *) (index + num_indexes);
+
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, base + index[mid] + 1);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+
+  res = hi * sd->sdh.lookback;
+  base += index[hi];
+
+  for (;;)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      if (res >= *num_words)
+	return (-1);
+      copy = *base++;
+      suff = *base++;
+      bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+      base += suff;
+      *prev = copy + suff;
+
+      c = casecompare (Word, prev);   /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	return (-1);
+
+      if (c == 0 && doc_count)
+	{
+	  bcopy ((char *) base, (char *) doc_count, sizeof (*doc_count));
+	  NTOHUL(*doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*doc_count);
+
+      if (c == 0 && count)
+	{
+	  bcopy ((char *) base, (char *) count, sizeof (*count));
+	  NTOHUL(*count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*count);
+
+      if (c == 0 && invf_ptr)
+	{
+	  bcopy ((char *) base, (char *) &invfp, sizeof (invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_ptr = invfp;
+	}
+      base += sizeof (*invf_ptr);
+
+      if (c == 0)
+	{
+	  /* Calculate invf_len is necessary */
+	  unsigned long next_invfp;
+	  if (!invf_len)
+	    return (*first_word + res);
+
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    {
+	      *invf_len = *last_invf_len;
+	      return (*first_word + res);
+	    }
+
+	  /* Skip over most of the next word to get to the invf_ptr */
+	  base++;
+	  suff = *base++;
+	  base += suff + sizeof (unsigned long) * 2;
+	  bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	  NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_len = next_invfp - invfp;
+	  return (*first_word + res);
+	}
+      res++;
+    }
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int 
+FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl)
+{
+  register unsigned int res;
+  unsigned int idx_res;
+  unsigned copy, suff;
+  int j, k;
+
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  int idx_block, idx_num_indexes;
+  unsigned long *idx_first_word;
+  unsigned short *idx_num_words;
+  u_char *idx_base;
+  unsigned short *idx_index;
+  u_char idx_prev[MAXSTEMLEN + 1];
+
+  unsigned int num_entries, num_cases;
+  unsigned short blk_index, offset;
+  stemmed_idx * si = NULL;
+
+  /* handle stem_method 0 seperately */
+  if (stem_method == 0) {
+    TermEntry te;
+    if ((te.WE.word_num = FindWord (sd, sWord, &te.WE.count, &te.WE.doc_count,
+				   &te.WE.invf_ptr, &te.WE.invf_len)) != -1) {
+      te.WE.max_doc_count = te.WE.doc_count;
+
+      te.Count = 1;
+      te.Word = copy_string (sWord);
+      if (!te.Word)
+	FatalError (1, "Could NOT create memory to add term");
+      te.Stem = copy_string (sWord);
+      if (!te.Stem)
+	FatalError (1, "Could NOT create memory to add term");
+      /*      te.query_mask = NULL;*/
+      
+      AddTermEntry (tl, &te);
+      return (*tl)->num;
+      
+    } else {
+      /* didn't match */
+      return 0;
+    }
+  }
+  
+  if (stem_method == 1)
+    si = sd->stem1;
+  else if (stem_method == 2)
+    si = sd->stem2;
+  else
+    si = sd->stem3;
+
+  /* Locate block */
+  idx_block = GetIdxBlock (si, sWord);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (si->active != si->pos[idx_block])
+    {
+      Fseek (si->stem_idx_file, si->pos[idx_block] + si->sih.blocks_start, 0);
+      Fread (si->buffer, si->sih.block_size, sizeof (u_char), si->stem_idx_file);
+      si->active = si->pos[idx_block];
+
+      idx_first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*idx_first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      NTOHUS(*idx_num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < idx_num_indexes; j++)
+	NTOHUS(idx_index[j]);
+    }
+  else
+    {
+      idx_first_word = (unsigned long *) (si->buffer);
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+    }
+  idx_base = (u_char *) (idx_index + idx_num_indexes);
+  
+  {
+    /* Locate 3-in-4 block */
+    register int lo, hi, mid, c;
+    lo = 0;
+    hi = idx_num_indexes - 1;
+    while (lo <= hi)
+      {
+	mid = (lo + hi) / 2;
+	c = casecompare (sWord, idx_base + idx_index[mid] + 1);
+	if (c < 0)
+	  hi = mid - 1;
+	else if (c > 0)
+	  lo = mid + 1;
+	else
+	  {
+	    hi = mid;
+	    break;
+	  }
+      }
+    if (hi < 0)
+      hi = 0;
+
+    idx_res = hi * si->sih.lookback;
+    idx_base += idx_index[hi];
+  }
+
+  /* Locate actual word entry */
+  for (;;)
+    {
+      int c;
+      if (idx_res >= *idx_num_words)
+	return (-1);
+      copy = *idx_base++;
+      suff = *idx_base++;
+      bcopy ((char *) idx_base, (char *) (idx_prev + copy + 1), suff);
+      idx_base += suff;
+      *idx_prev = copy + suff;
+
+      c = casecompare (sWord, idx_prev);
+      if (c < 0)
+	return (-1);
+     
+      bcopy ((char *) idx_base, (char *) &num_entries, sizeof (num_entries));
+      NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_entries);
+      
+      if (c > 0)
+	idx_base += num_entries * (sizeof (num_cases) + sizeof (block) + 
+				   sizeof (blk_index) + sizeof (offset));
+
+      else
+	break;
+
+      idx_res++;
+    }
+
+  for (k = 0; k < num_entries; k++)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      /* Read next stem index pos */
+      bcopy ((char *) idx_base, (char *) &num_cases, sizeof (num_cases));
+      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_cases);
+      bcopy ((char *) idx_base, (char *) &block, sizeof (block));
+      NTOHUI(block);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (block);
+      bcopy ((char *) idx_base, (char *) &blk_index, sizeof (blk_index));
+      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (blk_index);
+      bcopy ((char *) idx_base, (char *) &offset, sizeof (offset));
+      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (offset);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      if (sd->active != sd->pos[block])
+	{
+	  Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	  Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	  sd->active = sd->pos[block];
+
+	  first_word = (unsigned long *) (sd->buffer);
+	  NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (j = 0; j < num_indexes; j++)
+	    NTOHUS(index[j]);
+	}
+      else
+	{
+	  first_word = (unsigned long *) (sd->buffer);
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+	}
+      base = (u_char *) (index + num_indexes);
+      
+      res = blk_index * sd->sdh.lookback;
+      base += index[blk_index];
+      
+      for (j = 0; j < offset; j++)
+	{
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  base += sizeof (unsigned long);   /* skip doc_count */
+	  base += sizeof (unsigned long);   /* skip count */
+	  base += sizeof (unsigned long);   /* skip invf_ptr */
+	  res++;
+	}
+
+      for (j = 0; j < num_cases; j++)
+	{
+	  TermEntry te;
+	  
+	  if (res >= *num_words)
+	    return (-1);
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	      
+	  te.Word = copy_string (prev);
+	  if (!te.Word)
+	    FatalError (1, "Could NOT create memory to add term");
+	  te.Stem = copy_string (prev);
+	  if (!te.Stem)
+	    FatalError (1, "Could NOT create memory to add term");
+	  stemmer (2, sd->sdh.stemmer_num, te.Stem);
+
+	  te.Count = 1;
+	  te.WE.word_num = *first_word + res;
+	  bcopy ((char *) base, (char *) &te.WE.doc_count, sizeof (te.WE.doc_count));
+	  NTOHUL(te.WE.doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.max_doc_count = te.WE.doc_count;
+	  base += sizeof (te.WE.doc_count);
+	      
+	  bcopy ((char *) base, (char *) &te.WE.count, sizeof (te.WE.count));
+	  NTOHUL(te.WE.count);
+	  base += sizeof (te.WE.count);
+	      
+	  bcopy ((char *) base, (char *) &invfp, sizeof (te.WE.invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.invf_ptr = invfp;
+	  base += sizeof (te.WE.invf_ptr);
+		
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    te.WE.invf_len = *last_invf_len;
+	  else
+	    {
+	      unsigned long next_invfp;
+	      u_char *oldbase = base;
+		  
+	      /* Skip over most of the next word to get to the invf_ptr */
+	      base++;
+	      suff = *base++;
+	      base += suff + sizeof (unsigned long) * 2;
+	      bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	      NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	      te.WE.invf_len = next_invfp - invfp;
+	      base = oldbase;
+	    }
+	      
+	  /* Add term entry to term list */
+	  /*	  te.query_mask = NULL;*/
+	  AddTermEntry (tl, &te);
+	      
+	  if (res == *num_words - 1 && j + 1 < num_cases)
+	    {
+	      int ii;
+	      /* Read in next block */
+	      block++;
+	      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	      sd->active = sd->pos[block];
+
+	      first_word = (unsigned long *) (sd->buffer);
+	      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	      last_invf_len = (unsigned long *) (first_word + 1);
+	      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	      num_words = (unsigned short *) (last_invf_len + 1);
+	      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	      index = num_words + 1;
+	      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	      /* [RPAP - Jan 97: Endian Ordering] */
+	      for (ii = 0; ii < num_indexes; ii++)
+		NTOHUS(index[ii]);
+
+	      base = (u_char *) (index + num_indexes);
+	      base += index[0];
+	      res = 0;
+	      blk_index = 0;
+	    }
+	  else
+	    res++;
+	} /* end for num_cases */
+    } /* end for num_entries */
+  return (*tl)->num;
+}
+
+
+void 
+FreeStemDict (stemmed_dict * sd)
+{
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (sd->stem1)
+    FreeStemIdx (sd->stem1);
+  if (sd->stem2)
+    FreeStemIdx (sd->stem2);
+  if (sd->stem3)
+    FreeStemIdx (sd->stem3);
+
+  Xfree (sd->index[0]);
+  Xfree (sd->index);
+  Xfree (sd->buffer);
+  Xfree (sd->pos);
+  Xfree (sd);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void
+FreeStemIdx (stemmed_idx * si)
+{
+  Xfree (si->index[0]);
+  Xfree (si->index);
+  Xfree (si->buffer);
+  Xfree (si->pos);
+  Xfree (si);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/stem_search.h	(revision 21112)
@@ -0,0 +1,43 @@
+/**************************************************************************
+ *
+ * stem_search.h -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_STEM_SEARCH
+#define H_STEM_SEARCH
+
+stemmed_dict *ReadStemDictBlk (File * stem_file);
+
+stemmed_idx *ReadStemIdxBlk (File * stem_idx_file);
+
+int FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	      unsigned long *doc_count, unsigned long *invf_ptr,
+	      unsigned long *invf_len);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl);
+
+void FreeStemDict (stemmed_dict * sd);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void FreeStemIdx (stemmed_idx * si);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.c	(revision 21112)
@@ -0,0 +1,148 @@
+/**************************************************************************
+ *
+ * stemmer.c -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+/*
+   $Log$
+   Revision 1.2  2004/06/22 04:17:06  kjdon
+   make stemmerdescription a const
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:23  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:42  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:05  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:10  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; i++) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (const u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); i++)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp (descript, "0") == 0) ||
+      (strcmp (descript, "english") == 0) ||
+      (strcmp (descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp (descript, "1") == 1) ||
+      (strcmp (descript, "french") == 0) ||
+      (strcmp (descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/stemmer.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (const u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.c	(revision 21112)
@@ -0,0 +1,334 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:43  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "term_lists.h"
+#include "messages.h"
+#include "stemmer.h"
+
+TermList *query_term_list = NULL;
+
+/* =========================================================================
+ * Function: MakeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+TermList *
+MakeTermList (int n)
+{
+  TermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (TermList) + (list_size - 1) * sizeof (TermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeTermList (TermList ** term_list)
+{
+  TermList *tl = *term_list;
+
+  if (tl->num > tl->list_size)
+    {
+      if (tl->list_size)
+	tl->list_size *= GROWTH_FACTOR;
+      else
+	tl->list_size = MIN_SIZE;
+    }
+  tl = Xrealloc (tl, sizeof (TermList) + (tl->list_size - 1) * sizeof (TermEntry));
+
+  if (!tl)
+    FatalError (1, "Unable to resize term list");
+
+  *term_list = tl;
+}
+
+/* =========================================================================
+ * Function: ConvertTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = term list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertTermsToString (TermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->TE[i].Word;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetTermList (TermList ** tl)
+{
+  if (*tl)
+    FreeTermList (tl);
+  *tl = MakeTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTermEntry (TermList ** query_term_list, TermEntry * te)
+{
+  TermList *tl = *query_term_list;
+
+  tl->num++;
+  ResizeTermList (query_term_list);
+  tl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) te, (char *) &(tl->TE[tl->num - 1]), sizeof (TermEntry));
+
+  return tl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddTerm
+ * Description: Used in boolean parser - see bool_tree   [RPAP - Feb 97: Term Frequency] 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	 u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, int stemmer_num)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  int j;
+  TermList *tl = *query_term_list;
+
+  /* Look for the word in the already identified terms */
+  for (j = 0; j < tl->num; j++)
+    {
+      TermEntry *te = &(tl->TE[j]);
+      if (compare (te->Word, Word) == 0)
+	{
+	  te->Count++;
+	  return j;
+	}
+    }
+
+
+  {
+    /* Create a new entry in the list for the new word */
+    TermEntry te;
+
+    /* [RPAP - Feb 97: Term Frequency] */
+    te.WE.word_num = word_num;
+    te.WE.count = count;
+    te.WE.doc_count = doc_count;
+    te.WE.max_doc_count = doc_count;   /* [RPAP - Jan 97: Stem Index Change] */
+    te.WE.invf_ptr = invf_ptr;
+    te.WE.invf_len = invf_len;
+    te.Count = Count;
+    te.Word = copy_string (Word);
+    if (!te.Word)
+      FatalError (1, "Could NOT create memory to add term");
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    te.Stem = copy_string (Word);
+    if (!te.Stem)
+      FatalError (1, "Could NOT create memory to add term");
+    stemmer (2, stemmer_num, te.Stem);
+
+    te.require_match = 0;  /* [RJM - 07/97: Ranked Required Terms] */
+
+    return AddTermEntry (query_term_list, &te);
+  }
+
+}
+
+/* =========================================================================
+ * Function: FreeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeTermList (TermList ** the_tl)
+{
+  int j;
+  TermList *tl = *the_tl;
+
+  for (j = 0; j < tl->num; j++)
+    {
+      if (tl->TE[j].Word)
+	Xfree (tl->TE[j].Word);
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (tl->TE[j].Stem)
+	Xfree (tl->TE[j].Stem);
+    }
+  Xfree (tl);
+
+  *the_tl = NULL;
+}
+
+/* =========================================================================
+ * Function: PrintWordEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintWordEntry (WordEntry * we, FILE * file)
+{
+  fprintf (file, "we->word_num = %d\n", we->word_num);
+  fprintf (file, "we->count = %ld\n", we->count);
+  fprintf (file, "we->doc_count = %ld\n", we->doc_count);
+  fprintf (file, "we->max_doc_count = %ld\n", we->max_doc_count);
+  fprintf (file, "we->invf_ptr = %ld\n", we->invf_ptr);
+  fprintf (file, "we->invf_len = %ld\n", we->invf_len);
+}
+
+/* =========================================================================
+ * Function: PrintTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermEntry (TermEntry * te, FILE * file)
+{
+
+  fprintf (file, "Term Entry\n");
+  fprintf (file, "te->Count = %d\n", te->Count);
+  fprintf (file, "te->Word = %s\n", str255_to_string (te->Word, NULL));
+  if (te->Stem != NULL)
+    fprintf (file, "te->Stem = %s\n", str255_to_string (te->Stem, NULL));  /* [RPAP - Jan 97: Stem Index Change] */
+  fprintf (file, "te->require_match = %i\n", te->require_match);  /* [RJM 07/97: Ranked Required Terms] */
+  PrintWordEntry (&(te->WE), file);
+
+}
+
+/* =========================================================================
+ * Function: PrintTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermList (TermList * tl, FILE * file)
+{
+  int i;
+
+  fprintf (file, "Term List\n");
+  fprintf (file, "tl->list_size = %d\n", tl->list_size);
+  fprintf (file, "tl->num = %d\n", tl->num);
+
+  for (i = 0; i < tl->num; i++)
+    {
+      fprintf (file, "[%d]\n", i);
+      PrintTermEntry (&(tl->TE[i]), file);
+    }
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/term_lists.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef TERM_LISTS_H
+#define TERM_LISTS_H
+
+#include "sysfuncs.h"
+
+#define MAXTERMSTRLEN 1023	/* maximum number of characters in term string */
+
+typedef struct WordEntry
+  {
+    int word_num;		/* Unique number for each different word */
+    u_long count;		/* Number of times the word occurs in the text */
+    u_long doc_count;		/* Number of documents that contain the word */
+    u_long max_doc_count;       /* [RPAP - Jan 97: Stem Index Change] */
+    u_long invf_ptr;		/* This is a byte position of the  
+				   inverted file entry corresponding to the word */
+    u_long invf_len;		/* This is the length of the inverted 
+				   file entry in bytes */
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Word;		/* The word. */
+    u_char *Stem;               /* [RPAP - Jan 97: Stem Index Change] - The stemmed non-cf version of Word */
+    int require_match;          /* [RJM 07/97: Ranked Required Terms] 0=optional match, 1=must match */
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry TE[1];
+  }
+TermList;
+
+#define GetNthWE(term_list, n) (&((term_list)->TE[(n)].WE))
+
+/* --- prototypes --- */
+void ConvertTermsToString (TermList * query_term_list, char *str);
+int AddTermEntry (TermList ** query_term_list, TermEntry * te);
+int AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	     u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, /* [RPAP - Feb 97: Term Frequency] */
+	     int stemmer_num);  
+void ResetTermList (TermList ** tl);
+void FreeTermList (TermList ** the_tl);
+void PrintWordEntry (WordEntry * we, FILE * file);
+void PrintTermEntry (TermEntry * te, FILE * file);
+void PrintTermList (TermList * tl, FILE * file);
+TermList *MakeTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.h	(revision 21112)
@@ -0,0 +1,176 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_TEXT
+#define H_TEXT
+
+#include "huffman.h"
+
+
+
+
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using binary codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_BINARY 1
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+/* This method codes novel words using hybrid version of delta and a MTF
+   operation. The novel words are stored in a auxillary dictionary which
+   is built by pass two. */
+#define MG_NOVEL_HYBRID_MTF 4
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+typedef struct compression_dict_header
+  {
+    u_long dict_type;
+    u_long novel_method;
+    u_long params[TEXT_PARAMS];
+    u_long num_words[2];
+    u_long num_word_chars[2];
+    u_long lookback;
+  }
+compression_dict_header;
+
+
+typedef struct comp_frags_header
+  {
+    huff_data hd;
+    u_long uncompressed_size;
+    u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  }
+comp_frags_header;
+
+
+
+typedef struct compressed_text_header
+  {
+    u_long num_of_docs;
+    u_long num_of_words;	/* number of words in collection */
+    double num_of_bytes; /* [RJM 07/97: 4G limit] */
+    double ratio;
+    u_long length_of_longest_doc;	/* (characters) */
+    u_long dummy;    /* added to make the structure the same on different architectures */
+  }
+compressed_text_header;
+
+
+typedef struct compression_stats_header
+  {
+    u_long num_docs;
+    u_long dummy;   /* added to make the structure the same on different architectures */
+    double num_bytes; /* [RJM 07/97: 4G limit] */
+  }
+compression_stats_header;
+
+
+typedef struct frags_stats_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+frags_stats_header;
+
+
+typedef struct aux_frags_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+aux_frags_header;
+
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass1.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass1.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass1.c	(revision 21112)
@@ -0,0 +1,405 @@
+/**************************************************************************
+ *
+ * text.pass1.c -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long LongestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh =
+{0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+int 
+init_text_1 (char *FileName)
+{
+  int which;
+  if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  for (which = 1; which >= 0; which--)
+    {
+      u_char *word;
+      hash_rec *ent;
+      dict_data *dd = &DictData[which];
+
+      dd->wordnum = 0;
+      dd->words_read = 0;
+      dd->bytes_diff = 0;
+      dd->HashSize = INITIAL_HASH_SIZE;
+      dd->HashUsed = 0;
+      if (!(dd->HashTable = Xmalloc (sizeof (hash_rec) * dd->HashSize)))
+	{
+	  Message ("Unable to allocate memory for table");
+	  return (COMPERROR);
+	}
+      ChangeMem (sizeof (hash_rec) * dd->HashSize);
+      bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);
+
+      word = Pool;
+      *Pool++ = '\0';
+      PoolLeft--;
+      {
+	register u_char *wptr;
+	register int hsize = dd->HashSize;
+	register unsigned long hashval, step;
+
+	HASH (hashval, step, word, hsize);
+	wptr = (dd->HashTable + hashval)->word;
+	while (wptr)
+	  {
+	    hashval += step;
+	    if (hashval >= hsize)
+	      hashval -= hsize;
+	    wptr = (dd->HashTable + hashval)->word;
+	  }
+	ent = dd->HashTable + hashval;
+      }
+      ent->wcnt = 1;
+      ent->word = word;
+      dd->HashUsed = 1;
+    }
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_text_1 (u_char * s_in, int l_in)
+{
+  int which;
+  u_char *end = s_in + l_in - 1;
+  if (l_in > LongestDoc)
+    LongestDoc = l_in;
+
+  csh.num_docs++;
+  csh.num_bytes += l_in;
+
+  which = inaword (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      dict_data *dd = &DictData[which];
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+      dd->wordnum++;
+      inputbytes += *Word;
+      dd->words_read++;
+
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = dd->HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = dd->HashTable + hashval;
+	    if (!ent->word)
+	      {
+		int len = *Word + 1;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->occurance_num = occurance_num++;
+		ent->wcnt = 1;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		dd->HashUsed++;
+		dd->bytes_diff += Word[0];
+		break;
+	      }
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		break;
+	      }
+	  }
+      }
+
+  
+      if (dd->HashUsed >= dd->HashSize >> 1)
+	{
+	  hash_rec *ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (dd->HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  ChangeMem (sizeof (hash_rec) * size);
+	  bzero ((char *) ht, sizeof (hash_rec) * size);
+
+	  for (i = 0; i < dd->HashSize; i++)
+	    if (dd->HashTable[i].word)
+	      {
+		register u_char *wptr;
+		register unsigned long hashval, step;
+
+		wptr = dd->HashTable[i].word;
+		HASH (hashval, step, wptr, size);
+		wptr = (ht + hashval)->word;
+		while (wptr)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    wptr = (ht + hashval)->word;
+		  }
+		ht[hashval] = dd->HashTable[i];
+	      }
+	  Xfree (dd->HashTable);
+	  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+	  dd->HashTable = ht;
+	  dd->HashSize = size;
+
+
+	}
+    }
+  return (COMPALLOK);
+}				/* encode */
+
+
+
+static int 
+PackHashTable (dict_data * dd)
+{
+  int s, d;
+  for (s = d = 0; s < dd->HashSize; s++)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  if (!(dd->HashTable = Xrealloc (dd->HashTable,
+				  sizeof (hash_rec) * dd->HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *s1, const void *s2)
+{
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+
+static void 
+WriteHashTable (FILE * fp, dict_data * dd)
+{
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR)
+    return;
+
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; j++)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+
+  for (j = 0; j < dd->HashSize; j++)
+    {
+      curr = dd->HashTable[j].word;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(dd->HashTable[j].wcnt);
+      HTONUL(dd->HashTable[j].occurance_num);
+
+      fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+      fwrite (&dd->HashTable[j].occurance_num,
+	      sizeof (dd->HashTable[j].occurance_num), 1, fp);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(dd->HashTable[j].wcnt);
+      NTOHUL(dd->HashTable[j].occurance_num);
+
+      fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+    }
+}
+
+
+int 
+done_text_1 (char *file_name)
+{
+  char *temp_str;
+  FILE *fp;
+
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+	      file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+	      strerror(errno)
+#else
+	      " "
+#endif
+	      );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  fclose(fp);
+  return COMPALLOK;
+}				/* done_encode */
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass2.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass2.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/text.pass2.c	(revision 21112)
@@ -0,0 +1,791 @@
+/**************************************************************************
+ *
+ * text.pass2.c -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "huffman.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2002/01/22 02:41:14  jrm21
+   Try to print out an error if the aux file couldn't be written to.
+
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:10  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:14  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define POOL_SIZE 1024*256
+
+typedef struct char_pool
+  {
+    struct char_pool *next;
+    u_long left;
+    u_char *ptr;
+    u_char pool[POOL_SIZE];
+  }
+char_pool;
+
+typedef struct novel_hash_rec
+  {
+    u_long ordinal_num;
+    u_char *word;
+  }
+novel_hash_rec;
+
+
+#define INITIAL_HASH_SIZE 7927
+#define MAX_SWAPS 10000
+
+typedef struct novel_hash_table
+  {
+    novel_hash_rec *HashTable;
+    u_long HashSize, HashUsed;
+    char_pool *first_pool;
+    char_pool *pool;
+    u_long next_num, binary_start;
+    novel_hash_rec **code_to_nhr;
+  }
+novel_hash_table;
+
+
+static FILE *text, *text_idx;
+
+static u_char *comp_buffer;
+
+static u_long text_length;
+
+/* [RJM 07/97: 4G limit] */
+static double stats_in_tot_bytes = 0.0;
+static double stats_in_bytes = 0.0;
+static double stats_out_bytes = 0.0;
+
+
+static novel_hash_table nht[2];
+
+static u_long prefix_len = 0;
+
+int blk_start[2][33], blk_end[2][33];
+
+
+static char_pool *
+new_pool (char_pool * pool)
+{
+  char_pool *p = Xmalloc (sizeof (char_pool));
+  if (!p)
+    FatalError (1, "Unable to allocate memory for pool");
+  if (pool)
+    pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+
+int 
+init_text_2 (char *file_name)
+{
+  char path[512];
+  int i;
+
+  if (LoadCompressionDictionary (make_name (file_name, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  if (!(text = create_file (file_name, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  bzero ((char *) &cth, sizeof (cth));
+
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  text_length = sizeof (u_long) + sizeof (cth);
+
+  if (!(text_idx = create_file (file_name, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  if (fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+
+  if (!(comp_buffer = Xmalloc (sizeof (u_char) * buf_size)))
+    {
+      Message ("No memory for compression buffer");
+      return (COMPERROR);
+    }
+
+#if 0
+  MaxMemInUse += sizeof (u_char) * buf_size;
+#endif
+
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; i++)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	bzero ((char *) nht[i].HashTable,
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  nht[i].code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+					((nht[i].HashSize >> 1) + 2));
+	else
+	  nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID ||
+	    cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		num++;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+
+int 
+ic (void *a, void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+
+
+/* #define DOCDUMP 477 */
+
+int 
+process_text_2 (u_char * s_in, int l_in)
+{
+  int which, byte_length;
+  u_char *end = s_in + l_in - 1;
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+
+  which = inaword (s_in, end);
+
+  ENCODE_START (comp_buffer, buf_size)
+
+    ENCODE_BIT (which);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_ENCODE_L (nht[0].binary_start, prefix_len);
+      DELTA_ENCODE_L (nht[1].binary_start, prefix_len);
+    }
+
+  novels_used[0] = novels_used[1] = 0;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("---------------------------------------------------\n");
+      printf ("which = %d\n", which);
+    }
+#endif
+
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      int res;
+
+      if (which)
+	cth.num_of_words++;
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+#ifdef DOCDUMP
+      if (cth.num_of_docs == DOCDUMP)
+	{
+	  printf ("%sword : \"%.*s\"", which ? "    " : "non-", Word[0], Word + 1);
+	}
+#endif
+
+      /* Search the hash table for Word */
+      if (ht[which])
+	{
+	  register unsigned long hashval, step;
+	  register int tsize = ht[which]->size;
+	  register u_char **wptr;
+	  HASH (hashval, step, Word, tsize);
+	  for (;;)
+	    {
+	      register u_char *s1;
+	      register u_char *s2;
+	      register int len;
+	      wptr = ht[which]->table[hashval];
+	      if (wptr == NULL)
+		{
+		  res = COMPERROR;
+		  break;
+		}
+
+	      /* Compare the words */
+	      s1 = Word;
+	      s2 = *wptr;
+	      len = *s1 + 1;
+	      for (; len; len--)
+		if (*s1++ != *s2++)
+		  break;
+
+	      if (len)
+		{
+		  hashval += step;
+		  if (hashval >= tsize)
+		    hashval -= tsize;
+		}
+	      else
+		{
+		  res = ht[which]->table[hashval] - ht[which]->words;
+		  break;
+		}
+	    }
+	}
+      else
+	res = COMPERROR;
+      /* Check that the word was found in the dictionary */
+      if (res == COMPERROR)
+	{
+	  if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	    {
+	      Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      return (COMPERROR);
+	    }
+	  if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	    {
+	      u_long i;
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      HUFF_ENCODE (Word[0], lens_codes[which], lens_huff[which].clens);
+	      for (i = 0; i < Word[0]; i++)
+		HUFF_ENCODE (Word[i + 1], char_codes[which],
+			     char_huff[which].clens);
+	    }
+	  if (cdh.dict_type == MG_SEED_DICTIONARY)
+	    {
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      switch (cdh.novel_method)
+		{
+		case MG_NOVEL_HUFFMAN_CHARS:
+		  {
+		    u_long i;
+		    HUFF_ENCODE (Word[0], lens_codes[which],
+				 lens_huff[which].clens);
+		    for (i = 0; i < Word[0]; i++)
+		      HUFF_ENCODE (Word[i + 1], char_codes[which],
+				   char_huff[which].clens);
+		  }
+		  break;
+		case MG_NOVEL_BINARY:
+		case MG_NOVEL_DELTA:
+		case MG_NOVEL_HYBRID:
+		case MG_NOVEL_HYBRID_MTF:
+		  {
+		    register unsigned long hashval, step;
+		    register novel_hash_table *h = &nht[which];
+		    register int hsize = h->HashSize;
+		    register novel_hash_rec *ent;
+		    HASH (hashval, step, Word, hsize);
+		    for (;;)
+		      {
+			register u_char *s1, *s2;
+			register int len;
+			ent = h->HashTable + hashval;
+			if (!ent->word)
+			  {
+			    int len = *Word + 1;
+			    if (len > h->pool->left)
+			      h->pool = new_pool (h->pool);
+			    ent->word = h->pool->ptr;
+			    ent->ordinal_num = h->next_num++;
+			    if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			      h->code_to_nhr[ent->ordinal_num - 1] = ent;
+			    memcpy (h->pool->ptr, Word, len);
+			    h->pool->ptr += len;
+			    h->pool->left -= len;
+			    h->HashUsed++;
+			    break;
+			  }
+			/* Compare the words */
+			s1 = Word;
+			s2 = ent->word;
+			len = *s1 + 1;
+			for (; len; len--)
+			  if (*s1++ != *s2++)
+			    break;
+
+			if (!len)
+			  break;
+
+			hashval = (hashval + step);
+			if (hashval >= hsize)
+			  hashval -= hsize;
+		      }
+
+		    switch (cdh.novel_method)
+		      {
+		      case MG_NOVEL_BINARY:
+			{
+			  BINARY_ENCODE (ent->ordinal_num, h->binary_start);
+			  if (ent->ordinal_num == h->binary_start)
+			    h->binary_start++;
+			}
+			break;
+		      case MG_NOVEL_DELTA:
+			{
+			  DELTA_ENCODE (ent->ordinal_num);
+			}
+			break;
+		      case MG_NOVEL_HYBRID:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+			}
+			break;
+		      case MG_NOVEL_HYBRID_MTF:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+
+			  if (ent->ordinal_num - 1 >= novels_used[which])
+			    {
+			      int a = novels_used[which];
+			      int b = ent->ordinal_num - 1;
+			      novel_hash_rec *temp;
+
+
+/*                            fprintf(stderr, "a = %d , b = %d\n", a, b);
+ */
+			      temp = h->code_to_nhr[a];
+			      h->code_to_nhr[a] = h->code_to_nhr[b];
+			      h->code_to_nhr[b] = temp;
+			      h->code_to_nhr[a]->ordinal_num = a + 1;
+			      h->code_to_nhr[b]->ordinal_num = b + 1;
+			      if (novels_used[which] == MAX_SWAPS)
+				FatalError (1, "Not enough mem for swapping");
+			      swaps[which][novels_used[which]] = b;
+			      novels_used[which]++;
+			    }
+			}
+			break;
+		      }
+		    if (h->HashUsed >= h->HashSize >> 1)
+		      {
+			novel_hash_rec *ht;
+			unsigned long size;
+			unsigned long i;
+			size = prime (h->HashSize * 2);
+			if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			  {
+			    Xfree (h->code_to_nhr);
+			    h->code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+						      ((size >> 1) + 2));
+			  }
+			if (!(ht = Xmalloc (sizeof (novel_hash_rec) * size)))
+			  {
+			    Message ("Unable to allocate memory for table");
+			    return (COMPERROR);
+			  }
+			bzero ((char *) ht, sizeof (novel_hash_rec) * size);
+
+			for (i = 0; i < h->HashSize; i++)
+			  if (h->HashTable[i].word)
+			    {
+			      register u_char *wptr;
+			      register unsigned long hashval, step;
+
+			      wptr = h->HashTable[i].word;
+			      HASH (hashval, step, wptr, size);
+			      wptr = (ht + hashval)->word;
+			      while (wptr)
+				{
+				  hashval += step;
+				  if (hashval >= size)
+				    hashval -= size;
+				  wptr = (ht + hashval)->word;
+				}
+			      ht[hashval] = h->HashTable[i];
+			      if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+				h->code_to_nhr[ht[hashval].ordinal_num - 1] =
+				  &ht[hashval];
+			    }
+			Xfree (h->HashTable);
+			h->HashTable = ht;
+			h->HashSize = size;
+		      }
+		  }
+		  break;
+		}
+	    }
+	}
+      else
+	{
+	  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+#ifdef DOCDUMP
+	  if (cth.num_of_docs == DOCDUMP)
+	    {
+	      printf ("   %d %d\n", ht[which]->hd->clens[res],
+		      ht[which]->codes[res]);
+	    }
+#endif
+	}
+    }
+
+
+  /* Add a 1 bit onto the end of the buffer the remaining bits in the last
+     byte will all be zero */
+
+  ENCODE_BIT (1);
+
+  ENCODE_FLUSH;
+
+  byte_length = __pos - __base;
+  if (!__remaining)
+    {
+      Message ("The end of the buffer was probably overrun");
+      return COMPERROR;
+    }
+
+  ENCODE_DONE
+
+#ifdef DOCDUMP
+    if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("unused bits = %d\n", bits_unused);
+    }
+#endif
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  text_length += byte_length;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      int i;
+      for (i = 0; i < byte_length; i++)
+	printf ("%02x ", comp_buffer[i]);
+      printf ("\n");
+    }
+#endif
+
+  if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  novel_hash_rec *temp;
+	  temp = nht[which].code_to_nhr[a];
+	  nht[which].code_to_nhr[a] = nht[which].code_to_nhr[b];
+	  nht[which].code_to_nhr[b] = temp;
+	  nht[which].code_to_nhr[a]->ordinal_num = a + 1;
+	  nht[which].code_to_nhr[b]->ordinal_num = b + 1;
+	}
+
+
+  fwrite (comp_buffer, sizeof (*comp_buffer), byte_length, text);
+
+  if ((double) l_in / (double) byte_length > cth.ratio)
+    cth.ratio = (double) l_in / (double) byte_length;
+
+  cth.num_of_docs++;
+  if (l_in > cth.length_of_longest_doc)
+    cth.length_of_longest_doc = l_in;
+
+  cth.num_of_bytes += l_in;
+
+  if (Comp_Stats)
+    {
+      stats_in_tot_bytes += l_in;
+      stats_in_bytes += l_in;
+      stats_out_bytes += byte_length;
+      if (stats_in_bytes >= comp_stat_point)
+	{
+	  fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+		   stats_in_bytes, stats_out_bytes,
+		   (double) stats_out_bytes / (double) stats_in_bytes);
+	  stats_in_bytes = 0.0;
+	  stats_out_bytes = 0.0;
+	}
+    }
+
+  return COMPALLOK;
+}
+
+
+
+
+
+
+int 
+write_aux_dict (char *FileName)
+{
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (FileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              FileName, TEXT_DICT_AUX_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+              " "
+#endif
+              );
+      return COMPERROR;
+    }
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+
+      fwrite (&afh, sizeof (afh), 1, aux);
+
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+void 
+estimate_compressed_aux_dict (void)
+{
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      bzero ((char *) chars, sizeof (chars));
+      bzero ((char *) lens, sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      lens[len]++;
+	      total_uncomp += len + 4;
+	      for (; len; len--)
+		chars[*buf++]++;
+	    }
+	}
+      for (j = 0; j < 256; j++)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; j++)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+
+      aux_compressed += (Calculate_Huffman_Size (16, flens, lens) +
+			 Calculate_Huffman_Size (256, fchars, chars)) / 8;
+
+    }
+
+  Message ("Aux dictionary (Uncompressed) %.2f Mb ( %u bytes   %0.3f %%)",
+	   total_uncomp / 1024.0 / 1024, total_uncomp,
+	   (total_uncomp * 100.0) / bytes_processed);
+  Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	   (aux_compressed * 100.0) / bytes_processed);
+}
+
+
+
+
+
+
+int 
+done_text_2 (char *FileName)
+{
+  if (Comp_Stats)
+    fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+	     stats_in_bytes, stats_out_bytes,
+	     (double) stats_out_bytes / (double) stats_in_bytes);
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth.num_of_docs);
+  HTOND(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth.num_of_words);
+  HTONUL(cth.length_of_longest_doc);
+  HTOND(cth.ratio);
+
+  if (fseek (text_idx, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+  fclose (text_idx);
+
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+  fclose (text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+  
+
+  Message ("Compressed Text %.2f Mb ( %u bytes   %0.3f %%)",
+	   text_length / 1024.0 / 1024.0, text_length,
+	   (text_length * 100.0) / bytes_processed);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / bytes_processed);
+
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_BINARY ||
+       cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID ||
+       cdh.novel_method == MG_NOVEL_HYBRID_MTF))
+    {
+      if (write_aux_dict (FileName) == COMPERROR)
+	return COMPERROR;
+      estimate_compressed_aux_dict ();
+    }
+  else
+    {
+      if (cdh.dict_type != MG_COMPLETE_DICTIONARY)
+	Message ("Huffman info for chars in dictionary %.2f Mb"
+		 " ( %u bytes   %0.3f %%)",
+		 Chars_disk / 1024.0 / 1024, Chars_disk,
+		 (Chars_disk * 100.0) / bytes_processed);
+      unlink (make_name (FileName, TEXT_DICT_AUX_SUFFIX, NULL));
+    }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.c	(revision 21112)
@@ -0,0 +1,1090 @@
+/**************************************************************************
+ *
+ * text_get.c -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "text_get.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg_errors.h"
+#include "local_strings.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:11  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:15  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+
+
+
+
+static double 
+FetchDocStartLev1 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  unsigned long data[2];
+  /* [TS:Sep/94] Fixed up the seek call to give the correct offset */
+  Fseek (td->TextIdxFile,
+	 sizeof (unsigned long) * (DN - 1) +	/* the doc offsets */
+	 sizeof (unsigned long) +	/* the magic number */
+	 sizeof (compressed_text_header),	/* the header */
+	 0);
+  Fread ((char *) &data, sizeof (data), 1, td->TextIdxFile);
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(data[0]);
+  NTOHUL(data[1]);
+
+  *seek_pos = data[0];
+  *len = data[1] - data[0];
+  return (1.0);
+}
+
+#define MG_PAGE_SIZE 2048
+
+static int 
+LoadIdx (text_data * td, unsigned long DN)
+{
+  if (!td->idx_data)
+    {
+      td->idx_data = Xmalloc (sizeof (*(td->idx_data)) * MG_PAGE_SIZE);
+      if (!td->idx_data)
+	FatalError (1, "Out of memory in FDSL2");
+    }
+  if (td->current_pos == -1 || DN >= td->current_pos + MG_PAGE_SIZE - 1 ||
+      DN < td->current_pos)
+    {
+      int i, num;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      long rn = (long) DN - (MG_PAGE_SIZE >> 1);
+      if (rn < 1)
+	rn = 1;
+      Fseek (td->TextIdxWgtFile, (sizeof (unsigned long) + sizeof (float)) *
+	       (rn - 1) + sizeof (unsigned long), 0);
+      num = Fread ((char *) td->idx_data, sizeof (*(td->idx_data)), MG_PAGE_SIZE, /* [RPAP - Jan 97: Endian Ordering] */
+		   td->TextIdxWgtFile);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num; i++)
+	{
+	  NTOHUL(td->idx_data[i].Start);
+	  NTOHF(td->idx_data[i].Weight);
+	}
+
+      td->current_pos = rn;
+    }
+  return DN - td->current_pos;
+}
+
+static double 
+FDSL2 (text_data * td, unsigned long DN, unsigned long *Pos)
+{
+  unsigned long pos = LoadIdx (td, DN);
+  *Pos = td->idx_data[pos].Start;
+  return (td->idx_data[pos].Weight);
+}
+
+
+static double 
+FetchDocStartLev2 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  double Weight;
+  unsigned long s1, s2;
+  Weight = FDSL2 (td, DN, &s1);
+  do
+    {
+      DN++;
+      FDSL2 (td, DN, &s2);
+    }
+  while (s2 == s1);
+  *seek_pos = s1;
+  *len = s2 - s1;
+  return (Weight);
+}
+
+
+
+
+double 
+FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len)
+{
+  qd->text_idx_lookups++;
+  if (qd->td->TextIdxWgtFile)
+    return FetchDocStartLev2 (qd->td, DN, seek_pos, len);
+  else
+    return FetchDocStartLev1 (qd->td, DN, seek_pos, len);
+}
+
+unsigned long 
+FetchInitialParagraph (text_data * td, unsigned long ParaNum)
+{
+  if (td->TextIdxWgtFile)
+    {
+      unsigned long pos;
+      unsigned long start;
+      int PN = ParaNum - 1;
+      pos = LoadIdx (td, ParaNum);
+      start = td->idx_data[pos].Start;
+      while (PN > 0)
+	{
+	  pos = LoadIdx (td, PN);
+	  if (td->idx_data[pos].Start != start)
+	    return PN + 1;
+	  PN--;
+	}
+      return PN + 1;
+    }
+  else
+    return ParaNum;
+}
+
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int 
+FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt)
+{
+  if (!DocEnt->SeekPos)
+    FetchDocStart (qd, DocEnt->DocNum, &DocEnt->SeekPos, &DocEnt->Len);
+  if (!(*DocBuff = Xmalloc (DocEnt->Len)))
+    return (-1);
+
+  if (Fseek (qd->td->TextFile, DocEnt->SeekPos, 0) == -1)
+    FatalError (1, "Error when seeking into text file");
+#if 0
+  printf ("Loading compressed text %d %d\n", DocEnt->SeekPos, DocEnt->Len);
+#endif
+  if (Fread (*DocBuff, 1, DocEnt->Len, qd->td->TextFile) != DocEnt->Len)
+    FatalError (1, "Error when reading data");
+
+  return (DocEnt->Len);
+
+}
+
+
+text_data *
+LoadTextData (File * text, File * text_idx_wgt, File * text_idx)
+{
+  text_data *td;
+
+  if (!(td = Xmalloc (sizeof (text_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  td->TextFile = text;
+  td->TextIdxWgtFile = text_idx_wgt;
+  td->TextIdxFile = text_idx;
+  td->current_pos = -1;
+  td->idx_data = NULL;
+  Fread (&td->cth, sizeof (td->cth), 1, text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(td->cth.num_of_docs);
+  NTOHD(td->cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(td->cth.num_of_words);
+  NTOHUL(td->cth.length_of_longest_doc);
+  NTOHD(td->cth.ratio);
+
+  return (td);
+}
+
+
+void 
+FreeTextData (text_data * td)
+{
+  if (td)
+    {
+      if (td->idx_data)
+	Xfree (td->idx_data);
+      Xfree (td);
+    }
+}
+
+
+static int 
+pts_comp (const void *A, const void *B)
+{
+  const DocEntry *const *a = A;
+  const DocEntry *const *b = B;
+  return (*a)->DocNum - (*b)->DocNum;
+}
+
+
+
+
+int 
+GetPosLens (query_data * qd, DocEntry * Docs, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+  for (i = j = 0; i < num; i++, Docs++)
+    if (!Docs->SeekPos)
+      pts[j++] = Docs;
+
+  if (j)
+    {
+      qsort (pts, j, sizeof (DocEntry *), pts_comp);
+      for (i = 0; i < j; i++)
+	FetchDocStart (qd, pts[i]->DocNum, &pts[i]->SeekPos, &pts[i]->Len);
+    }
+
+  Xfree (pts);
+  return (0);
+}
+
+
+
+
+
+int 
+LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  int mem;
+
+  if (!num)
+    return (0);
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+
+  mem = i = 0;
+  do
+    {
+      pts[i] = Docs;
+      mem += Docs->Len;
+      i++;
+      Docs++;
+    }
+  while (i < num && mem < max_mem);
+  if (i > 1)
+    qsort (pts, i, sizeof (DocEntry *), pts_comp);
+  for (j = 0; j < i; j++)
+    {
+      if (FetchCompressed (qd, &pts[j]->CompTextBuffer, pts[j]) == -1)
+	return (-1);
+      ChangeMemInUse (qd, pts[j]->Len);
+    }
+
+  Xfree (pts);
+
+  return (i);
+}
+
+
+
+
+
+void 
+FreeBuffers (query_data * qd, DocEntry * Docs, int num)
+{
+  int i;
+  for (i = 0; i < num; i++, Docs++)
+    if (Docs->CompTextBuffer)
+      {
+	Xfree (Docs->CompTextBuffer);
+	Docs->CompTextBuffer = NULL;
+	ChangeMemInUse (qd, -Docs->Len);
+      }
+}
+
+
+
+/****************************************************************************/
+
+static void 
+FreeAuxDict (auxiliary_dict * ad)
+{
+  if (!ad)
+    return;
+  if (ad->word_data[0])
+    Xfree (ad->word_data[0]);
+  if (ad->word_data[1])
+    Xfree (ad->word_data[1]);
+  if (ad->words[0])
+    Xfree (ad->words[0]);
+  if (ad->words[1])
+    Xfree (ad->words[1]);
+  Xfree (ad);
+}
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict * cd, File * text_aux_dict)
+{
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = Xmalloc (sizeof (auxiliary_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) ad, sizeof (*ad));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      Fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = Xmalloc (ad->afh[i].mem_for_frags)))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+      if (!(ad->words[i] = Xmalloc (ad->afh[i].num_frags * sizeof (u_char *))))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+
+      Fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	  cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd->cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+
+
+
+
+static u_char ***
+ReadInWords (File * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  if (!(vals = Xmalloc (ptrs_reqd * sizeof (*vals))))
+    return (NULL);
+
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **))))
+    return (NULL);
+
+  if (!(next_word[0] = Xmalloc (mem_reqd))) return (NULL);
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = Getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      Fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static compression_dict *
+Load_Comp_Dict (File * dict, File * aux_dict)
+{
+  int which;
+  compression_dict *cd;
+
+  if (!(cd = Xmalloc (sizeof (compression_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) cd, sizeof (compression_dict));
+
+  cd->MemForCompDict = sizeof (compression_dict);
+
+  if (F_Read_cdh (dict, &cd->cdh, &cd->MemForCompDict, NULL) == -1)
+    return NULL;
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+
+	  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						 NULL)))
+	    return NULL;
+	  cd->escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  cd->chars_huff[which] = hd;
+	  cd->chars_vals[which] = vals;
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  cd->lens_huff[which] = hd;
+	  cd->lens_vals[which] = vals;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->chars_huff[which] = hd;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict,
+				       NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->chars_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      cd->lens_huff[which] = hd;
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict
+				       ,NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->lens_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+
+      if (!(cd->ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+    }
+
+
+  mg_errno = MG_NOERROR;
+
+  cd->fast_loaded = 0;
+  return (cd);
+}
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p, cd) & 7))) != 0)
+
+
+static compression_dict *
+Load_Fast_Comp_Dict (File * text_fast_comp_dict)
+{
+  compression_dict *cd;
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  Fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (!(cd = Xmalloc (mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  Fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = Xmalloc (fixup_mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; p++)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (i = 0; i <= 1; i++)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; j++)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  Xfree (fixup);
+  return (cd);
+}
+
+
+
+
+compression_dict *
+LoadCompDict (File * text_comp_dict,
+	      File * text_aux_dict,
+	      File * text_fast_comp_dict)
+{
+  return text_fast_comp_dict ?
+    Load_Fast_Comp_Dict (text_fast_comp_dict) :
+    Load_Comp_Dict (text_comp_dict, text_aux_dict);
+}
+
+
+
+
+void 
+FreeCompDict (compression_dict * cd)
+{
+  int which;
+  if (cd->fast_loaded)
+    {
+      Xfree (cd);
+      return;
+    }
+  for (which = 0; which < 2; which++)
+    {
+      if (cd->cfh[which])
+	Xfree (cd->cfh[which]);
+      if (cd->chars_huff[which])
+	Xfree (cd->chars_huff[which]);
+      if (cd->lens_huff[which])
+	Xfree (cd->lens_huff[which]);
+      if (cd->values[which])
+	{
+	  Xfree (cd->values[which][0][0]); /* [RJM 07/98: Memory Leak] */
+	  Xfree (cd->values[which][0]);
+	  Xfree (cd->values[which]);
+	}
+      if (cd->chars_vals[which])
+	{
+	  Xfree (cd->chars_vals[which][0]);
+	  Xfree (cd->chars_vals[which]);
+	}
+      if (cd->lens_vals[which])
+	{
+	  Xfree (cd->lens_vals[which][0]);
+	  Xfree (cd->lens_vals[which]);
+	}
+    }
+  if (cd->ad)
+    FreeAuxDict (cd->ad);
+  Xfree (cd);
+}
+
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+/*#define DUMPDOC */
+
+#define MAX_SWAPS 10000
+
+int 
+DecodeText (compression_dict * cd,
+	    u_char * s_in, int l_in, u_char * s_out, int *l_out)
+{
+  auxiliary_dict *ad = cd->ad;
+  int which;
+  u_long num_bits, bits;
+  u_char *ptr = s_out;
+  static int num = 0;
+  u_long binary_start[2];
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+  novels_used[0] = novels_used[1] = 0;
+
+  {
+    unsigned char bf = s_in[l_in - 1];
+    num_bits = 1;
+    while ((bf & 1) != 1)
+      {
+	num_bits++;
+	bf >>= 1;
+      }
+    num_bits = l_in * 8 - num_bits;
+  }
+
+  DECODE_START (s_in, l_in)
+
+    which = DECODE_BIT;
+  bits = 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_DECODE_L (binary_start[0], bits);
+      DELTA_DECODE_L (binary_start[1], bits);
+    }
+
+
+  while (bits < num_bits)
+    {
+      register unsigned code, len;
+      register int r;
+      register u_char *t, *b = NULL;
+      u_char word[MAXWORDLEN + 1];
+
+#ifdef DUMPDOC
+      printf ("\n%d %d  ", bits, num_bits);
+#endif
+      if (cd->cfh[which])
+	{
+	  MY_HUFF_DECODE (len, code, cd->cfh[which]->hd.min_code);
+	  bits += len;
+
+	  r = code & ((1 << cd->cdh.lookback) - 1);
+	  t = cd->values[which][len][code >> cd->cdh.lookback];
+
+	  /* step through from base pointer */
+	  b = word + 1;
+	  while (r--)
+	    {
+	      register int copy = *t >> 4;
+	      memcpy (word + copy + 1, t + 1, *t & 0xf);
+	      word[0] = copy + (*t & 0xf);
+	      t += ((*t) & 0xf) + 1;
+	    }
+	}
+      else
+	t = NULL;
+      if (t == cd->escape[which])
+	{
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      {
+		int len, i;
+		int c;
+		HUFF_DECODE_L (len, cd->lens_huff[which]->min_code,
+			       cd->lens_vals[which], bits);
+		for (i = 0; i < len; i++)
+		  {
+		    HUFF_DECODE_L (c, cd->chars_huff[which]->min_code,
+				   cd->chars_vals[which], bits);
+		    *ptr++ = c;
+		  }
+	      }
+	      break;
+	    case MG_NOVEL_BINARY:
+	    case MG_NOVEL_DELTA:
+	    case MG_NOVEL_HYBRID:
+	    case MG_NOVEL_HYBRID_MTF:
+	      {
+		int idx = 0, len;
+		u_char *base;
+		switch (cd->cdh.novel_method)
+		  {
+		  case MG_NOVEL_BINARY:
+		    {
+		      BINARY_DECODE_L (idx, binary_start[which], bits);
+		      if (idx == binary_start[which])
+			binary_start[which]++;
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_DELTA:
+		    {
+		      DELTA_DECODE_L (idx, bits);
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID_MTF:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		      if (idx >= novels_used[which])
+			{
+			  u_char *temp;
+			  temp = ad->words[which][idx];
+			  ad->words[which][idx] =
+			    ad->words[which][novels_used[which]];
+			  ad->words[which][novels_used[which]] = temp;
+			  swaps[which][novels_used[which]] = idx;
+			  idx = novels_used[which]++;
+			}
+		    }
+		    break;
+		  }
+		base = ad->words[which][idx];
+		len = *base++;
+#ifdef DUMPDOC
+		printf ("[[");
+#endif
+		for (; len; len--)
+		  {
+		    *ptr++ = *base++;
+#ifdef DUMPDOC
+		    putchar (*(base - 1));
+#endif
+		  }
+#ifdef DUMPDOC
+		printf ("]]");
+#endif
+	      }
+	      break;
+	    }
+	}
+      else
+	{
+	  /* copy over the matching prefix */
+	  r = (*t >> 4);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *b++;
+#else
+	    {
+	      *ptr = *b++;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+
+	  /* and the stored suffix */
+	  r = ((*t) & 0xf);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *++t;
+#else
+	    {
+	      *ptr = *++t;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+	}
+      which = !which;
+    }
+
+  DECODE_DONE
+
+    * l_out = ptr - s_out;
+  num += *l_out + 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  u_char *temp;
+	  temp = ad->words[which][a];
+	  ad->words[which][a] = ad->words[which][b];
+	  ad->words[which][b] = temp;
+	}
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/text_get.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * text_get.h -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT_GET
+#define H_TEXT_GET
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+double FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		u_char * s_in, int l_in, u_char * s_out, int *l_out);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.h	(revision 21112)
@@ -0,0 +1,24 @@
+static char *warranty_str[] = {
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.src
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.src	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/warranty.src	(revision 21112)
@@ -0,0 +1,23 @@
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.c	(revision 21112)
@@ -0,0 +1,161 @@
+/**************************************************************************
+ *
+ * weights.c -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:27  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:18  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+approx_weights_data *
+LoadDocWeights (File * weight_file,
+		unsigned long num_of_docs)
+{
+  approx_weights_data *awd;
+  int num;
+
+  if (!(awd = Xmalloc (sizeof (*awd))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fseek (weight_file, sizeof (long), 0);
+  Fread (&awd->bits, sizeof (awd->bits), 1, weight_file);
+  Fread (&awd->L, sizeof (awd->L), 1, weight_file);
+  NTOHD(awd->L);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&awd->B, sizeof (awd->B), 1, weight_file);
+  NTOHD(awd->B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  awd->mask = awd->bits == 32 ? 0xffffffff : (1 << awd->bits) - 1;
+
+  num = (num_of_docs * awd->bits + 31) / 32;
+  if (!(awd->DocWeights = Xmalloc (sizeof (unsigned long) * num)))
+    {
+      Xfree (awd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (awd->DocWeights, sizeof (unsigned long), num, weight_file);
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    unsigned long i;
+    for (i = 0; i < num; i++)
+      NTOHUL(awd->DocWeights[i]);
+  }
+
+  awd->MemForWeights = num * sizeof (unsigned long);
+  awd->num_of_docs = num_of_docs;
+
+  mg_errno = MG_NOERROR;
+
+  if (awd->bits <= 12)
+    {
+      int i, size = (1 << awd->bits);
+      if (!(awd->table = (float *) Xmalloc (size * sizeof (float))))
+	  return (awd);
+      awd->table[0] = awd->L;
+      for (i = 1; i < size; i++)
+	awd->table[i] = awd->table[i - 1] * awd->B;
+    }
+  else
+    awd->table = NULL;
+  return awd;
+}
+
+
+
+
+
+
+/*
+ * the first document in the collection has DocNum = 0
+ */
+float 
+GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum)
+{
+  register unsigned long c, Pos;
+  register unsigned long *dw;
+  if (awd == NULL)
+    return 1.0;
+#if 0
+  if (DocNum < 0 || DocNum >= awd->num_of_docs)
+    FatalError (1, "Something is wrong in \"GetDocWeight\" DocNum = %d\n",
+		DocNum);
+#endif
+  Pos = DocNum * awd->bits;
+  dw = &(awd->DocWeights[Pos / MAXBITS]);
+  Pos &= (MAXBITS - 1);
+  c = *dw >> Pos;
+  if (Pos + awd->bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - Pos);
+  c &= awd->mask;
+  if (awd->table)
+    return (awd->table[c]);
+  else
+    return (awd->L * pow (awd->B, (double) c));
+
+}
+
+void 
+FreeWeights (approx_weights_data * awd)
+{
+  Xfree (awd->DocWeights);
+  if (awd->table)
+    Xfree (awd->table);
+  Xfree (awd);
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/weights.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * weights.h -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_WEIGHTS
+#define H_WEIGHTS
+
+approx_weights_data *LoadDocWeights (File * weight_file,
+				     unsigned long num_of_docs);
+
+float GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum);
+
+void FreeWeights (approx_weights_data * awd);
+
+#endif
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/win32.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/win32.mak	(revision 21112)
@@ -0,0 +1,241 @@
+###########################################################################
+#
+# win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..\..
+
+AR = lib
+
+CC = cl
+CPPFLAGS = 
+# DEFS = -DQUIET -D__WIN32__ -DSHORT_SUFFIX -DHAVE_STRING_H -DHAVE_FCNTL_H \
+#        -DSTDC_HEADERS -DHAVE_CONFIG_H
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I"." -I"$(MGHOME)" -I"$(MGHOME)\lib"
+COMPILE = $(CC) $(CPPFLAGS) -c $(DEFS) $(INCLUDES)
+
+LDFLAGS =
+LINK = $(CC) $(LDFLAGS)
+
+LIBS = "$(MGHOME)\lib\libmg.lib"
+
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h          query_term_list.h
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  mgpass.c                query_term_list.c       words.c              \
+  mg_passes_4jni.c
+
+EXEC = \
+  mgquery$e mg_weights_build$e mgstat$e \
+  mg_invf_dump$e     mg_invf_dict$e \
+  mg_invf_rebuild$e     mgdictlist$e \
+  mg_passes$e     mg_perf_hash_build$e \
+  mg_compression_dict$e     mg_text_estimate$e \
+  mg_fast_comp_dict$e \
+  mg_hilite_words$e     mg_invf_merge$e \
+  mg_text_merge$e     bool_tester$e  mgpass$e \
+  mg_stem_idx$e  mgstemidxlist$e \
+  mg_decompress_text$e 
+
+all: $(EXEC) libmgtextin.lib libmgpass.lib
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tester$o bool_tree$o bool_parser$o bool_optimiser$o  \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o \
+	    query_term_list$o words$o environment$o
+
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+PASS_OBJS = \
+	 mgpass$o words$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o environment$o
+mgpass$e: $(PASS_OBJS)
+	$(LINK) $(PASS_OBJS) $(LIBS)
+
+LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
+ advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
+ odbccp32.lib
+
+BUILDER_OBJS = \
+	builder$o
+builder$e: $(BUILDER_OBJS)
+	$(LINK) $(BUILDER_OBJS) $(LIBS) $(LINK32_FLAGS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o \
+		       mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+        bool_query$o words$o
+
+JNI_LIB_OBJS = \
+	mg_passes_4jni$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o  words$o environment$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o 
+
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+
+libmgtextin.lib : $(LIB_OBJS)
+        if exist libmgtextin.lib del libmgtextin.lib
+	$(AR) /out:libmgtextin.lib $(LIB_OBJS)
+
+libmgpass.lib : $(JNI_LIB_OBJS)
+ 	  if exist libmgpass.lib del libmgpass.lib
+	$(AR) /out:libmgpass.lib $(JNI_LIB_OBJS)
+
+install: 
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDL3HOME)\bin"
+
+clean:
+        if exist *$o del *$o 
+	if exist *$e del *$e
+        if exist libmgtextin.lib del libmgtextin.lib
+        if exist libmgpass.lib del libmgpass.lib
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.c	(revision 21112)
@@ -0,0 +1,34 @@
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skipped. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+
+/* It determines whether a given place in a UTF-8 encoded 
+   Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/src/text/words.h	(revision 21112)
@@ -0,0 +1,394 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+#include "unitool.h"
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+/*#define MAXNUMERIC	4*/
+
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded 
+	   Unicode string is a unicode space. */
+
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with 
+	   begining unicode spaces skipped. */ 
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+  /*  
+#define PARSE_WORD(Word, s_in, end)                                  \
+  do {                                                               \
+	  register u_char  *wptr = (Word)+1;                         \
+	  register int    length = 0;                                \
+	  register int    c = *(s_in);                               \
+	  register int	numeric = 0;                                 \
+                                                                     \
+	  while( length < MAXWORDLEN && INAWORD(c) && (s_in)<=(end)) \
+	    {                                                        \
+	      if ((numeric += INNUMBER(c)) > MAXNUMERIC)             \
+		break;                                               \
+	      *wptr++ = c;                                           \
+	      length++;                                              \
+	      c = *++(s_in);                                         \
+	    }                                                        \
+	  *(Word) = length;                                          \
+  }while(0)
+  */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+	  register u_char  *wptr = (Word)+1;                       \
+	  register int    length = 0;                              \
+	  register int    c = *(s_in);                             \
+                                                                   \
+	  while( length < MAXWORDLEN && !INAWORD(c) && (s_in)<=(end) ) \
+	    {                                                      \
+	      *wptr++ = c;                                         \
+	      length++;                                            \
+	      c = *++(s_in);                                       \
+	    }                                                      \
+	  *(Word) = length;                                        \
+  }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_STEM_WORD 
+ * Description: 
+ *      Extracts out Word.      
+ * Input: 
+ *      s_in points to 1st letter in buffer to test
+ *      end points to last letter in buffer
+ * Output: 
+ *      s_in is modified to move to next word
+ *      Returns Word filled in with length in 1st byte.
+ * ========================================================================= */
+#define PARSE_STEM_WORD(Word, s_in, end)                           \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXSTEMLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+    /*
+#define PARSE_STEM_WORD(Word, s_in, end)                      \
+  do                                                          \
+    {                                                         \
+      register u_char  *wptr = (Word)+1;                      \
+      register int    length = 0;                             \
+      register int    c = *(s_in);                            \
+      register int    numeric = 0;                            \
+                                                              \
+      while ( length < MAXSTEMLEN && INAWORD(c) && (s_in)<=(end)) \
+        {                                                     \
+ 	  if ((numeric += INNUMBER(c)) > MAXNUMERIC)          \
+	    break;                                            \
+	  *wptr++ = c;                                        \
+	  length++;                                           \
+	  c = *++(s_in);                                      \
+	}                                                     \
+      *(Word) = length;                                       \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD 
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD(s_in, end)                             \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD(s_in, end)           \
+  do                                             \
+    {                                            \
+      while (!INAWORD(*(s_in)) && (s_in)<=(end)) \
+	(s_in)++;                                \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD_OR_SGML_TAG 
+ * Description: 
+ *      Like PARSE_NON_STEM_WORD but also eats up SGML tags
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end)                 \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == '<') {                                              \
+	while (charlength > 0 && c != '>') {                       \
+	  (s_in) += charlength;                                    \
+	  charlength = parse_utf8_char((s_in),(end),&c);           \
+	}                                                          \
+      }                                                            \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end) \
+  do                                               \
+    {                                              \
+      register int    c = *(s_in);                 \
+                                                   \
+      while (!INAWORD(c) && (s_in)<=(end))         \
+        {                                          \
+	  if (c == '<')                            \
+            {                                      \
+	      while (c != '>' && (s_in)<=(end))    \
+		c = *++(s_in);                     \
+            }                                      \
+	  if ((s_in)<=(end))                       \
+	    c = *++(s_in);                         \
+	}                                          \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_OPT_TERM_PARAM     [RPAP - Jan 97: Stem Index Change]
+ * Description: 
+ *      Extracts out optional paramater for query term.
+ *      Needed only in parsing the query line !
+ *
+ *      Note: that this function has not been converted to use UTF-8
+ *            as it should still work as it is (only uses ascii
+ *            characters)
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_OPT_TERM_PARAM(Param, type, s_in, end)                       \
+  do                                                                       \
+    {                                                                      \
+	  register u_char  *wptr = (Param);                                \
+	  register int    length = 0;                                      \
+	  register int    c = *(s_in);                                     \
+                                                                           \
+          if (c == WEIGHTPARAM || c == STEMPARAM)                          \
+	    {                                                              \
+	      type = c;                                                    \
+	      c = *++(s_in);                                               \
+	      while( length < MAXPARAMLEN && isdigit(c) && (s_in)<=(end))  \
+		{                                                          \
+	           *wptr++ = c;                                            \
+	           length++;                                               \
+	           c = *++(s_in);                                          \
+	        }                                                          \
+	      *wptr = '\0';                                                \
+              for (; isdigit(c) && (s_in)<=(end); c = *++(s_in))           \
+                ;                                                          \
+            }							           \
+    }while(0)
+
+/* =========================================================================
+ * Macro: PARSE_RANKED_NON_STEM_WORD    [RJM 07/97: Ranked Required Terms]
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      If come across a match requirement store it in require_match
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above
+ * Output: the requirement mode for the next term. -1=must not match,
+ *      0=optional match, 1=must match
+ * ========================================================================= */
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)       \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+    (require_match) = 0;                                           \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == MUSTMATCHPARAM) (require_match) = 1;                \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)         \
+  do {                                            \
+    (require_match) = 0;                          \
+    while (!INAWORD(*(s_in)) && (s_in)<=(end)) {  \
+      if (*(s_in) == MUSTMATCHPARAM) {            \
+        (require_match) = 1;                      \
+      }                                           \
+      (s_in)++;                                   \
+    }                                             \
+  } while (0)
+    */
Index: /main/tags/3.01/gsdl3/src/packages/mg/stamp-h.in
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/gsdl3/src/packages/mg/stemmerinterface.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/stemmerinterface.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/stemmerinterface.c	(revision 21112)
@@ -0,0 +1,39 @@
+#include "lib/lovinstem.h"
+#include <stdio.h>
+#include <ctype.h>
+
+int main () {
+  char word[256];
+  char c;
+  int len;
+
+  c = fgetc (stdin);
+  while (!feof (stdin)) {
+    /* output non words */
+    while (!feof (stdin) && !isalnum (c)) {
+      fputc (c, stdout);
+      c = fgetc (stdin);
+    }
+
+    /* get the word */
+    len = 0;
+    while (!feof (stdin) && isalnum (c) && len < 255) {
+      len++;
+      word[len] = c;
+      c = fgetc (stdin);
+    }
+    word[0] = len;
+
+    /* if we found a word stem and output it */
+    if (len > 0) {
+      lovinstem (word);
+      len = 0;
+      while (len < word[0]) {
+        len++;
+        fputc(word[len], stdout);
+      }
+    }
+  }
+}
+
+
Index: /main/tags/3.01/gsdl3/src/packages/mg/sysfuncs.h
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/sysfuncs.h	(revision 21112)
@@ -0,0 +1,361 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#include <sys/stat.h>
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#if defined(__MSDOS__)
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#if defined(__GNUC__) && defined(__WIN32__)
+#include <limits.h>
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#if defined(__WIN32__)
+#include <io.h>
+#define lseek _lseek
+#elif defined(__MSDOS__)
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#include <pathmax.h>
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+#include <getopt.h>
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#if (HAVE_VPRINTF || HAVE_DOPRNT) && __STDC__
+/*void error __P ((int, int, const char *,...));*/
+void error (int, int, const char *,...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/Makefile
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/Makefile	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/Makefile	(revision 21112)
@@ -0,0 +1,10 @@
+DISTFILES = Makefile q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum simple_sum.c alice.len
+
+all: simple_sum
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+simple_sum: simple_sum.c
+	cc -o $@ $?
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/alice.len
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/alice.len	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/alice.len	(revision 21112)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19321
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/alice.sum
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/alice.sum	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/alice.sum	(revision 21112)
@@ -0,0 +1,8 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict.blocked: 1030626 33214
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc1	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query boolean
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc2
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc2	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc2	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query ranked
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc3
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc3	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/mgrc3	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode docnums
+.set query boolean
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/q1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/q1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/q1	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/q2
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/q2	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/q2	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/q3
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/q3	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/q3	(revision 21112)
@@ -0,0 +1,2 @@
+Rabbit & !(Alice | she)
+laser
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/r1
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/r1	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/r1	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/r2
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/r2	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/r2	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/r3
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/r3	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/r3	(revision 21112)
@@ -0,0 +1,21 @@
+> Rabbit & !(Alice | she)
+Rabbit Alice she
+     14.14     
+    126.126    
+    143.143    
+    478.478    
+    701.701    
+    704.704    
+    705.705    
+    755.755    
+    774.774    
+    786.786    
+    788.788    
+    791.791    
+    793.793    
+    801.801    
+    803.803    
+15 documents retrieved.
+> laser
+laser
+No entries correspond to that query.
Index: /main/tags/3.01/gsdl3/src/packages/mg/test/simple_sum.c
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/test/simple_sum.c	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/test/simple_sum.c	(revision 21112)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * sum.c -- byte summing program
+ * Copyright (C) 1995  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:20:24  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:18:33  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:36:00  rjmcnab
+*** empty log message ***
+
+*/
+
+/*
+ * If given -l option then just prints the length.
+ * Normally prints the sum and the length for the file.
+ */
+
+static int length_only = 0;
+
+#include <stdio.h>
+
+void 
+output_sum(fname, input_file, output_file)
+char *fname;
+FILE *input_file;
+FILE *output_file;
+{
+  int ch = '\0';
+  unsigned long sum = 0;
+  unsigned long num_bytes = 0;
+
+  while((ch = getc(input_file))!=EOF){
+    sum += ch;
+    num_bytes++;
+  }
+
+  if (length_only)
+    fprintf(output_file, "%s: %ld\n", fname, num_bytes);
+  else
+    fprintf(output_file, "%s: %ld %ld\n", fname, sum, num_bytes);
+}
+
+void
+main(argc, argv)
+int argc;
+char *argv[];
+{
+  int i = 0;
+
+  /* use stdin */
+  if (argc == 1){
+    output_sum("-", stdin, stdout);
+    exit(0);
+  }
+
+  /* use args as file names for input */
+  for (i=1;i<argc; i++){
+    char *fname = argv[i];
+    FILE *input_file = NULL;
+    
+    if (!length_only && (argv[i][0] == '-') && (argv[i][1] =='l') ) 
+      { 
+        length_only = 1; 
+        continue;
+      }
+
+    input_file = fopen(fname, "r");
+    if (!input_file){
+      fprintf(stderr,"Could not open %s\n", fname);
+      continue;
+    }
+    else{
+      output_sum(fname, input_file, stdout);
+      fclose(input_file);
+    }
+  }/*for*/
+  exit(0);
+
+}
Index: /main/tags/3.01/gsdl3/src/packages/mg/win32.mak
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/win32.mak	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/win32.mak	(revision 21112)
@@ -0,0 +1,76 @@
+###########################################################################
+#
+# win32 makefile -- mg
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+AR = lib
+CC = cl
+CPPFLAGS =
+
+MGHOME = $(MAKEDIR)
+DLL = 0
+DLLDEBUG = 0
+
+MDEFINES = /f win32.mak
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+SOURCES = 
+
+OBJECTS = 
+
+OTHERDIRS = lib 
+
+INSTALLDIRS = src\text jni 
+
+MAKECMD = $(MAKE) $(MDEFINES)
+!IF $(DLL)
+MAKECMD = $(MAKECMD) DLL=1
+!ENDIF
+!IF $(DLLDEBUG)
+MAKECMD = $(MAKECMD) DLLDEBUG=1
+!ENDIF
+
+all: win32.mak $(OTHERDIRS) $(INSTALLDIRS)
+
+java:	FORCE
+	cd "java\org\greenstone\mg"
+	call winMake.bat
+	call winMake.bat install 
+	cd "$(MGHOME)"
+
+
+install: win32.mak $(INSTALLDIRS)
+
+$(OTHERDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	cd "$(MGHOME)"
+
+$(INSTALLDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	$(MAKECMD) install
+	cd "$(MGHOME)"
+
+FORCE:
Index: /main/tags/3.01/gsdl3/src/packages/mg/winMake.bat
===================================================================
--- /main/tags/3.01/gsdl3/src/packages/mg/winMake.bat	(revision 21112)
+++ /main/tags/3.01/gsdl3/src/packages/mg/winMake.bat	(revision 21112)
@@ -0,0 +1,47 @@
+@echo off
+
+set MAKE=nmake
+set MAKE_OPTIONS=/f
+
+if ""%1"" == """" goto all
+if ""%1"" == ""compile"" goto go
+if ""%1"" == ""install"" goto go
+if ""%1"" == ""clean"" goto go
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:go
+	if ""%2"" == ""javaonly"" goto java
+	
+:all
+	cd lib
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd src\images
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..\..
+
+	cd src\text
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..\..
+
+	cd java\org\greenstone\mg
+	call winMake.bat %1
+	cd ..\..\..\..
+
+	cd jni
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+:java
+	cd java\org\greenstone\mg
+	call winMake.bat %1
+	cd ..\..\..\..
+	goto done
+
+:done
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractBrowse.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractBrowse.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractBrowse.properties	(revision 21112)
@@ -0,0 +1,3 @@
+ClassifierBrowse.name=Browse
+ClassifierBrowse.description=Browse pre-defined classification hierarchies
+ClassifierBrowse.classifier_help:Browse by {0}
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGPPSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGPPSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGPPSearch.properties	(revision 21112)
@@ -0,0 +1,30 @@
+#query params
+param.level=Granularity to search at
+param.sortBy=Document display order
+param.sortBy.1=ranked
+param.sortBy.0=natural
+param.fqv=Word or phrase 
+param.fqf=in field
+param.fqf.all=All fields
+param.fqs=stem
+param.fqc=case
+param.fqk=
+param.fqk.0=AND
+param.fqk.1=OR
+param.fqk.2=NOT
+
+#levels - this should go somewhere more central perhaps?
+level.Document=Document
+level.Doc=Document
+level.Section=Section
+level.Sec=Section
+level.Paragraph=Paragraph
+level.Para=Paragraph
+
+#service stuff
+FieldQuery.name=Form Search
+FieldQuery.description=Simple fielded search
+FieldQuery.submit=Search
+AdvancedFieldQuery.name=Advanced Search
+AdvancedFieldQuery.description=Advanced fielded searching
+AdvancedFieldQuery.submit=Search
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractMGSearch.properties	(revision 21112)
@@ -0,0 +1,1 @@
+# nothing extra needed at present
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/AbstractSearch.properties	(revision 21112)
@@ -0,0 +1,20 @@
+#service stuff
+TextQuery.name=Text Search
+TextQuery.description=Full text and/or metadata search service
+TextQuery.submit=Search
+
+#compulsory query params
+param.index=Index to search in
+param.maxDocs=Maximum hits to return
+param.query=Query string
+param.hitsPerPage=Hits per page
+param.startPage=Start from page
+
+#optional params
+param.stem=Turn stemming 
+param.case=Turn casefolding 
+param.matchMode=Match
+param.matchMode.all=all
+param.matchMode.some=some
+param.boolean.on=on
+param.boolean.off=off
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/GATEServices.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/GATEServices.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/GATEServices.properties	(revision 21112)
@@ -0,0 +1,11 @@
+# Service parameters
+param.annotationType=Annotation type
+param.annotationType.Address=Address
+param.annotationType.Date=Date
+param.annotationType.Location=Places
+param.annotationType.Organization=Organization
+param.annotationType.Person=People
+
+#service stuff
+GatePOSTag.name=Annotate document
+GatePOSTag.submit=Go
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct.properties	(revision 21112)
@@ -0,0 +1,67 @@
+#the params used by the services
+param.collection=Collection
+param.creator=Creator's email address
+param.collTitle=Collection title
+param.collAbout=Collection description
+param.buildType=Building type
+param.buildType.mg=MG
+param.buildType.mgpp=MGPP
+
+#service names and submit buttons
+DeleteCollection.name=Delete
+DeleteCollection.description=Delete a collection 
+DeleteCollection.submit=Delete
+NewCollection.name=Create
+NewCollection.description=Create a new collection
+NewCollection.submit=Create
+ImportCollection.name=Import
+ImportCollection.description=Import a collection: take the original documents and convert them to Greenstone internal format
+ImportCollection.submit=Import
+BuildCollection.name=Build
+BuildCollection.description=Build a collection: create the indexes and browsable hierarchies.
+BuildCollection.submit=Build
+ActivateCollection.name=Activate
+ActivateCollection.description=Activate a collection: Replace the old built collection with the new version
+ActivateCollection.submit=Activate
+ReloadCollection.name=Reload
+ReloadCollection.description=Reload a collection into the library
+ReloadCollection.submit=Reload
+
+#error messages used by each service - {0} is the collection name
+#DeleteCollection######
+delete.configure_error=Couldn't deactivate {0} from the library.
+delete.delete_error=Couldn't delete {0}.
+delete.success=Collection {0} deleted successfully.
+delete.exists_error=Collection {0} did not exist and couldn't be deleted.
+
+#NewCollection######
+new.success=Collection {0} created successfully.
+
+#ImportCollection#####
+import.success=Collection {0} imported successfully.
+
+#BuildCollection#####
+build.success=Collection {0} built successfully.
+
+#ActivateCollection#####
+activate.build_dir_error=Collection {0} had no building directory: couldn't be activated.
+activate.build_config_error=Collection {0} had no building config file: couldn't be activated.
+activate.rm_index_error=Couldn't remove the index directory.
+activate.rename_building_error=Couldn't rename building to index.
+activate.success=Collection {0} activated successfully.
+
+#ReloadCollection#####
+reload.success=Collection {0} reloaded successfully.
+reload.activate_error=Couldn't activate collection {0} in the library.
+
+#general stuff
+general.process_id_error=Invalid process id.
+general.configure_constructor_error=Couldn't configure the collection constructor.
+general.process_start=Starting process...
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct_fr.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct_fr.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/GS2Construct_fr.properties	(revision 21112)
@@ -0,0 +1,51 @@
+#the params used by the services
+param.collection=Collection <fr>
+param.creator=Creator's email address <fr>
+param.collectionName=Name of the collection <fr>
+
+#service names and submit buttons
+DeleteCollection.name=Delete a collection <fr>
+DeleteCollection.submit=Delete <fr>
+
+#messages for each service
+#DeleteCollection######
+delete.deactivate_error=Couldn't deactivate {0} from the library. <fr>
+delete.delete_error=Couldn't delete {0}. <fr>
+delete.success=Collection {0} deleted successfully. <fr>
+delete.submit=Delete <fr>
+
+#NewCollection######
+new.success=Collection {0} created successfully. <fr>
+new.submit=Create <fr>
+
+#ImportCollection#####
+import.success=Collection {0} imported successfully. <fr>
+import.submit=Import <fr>
+
+#BuildCollection#####
+build.success=Collection {0} built successfully. <fr>
+build.submit=Build <fr>
+
+#ActivateCollection#####
+#activate.build_dir_error=Collection {0} had no building directory: couldn't be activated. <fr>
+activate.build_config_error=Collection {0} had no building config file: couldn't be activated. <fr>
+activate.rm_index_error=Couldn't remove the index directory. <fr>
+activate.rename_building_error=Couldn't rename building to index. <fr>
+activate.success=Collection {0} activated successfully. <fr>
+activate.submit=Activate <fr>
+
+#ReloadCollection#####
+reload.success=Collection {0} reloaded successfully. <fr>
+reload.activate_error=Couldn't activate collection {0} in the library. <fr>
+reload.submit=Reload <fr>
+
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/IViaSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/IViaSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/IViaSearch.properties	(revision 21112)
@@ -0,0 +1,7 @@
+#query params
+param.index.au=Author
+param.index.ti=Title
+param.index.kw=Keywords
+param.index.su=Subject
+param.index.de=Description
+param.index.fu=Full Text
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/LuceneSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/LuceneSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/LuceneSearch.properties	(revision 21112)
@@ -0,0 +1,5 @@
+TextQuery.name=Search
+TextQuery.submit=Search
+TextQuery.description=A simple XML search service
+
+param.query=Enter your query string
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapRetrieve.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapRetrieve.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapRetrieve.properties	(revision 21112)
@@ -0,0 +1,3 @@
+#service stuff
+MapRetrieve.name=Maps Retrieved
+DocumentMetadataRetrieve.name=Retrieved Documents
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapSearch.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapSearch.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/MapSearch.properties	(revision 21112)
@@ -0,0 +1,4 @@
+#service stuff
+TextQuery.name=Map Search
+TextQuery.description=Search for placenames in maps
+TextQuery.submit=Search
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/PhindPhraseBrowse.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/PhindPhraseBrowse.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/PhindPhraseBrowse.properties	(revision 21112)
@@ -0,0 +1,2 @@
+PhindApplet.name=Phrase browse
+PhindApplet.description=Java applet that allows you to browse extracted phrase hierarchies
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/TestServlet.java
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/TestServlet.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/TestServlet.java	(revision 21112)
@@ -0,0 +1,15 @@
+import java.io.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+
+/** a test servlet to make sure the servlet container is working */
+public class TestServlet extends HttpServlet {
+
+    public void doGet(HttpServletRequest request,
+		      HttpServletResponse response)
+	throws ServletException, IOException {
+
+	PrintWriter out = response.getWriter();
+	out.println("Hello Greenstone!");
+    }
+}
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/Visualizer.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/Visualizer.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/Visualizer.properties	(revision 21112)
@@ -0,0 +1,2 @@
+VisApplet.name=Query Visualizer
+VisApplet.description=Visualize your query...
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/config_format.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/config_format.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/config_format.xsl	(revision 21112)
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:xslt="output.xsl"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:gsf="http://www.greenstone.org/configformat"
+  extension-element-prefixes="java">
+  
+  <xsl:output method="xml"/>
+  <xsl:namespace-alias
+    stylesheet-prefix="xslt" result-prefix="xsl"/>
+
+
+  <xsl:template match="format">
+    <format hello="yes">
+      <xsl:apply-templates/>
+    </format>
+  </xsl:template>
+  
+  <xsl:template match="gsf:template">
+    <xslt:template>
+      <xsl:copy-of select="@*"/>
+      <xsl:attribute name="priority">2</xsl:attribute> 
+      <xsl:if test=".//gsf:link/@type">
+	<xslt:param name="serviceName">service-name</xslt:param>
+      </xsl:if>
+      <xsl:if test=".//gsf:link">
+	<xslt:param name="collName">coll-name</xslt:param>
+	<xslt:variable name='library' select='ancestor::page/config/library_name'/>
+      </xsl:if>
+      <xsl:apply-templates/>
+    </xslt:template>
+  </xsl:template>
+  
+  <xsl:template match="gsf:link">
+    <xsl:choose>
+      <xsl:when test="@type='classifier'">
+	<a><xslt:attribute name='href'><xslt:value-of select='$library'/>?a=b&amp;s=<xslt:value-of select='$serviceName'/>&amp;c=<xslt:value-of select='$collName'/>&amp;cl=<xslt:value-of select='@nodeID'/><xslt:if test="classifierNode|documentNode">.pr</xslt:if></xslt:attribute>
+	  <xsl:apply-templates/>
+	</a>
+      </xsl:when>
+      <xsl:otherwise> <!-- a document link -->
+	<a><xsl:attribute name="href">{$library}?a=d&amp;c={$collName}&amp;d={@nodeID}</xsl:attribute>
+	  <xsl:apply-templates/>
+	</a>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="gsf:icon">
+    <xsl:choose>
+      <xsl:when test="@type='classifier'">
+	<img src='interfaces/default/images/bshelf.gif' border='0' width='20' height='16' />
+      </xsl:when>
+      <xsl:otherwise>
+	<xslt:apply-templates select="." mode="displayNodeIcon"/>
+	<!--
+	<img src='interfaces/default/images/book.gif' width='18' height='11' border='0'/>-->
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="gsf:metadata">    
+    <xslt:value-of><xsl:attribute name="select">metadataList/metadata[@name='<xsl:value-of select="@name"/>']</xsl:attribute></xslt:value-of>
+  </xsl:template>
+
+<!--
+  <xsl:template match="gsf:text">
+TODO
+  </xsl:template>
+-->
+  <xsl:template match="*">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+
+
+<!-- copied from style for now-->
+<!-- Display the appropriate image, depending on the node type -->
+<xsl:template match="documentNode" mode="displayNodeIcon">
+
+  <!-- Root node: book icon (open or closed) -->
+  <xsl:if test="@nodeType='root'">
+    <xsl:choose>
+      <xsl:when test="documentNode">
+	<img border="0" width="28" height="23"
+	     src="interfaces/default/images/openbook.gif"
+	     alt="Close this book"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<img border="0" width="18" height="11"
+	     src="interfaces/default/images/book.gif"
+	     alt="Open this document and view contents"/>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:if>
+
+  <!-- Interior node: folder icon (open or closed) -->
+  <xsl:if test="@nodeType='interior'">
+    <xsl:choose>
+      <xsl:when test="documentNode">
+	<img border="0" width="23" height="15"
+	     src="interfaces/default/images/openfldr.gif"
+	     alt="Close this folder"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<img border="0" width="23" height="15"
+	     src="interfaces/default/images/clsdfldr.gif"
+	     alt="Open this folder and view contents"/>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:if>
+
+  <!-- Leaf node: page icon -->
+  <xsl:if test="@nodeType='leaf'">
+    <img border="0" width="16" height="21"
+	 src="interfaces/default/images/itext.gif"
+	 alt="View the document"/>
+  </xsl:if>
+</xsl:template>
+
+</xsl:stylesheet>
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_classic.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_classic.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_classic.properties	(revision 21112)
@@ -0,0 +1,110 @@
+nzdl=New Zealand Digital Library
+home_b=HOME
+pref_b=PREFERENCES
+help_b=HELP
+# text for the titles (in the page header)
+help_t=help
+pref_t=preferences
+about_t=about
+search_t=search
+phrases_t=phrases
+texticonpdf=View the PDF document
+texticonps=View the PostScript document
+texticonrtf=View the RTF document
+texticonmsword=View the Microsoft Word document
+texticonppt=View the PowerPoint document
+texticonxls=View the Microsoft Excel document
+#################
+# home page
+#################
+home.feedback=Feedback
+home.aboutgs=About<br/>Greenstone
+home.people=People
+home.research=Research
+home.projects=Projects
+home.software=Software
+home.publications=Publications
+home.coll_warning=This interface is designed to look and behave like the classic Greenstone 2 style interface. It makes assumptions about the services that a collection might have: it looks for the TextQuery/FormQuery/AdvancedFormQuery and ClassifierBrowse services. If a query service is present, a search button is added to the navigation bar. If more than one query service is available they come under the same button, and are selectable via the preferences page. If ClassifierBrowse is present, the classifiers provided by that service are displayed on the navigation bar instead of the service itself. Any other services are ignored. Greenstone 2 collections work exactly as they would in a Greenstone 2 library, while some other collections do not work properly.
+##################
+# about page
+##################
+aboutpage=About page
+#about.services=Services available:
+#about.description=Description:
+about.aboutcoll=About this collection
+about.howtofind=How to find information in the {0} collection
+##################
+# prefs page
+##################
+pref.searchpref=Search preferences
+pref.prespref=Presentation preferences
+pref.interfacelang=Interface language:
+pref.encoding=Encoding:
+##################
+# query page
+##################
+query.wordcount=Word count:
+query.nodocsmatch=No documents matched the query.
+query.onedocsmatch=One document matched the query.
+query.manydocsmatch={0} documents matched the query.
+query.matches=Matches
+query.results=results
+##################
+# help page
+##################
+help.numbrowseoptions=There are {0} ways to find information in this collection:
+help.simplehelpheading=How to find information in the {0} collection
+help.Searchshort=search for particular words
+help.Searchlong=<p />You can <i>search for particular words</i> that appear in the text from the "search" page. This is the first page that comes up when you begin, and can be reached from other pages by pressing the <i>search</i> button.
+help.Titleshort=access publications by title
+help.Titlelong=<p />You can <i>access publications by title</i> by pressing the <i>titles</i> button.  This brings up a list of books in alphabetic order.
+help.Organizationshort=access publications by organisation
+help.Organizationlong=<p />You can <i>access publications by organisation</i> by pressing the <i>organisations</i> button.  This brings up a list of organisations.
+help.Organisationshort=access publications by organisation
+help.Organisationlong=<p />You can <i>access publications by organisation</i> by pressing the <i>organisations</i> button.  This brings up a list of organisations.
+help.HowToshort=access publications by "how to" listing
+help.HowTolong=<p />You can <i>access publications by "how to" listing</i> by pressing the <i>how to</i> button.  This brings up a list of how to strings.
+help.Howtoshort=access publications by "how to" listing
+help.Howtolong=<p />You can <i>access publications by "how to" listing</i> by pressing the <i>how to</i> button.  This brings up a list of how to strings.
+help.Creatorshort=access publications by author
+help.Creatorlong=<p />You can <i>access publications by author</i> by pressing the <i>authors a-z</i> button.  This brings up a list of books, sorted by author name.  
+help.Subjectshort=access publications by subject
+help.Subjectlong=<p/>You can <i>access publications by subject</i> by pressing the <i>subjects</i> button.  This brings up a list of subjects, represented by bookshelves.
+help.Topicshort=access publications by topic
+help.Topiclong=<p>You can <i>access publications by topic</i> by pressing the <i>topic</i> button.  This brings up a list of topics to browse.  
+help.Toshort=access publications by To field
+help.Tolong=<p>You can <i>access publications by To field</i> by pressing the <i>to</i> button.  This brings up a list of addressees.  
+help.Fromshort=access publications by From field
+help.Fromlong=<p>You can <i>access publications by From field</i> by pressing the <i>from</i> button.  This brings up a list of senders.  
+help.Seriesshort=access publications by series
+help.Serieslong=<p>You can <i>access publications by series</i> by pressing the <i>series</i> button.  This brings up a list of those series which are currently in the collection.  
+help.Dateshort=access publications by date
+help.Datelong=<p>You can <i>access publications by date</i> by pressing the <i>dates</i> button.  This brings up a list of all the issues, sorted chronologically.  
+help.Browseshort=browse publications
+help.Browselong=<p>You can <i>browse publications</i> by pressing the <i>browse</i> button.  
+help.Peopleshort=access publications by people
+help.Peoplelong=<p>You can <i>access publications by people</i> by pressing the <i>people a-z</i> button.  This brings up a list of entries, sorted by surname.  
+help.Languageshort=access publications by language
+help.Languagelong=<p>You can <i>access publications by language</i> by pressing the <i>languages</i> button.  This brings up a list of entries, sorted by language.  
+help.Acronymshort=access publications by acronym
+help.Acronymlong=<p>You can <i>access publications by acronym occurrence</i> bypressing the <i>acronyms</i> button.  This brings up a list of the acronyms, and the places that they occur.  
+help.Phraseshort=browse phrases occurring in publications
+help.Phraselong=<p>You can <i>browse phrases occurring in publications</i> by pressing the <i>phrases</i> button.  This uses the phind phrase browser.  
+help.Artistshort=access publications by artist
+help.Artistlong=<p>You can <i>access publications by artist</i> by pressing the <i>artists</i> button.  This brings up a list of entries, sorted by artist name.  
+help.Sourceshort=access publications by filename
+help.Sourcelong=<p>You can <i>access publications by filename</i> by pressing the <i>filenames</i> button.  This brings up a list of entries, sorted by original filename.  
+help.Keywordshort=access publications by keyword
+help.Keywordlong=<p>You can <i>access publications by keyword</i> by pressing the <i>keywords</i> button.  This brings up a list of keywords. 
+help.Volumeshort=access publications by volume
+help.Volumelong=<p>You can <i>access publications by volume</i> by pressing the <i>volumes</i> button. This brings up a list of volumes. 
+help.Captionsshort=access publications by caption
+help.Captionslong=<p>You can <i>access publications by caption</i> by pressing the <i>captions</i> button. This brings up a list of captions. 
+help.Countriesshort=access publications by country
+help.Countrieslong=<p>You can <i>access publications by country</i> by pressing the <i>countries</i> button. This brings up a list of countries. 
+help.short=undefined classification
+help.defaultshort=undefined classification
+help.long=<p>Click on the <i>unknown</i> button to browse publications (listing is sorted by an unknown field).  
+help.defaultlong=<p>Click on the <i>unknown</i> button to browse publications (listing is sorted by an unknown field).  
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default.properties	(revision 21112)
@@ -0,0 +1,65 @@
+gsdl=Greenstone Digital Library Software
+gs3power=powered by greenstone3
+# the buttons
+home_b=HOME
+help_b=HELP
+pref_b=PREFERENCES
+# tool tips for the buttons
+home_tip=Library home page
+help_tip=Help pages
+pref_tip=Change your interface preferences
+aboutpage=About page
+textframebrowser=You must have a frame enabled browser to view this.
+#######################
+# query page
+########################
+about.noservices=There are no services available for this collection.
+about.services=Services available:
+about.servicehelp=You can access the services by clicking on the buttons below or on the names on the navigation bar. The navigation bar is persistent across all pages in the collection.
+#about.description=Description:
+about.metadata=Metadata:
+
+#######################
+# query page
+########################
+query.wordcount=Word count:
+query.nodocsmatch=No documents matched the query.
+query.onedocsmatch=One document matched the query.
+query.manydocsmatch={0} documents matched the query.
+query.atleastdocsmatch=At least {0} documents matched the query.
+query.docsreturned={0} documents returned.
+query.matches=Matches
+query.results=Results
+###############
+# document page
+##################
+document.pageof=Page {0} of {1}.
+document.pages={0} pages.
+document.gotopage=Go to page
+
+##################
+# prefs page
+##################
+pref.searchpref=Search preferences
+pref.prespref=Presentation preferences
+pref.interfacelang=Interface language:
+pref.encoding=Encoding:
+pref.hitsperpage=Number of hits per page:
+pref.all=all
+pref.set_prefs=Set preferences
+
+
+##################
+# docs page
+##################
+doc.expand_doc_b=EXPAND<br />DOCUMENT
+doc.expand_doc_tip=Display the entire document
+doc.contract_doc_b=CONTRACT<br />DOCUMENT
+doc.contract_doc_tip=Return to previously selected section
+doc.expand_contents_b=EXPAND<br />CONTENTS 
+doc.expand_contents_tip=Expand only the table of contents
+doc.contract_contents_b=CONTRACT<br />CONTENTS
+doc.contract_contents_tip=Collapse the table of contents
+doc.detach_page_b=DETACH<br />PAGE
+doc.detach_page_tip=Open this page in a new window
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default_fr.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default_fr.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_default_fr.properties	(revision 21112)
@@ -0,0 +1,6 @@
+nzdl=Bibliothèque Numérique Greenstone
+home=ACCUEIL
+aboutpage=About page (fr)
+about.services=Services available (fr):
+about.description=Description (fr):
+query.matches=Résultats
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_nzdl.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_nzdl.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/interface_nzdl.properties	(revision 21112)
@@ -0,0 +1,107 @@
+nzdl=New Zealand Digital Library
+home_b=HOME
+pref_b=PREFERENCES
+help_b=HELP
+# text for the titles (in the page header)
+help_t=help
+pref_t=preferences
+about_t=about
+search_t=search
+phrases_t=phrases
+#################
+# home page
+#################
+home.feedback=Feedback
+home.aboutgs=About<br/>Greenstone
+home.people=People
+home.research=Research
+home.projects=Projects
+home.software=Software
+home.publications=Publications
+##################
+# about page
+##################
+aboutpage=About page
+#about.services=Services available:
+#about.description=Description:
+about.aboutcoll=About this collection
+about.howtofind=How to find information in the {0} collection
+##################
+# prefs page
+##################
+pref.searchpref=Search preferences
+pref.prespref=Presentation preferences
+pref.interfacelang=Interface language:
+pref.encoding=Encoding:
+##################
+# query page
+##################
+query.wordcount=Word count:
+query.nodocsmatch=No documents matched the query.
+query.onedocsmatch=One document matched the query.
+query.manydocsmatch={0} documents matched the query.
+query.matches=Matches
+query.results=results
+##################
+# help page
+##################
+help.numbrowseoptions=There are {0} ways to find information in this collection:
+help.simplehelpheading=How to find information in the {0} collection
+help.Searchshort=search for particular words
+help.Searchlong=<p />You can <i>search for particular words</i> that appear in the text from the "search" page. This is the first page that comes up when you begin, and can be reached from other pages by pressing the <i>search</i> button.
+help.Titleshort=access publications by title
+help.Titlelong=<p />You can <i>access publications by title</i> by pressing the <i>titles</i> button.  This brings up a list of books in alphabetic order.
+help.Organizationshort=access publications by organisation
+help.Organizationlong=<p />You can <i>access publications by organisation</i> by pressing the <i>organisations</i> button.  This brings up a list of organisations.
+help.Organisationshort=access publications by organisation
+help.Organisationlong=<p />You can <i>access publications by organisation</i> by pressing the <i>organisations</i> button.  This brings up a list of organisations.
+help.HowToshort=access publications by "how to" listing
+help.HowTolong=<p />You can <i>access publications by "how to" listing</i> by pressing the <i>how to</i> button.  This brings up a list of how to strings.
+help.Howtoshort=access publications by "how to" listing
+help.Howtolong=<p />You can <i>access publications by "how to" listing</i> by pressing the <i>how to</i> button.  This brings up a list of how to strings.
+help.Creatorshort=access publications by author
+help.Creatorlong=<p />You can <i>access publications by author</i> by pressing the <i>authors a-z</i> button.  This brings up a list of books, sorted by author name.  
+help.Subjectshort=access publications by subject
+help.Subjectlong=<p/>You can <i>access publications by subject</i> by pressing the <i>subjects</i> button.  This brings up a list of subjects, represented by bookshelves.
+help.Topicshort=access publications by topic
+help.Topiclong=<p>You can <i>access publications by topic</i> by pressing the <i>topic</i> button.  This brings up a list of topics to browse.  
+help.Toshort=access publications by To field
+help.Tolong=<p>You can <i>access publications by To field</i> by pressing the <i>to</i> button.  This brings up a list of addressees.  
+help.Fromshort=access publications by From field
+help.Fromlong=<p>You can <i>access publications by From field</i> by pressing the <i>from</i> button.  This brings up a list of senders.  
+help.Seriesshort=access publications by series
+help.Serieslong=<p>You can <i>access publications by series</i> by pressing the <i>series</i> button.  This brings up a list of those series which are currently in the collection.  
+help.Dateshort=access publications by date
+help.Datelong=<p>You can <i>access publications by date</i> by pressing the <i>dates</i> button.  This brings up a list of all the issues, sorted chronologically.  
+help.Browseshort=browse publications
+help.Browselong=<p>You can <i>browse publications</i> by pressing the <i>browse</i> button.  
+help.Peopleshort=access publications by people
+help.Peoplelong=<p>You can <i>access publications by people</i> by pressing the <i>people a-z</i> button.  This brings up a list of entries, sorted by surname.  
+help.Languageshort=access publications by language
+help.Languagelong=<p>You can <i>access publications by language</i> by pressing the <i>languages</i> button.  This brings up a list of entries, sorted by language.  
+help.Acronymshort=access publications by acronym
+help.Acronymlong=<p>You can <i>access publications by acronym occurrence</i> bypressing the <i>acronyms</i> button.  This brings up a list of the acronyms, and the places that they occur.  
+help.Phraseshort=browse phrases occurring in publications
+help.Phraselong=<p>You can <i>browse phrases occurring in publications</i> by pressing the <i>phrases</i> button.  This uses the phind phrase browser.  
+help.Artistshort=access publications by artist
+help.Artistlong=<p>You can <i>access publications by artist</i> by pressing the <i>artists</i> button.  This brings up a list of entries, sorted by artist name.  
+help.Sourceshort=access publications by filename
+help.Sourcelong=<p>You can <i>access publications by filename</i> by pressing the <i>filenames</i> button.  This brings up a list of entries, sorted by original filename.  
+help.Keywordshort=access publications by keyword
+help.Keywordlong=<p>You can <i>access publications by keyword</i> by pressing the <i>keywords</i> button.  This brings up a list of keywords. 
+help.Volumeshort=access publications by volume
+help.Volumelong=<p>You can <i>access publications by volume</i> by pressing the <i>volumes</i> button. This brings up a list of volumes. 
+help.Captionsshort=access publications by caption
+help.Captionslong=<p>You can <i>access publications by caption</i> by pressing the <i>captions</i> button. This brings up a list of captions. 
+help.Countriesshort=access publications by country
+help.Countrieslong=<p>You can <i>access publications by country</i> by pressing the <i>countries</i> button. This brings up a list of countries. 
+help.short=undefined classification
+help.defaultshort=undefined classification
+help.long=<p>Click on the <i>unknown</i> button to browse publications (listing is sorted by an unknown field).  
+help.defaultlong=<p>Click on the <i>unknown</i> button to browse publications (listing is sorted by an unknown field).  
+#################################
+#stuff for individual collections
+#################################
+collect.SearchBySender=search by sender
+
+
Index: /main/tags/3.01/gsdl3/web/WEB-INF/classes/metadata_names.properties
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/classes/metadata_names.properties	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/classes/metadata_names.properties	(revision 21112)
@@ -0,0 +1,34 @@
+Subject=Subject
+Subject.buttonname=subjects
+Title=Title
+Title.buttonname=titles
+Organisation=Organisation
+Organisation.buttonname=organisations
+Organization=Organisation
+Organization.buttonname=organisations
+Keyword=Keyword
+Keyword.buttonname=keywords
+HowTo=How To
+HowTo.buttonname=how to
+Howto=How To
+Howto.buttonname=how to
+Creator=Creator
+Creator.buttonname=creators
+Date=Date
+Date.buttonname=dates
+From=From
+From.buttonname=senders
+To=To
+To.buttonname=recipients
+Source=Source
+Source.buttonname=filenames
+Acronym=Acronym
+Acronym.buttonname=acronyms
+Topic=Topic
+Topic.buttonname=topics
+Collage=Collage
+Collage.buttonname=collage
+Countries=Countries
+Countries.buttonname=countries
+Language=Language
+Language.buttonname=languages
Index: /main/tags/3.01/gsdl3/web/WEB-INF/web.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/WEB-INF/web.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/WEB-INF/web.xml	(revision 21112)
@@ -0,0 +1,286 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+
+<!DOCTYPE web-app 
+    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" 
+    "http://java.sun.com/dtd/web-app_2_3.dtd">
+
+<!-- this is based on a sample web.xml file provided by 
+http://jakarta.apache.org/tomcat/tomcat-4.0-doc/appdev/web.xml.txt
+-->
+<web-app>
+
+
+  <!-- General description of your web application -->
+  
+  <display-name>GSDL3</display-name>
+  <description>
+    Greenstone digital library version 3.
+  </description>
+  
+  
+  <!-- Context initialization parameters that define shared
+  String constants used within your application, which
+  can be customized by the system administrator who is
+  installing your application.  The values actually
+  assigned to these parameters can be retrieved in a
+  servlet or JSP page by calling:
+  
+  String value =
+  getServletContext().getInitParameter("name");
+  
+  where "name" matches the <param-name> element of
+  one of these initialization parameters.
+  
+  You can define any number of context initialization
+  parameters, including zero.
+  -->
+  
+  <context-param>
+    <param-name>webmaster</param-name>
+    <param-value>greenstone3@list.scms.waikato.ac.nz</param-value>
+    <description>
+      The EMAIL address of the administrator to whom questions
+      and comments about this application should be addressed.
+    </description>
+  </context-param>
+  
+
+  <!-- Servlet definitions for the servlets that make up
+  your web application, including initialization
+  parameters.  With Tomcat, you can also send requests
+  to servlets not listed here with a request like this:
+
+  http://localhost:8080/{context-path}/servlet/{classname}
+  
+  but this usage is not guaranteed to be portable.  It also
+  makes relative references to images and other resources
+  required by your servlet more complicated, so defining
+  all of your servlets (and defining a mapping to them with
+  a servlet-mapping element) is recommended.
+  
+  Servlet initialization parameters can be retrieved in a
+  servlet or JSP page by calling:
+  
+  String value =
+  getServletConfig().getInitParameter("name");
+  
+  where "name" matches the <param-name> element of
+  one of these initialization parameters.
+  
+  You can define any number of servlets, including zero.
+  -->
+
+  <servlet>
+    <servlet-name>tester</servlet-name>
+    <description>a test servlet</description>
+    <servlet-class>TestServlet</servlet-class>
+  </servlet>
+  
+  <servlet>
+    <servlet-name>library</servlet-name>
+    <description>standard gsdl3 library program: uses localsite and the default interface</description>
+    <servlet-class>org.greenstone.gsdl3.LibraryServlet</servlet-class>
+    <init-param>
+      <param-name>library_name</param-name>
+      <param-value>library</param-value>
+    </init-param>
+    <init-param>
+      <param-name>site_name</param-name>
+      <param-value>localsite</param-value>
+    </init-param>
+    <init-param>
+      <param-name>interface_name</param-name>
+      <param-value>default</param-value>
+    </init-param>
+    <init-param>
+      <param-name>default_lang</param-name>
+      <param-value>en</param-value>
+    </init-param>
+  </servlet>
+  
+  <servlet>
+    <servlet-name>gateway</servlet-name>
+    <description>gsdl3 library based on gateway site and default interface. It uses SOAP to talk to localsite</description>
+    <servlet-class>org.greenstone.gsdl3.LibraryServlet</servlet-class>
+    <init-param>
+      <param-name>library_name</param-name>
+      <param-value>gateway</param-value>
+    </init-param>
+    <init-param>
+      <param-name>site_name</param-name>
+      <param-value>gateway</param-value>
+    </init-param>
+    <init-param>
+      <param-name>interface_name</param-name>
+      <param-value>default</param-value>
+    </init-param>
+    <init-param>
+      <param-name>default_lang</param-name>
+      <param-value>en</param-value>
+    </init-param>
+  </servlet>
+  
+  <servlet>
+    <servlet-name>classic-library</servlet-name>
+    <description>Greenstone 2 lookalike library program: uses localsite with the classic interface</description>
+    <servlet-class>org.greenstone.gsdl3.LibraryServlet</servlet-class>
+    <init-param>
+      <param-name>library_name</param-name>
+      <param-value>classic</param-value>
+    </init-param>
+    <init-param>
+      <param-name>site_name</param-name>
+      <param-value>localsite</param-value>
+    </init-param>
+    <init-param>
+      <param-name>interface_name</param-name>
+      <param-value>classic</param-value>
+    </init-param>
+    <init-param>
+      <param-name>default_lang</param-name>
+      <param-value>en</param-value>
+    </init-param>
+    <init-param>
+      <param-name>params_class</param-name>
+      <param-value>NZDLParams</param-value>
+    </init-param>
+  </servlet>
+
+  <!-- axis servlets -->
+  <servlet>
+    <servlet-name>AxisServlet</servlet-name>
+    <display-name>Apache-Axis Servlet</display-name>
+    <servlet-class>
+        org.apache.axis.transport.http.AxisServlet
+    </servlet-class>
+  </servlet>
+
+  <servlet>
+    <servlet-name>AdminServlet</servlet-name>
+    <display-name>Axis Admin Servlet</display-name>
+    <servlet-class>
+        org.apache.axis.transport.http.AdminServlet
+    </servlet-class>
+    <load-on-startup>100</load-on-startup>
+  </servlet>
+
+  <servlet>
+    <servlet-name>SOAPMonitorService</servlet-name>
+    <display-name>SOAPMonitorService</display-name>
+    <servlet-class>
+        org.apache.axis.monitor.SOAPMonitorService
+    </servlet-class>
+    <init-param>
+      <param-name>SOAPMonitorPort</param-name>
+      <param-value>5001</param-value>
+    </init-param>
+    <load-on-startup>100</load-on-startup>
+  </servlet>
+    <!-- Define mappings that are used by the servlet container to
+         translate a particular request URI (context-relative) to a
+         particular servlet.  The examples below correspond to the
+         servlet descriptions above.  Thus, a request URI like:
+
+           http://localhost:8080/{contextpath}/graph
+
+         will be mapped to the "graph" servlet, while a request like:
+
+           http://localhost:8080/{contextpath}/saveCustomer.do
+
+         will be mapped to the "controller" servlet.
+
+	 Note: context path is determined by the servlet container - 
+	 in Tomcat, specified in server.xml -kjdon
+         You may define any number of servlet mappings, including zero.
+         It is also legal to define more than one mapping for the same
+         servlet, if you wish to.
+    -->
+    <!--
+    <servlet-mapping>
+      <servlet-name>controller</servlet-name>
+      <url-pattern>*.do</url-pattern>
+    </servlet-mapping>
+
+    <servlet-mapping>
+      <servlet-name>graph</servlet-name>
+      <url-pattern>/graph</url-pattern>
+    </servlet-mapping>
+    -->
+  
+  <servlet-mapping>
+    <servlet-name>tester</servlet-name>
+    <url-pattern>/testing</url-pattern>
+  </servlet-mapping>
+  
+  <servlet-mapping>
+    <servlet-name>library</servlet-name>
+    <url-pattern>/library</url-pattern>
+  </servlet-mapping>
+  
+  <servlet-mapping>
+    <servlet-name>gateway</servlet-name>
+    <url-pattern>/gateway</url-pattern>
+  </servlet-mapping>
+  
+  <servlet-mapping>
+    <servlet-name>classic-library</servlet-name>
+    <url-pattern>/classic</url-pattern>
+  </servlet-mapping>
+  
+  <!-- axis mappings -->
+  <servlet-mapping>
+    <servlet-name>AxisServlet</servlet-name>
+    <url-pattern>/servlet/AxisServlet</url-pattern>
+  </servlet-mapping>
+
+  <servlet-mapping>
+    <servlet-name>AxisServlet</servlet-name>
+    <url-pattern>*.jws</url-pattern>
+  </servlet-mapping>
+
+  <servlet-mapping>
+    <servlet-name>AxisServlet</servlet-name>
+    <url-pattern>/services/*</url-pattern>
+  </servlet-mapping>
+
+  <!-- uncomment this if you want to use the SOAPMonitorService. You will also 
+  need to copy the SOAPMonitorApplet*.class files from 
+  gsdl3/comms/soap/axis/web/WEB-INF/classes to the gsdl3/web directory. See the 
+  comms/soap/axis/docs/install.html Appendix for details about how to use this -->
+  <!--
+  <servlet-mapping>
+    <servlet-name>SOAPMonitorService</servlet-name>
+    <url-pattern>/SOAPMonitor</url-pattern>
+  </servlet-mapping>
+  -->
+  <!-- uncomment this if you want the admin servlet -->
+  <!--
+  <servlet-mapping>
+    <servlet-name>AdminServlet</servlet-name>
+    <url-pattern>/servlet/AdminServlet</url-pattern>
+  </servlet-mapping>
+  -->
+
+  <!-- Define the default session timeout for your application,
+  in minutes.  From a servlet or JSP page, you can modify
+  the timeout for a particular session dynamically by using
+  HttpSession.getMaxInactiveInterval(). -->
+
+  <session-config>
+    <session-timeout>30</session-timeout>    <!-- 30 minutes -->
+  </session-config>
+
+  <!-- axis mime-mappings -->
+  <mime-mapping>
+    <extension>wsdl</extension>
+    <mime-type>text/xml</mime-type>
+  </mime-mapping>
+  
+  <mime-mapping>
+    <extension>xsd</extension>
+    <mime-type>text/xml</mime-type>
+  </mime-mapping>
+  
+</web-app>
+
+
Index: /main/tags/3.01/gsdl3/web/gs_ajax_utils.js
===================================================================
--- /main/tags/3.01/gsdl3/web/gs_ajax_utils.js	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/gs_ajax_utils.js	(revision 21112)
@@ -0,0 +1,129 @@
+
+
+function loadAsync(sUri, SOAPMessage) {
+   var xmlHttp = XmlHttp.create();
+   var async = true;
+   xmlHttp.open("POST", sUri, async);
+   xmlHttp.onreadystatechange = function () {
+      if (xmlHttp.readyState == 4){
+		var result = xmlHttp.responseText;
+			getTitle2(xmlHttp.responseXML, xmlHttp.responseText);
+    }     
+   }
+   xmlHttp.setRequestHeader("SOAPAction", " ");
+   xmlHttp.setRequestHeader("Content-Type", "Content-Type: text/xml; charset=utf-8");
+ 
+   xmlHttp.send(SOAPMessage);
+}
+
+
+function messageToSOAP(message) {
+    soapBody = '<soapenv:Body>' + message + '</soapenv:Body>'
+    soap = '<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">' + soapBody + '</soapenv:Envelope>'
+    x= '<?xml version="1.0" encoding="UTF-8"?>' + soap;
+    return x;
+}
+
+
+
+
+function getText(element) { //get the text from possibly multiple text nodes
+if (element.hasChildNodes()) {
+	tempString = '';
+    for (j=0; j < element.childNodes.length; j++) {
+    				if (element.childNodes[j].nodeType == Node.TEXT_NODE ) { // =2
+    					tempString += element.childNodes[j].nodeValue;
+    				} 
+					else {
+						tempString += 'non text node: ';
+					}
+    } 
+	return tempString;
+}
+else {
+	return 'getText: element has no ChildNodes from which to extract Text';
+}
+}
+
+
+function newOpenTag(name) {
+	return '<' + name + '>';
+}
+
+function newCloseTag(name) {
+	return '</' + name + '>';
+}
+
+function newEmptyTag(name) {
+	return '<' + name + '/>';
+}
+
+function newElement(name, content) {
+    e = '<' + name + '>' + content;
+    e += '</' + name + '>';
+    return e;
+}
+
+
+function newElementAtt1(name, content, attName, attValue) {
+    e = '<' + name + ' ' + attName + '="' + attValue +'">' + content;
+    e += '</' + name + '>';
+    return e;
+}
+
+
+
+
+function newElementAtt(name, content, nameArray, valueArray) {
+    e = '<' + name + ' ' ;
+    for (i=0; i < nameArray.length; i++) {
+        e += newAttribute(nameArray[i], valueArray[i])
+    }
+    e += '>' + content;
+    e += '</' + name + '>';
+    return e;
+}
+
+
+function newAttribute(name, value) {
+				 return ' ' + name + '="' + value + '"';
+}
+
+/*
+var a = [];
+var b = [];
+a[0] = 'title';
+b[0] = 'test';
+a[1] = 'title2';
+b[1] = 'test2';
+
+alert( newElement('message', 'some content', a=[], b=[]));
+
+*/
+
+function countElementChildren(node) {
+  	count= 0;
+	childList = node.childNodes;
+	for(var i=0; i < (childList.length); i++) {
+		childNode = childList.item(i);
+		if ((childNode.nodeType == 1))	{ // only count elements
+				count++;
+		}
+	}
+	return count;
+}
+
+function removeAllChildren(node) {
+	while (node.hasChildNodes()) {
+		node.removeChild(node.firstChild);
+	}
+}
+
+
+function isElement(node) {
+    if (node.nodeType == 1) {
+    		return true; }
+    else {
+    		return false;
+    }
+}
Index: /main/tags/3.01/gsdl3/web/index.html
===================================================================
--- /main/tags/3.01/gsdl3/web/index.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/index.html	(revision 21112)
@@ -0,0 +1,47 @@
+<html>
+
+  <head>
+    <meta http-equiv="Content-Type"
+      content="text/html; charset=iso-8859-1">
+      <title>Greenstone Version 3</title>
+  </head>
+  
+  <body bgcolor="#ffffff" text="#000000" link="#006666" 
+    alink="#cc9900" vlink="#666633" background="interfaces/default/images/chalk.gif">
+    
+    <!-- page banner (_style:pagebanner_) -->
+    <center>
+      <table width=537 cellspacing=0 cellpadding=0>
+	<tr valign=top>
+	  <td rowspan=2 align=left><center><img src="interfaces/default/images/gsdlhead.gif" width=300 height=100 alt="Greenstone Digital Library Software"></center></td>
+	  <td align=right></td>
+	</tr>
+	
+	<tr>
+	  <td align=right></td>
+	</tr>
+	
+	<tr>
+	  <td colspan=2></td>
+	</tr>
+      </table>
+    </center>
+    <!-- end of page banner -->
+    
+    <center>
+      
+      <table width=537>
+	<tr><td><h3>Hello! Welcome to your Greenstone3 installation</h3></td></tr>
+	<tr><td>&nbsp;</td></tr>
+	<tr><td>What do you want to do today?</td></tr>
+	<tr><td>&nbsp;</td></tr>
+	
+	
+	<tr><td><a href="testing">Run</a> the test servlet</td></tr>
+	<tr><td><a href="library">Run</a> the standard library servlet. View the demo collections that come with Greenstone, using the default look and feel.</td></tr>
+	<tr><td><a href="classic">Run</a> the 'classic' servlet. View the same collections using the 'classic' (or Greenstone 2) look and feel.</td></tr>
+	<tr><td><a href="gateway">Run</a> the gateway servlet. This uses the standard look and feel, and talks via SOAP to the site with demo collections (localsite). Note that SOAP needs to be installed, and a SOAP server needs to be running for localsite.</td></tr>
+      </table>
+    </center>
+  </body>
+</html>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/interfaceConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/interfaceConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/interfaceConfig.xml	(revision 21112)
@@ -0,0 +1,21 @@
+<interfaceConfig baseInterface="default">
+  <actionList>
+    <action name='p' class='PageAction'>
+      <subaction name='home' xslt='home.xsl'/>
+      <subaction name='about' xslt='about.xsl'/>
+      <subaction name='help' xslt='help.xsl'/>
+      <subaction name='pref' xslt='pref.xsl'/>
+      <subaction name='nav' xslt='nav.xsl'/>
+      <subaction name="html" xslt="html.xsl"/>
+    </action>
+    <action name='q' class='QueryAction' xslt='query.xsl'/>
+    <action name='b' class='GS2BrowseAction' xslt='classifier.xsl'/>
+    <action name='a' class='AppletAction' xslt='applet.xsl'/>
+    <action name='d' class='DocumentAction' xslt='document.xsl'/>
+    <action name='pr' class='ProcessAction' xslt='process.xsl'/>
+    <action name='s' class='SystemAction' xslt='system.xsl'/>
+  </actionList>
+  <languageList>
+    <language name='en' displayname='English' default_encoding='utf-8'/>
+  </languageList>
+</interfaceConfig>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/about.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/about.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/about.xsl	(revision 21112)
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="help-common.xsl"/>
+  <xsl:include href="query-common.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/collection/displayItem[@name='name']"/>  
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+<!--    <center>-->
+      <xsl:apply-templates select="collection"><xsl:with-param name="collName" select="$collName"/></xsl:apply-templates>
+    <!--</center>-->
+  </xsl:template>
+
+  <xsl:template match="collection">
+    <xsl:param name="collName"/>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+      <xsl:with-param name="pageType">about</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:if test="/page/pageRequest/paramList/param[@name='ct']/@value = 0">
+    <xsl:apply-templates select="serviceList/service[@name='TextQuery']">
+      <xsl:with-param name="collName" select="$collName"/></xsl:apply-templates>
+    <xsl:call-template name="dividerBar"/>
+    </xsl:if>
+    <xsl:if test="displayItem[@name='description']">
+    <h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'about.aboutcoll')"/></h3>
+    <p><xsl:value-of select="displayItem[@name='description']" disable-output-escaping='yes'/></p>
+    </xsl:if>
+    <xsl:apply-templates select="." mode="simplehelp"/>
+  </xsl:template>
+
+
+</xsl:stylesheet>  
+
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/applet.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/applet.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/applet.xsl	(revision 21112)
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:include href="style.xsl"/>
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="pageResponse/service/applet"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="../pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="pageType"><xsl:choose><xsl:when test="service[@name='PhindApplet']">phrases</xsl:when><xsl:otherwise>applet</xsl:otherwise></xsl:choose></xsl:with-param>
+      </xsl:call-template>
+      <p/>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+<!--      <xsl:call-template name="dividerBar">
+	<xsl:with-param name="text" select="service/displayItem[@name='name']"/>
+      </xsl:call-template>-->
+      <p/>
+      <xsl:copy-of select="service/applet"/>
+      <xsl:call-template name="dividerBar"/>
+    </center>
+  </xsl:template>
+</xsl:stylesheet>  
+
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/classifier.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/classifier.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/classifier.xsl	(revision 21112)
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="classifiertools.xsl"/>
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'nzdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <xsl:variable name="serviceName"><xsl:value-of select="service/@name"/></xsl:variable>
+    <xsl:variable name="cl" select="/page/pageRequest/paramList/param[@name='cl']/@value"/>
+    <xsl:variable name="clTop">
+      <xsl:choose>
+	<xsl:when test="contains($cl, '.')"><xsl:value-of select="substring-before($cl, '.')"/></xsl:when>
+	<xsl:otherwise>
+	  <xsl:value-of select="$cl"/>
+	</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="pageType">browse</xsl:with-param>
+	<xsl:with-param name="clTop" select="$clTop"/>
+      </xsl:call-template>
+      <xsl:call-template name="navigationBar">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="clTop" select="$clTop"/>
+      </xsl:call-template>      
+      <p/>
+      <xsl:apply-templates select="classifier">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="serviceName" select="$serviceName"/>
+      </xsl:apply-templates>
+    </center>
+  </xsl:template>
+  
+
+</xsl:stylesheet>
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/document.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/document.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/document.xsl	(revision 21112)
@@ -0,0 +1,416 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util gsf">
+
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+  <xsl:output method="html"/>
+
+  <!-- the main page layout template is here -->
+  <!-- override the one in style so we can display the doc over the whole page and get rid of the green bar down the side --> 
+  <xsl:template match="page" priority='2'>
+    <html>
+      <xsl:call-template name="pageHead"/>
+      <body style="background-image: none;">
+         <center>
+		    <xsl:apply-templates select="pageResponse"/>
+		    <xsl:call-template name="greenstoneFooter"/>
+	    </center>
+	  </body>
+    </html>
+  </xsl:template>
+  
+  <xsl:template name="pageTitle">
+    <xsl:variable name="docID" select="/page/pageResponse/document/@selectedNode"/>
+    <xsl:for-each select='/page/pageResponse/document/descendant::documentNode[@nodeID=$docID]/ancestor-or-self::documentNode'><xsl:if test='position()!=1'>::</xsl:if><xsl:value-of select="metadataList/metadata[@name='Title']"/></xsl:for-each>
+  </xsl:template>
+
+  <!-- this is hard coded for GATE, should somehow do it dynamically-->
+  <xsl:template name="pageStyle">
+    <style type="text/css">
+      <xsl:text disable-output-escaping="yes">
+	span.Location {  display:inline; color : red }
+	span.Person {  display:inline; color : green }
+	span.Organization {  display:inline; color : yellow }
+	span.Date {  display:inline; color : blue }
+	span.query_term {display: inline; background-color : yellow }
+      </xsl:text>
+    </style>
+  </xsl:template>
+  
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <table width="537">
+      <tr><td>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    </td></tr></table>      
+    <!-- display the document -->
+    <xsl:apply-templates select="document">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+    <table width='537'><tr><td>
+    <xsl:call-template name="dividerBar"/>
+    </td></tr></table>
+  </xsl:template>
+	
+  <xsl:template match="document">
+    <xsl:param name="collName"/>
+    <table width='537'><tr><td>
+    <xsl:call-template name="documentHeading">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    </td></tr></table>
+    <xsl:call-template name="documentContent">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+  </xsl:template>
+
+  <xsl:template name="documentHeading">
+    <xsl:param name="collName"/>
+    <xsl:variable name="doCoverImage" select="/page/pageResponse/format/gsf:option[@name='coverImages']/@value"/>
+    <xsl:variable name="doTOC" select="/page/pageResponse/format/gsf:option[@name='documentTOC']/@value"/>
+    <xsl:variable name="p.a" select="/page/pageRequest/paramList/param[@name='p.a']/@value"/>
+    <xsl:variable name="p.sa" select="/page/pageRequest/paramList/param[@name='p.sa']/@value"/>
+    <xsl:variable name="p.s" select="/page/pageRequest/paramList/param[@name='p.s']/@value"/>
+    <table><tr><td>
+	  <a href="{$library_name}?a={$p.a}&amp;sa={$p.sa}&amp;s={$p.s}&amp;c={$collName}&amp;rt=r"><xsl:call-template name="openbookimg"><xsl:with-param name="title">Close this book and return to search/browse</xsl:with-param>
+	  </xsl:call-template></a></td>
+	<td valign='top'><b><font size="+1">&#160;
+	      <xsl:choose><xsl:when test="@docType='simple'"><xsl:value-of select="metadataList/metadata[@name='Title']" disable-output-escaping="yes"/></xsl:when><xsl:otherwise><xsl:value-of select="documentNode/metadataList/metadata[@name='Title']" disable-output-escaping="yes"/></xsl:otherwise></xsl:choose>
+	    </font></b></td>
+      </tr></table>
+
+    <table cellpadding="10" cellspacing="0">
+      <tr>
+	<td valign="top" align="left">
+	  <xsl:if test="string($doCoverImage) != 'false' and (@docType='paged' or @docType='hierarchy')">
+	    <xsl:call-template name="coverImage"/>
+	    <p/>
+	  </xsl:if>
+	  
+	  <xsl:call-template name="documentButtons">
+	    <xsl:with-param name="collName" select="$collName"/>
+	  </xsl:call-template>
+	  <!--<xsl:call-template name="enrichServices">
+	    <xsl:with-param name="collName" select="$collName"/>
+	  </xsl:call-template>-->
+	</td>
+	<xsl:if test="string($doTOC) != 'false'">
+	  <td valign="top" align="left">
+	    <xsl:call-template name="TOC">
+	      <xsl:with-param name="collName" select="$collName"/>
+	    </xsl:call-template>
+	  </td>
+	</xsl:if>
+      </tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="coverImage">
+    <img alt="" onError="src='interfaces/classic/images/blank.gif'"><xsl:attribute name='src'><xsl:value-of select="/page/pageResponse/collection/metadataList/metadata[@name='httpPath']"/>/index/assoc/<xsl:value-of select="metadataList/metadata[@name='archivedir']"/>/cover.jpg</xsl:attribute></img>
+  </xsl:template>
+
+  <xsl:template name="documentButtons">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docID" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="sib" select="/page/pageRequest/paramList/param[@name='sib']/@value"/>
+    <xsl:variable name="ec" select="/page/pageRequest/paramList/param[@name='ec']/@value"/>
+    <xsl:variable name="ed" select="/page/pageRequest/paramList/param[@name='ed']/@value"/>
+    
+    <table>
+      <!-- expand document -->
+      <!-- this doesn't work with MG so comment it out for now -->
+      
+      <xsl:if test="@docType = 'hierarchy' or @docType = 'paged'">
+      <tr><td class="gsbutton">
+	  <xsl:choose><xsl:when test="string($ed)='1'">
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ed=0"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_doc_tip')"/></xsl:attribute>
+		  <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_doc_b')"/></font>
+	      </a>
+	    </xsl:when>
+	    <xsl:otherwise>
+	  <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ed=1"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_doc_tip')"/></xsl:attribute>
+	    <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_doc_b')"/></font>
+	  </a>
+	    </xsl:otherwise>
+	  </xsl:choose>
+	</td></tr>
+      </xsl:if> 
+      
+      <!-- expand contents -->
+      <xsl:if test="@docType = 'hierarchy' and string(/page/pageResponse/format/gsf:option[@name='documentTOC']/@value) != 'false'">
+      <tr><td class="gsbutton">
+	  <xsl:choose><xsl:when test="string($ec)='1'">
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ec=0"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_contents_tip')"/></xsl:attribute>
+		  <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_contents_b')"/></font>
+	      </a>
+	    </xsl:when>
+	    <xsl:otherwise>
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ec=1"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_contents_tip')"/></xsl:attribute>
+		<font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_contents_b')"/></font>
+	      </a>
+	    </xsl:otherwise>
+	  </xsl:choose>
+	</td></tr>
+      </xsl:if>
+      <!-- detach page -->
+      <xsl:variable name="paramList" select="/page/pageRequest/paramList"/>
+      <tr><td class="gsbutton">
+	  <a class="gsbutton" target="_blank"><xsl:attribute name="href"><xsl:value-of select="$library_name"/>?a=d&amp;d=<xsl:value-of select="$docID"/>&amp;c=<xsl:value-of select="$collName"/>&amp;sib=<xsl:value-of select="$sib"/>&amp;dt=<xsl:value-of select="$paramList/param[@name='dt']/@value"/>&amp;ec=<xsl:value-of select="$paramList/param[@name='ec']/@value"/>&amp;et=<xsl:value-of select="$paramList/param[@name='et']/@value"/>&amp;p.a=<xsl:value-of select="$paramList/param[@name='p.a']/@value"/>&amp;p.s=<xsl:value-of select="$paramList/param[@name='p.s']/@value"/>&amp;p.sa=<xsl:value-of select="$paramList/param[@name='p.sa']/@value"/></xsl:attribute><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.detach_page_tip')"/></xsl:attribute>
+	    <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.detach_page_b')"/></font>
+	  </a></td></tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="TOC">
+    <xsl:param name="collName"/>
+    <xsl:choose>
+      <xsl:when test="@docType='hierarchy'">
+	<xsl:call-template name="hierarchicalContents">
+	  <xsl:with-param name="collName" select="$collName"/>
+	</xsl:call-template>
+      </xsl:when>
+      <xsl:when test="@docType='paged'">
+	<xsl:call-template name="pagedContents">
+	  <xsl:with-param name="collName" select="$collName"/>
+	</xsl:call-template>
+      </xsl:when>
+    </xsl:choose>
+  </xsl:template>
+    
+  <xsl:template name="hierarchicalContents">
+    <xsl:param name="collName"/>
+    <xsl:variable name="oc" select="/page/pageRequest/paramList/param[@name='oc']/@value"/>
+    <xsl:variable name="d" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="sib" select="/page/pageRequest/paramList/param[@name='sib']/@value"/>
+    <xsl:variable name="ec" select="/page/pageRequest/paramList/param[@name='ec']/@value"/>
+    <xsl:if test="documentNode[@nodeID]">
+      <table>
+	<tr valign='top'><td>
+	    <xsl:choose><xsl:when test="string($oc)='0'">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={$d}&amp;sib={$sib}&amp;oc=1"><xsl:call-template name="closedfolderimg"><xsl:with-param name="title">open the table of contents</xsl:with-param></xsl:call-template></a>
+	      </xsl:when>
+	      <xsl:otherwise><a href="{$library_name}?a=d&amp;c={$collName}&amp;d={$d}&amp;sib={$sib}&amp;oc=0"><xsl:call-template name="openfolderimg"><xsl:with-param name="title">close the table of contents</xsl:with-param></xsl:call-template></a></xsl:otherwise></xsl:choose>
+	  </td><td>Table of Contents</td></tr>
+	<xsl:if test="string($oc)!='0'">
+	  <tr><td>&#160;</td><td>
+	      <table>
+		<xsl:for-each select="documentNode/documentNode[@nodeID]">
+		  <xsl:apply-templates select='.'><xsl:with-param name="collName" select="$collName"/><xsl:with-param name="ec" select="$ec"/></xsl:apply-templates>
+		</xsl:for-each>
+	      </table>
+	    </td></tr>
+	</xsl:if>
+      </table>
+    </xsl:if>
+  </xsl:template>
+  
+  <!-- each icon-title pair is a row in a table. children go in a table in another row -->
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:param name="ec"/>
+    <!-- Display the appropriate image, depending on the node type -->
+    <tr><td valign="top">
+	<a><xsl:attribute name="href"><xsl:value-of select='$library_name'/>?a=d&amp;c=<xsl:value-of select='$collName'/>&amp;d=<xsl:value-of select='@nodeID'/><xsl:if test="documentNode">.pr</xsl:if><xsl:if test="@nodeType='leaf'">&amp;sib=1</xsl:if><xsl:if test="string($ec) = '1'">&amp;ec=1</xsl:if></xsl:attribute>
+	  <xsl:apply-templates select="." mode="displayNodeIcon"/>
+	</a>
+      </td>
+      
+      <!-- Display associated title, bolded if the node has content -->
+      <td valign="top">
+	<xsl:choose>
+	  <xsl:when test="nodeContent">
+	    <b><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></b>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </td>
+    </tr>
+    <!-- Apply recursively to the children of this node -->
+    <xsl:if test="documentNode[@nodeID]">
+      <tr><td>&#160;</td><td><table>
+	    <xsl:apply-templates select="documentNode[@nodeID]">
+	      <xsl:with-param name="collName" select="$collName"/>
+	      <xsl:with-param name="ec" select="$ec"/>
+	      <!--<xsl:with-param name="depth" select="$depth + 1"/>-->
+	    </xsl:apply-templates>
+	  </table>
+	</td></tr>
+    </xsl:if>
+  </xsl:template>
+  
+  <!-- default content is to print the title and content of any documentNodes that have nodeContent -->
+  <xsl:template name="documentContent">
+    <div align="left">
+      <xsl:choose>
+        <xsl:when test="@docType='simple'">
+	  <xsl:apply-templates select="nodeContent"/>
+	</xsl:when>
+        <xsl:otherwise>
+	  <xsl:apply-templates select="descendant-or-self::node()/documentNode" mode="content"/>
+        </xsl:otherwise>
+      </xsl:choose>
+    </div>
+  </xsl:template>
+  
+  <xsl:template match="documentNode" mode="content">
+    <xsl:if test="nodeContent and string(nodeContent)">
+      <xsl:if test="metadataList/metadata[@name='Title']">
+	<h3><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></h3>
+      </xsl:if>
+      <xsl:apply-templates select="nodeContent"/>
+      </xsl:if>
+    
+  </xsl:template>
+
+  <!-- the actual text/content -->
+  <xsl:template match="nodeContent">
+    <p/>
+    <xsl:for-each select="node()">
+      <xsl:choose>
+	<xsl:when test="not(name())"><xsl:value-of select="." disable-output-escaping="yes"/></xsl:when>
+	<xsl:otherwise><xsl:apply-templates select="."/></xsl:otherwise>
+      </xsl:choose>
+    </xsl:for-each>
+  </xsl:template>
+  
+  <!-- match any file nodes -->
+  <xsl:template match="file">
+    <xsl:variable name="httpPath" select="/page/pageResponse/collection/metadataList/metadata[@name='httpPath']"/>
+    <xsl:choose>
+      <xsl:when test="util:isImage(@mimeType)">
+	<img src="{$httpPath}/{@href}"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<a href="{$httpPath}/{@href}"><xsl:value-of select="@href"/></a>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <!-- match any link nodes -->
+  <xsl:template match="link">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/> 
+    <xsl:variable name="actionargs">
+      <xsl:choose>
+	<xsl:when test="@type='document'">a=d</xsl:when>
+	<xsl:when test="@type='query'">a=q&amp;s=<xsl:value-of select="@service"/>&amp;rt=r</xsl:when>
+	<xsl:otherwise>
+	  p
+	</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <xsl:variable name="serviceargs">
+      <xsl:for-each select="param">&amp;s1.<xsl:value-of select="@name"/>=<xsl:value-of select="@value"/></xsl:for-each>
+  </xsl:variable>
+    <a href="{$library_name}?{$actionargs}&amp;c={$collName}{$serviceargs}"><xsl:value-of disable-output-escaping="yes" select="."/></a>
+  </xsl:template>
+
+  <!-- match any annotations and make them span elements -->
+  <xsl:template match="annotation">
+    <span class="{@type}"><xsl:value-of disable-output-escaping="yes" select="."/></span>
+  </xsl:template>
+  
+  <!-- paged naviagtion : INCOMPLETE!!-->
+  <xsl:template name="pagedContents">
+    <xsl:param name="collName"/>
+    <xsl:variable name="pos" select="nodeStructureInfo/info[@name='siblingPosition']/@value"/>
+    <xsl:variable name="length" select="nodeStructureInfo/info[@name='numSiblings']/@value"/>
+    <xsl:variable name="children" select="nodeStructureInfo/info[@name='numChildren']/@value"/>
+    <table>
+      <xsl:choose>
+	<xsl:when test="$pos=-1"><!-- a doc -->
+	  <tr valign="top">
+	    <td align="left">
+	    </td>
+	    <td align="center"><center>
+		<b><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.pages', $children)"/></b>
+	      </center></td>
+	    <td align="right">
+	      <a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.fc"><img src="interfaces/default/images/more.gif" border="0" align="absbottom" /></a>
+	    </td>
+	  </tr>
+	</xsl:when>
+	
+	<xsl:otherwise> <!-- an internal node -->	  
+	  <tr valign="top">
+	    <td align="left">
+	      <xsl:if test=" not ( $pos = 1 )">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.ps"><img src="interfaces/default/images/less.gif" border="0" align="absbottom" /></a>
+	      </xsl:if>
+	    </td>
+	    <td align="center"><center>
+		<b><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.pageof', concat($pos, ';', $length))"/></b>
+	      </center></td>
+	    <td align="right">
+	      <xsl:if test=" not($pos = $length)">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.ns"><img src="interfaces/default/images/more.gif" border="0" align="absbottom" /></a>
+	      </xsl:if>
+	    </td>
+	  </tr>
+	</xsl:otherwise>
+      </xsl:choose>
+      <tr valign="middle">
+	<td align='center' valign='top' colspan='3'>
+	  <form name="GotoForm" method="get" action="{$library_name}">
+	    <input type='hidden' name='a' value='d'/>
+	    <input type='hidden' name='c' value='{$collName}'/>
+	    <input type='hidden' name='d' value='{@selectedNode}'/>
+	    <input type="text" name="gp" size="3" maxlength="4"/>
+	    <input type="submit"><xsl:attribute name='value'><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.gotopage')"/></xsl:attribute></input>
+	  </form>
+	</td>
+      </tr>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="enrichServices">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docID" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="request-params" select="/page/pageRequest/paramList"/>
+    <xsl:for-each select="../serviceList/service">
+      <table border='1' cellspacing='0'>
+	<tr><td>
+	    <p /><xsl:value-of select="displayItem[@name='name']"/><p/>
+	    <form name="EnrichForm" method="get" action="{$library_name}">
+	      <xsl:apply-templates select="paramList"/>
+	      <input type='hidden' name='a' value='d'/>
+	      <input type='hidden' name='d' value='{$docID}'/>
+	      <input type='hidden' name='c' value='{$collName}'/>
+	      <xsl:if test='$request-params/param[@name="sib"]'>
+		<input type='hidden' name='sib'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="sib"]/@value'/></xsl:attribute></input></xsl:if>
+	      <input type='hidden' name='s' value='{@name}'/>
+	      <input type='hidden' name='p.a'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.a"]'/></xsl:attribute></input>
+	      <input type='hidden' name='p.sa'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.sa"]'/></xsl:attribute></input>
+	      <input type='hidden' name='p.s'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.s"]'/></xsl:attribute></input>
+	      <input type='hidden' name='end' value='1'/>
+	      <input type="submit"><xsl:attribute name="value"><xsl:value-of select="displayItem[@name='submit']"/></xsl:attribute></input>
+	    </form>
+	  </td></tr>
+      </table>
+    </xsl:for-each>
+  </xsl:template>
+
+  <xsl:template match="paramList" mode="hidden">
+    <xsl:for-each select="param">
+      <input type='hidden' name='{@name}' value='{@value}'/><xsl:text>
+      </xsl:text>
+    </xsl:for-each>
+  </xsl:template>
+  
+</xsl:stylesheet>
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help-common.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help-common.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help-common.xsl	(revision 21112)
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+
+  <xsl:template match="collection" mode="simplehelp">
+    <xsl:variable name="longCollName"><xsl:value-of select="displayItem[@name='name']" disable-output-escaping='yes'/></xsl:variable>
+    <!--<xsl:variable name="numoptions">5</xsl:variable>-->
+    <xsl:variable name="has_search"><xsl:choose><xsl:when test="serviceList/service[@name='TextQuery']">1</xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <xsl:variable name="has_phrase"><xsl:choose><xsl:when test="serviceList/service[@name='PhindApplet']">1</xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <xsl:variable name="numoptions" select="count(serviceList/service[@name='ClassifierBrowse']/classifierList/classifier)"/>
+    <h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.simplehelpheading', $longCollName)"/></h3>
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.numbrowseoptions', $numoptions+$has_search+$has_phrase)"/>
+    <ul>
+     <xsl:if test="$has_search =1">
+       <li><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.Searchshort')" disable-output-escaping='yes'/></li>
+     </xsl:if>
+     <xsl:for-each select="serviceList/service[@name='ClassifierBrowse']/classifierList/classifier">
+       <xsl:variable name="title">help.<xsl:value-of select="@content"/>short</xsl:variable>
+       <li><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, $title)" disable-output-escaping='yes'/></li>
+     </xsl:for-each>
+     <xsl:if test="$has_phrase =1">
+       <li><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.Phraseshort')" disable-output-escaping='yes'/></li>
+     </xsl:if>
+     </ul>
+     <xsl:if test="$has_search =1">
+       <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.Searchlong')" disable-output-escaping='yes'/>
+     </xsl:if>
+     <xsl:for-each select="serviceList/service[@name='ClassifierBrowse']/classifierList/classifier">
+       <xsl:variable name="title">help.<xsl:value-of select="@content"/>long</xsl:variable>
+       <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, $title)" disable-output-escaping='yes'/>
+     </xsl:for-each>
+     <xsl:if test="$has_phrase =1">
+       <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help.Phraselong')" disable-output-escaping='yes'/>
+     </xsl:if>
+     
+  </xsl:template>
+
+   
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/help.xsl	(revision 21112)
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <xsl:include href="style.xsl"/>
+  <!--<xsl:include href="help-text.xsl"/>-->
+  <xsl:include href="page-common.xsl"/>
+  
+  <xsl:output method="html"/>  
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_tip')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="pageType">help</xsl:with-param>
+      </xsl:call-template>
+      <xsl:call-template name="navigationBar">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>      
+      <p />Help text to go here!!
+    </center>
+  </xsl:template>
+
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/home.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/home.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/home.xsl	(revision 21112)
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/>
+  </xsl:template>
+  
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:call-template name="greenstonePageBanner"/> 
+    <xsl:apply-templates select="collectionList"/>   
+  </xsl:template>
+
+  <xsl:template match="collectionList">
+    <xsl:for-each select="collection"> 
+      <xsl:variable name="ct"><xsl:choose><xsl:when test="metadataList/metadata[@name='buildType']='mgpp'">1</xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+
+      <p>
+      <a href="{$library_name}?a=p&amp;sa=about&amp;c={@name}&amp;ct={$ct}">
+	  <img width="150" border="1">
+	    <xsl:attribute name="src">
+	      <xsl:value-of select="metadataList/metadata[@name='httpPath']"/>/images/<xsl:choose><xsl:when test="displayItem[@name='smallicon']"><xsl:value-of select="displayItem[@name='smallicon']"/></xsl:when><xsl:otherwise><xsl:value-of select="displayItem[@name='icon']"/></xsl:otherwise></xsl:choose>
+	    </xsl:attribute>
+	    <xsl:attribute name="alt">
+	      <xsl:value-of select="displayItem[@name='name']"/>
+	    </xsl:attribute>
+	  </img>
+	</a>
+      </p>
+    </xsl:for-each> 
+    <xsl:call-template name="collWarning"/>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template> 
+  
+  <xsl:template name="collWarning">
+    <p><xsl:value-of select="util:getInterfaceText('classic', /page/@lang, 'home.coll_warning')"/>
+    </p>
+  </xsl:template>
+</xsl:stylesheet>  
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/nav.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/nav.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/nav.xsl	(revision 21112)
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+   <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+  
+  <xsl:template match="page">
+    <html>
+      <head>
+	<title>
+	  <xsl:call-template name="pageTitle"/><xsl:text> </xsl:text>
+	</title>
+	<xsl:call-template name="globalStyle"/>
+	<base target="_top"/>
+      </head>
+      <body style="background-image: none;">
+	<center>
+	  <table width="537">
+	    <tr><td>
+		<xsl:apply-templates select="pageResponse"/>
+	      </td></tr>
+	  </table>
+	</center>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/*/displayItem[@name='name']"/>
+  </xsl:template>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <div align="right">
+    <xsl:call-template name="top-buttons">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+      </div>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/pref.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/pref.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/pref.xsl	(revision 21112)
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <xsl:include href="style.xsl"/>
+  <!--<xsl:include href="pref-text.xsl"/>-->
+  
+  <xsl:output method="html"/>  
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_tip')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="pageType">pref</xsl:with-param>
+      </xsl:call-template>
+      <xsl:call-template name="navigationBar">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>
+      <form name="PrefForm" method="get" action="{$library_name}">
+	<input type='hidden' name='a' value='p'/>
+	<input type='hidden' name='sa' value='pref'/>
+	<table>
+	  <tr><td><h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.prespref')"/></h3></td></tr>
+	  <tr><td><xsl:call-template name="pres-prefs"/></td></tr>
+	  <tr><td><xsl:call-template name="dividerBar"/></td></tr>
+	  <tr><td><h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.searchpref')"/></h3></td></tr>
+	  <tr><td><xsl:call-template name="search-prefs"/></td></tr>
+	  <tr><td><xsl:call-template name="dividerBar"/></td></tr>
+	</table>
+      </form>
+    </center>
+  </xsl:template>
+
+  
+  <xsl:template name="pres-prefs">
+    <table>
+      <tr><td><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.interfacelang')"/></td><td align='left'>English</td></tr>
+      <tr><td><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.encoding')"/></td><td align='left'>UTF-8</td></tr>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="search-prefs">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:variable name="ct"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='ct']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='ct']/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <table>
+      <tr><td> </td><td> </td><td align='right'><input type='submit' value='set prefs'/></td></tr>
+      <xsl:choose>
+	<xsl:when test="$ct=0">
+	  <xsl:call-template name='text-search-prefs'/>
+	</xsl:when>
+	<xsl:otherwise>
+      <!-- search type -->
+	  <xsl:variable name='querytype'><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qt']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qt']/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+	  <tr><td  rowspan='2' valign='baseline'>Type of search:</td><td><input type='radio' name='qt' value='0'><xsl:if test="$querytype=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td align='left'> text search</td></tr>
+	  <tr><td></td><td>
+	      <table>
+		<xsl:call-template name='text-search-prefs'/>
+	      </table></td></tr>
+	  
+	  <tr><td> </td><td><input type='radio' name='qt' value='1'><xsl:if test="$querytype=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> form search<br /></td></tr>
+	  <tr><td></td><td></td><td>
+	      <table>
+		<!-- query form mode -->
+		<xsl:variable name="queryfmode"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qfm']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qfm']/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+		<tr><td valign='baseline'>Form type:</td><td><input type='radio' name='qfm' value='0'><xsl:if test="$queryfmode=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input> simple <br />
+		    <input type='radio' name='qfm' value='1'><xsl:if test="$queryfmode=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input> advanced</td></tr>
+		<!-- query form num boxes -->
+		<xsl:variable name="numbox"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qfn']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qfn']/@value"/></xsl:when><xsl:otherwise>4</xsl:otherwise></xsl:choose></xsl:variable>
+		<tr><td colspan='2'>with <select name="qfn">
+		      <option value="2"><xsl:if test="$numbox=2"><xsl:attribute name="selected"></xsl:attribute></xsl:if>2</option> 
+		      <option value="4"><xsl:if test="$numbox=4"><xsl:attribute name="selected"></xsl:attribute></xsl:if>4</option>
+		      <option value="6"><xsl:if test="$numbox=6"><xsl:attribute name="selected"></xsl:attribute></xsl:if>6</option>
+		      <option value="8"><xsl:if test="$numbox=8"><xsl:attribute name="selected"></xsl:attribute></xsl:if>8</option>
+		    </select> fields</td></tr>
+	      </table></td></tr>
+	</xsl:otherwise>
+      </xsl:choose>
+      <!-- case folding -->
+      <xsl:variable name="casevar" select="concat($ns,'case')"/>
+      <xsl:variable name="case"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name=$casevar]"><xsl:value-of select="/page/pageRequest/paramList/param[@name=$casevar]/@value"/></xsl:when><xsl:otherwise>1</xsl:otherwise></xsl:choose></xsl:variable>
+      <tr><td rowspan='2' valign='baseline'>Case differences:</td><td><input type='radio' name='{$ns}case' value='1'><xsl:if test="$case=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> ignore case differences</td></tr>
+      <tr><td><input type='radio' name='{$ns}case' value='0'><xsl:if test="$case=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> upper/lower case must match</td></tr>
+      <!-- stemming -->
+      <xsl:variable name="stemvar" select="concat($ns,'stem')"/>
+      <xsl:variable name="stem"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name=$stemvar]"><xsl:value-of select="/page/pageRequest/paramList/param[@name=$stemvar]/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+      <tr><td  rowspan='2' valign='baseline'>Word endings:</td><td><input type='radio' name='{$ns}stem' value='1'><xsl:if test="$stem=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> ignore word endings</td></tr>
+      <tr><td><input type='radio' name='{$ns}stem' value='0'><xsl:if test="$stem=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> whole word must match</td></tr>
+      <!-- hit display -->
+      <xsl:variable name="maxvar" select="concat($ns,'maxDocs')"/>
+      <xsl:variable name="maxdocs"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name=$maxvar]"><xsl:value-of select="/page/pageRequest/paramList/param[@name=$maxvar]/@value"/></xsl:when><xsl:otherwise>50</xsl:otherwise></xsl:choose></xsl:variable>
+      <xsl:variable name="hits"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='hitsPerPage']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='hitsPerPage']/@value"/></xsl:when><xsl:otherwise>20</xsl:otherwise></xsl:choose></xsl:variable>
+      <tr><td colspan='3'>Return up to 
+	  <select name="{$ns}maxDocs">
+	    <option value="50"><xsl:if test="$maxdocs=50"><xsl:attribute name="selected"></xsl:attribute></xsl:if>50</option>
+	    <option value="100"><xsl:if test="$maxdocs=100"><xsl:attribute name="selected"></xsl:attribute></xsl:if>100</option>
+	    <option value="200"><xsl:if test="$maxdocs=200"><xsl:attribute name="selected"></xsl:attribute></xsl:if>200</option>
+	    <option value="-1"><xsl:if test="$maxdocs=-1"><xsl:attribute name="selected"></xsl:attribute></xsl:if>all</option>
+	  </select>
+	  hits with 
+	  <select name="hitsPerPage">
+	    <option value="10"><xsl:if test="$hits=10"><xsl:attribute name="selected"></xsl:attribute></xsl:if>10 </option> 
+	    <option value="20"><xsl:if test="$hits=20"><xsl:attribute name="selected"></xsl:attribute></xsl:if>20</option>
+	    <option value="50"><xsl:if test="$hits=50"><xsl:attribute name="selected"></xsl:attribute></xsl:if>50</option>
+	    <option value="-1"><xsl:if test="$hits=-1"><xsl:attribute name="selected"></xsl:attribute></xsl:if>all</option>
+	  </select>
+	  hits per page.
+	</td></tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="text-search-prefs">
+    <!-- query mode -->
+    <xsl:variable name="querymode"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qm']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qm']/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <tr><td rowspan='2' valign='baseline'>Query mode:</td><td><input type='radio' name='qm' value='0'><xsl:if test="$querymode=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> simple query mode</td></tr>
+	<tr><td><input type='radio' name='qm' value='1'><xsl:if test="$querymode=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> advanced query mode (allows boolean searching using !, &amp;, |, and parentheses)</td></tr>
+    
+    <!-- query box size -->
+    <xsl:variable name="boxsize"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qb']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qb']/@value"/></xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <tr><td rowspan='2' valign='baseline'>Query box size:</td><td><input type='radio' name='qb' value='0'><xsl:if test="$boxsize=0"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> regular query box</td></tr>
+	<tr><td><input type='radio' name='qb' value='1'><xsl:if test="$boxsize=1"><xsl:attribute name="checked"></xsl:attribute></xsl:if></input></td><td> large query box</td></tr>
+  </xsl:template>
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query-common.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query-common.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query-common.xsl	(revision 21112)
@@ -0,0 +1,160 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!--<xsl:include href="service-params.xsl"/>-->
+  <xsl:template match="service">
+    <xsl:param name="collName"/>
+    <xsl:variable name="subaction" select="../pageRequest/@subaction"/>
+    <form name="QueryForm" method="get" action="{$library_name}">
+      <input type="hidden" name="a" value="q"/>
+      <input type="hidden" name="sa" value="{$subaction}"/>
+      <input type="hidden" name="rt" value="rd"/>
+      <input type="hidden" name="s" value="{@name}"/>
+      <input type="hidden" name="c" value="{$collName}"/>
+      <input type="hidden" name="startPage" value="1"/>
+      <xsl:apply-templates select="paramList"/>
+    </form>
+  </xsl:template>
+  
+  <!-- a special handling of the param list - we override the one in service-params -->
+  <xsl:template match="paramList" priority='2'>
+    <table>
+      <tr><td>
+	  Search for <xsl:apply-templates select="param[@name='index']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='index']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates><xsl:if test="param[@name='level']">at <xsl:apply-templates select="param[@name='level']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='level']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates> level</xsl:if><xsl:call-template name='query_mode'/>
+	</td></tr>
+	<tr><td></td></tr>
+      <xsl:call-template name="query-and-submit"/>
+    </table>
+  </xsl:template>
+  
+  <!-- new template for match mode -->
+  <xsl:template name='query_mode'>
+    <xsl:variable name="qt" select="/page/pageRequest/paramList/param[@name='qt']/@value"/>
+    <xsl:choose>
+      <xsl:when test="$qt=1">
+	<xsl:variable name="qfm" select="/page/pageRequest/paramList/param[@name='qfm']/@value"/>
+	<xsl:choose>
+	  <xsl:when test="$qfm=1">
+	    and display results in <xsl:apply-templates select="param[@name='sortBy']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='sortBy']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates> order
+	  </xsl:when>
+	  <xsl:otherwise>
+	    which contain <xsl:apply-templates select="param[@name='matchMode']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='matchMode']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates> of
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:variable name="ct" select="/page/pageRequest/paramList/param[@name='ct']/@value"/>
+	<xsl:variable name="qm" select="/page/pageRequest/paramList/param[@name='qm']/@value"/>
+	<xsl:choose>
+	  <xsl:when test="$qm=1 and $ct=0">
+	    using ranked/bool query
+	  </xsl:when>
+	  <xsl:when test="$qm=1 and $ct=1">
+	    and display results in <xsl:apply-templates select="param[@name='sortBy']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='sortBy']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates> order
+	  </xsl:when>
+	  <xsl:otherwise>
+	    which contain <xsl:apply-templates select="param[@name='matchMode']"><xsl:with-param name="default"><xsl:apply-templates select="param[@name='matchMode']" mode="calculate-default"/></xsl:with-param></xsl:apply-templates> of the words
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="param[@name='matchMode']" mode='query1'>
+   <xsl:param name='default'/>
+    <xsl:variable name="qfm" select="/page/pageRequest/paramList/param[@name='qfm']/@value"/>
+    <xsl:choose>
+      <xsl:when test="$qfm=1">
+	and display results in <xsl:apply-templates select="../param[@name='sortBy']"><xsl:with-param name="default" select="$default"/></xsl:apply-templates> order
+      </xsl:when>
+      <xsl:otherwise>
+	which contain <xsl:apply-templates select='.'><xsl:with-param name="default" select="$default"/></xsl:apply-templates> of the words
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <xsl:template name="query-and-submit">
+    <xsl:variable name="qt" select="/page/pageRequest/paramList/param[@name='qt']/@value"/>
+    <xsl:choose>
+      <xsl:when test="$qt=1"> <!-- doing form query -->
+	<xsl:variable name="qfm" select="/page/pageRequest/paramList/param[@name='qfm']/@value"/>
+	<xsl:choose>
+	  <xsl:when test="$qfm=1">
+	    <tr><td><xsl:apply-templates select="param[@name='complexField']"/></td></tr>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <tr><td><xsl:apply-templates select="param[@name='simpleField']"/></td></tr>
+	  </xsl:otherwise>
+	</xsl:choose>
+	<tr><td align='right'>
+	    <input type="submit"><xsl:attribute name="value"><xsl:value-of select="../displayItem[@name='submit']"/></xsl:attribute></input>
+	  </td></tr>
+      </xsl:when>
+      <xsl:otherwise> <!-- doing text query -->
+    <xsl:variable name="qb" select="/page/pageRequest/paramList/param[@name='qb']/@value"/>
+	<xsl:choose>
+	  <xsl:when test="$qb=1"><!-- large query box -->
+	    <tr><td>
+		<textarea name="query" cols='63' rows='10'>
+		  <xsl:apply-templates select="param[@name='query']" mode="calculate-default"/><xsl:text> </xsl:text><!-- put a space here just in case there is no value- mozilla craps out if have a <textarea /> element -->
+		</textarea>
+	      </td></tr>
+	    <tr><td align='right'>
+		<input type="submit"><xsl:attribute name="value"><xsl:value-of select="../displayItem[@name='submit']"/></xsl:attribute></input>
+	      </td></tr>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <tr><td>
+		<xsl:variable name="qs"><xsl:apply-templates select="param[@name='query']" mode="calculate-default"/></xsl:variable>
+		<nobr><xsl:apply-templates select="param[@name='query']"><xsl:with-param name="default" select="java:org.greenstone.gsdl3.util.XSLTUtil.tidyWhitespace($qs)"/></xsl:apply-templates><input type="submit"><xsl:attribute name="value"><xsl:value-of select="../displayItem[@name='submit']"/></xsl:attribute></input></nobr>
+	      </td></tr>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <!-- overwrite the multi param to use our value of occurs, not the one with the param -->
+  <xsl:template match="param[@type='multi']" priority='2'>
+    <xsl:variable name="parent" select="@name"/>
+    <table>
+      <tr>
+	<xsl:for-each select="param">
+	  <xsl:variable name='pname' select='@name'/>
+	  <td><xsl:value-of select="displayItem[@name='name']"/></td>
+	</xsl:for-each>
+      </tr>
+      <!-- the number of times to display this is the qfn variable -->
+      <xsl:variable name="numbox"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qfn']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='qfn']/@value"/></xsl:when><xsl:otherwise>4</xsl:otherwise></xsl:choose></xsl:variable>
+      <xsl:apply-templates select="." mode="contents"><xsl:with-param name="occurs" select="$numbox"/><xsl:with-param name="pos" select="0"/></xsl:apply-templates>
+    </table>
+  </xsl:template>
+
+  <xsl:template match="param[@type='multi']" mode="contents">
+    <xsl:param name="occurs"/>
+    <xsl:param name="pos"/>
+ <!--   <xsl:variable name="pos" select="@occurs - $occurs"/>-->
+    <tr><xsl:for-each select="param">
+	<xsl:variable name="pname" select="@name"/>
+	<xsl:variable name="values" select="/page/pageRequest/paramList/param[@name=$pname]/@value"/>
+	<td><xsl:choose>
+	    <xsl:when test="not(@ignore) or  @ignore != $pos">      
+	      <xsl:apply-templates select='.'><xsl:with-param name="default" select="java:org.greenstone.gsdl3.util.XSLTUtil.getNumberedItem($values, $pos)"/></xsl:apply-templates>
+	    </xsl:when>
+	    <xsl:otherwise><!-- put in a hidden placeholder -->
+	      <input type="hidden" name='{@name}' value=''/>
+	    </xsl:otherwise>
+	  </xsl:choose></td>
+      </xsl:for-each></tr> 
+    <!-- recursively call this template to get multiple entries -->
+    <xsl:if test="$pos &lt; ($occurs - 1)">
+      <xsl:apply-templates select="." mode="contents"><xsl:with-param name="occurs" select="$occurs"/><xsl:with-param name="pos" select="$pos+1"/></xsl:apply-templates>
+    </xsl:if>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/query.xsl	(revision 21112)
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+  <xsl:include href="querytools.xsl"/>
+  <xsl:include href="query-common.xsl"/>  
+  <xsl:output method="html"/>
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'nzdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="pageType">search</xsl:with-param>
+      </xsl:call-template>
+      <xsl:call-template name="navigationBar">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>      
+      <xsl:apply-templates select="service">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:apply-templates>
+    </center>
+
+    <xsl:if test="documentNodeList">
+      <xsl:call-template name="query-response">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>
+    </xsl:if>
+    <center>
+      <xsl:call-template name="dividerBar"/>
+    </center>
+  </xsl:template>
+  
+  <xsl:template name="query-response">
+    <xsl:param name="collName"/>
+    <xsl:call-template name="dividerBar">
+      <xsl:with-param name='text'><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.results')"/></xsl:with-param></xsl:call-template>
+
+    <!-- If query term information is available, display it -->
+    <xsl:call-template name="termInfo"/>    
+    <!-- If the number of matching documents is known, display it -->
+    <xsl:call-template name="matchDocs"/>
+    
+    <!-- Display the matching documents -->
+    <xsl:call-template name="resultList">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <!-- next and prev links at bottom of page -->
+    <xsl:call-template name="resultNavigation"/>
+  </xsl:template>
+
+</xsl:stylesheet>  
Index: /main/tags/3.01/gsdl3/web/interfaces/classic/transform/style.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/classic/transform/style.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/classic/transform/style.xsl	(revision 21112)
@@ -0,0 +1,393 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <!-- some global parameters - these are set by whoever is invoking the transformation -->
+  <xsl:param name="interface_name"/>
+  <xsl:param name="library_name"/>
+
+  <!-- the main page layout template is here -->
+  <xsl:template match="page">
+    <html>
+      <xsl:call-template name="pageHead"/>
+      <body>
+	<table border="0" cellspacing="0" cellpadding="0" width="100%">
+	  <td valign="top" width="65">
+	    <img src="interfaces/default/images/spacer.gif" width="65" height="1" alt="" border="0"/>
+	  </td>
+	  <td>
+	    <center>
+	      <table width="537">
+		<tr><td>
+		    <xsl:apply-templates select="pageResponse"/>
+		    <xsl:call-template name="greenstoneFooter"/>
+		  </td></tr>
+	      </table>
+	    </center>
+	  </td>
+	</table>
+      </body>
+    </html>
+  </xsl:template>
+  
+  <!-- put a space in the title in case the actual value is missing - mozilla will not display a page with no title-->
+  <xsl:template name="pageHead">
+    <head>
+      <title>
+	<xsl:call-template name="pageTitle"/><xsl:text> </xsl:text>
+      </title>
+      <xsl:call-template name="globalStyle"/>
+      <xsl:call-template name="pageStyle"/>
+    </head>
+  </xsl:template>
+
+  <!-- any global style stuff should go in here -->
+  <xsl:template name="globalStyle">
+    <link rel="stylesheet" href="interfaces/default/style/core.css" type="text/css"/>
+  </xsl:template>
+  
+  <xsl:template name="greenstonePageBanner">
+    <center><img src="interfaces/default/images/gsdlhead.gif"><xsl:attribute name="alt"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/></xsl:attribute></img></center>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  <xsl:template name="genericPageBanner">
+    <xsl:param name='text'/>
+    <xsl:variable name="lang" select="/page/@lang"/>
+    <center>
+      <table width="100%">
+	<tr><td align="left"></td>
+	  <td align="right">
+	    <table>
+	      <tr><td class="gsbutton">
+		  <a class="gsbutton" href="{$library_name}?a=p&amp;sa=home">
+		    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'home_b')"/>
+		  </a>
+		</td></tr></table>
+	  </td>
+	</tr>
+      </table>
+    </center>
+    <xsl:call-template name="dividerBar"><xsl:with-param name='text' select="$text"/></xsl:call-template>
+  </xsl:template>
+  
+  <xsl:template name="standardPageBanner">
+    <xsl:param name="collName"/>
+    <xsl:param name="pageType"/>
+    <xsl:param name="clTop"/>
+    <xsl:variable name="this-element" select="/page/pageResponse/collection"/>
+    <table width="100%">
+      <tr>
+	<td align="left" rowspan="2">
+	  <a href="{$library_name}?a=p&amp;sa=about&amp;c={$collName}">
+	    <xsl:choose>
+	      <xsl:when test="$this-element/displayItem[@name='icon']">
+		<img border="0">
+		  <xsl:attribute name="src">
+		    <xsl:value-of select="$this-element/metadataList/metadata[@name='httpPath']"/>/images/<xsl:value-of select="$this-element/displayItem[@name='icon']"/>
+		  </xsl:attribute>	
+		  <xsl:attribute name="alt">
+		    <xsl:value-of select="$this-element/displayItem[@name='name']"/>
+		  </xsl:attribute>
+		  <xsl:attribute name="title">
+		    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'aboutpage')"/>
+		  </xsl:attribute>
+		</img>
+	      </xsl:when>
+	      <xsl:otherwise>
+		<b><xsl:value-of select="$this-element/displayItem[@name='name']"/></b>
+	      </xsl:otherwise>
+	    </xsl:choose>
+	  </a>
+	</td>
+	<td align="right">
+	  <xsl:call-template name="top-buttons">
+	    <xsl:with-param name="collName" select="$collName"/>
+	    <xsl:with-param name="pageType" select="$pageType"/>
+	  </xsl:call-template>
+	</td>
+      </tr>
+      <xsl:if test="not(/page/pageRequest/@action='d')">
+	<xsl:variable name="text">
+	  <xsl:choose>
+	    <xsl:when test="$pageType='browse'">
+	      <xsl:value-of select="/page/pageResponse/collection/serviceList/service[@name='ClassifierBrowse']/classifierList/classifier[@name=$clTop]/displayItem[@name='name']"/>
+	    </xsl:when>
+	    <xsl:otherwise>
+	      <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, concat($pageType, '_t'))"/>
+	    </xsl:otherwise>
+	  </xsl:choose>
+	</xsl:variable>
+	<tr>
+	  <td valign='top' align='right' width="200" height="57" style="background: url(interfaces/nzdl/images/titleicon.gif) no-repeat right bottom"><b><font size="+2" color='black'><xsl:value-of select="$text"/>&#160;&#160;</font></b></td>
+	</tr>
+      </xsl:if>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="top-buttons">
+    <xsl:param name="collName"/>
+    <xsl:param name="pageType"/>
+    <table cellspacing='4'>
+      <tr>
+	<td class="gsbutton">
+	  <a class="gsbutton" href="{$library_name}?a=p&amp;sa=home"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'home_tip')"/></xsl:attribute>
+	    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'home_b')"/>
+	  </a>
+	</td>
+	<!-- 	      <xsl:choose>
+	<xsl:when test="$pageType='help'">
+	<td class="gsbuttonoff">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_b')"/>
+      </td>
+      </xsl:when>
+	<xsl:otherwise>
+	<td class="gsbutton">
+	<a class="gsbutton" href="{$library_name}?a=p&amp;sa=help&amp;c={$collName}"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_tip')"/></xsl:attribute>
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_b')"/>
+      </a>
+      </td>
+      </xsl:otherwise>
+      </xsl:choose> -->
+	<xsl:choose>
+	  <xsl:when test="$pageType='pref'">
+	    <td class="gsbuttonoff">
+	      <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_b')"/>
+	    </td>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <td class="gsbutton">
+	      <a class="gsbutton" href="{$library_name}?a=p&amp;sa=pref&amp;c={$collName}"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_tip')"/></xsl:attribute>
+		<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_b')"/>
+	      </a>
+	    </td>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </tr>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="navigationBar">
+    <xsl:param name="collName"/>
+    <xsl:param name="clTop"/>
+    <xsl:variable name="this-element" select="/page/pageResponse/collection"/>
+    <xsl:variable name="this-service" select="/page/pageResponse/service/@name"/>
+    <xsl:variable name="classifiers" select="$this-element/serviceList/service[@name='ClassifierBrowse']/classifierList"/>
+    <table border='0' cellspacing='0' width='100%'>
+      <tr>
+	<xsl:if test="$this-element/serviceList/service[@type='query']">
+	    <xsl:call-template name="navbar-search"><xsl:with-param name="collName" select="$collName"/></xsl:call-template>
+	</xsl:if>
+	
+	<xsl:for-each select="$classifiers/classifier">
+	  <xsl:apply-templates select="." mode="navbar">
+	    <xsl:with-param name="collName" select="$collName"/>
+	    <xsl:with-param name="clTop" select="$clTop"/>
+	  </xsl:apply-templates>
+	</xsl:for-each>
+	
+	<xsl:if test="$this-element/serviceList/service[@name='PhindApplet']">
+	  <xsl:call-template name="navbar-phind"><xsl:with-param name="collName" select="$collName"/></xsl:call-template>
+	</xsl:if>
+      </tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="navbar-search">
+    <xsl:param name="collName"/>
+    <xsl:choose>
+      <xsl:when test="/page/pageRequest/@action='q'">	  
+	<td class="navbaroff"><nobr>search</nobr></td>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:variable name="service">
+	  <xsl:choose>
+	    <xsl:when test="not(/page/pageRequest/paramList/param[@name='ct'])">TextQuery</xsl:when>
+	    <xsl:when test="/page/pageRequest/paramList/param[@name='ct']/@value='0'">TextQuery</xsl:when>
+	    <xsl:when test="/page/pageRequest/paramList/param[@name='qt']/@value='1'"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='qfm']/@value='1'">AdvancedFieldQuery</xsl:when><xsl:otherwise>FieldQuery</xsl:otherwise></xsl:choose></xsl:when>
+	    <xsl:otherwise>TextQuery</xsl:otherwise>
+	  </xsl:choose>
+	</xsl:variable>
+	
+	<td class="navbar"><a class="navbar" href="{$library_name}?a=q&amp;rt=d&amp;s={$service}&amp;c={$collName}"><xsl:attribute name='title'><xsl:value-of select="/page/pageResponse/collection/service[@name=$service]/displayItem[@name='description']"/></xsl:attribute><nobr>search</nobr></a></td>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <xsl:template name="navbar-phind">
+    <xsl:param name="collName"/>
+    <xsl:choose>
+      <xsl:when test="/page/pageRequest/paramList/param[@name='s']/@value='PhindApplet'">
+	<td class="navbaroff"><nobr>phrases</nobr></td>
+      </xsl:when>
+      <xsl:otherwise>
+	<td class="navbar"><a class="navbar" href="{$library_name}?a=a&amp;rt=d&amp;s=PhindApplet&amp;c={$collName}"><xsl:if test="/page/pageResponse/collection/service[@name='PhindApplet']/displayItem[@name='description']"><xsl:attribute name='title'><xsl:value-of select="/page/pageResponse/collection/service[@name='PhindApplet']/displayItem[@name='description']"/></xsl:attribute></xsl:if><nobr>phrases</nobr></a></td>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="classifier" mode="navbar">
+    <xsl:param name="collName"/>
+    <xsl:param name="clTop"/>
+    <xsl:choose>
+      <xsl:when test="/page/pageRequest/@action='b' and $clTop=@name">
+	<td class="navbaroff"><nobr><xsl:value-of select="displayItem[@name='name']"/></nobr></td>
+      </xsl:when>
+      <xsl:otherwise>
+	<td class="navbar"><a class="navbar" href="{$library_name}?a=b&amp;rt=r&amp;s=ClassifierBrowse&amp;c={$collName}&amp;cl={@name}"><xsl:attribute name='title'><xsl:value-of select="displayItem[@name='description']"/></xsl:attribute><nobr><xsl:value-of select="displayItem[@name='name']"/></nobr></a></td>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template name="dividerBar">
+    <xsl:param name='text'/>
+    <p><table width='537' border="0" cellpadding="0" cellspacing="0">
+      <tr><td valign='bottom' height='20' background='interfaces/default/images/greenbar.gif'><font size="+1" color="black"><b>&#160;<xsl:value-of select='$text'/></b></font></td></tr></table></p>
+  </xsl:template>
+  
+  <xsl:template name="nzdlPageBanner">
+    <center>
+      <table width="537" cellspacing="0" cellpadding="0">
+	<tr valign="top">
+	  <td rowspan="2" align="left"><center><img src="interfaces/nzdl/images/nzdl2gr.gif" width="457" height="181" ><xsl:attribute name="alt"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'nzdl')"/></xsl:attribute></img></center></td>
+	  <td align="right"></td>
+	</tr>
+	
+	<tr>
+	  <td align="right"></td>
+	</tr>
+	
+	<tr>
+	  <td colspan="2"></td>
+	</tr>
+      </table>
+    </center>
+    <!-- end of page banner -->  
+  </xsl:template>
+
+  <xsl:template name="greenstoneFooter">
+    <center>
+      <p/>
+      <small>powered by greenstone3</small>
+    </center>
+  </xsl:template>
+  
+  <xsl:template name="imagegreenstone">
+    <img src="interfaces/nzdl/images/gsdl.gif" width="140" height="77" border="0" hspace='0'><xsl:attribute name='alt'><xsl:call-template name='textimagegreenstone'/></xsl:attribute></img>
+  </xsl:template>
+  
+  <!-- Display the appropriate image, depending on the node type -->
+  <xsl:template match="documentNode" mode="displayNodeIcon">
+    
+    <!-- Root node: book icon (open or closed) -->
+    <xsl:choose>
+      <xsl:when test="@nodeType='root'">
+	<xsl:choose>
+	  <xsl:when test="documentNode">
+	    <xsl:call-template name="openbookimg">
+	      <xsl:with-param name="title">Close this book</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:call-template name="closedbookimg">
+	      <xsl:with-param name="title">Open this document and view contents</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      
+      <!-- Internal node: folder icon (open or closed) -->
+      <xsl:when test="@nodeType='internal'">
+	<xsl:choose>
+	  <xsl:when test="documentNode">
+	    <xsl:call-template name="openfolderimg">
+	      <xsl:with-param name="title">Close this folder</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:call-template name="closedfolderimg">
+	      <xsl:with-param name="title">Open this folder and view contents</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      
+      <!-- Leaf node: page icon, and this is the default -->
+      <xsl:otherwise>
+	<xsl:call-template name="textpageimg">
+	  <xsl:with-param name="title">View the document</xsl:with-param>
+	</xsl:call-template>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+
+  <xsl:template name="openbookimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="28" height="23"
+      src="interfaces/default/images/openbook.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="closedbookimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="18" height="11"
+      src="interfaces/default/images/book.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="openfolderimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="23" height="15"
+      src="interfaces/default/images/openfldr.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="closedfolderimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="23" height="15"
+      src="interfaces/default/images/clsdfldr.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="textpageimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="16" height="21"
+      src="interfaces/default/images/itext.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="bookshelfimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="20" height="16"
+      src="interfaces/default/images/bshelf.gif"  
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="iconpdf">
+    <xsl:param name="alt">PDF</xsl:param>
+    <xsl:param name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'texticonpdf')"/></xsl:param>
+    <img border="0" width='26' height='26' 
+      src='interfaces/default/images/ipdf.gif' 
+      alt='{$alt}' title='{$title}'/>
+  </xsl:template>
+    
+  <xsl:template name="icondoc">
+    <xsl:param name="alt">Word</xsl:param>
+    <xsl:param name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'texticonmsword')"/></xsl:param>
+    <img border="0" width='26' height='26' 
+      src='interfaces/default/images/imsword.gif' 
+      alt='{$alt}' title='{$title}'/>
+  </xsl:template>
+
+</xsl:stylesheet>  
Index: /main/tags/3.01/gsdl3/web/interfaces/default/interfaceConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/interfaceConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/interfaceConfig.xml	(revision 21112)
@@ -0,0 +1,39 @@
+<interfaceConfig>
+  <actionList>
+    <action name='p' class='PageAction'>
+      <subaction name='home' xslt='home.xsl'/>
+      <subaction name='about' xslt='about.xsl'/>
+      <subaction name='help' xslt='help.xsl'/>
+      <subaction name='pref' xslt='pref.xsl'/>
+      <subaction name='nav' xslt='nav.xsl'/>
+      <subaction name="html" xslt="html.xsl"/>
+    </action>
+    <action name='q' class='QueryAction' xslt='query.xsl'/>
+    <action name='b' class='GS2BrowseAction' xslt='classifier.xsl'/>
+    <action name='a' class='AppletAction' xslt='applet.xsl'/>
+    <action name='d' class='DocumentAction' xslt='document.xsl'/>
+    <action name='xd' class='XMLDocumentAction'>
+      <subaction name='toc' xslt='document-toc.xsl'/>
+      <subaction name='text' xslt='document-content.xsl'/>
+    </action>
+    <action name='pr' class='ProcessAction' xslt='process.xsl'/>
+    <action name='s' class='SystemAction' xslt='system.xsl'/>
+  </actionList>
+  <languageList>
+    <language name="en">
+      <displayItem name='name'>English</displayItem>
+    </language>
+    <language name="fr">
+      <displayItem name='name'>FranÃ§ais</displayItem>
+    </language>
+    <language name='es'>
+      <displayItem name='name'>EspaÃ±ol</displayItem>
+    </language>
+    <language name="jp">
+      <displayItem name='name'>ã«ã»ãã</displayItem>
+    </language>
+  </languageList>
+  <optionList>
+    <option name="highlightQueryTerms" value="true"/>
+  </optionList>
+</interfaceConfig>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/style/core.css
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/style/core.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/style/core.css	(revision 21112)
@@ -0,0 +1,68 @@
+a.gsbutton,a:visited.gsbutton {
+    text-decoration:none; 
+    color:#345437; /* green colour */
+    font-weight: bold;
+}
+
+
+a.gsbutton:hover {
+    color: #cc9900 /* bright yellow */
+}
+	
+td.gsbutton { 
+    background-image: url(../images/greenbar.gif); 
+    height: 20; 
+    text-align: center;  
+}
+	
+td.gsbuttonoff { 
+    background-image: url(../images/greenbar.gif); 
+    height: 20; 
+    text-align: center; 
+    color:black; 
+    font-weight: bold; 
+}
+
+a.navbar, a:visited.navbar { 
+    text-decoration: none; 
+    font-size: larger; 
+    color:#345437;
+}
+
+a.navbar:hover {
+    color: #cc9900 /* bright yellow */
+}
+
+td.navbar { 
+    background-image: url(../images/greenbar.gif); 
+    height: 20; 
+    text-align: center;  
+    font-weight:bold;
+}
+	
+td.navbaroff {
+    background-image: url(../images/greenbar.gif); 
+    height: 20; 
+    text-align: center;  
+    font-weight:bold; 
+    color:black ; 
+    font-size: larger;
+}
+
+body {
+    background-color: #ffffff; 
+    background-image: url(../images/chalk.gif); 
+    color:#000000  
+}
+
+a[href] {
+    color: #006666; 
+}
+
+a[href]:visited {
+    color: #666633
+}
+
+a[href]:hover {
+    color: #cc9900 /* bright yellow */
+}
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/about.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/about.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/about.xsl	(revision 21112)
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/*/displayItem[@name='name']"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <xsl:apply-templates select="collection|serviceCluster"><xsl:with-param name="collName" select="$collName"/></xsl:apply-templates>
+  </xsl:template>
+  
+  <xsl:template match="collection|serviceCluster">
+    <xsl:param name="collName"/>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <p/><xsl:value-of select="displayItem[@name='description']" disable-output-escaping="yes"/>
+    <p/><xsl:apply-templates select="serviceList">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+  </xsl:template>
+  
+  
+  <xsl:template match="serviceList">
+    <xsl:param name="collName"/>
+    <h2><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'about.services')"/></h2>
+    <xsl:choose>
+      <xsl:when test="service">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'about.servicehelp')"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'about.noservices')"/>
+      </xsl:otherwise>
+    </xsl:choose>
+      <p/>
+    <xsl:if test="service">
+      <table border='0' cellspacing='4'>
+	<xsl:for-each select="service">
+	  <xsl:variable name="action"><xsl:choose>
+	      <xsl:when test="@type='query'">q</xsl:when>
+	      <xsl:when test="@type='browse'">b</xsl:when>
+	      <xsl:when test="@type='process'">pr</xsl:when>
+	      <xsl:when test="@type='applet'">a</xsl:when>
+	      <xsl:otherwise>DO_NOT_DISPLAY</xsl:otherwise>
+	    </xsl:choose></xsl:variable>
+	  <xsl:if test="$action !='DO_NOT_DISPLAY'">
+	    <tr><td class="gsbutton"><a class="gsbutton" href="{$library_name}?a={$action}&amp;rt=d&amp;s={@name}&amp;c={$collName}"><xsl:value-of select="displayItem[@name='name']"/></a></td><td><xsl:value-of select="displayItem[@name='description']"/></td></tr>
+	  </xsl:if>
+	</xsl:for-each>
+      </table>
+    </xsl:if>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+</xsl:stylesheet>  
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/applet.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/applet.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/applet.xsl	(revision 21112)
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:include href="style.xsl"/>
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="pageResponse/service/applet"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="../pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>
+      <p/>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+<!--      <xsl:call-template name="dividerBar">
+	<xsl:with-param name="text" select="service/displayItem[@name='name']"/>
+      </xsl:call-template>-->
+      <p/>
+      <xsl:copy-of select="service/applet"/>
+      <xsl:call-template name="dividerBar"/>
+    </center>
+  </xsl:template>
+</xsl:stylesheet>  
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifier.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifier.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifier.xsl	(revision 21112)
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="classifiertools.xsl"/>
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <xsl:variable name="serviceName"><xsl:value-of select="service/@name"/></xsl:variable>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>
+    <p/>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <p/>
+      <xsl:apply-templates select="service/classifierList">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="serviceName" select="$serviceName"/>
+      </xsl:apply-templates>
+      <p/>
+      <xsl:apply-templates select="classifier">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="serviceName" select="$serviceName"/>
+      </xsl:apply-templates>
+  </xsl:template>
+  
+  <xsl:template match="classifierList">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <xsl:variable name="selected" select="/page/pageResponse/classifier/@name"/>
+    <table width='100%'><tr>
+	<xsl:for-each select="classifier">
+	  <xsl:choose>
+	    <xsl:when test="@name=$selected">
+	      <td align='center'><font size="+1"><xsl:value-of select="displayItem[@name='name']"/></font></td>
+	    </xsl:when>
+	    <xsl:otherwise>
+	      <td align='center'><a href="{$library_name}?a=b&amp;rt=r&amp;s={$serviceName}&amp;c={$collName}&amp;cl={@name}"><font size="+1"><xsl:value-of select="displayItem[@name='name']"/></font></a></td></xsl:otherwise></xsl:choose>
+ 	</xsl:for-each>
+      </tr></table>
+  </xsl:template>
+
+</xsl:stylesheet>
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifiertools.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifiertools.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/classifiertools.xsl	(revision 21112)
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+ 
+  <xsl:template match="classifier">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <xsl:variable name="cl_name"><xsl:value-of select="@name"/></xsl:variable>
+    <xsl:choose>
+      <xsl:when test="/page/pageResponse/service/classifierList/classifier[@name=$cl_name]/@horizontalAtTop">
+	<xsl:apply-templates select="." mode="horizontal-at-top">
+	  <xsl:with-param name="collName" select="$collName"/>
+	  <xsl:with-param name="serviceName" select="$serviceName"/>
+	</xsl:apply-templates>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:apply-templates select="." mode="default">
+	  <xsl:with-param name="collName" select="$collName"/>
+	  <xsl:with-param name="serviceName" select="$serviceName"/>
+	</xsl:apply-templates>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+
+  <xsl:template match="classifier" mode="horizontal-at-top">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <xsl:choose>
+      <xsl:when test="classifierNode">
+      <div align='left'>
+	<xsl:for-each select='classifierNode'>
+	  <xsl:apply-templates select='.' mode="horizontal"><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates>
+	</xsl:for-each>
+	</div>
+	<table width='100%'>
+	  <tr valign='top'>
+	    <xsl:for-each select='classifierNode'>
+	      <xsl:apply-templates select='.' mode='process-all-children'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates>
+	    </xsl:for-each>
+	  </tr>
+	</table>
+	</xsl:when>
+      <xsl:otherwise>
+	<!-- there were no classifier nodes -->
+	<table width='100%'>  
+	  <xsl:for-each select='documentNode'>
+	    <tr valign='top'><xsl:apply-templates select='.'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates></tr>
+	  </xsl:for-each>
+	</table>
+      </xsl:otherwise>
+    </xsl:choose>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  <xsl:template match="classifier" mode="default"> <!-- the default -->
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <table width='100%'>
+      <xsl:for-each select='classifierNode|documentNode'>
+	<tr valign='top'><xsl:apply-templates select='.'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates></tr>
+	<xsl:if test='name()="classifierNode" and (child::classifierNode or child::documentNode)'>
+	  <!--recurse into the children-->
+	  <tr valign='top'><td><xsl:text> </xsl:text></td><xsl:apply-templates select='.' mode='process-all-children'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates></tr>
+	</xsl:if>
+      </xsl:for-each>
+    </table>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+  <!-- recursively process all nodes and documents -->
+  
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <td><a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@nodeID}&amp;dt={@docType}&amp;p.a=b&amp;p.s={$serviceName}"><xsl:apply-templates select="." mode="displayNodeIcon"/></a></td><td align='left'><xsl:value-of disable-output-escaping="yes"  select="metadataList/metadata[@name='Title']"/></td>
+  </xsl:template>
+  
+  <!-- icon + title template-->
+  <xsl:template match="classifierNode">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <td width='30'><a><xsl:attribute name='href'><xsl:value-of select='$library_name'/>?a=b&amp;rt=r&amp;s=<xsl:value-of select='$serviceName'/>&amp;c=<xsl:value-of select='$collName'/>&amp;cl=<xsl:value-of select='@nodeID'/><xsl:if test="classifierNode|documentNode">.pr</xsl:if></xsl:attribute><xsl:call-template name="bookshelfimg"/></a></td><td align='left'><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></td>
+  </xsl:template>
+  
+  <!-- the title is a link: an alternative template -->
+  <xsl:template match="classifierNode" mode="horizontal">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <xsl:choose>
+      <xsl:when test="classifierNode|documentNode"><b><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/><xsl:text> </xsl:text></b></xsl:when>
+      <xsl:otherwise>
+<a><xsl:attribute name='href'><xsl:value-of select='$library_name'/>?a=b&amp;rt=r&amp;s=<xsl:value-of select='$serviceName'/>&amp;c=<xsl:value-of select='$collName'/>&amp;cl=<xsl:value-of select='@nodeID'/></xsl:attribute><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></a><xsl:text> </xsl:text></xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+
+<!-- processing for the recursive bit -->
+<!-- if interleave=true, process all child nodes and documents, else just do nodes -->
+
+
+  <xsl:template match="classifierNode" mode="process-all-children">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <!-- only do this if there are children -->
+    <xsl:if test="classifierNode|documentNode">
+      <td>
+      <table>  
+	<xsl:for-each select='classifierNode|documentNode'>
+	  <tr valign='top'><xsl:apply-templates select='.'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates></tr>
+	  <xsl:if test='name()="classifierNode" and (child::classifierNode or child::documentNode)'>
+	    <tr valign='top'><td><xsl:text> </xsl:text></td><xsl:apply-templates select='.' mode='process-all-children'><xsl:with-param name='collName' select='$collName'/><xsl:with-param name='serviceName' select='$serviceName'/></xsl:apply-templates></tr>
+	  </xsl:if>	      
+	</xsl:for-each>
+      </table>
+      </td>
+    </xsl:if>
+  </xsl:template>
+    
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/config_format.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/config_format.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/config_format.xsl	(revision 21112)
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:xslt="output.xsl"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat"
+  extension-element-prefixes="java">
+  
+  <xsl:output method="xml"/>
+  <xsl:namespace-alias
+    stylesheet-prefix="xslt" result-prefix="xsl"/>
+
+  <xsl:template match="format">
+    <format>
+      <xsl:apply-templates/>
+    </format>
+  </xsl:template>
+  
+  <xsl:template match="gsf:template">
+    <xslt:template>
+      <xsl:copy-of select="@*"/>
+      <xsl:attribute name="priority">2</xsl:attribute> 
+      <xsl:if test=".//gsf:link">
+	<xslt:param name="serviceName"/>
+	<xslt:param name="collName"/>
+      </xsl:if>
+      <xsl:apply-templates/>
+    </xslt:template>
+  </xsl:template>
+  
+  <xsl:template match="gsf:link">
+    <xsl:choose>
+      <xsl:when test="@type='classifier'">
+	<a><xslt:attribute name='href'><xslt:value-of select='$library_name'/>?a=b&amp;rt=r&amp;s=<xslt:value-of select='$serviceName'/>&amp;c=<xslt:value-of select='$collName'/>&amp;cl=<xslt:value-of select='@nodeID'/><xslt:if test="classifierNode|documentNode">.pr</xslt:if><xslt:if test="parent::node()[@orientation='horizontal']">&amp;sib=1</xslt:if></xslt:attribute>
+	  <xsl:apply-templates/>
+	</a>
+      </xsl:when>
+      <xsl:otherwise> <!-- a document link -->
+	<a><xslt:attribute name="href"><xslt:value-of select='$library_name'/>?a=d&amp;c=<xslt:value-of select='$collName'/>&amp;d=<xslt:value-of select='@nodeID'/>&amp;dt=<xslt:value-of select='@docType'/><xslt:if test="@nodeType='leaf'">&amp;sib=1</xslt:if>&amp;p.a=<xslt:value-of select="/page/pageRequest/@action"/>&amp;p.sa=<xsl:value-of select="/page/pageRequest/@subaction"/>&amp;p.s=<xslt:value-of select="$serviceName"/></xslt:attribute><xsl:apply-templates/></a>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template match="gsf:icon">
+    <xsl:choose>
+      <xsl:when test="@type='classifier'">
+	<xslt:call-template name="bookshelfimg"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<xslt:apply-templates select="." mode="displayNodeIcon"/>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <xsl:template match="gsf:metadata[@format]">
+    <xslt:value-of disable-output-escaping="yes"><xsl:attribute name="select">java:org.greenstone.gsdl3.util.XSLTUtil.<xsl:value-of select="@format"/>(metadataList/metadata[@name='<xsl:apply-templates select="." mode="get-metadata-name"/>'], /page/@lang )</xsl:attribute></xslt:value-of>
+  </xsl:template>
+
+  <xsl:template match="gsf:metadata">
+    <xslt:value-of disable-output-escaping="yes"><xsl:attribute name="select">metadataList/metadata[@name='<xsl:apply-templates select="." mode="get-metadata-name"/>']</xsl:attribute></xslt:value-of>
+  </xsl:template>
+
+  <xsl:template match="gsf:metadata" mode="get-metadata-name">
+    <xsl:if test="@multiple='true'">all_</xsl:if><xsl:if test='@select'><xsl:value-of select='@select'/>_</xsl:if><xsl:if test="@separator">*<xsl:value-of select='@separator'/>*_</xsl:if><xsl:value-of select="@name"/>
+  </xsl:template>
+  
+  <xsl:template match="gsf:metadata-old">
+    <xslt:value-of disable-output-escaping="yes"><xsl:attribute name="select">metadataList/metadata[@name="<xsl:choose><xsl:when test="@select='parent'">parent_</xsl:when><xsl:when test="@select='root'">root_</xsl:when><xsl:when test="@select='ancestors'">ancestors'<xsl:value-of select='@separator'/>'_</xsl:when><xsl:when test="@select='siblings'">siblings_'<xsl:value-of select='@separator'/>'_</xsl:when></xsl:choose><xsl:value-of select="@name"/>"]</xsl:attribute></xslt:value-of>
+  </xsl:template>
+  
+  <xsl:template match="gsf:text">
+    <xslt:apply-templates select="nodeContent"/>
+  </xsl:template>
+  
+  <xsl:template match="gsf:choose-metadata">
+    <xslt:choose>
+      <xsl:for-each select="gsf:metadata">
+	<xslt:when><xsl:attribute name="test">metadataList/metadata[@name='<xsl:apply-templates select="." mode="get-metadata-name"/>']</xsl:attribute>
+	  <xsl:apply-templates select="."/>
+	</xslt:when>
+      </xsl:for-each>
+      <xsl:if test="gsf:default">
+	<xslt:otherwise><xsl:value-of select="gsf:default"/></xslt:otherwise>
+      </xsl:if>
+    </xslt:choose>
+  </xsl:template>
+  
+  <xsl:template match="gsf:switch">
+    <xsl:variable name="meta-name"><xsl:apply-templates select="gsf:metadata" mode="get-metadata-name"/></xsl:variable>
+    <xslt:variable name="meta"><xsl:choose><xsl:when test="@preprocess"><xslt:value-of select="util:{@preprocess}(metadataList/metadata[@name='{$meta-name}'])"/></xsl:when><xsl:otherwise><xslt:value-of select="metadataList/metadata[@name='{$meta-name}']"/></xsl:otherwise></xsl:choose></xslt:variable>
+    <xslt:choose>
+      <xsl:for-each select="gsf:when">
+	<xslt:when test="util:{@test}($meta, '{@test-value}')">
+	  <xsl:apply-templates/>
+	</xslt:when>
+      </xsl:for-each>
+      <xsl:if test="gsf:otherwise">
+	<xslt:otherwise>
+	  <xsl:apply-templates select="gsf:otherwise/node()"/>
+	</xslt:otherwise>
+      </xsl:if>
+    </xslt:choose>
+  </xsl:template>
+  
+  <xsl:template match="*">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+
+</xsl:stylesheet>
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/document.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/document.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/document.xsl	(revision 21112)
@@ -0,0 +1,393 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util gsf">
+
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+  <xsl:output method="html"/>
+  
+  <xsl:template name="pageTitle">
+    <xsl:variable name="docID" select="/page/pageResponse/document/@selectedNode"/>
+    <xsl:for-each select='/page/pageResponse/document/descendant::documentNode[@nodeID=$docID]/ancestor-or-self::documentNode'><xsl:if test='position()!=1'>::</xsl:if><xsl:value-of select="metadataList/metadata[@name='Title']"/></xsl:for-each>
+  </xsl:template>
+
+  <!-- this is hard coded for GATE, should somehow do it dynamically-->
+  <xsl:template name="pageStyle">
+    <style type="text/css">
+      <xsl:text disable-output-escaping="yes">
+	span.Location {  display:inline; color : red }
+	span.Person {  display:inline; color : green }
+	span.Organization {  display:inline; color : yellow }
+	span.Date {  display:inline; color : blue }
+	span.query_term {display: inline; background-color : yellow }
+      </xsl:text>
+    </style>
+  </xsl:template>
+  
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <!-- display the document -->
+    <xsl:apply-templates select="document">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+	
+  <xsl:template match="document">
+    <xsl:param name="collName"/>
+    <xsl:call-template name="documentHeading">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="documentContent">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+  </xsl:template>
+
+  <xsl:template name="documentHeading">
+    <xsl:param name="collName"/>
+    <xsl:variable name="doCoverImage" select="/page/pageResponse/format/gsf:option[@name='coverImages']/@value"/>
+    <xsl:variable name="doTOC" select="/page/pageResponse/format/gsf:option[@name='documentTOC']/@value"/>
+    <xsl:variable name="p.a" select="/page/pageRequest/paramList/param[@name='p.a']/@value"/>
+    <xsl:variable name="p.sa" select="/page/pageRequest/paramList/param[@name='p.sa']/@value"/>
+    <xsl:variable name="p.s" select="/page/pageRequest/paramList/param[@name='p.s']/@value"/>
+    <xsl:variable name="p.c"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='p.c']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='p.c']/@value"/></xsl:when><xsl:otherwise><xsl:value-of select="$collName"/></xsl:otherwise></xsl:choose></xsl:variable>
+    <table><tr><td>
+	  <a href="{$library_name}?a={$p.a}&amp;sa={$p.sa}&amp;s={$p.s}&amp;c={$p.c}&amp;rt=rd"><xsl:call-template name="openbookimg"><xsl:with-param name="title">Close this book and return to search/browse</xsl:with-param>
+	  </xsl:call-template></a></td>
+	<td valign='top'><b><font size="+1">&#160;
+	      <xsl:choose><xsl:when test="@docType='simple'"><xsl:value-of select="metadataList/metadata[@name='Title']" disable-output-escaping="yes"/></xsl:when><xsl:otherwise><xsl:value-of select="documentNode/metadataList/metadata[@name='Title']" disable-output-escaping="yes"/></xsl:otherwise></xsl:choose>
+	    </font></b></td>
+      </tr></table>
+
+    <table cellpadding="10" cellspacing="0">
+      <tr>
+	<td valign="top" align="left">
+	  <xsl:if test="string($doCoverImage) != 'false' and (@docType='paged' or @docType='hierarchy')">
+	    <xsl:call-template name="coverImage"/>
+	    <p/>
+	  </xsl:if>
+	  
+	  <xsl:call-template name="documentButtons">
+	    <xsl:with-param name="collName" select="$collName"/>
+	  </xsl:call-template>
+	  <!--<xsl:call-template name="enrichServices">
+	    <xsl:with-param name="collName" select="$collName"/>
+	  </xsl:call-template>-->
+	</td>
+	<xsl:if test="string($doTOC) != 'false'">
+	  <td valign="top" align="left">
+	    <xsl:call-template name="TOC">
+	      <xsl:with-param name="collName" select="$collName"/>
+	    </xsl:call-template>
+	  </td>
+	</xsl:if>
+      </tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="coverImage">
+    <img><xsl:attribute name='src'><xsl:value-of select="/page/pageResponse/collection/metadataList/metadata[@name='httpPath']"/>/index/assoc/<xsl:value-of select="metadataList/metadata[@name='archivedir']"/>/cover.jpg</xsl:attribute></img>
+  </xsl:template>
+
+  <xsl:template name="documentButtons">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docID" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="sib" select="/page/pageRequest/paramList/param[@name='sib']/@value"/>
+    <xsl:variable name="ec" select="/page/pageRequest/paramList/param[@name='ec']/@value"/>
+    <xsl:variable name="ed" select="/page/pageRequest/paramList/param[@name='ed']/@value"/>
+    
+    <table>
+      <!-- expand document -->
+      <!-- this doesn't work with MG so comment it out for now -->
+      
+      <xsl:if test="@docType = 'hierarchy' or @docType = 'paged'">
+      <tr><td class="gsbutton">
+	  <xsl:choose><xsl:when test="string($ed)='1'">
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ed=0"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_doc_tip')"/></xsl:attribute>
+		  <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_doc_b')"/></font>
+	      </a>
+	    </xsl:when>
+	    <xsl:otherwise>
+	  <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ed=1"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_doc_tip')"/></xsl:attribute>
+	    <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_doc_b')"/></font>
+	  </a>
+	    </xsl:otherwise>
+	  </xsl:choose>
+	</td></tr>
+      </xsl:if> 
+      
+      <!-- expand contents -->
+      <xsl:if test="@docType = 'hierarchy' and string(/page/pageResponse/format/gsf:option[@name='documentTOC']/@value) != 'false'">
+      <tr><td class="gsbutton">
+	  <xsl:choose><xsl:when test="string($ec)='1'">
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ec=0"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_contents_tip')"/></xsl:attribute>
+		  <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.contract_contents_b')"/></font>
+	      </a>
+	    </xsl:when>
+	    <xsl:otherwise>
+	      <a class="gsbutton" href="{$library_name}?a=d&amp;d={$docID}&amp;c={$collName}&amp;sib={$sib}&amp;ec=1"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_contents_tip')"/></xsl:attribute>
+		<font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.expand_contents_b')"/></font>
+	      </a>
+	    </xsl:otherwise>
+	  </xsl:choose>
+	</td></tr>
+      </xsl:if>
+      <!-- detach page -->
+      <xsl:variable name="paramList" select="/page/pageRequest/paramList"/>
+      <tr><td class="gsbutton">
+	  <a class="gsbutton" target="_blank"><xsl:attribute name="href"><xsl:value-of select="$library_name"/>?a=d&amp;d=<xsl:value-of select="$docID"/>&amp;c=<xsl:value-of select="$collName"/>&amp;sib=<xsl:value-of select="$sib"/>&amp;dt=<xsl:value-of select="$paramList/param[@name='dt']/@value"/>&amp;ec=<xsl:value-of select="$paramList/param[@name='ec']/@value"/>&amp;et=<xsl:value-of select="$paramList/param[@name='et']/@value"/>&amp;p.a=<xsl:value-of select="$paramList/param[@name='p.a']/@value"/>&amp;p.s=<xsl:value-of select="$paramList/param[@name='p.s']/@value"/>&amp;p.sa=<xsl:value-of select="$paramList/param[@name='p.sa']/@value"/></xsl:attribute><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'doc.detach_page_tip')"/></xsl:attribute>
+	    <font size='-2'><xsl:value-of disable-output-escaping='yes' select="util:getInterfaceText($interface_name, /page/@lang, 'doc.detach_page_b')"/></font>
+	  </a></td></tr>
+    </table>
+  </xsl:template>
+
+  <xsl:template name="TOC">
+    <xsl:param name="collName"/>
+    <xsl:choose>
+      <xsl:when test="@docType='hierarchy'">
+	<xsl:call-template name="hierarchicalContents">
+	  <xsl:with-param name="collName" select="$collName"/>
+	</xsl:call-template>
+      </xsl:when>
+      <xsl:when test="@docType='paged'">
+	<xsl:call-template name="pagedContents">
+	  <xsl:with-param name="collName" select="$collName"/>
+	</xsl:call-template>
+      </xsl:when>
+    </xsl:choose>
+  </xsl:template>
+    
+  <xsl:template name="hierarchicalContents">
+    <xsl:param name="collName"/>
+    <xsl:variable name="oc" select="/page/pageRequest/paramList/param[@name='oc']/@value"/>
+    <xsl:variable name="d" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="sib" select="/page/pageRequest/paramList/param[@name='sib']/@value"/>
+    <xsl:variable name="ec" select="/page/pageRequest/paramList/param[@name='ec']/@value"/>
+    <xsl:if test="documentNode[@nodeID]">
+      <table>
+	<tr valign='top'><td>
+	    <xsl:choose><xsl:when test="string($oc)='0'">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={$d}&amp;sib={$sib}&amp;oc=1"><xsl:call-template name="closedfolderimg"><xsl:with-param name="title">open the table of contents</xsl:with-param></xsl:call-template></a>
+	      </xsl:when>
+	      <xsl:otherwise><a href="{$library_name}?a=d&amp;c={$collName}&amp;d={$d}&amp;sib={$sib}&amp;oc=0"><xsl:call-template name="openfolderimg"><xsl:with-param name="title">close the table of contents</xsl:with-param></xsl:call-template></a></xsl:otherwise></xsl:choose>
+	  </td><td>Table of Contents</td></tr>
+	<xsl:if test="string($oc)!='0'">
+	  <tr><td>&#160;</td><td>
+	      <table>
+		<xsl:for-each select="documentNode/documentNode[@nodeID]">
+		  <xsl:apply-templates select='.'><xsl:with-param name="collName" select="$collName"/><xsl:with-param name="ec" select="$ec"/></xsl:apply-templates>
+		</xsl:for-each>
+	      </table>
+	    </td></tr>
+	</xsl:if>
+      </table>
+    </xsl:if>
+  </xsl:template>
+  
+  <!-- each icon-title pair is a row in a table. children go in a table in another row -->
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:param name="ec"/>
+    <!-- Display the appropriate image, depending on the node type -->
+    <tr><td valign="top">
+	<a><xsl:attribute name="href"><xsl:value-of select='$library_name'/>?a=d&amp;c=<xsl:value-of select='$collName'/>&amp;d=<xsl:value-of select='@nodeID'/><xsl:if test="documentNode">.pr</xsl:if><xsl:if test="@nodeType='leaf'">&amp;sib=1</xsl:if><xsl:if test="string($ec) = '1'">&amp;ec=1</xsl:if></xsl:attribute>
+	  <xsl:apply-templates select="." mode="displayNodeIcon"/>
+	</a>
+      </td>
+      
+      <!-- Display associated title, bolded if the node has content -->
+      <td valign="top">
+	<xsl:choose>
+	  <xsl:when test="nodeContent">
+	    <b><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></b>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </td>
+    </tr>
+    <!-- Apply recursively to the children of this node -->
+    <xsl:if test="documentNode[@nodeID]">
+      <tr><td>&#160;</td><td><table>
+	    <xsl:apply-templates select="documentNode[@nodeID]">
+	      <xsl:with-param name="collName" select="$collName"/>
+	      <xsl:with-param name="ec" select="$ec"/>
+	      <!--<xsl:with-param name="depth" select="$depth + 1"/>-->
+	    </xsl:apply-templates>
+	  </table>
+	</td></tr>
+    </xsl:if>
+  </xsl:template>
+  
+  <!-- default content is to print the title and content of any documentNodes that have nodeContent -->
+  <xsl:template name="documentContent">
+    <xsl:choose>
+      <xsl:when test="@docType='simple'">
+	<xsl:apply-templates select="nodeContent"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:apply-templates select="descendant-or-self::node()/documentNode" mode="content"/>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <xsl:template match="documentNode" mode="content">
+    <xsl:if test="nodeContent">
+      <xsl:if test="metadataList/metadata[@name='Title']">
+	<h3><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></h3>
+      </xsl:if>
+      <xsl:apply-templates select="nodeContent"/>
+    </xsl:if>
+  </xsl:template>
+
+  <!-- the actual text/content -->
+  <xsl:template match="nodeContent">
+    <p/>
+    <xsl:for-each select="node()">
+      <xsl:choose>
+	<xsl:when test="not(name())"><xsl:value-of select="." disable-output-escaping="yes"/></xsl:when>
+	<xsl:otherwise><xsl:apply-templates select="."/></xsl:otherwise>
+      </xsl:choose>
+    </xsl:for-each>
+  </xsl:template>
+  
+  <!-- match any file nodes -->
+  <xsl:template match="file">
+    <xsl:variable name="httpPath" select="/page/pageResponse/collection/metadataList/metadata[@name='httpPath']"/>
+    <xsl:choose>
+      <xsl:when test="util:isImage(@mimeType)">
+	<img src="{$httpPath}/{@href}"/>
+      </xsl:when>
+      <xsl:otherwise>
+	<a href="{$httpPath}/{@href}"><xsl:value-of select="@href"/></a>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  <!-- match any link nodes -->
+  <xsl:template match="link">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/> 
+    <xsl:variable name="actionargs">
+      <xsl:choose>
+	<xsl:when test="@type='document'">a=d</xsl:when>
+	<xsl:when test="@type='query'">a=q&amp;s=<xsl:value-of select="@service"/>&amp;rt=rd</xsl:when>
+	<xsl:otherwise>
+	  p
+	</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <xsl:variable name="serviceargs">
+      <xsl:for-each select="param">&amp;s1.<xsl:value-of select="@name"/>=<xsl:value-of select="@value"/></xsl:for-each>
+  </xsl:variable>
+    <a href="{$library_name}?{$actionargs}&amp;c={$collName}{$serviceargs}"><xsl:value-of disable-output-escaping="yes" select="."/></a>
+  </xsl:template>
+
+  <!-- match any annotations and make them span elements -->
+  <xsl:template match="annotation">
+    <span class="{@type}"><xsl:value-of disable-output-escaping="yes" select="."/></span>
+  </xsl:template>
+  
+  <!-- paged naviagtion : INCOMPLETE!!-->
+  <xsl:template name="pagedContents">
+    <xsl:param name="collName"/>
+    <xsl:variable name="pos" select="nodeStructureInfo/info[@name='siblingPosition']/@value"/>
+    <xsl:variable name="length" select="nodeStructureInfo/info[@name='numSiblings']/@value"/>
+    <xsl:variable name="children" select="nodeStructureInfo/info[@name='numChildren']/@value"/>
+    <table>
+      <xsl:choose>
+	<xsl:when test="$pos=-1"><!-- a doc -->
+	  <tr valign="top">
+	    <td align="left">
+	    </td>
+	    <td align="center"><center>
+		<b><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.pages', $children)"/></b>
+	      </center></td>
+	    <td align="right">
+	      <a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.fc"><img src="interfaces/default/images/more.gif" border="0" align="absbottom" /></a>
+	    </td>
+	  </tr>
+	</xsl:when>
+	
+	<xsl:otherwise> <!-- an internal node -->	  
+	  <tr valign="top">
+	    <td align="left">
+	      <xsl:if test=" not ( $pos = 1 )">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.ps"><img src="interfaces/default/images/less.gif" border="0" align="absbottom" /></a>
+	      </xsl:if>
+	    </td>
+	    <td align="center"><center>
+		<b><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.pageof', concat($pos, ';', $length))"/></b>
+	      </center></td>
+	    <td align="right">
+	      <xsl:if test=" not($pos = $length)">
+		<a href="{$library_name}?a=d&amp;c={$collName}&amp;d={@selectedNode}.ns"><img src="interfaces/default/images/more.gif" border="0" align="absbottom" /></a>
+	      </xsl:if>
+	    </td>
+	  </tr>
+	</xsl:otherwise>
+      </xsl:choose>
+      <tr valign="middle">
+	<td align='center' valign='top' colspan='3'>
+	  <form name="GotoForm" method="get" action="{$library_name}">
+	    <input type='hidden' name='a' value='d'/>
+	    <input type='hidden' name='c' value='{$collName}'/>
+	    <input type='hidden' name='d' value='{@selectedNode}'/>
+	    <input type="text" name="gp" size="3" maxlength="4"/>
+	    <input type="submit"><xsl:attribute name='value'><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'document.gotopage')"/></xsl:attribute></input>
+	  </form>
+	</td>
+      </tr>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="enrichServices">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docID" select="/page/pageRequest/paramList/param[@name='d']/@value"/>
+    <xsl:variable name="request-params" select="/page/pageRequest/paramList"/>
+    <xsl:for-each select="../serviceList/service">
+      <table border='1' cellspacing='0'>
+	<tr><td>
+	    <p /><xsl:value-of select="displayItem[@name='name']"/><p/>
+	    <form name="EnrichForm" method="get" action="{$library_name}">
+	      <xsl:apply-templates select="paramList"/>
+	      <input type='hidden' name='a' value='d'/>
+	      <input type='hidden' name='d' value='{$docID}'/>
+	      <input type='hidden' name='c' value='{$collName}'/>
+	      <xsl:if test='$request-params/param[@name="sib"]'>
+		<input type='hidden' name='sib'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="sib"]/@value'/></xsl:attribute></input></xsl:if>
+	      <input type='hidden' name='s' value='{@name}'/>
+	      <input type='hidden' name='p.a'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.a"]'/></xsl:attribute></input>
+	      <input type='hidden' name='p.sa'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.sa"]'/></xsl:attribute></input>
+	      <input type='hidden' name='p.s'><xsl:attribute name='value'><xsl:value-of select='$request-params/param[@name="p.s"]'/></xsl:attribute></input>
+	      <input type='hidden' name='end' value='1'/>
+	      <input type="submit"><xsl:attribute name="value"><xsl:value-of select="displayItem[@name='submit']"/></xsl:attribute></input>
+	    </form>
+	  </td></tr>
+      </table>
+    </xsl:for-each>
+  </xsl:template>
+
+  <xsl:template match="paramList" mode="hidden">
+    <xsl:for-each select="param">
+      <input type='hidden' name='{@name}' value='{@value}'/><xsl:text>
+      </xsl:text>
+    </xsl:for-each>
+  </xsl:template>
+  
+</xsl:stylesheet>
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/help.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/help.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/help.xsl	(revision 21112)
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+   <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+  
+  <xsl:template name="pageTitle">
+	<xsl:value-of select="/page/pageResponse/*/displayItem[@name='name']"/>: <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_t')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+      <xsl:with-param name="pageType">help</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    
+    some help text goes here
+  </xsl:template>
+ 
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/home.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/home.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/home.xsl	(revision 21112)
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:call-template name="greenstonePageBanner"/> 
+    <xsl:apply-templates select="collectionList"/>   
+    <xsl:apply-templates select="serviceClusterList"/>
+  </xsl:template>
+
+  <xsl:template match="collectionList">
+    <xsl:for-each select="collection"> 
+      <p>
+      <a href="{$library_name}?a=p&amp;sa=about&amp;c={@name}">
+	  <img width="150" border="1">
+	    <xsl:attribute name="src">
+	      <xsl:value-of select="metadataList/metadata[@name='httpPath']"/>/images/<xsl:value-of select="displayItem[@name='icon']"/>
+	    </xsl:attribute>
+	    <xsl:attribute name="alt">
+	      <xsl:value-of select="displayItem[@name='name']"/>
+	    </xsl:attribute>
+	  </img>
+	</a>
+      </p>
+    </xsl:for-each> 
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template> 
+  
+  <xsl:template match="serviceClusterList">
+    <xsl:for-each select="serviceCluster"> 
+      <p/><a href="{$library_name}?a=p&amp;sa=about&amp;c={@name}"><xsl:value-of select='@name'/><xsl:value-of select="displayItem[@name='name']"/></a>
+    </xsl:for-each>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+</xsl:stylesheet>  
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/html.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/html.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/html.xsl	(revision 21112)
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:output method="html"/>  
+
+  <xsl:template match="page">
+    <html>
+      <head></head>
+      <frameset rows="68,*" noresize="" border="0">
+	<frame frameborder="0"><xsl:attribute name="src">?a=p&amp;sa=nav&amp;c=<xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value" /></xsl:attribute></frame>
+	<frame frameborder="0"><xsl:attribute name="src"><xsl:value-of select="/page/pageRequest/paramList/param[@name='url']/@value"/></xsl:attribute></frame>
+	<noframes>
+	  <p><xsl:value-of select="util:getInterfaceText('default', /page/@lang, 'textframebrowser')"/></p>
+	</noframes>
+      </frameset>
+    </html>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/nav.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/nav.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/nav.xsl	(revision 21112)
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+   <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+  
+  <xsl:template match="page">
+    <html>
+      <head>
+	<title>
+	  <xsl:call-template name="pageTitle"/><xsl:text> </xsl:text>
+	</title>
+	<xsl:call-template name="globalStyle"/>
+	<base target="_top"/>
+      </head>
+      <body style="background-image: none;">
+	<xsl:apply-templates select="pageResponse"/>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/*/displayItem[@name='name']"/>
+  </xsl:template>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <div align="right">
+    <xsl:call-template name="top-buttons">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+      </div>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/pref.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/pref.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/pref.xsl	(revision 21112)
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+   <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/*/displayItem[@name='name']"/>: <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_t')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName" select="/page/pageRequest/paramList/param[@name='c']/@value"/>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+      <xsl:with-param name="pageType">pref</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    
+    <form name="PrefForm" method="get" action="{$library_name}">
+      <input type='hidden' name='a' value='p'/>
+      <input type='hidden' name='sa' value='pref'/>
+      <table width='100%'>
+	<tr><td><h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.prespref')"/></h3></td></tr>
+	<tr><td><xsl:call-template name="pres-prefs"/></td></tr>
+	<tr><td></td></tr>
+	<tr><td><h3><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.searchpref')"/></h3></td></tr>
+	<tr><td><xsl:call-template name="search-prefs"/></td></tr>
+	<tr><td><input type='submit'><xsl:attribute name="value"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.set_prefs')"/></xsl:attribute></input></td></tr>
+      </table>
+    </form>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+  
+  <xsl:template name="pres-prefs">
+    <table>
+      <tr><td><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.interfacelang')"/></td><td align='left'><xsl:call-template name="lang-list"/></td></tr>
+      <tr><td><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.encoding')"/></td><td align='left'>x</td></tr>
+    </table>
+  </xsl:template>
+ 
+  <xsl:template name="lang-list">
+    <xsl:variable name="current" select="/page/@lang"/>
+    <select name="l">
+      <xsl:for-each select="/page/pageResponse/languageList/language">
+	<option value="{@name}"><xsl:if test="@name=$current"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="displayItem[@name='name']"/></option>
+      </xsl:for-each>
+    </select>
+  </xsl:template>
+  
+  <xsl:template name="search-prefs">
+    <xsl:variable name="hits"><xsl:choose><xsl:when test="/page/pageRequest/paramList/param[@name='hitsPerPage']"><xsl:value-of select="/page/pageRequest/paramList/param[@name='hitsPerPage']/@value"/></xsl:when><xsl:otherwise>20</xsl:otherwise></xsl:choose></xsl:variable>
+    <table>
+      <tr>
+	<td><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.hitsperpage')"/></td>
+	<td align='left'><select name="hitsPerPage">
+	    <option value="20"><xsl:if test="$hits=20"><xsl:attribute name="selected"></xsl:attribute></xsl:if>20</option> 
+	    <option value="50"><xsl:if test="$hits=50"><xsl:attribute name="selected"></xsl:attribute></xsl:if>50</option>
+	    <option value="100"><xsl:if test="$hits=100"><xsl:attribute name="selected"></xsl:attribute></xsl:if>100</option>
+	    <option value="-1"><xsl:if test="$hits=-1"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref.all')"/></option>
+	  </select>
+	</td>
+      </tr>
+    </table>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/process.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/process.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/process.xsl	(revision 21112)
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/serviceCluster/metadataList/metadata[@name='Title']"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="clusterName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <center>
+      <xsl:call-template name="standardPageBanner">
+	<xsl:with-param name="collName" select="$clusterName"/>
+      </xsl:call-template>
+      <xsl:call-template name="navigationBar">
+	<xsl:with-param name="collName" select="$clusterName"/>
+      </xsl:call-template>      
+      <xsl:apply-templates select="service">
+	<xsl:with-param name="clusterName" select="$clusterName"/>
+      </xsl:apply-templates>
+      <xsl:apply-templates select="status">
+	<xsl:with-param name="clusterName" select="$clusterName"/>
+      </xsl:apply-templates>
+    </center>
+  </xsl:template>
+
+  <xsl:template match="service">
+    <xsl:param name="clusterName"/>
+    <xsl:variable name='subaction' select="/page/pageRequest/@subaction"/>
+    <xsl:variable name='action' select="/page/pageRequest/@action"/>
+    <xsl:variable name='lang' select="/page/@lang"/>
+    <p/>
+    <form name="BuildForm" method="get" action="{$library_name}">
+      <xsl:apply-templates select="paramList"/>
+      <input type="hidden" name="a" value="{$action}"/>
+      <input type="hidden" name="c" value="{$clusterName}"/>
+      <input type="hidden" name="sa" value="p"/>
+      <input type="hidden" name='l' value='{$lang}'/>
+      <input type="hidden" name="s" value="{@name}"/>
+      <input type="hidden" name="rt" value="r"/>
+      <input type="submit"><xsl:attribute name="value"><xsl:value-of select="displayItem[@name='submit']"/></xsl:attribute></input>
+    </form>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+  
+  <xsl:template match="status">
+    <xsl:param name="clusterName"/>
+    <center/>
+    <applet code="org.greenstone.gsdl3.build.StatusDisplay.class" codebase='lib' archive='gsdl3.jar, xercesImpl.jar, xml-apis.jar' width='600' height='150'>The status display applet.
+      <param name='library'><xsl:attribute name="value"><xsl:value-of select='$library_name'/>?a=pr&amp;rt=s&amp;c=<xsl:value-of select='$clusterName'/>&amp;s=<xsl:value-of select='/page/pageResponse/service/@name'/>&amp;o=xml&amp;ro=1&amp;l=<xsl:value-of select='/page/@lang'/>&amp;pid=<xsl:value-of select="@pid"/><xsl:apply-templates select="/page/pageResponse/service/paramList" mode="cgi"/></xsl:attribute></param>
+      <param name='initial_text'><xsl:attribute name="value"><xsl:value-of select="."/></xsl:attribute></param>
+      <param name='initial_code'><xsl:attribute name="value"><xsl:value-of  select="@code"/></xsl:attribute></param></applet>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  
+</xsl:stylesheet>  
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/query.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/query.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/query.xsl	(revision 21112)
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="service-params.xsl"/>
+  <xsl:include href="querytools.xsl"/>
+  
+  <xsl:output method="html"/>
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+   <!-- <center>-->
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+<!--    <xsl:call-template name="dividerBar">
+      <xsl:with-param name="text" select="service/displayItem[@name='name']"/>
+    </xsl:call-template>-->
+    <xsl:apply-templates select="service">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+  <!--  </center>-->
+    <xsl:if test="documentNodeList">
+      <xsl:call-template name="query-response">
+	<xsl:with-param name="collName" select="$collName"/>
+      </xsl:call-template>
+    </xsl:if>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+
+  <!-- layout the response -->
+  <xsl:template name="query-response">
+    <xsl:param name="collName"/>
+    <xsl:call-template name="dividerBar"><xsl:with-param name='text'><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.results')"/></xsl:with-param></xsl:call-template>
+    
+    <!-- If query term information is available, display it -->
+    <xsl:call-template name="termInfo"/>
+    <xsl:call-template name="matchDocs"/>
+    
+    <!-- next and prev links at top of results-->
+    <xsl:call-template name="resultNavigation">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <!-- Display the matching documents -->
+    <xsl:call-template name="resultList">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <!-- next and prev links at bottom of page -->
+    <xsl:call-template name="resultNavigation">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+  </xsl:template>
+  
+
+  <xsl:template match="service">
+    <xsl:param name="collName"/>
+    <xsl:variable name="subaction" select="../pageRequest/@subaction"/>
+    <form name="QueryForm" method="get" action="{$library_name}">
+      <input type="hidden" name="a" value="q"/>
+      <input type="hidden" name="sa" value="{$subaction}"/>
+      <input type="hidden" name="rt" value="rd"/>
+      <input type="hidden" name="s" value="{@name}"/>
+      <input type="hidden" name="c" value="{$collName}"/>
+      <xsl:if test="not(paramList/param[@name='startPage'])">
+	<input type="hidden" name="startPage" value="1"/>
+      </xsl:if>
+      <xsl:apply-templates select="paramList"/>
+      <br/>
+      <input type="submit"><xsl:attribute name="value"><xsl:value-of select="displayItem[@name='submit']"/></xsl:attribute></input>
+    </form>
+  </xsl:template>
+   
+</xsl:stylesheet>  
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/querytools.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/querytools.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/querytools.xsl	(revision 21112)
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- have changed this so it uses service hitsPerPage and startPage if the service description has a param called hitsPerPage,  otherwise uses interface ones -->
+  <xsl:template name="resultNavigation">
+    <xsl:param name="collName"/>
+    <!-- hits type -->
+    <xsl:variable name='ht'>
+      <xsl:choose>
+	<xsl:when test="/page/pageResponse/service/paramList/param[@name='hitsPerPage']">s</xsl:when>
+	<xsl:otherwise>i</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    
+    <xsl:variable name="param-list" select="/page/pageRequest/paramList"/>
+    <!-- hits per page -->
+    <xsl:variable name="hpp">
+      <xsl:choose>
+	<xsl:when test="$ht='s'"><xsl:value-of select="$param-list/param[@name='s1.hitsPerPage']/@value"/></xsl:when>
+	<xsl:when test="$param-list/param[@name='hitsPerPage']"><xsl:value-of select="$param-list/param[@name='hitsPerPage']/@value"/></xsl:when>
+	<xsl:otherwise>20</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <!-- total docs - this may be in numDocsMatched or numDocsReturned metadata -->
+    <xsl:variable name="td">
+      <xsl:choose>
+	<xsl:when test="/page/pageResponse/metadataList/metadata[@name='numDocsReturned']">
+	  <xsl:value-of select="/page/pageResponse/metadataList/metadata[@name='numDocsReturned']"/>
+	</xsl:when>
+	<xsl:when test="/page/pageResponse/metadataList/metadata[@name='numDocsMatched']">
+	  <xsl:value-of select="/page/pageResponse/metadataList/metadata[@name='numDocsMatched']"/>
+	</xsl:when>
+	<xsl:otherwise> <!-- this is just a fall back - shoudl always have the metadata -->
+	  <xsl:value-of select="count(/page/pageResponse/documentNodeList/documentNode)"/>
+	</xsl:otherwise>	
+      </xsl:choose>
+    </xsl:variable>
+
+    <!-- only continue if hpp != -1 and td > hpp -->
+    <xsl:if test="not($hpp=-1) and $td &gt; $hpp">
+      <!-- start page -->
+      <xsl:variable name="here">
+	<xsl:choose>
+	  <xsl:when test="$ht='s'"><xsl:value-of select="$param-list/param[@name='s1.startPage']/@value"/></xsl:when>
+	  <xsl:when test="$param-list/param[@name='startPage']"><xsl:value-of select="$param-list/param[@name='startPage']/@value"/></xsl:when>
+	  <xsl:otherwise>1</xsl:otherwise>
+	</xsl:choose>
+      </xsl:variable>
+      <xsl:variable name="sa" select="/page/pageRequest/@subaction"/>
+      <xsl:variable name="service" select="$param-list/param[@name='s']/@value"/>
+      <xsl:variable name="prev" select="$here - 1"/>
+      <xsl:variable name="next" select="$here + 1"/>
+      <xsl:variable name="page-param"><xsl:if test="$ht='s'">s1.</xsl:if>startPage</xsl:variable>
+      <p />
+      <center>
+	<table cellspacing='0' cellpadding='0' width='70%'>
+	  <tr>
+	    <td align='left'>
+	      <xsl:if test="$here &gt; 1"><a href="{$library_name}?a=q&amp;sa={$sa}&amp;c={$collName}&amp;s={$service}&amp;rt=r&amp;{$page-param}={$prev}"><img src="interfaces/default/images/less.gif" width='30' height='16' border='0' align='top'/><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.matches')"/><xsl:text> </xsl:text><xsl:value-of select="number(($prev - 1)*$hpp + 1)"/> - <xsl:value-of select="number(($prev * $hpp))"/></a></xsl:if>
+	    </td>
+	    <td align='right'>
+	      <xsl:if test="(($here * $hpp) + 1)  &lt; $td">
+		<xsl:variable name='m' select="number($next * $hpp)"/>
+		<xsl:variable name='mm'><xsl:choose><xsl:when test="$m &lt; $td"><xsl:value-of select='$m'/></xsl:when><xsl:otherwise><xsl:value-of select='$td'/></xsl:otherwise></xsl:choose></xsl:variable>
+		<a href="{$library_name}?a=q&amp;sa={$sa}&amp;c={$collName}&amp;s={$service}&amp;rt=r&amp;{$page-param}={$next}"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.matches')"/><xsl:text> </xsl:text><xsl:value-of select="number(($next - 1)*$hpp + 1)"/> - <xsl:value-of select="$mm"/><img src="interfaces/default/images/more.gif" width='30' height='16' border='0' align='top'/></a></xsl:if></td> </tr>
+	</table>
+      </center>
+    </xsl:if>
+  </xsl:template>
+    
+  <xsl:template name="matchDocs">
+    <!-- If the number of matching documents is known, display it -->
+    <xsl:variable name="numDocsMatched" select="metadataList/metadata[@name='numDocsMatched']"/>
+    <xsl:variable name="numDocsReturned" select="metadataList/metadata[@name='numDocsReturned']"/>
+    <xsl:choose>
+      <xsl:when test="$numDocsMatched='0' or $numDocsReturned='0'">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.nodocsmatch')"/>
+      </xsl:when>
+      <xsl:when test="$numDocsMatched='1' or $numDocsReturned='1'">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.onedocsmatch')"/>
+      </xsl:when>
+      <xsl:when test="$numDocsMatched">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.manydocsmatch', $numDocsMatched)"/>
+	<xsl:if test="$numDocsReturned"> (<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.docsreturned', $numDocsReturned)"/>)</xsl:if>
+      </xsl:when>
+      <xsl:when test="$numDocsReturned">
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.atleastdocsmatch', $numDocsReturned)"/>
+      </xsl:when>
+    </xsl:choose>
+  </xsl:template>
+
+  <!-- paging is now done by the query action, so here we just print out all the docs that we have -->
+  <xsl:template name="resultList">
+    <xsl:param name="collName"/>
+    <table>
+      <xsl:for-each select="documentNodeList/documentNode">
+	<tr>
+	  <xsl:apply-templates select=".">
+	    <xsl:with-param name="collName" select="$collName"/>
+	    <xsl:with-param name="serviceName" select="/page/pageResponse/service/@name"/>
+	  </xsl:apply-templates>
+	</tr>	    
+      </xsl:for-each>
+    </table>    
+  </xsl:template>
+
+
+  <!-- the default doc node template for the query results -->
+  <!-- eventually shouldn't need sib arg here -->
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:param name="serviceName"/>
+    <td>
+      <a><xsl:attribute name="href"><xsl:value-of select='$library_name'/>?a=d&amp;c=<xsl:value-of select='$collName'/>&amp;d=<xsl:value-of select='@nodeID'/><xsl:if test="@nodeType='leaf'">&amp;sib=1</xsl:if>&amp;dt=<xsl:value-of select='@docType'/>&amp;p.a=q&amp;&amp;p.sa=<xsl:value-of select="/page/pageRequest/@subaction"/>&amp;p.s=<xsl:value-of select="$serviceName"/></xsl:attribute>
+	<xsl:apply-templates select="." mode="displayNodeIcon"/>
+      </a>
+    </td>
+    <td><xsl:value-of disable-output-escaping="yes" select="metadataList/metadata[@name='Title']"/></td>
+  </xsl:template>
+
+  <xsl:template name="termInfo">
+    <xsl:if test="count(termList/term) > 0">
+      <small>
+	<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'query.wordcount')"/>
+	<xsl:for-each select="termList/term">
+	  <xsl:if test="position() > 1">, </xsl:if>
+	  <xsl:value-of select="@name"/>: <xsl:value-of select="@freq"/> 
+	</xsl:for-each>
+      </small>
+      <br/>
+    </xsl:if>
+  </xsl:template>
+
+</xsl:stylesheet>
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/service-params.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/service-params.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/service-params.xsl	(revision 21112)
@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  extension-element-prefixes="java">
+
+<!-- handling of the different types of params on a service form 
+  - these now only output the selection box/text box etc, not the name -->
+
+  <!-- the default param list handling -->
+  <xsl:template match="paramList">
+    <xsl:param name="ns">s1.</xsl:param>
+    <table>
+      <xsl:for-each select="param">
+	<xsl:choose>
+	  <xsl:when test="@type='multi'">
+	    <tr><td colspan='2'>
+		<xsl:apply-templates select='.'><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates></td></tr>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:variable name="pvalue"><xsl:apply-templates select="." mode="calculate-default"><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates></xsl:variable>
+	      <tr><td><xsl:value-of select="displayItem[@name='name']"/></td><td align="left"><xsl:apply-templates select="."><xsl:with-param name="default" select="$pvalue"/><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates></td></tr>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:for-each>
+    </table>
+  </xsl:template>
+ 
+  <!-- puts all the params into a=p&p=h type form - need to change this if use 
+  multi params  -->
+  <xsl:template match="paramList" mode="cgi">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:for-each select="param">
+      <xsl:variable name='pname' select="@name"/>
+      <xsl:text>&amp;</xsl:text><xsl:value-of select="$ns"/><xsl:value-of select="@name"/>=<xsl:apply-templates select="." mode="calculate-default"><xsl:with-param name='ns' select='$ns'/></xsl:apply-templates>
+    </xsl:for-each>
+  </xsl:template>
+
+  <xsl:template match="param" mode="calculate-default">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:variable name="pname"><xsl:value-of select="$ns"/><xsl:value-of select="@name"/></xsl:variable>
+    <xsl:choose>
+      <xsl:when test="/page/pageRequest/paramList/param[@name=$pname]">
+	<xsl:choose>
+	  <xsl:when test="@type='enum_multi'"><xsl:text>,</xsl:text>
+	    <xsl:for-each select="/page/pageRequest/paramList/param[@name=$pname]">
+	      <xsl:value-of select="@value"/>,
+	    </xsl:for-each>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:value-of select="/page/pageRequest/paramList/param[@name=$pname]/@value"/>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      <xsl:otherwise>
+	<xsl:value-of select="@default"/>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <!-- invisible params - used by other stuff. in the query form, we set to teh default -->
+  <xsl:template match="param[@type='invisible']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <input type='hidden' name='{$ns}{@name}' value='{@default}'/>
+  </xsl:template>
+  <!-- boolean params -->
+  <xsl:template match="param[@type='boolean']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <select name='{$ns}{@name}'>
+      <option value="0"><xsl:if test="$default='0'"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="option[@name='0']/displayItem[@name='name']"/></option>
+      <option value="1"><xsl:if test="$default='1'"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="option[@name='1']/displayItem[@name='name']"/></option>
+    </select>
+  </xsl:template>
+  
+  <!-- integer params -->
+  <xsl:template match="param[@type='integer']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <input type="text" name="{$ns}{@name}" size="3" value="{$default}"/>
+  </xsl:template>
+
+  <!-- single selection enum params -->
+  <xsl:template match="param[@type='enum_single']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <xsl:choose>
+      <xsl:when test="count(option) = 1">
+	<xsl:value-of select="option/displayItem[@name='name']"/>
+	<input type='hidden' name='{$ns}{@name}'><xsl:attribute name='value'><xsl:value-of  select='option/@name'/></xsl:attribute></input>
+      </xsl:when>
+      <xsl:otherwise>
+	<select name="{$ns}{@name}">
+	  <xsl:for-each select="option">
+	    <option value="{@name}"><xsl:if test="@name=$default"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="displayItem[@name='name']"/></option>
+	  </xsl:for-each>
+	</select>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+
+  <!-- multiple selection enum params -->
+  <!-- how to do defaults for this?? -->
+  <xsl:template match="param[@type='enum_multi']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <select name="{$ns}{@name}" size='2'><xsl:attribute name="multiple"></xsl:attribute>
+      <xsl:for-each select="option">
+	<option value="{@name}"><xsl:if test="contains($default, concat(',', @name, ','))"><xsl:attribute name="selected"></xsl:attribute></xsl:if><xsl:value-of select="displayItem[@name='name']"/></option>
+      </xsl:for-each>
+    </select>
+  </xsl:template>
+ 
+  <!-- string params -->
+  <xsl:template match="param[@type='string']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <input type="text" name="{$ns}{@name}" size="50" value="{$default}"/>
+  </xsl:template>
+  
+  <!-- large string  params -->
+  <xsl:template match="param[@type='text']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="default"/>
+    <textarea name="{$ns}{@name}" cols="50" rows="3"><xsl:value-of select='$default'/></textarea>
+  </xsl:template>
+
+  <!-- multi params - params that are combinations of other params -->
+  <xsl:template match="param[@type='multi']">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:variable name="parent" select="@name"/>
+    <table>
+      <tr><td colspan='10'><xsl:value-of select="displayItem[@name='name']"/></td></tr>
+      <tr>
+	<xsl:for-each select="param">
+	  <td><xsl:value-of select="displayItem[@name='name']"/></td>
+	</xsl:for-each>
+      </tr>
+      <xsl:apply-templates select="." mode="contents"><xsl:with-param name="occurs" select="@occurs"/><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates>
+    </table>
+  </xsl:template>
+
+  <xsl:template match="param[@type='multi']" mode="contents">
+    <xsl:param name="ns">s1.</xsl:param>
+    <xsl:param name="occurs">1</xsl:param>
+    <xsl:variable name="pos" select="@occurs - $occurs"/>	
+    <tr><xsl:for-each select="param">
+	<xsl:variable name="pname" select="@name"/>
+	<xsl:variable name="values" select="/page/pageRequest/paramList/param[@name=$pname]/@value"/>
+	<td><xsl:choose>
+	    <xsl:when test="not(@ignore) or  @ignore != $pos">      
+	      <xsl:apply-templates select='.'><xsl:with-param name="default" select="java:org.greenstone.gsdl3.util.XSLTUtil.getNumberedItem($values, $pos)"/><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates>
+	    </xsl:when>
+	    <xsl:otherwise><!-- put in a hidden placeholder -->
+	      <input type="hidden" name='{$ns}{@name}' value=''/>
+	    </xsl:otherwise>
+	  </xsl:choose></td>
+      </xsl:for-each></tr> 
+    <!-- recursively call this template to get multiple entries -->
+    <xsl:if test="$occurs &gt; 1">
+      <xsl:apply-templates select="." mode="contents"><xsl:with-param name="occurs" select="$occurs - 1"/><xsl:with-param name="ns" select="$ns"/></xsl:apply-templates>
+    </xsl:if>
+  </xsl:template>
+
+  
+</xsl:stylesheet>  
+
+  <!-- a param list that puts params in pairs- wont work as is with new
+  param handling stuff -->
+  <!--
+  <xsl:template match="paramList">
+  <p/><table width="537">
+  <xsl:choose>
+  <xsl:when test='count(param)>4'>
+  <xsl:for-each select="param[position() mod 2 = 1]"> 
+  <tr><xsl:apply-templates select="."/>
+  <xsl:if test="following-sibling::param[1]"><xsl:apply-templates select='following-sibling::param[1]'/></xsl:if></tr>
+</xsl:for-each>
+</xsl:when>
+  <xsl:otherwise>
+  <xsl:for-each select="param">
+  <tr><xsl:apply-templates select='.'/></tr>
+</xsl:for-each>
+</xsl:otherwise>
+</xsl:choose>
+</table>
+</xsl:template>
+  -->
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/style.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/style.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/style.xsl	(revision 21112)
@@ -0,0 +1,316 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- some global parameters - these are set by whoever is invoking the transformation -->
+  <xsl:param name="interface_name"/>
+  <xsl:param name="library_name"/>
+  
+  <!-- global style info goes here  -->
+  <xsl:template name="globalStyle">
+    <link rel="stylesheet" href="interfaces/default/style/core.css" type="text/css"/>
+  </xsl:template>
+  
+  <!-- the main page layout template is here -->
+  <xsl:template match="page">
+    <html>
+      <xsl:call-template name="pageHead"/>
+      <body>
+	<table border="0" cellspacing="0" cellpadding="0" width="100%">
+	  <td valign="top" width="65"><!-- a space for our side bar-->
+	    <img src="interfaces/default/images/spacer.gif" width="65" height="1" alt="" border="0"/><!-- an image cos netscape ignores the cell width -->
+	  </td>
+	  <td>
+	    <table width="100%">
+	      <tr><td>
+		  <xsl:apply-templates select="pageResponse"/>
+		  <xsl:if test="descendant::error">
+		    <script language="Javascript">
+		      <xsl:text disable-output-escaping="yes">
+			function removeAllChildren(node) {
+			 while (node.hasChildNodes()) {
+			  node.removeChild(node.firstChild);
+			 }
+			}
+
+		      function toggleHideError(obj) {
+		      if (obj.style.display == "none") {
+			obj.style.display = "";
+			hide_link = document.getElementById("hide");
+			removeAllChildren(hide_link);
+			hide_link.appendChild(document.createTextNode("Hide Error"));
+		      } else {
+			obj.style.display = "none";
+			hide_link = document.getElementById("hide");
+			removeAllChildren(hide_link);
+			hide_link.appendChild(document.createTextNode("Show Error"));
+			
+			}
+		      }
+		      </xsl:text>
+		    </script>
+		    <p align='right'><a id="hide" href="javascript:toggleHideError(error);">Show Error</a></p>
+		    <div id="error" style="display: none;">
+		      <xsl:apply-templates select="descendant::error"/>
+		    </div>
+		  </xsl:if>
+		  <xsl:call-template name="greenstoneFooter"/>
+		</td></tr>
+	    </table>
+	  </td>
+	</table>
+      </body>
+    </html>
+  </xsl:template>
+
+  <!-- put a space in the title in case the actual value is missing - mozilla will not display a page with no title-->
+  <xsl:template name="pageHead">
+    <head>
+      <title>
+	<xsl:call-template name="pageTitle"/><xsl:text> </xsl:text>
+      </title>
+      <xsl:call-template name="globalStyle"/>
+      <xsl:call-template name="pageStyle"/>
+    </head>
+  </xsl:template>
+
+
+  <xsl:template name="greenstonePageBanner">
+    <center><img src="interfaces/default/images/gsdlhead.gif"><xsl:attribute name="alt"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gsdl')"/></xsl:attribute></img></center>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  <xsl:template name="greenstoneFooter">
+    <center>
+      <p/>
+      <small><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'gs3power')"/></small>
+    </center>
+  </xsl:template>
+  
+  <xsl:template match="error">
+    Error: <xsl:value-of select="."/>
+  </xsl:template>
+
+  <xsl:template name="standardPageBanner">
+    <xsl:param name="collName"/>
+    <xsl:param name="pageType"/>
+    <xsl:variable name="this-element" select="/page/pageResponse/collection|/page/pageResponse/serviceCluster"/>
+    <table width="100%">
+      <tr>
+	<td align="left">
+	  <a href="{$library_name}?a=p&amp;sa=about&amp;c={$collName}">
+	    <xsl:choose>
+	      <xsl:when test="$this-element/displayItem[@name='icon']">
+		<img border="0">
+		  <xsl:attribute name="src">
+		    <xsl:value-of select="$this-element/metadataList/metadata[@name='httpPath']"/>/images/<xsl:value-of select="$this-element/displayItem[@name='icon']"/>
+		  </xsl:attribute>	
+		  <xsl:attribute name="alt">
+		    <xsl:value-of select="$this-element/displayItem[@name='name']"/>
+		  </xsl:attribute>
+		  <xsl:attribute name="title">
+		    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'aboutpage')"/>
+		  </xsl:attribute>
+		</img>
+	      </xsl:when>
+	      <xsl:otherwise>
+		<b><xsl:value-of select="$this-element/displayItem[@name='name']"/></b>
+	      </xsl:otherwise>
+	    </xsl:choose>
+	  </a>
+	</td>
+	<td align="right">
+	  <xsl:call-template name="top-buttons">
+	    <xsl:with-param name="collName" select="$collName"/>
+	    <xsl:with-param name="pageType" select="$pageType"/>
+	  </xsl:call-template>
+	</td>
+      </tr>
+    </table>
+    <p/>
+  </xsl:template>
+  
+  <xsl:template name="top-buttons">
+    <xsl:param name="collName"/>
+    <xsl:param name="pageType"/>
+    <table cellspacing='4'>
+      <tr>
+	<td class="gsbutton">
+	  <a class="gsbutton" href="{$library_name}?a=p&amp;sa=home"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'home_tip')"/></xsl:attribute>
+	    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'home_b')"/>
+	  </a>
+	</td>
+	<xsl:choose>
+	  <xsl:when test="$pageType='help'">
+	    <td class="gsbuttonoff">
+	      <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_b')"/>
+	    </td>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <td class="gsbutton">
+	      <a class="gsbutton" href="{$library_name}?a=p&amp;sa=help&amp;c={$collName}"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_tip')"/></xsl:attribute>
+		<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'help_b')"/>
+	      </a>
+	    </td>
+	  </xsl:otherwise>
+	</xsl:choose>
+	<xsl:choose>
+	  <xsl:when test="$pageType='pref'">
+	    <td class="gsbuttonoff">
+	      <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_b')"/>
+	    </td>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <td class="gsbutton">
+	      <a class="gsbutton" href="{$library_name}?a=p&amp;sa=pref&amp;c={$collName}"><xsl:attribute name="title"><xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_tip')"/></xsl:attribute>
+		<xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'pref_b')"/>
+	      </a>
+	    </td>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </tr>
+    </table>
+    
+  </xsl:template>
+
+  <xsl:template name="navigationBar">
+    <xsl:param name="collName"/>
+    <xsl:variable name="this-element" select="/page/pageResponse/collection|/page/pageResponse/serviceCluster"/>
+    <xsl:variable name="this-service" select="/page/pageResponse/service/@name"/>
+    <xsl:choose>
+      <xsl:when test="$this-element/serviceList/service">
+	<table border='0' cellspacing='0' width='100%'>
+	  <tr>
+	    <xsl:for-each select="$this-element/serviceList/service">
+	      <xsl:variable name="action"><xsl:choose>
+		  <xsl:when test="@name=$this-service">CURRENT</xsl:when>
+		  <xsl:when test="@type='query'">q</xsl:when>
+		  <xsl:when test="@type='browse'">b</xsl:when>
+		  <xsl:when test="@type='process'">pr</xsl:when>
+		  <xsl:when test="@type='applet'">a</xsl:when>
+		  <xsl:otherwise>DO_NOT_DISPLAY</xsl:otherwise>
+		</xsl:choose></xsl:variable>
+	      <xsl:choose>
+		<xsl:when test="$action='CURRENT'">
+		  <td class="navbaroff"><nobr><xsl:value-of select="displayItem[@name='name']"/></nobr></td>
+		</xsl:when>
+		<xsl:when test="$action !='DO_NOT_DISPLAY'">
+		  <td class="navbar"><a class="navbar" href="{$library_name}?a={$action}&amp;rt=d&amp;s={@name}&amp;c={$collName}"><xsl:if test="displayItem[@name='description']"><xsl:attribute name='title'><xsl:value-of select="displayItem[@name='description']"/></xsl:attribute></xsl:if><nobr><xsl:value-of select="displayItem[@name='name']"/></nobr></a></td>
+		</xsl:when>
+	      </xsl:choose>
+	    </xsl:for-each>
+	  </tr>
+	</table>
+      </xsl:when>
+      <xsl:otherwise>
+	<table border='0' cellspacing='0' width='100%'><tr><td class="navbar">&#160;</td></tr></table>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+  
+  
+  <xsl:template name="dividerBar">
+    <xsl:param name='text'/>
+    <br/><table width='100%' border="0" cellpadding="0" cellspacing="0">
+      <tr><td valign='bottom' height='20' background='interfaces/default/images/greenbar.gif'><font size="+1" color="black"><b>&#160;<xsl:value-of select='$text'/></b></font></td></tr></table><br/>
+  </xsl:template>
+
+  <!-- Display the appropriate image, depending on the node type -->
+  <xsl:template match="documentNode" mode="displayNodeIcon">
+    
+    <!-- Root node: book icon (open or closed) -->
+    <xsl:choose>
+      <xsl:when test="@nodeType='root'">
+	<xsl:choose>
+	  <xsl:when test="documentNode">
+	    <xsl:call-template name="openbookimg">
+	      <xsl:with-param name="title">Close this book</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:call-template name="closedbookimg">
+	      <xsl:with-param name="title">Open this document and view contents</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      
+      <!-- Internal node: folder icon (open or closed) -->
+      <xsl:when test="@nodeType='internal'">
+	<xsl:choose>
+	  <xsl:when test="documentNode">
+	    <xsl:call-template name="openfolderimg">
+	      <xsl:with-param name="title">Close this folder</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    <xsl:call-template name="closedfolderimg">
+	      <xsl:with-param name="title">Open this folder and view contents</xsl:with-param>
+	    </xsl:call-template>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </xsl:when>
+      
+      <!-- Leaf node: page icon, and this is the default -->
+      <xsl:otherwise>
+	<xsl:call-template name="textpageimg">
+	  <xsl:with-param name="title">View the document</xsl:with-param>
+	</xsl:call-template>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template name="openbookimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="28" height="23"
+      src="interfaces/default/images/openbook.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="closedbookimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="18" height="11"
+      src="interfaces/default/images/book.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="openfolderimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="23" height="15"
+      src="interfaces/default/images/openfldr.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="closedfolderimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="23" height="15"
+      src="interfaces/default/images/clsdfldr.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="textpageimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="16" height="21"
+      src="interfaces/default/images/itext.gif"
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+
+  <xsl:template name="bookshelfimg">
+    <xsl:param name="alt"/>
+    <xsl:param name="title"/>
+    <img border="0" width="20" height="16"
+      src="interfaces/default/images/bshelf.gif"  
+      alt="{$alt}" title="{$title}"/>
+  </xsl:template>
+</xsl:stylesheet>  
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/default/transform/system.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/default/transform/system.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/default/transform/system.xsl	(revision 21112)
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  extension-element-prefixes="java">
+  
+  <xsl:include href="style.xsl"/>
+  
+    <xsl:template name="pageTitle">
+    <xsl:value-of select="java:org.greenstone.gsdl3.util.XSLTUtil.getInterfaceText($interface_name, /page/@lang, 'gsdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:value-of select="status"/>
+  </xsl:template>
+  
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/interfaceConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/interfaceConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/interfaceConfig.xml	(revision 21112)
@@ -0,0 +1,24 @@
+<interfaceConfig baseInterface="classic">
+  <actionList>
+    <action name='p' class='PageAction'>
+      <subaction name='home' xslt='home.xsl'/>
+      <subaction name='about' xslt='about.xsl'/>
+      <subaction name='gsdl' xslt='gsdl.xsl'/>
+      <subaction name='help' xslt='help.xsl'/>
+      <subaction name='pref' xslt='pref.xsl'/>
+      <subaction name='nav' xslt='nav.xsl'/>
+      <subaction name="html" xslt="html.xsl"/>
+      <subaction name='abexpl' xslt='abexpl.xsl'/>
+    </action>
+    <action name='q' class='QueryAction' xslt='query.xsl'/>
+    <action name='b' class='GS2BrowseAction' xslt='classifier.xsl'/>
+    <action name='a' class='AppletAction' xslt='applet.xsl'/>
+    <action name='d' class='DocumentAction' xslt='document.xsl'/>
+    <action name='pr' class='ProcessAction' xslt='process.xsl'/>
+    <action name='s' class='SystemAction' xslt='system.xsl'/>
+  </actionList>
+  <languageList>
+    <language name='en' displayname='English' default_encoding='utf-8'/>
+    <language name='fr' displayname='French' default-encoding='utf-8'/>
+  </languageList>
+</interfaceConfig>
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl-text.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl-text.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl-text.xsl	(revision 21112)
@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  extension-element-prefixes="java">
+
+  <xsl:template name="aboutgs">about the greenstone software</xsl:template>
+  <xsl:template name="textgreenstone1">
+Greenstone is a suite of software which has the ability to serve digital
+library collections and build new collections.  It provides a new way of
+organizing information and publishing it on the Internet or on CD-ROM.
+Greenstone is produced by the New Zealand Digital Library Project at the
+University of Waikato, and distributed in cooperation with UNESCO and the
+Human Info NGO. It is open-source software, available from
+<i>http://greenstone.org</i> under the terms of the GNU General Public
+License.
+  </xsl:template>
+
+  <xsl:template name="textgreenstone2">
+The New Zealand Digital Library website (<a href="http://nzdl.org">http://nzdl.org</a>) contains numerous example
+collections, all created with the Greenstone software, which are publicly
+available for you to peruse. They exemplify various searching and browsing
+options, and include collections in Arabic, Chinese, French, Maori, and
+Spanish, as well as English. There are also some music collections.
+  </xsl:template>
+
+  <xsl:template name="textplatformtitle">platform</xsl:template>
+  <xsl:template name="textgreenstone3">
+Greenstone runs on Windows and Unix. The distribution includes ready-to-use
+binaries for all versions of Windows, and for Linux. It also includes
+complete source code for the system, which can be compiled using Microsoft
+C++ or gcc.  Greenstone works with associated software that is also freely
+available: the Apache Webserver and PERL. The user interface uses a Web
+    browser: typically Netscape Navigator or Internet Explorer.
+  </xsl:template>
+
+  <xsl:template name="textgreenstone4">
+Many document collections are distributed on CD-ROM using the Greenstone
+software. For example, the <i>Humanity Development Library</i> contains
+1,230 publications ranging from accounting to water sanitation. It runs on
+minimal computing facilities such as those typically found in developing
+countries. The information can be accessed by searching, browsing by
+subject, browsing by titles, browsing by organisation, browsing a list of
+how-tos, and by randomly viewing the book covers.
+  </xsl:template>
+
+  <xsl:template name="textcustomisationtitle">customisation</xsl:template>
+  <xsl:template name="textgreenstone5">
+Greenstone is specifically designed to be highly extensible and
+customisable. New document and metadata formats are accommodated by writing
+"plugins" (in Perl). Analogously, new metadata browsing structures can be
+implemented by writing "classifiers." The user interface look-and-feel can
+be altered using "macros" written in a simple macro language. A Corba
+protocol allows agents (e.g. in Java) to use all the facilities associated
+with document collections. Finally, the source code, in C++ and Perl, is
+available and accessible for modification.
+  </xsl:template>
+
+<xsl:template name="textdocumentationtitle">documentation</xsl:template>
+<xsl:template name="textdocuments">Extensive documentation for the Greenstone software is available.</xsl:template>
+
+  <xsl:template name="textmailinglisttitle">mailing list</xsl:template>
+  <xsl:template name="textmailinglist">
+There is a mailing list intended primarily for discussions about the
+Greenstone digital library software.  Active users of Greenstone should
+consider joining the mailing list and contributing to the discussions.
+To subscribe, go to  <a href="https://list.scms.waikato.ac.nz/mailman/listinfo/greenstone-users">https://list.scms.waikato.ac.nz/mailman/listinfo/greenstone-users</a>.
+
+To send a message to the list, address it to <a
+href="mailto:greenstone-users@list.scms.waikato.ac.nz"
+>greenstone-users@list.scms.waikato.ac.nz</a>.
+</xsl:template>
+
+  <xsl:template name="textbugstitle">bugs</xsl:template>
+  <xsl:template name="textreport">
+We want to ensure that this software works well for you.  Please report any
+bugs to <a href="mailto:greenstone@cs.waikato.ac.nz">greenstone@cs.waikato.ac.nz</a>
+  </xsl:template>
+
+<xsl:template name="textgs3title">in the works</xsl:template>
+
+<xsl:template name="textgs3">Greenstone 3 is a complete redesign and 
+reimplementation which 
+retains all the advantages of Greenstone 2 (the current version)--for example, 
+it is multilingual, multiplatform, and highly configurable. It 
+incorporates all the features of the existing system, and is backwards 
+compatible: that is, it can build and run existing collections without 
+modification. Written in Java, it is structured as a network of 
+independent modules that communicate using XML: thus it runs in a 
+distributed fashion and can be spread across different servers as 
+necessary. This modular design increases the flexibility and 
+extensibility of Greenstone. The new version is expected to be 
+available for experimental use by 23 December 2003. An initial design for 
+the system is outlined in "The design of Greenstone 3: An agent based 
+dynamic digital library" (download <a href="http://www.greenstone.org/manuals/gs3desig.pdf">PDF</a>).
+</xsl:template>
+
+
+  <xsl:template name="textcreditstitle">credits</xsl:template>
+
+  <xsl:template name="textwhoswho">
+The Greenstone software is a collaborative effort between many
+people. Rodger McNab and Stefan Boddie are the principal architects and
+implementors.  Contributions have been made by David Bainbridge, George
+Buchanan, Michael Dewsnip, Katherine Don, Hong Chen, Elke Duncker, 
+Carl Gutwin, Geoff Holmes, John McPherson, Craig Nevill-Manning, 
+Dynal Patel, Gordon Paynter, Bernhard Pfahringer, Todd
+Reed, Bill Rogers, John Thompson, and Stuart Yeates.
+Other members of the New Zealand Digital Library project provided advice 
+and inspiration in the design of
+the system: Mark Apperley, Sally Jo Cunningham, Matt Jones, Steve Jones, 
+Te Taka Keegan, Michel Loots, Malika Mahoui, Gary Marsden, Dave Nichols 
+and Lloyd Smith. We would also like to
+acknowledge all those who have contributed to the GNU-licensed packages
+included in this distribution: MG, GDBM, PDFTOHTML, WGET, WVWARE and XLHTML.
+  </xsl:template>
+
+  
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/gsdl.xsl	(revision 21112)
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="gsdl-text.xsl"/>
+  <xsl:include href="home-text.xsl"/>
+  <xsl:include href="page-common.xsl"/>
+
+  <xsl:output method="html"/>  
+
+ <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'nzdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <center>
+    <xsl:call-template name="genericPageBanner"><xsl:with-param name='text'><xsl:call-template name="aboutgs"/></xsl:with-param></xsl:call-template>
+    </center>
+    <p />
+    <table border='0' cellspacing='0' cellpadding='0'>
+      <tr>
+	<td valign='top' align='center'><xsl:call-template name="imagegreenstone"/></td>
+	<td valign='top'><xsl:call-template name="textgreenstone1"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><a href="http://nzdl.org">nzdl.org</a></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textgreenstone2"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><xsl:call-template name="textplatformtitle"/></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textgreenstone3"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <a href="http://nzdl.org/hdl"><img border='0' width='101' height='100' src="interfaces/nzdl/images/hdlimg.jpg"/></a>
+	</td>
+	<td valign='top'>
+	  <xsl:call-template name="textgreenstone4"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><xsl:call-template name="textcustomisationtitle"/></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textgreenstone5"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><a href="http://www.greenstone.org/english/docs.html"><xsl:call-template name="textdocumentationtitle"/></a></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textdocuments"/>
+	<br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><xsl:call-template name="textmailinglisttitle"/></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textmailinglist"/>
+	  <br /><br />
+	</td>
+      </tr>
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><xsl:call-template name="textbugstitle"/></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textreport"/>
+	  <br /><br />
+	</td>
+      </tr>
+       <tr>
+         <td valign='top' align='center'>
+	   <h2><xsl:call-template name="textgs3title"/></h2>
+	 </td>
+	 <td valign='top'><xsl:call-template name="textgs3"/>
+	   <br /><br />
+	 </td>
+      </tr>
+
+      <tr>
+	<td valign='top' align='center'>
+	  <h2><xsl:call-template name="textcreditstitle"/></h2>
+	</td>
+	<td valign='top'><xsl:call-template name="textwhoswho"/>
+	</td>
+      </tr>
+    </table>
+    <center>
+    <p /><xsl:call-template name="dividerBar"/>
+    <xsl:call-template name="greenstone_info"/>
+    <xsl:call-template name="nzdlpagefooter"/>
+    </center>
+  </xsl:template>
+
+</xsl:stylesheet>
+
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home-text.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home-text.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home-text.xsl	(revision 21112)
@@ -0,0 +1,192 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  extension-element-prefixes="java">
+
+  <!--<xsl:include href="style.xsl"/>-->
+<xsl:template name="textpagetitle">New Zealand Digital Library</xsl:template>
+<xsl:template name="textprojhead">The New Zealand Digital Library Project</xsl:template>
+
+<xsl:template name="textprojinfo">
+<p />
+The New Zealand Digital Library project is a research programme at
+The University of Waikato whose aim is to develop the underlying
+technology for digital libraries and make it available publicly so that
+others can use it to create their own collections. 
+<p />
+Our web site provides several document collections, including historical
+documents, humanitarian and development information, computer
+science technical reports and bibliographies, literary works, and
+magazines. All are available over the Web, and can be accessed
+through searching and browsing interfaces provided by the
+Greenstone digital library software. Behind the query interface lies a
+huge collection providing gigabytes of information. We hope you find
+what you want, or at least something intriguing!
+</xsl:template>
+
+<xsl:template name="titlesoftwareinfo">The Greenstone software</xsl:template>
+
+<xsl:template name="textsoftwareinfo">
+<p />
+The <a href="http://www.greenstone.org">Greenstone Digital Library software</a> provides a new way of
+organizing information and making it available over the Internet or on
+CD-ROM. It is open-source software, available under the terms of the
+Gnu public license. 
+<p />
+A digital library is made up of a set of collections. Each collection of
+information comprises several (typically several thousand, or even
+several million) documents, which share a uniform searching and
+browsing interface. Collections can be organized in many different
+ways while retaining a strong family resemblance. 
+
+<p>To subscribe to the Greenstone mailing list, go to <a 
+href="https://list.scms.waikato.ac.nz/mailman/listinfo/greenstone-users">https://list.scms.waikato.ac.nz/mailman/listinfo/greenstone-users</a>.</p>
+
+<p align='right'/>
+<a href="{$library_name}?a=p&amp;sa=gsdl">More...</a>
+</xsl:template>
+
+<xsl:template name="titleresearchinfo">Our research</xsl:template>
+
+<xsl:template name="textresearchinfo">
+<p />
+The goal of our research program is to explore the potential of
+internet-based digital libraries. Our vision is to develop systems that
+automatically impose structure on anarchic, uncatalogued, distributed
+repositories of information, thereby providing information consumers
+with effective tools to locate what they need and to peruse it
+conveniently and comfortably. 
+<p />
+Project members are actively working on techniques for creating,
+managing, and and mainatining collections; extracting metadata from
+legacy documents; analysing library usage and user needs; Maori,
+Arabic and Chinese language systems; internationalising the library
+interface; optical music recognition and musical collections; novel
+interfaces for formulating queries and visualising results; novel
+interfaces for browsing metadata; text mining for keyphrases,
+acronyms, and other metadata; keyphrase extraction and
+phrase-based browsing; and other research topics. 
+<p align='right'/>
+<a href="http://www.nzdl.org/html/research.html">More...</a>
+</xsl:template>
+
+<xsl:template name="titleaffiliateinfo">Our affiliates</xsl:template>
+
+<xsl:template name="textaffiliatehumaninfo">
+<p />
+<a href="http://humaninfo.org">Human Info NGO</a>
+is a registered charity responsible for the
+provision of universal low-cost
+information access through co-operation between UN Agencies,
+universities and NGOs. Human Info NGO collaborates extensively
+with the NZDL project, and use the Greenstone software. 
+</xsl:template>
+
+<xsl:template name="textaffiliateunesco">
+<p />
+<a href="http://www.unesco.org">United Nations Educational, Scientific and Cultural Organization</a>.
+The dissemination of educational, scientific and cultural information
+throughout the world, and particularly its availability
+</xsl:template>
+
+<xsl:template name="textpoem">
+<br /><h2>Kia papapounamu te moana</h2>
+
+<p />kia hora te marino,
+<br />kia tere te karohirohi,
+<br />kia papapounamu te moana
+
+<p />may peace and calmness surround you,
+<br />may you reside in the warmth of a summer's haze,
+<br />may the ocean of your travels be as smooth as the polished greenstone.
+</xsl:template>
+
+<xsl:template name="textgreenstone"> 
+<p />Greenstone is a semi-precious stone that (like this software) is sourced
+in New Zealand.  In traditional Maori society it was the most highly prized
+and sought after of all substances.  It can absorb and hold <i>wairua</i>,
+which is a spirit or life force, and is endowed with traditional virtues
+that make it an appropriate emblem for a public-domain digital library
+project.  Its lustre shows charity; its translucence, honesty; its
+toughness, courage; and the sharp edge it can take, justice.  The carved
+piece used in the Greenstone Digital Library Software logo is a <i>patu</i>
+or fighting club, and is a family heirloom of one of our project members.
+In hand-to-hand combat its delivery is very quick, very accurate, and very
+complete.  We like to think these qualities also apply to our software, the
+razor sharp edge of the <i>patu</i> symbolizing the leading edge of
+technology.
+</xsl:template>
+
+<xsl:template name="textaboutgreenstone">
+<p />Greenstone is a suite of software for building and distributing digital
+library collections. It provides a new way of organizing information and
+publishing it on the Internet or on CD-ROM.  Greenstone is produced by the
+<b>New Zealand Digital Library Project</b> at the <b>University of
+Waikato</b>, and developed and distributed in cooperation with
+<b>UNESCO</b> and the <b>Human Info NGO</b>.  It is open-source
+software, available from <a href="http://greenstone.org">http://greenstone.org</a> under the terms of
+the GNU General Public License.
+
+<p />The aim of the software is to empower users, particularly in
+universities, libraries, and other public service institutions, to
+build their own digital libraries.  Digital libraries are radically
+reforming how information is disseminated and acquired in UNESCO's
+partner communities and institutions in the fields of education,
+science and culture around the world, and particularly in developing
+countries.  We hope that this software will encourage the effective
+deployment of digital libraries to share information and place it in
+the public domain.
+
+
+<p />This software is developed and distributed as an international
+cooperative effort established in August 2000 among three parties.
+
+<table border="0">
+<tr valign="top">
+<td>
+<a href="http://nzdl.org"><b>New Zealand Digital Library Project at the University of Waikato</b></a>
+<br />
+Greenstone software grew out of this project, and this initiative
+has been endorsed by the Communication Sub-Commission of the New
+Zealand National Commission for UNESCO as part of New Zealand's
+contribution to UNESCO's programme.
+</td>
+<td></td>
+</tr>
+<tr valign="top">
+<td>
+<a href="http://www.unesco.org"><b>United Nations Educational, Scientific and Cultural Organization</b></a>
+<br />
+The dissemination of educational, scientific and cultural information
+throughout the world, and particularly its availability in developing
+countries, is central to UNESCO's goals as pursued within its
+intergovernmental Information for All Programme, and appropriate,
+accessible information and communication technology is seen as an important
+tool in this context.
+</td>
+<td><a href="http://www.unesco.org"><img src="interfaces/nzdl/images/unesco.gif" border="0"/></a></td>
+</tr>
+<tr valign="top">
+<td>
+<a href="http://humaninfo.org"><b>The Human Info NGO, based in Antwerp, Belgium</b></a>
+<br />
+This project works with UN agencies and other NGOs, and has established
+a worldwide reputation for digitizing documentation of interest to
+human development and making it widely available, free of charge to
+developing nations and on a cost-recovery basis to others.
+</td>
+<td><a href="http://humaninfo.org"><img src="interfaces/nzdl/images/ghproj2.jpg" border="0"/></a></td>
+</tr>
+</table>
+</xsl:template>
+
+  <xsl:template name="textimagegreenstone">Greenstone Digital Library Software</xsl:template>
+  
+  <xsl:template name="nzdlpagefooter">
+      <p /><xsl:call-template name="dividerBar"/>
+      <p /><a href="http://www.nzdl.org">New Zealand Digital Library Project</a>
+      <br /><a href="http://www.cs.waikato.ac.nz/cs">Department of Computer Science</a>, <a href="http://www.waikato.ac.nz">University of Waikato</a>, New Zealand
+    </xsl:template>
+    
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/home.xsl	(revision 21112)
@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <xsl:include href="style.xsl"/>
+  <xsl:include href="home-text.xsl"/>
+  <xsl:include href="page-common.xsl"/>
+
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="util:getInterfaceText($interface_name, /page/@lang, 'nzdl')"/>
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:call-template name="nzdlPageBanner"/>     
+    <center>
+    <xsl:apply-templates select="collectionList"/>
+    </center>
+    <center><xsl:call-template name="dividerBar"/></center>    
+    <p />
+    <center><h2><xsl:call-template name="textprojhead"/></h2></center>
+    
+    <table border='0' cellpadding='5'>
+      <xsl:call-template name="project_info"/>
+      <xsl:call-template name="software_info"/>
+      <xsl:call-template name="research_info"/>
+      <xsl:call-template name="affiliate_info"/>
+    </table>
+    <center>
+      <xsl:call-template name="dividerBar"/>
+      <xsl:call-template name="greenstone_info"/>
+      <p />
+      <xsl:call-template name="nzdlpagefooter"/> 
+      <br />September 2003
+    </center>
+  </xsl:template>
+
+  <xsl:template name="project_info">
+    <tr valign='top'>
+      <td>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="mailto:greenstone@cs.waikato.ac.nz">Feedback</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.nzdl.org/html/people.html">People</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://greenstone.cs.uct.ac.za/gsdl/cgi-bin/library">African Digital Library Centre</a></td></tr></table>
+<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://sadl.uleth.ca">Southern Alberta Digital Library Centre</a></td></tr></table>
+      </td>
+      <td><xsl:call-template name="textprojinfo"/></td>
+    </tr>
+  </xsl:template>
+
+  <xsl:template name="software_info">
+    <tr>
+      <td></td>
+      <th align='left'><xsl:call-template name="titlesoftwareinfo"/></th>
+    </tr>
+    <tr valign='top'>
+      <td>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="{$library_name}?a=p&amp;sa=gsdl">About<br />Greenstone 2</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.greenstone.org/english/docs.html">Greenstone 2<br />Documentation</a></td></tr></table>
+      </td>
+      <td><xsl:call-template name="textsoftwareinfo"/></td>
+    </tr>
+  </xsl:template>
+  
+  <xsl:template name="research_info">
+    <tr>
+    <td></td>
+    <th align='left'><xsl:call-template name="titleresearchinfo"/></th>
+  </tr>
+    <tr valign='top'>
+      <td>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.nzdl.org/html/research.html">Research</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.nzdl.org/html/projects.html">Projects</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.nzdl.org/html/software.html">Software<br />Downloads</a></td></tr></table>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://www.cs.waikato.ac.nz/~nzdl/publications/">Publications</a></td></tr></table>
+      </td>
+      <td><xsl:call-template name="textresearchinfo"/></td>
+    </tr>
+    
+<!--    <tr valign='top'>
+      <td>
+	<table><tr valign='middle'><td><img src="interfaces/nzdl/images/gbull.gif"/></td><td><a href="http://nzdl2.cs.waikato.ac.nz/cgi-bin/zdemo/library">Z39.50<br />demo</a></td></tr></table>
+      </td><td>
+	We have an experimental facility for searching Z39.50 collections using  Greenstone.  Ultimately we plan a full bibliographic search facility, with  the ability to combine tests on different fields, and to be able to search multiple collections, including cross-searching Greenstone and Z39.50 collections.
+      </td>
+    </tr>-->
+  </xsl:template>
+
+  <xsl:template name="affiliate_info">
+    <tr>
+      <td></td>
+      <th align='left'><xsl:call-template name="titleaffiliateinfo"/></th>
+    </tr>
+    <tr valign='top'>
+      <td>
+	<img src="interfaces/nzdl/images/ghproj.jpg" alt="Human Info NGO" width='100' height='90'/>
+      </td>
+      <td><xsl:call-template name="textaffiliatehumaninfo"/></td>
+    </tr>
+    <tr valign='top'>
+      <td><img src="interfaces/nzdl/images/unesco.gif" alt="UNESCO" width='100' height='90' />
+      </td>
+      <td><xsl:call-template name='textaffiliateunesco'/></td>
+      </tr>
+  </xsl:template>
+  
+  <xsl:template match="collection">
+    <xsl:variable name="ct"><xsl:choose><xsl:when test="metadataList/metadata[@name='buildType']='mgpp'">1</xsl:when><xsl:otherwise>0</xsl:otherwise></xsl:choose></xsl:variable>
+    <p />
+    <a href="{$library_name}?a=p&amp;sa=about&amp;c={@name}&amp;ct={$ct}">
+      <img width="150" border="1">
+	<xsl:attribute name="src">
+	  <xsl:value-of select="metadataList/metadata[@name='httpPath']"/>/images/<xsl:choose><xsl:when test="displayItem[@name='smallicon']"><xsl:value-of select="displayItem[@name='smallicon']"/></xsl:when><xsl:otherwise><xsl:value-of select="displayItem[@name='icon']"/></xsl:otherwise></xsl:choose>
+	</xsl:attribute>
+	<xsl:attribute name="alt">
+	  <xsl:value-of select="displayItem[@name='name']"/>
+	</xsl:attribute>
+      </img>
+    </a>
+    <p />
+  </xsl:template> 
+
+
+  <xsl:template match="collectionList">
+    <center>
+      <xsl:call-template name="dividerBar"><xsl:with-param name="text">Humanitarian and UN collections</xsl:with-param></xsl:call-template>
+      <p /><xsl:call-template name="collectfao"/>
+      <xsl:call-template name="dividerBar"><xsl:with-param name="text">Demonstration collections</xsl:with-param></xsl:call-template> 
+      <p /><xsl:call-template name="collectrest"/>
+    </center>
+  </xsl:template>
+
+  <xsl:template name="collectfao">
+    <table width="537">
+      <tr valign='top'>
+	<td align='center'>
+	  <p /><xsl:apply-templates select="collection[@name='hdl']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='fnl']"/>
+ 	  <p /><xsl:apply-templates select="collection[@name='envl']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='aginfo']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='gtz']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='cdl']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='edudev']"/>
+	  <p /><xsl:apply-templates select="collection[@name='nigeria']"/>
+	  <p /><xsl:apply-templates select="collection[@name='safem']"/>
+	  <p />firstaid<xsl:apply-templates select="collection[@name='firstaid']"/>
+	</td>
+	<td align='center'>
+	  <p /><xsl:apply-templates select="collection[@name='mhl']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='fnl1_1']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='paho']"/>
+	  <p /><xsl:apply-templates select="collection[@name='unesco']"/>
+          <p /><xsl:apply-templates select="collection[@name='muster']"/>
+	  <p /><xsl:apply-templates select="collection[@name='unescoen']"/>
+	  <p /><xsl:apply-templates select="collection[@name='dfid']"/>	
+	  <p /><xsl:apply-templates select="collection[@name='ewf']"/>	
+	  <p /><xsl:apply-templates select="collection[@name='fi1998']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='faodocs']"/>  
+	</td>
+	<td align='center'> 
+	  <p /><xsl:apply-templates select="collection[@name='ccgi']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='fnl2.2']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='who']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='tulane']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='unaids']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='aedl']"/>
+	  <p /><xsl:apply-templates select="collection[@name='whoedm']"/>
+	  <p /><xsl:apply-templates select="collection[@name='helid']"/>
+	  <p /><xsl:apply-templates select="collection[@name='ipc']"/>	
+	  <p /><xsl:apply-templates select="collection[@name='povsem']"/>
+	</td>
+      </tr>
+    </table>
+  </xsl:template>
+  
+
+  <xsl:template name="collectrest">
+    <table width='537'>
+      <tr valign='top'>
+	<td align='center'>
+	  <p /><xsl:apply-templates select="collection[@name='acrodemo']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='niupepa']"/> 
+	  <p />howto DL<xsl:apply-templates select="collection[@name='howto']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='arabic']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='chinese']"/> 
+ 	  <p /><xsl:apply-templates select="collection[@name='folktale']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='demooai']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='tidbits']"/> 
+	</td>
+	<td align='center'>
+	  <p />meldex<xsl:apply-templates select="collection[@name='meldex']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='wordpdf']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='musvid']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='gberg']"/> 
+ 	  <p /><xsl:apply-templates select="collection[@name='allshake']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='aircraft']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='csbib']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='coltbib']"/> 
+	</td>
+	<td align='center'>
+	  <p /><xsl:apply-templates select="collection[@name='cstr']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='gsarch']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='ohist']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='hcibib']"/> 
+ 	  <p /><xsl:apply-templates select="collection[@name='whist']"/> 
+	  <p /><xsl:apply-templates select="collection[@name='beowulf']"/>
+	  <p /><xsl:apply-templates select="collection[@name='tcc']"/>
+	  <p /><xsl:apply-templates select="collection[@name='jair']"/>  
+	</td>
+      </tr>
+    </table>
+  </xsl:template>
+  
+</xsl:stylesheet>  
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/page-common.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/page-common.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/interfaces/nzdl/transform/page-common.xsl	(revision 21112)
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:template name="greenstone_info">
+    <p />
+    <table>
+      <tr valign='top'>
+	<td><xsl:call-template name="textpoem"/></td>
+	<td><xsl:call-template name="imagegreenstone"/></td>
+      </tr></table>
+    <p />
+    <table><tr><td><xsl:call-template name="textgreenstone"/></td></tr></table>
+  </xsl:template>
+
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/search.html
===================================================================
--- /main/tags/3.01/gsdl3/web/search.html	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/search.html	(revision 21112)
@@ -0,0 +1,224 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+
+<html>
+<head>
+<title>AJAX GS3 Search</title>
+<META HTTP-EQUIV="PRAGMA" CONTENT="NO-CACHE">
+<META HTTP-EQUIV="CACHE-CONTROL" CONTENT="NO-CACHE">
+<style type="text/css">
+<!-- 
+BODY, P, LI { font-family: sans-serif; } 
+H2 {font-size: 90%;}
+BUTTON {font-size: 0.8em; margin: 0 1em; color: red;}
+#searchArea {background-color:white; }
+#resultArea {background-color:white; padding: 0em 0em;}
+#resultSummary {font-size: 80%; padding: 0.2 em 0.2em;}
+#resultSummary P {padding: 0.2em 0.3em;}
+#query {padding: 0.1em 0.5em; width: 400px; height: 150px; background-color:#d3d3d3;}
+#timing {float:right; width: 250px; border: thin solid #d3d3d3; padding: 0.2em 0.2em; display: none;}
+#resultList {	list-style-image: none; 
+						list-style-type: square;
+						font-size: 85%; padding: 1em 0em 1em 2em;}
+#resultList LI {padding: 0.3em 0em; }
+-->
+</style>
+<script type="text/javascript" src="xmlextras.js">
+</script> 
+<script type="text/javascript" src="gs_ajax_utils.js">
+</script> 
+<script type="text/javascript">
+<!--
+
+localsiteServerAddress = "/gsdl3/services/localsite";
+libraryAddress = "/gsdl3/library";
+function loadAsync(sUri, SOAPMessage, callbackFunction) {
+   var xmlHttp = XmlHttp.create();
+   var async = true;
+   xmlHttp.open("POST", sUri, async);
+   xmlHttp.onreadystatechange = function () {
+      if (xmlHttp.readyState == 4){
+		var result = xmlHttp.responseText;
+			//getTitle2(xmlHttp.responseXML, xmlHttp.responseText);
+			eval(callbackFunction);
+    }     
+   }
+   xmlHttp.setRequestHeader("SOAPAction", " ");
+   xmlHttp.setRequestHeader("Content-Type", "Content-Type: text/xml; charset=utf-8");
+ 
+   xmlHttp.send(SOAPMessage);
+}
+
+
+function ajaxSearch() {
+
+searchStartTime = new Date();
+
+resultList = document.getElementById('resultList');
+removeAllChildren(resultList);
+resultSummary = document.getElementById('resultSummary');
+removeAllChildren(resultSummary);
+
+/*
+for (i=0; i < 10; i++) {
+    listElement = document.createElement('li');
+    anchor = document.createElement('a');
+    anchor.setAttribute('href', url);
+    anchor.appendChild(document.createTextNode(  text));
+    listElement.appendChild(anchor);
+    resultList.appendChild(listElement);
+}
+*/
+
+queryText = document.getElementById('queryText').value;
+
+maxdocs = newElementAtt('param', '', new Array('name', 'value'), new Array('maxDocs', '100') ) ;
+level = newElementAtt('param', '', new Array('name', 'value'), new Array('level', 'Sec') ) ;
+indexValue = newElementAtt('param', '', new Array('name', 'value'), new Array('index', 'ZZ') ) ;
+matchMode = newElementAtt('param', '', new Array('name', 'value'), new Array('matchMode', 'some') ) ;
+query = newElementAtt('param', '', new Array('name', 'value'), new Array('query', queryText) ) ;
+caseValue = newElementAtt('param', '', new Array('name', 'value'), new Array('case', '1') ) ;
+sortBy = newElementAtt('param', '', new Array('name', 'value'), new Array('sortBy', '1') ) ;
+stem = newElementAtt('param', '', new Array('name', 'value'), new Array('stem', '1') ) ;
+
+
+paramList = newElement('paramList', maxdocs + level + indexValue + matchMode + query + caseValue + sortBy + stem);
+
+collection = document.getElementById('collectionText').value;
+request = newElementAtt('request', paramList, new Array('lang', 'to', 'type', 'uid'), new Array('en',collection + '/TextQuery', 'process', '1'  ) );
+message = newElement('message', request );
+soapMessage = messageToSOAP(message);
+loadAsync(localsiteServerAddress, soapMessage, 'getResults(xmlHttp.responseXML)');
+}
+
+function getResults(xmlDoc) {
+idsReturnedTime = new Date();
+    metadataList = xmlDoc.getElementsByTagName('metadata');
+    for (i=0; i < metadataList.length; i++) {
+    	metadata = metadataList[i];
+    	if (isElement(metadata)) {
+		  nameAtt = metadata.getAttribute('name')
+    		if ( nameAtt == 'numDocsMatched') {
+    			numDocsMatched = getText(metadata);
+    		}
+    		if (nameAtt == 'numDocsReturned') {
+    			numDocsReturned = getText(metadata);
+    		}
+    		if (nameAtt  == 'query') {
+    			query = getText(metadata);
+    		}
+    	}
+    }
+    
+    resultSummary = document.getElementById('resultSummary');
+    para = document.createElement('p');
+    para.appendChild(document.createTextNode('Query: ' + query + ' Number of Matching Docs: ' + numDocsMatched + ' Number of returned documents: ' + numDocsReturned));
+    resultSummary.appendChild(para);
+    
+    documentNodeList = xmlDoc.getElementsByTagName('documentNode');
+    	
+    for (i=0; i < documentNodeList.length; i++) {
+    	documentNode = documentNodeList[i];
+    	if (isElement(documentNode)) {
+    		docIDArray[i] = documentNode.getAttribute('nodeID');
+    	}
+    }
+   
+    var docsList = '';
+    for (i=0; i < docIDArray.length; i++) {
+    	docsList += newElementAtt1('documentNode', '', 'nodeID', docIDArray[i]);
+    }
+	
+    docNodeList = newElement('documentNodeList', docsList);
+	collection = document.getElementById('collectionText').value;
+    param = newElementAtt('param', '', new Array('name', 'value'), new Array('metadata', 'Title') );
+    paramList = newElement('paramList', param);
+	request = newElementAtt('request', paramList + docNodeList, new Array('lang', 'to', 'type', 'uid'), new Array('en',collection + '/DocumentMetadataRetrieve', 'process', '1'  ) );
+    message = newElement('message', request ); 
+    soapMessage = messageToSOAP(message);
+	//alert(soapMessage);
+    loadAsync(localsiteServerAddress, soapMessage, 'displayResults(xmlHttp.responseXML)');
+}
+
+
+function displayResults(xmlDoc) {
+	metadataReturnedTime = new Date();
+	url1 = libraryAddress+'?a=d&d=';
+	url2 = '&c=' + document.getElementById('collectionText').value + '&dt=hierarchy';
+	resultList = document.getElementById('resultList');
+    metadataList = xmlDoc.getElementsByTagName('metadata');
+	//alert('in display results ' + metadataList.length + xmlDoc.xml);
+        for (i=0; i < metadataList.length; i++) {
+			  //alert('in display results ' + i);
+        	metadata = metadataList[i];
+        	if (isElement(metadata)) {
+        		if (metadata.getAttribute('name') == 'Title') {
+					
+          		  listElement = document.createElement('li');
+                anchor = document.createElement('a');
+                anchor.setAttribute('href', url1 + docIDArray[i] + url2);
+                anchor.appendChild(document.createTextNode(getText(metadata)) );
+                listElement.appendChild(anchor);
+                resultList.appendChild(listElement);
+        		}   		
+        	}
+        }
+			finishedTime = new Date();
+			displayTime();
+			
+}
+
+function displayTime() {
+searchStart = document.getElementById('searchStart');
+idsReturned = document.getElementById('idsReturned');
+metadataReturned = document.getElementById('metadataReturned');
+//removeAllChildren(searchStart);
+//searchStart.appendChild(document.createTextNode(searchStartTime.getMilliseconds() + ' ms'))
+removeAllChildren(idsReturned);
+idsReturned.appendChild(document.createTextNode(idsReturnedTime.getMilliseconds() - searchStartTime.getMilliseconds()+ ' ms'))
+removeAllChildren(metadataReturned);
+metadataReturned.appendChild(document.createTextNode(metadataReturnedTime.getMilliseconds() - idsReturnedTime.getMilliseconds()+ ' ms'))
+//alert(searchStartTime.getMilliseconds() + ' ' + idsReturnedTime.getMilliseconds() + ' ' + metadataReturnedTime.getMilliseconds() + ' ' + ' ' + finishedTime.getMilliseconds());
+}
+
+
+var docIDArray = new Array();
+var searchStartTime, idsReturnedTime, metadataReturnedTime, finishedTime;
+
+
+
+
+// -->
+</script>
+</head>
+<body>
+
+
+
+<div id="searchArea">
+
+<div id="timing">
+<p><strong>Timing</strong></p>
+<table border="0" cellpadding="0" cellspacing="0" summary="">
+<!-- <tr><td>Search started:</td><td id="searchStart"></td></tr> -->
+<tr><td>Time to get IDs:</td><td id="idsReturned"></td></tr>
+<tr><td>Time to get Metadata:</td><td id="metadataReturned"></td></tr>
+</table>
+</div> 
+
+<div id="query"><h2>Search a Greenstone 3 Collection</h2>
+<form action="">
+<p><label for="collectionText">GS3 Collection:</label>
+<input id="collectionText" type="text" value="gs2mgppdemo"/></p>
+
+<p><input id="queryText" type="text" /><input id="searchButton" type="submit" value="Search" onclick="ajaxSearch();return false;"></input></p>
+
+</form></div>
+</div>
+
+<div id="resultArea">
+<div id="resultSummary"></div>
+<ul id="resultList"></ul>
+</div>
+ 
+</body>
+</html>
Index: /main/tags/3.01/gsdl3/web/sites/gateway/siteConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/gateway/siteConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/gateway/siteConfig.xml	(revision 21112)
@@ -0,0 +1,9 @@
+<siteConfig>
+  <siteList localSiteName="gateway">
+    <!-- a site element needs name and type, then either an addess attribute where the URL of the server can be accessed, or a localSite attribute, which specifies to use the local server on a local site. This second method is just for testing. The address of a SOAP server deployed by axis in greenstone would be like http://{tomcat_server}:{tomcat_port}/gsdl3/services/{site_name} -->
+    <site name="localsite" localSite="localsite" type="soap"/>
+  </siteList>
+  <serviceClusterList/>
+  <serviceRackList/>
+</siteConfig>
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/README
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/README	(revision 21112)
@@ -0,0 +1,57 @@
+The Sample XML texts collection.
+
+This README provides directions on how to rebuild the collection if necessary, 
+and how to create a new collection using this method.
+
+Building the gberg collection (linux):
+
+Starting from the gberg directory (where this README file is located):
+cd java
+javac *.java
+cd ..
+java -classpath $CLASSPATH:./java BuildXMLColl $GSDL3HOME/web/sites/localsite gberg
+mv building index
+cp etc/buildConfig.xml index
+cp import/*.dtd $GSDL3HOME/resources/dtd/
+
+The building process:
+
+There are two stages, import and build.
+Importing goes through the xml documents in the import directory and adds 
+gs3:id attributes to indexable nodes. Building makes another pass though the 
+documents, indexing any indexable nodes, and assigning them node ids. Node 
+ids are made up of several parts: document id, scope, tag name, gs3:id. 
+Document id refers to the work, and is generated from the original document 
+filename. For example, origin.xml becomes origin. The scope part is optional. 
+Tag name is the name of the node that is to be indexed, and gs3:id is its id.
+
+The class XMLTagInfo provides two methods, isIndexable and isScopable, and is 
+used to determine which tags provide scope, and which tags should be indexed. 
+This is the only part of the build code that is specific to the documents in 
+the collection.
+
+Indexable nodes are those that should be indexed individually. When a search 
+is done, these are the units that will be returned. For instance, suppose a 
+collection contains books with chapters and sections of chapters. To have 
+searching and retrieval at chapter level, the chapter nodes would be made 
+indexable. Alternatively, making the section nodes indexable would provide 
+searching at the smaller section level instead.
+
+Scopable nodes are not necessarily indexed, but are recorded as part of the 
+id as a scope. This is to speed up searching for the appropriate tag during 
+retrieval. Scopeable tags should only occur once per document.
+
+Building other collections:
+
+To use the java building stuff for other XML document collections, you can 
+just modify the XMLTagInfo.java file to include the appropriate tag names for 
+your XML documents. Then put import documents into the import directory, and 
+run the BuildXMLColl program as above. Configuration files will need to be 
+created for the collection (etc/collectionConfig.xml and 
+index/buildConfig.xml).
+
+The Greenstone runtime software has problems locating DTDs, so any DTDs for 
+your collection should be placed in the collection's resources directory.
+If DTD files are shared between collections, they can go into the 
+WEB-INF/classes directory (in gsdl3/web or tomcat/webapps/gsdl3, depending on 
+your setup).
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/buildConfig.xml	(revision 21112)
@@ -0,0 +1,16 @@
+<buildConfig xmlns:gsf="http://www.greenstone.org/configformat" >
+  <metadataList>
+    <metadata name="numDocs">4</metadata>
+    <metadata name="buildType">lucene</metadata>
+  </metadataList>
+  <serviceRackList>
+    <serviceRack name="GutBookRetrieve">
+      <paramList>
+	<param name="tocXSLT" value="produce-toc"/>
+	<param name="documentEncoding" value="UTF-8"/>
+	<param name="documentRootTag" value="book"/>
+      </paramList>
+    </serviceRack>
+    <serviceRack name="LuceneSearch"/>
+  </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,61 @@
+<collectionConfig 
+  xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" 
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
+  <metadataList>
+    <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Sample XML Texts Collection</displayItem>
+    <displayItem name='icon' lang='en'>gberg.gif</displayItem>
+    <displayItem name='smallicon' lang='en'>gbergsm.gif</displayItem>
+    <displayItem name='description' lang='en'>
+&lt;p&gt;This collection is designed to demonstrate some of the flexibility of the Greenstone 3 architecture. Collections are not restricted to those built using the standard building process, but rather can be anything you want, as long as suitable services are provided.&lt;/p&gt; 
+&lt;p&gt; This collection contains 4 documents obtained from the &lt;a href="http://gutenberg.net/"&gt;Gutenberg project&lt;/a&gt;. Project Gutenberg documents are marked up in XML, and come with DTDs and CSS stylesheets. The collection has not been built using either Greenstone2 or Greenstone3 style building. Instead, simple java programs using the Lucene API have been created to do the indexing, and document retrieval is simply done from the original (actually slightly modified) XML documents. A description of the building process can be found in the &lt;a href="sites/localsite/collect/gberg/README"&gt;README&lt;/a&gt; file for this collection.&lt;/p&gt; 
+&lt;p&gt; This collection has a different appearance to standard collections, accomplished through the use of custom XSLT files, found in the transform directory of the collection. The 'about' page for the collection shows a list of all the documents. This is achieved by using a new about.xsl file. Document display is quite different for this collection: the display can be either table of contents style, or text style. Two new xslts were written to handle the two display types. A new document action also had to be created (XMLDocumentAction), and the interface config file was altered to map the xd action code to this new Action class.&lt;/p&gt; 
+&lt;p&gt; &amp;lt;action name='xd' class='XMLDocumentAction'&amp;gt;&lt;br/&gt;
+&amp;nbsp;&amp;nbsp;&amp;lt;subaction name='toc' xslt='document-toc.xsl'/&amp;gt;&lt;br/&gt;
+&amp;nbsp;&amp;nbsp;&amp;lt;subaction name='text' xslt='document-text.xsl'/&amp;gt;&lt;br/&gt;
+&amp;lt;/action&amp;gt;&lt;/p&gt; 
+&lt;p&gt;The collection uses a generic Greenstone Lucene search service, and a specific retrieval service class, GutBookRetrieve. This extends the generic XMLRetrieve serviceRack to handle metadata extraction for the Gutenberg documents.&lt;/p&gt; 
+&lt;p&gt;The collection configuration file can be seen &lt;a href="sites/localsite/collect/gberg/etc/collectionConfig.xml"&gt;here&lt;/a&gt;, and the building configuration file can be seen &lt;a href="sites/localsite/collect/gberg/index/buildConfig.xml"&gt;here&lt;/a&gt;.&lt;/p&gt;
+</displayItem>
+  </displayItemList>
+  <search>
+    <format>
+      <gsf:template match="documentNode">
+	<xsl:param name="collName"/>
+	<xsl:param name="serviceName"/>
+	<td>
+	<b><a href="{$library_name}?a=xd&amp;sa=text&amp;c={$collName}&amp;d={@nodeID}&amp;p.a=q&amp;p.s={$serviceName}"><xsl:choose><xsl:when test="metadataList/metadata[@name='Title']"><gsf:metadata name="Title"/></xsl:when><xsl:otherwise>(section)</xsl:otherwise></xsl:choose></a></b> from <b><a href="{$library_name}?a=xd&amp;sa=toc&amp;c={$collName}&amp;d={@nodeID}.rt&amp;p.a=q&amp;p.s={$serviceName}"><gsf:metadata name="Title" select="root"/></a></b>
+	</td>
+      </gsf:template>
+    </format>
+  </search>
+  <browse/>
+  <documentList>
+    <document name='origin'>
+      <metadataList>
+	<metadata name='Title'>On the Origin of Species</metadata>
+	<metadata name="Author">Charles Darwin</metadata>
+      </metadataList>
+    </document>
+    <document name='iliad'>
+      <metadataList>
+	<metadata name='Title'>The Iliad</metadata>
+	<metadata name="Author">Homer</metadata>
+      </metadataList>
+    </document>
+    <document name='owlcreek'>
+      <metadataList>
+	<metadata name='Title'>An Occurrence At Owl Creek Bridge</metadata>
+	<metadata name="Author">Ambrose Bierce</metadata>
+      </metadataList>
+    </document>
+    <document name='fishpatrol'>
+      <metadataList>
+	<metadata name='Title'>Tales of the Fish Patrol</metadata>
+	<metadata name="Author">Jack London</metadata>
+      </metadataList>
+    </document>
+  </documentList>
+</collectionConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionInit.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionInit.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/etc/collectionInit.xml	(revision 21112)
@@ -0,0 +1,2 @@
+<collectionInit class="XMLCollection"/>
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/bookfrag.dtd
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/bookfrag.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/bookfrag.dtd	(revision 21112)
@@ -0,0 +1,529 @@
+<!--This is bookfrag DTD-->
+
+<!--entity declarations-->
+<!--#############################################-->
+
+<!ENTITY % stdatts 
+"ref   IDREF  #IMPLIED
+ id    ID     #IMPLIED
+ type  CDATA  #IMPLIED
+ role  CDATA  #IMPLIED
+ class CDATA  #IMPLIED"
+>
+
+<!ENTITY % inline.class "|quote|emph|ital|reference|date|place|name|graphic|txterr|mkuperr|misc ">
+
+<!ENTITY % block.class "|letter|blockquote|footnote|note|list|deflist|table|blockgraphic">
+
+
+<!-- the top level element-->
+<!--##############################################-->
+
+<!ELEMENT book (acknowledge?,meta*,frontmatter?,bookbody,backmatter?,endmeta*)>
+
+<!--begin top level element declarations-->
+<!--##############################################-->
+
+
+<!--the acknowledge element is for a simple statement 
+designed to be placed at the beginning of the visible document.
+Suggested content:
+  Name of Book
+  Author
+  PG and HWG blurb
+  Original Transcriber + date
+  Marker 
+  e.g.
+  "Origin of the Species by Charles Darwin
+  A project of Project Gutenberg and the HTML Writers Guild.
+  Transcribed by  
+  Markup by Frank Boumphrey. Jan 22 2000"
+  -->
+
+<!ELEMENT acknowledge (#PCDATA %inline.class;)*>
+<!ATTLIST acknowledge
+  %stdatts;
+>
+
+<!--the meta tags are here to provide a means for comment 
+at either the very beginning or end of the document-->
+<!ELEMENT meta EMPTY>
+<!ATTLIST meta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+<!ELEMENT endmeta EMPTY >
+<!ATTLIST endmeta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+
+
+<!--the bookbody element contains the body of the book it can 
+consist of either part or chapter divisions-->
+
+<!ELEMENT bookbody (part*|chapter*)>
+
+<!--
+the backmatter element contains the backmatter of the book -->
+
+<!ELEMENT backmatter ((appendix|index|glossary|biblio|note)*,colophon?)>
+
+<!--The endmeta tag is groupedwith its twin meta-->
+
+
+<!--end top level element declarations-->
+
+<!--begin frontmatter element declarations-->
+<!--##############################################-->
+
+<!ELEMENT frontmatter (htitlepage|copypage|epigraph|titlepage|
+                toc|acksect|dedication|preface|introduction|miscfm)*>
+
+<!ELEMENT htitlepage (#PCDATA|title|subtitle|author|para|poem|song %inline.class;)*>
+<!ATTLIST htitlepage
+  %stdatts;
+>
+
+<!ELEMENT copypage (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST copypage
+  %stdatts;
+>
+
+<!ELEMENT epigraph (#PCDATA|para|poem|song|note|blockquote %inline.class;)*>
+<!ATTLIST epigraph
+  %stdatts;
+>
+
+
+<!ELEMENT titlepage (#PCDATA|partnum|title|subtitle|author|pubinfo|para|poem|song|note|line %inline.class;)*>
+<!ATTLIST titlepage
+  %stdatts;
+>
+
+<!ELEMENT pubinfo (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST pubinfo
+  %stdatts;
+>
+
+
+<!ELEMENT toc (#PCDATA|title|subtitle|subsubtitle|item|list|deflist %inline.class;)*>
+<!--use type or role to describe toctype="other"-->
+<!ATTLIST toc
+  toctype (contents|maps|graphics|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT acksect (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST acksect
+  %stdatts;
+>
+
+<!ELEMENT dedication (#PCDATA|title|para|poem|song|note %inline.class;)*>
+<!ATTLIST dedication
+  %stdatts;
+>
+
+
+<!ELEMENT preface ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?,preauthor?)>
+<!ATTLIST preface  %stdatts;
+>
+
+<!ELEMENT introduction ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST introduction  %stdatts;
+>
+
+<!ELEMENT miscfm (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST miscfm
+  %stdatts;
+>
+
+<!ELEMENT preauthor (#PCDATA|author %inline.class;)*>
+<!ATTLIST preauthor
+  %stdatts;
+>
+
+
+
+<!--end frontmatter element declarations-->
+
+<!--begin bookbody element declarations-->
+<!--##############################################-->
+
+
+<!ELEMENT part (acknowledge?,(titlepage|toc|htitlepage)*,chapter*)>
+<!ATTLIST part
+  %stdatts;
+>
+
+<!ELEMENT chapter ((title|chapheader)?,(para|poem|song|sect1|simplesect|page %block.class;)*,endchap?)>
+<!ATTLIST chapter  %stdatts;
+>
+
+<!--end bookbody element declarations-->
+
+<!--the lowerlevel bookbody elements-->
+<!--##############################################-->
+
+<!--the chapter heading elements-->
+<!--##############################################-->
+
+
+
+
+<!ELEMENT title (#PCDATA %inline.class;)*>
+<!ATTLIST title
+  %stdatts;
+>
+
+<!ELEMENT subtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subtitle
+  %stdatts;
+>
+
+<!ELEMENT subsubtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subsubtitle
+  %stdatts;
+>
+
+
+
+<!--validity constraint. Only one title,chapnum and chapsummary -->
+<!ELEMENT chapheader (title|subtitle|chapnum|chapsummary|blockquote|para|note)*>
+<!ATTLIST chapheader
+  %stdatts;
+>
+
+
+<!ELEMENT chapnum (#PCDATA %inline.class;)*>
+<!ATTLIST chapnum
+  %stdatts;
+>
+
+<!ELEMENT chapsummary (#PCDATA %inline.class;)*>
+<!ATTLIST chapsummary
+  %stdatts;
+>
+
+
+<!--begin backmatter element declarations-->
+<!--##############################################-->
+
+<!--end backmatter element declarations-->
+
+<!ELEMENT index (title|item|list|deflist|note)*>
+<!ATTLIST index
+  indtype (contents|authors|firstlines|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT glossary (title|item|list|deflist|note)*>
+<!ATTLIST glossary  
+  %stdatts;
+>
+
+<!ELEMENT biblio (title|item|list|deflist|note)*>
+<!ATTLIST biblio  
+  %stdatts;
+>
+
+
+<!ELEMENT appendix ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST appendix  %stdatts;
+>
+
+<!ELEMENT colophon (#PCDATA|para|poem|song %inline.class;)*>
+<!ATTLIST colophon
+  %stdatts;
+>
+
+<!--the block elements-->
+<!--##############################################-->
+
+
+<!--validity constraint. Only one title per paragraph-->
+<!ELEMENT para (#PCDATA|title %inline.class;)*>
+<!ATTLIST para
+  %stdatts;
+>
+
+<!ELEMENT simplesect (title,(subtitle|para|poem|song %block.class;)*)>
+<!ATTLIST simplesect
+  %stdatts;
+>
+
+<!ELEMENT sect1 (title,(sect2|simplesect|para|poem|song %block.class;)*)>
+<!ATTLIST sect1
+  %stdatts;
+>
+
+<!ELEMENT sect2 (title,(sect3|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect2
+  %stdatts;
+>
+
+<!ELEMENT sect3 (title,(sect4|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect3
+  %stdatts;
+>
+
+<!ELEMENT sect4 (title,(simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect4
+  %stdatts;
+>
+
+<!ELEMENT page (#PCDATA %inline.class;)*>
+<!ATTLIST page
+  %stdatts;
+>
+
+<!ELEMENT misc (#PCDATA %inline.class;)*>
+<!ATTLIST misc
+  %stdatts;
+>
+
+
+
+<!ELEMENT blockquote (title?,(para|poem|song)*,attrib?)>
+<!ATTLIST blockquote
+  %stdatts;
+>
+
+<!ELEMENT attrib (#PCDATA %inline.class;)*>
+<!ATTLIST attrib
+  %stdatts;
+>
+
+<!ELEMENT caption (#PCDATA %inline.class;)*>
+<!ATTLIST caption
+  %stdatts;
+>
+<!--##############################################-->
+<!--poetry and songs in the text-->
+<!ELEMENT poem (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST poem
+  %stdatts;
+>
+
+<!ELEMENT song (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST song
+  %stdatts;
+>
+
+
+<!ELEMENT verse (title|subtitle|line|note)*>
+<!ATTLIST verse
+  %stdatts;
+>
+
+<!ELEMENT line (#PCDATA %inline.class;)*>
+<!ATTLIST line
+  %stdatts;
+>
+
+
+<!--validity constraint: Only one title per poem or per verse-->
+
+<!--end of poetry in the text-->
+
+<!--letter module-->
+
+<!ELEMENT letter (address|to|from|salut|sig|title|subtitle|para|poem|song|line|note)*>
+<!ATTLIST letter
+  %stdatts;
+>
+
+<!ELEMENT address (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST address
+  %stdatts;
+>
+
+<!ELEMENT to (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST to
+  %stdatts;
+>
+
+
+<!ELEMENT from (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST from
+  %stdatts;
+>
+
+
+<!ELEMENT salut (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST salut
+  %stdatts;
+>
+
+
+<!ELEMENT sig (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST sig
+  %stdatts;
+>
+
+
+<!--end letters module-->
+<!--endchap-->
+<!--##############################################-->
+<!--
+  Endchap should only be used when there is a 
+  considerable ammount of material to be organized.
+  For the occasional note footnote and note will suffice
+-->
+
+<!ELEMENT endchap (para %block.class;)*>
+<!ATTLIST endchap  %stdatts;
+>
+
+<!--lists-->
+<!--##############################################-->
+
+
+<!ELEMENT list (title?,(list|item)*)>
+<!ATTLIST list
+  %stdatts;
+>
+
+<!ELEMENT item (#PCDATA|para|poem|song|simplesect %block.class; %inline.class;)*>
+<!ATTLIST item
+  %stdatts;
+>
+
+<!ELEMENT deflist (title?,(item,desc?,def*)*)>
+<!ATTLIST deflist
+  %stdatts;
+>
+
+<!ELEMENT desc (#PCDATA %inline.class;)*>
+<!ATTLIST desc
+  %stdatts;
+>
+
+<!ELEMENT def (#PCDATA %inline.class;)*>
+<!ATTLIST def
+  %stdatts;
+>
+<!--end lists-->
+<!--table element-->
+<!--##############################################-->
+
+
+<!ELEMENT table (title?,row*,caption?)>
+<!ATTLIST table
+  %stdatts;
+>
+
+<!ELEMENT row (cell)*>
+<!ATTLIST row
+  %stdatts;
+>
+
+<!ELEMENT cell (#PCDATA %block.class; %inline.class;)*>
+<!ATTLIST cell
+  %stdatts;
+>
+
+<!--end table-->
+
+<!--graphics-->
+<!--##############################################-->
+
+<!--
+  Note that the required desc attribute is added 
+  text composed by the marker.
+-->
+
+<!ELEMENT graphic EMPTY >
+<!ATTLIST graphic
+  desc CDATA #REQUIRED
+  href CDATA #REQUIRED
+  %stdatts;
+>
+
+<!ELEMENT blockgraphic (title?,graphic,caption?)>
+<!ATTLIST blockgraphic
+  %stdatts;
+>
+
+
+<!--the inline elements-->
+<!--##############################################-->
+
+
+
+<!ELEMENT footnote (#PCDATA %inline.class;)*>
+<!ATTLIST footnote
+  %stdatts;
+>
+
+<!ELEMENT note (#PCDATA %inline.class;)*>
+<!ATTLIST note
+  %stdatts;
+>
+
+
+<!ELEMENT quote (#PCDATA %inline.class;)*>
+<!ATTLIST quote
+  %stdatts;
+>
+
+<!ELEMENT emph (#PCDATA %inline.class;)*>
+<!ATTLIST emph
+  %stdatts;
+>
+
+<!ELEMENT ital (#PCDATA %inline.class;)*>
+<!ATTLIST ital
+  %stdatts;
+>
+
+<!ELEMENT reference (#PCDATA %inline.class;)*>
+<!ATTLIST reference
+  %stdatts;
+>
+
+<!ELEMENT author (#PCDATA %inline.class;)*>
+<!ATTLIST author
+  %stdatts;
+>
+
+<!ELEMENT date (#PCDATA %inline.class;)*>
+<!ATTLIST date
+  %stdatts;
+>
+
+<!ELEMENT place (#PCDATA %inline.class;)*>
+<!ATTLIST place
+  %stdatts;
+>
+
+<!ELEMENT name (#PCDATA %inline.class;)*>
+<!ATTLIST name
+  %stdatts;
+>
+
+
+<!--use to enclose the text you think is in error
+optional explanation if error is not obvious-->
+<!ELEMENT txterr (#PCDATA)*>
+<!ATTLIST txterr
+  explain  CDATA  #IMPLIED
+  %stdatts;
+>
+
+<!--an explanation is required-->
+<!ELEMENT mkuperr EMPTY>
+<!ATTLIST mkuperr
+  explain  CDATA  #REQUIRED
+  %stdatts;
+>
+
+<!--added as titlepage content  Feb/6/00-->
+<!ELEMENT partnum (#PCDATA %inline.class;)*>
+<!ATTLIST partnum
+  %stdatts;
+>
+
+<!--End of bookfrag dtd-->
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/fishpatrol.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/fishpatrol.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/fishpatrol.xml	(revision 21112)
@@ -0,0 +1,5033 @@
+<?xml version="1.0"?>
+<!DOCTYPE gutbook SYSTEM "gutbook1.dtd">
+
+<?xml-stylesheet type="text/css" href="darwin.css"?>
+
+
+<gutbook>
+<gutblurb>
+<para>
+Project Gutenberg's Etext of Tales of the Fish Patrol, by London
+#8 in our series by Jack London
+</para>
+
+<para>
+
+Copyright laws are changing all over the world, be sure to check
+the copyright laws for your country before posting these files!!
+</para>
+
+<para>
+Please take a look at the important information in this header.
+We encourage you to keep this file on your own disk, keeping an
+electronic path open for the next readers.  Do not remove this.
+</para>
+
+<para>
+
+**Welcome To The World of Free Plain Vanilla Electronic Texts**
+</para>
+
+<para>
+**Etexts Readable By Both Humans and By Computers, Since 1971**
+</para>
+
+<para>
+*These Etexts Prepared By Hundreds of Volunteers and Donations*
+</para>
+
+<para>
+Information on contacting Project Gutenberg to get Etexts, and
+further information is included below.  We need your donations.
+</para>
+
+<para>
+
+Tales of the Fish Patrol
+</para>
+
+<para>
+by Jack London
+</para>
+
+<para>
+May, 1997  [Etext #911]
+</para>
+
+<para>
+
+Project Gutenberg's Etext of Tales of the Fish Patrol, by London
+*****This file should be named totfp10.txt or totfp10.zip******
+</para>
+
+<para>
+Corrected EDITIONS of our etexts get a new NUMBER, totfp11.txt.
+VERSIONS based on separate sources get new LETTER, totfp10a.txt.
+</para>
+
+<para>
+
+Scanned and proofed by David Price
+ccx074@coventry.ac.uk
+</para>
+
+<para>
+
+We are now trying to release all our books one month in advance
+of the official release dates, for time for better editing.
+</para>
+
+<para>
+Please note:  neither this list nor its contents are final till
+midnight of the last day of the month of any such announcement.
+The official release date of all Project Gutenberg Etexts is at
+Midnight, Central Time, of the last day of the stated month.  A
+preliminary version may often be posted for suggestion, comment
+and editing by those who wish to do so.  To be sure you have an
+up to date first edition [xxxxx10x.xxx] please check file sizes
+in the first week of the next month.  Since our ftp program has
+a bug in it that scrambles the date [tried to fix and failed] a
+look at the file size will have to do, but we will try to see a
+new copy has at least one byte more or less.
+</para>
+
+<para>
+
+Information about Project Gutenberg (one page)
+</para>
+
+<para>
+We produce about two million dollars for each hour we work.  The
+fifty hours is one conservative estimate for how long it we take
+to get any etext selected, entered, proofread, edited, copyright
+searched and analyzed, the copyright letters written, etc.  This
+projected audience is one hundred million readers.  If our value
+per text is nominally estimated at one dollar then we produce $2
+million dollars per hour this year as we release thirty-two text
+files per month:  or 400 more Etexts in 1996 for a total of 800.
+If these reach just 10% of the computerized population, then the
+total should reach 80 billion Etexts.
+</para>
+
+<para>
+The Goal of Project Gutenberg is to Give Away One Trillion Etext
+Files by the December 31, 2001.  [10,000 x 100,000,000=Trillion]
+This is ten thousand titles each to one hundred million readers,
+which is only 10% of the present number of computer users.  2001
+should have at least twice as many computer users as that, so it
+will require us reaching less than 5% of the users in 2001.
+</para>
+
+<para>
+
+We need your donations more than ever!
+</para>
+
+<para>
+
+All donations should be made to "Project Gutenberg/CMU": and are
+tax deductible to the extent allowable by law.  (CMU = Carnegie-
+Mellon University).
+</para>
+
+<para>
+For these and other matters, please mail to:
+</para>
+
+<para>
+Project Gutenberg
+P. O. Box  2782
+Champaign, IL 61825
+</para>
+
+<para>
+When all other email fails try our Executive Director:
+Michael S. Hart &lt;hart@pobox.com&gt;
+</para>
+
+<para>
+We would prefer to send you this information by email
+(Internet, Bitnet, Compuserve, ATTMAIL or MCImail).
+</para>
+
+<para>
+******
+If you have an FTP program (or emulator), please
+FTP directly to the Project Gutenberg archives:
+[Mac users, do NOT point and click. . .type]
+</para>
+
+<para>
+ftp uiarchive.cso.uiuc.edu
+login:  anonymous
+password:  your@login
+cd etext/etext90 through /etext96
+or cd etext/articles [get suggest gut for more information]
+dir [to see files]
+get or mget [to get files. . .set bin for zip files]
+GET INDEX?00.GUT
+for a list of books
+and
+GET NEW GUT for general information
+and
+MGET GUT* for newsletters.
+</para>
+
+<para>
+**Information prepared by the Project Gutenberg legal advisor**
+(Three Pages)
+</para>
+
+<para>
+
+***START**THE SMALL PRINT!**FOR PUBLIC DOMAIN ETEXTS**START***
+Why is this "Small Print!" statement here?  You know: lawyers.
+They tell us you might sue us if there is something wrong with
+your copy of this etext, even if you got it for free from
+someone other than us, and even if what's wrong is not our
+fault.  So, among other things, this "Small Print!" statement
+disclaims most of our liability to you.  It also tells you how
+you can distribute copies of this etext if you want to.
+</para>
+
+<para>
+*BEFORE!* YOU USE OR READ THIS ETEXT
+By using or reading any part of this PROJECT GUTENBERG-tm
+etext, you indicate that you understand, agree to and accept
+this "Small Print!" statement.  If you do not, you can receive
+a refund of the money (if any) you paid for this etext by
+sending a request within 30 days of receiving it to the person
+you got it from.  If you received this etext on a physical
+medium (such as a disk), you must return it with your request.
+</para>
+
+<para>
+ABOUT PROJECT GUTENBERG-TM ETEXTS
+This PROJECT GUTENBERG-tm etext, like most PROJECT GUTENBERG-
+tm etexts, is a "public domain" work distributed by Professor
+Michael S. Hart through the Project Gutenberg Association at
+Carnegie-Mellon University (the "Project").  Among other
+things, this means that no one owns a United States copyright
+on or for this work, so the Project (and you!) can copy and
+distribute it in the United States without permission and
+without paying copyright royalties.  Special rules, set forth
+below, apply if you wish to copy and distribute this etext
+under the Project's "PROJECT GUTENBERG" trademark.
+</para>
+
+<para>
+To create these etexts, the Project expends considerable
+efforts to identify, transcribe and proofread public domain
+works.  Despite these efforts, the Project's etexts and any
+medium they may be on may contain "Defects".  Among other
+things, Defects may take the form of incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other
+intellectual property infringement, a defective or damaged
+disk or other etext medium, a computer virus, or computer
+codes that damage or cannot be read by your equipment.
+</para>
+
+<para>
+LIMITED WARRANTY; DISCLAIMER OF DAMAGES
+But for the "Right of Replacement or Refund" described below,
+[1] the Project (and any other party you may receive this
+etext from as a PROJECT GUTENBERG-tm etext) disclaims all
+liability to you for damages, costs and expenses, including
+legal fees, and [2] YOU HAVE NO REMEDIES FOR NEGLIGENCE OR
+UNDER STRICT LIABILITY, OR FOR BREACH OF WARRANTY OR CONTRACT,
+INCLUDING BUT NOT LIMITED TO INDIRECT, CONSEQUENTIAL, PUNITIVE
+OR INCIDENTAL DAMAGES, EVEN IF YOU GIVE NOTICE OF THE
+POSSIBILITY OF SUCH DAMAGES.
+</para>
+
+<para>
+If you discover a Defect in this etext within 90 days of
+receiving it, you can receive a refund of the money (if any)
+you paid for it by sending an explanatory note within that
+time to the person you received it from.  If you received it
+on a physical medium, you must return it with your note, and
+such person may choose to alternatively give you a replacement
+copy.  If you received it electronically, such person may
+choose to alternatively give you a second opportunity to
+receive it electronically.
+</para>
+
+<para>
+THIS ETEXT IS OTHERWISE PROVIDED TO YOU "AS-IS".  NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, ARE MADE TO YOU AS
+TO THE ETEXT OR ANY MEDIUM IT MAY BE ON, INCLUDING BUT NOT
+LIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
+PARTICULAR PURPOSE.
+</para>
+
+<para>
+Some states do not allow disclaimers of implied warranties or
+the exclusion or limitation of consequential damages, so the
+above disclaimers and exclusions may not apply to you, and you
+may have other legal rights.
+</para>
+
+<para>
+INDEMNITY
+You will indemnify and hold the Project, its directors,
+officers, members and agents harmless from all liability, cost
+and expense, including legal fees, that arise directly or
+indirectly from any of the following that you do or cause:
+[1] distribution of this etext, [2] alteration, modification,
+or addition to the etext, or [3] any Defect.
+</para>
+
+<para>
+DISTRIBUTION UNDER "PROJECT GUTENBERG-tm"
+You may distribute copies of this etext electronically, or by
+disk, book or any other medium if you either delete this
+"Small Print!" and all other references to Project Gutenberg,
+or:
+</para>
+
+<para>
+[1]  Only give exact copies of it.  Among other things, this
+     requires that you do not remove, alter or modify the
+     etext or this "small print!" statement.  You may however,
+     if you wish, distribute this etext in machine readable
+     binary, compressed, mark-up, or proprietary form,
+     including any form resulting from conversion by word pro-
+     cessing or hypertext software, but only so long as
+     *EITHER*:
+</para>
+
+<para>
+     [*]  The etext, when displayed, is clearly readable, and
+          does *not* contain characters other than those
+          intended by the author of the work, although tilde
+          (~), asterisk (*) and underline (_) characters may
+          be used to convey punctuation intended by the
+          author, and additional characters may be used to
+          indicate hypertext links; OR
+</para>
+
+<para>
+     [*]  The etext may be readily converted by the reader at
+          no expense into plain ASCII, EBCDIC or equivalent
+          form by the program that displays the etext (as is
+          the case, for instance, with most word processors);
+          OR
+</para>
+
+<para>
+     [*]  You provide, or agree to also provide on request at
+          no additional cost, fee or expense, a copy of the
+          etext in its original plain ASCII form (or in EBCDIC
+          or other equivalent proprietary form).
+</para>
+
+<para>
+[2]  Honor the etext refund and replacement provisions of this
+     "Small Print!" statement.
+</para>
+
+<para>
+[3]  Pay a trademark license fee to the Project of 20% of the
+     net profits you derive calculated using the method you
+     already use to calculate your applicable taxes.  If you
+     don't derive profits, no royalty is due.  Royalties are
+     payable to "Project Gutenberg Association/Carnegie-Mellon
+     University" within the 60 days following each
+     date you prepare (or were legally required to prepare)
+     your annual (or equivalent periodic) tax return.
+</para>
+
+<para>
+WHAT IF YOU *WANT* TO SEND MONEY EVEN IF YOU DON'T HAVE TO?
+The Project gratefully accepts contributions in money, time,
+scanning machines, OCR software, public domain etexts, royalty
+free copyright licenses, and every other sort of contribution
+you can think of.  Money should be paid to "Project Gutenberg
+Association / Carnegie-Mellon University".
+</para>
+
+<para>
+*END*THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS*Ver.04.29.93*END*
+</para>
+
+
+
+<para>
+Tales of the Fish Patrol by Jack London
+Scanned and proofed by David Price
+ccx074@coventry.ac.uk
+</para>
+</gutblurb>
+
+<markupblurb>
+Original markup by Frank Boumphrey.  2/4/2000
+Validated against gutbook1.dtd using MSXML parser
+</markupblurb>
+
+
+<book>
+<acknowledge>A production of Project Gutenberg and the HTML Writers Guild. E-text by David Price. Markup by Frank Boumphrey</acknowledge>
+<frontmatter>
+<titlepage>
+<title>
+Tales of the Fish Patrol
+</title>
+<author>Jack London</author>
+</titlepage>
+</frontmatter>
+
+<bookbody>
+<chapter>
+<title>
+WHITE AND YELLOW</title>
+
+
+<para>
+San Francisco Bay is so large that often its storms are more
+disastrous to ocean-going craft than is the ocean itself in its
+violent moments.  The waters of the bay contain all manner of fish,
+wherefore its surface is ploughed by the keels of all manner of
+fishing boats manned by all manner of fishermen.  To protect the
+fish from this motley floating population many wise laws have been
+passed, and there is a fish patrol to see that these laws are
+enforced.  Exciting times are the lot of the fish patrol:  in its
+history more than one dead patrolman has marked defeat, and more
+often dead fishermen across their illegal nets have marked success.
+</para>
+
+<para>
+Wildest among the fisher-folk may be accounted the Chinese shrimp-
+catchers.  It is the habit of the shrimp to crawl along the bottom
+in vast armies till it reaches fresh water, when it turns about and
+crawls back again to the salt.  And where the tide ebbs and flows,
+the Chinese sink great bag-nets to the bottom, with gaping mouths,
+into which the shrimp crawls and from which it is transferred to
+the boiling-pot.  This in itself would not be bad, were it not for
+the small mesh of the nets, so small that the tiniest fishes,
+little new-hatched things not a quarter of an inch long, cannot
+pass through.  The beautiful beaches of Points Pedro and Pablo,
+where are the shrimp-catchers' villages, are made fearful by the
+stench from myriads of decaying fish, and against this wasteful
+destruction it has ever been the duty of the fish patrol to act.
+</para>
+
+<para>
+When I was a youngster of sixteen, a good sloop-sailor and all-
+round bay-waterman, my sloop, the Reindeer, was chartered by the
+Fish Commission, and I became for the time being a deputy
+patrolman.  After a deal of work among the Greek fishermen of the
+Upper Bay and rivers, where knives flashed at the beginning of
+trouble and men permitted themselves to be made prisoners only
+after a revolver was thrust in their faces, we hailed with delight
+an expedition to the Lower Bay against the Chinese shrimp-catchers.
+</para>
+
+<para>
+There were six of us, in two boats, and to avoid suspicion we ran
+down after dark and dropped anchor under a projecting bluff of land
+known as Point Pinole.  As the east paled with the first light of
+dawn we got under way again, and hauled close on the land breeze as
+we slanted across the bay toward Point Pedro.  The morning mists
+curled and clung to the water so that we could see nothing, but we
+busied ourselves driving the chill from our bodies with hot coffee.
+Also we had to devote ourselves to the miserable task of bailing,
+for in some incomprehensible way the Reindeer had sprung a generous
+leak.  Half the night had been spent in overhauling the ballast and
+exploring the seams, but the labor had been without avail.  The
+water still poured in, and perforce we doubled up in the cockpit
+and tossed it out again.
+</para>
+
+<para>
+After coffee, three of the men withdrew to the other boat, a
+Columbia River salmon boat, leaving three of us in the Reindeer.
+Then the two craft proceeded in company till the sun showed over
+the eastern sky-line.  Its fiery rays dispelled the clinging
+vapors, and there, before our eyes, like a picture, lay the shrimp
+fleet, spread out in a great half-moon, the tips of the crescent
+fully three miles apart, and each junk moored fast to the buoy of a
+shrimp-net.  But there was no stir, no sign of life.
+</para>
+
+<para>
+The situation dawned upon us.  While waiting for slack water, in
+which to lift their heavy nets from the bed of the bay, the Chinese
+had all gone to sleep below.  We were elated, and our plan of
+battle was swiftly formed.
+</para>
+
+<para>
+"Throw each of your two men on to a junk," whispered Le Grant to me
+from the salmon boat.  "And you make fast to a third yourself.
+We'll do the same, and there's no reason in the world why we
+shouldn't capture six junks at the least."
+</para>
+
+<para>
+Then we separated.  I put the Reindeer about on the other tack, ran
+up under the lee of a junk, shivered the mainsail into the wind and
+lost headway, and forged past the stern of the junk so slowly and
+so near that one of the patrolmen stepped lightly aboard.  Then I
+kept off, filled the mainsail, and bore away for a second junk.
+</para>
+
+<para>
+Up to this time there had been no noise, but from the first junk
+captured by the salmon boat an uproar now broke forth.  There was
+shrill Oriental yelling, a pistol shot, and more yelling.
+</para>
+
+<para>
+"It's all up.  They're warning the others," said George, the
+remaining patrolman, as he stood beside me in the cockpit.
+</para>
+
+<para>
+By this time we were in the thick of the fleet, and the alarm was
+spreading with incredible swiftness.  The decks were beginning to
+swarm with half-awakened and half-naked Chinese.  Cries and yells
+of warning and anger were flying over the quiet water, and
+somewhere a conch shell was being blown with great success.  To the
+right of us I saw the captain of a junk chop away his mooring line
+with an axe and spring to help his crew at the hoisting of the
+huge, outlandish lug-sail.  But to the left the first heads were
+popping up from below on another junk, and I rounded up the
+Reindeer alongside long enough for George to spring aboard.
+</para>
+
+<para>
+The whole fleet was now under way.  In addition to the sails they
+had gotten out long sweeps, and the bay was being ploughed in every
+direction by the fleeing junks.  I was now alone in the Reindeer,
+seeking feverishly to capture a third prize.  The first junk I took
+after was a clean miss, for it trimmed its sheets and shot away
+surprisingly into the wind.  By fully half a point it outpointed
+the Reindeer, and I began to feel respect for the clumsy craft.
+Realizing the hopelessness of the pursuit, I filled away, threw out
+the main-sheet, and drove down before the wind upon the junks to
+leeward, where I had them at a disadvantage.
+</para>
+
+<para>
+The one I had selected wavered indecisively before me, and, as I
+swung wide to make the boarding gentle, filled suddenly and darted
+away, the smart Mongols shouting a wild rhythm as they bent to the
+sweeps.  But I had been ready for this.  I luffed suddenly.
+Putting the tiller hard down, and holding it down with my body, I
+brought the main-sheet in, hand over hand, on the run, so as to
+retain all possible striking force.  The two starboard sweeps of
+the junk were crumpled up, and then the two boats came together
+with a crash.  The Reindeer's bowsprit, like a monstrous hand,
+reached over and ripped out the junk's chunky mast and towering
+sail.
+</para>
+
+<para>
+This was met by a curdling yell of rage.  A big Chinaman,
+remarkably evil-looking, with his head swathed in a yellow silk
+handkerchief and face badly pock-marked, planted a pike-pole on the
+Reindeer's bow and began to shove the entangled boats apart.
+Pausing long enough to let go the jib halyards, and just as the
+Reindeer cleared and began to drift astern, I leaped aboard the
+junk with a line and made fast.  He of the yellow handkerchief and
+pock-marked face came toward me threateningly, but I put my hand
+into my hip pocket, and he hesitated.  I was unarmed, but the
+Chinese have learned to be fastidiously careful of American hip
+pockets, and it was upon this that I depended to keep him and his
+savage crew at a distance.
+</para>
+
+<para>
+I ordered him to drop the anchor at the junk's bow, to which he
+replied, "No sabbe."  The crew responded in like fashion, and
+though I made my meaning plain by signs, they refused to
+understand.  Realizing the inexpediency of discussing the matter, I
+went forward myself, overran the line, and let the anchor go.
+</para>
+
+<para>
+"Now get aboard, four of you," I said in a loud voice, indicating
+with my fingers that four of them were to go with me and the fifth
+was to remain by the junk.  The Yellow Handkerchief hesitated; but
+I repeated the order fiercely (much more fiercely than I felt), at
+the same time sending my hand to my hip.  Again the Yellow
+Handkerchief was overawed, and with surly looks he led three of his
+men aboard the Reindeer.  I cast off at once, and, leaving the jib
+down, steered a course for George's junk.  Here it was easier, for
+there were two of us, and George had a pistol to fall back on if it
+came to the worst.  And here, as with my junk, four Chinese were
+transferred to the sloop and one left behind to take care of
+things.
+</para>
+
+<para>
+Four more were added to our passenger list from the third junk.  By
+this time the salmon boat had collected its twelve prisoners and
+came alongside, badly overloaded.  To make matters worse, as it was
+a small boat, the patrolmen were so jammed in with their prisoners
+that they would have little chance in case of trouble.
+</para>
+
+<para>
+"You'll have to help us out," said Le Grant.
+</para>
+
+<para>
+I looked over my prisoners, who had crowded into the cabin and on
+top of it.  "I can take three," I answered.
+</para>
+
+<para>
+"Make it four," he suggested, "and I'll take Bill with me."  (Bill
+was the third patrolman.)  "We haven't elbow room here, and in case
+of a scuffle one white to every two of them will be just about the
+right proportion."
+</para>
+
+<para>
+The exchange was made, and the salmon boat got up its spritsail and
+headed down the bay toward the marshes off San Rafael.  I ran up
+the jib and followed with the Reindeer.  San Rafael, where we were
+to turn our catch over to the authorities, communicated with the
+bay by way of a long and tortuous slough, or marshland creek, which
+could be navigated only when the tide was in.  Slack water had
+come, and, as the ebb was commencing, there was need for hurry if
+we cared to escape waiting half a day for the next tide.
+</para>
+
+<para>
+But the land breeze had begun to die away with the rising sun, and
+now came only in failing puffs.  The salmon boat got out its oars
+and soon left us far astern.  Some of the Chinese stood in the
+forward part of the cockpit, near the cabin doors, and once, as I
+leaned over the cockpit rail to flatten down the jib-sheet a bit, I
+felt some one brush against my hip pocket.  I made no sign, but out
+of the corner of my eye I saw that the Yellow Handkerchief had
+discovered the emptiness of the pocket which had hitherto overawed
+him.
+</para>
+
+<para>
+To make matters serious, during all the excitement of boarding the
+junks the Reindeer had not been bailed, and the water was beginning
+to slush over the cockpit floor.  The shrimp-catchers pointed at it
+and looked to me questioningly.
+</para>
+
+<para>
+"Yes," I said.  "Bime by, allee same dlown, velly quick, you no
+bail now.  Sabbe?"
+</para>
+
+<para>
+No, they did not "sabbe," or at least they shook their heads to
+that effect, though they chattered most comprehendingly to one
+another in their own lingo.  I pulled up three or four of the
+bottom boards, got a couple of buckets from a locker, and by
+unmistakable sign-language invited them to fall to.  But they
+laughed, and some crowded into the cabin and some climbed up on
+top.
+</para>
+
+<para>
+Their laughter was not good laughter.  There was a hint of menace
+in it, a maliciousness which their black looks verified.  The
+Yellow Handkerchief, since his discovery of my empty pocket, had
+become most insolent in his bearing, and he wormed about among the
+other prisoners, talking to them with great earnestness.
+</para>
+
+<para>
+Swallowing my chagrin, I stepped down into the cockpit and began
+throwing out the water.  But hardly had I begun, when the boom
+swung overhead, the mainsail filled with a jerk, and the Reindeer
+heeled over.  The day wind was springing up.  George was the
+veriest of landlubbers, so I was forced to give over bailing and
+take the tiller.  The wind was blowing directly off Point Pedro and
+the high mountains behind, and because of this was squally and
+uncertain, half the time bellying the canvas out and the other half
+flapping it idly.
+</para>
+
+<para>
+George was about the most all-round helpless man I had ever met.
+Among his other disabilities, he was a consumptive, and I knew that
+if he attempted to bail, it might bring on a hemorrhage.  Yet the
+rising water warned me that something must be done.  Again I
+ordered the shrimp-catchers to lend a hand with the buckets.  They
+laughed defiantly, and those inside the cabin, the water up to
+their ankles, shouted back and forth with those on top.
+</para>
+
+<para>
+"You'd better get out your gun and make them bail," I said to
+George.
+</para>
+
+<para>
+But he shook his head and showed all too plainly that he was
+afraid.  The Chinese could see the funk he was in as well as I
+could, and their insolence became insufferable.  Those in the cabin
+broke into the food lockers, and those above scrambled down and
+joined them in a feast on our crackers and canned goods.
+</para>
+
+<para>
+"What do we care?" George said weakly.
+</para>
+
+<para>
+I was fuming with helpless anger.  "If they get out of hand, it
+will be too late to care.  The best thing you can do is to get them
+in check right now."
+</para>
+
+<para>
+The water was rising higher and higher, and the gusts, forerunners
+of a steady breeze, were growing stiffer and stiffer.  And between
+the gusts, the prisoners, having gotten away with a week's grub,
+took to crowding first to one side and then to the other till the
+Reindeer rocked like a cockle-shell.  Yellow Handkerchief
+approached me, and, pointing out his village on the Point Pedro
+beach, gave me to understand that if I turned the Reindeer in that
+direction and put them ashore, they, in turn, would go to bailing.
+By now the water in the cabin was up to the bunks, and the bed-
+clothes were sopping.  It was a foot deep on the cockpit floor.
+Nevertheless I refused, and I could see by George's face that he
+was disappointed.
+</para>
+
+<para>
+"If you don't show some nerve, they'll rush us and throw us
+overboard," I said to him.  "Better give me your revolver, if you
+want to be safe."
+</para>
+
+<para>
+"The safest thing to do," he chattered cravenly, "is to put them
+ashore.  I, for one, don't want to be drowned for the sake of a
+handful of dirty Chinamen."
+</para>
+
+<para>
+"And I, for another, don't care to give in to a handful of dirty
+Chinamen to escape drowning," I answered hotly.
+</para>
+
+<para>
+"You'll sink the Reindeer under us all at this rate," he whined.
+"And what good that'll do I can't see."
+</para>
+
+<para>
+"Every man to his taste," I retorted.
+</para>
+
+<para>
+He made no reply, but I could see he was trembling pitifully.
+Between the threatening Chinese and the rising water he was beside
+himself with fright; and, more than the Chinese and the water, I
+feared him and what his fright might impel him to do.  I could see
+him casting longing glances at the small skiff towing astern, so in
+the next calm I hauled the skiff alongside.  As I did so his eyes
+brightened with hope; but before he could guess my intention, I
+stove the frail bottom through with a hand-axe, and the skiff
+filled to its gunwales.
+</para>
+
+<para>
+"It's sink or float together," I said.  "And if you'll give me your
+revolver, I'll have the Reindeer bailed out in a jiffy."
+</para>
+
+<para>
+"They're too many for us," he whimpered.  "We can't fight them
+all."
+</para>
+
+<para>
+I turned my back on him in disgust.  The salmon boat had long since
+passed from sight behind a little archipelago known as the Marin
+Islands, so no help could be looked for from that quarter.  Yellow
+Handkerchief came up to me in a familiar manner, the water in the
+cockpit slushing against his legs.  I did not like his looks.  I
+felt that beneath the pleasant smile he was trying to put on his
+face there was an ill purpose.  I ordered him back, and so sharply
+that he obeyed.
+</para>
+
+<para>
+"Now keep your distance," I commanded, "and don't you come closer!"
+</para>
+
+<para>
+"Wha' fo'?" he demanded indignantly.  "I t'ink-um talkee talkee
+heap good."
+</para>
+
+<para>
+"Talkee talkee," I answered bitterly, for I knew now that he had
+understood all that passed between George and me.  "What for talkee
+talkee?  You no sabbe talkee talkee."
+</para>
+
+<para>
+He grinned in a sickly fashion.  "Yep, I sabbe velly much.  I
+honest Chinaman."
+</para>
+
+<para>
+"All right," I answered.  "You sabbe talkee talkee, then you bail
+water plenty plenty.  After that we talkee talkee."
+</para>
+
+<para>
+He shook his head, at the same time pointing over his shoulder to
+his comrades.  "No can do.  Velly bad Chinamen, heap velly bad.  I
+t'ink-um - "
+</para>
+
+<para>
+"Stand back!" I shouted, for I had noticed his hand disappear
+beneath his blouse and his body prepare for a spring.
+</para>
+
+<para>
+Disconcerted, he went back into the cabin, to hold a council,
+apparently, from the way the jabbering broke forth.  The Reindeer
+was very deep in the water, and her movements had grown quite
+loggy.  In a rough sea she would have inevitably swamped; but the
+wind, when it did blow, was off the land, and scarcely a ripple
+disturbed the surface of the bay.
+</para>
+
+<para>
+"I think you'd better head for the beach," George said abruptly, in
+a manner that told me his fear had forced him to make up his mind
+to some course of action.
+</para>
+
+<para>
+"I think not," I answered shortly.
+</para>
+
+<para>
+"I command you," he said in a bullying tone.
+</para>
+
+<para>
+"I was commanded to bring these prisoners into San Rafael," was my
+reply.
+</para>
+
+<para>
+Our voices were raised, and the sound of the altercation brought
+the Chinese out of the cabin.
+</para>
+
+<para>
+"Now will you head for the beach?"
+</para>
+
+<para>
+This from George, and I found myself looking into the muzzle of his
+revolver - of the revolver he dared to use on me, but was too
+cowardly to use on the prisoners.
+</para>
+
+<para>
+My brain seemed smitten with a dazzling brightness.  The whole
+situation, in all its bearings, was focussed sharply before me -
+the shame of losing the prisoners, the worthlessness and cowardice
+of George, the meeting with Le Grant and the other patrol men and
+the lame explanation; and then there was the fight I had fought so
+hard, victory wrenched from me just as I thought I had it within my
+grasp.  And out of the tail of my eye I could see the Chinese
+crowding together by the cabin doors and leering triumphantly.  It
+would never do.
+</para>
+
+<para>
+I threw my hand up and my head down.  The first act elevated the
+muzzle, and the second removed my head from the path of the bullet
+which went whistling past.  One hand closed on George's wrist, the
+other on the revolver.  Yellow Handkerchief and his gang sprang
+toward me.  It was now or never.  Putting all my strength into a
+sudden effort, I swung George's body forward to meet them.  Then I
+pulled back with equal suddenness, ripping the revolver out of his
+fingers and jerking him off his feet.  He fell against Yellow
+Handkerchief's knees, who stumbled over him, and the pair wallowed
+in the bailing hole where the cockpit floor was torn open.  The
+next instant I was covering them with my revolver, and the wild
+shrimp-catchers were cowering and cringing away.
+</para>
+
+<para>
+But I swiftly discovered that there was all the difference in the
+world between shooting men who are attacking and men who are doing
+nothing more than simply refusing to obey.  For obey they would not
+when I ordered them into the bailing hole.  I threatened them with
+the revolver, but they sat stolidly in the flooded cabin and on the
+roof and would not move.
+</para>
+
+<para>
+Fifteen minutes passed, the Reindeer sinking deeper and deeper, her
+mainsail flapping in the calm.  But from off the Point Pedro shore
+I saw a dark line form on the water and travel toward us.  It was
+the steady breeze I had been expecting so long.  I called to the
+Chinese and pointed it out.  They hailed it with exclamations.
+Then I pointed to the sail and to the water in the Reindeer, and
+indicated by signs that when the wind reached the sail, what of the
+water aboard we would capsize.  But they jeered defiantly, for they
+knew it was in my power to luff the helm and let go the main-sheet,
+so as to spill the wind and escape damage.
+</para>
+
+<para>
+But my mind was made up.  I hauled in the main-sheet a foot or two,
+took a turn with it, and bracing my feet, put my back against the
+tiller.  This left me one hand for the sheet and one for the
+revolver.  The dark line drew nearer, and I could see them looking
+from me to it and back again with an apprehension they could not
+successfully conceal.  My brain and will and endurance were pitted
+against theirs, and the problem was which could stand the strain of
+imminent death the longer and not give in.
+</para>
+
+<para>
+Then the wind struck us.  The main-sheet tautened with a brisk
+rattling of the blocks, the boom uplifted, the sail bellied out,
+and the Reindeer heeled over - over, and over, till the lee-rail
+went under, the cabin windows went under, and the bay began to pour
+in over the cockpit rail.  So violently had she heeled over, that
+the men in the cabin had been thrown on top of one another into the
+lee bunk, where they squirmed and twisted and were washed about,
+those underneath being perilously near to drowning.
+</para>
+
+<para>
+The wind freshened a bit, and the Reindeer went over farther than
+ever.  For the moment I thought she was gone, and I knew that
+another puff like that and she surely would go.  While I pressed
+her under and debated whether I should give up or not, the Chinese
+cried for mercy.  I think it was the sweetest sound I have ever
+heard.  And then, and not until then, did I luff up and ease out
+the main-sheet.  The Reindeer righted very slowly, and when she was
+on an even keel was so much awash that I doubted if she could be
+saved.
+</para>
+
+<para>
+But the Chinese scrambled madly into the cockpit and fell to
+bailing with buckets, pots, pans, and everything they could lay
+hands on.  It was a beautiful sight to see that water flying over
+the side!  And when the Reindeer was high and proud on the water
+once more, we dashed away with the breeze on our quarter, and at
+the last possible moment crossed the mud flats and entered the
+slough.
+</para>
+
+<para>
+The spirit of the Chinese was broken, and so docile did they become
+that ere we made San Rafael they were out with the tow-rope, Yellow
+Handkerchief at the head of the line.  As for George, it was his
+last trip with the fish patrol.  He did not care for that sort of
+thing, he explained, and he thought a clerkship ashore was good
+enough for him.  And we thought so too.
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>THE KING OF THE GREEKS</title>
+
+
+<para>
+Big Alec had never been captured by the fish patrol.  It was his
+boast that no man could take him alive, and it was his history that
+of the many men who had tried to take him dead none had succeeded.
+It was also history that at least two patrolmen who had tried to
+take him dead had died themselves.  Further, no man violated the
+fish laws more systematically and deliberately than Big Alec.
+</para>
+
+<para>
+He was called "Big Alec" because of his gigantic stature.  His
+height was six feet three inches, and he was correspondingly broad-
+shouldered and deep-chested.  He was splendidly muscled and hard as
+steel, and there were innumerable stories in circulation among the
+fisher-folk concerning his prodigious strength.  He was as bold and
+dominant of spirit as he was strong of body, and because of this he
+was widely known by another name, that of "The King of the Greeks."
+The fishing population was largely composed of Greeks, and they
+looked up to him and obeyed him as their chief.  And as their
+chief, he fought their fights for them, saw that they were
+protected, saved them from the law when they fell into its
+clutches, and made them stand by one another and himself in time of
+trouble.
+</para>
+
+<para>
+In the old days, the fish patrol had attempted his capture many
+disastrous times and had finally given it over, so that when the
+word was out that he was coming to Benicia, I was most anxious to
+see him.  But I did not have to hunt him up.  In his usual bold
+way, the first thing he did on arriving was to hunt us up.  Charley
+Le Grant and I at the time were under a patrol-man named Carmintel,
+and the three of us were on the Reindeer, preparing for a trip,
+when Big Alec stepped aboard.  Carmintel evidently knew him, for
+they shook hands in recognition.  Big Alec took no notice of
+Charley or me.
+</para>
+
+<para>
+"I've come down to fish sturgeon a couple of months," he said to
+Carmintel.
+</para>
+
+<para>
+His eyes flashed with challenge as he spoke, and we noticed the
+patrolman's eyes drop before him.
+</para>
+
+<para>
+"That's all right, Alec," Carmintel said in a low voice.  "I'll not
+bother you.  Come on into the cabin, and we'll talk things over,"
+he added.
+</para>
+
+<para>
+When they had gone inside and shut the doors after them, Charley
+winked with slow deliberation at me.  But I was only a youngster,
+and new to men and the ways of some men, so I did not understand.
+Nor did Charley explain, though I felt there was something wrong
+about the business.
+</para>
+
+<para>
+Leaving them to their conference, at Charley's suggestion we
+boarded our skiff and pulled over to the Old Steamboat Wharf, where
+Big Alec's ark was lying.  An ark is a house-boat of small though
+comfortable dimensions, and is as necessary to the Upper Bay
+fisherman as are nets and boats.  We were both curious to see Big
+Alec's ark, for history said that it had been the scene of more
+than one pitched battle, and that it was riddled with bullet-holes.
+</para>
+
+<para>
+We found the holes (stopped with wooden plugs and painted over),
+but there were not so many as I had expected.  Charley noted my
+look of disappointment, and laughed; and then to comfort me he gave
+an authentic account of one expedition which had descended upon Big
+Alec's floating home to capture him, alive preferably, dead if
+necessary.  At the end of half a day's fighting, the patrolmen had
+drawn off in wrecked boats, with one of their number killed and
+three wounded.  And when they returned next morning with
+reinforcements they found only the mooring-stakes of Big Alec's
+ark; the ark itself remained hidden for months in the fastnesses of
+the Suisun tules.
+</para>
+
+<para>
+"But why was he not hanged for murder?" I demanded.  "Surely the
+United States is powerful enough to bring such a man to justice."
+</para>
+
+<para>
+"He gave himself up and stood trial," Charley answered.  "It cost
+him fifty thousand dollars to win the case, which he did on
+technicalities and with the aid of the best lawyers in the state.
+Every Greek fisherman on the river contributed to the sum.  Big
+Alec levied and collected the tax, for all the world like a king.
+The United States may be all-powerful, my lad, but the fact remains
+that Big Alec is a king inside the United States, with a country
+and subjects all his own."
+</para>
+
+<para>
+"But what are you going to do about his fishing for sturgeon?  He's
+bound to fish with a 'Chinese line.'"
+</para>
+
+<para>
+Charley shrugged his shoulders.  "We'll see what we will see," he
+said enigmatically.
+</para>
+
+<para>
+Now a "Chinese line" is a cunning device invented by the people
+whose name it bears.  By a simple system of floats, weights, and
+anchors, thousands of hooks, each on a separate leader, are
+suspended at a distance of from six inches to a foot above the
+bottom.  The remarkable thing about such a line is the hook.  It is
+barbless, and in place of the barb, the hook is filed long and
+tapering to a point as sharp as that of a needle.  These hoods are
+only a few inches apart, and when several thousand of them are
+suspended just above the bottom, like a fringe, for a couple of
+hundred fathoms, they present a formidable obstacle to the fish
+that travel along the bottom.
+</para>
+
+<para>
+Such a fish is the sturgeon, which goes rooting along like a pig,
+and indeed is often called "pig-fish."  Pricked by the first hook
+it touches, the sturgeon gives a startled leap and comes into
+contact with half a dozen more hooks.  Then it threshes about
+wildly, until it receives hook after hook in its soft flesh; and
+the hooks, straining from many different angles, hold the luckless
+fish fast until it is drowned.  Because no sturgeon can pass
+through a Chinese line, the device is called a trap in the fish
+laws; and because it bids fair to exterminate the sturgeon, it is
+branded by the fish laws as illegal.  And such a line, we were
+confident, Big Alec intended setting, in open and flagrant
+violation of the law.
+</para>
+
+<para>
+Several days passed after the visit of Big Alec, during which
+Charley and I kept a sharp watch on him.  He towed his ark around
+the Solano Wharf and into the big bight at Turner's Shipyard.  The
+bight we knew to be good ground for sturgeon, and there we felt
+sure the King of the Greeks intended to begin operations.  The tide
+circled like a mill-race in and out of this bight, and made it
+possible to raise, lower, or set a Chinese line only at slack
+water.  So between the tides Charley and I made it a point for one
+or the other of us to keep a lookout from the Solano Wharf.
+</para>
+
+<para>
+On the fourth day I was lying in the sun behind the stringer-piece
+of the wharf, when I saw a skiff leave the distant shore and pull
+out into the bight.  In an instant the glasses were at my eyes and
+I was following every movement of the skiff.  There were two men in
+it, and though it was a good mile away, I made out one of them to
+be Big Alec; and ere the skiff returned to shore I made out enough
+more to know that the Greek had set his line.
+</para>
+
+<para>
+"Big Alec has a Chinese line out in the bight off Turner's
+Shipyard," Charley Le Grant said that afternoon to Carmintel.
+</para>
+
+<para>
+A fleeting expression of annoyance passed over the patrolman's
+face, and then he said, "Yes?" in an absent way, and that was all.
+</para>
+
+<para>
+Charley bit his lip with suppressed anger and turned on his heel.
+</para>
+
+<para>
+"Are you game, my lad?" he said to me later on in the evening, just
+as we finished washing down the Reindeer's decks and were preparing
+to turn in.
+</para>
+
+<para>
+A lump came up in my throat, and I could only nod my head.
+</para>
+
+<para>
+"Well, then," and Charley's eyes glittered in a determined way,
+"we've got to capture Big Alec between us, you and I, and we've got
+to do it in spite of Carmintel.  Will you lend a hand?"
+</para>
+
+<para>
+"It's a hard proposition, but we can do it," he added after a
+pause.
+</para>
+
+<para>
+"Of course we can," I supplemented enthusiastically.
+</para>
+
+<para>
+And then he said, "Of course we can," and we shook hands on it and
+went to bed.
+</para>
+
+<para>
+But it was no easy task we had set ourselves.  In order to convict
+a man of illegal fishing, it was necessary to catch him in the act
+with all the evidence of the crime about him - the hooks, the
+lines, the fish, and the man himself.  This meant that we must take
+Big Alec on the open water, where he could see us coming and
+prepare for us one of the warm receptions for which he was noted.
+</para>
+
+<para>
+"There's no getting around it," Charley said one morning.  "If we
+can only get alongside it's an even toss, and there's nothing left
+for us but to try and get alongside.  Come on, lad."
+</para>
+
+<para>
+We were in the Columbia River salmon boat, the one we had used
+against the Chinese shrimp-catchers.  Slack water had come, and as
+we dropped around the end of the Solano Wharf we saw Big Alec at
+work, running his line and removing the fish.
+</para>
+
+<para>
+"Change places," Charley commanded, "and steer just astern of him
+as though you're going into the shipyard."
+</para>
+
+<para>
+I took the tiller, and Charley sat down on a thwart amidships,
+placing his revolver handily beside him.
+</para>
+
+<para>
+"If he begins to shoot," he cautioned, "get down in the bottom and
+steer from there, so that nothing more than your hand will be
+exposed."
+</para>
+
+<para>
+I nodded, and we kept silent after that, the boat slipping gently
+through the water and Big Alec growing nearer and nearer.  We could
+see him quite plainly, gaffing the sturgeon and throwing them into
+the boat while his companion ran the line and cleared the hooks as
+he dropped them back into the water.  Nevertheless, we were five
+hundred yards away when the big fisherman hailed us.
+</para>
+
+<para>
+"Here!  You!  What do you want?" he shouted.
+</para>
+
+<para>
+"Keep going," Charley whispered, "just as though you didn't hear
+him."
+</para>
+
+<para>
+The next few moments were very anxious ones.  The fisherman was
+studying us sharply, while we were gliding up on him every second.
+</para>
+
+<para>
+"You keep off if you know what's good for you!" he called out
+suddenly, as though he had made up his mind as to who and what we
+were.  "If you don't, I'll fix you!"
+</para>
+
+<para>
+He brought a rifle to his shoulder and trained it on me.
+</para>
+
+<para>
+"Now will you keep off?" he demanded.
+</para>
+
+<para>
+I could hear Charley groan with disappointment.  "Keep off," he
+whispered; "it's all up for this time."
+</para>
+
+<para>
+I put up the tiller and eased the sheet, and the salmon boat ran
+off five or six points.  Big Alec watched us till we were out of
+range, when he returned to his work.
+</para>
+
+<para>
+"You'd better leave Big Alec alone," Carmintel said, rather sourly,
+to Charley that night.
+</para>
+
+<para>
+"So he's been complaining to you, has he?"  Charley said
+significantly.
+</para>
+
+<para>
+Carmintel flushed painfully.  "You'd better leave him alone, I tell
+you," he repeated.  "He's a dangerous man, and it won't pay to fool
+with him."
+</para>
+
+<para>
+"Yes," Charley answered softly; "I've heard that it pays better to
+leave him alone."
+</para>
+
+<para>
+This was a direct thrust at Carmintel, and we could see by the
+expression of his face that it sank home.  For it was common
+knowledge that Big Alec was as willing to bribe as to fight, and
+that of late years more than one patrolman had handled the
+fisherman's money.
+</para>
+
+<para>
+"Do you mean to say - " Carmintel began, in a bullying tone.
+</para>
+
+<para>
+But Charley cut him off shortly.  "I mean to say nothing," he said.
+"You heard what I said, and if the cap fits, why - "
+</para>
+
+<para>
+He shrugged his shoulders, and Carmintel glowered at him,
+speechless.
+</para>
+
+<para>
+"What we want is imagination," Charley said to me one day, when we
+had attempted to creep upon Big Alec in the gray of dawn and had
+been shot at for our trouble.
+</para>
+
+<para>
+And thereafter, and for many days, I cudgelled my brains trying to
+imagine some possible way by which two men, on an open stretch of
+water, could capture another who knew how to use a rifle and was
+never to be found without one.  Regularly, every slack water,
+without slyness, boldly and openly in the broad day, Big Alec was
+to be seen running his line.  And what made it particularly
+exasperating was the fact that every fisherman, from Benicia to
+Vallejo knew that he was successfully defying us.  Carmintel also
+bothered us, for he kept us busy among the shad-fishers of San
+Pablo, so that we had little time to spare on the King of the
+Greeks.  But Charley's wife and children lived at Benicia, and we
+had made the place our headquarters, so that we always returned to
+it.
+</para>
+
+<para>
+"I'll tell you what we can do," I said, after several fruitless
+weeks had passed; "we can wait some slack water till Big Alec has
+run his line and gone ashore with the fish, and then we can go out
+and capture the line.  It will put him to time and expense to make
+another, and then we'll figure to capture that too.  If we can't
+capture him, we can discourage him, you see."
+</para>
+
+<para>
+Charley saw, and said it wasn't a bad idea.  We watched our chance,
+and the next low-water slack, after Big Alec had removed the fish
+from the line and returned ashore, we went out in the salmon boat.
+We had the bearings of the line from shore marks, and we knew we
+would have no difficulty in locating it.  The first of the flood
+tide was setting in, when we ran below where we thought the line
+was stretched and dropped over a fishing-boat anchor.  Keeping a
+short rope to the anchor, so that it barely touched the bottom, we
+dragged it slowly along until it stuck and the boat fetched up hard
+and fast.
+</para>
+
+<para>
+"We've got it," Charley cried.  "Come on and lend a hand to get it
+in."
+</para>
+
+<para>
+Together we hove up the rope till the anchor I came in sight with
+the sturgeon line caught across one of the flukes.  Scores of the
+murderous-looking hooks flashed into sight as we cleared the
+anchor, and we had just started to run along the line to the end
+where we could begin to lift it, when a sharp thud in the boat
+startled us.  We looked about, but saw nothing and returned to our
+work.  An instant later there was a similar sharp thud and the
+gunwale splintered between Charley's body and mine.
+</para>
+
+<para>
+"That's remarkably like a bullet, lad," he said reflectively.  "And
+it's a long shot Big Alec's making."
+</para>
+
+<para>
+"And he's using smokeless powder," he concluded, after an
+examination of the mile-distant shore.  "That's why we can't hear
+the report."
+</para>
+
+<para>
+I looked at the shore, but could see no sign of Big Alec, who was
+undoubtedly hidden in some rocky nook with us at his mercy.  A
+third bullet struck the water, glanced, passed singing over our
+heads, and struck the water again beyond.
+</para>
+
+<para>
+"I guess we'd better get out of this," Charley remarked coolly.
+"What do you think, lad?"
+</para>
+
+<para>
+I thought so, too, and said we didn't want the line anyway.
+Whereupon we cast off and hoisted the spritsail.  The bullets
+ceased at once, and we sailed away, unpleasantly confident that Big
+Alec was laughing at our discomfiture.
+</para>
+
+<para>
+And more than that, the next day on the fishing wharf, where we
+were inspecting nets, he saw fit to laugh and sneer at us, and this
+before all the fishermen.  Charley's face went black with anger;
+but beyond promising Big Alec that in the end he would surely land
+him behind the bars, he controlled himself and said nothing.  The
+King of the Greeks made his boast that no fish patrol had ever
+taken him or ever could take him, and the fishermen cheered him and
+said it was true.  They grew excited, and it looked like trouble
+for a while; but Big Alec asserted his kingship and quelled them.
+</para>
+
+<para>
+Carmintel also laughed at Charley, and dropped sarcastic remarks,
+and made it hard for him.  But Charley refused to be angered,
+though he told me in confidence that he intended to capture Big
+Alec if it took all the rest of his life to accomplish it.
+</para>
+
+<para>
+"I don't know how I'll do it," he said, "but do it I will, as sure
+as I am Charley Le Grant.  The idea will come to me at the right
+and proper time, never fear."
+</para>
+
+<para>
+And at the right time it came, and most unexpectedly.  Fully a
+month had passed, and we were constantly up and down the river, and
+down and up the bay, with no spare moments to devote to the
+particular fisherman who ran a Chinese line in the bight of
+Turner's Shipyard.  We had called in at Selby's Smelter one
+afternoon, while on patrol work, when all unknown to us our
+opportunity happened along.  It appeared in the guise of a helpless
+yacht loaded with seasick people, so we could hardly be expected to
+recognize it as the opportunity.  It was a large sloop-yacht, and
+it was helpless inasmuch as the trade-wind was blowing half a gale
+and there were no capable sailors aboard.
+</para>
+
+<para>
+From the wharf at Selby's we watched with careless interest the
+lubberly manoeuvre performed of bringing the yacht to anchor, and
+the equally lubberly manoeuvre of sending the small boat ashore.  A
+very miserable-looking man in draggled ducks, after nearly swamping
+the boat in the heavy seas, passed us the painter and climbed out.
+He staggered about as though the wharf were rolling, and told us
+his troubles, which were the troubles of the yacht.  The only
+rough-weather sailor aboard, the man on whom they all depended, had
+been called back to San Francisco by a telegram, and they had
+attempted to continue the cruise alone.  The high wind and big seas
+of San Pablo Bay had been too much for them; all hands were sick,
+nobody knew anything or could do anything; and so they had run in
+to the smelter either to desert the yacht or to get somebody to
+bring it to Benicia.  In short, did we know of any sailors who
+would bring the yacht into Benicia?
+</para>
+
+<para>
+Charley looked at me.  The Reindeer was lying in a snug place.  We
+had nothing on hand in the way of patrol work till midnight.  With
+the wind then blowing, we could sail the yacht into Benicia in a
+couple of hours, have several more hours ashore, and come back to
+the smelter on the evening train.
+</para>
+
+<para>
+"All right, captain," Charley said to the disconsolate yachtsman,
+who smiled in sickly fashion at the title.
+</para>
+
+<para>
+"I'm only the owner," he explained.
+</para>
+
+<para>
+We rowed him aboard in much better style than he had come ashore,
+and saw for ourselves the helplessness of the passengers.  There
+were a dozen men and women, and all of them too sick even to appear
+grateful at our coming.  The yacht was rolling savagely, broad on,
+and no sooner had the owner's feet touched the deck than he
+collapsed and joined, the others.  Not one was able to bear a hand,
+so Charley and I between us cleared the badly tangled running gear,
+got up sail, and hoisted anchor.
+</para>
+
+<para>
+It was a rough trip, though a swift one.  The Carquinez Straits
+were a welter of foam and smother, and we came through them wildly
+before the wind, the big mainsail alternately dipping and flinging
+its boom skyward as we tore along.  But the people did not mind.
+They did not mind anything.  Two or three, including the owner,
+sprawled in the cockpit, shuddering when the yacht lifted and raced
+and sank dizzily into the trough, and between-whiles regarding the
+shore with yearning eyes.  The rest were huddled on the cabin floor
+among the cushions.  Now and again some one groaned, but for the
+most part they were as limp as so many dead persons.
+</para>
+
+<para>
+As the bight at Turner's Shipyard opened out, Charley edged into it
+to get the smoother water.  Benicia was in view, and we were
+bowling along over comparatively easy water, when a speck of a boat
+danced up ahead of us, directly in our course.  It was low-water
+slack.  Charley and I looked at each other.  No word was spoken,
+but at once the yacht began a most astonishing performance, veering
+and yawing as though the greenest of amateurs was at the wheel.  It
+was a sight for sailormen to see.  To all appearances, a runaway
+yacht was careering madly over the bight, and now and again
+yielding a little bit to control in a desperate effort to make
+Benicia.
+</para>
+
+<para>
+The owner forgot his seasickness long enough to look anxious.  The
+speck of a boat grew larger and larger, till we could see Big Alec
+and his partner, with a turn of the sturgeon line around a cleat,
+resting from their labor to laugh at us.  Charley pulled his
+sou'wester over his eyes, and I followed his example, though I
+could not guess the idea he evidently had in mind and intended to
+carry into execution.
+</para>
+
+<para>
+We came foaming down abreast of the skiff, so close that we could
+hear above the wind the voices of Big Alec and his mate as they
+shouted at us with all the scorn that professional watermen feel
+for amateurs, especially when amateurs are making fools of
+themselves.
+</para>
+
+<para>
+We thundered on past the fishermen, and nothing had happened.
+Charley grinned at the disappointment he saw in my face, and then
+shouted:
+</para>
+
+<para>
+"Stand by the main-sheet to jibe!"
+</para>
+
+<para>
+He put the wheel hard over, and the yacht whirled around
+obediently.  The main-sheet slacked and dipped, then shot over our
+heads after the boom and tautened with a crash on the traveller.
+The yacht heeled over almost on her beam ends, and a great wail
+went up from the seasick passengers as they swept across the cabin
+floor in a tangled mass and piled into a heap in the starboard
+bunks.
+</para>
+
+<para>
+But we had no time for them.  The yacht, completing the manoeuvre,
+headed into the wind with slatting canvas, and righted to an even
+keel.  We were still plunging ahead, and directly in our path was
+the skiff.  I saw Big Alec dive overboard and his mate leap for our
+bowsprit.  Then came the crash as we struck the boat, and a series
+of grinding bumps as it passed under our bottom.
+</para>
+
+<para>
+"That fixes his rifle," I heard Charley mutter, as he sprang upon
+the deck to look for Big Alec somewhere astern.
+</para>
+
+<para>
+The wind and sea quickly stopped our forward movement, and we began
+to drift backward over the spot where the skiff had been.  Big
+Alec's black head and swarthy face popped up within arm's reach;
+and all unsuspecting and very angry with what he took to be the
+clumsiness of amateur sailors, he was hauled aboard.  Also he was
+out of breath, for he had dived deep and stayed down long to escape
+our keel.
+</para>
+
+<para>
+The next instant, to the perplexity and consternation of the owner,
+Charley was on top of Big Alec in the cockpit, and I was helping
+bind him with gaskets.  The owner was dancing excitedly about and
+demanding an explanation, but by that time Big Alec's partner had
+crawled aft from the bowsprit and was peering apprehensively over
+the rail into the cockpit.  Charley's arm shot around his neck and
+the man landed on his back beside Big Alec.
+</para>
+
+<para>
+"More gaskets!" Charley shouted, and I made haste to supply them.
+</para>
+
+<para>
+The wrecked skiff was rolling sluggishly a short distance to
+windward, and I trimmed the sheets while Charley took the wheel and
+steered for it.
+</para>
+
+<para>
+"These two men are old offenders," he explained to the angry owner;
+"and they are most persistent violators of the fish and game laws.
+You have seen them caught in the act, and you may expect to be
+subpoenaed as witness for the state when the trial comes off."
+</para>
+
+<para>
+As he spoke he rounded alongside the skiff.  It had been torn from
+the line, a section of which was dragging to it.  He hauled in
+forty or fifty feet with a young sturgeon still fast in a tangle of
+barbless hooks, slashed that much of the line free with his knife,
+and tossed it into the cockpit beside the prisoners.
+</para>
+
+<para>
+"And there's the evidence, Exhibit A, for the people," Charley
+continued.  "Look it over carefully so that you may identify it in
+the court-room with the time and place of capture."
+</para>
+
+<para>
+And then, in triumph, with no more veering and yawing, we sailed
+into Benicia, the King of the Greeks bound hard and fast in the
+cockpit, and for the first time in his life a prisoner of the fish
+patrol.
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>A RAID ON THE OYSTER PIRATES</title>
+
+
+<para>
+Of the fish patrolmen under whom we served at various times,
+Charley Le Grant and I were agreed, I think, that Neil Partington
+was the best.  He was neither dishonest nor cowardly; and while he
+demanded strict obedience when we were under his orders, at the
+same time our relations were those of easy comradeship, and he
+permitted us a freedom to which we were ordinarily unaccustomed, as
+the present story will show.
+</para>
+
+<para>
+Neil's family lived in Oakland, which is on the Lower Bay, not more
+than six miles across the water from San Francisco.  One day, while
+scouting among the Chinese shrimp-catchers of Point Pedro, he
+received word that his wife was very ill; and within the hour the
+Reindeer was bowling along for Oakland, with a stiff northwest
+breeze astern.  We ran up the Oakland Estuary and came to anchor,
+and in the days that followed, while Neil was ashore, we tightened
+up the Reindeer's rigging, overhauled the ballast, scraped down,
+and put the sloop into thorough shape.
+</para>
+
+<para>
+This done, time hung heavy on our hands.  Neil's wife was
+dangerously ill, and the outlook was a week's lie-over, awaiting
+the crisis.  Charley and I roamed the docks, wondering what we
+should do, and so came upon the oyster fleet lying at the Oakland
+City Wharf.  In the main they were trim, natty boats, made for
+speed and bad weather, and we sat down on the stringer-piece of the
+dock to study them.
+</para>
+
+<para>
+"A good catch, I guess," Charley said, pointing to the heaps of
+oysters, assorted in three sizes, which lay upon their decks.
+</para>
+
+<para>
+Pedlers were backing their wagons to the edge of the wharf, and
+from the bargaining and chaffering that went on, I managed to learn
+the selling price of the oysters.
+</para>
+
+<para>
+"That boat must have at least two hundred dollars' worth aboard," I
+calculated.  "I wonder how long it took to get the load?"
+</para>
+
+<para>
+"Three or four days," Charley answered.  "Not bad wages for two men
+- twenty-five dollars a day apiece."
+</para>
+
+<para>
+The boat we were discussing, the Ghost, lay directly beneath us.
+Two men composed its crew.  One was a squat, broad-shouldered
+fellow with remarkably long and gorilla-like arms, while the other
+was tall and well proportioned, with clear blue eyes and a mat of
+straight black hair.  So unusual and striking was this combination
+of hair and eyes that Charley and I remained somewhat longer than
+we intended.
+</para>
+
+<para>
+And it was well that we did.  A stout, elderly man, with the dress
+and carriage of a successful merchant, came up and stood beside us,
+looking down upon the deck of the Ghost.  He appeared angry, and
+the longer he looked the angrier he grew.
+</para>
+
+<para>
+"Those are my oysters," he said at last.  "I know they are my
+oysters.  You raided my beds last night and robbed me of them."
+</para>
+
+<para>
+The tall man and the short man on the Ghost looked up.
+</para>
+
+<para>
+"Hello, Taft," the short man said, with insolent familiarity.
+(Among the bayfarers he had gained the nickname of "The Centipede"
+on account of his long arms.)  "Hello, Taft," he repeated, with the
+same touch of insolence.  "Wot 'r you growling about now?"
+</para>
+
+<para>
+"Those are my oysters - that's what I said.  You've stolen them
+from my beds."
+</para>
+
+<para>
+"Yer mighty wise, ain't ye?" was the Centipede's sneering reply.
+"S'pose you can tell your oysters wherever you see 'em?"
+</para>
+
+<para>
+"Now, in my experience," broke in the tall man, "oysters is oysters
+wherever you find 'em, an' they're pretty much alike all the Bay
+over, and the world over, too, for that matter.  We're not wantin'
+to quarrel with you, Mr. Taft, but we jes' wish you wouldn't
+insinuate that them oysters is yours an' that we're thieves an'
+robbers till you can prove the goods."
+</para>
+
+<para>
+"I know they're mine; I'd stake my life on it!" Mr. Taft snorted.
+</para>
+
+<para>
+"Prove it," challenged the tall man, who we afterward learned was
+known as "The Porpoise" because of his wonderful swimming
+abilities.
+</para>
+
+<para>
+Mr. Taft shrugged his shoulders helplessly.  Of course he could not
+prove the oysters to be his, no matter how certain he might be.
+</para>
+
+<para>
+"I'd give a thousand dollars to have you men behind the bars!" he
+cried.  "I'll give fifty dollars a head for your arrest and
+conviction, all of you!"
+</para>
+
+<para>
+A roar of laughter went up from the different boats, for the rest
+of the pirates had been listening to the discussion.
+</para>
+
+<para>
+"There's more money in oysters," the Porpoise remarked dryly.
+</para>
+
+<para>
+Mr. Taft turned impatiently on his heel and walked away.  From out
+of the corner of his eye, Charley noted the way he went.  Several
+minutes later, when he had disappeared around a corner, Charley
+rose lazily to his feet.  I followed him, and we sauntered off in
+the opposite direction to that taken by Mr. Taft.
+</para>
+
+<para>
+"Come on!  Lively!" Charley whispered, when we passed from the view
+of the oyster fleet.
+</para>
+
+<para>
+Our course was changed at once, and we dodged around corners and
+raced up and down side-streets till Mr. Taft's generous form loomed
+up ahead of us.
+</para>
+
+<para>
+"I'm going to interview him about that reward," Charley explained,
+as we rapidly over-hauled the oyster-bed owner.  "Neil will be
+delayed here for a week, and you and I might as well be doing
+something in the meantime.  What do you say?"
+</para>
+
+<para>
+"Of course, of course," Mr. Taft said, when Charley had introduced
+himself and explained his errand.  "Those thieves are robbing me of
+thousands of dollars every year, and I shall be glad to break them
+up at any price, - yes, sir, at any price.  As I said, I'll give
+fifty dollars a head, and call it cheap at that.  They've robbed my
+beds, torn down my signs, terrorized my watchmen, and last year
+killed one of them.  Couldn't prove it.  All done in the blackness
+of night.  All I had was a dead watchman and no evidence.  The
+detectives could do nothing.  Nobody has been able to do anything
+with those men.  We have never succeeded in arresting one of them.
+So I say, Mr. -  What did you say your name was?"
+</para>
+
+<para>
+"Le Grant," Charley answered.
+</para>
+
+<para>
+"So I say, Mr. Le Grant, I am deeply obliged to you for the
+assistance you offer.  And I shall be glad, most glad, sir, to co-
+operate with you in every way.  My watchmen and boats are at your
+disposal.  Come and see me at the San Francisco offices any time,
+or telephone at my expense.  And don't be afraid of spending money.
+I'll foot your expenses, whatever they are, so long as they are
+within reason.  The situation is growing desperate, and something
+must be done to determine whether I or that band of ruffians own
+those oyster beds."
+</para>
+
+<para>
+"Now we'll see Neil," Charley said, when he had seen Mr. Taft upon
+his train to San Francisco.
+</para>
+
+<para>
+Not only did Neil Partington interpose no obstacle to our
+adventure, but he proved to be of the greatest assistance.  Charley
+and I knew nothing of the oyster industry, while his head was an
+encyclopaedia of facts concerning it.  Also, within an hour or so,
+he was able to bring to us a Greek boy of seventeen or eighteen who
+knew thoroughly well the ins and outs of oyster piracy.
+</para>
+
+<para>
+At this point I may as well explain that we of the fish patrol were
+free lances in a way.  While Neil Partington, who was a patrolman
+proper, received a regular salary, Charley and I, being merely
+deputies, received only what we earned - that is to say, a certain
+percentage of the fines imposed on convicted violators of the fish
+laws.  Also, any rewards that chanced our way were ours.  We
+offered to share with Partington whatever we should get from Mr.
+Taft, but the patrolman would not hear of it.  He was only too
+happy, he said, to do a good turn for us, who had done so many for
+him.
+</para>
+
+<para>
+We held a long council of war, and mapped out the following line of
+action.  Our faces were unfamiliar on the Lower Bay, but as the
+Reindeer was well known as a fish-patrol sloop, the Greek boy,
+whose name was Nicholas, and I were to sail some innocent-looking
+craft down to Asparagus Island and join the oyster pirates' fleet.
+Here, according to Nicholas's description of the beds and the
+manner of raiding, it was possible for us to catch the pirates in
+the act of stealing oysters, and at the same time to get them in
+our power.  Charley was to be on the shore, with Mr. Taft's
+watchmen and a posse of constables, to help us at the right time.
+</para>
+
+<para>
+"I know just the boat," Neil said, at the conclusion of the
+discussion, "a crazy old sloop that's lying over at Tiburon.  You
+and Nicholas can go over by the ferry, charter it for a song, and
+sail direct for the beds."
+</para>
+
+<para>
+"Good luck be with you, boys," he said at parting, two days later.
+"Remember, they are dangerous men, so be careful."
+</para>
+
+<para>
+Nicholas and I succeeded in chartering the sloop very cheaply; and
+between laughs, while getting up sail, we agreed that she was even
+crazier and older than she had been described.  She was a big,
+flat-bottomed, square-sterned craft, sloop-rigged, with a sprung
+mast, slack rigging, dilapidated sails, and rotten running-gear,
+clumsy to handle and uncertain in bringing about, and she smelled
+vilely of coal tar, with which strange stuff she had been smeared
+from stem to stern and from cabin-roof to centreboard.  And to cap
+it all, Coal Tar Maggie was printed in great white letters the
+whole length of either side.
+</para>
+
+<para>
+It was an uneventful though laughable run from Tiburon to Asparagus
+Island, where we arrived in the afternoon of the following day.
+The oyster pirates, a fleet of a dozen sloops, were lying at anchor
+on what was known as the "Deserted Beds."  The Coal Tar Maggie came
+sloshing into their midst with a light breeze astern, and they
+crowded on deck to see us.  Nicholas and I had caught the spirit of
+the crazy craft, and we handled her in most lubberly fashion.
+</para>
+
+<para>
+"Wot is it?" some one called.
+</para>
+
+<para>
+"Name it 'n' ye kin have it!" called another.
+</para>
+
+<para>
+"I swan naow, ef it ain't the old Ark itself!" mimicked the
+Centipede from the deck of the Ghost.
+</para>
+
+<para>
+"Hey!  Ahoy there, clipper ship!" another wag shouted.  "Wot's yer
+port?"
+</para>
+
+<para>
+We took no notice of the joking, but acted, after the manner of
+greenhorns, as though the Coal Tar Maggie required our undivided
+attention.  I rounded her well to windward of the Ghost, and
+Nicholas ran for'ard to drop the anchor.  To all appearances it was
+a bungle, the way the chain tangled and kept the anchor from
+reaching the bottom.  And to all appearances Nicholas and I were
+terribly excited as we strove to clear it.  At any rate, we quite
+deceived the pirates, who took huge delight in our predicament.
+</para>
+
+<para>
+But the chain remained tangled, and amid all kinds of mocking
+advice we drifted down upon and fouled the Ghost, whose bowsprit
+poked square through our mainsail and ripped a hole in it as big as
+a barn door.  The Centipede and the Porpoise doubled up on the
+cabin in paroxysms of laughter, and left us to get clear as best we
+could.  This, with much unseaman-like performance, we succeeded in
+doing, and likewise in clearing the anchor-chain, of which we let
+out about three hundred feet.  With only ten feet of water under
+us, this would permit the Coal Tar Maggie to swing in a circle six
+hundred feet in diameter, in which circle she would be able to foul
+at least half the fleet.
+</para>
+
+<para>
+The oyster pirates lay snugly together at short hawsers, the
+weather being fine, and they protested loudly at our ignorance in
+putting out such an unwarranted length of anchor-chain.  And not
+only did they protest, for they made us heave it in again, all but
+thirty feet.
+</para>
+
+<para>
+Having sufficiently impressed them with our general lubberliness,
+Nicholas and I went below to congratulate ourselves and to cook
+supper.  Hardly had we finished the meal and washed the dishes,
+when a skiff ground against the Coal Tar Maggie's side, and heavy
+feet trampled on deck.  Then the Centipede's brutal face appeared
+in the companionway, and he descended into the cabin, followed by
+the Porpoise.  Before they could seat themselves on a bunk, another
+skiff came alongside, and another, and another, till the whole
+fleet was represented by the gathering in the cabin.
+</para>
+
+<para>
+"Where'd you swipe the old tub?" asked a squat and hairy man, with
+cruel eyes and Mexican features.
+</para>
+
+<para>
+"Didn't swipe it," Nicholas answered, meeting them on their own
+ground and encouraging the idea that we had stolen the Coal Tar
+Maggie.  "And if we did, what of it?"
+</para>
+
+<para>
+"Well, I don't admire your taste, that's all," sneered he of the
+Mexican features.  "I'd rot on the beach first before I'd take a
+tub that couldn't get out of its own way."
+</para>
+
+<para>
+"How were we to know till we tried her?" Nicholas asked, so
+innocently as to cause a laugh.  "And how do you get the oysters?"
+he hurried on.  "We want a load of them; that's what we came for, a
+load of oysters."
+</para>
+
+<para>
+"What d'ye want 'em for?" demanded the Porpoise.
+</para>
+
+<para>
+"Oh, to give away to our friends, of course," Nicholas retorted.
+"That's what you do with yours, I suppose."
+</para>
+
+<para>
+This started another laugh, and as our visitors grew more genial we
+could see that they had not the slightest suspicion of our identity
+or purpose.
+</para>
+
+<para>
+"Didn't I see you on the dock in Oakland the other day?" the
+Centipede asked suddenly of me.
+</para>
+
+<para>
+"Yep," I answered boldly, taking the bull by the horns.  "I was
+watching you fellows and figuring out whether we'd go oystering or
+not.  It's a pretty good business, I calculate, and so we're going
+in for it.  That is," I hastened to add, "if you fellows don't
+mind."
+</para>
+
+<para>
+"I'll tell you one thing, which ain't two things," he replied, "and
+that is you'll have to hump yerself an' get a better boat.  We
+won't stand to be disgraced by any such box as this.  Understand?"
+</para>
+
+<para>
+"Sure," I said.  "Soon as we sell some oysters we'll outfit in
+style."
+</para>
+
+<para>
+"And if you show yerself square an' the right sort," he went on,
+"why, you kin run with us.  But if you don't" (here his voice
+became stern and menacing), "why, it'll be the sickest day of yer
+life.  Understand?"
+</para>
+
+<para>
+"Sure," I said.
+</para>
+
+<para>
+After that and more warning and advice of similar nature, the
+conversation became general, and we learned that the beds were to
+be raided that very night.  As they got into their boats, after an
+hour's stay, we were invited to join them in the raid with the
+assurance of "the more the merrier."
+</para>
+
+<para>
+"Did you notice that short, Mexican-looking chap?" Nicholas asked,
+when they had departed to their various sloops.  "He's Barchi, of
+the Sporting Life Gang, and the fellow that came with him is
+Skilling.  They're both out now on five thousand dollars' bail."
+</para>
+
+<para>
+I had heard of the Sporting Life Gang before, a crowd of hoodlums
+and criminals that terrorized the lower quarters of Oakland, and
+two-thirds of which were usually to be found in state's prison for
+crimes that ranged from perjury and ballot-box stuffing to murder.
+</para>
+
+<para>
+"They are not regular oyster pirates," Nicholas continued.
+"They've just come down for the lark and to make a few dollars.
+But we'll have to watch out for them."
+</para>
+
+<para>
+We sat in the cockpit and discussed the details of our plan till
+eleven o'clock had passed, when we heard the rattle of an oar in a
+boat from the direction of the Ghost.  We hauled up our own skiff,
+tossed in a few sacks, and rowed over.  There we found all the
+skiffs assembling, it being the intention to raid the beds in a
+body.
+</para>
+
+<para>
+To my surprise, I found barely a foot of water where we had dropped
+anchor in ten feet.  It was the big June run-out of the full moon,
+and as the ebb had yet an hour and a half to run, I knew that our
+anchorage would be dry ground before slack water.
+</para>
+
+<para>
+Mr. Taft's beds were three miles away, and for a long time we rowed
+silently in the wake of the other boats, once in a while grounding
+and our oar blades constantly striking bottom.  At last we came
+upon soft mud covered with not more than two inches of water - not
+enough to float the boats.  But the pirates at once were over the
+side, and by pushing and pulling on the flat-bottomed skiffs, we
+moved steadily along.
+</para>
+
+<para>
+The full moon was partly obscured by high-flying clouds, but the
+pirates went their way with the familiarity born of long practice.
+After half a mile of the mud, we came upon a deep channel, up which
+we rowed, with dead oyster shoals looming high and dry on either
+side.  At last we reached the picking grounds.  Two men, on one of
+the shoals, hailed us and warned us off.  But the Centipede, the
+Porpoise, Barchi, and Skilling took the lead, and followed by the
+rest of us, at least thirty men in half as many boats, rowed right
+up to the watchmen.
+</para>
+
+<para>
+"You'd better slide outa this here," Barchi said threateningly, "or
+we'll fill you so full of holes you wouldn't float in molasses."
+</para>
+
+<para>
+The watchmen wisely retreated before so overwhelming a force, and
+rowed their boat along the channel toward where the shore should
+be.  Besides, it was in the plan for them to retreat.
+</para>
+
+<para>
+We hauled the noses of the boats up on the shore side of a big
+shoal, and all hands, with sacks, spread out and began picking.
+Every now and again the clouds thinned before the face of the moon,
+and we could see the big oysters quite distinctly.  In almost no
+time sacks were filled and carried back to the boats, where fresh
+ones were obtained.  Nicholas and I returned often and anxiously to
+the boats with our little loads, but always found some one of the
+pirates coming or going.
+</para>
+
+<para>
+"Never mind," he said; "no hurry.  As they pick farther and farther
+away, it will take too long to carry to the boats.  Then they'll
+stand the full sacks on end and pick them up when the tide comes in
+and the skiffs will float to them."
+</para>
+
+<para>
+Fully half an hour went by, and the tide had begun to flood, when
+this came to pass.  Leaving the pirates at their work, we stole
+back to the boats.  One by one, and noiselessly, we shoved them off
+and made them fast in an awkward flotilla.  Just as we were shoving
+off the last skiff, our own, one of the men came upon us.  It was
+Barchi.  His quick eye took in the situation at a glance, and he
+sprang for us; but we went clear with a mighty shove, and he was
+left floundering in the water over his head.  As soon as he got
+back to the shoal he raised his voice and gave the alarm.
+</para>
+
+<para>
+We rowed with all our strength, but it was slow going with so many
+boats in tow.  A pistol cracked from the shoal, a second, and a
+third; then a regular fusillade began.  The bullets spat and spat
+all about us; but thick clouds had covered the moon, and in the dim
+darkness it was no more than random firing.  It was only by chance
+that we could be hit.
+</para>
+
+<para>
+"Wish we had a little steam launch," I panted.
+</para>
+
+<para>
+"I'd just as soon the moon stayed hidden," Nicholas panted back.
+</para>
+
+<para>
+It was slow work, but every stroke carried us farther away from the
+shoal and nearer the shore, till at last the shooting died down,
+and when the moon did come out we were too far away to be in
+danger.  Not long afterward we answered a shoreward hail, and two
+Whitehall boats, each pulled by three pairs of oars, darted up to
+us.  Charley's welcome face bent over to us, and he gripped us by
+the hands while he cried, "Oh, you joys!  You joys!  Both of you!"
+</para>
+
+<para>
+When the flotilla had been landed, Nicholas and I and a watchman
+rowed out in one of the Whitehalls, with Charley in the stern-
+sheets.  Two other Whitehalls followed us, and as the moon now
+shone brightly, we easily made out the oyster pirates on their
+lonely shoal.  As we drew closer, they fired a rattling volley from
+their revolvers, and we promptly retreated beyond range.
+</para>
+
+<para>
+"Lot of time," Charley said.  "The flood is setting in fast, and by
+the time it's up to their necks there won't be any fight left in
+them."
+</para>
+
+<para>
+So we lay on our oars and waited for the tide to do its work.  This
+was the predicament of the pirates:  because of the big run-out,
+the tide was now rushing back like a mill-race, and it was
+impossible for the strongest swimmer in the world to make against
+it the three miles to the sloops.  Between the pirates and the
+shore were we, precluding escape in that direction.  On the other
+hand, the water was rising rapidly over the shoals, and it was only
+a question of a few hours when it would be over their heads.
+</para>
+
+<para>
+It was beautifully calm, and in the brilliant white moonlight we
+watched them through our night glasses and told Charley of the
+voyage of the Coal Tar Maggie.  One o'clock came, and two o'clock,
+and the pirates were clustering on the highest shoal, waist-deep in
+water.
+</para>
+
+<para>
+"Now this illustrates the value of imagination," Charley was
+saying.  "Taft has been trying for years to get them, but he went
+at it with bull strength and failed.  Now we used our heads . . ."
+</para>
+
+<para>
+Just then I heard a scarcely audible gurgle of water, and holding
+up my hand for silence, I turned and pointed to a ripple slowly
+widening out in a growing circle.  It was not more than fifty feet
+from us.  We kept perfectly quiet and waited.  After a minute the
+water broke six feet away, and a black head and white shoulder
+showed in the moonlight.  With a snort of surprise and of suddenly
+expelled breath, the head and shoulder went down.
+</para>
+
+<para>
+We pulled ahead several strokes and drifted with the current.  Four
+pairs of eyes searched the surface of the water, but never another
+ripple showed, and never another glimpse did we catch of the black
+head and white shoulder.
+</para>
+
+<para>
+"It's the Porpoise," Nicholas said.  "It would take broad daylight
+for us to catch him."
+</para>
+
+<para>
+At a quarter to three the pirates gave their first sign of
+weakening.  We heard cries for help, in the unmistakable voice of
+the Centipede, and this time, on rowing closer, we were not fired
+upon.  The Centipede was in a truly perilous plight.  Only the
+heads and shoulders of his fellow-marauders showed above the water
+as they braced themselves against the current, while his feet were
+off the bottom and they were supporting him.
+</para>
+
+<para>
+"Now, lads," Charley said briskly, "we have got you, and you can't
+get away.  If you cut up rough, we'll have to leave you alone and
+the water will finish you.  But if you're good we'll take you
+aboard, one man at a time, and you'll all be saved.  What do you
+say?"
+</para>
+
+<para>
+"Ay," they chorused hoarsely between their chattering teeth.
+</para>
+
+<para>
+"Then one man at a time, and the short men first."
+</para>
+
+<para>
+The Centipede was the first to be pulled aboard, and he came
+willingly, though he objected when the constable put the handcuffs
+on him.  Barchi was next hauled in, quite meek and resigned from
+his soaking.  When we had ten in, our boat we drew back, and the
+second Whitehall was loaded.  The third Whitehall received nine
+prisoners only - a catch of twenty-nine in all.
+</para>
+
+<para>
+"You didn't get the Porpoise," the Centipede said exultantly, as
+though his escape materially diminished our success.
+</para>
+
+<para>
+Charley laughed.  "But we saw him just the same, a-snorting for
+shore like a puffing pig."
+</para>
+
+<para>
+It was a mild and shivering band of pirates that we marched up the
+beach to the oyster house.  In answer to Charley's knock, the door
+was flung open, and a pleasant wave of warm air rushed out upon us.
+</para>
+
+<para>
+"You can dry your clothes here, lads, and get some hot coffee,"
+Charley announced, as they filed in.
+</para>
+
+<para>
+And there, sitting ruefully by the fire, with a steaming mug in his
+hand, was the Porpoise.  With one accord Nicholas and I looked at
+Charley.  He laughed gleefully.
+</para>
+
+<para>
+"That comes of imagination," he said.  "When you see a thing,
+you've got to see it all around, or what's the good of seeing it at
+all?  I saw the beach, so I left a couple of constables behind to
+keep an eye on it.  That's all."
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>THE SIEGE OF THE "LANCASHIRE QUEEN"</title>
+
+
+<para>
+Possibly our most exasperating experience on the fish patrol was
+when Charley Le Grant and I laid a two weeks' siege to a big four-
+masted English ship.  Before we had finished with the affair, it
+became a pretty mathematical problem, and it was by the merest
+chance that we came into possession of the instrument that brought
+it to a successful termination.
+</para>
+
+<para>
+After our raid on the oyster pirates we had returned to Oakland,
+where two more weeks passed before Neil Partington's wife was out
+of danger and on the highroad to recovery.  So it was after an
+absence of a month, all told, that we turned the Reindeer's nose
+toward Benicia.  When the cat's away the mice will play, and in
+these four weeks the fishermen had become very bold in violating
+the law.  When we passed Point Pedro we noticed many signs of
+activity among the shrimp-catchers, and, well into San Pablo Bay,
+we observed a widely scattered fleet of Upper Bay fishing-boats
+hastily pulling in their nets and getting up sail.
+</para>
+
+<para>
+This was suspicious enough to warrant investigation, and the first
+and only boat we succeeded in boarding proved to have an illegal
+net.  The law permitted no smaller mesh for catching shad than one
+that measured seven and one-half inches inside the knots, while the
+mesh of this particular net measured only three inches.  It was a
+flagrant breach of the rules, and the two fishermen were forthwith
+put under arrest.  Neil Partington took one of them with him to
+help manage the Reindeer, while Charley and I went on ahead with
+the other in the captured boat.
+</para>
+
+<para>
+But the shad fleet had headed over toward the Petaluma shore in
+wild flight, and for the rest of the run through San Pablo Bay we
+saw no more fishermen at all.  Our prisoner, a bronzed and bearded
+Greek, sat sullenly on his net while we sailed his craft.  It was a
+new Columbia River salmon boat, evidently on its first trip, and it
+handled splendidly.  Even when Charley praised it, our prisoner
+refused to speak or to notice us, and we soon gave him up as a most
+unsociable fellow.
+</para>
+
+<para>
+We ran up the Carquinez Straits and edged into the bight at
+Turner's Shipyard for smoother water.  Here were lying several
+English steel sailing ships, waiting for the wheat harvest; and
+here, most unexpectedly, in the precise place where we had captured
+Big Alec, we came upon two Italians in a skiff that was loaded with
+a complete "Chinese" sturgeon line.  The surprise was mutual, and
+we were on top of them before either they or we were aware.
+Charley had barely time to luff into the wind and run up to them.
+I ran forward and tossed them a line with orders to make it fast.
+One of the Italians took a turn with it over a cleat, while I
+hastened to lower our big spritsail.  This accomplished, the salmon
+boat dropped astern, dragging heavily on the skiff.
+</para>
+
+<para>
+Charley came forward to board the prize, but when I proceeded to
+haul alongside by means of the line, the Italians cast it off.  We
+at once began drifting to leeward, while they got out two pairs of
+oars and rowed their light craft directly into the wind.  This
+manoeuvre for the moment disconcerted us, for in our large and
+heavily loaded boat we could not hope to catch them with the oars.
+But our prisoner came unexpectedly to our aid.  His black eyes were
+flashing eagerly, and his face was flushed with suppressed
+excitement, as he dropped the centre-board, sprang forward with a
+single leap, and put up the sail.
+</para>
+
+<para>
+"I've always heard that Greeks don't like Italians," Charley
+laughed, as he ran aft to the tiller.
+</para>
+
+<para>
+And never in my experience have I seen a man so anxious for the
+capture of another as was our prisoner in the chase that followed.
+His eyes fairly snapped, and his nostrils quivered and dilated in a
+most extraordinary way.  Charley steered while he tended the sheet;
+and though Charley was as quick and alert as a cat, the Greek could
+hardly control his impatience.
+</para>
+
+<para>
+The Italians were cut off from the shore, which was fully a mile
+away at its nearest point.  Did they attempt to make it, we could
+haul after them with the wind abeam, and overtake them before they
+had covered an eighth of the distance.  But they were too wise to
+attempt it, contenting themselves with rowing lustily to windward
+along the starboard side of a big ship, the Lancashire Queen.  But
+beyond the ship lay an open stretch of fully two miles to the shore
+in that direction.  This, also, they dared not attempt, for we were
+bound to catch them before they could cover it.  So, when they
+reached the bow of the Lancashire Queen, nothing remained but to
+pass around and row down her port side toward the stern, which
+meant rowing to leeward and giving us the advantage.
+</para>
+
+<para>
+We in the salmon boat, sailing close on the wind, tacked about and
+crossed the ship's bow.  Then Charley put up the tiller and headed
+down the port side of the ship, the Greek letting out the sheet and
+grinning with delight.  The Italians were already half-way down the
+ship's length; but the stiff breeze at our back drove us after them
+far faster than they could row.  Closer and closer we came, and I,
+lying down forward, was just reaching out to grasp the skiff, when
+it ducked under the great stern of the Lancashire Queen.
+</para>
+
+<para>
+The chase was virtually where it had begun.  The Italians were
+rowing up the starboard side of the ship, and we were hauled close
+on the wind and slowly edging out from the ship as we worked to
+windward.  Then they darted around her bow and began the row down
+her port side, and we tacked about, crossed her bow, and went
+plunging down the wind hot after them.  And again, just as I was
+reaching for the skiff, it ducked under the ship's stern and out of
+danger.  And so it went, around and around, the skiff each time
+just barely ducking into safety.
+</para>
+
+<para>
+By this time the ship's crew had become aware of what was taking
+place, and we could see their heads in a long row as they looked at
+us over the bulwarks.  Each time we missed the skiff at the stern,
+they set up a wild cheer and dashed across to the other side of the
+Lancashire Queen to see the chase to wind-ward.  They showered us
+and the Italians with jokes and advice, and made our Greek so angry
+that at least once on each circuit he raised his fist and shook it
+at them in a rage.  They came to look for this, and at each display
+greeted it with uproarious mirth.
+</para>
+
+<para>
+"Wot a circus!" cried one.
+</para>
+
+<para>
+"Tork about yer marine hippodromes, - if this ain't one, I'd like
+to know!" affirmed another.
+</para>
+
+<para>
+"Six-days-go-as-yer-please," announced a third.  "Who says the
+dagoes won't win?"
+</para>
+
+<para>
+On the next tack to windward the Greek offered to change places
+with Charley.
+</para>
+
+<para>
+"Let-a me sail-a de boat," he demanded.  "I fix-a them, I catch-a
+them, sure."
+</para>
+
+<para>
+This was a stroke at Charley's professional pride, for pride
+himself he did upon his boat-sailing abilities; but he yielded the
+tiller to the prisoner and took his place at the sheet.  Three
+times again we made the circuit, and the Greek found that he could
+get no more speed out of the salmon boat than Charley had.
+</para>
+
+<para>
+"Better give it up," one of the sailors advised from above.
+</para>
+
+<para>
+The Greek scowled ferociously and shook his fist in his customary
+fashion.  In the meanwhile my mind had not been idle, and I had
+finally evolved an idea.
+</para>
+
+<para>
+"Keep going, Charley, one time more," I said.
+</para>
+
+<para>
+And as we laid out on the next tack to wind-ward, I bent a piece of
+line to a small grappling hook I had seen lying in the bail-hole.
+The end of the line I made fast to the ring-bolt in the bow, and
+with the hook out of sight I waited for the next opportunity to use
+it.  Once more they made their leeward pull down the port side of
+the Lancashire Queen, and once more we churned down after them
+before the wind.  Nearer and nearer we drew, and I was making
+believe to reach for them as before.  The stern of the skiff was
+not six feet away, and they were laughing at me derisively as they
+ducked under the ship's stern.  At that instant I suddenly arose
+and threw the grappling iron.  It caught fairly and squarely on the
+rail of the skiff, which was jerked backward out of safety as the
+rope tautened and the salmon boat ploughed on.
+</para>
+
+<para>
+A groan went up from the row of sailors above, which quickly
+changed to a cheer as one of the Italians whipped out a long
+sheath-knife and cut the rope.  But we had drawn them out of
+safety, and Charley, from his place in the stern-sheets, reached
+over and clutched the stern of the skiff.  The whole thing happened
+in a second of time, for the first Italian was cutting the rope and
+Charley was clutching the skiff when the second Italian dealt him a
+rap over the head with an oar, Charley released his hold and
+collapsed, stunned, into the bottom of the salmon boat, and the
+Italians bent to their oars and escaped back under the ship's
+stern.
+</para>
+
+<para>
+The Greek took both tiller and sheet and continued the chase around
+the Lancashire Queen, while I attended to Charley, on whose head a
+nasty lump was rapidly rising.  Our sailor audience was wild with
+delight, and to a man encouraged the fleeing Italians.  Charley sat
+up, with one hand on his head, and gazed about him sheepishly.
+</para>
+
+<para>
+"It will never do to let them escape now," he said, at the same
+time drawing his revolver.
+</para>
+
+<para>
+On our next circuit, he threatened the Italians with the weapon;
+but they rowed on stolidly, keeping splendid stroke and utterly
+disregarding him.
+</para>
+
+<para>
+"If you don't stop, I'll shoot," Charley said menacingly.
+</para>
+
+<para>
+But this had no effect, nor were they to be frightened into
+surrendering even when he fired several shots dangerously close to
+them.  It was too much to expect him to shoot unarmed men, and this
+they knew as well as we did; so they continued to pull doggedly
+round and round the ship.
+</para>
+
+<para>
+"We'll run them down, then!" Charley exclaimed.  "We'll wear them
+out and wind them!"
+</para>
+
+<para>
+So the chase continued.  Twenty times more we ran them around the
+Lancashire Queen, and at last we could see that even their iron
+muscles were giving out.  They were nearly exhausted, and it was
+only a matter of a few more circuits, when the game took on a new
+feature.  On the row to windward they always gained on us, so that
+they were half-way down the ship's side on the row to leeward when
+we were passing the bow.  But this last time, as we passed the bow,
+we saw them escaping up the ship's gangway, which had been suddenly
+lowered.  It was an organized move on the part of the sailors,
+evidently countenanced by the captain; for by the time we arrived
+where the gangway had been, it was being hoisted up, and the skiff,
+slung in the ship's davits, was likewise flying aloft out of reach.
+</para>
+
+<para>
+The parley that followed with the captain was short and snappy.  He
+absolutely forbade us to board the Lancashire Queen, and as
+absolutely refused to give up the two men.  By this time Charley
+was as enraged as the Greek.  Not only had he been foiled in a long
+and ridiculous chase, but he had been knocked senseless into the
+bottom of his boat by the men who had escaped him.
+</para>
+
+<para>
+"Knock off my head with little apples," he declared emphatically,
+striking the fist of one hand into the palm of the other, "if those
+two men ever escape me!  I'll stay here to get them if it takes the
+rest of my natural life, and if I don't get them, then I promise
+you I'll live unnaturally long or until I do get them, or my name's
+not Charley Le Grant!"
+</para>
+
+<para>
+And then began the siege of the Lancashire Queen, a siege memorable
+in the annals of both fishermen and fish patrol.  When the Reindeer
+came along, after a fruitless pursuit of the shad fleet, Charley
+instructed Neil Partington to send out his own salmon boat, with
+blankets, provisions, and a fisherman's charcoal stove.  By sunset
+this exchange of boats was made, and we said good-by to our Greek,
+who perforce had to go into Benicia and be locked up for his own
+violation of the law.  After supper, Charley and I kept alternate
+four-hour watches till day-light.  The fishermen made no attempt to
+escape that night, though the ship sent out a boat for scouting
+purposes to find if the coast were clear.
+</para>
+
+<para>
+By the next day we saw that a steady siege was in order, and we
+perfected our plans with an eye to our own comfort.  A dock, known
+as the Solano Wharf, which ran out from the Benicia shore, helped
+us in this.  It happened that the Lancashire Queen, the shore at
+Turner's Shipyard, and the Solano Wharf were the corners of a big
+equilateral triangle.  From ship to shore, the side of the triangle
+along which the Italians had to escape, was a distance equal to
+that from the Solano Wharf to the shore, the side of the triangle
+along which we had to travel to get to the shore before the
+Italians.  But as we could sail much faster than they could row, we
+could permit them to travel about half their side of the triangle
+before we darted out along our side.  If we allowed them to get
+more than half-way, they were certain to beat us to shore; while if
+we started before they were half-way, they were equally certain to
+beat us back to the ship.
+</para>
+
+<para>
+We found that an imaginary line, drawn from the end of the wharf to
+a windmill farther along the shore, cut precisely in half the line
+of the triangle along which the Italians must escape to reach the
+land.  This line made it easy for us to determine how far to let
+them run away before we bestirred ourselves in pursuit.  Day after
+day we would watch them through our glasses as they rowed leisurely
+along toward the half-way point; and as they drew close into line
+with the windmill, we would leap into the boat and get up sail.  At
+sight of our preparation, they would turn and row slowly back to
+the Lancashire Queen, secure in the knowledge that we could not
+overtake them.
+</para>
+
+<para>
+To guard against calms - when our salmon boat would be useless - we
+also had in readiness a light rowing skiff equipped with spoon-
+oars.  But at such times, when the wind failed us, we were forced
+to row out from the wharf as soon as they rowed from the ship.  In
+the night-time, on the other hand, we were compelled to patrol the
+immediate vicinity of the ship; which we did, Charley and I
+standing four-hour watches turn and turn about.  The Italians,
+however, preferred the daytime in which to escape, and so our long
+night vigils were without result.
+</para>
+
+<para>
+"What makes me mad," said Charley, "is our being kept from our
+honest beds while those rascally lawbreakers are sleeping soundly
+every night.  But much good may it do them," he threatened.  "I'll
+keep them on that ship till the captain charges them board, as sure
+as a sturgeon's not a catfish!"
+</para>
+
+<para>
+It was a tantalizing problem that confronted us.  As long as we
+were vigilant, they could not escape; and as long as they were
+careful, we would be unable to catch them.  Charley cudgelled his
+brains continually, but for once his imagination failed him.  It
+was a problem apparently without other solution than that of
+patience.  It was a waiting game, and whichever waited the longer
+was bound to win.  To add to our irritation, friends of the
+Italians established a code of signals with them from the shore, so
+that we never dared relax the siege for a moment.  And besides
+this, there were always one or two suspicious-looking fishermen
+hanging around the Solano Wharf and keeping watch on our actions.
+We could do nothing but "grin and bear it," as Charley said, while
+it took up all our time and prevented us from doing other work.
+</para>
+
+<para>
+The days went by, and there was no change in the situation.  Not
+that no attempts were made to change it.  One night friends from
+the shore came out in a skiff and attempted to confuse us while the
+two Italians escaped.  That they did not succeed was due to the
+lack of a little oil on the ship's davits.  For we were drawn back
+from the pursuit of the strange boat by the creaking of the davits,
+and arrived at the Lancashire Queen just as the Italians were
+lowering their skiff.  Another night, fully half a dozen skiffs
+rowed around us in the darkness, but we held on like a leech to the
+side of the ship and frustrated their plan till they grew angry and
+showered us with abuse.  Charley laughed to himself in the bottom
+of the boat.
+</para>
+
+<para>
+"It's a good sign, lad," he said to me.  "When men begin to abuse,
+make sure they're losing patience; and shortly after they lose
+patience, they lose their heads.  Mark my words, if we only hold
+out, they'll get careless some fine day, and then we'll get them."
+</para>
+
+<para>
+But they did not grow careless, and Charley confessed that this was
+one of the times when all signs failed.  Their patience seemed
+equal to ours, and the second week of the siege dragged
+monotonously along.  Then Charley's lagging imagination quickened
+sufficiently to suggest a ruse.  Peter Boyelen, a new patrolman and
+one unknown to the fisher-folk, happened to arrive in Benicia and
+we took him into our plan.  We were as secret as possible about it,
+but in some unfathomable way the friends ashore got word to the
+beleaguered Italians to keep their eyes open.
+</para>
+
+<para>
+On the night we were to put our ruse into effect, Charley and I
+took up our usual station in our rowing skiff alongside the
+Lancashire Queen.  After it was thoroughly dark, Peter Boyelen came
+out in a crazy duck boat, the kind you can pick up and carry away
+under one arm.  When we heard him coming along, paddling noisily,
+we slipped away a short distance into the darkness, and rested on
+our oars.  Opposite the gangway, having jovially hailed the anchor-
+watch of the Lancashire Queen and asked the direction of the
+Scottish Chiefs, another wheat ship, he awkwardly capsized himself.
+The man who was standing the anchor-watch ran down the gangway and
+hauled him out of the water.  This was what he wanted, to get
+aboard the ship; and the next thing he expected was to be taken on
+deck and then below to warm up and dry out.  But the captain
+inhospitably kept him perched on the lowest gang-way step,
+shivering miserably and with his feet dangling in the water, till
+we, out of very pity, rowed in from the darkness and took him off.
+The jokes and gibes of the awakened crew sounded anything but sweet
+in our ears, and even the two Italians climbed up on the rail and
+laughed down at us long and maliciously.
+</para>
+
+<para>
+"That's all right," Charley said in a low voice, which I only could
+hear.  "I'm mighty glad it's not us that's laughing first.  We'll
+save our laugh to the end, eh, lad?"
+</para>
+
+<para>
+He clapped a hand on my shoulder as he finished, but it seemed to
+me that there was more determination than hope in his voice.
+</para>
+
+<para>
+It would have been possible for us to secure the aid of United
+States marshals and board the English ship, backed by Government
+authority.  But the instructions of the Fish Commission were to the
+effect that the patrolmen should avoid complications, and this one,
+did we call on the higher powers, might well end in a pretty
+international tangle.
+</para>
+
+<para>
+The second week of the siege drew to its close, and there was no
+sign of change in the situation.  On the morning of the fourteenth
+day the change came, and it came in a guise as unexpected and
+startling to us as it was to the men we were striving to capture.
+</para>
+
+<para>
+Charley and I, after our customary night vigil by the side of the
+Lancashire Queen, rowed into the Solana Wharf.
+</para>
+
+<para>
+"Hello!" cried Charley, in surprise.  "In the name of reason and
+common sense, what is that?  Of all unmannerly craft did you ever
+see the like?"
+</para>
+
+<para>
+Well might he exclaim, for there, tied up to the dock, lay the
+strangest looking launch I had ever seen.  Not that it could be
+called a launch, either, but it seemed to resemble a launch more
+than any other kind of boat.  It was seventy feet long, but so
+narrow was it, and so bare of superstructure, that it appeared much
+smaller than it really was.  It was built wholly of steel, and was
+painted black.  Three smokestacks, a good distance apart and raking
+well aft, arose in single file amidships; while the bow, long and
+lean and sharp as a knife, plainly advertised that the boat was
+made for speed.  Passing under the stern, we read Streak, painted
+in small white letters.
+</para>
+
+<para>
+Charley and I were consumed with curiosity.  In a few minutes we
+were on board and talking with an engineer who was watching the
+sunrise from the deck.  He was quite willing to satisfy our
+curiosity, and in a few minutes we learned that the Streak had come
+in after dark from San Francisco; that this was what might be
+called the trial trip; and that she was the property of Silas Tate,
+a young mining millionaire of California, whose fad was high-speed
+yachts.  There was some talk about turbine engines, direct
+application of steam, and the absence of pistons, rods, and cranks,
+- all of which was beyond me, for I was familiar only with sailing
+craft; but I did understand the last words of the engineer.
+</para>
+
+<para>
+"Four thousand horse-power and forty-five miles an hour, though you
+wouldn't think it," he concluded proudly.
+</para>
+
+<para>
+"Say it again, man!  Say it again!" Charley exclaimed in an excited
+voice.
+</para>
+
+<para>
+"Four thousand horse-power and forty-five miles an hour," the
+engineer repeated, grinning good-naturedly.
+</para>
+
+<para>
+"Where's the owner?" was Charley's next question.  "Is there any
+way I can speak to him?"
+</para>
+
+<para>
+The engineer shook his head.  "No, I'm afraid not.  He's asleep,
+you see."
+</para>
+
+<para>
+At that moment a young man in blue uniform came on deck farther aft
+and stood regarding the sunrise.
+</para>
+
+<para>
+"There he is, that's him, that's Mr. Tate," said the engineer.
+</para>
+
+<para>
+Charley walked aft and spoke to him, and while he talked earnestly
+the young man listened with an amused expression on his face.  He
+must have inquired about the depth of water close in to the shore
+at Turner's Shipyard, for I could see Charley making gestures and
+explaining.  A few minutes later he came back in high glee.
+</para>
+
+<para>
+"Come on lad," he said.  "On to the dock with you.  We've got
+them!"
+</para>
+
+<para>
+It was our good fortune to leave the Streak when we did, for a
+little later one of the spy fishermen appeared.  Charley and I took
+up our accustomed places, on the stringer-piece, a little ahead of
+the Streak and over our own boat, where we could comfortably watch
+the Lancashire Queen.  Nothing occurred till about nine o'clock,
+when we saw the two Italians leave the ship and pull along their
+side of the triangle toward the shore.  Charley looked as
+unconcerned as could be, but before they had covered a quarter of
+the distance, he whispered to me:
+</para>
+
+<para>
+"Forty-five miles an hour . . . nothing can save them . . . they
+are ours!"
+</para>
+
+<para>
+Slowly the two men rowed along till they were nearly in line with
+the windmill.  This was the point where we always jumped into our
+salmon boat and got up the sail, and the two men, evidently
+expecting it, seemed surprised when we gave no sign.
+</para>
+
+<para>
+When they were directly in line with the windmill, as near to the
+shore as to the ship, and nearer the shore than we had ever allowed
+them before, they grew suspicious.  We followed them through the
+glasses, and saw them standing up in the skiff and trying to find
+out what we were doing.  The spy fisherman, sitting beside us on
+the stringer-piece was likewise puzzled.  He could not understand
+our inactivity.  The men in the skiff rowed nearer the shore, but
+stood up again and scanned it, as if they thought we might be in
+hiding there.  But a man came out on the beach and waved a
+handkerchief to indicate that the coast was clear.  That settled
+them.  They bent to the oars to make a dash for it.  Still Charley
+waited.  Not until they had covered three-quarters of the distance
+from the Lancashire Queen, which left them hardly more than a
+quarter of a mile to gain the shore, did Charley slap me on the
+shoulder and cry:
+</para>
+
+<para>
+"They're ours!  They're ours!"
+</para>
+
+<para>
+We ran the few steps to the side of the Streak and jumped aboard.
+Stern and bow lines were cast off in a jiffy.  The Streak shot
+ahead and away from the wharf.  The spy fisherman we had left
+behind on the stringer-piece pulled out a revolver and fired five
+shots into the air in rapid succession.  The men in the skiff gave
+instant heed to the warning, for we could see them pulling away
+like mad.
+</para>
+
+<para>
+But if they pulled like mad, I wonder how our progress can be
+described?  We fairly flew.  So frightful was the speed with which
+we displaced the water, that a wave rose up on either side our bow
+and foamed aft in a series of three stiff, up-standing waves, while
+astern a great crested billow pursued us hungrily, as though at
+each moment it would fall aboard and destroy us.  The Streak was
+pulsing and vibrating and roaring like a thing alive.  The wind of
+our progress was like a gale - a forty-five-mile gale.  We could
+not face it and draw breath without choking and strangling.  It
+blew the smoke straight back from the mouths of the smoke-stacks at
+a direct right angle to the perpendicular.  In fact, we were
+travelling as fast as an express train.  "We just streaked it," was
+the way Charley told it afterward, and I think his description
+comes nearer than any I can give.
+</para>
+
+<para>
+As for the Italians in the skiff - hardly had we started, it seemed
+to me, when we were on top of them.  Naturally, we had to slow down
+long before we got to them; but even then we shot past like a
+whirlwind and were compelled to circle back between them and the
+shore.  They had rowed steadily, rising from the thwarts at every
+stroke, up to the moment we passed them, when they recognized
+Charley and me.  That took the last bit of fight out of them.  They
+hauled in their oars, and sullenly submitted to arrest.
+</para>
+
+<para>
+"Well, Charley," Neil Partington said, as we discussed it on the
+wharf afterward, "I fail to see where your boasted imagination came
+into play this time."
+</para>
+
+<para>
+But Charley was true to his hobby.  "Imagination?" he demanded,
+pointing to the Streak.  "Look at that! just look at it!  If the
+invention of that isn't imagination, I should like to know what
+is."
+</para>
+
+<para>
+"Of course," he added, "it's the other fellow's imagination, but it
+did the work all the same."
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>CHARLEY'S COUP</title>
+
+
+<para>
+Perhaps our most laughable exploit on the fish patrol, and at the
+same time our most dangerous one, was when we rounded in, at a
+single haul, an even score of wrathful fishermen.  Charley called
+it a "coop," having heard Neil Partington use the term; but I think
+he misunderstood the word, and thought it meant "coop," to catch,
+to trap.  The fishermen, however, coup or coop, must have called it
+a Waterloo, for it was the severest stroke ever dealt them by the
+fish patrol, while they had invited it by open and impudent
+defiance of the law.
+</para>
+
+<para>
+During what is called the "open season" the fishermen might catch
+as many salmon as their luck allowed and their boats could hold.
+But there was one important restriction.  From sun-down Saturday
+night to sun-up Monday morning, they were not permitted to set a
+net.  This was a wise provision on the part of the Fish Commission,
+for it was necessary to give the spawning salmon some opportunity
+to ascend the river and lay their eggs.  And this law, with only an
+occasional violation, had been obediently observed by the Greek
+fishermen who caught salmon for the canneries and the market.
+</para>
+
+<para>
+One Sunday morning, Charley received a telephone call from a friend
+in Collinsville, who told him that the full force of fishermen was
+out with its nets.  Charley and I jumped into our salmon boat and
+started for the scene of the trouble.  With a light favoring wind
+at our back we went through the Carquinez Straits, crossed Suisun
+Bay, passed the Ship Island Light, and came upon the whole fleet at
+work.
+</para>
+
+<para>
+But first let me describe the method by which they worked.  The net
+used is what is known as a gill-net.  It has a simple diamond-
+shaped mesh which measures at least seven and one-half inches
+between the knots.  From five to seven and even eight hundred feet
+in length, these nets are only a few feet wide.  They are not
+stationary, but float with the current, the upper edge supported on
+the surface by floats, the lower edge sunk by means of leaden
+weights,
+</para>
+
+<para>
+This arrangement keeps the net upright in the current and
+effectually prevents all but the smaller fish from ascending the
+river.  The salmon, swimming near the surface, as is their custom,
+run their heads through these meshes, and are prevented from going
+on through by their larger girth of body, and from going back
+because of their gills, which catch in the mesh.  It requires two
+fishermen to set such a net, - one to row the boat, while the
+other, standing in the stern, carefully pays out the net.  When it
+is all out, stretching directly across the stream, the men make
+their boat fast to one end of the net and drift along with it.
+</para>
+
+<para>
+As we came upon the fleet of law-breaking fishermen, each boat two
+or three hundred yards from its neighbors, and boats and nets
+dotting the river as far as we could see, Charley said:
+</para>
+
+<para>
+"I've only one regret, lad, and that is that I have'nt a thousand
+arms so as to be able to catch them all.  As it is, we'll only be
+able to catch one boat, for while we are tackling that one it will
+be up nets and away with the rest."
+</para>
+
+<para>
+As we drew closer, we observed none of the usual flurry and
+excitement which our appearance invariably produced.  Instead, each
+boat lay quietly by its net, while the fishermen favored us with
+not the slightest attention.
+</para>
+
+<para>
+"It's curious," Charley muttered.  "Can it be they don't recognize
+us?"
+</para>
+
+<para>
+I said that it was impossible, and Charley agreed; yet there was a
+whole fleet, manned by men who knew us only too well, and who took
+no more notice of us than if we were a hay scow or a pleasure
+yacht.
+</para>
+
+<para>
+This did not continue to be the case, however, for as we bore down
+upon the nearest net, the men to whom it belonged detached their
+boat and rowed slowly toward the shore.  The rest of the boats
+showed no, sign of uneasiness.
+</para>
+
+<para>
+"That's funny," was Charley's remark.  "But we can confiscate the
+net, at any rate."
+</para>
+
+<para>
+We lowered sail, picked up one end of the net, and began to heave
+it into the boat.  But at the first heave we heard a bullet zip-
+zipping past us on the water, followed by the faint report of a
+rifle.  The men who had rowed ashore were shooting at us.  At the
+next heave a second bullet went zipping past, perilously near.
+Charley took a turn around a pin and sat down.  There were no more
+shots.  But as soon as he began to heave in, the shooting
+recommenced.
+</para>
+
+<para>
+"That settles it," he said, flinging the end of the net overboard.
+"You fellows want it worse than we do, and you can have it."
+</para>
+
+<para>
+We rowed over toward the next net, for Charley was intent on
+finding out whether or not we were face to face with an organized
+defiance.  As we approached, the two fishermen proceeded to cast
+off from their net and row ashore, while the first two rowed back
+and made fast to the net we had abandoned.  And at the second net
+we were greeted by rifle shots till we desisted and went on to the
+third, where the manoeuvre was again repeated.
+</para>
+
+<para>
+Then we gave it up, completely routed, and hoisted sail and started
+on the long windward beat back to Benicia.  A number of Sundays
+went by, on each of which the law was persistently violated.  Yet,
+short of an armed force of soldiers, we could do nothing.  The
+fishermen had hit upon a new idea and were using it for all it was
+worth, while there seemed no way by which we could get the better
+of them.
+</para>
+
+<para>
+About this time Neil Partington happened along from the Lower Bay,
+where he had been for a number of weeks.  With him was Nicholas,
+the Greek boy who had helped us in our raid on the oyster pirates,
+and the pair of them took a hand.  We made our arrangements
+carefully.  It was planned that while Charley and I tackled the
+nets, they were to be hidden ashore so as to ambush the fishermen
+who landed to shoot at us.
+</para>
+
+<para>
+It was a pretty plan.  Even Charley said it was.  But we reckoned
+not half so well as the Greeks.  They forestalled us by ambushing
+Neil and Nicholas and taking them prisoners, while, as of old,
+bullets whistled about our ears when Charley and I attempted to
+take possession of the nets.  When we were again beaten off, Neil
+Partington and Nicholas were released.  They were rather shamefaced
+when they put in an appearance, and Charley chaffed them
+unmercifully.  But Neil chaffed back, demanding to know why
+Charley's imagination had not long since overcome the difficulty.
+</para>
+
+<para>
+"Just you wait; the idea'll come all right," Charley promised.
+</para>
+
+<para>
+"Most probably," Neil agreed.  "But I'm afraid the salmon will be
+exterminated first, and then there will be no need for it when it
+does come."
+</para>
+
+<para>
+Neil Partington, highly disgusted with his adventure, departed for
+the Lower Bay, taking Nicholas with him, and Charley and I were
+left to our own resources.  This meant that the Sunday fishing
+would be left to itself, too, until such time as Charley's idea
+happened along.  I puzzled my head a good deal to find out some way
+of checkmating the Greeks, as also did Charley, and we broached a
+thousand expedients which on discussion proved worthless.
+</para>
+
+<para>
+The fishermen, on the other hand, were in high feather, and their
+boasts went up and down the river to add to our discomfiture.
+Among all classes of them we became aware of a growing
+insubordination.  We were beaten, and they were losing respect for
+us.  With the loss of respect, contempt began to arise.  Charley
+began to be spoken of as the "olda woman," and I received my rating
+as the "pee-wee kid."  The situation was fast becoming unbearable,
+and we knew that we should have to deliver a stunning stroke at the
+Greeks in order to regain the old-time respect in which we had
+stood.
+</para>
+
+<para>
+Then one morning the idea came.  We were down on Steamboat Wharf,
+where the river steamers made their landings, and where we found a
+group of amused long-shoremen and loafers listening to the hard-
+luck tale of a sleepy-eyed young fellow in long sea-boots.  He was
+a sort of amateur fisherman, he said, fishing for the local market
+of Berkeley.  Now Berkeley was on the Lower Bay, thirty miles away.
+On the previous night, he said, he had set his net and dozed off to
+sleep in the bottom of the boat.
+</para>
+
+<para>
+The next he knew it was morning, and he opened his eyes to find his
+boat rubbing softly against the piles of Steamboat Wharf at
+Benicia.  Also he saw the river steamer Apache lying ahead of him,
+and a couple of deck-hands disentangling the shreds of his net from
+the paddle-wheel.  In short, after he had gone to sleep, his
+fisherman's riding light had gone out, and the Apache had run over
+his net.  Though torn pretty well to pieces, the net in some way
+still remained foul, and he had had a thirty-mile tow out of his
+course.
+</para>
+
+<para>
+Charley nudged me with his elbow.  I grasped his thought on the
+instant, but objected:
+</para>
+
+<para>
+"We can't charter a steamboat."
+</para>
+
+<para>
+"Don't intend to," he rejoined.  "But let's run over to Turner's
+Shipyard.  I've something in my mind there that may be of use to
+us."
+</para>
+
+<para>
+And over we went to the shipyard, where Charley led the way to the
+Mary Rebecca, lying hauled out on the ways, where she was being
+cleaned and overhauled.  She was a scow-schooner we both knew well,
+carrying a cargo of one hundred and forty tons and a spread of
+canvas greater than other schooner on the bay.
+</para>
+
+<para>
+"How d'ye do, Ole," Charley greeted a big blue-shirted Swede who
+was greasing the jaws of the main gaff with a piece of pork rind.
+</para>
+
+<para>
+Ole grunted, puffed away at his pipe, and went on greasing.  The
+captain of a bay schooner is supposed to work with his hands just
+as well as the men.
+</para>
+
+<para>
+Ole Ericsen verified Charley's conjecture that the Mary Rebecca, as
+soon as launched, would run up the San Joaquin River nearly to
+Stockton for a load of wheat.  Then Charley made his proposition,
+and Ole Ericsen shook his head.
+</para>
+
+<para>
+"Just a hook, one good-sized hook," Charley pleaded.
+</para>
+
+<para>
+"No, Ay tank not," said Ole Ericsen.  "Der Mary Rebecca yust hang
+up on efery mud-bank with that hook.  Ay don't want to lose der
+Mary Rebecca.  She's all Ay got."
+</para>
+
+<para>
+"No, no," Charley hurried to explain.  "We can put the end of the
+hook through the bottom from the outside, and fasten it on the
+inside with a nut.  After it's done its work, why, all we have to
+do is to go down into the hold, unscrew the nut, and out drops the
+hook.  Then drive a wooden peg into the hole, and the Mary Rebecca
+will be all right again."
+</para>
+
+<para>
+Ole Ericsen was obstinate for a long time; but in the end, after we
+had had dinner with him, he was brought round to consent.
+</para>
+
+<para>
+"Ay do it, by Yupiter!" he said, striking one huge fist into the
+palm of the other hand.  "But yust hurry you up wid der hook.  Der
+Mary Rebecca slides into der water to-night."
+</para>
+
+<para>
+It was Saturday, and Charley had need to hurry.  We headed for the
+shipyard blacksmith shop, where, under Charley's directions, a most
+generously curved book of heavy steel was made.  Back we hastened
+to the Mary Rebecca.  Aft of the great centre-board case, through
+what was properly her keel, a hole was bored.  The end of the hook
+was inserted from the outside, and Charley, on the inside, screwed
+the nut on tightly.  As it stood complete, the hook projected over
+a foot beneath the bottom of the schooner.  Its curve was something
+like the curve of a sickle, but deeper.
+</para>
+
+<para>
+In the late afternoon the Mary Rebecca was launched, and
+preparations were finished for the start up-river next morning.
+Charley and Ole intently studied the evening sky for signs of wind,
+for without a good breeze our project was doomed to failure.  They
+agreed that there were all the signs of a stiff westerly wind - not
+the ordinary afternoon sea-breeze, but a half-gale, which even then
+was springing up.
+</para>
+
+<para>
+Next morning found their predictions verified.  The sun was shining
+brightly, but something more than a half-gale was shrieking up the
+Carquinez Straits, and the Mary Rebecca got under way with two
+reefs in her mainsail and one in her foresail.  We found it quite
+rough in the Straits and in Suisun Bay; but as the water grew more
+land-locked it became calm, though without let-up in the wind.
+</para>
+
+<para>
+Off Ship Island Light the reefs were shaken out, and at Charley's
+suggestion a big fisherman's staysail was made all ready for
+hoisting, and the maintopsail, bunched into a cap at the masthead,
+was overhauled so that it could be set on an instant's notice.
+</para>
+
+<para>
+We were tearing along, wing-and-wing, before the wind, foresail to
+starboard and mainsail to port, as we came upon the salmon fleet.
+There they were, boats and nets, as on that first Sunday when they
+had bested us, strung out evenly over the river as far as we could
+see.  A narrow space on the right-hand side of the channel was left
+clear for steamboats, but the rest of the river was covered with
+the wide-stretching nets.  The narrow space was our logical course,
+but Charley, at the wheel, steered the Mary Rebecca straight for
+the nets.  This did not cause any alarm among the fishermen,
+because up-river sailing craft are always provided with "shoes" on
+the ends of their keels, which permit them to slip over the nets
+without fouling them.
+</para>
+
+<para>
+"Now she takes it!" Charley cried, as we dashed across the middle
+of a line of floats which marked a net.  At one end of this line
+was a small barrel buoy, at the other the two fishermen in their
+boat.  Buoy and boat at once began to draw together, and the
+fishermen to cry out, as they were jerked after us.  A couple of
+minutes later we hooked a second net, and then a third, and in this
+fashion we tore straight up through the centre of the fleet.
+</para>
+
+<para>
+The consternation we spread among the fishermen was tremendous.  As
+fast as we hooked a net the two ends of it, buoy and boat, came
+together as they dragged out astern; and so many buoys and boats,
+coming together at such breakneck speed, kept the fishermen on the
+jump to avoid smashing into one another.  Also, they shouted at us
+like mad to heave to into the wind, for they took it as some
+drunken prank on the part of scow-sailors, little dreaming that we
+were the fish patrol.
+</para>
+
+<para>
+The drag of a single net is very heavy, and Charley and Ole Ericsen
+decided that even in such a wind ten nets were all the Mary Rebecca
+could take along with her.  So when we had hooked ten nets, with
+ten boats containing twenty men streaming along behind us, we
+veered to the left out of the fleet and headed toward Collinsville.
+</para>
+
+<para>
+We were all jubilant.  Charley was handling the wheel as though he
+were steering the winning yacht home in a race.  The two sailors
+who made up the crew of the Mary Rebecca, were grinning and joking.
+Ole Ericsen was rubbing his huge hands in child-like glee.
+</para>
+
+<para>
+"Ay tank you fish patrol fallers never ban so lucky as when you
+sail with Ole Ericsen," he was saying, when a rifle cracked sharply
+astern, and a bullet gouged along the newly painted cabin, glanced
+on a nail, and sang shrilly onward into space.
+</para>
+
+<para>
+This was too much for Ole Ericsen.  At sight of his beloved
+paintwork thus defaced, he jumped up and shook his fist at the
+fishermen; but a second bullet smashed into the cabin not six
+inches from his head, and he dropped down to the deck under cover
+of the rail.
+</para>
+
+<para>
+All the fishermen had rifles, and they now opened a general
+fusillade.  We were all driven to cover - even Charley, who was
+compelled to desert the wheel.  Had it not been for the heavy drag
+of the nets, we would inevitably have broached to at the mercy of
+the enraged fishermen.  But the nets, fastened to the bottom of the
+Mary Rebecca well aft, held her stern into the wind, and she
+continued to plough on, though somewhat erratically.
+</para>
+
+<para>
+Charley, lying on the deck, could just manage to reach the lower
+spokes of the wheel; but while he could steer after a fashion, it
+was very awkward.  Ole Ericsen bethought himself of a large piece
+of sheet steel in the empty hold.
+</para>
+
+<para>
+It was in fact a plate from the side of the New Jersey, a steamer
+which had recently been wrecked outside the Golden Gate, and in the
+salving of which the Mary Rebecca had taken part.
+</para>
+
+<para>
+Crawling carefully along the deck, the two sailors, Ole, and myself
+got the heavy plate on deck and aft, where we reared it as a shield
+between the wheel and the fishermen.  The bullets whanged and
+banged against it till it rang like a bull's-eye, but Charley
+grinned in its shelter, and coolly went on steering.
+</para>
+
+<para>
+So we raced along, behind us a howling, screaming bedlam of
+wrathful Greeks, Collinsville ahead, and bullets spat-spatting all
+around us.
+</para>
+
+<para>
+"Ole," Charley said in a faint voice, "I don't know what we're
+going to do."
+</para>
+
+<para>
+Ole Ericsen, lying on his back close to the rail and grinning
+upward at the sky, turned over on his side and looked at him.  "Ay
+tank we go into Collinsville yust der same," he said.
+</para>
+
+<para>
+"But we can't stop," Charley groaned.  "I never thought of it, but
+we can't stop."
+</para>
+
+<para>
+A look of consternation slowly overspread Ole Ericsen's broad face.
+It was only too true.  We had a hornet's nest on our hands, and to
+stop at Collinsville would be to have it about our ears.
+</para>
+
+<para>
+"Every man Jack of them has a gun," one of the sailors remarked
+cheerfully.
+</para>
+
+<para>
+"Yes, and a knife, too," the other sailor added.
+</para>
+
+<para>
+It was Ole Ericsen's turn to groan.  "What for a Svaidish faller
+like me monkey with none of my biziness, I don't know," he
+soliloquized.
+</para>
+
+<para>
+A bullet glanced on the stern and sang off to starboard like a
+spiteful bee.  "There's nothing to do but plump the Mary Rebecca
+ashore and run for it," was the verdict of the first cheerful
+sailor.
+</para>
+
+<para>
+"And leaf der Mary Rebecca?" Ole demanded, with unspeakable horror
+in his voice.
+</para>
+
+<para>
+"Not unless you want to," was the response.  "But I don't want to
+be within a thousand miles of her when those fellers come aboard" -
+indicating the bedlam of excited Greeks towing behind.
+</para>
+
+<para>
+We were right in at Collinsville then, and went foaming by within
+biscuit-toss of the wharf.
+</para>
+
+<para>
+"I only hope the wind holds out," Charley said, stealing a glance
+at our prisoners.
+</para>
+
+<para>
+"What of der wind?" Ole demanded disconsolately.  "Der river will
+not hold out, and then . . . and then . . ."
+</para>
+
+<para>
+"It's head for tall timber, and the Greeks take the hindermost,"
+adjudged the cheerful sailor, while Ole was stuttering over what
+would happen when we came to the end of the river.
+</para>
+
+<para>
+We had now reached a dividing of the ways.  To the left was the
+mouth of the Sacramento River, to the right the mouth of the San
+Joaquin.  The cheerful sailor crept forward and jibed over the
+foresail as Charley put the helm to starboard and we swerved to the
+right into the San Joaquin.  The wind, from which we had been
+running away on an even keel, now caught us on our beam, and the
+Mary Rebecca was pressed down on her port side as if she were about
+to capsize.
+</para>
+
+<para>
+Still we dashed on, and still the fishermen dashed on behind.  The
+value of their nets was greater than the fines they would have to
+pay for violating the fish laws; so to cast off from their nets and
+escape, which they could easily do, would profit them nothing.
+Further, they remained by their nets instinctively, as a sailor
+remains by his ship.  And still further, the desire for vengeance
+was roused, and we could depend upon it that they would follow us
+to the ends of the earth, if we undertook to tow them that far.
+</para>
+
+<para>
+The rifle-firing had ceased, and we looked astern to see what our
+prisoners were doing.  The boats were strung along at unequal
+distances apart, and we saw the four nearest ones bunching
+together.  This was done by the boat ahead trailing a small rope
+astern to the one behind.  When this was caught, they would cast
+off from their net and heave in on the line till they were brought
+up to the boat in front.  So great was the speed at which we were
+travelling, however, that this was very slow work.  Sometimes the
+men would strain to their utmost and fail to get in an inch of the
+rope; at other times they came ahead more rapidly.
+</para>
+
+<para>
+When the four boats were near enough together for a man to pass
+from one to another, one Greek from each of three got into the
+nearest boat to us, taking his rifle with him.  This made five in
+the foremost boat, and it was plain that their intention was to
+board us.  This they undertook to do, by main strength and sweat,
+running hand over hand the float-line of a net.  And though it was
+slow, and they stopped frequently to rest, they gradually drew
+nearer.
+</para>
+
+<para>
+Charley smiled at their efforts, and said, "Give her the topsail,
+Ole."
+</para>
+
+<para>
+The cap at the mainmast head was broken out, and sheet and downhaul
+pulled flat, amid a scattering rifle fire from the boats; and the
+Mary Rebecca lay over and sprang ahead faster than ever.
+</para>
+
+<para>
+But the Greeks were undaunted.  Unable, at the increased speed, to
+draw themselves nearer by means of their hands, they rigged from
+the blocks of their boat sail what sailors call a "watch-tackle."
+One of them, held by the legs by his mates, would lean far over the
+bow and make the tackle fast to the float-line.  Then they would
+heave in on the tackle till the blocks were together, when the
+manoeuvre would be repeated.
+</para>
+
+<para>
+"Have to give her the staysail," Charley said.
+</para>
+
+<para>
+Ole Ericsen looked at the straining Mary Rebecca and shook his
+head.  "It will take der masts out of her," he said.
+</para>
+
+<para>
+"And we'll be taken out of her if you don't," Charley replied.
+</para>
+
+<para>
+Ole shot an anxious glance at his masts, another at the boat load
+of armed Greeks, and consented.
+</para>
+
+<para>
+The five men were in the bow of the boat - a bad place when a craft
+is towing.  I was watching the behavior of their boat as the great
+fisherman's staysail, far, far larger than the top-sail and used
+only in light breezes, was broken out.  As the Mary Rebecca lurched
+forward with a tremendous jerk, the nose of the boat ducked down
+into the water, and the men tumbled over one another in a wild rush
+into the stern to save the boat from being dragged sheer under
+water.
+</para>
+
+<para>
+"That settles them!" Charley remarked, though he was anxiously
+studying the behavior of the Mary Rebecca, which was being driven
+under far more canvas than she was rightly able to carry.
+</para>
+
+<para>
+"Next stop is Antioch!" announced the cheerful sailor, after the
+manner of a railway conductor.  "And next comes Merryweather!"
+</para>
+
+<para>
+"Come here, quick," Charley said to me.
+</para>
+
+<para>
+I crawled across the deck and stood upright beside him in the
+shelter of the sheet steel.
+</para>
+
+<para>
+"Feel in my inside pocket," he commanded, "and get my notebook.
+That's right.  Tear out a blank page and write what I tell you."
+</para>
+
+<para>
+And this is what I wrote:
+</para>
+
+<para>
+
+Telephone to Merryweather, to the sheriff, the constable, or the
+judge.  Tell them we are coming and to turn out the town.  Arm
+everybody.  Have them down on the wharf to meet us or we are gone
+gooses.
+</para>
+
+<para>
+
+Now make it good and fast to that marlin-spike, and stand by to
+toss it ashore."
+</para>
+
+<para>
+I did as he directed.  By then we were close to Antioch.  The wind
+was shouting through our rigging, the Mary Rebecca was half over on
+her side and rushing ahead like an ocean greyhound.  The seafaring
+folk of Antioch had seen us breaking out topsail and staysail, a
+most reckless performance in such weather, and had hurried to the
+wharf-ends in little groups to find out what was the matter.
+</para>
+
+<para>
+Straight down the water front we boomed, Charley edging in till a
+man could almost leap ashore.  When he gave the signal I tossed the
+marlinspike.  It struck the planking of the wharf a resounding
+smash, bounced along fifteen or twenty feet, and was pounced upon
+by the amazed onlookers.
+</para>
+
+<para>
+It all happened in a flash, for the next minute Antioch was behind
+and we were heeling it up the San Joaquin toward Merryweather, six
+miles away.  The river straightened out here into its general
+easterly course, and we squared away before the wind, wing-and-wing
+once more, the foresail bellying out to starboard.
+</para>
+
+<para>
+Ole Ericsen seemed sunk into a state of stolid despair.  Charley
+and the two sailors were looking hopeful, as they had good reason
+to be.  Merryweather was a coal-mining town, and, it being Sunday,
+it was reasonable to expect the men to be in town.  Further, the
+coal-miners had never lost any love for the Greek fishermen, and
+were pretty certain to render us hearty assistance.
+</para>
+
+<para>
+We strained our eyes for a glimpse of the town, and the first sight
+we caught of it gave us immense relief.  The wharves were black
+with men.  As we came closer, we could see them still arriving,
+stringing down the main street, guns in their hands and on the run.
+Charley glanced astern at the fishermen with a look of ownership in
+his eye which till then had been missing.  The Greeks were plainly
+overawed by the display of armed strength and were putting their
+own rifles away.
+</para>
+
+<para>
+We took in topsail and staysail, dropped the main peak, and as we
+got abreast of the principal wharf jibed the mainsail.  The Mary
+Rebecca shot around into the wind, the captive fishermen describing
+a great arc behind her, and forged ahead till she lost way, when
+lines we're flung ashore and she was made fast.  This was
+accomplished under a hurricane of cheers from the delighted miners.
+</para>
+
+<para>
+Ole Ericsen heaved a great sigh.  "Ay never tank Ay see my wife
+never again," he confessed.
+</para>
+
+<para>
+"Why, we were never in any danger," said Charley.
+</para>
+
+<para>
+Ole looked at him incredulously.
+</para>
+
+<para>
+"Sure, I mean it," Charley went on.  "All we had to do, any time,
+was to let go our end - as I am going to do now, so that those
+Greeks can untangle their nets."
+</para>
+
+<para>
+He went below with a monkey-wrench, unscrewed the nut, and let the
+hook drop off.  When the Greeks had hauled their nets into their
+boats and made everything shipshape, a posse of citizens took them
+off our hands and led them away to jail.
+</para>
+
+<para>
+"Ay tank Ay ban a great big fool," said Ole Ericsen.  But he
+changed his mind when the admiring townspeople crowded aboard to
+shake hands with him, and a couple of enterprising newspaper men
+took photographs of the Mary Rebecca and her captain.
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>DEMETRIOS CONTOS</title>
+
+
+<para>
+It must not be thought, from what I have told of the Greek
+fishermen, that they were altogether bad.  Far from it.  But they
+were rough men, gathered together in isolated communities and
+fighting with the elements for a livelihood.  They lived far away
+from the law and its workings, did not understand it, and thought
+it tyranny.  Especially did the fish laws seem tyrannical.  And
+because of this, they looked upon the men of the fish patrol as
+their natural enemies.
+</para>
+
+<para>
+We menaced their lives, or their living, which is the same thing,
+in many ways.  We confiscated illegal traps and nets, the materials
+of which had cost them considerable sums and the making of which
+required weeks of labor.  We prevented them from catching fish at
+many times and seasons, which was equivalent to preventing them
+from making as good a living as they might have made had we not
+been in existence.  And when we captured them, they were brought
+into the courts of law, where heavy cash fines were collected from
+them.  As a result, they hated us vindictively.  As the dog is the
+natural enemy of the cat, the snake of man, so were we of the fish
+patrol the natural enemies of the fishermen.
+</para>
+
+<para>
+But it is to show that they could act generously as well as hate
+bitterly that this story of Demetrios Contos is told.  Demetrios
+Contos lived in Vallejo.  Next to Big Alec, he was the largest,
+bravest, and most influential man among the Greeks.  He had given
+us no trouble, and I doubt if he would ever have clashed with us
+had he not invested in a new salmon boat.  This boat was the cause
+of all the trouble.  He had had it built upon his own model, in
+which the lines of the general salmon boat were somewhat modified.
+</para>
+
+<para>
+To his high elation he found his new boat very fast - in fact,
+faster than any other boat on the bay or rivers.  Forthwith he grew
+proud and boastful:  and, our raid with the Mary Rebecca on the
+Sunday salmon fishers having wrought fear in their hearts, he sent
+a challenge up to Benicia.  One of the local fishermen conveyed it
+to us; it was to the effect that Demetrios Contos would sail up
+from Vallejo on the following Sunday, and in the plain sight of
+Benicia set his net and catch salmon, and that Charley Le Grant,
+patrolman, might come and get him if he could.  Of course Charley
+and I had heard nothing of the new boat.  Our own boat was pretty
+fast, and we were not afraid to have a brush with any other that
+happened along.
+</para>
+
+<para>
+Sunday came.  The challenge had been bruited abroad, and the
+fishermen and seafaring folk of Benicia turned out to a man,
+crowding Steamboat Wharf till it looked like the grand stand at a
+football match.  Charley and I had been sceptical, but the fact of
+the crowd convinced us that there was something in Demetrios
+Contos's dare.
+</para>
+
+<para>
+In the afternoon, when the sea-breeze had picked up in strength,
+his sail hove into view as he bowled along before the wind.  He
+tacked a score of feet from the wharf, waved his hand theatrically,
+like a knight about to enter the lists, received a hearty cheer in
+return, and stood away into the Straits for a couple of hundred
+yards.  Then he lowered sail, and, drifting the boat sidewise by
+means of the wind, proceeded to set his net.  He did not set much
+of it, possibly fifty feet; yet Charley and I were thunderstruck at
+the man's effrontery.  We did not know at the time, but we learned
+afterward, that the net he used was old and worthless.  It could
+catch fish, true; but a catch of any size would have torn it to
+pieces.
+</para>
+
+<para>
+Charley shook his head and said:
+</para>
+
+<para>
+"I confess, it puzzles me.  What if he has out only fifty feet?  He
+could never get it in if we once started for him.  And why does he
+come here anyway, flaunting his law-breaking in our faces?  Right
+in our home town, too."
+</para>
+
+<para>
+Charley's voice took on an aggrieved tone, and he continued for
+some minutes to inveigh against the brazenness of Demetrios Contos.
+</para>
+
+<para>
+In the meantime, the man in question was lolling in the stern of
+his boat and watching the net floats.  When a large fish is meshed
+in a gill-net, the floats by their agitation advertise the fact.
+And they evidently advertised it to Demetrios, for he pulled in
+about a dozen feet of net, and held aloft for a moment, before he
+flung it into the bottom of the boat, a big, glistening salmon.  It
+was greeted by the audience on the wharf with round after round of
+cheers.  This was more than Charley could stand.
+</para>
+
+<para>
+"Come on, lad," he called to me; and we lost no time jumping into
+our salmon boat and getting up sail.
+</para>
+
+<para>
+The crowd shouted warning to Demetrios, and as we darted out from
+the wharf we saw him slash his worthless net clear with a long
+knife.  His sail was all ready to go up, and a moment later it
+fluttered in the sunshine.  He ran aft, drew in the sheet, and
+filled on the long tack toward the Contra Costa Hills.
+</para>
+
+<para>
+By this time we were not more than thirty feet astern.  Charley was
+jubilant.  He knew our boat was fast, and he knew, further, that in
+fine sailing few men were his equals.  He was confident that we
+should surely catch Demetrios, and I shared his confidence.  But
+somehow we did not seem to gain.
+</para>
+
+<para>
+It was a pretty sailing breeze.  We were gliding sleekly through
+the water, but Demetrios was slowly sliding away from us.  And not
+only was he going faster, but he was eating into the wind a
+fraction of a point closer than we.  This was sharply impressed
+upon us when he went about under the Contra Costa Hills and passed
+us on the other tack fully one hundred feet dead to windward.
+</para>
+
+<para>
+"Whew!" Charley exclaimed.  "Either that boat is a daisy, or we've
+got a five-gallon coal-oil can fast to our keel!"
+</para>
+
+<para>
+It certainly looked it one way or the other.  And by the time
+Demetrios made the Sonoma Hills, on the other side of the Straits,
+we were so hopelessly outdistanced that Charley told me to slack
+off the sheet, and we squared away for Benicia.  The fishermen on
+Steamboat Wharf showered us with ridicule when we returned and tied
+up.  Charley and I got out and walked away, feeling rather
+sheepish, for it is a sore stroke to one's pride when he thinks he
+has a good boat and knows how to sail it, and another man comes
+along and beats him.
+</para>
+
+<para>
+Charley mooned over it for a couple of days; then word was brought
+to us, as before, that on the next Sunday Demetrios Contos would
+repeat his performance.  Charley roused himself.  He had our boat
+out of the water, cleaned and repainted its bottom, made a trifling
+alteration about the centre-board, overhauled the running gear, and
+sat up nearly all of Saturday night sewing on a new and much larger
+sail.  So large did he make it, in fact, that additional ballast
+was imperative, and we stowed away nearly five hundred extra pounds
+of old railroad iron in the bottom of the boat.
+</para>
+
+<para>
+Sunday came, and with it came Demetrios Contos, to break the law
+defiantly in open day.  Again we had the afternoon sea-breeze, and
+again Demetrios cut loose some forty or more feet of his rotten
+net, and got up sail and under way under our very noses.  But he
+had anticipated Charley's move, and his own sail peaked higher than
+ever, while a whole extra cloth had been added to the after leech.
+</para>
+
+<para>
+It was nip and tuck across to the Contra Costa Hills, neither of us
+seeming to gain or to lose.  But by the time we had made the return
+tack to the Sonoma Hills, we could see that, while we footed it at
+about equal speed, Demetrios had eaten into the wind the least bit
+more than we.  Yet Charley was sailing our boat as finely and
+delicately as it was possible to sail it, and getting more out of
+it than he ever had before.
+</para>
+
+<para>
+Of course, he could have drawn his revolver and fired at Demetrios;
+but we had long since found it contrary to our natures to shoot at
+a fleeing man guilty of only a petty offence.  Also a sort of tacit
+agreement seemed to have been reached between the patrolmen and the
+fishermen.  If we did not shoot while they ran away, they, in turn,
+did not fight if we once laid hands on them.  Thus Demetrios Contos
+ran away from us, and we did no more than try our best to overtake
+him; and, in turn, if our boat proved faster than his, or was
+sailed better, he would, we knew, make no resistance when we caught
+up with him.
+</para>
+
+<para>
+With our large sails and the healthy breeze romping up the
+Carquinez Straits, we found that our sailing was what is called
+"ticklish."  We had to be constantly on the alert to avoid a
+capsize, and while Charley steered I held the main-sheet in my hand
+with but a single turn round a pin, ready to let go at any moment.
+Demetrios, we could see, sailing his boat alone, had his hands
+full.
+</para>
+
+<para>
+But it was a vain undertaking for us to attempt to catch him.  Out
+of his inner consciousness he had evolved a boat that was better
+than ours.  And though Charley sailed fully as well, if not the
+least bit better, the boat he sailed was not so good as the
+Greek's.
+</para>
+
+<para>
+"Slack away the sheet," Charley commanded; and as our boat fell off
+before the wind, Demetrios's mocking laugh floated down to us.
+</para>
+
+<para>
+Charley shook his head, saying, "It's no use.  Demetrios has the
+better boat.  If he tries his performance again, we must meet it
+with some new scheme."
+</para>
+
+<para>
+This time it was my imagination that came to the rescue.
+</para>
+
+<para>
+"What's the matter," I suggested, on the Wednesday following, "with
+my chasing Demetrios in the boat next Sunday, while you wait for
+him on the wharf at Vallejo when he arrives?"
+</para>
+
+<para>
+Charley considered it a moment and slapped his knee.
+</para>
+
+<para>
+"A good idea!  You're beginning to use that head of yours.  A
+credit to your teacher, I must say."
+</para>
+
+<para>
+"But you mustn't chase him too far," he went on, the next moment,
+"or he'll head out into San Pablo Bay instead of running home to
+Vallejo, and there I'll be, standing lonely on the wharf and
+waiting in vain for him to arrive."
+</para>
+
+<para>
+On Thursday Charley registered an objection to my plan.
+</para>
+
+<para>
+"Everybody'll know I've gone to Vallejo, and you can depend upon it
+that Demetrios will know, too.  I'm afraid we'll have to give up
+the idea."
+</para>
+
+<para>
+This objection was only too valid, and for the rest of the day I
+struggled under my disappointment.  But that night a new way seemed
+to open to me, and in my eagerness I awoke Charley from a sound
+sleep.
+</para>
+
+<para>
+"Well," he grunted, "what's the matter?  House afire?"
+</para>
+
+<para>
+"No," I replied, "but my head is.  Listen to this.  On Sunday you
+and I will be around Benicia up to the very moment Demetrios's sail
+heaves into sight.  This will lull everybody's suspicions.  Then,
+when Demetrios's sail does heave in sight, do you stroll leisurely
+away and up-town.  All the fishermen will think you're beaten and
+that you know you're beaten."
+</para>
+
+<para>
+"So far, so good," Charley commented, while I paused to catch
+breath.
+</para>
+
+<para>
+"And very good indeed," I continued proudly.  "You stroll
+carelessly up-town, but when you're once out of sight you leg it
+for all you're worth for Dan Maloney's.  Take the little mare of
+his, and strike out on the country road for Vallejo.  The road's in
+fine condition, and you can make it in quicker time than Demetrios
+can beat all the way down against the wind."
+</para>
+
+<para>
+"And I'll arrange right away for the mare, first thing in the
+morning," Charley said, accepting the modified plan without
+hesitation.
+</para>
+
+<para>
+"But, I say," he said, a little later, this time waking me out of a
+sound sleep.
+</para>
+
+<para>
+I could hear him chuckling in the dark.
+</para>
+
+<para>
+"I say, lad, isn't it rather a novelty for the fish patrol to be
+taking to horseback?"
+</para>
+
+<para>
+"Imagination," I answered.  "It's what you're always preaching -
+'keep thinking one thought ahead of the other fellow, and you're
+bound to win out.'"
+</para>
+
+<para>
+"He! he!" he chuckled.  "And if one thought ahead, including a
+mare, doesn't take the other fellow's breath away this time, I'm
+not your humble servant, Charley Le Grant."
+</para>
+
+<para>
+"But can you manage the boat alone?" he asked, on Friday.
+"Remember, we've a ripping big sail on her."
+</para>
+
+<para>
+I argued my proficiency so well that he did not refer to the matter
+again till Saturday, when he suggested removing one whole cloth
+from the after leech.  I guess it was the disappointment written on
+my face that made him desist; for I, also, had a pride in my boat-
+sailing abilities, and I was almost wild to get out alone with the
+big sail and go tearing down the Carquinez Straits in the wake of
+the flying Greek.
+</para>
+
+<para>
+As usual, Sunday and Demetrios Contos arrived together.  It had
+become the regular thing for the fishermen to assemble on Steamboat
+Wharf to greet his arrival and to laugh at our discomfiture.  He
+lowered sail a couple of hundred yards out and set his customary
+fifty feet of rotten net.
+</para>
+
+<para>
+"I suppose this nonsense will keep up as long as his old net holds
+out," Charley grumbled, with intention, in the hearing of several
+of the Greeks.
+</para>
+
+<para>
+"Den I give-a heem my old-a net-a," one of them spoke up, promptly
+and maliciously,
+</para>
+
+<para>
+"I don't care," Charley answered.  "I've got some old net myself he
+can have - if he'll come around and ask for it."
+</para>
+
+<para>
+They all laughed at this, for they could afford to be sweet-
+tempered with a man so badly outwitted as Charley was.
+</para>
+
+<para>
+"Well, so long, lad," Charley called to me a moment later.  "I
+think I'll go up-town to Maloney's."
+</para>
+
+<para>
+"Let me take the boat out?" I asked.
+</para>
+
+<para>
+"If you want to," was his answer, as he turned on his heel and
+walked slowly away.
+</para>
+
+<para>
+Demetrios pulled two large salmon out of his net, and I jumped into
+the boat.  The fishermen crowded around in a spirit of fun, and
+when I started to get up sail overwhelmed me with all sorts of
+jocular advice.  They even offered extravagant bets to one another
+that I would surely catch Demetrios, and two of them, styling
+themselves the committee of judges, gravely asked permission to
+come along with me to see how I did it.
+</para>
+
+<para>
+But I was in no hurry.  I waited to give Charley all the time I
+could, and I pretended dissatisfaction with the stretch of the sail
+and slightly shifted the small tackle by which the huge sprit
+forces up the peak.  It was not until I was sure that Charley had
+reached Dan Maloney's and was on the little mare's back, that I
+cast off from the wharf and gave the big sail to the wind.  A stout
+puff filled it and suddenly pressed the lee gunwale down till a
+couple of buckets of water came inboard.  A little thing like this
+will happen to the best small-boat sailors, and yet, though I
+instantly let go the sheet and righted, I was cheered
+sarcastically, as though I had been guilty of a very awkward
+blunder.
+</para>
+
+<para>
+When Demetrios saw only one person in the fish patrol boat, and
+that one a boy, he proceeded to play with me.  Making a short tack
+out, with me not thirty feet behind, he returned, with his sheet a
+little free, to Steamboat Wharf.  And there he made short tacks,
+and turned and twisted and ducked around, to the great delight of
+his sympathetic audience.  I was right behind him all the time, and
+I dared to do whatever he did, even when he squared away before the
+wind and jibed his big sail over - a most dangerous trick with such
+a sail in such a wind.
+</para>
+
+<para>
+He depended upon the brisk sea breeze and the strong ebb-tide,
+which together kicked up a nasty sea, to bring me to grief.  But I
+was on my mettle, and never in all my life did I sail a boat better
+than on that day.  I was keyed up to concert pitch, my brain was
+working smoothly and quickly, my hands never fumbled once, and it
+seemed that I almost divined the thousand little things which a
+small-boat sailor must be taking into consideration every second.
+</para>
+
+<para>
+It was Demetrios who came to grief instead.  Something went wrong
+with his centre-board, so that it jammed in the case and would not
+go all the way down.  In a moment's breathing space, which he had
+gained from me by a clever trick, I saw him working impatiently
+with the centre-board, trying to force it down.  I gave him little
+time, and he was compelled quickly to return to the tiller and
+sheet.
+</para>
+
+<para>
+The centre-board made him anxious.  He gave over playing with me,
+and started on the long beat to Vallejo.  To my joy, on the first
+long tack across, I found that I could eat into the wind just a
+little bit closer than he.  Here was where another man in the boat
+would have been of value to him; for, with me but a few feet
+astern, he did not dare let go the tiller and run amidships to try
+to force down the centre-board.
+</para>
+
+<para>
+Unable to hang on as close in the eye of the wind as formerly, he
+proceeded to slack his sheet a trifle and to ease off a bit, in
+order to outfoot me.  This I permitted him to do till I had worked
+to windward, when I bore down upon him.  As I drew close, he
+feinted at coming about.  This led me to shoot into the wind to
+forestall him.  But it was only a feint, cleverly executed, and he
+held back to his course while I hurried to make up lost ground.
+</para>
+
+<para>
+He was undeniably smarter than I when it came to manoeuvring.  Time
+after time I all but had him, and each time he tricked me and
+escaped.  Besides, the wind was freshening, constantly, and each of
+us had his hands full to avoid capsizing.  As for my boat, it could
+not have been kept afloat but for the extra ballast.  I sat cocked
+over the weather gunwale, tiller in one hand and sheet in the
+other; and the sheet, with a single turn around a pin, I was very
+often forced to let go in the severer puffs.  This allowed the sail
+to spill the wind, which was equivalent to taking off so much
+driving power, and of course I lost ground.  My consolation was
+that Demetrios was as often compelled to do the same thing.
+</para>
+
+<para>
+The strong ebb-tide, racing down the Straits in the teeth of the
+wind, caused an unusually heavy and spiteful sea, which dashed
+aboard continually.  I was dripping wet, and even the sail was wet
+half-way up the after leech.  Once I did succeed in outmanoeuvring
+Demetrios, so that my bow bumped into him amidships.  Here was
+where I should have had another man.  Before I could run forward
+and leap aboard, he shoved the boats apart with an oar, laughing
+mockingly in my face as he did so.
+</para>
+
+<para>
+We were now at the mouth of the Straits, in a bad stretch of water.
+Here the Vallejo Straits and the Carquinez Straits rushed directly
+at each other.  Through the first flowed all the water of Napa
+River and the great tide-lands; through the second flowed all the
+water of Suisun Bay and the Sacramento and San Joaquin rivers.  And
+where such immense bodies of water, flowing swiftly, clashed
+together, a terrible tide-rip was produced.  To make it worse, the
+wind howled up San Pablo Bay for fifteen miles and drove in a
+tremendous sea upon the tide-rip.
+</para>
+
+<para>
+Conflicting currents tore about in all directions, colliding,
+forming whirlpools, sucks, and boils, and shooting up spitefully
+into hollow waves which fell aboard as often from leeward as from
+windward.  And through it all, confused, driven into a madness of
+motion, thundered the great smoking seas from San Pablo Bay.
+</para>
+
+<para>
+I was as wildly excited as the water.  The boat was behaving
+splendidly, leaping and lurching through the welter like a race-
+horse.  I could hardly contain myself with the joy of it.  The huge
+sail, the howling wind, the driving seas, the plunging boat - I, a
+pygmy, a mere speck in the midst of it, was mastering the elemental
+strife, flying through it and over it, triumphant and victorious.
+</para>
+
+<para>
+And just then, as I roared along like a conquering hero, the boat
+received a frightful smash and came instantly to a dead stop.  I
+was flung forward and into the bottom.  As I sprang up I caught a
+fleeting glimpse of a greenish, barnacle-covered object, and knew
+it at once for what it was, that terror of navigation, a sunken
+pile.  No man may guard against such a thing.  Water-logged and
+floating just beneath the surface, it was impossible to sight it in
+the troubled water in time to escape.
+</para>
+
+<para>
+The whole bow of the boat must have been crushed in, for in a few
+seconds the boat was half full.  Then a couple of seas filled it,
+and it sank straight down, dragged to bottom by the heavy ballast.
+So quickly did it all happen that I was entangled in the sail and
+drawn under.  When I fought my way to the surface, suffocating, my
+lungs almost bursting, I could see nothing of the oars.  They must
+have been swept away by the chaotic currents.  I saw Demetrios
+Contos looking back from his boat, and heard the vindictive and
+mocking tones of his voice as he shouted exultantly.  He held
+steadily on his course, leaving me to perish.
+</para>
+
+<para>
+There was nothing to do but to swim for it, which, in that wild
+confusion, was at the best a matter of but a few moments.  Holding
+my breath and working with my hands, I managed to get off my heavy
+sea-boots and my jacket.  Yet there was very little breath I could
+catch to hold, and I swiftly discovered that it was not so much a
+matter of swimming as of breathing.
+</para>
+
+<para>
+I was beaten and buffeted, smashed under by the great San Pablo
+whitecaps, and strangled by the hollow tide-rip waves which flung
+themselves into my eyes, nose, and mouth.  Then the strange sucks
+would grip my legs and drag me under, to spout me up in some fierce
+boiling, where, even as I tried to catch my breath, a great
+whitecap would crash down upon my head.
+</para>
+
+<para>
+It was impossible to survive any length of time.  I was breathing
+more water than air, and drowning all the time.  My senses began to
+leave me, my head to whirl around.  I struggled on, spasmodically,
+instinctively, and was barely half conscious when I felt myself
+caught by the shoulders and hauled over the gunwale of a boat.
+</para>
+
+<para>
+For some time I lay across a seat where I had been flung, face
+downward, and with the water running out of my mouth.  After a
+while, still weak and faint, I turned around to see who was my
+rescuer.  And there, in the stern, sheet in one hand and tiller in
+the other, grinning and nodding good-naturedly, sat Demetrios
+Contos.  He had intended to leave me to drown, - he said so
+afterward, - but his better self had fought the battle, conquered,
+and sent him back to me.
+</para>
+
+<para>
+"You all-a right?" he asked.
+</para>
+
+<para>
+I managed to shape a "yes" on my lips, though I could not yet
+speak.
+</para>
+
+<para>
+"You sail-a de boat verr-a good-a," he said.  "So good-a as a man."
+</para>
+
+<para>
+A compliment from Demetrios Contos was a compliment indeed, and I
+keenly appreciated it, though I could only nod my head in
+acknowledgment.
+</para>
+
+<para>
+We held no more conversation, for I was busy recovering and he was
+busy with the boat.  He ran in to the wharf at Vallejo, made the
+boat fast, and helped me out.  Then it was, as we both stood on the
+wharf, that Charley stepped out from behind a net-rack and put his
+hand on Demetrios Contos's arm.
+</para>
+
+<para>
+"He saved my life, Charley," I protested; "and I don't think he
+ought to be arrested."
+</para>
+
+<para>
+A puzzled expression came into Charley's face, which cleared
+immediately after, in a way it had when he made up his mind.
+</para>
+
+<para>
+"I can't help it, lad," he said kindly.  "I can't go back on my
+duty, and it's plain duty to arrest him.  To-day is Sunday; there
+are two salmon in his boat which he caught to-day.  What else can I
+do?"
+</para>
+
+<para>
+"But he saved my life," I persisted, unable to make any other
+argument.
+</para>
+
+<para>
+Demetrios Contos's face went black with rage when he learned
+Charley's judgment.  He had a sense of being unfairly treated.  The
+better part of his nature had triumphed, he had performed a
+generous act and saved a helpless enemy, and in return the enemy
+was taking him to jail.
+</para>
+
+<para>
+Charley and I were out of sorts with each other when we went back
+to Benicia.  I stood for the spirit of the law and not the letter;
+but by the letter Charley made his stand.  As far as he could see,
+there was nothing else for him to do.  The law said distinctly that
+no salmon should be caught on Sunday.  He was a patrolman, and it
+was his duty to enforce that law.  That was all there was to it.
+He had done his duty, and his conscience was clear.  Nevertheless,
+the whole thing seemed unjust to me, and I felt very sorry for
+Demetrios Contos.
+</para>
+
+<para>
+Two days later we went down to Vallejo to the trial.  I had to go
+along as a witness, and it was the most hateful task that I ever
+performed in my life when I testified on the witness stand to
+seeing Demetrios catch the two salmon Charley had captured him
+with.
+</para>
+
+<para>
+Demetrios had engaged a lawyer, but his case was hopeless.  The
+jury was out only fifteen minutes, and returned a verdict of
+guilty.  The judge sentenced Demetrios to pay a fine of one hundred
+dollars or go to jail for fifty days.
+</para>
+
+<para>
+Charley stepped up to the clerk of the court.  "I want to pay that
+fine," he said, at the same time placing five twenty-dollar gold
+pieces on the desk.  "It - it was the only way out of it, lad," he
+stammered, turning to me.
+</para>
+
+<para>
+The moisture rushed into my eyes as I seized his hand.  "I want to
+pay - " I began.
+</para>
+
+<para>
+"To pay your half?" he interrupted.  "I certainly shall expect you
+to pay it."
+</para>
+
+<para>
+In the meantime Demetrios had been informed by his lawyer that his
+fee likewise had been paid by Charley.
+</para>
+
+<para>
+Demetrios came over to shake Charley's hand, and all his warm
+Southern blood flamed in his face.  Then, not to be outdone in
+generosity, he insisted on paying his fine and lawyer's fee
+himself, and flew half-way into a passion because Charley refused
+to let him.
+</para>
+
+<para>
+More than anything else we ever did, I think, this action of
+Charley's impressed upon the fishermen the deeper significance of
+the law.  Also Charley was raised high in their esteem, while I
+came in for a little share of praise as a boy who knew how to sail
+a boat.  Demetrios Contos not only never broke the law again, but
+he became a very good friend of ours, and on more than one occasion
+he ran up to Benicia to have a gossip with us.
+</para>
+
+
+</chapter>
+
+<chapter>
+<title>YELLOW HANDKERCHIEF</title>
+
+
+<para>
+"I'm not wanting to dictate to you, lad," Charley said; "but I'm
+very much against your making a last raid.  You've gone safely
+through rough times with rough men, and it would be a shame to have
+something happen to you at the very end."
+</para>
+
+<para>
+"But how can I get out of making a last raid?" I demanded, with the
+cocksureness of youth.  "There always has to be a last, you know,
+to anything."
+</para>
+
+<para>
+Charley crossed his legs, leaned back, and considered the problem.
+"Very true.  But why not call the capture of Demetrios Contos the
+last?  You're back from it safe and sound and hearty, for all your
+good wetting, and - and - "  His voice broke and he could not speak
+for a moment.  "And I could never forgive myself if anything
+happened to you now."
+</para>
+
+<para>
+I laughed at Charley's fears while I gave in to the claims of his
+affection, and agreed to consider the last raid already performed.
+We had been together for two years, and now I was leaving the fish
+patrol in order to go back and finish my education.  I had earned
+and saved money to put me through three years at the high school,
+and though the beginning of the term was several months away, I
+intended doing a lot of studying for the entrance examinations.
+</para>
+
+<para>
+My belongings were packed snugly in a sea-chest, and I was all
+ready to buy my ticket and ride down on the train to Oakland, when
+Neil Partington arrived in Benicia.  The Reindeer was needed
+immediately for work far down on the Lower Bay, and Neil said he
+intended to run straight for Oakland.  As that was his home and as
+I was to live with his family while going to school, he saw no
+reason, he said, why I should not put my chest aboard and come
+along.
+</para>
+
+<para>
+So the chest went aboard, and in the middle of the afternoon we
+hoisted the Reindeer's big mainsail and cast off.  It was
+tantalizing fall weather.  The sea-breeze, which had blown steadily
+all summer, was gone, and in its place were capricious winds and
+murky skies which made the time of arriving anywhere extremely
+problematical.  We started on the first of the ebb, and as we
+slipped down the Carquinez Straits, I looked my last for some time
+upon Benicia and the bight at Turner's Shipyard, where we had
+besieged the Lancashire Queen, and had captured Big Alec, the King
+of the Greeks.  And at the mouth of the Straits I looked with not a
+little interest upon the spot where a few days before I should have
+drowned but for the good that was in the nature of Demetrios
+Contos.
+</para>
+
+<para>
+A great wall of fog advanced across San Pablo Bay to meet us, and
+in a few minutes the Reindeer was running blindly through the damp
+obscurity.  Charley, who was steering, seemed to have an instinct
+for that kind of work.  How he did it, he himself confessed that he
+did not know; but he had a way of calculating winds, currents,
+distance, time, drift, and sailing speed that was truly marvellous.
+</para>
+
+<para>
+"It looks as though it were lifting," Neil Partington said, a
+couple of hours after we had entered the fog.  "Where do you say we
+are, Charley?"
+</para>
+
+<para>
+Charley looked at his watch, "Six o'clock, and three hours more of
+ebb," he remarked casually.
+</para>
+
+<para>
+"But where do you say we are?" Neil insisted.
+</para>
+
+<para>
+Charley pondered a moment, and then answered, "The tide has edged
+us over a bit out of our course, but if the fog lifts right now, as
+it is going to lift, you'll find we're not more than a thousand
+miles off McNear's Landing."
+</para>
+
+<para>
+"You might be a little more definite by a few miles, anyway," Neil
+grumbled, showing by his tone that he disagreed.
+</para>
+
+<para>
+"All right, then," Charley said, conclusively, "not less than a
+quarter of a mile, not more than a half."
+</para>
+
+<para>
+The wind freshened with a couple of little puffs, and the fog
+thinned perceptibly.
+</para>
+
+<para>
+"McNear's is right off there," Charley said, pointing directly into
+the fog on our weather beam.
+</para>
+
+<para>
+The three of us were peering intently in that direction, when the
+Reindeer struck with a dull crash and came to a standstill.  We ran
+forward, and found her bowsprit entangled in the tanned rigging of
+a short, chunky mast.  She had collided, head on, with a Chinese
+junk lying at anchor.
+</para>
+
+<para>
+At the moment we arrived forward, five Chinese, like so many bees,
+came swarming out of the little 'tween-decks cabin, the sleep still
+in their eyes.
+</para>
+
+<para>
+Leading them came a big, muscular man, conspicuous for his pock-
+marked face and the yellow silk handkerchief swathed about his
+head.  It was Yellow Handkerchief, the Chinaman whom we had
+arrested for illegal shrimp-fishing the year before, and who, at
+that time, had nearly sunk the Reindeer, as he had nearly sunk it
+now by violating the rules of navigation.
+</para>
+
+<para>
+"What d'ye mean, you yellow-faced heathen, lying here in a fairway
+without a horn a-going?" Charley cried hotly.
+</para>
+
+<para>
+"Mean?" Neil calmly answered.  "Just take a look - that's what he
+means."
+</para>
+
+<para>
+Our eyes followed the direction indicated by Neil's finger, and we
+saw the open amidships of the junk, half filled, as we found on
+closer examination, with fresh-caught shrimps.  Mingled with the
+shrimps were myriads of small fish, from a quarter of an inch
+upward in size.
+</para>
+
+<para>
+Yellow Handkerchief had lifted the trap-net at high-water slack,
+and, taking advantage of the concealment offered by the fog, had
+boldly been lying by, waiting to lift the net again at low-water
+slack.
+</para>
+
+<para>
+"Well," Neil hummed and hawed, "in all my varied and extensive
+experience as a fish patrolman, I must say this is the easiest
+capture I ever made.  What'll we do with them, Charley?"
+</para>
+
+<para>
+"Tow the junk into San Rafael, of course," came the answer.
+Charley turned to me.  "You stand by the junk, lad, and I'll pass
+you a towing line.  If the wind doesn't fail us, we'll make the
+creek before the tide gets too low, sleep at San Rafael, and arrive
+in Oakland to-morrow by midday."
+</para>
+
+<para>
+So saying, Charley and Neil returned to the Reindeer and got under
+way, the junk towing astern.  I went aft and took charge of the
+prize, steering by means of an antiquated tiller and a rudder with
+large, diamond-shaped holes, through which the water rushed back
+and forth.
+</para>
+
+<para>
+By now the last of the fog had vanished, and Charley's estimate of
+our position was confirmed by the sight of McNear's Landing a short
+half-mile away.  Following along the west shore, we rounded Point
+Pedro in plain view of the Chinese shrimp villages, and a great to-
+do was raised when they saw one of their junks towing behind the
+familiar fish patrol sloop.
+</para>
+
+<para>
+The wind, coming off the land, was rather puffy and uncertain, and
+it would have been more to our advantage had it been stronger.  San
+Rafael Creek, up which we had to go to reach the town and turn over
+our prisoners to the authorities, ran through wide-stretching
+marshes, and was difficult to navigate on a falling tide, while at
+low tide it was impossible to navigate at all.  So, with the tide
+already half-ebbed, it was necessary for us to make time.  This the
+heavy junk prevented, lumbering along behind and holding the
+Reindeer back by just so much dead weight.
+</para>
+
+<para>
+"Tell those coolies to get up that sail," Charley finally called to
+me.  "We don't want to hang up on the mud flats for the rest of the
+night."
+</para>
+
+<para>
+I repeated the order to Yellow Handkerchief, who mumbled it huskily
+to his men.  He was suffering from a bad cold, which doubled him up
+in convulsive coughing spells and made his eyes heavy and
+bloodshot.  This made him more evil-looking than ever, and when he
+glared viciously at me I remembered with a shiver the close shave I
+had had with him at the time of his previous arrest.
+</para>
+
+<para>
+His crew sullenly tailed on to the halyards, and the strange,
+outlandish sail, lateen in rig and dyed a warm brown, rose in the
+air.  We were sailing on the wind, and when Yellow Handkerchief
+flattened down the sheet the junk forged ahead and the tow-line
+went slack.  Fast as the Reindeer could sail, the junk outsailed
+her; and to avoid running her down I hauled a little closer on the
+wind.  But the junk likewise outpointed, and in a couple of minutes
+I was abreast of the Reindeer and to windward.  The tow-line had
+now tautened, at right angles to the two boats, and the predicament
+was laughable.
+</para>
+
+<para>
+"Cast off!" I shouted.
+</para>
+
+<para>
+Charley hesitated.
+</para>
+
+<para>
+"It's all right," I added.  "Nothing can happen.  We'll make the
+creek on this tack, and you'll be right behind me all the way up to
+San Rafael."
+</para>
+
+<para>
+At this Charley cast off, and Yellow Handkerchief sent one of his
+men forward to haul in the line.  In the gathering darkness I could
+just make out the mouth of San Rafael Creek, and by the time we
+entered it I could barely see its banks.  The Reindeer was fully
+five minutes astern, and we continued to leave her astern as we
+beat up the narrow, winding channel.  With Charley behind us, it
+seemed I had little to fear from my five prisoners; but the
+darkness prevented my keeping a sharp eye on them, so I transferred
+my revolver from my trousers pocket to the side pocket of my coat,
+where I could more quickly put my hand on it.
+</para>
+
+<para>
+Yellow Handkerchief was the one I feared, and that he knew it and
+made use of it, subsequent events will show.  He was sitting a few
+feet away from me, on what then happened to be the weather side of
+the junk.  I could scarcely see the outlines of his form, but I
+soon became convinced that he was slowly, very slowly, edging
+closer to me.  I watched him carefully.  Steering with my left
+hand, I slipped my right into my pocket and got hold of the
+revolver.
+</para>
+
+<para>
+I saw him shift along for a couple of inches, and I was just about
+to order him back - the words were trembling on the tip of my
+tongue - when I was struck with great force by a heavy figure that
+had leaped through the air upon me from the lee side.  It was one
+of the crew.  He pinioned my right arm so that I could not withdraw
+my hand from my pocket, and at the same time clapped his other hand
+over my mouth.  Of course, I could have struggled away from him and
+freed my hand or gotten my mouth clear so that I might cry an
+alarm, but in a trice Yellow Handkerchief was on top of me.
+</para>
+
+<para>
+I struggled around to no purpose in the bottom of the junk, while
+my legs and arms were tied and my mouth securely bound in what I
+afterward found to be a cotton shirt.  Then I was left lying in the
+bottom.  Yellow Handkerchief took the tiller, issuing his orders in
+whispers; and from our position at the time, and from the
+alteration of the sail, which I could dimly make out above me as a
+blot against the stars, I knew the junk was being headed into the
+mouth of a small slough which emptied at that point into San Rafael
+Creek.
+</para>
+
+<para>
+In a couple of minutes we ran softly alongside the bank, and the
+sail was silently lowered.  The Chinese kept very quiet.  Yellow
+Handkerchief sat down in the bottom alongside of me, and I could
+feel him straining to repress his raspy, hacking cough.  Possibly
+seven or eight minutes later I heard Charley's voice as the
+Reindeer went past the mouth of the slough.
+</para>
+
+<para>
+"I can't tell you how relieved I am," I could plainly hear him
+saying to Neil, "that the lad has finished with the fish patrol
+without accident."
+</para>
+
+<para>
+Here Neil said something which I could not catch, and then
+Charley's voice went on:
+</para>
+
+<para>
+"The youngster takes naturally to the water, and if, when he
+finishes high school, he takes a course in navigation and goes deep
+sea, I see no reason why he shouldn't rise to be master of the
+finest and biggest ship afloat."
+</para>
+
+<para>
+It was all very flattering to me, but lying there, bound and gagged
+by my own prisoners, with the voices growing faint and fainter as
+the Reindeer slipped on through the darkness toward San Rafael, I
+must say I was not in quite the proper situation to enjoy my
+smiling future.  With the Reindeer went my last hope.  What was to
+happen next I could not imagine, for the Chinese were a different
+race from mine, and from what I knew I was confident that fair play
+was no part of their make-up.
+</para>
+
+<para>
+After waiting a few minutes longer, the crew hoisted the lateen
+sail, and Yellow Handkerchief steered down toward the mouth of San
+Rafael Creek.  The tide was getting lower, and he had difficulty in
+escaping the mud-banks.  I was hoping he would run aground, but he
+succeeded in making the Bay without accident.
+</para>
+
+<para>
+As we passed out of the creek a noisy discussion arose, which I
+knew related to me.  Yellow Handkerchief was vehement, but the
+other four as vehemently opposed him.  It was very evident that he
+advocated doing away with me and that they were afraid of the
+consequences.  I was familiar enough with the Chinese character to
+know that fear alone restrained them.  But what plan they offered
+in place of Yellow Handkerchief's murderous one, I could not make
+out.
+</para>
+
+<para>
+My feelings, as my fate hung in the balance, may be guessed.  The
+discussion developed into a quarrel, in the midst of which Yellow
+Handkerchief unshipped the heavy tiller and sprang toward me.  But
+his four companions threw themselves between, and a clumsy struggle
+took place for possession of the tiller.  In the end Yellow
+Handkerchief was overcome, and sullenly returned to the steering,
+while they soundly berated him for his rashness.
+</para>
+
+<para>
+Not long after, the sail was run down and the junk slowly urged
+forward by means of the sweeps.  I felt it ground gently on the
+soft mud.  Three of the Chinese - they all wore long sea-boots -
+got over the side, and the other two passed me across the rail.
+With Yellow Handkerchief at my legs and his two companions at my
+shoulders, they began to flounder along through the mud.  After
+some time their feet struck firmer footing, and I knew they were
+carrying me up some beach.  The location of this beach was not
+doubtful in my mind.  It could be none other than one of the Marin
+Islands, a group of rocky islets which lay off the Marin County
+shore.
+</para>
+
+<para>
+When they reached the firm sand that marked high tide, I was
+dropped, and none too gently.  Yellow Handkerchief kicked me
+spitefully in the ribs, and then the trio floundered back through
+the mud to the junk.  A moment later I heard the sail go up and
+slat in the wind as they drew in the sheet.  Then silence fell, and
+I was left to my own devices for getting free.
+</para>
+
+<para>
+I remembered having seen tricksters writhe and squirm out of ropes
+with which they were bound, but though I writhed and squirmed like
+a good fellow, the knots remained as hard as ever, and there was no
+appreciable slack.  In the course of my squirming, however, I
+rolled over upon a heap of clam-shells - the remains, evidently, of
+some yachting party's clam-bake.  This gave me an idea.  My hands
+were tied behind my back; and, clutching a shell in them, I rolled
+over and over, up the beach, till I came to the rocks I knew to be
+there.
+</para>
+
+<para>
+Rolling around and searching, I finally discovered a narrow
+crevice, into which I shoved the shell.  The edge of it was sharp,
+and across the sharp edge I proceeded to saw the rope that bound my
+wrists.  The edge of the shell was also brittle, and I broke it by
+bearing too heavily upon it.  Then I rolled back to the heap and
+returned with as many shells as I could carry in both hands.  I
+broke many shells, cut my hands a number of times, and got cramps
+in my legs from my strained position and my exertions.
+</para>
+
+<para>
+While I was suffering from the cramps, and resting, I heard a
+familiar halloo drift across the water.  It was Charley, searching
+for me.  The gag in my mouth prevented me from replying, and I
+could only lie there, helplessly fuming, while he rowed past the
+island and his voice slowly lost itself in the distance.
+</para>
+
+<para>
+I returned to the sawing process, and at the end of half an hour
+succeeded in severing the rope.  The rest was easy.  My hands once
+free, it was a matter of minutes to loosen my legs and to take the
+gag out of my mouth.  I ran around the island to make sure it was
+an island and not by any chance a portion of the mainland.  An
+island it certainly was, one of the Marin group, fringed with a
+sandy beach and surrounded by a sea of mud.  Nothing remained but
+to wait till daylight and to keep warm; for it was a cold, raw
+night for California, with just enough wind to pierce the skin and
+cause one to shiver.
+</para>
+
+<para>
+To keep up the circulation, I ran around the island a dozen times
+or so, and clambered across its rocky backbone as many times more -
+all of which was of greater service to me, as I afterward
+discovered, than merely to warm me up.  In the midst of this
+exercise I wondered if I had lost anything out of my pockets while
+rolling over and over in the sand.  A search showed the absence of
+my revolver and pocket-knife.  The first Yellow Handkerchief had
+taken; but the knife had been lost in the sand.
+</para>
+
+<para>
+I was hunting for it when the sound of rowlocks came to my ears.
+At first, of course, I thought of Charley; but on second thought I
+knew Charley would be calling out as he rowed along.  A sudden
+premonition of danger seized me.  The Marin Islands are lonely
+places; chance visitors in the dead of night are hardly to be
+expected.  What if it were Yellow Handkerchief?  The sound made by
+the rowlocks grew more distinct.  I crouched in the sand and
+listened intently.  The boat, which I judged a small skiff from the
+quick stroke of the oars, was landing in the mud about fifty yards
+up the beach.  I heard a raspy, hacking cough, and my heart stood
+still.  It was Yellow Handkerchief.  Not to be robbed of his
+revenge by his more cautious companions, he had stolen away from
+the village and come back alone.
+</para>
+
+<para>
+I did some swift thinking.  I was unarmed and helpless on a tiny
+islet, and a yellow barbarian, whom I had reason to fear, was
+coming after me.  Any place was safer than the island, and I turned
+instinctively to the water, or rather to the mud.  As he began to
+flounder ashore through the mud, I started to flounder out into it,
+going over the same course which the Chinese had taken in landing
+me and in returning to the junk.
+</para>
+
+<para>
+Yellow Handkerchief, believing me to be lying tightly bound,
+exercised no care, but came ashore noisily.  This helped me, for,
+under the shield of his noise and making no more myself than
+necessary, I managed to cover fifty feet by the time he had made
+the beach.  Here I lay down in the mud.  It was cold and clammy,
+and made me shiver, but I did not care to stand up and run the risk
+of being discovered by his sharp eyes.
+</para>
+
+<para>
+He walked down the beach straight to where he had left me lying,
+and I had a fleeting feeling of regret at not being able to see his
+surprise when he did not find me.  But it was a very fleeting
+regret, for my teeth were chattering with the cold.
+</para>
+
+<para>
+What his movements were after that I had largely to deduce from the
+facts of the situation, for I could scarcely see him in the dim
+starlight.  But I was sure that the first thing he did was to make
+the circuit of the beach to learn if landings had been made by
+other boats.  This he would have known at once by the tracks
+through the mud.
+</para>
+
+<para>
+Convinced that no boat had removed me from the island, he next
+started to find out what had become of me.  Beginning at the pile
+of clamshells, he lighted matches to trace my tracks in the sand.
+At such times I could see his villanous face plainly, and, when the
+sulphur from the matches irritated his lungs, between the raspy
+cough that followed and the clammy mud in which I was lying, I
+confess I shivered harder than ever.
+</para>
+
+<para>
+The multiplicity of my footprints puzzled him.  Then the idea that
+I might be out in the mud must have struck him, for he waded out a
+few yards in my direction, and, stooping, with his eyes searched
+the dim surface long and carefully.  He could not have been more
+than fifteen feet from me, and had he lighted a match he would
+surely have discovered me.
+</para>
+
+<para>
+He returned to the beach and clambered about, over the rocky
+backbone, again hunting for me with lighted matches, The closeness
+of the shave impelled me to further flight.  Not daring to wade
+upright, on account of the noise made by floundering and by the
+suck of the mud, I remained lying down in the mud and propelled
+myself over its surface by means of my hands.  Still keeping the
+trail made by the Chinese in going from and to the junk, I held on
+until I reached the water.  Into this I waded to a depth of three
+feet, and then I turned off to the side on a line parallel with the
+beach.
+</para>
+
+<para>
+The thought came to me of going toward Yellow Handkerchief's skiff
+and escaping in it, but at that very moment he returned to the
+beach, and, as though fearing the very thing I had in mind, he
+slushed out through the mud to assure himself that the skiff was
+safe.  This turned me in the opposite direction.  Half swimming,
+half wading, with my head just out of water and avoiding splashing,
+I succeeded in putting about a hundred feet between myself and the
+spot where the Chinese had begun to wade ashore from the junk.  I
+drew myself out on the mud and remained lying flat.
+</para>
+
+<para>
+Again Yellow Handkerchief returned to the beach and made a search
+of the island, and again he returned to the heap of clam-shells.  I
+knew what was running in his mind as well as he did himself.  No
+one could leave or land without making tracks in the mud.  The only
+tracks to be seen were those leading from his skiff and from where
+the junk had been.  I was not on the island.  I must have left it
+by one or the other of those two tracks.  He had just been over the
+one to his skiff, and was certain I had not left that way.
+Therefore I could have left the island only by going over the
+tracks of the junk landing.  This he proceeded to verify by wading
+out over them himself, lighting matches as he came along.
+</para>
+
+<para>
+When he arrived at the point where I had first lain, I knew, by the
+matches he burned and the time he took, that he had discovered the
+marks left by my body.  These he followed straight to the water and
+into it, but in three feet of water he could no longer see them.
+On the other hand, as the tide was still falling, he could easily
+make out the impression made by the junk's bow, and could have
+likewise made out the impression of any other boat if it had landed
+at that particular spot.  But there was no such mark; and I knew
+that he was absolutely convinced that I was hiding somewhere in the
+mud.
+</para>
+
+<para>
+But to hunt on a dark night for a boy in a sea of mud would be like
+hunting for a needle in a haystack, and he did not attempt it.
+Instead he went back to the beach and prowled around for some time.
+I was hoping he would give me up and go, for by this time I was
+suffering severely from the cold.  At last he waded out to his
+skiff and rowed away.  What if this departure of Yellow
+Handkerchief's were a sham?  What if he had done it merely to
+entice me ashore?
+</para>
+
+<para>
+The more I thought of it the more certain I became that he had made
+a little too much noise with his oars as he rowed away.  So I
+remained, lying in the mud and shivering.  I shivered till the
+muscles of the small of my back ached and pained me as badly as the
+cold, and I had need of all my self-control to force myself to
+remain in my miserable situation.
+</para>
+
+<para>
+It was well that I did, however, for, possibly an hour later, I
+thought I could make out something moving on the beach.  I watched
+intently, but my ears were rewarded first, by a raspy cough I knew
+only too well.  Yellow Handkerchief had sneaked back, landed on the
+other side of the island, and crept around to surprise me if I had
+returned.
+</para>
+
+<para>
+After that, though hours passed without sign of him, I was afraid
+to return to the island at all.  On the other hand, I was almost
+equally afraid that I should die of the exposure I was undergoing.
+I had never dreamed one could suffer so.  I grew so cold and numb,
+finally, that I ceased to shiver.  But my muscles and bones began
+to ache in a way that was agony.  The tide had long since begun to
+rise, and, foot by foot, it drove me in toward the beach.  High
+water came at three o'clock, and at three o'clock I drew myself up
+on the beach, more dead than alive, and too helpless to have
+offered any resistance had Yellow Handkerchief swooped down upon
+me.
+</para>
+
+<para>
+But no Yellow Handkerchief appeared.  He had given me up and gone
+back to Point Pedro.  Nevertheless, I was in a deplorable, not to
+say dangerous, condition.  I could not stand upon my feet, much
+less walk.  My clammy, muddy garments clung to me like sheets of
+ice.  I thought I should never get them off.  So numb and lifeless
+were my fingers, and so weak was I, that it seemed to take an hour
+to get off my shoes.  I had not the strength to break the porpoise-
+hide laces, and the knots defied me.  I repeatedly beat my hands
+upon the rocks to get some sort of life into them.  Sometimes I
+felt sure I was going to die.
+</para>
+
+<para>
+But in the end, - after several centuries, it seemed to me, - I got
+off the last of my clothes.  The water was now close at hand, and I
+crawled painfully into it and washed the mud from my naked body.
+Still, I could not get on my feet and walk and I was afraid to lie
+still.  Nothing remained but to crawl weakly, like a snail, and at
+the cost of constant pain, up and down the sand.  I kept this up as
+long as possible, but as the east paled with the coming of dawn I
+began to succumb.  The sky grew rosy-red, and the golden rim of the
+sun, showing above the horizon, found me lying helpless and
+motionless among the clam-shells.
+</para>
+
+<para>
+As in a dream, I saw the familiar mainsail of the Reindeer as she
+slipped out of San Rafael Creek on a light puff of morning air.
+This dream was very much broken.  There are intervals I can never
+recollect on looking back over it.  Three things, however, I
+distinctly remember:  the first sight of the Reindeer's mainsail;
+her lying at anchor a few hundred feet away and a small boat
+leaving her side; and the cabin stove roaring red-hot, myself
+swathed all over with blankets, except on the chest and shoulders,
+which Charley was pounding and mauling unmercifully, and my mouth
+and throat burning with the coffee which Neil Partington was
+pouring down a trifle too hot.
+</para>
+
+<para>
+But burn or no burn, I tell you it felt good.  By the time we
+arrived in Oakland I was as limber and strong as ever, - though
+Charlie and Neil Partington were afraid I was going to have
+pneumonia, and Mrs. Partington, for my first six months of school,
+kept an anxious eye upon me to discover the first symptoms of
+consumption.
+</para>
+
+<para>
+Time flies.  It seems but yesterday that I was a lad of sixteen on
+the fish patrol.  Yet I know that I arrived this very morning from
+China, with a quick passage to my credit, and master of the
+barkentine Harvester.  And I know that to-morrow morning I shall
+run over to Oakland to see Neil Partington and his wife and family,
+and later on up to Benicia to see Charley Le Grant and talk over
+old times.  No; I shall not go to Benicia, now that I think about
+it.  I expect to be a highly interested party to a wedding, shortly
+to take place.  Her name is Alice Partington, and, since Charley
+has promised to be best man, he will have to come down to Oakland
+instead.
+</para>
+</chapter>
+
+</bookbody>
+</book>
+<endgutblurb>
+End of Project Gutenberg's Etext of Tales of the Fish Patrol, by London
+</endgutblurb>
+
+</gutbook>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/gutbook1.dtd
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/gutbook1.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/gutbook1.dtd	(revision 21112)
@@ -0,0 +1,40 @@
+<!--
+GUTBOOK DTD V 1.0
+==================
+
+GENERAL NOTES
++++++++++++++
+This is gutbook dtd version 1.0
+file name is gutpoems1.dtd
+This dtd is a first pass at a dtd for marking up 
+books for project gutenberg.
+It was originally developed for marking up the Darwins books.
+
+REVISION HISTORY
+++++++++++++++++
+Original Author
+Frank Boumphrey  1/19/2000
+
+COPYRIGHT NOTICE
+++++++++++++++++
+This DTD is the copyright of the HTML writers guild.
+Permission is given in perpetuity for any one to use 
+this DTD provided that these notes and this  copyright 
+notice is included.
+
+
+-->
+
+<!--Begin gutbook DTD-->
+<!ELEMENT gutbook (gutblurb,markupblurb,book*,endmarkupblurb?,endgutblurb?)>
+
+<!ELEMENT gutblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT markupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endmarkupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endgutblurb (#PCDATA|para|simplesect|title)*>
+
+<!--Include bookfrag.dtd-->
+
+<!ENTITY % bookfrag SYSTEM "bookfrag.dtd">
+%bookfrag;
+<!--end of file-->
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/iliad.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/iliad.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/iliad.xml	(revision 21112)
@@ -0,0 +1,16976 @@
+<?xml version="1.0"?>
+<!DOCTYPE gutbook SYSTEM "gutbook1.dtd">
+
+<?xml-stylesheet type="text/css" href="darwin.css"?>
+
+<gutbook>
+<gutblurb>
+<para>
+*******The Project Gutenberg Etext of The Iliad, by Homer*****
+translated by Samuel Butler
+#3 in our series by Homer
+</para>
+
+<para>
+Copyright laws are changing all over the world, be sure to check
+the copyright laws for your country before posting these files!!
+</para>
+
+<para>
+Please take a look at the important information in this header.
+We encourage you to keep this file on your own disk, keeping an
+electronic path open for the next readers.  Do not remove this.
+</para>
+
+<para>
+**Welcome To The World of Free Plain Vanilla Electronic Texts**
+</para>
+
+<para>
+**Etexts Readable By Both Humans and By Computers, Since 1971**
+</para>
+
+<para>
+*These Etexts Prepared By Hundreds of Volunteers and Donations*
+</para>
+
+<para>
+Information on contacting Project Gutenberg to get Etexts, and
+further information is included below.  We need your donations.
+</para>
+
+<para>
+The Iliad of Homer
+</para>
+
+<para>
+translated by Samuel Butler
+</para>
+
+<para>
+June, 2000  [Etext #2199]
+</para>
+
+<para>
+********The Project Gutenberg Etext of The Iliad, by Homer******
+******This file should be named iliad10.txt or iliad10.zip******
+</para>
+
+<para>
+Corrected EDITIONS of our etexts get a new NUMBER, iliad11.txt
+VERSIONS based on separate sources get new LETTER, iliad10a.txt
+</para>
+
+<para>
+Project Gutenberg Etexts are usually created from multiple editions,
+all of which are in the Public Domain in the United States, unless a
+copyright notice is included.  Therefore, we do NOT keep these books
+in compliance with any particular paper edition, usually otherwise.
+</para>
+
+<para>
+We are now trying to release all our books one month in advance
+of the official release dates, for time for better editing.
+</para>
+
+<para>
+Please note:  neither this list nor its contents are final till
+midnight of the last day of the month of any such announcement.
+The official release date of all Project Gutenberg Etexts is at
+Midnight, Central Time, of the last day of the stated month.  A
+preliminary version may often be posted for suggestion, comment
+and editing by those who wish to do so.  To be sure you have an
+up to date first edition [xxxxx10x.xxx] please check file sizes
+in the first week of the next month.  Since our ftp program has
+a bug in it that scrambles the date [tried to fix and failed] a
+look at the file size will have to do, but we will try to see a
+new copy has at least one byte more or less.
+</para>
+
+<para>
+Information about Project Gutenberg (one page)
+</para>
+
+<para>
+We produce about two million dollars for each hour we work.  The
+fifty hours is one conservative estimate for how long it we take
+to get any etext selected, entered, proofread, edited, copyright
+searched and analyzed, the copyright letters written, etc.  This
+projected audience is one hundred million readers.  If our value
+per text is nominally estimated at one dollar then we produce $2
+million dollars per hour this year as we release thirty-two text
+files per month, or 384 more Etexts in 1998 for a total of 1500+
+If these reach just 10% of the computerized population, then the
+total should reach over 150 billion Etexts given away.
+</para>
+
+<para>
+The Goal of Project Gutenberg is to Give Away One Trillion Etext
+Files by the December 31, 2001.  [10,000 x 100,000,000=Trillion]
+This is ten thousand titles each to one hundred million readers,
+which is only 10% of the present number of computer users.  2001
+should have at least twice as many computer users as that, so it
+will require us reaching less than 5% of the users in 2001.
+</para>
+
+<para>
+We need your donations more than ever!
+</para>
+
+<para>
+All donations should be made to "Project Gutenberg/CMU": and are
+tax deductible to the extent allowable by law.  (CMU = Carnegie-
+Mellon University).
+</para>
+
+<para>
+For these and other matters, please mail to:
+</para>
+
+<para>
+Project Gutenberg
+P. O. Box  2782
+Champaign, IL 61825
+</para>
+
+<para>
+When all other email fails try our Executive Director:
+Michael S. Hart hart@pobox.com
+</para>
+
+<para>
+We would prefer to send you this information by email
+(Internet, Bitnet, Compuserve, ATTMAIL or MCImail).
+</para>
+
+<para>
+******
+If you have an FTP program (or emulator), please
+FTP directly to the Project Gutenberg archives:
+[Mac users, do NOT point and click. . .type]
+</para>
+
+<para>
+ftp uiarchive.cso.uiuc.edu
+login:  anonymous
+password:  your@login
+cd etext/etext90 through /etext96
+or cd etext/articles [get suggest gut for more information]
+dir [to see files]
+get or mget [to get files. . .set bin for zip files]
+GET INDEX?00.GUT
+for a list of books
+and
+GET NEW GUT for general information
+and
+MGET GUT* for newsletters.
+</para>
+
+<para>
+**Information prepared by the Project Gutenberg legal advisor**
+(Three Pages)
+</para>
+
+<para>
+***START**THE SMALL PRINT!**FOR PUBLIC DOMAIN ETEXTS**START***
+Why is this "Small Print!" statement here?  You know: lawyers.
+They tell us you might sue us if there is something wrong with
+your copy of this etext, even if you got it for free from
+someone other than us, and even if what's wrong is not our
+fault.  So, among other things, this "Small Print!" statement
+disclaims most of our liability to you.  It also tells you how
+you can distribute copies of this etext if you want to.
+</para>
+
+<para>
+*BEFORE!* YOU USE OR READ THIS ETEXT
+By using or reading any part of this PROJECT GUTENBERG-tm
+etext, you indicate that you understand, agree to and accept
+this "Small Print!" statement.  If you do not, you can receive
+a refund of the money (if any) you paid for this etext by
+sending a request within 30 days of receiving it to the person
+you got it from.  If you received this etext on a physical
+medium (such as a disk), you must return it with your request.
+</para>
+
+<para>
+ABOUT PROJECT GUTENBERG-TM ETEXTS
+This PROJECT GUTENBERG-tm etext, like most PROJECT GUTENBERG-
+tm etexts, is a "public domain" work distributed by Professor
+Michael S. Hart through the Project Gutenberg Association at
+Carnegie-Mellon University (the "Project").  Among other
+things, this means that no one owns a United States copyright
+on or for this work, so the Project (and you!) can copy and
+distribute it in the United States without permission and
+without paying copyright royalties.  Special rules, set forth
+below, apply if you wish to copy and distribute this etext
+under the Project's "PROJECT GUTENBERG" trademark.
+</para>
+
+<para>
+To create these etexts, the Project expends considerable
+efforts to identify, transcribe and proofread public domain
+works.  Despite these efforts, the Project's etexts and any
+medium they may be on may contain "Defects".  Among other
+things, Defects may take the form of incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other
+intellectual property infringement, a defective or damaged
+disk or other etext medium, a computer virus, or computer
+codes that damage or cannot be read by your equipment.
+</para>
+
+<para>
+LIMITED WARRANTY; DISCLAIMER OF DAMAGES
+But for the "Right of Replacement or Refund" described below,
+[1] the Project (and any other party you may receive this
+etext from as a PROJECT GUTENBERG-tm etext) disclaims all
+liability to you for damages, costs and expenses, including
+legal fees, and [2] YOU HAVE NO REMEDIES FOR NEGLIGENCE OR
+UNDER STRICT LIABILITY, OR FOR BREACH OF WARRANTY OR CONTRACT,
+INCLUDING BUT NOT LIMITED TO INDIRECT, CONSEQUENTIAL, PUNITIVE
+OR INCIDENTAL DAMAGES, EVEN IF YOU GIVE NOTICE OF THE
+POSSIBILITY OF SUCH DAMAGES.
+</para>
+
+<para>
+If you discover a Defect in this etext within 90 days of
+receiving it, you can receive a refund of the money (if any)
+you paid for it by sending an explanatory note within that
+time to the person you received it from.  If you received it
+on a physical medium, you must return it with your note, and
+such person may choose to alternatively give you a replacement
+copy.  If you received it electronically, such person may
+choose to alternatively give you a second opportunity to
+receive it electronically.
+</para>
+
+<para>
+THIS ETEXT IS OTHERWISE PROVIDED TO YOU "AS-IS".  NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, ARE MADE TO YOU AS
+TO THE ETEXT OR ANY MEDIUM IT MAY BE ON, INCLUDING BUT NOT
+LIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
+PARTICULAR PURPOSE.
+</para>
+
+<para>
+Some states do not allow disclaimers of implied warranties or
+the exclusion or limitation of consequential damages, so the
+above disclaimers and exclusions may not apply to you, and you
+may have other legal rights.
+</para>
+
+<para>
+INDEMNITY
+You will indemnify and hold the Project, its directors,
+officers, members and agents harmless from all liability, cost
+and expense, including legal fees, that arise directly or
+indirectly from any of the following that you do or cause:
+[1] distribution of this etext, [2] alteration, modification,
+or addition to the etext, or [3] any Defect.
+</para>
+
+<para>
+DISTRIBUTION UNDER "PROJECT GUTENBERG-tm"
+You may distribute copies of this etext electronically, or by
+disk, book or any other medium if you either delete this
+"Small Print!" and all other references to Project Gutenberg,
+or:
+</para>
+
+<para>
+[1]  Only give exact copies of it.  Among other things, this
+     requires that you do not remove, alter or modify the
+     etext or this "small print!" statement.  You may however,
+     if you wish, distribute this etext in machine readable
+     binary, compressed, mark-up, or proprietary form,
+     including any form resulting from conversion by word pro-
+     cessing or hypertext software, but only so long as
+     *EITHER*:
+</para>
+
+<para>
+     [*]  The etext, when displayed, is clearly readable, and
+          does *not* contain characters other than those
+          intended by the author of the work, although tilde
+          (~), asterisk (*) and underline (_) characters may
+          be used to convey punctuation intended by the
+          author, and additional characters may be used to
+          indicate hypertext links; OR
+</para>
+
+<para>
+     [*]  The etext may be readily converted by the reader at
+          no expense into plain ASCII, EBCDIC or equivalent
+          form by the program that displays the etext (as is
+          the case, for instance, with most word processors);
+          OR
+</para>
+
+<para>
+     [*]  You provide, or agree to also provide on request at
+          no additional cost, fee or expense, a copy of the
+          etext in its original plain ASCII form (or in EBCDIC
+          or other equivalent proprietary form).
+</para>
+
+<para>
+[2]  Honor the etext refund and replacement provisions of this
+     "Small Print!" statement.
+</para>
+
+<para>
+[3]  Pay a trademark license fee to the Project of 20% of the
+     net profits you derive calculated using the method you
+     already use to calculate your applicable taxes.  If you
+     don't derive profits, no royalty is due.  Royalties are
+     payable to "Project Gutenberg Association/Carnegie-Mellon
+     University" within the 60 days following each
+     date you prepare (or were legally required to prepare)
+     your annual (or equivalent periodic) tax return.
+</para>
+
+<para>
+WHAT IF YOU *WANT* TO SEND MONEY EVEN IF YOU DON'T HAVE TO?
+The Project gratefully accepts contributions in money, time,
+scanning machines, OCR software, public domain etexts, royalty
+free copyright licenses, and every other sort of contribution
+you can think of.  Money should be paid to "Project Gutenberg
+Association / Carnegie-Mellon University".
+</para>
+
+<para>
+*END*THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS*Ver.04.29.93*END*
+</para>
+</gutblurb>
+
+<markupblurb>
+Marked up by Mark Stradling Feb 2000
+Validated against gutbook1.dtd using Mozilla M13 Feb 2000
+</markupblurb>
+
+<book>
+<acknowledge>
+A project of the HTML Writrs Guild and Project Gutenberg. Markup by Mark Stradling Feb 2000
+</acknowledge>
+<frontmatter>
+<titlepage>
+<title>THE ILIAD</title>
+<para> by</para>
+<author>HOMER</author>
+<para>
+Rendered into English Prose for
+the use of those who cannot
+read the original
+</para>
+
+<para>
+by Samuel Butler
+</para>
+</titlepage>
+</frontmatter>
+<bookbody>
+<chapter>
+<title>BOOK I</title>
+
+<para>
+Sing, O goddess, the anger of Achilles son of Peleus, that
+brought countless ills upon the Achaeans. Many a brave soul did
+it send hurrying down to Hades, and many a hero did it yield a
+prey to dogs and vultures, for so were the counsels of Jove
+fulfilled from the day on which the son of Atreus, king of men,
+and great Achilles, first fell out with one another.
+</para>
+
+<para>
+And which of the gods was it that set them on to quarrel? It was
+the son of Jove and Leto; for he was angry with the king and sent
+a pestilence upon the host to plague the people, because the son
+of Atreus had dishonoured Chryses his priest. Now Chryses had
+come to the ships of the Achaeans to free his daughter, and had
+brought with him a great ransom: moreover he bore in his hand the
+sceptre of Apollo wreathed with a suppliant's wreath, and he
+besought the Achaeans, but most of all the two sons of Atreus,
+who were their chiefs.
+</para>
+
+<para>
+"Sons of Atreus," he cried, "and all other Achaeans, may the gods
+who dwell in Olympus grant you to sack the city of Priam, and to
+reach your homes in safety; but free my daughter, and accept a
+ransom for her, in reverence to Apollo, son of Jove."
+</para>
+
+<para>
+On this the rest of the Achaeans with one voice were for
+respecting the priest and taking the ransom that he offered; but
+not so Agamemnon, who spoke fiercely to him and sent him roughly
+away. "Old man," said he, "let me not find you tarrying about our
+ships, nor yet coming hereafter. Your sceptre of the god and your
+wreath shall profit you nothing. I will not free her. She shall
+grow old in my house at Argos far from her own home, busying
+herself with her loom and visiting my couch; so go, and do not
+provoke me or it shall be the worse for you."
+</para>
+
+<para>
+The old man feared him and obeyed. Not a word he spoke, but went
+by the shore of the sounding sea and prayed apart to King Apollo
+whom lovely Leto had borne. "Hear me," he cried, "O god of the
+silver bow, that protectest Chryse and holy Cilla and rulest
+Tenedos with thy might, hear me oh thou of Sminthe. If I have
+ever decked your temple with garlands, or burned your thigh-bones
+in fat of bulls or goats, grant my prayer, and let your arrows
+avenge these my tears upon the Danaans."
+</para>
+
+<para>
+Thus did he pray, and Apollo heard his prayer. He came down
+furious from the summits of Olympus, with his bow and his quiver
+upon his shoulder, and the arrows rattled on his back with the
+rage that trembled within him. He sat himself down away from the
+ships with a face as dark as night, and his silver bow rang death
+as he shot his arrow in the midst of them. First he smote their
+mules and their hounds, but presently he aimed his shafts at the
+people themselves, and all day long the pyres of the dead were
+burning.
+</para>
+
+<para>
+For nine whole days he shot his arrows among the people, but upon
+the tenth day Achilles called them in assembly--moved thereto by
+Juno, who saw the Achaeans in their death-throes and had
+compassion upon them. Then, when they were got together, he rose
+and spoke among them.
+</para>
+
+<para>
+"Son of Atreus," said he, "I deem that we should now turn roving
+home if we would escape destruction, for we are being cut down by
+war and pestilence at once. Let us ask some priest or prophet, or
+some reader of dreams (for dreams, too, are of Jove) who can tell
+us why Phoebus Apollo is so angry, and say whether it is for some
+vow that we have broken, or hecatomb that we have not offered,
+and whether he will accept the savour of lambs and goats without
+blemish, so as to take away the plague from us."
+</para>
+
+<para>
+With these words he sat down, and Calchas son of Thestor, wisest
+of augurs, who knew things past present and to come, rose to
+speak. He it was who had guided the Achaeans with their fleet to
+Ilius, through the prophesyings with which Phoebus Apollo had
+inspired him. With all sincerity and goodwill he addressed them
+thus:--
+</para>
+
+<para>
+"Achilles, loved of heaven, you bid me tell you about the anger
+of King Apollo, I will therefore do so; but consider first and
+swear that you will stand by me heartily in word and deed, for I
+know that I shall offend one who rules the Argives with might, to
+whom all the Achaeans are in subjection. A plain man cannot stand
+against the anger of a king, who if he swallow his displeasure
+now, will yet nurse revenge till he has wreaked it. Consider,
+therefore, whether or no you will protect me."
+</para>
+
+<para>
+And Achilles answered, "Fear not, but speak as it is borne in
+upon you from heaven, for by Apollo, Calchas, to whom you pray,
+and whose oracles you reveal to us, not a Danaan at our ships
+shall lay his hand upon you, while I yet live to look upon the
+face of the earth--no, not though you name Agamemnon himself, who
+is by far the foremost of the Achaeans."
+</para>
+
+<para>
+Thereon the seer spoke boldly. "The god," he said, "is angry
+neither about vow nor hecatomb, but for his priest's sake, whom
+Agamemnon has dishonoured, in that he would not free his daughter
+nor take a ransom for her; therefore has he sent these evils upon
+us, and will yet send others. He will not deliver the Danaans
+from this pestilence till Agamemnon has restored the girl without
+fee or ransom to her father, and has sent a holy hecatomb to
+Chryse. Thus we may perhaps appease him."
+</para>
+
+<para>
+With these words he sat down, and Agamemnon rose in anger. His
+heart was black with rage, and his eyes flashed fire as he
+scowled on Calchas and said, "Seer of evil, you never yet
+prophesied smooth things concerning me, but have ever loved to
+foretell that which was evil. You have brought me neither comfort
+nor performance; and now you come seeing among Danaans, and
+saying that Apollo has plagued us because I would not take a
+ransom for this girl, the daughter of Chryses. I have set my
+heart on keeping her in my own house, for I love her better even
+than my own wife Clytemnestra, whose peer she is alike in form
+and feature, in understanding and accomplishments. Still I will
+give her up if I must, for I would have the people live, not die;
+but you must find me a prize instead, or I alone among the
+Argives shall be without one. This is not well; for you behold,
+all of you, that my prize is to go elsewhither."
+</para>
+
+<para>
+And Achilles answered, "Most noble son of Atreus, covetous beyond
+all mankind, how shall the Achaeans find you another prize? We
+have no common store from which to take one. Those we took from
+the cities have been awarded; we cannot disallow the awards that
+have been made already. Give this girl, therefore, to the god,
+and if ever Jove grants us to sack the city of Troy we will
+requite you three and fourfold."
+</para>
+
+<para>
+Then Agamemnon said, "Achilles, valiant though you be, you shall
+not thus outwit me. You shall not overreach and you shall not
+persuade me. Are you to keep your own prize, while I sit tamely
+under my loss and give up the girl at your bidding? Let the
+Achaeans find me a prize in fair exchange to my liking, or I will
+come and take your own, or that of Ajax or of Ulysses; and he to
+whomsoever I may come shall rue my coming. But of this we will
+take thought hereafter; for the present, let us draw a ship into
+the sea, and find a crew for her expressly; let us put a hecatomb
+on board, and let us send Chryseis also; further, let some chief
+man among us be in command, either Ajax, or Idomeneus, or
+yourself, son of Peleus, mighty warrior that you are, that we may
+offer sacrifice and appease the the anger of the god."
+</para>
+
+<para>
+Achilles scowled at him and answered, "You are steeped in
+insolence and lust of gain. With what heart can any of the
+Achaeans do your bidding, either on foray or in open fighting? I
+came not warring here for any ill the Trojans had done me. I have
+no quarrel with them. They have not raided my cattle nor my
+horses, nor cut down my harvests on the rich plains of Phthia;
+for between me and them there is a great space, both mountain and
+sounding sea. We have followed you, Sir Insolence! for your
+pleasure, not ours--to gain satisfaction from the Trojans for
+your shameless self and for Menelaus. You forget this, and
+threaten to rob me of the prize for which I have toiled, and
+which the sons of the Achaeans have given me. Never when the
+Achaeans sack any rich city of the Trojans do I receive so good a
+prize as you do, though it is my hands that do the better part of
+the fighting. When the sharing comes, your share is far the
+largest, and I, forsooth, must go back to my ships, take what I
+can get and be thankful, when my labour of fighting is done. Now,
+therefore, I shall go back to Phthia; it will be much better for
+me to return home with my ships, for I will not stay here
+dishonoured to gather gold and substance for you."
+</para>
+
+<para>
+And Agamemnon answered, "Fly if you will, I shall make you no
+prayers to stay you. I have others here who will do me honour,
+and above all Jove, the lord of counsel. There is no king here so
+hateful to me as you are, for you are ever quarrelsome and ill-
+affected. What though you be brave? Was it not heaven that made
+you so? Go home, then, with your ships and comrades to lord it
+over the Myrmidons. I care neither for you nor for your anger;
+and thus will I do: since Phoebus Apollo is taking Chryseis from
+me, I shall send her with my ship and my followers, but I shall
+come to your tent and take your own prize Briseis, that you may
+learn how much stronger I am than you are, and that another may
+fear to set himself up as equal or comparable with me."
+</para>
+
+<para>
+The son of Peleus was furious, and his heart within his shaggy
+breast was divided whether to draw his sword, push the others
+aside, and kill the son of Atreus, or to restrain himself and
+check his anger. While he was thus in two minds, and was drawing
+his mighty sword from its scabbard, Minerva came down from heaven
+(for Juno had sent her in the love she bore to them both), and
+seized the son of Peleus by his yellow hair, visible to him
+alone, for of the others no man could see her. Achilles turned in
+amaze, and by the fire that flashed from her eyes at once knew
+that she was Minerva. "Why are you here," said he, "daughter of
+aegis-bearing Jove? To see the pride of Agamemnon, son of Atreus?
+Let me tell you--and it shall surely be--he shall pay for this
+insolence with his life."
+</para>
+
+<para>
+And Minerva said, "I come from heaven, if you will hear me, to
+bid you stay your anger. Juno has sent me, who cares for both of
+you alike. Cease, then, this brawling, and do not draw your
+sword; rail at him if you will, and your railing will not be
+vain, for I tell you--and it shall surely be--that you shall
+hereafter receive gifts three times as splendid by reason of this
+present insult. Hold, therefore, and obey."
+</para>
+
+<para>
+"Goddess," answered Achilles, "however angry a man may be, he
+must do as you two command him. This will be best, for the gods
+ever hear the prayers of him who has obeyed them."
+</para>
+
+<para>
+He stayed his hand on the silver hilt of his sword, and thrust it
+back into the scabbard as Minerva bade him. Then she went back to
+Olympus among the other gods, and to the house of aegis-bearing
+Jove.
+</para>
+
+<para>
+But the son of Peleus again began railing at the son of Atreus,
+for he was still in a rage. "Wine-bibber," he cried, "with the
+face of a dog and the heart of a hind, you never dare to go out
+with the host in fight, nor yet with our chosen men in ambuscade.
+You shun this as you do death itself. You had rather go round and
+rob his prizes from any man who contradicts you. You devour your
+people, for you are king over a feeble folk; otherwise, son of
+Atreus, henceforward you would insult no man. Therefore I say,
+and swear it with a great oath--nay, by this my sceptre which
+shalt sprout neither leaf nor shoot, nor bud anew from the day on
+which it left its parent stem upon the mountains--for the axe
+stripped it of leaf and bark, and now the sons of the Achaeans
+bear it as judges and guardians of the decrees of heaven--so
+surely and solemnly do I swear that hereafter they shall look
+fondly for Achilles and shall not find him. In the day of your
+distress, when your men fall dying by the murderous hand of
+Hector, you shall not know how to help them, and shall rend your
+heart with rage for the hour when you offered insult to the
+bravest of the Achaeans."
+</para>
+
+<para>
+With this the son of Peleus dashed his gold-bestudded sceptre on
+the ground and took his seat, while the son of Atreus was
+beginning fiercely from his place upon the other side. Then
+uprose smooth-tongued Nestor, the facile speaker of the Pylians,
+and the words fell from his lips sweeter than honey. Two
+generations of men born and bred in Pylos had passed away under
+his rule, and he was now reigning over the third. With all
+sincerity and goodwill, therefore, he addressed them thus:--
+</para>
+
+<para>
+"Of a truth," he said, "a great sorrow has befallen the Achaean
+land. Surely Priam with his sons would rejoice, and the Trojans
+be glad at heart if they could hear this quarrel between you two,
+who are so excellent in fight and counsel. I am older than either
+of you; therefore be guided by me. Moreover I have been the
+familiar friend of men even greater than you are, and they did
+not disregard my counsels. Never again can I behold such men as
+Pirithous and Dryas shepherd of his people, or as Caeneus,
+Exadius, godlike Polyphemus, and Theseus son of Aegeus, peer of
+the immortals. These were the mightiest men ever born upon this
+earth: mightiest were they, and when they fought the fiercest
+tribes of mountain savages they utterly overthrew them. I came
+from distant Pylos, and went about among them, for they would
+have me come, and I fought as it was in me to do. Not a man now
+living could withstand them, but they heard my words, and were
+persuaded by them. So be it also with yourselves, for this is the
+more excellent way. Therefore, Agamemnon, though you be strong,
+take not this girl away, for the sons of the Achaeans have
+already given her to Achilles; and you, Achilles, strive not
+further with the king, for no man who by the grace of Jove wields
+a sceptre has like honour with Agamemnon. You are strong, and
+have a goddess for your mother; but Agamemnon is stronger than
+you, for he has more people under him. Son of Atreus, check your
+anger, I implore you; end this quarrel with Achilles, who in the
+day of battle is a tower of strength to the Achaeans."
+</para>
+
+<para>
+And Agamemnon answered, "Sir, all that you have said is true, but
+this fellow must needs become our lord and master: he must be
+lord of all, king of all, and captain of all, and this shall
+hardly be. Granted that the gods have made him a great warrior,
+have they also given him the right to speak with railing?"
+</para>
+
+<para>
+Achilles interrupted him. "I should be a mean coward," he cried,
+"were I to give in to you in all things. Order other people
+about, not me, for I shall obey no longer. Furthermore I say--and
+lay my saying to your heart--I shall fight neither you nor any
+man about this girl, for those that take were those also that
+gave. But of all else that is at my ship you shall carry away
+nothing by force. Try, that others may see; if you do, my spear
+shall be reddened with your blood."
+</para>
+
+<para>
+When they had quarrelled thus angrily, they rose, and broke up
+the assembly at the ships of the Achaeans. The son of Peleus went
+back to his tents and ships with the son of Menoetius and his
+company, while Agamemnon drew a vessel into the water and chose a
+crew of twenty oarsmen. He escorted Chryseis on board and sent
+moreover a hecatomb for the god. And Ulysses went as captain.
+</para>
+
+<para>
+These, then, went on board and sailed their ways over the sea.
+But the son of Atreus bade the people purify themselves; so they
+purified themselves and cast their filth into the sea. Then they
+offered hecatombs of bulls and goats without blemish on the
+sea-shore, and the smoke with the savour of their sacrifice rose
+curling up towards heaven.
+</para>
+
+<para>
+Thus did they busy themselves throughout the host. But Agamemnon
+did not forget the threat that he had made Achilles, and called
+his trusty messengers and squires Talthybius and Eurybates. "Go,"
+said he, "to the tent of Achilles, son of Peleus; take Briseis by
+the hand and bring her hither; if he will not give her I shall
+come with others and take her--which will press him harder."
+</para>
+
+<para>
+He charged them straightly further and dismissed them, whereon
+they went their way sorrowfully by the seaside, till they came to
+the tents and ships of the Myrmidons. They found Achilles sitting
+by his tent and his ships, and ill-pleased he was when he beheld
+them. They stood fearfully and reverently before him, and never a
+word did they speak, but he knew them and said, "Welcome,
+heralds, messengers of gods and men; draw near; my quarrel is not
+with you but with Agamemnon who has sent you for the girl
+Briseis. Therefore, Patroclus, bring her and give her to them,
+but let them be witnesses by the blessed gods, by mortal men, and
+by the fierceness of Agamemnon's anger, that if ever again there
+be need of me to save the people from ruin, they shall seek and
+they shall not find. Agamemnon is mad with rage and knows not how
+to look before and after that the Achaeans may fight by their
+ships in safety."
+</para>
+
+<para>
+Patroclus did as his dear comrade had bidden him. He brought
+Briseis from the tent and gave her over to the heralds, who took
+her with them to the ships of the Achaeans--and the woman was
+loth to go. Then Achilles went all alone by the side of the hoar
+sea, weeping and looking out upon the boundless waste of waters.
+He raised his hands in prayer to his immortal mother, "Mother,"
+he cried, "you bore me doomed to live but for a little season;
+surely Jove, who thunders from Olympus, might have made that
+little glorious. It is not so. Agamemnon, son of Atreus, has done
+me dishonour, and has robbed me of my prize by force."
+</para>
+
+<para>
+As he spoke he wept aloud, and his mother heard him where she was
+sitting in the depths of the sea hard by the old man her father.
+Forthwith she rose as it were a grey mist out of the waves, sat
+down before him as he stood weeping, caressed him with her hand,
+and said, "My son, why are you weeping? What is it that grieves
+you? Keep it not from me, but tell me, that we may know it
+together."
+</para>
+
+<para>
+Achilles drew a deep sigh and said, "You know it; why tell you
+what you know well already? We went to Thebe the strong city of
+Eetion, sacked it, and brought hither the spoil. The sons of the
+Achaeans shared it duly among themselves, and chose lovely
+Chryseis as the meed of Agamemnon; but Chryses, priest of Apollo,
+came to the ships of the Achaeans to free his daughter, and
+brought with him a great ransom: moreover he bore in his hand the
+sceptre of Apollo, wreathed with a suppliant's wreath, and he
+besought the Achaeans, but most of all the two sons of Atreus who
+were their chiefs.
+</para>
+
+<para>
+"On this the rest of the Achaeans with one voice were for
+respecting the priest and taking the ransom that he offered; but
+not so Agamemnon, who spoke fiercely to him and sent him roughly
+away. So he went back in anger, and Apollo, who loved him dearly,
+heard his prayer. Then the god sent a deadly dart upon the
+Argives, and the people died thick on one another, for the arrows
+went everywhither among the wide host of the Achaeans. At last a
+seer in the fulness of his knowledge declared to us the oracles
+of Apollo, and I was myself first to say that we should appease
+him. Whereon the son of Atreus rose in anger, and threatened that
+which he has since done. The Achaeans are now taking the girl in
+a ship to Chryse, and sending gifts of sacrifice to the god; but
+the heralds have just taken from my tent the daughter of Briseus,
+whom the Achaeans had awarded to myself.
+</para>
+
+<para>
+"Help your brave son, therefore, if you are able. Go to Olympus,
+and if you have ever done him service in word or deed, implore
+the aid of Jove. Ofttimes in my father's house have I heard you
+glory in that you alone of the immortals saved the son of Saturn
+from ruin, when the others, with Juno, Neptune, and Pallas
+Minerva would have put him in bonds. It was you, goddess, who
+delivered him by calling to Olympus the hundred-handed monster
+whom gods call Briareus, but men Aegaeon, for he is stronger even
+than his father; when therefore he took his seat all-glorious
+beside the son of Saturn, the other gods were afraid, and did not
+bind him. Go, then, to him, remind him of all this, clasp his
+knees, and bid him give succour to the Trojans. Let the Achaeans
+be hemmed in at the sterns of their ships, and perish on the
+sea-shore, that they may reap what joy they may of their king,
+and that Agamemnon may rue his blindness in offering insult to
+the foremost of the Achaeans."
+</para>
+
+<para>
+Thetis wept and answered, "My son, woe is me that I should have
+borne or suckled you. Would indeed that you had lived your span
+free from all sorrow at your ships, for it is all too brief;
+alas, that you should be at once short of life and long of sorrow
+above your peers: woe, therefore, was the hour in which I bore
+you; nevertheless I will go to the snowy heights of Olympus, and
+tell this tale to Jove, if he will hear our prayer: meanwhile
+stay where you are with your ships, nurse your anger against the
+Achaeans, and hold aloof from fight. For Jove went yesterday to
+Oceanus, to a feast among the Ethiopians, and the other gods went
+with him. He will return to Olympus twelve days hence; I will
+then go to his mansion paved with bronze and will beseech him;
+nor do I doubt that I shall be able to persuade him."
+</para>
+
+<para>
+On this she left him, still furious at the loss of her that had
+been taken from him. Meanwhile Ulysses reached Chryse with the
+hecatomb. When they had come inside the harbour they furled the
+sails and laid them in the ship's hold; they slackened the
+forestays, lowered the mast into its place, and rowed the ship to
+the place where they would have her lie; there they cast out
+their mooring-stones and made fast the hawsers. They then got out
+upon the sea-shore and landed the hecatomb for Apollo; Chryseis
+also left the ship, and Ulysses led her to the altar to deliver
+her into the hands of her father. "Chryses," said he, "King
+Agamemnon has sent me to bring you back your child, and to offer
+sacrifice to Apollo on behalf of the Danaans, that we may
+propitiate the god, who has now brought sorrow upon the Argives."
+</para>
+
+<para>
+So saying he gave the girl over to her father, who received her
+gladly, and they ranged the holy hecatomb all orderly round the
+altar of the god. They washed their hands and took up the
+barley-meal to sprinkle over the victims, while Chryses lifted up
+his hands and prayed aloud on their behalf. "Hear me," he cried,
+"O god of the silver bow, that protectest Chryse and holy Cilla,
+and rulest Tenedos with thy might. Even as thou didst hear me
+aforetime when I prayed, and didst press hardly upon the
+Achaeans, so hear me yet again, and stay this fearful pestilence
+from the Danaans."
+</para>
+
+<para>
+Thus did he pray, and Apollo heard his prayer. When they had done
+praying and sprinkling the barley-meal, they drew back the heads
+of the victims and killed and flayed them. They cut out the
+thigh-bones, wrapped them round in two layers of fat, set some
+pieces of raw meat on the top of them, and then Chryses laid them
+on the wood fire and poured wine over them, while the young men
+stood near him with five-pronged spits in their hands. When the
+thigh-bones were burned and they had tasted the inward meats,
+they cut the rest up small, put the pieces upon the spits,
+roasted them till they were done, and drew them off: then, when
+they had finished their work and the feast was ready, they ate
+it, and every man had his full share, so that all were satisfied.
+As soon as they had had enough to eat and drink, pages filled the
+mixing-bowl with wine and water and handed it round, after giving
+every man his drink-offering.
+</para>
+
+<para>
+Thus all day long the young men worshipped the god with song,
+hymning him and chaunting the joyous paean, and the god took
+pleasure in their voices; but when the sun went down, and it came
+on dark, they laid themselves down to sleep by the stern cables
+of the ship, and when the child of morning, rosy-fingered Dawn,
+appeared they again set sail for the host of the Achaeans. Apollo
+sent them a fair wind, so they raised their mast and hoisted
+their white sails aloft. As the sail bellied with the wind the
+ship flew through the deep blue water, and the foam hissed
+against her bows as she sped onward. When they reached the
+wide-stretching host of the Achaeans, they drew the vessel
+ashore, high and dry upon the sands, set her strong props beneath
+her, and went their ways to their own tents and ships.
+</para>
+
+<para>
+But Achilles abode at his ships and nursed his anger. He went not
+to the honourable assembly, and sallied not forth to fight, but
+gnawed at his own heart, pining for battle and the war-cry.
+</para>
+
+<para>
+Now after twelve days the immortal gods came back in a body to
+Olympus, and Jove led the way. Thetis was not unmindful of the
+charge her son had laid upon her, so she rose from under the sea
+and went through great heaven with early morning to Olympus,
+where she found the mighty son of Saturn sitting all alone upon
+its topmost ridges. She sat herself down before him, and with her
+left hand seized his knees, while with her right she caught him
+under the chin, and besought him, saying:--
+</para>
+
+<para>
+"Father Jove, if I ever did you service in word or deed among the
+immortals, hear my prayer, and do honour to my son, whose life is
+to be cut short so early. King Agamemnon has dishonoured him by
+taking his prize and keeping her. Honour him then yourself,
+Olympian lord of counsel, and grant victory to the Trojans, till
+the Achaeans give my son his due and load him with riches in
+requital."
+</para>
+
+<para>
+Jove sat for a while silent, and without a word, but Thetis still
+kept firm hold of his knees, and besought him a second time.
+"Incline your head," said she, "and promise me surely, or else
+deny me--for you have nothing to fear--that I may learn how
+greatly you disdain me."
+</para>
+
+<para>
+At this Jove was much troubled and answered, "I shall have
+trouble if you set me quarrelling with Juno, for she will provoke
+me with her taunting speeches; even now she is always railing at
+me before the other gods and accusing me of giving aid to the
+Trojans. Go back now, lest she should find out. I will consider
+the matter, and will bring it about as you wish. See, I incline
+my head that you may believe me. This is the most solemn promise
+that I can give to any god. I never recall my word, or deceive,
+or fail to do what I say, when I have nodded my head."
+</para>
+
+<para>
+As he spoke the son of Saturn bowed his dark brows, and the
+ambrosial locks swayed on his immortal head, till vast Olympus
+reeled.
+</para>
+
+<para>
+When the pair had thus laid their plans, they parted--Jove to his
+house, while the goddess quitted the splendour of Olympus, and
+plunged into the depths of the sea. The gods rose from their
+seats, before the coming of their sire. Not one of them dared to
+remain sitting, but all stood up as he came among them. There,
+then, he took his seat. But Juno, when she saw him, knew that he
+and the old merman's daughter, silver-footed Thetis, had been
+hatching mischief, so she at once began to upbraid him.
+"Trickster," she cried, "which of the gods have you been taking
+into your counsels now? You are always settling matters in secret
+behind my back, and have never yet told me, if you could help it,
+one word of your intentions."
+</para>
+
+<para>
+"Juno," replied the sire of gods and men, "you must not expect to
+be informed of all my counsels. You are my wife, but you would
+find it hard to understand them. When it is proper for you to
+hear, there is no one, god or man, who will be told sooner, but
+when I mean to keep a matter to myself, you must not pry nor ask
+questions."
+</para>
+
+<para>
+"Dread son of Saturn," answered Juno, "what are you talking
+about? I? Pry and ask questions? Never. I let you have your own
+way in everything. Still, I have a strong misgiving that the old
+merman's daughter Thetis has been talking you over, for she was
+with you and had hold of your knees this self-same morning. I
+believe, therefore, that you have been promising her to give
+glory to Achilles, and to kill much people at the ships of the
+Achaeans."
+</para>
+
+<para>
+"Wife," said Jove, "I can do nothing but you suspect me and find
+it out. You will take nothing by it, for I shall only dislike you
+the more, and it will go harder with you. Granted that it is as
+you say; I mean to have it so; sit down and hold your tongue as I
+bid you for if I once begin to lay my hands about you, though all
+heaven were on your side it would profit you nothing."
+</para>
+
+<para>
+On this Juno was frightened, so she curbed her stubborn will and
+sat down in silence. But the heavenly beings were disquieted
+throughout the house of Jove, till the cunning workman Vulcan
+began to try and pacify his mother Juno. "It will be
+intolerable," said he, "if you two fall to wrangling and setting
+heaven in an uproar about a pack of mortals. If such ill counsels
+are to prevail, we shall have no pleasure at our banquet. Let me
+then advise my mother--and she must herself know that it will be
+better--to make friends with my dear father Jove, lest he again
+scold her and disturb our feast. If the Olympian Thunderer wants
+to hurl us all from our seats, he can do so, for he is far the
+strongest, so give him fair words, and he will then soon be in a
+good humour with us."
+</para>
+
+<para>
+As he spoke, he took a double cup of nectar, and placed it in his
+mother's hand. "Cheer up, my dear mother," said he, "and make the
+best of it. I love you dearly, and should be very sorry to see
+you get a thrashing; however grieved I might be, I could not help,
+for there is no standing against Jove. Once before when I was
+trying to help you, he caught me by the foot and flung me from
+the heavenly threshold. All day long from morn till eve, was I
+falling, till at sunset I came to ground in the island of Lemnos,
+and there I lay, with very little life left in me, till the
+Sintians came and tended me."
+</para>
+
+<para>
+Juno smiled at this, and as she smiled she took the cup from her
+son's hands. Then Vulcan drew sweet nectar from the mixing-bowl,
+and served it round among the gods, going from left to right; and
+the blessed gods laughed out a loud applause as they saw him
+bustling about the heavenly mansion.
+</para>
+
+<para>
+Thus through the livelong day to the going down of the sun they
+feasted, and every one had his full share, so that all were
+satisfied. Apollo struck his lyre, and the Muses lifted up their
+sweet voices, calling and answering one another. But when the
+sun's glorious light had faded, they went home to bed, each in
+his own abode, which lame Vulcan with his consummate skill had
+fashioned for them. So Jove, the Olympian Lord of Thunder, hied
+him to the bed in which he always slept; and when he had got on
+to it he went to sleep, with Juno of the golden throne by his
+side.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK II</title>
+
+<para>
+Now the other gods and the armed warriors on the plain slept
+soundly, but Jove was wakeful, for he was thinking how to do
+honour to Achilles, and destroyed much people at the ships of the
+Achaeans. In the end he deemed it would be best to send a lying
+dream to King Agamemnon; so he called one to him and said to it,
+"Lying Dream, go to the ships of the Achaeans, into the tent of
+Agamemnon, and say to him word for word as I now bid you. Tell
+him to get the Achaeans instantly under arms, for he shall take
+Troy. There are no longer divided counsels among the gods; Juno
+has brought them to her own mind, and woe betides the Trojans."
+</para>
+
+<para>
+The dream went when it had heard its message, and soon reached
+the ships of the Achaeans. It sought Agamemnon son of Atreus and
+found him in his tent, wrapped in a profound slumber. It hovered
+over his head in the likeness of Nestor, son of Neleus, whom
+Agamemnon honoured above all his councillors, and said:--
+</para>
+
+<para>
+"You are sleeping, son of Atreus; one who has the welfare of his
+host and so much other care upon his shoulders should dock his
+sleep. Hear me at once, for I come as a messenger from Jove, who,
+though he be not near, yet takes thought for you and pities you.
+He bids you get the Achaeans instantly under arms, for you shall
+take Troy. There are no longer divided counsels among the gods;
+Juno has brought them over to her own mind, and woe betides the
+Trojans at the hands of Jove. Remember this, and when you wake
+see that it does not escape you."
+</para>
+
+<para>
+The dream then left him, and he thought of things that were,
+surely not to be accomplished. He thought that on that same day
+he was to take the city of Priam, but he little knew what was in
+the mind of Jove, who had many another hard-fought fight in store
+alike for Danaans and Trojans. Then presently he woke, with the
+divine message still ringing in his ears; so he sat upright, and
+put on his soft shirt so fair and new, and over this his heavy
+cloak. He bound his sandals on to his comely feet, and slung his
+silver-studded sword about his shoulders; then he took the
+imperishable staff of his father, and sallied forth to the ships
+of the Achaeans.
+</para>
+
+<para>
+The goddess Dawn now wended her way to vast Olympus that she
+might herald day to Jove and to the other immortals, and
+Agamemnon sent the criers round to call the people in assembly;
+so they called them and the people gathered thereon. But first he
+summoned a meeting of the elders at the ship of Nestor king of
+Pylos, and when they were assembled he laid a cunning counsel
+before them.
+</para>
+
+<para>
+"My friends," said he, "I have had a dream from heaven in the
+dead of night, and its face and figure resembled none but
+Nestor's. It hovered over my head and said, 'You are sleeping,
+son of Atreus; one who has the welfare of his host and so much
+other care upon his shoulders should dock his sleep. Hear me at
+once, for I am a messenger from Jove, who, though he be not near,
+yet takes thought for you and pities you. He bids you get the
+Achaeans instantly under arms, for you shall take Troy. There are
+no longer divided counsels among the gods; Juno has brought them
+over to her own mind, and woe betides the Trojans at the hands of
+Jove. Remember this.' The dream then vanished and I awoke. Let us
+now, therefore, arm the sons of the Achaeans. But it will be well
+that I should first sound them, and to this end I will tell them
+to fly with their ships; but do you others go about among the
+host and prevent their doing so."
+</para>
+
+<para>
+He then sat down, and Nestor the prince of Pylos with all
+sincerity and goodwill addressed them thus: "My friends," said
+he, "princes and councillors of the Argives, if any other man of
+the Achaeans had told us of this dream we should have declared it
+false, and would have had nothing to do with it. But he who has
+seen it is the foremost man among us; we must therefore set about
+getting the people under arms."
+</para>
+
+<para>
+With this he led the way from the assembly, and the other
+sceptred kings rose with him in obedience to the word of
+Agamemnon; but the people pressed forward to hear. They swarmed
+like bees that sally from some hollow cave and flit in countless
+throng among the spring flowers, bunched in knots and clusters;
+even so did the mighty multitude pour from ships and tents to the
+assembly, and range themselves upon the wide-watered shore, while
+among them ran Wildfire Rumour, messenger of Jove, urging them
+ever to the fore. Thus they gathered in a pell-mell of mad
+confusion, and the earth groaned under the tramp of men as the
+people sought their places. Nine heralds went crying about among
+them to stay their tumult and bid them listen to the kings, till
+at last they were got into their several places and ceased their
+clamour. Then King Agamemnon rose, holding his sceptre. This was
+the work of Vulcan, who gave it to Jove the son of Saturn. Jove
+gave it to Mercury, slayer of Argus, guide and guardian. King
+Mercury gave it to Pelops, the mighty charioteer, and Pelops to
+Atreus, shepherd of his people. Atreus, when he died, left it to
+Thyestes, rich in flocks, and Thyestes in his turn left it to be
+borne by Agamemnon, that he might be lord of all Argos and of the
+isles. Leaning, then, on his sceptre, he addressed the Argives.
+</para>
+
+<para>
+"My friends," he said, "heroes, servants of Mars, the hand of
+heaven has been laid heavily upon me. Cruel Jove gave me his
+solemn promise that I should sack the city of Priam before
+returning, but he has played me false, and is now bidding me go
+ingloriously back to Argos with the loss of much people. Such is
+the will of Jove, who has laid many a proud city in the dust, as
+he will yet lay others, for his power is above all. It will be a
+sorry tale hereafter that an Achaean host, at once so great and
+valiant, battled in vain against men fewer in number than
+themselves; but as yet the end is not in sight. Think that the
+Achaeans and Trojans have sworn to a solemn covenant, and that
+they have each been numbered--the Trojans by the roll of their
+householders, and we by companies of ten; think further that each
+of our companies desired to have a Trojan householder to pour out
+their wine; we are so greatly more in number that full many a
+company would have to go without its cup-bearer. But they have in
+the town allies from other places, and it is these that hinder me
+from being able to sack the rich city of Ilius. Nine of Jove's
+years are gone; the timbers of our ships have rotted; their
+tackling is sound no longer. Our wives and little ones at home
+look anxiously for our coming, but the work that we came hither
+to do has not been done. Now, therefore, let us all do as I say:
+let us sail back to our own land, for we shall not take Troy."
+</para>
+
+<para>
+With these words he moved the hearts of the multitude, so many of
+them as knew not the cunning counsel of Agamemnon. They surged to
+and fro like the waves of the Icarian Sea, when the east and
+south winds break from heaven's clouds to lash them; or as when
+the west wind sweeps over a field of corn and the ears bow
+beneath the blast, even so were they swayed as they flew with
+loud cries towards the ships, and the dust from under their feet
+rose heavenward. They cheered each other on to draw the ships
+into the sea; they cleared the channels in front of them; they
+began taking away the stays from underneath them, and the welkin
+rang with their glad cries, so eager were they to return.
+</para>
+
+<para>
+Then surely the Argives would have returned after a fashion that
+was not fated. But Juno said to Minerva, "Alas, daughter of
+aegis-bearing Jove, unweariable, shall the Argives fly home to
+their own land over the broad sea, and leave Priam and the
+Trojans the glory of still keeping Helen, for whose sake so many
+of the Achaeans have died at Troy, far from their homes? Go about
+at once among the host, and speak fairly to them, man by man,
+that they draw not their ships into the sea."
+</para>
+
+<para>
+Minerva was not slack to do her bidding. Down she darted from the
+topmost summits of Olympus, and in a moment she was at the ships
+of the Achaeans. There she found Ulysses, peer of Jove in
+counsel, standing alone. He had not as yet laid a hand upon his
+ship, for he was grieved and sorry; so she went close up to him
+and said, "Ulysses, noble son of Laertes, are you going to fling
+yourselves into your ships and be off home to your own land in
+this way? Will you leave Priam and the Trojans the glory of still
+keeping Helen, for whose sake so many of the Achaeans have died
+at Troy, far from their homes? Go about at once among the host,
+and speak fairly to them, man by man, that they draw not their
+ships into the sea."
+</para>
+
+<para>
+Ulysses knew the voice as that of the goddess: he flung his cloak
+from him and set off to run. His servant Eurybates, a man of
+Ithaca, who waited on him, took charge of the cloak, whereon
+Ulysses went straight up to Agamemnon and received from him his
+ancestral, imperishable staff. With this he went about among the
+ships of the Achaeans.
+</para>
+
+<para>
+Whenever he met a king or chieftain, he stood by him and spoke
+him fairly. "Sir," said he, "this flight is cowardly and
+unworthy. Stand to your post, and bid your people also keep their
+places. You do not yet know the full mind of Agamemnon; he was
+sounding us, and ere long will visit the Achaeans with his
+displeasure. We were not all of us at the council to hear what he
+then said; see to it lest he be angry and do us a mischief; for
+the pride of kings is great, and the hand of Jove is with them."
+</para>
+
+<para>
+But when he came across any common man who was making a noise, he
+struck him with his staff and rebuked him, saying, "Sirrah, hold
+your peace, and listen to better men than yourself. You are a
+coward and no soldier; you are nobody either in fight or council;
+we cannot all be kings; it is not well that there should be many
+masters; one man must be supreme--one king to whom the son of
+scheming Saturn has given the sceptre of sovereignty over you
+all."
+</para>
+
+<para>
+Thus masterfully did he go about among the host, and the people
+hurried back to the council from their tents and ships with a
+sound as the thunder of surf when it comes crashing down upon the
+shore, and all the sea is in an uproar.
+</para>
+
+<para>
+The rest now took their seats and kept to their own several
+places, but Thersites still went on wagging his unbridled
+tongue--a man of many words, and those unseemly; a monger of
+sedition, a railer against all who were in authority, who cared
+not what he said, so that he might set the Achaeans in a laugh.
+He was the ugliest man of all those that came before
+Troy--bandy-legged, lame of one foot, with his two shoulders
+rounded and hunched over his chest. His head ran up to a point,
+but there was little hair on the top of it. Achilles and Ulysses
+hated him worst of all, for it was with them that he was most
+wont to wrangle; now, however, with a shrill squeaky voice he
+began heaping his abuse on Agamemnon. The Achaeans were angry and
+disgusted, yet none the less he kept on brawling and bawling at
+the son of Atreus.
+</para>
+
+<para>
+"Agamemnon," he cried, "what ails you now, and what more do you
+want? Your tents are filled with bronze and with fair women, for
+whenever we take a town we give you the pick of them. Would you
+have yet more gold, which some Trojan is to give you as a ransom
+for his son, when I or another Achaean has taken him prisoner? or
+is it some young girl to hide and lie with? It is not well that
+you, the ruler of the Achaeans, should bring them into such
+misery. Weakling cowards, women rather than men, let us sail
+home, and leave this fellow here at Troy to stew in his own meeds
+of honour, and discover whether we were of any service to him or
+no. Achilles is a much better man than he is, and see how he has
+treated him--robbing him of his prize and keeping it himself.
+Achilles takes it meekly and shows no fight; if he did, son of
+Atreus, you would never again insult him."
+</para>
+
+<para>
+Thus railed Thersites, but Ulysses at once went up to him and
+rebuked him sternly. "Check your glib tongue, Thersites," said
+be, "and babble not a word further. Chide not with princes when
+you have none to back you. There is no viler creature come before
+Troy with the sons of Atreus. Drop this chatter about kings, and
+neither revile them nor keep harping about going home. We do not
+yet know how things are going to be, nor whether the Achaeans are
+to return with good success or evil. How dare you gibe at
+Agamemnon because the Danaans have awarded him so many prizes? I
+tell you, therefore--and it shall surely be--that if I again
+catch you talking such nonsense, I will either forfeit my own
+head and be no more called father of Telemachus, or I will take
+you, strip you stark naked, and whip you out of the assembly till
+you go blubbering back to the ships."
+</para>
+
+<para>
+On this he beat him with his staff about the back and shoulders
+till he dropped and fell a-weeping. The golden sceptre raised a
+bloody weal on his back, so he sat down frightened and in pain,
+looking foolish as he wiped the tears from his eyes. The people
+were sorry for him, yet they laughed heartily, and one would turn
+to his neighbour saying, "Ulysses has done many a good thing ere
+now in fight and council, but he never did the Argives a better
+turn than when he stopped this fellow's mouth from prating
+further. He will give the kings no more of his insolence."
+</para>
+
+<para>
+Thus said the people. Then Ulysses rose, sceptre in hand, and
+Minerva in the likeness of a herald bade the people be still,
+that those who were far off might hear him and consider his
+council. He therefore with all sincerity and goodwill addressed
+them thus:--
+</para>
+
+<para>
+"King Agamemnon, the Achaeans are for making you a by-word among
+all mankind. They forget the promise they made you when they set
+out from Argos, that you should not return till you had sacked
+the town of Troy, and, like children or widowed women, they
+murmur and would set off homeward. True it is that they have had
+toil enough to be disheartened. A man chafes at having to stay
+away from his wife even for a single month, when he is on
+shipboard, at the mercy of wind and sea, but it is now nine long
+years that we have been kept here; I cannot, therefore, blame the
+Achaeans if they turn restive; still we shall be shamed if we go
+home empty after so long a stay--therefore, my friends, be
+patient yet a little longer that we may learn whether the
+prophesyings of Calchas were false or true.
+</para>
+
+<para>
+"All who have not since perished must remember as though it were
+yesterday or the day before, how the ships of the Achaeans were
+detained in Aulis when we were on our way hither to make war on
+Priam and the Trojans. We were ranged round about a fountain
+offering hecatombs to the gods upon their holy altars, and there
+was a fine plane-tree from beneath which there welled a stream of
+pure water. Then we saw a prodigy; for Jove sent a fearful
+serpent out of the ground, with blood-red stains upon its back,
+and it darted from under the altar on to the plane-tree. Now
+there was a brood of young sparrows, quite small, upon the
+topmost bough, peeping out from under the leaves, eight in all,
+and their mother that hatched them made nine. The serpent ate the
+poor cheeping things, while the old bird flew about lamenting her
+little ones; but the serpent threw his coils about her and caught
+her by the wing as she was screaming. Then, when he had eaten
+both the sparrow and her young, the god who had sent him made him
+become a sign; for the son of scheming Saturn turned him into
+stone, and we stood there wondering at that which had come to
+pass. Seeing, then, that such a fearful portent had broken in
+upon our hecatombs, Calchas forthwith declared to us the oracles
+of heaven. 'Why, Achaeans,' said he, 'are you thus speechless?
+Jove has sent us this sign, long in coming, and long ere it be
+fulfilled, though its fame shall last for ever. As the serpent
+ate the eight fledglings and the sparrow that hatched them, which
+makes nine, so shall we fight nine years at Troy, but in the
+tenth shall take the town.' This was what he said, and now it is
+all coming true. Stay here, therefore, all of you, till we take
+the city of Priam."
+</para>
+
+<para>
+On this the Argives raised a shout, till the ships rang again
+with the uproar. Nestor, knight of Gerene, then addressed them.
+"Shame on you," he cried, "to stay talking here like children,
+when you should fight like men. Where are our covenants now, and
+where the oaths that we have taken? Shall our counsels be flung
+into the fire, with our drink-offerings and the right hands of
+fellowship wherein we have put our trust? We waste our time in
+words, and for all our talking here shall be no further forward.
+Stand, therefore, son of Atreus, by your own steadfast purpose;
+lead the Argives on to battle, and leave this handful of men to
+rot, who scheme, and scheme in vain, to get back to Argos ere
+they have learned whether Jove be true or a liar. For the mighty
+son of Saturn surely promised that we should succeed, when we
+Argives set sail to bring death and destruction upon the Trojans.
+He showed us favourable signs by flashing his lightning on our
+right hands; therefore let none make haste to go till he has
+first lain with the wife of some Trojan, and avenged the toil and
+sorrow that he has suffered for the sake of Helen. Nevertheless,
+if any man is in such haste to be at home again, let him lay his
+hand to his ship that he may meet his doom in the sight of all.
+But, O king, consider and give ear to my counsel, for the word
+that I say may not be neglected lightly. Divide your men,
+Agamemnon, into their several tribes and clans, that clans and
+tribes may stand by and help one another. If you do this, and if
+the Achaeans obey you, you will find out who, both chiefs and
+peoples, are brave, and who are cowards; for they will vie
+against the other. Thus you shall also learn whether it is
+through the counsel of heaven or the cowardice of man that you
+shall fail to take the town."
+</para>
+
+<para>
+And Agamemnon answered, "Nestor, you have again outdone the sons
+of the Achaeans in counsel. Would, by Father Jove, Minerva, and
+Apollo, that I had among them ten more such councillors, for the
+city of King Priam would then soon fall beneath our hands, and we
+should sack it. But the son of Saturn afflicts me with bootless
+wranglings and strife. Achilles and I are quarrelling about this
+girl, in which matter I was the first to offend; if we can be of
+one mind again, the Trojans will not stave off destruction for a
+day. Now, therefore, get your morning meal, that our hosts join
+in fight. Whet well your spears; see well to the ordering of your
+shields; give good feeds to your horses, and look your chariots
+carefully over, that we may do battle the livelong day; for we
+shall have no rest, not for a moment, till night falls to part
+us. The bands that bear your shields shall be wet with the sweat
+upon your shoulders, your hands shall weary upon your spears,
+your horses shall steam in front of your chariots, and if I see
+any man shirking the fight, or trying to keep out of it at the
+ships, there shall be no help for him, but he shall be a prey to
+dogs and vultures."
+</para>
+
+<para>
+Thus he spoke, and the Achaeans roared applause. As when the
+waves run high before the blast of the south wind and break on
+some lofty headland, dashing against it and buffeting it without
+ceasing, as the storms from every quarter drive them, even so did
+the Achaeans rise and hurry in all directions to their ships.
+There they lighted their fires at their tents and got dinner,
+offering sacrifice every man to one or other of the gods, and
+praying each one of them that he might live to come out of the
+fight. Agamemnon, king of men, sacrificed a fat five-year-old
+bull to the mighty son of Saturn, and invited the princes and
+elders of his host. First he asked Nestor and King Idomeneus,
+then the two Ajaxes and the son of Tydeus, and sixthly Ulysses,
+peer of gods in counsel; but Menelaus came of his own accord, for
+he knew how busy his brother then was. They stood round the bull
+with the barley-meal in their hands, and Agamemnon prayed,
+saying, "Jove, most glorious, supreme, that dwellest in heaven,
+and ridest upon the storm-cloud, grant that the sun may not go
+down, nor the night fall, till the palace of Priam is laid low,
+and its gates are consumed with fire. Grant that my sword may
+pierce the shirt of Hector about his heart, and that full many of
+his comrades may bite the dust as they fall dying round him."
+</para>
+
+<para>
+Thus he prayed, but the son of Saturn would not fulfil his
+prayer. He accepted the sacrifice, yet none the less increased
+their toil continually. When they had done praying and sprinkling
+the barley-meal upon the victim, they drew back its head, killed
+it, and then flayed it. They cut out the thigh-bones, wrapped
+them round in two layers of fat, and set pieces of raw meat on
+the top of them. These they burned upon the split logs of
+firewood, but they spitted the inward meats, and held them in the
+flames to cook. When the thigh-bones were burned, and they had
+tasted the inward meats, they cut the rest up small, put the
+pieces upon spits, roasted them till they were done, and drew
+them off; then, when they had finished their work and the feast
+was ready, they ate it, and every man had his full share, so that
+all were satisfied. As soon as they had had enough to eat and
+drink, Nestor, knight of Gerene, began to speak. "King
+Agamemnon," said he, "let us not stay talking here, nor be slack
+in the work that heaven has put into our hands. Let the heralds
+summon the people to gather at their several ships; we will then
+go about among the host, that we may begin fighting at once."
+</para>
+
+<para>
+Thus did he speak, and Agamemnon heeded his words. He at once
+sent the criers round to call the people in assembly. So they
+called them, and the people gathered thereon. The chiefs about
+the son of Atreus chose their men and marshalled them, while
+Minerva went among them holding her priceless aegis that knows
+neither age nor death. From it there waved a hundred tassels of
+pure gold, all deftly woven, and each one of them worth a hundred
+oxen. With this she darted furiously everywhere among the hosts
+of the Achaeans, urging them forward, and putting courage into
+the heart of each, so that he might fight and do battle without
+ceasing. Thus war became sweeter in their eyes even than
+returning home in their ships. As when some great forest fire is
+raging upon a mountain top and its light is seen afar, even so as
+they marched the gleam of their armour flashed up into the
+firmament of heaven.
+</para>
+
+<para>
+They were like great flocks of geese, or cranes, or swans on the
+plain about the waters of Cayster, that wing their way hither and
+thither, glorying in the pride of flight, and crying as they
+settle till the fen is alive with their screaming. Even thus did
+their tribes pour from ships and tents on to the plain of the
+Scamander, and the ground rang as brass under the feet of men and
+horses. They stood as thick upon the flower-bespangled field as
+leaves that bloom in summer.
+</para>
+
+<para>
+As countless swarms of flies buzz around a herdsman's homestead
+in the time of spring when the pails are drenched with milk, even
+so did the Achaeans swarm on to the plain to charge the Trojans
+and destroy them.
+</para>
+
+<para>
+The chiefs disposed their men this way and that before the fight
+began, drafting them out as easily as goatherds draft their
+flocks when they have got mixed while feeding; and among them
+went King Agamemnon, with a head and face like Jove the lord of
+thunder, a waist like Mars, and a chest like that of Neptune. As
+some great bull that lords it over the herds upon the plain, even
+so did Jove make the son of Atreus stand peerless among the
+multitude of heroes.
+</para>
+
+<para>
+And now, O Muses, dwellers in the mansions of Olympus, tell me--
+for you are goddesses and are in all places so that you see all
+things, while we know nothing but by report--who were the chiefs
+and princes of the Danaans? As for the common soldiers, they were
+so that I could not name every single one of them though I had
+ten tongues, and though my voice failed not and my heart were of
+bronze within me, unless you, O Olympian Muses, daughters of
+aegis-bearing Jove, were to recount them to me. Nevertheless, I
+will tell the captains of the ships and all the fleet together.
+</para>
+
+<para>
+Peneleos, Leitus, Arcesilaus, Prothoenor, and Clonius were
+captains of the Boeotians. These were they that dwelt in Hyria
+and rocky Aulis, and who held Schoenus, Scolus, and the highlands
+of Eteonus, with Thespeia, Graia, and the fair city of
+Mycalessus. They also held Harma, Eilesium, and Erythrae; and
+they had Eleon, Hyle, and Peteon; Ocalea and the strong fortress
+of Medeon; Copae, Eutresis, and Thisbe the haunt of doves;
+Coronea, and the pastures of Haliartus; Plataea and Glisas; the
+fortress of Thebes the less; holy Onchestus with its famous grove
+of Neptune; Arne rich in vineyards; Midea, sacred Nisa, and
+Anthedon upon the sea. From these there came fifty ships, and in
+each there were a hundred and twenty young men of the Boeotians.
+</para>
+
+<para>
+Ascalaphus and Ialmenus, sons of Mars, led the people that dwelt
+in Aspledon and Orchomenus the realm of Minyas. Astyoche a noble
+maiden bore them in the house of Actor son of Azeus; for she had
+gone with Mars secretly into an upper chamber, and he had lain
+with her. With these there came thirty ships.
+</para>
+
+<para>
+The Phoceans were led by Schedius and Epistrophus, sons of mighty
+Iphitus the son of Naubolus. These were they that held
+Cyparissus, rocky Pytho, holy Crisa, Daulis, and Panopeus; they
+also that dwelt in Anemorea and Hyampolis, and about the waters
+of the river Cephissus, and Lilaea by the springs of the
+Cephissus; with their chieftains came forty ships, and they
+marshalled the forces of the Phoceans, which were stationed next
+to the Boeotians, on their left.
+</para>
+
+<para>
+Ajax, the fleet son of Oileus, commanded the Locrians. He was not
+so great, nor nearly so great, as Ajax the son of Telamon. He was
+a little man, and his breastplate was made of linen, but in use
+of the spear he excelled all the Hellenes and the Achaeans.
+These dwelt in Cynus, Opous, Calliarus, Bessa, Scarphe, fair
+Augeae, Tarphe, and Thronium about the river Boagrius. With him
+there came forty ships of the Locrians who dwell beyond Euboea.
+</para>
+
+<para>
+The fierce Abantes held Euboea with its cities, Chalcis, Eretria,
+Histiaea rich in vines, Cerinthus upon the sea, and the
+rock-perched town of Dium; with them were also the men of
+Carystus and Styra; Elephenor of the race of Mars was in command
+of these; he was son of Chalcodon, and chief over all the
+Abantes. With him they came, fleet of foot and wearing their hair
+long behind, brave warriors, who would ever strive to tear open
+the corslets of their foes with their long ashen spears. Of these
+there came fifty ships.
+</para>
+
+<para>
+And they that held the strong city of Athens, the people of great
+Erechtheus, who was born of the soil itself, but Jove's daughter,
+Minerva, fostered him, and established him at Athens in her own
+rich sanctuary. There, year by year, the Athenian youths worship
+him with sacrifices of bulls and rams. These were commanded by
+Menestheus, son of Peteos. No man living could equal him in the
+marshalling of chariots and foot soldiers. Nestor could alone
+rival him, for he was older. With him there came fifty ships.
+</para>
+
+<para>
+Ajax brought twelve ships from Salamis, and stationed them
+alongside those of the Athenians.
+</para>
+
+<para>
+The men of Argos, again, and those who held the walls of Tiryns,
+with Hermione, and Asine upon the gulf; Troezene, Eionae, and the
+vineyard lands of Epidaurus; the Achaean youths, moreover, who
+came from Aegina and Mases; these were led by Diomed of the loud
+battle-cry, and Sthenelus son of famed Capaneus. With them in
+command was Euryalus, son of king Mecisteus, son of Talaus; but
+Diomed was chief over them all. With these there came eighty
+ships.
+</para>
+
+<para>
+Those who held the strong city of Mycenae, rich Corinth and
+Cleonae; Orneae, Araethyrea, and Licyon, where Adrastus reigned
+of old; Hyperesia, high Gonoessa, and Pellene; Aegium and all the
+coast-land round about Helice; these sent a hundred ships under
+the command of King Agamemnon, son of Atreus. His force was far
+both finest and most numerous, and in their midst was the king
+himself, all glorious in his armour of gleaming bronze--foremost
+among the heroes, for he was the greatest king, and had most men
+under him.
+</para>
+
+<para>
+And those that dwelt in Lacedaemon, lying low among the hills,
+Pharis, Sparta, with Messe the haunt of doves; Bryseae, Augeae,
+Amyclae, and Helos upon the sea; Laas, moreover, and Oetylus;
+these were led by Menelaus of the loud battle-cry, brother to
+Agamemnon, and of them there were sixty ships, drawn up apart
+from the others. Among them went Menelaus himself, strong in
+zeal, urging his men to fight; for he longed to avenge the toil
+and sorrow that he had suffered for the sake of Helen.
+</para>
+
+<para>
+The men of Pylos and Arene, and Thryum where is the ford of the
+river Alpheus; strong Aipy, Cyparisseis, and Amphigenea; Pteleum,
+Helos, and Dorium, where the Muses met Thamyris, and stilled his
+minstrelsy for ever. He was returning from Oechalia, where
+Eurytus lived and reigned, and boasted that he would surpass even
+the Muses, daughters of aegis-bearing Jove, if they should sing
+against him; whereon they were angry, and maimed him. They robbed
+him of his divine power of song, and thenceforth he could strike
+the lyre no more. These were commanded by Nestor, knight of
+Gerene, and with him there came ninety ships.
+</para>
+
+<para>
+And those that held Arcadia, under the high mountain of Cyllene,
+near the tomb of Aepytus, where the people fight hand to hand;
+the men of Pheneus also, and Orchomenus rich in flocks; of
+Rhipae, Stratie, and bleak Enispe; of Tegea and fair Mantinea; of
+Stymphelus and Parrhasia; of these King Agapenor son of Ancaeus
+was commander, and they had sixty ships. Many Arcadians, good
+soldiers, came in each one of them, but Agamemnon found them the
+ships in which to cross the sea, for they were not a people that
+occupied their business upon the waters.
+</para>
+
+<para>
+The men, moreover, of Buprasium and of Elis, so much of it as is
+enclosed between Hyrmine, Myrsinus upon the sea-shore, the rock
+Olene and Alesium. These had four leaders, and each of them had
+ten ships, with many Epeans on board. Their captains were
+Amphimachus and Thalpius--the one, son of Cteatus, and the other,
+of Eurytus--both of the race of Actor. The two others were
+Diores, son of Amarynces, and Polyxenus, son of King Agasthenes,
+son of Augeas.
+</para>
+
+<para>
+And those of Dulichium with the sacred Echinean islands, who
+dwelt beyond the sea off Elis; these were led by Meges, peer of
+Mars, and the son of valiant Phyleus, dear to Jove, who
+quarrelled with his father, and went to settle in Dulichium. With
+him there came forty ships.
+</para>
+
+<para>
+Ulysses led the brave Cephallenians, who held Ithaca, Neritum
+with its forests, Crocylea, rugged Aegilips, Samos and Zacynthus,
+with the mainland also that was over against the islands. These
+were led by Ulysses, peer of Jove in counsel, and with him there
+came twelve ships.
+</para>
+
+<para>
+Thoas, son of Andraemon, commanded the Aetolians, who dwelt in
+Pleuron, Olenus, Pylene, Chalcis by the sea, and rocky Calydon,
+for the great king Oeneus had now no sons living, and was himself
+dead, as was also golden-haired Meleager, who had been set over
+the Aetolians to be their king. And with Thoas there came forty
+ships.
+</para>
+
+<para>
+The famous spearsman Idomeneus led the Cretans, who held Cnossus,
+and the well-walled city of Gortys; Lyctus also, Miletus and
+Lycastus that lies upon the chalk; the populous towns of Phaestus
+and Rhytium, with the other peoples that dwelt in the hundred
+cities of Crete. All these were led by Idomeneus, and by
+Meriones, peer of murderous Mars. And with these there came
+eighty ships.
+</para>
+
+<para>
+Tlepolemus, son of Hercules, a man both brave and large of
+stature, brought nine ships of lordly warriors from Rhodes. These
+dwelt in Rhodes which is divided among the three cities of
+Lindus, Ielysus, and Cameirus, that lies upon the chalk. These
+were commanded by Tlepolemus, son of Hercules by Astyochea, whom
+he had carried off from Ephyra, on the river Selleis, after
+sacking many cities of valiant warriors. When Tlepolemus grew up,
+he killed his father's uncle Licymnius, who had been a famous
+warrior in his time, but was then grown old. On this he built
+himself a fleet, gathered a great following, and fled beyond the
+sea, for he was menaced by the other sons and grandsons of
+Hercules. After a voyage, during which he suffered great
+hardship, he came to Rhodes, where the people divided into three
+communities, according to their tribes, and were dearly loved by
+Jove, the lord of gods and men; wherefore the son of Saturn
+showered down great riches upon them.
+</para>
+
+<para>
+And Nireus brought three ships from Syme--Nireus, who was the
+handsomest man that came up under Ilius of all the Danaans after
+the son of Peleus--but he was a man of no substance, and had but
+a small following.
+</para>
+
+<para>
+And those that held Nisyrus, Crapathus, and Casus, with Cos, the
+city of Eurypylus, and the Calydnian islands, these were
+commanded by Pheidippus and Antiphus, two sons of King Thessalus
+the son of Hercules. And with them there came thirty ships.
+</para>
+
+<para>
+Those again who held Pelasgic Argos, Alos, Alope, and Trachis;
+and those of Phthia and Hellas the land of fair women, who were
+called Myrmidons, Hellenes, and Achaeans; these had fifty ships,
+over which Achilles was in command. But they now took no part in
+the war, inasmuch as there was no one to marshal them; for
+Achilles stayed by his ships, furious about the loss of the girl
+Briseis, whom he had taken from Lyrnessus at his own great peril,
+when he had sacked Lyrnessus and Thebe, and had overthrown Mynes
+and Epistrophus, sons of king Evenor, son of Selepus. For her
+sake Achilles was still grieving, but ere long he was again to
+join them.
+</para>
+
+<para>
+And those that held Phylace and the flowery meadows of Pyrasus,
+sanctuary of Ceres; Iton, the mother of sheep; Antrum upon the
+sea, and Pteleum that lies upon the grass lands. Of these brave
+Protesilaus had been captain while he was yet alive, but he was
+now lying under the earth. He had left a wife behind him in
+Phylace to tear her cheeks in sorrow, and his house was only half
+finished, for he was slain by a Dardanian warrior while leaping
+foremost of the Achaeans upon the soil of Troy. Still, though his
+people mourned their chieftain, they were not without a leader,
+for Podarces, of the race of Mars, marshalled them; he was son of
+Iphiclus, rich in sheep, who was the son of Phylacus, and he was
+own brother to Protesilaus, only younger, Protesilaus being at
+once the elder and the more valiant. So the people were not
+without a leader, though they mourned him whom they had lost.
+With him there came forty ships.
+</para>
+
+<para>
+And those that held Pherae by the Boebean lake, with Boebe,
+Glaphyrae, and the populous city of Iolcus, these with their
+eleven ships were led by Eumelus, son of Admetus, whom Alcestis
+bore to him, loveliest of the daughters of Pelias.
+</para>
+
+<para>
+And those that held Methone and Thaumacia, with Meliboea and
+rugged Olizon, these were led by the skilful archer Philoctetes,
+and they had seven ships, each with fifty oarsmen all of them
+good archers; but Philoctetes was lying in great pain in the
+Island of Lemnos, where the sons of the Achaeans left him, for he
+had been bitten by a poisonous water snake. There he lay sick and
+sorry, and full soon did the Argives come to miss him. But his
+people, though they felt his loss were not leaderless, for Medon,
+the bastard son of Oileus by Rhene, set them in array.
+</para>
+
+<para>
+Those, again, of Tricca and the stony region of Ithome, and they
+that held Oechalia, the city of Oechalian Eurytus, these were
+commanded by the two sons of Aesculapius, skilled in the art of
+healing, Podalirius and Machaon. And with them there came thirty
+ships.
+</para>
+
+<para>
+The men, moreover, of Ormenius, and by the fountain of Hypereia,
+with those that held Asterius, and the white crests of Titanus,
+these were led by Eurypylus, the son of Euaemon, and with them
+there came forty ships.
+</para>
+
+<para>
+Those that held Argissa and Gyrtone, Orthe, Elone, and the white
+city of Oloosson, of these brave Polypoetes was leader. He was
+son of Pirithous, who was son of Jove himself, for Hippodameia
+bore him to Pirithous on the day when he took his revenge on the
+shaggy mountain savages and drove them from Mt. Pelion to the
+Aithices. But Polypoetes was not sole in command, for with him
+was Leonteus, of the race of Mars, who was son of Coronus, the
+son of Caeneus. And with these there came forty ships.
+</para>
+
+<para>
+Guneus brought two and twenty ships from Cyphus, and he was
+followed by the Enienes and the valiant Peraebi, who dwelt about
+wintry Dodona, and held the lands round the lovely river
+Titaresius, which sends its waters into the Peneus. They do not
+mingle with the silver eddies of the Peneus, but flow on the top
+of them like oil; for the Titaresius is a branch of dread Orcus
+and of the river Styx.
+</para>
+
+<para>
+Of the Magnetes, Prothous son of Tenthredon was commander. They
+were they that dwelt about the river Peneus and Mt. Pelion.
+Prothous, fleet of foot, was their leader, and with him there
+came forty ships.
+</para>
+
+<para>
+Such were the chiefs and princes of the Danaans. Who, then, O
+Muse, was the foremost, whether man or horse, among those that
+followed after the sons of Atreus?
+</para>
+
+<para>
+Of the horses, those of the son of Pheres were by far the finest.
+They were driven by Eumelus, and were as fleet as birds. They
+were of the same age and colour, and perfectly matched in height.
+Apollo, of the silver bow, had bred them in Perea--both of them
+mares, and terrible as Mars in battle. Of the men, Ajax, son of
+Telamon, was much the foremost so long as Achilles' anger lasted,
+for Achilles excelled him greatly and he had also better horses;
+but Achilles was now holding aloof at his ships by reason of his
+quarrel with Agamemnon, and his people passed their time upon the
+sea shore, throwing discs or aiming with spears at a mark, and in
+archery. Their horses stood each by his own chariot, champing
+lotus and wild celery. The chariots were housed under cover, but
+their owners, for lack of leadership, wandered hither and thither
+about the host and went not forth to fight.
+</para>
+
+<para>
+Thus marched the host like a consuming fire, and the earth
+groaned beneath them when the lord of thunder is angry and lashes
+the land about Typhoeus among the Arimi, where they say Typhoeus
+lies. Even so did the earth groan beneath them as they sped over
+the plain.
+</para>
+
+<para>
+And now Iris, fleet as the wind, was sent by Jove to tell the bad
+news among the Trojans. They were gathered in assembly, old and
+young, at Priam's gates, and Iris came close up to Priam,
+speaking with the voice of Priam's son Polites, who, being fleet
+of foot, was stationed as watchman for the Trojans on the tomb of
+old Aesyetes, to look out for any sally of the Achaeans. In his
+likeness Iris spoke, saying, "Old man, you talk idly, as in time
+of peace, while war is at hand. I have been in many a battle, but
+never yet saw such a host as is now advancing. They are crossing
+the plain to attack the city as thick as leaves or as the sands
+of the sea. Hector, I charge you above all others, do as I say.
+There are many allies dispersed about the city of Priam from
+distant places and speaking divers tongues. Therefore, let each
+chief give orders to his own people, setting them severally in
+array and leading them forth to battle."
+</para>
+
+<para>
+Thus she spoke, but Hector knew that it was the goddess, and at
+once broke up the assembly. The men flew to arms; all the gates
+were opened, and the people thronged through them, horse and
+foot, with the tramp as of a great multitude.
+</para>
+
+<para>
+Now there is a high mound before the city, rising by itself upon
+the plain. Men call it Batieia, but the gods know that it is the
+tomb of lithe Myrine. Here the Trojans and their allies divided
+their forces.
+</para>
+
+<para>
+Priam's son, great Hector of the gleaming helmet, commanded the
+Trojans, and with him were arrayed by far the greater number and
+most valiant of those who were longing for the fray.
+</para>
+
+<para>
+The Dardanians were led by brave Aeneas, whom Venus bore to
+Anchises, when she, goddess though she was, had lain with him
+upon the mountain slopes of Ida. He was not alone, for with him
+were the two sons of Antenor, Archilochus and Acamas, both
+skilled in all the arts of war.
+</para>
+
+<para>
+They that dwelt in Telea under the lowest spurs of Mt. Ida, men
+of substance, who drink the limpid waters of the Aesepus, and are
+of Trojan blood--these were led by Pandarus son of Lycaon, whom
+Apollo had taught to use the bow.
+</para>
+
+<para>
+They that held Adresteia and the land of Apaesus, with Pityeia,
+and the high mountain of Tereia--these were led by Adrestus and
+Amphius, whose breastplate was of linen. These were the sons of
+Merops of Percote, who excelled in all kinds of divination. He
+told them not to take part in the war, but they gave him no heed,
+for fate lured them to destruction.
+</para>
+
+<para>
+They that dwelt about Percote and Practius, with Sestos, Abydos,
+and Arisbe--these were led by Asius, son of Hyrtacus, a brave
+commander--Asius, the son of Hyrtacus, whom his powerful dark bay
+steeds, of the breed that comes from the river Selleis, had
+brought from Arisbe.
+</para>
+
+<para>
+Hippothous led the tribes of Pelasgian spearsmen, who dwelt in
+fertile Larissa--Hippothous, and Pylaeus of the race of Mars, two
+sons of the Pelasgian Lethus, son of Teutamus.
+</para>
+
+<para>
+Acamas and the warrior Peirous commanded the Thracians and those
+that came from beyond the mighty stream of the Hellespont.
+</para>
+
+<para>
+Euphemus, son of Troezenus, the son of Ceos, was captain of the
+Ciconian spearsmen.
+</para>
+
+<para>
+Pyraechmes led the Paeonian archers from distant Amydon, by the
+broad waters of the river Axius, the fairest that flow upon the
+earth.
+</para>
+
+<para>
+The Paphlagonians were commanded by stout-hearted Pylaemanes from
+Enetae, where the mules run wild in herds. These were they that
+held Cytorus and the country round Sesamus, with the cities by
+the river Parthenius, Cromna, Aegialus, and lofty Erithini.
+</para>
+
+<para>
+Odius and Epistrophus were captains over the Halizoni from
+distant Alybe, where there are mines of silver.
+</para>
+
+<para>
+Chromis, and Ennomus the augur, led the Mysians, but his skill in
+augury availed not to save him from destruction, for he fell by
+the hand of the fleet descendant of Aeacus in the river, where he
+slew others also of the Trojans.
+</para>
+
+<para>
+Phorcys, again, and noble Ascanius led the Phrygians from the far
+country of Ascania, and both were eager for the fray.
+</para>
+
+<para>
+Mesthles and Antiphus commanded the Meonians, sons of Talaemenes,
+born to him of the Gygaean lake. These led the Meonians, who
+dwelt under Mt. Tmolus.
+</para>
+
+<para>
+Nastes led the Carians, men of a strange speech. These held
+Miletus and the wooded mountain of Phthires, with the water of
+the river Maeander and the lofty crests of Mt. Mycale. These were
+commanded by Nastes and Amphimachus, the brave sons of Nomion. He
+came into the fight with gold about him, like a girl; fool that
+he was, his gold was of no avail to save him, for he fell in the
+river by the hand of the fleet descendant of Aeacus, and Achilles
+bore away his gold.
+</para>
+
+<para>
+Sarpedon and Glaucus led the Lycians from their distant land, by
+the eddying waters of the Xanthus.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK III</title>
+
+<para>
+When the companies were thus arrayed, each under its own captain,
+the Trojans advanced as a flight of wild fowl or cranes that
+scream overhead when rain and winter drive them over the flowing
+waters of Oceanus to bring death and destruction on the Pygmies,
+and they wrangle in the air as they fly; but the Achaeans marched
+silently, in high heart, and minded to stand by one another.
+</para>
+
+<para>
+As when the south wind spreads a curtain of mist upon the
+mountain tops, bad for shepherds but better than night for
+thieves, and a man can see no further than he can throw a stone,
+even so rose the dust from under their feet as they made all
+speed over the plain.
+</para>
+
+<para>
+When they were close up with one another, Alexandrus came forward
+as champion on the Trojan side. On his shoulders he bore the skin
+of a panther, his bow, and his sword, and he brandished two
+spears shod with bronze as a challenge to the bravest of the
+Achaeans to meet him in single fight. Menelaus saw him thus
+stride out before the ranks, and was glad as a hungry lion that
+lights on the carcase of some goat or horned stag, and devours it
+there and then, though dogs and youths set upon him. Even thus
+was Menelaus glad when his eyes caught sight of Alexandrus, for
+he deemed that now he should be revenged. He sprang, therefore,
+from his chariot, clad in his suit of armour.
+</para>
+
+<para>
+Alexandrus quailed as he saw Menelaus come forward, and shrank in
+fear of his life under cover of his men. As one who starts back
+affrighted, trembling and pale, when he comes suddenly upon a
+serpent in some mountain glade, even so did Alexandrus plunge
+into the throng of Trojan warriors, terror-stricken at the sight
+of the son of Atreus.
+</para>
+
+<para>
+Then Hector upbraided him. "Paris," said he, "evil-hearted Paris,
+fair to see, but woman-mad, and false of tongue, would that you
+had never been born, or that you had died unwed. Better so, than
+live to be disgraced and looked askance at. Will not the Achaeans
+mock at us and say that we have sent one to champion us who is
+fair to see but who has neither wit nor courage? Did you not,
+such as you are, get your following together and sail beyond the
+seas? Did you not from your a far country carry off a lovely
+woman wedded among a people of warriors--to bring sorrow upon
+your father, your city, and your whole country, but joy to your
+enemies, and hang-dog shamefacedness to yourself? And now can you
+not dare face Menelaus and learn what manner of man he is whose
+wife you have stolen? Where indeed would be your lyre and your
+love-tricks, your comely locks and your fair favour, when you
+were lying in the dust before him? The Trojans are a weak-kneed
+people, or ere this you would have had a shirt of stones for the
+wrongs you have done them."
+</para>
+
+<para>
+And Alexandrus answered, "Hector, your rebuke is just. You are
+hard as the axe which a shipwright wields at his work, and
+cleaves the timber to his liking. As the axe in his hand, so keen
+is the edge of your scorn. Still, taunt me not with the gifts
+that golden Venus has given me; they are precious; let not a man
+disdain them, for the gods give them where they are minded, and
+none can have them for the asking. If you would have me do battle
+with Menelaus, bid the Trojans and Achaeans take their seats,
+while he and I fight in their midst for Helen and all her wealth.
+Let him who shall be victorious and prove to be the better man
+take the woman and all she has, to bear them to his home, but let
+the rest swear to a solemn covenant of peace whereby you Trojans
+shall stay here in Troy, while the others go home to Argos and
+the land of the Achaeans."
+</para>
+
+<para>
+When Hector heard this he was glad, and went about among the
+Trojan ranks holding his spear by the middle to keep them back,
+and they all sat down at his bidding: but the Achaeans still
+aimed at him with stones and arrows, till Agamemnon shouted to
+them saying, "Hold, Argives, shoot not, sons of the Achaeans;
+Hector desires to speak."
+</para>
+
+<para>
+They ceased taking aim and were still, whereon Hector spoke.
+"Hear from my mouth," said he, "Trojans and Achaeans, the saying
+of Alexandrus, through whom this quarrel has come about. He bids
+the Trojans and Achaeans lay their armour upon the ground, while
+he and Menelaus fight in the midst of you for Helen and all her
+wealth. Let him who shall be victorious and prove to be the
+better man take the woman and all she has, to bear them to his
+own home, but let the rest swear to a solemn covenant of peace."
+</para>
+
+<para>
+Thus he spoke, and they all held their peace, till Menelaus of
+the loud battle-cry addressed them. "And now," he said, "hear me
+too, for it is I who am the most aggrieved. I deem that the
+parting of Achaeans and Trojans is at hand, as well it may be,
+seeing how much have suffered for my quarrel with Alexandrus and
+the wrong he did me. Let him who shall die, die, and let the
+others fight no more. Bring, then, two lambs, a white ram and a
+black ewe, for Earth and Sun, and we will bring a third for Jove.
+Moreover, you shall bid Priam come, that he may swear to the
+covenant himself; for his sons are high-handed and ill to trust,
+and the oaths of Jove must not be transgressed or taken in vain.
+Young men's minds are light as air, but when an old man comes he
+looks before and after, deeming that which shall be fairest upon
+both sides."
+</para>
+
+<para>
+The Trojans and Achaeans were glad when they heard this, for they
+thought that they should now have rest. They backed their
+chariots toward the ranks, got out of them, and put off their
+armour, laying it down upon the ground; and the hosts were near
+to one another with a little space between them. Hector sent two
+messengers to the city to bring the lambs and to bid Priam come,
+while Agamemnon told Talthybius to fetch the other lamb from the
+ships, and he did as Agamemnon had said.
+</para>
+
+<para>
+Meanwhile Iris went to Helen in the form of her sister-in-law,
+wife of the son of Antenor, for Helicaon, son of Antenor, had
+married Laodice, the fairest of Priam's daughters. She found her
+in her own room, working at a great web of purple linen, on which
+she was embroidering the battles between Trojans and Achaeans,
+that Mars had made them fight for her sake. Iris then came close
+up to her and said, "Come hither, child, and see the strange
+doings of the Trojans and Achaeans. Till now they have been
+warring upon the plain, mad with lust of battle, but now they
+have left off fighting, and are leaning upon their shields,
+sitting still with their spears planted beside them. Alexandrus
+and Menelaus are going to fight about yourself, and you are to be
+the wife of him who is the victor."
+</para>
+
+<para>
+Thus spoke the goddess, and Helen's heart yearned after her
+former husband, her city, and her parents. She threw a white
+mantle over her head, and hurried from her room, weeping as she
+went, not alone, but attended by two of her handmaids, Aethrae,
+daughter of Pittheus, and Clymene. And straightway they were at
+the Scaean gates.
+</para>
+
+<para>
+The two sages, Ucalegon and Antenor, elders of the people, were
+seated by the Scaean gates, with Priam, Panthous, Thymoetes,
+Lampus, Clytius, and Hiketaon of the race of Mars. These were too
+old to fight, but they were fluent orators, and sat on the tower
+like cicales that chirrup delicately from the boughs of some high
+tree in a wood. When they saw Helen coming towards the tower,
+they said softly to one another, "Small wonder that Trojans and
+Achaeans should endure so much and so long, for the sake of a
+woman so marvellously and divinely lovely. Still, fair though she
+be, let them take her and go, or she will breed sorrow for us and
+for our children after us."
+</para>
+
+<para>
+But Priam bade her draw nigh. "My child," said he, "take your
+seat in front of me that you may see your former husband, your
+kinsmen and your friends. I lay no blame upon you, it is the
+gods, not you who are to blame. It is they that have brought
+about this terrible war with the Achaeans. Tell me, then, who is
+yonder huge hero so great and goodly? I have seen men taller by a
+head, but none so comely and so royal. Surely he must be a king."
+</para>
+
+<para>
+"Sir," answered Helen, "father of my husband, dear and reverend
+in my eyes, would that I had chosen death rather than to have
+come here with your son, far from my bridal chamber, my friends,
+my darling daughter, and all the companions of my girlhood. But
+it was not to be, and my lot is one of tears and sorrow. As for
+your question, the hero of whom you ask is Agamemnon, son of
+Atreus, a good king and a brave soldier, brother-in-law as surely
+as that he lives, to my abhorred and miserable self."
+</para>
+
+<para>
+The old man marvelled at him and said, "Happy son of Atreus,
+child of good fortune. I see that the Achaeans are subject to you
+in great multitudes. When I was in Phrygia I saw much horsemen,
+the people of Otreus and of Mygdon, who were camping upon the
+banks of the river Sangarius; I was their ally, and with them
+when the Amazons, peers of men, came up against them, but even
+they were not so many as the Achaeans."
+</para>
+
+<para>
+The old man next looked upon Ulysses; "Tell me," he said, "who is
+that other, shorter by a head than Agamemnon, but broader across
+the chest and shoulders? His armour is laid upon the ground, and
+he stalks in front of the ranks as it were some great woolly ram
+ordering his ewes."
+</para>
+
+<para>
+And Helen answered, "He is Ulysses, a man of great craft, son of
+Laertes. He was born in rugged Ithaca, and excels in all manner
+of stratagems and subtle cunning."
+</para>
+
+<para>
+On this Antenor said, "Madam, you have spoken truly. Ulysses once
+came here as envoy about yourself, and Menelaus with him. I
+received them in my own house, and therefore know both of them by
+sight and conversation. When they stood up in presence of the
+assembled Trojans, Menelaus was the broader shouldered, but when
+both were seated Ulysses had the more royal presence. After a
+time they delivered their message, and the speech of Menelaus ran
+trippingly on the tongue; he did not say much, for he was a man
+of few words, but he spoke very clearly and to the point, though
+he was the younger man of the two; Ulysses, on the other hand,
+when he rose to speak, was at first silent and kept his eyes
+fixed upon the ground. There was no play nor graceful movement of
+his sceptre; he kept it straight and stiff like a man unpractised
+in oratory--one might have taken him for a mere churl or
+simpleton; but when he raised his voice, and the words came
+driving from his deep chest like winter snow before the wind,
+then there was none to touch him, and no man thought further of
+what he looked like."
+</para>
+
+<para>
+Priam then caught sight of Ajax and asked, "Who is that great and
+goodly warrior whose head and broad shoulders tower above the
+rest of the Argives?"
+</para>
+
+<para>
+"That," answered Helen, "is huge Ajax, bulwark of the Achaeans,
+and on the other side of him, among the Cretans, stands Idomeneus
+looking like a god, and with the captains of the Cretans round
+him. Often did Menelaus receive him as a guest in our house when
+he came visiting us from Crete. I see, moreover, many other
+Achaeans whose names I could tell you, but there are two whom I
+can nowhere find, Castor, breaker of horses, and Pollux the
+mighty boxer; they are children of my mother, and own brothers to
+myself. Either they have not left Lacedaemon, or else, though
+they have brought their ships, they will not show themselves in
+battle for the shame and disgrace that I have brought upon them."
+</para>
+
+<para>
+She knew not that both these heroes were already lying under the
+earth in their own land of Lacedaemon.
+</para>
+
+<para>
+Meanwhile the heralds were bringing the holy oath-offerings
+through the city--two lambs and a goatskin of wine, the gift of
+earth; and Idaeus brought the mixing bowl and the cups of gold.
+He went up to Priam and said, "Son of Laomedon, the princes of
+the Trojans and Achaeans bid you come down on to the plain and
+swear to a solemn covenant. Alexandrus and Menelaus are to fight
+for Helen in single combat, that she and all her wealth may go
+with him who is the victor. We are to swear to a solemn covenant
+of peace whereby we others shall dwell here in Troy, while the
+Achaeans return to Argos and the land of the Achaeans."
+</para>
+
+<para>
+The old man trembled as he heard, but bade his followers yoke the
+horses, and they made all haste to do so. He mounted the chariot,
+gathered the reins in his hand, and Antenor took his seat beside
+him; they then drove through the Scaean gates on to the plain.
+When they reached the ranks of the Trojans and Achaeans they left
+the chariot, and with measured pace advanced into the space
+between the hosts.
+</para>
+
+<para>
+Agamemnon and Ulysses both rose to meet them. The attendants
+brought on the oath-offerings and mixed the wine in the
+mixing-bowls; they poured water over the hands of the chieftains,
+and the son of Atreus drew the dagger that hung by his sword, and
+cut wool from the lambs' heads; this the men-servants gave about
+among the Trojan and Achaean princes, and the son of Atreus
+lifted up his hands in prayer. "Father Jove," he cried, "that
+rulest in Ida, most glorious in power, and thou oh Sun, that
+seest and givest ear to all things, Earth and Rivers, and ye who
+in the realms below chastise the soul of him that has broken his
+oath, witness these rites and guard them, that they be not vain.
+If Alexandrus kills Menelaus, let him keep Helen and all her
+wealth, while we sail home with our ships; but if Menelaus kills
+Alexandrus, let the Trojans give back Helen and all that she has;
+let them moreover pay such fine to the Achaeans as shall be
+agreed upon, in testimony among those that shall be born
+hereafter. And if Priam and his sons refuse such fine when
+Alexandrus has fallen, then will I stay here and fight on till I
+have got satisfaction."
+</para>
+
+<para>
+As he spoke he drew his knife across the throats of the victims,
+and laid them down gasping and dying upon the ground, for the
+knife had reft them of their strength. Then they poured wine from
+the mixing-bowl into the cups, and prayed to the everlasting
+gods, saying, Trojans and Achaeans among one another, "Jove, most
+great and glorious, and ye other everlasting gods, grant that the
+brains of them who shall first sin against their oaths--of them
+and their children--may be shed upon the ground even as this
+wine, and let their wives become the slaves of strangers."
+</para>
+
+<para>
+Thus they prayed, but not as yet would Jove grant them their
+prayer. Then Priam, descendant of Dardanus, spoke, saying, "Hear
+me, Trojans and Achaeans, I will now go back to the wind-beaten
+city of Ilius: I dare not with my own eyes witness this fight
+between my son and Menelaus, for Jove and the other immortals
+alone know which shall fall."
+</para>
+
+<para>
+On this he laid the two lambs on his chariot and took his seat.
+He gathered the reins in his hand, and Antenor sat beside him;
+the two then went back to Ilius. Hector and Ulysses measured the
+ground, and cast lots from a helmet of bronze to see which should
+take aim first. Meanwhile the two hosts lifted up their hands and
+prayed saying, "Father Jove, that rulest from Ida, most glorious
+in power, grant that he who first brought about this war between
+us may die, and enter the house of Hades, while we others remain
+at peace and abide by our oaths."
+</para>
+
+<para>
+Great Hector now turned his head aside while he shook the helmet,
+and the lot of Paris flew out first. The others took their
+several stations, each by his horses and the place where his arms
+were lying, while Alexandrus, husband of lovely Helen, put on his
+goodly armour. First he greaved his legs with greaves of good
+make and fitted with ancle-clasps of silver; after this he donned
+the cuirass of his brother Lycaon, and fitted it to his own body;
+he hung his silver-studded sword of bronze about his shoulders,
+and then his mighty shield. On his comely head he set his helmet,
+well-wrought, with a crest of horse-hair that nodded menacingly
+above it, and he grasped a redoubtable spear that suited his
+hands. In like fashion Menelaus also put on his armour.
+</para>
+
+<para>
+When they had thus armed, each amid his own people, they strode
+fierce of aspect into the open space, and both Trojans and
+Achaeans were struck with awe as they beheld them. They stood
+near one another on the measured ground, brandishing their
+spears, and each furious against the other. Alexandrus aimed
+first, and struck the round shield of the son of Atreus, but the
+spear did not pierce it, for the shield turned its point.
+Menelaus next took aim, praying to Father Jove as he did so.
+"King Jove," he said, "grant me revenge on Alexandrus who has
+wronged me; subdue him under my hand that in ages yet to come a
+man may shrink from doing ill deeds in the house of his host."
+</para>
+
+<para>
+He poised his spear as he spoke, and hurled it at the shield of
+Alexandrus. Through shield and cuirass it went, and tore the
+shirt by his flank, but Alexandrus swerved aside, and thus saved
+his life. Then the son of Atreus drew his sword, and drove at the
+projecting part of his helmet, but the sword fell shivered in
+three or four pieces from his hand, and he cried, looking towards
+Heaven, "Father Jove, of all gods thou art the most despiteful; I
+made sure of my revenge, but the sword has broken in my hand, my
+spear has been hurled in vain, and I have not killed him."
+</para>
+
+<para>
+With this he flew at Alexandrus, caught him by the horsehair
+plume of his helmet, and began dragging him towards the Achaeans.
+The strap of the helmet that went under his chin was choking him,
+and Menelaus would have dragged him off to his own great glory
+had not Jove's daughter Venus been quick to mark and to break the
+strap of oxhide, so that the empty helmet came away in his hand.
+This he flung to his comrades among the Achaeans, and was again
+springing upon Alexandrus to run him through with a spear, but
+Venus snatched him up in a moment (as a god can do), hid him
+under a cloud of darkness, and conveyed him to his own
+bedchamber.
+</para>
+
+<para>
+Then she went to call Helen, and found her on a high tower with
+the Trojan women crowding round her. She took the form of an old
+woman who used to dress wool for her when she was still in
+Lacedaemon, and of whom she was very fond. Thus disguised she
+plucked her by perfumed robe and said, "Come hither; Alexandrus
+says you are to go to the house; he is on his bed in his own
+room, radiant with beauty and dressed in gorgeous apparel. No one
+would think he had just come from fighting, but rather that he
+was going to a dance, or had done dancing and was sitting down."
+</para>
+
+<para>
+With these words she moved the heart of Helen to anger. When she
+marked the beautiful neck of the goddess, her lovely bosom, and
+sparkling eyes, she marvelled at her and said, "Goddess, why do
+you thus beguile me? Are you going to send me afield still
+further to some man whom you have taken up in Phrygia or fair
+Meonia? Menelaus has just vanquished Alexandrus, and is to take
+my hateful self back with him. You are come here to betray me. Go
+sit with Alexandrus yourself; henceforth be goddess no longer;
+never let your feet carry you back to Olympus; worry about him
+and look after him till he make you his wife, or, for the matter
+of that, his slave--but me? I shall not go; I can garnish his bed
+no longer; I should be a by-word among all the women of Troy.
+Besides, I have trouble on my mind."
+</para>
+
+<para>
+Venus was very angry, and said, "Bold hussy, do not provoke me;
+if you do, I shall leave you to your fate and hate you as much as
+I have loved you. I will stir up fierce hatred between Trojans
+and Achaeans, and you shall come to a bad end."
+</para>
+
+<para>
+At this Helen was frightened. She wrapped her mantle about her
+and went in silence, following the goddess and unnoticed by the
+Trojan women.
+</para>
+
+<para>
+When they came to the house of Alexandrus the maid-servants set
+about their work, but Helen went into her own room, and the
+laughter-loving goddess took a seat and set it for her facing
+Alexandrus. On this Helen, daughter of aegis-bearing Jove, sat
+down, and with eyes askance began to upbraid her husband.
+</para>
+
+<para>
+"So you are come from the fight," said she; "would that you had
+fallen rather by the hand of that brave man who was my husband.
+You used to brag that you were a better man with hands and spear
+than Menelaus. Go, then, and challenge him again--but I should
+advise you not to do so, for if you are foolish enough to meet
+him in single combat, you will soon fall by his spear."
+</para>
+
+<para>
+And Paris answered, "Wife, do not vex me with your reproaches.
+This time, with the help of Minerva, Menelaus has vanquished me;
+another time I may myself be victor, for I too have gods that
+will stand by me. Come, let us lie down together and make
+friends. Never yet was I so passionately enamoured of you as at
+this moment--not even when I first carried you off from
+Lacedaemon and sailed away with you--not even when I had converse
+with you upon the couch of love in the island of Cranae was I so
+enthralled by desire of you as now." On this he led her towards
+the bed, and his wife went with him.
+</para>
+
+<para>
+Thus they laid themselves on the bed together; but the son of
+Atreus strode among the throng, looking everywhere for
+Alexandrus, and no man, neither of the Trojans nor of the allies,
+could find him. If they had seen him they were in no mind to hide
+him, for they all of them hated him as they did death itself.
+Then Agamemnon, king of men, spoke, saying, "Hear me, Trojans,
+Dardanians, and allies. The victory has been with Menelaus;
+therefore give back Helen with all her wealth, and pay such fine
+as shall be agreed upon, in testimony among them that shall be
+born hereafter."
+</para>
+
+<para>
+Thus spoke the son of Atreus, and the Achaeans shouted in
+applause.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK IV</title>
+
+<para>
+Now the gods were sitting with Jove in council upon the golden
+floor while Hebe went round pouring out nectar for them to drink,
+and as they pledged one another in their cups of gold they looked
+down upon the town of Troy. The son of Saturn then began to tease
+Juno, talking at her so as to provoke her. "Menelaus," said he,
+"has two good friends among the goddesses, Juno of Argos, and
+Minerva of Alalcomene, but they only sit still and look on, while
+Venus keeps ever by Alexandrus' side to defend him in any danger;
+indeed she has just rescued him when he made sure that it was all
+over with him--for the victory really did lie with Menelaus. We
+must consider what we shall do about all this; shall we set them
+fighting anew or make peace between them? If you will agree to
+this last Menelaus can take back Helen and the city of Priam may
+remain still inhabited."
+</para>
+
+<para>
+Minerva and Juno muttered their discontent as they sat side by
+side hatching mischief for the Trojans. Minerva scowled at her
+father, for she was in a furious passion with him, and said
+nothing, but Juno could not contain herself. "Dread son of
+Saturn," said she, "what, pray, is the meaning of all this? Is my
+trouble, then, to go for nothing, and the sweat that I have
+sweated, to say nothing of my horses, while getting the people
+together against Priam and his children? Do as you will, but we
+other gods shall not all of us approve your counsel."
+</para>
+
+<para>
+Jove was angry and answered, "My dear, what harm have Priam and
+his sons done you that you are so hotly bent on sacking the city
+of Ilius? Will nothing do for you but you must within their walls
+and eat Priam raw, with his sons and all the other Trojans to
+boot? Have it your own way then; for I would not have this matter
+become a bone of contention between us. I say further, and lay my
+saying to your heart, if ever I want to sack a city belonging to
+friends of yours, you must not try to stop me; you will have to
+let me do it, for I am giving in to you sorely against my will.
+Of all inhabited cities under the sun and stars of heaven, there
+was none that I so much respected as Ilius with Priam and his
+whole people. Equitable feasts were never wanting about my altar,
+nor the savour of burning fat, which is honour due to ourselves."
+</para>
+
+<para>
+"My own three favourite cities," answered Juno, "are Argos,
+Sparta, and Mycenae. Sack them whenever you may be displeased
+with them. I shall not defend them and I shall not care. Even if
+I did, and tried to stay you, I should take nothing by it, for
+you are much stronger than I am, but I will not have my own work
+wasted. I too am a god and of the same race with yourself. I am
+Saturn's eldest daughter, and am honourable not on this ground
+only, but also because I am your wife, and you are king over the
+gods. Let it be a case, then, of give-and-take between us, and
+the rest of the gods will follow our lead. Tell Minerva to go and
+take part in the fight at once, and let her contrive that the
+Trojans shall be the first to break their oaths and set upon the
+Achaeans."
+</para>
+
+<para>
+The sire of gods and men heeded her words, and said to Minerva,
+"Go at once into the Trojan and Achaean hosts, and contrive that
+the Trojans shall be the first to break their oaths and set upon
+the Achaeans."
+</para>
+
+<para>
+This was what Minerva was already eager to do, so down she darted
+from the topmost summits of Olympus. She shot through the sky as
+some brilliant meteor which the son of scheming Saturn has sent
+as a sign to mariners or to some great army, and a fiery train of
+light follows in its wake. The Trojans and Achaeans were struck
+with awe as they beheld, and one would turn to his neighbour,
+saying, "Either we shall again have war and din of combat, or
+Jove the lord of battle will now make peace between us."
+</para>
+
+<para>
+Thus did they converse. Then Minerva took the form of Laodocus,
+son of Antenor, and went through the ranks of the Trojans to find
+Pandarus, the redoubtable son of Lycaon. She found him standing
+among the stalwart heroes who had followed him from the banks of
+the Aesopus, so she went close up to him and said, "Brave son of
+Lycaon, will you do as I tell you? If you dare send an arrow at
+Menelaus you will win honour and thanks from all the Trojans, and
+especially from prince Alexandrus--he would be the first to
+requite you very handsomely if he could see Menelaus mount his
+funeral pyre, slain by an arrow from your hand. Take your home
+aim then, and pray to Lycian Apollo, the famous archer; vow that
+when you get home to your strong city of Zelea you will offer a
+hecatomb of firstling lambs in his honour."
+</para>
+
+<para>
+His fool's heart was persuaded, and he took his bow from its
+case. This bow was made from the horns of a wild ibex which he
+had killed as it was bounding from a rock; he had stalked it, and
+it had fallen as the arrow struck it to the heart. Its horns were
+sixteen palms long, and a worker in horn had made them into a
+bow, smoothing them well down, and giving them tips of gold. When
+Pandarus had strung his bow he laid it carefully on the ground,
+and his brave followers held their shields before him lest the
+Achaeans should set upon him before he had shot Menelaus. Then he
+opened the lid of his quiver and took out a winged arrow that had
+not yet been shot, fraught with the pangs of death. He laid the
+arrow on the string and prayed to Lycian Apollo, the famous
+archer, vowing that when he got home to his strong city of Zelea
+he would offer a hecatomb of firstling lambs in his honour. He
+laid the notch of the arrow on the oxhide bowstring, and drew
+both notch and string to his breast till the arrow-head was near
+the bow; then when the bow was arched into a half-circle he let
+fly, and the bow twanged, and the string sang as the arrow flew
+gladly on over the heads of the throng.
+</para>
+
+<para>
+But the blessed gods did not forget thee, O Menelaus, and Jove's
+daughter, driver of the spoil, was the first to stand before thee
+and ward off the piercing arrow. She turned it from his skin as a
+mother whisks a fly from off her child when it is sleeping
+sweetly; she guided it to the part where the golden buckles of
+the belt that passed over his double cuirass were fastened, so
+the arrow struck the belt that went tightly round him. It went
+right through this and through the cuirass of cunning
+workmanship; it also pierced the belt beneath it, which he wore
+next his skin to keep out darts or arrows; it was this that
+served him in the best stead, nevertheless the arrow went through
+it and grazed the top of the skin, so that blood began flowing
+from the wound.
+</para>
+
+<para>
+As when some woman of Meonia or Caria strains purple dye on to a
+piece of ivory that is to be the cheek-piece of a horse, and is
+to be laid up in a treasure house--many a knight is fain to bear
+it, but the king keeps it as an ornament of which both horse and
+driver may be proud--even so, O Menelaus, were your shapely
+thighs and your legs down to your fair ancles stained with blood.
+</para>
+
+<para>
+When King Agamemnon saw the blood flowing from the wound he was
+afraid, and so was brave Menelaus himself till he saw that the
+barbs of the arrow and the thread that bound the arrow-head to
+the shaft were still outside the wound. Then he took heart, but
+Agamemnon heaved a deep sigh as he held Menelaus's hand in his
+own, and his comrades made moan in concert. "Dear brother," he
+cried, "I have been the death of you in pledging this covenant
+and letting you come forward as our champion. The Trojans have
+trampled on their oaths and have wounded you; nevertheless the
+oath, the blood of lambs, the drink-offerings and the right hands
+of fellowship in which we have put our trust shall not be vain.
+If he that rules Olympus fulfil it not here and now, he will yet
+fulfil it hereafter, and they shall pay dearly with their lives
+and with their wives and children. The day will surely come when
+mighty Ilius shall be laid low, with Priam and Priam's people,
+when the son of Saturn from his high throne shall overshadow them
+with his awful aegis in punishment of their present treachery.
+This shall surely be; but how, Menelaus, shall I mourn you, if it
+be your lot now to die? I should return to Argos as a by-word,
+for the Achaeans will at once go home. We shall leave Priam and
+the Trojans the glory of still keeping Helen, and the earth will
+rot your bones as you lie here at Troy with your purpose not
+fulfilled. Then shall some braggart Trojan leap upon your tomb
+and say, 'Ever thus may Agamemnon wreak his vengeance; he brought
+his army in vain; he is gone home to his own land with empty
+ships, and has left Menelaus behind him.' Thus will one of them
+say, and may the earth then swallow me."
+</para>
+
+<para>
+But Menelaus reassured him and said, "Take heart, and do not
+alarm the people; the arrow has not struck me in a mortal part,
+for my outer belt of burnished metal first stayed it, and under
+this my cuirass and the belt of mail which the bronze-smiths made
+me."
+</para>
+
+<para>
+And Agamemnon answered, "I trust, dear Menelaus, that it may be
+even so, but the surgeon shall examine your wound and lay herbs
+upon it to relieve your pain."
+</para>
+
+<para>
+He then said to Talthybius, "Talthybius, tell Machaon, son to the
+great physician, Aesculapius, to come and see Menelaus
+immediately. Some Trojan or Lycian archer has wounded him with an
+arrow to our dismay, and to his own great glory."
+</para>
+
+<para>
+Talthybius did as he was told, and went about the host trying to
+find Machaon. Presently he found standing amid the brave warriors
+who had followed him from Tricca; thereon he went up to him and
+said, "Son of Aesculapius, King Agamemnon says you are to come
+and see Menelaus immediately. Some Trojan or Lycian archer has
+wounded him with an arrow to our dismay and to his own great
+glory."
+</para>
+
+<para>
+Thus did he speak, and Machaon was moved to go. They passed
+through the spreading host of the Achaeans and went on till they
+came to the place where Menelaus had been wounded and was lying
+with the chieftains gathered in a circle round him. Machaon
+passed into the middle of the ring and at once drew the arrow
+from the belt, bending its barbs back through the force with
+which he pulled it out. He undid the burnished belt, and beneath
+this the cuirass and the belt of mail which the bronze-smiths had
+made; then, when he had seen the wound, he wiped away the blood
+and applied some soothing drugs which Chiron had given to
+Aesculapius out of the good will he bore him.
+</para>
+
+<para>
+While they were thus busy about Menelaus, the Trojans came
+forward against them, for they had put on their armour, and now
+renewed the fight.
+</para>
+
+<para>
+You would not have then found Agamemnon asleep nor cowardly and
+unwilling to fight, but eager rather for the fray. He left his
+chariot rich with bronze and his panting steeds in charge of
+Eurymedon, son of Ptolemaeus the son of Peiraeus, and bade him
+hold them in readiness against the time his limbs should weary of
+going about and giving orders to so many, for he went among the
+ranks on foot. When he saw men hasting to the front he stood by
+them and cheered them on. "Argives," said he, "slacken not one
+whit in your onset; father Jove will be no helper of liars; the
+Trojans have been the first to break their oaths and to attack
+us; therefore they shall be devoured of vultures; we shall take
+their city and carry off their wives and children in our ships."
+</para>
+
+<para>
+But he angrily rebuked those whom he saw shirking and disinclined
+to fight. "Argives," he cried, "cowardly miserable creatures,
+have you no shame to stand here like frightened fawns who, when
+they can no longer scud over the plain, huddle together, but show
+no fight? You are as dazed and spiritless as deer. Would you wait
+till the Trojans reach the sterns of our ships as they lie on the
+shore, to see whether the son of Saturn will hold his hand over
+you to protect you?"
+</para>
+
+<para>
+Thus did he go about giving his orders among the ranks. Passing
+through the crowd, he came presently on the Cretans, arming round
+Idomeneus, who was at their head, fierce as a wild boar, while
+Meriones was bringing up the battalions that were in the rear.
+Agamemnon was glad when he saw him, and spoke him fairly.
+"Idomeneus," said he, "I treat you with greater distinction than
+I do any others of the Achaeans, whether in war or in other
+things, or at table. When the princes are mixing my choicest
+wines in the mixing-bowls, they have each of them a fixed
+allowance, but your cup is kept always full like my own, that you
+may drink whenever you are minded. Go, therefore, into battle,
+and show yourself the man you have been always proud to be."
+</para>
+
+<para>
+Idomeneus answered, "I will be a trusty comrade, as I promised
+you from the first I would be. Urge on the other Achaeans, that
+we may join battle at once, for the Trojans have trampled upon
+their covenants. Death and destruction shall be theirs, seeing
+they have been the first to break their oaths and to attack us."
+</para>
+
+<para>
+The son of Atreus went on, glad at heart, till he came upon the
+two Ajaxes arming themselves amid a host of foot-soldiers. As
+when a goat-herd from some high post watches a storm drive over
+the deep before the west wind--black as pitch is the offing and a
+mighty whirlwind draws towards him, so that he is afraid and
+drives his flock into a cave--even thus did the ranks of stalwart
+youths move in a dark mass to battle under the Ajaxes, horrid
+with shield and spear. Glad was King Agamemnon when he saw them.
+"No need," he cried, "to give orders to such leaders of the
+Argives as you are, for of your own selves you spur your men on
+to fight with might and main. Would, by father Jove, Minerva, and
+Apollo that all were so minded as you are, for the city of Priam
+would then soon fall beneath our hands, and we should sack it."
+</para>
+
+<para>
+With this he left them and went onward to Nestor, the facile
+speaker of the Pylians, who was marshalling his men and urging
+them on, in company with Pelagon, Alastor, Chromius, Haemon, and
+Bias shepherd of his people. He placed his knights with their
+chariots and horses in the front rank, while the foot-soldiers,
+brave men and many, whom he could trust, were in the rear. The
+cowards he drove into the middle, that they might fight whether
+they would or no. He gave his orders to the knights first,
+bidding them hold their horses well in hand, so as to avoid
+confusion. "Let no man," he said, "relying on his strength or
+horsemanship, get before the others and engage singly with the
+Trojans, nor yet let him lag behind or you will weaken your
+attack; but let each when he meets an enemy's chariot throw his
+spear from his own; this be much the best; this is how the men of
+old took towns and strongholds; in this wise were they minded."
+</para>
+
+<para>
+Thus did the old man charge them, for he had been in many a
+fight, and King Agamemnon was glad. "I wish," he said to him,
+"that your limbs were as supple and your strength as sure as your
+judgment is; but age, the common enemy of mankind, has laid his
+hand upon you; would that it had fallen upon some other, and that
+you were still young."
+</para>
+
+<para>
+And Nestor, knight of Gerene, answered, "Son of Atreus, I too
+would gladly be the man I was when I slew mighty Ereuthalion; but
+the gods will not give us everything at one and the same time. I
+was then young, and now I am old; still I can go with my knights
+and give them that counsel which old men have a right to give.
+The wielding of the spear I leave to those who are younger and
+stronger than myself."
+</para>
+
+<para>
+Agamemnon went his way rejoicing, and presently found Menestheus,
+son of Peteos, tarrying in his place, and with him were the
+Athenians loud of tongue in battle. Near him also tarried cunning
+Ulysses, with his sturdy Cephallenians round him; they had not
+yet heard the battle-cry, for the ranks of Trojans and Achaeans
+had only just begun to move, so they were standing still, waiting
+for some other columns of the Achaeans to attack the Trojans and
+begin the fighting. When he saw this Agamemnon rebuked them and
+said, "Son of Peteos, and you other, steeped in cunning, heart of
+guile, why stand you here cowering and waiting on others? You two
+should be of all men foremost when there is hard fighting to be
+done, for you are ever foremost to accept my invitation when we
+councillors of the Achaeans are holding feast. You are glad
+enough then to take your fill of roast meats and to drink wine as
+long as you please, whereas now you would not care though you saw
+ten columns of Achaeans engage the enemy in front of you."
+</para>
+
+<para>
+Ulysses glared at him and answered, "Son of Atreus, what are you
+talking about? How can you say that we are slack? When the
+Achaeans are in full fight with the Trojans, you shall see, if
+you care to do so, that the father of Telemachus will join battle
+with the foremost of them. You are talking idly."
+</para>
+
+<para>
+When Agamemnon saw that Ulysses was angry, he smiled pleasantly
+at him and withdrew his words. "Ulysses," said he, "noble son of
+Laertes, excellent in all good counsel, I have neither fault to
+find nor orders to give you, for I know your heart is right, and
+that you and I are of a mind. Enough; I will make you amends for
+what I have said, and if any ill has now been spoken may the gods
+bring it to nothing."
+</para>
+
+<para>
+He then left them and went on to others. Presently he saw the son
+of Tydeus, noble Diomed, standing by his chariot and horses, with
+Sthenelus the son of Capaneus beside him; whereon he began to
+upbraid him. "Son of Tydeus," he said, "why stand you cowering
+here upon the brink of battle? Tydeus did not shrink thus, but
+was ever ahead of his men when leading them on against the foe--
+so, at least, say they that saw him in battle, for I never set
+eyes upon him myself. They say that there was no man like him. He
+came once to Mycenae, not as an enemy but as a guest, in company
+with Polynices to recruit his forces, for they were levying war
+against the strong city of Thebes, and prayed our people for a
+body of picked men to help them. The men of Mycenae were willing
+to let them have one, but Jove dissuaded them by showing them
+unfavourable omens. Tydeus, therefore, and Polynices went their
+way. When they had got as far the deep-meadowed and rush-grown
+banks of the Aesopus, the Achaeans sent Tydeus as their envoy,
+and he found the Cadmeans gathered in great numbers to a banquet
+in the house of Eteocles. Stranger though he was, he knew no fear
+on finding himself single-handed among so many, but challenged
+them to contests of all kinds, and in each one of them was at
+once victorious, so mightily did Minerva help him. The Cadmeans
+were incensed at his success, and set a force of fifty youths
+with two captains--the godlike hero Maeon, son of Haemon, and
+Polyphontes, son of Autophonus--at their head, to lie in wait for
+him on his return journey; but Tydeus slew every man of them,
+save only Maeon, whom he let go in obedience to heaven's omens.
+Such was Tydeus of Aetolia. His son can talk more glibly, but he
+cannot fight as his father did."
+</para>
+
+<para>
+Diomed made no answer, for he was shamed by the rebuke of
+Agamemnon; but the son of Capaneus took up his words and said,
+"Son of Atreus, tell no lies, for you can speak truth if you
+will. We boast ourselves as even better men than our fathers; we
+took seven-gated Thebes, though the wall was stronger and our men
+were fewer in number, for we trusted in the omens of the gods and
+in the help of Jove, whereas they perished through their own
+sheer folly; hold not, then, our fathers in like honour with us."
+</para>
+
+<para>
+Diomed looked sternly at him and said, "Hold your peace, my
+friend, as I bid you. It is not amiss that Agamemnon should urge
+the Achaeans forward, for the glory will be his if we take the
+city, and his the shame if we are vanquished. Therefore let us
+acquit ourselves with valour."
+</para>
+
+<para>
+As he spoke he sprang from his chariot, and his armour rang so
+fiercely about his body that even a brave man might well have
+been scared to hear it.
+</para>
+
+<para>
+As when some mighty wave that thunders on the beach when the west
+wind has lashed it into fury--it has reared its head afar and now
+comes crashing down on the shore; it bows its arching crest high
+over the jagged rocks and spews its salt foam in all
+directions--even so did the serried phalanxes of the Danaans
+march steadfastly to battle. The chiefs gave orders each to his
+own people, but the men said never a word; no man would think it,
+for huge as the host was, it seemed as though there was not a
+tongue among them, so silent were they in their obedience; and as
+they marched the armour about their bodies glistened in the sun.
+But the clamour of the Trojan ranks was as that of many thousand
+ewes that stand waiting to be milked in the yards of some rich
+flockmaster, and bleat incessantly in answer to the bleating of
+their lambs; for they had not one speech nor language, but their
+tongues were diverse, and they came from many different places.
+These were inspired of Mars, but the others by Minerva--and with
+them came Panic, Rout, and Strife whose fury never tires, sister
+and friend of murderous Mars, who, from being at first but low in
+stature, grows till she uprears her head to heaven, though her
+feet are still on earth. She it was that went about among them
+and flung down discord to the waxing of sorrow with even hand
+between them.
+</para>
+
+<para>
+When they were got together in one place shield clashed with
+shield and spear with spear in the rage of battle. The bossed
+shields beat one upon another, and there was a tramp as of a
+great multitude--death-cry and shout of triumph of slain and
+slayers, and the earth ran red with blood. As torrents swollen
+with rain course madly down their deep channels till the angry
+floods meet in some gorge, and the shepherd on the hillside hears
+their roaring from afar--even such was the toil and uproar of the
+hosts as they joined in battle.
+</para>
+
+<para>
+First Antilochus slew an armed warrior of the Trojans, Echepolus,
+son of Thalysius, fighting in the foremost ranks. He struck at
+the projecting part of his helmet and drove the spear into his
+brow; the point of bronze pierced the bone, and darkness veiled
+his eyes; headlong as a tower he fell amid the press of the
+fight, and as he dropped King Elephenor, son of Chalcodon and
+captain of the proud Abantes began dragging him out of reach of
+the darts that were falling around him, in haste to strip him of
+his armour. But his purpose was not for long; Agenor saw him
+haling the body away, and smote him in the side with his
+bronze-shod spear--for as he stooped his side was left
+unprotected by his shield--and thus he perished. Then the fight
+between Trojans and Achaeans grew furious over his body, and they
+flew upon each other like wolves, man and man crushing one upon
+the other.
+</para>
+
+<para>
+Forthwith Ajax, son of Telamon, slew the fair youth Simoeisius,
+son of Anthemion, whom his mother bore by the banks of the
+Simois, as she was coming down from Mt. Ida, where she had been
+with her parents to see their flocks. Therefore he was named
+Simoeisius, but he did not live to pay his parents for his
+rearing, for he was cut off untimely by the spear of mighty Ajax,
+who struck him in the breast by the right nipple as he was coming
+on among the foremost fighters; the spear went right through his
+shoulder, and he fell as a poplar that has grown straight and
+tall in a meadow by some mere, and its top is thick with
+branches. Then the wheelwright lays his axe to its roots that he
+may fashion a felloe for the wheel of some goodly chariot, and it
+lies seasoning by the waterside. In such wise did Ajax fell to
+earth Simoeisius, son of Anthemion. Thereon Antiphus of the
+gleaming corslet, son of Priam, hurled a spear at Ajax from amid
+the crowd and missed him, but he hit Leucus, the brave comrade of
+Ulysses, in the groin, as he was dragging the body of Simoeisius
+over to the other side; so he fell upon the body and loosed his
+hold upon it. Ulysses was furious when he saw Leucus slain, and
+strode in full armour through the front ranks till he was quite
+close; then he glared round about him and took aim, and the
+Trojans fell back as he did so. His dart was not sped in vain,
+for it struck Democoon, the bastard son of Priam, who had come to
+him from Abydos, where he had charge of his father's mares.
+Ulysses, infuriated by the death of his comrade, hit him with his
+spear on one temple, and the bronze point came through on the
+other side of his forehead. Thereon darkness veiled his eyes, and
+his armour rang rattling round him as he fell heavily to the
+ground. Hector, and they that were in front, then gave round
+while the Argives raised a shout and drew off the dead, pressing
+further forward as they did so. But Apollo looked down from
+Pergamus and called aloud to the Trojans, for he was displeased.
+"Trojans," he cried, "rush on the foe, and do not let yourselves
+be thus beaten by the Argives. Their skins are not stone nor iron
+that when hit them you do them no harm. Moreover, Achilles, the
+son of lovely Thetis, is not fighting, but is nursing his anger
+at the ships."
+</para>
+
+<para>
+Thus spoke the mighty god, crying to them from the city, while
+Jove's redoubtable daughter, the Trito-born, went about among the
+host of the Achaeans, and urged them forward whenever she beheld
+them slackening.
+</para>
+
+<para>
+Then fate fell upon Diores, son of Amarynceus, for he was struck
+by a jagged stone near the ancle of his right leg. He that hurled
+it was Peirous, son of Imbrasus, captain of the Thracians, who
+had come from Aenus; the bones and both the tendons were crushed
+by the pitiless stone. He fell to the ground on his back, and in
+his death throes stretched out his hands towards his comrades.
+But Peirous, who had wounded him, sprang on him and thrust a
+spear into his belly, so that his bowels came gushing out upon
+the ground, and darkness veiled his eyes. As he was leaving the
+body, Thoas of Aetolia struck him in the chest near the nipple,
+and the point fixed itself in his lungs. Thoas came close up to
+him, pulled the spear out of his chest, and then drawing his
+sword, smote him in the middle of the belly so that he died; but
+he did not strip him of his armour, for his Thracian comrades,
+men who wear their hair in a tuft at the top of their heads,
+stood round the body and kept him off with their long spears for
+all his great stature and valour; so he was driven back. Thus the
+two corpses lay stretched on earth near to one another, the one
+captain of the Thracians and the other of the Epeans; and many
+another fell round them.
+</para>
+
+<para>
+And now no man would have made light of the fighting if he could
+have gone about among it scatheless and unwounded, with Minerva
+leading him by the hand, and protecting him from the storm of
+spears and arrows. For many Trojans and Achaeans on that day lay
+stretched side by side face downwards upon the earth.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK V</title>
+
+<para>
+Then Pallas Minerva put valour into the heart of Diomed, son of
+Tydeus, that he might excel all the other Argives, and cover
+himself with glory. She made a stream of fire flare from his
+shield and helmet like the star that shines most brilliantly in
+summer after its bath in the waters of Oceanus--even such a fire
+did she kindle upon his head and shoulders as she bade him speed
+into the thickest hurly-burly of the fight.
+</para>
+
+<para>
+Now there was a certain rich and honourable man among the
+Trojans, priest of Vulcan, and his name was Dares. He had two
+sons, Phegeus and Idaeus, both of them skilled in all the arts of
+war. These two came forward from the main body of Trojans, and
+set upon Diomed, he being on foot, while they fought from their
+chariot. When they were close up to one another, Phegeus took aim
+first, but his spear went over Diomed's left shoulder without
+hitting him. Diomed then threw, and his spear sped not in vain,
+for it hit Phegeus on the breast near the nipple, and he fell
+from his chariot. Idaeus did not dare to bestride his brother's
+body, but sprang from the chariot and took to flight, or he would
+have shared his brother's fate; whereon Vulcan saved him by
+wrapping him in a cloud of darkness, that his old father might
+not be utterly overwhelmed with grief; but the son of Tydeus
+drove off with the horses, and bade his followers take them to
+the ships. The Trojans were scared when they saw the two sons of
+Dares, one of them in fright and the other lying dead by his
+chariot. Minerva, therefore, took Mars by the hand and said,
+"Mars, Mars, bane of men, bloodstained stormer of cities, may we
+not now leave the Trojans and Achaeans to fight it out, and see
+to which of the two Jove will vouchsafe the victory? Let us go
+away, and thus avoid his anger."
+</para>
+
+<para>
+So saying, she drew Mars out of the battle, and set him down upon
+the steep banks of the Scamander. Upon this the Danaans drove the
+Trojans back, and each one of their chieftains killed his man.
+First King Agamemnon flung mighty Odius, captain of the Halizoni,
+from his chariot. The spear of Agamemnon caught him on the broad
+of his back, just as he was turning in flight; it struck him
+between the shoulders and went right through his chest, and his
+armour rang rattling round him as he fell heavily to the ground.
+</para>
+
+<para>
+Then Idomeneus killed Phaesus, son of Borus the Meonian, who had
+come from Varne. Mighty Idomeneus speared him on the right
+shoulder as he was mounting his chariot, and the darkness of
+death enshrouded him as he fell heavily from the car.
+</para>
+
+<para>
+The squires of Idomeneus spoiled him of his armour, while
+Menelaus, son of Atreus, killed Scamandrius the son of Strophius,
+a mighty huntsman and keen lover of the chase. Diana herself had
+taught him how to kill every kind of wild creature that is bred
+in mountain forests, but neither she nor his famed skill in
+archery could now save him, for the spear of Menelaus struck him
+in the back as he was flying; it struck him between the shoulders
+and went right through his chest, so that he fell headlong and
+his armour rang rattling round him.
+</para>
+
+<para>
+Meriones then killed Phereclus the son of Tecton, who was the son
+of Hermon, a man whose hand was skilled in all manner of cunning
+workmanship, for Pallas Minerva had dearly loved him. He it was
+that made the ships for Alexandrus, which were the beginning of
+all mischief, and brought evil alike both on the Trojans and on
+Alexandrus himself; for he heeded not the decrees of heaven.
+Meriones overtook him as he was flying, and struck him on the
+right buttock. The point of the spear went through the bone into
+the bladder, and death came upon him as he cried aloud and fell
+forward on his knees.
+</para>
+
+<para>
+Meges, moreover, slew Pedaeus, son of Antenor, who, though he was
+a bastard, had been brought up by Theano as one of her own
+children, for the love she bore her husband. The son of Phyleus
+got close up to him and drove a spear into the nape of his neck:
+it went under his tongue all among his teeth, so he bit the cold
+bronze, and fell dead in the dust.
+</para>
+
+<para>
+And Eurypylus, son of Euaemon, killed Hypsenor, the son of noble
+Dolopion, who had been made priest of the river Scamander, and
+was honoured among the people as though he were a god. Eurypylus
+gave him chase as he was flying before him, smote him with his
+sword upon the arm, and lopped his strong hand from off it. The
+bloody hand fell to the ground, and the shades of death, with
+fate that no man can withstand, came over his eyes.
+</para>
+
+<para>
+Thus furiously did the battle rage between them. As for the son
+of Tydeus, you could not say whether he was more among the
+Achaeans or the Trojans. He rushed across the plain like a winter
+torrent that has burst its barrier in full flood; no dykes, no
+walls of fruitful vineyards can embank it when it is swollen with
+rain from heaven, but in a moment it comes tearing onward, and
+lays many a field waste that many a strong man's hand has
+reclaimed--even so were the dense phalanxes of the Trojans driven
+in rout by the son of Tydeus, and many though they were, they
+dared not abide his onslaught.
+</para>
+
+<para>
+Now when the son of Lycaon saw him scouring the plain and driving
+the Trojans pell-mell before him, he aimed an arrow and hit the
+front part of his cuirass near the shoulder: the arrow went right
+through the metal and pierced the flesh, so that the cuirass was
+covered with blood. On this the son of Lycaon shouted in triumph,
+"Knights Trojans, come on; the bravest of the Achaeans is
+wounded, and he will not hold out much longer if King Apollo was
+indeed with me when I sped from Lycia hither."
+</para>
+
+<para>
+Thus did he vaunt; but his arrow had not killed Diomed, who
+withdrew and made for the chariot and horses of Sthenelus, the
+son of Capaneus. "Dear son of Capaneus," said he, "come down from
+your chariot, and draw the arrow out of my shoulder."
+</para>
+
+<para>
+Sthenelus sprang from his chariot, and drew the arrow from the
+wound, whereon the blood came spouting out through the hole that
+had been made in his shirt. Then Diomed prayed, saying, "Hear me,
+daughter of aegis-bearing Jove, unweariable, if ever you loved my
+father well and stood by him in the thick of a fight, do the like
+now by me; grant me to come within a spear's throw of that man
+and kill him. He has been too quick for me and has wounded me;
+and now he is boasting that I shall not see the light of the sun
+much longer."
+</para>
+
+<para>
+Thus he prayed, and Pallas Minerva heard him; she made his limbs
+supple and quickened his hands and his feet. Then she went up
+close to him and said, "Fear not, Diomed, to do battle with the
+Trojans, for I have set in your heart the spirit of your knightly
+father Tydeus. Moreover, I have withdrawn the veil from your
+eyes, that you know gods and men apart. If, then, any other god
+comes here and offers you battle, do not fight him; but should
+Jove's daughter Venus come, strike her with your spear and wound
+her."
+</para>
+
+<para>
+When she had said this Minerva went away, and the son of Tydeus
+again took his place among the foremost fighters, three times
+more fierce even than he had been before. He was like a lion that
+some mountain shepherd has wounded, but not killed, as he is
+springing over the wall of a sheep-yard to attack the sheep. The
+shepherd has roused the brute to fury but cannot defend his
+flock, so he takes shelter under cover of the buildings, while
+the sheep, panic-stricken on being deserted, are smothered in
+heaps one on top of the other, and the angry lion leaps out over
+the sheep-yard wall. Even thus did Diomed go furiously about
+among the Trojans.
+</para>
+
+<para>
+He killed Astynous, and Hypeiron shepherd of his people, the one
+with a thrust of his spear, which struck him above the nipple,
+the other with a sword-cut on the collar-bone, that severed his
+shoulder from his neck and back. He let both of them lie, and
+went in pursuit of Abas and Polyidus, sons of the old reader of
+dreams Eurydamas: they never came back for him to read them any
+more dreams, for mighty Diomed made an end of them. He then gave
+chase to Xanthus and Thoon, the two sons of Phaenops, both of
+them very dear to him, for he was now worn out with age, and
+begat no more sons to inherit his possessions. But Diomed took
+both their lives and left their father sorrowing bitterly, for he
+nevermore saw them come home from battle alive, and his kinsmen
+divided his wealth among themselves.
+</para>
+
+<para>
+Then he came upon two sons of Priam, Echemmon and Chromius, as
+they were both in one chariot. He sprang upon them as a lion
+fastens on the neck of some cow or heifer when the herd is
+feeding in a coppice. For all their vain struggles he flung them
+both from their chariot and stripped the armour from their
+bodies. Then he gave their horses to his comrades to take them
+back to the ships.
+</para>
+
+<para>
+When Aeneas saw him thus making havoc among the ranks, he went
+through the fight amid the rain of spears to see if he could find
+Pandarus. When he had found the brave son of Lycaon he said,
+"Pandarus, where is now your bow, your winged arrows, and your
+renown as an archer, in respect of which no man here can rival
+you nor is there any in Lycia that can beat you? Lift then your
+hands to Jove and send an arrow at this fellow who is going so
+masterfully about, and has done such deadly work among the
+Trojans. He has killed many a brave man--unless indeed he is some
+god who is angry with the Trojans about their sacrifices, and and
+has set his hand against them in his displeasure."
+</para>
+
+<para>
+And the son of Lycaon answered, "Aeneas, I take him for none
+other than the son of Tydeus. I know him by his shield, the visor
+of his helmet, and by his horses. It is possible that he may be a
+god, but if he is the man I say he is, he is not making all this
+havoc without heaven's help, but has some god by his side who is
+shrouded in a cloud of darkness, and who turned my arrow aside
+when it had hit him. I have taken aim at him already and hit him
+on the right shoulder; my arrow went through the breastpiece of
+his cuirass; and I made sure I should send him hurrying to the
+world below, but it seems that I have not killed him. There must
+be a god who is angry with me. Moreover I have neither horse nor
+chariot. In my father's stables there are eleven excellent
+chariots, fresh from the builder, quite new, with cloths spread
+over them; and by each of them there stand a pair of horses,
+champing barley and rye; my old father Lycaon urged me again and
+again when I was at home and on the point of starting, to take
+chariots and horses with me that I might lead the Trojans in
+battle, but I would not listen to him; it would have been much
+better if I had done so, but I was thinking about the horses,
+which had been used to eat their fill, and I was afraid that in
+such a great gathering of men they might be ill-fed, so I left
+them at home and came on foot to Ilius armed only with my bow and
+arrows. These it seems, are of no use, for I have already hit two
+chieftains, the sons of Atreus and of Tydeus, and though I drew
+blood surely enough, I have only made them still more furious. I
+did ill to take my bow down from its peg on the day I led my band
+of Trojans to Ilius in Hector's service, and if ever I get home
+again to set eyes on my native place, my wife, and the greatness
+of my house, may some one cut my head off then and there if I do
+not break the bow and set it on a hot fire--such pranks as it
+plays me."
+</para>
+
+<para>
+Aeneas answered, "Say no more. Things will not mend till we two
+go against this man with chariot and horses and bring him to a
+trial of arms. Mount my chariot, and note how cleverly the horses
+of Tros can speed hither and thither over the plain in pursuit or
+flight. If Jove again vouchsafes glory to the son of Tydeus they
+will carry us safely back to the city. Take hold, then, of the
+whip and reins while I stand upon the car to fight, or else do
+you wait this man's onset while I look after the horses."
+</para>
+
+<para>
+"Aeneas," replied the son of Lycaon, "take the reins and drive;
+if we have to fly before the son of Tydeus the horses will go
+better for their own driver. If they miss the sound of your voice
+when they expect it they may be frightened, and refuse to take us
+out of the fight. The son of Tydeus will then kill both of us and
+take the horses. Therefore drive them yourself and I will be
+ready for him with my spear."
+</para>
+
+<para>
+They then mounted the chariot and drove full-speed towards the
+son of Tydeus. Sthenelus, son of Capaneus, saw them coming and
+said to Diomed, "Diomed, son of Tydeus, man after my own heart, I
+see two heroes speeding towards you, both of them men of might
+the one a skilful archer, Pandarus son of Lycaon, the other,
+Aeneas, whose sire is Anchises, while his mother is Venus. Mount
+the chariot and let us retreat. Do not, I pray you, press so
+furiously forward, or you may get killed."
+</para>
+
+<para>
+Diomed looked angrily at him and answered: "Talk not of flight,
+for I shall not listen to you: I am of a race that knows neither
+flight nor fear, and my limbs are as yet unwearied. I am in no
+mind to mount, but will go against them even as I am; Pallas
+Minerva bids me be afraid of no man, and even though one of them
+escape, their steeds shall not take both back again. I say
+further, and lay my saying to your heart--if Minerva sees fit to
+vouchsafe me the glory of killing both, stay your horses here and
+make the reins fast to the rim of the chariot; then be sure you
+spring Aeneas' horses and drive them from the Trojan to the
+Achaean ranks. They are of the stock that great Jove gave to Tros
+in payment for his son Ganymede, and are the finest that live and
+move under the sun. King Anchises stole the blood by putting his
+mares to them without Laomedon's knowledge, and they bore him six
+foals. Four are still in his stables, but he gave the other two
+to Aeneas. We shall win great glory if we can take them."
+</para>
+
+<para>
+Thus did they converse, but the other two had now driven close up
+to them, and the son of Lycaon spoke first. "Great and mighty
+son," said he, "of noble Tydeus, my arrow failed to lay you low,
+so I will now try with my spear."
+</para>
+
+<para>
+He poised his spear as he spoke and hurled it from him. It struck
+the shield of the son of Tydeus; the bronze point pierced it and
+passed on till it reached the breastplate. Thereon the son of
+Lycaon shouted out and said, "You are hit clean through the
+belly; you will not stand out for long, and the glory of the
+fight is mine."
+</para>
+
+<para>
+But Diomed all undismayed made answer, "You have missed, not hit,
+and before you two see the end of this matter one or other of you
+shall glut tough-shielded Mars with his blood."
+</para>
+
+<para>
+With this he hurled his spear, and Minerva guided it on to
+Pandarus's nose near the eye. It went crashing in among his white
+teeth; the bronze point cut through the root of his tongue,
+coming out under his chin, and his glistening armour rang
+rattling round him as he fell heavily to the ground. The horses
+started aside for fear, and he was reft of life and strength.
+</para>
+
+<para>
+Aeneas sprang from his chariot armed with shield and spear,
+fearing lest the Achaeans should carry off the body. He bestrode
+it as a lion in the pride of strength, with shield and spear
+before him and a cry of battle on his lips resolute to kill the
+first that should dare face him. But the son of Tydeus caught up
+a mighty stone, so huge and great that as men now are it would
+take two to lift it; nevertheless he bore it aloft with ease
+unaided, and with this he struck Aeneas on the groin where the
+hip turns in the joint that is called the "cup-bone." The stone
+crushed this joint, and broke both the sinews, while its jagged
+edges tore away all the flesh. The hero fell on his knees, and
+propped himself with his hand resting on the ground till the
+darkness of night fell upon his eyes. And now Aeneas, king of
+men, would have perished then and there, had not his mother,
+Jove's daughter Venus, who had conceived him by Anchises when he
+was herding cattle, been quick to mark, and thrown her two white
+arms about the body of her dear son. She protected him by
+covering him with a fold of her own fair garment, lest some
+Danaan should drive a spear into his breast and kill him.
+</para>
+
+<para>
+Thus, then, did she bear her dear son out of the fight. But the
+son of Capaneus was not unmindful of the orders that Diomed had
+given him. He made his own horses fast, away from the
+hurly-burly, by binding the reins to the rim of the chariot. Then
+he sprang upon Aeneas's horses and drove them from the Trojan to
+the Achaean ranks. When he had so done he gave them over to his
+chosen comrade Deipylus, whom he valued above all others as the
+one who was most like-minded with himself, to take them on to the
+ships. He then remounted his own chariot, seized the reins, and
+drove with all speed in search of the son of Tydeus.
+</para>
+
+<para>
+Now the son of Tydeus was in pursuit of the Cyprian goddess,
+spear in hand, for he knew her to be feeble and not one of those
+goddesses that can lord it among men in battle like Minerva or
+Enyo the waster of cities, and when at last after a long chase he
+caught her up, he flew at her and thrust his spear into the flesh
+of her delicate hand. The point tore through the ambrosial robe
+which the Graces had woven for her, and pierced the skin between
+her wrist and the palm of her hand, so that the immortal blood,
+or ichor, that flows in the veins of the blessed gods, came
+pouring from the wound; for the gods do not eat bread nor drink
+wine, hence they have no blood such as ours, and are immortal.
+Venus screamed aloud, and let her son fall, but Phoebus Apollo
+caught him in his arms, and hid him in a cloud of darkness, lest
+some Danaan should drive a spear into his breast and kill him;
+and Diomed shouted out as he left her, "Daughter of Jove, leave
+war and battle alone, can you not be contented with beguiling
+silly women? If you meddle with fighting you will get what will
+make you shudder at the very name of war."
+</para>
+
+<para>
+The goddess went dazed and discomfited away, and Iris, fleet as
+the wind, drew her from the throng, in pain and with her fair
+skin all besmirched. She found fierce Mars waiting on the left of
+the battle, with his spear and his two fleet steeds resting on a
+cloud; whereon she fell on her knees before her brother and
+implored him to let her have his horses. "Dear brother," she
+cried, "save me, and give me your horses to take me to Olympus
+where the gods dwell. I am badly wounded by a mortal, the son of
+Tydeus, who would now fight even with father Jove."
+</para>
+
+<para>
+Thus she spoke, and Mars gave her his gold-bedizened steeds. She
+mounted the chariot sick and sorry at heart, while Iris sat
+beside her and took the reins in her hand. She lashed her horses
+on and they flew forward nothing loth, till in a trice they were
+at high Olympus, where the gods have their dwelling. There she
+stayed them, unloosed them from the chariot, and gave them their
+ambrosial forage; but Venus flung herself on to the lap of her
+mother Dione, who threw her arms about her and caressed her,
+saying, "Which of the heavenly beings has been treating you in
+this way, as though you had been doing something wrong in the
+face of day?"
+</para>
+
+<para>
+And laughter-loving Venus answered, "Proud Diomed, the son of
+Tydeus, wounded me because I was bearing my dear son Aeneas, whom
+I love best of all mankind, out of the fight. The war is no
+longer one between Trojans and Achaeans, for the Danaans have now
+taken to fighting with the immortals."
+</para>
+
+<para>
+"Bear it, my child," replied Dione, "and make the best of it. We
+dwellers in Olympus have to put up with much at the hands of men,
+and we lay much suffering on one another. Mars had to suffer when
+Otus and Ephialtes, children of Aloeus, bound him in cruel bonds,
+so that he lay thirteen months imprisoned in a vessel of bronze.
+Mars would have then perished had not fair Eeriboea, stepmother
+to the sons of Aloeus, told Mercury, who stole him away when he
+was already well-nigh worn out by the severity of his bondage.
+Juno, again, suffered when the mighty son of Amphitryon wounded
+her on the right breast with a three-barbed arrow, and nothing
+could assuage her pain. So, also, did huge Hades, when this same
+man, the son of aegis-bearing Jove, hit him with an arrow even at
+the gates of hell, and hurt him badly. Thereon Hades went to the
+house of Jove on great Olympus, angry and full of pain; and the
+arrow in his brawny shoulder caused him great anguish till Paeeon
+healed him by spreading soothing herbs on the wound, for Hades
+was not of mortal mould. Daring, head-strong, evildoer who recked
+not of his sin in shooting the gods that dwell in Olympus. And
+now Minerva has egged this son of Tydeus on against yourself,
+fool that he is for not reflecting that no man who fights with
+gods will live long or hear his children prattling about his
+knees when he returns from battle. Let, then, the son of Tydeus
+see that he does not have to fight with one who is stronger than
+you are. Then shall his brave wife Aegialeia, daughter of
+Adrestus, rouse her whole house from sleep, wailing for the loss
+of her wedded lord, Diomed the bravest of the Achaeans."
+</para>
+
+<para>
+So saying, she wiped the ichor from the wrist of her daughter
+with both hands, whereon the pain left her, and her hand was
+healed. But Minerva and Juno, who were looking on, began to taunt
+Jove with their mocking talk, and Minerva was first to speak.
+"Father Jove," said she, "do not be angry with me, but I think
+the Cyprian must have been persuading some one of the Achaean
+women to go with the Trojans of whom she is so very fond, and
+while caressing one or other of them she must have torn her
+delicate hand with the gold pin of the woman's brooch."
+</para>
+
+<para>
+The sire of gods and men smiled, and called golden Venus to his
+side. "My child," said he, "it has not been given you to be a
+warrior. Attend, henceforth, to your own delightful matrimonial
+duties, and leave all this fighting to Mars and to Minerva."
+</para>
+
+<para>
+Thus did they converse. But Diomed sprang upon Aeneas, though he
+knew him to be in the very arms of Apollo. Not one whit did he
+fear the mighty god, so set was he on killing Aeneas and
+stripping him of his armour. Thrice did he spring forward with
+might and main to slay him, and thrice did Apollo beat back his
+gleaming shield. When he was coming on for the fourth time, as
+though he were a god, Apollo shouted to him with an awful voice
+and said, "Take heed, son of Tydeus, and draw off; think not to
+match yourself against gods, for men that walk the earth cannot
+hold their own with the immortals."
+</para>
+
+<para>
+The son of Tydeus then gave way for a little space, to avoid the
+anger of the god, while Apollo took Aeneas out of the crowd and
+set him in sacred Pergamus, where his temple stood. There, within
+the mighty sanctuary, Latona and Diana healed him and made him
+glorious to behold, while Apollo of the silver bow fashioned a
+wraith in the likeness of Aeneas, and armed as he was. Round this
+the Trojans and Achaeans hacked at the bucklers about one
+another's breasts, hewing each other's round shields and light
+hide-covered targets. Then Phoebus Apollo said to Mars, "Mars,
+Mars, bane of men, blood-stained stormer of cities, can you not
+go to this man, the son of Tydeus, who would now fight even with
+father Jove, and draw him out of the battle? He first went up to
+the Cyprian and wounded her in the hand near her wrist, and
+afterwards sprang upon me too, as though he were a god."
+</para>
+
+<para>
+He then took his seat on the top of Pergamus, while murderous
+Mars went about among the ranks of the Trojans, cheering them on,
+in the likeness of fleet Acamas chief of the Thracians. "Sons of
+Priam," said he, "how long will you let your people be thus
+slaughtered by the Achaeans? Would you wait till they are at the
+walls of Troy? Aeneas the son of Anchises has fallen, he whom we
+held in as high honour as Hector himself. Help me, then, to
+rescue our brave comrade from the stress of the fight."
+</para>
+
+<para>
+With these words he put heart and soul into them all. Then
+Sarpedon rebuked Hector very sternly. "Hector," said he, "where
+is your prowess now? You used to say that though you had neither
+people nor allies you could hold the town alone with your
+brothers and brothers-in-law. I see not one of them here; they
+cower as hounds before a lion; it is we, your allies, who bear
+the brunt of the battle. I have come from afar, even from Lycia
+and the banks of the river Xanthus, where I have left my wife, my
+infant son, and much wealth to tempt whoever is needy;
+nevertheless, I head my Lycian soldiers and stand my ground
+against any who would fight me though I have nothing here for the
+Achaeans to plunder, while you look on, without even bidding your
+men stand firm in defence of their wives. See that you fall not
+into the hands of your foes as men caught in the meshes of a net,
+and they sack your fair city forthwith. Keep this before your
+mind night and day, and beseech the captains of your allies to
+hold on without flinching, and thus put away their reproaches
+from you."
+</para>
+
+<para>
+So spoke Sarpedon, and Hector smarted under his words. He sprang
+from his chariot clad in his suit of armour, and went about among
+the host brandishing his two spears, exhorting the men to fight
+and raising the terrible cry of battle. Then they rallied and
+again faced the Achaeans, but the Argives stood compact and firm,
+and were not driven back. As the breezes sport with the chaff
+upon some goodly threshing-floor, when men are winnowing--while
+yellow Ceres blows with the wind to sift the chaff from the
+grain, and the chaff-heaps grow whiter and whiter--even so did
+the Achaeans whiten in the dust which the horses' hoofs raised to
+the firmament of heaven, as their drivers turned them back to
+battle, and they bore down with might upon the foe. Fierce Mars,
+to help the Trojans, covered them in a veil of darkness, and went
+about everywhere among them, inasmuch as Phoebus Apollo had told
+him that when he saw Pallas, Minerva leave the fray he was to put
+courage into the hearts of the Trojans--for it was she who was
+helping the Danaans. Then Apollo sent Aeneas forth from his rich
+sanctuary, and filled his heart with valour, whereon he took his
+place among his comrades, who were overjoyed at seeing him alive,
+sound, and of a good courage; but they could not ask him how it
+had all happened, for they were too busy with the turmoil raised
+by Mars and by Strife, who raged insatiably in their midst.
+</para>
+
+<para>
+The two Ajaxes, Ulysses and Diomed, cheered the Danaans on,
+fearless of the fury and onset of the Trojans. They stood as
+still as clouds which the son of Saturn has spread upon the
+mountain tops when there is no air and fierce Boreas sleeps with
+the other boisterous winds whose shrill blasts scatter the clouds
+in all directions--even so did the Danaans stand firm and
+unflinching against the Trojans. The son of Atreus went about
+among them and exhorted them. "My friends," said he, "quit
+yourselves like brave men, and shun dishonour in one another's
+eyes amid the stress of battle. They that shun dishonour more
+often live than get killed, but they that fly save neither life
+nor name."
+</para>
+
+<para>
+As he spoke he hurled his spear and hit one of those who were in
+the front rank, the comrade of Aeneas, Deicoon son of Pergasus,
+whom the Trojans held in no less honour than the sons of Priam,
+for he was ever quick to place himself among the foremost. The
+spear of King Agamemnon struck his shield and went right through
+it, for the shield stayed it not. It drove through his belt into
+the lower part of his belly, and his armour rang rattling round
+him as he fell heavily to the ground.
+</para>
+
+<para>
+Then Aeneas killed two champions of the Danaans, Crethon and
+Orsilochus. Their father was a rich man who lived in the strong
+city of Phere and was descended from the river Alpheus, whose
+broad stream flows through the land of the Pylians. The river
+begat Orsilochus, who ruled over much people and was father to
+Diocles, who in his turn begat twin sons, Crethon and Orsilochus,
+well skilled in all the arts of war. These, when they grew up,
+went to Ilius with the Argive fleet in the cause of Menelaus and
+Agamemnon sons of Atreus, and there they both of them fell. As
+two lions whom their dam has reared in the depths of some
+mountain forest to plunder homesteads and carry off sheep and
+cattle till they get killed by the hand of man, so were these two
+vanquished by Aeneas, and fell like high pine-trees to the
+ground.
+</para>
+
+<para>
+Brave Menelaus pitied them in their fall, and made his way to the
+front, clad in gleaming bronze and brandishing his spear, for
+Mars egged him on to do so with intent that he should be killed
+by Aeneas; but Antilochus the son of Nestor saw him and sprang
+forward, fearing that the king might come to harm and thus bring
+all their labour to nothing; when, therefore Aeneas and Menelaus
+were setting their hands and spears against one another eager to
+do battle, Antilochus placed himself by the side of Menelaus.
+Aeneas, bold though he was, drew back on seeing the two heroes
+side by side in front of him, so they drew the bodies of Crethon
+and Orsilochus to the ranks of the Achaeans and committed the two
+poor fellows into the hands of their comrades. They then turned
+back and fought in the front ranks.
+</para>
+
+<para>
+They killed Pylaemenes peer of Mars, leader of the Paphlagonian
+warriors. Menelaus struck him on the collar-bone as he was
+standing on his chariot, while Antilochus hit his charioteer and
+squire Mydon, the son of Atymnius, who was turning his horses in
+flight. He hit him with a stone upon the elbow, and the reins,
+enriched with white ivory, fell from his hands into the dust.
+Antilochus rushed towards him and struck him on the temples with
+his sword, whereon he fell head first from the chariot to the
+ground. There he stood for a while with his head and shoulders
+buried deep in the dust--for he had fallen on sandy soil till his
+horses kicked him and laid him flat on the ground, as Antilochus
+lashed them and drove them off to the host of the Achaeans.
+</para>
+
+<para>
+But Hector marked them from across the ranks, and with a loud cry
+rushed towards them, followed by the strong battalions of the
+Trojans. Mars and dread Enyo led them on, she fraught with
+ruthless turmoil of battle, while Mars wielded a monstrous spear,
+and went about, now in front of Hector and now behind him.
+</para>
+
+<para>
+Diomed shook with passion as he saw them. As a man crossing a
+wide plain is dismayed to find himself on the brink of some great
+river rolling swiftly to the sea--he sees its boiling waters and
+starts back in fear--even so did the son of Tydeus give ground.
+Then he said to his men, "My friends, how can we wonder that
+Hector wields the spear so well? Some god is ever by his side to
+protect him, and now Mars is with him in the likeness of mortal
+man. Keep your faces therefore towards the Trojans, but give
+ground backwards, for we dare not fight with gods."
+</para>
+
+<para>
+As he spoke the Trojans drew close up, and Hector killed two men,
+both in one chariot, Menesthes and Anchialus, heroes well versed
+in war. Ajax son of Telamon pitied them in their fall; he came
+close up and hurled his spear, hitting Amphius the son of
+Selagus, a man of great wealth who lived in Paesus and owned much
+corn-growing land, but his lot had led him to come to the aid of
+Priam and his sons. Ajax struck him in the belt; the spear
+pierced the lower part of his belly, and he fell heavily to the
+ground. Then Ajax ran towards him to strip him of his armour, but
+the Trojans rained spears upon him, many of which fell upon his
+shield. He planted his heel upon the body and drew out his spear,
+but the darts pressed so heavily upon him that he could not strip
+the goodly armour from his shoulders. The Trojan chieftains,
+moreover, many and valiant, came about him with their spears, so
+that he dared not stay; great, brave and valiant though he was,
+they drove him from them and he was beaten back.
+</para>
+
+<para>
+Thus, then, did the battle rage between them. Presently the
+strong hand of fate impelled Tlepolemus, the son of Hercules, a
+man both brave and of great stature, to fight Sarpedon; so the
+two, son and grandson of great Jove, drew near to one another,
+and Tlepolemus spoke first. "Sarpedon," said he, "councillor of
+the Lycians, why should you come skulking here you who are a man
+of peace? They lie who call you son of aegis-bearing Jove, for
+you are little like those who were of old his children. Far other
+was Hercules, my own brave and lion-hearted father, who came here
+for the horses of Laomedon, and though he had six ships only, and
+few men to follow him, sacked the city of Ilius and made a
+wilderness of her highways. You are a coward, and your people are
+falling from you. For all your strength, and all your coming from
+Lycia, you will be no help to the Trojans but will pass the gates
+of Hades vanquished by my hand."
+</para>
+
+<para>
+And Sarpedon, captain of the Lycians, answered, "Tlepolemus, your
+father overthrew Ilius by reason of Laomedon's folly in refusing
+payment to one who had served him well. He would not give your
+father the horses which he had come so far to fetch. As for
+yourself, you shall meet death by my spear. You shall yield glory
+to myself, and your soul to Hades of the noble steeds."
+</para>
+
+<para>
+Thus spoke Sarpedon, and Tlepolemus upraised his spear. They
+threw at the same moment, and Sarpedon struck his foe in the
+middle of his throat; the spear went right through, and the
+darkness of death fell upon his eyes. Tlepolemus's spear struck
+Sarpedon on the left thigh with such force that it tore through
+the flesh and grazed the bone, but his father as yet warded off
+destruction from him.
+</para>
+
+<para>
+His comrades bore Sarpedon out of the fight, in great pain by the
+weight of the spear that was dragging from his wound. They were
+in such haste and stress as they bore him that no one thought of
+drawing the spear from his thigh so as to let him walk uprightly.
+Meanwhile the Achaeans carried off the body of Tlepolemus,
+whereon Ulysses was moved to pity, and panted for the fray as he
+beheld them. He doubted whether to pursue the son of Jove, or to
+make slaughter of the Lycian rank and file; it was not decreed,
+however, that he should slay the son of Jove; Minerva, therefore,
+turned him against the main body of the Lycians. He killed
+Coeranus, Alastor, Chromius, Alcandrus, Halius, Noemon, and
+Prytanis, and would have slain yet more, had not great Hector
+marked him, and sped to the front of the fight clad in his suit
+of mail, filling the Danaans with terror. Sarpedon was glad when
+he saw him coming, and besought him, saying, "Son of Priam, let
+me not be here to fall into the hands of the Danaans. Help me,
+and since I may not return home to gladden the hearts of my wife
+and of my infant son, let me die within the walls of your city."
+</para>
+
+<para>
+Hector made him no answer, but rushed onward to fall at once upon
+the Achaeans and kill many among them. His comrades then bore
+Sarpedon away and laid him beneath Jove's spreading oak tree.
+Pelagon, his friend and comrade, drew the spear out of his thigh,
+but Sarpedon fainted and a mist came over his eyes. Presently he
+came to himself again, for the breath of the north wind as it
+played upon him gave him new life, and brought him out of the
+deep swoon into which he had fallen.
+</para>
+
+<para>
+Meanwhile the Argives were neither driven towards their ships by
+Mars and Hector, nor yet did they attack them; when they knew
+that Mars was with the Trojans they retreated, but kept their
+faces still turned towards the foe. Who, then, was first and who
+last to be slain by Mars and Hector? They were valiant Teuthras,
+and Orestes the renowned charioteer, Trechus the Aetolian
+warrior, Oenomaus, Helenus the son of Oenops, and Oresbius of the
+gleaming girdle, who was possessed of great wealth, and dwelt by
+the Cephisian lake with the other Boeotians who lived near him,
+owners of a fertile country.
+</para>
+
+<para>
+Now when the goddess Juno saw the Argives thus falling, she said
+to Minerva, "Alas, daughter of aegis-bearing Jove, unweariable,
+the promise we made Menelaus that he should not return till he
+had sacked the city of Ilius will be of no effect if we let Mars
+rage thus furiously. Let us go into the fray at once."
+</para>
+
+<para>
+Minerva did not gainsay her. Thereon the august goddess, daughter
+of great Saturn, began to harness her gold-bedizened steeds. Hebe
+with all speed fitted on the eight-spoked wheels of bronze that
+were on either side of the iron axle-tree. The felloes of the
+wheels were of gold, imperishable, and over these there was a
+tire of bronze, wondrous to behold. The naves of the wheels were
+silver, turning round the axle upon either side. The car itself
+was made with plaited bands of gold and silver, and it had a
+double top-rail running all round it. From the body of the car
+there went a pole of silver, on to the end of which she bound the
+golden yoke, with the bands of gold that were to go under the
+necks of the horses Then Juno put her steeds under the yoke,
+eager for battle and the war-cry.
+</para>
+
+<para>
+Meanwhile Minerva flung her richly embroidered vesture, made with
+her own hands, on to her father's threshold, and donned the shirt
+of Jove, arming herself for battle. She threw her tasselled aegis
+about her shoulders, wreathed round with Rout as with a fringe,
+and on it were Strife, and Strength, and Panic whose blood runs
+cold; moreover there was the head of the dread monster Gorgon,
+grim and awful to behold, portent of aegis-bearing Jove. On her
+head she set her helmet of gold, with four plumes, and coming to
+a peak both in front and behind--decked with the emblems of a
+hundred cities; then she stepped into her flaming chariot and
+grasped the spear, so stout and sturdy and strong, with which she
+quells the ranks of heroes who have displeased her. Juno lashed
+the horses on, and the gates of heaven bellowed as they flew open
+of their own accord--gates over which the Hours preside, in whose
+hands are Heaven and Olympus, either to open the dense cloud that
+hides them, or to close it. Through these the goddesses drove
+their obedient steeds, and found the son of Saturn sitting all
+alone on the topmost ridges of Olympus. There Juno stayed her
+horses, and spoke to Jove the son of Saturn, lord of all. "Father
+Jove," said she, "are you not angry with Mars for these high
+doings? how great and goodly a host of the Achaeans he has
+destroyed to my great grief, and without either right or reason,
+while the Cyprian and Apollo are enjoying it all at their ease
+and setting this unrighteous madman on to do further mischief. I
+hope, Father Jove, that you will not be angry if I hit Mars hard,
+and chase him out of the battle."
+</para>
+
+<para>
+And Jove answered, "Set Minerva on to him, for she punishes him
+more often than any one else does."
+</para>
+
+<para>
+Juno did as he had said. She lashed her horses, and they flew
+forward nothing loth midway betwixt earth and sky. As far as a
+man can see when he looks out upon the sea from some high beacon,
+so far can the loud-neighing horses of the gods spring at a
+single bound. When they reached Troy and the place where its two
+flowing streams Simois and Scamander meet, there Juno stayed them
+and took them from the chariot. She hid them in a thick cloud,
+and Simois made ambrosia spring up for them to eat; the two
+goddesses then went on, flying like turtledoves in their
+eagerness to help the Argives. When they came to the part where
+the bravest and most in number were gathered about mighty Diomed,
+fighting like lions or wild boars of great strength and
+endurance, there Juno stood still and raised a shout like that of
+brazen-voiced Stentor, whose cry was as loud as that of fifty men
+together. "Argives," she cried; "shame on cowardly creatures,
+brave in semblance only; as long as Achilles was fighting, if his
+spear was so deadly that the Trojans dared not show themselves
+outside the Dardanian gates, but now they sally far from the city
+and fight even at your ships."
+</para>
+
+<para>
+With these words she put heart and soul into them all, while
+Minerva sprang to the side of the son of Tydeus, whom she found
+near his chariot and horses, cooling the wound that Pandarus had
+given him. For the sweat caused by the hand that bore the weight
+of his shield irritated the hurt: his arm was weary with pain,
+and he was lifting up the strap to wipe away the blood. The
+goddess laid her hand on the yoke of his horses and said, "The
+son of Tydeus is not such another as his father. Tydeus was a
+little man, but he could fight, and rushed madly into the fray
+even when I told him not to do so. When he went all unattended as
+envoy to the city of Thebes among the Cadmeans, I bade him feast
+in their houses and be at peace; but with that high spirit which
+was ever present with him, he challenged the youth of the
+Cadmeans, and at once beat them in all that he attempted, so
+mightily did I help him. I stand by you too to protect you, and I
+bid you be instant in fighting the Trojans; but either you are
+tired out, or you are afraid and out of heart, and in that case I
+say that you are no true son of Tydeus the son of Oeneus."
+</para>
+
+<para>
+Diomed answered, "I know you, goddess, daughter of aegis-bearing
+Jove, and will hide nothing from you. I am not afraid nor out of
+heart, nor is there any slackness in me. I am only following your
+own instructions; you told me not to fight any of the blessed
+gods; but if Jove's daughter Venus came into battle I was to
+wound her with my spear. Therefore I am retreating, and bidding
+the other Argives gather in this place, for I know that Mars is
+now lording it in the field."
+</para>
+
+<para>
+"Diomed, son of Tydeus," replied Minerva, "man after my own
+heart, fear neither Mars nor any other of the immortals, for I
+will befriend you. Nay, drive straight at Mars, and smite him in
+close combat; fear not this raging madman, villain incarnate,
+first on one side and then on the other. But now he was holding
+talk with Juno and myself, saying he would help the Argives and
+attack the Trojans; nevertheless he is with the Trojans, and has
+forgotten the Argives."
+</para>
+
+<para>
+With this she caught hold of Sthenelus and lifted him off the
+chariot on to the ground. In a second he was on the ground,
+whereupon the goddess mounted the car and placed herself by the
+side of Diomed. The oaken axle groaned aloud under the burden of
+the awful goddess and the hero; Pallas Minerva took the whip and
+reins, and drove straight at Mars. He was in the act of stripping
+huge Periphas, son of Ochesius and bravest of the Aetolians.
+Bloody Mars was stripping him of his armour, and Minerva donned
+the helmet of Hades, that he might not see her; when, therefore,
+he saw Diomed, he made straight for him and let Periphas lie
+where he had fallen. As soon as they were at close quarters he
+let fly with his bronze spear over the reins and yoke, thinking
+to take Diomed's life, but Minerva caught the spear in her hand
+and made it fly harmlessly over the chariot. Diomed then threw,
+and Pallas Minerva drove the spear into the pit of Mars's stomach
+where his under-girdle went round him. There Diomed wounded him,
+tearing his fair flesh and then drawing his spear out again. Mars
+roared as loudly as nine or ten thousand men in the thick of a
+fight, and the Achaeans and Trojans were struck with panic, so
+terrible was the cry he raised.
+</para>
+
+<para>
+As a dark cloud in the sky when it comes on to blow after heat,
+even so did Diomed son of Tydeus see Mars ascend into the broad
+heavens. With all speed he reached high Olympus, home of the
+gods, and in great pain sat down beside Jove the son of Saturn.
+He showed Jove the immortal blood that was flowing from his
+wound, and spoke piteously, saying, "Father Jove, are you not
+angered by such doings? We gods are continually suffering in the
+most cruel manner at one another's hands while helping mortals;
+and we all owe you a grudge for having begotten that mad
+termagant of a daughter, who is always committing outrage of some
+kind. We other gods must all do as you bid us, but her you
+neither scold nor punish; you encourage her because the pestilent
+creature is your daughter. See how she has been inciting proud
+Diomed to vent his rage on the immortal gods. First he went up to
+the Cyprian and wounded her in the hand near her wrist, and then
+he sprang upon me too as though he were a god. Had I not run for
+it I must either have lain there for long enough in torments
+among the ghastly corpes, or have been eaten alive with spears
+till I had no more strength left in me."
+</para>
+
+<para>
+Jove looked angrily at him and said, "Do not come whining here,
+Sir Facing-both-ways. I hate you worst of all the gods in
+Olympus, for you are ever fighting and making mischief. You have
+the intolerable and stubborn spirit of your mother Juno: it is
+all I can do to manage her, and it is her doing that you are now
+in this plight: still, I cannot let you remain longer in such
+great pain; you are my own offspring, and it was by me that your
+mother conceived you; if, however, you had been the son of any
+other god, you are so destructive that by this time you should
+have been lying lower than the Titans."
+</para>
+
+<para>
+He then bade Paeeon heal him, whereon Paeeon spread pain-killing
+herbs upon his wound and cured him, for he was not of mortal
+mould. As the juice of the fig-tree curdles milk, and thickens it
+in a moment though it is liquid, even so instantly did Paeeon
+cure fierce Mars. Then Hebe washed him, and clothed him in goodly
+raiment, and he took his seat by his father Jove all glorious to
+behold.
+</para>
+
+<para>
+But Juno of Argos and Minerva of Alalcomene, now that they had
+put a stop to the murderous doings of Mars, went back again to
+the house of Jove.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK VI</title>
+
+<para>
+THE fight between Trojans and Achaeans was now left to rage as it
+would, and the tide of war surged hither and thither over the
+plain as they aimed their bronze-shod spears at one another
+between the streams of Simois and Xanthus.
+</para>
+
+<para>
+First, Ajax son of Telamon, tower of strength to the Achaeans,
+broke a phalanx of the Trojans, and came to the assistance of his
+comrades by killing Acamas son of Eussorus, the best man among
+the Thracians, being both brave and of great stature. The spear
+struck the projecting peak of his helmet: its bronze point then
+went through his forehead into the brain, and darkness veiled his
+eyes.
+</para>
+
+<para>
+Then Diomed killed Axylus son of Teuthranus, a rich man who lived
+in the strong city of Arisbe, and was beloved by all men; for he
+had a house by the roadside, and entertained every one who
+passed; howbeit not one of his guests stood before him to save
+his life, and Diomed killed both him and his squire Calesius, who
+was then his charioteer--so the pair passed beneath the earth.
+</para>
+
+<para>
+Euryalus killed Dresus and Opheltius, and then went in pursuit of
+Aesepus and Pedasus, whom the naiad nymph Abarbarea had borne to
+noble Bucolion. Bucolion was eldest son to Laomedon, but he was a
+bastard. While tending his sheep he had converse with the nymph,
+and she conceived twin sons; these the son of Mecisteus now slew,
+and he stripped the armour from their shoulders. Polypoetes then
+killed Astyalus, Ulysses Pidytes of Percote, and Teucer Aretaon.
+Ablerus fell by the spear of Nestor's son Antilochus, and
+Agamemnon, king of men, killed Elatus who dwelt in Pedasus by the
+banks of the river Satnioeis. Leitus killed Phylacus as he was
+flying, and Eurypylus slew Melanthus.
+</para>
+
+<para>
+Then Menelaus of the loud war-cry took Adrestus alive, for his
+horses ran into a tamarisk bush, as they were flying wildly over
+the plain, and broke the pole from the car; they went on towards
+the city along with the others in full flight, but Adrestus
+rolled out, and fell in the dust flat on his face by the wheel of
+his chariot; Menelaus came up to him spear in hand, but Adrestus
+caught him by the knees begging for his life. "Take me alive," he
+cried, "son of Atreus, and you shall have a full ransom for me:
+my father is rich and has much treasure of gold, bronze, and
+wrought iron laid by in his house. From this store he will give
+you a large ransom should he hear of my being alive and at the
+ships of the Achaeans."
+</para>
+
+<para>
+Thus did he plead, and Menelaus was for yielding and giving him
+to a squire to take to the ships of the Achaeans, but Agamemnon
+came running up to him and rebuked him. "My good Menelaus," said
+he, "this is no time for giving quarter. Has, then, your house
+fared so well at the hands of the Trojans? Let us not spare a
+single one of them--not even the child unborn and in its mother's
+womb; let not a man of them be left alive, but let all in Ilius
+perish, unheeded and forgotten."
+</para>
+
+<para>
+Thus did he speak, and his brother was persuaded by him, for his
+words were just. Menelaus, therefore, thrust Adrestus from him,
+whereon King Agamemnon struck him in the flank, and he fell: then
+the son of Atreus planted his foot upon his breast to draw his
+spear from the body.
+</para>
+
+<para>
+Meanwhile Nestor shouted to the Argives, saying, "My friends,
+Danaan warriors, servants of Mars, let no man lag that he may
+spoil the dead, and bring back much booty to the ships. Let us
+kill as many as we can; the bodies will lie upon the plain, and
+you can despoil them later at your leisure."
+</para>
+
+<para>
+With these words he put heart and soul into them all. And now the
+Trojans would have been routed and driven back into Ilius, had
+not Priam's son Helenus, wisest of augurs, said to Hector and
+Aeneas, "Hector and Aeneas, you two are the mainstays of the
+Trojans and Lycians, for you are foremost at all times, alike in
+fight and counsel; hold your ground here, and go about among the
+host to rally them in front of the gates, or they will fling
+themselves into the arms of their wives, to the great joy of our
+foes. Then, when you have put heart into all our companies, we
+will stand firm here and fight the Danaans however hard they
+press us, for there is nothing else to be done. Meanwhile do you,
+Hector, go to the city and tell our mother what is happening.
+Tell her to bid the matrons gather at the temple of Minerva in
+the acropolis; let her then take her key and open the doors of
+the sacred building; there, upon the knees of Minerva, let her
+lay the largest, fairest robe she has in her house--the one she
+sets most store by; let her, moreover, promise to sacrifice
+twelve yearling heifers that have never yet felt the goad, in the
+temple of the goddess, if she will take pity on the town, with
+the wives and little ones of the Trojans, and keep the son of
+Tydeus from falling on the goodly city of Ilius; for he fights
+with fury and fills men's souls with panic. I hold him mightiest
+of them all; we did not fear even their great champion Achilles,
+son of a goddess though he be, as we do this man: his rage is
+beyond all bounds, and there is none can vie with him in prowess"
+</para>
+
+<para>
+Hector did as his brother bade him. He sprang from his chariot,
+and went about everywhere among the host, brandishing his spears,
+urging the men on to fight, and raising the dread cry of battle.
+Thereon they rallied and again faced the Achaeans, who gave
+ground and ceased their murderous onset, for they deemed that
+some one of the immortals had come down from starry heaven to
+help the Trojans, so strangely had they rallied. And Hector
+shouted to the Trojans, "Trojans and allies, be men, my friends,
+and fight with might and main, while I go to Ilius and tell the
+old men of our council and our wives to pray to the gods and vow
+hecatombs in their honour."
+</para>
+
+<para>
+With this he went his way, and the black rim of hide that went
+round his shield beat against his neck and his ancles.
+</para>
+
+<para>
+Then Glaucus son of Hippolochus, and the son of Tydeus went into
+the open space between the hosts to fight in single combat. When
+they were close up to one another Diomed of the loud war-cry was
+the first to speak. "Who, my good sir," said he, "who are you
+among men? I have never seen you in battle until now, but you are
+daring beyond all others if you abide my onset. Woe to those
+fathers whose sons face my might. If, however, you are one of the
+immortals and have come down from heaven, I will not fight you;
+for even valiant Lycurgus, son of Dryas, did not live long when
+he took to fighting with the gods. He it was that drove the
+nursing women who were in charge of frenzied Bacchus through the
+land of Nysa, and they flung their thyrsi on the ground as
+murderous Lycurgus beat them with his oxgoad. Bacchus himself
+plunged terror-stricken into the sea, and Thetis took him to her
+bosom to comfort him, for he was scared by the fury with which
+the man reviled him. Thereon the gods who live at ease were angry
+with Lycurgus and the son of Saturn struck him blind, nor did he
+live much longer after he had become hateful to the immortals.
+Therefore I will not fight with the blessed gods; but if you are
+of them that eat the fruit of the ground, draw near and meet your
+doom."
+</para>
+
+<para>
+And the son of Hippolochus answered, son of Tydeus, why ask me of
+my lineage? Men come and go as leaves year by year upon the
+trees. Those of autumn the wind sheds upon the ground, but when
+spring returns the forest buds forth with fresh vines. Even so is
+it with the generations of mankind, the new spring up as the old
+are passing away. If, then, you would learn my descent, it is one
+that is well known to many. There is a city in the heart of
+Argos, pasture land of horses, called Ephyra, where Sisyphus
+lived, who was the craftiest of all mankind. He was the son of
+Aeolus, and had a son named Glaucus, who was father to
+Bellerophon, whom heaven endowed with the most surpassing
+comeliness and beauty. But Proetus devised his ruin, and being
+stronger than he, drove him from the land of the Argives, over
+which Jove had made him ruler. For Antea, wife of Proetus, lusted
+after him, and would have had him lie with her in secret; but
+Bellerophon was an honourable man and would not, so she told lies
+about him to Proteus. 'Proetus,' said she, 'kill Bellerophon or
+die, for he would have had converse with me against my will.' The
+king was angered, but shrank from killing Bellerophon, so he sent
+him to Lycia with lying letters of introduction, written on a
+folded tablet, and containing much ill against the bearer. He
+bade Bellerophon show these letters to his father-in-law, to the
+end that he might thus perish; Bellerophon therefore went to
+Lycia, and the gods convoyed him safely.
+</para>
+
+<para>
+"When he reached the river Xanthus, which is in Lycia, the king
+received him with all goodwill, feasted him nine days, and killed
+nine heifers in his honour, but when rosy-fingered morning
+appeared upon the tenth day, he questioned him and desired to see
+the letter from his son-in-law Proetus. When he had received the
+wicked letter he first commanded Bellerophon to kill that savage
+monster, the Chimaera, who was not a human being, but a goddess,
+for she had the head of a lion and the tail of a serpent, while
+her body was that of a goat, and she breathed forth flames of
+fire; but Bellerophon slew her, for he was guided by signs from
+heaven. He next fought the far-famed Solymi, and this, he said,
+was the hardest of all his battles. Thirdly, he killed the
+Amazons, women who were the peers of men, and as he was returning
+thence the king devised yet another plan for his destruction; he
+picked the bravest warriors in all Lycia, and placed them in
+ambuscade, but not a man ever came back, for Bellerophon killed
+every one of them. Then the king knew that he must be the valiant
+offspring of a god, so he kept him in Lycia, gave him his
+daughter in marriage, and made him of equal honour in the kingdom
+with himself; and the Lycians gave him a piece of land, the best
+in all the country, fair with vineyards and tilled fields, to
+have and to hold.
+</para>
+
+<para>
+"The king's daughter bore Bellerophon three children, Isander,
+Hippolochus, and Laodameia. Jove, the lord of counsel, lay with
+Laodameia, and she bore him noble Sarpedon; but when Bellerophon
+came to be hated by all the gods, he wandered all desolate and
+dismayed upon the Alean plain, gnawing at his own heart, and
+shunning the path of man. Mars, insatiate of battle, killed his
+son Isander while he was fighting the Solymi; his daughter was
+killed by Diana of the golden reins, for she was angered with
+her; but Hippolochus was father to myself, and when he sent me to
+Troy he urged me again and again to fight ever among the foremost
+and outvie my peers, so as not to shame the blood of my fathers
+who were the noblest in Ephyra and in all Lycia. This, then, is
+the descent I claim."
+</para>
+
+<para>
+Thus did he speak, and the heart of Diomed was glad. He planted
+his spear in the ground, and spoke to him with friendly words.
+"Then," he said, "you are an old friend of my father's house.
+Great Oeneus once entertained Bellerophon for twenty days, and
+the two exchanged presents. Oeneus gave a belt rich with purple,
+and Bellerophon a double cup, which I left at home when I set out
+for Troy. I do not remember Tydeus, for he was taken from us
+while I was yet a child, when the army of the Achaeans was cut to
+pieces before Thebes. Henceforth, however, I must be your host in
+middle Argos, and you mine in Lycia, if I should ever go there;
+let us avoid one another's spears even during a general
+engagement; there are many noble Trojans and allies whom I can
+kill, if I overtake them and heaven delivers them into my hand;
+so again with yourself, there are many Achaeans whose lives you
+may take if you can; we two, then, will exchange armour, that all
+present may know of the old ties that subsist between us."
+</para>
+
+<para>
+With these words they sprang from their chariots, grasped one
+another's hands, and plighted friendship. But the son of Saturn
+made Glaucus take leave of his wits, for he exchanged golden
+armour for bronze, the worth of a hundred head of cattle for the
+worth of nine.
+</para>
+
+<para>
+Now when Hector reached the Scaean gates and the oak tree, the
+wives and daughters of the Trojans came running towards him to
+ask after their sons, brothers, kinsmen, and husbands: he told
+them to set about praying to the gods, and many were made
+sorrowful as they heard him.
+</para>
+
+<para>
+Presently he reached the splendid palace of King Priam, adorned
+with colonnades of hewn stone. In it there were fifty
+bedchambers--all of hewn stone--built near one another, where the
+sons of Priam slept, each with his wedded wife. Opposite these,
+on the other side the courtyard, there were twelve upper rooms
+also of hewn stone for Priam's daughters, built near one another,
+where his sons-in-law slept with their wives. When Hector got
+there, his fond mother came up to him with Laodice the fairest of
+her daughters. She took his hand within her own and said, "My
+son, why have you left the battle to come hither? Are the
+Achaeans, woe betide them, pressing you hard about the city that
+you have thought fit to come and uplift your hands to Jove from
+the citadel? Wait till I can bring you wine that you may make
+offering to Jove and to the other immortals, and may then drink
+and be refreshed. Wine gives a man fresh strength when he is
+wearied, as you now are with fighting on behalf of your kinsmen."
+</para>
+
+<para>
+And Hector answered, "Honoured mother, bring no wine, lest you
+unman me and I forget my strength. I dare not make a
+drink-offering to Jove with unwashed hands; one who is
+bespattered with blood and filth may not pray to the son of
+Saturn. Get the matrons together, and go with offerings to the
+temple of Minerva driver of the spoil; there, upon the knees of
+Minerva, lay the largest and fairest robe you have in your
+house--the one you set most store by; promise, moreover, to
+sacrifice twelve yearling heifers that have never yet felt the
+goad, in the temple of the goddess if she will take pity on the
+town, with the wives and little ones of the Trojans, and keep the
+son of Tydeus from off the goodly city of Ilius, for he fights
+with fury, and fills men's souls with panic. Go, then, to the
+temple of Minerva, while I seek Paris and exhort him, if he will
+hear my words. Would that the earth might open her jaws and
+swallow him, for Jove bred him to be the bane of the Trojans, and
+of Priam and Priam's sons. Could I but see him go down into the
+house of Hades, my heart would forget its heaviness."
+</para>
+
+<para>
+His mother went into the house and called her waiting-women who
+gathered the matrons throughout the city. She then went down into
+her fragrant store-room, where her embroidered robes were kept,
+the work of Sidonian women, whom Alexandrus had brought over from
+Sidon when he sailed the seas upon that voyage during which he
+carried off Helen. Hecuba took out the largest robe, and the one
+that was most beautifully enriched with embroidery, as an
+offering to Minerva: it glittered like a star, and lay at the
+very bottom of the chest. With this she went on her way and many
+matrons with her.
+</para>
+
+<para>
+When they reached the temple of Minerva, lovely Theano, daughter
+of Cisseus and wife of Antenor, opened the doors, for the Trojans
+had made her priestess of Minerva. The women lifted up their
+hands to the goddess with a loud cry, and Theano took the robe to
+lay it upon the knees of Minerva, praying the while to the
+daughter of great Jove. "Holy Minerva," she cried, "protectress
+of our city, mighty goddess, break the spear of Diomed and lay
+him low before the Scaean gates. Do this, and we will sacrifice
+twelve heifers that have never yet known the goad, in your
+temple, if you will have pity upon the town, with the wives and
+little ones of the Trojans." Thus she prayed, but Pallas Minerva
+granted not her prayer.
+</para>
+
+<para>
+While they were thus praying to the daughter of great Jove,
+Hector went to the fair house of Alexandrus, which he had built
+for him by the foremost builders in the land. They had built him
+his house, storehouse, and courtyard near those of Priam and
+Hector on the acropolis. Here Hector entered, with a spear eleven
+cubits long in his hand; the bronze point gleamed in front of
+him, and was fastened to the shaft of the spear by a ring of
+gold. He found Alexandrus within the house, busied about his
+armour, his shield and cuirass, and handling his curved bow;
+there, too, sat Argive Helen with her women, setting them their
+several tasks; and as Hector saw him he rebuked him with words of
+scorn. "Sir," said he, "you do ill to nurse this rancour; the
+people perish fighting round this our town; you would yourself
+chide one whom you saw shirking his part in the combat. Up then,
+or ere long the city will be in a blaze."
+</para>
+
+<para>
+And Alexandrus answered, "Hector, your rebuke is just; listen
+therefore, and believe me when I tell you that I am not here so
+much through rancour or ill-will towards the Trojans, as from a
+desire to indulge my grief. My wife was even now gently urging me
+to battle, and I hold it better that I should go, for victory is
+ever fickle. Wait, then, while I put on my armour, or go first
+and I will follow. I shall be sure to overtake you."
+</para>
+
+<para>
+Hector made no answer, but Helen tried to soothe him. "Brother,"
+said she, "to my abhorred and sinful self, would that a whirlwind
+had caught me up on the day my mother brought me forth, and had
+borne me to some mountain or to the waves of the roaring sea that
+should have swept me away ere this mischief had come about. But,
+since the gods have devised these evils, would, at any rate, that
+I had been wife to a better man--to one who could smart under
+dishonour and men's evil speeches. This fellow was never yet to
+be depended upon, nor never will be, and he will surely reap what
+he has sown. Still, brother, come in and rest upon this seat, for
+it is you who bear the brunt of that toil that has been caused by
+my hateful self and by the sin of Alexandrus--both of whom Jove
+has doomed to be a theme of song among those that shall be born
+hereafter."
+</para>
+
+<para>
+And Hector answered, "Bid me not be seated, Helen, for all the
+goodwill you bear me. I cannot stay. I am in haste to help the
+Trojans, who miss me greatly when I am not among them; but urge
+your husband, and of his own self also let him make haste to
+overtake me before I am out of the city. I must go home to see my
+household, my wife and my little son, for I know not whether I
+shall ever again return to them, or whether the gods will cause
+me to fill by the hands of the Achaeans."
+</para>
+
+<para>
+Then Hector left her, and forthwith was at his own house. He did
+not find Andromache, for she was on the wall with her child and
+one of her maids, weeping bitterly. Seeing, then, that she was
+not within, he stood on the threshold of the women's rooms and
+said, "Women, tell me, and tell me true, where did Andromache go
+when she left the house? Was it to my sisters, or to my brothers'
+wives? or is she at the temple of Minerva where the other women
+are propitiating the awful goddess?"
+</para>
+
+<para>
+His good housekeeper answered, "Hector, since you bid me tell you
+truly, she did not go to your sisters nor to your brothers'
+wives, nor yet to the temple of Minerva, where the other women
+are propitiating the awful goddess, but she is on the high wall
+of Ilius, for she had heard the Trojans were being hard pressed,
+and that the Achaeans were in great force: she went to the wall
+in frenzied haste, and the nurse went with her carrying the
+child."
+</para>
+
+<para>
+Hector hurried from the house when she had done speaking, and
+went down the streets by the same way that he had come. When he
+had gone through the city and had reached the Scaean gates
+through which he would go out on to the plain, his wife came
+running towards him, Andromache, daughter of great Eetion who
+ruled in Thebe under the wooded slopes of Mt. Placus, and was
+king of the Cilicians. His daughter had married Hector, and now
+came to meet him with a nurse who carried his little child in her
+bosom--a mere babe. Hector's darling son, and lovely as a star.
+Hector had named him Scamandrius, but the people called him
+Astyanax, for his father stood alone as chief guardian of Ilius.
+Hector smiled as he looked upon the boy, but he did not speak,
+and Andromache stood by him weeping and taking his hand in her
+own. "Dear husband," said she, "your valour will bring you to
+destruction; think on your infant son, and on my hapless self who
+ere long shall be your widow--for the Achaeans will set upon you
+in a body and kill you. It would be better for me, should I lose
+you, to lie dead and buried, for I shall have nothing left to
+comfort me when you are gone, save only sorrow. I have neither
+father nor mother now. Achilles slew my father when he sacked
+Thebe the goodly city of the Cilicians. He slew him, but did not
+for very shame despoil him; when he had burned him in his
+wondrous armour, he raised a barrow over his ashes and the
+mountain nymphs, daughters of aegis-bearing Jove, planted a grove
+of elms about his tomb. I had seven brothers in my father's
+house, but on the same day they all went within the house of
+Hades. Achilles killed them as they were with their sheep and
+cattle. My mother--her who had been queen of all the land under
+Mt. Placus--he brought hither with the spoil, and freed her for a
+great sum, but the archer-queen Diana took her in the house of
+your father. Nay--Hector--you who to me are father, mother,
+brother, and dear husband--have mercy upon me; stay here upon
+this wall; make not your child fatherless, and your wife a widow;
+as for the host, place them near the fig-tree, where the city can
+be best scaled, and the wall is weakest. Thrice have the bravest
+of them come thither and assailed it, under the two Ajaxes,
+Idomeneus, the sons of Atreus, and the brave son of Tydeus,
+either of their own bidding, or because some soothsayer had told
+them."
+</para>
+
+<para>
+And Hector answered, "Wife, I too have thought upon all this, but
+with what face should I look upon the Trojans, men or women, if I
+shirked battle like a coward? I cannot do so: I know nothing save
+to fight bravely in the forefront of the Trojan host and win
+renown alike for my father and myself. Well do I know that the
+day will surely come when mighty Ilius shall be destroyed with
+Priam and Priam's people, but I grieve for none of these--not
+even for Hecuba, nor King Priam, nor for my brothers many and
+brave who may fall in the dust before their foes--for none of
+these do I grieve as for yourself when the day shall come on
+which some one of the Achaeans shall rob you for ever of your
+freedom, and bear you weeping away. It may be that you will have
+to ply the loom in Argos at the bidding of a mistress, or to
+fetch water from the springs Messeis or Hypereia, treated
+brutally by some cruel task-master; then will one say who sees
+you weeping, 'She was wife to Hector, the bravest warrior among
+the Trojans during the war before Ilius.' On this your tears will
+break forth anew for him who would have put away the day of
+captivity from you. May I lie dead under the barrow that is
+heaped over my body ere I hear your cry as they carry you into
+bondage."
+</para>
+
+<para>
+He stretched his arms towards his child, but the boy cried and
+nestled in his nurse's bosom, scared at the sight of his father's
+armour, and at the horse-hair plume that nodded fiercely from his
+helmet. His father and mother laughed to see him, but Hector took
+the helmet from his head and laid it all gleaming upon the
+ground. Then he took his darling child, kissed him, and dandled
+him in his arms, praying over him the while to Jove and to all
+the gods. "Jove," he cried, "grant that this my child may be even
+as myself, chief among the Trojans; let him be not less excellent
+in strength, and let him rule Ilius with his might. Then may one
+say of him as he comes from battle, 'The son is far better than
+the father.' May he bring back the blood-stained spoils of him
+whom he has laid low, and let his mother's heart be glad.'"
+</para>
+
+<para>
+With this he laid the child again in the arms of his wife, who
+took him to her own soft bosom, smiling through her tears. As her
+husband watched her his heart yearned towards her and he caressed
+her fondly, saying, "My own wife, do not take these things too
+bitterly to heart. No one can hurry me down to Hades before my
+time, but if a man's hour is come, be he brave or be he coward,
+there is no escape for him when he has once been born. Go, then,
+within the house, and busy yourself with your daily duties, your
+loom, your distaff, and the ordering of your servants; for war is
+man's matter, and mine above all others of them that have been
+born in Ilius."
+</para>
+
+<para>
+He took his plumed helmet from the ground, and his wife went back
+again to her house, weeping bitterly and often looking back
+towards him. When she reached her home she found her maidens
+within, and bade them all join in her lament; so they mourned
+Hector in his own house though he was yet alive, for they deemed
+that they should never see him return safe from battle, and from
+the furious hands of the Achaeans.
+</para>
+
+<para>
+Paris did not remain long in his house. He donned his goodly
+armour overlaid with bronze, and hasted through the city as fast
+as his feet could take him. As a horse, stabled and fed, breaks
+loose and gallops gloriously over the plain to the place where he
+is wont to bathe in the fair-flowing river--he holds his head
+high, and his mane streams upon his shoulders as he exults in his
+strength and flies like the wind to the haunts and feeding ground
+of the mares--even so went forth Paris from high Pergamus,
+gleaming like sunlight in his armour, and he laughed aloud as he
+sped swiftly on his way. Forthwith he came upon his brother
+Hector, who was then turning away from the place where he had
+held converse with his wife, and he was himself the first to
+speak. "Sir," said he, "I fear that I have kept you waiting when
+you are in haste, and have not come as quickly as you bade me."
+</para>
+
+<para>
+"My good brother," answered Hector, "you fight bravely, and no
+man with any justice can make light of your doings in battle. But
+you are careless and wilfully remiss. It grieves me to the heart
+to hear the ill that the Trojans speak about you, for they have
+suffered much on your account. Let us be going, and we will make
+things right hereafter, should Jove vouchsafe us to set the cup
+of our deliverance before ever-living gods of heaven in our own
+homes, when we have chased the Achaeans from Troy."
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK VII</title>
+
+<para>
+WITH these words Hector passed through the gates, and his brother
+Alexandrus with him, both eager for the fray. As when heaven
+sends a breeze to sailors who have long looked for one in vain,
+and have laboured at their oars till they are faint with toil,
+even so welcome was the sight of these two heroes to the Trojans.
+</para>
+
+<para>
+Thereon Alexandrus killed Menesthius the son of Areithous; he
+lived in Arne, and was son of Areithous the Mace-man, and of
+Phylomedusa. Hector threw a spear at Eioneus and struck him dead
+with a wound in the neck under the bronze rim of his helmet.
+Glaucus, moreover, son of Hippolochus, captain of the Lycians, in
+hard hand-to-hand fight smote Iphinous son of Dexius on the
+shoulder, as he was springing on to his chariot behind his fleet
+mares; so he fell to earth from the car, and there was no life
+left in him.
+</para>
+
+<para>
+When, therefore, Minerva saw these men making havoc of the
+Argives, she darted down to Ilius from the summits of Olympus,
+and Apollo, who was looking on from Pergamus, went out to meet
+her; for he wanted the Trojans to be victorious. The pair met by
+the oak tree, and King Apollo son of Jove was first to speak.
+"What would you have", said he, "daughter of great Jove, that
+your proud spirit has sent you hither from Olympus? Have you no
+pity upon the Trojans, and would you incline the scales of
+victory in favour of the Danaans? Let me persuade you--for it
+will be better thus--stay the combat for to-day, but let them
+renew the fight hereafter till they compass the doom of Ilius,
+since you goddesses have made up your minds to destroy the city."
+</para>
+
+<para>
+And Minerva answered, "So be it, Far-Darter; it was in this mind
+that I came down from Olympus to the Trojans and Achaeans. Tell
+me, then, how do you propose to end this present fighting?"
+</para>
+
+<para>
+Apollo, son of Jove, replied, "Let us incite great Hector to
+challenge some one of the Danaans in single combat; on this the
+Achaeans will be shamed into finding a man who will fight him."
+</para>
+
+<para>
+Minerva assented, and Helenus son of Priam divined the counsel of
+the gods; he therefore went up to Hector and said, "Hector son of
+Priam, peer of gods in counsel, I am your brother, let me then
+persuade you. Bid the other Trojans and Achaeans all of them take
+their seats, and challenge the best man among the Achaeans to
+meet you in single combat. I have heard the voice of the
+ever-living gods, and the hour of your doom is not yet come."
+</para>
+
+<para>
+Hector was glad when he heard this saying, and went in among the
+Trojans, grasping his spear by the middle to hold them back, and
+they all sat down. Agamemnon also bade the Achaeans be seated.
+But Minerva and Apollo, in the likeness of vultures, perched on
+father Jove's high oak tree, proud of their men; and the ranks
+sat close ranged together, bristling with shield and helmet and
+spear. As when the rising west wind furs the face of the sea and
+the waters grow dark beneath it, so sat the companies of Trojans
+and Achaeans upon the plain. And Hector spoke thus:--
+</para>
+
+<para>
+"Hear me, Trojans and Achaeans, that I may speak even as I am
+minded; Jove on his high throne has brought our oaths and
+covenants to nothing, and foreshadows ill for both of us, till
+you either take the towers of Troy, or are yourselves vanquished
+at your ships. The princes of the Achaeans are here present in
+the midst of you; let him, then, that will fight me stand forward
+as your champion against Hector. Thus I say, and may Jove be
+witness between us. If your champion slay me, let him strip me of
+my armour and take it to your ships, but let him send my body
+home that the Trojans and their wives may give me my dues of fire
+when I am dead. In like manner, if Apollo vouchsafe me glory and
+I slay your champion, I will strip him of his armour and take it
+to the city of Ilius, where I will hang it in the temple of
+Apollo, but I will give up his body, that the Achaeans may bury
+him at their ships, and the build him a mound by the wide waters
+of the Hellespont. Then will one say hereafter as he sails his
+ship over the sea, 'This is the monument of one who died long
+since a champion who was slain by mighty Hector.' Thus will one
+say, and my fame shall not be lost."
+</para>
+
+<para>
+Thus did he speak, but they all held their peace, ashamed to
+decline the challenge, yet fearing to accept it, till at last
+Menelaus rose and rebuked them, for he was angry. "Alas," he
+cried, "vain braggarts, women forsooth not men, double-dyed
+indeed will be the stain upon us if no man of the Danaans will
+now face Hector. May you be turned every man of you into earth
+and water as you sit spiritless and inglorious in your places. I
+will myself go out against this man, but the upshot of the fight
+will be from on high in the hands of the immortal gods."
+</para>
+
+<para>
+With these words he put on his armour; and then, O Menelaus, your
+life would have come to an end at the hands of hands of Hector,
+for he was far better the man, had not the princes of the
+Achaeans sprung upon you and checked you. King Agamemnon caught
+him by the right hand and said, "Menelaus, you are mad; a truce
+to this folly. Be patient in spite of passion, do not think of
+fighting a man so much stronger than yourself as Hector son of
+Priam, who is feared by many another as well as you. Even
+Achilles, who is far more doughty than you are, shrank from
+meeting him in battle. Sit down your own people, and the Achaeans
+will send some other champion to fight Hector; fearless and fond
+of battle though he be, I ween his knees will bend gladly under
+him if he comes out alive from the hurly-burly of this fight."
+</para>
+
+<para>
+With these words of reasonable counsel he persuaded his brother,
+whereon his squires gladly stripped the armour from off his
+shoulders. Then Nestor rose and spoke, "Of a truth," said he,
+"the Achaean land is fallen upon evil times. The old knight
+Peleus, counsellor and orator among the Myrmidons, loved when I
+was in his house to question me concerning the race and lineage
+of all the Argives. How would it not grieve him could he hear of
+them as now quailing before Hector? Many a time would he lift his
+hands in prayer that his soul might leave his body and go down
+within the house of Hades. Would, by father Jove, Minerva, and
+Apollo, that I were still young and strong as when the Pylians
+and Arcadians were gathered in fight by the rapid river Celadon
+under the walls of Pheia, and round about the waters of the river
+Iardanus. The godlike hero Ereuthalion stood forward as their
+champion, with the armour of King Areithous upon his shoulders--
+Areithous whom men and women had surnamed 'the Mace-man,' because
+he fought neither with bow nor spear, but broke the battalions of
+the foe with his iron mace. Lycurgus killed him, not in fair
+fight, but by entrapping him in a narrow way where his mace
+served him in no stead; for Lycurgus was too quick for him and
+speared him through the middle, so he fell to earth on his back.
+Lycurgus then spoiled him of the armour which Mars had given him,
+and bore it in battle thenceforward; but when he grew old and
+stayed at home, he gave it to his faithful squire Ereuthalion,
+who in this same armour challenged the foremost men among us. The
+others quaked and quailed, but my high spirit bade me fight him
+though none other would venture; I was the youngest man of them
+all; but when I fought him Minerva vouchsafed me victory. He was
+the biggest and strongest man that ever I killed, and covered
+much ground as he lay sprawling upon the earth. Would that I were
+still young and strong as I then was, for the son of Priam would
+then soon find one who would face him. But you, foremost among
+the whole host though you be, have none of you any stomach for
+fighting Hector."
+</para>
+
+<para>
+Thus did the old man rebuke them, and forthwith nine men started
+to their feet. Foremost of all uprose King Agamemnon, and after
+him brave Diomed the son of Tydeus. Next were the two Ajaxes, men
+clothed in valour as with a garment, and then Idomeneus, and
+Meriones his brother in arms. After these Eurypylus son of
+Euaemon, Thoas the son of Andraemon, and Ulysses also rose. Then
+Nestor knight of Gerene again spoke, saying: "Cast lots among you
+to see who shall be chosen. If he come alive out of this fight he
+will have done good service alike to his own soul and to the
+Achaeans."
+</para>
+
+<para>
+Thus he spoke, and when each of them had marked his lot, and had
+thrown it into the helmet of Agamemnon son of Atreus, the people
+lifted their hands in prayer, and thus would one of them say as
+he looked into the vault of heaven, "Father Jove, grant that the
+lot fall on Ajax, or on the son of Tydeus, or upon the king of
+rich Mycene himself."
+</para>
+
+<para>
+As they were speaking, Nestor knight of Gerene shook the helmet,
+and from it there fell the very lot which they wanted--the lot of
+Ajax. The herald bore it about and showed it to all the
+chieftains of the Achaeans, going from left to right; but they
+none of them owned it. When, however, in due course he reached
+the man who had written upon it and had put it into the helmet,
+brave Ajax held out his hand, and the herald gave him the lot.
+When Ajax saw his mark he knew it and was glad; he threw it to
+the ground and said, "My friends, the lot is mine, and I rejoice,
+for I shall vanquish Hector. I will put on my armour; meanwhile,
+pray to King Jove in silence among yourselves that the Trojans
+may not hear you--or aloud if you will, for we fear no man. None
+shall overcome me, neither by force nor cunning, for I was born
+and bred in Salamis, and can hold my own in all things."
+</para>
+
+<para>
+With this they fell praying to King Jove the son of Saturn, and
+thus would one of them say as he looked into the vault of heaven,
+"Father Jove that rulest from Ida, most glorious in power,
+vouchsafe victory to Ajax, and let him win great glory: but if
+you wish well to Hector also and would protect him, grant to each
+of them equal fame and prowess."
+</para>
+
+<para>
+Thus they prayed, and Ajax armed himself in his suit of gleaming
+bronze. When he was in full array he sprang forward as monstrous
+Mars when he takes part among men whom Jove has set fighting with
+one another--even so did huge Ajax, bulwark of the Achaeans,
+spring forward with a grim smile on his face as he brandished his
+long spear and strode onward. The Argives were elated as they
+beheld him, but the Trojans trembled in every limb, and the heart
+even of Hector beat quickly, but he could not now retreat and
+withdraw into the ranks behind him, for he had been the
+challenger. Ajax came up bearing his shield in front of him like
+a wall--a shield of bronze with seven folds of oxhide--the work
+of Tychius, who lived in Hyle and was by far the best worker in
+leather. He had made it with the hides of seven full-fed bulls,
+and over these he had set an eighth layer of bronze. Holding this
+shield before him, Ajax son of Telamon came close up to Hector,
+and menaced him saying, "Hector, you shall now learn, man to man,
+what kind of champions the Danaans have among them even besides
+lion-hearted Achilles cleaver of the ranks of men. He now abides
+at the ships in anger with Agamemnon shepherd of his people, but
+there are many of us who are well able to face you; therefore
+begin the fight."
+</para>
+
+<para>
+And Hector answered, "Noble Ajax, son of Telamon, captain of the
+host, treat me not as though I were some puny boy or woman that
+cannot fight. I have been long used to the blood and butcheries
+of battle. I am quick to turn my leathern shield either to right
+or left, for this I deem the main thing in battle. I can charge
+among the chariots and horsemen, and in hand to hand fighting can
+delight the heart of Mars; howbeit I would not take such a man as
+you are off his guard--but I will smite you openly if I can."
+</para>
+
+<para>
+He poised his spear as he spoke, and hurled it from him. It
+struck the sevenfold shield in its outermost layer--the eighth,
+which was of bronze--and went through six of the layers but in
+the seventh hide it stayed. Then Ajax threw in his turn, and
+struck the round shield of the son of Priam. The terrible spear
+went through his gleaming shield, and pressed onward through his
+cuirass of cunning workmanship; it pierced the shirt against his
+side, but he swerved and thus saved his life. They then each of
+them drew out the spear from his shield, and fell on one another
+like savage lions or wild boars of great strength and endurance:
+the son of Priam struck the middle of Ajax's shield, but the
+bronze did not break, and the point of his dart was turned. Ajax
+then sprang forward and pierced the shield of Hector; the spear
+went through it and staggered him as he was springing forward to
+attack; it gashed his neck and the blood came pouring from the
+wound, but even so Hector did not cease fighting; he gave ground,
+and with his brawny hand seized a stone, rugged and huge, that
+was lying upon the plain; with this he struck the shield of Ajax
+on the boss that was in its middle, so that the bronze rang
+again. But Ajax in turn caught up a far larger stone, swung it
+aloft, and hurled it with prodigious force. This millstone of a
+rock broke Hector's shield inwards and threw him down on his back
+with the shield crushing him under it, but Apollo raised him at
+once. Thereon they would have hacked at one another in close
+combat with their swords, had not heralds, messengers of gods and
+men, come forward, one from the Trojans and the other from the
+Achaeans--Talthybius and Idaeus both of them honourable men;
+these parted them with their staves, and the good herald Idaeus
+said, "My sons, fight no longer, you are both of you valiant, and
+both are dear to Jove; we know this; but night is now falling,
+and the behests of night may not be well gainsaid."
+</para>
+
+<para>
+Ajax son of Telamon answered, "Idaeus, bid Hector say so, for it
+was he that challenged our princes. Let him speak first and I
+will accept his saying."
+</para>
+
+<para>
+Then Hector said, "Ajax, heaven has vouchsafed you stature and
+strength, and judgement; and in wielding the spear you excel all
+others of the Achaeans. Let us for this day cease fighting;
+hereafter we will fight anew till heaven decide between us, and
+give victory to one or to the other; night is now falling, and
+the behests of night may not be well gainsaid. Gladden, then, the
+hearts of the Achaeans at your ships, and more especially those
+of your own followers and clansmen, while I, in the great city of
+King Priam, bring comfort to the Trojans and their women, who vie
+with one another in their prayers on my behalf. Let us, moreover,
+exchange presents that it may be said among the Achaeans and
+Trojans, 'They fought with might and main, but were reconciled
+and parted in friendship.'"
+</para>
+
+<para>
+On this he gave Ajax a silver-studded sword with its sheath and
+leathern baldric, and in return Ajax gave him a girdle dyed with
+purple. Thus they parted, the one going to the host of the
+Achaeans, and the other to that of the Trojans, who rejoiced when
+they saw their hero come to them safe and unharmed from the
+strong hands of mighty Ajax. They led him, therefore, to the city
+as one that had been saved beyond their hopes. On the other side
+the Achaeans brought Ajax elated with victory to Agamemnon.
+</para>
+
+<para>
+When they reached the quarters of the son of Atreus, Agamemnon
+sacrificed for them a five-year-old bull in honour of Jove the
+son of Saturn. They flayed the carcass, made it ready, and
+divided it into joints; these they cut carefully up into smaller
+pieces, putting them on the spits, roasting them sufficiently,
+and then drawing them off. When they had done all this and had
+prepared the feast, they ate it, and every man had his full and
+equal share, so that all were satisfied, and King Agamemnon gave
+Ajax some slices cut lengthways down the loin, as a mark of
+special honour. As soon as they had had enough to eat and drink,
+old Nestor whose counsel was ever truest began to speak; with all
+sincerity and goodwill, therefore, he addressed them thus:--
+</para>
+
+<para>
+"Son of Atreus, and other chieftains, inasmuch as many of the
+Achaeans are now dead, whose blood Mars has shed by the banks of
+the Scamander, and their souls have gone down to the house of
+Hades, it will be well when morning comes that we should cease
+fighting; we will then wheel our dead together with oxen and
+mules and burn them not far from the ships, that when we sail
+hence we may take the bones of our comrades home to their
+children. Hard by the funeral pyre we will build a barrow that
+shall be raised from the plain for all in common; near this let
+us set about building a high wall, to shelter ourselves and our
+ships, and let it have well-made gates that there may be a way
+through them for our chariots. Close outside we will dig a deep
+trench all round it to keep off both horse and foot, that the
+Trojan chieftains may not bear hard upon us."
+</para>
+
+<para>
+Thus he spoke, and the princess shouted in applause. Meanwhile
+the Trojans held a council, angry and full of discord, on the
+acropolis by the gates of King Priam's palace; and wise Antenor
+spoke. "Hear me," he said, "Trojans, Dardanians, and allies, that
+I may speak even as I am minded. Let us give up Argive Helen and
+her wealth to the sons of Atreus, for we are now fighting in
+violation of our solemn covenants, and shall not prosper till we
+have done as I say."
+</para>
+
+<para>
+He then sat down and Alexandrus husband of lovely Helen rose to
+speak. "Antenor," said he, "your words are not to my liking; you
+can find a better saying than this if you will; if, however, you
+have spoken in good earnest, then indeed has heaven robbed you of
+your reason. I will speak plainly, and hereby notify to the
+Trojans that I will not give up the woman; but the wealth that I
+brought home with her from Argos I will restore, and will add yet
+further of my own."
+</para>
+
+<para>
+On this, when Paris had spoken and taken his seat, Priam of the
+race of Dardanus, peer of gods in council, rose and with all
+sincerity and goodwill addressed them thus: "Hear me, Trojans,
+Dardanians, and allies, that I may speak even as I am minded. Get
+your suppers now as hitherto throughout the city, but keep your
+watches and be wakeful. At daybreak let Idaeus go to the ships,
+and tell Agamemnon and Menelaus sons of Atreus the saying of
+Alexandrus through whom this quarrel has come about; and let him
+also be instant with them that they now cease fighting till we
+burn our dead; hereafter we will fight anew, till heaven decide
+between us and give victory to one or to the other."
+</para>
+
+<para>
+Thus did he speak, and they did even as he had said. They took
+supper in their companies and at daybreak Idaeus went his way to
+the ships. He found the Danaans, servants of Mars, in council at
+the stern of Agamemnon's ship, and took his place in the midst of
+them. "Son of Atreus," he said, "and princes of the Achaean host,
+Priam and the other noble Trojans have sent me to tell you the
+saying of Alexandrus through whom this quarrel has come about, if
+so be that you may find it acceptable. All the treasure he took
+with him in his ships to Troy--would that he had sooner
+perished--he will restore, and will add yet further of his own,
+but he will not give up the wedded wife of Menelaus, though the
+Trojans would have him do so. Priam bade me inquire further if
+you will cease fighting till we burn our dead; hereafter we will
+fight anew, till heaven decide between us and give victory to one
+or to the other."
+</para>
+
+<para>
+They all held their peace, but presently Diomed of the loud
+war-cry spoke, saying, "Let there be no taking, neither treasure,
+nor yet Helen, for even a child may see that the doom of the
+Trojans is at hand."
+</para>
+
+<para>
+The sons of the Achaeans shouted applause at the words that
+Diomed had spoken, and thereon King Agamemnon said to Idaeus,
+"Idaeus, you have heard the answer the Achaeans make you-and I
+with them. But as concerning the dead, I give you leave to burn
+them, for when men are once dead there should be no grudging them
+the rites of fire. Let Jove the mighty husband of Juno be witness
+to this covenant."
+</para>
+
+<para>
+As he spoke he upheld his sceptre in the sight of all the gods,
+and Idaeus went back to the strong city of Ilius. The Trojans and
+Dardanians were gathered in council waiting his return; when he
+came, he stood in their midst and delivered his message. As soon
+as they heard it they set about their twofold labour, some to
+gather the corpses, and others to bring in wood. The Argives on
+their part also hastened from their ships, some to gather the
+corpses, and others to bring in wood.
+</para>
+
+<para>
+The sun was beginning to beat upon the fields, fresh risen into
+the vault of heaven from the slow still currents of deep Oceanus,
+when the two armies met. They could hardly recognise their dead,
+but they washed the clotted gore from off them, shed tears over
+them, and lifted them upon their waggons. Priam had forbidden the
+Trojans to wail aloud, so they heaped their dead sadly and
+silently upon the pyre, and having burned them went back to the
+city of Ilius. The Achaeans in like manner heaped their dead
+sadly and silently on the pyre, and having burned them went back
+to their ships.
+</para>
+
+<para>
+Now in the twilight when it was not yet dawn, chosen bands of the
+Achaeans were gathered round the pyre and built one barrow that
+was raised in common for all, and hard by this they built a high
+wall to shelter themselves and their ships; they gave it strong
+gates that there might be a way through them for their chariots,
+and close outside it they dug a trench deep and wide, and they
+planted it within with stakes.
+</para>
+
+<para>
+Thus did the Achaeans toil, and the gods, seated by the side of
+Jove the lord of lightning, marvelled at their great work; but
+Neptune, lord of the earthquake, spoke, saying, "Father Jove,
+what mortal in the whole world will again take the gods into his
+counsel? See you not how the Achaeans have built a wall about
+their ships and driven a trench all round it, without offering
+hecatombs to the gods? The fame of this wall will reach as far as
+dawn itself, and men will no longer think anything of the one
+which Phoebus Apollo and myself built with so much labour for
+Laomedon."
+</para>
+
+<para>
+Jove was displeased and answered, "What, O shaker of the earth,
+are you talking about? A god less powerful than yourself might be
+alarmed at what they are doing, but your fame reaches as far as
+dawn itself. Surely when the Achaeans have gone home with their
+ships, you can shatter their wall and fling it into the sea; you
+can cover the beach with sand again, and the great wall of the
+Achaeans will then be utterly effaced."
+</para>
+
+<para>
+Thus did they converse, and by sunset the work of the Achaeans
+was completed; they then slaughtered oxen at their tents and got
+their supper. Many ships had come with wine from Lemnos, sent by
+Euneus the son of Jason, born to him by Hypsipyle. The son of
+Jason freighted them with ten thousand measures of wine, which he
+sent specially to the sons of Atreus, Agamemnon and Menelaus.
+From this supply the Achaeans bought their wine, some with
+bronze, some with iron, some with hides, some with whole heifers,
+and some again with captives. They spread a goodly banquet and
+feasted the whole night through, as also did the Trojans and
+their allies in the city. But all the time Jove boded them ill
+and roared with his portentous thunder. Pale fear got hold upon
+them, and they spilled the wine from their cups on to the ground,
+nor did any dare drink till he had made offerings to the most
+mighty son of Saturn. Then they laid themselves down to rest and
+enjoyed the boon of sleep.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK VIII</title>
+
+<para>
+NOW when Morning, clad in her robe of saffron, had begun to
+suffuse light over the earth, Jove called the gods in council on
+the topmost crest of serrated Olympus. Then he spoke and all the
+other gods gave ear. "Hear me," said he, "gods and goddesses,
+that I may speak even as I am minded. Let none of you neither
+goddess nor god try to cross me, but obey me every one of you
+that I may bring this matter to an end. If I see anyone acting
+apart and helping either Trojans or Danaans, he shall be beaten
+inordinately ere he come back again to Olympus; or I will hurl
+him down into dark Tartarus far into the deepest pit under the
+earth, where the gates are iron and the floor bronze, as far
+beneath Hades as heaven is high above the earth, that you may
+learn how much the mightiest I am among you. Try me and find out
+for yourselves. Hangs me a golden chain from heaven, and lay hold
+of it all of you, gods and goddesses together--tug as you will,
+you will not drag Jove the supreme counsellor from heaven to
+earth; but were I to pull at it myself I should draw you up with
+earth and sea into the bargain, then would I bind the chain about
+some pinnacle of Olympus and leave you all dangling in the mid
+firmament. So far am I above all others either of gods or men."
+</para>
+
+<para>
+They were frightened and all of them of held their peace, for he
+had spoken masterfully; but at last Minerva answered, "Father,
+son of Saturn, king of kings, we all know that your might is not
+to be gainsaid, but we are also sorry for the Danaan warriors,
+who are perishing and coming to a bad end. We will, however,
+since you so bid us, refrain from actual fighting, but we will
+make serviceable suggestions to the Argives that they may not all
+of them perish in your displeasure."
+</para>
+
+<para>
+Jove smiled at her and answered, "Take heart, my child,
+Trito-born; I am not really in earnest, and I wish to be kind to
+you."
+</para>
+
+<para>
+With this he yoked his fleet horses, with hoofs of bronze and
+manes of glittering gold. He girded himself also with gold about
+the body, seized his gold whip and took his seat in his chariot.
+Thereon he lashed his horses and they flew forward nothing loth
+midway twixt earth and starry heaven. After a while he reached
+many-fountained Ida, mother of wild beasts, and Gargarus, where
+are his grove and fragrant altar. There the father of gods and
+men stayed his horses, took them from the chariot, and hid them
+in a thick cloud; then he took his seat all glorious upon the
+topmost crests, looking down upon the city of Troy and the ships
+of the Achaeans.
+</para>
+
+<para>
+The Achaeans took their morning meal hastily at the ships, and
+afterwards put on their armour. The Trojans on the other hand
+likewise armed themselves throughout the city, fewer in numbers
+but nevertheless eager perforce to do battle for their wives and
+children. All the gates were flung wide open, and horse and foot
+sallied forth with the tramp as of a great multitude.
+</para>
+
+<para>
+When they were got together in one place, shield clashed with
+shield, and spear with spear, in the conflict of mail-clad men.
+Mighty was the din as the bossed shields pressed hard on one
+another--death--cry and shout of triumph of slain and slayers,
+and the earth ran red with blood.
+</para>
+
+<para>
+Now so long as the day waxed and it was still morning their
+weapons beat against one another, and the people fell, but when
+the sun had reached mid-heaven, the sire of all balanced his
+golden scales, and put two fates of death within them, one for
+the Trojans and the other for the Achaeans. He took the balance
+by the middle, and when he lifted it up the day of the Achaeans
+sank; the death-fraught scale of the Achaeans settled down upon
+the ground, while that of the Trojans rose heavenwards. Then he
+thundered aloud from Ida, and sent the glare of his lightning
+upon the Achaeans; when they saw this, pale fear fell upon them
+and they were sore afraid.
+</para>
+
+<para>
+Idomeneus dared not stay nor yet Agamemnon, nor did the two
+Ajaxes, servants of Mars, hold their ground. Nestor knight of
+Gerene alone stood firm, bulwark of the Achaeans, not of his own
+will, but one of his horses was disabled. Alexandrus husband of
+lovely Helen had hit it with an arrow just on the top of its head
+where the mane begins to grow away from the skull, a very deadly
+place. The horse bounded in his anguish as the arrow pierced his
+brain, and his struggles threw others into confusion. The old man
+instantly began cutting the traces with his sword, but Hector's
+fleet horses bore down upon him through the rout with their bold
+charioteer, even Hector himself, and the old man would have
+perished there and then had not Diomed been quick to mark, and
+with a loud cry called Ulysses to help him.
+</para>
+
+<para>
+"Ulysses," he cried, "noble son of Laertes where are you flying
+to, with your back turned like a coward? See that you are not
+struck with a spear between the shoulders. Stay here and help me
+to defend Nestor from this man's furious onset."
+</para>
+
+<para>
+Ulysses would not give ear, but sped onward to the ships of the
+Achaeans, and the son of Tydeus flinging himself alone into the
+thick of the fight took his stand before the horses of the son of
+Neleus. "Sir," said he, "these young warriors are pressing you
+hard, your force is spent, and age is heavy upon you, your squire
+is naught, and your horses are slow to move. Mount my chariot and
+see what the horses of Tros can do--how cleverly they can scud
+hither and thither over the plain either in flight or in pursuit.
+I took them from the hero Aeneas. Let our squires attend to your
+own steeds, but let us drive mine straight at the Trojans, that
+Hector may learn how furiously I too can wield my spear."
+</para>
+
+<para>
+Nestor knight of Gerene hearkened to his words. Thereon the
+doughty squires, Sthenelus and kind-hearted Eurymedon, saw to
+Nestor's horses, while the two both mounted Diomed's chariot.
+Nestor took the reins in his hands and lashed the horses on; they
+were soon close up with Hector, and the son of Tydeus aimed a
+spear at him as he was charging full speed towards them. He
+missed him, but struck his charioteer and squire Eniopeus son of
+noble Thebaeus in the breast by the nipple while the reins were
+in his hands, so that he died there and then, and the horses
+swerved as he fell headlong from the chariot. Hector was greatly
+grieved at the loss of his charioteer, but let him lie for all
+his sorrow, while he went in quest of another driver; nor did his
+steeds have to go long without one, for he presently found brave
+Archeptolemus the son of Iphitus, and made him get up behind the
+horses, giving the reins into his hand.
+</para>
+
+<para>
+All had then been lost and no help for it, for they would have
+been penned up in Ilius like sheep, had not the sire of gods and
+men been quick to mark, and hurled a fiery flaming thunderbolt
+which fell just in front of Diomed's horses with a flare of
+burning brimstone. The horses were frightened and tried to back
+beneath the car, while the reins dropped from Nestor's hands.
+Then he was afraid and said to Diomed, "Son of Tydeus, turn your
+horses in flight; see you not that the hand of Jove is against
+you? To-day he vouchsafes victory to Hector; to-morrow, if it so
+please him, he will again grant it to ourselves; no man, however
+brave, may thwart the purpose of Jove, for he is far stronger
+than any."
+</para>
+
+<para>
+Diomed answered, "All that you have said is true; there is a
+grief however which pierces me to the very heart, for Hector will
+talk among the Trojans and say, 'The son of Tydeus fled before me
+to the ships.' This is the vaunt he will make, and may earth then
+swallow me."
+</para>
+
+<para>
+"Son of Tydeus," replied Nestor, "what mean you? Though Hector
+say that you are a coward the Trojans and Dardanians will not
+believe him, nor yet the wives of the mighty warriors whom you
+have laid low."
+</para>
+
+<para>
+So saying he turned the horses back through the thick of the
+battle, and with a cry that rent the air the Trojans and Hector
+rained their darts after them. Hector shouted to him and said,
+"Son of Tydeus, the Danaans have done you honour hitherto as
+regards your place at table, the meals they give you, and the
+filling of your cup with wine. Henceforth they will despise you,
+for you are become no better than a woman. Be off, girl and
+coward that you are, you shall not scale our walls through any
+flinching upon my part; neither shall you carry off our wives in
+your ships, for I shall kill you with my own hand."
+</para>
+
+<para>
+The son of Tydeus was in two minds whether or no to turn his
+horses round again and fight him. Thrice did he doubt, and thrice
+did Jove thunder from the heights of Ida in token to the Trojans
+that he would turn the battle in their favour. Hector then
+shouted to them and said, "Trojans, Lycians, and Dardanians,
+lovers of close fighting, be men, my friends, and fight with
+might and with main; I see that Jove is minded to vouchsafe
+victory and great glory to myself, while he will deal destruction
+upon the Danaans. Fools, for having thought of building this weak
+and worthless wall. It shall not stay my fury; my horses will
+spring lightly over their trench, and when I am at their ships
+forget not to bring me fire that I may burn them, while I
+slaughter the Argives who will be all dazed and bewildered by the
+smoke."
+</para>
+
+<para>
+Then he cried to his horses, "Xanthus and Podargus, and you
+Aethon and goodly Lampus, pay me for your keep now and for all
+the honey-sweet corn with which Andromache daughter of great
+Eetion has fed you, and for she has mixed wine and water for you
+to drink whenever you would, before doing so even for me who am
+her own husband. Haste in pursuit, that we may take the shield of
+Nestor, the fame of which ascends to heaven, for it is of solid
+gold, arm-rods and all, and that we may strip from the shoulders
+of Diomed. the cuirass which Vulcan made him. Could we take these
+two things, the Achaeans would set sail in their ships this
+self-same night."
+</para>
+
+<para>
+Thus did he vaunt, but Queen Juno made high Olympus quake as she
+shook with rage upon her throne. Then said she to the mighty god
+of Neptune, "What now, wide ruling lord of the earthquake? Can
+you find no compassion in your heart for the dying Danaans, who
+bring you many a welcome offering to Helice and to Aegae? Wish
+them well then. If all of us who are with the Danaans were to
+drive the Trojans back and keep Jove from helping them, he would
+have to sit there sulking alone on Ida."
+</para>
+
+<para>
+King Neptune was greatly troubled and answered, "Juno, rash of
+tongue, what are you talking about? We other gods must not set
+ourselves against Jove, for he is far stronger than we are."
+</para>
+
+<para>
+Thus did they converse; but the whole space enclosed by the
+ditch, from the ships even to the wall, was filled with horses
+and warriors, who were pent up there by Hector son of Priam, now
+that the hand of Jove was with him. He would even have set fire
+to the ships and burned them, had not Queen Juno put it into the
+mind of Agamemnon, to bestir himself and to encourage the
+Achaeans. To this end he went round the ships and tents carrying
+a great purple cloak, and took his stand by the huge black hull
+of Ulysses' ship, which was middlemost of all; it was from this
+place that his voice would carry farthest, on the one hand
+towards the tents of Ajax son of Telamon, and on the other
+towards those of Achilles--for these two heroes, well assured of
+their own strength, had valorously drawn up their ships at the
+two ends of the line. From this spot then, with a voice that
+could be heard afar, he shouted to the Danaans, saying, "Argives,
+shame on you cowardly creatures, brave in semblance only; where
+are now our vaunts that we should prove victorious--the vaunts we
+made so vaingloriously in Lemnos, when we ate the flesh of horned
+cattle and filled our mixing-bowls to the brim? You vowed that
+you would each of you stand against a hundred or two hundred men,
+and now you prove no match even for one--for Hector, who will be
+ere long setting our ships in a blaze. Father Jove, did you ever
+so ruin a great king and rob him so utterly of his greatness?
+Yet, when to my sorrow I was coming hither, I never let my ship
+pass your altars without offering the fat and thigh-bones of
+heifers upon every one of them, so eager was I to sack the city
+of Troy. Vouchsafe me then this prayer--suffer us to escape at
+any rate with our lives, and let not the Achaeans be so utterly
+vanquished by the Trojans."
+</para>
+
+<para>
+Thus did he pray, and father Jove pitying his tears vouchsafed
+him that his people should live, not die; forthwith he sent them
+an eagle, most unfailingly portentous of all birds, with a young
+fawn in its talons; the eagle dropped the fawn by the altar on
+which the Achaeans sacrificed to Jove the lord of omens; when,
+therefore, the people saw that the bird had come from Jove, they
+sprang more fiercely upon the Trojans and fought more boldly.
+</para>
+
+<para>
+There was no man of all the many Danaans who could then boast
+that he had driven his horses over the trench and gone forth to
+fight sooner than the son of Tydeus; long before any one else
+could do so he slew an armed warrior of the Trojans, Agelaus the
+son of Phradmon. He had turned his horses in flight, but the
+spear struck him in the back midway between his shoulders and
+went right through his chest, and his armour rang rattling round
+him as he fell forward from his chariot.
+</para>
+
+<para>
+After him came Agamemnon and Menelaus, sons of Atreus, the two
+Ajaxes clothed in valour as with a garment, Idomeneus and his
+companion in arms Meriones, peer of murderous Mars, and Eurypylus
+the brave son of Euaemon. Ninth came Teucer with his bow, and
+took his place under cover of the shield of Ajax son of Telamon.
+When Ajax lifted his shield Teucer would peer round, and when he
+had hit any one in the throng, the man would fall dead; then
+Teucer would hie back to Ajax as a child to its mother, and again
+duck down under his shield.
+</para>
+
+<para>
+Which of the Trojans did brave Teucer first kill? Orsilochus, and
+then Ormenus and Ophelestes, Daetor, Chromius, and godlike
+Lycophontes, Amopaon son of Polyaemon, and Melanippus. these in
+turn did he lay low upon the earth, and King Agamemnon was glad
+when he saw him making havoc of the Trojans with his mighty bow.
+He went up to him and said, "Teucer, man after my own heart, son
+of Telamon, captain among the host, shoot on, and be at once the
+saving of the Danaans and the glory of your father Telamon, who
+brought you up and took care of you in his own house when you
+were a child, bastard though you were. Cover him with glory
+though he is far off; I will promise and I will assuredly
+perform; if aegis-bearing Jove and Minerva grant me to sack the
+city of Ilius, you shall have the next best meed of honour after
+my own--a tripod, or two horses with their chariot, or a woman
+who shall go up into your bed."
+</para>
+
+<para>
+And Teucer answered, "Most noble son of Atreus, you need not urge
+me; from the moment we began to drive them back to Ilius, I have
+never ceased so far as in me lies to look out for men whom I can
+shoot and kill; I have shot eight barbed shafts, and all of them
+have been buried in the flesh of warlike youths, but this mad dog
+I cannot hit."
+</para>
+
+<para>
+As he spoke he aimed another arrow straight at Hector, for he was
+bent on hitting him; nevertheless he missed him, and the arrow
+hit Priam's brave son Gorgythion in the breast. His mother, fair
+Castianeira, lovely as a goddess, had been married from Aesyme,
+and now he bowed his head as a garden poppy in full bloom when it
+is weighed down by showers in spring--even thus heavy bowed his
+head beneath the weight of his helmet.
+</para>
+
+<para>
+Again he aimed at Hector, for he was longing to hit him, and
+again his arrow missed, for Apollo turned it aside; but he hit
+Hector's brave charioteer Archeptolemus in the breast, by the
+nipple, as he was driving furiously into the fight. The horses
+swerved aside as he fell headlong from the chariot, and there was
+no life left in him. Hector was greatly grieved at the loss of
+his charioteer, but for all his sorrow he let him lie where he
+fell, and bade his brother Cebriones, who was hard by, take the
+reins. Cebriones did as he had said. Hector thereon with a loud
+cry sprang from his chariot to the ground, and seizing a great
+stone made straight for Teucer with intent kill him. Teucer had
+just taken an arrow from his quiver and had laid it upon the
+bow-string, but Hector struck him with the jagged stone as he was
+taking aim and drawing the string to his shoulder; he hit him
+just where the collar-bone divides the neck from the chest, a
+very deadly place, and broke the sinew of his arm so that his
+wrist was less, and the bow dropped from his hand as he fell
+forward on his knees. Ajax saw that his brother had fallen, and
+running towards him bestrode him and sheltered him with his
+shield. Meanwhile his two trusty squires, Mecisteus son of
+Echius, and Alastor, came up and bore him to the ships groaning
+in his great pain.
+</para>
+
+<para>
+Jove now again put heart into the Trojans, and they drove the
+Achaeans to their deep trench with Hector in all his glory at
+their head. As a hound grips a wild boar or lion in flank or
+buttock when he gives him chase, and watches warily for his
+wheeling, even so did Hector follow close upon the Achaeans, ever
+killing the hindmost as they rushed panic-stricken onwards. When
+they had fled through the set stakes and trench and many Achaeans
+had been laid low at the hands of the Trojans, they halted at
+their ships, calling upon one another and praying every man
+instantly as they lifted up their hands to the gods; but Hector
+wheeled his horses this way and that, his eyes glaring like those
+of Gorgo or murderous Mars.
+</para>
+
+<para>
+Juno when she saw them had pity upon them, and at once said to
+Minerva, "Alas, child of aegis-bearing Jove, shall you and I take
+no more thought for the dying Danaans, though it be the last time
+we ever do so? See how they perish and come to a bad end before
+the onset of but a single man. Hector the son of Priam rages with
+intolerable fury, and has already done great mischief."
+</para>
+
+<para>
+Minerva answered, "Would, indeed, this fellow might die in his
+own land, and fall by the hands of the Achaeans; but my father
+Jove is mad with spleen, ever foiling me, ever headstrong and
+unjust. He forgets how often I saved his son when he was worn out
+by the labours Eurystheus had laid on him. He would weep till his
+cry came up to heaven, and then Jove would send me down to help
+him; if I had had the sense to foresee all this, when Eurystheus
+sent him to the house of Hades, to fetch the hell-hound from
+Erebus, he would never have come back alive out of the deep
+waters of the river Styx. And now Jove hates me, while he lets
+Thetis have her way because she kissed his knees and took hold of
+his beard, when she was begging him to do honour to Achilles. I
+shall know what to do next time he begins calling me his
+grey-eyed darling. Get our horses ready, while I go within the
+house of aegis-bearing Jove and put on my armour; we shall then
+find out whether Priam's son Hector will be glad to meet us in
+the highways of battle, or whether the Trojans will glut hounds
+and vultures with the fat of their flesh as they be dead by the
+ships of the Achaeans."
+</para>
+
+<para>
+Thus did she speak and white-armed Juno, daughter of great
+Saturn, obeyed her words; she set about harnessing her
+gold-bedizened steeds, while Minerva daughter of aegis-bearing
+Jove flung her richly vesture, made with her own hands, on to the
+threshold of her father, and donned the shirt of Jove, arming
+herself for battle. Then she stepped into her flaming chariot,
+and grasped the spear so stout and sturdy and strong with which
+she quells the ranks of heroes who have displeased her. Juno
+lashed her horses, and the gates of heaven bellowed as they flew
+open of their own accord--gates over which the Hours preside, in
+whose hands are heaven and Olympus, either to open the dense
+cloud that hides them or to close it. Through these the goddesses
+drove their obedient steeds.
+</para>
+
+<para>
+But father Jove when he saw them from Ida was very angry, and
+sent winged Iris with a message to them. "Go," said he, "fleet
+Iris, turn them back, and see that they do not come near me, for
+if we come to fighting there will be mischief. This is what I
+say, and this is what I mean to do. I will lame their horses for
+them; I will hurl them from their chariot, and will break it in
+pieces. It will take them all ten years to heal the wounds my
+lightning shall inflict upon them; my grey-eyed daughter will
+then learn what quarrelling with her father means. I am less
+surprised and angry with Juno, for whatever I say she always
+contradicts me."
+</para>
+
+<para>
+With this Iris went her way, fleet as the wind, from the heights
+of Ida to the lofty summits of Olympus. She met the goddesses at
+the outer gates of its many valleys and gave them her message.
+"What," said she, "are you about? Are you mad? The son of Saturn
+forbids going. This is what he says, and this is he means to do,
+he will lame your horses for you, he will hurl you from your
+chariot, and will break it in pieces. It will take you all ten
+years to heal the wounds his lightning will inflict upon you,
+that you may learn, grey-eyed goddess, what quarrelling with your
+father means. He is less hurt and angry with Juno, for whatever
+he says she always contradicts him but you, bold hussy, will you
+really dare to raise your huge spear in defiance of Jove?"
+</para>
+
+<para>
+With this she left them, and Juno said to Minerva, "Of a truth,
+child of aegis-bearing Jove, I am not for fighting men's battles
+further in defiance of Jove. Let them live or die as luck will
+have it, and let Jove mete out his judgements upon the Trojans
+and Danaans according to his own pleasure."
+</para>
+
+<para>
+She turned her steeds; the Hours presently unyoked them, made
+them fast to their ambrosial mangers, and leaned the chariot
+against the end wall of the courtyard. The two goddesses then sat
+down upon their golden thrones, amid the company of the other
+gods; but they were very angry.
+</para>
+
+<para>
+Presently father Jove drove his chariot to Olympus, and entered
+the assembly of gods. The mighty lord of the earthquake unyoked
+his horses for him, set the car upon its stand, and threw a cloth
+over it. Jove then sat down upon his golden throne and Olympus
+reeled beneath him. Minerva and Juno sat alone, apart from Jove,
+and neither spoke nor asked him questions, but Jove knew what
+they meant, and said, "Minerva and Juno, why are you so angry?
+Are you fatigued with killing so many of your dear friends the
+Trojans? Be this as it may, such is the might of my hands that
+all the gods in Olympus cannot turn me; you were both of you
+trembling all over ere ever you saw the fight and its terrible
+doings. I tell you therefore-and it would have surely been--I
+should have struck you with lighting, and your chariots would
+never have brought you back again to Olympus."
+</para>
+
+<para>
+Minerva and Juno groaned in spirit as they sat side by side and
+brooded mischief for the Trojans. Minerva sat silent without a
+word, for she was in a furious passion and bitterly incensed
+against her father; but Juno could not contain herself and said,
+"What, dread son of Saturn, are you talking about? We know how
+great your power is, nevertheless we have compassion upon the
+Danaan warriors who are perishing and coming to a bad end. We
+will, however, since you so bid us, refrain from actual fighting,
+but we will make serviceable suggestions to the Argives, that
+they may not all of them perish in your displeasure."
+</para>
+
+<para>
+And Jove answered, "To-morrow morning, Juno, if you choose to do
+so, you will see the son of Saturn destroying large numbers of
+the Argives, for fierce Hector shall not cease fighting till he
+has roused the son of Peleus when they are fighting in dire
+straits at their ships' sterns about the body of Patroclus. Like
+it or no, this is how it is decreed; for aught I care, you may go
+to the lowest depths beneath earth and sea, where Iapetus and
+Saturn dwell in lone Tartarus with neither ray of light nor
+breath of wind to cheer them. You may go on and on till you get
+there, and I shall not care one whit for your displeasure; you
+are the greatest vixen living."
+</para>
+
+<para>
+Juno made him no answer. The sun's glorious orb now sank into
+Oceanus and drew down night over the land. Sorry indeed were the
+Trojans when light failed them, but welcome and thrice prayed for
+did darkness fall upon the Achaeans.
+</para>
+
+<para>
+Then Hector led the Trojans back from the ships, and held a
+council on the open space near the river, where there was a spot
+clear of corpses. They left their chariots and sat down on the
+ground to hear the speech he made them. He grasped a spear eleven
+cubits long, the bronze point of which gleamed in front of it,
+while the ring round the spear-head was of gold. Spear in hand he
+spoke. "Hear me," said he, "Trojans, Dardanians, and allies. I
+deemed but now that I should destroy the ships and all the
+Achaeans with them ere I went back to Ilius, but darkness came on
+too soon. It was this alone that saved them and their ships upon
+the seashore. Now, therefore, let us obey the behests of night,
+and prepare our suppers. Take your horses out of their chariots
+and give them their feeds of corn; then make speed to bring sheep
+and cattle from the city; bring wine also and corn for your
+horses and gather much wood, that from dark till dawn we may burn
+watchfires whose flare may reach to heaven. For the Achaeans may
+try to fly beyond the sea by night, and they must not embark
+scatheless and unmolested; many a man among them must take a dart
+with him to nurse at home, hit with spear or arrow as he is
+leaping on board his ship, that others may fear to bring war and
+weeping upon the Trojans. Moreover let the heralds tell it about
+the city that the growing youths and grey-bearded men are to camp
+upon its heaven-built walls. Let the women each of them light a
+great fire in her house, and let watch be safely kept lest the
+town be entered by surprise while the host is outside. See to it,
+brave Trojans, as I have said, and let this suffice for the
+moment; at daybreak I will instruct you further. I pray in hope
+to Jove and to the gods that we may then drive those fate-sped
+hounds from our land, for 'tis the fates that have borne them and
+their ships hither. This night, therefore, let us keep watch, but
+with early morning let us put on our armour and rouse fierce war
+at the ships of the Achaeans; I shall then know whether brave
+Diomed the son of Tydeus will drive me back from the ships to the
+wall, or whether I shall myself slay him and carry off his
+bloodstained spoils. To-morrow let him show his mettle, abide my
+spear if he dare. I ween that at break of day, he shall be among
+the first to fall and many another of his comrades round him.
+Would that I were as sure of being immortal and never growing
+old, and of being worshipped like Minerva and Apollo, as I am
+that this day will bring evil to the Argives."
+</para>
+
+<para>
+Thus spoke Hector and the Trojans shouted applause. They took
+their sweating steeds from under the yoke, and made them fast
+each by his own chariot. They made haste to bring sheep and
+cattle from the city, they brought wine also and corn from their
+houses and gathered much wood. They then offered unblemished
+hecatombs to the immortals, and the wind carried the sweet savour
+of sacrifice to heaven--but the blessed gods partook not thereof,
+for they bitterly hated Ilius with Priam and Priam's people. Thus
+high in hope they sat through the livelong night by the highways
+of war, and many a watchfire did they kindle. As when the stars
+shine clear, and the moon is bright--there is not a breath of
+air, not a peak nor glade nor jutting headland but it stands out
+in the ineffable radiance that breaks from the serene of heaven;
+the stars can all of them be told and the heart of the shepherd
+is glad--even thus shone the watchfires of the Trojans before
+Ilius midway between the ships and the river Xanthus. A thousand
+camp-fires gleamed upon the plain, and in the glow of each there
+sat fifty men, while the horses, champing oats and corn beside
+their chariots, waited till dawn should come.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK IX</title>
+
+<para>
+THUS did the Trojans watch. But Panic, comrade of blood-stained
+Rout, had taken fast hold of the Achaeans, and their princes were
+all of them in despair. As when the two winds that blow from
+Thrace--the north and the northwest--spring up of a sudden and
+rouse the fury of the main--in a moment the dark waves uprear
+their heads and scatter their sea-wrack in all directions--even
+thus troubled were the hearts of the Achaeans.
+</para>
+
+<para>
+The son of Atreus in dismay bade the heralds call the people to a
+council man by man, but not to cry the matter aloud; he made
+haste also himself to call them, and they sat sorry at heart in
+their assembly. Agamemnon shed tears as it were a running stream
+or cataract on the side of some sheer cliff; and thus, with many
+a heavy sigh he spoke to the Achaeans. "My friends," said he,
+"princes and councillors Of the Argives, the hand of heaven has
+been laid heavily upon me. Cruel Jove gave me his solemn promise
+that I should sack the city of Troy before returning, but he has
+played me false, and is now bidding me go ingloriously back to
+Argos with the loss of much people. Such is the will of Jove, who
+has laid many a proud city in the dust as he will yet lay others,
+for his power is above all. Now, therefore, let us all do as I
+say and sail back to our own country, for we shall not take
+Troy."
+</para>
+
+<para>
+Thus he spoke, and the sons of the Achaeans for a long while sat
+sorrowful there, but they all held their peace, till at last
+Diomed of the loud battle-cry made answer saying, "Son of Atreus,
+I will chide your folly, as is my right in council. Be not then
+aggrieved that I should do so. In the first place you attacked me
+before all the Danaans and said that I was a coward and no
+soldier. The Argives young and old know that you did so. But the
+son of scheming Saturn endowed you by halves only. He gave you
+honour as the chief ruler over us, but valour, which is the
+highest both right and might he did not give you. Sir, think you
+that the sons of the Achaeans are indeed as unwarlike and
+cowardly as you say they are? If your own mind is set upon going
+home--go--the way is open to you; the many ships that followed
+you from Mycene stand ranged upon the seashore; but the rest of
+us stay here till we have sacked Troy. Nay though these too
+should turn homeward with their ships, Sthenelus and myself will
+still fight on till we reach the goal of Ilius, for heaven was
+with us when we came."
+</para>
+
+<para>
+The sons of the Achaeans shouted applause at the words of Diomed,
+and presently Nestor rose to speak. "Son of Tydeus," said he, "in
+war your prowess is beyond question, and in council you excel all
+who are of your own years; no one of the Achaeans can make light
+of what you say nor gainsay it, but you have not yet come to the
+end of the whole matter. You are still young--you might be the
+youngest of my own children--still you have spoken wisely and
+have counselled the chief of the Achaeans not without discretion;
+nevertheless I am older than you and I will tell you everything;
+therefore let no man, not even King Agamemnon, disregard my
+saying, for he that foments civil discord is a clanless,
+hearthless outlaw.
+</para>
+
+<para>
+"Now, however, let us obey the behests of night and get our
+suppers, but let the sentinels every man of them camp by the
+trench that is without the wall. I am giving these instructions
+to the young men; when they have been attended to, do you, son of
+Atreus, give your orders, for you are the most royal among us
+all. Prepare a feast for your councillors; it is right and
+reasonable that you should do so; there is abundance of wine in
+your tents, which the ships of the Achaeans bring from Thrace
+daily. You have everything at your disposal wherewith to
+entertain guests, and you have many subjects. When many are got
+together, you can be guided by him whose counsel is wisest--and
+sorely do we need shrewd and prudent counsel, for the foe has lit
+his watchfires hard by our ships. Who can be other than dismayed?
+This night will either be the ruin of our host, or save it."
+</para>
+
+<para>
+Thus did he speak, and they did even as he had said. The
+sentinels went out in their armour under command of Nestor's son
+Thrasymedes, a captain of the host, and of the bold warriors
+Ascalaphus and Ialmenus: there were also Meriones, Aphareus and
+Deipyrus, and the son of Creion, noble Lycomedes. There were
+seven captains of the sentinels, and with each there went a
+hundred youths armed with long spears: they took their places
+midway between the trench and the wall, and when they had done so
+they lit their fires and got every man his supper.
+</para>
+
+<para>
+The son of Atreus then bade many councillors of the Achaeans to
+his quarters prepared a great feast in their honour. They laid
+their hands on the good things that were before them, and as soon
+as they had enough to eat and drink, old Nestor, whose counsel
+was ever truest, was the first to lay his mind before them. He,
+therefore, with all sincerity and goodwill addressed them thus.
+</para>
+
+<para>
+"With yourself, most noble son of Atreus, king of men, Agamemnon,
+will I both begin my speech and end it, for you are king over
+much people. Jove, moreover, has vouchsafed you to wield the
+sceptre and to uphold righteousness, that you may take thought
+for your people under you; therefore it behooves you above all
+others both to speak and to give ear, and to out the counsel of
+another who shall have been minded to speak wisely. All turns on
+you and on your commands, therefore I will say what I think will
+be best. No man will be of a truer mind than that which has been
+mine from the hour when you, sir, angered Achilles by taking the
+girl Briseis from his tent against my judgment. I urged you not
+to do so, but you yielded to your own pride, and dishonoured a
+hero whom heaven itself had honoured--for you still hold the
+prize that had been awarded to him. Now, however, let us think
+how we may appease him, both with presents and fair speeches that
+may conciliate him."
+</para>
+
+<para>
+And King Agamemnon answered, "Sir, you have reproved my folly
+justly. I was wrong. I own it. One whom heaven befriends is in
+himself a host, and Jove has shown that he befriends this man by
+destroying much people of the Achaeans. I was blinded with
+passion and yielded to my worser mind; therefore I will make
+amends, and will give him great gifts by way of atonement. I will
+tell them in the presence of you all. I will give him seven
+tripods that have never yet been on the fire, and ten talents of
+gold. I will give him twenty iron cauldrons and twelve strong
+horses that have won races and carried off prizes. Rich, indeed,
+both in land and gold is he that has as many prizes as my horses
+have won me. I will give him seven excellent workwomen, Lesbians,
+whom I chose for myself when he took Lesbos--all of surpassing
+beauty. I will give him these, and with them her whom I erewhile
+took from him, the daughter of Briseus; and I swear a great oath
+that I never went up into her couch, nor have been with her after
+the manner of men and women.
+</para>
+
+<para>
+"All these things will I give him now, and if hereafter the gods
+vouchsafe me to sack the city of Priam, let him come when we
+Achaeans are dividing the spoil, and load his ship with gold and
+bronze to his liking; furthermore let him take twenty Trojan
+women, the loveliest after Helen herself. Then, when we reach
+Achaean Argos, wealthiest of all lands, he shall be my son-in-law
+and I will show him like honour with my own dear son Orestes, who
+is being nurtured in all abundance. I have three daughters,
+Chrysothemis, Laodice, and lphianassa, let him take the one of
+his choice, freely and without gifts of wooing, to the house of
+Peleus; I will add such dower to boot as no man ever yet gave his
+daughter, and will give him seven well established cities,
+Cardamyle, Enope, and Hire, where there is grass; holy Pherae and
+the rich meadows of Anthea; Aepea also, and the vine-clad slopes
+of Pedasus, all near the sea, and on the borders of sandy Pylos.
+The men that dwell there are rich in cattle and sheep; they will
+honour him with gifts as though he were a god, and be obedient to
+his comfortable ordinances. All this will I do if he will now
+forgo his anger. Let him then yield; it is only Hades who is
+utterly ruthless and unyielding--and hence he is of all gods the
+one most hateful to mankind. Moreover I am older and more royal
+than himself. Therefore, let him now obey me."
+</para>
+
+<para>
+Then Nestor answered, "Most noble son of Atreus, king of men,
+Agamemnon. The gifts you offer are no small ones, let us then
+send chosen messengers, who may go to the tent of Achilles son of
+Peleus without delay. Let those go whom I shall name. Let
+Phoenix, dear to Jove, lead the way; let Ajax and Ulysses follow,
+and let the heralds Odius and Eurybates go with them. Now bring
+water for our hands, and bid all keep silence while we pray to
+Jove the son of Saturn, if so be that he may have mercy upon us."
+</para>
+
+<para>
+Thus did he speak, and his saying pleased them well.
+Men-servants poured water over the hands of the guests, while
+pages filled the mixing-bowls with wine and water, and handed it
+round after giving every man his drink-offering; then, when they
+had made their offerings, and had drunk each as much as he was
+minded, the envoys set out from the tent of Agamemnon son of
+Atreus; and Nestor, looking first to one and then to another, but
+most especially at Ulysses, was instant with them that they
+should prevail with the noble son of Peleus.
+</para>
+
+<para>
+They went their way by the shore of the sounding sea, and prayed
+earnestly to earth-encircling Neptune that the high spirit of the
+son of Aeacus might incline favourably towards them. When they
+reached the ships and tents of the Myrmidons, they found Achilles
+playing on a lyre, fair, of cunning workmanship, and its
+cross-bar was of silver. It was part of the spoils which he had
+taken when he sacked the city of Eetion, and he was now diverting
+himself with it and singing the feats of heroes. He was alone
+with Patroclus, who sat opposite to him and said nothing, waiting
+till he should cease singing. Ulysses and Ajax now came in--
+Ulysses leading the way--and stood before him. Achilles sprang
+from his seat with the lyre still in his hand, and Patroclus,
+when he saw the strangers, rose also. Achilles then greeted them
+saying, "All hail and welcome--you must come upon some great
+matter, you, who for all my anger are still dearest to me of the
+Achaeans."
+</para>
+
+<para>
+With this he led them forward, and bade them sit on seats covered
+with purple rugs; then he said to Patroclus who was close by him,
+"Son of Menoetius, set a larger bowl upon the table, mix less
+water with the wine, and give every man his cup, for these are
+very dear friends, who are now under my roof."
+</para>
+
+<para>
+Patroclus did as his comrade bade him; he set the chopping-block
+in front of the fire, and on it he laid the loin of a sheep, the
+loin also of a goat, and the chine of a fat hog. Automedon held
+the meat while Achilles chopped it; he then sliced the pieces and
+put them on spits while the son of Menoetius made the fire burn
+high. When the flame had died down, he spread the embers, laid
+the spits on top of them, lifting them up and setting them upon
+the spit-racks; and he sprinkled them with salt. When the meat
+was roasted, he set it on platters, and handed bread round the
+table in fair baskets, while Achilles dealt them their portions.
+Then Achilles took his seat facing Ulysses against the opposite
+wall, and bade his comrade Patroclus offer sacrifice to the gods;
+so he cast the offerings into the fire, and they laid their hands
+upon the good things that were before them. As soon as they had
+had enough to eat and drink, Ajax made a sign to Phoenix, and
+when he saw this, Ulysses filled his cup with wine and pledged
+Achilles.
+</para>
+
+<para>
+"Hail," said he, "Achilles, we have had no scant of good cheer,
+neither in the tent of Agamemnon, nor yet here; there has been
+plenty to eat and drink, but our thought turns upon no such
+matter. Sir, we are in the face of great disaster, and without
+your help know not whether we shall save our fleet or lose it.
+The Trojans and their allies have camped hard by our ships and by
+the wall; they have lit watchfires throughout their host and deem
+that nothing can now prevent them from falling on our fleet.
+Jove, moreover, has sent his lightnings on their right; Hector,
+in all his glory, rages like a maniac; confident that Jove is
+with him he fears neither god nor man, but is gone raving mad,
+and prays for the approach of day. He vows that he will hew the
+high sterns of our ships in pieces, set fire to their hulls, and
+make havoc of the Achaeans while they are dazed and smothered in
+smoke; I much fear that heaven will make good his boasting, and
+it will prove our lot to perish at Troy far from our home in
+Argos. Up, then, and late though it be, save the sons of the
+Achaeans who faint before the fury of the Trojans. You will
+repent bitterly hereafter if you do not, for when the harm is
+done there will be no curing it; consider ere it be too late, and
+save the Danaans from destruction.
+</para>
+
+<para>
+"My good friend, when your father Peleus sent you from Phthia to
+Agamemnon, did he not charge you saying, 'Son, Minerva and Juno
+will make you strong if they choose, but check your high temper,
+for the better part is in goodwill. Eschew vain quarrelling, and
+the Achaeans old and young will respect you more for doing so.'
+These were his words, but you have forgotten them. Even now,
+however, be appeased, and put away your anger from you. Agamemnon
+will make you great amends if you will forgive him; listen, and I
+will tell you what he has said in his tent that he will give you.
+He will give you seven tripods that have never yet been on the
+fire, and ten talents of gold; twenty iron cauldrons, and twelve
+strong horses that have won races and carried off prizes. Rich
+indeed both in land and gold is he who has as many prizes as
+these horses have won for Agamemnon. Moreover he will give you
+seven excellent workwomen, Lesbians, whom he chose for himself,
+when you took Lesbos--all of surpassing beauty. He will give you
+these, and with them her whom he erewhile took from you, the
+daughter of Briseus, and he will swear a great oath, he has never
+gone up into her couch nor been with her after the manner of men
+and women. All these things will he give you now down, and if
+hereafter the gods vouchsafe him to sack the city of Priam, you
+can come when we Achaeans are dividing the spoil, and load your
+ship with gold and bronze to your liking. You can take twenty
+Trojan women, the loveliest after Helen herself. Then, when we
+reach Achaean Argos, wealthiest of all lands, you shall be his
+son-in-law, and he will show you like honour with his own dear
+son Orestes, who is being nurtured in all abundance. Agamemnon
+has three daughters, Chrysothemis, Laodice, and Iphianassa; you
+may take the one of your choice, freely and without gifts of
+wooing, to the house of Peleus; he will add such dower to boot as
+no man ever yet gave his daughter, and will give you seven
+well-established cities, Cardamyle, Enope, and Hire where there
+is grass; holy Pheras and the rich meadows of Anthea; Aepea also,
+and the vine-clad slopes of Pedasus, all near the sea, and on the
+borders of sandy Pylos. The men that dwell there are rich in
+cattle and sheep; they will honour you with gifts as though were
+a god, and be obedient to your comfortable ordinances. All this
+will he do if you will now forgo your anger. Moreover, though you
+hate both him and his gifts with all your heart, yet pity the
+rest of the Achaeans who are being harassed in all their host;
+they will honour you as a god, and you will earn great glory at
+their hands. You might even kill Hector; he will come within your
+reach, for he is infatuated, and declares that not a Danaan whom
+the ships have brought can hold his own against him."
+</para>
+
+<para>
+Achilles answered, "Ulysses, noble son of Laertes, I should give
+you formal notice plainly and in all fixity of purpose that there
+be no more of this cajoling, from whatsoever quarter it may come.
+Him do I hate even as the gates of hell who says one thing while
+he hides another in his heart; therefore I will say what I mean.
+I will be appeased neither by Agamemnon son of Atreus nor by any
+other of the Danaans, for I see that I have no thanks for all my
+fighting. He that fights fares no better than he that does not;
+coward and hero are held in equal honour, and death deals like
+measure to him who works and him who is idle. I have taken
+nothing by all my hardships--with my life ever in my hand; as a
+bird when she has found a morsel takes it to her nestlings, and
+herself fares hardly, even so many a long night have I been
+wakeful, and many a bloody battle have I waged by day against
+those who were fighting for their women. With my ships I have
+taken twelve cities, and eleven round about Troy have I stormed
+with my men by land; I took great store of wealth from every one
+of them, but I gave all up to Agamemnon son of Atreus. He stayed
+where he was by his ships, yet of what came to him he gave
+little, and kept much himself.
+</para>
+
+<para>
+"Nevertheless he did distribute some meeds of honour among the
+chieftains and kings, and these have them still; from me alone of
+the Achaeans did he take the woman in whom I delighted--let him
+keep her and sleep with her. Why, pray, must the Argives needs
+fight the Trojans? What made the son of Atreus gather the host
+and bring them? Was it not for the sake of Helen? Are the sons of
+Atreus the only men in the world who love their wives? Any man of
+common right feeling will love and cherish her who is his own, as
+I this woman, with my whole heart, though she was but a fruitling
+of my spear. Agamemnon has taken her from me; he has played me
+false; I know him; let him tempt me no further, for he shall not
+move me. Let him look to you, Ulysses, and to the other princes
+to save his ships from burning. He has done much without me
+already. He has built a wall; he has dug a trench deep and wide
+all round it, and he has planted it within with stakes; but even
+so he stays not the murderous might of Hector. So long as I
+fought the Achaeans Hector suffered not the battle range far from
+the city walls; he would come to the Scaean gates and to the oak
+tree, but no further. Once he stayed to meet me and hardly did he
+escape my onset: now, however, since I am in no mood to fight
+him, I will to-morrow offer sacrifice to Jove and to all the
+gods; I will draw my ships into the water and then victual them
+duly; to-morrow morning, if you care to look, you will see my
+ships on the Hellespont, and my men rowing out to sea with might
+and main. If great Neptune vouchsafes me a fair passage, in three
+days I shall be in Phthia. I have much there that I left behind
+me when I came here to my sorrow, and I shall bring back still
+further store of gold, of red copper, of fair women, and of iron,
+my share of the spoils that we have taken; but one prize, he who
+gave has insolently taken away. Tell him all as I now bid you,
+and tell him in public that the Achaeans may hate him and beware
+of him should he think that he can yet dupe others for his
+effrontery never fails him.
+</para>
+
+<para>
+"As for me, hound that he is, he dares not look me in the face.
+I will take no counsel with him, and will undertake nothing in
+common with him. He has wronged me and deceived me enough, he
+shall not cozen me further; let him go his own way, for Jove has
+robbed him of his reason. I loathe his presents, and for himself
+care not one straw. He may offer me ten or even twenty times what
+he has now done, nay--not though it be all that he has in the
+world, both now or ever shall have; he may promise me the wealth
+of Orchomenus or of Egyptian Thebes, which is the richest city in
+the whole world, for it has a hundred gates through each of which
+two hundred men may drive at once with their chariots and horses;
+he may offer me gifts as the sands of the sea or the dust of the
+plain in multitude, but even so he shall not move me till I have
+been revenged in full for the bitter wrong he has done me. I will
+not marry his daughter; she may be fair as Venus, and skilful as
+Minerva, but I will have none of her: let another take her, who
+may be a good match for her and who rules a larger kingdom. If
+the gods spare me to return home, Peleus will find me a wife;
+there are Achaean women in Hellas and Phthia, daughters of kings
+that have cities under them; of these I can take whom I will and
+marry her. Many a time was I minded when at home in Phthia to woo
+and wed a woman who would make me a suitable wife, and to enjoy
+the riches of my old father Peleus. My life is more to me than
+all the wealth of Ilius while it was yet at peace before the
+Achaeans went there, or than all the treasure that lies on the
+stone floor of Apollo's temple beneath the cliffs of Pytho.
+Cattle and sheep are to be had for harrying, and a man buy both
+tripods and horses if he wants them, but when his life has once
+left him it can neither be bought nor harried back again.
+</para>
+
+<para>
+"My mother Thetis tells me that there are two ways in which I may
+meet my end. If I stay here and fight, I shall not return alive
+but my name will live for ever: whereas if I go home my name will
+die, but it will be long ere death shall take me. To the rest of
+you, then, I say, 'Go home, for you will not take Ilius.' Jove
+has held his hand over her to protect her, and her people have
+taken heart. Go, therefore, as in duty bound, and tell the
+princes of the Achaeans the message that I have sent them; tell
+them to find some other plan for the saving of their ships and
+people, for so long as my displeasure lasts the one that they
+have now hit upon may not be. As for Phoenix, let him sleep here
+that he may sail with me in the morning if he so will. But I
+will not take him by force."
+</para>
+
+<para>
+They all held their peace, dismayed at the sternness with which
+he had denied them, till presently the old knight Phoenix in his
+great fear for the ships of the Achaeans, burst into tears and
+said, "Noble Achilles, if you are now minded to return, and in
+the fierceness of your anger will do nothing to save the ships
+from burning, how, my son, can I remain here without you? Your
+father Peleus bade me go with you when he sent you as a mere lad
+from Phthia to Agamemnon. You knew nothing neither of war nor of
+the arts whereby men make their mark in council, and he sent me
+with you to train you in all excellence of speech and action.
+Therefore, my son, I will not stay here without you--no, not
+though heaven itself vouchsafe to strip my years from off me, and
+make me young as I was when I first left Hellas the land of fair
+women. I was then flying the anger of father Amyntor, son of
+Ormenus, who was furious with me in the matter of his concubine,
+of whom he was enamoured to the wronging of his wife my mother.
+My mother, therefore, prayed me without ceasing to lie with the
+woman myself, that so she hate my father, and in the course of
+time I yielded. But my father soon came to know, and cursed me
+bitterly, calling the dread Erinyes to witness. He prayed that no
+son of mine might ever sit upon knees--and the gods, Jove of the
+world below and awful Proserpine, fulfilled his curse. I took
+counsel to kill him, but some god stayed my rashness and bade me
+think on men's evil tongues and how I should be branded as the
+murderer of my father; nevertheless I could not bear to stay in
+my father's house with him so bitter a against me. My cousins and
+clansmen came about me, and pressed me sorely to remain; many a
+sheep and many an ox did they slaughter, and many a fat hog did
+they set down to roast before the fire; many a jar, too, did they
+broach of my father's wine. Nine whole nights did they set a
+guard over me taking it in turns to watch, and they kept a fire
+always burning, both in the cloister of the outer court and in
+the inner court at the doors of the room wherein I lay; but when
+the darkness of the tenth night came, I broke through the closed
+doors of my room, and climbed the wall of the outer court after
+passing quickly and unperceived through the men on guard and the
+women servants. I then fled through Hellas till I came to fertile
+Phthia, mother of sheep, and to King Peleus, who made me welcome
+and treated me as a father treats an only son who will be heir to
+all his wealth. He made me rich and set me over much people,
+establishing me on the borders of Phthia where I was chief ruler
+over the Dolopians.
+</para>
+
+<para>
+"It was I, Achilles, who had the making of you; I loved you with
+all my heart: for you would eat neither at home nor when you had
+gone out elsewhere, till I had first set you upon my knees, cut
+up the dainty morsel that you were to eat, and held the wine-cup
+to your lips. Many a time have you slobbered your wine in baby
+helplessness over my shirt; I had infinite trouble with you, but
+I knew that heaven had vouchsafed me no offspring of my own, and
+I made a son of you, Achilles, that in my hour of need you might
+protect me. Now, therefore, I say battle with your pride and beat
+it; cherish not your anger for ever; the might and majesty of
+heaven are more than ours, but even heaven may be appeased; and
+if a man has sinned he prays the gods, and reconciles them to
+himself by his piteous cries and by frankincense, with
+drink-offerings and the savour of burnt sacrifice. For prayers
+are as daughters to great Jove; halt, wrinkled, with eyes
+askance, they follow in the footsteps of sin, who, being fierce
+and fleet of foot, leaves them far behind him, and ever baneful
+to mankind outstrips them even to the ends of the world; but
+nevertheless the prayers come hobbling and healing after. If a
+man has pity upon these daughters of Jove when they draw near
+him, they will bless him and hear him too when he is praying; but
+if he deny them and will not listen to them, they go to Jove the
+son of Saturn and pray that he may presently fall into sin--to
+his ruing bitterly hereafter. Therefore, Achilles, give these
+daughters of Jove due reverence, and bow before them as all good
+men will bow. Were not the son of Atreus offering you gifts and
+promising others later--if he were still furious and implacable--
+I am not he that would bid you throw off your anger and help the
+Achaeans, no matter how great their need; but he is giving much
+now, and more hereafter; he has sent his captains to urge his
+suit, and has chosen those who of all the Argives are most
+acceptable to you; make not then their words and their coming to
+be of none effect. Your anger has been righteous so far. We have
+heard in song how heroes of old time quarrelled when they were
+roused to fury, but still they could be won by gifts, and fair
+words could soothe them.
+</para>
+
+<para>
+"I have an old story in my mind--a very old one--but you are all
+friends and I will tell it. The Curetes and the Aetolians were
+fighting and killing one another round Calydon--the Aetolians
+defending the city and the Curetes trying to destroy it. For
+Diana of the golden throne was angry and did them hurt because
+Oeneus had not offered her his harvest first-fruits. The other
+gods had all been feasted with hecatombs, but to the daughter of
+great Jove alone he had made no sacrifice. He had forgotten her,
+or somehow or other it had escaped him, and this was a grievous
+sin. Thereon the archer goddess in her displeasure sent a
+prodigious creature against him--a savage wild boar with great
+white tusks that did much harm to his orchard lands, uprooting
+apple-trees in full bloom and throwing them to the ground. But
+Meleager son of Oeneus got huntsmen and hounds from many cities
+and killed it--for it was so monstrous that not a few were
+needed, and many a man did it stretch upon his funeral pyre. On
+this the goddess set the Curetes and the Aetolians fighting
+furiously about the head and skin of the boar.
+</para>
+
+<para>
+"So long as Meleager was in the field things went badly with the
+Curetes, and for all their numbers they could not hold their
+ground under the city walls; but in the course of time Meleager
+was angered as even a wise man will sometimes be. He was incensed
+with his mother Althaea, and therefore stayed at home with his
+wedded wife fair Cleopatra, who was daughter of Marpessa daughter
+of Euenus, and of Ides the man then living. He it was who took
+his bow and faced King Apollo himself for fair Marpessa's sake;
+her father and mother then named her Alcyone, because her mother
+had mourned with the plaintive strains of the halcyon-bird when
+Phoebus Apollo had carried her off. Meleager, then, stayed at
+home with Cleopatra, nursing the anger which he felt by reason of
+his mother's curses. His mother, grieving for the death of her
+brother, prayed the gods, and beat the earth with her hands,
+calling upon Hades and on awful Proserpine; she went down upon
+her knees and her bosom was wet with tears as she prayed that
+they would kill her son--and Erinys that walks in darkness and
+knows no ruth heard her from Erebus.
+</para>
+
+<para>
+"Then was heard the din of battle about the gates of Calydon, and
+the dull thump of the battering against their walls. Thereon the
+elders of the Aetolians besought Meleager; they sent the chiefest
+of their priests, and begged him to come out and help them,
+promising him a great reward. They bade him choose fifty
+plough-gates, the most fertile in the plain of Calydon, the
+one-half vineyard and the other open plough-land. The old warrior
+Oeneus implored him, standing at the threshold of his room and
+beating the doors in supplication. His sisters and his mother
+herself besought him sore, but he the more refused them; those of
+his comrades who were nearest and dearest to him also prayed him,
+but they could not move him till the foe was battering at the
+very doors of his chamber, and the Curetes had scaled the walls
+and were setting fire to the city. Then at last his sorrowing
+wife detailed the horrors that befall those whose city is taken;
+she reminded him how the men are slain, and the city is given
+over to the flames, while the women and children are carried into
+captivity; when he heard all this, his heart was touched, and he
+donned his armour to go forth. Thus of his own inward motion he
+saved the city of the Aetolians; but they now gave him nothing of
+those rich rewards that they had offered earlier, and though he
+saved the city he took nothing by it. Be not then, my son, thus
+minded; let not heaven lure you into any such course. When the
+ships are burning it will be a harder matter to save them. Take
+the gifts, and go, for the Achaeans will then honour you as a
+god; whereas if you fight without taking them, you may beat the
+battle back, but you will not be held in like honour."
+</para>
+
+<para>
+And Achilles answered, "Phoenix, old friend and father, I have no
+need of such honour. I have honour from Jove himself, which will
+abide with me at my ships while I have breath in my body, and my
+limbs are strong. I say further--and lay my saying to your
+heart--vex me no more with this weeping and lamentation, all in
+the cause of the son of Atreus. Love him so well, and you may
+lose the love I bear you. You ought to help me rather in
+troubling those that trouble me; be king as much as I am, and
+share like honour with myself; the others shall take my answer;
+stay here yourself and sleep comfortably in your bed; at daybreak
+we will consider whether to remain or go."
+</para>
+
+<para>
+On this she nodded quietly to Patroclus as a sign that he was to
+prepare a bed for Phoenix, and that the others should take their
+leave. Ajax son of Telamon then said, "Ulysses, noble son of
+Laertes, let us be gone, for I see that our journey is vain. We
+must now take our answer, unwelcome though it be, to the Danaans
+who are waiting to receive it. Achilles is savage and
+remorseless; he is cruel, and cares nothing for the love his
+comrades lavished upon him more than on all the others. He is
+implacable--and yet if a man's brother or son has been slain he
+will accept a fine by way of amends from him that killed him, and
+the wrong-doer having paid in full remains in peace among his own
+people; but as for you, Achilles, the gods have put a wicked
+unforgiving spirit in your heart, and this, all about one single
+girl, whereas we now offer you the seven best we have, and much
+else into the bargain. Be then of a more gracious mind, respect
+the hospitality of your own roof. We are with you as messengers
+from the host of the Danaans, and would fain he held nearest and
+dearest to yourself of all the Achaeans."
+</para>
+
+<para>
+"Ajax," replied Achilles, "noble son of Telamon, you have spoken
+much to my liking, but my blood boils when I think it all over,
+and remember how the son of Atreus treated me with contumely as
+though I were some vile tramp, and that too in the presence of
+the Argives. Go, then, and deliver your message; say that I will
+have no concern with fighting till Hector, son of noble Priam,
+reaches the tents of the Myrmidons in his murderous course, and
+flings fire upon their ships. For all his lust of battle, I take
+it he will be held in check when he is at my own tent and ship."
+</para>
+
+<para>
+On this they took every man his double cup, made their
+drink-offerings, and went back to the ships, Ulysses leading the
+way. But Patroclus told his men and the maid-servants to make
+ready a comfortable bed for Phoenix; they therefore did so with
+sheepskins, a rug, and a sheet of fine linen. The old man then
+laid himself down and waited till morning came. But Achilles
+slept in an inner room, and beside him the daughter of Phorbas
+lovely Diomede, whom he had carried off from Lesbos. Patroclus
+lay on the other side of the room, and with him fair Iphis whom
+Achilles had given him when he took Scyros the city of Enyeus.
+</para>
+
+<para>
+When the envoys reached the tents of the son of Atreus, the
+Achaeans rose, pledged them in cups of gold, and began to
+question them. King Agamemnon was the first to do so. "Tell me,
+Ulysses," said he, "will he save the ships from burning, or did
+be refuse, and is he still furious?"
+</para>
+
+<para>
+Ulysses answered, "Most noble son of Atreus, king of men,
+Agamemnon, Achilles will not be calmed, but is more fiercely
+angry than ever, and spurns both you and your gifts. He bids you
+take counsel with the Achaeans to save the ships and host as you
+best may; as for himself, he said that at daybreak he should draw
+his ships into the water. He said further that he should advise
+every one to sail home likewise, for that you will not reach the
+goal of Ilius. 'Jove,' he said, 'has laid his hand over the city
+to protect it, and the people have taken heart.' This is what he
+said, and the others who were with me can tell you the same
+story--Ajax and the two heralds, men, both of them, who may be
+trusted. The old man Phoenix stayed where he was to sleep, for so
+Achilles would have it, that he might go home with him in the
+morning if he so would; but he will not take him by force."
+</para>
+
+<para>
+They all held their peace, sitting for a long time silent and
+dejected, by reason of the sternness with which Achilles had
+refused them, till presently Diomed said, "Most noble son of
+Atreus, king of men, Agamemnon, you ought not to have sued the
+son of Peleus nor offered him gifts. He is proud enough as it is,
+and you have encouraged him in his pride still further. Let him
+stay or go as he will. He will fight later when he is in the
+humour, and heaven puts it in his mind to do so. Now, therefore,
+let us all do as I say; we have eaten and drunk our fill, let us
+then take our rest, for in rest there is both strength and stay.
+But when fair rosy-fingered morn appears, forthwith bring out
+your host and your horsemen in front of the ships, urging them
+on, and yourself fighting among the foremost."
+</para>
+
+<para>
+Thus he spoke, and the other chieftains approved his words. They
+then made their drink-offerings and went every man to his own
+tent, where they laid down to rest and enjoyed the boon of sleep.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK X</title>
+
+<para>
+NOW the other princes of the Achaeans slept soundly the whole
+night through, but Agamemnon son of Atreus was troubled, so that
+he could get no rest. As when fair Juno's lord flashes his
+lightning in token of great rain or hail or snow when the
+snow-flakes whiten the ground, or again as a sign that he will
+open the wide jaws of hungry war, even so did Agamemnon heave
+many a heavy sigh, for his soul trembled within him. When he
+looked upon the plain of Troy he marvelled at the many watchfires
+burning in front of Ilius, and at the sound of pipes and flutes
+and of the hum of men, but when presently he turned towards the
+ships and hosts of the Achaeans, he tore his hair by handfuls
+before Jove on high, and groaned aloud for the very disquietness
+of his soul. In the end he deemed it best to go at once to Nestor
+son of Neleus, and see if between them they could find any way of
+the Achaeans from destruction. He therefore rose, put on his
+shirt, bound his sandals about his comely feet, flung the skin of
+a huge tawny lion over his shoulders--a skin that reached his
+feet--and took his spear in his hand.
+</para>
+
+<para>
+Neither could Menelaus sleep, for he, too, boded ill for the
+Argives who for his sake had sailed from far over the seas to
+fight the Trojans. He covered his broad back with the skin of a
+spotted panther, put a casque of bronze upon his head, and took
+his spear in his brawny hand. Then he went to rouse his brother,
+who was by far the most powerful of the Achaeans, and was
+honoured by the people as though he were a god. He found him by
+the stern of his ship already putting his goodly array about his
+shoulders, and right glad was he that his brother had come.
+</para>
+
+<para>
+Menelaus spoke first. "Why," said he, "my dear brother, are you
+thus arming? Are you going to send any of our comrades to exploit
+the Trojans? I greatly fear that no one will do you this service,
+and spy upon the enemy alone in the dead of night. It will be a
+deed of great daring."
+</para>
+
+<para>
+And King Agamemnon answered, "Menelaus, we both of us need shrewd
+counsel to save the Argives and our ships, for Jove has changed
+his mind, and inclines towards Hector's sacrifices rather than
+ours. I never saw nor heard tell of any man as having wrought
+such ruin in one day as Hector has now wrought against the sons
+of the Achaeans--and that too of his own unaided self, for he is
+son neither to god nor goddess. The Argives will rue it long and
+deeply. Run, therefore, with all speed by the line of the ships,
+and call Ajax and Idomeneus. Meanwhile I will go to Nestor, and
+bid him rise and go about among the companies of our sentinels to
+give them their instructions; they will listen to him sooner than
+to any man, for his own son, and Meriones brother in arms to
+Idomeneus, are captains over them. It was to them more
+particularly that we gave this charge."
+</para>
+
+<para>
+Menelaus replied, "How do I take your meaning? Am I to stay with
+them and wait your coming, or shall I return here as soon as I
+have given your orders?" "Wait," answered King Agamemnon, "for
+there are so many paths about the camp that we might miss one
+another. Call every man on your way, and bid him be stirring;
+name him by his lineage and by his father's name, give each all
+titular observance, and stand not too much upon your own dignity;
+we must take our full share of toil, for at our birth Jove laid
+this heavy burden upon us."
+</para>
+
+<para>
+With these instructions he sent his brother on his way, and went
+on to Nestor shepherd of his people. He found him sleeping in his
+tent hard by his own ship; his goodly armour lay beside him--his
+shield, his two spears and his helmet; beside him also lay the
+gleaming girdle with which the old man girded himself when he
+armed to lead his people into battle--for his age stayed him not.
+He raised himself on his elbow and looked up at Agamemnon. "Who
+is it," said he, "that goes thus about the host and the ships
+alone and in the dead of night, when men are sleeping? Are you
+looking for one of your mules or for some comrade? Do not stand
+there and say nothing, but speak. What is your business?"
+</para>
+
+<para>
+And Agamemnon answered, "Nestor, son of Neleus, honour to the
+Achaean name, it is I, Agamemnon son of Atreus, on whom Jove has
+laid labour and sorrow so long as there is breath in my body and
+my limbs carry me. I am thus abroad because sleep sits not upon
+my eyelids, but my heart is big with war and with the jeopardy of
+the Achaeans. I am in great fear for the Danaans. I am at sea,
+and without sure counsel; my heart beats as though it would leap
+out of my body, and my limbs fail me. If then you can do
+anything--for you too cannot sleep--let us go the round of the
+watch, and see whether they are drowsy with toil and sleeping to
+the neglect of their duty. The enemy is encamped hard and we know
+not but he may attack us by night."
+</para>
+
+<para>
+Nestor replied, "Most noble son of Atreus, king of men,
+Agamemnon, Jove will not do all for Hector that Hector thinks he
+will; he will have troubles yet in plenty if Achilles will lay
+aside his anger. I will go with you, and we will rouse others,
+either the son of Tydeus, or Ulysses, or fleet Ajax and the
+valiant son of Phyleus. Some one had also better go and call Ajax
+and King Idomeneus, for their ships are not near at hand but the
+farthest of all. I cannot however refrain from blaming Menelaus,
+much as I love him and respect him--and I will say so plainly,
+even at the risk of offending you--for sleeping and leaving all
+this trouble to yourself. He ought to be going about imploring
+aid from all the princes of the Achaeans, for we are in extreme
+danger."
+</para>
+
+<para>
+And Agamemnon answered, "Sir, you may sometimes blame him justly,
+for he is often remiss and unwilling to exert himself--not
+indeed from sloth, nor yet heedlessness, but because he looks to
+me and expects me to take the lead. On this occasion, however, he
+was awake before I was, and came to me of his own accord. I have
+already sent him to call the very men whom you have named. And
+now let us be going. We shall find them with the watch outside
+the gates, for it was there I said that we would meet them."
+</para>
+
+<para>
+"In that case," answered Nestor, "the Argives will not blame him
+nor disobey his orders when he urges them to fight or gives them
+instructions."
+</para>
+
+<para>
+With this he put on his shirt, and bound his sandals about his
+comely feet. He buckled on his purple coat, of two thicknesses,
+large, and of a rough shaggy texture, grasped his redoubtable
+bronze-shod spear, and wended his way along the line of the
+Achaean ships. First he called loudly to Ulysses peer of gods in
+counsel and woke him, for he was soon roused by the sound of the
+battle-cry. He came outside his tent and said, "Why do you go
+thus alone about the host, and along the line of the ships in the
+stillness of the night? What is it that you find so urgent?" And
+Nestor knight of Gerene answered, "Ulysses, noble son of Laertes,
+take it not amiss, for the Achaeans are in great straits. Come
+with me and let us wake some other, who may advise well with us
+whether we shall fight or fly."
+</para>
+
+<para>
+On this Ulysses went at once into his tent, put his shield about
+his shoulders and came out with them. First they went to Diomed
+son of Tydeus, and found him outside his tent clad in his armour
+with his comrades sleeping round him and using their shields as
+pillows; as for their spears, they stood upright on the spikes of
+their butts that were driven into the ground, and the burnished
+bronze flashed afar like the lightning of father Jove. The hero
+was sleeping upon the skin of an ox, with a piece of fine carpet
+under his head; Nestor went up to him and stirred him with his
+heel to rouse him, upbraiding him and urging him to bestir
+himself. "Wake up," he exclaimed, "son of Tydeus. How can you
+sleep on in this way? Can you not see that the Trojans are
+encamped on the brow of the plain hard by our ships, with but a
+little space between us and them?"
+</para>
+
+<para>
+On these words Diomed leaped up instantly and said, "Old man,
+your heart is of iron; you rest not one moment from your labours.
+Are there no younger men among the Achaeans who could go about to
+rouse the princes? There is no tiring you."
+</para>
+
+<para>
+And Nestor knight of Gerene made answer, "My son, all that you
+have said is true. I have good sons, and also much people who
+might call the chieftains, but the Achaeans are in the gravest
+danger; life and death are balanced as it were on the edge of a
+razor. Go then, for you are younger than I, and of your courtesy
+rouse Ajax and the fleet son of Phyleus."
+</para>
+
+<para>
+Diomed threw the skin of a great tawny lion about his shoulders--
+a skin that reached his feet--and grasped his spear. When he had
+roused the heroes, he brought them back with him; they then went
+the round of those who were on guard, and found the captains not
+sleeping at their posts but wakeful and sitting with their arms
+about them. As sheep dogs that watch their flocks when they are
+yarded, and hear a wild beast coming through the mountain forest
+towards them--forthwith there is a hue and cry of dogs and men,
+and slumber is broken--even so was sleep chased from the eyes of
+the Achaeans as they kept the watches of the wicked night, for
+they turned constantly towards the plain whenever they heard any
+stir among the Trojans. The old man was glad bade them be of good
+cheer. "Watch on, my children," said he, "and let not sleep get
+hold upon you, lest our enemies triumph over us."
+</para>
+
+<para>
+With this he passed the trench, and with him the other chiefs of
+the Achaeans who had been called to the council. Meriones and the
+brave son of Nestor went also, for the princes bade them. When
+they were beyond the trench that was dug round the wall they held
+their meeting on the open ground where there was a space clear of
+corpses, for it was here that when night fell Hector had turned
+back from his onslaught on the Argives. They sat down, therefore,
+and held debate with one another.
+</para>
+
+<para>
+Nestor spoke first. "My friends," said he, "is there any man bold
+enough to venture the Trojans, and cut off some straggler, or us
+news of what the enemy mean to do whether they will stay here by
+the ships away from the city, or whether, now that they have
+worsted the Achaeans, they will retire within their walls. If he
+could learn all this and come back safely here, his fame would be
+high as heaven in the mouths of all men, and he would be rewarded
+richly; for the chiefs from all our ships would each of them give
+him a black ewe with her lamb--which is a present of surpassing
+value--and he would be asked as a guest to all feasts and
+clan-gatherings."
+</para>
+
+<para>
+They all held their peace, but Diomed of the loud war-cry spoke
+saying, "Nestor, gladly will I visit the host of the Trojans over
+against us, but if another will go with me I shall do so in
+greater confidence and comfort. When two men are together, one of
+them may see some opportunity which the other has not caught
+sight of; if a man is alone he is less full of resource, and his
+wit is weaker."
+</para>
+
+<para>
+On this several offered to go with Diomed. The two Ajaxes,
+servants of Mars, Meriones, and the son of Nestor all wanted to
+go, so did Menelaus son of Atreus; Ulysses also wished to go
+among the host of the Trojans, for he was ever full of daring,
+and thereon Agamemnon king of men spoke thus: "Diomed," said he,
+"son of Tydeus, man after my own heart, choose your comrade for
+yourself--take the best man of those that have offered, for many
+would now go with you. Do not through delicacy reject the better
+man, and take the worst out of respect for his lineage, because
+he is of more royal blood."
+</para>
+
+<para>
+He said this because he feared for Menelaus. Diomed answered, "If
+you bid me take the man of my own choice, how in that case can I
+fail to think of Ulysses, than whom there is no man more eager to
+face all kinds of danger--and Pallas Minerva loves him well? If
+he were to go with me we should pass safely through fire itself,
+for he is quick to see and understand."
+</para>
+
+<para>
+"Son of Tydeus," replied Ulysses, "say neither good nor ill about
+me, for you are among Argives who know me well. Let us be going,
+for the night wanes and dawn is at hand. The stars have gone
+forward, two-thirds of the night are already spent, and the third
+is alone left us."
+</para>
+
+<para>
+They then put on their armour. Brave Thrasymedes provided the son
+of Tydeus with a sword and a shield (for he had left his own at
+his ship) and on his head he set a helmet of bull's hide without
+either peak or crest; it is called a skull-cap and is a common
+headgear. Meriones found a bow and quiver for Ulysses, and on his
+head he set a leathern helmet that was lined with a strong
+plaiting of leathern thongs, while on the outside it was thickly
+studded with boar's teeth, well and skilfully set into it; next
+the head there was an inner lining of felt. This helmet had been
+stolen by Autolycus out of Eleon when he broke into the house of
+Amyntor son of Ormenus. He gave it to Amphidamas of Cythera to
+take to Scandea, and Amphidamas gave it as a guest-gift to Molus,
+who gave it to his son Meriones; and now it was set upon the head
+of Ulysses.
+</para>
+
+<para>
+When the pair had armed, they set out, and left the other
+chieftains behind them. Pallas Minerva sent them a heron by the
+wayside upon their right hands; they could not see it for the
+darkness, but they heard its cry. Ulysses was glad when he heard
+it and prayed to Minerva: "Hear me," he cried, "daughter of
+aegis-bearing Jove, you who spy out all my ways and who are with
+me in all my hardships; befriend me in this mine hour, and grant
+that we may return to the ships covered with glory after having
+achieved some mighty exploit that shall bring sorrow to the
+Trojans."
+</para>
+
+<para>
+Then Diomed of the loud war-cry also prayed: "Hear me too," said
+he, "daughter of Jove, unweariable; be with me even as you were
+with my noble father Tydeus when he went to Thebes as envoy sent
+by the Achaeans. He left the Achaeans by the banks of the river
+Aesopus, and went to the city bearing a message of peace to the
+Cadmeians; on his return thence, with your help, goddess, he did
+great deeds of daring, for you were his ready helper. Even so
+guide me and guard me now, and in return I will offer you in
+sacrifice a broad-browed heifer of a year old, unbroken, and
+never yet brought by man under the yoke. I will gild her horns
+and will offer her up to you in sacrifice."
+</para>
+
+<para>
+Thus they prayed, and Pallas Minerva heard their prayer. When
+they had done praying to the daughter of great Jove, they went
+their way like two lions prowling by night amid the armour and
+blood-stained bodies of them that had fallen.
+</para>
+
+<para>
+Neither again did Hector let the Trojans sleep; for he too called
+the princes and councillors of the Trojans that he might set his
+counsel before them. "Is there one," said he, "who for a great
+reward will do me the service of which I will tell you? He shall
+be well paid if he will. I will give him a chariot and a couple
+of horses, the fleetest that can be found at the ships of the
+Achaeans, if he will dare this thing; and he will win infinite
+honour to boot; he must go to the ships and find out whether they
+are still guarded as heretofore, or whether now that we have
+beaten them the Achaeans design to fly, and through sheer
+exhaustion are neglecting to keep their watches."
+</para>
+
+<para>
+They all held their peace; but there was among the Trojans a
+certain man named Dolon, son of Eumedes, the famous herald--a man
+rich in gold and bronze. He was ill-favoured, but a good runner,
+and was an only son among five sisters. He it was that now
+addressed the Trojans. "I, Hector," said he, "Will to the ships
+and will exploit them. But first hold up your sceptre and swear
+that you will give me the chariot, bedight with bronze, and the
+horses that now carry the noble son of Peleus. I will make you a
+good scout, and will not fail you. I will go through the host
+from one end to the other till I come to the ship of Agamemnon,
+where I take it the princes of the Achaeans are now consulting
+whether they shall fight or fly."
+</para>
+
+<para>
+When he had done speaking Hector held up his sceptre, and swore
+him his oath saying, "May Jove the thundering husband of Juno
+bear witness that no other Trojan but yourself shall mount those
+steeds, and that you shall have your will with them for ever."
+</para>
+
+<para>
+The oath he swore was bootless, but it made Dolon more keen on
+going. He hung his bow over his shoulder, and as an overall he
+wore the skin of a grey wolf, while on his head he set a cap of
+ferret skin. Then he took a pointed javelin, and left the camp
+for the ships, but he was not to return with any news for Hector.
+When he had left the horses and the troops behind him, he made
+all speed on his way, but Ulysses perceived his coming and said
+to Diomed, "Diomed, here is some one from the camp; I am not sure
+whether he is a spy, or whether it is some thief who would
+plunder the bodies of the dead; let him get a little past us, we
+can then spring upon him and take him. If, however, he is too
+quick for us, go after him with your spear and hem him in towards
+the ships away from the Trojan camp, to prevent his getting back
+to the town."
+</para>
+
+<para>
+With this they turned out of their way and lay down among the
+corpses. Dolon suspected nothing and soon passed them, but when
+he had got about as far as the distance by which a mule-plowed
+furrow exceeds one that has been ploughed by oxen (for mules can
+plow fallow land quicker than oxen) they ran after him, and when
+he heard their footsteps he stood still, for he made sure they
+were friends from the Trojan camp come by Hector's orders to bid
+him return; when, however, they were only a spear's cast, or less
+away form him, he saw that they were enemies as fast as his legs
+could take him. The others gave chase at once, and as a couple of
+well-trained hounds press forward after a doe or hare that runs
+screaming in front of them, even so did the son of Tydeus and
+Ulysses pursue Dolon and cut him off from his own people. But
+when he had fled so far towards the ships that he would soon have
+fallen in with the outposts, Minerva infused fresh strength into
+the son of Tydeus for fear some other of the Achaeans might have
+the glory of being first to hit him, and he might himself be only
+second; he therefore sprang forward with his spear and said,
+"Stand, or I shall throw my spear, and in that case I shall soon
+make an end of you."
+</para>
+
+<para>
+He threw as he spoke, but missed his aim on purpose. The dart
+flew over the man's right shoulder, and then stuck in the ground.
+He stood stock still, trembling and in great fear; his teeth
+chattered, and he turned pale with fear. The two came breathless
+up to him and seized his hands, whereon he began to weep and
+said, "Take me alive; I will ransom myself; we have great store
+of gold, bronze, and wrought iron, and from this my father will
+satisfy you with a very large ransom, should he hear of my being
+alive at the ships of the Achaeans."
+</para>
+
+<para>
+"Fear not," replied Ulysses, "let no thought of death be in your
+mind; but tell me, and tell me true, why are you thus going about
+alone in the dead of night away from your camp and towards the
+ships, while other men are sleeping? Is it to plunder the bodies
+of the slain, or did Hector send you to spy out what was going on
+at the ships? Or did you come here of your own mere notion?"
+</para>
+
+<para>
+Dolon answered, his limbs trembling beneath him: "Hector, with
+his vain flattering promises, lured me from my better judgement.
+He said he would give me the horses of the noble son of Peleus
+and his bronze-bedizened chariot; he bade me go through the
+darkness of the flying night, get close to the enemy, and find
+out whether the ships are still guarded as heretofore, or
+whether, now that we have beaten them, the Achaeans design to
+fly, and through sheer exhaustion are neglecting to keep their
+watches."
+</para>
+
+<para>
+Ulysses smiled at him and answered, "You had indeed set your
+heart upon a great reward, but the horses of the descendant of
+Aeacus are hardly to be kept in hand or driven by any other
+mortal man than Achilles himself, whose mother was an immortal.
+But tell me, and tell me true, where did you leave Hector when
+you started? Where lies his armour and his horses? How, too, are
+the watches and sleeping-ground of the Trojans ordered? What are
+their plans? Will they stay here by the ships and away from the
+city, or now that they have worsted the Achaeans, will they
+retire within their walls?"
+</para>
+
+<para>
+And Dolon answered, "I will tell you truly all. Hector and the
+other councillors are now holding conference by the monument of
+great Ilus, away from the general tumult; as for the guards about
+which you ask me, there is no chosen watch to keep guard over the
+host. The Trojans have their watchfires, for they are bound to
+have them; they, therefore, are awake and keep each other to
+their duty as sentinels; but the allies who have come from other
+places are asleep and leave it to the Trojans to keep guard, for
+their wives and children are not here."
+</para>
+
+<para>
+Ulysses then said, "Now tell me; are they sleeping among the
+Trojan troops, or do they lie apart? Explain this that I may
+understand it."
+</para>
+
+<para>
+"I will tell you truly all," replied Dolon. "To the seaward lie
+the Carians, the Paeonian bowmen, the Leleges, the Cauconians,
+and the noble Pelasgi. The Lysians and proud Mysians, with the
+Phrygians and Meonians, have their place on the side towards
+Thymbra; but why ask about an this? If you want to find your way
+into the host of the Trojans, there are the Thracians, who have
+lately come here and lie apart from the others at the far end of
+the camp; and they have Rhesus son of Eioneus for their king. His
+horses are the finest and strongest that I have ever seen, they
+are whiter than snow and fleeter than any wind that blows. His
+chariot is bedight with silver and gold, and he has brought his
+marvellous golden armour, of the rarest workmanship--too splendid
+for any mortal man to carry, and meet only for the gods. Now,
+therefore, take me to the ships or bind me securely here, until
+you come back and have proved my words whether they be false or
+true."
+</para>
+
+<para>
+Diomed looked sternly at him and answered, "Think not, Dolon, for
+all the good information you have given us, that you shall escape
+now you are in our hands, for if we ransom you or let you go, you
+will come some second time to the ships of the Achaeans either as
+a spy or as an open enemy, but if I kill you and an end of you,
+you will give no more trouble."
+</para>
+
+<para>
+On this Dolon would have caught him by the beard to beseech him
+further, but Diomed struck him in the middle of his neck with his
+sword and cut through both sinews so that his head fell rolling
+in the dust while he was yet speaking. They took the ferret-skin
+cap from his head, and also the wolf-skin, the bow, and his long
+spear. Ulysses hung them up aloft in honour of Minerva the
+goddess of plunder, and prayed saying, "Accept these, goddess,
+for we give them to you in preference to all the gods in Olympus:
+therefore speed us still further towards the horses and
+sleeping-ground of the Thracians."
+</para>
+
+<para>
+With these words he took the spoils and set them upon a tamarisk
+tree, and they marked the place by pulling up reeds and gathering
+boughs of tamarisk that they might not miss it as they came back
+through the' flying hours of darkness. The two then went onwards
+amid the fallen armour and the blood, and came presently to the
+company of Thracian soldiers, who were sleeping, tired out with
+their day's toil; their goodly armour was lying on the ground
+beside them all orderly in three rows, and each man had his yoke
+of horses beside him. Rhesus was sleeping in the middle, and hard
+by him his horses were made fast to the topmost rim of his
+chariot. Ulysses from some way off saw him and said, "This,
+Diomed, is the man, and these are the horses about which Dolon
+whom we killed told us. Do your very utmost; dally not about your
+armour, but loose the horses at once--or else kill the men
+yourself, while I see to the horses."
+</para>
+
+<para>
+Thereon Minerva put courage into the heart of Diomed, and he
+smote them right and left. They made a hideous groaning as they
+were being hacked about, and the earth was red with their blood.
+As a lion springs furiously upon a flock of sheep or goats when
+he finds without their shepherd, so did the son of Tydeus set
+upon the Thracian soldiers till he had killed twelve. As he
+killed them Ulysses came and drew them aside by their feet one by
+one, that the horses might go forward freely without being
+frightened as they passed over the dead bodies, for they were not
+yet used to them. When the son of Tydeus came to the king, he
+killed him too (which made thirteen), as he was breathing hard,
+for by the counsel of Minerva an evil dream, the seed of Oeneus,
+hovered that night over his head. Meanwhile Ulysses untied the
+horses, made them fast one to another and drove them off,
+striking them with his bow, for he had forgotten to take the whip
+from the chariot. Then he whistled as a sign to Diomed.
+</para>
+
+<para>
+But Diomed stayed where he was, thinking what other daring deed
+he might accomplish. He was doubting whether to take the chariot
+in which the king's armour was lying, and draw it out by the
+pole, or to lift the armour out and carry it off; or whether
+again, he should not kill some more Thracians. While he was thus
+hesitating Minerva came up to him and said, "Get back, Diomed, to
+the ships or you may be driven thither, should some other god
+rouse the Trojans."
+</para>
+
+<para>
+Diomed knew that it was the goddess, and at once sprang upon the
+horses. Ulysses beat them with his bow and they flew onward to
+the ships of the Achaeans.
+</para>
+
+<para>
+But Apollo kept no blind look-out when he saw Minerva with the
+son of Tydeus. He was angry with her, and coming to the host of
+the Trojans he roused Hippocoon, a counsellor of the Thracians
+and a noble kinsman of Rhesus. He started up out of his sleep and
+saw that the horses were no longer in their place, and that the
+men were gasping in their death-agony; on this he groaned aloud,
+and called upon his friend by name. Then the whole Trojan camp
+was in an uproar as the people kept hurrying together, and they
+marvelled at the deeds of the heroes who had now got away towards
+the ships.
+</para>
+
+<para>
+When they reached the place where they had killed Hector's scout,
+Ulysses stayed his horses, and the son of Tydeus, leaping to the
+ground, placed the blood-stained spoils in the hands of Ulysses
+and remounted: then he lashed the horses onwards, and they flew
+forward nothing loth towards the ships as though of their own
+free will. Nestor was first to hear the tramp of their feet. "My
+friends," said he, "princes and counsellors of the Argives, shall
+I guess right or wrong?--but I must say what I think: there is a
+sound in my ears as of the tramp of horses. I hope it may Diomed
+and Ulysses driving in horses from the Trojans, but I much fear
+that the bravest of the Argives may have come to some harm at
+their hands."
+</para>
+
+<para>
+He had hardly done speaking when the two men came in and
+dismounted, whereon the others shook hands right gladly with them
+and congratulated them. Nestor knight of Gerene was first to
+question them. "Tell me," said he, "renowned Ulysses, how did you
+two come by these horses? Did you steal in among the Trojan
+forces, or did some god meet you and give them to you? They are
+like sunbeams. I am well conversant with the Trojans, for old
+warrior though I am I never hold back by the ships, but I never
+yet saw or heard of such horses as these are. Surely some god
+must have met you and given them to you, for you are both of you
+dear to Jove, and to Jove's daughter Minerva."
+</para>
+
+<para>
+And Ulysses answered, "Nestor son of Neleus, honour to the
+Achaean name, heaven, if it so will, can give us even better
+horses than these, for the gods are far mightier than we are.
+These horses, however, about which you ask me, are freshly come
+from Thrace. Diomed killed their king with the twelve bravest of
+his companions. Hard by the ships we took a thirteenth man--a
+scout whom Hector and the other Trojans had sent as a spy upon
+our ships."
+</para>
+
+<para>
+He laughed as he spoke and drove the horses over the ditch, while
+the other Achaeans followed him gladly. When they reached the
+strongly built quarters of the son of Tydeus, they tied the
+horses with thongs of leather to the manger, where the steeds of
+Diomed stood eating their sweet corn, but Ulysses hung the
+blood-stained spoils of Dolon at the stern of his ship, that they
+might prepare a sacred offering to Minerva. As for themselves,
+they went into the sea and washed the sweat from their bodies,
+and from their necks and thighs. When the sea-water had taken all
+the sweat from off them, and had refreshed them, they went into
+the baths and washed themselves. After they had so done and had
+anointed themselves with oil, they sat down to table, and drawing
+from a full mixing-bowl, made a drink-offering of wine to
+Minerva.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XI</title>
+
+<para>
+AND now as Dawn rose from her couch beside Tithonus, harbinger of
+light alike to mortals and immortals, Jove sent fierce Discord
+with the ensign of war in her hands to the ships of the Achaeans.
+She took her stand by the huge black hull of Ulysses' ship which
+was middlemost of all, so that her voice might carry farthest on
+either side, on the one hand towards the tents of Ajax son of
+Telamon, and on the other towards those of Achilles--for these
+two heroes, well-assured of their own strength, had valorously
+drawn up their ships at the two ends of the line. There she took
+her stand, and raised a cry both loud and shrill that filled the
+Achaeans with courage, giving them heart to fight resolutely and
+with all their might, so that they had rather stay there and do
+battle than go home in their ships.
+</para>
+
+<para>
+The son of Atreus shouted aloud and bade the Argives gird
+themselves for battle while he put on his armour. First he girded
+his goodly greaves about his legs, making them fast with ankle-
+clasps of silver; and about his chest he set the breastplate
+which Cinyras had once given him as a guest-gift. It had been
+noised abroad as far as Cyprus that the Achaeans were about to
+sail for Troy, and therefore he gave it to the king. It had ten
+courses of dark cyanus, twelve of gold, and ten of tin. There
+were serpents of cyanus that reared themselves up towards the
+neck, three upon either side, like the rainbows which the son of
+Saturn has set in heaven as a sign to mortal men. About his
+shoulders he threw his sword, studded with bosses of gold; and
+the scabbard was of silver with a chain of gold wherewith to hang
+it. He took moreover the richly-dight shield that covered his
+body when he was in battle--fair to see, with ten circles of
+bronze running all round it. On the body of the shield there were
+twenty bosses of white tin, with another of dark cyanus in the
+middle: this last was made to show a Gorgon's head, fierce and
+grim, with Rout and Panic on either side. The band for the arm to
+go through was of silver, on which there was a writhing snake of
+cyanus with three heads that sprang from a single neck, and went
+in and out among one another. On his head Agamemnon set a helmet,
+with a peak before and behind, and four plumes of horse-hair that
+nodded menacingly above it; then he grasped two redoubtable
+bronze-shod spears, and the gleam of his armour shot from him as
+a flame into the firmament, while Juno and Minerva thundered in
+honour of the king of rich Mycene.
+</para>
+
+<para>
+Every man now left his horses in charge of his charioteer to hold
+them in readiness by the trench, while he went into battle on
+foot clad in full armour, and a mighty uproar rose on high into
+the dawning. The chiefs were armed and at the trench before the
+horses got there, but these came up presently. The son of Saturn
+sent a portent of evil sound about their host, and the dew fell
+red with blood, for he was about to send many a brave man
+hurrying down to Hades.
+</para>
+
+<para>
+The Trojans, on the other side upon the rising slope of the
+plain, were gathered round great Hector, noble Polydamas, Aeneas
+who was honoured by the Trojans like an immortal, and the three
+sons of Antenor, Polybus, Agenor, and young Acamas beauteous as a
+god. Hector's round shield showed in the front rank, and as some
+baneful star that shines for a moment through a rent in the
+clouds and is again hidden beneath them; even so was Hector now
+seen in the front ranks and now again in the hindermost, and his
+bronze armour gleamed like the lightning of aegis-bearing Jove.
+</para>
+
+<para>
+And now as a band of reapers mow swathes of wheat or barley upon
+a rich man's land, and the sheaves fall thick before them, even
+so did the Trojans and Achaeans fall upon one another; they were
+in no mood for yielding but fought like wolves, and neither side
+got the better of the other. Discord was glad as she beheld them,
+for she was the only god that went among them; the others were
+not there, but stayed quietly each in his own home among the
+dells and valleys of Olympus. All of them blamed the son of
+Saturn for wanting to give victory to the Trojans, but father
+Jove heeded them not: he held aloof from all, and sat apart in
+his all-glorious majesty, looking down upon the city of the
+Trojans, the ships of the Achaeans, the gleam of bronze, and
+alike upon the slayers and on the slain.
+</para>
+
+<para>
+Now so long as the day waxed and it was still morning, their
+darts rained thick on one another and the people perished, but as
+the hour drew nigh when a woodman working in some mountain forest
+will get his midday meal--for he has felled till his hands are
+weary; he is tired out, and must now have food--then the Danaans
+with a cry that rang through all their ranks, broke the
+battalions of the enemy. Agamemnon led them on, and slew first
+Bienor, a leader of his people, and afterwards his comrade and
+charioteer Oileus, who sprang from his chariot and was coming
+full towards him; but Agamemnon struck him on the forehead with
+his spear; his bronze visor was of no avail against the weapon,
+which pierced both bronze and bone, so that his brains were
+battered in and he was killed in full fight.
+</para>
+
+<para>
+Agamemnon stripped their shirts from off them and left them with
+their breasts all bare to lie where they had fallen. He then went
+on to kill Isus and Antiphus two sons of Priam, the one a
+bastard, the other born in wedlock; they were in the same
+chariot--the bastard driving, while noble Antiphus fought beside
+him. Achilles had once taken both of them prisoners in the glades
+of Ida, and had bound them with fresh withes as they were
+shepherding, but he had taken a ransom for them; now, however,
+Agamemnon son of Atreus smote Isus in the chest above the nipple
+with his spear, while he struck Antiphus hard by the ear and
+threw him from his chariot. Forthwith he stripped their goodly
+armour from off them and recognized them, for he had already seen
+them at ships when Achilles brought them in from Ida. As a lion
+fastens on the fawns of a hind and crushes them in his great
+jaws, robbing them of their tender life while he on his way back
+to his lair--the hind can do nothing for them even though she be
+close by, for she is in an agony of fear, and flies through the
+thick forest, sweating, and at her utmost speed before the mighty
+monster--so, no man of the Trojans could help Isus and Antiphus,
+for they were themselves flying panic before the Argives.
+</para>
+
+<para>
+Then King Agamemnon took the two sons of Antimachus, Pisander and
+brave Hippolochus. It was Antimachus who had been foremost in
+preventing Helen's being restored to Menelaus, for he was largely
+bribed by Alexandrus; and now Agamemnon took his two sons, both
+in the same chariot, trying to bring their horses to a stand--for
+they had lost hold of the reins and the horses were mad with
+fear. The son of Atreus sprang upon them like a lion, and the
+pair besought him from their chariot. "Take us alive," they
+cried, "son of Atreus, and you shall receive a great ransom for
+us. Our father Antimachus has great store of gold, bronze, and
+wrought iron, and from this he will satisfy you with a very large
+ransom should he hear of our being alive at the ships of the
+Achaeans."
+</para>
+
+<para>
+With such piteous words and tears did they beseech the king, but
+they heard no pitiful answer in return. "If," said Agamemnon,
+"you are sons of Antimachus, who once at a council of Trojans
+proposed that Menelaus and Ulysses, who had come to you as
+envoys, should be killed and not suffered to return, you shall
+now pay for the foul iniquity of your father."
+</para>
+
+<para>
+As he spoke he felled Pisander from his chariot to the earth,
+smiting him on the chest with his spear, so that he lay face
+uppermost upon the ground. Hippolochus fled, but him too did
+Agamemnon smite; he cut off his hands and his head--which he sent
+rolling in among the crowd as though it were a ball. There he let
+them both lie, and wherever the ranks were thickest thither he
+flew, while the other Achaeans followed. Foot soldiers drove the
+foot soldiers of the foe in rout before them, and slew them;
+horsemen did the like by horsemen, and the thundering tramp of
+the horses raised a cloud of dust from off the plain. King
+Agamemnon followed after, ever slaying them and cheering on the
+Achaeans. As when some mighty forest is all ablaze--the eddying
+gusts whirl fire in all directions till the thickets shrivel and
+are consumed before the blast of the flame--even so fell the
+heads of the flying Trojans before Agamemnon son of Atreus, and
+many a noble pair of steeds drew an empty chariot along the
+highways of war, for lack of drivers who were lying on the plain,
+more useful now to vultures than to their wives.
+</para>
+
+<para>
+Jove drew Hector away from the darts and dust, with the carnage
+and din of battle; but the son of Atreus sped onwards, calling
+out lustily to the Danaans. They flew on by the tomb of old Ilus,
+son of Dardanus, in the middle of the plain, and past the place
+of the wild fig-tree making always for the city--the son of
+Atreus still shouting, and with hands all bedrabbled in gore; but
+when they had reached the Scaean gates and the oak tree, there
+they halted and waited for the others to come up. Meanwhile the
+Trojans kept on flying over the middle of the plain like a herd
+of cows maddened with fright when a lion has attacked them in the
+dead of night--he springs on one of them, seizes her neck in the
+grip of his strong teeth and then laps up her blood and gorges
+himself upon her entrails--even so did King Agamemnon son of
+Atreus pursue the foe, ever slaughtering the hindmost as they
+fled pell-mell before him. Many a man was flung headlong from his
+chariot by the hand of the son of Atreus, for he wielded his
+spear with fury.
+</para>
+
+<para>
+But when he was just about to reach the high wall and the city,
+the father of gods and men came down from heaven and took his
+seat, thunderbolt in hand, upon the crest of many-fountained Ida.
+He then told Iris of the golden wings to carry a message for him.
+"Go," said he, "fleet Iris, and speak thus to Hector--say that so
+long as he sees Agamemnon heading his men and making havoc of the
+Trojan ranks, he is to keep aloof and bid the others bear the
+brunt of the battle, but when Agamemnon is wounded either by
+spear or arrow, and takes to his chariot, then will I vouchsafe
+him strength to slay till he reach the ships and night falls at
+the going down of the sun."
+</para>
+
+<para>
+Iris hearkened and obeyed. Down she went to strong Ilius from the
+crests of Ida, and found Hector son of Priam standing by his
+chariot and horses. Then she said, "Hector son of Priam, peer of
+gods in counsel, father Jove has sent me to bear you this
+message--so long as you see Agamemnon heading his men and making
+havoc of the Trojan ranks, you are to keep aloof and bid the
+others bear the brunt of the battle, but when Agamemnon is
+wounded either by spear or arrow, and takes to his chariot, then
+will Jove vouchsafe you strength to slay till you reach the
+ships, and till night falls at the going down of the sun."
+</para>
+
+<para>
+When she had thus spoken Iris left him, and Hector sprang full
+armed from his chariot to the ground, brandishing his spear as he
+went about everywhere among the host, cheering his men on to
+fight, and stirring the dread strife of battle. The Trojans then
+wheeled round, and again met the Achaeans, while the Argives on
+their part strengthened their battalions. The battle was now in
+array and they stood face to face with one another, Agamemnon
+ever pressing forward in his eagerness to be ahead of all others.
+</para>
+
+<para>
+Tell me now ye Muses that dwell in the mansions of Olympus, who,
+whether of the Trojans or of their allies, was first to face
+Agamemnon? It was Iphidamas son of Antenor, a man both brave and
+of great stature, who was brought up in fertile Thrace, the
+mother of sheep. Cisses, his mother's father, brought him up in
+his own house when he was a child--Cisses, father to fair Theano.
+When he reached manhood, Cisses would have kept him there, and
+was for giving him his daughter in marriage, but as soon as he
+had married he set out to fight the Achaeans with twelve ships
+that followed him: these he had left at Percote and had come on
+by land to Ilius. He it was that now met Agamemnon son of Atreus.
+When they were close up with one another, the son of Atreus
+missed his aim, and Iphidamas hit him on the girdle below the
+cuirass and then flung himself upon him, trusting to his strength
+of arm; the girdle, however, was not pierced, nor nearly so, for
+the point of the spear struck against the silver and was turned
+aside as though it had been lead: King Agamemnon caught it from
+his hand, and drew it towards him with the fury of a lion; he
+then drew his sword, and killed Iphidamas by striking him on the
+neck. So there the poor fellow lay, sleeping a sleep as it were
+of bronze, killed in the defence of his fellow-citizens, far from
+his wedded wife, of whom he had had no joy though he had given
+much for her: he had given a hundred-head of cattle down, and had
+promised later on to give a thousand sheep and goats mixed, from
+the countless flocks of which he was possessed. Agamemnon son of
+Atreus then despoiled him, and carried off his armour into the
+host of the Achaeans.
+</para>
+
+<para>
+When noble Coon, Antenor's eldest son, saw this, sore indeed were
+his eyes at the sight of his fallen brother. Unseen by Agamemnon
+he got beside him, spear in hand, and wounded him in the middle
+of his arm below the elbow, the point of the spear going right
+through the arm. Agamemnon was convulsed with pain, but still not
+even for this did he leave off struggling and fighting, but
+grasped his spear that flew as fleet as the wind, and sprang upon
+Coon who was trying to drag off the body of his brother--his
+father's son--by the foot, and was crying for help to all the
+bravest of his comrades; but Agamemnon struck him with a
+bronze-shod spear and killed him as he was dragging the dead body
+through the press of men under cover of his shield: he then cut
+off his head, standing over the body of Iphidamas. Thus did the
+sons of Antenor meet their fate at the hands of the son of
+Atreus, and go down into the house of Hades.
+</para>
+
+<para>
+As long as the blood still welled warm from his wound Agamemnon
+went about attacking the ranks of the enemy with spear and sword
+and with great handfuls of stone, but when the blood had ceased
+to flow and the wound grew dry, the pain became great. As the
+sharp pangs which the Eilithuiae, goddesses of childbirth,
+daughters of Juno and dispensers of cruel pain, send upon a woman
+when she is in labour--even so sharp were the pangs of the son of
+Atreus. He sprang on to his chariot, and bade his charioteer
+drive to the ships, for he was in great agony. With a loud clear
+voice he shouted to the Danaans, "My friends, princes and
+counsellors of the Argives, defend the ships yourselves, for Jove
+has not suffered me to fight the whole day through against the
+Trojans."
+</para>
+
+<para>
+With this the charioteer turned his horses towards the ships, and
+they flew forward nothing loth. Their chests were white with foam
+and their bellies with dust, as they drew the wounded king out of
+the battle.
+</para>
+
+<para>
+When Hector saw Agamemnon quit the field, he shouted to the
+Trojans and Lycians saying, "Trojans, Lycians, and Dardanian
+warriors, be men, my friends, and acquit yourselves in battle
+bravely; their best man has left them, and Jove has vouchsafed me
+a great triumph; charge the foe with your chariots that you may
+win still greater glory."
+</para>
+
+<para>
+With these words he put heart and soul into them all, and as a
+huntsman hounds his dogs on against a lion or wild boar, even so
+did Hector, peer of Mars, hound the proud Trojans on against the
+Achaeans. Full of hope he plunged in among the foremost, and fell
+on the fight like some fierce tempest that swoops down upon the
+sea, and lashes its deep blue waters into fury.
+</para>
+
+<para>
+What, then is the full tale of those whom Hector son of Priam
+killed in the hour of triumph which Jove then vouchsafed him?
+First Asaeus, Autonous, and Opites; Dolops son of Clytius,
+Opheltius and Agelaus; Aesymnus, Orus and Hipponous steadfast in
+battle; these chieftains of the Achaeans did Hector slay, and
+then he fell upon the rank and file. As when the west wind
+hustles the clouds of the white south and beats them down with
+the fierceness of its fury--the waves of the sea roll high, and
+the spray is flung aloft in the rage of the wandering wind--even
+so thick were the heads of them that fell by the hand of Hector.
+</para>
+
+<para>
+All had then been lost and no help for it, and the Achaeans would
+have fled pell-mell to their ships, had not Ulysses cried out to
+Diomed, "Son of Tydeus, what has happened to us that we thus
+forget our prowess? Come, my good fellow, stand by my side and
+help me, we shall be shamed for ever if Hector takes the ships."
+</para>
+
+<para>
+And Diomed answered, "Come what may, I will stand firm; but we
+shall have scant joy of it, for Jove is minded to give victory to
+the Trojans rather than to us."
+</para>
+
+<para>
+With these words he struck Thymbraeus from his chariot to the
+ground, smiting him in the left breast with his spear, while
+Ulysses killed Molion who was his squire. These they let lie, now
+that they had stopped their fighting; the two heroes then went on
+playing havoc with the foe, like two wild boars that turn in fury
+and rend the hounds that hunt them. Thus did they turn upon the
+Trojans and slay them, and the Achaeans were thankful to have
+breathing time in their flight from Hector.
+</para>
+
+<para>
+They then took two princes with their chariot, the two sons of
+Merops of Percote, who excelled all others in the arts of
+divination. He had forbidden his sons to go to the war, but they
+would not obey him, for fate lured them to their fall. Diomed son
+of Tydeus slew them both and stripped them of their armour, while
+Ulysses killed Hippodamus and Hypeirochus.
+</para>
+
+<para>
+And now the son of Saturn as he looked down from Ida ordained
+that neither side should have the advantage, and they kept on
+killing one another. The son of Tydeus speared Agastrophus son of
+Paeon in the hip-joint with his spear. His chariot was not at
+hand for him to fly with, so blindly confident had he been. His
+squire was in charge of it at some distance and he was fighting
+on foot among the foremost until he lost his life. Hector soon
+marked the havoc Diomed and Ulysses were making, and bore down
+upon them with a loud cry, followed by the Trojan ranks; brave
+Diomed was dismayed when he saw them, and said to Ulysses who was
+beside him, "Great Hector is bearing down upon us and we shall be
+undone; let us stand firm and wait his onset."
+</para>
+
+<para>
+He poised his spear as he spoke and hurled it, nor did he miss
+his mark. He had aimed at Hector's head near the top of his
+helmet, but bronze was turned by bronze, and Hector was
+untouched, for the spear was stayed by the visored helm made with
+three plates of metal, which Phoebus Apollo had given him. Hector
+sprang back with a great bound under cover of the ranks; he fell
+on his knees and propped himself with his brawny hand leaning on
+the ground, for darkness had fallen on his eyes. The son of
+Tydeus having thrown his spear dashed in among the foremost
+fighters, to the place where he had seen it strike the ground;
+meanwhile Hector recovered himself and springing back into his
+chariot mingled with the crowd, by which means he saved his life.
+But Diomed made at him with his spear and said, "Dog, you have
+again got away though death was close on your heels. Phoebus
+Apollo, to whom I ween you pray ere you go into battle, has again
+saved you, nevertheless I will meet you and make an end of you
+hereafter, if there is any god who will stand by me too and be my
+helper. For the present I must pursue those I can lay hands on."
+</para>
+
+<para>
+As he spoke he began stripping the spoils from the son of Paeon,
+but Alexandrus husband of lovely Helen aimed an arrow at him,
+leaning against a pillar of the monument which men had raised to
+Ilus son of Dardanus, a ruler in days of old. Diomed had taken
+the cuirass from off the breast of Agastrophus, his heavy helmet
+also, and the shield from off his shoulders, when Paris drew his
+bow and let fly an arrow that sped not from his hand in vain, but
+pierced the flat of Diomed's right foot, going right through it
+and fixing itself in the ground. Thereon Paris with a hearty
+laugh sprang forward from his hiding-place, and taunted him
+saying, "You are wounded--my arrow has not been shot in vain;
+would that it had hit you in the belly and killed you, for thus
+the Trojans, who fear you as goats fear a lion, would have had a
+truce from evil."
+</para>
+
+<para>
+Diomed all undaunted answered, "Archer, you who without your bow
+are nothing, slanderer and seducer, if you were to be tried in
+single combat fighting in full armour, your bow and your arrows
+would serve you in little stead. Vain is your boast in that you
+have scratched the sole of my foot. I care no more than if a girl
+or some silly boy had hit me. A worthless coward can inflict but
+a light wound; when I wound a man though I but graze his skin it
+is another matter, for my weapon will lay him low. His wife will
+tear her cheeks for grief and his children will be fatherless:
+there will he rot, reddening the earth with his blood, and
+vultures, not women, will gather round him."
+</para>
+
+<para>
+Thus he spoke, but Ulysses came up and stood over him. Under this
+cover he sat down to draw the arrow from his foot, and sharp was
+the pain he suffered as he did so. Then he sprang on to his
+chariot and bade the charioteer drive him to the ships, for he
+was sick at heart.
+</para>
+
+<para>
+Ulysses was now alone; not one of the Argives stood by him, for
+they were all panic-stricken. "Alas," said he to himself in his
+dismay, "what will become of me? It is ill if I turn and fly
+before these odds, but it will be worse if I am left alone and
+taken prisoner, for the son of Saturn has struck the rest of the
+Danaans with panic. But why talk to myself in this way? Well do I
+know that though cowards quit the field, a hero, whether he wound
+or be wounded, must stand firm and hold his own."
+</para>
+
+<para>
+While he was thus in two minds, the ranks of the Trojans advanced
+and hemmed him in, and bitterly did they come to rue it. As
+hounds and lusty youths set upon a wild boar that sallies from
+his lair whetting his white tusks--they attack him from every
+side and can hear the gnashing of his jaws, but for all his
+fierceness they still hold their ground--even so furiously did
+the Trojans attack Ulysses. First he sprang spear in hand upon
+Deiopites and wounded him on the shoulder with a downward blow;
+then he killed Thoon and Ennomus. After these he struck
+Chersidamas in the loins under his shield as he had just sprung
+down from his chariot; so he fell in the dust and clutched the
+earth in the hollow of his hand. These he let lie, and went on to
+wound Charops son of Hippasus own brother to noble Socus. Socus,
+hero that he was, made all speed to help him, and when he was
+close to Ulysses he said, "Far-famed Ulysses, insatiable of craft
+and toil, this day you shall either boast of having killed both
+the sons of Hippasus and stripped them of their armour, or you
+shall fall before my spear."
+</para>
+
+<para>
+With these words he struck the shield of Ulysses. The spear went
+through the shield and passed on through his richly wrought
+cuirass, tearing the flesh from his side, but Pallas Minerva did
+not suffer it to pierce the entrails of the hero. Ulysses knew
+that his hour was not yet come, but he gave ground and said to
+Socus, "Wretch, you shall now surely die. You have stayed me from
+fighting further with the Trojans, but you shall now fall by my
+spear, yielding glory to myself, and your soul to Hades of the
+noble steeds."
+</para>
+
+<para>
+Socus had turned in flight, but as he did so, the spear struck
+him in the back midway between the shoulders, and went right
+through his chest. He fell heavily to the ground and Ulysses
+vaunted over him saying, "O Socus, son of Hippasus tamer of
+horses, death has been too quick for you and you have not escaped
+him: poor wretch, not even in death shall your father and mother
+close your eyes, but the ravening vultures shall enshroud you
+with the flapping of their dark wings and devour you. Whereas
+even though I fall the Achaeans will give me my due rites of
+burial."
+</para>
+
+<para>
+So saying he drew Socus's heavy spear out of his flesh and from
+his shield, and the blood welled forth when the spear was
+withdrawn so that he was much dismayed. When the Trojans saw that
+Ulysses was bleeding they raised a great shout and came on in a
+body towards him; he therefore gave ground, and called his
+comrades to come and help him. Thrice did he cry as loudly as man
+can cry, and thrice did brave Menelaus hear him; he turned,
+therefore, to Ajax who was close beside him and said, "Ajax,
+noble son of Telamon, captain of your people, the cry of Ulysses
+rings in my ears, as though the Trojans had cut him off and were
+worsting him while he is single-handed. Let us make our way
+through the throng; it will be well that we defend him; I fear he
+may come to harm for all his valour if he be left without
+support, and the Danaans would miss him sorely."
+</para>
+
+<para>
+He led the way and mighty Ajax went with him. The Trojans had
+gathered round Ulysses like ravenous mountain jackals round the
+carcase of some horned stag that has been hit with an arrow--the
+stag has fled at full speed so long as his blood was warm and his
+strength has lasted, but when the arrow has overcome him, the
+savage jackals devour him in the shady glades of the forest. Then
+heaven sends a fierce lion thither, whereon the jackals fly in
+terror and the lion robs them of their prey--even so did Trojans
+many and brave gather round crafty Ulysses, but the hero stood at
+bay and kept them off with his spear. Ajax then came up with his
+shield before him like a wall, and stood hard by, whereon the
+Trojans fled in all directions. Menelaus took Ulysses by the
+hand, and led him out of the press while his squire brought up
+his chariot, but Ajax rushed furiously on the Trojans and killed
+Doryclus, a bastard son of Priam; then he wounded Pandocus,
+Lysandrus, Pyrasus, and Pylartes; as some swollen torrent comes
+rushing in full flood from the mountains on to the plain, big
+with the rain of heaven--many a dry oak and many a pine does it
+engulf, and much mud does it bring down and cast into the sea--
+even so did brave Ajax chase the foe furiously over the plain,
+slaying both men and horses.
+</para>
+
+<para>
+Hector did not yet know what Ajax was doing, for he was fighting
+on the extreme left of the battle by the banks of the river
+Scamander, where the carnage was thickest and the war-cry loudest
+round Nestor and brave Idomeneus. Among these Hector was making
+great slaughter with his spear and furious driving, and was
+destroying the ranks that were opposed to him; still the Achaeans
+would have given no ground, had not Alexandrus husband of lovely
+Helen stayed the prowess of Machaon, shepherd of his people, by
+wounding him in the right shoulder with a triple-barbed arrow.
+The Achaeans were in great fear that as the fight had turned
+against them the Trojans might take him prisoner, and Idomeneus
+said to Nestor, "Nestor son of Neleus, honour to the Achaean
+name, mount your chariot at once; take Machaon with you and drive
+your horses to the ships as fast as you can. A physician is worth
+more than several other men put together, for he can cut out
+arrows and spread healing herbs."
+</para>
+
+<para>
+Nestor knight of Gerene did as Idomeneus had counselled; he at
+once mounted his chariot, and Machaon son of the famed physician
+Aesculapius, went with him. He lashed his horses and they flew
+onward nothing loth towards the ships, as though of their own
+free will.
+</para>
+
+<para>
+Then Cebriones seeing the Trojans in confusion said to Hector
+from his place beside him, "Hector, here are we two fighting on
+the extreme wing of the battle, while the other Trojans are in
+pell-mell rout, they and their horses. Ajax son of Telamon is
+driving them before him; I know him by the breadth of his shield:
+let us turn our chariot and horses thither, where horse and foot
+are fighting most desperately, and where the cry of battle is
+loudest."
+</para>
+
+<para>
+With this he lashed his goodly steeds, and when they felt the
+whip they drew the chariot full speed among the Achaeans and
+Trojans, over the bodies and shields of those that had fallen:
+the axle was bespattered with blood, and the rail round the car
+was covered with splashes both from the horses' hoofs and from
+the tyres of the wheels. Hector tore his way through and flung
+himself into the thick of the fight, and his presence threw the
+Danaans into confusion, for his spear was not long idle;
+nevertheless though he went among the ranks with sword and spear,
+and throwing great stones, he avoided Ajax son of Telamon, for
+Jove would have been angry with him if he had fought a better man
+than himself.
+</para>
+
+<para>
+Then father Jove from his high throne struck fear into the heart
+of Ajax, so that he stood there dazed and threw his shield behind
+him--looking fearfully at the throng of his foes as though he
+were some wild beast, and turning hither and thither but
+crouching slowly backwards. As peasants with their hounds chase a
+lion from their stockyard, and watch by night to prevent his
+carrying off the pick of their herd--he makes his greedy spring,
+but in vain, for the darts from many a strong hand fall thick
+around him, with burning brands that scare him for all his fury,
+and when morning comes he slinks foiled and angry away--even so
+did Ajax, sorely against his will, retreat angrily before the
+Trojans, fearing for the ships of the Achaeans. Or as some lazy
+ass that has had many a cudgel broken about his back, when he
+into a field begins eating the corn--boys beat him but he is too
+many for them, and though they lay about with their sticks they
+cannot hurt him; still when he has had his fill they at last
+drive him from the field--even so did the Trojans and their
+allies pursue great Ajax, ever smiting the middle of his shield
+with their darts. Now and again he would turn and show fight,
+keeping back the battalions of the Trojans, and then he would
+again retreat; but he prevented any of them from making his way
+to the ships. Single-handed he stood midway between the Trojans
+and Achaeans: the spears that sped from their hands stuck some of
+them in his mighty shield, while many, though thirsting for his
+blood, fell to the ground ere they could reach him to the
+wounding of his fair flesh.
+</para>
+
+<para>
+Now when Eurypylus the brave son of Euaemon saw that Ajax was
+being overpowered by the rain of arrows, he went up to him and
+hurled his spear. He struck Apisaon son of Phausius in the liver
+below the midriff, and laid him low. Eurypylus sprang upon him,
+and stripped the armour from his shoulders; but when Alexandrus
+saw him, he aimed an arrow at him which struck him in the right
+thigh; the arrow broke, but the point that was left in the wound
+dragged on the thigh; he drew back, therefore, under cover of his
+comrades to save his life, shouting as he did so to the Danaans,
+"My friends, princes and counsellors of the Argives, rally to the
+defence of Ajax who is being overpowered, and I doubt whether he
+will come out of the fight alive. Hither, then, to the rescue of
+great Ajax son of Telamon."
+</para>
+
+<para>
+Even so did he cry when he was wounded; thereon the others came
+near, and gathered round him, holding their shields upwards from
+their shoulders so as to give him cover. Ajax then made towards
+them, and turned round to stand at bay as soon as he had reached
+his men.
+</para>
+
+<para>
+Thus then did they fight as it were a flaming fire. Meanwhile the
+mares of Neleus, all in a lather with sweat, were bearing Nestor
+out of the fight, and with him Machaon shepherd of his people.
+Achilles saw and took note, for he was standing on the stern of
+his ship watching the hard stress and struggle of the fight. He
+called from the ship to his comrade Patroclus, who heard him in
+the tent and came out looking like Mars himself--here indeed was
+the beginning of the ill that presently befell him. "Why," said
+he, "Achilles, do you call me? What do you want with me?" And
+Achilles answered, "Noble son of Menoetius, man after my own
+heart, I take it that I shall now have the Achaeans praying at my
+knees, for they are in great straits; go, Patroclus, and ask
+Nestor who it is that he is bearing away wounded from the field;
+from his back I should say it was Machaon son of Aesculapius, but
+I could not see his face for the horses went by me at full
+speed."
+</para>
+
+<para>
+Patroclus did as his dear comrade had bidden him, and set off
+running by the ships and tents of the Achaeans.
+</para>
+
+<para>
+When Nestor and Machaon had reached the tents of the son of
+Neleus, they dismounted, and an esquire, Eurymedon, took the
+horses from the chariot. The pair then stood in the breeze by the
+seaside to dry the sweat from their shirts, and when they had so
+done they came inside and took their seats. Fair Hecamede, whom
+Nestor had had awarded to him from Tenedos when Achilles took it,
+mixed them a mess; she was daughter of wise Arsinous, and the
+Achaeans had given her to Nestor because he excelled all of them
+in counsel. First she set for them a fair and well-made table
+that had feet of cyanus; on it there was a vessel of bronze and
+an onion to give relish to the drink, with honey and cakes of
+barley-meal. There was also a cup of rare workmanship which the
+old man had brought with him from home, studded with bosses of
+gold; it had four handles, on each of which there were two golden
+doves feeding, and it had two feet to stand on. Any one else
+would hardly have been able to lift it from the table when it was
+full, but Nestor could do so quite easily. In this the woman, as
+fair as a goddess, mixed them a mess with Pramnian wine; she
+grated goat's milk cheese into it with a bronze grater, threw in
+a handful of white barley-meal, and having thus prepared the mess
+she bade them drink it. When they had done so and had thus
+quenched their thirst, they fell talking with one another, and at
+this moment Patroclus appeared at the door.
+</para>
+
+<para>
+When the old man saw him he sprang from his seat, seized his
+hand, led him into the tent, and bade him take his place among
+them; but Patroclus stood where he was and said, "Noble sir, I
+may not stay, you cannot persuade me to come in; he that sent me
+is not one to be trifled with, and he bade me ask who the wounded
+man was whom you were bearing away from the field. I can now see
+for myself that he is Machaon, shepherd of his people. I must go
+back and tell Achilles. You, sir, know what a terrible man he is,
+and how ready to blame even where no blame should lie."
+</para>
+
+<para>
+And Nestor answered, "Why should Achilles care to know how many
+of the Achaeans may be wounded? He recks not of the dismay that
+reigns in our host; our most valiant chieftains lie disabled,
+brave Diomed, son of Tydeus, is wounded; so are Ulysses and
+Agamemnon; Eurypylus has been hit with an arrow in the thigh, and
+I have just been bringing this man from the field--he too wounded
+with an arrow. Nevertheless, Achilles, so valiant though he be,
+cares not and knows no ruth. Will he wait till the ships, do what
+we may, are in a blaze, and we perish one upon the other? As for
+me, I have no strength nor stay in me any longer; would that I
+were still young and strong as in the days when there was a fight
+between us and the men of Elis about some cattle-raiding. I then
+killed Itymoneus, the valiant son of Hypeirochus, a dweller in
+Elis, as I was driving in the spoil; he was hit by a dart thrown
+by my hand while fighting in the front rank in defence of his
+cows, so he fell and the country people around him were in great
+fear. We drove off a vast quantity of booty from the plain, fifty
+herds of cattle and as many flocks of sheep; fifty droves also of
+pigs, and as many wide-spreading flocks of goats. Of horses,
+moreover, we seized a hundred and fifty, all of them mares, and
+many had foals running with them. All these did we drive by night
+to Pylus, the city of Neleus, taking them within the city; and
+the heart of Neleus was glad in that I had taken so much, though
+it was the first time I had ever been in the field. At daybreak
+the heralds went round crying that all in Elis to whom there was
+a debt owing should come; and the leading Pylians assembled to
+divide the spoils. There were many to whom the Epeans owed
+chattels, for we men of Pylus were few and had been oppressed
+with wrong; in former years Hercules had come, and had laid his
+hand heavy upon us, so that all our best men had perished. Neleus
+had had twelve sons, but I alone was left; the others had all
+been killed. The Epeans presuming upon all this had looked down
+upon us and had done us much evil. My father chose a herd of
+cattle and a great flock of sheep--three hundred in all--and he
+took their shepherds with him, for there was a great debt due to
+him in Elis, to wit four horses, winners of prizes. They and
+their chariots with them had gone to the games and were to run
+for a tripod, but King Augeas took them, and sent back their
+driver grieving for the loss of his horses. Neleus was angered by
+what he had both said and done, and took great value in return,
+but he divided the rest, that no man might have less than his
+full share.
+</para>
+
+<para>
+"Thus did we order all things, and offer sacrifices to the gods
+throughout the city; but three days afterwards the Epeans came in
+a body, many in number, they and their chariots, in full array,
+and with them the two Moliones in their armour, though they were
+still lads and unused to fighting. Now there is a certain town,
+Thryoessa, perched upon a rock on the river Alpheus, the border
+city Pylus. This they would destroy, and pitched their camp about
+it, but when they had crossed their whole plain, Minerva darted
+down by night from Olympus and bade us set ourselves in array;
+and she found willing soldiers in Pylos, for the men meant
+fighting. Neleus would not let me arm, and hid my horses, for he
+said that as yet I could know nothing about war; nevertheless
+Minerva so ordered the fight that, all on foot as I was, I fought
+among our mounted forces and vied with the foremost of them.
+There is a river Minyeius that falls into the sea near Arene, and
+there they that were mounted (and I with them) waited till
+morning, when the companies of foot soldiers came up with us in
+force. Thence in full panoply and equipment we came towards noon
+to the sacred waters of the Alpheus, and there we offered victims
+to almighty Jove, with a bull to Alpheus, another to Neptune, and
+a herd-heifer to Minerva. After this we took supper in our
+companies, and laid us down to rest each in his armour by the
+river.
+</para>
+
+<para>
+"The Epeans were beleaguering the city and were determined to
+take it, but ere this might be there was a desperate fight in
+store for them. When the sun's rays began to fall upon the earth
+we joined battle, praying to Jove and to Minerva, and when the
+fight had begun, I was the first to kill my man and take his
+horses--to wit the warrior Mulius. He was son-in-law to Augeas,
+having married his eldest daughter, golden-haired Agamede, who
+knew the virtues of every herb which grows upon the face of the
+earth. I speared him as he was coming towards me, and when he
+fell headlong in the dust, I sprang upon his chariot and took my
+place in the front ranks. The Epeans fled in all directions when
+they saw the captain of their horsemen (the best man they had)
+laid low, and I swept down on them like a whirlwind, taking fifty
+chariots--and in each of them two men bit the dust, slain by my
+spear. I should have even killed the two Moliones, sons of Actor,
+unless their real father, Neptune lord of the earthquake, had
+hidden them in a thick mist and borne them out of the fight.
+Thereon Jove vouchsafed the Pylians a great victory, for we
+chased them far over the plain, killing the men and bringing in
+their armour, till we had brought our horses to Buprasium, rich
+in wheat, and to the Olenian rock, with the hill that is called
+Alision, at which point Minerva turned the people back. There I
+slew the last man and left him; then the Achaeans drove their
+horses back from Buprasium to Pylos and gave thanks to Jove among
+the gods, and among mortal men to Nestor.
+</para>
+
+<para>
+"Such was I among my peers, as surely as ever was, but Achilles
+is for keeping all his valour for himself; bitterly will he rue
+it hereafter when the host is being cut to pieces. My good
+friend, did not Menoetius charge you thus, on the day when he
+sent you from Phthia to Agamemnon? Ulysses and I were in the
+house, inside, and heard all that he said to you; for we came to
+the fair house of Peleus while beating up recruits throughout all
+Achaea, and when we got there we found Menoetius and yourself,
+and Achilles with you. The old knight Peleus was in the outer
+court, roasting the fat thigh-bones of a heifer to Jove the lord
+of thunder; and he held a gold chalice in his hand from which he
+poured drink-offerings of wine over the burning sacrifice. You
+two were busy cutting up the heifer, and at that moment we stood
+at the gates, whereon Achilles sprang to his feet, led us by the
+hand into the house, placed us at table, and set before us such
+hospitable entertainment as guests expect. When we had satisfied
+ourselves with meat and drink, I said my say and urged both of
+you to join us. You were ready enough to do so, and the two old
+men charged you much and straitly. Old Peleus bade his son
+Achilles fight ever among the foremost and outvie his peers,
+while Menoetius the son of Actor spoke thus to you: 'My son,'
+said he, 'Achilles is of nobler birth than you are, but you are
+older than he, though he is far the better man of the two.
+Counsel him wisely, guide him in the right way, and he will
+follow you to his own profit.' Thus did your father charge you,
+but you have forgotten; nevertheless, even now, say all this to
+Achilles if he will listen to you. Who knows but with heaven's
+help you may talk him over, for it is good to take a friend's
+advice. If, however, he is fearful about some oracle, or if his
+mother has told him something from Jove, then let him send you,
+and let the rest of the Myrmidons follow with you, if perchance
+you may bring light and saving to the Danaans. And let him send
+you into battle clad in his own armour, that the Trojans may
+mistake you for him and leave off fighting; the sons of the
+Achaeans may thus have time to get their breath, for they are
+hard pressed and there is little breathing time in battle. You,
+who are fresh, might easily drive a tired enemy back to his walls
+and away from the tents and ships."
+</para>
+
+<para>
+With these words he moved the heart of Patroclus, who set off
+running by the line of the ships to Achilles, descendant of
+Aeacus. When he had got as far as the ships of Ulysses, where was
+their place of assembly and court of justice, with their altars
+dedicated to the gods, Eurypylus son of Euaemon, met him, wounded
+in the thigh with an arrow, and limping out of the fight. Sweat
+rained from his head and shoulders, and black blood welled from
+his cruel wound, but his mind did not wander. The son of
+Menoetius when he saw him had compassion upon him and spoke
+piteously saying, "O unhappy princes and counsellors of the
+Danaans, are you then doomed to feed the hounds of Troy with your
+fat, far from your friends and your native land? Say, noble
+Eurypylus, will the Achaeans be able to hold great Hector in
+check, or will they fall now before his spear?"
+</para>
+
+<para>
+Wounded Eurypylus made answer, "Noble Patroclus, there is no hope
+left for the Achaeans but they will perish at their ships. All
+they that were princes among us are lying struck down and wounded
+at the hands of the Trojans, who are waxing stronger and
+stronger. But save me and take me to your ship; cut out the arrow
+from my thigh; wash the black blood from off it with warm water,
+and lay upon it those gracious herbs which, so they say, have
+been shown you by Achilles, who was himself shown them by Chiron,
+most righteous of all the centaurs. For of the physicians
+Podalirius and Machaon, I hear that the one is lying wounded in
+his tent and is himself in need of healing, while the other is
+fighting the Trojans upon the plain."
+</para>
+
+<para>
+"Hero Eurypylus," replied the brave son of Menoetius, "how may
+these things be? What can I do? I am on my way to bear a message
+to noble Achilles from Nestor of Gerene, bulwark of the Achaeans,
+but even so I will not be unmindful of your distress."
+</para>
+
+<para>
+With this he clasped him round the middle and led him into the
+tent, and a servant, when he saw him, spread bullock-skins on the
+ground for him to lie on. He laid him at full length and cut out
+the sharp arrow from his thigh; he washed the black blood from
+the wound with warm water; he then crushed a bitter herb, rubbing
+it between his hands, and spread it upon the wound; this was a
+virtuous herb which killed all pain; so the wound presently dried
+and the blood left off flowing.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XII</title>
+
+<para>
+SO THE son of Menoetius was attending to the hurt of Eurypylus
+within the tent, but the Argives and Trojans still fought
+desperately, nor were the trench and the high wall above it, to
+keep the Trojans in check longer. They had built it to protect
+their ships, and had dug the trench all round it that it might
+safeguard both the ships and the rich spoils which they had
+taken, but they had not offered hecatombs to the gods. It had
+been built without the consent of the immortals, and therefore it
+did not last. So long as Hector lived and Achilles nursed his
+anger, and so long as the city of Priam remained untaken, the
+great wall of the Achaeans stood firm; but when the bravest of
+the Trojans were no more, and many also of the Argives, though
+some were yet left alive--when, moreover, the city was sacked in
+the tenth year, and the Argives had gone back with their ships to
+their own country--then Neptune and Apollo took counsel to
+destroy the wall, and they turned on to it the streams of all the
+rivers from Mount Ida into the sea, Rhesus, Heptaporus, Caresus,
+Rhodius, Grenicus, Aesopus, and goodly Scamander, with Simois,
+where many a shield and helm had fallen, and many a hero of the
+race of demigods had bitten the dust. Phoebus Apollo turned the
+mouths of all these rivers together and made them flow for nine
+days against the wall, while Jove rained the whole time that he
+might wash it sooner into the sea. Neptune himself, trident in
+hand, surveyed the work and threw into the sea all the
+foundations of beams and stones which the Achaeans had laid with
+so much toil; he made all level by the mighty stream of the
+Hellespont, and then when he had swept the wall away he spread a
+great beach of sand over the place where it had been. This done
+he turned the rivers back into their old courses.
+</para>
+
+<para>
+This was what Neptune and Apollo were to do in after time; but as
+yet battle and turmoil were still raging round the wall till its
+timbers rang under the blows that rained upon them. The Argives,
+cowed by the scourge of Jove, were hemmed in at their ships in
+fear of Hector the mighty minister of Rout, who as heretofore
+fought with the force and fury of a whirlwind. As a lion or wild
+boar turns fiercely on the dogs and men that attack him, while
+these form solid wall and shower their javelins as they face
+him--his courage is all undaunted, but his high spirit will be
+the death of him; many a time does he charge at his pursuers to
+scatter them, and they fall back as often as he does so--even so
+did Hector go about among the host exhorting his men, and
+cheering them on to cross the trench.
+</para>
+
+<para>
+But the horses dared not do so, and stood neighing upon its
+brink, for the width frightened them. They could neither jump it
+nor cross it, for it had overhanging banks all round upon either
+side, above which there were the sharp stakes that the sons of
+the Achaeans had planted so close and strong as a defence against
+all who would assail it; a horse, therefore, could not get into
+it and draw his chariot after him, but those who were on foot
+kept trying their very utmost. Then Polydamas went up to Hector
+and said, "Hector, and you other captains of the Trojans and
+allies, it is madness for us to try and drive our horses across
+the trench; it will be very hard to cross, for it is full of
+sharp stakes, and beyond these there is the wall. Our horses
+therefore cannot get down into it, and would be of no use if they
+did; moreover it is a narrow place and we should come to harm.
+If, indeed, great Jove is minded to help the Trojans, and in his
+anger will utterly destroy the Achaeans, I would myself gladly
+see them perish now and here far from Argos; but if they should
+rally and we are driven back from the ships pell-mell into the
+trench there will be not so much as a man get back to the city to
+tell the tale. Now, therefore, let us all do as I say; let our
+squires hold our horses by the trench, but let us follow Hector
+in a body on foot, clad in full armour, and if the day of their
+doom is at hand the Achaeans will not be able to withstand us."
+</para>
+
+<para>
+Thus spoke Polydamas and his saying pleased Hector, who sprang in
+full armour to the ground, and all the other Trojans, when they
+saw him do so, also left their chariots. Each man then gave his
+horses over to his charioteer in charge to hold them ready for
+him at the trench. Then they formed themselves into companies,
+made themselves ready, and in five bodies followed their leaders.
+Those that went with Hector and Polydamas were the bravest and
+most in number, and the most determined to break through the wall
+and fight at the ships. Cebriones was also joined with them as
+third in command, for Hector had left his chariot in charge of a
+less valiant soldier. The next company was led by Paris,
+Alcathous, and Agenor; the third by Helenus and Deiphobus, two
+sons of Priam, and with them was the hero Asius--Asius, the son
+of Hyrtacus, whose great black horses of the breed that comes
+from the river Selleis had brought him from Arisbe. Aeneas, the
+valiant son of Anchises, led the fourth; he and the two sons of
+Antenor, Archelochus and Acamas, men well versed in all the arts
+of war. Sarpedon was captain over the allies, and took with him
+Glaucus and Asteropaeus whom he deemed most valiant after
+himself--for he was far the best man of them all. These helped to
+array one another in their ox-hide shields, and then charged
+straight at the Danaans, for they felt sure that they would not
+hold out longer and that they should themselves now fall upon the
+ships.
+</para>
+
+<para>
+The rest of the Trojans and their allies now followed the counsel
+of Polydamas but Asius, son of Hyrtacus, would not leave his
+horses and his esquire behind him; in his foolhardiness he took
+them on with him towards the ships, nor did he fail to come by
+his end in consequence. Nevermore was he to return to wind-beaten
+Ilius, exulting in his chariot and his horses; ere he could do
+so, death of ill-omened name had overshadowed him and he had
+fallen by the spear of Idomeneus the noble son of Deucalion. He
+had driven towards the left wing of the ships, by which way the
+Achaeans used to return with their chariots and horses from the
+plain. Hither he drove and found the gates with their doors
+opened wide, and the great bar down--for the gatemen kept them
+open so as to let those of their comrades enter who might be
+flying towards the ships. Hither of set purpose did he direct his
+horses, and his men followed him with a loud cry, for they felt
+sure that the Achaeans would not hold out longer, and that they
+should now fall upon the ships. Little did they know that at the
+gates they should find two of the bravest chieftains, proud sons
+of the fighting Lapithae--the one, Polypoetes, mighty son of
+Pirithous, and the other Leonteus, peer of murderous Mars. These
+stood before the gates like two high oak trees upon the
+mountains, that tower from their wide-spreading roots, and year
+after year battle with wind and rain--even so did these two men
+await the onset of great Asius confidently and without flinching.
+The Trojans led by him and by Iamenus, Orestes, Adamas the son of
+Asius, Thoon and Oenomaus, raised a loud cry of battle and made
+straight for the wall, holding their shields of dry ox-hide above
+their heads; for a while the two defenders remained inside and
+cheered the Achaeans on to stand firm in the defence of their
+ships; when, however, they saw that the Trojans were attacking
+the wall, while the Danaans were crying out for help and being
+routed, they rushed outside and fought in front of the gates like
+two wild boars upon the mountains that abide the attack of men
+and dogs, and charging on either side break down the wood all
+round them tearing it up by the roots, and one can hear the
+clattering of their tusks, till some one hits them and makes an
+end of them--even so did the gleaming bronze rattle about their
+breasts, as the weapons fell upon them; for they fought with
+great fury, trusting to their own prowess and to those who were
+on the wall above them. These threw great stones at their
+assailants in defence of themselves their tents and their ships.
+The stones fell thick as the flakes of snow which some fierce
+blast drives from the dark clouds and showers down in sheets upon
+the earth--even so fell the weapons from the hands alike of
+Trojans and Achaeans. Helmet and shield rang out as the great
+stones rained upon them, and Asius, the son of Hyrtacus, in his
+dismay cried aloud and smote his two thighs. "Father Jove," he
+cried, "of a truth you too are altogether given to lying. I made
+sure the Argive heroes could not withstand us, whereas like
+slim-waisted wasps, or bees that have their nests in the rocks by
+the wayside--they leave not the holes wherein they have built
+undefended, but fight for their little ones against all who would
+take them--even so these men, though they be but two, will not be
+driven from the gates, but stand firm either to slay or be
+slain."
+</para>
+
+<para>
+He spoke, but moved not the mind of Jove, whose counsel it then
+was to give glory to Hector. Meanwhile the rest of the Trojans
+were fighting about the other gates; I, however, am no god to be
+able to tell about all these things, for the battle raged
+everywhere about the stone wall as it were a fiery furnace. The
+Argives, discomfited though they were, were forced to defend
+their ships, and all the gods who were defending the Achaeans
+were vexed in spirit; but the Lapithae kept on fighting with
+might and main.
+</para>
+
+<para>
+Thereon Polypoetes, mighty son of Pirithous, hit Damasus with a
+spear upon his cheek-pierced helmet. The helmet did not protect
+him, for the point of the spear went through it, and broke the
+bone, so that the brain inside was scattered about, and he died
+fighting. He then slew Pylon and Ormenus. Leonteus, of the race
+of Mars, killed Hippomachus the son of Antimachus by striking him
+with his spear upon the girdle. He then drew his sword and sprang
+first upon Antiphates whom he killed in combat, and who fell face
+upwards on the earth. After him he killed Menon, Iamenus, and
+Orestes, and laid them low one after the other.
+</para>
+
+<para>
+While they were busy stripping the armour from these heroes, the
+youths who were led on by Polydamas and Hector (and these were
+the greater part and the most valiant of those that were trying
+to break through the wall and fire the ships) were still standing
+by the trench, uncertain what they should do; for they had seen a
+sign from heaven when they had essayed to cross it--a soaring
+eagle that flew skirting the left wing of their host, with a
+monstrous blood-red snake in its talons still alive and
+struggling to escape. The snake was still bent on revenge,
+wriggling and twisting itself backwards till it struck the bird
+that held it, on the neck and breast; whereon the bird being in
+pain, let it fall, dropping it into the middle of the host, and
+then flew down the wind with a sharp cry. The Trojans were struck
+with terror when they saw the snake, portent of aegis-bearing
+Jove, writhing in the midst of them, and Polydamas went up to
+Hector and said, "Hector, at our councils of war you are ever
+given to rebuke me, even when I speak wisely, as though it were
+not well, forsooth, that one of the people should cross your will
+either in the field or at the council board; you would have them
+support you always: nevertheless I will say what I think will be
+best; let us not now go on to fight the Danaans at their ships,
+for I know what will happen if this soaring eagle which skirted
+the left wing of our host with a monstrous blood-red snake in its
+talons (the snake being still alive) was really sent as an omen
+to the Trojans on their essaying to cross the trench. The eagle
+let go her hold; she did not succeed in taking it home to her
+little ones, and so will it be--with ourselves; even though by a
+mighty effort we break through the gates and wall of the
+Achaeans, and they give way before us, still we shall not return
+in good order by the way we came, but shall leave many a man
+behind us whom the Achaeans will do to death in defence of their
+ships. Thus would any seer who was expert in these matters, and
+was trusted by the people, read the portent."
+</para>
+
+<para>
+Hector looked fiercely at him and said, "Polydamas, I like not of
+your reading. You can find a better saying than this if you will.
+If, however, you have spoken in good earnest, then indeed has
+heaven robbed you of your reason. You would have me pay no heed
+to the counsels of Jove, nor to the promises he made me--and he
+bowed his head in confirmation; you bid me be ruled rather by the
+flight of wild-fowl. What care I whether they fly towards dawn or
+dark, and whether they be on my right hand or on my left? Let us
+put our trust rather in the counsel of great Jove, king of
+mortals and immortals. There is one omen, and one only--that a
+man should fight for his country. Why are you so fearful? Though
+we be all of us slain at the ships of the Argives you are not
+likely to be killed yourself, for you are not steadfast nor
+courageous. If you will not fight, or would talk others over from
+doing so, you shall fall forthwith before my spear."
+</para>
+
+<para>
+With these words he led the way, and the others followed after
+with a cry that rent the air. Then Jove the lord of thunder sent
+the blast of a mighty wind from the mountains of Ida, that bore
+the dust down towards the ships; he thus lulled the Achaeans into
+security, and gave victory to Hector and to the Trojans, who,
+trusting to their own might and to the signs he had shown them,
+essayed to break through the great wall of the Achaeans. They
+tore down the breastworks from the walls, and overthrew the
+battlements; they upheaved the buttresses, which the Achaeans had
+set in front of the wall in order to support it; when they had
+pulled these down they made sure of breaking through the wall,
+but the Danaans still showed no sign of giving ground; they still
+fenced the battlements with their shields of ox-hide, and hurled
+their missiles down upon the foe as soon as any came below the
+wall.
+</para>
+
+<para>
+The two Ajaxes went about everywhere on the walls cheering on the
+Achaeans, giving fair words to some while they spoke sharply to
+any one whom they saw to be remiss. "My friends," they cried,
+"Argives one and all--good bad and indifferent, for there was
+never fight yet, in which all were of equal prowess--there is now
+work enough, as you very well know, for all of you. See that you
+none of you turn in flight towards the ships, daunted by the
+shouting of the foe, but press forward and keep one another in
+heart, if it may so be that Olympian Jove the lord of lightning
+will vouchsafe us to repel our foes, and drive them back towards
+the city."
+</para>
+
+<para>
+Thus did the two go about shouting and cheering the Achaeans on.
+As the flakes that fall thick upon a winter's day, when Jove is
+minded to snow and to display these his arrows to mankind--he
+lulls the wind to rest, and snows hour after hour till he has
+buried the tops of the high mountains, the headlands that jut
+into the sea, the grassy plains, and the tilled fields of men;
+the snow lies deep upon the forelands, and havens of the grey
+sea, but the waves as they come rolling in stay it that it can
+come no further, though all else is wrapped as with a mantle, so
+heavy are the heavens with snow--even thus thickly did the stones
+fall on one side and on the other, some thrown at the Trojans,
+and some by the Trojans at the Achaeans; and the whole wall was
+in an uproar.
+</para>
+
+<para>
+Still the Trojans and brave Hector would not yet have broken down
+the gates and the great bar, had not Jove turned his son Sarpedon
+against the Argives as a lion against a herd of horned cattle.
+Before him he held his shield of hammered bronze, that the smith
+had beaten so fair and round, and had lined with ox hides which
+he had made fast with rivets of gold all round the shield; this
+he held in front of him, and brandishing his two spears came on
+like some lion of the wilderness, who has been long famished for
+want of meat and will dare break even into a well-fenced
+homestead to try and get at the sheep. He may find the shepherds
+keeping watch over their flocks with dogs and spears, but he is
+in no mind to be driven from the fold till he has had a try for
+it; he will either spring on a sheep and carry it off, or be hit
+by a spear from some strong hand--even so was Sarpedon fain to
+attack the wall and break down its battlements. Then he said to
+Glaucus son of Hippolochus, "Glaucus, why in Lycia do we receive
+especial honour as regards our place at table? Why are the
+choicest portions served us and our cups kept brimming, and why
+do men look up to us as though we were gods? Moreover we hold a
+large estate by the banks of the river Xanthus, fair with orchard
+lawns and wheat-growing land; it becomes us, therefore, to take
+our stand at the head of all the Lycians and bear the brunt of
+the fight, that one may say to another, 'Our princes in Lycia eat
+the fat of the land and drink best of wine, but they are fine
+fellows; they fight well and are ever at the front in battle.' My
+good friend, if, when we were once out of this fight, we could
+escape old age and death thenceforward and forever, I should
+neither press forward myself nor bid you do so, but death in ten
+thousand shapes hangs ever over our heads, and no man can elude
+him; therefore let us go forward and either win glory for
+ourselves, or yield it to another."
+</para>
+
+<para>
+Glaucus heeded his saying, and the pair forthwith led on the host
+of Lycians. Menestheus son of Peteos was dismayed when he saw
+them, for it was against his part of the wall that they came--
+bringing destruction with them; he looked along the wall for some
+chieftain to support his comrades and saw the two Ajaxes, men
+ever eager for the fray, and Teucer, who had just come from his
+tent, standing near them; but he could not make his voice heard
+by shouting to them, so great an uproar was there from crashing
+shields and helmets and the battering of gates with a din which
+reached the skies. For all the gates had been closed, and the
+Trojans were hammering at them to try and break their way through
+them. Menestheus, therefore, sent Thootes with a message to Ajax.
+"Run, good Thootes," he said, "and call Ajax, or better still bid
+both come, for it will be all over with us here directly; the
+leaders of the Lycians are upon us, men who have ever fought
+desperately heretofore. But if they have too much on their hands
+to let them come, at any rate let Ajax son of Telamon do so, and
+let Teucer, the famous bowman, come with him."
+</para>
+
+<para>
+The messenger did as he was told, and set off running along the
+wall of the Achaeans. When he reached the Ajaxes he said to them,
+"Sirs, princes of the Argives, the son of noble Peteos bids you
+come to him for a while and help him. You had better both come if
+you can, or it will be all over with him directly; the leaders of
+the Lycians are upon him, men who have ever fought desperately
+heretofore; if you have too much on your hands to let both come,
+at any rate let Ajax, son of Telamon, do so, and let Teucer, the
+famous bowman, come with him."
+</para>
+
+<para>
+Great Ajax son of Telamon heeded the message, and at once spoke
+to the son of Oileus. "Ajax," said he, "do you two, yourself and
+brave Lycomedes, stay here and keep the Danaans in heart to fight
+their hardest. I will go over yonder, and bear my part in the
+fray, but I will come back here at once as soon as I have given
+them the help they need."
+</para>
+
+<para>
+With this, Ajax son of Telamon set off, and Teucer, his brother
+by the same father, went also, with Pandion to carry Teucer's
+bow. They went along inside the wall, and when they came to the
+tower where Menestheus was (and hard pressed indeed did they find
+him) the brave captains and leaders of the Lycians were storming
+the battlements as it were a thick dark cloud, fighting in close
+quarters, and raising the battle-cry aloud.
+</para>
+
+<para>
+First, Ajax son of Telamon killed brave Epicles, a comrade of
+Sarpedon, hitting him with a jagged stone that lay by the
+battlements at the very top of the wall. As men now are, even one
+who is in the bloom of youth could hardly lift it with his two
+hands, but Ajax raised it high aloft and flung it down, smashing
+Epicles' four-crested helmet so that the bones of his head were
+crushed to pieces, and he fell from the high wall as though he
+were diving, with no more life left in him. Then Teucer wounded
+Glaucus the brave son of Hippolochus as he was coming on to
+attack the wall. He saw his shoulder bare and aimed an arrow at
+it, which made Glaucus leave off fighting. Thereon he sprang
+covertly down for fear some of the Achaeans might see that he was
+wounded and taunt him. Sarpedon was stung with grief when he saw
+Glaucus leave him, still he did not leave off fighting, but aimed
+his spear at Alcmaon the son of Thestor and hit him. He drew his
+spear back again and Alcmaon came down headlong after it with his
+bronzed armour rattling round him. Then Sarpedon seized the
+battlement in his strong hands, and tugged at it till it all gave
+way together, and a breach was made through which many might
+pass.
+</para>
+
+<para>
+Ajax and Teucer then both of them attacked him. Teucer hit him
+with an arrow on the band that bore the shield which covered his
+body, but Jove saved his son from destruction that he might not
+fall by the ships' sterns. Meanwhile Ajax sprang on him and
+pierced his shield, but the spear did not go clean through,
+though it hustled him back that he could come on no further. He
+therefore retired a little space from the battlement, yet without
+losing all his ground, for he still thought to cover himself with
+glory. Then he turned round and shouted to the brave Lycians
+saying, "Lycians, why do you thus fail me? For all my prowess I
+cannot break through the wall and open a way to the ships
+single-handed. Come close on behind me, for the more there are of
+us the better."
+</para>
+
+<para>
+The Lycians, shamed by his rebuke, pressed closer round him who
+was their counsellor and their king. The Argives on their part
+got their men in fighting order within the wall, and there was a
+deadly struggle between them. The Lycians could not break through
+the wall and force their way to the ships, nor could the Danaans
+drive the Lycians from the wall now that they had once reached
+it. As two men, measuring-rods in hand, quarrel about their
+boundaries in a field that they own in common, and stickle for
+their rights though they be but in a mere strip, even so did the
+battlements now serve as a bone of contention, and they beat one
+another's round shields for their possession. Many a man's body
+was wounded with the pitiless bronze, as he turned round and
+bared his back to the foe, and many were struck clean through
+their shields; the wall and battlements were everywhere deluged
+with the blood alike of Trojans and of Achaeans. But even so the
+Trojans could not rout the Achaeans, who still held on; and as
+some honest hard-working woman weighs wool in her balance and
+sees that the scales be true, for she would gain some pitiful
+earnings for her little ones, even so was the fight balanced
+evenly between them till the time came when Jove gave the greater
+glory to Hector son of Priam, who was first to spring towards the
+wall of the Achaeans. When he had done so, he cried aloud to the
+Trojans, "Up, Trojans, break the wall of the Argives, and fling
+fire upon their ships."
+</para>
+
+<para>
+Thus did he hound them on, and in one body they rushed straight
+at the wall as he had bidden them, and scaled the battlements
+with sharp spears in their hands. Hector laid hold of a stone
+that lay just outside the gates and was thick at one end but
+pointed at the other; two of the best men in a town, as men now
+are, could hardly raise it from the ground and put it on to a
+waggon, but Hector lifted it quite easily by himself, for the son
+of scheming Saturn made it light for him. As a shepherd picks up
+a ram's fleece with one hand and finds it no burden, so easily
+did Hector lift the great stone and drive it right at the doors
+that closed the gates so strong and so firmly set. These doors
+were double and high, and were kept closed by two cross-bars to
+which there was but one key. When he had got close up to them,
+Hector strode towards them that his blow might gain in force and
+struck them in the middle, leaning his whole weight against them.
+He broke both hinges, and the stone fell inside by reason of its
+great weight. The portals re-echoed with the sound, the bars held
+no longer, and the doors flew open, one one way, and the other
+the other, through the force of the blow. Then brave Hector
+leaped inside with a face as dark as that of flying night. The
+gleaming bronze flashed fiercely about his body and he had two
+spears in his hand. None but a god could have withstood him as he
+flung himself into the gateway, and his eyes glared like fire.
+Then he turned round towards the Trojans and called on them to
+scale the wall, and they did as he bade them--some of them at
+once climbing over the wall, while others passed through the
+gates. The Danaans then fled panic-stricken towards their ships,
+and all was uproar and confusion.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XIII</title>
+
+<para>
+NOW when Jove had thus brought Hector and the Trojans to the
+ships, he left them to their never-ending toil, and turned his
+keen eyes away, looking elsewhither towards the horse-breeders of
+Thrace, the Mysians, fighters at close quarters, the noble
+Hippemolgi, who live on milk, and the Abians, justest of mankind.
+He no longer turned so much as a glance towards Troy, for he did
+not think that any of the immortals would go and help either
+Trojans or Danaans.
+</para>
+
+<para>
+But King Neptune had kept no blind look-out; he had been looking
+admiringly on the battle from his seat on the topmost crests of
+wooded Samothrace, whence he could see all Ida, with the city of
+Priam and the ships of the Achaeans. He had come from under the
+sea and taken his place here, for he pitied the Achaeans who were
+being overcome by the Trojans; and he was furiously angry with
+Jove.
+</para>
+
+<para>
+Presently he came down from his post on the mountain top, and as
+he strode swiftly onwards the high hills and the forest quaked
+beneath the tread of his immortal feet. Three strides he took,
+and with the fourth he reached his goal--Aegae, where is his
+glittering golden palace, imperishable, in the depths of the sea.
+When he got there, he yoked his fleet brazen-footed steeds with
+their manes of gold all flying in the wind; he clothed himself in
+raiment of gold, grasped his gold whip, and took his stand upon
+his chariot. As he went his way over the waves the sea-monsters
+left their lairs, for they knew their lord, and came gambolling
+round him from every quarter of the deep, while the sea in her
+gladness opened a path before his chariot. So lightly did the
+horses fly that the bronze axle of the car was not even wet
+beneath it; and thus his bounding steeds took him to the ships of
+the Achaeans.
+</para>
+
+<para>
+Now there is a certain huge cavern in the depths of the sea
+midway between Tenedos and rocky Imbrus; here Neptune lord of the
+earthquake stayed his horses, unyoked them, and set before them
+their ambrosial forage. He hobbled their feet with hobbles of
+gold which none could either unloose or break, so that they might
+stay there in that place until their lord should return. This
+done he went his way to the host of the Achaeans.
+</para>
+
+<para>
+Now the Trojans followed Hector son of Priam in close array like
+a storm-cloud or flame of fire, fighting with might and main and
+raising the cry battle; for they deemed that they should take the
+ships of the Achaeans and kill all their chiefest heroes then and
+there. Meanwhile earth-encircling Neptune lord of the earthquake
+cheered on the Argives, for he had come up out of the sea and had
+assumed the form and voice of Calchas.
+</para>
+
+<para>
+First he spoke to the two Ajaxes, who were doing their best
+already, and said, "Ajaxes, you two can be the saving of the
+Achaeans if you will put out all your strength and not let
+yourselves be daunted. I am not afraid that the Trojans, who have
+got over the wall in force, will be victorious in any other part,
+for the Achaeans can hold all of them in check, but I much fear
+that some evil will befall us here where furious Hector, who
+boasts himself the son of great Jove himself, is leading them on
+like a pillar of flame. May some god, then, put it into your
+hearts to make a firm stand here, and to incite others to do the
+like. In this case you will drive him from the ships even though
+he be inspired by Jove himself."
+</para>
+
+<para>
+As he spoke the earth-encircling lord of the earthquake struck
+both of them with his sceptre and filled their hearts with
+daring. He made their legs light and active, as also their hands
+and their feet. Then, as the soaring falcon poises on the wing
+high above some sheer rock, and presently swoops down to chase
+some bird over the plain, even so did Neptune lord of the
+earthquake wing his flight into the air and leave them. Of the
+two, swift Ajax son of Oileus was the first to know who it was
+that had been speaking with them, and said to Ajax son of
+Telamon, "Ajax, this is one of the gods that dwell on Olympus,
+who in the likeness of the prophet is bidding us fight hard by
+our ships. It was not Calchas the seer and diviner of omens; I
+knew him at once by his feet and knees as he turned away, for the
+gods are soon recognised. Moreover I feel the lust of battle burn
+more fiercely within me, while my hands and my feet under me are
+more eager for the fray."
+</para>
+
+<para>
+And Ajax son of Telamon answered, "I too feel my hands grasp my
+spear more firmly; my strength is greater, and my feet more
+nimble; I long, moreover, to meet furious Hector son of Priam,
+even in single combat."
+</para>
+
+<para>
+Thus did they converse, exulting in the hunger after battle with
+which the god had filled them. Meanwhile the earth-encircler
+roused the Achaeans, who were resting in the rear by the ships
+overcome at once by hard fighting and by grief at seeing that the
+Trojans had got over the wall in force. Tears began falling from
+their eyes as they beheld them, for they made sure that they
+should not escape destruction; but the lord of the earthquake
+passed lightly about among them and urged their battalions to the
+front.
+</para>
+
+<para>
+First he went up to Teucer and Leitus, the hero Peneleos, and
+Thoas and Deipyrus; Meriones also and Antilochus, valiant
+warriors; all did he exhort. "Shame on you young Argives," he
+cried, "it was on your prowess I relied for the saving of our
+ships; if you fight not with might and main, this very day will
+see us overcome by the Trojans. Of a truth my eyes behold a great
+and terrible portent which I had never thought to see--the
+Trojans at our ships--they, who were heretofore like
+panic-stricken hinds, the prey of jackals and wolves in a forest,
+with no strength but in flight for they cannot defend themselves.
+Hitherto the Trojans dared not for one moment face the attack of
+the Achaeans, but now they have sallied far from their city and
+are fighting at our very ships through the cowardice of our
+leader and the disaffection of the people themselves, who in
+their discontent care not to fight in defence of the ships but
+are being slaughtered near them. True, King Agamemnon son of
+Atreus is the cause of our disaster by having insulted the son of
+Peleus, still this is no reason why we should leave off fighting.
+Let us be quick to heal, for the hearts of the brave heal
+quickly. You do ill to be thus remiss, you, who are the finest
+soldiers in our whole army. I blame no man for keeping out of
+battle if he is a weakling, but I am indignant with such men as
+you are. My good friends, matters will soon become even worse
+through this slackness; think, each one of you, of his own honour
+and credit, for the hazard of the fight is extreme. Great Hector
+is now fighting at our ships; he has broken through the gates and
+the strong bolt that held them."
+</para>
+
+<para>
+Thus did the earth-encircler address the Achaeans and urge them
+on. Thereon round the two Ajaxes there gathered strong bands of
+men, of whom not even Mars nor Minerva, marshaller of hosts could
+make light if they went among them, for they were the picked men
+of all those who were now awaiting the onset of Hector and the
+Trojans. They made a living fence, spear to spear, shield to
+shield, buckler to buckler, helmet to helmet, and man to man. The
+horse-hair crests on their gleaming helmets touched one another
+as they nodded forward, so closely serried were they; the spears
+they brandished in their strong hands were interlaced, and their
+hearts were set on battle.
+</para>
+
+<para>
+The Trojans advanced in a dense body, with Hector at their head
+pressing right on as a rock that comes thundering down the side
+of some mountain from whose brow the winter torrents have torn
+it; the foundations of the dull thing have been loosened by
+floods of rain, and as it bounds headlong on its way it sets the
+whole forest in an uproar; it swerves neither to right nor left
+till it reaches level ground, but then for all its fury it can go
+no further--even so easily did Hector for a while seem as though
+he would career through the tents and ships of the Achaeans till
+he had reached the sea in his murderous course; but the closely
+serried battalions stayed him when he reached them, for the sons
+of the Achaeans thrust at him with swords and spears pointed at
+both ends, and drove him from them so that he staggered and gave
+ground; thereon he shouted to the Trojans, "Trojans, Lycians, and
+Dardanians, fighters in close combat, stand firm: the Achaeans
+have set themselves as a wall against me, but they will not check
+me for long; they will give ground before me if the mightiest of
+the gods, the thundering spouse of Juno, has indeed inspired my
+onset."
+</para>
+
+<para>
+With these words he put heart and soul into them all. Deiphobus
+son of Priam went about among them intent on deeds of daring with
+his round shield before him, under cover of which he strode
+quickly forward. Meriones took aim at him with a spear, nor did
+he fail to hit the broad orb of ox-hide; but he was far from
+piercing it for the spear broke in two pieces long ere he could
+do so; moreover Deiphobus had seen it coming and had held his
+shield well away from him. Meriones drew back under cover of his
+comrades, angry alike at having failed to vanquish Deiphobus, and
+having broken his spear. He turned therefore towards the ships
+and tents to fetch a spear which he had left behind in his tent.
+</para>
+
+<para>
+The others continued fighting, and the cry of battle rose up into
+the heavens. Teucer son of Telamon was the first to kill his man,
+to wit, the warrior Imbrius, son of Mentor, rich in horses.
+Until the Achaeans came he had lived in Pedaeum, and had married
+Medesicaste, a bastard daughter of Priam; but on the arrival of
+the Danaan fleet he had gone back to Ilius, and was a great man
+among the Trojans, dwelling near Priam himself, who gave him like
+honour with his own sons. The son of Telamon now struck him under
+the ear with a spear which he then drew back again, and Imbrius
+fell headlong as an ash-tree when it is felled on the crest of
+some high mountain beacon, and its delicate green foliage comes
+toppling down to the ground. Thus did he fall with his
+bronze-dight armour ringing harshly round him, and Teucer sprang
+forward with intent to strip him of his armour; but as he was
+doing so, Hector took aim at him with a spear. Teucer saw the
+spear coming and swerved aside, whereon it hit Amphimachus, son
+of Cteatus son of Actor, in the chest as he was coming into
+battle, and his armour rang rattling round him as he fell heavily
+to the ground. Hector sprang forward to take Amphimachus's helmet
+from off his temples, and in a moment Ajax threw a spear at him,
+but did not wound him, for he was encased all over in his
+terrible armour; nevertheless the spear struck the boss of his
+shield with such force as to drive him back from the two corpses,
+which the Achaeans then drew off. Stichius and Menestheus,
+captains of the Athenians, bore away Amphimachus to the host of
+the Achaeans, while the two brave and impetuous Ajaxes did the
+like by Imbrius. As two lions snatch a goat from the hounds that
+have it in their fangs, and bear it through thick brushwood high
+above the ground in their jaws, thus did the Ajaxes bear aloft
+the body of Imbrius, and strip it of its armour. Then the son of
+Oileus severed the head from the neck in revenge for the death of
+Amphimachus, and sent it whirling over the crowd as though it had
+been a ball, till it fell in the dust at Hector's feet.
+</para>
+
+<para>
+Neptune was exceedingly angry that his grandson Amphimachus
+should have fallen; he therefore went to the tents and ships of
+the Achaeans to urge the Danaans still further, and to devise
+evil for the Trojans. Idomeneus met him, as he was taking leave
+of a comrade, who had just come to him from the fight, wounded in
+the knee. His fellow-soldiers bore him off the field, and
+Idomeneus having given orders to the physicians went on to his
+tent, for he was still thirsting for battle. Neptune spoke in the
+likeness and with the voice of Thoas son of Andraemon who ruled
+the Aetolians of all Pleuron and high Calydon, and was honoured
+among his people as though he were a god. "Idomeneus," said he,
+"lawgiver to the Cretans, what has now become of the threats with
+which the sons of the Achaeans used to threaten the Trojans?"
+</para>
+
+<para>
+And Idomeneus chief among the Cretans answered, "Thoas, no one,
+so far as I know, is in fault, for we can all fight. None are
+held back neither by fear nor slackness, but it seems to be the
+will of almighty Jove that the Achaeans should perish
+ingloriously here far from Argos: you, Thoas, have been always
+staunch, and you keep others in heart if you see any fail in
+duty; be not then remiss now, but exhort all to do their utmost."
+</para>
+
+<para>
+To this Neptune lord of the earthquake made answer, "Idomeneus,
+may he never return from Troy, but remain here for dogs to batten
+upon, who is this day wilfully slack in fighting. Get your armour
+and go, we must make all haste together if we may be of any use,
+though we are only two. Even cowards gain courage from
+companionship, and we two can hold our own with the bravest."
+</para>
+
+<para>
+Therewith the god went back into the thick of the fight, and
+Idomeneus when he had reached his tent donned his armour, grasped
+his two spears, and sallied forth. As the lightning which the son
+of Saturn brandishes from bright Olympus when he would show a
+sign to mortals, and its gleam flashes far and wide--even so did
+his armour gleam about him as he ran. Meriones his sturdy squire
+met him while he was still near his tent (for he was going to
+fetch his spear) and Idomeneus said:
+</para>
+
+<para>
+"Meriones, fleet son of Molus, best of comrades, why have you
+left the field? Are you wounded, and is the point of the weapon
+hurting you? or have you been sent to fetch me? I want no
+fetching; I had far rather fight than stay in my tent."
+</para>
+
+<para>
+"Idomeneus," answered Meriones, "I come for a spear, if I can
+find one in my tent; I have broken the one I had, in throwing it
+at the shield of Deiphobus."
+</para>
+
+<para>
+And Idomeneus captain of the Cretans answered, "You will find one
+spear, or twenty if you so please, standing up against the end
+wall of my tent. I have taken them from Trojans whom I have
+killed, for I am not one to keep my enemy at arm's length;
+therefore I have spears, bossed shields, helmets, and burnished
+corslets."
+</para>
+
+<para>
+Then Meriones said, "I too in my tent and at my ship have spoils
+taken from the Trojans, but they are not at hand. I have been at
+all times valorous, and wherever there has been hard fighting
+have held my own among the foremost. There may be those among the
+Achaeans who do not know how I fight, but you know it well enough
+yourself."
+</para>
+
+<para>
+Idomeneus answered, "I know you for a brave man: you need not
+tell me. If the best men at the ships were being chosen to go on
+an ambush--and there is nothing like this for showing what a man
+is made of; it comes out then who is cowardly and who brave; the
+coward will change colour at every touch and turn; he is full of
+fears, and keeps shifting his weight first on one knee and then
+on the other; his heart beats fast as he thinks of death, and one
+can hear the chattering of his teeth; whereas the brave man will
+not change colour nor be frightened on finding himself in ambush,
+but is all the time longing to go into action--if the best men
+were being chosen for such a service, no one could make light of
+your courage nor feats of arms. If you were struck by a dart or
+smitten in close combat, it would not be from behind, in your
+neck nor back, but the weapon would hit you in the chest or belly
+as you were pressing forward to a place in the front ranks. But
+let us no longer stay here talking like children, lest we be ill
+spoken of; go, fetch your spear from the tent at once."
+</para>
+
+<para>
+On this Meriones, peer of Mars, went to the tent and got himself
+a spear of bronze. He then followed after Idomeneus, big with
+great deeds of valour. As when baneful Mars sallies forth to
+battle, and his son Panic so strong and dauntless goes with him,
+to strike terror even into the heart of a hero--the pair have
+gone from Thrace to arm themselves among the Ephyri or the brave
+Phlegyans, but they will not listen to both the contending hosts,
+and will give victory to one side or to the other--even so did
+Meriones and Idomeneus, captains of men, go out to battle clad in
+their bronze armour. Meriones was first to speak. "Son of
+Deucalion," said he, "where would you have us begin fighting? On
+the right wing of the host, in the centre, or on the left wing,
+where I take it the Achaeans will be weakest?"
+</para>
+
+<para>
+Idomeneus answered, "There are others to defend the centre--the
+two Ajaxes and Teucer, who is the finest archer of all the
+Achaeans, and is good also in a hand-to-hand fight. These will
+give Hector son of Priam enough to do; fight as he may, he will
+find it hard to vanquish their indomitable fury, and fire the
+ships, unless the son of Saturn fling a firebrand upon them with
+his own hand. Great Ajax son of Telamon will yield to no man who
+is in mortal mould and eats the grain of Ceres, if bronze and
+great stones can overthrow him. He would not yield even to
+Achilles in hand-to-hand fight, and in fleetness of foot there is
+none to beat him; let us turn therefore towards the left wing,
+that we may know forthwith whether we are to give glory to some
+other, or he to us."
+</para>
+
+<para>
+Meriones, peer of fleet Mars, then led the way till they came to
+the part of the host which Idomeneus had named.
+</para>
+
+<para>
+Now when the Trojans saw Idomeneus coming on like a flame of
+fire, him and his squire clad in their richly wrought armour,
+they shouted and made towards him all in a body, and a furious
+hand-to-hand fight raged under the ships' sterns. Fierce as the
+shrill winds that whistle upon a day when dust lies deep on the
+roads, and the gusts raise it into a thick cloud--even such was
+the fury of the combat, and might and main did they hack at each
+other with spear and sword throughout the host. The field
+bristled with the long and deadly spears which they bore.
+Dazzling was the sheen of their gleaming helmets, their
+fresh-burnished breastplates, and glittering shields as they
+joined battle with one another. Iron indeed must be his courage
+who could take pleasure in the sight of such a turmoil, and look
+on it without being dismayed.
+</para>
+
+<para>
+Thus did the two mighty sons of Saturn devise evil for mortal
+heroes. Jove was minded to give victory to the Trojans and to
+Hector, so as to do honour to fleet Achilles, nevertheless he did
+not mean to utterly overthrow the Achaean host before Ilius, and
+only wanted to glorify Thetis and her valiant son. Neptune on the
+other hand went about among the Argives to incite them, having
+come up from the grey sea in secret, for he was grieved at seeing
+them vanquished by the Trojans, and was furiously angry with
+Jove. Both were of the same race and country, but Jove was elder
+born and knew more, therefore Neptune feared to defend the
+Argives openly, but in the likeness of man, he kept on
+encouraging them throughout their host. Thus, then, did these two
+devise a knot of war and battle, that none could unloose or
+break, and set both sides tugging at it, to the failing of men's
+knees beneath them.
+</para>
+
+<para>
+And now Idomeneus, though his hair was already flecked with grey,
+called loud on the Danaans and spread panic among the Trojans as
+he leaped in among them. He slew Othryoneus from Cabesus, a
+sojourner, who had but lately come to take part in the war. He
+sought Cassandra, the fairest of Priam's daughters, in marriage,
+but offered no gifts of wooing, for he promised a great thing, to
+wit, that he would drive the sons of the Achaeans willy nilly
+from Troy; old King Priam had given his consent and promised her
+to him, whereon he fought on the strength of the promises thus
+made to him. Idomeneus aimed a spear, and hit him as he came
+striding on. His cuirass of bronze did not protect him, and the
+spear stuck in his belly, so that he fell heavily to the ground.
+Then Idomeneus vaunted over him saying, "Othryoneus, there is no
+one in the world whom I shall admire more than I do you, if you
+indeed perform what you have promised Priam son of Dardanus in
+return for his daughter. We too will make you an offer; we will
+give you the loveliest daughter of the son of Atreus, and will
+bring her from Argos for you to marry, if you will sack the
+goodly city of Ilius in company with ourselves; so come along
+with me, that we may make a covenant at the ships about the
+marriage, and we will not be hard upon you about gifts of
+wooing."
+</para>
+
+<para>
+With this Idomeneus began dragging him by the foot through the
+thick of the fight, but Asius came up to protect the body, on
+foot, in front of his horses which his esquire drove so close
+behind him that he could feel their breath upon his shoulder. He
+was longing to strike down Idomeneus, but ere he could do so
+Idomeneus smote him with his spear in the throat under the chin,
+and the bronze point went clean through it. He fell as an oak, or
+poplar, or pine which shipwrights have felled for ship's timber
+upon the mountains with whetted axes--even thus did he lie full
+length in front of his chariot and horses, grinding his teeth and
+clutching at the bloodstained dust. His charioteer was struck
+with panic and did not dare turn his horses round and escape:
+thereupon Antilochus hit him in the middle of his body with a
+spear; his cuirass of bronze did not protect him, and the spear
+stuck in his belly. He fell gasping from his chariot and
+Antilochus, great Nestor's son, drove his horses from the Trojans
+to the Achaeans.
+</para>
+
+<para>
+Deiphobus then came close up to Idomeneus to avenge Asius, and
+took aim at him with a spear, but Idomeneus was on the look-out
+and avoided it, for he was covered by the round shield he always
+bore--a shield of oxhide and bronze with two arm-rods on the
+inside. He crouched under cover of this, and the spear flew over
+him, but the shield rang out as the spear grazed it, and the
+weapon sped not in vain from the strong hand of Deiphobus, for it
+struck Hypsenor son of Hippasus, shepherd of his people, in the
+liver under the midriff, and his limbs failed beneath him.
+Deiphobus vaunted over him and cried with a loud voice saying,
+"Of a truth Asius has not fallen unavenged; he will be glad even
+while passing into the house of Hades, strong warden of the gate,
+that I have sent some one to escort him."
+</para>
+
+<para>
+Thus did he vaunt, and the Argives were stung by his saying.
+Noble Antilochus was more angry than any one, but grief did not
+make him forget his friend and comrade. He ran up to him,
+bestrode him, and covered him with his shield; then two of his
+staunch comrades, Mecisteus son of Echius, and Alastor, stooped
+down, and bore him away groaning heavily to the ships. But
+Idomeneus ceased not his fury. He kept on striving continually
+either to enshroud some Trojan in the darkness of death, or
+himself to fall while warding off the evil day from the Achaeans.
+Then fell Alcathous son of noble Aesyetes; he was son-in-law to
+Anchises, having married his eldest daughter Hippodameia, who was
+the darling of her father and mother, and excelled all her
+generation in beauty, accomplishments, and understanding,
+wherefore the bravest man in all Troy had taken her to wife--him
+did Neptune lay low by the hand of Idomeneus, blinding his bright
+eyes and binding his strong limbs in fetters so that he could
+neither go back nor to one side, but stood stock still like
+pillar or lofty tree when Idomeneus struck him with a spear in
+the middle of his chest. The coat of mail that had hitherto
+protected his body was now broken, and rang harshly as the spear
+tore through it. He fell heavily to the ground, and the spear
+stuck in his heart, which still beat, and made the butt-end of
+the spear quiver till dread Mars put an end to his life.
+Idomeneus vaunted over him and cried with a loud voice saying,
+"Deiphobus, since you are in a mood to vaunt, shall we cry quits
+now that we have killed three men to your one? Nay, sir, stand in
+fight with me yourself, that you may learn what manner of
+Jove-begotten man am I that have come hither. Jove first begot
+Minos, chief ruler in Crete, and Minos in his turn begot a son,
+noble Deucalion. Deucalion begot me to be a ruler over many men
+in Crete, and my ships have now brought me hither, to be the bane
+of yourself, your father, and the Trojans."
+</para>
+
+<para>
+Thus did he speak, and Deiphobus was in two minds, whether to go
+back and fetch some other Trojan to help him, or to take up the
+challenge single-handed. In the end, he deemed it best to go and
+fetch Aeneas, whom he found standing in the rear, for he had long
+been aggrieved with Priam because in spite of his brave deeds he
+did not give him his due share of honour. Deiphobus went up to
+him and said, "Aeneas, prince among the Trojans, if you know any
+ties of kinship, help me now to defend the body of your sister's
+husband; come with me to the rescue of Alcathous, who being
+husband to your sister brought you up when you were a child in
+his house, and now Idomeneus has slain him."
+</para>
+
+<para>
+With these words he moved the heart of Aeneas, and he went in
+pursuit of Idomeneus, big with great deeds of valour; but
+Idomeneus was not to be thus daunted as though he were a mere
+child; he held his ground as a wild boar at bay upon the
+mountains, who abides the coming of a great crowd of men in some
+lonely place--the bristles stand upright on his back, his eyes
+flash fire, and he whets his tusks in his eagerness to defend
+himself against hounds and men--even so did famed Idomeneus hold
+his ground and budge not at the coming of Aeneas. He cried aloud
+to his comrades looking towards Ascalaphus, Aphareus, Deipyrus,
+Meriones, and Antilochus, all of them brave soldiers--"Hither my
+friends," he cried, "and leave me not single-handed--I go in
+great fear by fleet Aeneas, who is coming against me, and is a
+redoubtable dispenser of death battle. Moreover he is in the
+flower of youth when a man's strength is greatest; if I was of
+the same age as he is and in my present mind, either he or I
+should soon bear away the prize of victory."
+</para>
+
+<para>
+On this, all of them as one man stood near him, shield on
+shoulder. Aeneas on the other side called to his comrades,
+looking towards Deiphobus, Paris, and Agenor, who were leaders of
+the Trojans along with himself, and the people followed them as
+sheep follow the ram when they go down to drink after they have
+been feeding, and the heart of the shepherd is glad--even so was
+the heart of Aeneas gladdened when he saw his people follow him.
+</para>
+
+<para>
+Then they fought furiously in close combat about the body of
+Alcathous, wielding their long spears; and the bronze armour
+about their bodies rang fearfully as they took aim at one another
+in the press of the fight, while the two heroes Aeneas and
+Idomeneus, peers of Mars, outvied everyone in their desire to
+hack at each other with sword and spear. Aeneas took aim first,
+but Idomeneus was on the lookout and avoided the spear, so that
+it sped from Aeneas' strong hand in vain, and fell quivering in
+the ground. Idomeneus meanwhile smote Oenomaus in the middle of
+his belly, and broke the plate of his corslet, whereon his bowels
+came gushing out and he clutched the earth in the palms of his
+hands as he fell sprawling in the dust. Idomeneus drew his spear
+out of the body, but could not strip him of the rest of his
+armour for the rain of darts that were showered upon him:
+moreover his strength was now beginning to fail him so that he
+could no longer charge, and could neither spring forward to
+recover his own weapon nor swerve aside to avoid one that was
+aimed at him; therefore, though he still defended himself in
+hand-to-hand fight, his heavy feet could not bear him swiftly out
+of the battle. Deiphobus aimed a spear at him as he was
+retreating slowly from the field, for his bitterness against him
+was as fierce as ever, but again he missed him, and hit
+Ascalaphus, the son of Mars; the spear went through his shoulder,
+and he clutched the earth in the palms of his hands as he fell
+sprawling in the dust.
+</para>
+
+<para>
+Grim Mars of awful voice did not yet know that his son had
+fallen, for he was sitting on the summits of Olympus under the
+golden clouds, by command of Jove, where the other gods were also
+sitting, forbidden to take part in the battle. Meanwhile men
+fought furiously about the body. Deiphobus tore the helmet from
+off his head, but Meriones sprang upon him, and struck him on the
+arm with a spear so that the visored helmet fell from his hand
+and came ringing down upon the ground. Thereon Meriones sprang
+upon him like a vulture, drew the spear from his shoulder, and
+fell back under cover of his men. Then Polites, own brother of
+Deiphobus passed his arms around his waist, and bore him away
+from the battle till he got to his horses that were standing in
+the rear of the fight with the chariot and their driver. These
+took him towards the city groaning and in great pain, with the
+blood flowing from his arm.
+</para>
+
+<para>
+The others still fought on, and the battle-cry rose to heaven
+without ceasing. Aeneas sprang on Aphareus son of Caletor, and
+struck him with a spear in his throat which was turned towards
+him; his head fell on one side, his helmet and shield came down
+along with him, and death, life's foe, was shed around him.
+Antilochus spied his chance, flew forward towards Thoon, and
+wounded him as he was turning round. He laid open the vein that
+runs all the way up the back to the neck; he cut this vein clean
+away throughout its whole course, and Thoon fell in the dust face
+upwards, stretching out his hands imploringly towards his
+comrades. Antilochus sprang upon him and stripped the armour from
+his shoulders, glaring round him fearfully as he did so. The
+Trojans came about him on every side and struck his broad and
+gleaming shield, but could not wound his body, for Neptune stood
+guard over the son of Nestor, though the darts fell thickly round
+him. He was never clear of the foe, but was always in the thick
+of the fight; his spear was never idle; he poised and aimed it in
+every direction, so eager was he to hit someone from a distance
+or to fight him hand to hand.
+</para>
+
+<para>
+As he was thus aiming among the crowd, he was seen by Adamas, son
+of Asius, who rushed towards him and struck him with a spear in
+the middle of his shield, but Neptune made its point without
+effect, for he grudged him the life of Antilochus. One half,
+therefore, of the spear stuck fast like a charred stake in
+Antilochus's shield, while the other lay on the ground. Adamas
+then sought shelter under cover of his men, but Meriones followed
+after and hit him with a spear midway between the private parts
+and the navel, where a wound is particualrly painful to wretched
+mortals. There did Meriones transfix him, and he writhed
+convulsively about the spear as some bull whom mountain herdsmen
+have bound with ropes of withes and are taking away perforce.
+Even so did he move convulsively for a while, but not for very
+long, till Meriones came up and drew the spear out of his body,
+and his eyes were veiled in darkness.
+</para>
+
+<para>
+Helenus then struck Deipyrus with a great Thracian sword, hitting
+him on the temple in close combat and tearing the helmet from his
+head; the helmet fell to the ground, and one of those who were
+fighting on the Achaean side took charge of it as it rolled at
+his feet, but the eyes of Deipyrus were closed in the darkness of
+death.
+</para>
+
+<para>
+On this Menelaus was grieved, and made menacingly towards
+Helenus, brandishing his spear; but Helenus drew his bow, and the
+two attacked one another at one and the same moment, the one with
+his spear, and the other with his bow and arrow. The son of Priam
+hit the breastplate of Menelaus's corslet, but the arrow glanced
+from off it. As black beans or pulse come pattering down on to a
+threshing-floor from the broad winnowing-shovel, blown by shrill
+winds and shaken by the shovel--even so did the arrow glance off
+and recoil from the shield of Menelaus, who in his turn wounded
+the hand with which Helenus carried his bow; the spear went right
+through his hand and stuck in the bow itself, so that to his life
+he retreated under cover of his men, with his hand dragging by
+his side--for the spear weighed it down till Agenor drew it out
+and bound the hand carefully up in a woollen sling which his
+esquire had with him.
+</para>
+
+<para>
+Pisander then made straight at Menelaus--his evil destiny luring
+him on to his doom, for he was to fall in fight with you, O
+Menelaus. When the two were hard by one another the spear of the
+son of Atreus turned aside and he missed his aim; Pisander then
+struck the shield of brave Menelaus but could not pierce it, for
+the shield stayed the spear and broke the shaft; nevertheless he
+was glad and made sure of victory; forthwith, however, the son of
+Atreus drew his sword and sprang upon him. Pisander then seized
+the bronze battle-axe, with its long and polished handle of olive
+wood that hung by his side under his shield, and the two made at
+one another. Pisander struck the peak of Menelaus's crested
+helmet just under the crest itself, and Menelaus hit Pisander as
+he was coming towards him, on the forehead, just at the rise of
+his nose; the bones cracked and his two gore-bedrabbled eyes fell
+by his feet in the dust. He fell backwards to the ground, and
+Menelaus set his heel upon him, stripped him of his armour, and
+vaunted over him saying, "Even thus shall you Trojans leave the
+ships of the Achaeans, proud and insatiate of battle though you
+be, nor shall you lack any of the disgrace and shame which you
+have heaped upon myself. Cowardly she-wolves that you are, you
+feared not the anger of dread Jove, avenger of violated
+hospitality, who will one day destroy your city; you stole my
+wedded wife and wickedly carried off much treasure when you were
+her guest, and now you would fling fire upon our ships, and kill
+our heroes. A day will come when, rage as you may, you shall be
+stayed. O father Jove, you, who they say art above all, both gods
+and men, in wisdom, and from whom all things that befall us do
+proceed, how can you thus favour the Trojans--men so proud and
+overweening, that they are never tired of fighting? All things
+pall after a while--sleep, love, sweet song, and stately dance--
+still these are things of which a man would surely have his fill
+rather than of battle, whereas it is of battle that the Trojans
+are insatiate."
+</para>
+
+<para>
+So saying Menelaus stripped the blood-stained armour from the
+body of Pisander, and handed it over to his men; then he again
+ranged himself among those who were in the front of the fight.
+</para>
+
+<para>
+Harpalion son of King Pylaemenes then sprang upon him; he had
+come to fight at Troy along with his father, but he did not go
+home again. He struck the middle of Menelaus's shield with his
+spear but could not pierce it, and to save his life drew back
+under cover of his men, looking round him on every side lest he
+should be wounded. But Meriones aimed a bronze-tipped arrow at
+him as he was leaving the field, and hit him on the right
+buttock; the arrow pierced the bone through and through, and
+penetrated the bladder, so he sat down where he was and breathed
+his last in the arms of his comrades, stretched like a worm upon
+the ground and watering the earth with the blood that flowed from
+his wound. The brave Paphlagonians tended him with all due care;
+they raised him into his chariot, and bore him sadly off to the
+city of Troy; his father went also with him weeping bitterly, but
+there was no ransom that could bring his dead son to life again.
+</para>
+
+<para>
+Paris was deeply grieved by the death of Harpalion, who was his
+host when he went among the Paphlagonians; he aimed an arrow,
+therefore, in order to avenge him. Now there was a certain man
+named Euchenor, son of Polyidus the prophet, a brave man and
+wealthy, whose home was in Corinth. This Euchenor had set sail
+for Troy well knowing that it would be the death of him, for his
+good old father Polyidus had often told him that he must either
+stay at home and die of a terrible disease, or go with the
+Achaeans and perish at the hands of the Trojans; he chose,
+therefore, to avoid incurring the heavy fine the Achaeans would
+have laid upon him, and at the same time to escape the pain and
+suffering of disease. Paris now smote him on the jaw under his
+ear, whereon the life went out of him and he was enshrouded in
+the darkness of death.
+</para>
+
+<para>
+Thus then did they fight as it were a flaming fire. But Hector
+had not yet heard, and did not know that the Argives were making
+havoc of his men on the left wing of the battle, where the
+Achaeans ere long would have triumphed over them, so vigorously
+did Neptune cheer them on and help them. He therefore held on at
+the point where he had first forced his way through the gates and
+the wall, after breaking through the serried ranks of Danaan
+warriors. It was here that the ships of Ajax and Protesilaus were
+drawn up by the sea-shore; here the wall was at its lowest, and
+the fight both of man and horse raged most fiercely. The
+Boeotians and the Ionians with their long tunics, the Locrians,
+the men of Phthia, and the famous force of the Epeans could
+hardly stay Hector as he rushed on towards the ships, nor could
+they drive him from them, for he was as a wall of fire. The
+chosen men of the Athenians were in the van, led by Menestheus
+son of Peteos, with whom were also Pheidas, Stichius, and
+stalwart Bias; Meges son of Phyleus, Amphion, and Dracius
+commanded the Epeans, while Medon and staunch Podarces led the
+men of Phthia. Of these, Medon was bastard son to Oileus and
+brother of Ajax, but he lived in Phylace away from his own
+country, for he had killed the brother of his stepmother Eriopis,
+the wife of Oileus; the other, Podarces, was the son of Iphiclus,
+son of Phylacus. These two stood in the van of the Phthians, and
+defended the ships along with the Boeotians.
+</para>
+
+<para>
+Ajax son of Oileus, never for a moment left the side of Ajax, son
+of Telamon, but as two swart oxen both strain their utmost at the
+plough which they are drawing in a fallow field, and the sweat
+steams upwards from about the roots of their horns--nothing but
+the yoke divides them as they break up the ground till they reach
+the end of the field--even so did the two Ajaxes stand shoulder
+to shoulder by one another. Many and brave comrades followed the
+son of Telamon, to relieve him of his shield when he was overcome
+with sweat and toil, but the Locrians did not follow so close
+after the son of Oileus, for they could not hold their own in a
+hand-to-hand fight. They had no bronze helmets with plumes of
+horse-hair, neither had they shields nor ashen spears, but they
+had come to Troy armed with bows, and with slings of twisted wool
+from which they showered their missiles to break the ranks of the
+Trojans. The others, therefore, with their heavy armour bore the
+brunt of the fight with the Trojans and with Hector, while the
+Locrians shot from behind, under their cover; and thus the
+Trojans began to lose heart, for the arrows threw them into
+confusion.
+</para>
+
+<para>
+The Trojans would now have been driven in sorry plight from the
+ships and tents back to windy Ilius, had not Polydamas presently
+said to Hector, "Hector, there is no persuading you to take
+advice. Because heaven has so richly endowed you with the arts of
+war, you think that you must therefore excel others in counsel;
+but you cannot thus claim preeminence in all things. Heaven has
+made one man an excellent soldier; of another it has made a
+dancer or a singer and player on the lyre; while yet in another
+Jove has implanted a wise understanding of which men reap fruit
+to the saving of many, and he himself knows more about it than
+any one; therefore I will say what I think will be best. The
+fight has hemmed you in as with a circle of fire, and even now
+that the Trojans are within the wall some of them stand aloof in
+full armour, while others are fighting scattered and outnumbered
+near the ships. Draw back, therefore, and call your chieftains
+round you, that we may advise together whether to fall now upon
+the ships in the hope that heaven may vouchsafe us victory, or to
+beat a retreat while we can yet safely do so. I greatly fear that
+the Achaeans will pay us their debt of yesterday in full, for
+there is one abiding at their ships who is never weary of battle,
+and who will not hold aloof much longer."
+</para>
+
+<para>
+Thus spoke Polydamas, and his words pleased Hector well. He
+sprang in full armour from his chariot and said, "Polydamas,
+gather the chieftains here; I will go yonder into the fight, but
+will return at once when I have given them their orders."
+</para>
+
+<para>
+He then sped onward, towering like a snowy mountain, and with a
+loud cry flew through the ranks of the Trojans and their allies.
+When they heard his voice they all hastened to gather round
+Polydamas, the excellent son of Panthous, but Hector kept on
+among the foremost, looking everywhere to find Deiphobus and
+prince Helenus, Adamas son of Asius, and Asius son of Hyrtacus;
+living, indeed, and scatheless he could no longer find them, for
+the two last were lying by the sterns of the Achaean ships, slain
+by the Argives, while the others had been also stricken and
+wounded by them; but upon the left wing of the dread battle he
+found Alexandrus, husband of lovely Helen, cheering his men and
+urging them on to fight. He went up to him and upbraided him.
+"Paris," said he, "evil-hearted Paris, fair to see but woman-mad
+and false of tongue, where are Deiphobus and King Helenus? Where
+are Adamas son of Asius, and Asius son of Hyrtacus? Where too is
+Othryoneus? Ilius is undone and will now surely fall!"
+</para>
+
+<para>
+Alexandrus answered, "Hector, why find fault when there is no one
+to find fault with? I should hold aloof from battle on any day
+rather than this, for my mother bore me with nothing of the
+coward about me. From the moment when you set our men fighting
+about the ships we have been staying here and doing battle with
+the Danaans. Our comrades about whom you ask me are dead;
+Deiphobus and King Helenus alone have left the field, wounded
+both of them in the hand, but the son of Saturn saved them alive.
+Now, therefore, lead on where you would have us go, and we will
+follow with right goodwill; you shall not find us fail you in so
+far as our strength holds out, but no man can do more than in him
+lies, no matter how willing he may be."
+</para>
+
+<para>
+With these words he satisfied his brother, and the two went
+towards the part of the battle where the fight was thickest,
+about Cebriones, brave Polydamas, Phalces, Orthaeus, godlike
+Polyphetes, Palmys, Ascanius, and Morys son of Hippotion, who had
+come from fertile Ascania on the preceding day to relieve other
+troops. Then Jove urged them on to fight. They flew forth like
+the blasts of some fierce wind that strike earth in the van of a
+thunderstorm--they buffet the salt sea into an uproar; many and
+mighty are the great waves that come crashing in one after the
+other upon the shore with their arching heads all crested with
+foam--even so did rank behind rank of Trojans arrayed in gleaming
+armour follow their leaders onward. The way was led by Hector son
+of Priam, peer of murderous Mars, with his round shield before
+him--his shield of ox-hides covered with plates of bronze--and
+his gleaming helmet upon his temples. He kept stepping forward
+under cover of his shield in every direction, making trial of the
+ranks to see if they would give way before him, but he could not
+daunt the courage of the Achaeans. Ajax was the first to stride
+out and challenge him. "Sir," he cried, "draw near; why do you
+think thus vainly to dismay the Argives? We Achaeans are
+excellent soldiers, but the scourge of Jove has fallen heavily
+upon us. Your heart, forsooth, is set on destroying our ships,
+but we too have hands that can keep you at bay, and your own fair
+town shall be sooner taken and sacked by ourselves. The time is
+near when you shall pray Jove and all the gods in your flight,
+that your steeds may be swifter than hawks as they raise the dust
+on the plain and bear you back to your city."
+</para>
+
+<para>
+As he was thus speaking a bird flew by upon his right hand, and
+the host of the Achaeans shouted, for they took heart at the
+omen. But Hector answered, "Ajax, braggart and false of tongue,
+would that I were as sure of being son for evermore to
+aegis-bearing Jove, with Queen Juno for my mother, and of being
+held in like honour with Minerva and Apollo, as I am that this
+day is big with the destruction of the Achaeans; and you shall
+fall among them if you dare abide my spear; it shall rend your
+fair body and bid you glut our hounds and birds of prey with your
+fat and your flesh, as you fall by the ships of the Achaeans."
+</para>
+
+<para>
+With these words he led the way and the others followed after
+with a cry that rent the air, while the host shouted behind them.
+The Argives on their part raised a shout likewise, nor did they
+forget their prowess, but stood firm against the onslaught of the
+Trojan chieftains, and the cry from both the hosts rose up to
+heaven and to the brightness of Jove's presence.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XIV</title>
+
+<para>
+NESTOR was sitting over his wine, but the cry of battle did not
+escape him, and he said to the son of Aesculapius, "What, noble
+Machaon, is the meaning of all this? The shouts of men fighting
+by our ships grow stronger and stronger; stay here, therefore,
+and sit over your wine, while fair Hecamede heats you a bath and
+washes the clotted blood from off you. I will go at once to the
+look-out station and see what it is all about."
+</para>
+
+<para>
+As he spoke he took up the shield of his son Thrasymedes that was
+lying in his tent, all gleaming with bronze, for Thrasymedes had
+taken his father's shield; he grasped his redoubtable bronze-shod
+spear, and as soon as he was outside saw the disastrous rout of
+the Achaeans who, now that their wall was overthrown, were flying
+pell-mell before the Trojans. As when there is a heavy swell upon
+the sea, but the waves are dumb--they keep their eyes on the
+watch for the quarter whence the fierce winds may spring upon
+them, but they stay where they are and set neither this way nor
+that, till some particular wind sweeps down from heaven to
+determine them--even so did the old man ponder whether to make
+for the crowd of Danaans, or go in search of Agamemnon. In the
+end he deemed it best to go to the son of Atreus; but meanwhile
+the hosts were fighting and killing one another, and the hard
+bronze rattled on their bodies, as they thrust at one another
+with their swords and spears.
+</para>
+
+<para>
+The wounded kings, the son of Tydeus, Ulysses, and Agamemnon son
+of Atreus, fell in Nestor as they were coming up from their
+ships--for theirs were drawn up some way from where the fighting
+was going on, being on the shore itself inasmuch as they had been
+beached first, while the wall had been built behind the
+hindermost. The stretch of the shore, wide though it was, did not
+afford room for all the ships, and the host was cramped for
+space, therefore they had placed the ships in rows one behind the
+other, and had filled the whole opening of the bay between the
+two points that formed it. The kings, leaning on their spears,
+were coming out to survey the fight, being in great anxiety, and
+when old Nestor met them they were filled with dismay. Then King
+Agamemnon said to him, "Nestor son of Neleus, honour to the
+Achaean name, why have you left the battle to come hither? I fear
+that what dread Hector said will come true, when he vaunted among
+the Trojans saying that he would not return to Ilius till he had
+fired our ships and killed us; this is what he said, and now it
+is all coming true. Alas! others of the Achaeans, like Achilles,
+are in anger with me that they refuse to fight by the sterns of
+our ships."
+</para>
+
+<para>
+Then Nestor knight of Gerene, answered, "It is indeed as you say;
+it is all coming true at this moment, and even Jove who thunders
+from on high cannot prevent it. Fallen is the wall on which we
+relied as an impregnable bulwark both for us and our fleet. The
+Trojans are fighting stubbornly and without ceasing at the ships;
+look where you may you cannot see from what quarter the rout of
+the Achaeans is coming; they are being killed in a confused mass
+and the battle-cry ascends to heaven; let us think, if counsel
+can be of any use, what we had better do; but I do not advise our
+going into battle ourselves, for a man cannot fight when he is
+wounded."
+</para>
+
+<para>
+And King Agamemnon answered, "Nestor, if the Trojans are indeed
+fighting at the rear of our ships, and neither the wall nor the
+trench has served us--over which the Danaans toiled so hard, and
+which they deemed would be an impregnable bulwark both for us and
+our fleet--I see it must be the will of Jove that the Achaeans
+should perish ingloriously here, far from Argos. I knew when Jove
+was willing to defend us, and I know now that he is raising the
+Trojans to like honour with the gods, while us, on the other
+hand, he bas bound hand and foot. Now, therefore, let us all do
+as I say; let us bring down the ships that are on the beach and
+draw them into the water; let us make them fast to their
+mooring-stones a little way out, against the fall of night--if
+even by night the Trojans will desist from fighting; we may then
+draw down the rest of the fleet. There is nothing wrong in flying
+ruin even by night. It is better for a man that he should fly and
+be saved than be caught and killed."
+</para>
+
+<para>
+Ulysses looked fiercely at him and said, "Son of Atreus, what are
+you talking about? Wretch, you should have commanded some other
+and baser army, and not been ruler over us to whom Jove has
+allotted a life of hard fighting from youth to old age, till we
+every one of us perish. Is it thus that you would quit the city
+of Troy, to win which we have suffered so much hardship? Hold
+your peace, lest some other of the Achaeans hear you say what no
+man who knows how to give good counsel, no king over so great a
+host as that of the Argives should ever have let fall from his
+lips. I despise your judgement utterly for what you have been
+saying. Would you, then, have us draw down our ships into the
+water while the battle is raging, and thus play further into the
+hands of the conquering Trojans? It would be ruin; the Achaeans
+will not go on fighting when they see the ships being drawn into
+the water, but will cease attacking and keep turning their eyes
+towards them; your counsel, therefore, sir captain, would be our
+destruction."
+</para>
+
+<para>
+Agamemnon answered, "Ulysses, your rebuke has stung me to the
+heart. I am not, however, ordering the Achaeans to draw their
+ships into the sea whether they will or no. Someone, it may be,
+old or young, can offer us better counsel which I shall rejoice
+to hear."
+</para>
+
+<para>
+Then said Diomed, "Such an one is at hand; he is not far to seek,
+if you will listen to me and not resent my speaking though I am
+younger than any of you. I am by lineage son to a noble sire,
+Tydeus, who lies buried at Thebes. For Portheus had three noble
+sons, two of whom, Agrius and Melas, abode in Pleuron and rocky
+Calydon. The third was the knight Oeneus, my father's father, and
+he was the most valiant of them all. Oeneus remained in his own
+country, but my father (as Jove and the other gods ordained it)
+migrated to Argos. He married into the family of Adrastus, and
+his house was one of great abundance, for he had large estates of
+rich corn-growing land, with much orchard ground as well, and he
+had many sheep; moreover he excelled all the Argives in the use
+of the spear. You must yourselves have heard whether these things
+are true or no; therefore when I say well despise not my words as
+though I were a coward or of ignoble birth. I say, then, let us
+go to the fight as we needs must, wounded though we be. When
+there, we may keep out of the battle and beyond the range of the
+spears lest we get fresh wounds in addition to what we have
+already, but we can spur on others, who have been indulging their
+spleen and holding aloof from battle hitherto."
+</para>
+
+<para>
+Thus did he speak; whereon they did even as he had said and set
+out, King Agamemnon leading the way.
+</para>
+
+<para>
+Meanwhile Neptune had kept no blind look-out, and came up to them
+in the semblance of an old man. He took Agamemnon's right hand in
+his own and said, "Son of Atreus, I take it Achilles is glad now
+that he sees the Achaeans routed and slain, for he is utterly
+without remorse--may he come to a bad end and heaven confound
+him. As for yourself, the blessed gods are not yet so bitterly
+angry with you but that the princes and counsellors of the
+Trojans shall again raise the dust upon the plain, and you shall
+see them flying from the ships and tents towards their city."
+</para>
+
+<para>
+With this he raised a mighty cry of battle, and sped forward to
+the plain. The voice that came from his deep chest was as that of
+nine or ten thousand men when they are shouting in the thick of a
+fight, and it put fresh courage into the hearts of the Achaeans
+to wage war and do battle without ceasing.
+</para>
+
+<para>
+Juno of the golden throne looked down as she stood upon a peak of
+Olympus and her heart was gladdened at the sight of him who was
+at once her brother and her brother-in-law, hurrying hither and
+thither amid the fighting. Then she turned her eyes to Jove as he
+sat on the topmost crests of many-fountained Ida, and loathed
+him. She set herself to think how she might hoodwink him, and in
+the end she deemed that it would be best for her to go to Ida and
+array herself in rich attire, in the hope that Jove might become
+enamoured of her, and wish to embrace her. While he was thus
+engaged a sweet and careless sleep might be made to steal over
+his eyes and senses.
+</para>
+
+<para>
+She went, therefore, to the room which her son Vulcan had made
+her, and the doors of which he had cunningly fastened by means of
+a secret key so that no other god could open them. Here she
+entered and closed the doors behind her. She cleansed all the
+dirt from her fair body with ambrosia, then she anointed herself
+with olive oil, ambrosial, very soft, and scented specially for
+herself--if it were so much as shaken in the bronze-floored house
+of Jove, the scent pervaded the universe of heaven and earth.
+With this she anointed her delicate skin, and then she plaited
+the fair ambrosial locks that flowed in a stream of golden
+tresses from her immortal head. She put on the wondrous robe
+which Minerva had worked for her with consummate art, and had
+embroidered with manifold devices; she fastened it about her
+bosom with golden clasps, and she girded herself with a girdle
+that had a hundred tassels: then she fastened her earrings, three
+brilliant pendants that glistened most beautifully, through the
+pierced lobes of her ears, and threw a lovely new veil over her
+head. She bound her sandals on to her feet, and when she had
+arrayed herself perfectly to her satisfaction, she left her room
+and called Venus to come aside and speak to her. "My dear child,"
+said she, "will you do what I am going to ask of you, or will
+refuse me because you are angry at my being on the Danaan side,
+while you are on the Trojan?"
+</para>
+
+<para>
+Jove's daughter Venus answered, "Juno, august queen of goddesses,
+daughter of mighty Saturn, say what you want, and I will do it
+for you at once, if I can, and if it can be done at all."
+</para>
+
+<para>
+Then Juno told her a lying tale and said, "I want you to endow me
+with some of those fascinating charms, the spells of which bring
+all things mortal and immortal to your feet. I am going to the
+world's end to visit Oceanus (from whom all we gods proceed) and
+mother Tethys: they received me in their house, took care of me,
+and brought me up, having taken me over from Rhaea when Jove
+imprisoned great Saturn in the depths that are under earth and
+sea. I must go and see them that I may make peace between them;
+they have been quarrelling, and are so angry that they have not
+slept with one another this long while; if I can bring them round
+and restore them to one another's embraces, they will be grateful
+to me and love me for ever afterwards."
+</para>
+
+<para>
+Thereon laughter-loving Venus said, "I cannot and must not refuse
+you, for you sleep in the arms of Jove who is our king."
+</para>
+
+<para>
+As she spoke she loosed from her bosom the curiously embroidered
+girdle into which all her charms had been wrought--love, desire,
+and that sweet flattery which steals the judgement even of the
+most prudent. She gave the girdle to Juno and said, "Take this
+girdle wherein all my charms reside and lay it in your bosom. If
+you will wear it I promise you that your errand, be it what it
+may, will not be bootless."
+</para>
+
+<para>
+When she heard this Juno smiled, and still smiling she laid the
+girdle in her bosom.
+</para>
+
+<para>
+Venus now went back into the house of Jove, while Juno darted
+down from the summits of Olympus. She passed over Pieria and fair
+Emathia, and went on and on till she came to the snowy ranges of
+the Thracian horsemen, over whose topmost crests she sped without
+ever setting foot to ground. When she came to Athos she went on
+over the, waves of the sea till she reached Lemnos, the city of
+noble Thoas. There she met Sleep, own brother to Death, and
+caught him by the hand, saying, "Sleep, you who lord it alike
+over mortals and immortals, if you ever did me a service in times
+past, do one for me now, and I shall be grateful to you ever
+after. Close Jove's keen eyes for me in slumber while I hold him
+clasped in my embrace, and I will give you a beautiful golden
+seat, that can never fall to pieces; my clubfooted son Vulcan
+shall make it for you, and he shall give it a footstool for you
+to rest your fair feet upon when you are at table."
+</para>
+
+<para>
+Then Sleep answered, "Juno, great queen of goddesses, daughter of
+mighty Saturn, I would lull any other of the gods to sleep
+without compunction, not even excepting the waters of Oceanus
+from whom all of them proceed, but I dare not go near Jove, nor
+send him to sleep unless he bids me. I have had one lesson
+already through doing what you asked me, on the day when Jove's
+mighty son Hercules set sail from Ilius after having sacked the
+city of the Trojans. At your bidding I suffused my sweet self
+over the mind of aegis-bearing Jove, and laid him to rest;
+meanwhile you hatched a plot against Hercules, and set the blasts
+of the angry winds beating upon the sea, till you took him to the
+goodly city of Cos, away from all his friends. Jove was furious
+when he awoke, and began hurling the gods about all over the
+house; he was looking more particularly for myself, and would
+have flung me down through space into the sea where I should
+never have been heard of any more, had not Night who cows both
+men and gods protected me. I fled to her and Jove left off
+looking for me in spite of his being so angry, for he did not
+dare do anything to displease Night. And now you are again asking
+me to do something on which I cannot venture."
+</para>
+
+<para>
+And Juno said, "Sleep, why do you take such notions as those into
+your head? Do you think Jove will be as anxious to help the
+Trojans, as he was about his own son? Come, I will marry you to
+one of the youngest of the Graces, and she shall be your own--
+Pasithea, whom you have always wanted to marry."
+</para>
+
+<para>
+Sleep was pleased when he heard this, and answered, "Then swear
+it to me by the dread waters of the river Styx; lay one hand on
+the bounteous earth, and the other on the sheen of the sea, so
+that all the gods who dwell down below with Saturn may be our
+witnesses, and see that you really do give me one of the youngest
+of the Graces--Pasithea, whom I have always wanted to marry."
+</para>
+
+<para>
+Juno did as he had said. She swore, and invoked all the gods of
+the nether world, who are called Titans, to witness. When she had
+completed her oath, the two enshrouded themselves in a thick mist
+and sped lightly forward, leaving Lemnos and Imbrus behind them.
+Presently they reached many-fountained Ida, mother of wild
+beasts, and Lectum where they left the sea to go on by land, and
+the tops of the trees of the forest soughed under the going of
+their feet. Here Sleep halted, and ere Jove caught sight of him
+he climbed a lofty pine-tree--the tallest that reared its head
+towards heaven on all Ida. He hid himself behind the branches and
+sat there in the semblance of the sweet-singing bird that haunts
+the mountains and is called Chalcis by the gods, but men call it
+Cymindis. Juno then went to Gargarus, the topmost peak of Ida,
+and Jove, driver of the clouds, set eyes upon her. As soon as he
+did so he became inflamed with the same passionate desire for her
+that he had felt when they had first enjoyed each other's
+embraces, and slept with one another without their dear parents
+knowing anything about it. He went up to her and said, "What do
+you want that you have come hither from Olympus--and that too
+with neither chariot nor horses to convey you?"
+</para>
+
+<para>
+Then Juno told him a lying tale and said, "I am going to the
+world's end, to visit Oceanus, from whom all we gods proceed, and
+mother Tethys; they received me into their house, took care of
+me, and brought me up. I must go and see them that I may make
+peace between them: they have been quarrelling, and are so angry
+that they have not slept with one another this long time. The
+horses that will take me over land and sea are stationed on the
+lowermost spurs of many-fountained Ida, and I have come here from
+Olympus on purpose to consult you. I was afraid you might be
+angry with me later on, if I went to the house of Oceanus without
+letting you know."
+</para>
+
+<para>
+And Jove said, "Juno, you can choose some other time for paying
+your visit to Oceanus--for the present let us devote ourselves to
+love and to the enjoyment of one another. Never yet have I been
+so overpowered by passion neither for goddess nor mortal woman as
+I am at this moment for yourself--not even when I was in love
+with the wife of Ixion who bore me Pirithous, peer of gods in
+counsel, nor yet with Danae the daintily-ancled daughter of
+Acrisius, who bore me the famed hero Perseus. Then there was the
+daughter of Phoenix, who bore me Minos and Rhadamanthus: there
+was Semele, and Alcmena in Thebes by whom I begot my lion-hearted
+son Hercules, while Semele became mother to Bacchus the comforter
+of mankind. There was queen Ceres again, and lovely Leto, and
+yourself--but with none of these was I ever so much enamoured as
+I now am with you."
+</para>
+
+<para>
+Juno again answered him with a lying tale. "Most dread son of
+Saturn," she exclaimed, "what are you talking about? Would you
+have us enjoy one another here on the top of Mount Ida, where
+everything can be seen? What if one of the ever-living gods
+should see us sleeping together, and tell the others? It would be
+such a scandal that when I had risen from your embraces I could
+never show myself inside your house again; but if you are so
+minded, there is a room which your son Vulcan has made me, and he
+has given it good strong doors; if you would so have it, let us
+go thither and lie down."
+</para>
+
+<para>
+And Jove answered, "Juno, you need not be afraid that either god
+or man will see you, for I will enshroud both of us in such a
+dense golden cloud, that the very sun for all his bright piercing
+beams shall not see through it."
+</para>
+
+<para>
+With this the son of Saturn caught his wife in his embrace;
+whereon the earth sprouted them a cushion of young grass, with
+dew-bespangled lotus, crocus, and hyacinth, so soft and thick
+that it raised them well above the ground. Here they laid
+themselves down and overhead they were covered by a fair cloud of
+gold, from which there fell glittering dew-drops.
+</para>
+
+<para>
+Thus, then, did the sire of all things repose peacefully on the
+crest of Ida, overcome at once by sleep and love, and he held his
+spouse in his arms. Meanwhile Sleep made off to the ships of the
+Achaeans, to tell earth-encircling Neptune, lord of the
+earthquake. When he had found him he said, "Now, Neptune, you can
+help the Danaans with a will, and give them victory though it be
+only for a short time while Jove is still sleeping. I have sent
+him into a sweet slumber, and Juno has beguiled him into going to
+bed with her."
+</para>
+
+<para>
+Sleep now departed and went his ways to and fro among mankind,
+leaving Neptune more eager than ever to help the Danaans. He
+darted forward among the first ranks and shouted saying,
+"Argives, shall we let Hector son of Priam have the triumph of
+taking our ships and covering himself with glory? This is what he
+says that he shall now do, seeing that Achilles is still in
+dudgeon at his ship; we shall get on very well without him if we
+keep each other in heart and stand by one another. Now,
+therefore, let us all do as I say. Let us each take the best and
+largest shield we can lay hold of, put on our helmets, and sally
+forth with our longest spears in our hands; I will lead you on,
+and Hector son of Priam, rage as he may, will not dare to hold
+out against us. If any good staunch soldier has only a small
+shield, let him hand it over to a worse man, and take a larger
+one for himself."
+</para>
+
+<para>
+Thus did he speak, and they did even as he had said. The son of
+Tydeus, Ulysses, and Agamemnon, wounded though they were, set the
+others in array, and went about everywhere effecting the
+exchanges of armour; the most valiant took the best armour, and
+gave the worse to the worse man. When they had donned their
+bronze armour they marched on with Neptune at their head. In his
+strong hand he grasped his terrible sword, keen of edge and
+flashing like lightning; woe to him who comes across it in the
+day of battle; all men quake for fear and keep away from it.
+</para>
+
+<para>
+Hector on the other side set the Trojans in array. Thereon
+Neptune and Hector waged fierce war on one another--Hector on the
+Trojan and Neptune on the Argive side. Mighty was the uproar as
+the two forces met; the sea came rolling in towards the ships and
+tents of the Achaeans, but waves do not thunder on the shore more
+loudly when driven before the blast of Boreas, nor do the flames
+of a forest fire roar more fiercely when it is well alight upon
+the mountains, nor does the wind bellow with ruder music as it
+tears on through the tops of when it is blowing its hardest, than
+the terrible shout which the Trojans and Achaeans raised as they
+sprang upon one another.
+</para>
+
+<para>
+Hector first aimed his spear at Ajax, who was turned full towards
+him, nor did he miss his aim. The spear struck him where two
+bands passed over his chest--the band of his shield and that of
+his silver-studded sword--and these protected his body. Hector
+was angry that his spear should have been hurled in vain, and
+withdrew under cover of his men. As he was thus retreating, Ajax
+son of Telamon, struck him with a stone, of which there were many
+lying about under the men's feet as they fought--brought there to
+give support to the ships' sides as they lay on the shore. Ajax
+caught up one of them and struck Hector above the rim of his
+shield close to his neck; the blow made him spin round like a top
+and reel in all directions. As an oak falls headlong when
+uprooted by the lightning flash of father Jove, and there is a
+terrible smell of brimstone--no man can help being dismayed if he
+is standing near it, for a thunderbolt is a very awful thing--
+even so did Hector fall to earth and bite the dust. His spear
+fell from his hand, but his shield and helmet were made fast
+about his body, and his bronze armour rang about him.
+</para>
+
+<para>
+The sons of the Achaeans came running with a loud cry towards
+him, hoping to drag him away, and they showered their darts on
+the Trojans, but none of them could wound him before he was
+surrounded and covered by the princes Polydamas, Aeneas, Agenor,
+Sarpedon captain of the Lycians, and noble Glaucus. Of the
+others, too, there was not one who was unmindful of him, and they
+held their round shields over him to cover him. His comrades then
+lifted him off the ground and bore him away from the battle to
+the place where his horses stood waiting for him at the rear of
+the fight with their driver and the chariot; these then took him
+towards the city groaning and in great pain. When they reached
+the ford of the fair stream of Xanthus, begotten of Immortal
+Jove, they took him from off his chariot and laid him down on the
+ground; they poured water over him, and as they did so he
+breathed again and opened his eyes. Then kneeling on his knees he
+vomited blood, but soon fell back on to the ground, and his eyes
+were again closed in darkness for he was still stunned by the
+blow.
+</para>
+
+<para>
+When the Argives saw Hector leaving the field, they took heart
+and set upon the Trojans yet more furiously. Ajax fleet son of
+Oileus began by springing on Satnius son of Enops, and wounding
+him with his spear: a fair naiad nymph had borne him to Enops as
+he was herding cattle by the banks of the river Satnioeis. The
+son of Oileus came up to him and struck him in the flank so that
+he fell, and a fierce fight between Trojans and Danaans raged
+round his body. Polydamas son of Panthous drew near to avenge
+him, and wounded Prothoenor son of Areilycus on the right
+shoulder; the terrible spear went right through his shoulder, and
+he clutched the earth as he fell in the dust. Polydamas vaunted
+loudly over him saying, "Again I take it that the spear has not
+sped in vain from the strong hand of the son of Panthous; an
+Argive has caught it in his body, and it will serve him for a
+staff as he goes down into the house of Hades."
+</para>
+
+<para>
+The Argives were maddened by this boasting. Ajax son of Telamon
+was more angry than any, for the man had fallen close beside him;
+so he aimed at Polydamas as he was retreating, but Polydamas
+saved himself by swerving aside and the spear struck Archelochus
+son of Antenor, for heaven counselled his destruction; it struck
+him where the head springs from the neck at the top joint of the
+spine, and severed both the tendons at the back of the head. His
+head, mouth, and nostrils reached the ground long before his legs
+and knees could do so, and Ajax shouted to Polydamas saying,
+"Think, Polydamas, and tell me truly whether this man is not as
+well worth killing as Prothoenor was: he seems rich, and of rich
+family, a brother, it may be, or son of the knight Antenor, for
+he is very like him."
+</para>
+
+<para>
+But he knew well who it was, and the Trojans were greatly
+angered. Acamas then bestrode his brother's body and wounded
+Promachus the Boeotian with his spear, for he was trying to drag
+his brother's body away. Acamas vaunted loudly over him saying,
+"Argive archers, braggarts that you are, toil and suffering shall
+not be for us only, but some of you too shall fall here as well
+as ourselves. See how Promachus now sleeps, vanquished by my
+spear; payment for my brother's blood has not been long delayed;
+a man, therefore, may well be thankful if he leaves a kinsman in
+his house behind him to avenge his fall."
+</para>
+
+<para>
+His taunts infuriated the Argives, and Peneleos was more enraged
+than any of them. He sprang towards Acamas, but Acamas did not
+stand his ground, and he killed Ilioneus son of the rich
+flock-master Phorbas, whom Mercury had favoured and endowed with
+greater wealth than any other of the Trojans. Ilioneus was his
+only son, and Peneleos now wounded him in the eye under his
+eyebrows, tearing the eye-ball from its socket: the spear went
+right through the eye into the nape of the neck, and he fell,
+stretching out both hands before him. Peneleos then drew his
+sword and smote him on the neck, so that both head and helmet
+came tumbling down to the ground with the spear still sticking in
+the eye; he then held up the head, as though it had been a
+poppy-head, and showed it to the Trojans, vaunting over them as
+he did so. "Trojans," he cried, "bid the father and mother of
+noble Ilioneus make moan for him in their house, for the wife
+also of Promachus son of Alegenor will never be gladdened by the
+coming of her dear husband--when we Argives return with our ships
+from Troy."
+</para>
+
+<para>
+As he spoke fear fell upon them, and every man looked round about
+to see whither he might fly for safety.
+</para>
+
+<para>
+Tell me now, O Muses that dwell on Olympus, who was the first of
+the Argives to bear away blood-stained spoils after Neptune lord
+of the earthquake had turned the fortune of war. Ajax son of
+Telamon was first to wound Hyrtius son of Gyrtius, captain of the
+staunch Mysians. Antilochus killed Phalces and Mermerus, while
+Meriones slew Morys and Hippotion, Teucer also killed Prothoon
+and Periphetes. The son of Atreus then wounded Hyperenor shepherd
+of his people, in the flank, and the bronze point made his
+entrails gush out as it tore in among them; on this his life came
+hurrying out of him at the place where he had been wounded, and
+his eyes were closed in darkness. Ajax son of Oileus killed more
+than any other, for there was no man so fleet as he to pursue
+flying foes when Jove had spread panic among them.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XV</title>
+
+<para>
+BUT when their flight had taken them past the trench and the set
+stakes, and many had fallen by the hands of the Danaans, the
+Trojans made a halt on reaching their chariots, routed and pale
+with fear. Jove now woke on the crests of Ida, where he was lying
+with golden-throned Juno by his side, and starting to his feet he
+saw the Trojans and Achaeans, the one thrown into confusion, and
+the others driving them pell-mell before them with King Neptune
+in their midst. He saw Hector lying on the ground with his
+comrades gathered round him, gasping for breath, wandering in
+mind and vomiting blood, for it was not the feeblest of the
+Achaeans who struck him.
+</para>
+
+<para>
+The sire of gods and men had pity on him, and looked fiercely on
+Juno. "I see, Juno," said he, "you mischief-making trickster,
+that your cunning has stayed Hector from fighting and has caused
+the rout of his host. I am in half a mind to thrash you, in which
+case you will be the first to reap the fruits of your scurvy
+knavery. Do you not remember how once upon a time I had you
+hanged? I fastened two anvils on to your feet, and bound your
+hands in a chain of gold which none might break, and you hung in
+mid-air among the clouds. All the gods in Olympus were in a fury,
+but they could not reach you to set you free; when I caught any
+one of them I gripped him and hurled him from the heavenly
+threshold till he came fainting down to earth; yet even this did
+not relieve my mind from the incessant anxiety which I felt about
+noble Hercules whom you and Boreas had spitefully conveyed beyond
+the seas to Cos, after suborning the tempests; but I rescued him,
+and notwithstanding all his mighty labours I brought him back
+again to Argos. I would remind you of this that you may learn to
+leave off being so deceitful, and discover how much you are
+likely to gain by the embraces out of which you have come here to
+trick me."
+</para>
+
+<para>
+Juno trembled as he spoke, and said, "May heaven above and earth
+below be my witnesses, with the waters of the river Styx--and
+this is the most solemn oath that a blessed god can take--nay, I
+swear also by your own almighty head and by our bridal bed--
+things over which I could never possibly perjure myself--that
+Neptune is not punishing Hector and the Trojans and helping the
+Achaeans through any doing of mine; it is all of his own mere
+motion because he was sorry to see the Achaeans hard pressed at
+their ships: if I were advising him, I should tell him to do as
+you bid him."
+</para>
+
+<para>
+The sire of gods and men smiled and answered, "If you, Juno, were
+always to support me when we sit in council of the gods, Neptune,
+like it or no, would soon come round to your and my way of
+thinking. If, then, you are speaking the truth and mean what you
+say, go among the rank and file of the gods, and tell Iris and
+Apollo lord of the bow, that I want them--Iris, that she may go
+to the Achaean host and tell Neptune to leave off fighting and go
+home, and Apollo, that he may send Hector again into battle and
+give him fresh strength; he will thus forget his present
+sufferings, and drive the Achaeans back in confusion till they
+fall among the ships of Achilles son of Peleus. Achilles will
+then send his comrade Patroclus into battle, and Hector will kill
+him in front of Ilius after he has slain many warriors, and among
+them my own noble son Sarpedon. Achilles will kill Hector to
+avenge Patroclus, and from that time I will bring it about that
+the Achaeans shall persistently drive the Trojans back till they
+fulfil the counsels of Minerva and take Ilius. But I will not
+stay my anger, nor permit any god to help the Danaans till I have
+accomplished the desire of the son of Peleus, according to the
+promise I made by bowing my head on the day when Thetis touched
+my knees and besought me to give him honour."
+</para>
+
+<para>
+Juno heeded his words and went from the heights of Ida to great
+Olympus. Swift as the thought of one whose fancy carries him over
+vast continents, and he says to himself, "Now I will be here, or
+there," and he would have all manner of things--even so swiftly
+did Juno wing her way till she came to high Olympus and went in
+among the gods who were gathered in the house of Jove. When they
+saw her they all of them came up to her, and held out their cups
+to her by way of greeting. She let the others be, but took the
+cup offered her by lovely Themis, who was first to come running
+up to her. "Juno," said she, "why are you here? And you seem
+troubled--has your husband the son of Saturn been frightening
+you?"
+</para>
+
+<para>
+And Juno answered, "Themis, do not ask me about it. You know what
+a proud and cruel disposition my husband has. Lead the gods to
+table, where you and all the immortals can hear the wicked
+designs which he has avowed. Many a one, mortal and immortal,
+will be angered by them, however peaceably he may be feasting
+now."
+</para>
+
+<para>
+On this Juno sat down, and the gods were troubled throughout the
+house of Jove. Laughter sat on her lips but her brow was furrowed
+with care, and she spoke up in a rage. "Fools that we are," she
+cried, "to be thus madly angry with Jove; we keep on wanting to
+go up to him and stay him by force or by persuasion, but he sits
+aloof and cares for nobody, for he knows that he is much stronger
+than any other of the immortals. Make the best, therefore, of
+whatever ills he may choose to send each one of you; Mars, I take
+it, has had a taste of them already, for his son Ascalaphus has
+fallen in battle--the man whom of all others he loved most dearly
+and whose father he owns himself to be."
+</para>
+
+<para>
+When he heard this Mars smote his two sturdy thighs with the flat
+of his hands, and said in anger, "Do not blame me, you gods that
+dwell in heaven, if I go to the ships of the Achaeans and avenge
+the death of my son, even though it end in my being struck by
+Jove's lightning and lying in blood and dust among the corpses."
+</para>
+
+<para>
+As he spoke he gave orders to yoke his horses Panic and Rout,
+while he put on his armour. On this, Jove would have been roused
+to still more fierce and implacable enmity against the other
+immortals, had not Minerva, alarmed for the safety of the gods,
+sprung from her seat and hurried outside. She tore the helmet
+from his head and the shield from his shoulders, and she took the
+bronze spear from his strong hand and set it on one side; then
+she said to Mars, "Madman, you are undone; you have ears that
+hear not, or you have lost all judgement and understanding; have
+you not heard what Juno has said on coming straight from the
+presence of Olympian Jove? Do you wish to go through all kinds of
+suffering before you are brought back sick and sorry to Olympus,
+after having caused infinite mischief to all us others? Jove
+would instantly leave the Trojans and Achaeans to themselves; he
+would come to Olympus to punish us, and would grip us up one
+after another, guilty or not guilty. Therefore lay aside your
+anger for the death of your son; better men than he have either
+been killed already or will fall hereafter, and one cannot
+protect every one's whole family."
+</para>
+
+<para>
+With these words she took Mars back to his seat. Meanwhile Juno
+called Apollo outside, with Iris the messenger of the gods.
+"Jove," she said to them, "desires you to go to him at once on
+Mt. Ida; when you have seen him you are to do as he may then bid
+you."
+</para>
+
+<para>
+Thereon Juno left them and resumed her seat inside, while Iris
+and Apollo made all haste on their way. When they reached
+many-fountained Ida, mother of wild beasts, they found Jove
+seated on topmost Gargarus with a fragrant cloud encircling his
+head as with a diadem. They stood before his presence, and he was
+pleased with them for having been so quick in obeying the orders
+his wife had given them.
+</para>
+
+<para>
+He spoke to Iris first. "Go," said he, "fleet Iris, tell King
+Neptune what I now bid you--and tell him true. Bid him leave off
+fighting, and either join the company of the gods, or go down
+into the sea. If he takes no heed and disobeys me, let him
+consider well whether he is strong enough to hold his own against
+me if I attack him. I am older and much stronger than he is; yet
+he is not afraid to set himself up as on a level with myself, of
+whom all the other gods stand in awe."
+</para>
+
+<para>
+Iris, fleet as the wind, obeyed him, and as the cold hail or
+snowflakes that fly from out the clouds before the blast of
+Boreas, even so did she wing her way till she came close up to
+the great shaker of the earth. Then she said, "I have come, O
+dark-haired king that holds the world in his embrace, to bring
+you a message from Jove. He bids you leave off fighting, and
+either join the company of the gods or go down into the sea; if,
+however, you take no heed and disobey him, he says he will come
+down here and fight you. He would have you keep out of his reach,
+for he is older and much stronger than you are, and yet you are
+not afraid to set yourself up as on a level with himself, of whom
+all the other gods stand in awe."
+</para>
+
+<para>
+Neptune was very angry and said, "Great heavens! strong as Jove
+may be, he has said more than he can do if he has threatened
+violence against me, who am of like honour with himself. We were
+three brothers whom Rhea bore to Saturn--Jove, myself, and Hades
+who rules the world below. Heaven and earth were divided into
+three parts, and each of us was to have an equal share. When we
+cast lots, it fell to me to have my dwelling in the sea for
+evermore; Hades took the darkness of the realms under the earth,
+while air and sky and clouds were the portion that fell to Jove;
+but earth and great Olympus are the common property of all.
+Therefore I will not walk as Jove would have me. For all his
+strength, let him keep to his own third share and be contented
+without threatening to lay hands upon me as though I were nobody.
+Let him keep his bragging talk for his own sons and daughters,
+who must perforce obey him."
+</para>
+
+<para>
+Iris fleet as the wind then answered, "Am I really, Neptune, to
+take this daring and unyielding message to Jove, or will you
+reconsider your answer? Sensible people are open to argument, and
+you know that the Erinyes always range themselves on the side of
+the older person."
+</para>
+
+<para>
+Neptune answered, "Goddess Iris, your words have been spoken in
+season. It is well when a messenger shows so much discretion.
+Nevertheless it cuts me to the very heart that any one should
+rebuke so angrily another who is his own peer, and of like empire
+with himself. Now, however, I will give way in spite of my
+displeasure; furthermore let me tell you, and I mean what I say--
+if contrary to the desire of myself, Minerva driver of the spoil,
+Juno, Mercury, and King Vulcan, Jove spares steep Ilius, and will
+not let the Achaeans have the great triumph of sacking it, let
+him understand that he will incur our implacable resentment."
+</para>
+
+<para>
+Neptune now left the field to go down under the sea, and sorely
+did the Achaeans miss him. Then Jove said to Apollo, "Go, dear
+Phoebus, to Hector, for Neptune who holds the earth in his
+embrace has now gone down under the sea to avoid the severity of
+my displeasure. Had he not done so those gods who are below with
+Saturn would have come to hear of the fight between us. It is
+better for both of us that he should have curbed his anger and
+kept out of my reach, for I should have had much trouble with
+him. Take, then, your tasselled aegis, and shake it furiously, so
+as to set the Achaean heroes in a panic; take, moreover, brave
+Hector, O Far-Darter, into your own care, and rouse him to deeds
+of daring, till the Achaeans are sent flying back to their ships
+and to the Hellespont. From that point I will think it well over,
+how the Achaeans may have a respite from their troubles."
+</para>
+
+<para>
+Apollo obeyed his father's saying, and left the crests of Ida,
+flying like a falcon, bane of doves and swiftest of all birds. He
+found Hector no longer lying upon the ground, but sitting up, for
+he had just come to himself again. He knew those who were about
+him, and the sweat and hard breathing had left him from the
+moment when the will of aegis-bearing Jove had revived him.
+Apollo stood beside him and said, "Hector son of Priam, why are
+you so faint, and why are you here away from the others? Has any
+mishap befallen you?"
+</para>
+
+<para>
+Hector in a weak voice answered, "And which, kind sir, of the
+gods are you, who now ask me thus? Do you not know that Ajax
+struck me on the chest with a stone as I was killing his comrades
+at the ships of the Achaeans, and compelled me to leave off
+fighting? I made sure that this very day I should breathe my last
+and go down into the house of Hades."
+</para>
+
+<para>
+Then King Apollo said to him, "Take heart; the son of Saturn has
+sent you a mighty helper from Ida to stand by you and defend you,
+even me, Phoebus Apollo of the golden sword, who have been
+guardian hitherto not only of yourself but of your city. Now,
+therefore, order your horsemen to drive their chariots to the
+ships in great multitudes. I will go before your horses to smooth
+the way for them, and will turn the Achaeans in flight."
+</para>
+
+<para>
+As he spoke he infused great strength into the shepherd of his
+people. And as a horse, stabled and full-fed, breaks loose and
+gallops gloriously over the plain to the place where he is wont
+to take his bath in the river--he tosses his head, and his mane
+streams over his shoulders as in all the pride of his strength he
+flies full speed to the pastures where the mares are feeding--
+even so Hector, when he heard what the god said, urged his
+horsemen on, and sped forward as fast as his limbs could take
+him. As country peasants set their hounds on to a homed stag or
+wild goat--he has taken shelter under rock or thicket, and they
+cannot find him, but, lo, a bearded lion whom their shouts have
+roused stands in their path, and they are in no further humour
+for the chase--even so the Achaeans were still charging on in a
+body, using their swords and spears pointed at both ends, but
+when they saw Hector going about among his men they were afraid,
+and their hearts fell down into their feet.
+</para>
+
+<para>
+Then spoke Thoas son of Andraemon, leader of the Aetolians, a man
+who could throw a good throw, and who was staunch also in close
+fight, while few could surpass him in debate when opinions were
+divided. He then with all sincerity and goodwill addressed them
+thus: "What, in heaven's name, do I now see? Is it not Hector
+come to life again? Every one made sure he had been killed by
+Ajax son of Telamon, but it seems that one of the gods has again
+rescued him. He has killed many of us Danaans already, and I take
+it will yet do so, for the hand of Jove must be with him or he
+would never dare show himself so masterful in the forefront of
+the battle. Now, therefore, let us all do as I say; let us order
+the main body of our forces to fall back upon the ships, but let
+those of us who profess to be the flower of the army stand firm,
+and see whether we cannot hold Hector back at the point of our
+spears as soon as he comes near us; I conceive that he will then
+think better of it before he tries to charge into the press of
+the Danaans."
+</para>
+
+<para>
+Thus did he speak, and they did even as he had said. Those who
+were about Ajax and King Idomeneus, the followers moreover of
+Teucer, Meriones, and Meges peer of Mars called all their best
+men about them and sustained the fight against Hector and the
+Trojans, but the main body fell back upon the ships of the
+Achaeans.
+</para>
+
+<para>
+The Trojans pressed forward in a dense body, with Hector striding
+on at their head. Before him went Phoebus Apollo shrouded in
+cloud about his shoulders. He bore aloft the terrible aegis with
+its shaggy fringe, which Vulcan the smith had given Jove to
+strike terror into the hearts of men. With this in his hand he
+led on the Trojans.
+</para>
+
+<para>
+The Argives held together and stood their ground. The cry of
+battle rose high from either side, and the arrows flew from the
+bowstrings. Many a spear sped from strong hands and fastened in
+the bodies of many a valiant warrior, while others fell to earth
+midway, before they could taste of man's fair flesh and glut
+themselves with blood. So long as Phoebus Apollo held his aegis
+quietly and without shaking it, the weapons on either side took
+effect and the people fell, but when he shook it straight in the
+face of the Danaans and raised his mighty battle-cry their hearts
+fainted within them and they forgot their former prowess. As when
+two wild beasts spring in the dead of night on a herd of cattle
+or a large flock of sheep when the herdsman is not there--even so
+were the Danaans struck helpless, for Apollo filled them with
+panic and gave victory to Hector and the Trojans.
+</para>
+
+<para>
+The fight then became more scattered and they killed one another
+where they best could. Hector killed Stichius and Arcesilaus, the
+one, leader of the Boeotians, and the other, friend and comrade
+of Menestheus. Aeneas killed Medon and Iasus. The first was
+bastard son to Oileus, and brother to Ajax, but he lived in
+Phylace away from his own country, for he had killed a man, a
+kinsman of his stepmother Eriopis whom Oileus had married. Iasus
+had become a leader of the Athenians, and was son of Sphelus the
+son of Boucolos. Polydamas killed Mecisteus, and Polites Echius,
+in the front of the battle, while Agenor slew Clonius. Paris
+struck Deiochus from behind in the lower part of the shoulder, as
+he was flying among the foremost, and the point of the spear went
+clean through him.
+</para>
+
+<para>
+While they were spoiling these heroes of their armour, the
+Achaeans were flying pell-mell to the trench and the set stakes,
+and were forced back within their wall. Hector then cried out to
+the Trojans, "Forward to the ships, and let the spoils be. If I
+see any man keeping back on the other side the wall away from the
+ships I will have him killed: his kinsmen and kinswomen shall not
+give him his dues of fire, but dogs shall tear him in pieces in
+front of our city."
+</para>
+
+<para>
+As he spoke he laid his whip about his horses' shoulders and
+called to the Trojans throughout their ranks; the Trojans shouted
+with a cry that rent the air, and kept their horses neck and neck
+with his own. Phoebus Apollo went before, and kicked down the
+banks of the deep trench into its middle so as to make a great
+broad bridge, as broad as the throw of a spear when a man is
+trying his strength. The Trojan battalions poured over the
+bridge, and Apollo with his redoubtable aegis led the way. He
+kicked down the wall of the Achaeans as easily as a child who
+playing on the sea-shore has built a house of sand and then kicks
+it down again and destroys it--even so did you, O Apollo, shed
+toil and trouble upon the Argives, filling them with panic and
+confusion.
+</para>
+
+<para>
+Thus then were the Achaeans hemmed in at their ships, calling out
+to one another and raising their hands with loud cries every man
+to heaven. Nestor of Gerene, tower of strength to the Achaeans,
+lifted up his hands to the starry firmament of heaven, and prayed
+more fervently than any of them. "Father Jove," said he, "if ever
+any one in wheat-growing Argos burned you fat thigh-bones of
+sheep or heifer and prayed that he might return safely home,
+whereon you bowed your head to him in assent, bear it in mind
+now, and suffer not the Trojans to triumph thus over the
+Achaeans."
+</para>
+
+<para>
+All-counselling Jove thundered loudly in answer to the prayer of
+the aged son of Neleus. When they heard Jove thunder they flung
+themselves yet more fiercely on the Achaeans. As a wave breaking
+over the bulwarks of a ship when the sea runs high before a
+gale--for it is the force of the wind that makes the waves so
+great--even so did the Trojans spring over the wall with a
+shout, and drive their chariots onwards. The two sides fought
+with their double-pointed spears in hand-to-hand encounter-the
+Trojans from their chariots, and the Achaeans climbing up into
+their ships and wielding the long pikes that were lying on the
+decks ready for use in a sea-fight, jointed and shod with bronze.
+</para>
+
+<para>
+Now Patroclus, so long as the Achaeans and Trojans were fighting
+about the wall, but were not yet within it and at the ships,
+remained sitting in the tent of good Eurypylus, entertaining him
+with his conversation and spreading herbs over his wound to ease
+his pain. When, however, he saw the Trojans swarming through the
+breach in the wall, while the Achaeans were clamouring and struck
+with panic, he cried aloud, and smote his two thighs with the
+flat of his hands. "Eurypylus," said he in his dismay, "I know
+you want me badly, but I cannot stay with you any longer, for
+there is hard fighting going on; a servant shall take care of you
+now, for I must make all speed to Achilles, and induce him to
+fight if I can; who knows but with heaven's help I may persuade
+him. A man does well to listen to the advice of a friend."
+</para>
+
+<para>
+When he had thus spoken he went his way. The Achaeans stood firm
+and resisted the attack of the Trojans, yet though these were
+fewer in number, they could not drive them back from the ships,
+neither could the Trojans break the Achaean ranks and make their
+way in among the tents and ships. As a carpenter's line gives a
+true edge to a piece of ship's timber, in the hand of some
+skilled workman whom Minerva has instructed in all kinds of
+useful arts--even so level was the issue of the fight between the
+two sides, as they fought some round one and some round another.
+</para>
+
+<para>
+Hector made straight for Ajax, and the two fought fiercely about
+the same ship. Hector could not force Ajax back and fire the
+ship, nor yet could Ajax drive Hector from the spot to which
+heaven had brought him.
+</para>
+
+<para>
+Then Ajax struck Caletor son of Clytius in the chest with a spear
+as he was bringing fire towards the ship. He fell heavily to the
+ground and the torch dropped from his hand. When Hector saw his
+cousin fallen in front of the ship he shouted to the Trojans and
+Lycians saying, "Trojans, Lycians, and Dardanians good in close
+fight, bate not a jot, but rescue the son of Clytius lest the
+Achaeans strip him of his armour now that he has fallen."
+</para>
+
+<para>
+He then aimed a spear at Ajax, and missed him, but he hit
+Lycophron a follower of Ajax, who came from Cythera, but was
+living with Ajax inasmuch as he had killed a man among the
+Cythereans. Hector's spear struck him on the head below the ear,
+and he fell headlong from the ship's prow on to the ground with
+no life left in him. Ajax shook with rage and said to his
+brother, "Teucer, my good fellow, our trusty comrade the son of
+Mastor has fallen, he came to live with us from Cythera and whom
+we honoured as much as our own parents. Hector has just killed
+him; fetch your deadly arrows at once and the bow which Phoebus
+Apollo gave you."
+</para>
+
+<para>
+Teucer heard him and hastened towards him with his bow and quiver
+in his hands. Forthwith he showered his arrows on the Trojans,
+and hit Cleitus the son of Pisenor, comrade of Polydamas the
+noble son of Panthous, with the reins in his hands as he was
+attending to his horses; he was in the middle of the very
+thickest part of the fight, doing good service to Hector and the
+Trojans, but evil had now come upon him, and not one of those who
+were fain to do so could avert it, for the arrow struck him on
+the back of the neck. He fell from his chariot and his horses
+shook the empty car as they swerved aside. King Polydamas saw
+what had happened, and was the first to come up to the horses; he
+gave them in charge to Astynous son of Protiaon, and ordered him
+to look on, and to keep the horses near at hand. He then went
+back and took his place in the front ranks.
+</para>
+
+<para>
+Teucer then aimed another arrow at Hector, and there would have
+been no more fighting at the ships if he had hit him and killed
+him then and there: Jove, however, who kept watch over Hector,
+had his eyes on Teucer, and deprived him of his triumph, by
+breaking his bowstring for him just as he was drawing it and
+about to take his aim; on this the arrow went astray and the bow
+fell from his hands. Teucer shook with anger and said to his
+brother, "Alas, see how heaven thwarts us in all we do; it has
+broken my bowstring and snatched the bow from my hand, though I
+strung it this selfsame morning that it might serve me for many
+an arrow."
+</para>
+
+<para>
+Ajax son of Telamon answered, "My good fellow, let your bow and
+your arrows be, for Jove has made them useless in order to spite
+the Danaans. Take your spear, lay your shield upon your shoulder,
+and both fight the Trojans yourself and urge others to do so.
+They may be successful for the moment but if we fight as we ought
+they will find it a hard matter to take the ships."
+</para>
+
+<para>
+Teucer then took his bow and put it by in his tent. He hung a
+shield four hides thick about his shoulders, and on his comely
+head he set his helmet well wrought with a crest of horse-hair
+that nodded menacingly above it; he grasped his redoubtable
+bronze-shod spear, and forthwith he was by the side of Ajax.
+</para>
+
+<para>
+When Hector saw that Teucer's bow was of no more use to him, he
+shouted out to the Trojans and Lycians, "Trojans, Lycians, and
+Dardanians good in close fight, be men, my friends, and show your
+mettle here at the ships, for I see the weapon of one of their
+chieftains made useless by the hand of Jove. It is easy to see
+when Jove is helping people and means to help them still further,
+or again when he is bringing them down and will do nothing for
+them; he is now on our side, and is going against the Argives.
+Therefore swarm round the ships and fight. If any of you is
+struck by spear or sword and loses his life, let him die; he dies
+with honour who dies fighting for his country; and he will leave
+his wife and children safe behind him, with his house and
+allotment unplundered if only the Achaeans can be driven back to
+their own land, they and their ships."
+</para>
+
+<para>
+With these words he put heart and soul into them all. Ajax on the
+other side exhorted his comrades saying, "Shame on you Argives,
+we are now utterly undone, unless we can save ourselves by
+driving the enemy from our ships. Do you think, if Hector takes
+them, that you will be able to get home by land? Can you not hear
+him cheering on his whole host to fire our fleet, and bidding
+them remember that they are not at a dance but in battle? Our
+only course is to fight them with might and main; we had better
+chance it, life or death, once for all, than fight long and
+without issue hemmed in at our ships by worse men than
+ourselves."
+</para>
+
+<para>
+With these words he put life and soul into them all. Hector then
+killed Schedius son of Perimedes, leader of the Phoceans, and
+Ajax killed Laodamas captain of foot soldiers and son to Antenor.
+Polydamas killed Otus of Cyllene a comrade of the son of Phyleus
+and chief of the proud Epeans. When Meges saw this he sprang upon
+him, but Polydamas crouched down, and he missed him, for Apollo
+would not suffer the son of Panthous to fall in battle; but the
+spear hit Croesmus in the middle of his chest, whereon he fell
+heavily to the ground, and Meges stripped him of his armour. At
+that moment the valiant soldier Dolops son of Lampus sprang upon
+Lampus was son of Laomedon and for his valour, while his son
+Dolops was versed in all the ways of war. He then struck the
+middle of the son of Phyleus' shield with his spear, setting on
+him at close quarters, but his good corslet made with plates of
+metal saved him; Phyleus had brought it from Ephyra and the river
+Selleis, where his host, King Euphetes, had given it him to wear
+in battle and protect him. It now served to save the life of his
+son. Then Meges struck the topmost crest of Dolops's bronze
+helmet with his spear and tore away its plume of horse-hair, so
+that all newly dyed with scarlet as it was it tumbled down into
+the dust. While he was still fighting and confident of victory,
+Menelaus came up to help Meges, and got by the side of Dolops
+unperceived; he then speared him in the shoulder, from behind,
+and the point, driven so furiously, went through into his chest,
+whereon he fell headlong. The two then made towards him to strip
+him of his armour, but Hector called on all his brothers for
+help, and he especially upbraided brave Melanippus son of
+Hiketaon, who erewhile used to pasture his herds of cattle in
+Percote before the war broke out; but when the ships of the
+Danaans came, he went back to Ilius, where he was eminent among
+the Trojans, and lived near Priam who treated him as one of his
+own sons. Hector now rebuked him and said, "Why, Melanippus, are
+we thus remiss? do you take no note of the death of your kinsman,
+and do you not see how they are trying to take Dolops's armour?
+Follow me; there must be no fighting the Argives from a distance
+now, but we must do so in close combat till either we kill them
+or they take the high wall of Ilius and slay her people."
+</para>
+
+<para>
+He led on as he spoke, and the hero Melanippus followed after.
+Meanwhile Ajax son of Telamon was cheering on the Argives. "My
+friends," he cried, "be men, and fear dishonour; quit yourselves
+in battle so as to win respect from one another. Men who respect
+each other's good opinion are less likely to be killed than those
+who do not, but in flight there is neither gain nor glory."
+</para>
+
+<para>
+Thus did he exhort men who were already bent upon driving back
+the Trojans. They laid his words to heart and hedged the ships as
+with a wall of bronze, while Jove urged on the Trojans. Menelaus
+of the loud battle-cry urged Antilochus on. "Antilochus," said
+he, "you are young and there is none of the Achaeans more fleet
+of foot or more valiant than you are. See if you cannot spring
+upon some Trojan and kill him."
+</para>
+
+<para>
+He hurried away when he had thus spurred Antilochus, who at once
+darted out from the front ranks and aimed a spear, after looking
+carefully round him. The Trojans fell back as he threw, and the
+dart did not speed from his hand without effect, for it struck
+Melanippus the proud son of Hiketaon in the breast by the nipple
+as he was coming forward, and his armour rang rattling round him
+as he fell heavily to the ground. Antilochus sprang upon him as a
+dog springs on a fawn which a hunter has hit as it was breaking
+away from its covert, and killed it. Even so, O Melanippus, did
+stalwart Antilochus spring upon you to strip you of your armour;
+but noble Hector marked him, and came running up to him through
+the thick of the battle. Antilochus, brave soldier though he was,
+would not stay to face him, but fled like some savage creature
+which knows it has done wrong, and flies, when it has killed a
+dog or a man who is herding his cattle, before a body of men can
+be gathered to attack it. Even so did the son of Nestor fly, and
+the Trojans and Hector with a cry that rent the air showered
+their weapons after him; nor did he turn round and stay his
+flight till he had reached his comrades.
+</para>
+
+<para>
+The Trojans, fierce as lions, were still rushing on towards the
+ships in fulfilment of the behests of Jove who kept spurring them
+on to new deeds of daring, while he deadened the courage of the
+Argives and defeated them by encouraging the Trojans. For he
+meant giving glory to Hector son of Priam, and letting him throw
+fire upon the ships, till he had fulfilled the unrighteous prayer
+that Thetis had made him; Jove, therefore, bided his time till he
+should see the glare of a blazing ship. From that hour he was
+about so to order that the Trojans should be driven back from the
+ships and to vouchsafe glory to the Achaeans. With this purpose
+he inspired Hector son of Priam, who was cager enough already, to
+assail the ships. His fury was as that of Mars, or as when a fire
+is raging in the glades of some dense forest upon the mountains;
+he foamed at the mouth, his eyes glared under his terrible
+eye-brows, and his helmet quivered on his temples by reason of
+the fury with which he fought. Jove from heaven was with him, and
+though he was but one against many, vouchsafed him victory and
+glory; for he was doomed to an early death, and already Pallas
+Minerva was hurrying on the hour of his destruction at the hands
+of the son of Peleus. Now, however, he kept trying to break the
+ranks of the enemy wherever he could see them thickest, and in
+the goodliest armour; but do what he might he could not break
+through them, for they stood as a tower foursquare, or as some
+high cliff rising from the grey sea that braves the anger of the
+gale, and of the waves that thunder up against it. He fell upon
+them like flames of fire from every quarter. As when a wave,
+raised mountain high by wind and storm, breaks over a ship and
+covers it deep in foam, the fierce winds roar against the mast,
+the hearts of the sailors fail them for fear, and they are saved
+but by a very little from destruction--even so were the hearts of
+the Achaeans fainting within them. Or as a savage lion attacking
+a herd of cows while they are feeding by thousands in the
+low-lying meadows by some wide-watered shore--the herdsman is at
+his wit's end how to protect his herd and keeps going about now
+in the van and now in the rear of his cattle, while the lion
+springs into the thick of them and fastens on a cow so that they
+all tremble for fear--even so were the Achaeans utterly
+panic-stricken by Hector and father Jove. Nevertheless Hector
+only killed Periphetes of Mycenae; he was son of Copreus who was
+wont to take the orders of King Eurystheus to mighty Hercules,
+but the son was a far better man than the father in every way; he
+was fleet of foot, a valiant warrior, and in understanding ranked
+among the foremost men of Mycenae. He it was who then afforded
+Hector a triumph, for as he was turning back he stumbled against
+the rim of his shield which reached his feet, and served to keep
+the javelins off him. He tripped against this and fell face
+upward, his helmet ringing loudly about his head as he did so.
+Hector saw him fall and ran up to him; he then thrust a spear
+into his chest, and killed him close to his own comrades. These,
+for all their sorrow, could not help him for they were themselves
+terribly afraid of Hector.
+</para>
+
+<para>
+They had now reached the ships and the prows of those that had
+been drawn up first were on every side of them, but the Trojans
+came pouring after them. The Argives were driven back from the
+first row of ships, but they made a stand by their tents without
+being broken up and scattered; shame and fear restrained them.
+They kept shouting incessantly to one another, and Nestor of
+Gerene, tower of strength to the Achaeans, was loudest in
+imploring every man by his parents, and beseeching him to stand
+firm.
+</para>
+
+<para>
+"Be men, my friends," he cried, "and respect one another's good
+opinion. Think, all of you, on your children, your wives, your
+property, and your parents whether these be alive or dead. On
+their behalf though they are not here, I implore you to stand
+firm, and not to turn in flight."
+</para>
+
+<para>
+With these words he put heart and soul into them all. Minerva
+lifted the thick veil of darkness from their eyes, and much light
+fell upon them, alike on the side of the ships and on that where
+the fight was raging. They could see Hector and all his men, both
+those in the rear who were taking no part in the battle, and
+those who were fighting by the ships.
+</para>
+
+<para>
+Ajax could not bring himself to retreat along with the rest, but
+strode from deck to deck with a great sea-pike in his hands
+twelve cubits long and jointed with rings. As a man skilled in
+feats of horsemanship couples four horses together and comes
+tearing full speed along the public way from the country into
+some large town--many both men and women marvel as they see him
+for he keeps all the time changing his horse, springing from one
+to another without ever missing his feet while the horses are at
+a gallop--even so did Ajax go striding from one ship's deck to
+another, and his voice went up into the heavens. He kept on
+shouting his orders to the Danaans and exhorting them to defend
+their ships and tents; neither did Hector remain within the main
+body of the Trojan warriors, but as a dun eagle swoops down upon
+a flock of wild-fowl feeding near a river-geese, it may be, or
+cranes, or long-necked swans--even so did Hector make straight
+for a dark-prowed ship, rushing right towards it; for Jove with
+his mighty hand impelled him forward, and roused his people to
+follow him.
+</para>
+
+<para>
+And now the battle again raged furiously at the ships. You would
+have thought the men were coming on fresh and unwearied, so
+fiercely did they fight; and this was the mind in which they
+were--the Achaeans did not believe they should escape destruction
+but thought themselves doomed, while there was not a Trojan but
+his heart beat high with the hope of firing the ships and putting
+the Achaean heroes to the sword.
+</para>
+
+<para>
+Thus were the two sides minded. Then Hector seized the stern of
+the good ship that had brought Protesilaus to Troy, but never
+bore him back to his native land. Round this ship there raged a
+close hand-to-hand fight between Danaans and Trojans. They did
+not fight at a distance with bows and javelins, but with one mind
+hacked at one another in close combat with their mighty swords
+and spears pointed at both ends; they fought moreover with keen
+battle-axes and with hatchets. Many a good stout blade hilted and
+scabbarded with iron, fell from hand or shoulder as they fought,
+and the earth ran red with blood. Hector, when he had seized the
+ship, would not loose his hold but held on to its curved stern
+and shouted to the Trojans, "Bring fire, and raise the battle-cry
+all of you with a single voice. Now has Jove vouchsafed us a day
+that will pay us for all the rest; this day we shall take the
+ships which came hither against heaven's will, and which have
+caused us such infinite suffering through the cowardice of our
+councillors, who when I would have done battle at the ships held
+me back and forbade the host to follow me; if Jove did then
+indeed warp our judgements, himself now commands me and cheers me
+on."
+</para>
+
+<para>
+As he spoke thus the Trojans sprang yet more fiercely on the
+Achaeans, and Ajax no longer held his ground, for he was overcome
+by the darts that were flung at him, and made sure that he was
+doomed. Therefore he left the raised deck at the stern, and
+stepped back on to the seven-foot bench of the oarsmen. Here he
+stood on the look-out, and with his spear held back Trojan whom
+he saw bringing fire to the ships. All the time he kept on
+shouting at the top of his voice and exhorting the Danaans. "My
+friends," he cried, "Danaan heroes, servants of Mars, be men my
+friends, and fight with might and with main. Can we hope to find
+helpers hereafter, or a wall to shield us more surely than the
+one we have? There is no strong city within reach, whence we may
+draw fresh forces to turn the scales in our favour. We are on the
+plain of the armed Trojans with the sea behind us, and far from
+our own country. Our salvation, therefore, is in the might of our
+hands and in hard fighting."
+</para>
+
+<para>
+As he spoke he wielded his spear with still greater fury, and
+when any Trojan made towards the ships with fire at Hector's
+bidding, he would be on the look-out for him, and drive at him
+with his long spear. Twelve men did he thus kill in hand-to-hand
+fight before the ships.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XVI</title>
+
+<para>
+THUS did they fight about the ship of Protesilaus. Then Patroclus
+drew near to Achilles with tears welling from his eyes, as from
+some spring whose crystal stream falls over the ledges of a high
+precipice. When Achilles saw him thus weeping he was sorry for
+him and said, "Why, Patroclus, do you stand there weeping like
+some silly child that comes running to her mother, and begs to be
+taken up and carried--she catches hold of her mother's dress to
+stay her though she is in a hurry, and looks tearfully up until
+her mother carries her--even such tears, Patroclus, are you now
+shedding. Have you anything to say to the Myrmidons or to myself?
+or have you had news from Phthia which you alone know? They tell
+me Menoetius son of Actor is still alive, as also Peleus son of
+Aeacus, among the Myrmidons--men whose loss we two should
+bitterly deplore; or are you grieving about the Argives and the
+way in which they are being killed at the ships, through their
+own high-handed doings? Do not hide anything from me but tell me
+that both of us may know about it."
+</para>
+
+<para>
+Then, O knight Patroclus, with a deep sigh you answered,
+"Achilles, son of Peleus, foremost champion of the Achaeans, do
+not be angry, but I weep for the disaster that has now befallen
+the Argives. All those who have been their champions so far are
+lying at the ships, wounded by sword or spear. Brave Diomed son
+of Tydeus has been hit with a spear, while famed Ulysses and
+Agamemnon have received sword-wounds; Eurypylus again has been
+struck with an arrow in the thigh; skilled apothecaries are
+attending to these heroes, and healing them of their wounds; are
+you still, O Achilles, so inexorable? May it never be my lot to
+nurse such a passion as you have done, to the baning of your own
+good name. Who in future story will speak well of you unless you
+now save the Argives from ruin? You know no pity; knight Peleus
+was not your father nor Thetis your mother, but the grey sea bore
+you and the sheer cliffs begot you, so cruel and remorseless are
+you. If however you are kept back through knowledge of some
+oracle, or if your mother Thetis has told you something from the
+mouth of Jove, at least send me and the Myrmidons with me, if I
+may bring deliverance to the Danaans. Let me moreover wear your
+armour; the Trojans may thus mistake me for you and quit the
+field, so that the hard-pressed sons of the Achaeans may have
+breathing time--which while they are fighting may hardly be. We
+who are fresh might soon drive tired men back from our ships and
+tents to their own city."
+</para>
+
+<para>
+He knew not what he was asking, nor that he was suing for his own
+destruction. Achilles was deeply moved and answered, "What, noble
+Patroclus, are you saying? I know no prophesyings which I am
+heeding, nor has my mother told me anything from the mouth of
+Jove, but I am cut to the very heart that one of my own rank
+should dare to rob me because he is more powerful than I am.
+This, after all that I have gone through, is more than I can
+endure. The girl whom the sons of the Achaeans chose for me, whom
+I won as the fruit of my spear on having sacked a city--her has
+King Agamemnon taken from me as though I were some common
+vagrant. Still, let bygones be bygones: no man may keep his anger
+for ever; I said I would not relent till battle and the cry of
+war had reached my own ships; nevertheless, now gird my armour
+about your shoulders, and lead the Myrmidons to battle, for the
+dark cloud of Trojans has burst furiously over our fleet; the
+Argives are driven back on to the beach, cooped within a narrow
+space, and the whole people of Troy has taken heart to sally out
+against them, because they see not the visor of my helmet
+gleaming near them. Had they seen this, there would not have been
+a creek nor grip that had not been filled with their dead as they
+fled back again. And so it would have been, if only King
+Agamemnon had dealt fairly by me. As it is the Trojans have beset
+our host. Diomed son of Tydeus no longer wields his spear to
+defend the Danaans, neither have I heard the voice of the son of
+Atreus coming from his hated head, whereas that of murderous
+Hector rings in my cars as he gives orders to the Trojans, who
+triumph over the Achaeans and fill the whole plain with their cry
+of battle. But even so, Patroclus, fall upon them and save the
+fleet, lest the Trojans fire it and prevent us from being able to
+return. Do, however, as I now bid you, that you may win me great
+honour from all the Danaans, and that they may restore the girl
+to me again and give me rich gifts into the bargain. When you
+have driven the Trojans from the ships, come back again. Though
+Juno's thundering husband should put triumph within your reach,
+do not fight the Trojans further in my absence, or you will rob
+me of glory that should be mine. And do not for lust of battle go
+on killing the Trojans nor lead the Achaeans on to Ilius, lest
+one of the ever-living gods from Olympus attack you--for Phoebus
+Apollo loves them well: return when you have freed the ships from
+peril, and let others wage war upon the plain. Would, by father
+Jove, Minerva, and Apollo, that not a single man of all the
+Trojans might be left alive, nor yet of the Argives, but that we
+two might be alone left to tear aside the mantle that veils the
+brow of Troy."
+</para>
+
+<para>
+Thus did they converse. But Ajax could no longer hold his ground
+for the shower of darts that rained upon him; the will of Jove
+and the javelins of the Trojans were too much for him; the helmet
+that gleamed about his temples rang with the continuous clatter
+of the missiles that kept pouring on to it and on to the
+cheek-pieces that protected his face. Moreover his left shoulder
+was tired with having held his shield so long, yet for all this,
+let fly at him as they would, they could not make him give
+ground. He could hardly draw his breath, the sweat rained from
+every pore of his body, he had not a moment's respite, and on all
+sides he was beset by danger upon danger.
+</para>
+
+<para>
+And now, tell me, O Muses that hold your mansions on Olympus, how
+fire was thrown upon the ships of the Achaeans. Hector came close
+up and let drive with his great sword at the ashen spear of Ajax.
+He cut it clean in two just behind where the point was fastened
+on to the shaft of the spear. Ajax, therefore, had now nothing
+but a headless spear, while the bronze point flew some way off
+and came ringing down on to the ground. Ajax knew the hand of
+heaven in this, and was dismayed at seeing that Jove had now left
+him utterly defenceless and was willing victory for the Trojans.
+Therefore he drew back, and the Trojans flung fire upon the ship
+which was at once wrapped in flame.
+</para>
+
+<para>
+The fire was now flaring about the ship's stern, whereon Achilles
+smote his two thighs and said to Patroclus, "Up, noble knight,
+for I see the glare of hostile fire at our fleet; up, lest they
+destroy our ships, and there be no way by which we may retreat.
+Gird on your armour at once while I call our people together."
+</para>
+
+<para>
+As he spoke Patroclus put on his armour. First he greaved his
+legs with greaves of good make, and fitted with ancle-clasps of
+silver; after this he donned the cuirass of the son of Aeacus,
+richly inlaid and studded. He hung his silver-studded sword of
+bronze about his shoulders, and then his mighty shield. On his
+comely head he set his helmet, well wrought, with a crest of
+horse-hair that nodded menacingly above it. He grasped two
+redoubtable spears that suited his hands, but he did not take the
+spear of noble Achilles, so stout and strong, for none other of
+the Achaeans could wield it, though Achilles could do so easily.
+This was the ashen spear from Mount Pelion, which Chiron had cut
+upon a mountain top and had given to Peleus, wherewith to deal
+out death among heroes. He bade Automedon yoke his horses with
+all speed, for he was the man whom he held in honour next after
+Achilles, and on whose support in battle he could rely most
+firmly. Automedon therefore yoked the fleet horses Xanthus and
+Balius, steeds that could fly like the wind: these were they whom
+the harpy Podarge bore to the west wind, as she was grazing in a
+meadow by the waters of the river Oceanus. In the side traces he
+set the noble horse Pedasus, whom Achilles had brought away with
+him when he sacked the city of Eetion, and who, mortal steed
+though he was, could take his place along with those that were
+immortal.
+</para>
+
+<para>
+Meanwhile Achilles went about everywhere among the tents, and
+bade his Myrmidons put on their armour. Even as fierce ravening
+wolves that are feasting upon a homed stag which they have killed
+upon the mountains, and their jaws are red with blood--they go in
+a pack to lap water from the clear spring with their long thin
+tongues; and they reek of blood and slaughter; they know not what
+fear is, for it is hunger drives them--even so did the leaders
+and counsellors of the Myrmidons gather round the good squire of
+the fleet descendant of Aeacus, and among them stood Achilles
+himself cheering on both men and horses.
+</para>
+
+<para>
+Fifty ships had noble Achilles brought to Troy, and in each there
+was a crew of fifty oarsmen. Over these he set five captains whom
+he could trust, while he was himself commander over them all.
+Menesthius of the gleaming corslet, son to the river Spercheius
+that streams from heaven, was captain of the first company. Fair
+Polydora daughter of Peleus bore him to ever-flowing
+Spercheius--a woman mated with a god--but he was called son of
+Borus son of Perieres, with whom his mother was living as his
+wedded wife, and who gave great wealth to gain her. The second
+company was led by noble Eudorus, son to an unwedded woman.
+Polymele, daughter of Phylas the graceful dancer, bore him; the
+mighty slayer of Argos was enamoured of her as he saw her among
+the singing women at a dance held in honour of Diana the rushing
+huntress of the golden arrows; he therefore--Mercury, giver of
+all good--went with her into an upper chamber, and lay with her
+in secret, whereon she bore him a noble son Eudorus, singularly
+fleet of foot and in fight valiant. When Ilithuia goddess of the
+pains of child-birth brought him to the light of day, and he saw
+the face of the sun, mighty Echecles son of Actor took the mother
+to wife, and gave great wealth to gain her, but her father Phylas
+brought the child up, and took care of him, doting as fondly upon
+him as though he were his own son. The third company was led by
+Pisander son of Maemalus, the finest spearman among all the
+Myrmidons next to Achilles' own comrade Patroclus. The old knight
+Phoenix was captain of the fourth company, and Alcimedon, noble
+son of Laerceus of the fifth.
+</para>
+
+<para>
+When Achilles had chosen his men and had stationed them all with
+their captains, he charged them straitly saying, "Myrmidons,
+remember your threats against the Trojans while you were at the
+ships in the time of my anger, and you were all complaining of
+me. 'Cruel son of Peleus,' you would say, 'your mother must have
+suckled you on gall, so ruthless are you. You keep us here at the
+ships against our will; if you are so relentless it were better
+we went home over the sea.' Often have you gathered and thus
+chided with me. The hour is now come for those high feats of arms
+that you have so long been pining for, therefore keep high hearts
+each one of you to do battle with the Trojans."
+</para>
+
+<para>
+With these words he put heart and soul into them all, and they
+serried their companies yet more closely when they heard the of
+their king. As the stones which a builder sets in the wall of
+some high house which is to give shelter from the winds--even so
+closely were the helmets and bossed shields set against one
+another. Shield pressed on shield, helm on helm, and man on man;
+so close were they that the horse-hair plumes on the gleaming
+ridges of their helmets touched each other as they bent their
+heads.
+</para>
+
+<para>
+In front of them all two men put on their armour--Patroclus and
+Automedon--two men, with but one mind to lead the Myrmidons. Then
+Achilles went inside his tent and opened the lid of the strong
+chest which silver-footed Thetis had given him to take on board
+ship, and which she had filled with shirts, cloaks to keep out
+the cold, and good thick rugs. In this chest he had a cup of rare
+workmanship, from which no man but himself might drink, nor would
+he make offering from it to any other god save only to father
+Jove. He took the cup from the chest and cleansed it with
+sulphur; this done he rinsed it clean water, and after he had
+washed his hands he drew wine. Then he stood in the middle of the
+court and prayed, looking towards heaven, and making his
+drink-offering of wine; nor was he unseen of Jove whose joy is in
+thunder. "King Jove," he cried, "lord of Dodona, god of the
+Pelasgi, who dwellest afar, you who hold wintry Dodona in your
+sway, where your prophets the Selli dwell around you with their
+feet unwashed and their couches made upon the ground--if you
+heard me when I prayed to you aforetime, and did me honour while
+you sent disaster on the Achaeans, vouchsafe me now the
+fulfilment of yet this further prayer. I shall stay here where my
+ships are lying, but I shall send my comrade into battle at the
+head of many Myrmidons. Grant, O all-seeing Jove, that victory
+may go with him; put your courage into his heart that Hector may
+learn whether my squire is man enough to fight alone, or whether
+his might is only then so indomitable when I myself enter the
+turmoil of war. Afterwards when he has chased the fight and the
+cry of battle from the ships, grant that he may return unharmed,
+with his armour and his comrades, fighters in close combat."
+</para>
+
+<para>
+Thus did he pray, and all-counselling Jove heard his prayer.
+Part of it he did indeed vouchsafe him--but not the whole. He
+granted that Patroclus should thrust back war and battle from the
+ships, but refused to let him come safely out of the fight.
+</para>
+
+<para>
+When he had made his drink-offering and had thus prayed, Achilles
+went inside his tent and put back the cup into his chest.
+</para>
+
+<para>
+Then he again came out, for he still loved to look upon the
+fierce fight that raged between the Trojans and Achaeans.
+</para>
+
+<para>
+Meanwhile the armed band that was about Patroclus marched on till
+they sprang high in hope upon the Trojans. They came swarming out
+like wasps whose nests are by the roadside, and whom silly
+children love to tease, whereon any one who happens to be passing
+may get stung--or again, if a wayfarer going along the road vexes
+them by accident, every wasp will come flying out in a fury to
+defend his little ones--even with such rage and courage did the
+Myrmidons swarm from their ships, and their cry of battle rose
+heavenwards. Patroclus called out to his men at the top of his
+voice, "Myrmidons, followers of Achilles son of Peleus, be men my
+friends, fight with might and with main, that we may win glory
+for the son of Peleus, who is far the foremost man at the ships
+of the Argives--he, and his close fighting followers. The son of
+Atreus King Agamemnon will thus learn his folly in showing no
+respect to the bravest of the Achaeans."
+</para>
+
+<para>
+With these words he put heart and soul into them all, and they
+fell in a body upon the Trojans. The ships rang again with the
+cry which the Achaeans raised, and when the Trojans saw the brave
+son of Menoetius and his squire all gleaming in their armour,
+they were daunted and their battalions were thrown into
+confusion, for they thought the fleet son of Peleus must now have
+put aside his anger, and have been reconciled to Agamemnon; every
+one, therefore, looked round about to see whither he might fly
+for safety.
+</para>
+
+<para>
+Patroclus first aimed a spear into the middle of the press where
+men were packed most closely, by the stern of the ship of
+Protesilaus. He hit Pyraechmes who had led his Paeonian horsemen
+from the Amydon and the broad waters of the river Axius; the
+spear struck him on the right shoulder, and with a groan he fell
+backwards in the dust; on this his men were thrown into
+confusion, for by killing their leader, who was the finest
+soldier among them, Patroclus struck panic into them all. He thus
+drove them from the ship and quenched the fire that was then
+blazing--leaving the half-burnt ship to lie where it was. The
+Trojans were now driven back with a shout that rent the skies,
+while the Danaans poured after them from their ships, shouting
+also without ceasing. As when Jove, gatherer of the
+thunder-cloud, spreads a dense canopy on the top of some lofty
+mountain, and all the peaks, the jutting headlands, and forest
+glades show out in the great light that flashes from the bursting
+heavens, even so when the Danaans had now driven back the fire
+from their ships, they took breath for a little while; but the
+fury of the fight was not yet over, for the Trojans were not
+driven back in utter rout, but still gave battle, and were ousted
+from their ground only by sheer fighting.
+</para>
+
+<para>
+The fight then became more scattered, and the chieftains killed
+one another when and how they could. The valiant son of Menoetius
+first drove his spear into the thigh of Areilycus just as he was
+turning round; the point went clean through, and broke the bone
+so that he fell forward. Meanwhile Menelaus struck Thoas in the
+chest, where it was exposed near the rim of his shield, and he
+fell dead. The son of Phyleus saw Amphiclus about to attack him,
+and ere he could do so took aim at the upper part of his thigh,
+where the muscles are thicker than in any other part; the spear
+tore through all the sinews of the leg, and his eyes were closed
+in darkness. Of the sons of Nestor one, Antilochus, speared
+Atymnius, driving the point of the spear through his throat, and
+down he fell. Maris then sprang on Antilochus in hand-to-hand
+fight to avenge his brother, and bestrode the body spear in hand;
+but valiant Thrasymedes was too quick for him, and in a moment
+had struck him in the shoulder ere he could deal his blow; his
+aim was true, and the spear severed all the muscles at the root
+of his arm, and tore them right down to the bone, so he fell
+heavily to the ground and his eyes were closed in darkness. Thus
+did these two noble comrades of Sarpedon go down to Erebus slain
+by the two sons of Nestor; they were the warrior sons of
+Amisodorus, who had reared the invincible Chimaera, to the bane
+of many. Ajax son of Oileus sprang on Cleobulus and took him
+alive as he was entangled in the crush; but he killed him then
+and there by a sword-blow on the neck. The sword reeked with his
+blood, while dark death and the strong hand of fate gripped him
+and closed his eyes.
+</para>
+
+<para>
+Peneleos and Lycon now met in close fight, for they had missed
+each other with their spears. They had both thrown without
+effect, so now they drew their swords. Lycon struck the plumed
+crest of Peneleos' helmet but his sword broke at the hilt, while
+Peneleos smote Lycon on the neck under the ear. The blade sank so
+deep that the head was held on by nothing but the skin, and there
+was no more life left in him. Meriones gave chase to Acamas on
+foot and caught him up just as he was about to mount his chariot;
+he drove a spear through his right shoulder so that he fell
+headlong from the car, and his eyes were closed in darkness.
+Idomeneus speared Erymas in the mouth; the bronze point of the
+spear went clean through it beneath the brain, crashing in among
+the white bones and smashing them up. His teeth were all of them
+knocked out and the blood came gushing in a stream from both his
+eyes; it also came gurgling up from his mouth and nostrils, and
+the darkness of death enfolded him round about.
+</para>
+
+<para>
+Thus did these chieftains of the Danaans each of them kill his
+man. As ravening wolves seize on kids or lambs, fastening on them
+when they are alone on the hillsides and have strayed from the
+main flock through the carelessness of the shepherd--and when the
+wolves see this they pounce upon them at once because they cannot
+defend themselves--even so did the Danaans now fall on the
+Trojans, who fled with ill-omened cries in their panic and had no
+more fight left in them.
+</para>
+
+<para>
+Meanwhile great Ajax kept on trying to drive a spear into Hector,
+but Hector was so skilful that he held his broad shoulders well
+under cover of his ox-hide shield, ever on the look-out for the
+whizzing of the arrows and the heavy thud of the spears. He well
+knew that the fortunes of the day had changed, but still stood
+his ground and tried to protect his comrades.
+</para>
+
+<para>
+As when a cloud goes up into heaven from Olympus, rising out of a
+clear sky when Jove is brewing a gale--even with such panic
+stricken rout did the Trojans now fly, and there was no order in
+their going. Hector's fleet horses bore him and his armour out of
+the fight, and he left the Trojan host penned in by the deep
+trench against their will. Many a yoke of horses snapped the pole
+of their chariots in the trench and left their master's car
+behind them. Patroclus gave chase, calling impetuously on the
+Danaans and full of fury against the Trojans, who, being now no
+longer in a body, filled all the ways with their cries of panic
+and rout; the air was darkened with the clouds of dust they
+raised, and the horses strained every nerve in their flight from
+the tents and ships towards the city.
+</para>
+
+<para>
+Patroclus kept on heading his horses wherever he saw most men
+flying in confusion, cheering on his men the while. Chariots were
+being smashed in all directions, and many a man came tumbling
+down from his own car to fall beneath the wheels of that of
+Patroclus, whose immortal steeds, given by the gods to Peleus,
+sprang over the trench at a bound as they sped onward. He was
+intent on trying to get near Hector, for he had set his heart on
+spearing him, but Hector's horses were now hurrying him away. As
+the whole dark earth bows before some tempest on an autumn day
+when Jove rains his hardest to punish men for giving crooked
+judgement in their courts, and arriving justice therefrom without
+heed to the decrees of heaven--all the rivers run full and the
+torrents tear many a new channel as they roar headlong from the
+mountains to the dark sea, and it fares ill with the works of
+men--even such was the stress and strain of the Trojan horses in
+their flight.
+</para>
+
+<para>
+Patroclus now cut off the battalions that were nearest to him and
+drove them back to the ships. They were doing their best to reach
+the city, but he would not let them, and bore down on them
+between the river and the ships and wall. Many a fallen comrade
+did he then avenge. First he hit Pronous with a spear on the
+chest where it was exposed near the rim of his shield, and he
+fell heavily to the ground. Next he sprang on Thestor son of
+Enops, who was sitting all huddled up in his chariot, for he had
+lost his head and the reins had been torn out of his hands.
+Patroclus went up to him and drove a spear into his right jaw; he
+thus hooked him by the teeth and the spear pulled him over the
+rim of his car, as one who sits at the end of some jutting rock
+and draws a strong fish out of the sea with a hook and a line--
+even so with his spear did he pull Thestor all gaping from his
+chariot; he then threw him down on his face and he died while
+falling. On this, as Erylaus was on to attack him, he struck him
+full on the head with a stone, and his brains were all battered
+inside his helmet, whereon he fell headlong to the ground and the
+pangs of death took hold upon him. Then he laid low, one after
+the other, Erymas, Amphoterus, Epaltes, Tlepolemus, Echius son of
+Damastor, Pyris, lpheus, Euippus and Polymelus son of Argeas.
+</para>
+
+<para>
+Now when Sarpedon saw his comrades, men who wore ungirdled
+tunics, being overcome by Patroclus son of Menoetius, he rebuked
+the Lycians saying. "Shame on you, where are you flying to? Show
+your mettle; I will myself meet this man in fight and learn who
+it is that is so masterful; he has done us much hurt, and has
+stretched many a brave man upon the ground."
+</para>
+
+<para>
+He sprang from his chariot as he spoke, and Patroclus, when he
+saw this, leaped on to the ground also. The two then rushed at
+one another with loud cries like eagle-beaked crook-taloned
+vultures that scream and tear at one another in some high
+mountain fastness.
+</para>
+
+<para>
+The son of scheming Saturn looked down upon them in pity and said
+to Juno who was his wife and sister, "Alas, that it should be the
+lot of Sarpedon whom I love so dearly to perish by the hand of
+Patroclus. I am in two minds whether to catch him up out of the
+fight and set him down safe and sound in the fertile land of
+Lycia, or to let him now fall by the hand of the son of
+Menoetius."
+</para>
+
+<para>
+And Juno answered, "Most dread son of Saturn, what is this that
+you are saying? Would you snatch a mortal man, whose doom has
+long been fated, out of the jaws of death? Do as you will, but we
+shall not all of us be of your mind. I say further, and lay my
+saying to your heart, that if you send Sarpedon safely to his own
+home, some other of the gods will be also wanting to escort his
+son out of battle, for there are many sons of gods fighting round
+the city of Troy, and you will make every one jealous. If,
+however, you are fond of him and pity him, let him indeed fall by
+the hand of Patroclus, but as soon as the life is gone out of
+him, send Death and sweet Sleep to bear him off the field and
+take him to the broad lands of Lycia, where his brothers and his
+kinsmen will bury him with mound and pillar, in due honour to the
+dead."
+</para>
+
+<para>
+The sire of gods and men assented, but he shed a rain of blood
+upon the earth in honour of his son whom Patroclus was about to
+kill on the rich plain of Troy far from his home.
+</para>
+
+<para>
+When they were now come close to one another Patroclus struck
+Thrasydemus, the brave squire of Sarpedon, in the lower part of
+the belly, and killed him. Sarpedon then aimed a spear at
+Patroclus and missed him, but he struck the horse Pedasus in the
+right shoulder, and it screamed aloud as it lay, groaning in the
+dust until the life went out of it. The other two horses began to
+plunge; the pole of the chariot cracked and they got entangled in
+the reins through the fall of the horse that was yoked along with
+them; but Automedon knew what to do; without the loss of a moment
+he drew the keen blade that hung by his sturdy thigh and cut the
+third horse adrift; whereon the other two righted themselves, and
+pulling hard at the reins again went together into battle.
+</para>
+
+<para>
+Sarpedon now took a second aim at Patroclus, and again missed
+him, the point of the spear passed over his left shoulder without
+hitting him. Patroclus then aimed in his turn, and the spear sped
+not from his hand in vain, for he hit Sarpedon just where the
+midriff surrounds the ever-beating heart. He fell like some oak
+or silver poplar or tall pine to which woodmen have laid their
+axes upon the mountains to make timber for ship-building--even so
+did he lie stretched at full length in front of his chariot and
+horses, moaning and clutching at the blood-stained dust. As when
+a lion springs with a bound upon a herd of cattle and fastens on
+a great black bull which dies bellowing in its clutches--even so
+did the leader of the Lycian warriors struggle in death as he
+fell by the hand of Patroclus. He called on his trusty comrade
+and said, "Glaucus, my brother, hero among heroes, put forth all
+your strength, fight with might and main, now if ever quit
+yourself like a valiant soldier. First go about among the Lycian
+captains and bid them fight for Sarpedon; then yourself also do
+battle to save my armour from being taken. My name will haunt you
+henceforth and for ever if the Achaeans rob me of my armour now
+that I have fallen at their ships. Do your very utmost and call
+all my people together."
+</para>
+
+<para>
+Death closed his eyes as he spoke. Patroclus planted his heel on
+his breast and drew the spear from his body, whereon his senses
+came out along with it, and he drew out both spear-point and
+Sarpedon's soul at the same time. Hard by the Myrmidons held his
+snorting steeds, who were wild with panic at finding themselves
+deserted by their lords.
+</para>
+
+<para>
+Glaucus was overcome with grief when he heard what Sarpedon said,
+for he could not help him. He had to support his arm with his
+other hand, being in great pain through the wound which Teucer's
+arrow had given him when Teucer was defending the wall as he,
+Glaucus, was assailing it. Therefore he prayed to far-darting
+Apollo saying, "Hear me O king from your seat, may be in the rich
+land of Lycia, or may be in Troy, for in all places you can hear
+the prayer of one who is in distress, as I now am. I have a
+grievous wound; my hand is aching with pain, there is no
+staunching the blood, and my whole arm drags by reason of my
+hurt, so that I cannot grasp my sword nor go among my foes and
+fight them, thou our prince, Jove's son Sarpedon, is slain. Jove
+defended not his son, do you, therefore, O king, heal me of my
+wound, ease my pain and grant me strength both to cheer on the
+Lycians and to fight along with them round the body of him who
+has fallen."
+</para>
+
+<para>
+Thus did he pray, and Apollo heard his prayer. He eased his pain,
+staunched the black blood from the wound, and gave him new
+strength. Glaucus perceived this, and was thankful that the
+mighty god had answered his prayer; forthwith, therefore, he went
+among the Lycian captains, and bade them come to fight about the
+body of Sarpedon. From these he strode on among the Trojans to
+Polydamas son of Panthous and Agenor; he then went in search of
+Aeneas and Hector, and when he had found them he said, "Hector,
+you have utterly forgotten your allies, who languish here for
+your sake far from friends and home while you do nothing to
+support them. Sarpedon leader of the Lycian warriors has fallen--
+he who was at once the right and might of Lycia; Mars has laid
+him low by the spear of Patroclus. Stand by him, my friends, and
+suffer not the Myrmidons to strip him of his armour, nor to treat
+his body with contumely in revenge for all the Danaans whom we
+have speared at the ships."
+</para>
+
+<para>
+As he spoke the Trojans were plunged in extreme and ungovernable
+grief; for Sarpedon, alien though he was, had been one of the
+main stays of their city, both as having much people with him,
+and himself the foremost among them all. Led by Hector, who was
+infuriated by the fall of Sarpedon, they made instantly for the
+Danaans with all their might, while the undaunted spirit of
+Patroclus son of Menoetius cheered on the Achaeans. First he
+spoke to the two Ajaxes, men who needed no bidding. "Ajaxes,"
+said he, "may it now please you to show yourselves the men you
+have always been, or even better--Sarpedon is fallen--he who was
+first to overleap the wall of the Achaeans; let us take the body
+and outrage it; let us strip the armour from his shoulders, and
+kill his comrades if they try to rescue his body."
+</para>
+
+<para>
+He spoke to men who of themselves were full eager; both sides,
+therefore, the Trojans and Lycians on the one hand, and the
+Myrmidons and Achaeans on the other, strengthened their
+battalions, and fought desperately about the body of Sarpedon,
+shouting fiercely the while. Mighty was the din of their armour
+as they came together, and Jove shed a thick darkness over the
+fight, to increase the toil of the battle over the body of his
+son.
+</para>
+
+<para>
+At first the Trojans made some headway against the Achaeans, for
+one of the best men among the Myrmidons was killed, Epeigeus, son
+of noble Agacles who had erewhile been king in the good city of
+Budeum; but presently, having killed a valiant kinsman of his
+own, he took refuge with Peleus and Thetis, who sent him to Ilius
+the land of noble steeds to fight the Trojans under Achilles.
+Hector now struck him on the head with a stone just as he had
+caught hold of the body, and his brains inside his helmet were
+all battered in, so that he fell face foremost upon the body of
+Sarpedon, and there died. Patroclus was enraged by the death of
+his comrade, and sped through the front ranks as swiftly as a
+hawk that swoops down on a flock of daws or starlings. Even so
+swiftly, O noble knight Patroclus, did you make straight for the
+Lycians and Trojans to avenge your comrade. Forthwith he struck
+Sthenelaus the son of Ithaemenes on the neck with a stone, and
+broke the tendons that join it to the head and spine. On this
+Hector and the front rank of his men gave ground. As far as a man
+can throw a javelin when competing for some prize, or even in
+battle--so far did the Trojans now retreat before the Achaeans.
+Glaucus, captain of the Lycians, was the first to rally them, by
+killing Bathycles son of Chalcon who lived in Hellas and was the
+richest man among the Myrmidons. Glaucus turned round suddenly,
+just as Bathycles who was pursuing him was about to lay hold of
+him, and drove his spear right into the middle of his chest,
+whereon he fell heavily to the ground, and the fall of so good a
+man filled the Achaeans with dismay, while the Trojans were
+exultant, and came up in a body round the corpse. Nevertheless
+the Achaeans, mindful of their prowess, bore straight down upon
+them.
+</para>
+
+<para>
+Meriones then killed a helmed warrior of the Trojans, Laogonus
+son of Onetor, who was priest of Jove of Mt. Ida, and was
+honoured by the people as though he were a god. Meriones struck
+him under the jaw and ear, so that life went out of him and the
+darkness of death laid hold upon him. Aeneas then aimed a spear
+at Meriones, hoping to hit him under the shield as he was
+advancing, but Meriones saw it coming and stooped forward to
+avoid it, whereon the spear flew past him and the point stuck in
+the ground, while the butt-end went on quivering till Mars robbed
+it of its force. The spear, therefore, sped from Aeneas's hand in
+vain and fell quivering to the ground. Aeneas was angry and said,
+"Meriones, you are a good dancer, but if I had hit you my spear
+would soon have made an end of you."
+</para>
+
+<para>
+And Meriones answered, "Aeneas, for all your bravery, you will
+not be able to make an end of every one who comes against you.
+You are only a mortal like myself, and if I were to hit you in
+the middle of your shield with my spear, however strong and
+self-confident you may be, I should soon vanquish you, and you
+would yield your life to Hades of the noble steeds."
+</para>
+
+<para>
+On this the son of Menoetius rebuked him and said, "Meriones,
+hero though you be, you should not speak thus; taunting speeches,
+my good friend, will not make the Trojans draw away from the dead
+body; some of them must go under ground first; blows for battle,
+and words for council; fight, therefore, and say nothing."
+</para>
+
+<para>
+He led the way as he spoke and the hero went forward with him.
+As the sound of woodcutters in some forest glade upon the
+mountains--and the thud of their axes is heard afar--even such a
+din now rose from earth-clash of bronze armour and of good
+ox-hide shields, as men smote each other with their swords and
+spears pointed at both ends. A man had need of good eyesight now
+to know Sarpedon, so covered was he from head to foot with spears
+and blood and dust. Men swarmed about the body, as flies that
+buzz round the full milk-pails in spring when they are brimming
+with milk--even so did they gather round Sarpedon; nor did Jove
+turn his keen eyes away for one moment from the fight, but kept
+looking at it all the time, for he was settling how best to kill
+Patroclus, and considering whether Hector should be allowed to
+end him now in the fight round the body of Sarpedon, and strip
+him of his armour, or whether he should let him give yet further
+trouble to the Trojans. In the end, he deemed it best that the
+brave squire of Achilles son of Peleus should drive Hector and
+the Trojans back towards the city and take the lives of many.
+First, therefore, he made Hector turn fainthearted, whereon he
+mounted his chariot and fled, bidding the other Trojans fly also,
+for he saw that the scales of Jove had turned against him.
+Neither would the brave Lycians stand firm; they were dismayed
+when they saw their king lying struck to the heart amid a heap of
+corpses--for when the son of Saturn made the fight wax hot many
+had fallen above him. The Achaeans, therefore stripped the
+gleaming armour from his shoulders and the brave son of Menoetius
+gave it to his men to take to the ships. Then Jove lord of the
+storm-cloud said to Apollo, "Dear Phoebus, go, I pray you, and
+take Sarpedon out of range of the weapons; cleanse the black
+blood from off him, and then bear him a long way off where you
+may wash him in the river, anoint him with ambrosia, and clothe
+him in immortal raiment; this done, commit him to the arms of the
+two fleet messengers, Death, and Sleep, who will carry him
+straightway to the rich land of Lycia, where his brothers and
+kinsmen will inter him, and will raise both mound and pillar to
+his memory, in due honour to the dead."
+</para>
+
+<para>
+Thus he spoke. Apollo obeyed his father's saying, and came down
+from the heights of Ida into the thick of the fight; forthwith he
+took Sarpedon out of range of the weapons, and then bore him a
+long way off, where he washed him in the river, anointed him with
+ambrosia and clothed him in immortal raiment; this done, he
+committed him to the arms of the two fleet messengers, Death, and
+Sleep, who presently set him down in the rich land of Lycia.
+</para>
+
+<para>
+Meanwhile Patroclus, with many a shout to his horses and to
+Automedon, pursued the Trojans and Lycians in the pride and
+foolishness of his heart. Had he but obeyed the bidding of the
+son of Peleus, he would have, escaped death and have been
+scatheless; but the counsels of Jove pass man's understanding; he
+will put even a brave man to flight and snatch victory from his
+grasp, or again he will set him on to fight, as he now did when
+he put a high spirit into the heart of Patroclus.
+</para>
+
+<para>
+Who then first, and who last, was slain by you, O Patroclus, when
+the gods had now called you to meet your doom? First Adrestus,
+Autonous, Echeclus, Perimus the son of Megas, Epistor and
+Melanippus; after these he killed Elasus, Mulius, and Pylartes.
+These he slew, but the rest saved themselves by flight.
+</para>
+
+<para>
+The sons of the Achaeans would now have taken Troy by the hands
+of Patroclus, for his spear flew in all directions, had not
+Phoebus Apollo taken his stand upon the wall to defeat his
+purpose and to aid the Trojans. Thrice did Patroclus charge at an
+angle of the high wall, and thrice did Apollo beat him back,
+striking his shield with his own immortal hands. When Patroclus
+was coming on like a god for yet a fourth time, Apollo shouted to
+him with an awful voice and said, "Draw back, noble Patroclus, it
+is not your lot to sack the city of the Trojan chieftains, nor
+yet will it be that of Achilles who is a far better man than you
+are." On hearing this, Patroclus withdrew to some distance and
+avoided the anger of Apollo.
+</para>
+
+<para>
+Meanwhile Hector was waiting with his horses inside the Scaean
+gates, in doubt whether to drive out again and go on fighting, or
+to call the army inside the gates. As he was thus doubting
+Phoebus Apollo drew near him in the likeness of a young and lusty
+warrior Asius, who was Hector's uncle, being own brother to
+Hecuba, and son of Dymas who lived in Phrygia by the waters of
+the river Sangarius; in his likeness Jove's son Apollo now spoke
+to Hector saying, "Hector, why have you left off fighting? It is
+ill done of you. If I were as much better a man than you, as I am
+worse, you should soon rue your slackness. Drive straight towards
+Patroclus, if so be that Apollo may grant you a triumph over him,
+and you may rule him."
+</para>
+
+<para>
+With this the god went back into the hurly-burly, and Hector bade
+Cebriones drive again into the fight. Apollo passed in among
+them, and struck panic into the Argives, while he gave triumph to
+Hector and the Trojans. Hector let the other Danaans alone and
+killed no man, but drove straight at Patroclus. Patroclus then
+sprang from his chariot to the ground, with a spear in his left
+hand, and in his right a jagged stone as large as his hand could
+hold. He stood still and threw it, nor did it go far without
+hitting some one; the cast was not in vain, for the stone struck
+Cebriones, Hector's charioteer, a bastard son of Priam, as he
+held the reins in his hands. The stone hit him on the forehead
+and drove his brows into his head for the bone was smashed, and
+his eyes fell to the ground at his feet. He dropped dead from his
+chariot as though he were diving, and there was no more life left
+in him. Over him did you then vaunt, O knight Patroclus, saying,
+"Bless my heart, how active he is, and how well he dives. If we
+had been at sea this fellow would have dived from the ship's side
+and brought up as many oysters as the whole crew could stomach,
+even in rough water, for he has dived beautifully off his chariot
+on to the ground. It seems, then, that there are divers also
+among the Trojans."
+</para>
+
+<para>
+As he spoke he flung himself on Cebriones with the spring, as it
+were, of a lion that while attacking a stockyard is himself
+struck in the chest, and his courage is his own bane--even so
+furiously, O Patroclus, did you then spring upon Cebriones.
+Hector sprang also from his chariot to the ground. The pair then
+fought over the body of Cebriones. As two lions fight fiercely on
+some high mountain over the body of a stag that they have killed,
+even so did these two mighty warriors, Patroclus son of Menoetius
+and brave Hector, hack and hew at one another over the corpse of
+Cebriones. Hector would not let him go when he had once got him
+by the head, while Patroclus kept fast hold of his feet, and a
+fierce fight raged between the other Danaans and Trojans. As the
+east and south wind buffet one another when they beat upon some
+dense forest on the mountains--there is beech and ash and
+spreading cornel; the top of the trees roar as they beat on one
+another, and one can hear the boughs cracking and breaking--even
+so did the Trojans and Achaeans spring upon one another and lay
+about each other, and neither side would give way. Many a pointed
+spear fell to ground and many a winged arrow sped from its
+bow-string about the body of Cebriones; many a great stone,
+moreover, beat on many a shield as they fought around his body,
+but there he lay in the whirling clouds of dust, all huge and
+hugely, heedless of his driving now.
+</para>
+
+<para>
+So long as the sun was still high in mid-heaven the weapons of
+either side were alike deadly, and the people fell; but when he
+went down towards the time when men loose their oxen, the
+Achaeans proved to be beyond all forecast stronger, so that they
+drew Cebriones out of range of the darts and tumult of the
+Trojans, and stripped the armour from his shoulders. Then
+Patroclus sprang like Mars with fierce intent and a terrific
+shout upon the Trojans, and thrice did he kill nine men; but as
+he was coming on like a god for a time, then, O Patroclus, was
+the hour of your end approaching, for Phoebus fought you in fell
+earnest. Patroclus did not see him as he moved about in the
+crush, for he was enshrouded in thick darkness, and the god
+struck him from behind on his back and his broad shoulders with
+the flat of his hand, so that his eyes turned dizzy. Phoebus
+Apollo beat the helmet from off his head, and it rolled rattling
+off under the horses' feet, where its horse-hair plumes were all
+begrimed with dust and blood. Never indeed had that helmet fared
+so before, for it had served to protect the head and comely
+forehead of the godlike hero Achilles. Now, however, Zeus
+delivered it over to be worn by Hector. Nevertheless the end of
+Hector also was near. The bronze-shod spear, so great and so
+strong, was broken in the hand of Patroclus, while his shield
+that covered him from head to foot fell to the ground as did also
+the band that held it, and Apollo undid the fastenings of his
+corslet.
+</para>
+
+<para>
+On this his mind became clouded; his limbs failed him, and he
+stood as one dazed; whereon Euphorbus son of Panthous a
+Dardanian, the best spearman of his time, as also the finest
+horseman and fleetest runner, came behind him and struck him in
+the back with a spear, midway between the shoulders. This man as
+soon as ever he had come up with his chariot had dismounted
+twenty men, so proficient was he in all the arts of war--he it
+was, O knight Patroclus, that first drove a weapon into you, but
+he did not quite overpower you. Euphorbus then ran back into the
+crowd, after drawing his ashen spear out of the wound; he would
+not stand firm and wait for Patroclus, unarmed though he now was,
+to attack him; but Patroclus unnerved, alike by the blow the god
+had given him and by the spear-wound, drew back under cover of
+his men in fear for his life. Hector on this, seeing him to be
+wounded and giving ground, forced his way through the ranks, and
+when close up with him struck him in the lower part of the belly
+with a spear, driving the bronze point right through it, so that
+he fell heavily to the ground to the great of the Achaeans. As
+when a lion has fought some fierce wild-boar and worsted him--the
+two fight furiously upon the mountains over some little fountain
+at which they would both drink, and the lion has beaten the boar
+till he can hardly breathe--even so did Hector son of Priam take
+the life of the brave son of Menoetius who had killed so many,
+striking him from close at hand, and vaunting over him the while.
+"Patroclus," said he, "you deemed that you should sack our city,
+rob our Trojan women of their freedom, and carry them off in your
+ships to your own country. Fool; Hector and his fleet horses were
+ever straining their utmost to defend them. I am foremost of all
+the Trojan warriors to stave the day of bondage from off them; as
+for you, vultures shall devour you here. Poor wretch, Achilles
+with all his bravery availed you nothing; and yet I ween when you
+left him he charged you straitly saying, 'Come not back to the
+ships, knight Patroclus, till you have rent the bloodstained
+shirt of murderous Hector about his body. Thus I ween did he
+charge you, and your fool's heart answered him 'yea' within you."
+</para>
+
+<para>
+Then, as the life ebbed out of you, you answered, O knight
+Patroclus: "Hector, vaunt as you will, for Jove the son of Saturn
+and Apollo have vouchsafed you victory; it is they who have
+vanquished me so easily, and they who have stripped the armour
+from my shoulders; had twenty such men as you attacked me, all of
+them would have fallen before my spear. Fate and the son of Leto
+have overpowered me, and among mortal men Euphorbus; you are
+yourself third only in the killing of me. I say further, and lay
+my saying to your heart, you too shall live but for a little
+season; death and the day of your doom are close upon you, and
+they will lay you low by the hand of Achilles son of Aeacus."
+</para>
+
+<para>
+When he had thus spoken his eyes were closed in death, his soul
+left his body and flitted down to the house of Hades, mourning
+its sad fate and bidding farewell to the youth and vigor of its
+manhood. Dead though he was, Hector still spoke to him saying,
+"Patroclus, why should you thus foretell my doom? Who knows but
+Achilles, son of lovely Thetis, may be smitten by my spear and
+die before me?"
+</para>
+
+<para>
+As he spoke he drew the bronze spear from the wound, planting his
+foot upon the body, which he thrust off and let lie on its back.
+He then went spear in hand after Automedon, squire of the fleet
+descendant of Aeacus, for he longed to lay him low, but the
+immortal steeds which the gods had given as a rich gift to Peleus
+bore him swiftly from the field.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XVII</title>
+
+<para>
+BRAVE Menelaus son of Atreus now came to know that Patroclus had
+fallen, and made his way through the front ranks clad in full
+armour to bestride him. As a cow stands lowing over her first
+calf, even so did yellow-haired Menelaus bestride Patroclus. He
+held his round shield and his spear in front of him, resolute to
+kill any who should dare face him. But the son of Panthous had
+also noted the body, and came up to Menelaus saying, "Menelaus,
+son of Atreus, draw back, leave the body, and let the
+bloodstained spoils be. I was first of the Trojans and their
+brave allies to drive my spear into Patroclus, let me, therefore,
+have my full glory among the Trojans, or I will take aim and kill
+you."
+</para>
+
+<para>
+To this Menelaus answered in great anger "By father Jove,
+boasting is an ill thing. The pard is not more bold, nor the lion
+nor savage wild-boar, which is fiercest and most dauntless of all
+creatures, than are the proud sons of Panthous. Yet Hyperenor did
+not see out the days of his youth when he made light of me and
+withstood me, deeming me the meanest soldier among the Danaans.
+His own feet never bore him back to gladden his wife and parents.
+Even so shall I make an end of you too, if you withstand me; get
+you back into the crowd and do not face me, or it shall be worse
+for you. Even a fool may be wise after the event."
+</para>
+
+<para>
+Euphorbus would not listen, and said, "Now indeed, Menelaus,
+shall you pay for the death of my brother over whom you vaunted,
+and whose wife you widowed in her bridal chamber, while you
+brought grief unspeakable on his parents. I shall comfort these
+poor people if I bring your head and armour and place them in the
+hands of Panthous and noble Phrontis. The time is come when this
+matter shall be fought out and settled, for me or against me."
+</para>
+
+<para>
+As he spoke he struck Menelaus full on the shield, but the spear
+did not go through, for the shield turned its point. Menelaus
+then took aim, praying to father Jove as he did so; Euphorbus was
+drawing back, and Menelaus struck him about the roots of his
+throat, leaning his whole weight on the spear, so as to drive it
+home. The point went clean through his neck, and his armour rang
+rattling round him as he fell heavily to the ground. His hair
+which was like that of the Graces, and his locks so deftly bound
+in bands of silver and gold, were all bedrabbled with blood. As
+one who has grown a fine young olive tree in a clear space where
+there is abundance of water--the plant is full of promise, and
+though the winds beat upon it from every quarter it puts forth
+its white blossoms till the blasts of some fierce hurricane sweep
+down upon it and level it with the ground--even so did Menelaus
+strip the fair youth Euphorbus of his armour after he had slain
+him. Or as some fierce lion upon the mountains in the pride of
+his strength fastens on the finest heifer in a herd as it is
+feeding--first he breaks her neck with his strong jaws, and then
+gorges on her blood and entrails; dogs and shepherds raise a hue
+and cry against him, but they stand aloof and will not come close
+to him, for they are pale with fear--even so no one had the
+courage to face valiant Menelaus. The son of Atreus would have
+then carried off the armour of the son of Panthous with ease, had
+not Phoebus Apollo been angry, and in the guise of Mentes chief
+of the Cicons incited Hector to attack him. "Hector," said he,
+"you are now going after the horses of the noble son of Aeacus,
+but you will not take them; they cannot be kept in hand and
+driven by mortal man, save only by Achilles, who is son to an
+immortal mother. Meanwhile Menelaus son of Atreus has bestridden
+the body of Patroclus and killed the noblest of the Trojans,
+Euphorbus son of Panthous, so that he can fight no more."
+</para>
+
+<para>
+The god then went back into the toil and turmoil, but the soul of
+Hector was darkened with a cloud of grief; he looked along the
+ranks and saw Euphorbus lying on the ground with the blood still
+flowing from his wound, and Menelaus stripping him of his armour.
+On this he made his way to the front like a flame of fire, clad
+in his gleaming armour, and crying with a loud voice. When the
+son of Atreus heard him, he said to himself in his dismay, "Alas!
+what shall I do? I may not let the Trojans take the armour of
+Patroclus who has fallen fighting on my behalf, lest some Danaan
+who sees me should cry shame upon me. Still if for my honour's
+sake I fight Hector and the Trojans single-handed, they will
+prove too many for me, for Hector is bringing them up in force.
+Why, however, should I thus hesitate? When a man fights in
+despite of heaven with one whom a god befriends, he will soon rue
+it. Let no Danaan think ill of me if I give place to Hector, for
+the hand of heaven is with him. Yet, if I could find Ajax, the
+two of us would fight Hector and heaven too, if we might only
+save the body of Patroclus for Achilles son of Peleus. This, of
+many evils would be the least."
+</para>
+
+<para>
+While he was thus in two minds, the Trojans came up to him with
+Hector at their head; he therefore drew back and left the body,
+turning about like some bearded lion who is being chased by dogs
+and men from a stockyard with spears and hue and cry, whereon he
+is daunted and slinks sulkily off--even so did Menelaus son of
+Atreus turn and leave the body of Patroclus. When among the body
+of his men, he looked around for mighty Ajax son of Telamon, and
+presently saw him on the extreme left of the fight, cheering on
+his men and exhorting them to keep on fighting, for Phoebus
+Apollo had spread a great panic among them. He ran up to him and
+said, "Ajax, my good friend, come with me at once to dead
+Patroclus, if so be that we may take the body to Achilles--as for
+his armour, Hector already has it."
+</para>
+
+<para>
+These words stirred the heart of Ajax, and he made his way among
+the front ranks, Menelaus going with him. Hector had stripped
+Patroclus of his armour, and was dragging him away to cut off his
+head and take the body to fling before the dogs of Troy. But Ajax
+came up with his shield like wall before him, on which Hector
+withdrew under shelter of his men, and sprang on to his chariot,
+giving the armour over to the Trojans to take to the city, as a
+great trophy for himself; Ajax, therefore, covered the body of
+Patroclus with his broad shield and bestrode him; as a lion
+stands over his whelps if hunters have come upon him in a forest
+when he is with his little ones--in the pride and fierceness of
+his strength he draws his knit brows down till they cover his
+eyes--even so did Ajax bestride the body of Patroclus, and by his
+side stood Menelaus son of Atreus, nursing great sorrow in his
+heart.
+</para>
+
+<para>
+Then Glaucus son of Hippolochus looked fiercely at Hector and
+rebuked him sternly. "Hector," said he, "you make a brave show,
+but in fight you are sadly wanting. A runaway like yourself has
+no claim to so great a reputation. Think how you may now save
+your town and citadel by the hands of your own people born in
+Ilius; for you will get no Lycians to fight for you, seeing what
+thanks they have had for their incessant hardships. Are you
+likely, sir, to do anything to help a man of less note, after
+leaving Sarpedon, who was at once your guest and comrade in arms,
+to be the spoil and prey of the Danaans? So long as he lived he
+did good service both to your city and yourself; yet you had no
+stomach to save his body from the dogs. If the Lycians will
+listen to me, they will go home and leave Troy to its fate. If
+the Trojans had any of that daring fearless spirit which lays
+hold of men who are fighting for their country and harassing
+those who would attack it, we should soon bear off Patroclus into
+Ilius. Could we get this dead man away and bring him into the
+city of Priam, the Argives would readily give up the armour of
+Sarpedon, and we should get his body to boot. For he whose squire
+has been now killed is the foremost man at the ships of the
+Achaeans--he and his close-fighting followers. Nevertheless you
+dared not make a stand against Ajax, nor face him, eye to eye,
+with battle all round you, for he is a braver man than you are."
+</para>
+
+<para>
+Hector scowled at him and answered, "Glaucus, you should know
+better. I have held you so far as a man of more understanding
+than any in all Lycia, but now I despise you for saying that I am
+afraid of Ajax. I fear neither battle nor the din of chariots,
+but Jove's will is stronger than ours; Jove at one time makes
+even a strong man draw back and snatches victory from his grasp,
+while at another he will set him on to fight. Come hither then,
+my friend, stand by me and see indeed whether I shall play the
+coward the whole day through as you say, or whether I shall not
+stay some even of the boldest Danaans from fighting round the
+body of Patroclus."
+</para>
+
+<para>
+As he spoke he called loudly on the Trojans saying, "Trojans,
+Lycians, and Dardanians, fighters in close combat, be men, my
+friends, and fight might and main, while I put on the goodly
+armour of Achilles, which I took when I killed Patroclus."
+</para>
+
+<para>
+With this Hector left the fight, and ran full speed after his men
+who were taking the armour of Achilles to Troy, but had not yet
+got far. Standing for a while apart from the woeful fight, he
+changed his armour. His own he sent to the strong city of Ilius
+and to the Trojans, while he put on the immortal armour of the
+son of Peleus, which the gods had given to Peleus, who in his age
+gave it to his son; but the son did not grow old in his father's
+armour.
+</para>
+
+<para>
+When Jove, lord of the storm-cloud, saw Hector standing aloof and
+arming himself in the armour of the son of Peleus, he wagged his
+head and muttered to himself saying, "A! poor wretch, you arm in
+the armour of a hero, before whom many another trembles, and you
+reck nothing of the doom that is already close upon you. You have
+killed his comrade so brave and strong, but it was not well that
+you should strip the armour from his head and shoulders. I do
+indeed endow you with great might now, but as against this you
+shall not return from battle to lay the armour of the son of
+Peleus before Andromache."
+</para>
+
+<para>
+The son of Saturn bowed his portentous brows, and Hector fitted
+the armour to his body, while terrible Mars entered into him, and
+filled his whole body with might and valour. With a shout he
+strode in among the allies, and his armour flashed about him so
+that he seemed to all of them like the great son of Peleus
+himself. He went about among them and cheered them on--Mesthles,
+Glaucus, Medon, Thersilochus, Asteropaeus, Deisenor and
+Hippothous, Phorcys, Chromius and Ennomus the augur. All these
+did he exhort saying, "Hear me, allies from other cities who are
+here in your thousands, it was not in order to have a crowd about
+me that I called you hither each from his several city, but that
+with heart and soul you might defend the wives and little ones of
+the Trojans from the fierce Achaeans. For this do I oppress my
+people with your food and the presents that make you rich.
+Therefore turn, and charge at the foe, to stand or fall as is the
+game of war; whoever shall bring Patroclus, dead though he be,
+into the hands of the Trojans, and shall make Ajax give way
+before him, I will give him one half of the spoils while I keep
+the other. He will thus share like honour with myself."
+</para>
+
+<para>
+When he had thus spoken they charged full weight upon the Danaans
+with their spears held out before them, and the hopes of each ran
+high that he should force Ajax son of Telamon to yield up the
+body--fools that they were, for he was about to take the lives of
+many. Then Ajax said to Menelaus, "My good friend Menelaus, you
+and I shall hardly come out of this fight alive. I am less
+concerned for the body of Patroclus, who will shortly become meat
+for the dogs and vultures of Troy, than for the safety of my own
+head and yours. Hector has wrapped us round in a storm of battle
+from every quarter, and our destruction seems now certain. Call
+then upon the princes of the Danaans if there is any who can hear
+us."
+</para>
+
+<para>
+Menelaus did as he said, and shouted to the Danaans for help at
+the top of his voice. "My friends," he cried, "princes and
+counsellors of the Argives, all you who with Agamemnon and
+Menelaus drink at the public cost, and give orders each to his
+own people as Jove vouchsafes him power and glory, the fight is
+so thick about me that I cannot distinguish you severally; come
+on, therefore, every man unbidden, and think it shame that
+Patroclus should become meat and morsel for Trojan hounds."
+</para>
+
+<para>
+Fleet Ajax son of Oileus heard him and was first to force his way
+through the fight and run to help him. Next came Idomeneus and
+Meriones his esquire, peer of murderous Mars. As for the others
+that came into the fight after these, who of his own self could
+name them?
+</para>
+
+<para>
+The Trojans with Hector at their head charged in a body. As a
+great wave that comes thundering in at the mouth of some
+heaven-born river, and the rocks that jut into the sea ring with
+the roar of the breakers that beat and buffet them--even with
+such a roar did the Trojans come on; but the Achaeans in
+singleness of heart stood firm about the son of Menoetius, and
+fenced him with their bronze shields. Jove, moreover, hid the
+brightness of their helmets in a thick cloud, for he had borne no
+grudge against the son of Menoetius while he was still alive and
+squire to the descendant of Aeacus; therefore he was loth to let
+him fall a prey to the dogs of his foes the Trojans, and urged
+his comrades on to defend him.
+</para>
+
+<para>
+At first the Trojans drove the Achaeans back, and they withdrew
+from the dead man daunted. The Trojans did not succeed in killing
+any one, nevertheless they drew the body away. But the Achaeans
+did not lose it long, for Ajax, foremost of all the Danaans after
+the son of Peleus alike in stature and prowess, quickly rallied
+them and made towards the front like a wild boar upon the
+mountains when he stands at bay in the forest glades and routs
+the hounds and lusty youths that have attacked him--even so did
+Ajax son of Telamon passing easily in among the phalanxes of the
+Trojans, disperse those who had bestridden Patroclus and were
+most bent on winning glory by dragging him off to their city. At
+this moment Hippothous brave son of the Pelasgian Lethus, in his
+zeal for Hector and the Trojans, was dragging the body off by the
+foot through the press of the fight, having bound a strap round
+the sinews near the ancle; but a mischief soon befell him from
+which none of those could save him who would have gladly done so,
+for the son of Telamon sprang forward and smote him on his
+bronze-cheeked helmet. The plumed headpiece broke about the point
+of the weapon, struck at once by the spear and by the strong hand
+of Ajax, so that the bloody brain came oozing out through the
+crest-socket. His strength then failed him and he let Patroclus'
+foot drop from his hand, as he fell full length dead upon the
+body; thus he died far from the fertile land of Larissa, and
+never repaid his parents the cost of bringing him up, for his
+life was cut short early by the spear of mighty Ajax. Hector then
+took aim at Ajax with a spear, but he saw it coming and just
+managed to avoid it; the spear passed on and struck Schedius son
+of noble Iphitus, captain of the Phoceans, who dwelt in famed
+Panopeus and reigned over much people; it struck him under the
+middle of the collar-bone the bronze point went right through
+him, coming out at the bottom of his shoulder-blade, and his
+armour rang rattling round him as he fell heavily to the ground.
+Ajax in his turn struck noble Phorcys son of Phaenops in the
+middle of the belly as he was bestriding Hippothous, and broke
+the plate of his cuirass; whereon the spear tore out his entrails
+and he clutched the ground in his palm as he fell to earth.
+Hector and those who were in the front rank then gave ground,
+while the Argives raised a loud cry of triumph, and drew off the
+bodies of Phorcys and Hippothous which they stripped presently of
+their armour.
+</para>
+
+<para>
+The Trojans would now have been worsted by the brave Achaeans and
+driven back to Ilius through their own cowardice, while the
+Argives, so great was their courage and endurance, would have
+achieved a triumph even against the will of Jove, if Apollo had
+not roused Aeneas, in the likeness of Periphas son of Epytus, an
+attendant who had grown old in the service of Aeneas' aged
+father, and was at all times devoted to him. In his likeness,
+then, Apollo said, "Aeneas, can you not manage, even though
+heaven be against us, to save high Ilius? I have known men, whose
+numbers, courage, and self-reliance have saved their people in
+spite of Jove, whereas in this case he would much rather give
+victory to us than to the Danaans, if you would only fight
+instead of being so terribly afraid."
+</para>
+
+<para>
+Aeneas knew Apollo when he looked straight at him, and shouted to
+Hector saying, "Hector and all other Trojans and allies, shame on
+us if we are beaten by the Achaeans and driven back to Ilius
+through our own cowardice. A god has just come up to me and told
+me that Jove the supreme disposer will be with us. Therefore let
+us make for the Danaans, that it may go hard with them ere they
+bear away dead Patroclus to the ships."
+</para>
+
+<para>
+As he spoke he sprang out far in front of the others, who then
+rallied and again faced the Achaeans. Aeneas speared Leiocritus
+son of Arisbas, a valiant follower of Lycomedes, and Lycomedes
+was moved with pity as he saw him fall; he therefore went close
+up, and speared Apisaon son of Hippasus shepherd of his people in
+the liver under the midriff, so that he died; he had come from
+fertile Paeonia and was the best man of them all after
+Asteropaeus. Asteropaeus flew forward to avenge him and attack
+the Danaans, but this might no longer be, inasmuch as those about
+Patroclus were well covered by their shields, and held their
+spears in front of them, for Ajax had given them strict orders
+that no man was either to give ground, or to stand out before the
+others, but all were to hold well together about the body and
+fight hand to hand. Thus did huge Ajax bid them, and the earth
+ran red with blood as the corpses fell thick on one another alike
+on the side of the Trojans and allies, and on that of the
+Danaans; for these last, too, fought no bloodless fight though
+many fewer of them perished, through the care they took to defend
+and stand by one another.
+</para>
+
+<para>
+Thus did they fight as it were a flaming fire; it seemed as
+though it had gone hard even with the sun and moon, for they were
+hidden over all that part where the bravest heroes were fighting
+about the dead son of Menoetius, whereas the other Danaans and
+Achaeans fought at their ease in full daylight with brilliant
+sunshine all round them, and there was not a cloud to be seen
+neither on plain nor mountain. These last moreover would rest for
+a while and leave off fighting, for they were some distance apart
+and beyond the range of one another's weapons, whereas those who
+were in the thick of the fray suffered both from battle and
+darkness. All the best of them were being worn out by the great
+weight of their armour, but the two valiant heroes, Thrasymedes
+and Antilochus, had not yet heard of the death of Patroclus, and
+believed him to be still alive and leading the van against the
+Trojans; they were keeping themselves in reserve against the
+death or rout of their own comrades, for so Nestor had ordered
+when he sent them from the ships into battle.
+</para>
+
+<para>
+Thus through the livelong day did they wage fierce war, and the
+sweat of their toil rained ever on their legs under them, and on
+their hands and eyes, as they fought over the squire of the fleet
+son of Peleus. It was as when a man gives a great ox-hide all
+drenched in fat to his men, and bids them stretch it; whereon
+they stand round it in a ring and tug till the moisture leaves
+it, and the fat soaks in for the many that pull at it, and it is
+well stretched--even so did the two sides tug the dead body
+hither and thither within the compass of but a little space--the
+Trojans steadfastly set on dragging it into Ilius, while the
+Achaeans were no less so on taking it to their ships; and fierce
+was the fight between them. Not Mars himself the lord of hosts,
+nor yet Minerva, even in their fullest fury could make light of
+such a battle.
+</para>
+
+<para>
+Such fearful turmoil of men and horses did Jove on that day
+ordain round the body of Patroclus. Meanwhile Achilles did not
+know that he had fallen, for the fight was under the wall of Troy
+a long way off the ships. He had no idea, therefore, that
+Patroclus was dead, and deemed that he would return alive as soon
+as he had gone close up to the gates. He knew that he was not to
+sack the city neither with nor without himself, for his mother
+had often told him this when he had sat alone with her, and she
+had informed him of the counsels of great Jove. Now, however, she
+had not told him how great a disaster had befallen him in the
+death of the one who was far dearest to him of all his comrades.
+</para>
+
+<para>
+The others still kept on charging one another round the body with
+their pointed spears and killing each other. Then would one say,
+"My friends, we can never again show our faces at the ships--
+better, and greatly better, that earth should open and swallow us
+here in this place, than that we should let the Trojans have the
+triumph of bearing off Patroclus to their city."
+</para>
+
+<para>
+The Trojans also on their part spoke to one another saying,
+"Friends, though we fall to a man beside this body, let none
+shrink from fighting." With such words did they exhort each
+other. They fought and fought, and an iron clank rose through the
+void air to the brazen vault of heaven. The horses of the
+descendant of Aeacus stood out of the fight and wept when they
+heard that their driver had been laid low by the hand of
+murderous Hector. Automedon, valiant son of Diores, lashed them
+again and again; many a time did he speak kindly to them, and
+many a time did he upbraid them, but they would neither go back
+to the ships by the waters of the broad Hellespont, nor yet into
+battle among the Achaeans; they stood with their chariot stock
+still, as a pillar set over the tomb of some dead man or woman,
+and bowed their heads to the ground. Hot tears fell from their
+eyes as they mourned the loss of their charioteer, and their
+noble manes drooped all wet from under the yokestraps on either
+side the yoke.
+</para>
+
+<para>
+The son of Saturn saw them and took pity upon their sorrow. He
+wagged his head, and muttered to himself, saying, "Poor things,
+why did we give you to King Peleus who is a mortal, while you are
+yourselves ageless and immortal? Was it that you might share the
+sorrows that befall mankind? for of all creatures that live and
+move upon the earth there is none so pitiable as he is--still,
+Hector son of Priam shall drive neither you nor your chariot. I
+will not have it. It is enough that he should have the armour
+over which he vaunts so vainly. Furthermore I will give you
+strength of heart and limb to bear Automedon safely to the ships
+from battle, for I shall let the Trojans triumph still further,
+and go on killing till they reach the ships; whereon night shall
+fall and darkness overshadow the land."
+</para>
+
+<para>
+As he spoke he breathed heart and strength into the horses so
+that they shook the dust from out of their manes, and bore their
+chariot swiftly into the fight that raged between Trojans and
+Achaeans. Behind them fought Automedon full of sorrow for his
+comrade, as a vulture amid a flock of geese. In and out, and here
+and there, full speed he dashed amid the throng of the Trojans,
+but for all the fury of his pursuit he killed no man, for he
+could not wield his spear and keep his horses in hand when alone
+in the chariot; at last, however, a comrade, Alcimedon, son of
+Laerces son of Haemon caught sight of him and came up behind his
+chariot. "Automedon," said he, "what god has put this folly into
+your heart and robbed you of your right mind, that you fight the
+Trojans in the front rank single-handed? He who was your comrade
+is slain, and Hector plumes himself on being armed in the armour
+of the descendant of Aeacus."
+</para>
+
+<para>
+Automedon son of Diores answered, "Alcimedon, there is no one
+else who can control and guide the immortal steeds so well as you
+can, save only Patroclus--while he was alive--peer of gods in
+counsel. Take then the whip and reins, while I go down from the
+car and fight."
+</para>
+
+<para>
+Alcimedon sprang on to the chariot, and caught up the whip and
+reins, while Automedon leaped from off the car. When Hector saw
+him he said to Aeneas who was near him, "Aeneas, counsellor of
+the mail-clad Trojans, I see the steeds of the fleet son of
+Aeacus come into battle with weak hands to drive them. I am sure,
+if you think well, that we might take them; they will not dare
+face us if we both attack them."
+</para>
+
+<para>
+The valiant son of Anchises was of the same mind, and the pair
+went right on, with their shoulders covered under shields of
+tough dry ox-hide, overlaid with much bronze. Chromius and Aretus
+went also with them, and their hearts beat high with hope that
+they might kill the men and capture the horses--fools that they
+were, for they were not to return scatheless from their meeting
+with Automedon, who prayed to father Jove and was forthwith
+filled with courage and strength abounding. He turned to his
+trusty comrade Alcimedon and said, "Alcimedon, keep your horses
+so close up that I may feel their breath upon my back; I doubt
+that we shall not stay Hector son of Priam till he has killed us
+and mounted behind the horses; he will then either spread panic
+among the ranks of the Achaeans, or himself be killed among the
+foremost."
+</para>
+
+<para>
+On this he cried out to the two Ajaxes and Menelaus, "Ajaxes
+captains of the Argives, and Menelaus, give the dead body over to
+them that are best able to defend it, and come to the rescue of
+us living; for Hector and Aeneas who are the two best men among
+the Trojans, are pressing us hard in the full tide of war.
+Nevertheless the issue lies on the lap of heaven, I will
+therefore hurl my spear and leave the rest to Jove."
+</para>
+
+<para>
+He poised and hurled as he spoke, whereon the spear struck the
+round shield of Aretus, and went right through it for the shield
+stayed it not, so that it was driven through his belt into the
+lower part of his belly. As when some sturdy youth, axe in hand,
+deals his blow behind the horns of an ox and severs the tendons
+at the back of its neck so that it springs forward and then
+drops, even so did Aretus give one bound and then fall on his
+back the spear quivering in his body till it made an end of him.
+Hector then aimed a spear at Automedon but he saw it coming and
+stooped forward to avoid it, so that it flew past him and the
+point stuck in the ground, while the butt-end went on quivering
+till Mars robbed it of its force. They would then have fought
+hand to hand with swords had not the two Ajaxes forced their way
+through the crowd when they heard their comrade calling, and
+parted them for all their fury--for Hector, Aeneas, and Chromius
+were afraid and drew back, leaving Aretus to lie there struck to
+the heart. Automedon, peer of fleet Mars, then stripped him of
+his armour and vaunted over him saying, "I have done little to
+assuage my sorrow for the son of Menoetius, for the man I have
+killed is not so good as he was."
+</para>
+
+<para>
+As he spoke he took the blood-stained spoils and laid them upon
+his chariot; then he mounted the car with his hands and feet all
+steeped in gore as a lion that has been gorging upon a bull.
+</para>
+
+<para>
+And now the fierce groanful fight again raged about Patroclus,
+for Minerva came down from heaven and roused its fury by the
+command of far-seeing Jove, who had changed his mind and sent her
+to encourage the Danaans. As when Jove bends his bright bow in
+heaven in token to mankind either of war or of the chill storms
+that stay men from their labour and plague the flocks--even so,
+wrapped in such radiant raiment, did Minerva go in among the host
+and speak man by man to each. First she took the form and voice
+of Phoenix and spoke to Menelaus son of Atreus, who was standing
+near her. "Menelaus," said she, "it will be shame and dishonour
+to you, if dogs tear the noble comrade of Achilles under the
+walls of Troy. Therefore be staunch, and urge your men to be so
+also."
+</para>
+
+<para>
+Menelaus answered, "Phoenix, my good old friend, may Minerva
+vouchsafe me strength and keep the darts from off me, for so
+shall I stand by Patroclus and defend him; his death has gone to
+my heart, but Hector is as a raging fire and deals his blows
+without ceasing, for Jove is now granting him a time of triumph."
+</para>
+
+<para>
+Minerva was pleased at his having named herself before any of the
+other gods. Therefore she put strength into his knees and
+shoulders, and made him as bold as a fly, which, though driven
+off will yet come again and bite if it can, so dearly does it
+love man's blood--even so bold as this did she make him as he
+stood over Patroclus and threw his spear. Now there was among the
+Trojans a man named Podes, son of Eetion, who was both rich and
+valiant. Hector held him in the highest honour for he was his
+comrade and boon companion; the spear of Menelaus struck this man
+in the girdle just as he had turned in flight, and went right
+through him. Whereon he fell heavily forward, and Menelaus son of
+Atreus drew off his body from the Trojans into the ranks of his
+own people.
+</para>
+
+<para>
+Apollo then went up to Hector and spurred him on to fight, in the
+likeness of Phaenops son of Asius who lived in Abydos and was the
+most favoured of all Hector's guests. In his likeness Apollo
+said, "Hector, who of the Achaeans will fear you henceforward now
+that you have quailed before Menelaus who has ever been rated
+poorly as a soldier? Yet he has now got a corpse away from the
+Trojans single-handed, and has slain your own true comrade, a man
+brave among the foremost, Podes son of Eetion."
+</para>
+
+<para>
+A dark cloud of grief fell upon Hector as he heard, and he made
+his way to the front clad in full armour. Thereon the son of
+Saturn seized his bright tasselled aegis, and veiled Ida in
+cloud: he sent forth his lightnings and his thunders, and as he
+shook his aegis he gave victory to the Trojans and routed the
+Achaeans.
+</para>
+
+<para>
+The panic was begun by Peneleos the Boeotian, for while keeping
+his face turned ever towards the foe he had been hit with a spear
+on the upper part of the shoulder; a spear thrown by Polydamas
+had grazed the top of the bone, for Polydamas had come up to him
+and struck him from close at hand. Then Hector in close combat
+struck Leitus son of noble Alectryon in the hand by the wrist,
+and disabled him from fighting further. He looked about him in
+dismay, knowing that never again should he wield spear in battle
+with the Trojans. While Hector was in pursuit of Leitus,
+Idomeneus struck him on the breastplate over his chest near the
+nipple; but the spear broke in the shaft, and the Trojans cheered
+aloud. Hector then aimed at Idomeneus son of Deucalion as he was
+standing on his chariot, and very narrowly missed him, but the
+spear hit Coiranus, a follower and charioteer of Meriones who had
+come with him from Lyctus. Idomeneus had left the ships on foot
+and would have afforded a great triumph to the Trojans if
+Coiranus had not driven quickly up to him, he therefore brought
+life and rescue to Idomeneus, but himself fell by the hand of
+murderous Hector. For Hector hit him on the jaw under the ear;
+the end of the spear drove out his teeth and cut his tongue in
+two pieces, so that he fell from his chariot and let the reins
+fall to the ground. Meriones gathered them up from the ground and
+took them into his own hands, then he said to Idomeneus, "Lay on,
+till you get back to the ships, for you must see that the day is
+no longer ours."
+</para>
+
+<para>
+On this Idomeneus lashed the horses to the ships, for fear had
+taken hold upon him.
+</para>
+
+<para>
+Ajax and Menelaus noted how Jove had turned the scale in favour
+of the Trojans, and Ajax was first to speak. "Alas," said he,
+"even a fool may see that father Jove is helping the Trojans. All
+their weapons strike home; no matter whether it be a brave man or
+a coward that hurls them, Jove speeds all alike, whereas ours
+fall each one of them without effect. What, then, will be best
+both as regards rescuing the body, and our return to the joy of
+our friends who will be grieving as they look hitherwards; for
+they will make sure that nothing can now check the terrible hands
+of Hector, and that he will fling himself upon our ships. I wish
+that some one would go and tell the son of Peleus at once, for I
+do not think he can have yet heard the sad news that the dearest
+of his friends has fallen. But I can see not a man among the
+Achaeans to send, for they and their chariots are alike hidden in
+darkness. O father Jove, lift this cloud from over the sons of
+the Achaeans; make heaven serene, and let us see; if you will
+that we perish, let us fall at any rate by daylight."
+</para>
+
+<para>
+Father Jove heard him and had compassion upon his tears.
+Forthwith he chased away the cloud of darkness, so that the sun
+shone out and all the fighting was revealed. Ajax then said to
+Menelaus, "Look, Menelaus, and if Antilochus son of Nestor be
+still living, send him at once to tell Achilles that by far the
+dearest to him of all his comrades has fallen."
+</para>
+
+<para>
+Menelaus heeded his words and went his way as a lion from a
+stockyard--the lion is tired of attacking the men and hounds, who
+keep watch the whole night through and will not let him feast on
+the fat of their herd. In his lust of meat he makes straight at
+them but in vain, for darts from strong hands assail him, and
+burning brands which daunt him for all his hunger, so in the
+morning he slinks sulkily away--even so did Menelaus sorely
+against his will leave Patroclus, in great fear lest the Achaeans
+should be driven back in rout and let him fall into the hands of
+the foe. He charged Meriones and the two Ajaxes straitly saying,
+"Ajaxes and Meriones, leaders of the Argives, now indeed remember
+how good Patroclus was; he was ever courteous while alive, bear
+it in mind now that he is dead."
+</para>
+
+<para>
+With this Menelaus left them, looking round him as keenly as an
+eagle, whose sight they say is keener than that of any other
+bird--however high he may be in the heavens, not a hare that runs
+can escape him by crouching under bush or thicket, for he will
+swoop down upon it and make an end of it--even so, O Menelaus,
+did your keen eyes range round the mighty host of your followers
+to see if you could find the son of Nestor still alive. Presently
+Menelaus saw him on the extreme left of the battle cheering on
+his men and exhorting them to fight boldly. Menelaus went up to
+him and said, "Antilochus, come here and listen to sad news,
+which I would indeed were untrue. You must see with your own eyes
+that heaven is heaping calamity upon the Danaans, and giving
+victory to the Trojans. Patroclus has fallen, who was the bravest
+of the Achaeans, and sorely will the Danaans miss him. Run
+instantly to the ships and tell Achilles, that he may come to
+rescue the body and bear it to the ships. As for the armour,
+Hector already has it."
+</para>
+
+<para>
+Antilochus was struck with horror. For a long time he was
+speechless; his eyes filled with tears and he could find no
+utterance, but he did as Menelaus had said, and set off running
+as soon as he had given his armour to a comrade, Laodocus, who
+was wheeling his horses round, close beside him.
+</para>
+
+<para>
+Thus, then, did he run weeping from the field, to carry the bad
+news to Achilles son of Peleus. Nor were you, O Menelaus, minded
+to succour his harassed comrades, when Antilochus had left the
+Pylians--and greatly did they miss him--but he sent them noble
+Thrasymedes, and himself went back to Patroclus. He came running
+up to the two Ajaxes and said, "I have sent Antilochus to the
+ships to tell Achilles, but rage against Hector as he may, he
+cannot come, for he cannot fight without armour. What then will
+be our best plan both as regards rescuing the dead, and our own
+escape from death amid the battle-cries of the Trojans?"
+</para>
+
+<para>
+Ajax answered, "Menelaus, you have said well: do you, then, and
+Meriones stoop down, raise the body, and bear it out of the fray,
+while we two behind you keep off Hector and the Trojans, one in
+heart as in name, and long used to fighting side by side with one
+another."
+</para>
+
+<para>
+On this Menelaus and Meriones took the dead man in their arms and
+lifted him high aloft with a great effort. The Trojan host raised
+a hue and cry behind them when they saw the Achaeans bearing the
+body away, and flew after them like hounds attacking a wounded
+boar at the loo of a band of young huntsmen. For a while the
+hounds fly at him as though they would tear him in pieces, but
+now and again he turns on them in a fury, scaring and scattering
+them in all directions--even so did the Trojans for a while
+charge in a body, striking with sword and with spears pointed ai
+both the ends, but when the two Ajaxes faced them and stood at
+bay, they would turn pale and no man dared press on to fight
+further about the dead.
+</para>
+
+<para>
+In this wise did the two heroes strain every nerve to bear the
+body to the ships out of the fight. The battle raged round them
+like fierce flames that when once kindled spread like wildfire
+over a city, and the houses fall in the glare of its burning--
+even such was the roar and tramp of men and horses that pursued
+them as they bore Patroclus from the field. Or as mules that put
+forth all their strength to draw some beam or great piece of
+ship's timber down a rough mountain-track, and they pant and
+sweat as they, go even so did Menelaus and pant and sweat as they
+bore the body of Patroclus. Behind them the two Ajaxes held
+stoutly out. As some wooded mountain-spur that stretches across a
+plain will turn water and check the flow even of a great river,
+nor is there any stream strong enough to break through it--even
+so did the two Ajaxes face the Trojans and stem the tide of their
+fighting though they kept pouring on towards them and foremost
+among them all was Aeneas son of Anchises with valiant Hector. As
+a flock of daws or starlings fall to screaming and chattering
+when they see a falcon, foe to all small birds, come soaring near
+them, even so did the Achaean youth raise a babel of cries as
+they fled before Aeneas and Hector, unmindful of their former
+prowess. In the rout of the Danaans much goodly armour fell round
+about the trench, and of fighting there was no end.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XVIII</title>
+
+<para>
+THUS then did they fight as it were a flaming fire. Meanwhile the
+fleet runner Antilochus, who had been sent as messenger, reached
+Achilles, and found him sitting by his tall ships and boding that
+which was indeed too surely true. "Alas," said he to himself in
+the heaviness of his heart, "why are the Achaeans again scouring
+the plain and flocking towards the ships? Heaven grant the gods
+be not now bringing that sorrow upon me of which my mother Thetis
+spoke, saying that while I was yet alive the bravest of the
+Myrmidons should fall before the Trojans, and see the light of
+the sun no longer. I fear the brave son of Menoetius has fallen
+through his own daring and yet I bade him return to the ships as
+soon as he had driven back those that were bringing fire against
+them, and not join battle with Hector."
+</para>
+
+<para>
+As he was thus pondering, the son of Nestor came up to him and
+told his sad tale, weeping bitterly the while. "Alas," he cried,
+"son of noble Peleus, I bring you bad tidings, would indeed that
+they were untrue. Patroclus has fallen, and a fight is raging
+about his naked body--for Hector holds his armour."
+</para>
+
+<para>
+A dark cloud of grief fell upon Achilles as he listened. He
+filled both hands with dust from off the ground, and poured it
+over his head, disfiguring his comely face, and letting the
+refuse settle over his shirt so fair and new. He flung himself
+down all huge and hugely at full length, and tore his hair with
+his hands. The bondswomen whom Achilles and Patroclus had taken
+captive screamed aloud for grief, beating their breasts, and with
+their limbs failing them for sorrow. Antilochus bent over him the
+while, weeping and holding both his hands as he lay groaning for
+he feared that he might plunge a knife into his own throat. Then
+Achilles gave a loud cry and his mother heard him as she was
+sitting in the depths of the sea by the old man her father,
+whereon she screamed, and all the goddesses daughters of Nereus
+that dwelt at the bottom of the sea, came gathering round her.
+There were Glauce, Thalia and Cymodoce, Nesaia, Speo, Thoe and
+dark-eyed Halie, Cymothoe, Actaea and Limnorea, Melite, Iaera,
+Amphithoe and Agave, Doto and Proto, Pherusa and Dynamene,
+Dexamene, Amphinome and Callianeira, Doris, Panope, and the
+famous sea-nymph Galatea, Nemertes, Apseudes and Callianassa.
+There were also Clymene, Ianeira and Ianassa, Maera, Oreithuia
+and Amatheia of the lovely locks, with other Nereids who dwell in
+the depths of the sea. The crystal cave was filled with their
+multitude and they all beat their breasts while Thetis led them
+in their lament.
+</para>
+
+<para>
+"Listen," she cried, "sisters, daughters of Nereus, that you may
+hear the burden of my sorrows. Alas, woe is me, woe in that I
+have borne the most glorious of offspring. I bore him fair and
+strong, hero among heroes, and he shot up as a sapling; I tended
+him as a plant in a goodly garden, and sent him with his ships to
+Ilius to fight the Trojans, but never shall I welcome him back to
+the house of Peleus. So long as he lives to look upon the light
+of the sun he is in heaviness, and though I go to him I cannot
+help him. Nevertheless I will go, that I may see my dear son and
+learn what sorrow has befallen him though he is still holding
+aloof from battle."
+</para>
+
+<para>
+She left the cave as she spoke, while the others followed weeping
+after, and the waves opened a path before them. When they reached
+the rich plain of Troy, they came up out of the sea in a long
+line on to the sands, at the place where the ships of the
+Myrmidons were drawn up in close order round the tents of
+Achilles. His mother went up to him as he lay groaning; she laid
+her hand upon his head and spoke piteously, saying, "My son, why
+are you thus weeping? What sorrow has now befallen you? Tell me;
+hide it not from me. Surely Jove has granted you the prayer you
+made him, when you lifted up your hands and besought him that the
+Achaeans might all of them be pent up at their ships, and rue it
+bitterly in that you were no longer with them."
+</para>
+
+<para>
+Achilles groaned and answered, "Mother, Olympian Jove has indeed
+vouchsafed me the fulfilment of my prayer, but what boots it to
+me, seeing that my dear comrade Patroclus has fallen--he whom I
+valued more than all others, and loved as dearly as my own life?
+I have lost him; aye, and Hector when he had killed him stripped
+the wondrous armour, so glorious to behold, which the gods gave
+to Peleus when they laid you in the couch of a mortal man. Would
+that you were still dwelling among the immortal sea-nymphs, and
+that Peleus had taken to himself some mortal bride. For now you
+shall have grief infinite by reason of the death of that son whom
+you can never welcome home--nay, I will not live nor go about
+among mankind unless Hector fall by my spear, and thus pay me for
+having slain Patroclus son of Menoetius."
+</para>
+
+<para>
+Thetis wept and answered, "Then, my son, is your end near at
+hand--for your own death awaits you full soon after that of
+Hector."
+</para>
+
+<para>
+Then said Achilles in his great grief, "I would die here and now,
+in that I could not save my comrade. He has fallen far from home,
+and in his hour of need my hand was not there to help him. What
+is there for me? Return to my own land I shall not, and I have
+brought no saving neither to Patroclus nor to my other comrades
+of whom so many have been slain by mighty Hector; I stay here by
+my ships a bootless burden upon the earth, I, who in fight have
+no peer among the Achaeans, though in council there are better
+than I. Therefore, perish strife both from among gods and men,
+and anger, wherein even a righteous man will harden his
+heart--which rises up in the soul of a man like smoke, and the
+taste thereof is sweeter than drops of honey. Even so has
+Agamemnon angered me. And yet--so be it, for it is over; I will
+force my soul into subjection as I needs must; I will go; I will
+pursue Hector who has slain him whom I loved so dearly, and will
+then abide my doom when it may please Jove and the other gods to
+send it. Even Hercules, the best beloved of Jove--even he could
+not escape the hand of death, but fate and Juno's fierce anger
+laid him low, as I too shall lie when I am dead if a like doom
+awaits me. Till then I will win fame, and will bid Trojan and
+Dardanian women wring tears from their tender cheeks with both
+their hands in the grievousness of their great sorrow; thus shall
+they know that he who has held aloof so long will hold aloof no
+longer. Hold me not back, therefore, in the love you bear me, for
+you shall not move me."
+</para>
+
+<para>
+Then silver-footed Thetis answered, "My son, what you have said
+is true. It is well to save your comrades from destruction, but
+your armour is in the hands of the Trojans; Hector bears it in
+triumph upon his own shoulders. Full well I know that his vaunt
+shall not be lasting, for his end is close at hand; go not,
+however, into the press of battle till you see me return hither;
+to-morrow at break of day I shall be here, and will bring you
+goodly armour from King Vulcan."
+</para>
+
+<para>
+On this she left her brave son, and as she turned away she said
+to the sea-nymphs her sisters, "Dive into the bosom of the sea
+and go to the house of the old sea-god my father. Tell him
+everything; as for me, I will go to the cunning workman Vulcan on
+high Olympus, and ask him to provide my son with a suit of
+splendid armour."
+</para>
+
+<para>
+When she had so said, they dived forthwith beneath the waves,
+while silver-footed Thetis went her way that she might bring the
+armour for her son.
+</para>
+
+<para>
+Thus, then, did her feet bear the goddess to Olympus, and
+meanwhile the Achaeans were flying with loud cries before
+murderous Hector till they reached the ships and the Hellespont,
+and they could not draw the body of Mars's servant Patroclus out
+of reach of the weapons that were showered upon him, for Hector
+son of Priam with his host and horsemen had again caught up to
+him like the flame of a fiery furnace; thrice did brave Hector
+seize him by the feet, striving with might and main to draw him
+away and calling loudly on the Trojans, and thrice did the two
+Ajaxes, clothed in valour as with a garment, beat him from off
+the body; but all undaunted he would now charge into the thick of
+the fight, and now again he would stand still and cry aloud, but
+he would give no ground. As upland shepherds that cannot chase
+some famished lion from a carcase, even so could not the two
+Ajaxes scare Hector son of Priam from the body of Patroclus.
+</para>
+
+<para>
+And now he would even have dragged it off and have won
+imperishable glory, had not Iris fleet as the wind, winged her
+way as messenger from Olympus to the son of Peleus and bidden him
+arm. She came secretly without the knowledge of Jove and of the
+other gods, for Juno sent her, and when she had got close to him
+she said, "Up, son of Peleus, mightiest of all mankind; rescue
+Patroclus about whom this fearful fight is now raging by the
+ships. Men are killing one another, the Danaans in defence of the
+dead body, while the Trojans are trying to hale it away, and take
+it to windy Ilius: Hector is the most furious of them all; he is
+for cutting the head from the body and fixing it on the stakes of
+the wall. Up, then, and bide here no longer; shrink from the
+thought that Patroclus may become meat for the dogs of Troy.
+Shame on you, should his body suffer any kind of outrage."
+</para>
+
+<para>
+And Achilles said, "Iris, which of the gods was it that sent you
+to me?"
+</para>
+
+<para>
+Iris answered, "It was Juno the royal spouse of Jove, but the son
+of Saturn does not know of my coming, nor yet does any other of
+the immortals who dwell on the snowy summits of Olympus."
+</para>
+
+<para>
+Then fleet Achilles answered her saying, "How can I go up into
+the battle? They have my armour. My mother forbade me to arm till
+I should see her come, for she promised to bring me goodly armour
+from Vulcan; I know no man whose arms I can put on, save only the
+shield of Ajax son of Telamon, and he surely must be fighting in
+the front rank and wielding his spear about the body of dead
+Patroclus."
+</para>
+
+<para>
+Iris said, "We know that your armour has been taken, but go as
+you are; go to the deep trench and show yourself before the
+Trojans, that they may fear you and cease fighting. Thus will the
+fainting sons of the Achaeans gain some brief breathing-time,
+which in battle may hardly be."
+</para>
+
+<para>
+Iris left him when she had so spoken. But Achilles dear to Jove
+arose, and Minerva flung her tasselled aegis round his strong
+shoulders; she crowned his head with a halo of golden cloud from
+which she kindled a glow of gleaming fire. As the smoke that goes
+up into heaven from some city that is being beleaguered on an
+island far out at sea--all day long do men sally from the city
+and fight their hardest, and at the going down of the sun the
+line of beacon-fires blazes forth, flaring high for those that
+dwell near them to behold, if so be that they may come with their
+ships and succour them--even so did the light flare from the head
+of Achilles, as he stood by the trench, going beyond the wall--
+but he aid not join the Achaeans for he heeded the charge which
+his mother laid upon him.
+</para>
+
+<para>
+There did he stand and shout aloud. Minerva also raised her voice
+from afar, and spread terror unspeakable among the Trojans.
+Ringing as the note of a trumpet that sounds alarm then the foe
+is at the gates of a city, even so brazen was the voice of the
+son of Aeacus, and when the Trojans heard its clarion tones they
+were dismayed; the horses turned back with their chariots for
+they boded mischief, and their drivers were awe-struck by the
+steady flame which the grey-eyed goddess had kindled above the
+head of the great son of Peleus.
+</para>
+
+<para>
+Thrice did Achilles raise his loud cry as he stood by the trench,
+and thrice were the Trojans and their brave allies thrown into
+confusion; whereon twelve of their noblest champions fell beneath
+the wheels of their chariots and perished by their own spears.
+The Achaeans to their great joy then drew Patroclus out of reach
+of the weapons, and laid him on a litter: his comrades stood
+mourning round him, and among them fleet Achilles who wept
+bitterly as he saw his true comrade lying dead upon his bier. He
+had sent him out with horses and chariots into battle, but his
+return he was not to welcome.
+</para>
+
+<para>
+Then Juno sent the busy sun, loth though he was, into the waters
+of Oceanus; so he set, and the Achaeans had rest from the tug and
+turmoil of war.
+</para>
+
+<para>
+Now the Trojans when they had come out of the fight, unyoked
+their horses and gathered in assembly before preparing their
+supper. They kept their feet, nor would any dare to sit down, for
+fear had fallen upon them all because Achilles had shown himself
+after having held aloof so long from battle. Polydamas son of
+Panthous was first to speak, a man of judgement, who alone among
+them could look both before and after. He was comrade to Hector,
+and they had been born upon the same night; with all sincerity
+and goodwill, therefore, he addressed them thus:--
+</para>
+
+<para>
+"Look to it well, my friends; I would urge you to go back now to
+your city and not wait here by the ships till morning, for we are
+far from our walls. So long as this man was at enmity with
+Agamemnon the Achaeans were easier to deal with, and I would have
+gladly camped by the ships in the hope of taking them; but now I
+go in great fear of the fleet son of Peleus; he is so daring that
+he will never bide here on the plain whereon the Trojans and
+Achaeans fight with equal valour, but he will try to storm our
+city and carry off our women. Do then as I say, and let us
+retreat. For this is what will happen. The darkness of night will
+for a time stay the son of Peleus, but if he find us here in the
+morning when he sallies forth in full armour, we shall have
+knowledge of him in good earnest. Glad indeed will he be who can
+escape and get back to Ilius, and many a Trojan will become meat
+for dogs and vultures may I never live to hear it. If we do as I
+say, little though we may like it, we shall have strength in
+counsel during the night, and the great gates with the doors that
+close them will protect the city. At dawn we can arm and take our
+stand on the walls; he will then rue it if he sallies from the
+ships to fight us. He will go back when he has given his horses
+their fill of being driven all whithers under our walls, and will
+be in no mind to try and force his way into the city. Neither
+will he ever sack it, dogs shall devour him ere he do so."
+</para>
+
+<para>
+Hector looked fiercely at him and answered, "Polydamas, your
+words are not to my liking in that you bid us go back and be pent
+within the city. Have you not had enough of being cooped up
+behind walls? In the old-days the city of Priam was famous the
+whole world over for its wealth of gold and bronze, but our
+treasures are wasted out of our houses, and much goods have been
+sold away to Phrygia and fair Meonia, for the hand of Jove has
+been laid heavily upon us. Now, therefore, that the son of
+scheming Saturn has vouchsafed me to win glory here and to hem
+the Achaeans in at their ships, prate no more in this fool's wise
+among the people. You will have no man with you; it shall not be;
+do all of you as I now say;--take your suppers in your companies
+throughout the host, and keep your watches and be wakeful every
+man of you. If any Trojan is uneasy about his possessions, let
+him gather them and give them out among the people. Better let
+these, rather than the Achaeans, have them. At daybreak we will
+arm and fight about the ships; granted that Achilles has again
+come forward to defend them, let it be as he will, but it shall
+go hard with him. I shall not shun him, but will fight him, to
+fall or conquer. The god of war deals out like measure to all,
+and the slayer may yet be slain."
+</para>
+
+<para>
+Thus spoke Hector; and the Trojans, fools that they were, shouted
+in applause, for Pallas Minerva had robbed them of their
+understanding. They gave ear to Hector with his evil counsel, but
+the wise words of Polydamas no man would heed. They took their
+supper throughout the host, and meanwhile through the whole night
+the Achaeans mourned Patroclus, and the son of Peleus led them in
+their lament. He laid his murderous hands upon the breast of his
+comrade, groaning again and again as a bearded lion when a man
+who was chasing deer has robbed him of his young in some dense
+forest; when the lion comes back he is furious, and searches
+dingle and dell to track the hunter if he can find him, for he is
+mad with rage--even so with many a sigh did Achilles speak among
+the Myrmidons saying, "Alas! vain were the words with which I
+cheered the hero Menoetius in his own house; I said that I would
+bring his brave son back again to Opoeis after he had sacked
+Ilius and taken his share of the spoils--but Jove does not give
+all men their heart's desire. The same soil shall be reddened
+here at Troy by the blood of us both, for I too shall never be
+welcomed home by the old knight Peleus, nor by my mother Thetis,
+but even in this place shall the earth cover me. Nevertheless, O
+Patroclus, now that I am left behind you, I will not bury you,
+till I have brought hither the head and armour of mighty Hector
+who has slain you. Twelve noble sons of Trojans will I behead
+before your bier to avenge you; till I have done so you shall lie
+as you are by the ships, and fair women of Troy and Dardanus,
+whom we have taken with spear and strength of arm when we sacked
+men's goodly cities, shall weep over you both night and day."
+</para>
+
+<para>
+Then Achilles told his men to set a large tripod upon the fire
+that they might wash the clotted gore from off Patroclus. Thereon
+they set a tripod full of bath water on to a clear fire: they
+threw sticks on to it to make it blaze, and the water became hot
+as the flame played about the belly of the tripod. When the water
+in the cauldron was boiling they washed the body, anointed it
+with oil, and closed its wounds with ointment that had been kept
+nine years. Then they laid it on a bier and covered it with a
+linen cloth from head to foot, and over this they laid a fair
+white robe. Thus all night long did the Myrmidons gather round
+Achilles to mourn Patroclus.
+</para>
+
+<para>
+Then Jove said to Juno his sister-wife, "So, Queen Juno, you have
+gained your end, and have roused fleet Achilles. One would think
+that the Achaeans were of your own flesh and blood."
+</para>
+
+<para>
+And Juno answered, "Dread son of Saturn, why should you say this
+thing? May not a man though he be only mortal and knows less than
+we do, do what he can for another person? And shall not I--
+foremost of all goddesses both by descent and as wife to you who
+reign in heaven--devise evil for the Trojans if I am angry with
+them?"
+</para>
+
+<para>
+Thus did they converse. Meanwhile Thetis came to the house of
+Vulcan, imperishable, star-bespangled, fairest of the abodes in
+heaven, a house of bronze wrought by the lame god's own hands.
+She found him busy with his bellows, sweating and hard at work,
+for he was making twenty tripods that were to stand by the wall
+of his house, and he set wheels of gold under them all that they
+might go of their own selves to the assemblies of the gods, and
+come back again--marvels indeed to see. They were finished all
+but the ears of cunning workmanship which yet remained to be
+fixed to them: these he was now fixing, and he was hammering at
+the rivets. While he was thus at work silver-footed Thetis came
+to the house. Charis, of graceful head-dress, wife to the
+far-famed lame god, came towards her as soon as she saw her, and
+took her hand in her own, saying, "Why have you come to our
+house, Thetis, honoured and ever welcome--for you do not visit us
+often? Come inside and let me set refreshment before you."
+</para>
+
+<para>
+The goddess led the way as she spoke, and bade Thetis sit on a
+richly decorated seat inlaid with silver; there was a footstool
+also under her feet. Then she called Vulcan and said, "Vulcan,
+come here, Thetis wants you"; and the far-famed lame god
+answered, "Then it is indeed an august and honoured goddess who
+has come here; she it was that took care of me when I was
+suffering from the heavy fall which I had through my cruel
+mother's anger--for she would have got rid of me because I was
+lame. It would have gone hardly with me had not Eurynome,
+daughter of the ever-encircling waters of Oceanus, and Thetis,
+taken me to their bosom. Nine years did I stay with them, and
+many beautiful works in bronze, brooches, spiral armlets, cups,
+and chains, did I make for them in their cave, with the roaring
+waters of Oceanus foaming as they rushed ever past it; and no one
+knew, neither of gods nor men, save only Thetis and Eurynome who
+took care of me. If, then, Thetis has come to my house I must
+make her due requital for having saved me; entertain her,
+therefore, with all hospitality, while I put by my bellows and
+all my tools."
+</para>
+
+<para>
+On this the mighty monster hobbled off from his anvil, his thin
+legs plying lustily under him. He set the bellows away from the
+fire, and gathered his tools into a silver chest. Then he took a
+sponge and washed his face and hands, his shaggy chest and brawny
+neck; he donned his shirt, grasped his strong staff, and limped
+towards the door. There were golden handmaids also who worked for
+him, and were like real young women, with sense and reason, voice
+also and strength, and all the learning of the immortals; these
+busied themselves as the king bade them, while he drew near to
+Thetis, seated her upon a goodly seat, and took her hand in his
+own, saying, "Why have you come to our house, Thetis honoured and
+ever welcome--for you do not visit us often? Say what you want,
+and I will do it for you at once if I can, and if it can be done
+at all."
+</para>
+
+<para>
+Thetis wept and answered, "Vulcan, is there another goddess in
+Olympus whom the son of Saturn has been pleased to try with so
+much affliction as he has me? Me alone of the marine goddesses
+did he make subject to a mortal husband, Peleus son of Aeacus,
+and sorely against my will did I submit to the embraces of one
+who was but mortal, and who now stays at home worn out with age.
+Neither is this all. Heaven vouchsafed me a son, hero among
+heroes, and he shot up as a sapling. I tended him as a plant in a
+goodly garden and sent him with his ships to Ilius to fight the
+Trojans, but never shall I welcome him back to the house of
+Peleus. So long as he lives to look upon the light of the sun, he
+is in heaviness, and though I go to him I cannot help him; King
+Agamemnon has made him give up the maiden whom the sons of the
+Achaeans had awarded him, and he wastes with sorrow for her sake.
+Then the Trojans hemmed the Achaeans in at their ships' sterns
+and would not let them come forth; the elders, therefore, of the
+Argives besought Achilles and offered him great treasure, whereon
+he refused to bring deliverance to them himself, but put his own
+armour on Patroclus and sent him into the fight with much people
+after him. All day long they fought by the Scaean gates and would
+have taken the city there and then, had not Apollo vouchsafed
+glory to Hector and slain the valiant son of Menoetius after he
+had done the Trojans much evil. Therefore I am suppliant at your
+knees if haply you may be pleased to provide my son, whose end is
+near at hand, with helmet and shield, with goodly greaves fitted
+with ancle-clasps, and with a breastplate, for he lost his own
+when his true comrade fell at the hands of the Trojans, and he
+now lies stretched on earth in the bitterness of his soul."
+</para>
+
+<para>
+And Vulcan answered, "Take heart, and be no more disquieted about
+this matter; would that I could hide him from death's sight when
+his hour is come, so surely as I can find him armour that shall
+amaze the eyes of all who behold it."
+</para>
+
+<para>
+When he had so said he left her and went to his bellows, turning
+them towards the fire and bidding them do their office. Twenty
+bellows blew upon the melting-pots, and they blew blasts of every
+kind, some fierce to help him when he had need of them, and
+others less strong as Vulcan willed it in the course of his work.
+He threw tough copper into the fire, and tin, with silver and
+gold; he set his great anvil on its block, and with one hand
+grasped his mighty hammer while he took the tongs in the other.
+</para>
+
+<para>
+First he shaped the shield so great and strong, adorning it all
+over and binding it round with a gleaming circuit in three
+layers; and the baldric was made of silver. He made the shield in
+five thicknesses, and with many a wonder did his cunning hand
+enrich it.
+</para>
+
+<para>
+He wrought the earth, the heavens, and the sea; the moon also at
+her full and the untiring sun, with all the signs that glorify
+the face of heaven--the Pleiads, the Hyads, huge Orion, and the
+Bear, which men also call the Wain and which turns round ever in
+one place, facing. Orion, and alone never dips into the stream of
+Oceanus.
+</para>
+
+<para>
+He wrought also two cities, fair to see and busy with the hum of
+men. In the one were weddings and wedding-feasts, and they were
+going about the city with brides whom they were escorting by
+torchlight from their chambers. Loud rose the cry of Hymen, and
+the youths danced to the music of flute and lyre, while the women
+stood each at her house door to see them.
+</para>
+
+<para>
+Meanwhile the people were gathered in assembly, for there was a
+quarrel, and two men were wrangling about the blood-money for a
+man who had been killed, the one saying before the people that he
+had paid damages in full, and the other that he had not been
+paid. Each was trying to make his own case good, and the people
+took sides, each man backing the side that he had taken; but the
+heralds kept them back, and the elders sate on their seats of
+stone in a solemn circle, holding the staves which the heralds
+had put into their hands. Then they rose and each in his turn
+gave judgement, and there were two talents laid down, to be given
+to him whose judgement should be deemed the fairest.
+</para>
+
+<para>
+About the other city there lay encamped two hosts in gleaming
+armour, and they were divided whether to sack it, or to spare it
+and accept the half of what it contained. But the men of the city
+would not yet consent, and armed themselves for a surprise; their
+wives and little children kept guard upon the walls, and with
+them were the men who were past fighting through age; but the
+others sallied forth with Mars and Pallas Minerva at their head--
+both of them wrought in gold and clad in golden raiment, great
+and fair with their armour as befitting gods, while they that
+followed were smaller. When they reached the place where they
+would lay their ambush, it was on a riverbed to which live stock
+of all kinds would come from far and near to water; here, then,
+they lay concealed, clad in full armour. Some way off them there
+were two scouts who were on the look-out for the coming of sheep
+or cattle, which presently came, followed by two shepherds who
+were playing on their pipes, and had not so much as a thought of
+danger. When those who were in ambush saw this, they cut off the
+flocks and herds and killed the shepherds. Meanwhile the
+besiegers, when they heard much noise among the cattle as they
+sat in council, sprang to their horses, and made with all speed
+towards them; when they reached them they set battle in array by
+the banks of the river, and the hosts aimed their bronze-shod
+spears at one another. With them were Strife and Riot, and fell
+Fate who was dragging three men after her, one with a fresh
+wound, and the other unwounded, while the third was dead, and she
+was dragging him along by his heel: and her robe was bedrabbled
+in men's blood. They went in and out with one another and fought
+as though they were living people haling away one another's dead.
+</para>
+
+<para>
+He wrought also a fair fallow field, large and thrice ploughed
+already. Many men were working at the plough within it, turning
+their oxen to and fro, furrow after furrow. Each time that they
+turned on reaching the headland a man would come up to them and
+give them a cup of wine, and they would go back to their furrows
+looking forward to the time when they should again reach the
+headland. The part that they had ploughed was dark behind them,
+so that the field, though it was of gold, still looked as if it
+were being ploughed--very curious to behold.
+</para>
+
+<para>
+He wrought also a field of harvest corn, and the reapers were
+reaping with sharp sickles in their hands. Swathe after swathe
+fell to the ground in a straight line behind them, and the
+binders bound them in bands of twisted straw. There were three
+binders, and behind them there were boys who gathered the cut
+corn in armfuls and kept on bringing them to be bound: among them
+all the owner of the land stood by in silence and was glad. The
+servants were getting a meal ready under an oak, for they had
+sacrificed a great ox, and were busy cutting him up, while the
+women were making a porridge of much white barley for the
+labourers' dinner.
+</para>
+
+<para>
+He wrought also a vineyard, golden and fair to see, and the vines
+were loaded with grapes. The bunches overhead were black, but the
+vines were trained on poles of silver. He ran a ditch of dark
+metal all round it, and fenced it with a fence of tin; there was
+only one path to it, and by this the vintagers went when they
+would gather the vintage. Youths and maidens all blithe and full
+of glee, carried the luscious fruit in plaited baskets; and with
+them there went a boy who made sweet music with his lyre, and
+sang the Linos-song with his clear boyish voice.
+</para>
+
+<para>
+He wrought also a herd of horned cattle. He made the cows of gold
+and tin, and they lowed as they came full speed out of the yards
+to go and feed among the waving reeds that grow by the banks of
+the river. Along with the cattle there went four shepherds, all
+of them in gold, and their nine fleet dogs went with them. Two
+terrible lions had fastened on a bellowing bull that was with the
+foremost cows, and bellow as he might they haled him, while the
+dogs and men gave chase: the lions tore through the bull's thick
+hide and were gorging on his blood and bowels, but the herdsmen
+were afraid to do anything, and only hounded on their dogs; the
+dogs dared not fasten on the lions but stood by barking and
+keeping out of harm's way.
+</para>
+
+<para>
+The god wrought also a pasture in a fair mountain dell, and a
+large flock of sheep, with a homestead and huts, and sheltered
+sheepfolds.
+</para>
+
+<para>
+Furthermore he wrought a green, like that which Daedalus once
+made in Cnossus for lovely Ariadne. Hereon there danced youths
+and maidens whom all would woo, with their hands on one another's
+wrists. The maidens wore robes of light linen, and the youths
+well woven shirts that were slightly oiled. The girls were
+crowned with garlands, while the young men had daggers of gold
+that hung by silver baldrics; sometimes they would dance deftly
+in a ring with merry twinkling feet, as it were a potter sitting
+at his work and making trial of his wheel to see whether it will
+run, and sometimes they would go all in line with one another,
+and much people was gathered joyously about the green. There was
+a bard also to sing to them and play his lyre, while two tumblers
+went about performing in the midst of them when the man struck up
+with his tune.
+</para>
+
+<para>
+All round the outermost rim of the shield he set the mighty
+stream of the river Oceanus.
+</para>
+
+<para>
+Then when he had fashioned the shield so great and strong, he
+made a breastplate also that shone brighter than fire. He made a
+helmet, close fitting to the brow, and richly worked, with a
+golden plume overhanging it; and he made greaves also of beaten
+tin.
+</para>
+
+<para>
+Lastly, when the famed lame god had made all the armour, he took
+it and set it before the mother of Achilles; whereon she darted
+like a falcon from the snowy summits of Olympus and bore away the
+gleaming armour from the house of Vulcan.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XIX</title>
+
+<para>
+NOW when Dawn in robe of saffron was hasting from the streams of
+Oceanus, to bring light to mortals and immortals, Thetis reached
+the ships with the armour that the god had given her. She found
+her son fallen about the body of Patroclus and weeping bitterly.
+Many also of his followers were weeping round him, but when the
+goddess came among them she clasped his hand in her own, saying,
+"My son, grieve as we may we must let this man lie, for it is by
+heaven's will that he has fallen; now, therefore, accept from
+Vulcan this rich and goodly armour, which no man has ever yet
+borne upon his shoulders."
+</para>
+
+<para>
+As she spoke she set the armour before Achilles, and it rang out
+bravely as she did so. The Myrmidons were struck with awe, and
+none dared look full at it, for they were afraid; but Achilles
+was roused to still greater fury, and his eyes gleamed with a
+fierce light, for he was glad when he handled the splendid
+present which the god had made him. Then, as soon as he had
+satisfied himself with looking at it, he said to his mother,
+"Mother, the god has given me armour, meet handiwork for an
+immortal and such as no-one living could have fashioned; I will
+now arm, but I much fear that flies will settle upon the son of
+Menoetius and breed worms about his wounds, so that his body, now
+he is dead, will be disfigured and the flesh will rot."
+</para>
+
+<para>
+Silver-footed Thetis answered, "My son, be not disquieted about
+this matter. I will find means to protect him from the swarms of
+noisome flies that prey on the bodies of men who have been killed
+in battle. He may lie for a whole year, and his flesh shall still
+be as sound as ever, or even sounder. Call, therefore, the
+Achaean heroes in assembly; unsay your anger against Agamemnon;
+arm at once, and fight with might and main."
+</para>
+
+<para>
+As she spoke she put strength and courage into his heart, and she
+then dropped ambrosia and red nectar into the wounds of
+Patroclus, that his body might suffer no change.
+</para>
+
+<para>
+Then Achilles went out upon the seashore, and with a loud cry
+called on the Achaean heroes. On this even those who as yet had
+stayed always at the ships, the pilots and helmsmen, and even the
+stewards who were about the ships and served out rations, all
+came to the place of assembly because Achilles had shown himself
+after having held aloof so long from fighting. Two sons of Mars,
+Ulysses and the son of Tydeus, came limping, for their wounds
+still pained them; nevertheless they came, and took their seats
+in the front row of the assembly. Last of all came Agamemnon,
+king of men, he too wounded, for Coon son of Antenor had struck
+him with a spear in battle.
+</para>
+
+<para>
+When the Achaeans were got together Achilles rose and said, "Son
+of Atreus, surely it would have been better alike for both you
+and me, when we two were in such high anger about Briseis, surely
+it would have been better, had Diana's arrow slain her at the
+ships on the day when I took her after having sacked Lyrnessus.
+For so, many an Achaean the less would have bitten dust before
+the foe in the days of my anger. It has been well for Hector and
+the Trojans, but the Achaeans will long indeed remember our
+quarrel. Now, however, let it be, for it is over. If we have been
+angry, necessity has schooled our anger. I put it from me: I dare
+not nurse it for ever; therefore, bid the Achaeans arm forthwith
+that I may go out against the Trojans, and learn whether they
+will be in a mind to sleep by the ships or no. Glad, I ween, will
+he be to rest his knees who may fly my spear when I wield it."
+</para>
+
+<para>
+Thus did he speak, and the Achaeans rejoiced in that he had put
+away his anger.
+</para>
+
+<para>
+Then Agamemnon spoke, rising in his place, and not going into the
+middle of the assembly. "Danaan heroes," said he, "servants of
+Mars, it is well to listen when a man stands up to speak, and it
+is not seemly to interrupt him, or it will go hard even with a
+practised speaker. Who can either hear or speak in an uproar?
+Even the finest orator will be disconcerted by it. I will expound
+to the son of Peleus, and do you other Achaeans heed me and mark
+me well. Often have the Achaeans spoken to me of this matter and
+upbraided me, but it was not I that did it: Jove, and Fate, and
+Erinys that walks in darkness struck me mad when we were
+assembled on the day that I took from Achilles the meed that had
+been awarded to him. What could I do? All things are in the hand
+of heaven, and Folly, eldest of Jove's daughters, shuts men's
+eyes to their destruction. She walks delicately, not on the solid
+earth, but hovers over the heads of men to make them stumble or
+to ensnare them.
+</para>
+
+<para>
+"Time was when she fooled Jove himself, who they say is greatest
+whether of gods or men; for Juno, woman though she was, beguiled
+him on the day when Alcmena was to bring forth mighty Hercules in
+the fair city of Thebes. He told it out among the gods saying,
+'Hear me, all gods and goddesses, that I may speak even as I am
+minded; this day shall an Ilithuia, helper of women who are in
+labour, bring a man child into the world who shall be lord over
+all that dwell about him who are of my blood and lineage.' Then
+said Juno all crafty and full of guile, 'You will play false, and
+will not hold to your word. Swear me, O Olympian, swear me a
+great oath, that he who shall this day fall between the feet of a
+woman, shall be lord over all that dwell about him who are of
+your blood and lineage.'
+</para>
+
+<para>
+"Thus she spoke, and Jove suspected her not, but swore the great
+oath, to his much ruing thereafter. For Juno darted down from the
+high summit of Olympus, and went in haste to Achaean Argos where
+she knew that the noble wife of Sthenelus son of Perseus then
+was. She being with child and in her seventh month, Juno brought
+the child to birth though there was a month still wanting, but
+she stayed the offspring of Alcmena, and kept back the Ilithuiae.
+Then she went to tell Jove the son of Saturn, and said, 'Father
+Jove, lord of the lightning--I have a word for your ear. There is
+a fine child born this day, Eurystheus, son to Sthenelus the son
+of Perseus; he is of your lineage; it is well, therefore, that he
+should reign over the Argives.'
+</para>
+
+<para>
+"On this Jove was stung to the very quick, and in his rage he
+caught Folly by the hair, and swore a great oath that never
+should she again invade starry heaven and Olympus, for she was
+the bane of all. Then he whirled her round with a twist of his
+hand, and flung her down from heaven so that she fell on to the
+fields of mortal men; and he was ever angry with her when he saw
+his son groaning under the cruel labours that Eurystheus laid
+upon him. Even so did I grieve when mighty Hector was killing the
+Argives at their ships, and all the time I kept thinking of Folly
+who had so baned me. I was blind, and Jove robbed me of my
+reason; I will now make atonement, and will add much treasure by
+way of amends. Go, therefore, into battle, you and your people
+with you. I will give you all that Ulysses offered you yesterday
+in your tents: or if it so please you, wait, though you would
+fain fight at once, and my squires shall bring the gifts from my
+ship, that you may see whether what I give you is enough."
+</para>
+
+<para>
+And Achilles answered, "Son of Atreus, king of men Agamemnon, you
+can give such gifts as you think proper, or you can withhold
+them: it is in your own hands. Let us now set battle in array; it
+is not well to tarry talking about trifles, for there is a deed
+which is as yet to do. Achilles shall again be seen fighting
+among the foremost, and laying low the ranks of the Trojans: bear
+this in mind each one of you when he is fighting."
+</para>
+
+<para>
+Then Ulysses said, "Achilles, godlike and brave, send not the
+Achaeans thus against Ilius to fight the Trojans fasting, for the
+battle will be no brief one, when it is once begun, and heaven
+has filled both sides with fury; bid them first take food both
+bread and wine by the ships, for in this there is strength and
+stay. No man can do battle the livelong day to the going down of
+the sun if he is without food; however much he may want to fight
+his strength will fail him before he knows it; hunger and thirst
+will find him out, and his limbs will grow weary under him. But a
+man can fight all day if he is full fed with meat and wine; his
+heart beats high, and his strength will stay till he has routed
+all his foes; therefore, send the people away and bid them
+prepare their meal; King Agamemnon will bring out the gifts in
+presence of the assembly, that all may see them and you may be
+satisfied. Moreover let him swear an oath before the Argives that
+he has never gone up into the couch of Briseis, nor been with her
+after the manner of men and women; and do you, too, show yourself
+of a gracious mind; let Agamemnon entertain you in his tents with
+a feast of reconciliation, that so you may have had your dues in
+full. As for you, son of Atreus, treat people more righteously in
+future; it is no disgrace even to a king that he should make
+amends if he was wrong in the first instance."
+</para>
+
+<para>
+And King Agamemnon answered, "Son of Laertes, your words please
+me well, for throughout you have spoken wisely. I will swear as
+you would have me do; I do so of my own free will, neither shall
+I take the name of heaven in vain. Let, then, Achilles wait,
+though he would fain fight at once, and do you others wait also,
+till the gifts come from my tent and we ratify the oath with
+sacrifice. Thus, then, do I charge you: take some noble young
+Achaeans with you, and bring from my tents the gifts that I
+promised yesterday to Achilles, and bring the women also;
+furthermore let Talthybius find me a boar from those that are
+with the host, and make it ready for sacrifice to Jove and to the
+sun."
+</para>
+
+<para>
+Then said Achilles, "Son of Atreus, king of men Agamemnon, see to
+these matters at some other season, when there is breathing time
+and when I am calmer. Would you have men eat while the bodies of
+those whom Hector son of Priam slew are still lying mangled upon
+the plain? Let the sons of the Achaeans, say I, fight fasting and
+without food, till we have avenged them; afterwards at the going
+down of the sun let them eat their fill. As for me, Patroclus is
+lying dead in my tent, all hacked and hewn, with his feet to the
+door, and his comrades are mourning round him. Therefore I can
+take thought of nothing save only slaughter and blood and the
+rattle in the throat of the dying."
+</para>
+
+<para>
+Ulysses answered, "Achilles, son of Peleus, mightiest of all the
+Achaeans, in battle you are better than I, and that more than a
+little, but in counsel I am much before you, for I am older and
+of greater knowledge. Therefore be patient under my words.
+Fighting is a thing of which men soon surfeit, and when Jove, who
+is war's steward, weighs the upshot, it may well prove that the
+straw which our sickles have reaped is far heavier than the
+grain. It may not be that the Achaeans should mourn the dead with
+their bellies; day by day men fall thick and threefold
+continually; when should we have respite from our sorrow? Let us
+mourn our dead for a day and bury them out of sight and mind, but
+let those of us who are left eat and drink that we may arm and
+fight our foes more fiercely. In that hour let no man hold back,
+waiting for a second summons; such summons shall bode ill for him
+who is found lagging behind at our ships; let us rather sally as
+one man and loose the fury of war upon the Trojans."
+</para>
+
+<para>
+When he had thus spoken he took with him the sons of Nestor, with
+Meges son of Phyleus, Thoas, Meriones, Lycomedes son of Creontes,
+and Melanippus, and went to the tent of Agamemnon son of Atreus.
+The word was not sooner said than the deed was done: they
+brought out the seven tripods which Agamemnon had promised, with
+the twenty metal cauldrons and the twelve horses; they also
+brought the women skilled in useful arts, seven in number, with
+Briseis, which made eight. Ulysses weighed out the ten talents of
+gold and then led the way back, while the young Achaeans brought
+the rest of the gifts, and laid them in the middle of the
+assembly.
+</para>
+
+<para>
+Agamemnon then rose, and Talthybius whose voice was like that of
+a god came to him with the boar. The son of Atreus drew the knife
+which he wore by the scabbard of his mighty sword, and began by
+cutting off some bristles from the boar, lifting up his hands in
+prayer as he did so. The other Achaeans sat where they were all
+silent and orderly to hear the king, and Agamemnon looked into
+the vault of heaven and prayed saying, "I call Jove the first and
+mightiest of all gods to witness, I call also Earth and Sun and
+the Erinyes who dwell below and take vengeance on him who shall
+swear falsely, that I have laid no hand upon the girl Briseis,
+neither to take her to my bed nor otherwise, but that she has
+remained in my tents inviolate. If I swear falsely may heaven
+visit me with all the penalties which it metes out to those who
+perjure themselves."
+</para>
+
+<para>
+He cut the boar's throat as he spoke, whereon Talthybius whirled
+it round his head, and flung it into the wide sea to feed the
+fishes. Then Achilles also rose and said to the Argives, "Father
+Jove, of a truth you blind men's eyes and bane them. The son of
+Atreus had not else stirred me to so fierce an anger, nor so
+stubbornly taken Briseis from me against my will. Surely Jove
+must have counselled the destruction of many an Argive. Go, now,
+and take your food that we may begin fighting."
+</para>
+
+<para>
+On this he broke up the assembly, and every man went back to his
+own ship. The Myrmidons attended to the presents and took them
+away to the ship of Achilles. They placed them in his tents,
+while the stable-men drove the horses in among the others.
+</para>
+
+<para>
+Briseis, fair as Venus, when she saw the mangled body of
+Patroclus, flung herself upon it and cried aloud, tearing her
+breast, her neck, and her lovely face with both her hands.
+Beautiful as a goddess she wept and said, "Patroclus, dearest
+friend, when I went hence I left you living; I return, O prince,
+to find you dead; thus do fresh sorrows multiply upon me one
+after the other. I saw him to whom my father and mother married
+me, cut down before our city, and my three own dear brothers
+perished with him on the self-same day; but you, Patroclus, even
+when Achilles slew my husband and sacked the city of noble Mynes,
+told me that I was not to weep, for you said you would make
+Achilles marry me, and take me back with him to Phthia, we should
+have a wedding feast among the Myrmidons. You were always kind to
+me and I shall never cease to grieve for you."
+</para>
+
+<para>
+She wept as she spoke, and the women joined in her lament-making
+as though their tears were for Patroclus, but in truth each was
+weeping for her own sorrows. The elders of the Achaeans gathered
+round Achilles and prayed him to take food, but he groaned and
+would not do so. "I pray you," said he, "if any comrade will hear
+me, bid me neither eat nor drink, for I am in great heaviness,
+and will stay fasting even to the going down of the sun."
+</para>
+
+<para>
+On this he sent the other princes away, save only the two sons of
+Atreus and Ulysses, Nestor, Idomeneus, and the knight Phoenix,
+who stayed behind and tried to comfort him in the bitterness of
+his sorrow: but he would not be comforted till he should have
+flung himself into the jaws of battle, and he fetched sigh on
+sigh, thinking ever of Patroclus. Then he said--
+</para>
+
+<para>
+"Hapless and dearest comrade, you it was who would get a good
+dinner ready for me at once and without delay when the Achaeans
+were hasting to fight the Trojans; now, therefore, though I have
+meat and drink in my tents, yet will I fast for sorrow. Grief
+greater than this I could not know, not even though I were to
+hear of the death of my father, who is now in Phthia weeping for
+the loss of me his son, who am here fighting the Trojans in a
+strange land for the accursed sake of Helen, nor yet though I
+should hear that my son is no more--he who is being brought up in
+Scyros--if indeed Neoptolemus is still living. Till now I made
+sure that I alone was to fall here at Troy away from Argos, while
+you were to return to Phthia, bring back my son with you in your
+own ship, and show him all my property, my bondsmen, and the
+greatness of my house--for Peleus must surely be either dead, or
+what little life remains to him is oppressed alike with the
+infirmities of age and ever present fear lest he should hear the
+sad tidings of my death."
+</para>
+
+<para>
+He wept as he spoke, and the elders sighed in concert as each
+thought on what he had left at home behind him. The son of Saturn
+looked down with pity upon them, and said presently to Minerva,
+"My child, you have quite deserted your hero; is he then gone so
+clean out of your recollection? There he sits by the ships all
+desolate for the loss of his dear comrade, and though the others
+are gone to their dinner he will neither eat nor drink. Go then
+and drop nectar and ambrosia into his breast, that he may know no
+hunger."
+</para>
+
+<para>
+With these words he urged Minerva, who was already of the same
+mind. She darted down from heaven into the air like some falcon
+sailing on his broad wings and screaming. Meanwhile the Achaeans
+were arming throughout the host, and when Minerva had dropped
+nectar and ambrosia into Achilles so that no cruel hunger should
+cause his limbs to fail him, she went back to the house of her
+mighty father. Thick as the chill snow-flakes shed from the hand
+of Jove and borne on the keen blasts of the north wind, even so
+thick did the gleaming helmets, the bossed shields, the strongly
+plated breastplates, and the ashen spears stream from the ships.
+The sheen pierced the sky, the whole land was radiant with their
+flashing armour, and the sound of the tramp of their treading
+rose from under their feet. In the midst of them all Achilles put
+on his armour; he gnashed his teeth, his eyes gleamed like fire,
+for his grief was greater than he could bear. Thus, then, full of
+fury against the Trojans, did he don the gift of the god, the
+armour that Vulcan had made him.
+</para>
+
+<para>
+First he put on the goodly greaves fitted with ancle-clasps, and
+next he did on the breastplate about his chest. He slung the
+silver-studded sword of bronze about his shoulders, and then took
+up the shield so great and strong that shone afar with a
+splendour as of the moon. As the light seen by sailors from out
+at sea, when men have lit a fire in their homestead high up among
+the mountains, but the sailors are carried out to sea by wind and
+storm far from the haven where they would be--even so did the
+gleam of Achilles' wondrous shield strike up into the heavens. He
+lifted the redoubtable helmet, and set it upon his head, from
+whence it shone like a star, and the golden plumes which Vulcan
+had set thick about the ridge of the helmet, waved all around it.
+Then Achilles made trial of himself in his armour to see whether
+it fitted him, so that his limbs could play freely under it, and
+it seemed to buoy him up as though it had been wings.
+</para>
+
+<para>
+He also drew his father's spear out of the spear-stand, a spear
+so great and heavy and strong that none of the Achaeans save only
+Achilles had strength to wield it; this was the spear of Pelian
+ash from the topmost ridges of Mt. Pelion, which Chiron had once
+given to Peleus, fraught with the death of heroes. Automedon and
+Alcimus busied themselves with the harnessing of his horses; they
+made the bands fast about them, and put the bit in their mouths,
+drawing the reins back towards the chariot. Automedon, whip in
+hand, sprang up behind the horses, and after him Achilles mounted
+in full armour, resplendent as the sun-god Hyperion. Then with a
+loud voice he chided with his father's horses saying, "Xanthus
+and Balius, famed offspring of Podarge--this time when we have
+done fighting be sure and bring your driver safely back to the
+host of the Achaeans, and do not leave him dead on the plain as
+you did Patroclus."
+</para>
+
+<para>
+Then fleet Xanthus answered under the yoke--for white-armed Juno
+had endowed him with human speech--and he bowed his head till his
+mane touched the ground as it hung down from under the yoke-band.
+"Dread Achilles," said he, "we will indeed save you now, but the
+day of your death is near, and the blame will not be ours, for it
+will be heaven and stern fate that will destroy you. Neither was
+it through any sloth or slackness on our part that the Trojans
+stripped Patroclus of his armour; it was the mighty god whom
+lovely Leto bore that slew him as he fought among the foremost,
+and vouchsafed a triumph to Hector. We two can fly as swiftly as
+Zephyrus who they say is fleetest of all winds; nevertheless it
+is your doom to fall by the hand of a man and of a god."
+</para>
+
+<para>
+When he had thus said the Erinyes stayed his speech, and Achilles
+answered him in great sadness, saying, "Why, O Xanthus, do you
+thus foretell my death? You need not do so, for I well know that
+I am to fall here, far from my dear father and mother; none the
+more, however, shall I stay my hand till I have given the Trojans
+their fill of fighting."
+</para>
+
+<para>
+So saying, with a loud cry he drove his horses to the front.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XX</title>
+
+<para>
+THUS, then, did the Achaeans arm by their ships round you, O son
+of Peleus, who were hungering for battle; while the Trojans over
+against them armed upon the rise of the plain.
+</para>
+
+<para>
+Meanwhile Jove from the top of many-delled Olympus, bade Themis
+gather the gods in council, whereon she went about and called
+them to the house of Jove. There was not a river absent except
+Oceanus, nor a single one of the nymphs that haunt fair groves,
+or springs of rivers and meadows of green grass. When they
+reached the house of cloud-compelling Jove, they took their seats
+in the arcades of polished marble which Vulcan with his
+consummate skill had made for father Jove.
+</para>
+
+<para>
+In such wise, therefore, did they gather in the house of Jove.
+Neptune also, lord of the earthquake, obeyed the call of the
+goddess, and came up out of the sea to join them. There, sitting
+in the midst of them, he asked what Jove's purpose might be.
+"Why," said he, "wielder of the lightning, have you called the
+gods in council? Are you considering some matter that concerns
+the Trojans and Achaeans--for the blaze of battle is on the point
+of being kindled between them?"
+</para>
+
+<para>
+And Jove answered, "You know my purpose, shaker of earth, and
+wherefore I have called you hither. I take thought for them even
+in their destruction. For my own part I shall stay here seated on
+Mt. Olympus and look on in peace, but do you others go about
+among Trojans and Achaeans, and help either side as you may be
+severally disposed. If Achilles fights the Trojans without
+hindrance they will make no stand against him; they have ever
+trembled at the sight of him, and now that he is roused to such
+fury about his comrade, he will override fate itself and storm
+their city."
+</para>
+
+<para>
+Thus spoke Jove and gave the word for war, whereon the gods took
+their several sides and went into battle. Juno, Pallas Minerva,
+earth-encircling Neptune, Mercury bringer of good luck and
+excellent in all cunning--all these joined the host that came
+from the ships; with them also came Vulcan in all his glory,
+limping, but yet with his thin legs plying lustily under him.
+Mars of gleaming helmet joined the Trojans, and with him Apollo
+of locks unshorn, and the archer goddess Diana, Leto, Xanthus,
+and laughter-loving Venus.
+</para>
+
+<para>
+So long as the gods held themselves aloof from mortal warriors
+the Achaeans were triumphant, for Achilles who had long refused
+to fight was now with them. There was not a Trojan but his limbs
+failed him for fear as he beheld the fleet son of Peleus all
+glorious in his armour, and looking like Mars himself. When,
+however, the Olympians came to take their part among men,
+forthwith uprose strong Strife, rouser of hosts, and Minerva
+raised her loud voice, now standing by the deep trench that ran
+outside the wall, and now shouting with all her might upon the
+shore of the sounding sea. Mars also bellowed out upon the other
+side, dark as some black thunder-cloud, and called on the Trojans
+at the top of his voice, now from the acropolis, and now speeding
+up the side of the river Simois till he came to the hill
+Callicolone.
+</para>
+
+<para>
+Thus did the gods spur on both hosts to fight, and rouse fierce
+contention also among themselves. The sire of gods and men
+thundered from heaven above, while from beneath Neptune shook the
+vast earth, and bade the high hills tremble. The spurs and crests
+of many-fountained Ida quaked, as also the city of the Trojans
+and the ships of the Achaeans. Hades, king of the realms below,
+was struck with fear; he sprang panic-stricken from his throne
+and cried aloud in terror lest Neptune, lord of the earthquake,
+should crack the ground over his head, and lay bare his mouldy
+mansions to the sight of mortals and immortals--mansions so
+ghastly grim that even the gods shudder to think of them. Such
+was the uproar as the gods came together in battle. Apollo with
+his arrows took his stand to face King Neptune, while Minerva
+took hers against the god of war; the archer-goddess Diana with
+her golden arrows, sister of far-darting Apollo, stood to face
+Juno; Mercury the lusty bringer of good luck faced Leto, while
+the mighty eddying river whom men can Scamander, but gods
+Xanthus, matched himself against Vulcan.
+</para>
+
+<para>
+The gods, then, were thus ranged against one another. But the
+heart of Achilles was set on meeting Hector son of Priam, for it
+was with his blood that he longed above all things else to glut
+the stubborn lord of battle. Meanwhile Apollo set Aeneas on to
+attack the son of Peleus, and put courage into his heart,
+speaking with the voice of Lycaon son of Priam. In his likeness
+therefore, he said to Aeneas, "Aeneas, counsellor of the Trojans,
+where are now the brave words with which you vaunted over your
+wine before the Trojan princes, saying that you would fight
+Achilles son of Peleus in single combat?"
+</para>
+
+<para>
+And Aeneas answered, "Why do you thus bid me fight the proud son
+of Peleus, when I am in no mind to do so? Were I to face him now,
+it would not be for the first time. His spear has already put me
+to Right from Ida, when he attacked our cattle and sacked
+Lyrnessus and Pedasus; Jove indeed saved me in that he vouchsafed
+me strength to fly, else had the fallen by the hands of Achilles
+and Minerva, who went before him to protect him and urged him to
+fall upon the Lelegae and Trojans. No man may fight Achilles, for
+one of the gods is always with him as his guardian angel, and
+even were it not so, his weapon flies ever straight, and fails
+not to pierce the flesh of him who is against him; if heaven
+would let me fight him on even terms he should not soon overcome
+me, though he boasts that he is made of bronze."
+</para>
+
+<para>
+Then said King Apollo, son to Jove, "Nay, hero, pray to the
+ever-living gods, for men say that you were born of Jove's
+daughter Venus, whereas Achilles is son to a goddess of inferior
+rank. Venus is child to Jove, while Thetis is but daughter to the
+old man of the sea. Bring, therefore, your spear to bear upon
+him, and let him not scare you with his taunts and menaces."
+</para>
+
+<para>
+As he spoke he put courage into the heart of the shepherd of his
+people, and he strode in full armour among the ranks of the
+foremost fighters. Nor did the son of Anchises escape the notice
+of white-armed Juno, as he went forth into the throng to meet
+Achilles. She called the gods about her, and said, "Look to it,
+you two, Neptune and Minerva, and consider how this shall be;
+Phoebus Apollo has been sending Aeneas clad in full armour to
+fight Achilles. Shall we turn him back at once, or shall one of
+us stand by Achilles and endow him with strength so that his
+heart fail not, and he may learn that the chiefs of the immortals
+are on his side, while the others who have all along been
+defending the Trojans are but vain helpers? Let us all come down
+from Olympus and join in the fight, that this day he may take no
+hurt at the hands of the Trojans. Hereafter let him suffer
+whatever fate may have spun out for him when he was begotten and
+his mother bore him. If Achilles be not thus assured by the voice
+of a god, he may come to fear presently when one of us meets him
+in battle, for the gods are terrible if they are seen face to
+face."
+</para>
+
+<para>
+Neptune lord of the earthquake answered her saying, "Juno,
+restrain your fury; it is not well; I am not in favour of forcing
+the other gods to fight us, for the advantage is too greatly on
+our own side; let us take our places on some hill out of the
+beaten track, and let mortals fight it out among themselves. If
+Mars or Phoebus Apollo begin fighting, or keep Achilles in check
+so that he cannot fight, we too, will at once raise the cry of
+battle, and in that case they will soon leave the field and go
+back vanquished to Olympus among the other gods."
+</para>
+
+<para>
+With these words the dark-haired god led the way to the high
+earth-barrow of Hercules, built round solid masonry, and made by
+the Trojans and Pallas Minerva for him fly to when the
+sea-monster was chasing him from the shore on to the plain. Here
+Neptune and those that were with him took their seats, wrapped in
+a thick cloud of darkness; but the other gods seated themselves
+on the brow of Callicolone round you, O Phoebus, and Mars the
+waster of cities.
+</para>
+
+<para>
+Thus did the gods sit apart and form their plans, but neither
+side was willing to begin battle with the other, and Jove from
+his seat on high was in command over them all. Meanwhile the
+whole plain was alive with men and horses, and blazing with the
+gleam of armour. The earth rang again under the tramp of their
+feet as they rushed towards each other, and two champions, by far
+the foremost of them all, met between the hosts to fight--to wit,
+Aeneas son of Anchises, and noble Achilles.
+</para>
+
+<para>
+Aeneas was first to stride forward in attack, his doughty helmet
+tossing defiance as he came on. He held his strong shield before
+his breast, and brandished his bronze spear. The son of Peleus
+from the other side sprang forth to meet him, like some fierce
+lion that the whole country-side has met to hunt and kill--at
+first he bodes no ill, but when some daring youth has struck him
+with a spear, he crouches openmouthed, his jaws foam, he roars
+with fury, he lashes his tail from side to side about his ribs
+and loins, and glares as he springs straight before him, to find
+out whether he is to slay, or be slain among the foremost of his
+foes--even with such fury did Achilles burn to spring upon
+Aeneas.
+</para>
+
+<para>
+When they were now close up with one another Achilles was first
+to speak. "Aeneas," said he, "why do you stand thus out before
+the host to fight me? Is it that you hope to reign over the
+Trojans in the seat of Priam? Nay, though you kill me Priam will
+not hand his kingdom over to you. He is a man of sound judgement,
+and he has sons of his own. Or have the Trojans been allotting
+you a demesne of passing richness, fair with orchard lawns and
+corn lands, if you should slay me? This you shall hardly do. I
+have discomfited you once already. Have you forgotten how when
+you were alone I chased you from your herds helter-skelter down
+the slopes of Ida? You did not turn round to look behind you; you
+took refuge in Lyrnessus, but I attacked the city, and with the
+help of Minerva and father Jove I sacked it and carried its women
+into captivity, though Jove and the other gods rescued you. You
+think they will protect you now, but they will not do so;
+therefore I say go back into the host, and do not face me, or you
+will rue it. Even a fool may be wise after the event."
+</para>
+
+<para>
+Then Aeneas answered, "Son of Peleus, think not that your words
+can scare me as though I were a child. I too, if I will, can brag
+and talk unseemly. We know one another's race and parentage as
+matters of common fame, though neither have you ever seen my
+parents nor I yours. Men say that you are son to noble Peleus,
+and that your mother is Thetis, fair-haired daughter of the sea.
+I have noble Anchises for my father, and Venus for my mother; the
+parents of one or other of us shall this day mourn a son, for it
+will be more than silly talk that shall part us when the fight is
+over. Learn, then, my lineage if you will--and it is known to
+many.
+</para>
+
+<para>
+"In the beginning Dardanus was the son of Jove, and founded
+Dardania, for Ilius was not yet stablished on the plain for men
+to dwell in, and her people still abode on the spurs of
+many-fountained Ida. Dardanus had a son, king Erichthonius, who
+was wealthiest of all men living; he had three thousand mares
+that fed by the water-meadows, they and their foals with them.
+Boreas was enamoured of them as they were feeding, and covered
+them in the semblance of a dark-maned stallion. Twelve filly
+foals did they conceive and bear him, and these, as they sped
+over the rich plain, would go bounding on over the ripe ears of
+corn and not break them; or again when they would disport
+themselves on the broad back of Ocean they could gallop on the
+crest of a breaker. Erichthonius begat Tros, king of the Trojans,
+and Tros had three noble sons, Ilus, Assaracus, and Ganymede who
+was comeliest of mortal men; wherefore the gods carried him off
+to be Jove's cupbearer, for his beauty's sake, that he might
+dwell among the immortals. Ilus begat Laomedon, and Laomedon
+begat Tithonus, Priam, Lampus, Clytius, and Hiketaon of the stock
+of Mars. But Assaracus was father to Capys, and Capys to
+Anchises, who was my father, while Hector is son to Priam.
+</para>
+
+<para>
+"Such do I declare my blood and lineage, but as for valour, Jove
+gives it or takes it as he will, for he is lord of all. And now
+let there be no more of this prating in mid-battle as though we
+were children. We could fling taunts without end at one another;
+a hundred-oared galley would not hold them. The tongue can run
+all whithers and talk all wise; it can go here and there, and as
+a man says, so shall he be gainsaid. What is the use of our
+bandying hard like women who when they fall foul of one another
+go out and wrangle in the streets, one half true and the other
+lies, as rage inspires them? No words of yours shall turn me now
+that I am fain to fight--therefore let us make trial of one
+another with our spears."
+</para>
+
+<para>
+As he spoke he drove his spear at the great and terrible shield
+of Achilles, which rang out as the point struck it. The son of
+Peleus held the shield before him with his strong hand, and he
+was afraid, for he deemed that Aeneas's spear would go through it
+quite easily, not reflecting that the god's glorious gifts were
+little likely to yield before the blows of mortal men; and indeed
+Aeneas's spear did not pierce the shield, for the layer of gold,
+gift of the god, stayed the point. It went through two layers,
+but the god had made the shield in five, two of bronze, the two
+innermost ones of tin, and one of gold; it was in this that the
+spear was stayed.
+</para>
+
+<para>
+Achilles in his turn threw, and struck the round shield of Aeneas
+at the very edge, where the bronze was thinnest; the spear of
+Pelian ash went clean through, and the shield rang under the
+blow; Aeneas was afraid, and crouched backwards, holding the
+shield away from him; the spear, however, flew over his back, and
+stuck quivering in the ground, after having gone through both
+circles of the sheltering shield. Aeneas though he had avoided
+the spear, stood still, blinded with fear and grief because the
+weapon had gone so near him; then Achilles sprang furiously upon
+him, with a cry as of death and with his keen blade drawn, and
+Aeneas seized a great stone, so huge that two men, as men now
+are, would be unable to lift it, but Aeneas wielded it quite
+easily.
+</para>
+
+<para>
+Aeneas would then have struck Achilles as he was springing
+towards him, either on the helmet, or on the shield that covered
+him, and Achilles would have closed with him and despatched him
+with his sword, had not Neptune lord of the earthquake been quick
+to mark, and said forthwith to the immortals, "Alas, I am sorry
+for great Aeneas, who will now go down to the house of Hades,
+vanquished by the son of Peleus. Fool that he was to give ear to
+the counsel of Apollo. Apollo will never save him from
+destruction. Why should this man suffer when he is guiltless, to
+no purpose, and in another's quarrel? Has he not at all times
+offered acceptable sacrifice to the gods that dwell in heaven?
+Let us then snatch him from death's jaws, lest the son of Saturn
+be angry should Achilles slay him. It is fated, moreover, that he
+should escape, and that the race of Dardanus, whom Jove loved
+above all the sons born to him of mortal women, shall not perish
+utterly without seed or sign. For now indeed has Jove hated the
+blood of Priam, while Aeneas shall reign over the Trojans, he and
+his children's children that shall be born hereafter."
+</para>
+
+<para>
+Then answered Juno, "Earth-shaker, look to this matter yourself,
+and consider concerning Aeneas, whether you will save him, or
+suffer him, brave though he be, to fall by the hand of Achilles
+son of Peleus. For of a truth we two, I and Pallas Minerva, have
+sworn full many a time before all the immortals, that never would
+we shield Trojans from destruction, not even when all Troy is
+burning in the flames that the Achaeans shall kindle."
+</para>
+
+<para>
+When earth-encircling Neptune heard this he went into the battle
+amid the clash of spears, and came to the place where Achilles
+and Aeneas were. Forthwith he shed a darkness before the eyes of
+the son of Peleus, drew the bronze-headed ashen spear from the
+shield of Aeneas, and laid it at the feet of Achilles. Then he
+lifted Aeneas on high from off the earth and hurried him away.
+Over the heads of many a band of warriors both horse and foot did
+he soar as the god's hand sped him, till he came to the very
+fringe of the battle where the Cauconians were arming themselves
+for fight. Neptune, shaker of the earth, then came near to him
+and said, "Aeneas, what god has egged you on to this folly in
+fighting the son of Peleus, who is both a mightier man of valour
+and more beloved of heaven than you are? Give way before him
+whensoever you meet him, lest you go down to the house of Hades
+even though fate would have it otherwise. When Achilles is dead
+you may then fight among the foremost undaunted, for none other
+of the Achaeans shall slay you."
+</para>
+
+<para>
+The god left him when he had given him these instructions, and at
+once removed the darkness from before the eyes of Achilles, who
+opened them wide indeed and said in great anger, "Alas! what
+marvel am I now beholding? Here is my spear upon the ground, but
+I see not him whom I meant to kill when I hurled it. Of a truth
+Aeneas also must be under heaven's protection, although I had
+thought his boasting was idle. Let him go hang; he will be in no
+mood to fight me further, seeing how narrowly he has missed being
+killed. I will now give my orders to the Danaans and attack some
+other of the Trojans."
+</para>
+
+<para>
+He sprang forward along the line and cheered his men on as he did
+so. "Let not the Trojans," he cried, "keep you at arm's length,
+Achaeans, but go for them and fight them man for man. However
+valiant I may be, I cannot give chase to so many and fight all of
+them. Even Mars, who is an immortal, or Minerva, would shrink
+from flinging himself into the jaws of such a fight and laying
+about him; nevertheless, so far as in me lies I will show no
+slackness of hand or foot nor want of endurance, not even for a
+moment; I will utterly break their ranks, and woe to the Trojan
+who shall venture within reach of my spear."
+</para>
+
+<para>
+Thus did he exhort them. Meanwhile Hector called upon the Trojans
+and declared that he would fight Achilles. "Be not afraid, proud
+Trojans," said he, "to face the son of Peleus; I could fight gods
+myself if the battle were one of words only, but they would be
+more than a match for me, if we had to use our spears. Even so
+the deed of Achilles will fall somewhat short of his word; he
+will do in part, and the other part he will clip short. I will go
+up against him though his hands be as fire--though his hands be
+fire and his strength iron."
+</para>
+
+<para>
+Thus urged the Trojans lifted up their spears against the
+Achaeans, and raised the cry of battle as they flung themselves
+into the midst of their ranks. But Phoebus Apollo came up to
+Hector and said, "Hector, on no account must you challenge
+Achilles to single combat; keep a lookout for him while you are
+under cover of the others and away from the thick of the fight,
+otherwise he will either hit you with a spear or cut you down at
+close quarters."
+</para>
+
+<para>
+Thus he spoke, and Hector drew back within the crowd, for he was
+afraid when he heard what the god had said to him. Achilles then
+sprang upon the Trojans with a terrible cry, clothed in valour as
+with a garment. First he killed Iphition son of Otrynteus, a
+leader of much people whom a naiad nymph had borne to Otrynteus
+waster of cities, in the land of Hyde under the snowy heights of
+Mt. Tmolus. Achilles struck him full on the head as he was coming
+on towards him, and split it clean in two; whereon he fell
+heavily to the ground and Achilles vaunted over him saying, "You
+be low, son of Otrynteus, mighty hero; your death is here, but
+your lineage is on the Gygaean lake where your father's estate
+lies, by Hyllus, rich in fish, and the eddying waters of Hermus."
+</para>
+
+<para>
+Thus did he vaunt, but darkness closed the eyes of the other.
+The chariots of the Achaeans cut him up as their wheels passed
+over him in the front of the battle, and after him Achilles
+killed Demoleon, a valiant man of war and son to Antenor. He
+struck him on the temple through his bronze-cheeked helmet. The
+helmet did not stay the spear, but it went right on, crushing the
+bone so that the brain inside was shed in all directions, and his
+lust of fighting was ended. Then he struck Hippodamas in the
+midriff as he was springing down from his chariot in front of
+him, and trying to escape. He breathed his last, bellowing like a
+bull bellows when young men are dragging him to offer him in
+sacrifice to the King of Helice, and the heart of the
+earth-shaker is glad; even so did he bellow as he lay dying.
+Achilles then went in pursuit of Polydorus son of Priam, whom his
+father had always forbidden to fight because he was the youngest
+of his sons, the one he loved best, and the fastest runner. He,
+in his folly and showing off the fleetness of his feet, was
+rushing about among front ranks until he lost his life, for
+Achilles struck him in the middle of the back as he was darting
+past him: he struck him just at the golden fastenings of his belt
+and where the two pieces of the double breastplate overlapped.
+The point of the spear pierced him through and came out by the
+navel, whereon he fell groaning on to his knees and a cloud of
+darkness overshadowed him as he sank holding his entrails in his
+hands.
+</para>
+
+<para>
+When Hector saw his brother Polydorus with his entrails in his
+hands and sinking down upon the ground, a mist came over his
+eyes, and he could not bear to keep longer at a distance; he
+therefore poised his spear and darted towards Achilles like a
+flame of fire. When Achilles saw him he bounded forward and
+vaunted saying, "This is he that has wounded my heart most deeply
+and has slain my beloved comrade. Not for long shall we two quail
+before one another on the highways of war."
+</para>
+
+<para>
+He looked fiercely on Hector and said, "Draw near, that you may
+meet your doom the sooner." Hector feared him not and answered,
+"Son of Peleus, think not that your words can scare me as though
+I were a child; I too if I will can brag and talk unseemly; I
+know that you are a mighty warrior, mightier by far than I,
+nevertheless the issue lies in the lap of heaven whether I, worse
+man though I be, may not slay you with my spear, for this too has
+been found keen ere now."
+</para>
+
+<para>
+He hurled his spear as he spoke, but Minerva breathed upon it,
+and though she breathed but very lightly she turned it back from
+going towards Achilles, so that it returned to Hector and lay at
+his feet in front of him. Achilles then sprang furiously on him
+with a loud cry, bent on killing him, but Apollo caught him up
+easily as a god can, and hid him in a thick darkness. Thrice did
+Achilles spring towards him spear in hand, and thrice did he
+waste his blow upon the air. When he rushed forward for the
+fourth time as though he were a god, he shouted aloud saying,
+"Hound, this time too you have escaped death--but of a truth it
+came exceedingly near you. Phoebus Apollo, to whom it seems you
+pray before you go into battle, has again saved you; but if I too
+have any friend among the gods I will surely make an end of you
+when I come across you at some other time. Now, however, I will
+pursue and overtake other Trojans."
+</para>
+
+<para>
+On this he struck Dryops with his spear, about the middle of his
+neck, and he fell headlong at his feet. There he let him lie and
+stayed Demouchus son of Philetor, a man both brave and of great
+stature, by hitting him on the knee with a spear; then he smote
+him with his sword and killed him. After this he sprang on
+Laogonus and Dardanus, sons of Bias, and threw them from their
+chariot, the one with a blow from a thrown spear, while the other
+he cut down in hand-to-hand fight. There was also Tros the son of
+Alastor--he came up to Achilles and clasped his knees in the hope
+that he would spare him and not kill him but let him go, because
+they were both of the same age. Fool, he might have known that he
+should not prevail with him, for the man was in no mood for pity
+or forbearance but was in grim earnest. Therefore when Tros laid
+hold of his knees and sought a hearing for his prayers, Achilles
+drove his sword into his liver, and the liver came rolling out,
+while his bosom was all covered with the black blood that welled
+from the wound. Thus did death close his eyes as he lay lifeless.
+</para>
+
+<para>
+Achilles then went up to Mulius and struck him on the ear with a
+spear, and the bronze spear-head came right out at the other ear.
+He also struck Echeclus son of Agenor on the head with his sword,
+which became warm with the blood, while death and stern fate
+closed the eyes of Echeclus. Next in order the bronze point of
+his spear wounded Deucalion in the fore-arm where the sinews of
+the elbow are united, whereon he waited Achilles' onset with his
+arm hanging down and death staring him in the face. Achilles cut
+his head off with a blow from his sword and flung it helmet and
+all away from him, and the marrow came oozing out of his backbone
+as he lay. He then went in pursuit of Rhigmus, noble son of
+Peires, who had come from fertile Thrace, and struck him through
+the middle with a spear which fixed itself in his belly, so that
+he fell headlong from his chariot. He also speared Areithous
+squire to Rhigmus in the back as he was turning his horses in
+flight, and thrust him from his chariot, while the horses were
+struck with panic.
+</para>
+
+<para>
+As a fire raging in some mountain glen after long drought--and
+the dense forest is in a blaze, while the wind carries great
+tongues of fire in every direction--even so furiously did
+Achilles rage, wielding his spear as though he were a god, and
+giving chase to those whom he would slay, till the dark earth ran
+with blood. Or as one who yokes broad-browed oxen that they may
+tread barley in a threshing-floor--and it is soon bruised small
+under the feet of the lowing cattle--even so did the horses of
+Achilles trample on the shields and bodies of the slain. The axle
+underneath and the railing that ran round the car were
+bespattered with clots of blood thrown up by the horses' hoofs,
+and from the tyres of the wheels; but the son of Peleus pressed
+on to win still further glory, and his hands were bedrabbled with
+gore.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XXI</title>
+
+<para>
+NOW when they came to the ford of the full-flowing river Xanthus,
+begotten of immortal Jove, Achilles cut their forces in two: one
+half he chased over the plain towards the city by the same way
+that the Achaeans had taken when flying panic-stricken on the
+preceding day with Hector in full triumph; this way did they fly
+pell-mell, and Juno sent down a thick mist in front of them to
+stay them. The other half were hemmed in by the deep
+silver-eddying stream, and fell into it with a great uproar. The
+waters resounded, and the banks rang again, as they swam hither
+and thither with loud cries amid the whirling eddies. As locusts
+flying to a river before the blast of a grass fire--the flame
+comes on and on till at last it overtakes them and they huddle
+into the water--even so was the eddying stream of Xanthus filled
+with the uproar of men and horses, all struggling in confusion
+before Achilles.
+</para>
+
+<para>
+Forthwith the hero left his spear upon the bank, leaning it
+against a tamarisk bush, and plunged into the river like a god,
+armed with his sword only. Fell was his purpose as he hewed the
+Trojans down on every side. Their dying groans rose hideous as
+the sword smote them, and the river ran red with blood. As when
+fish fly scared before a huge dolphin, and fill every nook and
+corner of some fair haven--for he is sure to eat all he can
+catch--even so did the Trojans cower under the banks of the
+mighty river, and when Achilles' arms grew weary with killing
+them, he drew twelve youths alive out of the water, to sacrifice
+in revenge for Patroclus son of Menoetius. He drew them out like
+dazed fawns, bound their hands behind them with the girdles of
+their own shirts, and gave them over to his men to take back to
+the ships. Then he sprang into the river, thirsting for still
+further blood.
+</para>
+
+<para>
+There he found Lycaon, son of Priam seed of Dardanus, as he was
+escaping out of the water; he it was whom he had once taken
+prisoner when he was in his father's vineyard, having set upon
+him by night, as he was cutting young shoots from a wild fig-tree
+to make the wicker sides of a chariot. Achilles then caught him
+to his sorrow unawares, and sent him by sea to Lemnos, where the
+son of Jason bought him. But a guest-friend, Eetion of Imbros,
+freed him with a great sum, and sent him to Arisbe, whence he had
+escaped and returned to his father's house. He had spent eleven
+days happily with his friends after he had come from Lemnos, but
+on the twelfth heaven again delivered him into the hands of
+Achilles, who was to send him to the house of Hades sorely
+against his will. He was unarmed when Achilles caught sight of
+him, and had neither helmet nor shield; nor yet had he any spear,
+for he had thrown all his armour from him on to the bank, and was
+sweating with his struggles to get out of the river, so that his
+strength was now failing him.
+</para>
+
+<para>
+Then Achilles said to himself in his surprise, "What marvel do I
+see here? If this man can come back alive after having been sold
+over into Lemnos, I shall have the Trojans also whom I have slain
+rising from the world below. Could not even the waters of the
+grey sea imprison him, as they do many another whether he will or
+no? This time let him taste my spear, that I may know for certain
+whether mother earth who can keep even a strong man down, will be
+able to hold him, or whether thence too he will return."
+</para>
+
+<para>
+Thus did he pause and ponder. But Lycaon came up to him dazed and
+trying hard to embrace his knees, for he would fain live, not
+die. Achilles thrust at him with his spear, meaning to kill him,
+but Lycaon ran crouching up to him and caught his knees, whereby
+the spear passed over his back, and stuck in the ground,
+hungering though it was for blood. With one hand he caught
+Achilles' knees as he besought him, and with the other he
+clutched the spear and would not let it go. Then he said,
+"Achilles, have mercy upon me and spare me, for I am your
+suppliant. It was in your tents that I first broke bread on the
+day when you took me prisoner in the vineyard; after which you
+sold me away to Lemnos far from my father and my friends, and I
+brought you the price of a hundred oxen. I have paid three times
+as much to gain my freedom; it is but twelve days that I have
+come to Ilius after much suffering, and now cruel fate has again
+thrown me into your hands. Surely father Jove must hate me, that
+he has given me over to you a second time. Short of life indeed
+did my mother Laothoe bear me, daughter of aged Altes--of Altes
+who reigns over the warlike Lelegae and holds steep Pedasus on
+the river Satnioeis. Priam married his daughter along with many
+other women and two sons were born of her, both of whom you will
+have slain. Your spear slew noble Polydorus as he was fighting in
+the front ranks, and now evil will here befall me, for I fear
+that I shall not escape you since heaven has delivered me over to
+you. Furthermore I say, and lay my saying to your heart, spare
+me, for I am not of the same womb as Hector who slew your brave
+and noble comrade."
+</para>
+
+<para>
+With such words did the princely son of Priam beseech Achilles;
+but Achilles answered him sternly. "Idiot," said he, "talk not to
+me of ransom. Until Patroclus fell I preferred to give the
+Trojans quarter, and sold beyond the sea many of those whom I had
+taken alive; but now not a man shall live of those whom heaven
+delivers into my hands before the city of Ilius--and of all
+Trojans it shall fare hardest with the sons of Priam. Therefore,
+my friend, you too shall die. Why should you whine in this way?
+Patroclus fell, and he was a better man than you are. I too--see
+you not how I am great and goodly? I am son to a noble father,
+and have a goddess for my mother, but the hands of doom and death
+overshadow me all as surely. The day will come, either at dawn or
+dark, or at the noontide, when one shall take my life also in
+battle, either with his spear, or with an arrow sped from his
+bow."
+</para>
+
+<para>
+Thus did he speak, and Lycaon's heart sank within him. He loosed
+his hold of the spear, and held out both hands before him; but
+Achilles drew his keen blade, and struck him by the collar-bone
+on his neck; he plunged his two-edged sword into him to the very
+hilt, whereon he lay at full length on the ground, with the dark
+blood welling from him till the earth was soaked. Then Achilles
+caught him by the foot and flung him into the river to go down
+stream, vaunting over him the while, and saying, "Lie there among
+the fishes, who will lick the blood from your wound and gloat
+over it; your mother shall not lay you on any bier to mourn you,
+but the eddies of Scamander shall bear you into the broad bosom
+of the sea. There shall the fishes feed on the fat of Lycaon as
+they dart under the dark ripple of the waters--so perish all of
+you till we reach the citadel of strong Ilius--you in flight, and
+I following after to destroy you. The river with its broad silver
+stream shall serve you in no stead, for all the bulls you offered
+him and all the horses that you flung living into his waters.
+None the less miserably shall you perish till there is not a man
+of you but has paid in full for the death of Patroclus and the
+havoc you wrought among the Achaeans whom you have slain while I
+held aloof from battle."
+</para>
+
+<para>
+So spoke Achilles, but the river grew more and more angry, and
+pondered within himself how he should stay the hand of Achilles
+and save the Trojans from disaster. Meanwhile the son of Peleus,
+spear in hand, sprang upon Asteropaeus son of Pelegon to kill
+him. He was son to the broad river Axius and Periboea eldest
+daughter of Acessamenus; for the river had lain with her.
+Asteropaeus stood up out of the water to face him with a spear in
+either hand, and Xanthus filled him with courage, being angry for
+the death of the youths whom Achilles was slaying ruthlessly
+within his waters. When they were close up with one another
+Achilles was first to speak. "Who and whence are you," said he,
+"who dare to face me? Woe to the parents whose son stands up
+against me." And the son of Pelegon answered, "Great son of
+Peleus, why should you ask my lineage. I am from the fertile land
+of far Paeonia, captain of the Paeonians, and it is now eleven
+days that I am at Ilius. I am of the blood of the river Axius--of
+Axius that is the fairest of all rivers that run. He begot the
+famed warrior Pelegon, whose son men call me. Let us now fight,
+Achilles."
+</para>
+
+<para>
+Thus did he defy him, and Achilles raised his spear of Pelian
+ash. Asteropaeus failed with both his spears, for he could use
+both hands alike; with the one spear he struck Achilles' shield,
+but did not pierce it, for the layer of gold, gift of the god,
+stayed the point; with the other spear he grazed the elbow of
+Achilles' right arm drawing dark blood, but the spear itself went
+by him and fixed itself in the ground, foiled of its bloody
+banquet. Then Achilles, fain to kill him, hurled his spear at
+Asteropaeus, but failed to hit him and struck the steep bank of
+the river, driving the spear half its length into the earth. The
+son of Peleus then drew his sword and sprang furiously upon him.
+Asteropaeus vainly tried to draw Achilles' spear out of the bank
+by main force; thrice did he tug at it, trying with all his might
+to draw it out, and thrice he had to leave off trying; the fourth
+time he tried to bend and break it, but ere he could do so
+Achilles smote him with his sword and killed him. He struck him
+in the belly near the navel, so that all his bowels came gushing
+out on to the ground, and the darkness of death came over him as
+he lay gasping. Then Achilles set his foot on his chest and
+spoiled him of his armour, vaunting over him and saying, "Lie
+there--begotten of a river though you be, it is hard for you to
+strive with the offspring of Saturn's son. You declare yourself
+sprung from the blood of a broad river, but I am of the seed of
+mighty Jove. My father is Peleus, son of Aeacus ruler over the
+many Myrmidons, and Aeacus was the son of Jove. Therefore as Jove
+is mightier than any river that flows into the sea, so are his
+children stronger than those of any river whatsoever. Moreover
+you have a great river hard by if he can be of any use to you,
+but there is no fighting against Jove the son of Saturn, with
+whom not even King Achelous can compare, nor the mighty stream of
+deep-flowing Oceanus, from whom all rivers and seas with all
+springs and deep wells proceed; even Oceanus fears the lightnings
+of great Jove, and his thunder that comes crashing out of
+heaven."
+</para>
+
+<para>
+With this he drew his bronze spear out of the bank, and now that
+he had killed Asteropaeus, he let him lie where he was on the
+sand, with the dark water flowing over him and the eels and
+fishes busy nibbling and gnawing the fat that was about his
+kidneys. Then he went in chase of the Paeonians, who were flying
+along the bank of the river in panic when they saw their leader
+slain by the hands of the son of Peleus. Therein he slew
+Thersilochus, Mydon, Astypylus, Mnesus, Thrasius, Oeneus, and
+Ophelestes, and he would have slain yet others, had not the river
+in anger taken human form, and spoken to him from out the deep
+waters saying, "Achilles, if you excel all in strength, so do you
+also in wickedness, for the gods are ever with you to protect
+you: if, then, the son of Saturn has vouchsafed it to you to
+destroy all the Trojans, at any rate drive them out of my stream,
+and do your grim work on land. My fair waters are now filled with
+corpses, nor can I find any channel by which I may pour myself
+into the sea for I am choked with dead, and yet you go on
+mercilessly slaying. I am in despair, therefore, O captain of
+your host, trouble me no further."
+</para>
+
+<para>
+Achilles answered, "So be it, Scamander, Jove-descended; but I
+will never cease dealing out death among the Trojans, till I have
+pent them up in their city, and made trial of Hector face to
+face, that I may learn whether he is to vanquish me, or I him."
+</para>
+
+<para>
+As he spoke he set upon the Trojans with a fury like that of the
+gods. But the river said to Apollo, "Surely, son of Jove, lord of
+the silver bow, you are not obeying the commands of Jove who
+charged you straitly that you should stand by the Trojans and
+defend them, till twilight fades, and darkness is over an the
+earth."
+</para>
+
+<para>
+Meanwhile Achilles sprang from the bank into mid-stream, whereon
+the river raised a high wave and attacked him. He swelled his
+stream into a torrent, and swept away the many dead whom Achilles
+had slain and left within his waters. These he cast out on to the
+land, bellowing like a bull the while, but the living he saved
+alive, hiding them in his mighty eddies. The great and terrible
+wave gathered about Achilles, falling upon him and beating on his
+shield, so that he could not keep his feet; he caught hold of a
+great elm-tree, but it came up by the roots, and tore away the
+bank, damming the stream with its thick branches and bridging it
+all across; whereby Achilles struggled out of the stream, and
+fled full speed over the plain, for he was afraid.
+</para>
+
+<para>
+But the mighty god ceased not in his pursuit, and sprang upon him
+with a dark-crested wave, to stay his hands and save the Trojans
+from destruction. The son of Peleus darted away a spear's throw
+from him; swift as the swoop of a black hunter-eagle which is the
+strongest and fleetest of all birds, even so did he spring
+forward, and the armour rang loudly about his breast. He fled on
+in front, but the river with a loud roar came tearing after. As
+one who would water his garden leads a stream from some fountain
+over his plants, and all his ground-spade in hand he clears away
+the dams to free the channels, and the little stones run rolling
+round and round with the water as it goes merrily down the bank
+faster than the man can follow--even so did the river keep
+catching up with Achilles albeit he was a fleet runner, for the
+gods are stronger than men. As often as he would strive to stand
+his ground, and see whether or no all the gods in heaven were in
+league against him, so often would the mighty wave come beating
+down upon his shoulders, and be would have to keep flying on and
+on in great dismay; for the angry flood was tiring him out as it
+flowed past him and ate the ground from under his feet.
+</para>
+
+<para>
+Then the son of Peleus lifted up his voice to heaven saying,
+"Father Jove, is there none of the gods who will take pity upon
+me, and save me from the river? I do not care what may happen to
+me afterwards. I blame none of the other dwellers on Olympus so
+severely as I do my dear mother, who has beguiled and tricked me.
+She told me I was to fall under the walls of Troy by the flying
+arrows of Apollo; would that Hector, the best man among the
+Trojans, might there slay me; then should I fall a hero by the
+hand of a hero; whereas now it seems that I shall come to a most
+pitiable end, trapped in this river as though I were some
+swineherd's boy, who gets carried down a torrent while trying to
+cross it during a storm."
+</para>
+
+<para>
+As soon as he had spoken thus, Neptune and Minerva came up to him
+in the likeness of two men, and took him by the hand to reassure
+him. Neptune spoke first. "Son of Peleus," said he, "be not so
+exceeding fearful; we are two gods, come with Jove's sanction to
+assist you, I, and Pallas Minerva. It is not your fate to perish
+in this river; he will abate presently as you will see; moreover
+we strongly advise you, if you will be guided by us, not to stay
+your hand from fighting till you have pent the Trojan host within
+the famed walls of Ilius--as many of them as may escape. Then
+kill Hector and go back to the ships, for we will vouchsafe you a
+triumph over him."
+</para>
+
+<para>
+When they had so said they went back to the other immortals, but
+Achilles strove onward over the plain, encouraged by the charge
+the gods had laid upon him. All was now covered with the flood of
+waters, and much goodly armour of the youths that had been slain
+was rifting about, as also many corpses, but he forced his way
+against the stream, speeding right onwards, nor could the broad
+waters stay him, for Minerva had endowed him with great strength.
+Nevertheless Scamander did not slacken in his pursuit, but was
+still more furious with the son of Peleus. He lifted his waters
+into a high crest and cried aloud to Simois saying, "Dear
+brother, let the two of us unite to save this man, or he will
+sack the mighty city of King Priam, and the Trojans will not hold
+out against him. Help me at once; fill your streams with water
+from their sources, rouse all your torrents to a fury; raise your
+wave on high, and let snags and stones come thundering down you
+that we may make an end of this savage creature who is now
+lording it as though he were a god. Nothing shall serve him
+longer, not strength nor comeliness, nor his fine armour, which
+forsooth shall soon be lying low in the deep waters covered over
+with mud. I will wrap him in sand, and pour tons of shingle round
+him, so that the Achaeans shall not know how to gather his bones
+for the silt in which I shall have hidden him, and when they
+celebrate his funeral they need build no barrow."
+</para>
+
+<para>
+On this he upraised his tumultuous flood high against Achilles,
+seething as it was with foam and blood and the bodies of the
+dead. The dark waters of the river stood upright and would have
+overwhelmed the son of Peleus, but Juno, trembling lest Achilles
+should be swept away in the mighty torrent, lifted her voice on
+high and called out to Vulcan her son. "Crook-foot," she cried,
+"my child, be up and doing, for I deem it is with you that
+Xanthus is fain to fight; help us at once, kindle a fierce fire;
+I will then bring up the west and the white south wind in a
+mighty hurricane from the sea, that shall bear the flames against
+the heads and armour of the Trojans and consume them, while you
+go along the banks of Xanthus burning his trees and wrapping him
+round with fire. Let him not turn you back neither by fair words
+nor foul, and slacken not till I shout and tell you. Then you may
+stay your flames."
+</para>
+
+<para>
+On this Vulcan kindled a fierce fire, which broke out first upon
+the plain and burned the many dead whom Achilles had killed and
+whose bodies were lying about in great numbers; by this means the
+plain was dried and the flood stayed. As the north wind, blowing
+on an orchard that has been sodden with autumn rain, soon dries
+it, and the heart of the owner is glad--even so the whole plain
+was dried and the dead bodies were consumed. Then he turned
+tongues of fire on to the river. He burned the elms the willows
+and the tamarisks, the lotus also, with the rushes and marshy
+herbage that grew abundantly by the banks of the river. The eels
+and fishes that go darting about everywhere in the water, these,
+too, were sorely harassed by the flames that cunning Vulcan had
+kindled, and the river himself was scalded, so that he spoke
+saying, "Vulcan, there is no god can hold his own against you. I
+cannot fight you when you flare out your flames in this way;
+strive with me no longer. Let Achilles drive the Trojans out of
+city immediately. What have I to do with quarrelling and helping
+people?"
+</para>
+
+<para>
+He was boiling as he spoke, and all his waters were seething. As
+a cauldron upon a large fire boils when it is melting the lard of
+some fatted hog, and the lard keeps bubbling up all over when the
+dry faggots blaze under it--even so were the goodly waters of
+Xanthus heated with the fire till they were boiling. He could
+flow no longer but stayed his stream, so afflicted was he by the
+blasts of fire which cunning Vulcan had raised. Then he prayed to
+Juno and besought her saying, "Juno, why should your son vex my
+stream with such especial fury? I am not so much to blame as all
+the others are who have been helping the Trojans. I will leave
+off, since you so desire it, and let son leave off also.
+Furthermore I swear never again will I do anything to save the
+Trojans from destruction, not even when all Troy is burning in
+the flames which the Achaeans will kindle."
+</para>
+
+<para>
+As soon as Juno heard this she said to her son Vulcan, "Son
+Vulcan, hold now your flames; we ought not to use such violence
+against a god for the sake of mortals."
+</para>
+
+<para>
+When she had thus spoken Vulcan quenched his flames, and the
+river went back once more into his own fair bed.
+</para>
+
+<para>
+Xanthus was now beaten, so these two left off fighting, for Juno
+stayed them though she was still angry; but a furious quarrel
+broke out among the other gods, for they were of divided
+counsels. They fell on one another with a mighty uproar--earth
+groaned, and the spacious firmament rang out as with a blare of
+trumpets. Jove heard as he was sitting on Olympus, and laughed
+for joy when he saw the gods coming to blows among themselves.
+They were not long about beginning, and Mars piercer of shields
+opened the battle. Sword in hand he sprang at once upon Minerva
+and reviled her. "Why, vixen," said he, "have you again set the
+gods by the ears in the pride and haughtiness of your heart? Have
+you forgotten how you set Diomed son of Tydeus on to wound me,
+and yourself took visible spear and drove it into me to the hurt
+of my fair body? You shall now suffer for what you then did to
+me."
+</para>
+
+<para>
+As he spoke he struck her on the terrible tasselled aegis--so
+terrible that not even can Jove's lightning pierce it. Here did
+murderous Mars strike her with his great spear. She drew back and
+with her strong hand seized a stone that was lying on the plain--
+great and rugged and black--which men of old had set for the
+boundary of a field. With this she struck Mars on the neck, and
+brought him down. Nine roods did he cover in his fall, and his
+hair was all soiled in the dust, while his armour rang rattling
+round him. But Minerva laughed and vaunted over him saying,
+"Idiot, have you not learned how far stronger I am than you, but
+you must still match yourself against me? Thus do your mother's
+curses now roost upon you, for she is angry and would do you
+mischief because you have deserted the Achaeans and are helping
+the Trojans."
+</para>
+
+<para>
+She then turned her two piercing eyes elsewhere, whereon Jove's
+daughter Venus took Mars by the hand and led him away groaning
+all the time, for it was only with great difficulty that he had
+come to himself again. When Queen Juno saw her, she said to
+Minerva, "Look, daughter of aegis-bearing Jove, unweariable, that
+vixen Venus is again taking Mars through the crowd out of the
+battle; go after her at once."
+</para>
+
+<para>
+Thus she spoke. Minerva sped after Venus with a will, and made at
+her, striking her on the bosom with her strong hand so that she
+fell fainting to the ground, and there they both lay stretched at
+full length. Then Minerva vaunted over her saying, "May all who
+help the Trojans against the Argives prove just as redoubtable
+and stalwart as Venus did when she came across me while she was
+helping Mars. Had this been so, we should long since have ended
+the war by sacking the strong city of Ilius."
+</para>
+
+<para>
+Juno smiled as she listened. Meanwhile King Neptune turned to
+Apollo saying, "Phoebus, why should we keep each other at arm's
+length? it is not well, now that the others have begun fighting;
+it will be disgraceful to us if we return to Jove's
+bronze-floored mansion on Olympus without having fought each
+other; therefore come on, you are the younger of the two, and I
+ought not to attack you, for I am older and have had more
+experience. Idiot, you have no sense, and forget how we two alone
+of all the gods fared hardly round about Ilius when we came from
+Jove's house and worked for Laomedon a whole year at a stated
+wage and he gave us his orders. I built the Trojans the wall
+about their city, so wide and fair that it might be impregnable,
+while you, Phoebus, herded cattle for him in the dales of many
+valleyed Ida. When, however, the glad hours brought round the
+time of payment, mighty Laomedon robbed us of all our hire and
+sent us off with nothing but abuse. He threatened to bind us hand
+and foot and sell us over into some distant island. He tried,
+moreover, to cut off the ears of both of us, so we went away in a
+rage, furious about the payment he had promised us, and yet
+withheld; in spite of all this, you are now showing favour to his
+people, and will not join us in compassing the utter ruin of the
+proud Trojans with their wives and children."
+</para>
+
+<para>
+And King Apollo answered, "Lord of the earthquake, you would have
+no respect for me if I were to fight you about a pack of
+miserable mortals, who come out like leaves in summer and eat the
+fruit of the field, and presently fall lifeless to the ground.
+Let us stay this fighting at once and let them settle it among
+themselves."
+</para>
+
+<para>
+He turned away as he spoke, for he would lay no hand on the
+brother of his own father. But his sister the huntress Diana,
+patroness of wild beasts, was very angry with him and said, "So
+you would fly, Far-Darter, and hand victory over to Neptune with
+a cheap vaunt to boot. Baby, why keep your bow thus idle? Never
+let me again hear you bragging in my father's house, as you have
+often done in the presence of the immortals, that you would stand
+up and fight with Neptune."
+</para>
+
+<para>
+Apollo made her no answer, but Jove's august queen was angry and
+upbraided her bitterly. "Bold vixen," she cried, "how dare you
+cross me thus? For all your bow you will find it hard to hold
+your own against me. Jove made you as a lion among women, and
+lets you kill them whenever you choose. You will find it better
+to chase wild beasts and deer upon the mountains than to fight
+those who are stronger than you are. If you would try war, do so,
+and find out by pitting yourself against me, how far stronger I
+am than you are."
+</para>
+
+<para>
+She caught both Diana's wrists with her left hand as she spoke,
+and with her right she took the bow from her shoulders, and
+laughed as she beat her with it about the ears while Diana
+wriggled and writhed under her blows. Her swift arrows were shed
+upon the ground, and she fled weeping from under Juno's hand as a
+dove that flies before a falcon to the cleft of some hollow rock,
+when it is her good fortune to escape. Even so did she fly
+weeping away, leaving her bow and arrows behind her.
+</para>
+
+<para>
+Then the slayer of Argus, guide and guardian, said to Leto,
+"Leto, I shall not fight you; it is ill to come to blows with any
+of Jove's wives. Therefore boast as you will among the immortals
+that you worsted me in fair fight."
+</para>
+
+<para>
+Leto then gathered up Diana's bow and arrows that had fallen
+about amid the whirling dust, and when she had got them she made
+all haste after her daughter. Diana had now reached Jove's
+bronze-floored mansion on Olympus, and sat herself down with many
+tears on the knees of her father, while her ambrosial raiment was
+quivering all about her. The son of Saturn drew her towards him,
+and laughing pleasantly the while began to question her saying,
+"Which of the heavenly beings, my dear child, has been treating
+you in this cruel manner, as though you had been misconducting
+yourself in the face of everybody?" and the fair-crowned goddess
+of the chase answered, "It was your wife Juno, father, who has
+been beating me; it is always her doing when there is any
+quarrelling among the immortals."
+</para>
+
+<para>
+Thus did they converse, and meanwhile Phoebus Apollo entered the
+strong city of Ilius, for he was uneasy lest the wall should not
+hold out and the Danaans should take the city then and there,
+before its hour had come; but the rest of the ever-living gods
+went back, some angry and some triumphant to Olympus, where they
+took their seats beside Jove lord of the storm cloud, while
+Achilles still kept on dealing out death alike on the Trojans and
+on their horses. As when the smoke from some burning city ascends
+to heaven when the anger of the gods has kindled it--there is
+then toil for all, and sorrow for not a few--even so did Achilles
+bring toil and sorrow on the Trojans.
+</para>
+
+<para>
+Old King Priam stood on a high tower of the wall looking down on
+huge Achilles as the Trojans fled panic-stricken before him, and
+there was none to help them. Presently he came down from off the
+tower and with many a groan went along the wall to give orders to
+the brave warders of the gate. "Keep the gates," said he, "wide
+open till the people come flying into the city, for Achilles is
+hard by and is driving them in rout before him. I see we are in
+great peril. As soon as our people are inside and in safety,
+close the strong gates for I fear lest that terrible man should
+come bounding inside along with the others."
+</para>
+
+<para>
+As he spoke they drew back the bolts and opened the gates, and
+when these were opened there was a haven of refuge for the
+Trojans. Apollo then came full speed out of the city to meet them
+and protect them. Right for the city and the high wall, parched
+with thirst and grimy with dust, still they fied on, with
+Achilles wielding his spear furiously behind them. For he was as
+one possessed, and was thirsting after glory.
+</para>
+
+<para>
+Then had the sons of the Achaeans taken the lofty gates of Troy
+if Apollo had not spurred on Agenor, valiant and noble son to
+Antenor. He put courage into his heart, and stood by his side to
+guard him, leaning against a beech tree and shrouded in thick
+darkness. When Agenor saw Achilles he stood still and his heart
+was clouded with care. "Alas," said he to himself in his dismay,
+"if I fly before mighty Achilles, and go where all the others are
+being driven in rout, he will none the less catch me and kill me
+for a coward. How would it be were I to let Achilles drive the
+others before him, and then fly from the wall to the plain that
+is behind Ilius till I reach the spurs of Ida and can hide in the
+underwood that is thereon? I could then wash the sweat from off
+me in the river and in the evening return to Ilius. But why
+commune with myself in this way? Like enough he would see me as I
+am hurrying from the city over the plain, and would speed after
+me till he had caught me--I should stand no chance against him,
+for he is mightiest of all mankind. What, then, if I go out and
+meet him in front of the city? His flesh too, I take it, can be
+pierced by pointed bronze. Life is the same in one and all, and
+men say that he is but mortal despite the triumph that Jove son
+of Saturn vouchsafes him."
+</para>
+
+<para>
+So saying he stood on his guard and awaited Achilles, for he was
+now fain to fight him. As a leopardess that bounds from out a
+thick covert to attack a hunter--she knows no fear and is not
+dismayed by the baying of the hounds; even though the man be too
+quick for her and wound her either with thrust or spear, still,
+though the spear has pierced her she will not give in till she
+has either caught him in her grip or been killed outright--even
+so did noble Agenor son of Antenor refuse to fly till he had made
+trial of Achilles, and took aim at him with his spear, holding
+his round shield before him and crying with a loud voice. "Of a
+truth," said he, "noble Achilles, you deem that you shall this
+day sack the city of the proud Trojans. Fool, there will be
+trouble enough yet before it, for there is many a brave man of us
+still inside who will stand in front of our dear parents with our
+wives and children, to defend Ilius. Here therefore, huge and
+mighty warrior though you be, here shall you die."
+</para>
+
+<para>
+As he spoke his strong hand hurled his javelin from him, and the
+spear struck Achilles on the leg beneath the knee; the greave of
+newly wrought tin rang loudly, but the spear recoiled from the
+body of him whom it had struck, and did not pierce it, for the
+gods gift stayed it. Achilles in his turn attacked noble Agenor,
+but Apollo would not vouchsafe him glory, for he snatched Agenor
+away and hid him in a thick mist, sending him out of the battle
+unmolested Then he craftily drew the son of Peleus away from
+going after the host, for he put on the semblance of Agenor and
+stood in front of Achilles, who ran towards him to give him chase
+and pursued him over the corn lands of the plain, turning him
+towards the deep waters of the river Scamander. Apollo ran but a
+little way before him and beguiled Achilles by making him think
+all the time that he was on the point of overtaking him.
+Meanwhile the rabble of routed Trojans was thankful to crowd
+within the city till their numbers thronged it; no longer did
+they dare wait for one another outside the city walls, to learn
+who had escaped and who were fallen in fight, but all whose feet
+and knees could still carry them poured pell-mell into the town.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XXII</title>
+
+<para>
+THUS the Trojans in the city, scared like fawns, wiped the sweat
+from off them and drank to quench their thirst, leaning against
+the goodly battlements, while the Achaeans with their shields
+laid upon their shoulders drew close up to the walls. But stern
+fate bade Hector stay where he was before Ilius and the Scaean
+gates. Then Phoebus Apollo spoke to the son of Peleus saying,
+"Why, son of Peleus, do you, who are but man, give chase to me
+who am immortal? Have you not yet found out that it is a god whom
+you pursue so furiously? You did not harass the Trojans whom you
+had routed, and now they are within their walls, while you have
+been decoyed hither away from them. Me you cannot kill, for death
+can take no hold upon me."
+</para>
+
+<para>
+Achilles was greatly angered and said, "You have baulked me,
+Far-Darter, most malicious of all gods, and have drawn me away
+from the wall, where many another man would have bitten the dust
+ere he got within Ilius; you have robbed me of great glory and
+have saved the Trojans at no risk to yourself, for you have
+nothing to fear, but I would indeed have my revenge if it were in
+my power to do so."
+</para>
+
+<para>
+On this, with fell intent he made towards the city, and as the
+winning horse in a chariot race strains every nerve when he is
+flying over the plain, even so fast and furiously did the limbs
+of Achilles bear him onwards. King Priam was first to note him as
+he scoured the plain, all radiant as the star which men call
+Orion's Hound, and whose beams blaze forth in time of harvest
+more brilliantly than those of any other that shines by night;
+brightest of them all though he be, he yet bodes ill for mortals,
+for he brings fire and fever in his train--even so did Achilles'
+armour gleam on his breast as he sped onwards. Priam raised a cry
+and beat his head with his hands as he lifted them up and shouted
+out to his dear son, imploring him to return; but Hector still
+stayed before the gates, for his heart was set upon doing battle
+with Achilles. The old man reached out his arms towards him and
+bade him for pity's sake come within the walls. "Hector," he
+cried, "my son, stay not to face this man alone and unsupported,
+or you will meet death at the hands of the son of Peleus, for he
+is mightier than you. Monster that he is; would indeed that the
+gods loved him no better than I do, for so, dogs and vultures
+would soon devour him as he lay stretched on earth, and a load of
+grief would be lifted from my heart, for many a brave son has he
+reft from me, either by killing them or selling them away in the
+islands that are beyond the sea: even now I miss two sons from
+among the Trojans who have thronged within the city, Lycaon and
+Polydorus, whom Laothoe peeress among women bore me. Should they
+be still alive and in the hands of the Achaeans, we will ransom
+them with gold and bronze, of which we have store, for the old
+man Altes endowed his daughter richly; but if they are already
+dead and in the house of Hades, sorrow will it be to us two who
+were their parents; albeit the grief of others will be more
+short-lived unless you too perish at the hands of Achilles. Come,
+then, my son, within the city, to be the guardian of Trojan men
+and Trojan women, or you will both lose your own life and afford
+a mighty triumph to the son of Peleus. Have pity also on your
+unhappy father while life yet remains to him--on me, whom the son
+of Saturn will destroy by a terrible doom on the threshold of old
+age, after I have seen my sons slain and my daughters haled away
+as captives, my bridal chambers pillaged, little children dashed
+to earth amid the rage of battle, and my sons' wives dragged away
+by the cruel hands of the Achaeans; in the end fierce hounds will
+tear me in pieces at my own gates after some one has beaten the
+life out of my body with sword or spear-hounds that I myself
+reared and fed at my own table to guard my gates, but who will
+yet lap my blood and then lie all distraught at my doors. When a
+young man falls by the sword in battle, he may lie where he is
+and there is nothing unseemly; let what will be seen, all is
+honourable in death, but when an old man is slain there is
+nothing in this world more pitiable than that dogs should defile
+his grey hair and beard and all that men hide for shame."
+</para>
+
+<para>
+The old man tore his grey hair as he spoke, but he moved not the
+heart of Hector. His mother hard by wept and moaned aloud as she
+bared her bosom and pointed to the breast which had suckled him.
+"Hector," she cried, weeping bitterly the while, "Hector, my son,
+spurn not this breast, but have pity upon me too: if I have ever
+given you comfort from my own bosom, think on it now, dear son,
+and come within the wall to protect us from this man; stand not
+without to meet him. Should the wretch kill you, neither I nor
+your richly dowered wife shall ever weep, dear offshoot of
+myself, over the bed on which you lie, for dogs will devour you
+at the ships of the Achaeans."
+</para>
+
+<para>
+Thus did the two with many tears implore their son, but they
+moved not the heart of Hector, and he stood his ground awaiting
+huge Achilles as he drew nearer towards him. As serpent in its
+den upon the mountains, full fed with deadly poisons, waits for
+the approach of man--he is filled with fury and his eyes glare
+terribly as he goes writhing round his den--even so Hector leaned
+his shield against a tower that jutted out from the wall and
+stood where he was, undaunted.
+</para>
+
+<para>
+"Alas," said he to himself in the heaviness of his heart, "if I
+go within the gates, Polydamas will be the first to heap reproach
+upon me, for it was he that urged me to lead the Trojans back to
+the city on that awful night when Achilles again came forth
+against us. I would not listen, but it would have been indeed
+better if I had done so. Now that my folly has destroyed the
+host, I dare not look Trojan men and Trojan women in the face,
+lest a worse man should say, 'Hector has ruined us by his
+self-confidence.' Surely it would be better for me to return
+after having fought Achilles and slain him, or to die gloriously
+here before the city. What, again, if I were to lay down my
+shield and helmet, lean my spear against the wall and go straight
+up to noble Achilles? What if I were to promise to give up Helen,
+who was the fountainhead of all this war, and all the treasure
+that Alexandrus brought with him in his ships to Troy, aye, and
+to let the Achaeans divide the half of everything that the city
+contains among themselves? I might make the Trojans, by the
+mouths of their princes, take a solemn oath that they would hide
+nothing, but would divide into two shares all that is within the
+city--but why argue with myself in this way? Were I to go up to
+him he would show me no kind of mercy; he would kill me then and
+there as easily as though I were a woman, when I had off my
+armour. There is no parleying with him from some rock or oak
+tree as young men and maidens prattle with one another. Better
+fight him at once, and learn to which of us Jove will vouchsafe
+victory."
+</para>
+
+<para>
+Thus did he stand and ponder, but Achilles came up to him as it
+were Mars himself, plumed lord of battle. From his right shoulder
+he brandished his terrible spear of Pelian ash, and the bronze
+gleamed around him like flashing fire or the rays of the rising
+sun. Fear fell upon Hector as he beheld him, and he dared not
+stay longer where he was but fled in dismay from before the
+gates, while Achilles darted after him at his utmost speed. As a
+mountain falcon, swiftest of all birds, swoops down upon some
+cowering dove--the dove flies before him but the falcon with a
+shrill scream follows close after, resolved to have her--even so
+did Achilles make straight for Hector with all his might, while
+Hector fled under the Trojan wall as fast as his limbs could take
+him.
+</para>
+
+<para>
+On they flew along the waggon-road that ran hard by under the
+wall, past the lookout station, and past the weather-beaten wild
+fig-tree, till they came to two fair springs which feed the river
+Scamander. One of these two springs is warm, and steam rises from
+it as smoke from a burning fire, but the other even in summer is
+as cold as hail or snow, or the ice that forms on water. Here,
+hard by the springs, are the goodly washing-troughs of stone,
+where in the time of peace before the coming of the Achaeans the
+wives and fair daughters of the Trojans used to wash their
+clothes. Past these did they fly, the one in front and the other
+giving chase behind him: good was the man that fled, but better
+far was he that followed after, and swiftly indeed did they run,
+for the prize was no mere beast for sacrifice or bullock's hide,
+as it might be for a common foot-race, but they ran for the life
+of Hector. As horses in a chariot race speed round the
+turning-posts when they are running for some great prize--a
+tripod or woman--at the games in honour of some dead hero, so did
+these two run full speed three times round the city of Priam. All
+the gods watched them, and the sire of gods and men was the first
+to speak.
+</para>
+
+<para>
+"Alas," said he, "my eyes behold a man who is dear to me being
+pursued round the walls of Troy; my heart is full of pity for
+Hector, who has burned the thigh-bones of many a heifer in my
+honour, one while on the crests of many-valleyed Ida, and again
+on the citadel of Troy; and now I see noble Achilles in full
+pursuit of him round the city of Priam. What say you? Consider
+among yourselves and decide whether we shall now save him or let
+him fall, valiant though he be, before Achilles, son of Peleus."
+</para>
+
+<para>
+Then Minerva said, "Father, wielder of the lightning, lord of
+cloud and storm, what mean you? Would you pluck this mortal whose
+doom has long been decreed out of the jaws of death? Do as you
+will, but we others shall not be of a mind with you."
+</para>
+
+<para>
+And Jove answered, "My child, Trito-born, take heart. I did not
+speak in full earnest, and I will let you have your way. Do
+without let or hindrance as you are minded."
+</para>
+
+<para>
+Thus did he urge Minerva who was already eager, and down she
+darted from the topmost summits of Olympus.
+</para>
+
+<para>
+Achilles was still in full pursuit of Hector, as a hound chasing
+a fawn which he has started from its covert on the mountains, and
+hunts through glade and thicket. The fawn may try to elude him by
+crouching under cover of a bush, but he will scent her out and
+follow her up until he gets her--even so there was no escape for
+Hector from the fleet son of Peleus. Whenever he made a set to
+get near the Dardanian gates and under the walls, that his people
+might help him by showering down weapons from above, Achilles
+would gain on him and head him back towards the plain, keeping
+himself always on the city side. As a man in a dream who fails to
+lay hands upon another whom he is pursuing--the one cannot escape
+nor the other overtake--even so neither could Achilles come up
+with Hector, nor Hector break away from Achilles; nevertheless he
+might even yet have escaped death had not the time come when
+Apollo, who thus far had sustained his strength and nerved his
+running, was now no longer to stay by him. Achilles made signs to
+the Achaean host, and shook his head to show that no man was to
+aim a dart at Hector, lest another might win the glory of having
+hit him and he might himself come in second. Then, at last, as
+they were nearing the fountains for the fourth time, the father
+of all balanced his golden scales and placed a doom in each of
+them, one for Achilles and the other for Hector. As he held the
+scales by the middle, the doom of Hector fell down deep into the
+house of Hades--and then Phoebus Apollo left him. Thereon Minerva
+went close up to the son of Peleus and said, "Noble Achilles,
+favoured of heaven, we two shall surely take back to the ships a
+triumph for the Achaeans by slaying Hector, for all his lust of
+battle. Do what Apollo may as he lies grovelling before his
+father, aegis-bearing Jove, Hector cannot escape us longer. Stay
+here and take breath, while I go up to him and persuade him to
+make a stand and fight you."
+</para>
+
+<para>
+Thus spoke Minerva. Achilles obeyed her gladly, and stood still,
+leaning on his bronze-pointed ashen spear, while Minerva left him
+and went after Hector in the form and with the voice of
+Deiphobus. She came close up to him and said, "Dear brother, I
+see you are hard pressed by Achilles who is chasing you at full
+speed round the city of Priam, let us await his onset and stand
+on our defence."
+</para>
+
+<para>
+And Hector answered, "Deiphobus, you have always been dearest to
+me of all my brothers, children of Hecuba and Priam, but
+henceforth I shall rate you yet more highly, inasmuch as you have
+ventured outside the wall for my sake when all the others remain
+inside."
+</para>
+
+<para>
+Then Minerva said, "Dear brother, my father and mother went down
+on their knees and implored me, as did all my comrades, to remain
+inside, so great a fear has fallen upon them all; but I was in an
+agony of grief when I beheld you; now, therefore, let us two make
+a stand and fight, and let there be no keeping our spears in
+reserve, that we may learn whether Achilles shall kill us and
+bear off our spoils to the ships, or whether he shall fall before
+you."
+</para>
+
+<para>
+Thus did Minerva inveigle him by her cunning, and when the two
+were now close to one another great Hector was first to speak. "I
+will-no longer fly you, son of Peleus," said he, "as I have been
+doing hitherto. Three times have I fled round the mighty city of
+Priam, without daring to withstand you, but now, let me either
+slay or be slain, for I am in the mind to face you. Let us, then,
+give pledges to one another by our gods, who are the fittest
+witnesses and guardians of all covenants; let it be agreed
+between us that if Jove vouchsafes me the longer stay and I take
+your life, I am not to treat your dead body in any unseemly
+fashion, but when I have stripped you of your armour, I am to
+give up your body to the Achaeans. And do you likewise."
+</para>
+
+<para>
+Achilles glared at him and answered, "Fool, prate not to me about
+covenants. There can be no covenants between men and lions,
+wolves and lambs can never be of one mind, but hate each other
+out and out an through. Therefore there can be no understanding
+between you and me, nor may there be any covenants between us,
+till one or other shall fall and glut grim Mars with his life's
+blood. Put forth all your strength; you have need now to prove
+yourself indeed a bold soldier and man of war. You have no more
+chance, and Pallas Minerva will forthwith vanquish you by my
+spear: you shall now pay me in full for the grief you have caused
+me on account of my comrades whom you have killed in battle."
+</para>
+
+<para>
+He poised his spear as he spoke and hurled it. Hector saw it
+coming and avoided it; he watched it and crouched down so that it
+flew over his head and stuck in the ground beyond; Minerva then
+snatched it up and gave it back to Achilles without Hector's
+seeing her; Hector thereon said to the son of Peleus, "You have
+missed your aim, Achilles, peer of the gods, and Jove has not yet
+revealed to you the hour of my doom, though you made sure that he
+had done so. You were a false-tongued liar when you deemed that I
+should forget my valour and quail before you. You shall not drive
+spear into the back of a runaway--drive it, should heaven so
+grant you power, drive it into me as I make straight towards you;
+and now for your own part avoid my spear if you can--would that
+you might receive the whole of it into your body; if you were
+once dead the Trojans would find the war an easier matter, for it
+is you who have harmed them most."
+</para>
+
+<para>
+He poised his spear as he spoke and hurled it. His aim was true
+for he hit the middle of Achilles' shield, but the spear
+rebounded from it, and did not pierce it. Hector was angry when
+he saw that the weapon had sped from his hand in vain, and stood
+there in dismay for he had no second spear. With a loud cry he
+called Deiphobus and asked him for one, but there was no man;
+then he saw the truth and said to himself, "Alas! the gods have
+lured me on to my destruction. I deemed that the hero Deiphobus
+was by my side, but he is within the wall, and Minerva has
+inveigled me; death is now indeed exceedingly near at hand and
+there is no way out of it--for so Jove and his son Apollo the
+far-darter have willed it, though heretofore they have been ever
+ready to protect me. My doom has come upon me; let me not then
+die ingloriously and without a struggle, but let me first do some
+great thing that shall be told among men hereafter."
+</para>
+
+<para>
+As he spoke he drew the keen blade that hung so great and strong
+by his side, and gathering himself together be sprang on Achilles
+like a soaring eagle which swoops down from the clouds on to some
+lamb or timid hare--even so did Hector brandish his sword and
+spring upon Achilles. Achilles mad with rage darted towards him,
+with his wondrous shield before his breast, and his gleaming
+helmet, made with four layers of metal, nodding fiercely forward.
+The thick tresses of gold with which Vulcan had crested the
+helmet floated round it, and as the evening star that shines
+brighter than all others through the stillness of night, even
+such was the gleam of the spear which Achilles poised in his
+right hand, fraught with the death of noble Hector. He eyed his
+fair flesh over and over to see where he could best wound it, but
+all was protected by the goodly armour of which Hector had
+spoiled Patroclus after he had slain him, save only the throat
+where the collar-bones divide the neck from the shoulders, and
+this is a most deadly place: here then did Achilles strike him as
+he was coming on towards him, and the point of his spear went
+right through the fleshy part of the neck, but it did not sever
+his windpipe so that he could still speak. Hector fell headlong,
+and Achilles vaunted over him saying, "Hector, you deemed that
+you should come off scatheless when you were spoiling Patroclus,
+and recked not of myself who was not with him. Fool that you
+were: for I, his comrade, mightier far than he, was still left
+behind him at the ships, and now I have laid you low. The
+Achaeans shall give him all due funeral rites, while dogs and
+vultures shall work their will upon yourself."
+</para>
+
+<para>
+Then Hector said, as the life ebbed out of him, "I pray you by
+your life and knees, and by your parents, let not dogs devour me
+at the ships of the Achaeans, but accept the rich treasure of
+gold and bronze which my father and mother will offer you, and
+send my body home, that the Trojans and their wives may give me
+my dues of fire when I am dead."
+</para>
+
+<para>
+Achilles glared at him and answered, "Dog, talk not to me neither
+of knees nor parents; would that I could be as sure of being able
+to cut your flesh into pieces and eat it raw, for the ill you
+have done me, as I am that nothing shall save you from the
+dogs--it shall not be, though they bring ten or twenty-fold
+ransom and weigh it out for me on the spot, with promise of yet
+more hereafter. Though Priam son of Dardanus should bid them
+offer me your weight in gold, even so your mother shall never lay
+you out and make lament over the son she bore, but dogs and
+vultures shall eat you utterly up."
+</para>
+
+<para>
+Hector with his dying breath then said, "I know you what you are,
+and was sure that I should not move you, for your heart is hard
+as iron; look to it that I bring not heaven's anger upon you on
+the day when Paris and Phoebus Apollo, valiant though you be,
+shall slay you at the Scaean gates."
+</para>
+
+<para>
+When he had thus said the shrouds of death enfolded him, whereon
+his soul went out of him and flew down to the house of Hades,
+lamenting its sad fate that it should enjoy youth and strength no
+longer. But Achilles said, speaking to the dead body, "Die; for
+my part I will accept my fate whensoever Jove and the other gods
+see fit to send it."
+</para>
+
+<para>
+As he spoke he drew his spear from the body and set it on one
+side; then he stripped the blood-stained armour from Hector's
+shoulders while the other Achaeans came running up to view his
+wondrous strength and beauty; and no one came near him without
+giving him a fresh wound. Then would one turn to his neighbour
+and say, "It is easier to handle Hector now than when he was
+flinging fire on to our ships" and as he spoke he would thrust
+his spear into him anew.
+</para>
+
+<para>
+When Achilles had done spoiling Hector of his armour, he stood
+among the Argives and said, "My friends, princes and counsellors
+of the Argives, now that heaven has vouchsafed us to overcome
+this man, who has done us more hurt than all the others together,
+consider whether we should not attack the city in force, and
+discover in what mind the Trojans may be. We should thus learn
+whether they will desert their city now that Hector has fallen,
+or will still hold out even though he is no longer living. But
+why argue with myself in this way, while Patroclus is still lying
+at the ships unburied, and unmourned--he whom I can never forget
+so long as I am alive and my strength fails not? Though men
+forget their dead when once they are within the house of Hades,
+yet not even there will I forget the comrade whom I have lost.
+Now, therefore, Achaean youths, let us raise the song of victory
+and go back to the ships taking this man along with us; for we
+have achieved a mighty triumph and have slain noble Hector to
+whom the Trojans prayed throughout their city as though he were a
+god."
+</para>
+
+<para>
+On this he treated the body of Hector with contumely: he pierced
+the sinews at the back of both his feet from heel to ancle and
+passed thongs of ox-hide through the slits he had made: thus he
+made the body fast to his chariot, letting the head trail upon
+the ground. Then when he had put the goodly armour on the chariot
+and had himself mounted, he lashed his horses on and they flew
+forward nothing loth. The dust rose from Hector as he was being
+dragged along, his dark hair flew all abroad, and his head once
+so comely was laid low on earth, for Jove had now delivered him
+into the hands of his foes to do him outrage in his own land.
+</para>
+
+<para>
+Thus was the head of Hector being dishonoured in the dust. His
+mother tore her hair, and flung her veil from her with a loud cry
+as she looked upon her son. His father made piteous moan, and
+throughout the city the people fell to weeping and wailing. It
+was as though the whole of frowning Ilius was being smirched with
+fire. Hardly could the people hold Priam back in his hot haste to
+rush without the gates of the city. He grovelled in the mire and
+besought them, calling each one of them by his name. "Let be, my
+friends," he cried, "and for all your sorrow, suffer me to go
+single-handed to the ships of the Achaeans. Let me beseech this
+cruel and terrible man, if maybe he will respect the feeling of
+his fellow-men, and have compassion on my old age. His own father
+is even such another as myself--Peleus, who bred him and reared
+him to be the bane of us Trojans, and of myself more than of all
+others. Many a son of mine has he slain in the flower of his
+youth, and yet, grieve for these as I may, I do so for one--
+Hector--more than for them all, and the bitterness of my sorrow
+will bring me down to the house of Hades. Would that he had died
+in my arms, for so both his ill-starred mother who bore him, and
+myself, should have had the comfort of weeping and mourning over
+him."
+</para>
+
+<para>
+Thus did he speak with many tears, and all the people of the city
+joined in his lament. Hecuba then raised the cry of wailing among
+the Trojans. "Alas, my son," she cried, "what have I left to live
+for now that you are no more? Night and day did I glory in you
+throughout the city, for you were a tower of strength to all in
+Troy, and both men and women alike hailed you as a god. So long
+as you lived you were their pride, but now death and destruction
+have fallen upon you."
+</para>
+
+<para>
+Hector's wife had as yet heard nothing, for no one had come to
+tell her that her husband had remained without the gates. She was
+at her loom in an inner part of the house, weaving a double
+purple web, and embroidering it with many flowers. She told her
+maids to set a large tripod on the fire, so as to have a warm
+bath ready for Hector when he came out of battle; poor woman, she
+knew not that he was now beyond the reach of baths, and that
+Minerva had laid him low by the hands of Achilles. She heard the
+cry coming as from the wall, and trembled in every limb; the
+shuttle fell from her hands, and again she spoke to her
+waiting-women. "Two of you," she said, "come with me that I may
+learn what it is that has befallen; I heard the voice of my
+husband's honoured mother; my own heart beats as though it would
+come into my mouth and my limbs refuse to carry me; some great
+misfortune for Priam's children must be at hand. May I never live
+to hear it, but I greatly fear that Achilles has cut off the
+retreat of brave Hector and has chased him on to the plain where
+he was singlehanded; I fear he may have put an end to the
+reckless daring which possessed my husband, who would never
+remain with the body of his men, but would dash on far in front,
+foremost of them all in valour."
+</para>
+
+<para>
+Her heart beat fast, and as she spoke she flew from the house
+like a maniac, with her waiting-women following after. When she
+reached the battlements and the crowd of people, she stood
+looking out upon the wall, and saw Hector being borne away in
+front of the city--the horses dragging him without heed or care
+over the ground towards the ships of the Achaeans. Her eyes were
+then shrouded as with the darkness of night and she fell fainting
+backwards. She tore the attiring from her head and flung it from
+her, the frontlet and net with its plaited band, and the veil
+which golden Venus had given her on the day when Hector took her
+with him from the house of Eetion, after having given countless
+gifts of wooing for her sake. Her husband's sisters and the wives
+of his brothers crowded round her and supported her, for she was
+fain to die in her distraction; when she again presently breathed
+and came to herself, she sobbed and made lament among the Trojans
+saying, "Woe is me, O Hector; woe, indeed, that to share a common
+lot we were born, you at Troy in the house of Priam, and I at
+Thebes under the wooded mountain of Placus in the house of Eetion
+who brought me up when I was a child--ill-starred sire of an
+ill-starred daughter--would that he had never begotten me. You
+are now going into the house of Hades under the secret places of
+the earth, and you leave me a sorrowing widow in your house. The
+child, of whom you and I are the unhappy parents, is as yet a
+mere infant. Now that you are gone, O Hector, you can do nothing
+for him nor he for you. Even though he escape the horrors of this
+woeful war with the Achaeans, yet shall his life henceforth be
+one of labour and sorrow, for others will seize his lands. The
+day that robs a child of his parents severs him from his own
+kind; his head is bowed, his cheeks are wet with tears, and he
+will go about destitute among the friends of his father, plucking
+one by the cloak and another by the shirt. Some one or other of
+these may so far pity him as to hold the cup for a moment towards
+him and let him moisten his lips, but he must not drink enough to
+wet the roof of his mouth; then one whose parents are alive will
+drive him from the table with blows and angry words. 'Out with
+you,' he will say, 'you have no father here,' and the child will
+go crying back to his widowed mother--he, Astyanax, who erewhile
+would sit upon his father's knees, and have none but the
+daintiest and choicest morsels set before him. When he had played
+till he was tired and went to sleep, he would lie in a bed, in
+the arms of his nurse, on a soft couch, knowing neither want nor
+care, whereas now that he has lost his father his lot will be
+full of hardship--he, whom the Trojans name Astyanax, because
+you, O Hector, were the only defence of their gates and
+battlements. The wriggling writhing worms will now eat you at the
+ships, far from your parents, when the dogs have glutted
+themselves upon you. You will lie naked, although in your house
+you have fine and goodly raiment made by hands of women. This
+will I now burn; it is of no use to you, for you can never again
+wear it, and thus you will have respect shown you by the Trojans
+both men and women."
+</para>
+
+<para>
+In such wise did she cry aloud amid her tears, and the women
+joined in her lament.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XXIII</title>
+
+<para>
+THUS did they make their moan throughout the city, while the
+Achaeans when they reached the Hellespont went back every man to
+his own ship. But Achilles would not let the Myrmidons go, and
+spoke to his brave comrades saying, "Myrmidons, famed horsemen
+and my own trusted friends, not yet, forsooth, let us unyoke, but
+with horse and chariot draw near to the body and mourn Patroclus,
+in due honour to the dead. When we have had full comfort of
+lamentation we will unyoke our horses and take supper all of us
+here."
+</para>
+
+<para>
+On this they all joined in a cry of wailing and Achilles led them
+in their lament. Thrice did they drive their chariots all
+sorrowing round the body, and Thetis stirred within them a still
+deeper yearning. The sands of the seashore and the men's armour
+were wet with their weeping, so great a minister of fear was he
+whom they had lost. Chief in all their mourning was the son of
+Peleus: he laid his bloodstained hand on the breast of his
+friend. "Fare well," he cried, "Patroclus, even in the house of
+Hades. I will now do all that I erewhile promised you; I will
+drag Hector hither and let dogs devour him raw; twelve noble sons
+of Trojans will I also slay before your pyre to avenge you."
+</para>
+
+<para>
+As he spoke he treated the body of noble Hector with contumely,
+laying it at full length in the dust beside the bier of
+Patroclus. The others then put off every man his armour, took the
+horses from their chariots, and seated themselves in great
+multitude by the ship of the fleet descendant of Aeacus, who
+thereon feasted them with an abundant funeral banquet. Many a
+goodly ox, with many a sheep and bleating goat did they butcher
+and cut up; many a tusked boar moreover, fat and well-fed, did
+they singe and set to roast in the flames of Vulcan; and rivulets
+of blood flowed all round the place where the body was lying.
+</para>
+
+<para>
+Then the princes of the Achaeans took the son of Peleus to
+Agamemnon, but hardly could they persuade him to come with them,
+so wroth was he for the death of his comrade. As soon as they
+reached Agamemnon's tent they told the serving-men to set a large
+tripod over the fire in case they might persuade the son of
+Peleus to wash the clotted gore from this body, but he denied
+them sternly, and swore it with a solemn oath, saying, "Nay, by
+King Jove, first and mightiest of all gods, it is not meet that
+water should touch my body, till I have laid Patroclus on the
+flames, have built him a barrow, and shaved my head--for so long
+as I live no such second sorrow shall ever draw nigh me. Now,
+therefore, let us do all that this sad festival demands, but at
+break of day, King Agamemnon, bid your men bring wood, and
+provide all else that the dead may duly take into the realm of
+darkness; the fire shall thus burn him out of our sight the
+sooner, and the people shall turn again to their own labours."
+</para>
+
+<para>
+Thus did he speak, and they did even as he had said. They made
+haste to prepare the meal, they ate, and every man had his full
+share so that all were satisfied. As soon as they had had enough
+to eat and drink, the others went to their rest each in his own
+tent, but the son of Peleus lay grieving among his Myrmidons by
+the shore of the sounding sea, in an open place where the waves
+came surging in one after another. Here a very deep slumber took
+hold upon him and eased the burden of his sorrows, for his limbs
+were weary with chasing Hector round windy Ilius. Presently the
+sad spirit of Patroclus drew near him, like what he had been in
+stature, voice, and the light of his beaming eyes, clad, too, as
+he had been clad in life. The spirit hovered over his head and
+said--
+</para>
+
+<para>
+"You sleep, Achilles, and have forgotten me; you loved me living,
+but now that I am dead you think for me no further. Bury me with
+all speed that I may pass the gates of Hades; the ghosts, vain
+shadows of men that can labour no more, drive me away from them;
+they will not yet suffer me to join those that are beyond the
+river, and I wander all desolate by the wide gates of the house
+of Hades. Give me now your hand I pray you, for when you have
+once given me my dues of fire, never shall I again come forth out
+of the house of Hades. Nevermore shall we sit apart and take
+sweet counsel among the living; the cruel fate which was my
+birth-right has yawned its wide jaws around me--nay, you too
+Achilles, peer of gods, are doomed to die beneath the wall of the
+noble Trojans.
+</para>
+
+<para>
+"One prayer more will I make you, if you will grant it; let not
+my bones be laid apart from yours, Achilles, but with them; even
+as we were brought up together in your own home, what time
+Menoetius brought me to you as a child from Opoeis because by a
+sad spite I had killed the son of Amphidamas--not of set purpose,
+but in childish quarrel over the dice. The knight Peleus took me
+into his house, entreated me kindly, and named me to be your
+squire; therefore let our bones lie in but a single urn, the
+two-handled golden vase given to you by your mother."
+</para>
+
+<para>
+And Achilles answered, "Why, true heart, are you come hither to
+lay these charges upon me? will of my own self do all as you have
+bidden me. Draw closer to me, let us once more throw our arms
+around one another, and find sad comfort in the sharing of our
+sorrows."
+</para>
+
+<para>
+He opened his arms towards him as he spoke and would have clasped
+him in them, but there was nothing, and the spirit vanished as a
+vapour, gibbering and whining into the earth. Achilles sprang to
+his feet, smote his two hands, and made lamentation saying, "Of a
+truth even in the house of Hades there are ghosts and phantoms
+that have no life in them; all night long the sad spirit of
+Patroclus has hovered over head making piteous moan, telling me
+what I am to do for him, and looking wondrously like himself."
+</para>
+
+<para>
+Thus did he speak and his words set them all weeping and mourning
+about the poor dumb dead, till rosy-fingered morn appeared. Then
+King Agamemnon sent men and mules from all parts of the camp, to
+bring wood, and Meriones, squire to Idomeneus, was in charge over
+them. They went out with woodmen's axes and strong ropes in their
+hands, and before them went the mules. Up hill and down dale did
+they go, by straight ways and crooked, and when they reached the
+heights of many-fountained Ida, they laid their axes to the roots
+of many a tall branching oak that came thundering down as they
+felled it. They split the trees and bound them behind the mules,
+which then wended their way as they best could through the thick
+brushwood on to the plain. All who had been cutting wood bore
+logs, for so Meriones squire to Idomeneus had bidden them, and
+they threw them down in a line upon the seashore at the place
+where Achilles would make a mighty monument for Patroclus and for
+himself.
+</para>
+
+<para>
+When they had thrown down their great logs of wood over the whole
+ground, they stayed all of them where they were, but Achilles
+ordered his brave Myrmidons to gird on their armour, and to yoke
+each man his horses; they therefore rose, girded on their armour
+and mounted each his chariot--they and their charioteers with
+them. The chariots went before, and they that were on foot
+followed as a cloud in their tens of thousands after. In the
+midst of them his comrades bore Patroclus and covered him with
+the locks of their hair which they cut off and threw upon his
+body. Last came Achilles with his head bowed for sorrow, so noble
+a comrade was he taking to the house of Hades.
+</para>
+
+<para>
+When they came to the place of which Achilles had told them they
+laid the body down and built up the wood. Achilles then bethought
+him of another matter. He went a space away from the pyre, and
+cut off the yellow lock which he had let grow for the river
+Spercheius. He looked all sorrowfully out upon the dark sea, and
+said, "Spercheius, in vain did my father Peleus vow to you that
+when I returned home to my loved native land I should cut off
+this lock and offer you a holy hecatomb; fifty she-goats was I to
+sacrifice to you there at your springs, where is your grove and
+your altar fragrant with burnt-offerings. Thus did my father vow,
+but you have not fulfilled his prayer; now, therefore, that I
+shall see my home no more, I give this lock as a keepsake to the
+hero Patroclus."
+</para>
+
+<para>
+As he spoke he placed the lock in the hands of his dear comrade,
+and all who stood by were filled with yearning and lamentation.
+The sun would have gone down upon their mourning had not Achilles
+presently said to Agamemnon, "Son of Atreus, for it is to you
+that the people will give ear, there is a time to mourn and a
+time to cease from mourning; bid the people now leave the pyre
+and set about getting their dinners: we, to whom the dead is
+dearest, will see to what is wanted here, and let the other
+princes also stay by me."
+</para>
+
+<para>
+When King Agamemnon heard this he dismissed the people to their
+ships, but those who were about the dead heaped up wood and built
+a pyre a hundred feet this way and that; then they laid the dead
+all sorrowfully upon the top of it. They flayed and dressed many
+fat sheep and oxen before the pyre, and Achilles took fat from
+all of them and wrapped the body therein from head to foot,
+heaping the flayed carcases all round it. Against the bier he
+leaned two-handled jars of honey and unguents; four proud horses
+did he then cast upon the pyre, groaning the while he did so. The
+dead hero had had house-dogs; two of them did Achilles slay and
+threw upon the pyre; he also put twelve brave sons of noble
+Trojans to the sword and laid them with the rest, for he was full
+of bitterness and fury. Then he committed all to the resistless
+and devouring might of the fire; he groaned aloud and called on
+his dead comrade by name. "Fare well," he cried, "Patroclus, even
+in the house of Hades; I am now doing all that I have promised
+you. Twelve brave sons of noble Trojans shall the flames consume
+along with yourself, but dogs, not fire, shall devour the flesh
+of Hector son of Priam."
+</para>
+
+<para>
+Thus did he vaunt, but the dogs came not about the body of
+Hector, for Jove's daughter Venus kept them off him night and
+day, and anointed him with ambrosial oil of roses that his flesh
+might not be torn when Achilles was dragging him about. Phoebus
+Apollo moreover sent a dark cloud from heaven to earth, which
+gave shade to the whole place where Hector lay, that the heat of
+the sun might not parch his body.
+</para>
+
+<para>
+Now the pyre about dead Patroclus would not kindle. Achilles
+therefore bethought him of another matter; he went apart and
+prayed to the two winds Boreas and Zephyrus vowing them goodly
+offerings. He made them many drink-offerings from the golden cup
+and besought them to come and help him that the wood might make
+haste to kindle and the dead bodies be consumed. Fleet Iris heard
+him praying and started off to fetch the winds. They were holding
+high feast in the house of boisterous Zephyrus when Iris came
+running up to the stone threshold of the house and stood there,
+but as soon as they set eyes on her they all came towards her and
+each of them called her to him, but Iris would not sit down. "I
+cannot stay," she said, "I must go back to the streams of Oceanus
+and the land of the Ethiopians who are offering hecatombs to the
+immortals, and I would have my share; but Achilles prays that
+Boreas and shrill Zephyrus will come to him, and he vows them
+goodly offerings; he would have you blow upon the pyre of
+Patroclus for whom all the Achaeans are lamenting."
+</para>
+
+<para>
+With this she left them, and the two winds rose with a cry that
+rent the air and swept the clouds before them. They blew on and
+on until they came to the sea, and the waves rose high beneath
+them, but when they reached Troy they fell upon the pyre till the
+mighty flames roared under the blast that they blew. All night
+long did they blow hard and beat upon the fire, and all night
+long did Achilles grasp his double cup, drawing wine from a
+mixing-bowl of gold, and calling upon the spirit of dead
+Patroclus as he poured it upon the ground until the earth was
+drenched. As a father mourns when he is burning the bones of his
+bridegroom son whose death has wrung the hearts of his parents,
+even so did Achilles mourn while burning the body of his comrade,
+pacing round the bier with piteous groaning and lamentation.
+</para>
+
+<para>
+At length as the Morning Star was beginning to herald the light
+which saffron-mantled Dawn was soon to suffuse over the sea, the
+flames fell and the fire began to die. The winds then went home
+beyond the Thracian sea, which roared and boiled as they swept
+over it. The son of Peleus now turned away from the pyre and lay
+down, overcome with toil, till he fell into a sweet slumber.
+Presently they who were about the son of Atreus drew near in a
+body, and roused him with the noise and tramp of their coming. He
+sat upright and said, "Son of Atreus, and all other princes of
+the Achaeans, first pour red wine everywhere upon the fire and
+quench it; let us then gather the bones of Patroclus son of
+Menoetius, singling them out with care; they are easily found,
+for they lie in the middle of the pyre, while all else, both men
+and horses, has been thrown in a heap and burned at the outer
+edge. We will lay the bones in a golden urn, in two layers of
+fat, against the time when I shall myself go down into the house
+of Hades. As for the barrow, labour not to raise a great one now,
+but such as is reasonable. Afterwards, let those Achaeans who may
+be left at the ships when I am gone, build it both broad and
+high."
+</para>
+
+<para>
+Thus he spoke and they obeyed the word of the son of Peleus.
+First they poured red wine upon the thick layer of ashes and
+quenched the fire. With many tears they singled out the whitened
+bones of their loved comrade and laid them within a golden urn in
+two layers of fat: they then covered the urn with a linen cloth
+and took it inside the tent. They marked off the circle where the
+barrow should be, made a foundation for it about the pyre, and
+forthwith heaped up the earth. When they had thus raised a mound
+they were going away, but Achilles stayed the people and made
+them sit in assembly. He brought prizes from the
+ships--cauldrons, tripods, horses and mules, noble oxen, women
+with fair girdles, and swart iron.
+</para>
+
+<para>
+The first prize he offered was for the chariot races--a woman
+skilled in all useful arts, and a three-legged cauldron that had
+ears for handles, and would hold twenty-two measures. This was
+for the man who came in first. For the second there was a
+six-year old mare, unbroken, and in foal to a he-ass; the third
+was to have a goodly cauldron that had never yet been on the
+fire; it was still bright as when it left the maker, and would
+hold four measures. The fourth prize was two talents of gold, and
+the fifth a two-handled urn as yet unsoiled by smoke. Then he
+stood up and spoke among the Argives saying--
+</para>
+
+<para>
+"Son of Atreus, and all other Achaeans, these are the prizes that
+lie waiting the winners of the chariot races. At any other time I
+should carry off the first prize and take it to my own tent; you
+know how far my steeds excel all others--for they are immortal;
+Neptune gave them to my father Peleus, who in his turn gave them
+to myself; but I shall hold aloof, I and my steeds that have lost
+their brave and kind driver, who many a time has washed them in
+clear water and anointed their manes with oil. See how they stand
+weeping here, with their manes trailing on the ground in the
+extremity of their sorrow. But do you others set yourselves in
+order throughout the host, whosoever has confidence in his horses
+and in the strength of his chariot."
+</para>
+
+<para>
+Thus spoke the son of Peleus and the drivers of chariots
+bestirred themselves. First among them all uprose Eumelus, king
+of men, son of Admetus, a man excellent in horsemanship. Next to
+him rose mighty Diomed son of Tydeus; he yoked the Trojan horses
+which he had taken from Aeneas, when Apollo bore him out of the
+fight. Next to him, yellow-haired Menelaus son of Atreus rose and
+yoked his fleet horses, Agamemnon's mare Aethe, and his own horse
+Podargus. The mare had been given to Agamemnon by Echepolus son
+of Anchises, that he might not have to follow him to Ilius, but
+might stay at home and take his ease; for Jove had endowed him
+with great wealth and he lived in spacious Sicyon. This mare, all
+eager for the race, did Menelaus put under the yoke.
+</para>
+
+<para>
+Fourth in order Antilochus, son to noble Nestor son of Neleus,
+made ready his horses. These were bred in Pylos, and his father
+came up to him to give him good advice of which, however, he
+stood in but little need. "Antilochus," said Nestor, "you are
+young, but Jove and Neptune have loved you well, and have made
+you an excellent horseman. I need not therefore say much by way
+of instruction. You are skilful at wheeling your horses round the
+post, but the horses themselves are very slow, and it is this
+that will, I fear, mar your chances. The other drivers know less
+than you do, but their horses are fleeter; therefore, my dear
+son, see if you cannot hit upon some artifice whereby you may
+insure that the prize shall not slip through your fingers. The
+woodman does more by skill than by brute force; by skill the
+pilot guides his storm-tossed barque over the sea, and so by
+skill one driver can beat another. If a man go wide in rounding
+this way and that, whereas a man who knows what he is doing may
+have worse horses, but he will keep them well in hand when he
+sees the doubling-post; he knows the precise moment at which to
+pull the rein, and keeps his eye well on the man in front of him.
+I will give you this certain token which cannot escape your
+notice. There is a stump of a dead tree--oak or pine as it may
+be--some six feet above the ground, and not yet rotted away by
+rain; it stands at the fork of the road; it has two white stones
+set one on each side, and there is a clear course all round it.
+It may have been a monument to some one long since dead, or it
+may have been used as a doubling-post in days gone by; now,
+however, it has been fixed on by Achilles as the mark round which
+the chariots shall turn; hug it as close as you can, but as you
+stand in your chariot lean over a little to the left; urge on
+your right-hand horse with voice and lash, and give him a loose
+rein, but let the left-hand horse keep so close in, that the nave
+of your wheel shall almost graze the post; but mind the stone, or
+you will wound your horses and break your chariot in pieces,
+which would be sport for others but confusion for yourself.
+Therefore, my dear son, mind well what you are about, for if you
+can be first to round the post there is no chance of any one
+giving you the go-by later, not even though you had Adrestus's
+horse Arion behind you--a horse which is of divine race--or those
+of Laomedon, which are the noblest in this country."
+</para>
+
+<para>
+When Nestor had made an end of counselling his son he sat down in
+his place, and fifth in order Meriones got ready his horses.
+They then all mounted their chariots and cast lots. Achilles
+shook the helmet, and the lot of Antilochus son of Nestor fell
+out first; next came that of King Eumelus, and after his, those
+of Menelaus son of Atreus and of Meriones. The last place fell to
+the lot of Diomed son of Tydeus, who was the best man of them
+all. They took their places in line; Achilles showed them the
+doubling-post round which they were to turn, some way off upon
+the plain; here he stationed his father's follower Phoenix as
+umpire, to note the running, and report truly.
+</para>
+
+<para>
+At the same instant they all of them lashed their horses, struck
+them with the reins, and shouted at them with all their might.
+They flew full speed over the plain away from the ships, the dust
+rose from under them as it were a cloud or whirlwind, and their
+manes were all flying in the wind. At one moment the chariots
+seemed to touch the ground, and then again they bounded into the
+air; the drivers stood erect, and their hearts beat fast and
+furious in their lust of victory. Each kept calling on his
+horses, and the horses scoured the plain amid the clouds of dust
+that they raised.
+</para>
+
+<para>
+It was when they were doing the last part of the course on their
+way back towards the sea that their pace was strained to the
+utmost and it was seen what each could do. The horses of the
+descendant of Pheres now took the lead, and close behind them
+came the Trojan stallions of Diomed. They seemed as if about to
+mount Eumelus's chariot, and he could feel their warm breath on
+his back and on his broad shoulders, for their heads were close
+to him as they flew over the course. Diomed would have now passed
+him, or there would have been a dead heat, but Phoebus Apollo to
+spite him made him drop his whip. Tears of anger fell from his
+eyes as he saw the mares going on faster than ever, while his own
+horses lost ground through his having no whip. Minerva saw the
+trick which Apollo had played the son of Tydeus, so she brought
+him his whip and put spirit into his horses; moreover she went
+after the son of Admetus in a rage and broke his yoke for him;
+the mares went one to one side of the course, and the other to
+the other, and the pole was broken against the ground. Eumelus
+was thrown from his chariot close to the wheel; his elbows,
+mouth, and nostrils were all torn, and his forehead was bruised
+above his eyebrows; his eyes filled with tears and he could find
+no utterance. But the son of Tydeus turned his horses aside and
+shot far ahead, for Minerva put fresh strength into them and
+covered Diomed himself with glory.
+</para>
+
+<para>
+Menelaus son of Atreus came next behind him, but Antilochus
+called to his father's horses. "On with you both," he cried, "and
+do your very utmost. I do not bid you try to beat the steeds of
+the son of Tydeus, for Minerva has put running into them, and has
+covered Diomed with glory; but you must overtake the horses of
+the son of Atreus and not be left behind, or Aethe who is so
+fleet will taunt you. Why, my good fellows, are you lagging? I
+tell you, and it shall surely be--Nestor will keep neither of
+you, but will put both of you to the sword, if we win any the
+worse a prize through your carelessness. Fly after them at your
+utmost speed; I will hit on a plan for passing them in a narrow
+part of the way, and it shall not fail me."
+</para>
+
+<para>
+They feared the rebuke of their master, and for a short space
+went quicker. Presently Antilochus saw a narrow place where the
+road had sunk. The ground was broken, for the winter's rain had
+gathered and had worn the road so that the whole place was
+deepened. Menelaus was making towards it so as to get there
+first, for fear of a foul, but Antilochus turned his horses out
+of the way, and followed him a little on one side. The son of
+Atreus was afraid and shouted out, "Antilochus, you are driving
+recklessly; rein in your horses; the road is too narrow here, it
+will be wider soon, and you can pass me then; if you foul my
+chariot you may bring both of us to a mischief."
+</para>
+
+<para>
+But Antilochus plied his whip, and drove faster, as though he had
+not heard him. They went side by side for about as far as a young
+man can hurl a disc from his shoulder when he is trying his
+strength, and then Menelaus's mares drew behind, for he left off
+driving for fear the horses should foul one another and upset the
+chariots; thus, while pressing on in quest of victory, they might
+both come headlong to the ground. Menelaus then upbraided
+Antilochus and said, "There is no greater trickster living than
+you are; go, and bad luck go with you; the Achaeans say not well
+that you have understanding, and come what may you shall not bear
+away the prize without sworn protest on my part."
+</para>
+
+<para>
+Then he called on his horses and said to them, "Keep your pace,
+and slacken not; the limbs of the other horses will weary sooner
+than yours, for they are neither of them young."
+</para>
+
+<para>
+The horses feared the rebuke of their master, and went faster, so
+that they were soon nearly up with the others.
+</para>
+
+<para>
+Meanwhile the Achaeans from their seats were watching how the
+horses went, as they scoured the plain amid clouds of their own
+dust. Idomeneus captain of the Cretans was first to make out the
+running, for he was not in the thick of the crowd, but stood on
+the most commanding part of the ground. The driver was a long way
+off, but Idomeneus could hear him shouting, and could see the
+foremost horse quite plainly--a chestnut with a round white star,
+like the moon, on its forehead. He stood up and said among the
+Argives, "My friends, princes and counsellors of the Argives, can
+you see the running as well as I can? There seems to be another
+pair in front now, and another driver; those that led off at the
+start must have been disabled out on the plain. I saw them at
+first making their way round the doubling-post, but now, though I
+search the plain of Troy, I cannot find them. Perhaps the reins
+fell from the driver's hand so that he lost command of his horses
+at the doubling-post, and could not turn it. I suppose he must
+have been thrown out there, and broken his chariot, while his
+mares have left the course and gone off wildly in a panic. Come
+up and see for yourselves, I cannot make out for certain, but the
+driver seems an Aetolian by descent, ruler over the Argives,
+brave Diomed the son of Tydeus."
+</para>
+
+<para>
+Ajax the son of Oileus took him up rudely and said, "Idomeneus,
+why should you be in such a hurry to tell us all about it, when
+the mares are still so far out upon the plain? You are none of
+the youngest, nor your eyes none of the sharpest, but you are
+always laying down the law. You have no right to do so, for there
+are better men here than you are. Eumelus's horses are in front
+now, as they always have been, and he is on the chariot holding
+the reins."
+</para>
+
+<para>
+The captain of the Cretans was angry, and answered, "Ajax you are
+an excellent railer, but you have no judgement, and are wanting
+in much else as well, for you have a vile temper. I will wager
+you a tripod or cauldron, and Agamemnon son of Atreus shall
+decide whose horses are first. You will then know to your cost."
+</para>
+
+<para>
+Ajax son of Oileus was for making him an angry answer, and there
+would have been yet further brawling between them, had not
+Achilles risen in his place and said, "Cease your railing, Ajax
+and Idomeneus; is it not you would be scandalised if you saw any
+one else do the like: sit down and keep your eyes on the horses;
+they are speeding towards the winning-post and will be bere
+directly. You will then both of you know whose horses are first,
+and whose come after."
+</para>
+
+<para>
+As he was speaking, the son of Tydeus came driving in, plying his
+whip lustily from his shoulder, and his horses stepping high as
+they flew over the course. The sand and grit rained thick on the
+driver, and the chariot inlaid with gold and tin ran close behind
+his fleet horses. There was little trace of wheel-marks in the
+fine dust, and the horses came flying in at their utmost speed.
+Diomed stayed them in the middle of the crowd, and the sweat from
+their manes and chests fell in streams on to the ground.
+Forthwith he sprang from his goodly chariot, and leaned his whip
+against his horses' yoke; brave Sthenelus now lost no time, but
+at once brought on the prize, and gave the woman and the
+ear-handled cauldron to his comrades to take away. Then he
+unyoked the horses.
+</para>
+
+<para>
+Next after him came in Antilochus of the race of Neleus, who had
+passed Menelaus by a trick and not by the fleetness of his
+horses; but even so Menelaus came in as close behind him as the
+wheel is to the horse that draws both the chariot and its master.
+The end hairs of a horse's tail touch the tyre of the wheel, and
+there is never much space between wheel and horse when the
+chariot is going; Menelaus was no further than this behind
+Antilochus, though at first he had been a full disc's throw
+behind him. He had soon caught him up again, for Agamemnon's mare
+Aethe kept pulling stronger and stronger, so that if the course
+had been longer he would have passed him, and there would not
+even have been a dead heat. Idomeneus's brave squire Meriones was
+about a spear's cast behind Menelaus. His horses were slowest of
+all, and he was the worst driver. Last of them all came the son
+of Admetus, dragging his chariot and driving his horses on in
+front. When Achilles saw him he was sorry, and stood up among the
+Argives saying, "The best man is coming in last. Let us give him
+a prize for it is reasonable. He shall have the second, but the
+first must go to the son of Tydeus."
+</para>
+
+<para>
+Thus did he speak and the others all of them applauded his
+saying, and were for doing as he had said, but Nestor's son
+Antilochus stood up and claimed his rights from the son of
+Peleus. "Achilles," said he, "I shall take it much amiss if you
+do this thing; you would rob me of my prize, because you think
+Eumelus's chariot and horses were thrown out, and himself too,
+good man that he is. He should have prayed duly to the immortals;
+he would not have come in last if he had done so. If you are
+sorry for him and so choose, you have much gold in your tents,
+with bronze, sheep, cattle and horses. Take something from this
+store if you would have the Achaeans speak well of you, and give
+him a better prize even than that which you have now offered; but
+I will not give up the mare, and he that will fight me for her,
+let him come on."
+</para>
+
+<para>
+Achilles smiled as he heard this, and was pleased with
+Antilochus, who was one of his dearest comrades. So he said--
+</para>
+
+<para>
+"Antilochus, if you would have me find Eumelus another prize, I
+will give him the bronze breastplate with a rim of tin running
+all round it which I took from Asteropaeus. It will be worth much
+money to him."
+</para>
+
+<para>
+He bade his comrade Automedon bring the breastplate from his
+tent, and he did so. Achilles then gave it over to Eumelus, who
+received it gladly.
+</para>
+
+<para>
+But Menelaus got up in a rage, furiously angry with Antilochus.
+An attendant placed his staff in his hands and bade the Argives
+keep silence: the hero then addressed them. "Antilochus," said
+he, "what is this from you who have been so far blameless? You
+have made me cut a poor figure and baulked my horses by flinging
+your own in front of them, though yours are much worse than mine
+are; therefore, O princes and counsellors of the Argives, judge
+between us and show no favour, lest one of the Achaeans say,
+'Menelaus has got the mare through lying and corruption; his
+horses were far inferior to Antilochus's, but he has greater
+weight and influence.' Nay, I will determine the matter myself,
+and no man will blame me, for I shall do what is just. Come here,
+Antilochus, and stand, as our custom is, whip in hand before your
+chariot and horses; lay your hand on your steeds, and swear by
+earth-encircling Neptune that you did not purposely and
+guilefully get in the way of my horses."
+</para>
+
+<para>
+And Antilochus answered, "Forgive me; I am much younger, King
+Menelaus, than you are; you stand higher than I do and are the
+better man of the two; you know how easily young men are betrayed
+into indiscretion; their tempers are more hasty and they have
+less judgement; make due allowances therefore, and bear with me;
+I will of my own accord give up the mare that I have won, and if
+you claim any further chattel from my own possessions, I would
+rather yield it to you, at once, than fall from your good graces
+henceforth, and do wrong in the sight of heaven."
+</para>
+
+<para>
+The son of Nestor then took the mare and gave her over to
+Menelaus, whose anger was thus appeased; as when dew falls upon a
+field of ripening corn, and the lands are bristling with the
+harvest--even so, O Menelaus, was your heart made glad within
+you. He turned to Antilochus and said, "Now, Antilochus, angry
+though I have been, I can give way to you of my own free will;
+you have never been headstrong nor ill-disposed hitherto, but
+this time your youth has got the better of your judgement; be
+careful how you outwit your betters in future; no one else could
+have brought me round so easily, but your good father, your
+brother, and yourself have all of you had infinite trouble on my
+behalf; I therefore yield to your entreaty, and will give up the
+mare to you, mine though it indeed be; the people will thus see
+that I am neither harsh nor vindictive."
+</para>
+
+<para>
+With this he gave the mare over to Antilochus's comrade Noemon,
+and then took the cauldron. Meriones, who had come in fourth,
+carried off the two talents of gold, and the fifth prize, the
+two-handled urn, being unawarded, Achilles gave it to Nestor,
+going up to him among the assembled Argives and saying, "Take
+this, my good old friend, as an heirloom and memorial of the
+funeral of Patroclus--for you shall see him no more among the
+Argives. I give you this prize though you cannot win one; you can
+now neither wrestle nor fight, and cannot enter for the
+javelin-match nor foot-races, for the hand of age has been laid
+heavily upon you."
+</para>
+
+<para>
+So saying he gave the urn over to Nestor, who received it gladly
+and answered, "My son, all that you have said is true; there is
+no strength now in my legs and feet, nor can I hit out with my
+hands from either shoulder. Would that I were still young and
+strong as when the Epeans were burying King Amarynceus in
+Buprasium, and his sons offered prizes in his honour. There was
+then none that could vie with me neither of the Epeans nor the
+Pylians themselves nor the Aetolians. In boxing I overcame
+Clytomedes son of Enops, and in wrestling, Ancaeus of Pleuron who
+had come forward against me. Iphiclus was a good runner, but I
+beat him, and threw farther with my spear than either Phyleus or
+Polydorus. In chariot-racing alone did the two sons of Actor
+surpass me by crowding their horses in front of me, for they were
+angry at the way victory had gone, and at the greater part of the
+prizes remaining in the place in which they had been offered.
+They were twins, and the one kept on holding the reins, and
+holding the reins, while the other plied the whip. Such was I
+then, but now I must leave these matters to younger men; I must
+bow before the weight of years, but in those days I was eminent
+among heroes. And now, sir, go on with the funeral contests in
+honour of your comrade: gladly do I accept this urn, and my heart
+rejoices that you do not forget me but are ever mindful of my
+goodwill towards you, and of the respect due to me from the
+Achaeans. For all which may the grace of heaven be vouchsafed you
+in great abundance."
+</para>
+
+<para>
+Thereon the son of Peleus, when he had listened to all the thanks
+of Nestor, went about among the concourse of the Achaeans, and
+presently offered prizes for skill in the painful art of boxing.
+He brought out a strong mule, and made it fast in the middle of
+the crowd--a she-mule never yet broken, but six years old--when
+it is hardest of all to break them: this was for the victor, and
+for the vanquished he offered a double cup. Then he stood up and
+said among the Argives, "Son of Atreus, and all other Achaeans, I
+invite our two champion boxers to lay about them lustily and
+compete for these prizes. He to whom Apollo vouchsafes the
+greater endurance, and whom the Achaeans acknowledge as victor,
+shall take the mule back with him to his own tent, while he that
+is vanquished shall have the double cup."
+</para>
+
+<para>
+As he spoke there stood up a champion both brave and great
+stature, a skilful boxer, Epeus, son of Panopeus. He laid his
+hand on the mule and said, "Let the man who is to have the cup
+come hither, for none but myself will take the mule. I am the
+best boxer of all here present, and none can beat me. Is it not
+enough that I should fall short of you in actual fighting? Still,
+no man can be good at everything. I tell you plainly, and it
+shall come true; if any man will box with me I will bruise his
+body and break his bones; therefore let his friends stay here in
+a body and be at hand to take him away when I have done with
+him."
+</para>
+
+<para>
+They all held their peace, and no man rose save Euryalus son of
+Mecisteus, who was son of Talaus. Mecisteus went once to Thebes
+after the fall of Oedipus, to attend his funeral, and he beat all
+the people of Cadmus. The son of Tydeus was Euryalus's second,
+cheering him on and hoping heartily that he would win. First he
+put a waistband round him and then he gave him some well-cut
+thongs of ox-hide; the two men being now girt went into the
+middle of the ring, and immediately fell to; heavily indeed did
+they punish one another and lay about them with their brawny
+fists. One could hear the horrid crashing of their jaws, and they
+sweated from every pore of their skin. Presently Epeus came on
+and gave Euryalus a blow on the jaw as he was looking round;
+Euryalus could not keep his legs; they gave way under him in a
+moment and he sprang up with a bound, as a fish leaps into the
+air near some shore that is all bestrewn with sea-wrack, when
+Boreas furs the top of the waves, and then falls back into deep
+water. But noble Epeus caught hold of him and raised him up; his
+comrades also came round him and led him from the ring, unsteady
+in his gait, his head hanging on one side, and spitting great
+clots of gore. They set him down in a swoon and then went to
+fetch the double cup.
+</para>
+
+<para>
+The son of Peleus now brought out the prizes for the third
+contest and showed them to the Argives. These were for the
+painful art of wrestling. For the winner there was a great tripod
+ready for setting upon the fire, and the Achaeans valued it among
+themselves at twelve oxen. For the loser he brought out a woman
+skilled in all manner of arts, and they valued her at four oxen.
+He rose and said among the Argives, "Stand forward, you who will
+essay this contest."
+</para>
+
+<para>
+Forthwith uprose great Ajax the son of Telamon, and crafty
+Ulysses, full of wiles, rose also. The two girded themselves and
+went into the middle of the ring. They gripped each other in
+their strong hands like the rafters which some master-builder
+frames for the roof of a high house to keep the wind out. Their
+backbones cracked as they tugged at one another with their mighty
+arms--and sweat rained from them in torrents. Many a bloody weal
+sprang up on their sides and shoulders, but they kept on striving
+with might and main for victory and to win the tripod. Ulysses
+could not throw Ajax, nor Ajax him; Ulysses was too strong for
+him; but when the Achaeans began to tire of watching them, Ajax
+said to Ulysses, "Ulysses, noble son of Laertes, you shall either
+lift me, or I you, and let Jove settle it between us."
+</para>
+
+<para>
+He lifted him from the ground as he spoke, but Ulysses did not
+forget his cunning. He hit Ajax in the hollow at back of his
+knee, so that he could not keep his feet, but fell on his back
+with Ulysses lying upon his chest, and all who saw it marvelled.
+Then Ulysses in turn lifted Ajax and stirred him a little from
+the ground but could not lift him right off it, his knee sank
+under him, and the two fell side by side on the ground and were
+all begrimed with dust. They now sprang towards one another and
+were for wrestling yet a third time, but Achilles rose and stayed
+them. "Put not each other further," said he, "to such cruel
+suffering; the victory is with both alike, take each of you an
+equal prize, and let the other Achaeans now compete."
+</para>
+
+<para>
+Thus did he speak and they did even as he had said, and put on
+their shirts again after wiping the dust from off their bodies.
+</para>
+
+<para>
+The son of Peleus then offered prizes for speed in running--a
+mixing-bowl beautifully wrought, of pure silver. It would hold
+six measures, and far exceeded all others in the whole world for
+beauty; it was the work of cunning artificers in Sidon, and had
+been brought into port by Phoenicians from beyond the sea, who
+had made a present of it to Thoas. Eueneus son of Jason had given
+it to Patroclus in ransom of Priam's son Lycaon, and Achilles now
+offered it as a prize in honour of his comrade to him who should
+be the swiftest runner. For the second prize he offered a large
+ox, well fattened, while for the last there was to be half a
+talent of gold. He then rose and said among the Argives, "Stand
+forward, you who will essay this contest."
+</para>
+
+<para>
+Forthwith uprose fleet Ajax son of Oileus, with cunning Ulysses,
+and Nestor's son Antilochus, the fastest runner among all the
+youth of his time. They stood side by side and Achilles showed
+them the goal. The course was set out for them from the
+starting-post, and the son of Oileus took the lead at once, with
+Ulysses as close behind him as the shuttle is to a woman's bosom
+when she throws the woof across the warp and holds it close up to
+her; even so close behind him was Ulysses--treading in his
+footprints before the dust could settle there, and Ajax could
+feel his breath on the back of his head as he ran swiftly on. The
+Achaeans all shouted applause as they saw him straining his
+utmost, and cheered him as he shot past them; but when they were
+now nearing the end of the course Ulysses prayed inwardly to
+Minerva. "Hear me," he cried, "and help my feet, O goddess." Thus
+did he pray, and Pallas Minerva heard his prayer; she made his
+hands and his feet feel light, and when the runners were at the
+point of pouncing upon the prize, Ajax, through Minerva's spite
+slipped upon some offal that was lying there from the cattle
+which Achilles had slaughtered in honour of Patroclus, and his
+mouth and nostrils were all filled with cow dung. Ulysses
+therefore carried off the mixing-bowl, for he got before Ajax and
+came in first. But Ajax took the ox and stood with his hand on
+one of its horns, spitting the dung out of his mouth. Then he
+said to the Argives, "Alas, the goddess has spoiled my running;
+she watches over Ulysses and stands by him as though she were his
+own mother." Thus did he speak and they all of them laughed
+heartily.
+</para>
+
+<para>
+Antilochus carried off the last prize and smiled as he said to
+the bystanders, "You all see, my friends, that now too the gods
+have shown their respect for seniority. Ajax is somewhat older
+than I am, and as for Ulysses, he belongs to an earlier
+generation, but he is hale in spite of his years, and no man of
+the Achaeans can run against him save only Achilles."
+</para>
+
+<para>
+He said this to pay a compliment to the son of Peleus, and
+Achilles answered, "Antilochus, you shall not have praised me to
+no purpose; I shall give you an additional half talent of gold."
+He then gave the half talent to Antilochus, who received it
+gladly.
+</para>
+
+<para>
+Then the son of Peleus brought out the spear, helmet and shield
+that had been borne by Sarpedon, and were taken from him by
+Patroclus. He stood up and said among the Argives, "We bid two
+champions put on their armour, take their keen blades, and make
+trial of one another in the presence of the multitude; whichever
+of them can first wound the flesh of the other, cut through his
+armour, and draw blood, to him will I give this goodly Thracian
+sword inlaid with silver, which I took from Asteropaeus, but the
+armour let both hold in partnership, and I will give each of them
+a hearty meal in my own tent."
+</para>
+
+<para>
+Forthwith uprose great Ajax the son of Telamon, as also mighty
+Diomed son of Tydeus. When they had put on their armour each on
+his own side of the ring, they both went into the middle eager to
+engage, and with fire flashing from their eyes. The Achaeans
+marvelled as they beheld them, and when the two were now close up
+with one another, thrice did they spring forward and thrice try
+to strike each other in close combat. Ajax pierced Diomed's round
+shield, but did not draw blood, for the cuirass beneath the
+shield protected him; thereon the son of Tydeus from over his
+huge shield kept aiming continually at Ajax's neck with the point
+of his spear, and the Achaeans alarmed for his safety bade them
+leave off fighting and divide the prize between them. Achilles
+then gave the great sword to the son of Tydeus, with its
+scabbard, and the leathern belt with which to hang it.
+</para>
+
+<para>
+Achilles next offered the massive iron quoit which mighty Eetion
+had erewhile been used to hurl, until Achilles had slain him and
+carried it off in his ships along with other spoils. He stood up
+and said among the Argives, "Stand forward, you who would essay
+this contest. He who wins it will have a store of iron that will
+last him five years as they go rolling round, and if his fair
+fields lie far from a town his shepherd or ploughman will not
+have to make a journey to buy iron, for he will have a stock of
+it on his own premises."
+</para>
+
+<para>
+Then uprose the two mighty men Polypoetes and Leonteus, with Ajax
+son of Telamon and noble Epeus. They stood up one after the other
+and Epeus took the quoit, whirled it, and flung it from him,
+which set all the Achaeans laughing. After him threw Leonteus of
+the race of Mars. Ajax son of Telamon threw third, and sent the
+quoit beyond any mark that had been made yet, but when mighty
+Polypoetes took the quoit he hurled it as though it had been a
+stockman's stick which he sends flying about among his cattle
+when he is driving them, so far did his throw out-distance those
+of the others. All who saw it roared applause, and his comrades
+carried the prize for him and set it on board his ship.
+</para>
+
+<para>
+Achilles next offered a prize of iron for archery--ten
+double-edged axes and ten with single edges: he set up a ship's
+mast, some way off upon the sands, and with a fine string tied a
+pigeon to it by the foot; this was what they were to aim at.
+"Whoever," he said, "can hit the pigeon shall have all the axes
+and take them away with him; he who hits the string without
+hitting the bird will have taken a worse aim and shall have the
+single-edged axes."
+</para>
+
+<para>
+Then uprose King Teucer, and Meriones the stalwart squire of
+Idomeneus rose also, They cast lots in a bronze helmet and the
+lot of Teucer fell first. He let fly with his arrow forthwith,
+but he did not promise hecatombs of firstling lambs to King
+Apollo, and missed his bird, for Apollo foiled his aim; but he
+hit the string with which the bird was tied, near its foot; the
+arrow cut the string clean through so that it hung down towards
+the ground, while the bird flew up into the sky, and the Achaeans
+shouted applause. Meriones, who had his arrow ready while Teucer
+was aiming, snatched the bow out of his hand, and at once
+promised that he would sacrifice a hecatomb of firstling lambs to
+Apollo lord of the bow; then espying the pigeon high up under the
+clouds, he hit her in the middle of the wing as she was circling
+upwards; the arrow went clean through the wing and fixed itself
+in the ground at Meriones' feet, but the bird perched on the
+ship's mast hanging her head and with all her feathers drooping;
+the life went out of her, and she fell heavily from the mast.
+Meriones, therefore, took all ten double-edged axes, while Teucer
+bore off the single-edged ones to his ships.
+</para>
+
+<para>
+Then the son of Peleus brought in a spear and a cauldron that had
+never been on the fire; it was worth an ox, and was chased with a
+pattern of flowers; and those that throw the javelin stood up--to
+wit the son of Atreus, king of men Agamemnon, and Meriones,
+stalwart squire of Idomeneus. But Achilles spoke saying, "Son of
+Atreus, we know how far you excel all others both in power and in
+throwing the javelin; take the cauldron back with you to your
+ships, but if it so please you, let us give the spear to
+Meriones; this at least is what I should myself wish."
+</para>
+
+<para>
+King Agamemnon assented. So he gave the bronze spear to Meriones,
+and handed the goodly cauldron to Talthybius his esquire.
+</para>
+</chapter>
+
+<chapter>
+<title>BOOK XXIV</title>
+
+<para>
+THE assembly now broke up and the people went their ways each to
+his own ship. There they made ready their supper, and then
+bethought them of the blessed boon of sleep; but Achilles still
+wept for thinking of his dear comrade, and sleep, before whom all
+things bow, could take no hold upon him. This way and that did he
+turn as he yearned after the might and manfulness of Patroclus;
+he thought of all they had done together, and all they had gone
+through both on the field of battle and on the waves of the weary
+sea. As he dwelt on these things he wept bitterly and lay now on
+his side, now on his back, and now face downwards, till at last
+he rose and went out as one distraught to wander upon the
+seashore. Then, when he saw dawn breaking over beach and sea, he
+yoked his horses to his chariot, and bound the body of Hector
+behind it that he might drag it about. Thrice did he drag it
+round the tomb of the son of Menoetius, and then went back into
+his tent, leaving the body on the ground full length and with its
+face downwards. But Apollo would not suffer it to be disfigured,
+for he pitied the man, dead though he now was; therefore he
+shielded him with his golden aegis continually, that he might
+take no hurt while Achilles was dragging him.
+</para>
+
+<para>
+Thus shamefully did Achilles in his fury dishonour Hector; but
+the blessed gods looked down in pity from heaven, and urged
+Mercury, slayer of Argus, to steal the body. All were of this
+mind save only Juno, Neptune, and Jove's grey-eyed daughter, who
+persisted in the hate which they had ever borne towards Ilius
+with Priam and his people; for they forgave not the wrong done
+them by Alexandrus in disdaining the goddesses who came to him
+when he was in his sheepyards, and preferring her who had offered
+him a wanton to his ruin.
+</para>
+
+<para>
+When, therefore, the morning of the twelfth day had now come,
+Phoebus Apollo spoke among the immortals saying, "You gods ought
+to be ashamed of yourselves; you are cruel and hard-hearted. Did
+not Hector burn you thigh-bones of heifers and of unblemished
+goats? And now dare you not rescue even his dead body, for his
+wife to look upon, with his mother and child, his father Priam,
+and his people, who would forthwith commit him to the flames, and
+give him his due funeral rites? So, then, you would all be on the
+side of mad Achilles, who knows neither right nor ruth? He is
+like some savage lion that in the pride of his great strength and
+daring springs upon men's flocks and gorges on them. Even so has
+Achilles flung aside all pity, and all that conscience which at
+once so greatly banes yet greatly boons him that will heed it.
+man may lose one far dearer than Achilles has lost--a son, it may
+be, or a brother born from his own mother's womb; yet when he has
+mourned him and wept over him he will let him bide, for it takes
+much sorrow to kill a man; whereas Achilles, now that he has
+slain noble Hector, drags him behind his chariot round the tomb
+of his comrade. It were better of him, and for him, that he
+should not do so, for brave though he be we gods may take it ill
+that he should vent his fury upon dead clay."
+</para>
+
+<para>
+Juno spoke up in a rage. "This were well," she cried, "O lord of
+the silver bow, if you would give like honour to Hector and to
+Achilles; but Hector was mortal and suckled at a woman's breast,
+whereas Achilles is the offspring of a goddess whom I myself
+reared and brought up. I married her to Peleus, who is above
+measure dear to the immortals; you gods came all of you to her
+wedding; you feasted along with them yourself and brought your
+lyre--false, and fond of low company, that you have ever been."
+</para>
+
+<para>
+Then said Jove, "Juno, be not so bitter. Their honour shall not
+be equal, but of all that dwell in Ilius, Hector was dearest to
+the gods, as also to myself, for his offerings never failed me.
+Never was my altar stinted of its dues, nor of the
+drink-offerings and savour of sacrifice which we claim of right.
+I shall therefore permit the body of mighty Hector to be stolen;
+and yet this may hardly be without Achilles coming to know it,
+for his mother keeps night and day beside him. Let some one of
+you, therefore, send Thetis to me, and I will impart my counsel
+to her, namely that Achilles is to accept a ransom from Priam,
+and give up the body."
+</para>
+
+<para>
+On this Iris fleet as the wind went forth to carry his message.
+Down she plunged into the dark sea midway between Samos and rocky
+Imbrus; the waters hissed as they closed over her, and she sank
+into the bottom as the lead at the end of an ox-horn, that is
+sped to carry death to fishes. She found Thetis sitting in a
+great cave with the other sea-goddesses gathered round her; there
+she sat in the midst of them weeping for her noble son who was to
+fall far from his own land, on the rich plains of Troy. Iris went
+up to her and said, "Rise Thetis; Jove, whose counsels fail not,
+bids you come to him." And Thetis answered, "Why does the mighty
+god so bid me? I am in great grief, and shrink from going in and
+out among the immortals. Still, I will go, and the word that he
+may speak shall not be spoken in vain."
+</para>
+
+<para>
+The goddess took her dark veil, than which there can be no robe
+more sombre, and went forth with fleet Iris leading the way
+before her. The waves of the sea opened them a path, and when
+they reached the shore they flew up into the heavens, where they
+found the all-seeing son of Saturn with the blessed gods that
+live for ever assembled near him. Minerva gave up her seat to
+her, and she sat down by the side of father Jove. Juno then
+placed a fair golden cup in her hand, and spoke to her in words
+of comfort, whereon Thetis drank and gave her back the cup; and
+the sire of gods and men was the first to speak.
+</para>
+
+<para>
+"So, goddess," said he, "for all your sorrow, and the grief that
+I well know reigns ever in your heart, you have come hither to
+Olympus, and I will tell you why I have sent for you. This nine
+days past the immortals have been quarrelling about Achilles
+waster of cities and the body of Hector. The gods would have
+Mercury slayer of Argus steal the body, but in furtherance of our
+peace and amity henceforward, I will concede such honour to your
+son as I will now tell you. Go, then, to the host and lay these
+commands upon him; say that the gods are angry with him, and that
+I am myself more angry than them all, in that he keeps Hector at
+the ships and will not give him up. He may thus fear me and let
+the body go. At the same time I will send Iris to great Priam to
+bid him go to the ships of the Achaeans, and ransom his son,
+taking with him such gifts for Achilles as may give him
+satisfaction."
+</para>
+
+<para>
+Silver-footed Thetis did as the god had told her, and forthwith
+down she darted from the topmost summits of Olympus. She went to
+her son's tents where she found him grieving bitterly, while his
+trusty comrades round him were busy preparing their morning meal,
+for which they had killed a great woolly sheep. His mother sat
+down beside him and caressed him with her hand saying, "My son,
+how long will you keep on thus grieving and making moan? You are
+gnawing at your own heart, and think neither of food nor of
+woman's embraces; and yet these too were well, for you have no
+long time to live, and death with the strong hand of fate are
+already close beside you. Now, therefore, heed what I say, for I
+come as a messenger from Jove; he says that the gods are angry
+with you, and himself more angry than them all, in that you keep
+Hector at the ships and will not give him up. Therefore let him
+go, and accept a ransom for his body."
+</para>
+
+<para>
+And Achilles answered, "So be it. If Olympian Jove of his own
+motion thus commands me, let him that brings the ransom bear the
+body away."
+</para>
+
+<para>
+Thus did mother and son talk together at the ships in long
+discourse with one another. Meanwhile the son of Saturn sent Iris
+to the strong city of Ilius. "Go," said he, "fleet Iris, from the
+mansions of Olympus, and tell King Priam in Ilius, that he is to
+go to the ships of the Achaeans and free the body of his dear
+son. He is to take such gifts with him as shall give satisfaction
+to Achilles, and he is to go alone, with no other Trojan, save
+only some honoured servant who may drive his mules and waggon,
+and bring back the body of him whom noble Achilles has slain. Let
+him have no thought nor fear of death in his heart, for we will
+send the slayer of Argus to escort him, and bring him within the
+tent of Achilles. Achilles will not kill him nor let another do
+so, for he will take heed to his ways and sin not, and he will
+entreat a suppliant with all honourable courtesy."
+</para>
+
+<para>
+On this Iris, fleet as the wind, sped forth to deliver her
+message. She went to Priam's house, and found weeping and
+lamentation therein. His sons were seated round their father in
+the outer courtyard, and their raiment was wet with tears: the
+old man sat in the midst of them with his mantle wrapped close
+about his body, and his head and neck all covered with the filth
+which he had clutched as he lay grovelling in the mire. His
+daughters and his sons' wives went wailing about the house, as
+they thought of the many and brave men who lay dead, slain by the
+Argives. The messenger of Jove stood by Priam and spoke softly to
+him, but fear fell upon him as she did so. "Take heart," she
+said, "Priam offspring of Dardanus, take heart and fear not. I
+bring no evil tidings, but am minded well towards you. I come as
+a messenger from Jove, who though he be not near, takes thought
+for you and pities you. The lord of Olympus bids you go and
+ransom noble Hector, and take with you such gifts as shall give
+satisfaction to Achilles. You are to go alone, with no Trojan,
+save only some honoured servant who may drive your mules and
+waggon, and bring back to the city the body of him whom noble
+Achilles has slain. You are to have no thought, nor fear of
+death, for Jove will send the slayer of Argus to escort you. When
+he has brought you within Achilles' tent, Achilles will not kill
+you nor let another do so, for he will take heed to his ways and
+sin not, and he will entreat a suppliant with all honourable
+courtesy."
+</para>
+
+<para>
+Iris went her way when she had thus spoken, and Priam told his
+sons to get a mule-waggon ready, and to make the body of the
+waggon fast upon the top of its bed. Then he went down into his
+fragrant store-room, high-vaulted, and made of cedar-wood, where
+his many treasures were kept, and he called Hecuba his wife.
+"Wife," said he, "a messenger has come to me from Olympus, and
+has told me to go to the ships of the Achaeans to ransom my dear
+son, taking with me such gifts as shall give satisfaction to
+Achilles. What think you of this matter? for my own part I am
+greatly moved to pass through the camps of the Achaeans and go to
+their ships."
+</para>
+
+<para>
+His wife cried aloud as she heard him, and said, "Alas, what has
+become of that judgement for which you have been ever famous both
+among strangers and your own people? How can you venture alone to
+the ships of the Achaeans, and look into the face of him who has
+slain so many of your brave sons? You must have iron courage, for
+if the cruel savage sees you and lays hold on you, he will know
+neither respect nor pity. Let us then weep Hector from afar here
+in our own house, for when I gave him birth the threads of
+overruling fate were spun for him that dogs should eat his flesh
+far from his parents, in the house of that terrible man on whose
+liver I would fain fasten and devour it. Thus would I avenge my
+son, who showed no cowardice when Achilles slew him, and thought
+neither of flight nor of avoiding battle as he stood in defence
+of Trojan men and Trojan women."
+</para>
+
+<para>
+Then Priam said, "I would go, do not therefore stay me nor be as
+a bird of ill omen in my house, for you will not move me. Had it
+been some mortal man who had sent me some prophet or priest who
+divines from sacrifice--I should have deemed him false and have
+given him no heed; but now I have heard the goddess and seen her
+face to face, therefore I will go and her saying shall not be in
+vain. If it be my fate to die at the ships of the Achaeans even
+so would I have it; let Achilles slay me, if I may but first have
+taken my son in my arms and mourned him to my heart's
+comforting."
+</para>
+
+<para>
+So saying he lifted the lids of his chests, and took out twelve
+goodly vestments. He took also twelve cloaks of single fold,
+twelve rugs, twelve fair mantles, and an equal number of shirts.
+He weighed out ten talents of gold, and brought moreover two
+burnished tripods, four cauldrons, and a very beautiful cup which
+the Thracians had given him when he had gone to them on an
+embassy; it was very precious, but he grudged not even this, so
+eager was he to ransom the body of his son. Then he chased all
+the Trojans from the court and rebuked them with words of anger.
+"Out," he cried, "shame and disgrace to me that you are. Have you
+no grief in your own homes that you are come to plague me here?
+Is it a small thing, think you, that the son of Saturn has sent
+this sorrow upon me, to lose the bravest of my sons? Nay, you
+shall prove it in person, for now he is gone the Achaeans will
+have easier work in killing you. As for me, let me go down within
+the house of Hades, ere mine eyes behold the sacking and wasting
+of the city."
+</para>
+
+<para>
+He drove the men away with his staff, and they went forth as the
+old man sped them. Then he called to his sons, upbraiding
+Helenus, Paris, noble Agathon, Pammon, Antiphonus, Polites of the
+loud battle-cry, Deiphobus, Hippothous, and Dius. These nine did
+the old man call near him. "Come to me at once," he cried,
+"worthless sons who do me shame; would that you had all been
+killed at the ships rather than Hector. Miserable man that I am,
+I have had the bravest sons in all Troy--noble Nestor, Troilus
+the dauntless charioteer, and Hector who was a god among men, so
+that one would have thought he was son to an immortal--yet there
+is not one of them left. Mars has slain them and those of whom I
+am ashamed are alone left me. Liars, and light of foot, heroes of
+the dance, robbers of lambs and kids from your own people, why do
+you not get a waggon ready for me at once, and put all these
+things upon it that I may set out on my way?"
+</para>
+
+<para>
+Thus did he speak, and they feared the rebuke of their father.
+They brought out a strong mule-waggon, newly made, and set the
+body of the waggon fast on its bed. They took the mule-yoke from
+the peg on which it hung, a yoke of boxwood with a knob on the
+top of it and rings for the reins to go through. Then they
+brought a yoke-band eleven cubits long, to bind the yoke to the
+pole; they bound it on at the far end of the pole, and put the
+ring over the upright pin making it fast with three turns of the
+band on either side the knob, and bending the thong of the yoke
+beneath it. This done, they brought from the store-chamber the
+rich ransom that was to purchase the body of Hector, and they set
+it all orderly on the waggon; then they yoked the strong
+harness-mules which the Mysians had on a time given as a goodly
+present to Priam; but for Priam himself they yoked horses which
+the old king had bred, and kept for own use.
+</para>
+
+<para>
+Thus heedfully did Priam and his servant see to the yolking of
+their cars at the palace. Then Hecuba came to them all sorrowful,
+with a golden goblet of wine in her right hand, that they might
+make a drink-offering before they set out. She stood in front of
+the horses and said, "Take this, make a drink-offering to father
+Jove, and since you are minded to go to the ships in spite of me,
+pray that you may come safely back from the hands of your
+enemies. Pray to the son of Saturn lord of the whirlwind, who
+sits on Ida and looks down over all Troy, pray him to send his
+swift messenger on your right hand, the bird of omen which is
+strongest and most dear to him of all birds, that you may see it
+with your own eyes and trust it as you go forth to the ships of
+the Danaans. If all-seeing Jove will not send you this messenger,
+however set upon it you may be, I would not have you go to the
+ships of the Argives."
+</para>
+
+<para>
+And Priam answered, "Wife, I will do as you desire me; it is well
+to lift hands in prayer to Jove, if so be he may have mercy upon
+me."
+</para>
+
+<para>
+With this the old man bade the serving-woman pour pure water over
+his hands, and the woman came, bearing the water in a bowl. He
+washed his hands and took the cup from his wife; then he made the
+drink-offering and prayed, standing in the middle of the
+courtyard and turning his eyes to heaven. "Father Jove," he said,
+"that rulest from Ida, most glorious and most great, grant that I
+may be received kindly and compassionately in the tents of
+Achilles; and send your swift messenger upon my right hand, the
+bird of omen which is strongest and most dear to you of all
+birds, that I may see it with my own eyes and trust it as I go
+forth to the ships of the Danaans."
+</para>
+
+<para>
+So did he pray, and Jove the lord of counsel heard his prayer.
+Forthwith he sent an eagle, the most unerring portent of all
+birds that fly, the dusky hunter that men also call the Black
+Eagle. His wings were spread abroad on either side as wide as the
+well-made and well-bolted door of a rich man's chamber. He came
+to them flying over the city upon their right hands, and when
+they saw him they were glad and their hearts took comfort within
+them. The old man made haste to mount his chariot, and drove out
+through the inner gateway and under the echoing gatehouse of the
+outer court. Before him went the mules drawing the four-wheeled
+waggon, and driven by wise Idaeus; behind these were the horses,
+which the old man lashed with his whip and drove swiftly through
+the city, while his friends followed after, wailing and lamenting
+for him as though he were on his road to death. As soon as they
+had come down from the city and had reached the plain, his sons
+and sons-in-law who had followed him went back to Ilius.
+</para>
+
+<para>
+But Priam and Idaeus as they showed out upon the plain did not
+escape the ken of all-seeing Jove, who looked down upon the old
+man and pitied him; then he spoke to his son Mercury and said,
+"Mercury, for it is you who are the most disposed to escort men
+on their way, and to hear those whom you will hear, go, and so
+conduct Priam to the ships of the Achaeans that no other of the
+Danaans shall see him nor take note of him until he reach the son
+of Peleus."
+</para>
+
+<para>
+Thus he spoke and Mercury, guide and guardian, slayer of Argus,
+did as he was told. Forthwith he bound on his glittering golden
+sandals with which he could fly like the wind over land and sea;
+he took the wand with which he seals men's eyes in sleep, or
+wakes them just as he pleases, and flew holding it in his hand
+till he came to Troy and to the Hellespont. To look at, he was
+like a young man of noble birth in the hey-day of his youth and
+beauty with the down just coming upon his face.
+</para>
+
+<para>
+Now when Priam and Idaeus had driven past the great tomb of
+Ilius, they stayed their mules and horses that they might drink
+in the river, for the shades of night were falling, when,
+therefore, Idaeus saw Mercury standing near them he said to
+Priam, "Take heed, descendant of Dardanus; here is matter which
+demands consideration. I see a man who I think will presently
+fall upon us; let us fly with our horses, or at least embrace his
+knees and implore him to take compassion upon us?"
+</para>
+
+<para>
+When he heard this the old man's heart failed him, and he was in
+great fear; he stayed where he was as one dazed, and the hair
+stood on end over his whole body; but the bringer of good luck
+came up to him and took him by the hand, saying, "Whither,
+father, are you thus driving your mules and horses in the dead of
+night when other men are asleep? Are you not afraid of the fierce
+Achaeans who are hard by you, so cruel and relentless? Should
+some one of them see you bearing so much treasure through the
+darkness of the flying night, what would not your state then be?
+You are no longer young, and he who is with you is too old to
+protect you from those who would attack you. For myself, I will
+do you no harm, and I will defend you from any one else, for you
+remind me of my own father."
+</para>
+
+<para>
+And Priam answered, "It is indeed as you say, my dear son;
+nevertheless some god has held his hand over me, in that he has
+sent such a wayfarer as yourself to meet me so opportunely; you
+are so comely in mien and figure, and your judgement is so
+excellent that you must come of blessed parents."
+</para>
+
+<para>
+Then said the slayer of Argus, guide and guardian, "Sir, all that
+you have said is right; but tell me and tell me true, are you
+taking this rich treasure to send it to a foreign people where it
+may be safe, or are you all leaving strong Ilius in dismay now
+that your son has fallen who was the bravest man among you and
+was never lacking in battle with the Achaeans?"
+</para>
+
+<para>
+And Priam said, "Who are you, my friend, and who are your
+parents, that you speak so truly about the fate of my unhappy
+son?"
+</para>
+
+<para>
+The slayer of Argus, guide and guardian, answered him, "Sir, you
+would prove me, that you question me about noble Hector. Many a
+time have I set eyes upon him in battle when he was driving the
+Argives to their ships and putting them to the sword. We stood
+still and marvelled, for Achilles in his anger with the son of
+Atreus suffered us not to fight. I am his squire, and came with
+him in the same ship. I am a Myrmidon, and my father's name is
+Polyctor: he is a rich man and about as old as you are; he has
+six sons besides myself, and I am the seventh. We cast lots, and
+it fell upon me to sail hither with Achilles. I am now come from
+the ships on to the plain, for with daybreak the Achaeans will
+set battle in array about the city. They chafe at doing nothing,
+and are so eager that their princes cannot hold them back."
+</para>
+
+<para>
+Then answered Priam, "If you are indeed the squire of Achilles
+son of Peleus, tell me now the whole truth. Is my son still at
+the ships, or has Achilles hewn him limb from limb, and given him
+to his hounds?"
+</para>
+
+<para>
+"Sir," replied the slayer of Argus, guide and guardian, "neither
+hounds nor vultures have yet devoured him; he is still just lying
+at the tents by the ship of Achilles, and though it is now twelve
+days that he has lain there, his flesh is not wasted nor have the
+worms eaten him although they feed on warriors. At daybreak
+Achilles drags him cruelly round the sepulchre of his dear
+comrade, but it does him no hurt. You should come yourself and
+see how he lies fresh as dew, with the blood all washed away, and
+his wounds every one of them closed though many pierced him with
+their spears. Such care have the blessed gods taken of your brave
+son, for he was dear to them beyond all measure."
+</para>
+
+<para>
+The old man was comforted as he heard him and said, "My son, see
+what a good thing it is to have made due offerings to the
+immortals; for as sure as that he was born my son never forgot
+the gods that hold Olympus, and now they requite it to him even
+in death. Accept therefore at my hands this goodly chalice; guard
+me and with heaven's help guide me till I come to the tent of the
+son of Peleus."
+</para>
+
+<para>
+Then answered the slayer of Argus, guide and guardian, "Sir, you
+are tempting me and playing upon my youth, but you shall not move
+me, for you are offering me presents without the knowledge of
+Achilles whom I fear and hold it great guilt to defraud, lest
+some evil presently befall me; but as your guide I would go with
+you even to Argos itself, and would guard you so carefully
+whether by sea or land, that no one should attack you through
+making light of him who was with you."
+</para>
+
+<para>
+The bringer of good luck then sprang on to the chariot, and
+seizing the whip and reins he breathed fresh spirit into the
+mules and horses. When they reached the trench and the wall that
+was before the ships, those who were on guard had just been
+getting their suppers, and the slayer of Argus threw them all
+into a deep sleep. Then he drew back the bolts to open the gates,
+and took Priam inside with the treasure he had upon his waggon.
+Ere long they came to the lofty dwelling of the son of Peleus for
+which the Myrmidons had cut pine and which they had built for
+their king; when they had built it they thatched it with coarse
+tussock-grass which they had mown out on the plain, and all round
+it they made a large courtyard, which was fenced with stakes set
+close together. The gate was barred with a single bolt of pine
+which it took three men to force into its place, and three to
+draw back so as to open the gate, but Achilles could draw it by
+himself. Mercury opened the gate for the old man, and brought in
+the treasure that he was taking with him for the son of Peleus.
+Then he sprang from the chariot on to the ground and said, "Sir,
+it is I, immortal Mercury, that am come with you, for my father
+sent me to escort you. I will now leave you, and will not enter
+into the presence of Achilles, for it might anger him that a god
+should befriend mortal men thus openly. Go you within, and
+embrace the knees of the son of Peleus: beseech him by his
+father, his lovely mother, and his son; thus you may move him."
+</para>
+
+<para>
+With these words Mercury went back to high Olympus. Priam sprang
+from his chariot to the ground, leaving Idaeus where he was, in
+charge of the mules and horses. The old man went straight into
+the house where Achilles, loved of the gods, was sitting. There
+he found him with his men seated at a distance from him: only
+two, the hero Automedon, and Alcimus of the race of Mars, were
+busy in attendance about his person, for he had but just done
+eating and drinking, and the table was still there. King Priam
+entered without their seeing him, and going right up to Achilles
+he clasped his knees and kissed the dread murderous hands that
+had slain so many of his sons.
+</para>
+
+<para>
+As when some cruel spite has befallen a man that he should have
+killed some one in his own country, and must fly to a great man's
+protection in a land of strangers, and all marvel who see him,
+even so did Achilles marvel as he beheld Priam. The others looked
+one to another and marvelled also, but Priam besought Achilles
+saying, "Think of your father, O Achilles like unto the gods, who
+is such even as I am, on the sad threshold of old age. It may be
+that those who dwell near him harass him, and there is none to
+keep war and ruin from him. Yet when he hears of you being still
+alive, he is glad, and his days are full of hope that he shall
+see his dear son come home to him from Troy; but I, wretched man
+that I am, had the bravest in all Troy for my sons, and there is
+not one of them left. I had fifty sons when the Achaeans came
+here; nineteen of them were from a single womb, and the others
+were borne to me by the women of my household. The greater part
+of them has fierce Mars laid low, and Hector, him who was alone
+left, him who was the guardian of the city and ourselves, him
+have you lately slain; therefore I am now come to the ships of
+the Achaeans to ransom his body from you with a great ransom.
+Fear, O Achilles, the wrath of heaven; think on your own father
+and have compassion upon me, who am the more pitiable, for I have
+steeled myself as no man yet has ever steeled himself before me,
+and have raised to my lips the hand of him who slew my son."
+</para>
+
+<para>
+Thus spoke Priam, and the heart of Achilles yearned as he
+bethought him of his father. He took the old man's hand and moved
+him gently away. The two wept bitterly--Priam, as he lay at
+Achilles' feet, weeping for Hector, and Achilles now for his
+father and now for Patroclous, till the house was filled with
+their lamentation. But when Achilles was now sated with grief and
+had unburthened the bitterness of his sorrow, he left his seat
+and raised the old man by the hand, in pity for his white hair
+and beard; then he said, "Unhappy man, you have indeed been
+greatly daring; how could you venture to come alone to the ships
+of the Achaeans, and enter the presence of him who has slain so
+many of your brave sons? You must have iron courage: sit now upon
+this seat, and for all our grief we will hide our sorrows in our
+hearts, for weeping will not avail us. The immortals know no
+care, yet the lot they spin for man is full of sorrow; on the
+floor of Jove's palace there stand two urns, the one filled with
+evil gifts, and the other with good ones. He for whom Jove the
+lord of thunder mixes the gifts he sends, will meet now with good
+and now with evil fortune; but he to whom Jove sends none but
+evil gifts will be pointed at by the finger of scorn, the hand of
+famine will pursue him to the ends of the world, and he will go
+up and down the face of the earth, respected neither by gods nor
+men. Even so did it befall Peleus; the gods endowed him with all
+good things from his birth upwards, for he reigned over the
+Myrmidons excelling all men in prosperity and wealth, and mortal
+though he was they gave him a goddess for his bride. But even on
+him too did heaven send misfortune, for there is no race of royal
+children born to him in his house, save one son who is doomed to
+die all untimely; nor may I take care of him now that he is
+growing old, for I must stay here at Troy to be the bane of you
+and your children. And you too, O Priam, I have heard that you
+were aforetime happy. They say that in wealth and plenitude of
+offspring you surpassed all that is in Lesbos, the realm of Makar
+to the northward, Phrygia that is more inland, and those that
+dwell upon the great Hellespont; but from the day when the
+dwellers in heaven sent this evil upon you, war and slaughter
+have been about your city continually. Bear up against it, and
+let there be some intervals in your sorrow. Mourn as you may for
+your brave son, you will take nothing by it. You cannot raise him
+from the dead, ere you do so yet another sorrow shall befall
+you."
+</para>
+
+<para>
+And Priam answered, "O king, bid me not be seated, while Hector
+is still lying uncared for in your tents, but accept the great
+ransom which I have brought you, and give him to me at once that
+I may look upon him. May you prosper with the ransom and reach
+your own land in safety, seeing that you have suffered me to live
+and to look upon the light of the sun."
+</para>
+
+<para>
+Achilles looked at him sternly and said, "Vex me, sir, no longer;
+I am of myself minded to give up the body of Hector. My mother,
+daughter of the old man of the sea, came to me from Jove to bid
+me deliver it to you. Moreover I know well, O Priam, and you
+cannot hide it, that some god has brought you to the ships of the
+Achaeans, for else, no man however strong and in his prime would
+dare to come to our host; he could neither pass our guard unseen,
+nor draw the bolt of my gates thus easily; therefore, provoke me
+no further, lest I sin against the word of Jove, and suffer you
+not, suppliant though you are, within my tents."
+</para>
+
+<para>
+The old man feared him and obeyed. Then the son of Peleus sprang
+like a lion through the door of his house, not alone, but with
+him went his two squires Automedon and Alcimus who were closer to
+him than any others of his comrades now that Patroclus was no
+more. These unyoked the horses and mules, and bade Priam's herald
+and attendant be seated within the house. They lifted the ransom
+for Hector's body from the waggon. but they left two mantles and
+a goodly shirt, that Achilles might wrap the body in them when he
+gave it to be taken home. Then he called to his servants and
+ordered them to wash the body and anoint it, but he first took it
+to a place where Priam should not see it, lest if he did so, he
+should break out in the bitterness of his grief, and enrage
+Achilles, who might then kill him and sin against the word of
+Jove. When the servants had washed the body and anointed it, and
+had wrapped it in a fair shirt and mantle, Achilles himself
+lifted it on to a bier, and he and his men then laid it on the
+waggon. He cried aloud as he did so and called on the name of his
+dear comrade, "Be not angry with me, Patroclus," he said, "if you
+hear even in the house of Hades that I have given Hector to his
+father for a ransom. It has been no unworthy one, and I will
+share it equitably with you."
+</para>
+
+<para>
+Achilles then went back into the tent and took his place on the
+richly inlaid seat from which he had risen, by the wall that was
+at right angles to the one against which Priam was sitting.
+"Sir," he said, "your son is now laid upon his bier and is
+ransomed according to desire; you shall look upon him when you
+him away at daybreak; for the present let us prepare our supper.
+Even lovely Niobe had to think about eating, though her twelve
+children--six daughters and six lusty sons--had been all slain in
+her house. Apollo killed the sons with arrows from his silver
+bow, to punish Niobe, and Diana slew the daughters, because Niobe
+had vaunted herself against Leto; she said Leto had borne two
+children only, whereas she had herself borne many--whereon the
+two killed the many. Nine days did they lie weltering, and there
+was none to bury them, for the son of Saturn turned the people
+into stone; but on the tenth day the gods in heaven themselves
+buried them, and Niobe then took food, being worn out with
+weeping. They say that somewhere among the rocks on the mountain
+pastures of Sipylus, where the nymphs live that haunt the river
+Achelous, there, they say, she lives in stone and still nurses
+the sorrows sent upon her by the hand of heaven. Therefore, noble
+sir, let us two now take food; you can weep for your dear son
+hereafter as you are bearing him back to Ilius--and many a tear
+will he cost you."
+</para>
+
+<para>
+With this Achilles sprang from his seat and killed a sheep of
+silvery whiteness, which his followers skinned and made ready all
+in due order. They cut the meat carefully up into smaller pieces,
+spitted them, and drew them off again when they were well
+roasted. Automedon brought bread in fair baskets and served it
+round the table, while Achilles dealt out the meat, and they laid
+their hands on the good things that were before them. As soon as
+they had had enough to eat and drink, Priam, descendant of
+Dardanus, marvelled at the strength and beauty of Achilles for he
+was as a god to see, and Achilles marvelled at Priam as he
+listened to him and looked upon his noble presence. When they had
+gazed their fill Priam spoke first. "And now, O king," he said,
+"take me to my couch that we may lie down and enjoy the blessed
+boon of sleep. Never once have my eyes been closed from the day
+your hands took the life of my son; I have grovelled without
+ceasing in the mire of my stable-yard, making moan and brooding
+over my countless sorrows. Now, moreover, I have eaten bread and
+drunk wine; hitherto I have tasted nothing."
+</para>
+
+<para>
+As he spoke Achilles told his men and the women-servants to set
+beds in the room that was in the gatehouse, and make them with
+good red rugs, and spread coverlets on the top of them with
+woollen cloaks for Priam and Idaeus to wear. So the maids went
+out carrying a torch and got the two beds ready in all haste.
+Then Achilles said laughingly to Priam, "Dear sir, you shall lie
+outside, lest some counsellor of those who in due course keep
+coming to advise with me should see you here in the darkness of
+the flying night, and tell it to Agamemnon. This might cause
+delay in the delivery of the body. And now tell me and tell me
+true, for how many days would you celebrate the funeral rites of
+noble Hector? Tell me, that I may hold aloof from war and
+restrain the host."
+</para>
+
+<para>
+And Priam answered, "Since, then, you suffer me to bury my noble
+son with all due rites, do thus, Achilles, and I shall be
+grateful. You know how we are pent up within our city; it is far
+for us to fetch wood from the mountain, and the people live in
+fear. Nine days, therefore, will we mourn Hector in my house; on
+the tenth day we will bury him and there shall be a public feast
+in his honour; on the eleventh we will build a mound over his
+ashes, and on the twelfth, if there be need, we will fight."
+</para>
+
+<para>
+And Achilles answered, "All, King Priam, shall be as you have
+said. I will stay our fighting for as long a time as you have
+named."
+</para>
+
+<para>
+As he spoke he laid his hand on the old man's right wrist, in
+token that he should have no fear; thus then did Priam and his
+attendant sleep there in the forecourt, full of thought, while
+Achilles lay in an inner room of the house, with fair Briseis by
+his side.
+</para>
+
+<para>
+And now both gods and mortals were fast asleep through the
+livelong night, but upon Mercury alone, the bringer of good luck,
+sleep could take no hold for he was thinking all the time how to
+get King Priam away from the ships without his being seen by the
+strong force of sentinels. He hovered therefore over Priam's head
+and said, "Sir, now that Achilles has spared your life, you seem
+to have no fear about sleeping in the thick of your foes. You
+have paid a great ransom, and have received the body of your son;
+were you still alive and a prisoner the sons whom you have left
+at home would have to give three times as much to free you; and
+so it would be if Agamemnon and the other Achaeans were to know
+of your being here."
+</para>
+
+<para>
+When he heard this the old man was afraid and roused his servant.
+Mercury then yoked their horses and mules, and drove them quickly
+through the host so that no man perceived them. When they came to
+the ford of eddying Xanthus, begotten of immortal Jove, Mercury
+went back to high Olympus, and dawn in robe of saffron began to
+break over all the land. Priam and Idaeus then drove on toward
+the city lamenting and making moan, and the mules drew the body
+of Hector. No one neither man nor woman saw them, till Cassandra,
+fair as golden Venus standing on Pergamus, caught sight of her
+dear father in his chariot, and his servant that was the city's
+herald with him. Then she saw him that was lying upon the bier,
+drawn by the mules, and with a loud cry she went about the city
+saying, "Come hither Trojans, men and women, and look on Hector;
+if ever you rejoiced to see him coming from battle when he was
+alive, look now on him that was the glory of our city and all our
+people."
+</para>
+
+<para>
+At this there was not man nor woman left in the city, so great a
+sorrow had possessed them. Hard by the gates they met Priam as he
+was bringing in the body. Hector's wife and his mother were the
+first to mourn him: they flew towards the waggon and laid their
+hands upon his head, while the crowd stood weeping round them.
+They would have stayed before the gates, weeping and lamenting
+the livelong day to the going down of the sun, had not Priam
+spoken to them from the chariot and said, "Make way for the mules
+to pass you. Afterwards when I have taken the body home you shall
+have your fill of weeping."
+</para>
+
+<para>
+On this the people stood asunder, and made a way for the waggon.
+When they had borne the body within the house they laid it upon a
+bed and seated minstrels round it to lead the dirge, whereon the
+women joined in the sad music of their lament. Foremost among
+them all Andromache led their wailing as she clasped the head of
+mighty Hector in her embrace. "Husband," she cried, "you have
+died young, and leave me in your house a widow; he of whom we are
+the ill-starred parents is still a mere child, and I fear he may
+not reach manhood. Ere he can do so our city will be razed and
+overthrown, for you who watched over it are no more--you who were
+its saviour, the guardian of our wives and children. Our women
+will be carried away captives to the ships, and I among them;
+while you, my child, who will be with me will be put to some
+unseemly tasks, working for a cruel master. Or, may be, some
+Achaean will hurl you (O miserable death) from our walls, to
+avenge some brother, son, or father whom Hector slew; many of
+them have indeed bitten the dust at his hands, for your father's
+hand in battle was no light one. Therefore do the people mourn
+him. You have left, O Hector, sorrow unutterable to your parents,
+and my own grief is greatest of all, for you did not stretch
+forth your arms and embrace me as you lay dying, nor say to me
+any words that might have lived with me in my tears night and day
+for evermore."
+</para>
+
+<para>
+Bitterly did she weep the while, and the women joined in her
+lament. Hecuba in her turn took up the strains of woe. "Hector,"
+she cried, "dearest to me of all my children. So long as you were
+alive the gods loved you well, and even in death they have not
+been utterly unmindful of you; for when Achilles took any other
+of my sons, he would sell him beyond the seas, to Samos Imbrus or
+rugged Lemnos; and when he had slain you too with his sword, many
+a time did he drag you round the sepulchre of his comrade--though
+this could not give him life--yet here you lie all fresh as dew,
+and comely as one whom Apollo has slain with his painless
+shafts."
+</para>
+
+<para>
+Thus did she too speak through her tears with bitter moan, and
+then Helen for a third time took up the strain of lamentation.
+"Hector," said she, "dearest of all my brothers-in-law--for I am
+wife to Alexandrus who brought me hither to Troy--would that I
+had died ere he did so--twenty years are come and gone since I
+left my home and came from over the sea, but I have never heard
+one word of insult or unkindness from you. When another would
+chide with me, as it might be one of your brothers or sisters or
+of your brothers' wives, or my mother-in-law--for Priam was as
+kind to me as though he were my own father--you would rebuke and
+check them with words of gentleness and goodwill. Therefore my
+tears flow both for you and for my unhappy self, for there is no
+one else in Troy who is kind to me, but all shrink and shudder as
+they go by me."
+</para>
+
+<para>
+She wept as she spoke and the vast crowd that was gathered round
+her joined in her lament. Then King Priam spoke to them saying,
+"Bring wood, O Trojans, to the city, and fear no cunning ambush
+of the Argives, for Achilles when he dismissed me from the ships
+gave me his word that they should not attack us until the morning
+of the twelfth day."
+</para>
+
+<para>
+Forthwith they yoked their oxen and mules and gathered together
+before the city. Nine days long did they bring in great heaps of
+wood, and on the morning of the tenth day with many tears they
+took brave Hector forth, laid his dead body upon the summit of
+the pile, and set the fire thereto. Then when the child of
+morning, rosy-fingered dawn, appeared on the eleventh day, the
+people again assembled, round the pyre of mighty Hector. When
+they were got together, they first quenched the fire with wine
+wherever it was burning, and then his brothers and comrades with
+many a bitter tear gathered his white bones, wrapped them in soft
+robes of purple, and laid them in a golden urn, which they placed
+in a grave and covered over with large stones set close together.
+Then they built a barrow hurriedly over it keeping guard on every
+side lest the Achaeans should attack them before they had
+finished. When they had heaped up the barrow they went back again
+into the city, and being well assembled they held high feast in
+the house of Priam their king.
+</para>
+
+<para>
+Thus, then, did they celebrate the funeral of Hector tamer of
+horses.
+</para>
+</chapter>
+</bookbody>
+</book>
+<endgutblurb>
+End of The Project Gutenberg Etext of The Iliad of Homer,
+translated by Samuel Butler
+</endgutblurb>
+
+</gutbook>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/origin.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/origin.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/origin.xml	(revision 21112)
@@ -0,0 +1,17770 @@
+<?xml version="1.0"?>
+<!DOCTYPE gutbook SYSTEM "gutbook1.dtd">
+
+<?xml-stylesheet type="text/css" href="darwin.css"?>
+
+<gutbook>
+<gutblurb>
+<para>
+
+Project Gutenberg's Etext of On the Origin of Species, by Darwin
+#3 in our series by Charles Darwin
+</para>
+
+<para>
+This Etext is based on the First Edition, we are also working on
+the Sixth Edition, which many believe to be the best edition.
+</para>
+
+<para>
+
+Copyright laws are changing all over the world, be sure to check
+the copyright laws for your country before posting these files!!
+</para>
+
+<para>
+Please take a look at the important information in this header.
+We encourage you to keep this file on your own disk, keeping an
+electronic path open for the next readers.  Do not remove this.
+</para>
+
+<para>
+
+**Welcome To The World of Free Plain Vanilla Electronic Texts**
+</para>
+
+<para>
+**Etexts Readable By Both Humans and By Computers, Since 1971**
+</para>
+
+<para>
+*These Etexts Prepared By Hundreds of Volunteers and Donations*
+</para>
+
+<para>
+Information on contacting Project Gutenberg to get Etexts, and
+further information is included below.  We need your donations.
+</para>
+
+<para>
+
+On the Origin of Species
+</para>
+
+<para>
+by Charles Darwin
+</para>
+
+<para>
+March, 1998  [Etext #1228]
+</para>
+
+<para>
+
+Project Gutenberg's Etext of On the Origin of Species, by Darwin
+******This file should be named 1rbnh10.txt or 1rbnh10.zip******
+</para>
+
+<para>
+Corrected EDITIONS of our etexts get a new NUMBER, 1rbnh11.txt
+VERSIONS based on separate sources get new LETTER, 1rbnh10a.txt
+</para>
+
+<para>
+
+This etext was prepared by Joseph S. Miller, Pensacola, FL and
+Embry-Riddle Aeronautical University Library, Prescott, AZ.
+</para>
+
+<para>
+Project Gutenberg Etexts are usually created from multiple editions,
+all of which are in the Public Domain in the United States, unless a
+copyright notice is included.  Therefore, we do NOT keep these books
+in compliance with any particular paper edition, usually otherwise.
+</para>
+
+<para>
+
+We are now trying to release all our books one month in advance
+of the official release dates, for time for better editing.
+</para>
+
+<para>
+Please note:  neither this list nor its contents are final till
+midnight of the last day of the month of any such announcement.
+The official release date of all Project Gutenberg Etexts is at
+Midnight, Central Time, of the last day of the stated month.  A
+preliminary version may often be posted for suggestion, comment
+and editing by those who wish to do so.  To be sure you have an
+up to date first edition [xxxxx10x.xxx] please check file sizes
+in the first week of the next month.  Since our ftp program has
+a bug in it that scrambles the date [tried to fix and failed] a
+look at the file size will have to do, but we will try to see a
+new copy has at least one byte more or less.
+</para>
+
+<para>
+
+Information about Project Gutenberg (one page)
+</para>
+
+<para>
+We produce about two million dollars for each hour we work.  The
+fifty hours is one conservative estimate for how long it we take
+to get any etext selected, entered, proofread, edited, copyright
+searched and analyzed, the copyright letters written, etc.  This
+projected audience is one hundred million readers.  If our value
+per text is nominally estimated at one dollar then we produce $2
+million dollars per hour this year as we release thirty-two text
+files per month, or 384 more Etexts in 1998 for a total of 1500+
+If these reach just 10% of the computerized population, then the
+total should reach over 150 billion Etexts given away.
+</para>
+
+<para>
+The Goal of Project Gutenberg is to Give Away One Trillion Etext
+Files by the December 31, 2001.  [10,000 x 100,000,000=Trillion]
+This is ten thousand titles each to one hundred million readers,
+which is only 10% of the present number of computer users.  2001
+should have at least twice as many computer users as that, so it
+will require us reaching less than 5% of the users in 2001.
+</para>
+
+<para>
+
+We need your donations more than ever!
+</para>
+
+<para>
+
+All donations should be made to "Project Gutenberg/CMU": and are
+tax deductible to the extent allowable by law.  (CMU = Carnegie-
+Mellon University).
+</para>
+
+<para>
+For these and other matters, please mail to:
+</para>
+
+<para>
+Project Gutenberg
+P. O. Box  2782
+Champaign, IL 61825
+</para>
+
+<para>
+When all other email fails try our Executive Director:
+Michael S. Hart &lt;hart@pobox.com>
+</para>
+
+<para>
+We would prefer to send you this information by email
+(Internet, Bitnet, Compuserve, ATTMAIL or MCImail).
+</para>
+
+<para>
+******
+If you have an FTP program (or emulator), please
+FTP directly to the Project Gutenberg archives:
+[Mac users, do NOT point and click. . .type]
+</para>
+
+<para>
+ftp uiarchive.cso.uiuc.edu
+login:  anonymous
+password:  your@login
+cd etext/etext90 through /etext96
+or cd etext/articles [get suggest gut for more information]
+dir [to see files]
+get or mget [to get files. . .set bin for zip files]
+GET INDEX?00.GUT
+for a list of books
+and
+GET NEW GUT for general information
+and
+MGET GUT* for newsletters.
+</para>
+
+<para>
+**Information prepared by the Project Gutenberg legal advisor**
+(Three Pages)
+</para>
+
+<para>
+
+***START**THE SMALL PRINT!**FOR PUBLIC DOMAIN ETEXTS**START***
+Why is this "Small Print!" statement here?  You know: lawyers.
+They tell us you might sue us if there is something wrong with
+your copy of this etext, even if you got it for free from
+someone other than us, and even if what's wrong is not our
+fault.  So, among other things, this "Small Print!" statement
+disclaims most of our liability to you.  It also tells you how
+you can distribute copies of this etext if you want to.
+</para>
+
+<para>
+*BEFORE!* YOU USE OR READ THIS ETEXT
+By using or reading any part of this PROJECT GUTENBERG-tm
+etext, you indicate that you understand, agree to and accept
+this "Small Print!" statement.  If you do not, you can receive
+a refund of the money (if any) you paid for this etext by
+sending a request within 30 days of receiving it to the person
+you got it from.  If you received this etext on a physical
+medium (such as a disk), you must return it with your request.
+</para>
+
+<para>
+ABOUT PROJECT GUTENBERG-TM ETEXTS
+This PROJECT GUTENBERG-tm etext, like most PROJECT GUTENBERG-
+tm etexts, is a "public domain" work distributed by Professor
+Michael S. Hart through the Project Gutenberg Association at
+Carnegie-Mellon University (the "Project").  Among other
+things, this means that no one owns a United States copyright
+on or for this work, so the Project (and you!) can copy and
+distribute it in the United States without permission and
+without paying copyright royalties.  Special rules, set forth
+below, apply if you wish to copy and distribute this etext
+under the Project's "PROJECT GUTENBERG" trademark.
+</para>
+
+<para>
+To create these etexts, the Project expends considerable
+efforts to identify, transcribe and proofread public domain
+works.  Despite these efforts, the Project's etexts and any
+medium they may be on may contain "Defects".  Among other
+things, Defects may take the form of incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other
+intellectual property infringement, a defective or damaged
+disk or other etext medium, a computer virus, or computer
+codes that damage or cannot be read by your equipment.
+</para>
+
+<para>
+LIMITED WARRANTY; DISCLAIMER OF DAMAGES
+But for the "Right of Replacement or Refund" described below,
+[1] the Project (and any other party you may receive this
+etext from as a PROJECT GUTENBERG-tm etext) disclaims all
+liability to you for damages, costs and expenses, including
+legal fees, and [2] YOU HAVE NO REMEDIES FOR NEGLIGENCE OR
+UNDER STRICT LIABILITY, OR FOR BREACH OF WARRANTY OR CONTRACT,
+INCLUDING BUT NOT LIMITED TO INDIRECT, CONSEQUENTIAL, PUNITIVE
+OR INCIDENTAL DAMAGES, EVEN IF YOU GIVE NOTICE OF THE
+POSSIBILITY OF SUCH DAMAGES.
+</para>
+
+<para>
+If you discover a Defect in this etext within 90 days of
+receiving it, you can receive a refund of the money (if any)
+you paid for it by sending an explanatory note within that
+time to the person you received it from.  If you received it
+on a physical medium, you must return it with your note, and
+such person may choose to alternatively give you a replacement
+copy.  If you received it electronically, such person may
+choose to alternatively give you a second opportunity to
+receive it electronically.
+</para>
+
+<para>
+THIS ETEXT IS OTHERWISE PROVIDED TO YOU "AS-IS".  NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, ARE MADE TO YOU AS
+TO THE ETEXT OR ANY MEDIUM IT MAY BE ON, INCLUDING BUT NOT
+LIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
+PARTICULAR PURPOSE.
+</para>
+
+<para>
+Some states do not allow disclaimers of implied warranties or
+the exclusion or limitation of consequential damages, so the
+above disclaimers and exclusions may not apply to you, and you
+may have other legal rights.
+</para>
+
+<para>
+INDEMNITY
+You will indemnify and hold the Project, its directors,
+officers, members and agents harmless from all liability, cost
+and expense, including legal fees, that arise directly or
+indirectly from any of the following that you do or cause:
+[1] distribution of this etext, [2] alteration, modification,
+or addition to the etext, or [3] any Defect.
+</para>
+
+<para>
+DISTRIBUTION UNDER "PROJECT GUTENBERG-tm"
+You may distribute copies of this etext electronically, or by
+disk, book or any other medium if you either delete this
+"Small Print!" and all other references to Project Gutenberg,
+or:
+</para>
+
+<para>
+[1]  Only give exact copies of it.  Among other things, this
+     requires that you do not remove, alter or modify the
+     etext or this "small print!" statement.  You may however,
+     if you wish, distribute this etext in machine readable
+     binary, compressed, mark-up, or proprietary form,
+     including any form resulting from conversion by word pro-
+     cessing or hypertext software, but only so long as
+     *EITHER*:
+</para>
+
+<para>
+     [*]  The etext, when displayed, is clearly readable, and
+          does *not* contain characters other than those
+          intended by the author of the work, although tilde
+          (~), asterisk (*) and underline (_) characters may
+          be used to convey punctuation intended by the
+          author, and additional characters may be used to
+          indicate hypertext links; OR
+</para>
+
+<para>
+     [*]  The etext may be readily converted by the reader at
+          no expense into plain ASCII, EBCDIC or equivalent
+          form by the program that displays the etext (as is
+          the case, for instance, with most word processors);
+          OR
+</para>
+
+<para>
+     [*]  You provide, or agree to also provide on request at
+          no additional cost, fee or expense, a copy of the
+          etext in its original plain ASCII form (or in EBCDIC
+          or other equivalent proprietary form).
+</para>
+
+<para>
+[2]  Honor the etext refund and replacement provisions of this
+     "Small Print!" statement.
+</para>
+
+<para>
+[3]  Pay a trademark license fee to the Project of 20% of the
+     net profits you derive calculated using the method you
+     already use to calculate your applicable taxes.  If you
+     don't derive profits, no royalty is due.  Royalties are
+     payable to "Project Gutenberg Association/Carnegie-Mellon
+     University" within the 60 days following each
+     date you prepare (or were legally required to prepare)
+     your annual (or equivalent periodic) tax return.
+</para>
+
+<para>
+WHAT IF YOU *WANT* TO SEND MONEY EVEN IF YOU DON'T HAVE TO?
+The Project gratefully accepts contributions in money, time,
+scanning machines, OCR software, public domain etexts, royalty
+free copyright licenses, and every other sort of contribution
+you can think of.  Money should be paid to "Project Gutenberg
+Association / Carnegie-Mellon University".
+</para>
+
+<para>
+*END*THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS*Ver.04.29.93*END*
+</para>
+
+</gutblurb>
+<markupblurb>
+Frank Boumphrey 9:25 AM 1/24/00
+Validated using MSXML and gutbook1.dtd
+</markupblurb>
+
+<book>
+<acknowledge>'On the Origin of Species' by Charles Darwin.
+  A project of Project Gutenberg and the HTML Writers Guild.
+  Transcribed by  Joseph S. Miller and
+Embry-Riddle Aeronautical University Library.
+  Markup by Frank Boumphrey. Jan 22 2000.
+For more details view the source document.
+</acknowledge>
+<frontmatter>
+<htitlepage>
+<title>
+On the Origin of Species
+</title>
+
+<author>
+by Charles Darwin
+</author>
+</htitlepage>
+
+<epigraph>
+<blockquote>
+<para>
+'But with regard to the material world, we can at least go so far as this--
+we can perceive that events are brought about not by insulated
+interpositions of Divine power, exerted in each particular case, but by the
+establishment of general laws.'
+</para>
+
+<attrib>
+W. Whewell: Bridgewater Treatise.
+</attrib>
+</blockquote>
+
+<blockquote>
+<para>
+'To conclude, therefore, let no man out of a weak conceit of sobriety, or
+an ill-applied moderation, think or maintain, that a man can search too far
+or be too well studied in the book of God's word, or in the book of God's
+works; divinity or philosophy; but rather let men endeavour an endless
+progress or proficience in both.'
+</para>
+
+<attrib>
+Bacon: Advancement of Learning.
+</attrib>
+</blockquote>
+
+
+<para>
+
+<place>Down, Bromley, Kent,</place> 
+</para>
+
+<para>
+<date>October 1st, 1859.</date>
+</para>
+</epigraph>
+
+<titlepage>
+<title>
+On the Origin of Species by Means of Natural Selection, 
+</title>
+
+<para>
+or the
+</para>
+
+<subtitle>
+Preservation of Favoured Races in the Struggle for Life.
+</subtitle>
+
+<author>
+By Charles Darwin
+</author>
+
+</titlepage>
+
+<toc>
+<title>Contents</title>
+
+<deflist>
+
+<item>
+Introduction
+</item>
+
+
+<item>
+Chapter I
+</item>
+
+<desc>
+Variation under Domestication
+</desc>
+
+<def>
+Causes of Variability -- Effects of Habit -- Correlation of Growth --
+Inheritance -- Character of Domestic Varieties -- Difficulty of
+distinguishing between Varieties and Species -- Origin of Domestic
+Varieties from one or more Species -- Domestic Pigeons, their Differences
+and Origin -- Principle of Selection anciently followed, its Effects --
+Methodical and Unconscious Selection -- Unknown Origin of our Domestic
+Productions -- Circumstances favourable to Man's power of Selection.
+</def>
+
+
+
+<item>
+Chapter II
+</item>
+
+<desc>
+Variation under Nature
+</desc>
+
+<def>
+Variability -- Individual Differences -- Doubtful species -- Wide ranging,
+much diffused, and common species vary most -- Species of the larger genera
+in any country vary more than the species of the smaller genera -- Many of
+the species of the larger genera resemble varieties in being very closely,
+but unequally, related to each other, and in having restricted ranges.
+</def>
+
+
+
+<item>
+Chapter III
+</item>
+
+<desc>
+Struggle for Existence
+</desc>
+
+<def>
+Bears on natural selection -- The term used in a wide sense -- Geometrical
+powers of increase -- Rapid increase of naturalised animals and plants --
+Nature of the checks to increase -- Competition universal -- Effects of
+climate -- Protection from the number of individuals -- Complex relations
+of all animals and plants throughout nature -- Struggle for life most
+severe between individuals and varieties of the same species; often severe
+between species of the same genus -- The relation of organism to organism
+the most important of all relations.
+</def>
+
+
+
+<item>
+Chapter IV
+</item>
+
+<desc>
+Natural Selection
+</desc>
+
+<def>
+Natural Selection -- its power compared with man's selection -- its power
+on characters of trifling importance -- its power at all ages and on both
+sexes -- Sexual Selection -- On the generality of intercrosses between
+individuals of the same species -- Circumstances favourable and
+unfavourable to Natural Selection, namely, intercrossing, isolation, number
+of individuals -- Slow action -- Extinction caused by Natural Selection --
+Divergence of Character, related to the diversity of inhabitants of any
+small area, and to naturalisation -- Action of Natural Selection, through
+Divergence of Character and Extinction, on the descendants from a common
+parent -- Explains the Grouping of all organic beings.
+</def>
+
+
+
+<item>
+Chapter V
+</item>
+
+<desc>
+Laws of Variation
+</desc>
+
+<def>
+Effects of external conditions -- Use and disuse, combined with natural
+selection; organs of flight and of vision -- Acclimatisation -- Correlation
+of growth -- Compensation and economy of growth -- False correlations --
+Multiple, rudimentary, and lowly organised structures variable -- Parts
+developed in an unusual manner are highly variable:  specific characters
+more variable than generic:  secondary sexual characters variable --
+Species of the same genus vary in an analogous manner -- Reversions to
+long-lost characters -- Summary.
+</def>
+
+
+
+<item>
+Chapter VI
+</item>
+
+<desc>
+Difficulties on Theory
+</desc>
+
+<def>
+Difficulties on the theory of descent with modification -- Transitions --
+Absence or rarity of transitional varieties -- Transitions in habits of
+life -- Diversified habits in the same species -- Species with habits
+widely different from those of their allies -- Organs of extreme perfection
+-- Means of transition -- Cases of difficulty -- Natura non facit saltum --
+Organs of small importance -- Organs not in all cases absolutely perfect --
+The law of Unity of Type and of the Conditions of Existence embraced by the
+theory of Natural Selection.
+</def>
+
+
+
+<item>
+Chapter VII
+</item>
+
+<desc>
+Instinct
+</desc>
+
+<def>
+Instincts comparable with habits, but different in their origin --
+Instincts graduated -- Aphides and ants -- Instincts variable -- Domestic
+instincts, their origin -- Natural instincts of the cuckoo, ostrich, and
+parasitic bees -- Slave-making ants -- Hive-bee, its cell-making instinct -
+- Difficulties on the theory of the Natural Selection of instincts --
+Neuter or sterile insects -- Summary.
+</def>
+
+
+
+<item>
+Chapter VIII
+</item>
+
+<desc>
+Hybridism
+</desc>
+
+<def>
+Distinction between the sterility of first crosses and of hybrids --
+Sterility various in degree, not universal, affected by close
+interbreeding, removed by domestication -- Laws governing the sterility of
+hybrids -- Sterility not a special endowment, but incidental on other
+differences -- Causes of the sterility of first crosses and of hybrids --
+Parallelism between the effects of changed conditions of life and crossing
+-- Fertility of varieties when crossed and of their mongrel offspring not
+universal -- Hybrids and mongrels compared independently of their fertility
+-- Summary.
+</def>
+
+
+
+<item>
+Chapter IX
+</item>
+
+<desc>
+On the Imperfection of the Geological Record
+</desc>
+
+<def>
+On the absence of intermediate varieties at the present day -- On the
+nature of extinct intermediate varieties; on their number -- On the vast
+lapse of time, as inferred from the rate of deposition and of denudation --
+On the poorness of our palaeontological collections -- On the intermittence
+of geological formations -- On the absence of intermediate varieties in any
+one formation -- On the sudden appearance of groups of species -- On their
+sudden appearance in the lowest known fossiliferous strata.
+</def>
+
+
+
+<item>
+Chapter X
+</item>
+
+<desc>
+On the Geological Succession of Organic Beings
+</desc>
+
+<def>
+On the slow and successive appearance of new species -- On their different
+rates of change -- Species once lost do not reappear -- Groups of species
+follow the same general rules in their appearance and disappearance as do
+single species -- On Extinction -- On simultaneous changes in the forms of
+life throughout the world -- On the affinities of extinct species to each
+other and to living species -- On the state of development of ancient forms
+-- On the succession of the same types within the same areas -- Summary of
+preceding and present chapters.
+</def>
+
+
+
+<item>
+Chapter XI
+</item>
+
+<desc>
+Geographical Distribution
+</desc>
+
+<def>
+Present distribution cannot be accounted for by differences in physical
+conditions -- Importance of barriers -- Affinity of the productions of the
+same continent -- Centres of creation -- Means of dispersal, by changes of
+climate and of the level of the land, and by occasional means -- Dispersal
+during the Glacial period co-extensive with the world.
+</def>
+
+
+
+<item>
+Chapter XII
+</item>
+
+<desc>
+Geographical Distribution -- continued
+</desc>
+
+<def>
+Distribution of fresh-water productions -- On the inhabitants of oceanic
+islands -- Absence of Batrachians and of terrestrial Mammals -- On the
+relation of the inhabitants of islands to those of the nearest mainland --
+On colonisation from the nearest source with subsequent modification --
+Summary of the last and present chapters.
+</def>
+
+
+
+<item>
+Chapter XIII
+</item>
+
+<desc>
+Mutual Affinities of Organic Beings:
+Morphology:  Embryology:  Rudimentary Organs
+</desc>
+
+<def>
+Classification, groups subordinate to groups -- Natural system -- Rules and
+difficulties in classification, explained on the theory of descent with
+modification -- Classification of varieties -- Descent always used in
+classification -- Analogical or adaptive characters -- Affinities, general,
+complex and radiating -- Extinction separates and defines groups --
+Morphology, between members of the same class, between parts of the same
+individual -- Embryology, laws of, explained by variations not supervening
+at an early age, and being inherited at a corresponding age -- Rudimentary
+Organs; their origin explained -- Summary.
+</def>
+
+
+
+<item>
+Chapter XIV
+</item>
+
+<desc>
+Recapitulation and Conclusion
+</desc>
+
+<def>
+Recapitulation of the difficulties on the theory of Natural Selection --
+Recapitulation of the general and special circumstances in its favour --
+Causes of the general belief in the immutability of species -- How far the
+theory of natural selection may be extended -- Effects of its adoption on
+the study of Natural history -- Concluding remarks.
+</def>
+</deflist>
+</toc>
+
+
+
+
+<introduction>
+<title>
+On the Origin of Species.
+
+Introduction.
+</title>
+
+<para>
+When on board H.M.S. 'Beagle,' as naturalist, I was much struck with
+certain facts in the distribution of the inhabitants of South America, and
+in the geological relations of the present to the past inhabitants of that
+continent.  These facts seemed to me to throw some light on the origin of
+species--that mystery of mysteries, as it has been called by one of our
+greatest philosophers.  On my return home, it occurred to me, in 1837, that
+something might perhaps be made out on this question by patiently
+accumulating and reflecting on all sorts of facts which could possibly have
+any bearing on it.  After five years' work I allowed myself to speculate on
+the subject, and drew up some short notes; these I enlarged in 1844 into a
+sketch of the conclusions, which then seemed to me probable:  from that
+period to the present day I have steadily pursued the same object.  I hope
+that I may be excused for entering on these personal details, as I give
+them to show that I have not been hasty in coming to a decision.
+</para>
+
+<para>
+My work is now nearly finished; but as it will take me two or three more
+years to complete it, and as my health is far from strong, I have been
+urged to publish this Abstract.  I have more especially been induced to do
+this, as Mr. Wallace, who is now studying the natural history of the Malay
+archipelago, has arrived at almost exactly the same general conclusions
+that I have on the origin of species.  Last year he sent to me a memoir on
+this subject, with a request that I would forward it to Sir Charles Lyell,
+who sent it to the Linnean Society, and it is published in the third volume
+of the Journal of that Society.  Sir C. Lyell and Dr. Hooker, who both knew
+of my work--the latter having read my sketch of 1844--honoured me by
+thinking it advisable to publish, with Mr. Wallace's excellent memoir, some
+brief extracts from my manuscripts.
+</para>
+
+<para>
+This Abstract, which I now publish, must necessarily be imperfect.  I
+cannot here give references and authorities for my several statements; and
+I must trust to the reader reposing some confidence in my accuracy.  No
+doubt errors will have crept in, though I hope I have always been cautious
+in trusting to good authorities alone.  I can here give only the general
+conclusions at which I have arrived, with a few facts in illustration, but
+which, I hope, in most cases will suffice.  No one can feel more sensible
+than I do of the necessity of hereafter publishing in detail all the facts,
+with references, on which my conclusions have been grounded; and I hope in
+a future work to do this.  For I am well aware that scarcely a single point
+is discussed in this volume on which facts cannot be adduced, often
+apparently leading to conclusions directly opposite to those at which I
+have arrived.  A fair result can be obtained only by fully stating and
+balancing the facts and arguments on both sides of each question; and this
+cannot possibly be here done.
+</para>
+
+<para>
+I much regret that want of space prevents my having the satisfaction of
+acknowledging the generous assistance which I have received from very many
+naturalists, some of them personally unknown to me.  I cannot, however, let
+this opportunity pass without expressing my deep obligations to Dr. Hooker,
+who for the last fifteen years has aided me in every possible way by his
+large stores of knowledge and his excellent judgment.
+</para>
+
+<para>
+In considering the Origin of Species, it is quite conceivable that a
+naturalist, reflecting on the mutual affinities of organic beings, on their
+embryological relations, their geographical distribution, geological
+succession, and other such facts, might come to the conclusion that each
+species had not been independently created, but had descended, like
+varieties, from other species.  Nevertheless, such a conclusion, even if
+well founded, would be unsatisfactory, until it could be shown how the
+innumerable species inhabiting this world have been modified, so as to
+acquire that perfection of structure and coadaptation which most justly
+excites our admiration.  Naturalists continually refer to external
+conditions, such as climate, food, &amp;c., as the only possible cause of
+variation.  In one very limited sense, as we shall hereafter see, this may
+be true; but it is preposterous to attribute to mere external conditions,
+the structure, for instance, of the woodpecker, with its feet, tail, beak,
+and tongue, so admirably adapted to catch insects under the bark of trees. 
+In the case of the misseltoe, which draws its nourishment from certain
+trees, which has seeds that must be transported by certain birds, and which
+has flowers with separate sexes absolutely requiring the agency of certain
+insects to bring pollen from one flower to the other, it is equally
+preposterous to account for the structure of this parasite, with its
+relations to several distinct organic beings, by the effects of external
+conditions, or of habit, or of the volition of the plant itself.
+</para>
+
+<para>
+The author of the 'Vestiges of Creation' would, I presume, say that, after
+a certain unknown number of generations, some bird had given birth to a
+woodpecker, and some plant to the misseltoe, and that these had been
+produced perfect as we now see them; but this assumption seems to me to be
+no explanation, for it leaves the case of the coadaptations of organic
+beings to each other and to their physical conditions of life, untouched
+and unexplained.
+</para>
+
+<para>
+It is, therefore, of the highest importance to gain a clear insight into
+the means of modification and coadaptation.  At the commencement of my
+observations it seemed to me probable that a careful study of domesticated
+animals and of cultivated plants would offer the best chance of making out
+this obscure problem.  Nor have I been disappointed; in this and in all
+other perplexing cases I have invariably found that our knowledge,
+imperfect though it be, of variation under domestication, afforded the best
+and safest clue.  I may venture to express my conviction of the high value
+of such studies, although they have been very commonly neglected by
+naturalists.
+</para>
+
+<para>
+From these considerations, I shall devote the first chapter of this
+Abstract to Variation under Domestication.  We shall thus see that a large
+amount of hereditary modification is at least possible, and, what is
+equally or more important, we shall see how great is the power of man in
+accumulating by his Selection successive slight variations.  I will then
+pass on to the variability of species in a state of nature; but I shall,
+unfortunately, be compelled to treat this subject far too briefly, as it
+can be treated properly only by giving long catalogues of facts.  We shall,
+however, be enabled to discuss what circumstances are most favourable to
+variation.  In the next chapter the Struggle for Existence amongst all
+organic beings throughout the world, which inevitably follows from their
+high geometrical powers of increase, will be treated of.  This is the
+doctrine of Malthus, applied to the whole animal and vegetable kingdoms. 
+As many more individuals of each species are born than can possibly
+survive; and as, consequently, there is a frequently recurring struggle for
+existence, it follows that any being, if it vary however slightly in any
+manner profitable to itself, under the complex and sometimes varying
+conditions of life, will have a better chance of surviving, and thus be
+naturally selected.  From the strong principle of inheritance, any selected
+variety will tend to propagate its new and modified form.
+</para>
+
+<para>
+This fundamental subject of Natural Selection will be treated at some
+length in the fourth chapter; and we shall then see how Natural Selection
+almost inevitably causes much Extinction of the less improved forms of life
+and induces what I have called Divergence of Character.  In the next
+chapter I shall discuss the complex and little known laws of variation and
+of correlation of growth.  In the four succeeding chapters, the most
+apparent and gravest difficulties on the theory will be given:  namely,
+first, the difficulties of transitions, or in understanding how a simple
+being or a simple organ can be changed and perfected into a highly
+developed being or elaborately constructed organ; secondly the subject of
+Instinct, or the mental powers of animals, thirdly, Hybridism, or the
+infertility of species and the fertility of varieties when intercrossed;
+and fourthly, the imperfection of the Geological Record.  In the next
+chapter I shall consider the geological succession of organic beings
+throughout time; in the eleventh and twelfth, their geographical
+distribution throughout space; in the thirteenth, their classification or
+mutual affinities, both when mature and in an embryonic condition.  In the
+last chapter I shall give a brief recapitulation of the whole work, and a
+few concluding remarks.
+</para>
+
+<para>
+No one ought to feel surprise at much remaining as yet unexplained in
+regard to the origin of species and varieties, if he makes due allowance
+for our profound ignorance in regard to the mutual relations of all the
+beings which live around us.  Who can explain why one species ranges widely
+and is very numerous, and why another allied species has a narrow range and
+is rare?  Yet these relations are of the highest importance, for they
+determine the present welfare, and, as I believe, the future success and
+modification of every inhabitant of this world.  Still less do we know of
+the mutual relations of the innumerable inhabitants of the world during the
+many past geological epochs in its history.  Although much remains obscure,
+and will long remain obscure, I can entertain no doubt, after the most
+deliberate study and dispassionate judgment of which I am capable, that the
+view which most naturalists entertain, and which I formerly
+entertained--namely, that each species has been independently created--is
+erroneous.  I am fully convinced that species are not immutable; but that
+those belonging to what are called the same genera are lineal descendants
+of some other and generally extinct species, in the same manner as the
+acknowledged varieties of any one species are the descendants of that
+species.  Furthermore, I am convinced that Natural Selection has been the
+main but not exclusive means of modification.
+</para>
+</introduction>
+</frontmatter>
+<bookbody>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter I</chapnum>
+
+<title>Variation under Domestication</title>
+
+<chapsummary>
+Causes of Variability -- Effects of Habit -- Correlation of Growth --
+Inheritance -- Character of Domestic Varieties -- Difficulty of
+distinguishing between Varieties and Species -- Origin of Domestic
+Varieties from one or more Species -- Domestic Pigeons, their Differences
+and Origin -- Principle of Selection anciently followed, its Effects --
+Methodical and Unconscious Selection -- Unknown Origin of our Domestic
+Productions -- Circumstances favourable to Man's power of Selection.
+</chapsummary>
+</chapheader>
+
+<para>
+When we look to the individuals of the same variety or sub-variety of our
+older cultivated plants and animals, one of the first points which strikes
+us, is, that they generally differ much more from each other, than do the
+individuals of any one species or variety in a state of nature.  When we
+reflect on the vast diversity of the plants and animals which have been
+cultivated, and which have varied during all ages under the most different
+climates and treatment, I think we are driven to conclude that this greater
+variability is simply due to our domestic productions having been raised
+under conditions of life not so uniform as, and somewhat different from,
+those to which the parent-species have been exposed under nature.  There
+is, also, I think, some probability in the view propounded by Andrew
+Knight, that this variability may be partly connected with excess of food. 
+It seems pretty clear that organic beings must be exposed during several
+generations to the new conditions of life to cause any appreciable amount
+of variation; and that when the organisation has once begun to vary, it
+generally continues to vary for many generations.  No case is on record of
+a variable being ceasing to be variable under cultivation.  Our oldest
+cultivated plants, such as wheat, still often yield new varieties:  our
+oldest domesticated animals are still capable of rapid improvement or
+modification.
+</para>
+
+<para>
+It has been disputed at what period of life the causes of variability,
+whatever they may be, generally act; whether during the early or late
+period of development of the embryo, or at the instant of conception. 
+Geoffroy St. Hilaire's experiments show that unnatural treatment of the
+embryo causes monstrosities; and monstrosities cannot be separated by any
+clear line of distinction from mere variations.  But I am strongly inclined
+to suspect that the most frequent cause of variability may be attributed to
+the male and female reproductive elements having been affected prior to the
+act of conception.  Several reasons make me believe in this; but the chief
+one is the remarkable effect which confinement or cultivation has on the
+functions of the reproductive system; this system appearing to be far more
+susceptible than any other part of the organisation, to the action of any
+change in the conditions of life.  Nothing is more easy than to tame an
+animal, and few things more difficult than to get it to breed freely under
+confinement, even in the many cases when the male and female unite.  How
+many animals there are which will not breed, though living long under not
+very close confinement in their native country!  This is generally
+attributed to vitiated instincts; but how many cultivated plants display
+the utmost vigour, and yet rarely or never seed!  In some few such cases it
+has been found out that very trifling changes, such as a little more or
+less water at some particular period of growth, will determine whether or
+not the plant sets a seed.  I cannot here enter on the copious details
+which I have collected on this curious subject; but to show how singular
+the laws are which determine the reproduction of animals under confinement,
+I may just mention that carnivorous animals, even from the tropics, breed
+in this country pretty freely under confinement, with the exception of the
+plantigrades or bear family; whereas, carnivorous birds, with the rarest
+exceptions, hardly ever lay fertile eggs.  Many exotic plants have pollen
+utterly worthless, in the same exact condition as in the most sterile
+hybrids.  When, on the one hand, we see domesticated animals and plants,
+though often weak and sickly, yet breeding quite freely under confinement;
+and when, on the other hand, we see individuals, though taken young from a
+state of nature, perfectly tamed, long-lived, and healthy (of which I could
+give numerous instances), yet having their reproductive system so seriously
+affected by unperceived causes as to fail in acting, we need not be
+surprised at this system, when it does act under confinement, acting not
+quite regularly, and producing offspring not perfectly like their parents
+or variable.
+</para>
+
+<para>
+Sterility has been said to be the bane of horticulture; but on this view we
+owe variability to the same cause which produces sterility; and variability
+is the source of all the choicest productions of the garden.  I may add,
+that as some organisms will breed most freely under the most unnatural
+conditions (for instance, the rabbit and ferret kept in hutches), showing
+that their reproductive system has not been thus affected; so will some
+animals and plants withstand domestication or cultivation, and vary very
+slightly--perhaps hardly more than in a state of nature.
+</para>
+
+<para>
+A long list could easily be given of 'sporting plants;' by this term
+gardeners mean a single bud or offset, which suddenly assumes a new and
+sometimes very different character from that of the rest of the plant. 
+Such buds can be propagated by grafting, &amp;c., and sometimes by seed.  These
+'sports' are extremely rare under nature, but far from rare under
+cultivation; and in this case we see that the treatment of the parent has
+affected a bud or offset, and not the ovules or pollen.  But it is the
+opinion of most physiologists that there is no essential difference between
+a bud and an ovule in their earliest stages of formation; so that, in fact,
+'sports' support my view, that variability may be largely attributed to the
+ovules or pollen, or to both, having been affected by the treatment of the
+parent prior to the act of conception.  These cases anyhow show that
+variation is not necessarily connected, as some authors have supposed, with
+the act of generation.
+</para>
+
+<para>
+Seedlings from the same fruit, and the young of the same litter, sometimes
+differ considerably from each other, though both the young and the parents,
+as Muller has remarked, have apparently been exposed to exactly the same
+conditions of life; and this shows how unimportant the direct effects of
+the conditions of life are in comparison with the laws of reproduction, and
+of growth, and of inheritance; for had the action of the conditions been
+direct, if any of the young had varied, all would probably have varied in
+the same manner.  To judge how much, in the case of any variation, we
+should attribute to the direct action of heat, moisture, light, food, &amp;c.,
+is most difficult:  my impression is, that with animals such agencies have
+produced very little direct effect, though apparently more in the case of
+plants.  Under this point of view, Mr. Buckman's recent experiments on
+plants seem extremely valuable.  When all or nearly all the individuals
+exposed to certain conditions are affected in the same way, the change at
+first appears to be directly due to such conditions; but in some cases it
+can be shown that quite opposite conditions produce similar changes of
+structure.  Nevertheless some slight amount of change may, I think, be
+attributed to the direct action of the conditions of life--as, in some
+cases, increased size from amount of food, colour from particular kinds of
+food and from light, and perhaps the thickness of fur from climate.
+</para>
+
+<para>
+Habit also has a deciding influence, as in the period of flowering with
+plants when transported from one climate to another.  In animals it has a
+more marked effect; for instance, I find in the domestic duck that the
+bones of the wing weigh less and the bones of the leg more, in proportion
+to the whole skeleton, than do the same bones in the wild-duck; and I
+presume that this change may be safely attributed to the domestic duck
+flying much less, and walking more, than its wild parent.  The great and
+inherited development of the udders in cows and goats in countries where
+they are habitually milked, in comparison with the state of these organs in
+other countries, is another instance of the effect of use.  Not a single
+domestic animal can be named which has not in some country drooping ears;
+and the view suggested by some authors, that the drooping is due to the
+disuse of the muscles of the ear, from the animals not being much alarmed
+by danger, seems probable.
+</para>
+
+<para>
+There are many laws regulating variation, some few of which can be dimly
+seen, and will be hereafter briefly mentioned.  I will here only allude to
+what may be called correlation of growth.  Any change in the embryo or
+larva will almost certainly entail changes in the mature animal.  In
+monstrosities, the correlations between quite distinct parts are very
+curious; and many instances are given in Isidore Geoffroy St. Hilaire's
+great work on this subject.  Breeders believe that long limbs are almost
+always accompanied by an elongated head.  Some instances of correlation are
+quite whimsical; thus cats with blue eyes are invariably deaf; colour and
+constitutional peculiarities go together, of which many remarkable cases
+could be given amongst animals and plants.  From the facts collected by
+Heusinger, it appears that white sheep and pigs are differently affected
+from coloured individuals by certain vegetable poisons.  Hairless dogs have
+imperfect teeth; long-haired and coarse-haired animals are apt to have, as
+is asserted, long or many horns; pigeons with feathered feet have skin
+between their outer toes; pigeons with short beaks have small feet, and
+those with long beaks large feet.  Hence, if man goes on selecting, and
+thus augmenting, any peculiarity, he will almost certainly unconsciously
+modify other parts of the structure, owing to the mysterious laws of the
+correlation of growth.
+</para>
+
+<para>
+The result of the various, quite unknown, or dimly seen laws of variation
+is infinitely complex and diversified.  It is well worth while carefully to
+study the several treatises published on some of our old cultivated plants,
+as on the hyacinth, potato, even the dahlia, &amp;c.; and it is really
+surprising to note the endless points in structure and constitution in
+which the varieties and sub-varieties differ slightly from each other.  The
+whole organisation seems to have become plastic, and tends to depart in
+some small degree from that of the parental type.
+</para>
+
+<para>
+Any variation which is not inherited is unimportant for us.  But the number
+and diversity of inheritable deviations of structure, both those of slight
+and those of considerable physiological importance, is endless.  Dr.
+Prosper Lucas's treatise, in two large volumes, is the fullest and the best
+on this subject.  No breeder doubts how strong is the tendency to
+inheritance:  like produces like is his fundamental belief:  doubts have
+been thrown on this principle by theoretical writers alone.  When a
+deviation appears not unfrequently, and we see it in the father and child,
+we cannot tell whether it may not be due to the same original cause acting
+on both; but when amongst individuals, apparently exposed to the same
+conditions, any very rare deviation, due to some extraordinary combination
+of circumstances, appears in the parent--say, once amongst several million
+individuals--and it reappears in the child, the mere doctrine of chances
+almost compels us to attribute its reappearance to inheritance.  Every one
+must have heard of cases of albinism, prickly skin, hairy bodies, &amp;c.,
+appearing in several members of the same family.  If strange and rare
+deviations of structure are truly inherited, less strange and commoner
+deviations may be freely admitted to be inheritable.  Perhaps the correct
+way of viewing the whole subject, would be, to look at the inheritance of
+every character whatever as the rule, and non-inheritance as the anomaly.
+</para>
+
+<para>
+The laws governing inheritance are quite unknown; no one can say why the
+same peculiarity in different individuals of the same species, and in
+individuals of different species, is sometimes inherited and sometimes not
+so; why the child often reverts in certain characters to its grandfather or
+grandmother or other much more remote ancestor; why a peculiarity is often
+transmitted from one sex to both sexes or to one sex alone, more commonly
+but not exclusively to the like sex.  It is a fact of some little
+importance to us, that peculiarities appearing in the males of our domestic
+breeds are often transmitted either exclusively, or in a much greater
+degree, to males alone.  A much more important rule, which I think may be
+trusted, is that, at whatever period of life a peculiarity first appears,
+it tends to appear in the offspring at a corresponding age, though
+sometimes earlier.  In many cases this could not be otherwise:  thus the
+inherited peculiarities in the horns of cattle could appear only in the
+offspring when nearly mature; peculiarities in the silkworm are known to
+appear at the corresponding caterpillar or cocoon stage.  But hereditary
+diseases and some other facts make me believe that the rule has a wider
+extension, and that when there is no apparent reason why a peculiarity
+should appear at any particular age, yet that it does tend to appear in the
+offspring at the same period at which it first appeared in the parent.  I
+believe this rule to be of the highest importance in explaining the laws of
+embryology.  These remarks are of course confined to the first appearance
+of the peculiarity, and not to its primary cause, which may have acted on
+the ovules or male element; in nearly the same manner as in the crossed
+offspring from a short-horned cow by a long-horned bull, the greater length
+of horn, though appearing late in life, is clearly due to the male element.
+</para>
+
+<para>
+Having alluded to the subject of reversion, I may here refer to a statement
+often made by naturalists--namely, that our domestic varieties, when run
+wild, gradually but certainly revert in character to their aboriginal
+stocks.  Hence it has been argued that no deductions can be drawn from
+domestic races to species in a state of nature.  I have in vain endeavoured
+to discover on what decisive facts the above statement has so often and so
+boldly been made.  There would be great difficulty in proving its truth: 
+we may safely conclude that very many of the most strongly-marked domestic
+varieties could not possibly live in a wild state.  In many cases we do not
+know what the aboriginal stock was, and so could not tell whether or not
+nearly perfect reversion had ensued.  It would be quite necessary, in order
+to prevent the effects of intercrossing, that only a single variety should
+be turned loose in its new home.  Nevertheless, as our varieties certainly
+do occasionally revert in some of their characters to ancestral forms, it
+seems to me not improbable, that if we could succeed in naturalising, or
+were to cultivate, during many generations, the several races, for
+instance, of the cabbage, in very poor soil (in which case, however, some
+effect would have to be attributed to the direct action of the poor soil),
+that they would to a large extent, or even wholly, revert to the wild
+aboriginal stock.  Whether or not the experiment would succeed, is not of
+great importance for our line of argument; for by the experiment itself the
+conditions of life are changed.  If it could be shown that our domestic
+varieties manifested a strong tendency to reversion,--that is, to lose
+their acquired characters, whilst kept under unchanged conditions, and
+whilst kept in a considerable body, so that free intercrossing might check,
+by blending together, any slight deviations of structure, in such case, I
+grant that we could deduce nothing from domestic varieties in regard to
+species.  But there is not a shadow of evidence in favour of this view:  to
+assert that we could not breed our cart and race-horses, long and
+short-horned cattle, and poultry of various breeds, and esculent
+vegetables, for an almost infinite number of generations, would be opposed
+to all experience.  I may add, that when under nature the conditions of
+life do change, variations and reversions of character probably do occur;
+but natural selection, as will hereafter be explained, will determine how
+far the new characters thus arising shall be preserved.
+</para>
+
+<para>
+When we look to the hereditary varieties or races of our domestic animals
+and plants, and compare them with species closely allied together, we
+generally perceive in each domestic race, as already remarked, less
+uniformity of character than in true species.  Domestic races of the same
+species, also, often have a somewhat monstrous character; by which I mean,
+that, although differing from each other, and from the other species of the
+same genus, in several trifling respects, they often differ in an extreme
+degree in some one part, both when compared one with another, and more
+especially when compared with all the species in nature to which they are
+nearest allied.  With these exceptions (and with that of the perfect
+fertility of varieties when crossed,--a subject hereafter to be discussed),
+domestic races of the same species differ from each other in the same
+manner as, only in most cases in a lesser degree than, do closely-allied
+species of the same genus in a state of nature.  I think this must be
+admitted, when we find that there are hardly any domestic races, either
+amongst animals or plants, which have not been ranked by some competent
+judges as mere varieties, and by other competent judges as the descendants
+of aboriginally distinct species.  If any marked distinction existed
+between domestic races and species, this source of doubt could not so
+perpetually recur.  It has often been stated that domestic races do not
+differ from each other in characters of generic value.  I think it could be
+shown that this statement is hardly correct; but naturalists differ most
+widely in determining what characters are of generic value; all such
+valuations being at present empirical.  Moreover, on the view of the origin
+of genera which I shall presently give, we have no right to expect often to
+meet with generic differences in our domesticated productions.
+</para>
+
+<para>
+When we attempt to estimate the amount of structural difference between the
+domestic races of the same species, we are soon involved in doubt, from not
+knowing whether they have descended from one or several parent-species. 
+This point, if it could be cleared up, would be interesting; if, for
+instance, it could be shown that the greyhound, bloodhound, terrier,
+spaniel, and bull-dog, which we all know propagate their kind so truly,
+were the offspring of any single species, then such facts would have great
+weight in making us doubt about the immutability of the many very closely
+allied and natural species--for instance, of the many foxes--inhabiting
+different quarters of the world.  I do not believe, as we shall presently
+see, that all our dogs have descended from any one wild species; but, in
+the case of some other domestic races, there is presumptive, or even
+strong, evidence in favour of this view.
+</para>
+
+<para>
+It has often been assumed that man has chosen for domestication animals and
+plants having an extraordinary inherent tendency to vary, and likewise to
+withstand diverse climates.  I do not dispute that these capacities have
+added largely to the value of most of our domesticated productions; but how
+could a savage possibly know, when he first tamed an animal, whether it
+would vary in succeeding generations, and whether it would endure other
+climates?  Has the little variability of the ass or guinea-fowl, or the
+small power of endurance of warmth by the rein-deer, or of cold by the
+common camel, prevented their domestication?  I cannot doubt that if other
+animals and plants, equal in number to our domesticated productions, and
+belonging to equally diverse classes and countries, were taken from a state
+of nature, and could be made to breed for an equal number of generations
+under domestication, they would vary on an average as largely as the parent
+species of our existing domesticated productions have varied.
+</para>
+
+<para>
+In the case of most of our anciently domesticated animals and plants, I do
+not think it is possible to come to any definite conclusion, whether they
+have descended from one or several species.  The argument mainly relied on
+by those who believe in the multiple origin of our domestic animals is,
+that we find in the most ancient records, more especially on the monuments
+of Egypt, much diversity in the breeds; and that some of the breeds closely
+resemble, perhaps are identical with, those still existing.  Even if this
+latter fact were found more strictly and generally true than seems to me to
+be the case, what does it show, but that some of our breeds originated
+there, four or five thousand years ago?  But Mr. Horner's researches have
+rendered it in some degree probable that man sufficiently civilized to have
+manufactured pottery existed in the valley of the Nile thirteen or fourteen
+thousand years ago; and who will pretend to say how long before these
+ancient periods, savages, like those of Tierra del Fuego or Australia, who
+possess a semi-domestic dog, may not have existed in Egypt?
+</para>
+
+<para>
+The whole subject must, I think, remain vague; nevertheless, I may, without
+here entering on any details, state that, from geographical and other
+considerations, I think it highly probable that our domestic dogs have
+descended from several wild species.  In regard to sheep and goats I can
+form no opinion.  I should think, from facts communicated to me by Mr.
+Blyth, on the habits, voice, and constitution, &amp;c., of the humped Indian
+cattle, that these had descended from a different aboriginal stock from our
+European cattle; and several competent judges believe that these latter
+have had more than one wild parent.  With respect to horses, from reasons
+which I cannot give here, I am doubtfully inclined to believe, in
+opposition to several authors, that all the races have descended from one
+wild stock.  Mr. Blyth, whose opinion, from his large and varied stores of
+knowledge, I should value more than that of almost any one, thinks that all
+the breeds of poultry have proceeded from the common wild Indian fowl
+(Gallus bankiva).  In regard to ducks and rabbits, the breeds of which
+differ considerably from each other in structure, I do not doubt that they
+all have descended from the common wild duck and rabbit.
+</para>
+
+<para>
+The doctrine of the origin of our several domestic races from several
+aboriginal stocks, has been carried to an absurd extreme by some authors. 
+They believe that every race which breeds true, let the distinctive
+characters be ever so slight, has had its wild prototype.  At this rate
+there must have existed at least a score of species of wild cattle, as many
+sheep, and several goats in Europe alone, and several even within Great
+Britain.  One author believes that there formerly existed in Great Britain
+eleven wild species of sheep peculiar to it!  When we bear in mind that
+Britain has now hardly one peculiar mammal, and France but few distinct
+from those of Germany and conversely, and so with Hungary, Spain, &amp;c., but
+that each of these kingdoms possesses several peculiar breeds of cattle,
+sheep, &amp;c., we must admit that many domestic breeds have originated in
+Europe; for whence could they have been derived, as these several countries
+do not possess a number of peculiar species as distinct parent-stocks?  So
+it is in India.  Even in the case of the domestic dogs of the whole world,
+which I fully admit have probably descended from several wild species, I
+cannot doubt that there has been an immense amount of inherited variation. 
+Who can believe that animals closely resembling the Italian greyhound, the
+bloodhound, the bull-dog, or Blenheim spaniel, &amp;c.--so unlike all wild
+Canidae--ever existed freely in a state of nature?  It has often been
+loosely said that all our races of dogs have been produced by the crossing
+of a few aboriginal species; but by crossing we can get only forms in some
+degree intermediate between their parents; and if we account for our
+several domestic races by this process, we must admit the former existence
+of the most extreme forms, as the Italian greyhound, bloodhound, bull-dog,
+&amp;c., in the wild state.  Moreover, the possibility of making distinct races
+by crossing has been greatly exaggerated.  There can be no doubt that a
+race may be modified by occasional crosses, if aided by the careful
+selection of those individual mongrels, which present any desired
+character; but that a race could be obtained nearly intermediate between
+two extremely different races or species, I can hardly believe.  Sir J.
+Sebright expressly experimentised for this object, and failed.  The
+offspring from the first cross between two pure breeds is tolerably and
+sometimes (as I have found with pigeons) extremely uniform, and everything
+seems simple enough; but when these mongrels are crossed one with another
+for several generations, hardly two of them will be alike, and then the
+extreme difficulty, or rather utter hopelessness, of the task becomes
+apparent.  Certainly, a breed intermediate between two very distinct breeds
+could not be got without extreme care and long-continued selection; nor can
+I find a single case on record of a permanent race having been thus formed.
+</para>
+
+<para>
+On the Breeds of the Domestic Pigeon. -- Believing that it is always best
+to study some special group, I have, after deliberation, taken up domestic
+pigeons.  I have kept every breed which I could purchase or obtain, and
+have been most kindly favoured with skins from several quarters of the
+world, more especially by the Hon. W. Elliot from India, and by the Hon. C.
+Murray from Persia.  Many treatises in different languages have been
+published on pigeons, and some of them are very important, as being of
+considerably antiquity.  I have associated with several eminent fanciers,
+and have been permitted to join two of the London Pigeon Clubs.  The
+diversity of the breeds is something astonishing.  Compare the English
+carrier and the short-faced tumbler, and see the wonderful difference in
+their beaks, entailing corresponding differences in their skulls.  The
+carrier, more especially the male bird, is also remarkable from the
+wonderful development of the carunculated skin about the head, and this is
+accompanied by greatly elongated eyelids, very large external orifices to
+the nostrils, and a wide gape of mouth.  The short-faced tumbler has a beak
+in outline almost like that of a finch; and the common tumbler has the
+singular and strictly inherited habit of flying at a great height in a
+compact flock, and tumbling in the air head over heels.  The runt is a bird
+of great size, with long, massive beak and large feet; some of the
+sub-breeds of runts have very long necks, others very long wings and tails,
+others singularly short tails.  The barb is allied to the carrier, but,
+instead of a very long beak, has a very short and very broad one.  The
+pouter has a much elongated body, wings, and legs; and its enormously
+developed crop, which it glories in inflating, may well excite astonishment
+and even laughter.  The turbit has a very short and conical beak, with a
+line of reversed feathers down the breast; and it has the habit of
+continually expanding slightly the upper part of the oesophagus.  The
+Jacobin has the feathers so much reversed along the back of the neck that
+they form a hood, and it has, proportionally to its size, much elongated
+wing and tail feathers.  The trumpeter and laugher, as their names express,
+utter a very different coo from the other breeds.  The fantail has thirty
+or even forty tail-feathers, instead of twelve or fourteen, the normal
+number in all members of the great pigeon family; and these feathers are
+kept expanded, and are carried so erect that in good birds the head and
+tail touch; the oil-gland is quite aborted.  Several other less distinct
+breeds might have been specified.
+</para>
+
+<para>
+In the skeletons of the several breeds, the development of the bones of the
+face in length and breadth and curvature differs enormously.  The shape, as
+well as the breadth and length of the ramus of the lower jaw, varies in a
+highly remarkable manner.  The number of the caudal and sacral vertebrae
+vary; as does the number of the ribs, together with their relative breadth
+and the presence of processes.  The size and shape of the apertures in the
+sternum are highly variable; so is the degree of divergence and relative
+size of the two arms of the furcula.  The proportional width of the gape of
+mouth, the proportional length of the eyelids, of the orifice of the
+nostrils, of the tongue (not always in strict correlation with the length
+of beak), the size of the crop and of the upper part of the oesophagus; the
+development and abortion of the oil-gland; the number of the primary wing
+and caudal feathers; the relative length of wing and tail to each other and
+to the body; the relative length of leg and of the feet; the number of
+scutellae on the toes, the development of skin between the toes, are all
+points of structure which are variable.  The period at which the perfect
+plumage is acquired varies, as does the state of the down with which the
+nestling birds are clothed when hatched.  The shape and size of the eggs
+vary.  The manner of flight differs remarkably; as does in some breeds the
+voice and disposition.  Lastly, in certain breeds, the males and females
+have come to differ to a slight degree from each other.
+</para>
+
+<para>
+Altogether at least a score of pigeons might be chosen, which if shown to
+an ornithologist, and he were told that they were wild birds, would
+certainly, I think, be ranked by him as well-defined species.  Moreover, I
+do not believe that any ornithologist would place the English carrier, the
+short-faced tumbler, the runt, the barb, pouter, and fantail in the same
+genus; more especially as in each of these breeds several truly-inherited
+sub-breeds, or species as he might have called them, could be shown him.
+</para>
+
+<para>
+Great as the differences are between the breeds of pigeons, I am fully
+convinced that the common opinion of naturalists is correct, namely, that
+all have descended from the rock-pigeon (Columba livia), including under
+this term several geographical races or sub-species, which differ from each
+other in the most trifling respects.  As several of the reasons which have
+led me to this belief are in some degree applicable in other cases, I will
+here briefly give them.  If the several breeds are not varieties, and have
+not proceeded from the rock-pigeon, they must have descended from at least
+seven or eight aboriginal stocks; for it is impossible to make the present
+domestic breeds by the crossing of any lesser number:  how, for instance,
+could a pouter be produced by crossing two breeds unless one of the
+parent-stocks possessed the characteristic enormous crop?  The supposed
+aboriginal stocks must all have been rock-pigeons, that is, not breeding or
+willingly perching on trees.  But besides C. livia, with its geographical
+sub-species, only two or three other species of rock-pigeons are known; and
+these have not any of the characters of the domestic breeds.  Hence the
+supposed aboriginal stocks must either still exist in the countries where
+they were originally domesticated, and yet be unknown to ornithologists;
+and this, considering their size, habits, and remarkable characters, seems
+very improbable; or they must have become extinct in the wild state.  But
+birds breeding on precipices, and good fliers, are unlikely to be
+exterminated; and the common rock-pigeon, which has the same habits with
+the domestic breeds, has not been exterminated even on several of the
+smaller British islets, or on the shores of the Mediterranean.  Hence the
+supposed extermination of so many species having similar habits with the
+rock-pigeon seems to me a very rash assumption.  Moreover, the several
+above-named domesticated breeds have been transported to all parts of the
+world, and, therefore, some of them must have been carried back again into
+their native country; but not one has ever become wild or feral, though the
+dovecot-pigeon, which is the rock-pigeon in a very slightly altered state,
+has become feral in several places.  Again, all recent experience shows
+that it is most difficult to get any wild animal to breed freely under
+domestication; yet on the hypothesis of the multiple origin of our pigeons,
+it must be assumed that at least seven or eight species were so thoroughly
+domesticated in ancient times by half-civilized man, as to be quite
+prolific under confinement.
+</para>
+
+<para>
+An argument, as it seems to me, of great weight, and applicable in several
+other cases, is, that the above-specified breeds, though agreeing generally
+in constitution, habits, voice, colouring, and in most parts of their
+structure, with the wild rock-pigeon, yet are certainly highly abnormal in
+other parts of their structure:  we may look in vain throughout the whole
+great family of Columbidae for a beak like that of the English carrier, or
+that of the short-faced tumbler, or barb; for reversed feathers like those
+of the jacobin; for a crop like that of the pouter; for tail-feathers like
+those of the fantail.  Hence it must be assumed not only that
+half-civilized man succeeded in thoroughly domesticating several species,
+but that he intentionally or by chance picked out extraordinarily abnormal
+species; and further, that these very species have since all become extinct
+or unknown.  So many strange contingencies seem to me improbable in the
+highest degree.
+</para>
+
+<para>
+Some facts in regard to the colouring of pigeons well deserve
+consideration.  The rock-pigeon is of a slaty-blue, and has a white rump
+(the Indian sub-species, C. intermedia of Strickland, having it bluish);
+the tail has a terminal dark bar, with the bases of the outer feathers
+externally edged with white; the wings have two black bars; some
+semi-domestic breeds and some apparently truly wild breeds have, besides
+the two black bars, the wings chequered with black.  These several marks do
+not occur together in any other species of the whole family.  Now, in every
+one of the domestic breeds, taking thoroughly well-bred birds, all the
+above marks, even to the white edging of the outer tail-feathers, sometimes
+concur perfectly developed.  Moreover, when two birds belonging to two
+distinct breeds are crossed, neither of which is blue or has any of the
+above-specified marks, the mongrel offspring are very apt suddenly to
+acquire these characters; for instance, I crossed some uniformly white
+fantails with some uniformly black barbs, and they produced mottled brown
+and black birds; these I again crossed together, and one grandchild of the
+pure white fantail and pure black barb was of as beautiful a blue colour,
+with the white rump, double black wing-bar, and barred and white-edged
+tail-feathers, as any wild rock-pigeon!  We can understand these facts, on
+the well-known principle of reversion to ancestral characters, if all the
+domestic breeds have descended from the rock-pigeon.  But if we deny this,
+we must make one of the two following highly improbable suppositions. 
+Either, firstly, that all the several imagined aboriginal stocks were
+coloured and marked like the rock-pigeon, although no other existing
+species is thus coloured and marked, so that in each separate breed there
+might be a tendency to revert to the very same colours and markings.  Or,
+secondly, that each breed, even the purest, has within a dozen or, at most,
+within a score of generations, been crossed by the rock-pigeon:  I say
+within a dozen or twenty generations, for we know of no fact countenancing
+the belief that the child ever reverts to some one ancestor, removed by a
+greater number of generations.  In a breed which has been crossed only once
+with some distinct breed, the tendency to reversion to any character
+derived from such cross will naturally become less and less, as in each
+succeeding generation there will be less of the foreign blood; but when
+there has been no cross with a distinct breed, and there is a tendency in
+both parents to revert to a character, which has been lost during some
+former generation, this tendency, for all that we can see to the contrary,
+may be transmitted undiminished for an indefinite number of generations. 
+These two distinct cases are often confounded in treatises on inheritance.
+</para>
+
+<para>
+Lastly, the hybrids or mongrels from between all the domestic breeds of
+pigeons are perfectly fertile.  I can state this from my own observations,
+purposely made on the most distinct breeds.  Now, it is difficult, perhaps
+impossible, to bring forward one case of the hybrid offspring of two
+animals clearly distinct being themselves perfectly fertile.  Some authors
+believe that long-continued domestication eliminates this strong tendency
+to sterility:  from the history of the dog I think there is some
+probability in this hypothesis, if applied to species closely related
+together, though it is unsupported by a single experiment.  But to extend
+the hypothesis so far as to suppose that species, aboriginally as distinct
+as carriers, tumblers, pouters, and fantails now are, should yield
+offspring perfectly fertile, inter se, seems to me rash in the extreme.
+</para>
+
+<para>
+From these several reasons, namely, the improbability of man having
+formerly got seven or eight supposed species of pigeons to breed freely
+under domestication; these supposed species being quite unknown in a wild
+state, and their becoming nowhere feral; these species having very abnormal
+characters in certain respects, as compared with all other Columbidae,
+though so like in most other respects to the rock-pigeon; the blue colour
+and various marks occasionally appearing in all the breeds, both when kept
+pure and when crossed; the mongrel offspring being perfectly fertile;--from
+these several reasons, taken together, I can feel no doubt that all our
+domestic breeds have descended from the Columba livia with its geographical
+sub-species.
+</para>
+
+<para>
+In favour of this view, I may add, firstly, that C. livia, or the
+rock-pigeon, has been found capable of domestication in Europe and in
+India; and that it agrees in habits and in a great number of points of
+structure with all the domestic breeds.  Secondly, although an English
+carrier or short-faced tumbler differs immensely in certain characters from
+the rock-pigeon, yet by comparing the several sub-breeds of these breeds,
+more especially those brought from distant countries, we can make an almost
+perfect series between the extremes of structure.  Thirdly, those
+characters which are mainly distinctive of each breed, for instance the
+wattle and length of beak of the carrier, the shortness of that of the
+tumbler, and the number of tail-feathers in the fantail, are in each breed
+eminently variable; and the explanation of this fact will be obvious when
+we come to treat of selection.  Fourthly, pigeons have been watched, and
+tended with the utmost care, and loved by many people.  They have been
+domesticated for thousands of years in several quarters of the world; the
+earliest known record of pigeons is in the fifth Aegyptian dynasty, about
+3000 B.C., as was pointed out to me by Professor Lepsius; but Mr. Birch
+informs me that pigeons are given in a bill of fare in the previous
+dynasty.  In the time of the Romans, as we hear from Pliny, immense prices
+were given for pigeons; 'nay, they are come to this pass, that they can
+reckon up their pedigree and race.'  Pigeons were much valued by Akber Khan
+in India, about the year 1600; never less than 20,000 pigeons were taken
+with the court.  'The monarchs of Iran and Turan sent him some very rare
+birds;' and, continues the courtly historian, 'His Majesty by crossing the
+breeds, which method was never practised before, has improved them
+astonishingly.'  About this same period the Dutch were as eager about
+pigeons as were the old Romans.  The paramount importance of these
+considerations in explaining the immense amount of variation which pigeons
+have undergone, will be obvious when we treat of Selection.  We shall then,
+also, see how it is that the breeds so often have a somewhat monstrous
+character.  It is also a most favourable circumstance for the production of
+distinct breeds, that male and female pigeons can be easily mated for life;
+and thus different breeds can be kept together in the same aviary.
+</para>
+
+<para>
+I have discussed the probable origin of domestic pigeons at some, yet quite
+insufficient, length; because when I first kept pigeons and watched the
+several kinds, knowing well how true they bred, I felt fully as much
+difficulty in believing that they could ever have descended from a common
+parent, as any naturalist could in coming to a similar conclusion in regard
+to the many species of finches, or other large groups of birds, in nature. 
+One circumstance has struck me much; namely, that all the breeders of the
+various domestic animals and the cultivators of plants, with whom I have
+ever conversed, or whose treatises I have read, are firmly convinced that
+the several breeds to which each has attended, are descended from so many
+aboriginally distinct species.  Ask, as I have asked, a celebrated raiser
+of Hereford cattle, whether his cattle might not have descended from long
+horns, and he will laugh you to scorn.  I have never met a pigeon, or
+poultry, or duck, or rabbit fancier, who was not fully convinced that each
+main breed was descended from a distinct species.  Van Mons, in his
+treatise on pears and apples, shows how utterly he disbelieves that the
+several sorts, for instance a Ribston-pippin or Codlin-apple, could ever
+have proceeded from the seeds of the same tree.  Innumerable other examples
+could be given.  The explanation, I think, is simple:  from long-continued
+study they are strongly impressed with the differences between the several
+races; and though they well know that each race varies slightly, for they
+win their prizes by selecting such slight differences, yet they ignore all
+general arguments, and refuse to sum up in their minds slight differences
+accumulated during many successive generations.  May not those naturalists
+who, knowing far less of the laws of inheritance than does the breeder, and
+knowing no more than he does of the intermediate links in the long lines of
+descent, yet admit that many of our domestic races have descended from the
+same parents--may they not learn a lesson of caution, when they deride the
+idea of species in a state of nature being lineal descendants of other
+species?
+</para>
+
+<para>
+
+Selection. -- Let us now briefly consider the steps by which domestic races
+have been produced, either from one or from several allied species.  Some
+little effect may, perhaps, be attributed to the direct action of the
+external conditions of life, and some little to habit; but he would be a
+bold man who would account by such agencies for the differences of a dray
+and race horse, a greyhound and bloodhound, a carrier and tumbler pigeon. 
+One of the most remarkable features in our domesticated races is that we
+see in them adaptation, not indeed to the animal's or plant's own good, but
+to man's use or fancy.  Some variations useful to him have probably arisen
+suddenly, or by one step; many botanists, for instance, believe that the
+fuller's teazle, with its hooks, which cannot be rivalled by any mechanical
+contrivance, is only a variety of the wild Dipsacus; and this amount of
+change may have suddenly arisen in a seedling.  So it has probably been
+with the turnspit dog; and this is known to have been the case with the
+ancon sheep.  But when we compare the dray-horse and race-horse, the
+dromedary and camel, the various breeds of sheep fitted either for
+cultivated land or mountain pasture, with the wool of one breed good for
+one purpose, and that of another breed for another purpose; when we compare
+the many breeds of dogs, each good for man in very different ways; when we
+compare the game-cock, so pertinacious in battle, with other breeds so
+little quarrelsome, with 'everlasting layers' which never desire to sit,
+and with the bantam so small and elegant; when we compare the host of
+agricultural, culinary, orchard, and flower-garden races of plants, most
+useful to man at different seasons and for different purposes, or so
+beautiful in his eyes, we must, I think, look further than to mere
+variability.  We cannot suppose that all the breeds were suddenly produced
+as perfect and as useful as we now see them; indeed, in several cases, we
+know that this has not been their history.  The key is man's power of
+accumulative selection:  nature gives successive variations; man adds them
+up in certain directions useful to him.  In this sense he may be said to
+make for himself useful breeds.
+</para>
+
+<para>
+The great power of this principle of selection is not hypothetical.  It is
+certain that several of our eminent breeders have, even within a single
+lifetime, modified to a large extent some breeds of cattle and sheep.  In
+order fully to realise what they have done, it is almost necessary to read
+several of the many treatises devoted to this subject, and to inspect the
+animals.  Breeders habitually speak of an animal's organisation as
+something quite plastic, which they can model almost as they please.  If I
+had space I could quote numerous passages to this effect from highly
+competent authorities.  Youatt, who was probably better acquainted with the
+works of agriculturalists than almost any other individual, and who was
+himself a very good judge of an animal, speaks of the principle of
+selection as 'that which enables the agriculturist, not only to modify the
+character of his flock, but to change it altogether.  It is the magician's
+wand, by means of which he may summon into life whatever form and mould he
+pleases.'  Lord Somerville, speaking of what breeders have done for sheep,
+says:- 'It would seem as if they had chalked out upon a wall a form perfect
+in itself, and then had given it existence.'  That most skilful breeder,
+Sir John Sebright, used to say, with respect to pigeons, that 'he would
+produce any given feather in three years, but it would take him six years
+to obtain head and beak.'  In Saxony the importance of the principle of
+selection in regard to merino sheep is so fully recognised, that men follow
+it as a trade:  the sheep are placed on a table and are studied, like a
+picture by a connoisseur; this is done three times at intervals of months,
+and the sheep are each time marked and classed, so that the very best may
+ultimately be selected for breeding.
+</para>
+
+<para>
+What English breeders have actually effected is proved by the enormous
+prices given for animals with a good pedigree; and these have now been
+exported to almost every quarter of the world.  The improvement is by no
+means generally due to crossing different breeds; all the best breeders are
+strongly opposed to this practice, except sometimes amongst closely allied
+sub-breeds.  And when a cross has been made, the closest selection is far
+more indispensable even than in ordinary cases.  If selection consisted
+merely in separating some very distinct variety, and breeding from it, the
+principle would be so obvious as hardly to be worth notice; but its
+importance consists in the great effect produced by the accumulation in one
+direction, during successive generations, of differences absolutely
+inappreciable by an uneducated eye--differences which I for one have vainly
+attempted to appreciate.  Not one man in a thousand has accuracy of eye and
+judgment sufficient to become an eminent breeder.  If gifted with these
+qualities, and he studies his subject for years, and devotes his lifetime
+to it with indomitable perseverance, he will succeed, and may make great
+improvements; if he wants any of these qualities, he will assuredly fail. 
+Few would readily believe in the natural capacity and years of practice
+requisite to become even a skilful pigeon-fancier.
+</para>
+
+<para>
+The same principles are followed by horticulturists; but the variations are
+here often more abrupt.  No one supposes that our choicest productions have
+been produced by a single variation from the aboriginal stock.  We have
+proofs that this is not so in some cases, in which exact records have been
+kept; thus, to give a very trifling instance, the steadily-increasing size
+of the common gooseberry may be quoted.  We see an astonishing improvement
+in many florists' flowers, when the flowers of the present day are compared
+with drawings made only twenty or thirty years ago.  When a race of plants
+is once pretty well established, the seed-raisers do not pick out the best
+plants, but merely go over their seed-beds, and pull up the 'rogues,' as
+they call the plants that deviate from the proper standard.  With animals
+this kind of selection is, in fact, also followed; for hardly any one is so
+careless as to allow his worst animals to breed.
+</para>
+
+<para>
+In regard to plants, there is another means of observing the accumulated
+effects of selection--namely, by comparing the diversity of flowers in the
+different varieties of the same species in the flower-garden; the diversity
+of leaves, pods, or tubers, or whatever part is valued, in the
+kitchen-garden, in comparison with the flowers of the same varieties; and
+the diversity of fruit of the same species in the orchard, in comparison
+with the leaves and flowers of the same set of varieties.  See how
+different the leaves of the cabbage are, and how extremely alike the
+flowers; how unlike the flowers of the heartsease are, and how alike the
+leaves; how much the fruit of the different kinds of gooseberries differ in
+size, colour, shape, and hairiness, and yet the flowers present very slight
+differences.  It is not that the varieties which differ largely in some one
+point do not differ at all in other points; this is hardly ever, perhaps
+never, the case.  The laws of correlation of growth, the importance of
+which should never be overlooked, will ensure some differences; but, as a
+general rule, I cannot doubt that the continued selection of slight
+variations, either in the leaves, the flowers, or the fruit, will produce
+races differing from each other chiefly in these characters.
+</para>
+
+<para>
+It may be objected that the principle of selection has been reduced to
+methodical practice for scarcely more than three-quarters of a century; it
+has certainly been more attended to of late years, and many treatises have
+been published on the subject; and the result, I may add, has been, in a
+corresponding degree, rapid and important.  But it is very far from true
+that the principle is a modern discovery.  I could give several references
+to the full acknowledgment of the importance of the principle in works of
+high antiquity.  In rude and barbarous periods of English history choice
+animals were often imported, and laws were passed to prevent their
+exportation:  the destruction of horses under a certain size was ordered,
+and this may be compared to the 'roguing' of plants by nurserymen.  The
+principle of selection I find distinctly given in an ancient Chinese
+encyclopaedia.  Explicit rules are laid down by some of the Roman classical
+writers.  From passages in Genesis, it is clear that the colour of domestic
+animals was at that early period attended to.  Savages now sometimes cross
+their dogs with wild canine animals, to improve the breed, and they
+formerly did so, as is attested by passages in Pliny.  The savages in South
+Africa match their draught cattle by colour, as do some of the Esquimaux
+their teams of dogs.  Livingstone shows how much good domestic breeds are
+valued by the negroes of the interior of Africa who have not associated
+with Europeans.  Some of these facts do not show actual selection, but they
+show that the breeding of domestic animals was carefully attended to in
+ancient times, and is now attended to by the lowest savages.  It would,
+indeed, have been a strange fact, had attention not been paid to breeding,
+for the inheritance of good and bad qualities is so obvious.
+</para>
+
+<para>
+At the present time, eminent breeders try by methodical selection, with a
+distinct object in view, to make a new strain or sub-breed, superior to
+anything existing in the country.  But, for our purpose, a kind of
+Selection, which may be called Unconscious, and which results from every
+one trying to possess and breed from the best individual animals, is more
+important.  Thus, a man who intends keeping pointers naturally tries to get
+as good dogs as he can, and afterwards breeds from his own best dogs, but
+he has no wish or expectation of permanently altering the breed. 
+Nevertheless I cannot doubt that this process, continued during centuries,
+would improve and modify any breed, in the same way as Bakewell, Collins,
+&amp;c., by this very same process, only carried on more methodically, did
+greatly modify, even during their own lifetimes, the forms and qualities of
+their cattle.  Slow and insensible changes of this kind could never be
+recognised unless actual measurements or careful drawings of the breeds in
+question had been made long ago, which might serve for comparison.  In some
+cases, however, unchanged or but little changed individuals of the same
+breed may be found in less civilised districts, where the breed has been
+less improved.  There is reason to believe that King Charles's spaniel has
+been unconsciously modified to a large extent since the time of that
+monarch.  Some highly competent authorities are convinced that the setter
+is directly derived from the spaniel, and has probably been slowly altered
+from it.  It is known that the English pointer has been greatly changed
+within the last century, and in this case the change has, it is believed,
+been chiefly effected by crosses with the fox-hound; but what concerns us
+is, that the change has been effected unconsciously and gradually, and yet
+so effectually, that, though the old Spanish pointer certainly came from
+Spain, Mr. Borrow has not seen, as I am informed by him, any native dog in
+Spain like our pointer.
+</para>
+
+<para>
+By a similar process of selection, and by careful training, the whole body
+of English racehorses have come to surpass in fleetness and size the parent
+Arab stock, so that the latter, by the regulations for the Goodwood Races,
+are favoured in the weights they carry.  Lord Spencer and others have shown
+how the cattle of England have increased in weight and in early maturity,
+compared with the stock formerly kept in this country.  By comparing the
+accounts given in old pigeon treatises of carriers and tumblers with these
+breeds as now existing in Britain, India, and Persia, we can, I think,
+clearly trace the stages through which they have insensibly passed, and
+come to differ so greatly from the rock-pigeon.
+</para>
+
+<para>
+Youatt gives an excellent illustration of the effects of a course of
+selection, which may be considered as unconsciously followed, in so far
+that the breeders could never have expected or even have wished to have
+produced the result which ensued--namely, the production of two distinct
+strains.  The two flocks of Leicester sheep kept by Mr. Buckley and Mr.
+Burgess, as Mr. Youatt remarks, 'have been purely bred from the original
+stock of Mr. Bakewell for upwards of fifty years.  There is not a suspicion
+existing in the mind of any one at all acquainted with the subject that the
+owner of either of them has deviated in any one instance from the pure
+blood of Mr. Bakewell's flock, and yet the difference between the sheep
+possessed by these two gentlemen is so great that they have the appearance
+of being quite different varieties.'
+</para>
+
+<para>
+If there exist savages so barbarous as never to think of the inherited
+character of the offspring of their domestic animals, yet any one animal
+particularly useful to them, for any special purpose, would be carefully
+preserved during famines and other accidents, to which savages are so
+liable, and such choice animals would thus generally leave more offspring
+than the inferior ones; so that in this case there would be a kind of
+unconscious selection going on.  We see the value set on animals even by
+the barbarians of Tierra del Fuego, by their killing and devouring their
+old women, in times of dearth, as of less value than their dogs.
+</para>
+
+<para>
+In plants the same gradual process of improvement, through the occasional
+preservation of the best individuals, whether or not sufficiently distinct
+to be ranked at their first appearance as distinct varieties, and whether
+or not two or more species or races have become blended together by
+crossing, may plainly be recognised in the increased size and beauty which
+we now see in the varieties of the heartsease, rose, pelargonium, dahlia,
+and other plants, when compared with the older varieties or with their
+parent-stocks.  No one would ever expect to get a first-rate heartsease or
+dahlia from the seed of a wild plant.  No one would expect to raise a
+first-rate melting pear from the seed of a wild pear, though he might
+succeed from a poor seedling growing wild, if it had come from a
+garden-stock.  The pear, though cultivated in classical times, appears,
+from Pliny's description, to have been a fruit of very inferior quality.  I
+have seen great surprise expressed in horticultural works at the wonderful
+skill of gardeners, in having produced such splendid results from such poor
+materials; but the art, I cannot doubt, has been simple, and, as far as the
+final result is concerned, has been followed almost unconsciously.  It has
+consisted in always cultivating the best known variety, sowing its seeds,
+and, when a slightly better variety has chanced to appear, selecting it,
+and so onwards.  But the gardeners of the classical period, who cultivated
+the best pear they could procure, never thought what splendid fruit we
+should eat; though we owe our excellent fruit, in some small degree, to
+their having naturally chosen and preserved the best varieties they could
+anywhere find.
+</para>
+
+<para>
+A large amount of change in our cultivated plants, thus slowly and
+unconsciously accumulated, explains, as I believe, the well-known fact,
+that in a vast number of cases we cannot recognise, and therefore do not
+know, the wild parent-stocks of the plants which have been longest
+cultivated in our flower and kitchen gardens.  If it has taken centuries or
+thousands of years to improve or modify most of our plants up to their
+present standard of usefulness to man, we can understand how it is that
+neither Australia, the Cape of Good Hope, nor any other region inhabited by
+quite uncivilised man, has afforded us a single plant worth culture.  It is
+not that these countries, so rich in species, do not by a strange chance
+possess the aboriginal stocks of any useful plants, but that the native
+plants have not been improved by continued selection up to a standard of
+perfection comparable with that given to the plants in countries anciently
+civilised.
+</para>
+
+<para>
+In regard to the domestic animals kept by uncivilised man, it should not be
+overlooked that they almost always have to struggle for their own food, at
+least during certain seasons.  And in two countries very differently
+circumstanced, individuals of the same species, having slightly different
+constitutions or structure, would often succeed better in the one country
+than in the other, and thus by a process of 'natural selection,' as will
+hereafter be more fully explained, two sub-breeds might be formed.  This,
+perhaps, partly explains what has been remarked by some authors, namely,
+that the varieties kept by savages have more of the character of species
+than the varieties kept in civilised countries.
+</para>
+
+<para>
+On the view here given of the all-important part which selection by man has
+played, it becomes at once obvious, how it is that our domestic races show
+adaptation in their structure or in their habits to man's wants or fancies. 
+We can, I think, further understand the frequently abnormal character of
+our domestic races, and likewise their differences being so great in
+external characters and relatively so slight in internal parts or organs. 
+Man can hardly select, or only with much difficulty, any deviation of
+structure excepting such as is externally visible; and indeed he rarely
+cares for what is internal.  He can never act by selection, excepting on
+variations which are first given to him in some slight degree by nature. 
+No man would ever try to make a fantail, till he saw a pigeon with a tail
+developed in some slight degree in an unusual manner, or a pouter till he
+saw a pigeon with a crop of somewhat unusual size; and the more abnormal or
+unusual any character was when it first appeared, the more likely it would
+be to catch his attention.  But to use such an expression as trying to make
+a fantail, is, I have no doubt, in most cases, utterly incorrect.  The man
+who first selected a pigeon with a slightly larger tail, never dreamed what
+the descendants of that pigeon would become through long-continued, partly
+unconscious and partly methodical selection.  Perhaps the parent bird of
+all fantails had only fourteen tail-feathers somewhat expanded, like the
+present Java fantail, or like individuals of other and distinct breeds, in
+which as many as seventeen tail-feathers have been counted.  Perhaps the
+first pouter-pigeon did not inflate its crop much more than the turbit now
+does the upper part of its oesophagus,--a habit which is disregarded by all
+fanciers, as it is not one of the points of the breed.
+</para>
+
+<para>
+Nor let it be thought that some great deviation of structure would be
+necessary to catch the fancier's eye:  he perceives extremely small
+differences, and it is in human nature to value any novelty, however
+slight, in one's own possession.  Nor must the value which would formerly
+be set on any slight differences in the individuals of the same species, be
+judged of by the value which would now be set on them, after several breeds
+have once fairly been established.  Many slight differences might, and
+indeed do now, arise amongst pigeons, which are rejected as faults or
+deviations from the standard of perfection of each breed.  The common goose
+has not given rise to any marked varieties; hence the Thoulouse and the
+common breed, which differ only in colour, that most fleeting of
+characters, have lately been exhibited as distinct at our poultry-shows.
+</para>
+
+<para>
+I think these views further explain what has sometimes been noticed--namely
+that we know nothing about the origin or history of any of our domestic
+breeds.  But, in fact, a breed, like a dialect of a language, can hardly be
+said to have had a definite origin.  A man preserves and breeds from an
+individual with some slight deviation of structure, or takes more care than
+usual in matching his best animals and thus improves them, and the improved
+individuals slowly spread in the immediate neighbourhood.  But as yet they
+will hardly have a distinct name, and from being only slightly valued,
+their history will be disregarded.  When further improved by the same slow
+and gradual process, they will spread more widely, and will get recognised
+as something distinct and valuable, and will then probably first receive a
+provincial name.  In semi-civilised countries, with little free
+communication, the spreading and knowledge of any new sub-breed will be a
+slow process.  As soon as the points of value of the new sub-breed are once
+fully acknowledged, the principle, as I have called it, of unconscious
+selection will always tend,--perhaps more at one period than at another, as
+the breed rises or falls in fashion,--perhaps more in one district than in
+another, according to the state of civilisation of the inhabitants--slowly
+to add to the characteristic features of the breed, whatever they may be. 
+But the chance will be infinitely small of any record having been preserved
+of such slow, varying, and insensible changes.
+</para>
+
+<para>
+I must now say a few words on the circumstances, favourable, or the
+reverse, to man's power of selection.  A high degree of variability is
+obviously favourable, as freely giving the materials for selection to work
+on; not that mere individual differences are not amply sufficient, with
+extreme care, to allow of the accumulation of a large amount of
+modification in almost any desired direction.  But as variations manifestly
+useful or pleasing to man appear only occasionally, the chance of their
+appearance will be much increased by a large number of individuals being
+kept; and hence this comes to be of the highest importance to success.  On
+this principle Marshall has remarked, with respect to the sheep of parts of
+Yorkshire, that 'as they generally belong to poor people, and are mostly in
+small lots, they never can be improved.'  On the other hand, nurserymen,
+from raising large stocks of the same plants, are generally far more
+successful than amateurs in getting new and valuable varieties.  The
+keeping of a large number of individuals of a species in any country
+requires that the species should be placed under favourable conditions of
+life, so as to breed freely in that country.  When the individuals of any
+species are scanty, all the individuals, whatever their quality may be,
+will generally be allowed to breed, and this will effectually prevent
+selection.  But probably the most important point of all, is, that the
+animal or plant should be so highly useful to man, or so much valued by
+him, that the closest attention should be paid to even the slightest
+deviation in the qualities or structure of each individual.  Unless such
+attention be paid nothing can be effected.  I have seen it gravely
+remarked, that it was most fortunate that the strawberry began to vary just
+when gardeners began to attend closely to this plant.  No doubt the
+strawberry had always varied since it was cultivated, but the slight
+varieties had been neglected.  As soon, however, as gardeners picked out
+individual plants with slightly larger, earlier, or better fruit, and
+raised seedlings from them, and again picked out the best seedlings and
+bred from them, then, there appeared (aided by some crossing with distinct
+species) those many admirable varieties of the strawberry which have been
+raised during the last thirty or forty years.
+</para>
+
+<para>
+In the case of animals with separate sexes, facility in preventing crosses
+is an important element of success in the formation of new races,--at
+least, in a country which is already stocked with other races.  In this
+respect enclosure of the land plays a part.  Wandering savages or the
+inhabitants of open plains rarely possess more than one breed of the same
+species.  Pigeons can be mated for life, and this is a great convenience to
+the fancier, for thus many races may be kept true, though mingled in the
+same aviary; and this circumstance must have largely favoured the
+improvement and formation of new breeds.  Pigeons, I may add, can be
+propagated in great numbers and at a very quick rate, and inferior birds
+may be freely rejected, as when killed they serve for food.  On the other
+hand, cats, from their nocturnal rambling habits, cannot be matched, and,
+although so much valued by women and children, we hardly ever see a
+distinct breed kept up; such breeds as we do sometimes see are almost
+always imported from some other country, often from islands.  Although I do
+not doubt that some domestic animals vary less than others, yet the rarity
+or absence of distinct breeds of the cat, the donkey, peacock, goose, &amp;c.,
+may be attributed in main part to selection not having been brought into
+play:  in cats, from the difficulty in pairing them; in donkeys, from only
+a few being kept by poor people, and little attention paid to their
+breeding; in peacocks, from not being very easily reared and a large stock
+not kept; in geese, from being valuable only for two purposes, food and
+feathers, and more especially from no pleasure having been felt in the
+display of distinct breeds.
+</para>
+
+<para>
+To sum up on the origin of our Domestic Races of animals and plants.  I
+believe that the conditions of life, from their action on the reproductive
+system, are so far of the highest importance as causing variability.  I do
+not believe that variability is an inherent and necessary contingency,
+under all circumstances, with all organic beings, as some authors have
+thought.  The effects of variability are modified by various degrees of
+inheritance and of reversion.  Variability is governed by many unknown
+laws, more especially by that of correlation of growth.  Something may be
+attributed to the direct action of the conditions of life.  Something must
+be attributed to use and disuse.  The final result is thus rendered
+infinitely complex.  In some cases, I do not doubt that the intercrossing
+of species, aboriginally distinct, has played an important part in the
+origin of our domestic productions.  When in any country several domestic
+breeds have once been established, their occasional intercrossing, with the
+aid of selection, has, no doubt, largely aided in the formation of new
+sub-breeds; but the importance of the crossing of varieties has, I believe,
+been greatly exaggerated, both in regard to animals and to those plants
+which are propagated by seed.  In plants which are temporarily propagated
+by cuttings, buds, &amp;c., the importance of the crossing both of distinct
+species and of varieties is immense; for the cultivator here quite
+disregards the extreme variability both of hybrids and mongrels, and the
+frequent sterility of hybrids; but the cases of plants not propagated by
+seed are of little importance to us, for their endurance is only temporary. 
+Over all these causes of Change I am convinced that the accumulative action
+of Selection, whether applied methodically and more quickly, or
+unconsciously and more slowly, but more efficiently, is by far the
+predominant Power.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter II</chapnum>
+
+<title>Variation Under Nature</title>
+
+<chapsummary>
+Variability -- Individual differences -- Doubtful species -- Wide ranging,
+much diffused, and common species vary most -- Species of the larger genera
+in any country vary more than the species of the smaller genera -- Many of
+the species of the larger genera resemble varieties in being very closely,
+but unequally, related to each other, and in having restricted ranges.
+</chapsummary>
+</chapheader>
+
+<para>
+Before applying the principles arrived at in the last chapter to organic
+beings in a state of nature, we must briefly discuss whether these latter
+are subject to any variation.  To treat this subject at all properly, a
+long catalogue of dry facts should be given; but these I shall reserve for
+my future work.  Nor shall I here discuss the various definitions which
+have been given of the term species.  No one definition has as yet
+satisfied all naturalists; yet every naturalist knows vaguely what he means
+when he speaks of a species.  Generally the term includes the unknown
+element of a distinct act of creation.  The term 'variety' is almost
+equally difficult to define; but here community of descent is almost
+universally implied, though it can rarely be proved.  We have also what are
+called monstrosities; but they graduate into varieties.  By a monstrosity I
+presume is meant some considerable deviation of structure in one part,
+either injurious to or not useful to the species, and not generally
+propagated.  Some authors use the term 'variation' in a technical sense, as
+implying a modification directly due to the physical conditions of life;
+and 'variations' in this sense are supposed not to be inherited:  but who
+can say that the dwarfed condition of shells in the brackish waters of the
+Baltic, or dwarfed plants on Alpine summits, or the thicker fur of an
+animal from far northwards, would not in some cases be inherited for at
+least some few generations? and in this case I presume that the form would
+be called a variety.
+</para>
+
+<para>
+Again, we have many slight differences which may be called individual
+differences, such as are known frequently to appear in the offspring from
+the same parents, or which may be presumed to have thus arisen, from being
+frequently observed in the individuals of the same species inhabiting the
+same confined locality.  No one supposes that all the individuals of the
+same species are cast in the very same mould.  These individual differences
+are highly important for us, as they afford materials for natural selection
+to accumulate, in the same manner as man can accumulate in any given
+direction individual differences in his domesticated productions.  These
+individual differences generally affect what naturalists consider
+unimportant parts; but I could show by a long catalogue of facts, that
+parts which must be called important, whether viewed under a physiological
+or classificatory point of view, sometimes vary in the individuals of the
+same species.  I am convinced that the most experienced naturalist would be
+surprised at the number of the cases of variability, even in important
+parts of structure, which he could collect on good authority, as I have
+collected, during a course of years.  It should be remembered that
+systematists are far from pleased at finding variability in important
+characters, and that there are not many men who will laboriously examine
+internal and important organs, and compare them in many specimens of the
+same species.  I should never have expected that the branching of the main
+nerves close to the great central ganglion of an insect would have been
+variable in the same species; I should have expected that changes of this
+nature could have been effected only by slow degrees:  yet quite recently
+Mr. Lubbock has shown a degree of variability in these main nerves in
+Coccus, which may almost be compared to the irregular branching of the stem
+of a tree.  This philosophical naturalist, I may add, has also quite
+recently shown that the muscles in the larvae of certain insects are very
+far from uniform.  Authors sometimes argue in a circle when they state that
+important organs never vary; for these same authors practically rank that
+character as important (as some few naturalists have honestly confessed)
+which does not vary; and, under this point of view, no instance of any
+important part varying will ever be found:  but under any other point of
+view many instances assuredly can be given.
+</para>
+
+<para>
+There is one point connected with individual differences, which seems to me
+extremely perplexing:  I refer to those genera which have sometimes been
+called 'protean' or 'polymorphic,' in which the species present an
+inordinate amount of variation; and hardly two naturalists can agree which
+forms to rank as species and which as varieties.  We may instance Rubus,
+Rosa, and Hieracium amongst plants, several genera of insects, and several
+genera of Brachiopod shells.  In most polymorphic genera some of the
+species have fixed and definite characters.  Genera which are polymorphic
+in one country seem to be, with some few exceptions, polymorphic in other
+countries, and likewise, judging from Brachiopod shells, at former periods
+of time.  These facts seem to be very perplexing, for they seem to show
+that this kind of variability is independent of the conditions of life.  I
+am inclined to suspect that we see in these polymorphic genera variations
+in points of structure which are of no service or disservice to the
+species, and which consequently have not been seized on and rendered
+definite by natural selection, as hereafter will be explained.
+</para>
+
+<para>
+Those forms which possess in some considerable degree the character of
+species, but which are so closely similar to some other forms, or are so
+closely linked to them by intermediate gradations, that naturalists do not
+like to rank them as distinct species, are in several respects the most
+important for us.  We have every reason to believe that many of these
+doubtful and closely-allied forms have permanently retained their
+characters in their own country for a long time; for as long, as far as we
+know, as have good and true species.  Practically, when a naturalist can
+unite two forms together by others having intermediate characters, he
+treats the one as a variety of the other, ranking the most common, but
+sometimes the one first described, as the species, and the other as the
+variety.  But cases of great difficulty, which I will not here enumerate,
+sometimes occur in deciding whether or not to rank one form as a variety of
+another, even when they are closely connected by intermediate links; nor
+will the commonly-assumed hybrid nature of the intermediate links always
+remove the difficulty.  In very many cases, however, one form is ranked as
+a variety of another, not because the intermediate links have actually been
+found, but because analogy leads the observer to suppose either that they
+do now somewhere exist, or may formerly have existed; and here a wide door
+for the entry of doubt and conjecture is opened.
+</para>
+
+<para>
+Hence, in determining whether a form should be ranked as a species or a
+variety, the opinion of naturalists having sound judgment and wide
+experience seems the only guide to follow.  We must, however, in many
+cases, decide by a majority of naturalists, for few well-marked and
+well-known varieties can be named which have not been ranked as species by
+at least some competent judges.
+</para>
+
+<para>
+That varieties of this doubtful nature are far from uncommon cannot be
+disputed.  Compare the several floras of Great Britain, of France or of the
+United States, drawn up by different botanists, and see what a surprising
+number of forms have been ranked by one botanist as good species, and by
+another as mere varieties.  Mr. H. C. Watson, to whom I lie under deep
+obligation for assistance of all kinds, has marked for me 182 British
+plants, which are generally considered as varieties, but which have all
+been ranked by botanists as species; and in making this list he has omitted
+many trifling varieties, but which nevertheless have been ranked by some
+botanists as species, and he has entirely omitted several highly
+polymorphic genera.  Under genera, including the most polymorphic forms,
+Mr. Babington gives 251 species, whereas Mr. Bentham gives only 112,--a
+difference of 139 doubtful forms!  Amongst animals which unite for each
+birth, and which are highly locomotive, doubtful forms, ranked by one
+zoologist as a species and by another as a variety, can rarely be found
+within the same country, but are common in separated areas.  How many of
+those birds and insects in North America and Europe, which differ very
+slightly from each other, have been ranked by one eminent naturalist as
+undoubted species, and by another as varieties, or, as they are often
+called, as geographical races!  Many years ago, when comparing, and seeing
+others compare, the birds from the separate islands of the Galapagos
+Archipelago, both one with another, and with those from the American
+mainland, I was much struck how entirely vague and arbitrary is the
+distinction between species and varieties.  On the islets of the little
+Madeira group there are many insects which are characterized as varieties
+in Mr. Wollaston's admirable work, but which it cannot be doubted would be
+ranked as distinct species by many entomologists.  Even Ireland has a few
+animals, now generally regarded as varieties, but which have been ranked as
+species by some zoologists.  Several most experienced ornithologists
+consider our British red grouse as only a strongly-marked race of a
+Norwegian species, whereas the greater number rank it as an undoubted
+species peculiar to Great Britain.  A wide distance between the homes of
+two doubtful forms leads many naturalists to rank both as distinct species;
+but what distance, it has been well asked, will suffice? if that between
+America and Europe is ample, will that between the Continent and the
+Azores, or Madeira, or the Canaries, or Ireland, be sufficient?  It must be
+admitted that many forms, considered by highly-competent judges as
+varieties, have so perfectly the character of species that they are ranked
+by other highly-competent judges as good and true species.  But to discuss
+whether they are rightly called species or varieties, before any definition
+of these terms has been generally accepted, is vainly to beat the air.
+</para>
+
+<para>
+Many of the cases of strongly-marked varieties or doubtful species well
+deserve consideration; for several interesting lines of argument, from
+geographical distribution, analogical variation, hybridism, &amp;c., have been
+brought to bear on the attempt to determine their rank.  I will here give
+only a single instance,--the well-known one of the primrose and cowslip, or
+Primula veris and elatior.  These plants differ considerably in appearance;
+they have a different flavour and emit a different odour; they flower at
+slightly different periods; they grow in somewhat different stations; they
+ascend mountains to different heights; they have different geographical
+ranges; and lastly, according to very numerous experiments made during
+several years by that most careful observer Gartner, they can be crossed
+only with much difficulty.  We could hardly wish for better evidence of the
+two forms being specifically distinct.  On the other hand, they are united
+by many intermediate links, and it is very doubtful whether these links are
+hybrids; and there is, as it seems to me, an overwhelming amount of
+experimental evidence, showing that they descend from common parents, and
+consequently must be ranked as varieties.
+</para>
+
+<para>
+Close investigation, in most cases, will bring naturalists to an agreement
+how to rank doubtful forms.  Yet it must be confessed, that it is in the
+best-known countries that we find the greatest number of forms of doubtful
+value.  I have been struck with the fact, that if any animal or plant in a
+state of nature be highly useful to man, or from any cause closely attract
+his attention, varieties of it will almost universally be found recorded. 
+These varieties, moreover, will be often ranked by some authors as species. 
+Look at the common oak, how closely it has been studied; yet a German
+author makes more than a dozen species out of forms, which are very
+generally considered as varieties; and in this country the highest
+botanical authorities and practical men can be quoted to show that the
+sessile and pedunculated oaks are either good and distinct species or mere
+varieties.
+</para>
+
+<para>
+When a young naturalist commences the study of a group of organisms quite
+unknown to him, he is at first much perplexed to determine what differences
+to consider as specific, and what as varieties; for he knows nothing of the
+amount and kind of variation to which the group is subject; and this shows,
+at least, how very generally there is some variation.  But if he confine
+his attention to one class within one country, he will soon make up his
+mind how to rank most of the doubtful forms.  His general tendency will be
+to make many species, for he will become impressed, just like the pigeon or
+poultry-fancier before alluded to, with the amount of difference in the
+forms which he is continually studying; and he has little general knowledge
+of analogical variation in other groups and in other countries, by which to
+correct his first impressions.  As he extends the range of his
+observations, he will meet with more cases of difficulty; for he will
+encounter a greater number of closely-allied forms.  But if his
+observations be widely extended, he will in the end generally be enabled to
+make up his own mind which to call varieties and which species; but he will
+succeed in this at the expense of admitting much variation,--and the truth
+of this admission will often be disputed by other naturalists.  When,
+moreover, he comes to study allied forms brought from countries not now
+continuous, in which case he can hardly hope to find the intermediate links
+between his doubtful forms, he will have to trust almost entirely to
+analogy, and his difficulties will rise to a climax.
+</para>
+
+<para>
+Certainly no clear line of demarcation has as yet been drawn between
+species and sub-species--that is, the forms which in the opinion of some
+naturalists come very near to, but do not quite arrive at the rank of
+species; or, again, between sub-species and well-marked varieties, or
+between lesser varieties and individual differences.  These differences
+blend into each other in an insensible series; and a series impresses the
+mind with the idea of an actual passage.
+</para>
+
+<para>
+Hence I look at individual differences, though of small interest to the
+systematist, as of high importance for us, as being the first step towards
+such slight varieties as are barely thought worth recording in works on
+natural history.  And I look at varieties which are in any degree more
+distinct and permanent, as steps leading to more strongly marked and more
+permanent varieties; and at these latter, as leading to sub-species, and to
+species.  The passage from one stage of difference to another and higher
+stage may be, in some cases, due merely to the long-continued action of
+different physical conditions in two different regions; but I have not much
+faith in this view; and I attribute the passage of a variety, from a state
+in which it differs very slightly from its parent to one in which it
+differs more, to the action of natural selection in accumulating (as will
+hereafter be more fully explained) differences of structure in certain
+definite directions.  Hence I believe a well-marked variety may be justly
+called an incipient species; but whether this belief be justifiable must be
+judged of by the general weight of the several facts and views given
+throughout this work.
+</para>
+
+<para>
+It need not be supposed that all varieties or incipient species necessarily
+attain the rank of species.  They may whilst in this incipient state become
+extinct, or they may endure as varieties for very long periods, as has been
+shown to be the case by Mr. Wollaston with the varieties of certain fossil
+land-shells in Madeira.  If a variety were to flourish so as to exceed in
+numbers the parent species, it would then rank as the species, and the
+species as the variety; or it might come to supplant and exterminate the
+parent species; or both might co-exist, and both rank as independent
+species.  But we shall hereafter have to return to this subject.
+</para>
+
+<para>
+From these remarks it will be seen that I look at the term species, as one
+arbitrarily given for the sake of convenience to a set of individuals
+closely resembling each other, and that it does not essentially differ from
+the term variety, which is given to less distinct and more fluctuating
+forms.  The term variety, again, in comparison with mere individual
+differences, is also applied arbitrarily, and for mere convenience sake.
+</para>
+
+<para>
+Guided by theoretical considerations, I thought that some interesting
+results might be obtained in regard to the nature and relations of the
+species which vary most, by tabulating all the varieties in several
+well-worked floras.  At first this seemed a simple task; but Mr. H. C.
+Watson, to whom I am much indebted for valuable advice and assistance on
+this subject, soon convinced me that there were many difficulties, as did
+subsequently Dr. Hooker, even in stronger terms.  I shall reserve for my
+future work the discussion of these difficulties, and the tables themselves
+of the proportional numbers of the varying species.  Dr. Hooker permits me
+to add, that after having carefully read my manuscript, and examined the
+tables, he thinks that the following statements are fairly well
+established.  The whole subject, however, treated as it necessarily here is
+with much brevity, is rather perplexing, and allusions cannot be avoided to
+the 'struggle for existence,' 'divergence of character,' and other
+questions, hereafter to be discussed.
+</para>
+
+<para>
+Alph. De Candolle and others have shown that plants which have very wide
+ranges generally present varieties; and this might have been expected, as
+they become exposed to diverse physical conditions, and as they come into
+competition (which, as we shall hereafter see, is a far more important
+circumstance) with different sets of organic beings.  But my tables further
+show that, in any limited country, the species which are most common, that
+is abound most in individuals, and the species which are most widely
+diffused within their own country (and this is a different consideration
+from wide range, and to a certain extent from commonness), often give rise
+to varieties sufficiently well-marked to have been recorded in botanical
+works.  Hence it is the most flourishing, or, as they may be called, the
+dominant species,--those which range widely over the world, are the most
+diffused in their own country, and are the most numerous in
+individuals,--which oftenest produce well-marked varieties, or, as I
+consider them, incipient species.  And this, perhaps, might have been
+anticipated; for, as varieties, in order to become in any degree permanent,
+necessarily have to struggle with the other inhabitants of the country, the
+species which are already dominant will be the most likely to yield
+offspring which, though in some slight degree modified, will still inherit
+those advantages that enabled their parents to become dominant over their
+compatriots.
+</para>
+
+<para>
+If the plants inhabiting a country and described in any Flora be divided
+into two equal masses, all those in the larger genera being placed on one
+side, and all those in the smaller genera on the other side, a somewhat
+larger number of the very common and much diffused or dominant species will
+be found on the side of the larger genera.  This, again, might have been
+anticipated; for the mere fact of many species of the same genus inhabiting
+any country, shows that there is something in the organic or inorganic
+conditions of that country favourable to the genus; and, consequently, we
+might have expected to have found in the larger genera, or those including
+many species, a large proportional number of dominant species.  But so many
+causes tend to obscure this result, that I am surprised that my tables show
+even a small majority on the side of the larger genera.  I will here allude
+to only two causes of obscurity.  Fresh-water and salt-loving plants have
+generally very wide ranges and are much diffused, but this seems to be
+connected with the nature of the stations inhabited by them, and has little
+or no relation to the size of the genera to which the species belong. 
+Again, plants low in the scale of organisation are generally much more
+widely diffused than plants higher in the scale; and here again there is no
+close relation to the size of the genera.  The cause of lowly-organised
+plants ranging widely will be discussed in our chapter on geographical
+distribution.
+</para>
+
+<para>
+From looking at species as only strongly-marked and well-defined varieties,
+I was led to anticipate that the species of the larger genera in each
+country would oftener present varieties, than the species of the smaller
+genera; for wherever many closely related species (i.e. species of the same
+genus) have been formed, many varieties or incipient species ought, as a
+general rule, to be now forming.  Where many large trees grow, we expect to
+find saplings.  Where many species of a genus have been formed through
+variation, circumstances have been favourable for variation; and hence we
+might expect that the circumstances would generally be still favourable to
+variation.  On the other hand, if we look at each species as a special act
+of creation, there is no apparent reason why more varieties should occur in
+a group having many species, than in one having few.
+</para>
+
+<para>
+To test the truth of this anticipation I have arranged the plants of twelve
+countries, and the coleopterous insects of two districts, into two nearly
+equal masses, the species of the larger genera on one side, and those of
+the smaller genera on the other side, and it has invariably proved to be
+the case that a larger proportion of the species on the side of the larger
+genera present varieties, than on the side of the smaller genera. 
+Moreover, the species of the large genera which present any varieties,
+invariably present a larger average number of varieties than do the species
+of the small genera.  Both these results follow when another division is
+made, and when all the smallest genera, with from only one to four species,
+are absolutely excluded from the tables.  These facts are of plain
+signification on the view that species are only strongly marked and
+permanent varieties; for whenever many species of the same genus have been
+formed, or where, if we may use the expression, the manufactory of species
+has been active, we ought generally to find the manufactory still in
+action, more especially as we have every reason to believe the process of
+manufacturing new species to be a slow one.  And this certainly is the
+case, if varieties be looked at as incipient species; for my tables clearly
+show as a general rule that, wherever many species of a genus have been
+formed, the species of that genus present a number of varieties, that is of
+incipient species, beyond the average.  It is not that all large genera are
+now varying much, and are thus increasing in the number of their species,
+or that no small genera are now varying and increasing; for if this had
+been so, it would have been fatal to my theory; inasmuch as geology plainly
+tells us that small genera have in the lapse of time often increased
+greatly in size; and that large genera have often come to their maxima,
+declined, and disappeared.  All that we want to show is, that where many
+species of a genus have been formed, on an average many are still forming;
+and this holds good.
+</para>
+
+<para>
+There are other relations between the species of large genera and their
+recorded varieties which deserve notice.  We have seen that there is no
+infallible criterion by which to distinguish species and well-marked
+varieties; and in those cases in which intermediate links have not been
+found between doubtful forms, naturalists are compelled to come to a
+determination by the amount of difference between them, judging by analogy
+whether or not the amount suffices to raise one or both to the rank of
+species.  Hence the amount of difference is one very important criterion in
+settling whether two forms should be ranked as species or varieties.  Now
+Fries has remarked in regard to plants, and Westwood in regard to insects,
+that in large genera the amount of difference between the species is often
+exceedingly small.  I have endeavoured to test this numerically by
+averages, and, as far as my imperfect results go, they always confirm the
+view.  I have also consulted some sagacious and most experienced observers,
+and, after deliberation, they concur in this view.  In this respect,
+therefore, the species of the larger genera resemble varieties, more than
+do the species of the smaller genera.  Or the case may be put in another
+way, and it may be said, that in the larger genera, in which a number of
+varieties or incipient species greater than the average are now
+manufacturing, many of the species already manufactured still to a certain
+extent resemble varieties, for they differ from each other by a less than
+usual amount of difference.
+</para>
+
+<para>
+Moreover, the species of the large genera are related to each other, in the
+same manner as the varieties of any one species are related to each other. 
+No naturalist pretends that all the species of a genus are equally distinct
+from each other; they may generally be divided into sub-genera, or
+sections, or lesser groups.  As Fries has well remarked, little groups of
+species are generally clustered like satellites around certain other
+species.  And what are varieties but groups of forms, unequally related to
+each other, and clustered round certain forms--that is, round their
+parent-species?  Undoubtedly there is one most important point of
+difference between varieties and species; namely, that the amount of
+difference between varieties, when compared with each other or with their
+parent-species, is much less than that between the species of the same
+genus.  But when we come to discuss the principle, as I call it, of
+Divergence of Character, we shall see how this may be explained, and how
+the lesser differences between varieties will tend to increase into the
+greater differences between species.
+</para>
+
+<para>
+There is one other point which seems to me worth notice.  Varieties
+generally have much restricted ranges:  this statement is indeed scarcely
+more than a truism, for if a variety were found to have a wider range than
+that of its supposed parent-species, their denominations ought to be
+reversed.  But there is also reason to believe, that those species which
+are very closely allied to other species, and in so far resemble varieties,
+often have much restricted ranges.  For instance, Mr. H. C. Watson has
+marked for me in the well-sifted London Catalogue of plants (4th edition)
+63 plants which are therein ranked as species, but which he considers as so
+closely allied to other species as to be of doubtful value:  these 63
+reputed species range on an average over 6.9 of the provinces into which
+Mr. Watson has divided Great Britain.  Now, in this same catalogue, 53
+acknowledged varieties are recorded, and these range over 7.7 provinces;
+whereas, the species to which these varieties belong range over 14.3
+provinces.  So that the acknowledged varieties have very nearly the same
+restricted average range, as have those very closely allied forms, marked
+for me by Mr. Watson as doubtful species, but which are almost universally
+ranked by British botanists as good and true species.
+</para>
+
+<para>
+Finally, then, varieties have the same general characters as species, for
+they cannot be distinguished from species,--except, firstly, by the
+discovery of intermediate linking forms, and the occurrence of such links
+cannot affect the actual characters of the forms which they connect; and
+except, secondly, by a certain amount of difference, for two forms, if
+differing very little, are generally ranked as varieties, notwithstanding
+that intermediate linking forms have not been discovered; but the amount of
+difference considered necessary to give to two forms the rank of species is
+quite indefinite.  In genera having more than the average number of species
+in any country, the species of these genera have more than the average
+number of varieties.  In large genera the species are apt to be closely,
+but unequally, allied together, forming little clusters round certain
+species.  Species very closely allied to other species apparently have
+restricted ranges.  In all these several respects the species of large
+genera present a strong analogy with varieties.  And we can clearly
+understand these analogies, if species have once existed as varieties, and
+have thus originated:  whereas, these analogies are utterly inexplicable if
+each species has been independently created.
+</para>
+
+<para>
+We have, also, seen that it is the most flourishing and dominant species of
+the larger genera which on an average vary most; and varieties, as we shall
+hereafter see, tend to become converted into new and distinct species.  The
+larger genera thus tend to become larger; and throughout nature the forms
+of life which are now dominant tend to become still more dominant by
+leaving many modified and dominant descendants.  But by steps hereafter to
+be explained, the larger genera also tend to break up into smaller genera. 
+And thus, the forms of life throughout the universe become divided into
+groups subordinate to groups.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter III</chapnum>
+
+<title>Struggle for Existence</title>
+
+<chapsummary>
+Bears on natural selection -- The term used in a wide sense -- Geometrical
+powers of increase -- Rapid increase of naturalised animals and plants --
+Nature of the checks to increase -- Competition universal -- Effects of
+climate -- Protection from the number of individuals -- Complex relations
+of all animals and plants throughout nature -- Struggle for life most
+severe between individuals and varieties of the same species; often severe
+between species of the same genus -- The relation of organism to organism
+the most important of all relations.
+</chapsummary>
+</chapheader>
+
+<para> 
+Before entering on the subject of this chapter, I must make a few
+preliminary remarks, to show how the struggle for existence bears on
+Natural Selection.  It has been seen in the last chapter that amongst
+organic beings in a state of nature there is some individual variability;
+indeed I am not aware that this has ever been disputed.  It is immaterial
+for us whether a multitude of doubtful forms be called species or
+sub-species or varieties; what rank, for instance, the two or three hundred
+doubtful forms of British plants are entitled to hold, if the existence of
+any well-marked varieties be admitted.  But the mere existence of
+individual variability and of some few well-marked varieties, though
+necessary as the foundation for the work, helps us but little in
+understanding how species arise in nature.  How have all those exquisite
+adaptations of one part of the organisation to another part, and to the
+conditions of life, and of one distinct organic being to another being,
+been perfected?  We see these beautiful co-adaptations most plainly in the
+woodpecker and missletoe; and only a little less plainly in the humblest
+parasite which clings to the hairs of a quadruped or feathers of a bird; in
+the structure of the beetle which dives through the water; in the plumed
+seed which is wafted by the gentlest breeze; in short, we see beautiful
+adaptations everywhere and in every part of the organic world.
+</para>
+
+<para>
+Again, it may be asked, how is it that varieties, which I have called
+incipient species, become ultimately converted into good and distinct
+species, which in most cases obviously differ from each other far more than
+do the varieties of the same species?  How do those groups of species,
+which constitute what are called distinct genera, and which differ from
+each other more than do the species of the same genus, arise?  All these
+results, as we shall more fully see in the next chapter, follow inevitably
+from the struggle for life.  Owing to this struggle for life, any
+variation, however slight and from whatever cause proceeding, if it be in
+any degree profitable to an individual of any species, in its infinitely
+complex relations to other organic beings and to external nature, will tend
+to the preservation of that individual, and will generally be inherited by
+its offspring.  The offspring, also, will thus have a better chance of
+surviving, for, of the many individuals of any species which are
+periodically born, but a small number can survive.  I have called this
+principle, by which each slight variation, if useful, is preserved, by the
+term of Natural Selection, in order to mark its relation to man's power of
+selection.  We have seen that man by selection can certainly produce great
+results, and can adapt organic beings to his own uses, through the
+accumulation of slight but useful variations, given to him by the hand of
+Nature.  But Natural Selection, as we shall hereafter see, is a power
+incessantly ready for action, and is as immeasurably superior to man's
+feeble efforts, as the works of Nature are to those of Art.
+</para>
+
+<para>
+We will now discuss in a little more detail the struggle for existence.  In
+my future work this subject shall be treated, as it well deserves, at much
+greater length.  The elder De Candolle and Lyell have largely and
+philosophically shown that all organic beings are exposed to severe
+competition.  In regard to plants, no one has treated this subject with
+more spirit and ability than W. Herbert, Dean of Manchester, evidently the
+result of his great horticultural knowledge.  Nothing is easier than to
+admit in words the truth of the universal struggle for life, or more
+difficult--at least I have found it so--than constantly to bear this
+conclusion in mind.  Yet unless it be thoroughly engrained in the mind, I
+am convinced that the whole economy of nature, with every fact on
+distribution, rarity, abundance, extinction, and variation, will be dimly
+seen or quite misunderstood.  We behold the face of nature bright with
+gladness, we often see superabundance of food; we do not see, or we forget,
+that the birds which are idly singing round us mostly live on insects or
+seeds, and are thus constantly destroying life; or we forget how largely
+these songsters, or their eggs, or their nestlings, are destroyed by birds
+and beasts of prey; we do not always bear in mind, that though food may be
+now superabundant, it is not so at all seasons of each recurring year.
+</para>
+
+<para>
+I should premise that I use the term Struggle for Existence in a large and
+metaphorical sense, including dependence of one being on another, and
+including (which is more important) not only the life of the individual,
+but success in leaving progeny.  Two canine animals in a time of dearth,
+may be truly said to struggle with each other which shall get food and
+live.  But a plant on the edge of a desert is said to struggle for life
+against the drought, though more properly it should be said to be dependent
+on the moisture.  A plant which annually produces a thousand seeds, of
+which on an average only one comes to maturity, may be more truly said to
+struggle with the plants of the same and other kinds which already clothe
+the ground.  The missletoe is dependent on the apple and a few other trees,
+but can only in a far-fetched sense be said to struggle with these trees,
+for if too many of these parasites grow on the same tree, it will languish
+and die.  But several seedling missletoes, growing close together on the
+same branch, may more truly be said to struggle with each other.  As the
+missletoe is disseminated by birds, its existence depends on birds; and it
+may metaphorically be said to struggle with other fruit-bearing plants, in
+order to tempt birds to devour and thus disseminate its seeds rather than
+those of other plants.  In these several senses, which pass into each
+other, I use for convenience sake the general term of struggle for
+existence.
+</para>
+
+<para>
+A struggle for existence inevitably follows from the high rate at which all
+organic beings tend to increase.  Every being, which during its natural
+lifetime produces several eggs or seeds, must suffer destruction during
+some period of its life, and during some season or occasional year,
+otherwise, on the principle of geometrical increase, its numbers would
+quickly become so inordinately great that no country could support the
+product.  Hence, as more individuals are produced than can possibly
+survive, there must in every case be a struggle for existence, either one
+individual with another of the same species, or with the individuals of
+distinct species, or with the physical conditions of life.  It is the
+doctrine of Malthus applied with manifold force to the whole animal and
+vegetable kingdoms; for in this case there can be no artificial increase of
+food, and no prudential restraint from marriage.  Although some species may
+be now increasing, more or less rapidly, in numbers, all cannot do so, for
+the world would not hold them.
+</para>
+
+<para>
+There is no exception to the rule that every organic being naturally
+increases at so high a rate, that if not destroyed, the earth would soon be
+covered by the progeny of a single pair.  Even slow-breeding man has
+doubled in twenty-five years, and at this rate, in a few thousand years,
+there would literally not be standing room for his progeny.  Linnaeus has
+calculated that if an annual plant produced only two seeds--and there is no
+plant so unproductive as this--and their seedlings next year produced two,
+and so on, then in twenty years there would be a million plants.  The
+elephant is reckoned to be the slowest breeder of all known animals, and I
+have taken some pains to estimate its probable minimum rate of natural
+increase:  it will be under the mark to assume that it breeds when thirty
+years old, and goes on breeding till ninety years old, bringing forth three
+pair of young in this interval; if this be so, at the end of the fifth
+century there would be alive fifteen million elephants, descended from the
+first pair.
+</para>
+
+<para>
+But we have better evidence on this subject than mere theoretical
+calculations, namely, the numerous recorded cases of the astonishingly
+rapid increase of various animals in a state of nature, when circumstances
+have been favourable to them during two or three following seasons.  Still
+more striking is the evidence from our domestic animals of many kinds which
+have run wild in several parts of the world:  if the statements of the rate
+of increase of slow-breeding cattle and horses in South America, and
+latterly in Australia, had not been well authenticated, they would have
+been quite incredible.  So it is with plants:  cases could be given of
+introduced plants which have become common throughout whole islands in a
+period of less than ten years.  Several of the plants now most numerous
+over the wide plains of La Plata, clothing square leagues of surface almost
+to the exclusion of all other plants, have been introduced from Europe; and
+there are plants which now range in India, as I hear from Dr. Falconer,
+from Cape Comorin to the Himalaya, which have been imported from America
+since its discovery.  In such cases, and endless instances could be given,
+no one supposes that the fertility of these animals or plants has been
+suddenly and temporarily increased in any sensible degree.  The obvious
+explanation is that the conditions of life have been very favourable, and
+that there has consequently been less destruction of the old and young, and
+that nearly all the young have been enabled to breed.  In such cases the
+geometrical ratio of increase, the result of which never fails to be
+surprising, simply explains the extraordinarily rapid increase and wide
+diffusion of naturalised productions in their new homes.
+</para>
+
+<para>
+In a state of nature almost every plant produces seed, and amongst animals
+there are very few which do not annually pair.  Hence we may confidently
+assert, that all plants and animals are tending to increase at a
+geometrical ratio, that all would most rapidly stock every station in which
+they could any how exist, and that the geometrical tendency to increase
+must be checked by destruction at some period of life.  Our familiarity
+with the larger domestic animals tends, I think, to mislead us:  we see no
+great destruction falling on them, and we forget that thousands are
+annually slaughtered for food, and that in a state of nature an equal
+number would have somehow to be disposed of.
+</para>
+
+<para>
+The only difference between organisms which annually produce eggs or seeds
+by the thousand, and those which produce extremely few, is, that the
+slow-breeders would require a few more years to people, under favourable
+conditions, a whole district, let it be ever so large.  The condor lays a
+couple of eggs and the ostrich a score, and yet in the same country the
+condor may be the more numerous of the two:  the Fulmar petrel lays but one
+egg, yet it is believed to be the most numerous bird in the world.  One fly
+deposits hundreds of eggs, and another, like the hippobosca, a single one;
+but this difference does not determine how many individuals of the two
+species can be supported in a district.  A large number of eggs is of some
+importance to those species, which depend on a rapidly fluctuating amount
+of food, for it allows them rapidly to increase in number.  But the real
+importance of a large number of eggs or seeds is to make up for much
+destruction at some period of life; and this period in the great majority
+of cases is an early one.  If an animal can in any way protect its own eggs
+or young, a small number may be produced, and yet the average stock be
+fully kept up; but if many eggs or young are destroyed, many must be
+produced, or the species will become extinct.  It would suffice to keep up
+the full number of a tree, which lived on an average for a thousand years,
+if a single seed were produced once in a thousand years, supposing that
+this seed were never destroyed, and could be ensured to germinate in a
+fitting place.  So that in all cases, the average number of any animal or
+plant depends only indirectly on the number of its eggs or seeds.
+</para>
+
+<para>
+In looking at Nature, it is most necessary to keep the foregoing
+considerations always in mind--never to forget that every single organic
+being around us may be said to be striving to the utmost to increase in
+numbers; that each lives by a struggle at some period of its life; that
+heavy destruction inevitably falls either on the young or old, during each
+generation or at recurrent intervals.  Lighten any check, mitigate the 
+destruction ever so little, and the number of the species will almost
+instantaneously increase to any amount.  The face of Nature may be compared
+to a yielding surface, with ten thousand sharp wedges packed close together
+and driven inwards by incessant blows, sometimes one wedge being struck,
+and then another with greater force.
+</para>
+
+<para>
+What checks the natural tendency of each species to increase in number is
+most obscure.  Look at the most vigorous species; by as much as it swarms
+in numbers, by so much will its tendency to increase be still further
+increased.  We know not exactly what the checks are in even one single
+instance.  Nor will this surprise any one who reflects how ignorant we are
+on this head, even in regard to mankind, so incomparably better known than
+any other animal.  This subject has been ably treated by several authors,
+and I shall, in my future work, discuss some of the checks at considerable
+length, more especially in regard to the feral animals of South America. 
+Here I will make only a few remarks, just to recall to the reader's mind
+some of the chief points.  Eggs or very young animals seem generally to
+suffer most, but this is not invariably the case.  With plants there is a
+vast destruction of seeds, but, from some observations which I have made, I
+believe that it is the seedlings which suffer most from germinating in
+ground already thickly stocked with other plants.  Seedlings, also, are
+destroyed in vast numbers by various enemies; for instance, on a piece of
+ground three feet long and two wide, dug and cleared, and where there could
+be no choking from other plants, I marked all the seedlings of our native
+weeds as they came up, and out of the 357 no less than 295 were destroyed,
+chiefly by slugs and insects.  If turf which has long been mown, and the
+case would be the same with turf closely browsed by quadrupeds, be let to
+grow, the more vigorous plants gradually kill the less vigorous, though
+fully grown, plants:  thus out of twenty species growing on a little plot
+of turf (three feet by four) nine species perished from the other species
+being allowed to grow up freely.
+</para>
+
+<para>
+The amount of food for each species of course gives the extreme limit to
+which each can increase; but very frequently it is not the obtaining food,
+but the serving as prey to other animals, which determines the average
+numbers of a species.  Thus, there seems to be little doubt that the stock
+of partridges, grouse, and hares on any large estate depends chiefly on the
+destruction of vermin.  If not one head of game were shot during the next
+twenty years in England, and, at the same time, if no vermin were
+destroyed, there would, in all probability, be less game than at present,
+although hundreds of thousands of game animals are now annually killed.  On
+the other hand, in some cases, as with the elephant and rhinoceros, none
+are destroyed by beasts of prey:  even the tiger in India most rarely dares
+to attack a young elephant protected by its dam.
+</para>
+
+<para>
+Climate plays an important part in determining the average numbers of a
+species, and periodical seasons of extreme cold or drought, I believe to be
+the most effective of all checks.  I estimated that the winter of 1854-55
+destroyed four-fifths of the birds in my own grounds; and this is a
+tremendous destruction, when we remember that ten per cent. is an
+extraordinarily severe mortality from epidemics with man.  The action of
+climate seems at first sight to be quite independent of the struggle for
+existence; but in so far as climate chiefly acts in reducing food, it
+brings on the most severe struggle between the individuals, whether of the
+same or of distinct species, which subsist on the same kind of food.  Even
+when climate, for instance extreme cold, acts directly, it will be the
+least vigorous, or those which have got least food through the advancing
+winter, which will suffer most.  When we travel from south to north, or
+from a damp region to a dry, we invariably see some species gradually
+getting rarer and rarer, and finally disappearing; and the change of
+climate being conspicuous, we are tempted to attribute the whole effect to
+its direct action.  But this is a very false view:  we forget that each
+species, even where it most abounds, is constantly suffering enormous
+destruction at some period of its life, from enemies or from competitors
+for the same place and food; and if these enemies or competitors be in the
+least degree favoured by any slight change of climate, they will increase
+in numbers, and, as each area is already fully stocked with inhabitants,
+the other species will decrease.  When we travel southward and see a
+species decreasing in numbers, we may feel sure that the cause lies quite
+as much in other species being favoured, as in this one being hurt.  So it
+is when we travel northward, but in a somewhat lesser degree, for the
+number of species of all kinds, and therefore of competitors, decreases
+northwards; hence in going northward, or in ascending a mountain, we far
+oftener meet with stunted forms, due to the directly injurious action of
+climate, than we do in proceeding southwards or in descending a mountain. 
+When we reach the Arctic regions, or snow-capped summits, or absolute
+deserts, the struggle for life is almost exclusively with the elements.
+</para>
+
+<para>
+That climate acts in main part indirectly by favouring other species, we
+may clearly see in the prodigious number of plants in our gardens which can
+perfectly well endure our climate, but which never become naturalised, for
+they cannot compete with our native plants, nor resist destruction by our
+native animals.
+</para>
+
+<para>
+When a species, owing to highly favourable circumstances, increases
+inordinately in numbers in a small tract, epidemics--at least, this seems
+generally to occur with our game animals--often ensue:  and here we have a
+limiting check independent of the struggle for life.  But even some of
+these so-called epidemics appear to be due to parasitic worms, which have
+from some cause, possibly in part through facility of diffusion amongst the
+crowded animals, been disproportionably favoured:  and here comes in a sort
+of struggle between the parasite and its prey.
+</para>
+
+<para>
+On the other hand, in many cases, a large stock of individuals of the same
+species, relatively to the numbers of its enemies, is absolutely necessary
+for its preservation.  Thus we can easily raise plenty of corn and
+rape-seed, &amp;c., in our fields, because the seeds are in great excess
+compared with the number of birds which feed on them; nor can the birds,
+though having a superabundance of food at this one season, increase in
+number proportionally to the supply of seed, as their numbers are checked
+during winter:  but any one who has tried, knows how troublesome it is to
+get seed from a few wheat or other such plants in a garden; I have in this
+case lost every single seed.  This view of the necessity of a large stock
+of the same species for its preservation, explains, I believe, some
+singular facts in nature, such as that of very rare plants being sometimes
+extremely abundant in the few spots where they do occur; and that of some
+social plants being social, that is, abounding in individuals, even on the
+extreme confines of their range.  For in such cases, we may believe, that a
+plant could exist only where the conditions of its life were so favourable
+that many could exist together, and thus save each other from utter
+destruction.  I should add that the good effects of frequent intercrossing,
+and the ill effects of close interbreeding, probably come into play in some
+of these cases; but on this intricate subject I will not here enlarge.
+</para>
+
+<para>
+Many cases are on record showing how complex and unexpected are the checks
+and relations between organic beings, which have to struggle together in
+the same country.  I will give only a single instance, which, though a
+simple one, has interested me.  In Staffordshire, on the estate of a
+relation where I had ample means of investigation, there was a large and
+extremely barren heath, which had never been touched by the hand of man;
+but several hundred acres of exactly the same nature had been enclosed
+twenty-five years previously and planted with Scotch fir.  The change in
+the native vegetation of the planted part of the heath was most remarkable,
+more than is generally seen in passing from one quite different soil to
+another:  not only the proportional numbers of the heath-plants were wholly
+changed, but twelve species of plants (not counting grasses and carices)
+flourished in the plantations, which could not be found on the heath.  The
+effect on the insects must have been still greater, for six insectivorous
+birds were very common in the plantations, which were not to be seen on the
+heath; and the heath was frequented by two or three distinct insectivorous
+birds.  Here we see how potent has been the effect of the introduction of a
+single tree, nothing whatever else having been done, with the exception
+that the land had been enclosed, so that cattle could not enter.  But how
+important an element enclosure is, I plainly saw near Farnham, in Surrey. 
+Here there are extensive heaths, with a few clumps of old Scotch firs on
+the distant hill-tops:  within the last ten years large spaces have been
+enclosed, and self-sown firs are now springing up in multitudes, so close
+together that all cannot live.
+</para>
+
+<para>
+When I ascertained that these young trees had not been sown or planted, I
+was so much surprised at their numbers that I went to several points of
+view, whence I could examine hundreds of acres of the unenclosed heath, and
+literally I could not see a single Scotch fir, except the old planted
+clumps.  But on looking closely between the stems of the heath, I found a
+multitude of seedlings and little trees, which had been perpetually browsed
+down by the cattle.  In one square yard, at a point some hundreds yards
+distant from one of the old clumps, I counted thirty-two little trees; and
+one of them, judging from the rings of growth, had during twenty-six years
+tried to raise its head above the stems of the heath, and had failed.  No
+wonder that, as soon as the land was enclosed, it became thickly clothed
+with vigorously growing young firs.  Yet the heath was so extremely barren
+and so extensive that no one would ever have imagined that cattle would
+have so closely and effectually searched it for food.
+</para>
+
+<para>
+Here we see that cattle absolutely determine the existence of the Scotch
+fir; but in several parts of the world insects determine the existence of
+cattle.  Perhaps Paraguay offers the most curious instance of this; for
+here neither cattle nor horses nor dogs have ever run wild, though they
+swarm southward and northward in a feral state; and Azara and Rengger have
+shown that this is caused by the greater number in Paraguay of a certain
+fly, which lays its eggs in the navels of these animals when first born. 
+The increase of these flies, numerous as they are, must be habitually
+checked by some means, probably by birds.  Hence, if certain insectivorous
+birds (whose numbers are probably regulated by hawks or beasts of prey)
+were to increase in Paraguay, the flies would decrease--then cattle and
+horses would become feral, and this would certainly greatly alter (as
+indeed I have observed in parts of South America) the vegetation:  this
+again would largely affect the insects; and this, as we just have seen in
+Staffordshire, the insectivorous birds, and so onwards in ever-increasing
+circles of complexity.  We began this series by insectivorous birds, and we
+have ended with them.  Not that in nature the relations can ever be as
+simple as this.  Battle within battle must ever be recurring with varying
+success; and yet in the long-run the forces are so nicely balanced, that
+the face of nature remains uniform for long periods of time, though
+assuredly the merest trifle would often give the victory to one organic
+being over another.  Nevertheless so profound is our ignorance, and so high
+our presumption, that we marvel when we hear of the extinction of an
+organic being; and as we do not see the cause, we invoke cataclysms to
+desolate the world, or invent laws on the duration of the forms of life!
+</para>
+
+<para>
+I am tempted to give one more instance showing how plants and animals, most
+remote in the scale of nature, are bound together by a web of complex
+relations.  I shall hereafter have occasion to show that the exotic Lobelia
+fulgens, in this part of England, is never visited by insects, and
+consequently, from its peculiar structure, never can set a seed.  Many of
+our orchidaceous plants absolutely require the visits of moths to remove
+their pollen-masses and thus to fertilise them.  I have, also, reason to
+believe that humble-bees are indispensable to the fertilisation of the
+heartsease (Viola tricolor), for other bees do not visit this flower.  From
+experiments which I have tried, I have found that the visits of bees, if
+not indispensable, are at least highly beneficial to the fertilisation of
+our clovers; but humble-bees alone visit the common red clover (Trifolium
+pratense), as other bees cannot reach the nectar.  Hence I have very little
+doubt, that if the whole genus of humble-bees became extinct or very rare
+in England, the heartsease and red clover would become very rare, or wholly
+disappear.  The number of humble-bees in any district depends in a great
+degree on the number of field-mice, which destroy their combs and nests;
+and Mr. H. Newman, who has long attended to the habits of humble-bees,
+believes that 'more than two thirds of them are thus destroyed all over
+England.'  Now the number of mice is largely dependent, as every one knows,
+on the number of cats; and Mr. Newman says, 'Near villages and small towns
+I have found the nests of humble-bees more numerous than elsewhere, which I
+attribute to the number of cats that destroy the mice.'  Hence it is quite
+credible that the presence of a feline animal in large numbers in a
+district might determine, through the intervention first of mice and then
+of bees, the frequency of certain flowers in that district!
+</para>
+
+<para>
+In the case of every species, many different checks, acting at different
+periods of life, and during different seasons or years, probably come into
+play; some one check or some few being generally the most potent, but all
+concurring in determining the average number or even the existence of the
+species.  In some cases it can be shown that widely-different checks act on
+the same species in different districts.  When we look at the plants and
+bushes clothing an entangled bank, we are tempted to attribute their
+proportional numbers and kinds to what we call chance.  But how false a
+view is this!  Every one has heard that when an American forest is cut
+down, a very different vegetation springs up; but it has been observed that
+the trees now growing on the ancient Indian mounds, in the Southern United
+States, display the same beautiful diversity and proportion of kinds as in
+the surrounding virgin forests.  What a struggle between the several kinds
+of trees must here have gone on during long centuries, each annually
+scattering its seeds by the thousand; what war between insect and
+insect--between insects, snails, and other animals with birds and beasts of
+prey--all striving to increase, and all feeding on each other or on the
+trees or their seeds and seedlings, or on the other plants which first
+clothed the ground and thus checked the growth of the trees!  Throw up a
+handful of feathers, and all must fall to the ground according to definite
+laws; but how simple is this problem compared to the action and reaction of
+the innumerable plants and animals which have determined, in the course of
+centuries, the proportional numbers and kinds of trees now growing on the
+old Indian ruins!
+</para>
+
+<para>
+The dependency of one organic being on another, as of a parasite on its
+prey, lies generally between beings remote in the scale of nature.  This is
+often the case with those which may strictly be said to struggle with each
+other for existence, as in the case of locusts and grass-feeding
+quadrupeds.  But the struggle almost invariably will be most severe between
+the individuals of the same species, for they frequent the same districts,
+require the same food, and are exposed to the same dangers.  In the case of
+varieties of the same species, the struggle will generally be almost
+equally severe, and we sometimes see the contest soon decided:  for
+instance, if several varieties of wheat be sown together, and the mixed
+seed be resown, some of the varieties which best suit the soil or climate,
+or are naturally the most fertile, will beat the others and so yield more
+seed, and will consequently in a few years quite supplant the other
+varieties.  To keep up a mixed stock of even such extremely close varieties
+as the variously coloured sweet-peas, they must be each year harvested
+separately, and the seed then mixed in due proportion, otherwise the weaker
+kinds will steadily decrease in numbers and disappear.  So again with the
+varieties of sheep:  it has been asserted that certain mountain-varieties
+will starve out other mountain-varieties, so that they cannot be kept
+together.  The same result has followed from keeping together different
+varieties of the medicinal leech.  It may even be doubted whether the
+varieties of any one of our domestic plants or animals have so exactly the
+same strength, habits, and constitution, that the original proportions of a
+mixed stock could be kept up for half a dozen generations, if they were
+allowed to struggle together, like beings in a state of nature, and if the
+seed or young were not annually sorted.
+</para>
+
+<para>
+As species of the same genus have usually, though by no means invariably,
+some similarity in habits and constitution, and always in structure, the
+struggle will generally be more severe between species of the same genus,
+when they come into competition with each other, than between species of
+distinct genera.  We see this in the recent extension over parts of the
+United States of one species of swallow having caused the decrease of
+another species.  The recent increase of the missel-thrush in parts of
+Scotland has caused the decrease of the song-thrush.  How frequently we
+hear of one species of rat taking the place of another species under the
+most different climates!  In Russia the small Asiatic cockroach has
+everywhere driven before it its great congener.  One species of charlock
+will supplant another, and so in other cases.  We can dimly see why the
+competition should be most severe between allied forms, which fill nearly
+the same place in the economy of nature; but probably in no one case could
+we precisely say why one species has been victorious over another in the
+great battle of life.
+</para>
+
+<para>
+A corollary of the highest importance may be deduced from the foregoing
+remarks, namely, that the structure of every organic being is related, in
+the most essential yet often hidden manner, to that of all other organic
+beings, with which it comes into competition for food or residence, or from
+which it has to escape, or on which it preys.  This is obvious in the
+structure of the teeth and talons of the tiger; and in that of the legs and
+claws of the parasite which clings to the hair on the tiger's body.  But in
+the beautifully plumed seed of the dandelion, and in the flattened and
+fringed legs of the water-beetle, the relation seems at first confined to
+the elements of air and water.  Yet the advantage of plumed seeds no doubt
+stands in the closest relation to the land being already thickly clothed by
+other plants; so that the seeds may be widely distributed and fall on
+unoccupied ground.  In the water-beetle, the structure of its legs, so well
+adapted for diving, allows it to compete with other aquatic insects, to
+hunt for its own prey, and to escape serving as prey to other animals.
+</para>
+
+<para>
+The store of nutriment laid up within the seeds of many plants seems at
+first sight to have no sort of relation to other plants.  But from the
+strong growth of young plants produced from such seeds (as peas and beans),
+when sown in the midst of long grass, I suspect that the chief use of the
+nutriment in the seed is to favour the growth of the young seedling, whilst
+struggling with other plants growing vigorously all around.
+</para>
+
+<para>
+Look at a plant in the midst of its range, why does it not double or
+quadruple its numbers?  We know that it can perfectly well withstand a
+little more heat or cold, dampness or dryness, for elsewhere it ranges into
+slightly hotter or colder, damper or drier districts.  In this case we can
+clearly see that if we wished in imagination to give the plant the power of
+increasing in number, we should have to give it some advantage over its
+competitors, or over the animals which preyed on it.  On the confines of
+its geographical range, a change of constitution with respect to climate
+would clearly be an advantage to our plant; but we have reason to believe
+that only a few plants or animals range so far, that they are destroyed by
+the rigour of the climate alone.  Not until we reach the extreme confines
+of life, in the arctic regions or on the borders of an utter desert, will
+competition cease.  The land may be extremely cold or dry, yet there will
+be competition between some few species, or between the individuals of the
+same species, for the warmest or dampest spots.
+</para>
+
+<para>
+Hence, also, we can see that when a plant or animal is placed in a new
+country amongst new competitors, though the climate may be exactly the same
+as in its former home, yet the conditions of its life will generally be
+changed in an essential manner.  If we wished to increase its average
+numbers in its new home, we should have to modify it in a different way to
+what we should have done in its native country; for we should have to give
+it some advantage over a different set of competitors or enemies.
+</para>
+
+<para>
+It is good thus to try in our imagination to give any form some advantage
+over another.  Probably in no single instance should we know what to do, so
+as to succeed.  It will convince us of our ignorance on the mutual
+relations of all organic beings; a conviction as necessary, as it seems to
+be difficult to acquire.  All that we can do, is to keep steadily in mind
+that each organic being is striving to increase at a geometrical ratio;
+that each at some period of its life, during some season of the year,
+during each generation or at intervals, has to struggle for life, and to
+suffer great destruction.  When we reflect on this struggle, we may console
+ourselves with the full belief, that the war of nature is not incessant,
+that no fear is felt, that death is generally prompt, and that the
+vigorous, the healthy, and the happy survive and multiply.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter IV</chapnum>
+
+<title>Natural Selection</title>
+
+<chapsummary>
+Natural Selection -- its power compared with man's selection -- its power
+on characters of trifling importance -- its power at all ages and on both
+sexes -- Sexual Selection -- On the generality of intercrosses between
+individuals of the same species -- Circumstances favourable and
+unfavourable to Natural Selection, namely, intercrossing, isolation, number
+of individuals -- Slow action -- Extinction caused by Natural Selection --
+Divergence of Character, related to the diversity of inhabitants of any
+small area, and to naturalisation -- Action of Natural Selection, through
+Divergence of Character and Extinction, on the descendants from a common
+parent -- Explains the Grouping of all organic beings.
+</chapsummary>
+</chapheader>
+
+<para>
+How will the struggle for existence, discussed too briefly in the last
+chapter, act in regard to variation?  Can the principle of selection, which
+we have seen is so potent in the hands of man, apply in nature?  I think we
+shall see that it can act most effectually.  Let it be borne in mind in
+what an endless number of strange peculiarities our domestic productions,
+and, in a lesser degree, those under nature, vary; and how strong the
+hereditary tendency is.  Under domestication, it may be truly said that the
+whole organisation becomes in some degree plastic.  Let it be borne in mind
+how infinitely complex and close-fitting are the mutual relations of all
+organic beings to each other and to their physical conditions of life.  Can
+it, then, be thought improbable, seeing that variations useful to man have
+undoubtedly occurred, that other variations useful in some way to each
+being in the great and complex battle of life, should sometimes occur in
+the course of thousands of generations?  If such do occur, can we doubt
+(remembering that many more individuals are born than can possibly survive)
+that individuals having any advantage, however slight, over others, would
+have the best chance of surviving and of procreating their kind?  On the
+other hand, we may feel sure that any variation in the least degree
+injurious would be rigidly destroyed.  This preservation of favourable
+variations and the rejection of injurious variations, I call Natural
+Selection.  Variations neither useful nor injurious would not be affected
+by natural selection, and would be left a fluctuating element, as perhaps
+we see in the species called polymorphic.
+</para>
+
+<para>
+We shall best understand the probable course of natural selection by taking
+the case of a country undergoing some physical change, for instance, of
+climate.  The proportional numbers of its inhabitants would almost
+immediately undergo a change, and some species might become extinct.  We
+may conclude, from what we have seen of the intimate and complex manner in
+which the inhabitants of each country are bound together, that any change
+in the numerical proportions of some of the inhabitants, independently of
+the change of climate itself, would most seriously affect many of the
+others.  If the country were open on its borders, new forms would certainly
+immigrate, and this also would seriously disturb the relations of some of
+the former inhabitants.  Let it be remembered how powerful the influence of
+a single introduced tree or mammal has been shown to be.  But in the case
+of an island, or of a country partly surrounded by barriers, into which new
+and better adapted forms could not freely enter, we should then have places
+in the economy of nature which would assuredly be better filled up, if some
+of the original inhabitants were in some manner modified; for, had the area
+been open to immigration, these same places would have been seized on by
+intruders.  In such case, every slight modification, which in the course of
+ages chanced to arise, and which in any way favoured the individuals of any
+of the species, by better adapting them to their altered conditions, would
+tend to be preserved; and natural selection would thus have free scope for
+the work of improvement.
+</para>
+
+<para>
+We have reason to believe, as stated in the first chapter, that a change in
+the conditions of life, by specially acting on the reproductive system,
+causes or increases variability; and in the foregoing case the conditions
+of life are supposed to have undergone a change, and this would manifestly
+be favourable to natural selection, by giving a better chance of profitable
+variations occurring; and unless profitable variations do occur, natural
+selection can do nothing.  Not that, as I believe, any extreme amount of
+variability is necessary; as man can certainly produce great results by
+adding up in any given direction mere individual differences, so could
+Nature, but far more easily, from having incomparably longer time at her
+disposal.  Nor do I believe that any great physical change, as of climate,
+or any unusual degree of isolation to check immigration, is actually
+necessary to produce new and unoccupied places for natural selection to
+fill up by modifying and improving some of the varying inhabitants.  For as
+all the inhabitants of each country are struggling together with nicely
+balanced forces, extremely slight modifications in the structure or habits
+of one inhabitant would often give it an advantage over others; and still
+further modifications of the same kind would often still further increase
+the advantage.  No country can be named in which all the native inhabitants
+are now so perfectly adapted to each other and to the physical conditions
+under which they live, that none of them could anyhow be improved; for in
+all countries, the natives have been so far conquered by naturalised
+productions, that they have allowed foreigners to take firm possession of
+the land.  And as foreigners have thus everywhere beaten some of the
+natives, we may safely conclude that the natives might have been modified
+with advantage, so as to have better resisted such intruders.
+</para>
+
+<para>
+As man can produce and certainly has produced a great result by his
+methodical and unconscious means of selection, what may not nature effect? 
+Man can act only on external and visible characters:  nature cares nothing
+for appearances, except in so far as they may be useful to any being.  She
+can act on every internal organ, on every shade of constitutional
+difference, on the whole machinery of life.  Man selects only for his own
+good; Nature only for that of the being which she tends.  Every selected
+character is fully exercised by her; and the being is placed under
+well-suited conditions of life.  Man keeps the natives of many climates in
+the same country; he seldom exercises each selected character in some
+peculiar and fitting manner; he feeds a long and a short beaked pigeon on
+the same food; he does not exercise a long-backed or long-legged quadruped
+in any peculiar manner; he exposes sheep with long and short wool to the
+same climate.  He does not allow the most vigorous males to struggle for
+the females.  He does not rigidly destroy all inferior animals, but
+protects during each varying season, as far as lies in his power, all his
+productions.  He often begins his selection by some half-monstrous form; or
+at least by some modification prominent enough to catch his eye, or to be
+plainly useful to him.  Under nature, the slightest difference of structure
+or constitution may well turn the nicely-balanced scale in the struggle for
+life, and so be preserved.  How fleeting are the wishes and efforts of man!
+how short his time! and consequently how poor will his products be,
+compared with those accumulated by nature during whole geological periods. 
+Can we wonder, then, that nature's productions should be far 'truer' in
+character than man's productions; that they should be infinitely better
+adapted to the most complex conditions of life, and should plainly bear the
+stamp of far higher workmanship?
+</para>
+
+<para>
+It may be said that natural selection is daily and hourly scrutinising,
+throughout the world, every variation, even the slightest; rejecting that
+which is bad, preserving and adding up all that is good; silently and
+insensibly working, whenever and wherever opportunity offers, at the
+improvement of each organic being in relation to its organic and inorganic
+conditions of life.  We see nothing of these slow changes in progress,
+until the hand of time has marked the long lapse of ages, and then so
+imperfect is our view into long past geological ages, that we only see that
+the forms of life are now different from what they formerly were.
+</para>
+
+<para>
+Although natural selection can act only through and for the good of each
+being, yet characters and structures, which we are apt to consider as of
+very trifling importance, may thus be acted on.  When we see leaf-eating
+insects green, and bark-feeders mottled-grey; the alpine ptarmigan white in
+winter, the red-grouse the colour of heather, and the black-grouse that of
+peaty earth, we must believe that these tints are of service to these birds
+and insects in preserving them from danger.  Grouse, if not destroyed at
+some period of their lives, would increase in countless numbers; they are
+known to suffer largely from birds of prey; and hawks are guided by
+eyesight to their prey,--so much so, that on parts of the Continent persons
+are warned not to keep white pigeons, as being the most liable to
+destruction.  Hence I can see no reason to doubt that natural selection
+might be most effective in giving the proper colour to each kind of grouse,
+and in keeping that colour, when once acquired, true and constant.  Nor
+ought we to think that the occasional destruction of an animal of any
+particular colour would produce little effect:  we should remember how
+essential it is in a flock of white sheep to destroy every lamb with the
+faintest trace of black.  In plants the down on the fruit and the colour of
+the flesh are considered by botanists as characters of the most trifling
+importance:  yet we hear from an excellent horticulturist, Downing, that in
+the United States smooth-skinned fruits suffer far more from a beetle, a
+curculio, than those with down; that purple plums suffer far more from a
+certain disease than yellow plums; whereas another disease attacks
+yellow-fleshed peaches far more than those with other coloured flesh.  If,
+with all the aids of art, these slight differences make a great difference
+in cultivating the several varieties, assuredly, in a state of nature,
+where the trees would have to struggle with other trees and with a host of
+enemies, such differences would effectually settle which variety, whether a
+smooth or downy, a yellow or purple fleshed fruit, should succeed.
+</para>
+
+<para>
+In looking at many small points of difference between species, which, as
+far as our ignorance permits us to judge, seem to be quite unimportant, we
+must not forget that climate, food, &amp;c., probably produce some slight and
+direct effect.  It is, however, far more necessary to bear in mind that
+there are many unknown laws of correlation of growth, which, when one part
+of the organisation is modified through variation, and the modifications
+are accumulated by natural selection for the good of the being, will cause
+other modifications, often of the most unexpected nature.
+</para>
+
+<para>
+As we see that those variations which under domestication appear at any
+particular period of life, tend to reappear in the offspring at the same
+period;--for instance, in the seeds of the many varieties of our culinary
+and agricultural plants; in the caterpillar and cocoon stages of the
+varieties of the silkworm; in the eggs of poultry, and in the colour of the
+down of their chickens; in the horns of our sheep and cattle when nearly
+adult;--so in a state of nature, natural selection will be enabled to act
+on and modify organic beings at any age, by the accumulation of profitable
+variations at that age, and by their inheritance at a corresponding age. 
+If it profit a plant to have its seeds more and more widely disseminated by
+the wind, I can see no greater difficulty in this being effected through
+natural selection, than in the cotton-planter increasing and improving by
+selection the down in the pods on his cotton-trees.  Natural selection may
+modify and adapt the larva of an insect to a score of contingencies, wholly
+different from those which concern the mature insect.  These modifications
+will no doubt affect, through the laws of correlation, the structure of the
+adult; and probably in the case of those insects which live only for a few
+hours, and which never feed, a large part of their structure is merely the
+correlated result of successive changes in the structure of their larvae. 
+So, conversely, modifications in the adult will probably often affect the
+structure of the larva; but in all cases natural selection will ensure that
+modifications consequent on other modifications at a different period of
+life, shall not be in the least degree injurious:  for if they became so,
+they would cause the extinction of the species.
+</para>
+
+<para>
+Natural selection will modify the structure of the young in relation to the
+parent, and of the parent in relation to the young.  In social animals it
+will adapt the structure of each individual for the benefit of the
+community; if each in consequence profits by the selected change.  What
+natural selection cannot do, is to modify the structure of one species,
+without giving it any advantage, for the good of another species; and
+though statements to this effect may be found in works of natural history,
+I cannot find one case which will bear investigation.  A structure used
+only once in an animal's whole life, if of high importance to it, might be
+modified to any extent by natural selection; for instance, the great jaws
+possessed by certain insects, and used exclusively for opening the
+cocoon--or the hard tip to the beak of nestling birds, used for breaking
+the egg.  It has been asserted, that of the best short-beaked
+tumbler-pigeons more perish in the egg than are able to get out of it; so
+that fanciers assist in the act of hatching.  Now, if nature had to make
+the beak of a full-grown pigeon very short for the bird's own advantage,
+the process of modification would be very slow, and there would be
+simultaneously the most rigorous selection of the young birds within the
+egg, which had the most powerful and hardest beaks, for all with weak beaks
+would inevitably perish:  or, more delicate and more easily broken shells
+might be selected, the thickness of the shell being known to vary like
+every other structure.
+</para>
+
+<para>
+Sexual Selection. -- Inasmuch as peculiarities often appear under
+domestication in one sex and become hereditarily attached to that sex, the
+same fact probably occurs under nature, and if so, natural selection will
+be able to modify one sex in its functional relations to the other sex, or
+in relation to wholly different habits of life in the two sexes, as is
+sometimes the case with insects.  And this leads me to say a few words on
+what I call Sexual Selection.  This depends, not on a struggle for
+existence, but on a struggle between the males for possession of the
+females; the result is not death to the unsuccessful competitor, but few or
+no offspring.  Sexual selection is, therefore, less rigorous than natural
+selection.  Generally, the most vigorous males, those which are best fitted
+for their places in nature, will leave most progeny.  But in many cases,
+victory will depend not on general vigour, but on having special weapons,
+confined to the male sex.  A hornless stag or spurless cock would have a
+poor chance of leaving offspring.  Sexual selection by always allowing the
+victor to breed might surely give indomitable courage, length to the spur,
+and strength to the wing to strike in the spurred leg, as well as the
+brutal cock-fighter, who knows well that he can improve his breed by
+careful selection of the best cocks.  How low in the scale of nature this
+law of battle descends, I know not; male alligators have been described as
+fighting, bellowing, and whirling round, like Indians in a war-dance, for
+the possession of the females; male salmons have been seen fighting all day
+long; male stag-beetles often bear wounds from the huge mandibles of other
+males.  The war is, perhaps, severest between the males of polygamous
+animals, and these seem oftenest provided with special weapons.  The males
+of carnivorous animals are already well armed; though to them and to
+others, special means of defence may be given through means of sexual
+selection, as the mane to the lion, the shoulder-pad to the boar, and the
+hooked jaw to the male salmon; for the shield may be as important for
+victory, as the sword or spear.
+</para>
+
+<para>
+Amongst birds, the contest is often of a more peaceful character.  All
+those who have attended to the subject, believe that there is the severest
+rivalry between the males of many species to attract by singing the
+females.  The rock-thrush of Guiana, birds of Paradise, and some others,
+congregate; and successive males display their gorgeous plumage and perform
+strange antics before the females, which standing by as spectators, at last
+choose the most attractive partner.  Those who have closely attended to
+birds in confinement well know that they often take individual preferences
+and dislikes:  thus Sir R. Heron has described how one pied peacock was
+eminently attractive to all his hen birds.  It may appear childish to
+attribute any effect to such apparently weak means:  I cannot here enter on
+the details necessary to support this view; but if man can in a short time
+give elegant carriage and beauty to his bantams, according to his standard
+of beauty, I can see no good reason to doubt that female birds, by
+selecting, during thousands of generations, the most melodious or beautiful
+males, according to their standard of beauty, might produce a marked
+effect.  I strongly suspect that some well-known laws with respect to the
+plumage of male and female birds, in comparison with the plumage of the
+young, can be explained on the view of plumage having been chiefly modified
+by sexual selection, acting when the birds have come to the breeding age or
+during the breeding season; the modifications thus produced being inherited
+at corresponding ages or seasons, either by the males alone, or by the
+males and females; but I have not space here to enter on this subject.
+</para>
+
+<para>
+Thus it is, as I believe, that when the males and females of any animal
+have the same general habits of life, but differ in structure, colour, or
+ornament, such differences have been mainly caused by sexual selection;
+that is, individual males have had, in successive generations, some slight
+advantage over other males, in their weapons, means of defence, or charms;
+and have transmitted these advantages to their male offspring.  Yet, I
+would not wish to attribute all such sexual differences to this agency: 
+for we see peculiarities arising and becoming attached to the male sex in
+our domestic animals (as the wattle in male carriers, horn-like
+protuberances in the cocks of certain fowls, &amp;c.), which we cannot believe
+to be either useful to the males in battle, or attractive to the females. 
+We see analogous cases under nature, for instance, the tuft of hair on the
+breast of the turkey-cock, which can hardly be either useful or ornamental
+to this bird;--indeed, had the tuft appeared under domestication, it would
+have been called a monstrosity.
+</para>
+
+<para>
+
+Illustrations of the action of Natural Selection. -- In order to make it
+clear how, as I believe, natural selection acts, I must beg permission to
+give one or two imaginary illustrations.  Let us take the case of a wolf,
+which preys on various animals, securing some by craft, some by strength,
+and some by fleetness; and let us suppose that the fleetest prey, a deer
+for instance, had from any change in the country increased in numbers, or
+that other prey had decreased in numbers, during that season of the year
+when the wolf is hardest pressed for food.  I can under such circumstances
+see no reason to doubt that the swiftest and slimmest wolves would have the
+best chance of surviving, and so be preserved or selected,--provided always
+that they retained strength to master their prey at this or at some other
+period of the year, when they might be compelled to prey on other animals. 
+I can see no more reason to doubt this, than that man can improve the
+fleetness of his greyhounds by careful and methodical selection, or by that
+unconscious selection which results from each man trying to keep the best
+dogs without any thought of modifying the breed.
+</para>
+
+<para>
+Even without any change in the proportional numbers of the animals on which
+our wolf preyed, a cub might be born with an innate tendency to pursue
+certain kinds of prey.  Nor can this be thought very improbable; for we
+often observe great differences in the natural tendencies of our domestic
+animals; one cat, for instance, taking to catch rats, another mice; one
+cat, according to Mr. St. John, bringing home winged game, another hares or
+rabbits, and another hunting on marshy ground and almost nightly catching
+woodcocks or snipes.  The tendency to catch rats rather than mice is known
+to be inherited.  Now, if any slight innate change of habit or of structure
+benefited an individual wolf, it would have the best chance of surviving
+and of leaving offspring.  Some of its young would probably inherit the
+same habits or structure, and by the repetition of this process, a new
+variety might be formed which would either supplant or coexist with the
+parent-form of wolf.  Or, again, the wolves inhabiting a mountainous
+district, and those frequenting the lowlands, would naturally be forced to
+hunt different prey; and from the continued preservation of the individuals
+best fitted for the two sites, two varieties might slowly be formed.  These
+varieties would cross and blend where they met; but to this subject of
+intercrossing we shall soon have to return.  I may add, that, according to
+Mr. Pierce, there are two varieties of the wolf inhabiting the Catskill
+Mountains in the United States, one with a light greyhound-like form, which
+pursues deer, and the other more bulky, with shorter legs, which more
+frequently attacks the shepherd's flocks.
+</para>
+
+<para>
+Let us now take a more complex case.  Certain plants excrete a sweet juice,
+apparently for the sake of eliminating something injurious from their sap: 
+this is effected by glands at the base of the stipules in some Leguminosae,
+and at the back of the leaf of the common laurel.  This juice, though small
+in quantity, is greedily sought by insects.  Let us now suppose a little
+sweet juice or nectar to be excreted by the inner bases of the petals of a
+flower.  In this case insects in seeking the nectar would get dusted with
+pollen, and would certainly often transport the pollen from one flower to
+the stigma of another flower.  The flowers of two distinct individuals of
+the same species would thus get crossed; and the act of crossing, we have
+good reason to believe (as will hereafter be more fully alluded to), would
+produce very vigorous seedlings, which consequently would have the best
+chance of flourishing and surviving.  Some of these seedlings would
+probably inherit the nectar-excreting power.  Those individual flowers
+which had the largest glands or nectaries, and which excreted most nectar,
+would be oftenest visited by insects, and would be oftenest crossed; and so
+in the long-run would gain the upper hand.  Those flowers, also, which had
+their stamens and pistils placed, in relation to the size and habits of the
+particular insects which visited them, so as to favour in any degree the
+transportal of their pollen from flower to flower, would likewise be
+favoured or selected.  We might have taken the case of insects visiting
+flowers for the sake of collecting pollen instead of nectar; and as pollen
+is formed for the sole object of fertilisation, its destruction appears a
+simple loss to the plant; yet if a little pollen were carried, at first
+occasionally and then habitually, by the pollen-devouring insects from
+flower to flower, and a cross thus effected, although nine-tenths of the
+pollen were destroyed, it might still be a great gain to the plant; and
+those individuals which produced more and more pollen, and had larger and
+larger anthers, would be selected.
+</para>
+
+<para>
+When our plant, by this process of the continued preservation or natural
+selection of more and more attractive flowers, had been rendered highly
+attractive to insects, they would, unintentionally on their part, regularly
+carry pollen from flower to flower; and that they can most effectually do
+this, I could easily show by many striking instances.  I will give only
+one--not as a very striking case, but as likewise illustrating one step in
+the separation of the sexes of plants, presently to be alluded to.  Some
+holly-trees bear only male flowers, which have four stamens producing
+rather a small quantity of pollen, and a rudimentary pistil; other
+holly-trees bear only female flowers; these have a full-sized pistil, and
+four stamens with shrivelled anthers, in which not a grain of pollen can be
+detected.  Having found a female tree exactly sixty yards from a male tree,
+I put the stigmas of twenty flowers, taken from different branches, under
+the microscope, and on all, without exception, there were pollen-grains,
+and on some a profusion of pollen.  As the wind had set for several days
+from the female to the male tree, the pollen could not thus have been
+carried.  The weather had been cold and boisterous, and therefore not
+favourable to bees, nevertheless every female flower which I examined had
+been effectually fertilised by the bees, accidentally dusted with pollen,
+having flown from tree to tree in search of nectar.  But to return to our
+imaginary case:  as soon as the plant had been rendered so highly
+attractive to insects that pollen was regularly carried from flower to
+flower, another process might commence.  No naturalist doubts the advantage
+of what has been called the 'physiological division of labour;' hence we
+may believe that it would be advantageous to a plant to produce stamens
+alone in one flower or on one whole plant, and pistils alone in another
+flower or on another plant.  In plants under culture and placed under new
+conditions of life, sometimes the male organs and sometimes the female
+organs become more or less impotent; now if we suppose this to occur in
+ever so slight a degree under nature, then as pollen is already carried
+regularly from flower to flower, and as a more complete separation of the
+sexes of our plant would be advantageous on the principle of the division
+of labour, individuals with this tendency more and more increased, would be
+continually favoured or selected, until at last a complete separation of
+the sexes would be effected.
+</para>
+
+<para>
+Let us now turn to the nectar-feeding insects in our imaginary case:  we
+may suppose the plant of which we have been slowly increasing the nectar by
+continued selection, to be a common plant; and that certain insects
+depended in main part on its nectar for food.  I could give many facts,
+showing how anxious bees are to save time; for instance, their habit of
+cutting holes and sucking the nectar at the bases of certain flowers, which
+they can, with a very little more trouble, enter by the mouth.  Bearing
+such facts in mind, I can see no reason to doubt that an accidental
+deviation in the size and form of the body, or in the curvature and length
+of the proboscis, &amp;c., far too slight to be appreciated by us, might profit
+a bee or other insect, so that an individual so characterised would be able
+to obtain its food more quickly, and so have a better chance of living and
+leaving descendants.  Its descendants would probably inherit a tendency to
+a similar slight deviation of structure.  The tubes of the corollas of the
+common red and incarnate clovers (Trifolium pratense and incarnatum) do not
+on a hasty glance appear to differ in length; yet the hive-bee can easily
+suck the nectar out of the incarnate clover, but not out of the common red
+clover, which is visited by humble-bees alone; so that whole fields of the
+red clover offer in vain an abundant supply of precious nectar to the
+hive-bee.  Thus it might be a great advantage to the hive-bee to have a
+slightly longer or differently constructed proboscis.  On the other hand, I
+have found by experiment that the fertility of clover greatly depends on
+bees visiting and moving parts of the corolla, so as to push the pollen on
+to the stigmatic surface.  Hence, again, if humble-bees were to become rare
+in any country, it might be a great advantage to the red clover to have a
+shorter or more deeply divided tube to its corolla, so that the hive-bee
+could visit its flowers.  Thus I can understand how a flower and a bee
+might slowly become, either simultaneously or one after the other, modified
+and adapted in the most perfect manner to each other, by the continued
+preservation of individuals presenting mutual and slightly favourable
+deviations of structure.
+</para>
+
+<para>
+I am well aware that this doctrine of natural selection, exemplified in the
+above imaginary instances, is open to the same objections which were at
+first urged against Sir Charles Lyell's noble views on 'the modern changes
+of the earth, as illustrative of geology;' but we now very seldom hear the
+action, for instance, of the coast-waves, called a trifling and
+insignificant cause, when applied to the excavation of gigantic valleys or
+to the formation of the longest lines of inland cliffs.  Natural selection
+can act only by the preservation and accumulation of infinitesimally small
+inherited modifications, each profitable to the preserved being; and as
+modern geology has almost banished such views as the excavation of a great
+valley by a single diluvial wave, so will natural selection, if it be a
+true principle, banish the belief of the continued creation of new organic
+beings, or of any great and sudden modification in their structure.
+</para>
+
+<para>
+On the Intercrossing of Individuals. -- I must here introduce a short
+digression.  In the case of animals and plants with separated sexes, it is
+of course obvious that two individuals must always unite for each birth;
+but in the case of hermaphrodites this is far from obvious.  Nevertheless I
+am strongly inclined to believe that with all hermaphrodites two
+individuals, either occasionally or habitually, concur for the reproduction
+of their kind.  This view, I may add, was first suggested by Andrew Knight. 
+We shall presently see its importance; but I must here treat the subject
+with extreme brevity, though I have the materials prepared for an ample
+discussion.  All vertebrate animals, all insects, and some other large
+groups of animals, pair for each birth.  Modern research has much
+diminished the number of supposed hermaphrodites, and of real
+hermaphrodites a large number pair; that is, two individuals regularly
+unite for reproduction, which is all that concerns us.  But still there are
+many hermaphrodite animals which certainly do not habitually pair, and a
+vast majority of plants are hermaphrodites.  What reason, it may be asked,
+is there for supposing in these cases that two individuals ever concur in
+reproduction?  As it is impossible here to enter on details, I must trust
+to some general considerations alone.
+</para>
+
+<para>
+In the first place, I have collected so large a body of facts, showing, in
+accordance with the almost universal belief of breeders, that with animals
+and plants a cross between different varieties, or between individuals of
+the same variety but of another strain, gives vigour and fertility to the
+offspring; and on the other hand, that close interbreeding diminishes
+vigour and fertility; that these facts alone incline me to believe that it
+is a general law of nature (utterly ignorant though we be of the meaning of
+the law) that no organic being self-fertilises itself for an eternity of
+generations; but that a cross with another individual is
+occasionally--perhaps at very long intervals--indispensable.
+</para>
+
+<para>
+On the belief that this is a law of nature, we can, I think, understand
+several large classes of facts, such as the following, which on any other
+view are inexplicable.  Every hybridizer knows how unfavourable exposure to
+wet is to the fertilisation of a flower, yet what a multitude of flowers
+have their anthers and stigmas fully exposed to the weather! but if an
+occasional cross be indispensable, the fullest freedom for the entrance of
+pollen from another individual will explain this state of exposure, more
+especially as the plant's own anthers and pistil generally stand so close
+together that self-fertilisation seems almost inevitable.  Many flowers, on
+the other hand, have their organs of fructification closely enclosed, as in
+the great papilionaceous or pea-family; but in several, perhaps in all,
+such flowers, there is a very curious adaptation between the structure of
+the flower and the manner in which bees suck the nectar; for, in doing
+this, they either push the flower's own pollen on the stigma, or bring
+pollen from another flower.  So necessary are the visits of bees to
+papilionaceous flowers, that I have found, by experiments published
+elsewhere, that their fertility is greatly diminished if these visits be
+prevented.  Now, it is scarcely possible that bees should fly from flower
+to flower, and not carry pollen from one to the other, to the great good,
+as I believe, of the plant.  Bees will act like a camel-hair pencil, and it
+is quite sufficient just to touch the anthers of one flower and then the
+stigma of another with the same brush to ensure fertilisation; but it must
+not be supposed that bees would thus produce a multitude of hybrids between
+distinct species; for if you bring on the same brush a plant's own pollen
+and pollen from another species, the former will have such a prepotent
+effect, that it will invariably and completely destroy, as has been shown
+by Gartner, any influence from the foreign pollen.
+</para>
+
+<para>
+When the stamens of a flower suddenly spring towards the pistil, or slowly
+move one after the other towards it, the contrivance seems adapted solely
+to ensure self-fertilisation; and no doubt it is useful for this end:  but,
+the agency of insects is often required to cause the stamens to spring
+forward, as Kolreuter has shown to be the case with the barberry; and
+curiously in this very genus, which seems to have a special contrivance for
+self-fertilisation, it is well known that if very closely-allied forms or
+varieties are planted near each other, it is hardly possible to raise pure
+seedlings, so largely do they naturally cross.  In many other cases, far
+from there being any aids for self-fertilisation, there are special
+contrivances, as I could show from the writings of C. C. Sprengel and from
+my own observations, which effectually prevent the stigma receiving pollen
+from its own flower:  for instance, in Lobelia fulgens, there is a really
+beautiful and elaborate contrivance by which every one of the infinitely
+numerous pollen-granules are swept out of the conjoined anthers of each
+flower, before the stigma of that individual flower is ready to receive
+them; and as this flower is never visited, at least in my garden, by
+insects, it never sets a seed, though by placing pollen from one flower on
+the stigma of another, I raised plenty of seedlings; and whilst another
+species of Lobelia growing close by, which is visited by bees, seeds
+freely.  In very many other cases, though there be no special mechanical
+contrivance to prevent the stigma of a flower receiving its own pollen,
+yet, as C. C. Sprengel has shown, and as I can confirm, either the anthers
+burst before the stigma is ready for fertilisation, or the stigma is ready
+before the pollen of that flower is ready, so that these plants have in
+fact separated sexes, and must habitually be crossed.  How strange are
+these facts!  How strange that the pollen and stigmatic surface of the same
+flower, though placed so close together, as if for the very purpose of
+self-fertilisation, should in so many cases be mutually useless to each
+other!  How simply are these facts explained on the view of an occasional
+cross with a distinct individual being advantageous or indispensable!
+</para>
+
+<para>
+If several varieties of the cabbage, radish, onion, and of some other
+plants, be allowed to seed near each other, a large majority, as I have
+found, of the seedlings thus raised will turn out mongrels:  for instance,
+I raised 233 seedling cabbages from some plants of different varieties
+growing near each other, and of these only 78 were true to their kind, and
+some even of these were not perfectly true.  Yet the pistil of each
+cabbage-flower is surrounded not only by its own six stamens, but by those
+of the many other flowers on the same plant.  How, then, comes it that such
+a vast number of the seedlings are mongrelized?  I suspect that it must
+arise from the pollen of a distinct variety having a prepotent effect over
+a flower's own pollen; and that this is part of the general law of good
+being derived from the intercrossing of distinct individuals of the same
+species.  When distinct species are crossed the case is directly the
+reverse, for a plant's own pollen is always prepotent over foreign pollen;
+but to this subject we shall return in a future chapter.
+</para>
+
+<para>
+In the case of a gigantic tree covered with innumerable flowers, it may be
+objected that pollen could seldom be carried from tree to tree, and at most
+only from flower to flower on the same tree, and that flowers on the same
+tree can be considered as distinct individuals only in a limited sense.  I
+believe this objection to be valid, but that nature has largely provided
+against it by giving to trees a strong tendency to bear flowers with
+separated sexes.  When the sexes are separated, although the male and
+female flowers may be produced on the same tree, we can see that pollen
+must be regularly carried from flower to flower; and this will give a
+better chance of pollen being occasionally carried from tree to tree.  That
+trees belonging to all Orders have their sexes more often separated than
+other plants, I find to be the case in this country; and at my request Dr.
+Hooker tabulated the trees of New Zealand, and Dr. Asa Gray those of the
+United States, and the result was as I anticipated.  On the other hand, Dr.
+Hooker has recently informed me that he finds that the rule does not hold
+in Australia; and I have made these few remarks on the sexes of trees
+simply to call attention to the subject.
+</para>
+
+<para>
+Turning for a very brief space to animals:  on the land there are some
+hermaphrodites, as land-mollusca and earth-worms; but these all pair.  As
+yet I have not found a single case of a terrestrial animal which fertilises
+itself.  We can understand this remarkable fact, which offers so strong a
+contrast with terrestrial plants, on the view of an occasional cross being
+indispensable, by considering the medium in which terrestrial animals live,
+and the nature of the fertilising element; for we know of no means,
+analogous to the action of insects and of the wind in the case of plants,
+by which an occasional cross could be effected with terrestrial animals
+without the concurrence of two individuals.  Of aquatic animals, there are
+many self-fertilising hermaphrodites; but here currents in the water offer
+an obvious means for an occasional cross.  And, as in the case of flowers,
+I have as yet failed, after consultation with one of the highest
+authorities, namely, Professor Huxley, to discover a single case of an
+hermaphrodite animal with the organs of reproduction so perfectly enclosed
+within the body, that access from without and the occasional influence of a
+distinct individual can be shown to be physically impossible.  Cirripedes
+long appeared to me to present a case of very great difficulty under this
+point of view; but I have been enabled, by a fortunate chance, elsewhere to
+prove that two individuals, though both are self-fertilising
+hermaphrodites, do sometimes cross.
+</para>
+
+<para>
+It must have struck most naturalists as a strange anomaly that, in the case
+of both animals and plants, species of the same family and even of the same
+genus, though agreeing closely with each other in almost their whole
+organisation, yet are not rarely, some of them hermaphrodites, and some of
+them unisexual.  But if, in fact, all hermaphrodites do occasionally
+intercross with other individuals, the difference between hermaphrodites
+and unisexual species, as far as function is concerned, becomes very small.
+</para>
+
+<para>
+From these several considerations and from the many special facts which I
+have collected, but which I am not here able to give, I am strongly
+inclined to suspect that, both in the vegetable and animal kingdoms, an
+occasional intercross with a distinct individual is a law of nature.  I am
+well aware that there are, on this view, many cases of difficulty, some of
+which I am trying to investigate.  Finally then, we may conclude that in
+many organic beings, a cross between two individuals is an obvious
+necessity for each birth; in many others it occurs perhaps only at long
+intervals; but in none, as I suspect, can self-fertilisation go on for
+perpetuity.
+</para>
+
+<para>
+Circumstances favourable to Natural Selection. -- This is an extremely
+intricate subject.  A large amount of inheritable and diversified
+variability is favourable, but I believe mere individual differences
+suffice for the work.  A large number of individuals, by giving a better
+chance for the appearance within any given period of profitable variations,
+will compensate for a lesser amount of variability in each individual, and
+is, I believe, an extremely important element of success.  Though nature
+grants vast periods of time for the work of natural selection, she does not
+grant an indefinite period; for as all organic beings are striving, it may
+be said, to seize on each place in the economy of nature, if any one
+species does not become modified and improved in a corresponding degree
+with its competitors, it will soon be exterminated.
+</para>
+
+<para>
+In man's methodical selection, a breeder selects for some definite object,
+and free intercrossing will wholly stop his work.  But when many men,
+without intending to alter the breed, have a nearly common standard of
+perfection, and all try to get and breed from the best animals, much
+improvement and modification surely but slowly follow from this unconscious
+process of selection, notwithstanding a large amount of crossing with
+inferior animals.  Thus it will be in nature; for within a confined area,
+with some place in its polity not so perfectly occupied as might be,
+natural selection will always tend to preserve all the individuals varying
+in the right direction, though in different degrees, so as better to fill
+up the unoccupied place.  But if the area be large, its several districts
+will almost certainly present different conditions of life; and then if
+natural selection be modifying and improving a species in the several
+districts, there will be intercrossing with the other individuals of the
+same species on the confines of each.  And in this case the effects of
+intercrossing can hardly be counterbalanced by natural selection always
+tending to modify all the individuals in each district in exactly the same
+manner to the conditions of each; for in a continuous area, the conditions
+will generally graduate away insensibly from one district to another.  The
+intercrossing will most affect those animals which unite for each birth,
+which wander much, and which do not breed at a very quick rate.  Hence in
+animals of this nature, for instance in birds, varieties will generally be
+confined to separated countries; and this I believe to be the case.  In
+hermaphrodite organisms which cross only occasionally, and likewise in
+animals which unite for each birth, but which wander little and which can
+increase at a very rapid rate, a new and improved variety might be quickly
+formed on any one spot, and might there maintain itself in a body, so that
+whatever intercrossing took place would be chiefly between the individuals
+of the same new variety.  A local variety when once thus formed might
+subsequently slowly spread to other districts.  On the above principle,
+nurserymen always prefer getting seed from a large body of plants of the
+same variety, as the chance of intercrossing with other varieties is thus
+lessened.
+</para>
+
+<para>
+Even in the case of slow-breeding animals, which unite for each birth, we
+must not overrate the effects of intercrosses in retarding natural
+selection; for I can bring a considerable catalogue of facts, showing that
+within the same area, varieties of the same animal can long remain
+distinct, from haunting different stations, from breeding at slightly
+different seasons, or from varieties of the same kind preferring to pair
+together.
+</para>
+
+<para>
+Intercrossing plays a very important part in nature in keeping the
+individuals of the same species, or of the same variety, true and uniform
+in character.  It will obviously thus act far more efficiently with those
+animals which unite for each birth; but I have already attempted to show
+that we have reason to believe that occasional intercrosses take place with
+all animals and with all plants.  Even if these take place only at long
+intervals, I am convinced that the young thus produced will gain so much in
+vigour and fertility over the offspring from long-continued
+self-fertilisation, that they will have a better chance of surviving and
+propagating their kind; and thus, in the long run, the influence of
+intercrosses, even at rare intervals, will be great.  If there exist
+organic beings which never intercross, uniformity of character can be
+retained amongst them, as long as their conditions of life remain the same,
+only through the principle of inheritance, and through natural selection
+destroying any which depart from the proper type; but if their conditions
+of life change and they undergo modification, uniformity of character can
+be given to their modified offspring, solely by natural selection
+preserving the same favourable variations.
+</para>
+
+<para>
+Isolation, also, is an important element in the process of natural
+selection.  In a confined or isolated area, if not very large, the organic
+and inorganic conditions of life will generally be in a great degree
+uniform; so that natural selection will tend to modify all the individuals
+of a varying species throughout the area in the same manner in relation to
+the same conditions.  Intercrosses, also, with the individuals of the same
+species, which otherwise would have inhabited the surrounding and
+differently circumstanced districts, will be prevented.  But isolation
+probably acts more efficiently in checking the immigration of better
+adapted organisms, after any physical change, such as of climate or
+elevation of the land, &amp;c.; and thus new places in the natural economy of
+the country are left open for the old inhabitants to struggle for, and
+become adapted to, through modifications in their structure and
+constitution.  Lastly, isolation, by checking immigration and consequently
+competition, will give time for any new variety to be slowly improved; and
+this may sometimes be of importance in the production of new species.  If,
+however, an isolated area be very small, either from being surrounded by
+barriers, or from having very peculiar physical conditions, the total
+number of the individuals supported on it will necessarily be very small;
+and fewness of individuals will greatly retard the production of new
+species through natural selection, by decreasing the chance of the
+appearance of favourable variations.
+</para>
+
+<para>
+If we turn to nature to test the truth of these remarks, and look at any
+small isolated area, such as an oceanic island, although the total number
+of the species inhabiting it, will be found to be small, as we shall see in
+our chapter on geographical distribution; yet of these species a very large
+proportion are endemic,--that is, have been produced there, and nowhere
+else.  Hence an oceanic island at first sight seems to have been highly
+favourable for the production of new species.  But we may thus greatly
+deceive ourselves, for to ascertain whether a small isolated area, or a
+large open area like a continent, has been most favourable for the
+production of new organic forms, we ought to make the comparison within
+equal times; and this we are incapable of doing.
+</para>
+
+<para>
+Although I do not doubt that isolation is of considerable importance in the
+production of new species, on the whole I am inclined to believe that
+largeness of area is of more importance, more especially in the production
+of species, which will prove capable of enduring for a long period, and of
+spreading widely.  Throughout a great and open area, not only will there be
+a better chance of favourable variations arising from the large number of
+individuals of the same species there supported, but the conditions of life
+are infinitely complex from the large number of already existing species;
+and if some of these many species become modified and improved, others will
+have to be improved in a corresponding degree or they will be exterminated. 
+Each new form, also, as soon as it has been much improved, will be able to
+spread over the open and continuous area, and will thus come into
+competition with many others.  Hence more new places will be formed, and
+the competition to fill them will be more severe, on a large than on a
+small and isolated area.  Moreover, great areas, though now continuous,
+owing to oscillations of level, will often have recently existed in a
+broken condition, so that the good effects of isolation will generally, to
+a certain extent, have concurred.  Finally, I conclude that, although small
+isolated areas probably have been in some respects highly favourable for
+the production of new species, yet that the course of modification will
+generally have been more rapid on large areas; and what is more important,
+that the new forms produced on large areas, which already have been
+victorious over many competitors, will be those that will spread most
+widely, will give rise to most new varieties and species, and will thus
+play an important part in the changing history of the organic world.
+</para>
+
+<para>
+We can, perhaps, on these views, understand some facts which will be again
+alluded to in our chapter on geographical distribution; for instance, that
+the productions of the smaller continent of Australia have formerly
+yielded, and apparently are now yielding, before those of the larger
+Europaeo-Asiatic area.  Thus, also, it is that continental productions have
+everywhere become so largely naturalised on islands.  On a small island,
+the race for life will have been less severe, and there will have been less
+modification and less extermination.  Hence, perhaps, it comes that the
+flora of Madeira, according to Oswald Heer, resembles the extinct tertiary
+flora of Europe.  All fresh-water basins, taken together, make a small area
+compared with that of the sea or of the land; and, consequently, the
+competition between fresh-water productions will have been less severe than
+elsewhere; new forms will have been more slowly formed, and old forms more
+slowly exterminated.  And it is in fresh water that we find seven genera of
+Ganoid fishes, remnants of a once preponderant order:  and in fresh water
+we find some of the most anomalous forms now known in the world, as the
+Ornithorhynchus and Lepidosiren, which, like fossils, connect to a certain
+extent orders now widely separated in the natural scale.  These anomalous
+forms may almost be called living fossils; they have endured to the present
+day, from having inhabited a confined area, and from having thus been
+exposed to less severe competition.
+</para>
+
+<para>
+To sum up the circumstances favourable and unfavourable to natural
+selection, as far as the extreme intricacy of the subject permits.  I
+conclude, looking to the future, that for terrestrial productions a large
+continental area, which will probably undergo many oscillations of level,
+and which consequently will exist for long periods in a broken condition,
+will be the most favourable for the production of many new forms of life,
+likely to endure long and to spread widely.  For the area will first have
+existed as a continent, and the inhabitants, at this period numerous in
+individuals and kinds, will have been subjected to very severe competition. 
+When converted by subsidence into large separate islands, there will still
+exist many individuals of the same species on each island:  intercrossing
+on the confines of the range of each species will thus be checked:  after
+physical changes of any kind, immigration will be prevented, so that new
+places in the polity of each island will have to be filled up by
+modifications of the old inhabitants; and time will be allowed for the
+varieties in each to become well modified and perfected.  When, by renewed
+elevation, the islands shall be re-converted into a continental area, there
+will again be severe competition:  the most favoured or improved varieties
+will be enabled to spread:  there will be much extinction of the less
+improved forms, and the relative proportional numbers of the various
+inhabitants of the renewed continent will again be changed; and again there
+will be a fair field for natural selection to improve still further the
+inhabitants, and thus produce new species.
+</para>
+
+<para>
+That natural selection will always act with extreme slowness, I fully
+admit.  Its action depends on there being places in the polity of nature,
+which can be better occupied by some of the inhabitants of the country
+undergoing modification of some kind.  The existence of such places will
+often depend on physical changes, which are generally very slow, and on the
+immigration of better adapted forms having been checked.  But the action of
+natural selection will probably still oftener depend on some of the
+inhabitants becoming slowly modified; the mutual relations of many of the
+other inhabitants being thus disturbed.  Nothing can be effected, unless
+favourable variations occur, and variation itself is apparently always a
+very slow process.  The process will often be greatly retarded by free
+intercrossing.  Many will exclaim that these several causes are amply
+sufficient wholly to stop the action of natural selection.  I do not
+believe so.  On the other hand, I do believe that natural selection will
+always act very slowly, often only at long intervals of time, and generally
+on only a very few of the inhabitants of the same region at the same time. 
+I further believe, that this very slow, intermittent action of natural
+selection accords perfectly well with what geology tells us of the rate and
+manner at which the inhabitants of this world have changed.
+</para>
+
+<para>
+Slow though the process of selection may be, if feeble man can do much by
+his powers of artificial selection, I can see no limit to the amount of
+change, to the beauty and infinite complexity of the coadaptations between
+all organic beings, one with another and with their physical conditions of
+life, which may be effected in the long course of time by nature's power of
+selection.
+</para>
+
+<para>
+Extinction. -- This subject will be more fully discussed in our chapter on
+Geology; but it must be here alluded to from being intimately connected
+with natural selection.  Natural selection acts solely through the
+preservation of variations in some way advantageous, which consequently
+endure.  But as from the high geometrical powers of increase of all organic
+beings, each area is already fully stocked with inhabitants, it follows
+that as each selected and favoured form increases in number, so will the
+less favoured forms decrease and become rare.  Rarity, as geology tells us,
+is the precursor to extinction.  We can, also, see that any form
+represented by few individuals will, during fluctuations in the seasons or
+in the number of its enemies, run a good chance of utter extinction.  But
+we may go further than this; for as new forms are continually and slowly
+being produced, unless we believe that the number of specific forms goes on
+perpetually and almost indefinitely increasing, numbers inevitably must
+become extinct.  That the number of specific forms has not indefinitely
+increased, geology shows us plainly; and indeed we can see reason why they
+should not have thus increased, for the number of places in the polity of
+nature is not indefinitely great,--not that we have any means of knowing
+that any one region has as yet got its maximum of species.  Probably no
+region is as yet fully stocked, for at the Cape of Good Hope, where more
+species of plants are crowded together than in any other quarter of the
+world, some foreign plants have become naturalised, without causing, as far
+as we know, the extinction of any natives.
+</para>
+
+<para>
+Furthermore, the species which are most numerous in individuals will have
+the best chance of producing within any given period favourable variations. 
+We have evidence of this, in the facts given in the second chapter, showing
+that it is the common species which afford the greatest number of recorded
+varieties, or incipient species.  Hence, rare species will be less quickly
+modified or improved within any given period, and they will consequently be
+beaten in the race for life by the modified descendants of the commoner
+species.
+</para>
+
+<para>
+From these several considerations I think it inevitably follows, that as
+new species in the course of time are formed through natural selection,
+others will become rarer and rarer, and finally extinct.  The forms which
+stand in closest competition with those undergoing modification and
+improvement, will naturally suffer most.  And we have seen in the chapter
+on the Struggle for Existence that it is the most closely-allied
+forms,--varieties of the same species, and species of the same genus or of
+related genera,--which, from having nearly the same structure,
+constitution, and habits, generally come into the severest competition with
+each other.  Consequently, each new variety or species, during the progress
+of its formation, will generally press hardest on its nearest kindred, and
+tend to exterminate them.  We see the same process of extermination amongst
+our domesticated productions, through the selection of improved forms by
+man.  Many curious instances could be given showing how quickly new breeds
+of cattle, sheep, and other animals, and varieties of flowers, take the
+place of older and inferior kinds.  In Yorkshire, it is historically known
+that the ancient black cattle were displaced by the long-horns, and that
+these 'were swept away by the short-horns' (I quote the words of an
+agricultural writer) 'as if by some murderous pestilence.'
+</para>
+
+<para>
+Divergence of Character. -- The principle, which I have designated by this
+term, is of high importance on my theory, and explains, as I believe,
+several important facts.  In the first place, varieties, even
+strongly-marked ones, though having somewhat of the character of
+species--as is shown by the hopeless doubts in many cases how to rank
+them--yet certainly differ from each other far less than do good and
+distinct species.  Nevertheless, according to my view, varieties are
+species in the process of formation, or are, as I have called them,
+incipient species.  How, then, does the lesser difference between varieties
+become augmented into the greater difference between species?  That this
+does habitually happen, we must infer from most of the innumerable species
+throughout nature presenting well-marked differences; whereas varieties,
+the supposed prototypes and parents of future well-marked species, present
+slight and ill-defined differences.  Mere chance, as we may call it, might
+cause one variety to differ in some character from its parents, and the
+offspring of this variety again to differ from its parent in the very same
+character and in a greater degree; but this alone would never account for
+so habitual and large an amount of difference as that between varieties of
+the same species and species of the same genus.
+</para>
+
+<para>
+As has always been my practice, let us seek light on this head from our
+domestic productions.  We shall here find something analogous.  A fancier
+is struck by a pigeon having a slightly shorter beak; another fancier is
+struck by a pigeon having a rather longer beak; and on the acknowledged
+principle that 'fanciers do not and will not admire a medium standard, but
+like extremes,' they both go on (as has actually occurred with
+tumbler-pigeons) choosing and breeding from birds with longer and longer
+beaks, or with shorter and shorter beaks.  Again, we may suppose that at an
+early period one man preferred swifter horses; another stronger and more
+bulky horses.  The early differences would be very slight; in the course of
+time, from the continued selection of swifter horses by some breeders, and
+of stronger ones by others, the differences would become greater, and would
+be noted as forming two sub-breeds; finally, after the lapse of centuries,
+the sub-breeds would become converted into two well-established and
+distinct breeds.  As the differences slowly become greater, the inferior
+animals with intermediate characters, being neither very swift nor very
+strong, will have been neglected, and will have tended to disappear.  Here,
+then, we see in man's productions the action of what may be called the
+principle of divergence, causing differences, at first barely appreciable,
+steadily to increase, and the breeds to diverge in character both from each
+other and from their common parent.
+</para>
+
+<para>
+But how, it may be asked, can any analogous principle apply in nature?  I
+believe it can and does apply most efficiently, from the simple
+circumstance that the more diversified the descendants from any one species
+become in structure, constitution, and habits, by so much will they be
+better enabled to seize on many and widely diversified places in the polity
+of nature, and so be enabled to increase in numbers.
+</para>
+
+<para>
+We can clearly see this in the case of animals with simple habits.  Take
+the case of a carnivorous quadruped, of which the number that can be
+supported in any country has long ago arrived at its full average.  If its
+natural powers of increase be allowed to act, it can succeed in increasing
+(the country not undergoing any change in its conditions) only by its
+varying descendants seizing on places at present occupied by other animals: 
+some of them, for instance, being enabled to feed on new kinds of prey,
+either dead or alive; some inhabiting new stations, climbing trees,
+frequenting water, and some perhaps becoming less carnivorous.  The more
+diversified in habits and structure the descendants of our carnivorous
+animal became, the more places they would be enabled to occupy.  What
+applies to one animal will apply throughout all time to all animals--that
+is, if they vary--for otherwise natural selection can do nothing.  So it
+will be with plants.  It has been experimentally proved, that if a plot of
+ground be sown with several distinct genera of grasses, a greater number of
+plants and a greater weight of dry herbage can thus be raised.  The same
+has been found to hold good when first one variety and then several mixed
+varieties of wheat have been sown on equal spaces of ground.  Hence, if any
+one species of grass were to go on varying, and those varieties were
+continually selected which differed from each other in at all the same
+manner as distinct species and genera of grasses differ from each other, a
+greater number of individual plants of this species of grass, including its
+modified descendants, would succeed in living on the same piece of ground. 
+And we well know that each species and each variety of grass is annually
+sowing almost countless seeds; and thus, as it may be said, is striving its
+utmost to increase its numbers.  Consequently, I cannot doubt that in the
+course of many thousands of generations, the most distinct varieties of any
+one species of grass would always have the best chance of succeeding and of
+increasing in numbers, and thus of supplanting the less distinct varieties;
+and varieties, when rendered very distinct from each other, take the rank
+of species.
+</para>
+
+<para>
+The truth of the principle, that the greatest amount of life can be
+supported by great diversification of structure, is seen under many natural
+circumstances.  In an extremely small area, especially if freely open to
+immigration, and where the contest between individual and individual must
+be severe, we always find great diversity in its inhabitants.  For
+instance, I found that a piece of turf, three feet by four in size, which
+had been exposed for many years to exactly the same conditions, supported
+twenty species of plants, and these belonged to eighteen genera and to
+eight orders, which shows how much these plants differed from each other. 
+So it is with the plants and insects on small and uniform islets; and so in
+small ponds of fresh water.  Farmers find that they can raise most food by
+a rotation of plants belonging to the most different orders:  nature
+follows what may be called a simultaneous rotation.  Most of the animals
+and plants which live close round any small piece of ground, could live on
+it (supposing it not to be in any way peculiar in its nature), and may be
+said to be striving to the utmost to live there; but, it is seen, that
+where they come into the closest competition with each other, the
+advantages of diversification of structure, with the accompanying
+differences of habit and constitution, determine that the inhabitants,
+which thus jostle each other most closely, shall, as a general rule, belong
+to what we call different genera and orders.
+</para>
+
+<para>
+The same principle is seen in the naturalisation of plants through man's
+agency in foreign lands.  It might have been expected that the plants which
+have succeeded in becoming naturalised in any land would generally have
+been closely allied to the indigenes; for these are commonly looked at as
+specially created and adapted for their own country.  It might, also,
+perhaps have been expected that naturalised plants would have belonged to a
+few groups more especially adapted to certain stations in their new homes. 
+But the case is very different; and Alph. De Candolle has well remarked in
+his great and admirable work, that floras gain by naturalisation,
+proportionally with the number of the native genera and species, far more
+in new genera than in new species.  To give a single instance:  in the last
+edition of Dr. Asa Gray's 'Manual of the Flora of the Northern United
+States,' 260 naturalised plants are enumerated, and these belong to 162
+genera.  We thus see that these naturalised plants are of a highly
+diversified nature.  They differ, moreover, to a large extent from the
+indigenes, for out of the 162 genera, no less than 100 genera are not there
+indigenous, and thus a large proportional addition is made to the genera of
+these States.
+</para>
+
+<para>
+By considering the nature of the plants or animals which have struggled
+successfully with the indigenes of any country, and have there become
+naturalised, we can gain some crude idea in what manner some of the natives
+would have had to be modified, in order to have gained an advantage over
+the other natives; and we may, I think, at least safely infer that
+diversification of structure, amounting to new generic differences, would
+have been profitable to them.
+</para>
+
+<para>
+The advantage of diversification in the inhabitants of the same region is,
+in fact, the same as that of the physiological division of labour in the
+organs of the same individual body--a subject so well elucidated by Milne
+Edwards.  No physiologist doubts that a stomach by being adapted to digest
+vegetable matter alone, or flesh alone, draws most nutriment from these
+substances.  So in the general economy of any land, the more widely and
+perfectly the animals and plants are diversified for different habits of
+life, so will a greater number of individuals be capable of there
+supporting themselves.  A set of animals, with their organisation but
+little diversified, could hardly compete with a set more perfectly
+diversified in structure.  It may be doubted, for instance, whether the
+Australian marsupials, which are divided into groups differing but little
+from each other, and feebly representing, as Mr. Waterhouse and others have
+remarked, our carnivorous, ruminant, and rodent mammals, could successfully
+compete with these well-pronounced orders.  In the Australian mammals, we
+see the process of diversification in an early and incomplete stage of
+development.
+</para>
+
+<para>
+After the foregoing discussion, which ought to have been much amplified, we
+may, I think, assume that the modified descendants of any one species will
+succeed by so much the better as they become more diversified in structure,
+and are thus enabled to encroach on places occupied by other beings.  Now
+let us see how this principle of great benefit being derived from
+divergence of character, combined with the principles of natural selection
+and of extinction, will tend to act.
+</para>
+
+<para>
+The accompanying diagram will aid us in understanding this rather
+perplexing subject.  Let A to L represent the species of a genus large in
+its own country; these species are supposed to resemble each other in
+unequal degrees, as is so generally the case in nature, and as is
+represented in the diagram by the letters standing at unequal distances.  I
+have said a large genus, because we have seen in the second chapter, that
+on an average more of the species of large genera vary than of small
+genera; and the varying species of the large genera present a greater
+number of varieties.  We have, also, seen that the species, which are the
+commonest and the most widely-diffused, vary more than rare species with
+restricted ranges.  Let (A) be a common, widely-diffused, and varying
+species, belonging to a genus large in its own country.  The little fan of
+diverging dotted lines of unequal lengths proceeding from (A), may
+represent its varying offspring.  The variations are supposed to be
+extremely slight, but of the most diversified nature; they are not supposed
+all to appear simultaneously, but often after long intervals of time; nor
+are they all supposed to endure for equal periods.  Only those variations
+which are in some way profitable will be preserved or naturally selected. 
+And here the importance of the principle of benefit being derived from
+divergence of character comes in; for this will generally lead to the most
+different or divergent variations (represented by the outer dotted lines)
+being preserved and accumulated by natural selection.  When a dotted line
+reaches one of the horizontal lines, and is there marked by a small
+numbered letter, a sufficient amount of variation is supposed to have been
+accumulated to have formed a fairly well-marked variety, such as would be
+thought worthy of record in a systematic work.
+</para>
+
+<para>
+The intervals between the horizontal lines in the diagram, may represent
+each a thousand generations; but it would have been better if each had
+represented ten thousand generations.  After a thousand generations,
+species (A) is supposed to have produced two fairly well-marked varieties,
+namely a1 and m1.  These two varieties will generally continue to be
+exposed to the same conditions which made their parents variable, and the
+tendency to variability is in itself hereditary, consequently they will
+tend to vary, and generally to vary in nearly the same manner as their
+parents varied.  Moreover, these two varieties, being only slightly
+modified forms, will tend to inherit those advantages which made their
+common parent (A) more numerous than most of the other inhabitants of the
+same country; they will likewise partake of those more general advantages
+which made the genus to which the parent-species belonged, a large genus in
+its own country.  And these circumstances we know to be favourable to the
+production of new varieties.
+</para>
+
+<para>
+If, then, these two varieties be variable, the most divergent of their
+variations will generally be preserved during the next thousand
+generations.  And after this interval, variety a1 is supposed in the
+diagram to have produced variety a2, which will, owing to the principle of
+divergence, differ more from (A) than did variety  a1.  Variety m1 is
+supposed to have produced two varieties, namely m2 and s2, differing from
+each other, and more considerably from their common parent (A).  We may
+continue the process by similar steps for any length of time; some of the
+varieties, after each thousand generations, producing only a single
+variety, but in a more and more modified condition, some producing two or
+three varieties, and some failing to produce any.  Thus the varieties or
+modified descendants, proceeding from the common parent (A), will generally
+go on increasing in number and diverging in character.  In the diagram the
+process is represented up to the ten-thousandth generation, and under a
+condensed and simplified form up to the fourteen-thousandth generation.
+</para>
+
+<para>
+But I must here remark that I do not suppose that the process ever goes on
+so regularly as is represented in the diagram, though in itself made
+somewhat irregular.  I am far from thinking that the most divergent
+varieties will invariably prevail and multiply:  a medium form may often
+long endure, and may or may not produce more than one modified descendant;
+for natural selection will always act according to the nature of the places
+which are either unoccupied or not perfectly occupied by other beings; and
+this will depend on infinitely complex relations.  But as a general rule,
+the more diversified in structure the descendants from any one species can
+be rendered, the more places they will be enabled to seize on, and the more
+their modified progeny will be increased.  In our diagram the line of
+succession is broken at regular intervals by small numbered letters marking
+the successive forms which have become sufficiently distinct to be recorded
+as varieties.  But these breaks are imaginary, and might have been inserted
+anywhere, after intervals long enough to have allowed the accumulation of a
+considerable amount of divergent variation.
+</para>
+
+<para>
+As all the modified descendants from a common and widely-diffused species,
+belonging to a large genus, will tend to partake of the same advantages
+which made their parent successful in life, they will generally go on
+multiplying in number as well as diverging in character:  this is
+represented in the diagram by the several divergent branches proceeding
+from (A).  The modified offspring from the later and more highly improved
+branches in the lines of descent, will, it is probable, often take the
+place of, and so destroy, the earlier and less improved branches:  this is
+represented in the diagram by some of the lower branches not reaching to
+the upper horizontal lines.  In some cases I do not doubt that the process
+of modification will be confined to a single line of descent, and the
+number of the descendants will not be increased; although the amount of
+divergent modification may have been increased in the successive
+generations.  This case would be represented in the diagram, if all the
+lines proceeding from (A) were removed, excepting that from a1 to a10.  In
+the same way, for instance, the English race-horse and English pointer have
+apparently both gone on slowly diverging in character from their original
+stocks, without either having given off any fresh branches or races.
+</para>
+
+<para>
+After ten thousand generations, species (A) is supposed to have produced
+three forms, a10, f10, and m10, which, from having diverged in character
+during the successive generations, will have come to differ largely, but
+perhaps unequally, from each other and from their common parent.  If we
+suppose the amount of change between each horizontal line in our diagram to
+be excessively small, these three forms may still be only well-marked
+varieties; or they may have arrived at the doubtful category of
+sub-species; but we have only to suppose the steps in the process of
+modification to be more numerous or greater in amount, to convert these
+three forms into well-defined species:  thus the diagram illustrates the
+steps by which the small differences distinguishing varieties are increased
+into the larger differences distinguishing species.  By continuing the same
+process for a greater number of generations (as shown in the diagram in a
+condensed and simplified manner), we get eight species, marked by the
+letters between a14 and m14, all descended from (A).  Thus, as I believe,
+species are multiplied and genera are formed.
+</para>
+
+<para>
+In a large genus it is probable that more than one species would vary.  In
+the diagram I have assumed that a second species (I) has produced, by
+analogous steps, after ten thousand generations, either two well-marked
+varieties (w10 and z10) or two species, according to the amount of change
+supposed to be represented between the horizontal lines.  After fourteen
+thousand generations, six new species, marked by the letters n14 to z14,
+are supposed to have been produced.  In each genus, the species, which are
+already extremely different in character, will generally tend to produce
+the greatest number of modified descendants; for these will have the best
+chance of filling new and widely different places in the polity of nature: 
+hence in the diagram I have chosen the extreme species (A), and the nearly
+extreme species (I), as those which have largely varied, and have given
+rise to new varieties and species.  The other nine species (marked by
+capital letters) of our original genus, may for a long period continue
+transmitting unaltered descendants; and this is shown in the diagram by the
+dotted lines not prolonged far upwards from want of space.
+</para>
+
+<para>
+But during the process of modification, represented in the diagram, another
+of our principles, namely that of extinction, will have played an important
+part.  As in each fully stocked country natural selection necessarily acts
+by the selected form having some advantage in the struggle for life over
+other forms, there will be a constant tendency in the improved descendants
+of any one species to supplant and exterminate in each stage of descent
+their predecessors and their original parent.  For it should be remembered
+that the competition will generally be most severe between those forms
+which are most nearly related to each other in habits, constitution, and
+structure.  Hence all the intermediate forms between the earlier and later
+states, that is between the less and more improved state of a species, as
+well as the original parent-species itself, will generally tend to become
+extinct.  So it probably will be with many whole collateral lines of
+descent, which will be conquered by later and improved lines of descent. 
+If, however, the modified offspring of a species get into some distinct
+country, or become quickly adapted to some quite new station, in which
+child and parent do not come into competition, both may continue to exist.
+</para>
+
+<para>
+If then our diagram be assumed to represent a considerable amount of
+modification, species (A) and all the earlier varieties will have become
+extinct, having been replaced by eight new species (a14 to m14); and (I)
+will have been replaced by six (n14 to z14) new species.
+</para>
+
+<para>
+But we may go further than this.  The original species of our genus were
+supposed to resemble each other in unequal degrees, as is so generally the
+case in nature; species (A) being more nearly related to B, C, and D, than
+to the other species; and species (I) more to G, H, K, L, than to the
+others.  These two species (A) and (I), were also supposed to be very
+common and widely diffused species, so that they must originally have had
+some advantage over most of the other species of the genus.  Their modified
+descendants, fourteen in number at the fourteen-thousandth generation, will
+probably have inherited some of the same advantages:  they have also been
+modified and improved in a diversified manner at each stage of descent, so
+as to have become adapted to many related places in the natural economy of
+their country.  It seems, therefore, to me extremely probable that they
+will have taken the places of, and thus exterminated, not only their
+parents (A) and (I), but likewise some of the original species which were
+most nearly related to their parents.  Hence very few of the original
+species will have transmitted offspring to the fourteen-thousandth
+generation.  We may suppose that only one (F), of the two species which
+were least closely related to the other nine original species, has
+transmitted descendants to this late stage of descent.
+</para>
+
+<para>
+The new species in our diagram descended from the original eleven species,
+will now be fifteen in number.  Owing to the divergent tendency of natural
+selection, the extreme amount of difference in character between species
+a14 and z14 will be much greater than that between the most different of
+the original eleven species.  The new species, moreover, will be allied to
+each other in a widely different manner.  Of the eight descendants from (A)
+the three marked a14, q14, p14, will be nearly related from having recently
+branched off from a10; b14 and f14, from having diverged at an earlier
+period from a5, will be in some degree distinct from the three first-named
+species; and lastly, o14, e14, and m14, will be nearly related one to the
+other, but from having diverged at the first commencement of the process of
+modification, will be widely different from the other five species, and may
+constitute a sub-genus or even a distinct genus.  
+</para>
+
+<para>
+The six descendants from (I) will form two sub-genera or even genera.  But
+as the original species (I) differed largely from (A), standing nearly at
+the extreme points of the original genus, the six descendants from (I)
+will, owing to inheritance, differ considerably from the eight descendants
+from (A); the two groups, moreover, are supposed to have gone on diverging
+in different directions.  The intermediate species, also (and this is a
+very important consideration), which connected the original species (A) and
+(I), have all become, excepting (F), extinct, and have left no descendants. 
+Hence the six new species descended from (I), and the eight descended from
+(A), will have to be ranked as very distinct genera, or even as distinct
+sub-families.
+</para>
+
+<para>
+Thus it is, as I believe, that two or more genera are produced by descent,
+with modification, from two or more species of the same genus.  And the two
+or more parent-species are supposed to have descended from some one species
+of an earlier genus.  In our diagram, this is indicated by the broken
+lines, beneath the capital letters, converging in sub-branches downwards
+towards a single point; this point representing a single species, the
+supposed single parent of our several new sub-genera and genera.
+</para>
+
+<para>
+It is worth while to reflect for a moment on the character of the new
+species F14, which is supposed not to have diverged much in character, but
+to have retained the form of (F), either unaltered or altered only in a
+slight degree.  In this case, its affinities to the other fourteen new
+species will be of a curious and circuitous nature.  Having descended from
+a form which stood between the two parent-species (A) and (I), now supposed
+to be extinct and unknown, it will be in some degree intermediate in
+character between the two groups descended from these species.  But as
+these two groups have gone on diverging in character from the type of their
+parents, the new species (F14) will not be directly intermediate between
+them, but rather between types of the two groups; and every naturalist will
+be able to bring some such case before his mind.
+</para>
+
+<para>
+In the diagram, each horizontal line has hitherto been supposed to
+represent a thousand generations, but each may represent a million or
+hundred million generations, and likewise a section of the successive
+strata of the earth's crust including extinct remains.  We shall, when we
+come to our chapter on Geology, have to refer again to this subject, and I
+think we shall then see that the diagram throws light on the affinities of
+extinct beings, which, though generally belonging to the same orders, or
+families, or genera, with those now living, yet are often, in some degree,
+intermediate in character between existing groups; and we can understand
+this fact, for the extinct species lived at very ancient epochs when the
+branching lines of descent had diverged less.
+</para>
+
+<para>
+I see no reason to limit the process of modification, as now explained, to
+the formation of genera alone.  If, in our diagram, we suppose the amount
+of change represented by each successive group of diverging dotted lines to
+be very great, the forms marked a14 to p14, those marked b14 and f14, and
+those marked o14 to m14, will form three very distinct genera.  We shall
+also have two very distinct genera descended from (I) and as these latter
+two genera, both from continued divergence of character and from
+inheritance from a different parent, will differ widely from the three
+genera descended from (A), the two little groups of genera will form two
+distinct families, or even orders, according to the amount of divergent
+modification supposed to be represented in the diagram.  And the two new
+families, or orders, will have descended from two species of the original
+genus; and these two species are supposed to have descended from one
+species of a still more ancient and unknown genus.
+</para>
+
+<para>
+We have seen that in each country it is the species of the larger genera
+which oftenest present varieties or incipient species.  This, indeed, might
+have been expected; for as natural selection acts through one form having
+some advantage over other forms in the struggle for existence, it will
+chiefly act on those which already have some advantage; and the largeness
+of any group shows that its species have inherited from a common ancestor
+some advantage in common.  Hence, the struggle for the production of new
+and modified descendants, will mainly lie between the larger groups, which
+are all trying to increase in number.  One large group will slowly conquer
+another large group, reduce its numbers, and thus lessen its chance of
+further variation and improvement.  Within the same large group, the later
+and more highly perfected sub-groups, from branching out and seizing on
+many new places in the polity of Nature, will constantly tend to supplant
+and destroy the earlier and less improved sub-groups.  Small and broken
+groups and sub-groups will finally tend to disappear.  Looking to the
+future, we can predict that the groups of organic beings which are now
+large and triumphant, and which are least broken up, that is, which as yet
+have suffered least extinction, will for a long period continue to
+increase.  But which groups will ultimately prevail, no man can predict;
+for we well know that many groups, formerly most extensively developed,
+have now become extinct.  Looking still more remotely to the future, we may
+predict that, owing to the continued and steady increase of the larger
+groups, a multitude of smaller groups will become utterly extinct, and
+leave no modified descendants; and consequently that of the species living
+at any one period, extremely few will transmit descendants to a remote
+futurity.  I shall have to return to this subject in the chapter on
+Classification, but I may add that on this view of extremely few of the
+more ancient species having transmitted descendants, and on the view of all
+the descendants of the same species making a class, we can understand how
+it is that there exist but very few classes in each main division of the
+animal and vegetable kingdoms.  Although extremely few of the most ancient
+species may now have living and modified descendants, yet at the most
+remote geological period, the earth may have been as well peopled with many
+species of many genera, families, orders, and classes, as at the present
+day.
+</para>
+
+<para>
+Summary of Chapter -- If during the long course of ages and under varying
+conditions of life, organic beings vary at all in the several parts of
+their organisation, and I think this cannot be disputed; if there be, owing
+to the high geometrical powers of increase of each species, at some age,
+season, or year, a severe struggle for life, and this certainly cannot be
+disputed; then, considering the infinite complexity of the relations of all
+organic beings to each other and to their conditions of existence, causing
+an infinite diversity in structure, constitution, and habits, to be
+advantageous to them, I think it would be a most extraordinary fact if no
+variation ever had occurred useful to each being's own welfare, in the same
+way as so many variations have occurred useful to man.  But if variations
+useful to any organic being do occur, assuredly individuals thus
+characterised will have the best chance of being preserved in the struggle
+for life; and from the strong principle of inheritance they will tend to
+produce offspring similarly characterised.  This principle of preservation,
+I have called, for the sake of brevity, Natural Selection.  Natural
+selection, on the principle of qualities being inherited at corresponding
+ages, can modify the egg, seed, or young, as easily as the adult.  Amongst
+many animals, sexual selection will give its aid to ordinary selection, by
+assuring to the most vigorous and best adapted males the greatest number of
+offspring.  Sexual selection will also give characters useful to the males
+alone, in their struggles with other males.
+</para>
+
+<para>
+Whether natural selection has really thus acted in nature, in modifying and
+adapting the various forms of life to their several conditions and
+stations, must be judged of by the general tenour and balance of evidence
+given in the following chapters.  But we already see how it entails
+extinction; and how largely extinction has acted in the world's history,
+geology plainly declares.  Natural selection, also, leads to divergence of
+character; for more living beings can be supported on the same area the
+more they diverge in structure, habits, and constitution, of which we see
+proof by looking at the inhabitants of any small spot or at naturalised
+productions.  Therefore during the modification of the descendants of any
+one species, and during the incessant struggle of all species to increase
+in numbers, the more diversified these descendants become, the better will
+be their chance of succeeding in the battle of life.  Thus the small
+differences distinguishing varieties of the same species, will steadily
+tend to increase till they come to equal the greater differences between
+species of the same genus, or even of distinct genera.
+</para>
+
+<para>
+We have seen that it is the common, the widely-diffused, and widely-ranging
+species, belonging to the larger genera, which vary most; and these will
+tend to transmit to their modified offspring that superiority which now
+makes them dominant in their own countries.  Natural selection, as has just
+been remarked, leads to divergence of character and to much extinction of
+the less improved and intermediate forms of life.  On these principles, I
+believe, the nature of the affinities of all organic beings may be
+explained.  It is a truly wonderful fact--the wonder of which we are apt to
+overlook from familiarity--that all animals and all plants throughout all
+time and space should be related to each other in group subordinate to
+group, in the manner which we everywhere behold--namely, varieties of the
+same species most closely related together, species of the same genus less
+closely and unequally related together, forming sections and sub-genera,
+species of distinct genera much less closely related, and genera related in
+different degrees, forming sub-families, families, orders, sub-classes, and
+classes.  The several subordinate groups in any class cannot be ranked in a
+single file, but seem rather to be clustered round points, and these round
+other points, and so on in almost endless cycles.  On the view that each
+species has been independently created, I can see no explanation of this
+great fact in the classification of all organic beings; but, to the best of
+my judgment, it is explained through inheritance and the complex action of
+natural selection, entailing extinction and divergence of character, as we
+have seen illustrated in the diagram.
+</para>
+
+<para>
+The affinities of all the beings of the same class have sometimes been
+represented by a great tree.  I believe this simile largely speaks the
+truth.  The green and budding twigs may represent existing species; and
+those produced during each former year may represent the long succession of
+extinct species.  At each period of growth all the growing twigs have tried
+to branch out on all sides, and to overtop and kill the surrounding twigs
+and branches, in the same manner as species and groups of species have
+tried to overmaster other species in the great battle for life.  The limbs
+divided into great branches, and these into lesser and lesser branches,
+were themselves once, when the tree was small, budding twigs; and this
+connexion of the former and present buds by ramifying branches may well
+represent the classification of all extinct and living species in groups
+subordinate to groups.  Of the many twigs which flourished when the tree
+was a mere bush, only two or three, now grown into great branches, yet
+survive and bear all the other branches; so with the species which lived
+during long-past geological periods, very few now have living and modified
+descendants.  From the first growth of the tree, many a limb and branch has
+decayed and dropped off; and these lost branches of various sizes may
+represent those whole orders, families, and genera which have now no living
+representatives, and which are known to us only from having been found in a
+fossil state.  As we here and there see a thin straggling branch springing
+from a fork low down in a tree, and which by some chance has been favoured
+and is still alive on its summit, so we occasionally see an animal like the
+Ornithorhynchus or Lepidosiren, which in some small degree connects by its
+affinities two large branches of life, and which has apparently been saved
+from fatal competition by having inhabited a protected station.  As buds
+give rise by growth to fresh buds, and these, if vigorous, branch out and
+overtop on all sides many a feebler branch, so by generation I believe it
+has been with the great Tree of Life, which fills with its dead and broken
+branches the crust of the earth, and covers the surface with its ever
+branching and beautiful ramifications.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter V</chapnum>
+
+<title>Laws of Variation</title>
+
+<chapsummary>
+Effects of external conditions -- Use and disuse, combined with natural
+selection; organs of flight and of vision -- Acclimatisation -- Correlation
+of growth -- Compensation and economy of growth -- False correlations --
+Multiple, rudimentary, and lowly organised structures variable -- Parts
+developed in an unusual manner are highly variable: specific characters
+more variable than generic:  secondary sexual characters variable --
+Species of the same genus vary in an analogous manner -- Reversions to long
+lost characters -- Summary.
+</chapsummary>
+</chapheader>
+
+<para>
+I have hitherto sometimes spoken as if the variations--so common and
+multiform in organic beings under domestication, and in a lesser degree in
+those in a state of nature--had been due to chance.  This, of course, is a
+wholly incorrect expression, but it serves to acknowledge plainly our
+ignorance of the cause of each particular variation.  Some authors believe
+it to be as much the function of the reproductive system to produce
+individual differences, or very slight deviations of structure, as to make
+the child like its parents.  But the much greater variability, as well as
+the greater frequency of monstrosities, under domestication or cultivation,
+than under nature, leads me to believe that deviations of structure are in
+some way due to the nature of the conditions of life, to which the parents
+and their more remote ancestors have been exposed during several
+generations.  I have remarked in the first chapter--but a long catalogue of
+facts which cannot be here given would be necessary to show the truth of
+the remark--that the reproductive system is eminently susceptible to
+changes in the conditions of life; and to this system being functionally
+disturbed in the parents, I chiefly attribute the varying or plastic
+condition of the offspring.  The male and female sexual elements seem to be
+affected before that union takes place which is to form a new being.  In
+the case of 'sporting' plants, the bud, which in its earliest condition
+does not apparently differ essentially from an ovule, is alone affected. 
+But why, because the reproductive system is disturbed, this or that part
+should vary more or less, we are profoundly ignorant.  Nevertheless, we can
+here and there dimly catch a faint ray of light, and we may feel sure that
+there must be some cause for each deviation of structure, however slight.
+</para>
+
+<para>
+How much direct effect difference of climate, food, &amp;c., produces on any
+being is extremely doubtful.  My impression is, that the effect is
+extremely small in the case of animals, but perhaps rather more in that of
+plants.  We may, at least, safely conclude that such influences cannot have
+produced the many striking and complex co-adaptations of structure between
+one organic being and another, which we see everywhere throughout nature. 
+Some little influence may be attributed to climate, food, &amp;c.:  thus, E.
+Forbes speaks confidently that shells at their southern limit, and when
+living in shallow water, are more brightly coloured than those of the same
+species further north or from greater depths.  Gould believes that birds of
+the same species are more brightly coloured under a clear atmosphere, than
+when living on islands or near the coast.  So with insects, Wollaston is
+convinced that residence near the sea affects their colours.  Moquin-Tandon
+gives a list of plants which when growing near the sea-shore have their
+leaves in some degree fleshy, though not elsewhere fleshy.  Several other
+such cases could be given.
+</para>
+
+<para>
+The fact of varieties of one species, when they range into the zone of
+habitation of other species, often acquiring in a very slight degree some
+of the characters of such species, accords with our view that species of
+all kinds are only well-marked and permanent varieties.  Thus the species
+of shells which are confined to tropical and shallow seas are generally
+brighter-coloured than those confined to cold and deeper seas.  The birds
+which are confined to continents are, according to Mr. Gould,
+brighter-coloured than those of islands.  The insect-species confined to
+sea-coasts, as every collector knows, are often brassy or lurid.  Plants
+which live exclusively on the sea-side are very apt to have fleshy leaves. 
+He who believes in the creation of each species, will have to say that this
+shell, for instance, was created with bright colours for a warm sea; but
+that this other shell became bright-coloured by variation when it ranged
+into warmer or shallower waters.
+</para>
+
+<para>
+When a variation is of the slightest use to a being, we cannot tell how
+much of it to attribute to the accumulative action of natural selection,
+and how much to the conditions of life.  Thus, it is well known to furriers
+that animals of the same species have thicker and better fur the more
+severe the climate is under which they have lived; but who can tell how
+much of this difference may be due to the warmest-clad individuals having
+been favoured and preserved during many generations, and how much to the
+direct action of the severe climate? for it would appear that climate has
+some direct action on the hair of our domestic quadrupeds.
+</para>
+
+<para>
+Instances could be given of the same variety being produced under
+conditions of life as different as can well be conceived; and, on the other
+hand, of different varieties being produced from the same species under the
+same conditions.  Such facts show how indirectly the conditions of life
+must act.  Again, innumerable instances are known to every naturalist of
+species keeping true, or not varying at all, although living under the most
+opposite climates.  Such considerations as these incline me to lay very
+little weight on the direct action of the conditions of life.  Indirectly,
+as already remarked, they seem to play an important part in affecting the
+reproductive system, and in thus inducing variability; and natural
+selection will then accumulate all profitable variations, however slight,
+until they become plainly developed and appreciable by us.
+</para>
+
+<para>
+Effects of Use and Disuse. -- From the facts alluded to in the first
+chapter, I think there can be little doubt that use in our domestic animals
+strengthens and enlarges certain parts, and disuse diminishes them; and
+that such modifications are inherited.  Under free nature, we can have no
+standard of comparison, by which to judge of the effects of long-continued
+use or disuse, for we know not the parent-forms; but many animals have
+structures which can be explained by the effects of disuse.  As Professor
+Owen has remarked, there is no greater anomaly in nature than a bird that
+cannot fly; yet there are several in this state.  The logger-headed duck of
+South America can only flap along the surface of the water, and has its
+wings in nearly the same condition as the domestic Aylesbury duck.  As the
+larger ground-feeding birds seldom take flight except to escape danger, I
+believe that the nearly wingless condition of several birds, which now
+inhabit or have lately inhabited several oceanic islands, tenanted by no
+beast of prey, has been caused by disuse.  The ostrich indeed inhabits
+continents and is exposed to danger from which it cannot escape by flight,
+but by kicking it can defend itself from enemies, as well as any of the
+smaller quadrupeds.  We may imagine that the early progenitor of the
+ostrich had habits like those of a bustard, and that as natural selection
+increased in successive generations the size and weight of its body, its
+legs were used more, and its wings less, until they became incapable of
+flight.
+</para>
+
+<para>
+Kirby has remarked (and I have observed the same fact) that the anterior
+tarsi, or feet, of many male dung-feeding beetles are very often broken
+off; he examined seventeen specimens in his own collection, and not one had
+even a relic left.  In the Onites apelles the tarsi are so habitually lost,
+that the insect has been described as not having them.  In some other
+genera they are present, but in a rudimentary condition.  In the Ateuchus
+or sacred beetle of the Egyptians, they are totally deficient.  There is
+not sufficient evidence to induce us to believe that mutilations are ever
+inherited; and I should prefer explaining the entire absence of the
+anterior tarsi in Ateuchus, and their rudimentary condition in some other
+genera, by the long-continued effects of disuse in their progenitors; for
+as the tarsi are almost always lost in many dung-feeding beetles, they must
+be lost early in life, and therefore cannot be much used by these insects.
+</para>
+
+<para>
+In some cases we might easily put down to disuse modifications of structure
+which are wholly, or mainly, due to natural selection.  Mr. Wollaston has
+discovered the remarkable fact that 200 beetles, out of the 550 species
+inhabiting Madeira, are so far deficient in wings that they cannot fly; and
+that of the twenty-nine endemic genera, no less than twenty-three genera
+have all their species in this condition!  Several facts, namely, that
+beetles in many parts of the world are very frequently blown to sea and
+perish; that the beetles in Madeira, as observed by Mr. Wollaston, lie much
+concealed, until the wind lulls and the sun shines; that the proportion of
+wingless beetles is larger on the exposed Dezertas than in Madeira itself;
+and especially the extraordinary fact, so strongly insisted on by Mr.
+Wollaston, of the almost entire absence of certain large groups of beetles,
+elsewhere excessively numerous, and which groups have habits of life almost
+necessitating frequent flight;--these several considerations have made me
+believe that the wingless condition of so many Madeira beetles is mainly
+due to the action of natural selection, but combined probably with disuse. 
+For during thousands of successive generations each individual beetle which
+flew least, either from its wings having been ever so little less perfectly
+developed or from indolent habit, will have had the best chance of
+surviving from not being blown out to sea; and, on the other hand, those
+beetles which most readily took to flight will oftenest have been blown to
+sea and thus have been destroyed.
+</para>
+
+<para>
+The insects in Madeira which are not ground-feeders, and which, as the
+flower-feeding coleoptera and lepidoptera, must habitually use their wings
+to gain their subsistence, have, as Mr. Wollaston suspects, their wings not
+at all reduced, but even enlarged.  This is quite compatible with the
+action of natural selection.  For when a new insect first arrived on the
+island, the tendency of natural selection to enlarge or to reduce the
+wings, would depend on whether a greater number of individuals were saved
+by successfully battling with the winds, or by giving up the attempt and
+rarely or never flying.  As with mariners shipwrecked near a coast, it
+would have been better for the good swimmers if they had been able to swim
+still further, whereas it would have been better for the bad swimmers if
+they had not been able to swim at all and had stuck to the wreck.
+</para>
+
+<para>
+The eyes of moles and of some burrowing rodents are rudimentary in size,
+and in some cases are quite covered up by skin and fur.  This state of the
+eyes is probably due to gradual reduction from disuse, but aided perhaps by
+natural selection.  In South America, a burrowing rodent, the tuco-tuco, or
+Ctenomys, is even more subterranean in its habits than the mole; and I was
+assured by a Spaniard, who had often caught them, that they were frequently
+blind; one which I kept alive was certainly in this condition, the cause,
+as appeared on dissection, having been inflammation of the nictitating
+membrane.  As frequent inflammation of the eyes must be injurious to any
+animal, and as eyes are certainly not indispensable to animals with
+subterranean habits, a reduction in their size with the adhesion of the
+eyelids and growth of fur over them, might in such case be an advantage;
+and if so, natural selection would constantly aid the effects of disuse.
+</para>
+
+<para>
+It is well known that several animals, belonging to the most different
+classes, which inhabit the caves of Styria and of Kentucky, are blind.  In
+some of the crabs the foot-stalk for the eye remains, though the eye is
+gone; the stand for the telescope is there, though the telescope with its
+glasses has been lost.  As it is difficult to imagine that eyes, though
+useless, could be in any way injurious to animals living in darkness, I
+attribute their loss wholly to disuse.  In one of the blind animals,
+namely, the cave-rat, the eyes are of immense size; and Professor Silliman
+thought that it regained, after living some days in the light, some slight
+power of vision.  In the same manner as in Madeira the wings of some of the
+insects have been enlarged, and the wings of others have been reduced by
+natural selection aided by use and disuse, so in the case of the cave-rat
+natural selection seems to have struggled with the loss of light and to
+have increased the size of the eyes; whereas with all the other inhabitants
+of the caves, disuse by itself seems to have done its work.
+</para>
+
+<para>
+It is difficult to imagine conditions of life more similar than deep
+limestone caverns under a nearly similar climate; so that on the common
+view of the blind animals having been separately created for the American
+and European caverns, close similarity in their organisation and affinities
+might have been expected; but, as Schiodte and others have remarked, this
+is not the case, and the cave-insects of the two continents are not more
+closely allied than might have been anticipated from the general
+resemblance of the other inhabitants of North America and Europe.  On my
+view we must suppose that American animals, having ordinary powers of
+vision, slowly migrated by successive generations from the outer world into
+the deeper and deeper recesses of the Kentucky caves, as did European
+animals into the caves of Europe.  We have some evidence of this gradation
+of habit; for, as Schiodte remarks, 'animals not far remote from ordinary
+forms, prepare the transition from light to darkness.  Next follow those
+that are constructed for twilight; and, last of all, those destined for
+total darkness.'  By the time that an animal had reached, after numberless
+generations, the deepest recesses, disuse will on this view have more or
+less perfectly obliterated its eyes, and natural selection will often have
+effected other changes, such as an increase in the length of the antennae
+or palpi, as a compensation for blindness.  Notwithstanding such
+modifications, we might expect still to see in the cave-animals of America,
+affinities to the other inhabitants of that continent, and in those of
+Europe, to the inhabitants of the European continent.  And this is the case
+with some of the American cave-animals, as I hear from Professor Dana; and
+some of the European cave-insects are very closely allied to those of the
+surrounding country.  It would be most difficult to give any rational
+explanation of the affinities of the blind cave-animals to the other
+inhabitants of the two continents on the ordinary view of their independent
+creation.  That several of the inhabitants of the caves of the Old and New
+Worlds should be closely related, we might expect from the well-known
+relationship of most of their other productions.  Far from feeling any
+surprise that some of the cave-animals should be very anomalous, as Agassiz
+has remarked in regard to the blind fish, the Amblyopsis, and as is the
+case with the blind Proteus with reference to the reptiles of Europe, I am
+only surprised that more wrecks of ancient life have not been preserved,
+owing to the less severe competition to which the inhabitants of these dark
+abodes will probably have been exposed.
+</para>
+
+<para>
+Acclimatisation. -- Habit is hereditary with plants, as in the period of
+flowering, in the amount of rain requisite for seeds to germinate, in the
+time of sleep, &amp;c., and this leads me to say a few words on
+acclimatisation.  As it is extremely common for species of the same genus
+to inhabit very hot and very cold countries, and as I believe that all the
+species of the same genus have descended from a single parent, if this view
+be correct, acclimatisation must be readily effected during long-continued
+descent.  It is notorious that each species is adapted to the climate of
+its own home:  species from an arctic or even from a temperate region
+cannot endure a tropical climate, or conversely.  So again, many succulent
+plants cannot endure a damp climate.  But the degree of adaptation of
+species to the climates under which they live is often overrated.  We may
+infer this from our frequent inability to predict whether or not an
+imported plant will endure our climate, and from the number of plants and
+animals brought from warmer countries which here enjoy good health.  We
+have reason to believe that species in a state of nature are limited in
+their ranges by the competition of other organic beings quite as much as,
+or more than, by adaptation to particular climates.  But whether or not the
+adaptation be generally very close, we have evidence, in the case of some
+few plants, of their becoming, to a certain extent, naturally habituated to
+different temperatures, or becoming acclimatised:  thus the pines and
+rhododendrons, raised from seed collected by Dr. Hooker from trees growing
+at different heights on the Himalaya, were found in this country to possess
+different constitutional powers of resisting cold.  Mr. Thwaites informs me
+that he has observed similar facts in Ceylon, and analogous observations
+have been made by Mr. H. C. Watson on European species of plants brought
+from the Azores to England.  In regard to animals, several authentic cases
+could be given of species within historical times having largely extended
+their range from warmer to cooler latitudes, and conversely; but we do not
+positively know that these animals were strictly adapted to their native
+climate, but in all ordinary cases we assume such to be the case; nor do we
+know that they have subsequently become acclimatised to their new homes.
+</para>
+
+<para>
+As I believe that our domestic animals were originally chosen by
+uncivilised man because they were useful and bred readily under
+confinement, and not because they were subsequently found capable of
+far-extended transportation, I think the common and extraordinary capacity
+in our domestic animals of not only withstanding the most different
+climates but of being perfectly fertile (a far severer test) under them,
+may be used as an argument that a large proportion of other animals, now in
+a state of nature, could easily be brought to bear widely different
+climates.  We must not, however, push the foregoing argument too far, on
+account of the probable origin of some of our domestic animals from several
+wild stocks:  the blood, for instance, of a tropical and arctic wolf or
+wild dog may perhaps be mingled in our domestic breeds.  The rat and mouse
+cannot be considered as domestic animals, but they have been transported by
+man to many parts of the world, and now have a far wider range than any
+other rodent, living free under the cold climate of Faroe in the north and
+of the Falklands in the south, and on many islands in the torrid zones. 
+Hence I am inclined to look at adaptation to any special climate as a
+quality readily grafted on an innate wide flexibility of constitution,
+which is common to most animals.  On this view, the capacity of enduring
+the most different climates by man himself and by his domestic animals, and
+such facts as that former species of the elephant and rhinoceros were
+capable of enduring a glacial climate, whereas the living species are now
+all tropical or sub-tropical in their habits, ought not to be looked at as
+anomalies, but merely as examples of a very common flexibility of
+constitution, brought, under peculiar circumstances, into play.
+</para>
+
+<para>
+How much of the acclimatisation of species to any peculiar climate is due
+to mere habit, and how much to the natural selection of varieties having
+different innate constitutions, and how much to both means combined, is a
+very obscure question.  That habit or custom has some influence I must
+believe, both from analogy, and from the incessant advice given in
+agricultural works, even in the ancient Encyclopaedias of China, to be very
+cautious in transposing animals from one district to another; for it is not
+likely that man should have succeeded in selecting so many breeds and
+sub-breeds with constitutions specially fitted for their own districts: 
+the result must, I think, be due to habit.  On the other hand, I can see no
+reason to doubt that natural selection will continually tend to preserve
+those individuals which are born with constitutions best adapted to their
+native countries.  In treatises on many kinds of cultivated plants, certain
+varieties are said to withstand certain climates better than others:  this
+is very strikingly shown in works on fruit trees published in the United
+States, in which certain varieties are habitually recommended for the
+northern, and others for the southern States; and as most of these
+varieties are of recent origin, they cannot owe their constitutional
+differences to habit.  The case of the Jerusalem artichoke, which is never
+propagated by seed, and of which consequently new varieties have not been
+produced, has even been advanced--for it is now as tender as ever it
+was--as proving that acclimatisation cannot be effected!  The case, also,
+of the kidney-bean has been often cited for a similar purpose, and with
+much greater weight; but until some one will sow, during a score of
+generations, his kidney-beans so early that a very large proportion are
+destroyed by frost, and then collect seed from the few survivors, with care
+to prevent accidental crosses, and then again get seed from these
+seedlings, with the same precautions, the experiment cannot be said to have
+been even tried.  Nor let it be supposed that no differences in the
+constitution of seedling kidney-beans ever appear, for an account has been
+published how much more hardy some seedlings appeared to be than others.
+</para>
+
+<para>
+On the whole, I think we may conclude that habit, use, and disuse, have, in
+some cases, played a considerable part in the modification of the
+constitution, and of the structure of various organs; but that the effects
+of use and disuse have often been largely combined with, and sometimes
+overmastered by, the natural selection of innate differences.
+</para>
+
+<para>
+Correlation of Growth. -- I mean by this expression that the whole
+organisation is so tied together during its growth and development, that
+when slight variations in any one part occur, and are accumulated through
+natural selection, other parts become modified.  This is a very important
+subject, most imperfectly understood.  The most obvious case is, that
+modifications accumulated solely for the good of the young or larva, will,
+it may safely be concluded, affect the structure of the adult; in the same
+manner as any malconformation affecting the early embryo, seriously affects
+the whole organisation of the adult.  The several parts of the body which
+are homologous, and which, at an early embryonic period, are alike, seem
+liable to vary in an allied manner:  we see this in the right and left
+sides of the body varying in the same manner; in the front and hind legs,
+and even in the jaws and limbs, varying together, for the lower jaw is
+believed to be homologous with the limbs.  These tendencies, I do not
+doubt, may be mastered more or less completely by natural selection:  thus
+a family of stags once existed with an antler only on one side; and if this
+had been of any great use to the breed it might probably have been rendered
+permanent by natural selection.
+</para>
+
+<para>
+Homologous parts, as has been remarked by some authors, tend to cohere;
+this is often seen in monstrous plants; and nothing is more common than the
+union of homologous parts in normal structures, as the union of the petals
+of the corolla into a tube.  Hard parts seem to affect the form of
+adjoining soft parts; it is believed by some authors that the diversity in
+the shape of the pelvis in birds causes the remarkable diversity in the
+shape of their kidneys.  Others believe that the shape of the pelvis in the
+human mother influences by pressure the shape of the head of the child.  In
+snakes, according to Schlegel, the shape of the body and the manner of
+swallowing determine the position of several of the most important viscera.
+</para>
+
+<para>
+The nature of the bond of correlation is very frequently quite obscure.  M.
+Is. Geoffroy St. Hilaire has forcibly remarked, that certain
+malconformations very frequently, and that others rarely coexist, without
+our being able to assign any reason.  What can be more singular than the
+relation between blue eyes and deafness in cats, and the tortoise-shell
+colour with the female sex; the feathered feet and skin between the outer
+toes in pigeons, and the presence of more or less down on the young birds
+when first hatched, with the future colour of their plumage; or, again, the
+relation between the hair and teeth in the naked Turkish dog, though here
+probably homology comes into play?  With respect to this latter case of
+correlation, I think it can hardly be accidental, that if we pick out the
+two orders of mammalia which are most abnormal in their dermal coverings,
+viz. Cetacea (whales) and Edentata (armadilloes, scaly ant-eaters, &amp;c.),
+that these are likewise the most abnormal in their teeth.
+</para>
+
+<para>
+I know of no case better adapted to show the importance of the laws of
+correlation in modifying important structures, independently of utility
+and, therefore, of natural selection, than that of the difference between
+the outer and inner flowers in some Compositous and Umbelliferous plants. 
+Every one knows the difference in the ray and central florets of, for
+instance, the daisy, and this difference is often accompanied with the
+abortion of parts of the flower.  But, in some Compositous plants, the
+seeds also differ in shape and sculpture; and even the ovary itself, with
+its accessory parts, differs, as has been described by Cassini.  These
+differences have been attributed by some authors to pressure, and the shape
+of the seeds in the ray-florets in some Compositae countenances this idea;
+but, in the case of the corolla of the Umbelliferae, it is by no means, as
+Dr. Hooker informs me, in species with the densest heads that the inner and
+outer flowers most frequently differ.  It might have been thought that the
+development of the ray-petals by drawing nourishment from certain other
+parts of the flower had caused their abortion; but in some Compositae there
+is a difference in the seeds of the outer and inner florets without any
+difference in the corolla.  Possibly, these several differences may be
+connected with some difference in the flow of nutriment towards the central
+and external flowers:  we know, at least, that in irregular flowers, those
+nearest to the axis are oftenest subject to peloria, and become regular.  I
+may add, as an instance of this, and of a striking case of correlation,
+that I have recently observed in some garden pelargoniums, that the central
+flower of the truss often loses the patches of darker colour in the two
+upper petals; and that when this occurs, the adherent nectary is quite
+aborted; when the colour is absent from only one of the two upper petals,
+the nectary is only much shortened.
+</para>
+
+<para>
+With respect to the difference in the corolla of the central and exterior
+flowers of a head or umbel, I do not feel at all sure that C. C. Sprengel's
+idea that the ray-florets serve to attract insects, whose agency is highly
+advantageous in the fertilisation of plants of these two orders, is so
+far-fetched, as it may at first appear:  and if it be advantageous, natural
+selection may have come into play.  But in regard to the differences both
+in the internal and external structure of the seeds, which are not always
+correlated with any differences in the flowers, it seems impossible that
+they can be in any way advantageous to the plant:  yet in the Umbelliferae
+these differences are of such apparent importance--the seeds being in some
+cases, according to Tausch, orthospermous in the exterior flowers and
+coelospermous in the central flowers,--that the elder De Candolle founded
+his main divisions of the order on analogous differences.  Hence we see
+that modifications of structure, viewed by systematists as of high value,
+may be wholly due to unknown laws of correlated growth, and without being,
+as far as we can see, of the slightest service to the species.
+</para>
+
+<para>
+We may often falsely attribute to correlation of growth, structures which
+are common to whole groups of species, and which in truth are simply due to
+inheritance; for an ancient progenitor may have acquired through natural
+selection some one modification in structure, and, after thousands of
+generations, some other and independent modification; and these two
+modifications, having been transmitted to a whole group of descendants with
+diverse habits, would naturally be thought to be correlated in some
+necessary manner.  So, again, I do not doubt that some apparent
+correlations, occurring throughout whole orders, are entirely due to the
+manner alone in which natural selection can act.  For instance, Alph. De
+Candolle has remarked that winged seeds are never found in fruits which do
+not open:  I should explain the rule by the fact that seeds could not
+gradually become winged through natural selection, except in fruits which
+opened; so that the individual plants producing seeds which were a little
+better fitted to be wafted further, might get an advantage over those
+producing seed less fitted for dispersal; and this process could not
+possibly go on in fruit which did not open.
+</para>
+
+<para>
+The elder Geoffroy and Goethe propounded, at about the same period, their
+law of compensation or balancement of growth; or, as Goethe expressed it,
+'in order to spend on one side, nature is forced to economise on the other
+side.'  I think this holds true to a certain extent with our domestic
+productions:  if nourishment flows to one part or organ in excess, it
+rarely flows, at least in excess, to another part; thus it is difficult to
+get a cow to give much milk and to fatten readily.  The same varieties of
+the cabbage do not yield abundant and nutritious foliage and a copious
+supply of oil-bearing seeds.  When the seeds in our fruits become
+atrophied, the fruit itself gains largely in size and quality.  In our
+poultry, a large tuft of feathers on the head is generally accompanied by a
+diminished comb, and a large beard by diminished wattles.  With species in
+a state of nature it can hardly be maintained that the law is of universal
+application; but many good observers, more especially botanists, believe in
+its truth.  I will not, however, here give any instances, for I see hardly
+any way of distinguishing between the effects, on the one hand, of a part
+being largely developed through natural selection and another and adjoining
+part being reduced by this same process or by disuse, and, on the other
+hand, the actual withdrawal of nutriment from one part owing to the excess
+of growth in another and adjoining part.
+</para>
+
+<para>
+I suspect, also, that some of the cases of compensation which have been
+advanced, and likewise some other facts, may be merged under a more general
+principle, namely, that natural selection is continually trying to
+economise in every part of the organisation.  If under changed conditions
+of life a structure before useful becomes less useful, any diminution,
+however slight, in its development, will be seized on by natural selection,
+for it will profit the individual not to have its nutriment wasted in
+building up an useless structure.  I can thus only understand a fact with
+which I was much struck when examining cirripedes, and of which many other
+instances could be given:  namely, that when a cirripede is parasitic
+within another and is thus protected, it loses more or less completely its
+own shell or carapace.  This is the case with the male Ibla, and in a truly
+extraordinary manner with the Proteolepas:  for the carapace in all other
+cirripedes consists of the three highly-important anterior segments of the
+head enormously developed, and furnished with great nerves and muscles; but
+in the parasitic and protected Proteolepas, the whole anterior part of the
+head is reduced to the merest rudiment attached to the bases of the
+prehensile antennae.  Now the saving of a large and complex structure, when
+rendered superfluous by the parasitic habits of the Proteolepas, though
+effected by slow steps, would be a decided advantage to each successive
+individual of the species; for in the struggle for life to which every
+animal is exposed, each individual Proteolepas would have a better chance
+of supporting itself, by less nutriment being wasted in developing a
+structure now become useless.
+</para>
+
+<para>
+Thus, as I believe, natural selection will always succeed in the long run
+in reducing and saving every part of the organisation, as soon as it is
+rendered superfluous, without by any means causing some other part to be
+largely developed in a corresponding degree.  And, conversely, that natural
+selection may perfectly well succeed in largely developing any organ,
+without requiring as a necessary compensation the reduction of some
+adjoining part.
+</para>
+
+<para>
+It seems to be a rule, as remarked by Is. Geoffroy St. Hilaire, both in
+varieties and in species, that when any part or organ is repeated many
+times in the structure of the same individual (as the vertebrae in snakes,
+and the stamens in polyandrous flowers) the number is variable; whereas the
+number of the same part or organ, when it occurs in lesser numbers, is
+constant.  The same author and some botanists have further remarked that
+multiple parts are also very liable to variation in structure.  Inasmuch as
+this 'vegetative repetition,' to use Prof. Owen's expression, seems to be a
+sign of low organisation; the foregoing remark seems connected with the
+very general opinion of naturalists, that beings low in the scale of nature
+are more variable than those which are higher.  I presume that lowness in
+this case means that the several parts of the organisation have been but
+little specialised for particular functions; and as long as the same part
+has to perform diversified work, we can perhaps see why it should remain
+variable, that is, why natural selection should have preserved or rejected
+each little deviation of form less carefully than when the part has to
+serve for one special purpose alone.  In the same way that a knife which
+has to cut all sorts of things may be of almost any shape; whilst a tool
+for some particular object had better be of some particular shape.  Natural
+selection, it should never be forgotten, can act on each part of each
+being, solely through and for its advantage.
+</para>
+
+<para>
+Rudimentary parts, it has been stated by some authors, and I believe with
+truth, are apt to be highly variable.  We shall have to recur to the
+general subject of rudimentary and aborted organs; and I will here only add
+that their variability seems to be owing to their uselessness, and
+therefore to natural selection having no power to check deviations in their
+structure.  Thus rudimentary parts are left to the free play of the various
+laws of growth, to the effects of long-continued disuse, and to the
+tendency to reversion.
+</para>
+
+<para>
+A part developed in any species in an extraordinary degree or manner, in
+comparison with the same part in allied species, tends to be highly
+variable. -- Several years ago I was much struck with a remark, nearly to
+the above effect, published by Mr. Waterhouse.  I infer also from an
+observation made by Professor Owen, with respect to the length of the arms
+of the ourang-outang, that he has come to a nearly similar conclusion.  It
+is hopeless to attempt to convince any one of the truth of this proposition
+without giving the long array of facts which I have collected, and which
+cannot possibly be here introduced.  I can only state my conviction that it
+is a rule of high generality.  I am aware of several causes of error, but I
+hope that I have made due allowance for them.  It should be understood that
+the rule by no means applies to any part, however unusually developed,
+unless it be unusually developed in comparison with the same part in
+closely allied species.  Thus, the bat's wing is a most abnormal structure
+in the class mammalia; but the rule would not here apply, because there is
+a whole group of bats having wings; it would apply only if some one species
+of bat had its wings developed in some remarkable manner in comparison with
+the other species of the same genus.  The rule applies very strongly in the
+case of secondary sexual characters, when displayed in any unusual manner. 
+The term, secondary sexual characters, used by Hunter, applies to
+characters which are attached to one sex, but are not directly connected
+with the act of reproduction.  The rule applies to males and females; but
+as females more rarely offer remarkable secondary sexual characters, it
+applies more rarely to them.  The rule being so plainly applicable in the
+case of secondary sexual characters, may be due to the great variability of
+these characters, whether or not displayed in any unusual manner--of which
+fact I think there can be little doubt.  But that our rule is not confined
+to secondary sexual characters is clearly shown in the case of
+hermaphrodite cirripedes; and I may here add, that I particularly attended
+to Mr. Waterhouse's remark, whilst investigating this Order, and I am fully
+convinced that the rule almost invariably holds good with cirripedes.  I
+shall, in my future work, give a list of the more remarkable cases; I will
+here only briefly give one, as it illustrates the rule in its largest
+application.  The opercular valves of sessile cirripedes (rock barnacles)
+are, in every sense of the word, very important structures, and they differ
+extremely little even in different genera; but in the several species of
+one genus, Pyrgoma, these valves present a marvellous amount of
+diversification:  the homologous valves in the different species being
+sometimes wholly unlike in shape; and the amount of variation in the
+individuals of several of the species is so great, that it is no
+exaggeration to state that the varieties differ more from each other in the
+characters of these important valves than do other species of distinct
+genera.
+</para>
+
+<para>
+As birds within the same country vary in a remarkably small degree, I have
+particularly attended to them, and the rule seems to me certainly to hold
+good in this class.  I cannot make out that it applies to plants, and this
+would seriously have shaken my belief in its truth, had not the great
+variability in plants made it particularly difficult to compare their
+relative degrees of variability.
+</para>
+
+<para>
+When we see any part or organ developed in a remarkable degree or manner in
+any species, the fair presumption is that it is of high importance to that
+species; nevertheless the part in this case is eminently liable to
+variation.  Why should this be so?  On the view that each species has been
+independently created, with all its parts as we now see them, I can see no
+explanation.  But on the view that groups of species have descended from
+other species, and have been modified through natural selection, I think we
+can obtain some light.  In our domestic animals, if any part, or the whole
+animal, be neglected and no selection be applied, that part (for instance,
+the comb in the Dorking fowl) or the whole breed will cease to have a
+nearly uniform character.  The breed will then be said to have degenerated. 
+In rudimentary organs, and in those which have been but little specialised
+for any particular purpose, and perhaps in polymorphic groups, we see a
+nearly parallel natural case; for in such cases natural selection either
+has not or cannot come into full play, and thus the organisation is left in
+a fluctuating condition.  But what here more especially concerns us is,
+that in our domestic animals those points, which at the present time are
+undergoing rapid change by continued selection, are also eminently liable
+to variation.  Look at the breeds of the pigeon; see what a prodigious
+amount of difference there is in the beak of the different tumblers, in the
+beak and wattle of the different carriers, in the carriage and tail of our
+fantails, &amp;c., these being the points now mainly attended to by English
+fanciers.  Even in the sub-breeds, as in the short-faced tumbler, it is
+notoriously difficult to breed them nearly to perfection, and frequently
+individuals are born which depart widely from the standard.  There may be
+truly said to be a constant struggle going on between, on the one hand, the
+tendency to reversion to a less modified state, as well as an innate
+tendency to further variability of all kinds, and, on the other hand, the
+power of steady selection to keep the breed true.  In the long run
+selection gains the day, and we do not expect to fail so far as to breed a
+bird as coarse as a common tumbler from a good short-faced strain.  But as
+long as selection is rapidly going on, there may always be expected to be
+much variability in the structure undergoing modification.  It further
+deserves notice that these variable characters, produced by man's
+selection, sometimes become attached, from causes quite unknown to us, more
+to one sex than to the other, generally to the male sex, as with the wattle
+of carriers and the enlarged crop of pouters.
+</para>
+
+<para>
+Now let us turn to nature.  When a part has been developed in an
+extraordinary manner in any one species, compared with the other species of
+the same genus, we may conclude that this part has undergone an
+extraordinary amount of modification, since the period when the species
+branched off from the common progenitor of the genus.  This period will
+seldom be remote in any extreme degree, as species very rarely endure for
+more than one geological period.  An extraordinary amount of modification
+implies an unusually large and long-continued amount of variability, which
+has continually been accumulated by natural selection for the benefit of
+the species.  But as the variability of the extraordinarily-developed part
+or organ has been so great and long-continued within a period not
+excessively remote, we might, as a general rule, expect still to find more
+variability in such parts than in other parts of the organisation, which
+have remained for a much longer period nearly constant.  And this, I am
+convinced, is the case.  That the struggle between natural selection on the
+one hand, and the tendency to reversion and variability on the other hand,
+will in the course of time cease; and that the most abnormally developed
+organs may be made constant, I can see no reason to doubt.  Hence when an
+organ, however abnormal it may be, has been transmitted in approximately
+the same condition to many modified descendants, as in the case of the wing
+of the bat, it must have existed, according to my theory, for an immense
+period in nearly the same state; and thus it comes to be no more variable
+than any other structure.  It is only in those cases in which the
+modification has been comparatively recent and extraordinarily great that
+we ought to find the generative variability, as it may be called, still
+present in a high degree.  For in this case the variability will seldom as
+yet have been fixed by the continued selection of the individuals varying
+in the required manner and degree, and by the continued rejection of those
+tending to revert to a former and less modified condition.
+</para>
+
+<para>
+The principle included in these remarks may be extended.  It is notorious
+that specific characters are more variable than generic.  To explain by a
+simple example what is meant.  If some species in a large genus of plants
+had blue flowers and some had red, the colour would be only a specific
+character, and no one would be surprised at one of the blue species varying
+into red, or conversely; but if all the species had blue flowers, the
+colour would become a generic character, and its variation would be a more
+unusual circumstance.  I have chosen this example because an explanation is
+not in this case applicable, which most naturalists would advance, namely,
+that specific characters are more variable than generic, because they are
+taken from parts of less physiological importance than those commonly used
+for classing genera.  I believe this explanation is partly, yet only
+indirectly, true; I shall, however, have to return to this subject in our
+chapter on Classification.  It would be almost superfluous to adduce
+evidence in support of the above statement, that specific characters are
+more variable than generic; but I have repeatedly noticed in works on
+natural history, that when an author has remarked with surprise that some
+important organ or part, which is generally very constant throughout large
+groups of species, has differed considerably in closely-allied species,
+that it has, also, been variable in the individuals of some of the species. 
+And this fact shows that a character, which is generally of generic value,
+when it sinks in value and becomes only of specific value, often becomes
+variable, though its physiological importance may remain the same. 
+Something of the same kind applies to monstrosities:  at least Is. Geoffroy
+St. Hilaire seems to entertain no doubt, that the more an organ normally
+differs in the different species of the same group, the more subject it is
+to individual anomalies.
+</para>
+
+<para>
+On the ordinary view of each species having been independently created, why
+should that part of the structure, which differs from the same part in
+other independently-created species of the same genus, be more variable
+than those parts which are closely alike in the several species?  I do not
+see that any explanation can be given.  But on the view of species being
+only strongly marked and fixed varieties, we might surely expect to find
+them still often continuing to vary in those parts of their structure which
+have varied within a moderately recent period, and which have thus come to
+differ.  Or to state the case in another manner:--the points in which all
+the species of a genus resemble each other, and in which they differ from
+the species of some other genus, are called generic characters; and these
+characters in common I attribute to inheritance from a common progenitor,
+for it can rarely have happened that natural selection will have modified
+several species, fitted to more or less widely-different habits, in exactly
+the same manner:  and as these so-called generic characters have been
+inherited from a remote period, since that period when the species first
+branched off from their common progenitor, and subsequently have not varied
+or come to differ in any degree, or only in a slight degree, it is not
+probable that they should vary at the present day.  On the other hand, the
+points in which species differ from other species of the same genus, are
+called specific characters; and as these specific characters have varied
+and come to differ within the period of the branching off of the species
+from a common progenitor, it is probable that they should still often be in
+some degree variable,--at least more variable than those parts of the
+organisation which have for a very long period remained constant.
+</para>
+
+<para>
+In connexion with the present subject, I will make only two other remarks. 
+I think it will be admitted, without my entering on details, that secondary
+sexual characters are very variable; I think it also will be admitted that
+species of the same group differ from each other more widely in their
+secondary sexual characters, than in other parts of their organisation;
+compare, for instance, the amount of difference between the males of
+gallinaceous birds, in which secondary sexual characters are strongly
+displayed, with the amount of difference between their females; and the
+truth of this proposition will be granted.  The cause of the original
+variability of secondary sexual characters is not manifest; but we can see
+why these characters should not have been rendered as constant and uniform
+as other parts of the organisation; for secondary sexual characters have
+been accumulated by sexual selection, which is less rigid in its action
+than ordinary selection, as it does not entail death, but only gives fewer
+offspring to the less favoured males.  Whatever the cause may be of the
+variability of secondary sexual characters, as they are highly variable,
+sexual selection will have had a wide scope for action, and may thus
+readily have succeeded in giving to the species of the same group a greater
+amount of difference in their sexual characters, than in other parts of
+their structure.
+</para>
+
+<para>
+It is a remarkable fact, that the secondary sexual differences between the
+two sexes of the same species are generally displayed in the very same
+parts of the organisation in which the different species of the same genus
+differ from each other.  Of this fact I will give in illustration two
+instances, the first which happen to stand on my list; and as the
+differences in these cases are of a very unusual nature, the relation can
+hardly be accidental.  The same number of joints in the tarsi is a
+character generally common to very large groups of beetles, but in the
+Engidae, as Westwood has remarked, the number varies greatly; and the
+number likewise differs in the two sexes of the same species:  again in
+fossorial hymenoptera, the manner of neuration of the wings is a character
+of the highest importance, because common to large groups; but in certain
+genera the neuration differs in the different species, and likewise in the
+two sexes of the same species.  This relation has a clear meaning on my
+view of the subject:  I look at all the species of the same genus as having
+as certainly descended from the same progenitor, as have the two sexes of
+any one of the species.  Consequently, whatever part of the structure of
+the common progenitor, or of its early descendants, became variable;
+variations of this part would it is highly probable, be taken advantage of
+by natural and sexual selection, in order to fit the several species to
+their several places in the economy of nature, and likewise to fit the two
+sexes of the same species to each other, or to fit the males and females to
+different habits of life, or the males to struggle with other males for the
+possession of the females.
+</para>
+
+<para>
+Finally, then, I conclude that the greater variability of specific
+characters, or those which distinguish species from species, than of
+generic characters, or those which the species possess in common;--that the
+frequent extreme variability of any part which is developed in a species in
+an extraordinary manner in comparison with the same part in its congeners;
+and the not great degree of variability in a part, however extraordinarily
+it may be developed, if it be common to a whole group of species;--that the
+great variability of secondary sexual characters, and the great amount of
+difference in these same characters between closely allied species;--that
+secondary sexual and ordinary specific differences are generally displayed
+in the same parts of the organisation,--are all principles closely
+connected together.  All being mainly due to the species of the same group
+having descended from a common progenitor, from whom they have inherited
+much in common,--to parts which have recently and largely varied being more
+likely still to go on varying than parts which have long been inherited and
+have not varied,--to natural selection having more or less completely,
+according to the lapse of time, overmastered the tendency to reversion and
+to further variability,--to sexual selection being less rigid than ordinary
+selection,--and to variations in the same parts having been accumulated by
+natural and sexual selection, and thus adapted for secondary sexual, and
+for ordinary specific purposes.
+</para>
+
+<para>
+Distinct species present analogous variations; and a variety of one species
+often assumes some of the characters of an allied species, or reverts to
+some of the characters of an early progenitor. -- These propositions will
+be most readily understood by looking to our domestic races.  The most
+distinct breeds of pigeons, in countries most widely apart, present
+sub-varieties with reversed feathers on the head and feathers on the
+feet,--characters not possessed by the aboriginal rock-pigeon; these then
+are analogous variations in two or more distinct races.  The frequent
+presence of fourteen or even sixteen tail-feathers in the pouter, may be
+considered as a variation representing the normal structure of another
+race, the fantail.  I presume that no one will doubt that all such
+analogous variations are due to the several races of the pigeon having
+inherited from a common parent the same constitution and tendency to
+variation, when acted on by similar unknown influences.  In the vegetable
+kingdom we have a case of analogous variation, in the enlarged stems, or
+roots as commonly called, of the Swedish turnip and Ruta baga, plants which
+several botanists rank as varieties produced by cultivation from a common
+parent:  if this be not so, the case will then be one of analogous
+variation in two so-called distinct species; and to these a third may be
+added, namely, the common turnip.  According to the ordinary view of each
+species having been independently created, we should have to attribute this
+similarity in the enlarged stems of these three plants, not to the vera
+causa of community of descent, and a consequent tendency to vary in a like
+manner, but to three separate yet closely related acts of creation.
+</para>
+
+<para>
+With pigeons, however, we have another case, namely, the occasional
+appearance in all the breeds, of slaty-blue birds with two black bars on
+the wings, a white rump, a bar at the end of the tail, with the outer
+feathers externally edged near their bases with white.  As all these marks
+are characteristic of the parent rock-pigeon, I presume that no one will
+doubt that this is a case of reversion, and not of a new yet analogous
+variation appearing in the several breeds.  We may I think confidently come
+to this conclusion, because, as we have seen, these coloured marks are
+eminently liable to appear in the crossed offspring of two distinct and
+differently coloured breeds; and in this case there is nothing in the
+external conditions of life to cause the reappearance of the slaty-blue,
+with the several marks, beyond the influence of the mere act of crossing on
+the laws of inheritance.
+</para>
+
+<para>
+No doubt it is a very surprising fact that characters should reappear after
+having been lost for many, perhaps for hundreds of generations.  But when a
+breed has been crossed only once by some other breed, the offspring
+occasionally show a tendency to revert in character to the foreign breed
+for many generations--some say, for a dozen or even a score of generations. 
+After twelve generations, the proportion of blood, to use a common
+expression, of any one ancestor, is only 1 in 2048; and yet, as we see, it
+is generally believed that a tendency to reversion is retained by this very
+small proportion of foreign blood.  In a breed which has not been crossed,
+but in which both parents have lost some character which their progenitor
+possessed, the tendency, whether strong or weak, to reproduce the lost
+character might be, as was formerly remarked, for all that we can see to
+the contrary, transmitted for almost any number of generations.  When a
+character which has been lost in a breed, reappears after a great number of
+generations, the most probable hypothesis is, not that the offspring
+suddenly takes after an ancestor some hundred generations distant, but that
+in each successive generation there has been a tendency to reproduce the
+character in question, which at last, under unknown favourable conditions,
+gains an ascendancy.  For instance, it is probable that in each generation
+of the barb-pigeon, which produces most rarely a blue and black-barred
+bird, there has been a tendency in each generation in the plumage to assume
+this colour.  This view is hypothetical, but could be supported by some
+facts; and I can see no more abstract improbability in a tendency to
+produce any character being inherited for an endless number of generations,
+than in quite useless or rudimentary organs being, as we all know them to
+be, thus inherited.  Indeed, we may sometimes observe a mere tendency to
+produce a rudiment inherited:  for instance, in the common snapdragon
+(Antirrhinum) a rudiment of a fifth stamen so often appears, that this
+plant must have an inherited tendency to produce it.
+</para>
+
+<para>
+As all the species of the same genus are supposed, on my theory, to have
+descended from a common parent, it might be expected that they would
+occasionally vary in an analogous manner; so that a variety of one species
+would resemble in some of its characters another species; this other
+species being on my view only a well-marked and permanent variety.  But
+characters thus gained would probably be of an unimportant nature, for the
+presence of all important characters will be governed by natural selection,
+in accordance with the diverse habits of the species, and will not be left
+to the mutual action of the conditions of life and of a similar inherited
+constitution.  It might further be expected that the species of the same
+genus would occasionally exhibit reversions to lost ancestral characters. 
+As, however, we never know the exact character of the common ancestor of a
+group, we could not distinguish these two cases:  if, for instance, we did
+not know that the rock-pigeon was not feather-footed or turn-crowned, we
+could not have told, whether these characters in our domestic breeds were
+reversions or only analogous variations; but we might have inferred that
+the blueness was a case of reversion, from the number of the markings,
+which are correlated with the blue tint, and which it does not appear
+probable would all appear together from simple variation.  More especially
+we might have inferred this, from the blue colour and marks so often
+appearing when distinct breeds of diverse colours are crossed.  Hence,
+though under nature it must generally be left doubtful, what cases are
+reversions to an anciently existing character, and what are new but
+analogous variations, yet we ought, on my theory, sometimes to find the
+varying offspring of a species assuming characters (either from reversion
+or from analogous variation) which already occur in some other members of
+the same group.  And this undoubtedly is the case in nature.
+</para>
+
+<para>
+A considerable part of the difficulty in recognising a variable species in
+our systematic works, is due to its varieties mocking, as it were, some of
+the other species of the same genus.  A considerable catalogue, also, could
+be given of forms intermediate between two other forms, which themselves
+must be doubtfully ranked as either varieties or species; and this shows,
+unless all these forms be considered as independently created species, that
+the one in varying has assumed some of the characters of the other, so as
+to produce the intermediate form.  But the best evidence is afforded by
+parts or organs of an important and uniform nature occasionally varying so
+as to acquire, in some degree, the character of the same part or organ in
+an allied species.  I have collected a long list of such cases; but here,
+as before, I lie under a great disadvantage in not being able to give them. 
+I can only repeat that such cases certainly do occur, and seem to me very
+remarkable.
+</para>
+
+<para>
+I will, however, give one curious and complex case, not indeed as affecting
+any important character, but from occurring in several species of the same
+genus, partly under domestication and partly under nature.  It is a case
+apparently of reversion.  The ass not rarely has very distinct transverse
+bars on its legs, like those on the legs of a zebra:  it has been asserted
+that these are plainest in the foal, and from inquiries which I have made,
+I believe this to be true.  It has also been asserted that the stripe on
+each shoulder is sometimes double.  The shoulder stripe is certainly very
+variable in length and outline.  A white ass, but not an albino, has been
+described without either spinal or shoulder-stripe; and these stripes are
+sometimes very obscure, or actually quite lost, in dark-coloured asses. 
+The koulan of Pallas is said to have been seen with a double
+shoulder-stripe.  The hemionus has no shoulder-stripe; but traces of it, as
+stated by Mr. Blyth and others, occasionally appear:  and I have been
+informed by Colonel Poole that foals of this species are generally striped
+on the legs, and faintly on the shoulder.  The quagga, though so plainly
+barred like a zebra over the body, is without bars on the legs; but Dr.
+Gray has figured one specimen with very distinct zebra-like bars on the
+hocks.
+</para>
+
+<para>
+With respect to the horse, I have collected cases in England of the spinal
+stripe in horses of the most distinct breeds, and of all colours;
+transverse bars on the legs are not rare in duns, mouse-duns, and in one
+instance in a chestnut:  a faint shoulder-stripe may sometimes be seen in
+duns, and I have seen a trace in a bay horse.  My son made a careful
+examination and sketch for me of a dun Belgian cart-horse with a double
+stripe on each shoulder and with leg-stripes; and a man, whom I can
+implicitly trust, has examined for me a small dun Welch pony with three
+short parallel stripes on each shoulder.
+</para>
+
+<para>
+In the north-west part of India the Kattywar breed of horses is so
+generally striped, that, as I hear from Colonel Poole, who examined the
+breed for the Indian Government, a horse without stripes is not considered
+as purely-bred.  The spine is always striped; the legs are generally
+barred; and the shoulder-stripe, which is sometimes double and sometimes
+treble, is common; the side of the face, moreover, is sometimes striped. 
+The stripes are plainest in the foal; and sometimes quite disappear in old
+horses.  Colonel Poole has seen both gray and bay Kattywar horses striped
+when first foaled.  I have, also, reason to suspect, from information given
+me by Mr. W. W. Edwards, that with the English race-horse the spinal stripe
+is much commoner in the foal than in the full-grown animal.  Without here
+entering on further details, I may state that I have collected cases of leg
+and shoulder stripes in horses of very different breeds, in various
+countries from Britain to Eastern China; and from Norway in the north to
+the Malay Archipelago in the south.  In all parts of the world these
+stripes occur far oftenest in duns and mouse-duns; by the term dun a large
+range of colour is included, from one between brown and black to a close
+approach to cream-colour.
+</para>
+
+<para>
+I am aware that Colonel Hamilton Smith, who has written on this subject,
+believes that the several breeds of the horse have descended from several
+aboriginal species--one of which, the dun, was striped; and that the
+above-described appearances are all due to ancient crosses with the dun
+stock.  But I am not at all satisfied with this theory, and should be loth
+to apply it to breeds so distinct as the heavy Belgian cart-horse, Welch
+ponies, cobs, the lanky Kattywar race, &amp;c., inhabiting the most distant
+parts of the world.
+</para>
+
+<para>
+Now let us turn to the effects of crossing the several species of the
+horse-genus.  Rollin asserts, that the common mule from the ass and horse
+is particularly apt to have bars on its legs.  I once saw a mule with its
+legs so much striped that any one at first would have thought that it must
+have been the product of a zebra; and Mr. W. C. Martin, in his excellent
+treatise on the horse, has given a figure of a similar mule.  In four
+coloured drawings, which I have seen, of hybrids between the ass and zebra,
+the legs were much more plainly barred than the rest of the body; and in
+one of them there was a double shoulder-stripe.  In Lord Moreton's famous
+hybrid from a chestnut mare and male quagga, the hybrid, and even the pure
+offspring subsequently produced from the mare by a black Arabian sire, were
+much more plainly barred across the legs than is even the pure quagga. 
+Lastly, and this is another most remarkable case, a hybrid has been figured
+by Dr. Gray (and he informs me that he knows of a second case) from the ass
+and the hemionus; and this hybrid, though the ass seldom has stripes on its
+legs and the hemionus has none and has not even a shoulder-stripe,
+nevertheless had all four legs barred, and had three short
+shoulder-stripes, like those on the dun Welch pony, and even had some
+zebra-like stripes on the sides of its face.  With respect to this last
+fact, I was so convinced that not even a stripe of colour appears from what
+would commonly be called an accident, that I was led solely from the
+occurrence of the face-stripes on this hybrid from the ass and hemionus, to
+ask Colonel Poole whether such face-stripes ever occur in the eminently
+striped Kattywar breed of horses, and was, as we have seen, answered in the
+affirmative.
+</para>
+
+<para>
+What now are we to say to these several facts?  We see several very
+distinct species of the horse-genus becoming, by simple variation, striped
+on the legs like a zebra, or striped on the shoulders like an ass.  In the
+horse we see this tendency strong whenever a dun tint appears--a tint which
+approaches to that of the general colouring of the other species of the
+genus.  The appearance of the stripes is not accompanied by any change of
+form or by any other new character.  We see this tendency to become striped
+most strongly displayed in hybrids from between several of the most
+distinct species.  Now observe the case of the several breeds of pigeons: 
+they are descended from a pigeon (including two or three sub-species or
+geographical races) of a bluish colour, with certain bars and other marks;
+and when any breed assumes by simple variation a bluish tint, these bars
+and other marks invariably reappear; but without any other change of form
+or character.  When the oldest and truest breeds of various colours are
+crossed, we see a strong tendency for the blue tint and bars and marks to
+reappear in the mongrels.  I have stated that the most probable hypothesis
+to account for the reappearance of very ancient characters, is--that there
+is a tendency in the young of each successive generation to produce the
+long-lost character, and that this tendency, from unknown causes, sometimes
+prevails.  And we have just seen that in several species of the horse-genus
+the stripes are either plainer or appear more commonly in the young than in
+the old.  Call the breeds of pigeons, some of which have bred true for
+centuries, species; and how exactly parallel is the case with that of the
+species of the horse-genus!  For myself, I venture confidently to look back
+thousands on thousands of generations, and I see an animal striped like a
+zebra, but perhaps otherwise very differently constructed, the common
+parent of our domestic horse, whether or not it be descended from one or
+more wild stocks, of the ass, the hemionus, quagga, and zebra.
+</para>
+
+<para>
+He who believes that each equine species was independently created, will, I
+presume, assert that each species has been created with a tendency to vary,
+both under nature and under domestication, in this particular manner, so as
+often to become striped like other species of the genus; and that each has
+been created with a strong tendency, when crossed with species inhabiting
+distant quarters of the world, to produce hybrids resembling in their
+stripes, not their own parents, but other species of the genus.  To admit
+this view is, as it seems to me, to reject a real for an unreal, or at
+least for an unknown, cause.  It makes the works of God a mere mockery and
+deception; I would almost as soon believe with the old and ignorant
+cosmogonists, that fossil shells had never lived, but had been created in
+stone so as to mock the shells now living on the sea-shore.
+</para>
+
+<para>
+Summary. -- Our ignorance of the laws of variation is profound.  Not in one
+case out of a hundred can we pretend to assign any reason why this or that
+part differs, more or less, from the same part in the parents.  But
+whenever we have the means of instituting a comparison, the same laws
+appear to have acted in producing the lesser differences between varieties
+of the same species, and the greater differences between species of the
+same genus.  The external conditions of life, as climate and food, &amp;c.,
+seem to have induced some slight modifications.  Habit in producing
+constitutional differences, and use in strengthening, and disuse in
+weakening and diminishing organs, seem to have been more potent in their
+effects.  Homologous parts tend to vary in the same way, and homologous
+parts tend to cohere.  Modifications in hard parts and in external parts
+sometimes affect softer and internal parts.  When one part is largely
+developed, perhaps it tends to draw nourishment from the adjoining parts;
+and every part of the structure which can be saved without detriment to the
+individual, will be saved.  Changes of structure at an early age will
+generally affect parts subsequently developed; and there are very many
+other correlations of growth, the nature of which we are utterly unable to
+understand.  Multiple parts are variable in number and in structure,
+perhaps arising from such parts not having been closely specialised to any
+particular function, so that their modifications have not been closely
+checked by natural selection.  It is probably from this same cause that
+organic beings low in the scale of nature are more variable than those
+which have their whole organisation more specialised, and are higher in the
+scale.  Rudimentary organs, from being useless, will be disregarded by
+natural selection, and hence probably are variable.  Specific
+characters--that is, the characters which have come to differ since the
+several species of the same genus branched off from a common parent--are
+more variable than generic characters, or those which have long been
+inherited, and have not differed within this same period.  In these remarks
+we have referred to special parts or organs being still variable, because
+they have recently varied and thus come to differ; but we have also seen in
+the second Chapter that the same principle applies to the whole individual;
+for in a district where many species of any genus are found--that is, where
+there has been much former variation and differentiation, or where the
+manufactory of new specific forms has been actively at work--there, on an
+average, we now find most varieties or incipient species.  Secondary sexual
+characters are highly variable, and such characters differ much in the
+species of the same group.  Variability in the same parts of the
+organisation has generally been taken advantage of in giving secondary
+sexual differences to the sexes of the same species, and specific
+differences to the several species of the same genus.  Any part or organ
+developed to an extraordinary size or in an extraordinary manner, in
+comparison with the same part or organ in the allied species, must have
+gone through an extraordinary amount of modification since the genus arose;
+and thus we can understand why it should often still be variable in a much
+higher degree than other parts; for variation is a long-continued and slow
+process, and natural selection will in such cases not as yet have had time
+to overcome the tendency to further variability and to reversion to a less
+modified state.  But when a species with any extraordinarily-developed
+organ has become the parent of many modified descendants--which on my view
+must be a very slow process, requiring a long lapse of time--in this case,
+natural selection may readily have succeeded in giving a fixed character to
+the organ, in however extraordinary a manner it may be developed.  Species
+inheriting nearly the same constitution from a common parent and exposed to
+similar influences will naturally tend to present analogous variations, and
+these same species may occasionally revert to some of the characters of
+their ancient progenitors.  Although new and important modifications may
+not arise from reversion and analogous variation, such modifications will
+add to the beautiful and harmonious diversity of nature.
+</para>
+
+<para>
+Whatever the cause may be of each slight difference in the offspring from
+their parents--and a cause for each must exist--it is the steady
+accumulation, through natural selection, of such differences, when
+beneficial to the individual, that gives rise to all the more important
+modifications of structure, by which the innumerable beings on the face of
+this earth are enabled to struggle with each other, and the best adapted to
+survive.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter VI</chapnum>
+
+<title>Difficulties on Theory</title>
+
+<chapsummary>
+Difficulties on the theory of descent with modification -- Transitions --
+Absence or rarity of transitional varieties -- Transitions in habits of
+life -- Diversified habits in the same species -- Species with habits
+widely different from those of their allies -- Organs of extreme perfection
+-- Means of transition -- Cases of difficulty -- Natura non facit saltum --
+Organs of small importance -- Organs not in all cases absolutely perfect --
+The law of Unity of Type and of the Conditions of Existence embraced by the
+theory of Natural Selection.
+</chapsummary>
+</chapheader>
+
+<para>
+Long before having arrived at this part of my work, a crowd of difficulties
+will have occurred to the reader.  Some of them are so grave that to this
+day I can never reflect on them without being staggered; but, to the best
+of my judgment, the greater number are only apparent, and those that are
+real are not, I think, fatal to my theory.
+</para>
+
+<para>
+These difficulties and objections may be classed under the following
+heads:- Firstly, why, if species have descended from other species by
+insensibly fine gradations, do we not everywhere see innumerable
+transitional forms?  Why is not all nature in confusion instead of the
+species being, as we see them, well defined?
+</para>
+
+<para>
+Secondly, is it possible that an animal having, for instance, the structure
+and habits of a bat, could have been formed by the modification of some
+animal with wholly different habits?  Can we believe that natural selection
+could produce, on the one hand, organs of trifling importance, such as the
+tail of a giraffe, which serves as a fly-flapper, and, on the other hand,
+organs of such wonderful structure, as the eye, of which we hardly as yet
+fully understand the inimitable perfection?
+</para>
+
+<para>
+Thirdly, can instincts be acquired and modified through natural selection? 
+What shall we say to so marvellous an instinct as that which leads the bee
+to make cells, which have practically anticipated the discoveries of
+profound mathematicians?
+</para>
+
+<para>
+Fourthly, how can we account for species, when crossed, being sterile and
+producing sterile offspring, whereas, when varieties are crossed, their
+fertility is unimpaired?
+</para>
+
+<para>
+The two first heads shall be here discussed--Instinct and Hybridism in
+separate chapters.
+</para>
+
+<para>
+On the absence or rarity of transitional varieties. -- As natural selection
+acts solely by the preservation of profitable modifications, each new form
+will tend in a fully-stocked country to take the place of, and finally to
+exterminate, its own less improved parent or other less-favoured forms with
+which it comes into competition.  Thus extinction and natural selection
+will, as we have seen, go hand in hand.  Hence, if we look at each species
+as descended from some other unknown form, both the parent and all the
+transitional varieties will generally have been exterminated by the very
+process of formation and perfection of the new form.
+</para>
+
+<para>
+But, as by this theory innumerable transitional forms must have existed,
+why do we not find them embedded in countless numbers in the crust of the
+earth?  It will be much more convenient to discuss this question in the
+chapter on the Imperfection of the geological record; and I will here only
+state that I believe the answer mainly lies in the record being
+incomparably less perfect than is generally supposed; the imperfection of
+the record being chiefly due to organic beings not inhabiting profound
+depths of the sea, and to their remains being embedded and preserved to a
+future age only in masses of sediment sufficiently thick and extensive to
+withstand an enormous amount of future degradation; and such fossiliferous
+masses can be accumulated only where much sediment is deposited on the
+shallow bed of the sea, whilst it slowly subsides.  These contingencies
+will concur only rarely, and after enormously long intervals.  Whilst the
+bed of the sea is stationary or is rising, or when very little sediment is
+being deposited, there will be blanks in our geological history.  The crust
+of the earth is a vast museum; but the natural collections have been made
+only at intervals of time immensely remote.
+</para>
+
+<para>
+But it may be urged that when several closely-allied species inhabit the
+same territory we surely ought to find at the present time many
+transitional forms.  Let us take a simple case:  in travelling from north
+to south over a continent, we generally meet at successive intervals with
+closely allied or representative species, evidently filling nearly the same
+place in the natural economy of the land.  These representative species
+often meet and interlock; and as the one becomes rarer and rarer, the other
+becomes more and more frequent, till the one replaces the other.  But if we
+compare these species where they intermingle, they are generally as
+absolutely distinct from each other in every detail of structure as are
+specimens taken from the metropolis inhabited by each.  By my theory these
+allied species have descended from a common parent; and during the process
+of modification, each has become adapted to the conditions of life of its
+own region, and has supplanted and exterminated its original parent and all
+the transitional varieties between its past and present states.  Hence we
+ought not to expect at the present time to meet with numerous transitional
+varieties in each region, though they must have existed there, and may be
+embedded there in a fossil condition.  But in the intermediate region,
+having intermediate conditions of life, why do we not now find
+closely-linking intermediate varieties?  This difficulty for a long time
+quite confounded me.  But I think it can be in large part explained.
+</para>
+
+<para>
+In the first place we should be extremely cautious in inferring, because an
+area is now continuous, that it has been continuous during a long period. 
+Geology would lead us to believe that almost every continent has been
+broken up into islands even during the later tertiary periods; and in such
+islands distinct species might have been separately formed without the
+possibility of intermediate varieties existing in the intermediate zones. 
+By changes in the form of the land and of climate, marine areas now
+continuous must often have existed within recent times in a far less
+continuous and uniform condition than at present.  But I will pass over
+this way of escaping from the difficulty; for I believe that many perfectly
+defined species have been formed on strictly continuous areas; though I do
+not doubt that the formerly broken condition of areas now continuous has
+played an important part in the formation of new species, more especially
+with freely-crossing and wandering animals.
+</para>
+
+<para>
+In looking at species as they are now distributed over a wide area, we
+generally find them tolerably numerous over a large territory, then
+becoming somewhat abruptly rarer and rarer on the confines, and finally
+disappearing.  Hence the neutral territory between two representative
+species is generally narrow in comparison with the territory proper to
+each.  We see the same fact in ascending mountains, and sometimes it is
+quite remarkable how abruptly, as Alph. De Candolle has observed, a common
+alpine species disappears.  The same fact has been noticed by Forbes in
+sounding the depths of the sea with the dredge.  To those who look at
+climate and the physical conditions of life as the all-important elements
+of distribution, these facts ought to cause surprise, as climate and height
+or depth graduate away insensibly.  But when we bear in mind that almost
+every species, even in its metropolis, would increase immensely in numbers,
+were it not for other competing species; that nearly all either prey on or
+serve as prey for others; in short, that each organic being is either
+directly or indirectly related in the most important manner to other
+organic beings, we must see that the range of the inhabitants of any
+country by no means exclusively depends on insensibly changing physical
+conditions, but in large part on the presence of other species, on which it
+depends, or by which it is destroyed, or with which it comes into
+competition; and as these species are already defined objects (however they
+may have become so), not blending one into another by insensible
+gradations, the range of any one species, depending as it does on the range
+of others, will tend to be sharply defined.  Moreover, each species on the
+confines of its range, where it exists in lessened numbers, will, during
+fluctuations in the number of its enemies or of its prey, or in the
+seasons, be extremely liable to utter extermination; and thus its
+geographical range will come to be still more sharply defined.
+</para>
+
+<para>
+If I am right in believing that allied or representative species, when
+inhabiting a continuous area, are generally so distributed that each has a
+wide range, with a comparatively narrow neutral territory between them, in
+which they become rather suddenly rarer and rarer; then, as varieties do
+not essentially differ from species, the same rule will probably apply to
+both; and if we in imagination adapt a varying species to a very large
+area, we shall have to adapt two varieties to two large areas, and a third
+variety to a narrow intermediate zone.  The intermediate variety,
+consequently, will exist in lesser numbers from inhabiting a narrow and
+lesser area; and practically, as far as I can make out, this rule holds
+good with varieties in a state of nature.  I have met with striking
+instances of the rule in the case of varieties intermediate between
+well-marked varieties in the genus Balanus.  And it would appear from
+information given me by Mr. Watson, Dr. Asa Gray, and Mr. Wollaston, that
+generally when varieties intermediate between two other forms occur, they
+are much rarer numerically than the forms which they connect.  Now, if we
+may trust these facts and inferences, and therefore conclude that varieties
+linking two other varieties together have generally existed in lesser
+numbers than the forms which they connect, then, I think, we can understand
+why intermediate varieties should not endure for very long periods;--why as
+a general rule they should be exterminated and disappear, sooner than the
+forms which they originally linked together.
+</para>
+
+<para>
+For any form existing in lesser numbers would, as already remarked, run a
+greater chance of being exterminated than one existing in large numbers;
+and in this particular case the intermediate form would be eminently liable
+to the inroads of closely allied forms existing on both sides of it.  But a
+far more important consideration, as I believe, is that, during the process
+of further modification, by which two varieties are supposed on my theory
+to be converted and perfected into two distinct species, the two which
+exist in larger numbers from inhabiting larger areas, will have a great
+advantage over the intermediate variety, which exists in smaller numbers in
+a narrow and intermediate zone.  For forms existing in larger numbers will
+always have a better chance, within any given period, of presenting further
+favourable variations for natural selection to seize on, than will the
+rarer forms which exist in lesser numbers.  Hence, the more common forms,
+in the race for life, will tend to beat and supplant the less common forms,
+for these will be more slowly modified and improved.  It is the same
+principle which, as I believe, accounts for the common species in each
+country, as shown in the second chapter, presenting on an average a greater
+number of well-marked varieties than do the rarer species.  I may
+illustrate what I mean by supposing three varieties of sheep to be kept,
+one adapted to an extensive mountainous region; a second to a comparatively
+narrow, hilly tract; and a third to wide plains at the base; and that the
+inhabitants are all trying with equal steadiness and skill to improve their
+stocks by selection; the chances in this case will be strongly in favour of
+the great holders on the mountains or on the plains improving their breeds
+more quickly than the small holders on the intermediate narrow, hilly
+tract; and consequently the improved mountain or plain breed will soon take
+the place of the less improved hill breed; and thus the two breeds, which
+originally existed in greater numbers, will come into close contact with
+each other, without the interposition of the supplanted, intermediate
+hill-variety.
+</para>
+
+<para>
+To sum up, I believe that species come to be tolerably well-defined
+objects, and do not at any one period present an inextricable chaos of
+varying and intermediate links:  firstly, because new varieties are very
+slowly formed, for variation is a very slow process, and natural selection
+can do nothing until favourable variations chance to occur, and until a
+place in the natural polity of the country can be better filled by some
+modification of some one or more of its inhabitants.  And such new places
+will depend on slow changes of climate, or on the occasional immigration of
+new inhabitants, and, probably, in a still more important degree, on some
+of the old inhabitants becoming slowly modified, with the new forms thus
+produced and the old ones acting and reacting on each other.  So that, in
+any one region and at any one time, we ought only to see a few species
+presenting slight modifications of structure in some degree permanent; and
+this assuredly we do see.
+</para>
+
+<para>
+Secondly, areas now continuous must often have existed within the recent
+period in isolated portions, in which many forms, more especially amongst
+the classes which unite for each birth and wander much, may have separately
+been rendered sufficiently distinct to rank as representative species.  In
+this case, intermediate varieties between the several representative
+species and their common parent, must formerly have existed in each broken
+portion of the land, but these links will have been supplanted and
+exterminated during the process of natural selection, so that they will no
+longer exist in a living state.
+</para>
+
+<para>
+Thirdly, when two or more varieties have been formed in different portions
+of a strictly continuous area, intermediate varieties will, it is probable,
+at first have been formed in the intermediate zones, but they will
+generally have had a short duration.  For these intermediate varieties
+will, from reasons already assigned (namely from what we know of the actual
+distribution of closely allied or representative species, and likewise of
+acknowledged varieties), exist in the intermediate zones in lesser numbers
+than the varieties which they tend to connect.  From this cause alone the
+intermediate varieties will be liable to accidental extermination; and
+during the process of further modification through natural selection, they
+will almost certainly be beaten and supplanted by the forms which they
+connect; for these from existing in greater numbers will, in the aggregate,
+present more variation, and thus be further improved through natural
+selection and gain further advantages.
+</para>
+
+<para>
+Lastly, looking not to any one time, but to all time, if my theory be true,
+numberless intermediate varieties, linking most closely all the species of
+the same group together, must assuredly have existed; but the very process
+of natural selection constantly tends, as has been so often remarked, to
+exterminate the parent forms and the intermediate links.  Consequently
+evidence of their former existence could be found only amongst fossil
+remains, which are preserved, as we shall in a future chapter attempt to
+show, in an extremely imperfect and intermittent record.
+</para>
+
+<para>
+On the origin and transitions of organic beings with peculiar habits and
+structure. -- It has been asked by the opponents of such views as I hold,
+how, for instance, a land carnivorous animal could have been converted into
+one with aquatic habits; for how could the animal in its transitional state
+have subsisted?  It would be easy to show that within the same group
+carnivorous animals exist having every intermediate grade between truly
+aquatic and strictly terrestrial habits; and as each exists by a struggle
+for life, it is clear that each is well adapted in its habits to its place
+in nature.  Look at the Mustela vison of North America, which has webbed
+feet and which resembles an otter in its fur, short legs, and form of tail;
+during summer this animal dives for and preys on fish, but during the long
+winter it leaves the frozen waters, and preys like other polecats on mice
+and land animals.  If a different case had been taken, and it had been
+asked how an insectivorous quadruped could possibly have been converted
+into a flying bat, the question would have been far more difficult, and I
+could have given no answer.  Yet I think such difficulties have very little
+weight.
+</para>
+
+<para>
+Here, as on other occasions, I lie under a heavy disadvantage, for out of
+the many striking cases which I have collected, I can give only one or two
+instances of transitional habits and structures in closely allied species
+of the same genus; and of diversified habits, either constant or
+occasional, in the same species.  And it seems to me that nothing less than
+a long list of such cases is sufficient to lessen the difficulty in any
+particular case like that of the bat.
+</para>
+
+<para>
+Look at the family of squirrels; here we have the finest gradation from
+animals with their tails only slightly flattened, and from others, as Sir
+J. Richardson has remarked, with the posterior part of their bodies rather
+wide and with the skin on their flanks rather full, to the so-called flying
+squirrels; and flying squirrels have their limbs and even the base of the
+tail united by a broad expanse of skin, which serves as a parachute and
+allows them to glide through the air to an astonishing distance from tree
+to tree.  We cannot doubt that each structure is of use to each kind of
+squirrel in its own country, by enabling it to escape birds or beasts of
+prey, or to collect food more quickly, or, as there is reason to believe,
+by lessening the danger from occasional falls.  But it does not follow from
+this fact that the structure of each squirrel is the best that it is
+possible to conceive under all natural conditions.  Let the climate and
+vegetation change, let other competing rodents or new beasts of prey
+immigrate, or old ones become modified, and all analogy would lead us to
+believe that some at least of the squirrels would decrease in numbers or
+become exterminated, unless they also became modified and improved in
+structure in a corresponding manner.  Therefore, I can see no difficulty,
+more especially under changing conditions of life, in the continued
+preservation of individuals with fuller and fuller flank-membranes, each
+modification being useful, each being propagated, until by the accumulated
+effects of this process of natural selection, a perfect so-called flying
+squirrel was produced.
+</para>
+
+<para>
+Now look at the Galeopithecus or flying lemur, which formerly was falsely
+ranked amongst bats.  It has an extremely wide flank-membrane, stretching
+from the corners of the jaw to the tail, and including the limbs and the
+elongated fingers:  the flank membrane is, also, furnished with an extensor
+muscle.  Although no graduated links of structure, fitted for gliding
+through the air, now connect the Galeopithecus with the other Lemuridae,
+yet I can see no difficulty in supposing that such links formerly existed,
+and that each had been formed by the same steps as in the case of the less
+perfectly gliding squirrels; and that each grade of structure had been
+useful to its possessor.  Nor can I see any insuperable difficulty in
+further believing it possible that the membrane-connected fingers and
+fore-arm of the Galeopithecus might be greatly lengthened by natural
+selection; and this, as far as the organs of flight are concerned, would
+convert it into a bat.  In bats which have the wing-membrane extended from
+the top of the shoulder to the tail, including the hind-legs, we perhaps
+see traces of an apparatus originally constructed for gliding through the
+air rather than for flight.
+</para>
+
+<para>
+If about a dozen genera of birds had become extinct or were unknown, who
+would have ventured to have surmised that birds might have existed which
+used their wings solely as flappers, like the logger-headed duck
+(Micropterus of Eyton); as fins in the water and front legs on the land,
+like the penguin; as sails, like the ostrich; and functionally for no
+purpose, like the Apteryx.  Yet the structure of each of these birds is
+good for it, under the conditions of life to which it is exposed, for each
+has to live by a struggle; but it is not necessarily the best possible
+under all possible conditions.  It must not be inferred from these remarks
+that any of the grades of wing-structure here alluded to, which perhaps may
+all have resulted from disuse, indicate the natural steps by which birds
+have acquired their perfect power of flight; but they serve, at least, to
+show what diversified means of transition are possible.
+</para>
+
+<para>
+Seeing that a few members of such water-breathing classes as the Crustacea
+and Mollusca are adapted to live on the land, and seeing that we have
+flying birds and mammals, flying insects of the most diversified types, and
+formerly had flying reptiles, it is conceivable that flying-fish, which now
+glide far through the air, slightly rising and turning by the aid of their
+fluttering fins, might have been modified into perfectly winged animals. 
+If this had been effected, who would have ever imagined that in an early
+transitional state they had been inhabitants of the open ocean, and had
+used their incipient organs of flight exclusively, as far as we know, to
+escape being devoured by other fish?
+</para>
+
+<para>
+When we see any structure highly perfected for any particular habit, as the
+wings of a bird for flight, we should bear in mind that animals displaying
+early transitional grades of the structure will seldom continue to exist to
+the present day, for they will have been supplanted by the very process of
+perfection through natural selection.  Furthermore, we may conclude that
+transitional grades between structures fitted for very different habits of
+life will rarely have been developed at an early period in great numbers
+and under many subordinate forms.  Thus, to return to our imaginary
+illustration of the flying-fish, it does not seem probable that fishes
+capable of true flight would have been developed under many subordinate
+forms, for taking prey of many kinds in many ways, on the land and in the
+water, until their organs of flight had come to a high stage of perfection,
+so as to have given them a decided advantage over other animals in the
+battle for life.  Hence the chance of discovering species with transitional
+grades of structure in a fossil condition will always be less, from their
+having existed in lesser numbers, than in the case of species with fully
+developed structures.
+</para>
+
+<para>
+I will now give two or three instances of diversified and of changed habits
+in the individuals of the same species.  When either case occurs, it would
+be easy for natural selection to fit the animal, by some modification of
+its structure, for its changed habits, or exclusively for one of its
+several different habits.  But it is difficult to tell, and immaterial for
+us, whether habits generally change first and structure afterwards; or
+whether slight modifications of structure lead to changed habits; both
+probably often change almost simultaneously.  Of cases of changed habits it
+will suffice merely to allude to that of the many British insects which now
+feed on exotic plants, or exclusively on artificial substances.  Of
+diversified habits innumerable instances could be given:  I have often
+watched a tyrant flycatcher (Saurophagus sulphuratus) in South America,
+hovering over one spot and then proceeding to another, like a kestrel, and
+at other times standing stationary on the margin of water, and then dashing
+like a kingfisher at a fish.  In our own country the larger titmouse (Parus
+major) may be seen climbing branches, almost like a creeper; it often, like
+a shrike, kills small birds by blows on the head; and I have many times
+seen and heard it hammering the seeds of the yew on a branch, and thus
+breaking them like a nuthatch.  In North America the black bear was seen by
+Hearne swimming for hours with widely open mouth, thus catching, like a
+whale, insects in the water.  Even in so extreme a case as this, if the
+supply of insects were constant, and if better adapted competitors did not
+already exist in the country, I can see no difficulty in a race of bears
+being rendered, by natural selection, more and more aquatic in their
+structure and habits, with larger and larger mouths, till a creature was
+produced as monstrous as a whale.
+</para>
+
+<para>
+As we sometimes see individuals of a species following habits widely
+different from those both of their own species and of the other species of
+the same genus, we might expect, on my theory, that such individuals would
+occasionally have given rise to new species, having anomalous habits, and
+with their structure either slightly or considerably modified from that of
+their proper type.  And such instances do occur in nature.  Can a more
+striking instance of adaptation be given than that of a woodpecker for
+climbing trees and for seizing insects in the chinks of the bark?  Yet in
+North America there are woodpeckers which feed largely on fruit, and others
+with elongated wings which chase insects on the wing; and on the plains of
+La Plata, where not a tree grows, there is a woodpecker, which in every
+essential part of its organisation, even in its colouring, in the harsh
+tone of its voice, and undulatory flight, told me plainly of its close
+blood-relationship to our common species; yet it is a woodpecker which
+never climbs a tree!
+</para>
+
+<para>
+Petrels are the most aerial and oceanic of birds, yet in the quiet Sounds
+of Tierra del Fuego, the Puffinuria berardi, in its general habits, in its
+astonishing power of diving, its manner of swimming, and of flying when
+unwillingly it takes flight, would be mistaken by any one for an auk or
+grebe; nevertheless, it is essentially a petrel, but with many parts of its
+organisation profoundly modified.  On the other hand, the acutest observer
+by examining the dead body of the water-ouzel would never have suspected
+its sub-aquatic habits; yet this anomalous member of the strictly
+terrestrial thrush family wholly subsists by diving,--grasping the stones
+with its feet and using its wings under water.
+</para>
+
+<para>
+He who believes that each being has been created as we now see it, must
+occasionally have felt surprise when he has met with an animal having
+habits and structure not at all in agreement.  What can be plainer than
+that the webbed feet of ducks and geese are formed for swimming? yet there
+are upland geese with webbed feet which rarely or never go near the water;
+and no one except Audubon has seen the frigate-bird, which has all its four
+toes webbed, alight on the surface of the sea.  On the other hand, grebes
+and coots are eminently aquatic, although their toes are only bordered by
+membrane.  What seems plainer than that the long toes of grallatores are
+formed for walking over swamps and floating plants, yet the water-hen is
+nearly as aquatic as the coot; and the landrail nearly as terrestrial as
+the quail or partridge.  In such cases, and many others could be given,
+habits have changed without a corresponding change of structure.  The
+webbed feet of the upland goose may be said to have become rudimentary in
+function, though not in structure.  In the frigate-bird, the deeply-scooped
+membrane between the toes shows that structure has begun to change.
+</para>
+
+<para>
+He who believes in separate and innumerable acts of creation will say, that
+in these cases it has pleased the Creator to cause a being of one type to
+take the place of one of another type; but this seems to me only restating
+the fact in dignified language.  He who believes in the struggle for
+existence and in the principle of natural selection, will acknowledge that
+every organic being is constantly endeavouring to increase in numbers; and
+that if any one being vary ever so little, either in habits or structure,
+and thus gain an advantage over some other inhabitant of the country, it
+will seize on the place of that inhabitant, however different it may be
+from its own place.  Hence it will cause him no surprise that there should
+be geese and frigate-birds with webbed feet, either living on the dry land
+or most rarely alighting on the water; that there should be long-toed
+corncrakes living in meadows instead of in swamps; that there should be
+woodpeckers where not a tree grows; that there should be diving thrushes,
+and petrels with the habits of auks.
+</para>
+
+<para>
+Organs of extreme perfection and complication. -- To suppose that the eye,
+with all its inimitable contrivances for adjusting the focus to different
+distances, for admitting different amounts of light, and for the correction
+of spherical and chromatic aberration, could have been formed by natural
+selection, seems, I freely confess, absurd in the highest possible degree. 
+Yet reason tells me, that if numerous gradations from a perfect and complex
+eye to one very imperfect and simple, each grade being useful to its
+possessor, can be shown to exist; if further, the eye does vary ever so
+slightly, and the variations be inherited, which is certainly the case; and
+if any variation or modification in the organ be ever useful to an animal
+under changing conditions of life, then the difficulty of believing that a
+perfect and complex eye could be formed by natural selection, though
+insuperable by our imagination, can hardly be considered real.  How a nerve
+comes to be sensitive to light, hardly concerns us more than how life
+itself first originated; but I may remark that several facts make me
+suspect that any sensitive nerve may be rendered sensitive to light, and
+likewise to those coarser vibrations of the air which produce sound.
+</para>
+
+<para>
+In looking for the gradations by which an organ in any species has been
+perfected, we ought to look exclusively to its lineal ancestors; but this
+is scarcely ever possible, and we are forced in each case to look to
+species of the same group, that is to the collateral descendants from the
+same original parent-form, in order to see what gradations are possible,
+and for the chance of some gradations having been transmitted from the
+earlier stages of descent, in an unaltered or little altered condition. 
+Amongst existing Vertebrata, we find but a small amount of gradation in the
+structure of the eye, and from fossil species we can learn nothing on this
+head.  In this great class we should probably have to descend far beneath
+the lowest known fossiliferous stratum to discover the earlier stages, by
+which the eye has been perfected.
+</para>
+
+<para>
+In the Articulata we can commence a series with an optic nerve merely
+coated with pigment, and without any other mechanism; and from this low
+stage, numerous gradations of structure, branching off in two fundamentally
+different lines, can be shown to exist, until we reach a moderately high
+stage of perfection.  In certain crustaceans, for instance, there is a
+double cornea, the inner one divided into facets, within each of which
+there is a lens-shaped swelling.  In other crustaceans the transparent
+cones which are coated by pigment, and which properly act only by excluding
+lateral pencils of light, are convex at their upper ends and must act by
+convergence; and at their lower ends there seems to be an imperfect
+vitreous substance.  With these facts, here far too briefly and imperfectly
+given, which show that there is much graduated diversity in the eyes of
+living crustaceans, and bearing in mind how small the number of living
+animals is in proportion to those which have become extinct, I can see no
+very great difficulty (not more than in the case of many other structures)
+in believing that natural selection has converted the simple apparatus of
+an optic nerve merely coated with pigment and invested by transparent
+membrane, into an optical instrument as perfect as is possessed by any
+member of the great Articulate class.
+</para>
+
+<para>
+He who will go thus far, if he find on finishing this treatise that large
+bodies of facts, otherwise inexplicable, can be explained by the theory of
+descent, ought not to hesitate to go further, and to admit that a structure
+even as perfect as the eye of an eagle might be formed by natural
+selection, although in this case he does not know any of the transitional
+grades.  His reason ought to conquer his imagination; though I have felt
+the difficulty far too keenly to be surprised at any degree of hesitation
+in extending the principle of natural selection to such startling lengths.
+</para>
+
+<para>
+It is scarcely possible to avoid comparing the eye to a telescope.  We know
+that this instrument has been perfected by the long-continued efforts of
+the highest human intellects; and we naturally infer that the eye has been
+formed by a somewhat analogous process.  But may not this inference be
+presumptuous?  Have we any right to assume that the Creator works by
+intellectual powers like those of man?  If we must compare the eye to an
+optical instrument, we ought in imagination to take a thick layer of
+transparent tissue, with a nerve sensitive to light beneath, and then
+suppose every part of this layer to be continually changing slowly in
+density, so as to separate into layers of different densities and
+thicknesses, placed at different distances from each other, and with the
+surfaces of each layer slowly changing in form.  Further we must suppose
+that there is a power always intently watching each slight accidental
+alteration in the transparent layers; and carefully selecting each
+alteration which, under varied circumstances, may in any way, or in any
+degree, tend to produce a distincter image.  We must suppose each new state
+of the instrument to be multiplied by the million; and each to be preserved
+till a better be produced, and then the old ones to be destroyed.  In
+living bodies, variation will cause the slight alterations, generation will
+multiply them almost infinitely, and natural selection will pick out with
+unerring skill each improvement.  Let this process go on for millions on
+millions of years; and during each year on millions of individuals of many
+kinds; and may we not believe that a living optical instrument might thus
+be formed as superior to one of glass, as the works of the Creator are to
+those of man?
+</para>
+
+<para>
+If it could be demonstrated that any complex organ existed, which could not
+possibly have been formed by numerous, successive, slight modifications, my
+theory would absolutely break down.  But I can find out no such case.  No
+doubt many organs exist of which we do not know the transitional grades,
+more especially if we look to much-isolated species, round which, according
+to my theory, there has been much extinction.  Or again, if we look to an
+organ common to all the members of a large class, for in this latter case
+the organ must have been first formed at an extremely remote period, since
+which all the many members of the class have been developed; and in order
+to discover the early transitional grades through which the organ has
+passed, we should have to look to very ancient ancestral forms, long since
+become extinct.
+</para>
+
+<para>
+We should be extremely cautious in concluding that an organ could not have
+been formed by transitional gradations of some kind.  Numerous cases could
+be given amongst the lower animals of the same organ performing at the same
+time wholly distinct functions; thus the alimentary canal respires,
+digests, and excretes in the larva of the dragon-fly and in the fish
+Cobites.  In the Hydra, the animal may be turned inside out, and the
+exterior surface will then digest and the stomach respire.  In such cases
+natural selection might easily specialise, if any advantage were thus
+gained, a part or organ, which had performed two functions, for one
+function alone, and thus wholly change its nature by insensible steps.  Two
+distinct organs sometimes perform simultaneously the same function in the
+same individual; to give one instance, there are fish with gills or
+branchiae that breathe the air dissolved in the water, at the same time
+that they breathe free air in their swimbladders, this latter organ having
+a ductus pneumaticus for its supply, and being divided by highly vascular
+partitions.  In these cases, one of the two organs might with ease be
+modified and perfected so as to perform all the work by itself, being aided
+during the process of modification by the other organ; and then this other
+organ might be modified for some other and quite distinct purpose, or be
+quite obliterated.
+</para>
+
+<para>
+The illustration of the swimbladder in fishes is a good one, because it
+shows us clearly the highly important fact that an organ originally
+constructed for one purpose, namely flotation, may be converted into one
+for a wholly different purpose, namely respiration.  The swimbladder has,
+also, been worked in as an accessory to the auditory organs of certain
+fish, or, for I do not know which view is now generally held, a part of the
+auditory apparatus has been worked in as a complement to the swimbladder. 
+All physiologists admit that the swimbladder is homologous, or 'ideally
+similar,' in position and structure with the lungs of the higher vertebrate
+animals:  hence there seems to me to be no great difficulty in believing
+that natural selection has actually converted a swimbladder into a lung, or
+organ used exclusively for respiration.
+</para>
+
+<para>
+I can, indeed, hardly doubt that all vertebrate animals having true lungs
+have descended by ordinary generation from an ancient prototype, of which
+we know nothing, furnished with a floating apparatus or swimbladder.  We
+can thus, as I infer from Professor Owen's interesting description of these
+parts, understand the strange fact that every particle of food and drink
+which we swallow has to pass over the orifice of the trachea, with some
+risk of falling into the lungs, notwithstanding the beautiful contrivance
+by which the glottis is closed.  In the higher Vertebrata the branchiae
+have wholly disappeared--the slits on the sides of the neck and the
+loop-like course of the arteries still marking in the embryo their former
+position.  But it is conceivable that the now utterly lost branchiae might
+have been gradually worked in by natural selection for some quite distinct
+purpose:  in the same manner as, on the view entertained by some
+naturalists that the branchiae and dorsal scales of Annelids are homologous
+with the wings and wing-covers of insects, it is probable that organs which
+at a very ancient period served for respiration have been actually
+converted into organs of flight.
+</para>
+
+<para>
+In considering transitions of organs, it is so important to bear in mind
+the probability of conversion from one function to another, that I will
+give one more instance.  Pedunculated cirripedes have two minute folds of
+skin, called by me the ovigerous frena, which serve, through the means of a
+sticky secretion, to retain the eggs until they are hatched within the
+sack.  These cirripedes have no branchiae, the whole surface of the body
+and sack, including the small frena, serving for respiration.  The
+Balanidae or sessile cirripedes, on the other hand, have no ovigerous
+frena, the eggs lying loose at the bottom of the sack, in the well-enclosed
+shell; but they have large folded branchiae.  Now I think no one will
+dispute that the ovigerous frena in the one family are strictly homologous
+with the branchiae of the other family; indeed, they graduate into each
+other.  Therefore I do not doubt that little folds of skin, which
+originally served as ovigerous frena, but which, likewise, very slightly
+aided the act of respiration, have been gradually converted by natural
+selection into branchiae, simply through an increase in their size and the
+obliteration of their adhesive glands.  If all pedunculated cirripedes had
+become extinct, and they have already suffered far more extinction than
+have sessile cirripedes, who would ever have imagined that the branchiae in
+this latter family had originally existed as organs for preventing the ova
+from being washed out of the sack?
+</para>
+
+<para>
+Although we must be extremely cautious in concluding that any organ could
+not possibly have been produced by successive transitional gradations, yet,
+undoubtedly, grave cases of difficulty occur, some of which will be
+discussed in my future work.
+</para>
+
+<para>
+One of the gravest is that of neuter insects, which are often very
+differently constructed from either the males or fertile females; but this
+case will be treated of in the next chapter.  The electric organs of fishes
+offer another case of special difficulty; it is impossible to conceive by
+what steps these wondrous organs have been produced; but, as Owen and
+others have remarked, their intimate structure closely resembles that of
+common muscle; and as it has lately been shown that Rays have an organ
+closely analogous to the electric apparatus, and yet do not, as Matteuchi
+asserts, discharge any electricity, we must own that we are far too
+ignorant to argue that no transition of any kind is possible.
+</para>
+
+<para>
+The electric organs offer another and even more serious difficulty; for
+they occur in only about a dozen fishes, of which several are widely remote
+in their affinities.  Generally when the same organ appears in several
+members of the same class, especially if in members having very different
+habits of life, we may attribute its presence to inheritance from a common
+ancestor; and its absence in some of the members to its loss through disuse
+or natural selection.  But if the electric organs had been inherited from
+one ancient progenitor thus provided, we might have expected that all
+electric fishes would have been specially related to each other.  Nor does
+geology at all lead to the belief that formerly most fishes had electric
+organs, which most of their modified descendants have lost.  The presence
+of luminous organs in a few insects, belonging to different families and
+orders, offers a parallel case of difficulty.  Other cases could be given;
+for instance in plants, the very curious contrivance of a mass of
+pollen-grains, borne on a foot-stalk with a sticky gland at the end, is the
+same in Orchis and Asclepias,--genera almost as remote as possible amongst
+flowering plants.  In all these cases of two very distinct species
+furnished with apparently the same anomalous organ, it should be observed
+that, although the general appearance and function of the organ may be the
+same, yet some fundamental difference can generally be detected.  I am
+inclined to believe that in nearly the same way as two men have sometimes
+independently hit on the very same invention, so natural selection, working
+for the good of each being and taking advantage of analogous variations,
+has sometimes modified in very nearly the same manner two parts in two
+organic beings, which owe but little of their structure in common to
+inheritance from the same ancestor.
+</para>
+
+<para>
+Although in many cases it is most difficult to conjecture by what
+transitions an organ could have arrived at its present state; yet,
+considering that the proportion of living and known forms to the extinct
+and unknown is very small, I have been astonished how rarely an organ can
+be named, towards which no transitional grade is known to lead.  The truth
+of this remark is indeed shown by that old canon in natural history of
+'Natura non facit saltum.'  We meet with this admission in the writings of
+almost every experienced naturalist; or, as Milne Edwards has well
+expressed it, nature is prodigal in variety, but niggard in innovation. 
+Why, on the theory of Creation, should this be so?  Why should all the
+parts and organs of many independent beings, each supposed to have been
+separately created for its proper place in nature, be so invariably linked
+together by graduated steps?  Why should not Nature have taken a leap from
+structure to structure?  On the theory of natural selection, we can clearly
+understand why she should not; for natural selection can act only by taking
+advantage of slight successive variations; she can never take a leap, but
+must advance by the shortest and slowest steps.
+</para>
+
+<para>
+Organs of little apparent importance. -- As natural selection acts by life
+and death,--by the preservation of individuals with any favourable
+variation, and by the destruction of those with any unfavourable deviation
+of structure,--I have sometimes felt much difficulty in understanding the
+origin of simple parts, of which the importance does not seem sufficient to
+cause the preservation of successively varying individuals.  I have
+sometimes felt as much difficulty, though of a very different kind, on this
+head, as in the case of an organ as perfect and complex as the eye.
+</para>
+
+<para>
+In the first place, we are much too ignorant in regard to the whole economy
+of any one organic being, to say what slight modifications would be of
+importance or not.  In a former chapter I have given instances of most
+trifling characters, such as the down on fruit and the colour of the flesh,
+which, from determining the attacks of insects or from being correlated
+with constitutional differences, might assuredly be acted on by natural
+selection.  The tail of the giraffe looks like an artificially constructed
+fly-flapper; and it seems at first incredible that this could have been
+adapted for its present purpose by successive slight modifications, each
+better and better, for so trifling an object as driving away flies; yet we
+should pause before being too positive even in this case, for we know that
+the distribution and existence of cattle and other animals in South America
+absolutely depends on their power of resisting the attacks of insects:  so
+that individuals which could by any means defend themselves from these
+small enemies, would be able to range into new pastures and thus gain a
+great advantage.  It is not that the larger quadrupeds are actually
+destroyed (except in some rare cases) by the flies, but they are
+incessantly harassed and their strength reduced, so that they are more
+subject to disease, or not so well enabled in a coming dearth to search for
+food, or to escape from beasts of prey.
+</para>
+
+<para>
+Organs now of trifling importance have probably in some cases been of high
+importance to an early progenitor, and, after having been slowly perfected
+at a former period, have been transmitted in nearly the same state,
+although now become of very slight use; and any actually injurious
+deviations in their structure will always have been checked by natural
+selection.  Seeing how important an organ of locomotion the tail is in most
+aquatic animals, its general presence and use for many purposes in so many
+land animals, which in their lungs or modified swim-bladders betray their
+aquatic origin, may perhaps be thus accounted for.  A well-developed tail
+having been formed in an aquatic animal, it might subsequently come to be
+worked in for all sorts of purposes, as a fly-flapper, an organ of
+prehension, or as an aid in turning, as with the dog, though the aid must
+be slight, for the hare, with hardly any tail, can double quickly enough.
+</para>
+
+<para>
+In the second place, we may sometimes attribute importance to characters
+which are really of very little importance, and which have originated from
+quite secondary causes, independently of natural selection.  We should
+remember that climate, food, &amp;c., probably have some little direct
+influence on the organisation; that characters reappear from the law of
+reversion; that correlation of growth will have had a most important
+influence in modifying various structures; and finally, that sexual
+selection will often have largely modified the external characters of
+animals having a will, to give one male an advantage in fighting with
+another or in charming the females.  Moreover when a modification of
+structure has primarily arisen from the above or other unknown causes, it
+may at first have been of no advantage to the species, but may subsequently
+have been taken advantage of by the descendants of the species under new
+conditions of life and with newly acquired habits.
+</para>
+
+<para>
+To give a few instances to illustrate these latter remarks.  If green
+woodpeckers alone had existed, and we did not know that there were many
+black and pied kinds, I dare say that we should have thought that the green
+colour was a beautiful adaptation to hide this tree-frequenting bird from
+its enemies; and consequently that it was a character of importance and
+might have been acquired through natural selection; as it is, I have no
+doubt that the colour is due to some quite distinct cause, probably to
+sexual selection.  A trailing bamboo in the Malay Archipelago climbs the
+loftiest trees by the aid of exquisitely constructed hooks clustered around
+the ends of the branches, and this contrivance, no doubt, is of the highest
+service to the plant; but as we see nearly similar hooks on many trees
+which are not climbers, the hooks on the bamboo may have arisen from
+unknown laws of growth, and have been subsequently taken advantage of by
+the plant undergoing further modification and becoming a climber.  The
+naked skin on the head of a vulture is generally looked at as a direct
+adaptation for wallowing in putridity; and so it may be, or it may possibly
+be due to the direct action of putrid matter; but we should be very
+cautious in drawing any such inference, when we see that the skin on the
+head of the clean-feeding male turkey is likewise naked.  The sutures in
+the skulls of young mammals have been advanced as a beautiful adaptation
+for aiding parturition, and no doubt they facilitate, or may be
+indispensable for this act; but as sutures occur in the skulls of young
+birds and reptiles, which have only to escape from a broken egg, we may
+infer that this structure has arisen from the laws of growth, and has been
+taken advantage of in the parturition of the higher animals.
+</para>
+
+<para>
+We are profoundly ignorant of the causes producing slight and unimportant
+variations; and we are immediately made conscious of this by reflecting on
+the differences in the breeds of our domesticated animals in different
+countries,--more especially in the less civilized countries where there has
+been but little artificial selection.  Careful observers are convinced that
+a damp climate affects the growth of the hair, and that with the hair the
+horns are correlated.  Mountain breeds always differ from lowland breeds;
+and a mountainous country would probably affect the hind limbs from
+exercising them more, and possibly even the form of the pelvis; and then by
+the law of homologous variation, the front limbs and even the head would
+probably be affected.  The shape, also, of the pelvis might affect by
+pressure the shape of the head of the young in the womb.  The laborious
+breathing necessary in high regions would, we have some reason to believe,
+increase the size of the chest; and again correlation would come into play. 
+Animals kept by savages in different countries often have to struggle for
+their own subsistence, and would be exposed to a certain extent to natural
+selection, and individuals with slightly different constitutions would
+succeed best under different climates; and there is reason to believe that
+constitution and colour are correlated.  A good observer, also, states that
+in cattle susceptibility to the attacks of flies is correlated with colour,
+as is the liability to be poisoned by certain plants; so that colour would
+be thus subjected to the action of natural selection.  But we are far too
+ignorant to speculate on the relative importance of the several known and
+unknown laws of variation; and I have here alluded to them only to show
+that, if we are unable to account for the characteristic differences of our
+domestic breeds, which nevertheless we generally admit to have arisen
+through ordinary generation, we ought not to lay too much stress on our
+ignorance of the precise cause of the slight analogous differences between
+species.  I might have adduced for this same purpose the differences
+between the races of man, which are so strongly marked; I may add that some
+little light can apparently be thrown on the origin of these differences,
+chiefly through sexual selection of a particular kind, but without here
+entering on copious details my reasoning would appear frivolous.
+</para>
+
+<para>
+The foregoing remarks lead me to say a few words on the protest lately made
+by some naturalists, against the utilitarian doctrine that every detail of
+structure has been produced for the good of its possessor.  They believe
+that very many structures have been created for beauty in the eyes of man,
+or for mere variety.  This doctrine, if true, would be absolutely fatal to
+my theory.  Yet I fully admit that many structures are of no direct use to
+their possessors.  Physical conditions probably have had some little effect
+on structure, quite independently of any good thus gained.  Correlation of
+growth has no doubt played a most important part, and a useful modification
+of one part will often have entailed on other parts diversified changes of
+no direct use.  So again characters which formerly were useful, or which
+formerly had arisen from correlation of growth, or from other unknown
+cause, may reappear from the law of reversion, though now of no direct use. 
+The effects of sexual selection, when displayed in beauty to charm the
+females, can be called useful only in rather a forced sense.  But by far
+the most important consideration is that the chief part of the organisation
+of every being is simply due to inheritance; and consequently, though each
+being assuredly is well fitted for its place in nature, many structures now
+have no direct relation to the habits of life of each species.  Thus, we
+can hardly believe that the webbed feet of the upland goose or of the
+frigate-bird are of special use to these birds; we cannot believe that the
+same bones in the arm of the monkey, in the fore leg of the horse, in the
+wing of the bat, and in the flipper of the seal, are of special use to
+these animals.  We may safely attribute these structures to inheritance. 
+But to the progenitor of the upland goose and of the frigate-bird, webbed
+feet no doubt were as useful as they now are to the most aquatic of
+existing birds.  So we may believe that the progenitor of the seal had not
+a flipper, but a foot with five toes fitted for walking or grasping; and we
+may further venture to believe that the several bones in the limbs of the
+monkey, horse, and bat, which have been inherited from a common progenitor,
+were formerly of more special use to that progenitor, or its progenitors,
+than they now are to these animals having such widely diversified habits. 
+Therefore we may infer that these several bones might have been acquired
+through natural selection, subjected formerly, as now, to the several laws
+of inheritance, reversion, correlation of growth, &amp;c.  Hence every detail
+of structure in every living creature (making some little allowance for the
+direct action of physical conditions) may be viewed, either as having been
+of special use to some ancestral form, or as being now of special use to
+the descendants of this form--either directly, or indirectly through the
+complex laws of growth.
+</para>
+
+<para>
+Natural selection cannot possibly produce any modification in any one
+species exclusively for the good of another species; though throughout
+nature one species incessantly takes advantage of, and profits by, the
+structure of another.  But natural selection can and does often produce
+structures for the direct injury of other species, as we see in the fang of
+the adder, and in the ovipositor of the ichneumon, by which its eggs are
+deposited in the living bodies of other insects.  If it could be proved
+that any part of the structure of any one species had been formed for the
+exclusive good of another species, it would annihilate my theory, for such
+could not have been produced through natural selection.  Although many
+statements may be found in works on natural history to this effect, I
+cannot find even one which seems to me of any weight.  It is admitted that
+the rattlesnake has a poison-fang for its own defence and for the
+destruction of its prey; but some authors suppose that at the same time
+this snake is furnished with a rattle for its own injury, namely, to warn
+its prey to escape.  I would almost as soon believe that the cat curls the
+end of its tail when preparing to spring, in order to warn the doomed
+mouse.  But I have not space here to enter on this and other such cases.
+</para>
+
+<para>
+Natural selection will never produce in a being anything injurious to
+itself, for natural selection acts solely by and for the good of each.  No
+organ will be formed, as Paley has remarked, for the purpose of causing
+pain or for doing an injury to its possessor.  If a fair balance be struck
+between the good and evil caused by each part, each will be found on the
+whole advantageous.  After the lapse of time, under changing conditions of
+life, if any part comes to be injurious, it will be modified; or if it be
+not so, the being will become extinct, as myriads have become extinct.
+</para>
+
+<para>
+Natural selection tends only to make each organic being as perfect as, or
+slightly more perfect than, the other inhabitants of the same country with
+which it has to struggle for existence.  And we see that this is the degree
+of perfection attained under nature.  The endemic productions of New
+Zealand, for instance, are perfect one compared with another; but they are
+now rapidly yielding before the advancing legions of plants and animals
+introduced from Europe.  Natural selection will not produce absolute
+perfection, nor do we always meet, as far as we can judge, with this high
+standard under nature.  The correction for the aberration of light is said,
+on high authority, not to be perfect even in that most perfect organ, the
+eye.  If our reason leads us to admire with enthusiasm a multitude of
+inimitable contrivances in nature, this same reason tells us, though we may
+easily err on both sides, that some other contrivances are less perfect. 
+Can we consider the sting of the wasp or of the bee as perfect, which, when
+used against many attacking animals, cannot be withdrawn, owing to the
+backward serratures, and so inevitably causes the death of the insect by
+tearing out its viscera?
+</para>
+
+<para>
+If we look at the sting of the bee, as having originally existed in a
+remote progenitor as a boring and serrated instrument, like that in so many
+members of the same great order, and which has been modified but not
+perfected for its present purpose, with the poison originally adapted to
+cause galls subsequently intensified, we can perhaps understand how it is
+that the use of the sting should so often cause the insect's own death: 
+for if on the whole the power of stinging be useful to the community, it
+will fulfil all the requirements of natural selection, though it may cause
+the death of some few members.  If we admire the truly wonderful power of
+scent by which the males of many insects find their females, can we admire
+the production for this single purpose of thousands of drones, which are
+utterly useless to the community for any other end, and which are
+ultimately slaughtered by their industrious and sterile sisters?  It may be
+difficult, but we ought to admire the savage instinctive hatred of the
+queen-bee, which urges her instantly to destroy the young queens her
+daughters as soon as born, or to perish herself in the combat; for
+undoubtedly this is for the good of the community; and maternal love or
+maternal hatred, though the latter fortunately is most rare, is all the
+same to the inexorable principle of natural selection.  If we admire the
+several ingenious contrivances, by which the flowers of the orchis and of
+many other plants are fertilised through insect agency, can we consider as
+equally perfect the elaboration by our fir-trees of dense clouds of pollen,
+in order that a few granules may be wafted by a chance breeze on to the
+ovules?
+</para>
+
+<para>
+Summary of Chapter -- We have in this chapter discussed some of the
+difficulties and objections which may be urged against my theory.  Many of
+them are very grave; but I think that in the discussion light has been
+thrown on several facts, which on the theory of independent acts of
+creation are utterly obscure.  We have seen that species at any one period
+are not indefinitely variable, and are not linked together by a multitude
+of intermediate gradations, partly because the process of natural selection
+will always be very slow, and will act, at any one time, only on a very few
+forms; and partly because the very process of natural selection almost
+implies the continual supplanting and extinction of preceding and
+intermediate gradations.  Closely allied species, now living on a
+continuous area, must often have been formed when the area was not
+continuous, and when the conditions of life did not insensibly graduate
+away from one part to another.  When two varieties are formed in two
+districts of a continuous area, an intermediate variety will often be
+formed, fitted for an intermediate zone; but from reasons assigned, the
+intermediate variety will usually exist in lesser numbers than the two
+forms which it connects; consequently the two latter, during the course of
+further modification, from existing in greater numbers, will have a great
+advantage over the less numerous intermediate variety, and will thus
+generally succeed in supplanting and exterminating it.
+</para>
+
+<para>
+We have seen in this chapter how cautious we should be in concluding that
+the most different habits of life could not graduate into each other; that
+a bat, for instance, could not have been formed by natural selection from
+an animal which at first could only glide through the air.
+</para>
+
+<para>
+We have seen that a species may under new conditions of life change its
+habits, or have diversified habits, with some habits very unlike those of
+its nearest congeners.  Hence we can understand, bearing in mind that each
+organic being is trying to live wherever it can live, how it has arisen
+that there are upland geese with webbed feet, ground woodpeckers, diving
+thrushes, and petrels with the habits of auks.
+</para>
+
+<para>
+Although the belief that an organ so perfect as the eye could have been
+formed by natural selection, is more than enough to stagger any one; yet in
+the case of any organ, if we know of a long series of gradations in
+complexity, each good for its possessor, then, under changing conditions of
+life, there is no logical impossibility in the acquirement of any
+conceivable degree of perfection through natural selection.  In the cases
+in which we know of no intermediate or transitional states, we should be
+very cautious in concluding that none could have existed, for the
+homologies of many organs and their intermediate states show that wonderful
+metamorphoses in function are at least possible.  For instance, a
+swim-bladder has apparently been converted into an air-breathing lung.  The
+same organ having performed simultaneously very different functions, and
+then having been specialised for one function; and two very distinct organs
+having performed at the same time the same function, the one having been
+perfected whilst aided by the other, must often have largely facilitated
+transitions.
+</para>
+
+<para>
+We are far too ignorant, in almost every case, to be enabled to assert that
+any part or organ is so unimportant for the welfare of a species, that
+modifications in its structure could not have been slowly accumulated by
+means of natural selection.  But we may confidently believe that many
+modifications, wholly due to the laws of growth, and at first in no way
+advantageous to a species, have been subsequently taken advantage of by the
+still further modified descendants of this species.  We may, also, believe
+that a part formerly of high importance has often been retained (as the
+tail of an aquatic animal by its terrestrial descendants), though it has
+become of such small importance that it could not, in its present state,
+have been acquired by natural selection,--a power which acts solely by the
+preservation of profitable variations in the struggle for life.
+</para>
+
+<para>
+Natural selection will produce nothing in one species for the exclusive
+good or injury of another; though it may well produce parts, organs, and
+excretions highly useful or even indispensable, or highly injurious to
+another species, but in all cases at the same time useful to the owner. 
+Natural selection in each well-stocked country, must act chiefly through
+the competition of the inhabitants one with another, and consequently will
+produce perfection, or strength in the battle for life, only according to
+the standard of that country.  Hence the inhabitants of one country,
+generally the smaller one, will often yield, as we see they do yield, to
+the inhabitants of another and generally larger country.  For in the larger
+country there will have existed more individuals, and more diversified
+forms, and the competition will have been severer, and thus the standard of
+perfection will have been rendered higher.  Natural selection will not
+necessarily produce absolute perfection; nor, as far as we can judge by our
+limited faculties, can absolute perfection be everywhere found.
+</para>
+
+<para>
+On the theory of natural selection we can clearly understand the full
+meaning of that old canon in natural history, 'Natura non facit saltum.' 
+This canon, if we look only to the present inhabitants of the world, is not
+strictly correct, but if we include all those of past times, it must by my
+theory be strictly true.
+</para>
+
+<para>
+It is generally acknowledged that all organic beings have been formed on
+two great laws--Unity of Type, and the Conditions of Existence.  By unity
+of type is meant that fundamental agreement in structure, which we see in
+organic beings of the same class, and which is quite independent of their
+habits of life.  On my theory, unity of type is explained by unity of
+descent.  The expression of conditions of existence, so often insisted on
+by the illustrious Cuvier, is fully embraced by the principle of natural
+selection.  For natural selection acts by either now adapting the varying
+parts of each being to its organic and inorganic conditions of life; or by
+having adapted them during long-past periods of time:  the adaptations
+being aided in some cases by use and disuse, being slightly affected by the
+direct action of the external conditions of life, and being in all cases
+subjected to the several laws of growth.  Hence, in fact, the law of the
+Conditions of Existence is the higher law; as it includes, through the
+inheritance of former adaptations, that of Unity of Type.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter VII</chapnum>
+
+<title>Instinct</title>
+
+<chapsummary>
+Instincts comparable with habits, but different in their origin --
+Instincts graduated -- Aphides and ants -- Instincts variable -- Domestic
+instincts, their origin -- Natural instincts of the cuckoo, ostrich, and
+parasitic bees -- Slave-making ants -- Hive-bee, its cell-making instinct -
+- Difficulties on the theory of the Natural Selection of instincts --
+Neuter or sterile insects -- Summary.
+</chapsummary>
+</chapheader>
+
+<para>
+The subject of instinct might have been worked into the previous chapters;
+but I have thought that it would be more convenient to treat the subject
+separately, especially as so wonderful an instinct as that of the hive-bee
+making its cells will probably have occurred to many readers, as a
+difficulty sufficient to overthrow my whole theory.  I must premise, that I
+have nothing to do with the origin of the primary mental powers, any more
+than I have with that of life itself.  We are concerned only with the
+diversities of instinct and of the other mental qualities of animals within
+the same class.
+</para>
+
+<para>
+I will not attempt any definition of instinct.  It would be easy to show
+that several distinct mental actions are commonly embraced by this term;
+but every one understands what is meant, when it is said that instinct
+impels the cuckoo to migrate and to lay her eggs in other birds' nests.  An
+action, which we ourselves should require experience to enable us to
+perform, when performed by an animal, more especially by a very young one,
+without any experience, and when performed by many individuals in the same
+way, without their knowing for what purpose it is performed, is usually
+said to be instinctive.  But I could show that none of these characters of
+instinct are universal.  A little dose, as Pierre Huber expresses it, of
+judgment or reason, often comes into play, even in animals very low in the
+scale of nature.
+</para>
+
+<para>
+Frederick Cuvier and several of the older metaphysicians have compared
+instinct with habit.  This comparison gives, I think, a remarkably accurate
+notion of the frame of mind under which an instinctive action is performed,
+but not of its origin.  How unconsciously many habitual actions are
+performed, indeed not rarely in direct opposition to our conscious will!
+yet they may be modified by the will or reason.  Habits easily become
+associated with other habits, and with certain periods of time and states
+of the body.  When once acquired, they often remain constant throughout
+life.  Several other points of resemblance between instincts and habits
+could be pointed out.  As in repeating a well-known song, so in instincts,
+one action follows another by a sort of rhythm; if a person be interrupted
+in a song, or in repeating anything by rote, he is generally forced to go
+back to recover the habitual train of thought:  so P. Huber found it was
+with a caterpillar, which makes a very complicated hammock; for if he took
+a caterpillar which had completed its hammock up to, say, the sixth stage
+of construction, and put it into a hammock completed up only to the third
+stage, the caterpillar simply re-performed the fourth, fifth, and sixth
+stages of construction.  If, however, a caterpillar were taken out of a
+hammock made up, for instance, to the third stage, and were put into one
+finished up to the sixth stage, so that much of its work was already done
+for it, far from feeling the benefit of this, it was much embarrassed, and,
+in order to complete its hammock, seemed forced to start from the third
+stage, where it had left off, and thus tried to complete the already
+finished work.  If we suppose any habitual action to become inherited--and
+I think it can be shown that this does sometimes happen--then the
+resemblance between what originally was a habit and an instinct becomes so
+close as not to be distinguished.  If Mozart, instead of playing the
+pianoforte at three years old with wonderfully little practice, had played
+a tune with no practice at all, be might truly be said to have done so
+instinctively.  But it would be the most serious error to suppose that the
+greater number of instincts have been acquired by habit in one generation,
+and then transmitted by inheritance to succeeding generations.  It can be
+clearly shown that the most wonderful instincts with which we are
+acquainted, namely, those of the hive-bee and of many ants, could not
+possibly have been thus acquired.
+</para>
+
+<para>
+It will be universally admitted that instincts are as important as
+corporeal structure for the welfare of each species, under its present
+conditions of life.  Under changed conditions of life, it is at least
+possible that slight modifications of instinct might be profitable to a
+species; and if it can be shown that instincts do vary ever so little, then
+I can see no difficulty in natural selection preserving and continually
+accumulating variations of instinct to any extent that may be profitable. 
+It is thus, as I believe, that all the most complex and wonderful instincts
+have originated.  As modifications of corporeal structure arise from, and
+are increased by, use or habit, and are diminished or lost by disuse, so I
+do not doubt it has been with instincts.  But I believe that the effects of
+habit are of quite subordinate importance to the effects of the natural
+selection of what may be called accidental variations of instincts;--that
+is of variations produced by the same unknown causes which produce slight
+deviations of bodily structure.
+</para>
+
+<para>
+No complex instinct can possibly be produced through natural selection,
+except by the slow and gradual accumulation of numerous, slight, yet
+profitable, variations.  Hence, as in the case of corporeal structures, we
+ought to find in nature, not the actual transitional gradations by which
+each complex instinct has been acquired--for these could be found only in
+the lineal ancestors of each species--but we ought to find in the
+collateral lines of descent some evidence of such gradations; or we ought
+at least to be able to show that gradations of some kind are possible; and
+this we certainly can do.  I have been surprised to find, making allowance
+for the instincts of animals having been but little observed except in
+Europe and North America, and for no instinct being known amongst extinct
+species, how very generally gradations, leading to the most complex
+instincts, can be discovered.  The canon of 'Natura non facit saltum'
+applies with almost equal force to instincts as to bodily organs.  Changes
+of instinct may sometimes be facilitated by the same species having
+different instincts at different periods of life, or at different seasons
+of the year, or when placed under different circumstances, &amp;c.; in which
+case either one or the other instinct might be preserved by natural
+selection.  And such instances of diversity of instinct in the same species
+can be shown to occur in nature.
+</para>
+
+<para>
+Again as in the case of corporeal structure, and conformably with my
+theory, the instinct of each species is good for itself, but has never, as
+far as we can judge, been produced for the exclusive good of others.  One
+of the strongest instances of an animal apparently performing an action for
+the sole good of another, with which I am acquainted, is that of aphides
+voluntarily yielding their sweet excretion to ants:  that they do so
+voluntarily, the following facts show.  I removed all the ants from a group
+of about a dozen aphides on a dock-plant, and prevented their attendance
+during several hours.  After this interval, I felt sure that the aphides
+would want to excrete.  I watched them for some time through a lens, but
+not one excreted; I then tickled and stroked them with a hair in the same
+manner, as well as I could, as the ants do with their antennae; but not one
+excreted.  Afterwards I allowed an ant to visit them, and it immediately
+seemed, by its eager way of running about, to be well aware what a rich
+flock it had discovered; it then began to play with its antennae on the
+abdomen first of one aphis and then of another; and each aphis, as soon as
+it felt the antennae, immediately lifted up its abdomen and excreted a
+limpid drop of sweet juice, which was eagerly devoured by the ant.  Even
+the quite young aphides behaved in this manner, showing that the action was
+instinctive, and not the result of experience.  But as the excretion is
+extremely viscid, it is probably a convenience to the aphides to have it
+removed; and therefore probably the aphides do not instinctively excrete
+for the sole good of the ants.  Although I do not believe that any animal
+in the world performs an action for the exclusive good of another of a
+distinct species, yet each species tries to take advantage of the instincts
+of others, as each takes advantage of the weaker bodily structure of
+others.  So again, in some few cases, certain instincts cannot be
+considered as absolutely perfect; but as details on this and other such
+points are not indispensable, they may be here passed over.
+</para>
+
+<para>
+As some degree of variation in instincts under a state of nature, and the
+inheritance of such variations, are indispensable for the action of natural
+selection, as many instances as possible ought to have been here given; but
+want of space prevents me.  I can only assert, that instincts certainly do
+vary--for instance, the migratory instinct, both in extent and direction,
+and in its total loss.  So it is with the nests of birds, which vary partly
+in dependence on the situations chosen, and on the nature and temperature
+of the country inhabited, but often from causes wholly unknown to us: 
+Audubon has given several remarkable cases of differences in nests of the
+same species in the northern and southern United States.  Fear of any
+particular enemy is certainly an instinctive quality, as may be seen in
+nestling birds, though it is strengthened by experience, and by the sight
+of fear of the same enemy in other animals.  But fear of man is slowly
+acquired, as I have elsewhere shown, by various animals inhabiting desert
+islands; and we may see an instance of this, even in England, in the
+greater wildness of all our large birds than of our small birds; for the
+large birds have been most persecuted by man.  We may safely attribute the
+greater wildness of our large birds to this cause; for in uninhabited
+islands large birds are not more fearful than small; and the magpie, so
+wary in England, is tame in Norway, as is the hooded crow in Egypt.
+</para>
+
+<para>
+That the general disposition of individuals of the same species, born in a
+state of nature, is extremely diversified, can be shown by a multitude of
+facts.  Several cases also, could be given, of occasional and strange
+habits in certain species, which might, if advantageous to the species,
+give rise, through natural selection, to quite new instincts.  But I am
+well aware that these general statements, without facts given in detail,
+can produce but a feeble effect on the reader's mind.  I can only repeat my
+assurance, that I do not speak without good evidence.
+</para>
+
+<para>
+The possibility, or even probability, of inherited variations of instinct
+in a state of nature will be strengthened by briefly considering a few
+cases under domestication.  We shall thus also be enabled to see the
+respective parts which habit and the selection of so-called accidental
+variations have played in modifying the mental qualities of our domestic
+animals.  A number of curious and authentic instances could be given of the
+inheritance of all shades of disposition and tastes, and likewise of the
+oddest tricks, associated with certain frames of mind or periods of time. 
+But let us look to the familiar case of the several breeds of dogs:  it
+cannot be doubted that young pointers (I have myself seen a striking
+instance) will sometimes point and even back other dogs the very first time
+that they are taken out; retrieving is certainly in some degree inherited
+by retrievers; and a tendency to run round, instead of at, a flock of
+sheep, by shepherd-dogs.  I cannot see that these actions, performed
+without experience by the young, and in nearly the same manner by each
+individual, performed with eager delight by each breed, and without the end
+being known,--for the young pointer can no more know that he points to aid
+his master, than the white butterfly knows why she lays her eggs on the
+leaf of the cabbage,--I cannot see that these actions differ essentially
+from true instincts.  If we were to see one kind of wolf, when young and
+without any training, as soon as it scented its prey, stand motionless like
+a statue, and then slowly crawl forward with a peculiar gait; and another
+kind of wolf rushing round, instead of at, a herd of deer, and driving them
+to a distant point, we should assuredly call these actions instinctive. 
+Domestic instincts, as they may be called, are certainly far less fixed or
+invariable than natural instincts; but they have been acted on by far less
+rigorous selection, and have been transmitted for an incomparably shorter
+period, under less fixed conditions of life.
+</para>
+
+<para>
+How strongly these domestic instincts, habits, and dispositions are
+inherited, and how curiously they become mingled, is well shown when
+different breeds of dogs are crossed.  Thus it is known that a cross with a
+bull-dog has affected for many generations the courage and obstinacy of
+greyhounds; and a cross with a greyhound has given to a whole family of
+shepherd-dogs a tendency to hunt hares.  These domestic instincts, when
+thus tested by crossing, resemble natural instincts, which in a like manner
+become curiously blended together, and for a long period exhibit traces of
+the instincts of either parent:  for example, Le Roy describes a dog, whose
+great-grandfather was a wolf, and this dog showed a trace of its wild
+parentage only in one way, by not coming in a straight line to his master
+when called.
+</para>
+
+<para>
+Domestic instincts are sometimes spoken of as actions which have become
+inherited solely from long-continued and compulsory habit, but this, I
+think, is not true.  No one would ever have thought of teaching, or
+probably could have taught, the tumbler-pigeon to tumble,--an action which,
+as I have witnessed, is performed by young birds, that have never seen a
+pigeon tumble.  We may believe that some one pigeon showed a slight
+tendency to this strange habit, and that the long-continued selection of
+the best individuals in successive generations made tumblers what they now
+are; and near Glasgow there are house-tumblers, as I hear from Mr. Brent,
+which cannot fly eighteen inches high without going head over heels.  It
+may be doubted whether any one would have thought of training a dog to
+point, had not some one dog naturally shown a tendency in this line; and
+this is known occasionally to happen, as I once saw in a pure terrier. 
+When the first tendency was once displayed, methodical selection and the
+inherited effects of compulsory training in each successive generation
+would soon complete the work; and unconscious selection is still at work,
+as each man tries to procure, without intending to improve the breed, dogs
+which will stand and hunt best.  On the other hand, habit alone in some
+cases has sufficed; no animal is more difficult to tame than the young of
+the wild rabbit; scarcely any animal is tamer than the young of the tame
+rabbit; but I do not suppose that domestic rabbits have ever been selected
+for tameness; and I presume that we must attribute the whole of the
+inherited change from extreme wildness to extreme tameness, simply to habit
+and long-continued close confinement.
+</para>
+
+<para>
+Natural instincts are lost under domestication:  a remarkable instance of
+this is seen in those breeds of fowls which very rarely or never become
+'broody,' that is, never wish to sit on their eggs.  Familiarity alone
+prevents our seeing how universally and largely the minds of our domestic
+animals have been modified by domestication.  It is scarcely possible to
+doubt that the love of man has become instinctive in the dog.  All wolves,
+foxes, jackals, and species of the cat genus, when kept tame, are most
+eager to attack poultry, sheep, and pigs; and this tendency has been found
+incurable in dogs which have been brought home as puppies from countries,
+such as Tierra del Fuego and Australia, where the savages do not keep these
+domestic animals.  How rarely, on the other hand, do our civilised dogs,
+even when quite young, require to be taught not to attack poultry, sheep,
+and pigs!  No doubt they occasionally do make an attack, and are then
+beaten; and if not cured, they are destroyed; so that habit, with some
+degree of selection, has probably concurred in civilising by inheritance
+our dogs.  On the other hand, young chickens have lost, wholly by habit,
+that fear of the dog and cat which no doubt was originally instinctive in
+them, in the same way as it is so plainly instinctive in young pheasants,
+though reared under a hen.  It is not that chickens have lost all fear, but
+fear only of dogs and cats, for if the hen gives the danger-chuckle, they
+will run (more especially young turkeys) from under her, and conceal
+themselves in the surrounding grass or thickets; and this is evidently done
+for the instinctive purpose of allowing, as we see in wild ground-birds,
+their mother to fly away.  But this instinct retained by our chickens has
+become useless under domestication, for the mother-hen has almost lost by
+disuse the power of flight.
+</para>
+
+<para>
+Hence, we may conclude, that domestic instincts have been acquired and
+natural instincts have been lost partly by habit, and partly by man
+selecting and accumulating during successive generations, peculiar mental
+habits and actions, which at first appeared from what we must in our
+ignorance call an accident.  In some cases compulsory habit alone has
+sufficed to produce such inherited mental changes; in other cases
+compulsory habit has done nothing, and all has been the result of
+selection, pursued both methodically and unconsciously; but in most cases,
+probably, habit and selection have acted together.
+</para>
+
+<para>
+We shall, perhaps, best understand how instincts in a state of nature have
+become modified by selection, by considering a few cases.  I will select
+only three, out of the several which I shall have to discuss in my future
+work,--namely, the instinct which leads the cuckoo to lay her eggs in other
+birds' nests; the slave-making instinct of certain ants; and the
+comb-making power of the hive-bee:  these two latter instincts have
+generally, and most justly, been ranked by naturalists as the most
+wonderful of all known instincts.
+</para>
+
+<para>
+It is now commonly admitted that the more immediate and final cause of the
+cuckoo's instinct is, that she lays her eggs, not daily, but at intervals
+of two or three days; so that, if she were to make her own nest and sit on
+her own eggs, those first laid would have to be left for some time
+unincubated, or there would be eggs and young birds of different ages in
+the same nest.  If this were the case, the process of laying and hatching
+might be inconveniently long, more especially as she has to migrate at a
+very early period; and the first hatched young would probably have to be
+fed by the male alone.  But the American cuckoo is in this predicament; for
+she makes her own nest and has eggs and young successively hatched, all at
+the same time.  It has been asserted that the American cuckoo occasionally
+lays her eggs in other birds' nests; but I hear on the high authority of
+Dr. Brewer, that this is a mistake.  Nevertheless, I could give several
+instances of various birds which have been known occasionally to lay their
+eggs in other birds' nests.  Now let us suppose that the ancient progenitor
+of our European cuckoo had the habits of the American cuckoo; but that
+occasionally she laid an egg in another bird's nest.  If the old bird
+profited by this occasional habit, or if the young were made more vigorous
+by advantage having been taken of the mistaken maternal instinct of another
+bird, than by their own mother's care, encumbered as she can hardly fail to
+be by having eggs and young of different ages at the same time; then the
+old birds or the fostered young would gain an advantage.  And analogy would
+lead me to believe, that the young thus reared would be apt to follow by
+inheritance the occasional and aberrant habit of their mother, and in their
+turn would be apt to lay their eggs in other birds' nests, and thus be
+successful in rearing their young.  By a continued process of this nature,
+I believe that the strange instinct of our cuckoo could be, and has been,
+generated.  I may add that, according to Dr. Gray and to some other
+observers, the European cuckoo has not utterly lost all maternal love and
+care for her own offspring.
+</para>
+
+<para>
+The occasional habit of birds laying their eggs in other birds' nests,
+either of the same or of a distinct species, is not very uncommon with the
+Gallinaceae; and this perhaps explains the origin of a singular instinct in
+the allied group of ostriches.  For several hen ostriches, at least in the
+case of the American species, unite and lay first a few eggs in one nest
+and then in another; and these are hatched by the males.  This instinct may
+probably be accounted for by the fact of the hens laying a large number of
+eggs; but, as in the case of the cuckoo, at intervals of two or three days. 
+This instinct, however, of the American ostrich has not as yet been
+perfected; for a surprising number of eggs lie strewed over the plains, so
+that in one day's hunting I picked up no less than twenty lost and wasted
+eggs.
+</para>
+
+<para>
+Many bees are parasitic, and always lay their eggs in the nests of bees of
+other kinds.  This case is more remarkable than that of the cuckoo; for
+these bees have not only their instincts but their structure modified in
+accordance with their parasitic habits; for they do not possess the
+pollen-collecting apparatus which would be necessary if they had to store
+food for their own young.  Some species, likewise, of Sphegidae (wasp-like
+insects) are parasitic on other species; and M. Fabre has lately shown good
+reason for believing that although the Tachytes nigra generally makes its
+own burrow and stores it with paralysed prey for its own larvae to feed on,
+yet that when this insect finds a burrow already made and stored by another
+sphex, it takes advantage of the prize, and becomes for the occasion
+parasitic.  In this case, as with the supposed case of the cuckoo, I can
+see no difficulty in natural selection making an occasional habit
+permanent, if of advantage to the species, and if the insect whose nest and
+stored food are thus feloniously appropriated, be not thus exterminated.
+</para>
+
+<para>
+Slave-making instinct. -- This remarkable instinct was first discovered in
+the Formica (Polyerges) rufescens by Pierre Huber, a better observer even
+than his celebrated father.  This ant is absolutely dependent on its
+slaves; without their aid, the species would certainly become extinct in a
+single year.  The males and fertile females do no work.  The workers or
+sterile females, though most energetic and courageous in capturing slaves,
+do no other work.  They are incapable of making their own nests, or of
+feeding their own larvae.  When the old nest is found inconvenient, and
+they have to migrate, it is the slaves which determine the migration, and
+actually carry their masters in their jaws.  So utterly helpless are the
+masters, that when Huber shut up thirty of them without a slave, but with
+plenty of the food which they like best, and with their larvae and pupae to
+stimulate them to work, they did nothing; they could not even feed
+themselves, and many perished of hunger.  Huber then introduced a single
+slave (F. fusca), and she instantly set to work, fed and saved the
+survivors; made some cells and tended the larvae, and put all to rights. 
+What can be more extraordinary than these well-ascertained facts?  If we
+had not known of any other slave-making ant, it would have been hopeless to
+have speculated how so wonderful an instinct could have been perfected.
+</para>
+
+<para>
+Formica sanguinea was likewise first discovered by P. Huber to be a
+slave-making ant.  This species is found in the southern parts of England,
+and its habits have been attended to by Mr. F. Smith, of the British
+Museum, to whom I am much indebted for information on this and other
+subjects.  Although fully trusting to the statements of Huber and Mr.
+Smith, I tried to approach the subject in a sceptical frame of mind, as any
+one may well be excused for doubting the truth of so extraordinary and
+odious an instinct as that of making slaves.  Hence I will give the
+observations which I have myself made, in some little detail.  I opened
+fourteen nests of F. sanguinea, and found a few slaves in all.  Males and
+fertile females of the slave-species are found only in their own proper
+communities, and have never been observed in the nests of F. sanguinea. 
+The slaves are black and not above half the size of their red masters, so
+that the contrast in their appearance is very great.  When the nest is
+slightly disturbed, the slaves occasionally come out, and like their
+masters are much agitated and defend their nest:  when the nest is much
+disturbed and the larvae and pupae are exposed, the slaves work
+energetically with their masters in carrying them away to a place of
+safety.  Hence, it is clear, that the slaves feel quite at home.  During
+the months of June and July, on three successive years, I have watched for
+many hours several nests in Surrey and Sussex, and never saw a slave either
+leave or enter a nest.  As, during these months, the slaves are very few in
+number, I thought that they might behave differently when more numerous;
+but Mr. Smith informs me that he has watched the nests at various hours
+during May, June and August, both in Surrey and Hampshire, and has never
+seen the slaves, though present in large numbers in August, either leave or
+enter the nest.  Hence he considers them as strictly household slaves.  The
+masters, on the other hand, may be constantly seen bringing in materials
+for the nest, and food of all kinds.  During the present year, however, in
+the month of July, I came across a community with an unusually large stock
+of slaves, and I observed a few slaves mingled with their masters leaving
+the nest, and marching along the same road to a tall Scotch-fir-tree,
+twenty-five yards distant, which they ascended together, probably in search
+of aphides or cocci.  According to Huber, who had ample opportunities for
+observation, in Switzerland the slaves habitually work with their masters
+in making the nest, and they alone open and close the doors in the morning
+and evening; and, as Huber expressly states, their principal office is to
+search for aphides.  This difference in the usual habits of the masters and
+slaves in the two countries, probably depends merely on the slaves being
+captured in greater numbers in Switzerland than in England.
+</para>
+
+<para>
+One day I fortunately chanced to witness a migration from one nest to
+another, and it was a most interesting spectacle to behold the masters
+carefully carrying, as Huber has described, their slaves in their jaws. 
+Another day my attention was struck by about a score of the slave-makers
+haunting the same spot, and evidently not in search of food; they
+approached and were vigorously repulsed by an independent community of the
+slave species (F. fusca); sometimes as many as three of these ants clinging
+to the legs of the slave-making F. sanguinea.  The latter ruthlessly killed
+their small opponents, and carried their dead bodies as food to their nest,
+twenty-nine yards distant; but they were prevented from getting any pupae
+to rear as slaves.  I then dug up a small parcel of the pupae of F. fusca
+from another nest, and put them down on a bare spot near the place of
+combat; they were eagerly seized, and carried off by the tyrants, who
+perhaps fancied that, after all, they had been victorious in their late
+combat.
+</para>
+
+<para>
+At the same time I laid on the same place a small parcel of the pupae of
+another species, F. flava, with a few of these little yellow ants still
+clinging to the fragments of the nest.  This species is sometimes, though
+rarely, made into slaves, as has been described by Mr. Smith.  Although so
+small a species, it is very courageous, and I have seen it ferociously
+attack other ants.  In one instance I found to my surprise an independent
+community of F. flava under a stone beneath a nest of the slave-making F.
+sanguinea; and when I had accidentally disturbed both nests, the little
+ants attacked their big neighbours with surprising courage.  Now I was
+curious to ascertain whether F. sanguinea could distinguish the pupae of F.
+fusca, which they habitually make into slaves, from those of the little and
+furious F. flava, which they rarely capture, and it was evident that they
+did at once distinguish them:  for we have seen that they eagerly and
+instantly seized the pupae of F. fusca, whereas they were much terrified
+when they came across the pupae, or even the earth from the nest of F.
+flava, and quickly ran away; but in about a quarter of an hour, shortly
+after all the little yellow ants had crawled away, they took heart and
+carried off the pupae.
+</para>
+
+<para>
+One evening I visited another community of F. sanguinea, and found a number
+of these ants entering their nest, carrying the dead bodies of F. fusca
+(showing that it was not a migration) and numerous pupae.  I traced the
+returning file burthened with booty, for about forty yards, to a very thick
+clump of heath, whence I saw the last individual of F. sanguinea emerge,
+carrying a pupa; but I was not able to find the desolated nest in the thick
+heath.  The nest, however, must have been close at hand, for two or three
+individuals of F. fusca were rushing about in the greatest agitation, and
+one was perched motionless with its own pupa in its mouth on the top of a
+spray of heath over its ravaged home.
+</para>
+
+<para>
+Such are the facts, though they did not need confirmation by me, in regard
+to the wonderful instinct of making slaves.  Let it be observed what a
+contrast the instinctive habits of F. sanguinea present with those of the
+F. rufescens.  The latter does not build its own nest, does not determine
+its own migrations, does not collect food for itself or its young, and
+cannot even feed itself:  it is absolutely dependent on its numerous
+slaves.  Formica sanguinea, on the other hand, possesses much fewer slaves,
+and in the early part of the summer extremely few.  The masters determine
+when and where a new nest shall be formed, and when they migrate, the
+masters carry the slaves.  Both in Switzerland and England the slaves seem
+to have the exclusive care of the larvae, and the masters alone go on
+slave-making expeditions.  In Switzerland the slaves and masters work
+together, making and bringing materials for the nest:  both, but chiefly
+the slaves, tend, and milk as it may be called, their aphides; and thus
+both collect food for the community.  In England the masters alone usually
+leave the nest to collect building materials and food for themselves, their
+slaves and larvae.  So that the masters in this country receive much less
+service from their slaves than they do in Switzerland.
+</para>
+
+<para>
+By what steps the instinct of F. sanguinea originated I will not pretend to
+conjecture.  But as ants, which are not slave-makers, will, as I have seen,
+carry off pupae of other species, if scattered near their nests, it is
+possible that pupae originally stored as food might become developed; and
+the ants thus unintentionally reared would then follow their proper
+instincts, and do what work they could.  If their presence proved useful to
+the species which had seized them--if it were more advantageous to this
+species to capture workers than to procreate them--the habit of collecting
+pupae originally for food might by natural selection be strengthened and
+rendered permanent for the very different purpose of raising slaves.  When
+the instinct was once acquired, if carried out to a much less extent even
+than in our British F. sanguinea, which, as we have seen, is less aided by
+its slaves than the same species in Switzerland, I can see no difficulty in
+natural selection increasing and modifying the instinct--always supposing
+each modification to be of use to the species--until an ant was formed as
+abjectly dependent on its slaves as is the Formica rufescens.
+</para>
+
+<para>
+Cell-making instinct of the Hive-Bee. -- I will not here enter on minute
+details on this subject, but will merely give an outline of the conclusions
+at which I have arrived.  He must be a dull man who can examine the
+exquisite structure of a comb, so beautifully adapted to its end, without
+enthusiastic admiration.  We hear from mathematicians that bees have
+practically solved a recondite problem, and have made their cells of the
+proper shape to hold the greatest possible amount of honey, with the least
+possible consumption of precious wax in their construction.  It has been
+remarked that a skilful workman, with fitting tools and measures, would
+find it very difficult to make cells of wax of the true form, though this
+is perfectly effected by a crowd of bees working in a dark hive.  Grant
+whatever instincts you please, and it seems at first quite inconceivable
+how they can make all the necessary angles and planes, or even perceive
+when they are correctly made.  But the difficulty is not nearly so great as
+it at first appears:  all this beautiful work can be shown, I think, to
+follow from a few very simple instincts.
+</para>
+
+<para>
+I was led to investigate this subject by Mr. Waterhouse, who has shown that
+the form of the cell stands in close relation to the presence of adjoining
+cells; and the following view may, perhaps, be considered only as a
+modification of this theory.  Let us look to the great principle of
+gradation, and see whether Nature does not reveal to us her method of work. 
+At one end of a short series we have humble-bees, which use their old
+cocoons to hold honey, sometimes adding to them short tubes of wax, and
+likewise making separate and very irregular rounded cells of wax.  At the
+other end of the series we have the cells of the hive-bee, placed in a
+double layer:  each cell, as is well known, is an hexagonal prism, with the
+basal edges of its six sides bevelled so as to join on to a pyramid, formed
+of three rhombs.  These rhombs have certain angles, and the three which
+form the pyramidal base of a single cell on one side of the comb, enter
+into the composition of the bases of three adjoining cells on the opposite
+side.  In the series between the extreme perfection of the cells of the
+hive-bee and the simplicity of those of the humble-bee, we have the cells
+of the Mexican Melipona domestica, carefully described and figured by
+Pierre Huber.  The Melipona itself is intermediate in structure between the
+hive and humble bee, but more nearly related to the latter:  it forms a
+nearly regular waxen comb of cylindrical cells, in which the young are
+hatched, and, in addition, some large cells of wax for holding honey. 
+These latter cells are nearly spherical and of nearly equal sizes, and are
+aggregated into an irregular mass.  But the important point to notice, is
+that these cells are always made at that degree of nearness to each other,
+that they would have intersected or broken into each other, if the spheres
+had been completed; but this is never permitted, the bees building
+perfectly flat walls of wax between the spheres which thus tend to
+intersect.  Hence each cell consists of an outer spherical portion and of
+two, three, or more perfectly flat surfaces, according as the cell adjoins
+two, three or more other cells.  When one cell comes into contact with
+three other cells, which, from the spheres being nearly of the same size,
+is very frequently and necessarily the case, the three flat surfaces are
+united into a pyramid; and this pyramid, as Huber has remarked, is
+manifestly a gross imitation of the three-sided pyramidal basis of the cell
+of the hive-bee.  As in the cells of the hive-bee, so here, the three plane
+surfaces in any one cell necessarily enter into the construction of three
+adjoining cells.  It is obvious that the Melipona saves wax by this manner
+of building; for the flat walls between the adjoining cells are not double,
+but are of the same thickness as the outer spherical portions, and yet each
+flat portion forms a part of two cells.
+</para>
+
+<para>
+Reflecting on this case, it occurred to me that if the Melipona had made
+its spheres at some given distance from each other, and had made them of
+equal sizes and had arranged them symmetrically in a double layer, the
+resulting structure would probably have been as perfect as the comb of the
+hive-bee.  Accordingly I wrote to Professor Miller, of Cambridge, and this
+geometer has kindly read over the following statement, drawn up from his
+information, and tells me that it is strictly correct:-
+</para>
+
+<para>
+If a number of equal spheres be described with their centres placed in two
+parallel layers; with the centre of each sphere at the distance of radius x
+sqrt(2) or radius x 1.41421 (or at some lesser distance), from the centres
+of the six surrounding spheres in the same layer; and at the same distance
+from the centres of the adjoining spheres in the other and parallel layer;
+then, if planes of intersection between the several spheres in both layers
+be formed, there will result a double layer of hexagonal prisms united
+together by pyramidal bases formed of three rhombs; and the rhombs and the
+sides of the hexagonal prisms will have every angle identically the same
+with the best measurements which have been made of the cells of the
+hive-bee.
+</para>
+
+<para>
+Hence we may safely conclude that if we could slightly modify the instincts
+already possessed by the Melipona, and in themselves not very wonderful,
+this bee would make a structure as wonderfully perfect as that of the
+hive-bee.  We must suppose the Melipona to make her cells truly spherical,
+and of equal sizes; and this would not be very surprising, seeing that she
+already does so to a certain extent, and seeing what perfectly cylindrical
+burrows in wood many insects can make, apparently by turning round on a
+fixed point.  We must suppose the Melipona to arrange her cells in level
+layers, as she already does her cylindrical cells; and we must further
+suppose, and this is the greatest difficulty, that she can somehow judge
+accurately at what distance to stand from her fellow-labourers when several
+are making their spheres; but she is already so far enabled to judge of
+distance, that she always describes her spheres so as to intersect largely;
+and then she unites the points of intersection by perfectly flat surfaces. 
+We have further to suppose, but this is no difficulty, that after hexagonal
+prisms have been formed by the intersection of adjoining spheres in the
+same layer, she can prolong the hexagon to any length requisite to hold the
+stock of honey; in the same way as the rude humble-bee adds cylinders of
+wax to the circular mouths of her old cocoons.  By such modifications of
+instincts in themselves not very wonderful,--hardly more wonderful than
+those which guide a bird to make its nest,--I believe that the hive-bee has
+acquired, through natural selection, her inimitable architectural powers.
+</para>
+
+<para>
+But this theory can be tested by experiment.  Following the example of Mr.
+Tegetmeier, I separated two combs, and put between them a long, thick,
+square strip of wax:  the bees instantly began to excavate minute circular
+pits in it; and as they deepened these little pits, they made them wider
+and wider until they were converted into shallow basins, appearing to the
+eye perfectly true or parts of a sphere, and of about the diameter of a
+cell.  It was most interesting to me to observe that wherever several bees
+had begun to excavate these basins near together, they had begun their work
+at such a distance from each other, that by the time the basins had
+acquired the above stated width (i.e. about the width of an ordinary cell),
+and were in depth about one sixth of the diameter of the sphere of which
+they formed a part, the rims of the basins intersected or broke into each
+other.  As soon as this occurred, the bees ceased to excavate, and began to
+build up flat walls of wax on the lines of intersection between the basins,
+so that each hexagonal prism was built upon the festooned edge of a smooth
+basin, instead of on the straight edges of a three-sided pyramid as in the
+case of ordinary cells.
+</para>
+
+<para>
+I then put into the hive, instead of a thick, square piece of wax, a thin
+and narrow, knife-edged ridge, coloured with vermilion.  The bees instantly
+began on both sides to excavate little basins near to each other, in the
+same way as before; but the ridge of wax was so thin, that the bottoms of
+the basins, if they had been excavated to the same depth as in the former
+experiment, would have broken into each other from the opposite sides.  The
+bees, however, did not suffer this to happen, and they stopped their
+excavations in due time; so that the basins, as soon as they had been a
+little deepened, came to have flat bottoms; and these flat bottoms, formed
+by thin little plates of the vermilion wax having been left ungnawed, were
+situated, as far as the eye could judge, exactly along the planes of
+imaginary intersection between the basins on the opposite sides of the
+ridge of wax.  In parts, only little bits, in other parts, large portions
+of a rhombic plate had been left between the opposed basins, but the work,
+from the unnatural state of things, had not been neatly performed.  The
+bees must have worked at very nearly the same rate on the opposite sides of
+the ridge of vermilion wax, as they circularly gnawed away and deepened the
+basins on both sides, in order to have succeeded in thus leaving flat
+plates between the basins, by stopping work along the intermediate planes
+or planes of intersection.
+</para>
+
+<para>
+Considering how flexible thin wax is, I do not see that there is any
+difficulty in the bees, whilst at work on the two sides of a strip of wax,
+perceiving when they have gnawed the wax away to the proper thinness, and
+then stopping their work.  In ordinary combs it has appeared to me that the
+bees do not always succeed in working at exactly the same rate from the
+opposite sides; for I have noticed half-completed rhombs at the base of a
+just-commenced cell, which were slightly concave on one side, where I
+suppose that the bees had excavated too quickly, and convex on the opposed
+side, where the bees had worked less quickly.  In one well-marked instance,
+I put the comb back into the hive, and allowed the bees to go on working
+for a short time, and again examined the cell, and I found that the rhombic
+plate had been completed, and had become perfectly flat:  it was absolutely
+impossible, from the extreme thinness of the little rhombic plate, that
+they could have effected this by gnawing away the convex side; and I
+suspect that the bees in such cases stand in the opposed cells and push and
+bend the ductile and warm wax (which as I have tried is easily done) into
+its proper intermediate plane, and thus flatten it.
+</para>
+
+<para>
+From the experiment of the ridge of vermilion wax, we can clearly see that
+if the bees were to build for themselves a thin wall of wax, they could
+make their cells of the proper shape, by standing at the proper distance
+from each other, by excavating at the same rate, and by endeavouring to
+make equal spherical hollows, but never allowing the spheres to break into
+each other.  Now bees, as may be clearly seen by examining the edge of a
+growing comb, do make a rough, circumferential wall or rim all round the
+comb; and they gnaw into this from the opposite sides, always working
+circularly as they deepen each cell.  They do not make the whole
+three-sided pyramidal base of any one cell at the same time, but only the
+one rhombic plate which stands on the extreme growing margin, or the two
+plates, as the case may be; and they never complete the upper edges of the
+rhombic plates, until the hexagonal walls are commenced.  Some of these
+statements differ from those made by the justly celebrated elder Huber, but
+I am convinced of their accuracy; and if I had space, I could show that
+they are conformable with my theory.
+</para>
+
+<para>
+Huber's statement that the very first cell is excavated out of a little
+parallel-sided wall of wax, is not, as far as I have seen, strictly
+correct; the first commencement having always been a little hood of wax;
+but I will not here enter on these details.  We see how important a part
+excavation plays in the construction of the cells; but it would be a great
+error to suppose that the bees cannot build up a rough wall of wax in the
+proper position--that is, along the plane of intersection between two
+adjoining spheres.  I have several specimens showing clearly that they can
+do this.  Even in the rude circumferential rim or wall of wax round a
+growing comb, flexures may sometimes be observed, corresponding in position
+to the planes of the rhombic basal plates of future cells.  But the rough
+wall of wax has in every case to be finished off, by being largely gnawed
+away on both sides.  The manner in which the bees build is curious; they
+always make the first rough wall from ten to twenty times thicker than the
+excessively thin finished wall of the cell, which will ultimately be left. 
+We shall understand how they work, by supposing masons first to pile up a
+broad ridge of cement, and then to begin cutting it away equally on both
+sides near the ground, till a smooth, very thin wall is left in the middle;
+the masons always piling up the cut-away cement, and adding fresh cement,
+on the summit of the ridge.  We shall thus have a thin wall steadily
+growing upward; but always crowned by a gigantic coping.  From all the
+cells, both those just commenced and those completed, being thus crowned by
+a strong coping of wax, the bees can cluster and crawl over the comb
+without injuring the delicate hexagonal walls, which are only about one
+four-hundredth of an inch in thickness; the plates of the pyramidal basis
+being about twice as thick.  By this singular manner of building, strength
+is continually given to the comb, with the utmost ultimate economy of wax.
+</para>
+
+<para>
+It seems at first to add to the difficulty of understanding how the cells
+are made, that a multitude of bees all work together; one bee after working
+a short time at one cell going to another, so that, as Huber has stated, a
+score of individuals work even at the commencement of the first cell.  I
+was able practically to show this fact, by covering the edges of the
+hexagonal walls of a single cell, or the extreme margin of the
+circumferential rim of a growing comb, with an extremely thin layer of
+melted vermilion wax; and I invariably found that the colour was most
+delicately diffused by the bees--as delicately as a painter could have done
+with his brush--by atoms of the coloured wax having been taken from the
+spot on which it had been placed, and worked into the growing edges of the
+cells all round.  The work of construction seems to be a sort of balance
+struck between many bees, all instinctively standing at the same relative
+distance from each other, all trying to sweep equal spheres, and then
+building up, or leaving ungnawed, the planes of intersection between these
+spheres.  It was really curious to note in cases of difficulty, as when two
+pieces of comb met at an angle, how often the bees would entirely pull down
+and rebuild in different ways the same cell, sometimes recurring to a shape
+which they had at first rejected.
+</para>
+
+<para>
+When bees have a place on which they can stand in their proper positions
+for working,--for instance, on a slip of wood, placed directly under the
+middle of a comb growing downwards so that the comb has to be built over
+one face of the slip--in this case the bees can lay the foundations of one
+wall of a new hexagon, in its strictly proper place, projecting beyond the
+other completed cells.  It suffices that the bees should be enabled to
+stand at their proper relative distances from each other and from the walls
+of the last completed cells, and then, by striking imaginary spheres, they
+can build up a wall intermediate between two adjoining spheres; but, as far
+as I have seen, they never gnaw away and finish off the angles of a cell
+till a large part both of that cell and of the adjoining cells has been
+built.  This capacity in bees of laying down under certain circumstances a
+rough wall in its proper place between two just-commenced cells, is
+important, as it bears on a fact, which seems at first quite subversive of
+the foregoing theory; namely, that the cells on the extreme margin of
+wasp-combs are sometimes strictly hexagonal; but I have not space here to
+enter on this subject.  Nor does there seem to me any great difficulty in a
+single insect (as in the case of a queen-wasp) making hexagonal cells, if
+she work alternately on the inside and outside of two or three cells
+commenced at the same time, always standing at the proper relative distance
+from the parts of the cells just begun, sweeping spheres or cylinders, and
+building up intermediate planes.  It is even conceivable that an insect
+might, by fixing on a point at which to commence a cell, and then moving
+outside, first to one point, and then to five other points, at the proper
+relative distances from the central point and from each other, strike the
+planes of intersection, and so make an isolated hexagon:  but I am not
+aware that any such case has been observed; nor would any good be derived
+from a single hexagon being built, as in its construction more materials
+would be required than for a cylinder.
+</para>
+
+<para>
+As natural selection acts only by the accumulation of slight modifications
+of structure or instinct, each profitable to the individual under its
+conditions of life, it may reasonably be asked, how a long and graduated
+succession of modified architectural instincts, all tending towards the
+present perfect plan of construction, could have profited the progenitors
+of the hive-bee?  I think the answer is not difficult:  it is known that
+bees are often hard pressed to get sufficient nectar; and I am informed by
+Mr. Tegetmeier that it has been experimentally found that no less than from
+twelve to fifteen pounds of dry sugar are consumed by a hive of bees for
+the secretion of each pound of wax; so that a prodigious quantity of fluid
+nectar must be collected and consumed by the bees in a hive for the
+secretion of the wax necessary for the construction of their combs. 
+Moreover, many bees have to remain idle for many days during the process of
+secretion.  A large store of honey is indispensable to support a large
+stock of bees during the winter; and the security of the hive is known
+mainly to depend on a large number of bees being supported.  Hence the
+saving of wax by largely saving honey must be a most important element of
+success in any family of bees.  Of course the success of any species of bee
+may be dependent on the number of its parasites or other enemies, or on
+quite distinct causes, and so be altogether independent of the quantity of
+honey which the bees could collect.  But let us suppose that this latter
+circumstance determined, as it probably often does determine, the numbers
+of a humble-bee which could exist in a country; and let us further suppose
+that the community lived throughout the winter, and consequently required a
+store of honey:  there can in this case be no doubt that it would be an
+advantage to our humble-bee, if a slight modification of her instinct led
+her to make her waxen cells near together, so as to intersect a little; for
+a wall in common even to two adjoining cells, would save some little wax. 
+Hence it would continually be more and more advantageous to our humble-bee,
+if she were to make her cells more and more regular, nearer together, and
+aggregated into a mass, like the cells of the Melipona; for in this case a
+large part of the bounding surface of each cell would serve to bound other
+cells, and much wax would be saved.  Again, from the same cause, it would
+be advantageous to the Melipona, if she were to make her cells closer
+together, and more regular in every way than at present; for then, as we
+have seen, the spherical surfaces would wholly disappear, and would all be
+replaced by plane surfaces; and the Melipona would make a comb as perfect
+as that of the hive-bee.  Beyond this stage of perfection in architecture,
+natural selection could not lead; for the comb of the hive-bee, as far as
+we can see, is absolutely perfect in economising wax.
+</para>
+
+<para>
+Thus, as I believe, the most wonderful of all known instincts, that of the
+hive-bee, can be explained by natural selection having taken advantage of
+numerous, successive, slight modifications of simpler instincts; natural
+selection having by slow degrees, more and more perfectly, led the bees to
+sweep equal spheres at a given distance from each other in a double layer,
+and to build up and excavate the wax along the planes of intersection.  The
+bees, of course, no more knowing that they swept their spheres at one
+particular distance from each other, than they know what are the several
+angles of the hexagonal prisms and of the basal rhombic plates.  The motive
+power of the process of natural selection having been economy of wax; that
+individual swarm which wasted least honey in the secretion of wax, having
+succeeded best, and having transmitted by inheritance its newly acquired
+economical instinct to new swarms, which in their turn will have had the
+best chance of succeeding in the struggle for existence.
+</para>
+
+<para>
+No doubt many instincts of very difficult explanation could be opposed to
+the theory of natural selection,--cases, in which we cannot see how an
+instinct could possibly have originated; cases, in which no intermediate
+gradations are known to exist; cases of instinct of apparently such
+trifling importance, that they could hardly have been acted on by natural
+selection; cases of instincts almost identically the same in animals so
+remote in the scale of nature, that we cannot account for their similarity
+by inheritance from a common parent, and must therefore believe that they
+have been acquired by independent acts of natural selection.  I will not
+here enter on these several cases, but will confine myself to one special
+difficulty, which at first appeared to me insuperable, and actually fatal
+to my whole theory.  I allude to the neuters or sterile females in
+insect-communities:  for these neuters often differ widely in instinct and
+in structure from both the males and fertile females, and yet, from being
+sterile, they cannot propagate their kind.
+</para>
+
+<para>
+The subject well deserves to be discussed at great length, but I will here
+take only a single case, that of working or sterile ants.  How the workers
+have been rendered sterile is a difficulty; but not much greater than that
+of any other striking modification of structure; for it can be shown that
+some insects and other articulate animals in a state of nature occasionally
+become sterile; and if such insects had been social, and it had been
+profitable to the community that a number should have been annually born
+capable of work, but incapable of procreation, I can see no very great
+difficulty in this being effected by natural selection.  But I must pass
+over this preliminary difficulty.  The great difficulty lies in the working
+ants differing widely from both the males and the fertile females in
+structure, as in the shape of the thorax and in being destitute of wings
+and sometimes of eyes, and in instinct.  As far as instinct alone is
+concerned, the prodigious difference in this respect between the workers
+and the perfect females, would have been far better exemplified by the
+hive-bee.  If a working ant or other neuter insect had been an animal in
+the ordinary state, I should have unhesitatingly assumed that all its
+characters had been slowly acquired through natural selection; namely, by
+an individual having been born with some slight profitable modification of
+structure, this being inherited by its offspring, which again varied and
+were again selected, and so onwards.  But with the working ant we have an
+insect differing greatly from its parents, yet absolutely sterile; so that
+it could never have transmitted successively acquired modifications of
+structure or instinct to its progeny.  It may well be asked how is it
+possible to reconcile this case with the theory of natural selection?
+</para>
+
+<para>
+First, let it be remembered that we have innumerable instances, both in our
+domestic productions and in those in a state of nature, of all sorts of
+differences of structure which have become correlated to certain ages, and
+to either sex.  We have differences correlated not only to one sex, but to
+that short period alone when the reproductive system is active, as in the
+nuptial plumage of many birds, and in the hooked jaws of the male salmon. 
+We have even slight differences in the horns of different breeds of cattle
+in relation to an artificially imperfect state of the male sex; for oxen of
+certain breeds have longer horns than in other breeds, in comparison with
+the horns of the bulls or cows of these same breeds.  Hence I can see no
+real difficulty in any character having become correlated with the sterile
+condition of certain members of insect-communities:  the difficulty lies in
+understanding how such correlated modifications of structure could have
+been slowly accumulated by natural selection.
+</para>
+
+<para>
+This difficulty, though appearing insuperable, is lessened, or, as I
+believe, disappears, when it is remembered that selection may be applied to
+the family, as well as to the individual, and may thus gain the desired
+end.  Thus, a well-flavoured vegetable is cooked, and the individual is
+destroyed; but the horticulturist sows seeds of the same stock, and
+confidently expects to get nearly the same variety; breeders of cattle wish
+the flesh and fat to be well marbled together; the animal has been
+slaughtered, but the breeder goes with confidence to the same family.  I
+have such faith in the powers of selection, that I do not doubt that a
+breed of cattle, always yielding oxen with extraordinarily long horns,
+could be slowly formed by carefully watching which individual bulls and
+cows, when matched, produced oxen with the longest horns; and yet no one ox
+could ever have propagated its kind.  Thus I believe it has been with
+social insects:  a slight modification of structure, or instinct,
+correlated with the sterile condition of certain members of the community,
+has been advantageous to the community:  consequently the fertile males and
+females of the same community flourished, and transmitted to their fertile
+offspring a tendency to produce sterile members having the same
+modification.  And I believe that this process has been repeated, until
+that prodigious amount of difference between the fertile and sterile
+females of the same species has been produced, which we see in many social
+insects.
+</para>
+
+<para>
+But we have not as yet touched on the climax of the difficulty; namely, the
+fact that the neuters of several ants differ, not only from the fertile
+females and males, but from each other, sometimes to an almost incredible
+degree, and are thus divided into two or even three castes.  The castes,
+moreover, do not generally graduate into each other, but are perfectly well
+defined; being as distinct from each other, as are any two species of the
+same genus, or rather as any two genera of the same family.  Thus in
+Eciton, there are working and soldier neuters, with jaws and instincts
+extraordinarily different:  in Cryptocerus, the workers of one caste alone
+carry a wonderful sort of shield on their heads, the use of which is quite
+unknown:  in the Mexican Myrmecocystus, the workers of one caste never
+leave the nest; they are fed by the workers of another caste, and they have
+an enormously developed abdomen which secretes a sort of honey, supplying
+the place of that excreted by the aphides, or the domestic cattle as they
+may be called, which our European ants guard or imprison.
+</para>
+
+<para>
+It will indeed be thought that I have an overweening confidence in the
+principle of natural selection, when I do not admit that such wonderful and
+well-established facts at once annihilate my theory.  In the simpler case
+of neuter insects all of one caste or of the same kind, which have been
+rendered by natural selection, as I believe to be quite possible, different
+from the fertile males and females,--in this case, we may safely conclude
+from the analogy of ordinary variations, that each successive, slight,
+profitable modification did not probably at first appear in all the
+individual neuters in the same nest, but in a few alone; and that by the
+long-continued selection of the fertile parents which produced most neuters
+with the profitable modification, all the neuters ultimately came to have
+the desired character.  On this view we ought occasionally to find
+neuter-insects of the same species, in the same nest, presenting gradations
+of structure; and this we do find, even often, considering how few
+neuter-insects out of Europe have been carefully examined.  Mr. F. Smith
+has shown how surprisingly the neuters of several British ants differ from
+each other in size and sometimes in colour; and that the extreme forms can
+sometimes be perfectly linked together by individuals taken out of the same
+nest:  I have myself compared perfect gradations of this kind.  It often
+happens that the larger or the smaller sized workers are the most numerous;
+or that both large and small are numerous, with those of an intermediate
+size scanty in numbers.  Formica flava has larger and smaller workers, with
+some of intermediate size; and, in this species, as Mr. F. Smith has
+observed, the larger workers have simple eyes (ocelli), which though small
+can be plainly distinguished, whereas the smaller workers have their ocelli
+rudimentary.  Having carefully dissected several specimens of these
+workers, I can affirm that the eyes are far more rudimentary in the smaller
+workers than can be accounted for merely by their proportionally lesser
+size; and I fully believe, though I dare not assert so positively, that the
+workers of intermediate size have their ocelli in an exactly intermediate
+condition.  So that we here have two bodies of sterile workers in the same
+nest, differing not only in size, but in their organs of vision, yet
+connected by some few members in an intermediate condition.  I may digress
+by adding, that if the smaller workers had been the most useful to the
+community, and those males and females had been continually selected, which
+produced more and more of the smaller workers, until all the workers had
+come to be in this condition; we should then have had a species of ant with
+neuters very nearly in the same condition with those of Myrmica.  For the
+workers of Myrmica have not even rudiments of ocelli, though the male and
+female ants of this genus have well-developed ocelli.
+</para>
+
+<para>
+I may give one other case:  so confidently did I expect to find gradations
+in important points of structure between the different castes of neuters in
+the same species, that I gladly availed myself of Mr. F. Smith's offer of
+numerous specimens from the same nest of the driver ant (Anomma) of West
+Africa.  The reader will perhaps best appreciate the amount of difference
+in these workers, by my giving not the actual measurements, but a strictly
+accurate illustration:  the difference was the same as if we were to see a
+set of workmen building a house of whom many were five feet four inches
+high, and many sixteen feet high; but we must suppose that the larger
+workmen had heads four instead of three times as big as those of the
+smaller men, and jaws nearly five times as big.  The jaws, moreover, of the
+working ants of the several sizes differed wonderfully in shape, and in the
+form and number of the teeth.  But the important fact for us is, that
+though the workers can be grouped into castes of different sizes, yet they
+graduate insensibly into each other, as does the widely-different structure
+of their jaws.  I speak confidently on this latter point, as Mr. Lubbock
+made drawings for me with the camera lucida of the jaws which I had
+dissected from the workers of the several sizes.
+</para>
+
+<para>
+With these facts before me, I believe that natural selection, by acting on
+the fertile parents, could form a species which should regularly produce
+neuters, either all of large size with one form of jaw, or all of small
+size with jaws having a widely different structure; or lastly, and this is
+our climax of difficulty, one set of workers of one size and structure, and
+simultaneously another set of workers of a different size and structure;--a
+graduated series having been first formed, as in the case of the driver
+ant, and then the extreme forms, from being the most useful to the
+community, having been produced in greater and greater numbers through the
+natural selection of the parents which generated them; until none with an
+intermediate structure were produced.
+</para>
+
+<para>
+Thus, as I believe, the wonderful fact of two distinctly defined castes of
+sterile workers existing in the same nest, both widely different from each
+other and from their parents, has originated.  We can see how useful their
+production may have been to a social community of insects, on the same
+principle that the division of labour is useful to civilised man.  As ants
+work by inherited instincts and by inherited tools or weapons, and not by
+acquired knowledge and manufactured instruments, a perfect division of
+labour could be effected with them only by the workers being sterile; for
+had they been fertile, they would have intercrossed, and their instincts
+and structure would have become blended.  And nature has, as I believe,
+effected this admirable division of labour in the communities of ants, by
+the means of natural selection.  But I am bound to confess, that, with all
+my faith in this principle, I should never have anticipated that natural
+selection could have been efficient in so high a degree, had not the case
+of these neuter insects convinced me of the fact.  I have, therefore,
+discussed this case, at some little but wholly insufficient length, in
+order to show the power of natural selection, and likewise because this is
+by far the most serious special difficulty, which my theory has
+encountered.  The case, also, is very interesting, as it proves that with
+animals, as with plants, any amount of modification in structure can be
+effected by the accumulation of numerous, slight, and as we must call them
+accidental, variations, which are in any manner profitable, without
+exercise or habit having come into play.  For no amount of exercise, or
+habit, or volition, in the utterly sterile members of a community could
+possibly have affected the structure or instincts of the fertile members,
+which alone leave descendants.  I am surprised that no one has advanced
+this demonstrative case of neuter insects, against the well-known doctrine
+of Lamarck.
+</para>
+
+<para>
+Summary. -- I have endeavoured briefly in this chapter to show that the
+mental qualities of our domestic animals vary, and that the variations are
+inherited.  Still more briefly I have attempted to show that instincts vary
+slightly in a state of nature.  No one will dispute that instincts are of
+the highest importance to each animal.  Therefore I can see no difficulty,
+under changing conditions of life, in natural selection accumulating slight
+modifications of instinct to any extent, in any useful direction.  In some
+cases habit or use and disuse have probably come into play.  I do not
+pretend that the facts given in this chapter strengthen in any great degree
+my theory; but none of the cases of difficulty, to the best of my judgment,
+annihilate it.  On the other hand, the fact that instincts are not always
+absolutely perfect and are liable to mistakes;--that no instinct has been
+produced for the exclusive good of other animals, but that each animal
+takes advantage of the instincts of others;--that the canon in natural
+history, of 'natura non facit saltum' is applicable to instincts as well as
+to corporeal structure, and is plainly explicable on the foregoing views,
+but is otherwise inexplicable,--all tend to corroborate the theory of
+natural selection.
+</para>
+
+<para>
+This theory is, also, strengthened by some few other facts in regard to
+instincts; as by that common case of closely allied, but certainly
+distinct, species, when inhabiting distant parts of the world and living
+under considerably different conditions of life, yet often retaining nearly
+the same instincts.  For instance, we can understand on the principle of
+inheritance, how it is that the thrush of South America lines its nest with
+mud, in the same peculiar manner as does our British thrush:  how it is
+that the male wrens (Troglodytes) of North America, build 'cock-nests,' to
+roost in, like the males of our distinct Kitty-wrens,--a habit wholly
+unlike that of any other known bird.  Finally, it may not be a logical
+deduction, but to my imagination it is far more satisfactory to look at
+such instincts as the young cuckoo ejecting its foster-brothers,--ants
+making slaves,--the larvae of ichneumonidae feeding within the live bodies
+of caterpillars,--not as specially endowed or created instincts, but as
+small consequences of one general law, leading to the advancement of all
+organic beings, namely, multiply, vary, let the strongest live and the
+weakest die.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter VIII</chapnum>
+
+<title>Hybridism</title>
+
+<chapsummary>
+Distinction between the sterility of first crosses and of hybrids --
+Sterility various in degree, not universal, affected by close
+interbreeding, removed by domestication -- Laws governing the sterility of
+hybrids -- Sterility not a special endowment, but incidental on other
+differences -- Causes of the sterility of first crosses and of hybrids --
+Parallelism between the effects of changed conditions of life and crossing
+-- Fertility of varieties when crossed and of their mongrel offspring not
+universal -- Hybrids and mongrels compared independently of their fertility
+-- Summary.
+</chapsummary>
+</chapheader>
+
+<para>
+The view generally entertained by naturalists is that species, when
+intercrossed, have been specially endowed with the quality of sterility, in
+order to prevent the confusion of all organic forms.  This view certainly
+seems at first probable, for species within the same country could hardly
+have kept distinct had they been capable of crossing freely.  The
+importance of the fact that hybrids are very generally sterile, has, I
+think, been much underrated by some late writers.  On the theory of natural
+selection the case is especially important, inasmuch as the sterility of
+hybrids could not possibly be of any advantage to them, and therefore could
+not have been acquired by the continued preservation of successive
+profitable degrees of sterility.  I hope, however, to be able to show that
+sterility is not a specially acquired or endowed quality, but is incidental
+on other acquired differences.
+</para>
+
+<para>
+In treating this subject, two classes of facts, to a large extent
+fundamentally different, have generally been confounded together; namely,
+the sterility of two species when first crossed, and the sterility of the
+hybrids produced from them.
+</para>
+
+<para>
+Pure species have of course their organs of reproduction in a perfect
+condition, yet when intercrossed they produce either few or no offspring. 
+Hybrids, on the other hand, have their reproductive organs functionally
+impotent, as may be clearly seen in the state of the male element in both
+plants and animals; though the organs themselves are perfect in structure,
+as far as the microscope reveals.  In the first case the two sexual
+elements which go to form the embryo are perfect; in the second case they
+are either not at all developed, or are imperfectly developed.  This
+distinction is important, when the cause of the sterility, which is common
+to the two cases, has to be considered.  The distinction has probably been
+slurred over, owing to the sterility in both cases being looked on as a
+special endowment, beyond the province of our reasoning powers.
+</para>
+
+<para>
+The fertility of varieties, that is of the forms known or believed to have
+descended from common parents, when intercrossed, and likewise the
+fertility of their mongrel offspring, is, on my theory, of equal importance
+with the sterility of species; for it seems to make a broad and clear
+distinction between varieties and species.
+</para>
+
+<para>
+First, for the sterility of species when crossed and of their hybrid
+offspring.  It is impossible to study the several memoirs and works of
+those two conscientious and admirable observers, Kolreuter and Gartner, who
+almost devoted their lives to this subject, without being deeply impressed
+with the high generality of some degree of sterility.  Kolreuter makes the
+rule universal; but then he cuts the knot, for in ten cases in which he
+found two forms, considered by most authors as distinct species, quite
+fertile together, he unhesitatingly ranks them as varieties.  Gartner,
+also, makes the rule equally universal; and he disputes the entire
+fertility of Kolreuter's ten cases.  But in these and in many other cases,
+Gartner is obliged carefully to count the seeds, in order to show that
+there is any degree of sterility.  He always compares the maximum number of
+seeds produced by two species when crossed and by their hybrid offspring,
+with the average number produced by both pure parent-species in a state of
+nature.  But a serious cause of error seems to me to be here introduced:  a
+plant to be hybridised must be castrated, and, what is often more
+important, must be secluded in order to prevent pollen being brought to it
+by insects from other plants.  Nearly all the plants experimentised on by
+Gartner were potted, and apparently were kept in a chamber in his house. 
+That these processes are often injurious to the fertility of a plant cannot
+be doubted; for Gartner gives in his table about a score of cases of plants
+which he castrated, and artificially fertilised with their own pollen, and
+(excluding all cases such as the Leguminosae, in which there is an
+acknowledged difficulty in the manipulation) half of these twenty plants
+had their fertility in some degree impaired.  Moreover, as Gartner during
+several years repeatedly crossed the primrose and cowslip, which we have
+such good reason to believe to be varieties, and only once or twice
+succeeded in getting fertile seed; as he found the common red and blue
+pimpernels (Anagallis arvensis and coerulea), which the best botanists rank
+as varieties, absolutely sterile together; and as he came to the same
+conclusion in several other analogous cases; it seems to me that we may
+well be permitted to doubt whether many other species are really so
+sterile, when intercrossed, as Gartner believes.
+</para>
+
+<para>
+It is certain, on the one hand, that the sterility of various species when
+crossed is so different in degree and graduates away so insensibly, and, on
+the other hand, that the fertility of pure species is so easily affected by
+various circumstances, that for all practical purposes it is most difficult
+to say where perfect fertility ends and sterility begins.  I think no
+better evidence of this can be required than that the two most experienced
+observers who have ever lived, namely, Kolreuter and Gartner, should have
+arrived at diametrically opposite conclusions in regard to the very same
+species.  It is also most instructive to compare--but I have not space here
+to enter on details--the evidence advanced by our best botanists on the
+question whether certain doubtful forms should be ranked as species or
+varieties, with the evidence from fertility adduced by different
+hybridisers, or by the same author, from experiments made during different
+years.  It can thus be shown that neither sterility nor fertility affords
+any clear distinction between species and varieties; but that the evidence
+from this source graduates away, and is doubtful in the same degree as is
+the evidence derived from other constitutional and structural differences.
+</para>
+
+<para>
+In regard to the sterility of hybrids in successive generations; though
+Gartner was enabled to rear some hybrids, carefully guarding them from a
+cross with either pure parent, for six or seven, and in one case for ten
+generations, yet he asserts positively that their fertility never
+increased, but generally greatly decreased.  I do not doubt that this is
+usually the case, and that the fertility often suddenly decreases in the
+first few generations.  Nevertheless I believe that in all these
+experiments the fertility has been diminished by an independent cause,
+namely, from close interbreeding.  I have collected so large a body of
+facts, showing that close interbreeding lessens fertility, and, on the
+other hand, that an occasional cross with a distinct individual or variety
+increases fertility, that I cannot doubt the correctness of this almost
+universal belief amongst breeders.  Hybrids are seldom raised by
+experimentalists in great numbers; and as the parent-species, or other
+allied hybrids, generally grow in the same garden, the visits of insects
+must be carefully prevented during the flowering season:  hence hybrids
+will generally be fertilised during each generation by their own individual
+pollen; and I am convinced that this would be injurious to their fertility,
+already lessened by their hybrid origin.  I am strengthened in this
+conviction by a remarkable statement repeatedly made by Gartner, namely,
+that if even the less fertile hybrids be artificially fertilised with
+hybrid pollen of the same kind, their fertility, notwithstanding the
+frequent ill effects of manipulation, sometimes decidedly increases, and
+goes on increasing.  Now, in artificial fertilisation pollen is as often
+taken by chance (as I know from my own experience) from the anthers of
+another flower, as from the anthers of the flower itself which is to be
+fertilised; so that a cross between two flowers, though probably on the
+same plant, would be thus effected.  Moreover, whenever complicated
+experiments are in progress, so careful an observer as Gartner would have
+castrated his hybrids, and this would have insured in each generation a
+cross with the pollen from a distinct flower, either from the same plant or
+from another plant of the same hybrid nature.  And thus, the strange fact
+of the increase of fertility in the successive generations of artificially
+fertilised hybrids may, I believe, be accounted for by close interbreeding
+having been avoided.
+</para>
+
+<para>
+Now let us turn to the results arrived at by the third most experienced
+hybridiser, namely, the Hon. and Rev. W. Herbert.  He is as emphatic in his
+conclusion that some hybrids are perfectly fertile--as fertile as the pure
+parent-species--as are Kolreuter and Gartner that some degree of sterility
+between distinct species is a universal law of nature.  He experimentised
+on some of the very same species as did Gartner.  The difference in their
+results may, I think, be in part accounted for by Herbert's great
+horticultural skill, and by his having hothouses at his command.  Of his
+many important statements I will here give only a single one as an example,
+namely, that 'every ovule in a pod of Crinum capense fertilised by C.
+revolutum produced a plant, which (he says) I never saw to occur in a case
+of its natural fecundation.'  So that we here have perfect, or even more
+than commonly perfect, fertility in a first cross between two distinct
+species.
+</para>
+
+<para>
+This case of the Crinum leads me to refer to a most singular fact, namely,
+that there are individual plants, as with certain species of Lobelia, and
+with all the species of the genus Hippeastrum, which can be far more easily
+fertilised by the pollen of another and distinct species, than by their own
+pollen.  For these plants have been found to yield seed to the pollen of a
+distinct species, though quite sterile with their own pollen,
+notwithstanding that their own pollen was found to be perfectly good, for
+it fertilised distinct species.  So that certain individual plants and all
+the individuals of certain species can actually be hybridised much more
+readily than they can be self-fertilised!  For instance, a bulb of
+Hippeastrum aulicum produced four flowers; three were fertilised by Herbert
+with their own pollen, and the fourth was subsequently fertilised by the
+pollen of a compound hybrid descended from three other and distinct
+species:  the result was that 'the ovaries of the three first flowers soon
+ceased to grow, and after a few days perished entirely, whereas the pod
+impregnated by the pollen of the hybrid made vigorous growth and rapid
+progress to maturity, and bore good seed, which vegetated freely.'  In a
+letter to me, in 1839, Mr. Herbert told me that he had then tried the
+experiment during five years, and he continued to try it during several
+subsequent years, and always with the same result.  This result has, also,
+been confirmed by other observers in the case of Hippeastrum with its
+sub-genera, and in the case of some other genera, as Lobelia, Passiflora
+and Verbascum.  Although the plants in these experiments appeared perfectly
+healthy, and although both the ovules and pollen of the same flower were
+perfectly good with respect to other species, yet as they were functionally
+imperfect in their mutual self-action, we must infer that the plants were
+in an unnatural state.  Nevertheless these facts show on what slight and
+mysterious causes the lesser or greater fertility of species when crossed,
+in comparison with the same species when self-fertilised, sometimes
+depends.
+</para>
+
+<para>
+The practical experiments of horticulturists, though not made with
+scientific precision, deserve some notice.  It is notorious in how
+complicated a manner the species of Pelargonium, Fuchsia, Calceolaria,
+Petunia, Rhododendron, &amp;c., have been crossed, yet many of these hybrids
+seed freely.  For instance, Herbert asserts that a hybrid from Calceolaria
+integrifolia and plantaginea, species most widely dissimilar in general
+habit, 'reproduced itself as perfectly as if it had been a natural species
+from the mountains of Chile.'  I have taken some pains to ascertain the
+degree of fertility of some of the complex crosses of Rhododendrons, and I
+am assured that many of them are perfectly fertile.  Mr. C. Noble, for
+instance, informs me that he raises stocks for grafting from a hybrid
+between Rhod. Ponticum and Catawbiense, and that this hybrid 'seeds as
+freely as it is possible to imagine.'  Had hybrids, when fairly treated,
+gone on decreasing in fertility in each successive generation, as Gartner
+believes to be the case, the fact would have been notorious to nurserymen. 
+Horticulturists raise large beds of the same hybrids, and such alone are
+fairly treated, for by insect agency the several individuals of the same
+hybrid variety are allowed to freely cross with each other, and the
+injurious influence of close interbreeding is thus prevented.  Any one may
+readily convince himself of the efficiency of insect-agency by examining
+the flowers of the more sterile kinds of hybrid rhododendrons, which
+produce no pollen, for he will find on their stigmas plenty of pollen
+brought from other flowers.
+</para>
+
+<para>
+In regard to animals, much fewer experiments have been carefully tried than
+with plants.  If our systematic arrangements can be trusted, that is if the
+genera of animals are as distinct from each other, as are the genera of
+plants, then we may infer that animals more widely separated in the scale
+of nature can be more easily crossed than in the case of plants; but the
+hybrids themselves are, I think, more sterile.  I doubt whether any case of
+a perfectly fertile hybrid animal can be considered as thoroughly well
+authenticated.  It should, however, be borne in mind that, owing to few
+animals breeding freely under confinement, few experiments have been fairly
+tried:  for instance, the canary-bird has been crossed with nine other
+finches, but as not one of these nine species breeds freely in confinement,
+we have no right to expect that the first crosses between them and the
+canary, or that their hybrids, should be perfectly fertile.  Again, with
+respect to the fertility in successive generations of the more fertile
+hybrid animals, I hardly know of an instance in which two families of the
+same hybrid have been raised at the same time from different parents, so as
+to avoid the ill effects of close interbreeding.  On the contrary, brothers
+and sisters have usually been crossed in each successive generation, in
+opposition to the constantly repeated admonition of every breeder.  And in
+this case, it is not at all surprising that the inherent sterility in the
+hybrids should have gone on increasing.  If we were to act thus, and pair
+brothers and sisters in the case of any pure animal, which from any cause
+had the least tendency to sterility, the breed would assuredly be lost in a
+very few generations.
+</para>
+
+<para>
+Although I do not know of any thoroughly well-authenticated cases of
+perfectly fertile hybrid animals, I have some reason to believe that the
+hybrids from Cervulus vaginalis and Reevesii, and from Phasianus colchicus
+with P. torquatus and with P. versicolor are perfectly fertile.  The
+hybrids from the common and Chinese geese (A. cygnoides), species which are
+so different that they are generally ranked in distinct genera, have often
+bred in this country with either pure parent, and in one single instance
+they have bred inter se.  This was effected by Mr. Eyton, who raised two
+hybrids from the same parents but from different hatches; and from these
+two birds he raised no less than eight hybrids (grandchildren of the pure
+geese) from one nest.  In India, however, these cross-bred geese must be
+far more fertile; for I am assured by two eminently capable judges, namely
+Mr. Blyth and Capt. Hutton, that whole flocks of these crossed geese are
+kept in various parts of the country; and as they are kept for profit,
+where neither pure parent-species exists, they must certainly be highly
+fertile.
+</para>
+
+<para>
+A doctrine which originated with Pallas, has been largely accepted by
+modern naturalists; namely, that most of our domestic animals have
+descended from two or more aboriginal species, since commingled by
+intercrossing.  On this view, the aboriginal species must either at first
+have produced quite fertile hybrids, or the hybrids must have become in
+subsequent generations quite fertile under domestication.  This latter
+alternative seems to me the most probable, and I am inclined to believe in
+its truth, although it rests on no direct evidence.  I believe, for
+instance, that our dogs have descended from several wild stocks; yet, with
+perhaps the exception of certain indigenous domestic dogs of South America,
+all are quite fertile together; and analogy makes me greatly doubt, whether
+the several aboriginal species would at first have freely bred together and
+have produced quite fertile hybrids.  So again there is reason to believe
+that our European and the humped Indian cattle are quite fertile together;
+but from facts communicated to me by Mr. Blyth, I think they must be
+considered as distinct species.  On this view of the origin of many of our
+domestic animals, we must either give up the belief of the almost universal
+sterility of distinct species of animals when crossed; or we must look at
+sterility, not as an indelible characteristic, but as one capable of being
+removed by domestication.
+</para>
+
+<para>
+Finally, looking to all the ascertained facts on the intercrossing of
+plants and animals, it may be concluded that some degree of sterility, both
+in first crosses and in hybrids, is an extremely general result; but that
+it cannot, under our present state of knowledge, be considered as
+absolutely universal.
+</para>
+
+<para>
+Laws governing the Sterility of first Crosses and of Hybrids. -- We will
+now consider a little more in detail the circumstances and rules governing
+the sterility of first crosses and of hybrids.  Our chief object will be to
+see whether or not the rules indicate that species have specially been
+endowed with this quality, in order to prevent their crossing and blending
+together in utter confusion.  The following rules and conclusions are
+chiefly drawn up from Gartner's admirable work on the hybridisation of
+plants.  I have taken much pains to ascertain how far the rules apply to
+animals, and considering how scanty our knowledge is in regard to hybrid
+animals, I have been surprised to find how generally the same rules apply
+to both kingdoms.
+</para>
+
+<para>
+It has been already remarked, that the degree of fertility, both of first
+crosses and of hybrids, graduates from zero to perfect fertility.  It is
+surprising in how many curious ways this gradation can be shown to exist;
+but only the barest outline of the facts can here be given.  When pollen
+from a plant of one family is placed on the stigma of a plant of a distinct
+family, it exerts no more influence than so much inorganic dust.  From this
+absolute zero of fertility, the pollen of different species of the same
+genus applied to the stigma of some one species, yields a perfect gradation
+in the number of seeds produced, up to nearly complete or even quite
+complete fertility; and, as we have seen, in certain abnormal cases, even
+to an excess of fertility, beyond that which the plant's own pollen will
+produce.  So in hybrids themselves, there are some which never have
+produced, and probably never would produce, even with the pollen of either
+pure parent, a single fertile seed:  but in some of these cases a first
+trace of fertility may be detected, by the pollen of one of the pure
+parent-species causing the flower of the hybrid to wither earlier than it
+otherwise would have done; and the early withering of the flower is well
+known to be a sign of incipient fertilisation.  From this extreme degree of
+sterility we have self-fertilised hybrids producing a greater and greater
+number of seeds up to perfect fertility.
+</para>
+
+<para>
+Hybrids from two species which are very difficult to cross, and which
+rarely produce any offspring, are generally very sterile; but the
+parallelism between the difficulty of making a first cross, and the
+sterility of the hybrids thus produced--two classes of facts which are
+generally confounded together--is by no means strict.  There are many
+cases, in which two pure species can be united with unusual facility, and
+produce numerous hybrid-offspring, yet these hybrids are remarkably
+sterile.  On the other hand, there are species which can be crossed very
+rarely, or with extreme difficulty, but the hybrids, when at last produced,
+are very fertile.  Even within the limits of the same genus, for instance
+in Dianthus, these two opposite cases occur.
+</para>
+
+<para>
+The fertility, both of first crosses and of hybrids, is more easily
+affected by unfavourable conditions, than is the fertility of pure species. 
+But the degree of fertility is likewise innately variable; for it is not
+always the same when the same two species are crossed under the same
+circumstances, but depends in part upon the constitution of the individuals
+which happen to have been chosen for the experiment.  So it is with
+hybrids, for their degree of fertility is often found to differ greatly in
+the several individuals raised from seed out of the same capsule and
+exposed to exactly the same conditions.
+</para>
+
+<para>
+By the term systematic affinity is meant, the resemblance between species
+in structure and in constitution, more especially in the structure of parts
+which are of high physiological importance and which differ little in the
+allied species.  Now the fertility of first crosses between species, and of
+the hybrids produced from them, is largely governed by their systematic
+affinity.  This is clearly shown by hybrids never having been raised
+between species ranked by systematists in distinct families; and on the
+other hand, by very closely allied species generally uniting with facility. 
+But the correspondence between systematic affinity and the facility of
+crossing is by no means strict.  A multitude of cases could be given of
+very closely allied species which will not unite, or only with extreme
+difficulty; and on the other hand of very distinct species which unite with
+the utmost facility.  In the same family there may be a genus, as Dianthus,
+in which very many species can most readily be crossed; and another genus,
+as Silene, in which the most persevering efforts have failed to produce
+between extremely close species a single hybrid.  Even within the limits of
+the same genus, we meet with this same difference; for instance, the many
+species of Nicotiana have been more largely crossed than the species of
+almost any other genus; but Gartner found that N. acuminata, which is not a
+particularly distinct species, obstinately failed to fertilise, or to be
+fertilised by, no less than eight other species of Nicotiana.  Very many
+analogous facts could be given.
+</para>
+
+<para>
+No one has been able to point out what kind, or what amount, of difference
+in any recognisable character is sufficient to prevent two species
+crossing.  It can be shown that plants most widely different in habit and
+general appearance, and having strongly marked differences in every part of
+the flower, even in the pollen, in the fruit, and in the cotyledons, can be
+crossed.  Annual and perennial plants, deciduous and evergreen trees,
+plants inhabiting different stations and fitted for extremely different
+climates, can often be crossed with ease.
+</para>
+
+<para>
+By a reciprocal cross between two species, I mean the case, for instance,
+of a stallion-horse being first crossed with a female-ass, and then a
+male-ass with a mare:  these two species may then be said to have been
+reciprocally crossed.  There is often the widest possible difference in the
+facility of making reciprocal crosses.  Such cases are highly important,
+for they prove that the capacity in any two species to cross is often
+completely independent of their systematic affinity, or of any recognisable
+difference in their whole organisation.  On the other hand, these cases
+clearly show that the capacity for crossing is connected with
+constitutional differences imperceptible by us, and confined to the
+reproductive system.  This difference in the result of reciprocal crosses
+between the same two species was long ago observed by Kolreuter.  To give
+an instance:  Mirabilis jalappa can easily be fertilised by the pollen of
+M. longiflora, and the hybrids thus produced are sufficiently fertile; but
+Kolreuter tried more than two hundred times, during eight following years,
+to fertilise reciprocally M. longiflora with the pollen of M. jalappa, and
+utterly failed.  Several other equally striking cases could be given. 
+Thuret has observed the same fact with certain sea-weeds or Fuci.  Gartner,
+moreover, found that this difference of facility in making reciprocal
+crosses is extremely common in a lesser degree.  He has observed it even
+between forms so closely related (as Matthiola annua and glabra) that many
+botanists rank them only as varieties.  It is also a remarkable fact, that
+hybrids raised from reciprocal crosses, though of course compounded of the
+very same two species, the one species having first been used as the father
+and then as the mother, generally differ in fertility in a small, and
+occasionally in a high degree.
+</para>
+
+<para>
+Several other singular rules could be given from Gartner:  for instance,
+some species have a remarkable power of crossing with other species; other
+species of the same genus have a remarkable power of impressing their
+likeness on their hybrid offspring; but these two powers do not at all
+necessarily go together.  There are certain hybrids which instead of
+having, as is usual, an intermediate character between their two parents,
+always closely resemble one of them; and such hybrids, though externally so
+like one of their pure parent-species, are with rare exceptions extremely
+sterile.  So again amongst hybrids which are usually intermediate in
+structure between their parents, exceptional and abnormal individuals
+sometimes are born, which closely resemble one of their pure parents; and
+these hybrids are almost always utterly sterile, even when the other
+hybrids raised from seed from the same capsule have a considerable degree
+of fertility.  These facts show how completely fertility in the hybrid is
+independent of its external resemblance to either pure parent.
+</para>
+
+<para>
+Considering the several rules now given, which govern the fertility of
+first crosses and of hybrids, we see that when forms, which must be
+considered as good and distinct species, are united, their fertility
+graduates from zero to perfect fertility, or even to fertility under
+certain conditions in excess.  That their fertility, besides being
+eminently susceptible to favourable and unfavourable conditions, is
+innately variable.  That it is by no means always the same in degree in the
+first cross and in the hybrids produced from this cross.  That the
+fertility of hybrids is not related to the degree in which they resemble in
+external appearance either parent.  And lastly, that the facility of making
+a first cross between any two species is not always governed by their
+systematic affinity or degree of resemblance to each other.  This latter
+statement is clearly proved by reciprocal crosses between the same two
+species, for according as the one species or the other is used as the
+father or the mother, there is generally some difference, and occasionally
+the widest possible difference, in the facility of effecting an union.  The
+hybrids, moreover, produced from reciprocal crosses often differ in
+fertility.
+</para>
+
+<para>
+Now do these complex and singular rules indicate that species have been
+endowed with sterility simply to prevent their becoming confounded in
+nature?  I think not.  For why should the sterility be so extremely
+different in degree, when various species are crossed, all of which we must
+suppose it would be equally important to keep from blending together?  Why
+should the degree of sterility be innately variable in the individuals of
+the same species?  Why should some species cross with facility, and yet
+produce very sterile hybrids; and other species cross with extreme
+difficulty, and yet produce fairly fertile hybrids?  Why should there often
+be so great a difference in the result of a reciprocal cross between the
+same two species?  Why, it may even be asked, has the production of hybrids
+been permitted? to grant to species the special power of producing hybrids,
+and then to stop their further propagation by different degrees of
+sterility, not strictly related to the facility of the first union between
+their parents, seems to be a strange arrangement.
+</para>
+
+<para>
+The foregoing rules and facts, on the other hand, appear to me clearly to
+indicate that the sterility both of first crosses and of hybrids is simply
+incidental or dependent on unknown differences, chiefly in the reproductive
+systems, of the species which are crossed.  The differences being of so
+peculiar and limited a nature, that, in reciprocal crosses between two
+species the male sexual element of the one will often freely act on the
+female sexual element of the other, but not in a reversed direction.  It
+will be advisable to explain a little more fully by an example what I mean
+by sterility being incidental on other differences, and not a specially
+endowed quality.  As the capacity of one plant to be grafted or budded on
+another is so entirely unimportant for its welfare in a state of nature, I
+presume that no one will suppose that this capacity is a specially endowed
+quality, but will admit that it is incidental on differences in the laws of
+growth of the two plants.  We can sometimes see the reason why one tree
+will not take on another, from differences in their rate of growth, in the
+hardness of their wood, in the period of the flow or nature of their sap,
+&amp;c.; but in a multitude of cases we can assign no reason whatever.  Great
+diversity in the size of two plants, one being woody and the other
+herbaceous, one being evergreen and the other deciduous, and adaptation to
+widely different climates, does not always prevent the two grafting
+together.  As in hybridisation, so with grafting, the capacity is limited
+by systematic affinity, for no one has been able to graft trees together
+belonging to quite distinct families; and, on the other hand, closely
+allied species, and varieties of the same species, can usually, but not
+invariably, be grafted with ease.  But this capacity, as in hybridisation,
+is by no means absolutely governed by systematic affinity.  Although many
+distinct genera within the same family have been grafted together, in other
+cases species of the same genus will not take on each other.  The pear can
+be grafted far more readily on the quince, which is ranked as a distinct
+genus, than on the apple, which is a member of the same genus.  Even
+different varieties of the pear take with different degrees of facility on
+the quince; so do different varieties of the apricot and peach on certain
+varieties of the plum.
+</para>
+
+<para>
+As Gartner found that there was sometimes an innate difference in different
+individuals of the same two species in crossing; so Sagaret believes this
+to be the case with different individuals of the same two species in being
+grafted together.  As in reciprocal crosses, the facility of effecting an
+union is often very far from equal, so it sometimes is in grafting; the
+common gooseberry, for instance, cannot be grafted on the currant, whereas
+the currant will take, though with difficulty, on the gooseberry.
+</para>
+
+<para>
+We have seen that the sterility of hybrids, which have their reproductive
+organs in an imperfect condition, is a very different case from the
+difficulty of uniting two pure species, which have their reproductive
+organs perfect; yet these two distinct cases run to a certain extent
+parallel.  Something analogous occurs in grafting; for Thouin found that
+three species of Robinia, which seeded freely on their own roots, and which
+could be grafted with no great difficulty on another species, when thus
+grafted were rendered barren.  On the other hand, certain species of
+Sorbus, when grafted on other species, yielded twice as much fruit as when
+on their own roots.  We are reminded by this latter fact of the
+extraordinary case of Hippeastrum, Lobelia, &amp;c., which seeded much more
+freely when fertilised with the pollen of distinct species, than when
+self-fertilised with their own pollen.
+</para>
+
+<para>
+We thus see, that although there is a clear and fundamental difference
+between the mere adhesion of grafted stocks, and the union of the male and
+female elements in the act of reproduction, yet that there is a rude degree
+of parallelism in the results of grafting and of crossing distinct species. 
+And as we must look at the curious and complex laws governing the facility
+with which trees can be grafted on each other as incidental on unknown
+differences in their vegetative systems, so I believe that the still more
+complex laws governing the facility of first crosses, are incidental on
+unknown differences, chiefly in their reproductive systems.  These
+differences, in both cases, follow to a certain extent, as might have been
+expected, systematic affinity, by which every kind of resemblance and
+dissimilarity between organic beings is attempted to be expressed.  The
+facts by no means seem to me to indicate that the greater or lesser
+difficulty of either grafting or crossing together various species has been
+a special endowment; although in the case of crossing, the difficulty is as
+important for the endurance and stability of specific forms, as in the case
+of grafting it is unimportant for their welfare.
+</para>
+
+<para>
+Causes of the Sterility of first Crosses and of Hybrids. -- We may now look
+a little closer at the probable causes of the sterility of first crosses
+and of hybrids.  These two cases are fundamentally different, for, as just
+remarked, in the union of two pure species the male and female sexual
+elements are perfect, whereas in hybrids they are imperfect.  Even in first
+crosses, the greater or lesser difficulty in effecting a union apparently
+depends on several distinct causes.  There must sometimes be a physical
+impossibility in the male element reaching the ovule, as would be the case
+with a plant having a pistil too long for the pollen-tubes to reach the
+ovarium.  It has also been observed that when pollen of one species is
+placed on the stigma of a distantly allied species, though the pollen-tubes
+protrude, they do not penetrate the stigmatic surface.  Again, the male
+element may reach the female element, but be incapable of causing an embryo
+to be developed, as seems to have been the case with some of Thuret's
+experiments on Fuci.  No explanation can be given of these facts, any more
+than why certain trees cannot be grafted on others.  Lastly, an embryo may
+be developed, and then perish at an early period.  This latter alternative
+has not been sufficiently attended to; but I believe, from observations
+communicated to me by Mr. Hewitt, who has had great experience in
+hybridising gallinaceous birds, that the early death of the embryo is a
+very frequent cause of sterility in first crosses.  I was at first very
+unwilling to believe in this view; as hybrids, when once born, are
+generally healthy and long-lived, as we see in the case of the common mule. 
+Hybrids, however, are differently circumstanced before and after birth: 
+when born and living in a country where their two parents can live, they
+are generally placed under suitable conditions of life.  But a hybrid
+partakes of only half of the nature and constitution of its mother, and
+therefore before birth, as long as it is nourished within its mother's womb
+or within the egg or seed produced by the mother, it may be exposed to
+conditions in some degree unsuitable, and consequently be liable to perish
+at an early period; more especially as all very young beings seem eminently
+sensitive to injurious or unnatural conditions of life.
+</para>
+
+<para>
+In regard to the sterility of hybrids, in which the sexual elements are
+imperfectly developed, the case is very different.  I have more than once
+alluded to a large body of facts, which I have collected, showing that when
+animals and plants are removed from their natural conditions, they are
+extremely liable to have their reproductive systems seriously affected. 
+This, in fact, is the great bar to the domestication of animals.  Between
+the sterility thus superinduced and that of hybrids, there are many points
+of similarity.  In both cases the sterility is independent of general
+health, and is often accompanied by excess of size or great luxuriance.  In
+both cases, the sterility occurs in various degrees; in both, the male
+element is the most liable to be affected; but sometimes the female more
+than the male.  In both, the tendency goes to a certain extent with
+systematic affinity, or whole groups of animals and plants are rendered
+impotent by the same unnatural conditions; and whole groups of species tend
+to produce sterile hybrids.  On the other hand, one species in a group will
+sometimes resist great changes of conditions with unimpaired fertility; and
+certain species in a group will produce unusually fertile hybrids.  No one
+can tell, till he tries, whether any particular animal will breed under
+confinement or any plant seed freely under culture; nor can he tell, till
+he tries, whether any two species of a genus will produce more or less
+sterile hybrids.  Lastly, when organic beings are placed during several
+generations under conditions not natural to them, they are extremely liable
+to vary, which is due, as I believe, to their reproductive systems having
+been specially affected, though in a lesser degree than when sterility
+ensues.  So it is with hybrids, for hybrids in successive generations are
+eminently liable to vary, as every experimentalist has observed.
+</para>
+
+<para>
+Thus we see that when organic beings are placed under new and unnatural
+conditions, and when hybrids are produced by the unnatural crossing of two
+species, the reproductive system, independently of the general state of
+health, is affected by sterility in a very similar manner.  In the one
+case, the conditions of life have been disturbed, though often in so slight
+a degree as to be inappreciable by us; in the other case, or that of
+hybrids,the external conditions have remained the same, but the
+organisation has been disturbed by two different structures and
+constitutions having been blended into one.  For it is scarcely possible
+that two organisations should be compounded into one, without some
+disturbance occurring in the development, or periodical action, or mutual
+relation of the different parts and organs one to another, or to the
+conditions of life.  When hybrids are able to breed inter se, they transmit
+to their offspring from generation to generation the same compounded
+organisation, and hence we need not be surprised that their sterility,
+though in some degree variable, rarely diminishes.
+</para>
+
+<para>
+It must, however, be confessed that we cannot understand, excepting on
+vague hypotheses, several facts with respect to the sterility of hybrids;
+for instance, the unequal fertility of hybrids produced from reciprocal
+crosses; or the increased sterility in those hybrids which occasionally and
+exceptionally resemble closely either pure parent.  Nor do I pretend that
+the foregoing remarks go to the root of the matter:  no explanation is
+offered why an organism, when placed under unnatural conditions, is
+rendered sterile.  All that I have attempted to show, is that in two cases,
+in some respects allied, sterility is the common result,--in the one case
+from the conditions of life having been disturbed, in the other case from
+the organisation having been disturbed by two organisations having been
+compounded into one.
+</para>
+
+<para>
+It may seem fanciful, but I suspect that a similar parallelism extends to
+an allied yet very different class of facts.  It is an old and almost
+universal belief, founded, I think, on a considerable body of evidence,
+that slight changes in the conditions of life are beneficial to all living
+things.  We see this acted on by farmers and gardeners in their frequent
+exchanges of seed, tubers, &amp;c., from one soil or climate to another, and
+back again.  During the convalescence of animals, we plainly see that great
+benefit is derived from almost any change in the habits of life.  Again,
+both with plants and animals, there is abundant evidence, that a cross
+between very distinct individuals of the same species, that is between
+members of different strains or sub-breeds, gives vigour and fertility to
+the offspring.  I believe, indeed, from the facts alluded to in our fourth
+chapter, that a certain amount of crossing is indispensable even with
+hermaphrodites; and that close interbreeding continued during several
+generations between the nearest relations, especially if these be kept
+under the same conditions of life, always induces weakness and sterility in
+the progeny.
+</para>
+
+<para>
+Hence it seems that, on the one hand, slight changes in the conditions of
+life benefit all organic beings, and on the other hand, that slight
+crosses, that is crosses between the males and females of the same species
+which have varied and become slightly different, give vigour and fertility
+to the offspring.  But we have seen that greater changes, or changes of a
+particular nature, often render organic beings in some degree sterile; and
+that greater crosses, that is crosses between males and females which have
+become widely or specifically different, produce hybrids which are
+generally sterile in some degree.  I cannot persuade myself that this
+parallelism is an accident or an illusion.  Both series of facts seem to be
+connected together by some common but unknown bond, which is essentially
+related to the principle of life.
+</para>
+
+<para>
+Fertility of Varieties when crossed, and of their Mongrel off-spring. -- It
+may be urged, as a most forcible argument, that there must be some
+essential distinction between species and varieties, and that there must be
+some error in all the foregoing remarks, inasmuch as varieties, however
+much they may differ from each other in external appearance, cross with
+perfect facility, and yield perfectly fertile offspring.  I fully admit
+that this is almost invariably the case.  But if we look to varieties
+produced under nature, we are immediately involved in hopeless
+difficulties; for if two hitherto reputed varieties be found in any degree
+sterile together, they are at once ranked by most naturalists as species. 
+For instance, the blue and red pimpernel, the primrose and cowslip, which
+are considered by many of our best botanists as varieties, are said by
+Gartner not to be quite fertile when crossed, and he consequently ranks
+them as undoubted species.  If we thus argue in a circle, the fertility of
+all varieties produced under nature will assuredly have to be granted.
+</para>
+
+<para>
+If we turn to varieties, produced, or supposed to have been produced, under
+domestication, we are still involved in doubt.  For when it is stated, for
+instance, that the German Spitz dog unites more easily than other dogs with
+foxes, or that certain South American indigenous domestic dogs do not
+readily cross with European dogs, the explanation which will occur to
+everyone, and probably the true one, is that these dogs have descended from
+several aboriginally distinct species.  Nevertheless the perfect fertility
+of so many domestic varieties, differing widely from each other in
+appearance, for instance of the pigeon or of the cabbage, is a remarkable
+fact; more especially when we reflect how many species there are, which,
+though resembling each other most closely, are utterly sterile when
+intercrossed.  Several considerations, however, render the fertility of
+domestic varieties less remarkable than at first appears.  It can, in the
+first place, be clearly shown that mere external dissimilarity between two
+species does not determine their greater or lesser degree of sterility when
+crossed; and we may apply the same rule to domestic varieties.  In the
+second place, some eminent naturalists believe that a long course of
+domestication tends to eliminate sterility in the successive generations of
+hybrids, which were at first only slightly sterile; and if this be so, we
+surely ought not to expect to find sterility both appearing and
+disappearing under nearly the same conditions of life.  Lastly, and this
+seems to me by far the most important consideration, new races of animals
+and plants are produced under domestication by man's methodical and
+unconscious power of selection, for his own use and pleasure:  he neither
+wishes to select, nor could select, slight differences in the reproductive
+system, or other constitutional differences correlated with the
+reproductive system.  He supplies his several varieties with the same food;
+treats them in nearly the same manner, and does not wish to alter their
+general habits of life.  Nature acts uniformly and slowly during vast
+periods of time on the whole organisation, in any way which may be for each
+creature's own good; and thus she may, either directly, or more probably
+indirectly, through correlation, modify the reproductive system in the
+several descendants from any one species.  Seeing this difference in the
+process of selection, as carried on by man and nature, we need not be
+surprised at some difference in the result.
+</para>
+
+<para>
+I have as yet spoken as if the varieties of the same species were
+invariably fertile when intercrossed.  But it seems to me impossible to
+resist the evidence of the existence of a certain amount of sterility in
+the few following cases, which I will briefly abstract.  The evidence is at
+least as good as that from which we believe in the sterility of a multitude
+of species.  The evidence is, also, derived from hostile witnesses, who in
+all other cases consider fertility and sterility as safe criterions of
+specific distinction.  Gartner kept during several years a dwarf kind of
+maize with yellow seeds, and a tall variety with red seeds, growing near
+each other in his garden; and although these plants have separated sexes,
+they never naturally crossed.  He then fertilised thirteen flowers of the
+one with the pollen of the other; but only a single head produced any seed,
+and this one head produced only five grains.  Manipulation in this case
+could not have been injurious, as the plants have separated sexes.  No one,
+I believe, has suspected that these varieties of maize are distinct
+species; and it is important to notice that the hybrid plants thus raised
+were themselves perfectly fertile; so that even Gartner did not venture to
+consider the two varieties as specifically distinct.
+</para>
+
+<para>
+Girou de Buzareingues crossed three varieties of gourd, which like the
+maize has separated sexes, and he asserts that their mutual fertilisation
+is by so much the less easy as their differences are greater.  How far
+these experiments may be trusted, I know not; but the forms experimentised
+on, are ranked by Sagaret, who mainly founds his classification by the test
+of infertility, as varieties.
+</para>
+
+<para>
+The following case is far more remarkable, and seems at first quite
+incredible; but it is the result of an astonishing number of experiments
+made during many years on nine species of Verbascum, by so good an observer
+and so hostile a witness, as Gartner:  namely, that yellow and white
+varieties of the same species of Verbascum when intercrossed produce less
+seed, than do either coloured varieties when fertilised with pollen from
+their own coloured flowers.  Moreover, he asserts that when yellow and
+white varieties of one species are crossed with yellow and white varieties
+of a distinct species, more seed is produced by the crosses between the
+same coloured flowers, than between those which are differently coloured. 
+Yet these varieties of Verbascum present no other difference besides the
+mere colour of the flower; and one variety can sometimes be raised from the
+seed of the other.
+</para>
+
+<para>
+From observations which I have made on certain varieties of hollyhock, I am
+inclined to suspect that they present analogous facts.
+</para>
+
+<para>
+Kolreuter, whose accuracy has been confirmed by every subsequent observer,
+has proved the remarkable fact, that one variety of the common tobacco is
+more fertile, when crossed with a widely distinct species, than are the
+other varieties.  He experimentised on five forms, which are commonly
+reputed to be varieties, and which he tested by the severest trial, namely,
+by reciprocal crosses, and he found their mongrel offspring perfectly
+fertile.  But one of these five varieties, when used either as father or
+mother, and crossed with the Nicotiana glutinosa, always yielded hybrids
+not so sterile as those which were produced from the four other varieties
+when crossed with N. glutinosa.  Hence the reproductive system of this one
+variety must have been in some manner and in some degree modified.
+</para>
+
+<para>
+From these facts; from the great difficulty of ascertaining the infertility
+of varieties in a state of nature, for a supposed variety if infertile in
+any degree would generally be ranked as species; from man selecting only
+external characters in the production of the most distinct domestic
+varieties, and from not wishing or being able to produce recondite and
+functional differences in the reproductive system; from these several
+considerations and facts, I do not think that the very general fertility of
+varieties can be proved to be of universal occurrence, or to form a
+fundamental distinction between varieties and species.  The general
+fertility of varieties does not seem to me sufficient to overthrow the view
+which I have taken with respect to the very general, but not invariable,
+sterility of first crosses and of hybrids, namely, that it is not a special
+endowment, but is incidental on slowly acquired modifications, more
+especially in the reproductive systems of the forms which are crossed.
+</para>
+
+<para>
+Hybrids and Mongrels compared, independently of their fertility. --
+Independently of the question of fertility, the offspring of species when
+crossed and of varieties when crossed may be compared in several other
+respects.  Gartner, whose strong wish was to draw a marked line of
+distinction between species and varieties, could find very few and, as it
+seems to me, quite unimportant differences between the so-called hybrid
+offspring of species, and the so-called mongrel offspring of varieties. 
+And, on the other hand, they agree most closely in very many important
+respects.
+</para>
+
+<para>
+I shall here discuss this subject with extreme brevity.  The most important
+distinction is, that in the first generation mongrels are more variable
+than hybrids; but Gartner admits that hybrids from species which have long
+been cultivated are often variable in the first generation; and I have
+myself seen striking instances of this fact.  Gartner further admits that
+hybrids between very closely allied species are more variable than those
+from very distinct species; and this shows that the difference in the
+degree of variability graduates away.  When mongrels and the more fertile
+hybrids are propagated for several generations an extreme amount of
+variability in their offspring is notorious; but some few cases both of
+hybrids and mongrels long retaining uniformity of character could be given. 
+The variability, however, in the successive generations of mongrels is,
+perhaps, greater than in hybrids.
+</para>
+
+<para>
+This greater variability of mongrels than of hybrids does not seem to me at
+all surprising.  For the parents of mongrels are varieties, and mostly
+domestic varieties (very few experiments having been tried on natural
+varieties), and this implies in most cases that there has been recent
+variability; and therefore we might expect that such variability would
+often continue and be super-added to that arising from the mere act of
+crossing.  The slight degree of variability in hybrids from the first cross
+or in the first generation, in contrast with their extreme variability in
+the succeeding generations, is a curious fact and deserves attention.  For
+it bears on and corroborates the view which I have taken on the cause of
+ordinary variability; namely, that it is due to the reproductive system
+being eminently sensitive to any change in the conditions of life, being
+thus often rendered either impotent or at least incapable of its proper
+function of producing offspring identical with the parent-form.  Now
+hybrids in the first generation are descended from species (excluding those
+long cultivated) which have not had their reproductive systems in any way
+affected, and they are not variable; but hybrids themselves have their
+reproductive systems seriously affected, and their descendants are highly
+variable.
+</para>
+
+<para>
+But to return to our comparison of mongrels and hybrids:  Gartner states
+that mongrels are more liable than hybrids to revert to either parent-form;
+but this, if it be true, is certainly only a difference in degree.  Gartner
+further insists that when any two species, although most closely allied to
+each other, are crossed with a third species, the hybrids are widely
+different from each other; whereas if two very distinct varieties of one
+species are crossed with another species, the hybrids do not differ much. 
+But this conclusion, as far as I can make out, is founded on a single
+experiment; and seems directly opposed to the results of several
+experiments made by Kolreuter.
+</para>
+
+<para>
+These alone are the unimportant differences, which Gartner is able to point
+out, between hybrid and mongrel plants.  On the other hand, the resemblance
+in mongrels and in hybrids to their respective parents, more especially in
+hybrids produced from nearly related species, follows according to Gartner
+the same laws.  When two species are crossed, one has sometimes a prepotent
+power of impressing its likeness on the hybrid; and so I believe it to be
+with varieties of plants.  With animals one variety certainly often has
+this prepotent power over another variety.  Hybrid plants produced from a
+reciprocal cross, generally resemble each other closely; and so it is with
+mongrels from a reciprocal cross.  Both hybrids and mongrels can be reduced
+to either pure parent-form, by repeated crosses in successive generations
+with either parent.
+</para>
+
+<para>
+These several remarks are apparently applicable to animals; but the subject
+is here excessively complicated, partly owing to the existence of secondary
+sexual characters; but more especially owing to prepotency in transmitting
+likeness running more strongly in one sex than in the other, both when one
+species is crossed with another, and when one variety is crossed with
+another variety.  For instance, I think those authors are right, who
+maintain that the ass has a prepotent power over the horse, so that both
+the mule and the hinny more resemble the ass than the horse; but that the
+prepotency runs more strongly in the male-ass than in the female, so that
+the mule, which is the offspring of the male-ass and mare, is more like an
+ass, than is the hinny, which is the offspring of the female-ass and
+stallion.
+</para>
+
+<para>
+Much stress has been laid by some authors on the supposed fact, that
+mongrel animals alone are born closely like one of their parents; but it
+can be shown that this does sometimes occur with hybrids; yet I grant much
+less frequently with hybrids than with mongrels.  Looking to the cases
+which I have collected of cross-bred animals closely resembling one parent,
+the resemblances seem chiefly confined to characters almost monstrous in
+their nature, and which have suddenly appeared--such as albinism, melanism,
+deficiency of tail or horns, or additional fingers and toes; and do not
+relate to characters which have been slowly acquired by selection. 
+Consequently, sudden reversions to the perfect character of either parent
+would be more likely to occur with mongrels, which are descended from
+varieties often suddenly produced and semi-monstrous in character, than
+with hybrids, which are descended from species slowly and naturally
+produced.  On the whole I entirely agree with Dr. Prosper Lucas, who, after
+arranging an enormous body of facts with respect to animals, comes to the
+conclusion, that the laws of resemblance of the child to its parents are
+the same, whether the two parents differ much or little from each other,
+namely in the union of individuals of the same variety, or of different
+varieties, or of distinct species.
+</para>
+
+<para>
+Laying aside the question of fertility and sterility, in all other respects
+there seems to be a general and close similarity in the offspring of
+crossed species, and of crossed varieties.  If we look at species as having
+been specially created, and at varieties as having been produced by
+secondary laws, this similarity would be an astonishing fact.  But it
+harmonises perfectly with the view that there is no essential distinction
+between species and varieties.
+</para>
+
+<para>
+Summary of Chapter -- First crosses between forms sufficiently distinct to
+be ranked as species, and their hybrids, are very generally, but not
+universally, sterile.  The sterility is of all degrees, and is often so
+slight that the two most careful experimentalists who have ever lived, have
+come to diametrically opposite conclusions in ranking forms by this test. 
+The sterility is innately variable in individuals of the same species, and
+is eminently susceptible of favourable and unfavourable conditions.  The
+degree of sterility does not strictly follow systematic affinity, but is
+governed by several curious and complex laws.  It is generally different,
+and sometimes widely different, in reciprocal crosses between the same two
+species.  It is not always equal in degree in a first cross and in the
+hybrid produced from this cross.
+</para>
+
+<para>
+In the same manner as in grafting trees, the capacity of one species or
+variety to take on another, is incidental on generally unknown differences
+in their vegetative systems, so in crossing, the greater or less facility
+of one species to unite with another, is incidental on unknown differences
+in their reproductive systems.  There is no more reason to think that
+species have been specially endowed with various degrees of sterility to
+prevent them crossing and blending in nature, than to think that trees have
+been specially endowed with various and somewhat analogous degrees of
+difficulty in being grafted together in order to prevent them becoming
+inarched in our forests.
+</para>
+
+<para>
+The sterility of first crosses between pure species, which have their
+reproductive systems perfect, seems to depend on several circumstances; in
+some cases largely on the early death of the embryo.  The sterility of
+hybrids, which have their reproductive systems imperfect, and which have
+had this system and their whole organisation disturbed by being compounded
+of two distinct species, seems closely allied to that sterility which so
+frequently affects pure species, when their natural conditions of life have
+been disturbed.  This view is supported by a parallelism of another
+kind;--namely, that the crossing of forms only slightly different is
+favourable to the vigour and fertility of their offspring; and that slight
+changes in the conditions of life are apparently favourable to the vigour
+and fertility of all organic beings.  It is not surprising that the degree
+of difficulty in uniting two species, and the degree of sterility of their
+hybrid-offspring should generally correspond, though due to distinct
+causes; for both depend on the amount of difference of some kind between
+the species which are crossed.  Nor is it surprising that the facility of
+effecting a first cross, the fertility of the hybrids produced, and the
+capacity of being grafted together--though this latter capacity evidently
+depends on widely different circumstances--should all run, to a certain
+extent, parallel with the systematic affinity of the forms which are
+subjected to experiment; for systematic affinity attempts to express all
+kinds of resemblance between all species.
+</para>
+
+<para>
+First crosses between forms known to be varieties, or sufficiently alike to
+be considered as varieties, and their mongrel offspring, are very
+generally, but not quite universally, fertile.  Nor is this nearly general
+and perfect fertility surprising, when we remember how liable we are to
+argue in a circle with respect to varieties in a state of nature; and when
+we remember that the greater number of varieties have been produced under
+domestication by the selection of mere external differences, and not of
+differences in the reproductive system.  In all other respects, excluding
+fertility, there is a close general resemblance between hybrids and
+mongrels.  Finally, then, the facts briefly given in this chapter do not
+seem to me opposed to, but even rather to support the view, that there is
+no fundamental distinction between species and varieties.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter IX</chapnum>
+
+<title>On the Imperfection of the Geological Record</title>
+
+<chapsummary>
+On the absence of intermediate varieties at the present day -- On the
+nature of extinct intermediate varieties; on their number -- On the vast
+lapse of time, as inferred from the rate of deposition and of denudation --
+On the poorness of our palaeontological collections -- On the intermittence
+of geological formations -- On the absence of intermediate varieties in any
+one formation -- On the sudden appearance of groups of species -- On their
+sudden appearance in the lowest known fossiliferous strata.
+</chapsummary>
+</chapheader>
+
+<para>
+In the sixth chapter I enumerated the chief objections which might be
+justly urged against the views maintained in this volume.  Most of them
+have now been discussed.  One, namely the distinctness of specific forms,
+and their not being blended together by innumerable transitional links, is
+a very obvious difficulty.  I assigned reasons why such links do not
+commonly occur at the present day, under the circumstances apparently most
+favourable for their presence, namely on an extensive and continuous area
+with graduated physical conditions.  I endeavoured to show, that the life
+of each species depends in a more important manner on the presence of other
+already defined organic forms, than on climate; and, therefore, that the
+really governing conditions of life do not graduate away quite insensibly
+like heat or moisture.  I endeavoured, also, to show that intermediate
+varieties, from existing in lesser numbers than the forms which they
+connect, will generally be beaten out and exterminated during the course of
+further modification and improvement.  The main cause, however, of
+innumerable intermediate links not now occurring everywhere throughout
+nature depends on the very process of natural selection, through which new
+varieties continually take the places of and exterminate their
+parent-forms.  But just in proportion as this process of extermination has
+acted on an enormous scale, so must the number of intermediate varieties,
+which have formerly existed on the earth, be truly enormous.  Why then is
+not every geological formation and every stratum full of such intermediate
+links?  Geology assuredly does not reveal any such finely graduated organic
+chain; and this, perhaps, is the most obvious and gravest objection which
+can be urged against my theory.  The explanation lies, as I believe, in the
+extreme imperfection of the geological record.
+</para>
+
+<para>
+In the first place it should always be borne in mind what sort of
+intermediate forms must, on my theory, have formerly existed.  I have found
+it difficult, when looking at any two species, to avoid picturing to
+myself, forms directly intermediate between them.  But this is a wholly
+false view; we should always look for forms intermediate between each
+species and a common but unknown progenitor; and the progenitor will
+generally have differed in some respects from all its modified descendants. 
+To give a simple illustration:  the fantail and pouter pigeons have both
+descended from the rock-pigeon; if we possessed all the intermediate
+varieties which have ever existed, we should have an extremely close series
+between both and the rock-pigeon; but we should have no varieties directly
+intermediate between the fantail and pouter; none, for instance, combining
+a tail somewhat expanded with a crop somewhat enlarged, the characteristic
+features of these two breeds.  These two breeds, moreover, have become so
+much modified, that if we had no historical or indirect evidence regarding
+their origin, it would not have been possible to have determined from a
+mere comparison of their structure with that of the rock-pigeon, whether
+they had descended from this species or from some other allied species,
+such as C. oenas.
+</para>
+
+<para>
+So with natural species, if we look to forms very distinct, for instance to
+the horse and tapir, we have no reason to suppose that links ever existed
+directly intermediate between them, but between each and an unknown common
+parent.  The common parent will have had in its whole organisation much
+general resemblance to the tapir and to the horse; but in some points of
+structure may have differed considerably from both, even perhaps more than
+they differ from each other.  Hence in all such cases, we should be unable
+to recognise the parent-form of any two or more species, even if we closely
+compared the structure of the parent with that of its modified descendants,
+unless at the same time we had a nearly perfect chain of the intermediate
+links.
+</para>
+
+<para>
+It is just possible by my theory, that one of two living forms might have
+descended from the other; for instance, a horse from a tapir; and in this
+case direct intermediate links will have existed between them.  But such a
+case would imply that one form had remained for a very long period
+unaltered, whilst its descendants had undergone a vast amount of change;
+and the principle of competition between organism and organism, between
+child and parent, will render this a very rare event; for in all cases the
+new and improved forms of life will tend to supplant the old and unimproved
+forms.
+</para>
+
+<para>
+By the theory of natural selection all living species have been connected
+with the parent-species of each genus, by differences not greater than we
+see between the varieties of the same species at the present day; and these
+parent-species, now generally extinct, have in their turn been similarly
+connected with more ancient species; and so on backwards, always converging
+to the common ancestor of each great class.  So that the number of
+intermediate and transitional links, between all living and extinct
+species, must have been inconceivably great.  But assuredly, if this theory
+be true, such have lived upon this earth.
+</para>
+
+<para>
+On the lapse of Time. -- Independently of our not finding fossil remains of
+such infinitely numerous connecting links, it may be objected, that time
+will not have sufficed for so great an amount of organic change, all
+changes having been effected very slowly through natural selection.  It is
+hardly possible for me even to recall to the reader, who may not be a
+practical geologist, the facts leading the mind feebly to comprehend the
+lapse of time.  He who can read Sir Charles Lyell's grand work on the
+Principles of Geology, which the future historian will recognise as having
+produced a revolution in natural science, yet does not admit how
+incomprehensibly vast have been the past periods of time, may at once close
+this volume.  Not that it suffices to study the Principles of Geology, or
+to read special treatises by different observers on separate formations,
+and to mark how each author attempts to give an inadequate idea of the
+duration of each formation or even each stratum.  A man must for years
+examine for himself great piles of superimposed strata, and watch the sea
+at work grinding down old rocks and making fresh sediment, before he can
+hope to comprehend anything of the lapse of time, the monuments of which we
+see around us.
+</para>
+
+<para>
+It is good to wander along lines of sea-coast, when formed of moderately
+hard rocks, and mark the process of degradation.  The tides in most cases
+reach the cliffs only for a short time twice a day, and the waves eat into
+them only when they are charged with sand or pebbles; for there is reason
+to believe that pure water can effect little or nothing in wearing away
+rock.  At last the base of the cliff is undermined, huge fragments fall
+down, and these remaining fixed, have to be worn away, atom by atom, until
+reduced in size they can be rolled about by the waves, and then are more
+quickly ground into pebbles, sand, or mud.  But how often do we see along
+the bases of retreating cliffs rounded boulders, all thickly clothed by
+marine productions, showing how little they are abraded and how seldom they
+are rolled about!  Moreover, if we follow for a few miles any line of rocky
+cliff, which is undergoing degradation, we find that it is only here and
+there, along a short length or round a promontory, that the cliffs are at
+the present time suffering.  The appearance of the surface and the
+vegetation show that elsewhere years have elapsed since the waters washed
+their base.
+</para>
+
+<para>
+He who most closely studies the action of the sea on our shores, will, I
+believe, be most deeply impressed with the slowness with which rocky coasts
+are worn away.  The observations on this head by Hugh Miller, and by that
+excellent observer Mr. Smith of Jordan Hill, are most impressive.  With the
+mind thus impressed, let any one examine beds of conglomerate many thousand
+feet in thickness, which, though probably formed at a quicker rate than
+many other deposits, yet, from being formed of worn and rounded pebbles,
+each of which bears the stamp of time, are good to show how slowly the mass
+has been accumulated.  Let him remember Lyell's profound remark, that the
+thickness and extent of sedimentary formations are the result and measure
+of the degradation which the earth's crust has elsewhere suffered.  And
+what an amount of degradation is implied by the sedimentary deposits of
+many countries!  Professor Ramsay has given me the maximum thickness, in
+most cases from actual measurement, in a few cases from estimate, of each
+formation in different parts of Great Britain; and this is the result:-
+</para>
+
+<list>
+                                                 <item>Feet</item>
+<item>Palaeozoic strata (not including igneous beds)..57,154</item>
+<item>Secondary strata................................13,190</item>
+<item>Tertiary strata..................................2,240</item>
+</list>
+
+<para>
+--making altogether 72,584 feet; that is, very nearly thirteen and
+three-quarters British miles.  Some of these formations, which are
+represented in England by thin beds, are thousands of feet in thickness on
+the Continent.  Moreover, between each successive formation, we have, in
+the opinion of most geologists, enormously long blank periods.  So that the
+lofty pile of sedimentary rocks in Britain, gives but an inadequate idea of
+the time which has elapsed during their accumulation; yet what time this
+must have consumed!  Good observers have estimated that sediment is
+deposited by the great Mississippi river at the rate of only 600 feet in a
+hundred thousand years.  This estimate may be quite erroneous; yet,
+considering over what wide spaces very fine sediment is transported by the
+currents of the sea, the process of accumulation in any one area must be
+extremely slow.
+</para>
+
+<para>
+But the amount of denudation which the strata have in many places suffered,
+independently of the rate of accumulation of the degraded matter, probably
+offers the best evidence of the lapse of time.  I remember having been much
+struck with the evidence of denudation, when viewing volcanic islands,
+which have been worn by the waves and pared all round into perpendicular
+cliffs of one or two thousand feet in height; for the gentle slope of the
+lava-streams, due to their formerly liquid state, showed at a glance how
+far the hard, rocky beds had once extended into the open ocean.  The same
+story is still more plainly told by faults,--those great cracks along which
+the strata have been upheaved on one side, or thrown down on the other, to
+the height or depth of thousands of feet; for since the crust cracked, the
+surface of the land has been so completely planed down by the action of the
+sea, that no trace of these vast dislocations is externally visible.
+</para>
+
+<para>
+The Craven fault, for instance, extends for upwards of 30 miles, and along
+this line the vertical displacement of the strata has varied from 600 to
+3000 feet.  Prof. Ramsay has published an account of a downthrow in
+Anglesea of 2300 feet; and he informs me that he fully believes there is
+one in Merionethshire of 12,000 feet; yet in these cases there is nothing
+on the surface to show such prodigious movements; the pile of rocks on the
+one or other side having been smoothly swept away.  The consideration of
+these facts impresses my mind almost in the same manner as does the vain
+endeavour to grapple with the idea of eternity.
+</para>
+
+<para>
+I am tempted to give one other case, the well-known one of the denudation
+of the Weald.  Though it must be admitted that the denudation of the Weald
+has been a mere trifle, in comparison with that which has removed masses of
+our palaeozoic strata, in parts ten thousand feet in thickness, as shown in
+Prof. Ramsay's masterly memoir on this subject.  Yet it is an admirable
+lesson to stand on the North Downs and to look at the distant South Downs;
+for, remembering that at no great distance to the west the northern and
+southern escarpments meet and close, one can safely picture to oneself the
+great dome of rocks which must have covered up the Weald within so limited
+a period as since the latter part of the Chalk formation.  The distance
+from the northern to the southern Downs is about 22 miles, and the
+thickness of the several formations is on an average about 1100 feet, as I
+am informed by Prof. Ramsay.  But if, as some geologists suppose, a range
+of older rocks underlies the Weald, on the flanks of which the overlying
+sedimentary deposits might have accumulated in thinner masses than
+elsewhere, the above estimate would be erroneous; but this source of doubt
+probably would not greatly affect the estimate as applied to the western
+extremity of the district.  If, then, we knew the rate at which the sea
+commonly wears away a line of cliff of any given height, we could measure
+the time requisite to have denuded the Weald.  This, of course, cannot be
+done; but we may, in order to form some crude notion on the subject, assume
+that the sea would eat into cliffs 500 feet in height at the rate of one
+inch in a century.  This will at first appear much too small an allowance;
+but it is the same as if we were to assume a cliff one yard in height to be
+eaten back along a whole line of coast at the rate of one yard in nearly
+every twenty-two years.  I doubt whether any rock, even as soft as chalk,
+would yield at this rate excepting on the most exposed coasts; though no
+doubt the degradation of a lofty cliff would be more rapid from the
+breakage of the fallen fragments.  On the other hand, I do not believe that
+any line of coast, ten or twenty miles in length, ever suffers degradation
+at the same time along its whole indented length; and we must remember that
+almost all strata contain harder layers or nodules, which from long
+resisting attrition form a breakwater at the base.  Hence, under ordinary
+circumstances, I conclude that for a cliff 500 feet in height, a denudation
+of one inch per century for the whole length would be an ample allowance. 
+At this rate, on the above data, the denudation of the Weald must have
+required 306,662,400 years; or say three hundred million years.
+</para>
+
+<para>
+The action of fresh water on the gently inclined Wealden district, when
+upraised, could hardly have been great, but it would somewhat reduce the
+above estimate.  On the other hand, during oscillations of level, which we
+know this area has undergone, the surface may have existed for millions of
+years as land, and thus have escaped the action of the sea:  when deeply
+submerged for perhaps equally long periods, it would, likewise, have
+escaped the action of the coast-waves.  So that in all probability a far
+longer period than 300 million years has elapsed since the latter part of
+the Secondary period.
+</para>
+
+<para>
+I have made these few remarks because it is highly important for us to gain
+some notion, however imperfect, of the lapse of years.  During each of
+these years, over the whole world, the land and the water has been peopled
+by hosts of living forms.  What an infinite number of generations, which
+the mind cannot grasp, must have succeeded each other in the long roll of
+years!  Now turn to our richest geological museums, and what a paltry
+display we behold!
+</para>
+
+<para>
+On the poorness of our Palaeontological collections. -- That our
+palaeontological collections are very imperfect, is admitted by every one. 
+The remark of that admirable palaeontologist, the late Edward Forbes,
+should not be forgotten, namely, that numbers of our fossil species are
+known and named from single and often broken specimens, or from a few
+specimens collected on some one spot.  Only a small portion of the surface
+of the earth has been geologically explored, and no part with sufficient
+care, as the important discoveries made every year in Europe prove.  No
+organism wholly soft can be preserved.  Shells and bones will decay and
+disappear when left on the bottom of the sea, where sediment is not
+accumulating.  I believe we are continually taking a most erroneous view,
+when we tacitly admit to ourselves that sediment is being deposited over
+nearly the whole bed of the sea, at a rate sufficiently quick to embed and
+preserve fossil remains.  Throughout an enormously large proportion of the
+ocean, the bright blue tint of the water bespeaks its purity.  The many
+cases on record of a formation conformably covered, after an enormous
+interval of time, by another and later formation, without the underlying
+bed having suffered in the interval any wear and tear, seem explicable only
+on the view of the bottom of the sea not rarely lying for ages in an
+unaltered condition.  The remains which do become embedded, if in sand or
+gravel, will when the beds are upraised generally be dissolved by the
+percolation of rain-water.  I suspect that but few of the very many animals
+which live on the beach between high and low watermark are preserved.  For
+instance, the several species of the Chthamalinae (a sub-family of sessile
+cirripedes) coat the rocks all over the world in infinite numbers:  they
+are all strictly littoral, with the exception of a single Mediterranean
+species, which inhabits deep water and has been found fossil in Sicily,
+whereas not one other species has hitherto been found in any tertiary
+formation:  yet it is now known that the genus Chthamalus existed during
+the chalk period.  The molluscan genus Chiton offers a partially analogous
+case.
+</para>
+
+<para>
+With respect to the terrestrial productions which lived during the
+Secondary and Palaeozoic periods, it is superfluous to state that our
+evidence from fossil remains is fragmentary in an extreme degree.  For
+instance, not a land shell is known belonging to either of these vast
+periods, with one exception discovered by Sir C. Lyell in the carboniferous
+strata of North America.  In regard to mammiferous remains, a single glance
+at the historical table published in the Supplement to Lyell's Manual, will
+bring home the truth, how accidental and rare is their preservation, far
+better than pages of detail.  Nor is their rarity surprising, when we
+remember how large a proportion of the bones of tertiary mammals have been
+discovered either in caves or in lacustrine deposits; and that not a cave
+or true lacustrine bed is known belonging to the age of our secondary or
+palaeozoic formations.
+</para>
+
+<para>
+But the imperfection in the geological record mainly results from another
+and more important cause than any of the foregoing; namely, from the
+several formations being separated from each other by wide intervals of
+time.  When we see the formations tabulated in written works, or when we
+follow them in nature, it is difficult to avoid believing that they are
+closely consecutive.  But we know, for instance, from Sir R. Murchison's
+great work on Russia, what wide gaps there are in that country between the
+superimposed formations; so it is in North America, and in many other parts
+of the world.  The most skilful geologist, if his attention had been
+exclusively confined to these large territories, would never have suspected
+that during the periods which were blank and barren in his own country,
+great piles of sediment, charged with new and peculiar forms of life, had
+elsewhere been accumulated.  And if in each separate territory, hardly any
+idea can be formed of the length of time which has elapsed between the
+consecutive formations, we may infer that this could nowhere be
+ascertained.  The frequent and great changes in the mineralogical
+composition of consecutive formations, generally implying great changes in
+the geography of the surrounding lands, whence the sediment has been
+derived, accords with the belief of vast intervals of time having elapsed
+between each formation.
+</para>
+
+<para>
+But we can, I think, see why the geological formations of each region are
+almost invariably intermittent; that is, have not followed each other in
+close sequence.  Scarcely any fact struck me more when examining many
+hundred miles of the South American coasts, which have been upraised
+several hundred feet within the recent period, than the absence of any
+recent deposits sufficiently extensive to last for even a short geological
+period.  Along the whole west coast, which is inhabited by a peculiar
+marine fauna, tertiary beds are so scantily developed, that no record of
+several successive and peculiar marine faunas will probably be preserved to
+a distant age.  A little reflection will explain why along the rising coast
+of the western side of South America, no extensive formations with recent
+or tertiary remains can anywhere be found, though the supply of sediment
+must for ages have been great, from the enormous degradation of the
+coast-rocks and from muddy streams entering the sea.  The explanation, no
+doubt, is, that the littoral and sub-littoral deposits are continually worn
+away, as soon as they are brought up by the slow and gradual rising of the
+land within the grinding action of the coast-waves.
+</para>
+
+<para>
+We may, I think, safely conclude that sediment must be accumulated in
+extremely thick, solid, or extensive masses, in order to withstand the
+incessant action of the waves, when first upraised and during subsequent
+oscillations of level.  Such thick and extensive accumulations of sediment
+may be formed in two ways; either, in profound depths of the sea, in which
+case, judging from the researches of E. Forbes, we may conclude that the
+bottom will be inhabited by extremely few animals, and the mass when
+upraised will give a most imperfect record of the forms of life which then
+existed; or, sediment may be accumulated to any thickness and extent over a
+shallow bottom, if it continue slowly to subside.  In this latter case, as
+long as the rate of subsidence and supply of sediment nearly balance each
+other, the sea will remain shallow and favourable for life, and thus a
+fossiliferous formation thick enough, when upraised, to resist any amount
+of degradation, may be formed.
+</para>
+
+<para>
+I am convinced that all our ancient formations, which are rich in fossils,
+have thus been formed during subsidence.  Since publishing my views on this
+subject in 1845, I have watched the progress of Geology, and have been
+surprised to note how author after author, in treating of this or that
+great formation, has come to the conclusion that it was accumulated during
+subsidence.  I may add, that the only ancient tertiary formation on the
+west coast of South America, which has been bulky enough to resist such
+degradation as it has as yet suffered, but which will hardly last to a
+distant geological age, was certainly deposited during a downward
+oscillation of level, and thus gained considerable thickness.
+</para>
+
+<para>
+All geological facts tell us plainly that each area has undergone numerous
+slow oscillations of level, and apparently these oscillations have affected
+wide spaces.  Consequently formations rich in fossils and sufficiently
+thick and extensive to resist subsequent degradation, may have been formed
+over wide spaces during periods of subsidence, but only where the supply of
+sediment was sufficient to keep the sea shallow and to embed and preserve
+the remains before they had time to decay.  On the other hand, as long as
+the bed of the sea remained stationary, thick deposits could not have been
+accumulated in the shallow parts, which are the most favourable to life. 
+Still less could this have happened during the alternate periods of
+elevation; or, to speak more accurately, the beds which were then
+accumulated will have been destroyed by being upraised and brought within
+the limits of the coast-action.
+</para>
+
+<para>
+Thus the geological record will almost necessarily be rendered
+intermittent.  I feel much confidence in the truth of these views, for they
+are in strict accordance with the general principles inculcated by Sir C.
+Lyell; and E. Forbes independently arrived at a similar conclusion.
+</para>
+
+<para>
+One remark is here worth a passing notice.  During periods of elevation the
+area of the land and of the adjoining shoal parts of the sea will be
+increased, and new stations will often be formed;--all circumstances most
+favourable, as previously explained, for the formation of new varieties and
+species; but during such periods there will generally be a blank in the
+geological record.  On the other hand, during subsidence, the inhabited
+area and number of inhabitants will decrease (excepting the productions on
+the shores of a continent when first broken up into an archipelago), and
+consequently during subsidence, though there will be much extinction, fewer
+new varieties or species will be formed; and it is during these very
+periods of subsidence, that our great deposits rich in fossils have been
+accumulated.  Nature may almost be said to have guarded against the
+frequent discovery of her transitional or linking forms.
+</para>
+
+<para>
+From the foregoing considerations it cannot be doubted that the geological
+record, viewed as a whole, is extremely imperfect; but if we confine our
+attention to any one formation, it becomes more difficult to understand,
+why we do not therein find closely graduated varieties between the allied
+species which lived at its commencement and at its close.  Some cases are
+on record of the same species presenting distinct varieties in the upper
+and lower parts of the same formation, but, as they are rare, they may be
+here passed over.  Although each formation has indisputably required a vast
+number of years for its deposition, I can see several reasons why each
+should not include a graduated series of links between the species which
+then lived; but I can by no means pretend to assign due proportional weight
+to the following considerations.
+</para>
+
+<para>
+Although each formation may mark a very long lapse of years, each perhaps
+is short compared with the period requisite to change one species into
+another.  I am aware that two palaeontologists, whose opinions are worthy
+of much deference, namely Bronn and Woodward, have concluded that the
+average duration of each formation is twice or thrice as long as the
+average duration of specific forms.  But insuperable difficulties, as it
+seems to me, prevent us coming to any just conclusion on this head.  When
+we see a species first appearing in the middle of any formation, it would
+be rash in the extreme to infer that it had not elsewhere previously
+existed.  So again when we find a species disappearing before the uppermost
+layers have been deposited, it would be equally rash to suppose that it
+then became wholly extinct.  We forget how small the area of Europe is
+compared with the rest of the world; nor have the several stages of the
+same formation throughout Europe been correlated with perfect accuracy.
+</para>
+
+<para>
+With marine animals of all kinds, we may safely infer a large amount of
+migration during climatal and other changes; and when we see a species
+first appearing in any formation, the probability is that it only then
+first immigrated into that area.  It is well known, for instance, that
+several species appeared somewhat earlier in the palaeozoic beds of North
+America than in those of Europe; time having apparently been required for
+their migration from the American to the European seas.  In examining the
+latest deposits of various quarters of the world, it has everywhere been
+noted, that some few still existing species are common in the deposit, but
+have become extinct in the immediately surrounding sea; or, conversely,
+that some are now abundant in the neighbouring sea, but are rare or absent
+in this particular deposit.  It is an excellent lesson to reflect on the
+ascertained amount of migration of the inhabitants of Europe during the
+Glacial period, which forms only a part of one whole geological period; and
+likewise to reflect on the great changes of level, on the inordinately
+great change of climate, on the prodigious lapse of time, all included
+within this same glacial period.  Yet it may be doubted whether in any
+quarter of the world, sedimentary deposits, including fossil remains, have
+gone on accumulating within the same area during the whole of this period. 
+It is not, for instance, probable that sediment was deposited during the
+whole of the glacial period near the mouth of the Mississippi, within that
+limit of depth at which marine animals can flourish; for we know what vast
+geographical changes occurred in other parts of America during this space
+of time.  When such beds as were deposited in shallow water near the mouth
+of the Mississippi during some part of the glacial period shall have been
+upraised, organic remains will probably first appear and disappear at
+different levels, owing to the migration of species and to geographical
+changes.  And in the distant future, a geologist examining these beds,
+might be tempted to conclude that the average duration of life of the
+embedded fossils had been less than that of the glacial period, instead of
+having been really far greater, that is extending from before the glacial
+epoch to the present day.
+</para>
+
+<para>
+In order to get a perfect gradation between two forms in the upper and
+lower parts of the same formation, the deposit must have gone on
+accumulating for a very long period, in order to have given sufficient time
+for the slow process of variation; hence the deposit will generally have to
+be a very thick one; and the species undergoing modification will have had
+to live on the same area throughout this whole time.  But we have seen that
+a thick fossiliferous formation can only be accumulated during a period of
+subsidence; and to keep the depth approximately the same, which is
+necessary in order to enable the same species to live on the same space,
+the supply of sediment must nearly have counterbalanced the amount of
+subsidence.  But this same movement of subsidence will often tend to sink
+the area whence the sediment is derived, and thus diminish the supply
+whilst the downward movement continues.  In fact, this nearly exact
+balancing between the supply of sediment and the amount of subsidence is
+probably a rare contingency; for it has been observed by more than one
+palaeontologist, that very thick deposits are usually barren of organic
+remains, except near their upper or lower limits.
+</para>
+
+<para>
+It would seem that each separate formation, like the whole pile of
+formations in any country, has generally been intermittent in its
+accumulation.  When we see, as is so often the case, a formation composed
+of beds of different mineralogical composition, we may reasonably suspect
+that the process of deposition has been much interrupted, as a change in
+the currents of the sea and a supply of sediment of a different nature will
+generally have been due to geographical changes requiring much time.  Nor
+will the closest inspection of a formation give any idea of the time which
+its deposition has consumed.  Many instances could be given of beds only a
+few feet in thickness, representing formations, elsewhere thousands of feet
+in thickness, and which must have required an enormous period for their
+accumulation; yet no one ignorant of this fact would have suspected the
+vast lapse of time represented by the thinner formation.  Many cases could
+be given of the lower beds of a formation having been upraised, denuded,
+submerged, and then re-covered by the upper beds of the same
+formation,--facts, showing what wide, yet easily overlooked, intervals have
+occurred in its accumulation.  In other cases we have the plainest evidence
+in great fossilised trees, still standing upright as they grew, of many
+long intervals of time and changes of level during the process of
+deposition, which would never even have been suspected, had not the trees
+chanced to have been preserved:  thus, Messrs. Lyell and Dawson found
+carboniferous beds 1400 feet thick in Nova Scotia, with ancient
+root-bearing strata, one above the other, at no less than sixty-eight
+different levels.  Hence, when the same species occur at the bottom,
+middle, and top of a formation, the probability is that they have not lived
+on the same spot during the whole period of deposition, but have
+disappeared and reappeared, perhaps many times, during the same geological
+period.  So that if such species were to undergo a considerable amount of
+modification during any one geological period, a section would not probably
+include all the fine intermediate gradations which must on my theory have
+existed between them, but abrupt, though perhaps very slight, changes of
+form.
+</para>
+
+<para>
+It is all-important to remember that naturalists have no golden rule by
+which to distinguish species and varieties; they grant some little
+variability to each species, but when they meet with a somewhat greater
+amount of difference between any two forms, they rank both as species,
+unless they are enabled to connect them together by close intermediate
+gradations.  And this from the reasons just assigned we can seldom hope to
+effect in any one geological section.  Supposing B and C to be two species,
+and a third, A, to be found in an underlying bed; even if A were strictly
+intermediate between B and C, it would simply be ranked as a third and
+distinct species, unless at the same time it could be most closely
+connected with either one or both forms by intermediate varieties.  Nor
+should it be forgotten, as before explained, that A might be the actual
+progenitor of B and C, and yet might not at all necessarily be strictly
+intermediate between them in all points of structure.  So that we might
+obtain the parent-species and its several modified descendants from the
+lower and upper beds of a formation, and unless we obtained numerous
+transitional gradations, we should not recognise their relationship, and
+should consequently be compelled to rank them all as distinct species.
+</para>
+
+<para>
+It is notorious on what excessively slight differences many
+palaeontologists have founded their species; and they do this the more
+readily if the specimens come from different sub-stages of the same
+formation.  Some experienced conchologists are now sinking many of the very
+fine species of D'Orbigny and others into the rank of varieties; and on
+this view we do find the kind of evidence of change which on my theory we
+ought to find.  Moreover, if we look to rather wider intervals, namely, to
+distinct but consecutive stages of the same great formation, we find that
+the embedded fossils, though almost universally ranked as specifically
+different, yet are far more closely allied to each other than are the
+species found in more widely separated formations; but to this subject I
+shall have to return in the following chapter.
+</para>
+
+<para>
+One other consideration is worth notice:  with animals and plants that can
+propagate rapidly and are not highly locomotive, there is reason to
+suspect, as we have formerly seen, that their varieties are generally at
+first local; and that such local varieties do not spread widely and
+supplant their parent-forms until they have been modified and perfected in
+some considerable degree.  According to this view, the chance of
+discovering in a formation in any one country all the early stages of
+transition between any two forms, is small, for the successive changes are
+supposed to have been local or confined to some one spot.  Most marine
+animals have a wide range; and we have seen that with plants it is those
+which have the widest range, that oftenest present varieties; so that with
+shells and other marine animals, it is probably those which have had the
+widest range, far exceeding the limits of the known geological formations
+of Europe, which have oftenest given rise, first to local varieties and
+ultimately to new species; and this again would greatly lessen the chance
+of our being able to trace the stages of transition in any one geological
+formation.
+</para>
+
+<para>
+It should not be forgotten, that at the present day, with perfect specimens
+for examination, two forms can seldom be connected by intermediate
+varieties and thus proved to be the same species, until many specimens have
+been collected from many places; and in the case of fossil species this
+could rarely be effected by palaeontologists.  We shall, perhaps, best
+perceive the improbability of our being enabled to connect species by
+numerous, fine, intermediate, fossil links, by asking ourselves whether,
+for instance, geologists at some future period will be able to prove, that
+our different breeds of cattle, sheep, horses, and dogs have descended from
+a single stock or from several aboriginal stocks; or, again, whether
+certain sea-shells inhabiting the shores of North America, which are ranked
+by some conchologists as distinct species from their European
+representatives, and by other conchologists as only varieties, are really
+varieties or are, as it is called, specifically distinct.  This could be
+effected only by the future geologist discovering in a fossil state
+numerous intermediate gradations; and such success seems to me improbable
+in the highest degree.
+</para>
+
+<para>
+Geological research, though it has added numerous species to existing and
+extinct genera, and has made the intervals between some few groups less
+wide than they otherwise would have been, yet has done scarcely anything in
+breaking down the distinction between species, by connecting them together
+by numerous, fine, intermediate varieties; and this not having been
+effected, is probably the gravest and most obvious of all the many
+objections which may be urged against my views.  Hence it will be worth
+while to sum up the foregoing remarks, under an imaginary illustration. 
+The Malay Archipelago is of about the size of Europe from the North Cape to
+the Mediterranean, and from Britain to Russia; and therefore equals all the
+geological formations which have been examined with any accuracy, excepting
+those of the United States of America.  I fully agree with Mr.
+Godwin-Austen, that the present condition of the Malay Archipelago, with
+its numerous large islands separated by wide and shallow seas, probably
+represents the former state of Europe, when most of our formations were
+accumulating.  The Malay Archipelago is one of the richest regions of the
+whole world in organic beings; yet if all the species were to be collected
+which have ever lived there, how imperfectly would they represent the
+natural history of the world!
+</para>
+
+<para>
+But we have every reason to believe that the terrestrial productions of the
+archipelago would be preserved in an excessively imperfect manner in the
+formations which we suppose to be there accumulating.  I suspect that not
+many of the strictly littoral animals, or of those which lived on naked
+submarine rocks, would be embedded; and those embedded in gravel or sand,
+would not endure to a distant epoch.  Wherever sediment did not accumulate
+on the bed of the sea, or where it did not accumulate at a sufficient rate
+to protect organic bodies from decay, no remains could be preserved.
+</para>
+
+<para>
+In our archipelago, I believe that fossiliferous formations could be formed
+of sufficient thickness to last to an age, as distant in futurity as the
+secondary formations lie in the past, only during periods of subsidence. 
+These periods of subsidence would be separated from each other by enormous
+intervals, during which the area would be either stationary or rising;
+whilst rising, each fossiliferous formation would be destroyed, almost as
+soon as accumulated, by the incessant coast-action, as we now see on the
+shores of South America.  During the periods of subsidence there would
+probably be much extinction of life; during the periods of elevation, there
+would be much variation, but the geological record would then be least
+perfect.
+</para>
+
+<para>
+It may be doubted whether the duration of any one great period of
+subsidence over the whole or part of the archipelago, together with a
+contemporaneous accumulation of sediment, would exceed the average duration
+of the same specific forms; and these contingencies are indispensable for
+the preservation of all the transitional gradations between any two or more
+species.  If such gradations were not fully preserved, transitional
+varieties would merely appear as so many distinct species.  It is, also,
+probable that each great period of subsidence would be interrupted by
+oscillations of level, and that slight climatal changes would intervene
+during such lengthy periods; and in these cases the inhabitants of the
+archipelago would have to migrate, and no closely consecutive record of
+their modifications could be preserved in any one formation.
+</para>
+
+<para>
+Very many of the marine inhabitants of the archipelago now range thousands
+of miles beyond its confines; and analogy leads me to believe that it would
+be chiefly these far-ranging species which would oftenest produce new
+varieties; and the varieties would at first generally be local or confined
+to one place, but if possessed of any decided advantage, or when further
+modified and improved, they would slowly spread and supplant their
+parent-forms.  When such varieties returned to their ancient homes, as they
+would differ from their former state, in a nearly uniform, though perhaps
+extremely slight degree, they would, according to the principles followed
+by many palaeontologists, be ranked as new and distinct species.
+</para>
+
+<para>
+If then, there be some degree of truth in these remarks, we have no right
+to expect to find in our geological formations, an infinite number of those
+fine transitional forms, which on my theory assuredly have connected all
+the past and present species of the same group into one long and branching
+chain of life.  We ought only to look for a few links, some more closely,
+some more distantly related to each other; and these links, let them be
+ever so close, if found in different stages of the same formation, would,
+by most palaeontologists, be ranked as distinct species.  But I do not
+pretend that I should ever have suspected how poor a record of the
+mutations of life, the best preserved geological section presented, had not
+the difficulty of our not discovering innumerable transitional links
+between the species which appeared at the commencement and close of each
+formation, pressed so hardly on my theory.
+</para>
+
+<para>
+On the sudden appearance of whole groups of Allied Species. -- The abrupt
+manner in which whole groups of species suddenly appear in certain
+formations, has been urged by several palaeontologists, for instance, by
+Agassiz, Pictet, and by none more forcibly than by Professor Sedgwick, as a
+fatal objection to the belief in the transmutation of species.  If numerous
+species, belonging to the same genera or families, have really started into
+life all at once, the fact would be fatal to the theory of descent with
+slow modification through natural selection.  For the development of a
+group of forms, all of which have descended from some one progenitor, must
+have been an extremely slow process; and the progenitors must have lived
+long ages before their modified descendants.  But we continually over-rate
+the perfection of the geological record, and falsely infer, because certain
+genera or families have not been found beneath a certain stage, that they
+did not exist before that stage.  We continually forget how large the world
+is, compared with the area over which our geological formations have been
+carefully examined; we forget that groups of species may elsewhere have
+long existed and have slowly multiplied before they invaded the ancient
+archipelagoes of Europe and of the United States.  We do not make due
+allowance for the enormous intervals of time, which have probably elapsed
+between our consecutive formations,--longer perhaps in some cases than the
+time required for the accumulation of each formation.  These intervals will
+have given time for the multiplication of species from some one or some few
+parent-forms; and in the succeeding formation such species will appear as
+if suddenly created.
+</para>
+
+<para>
+I may here recall a remark formerly made, namely that it might require a
+long succession of ages to adapt an organism to some new and peculiar line
+of life, for instance to fly through the air; but that when this had been
+effected, and a few species had thus acquired a great advantage over other
+organisms, a comparatively short time would be necessary to produce many
+divergent forms, which would be able to spread rapidly and widely
+throughout the world.
+</para>
+
+<para>
+I will now give a few examples to illustrate these remarks; and to show how
+liable we are to error in supposing that whole groups of species have
+suddenly been produced.  I may recall the well-known fact that in
+geological treatises, published not many years ago, the great class of
+mammals was always spoken of as having abruptly come in at the commencement
+of the tertiary series.  And now one of the richest known accumulations of
+fossil mammals belongs to the middle of the secondary series; and one true
+mammal has been discovered in the new red sandstone at nearly the
+commencement of this great series.  Cuvier used to urge that no monkey
+occurred in any tertiary stratum; but now extinct species have been
+discovered in India, South America, and in Europe even as far back as the
+eocene stage.  The most striking case, however, is that of the Whale
+family; as these animals have huge bones, are marine, and range over the
+world, the fact of not a single bone of a whale having been discovered in
+any secondary formation, seemed fully to justify the belief that this great
+and distinct order had been suddenly produced in the interval between the
+latest secondary and earliest tertiary formation.  But now we may read in
+the Supplement to Lyell's 'Manual,' published in 1858, clear evidence of
+the existence of whales in the upper greensand, some time before the close
+of the secondary period.
+</para>
+
+<para>
+I may give another instance, which from having passed under my own eyes has
+much struck me.  In a memoir on Fossil Sessile Cirripedes, I have stated
+that, from the number of existing and extinct tertiary species; from the
+extraordinary abundance of the individuals of many species all over the
+world, from the Arctic regions to the equator, inhabiting various zones of
+depths from the upper tidal limits to 50 fathoms; from the perfect manner
+in which specimens are preserved in the oldest tertiary beds; from the ease
+with which even a fragment of a valve can be recognised; from all these
+circumstances, I inferred that had sessile cirripedes existed during the
+secondary periods, they would certainly have been preserved and discovered;
+and as not one species had been discovered in beds of this age, I concluded
+that this great group had been suddenly developed at the commencement of
+the tertiary series.  This was a sore trouble to me, adding as I thought
+one more instance of the abrupt appearance of a great group of species. 
+But my work had hardly been published, when a skilful palaeontologist, M.
+Bosquet, sent me a drawing of a perfect specimen of an unmistakeable
+sessile cirripede, which he had himself extracted from the chalk of
+Belgium.  And, as if to make the case as striking as possible, this sessile
+cirripede was a Chthamalus, a very common, large, and ubiquitous genus, of
+which not one specimen has as yet been found even in any tertiary stratum. 
+Hence we now positively know that sessile cirripedes existed during the
+secondary period; and these cirripedes might have been the progenitors of
+our many tertiary and existing species.
+</para>
+
+<para>
+The case most frequently insisted on by palaeontologists of the apparently
+sudden appearance of a whole group of species, is that of the teleostean
+fishes, low down in the Chalk period.  This group includes the large
+majority of existing species.  Lately, Professor Pictet has carried their
+existence one sub-stage further back; and some palaeontologists believe
+that certain much older fishes, of which the affinities are as yet
+imperfectly known, are really teleostean.  Assuming, however, that the
+whole of them did appear, as Agassiz believes, at the commencement of the
+chalk formation, the fact would certainly be highly remarkable; but I
+cannot see that it would be an insuperable difficulty on my theory, unless
+it could likewise be shown that the species of this group appeared suddenly
+and simultaneously throughout the world at this same period.  It is almost
+superfluous to remark that hardly any fossil-fish are known from south of
+the equator; and by running through Pictet's Palaeontology it will be seen
+that very few species are known from several formations in Europe.  Some
+few families of fish now have a confined range; the teleostean fish might
+formerly have had a similarly confined range, and after having been largely
+developed in some one sea, might have spread widely.  Nor have we any right
+to suppose that the seas of the world have always been so freely open from
+south to north as they are at present.  Even at this day, if the Malay
+Archipelago were converted into land, the tropical parts of the Indian
+Ocean would form a large and perfectly enclosed basin, in which any great
+group of marine animals might be multiplied; and here they would remain
+confined, until some of the species became adapted to a cooler climate, and
+were enabled to double the southern capes of Africa or Australia, and thus
+reach other and distant seas.
+</para>
+
+<para>
+From these and similar considerations, but chiefly from our ignorance of
+the geology of other countries beyond the confines of Europe and the United
+States; and from the revolution in our palaeontological ideas on many
+points, which the discoveries of even the last dozen years have effected,
+it seems to me to be about as rash in us to dogmatize on the succession of
+organic beings throughout the world, as it would be for a naturalist to
+land for five minutes on some one barren point in Australia, and then to
+discuss the number and range of its productions.
+</para>
+
+<para>
+On the sudden appearance of groups of Allied Species in the lowest known
+fossiliferous strata. -- There is another and allied difficulty, which is
+much graver.  I allude to the manner in which numbers of species of the
+same group, suddenly appear in the lowest known fossiliferous rocks.  Most
+of the arguments which have convinced me that all the existing species of
+the same group have descended from one progenitor, apply with nearly equal
+force to the earliest known species.  For instance, I cannot doubt that all
+the Silurian trilobites have descended from some one crustacean, which must
+have lived long before the Silurian age, and which probably differed
+greatly from any known animal.  Some of the most ancient Silurian animals,
+as the Nautilus, Lingula, &amp;c., do not differ much from living species; and
+it cannot on my theory be supposed, that these old species were the
+progenitors of all the species of the orders to which they belong, for they
+do not present characters in any degree intermediate between them.  If,
+moreover, they had been the progenitors of these orders, they would almost
+certainly have been long ago supplanted and exterminated by their numerous
+and improved descendants.
+</para>
+
+<para>
+Consequently, if my theory be true, it is indisputable that before the
+lowest Silurian stratum was deposited, long periods elapsed, as long as, or
+probably far longer than, the whole interval from the Silurian age to the
+present day; and that during these vast, yet quite unknown, periods of
+time, the world swarmed with living creatures.
+</para>
+
+<para>
+To the question why we do not find records of these vast primordial
+periods, I can give no satisfactory answer.  Several of the most eminent
+geologists, with Sir R. Murchison at their head, are convinced that we see
+in the organic remains of the lowest Silurian stratum the dawn of life on
+this planet.  Other highly competent judges, as Lyell and the late E.
+Forbes, dispute this conclusion.  We should not forget that only a small
+portion of the world is known with accuracy.  M. Barrande has lately added
+another and lower stage to the Silurian system, abounding with new and
+peculiar species.  Traces of life have been detected in the Longmynd beds
+beneath Barrande's so-called primordial zone.  The presence of phosphatic
+nodules and bituminous matter in some of the lowest azoic rocks, probably
+indicates the former existence of life at these periods.  But the
+difficulty of understanding the absence of vast piles of fossiliferous
+strata, which on my theory no doubt were somewhere accumulated before the
+Silurian epoch, is very great.  If these most ancient beds had been wholly
+worn away by denudation, or obliterated by metamorphic action, we ought to
+find only small remnants of the formations next succeeding them in age, and
+these ought to be very generally in a metamorphosed condition.  But the
+descriptions which we now possess of the Silurian deposits over immense
+territories in Russia and in North America, do not support the view, that
+the older a formation is, the more it has suffered the extremity of
+denudation and metamorphism.
+</para>
+
+<para>
+The case at present must remain inexplicable; and may be truly urged as a
+valid argument against the views here entertained.  To show that it may
+hereafter receive some explanation, I will give the following hypothesis. 
+From the nature of the organic remains, which do not appear to have
+inhabited profound depths, in the several formations of Europe and of the
+United States; and from the amount of sediment, miles in thickness, of
+which the formations are composed, we may infer that from first to last
+large islands or tracts of land, whence the sediment was derived, occurred
+in the neighbourhood of the existing continents of Europe and North
+America.  But we do not know what was the state of things in the intervals
+between the successive formations; whether Europe and the United States
+during these intervals existed as dry land, or as a submarine surface near
+land, on which sediment was not deposited, or again as the bed of an open
+and unfathomable sea.
+</para>
+
+<para>
+Looking to the existing oceans, which are thrice as extensive as the land,
+we see them studded with many islands; but not one oceanic island is as yet
+known to afford even a remnant of any palaeozoic or secondary formation. 
+Hence we may perhaps infer, that during the palaeozoic and secondary
+periods, neither continents nor continental islands existed where our
+oceans now extend; for had they existed there, palaeozoic and secondary
+formations would in all probability have been accumulated from sediment
+derived from their wear and tear; and would have been at least partially
+upheaved by the oscillations of level, which we may fairly conclude must
+have intervened during these enormously long periods.  If then we may infer
+anything from these facts, we may infer that where our oceans now extend,
+oceans have extended from the remotest period of which we have any record;
+and on the other hand, that where continents now exist, large tracts of
+land have existed, subjected no doubt to great oscillations of level, since
+the earliest silurian period.  The coloured map appended to my volume on
+Coral Reefs, led me to conclude that the great oceans are still mainly
+areas of subsidence, the great archipelagoes still areas of oscillations of
+level, and the continents areas of elevation.  But have we any right to
+assume that things have thus remained from eternity?  Our continents seem
+to have been formed by a preponderance, during many oscillations of level,
+of the force of elevation; but may not the areas of preponderant movement
+have changed in the lapse of ages?  At a period immeasurably antecedent to
+the silurian epoch, continents may have existed where oceans are now spread
+out; and clear and open oceans may have existed where our continents now
+stand.  Nor should we be justified in assuming that if, for instance, the
+bed of the Pacific Ocean were now converted into a continent, we should
+there find formations older than the silurian strata, supposing such to
+have been formerly deposited; for it might well happen that strata which
+had subsided some miles nearer to the centre of the earth, and which had
+been pressed on by an enormous weight of superincumbent water, might have
+undergone far more metamorphic action than strata which have always
+remained nearer to the surface.  The immense areas in some parts of the
+world, for instance in South America, of bare metamorphic rocks, which must
+have been heated under great pressure, have always seemed to me to require
+some special explanation; and we may perhaps believe that we see in these
+large areas, the many formations long anterior to the silurian epoch in a
+completely metamorphosed condition.
+</para>
+
+<para>
+The several difficulties here discussed, namely our not finding in the
+successive formations infinitely numerous transitional links between the
+many species which now exist or have existed; the sudden manner in which
+whole groups of species appear in our European formations; the almost
+entire absence, as at present known, of fossiliferous formations beneath
+the Silurian strata, are all undoubtedly of the gravest nature.  We see
+this in the plainest manner by the fact that all the most eminent
+palaeontologists, namely Cuvier, Owen, Agassiz, Barrande, Falconer, E.
+Forbes, &amp;c., and all our greatest geologists, as Lyell, Murchison,
+Sedgwick, &amp;c., have unanimously, often vehemently, maintained the
+immutability of species.  But I have reason to believe that one great
+authority, Sir Charles Lyell, from further reflexion entertains grave
+doubts on this subject.  I feel how rash it is to differ from these great
+authorities, to whom, with others, we owe all our knowledge.  Those who
+think the natural geological record in any degree perfect, and who do not
+attach much weight to the facts and arguments of other kinds given in this
+volume, will undoubtedly at once reject my theory.  For my part, following
+out Lyell's metaphor, I look at the natural geological record, as a history
+of the world imperfectly kept, and written in a changing dialect; of this
+history we possess the last volume alone, relating only to two or three
+countries.  Of this volume, only here and there a short chapter has been
+preserved; and of each page, only here and there a few lines.  Each word of
+the slowly-changing language, in which the history is supposed to be
+written, being more or less different in the interrupted succession of
+chapters, may represent the apparently abruptly changed forms of life,
+entombed in our consecutive, but widely separated formations.  On this
+view, the difficulties above discussed are greatly diminished, or even
+disappear.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter X</chapnum>
+
+<title>On the Geological Succession of Organic Beings</title>
+
+<chapsummary>
+On the slow and successive appearance of new species -- On their different
+rates of change -- Species once lost do not reappear -- Groups of species
+follow the same general rules in their appearance and disappearance as do
+single species -- On Extinction -- On simultaneous changes in the forms of
+life throughout the world -- On the affinities of extinct species to each
+other and to living species -- On the state of development of ancient forms
+-- On the succession of the same types within the same areas -- Summary of
+preceding and present chapters.
+</chapsummary>
+</chapheader>
+
+<para>
+Let us now see whether the several facts and rules relating to the
+geological succession of organic beings, better accord with the common view
+of the immutability of species, or with that of their slow and gradual
+modification, through descent and natural selection.
+</para>
+
+<para>
+New species have appeared very slowly, one after another, both on the land
+and in the waters.  Lyell has shown that it is hardly possible to resist
+the evidence on this head in the case of the several tertiary stages; and
+every year tends to fill up the blanks between them, and to make the
+percentage system of lost and new forms more gradual.  In some of the most
+recent beds, though undoubtedly of high antiquity if measured by years,
+only one or two species are lost forms, and only one or two are new forms,
+having here appeared for the first time, either locally, or, as far as we
+know, on the face of the earth.  If we may trust the observations of
+Philippi in Sicily, the successive changes in the marine inhabitants of
+that island have been many and most gradual.  The secondary formations are
+more broken; but, as Bronn has remarked, neither the appearance nor
+disappearance of their many now extinct species has been simultaneous in
+each separate formation.
+</para>
+
+<para>
+Species of different genera and classes have not changed at the same rate,
+or in the same degree.  In the oldest tertiary beds a few living shells may
+still be found in the midst of a multitude of extinct forms.  Falconer has
+given a striking instance of a similar fact, in an existing crocodile
+associated with many strange and lost mammals and reptiles in the
+sub-Himalayan deposits.  The Silurian Lingula differs but little from the
+living species of this genus; whereas most of the other Silurian Molluscs
+and all the Crustaceans have changed greatly.  The productions of the land
+seem to change at a quicker rate than those of the sea, of which a striking
+instance has lately been observed in Switzerland.  There is some reason to
+believe that organisms, considered high in the scale of nature, change more
+quickly than those that are low:  though there are exceptions to this rule. 
+The amount of organic change, as Pictet has remarked, does not strictly
+correspond with the succession of our geological formations; so that
+between each two consecutive formations, the forms of life have seldom
+changed in exactly the same degree.  Yet if we compare any but the most
+closely related formations, all the species will be found to have undergone
+some change.  When a species has once disappeared from the face of the
+earth, we have reason to believe that the same identical form never
+reappears.  The strongest apparent exception to this latter rule, is that
+of the so-called 'colonies' of M. Barrande, which intrude for a period in
+the midst of an older formation, and then allow the pre-existing fauna to
+reappear; but Lyell's explanation, namely, that it is a case of temporary
+migration from a distinct geographical province, seems to me satisfactory.
+</para>
+
+<para>
+These several facts accord well with my theory.  I believe in no fixed law
+of development, causing all the inhabitants of a country to change
+abruptly, or simultaneously, or to an equal degree.  The process of
+modification must be extremely slow.  The variability of each species is
+quite independent of that of all others.  Whether such variability be taken
+advantage of by natural selection, and whether the variations be
+accumulated to a greater or lesser amount, thus causing a greater or lesser
+amount of modification in the varying species, depends on many complex
+contingencies,--on the variability being of a beneficial nature, on the
+power of intercrossing, on the rate of breeding, on the slowly changing
+physical conditions of the country, and more especially on the nature of
+the other inhabitants with which the varying species comes into
+competition.  Hence it is by no means surprising that one species should
+retain the same identical form much longer than others; or, if changing,
+that it should change less.  We see the same fact in geographical
+distribution; for instance, in the land-shells and coleopterous insects of
+Madeira having come to differ considerably from their nearest allies on the
+continent of Europe, whereas the marine shells and birds have remained
+unaltered.  We can perhaps understand the apparently quicker rate of change
+in terrestrial and in more highly organised productions compared with
+marine and lower productions, by the more complex relations of the higher
+beings to their organic and inorganic conditions of life, as explained in a
+former chapter.  When many of the inhabitants of a country have become
+modified and improved, we can understand, on the principle of competition,
+and on that of the many all-important relations of organism to organism,
+that any form which does not become in some degree modified and improved,
+will be liable to be exterminated.  Hence we can see why all the species in
+the same region do at last, if we look to wide enough intervals of time,
+become modified; for those which do not change will become extinct.
+</para>
+
+<para>
+In members of the same class the average amount of change, during long and
+equal periods of time, may, perhaps, be nearly the same; but as the
+accumulation of long-enduring fossiliferous formations depends on great
+masses of sediment having been deposited on areas whilst subsiding, our
+formations have been almost necessarily accumulated at wide and irregularly
+intermittent intervals; consequently the amount of organic change exhibited
+by the fossils embedded in consecutive formations is not equal.  Each
+formation, on this view, does not mark a new and complete act of creation,
+but only an occasional scene, taken almost at hazard, in a slowly changing
+drama.
+</para>
+
+<para>
+We can clearly understand why a species when once lost should never
+reappear, even if the very same conditions of life, organic and inorganic,
+should recur.  For though the offspring of one species might be adapted
+(and no doubt this has occurred in innumerable instances) to fill the exact
+place of another species in the economy of nature, and thus supplant it;
+yet the two forms--the old and the new--would not be identically the same;
+for both would almost certainly inherit different characters from their
+distinct progenitors.  For instance, it is just possible, if our
+fantail-pigeons were all destroyed, that fanciers, by striving during long
+ages for the same object, might make a new breed hardly distinguishable
+from our present fantail; but if the parent rock-pigeon were also
+destroyed, and in nature we have every reason to believe that the
+parent-form will generally be supplanted and exterminated by its improved
+offspring, it is quite incredible that a fantail, identical with the
+existing breed, could be raised from any other species of pigeon, or even
+from the other well-established races of the domestic pigeon, for the
+newly-formed fantail would be almost sure to inherit from its new
+progenitor some slight characteristic differences.
+</para>
+
+<para>
+Groups of species, that is, genera and families, follow the same general
+rules in their appearance and disappearance as do single species, changing
+more or less quickly, and in a greater or lesser degree.  A group does not
+reappear after it has once disappeared; or its existence, as long as it
+lasts, is continuous.  I am aware that there are some apparent exceptions
+to this rule, but the exceptions are surprisingly few, so few, that E.
+Forbes, Pictet, and Woodward (though all strongly opposed to such views as
+I maintain) admit its truth; and the rule strictly accords with my theory. 
+For as all the species of the same group have descended from some one
+species, it is clear that as long as any species of the group have appeared
+in the long succession of ages, so long must its members have continuously
+existed, in order to have generated either new and modified or the same old
+and unmodified forms.  Species of the genus Lingula, for instance, must
+have continuously existed by an unbroken succession of generations, from
+the lowest Silurian stratum to the present day.
+</para>
+
+<para>
+We have seen in the last chapter that the species of a group sometimes
+falsely appear to have come in abruptly; and I have attempted to give an
+explanation of this fact, which if true would have been fatal to my views. 
+But such cases are certainly exceptional; the general rule being a gradual
+increase in number, till the group reaches its maximum, and then, sooner or
+later, it gradually decreases.  If the number of the species of a genus, or
+the number of the genera of a family, be represented by a vertical line of
+varying thickness, crossing the successive geological formations in which
+the species are found, the line will sometimes falsely appear to begin at
+its lower end, not in a sharp point, but abruptly; it then gradually
+thickens upwards, sometimes keeping for a space of equal thickness, and
+ultimately thins out in the upper beds, marking the decrease and final
+extinction of the species.  This gradual increase in number of the species
+of a group is strictly conformable with my theory; as the species of the
+same genus, and the genera of the same family, can increase only slowly and
+progressively; for the process of modification and the production of a
+number of allied forms must be slow and gradual,--one species giving rise
+first to two or three varieties, these being slowly converted into species,
+which in their turn produce by equally slow steps other species, and so on,
+like the branching of a great tree from a single stem, till the group
+becomes large.
+</para>
+
+<para>
+On Extinction. -- We have as yet spoken only incidentally of the
+disappearance of species and of groups of species.  On the theory of
+natural selection the extinction of old forms and the production of new and
+improved forms are intimately connected together.  The old notion of all
+the inhabitants of the earth having been swept away at successive periods
+by catastrophes, is very generally given up, even by those geologists, as
+Elie de Beaumont, Murchison, Barrande, &amp;c., whose general views would
+naturally lead them to this conclusion.  On the contrary, we have every
+reason to believe, from the study of the tertiary formations, that species
+and groups of species gradually disappear, one after another, first from
+one spot, then from another, and finally from the world.  Both single
+species and whole groups of species last for very unequal periods; some
+groups, as we have seen, having endured from the earliest known dawn of
+life to the present day; some having disappeared before the close of the
+palaeozoic period.  No fixed law seems to determine the length of time
+during which any single species or any single genus endures.  There is
+reason to believe that the complete extinction of the species of a group is
+generally a slower process than their production:  if the appearance and
+disappearance of a group of species be represented, as before, by a
+vertical line of varying thickness, the line is found to taper more
+gradually at its upper end, which marks the progress of extermination, than
+at its lower end, which marks the first appearance and increase in numbers
+of the species.  In some cases, however, the extermination of whole groups
+of beings, as of ammonites towards the close of the secondary period, has
+been wonderfully sudden.
+</para>
+
+<para>
+The whole subject of the extinction of species has been involved in the
+most gratuitous mystery.  Some authors have even supposed that as the
+individual has a definite length of life, so have species a definite
+duration.  No one I think can have marvelled more at the extinction of
+species, than I have done.  When I found in La Plata the tooth of a horse
+embedded with the remains of Mastodon, Megatherium, Toxodon, and other
+extinct monsters, which all co-existed with still living shells at a very
+late geological period, I was filled with astonishment; for seeing that the
+horse, since its introduction by the Spaniards into South America, has run
+wild over the whole country and has increased in numbers at an unparalleled
+rate, I asked myself what could so recently have exterminated the former
+horse under conditions of life apparently so favourable.  But how utterly
+groundless was my astonishment!  Professor Owen soon perceived that the
+tooth, though so like that of the existing horse, belonged to an extinct
+species.  Had this horse been still living, but in some degree rare, no
+naturalist would have felt the least surprise at its rarity; for rarity is
+the attribute of a vast number of species of all classes, in all countries. 
+If we ask ourselves why this or that species is rare, we answer that
+something is unfavourable in its conditions of life; but what that
+something is, we can hardly ever tell.  On the supposition of the fossil
+horse still existing as a rare species, we might have felt certain from the
+analogy of all other mammals, even of the slow-breeding elephant, and from
+the history of the naturalisation of the domestic horse in South America,
+that under more favourable conditions it would in a very few years have
+stocked the whole continent.  But we could not have told what the
+unfavourable conditions were which checked its increase, whether some one
+or several contingencies, and at what period of the horse's life, and in
+what degree, they severally acted.  If the conditions had gone on, however
+slowly, becoming less and less favourable, we assuredly should not have
+perceived the fact, yet the fossil horse would certainly have become rarer
+and rarer, and finally extinct;--its place being seized on by some more
+successful competitor.
+</para>
+
+<para>
+It is most difficult always to remember that the increase of every living
+being is constantly being checked by unperceived injurious agencies; and
+that these same unperceived agencies are amply sufficient to cause rarity,
+and finally extinction.  We see in many cases in the more recent tertiary
+formations, that rarity precedes extinction; and we know that this has been
+the progress of events with those animals which have been exterminated,
+either locally or wholly, through man's agency.  I may repeat what I
+published in 1845, namely, that to admit that species generally become rare
+before they become extinct--to feel no surprise at the rarity of a species,
+and yet to marvel greatly when it ceases to exist, is much the same as to
+admit that sickness in the individual is the forerunner of death--to feel
+no surprise at sickness, but when the sick man dies, to wonder and to
+suspect that he died by some unknown deed of violence.
+</para>
+
+<para>
+The theory of natural selection is grounded on the belief that each new
+variety, and ultimately each new species, is produced and maintained by
+having some advantage over those with which it comes into competition; and
+the consequent extinction of less-favoured forms almost inevitably follows. 
+It is the same with our domestic productions:  when a new and slightly
+improved variety has been raised, it at first supplants the less improved
+varieties in the same neighbourhood; when much improved it is transported
+far and near, like our short-horn cattle, and takes the place of other
+breeds in other countries.  Thus the appearance of new forms and the
+disappearance of old forms, both natural and artificial, are bound
+together.  In certain flourishing groups, the number of new specific forms
+which have been produced within a given time is probably greater than that
+of the old forms which have been exterminated; but we know that the number
+of species has not gone on indefinitely increasing, at least during the
+later geological periods, so that looking to later times we may believe
+that the production of new forms has caused the extinction of about the
+same number of old forms.
+</para>
+
+<para>
+The competition will generally be most severe, as formerly explained and
+illustrated by examples, between the forms which are most like each other
+in all respects.  Hence the improved and modified descendants of a species
+will generally cause the extermination of the parent-species; and if many
+new forms have been developed from any one species, the nearest allies of
+that species, i.e. the species of the same genus, will be the most liable
+to extermination.  Thus, as I believe, a number of new species descended
+from one species, that is a new genus, comes to supplant an old genus,
+belonging to the same family.  But it must often have happened that a new
+species belonging to some one group will have seized on the place occupied
+by a species belonging to a distinct group, and thus caused its
+extermination; and if many allied forms be developed from the successful
+intruder, many will have to yield their places; and it will generally be
+allied forms, which will suffer from some inherited inferiority in common. 
+But whether it be species belonging to the same or to a distinct class,
+which yield their places to other species which have been modified and
+improved, a few of the sufferers may often long be preserved, from being
+fitted to some peculiar line of life, or from inhabiting some distant and
+isolated station, where they have escaped severe competition.  For
+instance, a single species of Trigonia, a great genus of shells in the
+secondary formations, survives in the Australian seas; and a few members of
+the great and almost extinct group of Ganoid fishes still inhabit our fresh
+waters.  Therefore the utter extinction of a group is generally, as we have
+seen, a slower process than its production.
+</para>
+
+<para>
+With respect to the apparently sudden extermination of whole families or
+orders, as of Trilobites at the close of the palaeozoic period and of
+Ammonites at the close of the secondary period, we must remember what has
+been already said on the probable wide intervals of time between our
+consecutive formations; and in these intervals there may have been much
+slow extermination.  Moreover, when by sudden immigration or by unusually
+rapid development, many species of a new group have taken possession of a
+new area, they will have exterminated in a correspondingly rapid manner
+many of the old inhabitants; and the forms which thus yield their places
+will commonly be allied, for they will partake of some inferiority in
+common.
+</para>
+
+<para>
+Thus, as it seems to me, the manner in which single species and whole
+groups of species become extinct, accords well with the theory of natural
+selection.  We need not marvel at extinction; if we must marvel, let it be
+at our presumption in imagining for a moment that we understand the many
+complex contingencies, on which the existence of each species depends.  If
+we forget for an instant, that each species tends to increase inordinately,
+and that some check is always in action, yet seldom perceived by us, the
+whole economy of nature will be utterly obscured.  Whenever we can
+precisely say why this species is more abundant in individuals than that;
+why this species and not another can be naturalised in a given country;
+then, and not till then, we may justly feel surprise why we cannot account
+for the extinction of this particular species or group of species.
+</para>
+
+<para>
+On the Forms of Life changing almost simultaneously throughout the World. -
+- Scarcely any palaeontological discovery is more striking than the fact,
+that the forms of life change almost simultaneously throughout the world. 
+Thus our European Chalk formation can be recognised in many distant parts
+of the world, under the most different climates, where not a fragment of
+the mineral chalk itself can be found; namely, in North America, in
+equatorial South America, in Tierra del Fuego, at the Cape of Good Hope,
+and in the peninsula of India.  For at these distant points, the organic
+remains in certain beds present an unmistakeable degree of resemblance to
+those of the Chalk.  It is not that the same species are met with; for in
+some cases not one species is identically the same, but they belong to the
+same families, genera, and sections of genera, and sometimes are similarly
+characterised in such trifling points as mere superficial sculpture. 
+Moreover other forms, which are not found in the Chalk of Europe, but which
+occur in the formations either above or below, are similarly absent at
+these distant points of the world.  In the several successive palaeozoic
+formations of Russia, Western Europe and North America, a similar
+parallelism in the forms of life has been observed by several authors:  so
+it is, according to Lyell, with the several European and North American
+tertiary deposits.  Even if the few fossil species which are common to the
+Old and New Worlds be kept wholly out of view, the general parallelism in
+the successive forms of life, in the stages of the widely separated
+palaeozoic and tertiary periods, would still be manifest, and the several
+formations could be easily correlated.
+</para>
+
+<para>
+These observations, however, relate to the marine inhabitants of distant
+parts of the world:  we have not sufficient data to judge whether the
+productions of the land and of fresh water change at distant points in the
+same parallel manner.  We may doubt whether they have thus changed:  if the
+Megatherium, Mylodon, Macrauchenia, and Toxodon had been brought to Europe
+from La Plata, without any information in regard to their geological
+position, no one would have suspected that they had coexisted with still
+living sea-shells; but as these anomalous monsters coexisted with the
+Mastodon and Horse, it might at least have been inferred that they had
+lived during one of the latter tertiary stages.
+</para>
+
+<para>
+When the marine forms of life are spoken of as having changed
+simultaneously throughout the world, it must not be supposed that this
+expression relates to the same thousandth or hundred-thousandth year, or
+even that it has a very strict geological sense; for if all the marine
+animals which live at the present day in Europe, and all those that lived
+in Europe during the pleistocene period (an enormously remote period as
+measured by years, including the whole glacial epoch), were to be compared
+with those now living in South America or in Australia, the most skilful
+naturalist would hardly be able to say whether the existing or the
+pleistocene inhabitants of Europe resembled most closely those of the
+southern hemisphere.  So, again, several highly competent observers believe
+that the existing productions of the United States are more closely related
+to those which lived in Europe during certain later tertiary stages, than
+to those which now live here; and if this be so, it is evident that
+fossiliferous beds deposited at the present day on the shores of North
+America would hereafter be liable to be classed with somewhat older
+European beds.  Nevertheless, looking to a remotely future epoch, there
+can, I think, be little doubt that all the more modern marine formations,
+namely, the upper pliocene, the pleistocene and strictly modern beds, of
+Europe, North and South America, and Australia, from containing fossil
+remains in some degree allied, and from not including those forms which are
+only found in the older underlying deposits, would be correctly ranked as
+simultaneous in a geological sense.
+</para>
+
+<para>
+The fact of the forms of life changing simultaneously, in the above large
+sense, at distant parts of the world, has greatly struck those admirable
+observers, MM. de Verneuil and d'Archiac.  After referring to the
+parallelism of the palaeozoic forms of life in various parts of Europe,
+they add, 'If struck by this strange sequence, we turn our attention to
+North America, and there discover a series of analogous phenomena, it will
+appear certain that all these modifications of species, their extinction,
+and the introduction of new ones, cannot be owing to mere changes in marine
+currents or other causes more or less local and temporary, but depend on
+general laws which govern the whole animal kingdom.'  M. Barrande has made
+forcible remarks to precisely the same effect.  It is, indeed, quite futile
+to look to changes of currents, climate, or other physical conditions, as
+the cause of these great mutations in the forms of life throughout the
+world, under the most different climates.  We must, as Barrande has
+remarked, look to some special law.  We shall see this more clearly when we
+treat of the present distribution of organic beings, and find how slight is
+the relation between the physical conditions of various countries, and the
+nature of their inhabitants.
+</para>
+
+<para>
+This great fact of the parallel succession of the forms of life throughout
+the world, is explicable on the theory of natural selection.  New species
+are formed by new varieties arising, which have some advantage over older
+forms; and those forms, which are already dominant, or have some advantage
+over the other forms in their own country, would naturally oftenest give
+rise to new varieties or incipient species; for these latter must be
+victorious in a still higher degree in order to be preserved and to
+survive.  We have distinct evidence on this head, in the plants which are
+dominant, that is, which are commonest in their own homes, and are most
+widely diffused, having produced the greatest number of new varieties.  It
+is also natural that the dominant, varying, and far-spreading species,
+which already have invaded to a certain extent the territories of other
+species, should be those which would have the best chance of spreading
+still further, and of giving rise in new countries to new varieties and
+species.  The process of diffusion may often be very slow, being dependent
+on climatal and geographical changes, or on strange accidents, but in the
+long run the dominant forms will generally succeed in spreading.  The
+diffusion would, it is probable, be slower with the terrestrial inhabitants
+of distinct continents than with the marine inhabitants of the continuous
+sea.  We might therefore expect to find, as we apparently do find, a less
+strict degree of parallel succession in the productions of the land than of
+the sea.
+</para>
+
+<para>
+Dominant species spreading from any region might encounter still more
+dominant species, and then their triumphant course, or even their
+existence, would cease.  We know not at all precisely what are all the
+conditions most favourable for the multiplication of new and dominant
+species; but we can, I think, clearly see that a number of individuals,
+from giving a better chance of the appearance of favourable variations, and
+that severe competition with many already existing forms, would be highly
+favourable, as would be the power of spreading into new territories.  A
+certain amount of isolation, recurring at long intervals of time, would
+probably be also favourable, as before explained.  One quarter of the world
+may have been most favourable for the production of new and dominant
+species on the land, and another for those in the waters of the sea.  If
+two great regions had been for a long period favourably circumstanced in an
+equal degree, whenever their inhabitants met, the battle would be prolonged
+and severe; and some from one birthplace and some from the other might be
+victorious.  But in the course of time, the forms dominant in the highest
+degree, wherever produced, would tend everywhere to prevail.  As they
+prevailed, they would cause the extinction of other and inferior forms; and
+as these inferior forms would be allied in groups by inheritance, whole
+groups would tend slowly to disappear; though here and there a single
+member might long be enabled to survive.
+</para>
+
+<para>
+Thus, as it seems to me, the parallel, and, taken in a large sense,
+simultaneous, succession of the same forms of life throughout the world,
+accords well with the principle of new species having been formed by
+dominant species spreading widely and varying; the new species thus
+produced being themselves dominant owing to inheritance, and to having
+already had some advantage over their parents or over other species; these
+again spreading, varying, and producing new species.  The forms which are
+beaten and which yield their places to the new and victorious forms, will
+generally be allied in groups, from inheriting some inferiority in common;
+and therefore as new and improved groups spread throughout the world, old
+groups will disappear from the world; and the succession of forms in both
+ways will everywhere tend to correspond.
+</para>
+
+<para>
+There is one other remark connected with this subject worth making.  I have
+given my reasons for believing that all our greater fossiliferous
+formations were deposited during periods of subsidence; and that blank
+intervals of vast duration occurred during the periods when the bed of the
+sea was either stationary or rising, and likewise when sediment was not
+thrown down quickly enough to embed and preserve organic remains.  During
+these long and blank intervals I suppose that the inhabitants of each
+region underwent a considerable amount of modification and extinction, and
+that there was much migration from other parts of the world.  As we have
+reason to believe that large areas are affected by the same movement, it is
+probable that strictly contemporaneous formations have often been
+accumulated over very wide spaces in the same quarter of the world; but we
+are far from having any right to conclude that this has invariably been the
+case, and that large areas have invariably been affected by the same
+movements.  When two formations have been deposited in two regions during
+nearly, but not exactly the same period, we should find in both, from the
+causes explained in the foregoing paragraphs, the same general succession
+in the forms of life; but the species would not exactly correspond; for
+there will have been a little more time in the one region than in the other
+for modification, extinction, and immigration.
+</para>
+
+<para>
+I suspect that cases of this nature have occurred in Europe.  Mr.
+Prestwich, in his admirable Memoirs on the eocene deposits of England and
+France, is able to draw a close general parallelism between the successive
+stages in the two countries; but when he compares certain stages in England
+with those in France, although he finds in both a curious accordance in the
+numbers of the species belonging to the same genera, yet the species
+themselves differ in a manner very difficult to account for, considering
+the proximity of the two areas,--unless, indeed, it be assumed that an
+isthmus separated two seas inhabited by distinct, but contemporaneous,
+faunas.  Lyell has made similar observations on some of the later tertiary
+formations.  Barrande, also, shows that there is a striking general
+parallelism in the successive Silurian deposits of Bohemia and Scandinavia;
+nevertheless he finds a surprising amount of difference in the species.  If
+the several formations in these regions have not been deposited during the
+same exact periods,--a formation in one region often corresponding with a
+blank interval in the other,--and if in both regions the species have gone
+on slowly changing during the accumulation of the several formations and
+during the long intervals of time between them; in this case, the several
+formations in the two regions could be arranged in the same order, in
+accordance with the general succession of the form of life, and the order
+would falsely appear to be strictly parallel; nevertheless the species
+would not all be the same in the apparently corresponding stages in the two
+regions.
+</para>
+
+<para>
+On the Affinities of extinct Species to each other, and to living forms. --
+Let us now look to the mutual affinities of extinct and living species. 
+They all fall into one grand natural system; and this fact is at once
+explained on the principle of descent.  The more ancient any form is, the
+more, as a general rule, it differs from living forms.  But, as Buckland
+long ago remarked, all fossils can be classed either in still existing
+groups, or between them.  That the extinct forms of life help to fill up
+the wide intervals between existing genera, families, and orders, cannot be
+disputed.  For if we confine our attention either to the living or to the
+extinct alone, the series is far less perfect than if we combine both into
+one general system.  With respect to the Vertebrata, whole pages could be
+filled with striking illustrations from our great palaeontologist, Owen,
+showing how extinct animals fall in between existing groups.  Cuvier ranked
+the Ruminants and Pachyderms, as the two most distinct orders of mammals;
+but Owen has discovered so many fossil links, that he has had to alter the
+whole classification of these two orders; and has placed certain pachyderms
+in the same sub-order with ruminants:  for example, he dissolves by fine
+gradations the apparently wide difference between the pig and the camel. 
+In regard to the Invertebrata, Barrande, and a higher authority could not
+be named, asserts that he is every day taught that palaeozoic animals,
+though belonging to the same orders, families, or genera with those living
+at the present day, were not at this early epoch limited in such distinct
+groups as they now are.
+</para>
+
+<para>
+Some writers have objected to any extinct species or group of species being
+considered as intermediate between living species or groups.  If by this
+term it is meant that an extinct form is directly intermediate in all its
+characters between two living forms, the objection is probably valid.  But
+I apprehend that in a perfectly natural classification many fossil species
+would have to stand between living species, and some extinct genera between
+living genera, even between genera belonging to distinct families.  The
+most common case, especially with respect to very distinct groups, such as
+fish and reptiles, seems to be, that supposing them to be distinguished at
+the present day from each other by a dozen characters, the ancient members
+of the same two groups would be distinguished by a somewhat lesser number
+of characters, so that the two groups, though formerly quite distinct, at
+that period made some small approach to each other.
+</para>
+
+<para>
+It is a common belief that the more ancient a form is, by so much the more
+it tends to connect by some of its characters groups now widely separated
+from each other.  This remark no doubt must be restricted to those groups
+which have undergone much change in the course of geological ages; and it
+would be difficult to prove the truth of the proposition, for every now and
+then even a living animal, as the Lepidosiren, is discovered having
+affinities directed towards very distinct groups.  Yet if we compare the
+older Reptiles and Batrachians, the older Fish, the older Cephalopods, and
+the eocene Mammals, with the more recent members of the same classes, we
+must admit that there is some truth in the remark.
+</para>
+
+<para>
+Let us see how far these several facts and inferences accord with the
+theory of descent with modification.  As the subject is somewhat complex, I
+must request the reader to turn to the diagram in the fourth chapter.  We
+may suppose that the numbered letters represent genera, and the dotted
+lines diverging from them the species in each genus.  The diagram is much
+too simple, too few genera and too few species being given, but this is
+unimportant for us.  The horizontal lines may represent successive
+geological formations, and all the forms beneath the uppermost line may be
+considered as extinct.  The three existing genera, a14, q14, p14, will form
+a small family; b14 and f14 a closely allied family or sub-family; and o14,
+e14, m14, a third family.  These three families, together with the many
+extinct genera on the several lines of descent diverging from the
+parent-form A, will form an order; for all will have inherited something in
+common from their ancient and common progenitor.  On the principle of the
+continued tendency to divergence of character, which was formerly
+illustrated by this diagram, the more recent any form is, the more it will
+generally differ from its ancient progenitor.  Hence we can understand the
+rule that the most ancient fossils differ most from existing forms.  We
+must not, however, assume that divergence of character is a necessary
+contingency; it depends solely on the descendants from a species being thus
+enabled to seize on many and different places in the economy of nature. 
+Therefore it is quite possible, as we have seen in the case of some
+Silurian forms, that a species might go on being slightly modified in
+relation to its slightly altered conditions of life, and yet retain
+throughout a vast period the same general characteristics.  This is
+represented in the diagram by the letter F14.
+</para>
+
+<para>
+All the many forms, extinct and recent, descended from A, make, as before
+remarked, one order; and this order, from the continued effects of
+extinction and divergence of character, has become divided into several
+sub-families and families, some of which are supposed to have perished at
+different periods, and some to have endured to the present day.
+</para>
+
+<para>
+By looking at the diagram we can see that if many of the extinct forms,
+supposed to be embedded in the successive formations, were discovered at
+several points low down in the series, the three existing families on the
+uppermost line would be rendered less distinct from each other.  If, for
+instance, the genera a1, a5, a10, f8, m3, m6, m9 were disinterred, these
+three families would be so closely linked together that they probably would
+have to be united into one great family, in nearly the same manner as has
+occurred with ruminants and pachyderms.  Yet he who objected to call the
+extinct genera, which thus linked the living genera of three families
+together, intermediate in character, would be justified, as they are
+intermediate, not directly, but only by a long and circuitous course
+through many widely different forms.  If many extinct forms were to be
+discovered above one of the middle horizontal lines or geological
+formations--for instance, above No. VI.--but none from beneath this line,
+then only the two families on the left hand (namely, a14, &amp;c., and b14,
+&amp;c.) would have to be united into one family; and the two other families
+(namely, a14 to f14 now including five genera, and o14 to m14) would yet
+remain distinct.  These two families, however, would be less distinct from
+each other than they were before the discovery of the fossils.  If, for
+instance, we suppose the existing genera of the two families to differ from
+each other by a dozen characters, in this case the genera, at the early
+period marked VI., would differ by a lesser number of characters; for at
+this early stage of descent they have not diverged in character from the
+common progenitor of the order, nearly so much as they subsequently
+diverged.  Thus it comes that ancient and extinct genera are often in some
+slight degree intermediate in character between their modified descendants,
+or between their collateral relations.
+</para>
+
+<para>
+In nature the case will be far more complicated than is represented in the
+diagram; for the groups will have been more numerous, they will have
+endured for extremely unequal lengths of time, and will have been modified
+in various degrees.  As we possess only the last volume of the geological
+record, and that in a very broken condition, we have no right to expect,
+except in very rare cases, to fill up wide intervals in the natural system,
+and thus unite distinct families or orders.  All that we have a right to
+expect, is that those groups, which have within known geological periods
+undergone much modification, should in the older formations make some
+slight approach to each other; so that the older members should differ less
+from each other in some of their characters than do the existing members of
+the same groups; and this by the concurrent evidence of our best
+palaeontologists seems frequently to be the case.
+</para>
+
+<para>
+Thus, on the theory of descent with modification, the main facts with
+respect to the mutual affinities of the extinct forms of life to each other
+and to living forms, seem to me explained in a satisfactory manner.  And
+they are wholly inexplicable on any other view.
+</para>
+
+<para>
+On this same theory, it is evident that the fauna of any great period in
+the earth's history will be intermediate in general character between that
+which preceded and that which succeeded it.  Thus, the species which lived
+at the sixth great stage of descent in the diagram are the modified
+offspring of those which lived at the fifth stage, and are the parents of
+those which became still more modified at the seventh stage; hence they
+could hardly fail to be nearly intermediate in character between the forms
+of life above and below.  We must, however, allow for the entire extinction
+of some preceding forms, and for the coming in of quite new forms by
+immigration, and for a large amount of modification, during the long and
+blank intervals between the successive formations.  Subject to these
+allowances, the fauna of each geological period undoubtedly is intermediate
+in character, between the preceding and succeeding faunas.  I need give
+only one instance, namely, the manner in which the fossils of the Devonian
+system, when this system was first discovered, were at once recognised by
+palaeontologists as intermediate in character between those of the
+overlying carboniferous, and underlying Silurian system.  But each fauna is
+not necessarily exactly intermediate, as unequal intervals of time have
+elapsed between consecutive formations.
+</para>
+
+<para>
+It is no real objection to the truth of the statement, that the fauna of
+each period as a whole is nearly intermediate in character between the
+preceding and succeeding faunas, that certain genera offer exceptions to
+the rule.  For instance, mastodons and elephants, when arranged by Dr.
+Falconer in two series, first according to their mutual affinities and then
+according to their periods of existence, do not accord in arrangement.  The
+species extreme in character are not the oldest, or the most recent; nor
+are those which are intermediate in character, intermediate in age.  But
+supposing for an instant, in this and other such cases, that the record of
+the first appearance and disappearance of the species was perfect, we have
+no reason to believe that forms successively produced necessarily endure
+for corresponding lengths of time:  a very ancient form might occasionally
+last much longer than a form elsewhere subsequently produced, especially in
+the case of terrestrial productions inhabiting separated districts.  To
+compare small things with great:  if the principal living and extinct races
+of the domestic pigeon were arranged as well as they could be in serial
+affinity, this arrangement would not closely accord with the order in time
+of their production, and still less with the order of their disappearance;
+for the parent rock-pigeon now lives; and many varieties between the
+rock-pigeon and the carrier have become extinct; and carriers which are
+extreme in the important character of length of beak originated earlier
+than short-beaked tumblers, which are at the opposite end of the series in
+this same respect.
+</para>
+
+<para>
+Closely connected with the statement, that the organic remains from an
+intermediate formation are in some degree intermediate in character, is the
+fact, insisted on by all palaeontologists, that fossils from two
+consecutive formations are far more closely related to each other, than are
+the fossils from two remote formations.  Pictet gives as a well-known
+instance, the general resemblance of the organic remains from the several
+stages of the chalk formation, though the species are distinct in each
+stage.  This fact alone, from its generality, seems to have shaken
+Professor Pictet in his firm belief in the immutability of species.  He who
+is acquainted with the distribution of existing species over the globe,
+will not attempt to account for the close resemblance of the distinct
+species in closely consecutive formations, by the physical conditions of
+the ancient areas having remained nearly the same.  Let it be remembered
+that the forms of life, at least those inhabiting the sea, have changed
+almost simultaneously throughout the world, and therefore under the most
+different climates and conditions.  Consider the prodigious vicissitudes of
+climate during the pleistocene period, which includes the whole glacial
+period, and note how little the specific forms of the inhabitants of the
+sea have been affected.
+</para>
+
+<para>
+On the theory of descent, the full meaning of the fact of fossil remains
+from closely consecutive formations, though ranked as distinct species,
+being closely related, is obvious.  As the accumulation of each formation
+has often been interrupted, and as long blank intervals have intervened
+between successive formations, we ought not to expect to find, as I
+attempted to show in the last chapter, in any one or two formations all the
+intermediate varieties between the species which appeared at the
+commencement and close of these periods; but we ought to find after
+intervals, very long as measured by years, but only moderately long as
+measured geologically, closely allied forms, or, as they have been called
+by some authors, representative species; and these we assuredly do find. 
+We find, in short, such evidence of the slow and scarcely sensible mutation
+of specific forms, as we have a just right to expect to find.
+</para>
+
+<para>
+On the state of Development of Ancient Forms. -- There has been much
+discussion whether recent forms are more highly developed than ancient.  I
+will not here enter on this subject, for naturalists have not as yet
+defined to each other's satisfaction what is meant by high and low forms. 
+But in one particular sense the more recent forms must, on my theory, be
+higher than the more ancient; for each new species is formed by having had
+some advantage in the struggle for life over other and preceding forms.  If
+under a nearly similar climate, the eocene inhabitants of one quarter of
+the world were put into competition with the existing inhabitants of the
+same or some other quarter, the eocene fauna or flora would certainly be
+beaten and exterminated; as would a secondary fauna by an eocene, and a
+palaeozoic fauna by a secondary fauna.  I do not doubt that this process of
+improvement has affected in a marked and sensible manner the organisation
+of the more recent and victorious forms of life, in comparison with the
+ancient and beaten forms; but I can see no way of testing this sort of
+progress.  Crustaceans, for instance, not the highest in their own class,
+may have beaten the highest molluscs.  From the extraordinary manner in
+which European productions have recently spread over New Zealand, and have
+seized on places which must have been previously occupied, we may believe,
+if all the animals and plants of Great Britain were set free in New
+Zealand, that in the course of time a multitude of British forms would
+become thoroughly naturalized there, and would exterminate many of the
+natives.  On the other hand, from what we see now occurring in New Zealand,
+and from hardly a single inhabitant of the southern hemisphere having
+become wild in any part of Europe, we may doubt, if all the productions of
+New Zealand were set free in Great Britain, whether any considerable number
+would be enabled to seize on places now occupied by our native plants and
+animals.  Under this point of view, the productions of Great Britain may be
+said to be higher than those of New Zealand.  Yet the most skilful
+naturalist from an examination of the species of the two countries could
+not have foreseen this result.
+</para>
+
+<para>
+Agassiz insists that ancient animals resemble to a certain extent the
+embryos of recent animals of the same classes; or that the geological
+succession of extinct forms is in some degree parallel to the embryological
+development of recent forms.  I must follow Pictet and Huxley in thinking
+that the truth of this doctrine is very far from proved.  Yet I fully
+expect to see it hereafter confirmed, at least in regard to subordinate
+groups, which have branched off from each other within comparatively recent
+times.  For this doctrine of Agassiz accords well with the theory of
+natural selection.  In a future chapter I shall attempt to show that the
+adult differs from its embryo, owing to variations supervening at a not
+early age, and being inherited at a corresponding age.  This process,
+whilst it leaves the embryo almost unaltered, continually adds, in the
+course of successive generations, more and more difference to the adult.
+</para>
+
+<para>
+Thus the embryo comes to be left as a sort of picture, preserved by nature,
+of the ancient and less modified condition of each animal.  This view may
+be true, and yet it may never be capable of full proof.  Seeing, for
+instance, that the oldest known mammals, reptiles, and fish strictly belong
+to their own proper classes, though some of these old forms are in a slight
+degree less distinct from each other than are the typical members of the
+same groups at the present day, it would be vain to look for animals having
+the common embryological character of the Vertebrata, until beds far
+beneath the lowest Silurian strata are discovered--a discovery of which the
+chance is very small.
+</para>
+
+<para>
+On the Succession of the same Types within the same areas, during the later
+tertiary periods. -- Mr. Clift many years ago showed that the fossil
+mammals from the Australian caves were closely allied to the living
+marsupials of that continent.  In South America, a similar relationship is
+manifest, even to an uneducated eye, in the gigantic pieces of armour like
+those of the armadillo, found in several parts of La Plata; and Professor
+Owen has shown in the most striking manner that most of the fossil mammals,
+buried there in such numbers, are related to South American types.  This
+relationship is even more clearly seen in the wonderful collection of
+fossil bones made by MM. Lund and Clausen in the caves of Brazil.  I was so
+much impressed with these facts that I strongly insisted, in 1839 and 1845,
+on this 'law of the succession of types,'--on 'this wonderful relationship
+in the same continent between the dead and the living.'  Professor Owen has
+subsequently extended the same generalisation to the mammals of the Old
+World.  We see the same law in this author's restorations of the extinct
+and gigantic birds of New Zealand.  We see it also in the birds of the
+caves of Brazil.  Mr. Woodward has shown that the same law holds good with
+sea-shells, but from the wide distribution of most genera of molluscs, it
+is not well displayed by them.  Other cases could be added, as the relation
+between the extinct and living land-shells of Madeira; and between the
+extinct and living brackish-water shells of the Aralo-Caspian Sea.
+</para>
+
+<para>
+Now what does this remarkable law of the succession of the same types
+within the same areas mean?  He would be a bold man, who after comparing
+the present climate of Australia and of parts of South America under the
+same latitude, would attempt to account, on the one hand, by dissimilar
+physical conditions for the dissimilarity of the inhabitants of these two
+continents, and, on the other hand, by similarity of conditions, for the
+uniformity of the same types in each during the later tertiary periods. 
+Nor can it be pretended that it is an immutable law that marsupials should
+have been chiefly or solely produced in Australia; or that Edentata and
+other American types should have been solely produced in South America. 
+For we know that Europe in ancient times was peopled by numerous
+marsupials; and I have shown in the publications above alluded to, that in
+America the law of distribution of terrestrial mammals was formerly
+different from what it now is.  North America formerly partook strongly of
+the present character of the southern half of the continent; and the
+southern half was formerly more closely allied, than it is at present, to
+the northern half.  In a similar manner we know from Falconer and Cautley's
+discoveries, that northern India was formerly more closely related in its
+mammals to Africa than it is at the present time.  Analogous facts could be
+given in relation to the distribution of marine animals.
+</para>
+
+<para>
+On the theory of descent with modification, the great law of the long
+enduring, but not immutable, succession of the same types within the same
+areas, is at once explained; for the inhabitants of each quarter of the
+world will obviously tend to leave in that quarter, during the next
+succeeding period of time, closely allied though in some degree modified
+descendants.  If the inhabitants of one continent formerly differed greatly
+from those of another continent, so will their modified descendants still
+differ in nearly the same manner and degree.  But after very long intervals
+of time and after great geographical changes, permitting much
+inter-migration, the feebler will yield to the more dominant forms, and
+there will be nothing immutable in the laws of past and present
+distribution.
+</para>
+
+<para>
+It may be asked in ridicule, whether I suppose that the megatherium and
+other allied huge monsters have left behind them in South America the
+sloth, armadillo, and anteater, as their degenerate descendants.  This
+cannot for an instant be admitted.  These huge animals have become wholly
+extinct, and have left no progeny.  But in the caves of Brazil, there are
+many extinct species which are closely allied in size and in other
+characters to the species still living in South America; and some of these
+fossils may be the actual progenitors of living species.  It must not be
+forgotten that, on my theory, all the species of the same genus have
+descended from some one species; so that if six genera, each having eight
+species, be found in one geological formation, and in the next succeeding
+formation there be six other allied or representative genera with the same
+number of species, then we may conclude that only one species of each of
+the six older genera has left modified descendants, constituting the six
+new genera.  The other seven species of the old genera have all died out
+and have left no progeny.  Or, which would probably be a far commoner case,
+two or three species of two or three alone of the six older genera will
+have been the parents of the six new genera; the other old species and the
+other whole genera having become utterly extinct.  In failing orders, with
+the genera and species decreasing in numbers, as apparently is the case of
+the Edentata of South America, still fewer genera and species will have
+left modified blood-descendants.
+</para>
+
+<para>
+Summary of the preceding and present Chapters -- I have attempted to show
+that the geological record is extremely imperfect; that only a small
+portion of the globe has been geologically explored with care; that only
+certain classes of organic beings have been largely preserved in a fossil
+state; that the number both of specimens and of species, preserved in our
+museums, is absolutely as nothing compared with the incalculable number of
+generations which must have passed away even during a single formation;
+that, owing to subsidence being necessary for the accumulation of
+fossiliferous deposits thick enough to resist future degradation, enormous
+intervals of time have elapsed between the successive formations; that
+there has probably been more extinction during the periods of subsidence,
+and more variation during the periods of elevation, and during the latter
+the record will have been least perfectly kept; that each single formation
+has not been continuously deposited; that the duration of each formation
+is, perhaps, short compared with the average duration of specific forms;
+that migration has played an important part in the first appearance of new
+forms in any one area and formation; that widely ranging species are those
+which have varied most, and have oftenest given rise to new species; and
+that varieties have at first often been local.  All these causes taken
+conjointly, must have tended to make the geological record extremely
+imperfect, and will to a large extent explain why we do not find
+interminable varieties, connecting together all the extinct and existing
+forms of life by the finest graduated steps.
+</para>
+
+<para>
+He who rejects these views on the nature of the geological record, will
+rightly reject my whole theory.  For he may ask in vain where are the
+numberless transitional links which must formerly have connected the
+closely allied or representative species, found in the several stages of
+the same great formation.  He may disbelieve in the enormous intervals of
+time which have elapsed between our consecutive formations; he may overlook
+how important a part migration must have played, when the formations of any
+one great region alone, as that of Europe, are considered; he may urge the
+apparent, but often falsely apparent, sudden coming in of whole groups of
+species.  He may ask where are the remains of those infinitely numerous
+organisms which must have existed long before the first bed of the Silurian
+system was deposited:  I can answer this latter question only
+hypothetically, by saying that as far as we can see, where our oceans now
+extend they have for an enormous period extended, and where our oscillating
+continents now stand they have stood ever since the Silurian epoch; but
+that long before that period, the world may have presented a wholly
+different aspect; and that the older continents, formed of formations older
+than any known to us, may now all be in a metamorphosed condition, or may
+lie buried under the ocean.
+</para>
+
+<para>
+Passing from these difficulties, all the other great leading facts in
+palaeontology seem to me simply to follow on the theory of descent with
+modification through natural selection.  We can thus understand how it is
+that new species come in slowly and successively; how species of different
+classes do not necessarily change together, or at the same rate, or in the
+same degree; yet in the long run that all undergo modification to some
+extent.  The extinction of old forms is the almost inevitable consequence
+of the production of new forms.  We can understand why when a species has
+once disappeared it never reappears.  Groups of species increase in numbers
+slowly, and endure for unequal periods of time; for the process of
+modification is necessarily slow, and depends on many complex
+contingencies.  The dominant species of the larger dominant groups tend to
+leave many modified descendants, and thus new sub-groups and groups are
+formed.  As these are formed, the species of the less vigorous groups, from
+their inferiority inherited from a common progenitor, tend to become
+extinct together, and to leave no modified offspring on the face of the
+earth.  But the utter extinction of a whole group of species may often be a
+very slow process, from the survival of a few descendants, lingering in
+protected and isolated situations.  When a group has once wholly
+disappeared, it does not reappear; for the link of generation has been
+broken.
+</para>
+
+<para>
+We can understand how the spreading of the dominant forms of life, which
+are those that oftenest vary, will in the long run tend to people the world
+with allied, but modified, descendants; and these will generally succeed in
+taking the places of those groups of species which are their inferiors in
+the struggle for existence.  Hence, after long intervals of time, the
+productions of the world will appear to have changed simultaneously.
+</para>
+
+<para>
+We can understand how it is that all the forms of life, ancient and recent,
+make together one grand system; for all are connected by generation.  We
+can understand, from the continued tendency to divergence of character, why
+the more ancient a form is, the more it generally differs from those now
+living.  Why ancient and extinct forms often tend to fill up gaps between
+existing forms, sometimes blending two groups previously classed as
+distinct into one; but more commonly only bringing them a little closer
+together.  The more ancient a form is, the more often, apparently, it
+displays characters in some degree intermediate between groups now
+distinct; for the more ancient a form is, the more nearly it will be
+related to, and consequently resemble, the common progenitor of groups,
+since become widely divergent.  Extinct forms are seldom directly
+intermediate between existing forms; but are intermediate only by a long
+and circuitous course through many extinct and very different forms.  We
+can clearly see why the organic remains of closely consecutive formations
+are more closely allied to each other, than are those of remote formations;
+for the forms are more closely linked together by generation:  we can
+clearly see why the remains of an intermediate formation are intermediate
+in character.
+</para>
+
+<para>
+The inhabitants of each successive period in the world's history have
+beaten their predecessors in the race for life, and are, in so far, higher
+in the scale of nature; and this may account for that vague yet ill-defined
+sentiment, felt by many palaeontologists, that organisation on the whole
+has progressed.  If it should hereafter be proved that ancient animals
+resemble to a certain extent the embryos of more recent animals of the same
+class, the fact will be intelligible.  The succession of the same types of
+structure within the same areas during the later geological periods ceases
+to be mysterious, and is simply explained by inheritance.
+</para>
+
+<para>
+If then the geological record be as imperfect as I believe it to be, and it
+may at least be asserted that the record cannot be proved to be much more
+perfect, the main objections to the theory of natural selection are greatly
+diminished or disappear.  On the other hand, all the chief laws of
+palaeontology plainly proclaim, as it seems to me, that species have been
+produced by ordinary generation:  old forms having been supplanted by new
+and improved forms of life, produced by the laws of variation still acting
+round us, and preserved by Natural Selection.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter XI</chapnum>
+
+<title>Geographical Distribution</title>
+
+<chapsummary>
+Present distribution cannot be accounted for by differences in physical
+conditions -- Importance of barriers -- Affinity of the productions of the
+same continent -- Centres of creation -- Means of dispersal, by changes of
+climate and of the level of the land, and by occasional means -- Dispersal
+during the Glacial period co-extensive with the world.
+</chapsummary>
+</chapheader>
+
+<para>
+In considering the distribution of organic beings over the face of the
+globe, the first great fact which strikes us is, that neither the
+similarity nor the dissimilarity of the inhabitants of various regions can
+be accounted for by their climatal and other physical conditions.  Of late,
+almost every author who has studied the subject has come to this
+conclusion.  The case of America alone would almost suffice to prove its
+truth:  for if we exclude the northern parts where the circumpolar land is
+almost continuous, all authors agree that one of the most fundamental
+divisions in geographical distribution is that between the New and Old
+Worlds; yet if we travel over the vast American continent, from the central
+parts of the United States to its extreme southern point, we meet with the
+most diversified conditions; the most humid districts, arid deserts, lofty
+mountains, grassy plains, forests, marshes, lakes, and great rivers, under
+almost every temperature.  There is hardly a climate or condition in the
+Old World which cannot be paralleled in the New--at least as closely as the
+same species generally require; for it is a most rare case to find a group
+of organisms confined to any small spot, having conditions peculiar in only
+a slight degree; for instance, small areas in the Old World could be
+pointed out hotter than any in the New World, yet these are not inhabited
+by a peculiar fauna or flora.  Notwithstanding this parallelism in the
+conditions of the Old and New Worlds, how widely different are their living
+productions!
+</para>
+
+<para>
+In the southern hemisphere, if we compare large tracts of land in
+Australia, South Africa, and western South America, between latitudes 25
+deg and 35 deg, we shall find parts extremely similar in all their
+conditions, yet it would not be possible to point out three faunas and
+floras more utterly dissimilar.  Or again we may compare the productions of
+South America south of lat. 35 deg with those north of 25 deg, which
+consequently inhabit a considerably different climate, and they will be
+found incomparably more closely related to each other, than they are to the
+productions of Australia or Africa under nearly the same climate. 
+Analogous facts could be given with respect to the inhabitants of the sea.
+</para>
+
+<para>
+A second great fact which strikes us in our general review is, that
+barriers of any kind, or obstacles to free migration, are related in a
+close and important manner to the differences between the productions of
+various regions.  We see this in the great difference of nearly all the
+terrestrial productions of the New and Old Worlds, excepting in the
+northern parts, where the land almost joins, and where, under a slightly
+different climate, there might have been free migration for the northern
+temperate forms, as there now is for the strictly arctic productions.  We
+see the same fact in the great difference between the inhabitants of
+Australia, Africa, and South America under the same latitude:  for these
+countries are almost as much isolated from each other as is possible.  On
+each continent, also, we see the same fact; for on the opposite sides of
+lofty and continuous mountain-ranges, and of great deserts, and sometimes
+even of large rivers, we find different productions; though as mountain
+chains, deserts, &amp;c., are not as impassable, or likely to have endured so
+long as the oceans separating continents, the differences are very inferior
+in degree to those characteristic of distinct continents.
+</para>
+
+<para>
+Turning to the sea, we find the same law.  No two marine faunas are more
+distinct, with hardly a fish, shell, or crab in common, than those of the
+eastern and western shores of South and Central America; yet these great
+faunas are separated only by the narrow, but impassable, isthmus of Panama. 
+Westward of the shores of America, a wide space of open ocean extends, with
+not an island as a halting-place for emigrants; here we have a barrier of
+another kind, and as soon as this is passed we meet in the eastern islands
+of the Pacific, with another and totally distinct fauna.  So that here
+three marine faunas range far northward and southward, in parallel lines
+not far from each other, under corresponding climates; but from being
+separated from each other by impassable barriers, either of land or open
+sea, they are wholly distinct.  On the other hand, proceeding still further
+westward from the eastern islands of the tropical parts of the Pacific, we
+encounter no impassable barriers, and we have innumerable islands as
+halting-places, until after travelling over a hemisphere we come to the
+shores of Africa; and over this vast space we meet with no well-defined and
+distinct marine faunas.  Although hardly one shell, crab or fish is common
+to the above-named three approximate faunas of Eastern and Western America
+and the eastern Pacific islands, yet many fish range from the Pacific into
+the Indian Ocean, and many shells are common to the eastern islands of the
+Pacific and the eastern shores of Africa, on almost exactly opposite
+meridians of longitude.
+</para>
+
+<para>
+A third great fact, partly included in the foregoing statements, is the
+affinity of the productions of the same continent or sea, though the
+species themselves are distinct at different points and stations.  It is a
+law of the widest generality, and every continent offers innumerable
+instances.  Nevertheless the naturalist in travelling, for instance, from
+north to south never fails to be struck by the manner in which successive
+groups of beings, specifically distinct, yet clearly related, replace each
+other.  He hears from closely allied, yet distinct kinds of birds, notes
+nearly similar, and sees their nests similarly constructed, but not quite
+alike, with eggs coloured in nearly the same manner.  The plains near the
+Straits of Magellan are inhabited by one species of Rhea (American
+ostrich), and northward the plains of La Plata by another species of the
+same genus; and not by a true ostrich or emeu, like those found in Africa
+and Australia under the same latitude.  On these same plains of La Plata,
+we see the agouti and bizcacha, animals having nearly the same habits as
+our hares and rabbits and belonging to the same order of Rodents, but they
+plainly display an American type of structure.  We ascend the lofty peaks
+of the Cordillera and we find an alpine species of bizcacha; we look to the
+waters, and we do not find the beaver or musk-rat, but the coypu and
+capybara, rodents of the American type.  Innumerable other instances could
+be given.  If we look to the islands off the American shore, however much
+they may differ in geological structure, the inhabitants, though they may
+be all peculiar species, are essentially American.  We may look back to
+past ages, as shown in the last chapter, and we find American types then
+prevalent on the American continent and in the American seas.  We see in
+these facts some deep organic bond, prevailing throughout space and time,
+over the same areas of land and water, and independent of their physical
+conditions.  The naturalist must feel little curiosity, who is not led to
+inquire what this bond is.
+</para>
+
+<para>
+This bond, on my theory, is simply inheritance, that cause which alone, as
+far as we positively know, produces organisms quite like, or, as we see in
+the case of varieties nearly like each other.  The dissimilarity of the
+inhabitants of different regions may be attributed to modification through
+natural selection, and in a quite subordinate degree to the direct
+influence of different physical conditions.  The degree of dissimilarity
+will depend on the migration of the more dominant forms of life from one
+region into another having been effected with more or less ease, at periods
+more or less remote;--on the nature and number of the former
+immigrants;--and on their action and reaction, in their mutual struggles
+for life;--the relation of organism to organism being, as I have already
+often remarked, the most important of all relations.  Thus the high
+importance of barriers comes into play by checking migration; as does time
+for the slow process of modification through natural selection. 
+Widely-ranging species, abounding in individuals, which have already
+triumphed over many competitors in their own widely-extended homes will
+have the best chance of seizing on new places, when they spread into new
+countries.  In their new homes they will be exposed to new conditions, and
+will frequently undergo further modification and improvement; and thus they
+will become still further victorious, and will produce groups of modified
+descendants.  On this principle of inheritance with modification, we can
+understand how it is that sections of genera, whole genera, and even
+families are confined to the same areas, as is so commonly and notoriously
+the case.
+</para>
+
+<para>
+I believe, as was remarked in the last chapter, in no law of necessary
+development.  As the variability of each species is an independent
+property, and will be taken advantage of by natural selection, only so far
+as it profits the individual in its complex struggle for life, so the
+degree of modification in different species will be no uniform quantity. 
+If, for instance, a number of species, which stand in direct competition
+with each other, migrate in a body into a new and afterwards isolated
+country, they will be little liable to modification; for neither migration
+nor isolation in themselves can do anything.  These principles come into
+play only by bringing organisms into new relations with each other, and in
+a lesser degree with the surrounding physical conditions.  As we have seen
+in the last chapter that some forms have retained nearly the same character
+from an enormously remote geological period, so certain species have
+migrated over vast spaces, and have not become greatly modified.
+</para>
+
+<para>
+On these views, it is obvious, that the several species of the same genus,
+though inhabiting the most distant quarters of the world, must originally
+have proceeded from the same source, as they have descended from the same
+progenitor.  In the case of those species, which have undergone during
+whole geological periods but little modification, there is not much
+difficulty in believing that they may have migrated from the same region;
+for during the vast geographical and climatal changes which will have
+supervened since ancient times, almost any amount of migration is possible. 
+But in many other cases, in which we have reason to believe that the
+species of a genus have been produced within comparatively recent times,
+there is great difficulty on this head.  It is also obvious that the
+individuals of the same species, though now inhabiting distant and isolated
+regions, must have proceeded from one spot, where their parents were first
+produced:  for, as explained in the last chapter, it is incredible that
+individuals identically the same should ever have been produced through
+natural selection from parents specifically distinct.
+</para>
+
+<para>
+We are thus brought to the question which has been largely discussed by
+naturalists, namely, whether species have been created at one or more
+points of the earth's surface.  Undoubtedly there are very many cases of
+extreme difficulty, in understanding how the same species could possibly
+have migrated from some one point to the several distant and isolated
+points, where now found.  Nevertheless the simplicity of the view that each
+species was first produced within a single region captivates the mind.  He
+who rejects it, rejects the vera causa of ordinary generation with
+subsequent migration, and calls in the agency of a miracle.  It is
+universally admitted, that in most cases the area inhabited by a species is
+continuous; and when a plant or animal inhabits two points so distant from
+each other, or with an interval of such a nature, that the space could not
+be easily passed over by migration, the fact is given as something
+remarkable and exceptional.  The capacity of migrating across the sea is
+more distinctly limited in terrestrial mammals, than perhaps in any other
+organic beings; and, accordingly, we find no inexplicable cases of the same
+mammal inhabiting distant points of the world.  No geologist will feel any
+difficulty in such cases as Great Britain having been formerly united to
+Europe, and consequently possessing the same quadrupeds.  But if the same
+species can be produced at two separate points, why do we not find a single
+mammal common to Europe and Australia or South America?  The conditions of
+life are nearly the same, so that a multitude of European animals and
+plants have become naturalised in America and Australia; and some of the
+aboriginal plants are identically the same at these distant points of the
+northern and southern hemispheres?  The answer, as I believe, is, that
+mammals have not been able to migrate, whereas some plants, from their
+varied means of dispersal, have migrated across the vast and broken
+interspace.  The great and striking influence which barriers of every kind
+have had on distribution, is intelligible only on the view that the great
+majority of species have been produced on one side alone, and have not been
+able to migrate to the other side.  Some few families, many sub-families,
+very many genera, and a still greater number of sections of genera are
+confined to a single region; and it has been observed by several
+naturalists, that the most natural genera, or those genera in which the
+species are most closely related to each other, are generally local, or
+confined to one area.  What a strange anomaly it would be, if, when coming
+one step lower in the series, to the individuals of the same species, a
+directly opposite rule prevailed; and species were not local, but had been
+produced in two or more distinct areas!
+</para>
+
+<para>
+Hence it seems to me, as it has to many other naturalists, that the view of
+each species having been produced in one area alone, and having
+subsequently migrated from that area as far as its powers of migration and
+subsistence under past and present conditions permitted, is the most
+probable.  Undoubtedly many cases occur, in which we cannot explain how the
+same species could have passed from one point to the other.  But the
+geographical and climatal changes, which have certainly occurred within
+recent geological times, must have interrupted or rendered discontinuous
+the formerly continuous range of many species.  So that we are reduced to
+consider whether the exceptions to continuity of range are so numerous and
+of so grave a nature, that we ought to give up the belief, rendered
+probable by general considerations, that each species has been produced
+within one area, and has migrated thence as far as it could.  It would be
+hopelessly tedious to discuss all the exceptional cases of the same
+species, now living at distant and separated points; nor do I for a moment
+pretend that any explanation could be offered of many such cases.  But
+after some preliminary remarks, I will discuss a few of the most striking
+classes of facts; namely, the existence of the same species on the summits
+of distant mountain-ranges, and at distant points in the arctic and
+antarctic regions; and secondly (in the following chapter), the wide
+distribution of freshwater productions; and thirdly, the occurrence of the
+same terrestrial species on islands and on the mainland, though separated
+by hundreds of miles of open sea.  If the existence of the same species at
+distant and isolated points of the earth's surface, can in many instances
+be explained on the view of each species having migrated from a single
+birthplace; then, considering our ignorance with respect to former climatal
+and geographical changes and various occasional means of transport, the
+belief that this has been the universal law, seems to me incomparably the
+safest.
+</para>
+
+<para>
+In discussing this subject, we shall be enabled at the same time to
+consider a point equally important for us, namely, whether the several
+distinct species of a genus, which on my theory have all descended from a
+common progenitor, can have migrated (undergoing modification during some
+part of their migration) from the area inhabited by their progenitor.  If
+it can be shown to be almost invariably the case, that a region, of which
+most of its inhabitants are closely related to, or belong to the same
+genera with the species of a second region, has probably received at some
+former period immigrants from this other region, my theory will be
+strengthened; for we can clearly understand, on the principle of
+modification, why the inhabitants of a region should be related to those of
+another region, whence it has been stocked.  A volcanic island, for
+instance, upheaved and formed at the distance of a few hundreds of miles
+from a continent, would probably receive from it in the course of time a
+few colonists, and their descendants, though modified, would still be
+plainly related by inheritance to the inhabitants of the continent.  Cases
+of this nature are common, and are, as we shall hereafter more fully see,
+inexplicable on the theory of independent creation.  This view of the
+relation of species in one region to those in another, does not differ much
+(by substituting the word variety for species) from that lately advanced in
+an ingenious paper by Mr. Wallace, in which he concludes, that 'every
+species has come into existence coincident both in space and time with a
+pre-existing closely allied species.'  And I now know from correspondence,
+that this coincidence he attributes to generation with modification.
+</para>
+
+<para>
+The previous remarks on 'single and multiple centres of creation' do not
+directly bear on another allied question,--namely whether all the
+individuals of the same species have descended from a single pair, or
+single hermaphrodite, or whether, as some authors suppose, from many
+individuals simultaneously created.  With those organic beings which never
+intercross (if such exist), the species, on my theory, must have descended
+from a succession of improved varieties, which will never have blended with
+other individuals or varieties, but will have supplanted each other; so
+that, at each successive stage of modification and improvement, all the
+individuals of each variety will have descended from a single parent.  But
+in the majority of cases, namely, with all organisms which habitually unite
+for each birth, or which often intercross, I believe that during the slow
+process of modification the individuals of the species will have been kept
+nearly uniform by intercrossing; so that many individuals will have gone on
+simultaneously changing, and the whole amount of modification will not have
+been due, at each stage, to descent from a single parent.  To illustrate
+what I mean:  our English racehorses differ slightly from the horses of
+every other breed; but they do not owe their difference and superiority to
+descent from any single pair, but to continued care in selecting and
+training many individuals during many generations.
+</para>
+
+<para>
+Before discussing the three classes of facts, which I have selected as
+presenting the greatest amount of difficulty on the theory of 'single
+centres of creation,' I must say a few words on the means of dispersal.
+</para>
+
+<para>
+Means of Dispersal. -- Sir C. Lyell and other authors have ably treated
+this subject.  I can give here only the briefest abstract of the more
+important facts.  Change of climate must have had a powerful influence on
+migration:  a region when its climate was different may have been a high
+road for migration, but now be impassable; I shall, however, presently have
+to discuss this branch of the subject in some detail.  Changes of level in
+the land must also have been highly influential:  a narrow isthmus now
+separates two marine faunas; submerge it, or let it formerly have been
+submerged, and the two faunas will now blend or may formerly have blended: 
+where the sea now extends, land may at a former period have connected
+islands or possibly even continents together, and thus have allowed
+terrestrial productions to pass from one to the other.  No geologist will
+dispute that great mutations of level have occurred within the period of
+existing organisms.  Edward Forbes insisted that all the islands in the
+Atlantic must recently have been connected with Europe or Africa, and
+Europe likewise with America.  Other authors have thus hypothetically
+bridged over every ocean, and have united almost every island to some
+mainland.  If indeed the arguments used by Forbes are to be trusted, it
+must be admitted that scarcely a single island exists which has not
+recently been united to some continent.  This view cuts the Gordian knot of
+the dispersal of the same species to the most distant points, and removes
+many a difficulty:  but to the best of any judgment we are not authorized
+in admitting such enormous geographical changes within the period of
+existing species.  It seems to me that we have abundant evidence of great
+oscillations of level in our continents; but not of such vast changes in
+their position and extension, as to have united them within the recent
+period to each other and to the several intervening oceanic islands.  I
+freely admit the former existence of many islands, now buried beneath the
+sea, which may have served as halting places for plants and for many
+animals during their migration.  In the coral-producing oceans such sunken
+islands are now marked, as I believe, by rings of coral or atolls standing
+over them.  Whenever it is fully admitted, as I believe it will some day
+be, that each species has proceeded from a single birthplace, and when in
+the course of time we know something definite about the means of
+distribution, we shall be enabled to speculate with security on the former
+extension of the land.  But I do not believe that it will ever be proved
+that within the recent period continents which are now quite separate, have
+been continuously, or almost continuously, united with each other, and with
+the many existing oceanic islands.  Several facts in distribution,--such as
+the great difference in the marine faunas on the opposite sides of almost
+every continent,--the close relation of the tertiary inhabitants of several
+lands and even seas to their present inhabitants,--a certain degree of
+relation (as we shall hereafter see) between the distribution of mammals
+and the depth of the sea,--these and other such facts seem to me opposed to
+the admission of such prodigious geographical revolutions within the recent
+period, as are necessitated on the view advanced by Forbes and admitted by
+his many followers.  The nature and relative proportions of the inhabitants
+of oceanic islands likewise seem to me opposed to the belief of their
+former continuity with continents.  Nor does their almost universally
+volcanic composition favour the admission that they are the wrecks of
+sunken continents;--if they had originally existed as mountain-ranges on
+the land, some at least of the islands would have been formed, like other
+mountain-summits, of granite, metamorphic schists, old fossiliferous or
+other such rocks, instead of consisting of mere piles of volcanic matter.
+</para>
+
+<para>
+I must now say a few words on what are called accidental means, but which
+more properly might be called occasional means of distribution.  I shall
+here confine myself to plants.  In botanical works, this or that plant is
+stated to be ill adapted for wide dissemination; but for transport across
+the sea, the greater or less facilities may be said to be almost wholly
+unknown.  Until I tried, with Mr. Berkeley's aid, a few experiments, it was
+not even known how far seeds could resist the injurious action of
+sea-water.  To my surprise I found that out of 87 kinds, 64 germinated
+after an immersion of 28 days, and a few survived an immersion of 137 days. 
+For convenience sake I chiefly tried small seeds, without the capsule or
+fruit; and as all of these sank in a few days, they could not be floated
+across wide spaces of the sea, whether or not they were injured by the
+salt-water.  Afterwards I tried some larger fruits, capsules, &amp;c., and some
+of these floated for a long time.  It is well known what a difference there
+is in the buoyancy of green and seasoned timber; and it occurred to me that
+floods might wash down plants or branches, and that these might be dried on
+the banks, and then by a fresh rise in the stream be washed into the sea. 
+Hence I was led to dry stems and branches of 94 plants with ripe fruit, and
+to place them on sea water.  The majority sank quickly, but some which
+whilst green floated for a very short time, when dried floated much longer;
+for instance, ripe hazel-nuts sank immediately, but when dried, they
+floated for 90 days and afterwards when planted they germinated; an
+asparagus plant with ripe berries floated for 23 days, when dried it
+floated for 85 days, and the seeds afterwards germinated:  the ripe seeds
+of Helosciadium sank in two days, when dried they floated for above 90
+days, and afterwards germinated.  Altogether out of the 94 dried plants, 18
+floated for above 28 days, and some of the 18 floated for a very much
+longer period.  So that as 64/87 seeds germinated after an immersion of 28
+days; and as 18/94 plants with ripe fruit (but not all the same species as
+in the foregoing experiment) floated, after being dried, for above 28 days,
+as far as we may infer anything from these scanty facts, we may conclude
+that the seeds of 14/100 plants of any country might be floated by
+sea-currents during 28 days, and would retain their power of germination. 
+In Johnston's Physical Atlas, the average rate of the several Atlantic
+currents is 33 miles per diem (some currents running at the rate of 60
+miles per diem); on this average, the seeds of 14/100 plants belonging to
+one country might be floated across 924 miles of sea to another country;
+and when stranded, if blown to a favourable spot by an inland gale, they
+would germinate.
+</para>
+
+<para>
+Subsequently to my experiments, M. Martens tried similar ones, but in a
+much better manner, for he placed the seeds in a box in the actual sea, so
+that they were alternately wet and exposed to the air like really floating
+plants.  He tried 98 seeds, mostly different from mine; but he chose many
+large fruits and likewise seeds from plants which live near the sea; and
+this would have favoured the average length of their flotation and of their
+resistance to the injurious action of the salt-water.  On the other hand he
+did not previously dry the plants or branches with the fruit; and this, as
+we have seen, would have caused some of them to have floated much longer. 
+The result was that 18/98 of his seeds floated for 42 days, and were then
+capable of germination.  But I do not doubt that plants exposed to the
+waves would float for a less time than those protected from violent
+movement as in our experiments.  Therefore it would perhaps be safer to
+assume that the seeds of about 10/100 plants of a flora, after having been
+dried, could be floated across a space of sea 900 miles in width, and would
+then germinate.  The fact of the larger fruits often floating longer than
+the small, is interesting; as plants with large seeds or fruit could hardly
+be transported by any other means; and Alph. de Candolle has shown that
+such plants generally have restricted ranges.
+</para>
+
+<para>
+But seeds may be occasionally transported in another manner.  Drift timber
+is thrown up on most islands, even on those in the midst of the widest
+oceans; and the natives of the coral-islands in the Pacific, procure stones
+for their tools, solely from the roots of drifted trees, these stones being
+a valuable royal tax.  I find on examination, that when irregularly shaped
+stones are embedded in the roots of trees, small parcels of earth are very
+frequently enclosed in their interstices and behind them,--so perfectly
+that not a particle could be washed away in the longest transport:  out of
+one small portion of earth thus completely enclosed by wood in an oak about
+50 years old, three dicotyledonous plants germinated:  I am certain of the
+accuracy of this observation.  Again, I can show that the carcasses of
+birds, when floating on the sea, sometimes escape being immediately
+devoured; and seeds of many kinds in the crops of floating birds long
+retain their vitality:  peas and vetches, for instance, are killed by even
+a few days' immersion in sea-water; but some taken out of the crop of a
+pigeon, which had floated on artificial salt-water for 30 days, to my
+surprise nearly all germinated.
+</para>
+
+<para>
+Living birds can hardly fail to be highly effective agents in the
+transportation of seeds.  I could give many facts showing how frequently
+birds of many kinds are blown by gales to vast distances across the ocean. 
+We may I think safely assume that under such circumstances their rate of
+flight would often be 35 miles an hour; and some authors have given a far
+higher estimate.  I have never seen an instance of nutritious seeds passing
+through the intestines of a bird; but hard seeds of fruit will pass
+uninjured through even the digestive organs of a turkey.  In the course of
+two months, I picked up in my garden 12 kinds of seeds, out of the
+excrement of small birds, and these seemed perfect, and some of them, which
+I tried, germinated.  But the following fact is more important:  the crops
+of birds do not secrete gastric juice, and do not in the least injure, as I
+know by trial, the germination of seeds; now after a bird has found and
+devoured a large supply of food, it is positively asserted that all the
+grains do not pass into the gizzard for 12 or even 18 hours.  A bird in
+this interval might easily be blown to the distance of 500 miles, and hawks
+are known to look out for tired birds, and the contents of their torn crops
+might thus readily get scattered.  Mr. Brent informs me that a friend of
+his had to give up flying carrier-pigeons from France to England, as the
+hawks on the English coast destroyed so many on their arrival.  Some hawks
+and owls bolt their prey whole, and after an interval of from twelve to
+twenty hours, disgorge pellets, which, as I know from experiments made in
+the Zoological Gardens, include seeds capable of germination.  Some seeds
+of the oat, wheat, millet, canary, hemp, clover, and beet germinated after
+having been from twelve to twenty-one hours in the stomachs of different
+birds of prey; and two seeds of beet grew after having been thus retained
+for two days and fourteen hours.  Freshwater fish, I find, eat seeds of
+many land and water plants:  fish are frequently devoured by birds, and
+thus the seeds might be transported from place to place.  I forced many
+kinds of seeds into the stomachs of dead fish, and then gave their bodies
+to fishing-eagles, storks, and pelicans; these birds after an interval of
+many hours, either rejected the seeds in pellets or passed them in their
+excrement; and several of these seeds retained their power of germination. 
+Certain seeds, however, were always killed by this process.
+</para>
+
+<para>
+Although the beaks and feet of birds are generally quite clean, I can show
+that earth sometimes adheres to them:  in one instance I removed twenty-two
+grains of dry argillaceous earth from one foot of a partridge, and in this
+earth there was a pebble quite as large as the seed of a vetch.  Thus seeds
+might occasionally be transported to great distances; for many facts could
+be given showing that soil almost everywhere is charged with seeds. 
+Reflect for a moment on the millions of quails which annually cross the
+Mediterranean; and can we doubt that the earth adhering to their feet would
+sometimes include a few minute seeds?  But I shall presently have to recur
+to this subject.
+</para>
+
+<para>
+As icebergs are known to be sometimes loaded with earth and stones, and
+have even carried brushwood, bones, and the nest of a land-bird, I can
+hardly doubt that they must occasionally have transported seeds from one
+part to another of the arctic and antarctic regions, as suggested by Lyell;
+and during the Glacial period from one part of the now temperate regions to
+another.  In the Azores, from the large number of the species of plants
+common to Europe, in comparison with the plants of other oceanic islands
+nearer to the mainland, and (as remarked by Mr. H. C. Watson) from the
+somewhat northern character of the flora in comparison with the latitude, I
+suspected that these islands had been partly stocked by ice-borne seeds,
+during the Glacial epoch.  At my request Sir C. Lyell wrote to M. Hartung
+to inquire whether he had observed erratic boulders on these islands, and
+he answered that he had found large fragments of granite and other rocks,
+which do not occur in the archipelago.  Hence we may safely infer that
+icebergs formerly landed their rocky burthens on the shores of these
+mid-ocean islands, and it is at least possible that they may have brought
+thither the seeds of northern plants.
+</para>
+
+<para>
+Considering that the several above means of transport, and that several
+other means, which without doubt remain to be discovered, have been in
+action year after year, for centuries and tens of thousands of years, it
+would I think be a marvellous fact if many plants had not thus become
+widely transported.  These means of transport are sometimes called
+accidental, but this is not strictly correct:  the currents of the sea are
+not accidental, nor is the direction of prevalent gales of wind.  It should
+be observed that scarcely any means of transport would carry seeds for very
+great distances; for seeds do not retain their vitality when exposed for a
+great length of time to the action of seawater; nor could they be long
+carried in the crops or intestines of birds.  These means, however, would
+suffice for occasional transport across tracts of sea some hundred miles in
+breadth, or from island to island, or from a continent to a neighbouring
+island, but not from one distant continent to another.  The floras of
+distant continents would not by such means become mingled in any great
+degree; but would remain as distinct as we now see them to be.  The
+currents, from their course, would never bring seeds from North America to
+Britain, though they might and do bring seeds from the West Indies to our
+western shores, where, if not killed by so long an immersion in salt-water,
+they could not endure our climate.  Almost every year, one or two
+land-birds are blown across the whole Atlantic Ocean, from North America to
+the western shores of Ireland and England; but seeds could be transported
+by these wanderers only by one means, namely, in dirt sticking to their
+feet, which is in itself a rare accident.  Even in this case, how small
+would the chance be of a seed falling on favourable soil, and coming to
+maturity!  But it would be a great error to argue that because a
+well-stocked island, like Great Britain, has not, as far as is known (and
+it would be very difficult to prove this), received within the last few
+centuries, through occasional means of transport, immigrants from Europe or
+any other continent, that a poorly-stocked island, though standing more
+remote from the mainland, would not receive colonists by similar means.  I
+do not doubt that out of twenty seeds or animals transported to an island,
+even if far less well-stocked than Britain, scarcely more than one would be
+so well fitted to its new home, as to become naturalised.  But this, as it
+seems to me, is no valid argument against what would be effected by
+occasional means of transport, during the long lapse of geological time,
+whilst an island was being upheaved and formed, and before it had become
+fully stocked with inhabitants.  On almost bare land, with few or no
+destructive insects or birds living there, nearly every seed, which chanced
+to arrive, would be sure to germinate and survive.
+</para>
+
+<para>
+Dispersal during the Glacial period. -- The identity of many plants and
+animals, on mountain-summits, separated from each other by hundreds of
+miles of lowlands, where the Alpine species could not possibly exist, is
+one of the most striking cases known of the same species living at distant
+points, without the apparent possibility of their having migrated from one
+to the other.  It is indeed a remarkable fact to see so many of the same
+plants living on the snowy regions of the Alps or Pyrenees, and in the
+extreme northern parts of Europe; but it is far more remarkable, that the
+plants on the White Mountains, in the United States of America, are all the
+same with those of Labrador, and nearly all the same, as we hear from Asa
+Gray, with those on the loftiest mountains of Europe.  Even as long ago as
+1747, such facts led Gmelin to conclude that the same species must have
+been independently created at several distinct points; and we might have
+remained in this same belief, had not Agassiz and others called vivid
+attention to the Glacial period, which, as we shall immediately see,
+affords a simple explanation of these facts.  We have evidence of almost
+every conceivable kind, organic and inorganic, that within a very recent
+geological period, central Europe and North America suffered under an
+Arctic climate.  The ruins of a house burnt by fire do not tell their tale
+more plainly, than do the mountains of Scotland and Wales, with their
+scored flanks, polished surfaces, and perched boulders, of the icy streams
+with which their valleys were lately filled.  So greatly has the climate of
+Europe changed, that in Northern Italy, gigantic moraines, left by old
+glaciers, are now clothed by the vine and maize.  Throughout a large part
+of the United States, erratic boulders, and rocks scored by drifted
+icebergs and coast-ice, plainly reveal a former cold period.
+</para>
+
+<para>
+The former influence of the glacial climate on the distribution of the
+inhabitants of Europe, as explained with remarkable clearness by Edward
+Forbes, is substantially as follows.  But we shall follow the changes more
+readily, by supposing a new glacial period to come slowly on, and then pass
+away, as formerly occurred.  As the cold came on, and as each more southern
+zone became fitted for arctic beings and ill-fitted for their former more
+temperate inhabitants, the latter would be supplanted and arctic
+productions would take their places.  The inhabitants of the more temperate
+regions would at the same time travel southward, unless they were stopped
+by barriers, in which case they would perish.  The mountains would become
+covered with snow and ice, and their former Alpine inhabitants would
+descend to the plains.  By the time that the cold had reached its maximum,
+we should have a uniform arctic fauna and flora, covering the central parts
+of Europe, as far south as the Alps and Pyrenees, and even stretching into
+Spain.  The now temperate regions of the United States would likewise be
+covered by arctic plants and animals, and these would be nearly the same
+with those of Europe; for the present circumpolar inhabitants, which we
+suppose to have everywhere travelled southward, are remarkably uniform
+round the world.  We may suppose that the Glacial period came on a little
+earlier or later in North America than in Europe, so will the southern
+migration there have been a little earlier or later; but this will make no
+difference in the final result.
+</para>
+
+<para>
+As the warmth returned, the arctic forms would retreat northward, closely
+followed up in their retreat by the productions of the more temperate
+regions.  And as the snow melted from the bases of the mountains, the
+arctic forms would seize on the cleared and thawed ground, always ascending
+higher and higher, as the warmth increased, whilst their brethren were
+pursuing their northern journey.  Hence, when the warmth had fully
+returned, the same arctic species, which had lately lived in a body
+together on the lowlands of the Old and New Worlds, would be left isolated
+on distant mountain-summits (having been exterminated on all lesser
+heights) and in the arctic regions of both hemispheres.
+</para>
+
+<para>
+Thus we can understand the identity of many plants at points so immensely
+remote as on the mountains of the United States and of Europe.  We can thus
+also understand the fact that the Alpine plants of each mountain-range are
+more especially related to the arctic forms living due north or nearly due
+north of them:  for the migration as the cold came on, and the re-migration
+on the returning warmth, will generally have been due south and north.  The
+Alpine plants, for example, of Scotland, as remarked by Mr. H. C. Watson,
+and those of the Pyrenees, as remarked by Ramond, are more especially
+allied to the plants of northern Scandinavia; those of the United States to
+Labrador; those of the mountains of Siberia to the arctic regions of that
+country.  These views, grounded as they are on the perfectly
+well-ascertained occurrence of a former Glacial period, seem to me to
+explain in so satisfactory a manner the present distribution of the Alpine
+and Arctic productions of Europe and America, that when in other regions we
+find the same species on distant mountain-summits, we may almost conclude
+without other evidence, that a colder climate permitted their former
+migration across the low intervening tracts, since become too warm for
+their existence.
+</para>
+
+<para>
+If the climate, since the Glacial period, has ever been in any degree
+warmer than at present (as some geologists in the United States believe to
+have been the case, chiefly from the distribution of the fossil Gnathodon),
+then the arctic and temperate productions will at a very late period have
+marched a little further north, and subsequently have retreated to their
+present homes; but I have met with no satisfactory evidence with respect to
+this intercalated slightly warmer period, since the Glacial period.
+</para>
+
+<para>
+The arctic forms, during their long southern migration and re-migration
+northward, will have been exposed to nearly the same climate, and, as is
+especially to be noticed, they will have kept in a body together;
+consequently their mutual relations will not have been much disturbed, and,
+in accordance with the principles inculcated in this volume, they will not
+have been liable to much modification.  But with our Alpine productions,
+left isolated from the moment of the returning warmth, first at the bases
+and ultimately on the summits of the mountains, the case will have been
+somewhat different; for it is not likely that all the same arctic species
+will have been left on mountain ranges distant from each other, and have
+survived there ever since; they will, also, in all probability have become
+mingled with ancient Alpine species, which must have existed on the
+mountains before the commencement of the Glacial epoch, and which during
+its coldest period will have been temporarily driven down to the plains;
+they will, also, have been exposed to somewhat different climatal
+influences.  Their mutual relations will thus have been in some degree
+disturbed; consequently they will have been liable to modification; and
+this we find has been the case; for if we compare the present Alpine plants
+and animals of the several great European mountain-ranges, though very many
+of the species are identically the same, some present varieties, some are
+ranked as doubtful forms, and some few are distinct yet closely allied or
+representative species.
+</para>
+
+<para>
+In illustrating what, as I believe, actually took place during the Glacial
+period, I assumed that at its commencement the arctic productions were as
+uniform round the polar regions as they are at the present day.  But the
+foregoing remarks on distribution apply not only to strictly arctic forms,
+but also to many sub-arctic and to some few northern temperate forms, for
+some of these are the same on the lower mountains and on the plains of
+North America and Europe; and it may be reasonably asked how I account for
+the necessary degree of uniformity of the sub-arctic and northern temperate
+forms round the world, at the commencement of the Glacial period.  At the
+present day, the sub-arctic and northern temperate productions of the Old
+and New Worlds are separated from each other by the Atlantic Ocean and by
+the extreme northern part of the Pacific.  During the Glacial period, when
+the inhabitants of the Old and New Worlds lived further southwards than at
+present, they must have been still more completely separated by wider
+spaces of ocean.  I believe the above difficulty may be surmounted by
+looking to still earlier changes of climate of an opposite nature.  We have
+good reason to believe that during the newer Pliocene period, before the
+Glacial epoch, and whilst the majority of the inhabitants of the world were
+specifically the same as now, the climate was warmer than at the present
+day.  Hence we may suppose that the organisms now living under the climate
+of latitude 60 deg, during the Pliocene period lived further north under
+the Polar Circle, in latitude 66 deg-67 deg; and that the strictly arctic
+productions then lived on the broken land still nearer to the pole.  Now if
+we look at a globe, we shall see that under the Polar Circle there is
+almost continuous land from western Europe, through Siberia, to eastern
+America.  And to this continuity of the circumpolar land, and to the
+consequent freedom for intermigration under a more favourable climate, I
+attribute the necessary amount of uniformity in the sub-arctic and northern
+temperate productions of the Old and New Worlds, at a period anterior to
+the Glacial epoch.
+</para>
+
+<para>
+Believing, from reasons before alluded to, that our continents have long
+remained in nearly the same relative position, though subjected to large,
+but partial oscillations of level, I am strongly inclined to extend the
+above view, and to infer that during some earlier and still warmer period,
+such as the older Pliocene period, a large number of the same plants and
+animals inhabited the almost continuous circumpolar land; and that these
+plants and animals, both in the Old and New Worlds, began slowly to migrate
+southwards as the climate became less warm, long before the commencement of
+the Glacial period.  We now see, as I believe, their descendants, mostly in
+a modified condition, in the central parts of Europe and the United States. 
+On this view we can understand the relationship, with very little identity,
+between the productions of North America and Europe,--a relationship which
+is most remarkable, considering the distance of the two areas, and their
+separation by the Atlantic Ocean.  We can further understand the singular
+fact remarked on by several observers, that the productions of Europe and
+America during the later tertiary stages were more closely related to each
+other than they are at the present time; for during these warmer periods
+the northern parts of the Old and New Worlds will have been almost
+continuously united by land, serving as a bridge, since rendered impassable
+by cold, for the inter-migration of their inhabitants.
+</para>
+
+<para>
+During the slowly decreasing warmth of the Pliocene period, as soon as the
+species in common, which inhabited the New and Old Worlds, migrated south
+of the Polar Circle, they must have been completely cut off from each
+other.  This separation, as far as the more temperate productions are
+concerned, took place long ages ago.  And as the plants and animals
+migrated southward, they will have become mingled in the one great region
+with the native American productions, and have had to compete with them;
+and in the other great region, with those of the Old World.  Consequently
+we have here everything favourable for much modification,--for far more
+modification than with the Alpine productions, left isolated, within a much
+more recent period, on the several mountain-ranges and on the arctic lands
+of the two Worlds.  Hence it has come, that when we compare the now living
+productions of the temperate regions of the New and Old Worlds, we find
+very few identical species (though Asa Gray has lately shown that more
+plants are identical than was formerly supposed), but we find in every
+great class many forms, which some naturalists rank as geographical races,
+and others as distinct species; and a host of closely allied or
+representative forms which are ranked by all naturalists as specifically
+distinct.
+</para>
+
+<para>
+As on the land, so in the waters of the sea, a slow southern migration of a
+marine fauna, which during the Pliocene or even a somewhat earlier period,
+was nearly uniform along the continuous shores of the Polar Circle, will
+account, on the theory of modification, for many closely allied forms now
+living in areas completely sundered.  Thus, I think, we can understand the
+presence of many existing and tertiary representative forms on the eastern
+and western shores of temperate North America; and the still more striking
+case of many closely allied crustaceans (as described in Dana's admirable
+work), of some fish and other marine animals, in the Mediterranean and in
+the seas of Japan,--areas now separated by a continent and by nearly a
+hemisphere of equatorial ocean.
+</para>
+
+<para>
+These cases of relationship, without identity, of the inhabitants of seas
+now disjoined, and likewise of the past and present inhabitants of the
+temperate lands of North America and Europe, are inexplicable on the theory
+of creation.  We cannot say that they have been created alike, in
+correspondence with the nearly similar physical conditions of the areas;
+for if we compare, for instance, certain parts of South America with the
+southern continents of the Old World, we see countries closely
+corresponding in all their physical conditions, but with their inhabitants
+utterly dissimilar.
+</para>
+
+<para>
+But we must return to our more immediate subject, the Glacial period.  I am
+convinced that Forbes's view may be largely extended.  In Europe we have
+the plainest evidence of the cold period, from the western shores of
+Britain to the Oural range, and southward to the Pyrenees.  We may infer,
+from the frozen mammals and nature of the mountain vegetation, that Siberia
+was similarly affected.  Along the Himalaya, at points 900 miles apart,
+glaciers have left the marks of their former low descent; and in Sikkim,
+Dr. Hooker saw maize growing on gigantic ancient moraines.  South of the
+equator, we have some direct evidence of former glacial action in New
+Zealand; and the same plants, found on widely separated mountains in this
+island, tell the same story.  If one account which has been published can
+be trusted, we have direct evidence of glacial action in the south-eastern
+corner of Australia.
+</para>
+
+<para>
+Looking to America; in the northern half, ice-borne fragments of rock have
+been observed on the eastern side as far south as lat. 36 deg-37 deg, and
+on the shores of the Pacific, where the climate is now so different, as far
+south as lat. 46 deg; erratic boulders have, also, been noticed on the
+Rocky Mountains.  In the Cordillera of Equatorial South America, glaciers
+once extended far below their present level.  In central Chile I was
+astonished at the structure of a vast mound of detritus, about 800 feet in
+height, crossing a valley of the Andes; and this I now feel convinced was a
+gigantic moraine, left far below any existing glacier.  Further south on
+both sides of the continent, from lat. 41 deg to the southernmost
+extremity, we have the clearest evidence of former glacial action, in huge
+boulders transported far from their parent source.
+</para>
+
+<para>
+We do not know that the Glacial epoch was strictly simultaneous at these
+several far distant points on opposite sides of the world.  But we have
+good evidence in almost every case, that the epoch was included within the
+latest geological period.  We have, also, excellent evidence, that it
+endured for an enormous time, as measured by years, at each point.  The
+cold may have come on, or have ceased, earlier at one point of the globe
+than at another, but seeing that it endured for long at each, and that it
+was contemporaneous in a geological sense, it seems to me probable that it
+was, during a part at least of the period, actually simultaneous throughout
+the world.  Without some distinct evidence to the contrary, we may at least
+admit as probable that the glacial action was simultaneous on the eastern
+and western sides of North America, in the Cordillera under the equator and
+under the warmer temperate zones, and on both sides of the southern
+extremity of the continent.  If this be admitted, it is difficult to avoid
+believing that the temperature of the whole world was at this period
+simultaneously cooler.  But it would suffice for my purpose, if the
+temperature was at the same time lower along certain broad belts of
+longitude.
+</para>
+
+<para>
+On this view of the whole world, or at least of broad longitudinal belts,
+having been simultaneously colder from pole to pole, much light can be
+thrown on the present distribution of identical and allied species.  In
+America, Dr. Hooker has shown that between forty and fifty of the flowering
+plants of Tierra del Fuego, forming no inconsiderable part of its scanty
+flora, are common to Europe, enormously remote as these two points are; and
+there are many closely allied species.  On the lofty mountains of
+equatorial America a host of peculiar species belonging to European genera
+occur.  On the highest mountains of Brazil, some few European genera were
+found by Gardner, which do not exist in the wide intervening hot countries. 
+So on the Silla of Caraccas the illustrious Humboldt long ago found species
+belonging to genera characteristic of the Cordillera.  On the mountains of
+Abyssinia, several European forms and some few representatives of the
+peculiar flora of the Cape of Good Hope occur.  At the Cape of Good Hope a
+very few European species, believed not to have been introduced by man, and
+on the mountains, some few representative European forms are found, which
+have not been discovered in the intertropical parts of Africa.  On the
+Himalaya, and on the isolated mountain-ranges of the peninsula of India, on
+the heights of Ceylon, and on the volcanic cones of Java, many plants
+occur, either identically the same or representing each other, and at the
+same time representing plants of Europe, not found in the intervening hot
+lowlands.  A list of the genera collected on the loftier peaks of Java
+raises a picture of a collection made on a hill in Europe!  Still more
+striking is the fact that southern Australian forms are clearly represented
+by plants growing on the summits of the mountains of Borneo.  Some of these
+Australian forms, as I hear from Dr. Hooker, extend along the heights of
+the peninsula of Malacca, and are thinly scattered, on the one hand over
+India and on the other as far north as Japan.
+</para>
+
+<para>
+On the southern mountains of Australia, Dr. F. Muller has discovered
+several European species; other species, not introduced by man, occur on
+the lowlands; and a long list can be given, as I am informed by Dr. Hooker,
+of European genera, found in Australia, but not in the intermediate torrid
+regions.  In the admirable 'Introduction to the Flora of New Zealand,' by
+Dr. Hooker, analogous and striking facts are given in regard to the plants
+of that large island.  Hence we see that throughout the world, the plants
+growing on the more lofty mountains, and on the temperate lowlands of the
+northern and southern hemispheres, are sometimes identically the same; but
+they are much oftener specifically distinct, though related to each other
+in a most remarkable manner.
+</para>
+
+<para>
+This brief abstract applies to plants alone:  some strictly analogous facts
+could be given on the distribution of terrestrial animals.  In marine
+productions, similar cases occur; as an example, I may quote a remark by
+the highest authority, Prof. Dana, that 'it is certainly a wonderful fact
+that New Zealand should have a closer resemblance in its crustacea to Great
+Britain, its antipode, than to any other part of the world.'  Sir J.
+Richardson, also, speaks of the reappearance on the shores of New Zealand,
+Tasmania, &amp;c., of northern forms of fish.  Dr. Hooker informs me that
+twenty-five species of Algae are common to New Zealand and to Europe, but
+have not been found in the intermediate tropical seas.
+</para>
+
+<para>
+It should be observed that the northern species and forms found in the
+southern parts of the southern hemisphere, and on the mountain-ranges of
+the intertropical regions, are not arctic, but belong to the northern
+temperate zones.  As Mr. H. C. Watson has recently remarked, 'In receding
+from polar towards equatorial latitudes, the Alpine or mountain floras
+really become less and less arctic.'  Many of the forms living on the
+mountains of the warmer regions of the earth and in the southern hemisphere
+are of doubtful value, being ranked by some naturalists as specifically
+distinct, by others as varieties; but some are certainly identical, and
+many, though closely related to northern forms, must be ranked as distinct
+species.
+</para>
+
+<para>
+Now let us see what light can be thrown on the foregoing facts, on the
+belief, supported as it is by a large body of geological evidence, that the
+whole world, or a large part of it, was during the Glacial period
+simultaneously much colder than at present.  The Glacial period, as
+measured by years, must have been very long; and when we remember over what
+vast spaces some naturalised plants and animals have spread within a few
+centuries, this period will have been ample for any amount of migration. 
+As the cold came slowly on, all the tropical plants and other productions
+will have retreated from both sides towards the equator, followed in the
+rear by the temperate productions, and these by the arctic; but with the
+latter we are not now concerned.  The tropical plants probably suffered
+much extinction; how much no one can say; perhaps formerly the tropics
+supported as many species as we see at the present day crowded together at
+the Cape of Good Hope, and in parts of temperate Australia.  As we know
+that many tropical plants and animals can withstand a considerable amount
+of cold, many might have escaped extermination during a moderate fall of
+temperature, more especially by escaping into the warmest spots.  But the
+great fact to bear in mind is, that all tropical productions will have
+suffered to a certain extent.  On the other hand, the temperate
+productions, after migrating nearer to the equator, though they will have
+been placed under somewhat new conditions, will have suffered less.  And it
+is certain that many temperate plants, if protected from the inroads of
+competitors, can withstand a much warmer climate than their own.  Hence, it
+seems to me possible, bearing in mind that the tropical productions were in
+a suffering state and could not have presented a firm front against
+intruders, that a certain number of the more vigorous and dominant
+temperate forms might have penetrated the native ranks and have reached or
+even crossed the equator.  The invasion would, of course, have been greatly
+favoured by high land, and perhaps by a dry climate; for Dr. Falconer
+informs me that it is the damp with the heat of the tropics which is so
+destructive to perennial plants from a temperate climate.  On the other
+hand, the most humid and hottest districts will have afforded an asylum to
+the tropical natives.  The mountain-ranges north-west of the Himalaya, and
+the long line of the Cordillera, seem to have afforded two great lines of
+invasion:  and it is a striking fact, lately communicated to me by Dr.
+Hooker, that all the flowering plants, about forty-six in number, common to
+Tierra del Fuego and to Europe still exist in North America, which must
+have lain on the line of march.  But I do not doubt that some temperate
+productions entered and crossed even the lowlands of the  tropics at the
+period when the cold was most intense,--when arctic forms had migrated some
+twenty-five degrees of latitude from their native country and covered the
+land at the foot of the Pyrenees.  At this period of extreme cold, I
+believe that the climate under the equator at the level of the sea was
+about the same with that now felt there at the height of six or seven
+thousand feet.  During this the coldest period, I suppose that large spaces
+of the tropical lowlands were clothed with a mingled tropical and temperate
+vegetation, like that now growing with strange luxuriance at the base of
+the Himalaya, as graphically described by Hooker.
+</para>
+
+<para>
+Thus, as I believe, a considerable number of plants, a few terrestrial
+animals, and some marine productions, migrated during the Glacial period
+from the northern and southern temperate zones into the intertropical
+regions, and some even crossed the equator.  As the warmth returned, these
+temperate forms would naturally ascend the higher mountains, being
+exterminated on the lowlands; those which had not reached the equator,
+would re-migrate northward or southward towards their former homes; but the
+forms, chiefly northern, which had crossed the equator, would travel still
+further from their homes into the more temperate latitudes of the opposite
+hemisphere.  Although we have reason to believe from geological evidence
+that the whole body of arctic shells underwent scarcely any modification
+during their long southern migration and re-migration northward, the case
+may have been wholly different with those intruding forms which settled
+themselves on the intertropical mountains, and in the southern hemisphere. 
+These being surrounded by strangers will have had to compete with many new
+forms of life; and it is probable that selected modifications in their
+structure, habits, and constitutions will have profited them.  Thus many of
+these wanderers, though still plainly related by inheritance to their
+brethren of the northern or southern hemispheres, now exist in their new
+homes as well-marked varieties or as distinct species.
+</para>
+
+<para>
+It is a remarkable fact, strongly insisted on by Hooker in regard to
+America, and by Alph. de Candolle in regard to Australia, that many more
+identical plants and allied forms have apparently migrated from the north
+to the south, than in a reversed direction.  We see, however, a few
+southern vegetable forms on the mountains of Borneo and Abyssinia.  I
+suspect that this preponderant migration from north to south is due to the
+greater extent of land in the north, and to the northern forms having
+existed in their own homes in greater numbers, and having consequently been
+advanced through natural selection and competition to a higher stage of
+perfection or dominating power, than the southern forms.  And thus, when
+they became commingled during the Glacial period, the northern forms were
+enabled to beat the less powerful southern forms.  Just in the same manner
+as we see at the present day, that very many European productions cover the
+ground in La Plata, and in a lesser degree in Australia, and have to a
+certain extent beaten the natives; whereas extremely few southern forms
+have become naturalised in any part of Europe, though hides, wool, and
+other objects likely to carry seeds have been largely imported into Europe
+during the last two or three centuries from La Plata, and during the last
+thirty or forty years from Australia.  Something of the same kind must have
+occurred on the intertropical mountains:  no doubt before the Glacial
+period they were stocked with endemic Alpine forms; but these have almost
+everywhere largely yielded to the more dominant forms, generated in the
+larger areas and more efficient workshops of the north.  In many islands
+the native productions are nearly equalled or even outnumbered by the
+naturalised; and if the natives have not been actually exterminated, their
+numbers have been greatly reduced, and this is the first stage towards
+extinction.  A mountain is an island on the land; and the intertropical
+mountains before the Glacial period must have been completely isolated; and
+I believe that the productions of these islands on the land yielded to
+those produced within the larger areas of the north, just in the same way
+as the productions of real islands have everywhere lately yielded to
+continental forms, naturalised by man's agency.
+</para>
+
+<para>
+I am far from supposing that all difficulties are removed on the view here
+given in regard to the range and affinities of the allied species which
+live in the northern and southern temperate zones and on the mountains of
+the intertropical regions.  Very many difficulties remain to be solved.  I
+do not pretend to indicate the exact lines and means of migration, or the
+reason why certain species and not others have migrated; why certain
+species have been modified and have given rise to new groups of forms, and
+others have remained unaltered.  We cannot hope to explain such facts,
+until we can say why one species and not another becomes naturalised by
+man's agency in a foreign land; why one ranges twice or thrice as far, and
+is twice or thrice as common, as another species within their own homes.
+</para>
+
+<para>
+I have said that many difficulties remain to be solved:  some of the most
+remarkable are stated with admirable clearness by Dr. Hooker in his
+botanical works on the antarctic regions.  These cannot be here discussed. 
+I will only say that as far as regards the occurrence of identical species
+at points so enormously remote as Kerguelen Land, New Zealand, and Fuegia,
+I believe that towards the close of the Glacial period, icebergs, as
+suggested by Lyell, have been largely concerned in their dispersal.  But
+the existence of several quite distinct species, belonging to genera
+exclusively confined to the south, at these and other distant points of the
+southern hemisphere, is, on my theory of descent with modification, a far
+more remarkable case of difficulty.  For some of these species are so
+distinct, that we cannot suppose that there has been time since the
+commencement of the Glacial period for their migration, and for their
+subsequent modification to the necessary degree.  The facts seem to me to
+indicate that peculiar and very distinct species have migrated in radiating
+lines from some common centre; and I am inclined to look in the southern,
+as in the northern hemisphere, to a former and warmer period, before the
+commencement of the Glacial period, when the antarctic lands, now covered
+with ice, supported a highly peculiar and isolated flora.  I suspect that
+before this flora was exterminated by the Glacial epoch, a few forms were
+widely dispersed to various points of the southern hemisphere by occasional
+means of transport, and by the aid, as halting-places, of existing and now
+sunken islands, and perhaps at the commencement of the Glacial period, by
+icebergs.  By these means, as I believe, the southern shores of America,
+Australia, New Zealand have become slightly tinted by the same peculiar
+forms of vegetable life.
+</para>
+
+<para>
+Sir C. Lyell in a striking passage has speculated, in language almost
+identical with mine, on the effects of great alternations of climate on
+geographical distribution.  I believe that the world has recently felt one
+of his great cycles of change; and that on this view, combined with
+modification through natural selection, a multitude of facts in the present
+distribution both of the same and of allied forms of life can be explained. 
+The living waters may be said to have flowed during one short period from
+the north and from the south, and to have crossed at the equator; but to
+have flowed with greater force from the north so as to have freely
+inundated the south.  As the tide leaves its drift in horizontal lines,
+though rising higher on the shores where the tide rises highest, so have
+the living waters left their living drift on our mountain-summits, in a
+line gently rising from the arctic lowlands to a great height under the
+equator.  The various beings thus left stranded may be compared with savage
+races of man, driven up and surviving in the mountain-fastnesses of almost
+every land, which serve as a record, full of interest to us, of the former
+inhabitants of the surrounding lowlands.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter XII</chapnum>
+
+<title>Geographical Distribution--continued</title>
+
+<chapsummary>
+Distribution of fresh-water productions -- On the inhabitants of oceanic
+islands -- Absence of Batrachians and of terrestrial Mammals -- On the
+relation of the inhabitants of islands to those of the nearest mainland --
+On colonisation from the nearest source with subsequent modification --
+Summary of the last and present chapters.
+</chapsummary>
+</chapheader>
+
+<para>
+As lakes and river-systems are separated from each other by barriers of
+land, it might have been thought that fresh-water productions would not
+have ranged widely within the same country, and as the sea is apparently a
+still more impassable barrier, that they never would have extended to
+distant countries.  But the case is exactly the reverse.  Not only have
+many fresh-water species, belonging to quite different classes, an enormous
+range, but allied species prevail in a remarkable manner throughout the
+world.  I well remember, when first collecting in the fresh waters of
+Brazil, feeling much surprise at the similarity of the fresh-water insects,
+shells, &amp;c., and at the dissimilarity of the surrounding terrestrial
+beings, compared with those of Britain.
+</para>
+
+<para>
+But this power in fresh-water productions of ranging widely, though so
+unexpected, can, I think, in most cases be explained by their having become
+fitted, in a manner highly useful to them, for short and frequent
+migrations from pond to pond, or from stream to stream; and liability to
+wide dispersal would follow from this capacity as an almost necessary
+consequence.  We can here consider only a few cases.  In regard to fish, I
+believe that the same species never occur in the fresh waters of distant
+continents.  But on the same continent the species often range widely and
+almost capriciously; for two river-systems will have some fish in common
+and some different.  A few facts seem to favour the possibility of their
+occasional transport by accidental means; like that of the live fish not
+rarely dropped by whirlwinds in India, and the vitality of their ova when
+removed from the water.  But I am inclined to attribute the dispersal of
+fresh-water fish mainly to slight changes within the recent period in the
+level of the land, having caused rivers to flow into each other. 
+Instances, also, could be given of this having occurred during floods,
+without any change of level.  We have evidence in the loess of the Rhine of
+considerable changes of level in the land within a very recent geological
+period, and when the surface was peopled by existing land and fresh-water
+shells.  The wide difference of the fish on opposite sides of continuous
+mountain-ranges, which from an early period must have parted river-systems
+and completely prevented their inosculation, seems to lead to this same
+conclusion.  With respect to allied fresh-water fish occurring at very
+distant points of the world, no doubt there are many cases which cannot at
+present be explained:  but some fresh-water fish belong to very ancient
+forms, and in such cases there will have been ample time for great
+geographical changes, and consequently time and means for much migration. 
+In the second place, salt-water fish can with care be slowly accustomed to
+live in fresh water; and, according to Valenciennes, there is hardly a
+single group of fishes confined exclusively to fresh water, so that we may
+imagine that a marine member of a fresh-water group might travel far along
+the shores of the sea, and subsequently become modified and adapted to the
+fresh waters of a distant land.
+</para>
+
+<para>
+Some species of fresh-water shells have a very wide range, and allied
+species, which, on my theory, are descended from a common parent and must
+have proceeded from a single source, prevail throughout the world.  Their
+distribution at first perplexed me much, as their ova are not likely to be
+transported by birds, and they are immediately killed by sea water, as are
+the adults.  I could not even understand how some naturalised species have
+rapidly spread throughout the same country.  But two facts, which I have
+observed--and no doubt many others remain to be observed--throw some light
+on this subject.  When a duck suddenly emerges from a pond covered with
+duck-weed, I have twice seen these little plants adhering to its back; and
+it has happened to me, in removing a little duck-weed from one aquarium to
+another, that I have quite unintentionally stocked the one with fresh-water
+shells from the other.  But another agency is perhaps more effectual:  I
+suspended a duck's feet, which might represent those of a bird sleeping in
+a natural pond, in an aquarium, where many ova of fresh-water shells were
+hatching; and I found that numbers of the extremely minute and just hatched
+shells crawled on the feet, and clung to them so firmly that when taken out
+of the water they could not be jarred off, though at a somewhat more
+advanced age they would voluntarily drop off.  These just hatched molluscs,
+though aquatic in their nature, survived on the duck's feet, in damp air,
+from twelve to twenty hours; and in this length of time a duck or heron
+might fly at least six or seven hundred miles, and would be sure to alight
+on a pool or rivulet, if blown across sea to an oceanic island or to any
+other distant point.  Sir Charles Lyell also informs me that a Dyticus has
+been caught with an Ancylus (a fresh-water shell like a limpet) firmly
+adhering to it; and a water-beetle of the same family, a Colymbetes, once
+flew on board the 'Beagle,' when forty-five miles distant from the nearest
+land:  how much farther it might have flown with a favouring gale no one
+can tell.
+</para>
+
+<para>
+With respect to plants, it has long been known what enormous ranges many
+fresh-water and even marsh-species have, both over continents and to the
+most remote oceanic islands.  This is strikingly shown, as remarked by
+Alph. de Candolle, in large groups of terrestrial plants, which have only a
+very few aquatic members; for these latter seem immediately to acquire, as
+if in consequence, a very wide range.  I think favourable means of
+dispersal explain this fact.  I have before mentioned that earth
+occasionally, though rarely, adheres in some quantity to the feet and beaks
+of birds.  Wading birds, which frequent the muddy edges of ponds, if
+suddenly flushed, would be the most likely to have muddy feet.  Birds of
+this order I can show are the greatest wanderers, and are occasionally
+found on the most remote and barren islands in the open ocean; they would
+not be likely to alight on the surface of the sea, so that the dirt would
+not be washed off their feet; when making land, they would be sure to fly
+to their natural fresh-water haunts.  I do not believe that botanists are
+aware how charged the mud of ponds is with seeds:  I have tried several
+little experiments, but will here give only the most striking case:  I took
+in February three table-spoonfuls of mud from three different points,
+beneath water, on the edge of a little pond; this mud when dry weighed only
+6 3/4 ounces; I kept it covered up in my study for six months, pulling up
+and counting each plant as it grew; the plants were of many kinds, and were
+altogether 537 in number; and yet the viscid mud was all contained in a
+breakfast cup!  Considering these facts, I think it would be an
+inexplicable circumstance if water-birds did not transport the seeds of
+fresh-water plants to vast distances, and if consequently the range of
+these plants was not very great.  The same agency may have come into play
+with the eggs of some of the smaller fresh-water animals.
+</para>
+
+<para>
+Other and unknown agencies probably have also played a part.  I have stated
+that fresh-water fish eat some kinds of seeds, though they reject many
+other kinds after having swallowed them; even small fish swallow seeds of
+moderate size, as of the yellow water-lily and Potamogeton.  Herons and
+other birds, century after century, have gone on daily devouring fish; they
+then take flight and go to other waters, or are blown across the sea; and
+we have seen that seeds retain their power of germination, when rejected in
+pellets or in excrement, many hours afterwards.  When I saw the great size
+of the seeds of that fine water-lily, the Nelumbium, and remembered Alph.
+de Candolle's remarks on this plant, I thought that its distribution must
+remain quite inexplicable; but Audubon states that he found the seeds of
+the great southern water-lily (probably, according to Dr. Hooker, the
+Nelumbium luteum) in a heron's stomach; although I do not know the fact,
+yet analogy makes me believe that a heron flying to another pond and
+getting a hearty meal of fish, would probably reject from its stomach a
+pellet containing the seeds of the Nelumbium undigested; or the seeds might
+be dropped by the bird whilst feeding its young, in the same way as fish
+are known sometimes to be dropped.
+</para>
+
+<para>
+In considering these several means of distribution, it should be remembered
+that when a pond or stream is first formed, for instance, on a rising
+islet, it will be unoccupied; and a single seed or egg will have a good
+chance of succeeding.  Although there will always be a struggle for life
+between the individuals of the species, however few, already occupying any
+pond, yet as the number of kinds is small, compared with those on the land,
+the competition will probably be less severe between aquatic than between
+terrestrial species; consequently an intruder from the waters of a foreign
+country, would have a better chance of seizing on a place, than in the case
+of terrestrial colonists.  We should, also, remember that some, perhaps
+many, fresh-water productions are low in the scale of nature, and that we
+have reason to believe that such low beings change or become modified less
+quickly than the high; and this will give longer time than the average for
+the migration of the same aquatic species.  We should not forget the
+probability of many species having formerly ranged as continuously as
+fresh-water productions ever can range, over immense areas, and having
+subsequently become extinct in intermediate regions.  But the wide
+distribution of fresh-water plants and of the lower animals, whether
+retaining the same identical form or in some degree modified, I believe
+mainly depends on the wide dispersal of their seeds and eggs by animals,
+more especially by fresh-water birds, which have large powers of flight,
+and naturally travel from one to another and often distant piece of water. 
+Nature, like a careful gardener, thus takes her seeds from a bed of a
+particular nature, and drops them in another equally well fitted for them.
+</para>
+
+<para>
+On the Inhabitants of Oceanic Islands. -- We now come to the last of the
+three classes of facts, which I have selected as presenting the greatest
+amount of difficulty, on the view that all the individuals both of the same
+and of allied species have descended from a single parent; and therefore
+have all proceeded from a common birthplace, notwithstanding that in the
+course of time they have come to inhabit distant points of the globe.  I
+have already stated that I cannot honestly admit Forbes's view on
+continental extensions, which, if legitimately followed out, would lead to
+the belief that within the recent period all existing islands have been
+nearly or quite joined to some continent.  This view would remove many
+difficulties, but it would not, I think, explain all the facts in regard to
+insular productions.  In the following remarks I shall not confine myself
+to the mere question of dispersal; but shall consider some other facts,
+which bear on the truth of the two theories of independent creation and of
+descent with modification.
+</para>
+
+<para>
+The species of all kinds which inhabit oceanic islands are few in number
+compared with those on equal continental areas:  Alph. de Candolle admits
+this for plants, and Wollaston for insects.  If we look to the large size
+and varied stations of New Zealand, extending over 780 miles of latitude,
+and compare its flowering plants, only 750 in number, with those on an
+equal area at the Cape of Good Hope or in Australia, we must, I think,
+admit that something quite independently of any difference in physical
+conditions has caused so great a difference in number.  Even the uniform
+county of Cambridge has 847 plants, and the little island of Anglesea 764,
+but a few ferns and a few introduced plants are included in these numbers,
+and the comparison in some other respects is not quite fair.  We have
+evidence that the barren island of Ascension aboriginally possessed under
+half-a-dozen flowering plants; yet many have become naturalised on it, as
+they have on New Zealand and on every other oceanic island which can be
+named.  In St. Helena there is reason to believe that the naturalised
+plants and animals have nearly or quite exterminated many native
+productions.  He who admits the doctrine of the creation of each separate
+species, will have to admit, that a sufficient number of the best adapted
+plants and animals have not been created on oceanic islands; for man has
+unintentionally stocked them from various sources far more fully and
+perfectly than has nature.
+</para>
+
+<para>
+Although in oceanic islands the number of kinds of inhabitants is scanty,
+the proportion of endemic species (i.e. those found nowhere else in the
+world) is often extremely large.  If we compare, for instance, the number
+of the endemic land-shells in Madeira, or of the endemic birds in the
+Galapagos Archipelago, with the number found on any continent, and then
+compare the area of the islands with that of the continent, we shall see
+that this is true.  This fact might have been expected on my theory, for,
+as already explained, species occasionally arriving after long intervals in
+a new and isolated district, and having to compete with new associates,
+will be eminently liable to modification, and will often produce groups of
+modified descendants.  But it by no means follows, that, because in an
+island nearly all the species of one class are peculiar, those of another
+class, or of another section of the same class, are peculiar; and this
+difference seems to depend on the species which do not become modified
+having immigrated with facility and in a body, so that their mutual
+relations have not been much disturbed.  Thus in the Galapagos Islands
+nearly every land-bird, but only two out of the eleven marine birds, are
+peculiar; and it is obvious that marine birds could arrive at these islands
+more easily than land-birds.  Bermuda, on the other hand, which lies at
+about the same distance from North America as the Galapagos Islands do from
+South America, and which has a very peculiar soil, does not possess one
+endemic land bird; and we know from Mr. J. M. Jones's admirable account of
+Bermuda, that very many North American birds, during their great annual
+migrations, visit either periodically or occasionally this island.  Madeira
+does not possess one peculiar bird, and many European and African birds are
+almost every year blown there, as I am informed by Mr. E. V. Harcourt.  So
+that these two islands of Bermuda and Madeira have been stocked by birds,
+which for long ages have struggled together in their former homes, and have
+become mutually adapted to each other; and when settled in their new homes,
+each kind will have been kept by the others to their proper places and
+habits, and will consequently have been little liable to modification. 
+Madeira, again, is inhabited by a wonderful number of peculiar land-shells,
+whereas not one species of sea-shell is confined to its shores:  now,
+though we do not know how seashells are dispersed, yet we can see that
+their eggs or larvae, perhaps attached to seaweed or floating timber, or to
+the feet of wading-birds, might be transported far more easily than
+land-shells, across three or four hundred miles of open sea.  The different
+orders of insects in Madeira apparently present analogous facts.
+</para>
+
+<para>
+Oceanic islands are sometimes deficient in certain classes, and their
+places are apparently occupied by the other inhabitants; in the Galapagos
+Islands reptiles, and in New Zealand gigantic wingless birds, take the
+place of mammals.  In the plants of the Galapagos Islands, Dr. Hooker has
+shown that the proportional numbers of the different orders are very
+different from what they are elsewhere.  Such cases are generally accounted
+for by the physical conditions of the islands; but this explanation seems
+to me not a little doubtful.  Facility of immigration, I believe, has been
+at least as important as the nature of the conditions.
+</para>
+
+<para>
+Many remarkable little facts could be given with respect to the inhabitants
+of remote islands.  For instance, in certain islands not tenanted by
+mammals, some of the endemic plants have beautifully hooked seeds; yet few
+relations are more striking than the adaptation of hooked seeds for
+transportal by the wool and fur of quadrupeds.  This case presents no
+difficulty on my view, for a hooked seed might be transported to an island
+by some other means; and the plant then becoming slightly modified, but
+still retaining its hooked seeds, would form an endemic species, having as
+useless an appendage as any rudimentary organ,--for instance, as the
+shrivelled wings under the soldered elytra of many insular beetles.  Again,
+islands often possess trees or bushes belonging to orders which elsewhere
+include only herbaceous species; now trees, as Alph. de Candolle has shown,
+generally have, whatever the cause may be, confined ranges.  Hence trees
+would be little likely to reach distant oceanic islands; and an herbaceous
+plant, though it would have no chance of successfully competing in stature
+with a fully developed tree, when established on an island and having to
+compete with herbaceous plants alone, might readily gain an advantage by
+growing taller and taller and overtopping the other plants.  If so, natural
+selection would often tend to add to the stature of herbaceous plants when
+growing on an island, to whatever order they belonged, and thus convert
+them first into bushes and ultimately into trees.
+</para>
+
+<para>
+With respect to the absence of whole orders on oceanic islands, Bory St.
+Vincent long ago remarked that Batrachians (frogs, toads, newts) have never
+been found on any of the many islands with which the great oceans are
+studded.  I have taken pains to verify this assertion, and I have found it
+strictly true.  I have, however, been assured that a frog exists on the
+mountains of the great island of New Zealand; but I suspect that this
+exception (if the information be correct) may be explained through glacial
+agency.  This general absence of frogs, toads, and newts on so many oceanic
+islands cannot be accounted for by their physical conditions; indeed it
+seems that islands are peculiarly well fitted for these animals; for frogs
+have been introduced into Madeira, the Azores, and Mauritius, and have
+multiplied so as to become a nuisance.  But as these animals and their
+spawn are known to be immediately killed by sea-water, on my view we can
+see that there would be great difficulty in their transportal across the
+sea, and therefore why they do not exist on any oceanic island.  But why,
+on the theory of creation, they should not have been created there, it
+would be very difficult to explain.
+</para>
+
+<para>
+Mammals offer another and similar case.  I have carefully searched the
+oldest voyages, but have not finished my search; as yet I have not found a
+single instance, free from doubt, of a terrestrial mammal (excluding
+domesticated animals kept by the natives) inhabiting an island situated
+above 300 miles from a continent or great continental island; and many
+islands situated at a much less distance are equally barren.  The Falkland
+Islands, which are inhabited by a wolf-like fox, come nearest to an
+exception; but this group cannot be considered as oceanic, as it lies on a
+bank connected with the mainland; moreover, icebergs formerly brought
+boulders to its western shores, and they may have formerly transported
+foxes, as so frequently now happens in the arctic regions.  Yet it cannot
+be said that small islands will not support small mammals, for they occur
+in many parts of the world on very small islands, if close to a continent;
+and hardly an island can be named on which our smaller quadrupeds have not
+become naturalised and greatly multiplied.  It cannot be said, on the
+ordinary view of creation, that there has not been time for the creation of
+mammals; many volcanic islands are sufficiently ancient, as shown by the
+stupendous degradation which they have suffered and by their tertiary
+strata:  there has also been time for the production of endemic species
+belonging to other classes; and on continents it is thought that mammals
+appear and disappear at a quicker rate than other and lower animals. 
+Though terrestrial mammals do not occur on oceanic islands, aerial mammals
+do occur on almost every island.  New Zealand possesses two bats found
+nowhere else in the world:  Norfolk Island, the Viti Archipelago, the Bonin
+Islands, the Caroline and Marianne Archipelagoes, and Mauritius, all
+possess their peculiar bats.  Why, it may be asked, has the supposed
+creative force produced bats and no other mammals on remote islands?  On my
+view this question can easily be answered; for no terrestrial mammal can be
+transported across a wide space of sea, but bats can fly across.  Bats have
+been seen wandering by day far over the Atlantic Ocean; and two North
+American species either regularly or occasionally visit Bermuda, at the
+distance of 600 miles from the mainland.  I hear from Mr. Tomes, who has
+specially studied this family, that many of the same species have enormous
+ranges, and are found on continents and on far distant islands.  Hence we
+have only to suppose that such wandering species have been modified through
+natural selection in their new homes in relation to their new position, and
+we can understand the presence of endemic bats on islands, with the absence
+of all terrestrial mammals.
+</para>
+
+<para>
+Besides the absence of terrestrial mammals in relation to the remoteness of
+islands from continents, there is also a relation, to a certain extent
+independent of distance, between the depth of the sea separating an island
+from the neighbouring mainland, and the presence in both of the same
+mammiferous species or of allied species in a more or less modified
+condition.  Mr. Windsor Earl has made some striking observations on this
+head in regard to the great Malay Archipelago, which is traversed near
+Celebes by a space of deep ocean; and this space separates two widely
+distinct mammalian faunas.  On either side the islands are situated on
+moderately deep submarine banks, and they are inhabited by closely allied
+or identical quadrupeds.  No doubt some few anomalies occur in this great
+archipelago, and there is much difficulty in forming a judgment in some
+cases owing to the probable naturalisation of certain mammals through man's
+agency; but we shall soon have much light thrown on the natural history of
+this archipelago by the admirable zeal and researches of Mr. Wallace.  I
+have not as yet had time to follow up this subject in all other quarters of
+the world; but as far as I have gone, the relation generally holds good. 
+We see Britain separated by a shallow channel from Europe, and the mammals
+are the same on both sides; we meet with analogous facts on many islands
+separated by similar channels from Australia.  The West Indian Islands
+stand on a deeply submerged bank, nearly 1000 fathoms in depth, and here we
+find American forms, but the species and even the genera are distinct.  As
+the amount of modification in all cases depends to a certain degree on the
+lapse of time, and as during changes of level it is obvious that islands
+separated by shallow channels are more likely to have been continuously
+united within a recent period to the mainland than islands separated by
+deeper channels, we can understand the frequent relation between the depth
+of the sea and the degree of affinity of the mammalian inhabitants of
+islands with those of a neighbouring continent,--an inexplicable relation
+on the view of independent acts of creation.
+</para>
+
+<para>
+All the foregoing remarks on the inhabitants of oceanic islands,--namely,
+the scarcity of kinds--the richness in endemic forms in particular classes
+or sections of classes,--the absence of whole groups, as of batrachians,
+and of terrestrial mammals notwithstanding the presence of aerial
+bats,--the singular proportions of certain orders of plants,--herbaceous
+forms having been developed into trees, &amp;c.,--seem to me to accord better
+with the view of occasional means of transport having been largely
+efficient in the long course of time, than with the view of all our oceanic
+islands having been formerly connected by continuous land with the nearest
+continent; for on this latter view the migration would probably have been
+more complete; and if modification be admitted, all the forms of life would
+have been more equally modified, in accordance with the paramount
+importance of the relation of organism to organism.
+</para>
+
+<para>
+I do not deny that there are many and grave difficulties in understanding
+how several of the inhabitants of the more remote islands, whether still
+retaining the same specific form or modified since their arrival, could
+have reached their present homes.  But the probability of many islands
+having existed as halting-places, of which not a wreck now remains, must
+not be overlooked.  I will here give a single instance of one of the cases
+of difficulty.  Almost all oceanic islands, even the most isolated and
+smallest, are inhabited by land-shells, generally by endemic species, but
+sometimes by species found elsewhere.  Dr. Aug. A. Gould has given several
+interesting cases in regard to the land-shells of the islands of the
+Pacific.  Now it is notorious that land-shells are very easily killed by
+salt; their eggs, at least such as I have tried, sink in sea-water and are
+killed by it.  Yet there must be, on my view, some unknown, but highly
+efficient means for their transportal.  Would the just-hatched young
+occasionally crawl on and adhere to the feet of birds roosting on the
+ground, and thus get transported?  It occurred to me that land-shells, when
+hybernating and having a membranous diaphragm over the mouth of the shell,
+might be floated in chinks of drifted timber across moderately wide arms of
+the sea.  And I found that several species did in this state withstand
+uninjured an immersion in sea-water during seven days:  one of these shells
+was the Helix pomatia, and after it had again hybernated I put it in
+sea-water for twenty days, and it perfectly recovered.  As this species has
+a thick calcareous operculum, I removed it, and when it had formed a new
+membranous one, I immersed it for fourteen days in sea-water, and it
+recovered and crawled away:  but more experiments are wanted on this head.
+</para>
+
+<para>
+The most striking and important fact for us in regard to the inhabitants of
+islands, is their affinity to those of the nearest mainland, without being
+actually the same species.  Numerous instances could be given of this fact. 
+I will give only one, that of the Galapagos Archipelago, situated under the
+equator, between 500 and 600 miles from the shores of South America.  Here
+almost every product of the land and water bears the unmistakeable stamp of
+the American continent.  There are twenty-six land birds, and twenty-five
+of these are ranked by Mr. Gould as distinct species, supposed to have been
+created here; yet the close affinity of most of these birds to American
+species in every character, in their habits, gestures, and tones of voice,
+was manifest.  So it is with the other animals, and with nearly all the
+plants, as shown by Dr. Hooker in his admirable memoir on the Flora of this
+archipelago.  The naturalist, looking at the inhabitants of these volcanic
+islands in the Pacific, distant several hundred miles from the continent,
+yet feels that he is standing on American land.  Why should this be so? why
+should the species which are supposed to have been created in the Galapagos
+Archipelago, and nowhere else, bear so plain a stamp of affinity to those
+created in America?  There is nothing in the conditions of life, in the
+geological nature of the islands, in their height or climate, or in the
+proportions in which the several classes are associated together, which
+resembles closely the conditions of the South American coast:  in fact
+there is a considerable dissimilarity in all these respects.  On the other
+hand, there is a considerable degree of resemblance in the volcanic nature
+of the soil, in climate, height, and size of the islands, between the
+Galapagos and Cape de Verde Archipelagos:  but what an entire and absolute
+difference in their inhabitants!  The inhabitants of the Cape de Verde
+Islands are related to those of Africa, like those of the Galapagos to
+America.  I believe this grand fact can receive no sort of explanation on
+the ordinary view of independent creation; whereas on the view here
+maintained, it is obvious that the Galapagos Islands would be likely to
+receive colonists, whether by occasional means of transport or by formerly
+continuous land, from America; and the Cape de Verde Islands from Africa;
+and that such colonists would be liable to modification;--the principle of
+inheritance still betraying their original birthplace.
+</para>
+
+<para>
+Many analogous facts could be given:  indeed it is an almost universal rule
+that the endemic productions of islands are related to those of the nearest
+continent, or of other near islands.  The exceptions are few, and most of
+them can be explained.  Thus the plants of Kerguelen Land, though standing
+nearer to Africa than to America, are related, and that very closely, as we
+know from Dr. Hooker's account, to those of America:  but on the view that
+this island has been mainly stocked by seeds brought with earth and stones
+on icebergs, drifted by the prevailing currents, this anomaly disappears. 
+New Zealand in its endemic plants is much more closely related to
+Australia, the nearest mainland, than to any other region:  and this is
+what might have been expected; but it is also plainly related to South
+America, which, although the next nearest continent, is so enormously
+remote, that the fact becomes an anomaly.  But this difficulty almost
+disappears on the view that both New Zealand, South America, and other
+southern lands were long ago partially stocked from a nearly intermediate
+though distant point, namely from the antarctic islands, when they were
+clothed with vegetation, before the commencement of the Glacial period. 
+The affinity, which, though feeble, I am assured by Dr. Hooker is real,
+between the flora of the south-western corner of Australia and of the Cape
+of Good Hope, is a far more remarkable case, and is at present
+inexplicable:  but this affinity is confined to the plants, and will, I do
+not doubt, be some day explained.
+</para>
+
+<para>
+The law which causes the inhabitants of an archipelago, though specifically
+distinct, to be closely allied to those of the nearest continent, we
+sometimes see displayed on a small scale, yet in a most interesting manner,
+within the limits of the same archipelago.  Thus the several islands of the
+Galapagos Archipelago are tenanted, as I have elsewhere shown, in a quite
+marvellous manner, by very closely related species; so that the inhabitants
+of each separate island, though mostly distinct, are related in an
+incomparably closer degree to each other than to the inhabitants of any
+other part of the world.  And this is just what might have been expected on
+my view, for the islands are situated so near each other that they would
+almost certainly receive immigrants from the same original source, or from
+each other.  But this dissimilarity between the endemic inhabitants of the
+islands may be used as an argument against my views; for it may be asked,
+how has it happened in the several islands situated within sight of each
+other, having the same geological nature, the same height, climate, &amp;c.,
+that many of the immigrants should have been differently modified, though
+only in a small degree.  This long appeared to me a great difficulty:  but
+it arises in chief part from the deeply-seated error of considering the
+physical conditions of a country as the most important for its inhabitants;
+whereas it cannot, I think, be disputed that the nature of the other
+inhabitants, with which each has to compete, is at least as important, and
+generally a far more important element of success.  Now if we look to those
+inhabitants of the Galapagos Archipelago which are found in other parts of
+the world (laying on one side for the moment the endemic species, which
+cannot be here fairly included, as we are considering how they have come to
+be modified since their arrival), we find a considerable amount of
+difference in the several islands.  This difference might indeed have been
+expected on the view of the islands having been stocked by occasional means
+of transport--a seed, for instance, of one plant having been brought to one
+island, and that of another plant to another island.  Hence when in former
+times an immigrant settled on any one or more of the islands, or when it
+subsequently spread from one island to another, it would undoubtedly be
+exposed to different conditions of life in the different islands, for it
+would have to compete with different sets of organisms:  a plant, for
+instance, would find the best-fitted ground more perfectly occupied by
+distinct plants in one island than in another, and it would be exposed to
+the attacks of somewhat different enemies.  If then it varied, natural
+selection would probably favour different varieties in the different
+islands.  Some species, however, might spread and yet retain the same
+character throughout the group, just as we see on continents some species
+spreading widely and remaining the same.
+</para>
+
+<para>
+The really surprising fact in this case of the Galapagos Archipelago, and
+in a lesser degree in some analogous instances, is that the new species
+formed in the separate islands have not quickly spread to the other
+islands.  But the islands, though in sight of each other, are separated by
+deep arms of the sea, in most cases wider than the British Channel, and
+there is no reason to suppose that they have at any former period been
+continuously united.  The currents of the sea are rapid and sweep across
+the archipelago, and gales of wind are extraordinarily rare; so that the
+islands are far more effectually separated from each other than they appear
+to be on a map.  Nevertheless a good many species, both those found in
+other parts of the world and those confined to the archipelago, are common
+to the several islands, and we may infer from certain facts that these have
+probably spread from some one island to the others.  But we often take, I
+think, an erroneous view of the probability of closely allied species
+invading each other's territory, when put into free intercommunication. 
+Undoubtedly if one species has any advantage whatever over another, it will
+in a very brief time wholly or in part supplant it; but if both are equally
+well fitted for their own places in nature, both probably will hold their
+own places and keep separate for almost any length of time.  Being familiar
+with the fact that many species, naturalised through man's agency, have
+spread with astonishing rapidity over new countries, we are apt to infer
+that most species would thus spread; but we should remember that the forms
+which become naturalised in new countries are not generally closely allied
+to the aboriginal inhabitants, but are very distinct species, belonging in
+a large proportion of cases, as shown by Alph. de Candolle, to distinct
+genera.  In the Galapagos Archipelago, many even of the birds, though so
+well adapted for flying from island to island, are distinct on each; thus
+there are three closely-allied species of mocking-thrush, each confined to
+its own island.  Now let us suppose the mocking-thrush of Chatham Island to
+be blown to Charles Island, which has its own mocking-thrush:  why should
+it succeed in establishing itself there?  We may safely infer that Charles
+Island is well stocked with its own species, for annually more eggs are
+laid there than can possibly be reared; and we may infer that the
+mocking-thrush peculiar to Charles Island is at least as well fitted for
+its home as is the species peculiar to Chatham Island.  Sir C. Lyell and
+Mr. Wollaston have communicated to me a remarkable fact bearing on this
+subject; namely, that Madeira and the adjoining islet of Porto Santo
+possess many distinct but representative land-shells, some of which live in
+crevices of stone; and although large quantities of stone are annually
+transported from Porto Santo to Madeira, yet this latter island has not
+become colonised by the Porto Santo species:  nevertheless both islands
+have been colonised by some European land-shells, which no doubt had some
+advantage over the indigenous species.  From these considerations I think
+we need not greatly marvel at the endemic and representative species, which
+inhabit the several islands of the Galapagos Archipelago, not having
+universally spread from island to island.  In many other instances, as in
+the several districts of the same continent, pre-occupation has probably
+played an important part in checking the commingling of species under the
+same conditions of life.  Thus, the south-east and south-west corners of
+Australia have nearly the same physical conditions, and are united by
+continuous land, yet they are inhabited by a vast number of distinct
+mammals, birds, and plants.
+</para>
+
+<para>
+The principle which determines the general character of the fauna and flora
+of oceanic islands, namely, that the inhabitants, when not identically the
+same, yet are plainly related to the inhabitants of that region whence
+colonists could most readily have been derived,--the colonists having been
+subsequently modified and better fitted to their new homes,--is of the
+widest application throughout nature.  We see this on every mountain, in
+every lake and marsh.  For Alpine species, excepting in so far as the same
+forms, chiefly of plants, have spread widely throughout the world during
+the recent Glacial epoch, are related to those of the surrounding
+lowlands;--thus we have in South America, Alpine humming-birds, Alpine
+rodents, Alpine plants, &amp;c., all of strictly American forms, and it is
+obvious that a mountain, as it became slowly upheaved, would naturally be
+colonised from the surrounding lowlands.  So it is with the inhabitants of
+lakes and marshes, excepting in so far as great facility of transport has
+given the same general forms to the whole world.  We see this same
+principle in the blind animals inhabiting the caves of America and of
+Europe.  Other analogous facts could be given.  And it will, I believe, be
+universally found to be true, that wherever in two regions, let them be
+ever so distant, many closely allied or representative species occur, there
+will likewise be found some identical species, showing, in accordance with
+the foregoing view, that at some former period there has been
+intercommunication or migration between the two regions.  And wherever many
+closely-allied species occur, there will be found many forms which some
+naturalists rank as distinct species, and some as varieties; these doubtful
+forms showing us the steps in the process of modification.
+</para>
+
+<para>
+This relation between the power and extent of migration of a species,
+either at the present time or at some former period under different
+physical conditions, and the existence at remote points of the world of
+other species allied to it, is shown in another and more general way.  Mr.
+Gould remarked to me long ago, that in those genera of birds which range
+over the world, many of the species have very wide ranges.  I can hardly
+doubt that this rule is generally true, though it would be difficult to
+prove it.  Amongst mammals, we see it strikingly displayed in Bats, and in
+a lesser degree in the Felidae and Canidae.  We see it, if we compare the
+distribution of butterflies and beetles.  So it is with most fresh-water
+productions, in which so many genera range over the world, and many
+individual species have enormous ranges.  It is not meant that in
+world-ranging genera all the species have a wide range, or even that they
+have on an average a wide range; but only that some of the species range
+very widely; for the facility with which widely-ranging species vary and
+give rise to new forms will largely determine their average range.  For
+instance, two varieties of the same species inhabit America and Europe, and
+the species thus has an immense range; but, if the variation had been a
+little greater, the two varieties would have been ranked as distinct
+species, and the common range would have been greatly reduced.  Still less
+is it meant, that a species which apparently has the capacity of crossing
+barriers and ranging widely, as in the case of certain powerfully-winged
+birds, will necessarily range widely; for we should never forget that to
+range widely implies not only the power of crossing barriers, but the more
+important power of being victorious in distant lands in the struggle for
+life with foreign associates.  But on the view of all the species of a
+genus having descended from a single parent, though now distributed to the
+most remote points of the world, we ought to find, and I believe as a
+general rule we do find, that some at least of the species range very
+widely; for it is necessary that the unmodified parent should range widely,
+undergoing modification during its diffusion, and should place itself under
+diverse conditions favourable for the conversion of its offspring, firstly
+into new varieties and ultimately into new species.
+</para>
+
+<para>
+In considering the wide distribution of certain genera, we should bear in
+mind that some are extremely ancient, and must have branched off from a
+common parent at a remote epoch; so that in such cases there will have been
+ample time for great climatal and geographical changes and for accidents of
+transport; and consequently for the migration of some of the species into
+all quarters of the world, where they may have become slightly modified in
+relation to their new conditions.  There is, also, some reason to believe
+from geological evidence that organisms low in the scale within each great
+class, generally change at a slower rate than the higher forms; and
+consequently the lower forms will have had a better chance of ranging
+widely and of still retaining the same specific character.  This fact,
+together with the seeds and eggs of many low forms being very minute and
+better fitted for distant transportation, probably accounts for a law which
+has long been observed, and which has lately been admirably discussed by
+Alph. de Candolle in regard to plants, namely, that the lower any group of
+organisms is, the more widely it is apt to range.
+</para>
+
+<para>
+The relations just discussed,--namely, low and slowly-changing organisms
+ranging more widely than the high,--some of the species of widely-ranging
+genera themselves ranging widely,--such facts, as alpine, lacustrine, and
+marsh productions being related (with the exceptions before specified) to
+those on the surrounding low lands and dry lands, though these stations are
+so different--the very close relation of the distinct species which inhabit
+the islets of the same archipelago,--and especially the striking relation
+of the inhabitants of each whole archipelago or island to those of the
+nearest mainland,--are, I think, utterly inexplicable on the ordinary view
+of the independent creation of each species, but are explicable on the view
+of colonisation from the nearest and readiest source, together with the
+subsequent modification and better adaptation of the colonists to their new
+homes.
+</para>
+
+<para>
+Summary of last and present Chapters -- In these chapters I have
+endeavoured to show, that if we make due allowance for our ignorance of the
+full effects of all the changes of climate and of the level of the land,
+which have certainly occurred within the recent period, and of other
+similar changes which may have occurred within the same period; if we
+remember how profoundly ignorant we are with respect to the many and
+curious means of occasional transport,--a subject which has hardly ever
+been properly experimentised on; if we bear in mind how often a species may
+have ranged continuously over a wide area, and then have become extinct in
+the intermediate tracts, I think the difficulties in believing that all the
+individuals of the same species, wherever located, have descended from the
+same parents, are not insuperable.  And we are led to this conclusion,
+which has been arrived at by many naturalists under the designation of
+single centres of creation, by some general considerations, more especially
+from the importance of barriers and from the analogical distribution of
+sub-genera, genera, and families.
+</para>
+
+<para>
+With respect to the distinct species of the same genus, which on my theory
+must have spread from one parent-source; if we make the same allowances as
+before for our ignorance, and remember that some forms of life change most
+slowly, enormous periods of time being thus granted for their migration, I
+do not think that the difficulties are insuperable; though they often are
+in this case, and in that of the individuals of the same species, extremely
+grave.
+</para>
+
+<para>
+As exemplifying the effects of climatal changes on distribution, I have
+attempted to show how important has been the influence of the modern
+Glacial period, which I am fully convinced simultaneously affected the
+whole world, or at least great meridional belts.  As showing how
+diversified are the means of occasional transport, I have discussed at some
+little length the means of dispersal of fresh-water productions.
+</para>
+
+<para>
+If the difficulties be not insuperable in admitting that in the long course
+of time the individuals of the same species, and likewise of allied
+species, have proceeded from some one source; then I think all the grand
+leading facts of geographical distribution are explicable on the theory of
+migration (generally of the more dominant forms of life), together with
+subsequent modification and the multiplication of new forms.  We can thus
+understand the high importance of barriers, whether of land or water, which
+separate our several zoological and botanical provinces.  We can thus
+understand the localisation of sub-genera, genera, and families; and how it
+is that under different latitudes, for instance in South America, the
+inhabitants of the plains and mountains, of the forests, marshes, and
+deserts, are in so mysterious a manner linked together by affinity, and are
+likewise linked to the extinct beings which formerly inhabited the same
+continent.  Bearing in mind that the mutual relations of organism to
+organism are of the highest importance, we can see why two areas having
+nearly the same physical conditions should often be inhabited by very
+different forms of life; for according to the length of time which has
+elapsed since new inhabitants entered one region; according to the nature
+of the communication which allowed certain forms and not others to enter,
+either in greater or lesser numbers; according or not, as those which
+entered happened to come in more or less direct competition with each other
+and with the aborigines; and according as the immigrants were capable of
+varying more or less rapidly, there would ensue in different regions,
+independently of their physical conditions, infinitely diversified
+conditions of life,--there would be an almost endless amount of organic
+action and reaction,--and we should find, as we do find, some groups of
+beings greatly, and some only slightly modified,--some developed in great
+force, some existing in scanty numbers--in the different great geographical
+provinces of the world.
+</para>
+
+<para>
+On these same principles, we can understand, as I have endeavoured to show,
+why oceanic islands should have few inhabitants, but of these a great
+number should be endemic or peculiar; and why, in relation to the means of
+migration, one group of beings, even within the same class, should have all
+its species endemic, and another group should have all its species common
+to other quarters of the world.  We can see why whole groups of organisms,
+as batrachians and terrestrial mammals, should be absent from oceanic
+islands, whilst the most isolated islands possess their own peculiar
+species of aerial mammals or bats.  We can see why there should be some
+relation between the presence of mammals, in a more or less modified
+condition, and the depth of the sea between an island and the mainland.  We
+can clearly see why all the inhabitants of an archipelago, though
+specifically distinct on the several islets, should be closely related to
+each other, and likewise be related, but less closely, to those of the
+nearest continent or other source whence immigrants were probably derived. 
+We can see why in two areas, however distant from each other, there should
+be a correlation, in the presence of identical species, of varieties, of
+doubtful species, and of distinct but representative species.
+</para>
+
+<para>
+As the late Edward Forbes often insisted, there is a striking parallelism
+in the laws of life throughout time and space:  the laws governing the
+succession of forms in past times being nearly the same with those
+governing at the present time the differences in different areas.  We see
+this in many facts.  The endurance of each species and group of species is
+continuous in time; for the exceptions to the rule are so few, that they
+may fairly be attributed to our not having as yet discovered in an
+intermediate deposit the forms which are therein absent, but which occur
+above and below:  so in space, it certainly is the general rule that the
+area inhabited by a single species, or by a group of species, is
+continuous; and the exceptions, which are not rare, may, as I have
+attempted to show, be accounted for by migration at some former period
+under different conditions or by occasional means of transport, and by the
+species having become extinct in the intermediate tracts.  Both in time and
+space, species and groups of species have their points of maximum
+development.  Groups of species, belonging either to a certain period of
+time, or to a certain area, are often characterised by trifling characters
+in common, as of sculpture or colour.  In looking to the long succession of
+ages, as in now looking to distant provinces throughout the world, we find
+that some organisms differ little, whilst others belonging to a different
+class, or to a different order, or even only to a different family of the
+same order, differ greatly.  In both time and space the lower members of
+each class generally change less than the higher; but there are in both
+cases marked exceptions to the rule.  On my theory these several relations
+throughout time and space are intelligible; for whether we look to the
+forms of life which have changed during successive ages within the same
+quarter of the world, or to those which have changed after having migrated
+into distant quarters, in both cases the forms within each class have been
+connected by the same bond of ordinary generation; and the more nearly any
+two forms are related in blood, the nearer they will generally stand to
+each other in time and space; in both cases the laws of variation have been
+the same, and modifications have been accumulated by the same power of
+natural selection.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+
+<chapnum>Chapter XIII</chapnum>
+
+<title>Mutual Affinities of Organic Beings:
+Morphology:  Embryology:  Rudimentary Organs</title>
+
+<chapsummary>
+Classification, groups subordinate to groups -- Natural system -- Rules and
+difficulties in classification, explained on the theory of descent with
+modification -- Classification of varieties -- Descent always used in
+classification -- Analogical or adaptive characters -- Affinities, general,
+complex and radiating -- Extinction separates and defines groups --
+Morphology, between members of the same class, between parts of the same
+individual -- Embryology, laws of, explained by variations not supervening
+at an early age, and being inherited at a corresponding age -- Rudimentary
+Organs; their origin explained -- Summary.
+</chapsummary>
+</chapheader>
+
+<para>
+From the first dawn of life, all organic beings are found to resemble each
+other in descending degrees, so that they can be classed in groups under
+groups.  This classification is evidently not arbitrary like the grouping
+of the stars in constellations.  The existence of groups would have been of
+simple signification, if one group had been exclusively fitted to inhabit
+the land, and another the water; one to feed on flesh, another on vegetable
+matter, and so on; but the case is widely different in nature; for it is
+notorious how commonly members of even the same subgroup have different
+habits.  In our second and fourth chapters, on Variation and on Natural
+Selection, I have attempted to show that it is the widely ranging, the much
+diffused and common, that is the dominant species belonging to the larger
+genera, which vary most.  The varieties, or incipient species, thus
+produced ultimately become converted, as I believe, into new and distinct
+species; and these, on the principle of inheritance, tend to produce other
+new and dominant species.  Consequently the groups which are now large, and
+which generally include many dominant species, tend to go on increasing
+indefinitely in size.  I further attempted to show that from the varying
+descendants of each species trying to occupy as many and as different
+places as possible in the economy of nature, there is a constant tendency
+in their characters to diverge.  This conclusion was supported by looking
+at the great diversity of the forms of life which, in any small area, come
+into the closest competition, and by looking to certain facts in
+naturalisation.
+</para>
+
+<para>
+I attempted also to show that there is a constant tendency in the forms
+which are increasing in number and diverging in character, to supplant and
+exterminate the less divergent, the less improved, and preceding forms.  I
+request the reader to turn to the diagram illustrating the action, as
+formerly explained, of these several principles; and he will see that the
+inevitable result is that the modified descendants proceeding from one
+progenitor become broken up into groups subordinate to groups.  In the
+diagram each letter on the uppermost line may represent a genus including
+several species; and all the genera on this line form together one class,
+for all have descended from one ancient but unseen parent, and,
+consequently, have inherited something in common.  But the three genera on
+the left hand have, on this same principle, much in common, and form a
+sub-family, distinct from that including the next two genera on the right
+hand, which diverged from a common parent at the fifth stage of descent. 
+These five genera have also much, though less, in common; and they form a
+family distinct from that including the three genera still further to the
+right hand, which diverged at a still earlier period.  And all these
+genera, descended from (A), form an order distinct from the genera
+descended from (I).  So that we here have many species descended from a
+single progenitor grouped into genera; and the genera are included in, or
+subordinate to, sub-families, families, and orders, all united into one
+class.  Thus, the grand fact in natural history of the subordination of
+group under group, which, from its familiarity, does not always
+sufficiently strike us, is in my judgment fully explained.
+</para>
+
+<para>
+Naturalists try to arrange the species, genera, and families in each class,
+on what is called the Natural System.  But what is meant by this system? 
+Some authors look at it merely as a scheme for arranging together those
+living objects which are most alike, and for separating those which are
+most unlike; or as an artificial means for enunciating, as briefly as
+possible, general propositions,--that is, by one sentence to give the
+characters common, for instance, to all mammals, by another those common to
+all carnivora, by another those common to the dog-genus, and then by adding
+a single sentence, a full description is given of each kind of dog.  The
+ingenuity and utility of this system are indisputable.  But many
+naturalists think that something more is meant by the Natural System; they
+believe that it reveals the plan of the Creator; but unless it be specified
+whether order in time or space, or what else is meant by the plan of the
+Creator, it seems to me that nothing is thus added to our knowledge.  Such
+expressions as that famous one of Linnaeus, and which we often meet with in
+a more or less concealed form, that the characters do not make the genus,
+but that the genus gives the characters, seem to imply that something more
+is included in our classification, than mere resemblance.  I believe that
+something more is included; and that propinquity of descent,--the only
+known cause of the similarity of organic beings,--is the bond, hidden as it
+is by various degrees of modification, which is partially revealed to us by
+our classifications.
+</para>
+
+<para>
+Let us now consider the rules followed in classification, and the
+difficulties which are encountered on the view that classification either
+gives some unknown plan of creation, or is simply a scheme for enunciating
+general propositions and of placing together the forms most like each
+other.  It might have been thought (and was in ancient times thought) that
+those parts of the structure which determined the habits of life, and the
+general place of each being in the economy of nature, would be of very high
+importance in classification.  Nothing can be more false.  No one regards
+the external similarity of a mouse to a shrew, of a dugong to a whale, of a
+whale to a fish, as of any importance.  These resemblances, though so
+intimately connected with the whole life of the being, are ranked as merely
+'adaptive or analogical characters;' but to the consideration of these
+resemblances we shall have to recur.  It may even be given as a general
+rule, that the less any part of the organisation is concerned with special
+habits, the more important it becomes for classification.  As an instance: 
+Owen, in speaking of the dugong, says, 'The generative organs being those
+which are most remotely related to the habits and food of an animal, I have
+always regarded as affording very clear indications of its true affinities. 
+We are least likely in the modifications of these organs to mistake a
+merely adaptive for an essential character.'  So with plants, how
+remarkable it is that the organs of vegetation, on which their whole life
+depends, are of little signification, excepting in the first main
+divisions; whereas the organs of reproduction, with their product the seed,
+are of paramount importance!
+</para>
+
+<para>
+We must not, therefore, in classifying, trust to resemblances in parts of
+the organisation, however important they may be for the welfare of the
+being in relation to the outer world.  Perhaps from this cause it has
+partly arisen, that almost all naturalists lay the greatest stress on
+resemblances in organs of high vital or physiological importance.  No doubt
+this view of the classificatory importance of organs which are important is
+generally, but by no means always, true.  But their importance for
+classification, I believe, depends on their greater constancy throughout
+large groups of species; and this constancy depends on such organs having
+generally been subjected to less change in the adaptation of the species to
+their conditions of life.  That the mere physiological importance of an
+organ does not determine its classificatory value, is almost shown by the
+one fact, that in allied groups, in which the same organ, as we have every
+reason to suppose, has nearly the same physiological value, its
+classificatory value is widely different.  No naturalist can have worked at
+any group without being struck with this fact; and it has been most fully
+acknowledged in the writings of almost every author.  It will suffice to
+quote the highest authority, Robert Brown, who in speaking of certain
+organs in the Proteaceae, says their generic importance, 'like that of all
+their parts, not only in this but, as I apprehend, in every natural family,
+is very unequal, and in some cases seems to be entirely lost.'  Again in
+another work he says, the genera of the Connaraceae 'differ in having one
+or more ovaria, in the existence or absence of albumen, in the imbricate or
+valvular aestivation.  Any one of these characters singly is frequently of
+more than generic importance, though here even when all taken together they
+appear insufficient to separate Cnestis from Connarus.'  To give an example
+amongst insects, in one great division of the Hymenoptera, the antennae, as
+Westwood has remarked, are most constant in structure; in another division
+they differ much, and the differences are of quite subordinate value in
+classification; yet no one probably will say that the antennae in these two
+divisions of the same order are of unequal physiological importance.  Any
+number of instances could be given of the varying importance for
+classification of the same important organ within the same group of beings.
+</para>
+
+<para>
+Again, no one will say that rudimentary or atrophied organs are of high
+physiological or vital importance; yet, undoubtedly, organs in this
+condition are often of high value in classification.  No one will dispute
+that the rudimentary teeth in the upper jaws of young ruminants, and
+certain rudimentary bones of the leg, are highly serviceable in exhibiting
+the close affinity between Ruminants and Pachyderms.  Robert Brown has
+strongly insisted on the fact that the rudimentary florets are of the
+highest importance in the classification of the Grasses.
+</para>
+
+<para>
+Numerous instances could be given of characters derived from parts which
+must be considered of very trifling physiological importance, but which are
+universally admitted as highly serviceable in the definition of whole
+groups.  For instance, whether or not there is an open passage from the
+nostrils to the mouth, the only character, according to Owen, which
+absolutely distinguishes fishes and reptiles--the inflection of the angle
+of the jaws in Marsupials--the manner in which the wings of insects are
+folded--mere colour in certain Algae--mere pubescence on parts of the
+flower in grasses--the nature of the dermal covering, as hair or feathers,
+in the Vertebrata.  If the Ornithorhynchus had been covered with feathers
+instead of hair, this external and trifling character would, I think, have
+been considered by naturalists as important an aid in determining the
+degree of affinity of this strange creature to birds and reptiles, as an
+approach in structure in any one internal and important organ.
+</para>
+
+<para>
+The importance, for classification, of trifling characters, mainly depends
+on their being correlated with several other characters of more or less
+importance.  The value indeed of an aggregate of characters is very evident
+in natural history.  Hence, as has often been remarked, a species may
+depart from its allies in several characters, both of high physiological
+importance and of almost universal prevalence, and yet leave us in no doubt
+where it should be ranked.  Hence, also, it has been found, that a
+classification founded on any single character, however important that may
+be, has always failed; for no part of the organisation is universally
+constant.  The importance of an aggregate of characters, even when none are
+important, alone explains, I think, that saying of Linnaeus, that the
+characters do not give the genus, but the genus gives the characters; for
+this saying seems founded on an appreciation of many trifling points of
+resemblance, too slight to be defined.  Certain plants, belonging to the
+Malpighiaceae, bear perfect and degraded flowers; in the latter, as A. de
+Jussieu has remarked, 'the greater number of the characters proper to the
+species, to the genus, to the family, to the class, disappear, and thus
+laugh at our classification.'  But when Aspicarpa produced in France,
+during several years, only degraded flowers, departing so wonderfully in a
+number of the most important points of structure from the proper type of
+the order, yet M. Richard sagaciously saw, as Jussieu observes, that this
+genus should still be retained amongst the Malpighiaceae.  This case seems
+to me well to illustrate the spirit with which our classifications are
+sometimes necessarily founded.
+</para>
+
+<para>
+Practically when naturalists are at work, they do not trouble themselves
+about the physiological value of the characters which they use in defining
+a group, or in allocating any particular species.  If they find a character
+nearly uniform, and common to a great number of forms, and not common to
+others, they use it as one of high value; if common to some lesser number,
+they use it as of subordinate value.  This principle has been broadly
+confessed by some naturalists to be the true one; and by none more clearly
+than by that excellent botanist, Aug. St. Hilaire.  If certain characters
+are always found correlated with others, though no apparent bond of
+connexion can be discovered between them, especial value is set on them. 
+As in most groups of animals, important organs, such as those for
+propelling the blood, or for aerating it, or those for propagating the
+race, are found nearly uniform, they are considered as highly serviceable
+in classification; but in some groups of animals all these, the most
+important vital organs, are found to offer characters of quite subordinate
+value.
+</para>
+
+<para>
+We can see why characters derived from the embryo should be of equal
+importance with those derived from the adult, for our classifications of
+course include all ages of each species.  But it is by no means obvious, on
+the ordinary view, why the structure of the embryo should be more important
+for this purpose than that of the adult, which alone plays its full part in
+the economy of nature.  Yet it has been strongly urged by those great
+naturalists, Milne Edwards and Agassiz, that embryonic characters are the
+most important of any in the classification of animals; and this doctrine
+has very generally been admitted as true.  The same fact holds good with
+flowering plants, of which the two main divisions have been founded on
+characters derived from the embryo,--on the number and position of the
+embryonic leaves or cotyledons, and on the mode of development of the
+plumule and radicle.  In our discussion on embryology, we shall see why
+such characters are so valuable, on the view of classification tacitly
+including the idea of descent.
+</para>
+
+<para>
+Our classifications are often plainly influenced by chains of affinities. 
+Nothing can be easier than to define a number of characters common to all
+birds; but in the case of crustaceans, such definition has hitherto been
+found impossible.  There are crustaceans at the opposite ends of the
+series, which have hardly a character in common; yet the species at both
+ends, from being plainly allied to others, and these to others, and so
+onwards, can be recognised as unequivocally belonging to this, and to no
+other class of the Articulata.
+</para>
+
+<para>
+Geographical distribution has often been used, though perhaps not quite
+logically, in classification, more especially in very large groups of
+closely allied forms.  Temminck insists on the utility or even necessity of
+this practice in certain groups of birds; and it has been followed by
+several entomologists and botanists.
+</para>
+
+<para>
+Finally, with respect to the comparative value of the various groups of
+species, such as orders, sub-orders, families, sub-families, and genera,
+they seem to be, at least at present, almost arbitrary.  Several of the
+best botanists, such as Mr. Bentham and others, have strongly insisted on
+their arbitrary value.  Instances could be given amongst plants and
+insects, of a group of forms, first ranked by practised naturalists as only
+a genus, and then raised to the rank of a sub-family or family; and this
+has been done, not because further research has detected important
+structural differences, at first overlooked, but because numerous allied
+species, with slightly different grades of difference, have been
+subsequently discovered.
+</para>
+	
+<para>
+All the foregoing rules and aids and difficulties in classification are
+explained, if I do not greatly deceive myself, on the view that the natural
+system is founded on descent with modification; that the characters which
+naturalists consider as showing true affinity between any two or more
+species, are those which have been inherited from a common parent, and, in
+so far, all true classification is genealogical; that community of descent
+is the hidden bond which naturalists have been unconsciously seeking, and
+not some unknown plan of creation, or the enunciation of general
+propositions, and the mere putting together and separating objects more or
+less alike.
+</para>
+
+<para>
+But I must explain my meaning more fully.  I believe that the arrangement
+of the groups within each class, in due subordination and relation to the
+other groups, must be strictly genealogical in order to be natural; but
+that the amount of difference in the several branches or groups, though
+allied in the same degree in blood to their common progenitor, may differ
+greatly, being due to the different degrees of modification which they have
+undergone; and this is expressed by the forms being ranked under different
+genera, families, sections, or orders.  The reader will best understand
+what is meant, if he will take the trouble of referring to the diagram in
+the fourth chapter.  We will suppose the letters A to L to represent allied
+genera, which lived during the Silurian epoch, and these have descended
+from a species which existed at an unknown anterior period.  Species of
+three of these genera (A, F, and I) have transmitted modified descendants
+to the present day, represented by the fifteen genera (a14 to z14) on the
+uppermost horizontal line.  Now all these modified descendants from a
+single species, are represented as related in blood or descent to the same
+degree; they may metaphorically be called cousins to the same millionth
+degree; yet they differ widely and in different degrees from each other. 
+The forms descended from A, now broken up into two or three families,
+constitute a distinct order from those descended from I, also broken up
+into two families.  Nor can the existing species, descended from A, be
+ranked in the same genus with the parent A; or those from I, with the
+parent I.  But the existing genus F14 may be supposed to have been but
+slightly modified; and it will then rank with the parent-genus F; just as
+some few still living organic beings belong to Silurian genera.  So that
+the amount or value of the differences between organic beings all related
+to each other in the same degree in blood, has come to be widely different. 
+Nevertheless their genealogical arrangement remains strictly true, not only
+at the present time, but at each successive period of descent.  All the
+modified descendants from A will have inherited something in common from
+their common parent, as will all the descendants from I; so will it be with
+each subordinate branch of descendants, at each successive period.  If,
+however, we choose to suppose that any of the descendants of A or of I have
+been so much modified as to have more or less completely lost traces of
+their parentage, in this case, their places in a natural classification
+will have been more or less completely lost,--as sometimes seems to have
+occurred with existing organisms.  All the descendants of the genus F,
+along its whole line of descent, are supposed to have been but little
+modified, and they yet form a single genus.  But this genus, though much
+isolated, will still occupy its proper intermediate position; for F
+originally was intermediate in character between A and I, and the several
+genera descended from these two genera will have inherited to a certain
+extent their characters.  This natural arrangement is shown, as far as is
+possible on paper, in the diagram, but in much too simple a manner.  If a
+branching diagram had not been used, and only the names of the groups had
+been written in a linear series, it would have been still less possible to
+have given a natural arrangement; and it is notoriously not possible to
+represent in a series, on a flat surface, the affinities which we discover
+in nature amongst the beings of the same group.  Thus, on the view which I
+hold, the natural system is genealogical in its arrangement, like a
+pedigree; but the degrees of modification which the different groups have
+undergone, have to be expressed by ranking them under different so-called
+genera, sub-families, families, sections, orders, and classes.
+</para>
+
+<para>
+It may be worth while to illustrate this view of classification, by taking
+the case of languages.  If we possessed a perfect pedigree of mankind, a
+genealogical arrangement of the races of man would afford the best
+classification of the various languages now spoken throughout the world;
+and if all extinct languages, and all intermediate and slowly changing
+dialects, had to be included, such an arrangement would, I think, be the
+only possible one.  Yet it might be that some very ancient language had
+altered little, and had given rise to few new languages, whilst others
+(owing to the spreading and subsequent isolation and states of civilisation
+of the several races, descended from a common race) had altered much, and
+had given rise to many new languages and dialects.  The various degrees of
+difference in the languages from the same stock, would have to be expressed
+by groups subordinate to groups; but the proper or even only possible
+arrangement would still be genealogical; and this would be strictly
+natural, as it would connect together all languages, extinct and modern, by
+the closest affinities, and would give the filiation and origin of each
+tongue.
+</para>
+
+<para>
+In confirmation of this view, let us glance at the classification of
+varieties, which are believed or known to have descended from one species. 
+These are grouped under species, with sub-varieties under varieties; and
+with our domestic productions, several other grades of difference are
+requisite, as we have seen with pigeons.  The origin of the existence of
+groups subordinate to groups, is the same with varieties as with species,
+namely, closeness of descent with various degrees of modification.  Nearly
+the same rules are followed in classifying varieties, as with species. 
+Authors have insisted on the necessity of classing varieties on a natural
+instead of an artificial system; we are cautioned, for instance, not to
+class two varieties of the pine-apple together, merely because their fruit,
+though the most important part, happens to be nearly identical; no one puts
+the swedish and common turnips together, though the esculent and thickened
+stems are so similar.  Whatever part is found to be most constant, is used
+in classing varieties:  thus the great agriculturist Marshall says the
+horns are very useful for this purpose with cattle, because they are less
+variable than the shape or colour of the body, &amp;c.; whereas with sheep the
+horns are much less serviceable, because less constant.  In classing
+varieties, I apprehend if we had a real pedigree, a genealogical
+classification would be universally preferred; and it has been attempted by
+some authors.  For we might feel sure, whether there had been more or less
+modification, the principle of inheritance would keep the forms together
+which were allied in the greatest number of points.  In tumbler pigeons,
+though some sub-varieties differ from the others in the important character
+of having a longer beak, yet all are kept together from having the common
+habit of tumbling; but the short-faced breed has nearly or quite lost this
+habit; nevertheless, without any reasoning or thinking on the subject,
+these tumblers are kept in the same group, because allied in blood and
+alike in some other respects.  If it could be proved that the Hottentot had
+descended from the Negro, I think he would be classed under the Negro
+group, however much he might differ in colour and other important
+characters from negroes.
+</para>
+
+<para>
+With species in a state of nature, every naturalist has in fact brought
+descent into his classification; for he includes in his lowest grade, or
+that of a species, the two sexes; and how enormously these sometimes differ
+in the most important characters, is known to every naturalist:  scarcely a
+single fact can be predicated in common of the males and hermaphrodites of
+certain cirripedes, when adult, and yet no one dreams of separating them. 
+The naturalist includes as one species the several larval stages of the
+same individual, however much they may differ from each other and from the
+adult; as he likewise includes the so-called alternate generations of
+Steenstrup, which can only in a technical sense be considered as the same
+individual.  He includes monsters; he includes varieties, not solely
+because they closely resemble the parent-form, but because they are
+descended from it.  He who believes that the cowslip is descended from the
+primrose, or conversely, ranks them together as a single species, and gives
+a single definition.  As soon as three Orchidean forms (Monochanthus,
+Myanthus, and Catasetum), which had previously been ranked as three
+distinct genera, were known to be sometimes produced on the same spike,
+they were immediately included as a single species.  But it may be asked,
+what ought we to do, if it could be proved that one species of kangaroo had
+been produced, by a long course of modification, from a bear?  Ought we to
+rank this one species with bears, and what should we do with the other
+species?  The supposition is of course preposterous; and I might answer by
+the argumentum ad hominem, and ask what should be done if a perfect
+kangaroo were seen to come out of the womb of a bear?  According to all
+analogy, it would be ranked with bears; but then assuredly all the other
+species of the kangaroo family would have to be classed under the bear
+genus.  The whole case is preposterous; for where there has been close
+descent in common, there will certainly be close resemblance or affinity.
+</para>
+
+<para>
+As descent has universally been used in classing together the individuals
+of the same species, though the males and females and larvae are sometimes
+extremely different; and as it has been used in classing varieties which
+have undergone a certain, and sometimes a considerable amount of
+modification, may not this same element of descent have been unconsciously
+used in grouping species under genera, and genera under higher groups,
+though in these cases the modification has been greater in degree, and has
+taken a longer time to complete?  I believe it has thus been unconsciously
+used; and only thus can I understand the several rules and guides which
+have been followed by our best systematists.  We have no written pedigrees;
+we have to make out community of descent by resemblances of any kind. 
+Therefore we choose those characters which, as far as we can judge, are the
+least likely to have been modified in relation to the conditions of life to
+which each species has been recently exposed.  Rudimentary structures on
+this view are as good as, or even sometimes better than, other parts of the
+organisation.  We care not how trifling a character may be--let it be the
+mere inflection of the angle of the jaw, the manner in which an insect's
+wing is folded, whether the skin be covered by hair or feathers--if it
+prevail throughout many and different species, especially those having very
+different habits of life, it assumes high value; for we can account for its
+presence in so many forms with such different habits, only by its
+inheritance from a common parent.  We may err in this respect in regard to
+single points of structure, but when several characters, let them be ever
+so trifling, occur together throughout a large group of beings having
+different habits, we may feel almost sure, on the theory of descent, that
+these characters have been inherited from a common ancestor.  And we know
+that such correlated or aggregated characters have especial value in
+classification.
+</para>
+
+<para>
+We can understand why a species or a group of species may depart, in
+several of its most important characteristics, from its allies, and yet be
+safely classed with them.  This may be safely done, and is often done, as
+long as a sufficient number of characters, let them be ever so unimportant,
+betrays the hidden bond of community of descent.  Let two forms have not a
+single character in common, yet if these extreme forms are connected
+together by a chain of intermediate groups, we may at once infer their
+community of descent, and we put them all into the same class.  As we find
+organs of high physiological importance--those which serve to preserve life
+under the most diverse conditions of existence--are generally the most
+constant, we attach especial value to them; but if these same organs, in
+another group or section of a group, are found to differ much, we at once
+value them less in our classification.  We shall hereafter, I think,
+clearly see why embryological characters are of such high classificatory
+importance.  Geographical distribution may sometimes be brought usefully
+into play in classing large and widely-distributed genera, because all the
+species of the same genus, inhabiting any distinct and isolated region,
+have in all probability descended from the same parents.
+</para>
+
+<para>
+We can understand, on these views, the very important distinction between
+real affinities and analogical or adaptive resemblances.  Lamarck first
+called attention to this distinction, and he has been ably followed by
+Macleay and others.  The resemblance, in the shape of the body and in the
+fin-like anterior limbs, between the dugong, which is a pachydermatous
+animal, and the whale, and between both these mammals and fishes, is
+analogical.  Amongst insects there are innumerable instances:  thus
+Linnaeus, misled by external appearances, actually classed an homopterous
+insect as a moth.  We see something of the same kind even in our domestic
+varieties, as in the thickened stems of the common and swedish turnip.  The
+resemblance of the greyhound and racehorse is hardly more fanciful than the
+analogies which have been drawn by some authors between very distinct
+animals.  On my view of characters being of real importance for
+classification, only in so far as they reveal descent, we can clearly
+understand why analogical or adaptive character, although of the utmost
+importance to the welfare of the being, are almost valueless to the
+systematist.  For animals, belonging to two most distinct lines of descent,
+may readily become adapted to similar conditions, and thus assume a close
+external resemblance; but such resemblances will not reveal--will rather
+tend to conceal their blood-relationship to their proper lines of descent. 
+We can also understand the apparent paradox, that the very same characters
+are analogical when one class or order is compared with another, but give
+true affinities when the members of the same class or order are compared
+one with another:  thus the shape of the body and fin-like limbs are only
+analogical when whales are compared with fishes, being adaptations in both
+classes for swimming through the water; but the shape of the body and
+fin-like limbs serve as characters exhibiting true affinity between the
+several members of the whale family; for these cetaceans agree in so many
+characters, great and small, that we cannot doubt that they have inherited
+their general shape of body and structure of limbs from a common ancestor. 
+So it is with fishes.
+</para>
+
+<para>
+As members of distinct classes have often been adapted by successive slight
+modifications to live under nearly similar circumstances,--to inhabit for
+instance the three elements of land, air, and water,--we can perhaps
+understand how it is that a numerical parallelism has sometimes been
+observed between the sub-groups in distinct classes.  A naturalist, struck
+by a parallelism of this nature in any one class, by arbitrarily raising or
+sinking the value of the groups in other classes (and all our experience
+shows that this valuation has hitherto been arbitrary), could easily extend
+the parallelism over a wide range; and thus the septenary, quinary,
+quaternary, and ternary classifications have probably arisen.
+</para>
+
+<para>
+As the modified descendants of dominant species, belonging to the larger
+genera, tend to inherit the advantages, which made the groups to which they
+belong large and their parents dominant, they are almost sure to spread
+widely, and to seize on more and more places in the economy of nature.  The
+larger and more dominant groups thus tend to go on increasing in size; and
+they consequently supplant many smaller and feebler groups.  Thus we can
+account for the fact that all organisms, recent and extinct, are included
+under a few great orders, under still fewer classes, and all in one great
+natural system.  As showing how few the higher groups are in number, and
+how widely spread they are throughout the world, the fact is striking, that
+the discovery of Australia has not added a single insect belonging to a new
+order; and that in the vegetable kingdom, as I learn from Dr. Hooker, it
+has added only two or three orders of small size.
+</para>
+
+<para>
+In the chapter on geological succession I attempted to show, on the
+principle of each group having generally diverged much in character during
+the long-continued process of modification, how it is that the more ancient
+forms of life often present characters in some slight degree intermediate
+between existing groups.  A few old and intermediate parent-forms having
+occasionally transmitted to the present day descendants but little
+modified, will give to us our so-called osculant or aberrant groups.  The
+more aberrant any form is, the greater must be the number of connecting
+forms which on my theory have been exterminated and utterly lost.  And we
+have some evidence of aberrant forms having suffered severely from
+extinction, for they are generally represented by extremely few species;
+and such species as do occur are generally very distinct from each other,
+which again implies extinction.  The genera Ornithorhynchus and
+Lepidosiren, for example, would not have been less aberrant had each been
+represented by a dozen species instead of by a single one; but such
+richness in species, as I find after some investigation, does not commonly
+fall to the lot of aberrant genera.  We can, I think, account for this fact
+only by looking at aberrant forms as failing groups conquered by more
+successful competitors, with a few members preserved by some unusual
+coincidence of favourable circumstances.
+</para>
+
+<para>
+Mr. Waterhouse has remarked that, when a member belonging to one group of
+animals exhibits an affinity to a quite distinct group, this affinity in
+most cases is general and not special:  thus, according to Mr. Waterhouse,
+of all Rodents, the bizcacha is most nearly related to Marsupials; but in
+the points in which it approaches this order, its relations are general,
+and not to any one marsupial species more than to another.  As the points
+of affinity of the bizcacha to Marsupials are believed to be real and not
+merely adaptive, they are due on my theory to inheritance in common. 
+Therefore we must suppose either that all Rodents, including the bizcacha,
+branched off from some very ancient Marsupial, which will have had a
+character in some degree intermediate with respect to all existing
+Marsupials; or that both Rodents and Marsupials branched off from a common
+progenitor, and that both groups have since undergone much modification in
+divergent directions.  On either view we may suppose that the bizcacha has
+retained, by inheritance, more of the character of its ancient progenitor
+than have other Rodents; and therefore it will not be specially related to
+any one existing Marsupial, but indirectly to all or nearly all Marsupials,
+from having partially retained the character of their common progenitor, or
+of an early member of the group.  On the other hand, of all Marsupials, as
+Mr. Waterhouse has remarked, the phascolomys resembles most nearly, not any
+one species, but the general order of Rodents.  In this case, however, it
+may be strongly suspected that the resemblance is only analogical, owing to
+the phascolomys having become adapted to habits like those of a Rodent. 
+The elder De Candolle has made nearly similar observations on the general
+nature of the affinities of distinct orders of plants.
+</para>
+
+<para>
+On the principle of the multiplication and gradual divergence in character
+of the species descended from a common parent, together with their
+retention by inheritance of some characters in common, we can understand
+the excessively complex and radiating affinities by which all the members
+of the same family or higher group are connected together.  For the common
+parent of a whole family of species, now broken up by extinction into
+distinct groups and sub-groups, will have transmitted some of its
+characters, modified in various ways and degrees, to all; and the several
+species will consequently be related to each other by circuitous lines of
+affinity of various lengths (as may be seen in the diagram so often
+referred to), mounting up through many predecessors.  As it is difficult to
+show the blood-relationship between the numerous kindred of any ancient and
+noble family, even by the aid of a genealogical tree, and almost impossible
+to do this without this aid, we can understand the extraordinary difficulty
+which naturalists have experienced in describing, without the aid of a
+diagram, the various affinities which they perceive between the many living
+and extinct members of the same great natural class.
+</para>
+
+<para>
+Extinction, as we have seen in the fourth chapter, has played an important
+part in defining and widening the intervals between the several groups in
+each class.  We may thus account even for the distinctness of whole classes
+from each other--for instance, of birds from all other vertebrate
+animals--by the belief that many ancient forms of life have been utterly
+lost, through which the early progenitors of birds were formerly connected
+with the early progenitors of the other vertebrate classes.  There has been
+less entire extinction of the forms of life which once connected fishes
+with batrachians.  There has been still less in some other classes, as in
+that of the Crustacea, for here the most wonderfully diverse forms are
+still tied together by a long, but broken, chain of affinities.  Extinction
+has only separated groups:  it has by no means made them; for if every form
+which has ever lived on this earth were suddenly to reappear, though it
+would be quite impossible to give definitions by which each group could be
+distinguished from other groups, as all would blend together by steps as
+fine as those between the finest existing varieties, nevertheless a natural
+classification, or at least a natural arrangement, would be possible.  We
+shall see this by turning to the diagram:  the letters, A to L, may
+represent eleven Silurian genera, some of which have produced large groups
+of modified descendants.  Every intermediate link between these eleven
+genera and their primordial parent, and every intermediate link in each
+branch and sub-branch of their descendants, may be supposed to be still
+alive; and the links to be as fine as those between the finest varieties. 
+In this case it would be quite impossible to give any definition by which
+the several members of the several groups could be distinguished from their
+more immediate parents; or these parents from their ancient and unknown
+progenitor.  Yet the natural arrangement in the diagram would still hold
+good; and, on the principle of inheritance, all the forms descended from A,
+or from I, would have something in common.  In a tree we can specify this
+or that branch, though at the actual fork the two unite and blend together. 
+We could not, as I have said, define the several groups; but we could pick
+out types, or forms, representing most of the characters of each group,
+whether large or small, and thus give a general idea of the value of the
+differences between them.  This is what we should be driven to, if we were
+ever to succeed in collecting all the forms in any class which have lived
+throughout all time and space.  We shall certainly never succeed in making
+so perfect a collection:  nevertheless, in certain classes, we are tending
+in this direction; and Milne Edwards has lately insisted, in an able paper,
+on the high importance of looking to types, whether or not we can separate
+and define the groups to which such types belong.
+</para>
+
+<para>
+Finally, we have seen that natural selection, which results from the
+struggle for existence, and which almost inevitably induces extinction and
+divergence of character in the many descendants from one dominant
+parent-species, explains that great and universal feature in the affinities
+of all organic beings, namely, their subordination in group under group. 
+We use the element of descent in classing the individuals of both sexes and
+of all ages, although having few characters in common, under one species;
+we use descent in classing acknowledged varieties, however different they
+may be from their parent; and I believe this element of descent is the
+hidden bond of connexion which naturalists have sought under the term of
+the Natural System.  On this idea of the natural system being, in so far as
+it has been perfected, genealogical in its arrangement, with the grades of
+difference between the descendants from a common parent, expressed by the
+terms genera, families, orders, &amp;c., we can understand the rules which we
+are compelled to follow in our classification.  We can understand why we
+value certain resemblances far more than others; why we are permitted to
+use rudimentary and useless organs, or others of trifling physiological
+importance; why, in comparing one group with a distinct group, we summarily
+reject analogical or adaptive characters, and yet use these same characters
+within the limits of the same group.  We can clearly see how it is that all
+living and extinct forms can be grouped together in one great system; and
+how the several members of each class are connected together by the most
+complex and radiating lines of affinities.  We shall never, probably,
+disentangle the inextricable web of affinities between the members of any
+one class; but when we have a distinct object in view, and do not look to
+some unknown plan of creation, we may hope to make sure but slow progress.
+</para>
+
+<para>
+Morphology. -- We have seen that the members of the same class,
+independently of their habits of life, resemble each other in the general
+plan of their organisation.  This resemblance is often expressed by the
+term 'unity of type;' or by saying that the several parts and organs in the
+different species of the class are homologous.  The whole subject is
+included under the general name of Morphology.  This is the most
+interesting department of natural history, and may be said to be its very
+soul.  What can be more curious than that the hand of a man, formed for
+grasping, that of a mole for digging, the leg of the horse, the paddle of
+the porpoise, and the wing of the bat, should all be constructed on the
+same pattern, and should include the same bones, in the same relative
+positions?  Geoffroy St. Hilaire has insisted strongly on the high
+importance of relative connexion in homologous organs:  the parts may
+change to almost any extent in form and size, and yet they always remain
+connected together in the same order.  We never find, for instance, the
+bones of the arm and forearm, or of the thigh and leg, transposed.  Hence
+the same names can be given to the homologous bones in widely different
+animals.  We see the same great law in the construction of the mouths of
+insects:  what can be more different than the immensely long spiral
+proboscis of a sphinx-moth, the curious folded one of a bee or bug, and the
+great jaws of a beetle?--yet all these organs, serving for such different
+purposes, are formed by infinitely numerous modifications of an upper lip,
+mandibles, and two pairs of maxillae.  Analogous laws govern the
+construction of the mouths and limbs of crustaceans.  So it is with the
+flowers of plants.
+</para>
+
+<para>
+Nothing can be more hopeless than to attempt to explain this similarity of
+pattern in members of the same class, by utility or by the doctrine of
+final causes.  The hopelessness of the attempt has been expressly admitted
+by Owen in his most interesting work on the 'Nature of Limbs.'  On the
+ordinary view of the independent creation of each being, we can only say
+that so it is;--that it has so pleased the Creator to construct each animal
+and plant.
+</para>
+
+<para>
+The explanation is manifest on the theory of the natural selection of
+successive slight modifications,--each modification being profitable in
+some way to the modified form, but often affecting by correlation of growth
+other parts of the organisation.  In changes of this nature, there will be
+little or no tendency to modify the original pattern, or to transpose
+parts.  The bones of a limb might be shortened and widened to any extent,
+and become gradually enveloped in thick membrane, so as to serve as a fin;
+or a webbed foot might have all its bones, or certain bones, lengthened to
+any extent, and the membrane connecting them increased to any extent, so as
+to serve as a wing:  yet in all this great amount of modification there
+will be no tendency to alter the framework of bones or the relative
+connexion of the several parts.  If we suppose that the ancient progenitor,
+the archetype as it may be called, of all mammals, had its limbs
+constructed on the existing general pattern, for whatever purpose they
+served, we can at once perceive the plain signification of the homologous
+construction of the limbs throughout the whole class.  So with the mouths
+of insects, we have only to suppose that their common progenitor had an
+upper lip, mandibles, and two pair of maxillae, these parts being perhaps
+very simple in form; and then natural selection will account for the
+infinite diversity in structure and function of the mouths of insects. 
+Nevertheless, it is conceivable that the general pattern of an organ might
+become so much obscured as to be finally lost, by the atrophy and
+ultimately by the complete abortion of certain parts, by the soldering
+together of other parts, and by the doubling or multiplication of
+others,--variations which we know to be within the limits of possibility. 
+In the paddles of the extinct gigantic sea-lizards, and in the mouths of
+certain suctorial crustaceans, the general pattern seems to have been thus
+to a certain extent obscured.
+</para>
+
+<para>
+There is another and equally curious branch of the present subject; namely,
+the comparison not of the same part in different members of a class, but of
+the different parts or organs in the same individual.  Most physiologists
+believe that the bones of the skull are homologous with--that is correspond
+in number and in relative connexion with--the elemental parts of a certain
+number of vertebrae.  The anterior and posterior limbs in each member of
+the vertebrate and articulate classes are plainly homologous.  We see the
+same law in comparing the wonderfully complex jaws and legs in crustaceans. 
+It is familiar to almost every one, that in a flower the relative position
+of the sepals, petals, stamens, and pistils, as well as their intimate
+structure, are intelligible on the view that they consist of metamorphosed
+leaves, arranged in a spire.  In monstrous plants, we often get direct
+evidence of the possibility of one organ being transformed into another;
+and we can actually see in embryonic crustaceans and in many other animals,
+and in flowers, that organs, which when mature become extremely different,
+are at an early stage of growth exactly alike.
+</para>
+
+<para>
+How inexplicable are these facts on the ordinary view of creation!  Why
+should the brain be enclosed in a box composed of such numerous and such
+extraordinarily shaped pieces of bone?  As Owen has remarked, the benefit
+derived from the yielding of the separate pieces in the act of parturition
+of mammals, will by no means explain the same construction in the skulls of
+birds.  Why should similar bones have been created in the formation of the
+wing and leg of a bat, used as they are for such totally different
+purposes?  Why should one crustacean, which has an extremely complex mouth
+formed of many parts, consequently always have fewer legs; or conversely,
+those with many legs have simpler mouths?  Why should the sepals, petals,
+stamens, and pistils in any individual flower, though fitted for such
+widely different purposes, be all constructed on the same pattern?
+</para>
+
+<para>
+On the theory of natural selection, we can satisfactorily answer these
+questions.  In the vertebrata, we see a series of internal vertebrae
+bearing certain processes and appendages; in the articulata, we see the
+body divided into a series of segments, bearing external appendages; and in
+flowering plants, we see a series of successive spiral whorls of leaves. 
+An indefinite repetition of the same part or organ is the common
+characteristic (as Owen has observed) of all low or little-modified forms;
+therefore we may readily believe that the unknown progenitor of the
+vertebrata possessed many vertebrae; the unknown progenitor of the
+articulata, many segments; and the unknown progenitor of flowering plants,
+many spiral whorls of leaves.  We have formerly seen that parts many times
+repeated are eminently liable to vary in number and structure; consequently
+it is quite probable that natural selection, during a long-continued course
+of modification, should have seized on a certain number of the primordially
+similar elements, many times repeated, and have adapted them to the most
+diverse purposes.  And as the whole amount of modification will have been
+effected by slight successive steps, we need not wonder at discovering in
+such parts or organs, a certain degree of fundamental resemblance, retained
+by the strong principle of inheritance.
+</para>
+
+<para>
+In the great class of molluscs, though we can homologise the parts of one
+species with those of another and distinct species, we can indicate but few
+serial homologies; that is, we are seldom enabled to say that one part or
+organ is homologous with another in the same individual.  And we can
+understand this fact; for in molluscs, even in the lowest members of the
+class, we do not find nearly so much indefinite repetition of any one part,
+as we find in the other great classes of the animal and vegetable kingdoms.
+</para>
+
+<para>
+Naturalists frequently speak of the skull as formed of metamorphosed
+vertebrae:  the jaws of crabs as metamorphosed legs; the stamens and
+pistils of flowers as metamorphosed leaves; but it would in these cases
+probably be more correct, as Professor Huxley has remarked, to speak of
+both skull and vertebrae, both jaws and legs, &amp;c.,--as having been
+metamorphosed, not one from the other, but from some common element. 
+Naturalists, however, use such language only in a metaphorical sense:  they
+are far from meaning that during a long course of descent, primordial
+organs of any kind--vertebrae in the one case and legs in the other--have
+actually been modified into skulls or jaws.  Yet so strong is the
+appearance of a modification of this nature having occurred, that
+naturalists can hardly avoid employing language having this plain
+signification.  On my view these terms may be used literally; and the
+wonderful fact of the jaws, for instance, of a crab retaining numerous
+characters, which they would probably have retained through inheritance, if
+they had really been metamorphosed during a long course of descent from
+true legs, or from some simple appendage, is explained.
+</para>
+
+<para>
+Embryology. -- It has already been casually remarked that certain organs in
+the individual, which when mature become widely different and serve for
+different purposes, are in the embryo exactly alike.  The embryos, also, of
+distinct animals within the same class are often strikingly similar:  a
+better proof of this cannot be given, than a circumstance mentioned by
+Agassiz, namely, that having forgotten to ticket the embryo of some
+vertebrate animal, he cannot now tell whether it be that of a mammal, bird,
+or reptile.  The vermiform larvae of moths, flies, beetles, &amp;c., resemble
+each other much more closely than do the mature insects; but in the case of
+larvae, the embryos are active, and have been adapted for special lines of
+life.  A trace of the law of embryonic resemblance, sometimes lasts till a
+rather late age:  thus birds of the same genus, and of closely allied
+genera, often resemble each other in their first and second plumage; as we
+see in the spotted feathers in the thrush group.  In the cat tribe, most of
+the species are striped or spotted in lines; and stripes can be plainly
+distinguished in the whelp of the lion.  We occasionally though rarely see
+something of this kind in plants:  thus the embryonic leaves of the ulex or
+furze, and the first leaves of the phyllodineous acaceas, are pinnate or
+divided like the ordinary leaves of the leguminosae.
+</para>
+
+<para>
+The points of structure, in which the embryos of widely different animals
+of the same class resemble each other, often have no direct relation to
+their conditions of existence.  We cannot, for instance, suppose that in
+the embryos of the vertebrata the peculiar loop-like course of the arteries
+near the branchial slits are related to similar conditions,--in the young
+mammal which is nourished in the womb of its mother, in the egg of the bird
+which is hatched in a nest, and in the spawn of a frog under water.  We
+have no more reason to believe in such a relation, than we have to believe
+that the same bones in the hand of a man, wing of a bat, and fin of a
+porpoise, are related to similar conditions of life.  No one will suppose
+that the stripes on the whelp of a lion, or the spots on the young
+blackbird, are of any use to these animals, or are related to the
+conditions to which they are exposed.
+</para>
+
+<para>
+The case, however, is different when an animal during any part of its
+embryonic career is active, and has to provide for itself.  The period of
+activity may come on earlier or later in life; but whenever it comes on,
+the adaptation of the larva to its conditions of life is just as perfect
+and as beautiful as in the adult animal.  From such special adaptations,
+the similarity of the larvae or active embryos of allied animals is
+sometimes much obscured; and cases could be given of the larvae of two
+species, or of two groups of species, differing quite as much, or even
+more, from each other than do their adult parents.  In most cases, however,
+the larvae, though active, still obey more or less closely the law of
+common embryonic resemblance.  Cirripedes afford a good instance of this: 
+even the illustrious Cuvier did not perceive that a barnacle was, as it
+certainly is, a crustacean; but a glance at the larva shows this to be the
+case in an unmistakeable manner.  So again the two main divisions of
+cirripedes, the pedunculated and sessile, which differ widely in external
+appearance, have larvae in all their several stages barely distinguishable.
+</para>
+
+<para>
+The embryo in the course of development generally rises in organisation:  I
+use this expression, though I am aware that it is hardly possible to define
+clearly what is meant by the organisation being higher or lower.  But no
+one probably will dispute that the butterfly is higher than the
+caterpillar.  In some cases, however, the mature animal is generally
+considered as lower in the scale than the larva, as with certain parasitic
+crustaceans.  To refer once again to cirripedes:  the larvae in the first
+stage have three pairs of legs, a very simple single eye, and a
+probosciformed mouth, with which they feed largely, for they increase much
+in size.  In the second stage, answering to the chrysalis stage of
+butterflies, they have six pairs of beautifully constructed natatory legs,
+a pair of magnificent compound eyes, and extremely complex antennae; but
+they have a closed and imperfect mouth, and cannot feed:  their function at
+this stage is, to search by their well-developed organs of sense, and to
+reach by their active powers of swimming, a proper place on which to become
+attached and to undergo their final metamorphosis.  When this is completed
+they are fixed for life:  their legs are now converted into prehensile
+organs; they again obtain a well-constructed mouth; but they have no
+antennae, and their two eyes are now reconverted into a minute, single, and
+very simple eye-spot.  In this last and complete state, cirripedes may be
+considered as either more highly or more lowly organised than they were in
+the larval condition.  But in some genera the larvae become developed
+either into hermaphrodites having the ordinary structure, or into what I
+have called complemental males:  and in the latter, the development has
+assuredly been retrograde; for the male is a mere sack, which lives for a
+short time, and is destitute of mouth, stomach, or other organ of
+importance, excepting for reproduction.
+</para>
+
+<para>
+We are so much accustomed to see differences in structure between the
+embryo and the adult, and likewise a close similarity in the embryos of
+widely different animals within the same class, that we might be led to
+look at these facts as necessarily contingent in some manner on growth. 
+But there is no obvious reason why, for instance, the wing of a bat, or the
+fin of a porpoise, should not have been sketched out with all the parts in
+proper proportion, as soon as any structure became visible in the embryo. 
+And in some whole groups of animals and in certain members of other groups,
+the embryo does not at any period differ widely from the adult:  thus Owen
+has remarked in regard to cuttle-fish, 'there is no metamorphosis; the
+cephalopodic character is manifested long before the parts of the embryo
+are completed;' and again in spiders, 'there is nothing worthy to be called
+a metamorphosis.'  The larvae of insects, whether adapted to the most
+diverse and active habits, or quite inactive, being fed by their parents or
+placed in the midst of proper nutriment, yet nearly all pass through a
+similar worm-like stage of development; but in some few cases, as in that
+of Aphis, if we look to the admirable drawings by Professor Huxley of the
+development of this insect, we see no trace of the vermiform stage.
+</para>
+
+<para>
+How, then, can we explain these several facts in embryology,--namely the
+very general, but not universal difference in structure between the embryo
+and the adult;--of parts in the same individual embryo, which ultimately
+become very unlike and serve for diverse purposes, being at this early
+period of growth alike;--of embryos of different species within the same
+class, generally, but not universally, resembling each other;--of the
+structure of the embryo not being closely related to its conditions of
+existence, except when the embryo becomes at any period of life active and
+has to provide for itself;--of the embryo apparently having sometimes a
+higher organisation than the mature animal, into which it is developed.  I
+believe that all these facts can be explained, as follows, on the view of
+descent with modification.
+</para>
+
+<para>
+It is commonly assumed, perhaps from monstrosities often affecting the
+embryo at a very early period, that slight variations necessarily appear at
+an equally early period.  But we have little evidence on this head--indeed
+the evidence rather points the other way; for it is notorious that breeders
+of cattle, horses, and various fancy animals, cannot positively tell, until
+some time after the animal has been born, what its merits or form will
+ultimately turn out.  We see this plainly in our own children; we cannot
+always tell whether the child will be tall or short, or what its precise
+features will be.  The question is not, at what period of life any
+variation has been caused, but at what period it is fully displayed.  The
+cause may have acted, and I believe generally has acted, even before the
+embryo is formed; and the variation may be due to the male and female
+sexual elements having been affected by the conditions to which either
+parent, or their ancestors, have been exposed.  Nevertheless an effect thus
+caused at a very early period, even before the formation of the embryo, may
+appear late in life; as when an hereditary disease, which appears in old
+age alone, has been communicated to the offspring from the reproductive
+element of one parent.  Or again, as when the horns of cross-bred cattle
+have been affected by the shape of the horns of either parent.  For the
+welfare of a very young animal, as long as it remains in its mother's womb,
+or in the egg, or as long as it is nourished and protected by its parent,
+it must be quite unimportant whether most of its characters are fully
+acquired a little earlier or later in life.  It would not signify, for
+instance, to a bird which obtained its food best by having a long beak,
+whether or not it assumed a beak of this particular length, as long as it
+was fed by its parents.  Hence, I conclude, that it is quite possible, that
+each of the many successive modifications, by which each species has
+acquired its present structure, may have supervened at a not very early
+period of life; and some direct evidence from our domestic animals supports
+this view.  But in other cases it is quite possible that each successive
+modification, or most of them, may have appeared at an extremely early
+period.
+</para>
+
+<para>
+I have stated in the first chapter, that there is some evidence to render
+it probable, that at whatever age any variation first appears in the
+parent, it tends to reappear at a corresponding age in the offspring. 
+Certain variations can only appear at corresponding ages, for instance,
+peculiarities in the caterpillar, cocoon, or imago states of the silk-moth;
+or, again, in the horns of almost full-grown cattle.  But further than
+this, variations which, for all that we can see, might have appeared
+earlier or later in life, tend to appear at a corresponding age in the
+offspring and parent.  I am far from meaning that this is invariably the
+case; and I could give a good many cases of variations (taking the word in
+the largest sense) which have supervened at an earlier age in the child
+than in the parent.
+</para>
+
+<para>
+These two principles, if their truth be admitted, will, I believe, explain
+all the above specified leading facts in embryology.  But first let us look
+at a few analogous cases in domestic varieties.  Some authors who have
+written on Dogs, maintain that the greyhound and bulldog, though appearing
+so different, are really varieties most closely allied, and have probably
+descended from the same wild stock; hence I was curious to see how far
+their puppies differed from each other:  I was told by breeders that they
+differed just as much as their parents, and this, judging by the eye,
+seemed almost to be the case; but on actually measuring the old dogs and
+their six-days old puppies, I found that the puppies had not nearly
+acquired their full amount of proportional difference.  So, again, I was
+told that the foals of cart and race-horses differed as much as the
+full-grown animals; and this surprised me greatly, as I think it probable
+that the difference between these two breeds has been wholly caused by
+selection under domestication; but having had careful measurements made of
+the dam and of a three-days old colt of a race and heavy cart-horse, I find
+that the colts have by no means acquired their full amount of proportional
+difference.
+</para>
+
+<para>
+As the evidence appears to me conclusive, that the several domestic breeds
+of Pigeon have descended from one wild species, I compared young pigeons of
+various breeds, within twelve hours after being hatched; I carefully
+measured the proportions (but will not here give details) of the beak,
+width of mouth, length of nostril and of eyelid, size of feet and length of
+leg, in the wild stock, in pouters, fantails, runts, barbs, dragons,
+carriers, and tumblers.  Now some of these birds, when mature, differ so
+extraordinarily in length and form of beak, that they would, I cannot
+doubt, be ranked in distinct genera, had they been natural productions. 
+But when the nestling birds of these several breeds were placed in a row,
+though most of them could be distinguished from each other, yet their
+proportional differences in the above specified several points were
+incomparably less than in the full-grown birds.  Some characteristic points
+of difference--for instance, that of the width of mouth--could hardly be
+detected in the young.  But there was one remarkable exception to this
+rule, for the young of the short-faced tumbler differed from the young of
+the wild rock-pigeon and of the other breeds, in all its proportions,
+almost exactly as much as in the adult state.
+</para>
+
+<para>
+The two principles above given seem to me to explain these facts in regard
+to the later embryonic stages of our domestic varieties.  Fanciers select
+their horses, dogs, and pigeons, for breeding, when they are nearly grown
+up:  they are indifferent whether the desired qualities and structures have
+been acquired earlier or later in life, if the full-grown animal possesses
+them.  And the cases just given, more especially that of pigeons, seem to
+show that the characteristic differences which give value to each breed,
+and which have been accumulated by man's selection, have not generally
+first appeared at an early period of life, and have been inherited by the
+offspring at a corresponding not early period.  But the case of the
+short-faced tumbler, which when twelve hours old had acquired its proper
+proportions, proves that this is not the universal rule; for here the
+characteristic differences must either have appeared at an earlier period
+than usual, or, if not so, the differences must have been inherited, not at
+the corresponding, but at an earlier age.
+</para>
+
+<para>
+Now let us apply these facts and the above two principles--which latter,
+though not proved true, can be shown to be in some degree probable--to
+species in a state of nature.  Let us take a genus of birds, descended on
+my theory from some one parent-species, and of which the several new
+species have become modified through natural selection in accordance with
+their diverse habits.  Then, from the many slight successive steps of
+variation having supervened at a rather late age, and having been inherited
+at a corresponding age, the young of the new species of our supposed genus
+will manifestly tend to resemble each other much more closely than do the
+adults, just as we have seen in the case of pigeons.  We may extend this
+view to whole families or even classes.  The fore-limbs, for instance,
+which served as legs in the parent-species, may become, by a long course of
+modification, adapted in one descendant to act as hands, in another as
+paddles, in another as wings; and on the above two principles--namely of
+each successive modification supervening at a rather late age, and being
+inherited at a corresponding late age--the fore-limbs in the embryos of the
+several descendants of the parent-species will still resemble each other
+closely, for they will not have been modified.  But in each individual new
+species, the embryonic fore-limbs will differ greatly from the fore-limbs
+in the mature animal; the limbs in the latter having undergone much
+modification at a rather late period of life, and having thus been
+converted into hands, or paddles, or wings.  Whatever influence
+long-continued exercise or use on the one hand, and disuse on the other,
+may have in modifying an organ, such influence will mainly affect the
+mature animal, which has come to its full powers of activity and has to
+gain its own living; and the effects thus produced will be inherited at a
+corresponding mature age.  Whereas the young will remain unmodified, or be
+modified in a lesser degree, by the effects of use and disuse.
+</para>
+
+<para>
+In certain cases the successive steps of variation might supervene, from
+causes of which we are wholly ignorant, at a very early period of life, or
+each step might be inherited at an earlier period than that at which it
+first appeared.  In either case (as with the short-faced tumbler) the young
+or embryo would closely resemble the mature parent-form.  We have seen that
+this is the rule of development in certain whole groups of animals, as with
+cuttle-fish and spiders, and with a few members of the great class of
+insects, as with Aphis.  With respect to the final cause of the young in
+these cases not undergoing any metamorphosis, or closely resembling their
+parents from their earliest age, we can see that this would result from the
+two following contingencies; firstly, from the young, during a course of
+modification carried on for many generations, having to provide for their
+own wants at a very early stage of development, and secondly, from their
+following exactly the same habits of life with their parents; for in this
+case, it would be indispensable for the existence of the species, that the
+child should be modified at a very early age in the same manner with its
+parents, in accordance with their similar habits.  Some further
+explanation, however, of the embryo not undergoing any metamorphosis is
+perhaps requisite.  If, on the other hand, it profited the young to follow
+habits of life in any degree different from those of their parent, and
+consequently to be constructed in a slightly different manner, then, on the
+principle of inheritance at corresponding ages, the active young or larvae
+might easily be rendered by natural selection different to any conceivable
+extent from their parents.  Such differences might, also, become correlated
+with successive stages of development; so that the larvae, in the first
+stage, might differ greatly from the larvae in the second stage, as we have
+seen to be the case with cirripedes.  The adult might become fitted for
+sites or habits, in which organs of locomotion or of the senses, &amp;c., would
+be useless; and in this case the final metamorphosis would be said to be
+retrograde.
+</para>
+
+<para>
+As all the organic beings, extinct and recent, which have ever lived on
+this earth have to be classed together, and as all have been connected by
+the finest gradations, the best, or indeed, if our collections were nearly
+perfect, the only possible arrangement, would be genealogical.  Descent
+being on my view the hidden bond of connexion which naturalists have been
+seeking under the term of the natural system.  On this view we can
+understand how it is that, in the eyes of most naturalists, the structure
+of the embryo is even more important for classification than that of the
+adult.  For the embryo is the animal in its less modified state; and in so
+far it reveals the structure of its progenitor.  In two groups of animal,
+however much they may at present differ from each other in structure and
+habits, if they pass through the same or similar embryonic stages, we may
+feel assured that they have both descended from the same or nearly similar
+parents, and are therefore in that degree closely related.  Thus, community
+in embryonic structure reveals community of descent.  It will reveal this
+community of descent, however much the structure of the adult may have been
+modified and obscured; we have seen, for instance, that cirripedes can at
+once be recognised by their larvae as belonging to the great class of
+crustaceans.  As the embryonic state of each species and group of species
+partially shows us the structure of their less modified ancient
+progenitors, we can clearly see why ancient and extinct forms of life
+should resemble the embryos of their descendants,--our existing species. 
+Agassiz believes this to be a law of nature; but I am bound to confess that
+I only hope to see the law hereafter proved true.  It can be proved true in
+those cases alone in which the ancient state, now supposed to be
+represented in many embryos, has not been obliterated, either by the
+successive variations in a long course of modification having supervened at
+a very early age, or by the variations having been inherited at an earlier
+period than that at which they first appeared.  It should also be borne in
+mind, that the supposed law of resemblance of ancient forms of life to the
+embryonic stages of recent forms, may be true, but yet, owing to the
+geological record not extending far enough back in time, may remain for a
+long period, or for ever, incapable of demonstration.
+</para>
+
+<para>
+Thus, as it seems to me, the leading facts in embryology, which are second
+in importance to none in natural history, are explained on the principle of
+slight modifications not appearing, in the many descendants from some one
+ancient progenitor, at a very early period in the life of each, though
+perhaps caused at the earliest, and being inherited at a corresponding not
+early period.  Embryology rises greatly in interest, when we thus look at
+the embryo as a picture, more or less obscured, of the common parent-form
+of each great class of animals.
+</para>
+
+<para>
+Rudimentary, atrophied, or aborted organs. -- Organs or parts in this
+strange condition, bearing the stamp of inutility, are extremely common
+throughout nature.  For instance, rudimentary mammae are very general in
+the males of mammals:  I presume that the 'bastard-wing' in birds may be
+safely considered as a digit in a rudimentary state:  in very many snakes
+one lobe of the lungs is rudimentary; in other snakes there are rudiments
+of the pelvis and hind limbs.  Some of the cases of rudimentary organs are
+extremely curious; for instance, the presence of teeth in foetal whales,
+which when grown up have not a tooth in their heads; and the presence of
+teeth, which never cut through the gums, in the upper jaws of our unborn
+calves.  It has even been stated on good authority that rudiments of teeth
+can be detected in the beaks of certain embryonic birds.  Nothing can be
+plainer than that wings are formed for flight, yet in how many insects do
+we see wings so reduced in size as to be utterly incapable of flight, and
+not rarely lying under wing-cases, firmly soldered together!
+</para>
+
+<para>
+The meaning of rudimentary organs is often quite unmistakeable:  for
+instance there are beetles of the same genus (and even of the same species)
+resembling each other most closely in all respects, one of which will have
+full-sized wings, and another mere rudiments of membrane; and here it is
+impossible to doubt, that the rudiments represent wings.  Rudimentary
+organs sometimes retain their potentiality, and are merely not developed: 
+this seems to be the case with the mammae of male mammals, for many
+instances are on record of these organs having become well developed in
+full-grown males, and having secreted milk.  So again there are normally
+four developed and two rudimentary teats in the udders of the genus Bos,
+but in our domestic cows the two sometimes become developed and give milk. 
+In individual plants of the same species the petals sometimes occur as mere
+rudiments, and sometimes in a well-developed state.  In plants with
+separated sexes, the male flowers often have a rudiment of a pistil; and
+Kolreuter found that by crossing such male plants with an hermaphrodite
+species, the rudiment of the pistil in the hybrid offspring was much
+increased in size; and this shows that the rudiment and the perfect pistil
+are essentially alike in nature.
+</para>
+
+<para>
+An organ serving for two purposes, may become rudimentary or utterly
+aborted for one, even the more important purpose; and remain perfectly
+efficient for the other.  Thus in plants, the office of the pistil is to
+allow the pollen-tubes to reach the ovules protected in the ovarium at its
+base.  The pistil consists of a stigma supported on the style; but in some
+Compositae, the male florets, which of course cannot be fecundated, have a
+pistil, which is in a rudimentary state, for it is not crowned with a
+stigma; but the style remains well developed, and is clothed with hairs as
+in other compositae, for the purpose of brushing the pollen out of the
+surrounding anthers.  Again, an organ may become rudimentary for its proper
+purpose, and be used for a distinct object:  in certain fish the
+swim-bladder seems to be rudimentary for its proper function of giving
+buoyancy, but has become converted into a nascent breathing organ or lung. 
+Other similar instances could be given.
+</para>
+
+<para>
+Rudimentary organs in the individuals of the same species are very liable
+to vary in degree of development and in other respects.  Moreover, in
+closely allied species, the degree to which the same organ has been
+rendered rudimentary occasionally differs much.  This latter fact is well
+exemplified in the state of the wings of the female moths in certain
+groups.  Rudimentary organs may be utterly aborted; and this implies, that
+we find in an animal or plant no trace of an organ, which analogy would
+lead us to expect to find, and which is occasionally found in monstrous
+individuals of the species.  Thus in the snapdragon (antirrhinum) we
+generally do not find a rudiment of a fifth stamen; but this may sometimes
+be seen.  In tracing the homologies of the same part in different members
+of a class, nothing is more common, or more necessary, than the use and
+discovery of rudiments.  This is well shown in the drawings given by Owen
+of the bones of the leg of the horse, ox, and rhinoceros.
+</para>
+
+<para>
+It is an important fact that rudimentary organs, such as teeth in the upper
+jaws of whales and ruminants, can often be detected in the embryo, but
+afterwards wholly disappear.  It is also, I believe, a universal rule, that
+a rudimentary part or organ is of greater size relatively to the adjoining
+parts in the embryo, than in the adult; so that the organ at this early age
+is less rudimentary, or even cannot be said to be in any degree
+rudimentary.  Hence, also, a rudimentary organ in the adult, is often said
+to have retained its embryonic condition.
+</para>
+
+<para>
+I have now given the leading facts with respect to rudimentary organs.  In
+reflecting on them, every one must be struck with astonishment:  for the
+same reasoning power which tells us plainly that most parts and organs are
+exquisitely adapted for certain purposes, tells us with equal plainness
+that these rudimentary or atrophied organs, are imperfect and useless.  In
+works on natural history rudimentary organs are generally said to have been
+created 'for the sake of symmetry,' or in order 'to complete the scheme of
+nature;' but this seems to me no explanation, merely a restatement of the
+fact.  Would it be thought sufficient to say that because planets revolve
+in elliptic courses round the sun, satellites follow the same course round
+the planets, for the sake of symmetry, and to complete the scheme of
+nature?  An eminent physiologist accounts for the presence of rudimentary
+organs, by supposing that they serve to excrete matter in excess, or
+injurious to the system; but can we suppose that the minute papilla, which
+often represents the pistil in male flowers, and which is formed merely of
+cellular tissue, can thus act?  Can we suppose that the formation of
+rudimentary teeth which are subsequently absorbed, can be of any service to
+the rapidly growing embryonic calf by the excretion of precious phosphate
+of lime?  When a man's fingers have been amputated, imperfect nails
+sometimes appear on the stumps:  I could as soon believe that these
+vestiges of nails have appeared, not from unknown laws of growth, but in
+order to excrete horny matter, as that the rudimentary nails on the fin of
+the manatee were formed for this purpose.
+</para>
+
+<para>
+On my view of descent with modification, the origin of rudimentary organs
+is simple.  We have plenty of cases of rudimentary organs in our domestic
+productions,--as the stump of a tail in tailless breeds,--the vestige of an
+ear in earless breeds,--the reappearance of minute dangling horns in
+hornless breeds of cattle, more especially, according to Youatt, in young
+animals,--and the state of the whole flower in the cauliflower.  We often
+see rudiments of various parts in monsters.  But I doubt whether any of
+these cases throw light on the origin of rudimentary organs in a state of
+nature, further than by showing that rudiments can be produced; for I doubt
+whether species under nature ever undergo abrupt changes.  I believe that
+disuse has been the main agency; that it has led in successive generations
+to the gradual reduction of various organs, until they have become
+rudimentary,--as in the case of the eyes of animals inhabiting dark
+caverns, and of the wings of birds inhabiting oceanic islands, which have
+seldom been forced to take flight, and have ultimately lost the power of
+flying.  Again, an organ useful under certain conditions, might become
+injurious under others, as with the wings of beetles living on small and
+exposed islands; and in this case natural selection would continue slowly
+to reduce the organ, until it was rendered harmless and rudimentary.
+</para>
+
+<para>
+Any change in function, which can be effected by insensibly small steps, is
+within the power of natural selection; so that an organ rendered, during
+changed habits of life, useless or injurious for one purpose, might easily
+be modified and used for another purpose.  Or an organ might be retained
+for one alone of its former functions.  An organ, when rendered useless,
+may well be variable, for its variations cannot be checked by natural
+selection.  At whatever period of life disuse or selection reduces an
+organ, and this will generally be when the being has come to maturity and
+to its full powers of action, the principle of inheritance at corresponding
+ages will reproduce the organ in its reduced state at the same age, and
+consequently will seldom affect or reduce it in the embryo.  Thus we can
+understand the greater relative size of rudimentary organs in the embryo,
+and their lesser relative size in the adult.  But if each step of the
+process of reduction were to be inherited, not at the corresponding age,
+but at an extremely early period of life (as we have good reason to believe
+to be possible) the rudimentary part would tend to be wholly lost, and we
+should have a case of complete abortion.  The principle, also, of economy,
+explained in a former chapter, by which the materials forming any part or
+structure, if not useful to the possessor, will be saved as far as is
+possible, will probably often come into play; and this will tend to cause
+the entire obliteration of a rudimentary organ.
+</para>
+
+<para>
+As the presence of rudimentary organs is thus due to the tendency in every
+part of the organisation, which has long existed, to be inherited--we can
+understand, on the genealogical view of classification, how it is that
+systematists have found rudimentary parts as useful as, or even sometimes
+more useful than, parts of high physiological importance.  Rudimentary
+organs may be compared with the letters in a word, still retained in the
+spelling, but become useless in the pronunciation, but which serve as a
+clue in seeking for its derivation.  On the view of descent with
+modification, we may conclude that the existence of organs in a
+rudimentary, imperfect, and useless condition, or quite aborted, far from
+presenting a strange difficulty, as they assuredly do on the ordinary
+doctrine of creation, might even have been anticipated, and can be
+accounted for by the laws of inheritance.
+</para>
+
+<para>
+Summary. -- In this chapter I have attempted to show, that the
+subordination of group to group in all organisms throughout all time; that
+the nature of the relationship, by which all living and extinct beings are
+united by complex, radiating, and circuitous lines of affinities into one
+grand system; the rules followed and the difficulties encountered by
+naturalists in their classifications; the value set upon characters, if
+constant and prevalent, whether of high vital importance, or of the most
+trifling importance, or, as in rudimentary organs, of no importance; the
+wide opposition in value between analogical or adaptive characters, and
+characters of true affinity; and other such rules;--all naturally follow on
+the view of the common parentage of those forms which are considered by
+naturalists as allied, together with their modification through natural
+selection, with its contingencies of extinction and divergence of
+character.  In considering this view of classification, it should be borne
+in mind that the element of descent has been universally used in ranking
+together the sexes, ages, and acknowledged varieties of the same species,
+however different they may be in structure.  If we extend the use of this
+element of descent,--the only certainly known cause of similarity in
+organic beings,--we shall understand what is meant by the natural system: 
+it is genealogical in its attempted arrangement, with the grades of
+acquired difference marked by the terms varieties, species, genera,
+families, orders, and classes.
+</para>
+
+<para>
+On this same view of descent with modification, all the great facts in
+Morphology become intelligible,--whether we look to the same pattern
+displayed in the homologous organs, to whatever purpose applied, of the
+different species of a class; or to the homologous parts constructed on the
+same pattern in each individual animal and plant.
+</para>
+
+<para>
+On the principle of successive slight variations, not necessarily or
+generally supervening at a very early period of life, and being inherited
+at a corresponding period, we can understand the great leading facts in
+Embryology; namely, the resemblance in an individual embryo of the
+homologous parts, which when matured will become widely different from each
+other in structure and function; and the resemblance in different species
+of a class of the homologous parts or organs, though fitted in the adult
+members for purposes as different as possible.  Larvae are active embryos,
+which have become specially modified in relation to their habits of life,
+through the principle of modifications being inherited at corresponding
+ages.  On this same principle--and bearing in mind, that when organs are
+reduced in size, either from disuse or selection, it will generally be at
+that period of life when the being has to provide for its own wants, and
+bearing in mind how strong is the principle of inheritance--the occurrence
+of rudimentary organs and their final abortion, present to us no
+inexplicable difficulties; on the contrary, their presence might have been
+even anticipated.  The importance of embryological characters and of
+rudimentary organs in classification is intelligible, on the view that an
+arrangement is only so far natural as it is genealogical.
+</para>
+
+<para>
+Finally, the several classes of facts which have been considered in this
+chapter, seem to me to proclaim so plainly, that the innumerable species,
+genera, and families of organic beings, with which this world is peopled,
+have all descended, each within its own class or group, from common
+parents, and have all been modified in the course of descent, that I should
+without hesitation adopt this view, even if it were unsupported by other
+facts or arguments.
+</para>
+
+</chapter>
+
+<chapter>
+<chapheader>
+
+<chapnum>Chapter XIV</chapnum>
+
+<title>Recapitulation and Conclusion</title>
+
+<chapsummary>
+Recapitulation of the difficulties on the theory of Natural Selection --
+Recapitulation of the general and special circumstances in its favour --
+Causes of the general belief in the immutability of species -- How far the
+theory of natural selection may be extended -- Effects of its adoption on
+the study of Natural history -- Concluding remarks.
+</chapsummary>
+</chapheader>
+
+<para>
+As this whole volume is one long argument, it may be convenient to the
+reader to have the leading facts and inferences briefly recapitulated.
+</para>
+
+<para>
+That many and grave objections may be advanced against the theory of
+descent with modification through natural selection, I do not deny.  I have
+endeavoured to give to them their full force.  Nothing at first can appear
+more difficult to believe than that the more complex organs and instincts
+should have been perfected, not by means superior to, though analogous
+with, human reason, but by the accumulation of innumerable slight
+variations, each good for the individual possessor.  Nevertheless, this
+difficulty, though appearing to our imagination insuperably great, cannot
+be considered real if we admit the following propositions, namely,--that
+gradations in the perfection of any organ or instinct, which we may
+consider, either do now exist or could have existed, each good of its
+kind,--that all organs and instincts are, in ever so slight a degree,
+variable,--and, lastly, that there is a struggle for existence leading to
+the preservation of each profitable deviation of structure or instinct. 
+The truth of these propositions cannot, I think, be disputed.
+</para>
+
+<para>
+It is, no doubt, extremely difficult even to conjecture by what gradations
+many structures have been perfected, more especially amongst broken and
+failing groups of organic beings; but we see so many strange gradations in
+nature, as is proclaimed by the canon, 'Natura non facit saltum,' that we
+ought to be extremely cautious in saying that any organ or instinct, or any
+whole being, could not have arrived at its present state by many graduated
+steps.  There are, it must be admitted, cases of special difficulty on the
+theory of natural selection; and one of the most curious of these is the
+existence of two or three defined castes of workers or sterile females in
+the same community of ants; but I have attempted to show how this
+difficulty can be mastered.
+</para>
+
+<para>
+With respect to the almost universal sterility of species when first
+crossed, which forms so remarkable a contrast with the almost universal
+fertility of varieties when crossed, I must refer the reader to the
+recapitulation of the facts given at the end of the eighth chapter, which
+seem to me conclusively to show that this sterility is no more a special
+endowment than is the incapacity of two trees to be grafted together, but
+that it is incidental on constitutional differences in the reproductive
+systems of the intercrossed species.  We see the truth of this conclusion
+in the vast difference in the result, when the same two species are crossed
+reciprocally; that is, when one species is first used as the father and
+then as the mother.
+</para>
+
+<para>
+The fertility of varieties when intercrossed and of their mongrel offspring
+cannot be considered as universal; nor is their very general fertility
+surprising when we remember that it is not likely that either their
+constitutions or their reproductive systems should have been profoundly
+modified.  Moreover, most of the varieties which have been experimentised
+on have been produced under domestication; and as domestication apparently
+tends to eliminate sterility, we ought not to expect it also to produce
+sterility.
+</para>
+
+<para>
+The sterility of hybrids is a very different case from that of first
+crosses, for their reproductive organs are more or less functionally
+impotent; whereas in first crosses the organs on both sides are in a
+perfect condition.  As we continually see that organisms of all kinds are
+rendered in some degree sterile from their constitutions having been
+disturbed by slightly different and new conditions of life, we need not
+feel surprise at hybrids being in some degree sterile, for their
+constitutions can hardly fail to have been disturbed from being compounded
+of two distinct organisations.  This parallelism is supported by another
+parallel, but directly opposite, class of facts; namely, that the vigour
+and fertility of all organic beings are increased by slight changes in
+their conditions of life, and that the offspring of slightly modified forms
+or varieties acquire from being crossed increased vigour and fertility.  So
+that, on the one hand, considerable changes in the conditions of life and
+crosses between greatly modified forms, lessen fertility; and on the other
+hand, lesser changes in the conditions of life and crosses between less
+modified forms, increase fertility.
+</para>
+
+<para>
+Turning to geographical distribution, the difficulties encountered on the
+theory of descent with modification are grave enough.  All the individuals
+of the same species, and all the species of the same genus, or even higher
+group, must have descended from common parents; and therefore, in however
+distant and isolated parts of the world they are now found, they must in
+the course of successive generations have passed from some one part to the
+others.  We are often wholly unable even to conjecture how this could have
+been effected.  Yet, as we have reason to believe that some species have
+retained the same specific form for very long periods, enormously long as
+measured by years, too much stress ought not to be laid on the occasional
+wide diffusion of the same species; for during very long periods of time
+there will always be a good chance for wide migration by many means.  A
+broken or interrupted range may often be accounted for by the extinction of
+the species in the intermediate regions.  It cannot be denied that we are
+as yet very ignorant of the full extent of the various climatal and
+geographical changes which have affected the earth during modern periods;
+and such changes will obviously have greatly facilitated migration.  As an
+example, I have attempted to show how potent has been the influence of the
+Glacial period on the distribution both of the same and of representative
+species throughout the world.  We are as yet profoundly ignorant of the
+many occasional means of transport.  With respect to distinct species of
+the same genus inhabiting very distant and isolated regions, as the process
+of modification has necessarily been slow, all the means of migration will
+have been possible during a very long period; and consequently the
+difficulty of the wide diffusion of species of the same genus is in some
+degree lessened.
+</para>
+
+<para>
+As on the theory of natural selection an interminable number of
+intermediate forms must have existed, linking together all the species in
+each group by gradations as fine as our present varieties, it may be asked,
+Why do we not see these linking forms all around us?  Why are not all
+organic beings blended together in an inextricable chaos?  With respect to
+existing forms, we should remember that we have no right to expect
+(excepting in rare cases) to discover directly connecting links between
+them, but only between each and some extinct and supplanted form.  Even on
+a wide area, which has during a long period remained continuous, and of
+which the climate and other conditions of life change insensibly in going
+from a district occupied by one species into another district occupied by a
+closely allied species, we have no just right to expect often to find
+intermediate varieties in the intermediate zone.  For we have reason to
+believe that only a few species are undergoing change at any one period;
+and all changes are slowly effected.  I have also shown that the
+intermediate varieties which will at first probably exist in the
+intermediate zones, will be liable to be supplanted by the allied forms on
+either hand; and the latter, from existing in greater numbers, will
+generally be modified and improved at a quicker rate than the intermediate
+varieties, which exist in lesser numbers; so that the intermediate
+varieties will, in the long run, be supplanted and exterminated.
+</para>
+
+<para>
+On this doctrine of the extermination of an infinitude of connecting links,
+between the living and extinct inhabitants of the world, and at each
+successive period between the extinct and still older species, why is not
+every geological formation charged with such links?  Why does not every
+collection of fossil remains afford plain evidence of the gradation and
+mutation of the forms of life?  We meet with no such evidence, and this is
+the most obvious and forcible of the many objections which may be urged
+against my theory.  Why, again, do whole groups of allied species appear,
+though certainly they often falsely appear, to have come in suddenly on the
+several geological stages?  Why do we not find great piles of strata
+beneath the Silurian system, stored with the remains of the progenitors of
+the Silurian groups of fossils?  For certainly on my theory such strata
+must somewhere have been deposited at these ancient and utterly unknown
+epochs in the world's history.
+</para>
+
+<para>
+I can answer these questions and grave objections only on the supposition
+that the geological record is far more imperfect than most geologists
+believe.  It cannot be objected that there has not been time sufficient for
+any amount of organic change; for the lapse of time has been so great as to
+be utterly inappreciable by the human intellect.  The number of specimens
+in all our museums is absolutely as nothing compared with the countless
+generations of countless species which certainly have existed.  We should
+not be able to recognise a species as the parent of any one or more species
+if we were to examine them ever so closely, unless we likewise possessed
+many of the intermediate links between their past or parent and present
+states; and these many links we could hardly ever expect to discover, owing
+to the imperfection of the geological record.  Numerous existing doubtful
+forms could be named which are probably varieties; but who will pretend
+that in future ages so many fossil links will be discovered, that
+naturalists will be able to decide, on the common view, whether or not
+these doubtful forms are varieties?  As long as most of the links between
+any two species are unknown, if any one link or intermediate variety be
+discovered, it will simply be classed as another and distinct species. 
+Only a small portion of the world has been geologically explored.  Only
+organic beings of certain classes can be preserved in a fossil condition,
+at least in any great number.  Widely ranging species vary most, and
+varieties are often at first local,--both causes rendering the discovery of
+intermediate links less likely.  Local varieties will not spread into other
+and distant regions until they are considerably modified and improved; and
+when they do spread, if discovered in a geological formation, they will
+appear as if suddenly created there, and will be simply classed as new
+species.  Most formations have been intermittent in their accumulation; and
+their duration, I am inclined to believe, has been shorter than the average
+duration of specific forms.  Successive formations are separated from each
+other by enormous blank intervals of time; for fossiliferous formations,
+thick enough to resist future degradation, can be accumulated only where
+much sediment is deposited on the subsiding bed of the sea.  During the
+alternate periods of elevation and of stationary level the record will be
+blank.  During these latter periods there will probably be more variability
+in the forms of life; during periods of subsidence, more extinction.
+</para>
+
+<para>
+With respect to the absence of fossiliferous formations beneath the lowest
+Silurian strata, I can only recur to the hypothesis given in the ninth
+chapter.  That the geological record is imperfect all will admit; but that
+it is imperfect to the degree which I require, few will be inclined to
+admit.  If we look to long enough intervals of time, geology plainly
+declares that all species have changed; and they have changed in the manner
+which my theory requires, for they have changed slowly and in a graduated
+manner.  We clearly see this in the fossil remains from consecutive
+formations invariably being much more closely related to each other, than
+are the fossils from formations distant from each other in time.
+</para>
+
+<para>
+Such is the sum of the several chief objections and difficulties which may
+justly be urged against my theory; and I have now briefly recapitulated the
+answers and explanations which can be given to them.  I have felt these
+difficulties far too heavily during many years to doubt their weight.  But
+it deserves especial notice that the more important objections relate to
+questions on which we are confessedly ignorant; nor do we know how ignorant
+we are.  We do not know all the possible transitional gradations between
+the simplest and the most perfect organs; it cannot be pretended that we
+know all the varied means of Distribution during the long lapse of years,
+or that we know how imperfect the Geological Record is.  Grave as these
+several difficulties are, in my judgment they do not overthrow the theory
+of descent with modification.
+</para>
+
+<para>
+Now let us turn to the other side of the argument.  Under domestication we
+see much variability.  This seems to be mainly due to the reproductive
+system being eminently susceptible to changes in the conditions of life; so
+that this system, when not rendered impotent, fails to reproduce offspring
+exactly like the parent-form.  Variability is governed by many complex
+laws,--by correlation of growth, by use and disuse, and by the direct
+action of the physical conditions of life.  There is much difficulty in
+ascertaining how much modification our domestic productions have undergone;
+but we may safely infer that the amount has been large, and that
+modifications can be inherited for long periods.  As long as the conditions
+of life remain the same, we have reason to believe that a modification,
+which has already been inherited for many generations, may continue to be
+inherited for an almost infinite number of generations.  On the other hand
+we have evidence that variability, when it has once come into play, does
+not wholly cease; for new varieties are still occasionally produced by our
+most anciently domesticated productions.
+</para>
+
+<para>
+Man does not actually produce variability; he only unintentionally exposes
+organic beings to new conditions of life, and then nature acts on the
+organisation, and causes variability.  But man can and does select the
+variations given to him by nature, and thus accumulate them in any desired
+manner.  He thus adapts animals and plants for his own benefit or pleasure. 
+He may do this methodically, or he may do it unconsciously by preserving
+the individuals most useful to him at the time, without any thought of
+altering the breed.  It is certain that he can largely influence the
+character of a breed by selecting, in each successive generation,
+individual differences so slight as to be quite inappreciable by an
+uneducated eye.  This process of selection has been the great agency in the
+production of the most distinct and useful domestic breeds.  That many of
+the breeds produced by man have to a large extent the character of natural
+species, is shown by the inextricable doubts whether very many of them are
+varieties or aboriginal species.
+</para>
+
+<para>
+There is no obvious reason why the principles which have acted so
+efficiently under domestication should not have acted under nature.  In the
+preservation of favoured individuals and races, during the
+constantly-recurrent Struggle for Existence, we see the most powerful and
+ever-acting means of selection.  The struggle for existence inevitably
+follows from the high geometrical ratio of increase which is common to all
+organic beings.  This high rate of increase is proved by calculation, by
+the effects of a succession of peculiar seasons, and by the results of
+naturalisation, as explained in the third chapter.  More individuals are
+born than can possibly survive.  A grain in the balance will determine
+which individual shall live and which shall die,--which variety or species
+shall increase in number, and which shall decrease, or finally become
+extinct.  As the individuals of the same species come in all respects into
+the closest competition with each other, the struggle will generally be
+most severe between them; it will be almost equally severe between the
+varieties of the same species, and next in severity between the species of
+the same genus.  But the struggle will often be very severe between beings
+most remote in the scale of nature.  The slightest advantage in one being,
+at any age or during any season, over those with which it comes into
+competition, or better adaptation in however slight a degree to the
+surrounding physical conditions, will turn the balance.
+</para>
+
+<para>
+With animals having separated sexes there will in most cases be a struggle
+between the males for possession of the females.  The most vigorous
+individuals, or those which have most successfully struggled with their
+conditions of life, will generally leave most progeny.  But success will
+often depend on having special weapons or means of defence, or on the
+charms of the males; and the slightest advantage will lead to victory.
+</para>
+
+<para>
+As geology plainly proclaims that each land has undergone great physical
+changes, we might have expected that organic beings would have varied under
+nature, in the same way as they generally have varied under the changed
+conditions of domestication.  And if there be any variability under nature,
+it would be an unaccountable fact if natural selection had not come into
+play.  It has often been asserted, but the assertion is quite incapable of
+proof, that the amount of variation under nature is a strictly limited
+quantity.  Man, though acting on external characters alone and often
+capriciously, can produce within a short period a great result by adding up
+mere individual differences in his domestic productions; and every one
+admits that there are at least individual differences in species under
+nature.  But, besides such differences, all naturalists have admitted the
+existence of varieties, which they think sufficiently distinct to be worthy
+of record in systematic works.  No one can draw any clear distinction
+between individual differences and slight varieties; or between more
+plainly marked varieties and sub-species, and species.  Let it be observed
+how naturalists differ in the rank which they assign to the many
+representative forms in Europe and North America.
+</para>
+
+<para>
+If then we have under nature variability and a powerful agent always ready
+to act and select, why should we doubt that variations in any way useful to
+beings, under their excessively complex relations of life, would be
+preserved, accumulated, and inherited?  Why, if man can by patience select
+variations most useful to himself, should nature fail in selecting
+variations useful, under changing conditions of life, to her living
+products?  What limit can be put to this power, acting during long ages and
+rigidly scrutinising the whole constitution, structure, and habits of each
+creature,--favouring the good and rejecting the bad?  I can see no limit to
+this power, in slowly and beautifully adapting each form to the most
+complex relations of life.  The theory of natural selection, even if we
+looked no further than this, seems to me to be in itself probable.  I have
+already recapitulated, as fairly as I could, the opposed difficulties and
+objections:  now let us turn to the special facts and arguments in favour
+of the theory.
+</para>
+
+<para>
+On the view that species are only strongly marked and permanent varieties,
+and that each species first existed as a variety, we can see why it is that
+no line of demarcation can be drawn between species, commonly supposed to
+have been produced by special acts of creation, and varieties which are
+acknowledged to have been produced by secondary laws.  On this same view we
+can understand how it is that in each region where many species of a genus
+have been produced, and where they now flourish, these same species should
+present many varieties; for where the manufactory of species has been
+active, we might expect, as a general rule, to find it still in action; and
+this is the case if varieties be incipient species.  Moreover, the species
+of the large genera, which afford the greater number of varieties or
+incipient species, retain to a certain degree the character of varieties;
+for they differ from each other by a less amount of difference than do the
+species of smaller genera.  The closely allied species also of the larger
+genera apparently have restricted ranges, and they are clustered in little
+groups round other species--in which respects they resemble varieties. 
+These are strange relations on the view of each species having been
+independently created, but are intelligible if all species first existed as
+varieties.
+</para>
+
+<para>
+As each species tends by its geometrical ratio of reproduction to increase
+inordinately in number; and as the modified descendants of each species
+will be enabled to increase by so much the more as they become more
+diversified in habits and structure, so as to be enabled to seize on many
+and widely different places in the economy of nature, there will be a
+constant tendency in natural selection to preserve the most divergent
+offspring of any one species.  Hence during a long-continued course of
+modification, the slight differences, characteristic of varieties of the
+same species, tend to be augmented into the greater differences
+characteristic of species of the same genus.  New and improved varieties
+will inevitably supplant and exterminate the older, less improved and
+intermediate varieties; and thus species are rendered to a large extent
+defined and distinct objects.  Dominant species belonging to the larger
+groups tend to give birth to new and dominant forms; so that each large
+group tends to become still larger, and at the same time more divergent in
+character.  But as all groups cannot thus succeed in increasing in size,
+for the world would not hold them, the more dominant groups beat the less
+dominant.  This tendency in the large groups to go on increasing in size
+and diverging in character, together with the almost inevitable contingency
+of much extinction, explains the arrangement of all the forms of life, in
+groups subordinate to groups, all within a few great classes, which we now
+see everywhere around us, and which has prevailed throughout all time. 
+This grand fact of the grouping of all organic beings seems to me utterly
+inexplicable on the theory of creation.
+</para>
+
+<para>
+As natural selection acts solely by accumulating slight, successive,
+favourable variations, it can produce no great or sudden modification; it
+can act only by very short and slow steps.  Hence the canon of 'Natura non
+facit saltum,' which every fresh addition to our knowledge tends to make
+more strictly correct, is on this theory simply intelligible.  We can
+plainly see why nature is prodigal in variety, though niggard in
+innovation.  But why this should be a law of nature if each species has
+been independently created, no man can explain.
+</para>
+
+<para>
+Many other facts are, as it seems to me, explicable on this theory.  How
+strange it is that a bird, under the form of woodpecker, should have been
+created to prey on insects on the ground; that upland geese, which never or
+rarely swim, should have been created with webbed feet; that a thrush
+should have been created to dive and feed on sub-aquatic insects; and that
+a petrel should have been created with habits and structure fitting it for
+the life of an auk or grebe! and so on in endless other cases.  But on the
+view of each species constantly trying to increase in number, with natural
+selection always ready to adapt the slowly varying descendants of each to
+any unoccupied or ill-occupied place in nature, these facts cease to be
+strange, or perhaps might even have been anticipated.
+</para>
+
+<para>
+As natural selection acts by competition, it adapts the inhabitants of each
+country only in relation to the degree of perfection of their associates;
+so that we need feel no surprise at the inhabitants of any one country,
+although on the ordinary view supposed to have been specially created and
+adapted for that country, being beaten and supplanted by the naturalised
+productions from another land.  Nor ought we to marvel if all the
+contrivances in nature be not, as far as we can judge, absolutely perfect;
+and if some of them be abhorrent to our ideas of fitness.  We need not
+marvel at the sting of the bee causing the bee's own death; at drones being
+produced in such vast numbers for one single act, and being then
+slaughtered by their sterile sisters; at the astonishing waste of pollen by
+our fir-trees; at the instinctive hatred of the queen bee for her own
+fertile daughters; at ichneumonidae feeding within the live bodies of
+caterpillars; and at other such cases.  The wonder indeed is, on the theory
+of natural selection, that more cases of the want of absolute perfection
+have not been observed.
+</para>
+
+<para>
+The complex and little known laws governing variation are the same, as far
+as we can see, with the laws which have governed the production of
+so-called specific forms.  In both cases physical conditions seem to have
+produced but little direct effect; yet when varieties enter any zone, they
+occasionally assume some of the characters of the species proper to that
+zone.  In both varieties and species, use and disuse seem to have produced
+some effect; for it is difficult to resist this conclusion when we look,
+for instance, at the logger-headed duck, which has wings incapable of
+flight, in nearly the same condition as in the domestic duck; or when we
+look at the burrowing tucutucu, which is occasionally blind, and then at
+certain moles, which are habitually blind and have their eyes covered with
+skin; or when we look at the blind animals inhabiting the dark caves of
+America and Europe.  In both varieties and species correlation of growth
+seems to have played a most important part, so that when one part has been
+modified other parts are necessarily modified.  In both varieties and
+species reversions to long-lost characters occur.  How inexplicable on the
+theory of creation is the occasional appearance of stripes on the shoulder
+and legs of the several species of the horse-genus and in their hybrids! 
+How simply is this fact explained if we believe that these species have
+descended from a striped progenitor, in the same manner as the several
+domestic breeds of pigeon have descended from the blue and barred
+rock-pigeon!
+</para>
+
+<para>
+On the ordinary view of each species having been independently created, why
+should the specific characters, or those by which the species of the same
+genus differ from each other, be more variable than the generic characters
+in which they all agree?  Why, for instance, should the colour of a flower
+be more likely to vary in any one species of a genus, if the other species,
+supposed to have been created independently, have differently coloured
+flowers, than if all the species of the genus have the same coloured
+flowers?  If species are only well-marked varieties, of which the
+characters have become in a high degree permanent, we can understand this
+fact; for they have already varied since they branched off from a common
+progenitor in certain characters, by which they have come to be
+specifically distinct from each other; and therefore these same characters
+would be more likely still to be variable than the generic characters which
+have been inherited without change for an enormous period.  It is
+inexplicable on the theory of creation why a part developed in a very
+unusual manner in any one species of a genus, and therefore, as we may
+naturally infer, of great importance to the species, should be eminently
+liable to variation; but, on my view, this part has undergone, since the
+several species branched off from a common progenitor, an unusual amount of
+variability and modification, and therefore we might expect this part
+generally to be still variable.  But a part may be developed in the most
+unusual manner, like the wing of a bat, and yet not be more variable than
+any other structure, if the part be common to many subordinate forms, that
+is, if it has been inherited for a very long period; for in this case it
+will have been rendered constant by long-continued natural selection.
+</para>
+
+<para>
+Glancing at instincts, marvellous as some are, they offer no greater
+difficulty than does corporeal structure on the theory of the natural
+selection of successive, slight, but profitable modifications.  We can thus
+understand why nature moves by graduated steps in endowing different
+animals of the same class with their several instincts.  I have attempted
+to show how much light the principle of gradation throws on the admirable
+architectural powers of the hive-bee.  Habit no doubt sometimes comes into
+play in modifying instincts; but it certainly is not indispensable, as we
+see, in the case of neuter insects, which leave no progeny to inherit the
+effects of long-continued habit.  On the view of all the species of the
+same genus having descended from a common parent, and having inherited much
+in common, we can understand how it is that allied species, when placed
+under considerably different conditions of life, yet should follow nearly
+the same instincts; why the thrush of South America, for instance, lines
+her nest with mud like our British species.  On the view of instincts
+having been slowly acquired through natural selection we need not marvel at
+some instincts being apparently not perfect and liable to mistakes, and at
+many instincts causing other animals to suffer.
+</para>
+
+<para>
+If species be only well-marked and permanent varieties, we can at once see
+why their crossed offspring should follow the same complex laws in their
+degrees and kinds of resemblance to their parents,--in being absorbed into
+each other by successive crosses, and in other such points,--as do the
+crossed offspring of acknowledged varieties.  On the other hand, these
+would be strange facts if species have been independently created, and
+varieties have been produced by secondary laws.
+</para>
+
+<para>
+If we admit that the geological record is imperfect in an extreme degree,
+then such facts as the record gives, support the theory of descent with
+modification.  New species have come on the stage slowly and at successive
+intervals; and the amount of change, after equal intervals of time, is
+widely different in different groups.  The extinction of species and of
+whole groups of species, which has played so conspicuous a part in the
+history of the organic world, almost inevitably follows on the principle of
+natural selection; for old forms will be supplanted by new and improved
+forms.  Neither single species nor groups of species reappear when the
+chain of ordinary generation has once been broken.  The gradual diffusion
+of dominant forms, with the slow modification of their descendants, causes
+the forms of life, after long intervals of time, to appear as if they had
+changed simultaneously throughout the world.  The fact of the fossil
+remains of each formation being in some degree intermediate in character
+between the fossils in the formations above and below, is simply explained
+by their intermediate position in the chain of descent.  The grand fact
+that all extinct organic beings belong to the same system with recent
+beings, falling either into the same or into intermediate groups, follows
+from the living and the extinct being the offspring of common parents.  As
+the groups which have descended from an ancient progenitor have generally
+diverged in character, the progenitor with its early descendants will often
+be intermediate in character in comparison with its later descendants; and
+thus we can see why the more ancient a fossil is, the oftener it stands in
+some degree intermediate between existing and allied groups.  Recent forms
+are generally looked at as being, in some vague sense, higher than ancient
+and extinct forms; and they are in so far higher as the later and more
+improved forms have conquered the older and less improved organic beings in
+the struggle for life.  Lastly, the law of the long endurance of allied
+forms on the same continent,--of marsupials in Australia, of edentata in
+America, and other such cases,--is intelligible, for within a confined
+country, the recent and the extinct will naturally be allied by descent.
+</para>
+
+<para>
+Looking to geographical distribution, if we admit that there has been
+during the long course of ages much migration from one part of the world to
+another, owing to former climatal and geographical changes and to the many
+occasional and unknown means of dispersal, then we can understand, on the
+theory of descent with modification, most of the great leading facts in
+Distribution.  We can see why there should be so striking a parallelism in
+the distribution of organic beings throughout space, and in their
+geological succession throughout time; for in both cases the beings have
+been connected by the bond of ordinary generation, and the means of
+modification have been the same.  We see the full meaning of the wonderful
+fact, which must have struck every traveller, namely, that on the same
+continent, under the most diverse conditions, under heat and cold, on
+mountain and lowland, on deserts and marshes, most of the inhabitants
+within each great class are plainly related; for they will generally be
+descendants of the same progenitors and early colonists.  On this same
+principle of former migration, combined in most cases with modification, we
+can understand, by the aid of the Glacial period, the identity of some few
+plants, and the close alliance of many others, on the most distant
+mountains, under the most different climates; and likewise the close
+alliance of some of the inhabitants of the sea in the northern and southern
+temperate zones, though separated by the whole intertropical ocean. 
+Although two areas may present the same physical conditions of life, we
+need feel no surprise at their inhabitants being widely different, if they
+have been for a long period completely separated from each other; for as
+the relation of organism to organism is the most important of all
+relations, and as the two areas will have received colonists from some
+third source or from each other, at various periods and in different
+proportions, the course of modification in the two areas will inevitably be
+different.
+</para>
+
+<para>
+On this view of migration, with subsequent modification, we can see why
+oceanic islands should be inhabited by few species, but of these, that many
+should be peculiar.  We can clearly see why those animals which cannot
+cross wide spaces of ocean, as frogs and terrestrial mammals, should not
+inhabit oceanic islands; and why, on the other hand, new and peculiar
+species of bats, which can traverse the ocean, should so often be found on
+islands far distant from any continent.  Such facts as the presence of
+peculiar species of bats, and the absence of all other mammals, on oceanic
+islands, are utterly inexplicable on the theory of independent acts of
+creation.
+</para>
+
+<para>
+The existence of closely allied or representative species in any two areas,
+implies, on the theory of descent with modification, that the same parents
+formerly inhabited both areas; and we almost invariably find that wherever
+many closely allied species inhabit two areas, some identical species
+common to both still exist.  Wherever many closely allied yet distinct
+species occur, many doubtful forms and varieties of the same species
+likewise occur.  It is a rule of high generality that the inhabitants of
+each area are related to the inhabitants of the nearest source whence
+immigrants might have been derived.  We see this in nearly all the plants
+and animals of the Galapagos archipelago, of Juan Fernandez, and of the
+other American islands being related in the most striking manner to the
+plants and animals of the neighbouring American mainland; and those of the
+Cape de Verde archipelago and other African islands to the African
+mainland.  It must be admitted that these facts receive no explanation on
+the theory of creation.
+</para>
+
+<para>
+The fact, as we have seen, that all past and present organic beings
+constitute one grand natural system, with group subordinate to group, and
+with extinct groups often falling in between recent groups, is intelligible
+on the theory of natural selection with its contingencies of extinction and
+divergence of character.  On these same principles we see how it is, that
+the mutual affinities of the species and genera within each class are so
+complex and circuitous.  We see why certain characters are far more
+serviceable than others for classification;--why adaptive characters,
+though of paramount importance to the being, are of hardly any importance
+in classification; why characters derived from rudimentary parts, though of
+no service to the being, are often of high classificatory value; and why
+embryological characters are the most valuable of all.  The real affinities
+of all organic beings are due to inheritance or community of descent.  The
+natural system is a genealogical arrangement, in which we have to discover
+the lines of descent by the most permanent characters, however slight their
+vital importance may be.
+</para>
+
+<para>
+The framework of bones being the same in the hand of a man, wing of a bat,
+fin of the porpoise, and leg of the horse,--the same number of vertebrae
+forming the neck of the giraffe and of the elephant,--and innumerable other
+such facts, at once explain themselves on the theory of descent with slow
+and slight successive modifications.  The similarity of pattern in the wing
+and leg of a bat, though used for such different purpose,--in the jaws and
+legs of a crab,--in the petals, stamens, and pistils of a flower, is
+likewise intelligible on the view of the gradual modification of parts or
+organs, which were alike in the early progenitor of each class.  On the
+principle of successive variations not always supervening at an early age,
+and being inherited at a corresponding not early period of life, we can
+clearly see why the embryos of mammals, birds, reptiles, and fishes should
+be so closely alike, and should be so unlike the adult forms.  We may cease
+marvelling at the embryo of an air-breathing mammal or bird having
+branchial slits and arteries running in loops, like those in a fish which
+has to breathe the air dissolved in water, by the aid of well-developed
+branchiae.
+</para>
+
+<para>
+Disuse, aided sometimes by natural selection, will often tend to reduce an
+organ, when it has become useless by changed habits or under changed
+conditions of life; and we can clearly understand on this view the meaning
+of rudimentary organs.  But disuse and selection will generally act on each
+creature, when it has come to maturity and has to play its full part in the
+struggle for existence, and will thus have little power of acting on an
+organ during early life; hence the organ will not be much reduced or
+rendered rudimentary at this early age.  The calf, for instance, has
+inherited teeth, which never cut through the gums of the upper jaw, from an
+early progenitor having well-developed teeth; and we may believe, that the
+teeth in the mature animal were reduced, during successive generations, by
+disuse or by the tongue and palate having been fitted by natural selection
+to browse without their aid; whereas in the calf, the teeth have been left
+untouched by selection or disuse, and on the principle of inheritance at
+corresponding ages have been inherited from a remote period to the present
+day.  On the view of each organic being and each separate organ having been
+specially created, how utterly inexplicable it is that parts, like the
+teeth in the embryonic calf or like the shrivelled wings under the soldered
+wing-covers of some beetles, should thus so frequently bear the plain stamp
+of inutility!  Nature may be said to have taken pains to reveal, by
+rudimentary organs and by homologous structures, her scheme of
+modification, which it seems that we wilfully will not understand.
+</para>
+
+<para>
+I have now recapitulated the chief facts and considerations which have
+thoroughly convinced me that species have changed, and are still slowly
+changing by the preservation and accumulation of successive slight
+favourable variations.  Why, it may be asked, have all the most eminent
+living naturalists and geologists rejected this view of the mutability of
+species?  It cannot be asserted that organic beings in a state of nature
+are subject to no variation; it cannot be proved that the amount of
+variation in the course of long ages is a limited quantity; no clear
+distinction has been, or can be, drawn between species and well-marked
+varieties.  It cannot be maintained that species when intercrossed are
+invariably sterile, and varieties invariably fertile; or that sterility is
+a special endowment and sign of creation.  The belief that species were
+immutable productions was almost unavoidable as long as the history of the
+world was thought to be of short duration; and now that we have acquired
+some idea of the lapse of time, we are too apt to assume, without proof,
+that the geological record is so perfect that it would have afforded us
+plain evidence of the mutation of species, if they had undergone mutation.
+</para>
+
+<para>
+But the chief cause of our natural unwillingness to admit that one species
+has given birth to other and distinct species, is that we are always slow
+in admitting any great change of which we do not see the intermediate
+steps.  The difficulty is the same as that felt by so many geologists, when
+Lyell first insisted that long lines of inland cliffs had been formed, and
+great valleys excavated, by the slow action of the coast-waves.  The mind
+cannot possibly grasp the full meaning of the term of a hundred million
+years; it cannot add up and perceive the full effects of many slight
+variations, accumulated during an almost infinite number of generations.
+</para>
+
+<para>
+Although I am fully convinced of the truth of the views given in this
+volume under the form of an abstract, I by no means expect to convince
+experienced naturalists whose minds are stocked with a multitude of facts
+all viewed, during a long course of years, from a point of view directly
+opposite to mine.  It is so easy to hide our ignorance under such
+expressions as the 'plan of creation,' 'unity of design,' &amp;c., and to think
+that we give an explanation when we only restate a fact.  Any one whose
+disposition leads him to attach more weight to unexplained difficulties
+than to the explanation of a certain number of facts will certainly reject
+my theory.  A few naturalists, endowed with much flexibility of mind, and
+who have already begun to doubt on the immutability of species, may be
+influenced by this volume; but I look with confidence to the future, to
+young and rising naturalists, who will be able to view both sides of the
+question with impartiality.  Whoever is led to believe that species are
+mutable will do good service by conscientiously expressing his conviction;
+for only thus can the load of prejudice by which this subject is
+overwhelmed be removed.
+</para>
+
+<para>
+Several eminent naturalists have of late published their belief that a
+multitude of reputed species in each genus are not real species; but that
+other species are real, that is, have been independently created.  This
+seems to me a strange conclusion to arrive at.  They admit that a multitude
+of forms, which till lately they themselves thought were special creations,
+and which are still thus looked at by the majority of naturalists, and
+which consequently have every external characteristic feature of true
+species,--they admit that these have been produced by variation, but they
+refuse to extend the same view to other and very slightly different forms. 
+Nevertheless they do not pretend that they can define, or even conjecture,
+which are the created forms of life, and which are those produced by
+secondary laws.  They admit variation as a vera causa in one case, they
+arbitrarily reject it in another, without assigning any distinction in the
+two cases.  The day will come when this will be given as a curious
+illustration of the blindness of preconceived opinion.  These authors seem
+no more startled at a miraculous act of creation than at an ordinary birth. 
+But do they really believe that at innumerable periods in the earth's
+history certain elemental atoms have been commanded suddenly to flash into
+living tissues?  Do they believe that at each supposed act of creation one
+individual or many were produced?  Were all the infinitely numerous kinds
+of animals and plants created as eggs or seed, or as full grown? and in the
+case of mammals, were they created bearing the false marks of nourishment
+from the mother's womb?  Although naturalists very properly demand a full
+explanation of every difficulty from those who believe in the mutability of
+species, on their own side they ignore the whole subject of the first
+appearance of species in what they consider reverent silence.
+</para>
+
+<para>
+It may be asked how far I extend the doctrine of the modification of
+species.  The question is difficult to answer, because the more distinct
+the forms are which we may consider, by so much the arguments fall away in
+force.  But some arguments of the greatest weight extend very far.  All the
+members of whole classes can be connected together by chains of affinities,
+and all can be classified on the same principle, in groups subordinate to
+groups.  Fossil remains sometimes tend to fill up very wide intervals
+between existing orders.  Organs in a rudimentary condition plainly show
+that an early progenitor had the organ in a fully developed state; and this
+in some instances necessarily implies an enormous amount of modification in
+the descendants.  Throughout whole classes various structures are formed on
+the same pattern, and at an embryonic age the species closely resemble each
+other.  Therefore I cannot doubt that the theory of descent with
+modification embraces all the members of the same class.  I believe that
+animals have descended from at most only four or five progenitors, and
+plants from an equal or lesser number.
+</para>
+
+<para>
+Analogy would lead me one step further, namely, to the belief that all
+animals and plants have descended from some one prototype.  But analogy may
+be a deceitful guide.  Nevertheless all living things have much in common,
+in their chemical composition, their germinal vesicles, their cellular
+structure, and their laws of growth and reproduction.  We see this even in
+so trifling a circumstance as that the same poison often similarly affects
+plants and animals; or that the poison secreted by the gall-fly produces
+monstrous growths on the wild rose or oak-tree.  Therefore I should infer
+from analogy that probably all the organic beings which have ever lived on
+this earth have descended from some one primordial form, into which life
+was first breathed.
+</para>
+
+<para>
+When the views entertained in this volume on the origin of species, or when
+analogous views are generally admitted, we can dimly foresee that there
+will be a considerable revolution in natural history.  Systematists will be
+able to pursue their labours as at present; but they will not be
+incessantly haunted by the shadowy doubt whether this or that form be in
+essence a species.  This I feel sure, and I speak after experience, will be
+no slight relief.  The endless disputes whether or not some fifty species
+of British brambles are true species will cease.  Systematists will have
+only to decide (not that this will be easy) whether any form be
+sufficiently constant and distinct from other forms, to be capable of
+definition; and if definable, whether the differences be sufficiently
+important to deserve a specific name.  This latter point will become a far
+more essential consideration than it is at present; for differences,
+however slight, between any two forms, if not blended by intermediate
+gradations, are looked at by most naturalists as sufficient to raise both
+forms to the rank of species.  Hereafter we shall be compelled to
+acknowledge that the only distinction between species and well-marked
+varieties is, that the latter are known, or believed, to be connected at
+the present day by intermediate gradations, whereas species were formerly
+thus connected.  Hence, without quite rejecting the consideration of the
+present existence of intermediate gradations between any two forms, we
+shall be led to weigh more carefully and to value higher the actual amount
+of difference between them.  It is quite possible that forms now generally
+acknowledged to be merely varieties may hereafter be thought worthy of
+specific names, as with the primrose and cowslip; and in this case
+scientific and common language will come into accordance.  In short, we
+shall have to treat species in the same manner as those naturalists treat
+genera, who admit that genera are merely artificial combinations made for
+convenience.  This may not be a cheering prospect; but we shall at least be
+freed from the vain search for the undiscovered and undiscoverable essence
+of the term species.
+</para>
+
+<para>
+The other and more general departments of natural history will rise greatly
+in interest.  The terms used by naturalists of affinity, relationship,
+community of type, paternity, morphology, adaptive characters, rudimentary
+and aborted organs, &amp;c., will cease to be metaphorical, and will have a
+plain signification.  When we no longer look at an organic being as a
+savage looks at a ship, as at something wholly beyond his comprehension;
+when we regard every production of nature as one which has had a history;
+when we contemplate every complex structure and instinct as the summing up
+of many contrivances, each useful to the possessor, nearly in the same way
+as when we look at any great mechanical invention as the summing up of the
+labour, the experience, the reason, and even the blunders of numerous
+workmen; when we thus view each organic being, how far more interesting, I
+speak from experience, will the study of natural history become!
+</para>
+
+<para>
+A grand and almost untrodden field of inquiry will be opened, on the causes
+and laws of variation, on correlation of growth, on the effects of use and
+disuse, on the direct action of external conditions, and so forth.  The
+study of domestic productions will rise immensely in value.  A new variety
+raised by man will be a far more important and interesting subject for
+study than one more species added to the infinitude of already recorded
+species.  Our classifications will come to be, as far as they can be so
+made, genealogies; and will then truly give what may be called the plan of
+creation.  The rules for classifying will no doubt become simpler when we
+have a definite object in view.  We possess no pedigrees or armorial
+bearings; and we have to discover and trace the many diverging lines of
+descent in our natural genealogies, by characters of any kind which have
+long been inherited.  Rudimentary organs will speak infallibly with respect
+to the nature of long-lost structures.  Species and groups of species,
+which are called aberrant, and which may fancifully be called living
+fossils, will aid us in forming a picture of the ancient forms of life. 
+Embryology will reveal to us the structure, in some degree obscured, of the
+prototypes of each great class.
+</para>
+
+<para>
+When we can feel assured that all the individuals of the same species, and
+all the closely allied species of most genera, have within a not very
+remote period descended from one parent, and have migrated from some one
+birthplace; and when we better know the many means of migration, then, by
+the light which geology now throws, and will continue to throw, on former
+changes of climate and of the level of the land, we shall surely be enabled
+to trace in an admirable manner the former migrations of the inhabitants of
+the whole world.  Even at present, by comparing the differences of the
+inhabitants of the sea on the opposite sides of a continent, and the nature
+of the various inhabitants of that continent in relation to their apparent
+means of immigration, some light can be thrown on ancient geography.
+</para>
+
+<para>
+The noble science of Geology loses glory from the extreme imperfection of
+the record.  The crust of the earth with its embedded remains must not be
+looked at as a well-filled museum, but as a poor collection made at hazard
+and at rare intervals.  The accumulation of each great fossiliferous
+formation will be recognised as having depended on an unusual concurrence
+of circumstances, and the blank intervals between the successive stages as
+having been of vast duration.  But we shall be able to gauge with some
+security the duration of these intervals by a comparison of the preceding
+and succeeding organic forms.  We must be cautious in attempting to
+correlate as strictly contemporaneous two formations, which include few
+identical species, by the general succession of their forms of life.  As
+species are produced and exterminated by slowly acting and still existing
+causes, and not by miraculous acts of creation and by catastrophes; and as
+the most important of all causes of organic change is one which is almost
+independent of altered and perhaps suddenly altered physical conditions,
+namely, the mutual relation of organism to organism,--the improvement of
+one being entailing the improvement or the extermination of others; it
+follows, that the amount of organic change in the fossils of consecutive
+formations probably serves as a fair measure of the lapse of actual time. 
+A number of species, however, keeping in a body might remain for a long
+period unchanged, whilst within this same period, several of these species,
+by migrating into new countries and coming into competition with foreign
+associates, might become modified; so that we must not overrate the
+accuracy of organic change as a measure of time.  During early periods of
+the earth's history, when the forms of life were probably fewer and
+simpler, the rate of change was probably slower; and at the first dawn of
+life, when very few forms of the simplest structure existed, the rate of
+change may have been slow in an extreme degree.  The whole history of the
+world, as at present known, although of a length quite incomprehensible by
+us, will hereafter be recognised as a mere fragment of time, compared with
+the ages which have elapsed since the first creature, the progenitor of
+innumerable extinct and living descendants, was created.
+</para>
+
+<para>
+In the distant future I see open fields for far more important researches. 
+Psychology will be based on a new foundation, that of the necessary
+acquirement of each mental power and capacity by gradation.  Light will be
+thrown on the origin of man and his history.
+</para>
+
+<para>
+Authors of the highest eminence seem to be fully satisfied with the view
+that each species has been independently created.  To my mind it accords
+better with what we know of the laws impressed on matter by the Creator,
+that the production and extinction of the past and present inhabitants of
+the world should have been due to secondary causes, like those determining
+the birth and death of the individual.  When I view all beings not as
+special creations, but as the lineal descendants of some few beings which
+lived long before the first bed of the Silurian system was deposited, they
+seem to me to become ennobled.  Judging from the past, we may safely infer
+that not one living species will transmit its unaltered likeness to a
+distant futurity.  And of the species now living very few will transmit
+progeny of any kind to a far distant futurity; for the manner in which all
+organic beings are grouped, shows that the greater number of species of
+each genus, and all the species of many genera, have left no descendants,
+but have become utterly extinct.  We can so far take a prophetic glance
+into futurity as to foretel that it will be the common and widely-spread
+species, belonging to the larger and dominant groups, which will ultimately
+prevail and procreate new and dominant species.  As all the living forms of
+life are the lineal descendants of those which lived long before the
+Silurian epoch, we may feel certain that the ordinary succession by
+generation has never once been broken, and that no cataclysm has desolated
+the whole world.  Hence we may look with some confidence to a secure future
+of equally inappreciable length.  And as natural selection works solely by
+and for the good of each being, all corporeal and mental endowments will
+tend to progress towards perfection.
+</para>
+
+<para>
+It is interesting to contemplate an entangled bank, clothed with many
+plants of many kinds, with birds singing on the bushes, with various
+insects flitting about, and with worms crawling through the damp earth, and
+to reflect that these elaborately constructed forms, so different from each
+other, and dependent on each other in so complex a manner, have all been
+produced by laws acting around us.  These laws, taken in the largest sense,
+being Growth with Reproduction; Inheritance which is almost implied by
+reproduction; Variability from the indirect and direct action of the
+external conditions of life, and from use and disuse; a Ratio of Increase
+so high as to lead to a Struggle for Life, and as a consequence to Natural
+Selection, entailing Divergence of Character and the Extinction of
+less-improved forms.  Thus, from the war of nature, from famine and death,
+the most exalted object which we are capable of conceiving, namely, the
+production of the higher animals, directly follows.  There is grandeur in
+this view of life, with its several powers, having been originally breathed
+into a few forms or into one; and that, whilst this planet has gone cycling
+on according to the fixed law of gravity, from so simple a beginning
+endless forms most beautiful and most wonderful have been, and are being,
+evolved.
+</para>
+</chapter>
+</bookbody>
+<backmatter>
+
+<index>
+
+<title>
+Subject Index
+</title>
+
+<item>
+Aberrant groups
+</item>
+
+<item>
+Abyssinia, plants of
+</item>
+
+<item>
+Acclimatisation
+</item>
+
+<item>
+Affinities of extinct species; of organic beings
+</item>
+
+<item>
+Agassiz on Amblyopsis; on groups of species suddenly appearing; on
+embryological succession; on the glacial period; on embryological
+characters; on the embryos of vertebrata; on parallelism of embryological
+development and geological succession
+</item>
+<item>
+Algae of New Zealand
+</item>
+
+<item>
+Alligators, males, fighting
+</item>
+
+<item>
+Amblyopsis, blind fish
+</item>
+
+<item>
+America, North, productions allied to those of Europe; boulders and
+glaciers of; South, no modern formations on west coast
+</item>
+
+<item>
+Ammonites, sudden extinction of
+</item>
+
+<item>
+Anagallis, sterility of
+</item>
+
+<item>
+Analog of variations
+</item>
+
+<item>
+Ancylus
+</item>
+
+<item>
+Animals, not domesticated from being variable; domestic; descended from
+several stocks; acclimatisation of; of Australia; with thicker fur in cold
+climates; blind, in caves; extinct, of Australia
+</item>
+
+<item>
+Anomma
+</item>
+
+<item>
+Antarctic islands, ancient flora of
+</item>
+
+<item>
+Antirrhinum
+</item>
+
+<item>
+Ants attending aphides; slave-making instinct
+</item>
+
+<item>
+Ants, neuter, structure of
+</item>
+
+<item>
+Aphides attended by ants
+</item>
+
+<item>
+Aphis, development of
+</item>
+
+<item>
+Apteryx
+</item>
+
+<item>
+Arab horses
+</item>
+
+<item>
+Aralo-Caspian Sea
+</item>
+
+<item>
+Archiac, M. de, on the succession of species
+</item>
+
+<item>
+Artichoke, Jerusalem
+</item>
+
+<item>
+Ascension, plants of
+</item>
+
+<item>
+Asclepias, pollen of
+</item>
+
+<item>
+Asparagus
+</item>
+
+<item>
+Aspicarpa
+</item>
+
+<item>
+Asses, striped
+</item>
+
+<item>
+Ateuchus
+</item>
+
+<item>
+Audubon, on habits of frigate-bird; on variation in birds'-nests; on heron
+eating seeds
+</item>
+
+<item>
+Australia, animals of; dogs of; extinct animals of; European plants in
+</item>
+
+<item>
+Azara on flies destroying cattle
+</item>
+
+<item>
+Azores, flora of
+</item>
+
+<item>
+Babington, Mr., on British plants
+</item>
+
+<item>
+Balancement of growth
+</item>
+
+<item>
+Bamboo with hooks
+</item>
+
+<item>
+Barberry, flowers of
+</item>
+
+<item>
+Barrande, M., on Silurian colonies; on the succession of species; on
+parallelism of palaeozoic formations; on affinities of ancient species
+</item>
+
+<item>
+Barriers, importance of
+</item>
+
+<item>
+Batrachians on islands
+</item>
+
+<item>
+Bats, how structure acquired; distribution of
+</item>
+
+<item>
+Bear, catching water-insects
+</item>
+
+<item>
+Bee, sting of; queen, killing rivals
+</item>
+
+<item>
+Bees fertilizing flowers; hive; not sucking the red clover; cell-making
+instinct; humble, cells of; parasitic
+</item>
+
+<item>
+Beetles, wingless, in Madeira; with deficient tarsi
+</item>
+
+<item>
+Bentham, Mr., on British plants; on classification
+</item>
+
+<item>
+Berkeley, Mr., on seeds in salt-water
+</item>
+
+<item>
+Bermuda, birds of
+</item>
+
+<item>
+Birds acquiring fear; annually cross the Atlantic; colour of, on
+continents; fossil, in caves of Brazil; of Madeira, Bermuda, and Galapagos;
+song of males; transporting seeds; waders; wingless; with traces of
+embryonic teeth
+</item>
+
+<item>
+Bizcacha, affinities of
+</item>
+
+<item>
+Bladder for swimming in fish
+</item>
+
+<item>
+Blindness of cave animals
+</item>
+
+<item>
+Blyth, Mr., on distinctness of Indian cattle; on striped Hemionus; on
+crossed geese
+</item>
+
+<item>
+Boar; shoulder-pad of
+</item>
+
+<item>
+Borrow, Mr., on the Spanish pointer
+</item>
+
+<item>
+Bory St. Vincent, on Batrachians
+</item>
+
+<item>
+Bosquet, M., on fossil Chthamalus
+</item>
+
+<item>
+Boulders, erratic, on the Azores
+</item>
+
+<item>
+Branchiae
+</item>
+
+<item>
+Brent, Mr., on house-tumblers; on hawks killing pigeons
+</item>
+
+<item>
+Brewer, Dr., on American cuckoo
+</item>
+
+<item>
+Britain, mammals of
+</item>
+
+<item>
+Bronn, on duration of specific forms
+</item>
+
+<item>
+Brown, Robert, on classification
+</item>
+
+<item>
+Buckman, on variation in plants
+</item>
+
+<item>
+Buzareingues on sterility of varieties
+</item>
+
+<item>
+Cabbage, varieties of, crossed
+</item>
+
+<item>
+Calceolaria
+</item>
+
+<item>
+Canary-birds, sterility of hybrids
+</item>
+
+<item>
+Cape de Verde islands
+</item>
+
+<item>
+Cape of Good Hope, plants of
+</item>
+
+<item>
+Carrier-pigeons killed by hawks
+</item>
+
+<item>
+Cassini on flowers of composita
+</item>
+
+<item>
+Catasetum
+</item>
+
+<item>
+Cats, with blue eyes, deaf; variation in habits of; curling tail when going
+to spring
+</item>
+
+<item>
+Cattle destroying fir-trees; destroyed by flies in La Plata; breeds of,
+locally extinct; fertility of Indian and European breeds
+</item>
+
+<item>
+Cave, inhabitants of, blind
+</item>
+
+<item>
+Centres of creation
+</item>
+
+<item>
+Cephalopodae, development of
+</item>
+
+<item>
+Cervulus
+</item>
+
+<item>
+Cetacea, teeth and hair
+</item>
+
+<item>
+Ceylon, plants of
+</item>
+
+<item>
+Chalk formation
+</item>
+
+<item>
+Charlock
+</item>
+
+<item>
+Checks to increase; mutual
+</item>
+
+<item>
+Chickens, instinctive tameness of
+</item>
+
+<item>
+Chthamalinae
+</item>
+
+<item>
+Chthamalus, cretacean species of
+</item>
+
+<item>
+Circumstances favourable to selection of domestic products; to natural
+selection
+</item>
+
+<item>
+Cirripedes capable of crossing; carapace aborted; their ovigerous frena;
+fossil; larvae of
+</item>
+
+<item>
+Clift, Mr., on the succession of types
+</item>
+
+<item>
+Climate, effects of, in checking increase of beings; adaptation of, to
+organisms
+</item>
+
+<item>
+Cobites, intestine of
+</item>
+
+<item>
+Cockroach
+</item>
+
+<item>
+Collections, palaeontological, poor
+</item>
+
+<item>
+Colour, influenced by climate; in relation to attacks by flies
+</item>
+
+<item>
+Columba livia, parent of domestic pigeons
+</item>
+
+<item>
+Colymbetes
+</item>
+
+<item>
+Compositae, outer and inner florets of; male flowers of
+</item>
+
+<item>
+Conclusion, general
+</item>
+
+<item>
+Conditions, slight changes in, favourable to fertility
+</item>
+
+<item>
+Coot
+</item>
+
+<item>
+Coral-islands, seeds drifted to; reefs, indicating movements of earth
+</item>
+
+<item>
+Corn-crake
+</item>
+
+<item>
+Correlation of growth in domestic productions; of growth
+</item>
+
+<item>
+Cowslip
+</item>
+
+<item>
+Creation, single centres of
+</item>
+
+<item>
+Crinum
+</item>
+
+<item>
+Crosses, reciprocal
+</item>
+
+<item>
+Crossing of domestic animals, importance in altering breeds; advantages of;
+unfavourable to selection
+</item>
+
+<item>
+Crustacea of New Zealand
+</item>
+
+<item>
+Crustacean, blind
+</item>
+
+<item>
+Cryptocerus
+</item>
+
+<item>
+Ctenomys, blind
+</item>
+
+<item>
+Cuckoo, instinct of
+</item>
+
+<item>
+Currants, grafts of
+</item>
+
+<item>
+Currents of sea, rate of
+</item>
+
+<item>
+Cuvier on conditions of existence; on fossil monkeys; Fred., on instinct
+</item>
+
+<item>
+Dana, Prof., on blind cave-animals; on relations of crustaceans of Japan,
+on crustaceans of New Zealand
+</item>
+
+<item>
+De Candolle on struggle for existence; on umbelliferae; on general
+affinities; Alph., on low plants, widely dispersed; on widely-ranging
+plants being variable; on naturalisation; on winged seeds; on Alpine
+species suddenly becoming rare; on distribution of plants with large seeds;
+on vegetation of Australia; on fresh-water plants; on insular plants
+</item>
+
+<item>
+Degradation of coast-rocks
+</item>
+
+<item>
+Denudation, rate of; of oldest rocks
+</item>
+
+<item>
+Development of ancient forms
+</item>
+
+<item>
+Devonian system
+</item>
+
+<item>
+Dianthus, fertility of crosses
+</item>
+
+<item>
+Dirt on feet of birds
+</item>
+
+<item>
+Division, physiological, of labour
+</item>
+
+<item>
+Dogs, hairless, with imperfect teeth; descended from several wild stocks;
+domestic instincts of; inherited civilisation of; fertility of breeds
+together; of crosses; proportions of, when young
+</item>
+
+<item>
+Downing, Mr., on fruit-trees in America
+</item>
+
+<item>
+Downs, North and South
+</item>
+
+<item>
+Dragon-flies, intestines of
+</item>
+
+<item>
+Drift-timber
+</item>
+
+<item>
+Driver-ant
+</item>
+
+<item>
+Drones killed by other bees
+</item>
+
+<item>
+Duck, domestic, wings of, reduced; logger-headed
+</item>
+
+<item>
+Duckweed
+</item>
+
+<item>
+Dugong, affinities of
+</item>
+
+<item>
+Dung-beetles with deficient tarsi
+</item>
+
+<item>
+Dyticus
+</item>
+
+<item>
+Earl, Mr., W., on the Malay Archipelago
+</item>
+
+<item>
+Ears, drooping, in domestic animals; rudimentary
+</item>
+
+<item>
+Earth, seeds in roots of trees
+</item>
+
+<item>
+Eciton
+</item>
+
+<item>
+Edentata, teeth and hair; fossil species of
+</item>
+
+<item>
+Edwards, Milne, on physiological divisions of labour; on gradations of
+structure; on embryological characters
+</item>
+
+<item>
+Eggs, young birds escaping from
+</item>
+
+<item>
+Electric organs
+</item>
+
+<item>
+Elephant, rate increase; of glacial period
+</item>
+
+<item>
+Existence, struggle for; conditions of
+</item>
+
+<item>
+Eye, structure of; correction for aberration
+</item>
+
+<item>
+Eyes reduced in moles
+</item>
+
+<item>
+Fabre, M., on parasitic sphex
+</item>
+
+<item>
+Falconer, Dr., on naturalisation of plants in India; on fossil crocodile;
+on elephants and mastodons; and Cautley on mammals of sub-Himalayan beds
+</item>
+
+<item>
+Falkland Island, wolf of
+</item>
+
+<item>
+Faults
+</item>
+
+<item>
+Fear, instinctive, in birds
+</item>
+
+<item>
+Feet of birds, young molluscs adhering to
+</item>
+
+<item>
+Fertility of hybrids, from slight changes in conditions; of crossed
+varieties
+</item>
+
+<item>
+Fir-trees destroyed by cattle; pollen of
+</item>
+
+<item>
+Fish, flying; teleostean, sudden appearance of; eating seeds; fresh-water,
+distribution of
+</item>
+
+<item>
+Fishes, ganoid, now confined to fresh water; electric organs of; of
+southern hemisphere
+</item>
+
+<item>
+Flight, powers of, how acquired
+</item>
+
+<item>
+Flowers, structure of; in relation to crossing; of composite and
+umbelliferae
+</item>
+
+<item>
+Forbes, E., on colours of shells; on abrupt range of shells in depth; on
+poorness of palaeontological collections; on continuous succession of
+genera; on continental extensions; on distribution during glacial period;
+on parallelism in time and space
+</item>
+
+<item>
+Forests, changes in, in America
+</item>
+
+<item>
+Formation, Devonian
+</item>
+
+<item>
+Formations, thickness of, in Britain; intermittent
+</item>
+
+<item>
+Formica rufescens; sanguinea; flava, neuter of
+</item>
+
+<item>
+Frena, ovigerous, of cirripedes
+</item>
+
+<item>
+Fresh-water productions, dispersal of
+</item>
+
+<item>
+Fries on species in large genera being closely allied to other species
+</item>
+
+<item>
+Frigate-bird
+</item>
+
+<item>
+Frogs on islands
+</item>
+
+<item>
+Fruit-trees, gradual improvement of; in United States; varieties of,
+acclimatised in United States
+</item>
+
+<item>
+Fuci, crossed
+</item>
+
+<item>
+Fur, thicker in cold climates
+</item>
+
+<item>
+Furze
+</item>
+
+<item>
+Galapagos Archipelago, birds of; productions of
+</item>
+
+<item>
+Galeopithecus
+</item>
+
+<item>
+Game, increase of, checked by vermin
+</item>
+
+<item>
+Gartner on sterility of hybrids; on reciprocal crosses; on crossed maize
+and verbascum; on comparison of hybrids and mongrels
+</item>
+
+<item>
+Geese, fertility when crossed; upland
+</item>
+
+<item>
+Genealogy important in classification
+</item>
+
+<item>
+Geoffroy St. Hilaire on balancement; on homologous organs; Isidore, on
+variability of repeated parts; on correlation in monstrosities; on
+correlation; on variable parts being often monstrous
+</item>
+
+<item>
+Geographical distribution
+</item>
+
+<item>
+Geology, future progress of
+</item>
+
+<item>
+Giraffe, tail of
+</item>
+
+<item>
+Glacial period
+</item>
+
+<item>
+Gmelin on Distribution
+</item>
+
+<item>
+Gnathodon, fossil
+</item>
+
+<item>
+Godwin-Austin, Mr., on the Malay Archipelago
+</item>
+
+<item>
+Goethe on compensation of growth
+</item>
+
+<item>
+Gooseberry, grafts of
+</item>
+
+<item>
+Gould, Dr. A., on land-shells; Mr., on colours of birds; on birds of the
+Galapagos; on distribution of genera of birds
+</item>
+
+<item>
+Gourds, crossed
+</item>
+
+<item>
+Grafts, capacity of
+</item>
+
+<item>
+Grasses, varieties of
+</item>
+
+<item>
+Gray, Dr. Asa, on trees of United States; on naturalised plants in the
+United States; on rarity of intermediate varieties; on Alpine plants; Dr.
+J.E., on striped mule
+</item>
+
+<item>
+Grebe
+</item>
+
+<item>
+Grouse, colours of; red, a doubtful species
+</item>
+
+<item>
+Growth, compensation of; correlation of, in domestic products; correlation
+of
+</item>
+
+<item>
+Habit, effect of, under domestication; effect of, under nature;
+diversified, of same species
+</item>
+
+<item>
+Hair and teeth, correlated
+</item>
+
+<item>
+Harcourt, Mr. E.V., on the birds of Madeira
+</item>
+
+<item>
+Hartung, M., on boulders in the Azores
+</item>
+
+<item>
+Hazel-nuts
+</item>
+
+<item>
+Hearne on habits of bears
+</item>
+
+<item>
+Heath, changes in vegetation
+</item>
+
+<item>
+Heer, O., on plants of Madeira
+</item>
+
+<item>
+Helix pomatia
+</item>
+
+<item>
+Helosciadium
+</item>
+
+<item>
+Hemionus, striped
+</item>
+
+<item>
+Herbert, W., on struggle for existence; on sterility of hybrids
+</item>
+
+<item>
+Hermaphrodites crossing
+</item>
+
+<item>
+Heron eating seed
+</item>
+
+<item>
+Heron, Sir R., on peacocks
+</item>
+
+<item>
+Heusinger, on white animals not poisoned by certain plants
+</item>
+
+<item>
+Hewitt, Mr., on sterility of first crosses
+</item>
+
+<item>
+Himalaya, glaciers of; plants of
+</item>
+
+<item>
+Hippeastrum
+</item>
+
+<item>
+Holly-trees, sexes of
+</item>
+
+<item>
+Hollyhock, varieties of, crossed
+</item>
+
+<item>
+Hooker, Dr., on trees of New Zealand
+</item>
+
+<item>
+Hooker, Dr., on acclimatisation of Himalayan trees; on flowers of
+umbelliferae; on glaciers of Himalaya; on algae of New Zealand; on
+vegetation at the base of the Himalaya; on plants of Tierra del Fuego; on
+Australian plants; on relations of flora of South America; on flora of the
+Antarctic lands; on the plants of the Galapagos
+</item>
+
+<item>
+Hooks on bamboos; to seeds on islands
+</item>
+
+<item>
+Horner, Mr., on the antiquity of Egyptians
+</item>
+
+<item>
+Horns, rudimentary
+</item>
+
+<item>
+Horse, fossil, in La Plata
+</item>
+
+<item>
+Horses destroyed by flies in La Plata; striped; proportions of, when young
+</item>
+
+<item>
+Horticulturists, selection applied by
+</item>
+
+<item>
+Huber on cells of bees; P., on reason blended with instinct; on habitual
+nature of instincts; on slave-making ants; on Melipona domestica
+</item>
+
+<item>
+Humble-bees; cells of
+</item>
+
+<item>
+Hunter, J., on secondary sexual characters
+</item>
+
+<item>
+Hutton, Captain, on crossed geese
+</item>
+
+<item>
+Huxley, Prof., on structure of hermaphrodites; on embryological succession;
+on homologous organs; on the development of aphis
+</item>
+
+<item>
+Hybrids and mongrels compared
+</item>
+
+<item>
+Hybridism
+</item>
+
+<item>
+Hydra, structure of
+</item>
+
+<item>
+Ibla
+</item>
+
+<item>
+Icebergs transporting seeds
+</item>
+
+<item>
+Individuals, numbers favourable to selection; many, whether simultaneously
+created
+</item>
+
+<item>
+Insects, colour of, fitted for habitation; sea-side colours of; blind, in
+caves; luminous; neuter
+</item>
+
+<item>
+Instincts, domestic
+</item>
+
+<item>
+Intercrossing, advantages of
+</item>
+
+<item>
+Japan, productions of
+</item>
+
+<item>
+Java, plants of
+</item>
+
+<item>
+Jones, Mr. J.M., on the birds of Bermuda
+</item>
+
+<item>
+Jussieu on classification
+</item>
+
+<item>
+Kentucky, caves of
+</item>
+
+<item>
+Kerguelen-land, flora of
+</item>
+
+<item>
+Kidney-bean, acclimatisation of
+</item>
+
+<item>
+Kidneys of birds
+</item>
+
+<item>
+Kirby on tarsi deficient in beetles
+</item>
+
+<item>
+Knight, Andrew, on cause of variation
+</item>
+
+<item>
+Kolreuter on the barberry; on sterility of hybrids; on reciprocal crosses;
+on crossed varieties of nicotiana; on crossing male and hermaphrodite
+flowers
+</item>
+
+<item>
+Land-shells, distribution of, of Madeira, naturalised
+</item>
+
+<item>
+Languages, classification of
+</item>
+
+<item>
+Lapse, great, of time
+</item>
+
+<item>
+Larvae
+</item>
+
+<item>
+Laurel, nectar secreted by the leaves
+</item>
+
+<item>
+Laws of variation
+</item>
+
+<item>
+Leech, varieties of
+</item>
+
+<item>
+Leguminosae, nectar secreted by glands
+</item>
+
+<item>
+Lepidosiren
+</item>
+
+<item>
+Life, struggle for
+</item>
+
+<item>
+Lingula, Silurian
+</item>
+
+<item>
+Linnaeus, aphorism of
+</item>
+
+<item>
+Lion, mane of; young of, striped
+</item>
+
+<item>
+Lobelia fulgens
+</item>
+
+<item>
+Lobelia, sterility of crosses
+</item>
+
+<item>
+Loess of the Rhine
+</item>
+
+<item>
+Lowness of structure connected with variability
+</item>
+
+<item>
+Lowness, related to wide distribution
+</item>
+
+<item>
+Lubbock, Mr., on the nerves of coccus
+</item>
+
+<item>
+Lucas, Dr. P., on inheritance; on resemblance of child to parent
+</item>
+
+<item>
+Lund and Clausen on fossils of Brazil
+</item>
+
+<item>
+Lyell, Sir C., on the struggle for existence; on modern changes of the
+earth; on measure of denudation; on a carboniferous land-shell; on fossil
+whales; on strata beneath Silurian system; on the imperfection of the
+geological record; on the appearance of species; on Barrande's colonies; on
+tertiary formations of Europe and North America; on parallelism of tertiary
+formations; on transport of seeds by icebergs; on great alternations of
+climate; on the distribution of fresh-water shells; on land-shells of
+Madeira
+</item>
+
+<item>
+Lyell and Dawson on fossilized trees in Nova Scotia
+</item>
+
+<item>
+Macleay on analogical characters
+</item>
+
+<item>
+Madeira, plants of; beetles of, wingless; fossil land-shells of; birds of
+</item>
+
+<item>
+Magpie tame in Norway
+</item>
+
+<item>
+Maize, crossed
+</item>
+
+<item>
+Malay Archipelago compared with Europe; mammals of
+</item>
+
+<item>
+Malpighiaceae
+</item>
+
+<item>
+Mammae, rudimentary
+</item>
+
+<item>
+Mammals, fossil, in secondary formation; insular
+</item>
+
+<item>
+Man, origin of races of
+</item>
+
+<item>
+Manatee, rudimentary nails of
+</item>
+
+<item>
+Marsupials of Australia, fossil species of
+</item>
+
+<item>
+Martens, M., experiment on seeds
+</item>
+
+<item>
+Martin, Mr. W.C., on striped mules
+</item>
+
+<item>
+Matteuchi on the electric organs of rays
+</item>
+
+<item>
+Matthiola, reciprocal crosses of
+</item>
+
+<item>
+Means of dispersal
+</item>
+
+<item>
+Melipona domestica
+</item>
+
+<item>
+Metamorphism of oldest rocks
+</item>
+
+<item>
+Mice destroying bees; acclimatisation of
+</item>
+
+<item>
+Migration, bears on first appearance of fossils
+</item>
+
+<item>
+Miller, Prof., on the cells of bees
+</item>
+
+<item>
+Mirabilis, crosses of
+</item>
+
+<item>
+Missel-thrush
+</item>
+
+<item>
+Misseltoe, complex relations of
+</item>
+
+<item>
+Mississippi, rate of deposition at mouth
+</item>
+
+<item>
+Mocking-thrush of the Galapagos
+</item>
+
+<item>
+Modification of species, how far applicable
+</item>
+
+<item>
+Moles, blind
+</item>
+
+<item>
+Mongrels, fertility and sterility of; and hybrids compared
+</item>
+
+<item>
+Monkeys, fossil
+</item>
+
+<item>
+Monocanthus
+</item>
+
+<item>
+Mons, Van, on the origin of fruit-trees
+</item>
+
+<item>
+Mozart, musical powers of
+</item>
+
+<item>
+Mud, seeds in
+</item>
+
+<item>
+Mules, striped
+</item>
+
+<item>
+Muller, Dr. F., on Alpine Australian plants
+</item>
+
+<item>
+Murchison, Sir, R., on the formations of Russia; on azoic formations; on
+extinction
+</item>
+
+<item>
+Mustela vison
+</item>
+
+<item>
+Myanthus
+</item>
+
+<item>
+Myrmecocystus
+</item>
+
+<item>
+Myrmica, eyes of
+</item>
+
+<item>
+Nails, rudimentary
+</item>
+
+<item>
+Natural history, future progress of
+</item>
+
+<item>
+Naturalisation of forms distinct from the indigenous species; in New
+Zealand
+</item>
+
+<item>
+Nautilus, Silurian
+</item>
+
+<item>
+Nectar of plants
+</item>
+
+<item>
+Nectaries, how formed
+</item>
+
+<item>
+Nelumbium luteum
+</item>
+
+<item>
+Nests, variation in
+</item>
+
+<item>
+Neuter insects
+</item>
+
+<item>
+New Zealand, productions of, not perfect; naturalised products of; fossil
+birds of; glacial action in; crustaceans of; algae of; number of plants of;
+flora of
+</item>
+
+<item>
+Nicotiana, crossed varieties of; certain species very sterile
+</item>
+
+<item>
+Noble, Mr., on fertility of Rhododendron
+</item>
+
+<item>
+Nodules, phosphatic, in azoic rocks
+</item>
+
+<item>
+Oak, varieties of
+</item>
+
+<item>
+Onites apelles
+</item>
+
+<item>
+Orchis, pollen of
+</item>
+
+<item>
+Ornithorhynchus
+</item>
+
+<item>
+Ostrich not capable of flight; habit of laying eggs together; American, two
+species of
+</item>
+
+<item>
+Otter, habits of, how acquired
+</item>
+
+<item>
+Ouzel, water
+</item>
+
+<item>
+Owen, Prof., on birds not flying; on vegetative repetition; on variable
+length of arms in ourang-outang; on the swim-bladder of fishes; on electric
+organs; on fossil horse of La Plata; on relations of ruminants and
+pachyderms; on fossil birds of New Zealand; on succession of types; on
+affinities of the dugong; on homologous organs; on the metamorphosis of
+cephalopods and spiders
+</item>
+
+<item>
+Pacific Ocean, faunas of
+</item>
+
+<item>
+Paley, on no organ formed to give pain
+</item>
+
+<item>
+Pallas, on the fertility of the wild stocks of domestic animals
+</item>
+
+<item>
+Paraguay, cattle destroyed by flies
+</item>
+
+<item>
+Parasites
+</item>
+
+<item>
+Partridge, dirt on feet
+</item>
+
+<item>
+Parts greatly developed, variable, degrees of utility of
+</item>
+
+<item>
+Parus major
+</item>
+
+<item>
+Passiflora
+</item>
+
+<item>
+Peaches in United States
+</item>
+
+<item>
+Pear, grafts of
+</item>
+
+<item>
+Pelargonium, flowers of, sterility of
+</item>
+
+<item>
+Peloria
+</item>
+
+<item>
+Pelvis of women
+</item>
+
+<item>
+Period, glacial
+</item>
+
+<item>
+Petrels, habits of
+</item>
+
+<item>
+Phasianus, fertility of hybrids
+</item>
+
+<item>
+Pheasant, young, wild
+</item>
+
+<item>
+Pictet, Prof., on groups of species suddenly appearing; on rate of organic
+change; on continuous succession of genera; on close alliance of fossils in
+consecutive formations; on embryological succession
+</item>
+
+<item>
+Pierce, Mr., on varieties of wolves
+</item>
+
+<item>
+Pigeons with feathered feet and skin between toes; breeds described, and
+origin of; breeds of, how produced; tumbler, not being able to get out of
+egg; reverting to blue colour; instinct of tumbling; carriers, killed by
+hawks; young of
+</item>
+
+<item>
+Pistil, rudimentary
+</item>
+
+<item>
+Plants, poisonous, not affecting certain coloured animals; selection
+applied to; gradual improvement of; not improved in barbarous countries;
+destroyed by insects; in midst of range, have to struggle with other
+plants; nectar of; fleshy, on sea-shores; fresh-water, distribution of; low
+in scale; widely distributed
+</item>
+
+<item>
+Plumage, laws of change in sexes of birds
+</item>
+
+<item>
+Plums in the United States
+</item>
+
+<item>
+Pointer dog, origin of; habits of
+</item>
+
+<item>
+Poison not affecting certain coloured animals
+</item>
+
+<item>
+Poison, similar effect of, on animals and plants
+</item>
+
+<item>
+Pollen of fir-trees
+</item>
+
+<item>
+Poole, Col., on striped hemionus
+</item>
+
+<item>
+Prestwich, Mr., on English and French eocene formation
+</item>
+
+<item>
+Primrose; sterility of
+</item>
+
+<item>
+Primula, varieties of
+</item>
+
+<item>
+Proteolepas
+</item>
+
+<item>
+Proteus
+</item>
+
+<item>
+Psychology, future progress of
+</item>
+
+<item>
+Quagga, striped
+</item>
+
+<item>
+Quince, grafts of
+</item>
+
+<item>
+Rabbit, disposition of young
+</item>
+
+<item>
+Race-horses, Arab; English
+</item>
+
+<item>
+Races, domestic, characters of
+</item>
+
+<item>
+Ramond on plants of Pyrenees
+</item>
+
+<item>
+Ramsay, Prof., on thickness of the British formations; on faults
+</item>
+
+<item>
+Ratio of increase
+</item>
+
+<item>
+Rats, supplanting each other; acclimatisation of; blind in cave
+</item>
+
+<item>
+Rattle-snake
+</item>
+
+<item>
+Reason and instinct
+</item>
+
+<item>
+Recapitulation, general
+</item>
+
+<item>
+Reciprocity of crosses
+</item>
+
+<item>
+Record, geological, imperfect
+</item>
+
+<item>
+Rengger on flies destroying cattle
+</item>
+
+<item>
+Resemblance to parents in mongrels and hybrids
+</item>
+
+<item>
+Reversion, law of inheritance
+</item>
+
+<item>
+Rhododendron, sterility of
+</item>
+
+<item>
+Richard, Prof., on Aspicarpa
+</item>
+
+<item>
+Richardson, Sir J., on structure of squirrels; on fishes of the southern
+hemisphere
+</item>
+
+<item>
+Robinia, grafts of
+</item>
+
+<item>
+Rodents, blind
+</item>
+
+<item>
+Rudimentary organs
+</item>
+
+<item>
+Rudiments important for classification
+</item>
+
+<item>
+Sageret on grafts
+</item>
+
+<item>
+Salmons, males fighting, and hooked jaws of
+</item>
+
+<item>
+Salt-water, how far injurious to seeds
+</item>
+
+<item>
+Saurophagus sulphuratus
+</item>
+
+<item>
+Schiodte on blind insects
+</item>
+
+<item>
+Schlegel on snakes
+</item>
+
+<item>
+Sea-water, how far injurious to seeds
+</item>
+
+<item>
+Sebright, Sir J., on crossed animals; on selection of pigeons
+</item>
+
+<item>
+Sedgwick, Prof., on groups of species suddenly appearing
+</item>
+
+<item>
+Seedlings destroyed by insects
+</item>
+
+<item>
+Seeds, nutriment in; winged; power of resisting salt-water; in crops and
+intestines of birds; eaten by fish; in mud; hooked, on islands
+</item>
+
+<item>
+Selection of domestic products, principle not of recent origin;
+unconscious; natural; sexual; natural, circumstances favourable to
+</item>
+
+<item>
+Sexes, relations of
+</item>
+
+<item>
+Sexual characters variable; selection
+</item>
+
+<item>
+Sheep, Merino, their selection; two sub-breeds unintentionally produced;
+mountain, varieties of
+</item>
+
+<item>
+Shells, colours of, littoral, seldom embedded; fresh-water, dispersal of;
+of Madeira; land, distribution of
+</item>
+
+<item>
+Silene, fertility of crosses
+</item>
+
+<item>
+Silliman, Prof., on blind rat
+</item>
+
+<item>
+Skulls of young mammals
+</item>
+
+<item>
+Slave-making instinct
+</item>
+
+<item>
+Smith, Col. Hamilton, on striped horses; Mr. Fred., on slave-making ants;
+Mr., of Jordan Hill, on degradation of coast-rocks
+</item>
+
+<item>
+Snap-dragon
+</item>
+
+<item>
+Somerville, Lord, on selection of sheep
+</item>
+
+<item>
+Sorbus, grafts of
+</item>
+
+<item>
+Spaniel, King Charles's breed
+</item>
+
+<item>
+Species, groups of, suddenly appearing; beneath Silurian formations;
+successively appearing; changing simultaneously throughout the world
+</item>
+
+<item>
+Spencer, Lord, on increase in size of cattle
+</item>
+
+<item>
+Sphex, parasitic
+</item>
+
+<item>
+Spiders, development of
+</item>
+
+<item>
+Spitz-dog crossed with fox
+</item>
+
+<item>
+Sports in plants
+</item>
+
+<item>
+Sprengel, C.C., on crossing; on ray-florets
+</item>
+
+<item>
+Squirrels, gradations in structure
+</item>
+
+<item>
+Staffordshire, heath, changes in
+</item>
+
+<item>
+Stag-beetles, fighting
+</item>
+
+<item>
+Sterility from changed conditions of life; of hybrids; laws of; causes of;
+from unfavourable conditions; of certain varieties
+</item>
+
+<item>
+St. Helena, productions of
+</item>
+
+<item>
+St. Hilaire, Aug., on classification
+</item>
+
+<item>
+St. John, Mr., on habits of cats
+</item>
+
+<item>
+Sting of bee
+</item>
+
+<item>
+Stocks, aboriginal, of domestic animal
+</item>
+
+<item>
+Strata, thickness of, in Britain
+</item>
+
+<item>
+Stripes of horses
+</item>
+
+<item>
+Swallow, one species supplanting another
+</item>
+
+<item>
+Swim-bladder
+</item>
+
+<item>
+Tail of giraffe; of aquatic animals; rudimentary
+</item>
+
+<item>
+Tarsi deficient
+</item>
+
+<item>
+Tausch on umbelliferous flowers
+</item>
+
+<item>
+Teeth and hair correlated; embryonic, traces of, in birds; rudimentary, in
+embryonic calf
+</item>
+
+<item>
+Tegetmeier, Mr., on cells of bees
+</item>
+
+<item>
+Temminck on distribution aiding classification
+</item>
+
+<item>
+Thouin on grafts
+</item>
+
+<item>
+Thrush, aquatic species of; mocking, of the Galapagos; young of, spotted;
+nest of
+</item>
+
+<item>
+Thuret, M., on crossed fuci
+</item>
+
+<item>
+Thwaites, Mr., on acclimatisation
+</item>
+
+<item>
+Tierra del Fuego, dogs of; plants of
+</item>
+
+<item>
+Timber-drift
+</item>
+
+<item>
+Time, lapse of
+</item>
+
+<item>
+Titmouse
+</item>
+
+<item>
+Toads on islands
+</item>
+
+<item>
+Tobacco, crossed varieties of
+</item>
+
+<item>
+Tomes, Mr., on the distribution of bats
+</item>
+
+<item>
+Transitions in varieties
+</item>
+
+<item>
+Trees on islands belong to peculiar orders; with separated sexes
+</item>
+
+<item>
+Trifolium pratense; incarnatum
+</item>
+
+<item>
+Trigonia
+</item>
+
+<item>
+Trilobites; sudden extinction of
+</item>
+
+<item>
+Troglodytes
+</item>
+
+<item>
+Tucutucu, blind
+</item>
+
+<item>
+Tumbler pigeons, habits of, hereditary; young of
+</item>
+
+<item>
+Turkey-cock, brush of hair on breast
+</item>
+
+<item>
+Turkey, naked skin on head; young, wild
+</item>
+
+<item>
+Turnip and cabbage, analogous variations of
+</item>
+
+<item>
+Type, unity of
+</item>
+
+<item>
+Types, succession of, in same areas
+</item>
+
+<item>
+Udders enlarged by use; rudimentary
+</item>
+
+<item>
+Ulex, young leaves of
+</item>
+
+<item>
+Umbelliferae, outer and inner florets of
+</item>
+
+<item>
+Unity of type
+</item>
+
+<item>
+Use, effects of, under domestication
+</item>
+
+<item>
+Utility, how far important in the construction of each part
+</item>
+
+<item>
+Valenciennes on fresh-water fish
+</item>
+
+<item>
+Variability of mongrels and hybrids
+</item>
+
+<item>
+Variations appear at corresponding ages; analogous in distinct species
+</item>
+
+<item>
+Varieties, natural; struggle between; domestic, extinction of;
+transitional; when crossed, fertile; when crossed, sterile; classification
+of
+</item>
+
+<item>
+Verbascum, sterility of; varieties of, crossed
+</item>
+
+<item>
+Verneuil, M. de, on the succession of species
+</item>
+
+<item>
+Viola tricolor
+</item>
+
+<item>
+Volcanic islands, denudation of
+</item>
+
+<item>
+Vulture, naked skin on head
+</item>
+
+<item>
+Wading-birds
+</item>
+
+<item>
+Wallace, Mr., on origin of species; on law of geographical distribution; on
+the Malay Archipelago
+</item>
+
+<item>
+Wasp, sting of
+</item>
+
+<item>
+Water, fresh, productions of
+</item>
+
+<item>
+Water-hen
+</item>
+
+<item>
+Waterhouse, Mr., on Australian marsupials; on greatly developed parts being
+variable; on the cells of bees; on general affinities
+</item>
+
+<item>
+Water-ouzel
+</item>
+
+<item>
+Watson, Mr. H.C., on range of varieties of British plants; on
+acclimatisation; on flora of Azores; on Alpine plants; on rarity of
+intermediate varieties
+</item>
+
+<item>
+Weald, denudation of
+</item>
+
+<item>
+Web of feet in water-birds
+</item>
+
+<item>
+West Indian islands, mammals of
+</item>
+
+<item>
+Westwood on species in large genera being closely allied to others; on the
+tarsi of Engidae; on the antennae of hymenopterous insects
+</item>
+
+<item>
+Whales, fossil
+</item>
+
+<item>
+Wheat, varieties of
+</item>
+
+<item>
+White Mountains, flora of
+</item>
+
+<item>
+Wings, reduction of size
+</item>
+
+<item>
+Wings of insects homologous with branchiae; rudimentary, in insects
+</item>
+
+<item>
+Wolf crossed with dog; of Falkland Isles
+</item>
+
+<item>
+Wollaston, Mr., on varieties of insects; on fossil varieties of land-shells
+in Madeira; on colours of insects on sea-shore; on wingless beetles; on
+rarity of intermediate varieties; on insular insects; on land-shells of
+Madeira, naturalised
+</item>
+
+<item>
+Wolves, varieties of
+</item>
+
+<item>
+Woodpecker, habits of; green colour of
+</item>
+
+<item>
+Woodward, Mr., on the duration of specific forms; on the continuous
+succession of genera; on the succession of types
+</item>
+
+<item>
+World, species changing simultaneously throughout
+</item>
+
+<item>
+Wrens, nest of
+</item>
+
+<item>
+Youatt, Mr., on selection, on sub-breeds of sheep; on rudimentary horns in
+young cattle
+</item>
+
+<item>
+Zebra, stripes on 
+</item>
+
+</index>
+</backmatter>
+</book>
+<endgutblurb>
+End of this Project Gutenberg Etext of the First Edition of On the Origin
+of Species by Charles Darwin
+</endgutblurb>
+</gutbook>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/owlcreek.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/owlcreek.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/import/owlcreek.xml	(revision 21112)
@@ -0,0 +1,869 @@
+<?xml version="1.0"?>
+<!DOCTYPE gutbook SYSTEM "gutbook1.dtd">
+
+<?xml-stylesheet type="text/css" href="darwin.css"?>
+
+<gutbook>
+<gutblurb>
+<para>
+The Project Gutenberg Etext of An Occurrence at Owl Creek Bridge
+</para>
+
+<para>
+[This was the #1 rated Twilight Zone epidose of all time]
+#2 in our Ambrose Bierce series
+</para>
+
+<para>
+Copyright laws are changing all over the world, be sure to check
+the copyright laws for your country before posting these files!!
+</para>
+
+<para>
+Please take a look at the important information in this header.
+We encourage you to keep this file on your own disk, keeping an
+electronic path open for the next readers.  Do not remove this.
+</para>
+
+<para>
+**Welcome To The World of Free Plain Vanilla Electronic Texts**
+</para>
+
+<para>
+**Etexts Readable By Both Humans and By Computers, Since 1971**
+</para>
+
+<para>
+*These Etexts Prepared By Hundreds of Volunteers and Donations*
+</para>
+
+<para>
+Information on contacting Project Gutenberg to get Etexts, and
+further information is included below.  We need your donations.
+</para>
+
+<para>
+An Occurrence At Owl Creek Bridge
+</para>
+
+<para>
+by Ambrose Bierce
+</para>
+
+<para>
+December, 1995  [Etext #375]
+</para>
+
+<para>
+The Project Gutenberg Etext of An Occurrence at Owl Creek Bridge
+*****This file should be named owlcr10.txt or owlcr10.zip******
+</para>
+
+<para>
+Corrected EDITIONS of our etexts get a new NUMBER, owlcr11.txt.
+VERSIONS based on separate sources get new LETTER, owlcr10a.txt.
+</para>
+
+<para>
+We are now trying to release all our books one month in advance
+of the official release dates, for time for better editing.
+</para>
+
+<para>
+Please note:  neither this list nor its contents are final till
+midnight of the last day of the month of any such announcement.
+The official release date of all Project Gutenberg Etexts is at
+Midnight, Central Time, of the last day of the stated month.  A
+preliminary version may often be posted for suggestion, comment
+and editing by those who wish to do so.  To be sure you have an
+up to date first edition [xxxxx10x.xxx] please check file sizes
+in the first week of the next month.  Since our ftp program has
+a bug in it that scrambles the date [tried to fix and failed] a
+look at the file size will have to do, but we will try to see a
+new copy has at least one byte more or less.
+</para>
+
+<para>
+Information about Project Gutenberg (one page)
+</para>
+
+<para>
+We produce about two million dollars for each hour we work.  The
+fifty hours is one conservative estimate for how long it we take
+to get any etext selected, entered, proofread, edited, copyright
+searched and analyzed, the copyright letters written, etc.  This
+projected audience is one hundred million readers.  If our value
+per text is nominally estimated at one dollar then we produce $4
+million dollars per hour this year as we release some eight text
+files per month:  thus upping our productivity from $2 million.
+</para>
+
+<para>
+The Goal of Project Gutenberg is to Give Away One Trillion Etext
+Files by the December 31, 2001.  [10,000 x 100,000,000=Trillion]
+This is ten thousand titles each to one hundred million readers,
+which is 10% of the expected number of computer users by the end
+of the year 2001.
+</para>
+
+<para>
+We need your donations more than ever!
+</para>
+
+<para>
+All donations should be made to "Project Gutenberg/IBC", and are
+tax deductible to the extent allowable by law ("IBC" is Illinois
+Benedictine College).  (Subscriptions to our paper newsletter go
+to IBC, too)
+</para>
+
+<para>
+For these and other matters, please mail to:
+</para>
+
+<para>
+Project Gutenberg
+P. O. Box  2782
+Champaign, IL 61825
+</para>
+
+<para>
+When all other email fails try our Michael S. Hart, Executive
+Director:
+hart@vmd.cso.uiuc.edu (internet)   hart@uiucvmd   (bitnet)
+</para>
+
+<para>
+We would prefer to send you this information by email
+(Internet, Bitnet, Compuserve, ATTMAIL or MCImail).
+</para>
+
+<para>
+******
+If you have an FTP program (or emulator), please
+FTP directly to the Project Gutenberg archives:
+[Mac users, do NOT point and click. . .type]
+</para>
+
+<para>
+ftp uiarchive.cso.uiuc.edu
+login:  anonymous
+password:  your@login
+cd etext/etext90 through /etext96
+or cd etext/articles [get suggest gut for more information]
+dir [to see files]
+get or mget [to get files. . .set bin for zip files]
+GET INDEX?00.GUT
+for a list of books
+and
+GET NEW GUT for general information
+and
+MGET GUT* for newsletters.
+</para>
+
+<para>
+**Information prepared by the Project Gutenberg legal advisor**
+(Three Pages)
+</para>
+
+<para>
+***START**THE SMALL PRINT!**FOR PUBLIC DOMAIN ETEXTS**START***
+Why is this "Small Print!" statement here?  You know: lawyers.
+They tell us you might sue us if there is something wrong with
+your copy of this etext, even if you got it for free from
+someone other than us, and even if what's wrong is not our
+fault.  So, among other things, this "Small Print!" statement
+disclaims most of our liability to you.  It also tells you how
+you can distribute copies of this etext if you want to.
+</para>
+
+<para>
+*BEFORE!* YOU USE OR READ THIS ETEXT
+By using or reading any part of this PROJECT GUTENBERG-tm
+etext, you indicate that you understand, agree to and accept
+this "Small Print!" statement.  If you do not, you can receive
+a refund of the money (if any) you paid for this etext by
+sending a request within 30 days of receiving it to the person
+you got it from.  If you received this etext on a physical
+medium (such as a disk), you must return it with your request.
+</para>
+
+<para>
+ABOUT PROJECT GUTENBERG-TM ETEXTS
+This PROJECT GUTENBERG-tm etext, like most PROJECT GUTENBERG-
+tm etexts, is a "public domain" work distributed by Professor
+Michael S. Hart through the Project Gutenberg Association at
+Illinois Benedictine College (the "Project").  Among other
+things, this means that no one owns a United States copyright
+on or for this work, so the Project (and you!) can copy and
+distribute it in the United States without permission and
+without paying copyright royalties.  Special rules, set forth
+below, apply if you wish to copy and distribute this etext
+under the Project's "PROJECT GUTENBERG" trademark.
+</para>
+
+<para>
+To create these etexts, the Project expends considerable
+efforts to identify, transcribe and proofread public domain
+works.  Despite these efforts, the Project's etexts and any
+medium they may be on may contain "Defects".  Among other
+things, Defects may take the form of incomplete, inaccurate or
+corrupt data, transcription errors, a copyright or other
+intellectual property infringement, a defective or damaged
+disk or other etext medium, a computer virus, or computer
+codes that damage or cannot be read by your equipment.
+</para>
+
+<para>
+LIMITED WARRANTY; DISCLAIMER OF DAMAGES
+But for the "Right of Replacement or Refund" described below,
+[1] the Project (and any other party you may receive this
+etext from as a PROJECT GUTENBERG-tm etext) disclaims all
+liability to you for damages, costs and expenses, including
+legal fees, and [2] YOU HAVE NO REMEDIES FOR NEGLIGENCE OR
+UNDER STRICT LIABILITY, OR FOR BREACH OF WARRANTY OR CONTRACT,
+INCLUDING BUT NOT LIMITED TO INDIRECT, CONSEQUENTIAL, PUNITIVE
+OR INCIDENTAL DAMAGES, EVEN IF YOU GIVE NOTICE OF THE
+POSSIBILITY OF SUCH DAMAGES.
+</para>
+
+<para>
+If you discover a Defect in this etext within 90 days of
+receiving it, you can receive a refund of the money (if any)
+you paid for it by sending an explanatory note within that
+time to the person you received it from.  If you received it
+on a physical medium, you must return it with your note, and
+such person may choose to alternatively give you a replacement
+copy.  If you received it electronically, such person may
+choose to alternatively give you a second opportunity to
+receive it electronically.
+</para>
+
+<para>
+THIS ETEXT IS OTHERWISE PROVIDED TO YOU "AS-IS".  NO OTHER
+WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, ARE MADE TO YOU AS
+TO THE ETEXT OR ANY MEDIUM IT MAY BE ON, INCLUDING BUT NOT
+LIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
+PARTICULAR PURPOSE.
+</para>
+
+<para>
+Some states do not allow disclaimers of implied warranties or
+the exclusion or limitation of consequential damages, so the
+above disclaimers and exclusions may not apply to you, and you
+may have other legal rights.
+</para>
+
+<para>
+INDEMNITY
+You will indemnify and hold the Project, its directors,
+officers, members and agents harmless from all liability, cost
+and expense, including legal fees, that arise directly or
+indirectly from any of the following that you do or cause:
+[1] distribution of this etext, [2] alteration, modification,
+or addition to the etext, or [3] any Defect.
+</para>
+
+<para>
+DISTRIBUTION UNDER "PROJECT GUTENBERG-tm"
+You may distribute copies of this etext electronically, or by
+disk, book or any other medium if you either delete this
+"Small Print!" and all other references to Project Gutenberg,
+or:
+</para>
+
+<para>
+[1]  Only give exact copies of it.  Among other things, this
+     requires that you do not remove, alter or modify the
+     etext or this "small print!" statement.  You may however,
+     if you wish, distribute this etext in machine readable
+     binary, compressed, mark-up, or proprietary form,
+     including any form resulting from conversion by word pro-
+     cessing or hypertext software, but only so long as
+     *EITHER*:
+</para>
+
+<para>
+     [*]  The etext, when displayed, is clearly readable, and
+          does *not* contain characters other than those
+          intended by the author of the work, although tilde
+          (~), asterisk (*) and underline (_) characters may
+          be used to convey punctuation intended by the
+          author, and additional characters may be used to
+          indicate hypertext links; OR
+</para>
+
+<para>
+     [*]  The etext may be readily converted by the reader at
+          no expense into plain ASCII, EBCDIC or equivalent
+          form by the program that displays the etext (as is
+          the case, for instance, with most word processors);
+          OR
+</para>
+
+<para>
+     [*]  You provide, or agree to also provide on request at
+          no additional cost, fee or expense, a copy of the
+          etext in its original plain ASCII form (or in EBCDIC
+          or other equivalent proprietary form).
+</para>
+
+<para>
+[2]  Honor the etext refund and replacement provisions of this
+     "Small Print!" statement.
+</para>
+
+<para>
+[3]  Pay a trademark license fee to the Project of 20% of the
+     net profits you derive calculated using the method you
+     already use to calculate your applicable taxes.  If you
+     don't derive profits, no royalty is due.  Royalties are
+     payable to "Project Gutenberg Association / Illinois
+     Benedictine College" within the 60 days following each
+     date you prepare (or were legally required to prepare)
+     your annual (or equivalent periodic) tax return.
+</para>
+
+<para>
+WHAT IF YOU *WANT* TO SEND MONEY EVEN IF YOU DON'T HAVE TO?
+The Project gratefully accepts contributions in money, time,
+scanning machines, OCR software, public domain etexts, royalty
+free copyright licenses, and every other sort of contribution
+you can think of.  Money should be paid to "Project Gutenberg
+Association / Illinois Benedictine College".
+</para>
+
+<para>
+*END*THE SMALL PRINT! FOR PUBLIC DOMAIN ETEXTS*Ver.04.29.93*END*
+</para>
+</gutblurb>
+
+<markupblurb>
+Marked up by John Hanna Feb 2000
+Validated against gutbook1.dtd using MSXML Feb 2000
+Checked FB Feb 2000
+</markupblurb>
+
+<book>
+<acknowledge>A project of the HTML Writrs Guild and Project Gutenberg. Markup by an anonymous Project Gutenberg Volunteer, Markup by John Hanna Feb 2000</acknowledge>
+<frontmatter>
+<titlepage>
+<title>AN OCCURRENCE AT OWL CREEK BRIDGE</title>
+<para>by</para>
+<author>Ambrose Bierce</author>
+<para>
+THE MILLENNIUM FULCRUM EDITION, 1988
+</para>
+</titlepage>
+</frontmatter>
+<bookbody>
+<part>
+
+<chapter>
+<title>I</title>
+<para>
+A man stood upon a railroad bridge in northern Alabama,
+looking down into the swift water twenty feet below.  The
+man's hands were behind his back, the wrists bound with a
+cord.  A rope closely encircled his neck.  It was attached to
+a stout cross-timber above his head and the slack feel to the
+level of his knees.  Some loose boards laid upon the ties
+supporting the rails of the railway supplied a footing for
+him and his executioners -- two private soldiers of the
+Federal army, directed by a sergeant who in civil life may
+have been a deputy sheriff.  At a short remove upon the same
+temporary platform was an officer in the uniform of his rank,
+armed.  He was a captain.  A sentinel at each end of the
+bridge stood with his rifle in the position known as
+"support," that is to say, vertical in front of the left
+shoulder, the hammer resting on the forearm thrown straight
+across the chest -- a formal and unnatural position,
+enforcing an erect carriage of the body.  It did not appear
+to be the duty of these two men to know what was occurring at
+the center of the bridge; they merely blockaded the two ends
+of the foot planking that traversed it.
+</para>
+
+<para>
+Beyond one of the sentinels nobody was in sight; the railroad
+ran straight away into a forest for a hundred yards, then,
+curving, was lost to view.  Doubtless there was an outpost
+farther along.  The other bank of the stream was open ground
+-- a gentle slope topped with a stockade of vertical tree
+trunks, loopholed for rifles, with a single embrasure
+through which protruded the muzzle of a brass cannon
+commanding the bridge.  Midway up the slope between the
+bridge and fort were the spectators -- a single company of
+infantry in line, at "parade rest," the butts of their rifles
+on the ground, the barrels inclining slightly backward
+against the right shoulder, the hands crossed upon the stock.
+A lieutenant stood at the right of the line, the point
+of his sword upon the ground, his left hand resting upon his
+right.  Excepting the group of four at the center of the
+bridge, not a man moved.  The company faced the bridge,
+staring stonily, motionless.  The sentinels, facing the
+banks of the stream, might have been statues to adorn the
+bridge.  The captain stood with folded arms, silent,
+observing the work of his subordinates, but making no sign.
+Death is a dignitary who when he comes announced is to be
+received with formal manifestations of respect, even by those
+most familiar with him.  In the code of military etiquette
+silence and fixity are forms of deference.
+</para>
+
+<para>
+The man who was engaged in being hanged was apparently about
+thirty-five years of age.  He was a civilian, if one might
+judge from his habit, which was that of a planter.  His
+features were good -- a straight nose, firm mouth, broad
+forehead, from which his long, dark hair was combed straight
+back, falling behind his ears to the collar of his well
+fitting frock coat.  He wore a moustache and pointed beard,
+but no whiskers; his eyes were large and dark gray, and had a
+kindly expression which one would hardly have expected in one
+whose neck was in the hemp.  Evidently this was no vulgar
+assassin.  The liberal military code makes provision for
+hanging many kinds of persons, and gentlemen are not
+excluded.
+</para>
+
+<para>
+The preparations being complete, the two private soldiers
+stepped aside and each drew away the plank upon which he had
+been standing.  The sergeant turned to the captain, saluted
+and placed himself immediately behind that officer, who in
+turn moved apart one pace.  These movements left the
+condemned man and the sergeant standing on the two ends of
+the same plank, which spanned three of the cross-ties of the
+bridge.  The end upon which the civilian stood almost, but
+not quite, reached a fourth.  This plank had been held in
+place by the weight of the captain; it was now held by that
+of the sergeant.  At a signal from the former the latter
+would step aside, the plank would tilt and the condemned man
+go down between two ties.  The arrangement commended itself
+to his judgement as simple and effective.  His face had not
+been covered nor his eyes bandaged.  He looked a moment at
+his "unsteadfast footing," then let his gaze wander to the
+swirling water of the stream racing madly beneath his feet.
+A piece of dancing driftwood caught his attention and his
+eyes followed it down the current.  How slowly it appeared
+to move!  What a sluggish stream!
+</para>
+
+<para>
+He closed his eyes in order to fix his last thoughts upon his
+wife and children.  The water, touched to gold by the early
+sun, the brooding mists under the banks at some distance down
+the stream, the fort, the soldiers, the piece of drift -- all
+had distracted him.  And now he became conscious of a new
+disturbance.  Striking through the thought of his dear
+ones was sound which he could neither ignore nor understand,
+a sharp, distinct, metallic percussion like the stroke of a
+blacksmith's hammer upon the anvil; it had the same ringing
+quality.  He wondered what it was, and whether immeasurably
+distant or near by -- it seemed both.  Its recurrence was
+regular, but as slow as the tolling of a death knell.  He
+awaited each new stroke with impatience and -- he knew not
+why -- apprehension.  The intervals of silence grew
+progressively longer; the delays became maddening.  With
+their greater infrequency the sounds increased in strength
+and sharpness.  They hurt his ear like the trust of a knife;
+he feared he would shriek.  What he heard was the ticking of
+his watch.
+</para>
+
+<para>
+He unclosed his eyes and saw again the water below him.  "If
+I could free my hands," he thought, "I might throw off the
+noose and spring into the stream.  By diving I could evade
+the bullets and, swimming vigorously, reach the bank, take
+to the woods and get away home.  My home, thank God, is as
+yet outside their lines; my wife and little ones are still
+beyond the invader's farthest advance."
+</para>
+
+<para>
+As these thoughts, which have here to be set down in words,
+were flashed into the doomed man's brain rather than evolved
+from it the captain nodded to the sergeant.  The sergeant
+stepped aside.
+</para>
+</chapter>
+
+<chapter>
+<title>II</title>
+<para>
+Peyton Fahrquhar was a well to do planter, of an old and
+highly respected Alabama family.  Being a slave owner and
+like other slave owners a politician, he was naturally an
+original secessionist and ardently devoted to the Southern
+cause.  Circumstances of an imperious nature, which it is
+unnecessary to relate here, had prevented him from taking
+service with that gallant army which had fought the
+disastrous campaigns ending with the fall of Corinth, and he
+chafed under the inglorious restraint, longing for the
+release of his energies, the larger life of the soldier, the
+opportunity for distinction.  That opportunity, he felt,
+would come, as it comes to all in wartime.  Meanwhile he
+did what he could.  No service was too humble for him to
+perform in the aid of the South, no adventure to perilous for
+him to undertake if consistent with the character of a
+civilian who was at heart a soldier, and who in good faith
+and without too much qualification assented to at least a
+part of the frankly villainous dictum that all is fair in
+love and war.
+</para>
+
+<para>
+One evening while Fahrquhar and his wife were sitting on a
+rustic bench near the entrance to his grounds, a gray-clad
+soldier rode up to the gate and asked for a drink of water.
+Mrs. Fahrquhar was only too happy to serve him with her own
+white hands.  While she was fetching the water her husband
+approached the dusty horseman and inquired eagerly for news
+from the front.
+</para>
+
+<para>
+"The Yanks are repairing the railroads," said the man, "and
+are getting ready for another advance.  They have reached the
+Owl Creek bridge, put it in order and built a stockade on the
+north bank.  The commandant has issued an order, which is
+posted everywhere, declaring that any civilian caught
+interfering with the railroad, its bridges, tunnels, or
+trains will be summarily hanged.  I saw the order."
+</para>
+
+<para>
+"How far is it to the Owl Creek bridge?" Fahrquhar asked.
+</para>
+
+<para>
+"About thirty miles."
+</para>
+
+<para>
+"Is there no force on this side of the creek?"
+</para>
+
+<para>
+"Only a picket post half a mile out, on the railroad, and a
+single sentinel at this end of the bridge."
+</para>
+
+<para>
+"Suppose a man -- a civilian and student of hanging --
+should elude the picket post and perhaps get the better of
+the sentinel," said Fahrquhar, smiling, "what could he
+accomplish?"
+</para>
+
+<para>
+The soldier reflected.  "I was there a month ago," he
+replied.  "I observed that the flood of last winter had
+lodged a great quantity of driftwood against the wooden pier
+at this end of the bridge.  It is now dry and would burn like
+tinder."
+</para>
+
+<para>
+The lady had now brought the water, which the soldier drank.
+He thanked her ceremoniously, bowed to her husband and rode
+away.  An hour later, after nightfall, he repassed the
+plantation, going northward in the direction from which he
+had come.  He was a Federal scout.
+</para>
+</chapter>
+
+<chapter>
+<title>III</title>
+<para>
+ As Peyton Fahrquhar fell straight downward through the
+bridge he lost consciousness and was as one already dead.
+From this state he was awakened -- ages later, it seemed to
+him -- by the pain of a sharp pressure upon his throat,
+followed by a sense of suffocation.  Keen, poignant agonies
+seemed to shoot from his neck downward through every fiber of
+his body and limbs.  These pains appeared to flash along well
+defined lines of ramification and to beat with an
+inconceivably rapid periodicity.  They seemed like streams of
+pulsating fire heating him to an intolerable temperature.  As
+to his head, he was conscious of nothing but a feeling of
+fullness -- of congestion.  These sensations were
+unaccompanied by thought.  The intellectual part of his
+nature was already effaced; he had power only to feel, and
+feeling was torment.  He was conscious of motion.
+Encompassed in a luminous cloud, of which he was now merely
+the fiery heart, without material substance, he swung
+through unthinkable arcs of oscillation, like a vast
+pendulum.  Then all at once, with terrible suddenness, the
+light about him shot upward with the noise of a loud splash;
+a frightful roaring was in his ears, and all was cold and
+dark.  The power of thought was restored; he knew that the
+rope had broken and he had fallen into the stream.  There was
+no additional strangulation; the noose about his neck
+was already suffocating him and kept the water from his
+lungs.  To die of hanging at the bottom of a river! -- the
+idea seemed to him ludicrous.  He opened his eyes in the
+darkness and saw above him a gleam of light, but how distant,
+how inaccessible!  He was still sinking, for the light became
+fainter and fainter until it was a mere glimmer.  Then it
+began to grow and brighten, and he knew that he was rising
+toward the surface -- knew it with reluctance, for he was now
+very comfortable.  "To be hanged and drowned," he thought,
+"that is not so bad; but I do not wish to be shot.  No; I
+will not be shot; that is not fair."
+</para>
+
+<para>
+He was not conscious of an effort, but a sharp pain in his
+wrist apprised him that he was trying to free his hands.  He
+gave the struggle his attention, as an idler might observe
+the feat of a juggler, without interest in the outcome.  What
+splendid effort! -- what magnificent, what superhuman
+strength!  Ah, that was a fine endeavor!  Bravo!  The cord
+fell away; his arms parted and floated upward, the hands
+dimly seen on each side in the growing light.  He watched
+them with a new interest as first one and then the other
+pounced upon the noose at his neck.  They tore it away and
+thrust it fiercely aside, its undulations resembling those of
+a water snake.  "Put it back, put it back!"  He thought he
+shouted these words to his hands, for the undoing of the
+noose had been succeeded by the direst pang that he had yet
+experienced.  His neck ached horribly; his brain was on fire,
+his heart, which had been fluttering faintly, gave a great
+leap, trying to force itself out at his mouth.  His whole
+body was racked and wrenched with an insupportable anguish!
+But his disobedient hands gave no heed to the command.  They
+beat the water vigorously with quick, downward strokes,
+forcing him to the surface.  He felt his head emerge; his
+eyes were blinded by the sunlight; his chest expanded
+convulsively, and with a supreme and crowning agony his lungs
+engulfed a great draught of air, which instantly he expelled
+in a shriek!
+</para>
+
+<para>
+He was now in full possession of his physical senses.  They
+were, indeed, preternaturally keen and alert.  Something in
+the awful disturbance of his organic system had so exalted
+and refined them that they made record of things never before
+perceived.  He felt the ripples upon his face and heard their
+separate sounds as they struck.  He looked at the forest on
+the bank of the stream, saw the individual trees, the leaves
+and the veining of each leaf -- he saw the very insects upon
+them:  the locusts, the brilliant bodied flies, the gray
+spiders stretching their webs from twig to twig.  He noted
+the prismatic colors in all the dewdrops upon a million
+blades of grass.  The humming of the gnats that danced above
+the eddies of the stream, the beating of the dragon flies'
+wings, the strokes of the water spiders' legs, like oars
+which had lifted their boat -- all these made audible
+music.  A fish slid along beneath his eyes and he heard the
+rush of its body parting the water.
+</para>
+
+<para>
+He had come to the surface facing down the stream; in a
+moment the visible world seemed to wheel slowly round,
+himself the pivotal point, and he saw the bridge, the fort,
+the soldiers upon the bridge, the captain, the sergeant, the
+two privates, his executioners.  They were in silhouette
+against the blue sky.  They shouted and gesticulated,
+pointing at him.  The captain had drawn his pistol, but did
+not fire; the others were unarmed.  Their movements were
+grotesque and horrible, their forms gigantic.
+</para>
+
+<para>
+Suddenly he heard a sharp report and something struck the
+water smartly within a few inches of his head, spattering his
+face with spray.  He heard a second report, and saw one of
+the sentinels with his rifle at his shoulder, a light cloud
+of blue smoke rising from the muzzle.  The man in the water
+saw the eye of the man on the bridge gazing into his own
+through the sights of the rifle.  He observed that it was a
+gray eye and remembered having read that gray eyes were
+keenest, and that all famous marksmen had them.
+Nevertheless, this one had missed.
+</para>
+
+<para>
+A counter-swirl had caught Fahrquhar and turned him half
+round; he was again looking at the forest on the bank
+opposite the fort.  The sound of a clear, high voice in a
+monotonous singsong now rang out behind him and came across
+the water with a distinctness that pierced and subdued all
+other sounds, even the beating of the ripples in his ears.
+Although no soldier, he had frequented camps enough to know
+the dread significance of that deliberate, drawling,
+aspirated chant; the lieutenant on shore was taking a part in
+the morning's work.  How coldly and pitilessly -- with what
+an even, calm intonation, presaging, and enforcing
+tranquility in the men -- with what accurately measured
+interval fell those cruel words:
+</para>
+
+<para>
+"Company! . . . Attention!  . . . Shoulder arms! . . . Ready!
+. . . Aim!  . . . Fire!"
+</para>
+
+<para>
+Fahrquhar dived -- dived as deeply as he could.  The water
+roared in his ears like the voice of Niagara, yet he heard
+the dull thunder of the volley and, rising again toward the
+surface, met shining bits of metal, singularly flattened,
+oscillating slowly downward.  Some of them touched him on the
+face and hands, then fell away, continuing their descent.
+One lodged between his collar and neck; it was uncomfortably
+warm and he snatched it out.
+</para>
+
+<para>
+As he rose to the surface, gasping for breath, he saw that he
+had been a long time under water; he was perceptibly farther
+downstream -- nearer to safety.  The soldiers had almost
+finished reloading; the metal ramrods flashed all at once in
+the sunshine as they were drawn from the barrels,
+turned in the air, and thrust into their sockets.  The two
+sentinels fired again, independently and ineffectually.
+</para>
+
+<para>
+The hunted man saw all this over his shoulder; he was now
+swimming vigorously with the current.  His brain was as
+energetic as his arms and legs; he thought with the rapidity
+of lightning:
+</para>
+
+<para>
+"The officer," he reasoned, "will not make that martinet's
+error a second time.  It is as easy to dodge a volley as a
+single shot.  He has probably already given the command to
+fire at will.  God help me, I cannot dodge them all!"
+</para>
+
+<para>
+An appalling splash within two yards of him was followed by a
+loud, rushing sound, DIMINUENDO, which seemed to travel back
+through the air to the fort and died in an explosion which
+stirred the very river to its deeps!  A rising sheet of water
+curved over him, fell down upon him, blinded him, strangled
+him!  The cannon had taken an hand in the game.  As he shook
+his head free from the commotion of the smitten water he
+heard the deflected shot humming through the air ahead, and
+in an instant it was cracking and smashing the branches in
+the forest beyond.
+</para>
+
+<para>
+"They will not do that again," he thought; "the next time
+they will use a charge of grape.  I must keep my eye upon
+the gun; the smoke will apprise me -- the report arrives too
+late; it lags behind the missile.  That is a good gun."
+</para>
+
+<para>
+Suddenly he felt himself whirled round and round -- spinning
+like a top.  The water, the banks, the forests, the now
+distant bridge, fort and men, all were commingled and
+blurred.  Objects were represented by their colors only;
+circular horizontal streaks of color -- that was all he saw.
+He had been caught in a vortex and was being whirled on with
+a velocity of advance and gyration that made him giddy and
+sick.  In few moments he was flung upon the gravel at the
+foot of the left bank of the stream -- the southern bank --
+and behind a projecting point which concealed him from his
+enemies.  The sudden arrest of his motion, the abrasion of
+one of his hands on the gravel, restored him, and he wept
+with delight.  He dug his fingers into the sand, threw it
+over himself in handfuls and audibly blessed it.  It looked
+like diamonds, rubies, emeralds; he could think of nothing
+beautiful which it did not resemble.  The trees upon the bank
+were giant garden plants; he noted a definite order in their
+arrangement, inhaled the fragrance of their blooms.  A
+strange roseate light shone through the spaces among their
+trunks and the wind made in their branches the music of
+AEolian harps.  He had not wish to perfect his escape -- he
+was content to remain in that enchanting spot until retaken.
+</para>
+
+<para>
+A whiz and a rattle of grapeshot among the branches high
+above his head roused him from his dream.  The baffled
+cannoneer had fired him a random farewell.  He sprang
+to his feet, rushed up the sloping bank, and plunged into the
+forest.
+</para>
+
+<para>
+All that day he traveled, laying his course by the rounding
+sun.  The forest seemed interminable; nowhere did he
+discover a break in it, not even a woodman's road.  He had
+not known that he lived in so wild a region.  There was
+something uncanny in the revelation.
+</para>
+
+<para>
+By nightfall he was fatigued, footsore, famished.  The
+thought of his wife and children urged him on.  At last he
+found a road which led him in what he knew to be the right
+direction.  It was as wide and straight as a city street, yet
+it seemed untraveled.  No fields bordered it, no dwelling
+anywhere.  Not so much as the barking of a dog suggested
+human habitation.  The black bodies of the trees formed a
+straight wall on both sides, terminating on the horizon in a
+point, like a diagram in a lesson in perspective.  Overhead,
+as he looked up through this rift in the wood, shone great
+golden stars looking unfamiliar and grouped in strange
+constellations.  He was sure they were arranged in some order
+which had a secret and malign significance.  The wood on
+either side was full of singular noises, among which -- once,
+twice, and again -- he distinctly heard whispers in an
+unknown tongue.
+</para>
+
+<para>
+His neck was in pain and lifting his hand to it found it
+horribly swollen.  He knew that it had a circle of black
+where the rope had bruised it.  His eyes felt congested; he
+could no longer close them.  His tongue was swollen with
+thirst; he relieved its fever by thrusting it forward from
+between his teeth into the cold air.  How softly the turf had
+carpeted the untraveled avenue -- he could no longer feel the
+roadway beneath his feet!
+</para>
+
+<para>
+Doubtless, despite his suffering, he had fallen asleep while
+walking, for now he sees another scene -- perhaps he has
+merely recovered from a delirium.  He stands at the gate of
+his own home.  All is as he left it, and all bright and
+beautiful in the morning sunshine.  He must have traveled the
+entire night.  As he pushes open the gate and passes up the
+wide white walk, he sees a flutter of female garments; his
+wife, looking fresh and cool and sweet, steps down from the
+veranda to meet him.  At the bottom of the steps she stands
+waiting, with a smile of ineffable joy, an attitude of
+matchless grace and dignity.  Ah, how beautiful she is!  He
+springs forwards with extended arms.  As he is about to clasp
+her he feels a stunning blow upon the back of the neck; a
+blinding white light blazes all about him with a sound like
+the shock of a cannon -- then all is darkness and silence!
+</para>
+
+<para>
+Peyton Fahrquhar was dead; his body, with a broken neck,
+swung gently from side to side beneath the timbers of the
+Owl Creek bridge.
+</para>
+
+<para>
+End of The Project Gutenberg Etext of An Occurrence at Owl Creek Bridge
+</para>
+</chapter>
+</part>
+</bookbody>
+</book>
+</gutbook>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,16 @@
+<buildConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" >
+  <metadataList>
+    <metadata name="numDocs">4</metadata>
+    <metadata name="buildType">lucene</metadata>
+  </metadataList>
+  <serviceRackList>
+    <serviceRack name="GutBookRetrieve">
+      <paramList>
+	<param name="tocXSLT" value="produce-toc"/>
+	<param name="documentEncoding" value="UTF-8"/>
+	<param name="documentRootTag" value="book"/>
+      </paramList>
+    </serviceRack>
+    <serviceRack name="LuceneSearch"/>
+  </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/BuildXMLColl.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/BuildXMLColl.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/BuildXMLColl.java	(revision 21112)
@@ -0,0 +1,59 @@
+/**
+ *
+ * class for building a simple  XML collection 
+ * @author  kjdon@cs.waikato.ac.nz
+ * @version 
+ */
+
+
+import org.greenstone.gsdl3.util.GSFile;
+
+import java.io.File;
+
+public class BuildXMLColl {
+
+    public static void main (String args[]) throws Exception { 
+	if (args.length != 2) {
+	    System.out.println("Usage: java BuildXMLColl site-home coll-name");
+	    return;
+	}
+ 
+	String site_home = args[0];
+	String coll_name = args[1];
+
+	File import_dir = new File(GSFile.collectionImportDir(site_home, coll_name));
+	File building_dir = new File(GSFile.collectionBuildDir(site_home, coll_name));
+
+	if (!import_dir.exists()) {
+	    System.out.println("Couldn't find import dir for collection "+coll_name);
+	    return;
+	}
+
+	if (building_dir.exists()) {
+	    // get rid of it
+	    GSFile.deleteFile(building_dir);
+	}
+	building_dir.mkdir();
+	
+	File idx_dir = new File(building_dir.getPath()+File.separator+"idx"+File.separator+"temp.txt");
+	idx_dir = idx_dir.getParentFile();
+	File text_dir = new File(building_dir.getPath()+File.separator+"text"+File.separator+"temp.txt");
+	text_dir = text_dir.getParentFile();
+	idx_dir.mkdir();
+	text_dir.mkdir();
+	
+	// first we import the coll
+	ImportXML importer = new ImportXML();
+	importer.setOutDir(text_dir);
+	importer.init();
+	importer.importFile(import_dir);
+	importer.finish();
+	
+	// then we index it
+	IndexXML indexer = new IndexXML();
+	indexer.setOutDir(idx_dir);
+	indexer.init();
+	indexer.indexFile(text_dir);
+	indexer.finish();
+    }
+}
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/ImportXML.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/ImportXML.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/ImportXML.java	(revision 21112)
@@ -0,0 +1,116 @@
+/**
+ *
+ * @author  kjdon@cs.waikato.ac.nz
+ * @version 
+ */
+
+
+// gsdl3 classes
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.util.GSFile;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.traversal.NodeIterator;
+import org.w3c.dom.traversal.NodeFilter;
+import org.w3c.dom.traversal.DocumentTraversal;
+import org.apache.xml.serialize.XMLSerializer;
+import org.xml.sax.InputSource;
+import org.xml.sax.EntityResolver;
+// java classes
+import java.io.File;
+import java.io.FileOutputStream;
+
+public class ImportXML
+    implements EntityResolver {
+    File out_dir = null;
+    XMLConverter converter = null;
+
+    String base_path = null;
+    public ImportXML() {
+	converter = new XMLConverter();
+	converter.setEntityResolver(this);
+	
+    }
+    public void setOutDir(File out_dir) {
+	this.out_dir = out_dir;
+    }
+    public void init() {
+
+    }
+    public void importFile(File file) throws Exception {
+	importFile(file, "");
+    }
+    protected void importFile(File file, String local_path) throws Exception {
+
+	if (file.isDirectory()) {
+	    File files []  = file.listFiles();
+	    for (int i=0; i<files.length; i++) {
+		//if (files[i].getName().endsWith(".xml")) {
+		importFile(files[i], local_path+File.separator+files[i].getName());
+		    //}
+	    }
+	    return;
+	}
+
+	base_path = file.getPath();
+	base_path = base_path.substring(0, base_path.lastIndexOf(File.separatorChar));
+	System.out.println("base path = "+base_path);
+	// now we have an actual file
+	System.out.println("processing file "+file.getPath());
+	File out_file = new File (out_dir, local_path);
+	String name = file.getName();
+	if (name.endsWith(".dtd")) {
+	    if (!GSFile.copyFile(file, out_file)) {
+		System.err.println("couldn't copy dtd file "+file.getPath()+" to "+out_file.getPath()+"- please do the copy yourself");
+	    }
+	    //copy the file
+	    return;
+	}
+	if (!name.endsWith(".xml")) {
+	    // now we ignore any that don't end in .xml
+	    return;
+	}
+	// now do the importing
+	Document doc = converter.getDOM(file);
+	
+        String gs3NS = "http://www.greenstone.org/gs3";
+	
+        Element rootNode = doc.getDocumentElement();
+        
+        rootNode.setAttribute("xmlns:gs3", gs3NS);
+       
+        DocumentTraversal traversal = (DocumentTraversal)doc;
+        NodeIterator i = traversal.createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, null, true);
+        
+        Element element = null;
+        Node node = null;
+        int id = 0;
+        while ((node = i.nextNode()) != null) {
+            element = (Element)node;
+	    if (XMLTagInfo.isIndexable(element.getNodeName())) {
+		element.setAttribute("gs3:id", Integer.toString(id++));
+	    }
+        }
+        
+        XMLSerializer gs3Serializer = new XMLSerializer(new FileOutputStream(out_file), null);
+        gs3Serializer.asDOMSerializer().serialize(doc);
+        
+    }
+
+    public void finish() {
+    }
+    
+    public InputSource resolveEntity (String public_id, String system_id) {
+	
+	if (system_id.startsWith("file://")) {
+	    return new InputSource(system_id);
+	}
+	if (!system_id.startsWith(File.separator)) {
+	    system_id = base_path+File.separatorChar+system_id;
+	}
+	return new InputSource("file://"+system_id);
+    }
+
+}
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/IndexXML.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/IndexXML.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/IndexXML.java	(revision 21112)
@@ -0,0 +1,43 @@
+
+/**
+ *
+ * @author  kjdon@cs.waikato.ac.nz
+ * @version 
+ */
+
+import java.io.File;
+
+public class IndexXML extends Object {
+
+    File out_dir = null;
+    Indexer indexer = null;
+    
+    public void setOutDir(File out_dir) {
+	this.out_dir = out_dir;
+    }
+
+    public void init() {
+	indexer = new Indexer(out_dir, true);
+    }
+    public void finish() {
+	indexer.finish();
+    }
+    public void indexFile(File file) {
+	
+	if (file.isDirectory()) {
+	    File[] files = file.listFiles();
+	    for (int i=0; i<files.length; i++) {
+		if (files[i].isDirectory() || files[i].getName().endsWith(".xml")) {
+		    indexFile(files[i]);
+		}
+	    }
+	    
+	} else {
+	    String name = file.getName();
+	    name = name.substring(0, name.lastIndexOf('.'));
+	    System.out.println("Indexing "+file.getPath()+" with id "+name);
+	    this.indexer.index(name, file);
+	}
+    }
+        
+}
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Indexer.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Indexer.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Indexer.java	(revision 21112)
@@ -0,0 +1,202 @@
+import org.greenstone.gsdl3.util.GSEntityResolver;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+//import org.xml.sax.SAXParseException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+//import org.apache.lucene.document.DateField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+//import org.apache.lucene.analysis.SimpleAnalyzer;
+
+import java.util.Stack;
+import java.io.FileInputStream;
+import java.io.File;
+import java.net.URL;
+
+
+
+public class Indexer extends DefaultHandler {
+    IndexWriter writer = null;
+    SAXParser sax_parser = null;
+    Stack stack = null;
+    String path = "";
+    String current_node = "";
+    String current_contents = "";
+    Document current_doc = null;
+    String scope = "";
+    protected String file_id = null;
+    private String base_path = null;
+    /** pass in true if want to create a new index, false if want to use the existing one */
+    public Indexer (File index_dir, boolean create) {
+	try {
+	    stack = new Stack();
+	    SAXParserFactory sax_factory = SAXParserFactory.newInstance();
+	    sax_parser = sax_factory.newSAXParser();
+	    writer = new IndexWriter(index_dir.getPath(), new StandardAnalyzer(), create);
+	    if (create) {
+		writer.optimize();
+	    }
+
+	} catch (Exception e) {
+
+	}
+    }
+
+    /** index one document */
+    public void index (String file_id, File file) {
+	this.file_id = file_id;
+	this.path = "";
+	this.base_path = file.getPath();
+	this.base_path = this.base_path.substring(0, this.base_path.lastIndexOf(File.separatorChar));
+	try {
+            sax_parser.parse(new InputSource(new FileInputStream(file)), this);
+        }
+        catch (Exception e) {
+	    println("parse error:");
+            e.printStackTrace();
+	}
+    }
+    
+    /** optimise the index */
+    public void finish() {
+	try {
+	    writer.optimize();
+	    writer.close();
+	} catch (Exception e) {}
+    }
+
+    protected void println(String s) { System.out.println(s); }
+
+    public void startDocument() throws SAXException {
+        println("Starting to index " + file_id);
+    }
+    public void endDocument() throws SAXException {
+        println("... indexing finished.");
+    }
+    public void startElement(String uri, String localName, String qName, Attributes atts) throws SAXException {
+	path = appendPathLink(path, qName, atts);
+	if (XMLTagInfo.isScopable(qName)) {
+	    scope = qName;
+	}
+	if (XMLTagInfo.isIndexable(qName)) {
+	    pushOnStack();
+	    current_node = qName;	
+	    System.out.println("going to index "+qName );
+	    String node_id = "";
+	    String id = "<"+qName;
+	    for (int i=0; i<atts.getLength(); i++) {
+		String name = atts.getQName(i);
+		String value = atts.getValue(i);
+		if (name!=null && value != null) {
+		    id += " "+name+"="+value;
+		}
+		if (name.equals("gs3:id")) {
+		    node_id = value;
+		}
+	    }
+	    id += "/>";
+
+	    if (scope.equals(qName)) {
+		current_doc.add(Field.UnIndexed("nodeID", this.file_id+"."+qName));
+	    } else {
+		current_doc.add(Field.UnIndexed("nodeID", this.file_id+"."+scope+"."+qName+"."+node_id));
+	    }
+	}
+    }
+    public void endElement(String uri, String localName, String qName) throws SAXException {
+	if (XMLTagInfo.isIndexable(qName) && qName.equals(current_node)) {
+	    current_doc.add(Field.UnStored("content", current_contents));
+	    try {
+		writer.addDocument(current_doc);
+	    } catch (java.io.IOException e) {
+		e.printStackTrace();
+	    }
+	    popOffStack();
+	}
+	
+	path = removePathLink(path);
+    }
+
+    public void characters(char ch[], int start, int length) throws SAXException {
+	String data = new String(ch, start, length).trim();
+	if (data.length() > 0 ) {
+	    current_contents += data;
+	}
+    }
+    
+    protected String appendPathLink(String path, String qName, Attributes atts) {
+
+	path = path + "/"+qName;
+	if (atts.getLength()>0) {
+	    String id = atts.getValue("gs3:id");
+	    if (id != null) {
+		path +=  "[@gs3:id='"+id+"']";
+	    }
+	}
+	return path;
+    }
+    protected String  removePathLink(String path) {
+
+	int i=path.lastIndexOf('/');
+	if (i==-1) {
+	    path="";
+	} else {
+	    path = path.substring(0, i);
+	}
+	return path;
+    }
+    /** these are what we save on the stack */
+    private class MyDocument {
+
+	public Document doc = null;
+	public String contents = null;
+	public String tagname = "";
+	
+    }
+    protected void pushOnStack() {
+	if (current_doc != null) {
+	    MyDocument save = new MyDocument();
+	    save.doc = current_doc;
+	    save.contents = current_contents;
+	    save.tagname = current_node;
+	    stack.push(save);
+	}
+	current_doc = new Document();
+	current_contents = "";
+	current_node = "";
+    }
+
+    protected void popOffStack() {
+	if (!stack.empty()) {
+	    MyDocument saved = (MyDocument)stack.pop();
+	    current_doc = saved.doc;
+	    current_contents = saved.contents;
+	    current_node = saved.tagname;
+	} else {
+	    current_doc = new Document();
+	    current_contents = "";
+	    current_node = "";
+	}
+    }
+
+    public InputSource resolveEntity (String public_id, String system_id) {
+	
+	if (system_id.startsWith("file://")) {
+	    return new InputSource(system_id);
+	}
+	if (!system_id.startsWith(File.separator)) {
+	    system_id = base_path+File.separatorChar+system_id;
+	}
+	return new InputSource("file://"+system_id);
+    }
+
+}
+
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Search.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Search.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/Search.java	(revision 21112)
@@ -0,0 +1,84 @@
+/*
+ * search.java
+ *
+ * Created on 25 February 2003, 02:25
+ */
+
+
+/**
+ *
+ * @author  kjdon@cs.waikato.ac.nz
+ * @version 
+ */
+
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.index.Term;
+
+public class Search  {
+
+    public static void main (String args[]) {
+	
+	if (args.length == 0) {
+	    System.out.println("Usage: Search <index directory>");
+	    return;
+	}
+        try {
+	    Searcher searcher = new IndexSearcher(args[0]);
+	    Analyzer analyzer = new StandardAnalyzer();
+
+	    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
+	    while (true) {
+		System.out.print("Query: ");
+		String line = in.readLine();
+		
+		if (line.length() == -1)
+		    break;
+		
+		Term term = new Term("content",line);
+		
+		Query query = new TermQuery(term);
+		System.out.println("Searching for: " + query.toString("content"));
+		
+		Hits hits = searcher.search(query);
+		System.out.println(hits.length() + " total matching documents");
+		
+		final int HITS_PER_PAGE=10;
+		
+		for (int start = 0; start < hits.length(); start += HITS_PER_PAGE) {
+		    int end = Math.min(hits.length(), start + HITS_PER_PAGE);
+		    for (int i = start; i < end; i++) {
+			Document doc = hits.doc(i);
+			String node_id= doc.get("nodeID");
+			System.out.println(i + ". ID: "+node_id);
+		    }
+		    
+		    if (hits.length() > end) {
+			System.out.print("more (y/n) ? ");
+			line = in.readLine();
+			if (line.length() == 0 || line.charAt(0) == 'n')
+			    break;
+		    }
+		}
+		
+	    }
+	    
+	    searcher.close();
+	}
+	catch (Exception e) {
+	    System.out.println(" caught a " + e.getClass() +
+			       "\n with message: " + e.getMessage());
+        }
+    }
+}
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/XMLTagInfo.java
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/XMLTagInfo.java	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/java/XMLTagInfo.java	(revision 21112)
@@ -0,0 +1,23 @@
+/** XMLTagInfo
+ * contains any methods that are specific to what kind of XML you are working on
+ */
+
+public class XMLTagInfo {
+
+    public static boolean isIndexable(String tag) {
+	if (tag.equals("chapter") || tag.equals("part") || tag.equals("frontmatter") || tag.equals("appendix") || tag.equals("index") || tag.equals("glossary") || tag.equals("biblio")) {
+	    return true;
+	} 
+	return false;
+    }
+
+    public static boolean isScopable(String tag) {
+
+	if (tag.equals("bookbody") || tag.equals("frontmatter") || tag.equals("backmatter")) {
+	    return true;
+	} 
+	return false;
+    }
+
+    
+}
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/bookfrag.dtd
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/bookfrag.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/bookfrag.dtd	(revision 21112)
@@ -0,0 +1,529 @@
+<!--This is bookfrag DTD-->
+
+<!--entity declarations-->
+<!--#############################################-->
+
+<!ENTITY % stdatts 
+"ref   IDREF  #IMPLIED
+ id    ID     #IMPLIED
+ type  CDATA  #IMPLIED
+ role  CDATA  #IMPLIED
+ class CDATA  #IMPLIED"
+>
+
+<!ENTITY % inline.class "|quote|emph|ital|reference|date|place|name|graphic|txterr|mkuperr|misc ">
+
+<!ENTITY % block.class "|letter|blockquote|footnote|note|list|deflist|table|blockgraphic">
+
+
+<!-- the top level element-->
+<!--##############################################-->
+
+<!ELEMENT book (acknowledge?,meta*,frontmatter?,bookbody,backmatter?,endmeta*)>
+
+<!--begin top level element declarations-->
+<!--##############################################-->
+
+
+<!--the acknowledge element is for a simple statement 
+designed to be placed at the beginning of the visible document.
+Suggested content:
+  Name of Book
+  Author
+  PG and HWG blurb
+  Original Transcriber + date
+  Marker 
+  e.g.
+  "Origin of the Species by Charles Darwin
+  A project of Project Gutenberg and the HTML Writers Guild.
+  Transcribed by  
+  Markup by Frank Boumphrey. Jan 22 2000"
+  -->
+
+<!ELEMENT acknowledge (#PCDATA %inline.class;)*>
+<!ATTLIST acknowledge
+  %stdatts;
+>
+
+<!--the meta tags are here to provide a means for comment 
+at either the very beginning or end of the document-->
+<!ELEMENT meta EMPTY>
+<!ATTLIST meta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+<!ELEMENT endmeta EMPTY >
+<!ATTLIST endmeta
+   content  CDATA  #REQUIRED
+   id       ID     #IMPLIED
+>
+
+
+
+<!--the bookbody element contains the body of the book it can 
+consist of either part or chapter divisions-->
+
+<!ELEMENT bookbody (part*|chapter*)>
+
+<!--
+the backmatter element contains the backmatter of the book -->
+
+<!ELEMENT backmatter ((appendix|index|glossary|biblio|note)*,colophon?)>
+
+<!--The endmeta tag is groupedwith its twin meta-->
+
+
+<!--end top level element declarations-->
+
+<!--begin frontmatter element declarations-->
+<!--##############################################-->
+
+<!ELEMENT frontmatter (htitlepage|copypage|epigraph|titlepage|
+                toc|acksect|dedication|preface|introduction|miscfm)*>
+
+<!ELEMENT htitlepage (#PCDATA|title|subtitle|author|para|poem|song %inline.class;)*>
+<!ATTLIST htitlepage
+  %stdatts;
+>
+
+<!ELEMENT copypage (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST copypage
+  %stdatts;
+>
+
+<!ELEMENT epigraph (#PCDATA|para|poem|song|note|blockquote %inline.class;)*>
+<!ATTLIST epigraph
+  %stdatts;
+>
+
+
+<!ELEMENT titlepage (#PCDATA|partnum|title|subtitle|author|pubinfo|para|poem|song|note|line %inline.class;)*>
+<!ATTLIST titlepage
+  %stdatts;
+>
+
+<!ELEMENT pubinfo (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST pubinfo
+  %stdatts;
+>
+
+
+<!ELEMENT toc (#PCDATA|title|subtitle|subsubtitle|item|list|deflist %inline.class;)*>
+<!--use type or role to describe toctype="other"-->
+<!ATTLIST toc
+  toctype (contents|maps|graphics|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT acksect (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST acksect
+  %stdatts;
+>
+
+<!ELEMENT dedication (#PCDATA|title|para|poem|song|note %inline.class;)*>
+<!ATTLIST dedication
+  %stdatts;
+>
+
+
+<!ELEMENT preface ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?,preauthor?)>
+<!ATTLIST preface  %stdatts;
+>
+
+<!ELEMENT introduction ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST introduction  %stdatts;
+>
+
+<!ELEMENT miscfm (#PCDATA|para|poem|song|note %inline.class;)*>
+<!ATTLIST miscfm
+  %stdatts;
+>
+
+<!ELEMENT preauthor (#PCDATA|author %inline.class;)*>
+<!ATTLIST preauthor
+  %stdatts;
+>
+
+
+
+<!--end frontmatter element declarations-->
+
+<!--begin bookbody element declarations-->
+<!--##############################################-->
+
+
+<!ELEMENT part (acknowledge?,(titlepage|toc|htitlepage)*,chapter*)>
+<!ATTLIST part
+  %stdatts;
+>
+
+<!ELEMENT chapter ((title|chapheader)?,(para|poem|song|sect1|simplesect|page %block.class;)*,endchap?)>
+<!ATTLIST chapter  %stdatts;
+>
+
+<!--end bookbody element declarations-->
+
+<!--the lowerlevel bookbody elements-->
+<!--##############################################-->
+
+<!--the chapter heading elements-->
+<!--##############################################-->
+
+
+
+
+<!ELEMENT title (#PCDATA %inline.class;)*>
+<!ATTLIST title
+  %stdatts;
+>
+
+<!ELEMENT subtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subtitle
+  %stdatts;
+>
+
+<!ELEMENT subsubtitle (#PCDATA %inline.class;)*>
+<!ATTLIST subsubtitle
+  %stdatts;
+>
+
+
+
+<!--validity constraint. Only one title,chapnum and chapsummary -->
+<!ELEMENT chapheader (title|subtitle|chapnum|chapsummary|blockquote|para|note)*>
+<!ATTLIST chapheader
+  %stdatts;
+>
+
+
+<!ELEMENT chapnum (#PCDATA %inline.class;)*>
+<!ATTLIST chapnum
+  %stdatts;
+>
+
+<!ELEMENT chapsummary (#PCDATA %inline.class;)*>
+<!ATTLIST chapsummary
+  %stdatts;
+>
+
+
+<!--begin backmatter element declarations-->
+<!--##############################################-->
+
+<!--end backmatter element declarations-->
+
+<!ELEMENT index (title|item|list|deflist|note)*>
+<!ATTLIST index
+  indtype (contents|authors|firstlines|tables|other) "contents" 
+  %stdatts;
+>
+
+<!ELEMENT glossary (title|item|list|deflist|note)*>
+<!ATTLIST glossary  
+  %stdatts;
+>
+
+<!ELEMENT biblio (title|item|list|deflist|note)*>
+<!ATTLIST biblio  
+  %stdatts;
+>
+
+
+<!ELEMENT appendix ((title|chapheader)?,(para|poem|song|sect1|simplesect %block.class;)*,endchap?)>
+<!ATTLIST appendix  %stdatts;
+>
+
+<!ELEMENT colophon (#PCDATA|para|poem|song %inline.class;)*>
+<!ATTLIST colophon
+  %stdatts;
+>
+
+<!--the block elements-->
+<!--##############################################-->
+
+
+<!--validity constraint. Only one title per paragraph-->
+<!ELEMENT para (#PCDATA|title %inline.class;)*>
+<!ATTLIST para
+  %stdatts;
+>
+
+<!ELEMENT simplesect (title,(subtitle|para|poem|song %block.class;)*)>
+<!ATTLIST simplesect
+  %stdatts;
+>
+
+<!ELEMENT sect1 (title,(sect2|simplesect|para|poem|song %block.class;)*)>
+<!ATTLIST sect1
+  %stdatts;
+>
+
+<!ELEMENT sect2 (title,(sect3|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect2
+  %stdatts;
+>
+
+<!ELEMENT sect3 (title,(sect4|simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect3
+  %stdatts;
+>
+
+<!ELEMENT sect4 (title,(simplesect|subtitle|para|poem|song  %block.class;)*)>
+<!ATTLIST sect4
+  %stdatts;
+>
+
+<!ELEMENT page (#PCDATA %inline.class;)*>
+<!ATTLIST page
+  %stdatts;
+>
+
+<!ELEMENT misc (#PCDATA %inline.class;)*>
+<!ATTLIST misc
+  %stdatts;
+>
+
+
+
+<!ELEMENT blockquote (title?,(para|poem|song)*,attrib?)>
+<!ATTLIST blockquote
+  %stdatts;
+>
+
+<!ELEMENT attrib (#PCDATA %inline.class;)*>
+<!ATTLIST attrib
+  %stdatts;
+>
+
+<!ELEMENT caption (#PCDATA %inline.class;)*>
+<!ATTLIST caption
+  %stdatts;
+>
+<!--##############################################-->
+<!--poetry and songs in the text-->
+<!ELEMENT poem (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST poem
+  %stdatts;
+>
+
+<!ELEMENT song (#PCDATA|title|subtitle|verse|line|note|footnote %inline.class;)*>
+<!ATTLIST song
+  %stdatts;
+>
+
+
+<!ELEMENT verse (title|subtitle|line|note)*>
+<!ATTLIST verse
+  %stdatts;
+>
+
+<!ELEMENT line (#PCDATA %inline.class;)*>
+<!ATTLIST line
+  %stdatts;
+>
+
+
+<!--validity constraint: Only one title per poem or per verse-->
+
+<!--end of poetry in the text-->
+
+<!--letter module-->
+
+<!ELEMENT letter (address|to|from|salut|sig|title|subtitle|para|poem|song|line|note)*>
+<!ATTLIST letter
+  %stdatts;
+>
+
+<!ELEMENT address (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST address
+  %stdatts;
+>
+
+<!ELEMENT to (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST to
+  %stdatts;
+>
+
+
+<!ELEMENT from (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST from
+  %stdatts;
+>
+
+
+<!ELEMENT salut (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST salut
+  %stdatts;
+>
+
+
+<!ELEMENT sig (#PCDATA|para|line %inline.class;)*>
+<!ATTLIST sig
+  %stdatts;
+>
+
+
+<!--end letters module-->
+<!--endchap-->
+<!--##############################################-->
+<!--
+  Endchap should only be used when there is a 
+  considerable ammount of material to be organized.
+  For the occasional note footnote and note will suffice
+-->
+
+<!ELEMENT endchap (para %block.class;)*>
+<!ATTLIST endchap  %stdatts;
+>
+
+<!--lists-->
+<!--##############################################-->
+
+
+<!ELEMENT list (title?,(list|item)*)>
+<!ATTLIST list
+  %stdatts;
+>
+
+<!ELEMENT item (#PCDATA|para|poem|song|simplesect %block.class; %inline.class;)*>
+<!ATTLIST item
+  %stdatts;
+>
+
+<!ELEMENT deflist (title?,(item,desc?,def*)*)>
+<!ATTLIST deflist
+  %stdatts;
+>
+
+<!ELEMENT desc (#PCDATA %inline.class;)*>
+<!ATTLIST desc
+  %stdatts;
+>
+
+<!ELEMENT def (#PCDATA %inline.class;)*>
+<!ATTLIST def
+  %stdatts;
+>
+<!--end lists-->
+<!--table element-->
+<!--##############################################-->
+
+
+<!ELEMENT table (title?,row*,caption?)>
+<!ATTLIST table
+  %stdatts;
+>
+
+<!ELEMENT row (cell)*>
+<!ATTLIST row
+  %stdatts;
+>
+
+<!ELEMENT cell (#PCDATA %block.class; %inline.class;)*>
+<!ATTLIST cell
+  %stdatts;
+>
+
+<!--end table-->
+
+<!--graphics-->
+<!--##############################################-->
+
+<!--
+  Note that the required desc attribute is added 
+  text composed by the marker.
+-->
+
+<!ELEMENT graphic EMPTY >
+<!ATTLIST graphic
+  desc CDATA #REQUIRED
+  href CDATA #REQUIRED
+  %stdatts;
+>
+
+<!ELEMENT blockgraphic (title?,graphic,caption?)>
+<!ATTLIST blockgraphic
+  %stdatts;
+>
+
+
+<!--the inline elements-->
+<!--##############################################-->
+
+
+
+<!ELEMENT footnote (#PCDATA %inline.class;)*>
+<!ATTLIST footnote
+  %stdatts;
+>
+
+<!ELEMENT note (#PCDATA %inline.class;)*>
+<!ATTLIST note
+  %stdatts;
+>
+
+
+<!ELEMENT quote (#PCDATA %inline.class;)*>
+<!ATTLIST quote
+  %stdatts;
+>
+
+<!ELEMENT emph (#PCDATA %inline.class;)*>
+<!ATTLIST emph
+  %stdatts;
+>
+
+<!ELEMENT ital (#PCDATA %inline.class;)*>
+<!ATTLIST ital
+  %stdatts;
+>
+
+<!ELEMENT reference (#PCDATA %inline.class;)*>
+<!ATTLIST reference
+  %stdatts;
+>
+
+<!ELEMENT author (#PCDATA %inline.class;)*>
+<!ATTLIST author
+  %stdatts;
+>
+
+<!ELEMENT date (#PCDATA %inline.class;)*>
+<!ATTLIST date
+  %stdatts;
+>
+
+<!ELEMENT place (#PCDATA %inline.class;)*>
+<!ATTLIST place
+  %stdatts;
+>
+
+<!ELEMENT name (#PCDATA %inline.class;)*>
+<!ATTLIST name
+  %stdatts;
+>
+
+
+<!--use to enclose the text you think is in error
+optional explanation if error is not obvious-->
+<!ELEMENT txterr (#PCDATA)*>
+<!ATTLIST txterr
+  explain  CDATA  #IMPLIED
+  %stdatts;
+>
+
+<!--an explanation is required-->
+<!ELEMENT mkuperr EMPTY>
+<!ATTLIST mkuperr
+  explain  CDATA  #REQUIRED
+  %stdatts;
+>
+
+<!--added as titlepage content  Feb/6/00-->
+<!ELEMENT partnum (#PCDATA %inline.class;)*>
+<!ATTLIST partnum
+  %stdatts;
+>
+
+<!--End of bookfrag dtd-->
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/gutbook1.dtd
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/gutbook1.dtd	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/resources/gutbook1.dtd	(revision 21112)
@@ -0,0 +1,40 @@
+<!--
+GUTBOOK DTD V 1.0
+==================
+
+GENERAL NOTES
++++++++++++++
+This is gutbook dtd version 1.0
+file name is gutpoems1.dtd
+This dtd is a first pass at a dtd for marking up 
+books for project gutenberg.
+It was originally developed for marking up the Darwins books.
+
+REVISION HISTORY
+++++++++++++++++
+Original Author
+Frank Boumphrey  1/19/2000
+
+COPYRIGHT NOTICE
+++++++++++++++++
+This DTD is the copyright of the HTML writers guild.
+Permission is given in perpetuity for any one to use 
+this DTD provided that these notes and this  copyright 
+notice is included.
+
+
+-->
+
+<!--Begin gutbook DTD-->
+<!ELEMENT gutbook (gutblurb,markupblurb,book*,endmarkupblurb?,endgutblurb?)>
+
+<!ELEMENT gutblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT markupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endmarkupblurb (#PCDATA|para|simplesect|title)*>
+<!ELEMENT endgutblurb (#PCDATA|para|simplesect|title)*>
+
+<!--Include bookfrag.dtd-->
+
+<!ENTITY % bookfrag SYSTEM "bookfrag.dtd">
+%bookfrag;
+<!--end of file-->
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/about.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/about.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/about.xsl	(revision 21112)
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/>  
+
+  <xsl:template name="pageTitle">
+    Sample XML Texts Collection
+  </xsl:template>
+
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+
+  <xsl:template match="pageResponse">
+    <xsl:apply-templates select="collection"/>
+  </xsl:template>
+
+  <xsl:template match="collection">
+    <xsl:variable name="collName" select="@name"/>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+      <xsl:with-param name="pageType">about</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <p/><xsl:value-of select="displayItem[@name='description']" disable-output-escaping='yes'/>
+    <p/><xsl:apply-templates select="documentList">
+      <xsl:with-param name="collName" select="@name"/>
+    </xsl:apply-templates>
+  </xsl:template>
+  
+  <xsl:template match="documentList">
+    <xsl:param name="collName"/>
+    <ul>
+      <xsl:for-each select="document">
+	<li><a href="{$library_name}?a=xd&amp;sa=toc&amp;sc=full&amp;c={$collName}&amp;d={@name}"><xsl:value-of select="metadataList/metadata[@name='Title']"/></a></li>
+      </xsl:for-each>
+    </ul>
+  </xsl:template>
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/darwin.css
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/darwin.css	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/darwin.css	(revision 21112)
@@ -0,0 +1,342 @@
+/*This is darwin.css*/
+
+ gutblurb{
+  display:none;
+ }
+
+ markupblurb
+  {
+  display:none;
+
+  }
+ endgutblurb{
+  display:none;
+ }
+
+ para,sect1,sect2,sect3,sect4,simplesect{
+  display:block;
+  }
+
+
+ title1,item,chapter,chapnum,chapsummary,desc,def,letter
+{
+  display:block;
+  }
+
+ blockquote,footnote,note,list,deflist,blockgraphic{
+  display:block;
+  }
+
+ quote,emph,ital,reference,date,place,name,graphic,misc{
+  display:inline;
+  }
+
+  acknowledge{
+  display:block;
+  text-align:center;
+  font-size:8pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+ htitlepage title1{
+  display:block;
+  text-align:center;
+  font-size:16pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+ htitlepage subtitle{
+  display:block;
+  text-align:center;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  htitlepage author{
+  display:block;
+  text-align:center;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  epigraph{
+  display:block;
+  text-align:center;
+  font-size:12pt;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+
+  epigraph blockquote{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  margin-top:8pt;
+  margin-left:24pt;
+  margin-right:24pt;
+  margin-bottom:12pt;
+  }
+
+  epigraph attrib{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  margin-top:8pt;
+  margin-right:24pt;
+  margin-left:24pt;
+  margin-bottom:12pt;
+  }
+
+ titlepage title1{
+  display:block;
+  text-align:center;
+  font-size:16pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  titlepage para{
+  display:block;
+  text-align:center;
+  font-size:12pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  titlepage line{
+  display:block;
+  text-align:center;
+  font-size:12pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  titlepage subtitle{
+  display:block;
+  text-align:center;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+
+
+  titlepage author{
+  display:block;
+  text-align:center;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+ titlepage partnum{
+  display:block;
+  text-align:center;
+  font-size:16pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+ dedication title1{
+  display:block;
+  text-align:center;
+  font-size:16pt;
+  font-family:arial,sans-serif;
+  margin-top:8pt;
+  margin-bottom:12pt;
+  }
+
+  toc title1{
+  display:block;
+  text-align:left;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:24pt;
+  margin-bottom:12pt;
+  }
+
+  toc subtitle{
+  display:block;
+  text-align:left;
+  font-weight:bold;
+  font-size:13pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:24pt;
+  margin-bottom:12pt;
+  }
+
+
+  toc item{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:36pt;
+  margin-bottom:12pt;
+  }
+
+
+  toc def{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-family:times new roman,serif;
+  margin-top:12pt;
+  margin-left:48pt;
+  margin-right:24pt;
+
+  margin-bottom:12pt;
+  }
+
+  toc desc{
+  display:block;
+  text-align:left;
+  font-size:10pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:42pt;
+  margin-bottom:12pt;
+  }
+
+ preface title1{
+  display:block;
+  text-align:left;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:12pt;
+  margin-bottom:12pt;
+  }
+
+  introduction title1{
+  display:block;
+  text-align:left;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:12pt;
+  margin-bottom:12pt;
+  }
+
+  salut,sig,para{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-family:times new roman,serif;
+  margin-top:12pt;
+  margin-left:24pt;
+  margin-right:24pt;
+  margin-bottom:12pt;
+  }
+
+  poem{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  margin-left:24pt;
+  margin-top:18pt;
+  margin-bottom:18pt;
+  }
+
+  line{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  margin-left:24pt;
+  margin-right:24pt;
+  }
+
+  chapter title1{
+  display:block;
+  text-align:left;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:12pt;
+  margin-bottom:12pt;
+  }
+
+  chapter chapnum{
+  display:block;
+  text-align:center;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-bottom:12pt;
+  }
+
+  chapter chapsummary{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-style:italic;
+  font-family:times new roman,serif;
+  margin-top:12pt;
+  margin-left:48pt;
+  margin-right:36pt;
+  margin-bottom:12pt;
+  }
+
+  chapter blockquote para{
+  display:block;
+  text-align:left;
+  font-size:11pt;
+  
+  margin-top:8pt;
+  margin-left:48pt;
+  margin-right:48pt;
+  margin-bottom:12pt;
+  }
+
+
+  index item{
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-family:times new roman,serif;
+  margin-top:8pt;
+  margin-left:12pt;
+  margin-bottom:8pt;
+  }
+
+  index title1{
+  display:block;
+  text-align:left;
+  font-size:14pt;
+  font-family:arial,sans-serif;
+  margin-top:12pt;
+  margin-left:12pt;
+  margin-bottom:12pt;
+  }
+
+  list item{
+  color:black;
+  display:block;
+  text-align:left;
+  font-size:12pt;
+  font-family:times new roman,serif;
+  margin-top:8pt;
+  margin-left:48pt;
+  margin-bottom:8pt;
+  }
+
+
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-content.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-content.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-content.xsl	(revision 21112)
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/> 
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/documentNode/metadataList/metadata[@name='root_Title']"/>
+  </xsl:template>
+  
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle">
+    <link type="text/css" href="sites/localsite/collect/gberg/transform/darwin.css" rel="stylesheet"/>
+  </xsl:template>
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <!-- display the document -->
+    <xsl:apply-templates select="documentNode">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docname"><xsl:choose><xsl:when test="contains(@nodeID, '.')"><xsl:value-of select="substring-before(@nodeID, '.')"/></xsl:when><xsl:otherwise><xsl:value-of select="@nodeID"/></xsl:otherwise></xsl:choose></xsl:variable>
+    <h1>
+      <center>
+	<a href="{$library_name}?a=xd&amp;sa=toc&amp;c={$collName}&amp;d={$docname}"><xsl:attribute name="title">Go to the document contents</xsl:attribute><xsl:value-of select="metadataList/metadata[@name='root_Title']"/>&#160;</a>
+      </center>
+    </h1>
+    <xsl:apply-templates select="nodeContent/*"/>
+  </xsl:template>
+
+  <xsl:template match="gutblurb | markupblurb | endmarkupblurb | endgutblurb">
+  </xsl:template>
+
+  <xsl:template match="gutbook | book | frontmatter | bookbody | backmatter | titlepage| htitlepage | toc | dedication | preface | introduction | chapter |  part | chapheader | index | appendix | glossary">
+    <xsl:copy>
+      <xsl:copy-of select="@*"/>
+      <xsl:apply-templates/>
+    </xsl:copy>
+  </xsl:template>
+
+<!-- dont seem to be able to apply styles to title tags - cos they are special to html. so we rename title to title1-->
+  <xsl:template match="title">
+    <title1>
+      <xsl:apply-templates/>
+    </title1>
+  </xsl:template>  
+    
+  <xsl:template match="*">
+    <xsl:copy-of select="."/>
+  </xsl:template>
+
+  <xsl:template match="metadataList">
+  </xsl:template>
+
+</xsl:stylesheet>
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-toc.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-toc.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/document-toc.xsl	(revision 21112)
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:gs3="http://www.greenstone.org/gs3"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+
+  <!-- style includes global params interface_name, library_name -->
+  <xsl:include href="style.xsl"/>
+
+  <xsl:output method="html"/> 
+  
+  <xsl:template name="pageTitle">
+    <xsl:value-of select="/page/pageResponse/documentNode/metadataList/metadata[@name='root_Title']"/>: Table of Contents
+  </xsl:template>
+  
+  <!-- page specific style goes here -->
+  <xsl:template name="pageStyle"/>
+  
+
+  <xsl:template match="pageResponse">
+    <xsl:variable name="collName"><xsl:value-of select="/page/pageRequest/paramList/param[@name='c']/@value"/></xsl:variable>
+    <xsl:call-template name="standardPageBanner">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>
+    <xsl:call-template name="navigationBar">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:call-template>      
+    <!-- display the document -->
+    <xsl:apply-templates select="documentNode">
+      <xsl:with-param name="collName" select="$collName"/>
+    </xsl:apply-templates>
+    <xsl:call-template name="dividerBar"/>
+  </xsl:template>
+
+  <xsl:template match="documentNode">
+    <xsl:param name="collName"/>
+    <xsl:variable name="docname"><xsl:choose><xsl:when test="contains(@nodeID, '.')"><xsl:value-of select="substring-before(@nodeID, '.')"/></xsl:when><xsl:otherwise><xsl:value-of select="@nodeID"/></xsl:otherwise></xsl:choose></xsl:variable>
+    <h1>
+      <center>
+	<xsl:value-of select="metadataList/metadata[@name='root_Title']"/>
+      </center>
+    </h1>
+    <xsl:apply-templates select="nodeStructure/gutbook/book"><xsl:with-param name="collName" select="$collName"/><xsl:with-param name="docName" select="$docname"/></xsl:apply-templates>
+  </xsl:template>
+  
+  <xsl:template match="book">
+    <xsl:param name="collName"/>
+    <xsl:param name="docName"/>
+    <h1>
+      Contents
+    </h1>
+    <ul class="toc">
+      <xsl:if test="acknowledge">
+	<li class='toc'>
+	  <a href="{$library_name}?a=xd&amp;sa=text&amp;d={$docName}.acknowledge&amp;c={$collName}">Acknowledgements</a>
+	</li>
+      </xsl:if>
+      <xsl:if test="frontmatter">
+	<li class='toc'>
+	  <a href="{$library_name}?a=xd&amp;sa=text&amp;d={$docName}.frontmatter&amp;c={$collName}">About this document</a>
+	</li>
+      </xsl:if>
+      <li class='toc'><a href="{$library_name}?a=xd&amp;sa=text&amp;d={$docName}&amp;c={$collName}">Full version</a>
+      </li>
+    </ul>
+    <xsl:apply-templates select="bookbody"><xsl:with-param name="collName" select="$collName"/><xsl:with-param name="docName" select="$docName"/></xsl:apply-templates>
+    <xsl:if test="backmatter">
+      <ul class='toc'>
+	<li class='toc'>
+	<a href="{$library_name}?a=xd&amp;sa=text&amp;d={$docName}.backmatter&amp;c={$collName}">Back matter (may include appendices, index, glossary etc)</a>
+	</li>
+      </ul>
+    </xsl:if>
+  </xsl:template>
+    
+  <xsl:template match="bookbody">
+    <xsl:param name="collName"/>
+    <xsl:param name="docName"/>
+    <ul class="toc">
+      <xsl:apply-templates select="part|chapter">
+	<xsl:with-param name="collName" select="$collName"/>
+	<xsl:with-param name="docName" select="$docName"/>
+      </xsl:apply-templates>
+    </ul>
+  </xsl:template>
+
+  <xsl:template match="part">
+    <xsl:param name="collName"/>
+    <xsl:param name="docName"/>
+    <li class='toc'>
+      <a href="{$library_name}?a=xd&amp;sa=text&amp;c={$collName}&amp;d={$docName}.bookbody.part.{@gs3:id}">Part <xsl:value-of select="position()"/></a>
+      <xsl:if test="chapter">
+	<ul class='toc'>
+	  <xsl:apply-templates select="chapter">
+	  <xsl:with-param name="collName" select="$collName"/>
+	  <xsl:with-param name="docName" select="$docName"/>
+	</xsl:apply-templates>
+	</ul>
+      </xsl:if>
+    </li>
+  </xsl:template>
+
+  <xsl:template match="chapter">
+    <xsl:param name="collName"/>
+    <xsl:param name="docName"/>
+    <li class='toc'>
+      <a href="{$library_name}?a=xd&amp;sa=text&amp;c={$collName}&amp;d={$docName}.bookbody.chapter.{@gs3:id}">
+	<xsl:choose>
+	  <xsl:when test="title">
+	    <xsl:value-of select="title"/>
+	  </xsl:when>
+	  <xsl:when test="chapheader">
+	    <xsl:value-of select="chapheader/chapnum"/>: <xsl:value-of select="chapheader/title"/>
+	  </xsl:when>
+	  <xsl:otherwise>
+	    Chapter <xsl:value-of select="position()"/>
+	  </xsl:otherwise>
+	</xsl:choose>
+      </a>
+    </li>
+  </xsl:template>
+   
+</xsl:stylesheet>
+
+
+
+
+
+
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/produce-toc.xsl
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/produce-toc.xsl	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gberg/transform/produce-toc.xsl	(revision 21112)
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:java="http://xml.apache.org/xslt/java"
+  xmlns:util="xalan://org.greenstone.gsdl3.util.XSLTUtil"
+  extension-element-prefixes="java util"
+  exclude-result-prefixes="java util">
+  
+  <xsl:output method="xml"/>
+
+  <xsl:strip-space elements='*'/>
+
+  <xsl:template match="gutbook | book | acknowledge | frontmatter | bookbody | backmatter | part | chapter | chapheader "><xsl:copy><xsl:for-each select="@*"><xsl:copy/></xsl:for-each><xsl:apply-templates/></xsl:copy></xsl:template>
+
+  <xsl:template match="title | chapnum">
+    <xsl:copy-of select="."/>
+  </xsl:template>
+
+  <!-- any thing else is ignored -->
+  <xsl:template match="*"></xsl:template>
+  
+</xsl:stylesheet>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/README
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/README	(revision 21112)
@@ -0,0 +1,19 @@
+This collection is based on the original demo collection that comes with 
+Greenstone2. I built the collection using Greenstone2, then copied it 
+across to Greenstone3. I ran the convert_coll_from_gs2.pl Perl script to 
+generate the collectionConfig.xml and buildConfig.xml files, then edited 
+them by hand. I also generated new images for the collection, based on its 
+new name.
+
+The changes I made were:
+
+collectionConfig.xml:
+--------------------
+* More readable XML
+* Changed the English name and description displayItems, and removed the other language items, as they no longer are an accurate translation.
+* added a new displayItem to classifier CL4.
+* changed the image names
+
+buildConfig.xml:
+----------------
+* More readable XML
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collect.cfg
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collect.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collect.cfg	(revision 21112)
@@ -0,0 +1,51 @@
+creator		greenstone@cs.waikato.ac.nz
+maintainer	greenstone@cs.waikato.ac.nz
+public		true
+
+indexes		section:text section:Title document:text
+
+plugin		GAPlug
+plugin		HTMLPlug -description_tags 
+plugin		ArcPlug 
+plugin		RecPlug -use_metadata_files 
+
+classify	Hierarchy -metadata dls.Subject -sort dls.Title -buttonname Subject  
+classify	AZList -metadata dls.Title 
+classify	AZCompactList -metadata dls.Organization -sort dls.Title -buttonname Organization -mingroup 1
+classify      List -metadata dls.Keyword -buttonname Howto
+
+format SearchVList "<td valign=top>[link][icon][/link]</td><td>{If}{[parent(All\': \'):Title],[parent(All\': \'):Title]:}[link][Title][/link]</td>"
+
+format        CL4VList     "<br>[link][dls.Keyword][/link]"
+
+format DocumentText "<h3>[Title]</h3>\\n\\n<p>[Text]"
+format DocumentImages true
+format DocumentButtons "Expand Text|Expand Contents|Detach|Highlight"
+
+collectionmeta	iconcollectionsmall [l=en] "_httpprefix_/collect/gs2mgdemo/images/gs2mgdemo_sm.gif"
+collectionmeta	iconcollection [l=en] "_httpprefix_/collect/gs2mgdemo/images/gs2mgdemo.gif"
+
+# -- English strings ----------------
+collectionmeta	collectionname [l=en] "Greenstone2 MG demo collection"
+collectionmeta	collectionextra [l=en] "This is a demonstration collection for the Greenstone3 digital library software. It contains a small subset (11 books) of the Humanitarian and Development Libraries. It is a Greenstone2 collection, built with MG. The Greenstone2 configuration files have been converted to Greenstone3 equivalents, enabling it to be served using Greenstone3."
+
+collectionmeta	.section:text [l=en] "chapters"
+collectionmeta	.section:Title [l=en] "section titles"
+collectionmeta	.document:text [l=en] "entire documents"
+
+# -- French strings ----------------
+collectionmeta	.section:text [l=fr] "chapitres"
+collectionmeta	.section:Title [l=fr] "titres des sections"
+collectionmeta	.document:text [l=fr] "documents entiers"
+
+# -- Spanish strings ----------------
+collectionmeta	.section:text [l=es] "capÃ­tulos"
+collectionmeta	.section:Title [l=es] "tÃ­tulos de las secciones"
+collectionmeta	.document:text [l=es] "documentos enteros"
+
+# -- Russian strings ----------------
+collectionmeta	.section:text [l=ru] "ÐÐ»Ð°Ð²Ñ"
+collectionmeta	.section:Title [l=ru] "ÐÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ°Ð·ÐŽÐµÐ»ÐŸÐ²"
+collectionmeta	.document:text [l=ru] "ÐÐŸÐ»ÐœÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ"
+
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/etc/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,51 @@
+<collectionConfig 
+  xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" 
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <metadataList>
+    <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Greenstone2 MG demo collection</displayItem>
+    <displayItem name='description' lang='en'>This is a demonstration collection for the Greenstone3 digital library software. It contains a small subset (11 books) of the Humanitarian and Development Libraries. It is a Greenstone2 collection, built with MG. The Greenstone2 configuration files have been converted to Greenstone3 equivalents, enabling it to be served using Greenstone3. </displayItem>
+    <displayItem name="smallicon" lang="en">gs2mgdemo_sm.gif</displayItem>
+    <displayItem name="icon" lang="en">gs2mgdemo.gif</displayItem>
+  </displayItemList>
+  <search>
+    <index name="dtx">
+      <displayItem name="name" lang="ru">ÐÐŸÐ»ÐœÑÐµ ÐŽÐŸÐºÑÐŒÐµÐœÑÑ</displayItem>
+      <displayItem name="name" lang="en">entire documents</displayItem>
+      <displayItem name="name" lang="fr">documents entiers</displayItem>
+      <displayItem name="name" lang="es">documentos enteros</displayItem>
+    </index>
+    <index name="stt">
+      <displayItem name="name" lang="ru">ÐÐ°Ð·Ð²Ð°ÐœÐžÑ ÑÐ°Ð·ÐŽÐµÐ»ÐŸÐ²</displayItem>
+      <displayItem name="name" lang="en">section titles</displayItem>
+      <displayItem name="name" lang="fr">titres des sections</displayItem>
+      <displayItem name="name" lang="es">tÃ­tulos de las secciones</displayItem>
+    </index>
+    <index name="stx">
+      <displayItem name="name" lang="ru">ÐÐ»Ð°Ð²Ñ</displayItem>
+      <displayItem name="name" lang="en">chapters</displayItem>
+      <displayItem name="name" lang="fr">chapitres</displayItem>
+      <displayItem name="name" lang="es">capÃ­tulos</displayItem>
+    </index>
+    <format>
+      <gsf:template match="documentNode">
+	<td valign='top'><gsf:link><gsf:icon/></gsf:link></td>
+	<td><gsf:switch><gsf:metadata name='Title' select='ancestors' separator=': '/><gsf:when test='exists'><gsf:metadata name='Title' select='ancestors' separator=': '/>: </gsf:when></gsf:switch><gsf:link><gsf:metadata name='Title' /></gsf:link></td>
+      </gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CL1"/>
+    <classifier name="CL2"/>
+    <classifier name="CL3"/>
+    <classifier name="CL4">
+      <format>
+	<gsf:template match="documentNode">
+	  <br /><gsf:link><gsf:metadata name="dls.Keyword"/></gsf:link>
+	</gsf:template>
+      </format>
+    </classifier>
+  </browse>
+</collectionConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/gs2mgdemo.col
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/gs2mgdemo.col	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/gs2mgdemo.col	(revision 21112)
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE GathererCollection [
+  <!ELEMENT GathererCollection (Argument*, GathererConfig, BuildConfig)>
+  <!ELEMENT GathererConfig    (Arguments*)>
+  <!ELEMENT BuildConfig       (Build, Import)>
+  <!ELEMENT Build             (Arguments*)>
+  <!ELEMENT Import            (Arguments*)>
+  <!ELEMENT Argument          (#PCDATA)>
+  <!ATTLIST Argument
+            name              CDATA #REQUIRED
+				enabled           CDATA "true">
+]>
+<GathererCollection>
+    <Argument name="built">false</Argument>
+    <Argument name="imported">false</Argument>
+    <GathererConfig/>
+    <BuildConfig>
+        <Build/>
+        <Import/>
+    </BuildConfig>
+</GathererCollection>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgdemo/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,30 @@
+<buildConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat">
+  <metadataList>
+    <metadata name="numDocs">11</metadata>
+    <metadata name="buildType">mg</metadata>
+  </metadataList>
+  <serviceRackList>
+    <serviceRack name="GS2Browse">
+      <indexStem name="gs2mgdemo" />
+      <classifierList>
+	<classifier name="CL1" content="Subject" />
+	<classifier name="CL2" content="Title" />
+	<classifier name="CL3" content="Organization" />
+	<classifier name="CL4" content="Howto" />
+      </classifierList>
+    </serviceRack>      
+    <serviceRack name="GS2MGRetrieve">
+      <defaultIndex name="stx" />
+      <indexStem name="gs2mgdemo" />
+    </serviceRack>
+    <serviceRack name="GS2MGSearch">
+      <defaultIndex name="stx" />
+      <indexStem name="gs2mgdemo" />
+      <indexList>
+	<index name="stx" />
+	<index name="stt" />
+	<index name="dtx" />
+      </indexList>
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/README
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/README	(revision 21112)
@@ -0,0 +1,17 @@
+This collection is based on the original demo collection that comes with 
+Greenstone2. I converted the collect.cfg file so that the collection used MGPP,
+then built the collection using Greenstone2, and copied it 
+across to Greenstone3. I ran the convert_coll_from_gs2.pl Perl script to 
+generate the collectionConfig.xml and buildConfig.xml files, then edited 
+them by hand. I also generated new images for the collection, based on its 
+new name.
+
+The changes I made were:
+
+collectionConfig.xml:
+--------------------
+* More readable XML
+
+buildConfig.xml:
+----------------
+* More readable XML
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collect.cfg
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collect.cfg	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collect.cfg	(revision 21112)
@@ -0,0 +1,44 @@
+creator		greenstone@cs.waikato.ac.nz
+maintainer	greenstone@cs.waikato.ac.nz
+public		true
+
+searchtype	form plain
+
+indexes		allfields text dls.Title dls.Subject dls.Organization
+
+levels		section document
+
+plugin		GAPlug 
+plugin		HTMLPlug -description_tags 
+plugin		ArcPlug 
+plugin		RecPlug -use_metadata_files 
+
+classify	AZList -metadata dls.Title 
+classify	Hierarchy -metadata dls.Subject -sort dls.Title 
+classify	AZCompactList -metadata dls.Organization -sort dls.Title -buttonname Organization -mingroup 1 
+classify	List -metadata dls.Keyword -buttonname Howto
+
+format SearchVList "<td valign=top>[link][icon][/link]</td><td>{If}{[parent(All\': \'):Title],[parent(All\': \'):Title]:}[link][Title][/link]</td>"
+
+format DocumentText "<h3>[Title]</h3>\\n\\n<p>[Text]"
+format DocumentImages true
+format DocumentButtons "Expand Text|Expand Contents|Detach|Highlight"
+
+format CL4VList "<td>[link][dls.Keyword][/link]</td>"
+
+collectionmeta	iconcollectionsmall [l=en] "_httpprefix_/collect/gs2mgppdemo/images/gs2mgppdemo_sm.gif"
+collectionmeta	iconcollection [l=en] "_httpprefix_/collect/gs2mgppdemo/images/gs2mgppdemo.gif"
+
+collectionmeta	collectionname [l=en] "Greenstone2 MGPP demo collection"
+collectionmeta	collectionextra [l=en] "This is a demonstration collection 
+for the Greenstone3 digital library software. It contains a small subset 
+(11 books) of the Humanitarian and Development Libraries. It is a Greenstone2 
+collection, built with MGPP, which provides form based searching. 
+The Greenstone2 configuration files have been converted to Greenstone3 
+equivalents, enabling it to be served using Greenstone3."
+
+collectionmeta	.text [l=en] "text"
+collectionmeta	.allfields [l=en] "all fields"
+collectionmeta	.dls.Subject [l=en] "subjects"
+collectionmeta	.dls.Title [l=en] " titles"
+collectionmeta	.dls.Organization [l=en] "organisations"
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/etc/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,36 @@
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+<metadataList>
+<metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name="description" lang="en">This is a demonstration collection  for the Greenstone3 digital library software. It contains a small subset  (11 books) of the Humanitarian and Development Libraries. It is a Greenstone2 collection, built with MGPP, which provides form based searching. The Greenstone2 configuration files have been converted to Greenstone3 equivalents, enabling it to be served using Greenstone3.</displayItem>\n<displayItem name="smallicon" lang="en">gs2mgppdemo_sm.gif</displayItem>
+    <displayItem name="icon" lang="en">gs2mgppdemo.gif</displayItem>
+    <displayItem name="smallicon" lang="en">gs2mgppdemo_sm.gif</displayItem>
+    <displayItem name="name" lang="en">Greenstone2 MGPP demo collection</displayItem>
+  </displayItemList>
+  <search>
+    <index name="idx">
+    </index>
+    <format>
+      <gsf:template match="documentNode">
+	<td valign='top'><gsf:link><gsf:icon/></gsf:link></td>
+	<td><gsf:switch><gsf:metadata name='Title' select='ancestors' separator=': '/><gsf:when test='exists'><gsf:metadata name='Title' select='ancestors' separator=': '/>: </gsf:when></gsf:switch><gsf:link><gsf:metadata name='Title' /></gsf:link></td>
+      </gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CL1">
+    </classifier>
+    <classifier name="CL2">
+    </classifier>
+    <classifier name="CL3">
+    </classifier>
+    <classifier name="CL4">
+      <format>
+	<gsf:template match="documentNode">
+	  <td><gsf:link><gsf:metadata name='dls.Keyword' /></gsf:link></td>
+	</gsf:template>
+      </format>
+    </classifier>
+  </browse>
+</collectionConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/gs2mgppdemo.col
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/gs2mgppdemo.col	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/gs2mgppdemo.col	(revision 21112)
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE GathererCollection [
+  <!ELEMENT GathererCollection (Argument*, GathererConfig, BuildConfig)>
+  <!ELEMENT GathererConfig    (Arguments*)>
+  <!ELEMENT BuildConfig       (Build, Import)>
+  <!ELEMENT Build             (Arguments*)>
+  <!ELEMENT Import            (Arguments*)>
+  <!ELEMENT Argument          (#PCDATA)>
+  <!ATTLIST Argument
+            name              CDATA #REQUIRED
+				enabled           CDATA "true">
+]>
+<GathererCollection>
+    <Argument name="built">false</Argument>
+    <Argument name="imported">false</Argument>
+    <GathererConfig/>
+    <BuildConfig>
+        <Build/>
+        <Import/>
+    </BuildConfig>
+</GathererCollection>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs2mgppdemo/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,41 @@
+<buildConfig>
+  <metadataList>
+    <metadata name="numDocs">11</metadata>
+    <metadata name="buildType">mgpp</metadata>
+  </metadataList>
+  <serviceRackList>
+    <serviceRack name="GS2Browse">
+      <classifierList>
+	<classifier name="CL1" content="Title"/>
+	<classifier name="CL2" content="Subject" />
+	<classifier name="CL3" content="Organization" />
+	<classifier name="CL4" content="Howto" />
+      </classifierList>
+    </serviceRack>      
+    <serviceRack name="GS2MGPPRetrieve">
+      <defaultLevel name="Sec" />
+    </serviceRack>
+    <serviceRack name="GS2MGPPSearch">
+      <defaultLevel name="Sec" />
+      <levelList>
+	<level name="Sec" />
+	<level name="Doc" />
+      </levelList>
+      <fieldList>
+	<field shortname="ZZ" name="allfields" />
+	<field shortname="TX" name="text" />
+	<field shortname="DL" name="dls.Title" />
+	<field shortname="DS" name="dls.Subject" />
+	<field shortname="DO" name="dls.Organization" />
+      </fieldList>
+      <searchTypeList>
+	<searchType name="form" />
+	<searchType name="plain" />
+      </searchTypeList>
+      <defaultIndex name="idx" />
+      <indexList>
+	<index name="idx" />
+      </indexList>
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,83 @@
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
+  <metadataList>
+    <metadata name="creator">greenstone@cs.waikato.ac.nz</metadata>
+    <metadata name="maintainer">greenstone@cs.waikato.ac.nz</metadata>
+  </metadataList>
+  <displayItemList>
+    <displayItem name='name' lang='en'>Greenstone3 MG demo collection</displayItem>
+    <displayItem name='description' lang='en'>This is a demonstration collection for the Greenstone3 digital library software. It contains a small subset (11 books) of the Humanities and Development Libraries. It is built with mg, using the new native Greenstone3 building software.</displayItem>
+    <displayItem name='icon' lang='en'>gs3mgdemo.gif</displayItem>
+    <displayItem name='smallicon' lang='en'>gs3mgdemo_sm.gif</displayItem>
+  </displayItemList>
+  <recognise>
+      <docType name="HTML"/>
+      <docType name="Text"/>
+      <docType name="Metadata"/>
+      <docType name="JPEG"/>
+  </recognise>
+		    
+  <search type="mg" name="mgsearch">
+    <index name="sectext">
+      <field>text</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">chapters</displayItem>
+      <displayItem name='name' lang="fr">chapitres</displayItem>
+      <displayItem name='name' lang="es">capÃ­tulos</displayItem>
+    </index>
+    <index name="doctext">
+      <field>text</field>
+      <level>document</level>
+      <displayItem name='name' lang="en">entire documents</displayItem>
+      <displayItem name='name' lang="fr">documents entiers</displayItem>
+      <displayItem name='name' lang="es">documentos enteros</displayItem>
+    </index>
+    <index name="sectitle">
+      <field>Title</field>
+      <level>section</level>
+      <displayItem name='name' lang="en">section titles</displayItem>
+      <displayItem name='name' lang="fr">titres des sections</displayItem>
+      <displayItem name='name' lang="es">tÃ­tulos de las secciones</displayItem>
+    </index>
+    <format>
+      <gsf:template match="documentNode"><td valign='top'><gsf:link><gsf:icon/></gsf:link></td><td><gsf:metadata name='Title'/></td></gsf:template>
+    </format>
+  </search>
+  <browse>
+    <classifier name="CLTit" type="AZList" horizontalAtTop='true'>
+      <field>Title</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Titles</displayItem>
+    </classifier>
+    <classifier name="CLOrg" type="Hierarchy">
+      <field>Organization</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Organizations</displayItem>
+      <!-- use an absolute URL or one relative to etc directory -->
+      <file URL="org.xml"/>
+    </classifier>
+    <classifier name="CLSubject" type="Hierarchy">
+      <field>Subject</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>Subjects</displayItem>
+      <file URL="sub.xml"/>
+    </classifier>
+    <classifier name="CLKeyword" type="Hierarchy">
+      <field>Keyword</field>
+      <sort>Title</sort>
+      <displayItem name='name' lang='en'>HowTo</displayItem>
+      <file URL="keyword.xml"/>
+      <!--<format>
+        <gsf:template match="documentNode">
+          <br /><gsf:link><gsf:metadata name='Keyword' />
+            </gsf:link></gsf:template>
+      </format>-->
+    </classifier>
+  </browse>
+  <display>
+    <format>
+      <gsf:option name="coverImages" value="false"/>
+      <!--<gsf:option name="documentTOC" value="false"/>-->
+    </format>
+  </display>
+</collectionConfig>
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/keyword.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/keyword.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/keyword.xml	(revision 21112)
@@ -0,0 +1,32 @@
+<Hierarchy>
+  <Classification>
+    <Name>introduce small animals and micro-livestock in your farm</Name>
+    <Path>1</Path>
+    <Description>introduce small animals and micro-livestock in your farm</Description>
+  </Classification>
+  <Classification>
+    <Name>introduce little-known Asian farm animals with a promising future</Name>
+    <Path>2</Path>
+    <Description>introduce little-known Asian farm animals with a promising future</Description>
+  </Classification>
+  <Classification>
+    <Name>utilize the Water Buffalo more effectively</Name>
+    <Path>3</Path>
+    <Description>utilize the Water Buffalo more effectively</Description>
+  </Classification>
+  <Classification>
+    <Name>start a butterfly farm</Name>
+    <Path>4</Path>
+    <Description>start a butterfly farm</Description>
+  </Classification>
+  <Classification>
+    <Name>farm snails</Name>
+    <Path>5</Path>
+    <Description>farm snails</Description>
+  </Classification>
+  <Classification>
+    <Name>achieve gender equality</Name>
+    <Path>6</Path>
+    <Description>achieve gender equality</Description>
+  </Classification>
+</Hierarchy>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/org.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/org.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/org.xml	(revision 21112)
@@ -0,0 +1,84 @@
+<Hierarchy>
+<Classification>
+  <Name>ACCU</Name>
+  <Path>1</Path>
+  <Description>ACCU</Description>
+</Classification>
+<Classification>
+  <Name>Agenda 21</Name>
+  <Path>2</Path>
+  <Description>Agenda 21</Description>
+</Classification>
+<Classification>
+  <Name>Alkire</Name>
+  <Path>3</Path>
+  <Description>Alkire</Description>
+</Classification>
+<Classification>
+  <Name>BOSTID</Name>
+  <Path>4</Path>
+  <Description>BOSTID</Description>
+</Classification>
+<Classification>
+  <Name>Chandra</Name>
+  <Path>5</Path>
+  <Description>Chandra</Description>
+</Classification>
+<Classification>
+  <Name>CPS</Name>
+  <Path>6</Path>
+  <Description>CPS</Description>
+</Classification>
+<Classification>
+  <Name>EC Courier</Name>
+  <Path>7</Path>
+  <Description>EC Courier</Description>
+</Classification>
+<Classification>
+  <Name>FAO</Name>
+  <Path>8</Path>
+  <Description>FAO</Description>
+  <Children>
+    <Classification>
+      <Name>FAO Better Farming series</Name>
+      <Path>8.1</Path>
+      <Description>FAO Better Farming Series</Description>
+    </Classification>
+  </Children>
+</Classification>
+<Classification>
+  <Name>Habitat</Name>
+  <Path>9</Path>
+  <Description>Habitat</Description>
+</Classification>
+<Classification>
+  <Name>Inforse</Name>
+  <Path>10</Path>
+  <Description>Inforse</Description>
+</Classification>
+<Classification>
+  <Name>International Institute for Rural Reconstruction</Name>
+  <Path>12</Path>
+  <Description>International Institute for Rural Reconstruction</Description>
+</Classification>
+<Classification>
+  <Name>NGLS</Name>
+  <Path>13</Path>
+  <Description>NGLS</Description>
+</Classification>
+<Classification>
+  <Name>Peace Corps</Name>
+  <Path>14</Path>
+  <Description>Peace Corps</Description>
+</Classification>
+<Classification>
+  <Name>UNESCO</Name>
+  <Path>15</Path>
+  <Description>UNESCO</Description>
+</Classification>
+<Classification>
+  <Name>World Bank</Name>
+  <Path>16</Path>
+  <Description>World Bank</Description>
+</Classification>
+</Hierarchy>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/sub.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/sub.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/etc/sub.xml	(revision 21112)
@@ -0,0 +1,94 @@
+<Hierarchy>
+  <Classification>
+    <Name>Society, Culture, Community, Woman, Youth, Population</Name>
+    <Path>10</Path>
+    <Description>Society, Culture, Community, Woman, Youth, Population</Description>
+    <Children>
+      <Classification>
+	<Name>Women, gender and development, women&apos;s organizations</Name>
+	<Path>10.5</Path>
+	<Description>Women, gender and development, women&apos;s organizations</Description>
+      </Classification>
+      <Classification>
+	<Name>Social sciences, sociology (works comprising several subgroups) incl. participatory research and training</Name>
+	<Path>10.6</Path>
+	<Description>Social sciences, sociology (works comprising several subgroups) incl. participatory research and training</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Communication, Information and Documentation</Name>
+    <Path>12</Path>
+    <Description>Communication, Information and Documentation</Description>
+    <Children>
+      <Classification>
+	<Name>Communication, telecommunication, mass communication, mass media, film-making</Name>
+	<Path>12.2</Path>
+	<Description>Communication, telecommunication, mass communication, mass media, film-making</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Agriculture and Food Processing</Name>
+    <Path>13</Path>
+    <Description>Agriculture and Food Processing</Description>
+    <Children>
+      <Classification>
+	<Name>Better Farming series of FAO and INADES - 46 booklets</Name>
+	<Path>13.8</Path>
+	<Description>Better Farming series of FAO and INADES - 46 booklets</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Animal Husbandry and Animal Product Processing</Name>
+    <Path>14</Path>
+    <Description>Animal Husbandry and Animal Product Processing</Description>
+    <Children>
+      <Classification>
+	<Name>Veterinary medicine, animal diseases</Name>
+	<Path>14.2</Path>
+	<Description>Veterinary medicine, animal diseases</Description>
+      </Classification>
+      <Classification>
+	<Name>Cattle</Name>
+	<Path>14.5</Path>
+	<Description>Cattle</Description>
+      </Classification>
+      <Classification>
+	<Name>Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)</Name>
+	<Path>14.6</Path>
+	<Description>Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Settlements, Housing, Building - Infrastructure Construction (Roads etc)</Name>
+    <Path>15</Path>
+    <Description>Settlements, Housing, Building - Infrastructure Construction (Roads etc)</Description>
+    <Children>
+      <Classification>
+	<Name>Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.</Name>
+	<Path>15.3</Path>
+	<Description>Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.</Description>
+      </Classification>
+    </Children>
+  </Classification>
+  <Classification>
+    <Name>Development Periodicals and Magazines</Name>
+    <Path>16</Path>
+    <Description>Development Periodicals and Magazines</Description>
+    <Children>
+      <Classification>
+	<Name>Sustainable Energy Newsletters of the INFORSE network</Name>
+	<Path>16.1</Path>
+	<Description>Sustainable Energy Newsletters of the INFORSE network</Description>
+      </Classification>
+      <Classification>
+	<Name>The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union</Name>
+	<Path>16.2</Path>
+	<Description>The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union</Description>
+      </Classification>
+    </Children>
+  </Classification>
+</Hierarchy>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,1 @@
+<buildConfig><metadataList /><serviceRackList><serviceRack name="GS3Browse"><classifierList><classifier name="CLTit" /><classifier name="CLOrg" /><classifier name="CLSubject" /><classifier name="CLKeyword" /></classifierList></serviceRack><serviceRack name="GS3MGSearch"><indexStem name="index"/><indexList><index name="sectext" /><index name="doctext" /><index name="sectitle" /></indexList><defaultIndex name="sectext" /><baseIndexPrefix name="mgsearch" /></serviceRack><serviceRack name="GS3MGRetrieve"><defaultIndex name="sectext" /><indexStem name="index"/><baseIndexPrefix name="mgsearch" /></serviceRack></serviceRackList></buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/gs3mgdemo/mysqldatadump.sql	(revision 21112)
@@ -0,0 +1,2812 @@
+-- MySQL dump 9.11
+--
+-- Host: localhost    Database: localsite_gs3mgdemo
+-- ------------------------------------------------------
+-- Server version	4.0.20-standard
+
+--
+-- Table structure for table `build`
+--
+
+CREATE TABLE build (
+  buildRef int(11) NOT NULL auto_increment,
+  buildKey varchar(64) default NULL,
+  buildValue tinytext,
+  PRIMARY KEY  (buildRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `build`
+--
+
+INSERT INTO build VALUES (1,'NextSeqNo','7');
+INSERT INTO build VALUES (2,'lastBuildDate','20050201');
+
+--
+-- Table structure for table `classdata`
+--
+
+CREATE TABLE classdata (
+  ClassifyRef int(11) default NULL,
+  Label tinytext,
+  Value tinytext
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classdata`
+--
+
+
+--
+-- Table structure for table `classdocuments`
+--
+
+CREATE TABLE classdocuments (
+  ClassifyRef int(11) default NULL,
+  DocID tinytext,
+  DocOrder int(11) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classdocuments`
+--
+
+INSERT INTO classdocuments VALUES (2,'20050201:5',1);
+INSERT INTO classdocuments VALUES (3,'20050201:2',1);
+INSERT INTO classdocuments VALUES (4,'20050201:3',1);
+INSERT INTO classdocuments VALUES (4,'20050201:4',2);
+INSERT INTO classdocuments VALUES (5,'20050201:1',1);
+INSERT INTO classdocuments VALUES (10,'20050201:1',1);
+INSERT INTO classdocuments VALUES (10,'20050201:2',2);
+INSERT INTO classdocuments VALUES (15,'20050201:3',1);
+INSERT INTO classdocuments VALUES (15,'20050201:4',2);
+INSERT INTO classdocuments VALUES (22,'20050201:5',1);
+INSERT INTO classdocuments VALUES (25,'20050201:5',1);
+INSERT INTO classdocuments VALUES (30,'20050201:3',1);
+INSERT INTO classdocuments VALUES (30,'20050201:4',2);
+INSERT INTO classdocuments VALUES (34,'20050201:1',1);
+INSERT INTO classdocuments VALUES (34,'20050201:2',2);
+INSERT INTO classdocuments VALUES (36,'20050201:3',1);
+INSERT INTO classdocuments VALUES (36,'20050201:4',2);
+INSERT INTO classdocuments VALUES (41,'20050201:2',1);
+INSERT INTO classdocuments VALUES (42,'20050201:1',1);
+INSERT INTO classdocuments VALUES (45,'20050201:4',1);
+INSERT INTO classdocuments VALUES (46,'20050201:5',1);
+
+--
+-- Table structure for table `classifiers`
+--
+
+CREATE TABLE classifiers (
+  ClassifyRef int(11) NOT NULL auto_increment,
+  ClassifyID tinytext,
+  ParentID tinytext,
+  ClassifyOrder tinytext,
+  Name tinytext,
+  Description tinytext,
+  NumLeafDocs int(11) default NULL,
+  PRIMARY KEY  (ClassifyRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `classifiers`
+--
+
+INSERT INTO classifiers VALUES (1,'CLTit','','0','','Classifier',5);
+INSERT INTO classifiers VALUES (2,'CLTit.d','CLTit','1','d','d',1);
+INSERT INTO classifiers VALUES (3,'CLTit.m','CLTit','2','m','m',1);
+INSERT INTO classifiers VALUES (4,'CLTit.f','CLTit','3','f','f',2);
+INSERT INTO classifiers VALUES (5,'CLTit.l','CLTit','4','l','l',1);
+INSERT INTO classifiers VALUES (6,'CLOrg','','0','','Classifier',5);
+INSERT INTO classifiers VALUES (7,'CLOrg.1','CLOrg','1','ACCU','ACCU',0);
+INSERT INTO classifiers VALUES (8,'CLOrg.2','CLOrg','2','Agenda 21','Agenda 21',0);
+INSERT INTO classifiers VALUES (9,'CLOrg.3','CLOrg','3','Alkire','Alkire',0);
+INSERT INTO classifiers VALUES (10,'CLOrg.4','CLOrg','4','BOSTID','BOSTID',2);
+INSERT INTO classifiers VALUES (11,'CLOrg.5','CLOrg','5','Chandra','Chandra',0);
+INSERT INTO classifiers VALUES (12,'CLOrg.6','CLOrg','6','CPS','CPS',0);
+INSERT INTO classifiers VALUES (13,'CLOrg.7','CLOrg','7','EC Courier','EC Courier',0);
+INSERT INTO classifiers VALUES (14,'CLOrg.8','CLOrg','8','FAO','FAO',2);
+INSERT INTO classifiers VALUES (15,'CLOrg.8.1','CLOrg.8','1','FAO Better Farming series','FAO Better Farming Series',2);
+INSERT INTO classifiers VALUES (16,'CLOrg.9','CLOrg','9','Habitat','Habitat',0);
+INSERT INTO classifiers VALUES (17,'CLOrg.10','CLOrg','10','Inforse','Inforse',0);
+INSERT INTO classifiers VALUES (18,'CLOrg.12','CLOrg','12','International Institute for Rural Reconstruction','International Institute for Rural Reconstruction',0);
+INSERT INTO classifiers VALUES (19,'CLOrg.13','CLOrg','13','NGLS','NGLS',0);
+INSERT INTO classifiers VALUES (20,'CLOrg.14','CLOrg','14','Peace Corps','Peace Corps',0);
+INSERT INTO classifiers VALUES (21,'CLOrg.15','CLOrg','15','UNESCO','UNESCO',0);
+INSERT INTO classifiers VALUES (22,'CLOrg.16','CLOrg','16','World Bank','World Bank',1);
+INSERT INTO classifiers VALUES (23,'CLSubject','','0','','Classifier',7);
+INSERT INTO classifiers VALUES (24,'CLSubject.10','CLSubject','10','Society, Culture, Community, Woman, Youth, Population','Society, Culture, Community, Woman, Youth, Population',1);
+INSERT INTO classifiers VALUES (25,'CLSubject.10.5','CLSubject.10','5','Women, gender and development, women\'s organizations','Women, gender and development, women\'s organizations',1);
+INSERT INTO classifiers VALUES (26,'CLSubject.10.6','CLSubject.10','6','Social sciences, sociology (works comprising several subgroups) incl. participatory research and training','Social sciences, sociology (works comprising several subgroups) incl. participatory research and training',0);
+INSERT INTO classifiers VALUES (27,'CLSubject.12','CLSubject','12','Communication, Information and Documentation','Communication, Information and Documentation',0);
+INSERT INTO classifiers VALUES (28,'CLSubject.12.2','CLSubject.12','2','Communication, telecommunication, mass communication, mass media, film-making','Communication, telecommunication, mass communication, mass media, film-making',0);
+INSERT INTO classifiers VALUES (29,'CLSubject.13','CLSubject','13','Agriculture and Food Processing','Agriculture and Food Processing',2);
+INSERT INTO classifiers VALUES (30,'CLSubject.13.8','CLSubject.13','8','Better Farming series of FAO and INADES - 46 booklets','Better Farming series of FAO and INADES - 46 booklets',2);
+INSERT INTO classifiers VALUES (31,'CLSubject.14','CLSubject','14','Animal Husbandry and Animal Product Processing','Animal Husbandry and Animal Product Processing',2);
+INSERT INTO classifiers VALUES (32,'CLSubject.14.2','CLSubject.14','2','Veterinary medicine, animal diseases','Veterinary medicine, animal diseases',0);
+INSERT INTO classifiers VALUES (33,'CLSubject.14.5','CLSubject.14','5','Cattle','Cattle',0);
+INSERT INTO classifiers VALUES (34,'CLSubject.14.6','CLSubject.14','6','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)',2);
+INSERT INTO classifiers VALUES (35,'CLSubject.15','CLSubject','15','Settlements, Housing, Building - Infrastructure Construction (Roads etc)','Settlements, Housing, Building - Infrastructure Construction (Roads etc)',2);
+INSERT INTO classifiers VALUES (36,'CLSubject.15.3','CLSubject.15','3','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.',2);
+INSERT INTO classifiers VALUES (37,'CLSubject.16','CLSubject','16','Development Periodicals and Magazines','Development Periodicals and Magazines',0);
+INSERT INTO classifiers VALUES (38,'CLSubject.16.1','CLSubject.16','1','Sustainable Energy Newsletters of the INFORSE network','Sustainable Energy Newsletters of the INFORSE network',0);
+INSERT INTO classifiers VALUES (39,'CLSubject.16.2','CLSubject.16','2','The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union','The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union',0);
+INSERT INTO classifiers VALUES (40,'CLKeyword','','0','','Classifier',4);
+INSERT INTO classifiers VALUES (41,'CLKeyword.1','CLKeyword','1','introduce small animals and micro-livestock in your farm','introduce small animals and micro-livestock in your farm',1);
+INSERT INTO classifiers VALUES (42,'CLKeyword.2','CLKeyword','2','introduce little-known Asian farm animals with a promising future','introduce little-known Asian farm animals with a promising future',1);
+INSERT INTO classifiers VALUES (43,'CLKeyword.3','CLKeyword','3','utilize the Water Buffalo more effectively','utilize the Water Buffalo more effectively',0);
+INSERT INTO classifiers VALUES (44,'CLKeyword.4','CLKeyword','4','start a butterfly farm','start a butterfly farm',0);
+INSERT INTO classifiers VALUES (45,'CLKeyword.5','CLKeyword','5','farm snails','farm snails',1);
+INSERT INTO classifiers VALUES (46,'CLKeyword.6','CLKeyword','6','achieve gender equality','achieve gender equality',1);
+
+--
+-- Table structure for table `divisionfilerefs`
+--
+
+CREATE TABLE divisionfilerefs (
+  DocID varchar(64) default NULL,
+  DivisionRef int(11) default NULL,
+  DivisionType varchar(16) default NULL,
+  FileID varchar(32) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisionfilerefs`
+--
+
+INSERT INTO divisionfilerefs VALUES ('20050201:1',1,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',2,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',3,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',4,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',5,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',6,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',7,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',8,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',9,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',10,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',11,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',12,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',13,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',14,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',15,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',16,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',17,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',18,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',19,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',20,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',21,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',22,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',23,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',24,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',25,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',26,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',27,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',28,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',30,'Group','FILEGROUP_PRELUDE51');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',31,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',32,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',33,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',34,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',35,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',36,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',37,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',38,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',39,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',40,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',41,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',42,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',43,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',44,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',45,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',46,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',47,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',48,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',49,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',50,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',51,'Group','FILEGROUP_PRELUDE32');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',52,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',53,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',54,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',55,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',56,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',57,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',58,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',59,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',60,'Group','FILEGROUP_PRELUDE29');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',61,'Group','FILEGROUP_PRELUDE30');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',62,'Group','FILEGROUP_PRELUDE31');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',63,'Group','FILEGROUP_PRELUDE40');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',64,'Group','FILEGROUP_PRELUDE33');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',65,'Group','FILEGROUP_PRELUDE34');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',66,'Group','FILEGROUP_PRELUDE35');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',67,'Group','FILEGROUP_PRELUDE36');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',68,'Group','FILEGROUP_PRELUDE37');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',69,'Group','FILEGROUP_PRELUDE38');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',70,'Group','FILEGROUP_PRELUDE39');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',71,'Group','FILEGROUP_PRELUDE43');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',72,'Group','FILEGROUP_PRELUDE41');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',73,'Group','FILEGROUP_PRELUDE42');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',74,'Group','FILEGROUP_PRELUDE45');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',75,'Group','FILEGROUP_PRELUDE44');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',76,'Group','FILEGROUP_PRELUDE49');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',77,'Group','FILEGROUP_PRELUDE46');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',78,'Group','FILEGROUP_PRELUDE47');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',79,'Group','FILEGROUP_PRELUDE48');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',80,'Group','FILEGROUP_PRELUDE50');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.65');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.68');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.67');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.66');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.40');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.41');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.42');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.43');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.44');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.45');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.46');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.47');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.48');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.49');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.50');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.51');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.52');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.53');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.54');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.55');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.56');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.57');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.58');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.59');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.60');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.61');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.69');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.62');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.63');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.64');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',82,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',83,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',84,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',85,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',86,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',87,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',88,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',89,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',90,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',91,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',92,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',93,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',94,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',95,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',96,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',97,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',98,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',99,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',100,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',101,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',102,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',103,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.79');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.78');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.77');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.76');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.75');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.74');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.73');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.72');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.71');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.70');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.69');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.68');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.67');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.66');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.65');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.64');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.63');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.62');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.61');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.60');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.59');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.58');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.57');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.56');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.55');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.54');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.53');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.52');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.51');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.50');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.49');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.48');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.47');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.46');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.45');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.44');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.43');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.42');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.41');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.40');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',105,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',106,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',107,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',108,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',109,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',110,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',111,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',112,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',113,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',114,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',115,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',116,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',117,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',118,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',119,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',120,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',121,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',122,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',123,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',124,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',125,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',126,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',127,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.35');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.38');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.37');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.36');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.17');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.16');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.18');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.19');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.20');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.21');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.22');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.23');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.24');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.25');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.26');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.27');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.30');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.39');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.32');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.33');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.34');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.12');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',129,'Group','FILEGROUP_PRELUDE46');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',130,'Group','FILEGROUP_PRELUDE1');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',131,'Group','FILEGROUP_PRELUDE2');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',132,'Group','FILEGROUP_PRELUDE9');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',133,'Group','FILEGROUP_PRELUDE3');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',134,'Group','FILEGROUP_PRELUDE4');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',135,'Group','FILEGROUP_PRELUDE5');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',136,'Group','FILEGROUP_PRELUDE6');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',137,'Group','FILEGROUP_PRELUDE7');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',138,'Group','FILEGROUP_PRELUDE8');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',139,'Group','FILEGROUP_PRELUDE10');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',140,'Group','FILEGROUP_PRELUDE15');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',141,'Group','FILEGROUP_PRELUDE11');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',142,'Group','FILEGROUP_PRELUDE12');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',143,'Group','FILEGROUP_PRELUDE13');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',144,'Group','FILEGROUP_PRELUDE14');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',145,'Group','FILEGROUP_PRELUDE26');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',146,'Group','FILEGROUP_PRELUDE16');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',147,'Group','FILEGROUP_PRELUDE17');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',148,'Group','FILEGROUP_PRELUDE18');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',149,'Group','FILEGROUP_PRELUDE19');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',150,'Group','FILEGROUP_PRELUDE20');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',151,'Group','FILEGROUP_PRELUDE21');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',152,'Group','FILEGROUP_PRELUDE22');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',153,'Group','FILEGROUP_PRELUDE23');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',154,'Group','FILEGROUP_PRELUDE24');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',155,'Group','FILEGROUP_PRELUDE25');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',156,'Group','FILEGROUP_PRELUDE43');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',157,'Group','FILEGROUP_PRELUDE27');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',158,'Group','FILEGROUP_PRELUDE28');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',159,'Group','FILEGROUP_PRELUDE29');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',160,'Group','FILEGROUP_PRELUDE30');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',161,'Group','FILEGROUP_PRELUDE31');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',162,'Group','FILEGROUP_PRELUDE32');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',163,'Group','FILEGROUP_PRELUDE33');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',164,'Group','FILEGROUP_PRELUDE34');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',165,'Group','FILEGROUP_PRELUDE35');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',166,'Group','FILEGROUP_PRELUDE36');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',167,'Group','FILEGROUP_PRELUDE37');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',168,'Group','FILEGROUP_PRELUDE38');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',169,'Group','FILEGROUP_PRELUDE39');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',170,'Group','FILEGROUP_PRELUDE40');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',171,'Group','FILEGROUP_PRELUDE41');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',172,'Group','FILEGROUP_PRELUDE42');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',173,'Group','FILEGROUP_PRELUDE44');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',174,'Group','FILEGROUP_PRELUDE45');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.11');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.9');
+INSERT INTO divisionfilerefs VALUES ('20050201:6',176,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.14');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.31');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.29');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.13');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.7');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.8');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.6');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.2');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.80');
+INSERT INTO divisionfilerefs VALUES ('20050201:5',175,'Group','default.10');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.15');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:3',104,'Group','default.5');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.4');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.1');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.28');
+INSERT INTO divisionfilerefs VALUES ('20050201:2',81,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:4',128,'Group','default.3');
+INSERT INTO divisionfilerefs VALUES ('20050201:1',29,'Group','default.27');
+
+--
+-- Table structure for table `divisionmetarefs`
+--
+
+CREATE TABLE divisionmetarefs (
+  DocID varchar(64) default NULL,
+  DivisionRef int(11) default NULL,
+  DivisionType varchar(16) default NULL,
+  MetaID varchar(64) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisionmetarefs`
+--
+
+INSERT INTO divisionmetarefs VALUES ('20050201:1',1,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',2,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',3,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',4,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',5,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',6,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',7,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',8,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',9,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',10,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',11,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',12,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',13,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',14,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',15,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',16,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',17,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',18,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',19,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',20,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',21,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',22,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',23,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',24,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',25,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',26,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',27,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',28,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:1',29,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',30,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',31,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',32,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',33,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',34,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',35,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',36,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',37,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',38,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',39,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',40,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',41,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',42,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',43,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',44,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',45,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',46,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',47,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',48,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',49,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',50,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',51,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',52,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',53,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',54,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',55,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',56,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',57,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',58,'Group','DM30');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',59,'Group','DM31');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',60,'Group','DM32');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',61,'Group','DM33');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',62,'Group','DM34');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',63,'Group','DM35');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',64,'Group','DM36');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',65,'Group','DM37');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',66,'Group','DM38');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',67,'Group','DM39');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',68,'Group','DM40');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',69,'Group','DM41');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',70,'Group','DM42');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',71,'Group','DM43');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',72,'Group','DM44');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',73,'Group','DM45');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',74,'Group','DM46');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',75,'Group','DM47');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',76,'Group','DM48');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',77,'Group','DM49');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',78,'Group','DM50');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',79,'Group','DM51');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',80,'Group','DM52');
+INSERT INTO divisionmetarefs VALUES ('20050201:2',81,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',82,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',83,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',84,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',85,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',86,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',87,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',88,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',89,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',90,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',91,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',92,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',93,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',94,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',95,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',96,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',97,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',98,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',99,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',100,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',101,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',102,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',103,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:3',104,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',105,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',106,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',107,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',108,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',109,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',110,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',111,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',112,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',113,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',114,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',115,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',116,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',117,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',118,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',119,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',120,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',121,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',122,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',123,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',124,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',125,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',126,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',127,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:4',128,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',129,'Group','DM2');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',130,'Group','DM3');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',131,'Group','DM4');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',132,'Group','DM5');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',133,'Group','DM6');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',134,'Group','DM7');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',135,'Group','DM8');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',136,'Group','DM9');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',137,'Group','DM10');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',138,'Group','DM11');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',139,'Group','DM12');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',140,'Group','DM13');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',141,'Group','DM14');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',142,'Group','DM15');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',143,'Group','DM16');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',144,'Group','DM17');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',145,'Group','DM18');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',146,'Group','DM19');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',147,'Group','DM20');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',148,'Group','DM21');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',149,'Group','DM22');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',150,'Group','DM23');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',151,'Group','DM24');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',152,'Group','DM25');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',153,'Group','DM26');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',154,'Group','DM27');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',155,'Group','DM28');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',156,'Group','DM29');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',157,'Group','DM30');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',158,'Group','DM31');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',159,'Group','DM32');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',160,'Group','DM33');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',161,'Group','DM34');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',162,'Group','DM35');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',163,'Group','DM36');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',164,'Group','DM37');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',165,'Group','DM38');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',166,'Group','DM39');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',167,'Group','DM40');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',168,'Group','DM41');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',169,'Group','DM42');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',170,'Group','DM43');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',171,'Group','DM44');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',172,'Group','DM45');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',173,'Group','DM46');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',174,'Group','DM47');
+INSERT INTO divisionmetarefs VALUES ('20050201:5',175,'Group','DM1');
+INSERT INTO divisionmetarefs VALUES ('20050201:6',176,'Group','DM1');
+
+--
+-- Table structure for table `divisions`
+--
+
+CREATE TABLE divisions (
+  DivisionRef int(11) NOT NULL auto_increment,
+  StructureRef int(11) default NULL,
+  DocID varchar(64) default NULL,
+  ParentType varchar(64) default NULL,
+  ParentRef varchar(64) default NULL,
+  SectionID varchar(64) default NULL,
+  DivisionType varchar(64) default NULL,
+  LabelOrder varchar(64) default NULL,
+  ShortLabel varchar(128) default NULL,
+  UserLabel varchar(255) default NULL,
+  PRIMARY KEY  (DivisionRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `divisions`
+--
+
+INSERT INTO divisions VALUES (1,NULL,'20050201:1','Structure','1','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (2,NULL,'20050201:1','Division','1','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (3,NULL,'20050201:1','Division','1','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (4,NULL,'20050201:1','Division','1','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (5,NULL,'20050201:1','Division','1','DS4','Section','4','1.4','1.4');
+INSERT INTO divisions VALUES (6,NULL,'20050201:1','Division','5','DS5','Section','5','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (7,NULL,'20050201:1','Division','5','DS6','Section','6','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (8,NULL,'20050201:1','Division','5','DS7','Section','7','1.4.3','1.4.3');
+INSERT INTO divisions VALUES (9,NULL,'20050201:1','Division','5','DS8','Section','8','1.4.4','1.4.4');
+INSERT INTO divisions VALUES (10,NULL,'20050201:1','Division','5','DS9','Section','9','1.4.5','1.4.5');
+INSERT INTO divisions VALUES (11,NULL,'20050201:1','Division','1','DS10','Section','10','1.5','1.5');
+INSERT INTO divisions VALUES (12,NULL,'20050201:1','Division','11','DS11','Section','11','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (13,NULL,'20050201:1','Division','11','DS12','Section','12','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (14,NULL,'20050201:1','Division','11','DS13','Section','13','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (15,NULL,'20050201:1','Division','11','DS14','Section','14','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (16,NULL,'20050201:1','Division','11','DS15','Section','15','1.5.5','1.5.5');
+INSERT INTO divisions VALUES (17,NULL,'20050201:1','Division','1','DS16','Section','16','1.6','1.6');
+INSERT INTO divisions VALUES (18,NULL,'20050201:1','Division','17','DS17','Section','17','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (19,NULL,'20050201:1','Division','17','DS18','Section','18','1.6.2','1.6.2');
+INSERT INTO divisions VALUES (20,NULL,'20050201:1','Division','17','DS19','Section','19','1.6.3','1.6.3');
+INSERT INTO divisions VALUES (21,NULL,'20050201:1','Division','17','DS20','Section','20','1.6.4','1.6.4');
+INSERT INTO divisions VALUES (22,NULL,'20050201:1','Division','17','DS21','Section','21','1.6.5','1.6.5');
+INSERT INTO divisions VALUES (23,NULL,'20050201:1','Division','1','DS22','Section','22','1.7','1.7');
+INSERT INTO divisions VALUES (24,NULL,'20050201:1','Division','1','DS23','Section','23','1.8','1.8');
+INSERT INTO divisions VALUES (25,NULL,'20050201:1','Division','24','DS24','Section','24','1.8.1','1.8.1');
+INSERT INTO divisions VALUES (26,NULL,'20050201:1','Division','24','DS25','Section','25','1.8.2','1.8.2');
+INSERT INTO divisions VALUES (27,NULL,'20050201:1','Division','1','DS26','Section','26','1.9','1.9');
+INSERT INTO divisions VALUES (28,NULL,'20050201:1','Division','1','DS27','Section','27','1.10','1.10');
+INSERT INTO divisions VALUES (29,NULL,'20050201:1','Structure','2','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (30,NULL,'20050201:2','Structure','3','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (31,NULL,'20050201:2','Division','30','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (32,NULL,'20050201:2','Division','30','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (33,NULL,'20050201:2','Division','30','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (34,NULL,'20050201:2','Division','30','DS4','Section','4','1.4','1.4');
+INSERT INTO divisions VALUES (35,NULL,'20050201:2','Division','34','DS5','Section','5','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (36,NULL,'20050201:2','Division','34','DS6','Section','6','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (37,NULL,'20050201:2','Division','34','DS7','Section','7','1.4.3','1.4.3');
+INSERT INTO divisions VALUES (38,NULL,'20050201:2','Division','34','DS8','Section','8','1.4.4','1.4.4');
+INSERT INTO divisions VALUES (39,NULL,'20050201:2','Division','30','DS9','Section','9','1.5','1.5');
+INSERT INTO divisions VALUES (40,NULL,'20050201:2','Division','39','DS10','Section','10','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (41,NULL,'20050201:2','Division','39','DS11','Section','11','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (42,NULL,'20050201:2','Division','39','DS12','Section','12','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (43,NULL,'20050201:2','Division','39','DS13','Section','13','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (44,NULL,'20050201:2','Division','39','DS14','Section','14','1.5.5','1.5.5');
+INSERT INTO divisions VALUES (45,NULL,'20050201:2','Division','39','DS15','Section','15','1.5.6','1.5.6');
+INSERT INTO divisions VALUES (46,NULL,'20050201:2','Division','39','DS16','Section','16','1.5.7','1.5.7');
+INSERT INTO divisions VALUES (47,NULL,'20050201:2','Division','39','DS17','Section','17','1.5.8','1.5.8');
+INSERT INTO divisions VALUES (48,NULL,'20050201:2','Division','39','DS18','Section','18','1.5.9','1.5.9');
+INSERT INTO divisions VALUES (49,NULL,'20050201:2','Division','30','DS19','Section','19','1.6','1.6');
+INSERT INTO divisions VALUES (50,NULL,'20050201:2','Division','49','DS20','Section','20','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (51,NULL,'20050201:2','Division','30','DS21','Section','21','1.7','1.7');
+INSERT INTO divisions VALUES (52,NULL,'20050201:2','Division','51','DS22','Section','22','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (53,NULL,'20050201:2','Division','51','DS23','Section','23','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (54,NULL,'20050201:2','Division','51','DS24','Section','24','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (55,NULL,'20050201:2','Division','51','DS25','Section','25','1.7.4','1.7.4');
+INSERT INTO divisions VALUES (56,NULL,'20050201:2','Division','51','DS26','Section','26','1.7.5','1.7.5');
+INSERT INTO divisions VALUES (57,NULL,'20050201:2','Division','51','DS27','Section','27','1.7.6','1.7.6');
+INSERT INTO divisions VALUES (58,NULL,'20050201:2','Division','51','DS28','Section','28','1.7.7','1.7.7');
+INSERT INTO divisions VALUES (59,NULL,'20050201:2','Division','51','DS29','Section','29','1.7.8','1.7.8');
+INSERT INTO divisions VALUES (60,NULL,'20050201:2','Division','51','DS30','Section','30','1.7.9','1.7.9');
+INSERT INTO divisions VALUES (61,NULL,'20050201:2','Division','51','DS31','Section','31','1.7.10','1.7.10');
+INSERT INTO divisions VALUES (62,NULL,'20050201:2','Division','51','DS32','Section','32','1.7.11','1.7.11');
+INSERT INTO divisions VALUES (63,NULL,'20050201:2','Division','30','DS33','Section','33','1.8','1.8');
+INSERT INTO divisions VALUES (64,NULL,'20050201:2','Division','63','DS34','Section','34','1.8.1','1.8.1');
+INSERT INTO divisions VALUES (65,NULL,'20050201:2','Division','63','DS35','Section','35','1.8.2','1.8.2');
+INSERT INTO divisions VALUES (66,NULL,'20050201:2','Division','63','DS36','Section','36','1.8.3','1.8.3');
+INSERT INTO divisions VALUES (67,NULL,'20050201:2','Division','63','DS37','Section','37','1.8.4','1.8.4');
+INSERT INTO divisions VALUES (68,NULL,'20050201:2','Division','63','DS38','Section','38','1.8.5','1.8.5');
+INSERT INTO divisions VALUES (69,NULL,'20050201:2','Division','63','DS39','Section','39','1.8.6','1.8.6');
+INSERT INTO divisions VALUES (70,NULL,'20050201:2','Division','63','DS40','Section','40','1.8.7','1.8.7');
+INSERT INTO divisions VALUES (71,NULL,'20050201:2','Division','30','DS41','Section','41','1.9','1.9');
+INSERT INTO divisions VALUES (72,NULL,'20050201:2','Division','71','DS42','Section','42','1.9.1','1.9.1');
+INSERT INTO divisions VALUES (73,NULL,'20050201:2','Division','71','DS43','Section','43','1.9.2','1.9.2');
+INSERT INTO divisions VALUES (74,NULL,'20050201:2','Division','30','DS44','Section','44','1.10','1.10');
+INSERT INTO divisions VALUES (75,NULL,'20050201:2','Division','74','DS45','Section','45','1.10.1','1.10.1');
+INSERT INTO divisions VALUES (76,NULL,'20050201:2','Division','30','DS46','Section','46','1.11','1.11');
+INSERT INTO divisions VALUES (77,NULL,'20050201:2','Division','76','DS47','Section','47','1.11.1','1.11.1');
+INSERT INTO divisions VALUES (78,NULL,'20050201:2','Division','76','DS48','Section','48','1.11.2','1.11.2');
+INSERT INTO divisions VALUES (79,NULL,'20050201:2','Division','76','DS49','Section','49','1.11.3','1.11.3');
+INSERT INTO divisions VALUES (80,NULL,'20050201:2','Division','30','DS50','Section','50','1.12','1.12');
+INSERT INTO divisions VALUES (81,NULL,'20050201:2','Structure','4','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (82,NULL,'20050201:3','Structure','5','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (83,NULL,'20050201:3','Division','82','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (84,NULL,'20050201:3','Division','83','DS2','Section','2','1.1.1','1.1.1');
+INSERT INTO divisions VALUES (85,NULL,'20050201:3','Division','82','DS3','Section','3','1.2','1.2');
+INSERT INTO divisions VALUES (86,NULL,'20050201:3','Division','85','DS4','Section','4','1.2.1','1.2.1');
+INSERT INTO divisions VALUES (87,NULL,'20050201:3','Division','85','DS5','Section','5','1.2.2','1.2.2');
+INSERT INTO divisions VALUES (88,NULL,'20050201:3','Division','85','DS6','Section','6','1.2.3','1.2.3');
+INSERT INTO divisions VALUES (89,NULL,'20050201:3','Division','85','DS7','Section','7','1.2.4','1.2.4');
+INSERT INTO divisions VALUES (90,NULL,'20050201:3','Division','85','DS8','Section','8','1.2.5','1.2.5');
+INSERT INTO divisions VALUES (91,NULL,'20050201:3','Division','85','DS9','Section','9','1.2.6','1.2.6');
+INSERT INTO divisions VALUES (92,NULL,'20050201:3','Division','85','DS10','Section','10','1.2.7','1.2.7');
+INSERT INTO divisions VALUES (93,NULL,'20050201:3','Division','85','DS11','Section','11','1.2.8','1.2.8');
+INSERT INTO divisions VALUES (94,NULL,'20050201:3','Division','85','DS12','Section','12','1.2.9','1.2.9');
+INSERT INTO divisions VALUES (95,NULL,'20050201:3','Division','85','DS13','Section','13','1.2.10','1.2.10');
+INSERT INTO divisions VALUES (96,NULL,'20050201:3','Division','85','DS14','Section','14','1.2.11','1.2.11');
+INSERT INTO divisions VALUES (97,NULL,'20050201:3','Division','85','DS15','Section','15','1.2.12','1.2.12');
+INSERT INTO divisions VALUES (98,NULL,'20050201:3','Division','82','DS16','Section','16','1.3','1.3');
+INSERT INTO divisions VALUES (99,NULL,'20050201:3','Division','98','DS17','Section','17','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (100,NULL,'20050201:3','Division','98','DS18','Section','18','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (101,NULL,'20050201:3','Division','82','DS19','Section','19','1.4','1.4');
+INSERT INTO divisions VALUES (102,NULL,'20050201:3','Division','82','DS20','Section','20','1.5','1.5');
+INSERT INTO divisions VALUES (103,NULL,'20050201:3','Division','102','DS21','Section','21','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (104,NULL,'20050201:3','Structure','6','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (105,NULL,'20050201:4','Structure','7','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (106,NULL,'20050201:4','Division','105','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (107,NULL,'20050201:4','Division','105','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (108,NULL,'20050201:4','Division','107','DS3','Section','3','1.2.1','1.2.1');
+INSERT INTO divisions VALUES (109,NULL,'20050201:4','Division','107','DS4','Section','4','1.2.2','1.2.2');
+INSERT INTO divisions VALUES (110,NULL,'20050201:4','Division','107','DS5','Section','5','1.2.3','1.2.3');
+INSERT INTO divisions VALUES (111,NULL,'20050201:4','Division','105','DS6','Section','6','1.3','1.3');
+INSERT INTO divisions VALUES (112,NULL,'20050201:4','Division','111','DS7','Section','7','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (113,NULL,'20050201:4','Division','111','DS8','Section','8','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (114,NULL,'20050201:4','Division','111','DS9','Section','9','1.3.3','1.3.3');
+INSERT INTO divisions VALUES (115,NULL,'20050201:4','Division','111','DS10','Section','10','1.3.4','1.3.4');
+INSERT INTO divisions VALUES (116,NULL,'20050201:4','Division','105','DS11','Section','11','1.4','1.4');
+INSERT INTO divisions VALUES (117,NULL,'20050201:4','Division','116','DS12','Section','12','1.4.1','1.4.1');
+INSERT INTO divisions VALUES (118,NULL,'20050201:4','Division','116','DS13','Section','13','1.4.2','1.4.2');
+INSERT INTO divisions VALUES (119,NULL,'20050201:4','Division','105','DS14','Section','14','1.5','1.5');
+INSERT INTO divisions VALUES (120,NULL,'20050201:4','Division','119','DS15','Section','15','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (121,NULL,'20050201:4','Division','119','DS16','Section','16','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (122,NULL,'20050201:4','Division','119','DS17','Section','17','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (123,NULL,'20050201:4','Division','105','DS18','Section','18','1.6','1.6');
+INSERT INTO divisions VALUES (124,NULL,'20050201:4','Division','105','DS19','Section','19','1.7','1.7');
+INSERT INTO divisions VALUES (125,NULL,'20050201:4','Division','124','DS20','Section','20','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (126,NULL,'20050201:4','Division','124','DS21','Section','21','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (127,NULL,'20050201:4','Division','124','DS22','Section','22','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (128,NULL,'20050201:4','Structure','8','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (129,NULL,'20050201:5','Structure','9','DS0','Section','0','1','1');
+INSERT INTO divisions VALUES (130,NULL,'20050201:5','Division','129','DS1','Section','1','1.1','1.1');
+INSERT INTO divisions VALUES (131,NULL,'20050201:5','Division','129','DS2','Section','2','1.2','1.2');
+INSERT INTO divisions VALUES (132,NULL,'20050201:5','Division','129','DS3','Section','3','1.3','1.3');
+INSERT INTO divisions VALUES (133,NULL,'20050201:5','Division','132','DS4','Section','4','1.3.1','1.3.1');
+INSERT INTO divisions VALUES (134,NULL,'20050201:5','Division','132','DS5','Section','5','1.3.2','1.3.2');
+INSERT INTO divisions VALUES (135,NULL,'20050201:5','Division','132','DS6','Section','6','1.3.3','1.3.3');
+INSERT INTO divisions VALUES (136,NULL,'20050201:5','Division','132','DS7','Section','7','1.3.4','1.3.4');
+INSERT INTO divisions VALUES (137,NULL,'20050201:5','Division','132','DS8','Section','8','1.3.5','1.3.5');
+INSERT INTO divisions VALUES (138,NULL,'20050201:5','Division','132','DS9','Section','9','1.3.6','1.3.6');
+INSERT INTO divisions VALUES (139,NULL,'20050201:5','Division','129','DS10','Section','10','1.4','1.4');
+INSERT INTO divisions VALUES (140,NULL,'20050201:5','Division','129','DS11','Section','11','1.5','1.5');
+INSERT INTO divisions VALUES (141,NULL,'20050201:5','Division','140','DS12','Section','12','1.5.1','1.5.1');
+INSERT INTO divisions VALUES (142,NULL,'20050201:5','Division','140','DS13','Section','13','1.5.2','1.5.2');
+INSERT INTO divisions VALUES (143,NULL,'20050201:5','Division','140','DS14','Section','14','1.5.3','1.5.3');
+INSERT INTO divisions VALUES (144,NULL,'20050201:5','Division','140','DS15','Section','15','1.5.4','1.5.4');
+INSERT INTO divisions VALUES (145,NULL,'20050201:5','Division','129','DS16','Section','16','1.6','1.6');
+INSERT INTO divisions VALUES (146,NULL,'20050201:5','Division','145','DS17','Section','17','1.6.1','1.6.1');
+INSERT INTO divisions VALUES (147,NULL,'20050201:5','Division','145','DS18','Section','18','1.6.2','1.6.2');
+INSERT INTO divisions VALUES (148,NULL,'20050201:5','Division','145','DS19','Section','19','1.6.3','1.6.3');
+INSERT INTO divisions VALUES (149,NULL,'20050201:5','Division','145','DS20','Section','20','1.6.4','1.6.4');
+INSERT INTO divisions VALUES (150,NULL,'20050201:5','Division','145','DS21','Section','21','1.6.5','1.6.5');
+INSERT INTO divisions VALUES (151,NULL,'20050201:5','Division','145','DS22','Section','22','1.6.6','1.6.6');
+INSERT INTO divisions VALUES (152,NULL,'20050201:5','Division','145','DS23','Section','23','1.6.7','1.6.7');
+INSERT INTO divisions VALUES (153,NULL,'20050201:5','Division','145','DS24','Section','24','1.6.8','1.6.8');
+INSERT INTO divisions VALUES (154,NULL,'20050201:5','Division','145','DS25','Section','25','1.6.9','1.6.9');
+INSERT INTO divisions VALUES (155,NULL,'20050201:5','Division','145','DS26','Section','26','1.6.10','1.6.10');
+INSERT INTO divisions VALUES (156,NULL,'20050201:5','Division','129','DS27','Section','27','1.7','1.7');
+INSERT INTO divisions VALUES (157,NULL,'20050201:5','Division','156','DS28','Section','28','1.7.1','1.7.1');
+INSERT INTO divisions VALUES (158,NULL,'20050201:5','Division','156','DS29','Section','29','1.7.2','1.7.2');
+INSERT INTO divisions VALUES (159,NULL,'20050201:5','Division','156','DS30','Section','30','1.7.3','1.7.3');
+INSERT INTO divisions VALUES (160,NULL,'20050201:5','Division','156','DS31','Section','31','1.7.4','1.7.4');
+INSERT INTO divisions VALUES (161,NULL,'20050201:5','Division','156','DS32','Section','32','1.7.5','1.7.5');
+INSERT INTO divisions VALUES (162,NULL,'20050201:5','Division','156','DS33','Section','33','1.7.6','1.7.6');
+INSERT INTO divisions VALUES (163,NULL,'20050201:5','Division','156','DS34','Section','34','1.7.7','1.7.7');
+INSERT INTO divisions VALUES (164,NULL,'20050201:5','Division','156','DS35','Section','35','1.7.8','1.7.8');
+INSERT INTO divisions VALUES (165,NULL,'20050201:5','Division','156','DS36','Section','36','1.7.9','1.7.9');
+INSERT INTO divisions VALUES (166,NULL,'20050201:5','Division','156','DS37','Section','37','1.7.10','1.7.10');
+INSERT INTO divisions VALUES (167,NULL,'20050201:5','Division','156','DS38','Section','38','1.7.11','1.7.11');
+INSERT INTO divisions VALUES (168,NULL,'20050201:5','Division','156','DS39','Section','39','1.7.12','1.7.12');
+INSERT INTO divisions VALUES (169,NULL,'20050201:5','Division','156','DS40','Section','40','1.7.13','1.7.13');
+INSERT INTO divisions VALUES (170,NULL,'20050201:5','Division','156','DS41','Section','41','1.7.14','1.7.14');
+INSERT INTO divisions VALUES (171,NULL,'20050201:5','Division','156','DS42','Section','42','1.7.15','1.7.15');
+INSERT INTO divisions VALUES (172,NULL,'20050201:5','Division','156','DS43','Section','43','1.7.16','1.7.16');
+INSERT INTO divisions VALUES (173,NULL,'20050201:5','Division','129','DS44','Section','44','1.8','1.8');
+INSERT INTO divisions VALUES (174,NULL,'20050201:5','Division','129','DS45','Section','45','1.9','1.9');
+INSERT INTO divisions VALUES (175,NULL,'20050201:5','Structure','10','All','Document','All','All','Whole Document');
+INSERT INTO divisions VALUES (176,NULL,'20050201:6','Structure','11','All','Document','All','All','Whole Document');
+
+--
+-- Table structure for table `document`
+--
+
+CREATE TABLE document (
+  DocID varchar(64) NOT NULL default '',
+  DocType varchar(64) default NULL,
+  AccessionDate datetime default NULL,
+  IndexedDate datetime default NULL,
+  ModifiedDate datetime default NULL,
+  PRIMARY KEY  (DocID)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `document`
+--
+
+INSERT INTO document VALUES ('20050201:1','HTML','2005-02-01 22:03:55','2005-02-01 22:03:55','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:2','HTML','2005-02-01 22:03:56','2005-02-01 22:03:56','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:3','HTML','2005-02-01 22:03:59','2005-02-01 22:03:59','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:4','HTML','2005-02-01 22:03:59','2005-02-01 22:03:59','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:5','HTML','2005-02-01 22:04:00','2005-02-01 22:04:00','2004-07-20 04:36:03');
+INSERT INTO document VALUES ('20050201:6','GSMETADATA','2005-02-01 22:04:01','2005-02-01 22:04:01','2004-07-20 04:36:03');
+
+--
+-- Table structure for table `filegroups`
+--
+
+CREATE TABLE filegroups (
+  FileGroupRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  FileGroupID varchar(64) default NULL,
+  ParentRef int(11) default NULL,
+  ParentType varchar(16) default NULL,
+  PRIMARY KEY  (FileGroupRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `filegroups`
+--
+
+INSERT INTO filegroups VALUES (1,'20050201:1','FILEGROUP_PRELUDE4',1,'Section');
+INSERT INTO filegroups VALUES (2,'20050201:1','FILEGROUP_PRELUDE11',1,'Section');
+INSERT INTO filegroups VALUES (3,'20050201:1','FILEGROUP_PRELUDE1',1,'Section');
+INSERT INTO filegroups VALUES (4,'20050201:1','FILEGROUP_PRELUDE7',1,'Section');
+INSERT INTO filegroups VALUES (5,'20050201:1','FILEGROUP_PRELUDE25',1,'Section');
+INSERT INTO filegroups VALUES (6,'20050201:1','FILEGROUP_PRELUDE28',1,'Section');
+INSERT INTO filegroups VALUES (7,'20050201:1','FILEGROUP_PRELUDE14',1,'Section');
+INSERT INTO filegroups VALUES (8,'20050201:1','FILEGROUP_PRELUDE24',1,'Section');
+INSERT INTO filegroups VALUES (9,'20050201:1','FILEGROUP_PRELUDE6',1,'Section');
+INSERT INTO filegroups VALUES (10,'20050201:1','FILEGROUP_PRELUDE13',1,'Section');
+INSERT INTO filegroups VALUES (11,'20050201:1','FILEGROUP_PRELUDE26',1,'Section');
+INSERT INTO filegroups VALUES (12,'20050201:1','FILEGROUP_PRELUDE10',1,'Section');
+INSERT INTO filegroups VALUES (13,'20050201:1','FILEGROUP_PRELUDE27',1,'Section');
+INSERT INTO filegroups VALUES (14,'20050201:1','FILEGROUP_PRELUDE12',1,'Section');
+INSERT INTO filegroups VALUES (15,'20050201:1','FILEGROUP_PRELUDE5',1,'Section');
+INSERT INTO filegroups VALUES (16,'20050201:1','FILEGROUP_PRELUDE15',1,'Section');
+INSERT INTO filegroups VALUES (17,'20050201:1','FILEGROUP_PRELUDE8',1,'Section');
+INSERT INTO filegroups VALUES (18,'20050201:1','FILEGROUP_PRELUDE22',1,'Section');
+INSERT INTO filegroups VALUES (19,'20050201:1','FILEGROUP_PRELUDE18',1,'Section');
+INSERT INTO filegroups VALUES (20,'20050201:1','FILEGROUP_PRELUDE3',1,'Section');
+INSERT INTO filegroups VALUES (21,'20050201:1','default',1,'Section');
+INSERT INTO filegroups VALUES (22,'20050201:1','FILEGROUP_PRELUDE17',1,'Section');
+INSERT INTO filegroups VALUES (23,'20050201:1','FILEGROUP_PRELUDE2',1,'Section');
+INSERT INTO filegroups VALUES (24,'20050201:1','FILEGROUP_PRELUDE20',1,'Section');
+INSERT INTO filegroups VALUES (25,'20050201:1','FILEGROUP_PRELUDE23',1,'Section');
+INSERT INTO filegroups VALUES (26,'20050201:1','FILEGROUP_PRELUDE9',1,'Section');
+INSERT INTO filegroups VALUES (27,'20050201:1','FILEGROUP_PRELUDE16',1,'Section');
+INSERT INTO filegroups VALUES (28,'20050201:1','FILEGROUP_PRELUDE19',1,'Section');
+INSERT INTO filegroups VALUES (29,'20050201:1','FILEGROUP_PRELUDE21',1,'Section');
+INSERT INTO filegroups VALUES (30,'20050201:2','FILEGROUP_PRELUDE51',2,'Section');
+INSERT INTO filegroups VALUES (31,'20050201:2','FILEGROUP_PRELUDE33',2,'Section');
+INSERT INTO filegroups VALUES (32,'20050201:2','FILEGROUP_PRELUDE48',2,'Section');
+INSERT INTO filegroups VALUES (33,'20050201:2','FILEGROUP_PRELUDE36',2,'Section');
+INSERT INTO filegroups VALUES (34,'20050201:2','FILEGROUP_PRELUDE14',2,'Section');
+INSERT INTO filegroups VALUES (35,'20050201:2','FILEGROUP_PRELUDE35',2,'Section');
+INSERT INTO filegroups VALUES (36,'20050201:2','FILEGROUP_PRELUDE13',2,'Section');
+INSERT INTO filegroups VALUES (37,'20050201:2','FILEGROUP_PRELUDE34',2,'Section');
+INSERT INTO filegroups VALUES (38,'20050201:2','FILEGROUP_PRELUDE49',2,'Section');
+INSERT INTO filegroups VALUES (39,'20050201:2','FILEGROUP_PRELUDE22',2,'Section');
+INSERT INTO filegroups VALUES (40,'20050201:2','FILEGROUP_PRELUDE29',2,'Section');
+INSERT INTO filegroups VALUES (41,'20050201:2','FILEGROUP_PRELUDE40',2,'Section');
+INSERT INTO filegroups VALUES (42,'20050201:2','FILEGROUP_PRELUDE44',2,'Section');
+INSERT INTO filegroups VALUES (43,'20050201:2','default',2,'Section');
+INSERT INTO filegroups VALUES (44,'20050201:2','FILEGROUP_PRELUDE3',2,'Section');
+INSERT INTO filegroups VALUES (45,'20050201:2','FILEGROUP_PRELUDE42',2,'Section');
+INSERT INTO filegroups VALUES (46,'20050201:2','FILEGROUP_PRELUDE39',2,'Section');
+INSERT INTO filegroups VALUES (47,'20050201:2','FILEGROUP_PRELUDE23',2,'Section');
+INSERT INTO filegroups VALUES (48,'20050201:2','FILEGROUP_PRELUDE9',2,'Section');
+INSERT INTO filegroups VALUES (49,'20050201:2','FILEGROUP_PRELUDE43',2,'Section');
+INSERT INTO filegroups VALUES (50,'20050201:2','FILEGROUP_PRELUDE50',2,'Section');
+INSERT INTO filegroups VALUES (51,'20050201:2','FILEGROUP_PRELUDE45',2,'Section');
+INSERT INTO filegroups VALUES (52,'20050201:2','FILEGROUP_PRELUDE4',2,'Section');
+INSERT INTO filegroups VALUES (53,'20050201:2','FILEGROUP_PRELUDE1',2,'Section');
+INSERT INTO filegroups VALUES (54,'20050201:2','FILEGROUP_PRELUDE11',2,'Section');
+INSERT INTO filegroups VALUES (55,'20050201:2','FILEGROUP_PRELUDE7',2,'Section');
+INSERT INTO filegroups VALUES (56,'20050201:2','FILEGROUP_PRELUDE30',2,'Section');
+INSERT INTO filegroups VALUES (57,'20050201:2','FILEGROUP_PRELUDE25',2,'Section');
+INSERT INTO filegroups VALUES (58,'20050201:2','FILEGROUP_PRELUDE41',2,'Section');
+INSERT INTO filegroups VALUES (59,'20050201:2','FILEGROUP_PRELUDE28',2,'Section');
+INSERT INTO filegroups VALUES (60,'20050201:2','FILEGROUP_PRELUDE47',2,'Section');
+INSERT INTO filegroups VALUES (61,'20050201:2','FILEGROUP_PRELUDE46',2,'Section');
+INSERT INTO filegroups VALUES (62,'20050201:2','FILEGROUP_PRELUDE24',2,'Section');
+INSERT INTO filegroups VALUES (63,'20050201:2','FILEGROUP_PRELUDE6',2,'Section');
+INSERT INTO filegroups VALUES (64,'20050201:2','FILEGROUP_PRELUDE10',2,'Section');
+INSERT INTO filegroups VALUES (65,'20050201:2','FILEGROUP_PRELUDE26',2,'Section');
+INSERT INTO filegroups VALUES (66,'20050201:2','FILEGROUP_PRELUDE32',2,'Section');
+INSERT INTO filegroups VALUES (67,'20050201:2','FILEGROUP_PRELUDE27',2,'Section');
+INSERT INTO filegroups VALUES (68,'20050201:2','FILEGROUP_PRELUDE5',2,'Section');
+INSERT INTO filegroups VALUES (69,'20050201:2','FILEGROUP_PRELUDE12',2,'Section');
+INSERT INTO filegroups VALUES (70,'20050201:2','FILEGROUP_PRELUDE15',2,'Section');
+INSERT INTO filegroups VALUES (71,'20050201:2','FILEGROUP_PRELUDE8',2,'Section');
+INSERT INTO filegroups VALUES (72,'20050201:2','FILEGROUP_PRELUDE18',2,'Section');
+INSERT INTO filegroups VALUES (73,'20050201:2','FILEGROUP_PRELUDE37',2,'Section');
+INSERT INTO filegroups VALUES (74,'20050201:2','FILEGROUP_PRELUDE17',2,'Section');
+INSERT INTO filegroups VALUES (75,'20050201:2','FILEGROUP_PRELUDE2',2,'Section');
+INSERT INTO filegroups VALUES (76,'20050201:2','FILEGROUP_PRELUDE31',2,'Section');
+INSERT INTO filegroups VALUES (77,'20050201:2','FILEGROUP_PRELUDE20',2,'Section');
+INSERT INTO filegroups VALUES (78,'20050201:2','FILEGROUP_PRELUDE19',2,'Section');
+INSERT INTO filegroups VALUES (79,'20050201:2','FILEGROUP_PRELUDE16',2,'Section');
+INSERT INTO filegroups VALUES (80,'20050201:2','FILEGROUP_PRELUDE38',2,'Section');
+INSERT INTO filegroups VALUES (81,'20050201:2','FILEGROUP_PRELUDE21',2,'Section');
+INSERT INTO filegroups VALUES (82,'20050201:3','FILEGROUP_PRELUDE15',3,'Section');
+INSERT INTO filegroups VALUES (83,'20050201:3','FILEGROUP_PRELUDE4',3,'Section');
+INSERT INTO filegroups VALUES (84,'20050201:3','FILEGROUP_PRELUDE22',3,'Section');
+INSERT INTO filegroups VALUES (85,'20050201:3','FILEGROUP_PRELUDE8',3,'Section');
+INSERT INTO filegroups VALUES (86,'20050201:3','FILEGROUP_PRELUDE18',3,'Section');
+INSERT INTO filegroups VALUES (87,'20050201:3','FILEGROUP_PRELUDE1',3,'Section');
+INSERT INTO filegroups VALUES (88,'20050201:3','FILEGROUP_PRELUDE11',3,'Section');
+INSERT INTO filegroups VALUES (89,'20050201:3','FILEGROUP_PRELUDE7',3,'Section');
+INSERT INTO filegroups VALUES (90,'20050201:3','default',3,'Section');
+INSERT INTO filegroups VALUES (91,'20050201:3','FILEGROUP_PRELUDE3',3,'Section');
+INSERT INTO filegroups VALUES (92,'20050201:3','FILEGROUP_PRELUDE17',3,'Section');
+INSERT INTO filegroups VALUES (93,'20050201:3','FILEGROUP_PRELUDE14',3,'Section');
+INSERT INTO filegroups VALUES (94,'20050201:3','FILEGROUP_PRELUDE2',3,'Section');
+INSERT INTO filegroups VALUES (95,'20050201:3','FILEGROUP_PRELUDE20',3,'Section');
+INSERT INTO filegroups VALUES (96,'20050201:3','FILEGROUP_PRELUDE13',3,'Section');
+INSERT INTO filegroups VALUES (97,'20050201:3','FILEGROUP_PRELUDE6',3,'Section');
+INSERT INTO filegroups VALUES (98,'20050201:3','FILEGROUP_PRELUDE10',3,'Section');
+INSERT INTO filegroups VALUES (99,'20050201:3','FILEGROUP_PRELUDE19',3,'Section');
+INSERT INTO filegroups VALUES (100,'20050201:3','FILEGROUP_PRELUDE16',3,'Section');
+INSERT INTO filegroups VALUES (101,'20050201:3','FILEGROUP_PRELUDE9',3,'Section');
+INSERT INTO filegroups VALUES (102,'20050201:3','FILEGROUP_PRELUDE21',3,'Section');
+INSERT INTO filegroups VALUES (103,'20050201:3','FILEGROUP_PRELUDE5',3,'Section');
+INSERT INTO filegroups VALUES (104,'20050201:3','FILEGROUP_PRELUDE12',3,'Section');
+INSERT INTO filegroups VALUES (105,'20050201:4','FILEGROUP_PRELUDE15',4,'Section');
+INSERT INTO filegroups VALUES (106,'20050201:4','FILEGROUP_PRELUDE4',4,'Section');
+INSERT INTO filegroups VALUES (107,'20050201:4','FILEGROUP_PRELUDE22',4,'Section');
+INSERT INTO filegroups VALUES (108,'20050201:4','FILEGROUP_PRELUDE8',4,'Section');
+INSERT INTO filegroups VALUES (109,'20050201:4','FILEGROUP_PRELUDE18',4,'Section');
+INSERT INTO filegroups VALUES (110,'20050201:4','FILEGROUP_PRELUDE1',4,'Section');
+INSERT INTO filegroups VALUES (111,'20050201:4','FILEGROUP_PRELUDE11',4,'Section');
+INSERT INTO filegroups VALUES (112,'20050201:4','FILEGROUP_PRELUDE7',4,'Section');
+INSERT INTO filegroups VALUES (113,'20050201:4','default',4,'Section');
+INSERT INTO filegroups VALUES (114,'20050201:4','FILEGROUP_PRELUDE3',4,'Section');
+INSERT INTO filegroups VALUES (115,'20050201:4','FILEGROUP_PRELUDE17',4,'Section');
+INSERT INTO filegroups VALUES (116,'20050201:4','FILEGROUP_PRELUDE14',4,'Section');
+INSERT INTO filegroups VALUES (117,'20050201:4','FILEGROUP_PRELUDE2',4,'Section');
+INSERT INTO filegroups VALUES (118,'20050201:4','FILEGROUP_PRELUDE20',4,'Section');
+INSERT INTO filegroups VALUES (119,'20050201:4','FILEGROUP_PRELUDE13',4,'Section');
+INSERT INTO filegroups VALUES (120,'20050201:4','FILEGROUP_PRELUDE6',4,'Section');
+INSERT INTO filegroups VALUES (121,'20050201:4','FILEGROUP_PRELUDE23',4,'Section');
+INSERT INTO filegroups VALUES (122,'20050201:4','FILEGROUP_PRELUDE10',4,'Section');
+INSERT INTO filegroups VALUES (123,'20050201:4','FILEGROUP_PRELUDE19',4,'Section');
+INSERT INTO filegroups VALUES (124,'20050201:4','FILEGROUP_PRELUDE16',4,'Section');
+INSERT INTO filegroups VALUES (125,'20050201:4','FILEGROUP_PRELUDE9',4,'Section');
+INSERT INTO filegroups VALUES (126,'20050201:4','FILEGROUP_PRELUDE21',4,'Section');
+INSERT INTO filegroups VALUES (127,'20050201:4','FILEGROUP_PRELUDE5',4,'Section');
+INSERT INTO filegroups VALUES (128,'20050201:4','FILEGROUP_PRELUDE12',4,'Section');
+INSERT INTO filegroups VALUES (129,'20050201:5','FILEGROUP_PRELUDE4',5,'Section');
+INSERT INTO filegroups VALUES (130,'20050201:5','FILEGROUP_PRELUDE11',5,'Section');
+INSERT INTO filegroups VALUES (131,'20050201:5','FILEGROUP_PRELUDE1',5,'Section');
+INSERT INTO filegroups VALUES (132,'20050201:5','FILEGROUP_PRELUDE33',5,'Section');
+INSERT INTO filegroups VALUES (133,'20050201:5','FILEGROUP_PRELUDE30',5,'Section');
+INSERT INTO filegroups VALUES (134,'20050201:5','FILEGROUP_PRELUDE7',5,'Section');
+INSERT INTO filegroups VALUES (135,'20050201:5','FILEGROUP_PRELUDE25',5,'Section');
+INSERT INTO filegroups VALUES (136,'20050201:5','FILEGROUP_PRELUDE41',5,'Section');
+INSERT INTO filegroups VALUES (137,'20050201:5','FILEGROUP_PRELUDE28',5,'Section');
+INSERT INTO filegroups VALUES (138,'20050201:5','FILEGROUP_PRELUDE36',5,'Section');
+INSERT INTO filegroups VALUES (139,'20050201:5','FILEGROUP_PRELUDE14',5,'Section');
+INSERT INTO filegroups VALUES (140,'20050201:5','FILEGROUP_PRELUDE46',5,'Section');
+INSERT INTO filegroups VALUES (141,'20050201:5','FILEGROUP_PRELUDE24',5,'Section');
+INSERT INTO filegroups VALUES (142,'20050201:5','FILEGROUP_PRELUDE35',5,'Section');
+INSERT INTO filegroups VALUES (143,'20050201:5','FILEGROUP_PRELUDE6',5,'Section');
+INSERT INTO filegroups VALUES (144,'20050201:5','FILEGROUP_PRELUDE13',5,'Section');
+INSERT INTO filegroups VALUES (145,'20050201:5','FILEGROUP_PRELUDE26',5,'Section');
+INSERT INTO filegroups VALUES (146,'20050201:5','FILEGROUP_PRELUDE10',5,'Section');
+INSERT INTO filegroups VALUES (147,'20050201:5','FILEGROUP_PRELUDE32',5,'Section');
+INSERT INTO filegroups VALUES (148,'20050201:5','FILEGROUP_PRELUDE34',5,'Section');
+INSERT INTO filegroups VALUES (149,'20050201:5','FILEGROUP_PRELUDE27',5,'Section');
+INSERT INTO filegroups VALUES (150,'20050201:5','FILEGROUP_PRELUDE12',5,'Section');
+INSERT INTO filegroups VALUES (151,'20050201:5','FILEGROUP_PRELUDE5',5,'Section');
+INSERT INTO filegroups VALUES (152,'20050201:5','FILEGROUP_PRELUDE15',5,'Section');
+INSERT INTO filegroups VALUES (153,'20050201:5','FILEGROUP_PRELUDE8',5,'Section');
+INSERT INTO filegroups VALUES (154,'20050201:5','FILEGROUP_PRELUDE22',5,'Section');
+INSERT INTO filegroups VALUES (155,'20050201:5','FILEGROUP_PRELUDE40',5,'Section');
+INSERT INTO filegroups VALUES (156,'20050201:5','FILEGROUP_PRELUDE29',5,'Section');
+INSERT INTO filegroups VALUES (157,'20050201:5','FILEGROUP_PRELUDE18',5,'Section');
+INSERT INTO filegroups VALUES (158,'20050201:5','FILEGROUP_PRELUDE44',5,'Section');
+INSERT INTO filegroups VALUES (159,'20050201:5','FILEGROUP_PRELUDE37',5,'Section');
+INSERT INTO filegroups VALUES (160,'20050201:5','FILEGROUP_PRELUDE3',5,'Section');
+INSERT INTO filegroups VALUES (161,'20050201:5','default',5,'Section');
+INSERT INTO filegroups VALUES (162,'20050201:5','FILEGROUP_PRELUDE17',5,'Section');
+INSERT INTO filegroups VALUES (163,'20050201:5','FILEGROUP_PRELUDE31',5,'Section');
+INSERT INTO filegroups VALUES (164,'20050201:5','FILEGROUP_PRELUDE2',5,'Section');
+INSERT INTO filegroups VALUES (165,'20050201:5','FILEGROUP_PRELUDE42',5,'Section');
+INSERT INTO filegroups VALUES (166,'20050201:5','FILEGROUP_PRELUDE39',5,'Section');
+INSERT INTO filegroups VALUES (167,'20050201:5','FILEGROUP_PRELUDE20',5,'Section');
+INSERT INTO filegroups VALUES (168,'20050201:5','FILEGROUP_PRELUDE23',5,'Section');
+INSERT INTO filegroups VALUES (169,'20050201:5','FILEGROUP_PRELUDE9',5,'Section');
+INSERT INTO filegroups VALUES (170,'20050201:5','FILEGROUP_PRELUDE16',5,'Section');
+INSERT INTO filegroups VALUES (171,'20050201:5','FILEGROUP_PRELUDE19',5,'Section');
+INSERT INTO filegroups VALUES (172,'20050201:5','FILEGROUP_PRELUDE43',5,'Section');
+INSERT INTO filegroups VALUES (173,'20050201:5','FILEGROUP_PRELUDE38',5,'Section');
+INSERT INTO filegroups VALUES (174,'20050201:5','FILEGROUP_PRELUDE45',5,'Section');
+INSERT INTO filegroups VALUES (175,'20050201:5','FILEGROUP_PRELUDE21',5,'Section');
+INSERT INTO filegroups VALUES (176,'20050201:6','default',6,'Section');
+
+--
+-- Table structure for table `files`
+--
+
+CREATE TABLE files (
+  FileGroupRef int(11) default NULL,
+  FileLocType varchar(64) default NULL,
+  FileLocation tinytext,
+  MIMEType varchar(64) default NULL,
+  FileID varchar(32) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `files`
+--
+
+INSERT INTO files VALUES (1,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/166/comment()[1]/range-to(/1/2/213/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (2,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/437/comment()[1]/range-to(/1/2/489/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (3,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/3/comment()[1]/range-to(/1/2/119/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (4,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/301/comment()[1]/range-to(/1/2/343/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (5,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/861/comment()[1]/range-to(/1/2/1369/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (6,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (7,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/564/comment()[1]/range-to(/1/2/600/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (8,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1042/comment()[1]/range-to(/1/2/1369/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (9,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/263/comment()[1]/range-to(/1/2/301/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (10,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/532/comment()[1]/range-to(/1/2/564/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (11,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1369/comment()[1]/range-to(/1/2/1380/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (12,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/389/comment()[1]/range-to(/1/2/437/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (13,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/1380/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (14,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/489/comment()[1]/range-to(/1/2/532/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (15,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/213/comment()[1]/range-to(/1/2/263/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (16,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/374/comment()[1]/range-to(/1/2/600/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (17,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/343/comment()[1]/range-to(/1/2/374/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (18,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/793/comment()[1]/range-to(/1/2/861/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (19,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/685/comment()[1]/range-to(/1/2/718/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (20,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/131/comment()[1]/range-to(/1/2/151/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm','text/html','default.1');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p00.png','image/png','default.2');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p06.png','image/png','default.3');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p09.png','image/png','default.4');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p13.png','image/png','default.5');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p23.png','image/png','default.6');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p26.png','image/png','default.7');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p29.png','image/png','default.8');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p33.png','image/png','default.9');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p40.png','image/png','default.10');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p43.png','image/png','default.11');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p45.png','image/png','default.12');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p48.png','image/png','default.13');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p52.png','image/png','default.14');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p55.png','image/png','default.15');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p58.png','image/png','default.16');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p61.png','image/png','default.17');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p63.png','image/png','default.18');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p66.png','image/png','default.19');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p68.png','image/png','default.20');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p70.png','image/png','default.21');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p73.png','image/png','default.22');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p77.png','image/png','default.23');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p79.png','image/png','default.24');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p82.png','image/png','default.25');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p84.png','image/png','default.26');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p87.png','image/png','default.27');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p88.png','image/png','default.28');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p91.png','image/png','default.29');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p93.png','image/png','default.30');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p94.png','image/png','default.31');
+INSERT INTO files VALUES (21,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/p101.png','image/png','default.32');
+INSERT INTO files VALUES (22,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/648/comment()[1]/range-to(/1/2/685/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (23,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/119/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (24,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/751/comment()[1]/range-to(/1/2/793/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (25,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/862/comment()[1]/range-to(/1/2/1042/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (26,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/151/comment()[1]/range-to(/1/2/374/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (27,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/615/comment()[1]/range-to(/1/2/648/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (28,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/718/comment()[1]/range-to(/1/2/751/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (29,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b18ase/b18ase.htm#xpointer(/1/2/600/comment()[1]/range-to(/1/2/793/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (30,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE51');
+INSERT INTO files VALUES (31,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2189/comment()[1]/range-to(/1/2/2234/comment()[1]))','text/html','FILE33');
+INSERT INTO files VALUES (32,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/4085/comment()[1]/range-to(/1/2/4112/comment()[1]))','text/html','FILE48');
+INSERT INTO files VALUES (33,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2332/comment()[1]/range-to(/1/2/2363/comment()[1]))','text/html','FILE36');
+INSERT INTO files VALUES (34,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1151/comment()[1]/range-to(/1/2/1233/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (35,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2283/comment()[1]/range-to(/1/2/2332/comment()[1]))','text/html','FILE35');
+INSERT INTO files VALUES (36,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1072/comment()[1]/range-to(/1/2/1151/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (37,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2234/comment()[1]/range-to(/1/2/2283/comment()[1]))','text/html','FILE34');
+INSERT INTO files VALUES (38,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2732/comment()[1]/range-to(/1/2/4112/comment()[1]))','text/html','FILE49');
+INSERT INTO files VALUES (39,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1619/comment()[1]/range-to(/1/2/1683/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (40,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2004/comment()[1]/range-to(/1/2/2059/comment()[1]))','text/html','FILE29');
+INSERT INTO files VALUES (41,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2166/comment()[1]/range-to(/1/2/2538/comment()[1]))','text/html','FILE40');
+INSERT INTO files VALUES (42,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2662/comment()[1]/range-to(/1/2/2732/comment()[1]))','text/html','FILE44');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm','text/html','default.1');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g001a.png','image/png','default.2');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p9a.png','image/png','default.3');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g002a.png','image/png','default.4');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g003a.png','image/png','default.5');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g004a.png','image/png','default.6');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p24a.png','image/png','default.7');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p39a.png','image/png','default.8');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g005a.png','image/png','default.9');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g006a.png','image/png','default.10');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g007a.png','image/png','default.11');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g008a.png','image/png','default.12');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p81a.png','image/png','default.13');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g009a.png','image/png','default.14');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p96a.png','image/png','default.15');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g010a.png','image/png','default.16');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p112a.png','image/png','default.17');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g011a.png','image/png','default.18');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p117a.png','image/png','default.19');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g012a.png','image/png','default.20');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p129a.png','image/png','default.21');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g013a.png','image/png','default.22');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p138a.png','image/png','default.23');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g014a.png','image/png','default.24');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p149a.png','image/png','default.25');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g015a.png','image/png','default.26');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p159a.png','image/png','default.27');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g016a.png','image/png','default.28');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g017a.png','image/png','default.29');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g018a.png','image/png','default.30');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g019a.png','image/png','default.31');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g020a.png','image/png','default.32');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p203a.png','image/png','default.33');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g021a.png','image/png','default.34');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p210a.png','image/png','default.35');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g022a.png','image/png','default.36');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p219a.png','image/png','default.37');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p227a.png','image/png','default.38');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g023a.png','image/png','default.39');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p236a.png','image/png','default.40');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g024a.png','image/png','default.41');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p243a.png','image/png','default.42');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g025a.png','image/png','default.43');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p252a.png','image/png','default.44');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g026a.png','image/png','default.45');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p260a.png','image/png','default.46');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p265a.png','image/png','default.47');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g027a.png','image/png','default.48');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p274a.png','image/png','default.49');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g028a.png','image/png','default.50');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g029a.png','image/png','default.51');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p292a.png','image/png','default.52');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p293a.png','image/png','default.53');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g030a.png','image/png','default.54');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p301a.png','image/png','default.55');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p309a.png','image/png','default.56');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g031a.png','image/png','default.57');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g032a.png','image/png','default.58');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g033a.png','image/png','default.59');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g034a.png','image/png','default.60');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p323a.png','image/png','default.61');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g035a.png','image/png','default.62');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g036a.png','image/png','default.63');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g037a.png','image/png','default.64');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g038a.png','image/png','default.65');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p349a.png','image/png','default.66');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g039a.png','image/png','default.67');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/p357a.png','image/png','default.68');
+INSERT INTO files VALUES (43,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/g040a.png','image/png','default.69');
+INSERT INTO files VALUES (44,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/216/comment()[1]/range-to(/1/2/320/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (45,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2607/comment()[1]/range-to(/1/2/2658/comment()[1]))','text/html','FILE42');
+INSERT INTO files VALUES (46,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2474/comment()[1]/range-to(/1/2/2538/comment()[1]))','text/html','FILE39');
+INSERT INTO files VALUES (47,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1683/comment()[1]/range-to(/1/2/1731/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (48,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/760/comment()[1]/range-to(/1/2/842/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (49,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2538/comment()[1]/range-to(/1/2/2658/comment()[1]))','text/html','FILE43');
+INSERT INTO files VALUES (50,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/4112/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE50');
+INSERT INTO files VALUES (51,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2658/comment()[1]/range-to(/1/2/2732/comment()[1]))','text/html','FILE45');
+INSERT INTO files VALUES (52,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/327/comment()[1]/range-to(/1/2/424/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (53,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2/comment()[1]/range-to(/1/2/191/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (54,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/913/comment()[1]/range-to(/1/2/1000/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (55,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/652/comment()[1]/range-to(/1/2/733/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (56,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2059/comment()[1]/range-to(/1/2/2108/comment()[1]))','text/html','FILE30');
+INSERT INTO files VALUES (57,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1788/comment()[1]/range-to(/1/2/1841/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (58,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2557/comment()[1]/range-to(/1/2/2607/comment()[1]))','text/html','FILE41');
+INSERT INTO files VALUES (59,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1952/comment()[1]/range-to(/1/2/2004/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (60,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/3152/comment()[1]/range-to(/1/2/4085/comment()[1]))','text/html','FILE47');
+INSERT INTO files VALUES (61,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2733/comment()[1]/range-to(/1/2/3152/comment()[1]))','text/html','FILE46');
+INSERT INTO files VALUES (62,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1731/comment()[1]/range-to(/1/2/1788/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (63,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/531/comment()[1]/range-to(/1/2/652/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (64,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/842/comment()[1]/range-to(/1/2/913/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (65,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1841/comment()[1]/range-to(/1/2/1902/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (66,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1537/comment()[1]/range-to(/1/2/2166/comment()[1]))','text/html','FILE32');
+INSERT INTO files VALUES (67,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1902/comment()[1]/range-to(/1/2/1952/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (68,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/424/comment()[1]/range-to(/1/2/531/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (69,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1000/comment()[1]/range-to(/1/2/1072/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (70,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1233/comment()[1]/range-to(/1/2/1307/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (71,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/320/comment()[1]/range-to(/1/2/733/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (72,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/733/comment()[1]/range-to(/1/2/1447/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (73,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2363/comment()[1]/range-to(/1/2/2410/comment()[1]))','text/html','FILE37');
+INSERT INTO files VALUES (74,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1363/comment()[1]/range-to(/1/2/1447/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (75,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/191/comment()[1]/range-to(/1/2/216/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (76,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2108/comment()[1]/range-to(/1/2/2166/comment()[1]))','text/html','FILE31');
+INSERT INTO files VALUES (77,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1447/comment()[1]/range-to(/1/2/1537/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (78,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1464/comment()[1]/range-to(/1/2/1537/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (79,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1307/comment()[1]/range-to(/1/2/1363/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (80,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/2410/comment()[1]/range-to(/1/2/2474/comment()[1]))','text/html','FILE38');
+INSERT INTO files VALUES (81,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/b17mie/b17mie.htm#xpointer(/1/2/1560/comment()[1]/range-to(/1/2/1619/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (82,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/36/comment()[1]/range-to(/1/2/230/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (83,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/101/comment()[1]/range-to(/1/2/113/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (84,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (85,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/164/comment()[1]/range-to(/1/2/174/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (86,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/230/comment()[1]/range-to(/1/2/274/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (87,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/23/comment()[1]/range-to(/1/2/36/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (88,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/185/comment()[1]/range-to(/1/2/205/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (89,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/143/comment()[1]/range-to(/1/2/164/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm','text/html','default.1');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p60a.png','image/png','default.2');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61a.png','image/png','default.3');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61b.png','image/png','default.4');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61c.png','image/png','default.5');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p61d.png','image/png','default.6');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p62a.png','image/png','default.7');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p62b.png','image/png','default.8');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p63a.png','image/png','default.9');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p63b.png','image/png','default.10');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64a.png','image/png','default.11');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64b.png','image/png','default.12');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64c.png','image/png','default.13');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p64d.png','image/png','default.14');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65a.png','image/png','default.15');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65b.png','image/png','default.16');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65c.png','image/png','default.17');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p65d.png','image/png','default.18');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66a.png','image/png','default.19');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66b.png','image/png','default.20');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p66c.png','image/png','default.21');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p67a.png','image/png','default.22');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p67b.png','image/png','default.23');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68a.png','image/png','default.24');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68b.png','image/png','default.25');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68c.png','image/png','default.26');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p68d.png','image/png','default.27');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69a.png','image/png','default.28');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69b.png','image/png','default.29');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p69c.png','image/png','default.30');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70a.png','image/png','default.31');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70b.png','image/png','default.32');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p70c.png','image/png','default.33');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71a.png','image/png','default.34');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71b.png','image/png','default.35');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p71c.png','image/png','default.36');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p72a.png','image/png','default.37');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p72b.png','image/png','default.38');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p73a.png','image/png','default.39');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p73b.png','image/png','default.40');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p74a.png','image/png','default.41');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p74b.png','image/png','default.42');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75a.png','image/png','default.43');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75b.png','image/png','default.44');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p75c.png','image/png','default.45');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76a.png','image/png','default.46');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76b.png','image/png','default.47');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p76c.png','image/png','default.48');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77a.png','image/png','default.49');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77b.png','image/png','default.50');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p77c.png','image/png','default.51');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p78a.png','image/png','default.52');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p78b.png','image/png','default.53');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79a.png','image/png','default.54');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79b.png','image/png','default.55');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79c.png','image/png','default.56');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p79d.png','image/png','default.57');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p80a.png','image/png','default.58');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p80b.png','image/png','default.59');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p81a.png','image/png','default.60');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p81b.png','image/png','default.61');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82a.png','image/png','default.62');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82b.png','image/png','default.63');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82c.png','image/png','default.64');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p82d.png','image/png','default.65');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p83a.png','image/png','default.66');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84a.png','image/png','default.67');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84b.png','image/png','default.68');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84c.png','image/png','default.69');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p84d.png','image/png','default.70');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85a.png','image/png','default.71');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85b.png','image/png','default.72');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p85c.png','image/png','default.73');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86a.png','image/png','default.74');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86b.png','image/png','default.75');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p86c.png','image/png','default.76');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87a.png','image/png','default.77');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87b.png','image/png','default.78');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p87c.png','image/png','default.79');
+INSERT INTO files VALUES (90,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/f33p88a.png','image/png','default.80');
+INSERT INTO files VALUES (91,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/48/comment()[1]/range-to(/1/2/101/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (92,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/246/comment()[1]/range-to(/1/2/274/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (93,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/218/comment()[1]/range-to(/1/2/230/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (94,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/7/comment()[1]/range-to(/1/2/36/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (95,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/399/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (96,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/213/comment()[1]/range-to(/1/2/218/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (97,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/117/comment()[1]/range-to(/1/2/143/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (98,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/178/comment()[1]/range-to(/1/2/185/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (99,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/274/comment()[1]/range-to(/1/2/340/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (100,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/231/comment()[1]/range-to(/1/2/246/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (101,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/174/comment()[1]/range-to(/1/2/178/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (102,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/340/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (103,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/113/comment()[1]/range-to(/1/2/117/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (104,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb33fe/fb33fe.htm#xpointer(/1/2/205/comment()[1]/range-to(/1/2/213/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (105,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/172/comment()[1]/range-to(/1/2/183/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (106,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/66/comment()[1]/range-to(/1/2/73/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (107,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/225/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (108,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/106/comment()[1]/range-to(/1/2/115/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (109,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/200/comment()[1]/range-to(/1/2/225/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (110,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/7/comment()[1]/range-to(/1/2/12/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (111,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/139/comment()[1]/range-to(/1/2/147/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (112,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/100/comment()[1]/range-to(/1/2/106/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm','text/html','default.1');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p093a.png','image/png','default.2');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p094a.png','image/png','default.3');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p094b.png','image/png','default.4');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095a.png','image/png','default.5');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095b.png','image/png','default.6');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p095c.png','image/png','default.7');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096a.png','image/png','default.8');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096b.png','image/png','default.9');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p096c.png','image/png','default.10');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p097a.png','image/png','default.11');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p097b.png','image/png','default.12');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p098a.png','image/png','default.13');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p099a.png','image/png','default.14');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p099b.png','image/png','default.15');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p101a.png','image/png','default.16');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p101b.png','image/png','default.17');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p102a.png','image/png','default.18');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p103a.png','image/png','default.19');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p103b.png','image/png','default.20');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p104a.png','image/png','default.21');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p104b.png','image/png','default.22');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105a.png','image/png','default.23');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105b.png','image/png','default.24');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p105c.png','image/png','default.25');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106a.png','image/png','default.26');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106b.png','image/png','default.27');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106c.png','image/png','default.28');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p106d.png','image/png','default.29');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107a.png','image/png','default.30');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107b.png','image/png','default.31');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p107c.png','image/png','default.32');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p108a.png','image/png','default.33');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p108b.png','image/png','default.34');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p109a.png','image/png','default.35');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p109b.png','image/png','default.36');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p110a.png','image/png','default.37');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p110b.png','image/png','default.38');
+INSERT INTO files VALUES (113,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/f34p111a.png','image/png','default.39');
+INSERT INTO files VALUES (114,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/38/comment()[1]/range-to(/1/2/66/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (115,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/163/comment()[1]/range-to(/1/2/200/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (116,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/164/comment()[1]/range-to(/1/2/172/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (117,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/13/comment()[1]/range-to(/1/2/38/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (118,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/244/comment()[1]/range-to(/1/2/258/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (119,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/131/comment()[1]/range-to(/1/2/163/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (120,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/80/comment()[1]/range-to(/1/2/100/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (121,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (122,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/73/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (123,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/229/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (124,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/183/comment()[1]/range-to(/1/2/200/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (125,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/115/comment()[1]/range-to(/1/2/131/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (126,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/258/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (127,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/12/comment()[1]/range-to(/1/2/73/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (128,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/fb34fe/fb34fe.htm#xpointer(/1/2/147/comment()[1]/range-to(/1/2/163/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (129,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/41/comment()[1]/range-to(/1/2/51/comment()[1]))','text/html','FILE4');
+INSERT INTO files VALUES (130,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/91/comment()[1]/range-to(/1/2/93/comment()[1]))','text/html','FILE11');
+INSERT INTO files VALUES (131,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/21/comment()[1]/range-to(/1/2/29/comment()[1]))','text/html','FILE1');
+INSERT INTO files VALUES (132,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/275/comment()[1]/range-to(/1/2/281/comment()[1]))','text/html','FILE33');
+INSERT INTO files VALUES (133,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/254/comment()[1]/range-to(/1/2/267/comment()[1]))','text/html','FILE30');
+INSERT INTO files VALUES (134,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/65/comment()[1]/range-to(/1/2/78/comment()[1]))','text/html','FILE7');
+INSERT INTO files VALUES (135,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/241/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE25');
+INSERT INTO files VALUES (136,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/347/comment()[1]/range-to(/1/2/353/comment()[1]))','text/html','FILE41');
+INSERT INTO files VALUES (137,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/247/comment()[1]/range-to(/1/2/251/comment()[1]))','text/html','FILE28');
+INSERT INTO files VALUES (138,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/297/comment()[1]/range-to(/1/2/311/comment()[1]))','text/html','FILE36');
+INSERT INTO files VALUES (139,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/115/comment()[1]/range-to(/1/2/138/comment()[1]))','text/html','FILE14');
+INSERT INTO files VALUES (140,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/1/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE46');
+INSERT INTO files VALUES (141,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/229/comment()[1]/range-to(/1/2/241/comment()[1]))','text/html','FILE24');
+INSERT INTO files VALUES (142,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/284/comment()[1]/range-to(/1/2/297/comment()[1]))','text/html','FILE35');
+INSERT INTO files VALUES (143,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/56/comment()[1]/range-to(/1/2/65/comment()[1]))','text/html','FILE6');
+INSERT INTO files VALUES (144,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/102/comment()[1]/range-to(/1/2/115/comment()[1]))','text/html','FILE13');
+INSERT INTO files VALUES (145,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/138/comment()[1]/range-to(/1/2/244/comment()[1]))','text/html','FILE26');
+INSERT INTO files VALUES (146,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/88/comment()[1]/range-to(/1/2/90/comment()[1]))','text/html','FILE10');
+INSERT INTO files VALUES (147,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/269/comment()[1]/range-to(/1/2/275/comment()[1]))','text/html','FILE32');
+INSERT INTO files VALUES (148,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/281/comment()[1]/range-to(/1/2/284/comment()[1]))','text/html','FILE34');
+INSERT INTO files VALUES (149,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/245/comment()[1]/range-to(/1/2/247/comment()[1]))','text/html','FILE27');
+INSERT INTO files VALUES (150,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/93/comment()[1]/range-to(/1/2/102/comment()[1]))','text/html','FILE12');
+INSERT INTO files VALUES (151,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/51/comment()[1]/range-to(/1/2/56/comment()[1]))','text/html','FILE5');
+INSERT INTO files VALUES (152,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/90/comment()[1]/range-to(/1/2/138/comment()[1]))','text/html','FILE15');
+INSERT INTO files VALUES (153,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/78/comment()[1]/range-to(/1/2/88/comment()[1]))','text/html','FILE8');
+INSERT INTO files VALUES (154,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/208/comment()[1]/range-to(/1/2/218/comment()[1]))','text/html','FILE22');
+INSERT INTO files VALUES (155,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/339/comment()[1]/range-to(/1/2/347/comment()[1]))','text/html','FILE40');
+INSERT INTO files VALUES (156,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/251/comment()[1]/range-to(/1/2/254/comment()[1]))','text/html','FILE29');
+INSERT INTO files VALUES (157,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/150/comment()[1]/range-to(/1/2/176/comment()[1]))','text/html','FILE18');
+INSERT INTO files VALUES (158,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/357/comment()[1]/range-to(/1/2/366/comment()[1]))','text/html','FILE44');
+INSERT INTO files VALUES (159,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/311/comment()[1]/range-to(/1/2/328/comment()[1]))','text/html','FILE37');
+INSERT INTO files VALUES (160,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/36/comment()[1]/range-to(/1/2/41/comment()[1]))','text/html','FILE3');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm','text/html','default.1');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p16.png','image/png','default.2');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p17.png','image/png','default.3');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p18.png','image/png','default.4');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p19.png','image/png','default.5');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p20.png','image/png','default.6');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p23.png','image/png','default.7');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p24.png','image/png','default.8');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p28.png','image/png','default.9');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p29a.png','image/png','default.10');
+INSERT INTO files VALUES (161,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/p29b.png','image/png','default.11');
+INSERT INTO files VALUES (162,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/142/comment()[1]/range-to(/1/2/150/comment()[1]))','text/html','FILE17');
+INSERT INTO files VALUES (163,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/267/comment()[1]/range-to(/1/2/269/comment()[1]))','text/html','FILE31');
+INSERT INTO files VALUES (164,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/29/comment()[1]/range-to(/1/2/35/comment()[1]))','text/html','FILE2');
+INSERT INTO files VALUES (165,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/353/comment()[1]/range-to(/1/2/357/comment()[1]))','text/html','FILE42');
+INSERT INTO files VALUES (166,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/332/comment()[1]/range-to(/1/2/339/comment()[1]))','text/html','FILE39');
+INSERT INTO files VALUES (167,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/182/comment()[1]/range-to(/1/2/197/comment()[1]))','text/html','FILE20');
+INSERT INTO files VALUES (168,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/218/comment()[1]/range-to(/1/2/229/comment()[1]))','text/html','FILE23');
+INSERT INTO files VALUES (169,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/35/comment()[1]/range-to(/1/2/88/comment()[1]))','text/html','FILE9');
+INSERT INTO files VALUES (170,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/139/comment()[1]/range-to(/1/2/142/comment()[1]))','text/html','FILE16');
+INSERT INTO files VALUES (171,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/176/comment()[1]/range-to(/1/2/182/comment()[1]))','text/html','FILE19');
+INSERT INTO files VALUES (172,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/244/comment()[1]/range-to(/1/2/357/comment()[1]))','text/html','FILE43');
+INSERT INTO files VALUES (173,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/328/comment()[1]/range-to(/1/2/332/comment()[1]))','text/html','FILE38');
+INSERT INTO files VALUES (174,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/366/comment()[1]/range-to(/1/2/comment()[1]))','text/html','FILE45');
+INSERT INTO files VALUES (175,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/wb34te/wb34te.htm#xpointer(/1/2/197/comment()[1]/range-to(/1/2/208/comment()[1]))','text/html','FILE21');
+INSERT INTO files VALUES (176,'URL','file:/tmp/kjdon/gsdl3/web/sites/localsite/collect/gs3mgdemo/import/metadata.xml','application/xml','default.1');
+
+--
+-- Table structure for table `filesection`
+--
+
+CREATE TABLE filesection (
+  FileSectionRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  FileSecID varchar(64) default NULL,
+  PRIMARY KEY  (FileSectionRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `filesection`
+--
+
+INSERT INTO filesection VALUES (1,'20050201:1','test');
+INSERT INTO filesection VALUES (2,'20050201:2','test');
+INSERT INTO filesection VALUES (3,'20050201:3','test');
+INSERT INTO filesection VALUES (4,'20050201:4','test');
+INSERT INTO filesection VALUES (5,'20050201:5','test');
+INSERT INTO filesection VALUES (6,'20050201:6','test');
+
+--
+-- Table structure for table `mdgroups`
+--
+
+CREATE TABLE mdgroups (
+  NamespaceRef int(11) default NULL,
+  MdGroupRef int(11) NOT NULL auto_increment,
+  MdGroupID varchar(32) default NULL,
+  PRIMARY KEY  (MdGroupRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdgroups`
+--
+
+
+--
+-- Table structure for table `mdqualifiers`
+--
+
+CREATE TABLE mdqualifiers (
+  MdGroupRef int(11) default NULL,
+  QualifierName varchar(64) default NULL,
+  QualifierValue varchar(64) default NULL
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdqualifiers`
+--
+
+
+--
+-- Table structure for table `mdvalues`
+--
+
+CREATE TABLE mdvalues (
+  NamespaceRef int(11) default NULL,
+  GroupRef int(11) default NULL,
+  Label tinytext,
+  Value tinytext
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `mdvalues`
+--
+
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (20,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (51,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (51,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','BOSTID');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','achieve gender equality');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','Women, gender and development, women\'s organizations');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','introduce small animals and micro-livestock in your farm');
+INSERT INTO mdvalues VALUES (51,NULL,'Subject','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','introduce little-known Asian farm animals with a promising future');
+INSERT INTO mdvalues VALUES (20,NULL,'classified','BOSTID');
+INSERT INTO mdvalues VALUES (20,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (111,NULL,'Subject','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Subject','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (89,NULL,'mgseqno','mgsearch.84');
+INSERT INTO mdvalues VALUES (89,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (156,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (156,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (156,NULL,'mgseqno','mgsearch.133');
+INSERT INTO mdvalues VALUES (156,NULL,'Title','Development in practice: Toward Gender Equality (wb34te)');
+INSERT INTO mdvalues VALUES (20,NULL,'mgseqno','mgsearch.1');
+INSERT INTO mdvalues VALUES (20,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (51,NULL,'classified','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (89,NULL,'Title','Farming snails 1: Learning about snails; Building a pen; Food and shelter plants (fb33fe)');
+INSERT INTO mdvalues VALUES (111,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','farm snails');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (156,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (20,NULL,'Title','Little Known Asian Animals With a Promising Economic Future (b18ase)');
+INSERT INTO mdvalues VALUES (51,NULL,'Title','Microlivestock - Little-Known Small Animals with a Promising Economic Future (b17mie)');
+INSERT INTO mdvalues VALUES (51,NULL,'mgseqno','mgsearch.31');
+INSERT INTO mdvalues VALUES (89,NULL,'Subject','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (111,NULL,'mgseqno','mgsearch.108');
+INSERT INTO mdvalues VALUES (20,NULL,'Organization','BOSTID');
+INSERT INTO mdvalues VALUES (20,NULL,'Subject','Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)');
+INSERT INTO mdvalues VALUES (51,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (51,NULL,'Organization','BOSTID');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (111,NULL,'Organization','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (156,NULL,'Organization','World Bank');
+INSERT INTO mdvalues VALUES (156,NULL,'Subject','Women, gender and development, women\'s organizations');
+INSERT INTO mdvalues VALUES (20,NULL,'Keyword','introduce little-known Asian farm animals with a promising future');
+INSERT INTO mdvalues VALUES (51,NULL,'Keyword','introduce small animals and micro-livestock in your farm');
+INSERT INTO mdvalues VALUES (89,NULL,'Generator','Microsoft Word 97');
+INSERT INTO mdvalues VALUES (89,NULL,'Subject','Better Farming series of FAO and INADES - 46 booklets');
+INSERT INTO mdvalues VALUES (111,NULL,'classified','Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc.');
+INSERT INTO mdvalues VALUES (111,NULL,'Title','Farming snails 2: Choosing snails; Care and harvesting; Further improvement (fb34fe)');
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (89,NULL,'Organization','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (90,NULL,'Title','Dew and rain');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (111,NULL,'AZList','T.1');
+INSERT INTO mdvalues VALUES (111,NULL,'Keyword','farm snails');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (156,NULL,'classified','World Bank');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+INSERT INTO mdvalues VALUES (1,NULL,'Title','12 The Sulawesi Warty Pig');
+INSERT INTO mdvalues VALUES (2,NULL,'Title','Selected Readings');
+INSERT INTO mdvalues VALUES (3,NULL,'Title','6  Wild Banteng');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (5,NULL,'Title','Appendixes');
+INSERT INTO mdvalues VALUES (4,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (6,NULL,'Title','1 Domesticated Banteng');
+INSERT INTO mdvalues VALUES (7,NULL,'Title','13 Javan Warty Pig');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (8,NULL,'Title','7 Gaur');
+INSERT INTO mdvalues VALUES (9,NULL,'Title','Little Known Asian Animals With a Promising Economic Future');
+INSERT INTO mdvalues VALUES (11,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (10,NULL,'Title','Board on Science and Technology for International Development');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (12,NULL,'Title','3 Mithan');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (13,NULL,'Title','9 Tamaraw');
+INSERT INTO mdvalues VALUES (14,NULL,'Title','15 The Babirusa');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (15,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (16,NULL,'Title','Advisory Committee on Technology Innovation');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (17,NULL,'Title','10 Anoas');
+INSERT INTO mdvalues VALUES (18,NULL,'Title','Part IV : Recommendations and  General Research  Needs');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (19,NULL,'Title','4 Yak');
+INSERT INTO mdvalues VALUES (21,NULL,'mgseqno','mgsearch.11');
+INSERT INTO mdvalues VALUES (21,NULL,'Title','5 Yakows');
+INSERT INTO mdvalues VALUES (22,NULL,'Title','Part I : Domesticated Bovine Species');
+INSERT INTO mdvalues VALUES (23,NULL,'Title','11 The Bearded Pig');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (26,NULL,'Title','14 Pigmy Hog');
+INSERT INTO mdvalues VALUES (24,NULL,'Title','Part II : Wild Bovine Species');
+INSERT INTO mdvalues VALUES (25,NULL,'Title','Part IlI : Pig and Piglike Species');
+INSERT INTO mdvalues VALUES (27,NULL,'Title','2 Banteng Cattle Hybrids');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (28,NULL,'Title','8 Kouprey');
+INSERT INTO mdvalues VALUES (29,NULL,'Title','Research Contacts');
+INSERT INTO mdvalues VALUES (30,NULL,'Title','19 Grasscutter');
+INSERT INTO mdvalues VALUES (31,NULL,'Title','26 Mouse Deer');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (32,NULL,'Title','5 Chicken');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (33,NULL,'Title','17 Coypu');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (34,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (35,NULL,'Title','34 Black Iguana');
+INSERT INTO mdvalues VALUES (36,NULL,'Title','1 Microcattle');
+INSERT INTO mdvalues VALUES (37,NULL,'Title','20 Guinea Pig');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (38,NULL,'Title','31 Duikers');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (39,NULL,'Title','6 Ducks');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (40,NULL,'Title','35 Bees');
+INSERT INTO mdvalues VALUES (41,NULL,'Title','Board on Science and Technology for International Development (BOSTID)');
+INSERT INTO mdvalues VALUES (42,NULL,'Title','33 Green Iguana');
+INSERT INTO mdvalues VALUES (43,NULL,'Title','C BIOGRAPHICAL SKETCHES OF PANEL MEMBERS');
+INSERT INTO mdvalues VALUES (45,NULL,'Title','Part VII Others');
+INSERT INTO mdvalues VALUES (44,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (46,NULL,'Title','12 Turkey');
+INSERT INTO mdvalues VALUES (47,NULL,'Title','8 Guinea Fowl');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (48,NULL,'Title','18 Giant Rat');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (49,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (50,NULL,'Title','4 Micropigs');
+INSERT INTO mdvalues VALUES (52,NULL,'Title','25 Other Rodents');
+INSERT INTO mdvalues VALUES (53,NULL,'Title','Part I : Microbreeds');
+INSERT INTO mdvalues VALUES (54,NULL,'Title','Part II : Poultry');
+INSERT INTO mdvalues VALUES (55,NULL,'Title','32 Other Small Antelope');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (56,NULL,'Title','24 Vizcacha');
+INSERT INTO mdvalues VALUES (57,NULL,'Title','29 South America\'s Microdeer');
+INSERT INTO mdvalues VALUES (58,NULL,'Title','30 Water Deer');
+INSERT INTO mdvalues VALUES (59,NULL,'Title','Part V : Deer and Antelope');
+INSERT INTO mdvalues VALUES (60,NULL,'Title','Microlivestock   LittleKnown Small Animals with a Promising Economic Future');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (61,NULL,'Title','3 Microsheep');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (62,NULL,'Title','21 Hutia');
+INSERT INTO mdvalues VALUES (63,NULL,'Title','Part VI : Lizards');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (64,NULL,'Title','11 Quail');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (65,NULL,'Title','9 Muscovy');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (66,NULL,'Title','23 Paca');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (67,NULL,'Title','Appendixes');
+INSERT INTO mdvalues VALUES (68,NULL,'Title','Part IV : Rodents');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (69,NULL,'Title','10 Pigeon');
+INSERT INTO mdvalues VALUES (70,NULL,'Title','14 Domestic Rabbit');
+INSERT INTO mdvalues VALUES (71,NULL,'Title','Part III : Rabbits');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (72,NULL,'Title','16 Capybara');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (73,NULL,'Title','2 Microgoats');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (74,NULL,'Title','7 Geese');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (75,NULL,'Title','28 Musk Deer');
+INSERT INTO mdvalues VALUES (76,NULL,'Title','B Research Contacts');
+INSERT INTO mdvalues VALUES (77,NULL,'Title','A Selected Readings');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (78,NULL,'Title','15 Agouti');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (80,NULL,'Title','27 Muntjac');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (79,NULL,'Title','22 Mara');
+INSERT INTO mdvalues VALUES (81,NULL,'Title','13 Potential New Poultry');
+INSERT INTO mdvalues VALUES (82,NULL,'Title','Now you must decide');
+INSERT INTO mdvalues VALUES (83,NULL,'Title','When are the plants big enough for food and for shelter?');
+INSERT INTO mdvalues VALUES (84,NULL,'Title','Land that is wet');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (85,NULL,'Title','How to begin');
+INSERT INTO mdvalues VALUES (86,NULL,'Title','What do you need to start?');
+INSERT INTO mdvalues VALUES (87,NULL,'Title','Where can you farm snails?');
+INSERT INTO mdvalues VALUES (88,NULL,'Title','How much snail meat can you expect to get?');
+INSERT INTO mdvalues VALUES (91,NULL,'Title','Where can you get snails?');
+INSERT INTO mdvalues VALUES (92,NULL,'Title','Farming snails 1: Learning about snails; Building a pen; Food and shelter plants');
+INSERT INTO mdvalues VALUES (93,NULL,'Title','Building a pen');
+INSERT INTO mdvalues VALUES (94,NULL,'Title','Plant the food and shelter plants');
+INSERT INTO mdvalues VALUES (95,NULL,'Title','Choose a place on your land to put your snail farm');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (96,NULL,'Title','Introduction');
+INSERT INTO mdvalues VALUES (97,NULL,'Title','Plants for food and plants for shelter');
+INSERT INTO mdvalues VALUES (98,NULL,'Title','Where do you keep snails when you farm them?');
+INSERT INTO mdvalues VALUES (99,NULL,'Title','What kind of snails can you farm?');
+INSERT INTO mdvalues VALUES (100,NULL,'Title','Clean the site and prepare the ground');
+INSERT INTO mdvalues VALUES (101,NULL,'Title','The soil and the water in the soil');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (103,NULL,'Title','Wind');
+INSERT INTO mdvalues VALUES (102,NULL,'Title','Learning about snails');
+INSERT INTO mdvalues VALUES (104,NULL,'Title','How many snails will you need when you first begin?');
+INSERT INTO mdvalues VALUES (105,NULL,'Title','Taking better care of your snails');
+INSERT INTO mdvalues VALUES (106,NULL,'Title','When to harvest your snails');
+INSERT INTO mdvalues VALUES (107,NULL,'Title','Taking care of your snails');
+INSERT INTO mdvalues VALUES (108,NULL,'Title','When to take out the fully grown snails');
+INSERT INTO mdvalues VALUES (109,NULL,'Title','Further improvement');
+INSERT INTO mdvalues VALUES (110,NULL,'Title','When to build the second pen and how to build it');
+INSERT INTO mdvalues VALUES (112,NULL,'Title','How to harvest your snails');
+INSERT INTO mdvalues VALUES (113,NULL,'Title','Putting your snails in the pen');
+INSERT INTO mdvalues VALUES (114,NULL,'Title','Take away the weeds and creeping grass');
+INSERT INTO mdvalues VALUES (115,NULL,'Title','Farming snails 2: Choosing snails; Care and harvesting; Further improvement');
+INSERT INTO mdvalues VALUES (116,NULL,'Title','Building the second pen');
+INSERT INTO mdvalues VALUES (118,NULL,'Title','Harvesting your snails');
+INSERT INTO mdvalues VALUES (117,NULL,'Title','Preparing snails for cooking');
+INSERT INTO mdvalues VALUES (121,NULL,'Title','Wet the food and shelter plants and moisten the ground');
+INSERT INTO mdvalues VALUES (120,NULL,'Title','Putting snails in your second pen');
+INSERT INTO mdvalues VALUES (119,NULL,'Title','Building a bigger pen');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (123,NULL,'Title','Preface');
+INSERT INTO mdvalues VALUES (122,NULL,'Title','What to do with the snails you harvest');
+INSERT INTO mdvalues VALUES (124,NULL,'Title','Feeding your snails other kinds of food');
+INSERT INTO mdvalues VALUES (125,NULL,'Title','Building the third pen');
+INSERT INTO mdvalues VALUES (126,NULL,'Title','Choosing your snails');
+INSERT INTO mdvalues VALUES (127,NULL,'Title','Tend the food and shelter plants');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (128,NULL,'Title','The snails');
+INSERT INTO mdvalues VALUES (130,NULL,'Title','Household and Intrahousehold Resource Allocation');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (129,NULL,'Title','Family Law');
+INSERT INTO mdvalues VALUES (132,NULL,'Title','Gender Inequalities Persist');
+INSERT INTO mdvalues VALUES (131,NULL,'Title','Land and Property Rights');
+INSERT INTO mdvalues VALUES (135,NULL,'Title','Definitions and Data Notes');
+INSERT INTO mdvalues VALUES (134,NULL,'Title','Strengthening International Policies to Meet New Challenges');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (133,NULL,'Title','Notes');
+INSERT INTO mdvalues VALUES (137,NULL,'Title','Linkages between Education Health, and Nutritious');
+INSERT INTO mdvalues VALUES (136,NULL,'Title','Women\'s bargaining position in relation to household');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (138,NULL,'Title','Education');
+INSERT INTO mdvalues VALUES (139,NULL,'Title','Development in practice: Toward Gender Equality');
+INSERT INTO mdvalues VALUES (140,NULL,'Title','Working in Collaboration');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (141,NULL,'Title','Conclusions');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (142,NULL,'Title','Foreword');
+INSERT INTO mdvalues VALUES (143,NULL,'Title','Why Do Gender Inequalities Persist?');
+INSERT INTO mdvalues VALUES (144,NULL,'Title','Employment Work');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (145,NULL,'Title','Data Notes');
+INSERT INTO mdvalues VALUES (146,NULL,'Title','Formal Sector Employment');
+INSERT INTO mdvalues VALUES (147,NULL,'Title','Labor Market Policies and Employment Law');
+INSERT INTO mdvalues VALUES (148,NULL,'Title','Financial Laws and Regulations');
+INSERT INTO mdvalues VALUES (149,NULL,'Title','Acknowledgments');
+INSERT INTO mdvalues VALUES (150,NULL,'Title','Informal Sector');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (151,NULL,'Title','References');
+INSERT INTO mdvalues VALUES (152,NULL,'Title','Involving Beneficiaries in Public Policy');
+INSERT INTO mdvalues VALUES (154,NULL,'Title','Access to Extension Services');
+INSERT INTO mdvalues VALUES (153,NULL,'Title','Inflation tends to hit women harder than men.');
+INSERT INTO mdvalues VALUES (155,NULL,'Title','Chapter three');
+INSERT INTO mdvalues VALUES (156,NULL,'Keyword','achieve gender equality');
+INSERT INTO mdvalues VALUES (157,NULL,'Title','Using Targeting Measures to Narrow the Gender');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (160,NULL,'Title','Summary');
+INSERT INTO mdvalues VALUES (159,NULL,'Title','Access to Lund and Property');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (158,NULL,'Title','Conclusion');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (161,NULL,'Title','Conclusion');
+INSERT INTO mdvalues VALUES (162,NULL,'Title','Gender Inequalities Hamper Growth');
+INSERT INTO mdvalues VALUES (165,NULL,'Title','Household and Labor Market Linkages');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (164,NULL,'Title','Chapter two');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (163,NULL,'Title','Chapter one');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (167,NULL,'Title','Definitions');
+INSERT INTO mdvalues VALUES (166,NULL,'Title','Equalizing Opportunities by Modifying, the Legal Framework');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (168,NULL,'Title','Health');
+INSERT INTO mdvalues VALUES (169,NULL,'Title','Progress to Date');
+INSERT INTO mdvalues VALUES (170,NULL,'Title','Sectoral Investments');
+INSERT INTO mdvalues VALUES (171,NULL,'Title','Access to Financial Markets');
+INSERT INTO mdvalues VALUES (172,NULL,'Title','Public Policies Matter');
+INSERT INTO mdvalues VALUES (174,NULL,'Title','Macroeconomic: Policies');
+INSERT INTO mdvalues VALUES (173,NULL,'Title','Generating and Analyzing GenderDesegregated Data');
+INSERT INTO mdvalues VALUES (175,NULL,'Title','Strategies for the Future');
+INSERT INTO mdvalues VALUES (1,NULL,'mgseqno','mgsearch.20');
+INSERT INTO mdvalues VALUES (2,NULL,'mgseqno','mgsearch.26');
+INSERT INTO mdvalues VALUES (3,NULL,'mgseqno','mgsearch.13');
+INSERT INTO mdvalues VALUES (5,NULL,'mgseqno','mgsearch.25');
+INSERT INTO mdvalues VALUES (4,NULL,'mgseqno','mgsearch.5');
+INSERT INTO mdvalues VALUES (6,NULL,'mgseqno','mgsearch.7');
+INSERT INTO mdvalues VALUES (7,NULL,'mgseqno','mgsearch.21');
+INSERT INTO mdvalues VALUES (8,NULL,'mgseqno','mgsearch.14');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (9,NULL,'mgseqno','mgsearch.2');
+INSERT INTO mdvalues VALUES (11,NULL,'mgseqno','mgsearch.3');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (10,NULL,'mgseqno','mgsearch.29');
+INSERT INTO mdvalues VALUES (12,NULL,'mgseqno','mgsearch.9');
+INSERT INTO mdvalues VALUES (13,NULL,'mgseqno','mgsearch.16');
+INSERT INTO mdvalues VALUES (14,NULL,'mgseqno','mgsearch.23');
+INSERT INTO mdvalues VALUES (15,NULL,'mgseqno','mgsearch.4');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (16,NULL,'mgseqno','mgsearch.28');
+INSERT INTO mdvalues VALUES (17,NULL,'mgseqno','mgsearch.17');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (18,NULL,'mgseqno','mgsearch.24');
+INSERT INTO mdvalues VALUES (19,NULL,'mgseqno','mgsearch.10');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (22,NULL,'mgseqno','mgsearch.6');
+INSERT INTO mdvalues VALUES (23,NULL,'mgseqno','mgsearch.19');
+INSERT INTO mdvalues VALUES (26,NULL,'mgseqno','mgsearch.22');
+INSERT INTO mdvalues VALUES (24,NULL,'mgseqno','mgsearch.12');
+INSERT INTO mdvalues VALUES (25,NULL,'mgseqno','mgsearch.18');
+INSERT INTO mdvalues VALUES (27,NULL,'mgseqno','mgsearch.8');
+INSERT INTO mdvalues VALUES (28,NULL,'mgseqno','mgsearch.15');
+INSERT INTO mdvalues VALUES (29,NULL,'mgseqno','mgsearch.27');
+INSERT INTO mdvalues VALUES (30,NULL,'mgseqno','mgsearch.58');
+INSERT INTO mdvalues VALUES (31,NULL,'mgseqno','mgsearch.66');
+INSERT INTO mdvalues VALUES (32,NULL,'mgseqno','mgsearch.42');
+INSERT INTO mdvalues VALUES (33,NULL,'mgseqno','mgsearch.56');
+INSERT INTO mdvalues VALUES (34,NULL,'mgseqno','mgsearch.35');
+INSERT INTO mdvalues VALUES (35,NULL,'mgseqno','mgsearch.75');
+INSERT INTO mdvalues VALUES (36,NULL,'mgseqno','mgsearch.37');
+INSERT INTO mdvalues VALUES (37,NULL,'mgseqno','mgsearch.59');
+INSERT INTO mdvalues VALUES (38,NULL,'mgseqno','mgsearch.71');
+INSERT INTO mdvalues VALUES (39,NULL,'mgseqno','mgsearch.43');
+INSERT INTO mdvalues VALUES (40,NULL,'mgseqno','mgsearch.77');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (41,NULL,'mgseqno','mgsearch.82');
+INSERT INTO mdvalues VALUES (42,NULL,'mgseqno','mgsearch.74');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (43,NULL,'mgseqno','mgsearch.81');
+INSERT INTO mdvalues VALUES (45,NULL,'mgseqno','mgsearch.76');
+INSERT INTO mdvalues VALUES (44,NULL,'mgseqno','mgsearch.33');
+INSERT INTO mdvalues VALUES (46,NULL,'mgseqno','mgsearch.49');
+INSERT INTO mdvalues VALUES (47,NULL,'mgseqno','mgsearch.45');
+INSERT INTO mdvalues VALUES (48,NULL,'mgseqno','mgsearch.57');
+INSERT INTO mdvalues VALUES (49,NULL,'mgseqno','mgsearch.34');
+INSERT INTO mdvalues VALUES (50,NULL,'mgseqno','mgsearch.40');
+INSERT INTO mdvalues VALUES (52,NULL,'mgseqno','mgsearch.64');
+INSERT INTO mdvalues VALUES (53,NULL,'mgseqno','mgsearch.36');
+INSERT INTO mdvalues VALUES (54,NULL,'mgseqno','mgsearch.41');
+INSERT INTO mdvalues VALUES (55,NULL,'mgseqno','mgsearch.72');
+INSERT INTO mdvalues VALUES (56,NULL,'mgseqno','mgsearch.63');
+INSERT INTO mdvalues VALUES (57,NULL,'mgseqno','mgsearch.69');
+INSERT INTO mdvalues VALUES (58,NULL,'mgseqno','mgsearch.70');
+INSERT INTO mdvalues VALUES (59,NULL,'mgseqno','mgsearch.65');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (60,NULL,'mgseqno','mgsearch.32');
+INSERT INTO mdvalues VALUES (61,NULL,'mgseqno','mgsearch.39');
+INSERT INTO mdvalues VALUES (62,NULL,'mgseqno','mgsearch.60');
+INSERT INTO mdvalues VALUES (63,NULL,'mgseqno','mgsearch.73');
+INSERT INTO mdvalues VALUES (64,NULL,'mgseqno','mgsearch.48');
+INSERT INTO mdvalues VALUES (65,NULL,'mgseqno','mgsearch.46');
+INSERT INTO mdvalues VALUES (66,NULL,'mgseqno','mgsearch.62');
+INSERT INTO mdvalues VALUES (67,NULL,'mgseqno','mgsearch.78');
+INSERT INTO mdvalues VALUES (68,NULL,'mgseqno','mgsearch.53');
+INSERT INTO mdvalues VALUES (69,NULL,'mgseqno','mgsearch.47');
+INSERT INTO mdvalues VALUES (70,NULL,'mgseqno','mgsearch.52');
+INSERT INTO mdvalues VALUES (71,NULL,'mgseqno','mgsearch.51');
+INSERT INTO mdvalues VALUES (72,NULL,'mgseqno','mgsearch.55');
+INSERT INTO mdvalues VALUES (73,NULL,'mgseqno','mgsearch.38');
+INSERT INTO mdvalues VALUES (74,NULL,'mgseqno','mgsearch.44');
+INSERT INTO mdvalues VALUES (75,NULL,'mgseqno','mgsearch.68');
+INSERT INTO mdvalues VALUES (76,NULL,'mgseqno','mgsearch.80');
+INSERT INTO mdvalues VALUES (77,NULL,'mgseqno','mgsearch.79');
+INSERT INTO mdvalues VALUES (78,NULL,'mgseqno','mgsearch.54');
+INSERT INTO mdvalues VALUES (80,NULL,'mgseqno','mgsearch.67');
+INSERT INTO mdvalues VALUES (79,NULL,'mgseqno','mgsearch.61');
+INSERT INTO mdvalues VALUES (81,NULL,'mgseqno','mgsearch.50');
+INSERT INTO mdvalues VALUES (82,NULL,'mgseqno','mgsearch.100');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (83,NULL,'mgseqno','mgsearch.106');
+INSERT INTO mdvalues VALUES (84,NULL,'mgseqno','mgsearch.95');
+INSERT INTO mdvalues VALUES (85,NULL,'mgseqno','mgsearch.101');
+INSERT INTO mdvalues VALUES (86,NULL,'mgseqno','mgsearch.87');
+INSERT INTO mdvalues VALUES (87,NULL,'mgseqno','mgsearch.91');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (88,NULL,'mgseqno','mgsearch.99');
+INSERT INTO mdvalues VALUES (89,NULL,'classified','FAO Better Farming series');
+INSERT INTO mdvalues VALUES (89,NULL,'Language','English');
+INSERT INTO mdvalues VALUES (90,NULL,'mgseqno','mgsearch.94');
+INSERT INTO mdvalues VALUES (91,NULL,'mgseqno','mgsearch.90');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (92,NULL,'mgseqno','mgsearch.85');
+INSERT INTO mdvalues VALUES (93,NULL,'mgseqno','mgsearch.104');
+INSERT INTO mdvalues VALUES (94,NULL,'mgseqno','mgsearch.105');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (95,NULL,'mgseqno','mgsearch.102');
+INSERT INTO mdvalues VALUES (96,NULL,'mgseqno','mgsearch.86');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (97,NULL,'mgseqno','mgsearch.93');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (98,NULL,'mgseqno','mgsearch.97');
+INSERT INTO mdvalues VALUES (99,NULL,'mgseqno','mgsearch.89');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (100,NULL,'mgseqno','mgsearch.103');
+INSERT INTO mdvalues VALUES (101,NULL,'mgseqno','mgsearch.92');
+INSERT INTO mdvalues VALUES (103,NULL,'mgseqno','mgsearch.96');
+INSERT INTO mdvalues VALUES (102,NULL,'mgseqno','mgsearch.88');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (104,NULL,'mgseqno','mgsearch.98');
+INSERT INTO mdvalues VALUES (105,NULL,'mgseqno','mgsearch.131');
+INSERT INTO mdvalues VALUES (106,NULL,'mgseqno','mgsearch.124');
+INSERT INTO mdvalues VALUES (107,NULL,'mgseqno','mgsearch.115');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (108,NULL,'mgseqno','mgsearch.114');
+INSERT INTO mdvalues VALUES (109,NULL,'mgseqno','mgsearch.128');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (110,NULL,'mgseqno','mgsearch.121');
+INSERT INTO mdvalues VALUES (112,NULL,'mgseqno','mgsearch.125');
+INSERT INTO mdvalues VALUES (113,NULL,'mgseqno','mgsearch.113');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (114,NULL,'mgseqno','mgsearch.118');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (115,NULL,'mgseqno','mgsearch.109');
+INSERT INTO mdvalues VALUES (116,NULL,'mgseqno','mgsearch.120');
+INSERT INTO mdvalues VALUES (118,NULL,'mgseqno','mgsearch.123');
+INSERT INTO mdvalues VALUES (117,NULL,'mgseqno','mgsearch.127');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (121,NULL,'mgseqno','mgsearch.116');
+INSERT INTO mdvalues VALUES (120,NULL,'mgseqno','mgsearch.122');
+INSERT INTO mdvalues VALUES (119,NULL,'mgseqno','mgsearch.130');
+INSERT INTO mdvalues VALUES (123,NULL,'mgseqno','mgsearch.110');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (122,NULL,'mgseqno','mgsearch.126');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (124,NULL,'mgseqno','mgsearch.119');
+INSERT INTO mdvalues VALUES (125,NULL,'mgseqno','mgsearch.129');
+INSERT INTO mdvalues VALUES (126,NULL,'mgseqno','mgsearch.112');
+INSERT INTO mdvalues VALUES (127,NULL,'mgseqno','mgsearch.117');
+INSERT INTO mdvalues VALUES (128,NULL,'mgseqno','mgsearch.111');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (130,NULL,'mgseqno','mgsearch.152');
+INSERT INTO mdvalues VALUES (129,NULL,'mgseqno','mgsearch.166');
+INSERT INTO mdvalues VALUES (132,NULL,'mgseqno','mgsearch.146');
+INSERT INTO mdvalues VALUES (131,NULL,'mgseqno','mgsearch.164');
+INSERT INTO mdvalues VALUES (135,NULL,'mgseqno','mgsearch.137');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (134,NULL,'mgseqno','mgsearch.176');
+INSERT INTO mdvalues VALUES (133,NULL,'mgseqno','mgsearch.178');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (137,NULL,'mgseqno','mgsearch.153');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (136,NULL,'mgseqno','mgsearch.167');
+INSERT INTO mdvalues VALUES (138,NULL,'mgseqno','mgsearch.147');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (139,NULL,'mgseqno','mgsearch.134');
+INSERT INTO mdvalues VALUES (140,NULL,'mgseqno','mgsearch.175');
+INSERT INTO mdvalues VALUES (141,NULL,'mgseqno','mgsearch.177');
+INSERT INTO mdvalues VALUES (142,NULL,'mgseqno','mgsearch.135');
+INSERT INTO mdvalues VALUES (143,NULL,'mgseqno','mgsearch.142');
+INSERT INTO mdvalues VALUES (144,NULL,'mgseqno','mgsearch.149');
+INSERT INTO mdvalues VALUES (145,NULL,'mgseqno','mgsearch.139');
+INSERT INTO mdvalues VALUES (146,NULL,'mgseqno','mgsearch.155');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (147,NULL,'mgseqno','mgsearch.165');
+INSERT INTO mdvalues VALUES (148,NULL,'mgseqno','mgsearch.168');
+INSERT INTO mdvalues VALUES (149,NULL,'mgseqno','mgsearch.136');
+INSERT INTO mdvalues VALUES (150,NULL,'mgseqno','mgsearch.156');
+INSERT INTO mdvalues VALUES (151,NULL,'mgseqno','mgsearch.179');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (152,NULL,'mgseqno','mgsearch.173');
+INSERT INTO mdvalues VALUES (154,NULL,'mgseqno','mgsearch.159');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (153,NULL,'mgseqno','mgsearch.170');
+INSERT INTO mdvalues VALUES (155,NULL,'mgseqno','mgsearch.161');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (157,NULL,'mgseqno','mgsearch.172');
+INSERT INTO mdvalues VALUES (160,NULL,'mgseqno','mgsearch.140');
+INSERT INTO mdvalues VALUES (159,NULL,'mgseqno','mgsearch.158');
+INSERT INTO mdvalues VALUES (158,NULL,'mgseqno','mgsearch.160');
+INSERT INTO mdvalues VALUES (161,NULL,'mgseqno','mgsearch.144');
+INSERT INTO mdvalues VALUES (162,NULL,'mgseqno','mgsearch.151');
+INSERT INTO mdvalues VALUES (165,NULL,'mgseqno','mgsearch.154');
+INSERT INTO mdvalues VALUES (164,NULL,'mgseqno','mgsearch.150');
+INSERT INTO mdvalues VALUES (163,NULL,'mgseqno','mgsearch.145');
+INSERT INTO mdvalues VALUES (167,NULL,'mgseqno','mgsearch.138');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (166,NULL,'mgseqno','mgsearch.163');
+INSERT INTO mdvalues VALUES (168,NULL,'mgseqno','mgsearch.148');
+INSERT INTO mdvalues VALUES (169,NULL,'mgseqno','mgsearch.141');
+INSERT INTO mdvalues VALUES (170,NULL,'mgseqno','mgsearch.171');
+INSERT INTO mdvalues VALUES (171,NULL,'mgseqno','mgsearch.157');
+INSERT INTO mdvalues VALUES (172,NULL,'mgseqno','mgsearch.162');
+INSERT INTO mdvalues VALUES (174,NULL,'mgseqno','mgsearch.169');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (173,NULL,'mgseqno','mgsearch.174');
+INSERT INTO mdvalues VALUES (175,NULL,'mgseqno','mgsearch.143');
+
+--
+-- Table structure for table `metadata`
+--
+
+CREATE TABLE metadata (
+  MetadataRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  MetaID varchar(64) default NULL,
+  GroupID varchar(64) default NULL,
+  PRIMARY KEY  (MetadataRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `metadata`
+--
+
+INSERT INTO metadata VALUES (1,'20050201:1','DM20','1.6.2');
+INSERT INTO metadata VALUES (2,'20050201:1','DM26','1.8.1');
+INSERT INTO metadata VALUES (3,'20050201:1','DM13','1.5.1');
+INSERT INTO metadata VALUES (4,'20050201:1','DM5','1.3');
+INSERT INTO metadata VALUES (5,'20050201:1','DM25','1.8');
+INSERT INTO metadata VALUES (6,'20050201:1','DM7','1.4.1');
+INSERT INTO metadata VALUES (7,'20050201:1','DM21','1.6.3');
+INSERT INTO metadata VALUES (8,'20050201:1','DM14','1.5.2');
+INSERT INTO metadata VALUES (9,'20050201:1','DM2','1');
+INSERT INTO metadata VALUES (10,'20050201:1','DM29','1.10');
+INSERT INTO metadata VALUES (11,'20050201:1','DM3','1.1');
+INSERT INTO metadata VALUES (12,'20050201:1','DM9','1.4.3');
+INSERT INTO metadata VALUES (13,'20050201:1','DM16','1.5.4');
+INSERT INTO metadata VALUES (14,'20050201:1','DM23','1.6.5');
+INSERT INTO metadata VALUES (15,'20050201:1','DM4','1.2');
+INSERT INTO metadata VALUES (16,'20050201:1','DM28','1.9');
+INSERT INTO metadata VALUES (17,'20050201:1','DM17','1.5.5');
+INSERT INTO metadata VALUES (18,'20050201:1','DM24','1.7');
+INSERT INTO metadata VALUES (19,'20050201:1','DM10','1.4.4');
+INSERT INTO metadata VALUES (20,'20050201:1','DM1','default');
+INSERT INTO metadata VALUES (21,'20050201:1','DM11','1.4.5');
+INSERT INTO metadata VALUES (22,'20050201:1','DM6','1.4');
+INSERT INTO metadata VALUES (23,'20050201:1','DM19','1.6.1');
+INSERT INTO metadata VALUES (24,'20050201:1','DM12','1.5');
+INSERT INTO metadata VALUES (25,'20050201:1','DM18','1.6');
+INSERT INTO metadata VALUES (26,'20050201:1','DM22','1.6.4');
+INSERT INTO metadata VALUES (27,'20050201:1','DM8','1.4.2');
+INSERT INTO metadata VALUES (28,'20050201:1','DM15','1.5.3');
+INSERT INTO metadata VALUES (29,'20050201:1','DM27','1.8.2');
+INSERT INTO metadata VALUES (30,'20050201:2','DM28','1.7.5');
+INSERT INTO metadata VALUES (31,'20050201:2','DM36','1.8.1');
+INSERT INTO metadata VALUES (32,'20050201:2','DM12','1.5.1');
+INSERT INTO metadata VALUES (33,'20050201:2','DM26','1.7.3');
+INSERT INTO metadata VALUES (34,'20050201:2','DM5','1.3');
+INSERT INTO metadata VALUES (35,'20050201:2','DM45','1.9.2');
+INSERT INTO metadata VALUES (36,'20050201:2','DM7','1.4.1');
+INSERT INTO metadata VALUES (37,'20050201:2','DM29','1.7.6');
+INSERT INTO metadata VALUES (38,'20050201:2','DM41','1.8.6');
+INSERT INTO metadata VALUES (39,'20050201:2','DM13','1.5.2');
+INSERT INTO metadata VALUES (40,'20050201:2','DM47','1.10.1');
+INSERT INTO metadata VALUES (41,'20050201:2','DM52','1.12');
+INSERT INTO metadata VALUES (42,'20050201:2','DM44','1.9.1');
+INSERT INTO metadata VALUES (43,'20050201:2','DM51','1.11.3');
+INSERT INTO metadata VALUES (44,'20050201:2','DM3','1.1');
+INSERT INTO metadata VALUES (45,'20050201:2','DM46','1.10');
+INSERT INTO metadata VALUES (46,'20050201:2','DM19','1.5.8');
+INSERT INTO metadata VALUES (47,'20050201:2','DM15','1.5.4');
+INSERT INTO metadata VALUES (48,'20050201:2','DM27','1.7.4');
+INSERT INTO metadata VALUES (49,'20050201:2','DM4','1.2');
+INSERT INTO metadata VALUES (50,'20050201:2','DM10','1.4.4');
+INSERT INTO metadata VALUES (51,'20050201:2','DM1','default');
+INSERT INTO metadata VALUES (52,'20050201:2','DM34','1.7.11');
+INSERT INTO metadata VALUES (53,'20050201:2','DM6','1.4');
+INSERT INTO metadata VALUES (54,'20050201:2','DM11','1.5');
+INSERT INTO metadata VALUES (55,'20050201:2','DM42','1.8.7');
+INSERT INTO metadata VALUES (56,'20050201:2','DM33','1.7.10');
+INSERT INTO metadata VALUES (57,'20050201:2','DM39','1.8.4');
+INSERT INTO metadata VALUES (58,'20050201:2','DM40','1.8.5');
+INSERT INTO metadata VALUES (59,'20050201:2','DM35','1.8');
+INSERT INTO metadata VALUES (60,'20050201:2','DM2','1');
+INSERT INTO metadata VALUES (61,'20050201:2','DM9','1.4.3');
+INSERT INTO metadata VALUES (62,'20050201:2','DM30','1.7.7');
+INSERT INTO metadata VALUES (63,'20050201:2','DM43','1.9');
+INSERT INTO metadata VALUES (64,'20050201:2','DM18','1.5.7');
+INSERT INTO metadata VALUES (65,'20050201:2','DM16','1.5.5');
+INSERT INTO metadata VALUES (66,'20050201:2','DM32','1.7.9');
+INSERT INTO metadata VALUES (67,'20050201:2','DM48','1.11');
+INSERT INTO metadata VALUES (68,'20050201:2','DM23','1.7');
+INSERT INTO metadata VALUES (69,'20050201:2','DM17','1.5.6');
+INSERT INTO metadata VALUES (70,'20050201:2','DM22','1.6.1');
+INSERT INTO metadata VALUES (71,'20050201:2','DM21','1.6');
+INSERT INTO metadata VALUES (72,'20050201:2','DM25','1.7.2');
+INSERT INTO metadata VALUES (73,'20050201:2','DM8','1.4.2');
+INSERT INTO metadata VALUES (74,'20050201:2','DM14','1.5.3');
+INSERT INTO metadata VALUES (75,'20050201:2','DM38','1.8.3');
+INSERT INTO metadata VALUES (76,'20050201:2','DM50','1.11.2');
+INSERT INTO metadata VALUES (77,'20050201:2','DM49','1.11.1');
+INSERT INTO metadata VALUES (78,'20050201:2','DM24','1.7.1');
+INSERT INTO metadata VALUES (79,'20050201:2','DM31','1.7.8');
+INSERT INTO metadata VALUES (80,'20050201:2','DM37','1.8.2');
+INSERT INTO metadata VALUES (81,'20050201:2','DM20','1.5.9');
+INSERT INTO metadata VALUES (82,'20050201:3','DM17','1.2.12');
+INSERT INTO metadata VALUES (83,'20050201:3','DM23','1.5.1');
+INSERT INTO metadata VALUES (84,'20050201:3','DM12','1.2.7');
+INSERT INTO metadata VALUES (85,'20050201:3','DM18','1.3');
+INSERT INTO metadata VALUES (86,'20050201:3','DM4','1.1.1');
+INSERT INTO metadata VALUES (87,'20050201:3','DM8','1.2.3');
+INSERT INTO metadata VALUES (88,'20050201:3','DM16','1.2.11');
+INSERT INTO metadata VALUES (89,'20050201:3','DM1','default');
+INSERT INTO metadata VALUES (90,'20050201:3','DM11','1.2.6');
+INSERT INTO metadata VALUES (91,'20050201:3','DM7','1.2.2');
+INSERT INTO metadata VALUES (92,'20050201:3','DM2','1');
+INSERT INTO metadata VALUES (93,'20050201:3','DM21','1.4');
+INSERT INTO metadata VALUES (94,'20050201:3','DM22','1.5');
+INSERT INTO metadata VALUES (95,'20050201:3','DM19','1.3.1');
+INSERT INTO metadata VALUES (96,'20050201:3','DM3','1.1');
+INSERT INTO metadata VALUES (97,'20050201:3','DM10','1.2.5');
+INSERT INTO metadata VALUES (98,'20050201:3','DM14','1.2.9');
+INSERT INTO metadata VALUES (99,'20050201:3','DM6','1.2.1');
+INSERT INTO metadata VALUES (100,'20050201:3','DM20','1.3.2');
+INSERT INTO metadata VALUES (101,'20050201:3','DM9','1.2.4');
+INSERT INTO metadata VALUES (102,'20050201:3','DM5','1.2');
+INSERT INTO metadata VALUES (103,'20050201:3','DM13','1.2.8');
+INSERT INTO metadata VALUES (104,'20050201:3','DM15','1.2.10');
+INSERT INTO metadata VALUES (105,'20050201:4','DM24','1.7.3');
+INSERT INTO metadata VALUES (106,'20050201:4','DM17','1.5.1');
+INSERT INTO metadata VALUES (107,'20050201:4','DM8','1.3');
+INSERT INTO metadata VALUES (108,'20050201:4','DM7','1.2.3');
+INSERT INTO metadata VALUES (109,'20050201:4','DM21','1.7');
+INSERT INTO metadata VALUES (110,'20050201:4','DM14','1.4.1');
+INSERT INTO metadata VALUES (111,'20050201:4','DM1','default');
+INSERT INTO metadata VALUES (112,'20050201:4','DM18','1.5.2');
+INSERT INTO metadata VALUES (113,'20050201:4','DM6','1.2.2');
+INSERT INTO metadata VALUES (114,'20050201:4','DM11','1.3.3');
+INSERT INTO metadata VALUES (115,'20050201:4','DM2','1');
+INSERT INTO metadata VALUES (116,'20050201:4','DM13','1.4');
+INSERT INTO metadata VALUES (117,'20050201:4','DM20','1.6');
+INSERT INTO metadata VALUES (118,'20050201:4','DM16','1.5');
+INSERT INTO metadata VALUES (119,'20050201:4','DM23','1.7.2');
+INSERT INTO metadata VALUES (120,'20050201:4','DM15','1.4.2');
+INSERT INTO metadata VALUES (121,'20050201:4','DM9','1.3.1');
+INSERT INTO metadata VALUES (122,'20050201:4','DM19','1.5.3');
+INSERT INTO metadata VALUES (123,'20050201:4','DM3','1.1');
+INSERT INTO metadata VALUES (124,'20050201:4','DM12','1.3.4');
+INSERT INTO metadata VALUES (125,'20050201:4','DM22','1.7.1');
+INSERT INTO metadata VALUES (126,'20050201:4','DM5','1.2.1');
+INSERT INTO metadata VALUES (127,'20050201:4','DM10','1.3.2');
+INSERT INTO metadata VALUES (128,'20050201:4','DM4','1.2');
+INSERT INTO metadata VALUES (129,'20050201:5','DM34','1.7.5');
+INSERT INTO metadata VALUES (130,'20050201:5','DM20','1.6.2');
+INSERT INTO metadata VALUES (131,'20050201:5','DM32','1.7.3');
+INSERT INTO metadata VALUES (132,'20050201:5','DM14','1.5.1');
+INSERT INTO metadata VALUES (133,'20050201:5','DM46','1.8');
+INSERT INTO metadata VALUES (134,'20050201:5','DM44','1.7.15');
+INSERT INTO metadata VALUES (135,'20050201:5','DM5','1.3');
+INSERT INTO metadata VALUES (136,'20050201:5','DM35','1.7.6');
+INSERT INTO metadata VALUES (137,'20050201:5','DM21','1.6.3');
+INSERT INTO metadata VALUES (138,'20050201:5','DM15','1.5.2');
+INSERT INTO metadata VALUES (139,'20050201:5','DM2','1');
+INSERT INTO metadata VALUES (140,'20050201:5','DM43','1.7.14');
+INSERT INTO metadata VALUES (141,'20050201:5','DM45','1.7.16');
+INSERT INTO metadata VALUES (142,'20050201:5','DM3','1.1');
+INSERT INTO metadata VALUES (143,'20050201:5','DM10','1.3.5');
+INSERT INTO metadata VALUES (144,'20050201:5','DM17','1.5.4');
+INSERT INTO metadata VALUES (145,'20050201:5','DM7','1.3.2');
+INSERT INTO metadata VALUES (146,'20050201:5','DM23','1.6.5');
+INSERT INTO metadata VALUES (147,'20050201:5','DM33','1.7.4');
+INSERT INTO metadata VALUES (148,'20050201:5','DM36','1.7.7');
+INSERT INTO metadata VALUES (149,'20050201:5','DM4','1.2');
+INSERT INTO metadata VALUES (150,'20050201:5','DM24','1.6.6');
+INSERT INTO metadata VALUES (151,'20050201:5','DM47','1.9');
+INSERT INTO metadata VALUES (152,'20050201:5','DM41','1.7.12');
+INSERT INTO metadata VALUES (153,'20050201:5','DM38','1.7.9');
+INSERT INTO metadata VALUES (154,'20050201:5','DM27','1.6.9');
+INSERT INTO metadata VALUES (155,'20050201:5','DM29','1.7');
+INSERT INTO metadata VALUES (156,'20050201:5','DM1','default');
+INSERT INTO metadata VALUES (157,'20050201:5','DM40','1.7.11');
+INSERT INTO metadata VALUES (158,'20050201:5','DM28','1.6.10');
+INSERT INTO metadata VALUES (159,'20050201:5','DM26','1.6.8');
+INSERT INTO metadata VALUES (160,'20050201:5','DM8','1.3.3');
+INSERT INTO metadata VALUES (161,'20050201:5','DM12','1.4');
+INSERT INTO metadata VALUES (162,'20050201:5','DM19','1.6.1');
+INSERT INTO metadata VALUES (163,'20050201:5','DM13','1.5');
+INSERT INTO metadata VALUES (164,'20050201:5','DM18','1.6');
+INSERT INTO metadata VALUES (165,'20050201:5','DM22','1.6.4');
+INSERT INTO metadata VALUES (166,'20050201:5','DM31','1.7.2');
+INSERT INTO metadata VALUES (167,'20050201:5','DM6','1.3.1');
+INSERT INTO metadata VALUES (168,'20050201:5','DM16','1.5.3');
+INSERT INTO metadata VALUES (169,'20050201:5','DM9','1.3.4');
+INSERT INTO metadata VALUES (170,'20050201:5','DM39','1.7.10');
+INSERT INTO metadata VALUES (171,'20050201:5','DM25','1.6.7');
+INSERT INTO metadata VALUES (172,'20050201:5','DM30','1.7.1');
+INSERT INTO metadata VALUES (173,'20050201:5','DM42','1.7.13');
+INSERT INTO metadata VALUES (174,'20050201:5','DM37','1.7.8');
+INSERT INTO metadata VALUES (175,'20050201:5','DM11','1.3.6');
+INSERT INTO metadata VALUES (176,'20050201:6','DM1','default');
+
+--
+-- Table structure for table `namespaces`
+--
+
+CREATE TABLE namespaces (
+  NamespaceRef int(11) NOT NULL auto_increment,
+  MetadataRef int(11) default NULL,
+  DocID varchar(64) default NULL,
+  MetaID varchar(64) default NULL,
+  NamespaceID varchar(64) default NULL,
+  NamespaceType varchar(64) default NULL,
+  FileType varchar(64) default NULL,
+  FileLoc tinytext,
+  Creator varchar(128) default NULL,
+  PRIMARY KEY  (NamespaceRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `namespaces`
+--
+
+INSERT INTO namespaces VALUES (1,1,NULL,NULL,'1','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (2,2,NULL,NULL,'2','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (3,3,NULL,NULL,'3','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (4,4,NULL,NULL,'4','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (5,5,NULL,NULL,'5','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (6,6,NULL,NULL,'6','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (7,7,NULL,NULL,'7','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (8,8,NULL,NULL,'8','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (9,9,NULL,NULL,'9','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (10,10,NULL,NULL,'10','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (11,11,NULL,NULL,'11','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (12,12,NULL,NULL,'12','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (13,13,NULL,NULL,'13','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (14,14,NULL,NULL,'14','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (15,15,NULL,NULL,'15','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (16,16,NULL,NULL,'16','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (17,17,NULL,NULL,'17','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (18,18,NULL,NULL,'18','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (19,19,NULL,NULL,'19','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (20,20,NULL,NULL,'20','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (21,21,NULL,NULL,'21','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (22,22,NULL,NULL,'22','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (23,23,NULL,NULL,'23','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (24,24,NULL,NULL,'24','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (25,25,NULL,NULL,'25','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (26,26,NULL,NULL,'26','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (27,27,NULL,NULL,'27','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (28,28,NULL,NULL,'28','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (29,29,NULL,NULL,'29','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (30,30,NULL,NULL,'30','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (31,31,NULL,NULL,'31','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (32,32,NULL,NULL,'32','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (33,33,NULL,NULL,'33','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (34,34,NULL,NULL,'34','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (35,35,NULL,NULL,'35','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (36,36,NULL,NULL,'36','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (37,37,NULL,NULL,'37','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (38,38,NULL,NULL,'38','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (39,39,NULL,NULL,'39','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (40,40,NULL,NULL,'40','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (41,41,NULL,NULL,'41','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (42,42,NULL,NULL,'42','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (43,43,NULL,NULL,'43','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (44,44,NULL,NULL,'44','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (45,45,NULL,NULL,'45','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (46,46,NULL,NULL,'46','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (47,47,NULL,NULL,'47','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (48,48,NULL,NULL,'48','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (49,49,NULL,NULL,'49','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (50,50,NULL,NULL,'50','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (51,51,NULL,NULL,'51','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (52,52,NULL,NULL,'52','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (53,53,NULL,NULL,'53','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (54,54,NULL,NULL,'54','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (55,55,NULL,NULL,'55','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (56,56,NULL,NULL,'56','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (57,57,NULL,NULL,'57','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (58,58,NULL,NULL,'58','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (59,59,NULL,NULL,'59','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (60,60,NULL,NULL,'60','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (61,61,NULL,NULL,'61','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (62,62,NULL,NULL,'62','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (63,63,NULL,NULL,'63','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (64,64,NULL,NULL,'64','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (65,65,NULL,NULL,'65','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (66,66,NULL,NULL,'66','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (67,67,NULL,NULL,'67','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (68,68,NULL,NULL,'68','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (69,69,NULL,NULL,'69','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (70,70,NULL,NULL,'70','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (71,71,NULL,NULL,'71','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (72,72,NULL,NULL,'72','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (73,73,NULL,NULL,'73','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (74,74,NULL,NULL,'74','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (75,75,NULL,NULL,'75','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (76,76,NULL,NULL,'76','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (77,77,NULL,NULL,'77','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (78,78,NULL,NULL,'78','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (79,79,NULL,NULL,'79','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (80,80,NULL,NULL,'80','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (81,81,NULL,NULL,'81','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (82,82,NULL,NULL,'82','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (83,83,NULL,NULL,'83','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (84,84,NULL,NULL,'84','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (85,85,NULL,NULL,'85','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (86,86,NULL,NULL,'86','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (87,87,NULL,NULL,'87','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (88,88,NULL,NULL,'88','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (89,89,NULL,NULL,'89','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (90,90,NULL,NULL,'90','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (91,91,NULL,NULL,'91','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (92,92,NULL,NULL,'92','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (93,93,NULL,NULL,'93','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (94,94,NULL,NULL,'94','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (95,95,NULL,NULL,'95','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (96,96,NULL,NULL,'96','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (97,97,NULL,NULL,'97','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (98,98,NULL,NULL,'98','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (99,99,NULL,NULL,'99','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (100,100,NULL,NULL,'100','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (101,101,NULL,NULL,'101','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (102,102,NULL,NULL,'102','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (103,103,NULL,NULL,'103','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (104,104,NULL,NULL,'104','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (105,105,NULL,NULL,'105','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (106,106,NULL,NULL,'106','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (107,107,NULL,NULL,'107','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (108,108,NULL,NULL,'108','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (109,109,NULL,NULL,'109','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (110,110,NULL,NULL,'110','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (111,111,NULL,NULL,'111','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (112,112,NULL,NULL,'112','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (113,113,NULL,NULL,'113','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (114,114,NULL,NULL,'114','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (115,115,NULL,NULL,'115','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (116,116,NULL,NULL,'116','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (117,117,NULL,NULL,'117','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (118,118,NULL,NULL,'118','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (119,119,NULL,NULL,'119','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (120,120,NULL,NULL,'120','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (121,121,NULL,NULL,'121','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (122,122,NULL,NULL,'122','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (123,123,NULL,NULL,'123','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (124,124,NULL,NULL,'124','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (125,125,NULL,NULL,'125','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (126,126,NULL,NULL,'126','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (127,127,NULL,NULL,'127','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (128,128,NULL,NULL,'128','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (129,129,NULL,NULL,'129','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (130,130,NULL,NULL,'130','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (131,131,NULL,NULL,'131','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (132,132,NULL,NULL,'132','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (133,133,NULL,NULL,'133','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (134,134,NULL,NULL,'134','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (135,135,NULL,NULL,'135','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (136,136,NULL,NULL,'136','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (137,137,NULL,NULL,'137','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (138,138,NULL,NULL,'138','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (139,139,NULL,NULL,'139','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (140,140,NULL,NULL,'140','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (141,141,NULL,NULL,'141','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (142,142,NULL,NULL,'142','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (143,143,NULL,NULL,'143','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (144,144,NULL,NULL,'144','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (145,145,NULL,NULL,'145','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (146,146,NULL,NULL,'146','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (147,147,NULL,NULL,'147','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (148,148,NULL,NULL,'148','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (149,149,NULL,NULL,'149','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (150,150,NULL,NULL,'150','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (151,151,NULL,NULL,'151','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (152,152,NULL,NULL,'152','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (153,153,NULL,NULL,'153','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (154,154,NULL,NULL,'154','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (155,155,NULL,NULL,'155','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (156,156,NULL,NULL,'156','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (157,157,NULL,NULL,'157','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (158,158,NULL,NULL,'158','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (159,159,NULL,NULL,'159','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (160,160,NULL,NULL,'160','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (161,161,NULL,NULL,'161','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (162,162,NULL,NULL,'162','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (163,163,NULL,NULL,'163','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (164,164,NULL,NULL,'164','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (165,165,NULL,NULL,'165','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (166,166,NULL,NULL,'166','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (167,167,NULL,NULL,'167','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (168,168,NULL,NULL,'168','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (169,169,NULL,NULL,'169','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (170,170,NULL,NULL,'170','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (171,171,NULL,NULL,'171','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (172,172,NULL,NULL,'172','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (173,173,NULL,NULL,'173','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (174,174,NULL,NULL,'174','gsdl3',NULL,NULL,NULL);
+INSERT INTO namespaces VALUES (175,175,NULL,NULL,'175','gsdl3',NULL,NULL,NULL);
+
+--
+-- Table structure for table `structure`
+--
+
+CREATE TABLE structure (
+  StructureRef int(11) NOT NULL auto_increment,
+  DocID varchar(64) default NULL,
+  StructureID varchar(64) default NULL,
+  StructureType varchar(64) default NULL,
+  Label varchar(128) default NULL,
+  PRIMARY KEY  (StructureRef)
+) TYPE=MyISAM;
+
+--
+-- Dumping data for table `structure`
+--
+
+INSERT INTO structure VALUES (1,'20050201:1','Section','Section','Section');
+INSERT INTO structure VALUES (2,'20050201:1','All','Whole Document','All');
+INSERT INTO structure VALUES (3,'20050201:2','Section','Section','Section');
+INSERT INTO structure VALUES (4,'20050201:2','All','Whole Document','All');
+INSERT INTO structure VALUES (5,'20050201:3','Section','Section','Section');
+INSERT INTO structure VALUES (6,'20050201:3','All','Whole Document','All');
+INSERT INTO structure VALUES (7,'20050201:4','Section','Section','Section');
+INSERT INTO structure VALUES (8,'20050201:4','All','Whole Document','All');
+INSERT INTO structure VALUES (9,'20050201:5','Section','Section','Section');
+INSERT INTO structure VALUES (10,'20050201:5','All','Whole Document','All');
+INSERT INTO structure VALUES (11,'20050201:6','All','Whole Document','All');
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/README
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/README	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/README	(revision 21112)
@@ -0,0 +1,105 @@
+This collection talks to the iVia installation at INFOMINE, Riverside, 
+California. The software should work with any iVia installation. 
+
+Cgi scripts are called to do searching of and retrieval from the database. 
+All three are fairly generic CGI scripts; iVia's "theming" ability is used to
+ get the data back in the format we want-- there is a gsdl3 theme. 
+
+To create a new collection that talks to an iVia database, you need to create 
+the two configuration files, collectionConfig.xml, and buildConfig.xml. 
+The collectionConfig.xml needs to contain any display information for the 
+collection, and any metadata that is available. Also it can contain format 
+information for document display or search result lists if the default is 
+not useful.
+
+The buildConfig.xml file needs to specify the IViaProxy serviceRack 
+class. Inside this element should be the url of the iVia installation that
+you want to connect to. Here is an example:
+
+<buildConfig> 
+  <metadataList/>
+  <serviceRackList>                                                            
+    <serviceRack name="IViaProxy">
+      <iViaServer url="http://infominehelper.ucr.edu"/>
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
+
+
+The following describes the iVia cgi scripts.
+
+For performing a search:
+************************
+
+  <serverhome>/cgi-bin/canned_search
+
+Parameters:
+  Required parameters: theme=gsdl3&query=xxx
+  Useful parameters: no_of_records_per_page=10&start_page_no=1&fields='kw,su'
+  Other parameters: http://infomine.ucr.edu/help/adders/index.php?section=11
+  Example: http://infomine.ucr.edu/cgi-bin/canned_search?query=tree&
+              theme=gsdl3&no_of_records_per_page=10&start_page_no=1
+
+Fields include Title (ti), Subject (su), Keyword (kw), Author (au), 
+Description (de) and full text (fu). 'all' will search all fields.
+It's possible to do uch more advanced searches including phrase searching 
+and bolean queries.
+
+The canned_search result page will look something like this:
+  Resources: 58
+  Ids:  1015 22128 1730 6020 2981 1014 20147 2097 46569 4360
+
+For getting an HTML representation of a record:
+***********************************************
+
+  <serverhome>/cgi-bin/view_record
+
+Parameters:
+  Required parameters: theme=gsdl3&record_id=x
+  Example: http://infomine.ucr.edu/cgi-bin/view_record?theme=gsdl3&
+                record_id=2097
+
+The result page will be an HTML fragment; specifically, it will be an HTML
+table.  Again, the formatting is quite flexible.  There's one current
+problem: a lot of the data is displayed as hypertext links to new infomine
+searches; This should be changed eventually.
+
+For now, the Greenstone services process the records, replacing infomine 
+links with Greenstone links where possible. 
+
+For getting metadata for a set of records:
+******************************************
+
+  <serverhome>/cgi-bin/view_record_set
+
+Parameters:
+  Required parameters: theme=gsdl3&record_id_list=id1,id2,id3
+  Optional parameter: field_list=f1,f2,f3
+  Example: http://infomine.ucr.edu/cgi-bin/view_record_set?theme=gsdl3&
+                record_id_list=1015,2128,1730&field_list=url,title
+
+The result will look like this:
+
+  Record: 1015
+  url: http://www.fruit.affrc.go.jp/index-e.html
+  title: National Institute of Fruit Tree Science (Japan)
+
+  Record: 2128
+  url: http://www.publish.csiro.au/journals/ajb/
+  title: Australian Journal of Botany
+
+  Record: 1730
+  url: http://usda.mannlib.cornell.edu/reports/erssor/specialty/fts-bb/
+  title: Fruit and Tree Nuts : USDA Economics and Statistics System
+
+There are many different metadata fields that can be requested, including
+keywords, subjects, ivia_description, access, expert_created, foreign_source,
+foreign_id, and so on.  
+
+Note that the metadata always fits on one line (whitespace is explicitly
+normalised to space) to make parsing easier.  Generally, the metadata is
+in ISO-8859-15 format (i.e. ISO-8859-1 plus the Euro currency symbol). 
+Metadata is usually plain text, though the description field may contain
+HTML entities and limited set of HTML elements (bold tags, italic tags,
+links, and a few others).
+
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/etc/collectionConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/etc/collectionConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/etc/collectionConfig.xml	(revision 21112)
@@ -0,0 +1,39 @@
+<collectionConfig xmlns:gsf="http://www.greenstone.org/greenstone3/schema/ConfigFormat" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
+  <metadataList/>
+  <displayItemList>
+    <displayItem name='name' lang=''>Infomine proxy demo</displayItem>
+    <displayItem name='description' lang='en'>&lt;p&gt;This is a demonstration collection that acts as a proxy to the iVia database at &lt;a href="http://infomine.ucr.edu/"&gt;INFOMINE&lt;/a&gt;, Riverside, California. No data is stored locally; instead, the search and retrieval services invoke cgi scripts at infomine to query the database, and retrieve records and metadata.&lt;/p&gt;
+&lt;p&gt;A &lt;a
+    href="sites/localsite/collect/infomine/README"&gt;README&lt;/a&gt; file
+    describes the cgi scripts and how to invoke them. The configuration
+    files for the collection are very basic, and merely specify display 
+    information for the collection, and specify the IViaProxy service
+    rack. The collection configuration can be seen &lt;a
+    href="sites/localsite/collect/infomine/etc/collectionConfig.xml"&gt;here&lt;/a&gt;,
+    and the building configuration file can be seen &lt;a
+    href="sites/localsite/collect/infomine/index/buildConfig.xml"&gt;here&lt;/a&gt;.
+       You may need to "view page source" to see the raw XML. &lt;/p&gt; 
+</displayItem>    
+    <displayItem name='icon' lang='en'>infomine.gif</displayItem>  
+    <displayItem name='smallicon' lang='en'>infomine.gif</displayItem>  
+  </displayItemList>  
+  <search>
+    <format>
+      <gsf:template match="documentNode">
+	<td valign='top'><gsf:link><gsf:icon/></gsf:link></td><td>
+	  <table><tr><td><a><xsl:attribute name='href'><gsf:metadata name="url"/></xsl:attribute><b><gsf:metadata name="title"/></b></a></td></tr>
+	    <tr><td><gsf:metadata name="ivia_description"/></td></tr>
+	    <tr><td><b><gsf:metadata name="url"/></b></td></tr>
+	    <tr><td>[<gsf:link>View Full Record</gsf:link>]</td></tr></table>
+	</td>
+      </gsf:template>
+    </format>
+  </search>
+  <display>
+  <format>
+  <gsf:template name="documentHeading">
+  <gsf:metadata name="title"/>
+</gsf:template>	
+</format>
+</display>
+</collectionConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/infomine/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,11 @@
+<buildConfig> 
+  <metadataList/>
+  <serviceRackList>                                                            
+    <serviceRack name="IViaSearch">
+    <iViaServer url="http://infominehelper.ucr.edu"/>
+    </serviceRack>
+    <serviceRack name="IViaRetrieve">
+    <iViaServer url="http://infominehelper.ucr.edu"/>
+    </serviceRack>
+  </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/collect/xslt/index/buildConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/collect/xslt/index/buildConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/collect/xslt/index/buildConfig.xml	(revision 21112)
@@ -0,0 +1,10 @@
+<buildConfig>
+    <metadataList>
+      <metadata name="numDocs">Infinite</metadata>
+      <metadata name="colName">Infinity</metadata>
+      <metadata name="description">An infinite collection of generated documents. </metadata>
+    </metadataList>
+    <serviceRackList>
+      <serviceRack name="XSLTServices"/>
+    </serviceRackList>
+</buildConfig>
Index: /main/tags/3.01/gsdl3/web/sites/localsite/siteConfig.xml
===================================================================
--- /main/tags/3.01/gsdl3/web/sites/localsite/siteConfig.xml	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/sites/localsite/siteConfig.xml	(revision 21112)
@@ -0,0 +1,8 @@
+<siteConfig>
+  <siteList/>
+  <serviceRackList>
+    <!-- uncomment to provide the GATE tagging service -->
+    <!--<serviceRack name="GATEServices"/>-->
+  </serviceRackList>
+  <serviceClusterList/>
+</siteConfig>
Index: /main/tags/3.01/gsdl3/web/xmlextras.js
===================================================================
--- /main/tags/3.01/gsdl3/web/xmlextras.js	(revision 21112)
+++ /main/tags/3.01/gsdl3/web/xmlextras.js	(revision 21112)
@@ -0,0 +1,149 @@
+//<script>
+//////////////////
+// Helper Stuff //
+//////////////////
+
+// used to find the Automation server name
+function getDomDocumentPrefix() {
+	if (getDomDocumentPrefix.prefix)
+		return getDomDocumentPrefix.prefix;
+	
+	var prefixes = ["MSXML2", "Microsoft", "MSXML", "MSXML3"];
+	var o;
+	for (var i = 0; i < prefixes.length; i++) {
+		try {
+			// try to create the objects
+			o = new ActiveXObject(prefixes[i] + ".DomDocument");
+			return getDomDocumentPrefix.prefix = prefixes[i];
+		}
+		catch (ex) {};
+	}
+	
+	throw new Error("Could not find an installed XML parser");
+}
+
+function getXmlHttpPrefix() {
+	if (getXmlHttpPrefix.prefix)
+		return getXmlHttpPrefix.prefix;
+	
+	var prefixes = ["MSXML2", "Microsoft", "MSXML", "MSXML3"];
+	var o;
+	for (var i = 0; i < prefixes.length; i++) {
+		try {
+			// try to create the objects
+			o = new ActiveXObject(prefixes[i] + ".XmlHttp");
+			return getXmlHttpPrefix.prefix = prefixes[i];
+		}
+		catch (ex) {};
+	}
+	
+	throw new Error("Could not find an installed XML parser");
+}
+
+//////////////////////////
+// Start the Real stuff //
+//////////////////////////
+
+
+// XmlHttp factory
+function XmlHttp() {}
+
+XmlHttp.create = function () {
+	try {
+		if (window.XMLHttpRequest) {
+			var req = new XMLHttpRequest();
+			
+			// some versions of Moz do not support the readyState property
+			// and the onreadystate event so we patch it!
+			if (req.readyState == null) {
+				req.readyState = 1;
+				req.addEventListener("load", function () {
+					req.readyState = 4;
+					if (typeof req.onreadystatechange == "function")
+						req.onreadystatechange();
+				}, false);
+			}
+			
+			return req;
+		}
+		if (window.ActiveXObject) {
+			return new ActiveXObject(getXmlHttpPrefix() + ".XmlHttp");
+		}
+	}
+	catch (ex) {}
+	// fell through
+	throw new Error("Your browser does not support XmlHttp objects");
+};
+
+// XmlDocument factory
+function XmlDocument() {}
+
+XmlDocument.create = function () {
+	try {
+		// DOM2
+		if (document.implementation && document.implementation.createDocument) {
+			var doc = document.implementation.createDocument("", "", null);
+			
+			// some versions of Moz do not support the readyState property
+			// and the onreadystate event so we patch it!
+			if (doc.readyState == null) {
+				doc.readyState = 1;
+				doc.addEventListener("load", function () {
+					doc.readyState = 4;
+					if (typeof doc.onreadystatechange == "function")
+						doc.onreadystatechange();
+				}, false);
+			}
+			
+			return doc;
+		}
+		if (window.ActiveXObject)
+			return new ActiveXObject(getDomDocumentPrefix() + ".DomDocument");
+	}
+	catch (ex) {}
+	throw new Error("Your browser does not support XmlDocument objects");
+};
+
+// Create the loadXML method and xml getter for Mozilla
+if (window.DOMParser &&
+	window.XMLSerializer &&
+	window.Node && Node.prototype && Node.prototype.__defineGetter__) {
+
+	// XMLDocument did not extend the Document interface in some versions
+	// of Mozilla. Extend both!
+	//XMLDocument.prototype.loadXML = 
+	Document.prototype.loadXML = function (s) {
+		
+		// parse the string to a new doc	
+		var doc2 = (new DOMParser()).parseFromString(s, "text/xml");
+		
+		// remove all initial children
+		while (this.hasChildNodes())
+			this.removeChild(this.lastChild);
+			
+		// insert and import nodes
+		for (var i = 0; i < doc2.childNodes.length; i++) {
+			this.appendChild(this.importNode(doc2.childNodes[i], true));
+		}
+	};
+	
+	
+	/*
+	 * xml getter
+	 *
+	 * This serializes the DOM tree to an XML String
+	 *
+	 * Usage: var sXml = oNode.xml
+	 *
+	 */
+	// XMLDocument did not extend the Document interface in some versions
+	// of Mozilla. Extend both!
+	/*
+	XMLDocument.prototype.__defineGetter__("xml", function () {
+		return (new XMLSerializer()).serializeToString(this);
+	});
+	*/
+	Document.prototype.__defineGetter__("xml", function () {
+		return (new XMLSerializer()).serializeToString(this);
+	});
+}
Index: /main/tags/3.01/indexers/mg/COPYING
===================================================================
--- /main/tags/3.01/indexers/mg/COPYING	(revision 21112)
+++ /main/tags/3.01/indexers/mg/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/indexers/mg/INSTALL
===================================================================
--- /main/tags/3.01/indexers/mg/INSTALL	(revision 21112)
+++ /main/tags/3.01/indexers/mg/INSTALL	(revision 21112)
@@ -0,0 +1,176 @@
+Basic Installation
+==================
+
+   These are generic installation instructions.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, a file
+`config.cache' that saves the results of its tests to speed up
+reconfiguring, and a file `config.log' containing compiler output
+(useful mainly for debugging `configure').
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If at some point `config.cache'
+contains results you don't want to keep, you may remove or edit it.
+
+   The file `configure.in' is used to create `configure' by a program
+called `autoconf'.  You only need `configure.in' if you want to change
+it or regenerate `configure' using a newer version of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.  If you're
+     using `csh' on an old version of System V, you might need to type
+     `sh ./configure' instead to prevent `csh' from trying to execute
+     `configure' itself.
+
+     Running `configure' takes awhile.  While running, it prints some
+     messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+   Some systems require unusual options for compilation or linking that
+the `configure' script does not know about.  You can give `configure'
+initial values for variables by setting them in the environment.  Using
+a Bourne-compatible shell, you can do that on the command line like
+this:
+     CC=c89 CFLAGS=-O2 LIBS=-lposix ./configure
+
+Or on systems that have the `env' program, you can do it like this:
+     env CPPFLAGS=-I/usr/local/include LDFLAGS=-s ./configure
+
+Compiling For Multiple Architectures
+====================================
+
+   You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you must use a version of `make' that
+supports the `VPATH' variable, such as GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   If you have to use a `make' that does not supports the `VPATH'
+variable, you have to compile the package for one architecture at a time
+in the source code directory.  After you have installed the package for
+one architecture, use `make distclean' before reconfiguring for another
+architecture.
+
+Installation Names
+==================
+
+   By default, `make install' will install the package's files in
+`/usr/local/bin', `/usr/local/man', etc.  You can specify an
+installation prefix other than `/usr/local' by giving `configure' the
+option `--prefix=PATH'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+give `configure' the option `--exec-prefix=PATH', the package will use
+PATH as the prefix for installing programs and libraries.
+Documentation and other data files will still use the regular prefix.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+   Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+   There may be some features `configure' can not figure out
+automatically, but needs to determine by the type of host the package
+will run on.  Usually `configure' can figure that out, but if it prints
+a message saying it can not guess the host type, give it the
+`--host=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name with three fields:
+     CPU-COMPANY-SYSTEM
+
+See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the host type.
+
+   If you are building compiler tools for cross-compiling, you can also
+use the `--target=TYPE' option to select the type of system they will
+produce code for and the `--build=TYPE' option to select the type of
+system on which you are compiling the package.
+
+Sharing Defaults
+================
+
+   If you want to set default values for `configure' scripts to share,
+you can create a site shell script called `config.site' that gives
+default values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Operation Controls
+==================
+
+   `configure' recognizes the following options to control how it
+operates.
+
+`--cache-file=FILE'
+     Use and save the results of the tests in FILE instead of
+     `./config.cache'.  Set FILE to `/dev/null' to disable caching, for
+     debugging `configure'.
+
+`--help'
+     Print a summary of the options to `configure', and exit.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`--version'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`configure' also accepts some other, not widely useful, options.
+
Index: /main/tags/3.01/indexers/mg/INSTALL.mg
===================================================================
--- /main/tags/3.01/indexers/mg/INSTALL.mg	(revision 21112)
+++ /main/tags/3.01/indexers/mg/INSTALL.mg	(revision 21112)
@@ -0,0 +1,38 @@
+                     BUILDING AND INSTALLATION OF MG
+                     ===============================
+ 
+This file explains how to build the mg system.
+ 
+(1) Follow the general GNU install instructions in the INSTALL file.
+    NOTE: this procedure has changed from version mg-1.1 which did
+    not use a GNU configure script.
+    e.g.
+      cd ~/mg-1.2
+      ./configure --prefix=`pwd`
+      make
+      make check
+      make install
+    Report portability problems to tes@kbs.citri.edu.au .
+ 
+(2) You will need to set the environment variable MGDATA which 
+    indicates where the mg data files will be created. You will also
+    need to create the directory you specify.
+ 
+    e.g.
+        setenv MGDATA ~/mgdata
+	mkdir ~/mgdata
+
+(4) You may also need to set the environment variable MGSAMPLE which
+    is used by mg_get and indicates where the sample data included
+    with this package is located.
+    If this is not specified it defaults to ``./SampleData''
+
+(4) Included with mg is an X11 interface to mgquery called xmg. 
+    Xmg is a ``wish'' script which uses the Tcl/Tk packages available
+    by anonymous ftp from allspice.berkeley.edu [128.32.150.27].
+ 
+(5) Another environment variable that you may wish to set is
+    MGIMAGEVIEWER, this variable sets the image viewer to be used
+    to display images. The image viewer must take the image from stdin.
+    If this is not specified it defaults to ``xv -''.
+ 
Index: /main/tags/3.01/indexers/mg/MODIFICATIONS
===================================================================
--- /main/tags/3.01/indexers/mg/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/indexers/mg/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,1023 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Similarity variants
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  Can only use one type of similarity measure - the
+  standard cosine measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+  The standard measure can be broken up into 7 components.
+  The 7 components are  
+  Each one of these components has a number of alternatives.
+  The overall measure, S_qd, can also be altered.
+  Thus the particular similarity measure used can be specified
+  by an 8 dimensional vector.
+  What is desired is to be able to specify to mgquery an option
+  and a 8-digit string representing this vector (assuming that
+  any one alternative can have at most 9 (not using zero) variants).
+SOLUTION
+  The programs which had to be modified were:
+  (i) mgquery,
+  (ii) mg_weights_build.
+  The other mg programs in existence store the text, indexing info,
+  and the basic statistics such as N, n, ft, fdt.  
+  Other programs which had to be created were:
+  (i) mg_fmd_build,
+  (ii) mg_wt_build.
+  Mg_fmd_build will create the file to store the f_md statistic, 
+  where f_md is the largest (maximum) f_dt of any term in document, d.
+  Mg_wt_build will create the file to store the w_t primitive.
+  It only creates this for the w_t variants 6-9 which would require
+  extra passes of invf at query time if they were not stored here.
+  For details on similarity changes for mgquery and mg_weights_build,
+  please see the other modification entries.  
+FILES
+ *   mg_fmd_build.c
+ *   mg_wt_build.c
+ *   build_lib.c, build_lib.h
+*************************************************************
+TITLE
+  Similarity variants for mgquery
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mgquery" needs to be altered to allow modification of
+  the similarity measure. 
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+
+  Most of the similarity measures, Sqd, are of the 
+  form:   Aqd
+         -----
+          Bqd
+  where Bqd is an expression involving Wd and possibly Wq,
+  where Aqd is a sum over the common document/query terms
+  of w_qt and w_dt.
+  Building of Aqd
+  ===============
+  The calculation of Aqd is done in the file build_Aqd.c .
+  The functions for doing this used to be in the file invf_get.c .
+  Build_Aqd.c contains 4 different functions for building Aqd, each
+  of them building a different data structure:
+    (i) Array, (ii) Splay Tree, (iii) Hash Table, (iv) List Table.
+  Each of these routines seems to have been construction by modifications
+  to duplicated code. This is often the easiest way to construct variants
+  but is quite difficult to maintain consistency. 
+  As the aim of the exercise was to try out different sim. measures for
+  retrieval effectiveness, I only modified the code that constructed
+  an array. This routine was called "CosineDecode"; I changed it to
+  "build_Aqd_Array." This change reflects the fact that we are only 
+  calculating Aqd and this need not be used for the Cosine measure.
+  The other routines: "CosineDecodeSplay," "CosineDecodeHash," and
+  "CosineDecodeList" have been left unaltered - they need to be updated
+  in the future which would be best be done by abstracting out common code.
+  By the stage of building Aqd, the query terms have been looked up in
+  the inverted file dictionary and put into a list.
+  This list of common terms is traversed to lookup the corresponding
+  invf entries. Before the invf entry is processed, all query and term
+  relevant statistics and primitive quantities are calculated.
+  For example, fqt, ft, wt, rqt, wqt, Wq-partial-sum.
+  To save unnecessary calculations, there is a test for each value
+  to see whether it is needed e.g. "if (sim->variant_needs & NEEDS_wt) ... ".  
+  Aside: Variant Needs
+  --------------------
+  The idea behind the "variant_needs" field is to be able to have all
+  the code in the one place for each possible variant and this code would
+  get the information at the correct time/place only when it is needed.
+  The overhead is a "bit-and" and "test" for each component.
+  The important concern though, is that the "variant_needs" must be
+  accurate i.e. it should be carefully maintained.
+  Each possible need is stored as a bit position in a constant/macro
+  of the form "NEEDS_component" e.g. NEEDS_wt, NEEDS_nt 
+  More recently (Jan/95), I have found that the _need_ing of a component
+  may be relevant to a particular purpose, that is, it may be needed in
+  one situation and not another. This was the case for Wd. 
+  For rdt#6, Wd was needed, however, it might not be needed for Bqd, the
+  denominator of Sqd. So I changed from NEEDS_Wd to NEEDS_Wd_for_Sqd and
+  NEEDS_Wd_for_rdt. 
+  --------------------
+  To handle the different calculations of the variants, I wrote macros
+  based around a switch statement. All these macros are stored in the file
+  "similarity.h" . The point of doing this is to centralise most of the changes
+  and cut down on the number of files which have to be altered if a new
+  way of calculating a primitive is to be added.  
+  This is achieved by having a data record called, "Similarity_variant" whose
+  fields includes all the statistics and the similarity primitives.
+  So the standard procedure is to see if something is needed and if so, then
+  extract it from an mg file or calculate it using a macro - most of the
+  input and output to the macro is done via the "Similarity_variant" structure.
+  As well as calculating Aqd, it is also a convenient place to calculate Wq.
+  Previously, for Cosine measure, Wq was not needed because each Aqd was directly
+  divided by Wq and thus would not change the ordering. However, there are
+  some Sqd's which divide by a sum involving Wq and thus need Wq. 
+  Calculating Sqd and Ordering Documents
+  ====================================== 
+  The file query.ranked.c contains the code for calculating Sqd using the
+  approximate Wd in order to do the ranking of the documents.
+  The mg-1 file was cleaned up and modified slightly.
+  All the heap data structures and routines were taken out and placed in 
+  their own file, heap_weights.c/.h .
+  The major components/steps in the ranking process were abstracted into
+  macros and functions:
+    calc_MaxParas
+    insert_heap_entry
+    insert_greater_heap_entry
+    approx_guided_insert
+    fill_initial_heap
+    add_heap_remainders
+    change_heap_to_exact
+    add_remainder_exact_weights
+    Make_Exact_Root
+    build_doc_list
+  Aside: Zeroing of an Aqd Element
+  -------------------------
+  One interesting change concerned the use of zeroing out an Aqd element
+  so as to mark it as being used in the heap.
+  In the heap, the approx. Sqd is stored based on Aqd/Wd-approx.
+  Later when Aqd is required, it can be extracted from the approx. Sqd
+  by muliplication of Wd-approx i.e. Aqd = Sqd-approx * Wd-approx.
+  This, however, is not always possible for the various Sqd variants.
+  So, instead of zeroing out Aqd, I decided just to make it negative.
+  -------------------------
+  In some Sqd cases, Wd and Wd-approx. are not required. In which case,
+  step 3 which calls on "change_heap_to_exact", "Heap_Build" and
+  "add_remainder_exact_weights" is not required.
+ 
+FILES
+ *    build_Aqd.c
+ *    query.ranked.c
+ *    similarity.c/.h (in libmg)
+ *    heap_weights.c/.h
+ *    backend.c/.h
+*************************************************************
+TITLE
+  Similarity variants for mg_weights_build
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mg_weights_build" needs to be altered to allow modification of
+  the similarity measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+  The weight files which are generated for a particular similarity
+  measure have their names extended by a suffix.
+  In the case of Wd#1 no weights are generated.
+  In the case of the standard cosine weights, Wd#2, a 3 letter suffix
+  is used to represent .
+  In the case of the other Wd variants, a one letter suffix is used
+  to represent which Wd variant it is.
+  In each case, the variant input (e.g. -q 22222222) should be the whole
+  similarity variant string and the relevant fields will be extracted out.
+  This is done for consistency in code and interface. 
+  The code is fairly similar to the original.
+  A dependency check has been added so that the dates of files and the
+  type of needed files is verified before building.
+  The dependencies include, invf dictionary, invf index, invf, fmd and wt files.
+  The major change here, is the possible use of fmd and/or wt files.
+  Later when I was having to write mg_fmd_build and mg_wt_build,
+  I decided to abstract out some macros, namely:
+    Get_ft, Get_ft_Ft, loop_invf_entry,
+  which were put into "build_lib.c" .
+  "loop_invf_entry" takes a function/macro name as a parameter and applies
+  it to the sim record (with field fdt set), current doc number and modifies
+  the return value. 
+FILES
+ *   mg_weights_build.c
+ *   build_lib.c
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  Boolean tree optimisations
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  (i) tes@kbs.citri.edu.au - 28/Sep/94
+  (ii) tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  (i) tes@kbs.citri.edu.au - 18/Oct/94
+  (ii) tes@kbs.citri.edu.au - 21/Dec/94
+CLAIM
+  The initial prompt for investigating the optimisation of
+  boolean queries is noted in the mg-1 mod14.txt. 
+  The code for optimising seemed to have a number of faults.
+PROBLEM
+  Boolean optimisation was unreliable.
+SOLUTION
+  Initially (in case (i) above, see mg-1/mod14.txt), I rewrote 
+  all the boolean tree and optimising code. I converted the boolean
+  expression into DNF. I did this after reading some notes about 
+  the steps involved in optimisation and they suggested standardising
+  in some normal form. I thought that DNF would be appropriate so that
+  all the terms are converted to be part of "and" expressions and be
+  evaluated quickly using skipping.
+  This, however, can suffer quite badly if the distributive law is
+  applied to often and the query expands in size. If there was
+  some sort of cacheing of invf entries, then it might not be so
+  bad otherwise there is quite an overhead on reading the same
+  invf entry more than once.
+
+  As it happens, CNF queries are reasonably common, where the user
+  queries with a conjunction of disjunctions of similes:
+    e.g. (car | automobile | vehicle) & (fast | quick | speedy)
+  This sort of CNF query expands a hell of a lot !
+  So after speaking with Justin who wanted to benchmark Atlas with Mg on
+  these sort of queries, I looked up the MG book for other ideas.
+
+  The method that I implemented was the following:
+
+  -----------------------------------------------------------------
+  Steps of tree modifications:
+    Gets literals by pushing the nots down, detecting T/F at leaves
+    and collapsing the tree by detecting  'and' of 'and's and 'or' of
+    'or's.
+    Next it looks at the or nodes and if all the children are terms
+    then it marks the or-node as such.
+    Finally, the or-term-nodes are sorted by using the sum of their
+    ft's for comparison.
+  Steps at query evaluation:
+    If it comes across an 'and' of 'or-terms' then the evaluation is
+    done noting the distributive law.
+    I.e. a & (b | c | d) = (a & b) | (a & c) | (a & d)
+    Assuming 'a' is the c-set of documents.
+    All of 'a' is tested against 'b' and matching ones are marked.
+    Next, all the unmarked members of 'a' are tested against 'c'.
+    Likewise for 'd'.
+    Now all the marked members of our c-set are kept.
+    When we do the testing, we can use the skipping in the invf entries.
+  -----------------------------------------------------------------
+
+
+  After doing this, I added the choice of which type of optimisation
+  the user wanted by adding query-environment-variable, "optimise_type".
+  Type 0 = no parse tree modification. 
+  Type 1 = Or-term recognition and CNF query evaluation optimisation. 
+  Type 2 = Put into DNF form. [generally not recommended]
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+ *   query_env.c
+ *   invf_get.c [GetDocsOp]
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  TREC DocNo file   
+APPLICATION
+  mg-2
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 26/May/95
+FIX
+  tes@kbs.citri.edu.au - 26/May/95
+CLAIM
+  MG has problems dealing with trec docnos for trec disk 3.
+PROBLEM
+  Trec DocNos file didn't have a wide enough field
+  to handle disk 3. 
+SOLUTION
+  Allow different width fields for file.
+  It is still fixed width but a number in the header
+  says how wide the field is.
+FILES
+ *   passes/mg.special.c
+ *   query/mgquery.c
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+   Autoconfiguring mg-1  
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  many people - 94/95
+FIX
+  tes@kbs.citri.edu.au - Aug/95
+CLAIM
+  Portability is limited by setting up c-macros just for particular
+  machines and operating systems.
+  People had to make changes for HP, Next, Linux, Dec Alpha, ...
+PROBLEM
+  Porting was only targetting at the machines that the author had
+  access to.
+SOLUTION
+  Use GNU's autoconfigure program.
+  This allows checking of the systems features/characteristics.
+  It also allows some checking for specific machines/OS - although
+  I have not utilised this option.
+  I used GNU's tar-1.11.8 as an example to base my changes on.
+  I also used autoscan to generate the initial "configure.in".
+  The "Makefile.in"s were done very similarly to GNU tar's.
+  The "config.h" and "sysfuncs.h" files were scrapped and 
+  rewritten. The new "config.h" file is generated by the configure
+  script - it contains all the #define's for the system features.
+  The "sysfuncs.h" file wraps up a number of system headers. 
+  For example, some systems use , while some use ;
+  which one is included is decided in "sysfuncs.h".
+  I have also used GNU tar's use of ansi2knr in its Makefiles.
+  This should hopefully allow the package to work on a system with
+  only a K&R C compiler.
+  However, there are probably problems with what I have done.
+  I am concerned about <stdarg.h> for example.
+  I also noticed that "ansi2knr" require function definitions as
+  the GNU coding style recommends ie. with function name the first
+  string on the line. This prompted me to run all the package's code
+  thru GNU's indent.
+  Setting up the configure changes is difficult. It really seems
+  necessary to try the package out on many target machines so one
+  can know what is necessary.
+  A simple check target for the main Makefile has been written.
+  It is used to see if the installation is working - it does
+  not test much of the functionality of mg.
+  It does cmp's on data files and diff's on query/result files.
+FILES
+  Most of the files in the distribution.
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/indexers/mg/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,157 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib src/text src/images src/scripts java/org/greenstone/mg jni
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall: config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+check: all
+	echo making $@ in ./test directory
+	(cd ./test && $(MAKE) all) || exit 1
+	./test/testmg `pwd`
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+distclean: distclean-recursive distclean-local
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm -f config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile stamp-h config.status config.h config.log
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in sysfuncs.h
+
+dist: $(DISTFILES)
+#-----------------------------------------------------------------
+	#create distribution directory
+	-rm -rf $(distdir)
+	mkdir $(distdir)
+#
+	#link in local distribution files
+	ln $(DISTFILES) $(distdir)
+#
+	#link in SampleData
+	mkdir $(distdir)/SampleData
+	(cd SampleData; $(MAKE) "distdir=../$(distdir)/SampleData" dist)
+	# link in davinci stuff
+	mkdir $(distdir)/SampleData/davinci
+	(cd SampleData/davinci; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/davinci" dist)
+	# link in bible stuff
+	mkdir $(distdir)/SampleData/bible
+	(cd SampleData/bible; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/bible" dist)
+#
+	#link in tests
+	mkdir $(distdir)/test
+	(cd test; $(MAKE) "distdir=../$(distdir)/test" dist)
+#
+	#link in docs directory
+	mkdir $(distdir)/docs
+	(cd docs; $(MAKE) "distdir=../$(distdir)/docs" dist)
+#
+	#link in lib directory
+	mkdir $(distdir)/lib
+	(cd lib; $(MAKE) "distdir=../$(distdir)/lib" dist)
+#
+	#link in src distribution files
+	mkdir $(distdir)/src
+	for dir in src/text src/images src/scripts; do \
+	  echo makeing $@ in $$dir ; \
+	  mkdir $(distdir)/$$dir; \
+	  (cd $$dir; $(MAKE) "distdir=../../$(distdir)/$$dir" $@) ; \
+	done
+#
+	# archive and compress
+	tar -chf - $(distdir) | gzip -9 > $(distname).tar.gz
+	tar -chf - $(distdir) | compress > $(distname).tar.Z
+#
+	# cleanup
+	rm -rf $(distdir)
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/indexers/mg/README
===================================================================
--- /main/tags/3.01/indexers/mg/README	(revision 21112)
+++ /main/tags/3.01/indexers/mg/README	(revision 21112)
@@ -0,0 +1,156 @@
+                 MG INFORMATION RETRIEVAL SYSTEM
+                 ===============================
+
+The MG system is a suite of programs for compressing and indexing text
+and images. Most of the functionality implemented in the suite is as
+described in the book ``Managing Gigabytes:  Compressing and Indexing
+Documents and Images'', I.H. Witten, A.  Moffat, and T.C. Bell; Van
+Nostrand Reinhold, New York, 1994, ISBN 0-442-01863-0; US $54.95; call
+1 (800) 544-0550 to order.
+
+These features include:
+
+-- text compression using a Huffman-coded semi-static word-based scheme
+-- two-level context-based compression of bi-level images
+-- FELICS lossless compression of gray-scale images
+-- combined lossy/lossless compression for textual images
+-- indexing algorithms for large volumes of text in limited main memory
+-- index compression
+-- a retrieval system that processes Boolean and ranked queries
+-- an X windows interface to the retrieval system
+
+As one example, a collection of 2 Gb of text (1,700,000 documents) can
+be indexed (on a SPARC 10 Model 512) in about four hours and compressed
+in a further four hours to make a database that in total occupies less
+than 800 Mb, or 40% of the original size. This includes a full index to
+every word and number in the original text. Boolean queries such as
+``managing AND gigabytes'' run in a few seconds, and ranked queries of
+30--50 terms are evaluated in 10--30 seconds.
+
+Details of these methods and further performance results appear in the
+MG book.
+
+The MG system comes with ABSOLUTELY NO WARRANTY; for details see the
+file COPYING.
+
+Instructions on how to build and install mg are in the file INSTALL.
+
+
+** CHANGES FROM BOOK
+
+For copyright reasons the stemmer used in this distribution of MG is
+not the same as the one illustrated in Figure 3.8 on page 108 of the MG
+book.  This means that the numbers generated by the command ``mgstat
+alice'' will not match those numbers in Figure A.1 on page 394.
+Another stemmer was initially written as a simple stopgap for version
+1.0.  That stemmer has been replaced by a stemmer based on the Lovin's
+stemming algorithm for mg-1.1.
+
+The output format of ``mgstat'' has changed since Figure A.1 (page 394)
+was prepared. The same information is displayed but formatted
+differently.
+
+** MG VERSIONS
+
+The current version is mg-1.2, September 1995. The changes from earlier 
+versions are listed in the file MODIFICATIONS. This can be accessed
+with mg by building a database using ``mgbuild mods'' and can also be
+accessed from the mg web page (see below). 
+
+The mg-1.2 extensions include:
+
+-- Source modifications for use of GNU's autoconf.
+
+The mg-1.1 extensions include:
+
+-- A new highlighting mode.
+   The output mode ``hilite'' will highlight the query terms in the
+   retrieved text documents. The variable ``hilite_style'' can be set
+   to ``bold'' or ``underline''. It works best with the pager
+   ``less''.  A .mgrc to use would include:
+	.set pager less
+	.set mode hilite
+	.set hilite_style bold
+
+-- A web site containing manual pages, documentation, and a
+   mgquery demo page (utilising cgi scripts). 
+   See: http://www.kbs.citri.edu.au/mg
+   One of these pages ``about_mg.html'' is included in this
+   distribution.
+
+-- A revised mg_get script which uses a .mg_getrc file to map
+   specific collection names to filter types.  (Modifications by Bruce
+   McKenzie). See mg_get.1 for more details.
+
+-- Code to perform merging of existing databases. This code
+   was created by Shane Hudson and is documented in the mgmerge.README
+   file found in the docs subdirectory.  This code is maintained by
+   Shane Hudson (shane@cosc.canterbury.ac.nz).
+
+-- Revised man pages, including some new entries (thanks to Nelson
+   Beebe). See mg.1, mgintro.1, mgintro++.1.
+
+-- A real (rather than toy) stemmer.
+
+** PORTABILITY
+
+Please refer to "README.port".
+
+** CREDITS
+ 
+The MG development is largely the result of research collaboration
+between:
+
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Ian Witten             <ihw@cs.waikato.ac.nz>
+        Alistair Moffat        <alistair@cs.mu.oz.au>
+        Justin Zobel           <jz@cs.rmit.oz.au>
+
+The bulk of the programming work has been carried out by:
+
+        Stuart Inglis          <singlis@cs.waikato.ac.nz>
+        Craig Nevill-Manning   <cgn@cs.waikato.ac.nz>
+        Neil Sharman           <neil@kbs.citri.edu.au>
+        Tim Shimmin            <tes@kbs.citri.edu.au>
+
+In addition to these, the following people have contributed to the
+development of the MG software:
+
+        Lachlan Andrew         <lha@kbs.citri.edu.au>
+        Gary Eddy              <gary@cs.mu.oz.au>
+        Hugh Emberson          <hugh@cosc.canterbury.ac.nz>
+        Kerry Guise            <kerry@cs.waikato.ac.nz>
+	Shane Hudson           <shane@cosc.canterbury.ac.nz>
+        Linh Huynh             <linh@kbs.citri.edu.au>
+        Bohdan S. Majewski     <bohdan@cs.uq.oz.au>
+        Bruce McKenzie         <bruce@cosc.canterbury.ac.nz>
+
+In addition to these, the following people have submitted bug reports
+and suggestions/fixes:
+
+        Rex Barzee             <rex@python.cs.byu.edu>
+        Tim A.H. Bell          <bhat@cs.mu.oz.au>
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Nelson Beebe           <beebe@math.utah.edu>
+	Rodney Brown           <RBrown@cmutual.com.au>
+        Rok Sosic              <R.Sosic@cit.gu.edu.au>
+        Carl Staelin           <staelin@cello.hpl.hp.com>
+
+Development of the MG system was supported by the Australian Research
+Council; the Universities of Melbourne, Waikato, Canterbury, and
+Calgary; RMIT; and the Collaborative Information Technology Research
+Institute (Melbourne).
+
+** BUG REPORTS
+
+Send bug reports to <alistair@cs.mu.oz.au> and <tes@kbs.citri.edu.au>.
+Back-traces from gdb are always welcome but not mandatory :-)
+
+** FURTHER READING
+
+A bibliography of MG related research work appears in the files
+MG.Bibliography.ps and MG.Bibliography.bib on the ftp site and
+is accessible through the Web page.
+
+
+
Index: /main/tags/3.01/indexers/mg/README.port
===================================================================
--- /main/tags/3.01/indexers/mg/README.port	(revision 21112)
+++ /main/tags/3.01/indexers/mg/README.port	(revision 21112)
@@ -0,0 +1,125 @@
+P O R T A B I L I T Y
+=====================
+
+The main development of MG is done under Solaris on a Sun sparc.
+However, we would also like MG to be reasonably portable.
+Therefore, MG-1.2 now uses a configure stript generated by GNU's autoconf.
+Below, is a list of the machines that the s/ware has been ported to. 
+Most only need the configure script and some require manual modifications 
+mentioned in the "Problems" field.
+
+------------------------------------------------------------------------- 
+** Systems which have successfully been built.
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2 8 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2_DB 14 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.3_U1 1 sun4c
+Compiler:	gcc version 2.5.7
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.3 Generic_Patch sun4m sparc
+Compiler:	gcc version 2.4.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Silicon Graphics
+System:		IRIX 5.2 02282014 IP7
+Compiler:	gcc version 2.6.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.4 Generic_101945-27 sun4c sparc
+Compiler:	cc: SC3.0.1 13 Jul 1994
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:
+System:		SCO 3.2v4.1
+Compiler:	gcc 2.6.1
+Problems:	requires RANLIB=true, use of tcsh
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	HP
+System:		HP-UX A.09.04 D 9000/837 415303291 64-user license
+Compiler:	HP92453-01 A.09.73 HP C Compiler
+Problems:
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Data General
+System:		dgux 5.4R3.00 generic AViiON mc88110 
+Compiler:	gcc 2.4.5 (dg supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Pyramid MIPS R3000 box
+System:		
+Compiler:	cc (vendor supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	IBM RS6000
+System:		AIX 3.2
+Compiler:	AIX XL C Compiler/6000
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Linux 1.2.9
+System:		
+Compiler:	gcc 2.6.3
+Problems:	none
+
+-----------------------------------------------------------------------
+** Systems which have NOT successfully been built.
+
+Tester:		RBrown@cmutual.com.au, 
+		Lloyd.Parkes@vuw.ac.nz
+Date:		Oct, Nov, 1995
+Machine:	Dec Alpha OSF/1
+System:
+Compiler:	cc
+Problems: 	Many - see below.
+
+* Problems with Dec Alpha
+  - 64 bit length longs as opposed to 32 bit longs.
+  - Lloyd.Parkes@vuw.ac.nz says that there is some bit encoding problems 
+    but the decoding seems to work.
+  - RBrown@cmutual.com.au says that it will compile but will core dump
+    when  building the test database.
+  - tes.kbs.citri.edu.au says that he needs to look into the repercussions
+    of 64 bit longs, however, he does not have access to such a machine.
+		
+-----------------------------------------------------------------------
+** Other systems. 
+
+Systems other than the above may have installation problems.
+If this is the case, please send me (tes@kbs.citri.edu.au) a log of the output
+and I will modify the configuration files in an effort to fix the problems.
+
+-----------------------------------------------------------------------
Index: /main/tags/3.01/indexers/mg/SampleData/MODIFICATIONS
===================================================================
--- /main/tags/3.01/indexers/mg/SampleData/MODIFICATIONS	(revision 21112)
+++ /main/tags/3.01/indexers/mg/SampleData/MODIFICATIONS	(revision 21112)
@@ -0,0 +1,664 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /main/tags/3.01/indexers/mg/SampleData/Makefile
===================================================================
--- /main/tags/3.01/indexers/mg/SampleData/Makefile	(revision 21112)
+++ /main/tags/3.01/indexers/mg/SampleData/Makefile	(revision 21112)
@@ -0,0 +1,4 @@
+DISTFILES = Makefile alice13a.txt.Z MODIFICATIONS 
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/indexers/mg/SampleData/alice13a.txt.Z
===================================================================
--- /main/tags/3.01/indexers/mg/SampleData/alice13a.txt.Z	(revision 21112)
+++ /main/tags/3.01/indexers/mg/SampleData/alice13a.txt.Z	(revision 21112)
@@ -0,0 +1,2293 @@
+ 
+
+EÊ%EPq€
+"NrD'D@©âdÈERL)DÊ$ Š )Yä
+K
+`I³fMcRÄÈ#FÌ
+ Èy£ŠÌ:
+ëÐ)ãFL9g@ 	3D0dÊ#GN7gÊŽaJ§ê4tÐh¬ãfÌWeæá:F­7r@(H«L9tä€£4Í7 ÞYÛö­žsåÔQÑÀÓÁd°W9y@,ÅCÇE@'o¶9\&ô_sÞ°Ñ¶
+bB
+%z9éÒŠOÏ¶È
+2ÃÐ1ìŠêÔtÞè	TW1\Ã0v(¶ÜáÊ¥«(O
+y8dÀ
+2µ
+0iÎža"	6Ä_
+ Žà
+|òhŒöÚSÉQÆ
+\Á_
+oÔaU¡Áñ`  ¢tqÔVnäqOYžUUl
+U[ÇPføo F[uteÛpzÁDUVH¡üñegaÜbÌår
+XUsÔ1Öm¹¡Ñ
+ÜR#ªöOUµ%X^\þUb[Zý( !¡rÌU
+×qv
+FÍ¥eXvÔPnh&Z€Í1
+£e[JØ'ø}	[Yå5Æpä!Øh UŠ^7öØ_iäõãRgFG
+üñ%á]¹§ÆQG­¶Ú}sTÅW"v«tðwšnÚUkhU×WaÝVæyœqb§ÂF\Ã]¡
+$Uæ<Þâb_mkgÝV%lnÁ_l Üáí[§
+yâ³i^¶¡š²}Ÿ1V»Z¡ªVÂU\Á_¹rn(§Ã.F®¢çj:š_ZXzŠõÚUn°)ÇpIší
+²qÛlÝp_
+cX^5ó9GcmóS/n%ÆvžšF
+ þ8udUã
+iŽšyŠ\Ò¥QeTiuÕÙ+š³ÐñÆÕYoE6Ëß v€¡XøÝt
+!
+@
+[ÇvŽA
+b 'L:Îä_.ù.
+¶iš NyífiH|±UÔG(ÅSPÝ>èw!öØx©5Õ
+
+ä9ažq§N}s¿Ÿ§VUDµ
+dïú|ÅýøÆ_ŠýôS6ÕI
+QÄ	SXDQQEHõ'áWxF&Âd:kgÂ	œf\ÊÇÈŸ
+H(à&@Ä	IšB@`*0aR!e*$A 
+f°>
+ÒCÐ	Ob°
+Ç@<	Ž¡XÃ!¶ï$AÂC€ÐÔ°FÔ¹Ãì-sthd£
+B1TÀO01ŒFN9Ã"
+)ß
+Ír$tE
+o9m
+ì`c'ªé%&bÌR±ã5ðç+u\ÌT
+×F
+Ù¬eBãcÆ9æ,ŠrÀ¡
+eøämn7­\í
+k
+ÕÈªQš*ò
+X¬¥!
+Â=©:).¬Þ÷v·4¡
+?`@dÁŠâ#gñkZYvNiÈè%-bRÈš­á,Ø¬¥ºòKAê0ÃÔa~`M@d
+©Ñä«R=²êGQÑÓ
+§hÝ42ØÇh$ÏÕAhüQ°¶'
+
+CÛ
+ÍJãÄ6	eh	:š
+)àÏÊà«kò	pë§Å-|Ö
+øC
+Zð-=í `z^ÒÂPÌÔ¡JZQÀ¢ÞÇšxE·ç\ZEþe]s)Ó¥¿1
+
+`Ð€)\QDàTº
+Éš<Ó+è\pØ¥}òÔ+
+÷Èš
+RÈ$t9¬aoi3w ŒŒøå6Ü¬ä^DVÝu"zmÃäÔ!€,ªcjXÄzU
+X
+DË€ŽÈ
+
+f@æÔ®  «=Nd^ÛDMJÉ8NyÉB`M `€TåS8µÉ4Ë~o+fX
+DšÚ21cCèH±=ŠE9°áS6[UÝ0ôM@
+Xn]9÷ZånÙòÛÂ¢2Vmn§ I)UÀµëÙ¯¶·
+
+ð%XA ÂI@ð0Øý¡Bý\)(ø		6PP0%
+
+ØÞÊda÷HÝ
+bÐ0ÝuÕq?Äé&lâŽq¥Feµ\f+€Tò$¯>T=MÄ×EJAqÉŠãê·
+ÀÒ\Ž¢
+¯¬Ús]IµC!;J9-ðÐºµãAqY9!mVÙñ¡X£Ö>É0áãUvµÖ­
+¹Y
+Uèº\åÚ2Ä·/K¶¬¬*[ë×K±)?×CpÐ
+rZŠ
+Ž
+e¯þmÆK*ªUEdË[Ä|¥GF¶YÌÔ2Å+±TN.ÕPÜdªQ2šõLÃ=Ûš1{5M§r tAçZŠ7&ŠaÒV}A>Y¢æbÁf
+3ü7KD£²hØÂ&ã,46QËlBÉÄø¢MØ²âGoÆ«*œ
+äWÉÕVÃ6=2{k3`P5ÖšÀîŽqJ÷ì2R©|ZBoëÒ{×è¢Y	å¶CmÊŒÎÔ|šqæ
+rPÂapiãE?vÑ9ÄèXåÏ¹ùc¬Í2$
+Íe)MÐÏÚãe*­Öæ=²ûÆá-Õ2±;×ç0u³a7SnG©¶°Vp¬€8
+ðž2¹ÍZ®»ËŠÑ©
+èÐÒGHa6êW)ÀÒùª+aÛMq¥wŒ®r°yY£Ñzqd7ÅÅÌí
+Æ`ÚenSgéÂáj\in©Ùy²!ksÈ/ìšäÕSüiöº
+
+¶$âqú2pRaQ=OÁRïäyt®vš¡³f
+ŽX3Ÿ*
+õU+(³äBÂF
+S,0<A
+9Pòó»{u©«¹ fY÷Ð­
+."ðBT,pØã
+zXº>)
+EICiUt|y±PÕY²TåÓF'kWCzF08me·,µ€#$Ì×xÞâQ '¢WtA5sÆÁ,xWr`à.ÅÁ[{²]ÛTF'³YmqÈº6U#fà^4b^üza[sptÅXQqY^R%
+ÇOjvŽµEö"^Ñ7|'ð1ÍSJQ]3Žuh gBxSBã' dxHÈtžc
+PY#-!&ø<QEÑi
+ØPÐä(@¡,÷¡n¶rPeà)P$?ASÄ÷fSÔ.ACyZÑãEW5eòLAŽB^!uÅ&5úÂ)ýq}ã@ÇV{òË1ìôzøA^16Ã¡ªh\¹ET.
+nâzlÒYUdF$²R{È'ÃÑ@H1pRJÓohSáô4"f6²xMtp-ÐÖÀPÂ2eükRásp#gJFO!%¹)Urà#+F5Ù±(ÇuAbl0yÃ¥æõhþ§
+x¥WÏñÄp;Åh;g..¥D5ôb.äL-Á|€5­DE4(³
+s}ÎAbIå×SIeò4®a\O`Ðfå\¥
+'°%WSKUŽ*Ü3wbFíx-w­èJÇâKb#TyÁ2a£I@Zñ
+"ÓŸe"­5uH!ËAóP`¡b£e°£~x­±m)+Ê4"7CÈ!(!A"VPiôÑn0
+Åu»vÙå^¶žWùÁ]
+AQGUth
+æ¢)ävYh
+S-e2wm×^oRw<÷IšXà(%©TgŽ%z÷Æ\Ž
+)A"ip6AhR?²íøqKÊ±Gg*è).Q?NÂ|6ùøÔ+
+R{±^/ù\¥J!"]îU«wxR °-p-BHÆÑsVTJ
+óvI_Qó§ÂáÎ%,+yÃ)å"M'5ü¡ñ	­c/Z0 "bÇè³œP"VPáb×ÒUr±Qé4
+Ò-íØmC°?VYà?A#crÇqCQ¬ÂPø!ŽEØB&]ø¥Ehøihñ8 A bªož2ÿGtXjl{¶ÿè
+hfs°T`A[šQ¢t §
+Ðk
+
+ú§óIr0³W[Sñ1Vºò£iÌ8RâíÄ>x.(ªŽÕsåHoØ(kIo00©a±%O%T€.õÓDð#azj65%Yiö-)ÚŽ§18µ§.è[œZ2ø|â¥ŠÅš1dyzÊWb<z±GO39Ÿ¶èp®2-°È«Ÿªm­Å¥q¡
+iÇO®WgbxªR!¹d6Ô['-8èÂŠ~
+ ,fC\Ø:Ó€
+ÉLQ	n5²Ðt!Ù²¡Ã(µ
+}¢
+Uá~ž¡²ÛIr kz®g àl
+à%es.§g9kYbWwqW¢®v
+2¯°±Sãcz^­+]ØQ?À«<ª²bµ*Û
+%kîê\R{cW[µHAµœbJÑw­(2Ç('
+`,q(J(d ·Æ*¯Êah+R ÒÃ!Ø(ÛØoái&zpÐyžiÈXgÀ8sáWPögžJlÊ€O©=Óú«GŽ
+Ãl97 šqü±ºhÐ+b
+€8vJÓÂÂyÁ²wµœÈ_úvUz^¯%œöñZ
+WPn5
+%ÚÁY~šq¡'bszÛî5WXó%g©pž!",V­Ë'[cÉrT»ERi"pGûÇ'×¿/qîS}÷Ÿ:ÕPsõj%#m4}žŸZ5_Zs5Ÿ=÷üq¿ Ñ-ò¯4âH+kñR&¹¥VtÀ8ÍYÛvk2ørþys
+¹
+CieÂY=6@["r{lB^u£3ª%¯¡ŽÇyñÙ-àÓÂ·	*\
+«Ú[_ó!ŠT5Jy7UGÅš	Å
+
+¡%ì¹äÁaA©[SæÜ§c5Z<b³4bö2©l U;W³G&Uv*ö!Sm|ÃP9;	qõ.1
+x±%s¥­1·e,ry1é^12f¶_IÇtésN±uýråqÌgO¬wGUEBå+œòÂmŒôJb:IqÂ'Zd`oüTEfó7P0ëR¯¹ÈëFÜfºjªîóœ·œZÑœI{Éuz2h !eÐP.¢X.â^W!6ò³qåÉ%GÃU¡ª')-HA)f5G¡ñ
+B³uÝ¢gÍ#bž¶±9>Ùo"2±Üµ}2B1
+XAåÊàÓ|t[u(d¯E(§»ŒfÃ§[Pá"²XZÑÑ'œwsp·Ç[Ñ^)jhVèãjS}ÐÏ2
+RŒCENc­(&Ë3ÏPSÏŒW#7ÇAR"z²d>gÃgF9eçqü\<§ù¬iÖ
+>º¶u^Ã"E¶,dÇm§²Ao  Ô¥%Ö>ýcF°Û~ØVÂ]"#tj¯õÎöÕeÇ'2bwaq(7"¥N  ìþžD
+È{âTmÑØôâž¥bÙwv&È
+Ûy²
+#Y3{³¡·uÇ{\°7RXx2rÇÎÖf÷Ù\JÄD5	WKÍ3/žVN!zB
+ÿ©)7×&)Mku*²Qq$çÅÞÚ{ÕeÂál{ô/¥ÜÅÚÖYL"fà÷¢L«àá³ys¥¬ÂÖtF3ØžLVý¬¹uÐ
+
+MÙNHÁŽŽm2
+«%Æö¯€BX¡ÊÆbi
+«
+éRŽ%FžãX¢Šyž¡
+e/:¬¹ÓÖ[F"«!nKÎSY- íÊ"-ÄuÊ>©TÅºÕhÑ[yÁS³FØoEQö²Xø$Åâ,Çg,êheã!Í2Qr«ü^bqÇ¹å$,nSøQTžXEø
+X2
+«^³
+cfpÂ%Ð-,%*ËÈ'*]Øû©ª*Žbf&Õ¡Á³âFMÔ²žsJéÆRlMÄA5äsPäsð'ê®÷ÚûÙŠÔ#M3;ßÔ ¡fT éhó{K1)qìá¢%lþf!šLhÍPl²éBÓž¡ìõ×÷ášºµ}"¥= U!µRà?Öó£N
+aåh7r+ÉÂìiâi>MRÊ|ïyG#îuQ*Wi%µAVcÁí\iÄk5éA%¿5ìPV÷ºf¢Ê+±zòœÓ¢9š³ø1§#
+vÖTPOó°ÊQ ŽL$µGfì&å," 
+)äH'_;GË·Kïõ=þÈ>­âÑAvRÙfuq6+rcž
+¬ç~[ÉÇõLeb°F5l¡ëT!§x9[Fáï­zËÔ`L6
+®m ®ÎèEëÀÔµ¢)d0Eæs%ÂàÙû·Tü¹W¶mH1€|6ž$!,ý¥Î9ÙÏ5ÔLXìVy¥/,Ò
++#h`A5)[¡díÚ£
+ôbØ.""\SÕ\¬£5ÝIÊcp©IÆäb
+8ÝiAÊ
+îtX}µÍÄHôÁí:Ë«"Ó.×8
+í÷.Ú3&°Êå·d
+¿©üÕß¯q5Ù?O(ú
+±ýÝçýû¡æÅª!VnO²Ø
+Ðí±È0dI>#²Kêñ`¿°ÒÒÔâ4­€XJÀÉâÇx
+Q.C]x
+ ¡]}þ ]Ôâc¥Åq¶WÞÛ»&A¢£õ
+J,C9Êå±=
+JÍ!1`ä«("PRÉ* &DA0gB
+W
+JÁT¶©`Ž_
+rÁÉ`Ô^0
+Áw¶*vèQkhO\ÐÍC,2(Â	!É"tG%û qãë§Je3«iù5áO@	hlóË^899ØÚb
+«	ò(²d"LUú
+,ªõgE\ 1{h·É?ŠúT^yÄi»D7ª×)ùm²í}9(!Â02`gÇ°Í
+]43s0öüß!*Õb\
+V}6ýÀ+H°£±Íöìž
+°"ë±7óáJJ«o§(ËU¶zÁíîà·[?µ8ÄÀŽ¢áùh#ì
+D7Rx]àÝì]5
+$ÕUË-°1
+üNÆc£Fj
+\ÊSEDàÂ)+`×s¡Fd#V
+*ÁO*ÎîYŸD7øR:[$ËPË÷"ÌA5¢pÑÀÈT*Dk#«ÁS 9Š`ÄN_0
+ç#
+ž{O [á
+
+»üŠrX(
+âEÉ ×Â_Õ[r(e¢©X3£èßhÓc~ÊnëH:Zšá6
+Ó)Æyx¡é
+T)	h@
+Ç
+b§nèâq·Á	:²Q%^Ý\
+h¡±FD÷\ä
+@)Õ.4,gøÄB
+)G=Ä=â@Äo
+/S%6AÓ
+H|ÈÓA5?6ÔlÎurTyàøy±­C×:É
+I'ã<Œ#ï€ÙOZ!+æWEËûá'ÊDÚ=64d
+pÑ­=á'òKñXIÆþ¢wHZr£<€jä*B Ø8 V(
+kEX$ôM @ê'nÀwž °ÆÂÎ~Q¢oxÇÅpQ:y!
+ÃLÀDq.ÊßiÈZÜ
+â`.Z)Ql­ %eNÅF3ñðÞ¹ð=¢ô
+p1Ú»ZC\£1X<à
+*ZW£=ì1"
+Ll#'Ðd`HÓÈ¢DÔ8w}b(è-%s
+ÈUÒŠ9`»
+yÉÑá5BsäÄ]pQáFS#OÐ0eà8	D0`²ðNtŒ1,
+~8RÈ]ã·C&eBKþcÁ.É²ºn
+¡â+ÊÀÆÝlñ7>¢eH.í0P€ùæÀ'9
+
+VŽÃ0Ù<É{ÄDÎÁáœºáe
+ìb$¬âRJ
+ÞJ@¥±©69Æû¢1á5ç¶¥¬nÖôBáJ3)Þ
+(°ô5éskïßŒmôF	º.!øè
+ô»ÖÇ`YÇ€æmGn5n)})_š+ÆWd	eÇœ
+?+Zk@ØòÅiK7ñœhj·å)l§ðà'\^0
+EŠÁÓòkKÃÙÜÂ§xÝ`2YE$c
+¥,¹íGüa
+s Rþ5f2Ü[Ä"­k;~<mf1Z6h@¢ÉY²Õ*°1&4ÌÕe/Š§ ñà(D¬(ËÄpA­Eè
+Üå{ÊEÒ1J 1ðšDô'¢RtñÐŽÚ€ÌZã#­ð
+²¡É$Þ£}ìGÅ¡­<ÌD^èóq?ôD[ª7§áòõS
+Ó	
+C)
+%€ûQáZx90ÇÆÜhABi(ÂØ6àK²Ç!àeF_òQôâl (nI
+Ì6ÞÄáz+25Q0ð!I1G0K(Á £[è³A0:Kç4iP}ŽÓ©Å`ëDªÓuÁ×:g§ítbÄ2œ30RýaUÞyª@Hã
+HwòÀÀž3áPœ(	Mž5ñ:Ê
+¡ÈEvÐß!ú`>3(À#Zp
+±÷Ò'áÎ#é
+"lÀ/ª^Ì¢åæNR-Áì[V·Bž1Då
+:.¯¶
+ T ÌÎÑ1h¡ÈÅÌVbnÚY2OÇGÄå7édnãÏ²Ô;ÂÂ,bzK
+oÝ
+TÍ@
+M5Î*tÁF
+ØñJâÅ_jìñ1àÀUa+#Æ
+ÆLÓÂ
+Ê®0²Öþ7â
+$/päÖ±áÈŠñAsÖx?,JY€Ú9ÃâÊóG6©$û2ãXÇHjX Ò^h}_°l¥Z²ã(BTtòYp;7âÔ pT
+&]Ã 0³â9ÿdä°Ôð
+h6²
+Ýš
+0i
+Ôb×MØû°øÚ)i{Ô
+ÐðI®¥øTc€!1ËÒ{ZÔ<òåÊgUŠØË0b[ÄÑçé®UY:©æ	6I"2yE/gè&-Ôü`,H&Ö°±ø
+¥-ðy$ÈvÅõ8
+fôTrÚ§Mxñc~Ô{ 0 2L(9
+H)	®	6%â	JÉÓ F" )0P§@D-%ÀSùA?8ÃÀ/TÅz%U°
+h--¬ŠÍê ,
+J ésbÆä@8IÒ3Z³þ$c
+zbãÈ¥?¢>{
+l€q	°Ñá&¡¥DéìÜKÀØ¢F£lJí¶ç
+Ã¢e,f/ã©»É{x[ÂMX>
+9Š,è«zÖ± ³DWXVØëôRFÉP âÖ©3_Ó¬PEÊ©³¬,+>µ¡|NáLLÑ÷hlÄäžÁÍõHKuÜ
+IRelaÒŠÜšTÕô
+-iéí
+i2ñ¢,+Ö
+ãà±Øx"uõmâ%!Á@³QA2­6EîDºòm}è=0±qÑªLb3¡åjŽóYkFñož(Á ÅeábüÁBÑ>éÀz[pD[È8»~žÞÈg'°*L#îãTéð
+I¿p¢£4K&2X1â
+å¬!Sµ'ò-Caôj3R
+Ù
+w^+s
+#wÒø!B}
+)H*ö3
+%Ð9,ö&pÏ§)-9M,
+&Ý`>03Åtg y{rqSR4èT®bn)Ž¡Ç$Ki»xuì
+
+I#f¬ð>µ¡|ðê\ªÈUlÁ$Ùð¢0
+
+¿sv`Ÿ¶Ô7º
+EÛ#]ŠšD+{Ž
+aVpý«BýK>fR%l |¡¡Ã&öåYå/U$Ââ±§ùŒ€87ëvsÑ+Ê·Ýô
+1%÷äBB!rà;O=ëk
+Ç2
+@§NéÀÁpfMjÞ.Ø.ÆPTëZSŒ±ŽÙÊ[í
+«÷H¹©"ã-®Ús_Clø€f&s<)OïË¹ útjÚ0cÇ/ëcÖËÌ[
+
+ø:i)ITgÀI­È`fK@ènzkÀÆº¢%E
+)¶5 $ÂŠ€\Ð-wH³t48J#à3V8.ûÆ¶Î·š}ÆÒ€heÕ*è%
+^ÝqÌ$ç4&Bb×rÄ 8gšyd!tO5ð øÄU
+m€Ó°År
+=ZPèTv\3
+¹ë: @o/á\ÈPjÈ©ó.Ùhhè]&/ÁTÔ§øEF
+Ðz öDÈ«
+;8'éž3o1%»
+Å)nR®òË?ô4ŸÐ§i
+ŽIVWúR"¡Y$ý]2¶PºÆéj©LT"R¡
+
+
+0¥
+6\läËÜRÎÄÎÄhÇÜ¬e(
+çÈÔÂž ·©0Œ$eKŽâ"ÛŽøB\Ê­Ü¢à^BPÎœ€¶×xòõKIvÙÖDS\þ£-B .,
+ç¢äé<·?¬ÑÞká¶QÎè"]û£­qø
+GNO]š}»oÓÔ}·¿§@:I²ÂÁÍ€ãµ£	©)Ò@ßÐE¥Zº)£}ÍÚXH°ÜÚ$[åw
+[O±ÍÞû{]Û¢Æaù9°hkÄGzÁ.°Î6úhÀAY(E3šÄ@"%r`ŒUN¶<À8ïÆZŸG L:&8P`"ê×ª1%à
+òàðÆöGM
+8Ç#ÇB;âüÝúÑŸ Åo|¢å4iQ>xKŒKPfj\
+ÃŒÐyEò-Br£@ÐÝOYù¯
+é`M8Êµ
+Í7þ]Ç'o÷š ¯Rì
+Oáº¢BBjèt¯9]!!Þ'ÞQ?{ÑfŸm Æ¿~10(a ·³aÀ¡t]*£HG
+Ñ$-D
+ïÎöª0¬
+ù/Hj6t6Iöa(èÇ× I~SI
+
+U$Bñ~ÙÂ+(ª»ÓöñC*}ªŽÑ-dÑ1h*ñm:OàeMUèÚdÊ?"/îö4-lÐgî5|0pI%×Ð%šçf±MžÚÃ>:ãŠv6\Êñ©Æ3 È]hAwßYL`ÑÚ€Vz×·£,€É1S6¥îË}8Œíìm±²xñýÊ£1êZ¡	'(
+dhjY9Pì
+-(
+bfÃ÷kØeH%¢ë¬
+mA»\4P
+'
+òdi/-@
+ß»ÓŽÛÁè
+'õH»ÕYå·e×1­#C^Øš-±>*÷ò*dgÉJD5k«ç«ê©EX\s2º6°Å§È°xE7êH¶Ð
+¹yMÝÖµ=Iå^&a\]ªq
+ÛÜ­ÀsUU#xa·cZS~«Èc9
+'%á€£­áÔÅàHò9füdNÛ{QÚbl@.=Zfê¯õ
+ÙSÅ;"pb3 â¹2QE[òÎB»B}ÏdßsO÷FB
+Exûnc"±`7Ö8H	Û"Ò±E «îð"fV9
+
+ÂMaR4ï¥Y³÷(©üÐÑØ
+ûË)T0bä.ëí-ôÑRŠH`l+ºGøB$^ÇLzbŠ¢©"d/jÚŒ
+(UYFÖ$í)²LªðK@&#
+oüºÊç7k¹Wp4$š¬æ,GyªóõÉtàKÚPLXT?ç¡ÒphQÃBâRŒÂGÁ*	ÅÝ(*[`q¥tÁ€[
+Ìdû^ž/
+ xSOÒ3J¬f01
+hýâÆ¶[I}Øóä Š'=ÕÑnË:F³âÁD[#üã57("
++o¬J®}ê§?§£)böãQ§9Rô<áLEþ\(³º£A×Û}ê
+ÀºÈéŠ^&i§E[ªPœê¡
+l'ÌÇ¹)°ß
+u`ÃÅA©Y2Ù*o"qûh»ª)á(Ï%XiïVy¹&+ëIl[¡,Ë
+"ß	'²ÒZÝ¿
+V581B(+LcOQZ°³mYio"ßú}·ã
+k¥]áP\£ÖIk§K«ÁX
+w>dÔ$¯%r53Øµõ\ìÓÐ C¹xµN
+Æ]b6¡+ù5­±MãåâŽªÝ(
+XËÛo€]ÜØžoê0á°ØºQŠNóYç$3
+PÙ|2jñ.e"ÛÅù?ÅrùD¯sQQx
+²QÁÑê9.Â	AÂ'ègyÊWãþ
+€Ù<nÇ:W-Ü^
+r-xñ\UQ#V[]¿pÔHÁ¹
+Mi áhã}8³À1i¹Ëñ#R9É¹ð5€É0ªLâbjQT&@ÿþ¬ÍSÑ-8ÐÈÕ²b	L4gNÚžm,ÚI?¢éTS
+é+È-ì,ºg]ÐU3
+dØ@!åŸMcÈÃµã3þ­>4RpNsè*êðÆYÐ°Õª»Tszi}3.6¢t©Ú!ÜJ$W	Ù/ÀÄÛ× »Í-º=ï,¥ÊTÚžIäëŒîyËè[¿¡î*X®k8Ä¡$ÿÊžgs55©AQ¶6åv4$­1ÓçBa7ÈHrpP4Ðí!JŽe[Ç+jŸ±Ð©Ë9ZwÕŠ%Ò6î×~Ì,k¥ª°_
+×úà
+BÊØàžj"âÄá©²îjÕïM(÷9ìâ{
+8ÑÓqûH1R
+_Î
+¹~Ä<ÎêñYÕœTâº K[U9yoWGÕ\D<±Zà\ßxSä|c¶qx
+=1r¢··{¢Ü¶ÅeË±KNØæñA·ÊÝ!lÇE<&
+zpØ¹v|_€ò[T!wÞÏð}"éÚFÀ5)P ÛËo=ËU
+Gµž¥ãìRd%ŒØTÆn,œ€'Wh}«={/F¬ãíš¶phÒx%ocŒG+êÔ¢g\51gíË&_Â÷õ­«ÂCqÈ
+^%ÂŽtÃ9ÔÔcÙpSA§HíÓÀ,ò×Ûà²íç 
+ßžFj
+±Z^0ÂRò
+%  Ïp® Ò
+ÝÞ
+;£· +wÍê{Ù#ª%Ug«ôÌÃvåkDº=^s6
+*E[\¢7D'îfAVÐ€ÄšnW¡O0¥ÐŒfe×á[.Šp°TØ
+Áæ
+]-ÖÃP ÎÎ
+KÈŠ<?Ïü$æ¡èÎþ(Xèè€|ºjäó`(ÐBiD/¬û (åõ|Á
+À
+ä³
+¡3AkÆM~
+²!%D
+H[}â±',îÂôXT
+x³0dÛ§Z.pC¶R^Ás¬&
+Omú0©ôð|àÏ*
+4Hµ*ÜŸòŒY3ËžNÌ³*¬Zú
+ºuCn£Pâ	3sß\±Q|²
+S·"Æ¬ 
+¿œq¯Z³ÂŠ,€À»|Ã~QJ\hK
+ô$"Åèë¯/[,F³liÜ
+PXc€&Ö«{,X]ávòž~:FÖÄYjiÐ6¯qëÉ¡-;ãtÛûúõèrÚThäé&&LTDdÁÊÞ#÷¡ÇEªp^©FÒJ- 9ÜÎÇ¥^ÓB=5ÄGÓÉâÆê|Ü-ŒàÅ|JÅzC«}¹nÁ¢ÂP=F
+3'
+{ÔÎ
+
+vLO°õhFX)ñ5Ÿ÷é2+íù#ß~ÔÏ
+?
+(^.:vùVag0îtŠ÷±œŠ7š#ëÍÅ7BÓvÔuTð.âë4ŸÂÔê7»§š*]8DÓ`ŸnÅ-Ø%ÄÀî ŽÅ¡sBt$¥Df©
+'¯dªÅ8J`Ð37?ØÂ
+kb"rëðbRUÒY¢2IÒ7&FjB*Jì
+4Çz¶[²ò^ûÚÕgA¥Ê_ùñ!LÚÄvòÁ"CtòH¯!®$°äÜLü
+¹Ô$Œ)ï?6ì+Ð-³=eLÿ·s!¡hu|è8"~ï°·CÆGäúì³9},"_5ì(9op
+8Ò7mÒL€ž@ÈT
+nšÍLYî«wdQ¡G-¢D·Ô¯cz¢Ð\yDùzÜg©jB3F©y1As×5ògèÝ)š±Ìß°žÂäÎ	ÔÆ6PßådI;°Ð±pÍ
+XöÙid[rß²\ÆjcA&Î{uaÅ¶($ ð¥R`@ÃW%®
+HèïœÌj]±'­T¥ï0VlBÑÔµ
+q2Ï
+þŸE+v^Mù	;
+<W1ëw±lè©"b1>MØRÚà|c5 æ
+â·Ç¥%¡k¯xé3fÈ»fÇ ò»5-ËÜ5øžšSäÖgÞ}ýeŒèÛ/¡_Wçóz
+hEÔ¢§gqNÌT;)&úÅžã(ÊýïÜìn"<x6
+ºüŒë?ÀyËoÑ
+rœ Û£Å äébCÞ.ÇŸµÀ®.Œ]G&Tîßã|4FÕŽË0ûV1uáìD=<QJ
+ÊeŸLR¹
+Üh¢¯+ZÿJS+DÕ"
+i
+×$ò¬:)Ôéàï.qŸ­w|"sD&)Za5àûÜëÛÃugÉ/+FDKQøÔQhsxc9Ô%
+zÊ§zY_Çp°G³MŽÌ2Ë¶AÔtËlÖ±5+ÌäP	ŸÓ<	O_Ä1ÉA×Ðx»ÇŽ4
+X`Á<*ž
+·çX1²?svž 
+á
+aÏlÃ²
+a_w²ÂÕ/Û_8DåIä~<
+À@Õ
+|ÌÉT·\C>áRŽ1­J)Ñ@
+`ožÃñgdm÷R[`1(¡#sŽk2×Ô·(T(Ø+aãš3ïG$	;IÏaÅÔØÆ!*{
+ÁGÌÐsÑ
+@£a_(¥^RÕ}µiÆÝµåRô èál€djRó"}jØYŽ¡`6
+ËÇ`ù><Ã	HáyËô'nüsP0kZü§gyØá*~~ÑA±x4ÉUCÛq€Ùl6Ù 
+oûboPSðc%=Ø{$¿\5¡
+d/mÆrµIBO²{ak3)âžrÓað<bÐÁïbKl	ºÍùòpkeuÅ/²Í
+ôã5q`1ŠèÊB2>ø;Ø®°Qž7 
+ÄõíFÈÄ "š#eÏP@
+íIÄI%_LÀðôo«]W''0ÛÅ(Ç÷^
+rTý>÷
+µ³üL€ 7
+1NáL@ôÿ
+1ÌÖÅF`_
+S8ähä PÌb,=Q -WÍW#`
+4JD~ÿál/Y
+â© ?×ÞØ,;'-Ø;
+°ä<
+·³P0*Sµá ÌhÒÏõ^õÂâÒá	
+ZýD,ÙB£eä"8V>(|B9wÎa9à~£÷è?=8ôK	Qiy
+Vå
+¥åjÍ\Ÿ"U€ð¥`mÆSl
+LÅÄÛ-Pt *	Á
+`ÉQõEPË¥¥dÀAÈ·(4Oô 
+(š6ŽaÑpaqWÀ* n)Èì
+
+Â÷à·xŠÑàíiCáßç2çÂÏt-H
+Â%ã!)qœÊm0ÈqœTÖõž!C!
+úhgËw1\&MW÷°(
+4!Ž#BH¬`F
+	dagø&G¡¡[pÍÌ&Z@JCÑZ/ôŸ&[
+kK&DùÊ
+[åÃVs£
+=€@
+d1Altr@¡g;p
+ýM;B²AWÅq<îYSÇ'š*H¬kRyžVÔaÔ
+ÿa
+7@éQ¶pðéOm=Zsã©cXámÒ#þš7r
+!#Ul
+xÒ$äÐn§þ)qÖÜÕ$£@hÏÜgIpxApÒñv{
+öÙJGÿQE<ùÊÌÔUp
+qØ
+bÅ]£ÏðnßÕž0.µ#Ü-KY
+!>âÓœVß\gØæ|Á
+' ímfáry)<sR6xN§[»ÎA
+Kõ@LíEaŸÍ \©`2 \Ê
+Ä¢p¢Ž8ÃŒ7u7ÁÆ$!ÅŽÔ+œHÒwàH²&ú-
+ÞÕ÷wx>@aPÓ^¬,õ 7ÄäXcAòð¬x
+!s}XÐ¹È	_ÈäÅ
+åFÃ9Zä¡²¬bAœ¡gOEKä§Ad Ü'^È
+:F®Bš$ñàå	\ÄP1aš®`0ÎðFH
+à_HZlÐ ÇQõSŸ
+VEýGRXn]óuÑS
+>€q WîŽW9E&FGe¬ùTÛIÆž}¬
+}ÃÝ1Ü-cX1öKŒàqèr
+N³x
+è
+7 }È¶Šý
+{ID^
+wèÀU
+gÆQ<ŒB¹ØþÝ\Z
+r\ 1ß²d¶hÙUXŽ8rOátQ`
+Ô%`G`FP6jhcžuË¯ÝmÀÛ @#üf.b
+IÅ[H^€a©WyLOŸ#<D^!Tdp
+
+¿y¬'€«I
+^Èåµ!
+¢!Qc·ÙLpR×"9©AÛ*è	áOçBþqgÜÌRH,õ
+¯çØü
+Å¹pŽ
+oÙ
+8Í!<â 
+
+íÓèdo_"ÇT&ï
+0ŸeK~EÁmZ	è-v
+ÐX+èskÅsP
+ÔÂiÒ­<`sà2ž°jzà/BUjÄŽ%
+$°á5ÑVKà.ëau*þã$xYã^4À'ü
+x°A5šB×ä
+T 
+
+iÖÒ!J(	ÏÙld¥Ã7];]LA1@9]ßÇgtBãÕ4HÅ÷àHØ^An3úÞGŒ?M
+N<áÚçd
+eÚŸèb³Íã@i#äÁ"(||Ú,³Q|
+CÆ
+`ý;(Æ}`<2dò3InÇµ<ä[pâOÀ àeâ
+ø7Jh¡^sP· Óã ÷¥=-	äñËOtÈHìÜrìZíhIa3ÓÖkUA¡à vá 1Ð
+^E',Ø[Ã¥$ô
+ïSÕtÙ47œD9A3
+xfÊT7aDUl
+fÄÿ·šJy!¡d±86÷YP¹{qY
+yA+Tü±AÂ§PR¯\sKÆœÈÓ'Ñáét>òIÂ
+.zFí58@øc=ö¬î_s+CL!TFC "a£Fpô9l¡&æD~®R?ä¯5 IHŽT× C
+(BA KŸñä<É¬MNÂâÁePD#ÃY­Éàž
+gÉ>A5däáCõc,Ú4*Ýmiã\ `Åo4\èL÷wF"Cq
+=H[·=@
+_U0ðÈÃÈ¡3WG£Á|ÄêQ@eÄ2}
+"\åHŠ
+Ø¥¡šI¢#Cä© O3$¡$Ê#ê(ýQâ!&kãDÑ2ßEÊæQ>¢
+BIRô"ÕÖhÔ+
+$®ATªMZÎ¢ñ4J*Híp-!%~ 5
+^ ¹DRÕ9iqì\!Ê3!'ÐäÄEŽd%
+IxZ6ÃY90heQ³
+®
+a(TgxÎÊ%È)%P
+&üEhbpæúm#hQt¡fî0FGàZ2ÃMjäÖE±7šiÍÆè2ÊM'A08&ò"úÚ0Ì
+¢[08²Q}bvqK"MtEQB
+öAv× akHRÅ@EBaxä*É`éjUQ]%
+Tà:Î£D)ìHc#ïÈ#üA
+2ÁNr
+Ì±@58 è1w@Fyå1/þÄ C±(@ÛØ!ÂÜgSBíòKn3Ïä35	}¡ŒeçÕE'äögpRW9v°KDÈi$ILÔ£¡{
+@C6ŽÅ-^wMVB¬ ¬Ü1³_Ô°2Ðs ²TºC
+1òËø~¯ñh2Ž#C@Aè]TUpí£2Üi
+4ß3yb(&N{DQí
+ÒämåÖ r
+;k%I¥'ŠVYz>v£¬&
+ÖwžÙñ`r$*Ö'xx3¹¥	Hq
+ó§ñ¬Ÿ
+^yXâ`MûERòàQ=	B0Dº%f	>Ôð«Xvs,Vfb$h¿Pa±~Ïsü!$þÃ[Ô_ HbÞ
+AB©°ÕàQÇå`Š'fDI 1Dtd.B¬)©|"Dà?jÊvc Â
+°I©Š+È±­	FèuUÀp@dÎ`ŸŠ
+!û­vNB¶iCÐÓaâyíCNiÁQD"%0<¹©ù
+?I¶Iq¹8¯	75®
+`ôdS(m*u\¡ÖlÒ=
+ LÞS!äÀÌÀÙ>ìvúà5ãn.àéP9	[0q
+ÂÖjôFÄØøo*¯€Cnvî&òêç@ fQáqê6ÙÆ)ä'¯°o>A¿jOÒé<š«	L6§ËYs>Ë3Ãs_øt/Gt.ÉÑÓé)Š-çað1ÇæØZ`#;'c
+WÀQÚ
+&)'R
+²
+òòTmÒB]
+cI$>YåphgŸCR`ê©ÐE&»\šNAÀÐ[u!z§AK8üOZ÷bW0
+aN²
+ž×)cÝ$·`yi%éåq°T(­vn4ñ!7ÛåYr
+Í]Ó!ÿ
+(1Ê"çE
+,BE1
+ä=0ò
+¬q_9;ÅßyzlQÙî
+:RBe¢ªâ6çiw¥]±|T\ |ú-Ë\ Ñž rõwÉ ¢p¥°aç%Ù°Ü|EÑW/øf€)guK8&©ùGÂÜ¥}ëß ³QWmeMØ#@€kÛ€vá+À2ÆŸØ_Š	I#B¶dÚÉÏ1¬È
+r-Øydm§1,
+8£Íè
+dÁíò('&Æ.'_ð2;CÜUÖ#ïÈ-jÊG°
+n}šhÐ62WNSÜÚ* Hoê3XbÔ#MnH">
+4mØ,SH>TðsõXô·Åøv'J5Òá,ßòcv¬Ô&±ð/Ù²E+c·¹(aÀÄÈs	mÄº7h@ÄQùä+X
+hcJ)ZA¥Qž£'r
+Ëeb`)ß§ižEg·F¯qÛ	¬œÙ!DXç2è4TGâô
+fpPp¢	ã2ãñoII!#íP¯ïõÄET "
+C2ÕapÆ'¬e%±T&
+©ÞSp»°ÁzŸÁ¡qTþ
+(ŒK7õ]a+ÿ}ÇzåwYZaGÍçRxbœG^×^b@àI¢íšg·ó¬r`dJÔ
+Í
+4°zòÒ
+
+?š÷#(UÍÞA(¥ådPu¶ K_)ur Ô`?ßä0O\Dt?Zd€ZT'L€Qú(g@LYàœà
+ÖplºÑQÏUÂØíb[€Ù0±qAÅ­ØP
+ÍÀõ€wùN2Ô0(I€3!rQZGš
+gú
+Â¥.'ŽesÈDWFešIiÚÁºÝ+ÆZòg~q
+?C.ÂX0
+ldY
+¢
+$'u
+ò	jœ/Äq§L<Ý%žeL^¬ÁoHnÂÄ60Éïbí]tcq#Òâš¡Y°	-O)Zö&ÇÒ¹p-î5Ra^á4£ÕQ£
+Õ
+Y
+Ðµ+îséœb+}%¶i 0
+ÚÍÎ#,8>UÌ!9oyÁæ§Ðý&j hÒüeVÇÂ@h"ÆÍQzš
+º{šãÜ3lxfãCE@øYLX}õÜW
+c×JD*
+	VÒÚ
+4²q&5£à)XrýÆ¥Kü[ÍÂ¡7ôÅàRj€Åæ€<b(µ1ñ ðXŒØAèYSÀ)ÒiA&&šS#`V1VaE3ÊÎ2²ønßBÁ+ ìÙ1ÕÐÆÕ×Õ'é%ÊÃ
+oG0_
+})Í­³<Š
+nDD®LVëŠ<Y
+Æ[¬ŠEx÷æýš$ÉwõËü`ÌçLÌD)Ÿ4E1bsÐö,
+úæõ}ÂyÖDŒV'H
+D°©Ì2ÄÌ¡yd#³[%
+S&FÜ `ùÔki5Í5>lg lQouìâ%+ýmÝÍa,Æ9#ødûâ#h[AEœàz2Q;¢»œ!{`cznÂ§
+×ÐS!@ÕÝ±:VöÔd^X$GXX`SéP¢Yæj"ºSrX²Ë¬Â8éYvâ9ÐU§^¡
+étWÏ`ÊÂP¶Å­P5zé:ò!°ŠÀmÊKjÈÿŽ$NÀ¹!w¢Ð­¹6²šŒÂ^w6D
+;F
+m
+
+§ŸC·ÚÙÄøü'bIwXIš{$vtù95õòGÞš,0{m#Â©8xÉî
+Egµ3ÈÒ5ØH¯¯97
+¹°éFZñKu7ÌYø$-eï
+óaÛÛe
+µ°õ>?V@S4Péké©šG&çú
+~ŠIxÐ°ãgógâÖçuJÍX7ÄkÖJbÑQ³(ÜJÕÊ|BS+`$¡	aÉ5¬ Šø!-D;Áð
+
+³³žt6Åë
+òFÔ##UUwGñ1i
+GÕÍã¬){[UF5«¡Á³¬¶q¬ fSr~
+ÅK
+ÑÍª«
+ÏXn9"a_1U
+ CLp¶±~G®Õ)
+q#ÍÒG"C×Z EnÎ¯¬ïÛ	E*Ìùõ4M×ñ®6]¶J%õDRÓRzIeß[£7)q"À	hàŽ
+WÆ
+BK9*4L%ó
+
++#=\}zŽ%-GïÐ$
+c! \ádbOQ²ó<%JÀi*LáRÀr2då&êöu!À]59k\¥ãùÊyŒË135$Œf
+ÁÍaÊêå:§ªO@1¥Ìàê"ÔÖY4(YÅ-ô^U£ÄD-XBN€ÔÔrÆJ&ÑøA·1(ã¡Ýr*?#
+
+ÍÇ0Í#VÎ2j56ºhÔžÒ©nÊ
+
+byj{êxi
+SO#MöIñ$Æ#ØécËÇ`Ï`§
+þØ@.ÃG`ŽÏI:!ê a|J'\ñ
+jÂ{4GbÈèaïà;DÀíèµ¢Ò(*$x`99HRtwÌr²T£ŸA8)ŽM0Ÿ
+m¬W8uh
+³Öw	Å³C
+BËò2\<åK
+Ð¢
+ciŽ)ñŠKé
+sÊÒ&*X±ÁÊjÇ-Mmª/
+5:©£{N[Ü|ã:Øg4FÓ
+ÕŸÕ
+;
+u©7î ¶*.\EmÓe`ø0+Ý8°æŸ7Hš;.qZNŸL6
+I68 íÌð6gcnkæ`ú¯3SQrnîåN*AônnX"×-*)AM`AeÇÍ3ÿ,ëæªq@iPIU8¢hz8Êe×Ôb0íÈþ0+î(óÙPF(2?²
+ŠÑÜ²ÌÁ0pLŸÁšñLÇZüCËÁlÙQð1Y[øçÙIŽÝìm9­ßlqÆ®U¥ÒcA~*
+cd,Ch7xØÑŠ[«Ñ}Zâ"Y#%×ÐÉpj
+
+É(°WœÍ2ŽcšúÌ8
+rŽ íé 
+,.<	oÒUf	(í ]nÍ
+8@xæ@I
+
+ËLÇP°ÛÓVÝüSEu
+lÂóª±2pžËm¢y<N_Ù5
+Ç
+Þ€,Í WÀëætsè
+¶Ê¬1/UwšR©ªömH
+ìY€?áà4µu,Õ~Ø
+NÂ3ÓŸi&FrQÈ±eR]böeLR©HiçdY@¡L«$/Ç
+O*DÂb>íÚç$DÄ©qÃ 9 gótüÑŒM£2š7·sÁç'æÈ²UAÅ+!·
+_Xk]Ñ	7ÊmÓî'I/D]0oú
+ŸÐ¥kYoPoãŠó¡^}oí­p#à)]¿'&í5§ÊQ
+ú×V¶-ÕæûhQ-lZy© YAG',zAßû9bwÓÓÔ
+3Ešé\à!­WyR4M!·üHžãmâz¬¬T_GX,
+¬J¬UVŠC¹tÜ
+1ã¹\í©ëÄ£]Ä5µy;_°Ù
+"GÅ©7(
+Ëç.œ|]àLÉ6(b
+¡B5P)Ól
+¹çÞ¶@0n|A.bI"KH)n`Àl*"¢³Ñål4NÒ[°XoæÂ[Š±pQâÆÜ¡h5gÄL¬€3UJ%Y
+_lh[º4ºð51çæF{lAàÅ88E·Àµ&'Cý
+	ÌŒf¬¢^³#e
+åÙ/
+êùb éŠ	«µ¬n[n¬(u¹
+Ò @mŠžzApë-êEÎÃZÒ¡Ó=y£
+qñVÉŠi2íZ2³3Â~
+¬6}Q/Ùèr`.7y1iŸÂ^Uhš7wàÀÓ/ª¬¥
+Øu»2Óºô!ñFÒ1§iPiuRénñî³jjvhb«s
+Ó¹ÒV
+Ûàú[OÈ@Öø°¡\>|2ÖÅÑæYþÅ(RŽÄ<~×5£Þ
+¥_Ù¡Né­Öå&juEMó4AR!SëMá»Á}Ñ©¬µb¡úa.ñRÿ
+ìÆÔy;ÚîVMi&ž£EÏè°wÑªÊn\
+Hóñ;UQÕÐŽ
+i&É\`!eÙÔb$Öì(©²]ÃÑoöAç#µ .ØÀ?ŸC×iý±Z	5*#CDÆ~z
+1uøýZù«¡Àµ
+`§-
+7Ýrj&¶
+YLÅÇËsDD:J¹
+ÙèYJ
+ŸYøq­ÍÏÇ¢Óc{æ{ç.
+bÄ8m)gë\>®Cê³\Z²ŽT£øB±Å[·. )ìSSEIäÝø În
+Î¡|ÈmjÁÓ
+Öß}(1¡Ã^P¢¶H%TÛHÒEý&(>rFMÚÀbr\×NQrŽ"ÙÐëkQv
+
+Vª³ÎŒ×YTÒ'g<1¥úc£©b|
+X©âZ×ŒDÆþ¢t{%eðJÑ§Ò	ð§p!ŒL	€çõ*þï(cfÀh/{ïZ×¢êÇ'!h|Êë3Õ(d[ûÇNÑ
+
+O
+I`uû
+8ó8Öå.)&ðâ"¿&ãæú0Nxoä	 U5çTµðé3(µ¡(Çg4@
+ú	ý("$]à©D"lÐÂÖªC
+÷)¬eé
+ì±ÖB.ò$¶þîIÏ
+ Q
+ÌhQ§ú0
+z
+üû\
+ü'jqÈÆ²çºt¢
+¹,HÃ%Ö
+}ùjÑº
+ë0Ä»±ÚÕÔ°¢*Èè~rt
+Êq&î¥CÌlGV:1hbßèO°ßŸ4VÝ¿UüúÕ-p(Y·h
+?ð!á-ÂÆWŽR¶T
+HÎð!šC§tÒ
+í'ä
+dó@cS)A¹PjkÇækÓ®/Úôté
+µÂ
+Á/šJLBÝë[5×À¶ïP°ÎÚ°\TéÙlõ£:¿LÝùÅ[8!
+²ævAÄÉÞ2z
+ÊÂ&àÔè·,^|ÛŸµ¯þ&eô(j6ø=D@òÅhÞWVF¬ËL5ÌSà­1d¢¡}c¬Cç%ŒnÈ:Áìpyß
+€ýº1à}E
+ÉÀª
+®4ñž%°H÷+cô'Ù§²À=ÿF¬æJ¶¥þÊÀÑD>?@;2ëÒ·*Þ ðÊMu^ð gL<ÖÃíÈRÌŒÚÞ,êšH;b»¬phýyYÊOA;âò,ZÁWìÌ}MÚ
+}¿ÕÜB.9L
+±0
+RÒßrIÖÒÚ ?KXšÞdA¢¢
+²áVÌHÅ¯
+L8jÜfÉ"àEŠ
+MµöÄ,íÈ)b ¯ëÔZas
+|qÄW~q±Ù·EÁDü
+Za
+ãh)Àø»
+9aÓ{ªTŒDD®ÍuQR7zu×­ÁmìŒÃ'~òZP%éÚ0lc'Ž!@T
+úüÆÙœ¡wŠâ+èÓšÉ
+PÚ0®Â¥=Ï^+fe^ëé&YTÜSNp,Ñáµ­Á§CkÉ=R¬2lêÐæ$&œ	ÅÐOª¥R[Ã^IŠ«0åz{lR
+DÛ?YP,ÊzÙG"
+óc PiîRíF5Y¹PòÇ:Æ_»Tl:o>ó³6B$cÞ1vU%o=Äñt\jB±YC©oG_|ªvþÈö·(ÈHŽ),ß
+ûJA~±ŒÝ(MrH
+r9wSz0ñ%ßÐS²øcKÙš_€SèÆ«J·Ö6+geCŸ 7µŽÉj]WÞ4^tYqŽúþ5Z¢?RÊ¡ó
+$>qx«ú
+º»ã/±íí§ ° Ý­W¶AX`Ó1pÅ>jkùßap
+b]7ïùbpT(ÏITVT£õB;B
+nÃÌZ6hÅFb:8&H
+Ì .ÅÐþ^Ú 
+äPŸXBþŒÝPóÚÎzË²~Ÿ>k¬É&
+I3â(%uðv(9Yœm;¿
+Y¡W=Q0«¡*(.ëõÓZcÔÈf4
+®p
+ÐèEâSŽN¿8¬!yÍyA 4~-À°ÂtxäqT@\ lÞ·ÀiÊL1ð2€gÜ
+Ði
+êS+ß0¹æxr%ú0llV.FX
+2V")è=ÈÎ
+€­q)Zþ À3è)rÁ6Ò¿bÀ³ŠR&Â­
+_ÄI>7üŠyèÇj
+ØÃÈdê0EÄ÷$¶!`k»Þ+0ÖD^žxÝ§Ç«v
+Q[SAE\`v¯hBù4@PµÊ[°0*;Ã
+4CÌuJñFÀÎ¥ÂïDæÂËç1ÑäcŽqXgî$
+Ž|/ß¢0"&œÈ9lFx»xCÁÜ@b°cìãeÀ;k ÌÃE_ 
+,2ôî,Ë
+òÜœùpë4Â4ÐÂVq±5$ß/uÅsneU¥`«æÊÀå/
+ô(hN³yçýJ]¬ð)Ò¿°aa3­0¬â»Ú
+1¹ê7Ç@QÍ×
+QÎ@EP3°Wg&]5¯KN/ß*ÐY1mTë{¬>²D
+·KeâGŠ$ËÁ4na¿dFËc¬µcÚj×ÇjjãdÁHš±Ð¿
+ÑP4JÔ5£RÔ`þYGºZÓÍÜ7/Fu¹ÏB¬lÚXZeÂÏÊã®oëK¢©Uþ*Ü¬ÙªÛ	#+Í!y±EBI¡sÕÓKõEè©&G¢µ[ý)žØTçý
+
+)È 1àCJ2RŸjqÔè¡Çb
+úÐÂ*Îüq³¬Ã FaII
+
+ÏH8B
+°äI¹œNÞÃeþsDßq/ìBæ8ØÐ,7OÇÃT!T0
+ÏACë«¹
+^.9æ4ÍB[OâôÚ
+Ófâ±AÔILn¯§ðwêÇö¬qòæP&<ïx$$Î16µ	h¯N}C6ŽBª¬/CBeÍî&ü¬LËªåU
+%Ö.T0-X8¢NµqŠcçÓ¢
+Ö4eÚbx!þéÒ_AÐOþ¡­J¹È©niøl]VÁ€d¥_=
+ÆâÃÖèÔÕ5Nñ"Þv
+wIûO0ÌZ"Y ð®Qì0{zLlÙEêÂV3,
+µ,}ãÍe£
+ÿf)UG
+ Wt¹0É-fQo\
+0o«XaPuF*;îðrš2a,o&€ß¢l­®µá]4SÏ*
+;m·,qUÃ[Ëá?dqL¯KÖÔ>D=¬U&.¶KVÚL©Ÿâ²@ótÚDœú|Í'Ýâ(äªµ©¥NTšRvÔ²LGBèAïÊ.ÈÇÄ²ì5U1Ugï
+»R·ïLí `ËõBy¯ŸdR
+ùÈûè=Üð®É	_3(BQNv¡M
+q*eÀW/Ôóÿ!ŸÒTF)¥¯&}äÞ¶¯y»Ïë{.
+ M	%AZ ¯*z9ŸŽ«œ°Ø°2ò#Ž
+^ 'Óïïæ
+|9E_£
+šæE>Wu
+"vÎVÑ.€
+#Á4 stÁQj
+wt§æ+GšP§
+/êäµ(ä|,5±Ì6
+7g|9ÎXè5¥/¡¡í¥eÛ
+Ñª€z¹uÃ¢dßÜô9dGÙ°^.E*Â
+ ]ö/ú»HRð}Æžs]Ú	"`;]77rõJŽÈX7ZJtÔœìÌ<ÖÓyK»Put
+Q
+So×f
+Çs³3ÕE{Mysµ1Q
+ÈrLI>:
+Þ@o€`ÃiLÈÛô&
+ª»¡.':"÷(sF»mÏ&§¥º9ûÊÉê ÄàØ I)
+µ+þ:XZ~oLN!éDyô]mÆÛÍ[
+¥ÈÞÌ7_Ðåòº·527MK©qšO I®éÌ§xPÙ	0mv°ÛÑWùµOÒ*nÚa/<µ3¡*kÒŽ;ZÔkmÚÃKÝ°GÁÎÞé]î tì%« òI±2Y09®Êõb
+ª"€jB\ÒÂ%ÊbºZºÚX±I²Õ¥Eââ,Ò7¯«ÊZR
+7èŽ#¯HÛ8Å<œÔ¢é¢®nLAãv-Þ"ý£æ2(ÀXÞ.ÑÅ²
+£4ÉyR'j$Ž.Ð?µsùà%>×cºµÓ¡ü
+ÒLc?a@â`FyŽèëX
+êËåB?äÆœ£hŠ$kXP²£ŸeçI{¶16Y*K7³ÔÊòAà^)ãÝÚzæ$óÖre0õMeÜÓ²ŒfâKÅýôŸ³-õ5(F]LwÌ*Õ
+6óè<%+ÕA#¹UÏÊ¬11Cž*hóHþË€
+s
+¥ËQð
+³ÉÇÉF:\CÂýjÀÉË>æÅùl&UÒ¬*q*ë¯)êBq0J'ÑÈLF¡ËqÆËp!%òI±¬Hép
+¬K~£,ûÎÇ7Ý¹5
+«èkXñ43Uú§ºSß=þ§êJð;1*à-Dà]R'²ûjòSPu£
+}·*
+!èÓY@Â²^n)
+WAIqnPÄFNU)
+1(YE4Þ·Õ-È,
+64*ñ«®.'ckÙûÉß a	ÚL²¹à<>Fáw2èE°Pø1Cz?Þ¬è}¢pŠh_mIHÿ€Ç×.1Á
+@ŒÅ2Ö
+ž1G
+p»š"þìµ
+ | Ädd?o¡a€YnåyBöpM:¬|ÌêÇiGVè¿YÖ7Žª_Œø#ð×5KLtpH,âg1¥
+ß§7å|§
+Ê
+ºøK	ëêU¯šÅ÷Ðª ÉL °0,;oZi¡ÔÈÄbûê8+CF
+Ló(qæzÂûZévjÇ·å
+²A¹Wyh^
+SWÒú\ª]¿ÊÚwO+ÈµÁ÷M«eD
+Óîw'ø{38q75ÝÍaÅÈÎ*B0¡PresJFŒ >>_^³Â`Ç`ýøò9ySÝ>øçØ,Ia'Š ]rË XFÿ·]PÍ
+(š·-daÑÆlÁJt Á¹ö0%?ÃÇö¡[êÐh.ø·G>**-â@P0v»èzÏû%(¡Ø
+Én"Þ×þ
+v×á4-íàª^Ë w8"¥¯
+×unsâNLÒyÐ²ä^ÄËZBßáâEïV&è;g.ù÷àªwÅ°Ñ
+œuIè
+LÂ·§@|3k(ÓÔêªiZ@Ö±¥kÄñÉsTßº5ÄYhuPÝz	£
+ËÀñv?{·OÃÝ€X ÿÑ"óµG
+óô©rÙú}ëßShGdqYÉ0ÇÎ
+%à*S¹8'ß]5"L{lkfÒåRXÛýó;ŽÙ03Ü;8ÕxiÂBÎhyÚ ËYV9FYôÛÎmŒ¶Ugói
+SžÂõ¢ŒÖE2.GãRé·WÆgn	úH!µó%
+©	'fÁö)ÎaÛê"ã ïäÓÓÓDš5UÜµØKq(þ>£B9
+®zûmßñU|R	 1šg}vdÕ0ñ,gC²Ð 
+*ùP6$£úISFÜAÍÇfYÒ­ñ©w¯|c4©LØâ&î^C¥6H®Ñ##éÄŽ$ÄD
+5pÍ&VçS\ªDF*rrÈÞ|Ø]o¿œ Y_wUÛd
+ŒÛMYžµº
+ÈÂg®}yZs/àÙ(!¢6ÀåÄ­âßáÌ7$
+L	âÁx1Ðâ{n^ÕÀO_þ-^ÙØK7Ôä
+M
+ôÐž7ÌÄ4!Þ{,Õð7/e>;Þ³žwãdB­<
+!OÀ ð?(¡AÀTàA¯UÀ WHj¬Ñs-ÂLäÓdMg©d
+b4á9V±Ò(Žvéïm<u
+fp2Ç/Áð÷ðF@A0H3ü4,mÇâjù_­ã·àäÉaI
+ªYCÐÎÙÁ2Ùè5^^)sÏ
+Zª{H
+¢£2Š­[¢ÝTà¡d
+±"»à©ùÅ@°n	0€ºœ²kµAx)ß^w
+CºMëø	ED"=tGUœ5aéàÜYHRæ>>ÞfaÂÁPD1r·Ò±Œšhãd^Õú#TéAdLg]lãHåšPÎëú¥È9
+ÅÍq{bº{ùÙÄ£k>:BQ.	3v2QìAdb×pËPD8un.¹
+ÀÚŸ	'¡ršhE&ìähé!k`»Öçæ/Y3šæL ŒÜÛg	ã
+
+ÀŠI)©
+§Ãâ³ë®JÅ8KÇÒ~$"
+4<í±%ÇŽFb@UPs8ÐRg¢;ÂhÕÈkpµRdAïEbè$
+Å
+ëVº18ç¯ÃmjPè:Ïú§v&¢:,.­Gw+Ã9Ö)D¬ÛüEð|ywÅà³òIq«¹º¢zZQ0a}
+e¬©5Htpøº@p¬··ñ-ôR®£-°·Ñ:þw(=4cå°
+[²©€W'äE(ÈBÚKµúÃSŽÒŸÏÅ3\ty×Ü`Á­Tf2(m|Pc;{í#(@Màpë ³6}öè+EÁþ¢ìi$Œ
+
+lCÐ±®të?ÅR|HÈËÅ±;ÊRÂxú43
+a¬ä-
++ ì[0vá¿Í#
+Ë7,}
+ÈËË.±
+£
+ua¥ì<Ö6Ò¬,-ôKø4ËkP³ÃNÆèIéÐÖeÕdzp
+ŠüµaqE7ñ0ëK@&M{é!0ÇËc
+È©ÄÝ×2Ì­Ånäí,[6;ô	Éái9|ŸÇ]×aPŽÄñþD	áÀ8¶X
+Å:3ÒÚãN/iãÅ
+ÞL
+>X-ãBgŸíxöå1z'¬%E*è,ãÅ¢_Å³^Ž<æsñÔ_tÝWø9®cè?f95õËâáATþt1xç*íw3/áÿÛ#Ùjßù`Ù· ¬Š[4À§Ö×wÉæÂ
+Ÿ;'ÀB°ì±ïÊ»~<«Àbmx\:ÓBnjAõ ùGþ
+No^³¿YÐ%#HŠYþM¡?kÔ
+îyòz»ðc]µ
+Ã¶c 
+èi;0Ô
+vxŠG
+J­ÄMîÏMxUn={\ÆuuSY)45ï]Šî¥Åñº÷ûåuçÙ
+Â×us<ÄEúŸ
+ÃR:ch|©Æ'-Y.ûÓÅ`ºZåg²PÝò«;em%U¹ÃÓµL4v·Ø5Ž_û!IñE»J¶ðg[jvçL*&
+>BZÆÃÂOñ\ºŽtÅ­÷_!
+³œmnÞa¹kS§\aù;Ãc; O
+a®
+\ºtu|¥-iÂT ÉÅœ»Ä
+k|óñ«·ÚPžÞ·ù=,{ó)F
+@©9¿F8I§FRÕö¢27bS$
+ÈLUEt¿uI,äÑ$ç¥ù)ZWœÆÎÁa?
+Ûwé×üâžçŒŸ3ÎÍŽNI9ñÆ»€=ËD¯N)J)ÛÔŠ«k]-&eì~3X<mk[J8ÿœ1Hk)4gHD%ºpk°È;ÞÂ_Ýç7áH·]Ûø¢Kr¬ALšE(Ù MæóÓ448ÔIM
+N27}
+ìZ8ã®G]ŽhoYd%#Ô_2`Öî|âÇ<ê£áòE°Í`Ë	:6M&Î\ììÆB¶{ŠoYs/Êž7Ø
+KžJ¢ÍJ'4b¬È³²ô
+Ö°96AèN€,Ž«7Ì€fsÇ E÷@­îcú%GÊD€P9· P&ö|B¢±kß
+©o£UÄ;üâÐÑò;~iqtÑ[c*5cÊËXàTàKµé¬rt­ùûÎ@Ä°v.Ï«=µó(¿çºVë6hq®õžy.AìXV<
+bi
+i
+œGKv7B9A&êJÅF1°,ËsÐXR`Ünïè·w×÷)ž1ZL
+A÷0N;® í¯U¿²)œ±ÆðB8³6¡_šš[lÉQ¯]iLÐÎ%ÁbÒuî]v*t;Yø²SákAwí€á{ø>oý§¬Ó
+/âN§oÏ`#Òž°¬yÃÌc>IKûŽ$ÖË
+ïûÜk¥Ú ÄõTÃú$³Ók[¬{k© =§t5m[yo;r­%Ã¹-wyªl0GaÒÐñ­<Å¢PÎR2ÑÎl[&¬þPê|¡ó\0>€?®.¢-*mÏ2OgÁÒžÈK|Wyx©Å
+vÁæÏ£yÄáØBçèðÎŠž
+_?$¥¶OG>¢ÎPê"á\Œ"Âf·gá ¶i ÷Þ*Ù<Rq£TbšÝfà`}ùclæ1
+ÁtžÞÛi·{sfï
+ñÂ7 
+ñ
+LÂœäŠðoé¶Ì'€àHúDïÅV©àcG®ÐÏñ¬HjÐôªm4Fqmµf^PÎVNUæ~âœòVC¯ø«õ¥Á\4€ù!YúR¢ìä°
+ÂæÇÒ7D]7x46Ÿ|ËÀDµÊÙORŽøY<-%<FmÑÅö
+Z¯/Ã#ÂªïÍÓÀŸ€T.,€L$I
+?9§Éñ	Ð#QP
+;àÒ¡>Šµ÷f]
+ùAÄÒ&7Çøw±I~Ï~¿cræaV
+Ênœãï¡$šå>`ñ!üß6RQ
+¹æÈøÓOŽ
+@¯hr}ÕlBìxCôJ²;ïfdóóÙÎœJÃ?Å§5Òó~õçÎ©ÉXRÓ"È|VÈCd^¥ýÁVHé³ð}òá*
+Ó~Ïm
+ylNÉ&Q(V&çm¿Äöš?Òç!á­LI9#¢£ Žkn<ý^ÏxG5sRÑºâãP6ßäüe+æïÊ²ÞÕ$sK0®
+cGŸyšïÙ7>¿ÛŒUá%GìþbŠ8øü?ò
+Z¢É¢
+V¹}gT`ÁÏÞ#õ¿Ï(m+,
+`þÒ×Ib€áñåÔçÆÔ¯4Äz
+üfnù£Ö²&·à\ÆôcGdvíÞãÂ
+ÏpðTGPWðŠËxÿ·Ä¿&ŒfÆ
+èãa4Q
+#}7va/°K:F6&¡§7 uÚ?Ö
+(ÁíÐá-CZZ­ÉŸSäiè(3²Ãå€T3oÐFu©éZGhÈ-$v
+
+E^orýcØFcÙ­UÁËÆÈÓ2 ú8^AŽÛÐçwôÓó¯~ü±÷ïÒûRÌÙ}åzè3rGÐâZé¯ÇÀ9÷¢ÓýöNáKßÏ`Õ.r@rm' ÜðúA
+¹SL2Xpà_â ÕÓî1ôÂÖÞÍÄ©"«=hà
+BI¿áww	ö
+B0jõ
+¯\ät±9
+mg6ùãWð:È]ú
+.
+K,
+×
+­tcê!î
+R_:C Aš3°ñž(²ïMÆF[D
+,ú&€?ÖZ­K:rIP×ð-À¶
+ÏuMø]¬£.0~C
+Ð
+µC<ÿÎß§À¡¯DÐJž
+
+Al[èhÆÇ¥	J­
+$áÅËGhSýû8U<Ä¶c >ËT5x
+É'økG¿«+ïõ
+2l^,£#úúó4Í$iœË8Çû@Øã7c&#"Z`yØÌo(Å.¡æ+H×ÆjÖVh!!ÍÉóî,š
+ðëÇ8	²-FOPÔÉHAAÒ,(ýÉÞkItP#.!7Úä#ò
+Â+QR²j})øÑÊxô³1X®?©q/²Sc¡ÔRI?nàfç3eÃ=ENÐ?Uz ±&¶F,ÀÂJN,2Fz5và
+wšcó@`#£K ÍHVIû&ùâ«ÄôQj{Ô.Œš+ÅO±"ë>(þìrÚ  -ã
+
+L%|RdpAÉ"åü£B`è*j&lAäÃ%%r+É
+
+i&¡n±¡ÓV$ôI¯eë=SbJÄ€BIÑÍ§ePÂ|5š71³ÊZßÁ@"2
+K"ÏS°]#ZfŒ¿ÐKÒ.äwkÔÄ}!ÁXÀ[]\÷e€÷g±æÉ¿tKê>Gþ)D}ãk!
+`$UBOÇø3¡"07
+p¡)JT
+÷gë¹·yçø$®q£¢%©}Bè2Ÿ¯-ï¡5Ù9ð@K,Ðo*&¶
+Ô[º÷uW
+=Ø"øè{wóaþôm¹MÈþr3ÛýÆ¡§t»W¬\ª`6EîE2åÐ}2AÒBð
+wC€Ñ¢6
+R;ªš
+Ô8«{äfs
+­ŽÕb. ,é€Lpm~­úA{êÁÔ¯ÈáPJ2
+à
+ÑÊ¹`Ži
+
+O,TVÃH,¬GX¬£ö¢JŽ³¡IþüRl.é
+
+0¡OòÌ±¹È=ùtÊâ7öµ50ïÂ
+
+Dîµ
+DHm­{ØSÐVúSc »:=Ý oM
+£MH±xsæP
+ÃeÈ¢ÈE^NÃ×Y$ÁqÓô+Ze 
+xåb%ô»u*ÂhêŽõK1
+sŽxTÄ?M GõU¬<SÈ%B±Žz	ÂŠÄjwX×aÈ×Cg|;8ÎLêGb`Ûšý!ií!Y­¥9°x°ß\
+]ÔV
+KÊøÑè	ŽG§Å4£5Tõ;Ò$äÑ²KaŽõ
+r4+OŸ"6Þao=Lza
+TëšQg8QÐ
+^*R/êEBlk{¹Âgá+'Çtý¡ø;X~ËH(_FÊ8 HMÉÂ€ABÁ
+«út$Ì¥Üà¥ßpÕFL	nF qÅiìÜW]è>H<¢ô c ðžAnÔK,í8é[kF'iÝ!À£G+Tè	|gCÐî
+_âÖ«jui.	K 4ô!y-61þÁÖCÅ!RòæšfkfÆ ¥ia ,Á|,<'ËdA!
+1àqØ `]%ñÀù
+C"u3÷â
+YRu¬ß€;.²V¹Âó¯õÃ€èJÐjÈ¶á8@6ô
+Ô~_C9ãGhàq×ÐÒÀAÐ®ŽqKw
+zú»ÌâCÚóhü7ê'.¶¯©*ØÉ÷
+Cb1ÝfÌÇ¹ ÔÜBì<4œÈÝª4â	ä!ñä7â
+0P$#š7ü¶ª0Ã~ÖÌPYèõp¹Ô@Û¬ÉáÕ-±8
+
+k!l!È2pòÝ°Z`¯ ŠÞ©çzwëâ³ÉÐR
++	^Æ"m¬³Êâ^Æ¹¿æKÛê7R.
+â
+¢;1<ÍdY'$
+86LÌÂµBîEŽá#4b:­
+(ØDJÓèò
+,`ç
+§<0æTEµµ\óa1@s4PÂÆY³é	Vî¢TÝ«Ùd\º^
+¯BNð^Z+höL
+N4®gåžðHÊG ÉEJ§EBtŸP,&&E
+4dÀFÄoëp­
+Iý§
+Ôì.Œ"f3HAÂvè(ª2Ð>Q³4ÁÊ |juæhoSÏ5ÃÂÎX;9@à!$ÅÁõô²`dh$~\á@»IU4Y
+ãi%þ)@ý )ªqq/9ÏŽ
+DÂcmôNAë!_
+{xWê)PŠÔÜ
+ñ"43iŒïp>HÂ¶÷ål~bo@æh-T8lïé7Žz5íG"
+'ÁÂ¢uûè³š
+¹n$ZËV+  \áOL5`
+Ä¢ÚG#r7×»ž`eN­¶
+sm±êHrñ¡rër>ÀDyÝãÄAôÈ7e
+YÖ.qV@ÔXÌqÃpp,ÛÐ
+¬U	häá*
+œþ^
+kÕNÂaQÈP@Õ²ÄGä) Ä÷
+Z!;TGÎØ3"ÊžAZãøØÚŽiÊ*ÞµæBÓ8,ïDi
+°HÿÔ»B
+ðÔ
+áèÞDÐÙ
+$WÎa¢qD¶øS,xkÑO2{C*ÄÃÄ·(÷E$
+&Úv<1(Oîû)ð5Ë-ç9L#I9ì 
+ÃÙ'œú8
+Ð
+
+VV
+ñÉXùXÖeýAÎHÃlø¢HŸ%yáÆbù'È%Mœz
+3OQDõÕÒÜz@6(ÔGÈÆÊ
+%žÌ¯Ôn.e8ð%*Ú"2ÍÄ5ç0J*Û¹=*zt
+©E±xæª%¹ADf·
+ú©È\j#
+Â
+íÃ·¡ÁFŽz_xÏÓ-°aÀÔÓ=gG
+É`Õ³,H1i=×_îv»Ò.Ð
+uf6Å¬aW±V
+â­
+õE\*k©À×VYtæ
+5©§
+2¯h@ôùT¢]Ÿ!ãPFÃGX
+®|ÐxþSáføp|¢DŸWàìv 7ªÜž°p[Óð
+T5_TwB.#¿ÐR ãs+^Qv
+
+@Q>€'l5
+Ù
+Ä'T(X;è%t«ŠÎLïÏVÓÍüÑç 
+1'ªîEÉJC
+Þ¶êú
+p:m
+é"l@ÂXð.Pæ:Y
+ãlŒšgðcü
+$¹6
+KâÆR[-2ØÜDcÍ}À<T<¹&](
+	ââ	€ÕTT2E
+_œZ
+*!XµbÁ/a+t|ÊCHÉÁùµ\<n)D94âŒTçSÇx(Ø>8TpN€Ëûÿ
+4nY@IóG~ÃAªheY%?g 3]lH
+ÛŒþÝéÙqÿ3èù.
+Ù"ÆÎ
+g¬aŒ)6lENoºÝI2H|:0æüÀ,aÙ%bgÀ¬É×l¯~×oØüNö¢ø hÁØäË
+¬©6qªPT|oGAGÙèµX.9	KB©B
+a°}25­5Î5åÕùøèOú
+ôž¢ñ`p¥rtõLØ=
+=Áy	XNÑ
+jD%}jÆRå»S0<ÜEãÖ°YP§ÑÅ·Æj1Î
+š0rŸD%îÌ
+ïÐ"»hEá<¹8éÏ|A,d©zµ]_èGØæÅ-ÈA9ÚF&K·âÃ
+ÀEHE:O±TLø­ÏÞ#zÅJI£XfG€ÜT#ÌÀï¬w5¡èYü=
+tÜŸqÑúÏDn:héf]Öè~ÄzCHDp
+ôÅü¢ÿÂÑ!û
+ÖÞ
+ã=ËIÁg2Í áP^­ÁwÅ"B!b<·±eCÛ
+	A¿
+R6XCîk
+
+ošÜÆÅñwT4D=	
+r"dÐÂsXN;ÑŸ§UHß fDg?££«#Qlžðàð¢-mNýaE
+t_P#GIapsøHq@ ÖŸ
+hÑxâAà#^
+ÙpÎSP­Jc(xtUæ·éE,t3:V=¶{VE
+C¡®üÉU@Bà
+P <ž16KŽ~"ÉÑÊ*W\
+ {
+ QŠi\?€c
+ Î	ò
+Z
+P3
+3ÇŠI)¥e@žf)$,"FwÈÏcBt4ŒGÈCa/5pøY#õ Z	u®V¿] 6:.^5~°W$dÂ
+y:§
+DÎ5+J¡Ö&Ù
+úLB_Õ
+)åéâ9=Ñðæ
+JxZÁÉú%2Åj'#än \Ì_»lÁsÆñ9°H`ÏÞAAQGdpØí,Rb-ÇdÞ£÷I@ Ñ¿
+£q@ñ_ì¶è<²<ö=b¡'µxÞi¯DãåÄ|'À	
+	ªcáŠf[2/.'Ðô%Z­ á
+|"r!7f[°Æ:IënÓ4\Á5ª²8TþËL¡ücÀm'FdKÂÂ  ÒwIhd
+,.
+£±Nî	ïCl¢è;PiÎòc1²7Ø	6wU"'<*=[æcµÏ
+ÃÅ66ð]-ÀV7íùgq³Ëò'ýZ6GnH]'æg
+û";ßcã±S ê=(#Ô¢à°ÿ·³³ÒÀ"þÓÄh«	ÁÌ21ÁÓÞþÏ£¡@œ<€1«ÜN.0KºÒNß³SªÉ AU,FãÃ=Ä685é6N+2Xåb¶ôÍ"K;ªmã@\à³ñeq nè
+ÄfÙ®C¢Šµcá¶ç	È/* *7	qÜR
+$èÕD%š8ï"öÖ¢w#+<#€EdZ\
+Xí ÎT-i²2"/v1E¬Ø.8ø"É:1`tw"#
+ù-r&L!L
+@d$3h0TÅ]¶b¯DaŸÃT[ìíb<Œ 
+
+ÊTâR~
+J ÆàÚñÈ7Ø{= öNå@ØfÌÙ œ2B l62árÈêÓðdgC·ÃšxÂß«LxŸ0g'âtàU	-^l¬;CXa4±íŒð*	/RHögíh
+Ä6¢(G+vC¹Ž
+! ÉWñõCr®Å!:a7eÂid¡
+ð
+4¯îÈ%FŸÌ"(
+ÁôŸ
+c"7Ihoõ8aà 
+I¹
+¿glÅeºÛ€%EreBHšš'ŒçF(ìÄ7á-Å4i6NýD%KÑa?qV'³zÍÎ*~
+æ÷D0ø¬nP>d_ê»ô
+ÌÉüoN
+10cLD&~
+ÇEóÒ%ÁNð\šªÕ6Î
+ÂÌ¥l;'rO
+c06X:äòúK%
+-
+ÒI-ÉPö|äÙ  Æž¿§Íp=\vË#ÛgµsÚ$iLsÚ2£àìÈâcÃhøF \^ÒÂÝrTžÄ-&,cybpë;~=â>Í€ãÒœk3|¢DëÈ¥\®ÀB¡
+( à_çX0Sae
+7
+ï#ï³PTïg
+ÅN?1P ÊY$¬>kV!Gwê
+¶,Â#J PWuÊ±=nÜi#ªSi!òCB¢ADà5à±×ÍëpèX0eÈ4T#%J&W7ÈZM.ÑŒ*°²
+$:)á\õ!ÿ¬,ö1bŽèOTè8»¡&ÜÛµGH8Ÿô>gPÓòfq[è$v>
+;à)€
+Y)6pe%d¬ÚHÍÛew§
+¢Pj¬¥óÏBÂD¹~UDðžÍÃHä"Îq¡8èF'ï3pèBàüµ(¥ÌÇD©;øV\ýB}E`8 £ì
+Œª>%µL
+¹
+RßPeùIÊ1t
+â	Š¹çGìJ
+5g€¹ÆÎL{H#Pb ïn)«3²BVñžCÖkµÙÚ29ÂöG³¬	^ž&Ñ­Àé¹­<¬}ãÈLËí:cåã'ÔmVüªåÙìf
+sDxb^J­å© 6÷Qx{n*6ÁÝ¢÷WXhé$|4šüõ'29é>åº D,®ôÚ ø~®!£Ã²+6XÙòÔ+K
+õ 9PB@ÝÈÐI
+9©Ë24PvsÆ\iñØøVGäj'YŠŒ$ÍâÜä¿q³Å€z£òÄ©êÌj`°
+Ä[[sí£.øÁ ä2F$K
+ï8.lk¶RºÃuÈ6VTCYH<ò²%(G©B­+Ók-³ÄË<ž3ùM
+zFJ»Í_£Ecm	Š
+ì©ŽÐ¡Y£«.JS,
+Jô#nžºó¶?ŒÇÞbåòþteäsÐ1~9öÁªI9!Ó0øèdÃÄþ!Êæ°º}07Š1â
+É
+7ÕX
+ô<d2ãßdØÖÆù¢ð9Ž·	sdré)p
+B]#[pïfTAU¢ìÊ³q€g€2añµå®g@£¥,"1SùJtªX(2|
+dJDù
+
+¥KÁ!ÑÙd)§/ Èäª¡
+!3rÅ>`ŸIáq$Ç£È}b]z¯4]
+óYc
+÷²MŸO¡ewšEàÚ@8rãcQäy(
+­Ô@à#ít3i)£÷?æû.¡è«Z÷³Qd²þ@<i	Ât
+äÓµxüµìì;Àeh¬äHÉûŽ4øYÇuðÒh&	0IvÀMñ§ðcCG<;Yòõu§üDö(K'	,h	Y<- F
+h$¯}?Œ²$
+ÈÀÓ§Ë¡Z
+wj €FPjU;ø±xo®%cÌóÊÈ¢xú8zà
+ªÝ²dY#hVMŒj+ÀSa¢±±Œ,BØ@
+ÄdÇÐn«%Ø_i0%)|
+{£ &Ð0zÓÄÁ#
+åü·Ã
+äôeJ
+?°¶Z°T
+Ã`DáÌe)$öÀÐd1Ð7ËÓð€N
+CJ^ ïE$óÞ%bVILC¯ñ­~ÃKŒE
+afÙ@\YSRvÀ9
+§òà÷Óeêœ¶pi+L3ç¡é9šKÎgg'ÇDŒ}å7n<c¯
+
+î©Ñn°ü¥1ó00@
+îÌHë¹èHõ8dz#ú¢bK
+ì+­9ôÆá
+LÈÅØWæ^<	
+_È *ÀŽk¬
+èA8°h Å«©»Ì2Ç@€KCáaÃÀ?_nºæ	!µÕ÷ËÀÔ]él($
++
+
+CÄ¹
+CÐÎLôùCD¶ºJ³ÇGyÜœã7Cif§GËãMì<ar%X¢
+òÐªZ(ÕqŠÇsn>/iè@
+Á Ç7RÆáMî,ÎÜ32<hÄ[æÍ"24QLÂa
+é[xf 4°µóV3o¬ãÀúå<Üön- )*ÊrÊ¢!ZdíüSÐñìÆÚFÙòYŽÜLšJË"ã%JÚöq
+&-^þÌÂoÄ!µiR5§:,Ó.«)Í
+Þ#Å¬ëÉ£R¡©sM.è¶ØK;FŒ°5çÝ&!ÚeÖw4
+?
+ "Ó€`ü?d
+r@_Ï!(hÁÐkŽÎ§Ä
+eBF¹ª¹¥xæICd
+ïK*@*a-×@ÂE
+*T!ÖQXNëRŒC
+
+8ZzCfßfé"$
+µ57#äþHÿqÁTe}ñ¬°&Ç 6gsÉÏPADÚ
+ÉŠ%!Gp0lÎÁÚîÉŠ5¡	7#rvir£œ€y×1ªŽPªØÝðDûYíd	'j¿A°Ë`a°ïP0ÊçBUs8Qü°$À[JF(	 9=\Wt¥€ŒïC¥T[L gÄMß*4£Ž€D]
+S'ßEò¥o Â£*èÿJS¿ÏÚÅ~I\Ä#Ð
+ÔF"[¡Ñd\>4"C¶¡(L)Q%Má
+²Þëhª5Èg€oCL'îÐom*8¬ñÍM²c
+l>^Ñ+ÙÂXb[7x,2 S*É°¬3\Cä
+ŠÔ@Ì
+áÎ`/;"`ûåÃsæg÷;Ñ"ÂbÑ/0ö2F·'[LáæŠ)E`h44{äãÂÔ ò9RB!Ñûf~ùîÑ³<
+ bÂéyüòiæÞ@d
+ÈÃž|S/«+g¡PÇ44Ð)é«M S°$6
+O§INâ6 <
+­`ÄªÛEc"µÎš%àKNd$€@N(Ž:Û
+ËÍs[¹ÓžÔ
+Ô|S&_¡V"(hÝò$ø%Hçp¡
+°ÿDÛq
+ŠÃS +Ÿ$ÂZ 
+Öwµøh4µ
+Æ©€9. f
+ZµÅÊ,)H%ÎO§Ã¹-Dú{o
+
+Ú®%¹ŸzqøÀEÔA^dqÉba
+øyè&R^IÚG
+Lïo8Ä:5ë²É=[D?$ZÊrhGÒ
+n«Àà è¬ìýRÆàOS
+âwiKãŒ¯"_»ÑttbÁ2Öé*ŽÎMàÊŒåá¹JÜ€<l1Î7¯@u·Ú¬$|ÛhJmœjÄ¶tû»7<ÂpÞð0Õšåèà9ÒDµÑõSä
+rzOþÑS÷h[äs6oõb
+wh8
+~4&`=9UÄñOaòû,
+?sñ}?aó÷
+»l|
+"å¢Ü'Ó ³Ìrø1ÒPŠª×gÒn4 °æ¬V°&5RîWÊPJÙà;0Š|Ïâ£0DOžuŒ'ÊE
+ÏÔ!Q<(ÝL$ÆÑhtÒ/R¥¡Î Fä2=ìÍ®]ÓêÛðµ
+IŸíî<*ŽÚØgôOV±
+¡öŸ°ÈDÃsQhDd |ügbà¶JÜÇH<Pz	Tæ/àír/@§A=â£:ªã9\]3òÝMufYØÀÜkÄÙHfyMqW=&Uëlm
+ÿsŒ4)¥À5YëüµdŠÌUóÊµFÄ]dI¢V[Î[&Ûy
+ü¬nb»º1¯åo¢w%ºÏ±.£èBiòéóèj¥¥éË?M>¹râÊo(Ýö²à©õðtœÇ|0 ßPuÅïMÕOÐåhÑô;
+Ä3×uš¢žŒ©_ ýIZü§ jeWßš'^
+ywìÂË1¢!$¶=LNKÈ¥*0
+
+­ÔB§
+"=ÏÙ©šÆT­Ðº0P®äæ
+¿B
+ÌBQµÐ7Â-Ôú4!·Ðf¡ÎZhž!
+ÿd
+¡Ú ç24øe<ªj¡<öÅ6NLL@ùxæ(e¡@,œ²¶Àþ>¡Ë^èýçÐZýÊÜP7S@GØpÑ
+&9y
+ŠX\þÓrd
+ÙõØ0(AÑ|VÊàH¬^%,°PIÌàxsD/y¹ŠGzNàùZZRCÐsK\À[§TÓnŠ&
+â%(RÏH9Q9ÁO
+Å3myþhçyð*}\%JãWIÎ@Mê&ÈXÎ"k­ÃBŠ3(BUqºuïµ
+ÿóÅàöý£ÞnÇTÊd±Y€YÜhÕ
+×@d©lª/çYÛIwÈJûÐ{}Iä;
+d5áiQ
+î¢bF
+A'oÁÊ7L1pb@
+ÄËúãÿÔhJ¶þL@O
+Ào3¡ª"H,
+{QûIy=B0Åæ	*ÀÚ4"bHFìòh0ZþT 5Ê³<q@}Púß#
+pfeÄBÁhô>(
+(Ðíæ2`³sÅX,S·6°Å>ÅZ$<@ß
+<Îšþ)gÛ£w¬v
+Û²	Å)3?_ÐG[éz
+,@Ã7Gå§­Dæ%Ÿ	,ö¢ëÀdfo1äêrš×H1I>Wö`yf
+I\	·aÑÖ§l1åçðÏÄ£ŸªÒá¥œU°E³6ÅàÐÆØt?ô<=PæÂpUÌŠ@^¢	J0üI!X5}¿}NøÃVT2(üa
+3êFe$ÔIIê)q6
+ù¬+Ë+Ò±`¡Úö9çÈálHCKÀ.^J
+HŽF{ÀAEäôì¢2qÃsÀZÊþSÔügT1öÙcó;dØÉÆðUûÉ¯Lš.À&ÁtvÉXáø[¯°ë&ÎFànlºÕÙÂK³Ï
+ÐSCö"à6xð/hc.ÔÃzEÔ|O,]ànûI DCY
+ØºTÂÍs`XDùÌÒ28ÍPAúöéT-J€Ö#£Q	ã@XPbÔB«E¬ÃMJÂ°ìÔZFÙ#§%
+ÄÎ.€1«Â"
+
+ä±÷ìí5'.K.ÉÜóÉÔŸñtZ
+3PlÜ÷PžFzjàŽ
+ôÇ²$pÐí§¬.ÊyäÐÂ9_f}ã\Ó~>>vÓ2ãÓ2U<l%((¯ %aC@€-§ šmºf
+nžÄ
+@th0²Â$iËÒkTçœ¶qJNô&*XÌhðü!CÅc!Ž&Ï»`JñGÑ>LR*+œ;>Îæ4.ÿP£=]ãÁ¥ bùAáõ RôR1`gT HErð (åþ)Fô"-jj¢Œw€ßYªÚê:Èhòf}17HtA?ÑöàÜ
+I1€°¶Oå
+weq-€à2\Öb)ÂníÂ²A
+ÃObVqSx;MEKÓÑöÍFü4ä[H%Éxj`°ô
+*bÂiTQÊº¿ýL€ËôÇr!hê
+ç6mZ
+MÍÝñœžP
+RŠd¿qÇç1Ð07mddhºÄ(šC+Y9R
+²oÆÙ±#bÂlAÑhé^Ë4ùÂ/`äC8YÉ NÇ8TOÛ²èòg%úÞîÑ4ÊS;ÑDa çÍ£©
+S
+s-}\5Hk#€
+µ@9U|6Ôv¡b
+Ò
+®-·UdõRzÅ"Ù§èb:2o`Lœ¢©^ej³#õs`FÏgÄ± ð»ÈKÞ@ôís¥3o)h¬F(€#kê,PóúŒ
+íçú@PÈûr¿Èèî3;Êi8>Å1ú9tò6¢SN-g
+@ö ¿ŒMÝ!XBaÊÄòÎNÛÌÞÕ#ãèLÊ]§m×PX .¹ý.N
+2>Å
+¹s&hÄÀM(f×ž@TÈ+Z²z€ÎEcw»ð6uÆ}pv
+WÀ ÷=ñN0š
+Èã4*àùé¶b¡€C©g¬{
+âìq7I|È(á@JÀ¢rP5šÔ^ÁU:GðÌÂª¥NHïàEê{svüf¬bõ³¡1Ïl8Úl]'¡á§¥N6L¢BBC­ilÈÌ«ÅÌ`j
+šéE
+9ýIÌ[Œ«kÂz!Ëã
+á¢ ¯WŠÒÝÕrq÷4È²HÊ¬	Õš`ð§tj74UaÐ
+ª²	-ÊQ¹¬­E¥éq€¬4ý0ü)ÚÑ/¡= 7ýiXpS«àâ]bèu
+©z¡ ã3ÔÓ	Îªðq
+ÈíÂ
+§OAWìòõç«N¯Íh@Ðã#<%\ÖzÄ
+uÌØØ
+
+
+Ù>I9Ìòç0­Îù4%îctÇäa
+J4}íD.°
+;­¡îA#u±v*ML
+C
+-s(Ž*y
+
+Æb	
+±51
+Be;±»È¹IÀ1úT$>)0
+ç0knv|ãeàº5)ùøSÄâÇú€bYíºPV÷+§Å'ÀCEÌu¡ßEUÀWæ¿Úhºás!Ü%ÓÜó2;óvU-T&V èî>'Ò"(Gº¹Äšu J«
+#äÄS:fâÙPþš¯ª&Ègæ#ÖózŠÙorR¶*FáÇ¹­Wµ@Ù{'è²èã9XLôX>
+Û
+rX°ë(qÒ3ÆTÊ¥ñ_ð»0È'(¥3\Æ8ÀT]8
+`ÚOÉJJ,d& ?Á
+¬?ŽÔÐiÊ"£4q;~b
+š0ó&È
+ôx82+kÎzæÌU©$$	wŠšJ}
+«ÕGcx×ª'
+@mª¶ò8>eŽVÀ`IelÐkKHÈ~ôÒòÌ.³ñYî	%¡wÐ(
+içûl B»«°«]&PøFOÍa9C+­] r7>ž+~,(,®Zê41¡%~Lã¹[u©óQÎ°zñ®©ð
+ðœõKÕ3À`fõ)YDÜ€"¡ÐG,ÄÔ2ÿØ 9v:š>Ç¢qNáž}>Ô£eol³§F
+]X>i"¡¶ðXdš@N5^/G47|Áœe/mjæ:ó0 Äq,~û8èÕï*ö#ŒªS9PW;±Îp²Tš$:uR
+$®œlþ§áŠ[4;œÒéc&Qr)YÛ\7"«ÀÕê.L	ª	­€(g@$0š¶šÆ,+Uàc"gm|4 Ë`NïøhLº
+?E
+3Š1· z i. %
+èörxŠBg
+Ök``zÊÐŽ0õ°÷?Idº:ëJ1{OŸC7MQ"ñD÷ÄÊ
+
+ôèv²)pB®ÞÐØ[vE
+ÉÅÆÚŠ
+;²PŸè%ý²ãCq²)ÃªŠ
+qReãÛ\°r:ª!®Uäá¡°(
+ýküWõ9cß
+f5îÉåzöÔ°¹ÂHµ [2Þ.>PUèÁStzê¹¥^jwÔ0jÀé{RRyúGYfíš¯ÀÌ[üÌrŒ¢Œ¥n1WÕ=Ž
+DÁ¡D¡gÒ*1%ÁÃe7Ê	O
+iE9ñ(0*ÃrMéÚËõ°«y«Y#5àÔ÷Ôü=¯qekÆ9rXZ?*N<Ç®¡puçA\Å>Î{P3#OQäÑñ»f«xi¢óªÒ£^uŒ|Rà
+å|¿H.	°Ââ­Í_YÔ%
+)fpDÜ)¥ãaÐŽ
+d%Árª±UÇ¢ä|qÍ ³F66nV
+£>PÔu­u.	X
+ÊhÛ5²y»þ2d?³!³STæášE¬·l š=d.F'2QòšÎâûÞM5Î#äÇOØê¢¹c2)ÿöùÉq/U£ØYÚ¿²gWµÀrórG[ÕÚÀgïÄÖ )²
+œZIKØË~ØðóÊ4:8¬ÑKÔLjfà1-²53FgÒA»Àõ &â{tüÈ4×ÖÀõÃvv
+<_;€Ÿ·ÖA¹m}<äKV0[*¡yŸF_Õ6
+ÎÃe ûÊT]^
+a
+LžbIøÞZÍ)¥Qàäw5Šm#
+G<4Øñ"JºIÕôL€{ÄDã°Xz€ú]#
+íGµq­øxJ¿€qí€aWÏD°CåUç­
+Woe4\úØ£ORpªàdóäÚ,iuíûŒ:Ô
+^y!à©ÞW{]?
+ÌA"ÒRÚü3ê)pHOò	5ã¹žs4m}Ã<€ÊçŒÊ\mÒ)UÆ
+yõÄch2
+m®ŒÐ#ŠZí|&â%¿+%
+K&Ç°Ÿ(
+.ŸÏQAZÄ!Z!{ZÉ®,x]Z#©V¢AÄtöëCQCa6$\£Gg5#|Âv×T,º=NÅ[ÅÉÄÌ
+D«UÒ9B,pûÙM^-
+*óo!X4K¬B"P€²{è8,
+§@kšÔªãw_àb6°Ç@aQ'Šx³Kªj
+Ùoc
+÷CÕOæýºSØ>ÙGÚ/
+ÏØYœ@¯Ô`uÆJ4VF Ëxd+ÐØ{	ëVEÁæÁ§	næ¥ÊbØ<mµ;IG<¹Nú%°YêÐFß€»Ís°ø,d8£y#ÔÄóŒf\·bÞ&pY€<+BžÇOtÃ
+Á5hÀ2 ÉW%¹.åví ¬bÑ¿ LOtÏQ°ÎfVÜ9š\G">c¬ù­Å_Ï,ë
+Ò7iÎØ`äuÎ1SðÞÒßüEgýH AññaÅèW$Ñ(éRâB]î#Íç@OžüÑÇRî$\!:EGYŒ!p
+±ºÓ&T£ìå¯m&²b0ÙœÕ9Fæ0Hã4$çÃìqÔüÔ?+Sú°RVÉàÓpcÑ'Ä ®³ýî-}ãXj$ní^ðúÓ¥¶6ú
+³ë¡]
+
+)0vúv :Ç<á
+œTÕ+e
+)Q!¡NL	ÆŠAEQ¶ÈóæVèPv*ngoìyÌ4Âqnx^*Nv4k¯TñQ}
+1ÓÏböÐôÔ7ÈmÅ
+TÅß ÅM
+|7uç-Rð&rwŸ;ñ*JªLdc±z
+IZ
+WI*ØÃbÃô38à,T>
+/Ö
+öåiè
+xWïlÏDþ(oy2LjðDh8h±óXÌâ7Ÿ7(â£dbšúô­À­9È\Ñ(JŽÈö&]S.9º-i²Î]ßa°úÒÉjÚf`ŽÄÒ¡Í Dd
+Ì>õ­!òTfËðùŠ
+ÆY0BÊ
+%ÎP·#
+ÚÂÌ¬p$šêÂÈ¢Íiáòü)Š2j1:ŸíÖáâvîñ\6eQÅJBOÍ>ÜML	
+ØUçíá 
+m02uYr·àÑZnÔfÒYÚòÅny
+ž[º:oÈ:Èñàñßæ2x"óAÈK0n
+è2Ði¥n|þÔÏŸZr0@ìÖ1ÖQ§>ÚFa(=X*±£îÈ~,s¯²~ÙVÐÎH
+
+¥V%Ç
+³wjDÐJð9Àž­B"fÍ ÖŸq€sáÜ?k5/ù©t°Óp&W	:
+Þ$qi²sô£{á
+9¢
+Yb
+Òr
+8ÒAkè¶f%
+É]ÅéÌPÇ§?uJ(§tf9ÌKðdPüË1cCnšÉú
+:;+&ÁB5MÊÛÔ³§maTëšŸmH($õð%Q®®|ÞÌ²6îhÙXk[ô@6&
+©,&1lkZ0 1O
+dpÇ¹Ž!Á³L]3Ò L ¬HÒnB8ÍJªÞa4lz%A# ïsSî4×žm
+wršl&¢Y°E2çé€Œ@
+e©#
+[_v À6ñÔØšM
+µ0HÉÿó
+TW)ØUµUNSGµÌGÎm<ûìL£
+Ë¥Üm§`äÇA«xÊH~n>€Å¥`
+ñN-
+±)ÍÏ³6¡ÚµðmX)«9ÍÃýC Ühðò¡ÈÍ×à§PÖ°2Šg^-,M'Qµ-$$ìÖv7ôAÎ 764è¥<F Ì
+.ø[H h1ðAP*ZÃ¯ž¿­59
+ù(OÁ/À|
+wT³â<vÃ×Lòº*RÍ4
+ÿßv|Vš
+žóšUlNPJ>Ûë`
+Î@lzEÓQ+jüVw 4hÙK+óE
+Ñ/r
+ '
+çmpŒä)P%Ø¯ÿk5Ô2'JèÅ·
+
+KS!§b
+kß!O¡S~À«I¡Ú*¿°KfÃ:±
+ÒÈ
+ätF³M"hUHC·5akRÝ"]äÆÈÞžk<[N:ÞJc§e	P2àr. š>
+{ý/ªß>eé	§yËD0`ÉL7²i¢s žtàC£q!8"êÌ fC!îzùç@K@pX(@
+
+d@Â@®[îHtÀÞ1<Òv .$!©Ç
+ZÅDDž¥ÏŸ%
+·t«vÉ¡@; È.
+±ÛÌ)ñ!
+Q±pÑ)ÇÎÈì
+¡:mêØu4š
+ÙWÍ5Ÿ'.XAôÆñÿP
+iØÕ2_[
+ÑÌÁZÌ·MHß	3ËT!
+ã¢)Û:
+°Éj
+%
+.i
+;èÈ
+œ@áátVéŽtÐÑÃêÙGò,Cm3á/Àé¡¹Y>áj|®Èö{ø)N€KÃÌHÏ(àòâ?}f§PÈFNsòŒØ(8NPh1 	w/P=s*nŒ²^%l\4p\ÁC.;M-ìÓä5[øV!¢À¥éó$@áØ
+¢FØ*\E/ûE2G#\¢ûÀŠ#äz¶žWH®) ³ñ²Ú¢}
+
+On$@SÜåJú³
+F^4ëQy
+H%mùÚL%fÆÏ ,°-öõ	€£
+­²uóÚvB,w©
+[¥À$³mü\VÉ£ýCòš|õ¡Îœ
+ànÚ î¥Éï!F#èæîZ·ÈçÂ 
+¹é:I\XöG¡BËnIšm¬" ñ²¢+ÛÔ2$²ÓÌHÐ$
+éÌXã$ðÓÂ±NCþ3ÒJkÅtpQž	~aH·ÐÀaPºSq€ScŸÌcS8$Ý¢Q{`>óQA±íJ
+Hå	g2N¥
+2é&Q¯åÌp fÝ2-â%
+îQu;B*S&<±šZÄÚfç­Ë
+B(ç(3®@ªÓÖzæÉ%Ú
+Â¶j®
+òshiŸeL^MUUîw²1cH0.»¢]w[DU
+Gº
+~sú;
+DæÇñóçºuš!OONæYžÕÓà7š¶ÎP£c/;W­å Ý ?·*ÁÿÓånGx¢8ðÑr€h ¬r«
+¬³îçÂÓ·ýÚù2MYo	ué3ÍD
+¢+ï"ÞY)ê¿[Ébm
+}O
+	fÆ±­7/
+×\ÑN6woñ:õvÈk§§áÚ
+"$
+i1\ZoW)ç°FŒHe?×eâ¡¶Ë€áD€øèÈ@ÁUÏÂPJ©d¬Ú¡UŠ)µÈ%&lè!ÎÌÌdŒÛÎ4·"äImb(°
+lºÔÖë§;@©rKsJO×»[EÙ·²i)nZ*õ!«-÷¥²³e±öMÒ$UØg5ò`ºø
+xYÔM1%
+ë6[í$Ý
+Z¢j}šlJFú
+{Á
+jžªlÍg]`º ³;Óå2%¬"Þ§àÊ	ýXåw¿+ûÝ3LË;Ó`E©Ò&Ê<­±æ²ªófle"íËU×Ló(YÓÜ*)	œK>>BŠå9ZoûriÈ£Ö¶ñ(³hñÀ¥»AêEÌ
+Úž@£÷ÐZUiö&SÄRJrŠíŸC$Pjg·
+­z[u±;<5's3¿$µÇ:iüØy¯3><Iÿ¡@>R$>åpG*Æ$
+4Wè
+Æ¶!ŠbÁ±ªóìïêª±cn«EiCÝ¢ïÞYÅK¥š¿yJÚZûÏ7	Ji3ÑZ2$ÜV²tXMìbXÛÅ]í:
+àÕìoÃù
+)SruKN©
+îÖÓŸD£óöÑxg¡î#6fÑ0e<8çjmX
+êzeiÕööŽ§+°
+øi
+Ò|¬^2:°_W'oW@Ks
+*Àµy²ÆpwcE&€emZžY`ßž}è
+[ä×±Ž Úø;zjúO$K
+#Þk1aÄ'
+ú'OM59ö{jüºê
+b š\ïYz]tºøÇ&·zëÎvueY¥ÅkðÓÅA«õ62Ð]SI°
+«_K­
+Øšd
+p_
+fÜ %s>å6ÊB·+ÊA
+AÍ=bfù
+¹ êË«EaøHÔì¬è§M^³üÈ¥
+QYMÌu¶yÇŒØñ¬©ÆçÇé$z
+qôm×_°èíòÇûlÁcÜ¶ÇS¡àÉÀ'²Pb-üeðv(¥ T¿ïgô€c`_ÞEÔB
+y°*ºÜz@ÄppI}HàFgqËŽkæ
+©äQužWü
+HhxNè m@ÿäÒå}	âÃ¡eBÁÏåÚzHfîúm®õì£ËL®]ôL	
+kA,ky4ú}€Q»à±6Ö#LC\bèUj!Ò³ïU³È%
+{+Iä€=Xö-N&±éd×G2÷zê-
+ªj+¿NÈnèôû$u&¬4,ØèÌH®4[ÊÍHÿ'W_$ã·üâ
+â8Ë
+Xoªôc\bà9ÎŽ¯õ^âý)Ql	Bé÷ìCXµL 0ØÉlf!Q#{9³hØÎDHÛßKý4
+
+j¿·V¶²E Ýž€qÐÆi +°ÈîTm	~
+4¥ŠÂÅ{YUK)S}RÑ_kïüqÁ()
+~«µ¬BE3ôêØTö¶'^1å_Ã@×SÚÉŠ¹' ñJæ>ÛCÙiUãy?Â×ôÍLÙ
+ »	?õºçtÊµñøqÌ@ÀŸ_¢^«æð_»îù}Ñ¿IkHÇD8Äë
+ÄÜ7/)`Ð€²àRð>x­·ßSÄ ÀŒÖÕÌQ³°|L<Z4îbêýŠÆ(8ÕÖ3BŠ 5©Ð]6A^VëfgL9Õ°f
+{Þwp)2Ô%ÊÉpSÑ!ìTÌÉ]pãkËž9Æ
+C
+Ï;ç2IÉXKKí
+ý`t1{J 2g[ðÊÁu&¢îÕºº
+€¥àsçlkŠÝºŒŒ#8£ÉöI@bøzÝ0*]$Ð çbP?Qk
+
+ZaA!y&pøê6ÇS
+ÆZÆ²|±	jvX(Á»Õà${÷1[¬¥B0/ÅÁÓYÈ]Y~ª>M^¶€üÌ)K1]*å@Ç¯u
+Ùb!Š(k6CY@ËtŸ4
+|Rëqf^È 5#±óJÃÃ(¬@®æU­ûÝfü	ÚÅ#P2d8¢2Ï$ÍDJ
+åÅÏ68pTJÜ$ÈÝ}oR
+Ï`fWÅ
+nØŽ
+œ0(êhèqŠíßÃÞb%wåZ(`1]KŒqYæ§ZÓÐm4	Qç
+2&R€1Ýs=îíoù·aÔ+s|ÉÀ¬³Ê©^â}rÃ	îÒÜ°¥È9è_õkÁ+Ô'ºž:TU9Lõ),ØÌ²J
+ÆLõ­ÛJXÈN7²ÍÃÄ9_Wð
+îBÔ÷xPèÙ(¢
+c<Ü¡(2
+ý ®DäV3¢dâYàßVåSo.áŽ_Š¬äË+»Î×à-_žpÖ@é6å¯p`Õœk¢;š0ÕéW!žöÆ£4Ï²Ž!B±­!r]ú"Ù¹øã³âìû}ŸŽÓX¡D1Ë,;=.aædBBI°ÚœØÌè)ø³ë°pŸd!ºféÂÑûjfTâÈ>¬>ÒOŽHÐÕ[Ø?Ÿ¿ä78Ù=.¡Vxcô5ú!KH°ÀþaCòÕÊg¶#¢j-XïÉ@øU[€`Cª:EWU(õócf9.pÑjx)ªGJ4Åä®còhÌýônÆŠ¢ãŒ²ÍZt©ò|œR+Sñ¢BŒ
+at5_ªŸîGÂöFx$Ì¶)ÞQóØ0.sz]%4óÏBqìž.ZkI3ÜPTnœ¢/Óç
+Z¶À:<<yÞJ¹HOL
+.[_
+¬ÙmŽB&öñŸC$n<ÐI!°4Õq]¹qCºV"QRÎéÒPa,Ïš	ù·K&u®F|j,:ë$|ŒšØ¥ÐrÐë
+ª³ÆÜ¶ÂËíeÖ	ø·æ¶2![öqMÚr
+ Î
+k·ª4.öY^v
+šž
+ÙHùÌ~
+:Áš`w¹m|x«Áû©xbR¶°žù£p$/€¡ßÇ *ÖOýø"
+|&Ä®žŒfHO!76	 ©Ð¥
+2#C­òì¥ÆM'øB0ùJ#Ú=0íx:k®
+§h}xŠ¿ÏïçÈSÈü°³Ör\éÕu\qXâ
+	tIºŠàeÅc
+ Ââ%
+äÑH~œä+U~WÑO1êU»æ,Žs0Ä`èÌòCâ~b L
+³£Æ íÊF¢'H=0
+ÑTŒQ{VJ`åQ#ÂùÆÙ}ìÖT	~cZåÃ×w,Š	Ó2-+P
+$èß©ô^.
+1À²)E|KbÓjÛh/ñj`üÅH;M4"K¥îØÍz0?
+0{ÔñëÁ;[ù^lÀ+	®§ìÎŸ,â8%#Q&¯ùLÛBÃ
+0eäEa#Óz
+tÏ%æ8Îô&õþŽÓFÓsC³8>AÙ\÷ÑöBøÁ&íŸ
+ÿÃ^MÉNšËÿj×0`Ì7)ªÙ0®4Ý%eŒ
+P³PšÙG+"GìA»ÐŒÓø@§*r
+
+ÅâUÇÆQL¹jJÙBVìÛüÒÄÊj"Ý,$K
+µ¶u³5|+±­Û?µlhÄ>Só¡Tîv]Ü~Iå\E>1±Á%ãTå>¯K$Þ`yJ
+Ìÿ¥v8>ñª9Æcq^3Y$±£ÄF
+Qáì%ûž«X?]ëS¬È6È+±­F¢Ö®Ö"ËZ
+W\Õ	ÕÈÕh`#uë0Ù/BŒä1F!»ÁBAm7€ZÌ}€4>¶îX¿çPªFý&$ë°ãÈ·Ù<q}³ÎEµ¬7ÄéUÖûd_
+J
+<19€ mDìèd`ÈÝ7òæ"Œmûz
+ME%è?€!Â×@òJ4
+i©lNQÌ_Ê}
+©rdQPŒ@|bÐà*TÖKÉ'Cq|Â±­ÐÖ+
+$ó{
+
+S±5 Oå+PÙ=Q`©€ý
+ïæëµXšÆ<,Zd+îÏ;M6F
+[GÕØQrå(ïx-²4à-
+ éâåoêFlÈ£Nt`
+ÆÕµ&((ZYÈbZÐn	Jõ1Ÿ³r5q¥Žf²ÑFv$»Ü#AWCËGŒ€(ûØL:=3"OªGG+ðyûÀüÜ±C^"W0¡]gjÏ
+òEHôÈºHÂ"íq#ž;µ$M9	A
+è©?uaGÞŠNìÎ$:kAš?m:šq
+ËÀÆ·P	ìâ¬aÍÅe¥ŒMQjÐŽ
+PÙ·U)Ôîáµ`Üjo®ÓtÛTDSë
+j¶DštÒ¬³æºÖð7ëgåóŒÀzjrë
+6&ÄµdÏ|9Zf»P(ÁÝMÎúÓV	ØJWê<5k¢úpÁ7(Ñ	` 
+`4ªI¡»#
+£B'»šB~~©%)6Ik¢0Ñ)M5*»7¬žhXÌ
+B)Û£BLðniS[¡pZëMY¡
+Z®sm#ŸL]GC!°ûúÿØ»0°*n0ŠQPÄ©Æ¶Ó£
+ 
+ïÙ
+î0+¶å
+Lâò2
+ŽÀZÇgÄÑº€_ðéÃ"p_a­þ­œe`ŠÐ%hóŒwÕ~¢§ËžÑ1Ž+xÎ©Î©î#ÇáIxË-u
+dÊa_ôøŠ
+Tvè;O¶,Â+Ò¢D
+]Ymë^&Ëö,õìúfb*-³»ÍÒjîZËHoËY&ÛC}dDv3;ìÁg}òœ¶à[â¬í2ÌŠ§ b>Ù4ëQ²L,c$z.uÅIã.,ÓbN€|€0±z>	
+®Ø'±éåK°U]
+`ÓEÑÄ4PWxÁ"RèYfÉ±Í/]d)È2Ê> iÀcTßûÎ³
+8ó<KºÛlŒ
+òÔºcaŸ8d_Â2ÒÖPÁ<Wž}`)²w²ÑZ÷@IægdF
+°Áá:(`ÊW÷åLEÀè#~
+L.-PkËŒž§k*LÔ)ßè
+ÕiYaÆ/
+0Ìæ]îÁšb[7fgÐòX
+ÝØ4kæ6¯eXNú ÕT)<R!¡Ä9·Aß6;Q
+E°x
+XøÅæøEœüÄ±©}TªÃ
+,|R=/Ÿ?ÔØÆÆB&(&W£Ä]HNnbøššmØ¢.*¯JT¬R.-DwÍrJÌsHÄ
+tFs|Ž'YR25²|ÍxÌb`%ø
+5`>x>ºldHEæéK=Pôä¬
+Ìàó]0ô­Ý&Â]
+Wß(pF®5ò
+ÆÖRÐ
+ ÌË`j\!
+~¿rF!\qÅûtÅ`78ffåÔe y$iîÉpð+ñž*ÂL£Ï÷åãp³Ü²¡Æž8äñÙjÚsf³8ždv¯Á}¿h°
+Ó©Áÿ_Ìþº>ï4õp¡îéŒ^Èa,Gâ.å^Ç#ÞÜ7í» ÏU¿,àcë\
+£ ¿fÆAVô ]bD7g*€GZqè900*
+37dhfõª €%ÿ ðã
+øtFŠ2·aI8åd
+Ó`»(`Í@:Ê
+Â§B#|kr
+$Èü ÓMiKuVyf²ÐÆf[EIEúÕè%Nxz3	¿ÇÇàÆbA< ZU'@Y rŽ#utõÎÍuý9×,I>
+{dÚg@Bw€o*éx6.Xjšà€X
+Ô"¥ÏGfÖŒGäŒý%{5$
+.YÊL-yx,©·M³	ïé3{®3$D©ÊÇ(ïÌÕØYynwÐc1¶(ÚB¶*2TxÖç€]tZ6Ï«ÑtÇ#IB8|ÔÂnšqËkº4
+PX·9ò
+`kj¬oŒZH
+€i!|I
+þÙ
+ uJ
+eV#ÃÀ²¢
+TÛI1¡!ù:l'£®,Ro®sèû¶e¡DY*T
+[{Ãt`#·ÐêèëaþdëÆ<ÂÔÓ
+üfjL#	M5§g%úC>J#{]+Ò©!œñj(Ì¢Þ|è^'wPÄûnds©êà1côöÁÐ ¡ÌÒHË
+HOGŽSJ¿ÕW°QLÎÙ_p1ìLrr$à-æ,3Ñ0P"å êGJÆHŽò«€Ü-¥ïß«a&Ÿª%ð±j¡0íûñP&ÿ Ý	ê å[t<§îªD	BÂ¶`ê\	âªUºr
+p€EGSæq+©2Ž¬¹TNâÑ,»Y-pŒUs
+&¥/Ît
+gö5c
++/
+o7áQt :ODÒÈ2SGñÃx'Q8.®w,'D.ÔÝp
+ç×*šyà3LNdg
+	<Ÿ
+Má×Òñzú².ÕªÎðl<ØÅBÊ Èün¿9ÝÑÕØMæqº¡BQ¥Ê«ÇT²	6jµè°¹amî®@&)bŽ×º5Ü
+
+œ@Ÿy×Ò¢J+=%þMâèÅ'ãÁ[7;ë÷Ÿy
++1D8Ð±¡aæÈÖçÀÂÝªÅ,
+}>Âup&ÝýŒ*ÇÖ/t
+Ñ¬1ðh~Û¯Û ºKRhÉ[HUÈ/"'·¹9ÃŒ)ôÆêG&×»œ@T_*±ÔqCSÆYg#-Žxi[Šj{Xµ%âÇVUÀ
+î%K%4:Aã ± LŽœRÄp|F£ârkmÖ±Î ©¶tÍx8%VèÙ[-lÿ[ÆùdÛ|¡óÇNÃqÎâckùý`M(ò£íY(ÕºüPQV:AìÓz]F#5îÇ9+ 
+Ô`4Gä9!
+Î
+äŠÈ`:P°íëÁkltZÑÛÓ5ì
+
+Ûa©®[
+ï|5A]Ý#	âäÁ°$ÐãZ-pGDNwå4k
+ÃMó¹jŒŽÖ4m$
+9(á:äîÏÄ%
+x­Î5%
+jaWßÂñe=-
+ÊCÞ|
+ãÄêF,¡ÜZ6Ñ[oÎïàY9Zô2ç rPc`é%I£¹x
+š5óQÊP$D"â»"¹¥CJ*šµ#°LÃÍ`óeýÈéuw3¥_j
+u&wwúÔ:RuUÌWoÔÆEÍ.&:
+"2³âÑnÚöcº'5by)X[¬
+63Þí3¬{Æm¢®«Ü¥'¹Í,6:@ÄðuÖ3ÞóÛŽ÷ªš40å'Se
+e%Í~'¯)³ô8[ñ©
+°¶(PÏhÆZóVf¥?M~«<Š Ð}
+v<·Ù.;	ažûÈ-6ÔÜ@õ ZZà\pCÈ	41âÚÝ¹'rbå"î?y¢0ªbÅ>&@ËuzBÍÙ GÊÔ×8ùÌ>uÜyhâgvêI
+SOMØ,á/YPy)íRúÂ¥àÓ*%%Õ€04š
+è±°ótÑ
+üúAß4€Oá§Ï4qrPþD3¢5/ÜÁ»0GQä4·ÄÅ6x	Y=ŽÀ.D :¥K¶áëSšºÂ[
+_
+aÕà
+Î)÷n4e>ò¢cWÃ@FÊ	Âú«
+ÀcD;l,lð-AÈé¬ÿÂe×%îb|œ=T€3šÆšÇLz4  ŒJð°AÌyà£QÚÍÙÓ=JYÀÍ+¹ÎŽ9ðU5qbO­&/4'ØòÓ\wI) ^BÁW39téú;r>?gcZF­ªVeLšo³h¶ŽÖÅÓ® ížgí8yÎø`av9kÉŒrQëÍ5C{vÙçÐ¬à©œÒÇÀÖvPØkþ	Xªí¬àÄ7r)¯\KgP+NË'³_:§2
+Ö"
+õÜÂÔ%!*6õ+ýÃî
+NWÎXg
+7Öž£h*©ŸµÀ^õ7([õ«÷!VJÀeÄ
+Œ
+µh-
+¯ýôYRY#!übæq4£úáÕPÑa#]©j±
+Lª	qïæÆåž1ÁªÉ"ÈòEd¿8ËÕAIÔNëu#cŒÂ2yk @(f¢:až{
+¥2FyF³Žž@RoÚØ
+©e&Ä
+?ÈNcB\x>ìwhü?ZèYÀ$çLÑÃµ9A°ô?ÐOA?xÛcô]çŠdÌæ]d~c%Iö³[ŒlÄø^ÑžME&h}š2QLÔùÜhŽFW,©»Ær5NžïàÎÌZØc«L{Ð·À×¡}
+rÊ5À¡+oíìV×ÒVž
+Ú4^ÀY°ø*óG¡]èÐª¶*ËžÇ`ßäQ
+#[A&@ÇØ ÿ ç\ógö_%LÛ
+POtHÔÿjÝSð®<MÝyô»	=ûž\€A.€¹ä9J,gþt¿ŽÉçûiÏu+ç€6ÀÀMÚâMP	t°VâÈVTÅÀà
+iI°ÝEM`LäJ!\Ü0	vôÎ	u¶qË
+i"|U;¢
+ubB4hVTH6|é~=O[°Lª/±sª
+NXE\»e²
+l
+Æ+²ÓÁáà>	Ø­§µµ*Öb= 0²9ªqÙDÕkÜb íN€eñq KºåNP©)
+å¬Yk€Êa{"¢êôv>D
+ó£q#öÌGh,,i€6Kpí:kZó°,DÁÆ 1«¬2#õ«2#ï£
+,·v7ìØ)ÕàeLÇÙTñ<P¢ÌŒ{nSõÛž1FííÖcÖø*
+û,Â,ÔöÇC°8äõ9»­úCÉ
+|f¶Vž"[ûE
+Fß]E>#º^w(BiqlÄ\Z}õw¶÷(3 /Õíá?{ H¥ [àºmC_
+©¶ñÓþuðŽw ñL3cchÀ> šr)313e 
+ŒÆJÛ
+s¶,»²1RVCžXp­ÎUÍÒÑæ¬
+èK€CtŸ	Á
+]cš#ÿEº]÷¹±ô;¹
+ŠØ
+€`µF`ËöÒµ!¯lsÇ9ÛqjhS#ñ,×øŒv¥S
+)Ð`<¬ÖÁVqöçµ
+>Ó1@=#`P	8)$UVíAváùšü`=7uÞc¥öâÆ
+/æ9H±­l
+Öö	=PP()]Væ¶[;°qö[a°
+,7yeÚæcsyË&à\nß^9çX@]hõªc
+ðÀì@WúN×h[\ž×[°ýŽÆàé¹¢ý*_
+]­ Æ[šÛÊÁ1$&0ø$ÁzÝ!/À6t8QÀè	DíÔ×wâ2Þ,
+š/èÁŠEx"
+šT¹£:ÑmPÔH)ž>XZ2é5ŠãŸë0éª¬eQpgÀŽDÚçõÂ
+0­ÒfÉÈHéIí€Ÿj'PsÀÙ©<X,êÉä +ÏÚúÿV¥adn»Å
+'ŸåOpM-¶ÌŠF¢Q«š,À¡PŒfÚÕ
+û[r(â8:I]Ó¢¢Å±€¯¢·P
+RLZkÇF£Íª
+c+s¥Jk\Á&#g°Yñô{Ñ\ìcèH«­ŒÖŸ±@ŠÔ/£ŒÁÊsyÖ¬ìº"Î­\]WïlçŠ¡=HP
+æ>&2œ:­IÅ'|¿4a`Kw!(zD}hLÅF
+§G&Pp
+ž
+ 5ã~ù .ÈABîGnÅúR&ÌéÊµ³æÈÙ UL
+En]áµ|î©¡f?tu€R±»¥Mi|¬~ŠZ#(n|^ñy€m»&
+q
+
+­®©Tp£â¹ËÂâ
+Ö6_¶ñíS£ÀßÇÚ¿mÓ€MûZûåp
+üí¬?Û³;Ým·Ÿ¶#g?xªšž×ÎKEÊ
+Ä6
+Ôàþ°Ò üMsíZPM
+ûâ±
+Èh+]èãb[9VKÅÇ×¶y³øRMö/a8g
+¹
+
+WÉî7:è¡n fèQÂÄTùÜGk£RtPâØÌîé@RèÌpíyÎF#ÇmnÈÇ0A}ŠÉR)Ú5T¢²n5:çå$]Cîj7­bVü"í3'`c2Î®»%@8&@ÇóíÀBP|Ö'6=ÉRHp%X°ÇÏSÚ|³`.XVÏ$µe(° 
+-Ñè¹i<ØPHhð4Kzªçãçœª~kSœ¢öR¬
+Å
+Ì8y#¢Ò ìpk5=µeMÆHÌš·Qp-Ñ¶ž}EÒ²KâØïg,ß'MpP²1¥uH€í0g';å:(×ÖUXžÔItIæV^ÃSvkœV¶ÕžÎ­©`Î0îæÑö¯ª,®_SLç
+GýÐ?>
+JzB«nÝ3ÞÀ
+^#yC©­Âu&,¥^ŽTpw@àÑL³$
+é
+a}xÅIQí<p^¹5$(LêŒÞ²»ð
+Øh^I0Ž^Ìœ*5F91"éò#ÞÚïT÷€ÏM«Qv
+WHŠ¿*¯r|5ªïÀM
+æ+øA
+Iwå¿0ðžp¶o
+é$ÛwYL/`K
+1ŸÁ"ÓbD<XcŽ\
+N ù_G!wðTjW
+¿'¥lVe
+žÕ|m¶RÛÊU
+­}c¹oÐLÏãòèe]šÝ»YÒNxÁ=±^xß®÷¥v
+°àÇWÛÐ
+AýÀ}Ø>nÍHûhw):w»?Æ¢h²Lšð->]Øb4]£F)Ÿ
+ùå{º(5iLæ8P°C\w%ž#`Ï ÞîÎ7Øõo-õP(nSãG§ñ<®[1"8	©OðDØHsÚãñŽ\¬SÏX
+
+üÞ-lè5gÂó<k
+åä=Lš3ÂuÛù³
+nU^¹te&|Ã
+à¯èHN9
+U×Ûmþ·K×Løö|ÒÄ
+falfcŸ÷œMÂ+áì©
+³cÓadtr{ÿqÚj(
+­!àÀu§ª5¯PH»»ÖlÃù
+dQ'§Î5Ùú>_êdÎ_BN/ïjïfMŸ2,
+Œb÷G	µýXŒd\ÀÑÁ1PŒüyh³ŠÕ
+ÍúÞš
+@L5¬@3mÊfvÓk=¿'@ž
+æ|æ­×!°`
+ý£ ÷°òãÉÖjlGú£Ò²­ÍìVA€yVðïXÇ}Ivm:³Aùð
+á»ž(Ná"ø?
+/<FÖ#¹MDfYáŽ>ócV¹rRüè
+i~tÚ€5ŽÎuá·nXM
+q%t]%âWI!E+:¥výWÏìMtuîÝ!ÛxUb
+víß§ñÆ`©VO}>}MŒ
+7¬Î@
+5ŽŽ&PrdÊBê	æXž
+w}T.}JÛ¢Â¬dÀIÜÂôiPx¢· JÌŠEž²>XÂÖi1	X4Ó	ªøuÛÖ
+;Ò
+Û\AU·|FÙó'8m\Q5îúð*à¡èO¿
+À{C&vOÕîN&qëõÄõKïè×IÇUqÒñèž×»
+w" ÄÍ&Ê.h:>
+×§âäãqìøv\9^EhÇµ`Ì@õ58ü0ós0%);x 6áåeàLèâ2+Föi²õ0ä£)%mãOñp8s|êÄ ®WJ1ªÄ(Ò Ù» î
+æÇ
+\`&Ž,
+Zß
+r@b<%áÜc$|Yí
+Ûí§fÜéT"îx{
+ð
+ï§wnù!Ã¡ÜïÈÂH=Aa ÐTOÙÈñ:
+9\­$×çÈä1råø\ P$¿«>i~Ÿ!fêŠýââÜYX#>.ã
+)]
+I¢¬u>&]P¥ÏöÂ
+ëÇE0aU4w;z'Pœ@#-®±Op}9ç6±µú%5Í!k³¬`ÿCåVbñpÆN7pŽ2AFÎÚ0AÄã×qôxvÜ:Ä&OÊ+ÙFy!Öô
+¯ÝMÁÌãÒ1ê8³)=.)¯«ÇäN¹sŒâ\
+FbÆ>²&6yó¡à4&
+9ÉþNµRÁjØ·Š~
+ [%Ü^3°ŠäaÑÚDÛ
+ŽM
+øÝk«CjÙöÃ*ò¢B`[N Œe@âð
+²X`
+wqµçFx&®øóS¡¥B2§¿ ^Ú'?µs¢ÖÄŽ
+Qx(ÅÏÙDMw«2º	'Tt9¹åR[fùÛ1áuA{WaÖóiÌ:51ûN=Tf7æ)cÌ÷pWWC×gŽ\ÇÔñÈÊwEgåâ[ŸÖék[ü#sŽ±åî
+iHPE&e  5c(11+æ¬µwÓÏ¹Œ*
+uÞy&äÇÃ×VË.mV>mª; b^ë/ìJxUg
+mÓcÃß¡Ð@aÚHL±&a±šá§9ÝØòÉà«Äá)Û~lu:5¡v{Íù
+å+yâÀf%iž,>*æisÔÌl»Hæª²föpëì³D
+o"@cå"'6ŸemŒ|j?V^!ûTGS¶ÝXI1³?+Ì;¢µ\t«cÐ ¢VÝDØ¡¿/Ž	]ãrúGM
+­È}D'Hà "ŸÂ§Ôfž*ÙùGró]	/¡Éâ,q6ŒŠ!Éàx
+].k
+éhjSë
+e%ek¹Y¿FÊ9ÈÒRD&vàO!ÊUÆl÷: )nÅWA¢ë9nB+9aE@·
+ÊAV-N)¶ ÖÞÑ
+¬Å¹s¿lÕ<
+*Aãçg"Dh7çÈ<_õfxC0y©
+ÊgŸ`¡Ç9ûmg8s,0­Z;ñØ^xóKþoFLP¯ÆWØðYDè$Æèi&ñÎ
+è&ÜàKÍÓŽËÔû×ÑVÝÍÁâ¥
+À
+ø[Yn©43ö
+0LïZ-ñBÂ0NŽ2}ñ^+ddÆrÝÉæ±XHáR
+[
+Äcu€ìhK°eBh¹œw4+RÐiqÿM7¬¶ÂÕ1Ã­t ÇáJªþ®enäX4Në.;ÿ
+®s*aûášFªp`æ/Ó]ï;qyn1þa6³¢ goE¹ÃñE
+àOsµç3µ)æèl¡¯`'ìU¶©;©+°{¬	-$EÇÄ¡dôÁ6
+®jËÛQà{§Ý
+Þ9lÕ'7éSÂÒ{ðçÈÌë6È,FÔ+»ÄÀ
+8ØJ9u^.ûÛÎ}8LjÕK'¡©
+ÆåbqHCÚÒœ
+ö@
+àÌÞ&G%ø`ò»`àV6Åœ|BÝ,eOÎ#Ät­mPØjB³2
+¥Á\ô
+c4L
+ä¬§À0kœüŠ|ÿŸÐzc.ÖÚÕ
+éüqÊŽv°žo]Ÿ^N_·Ör¿ÅFÎo¢¢
+Â ^¯ì
+*BÚÌý
+óŠîr{Ü2ªÄD÷ð¢»ÔîÉÞ*8µSÏD:GÐà3×5%µD7&ÖÔuÖ®|5¬MAÄaÞìžiÝKQDwdÒ
+èüs(ºB`E7H¹Žä1C&öÊ
+ó 
+§ySéOëáwÛ<Am ­I"iÓÊrýOz7Ÿòª"Ë
+yá{ù·Ûs
+Ç6:èÀéd8ä ýðk5
+Eõ%Šíf]ÿDí0?£Ó7)CÁ+³-hõ×œçw&7j^OÇ>š­ŽBÙÞ
+Úg¹/^à@Þ
+çö·bØU)CÕ+Û)Œ¶Ñ
+×
+ð§àp_žS&lâP82¹è: õI5G{c$Qåá
+øk-ep/Í@y­¿aøŠ2/!ù³r³~P
+íÁì(åJž,0FÃêca}/r-`öÅ«
+?üö-ü~pT52ñ}¹ 
+iÃÕ·?C¯d\]RèiOž0ôÒ
+àÝ±^¿~©74+a
+ÚÀRG:;d°Û
+RÄÞÚó¹þ;
+Ð×ÎI ðäq³œTÀš³BBHxð6¥J'T.èñßSÇúsA¿ì­1fØÊY¶ñä
+_ ÐC
+ÅÈT}Õá"÷žÂÀÜ*-ô
+|×@àCÒdŒÔ°1êOóvcêKøä
+ì`1I¿2FQ·¯
+DV€Z4Pc`Ç`ÛÖpÃÉÙëí8z
+Q³
+ 
+087ÖÕÉpÝZ(Á[d:ûkmÖXkŸzo>wíŸËZXÖMÝ6³[)Øk8/U%ÍV2HÅTìKä5âëé'[×ŒCB
+:ö
+
+çq_
+÷á(%1äd9ÐæÊï­ÚÌ`*ÖXïYa
+ýTŽSËo~
+I
+	.Jè·v»ËE
+«ŽöÀÎs!n€jT¥M[/©ÞÍìÆÃ±¹g{Žñï¡1
+vÝlÝ
+2ê,ÉþÄ¯Ù®
+B­\zð4}ßAŒ]€"úâ'ü®VB1¹·{pÀ&Sy³AíL¥d±œQ@TG--,t¶ee
+Nê°m%û,­ðlšô?L#	*A},f]ì0nŽc°ÕÚËD
+KF@œicçXÄzBÇ*uïtp¯jë¶¯iân(¯Â`'Û{ŽOšêC"³re ÖR0&c§Œ1
+vì
+»A!ÝwD±ûüBÖ¥Î
+eÎUhT!ôZ9Ä@`­ÉP¹_²#âs"8Ç'Hpî)|QäsoÜnc¬HMÖð	¬îõ	nêN#€<|¡;paŠÉ2ì
+6Íœ€ÐŒr
+xWo£ŸA`Ár¢°fÒTÆ$B.ÁÀ/O
+!á:Û ³/
+Ö6^lä¿
+BHÅêó%ÃËèáÞ+ØUiò[ìÚGXD
+AÖâb¬	Qz¢ßÆ5\OÞ$ç
+üC3P¹ íÌÐP¡ÄÝÑÓ'Õ¯Ù\y		ÃžwD+
+£ÌqùRÅ
+3^?@»p(ž{Ý2Ðòí×ÓêGKÜÞºŠW\
+DªeKÿ`ÄBõáê;æÝdI0€¬±&tahEå»;¡ g_6s,×VzÃ`Y¡(Šáþ,F¢€ºC^PÖôý"Û8nœngŠmâ÷ÁöïBûÐkZþŸñ/â 8LÚ±ýöÇð²ßp6@Â¿>!ž
+BÜµ¥cÒvÌsº·kWàxaú{ù%0Ú)S°
+Jt2àT°NDWä
+T§
+;sLíÕ!CøOÏoZ#~ð~GÐãQ ÂÏb/þÞKHObÂS/RMPxwÙc
+¯Âoá¯ð5K
+#à)Ã#ái6,<û ÀgÎÍváð:5<Þ80ßm-pÁÐðäÐÌÍ-
+
+J
+>Ä
+ŒVÁý­#ïG}%S¥Ö€0ÌŠL|(
+d%%ü¯_Ä7âSxt.üQ?gÂÃñ,Œ~oÄ¹qÀðixá(Ÿ/ÿÂ;á[ñ§x2Œ€j	ËøTäð¯'N|;¢ð@
+U z
+ú0	 ø#°<þbé(tÆ`ý,¢PÕr.Æ|ãuÅ/¶2>žQ©Ø
+aô|äm,FîW9kš%kâÐãÇ(¯Õ/pÜÌ$¡`¿H#SÁ*ø§ ªöC>=MßŽ¶éA¹åÛ~nÑ€ñUnf3»w>y
+\rÆÔŠn/­×}!e}pì
+!Á}TN-Jí²ZÚî:Š"^è,!²3qs?øÍâg&2<-
+ÿ
+Â³á%€ø!üIEö
+F`wbàÄµ
+Q¹ÜñQ<Ç+0\²uQÁm{P:£çÔßî6ÿ
+Zªîø,ØÐ
+X5:¥ŒËl[ò	Kr&fhDjéw-öRòhµº€Ëèiçú0B»
+\QÓNr>qSQGt­àît"ab©ï
+¯àÎ]Eò<
+Ê¯É£ÙÞ
+ÕMsá4œš{­µÖñ*³®ªo|@Ô©
+ÐRå0LC¡X¡.}qA«1·Õ©'
+€kò!6"šnr[ôÖ<µ[¢)ý7FràQM#êü#.[§ŒÓÌÔìóG¬gíá
+:-
+úÔ|Œ5	Uäà\¶%Ëá})}Åy e(®=c£ú!q«ÕDY0~1 r3lx¿
+
+Mw|¹1^¥ž §GÈüÇî vuÜiŠäfmãhZ¶öáÅŠsŸOBõ¶fN¯O5õëgõç(Ù.g6
+?ïjAÑf4Ý
+N,c@¯LÌ¬Õ?kŸ"2ðl2
+kbò¿.ð­4ÍþKuji
+Ôÿ%ûù1\§ŠÇÌ
+xo,è
+æsN3~pL/\Þú¶9Ÿ¿
+ñ·o!rA%°l@¹9úèDWý{@
+6µEè YdÆsnÈvx€š,îÔØ7ª¿Œ28ŠÜg±CiÛçµ°v
+ð}Úß7Ð¥ÓÅ
+ZcÚ®ì
+J$7ß*Ð|B¯0:ÇU²lH8
+>
+ôàïŠøéŸ¢!òa	0`}ÒàW"ìèq\¡
+FñÒxk¥id=Wºr±ìB?ëR¢*Ä?%éS7ÝLO0SzŸŸyÀ»ÈVüÚè¯9ÿxq?ÌZÝÆÖæ±ñš±8Ç¶µ5@ó<Ïå«kè_pÝÚmŠå(	+ßÃ§Ib|ŽêßŒx1Æg¯átæÂ­dÝ<!çJG5Ö(¹2ýàðèÞ¯6ã.oþTó¯IdµŸ3KüÛ0·$Þ'uT`Æ\¬pgÖ%ØYfºÖËþ²h¬?œHNv\5W8š]÷(ÆÎ{
+ø]F}Š×Èéð8Ž¹šŠÄ|$)Q0ØŠWÂ¶Ô­CYTb¡œŒt?7[©xÂ`Å8ð°?ÉÊ£{ÐòêÛcœJg?f%'8c6E7Æò¬«JâÁpêŠ z
+èíÃ1aÎáÉÒ 8:ÓÐE²Lö)ys×ŸÞd(ù u«¿5(t¥|Mõ_ÉXlV¢jÖªT4¢$ûqÈ<ÇEËÑ×ž5Fÿá÷â ²ß?I4¶HÆiP¢tSéÆŽCò€ÐC$ÂG^Ñ[haxÚ?0ê
+®\HmÝi€G_ò£0ì³[
+6l>áE6
+é¢;Ð-8Nêb§¬²L
+©àòT-0ÿ0þ±Qlßl&k¢õ-Áó¢ì
+ŽoýE]ÏTÈºž°ôç_yÙC€0LÓöÛŽÀÆ<Àµ6dxjWŽDwË*I^  Ž­€å[¹dÿ
+3Úä°sJök^éD N€všÐûk
+^40×ª ÞvŽ9hÁÓ'³oK
+Vß
+^ N]¿D¯Òo4IÒ²šš
+C³#Ô³žËœ®íÿègŒf£F:îºx®úì®yî
+~þÕ
+
+GÆ`9ž>Š_=º?¡Žl?7Ïw/Ð35œVMí:á«å²[''ìÁf¢vÑý'5«+ä8À¯ILÃ§C[
+«|ö¢TÈüÞ)šó
+&d90iþ.·#øÏ\(Tjÿ}¡%ô*Ð&z¯Rr»~è	cÖ,è?Ú×ç&ïõžø
+KB{;ä»@ü
+öŸYðDK-Œ>
+6Ý
+x2ÖeªZý2FWÌ9#äwÕwane"]ŸÅ~Yó«
+ñ))"Ãï²æ-­©ÖPõA€<þŽn.­ã|ÚÑÀJQ°[¿gÔku	@9§Ù
+ªçòPc©ã­y0
+ü î~ñq³ß:>ùær'®õ
+,å%%Š$ÄRK£¢œuqºGnLêt%¿
++U×DkNÏëðI:ÖÙÉ¿à'ùå5£÷öYXŽL8åhcÙáÙF¿	8
+Œ°eãoyúéã×QÓ­V2hê±Ÿ!,,Kp2{ZÐá\ð2»~Ltàk×š]±A³@Jô9úÁ9ô°µ
+T­
+yh­rZðÇ	Ú~oÏ%­
+è€^6,.AÿÝŸd.:÷ÛÜþKO9^--üÃ¬×{Ý3ú
+82sÇW-
+µ
+\üÿ×ò
+IdþYUÑc
+çàÏý'=IÐÆýŽZY
+¿DÃw^]]ø*j%ÍäaœôÜðõl%dôÎŠ$E×ÄåÃ`H©æI8×À]CAs°ímÐØš*Éš ÑæŒ{Ñûný^Z©óºo©m
+œúqTdØ#Ì­·kh¡gŠ<Ñ/
+¢o:	÷u,e{Ît}l«^ÿÕ»L€GåqPáRÚé`äzÝ¡î.
+
+q±ÊrSG²ÕÊ+Zð0EEœ:/psRÝŽì
+!ÏQ@ßËü{Úž
+_.Wg±Ïc4ù
+¯Qv€
+Ý6Ù ekbýÃ{ÞÑŸ¥²þáª®TóA^D{£·FOŽ-åV[ô0%¹>Š<ËénÕý[ÿ\WoíòLOhD?·­K?mÀ:ëÇ5JÓvYdØø:í6OVöùoÍ@l5AÄlär®@îs©¹	ÃOÇ^ÓŠ
+`ùþBðï·'PÃlžß.ãt	öðHíf
+Íí6¹Ã:OµŠøãÒaÈÒ0
+°fq#A_ŽÐ§7(ôé÷Põ 9òà,¥èÇä'QIÁïg¿
+
+n<OÕëä}
+
+ôî
+N£3Áü!Í*E.(&­}«Ž¢É®HV
+ššîgXDúåß:°(%úàe
+ŸXSüÌèº.Ûy¶š
+TsÜÉ ÈÀô@NµüÜ6ŸŸÞ<É9Gœ*PíÛ
+bå¿ð ï °°
+*Àl/
+1¡#Ð rÖ2äS'p¢àÛÅñó?ÿ`ã}æ	¹Iª)ÊÛ,°Ÿ
+F3Ÿ!_tÔùæru©Ÿ|3Sà(1çÇ']áú7±ÌçÈ
+s
+À:@bØº¯¿Ke1Ý(p¯|m¡ïRÈŠ»¶÷/Â/š 
+b±ð C5û($»Í_Oá>Ô°k$U:çN_ ürÙ®ëX4¯ö^õªF^
+vÞ
+e×ÙÐè¹Ð ñúJÌ×f
+²ØQ|·cV¬Ë¿"øy'é"?yH`€1Y
+*P(÷nãy@ìÙðy/²V	Ò©ÇÀKqi=¢Aã?FZ#
+:rm;]$+=LX¹ÀM²Í÷C6\÷óíÐ{·)I£UK{á|_ _Dºí*£®VÏiD7aŠA¡ÀÜ6l<»8`
+)ŠÆyPŒ,Ô)JÏ;ÞWC@éPíVùÔ &ã§;eù²Uü,:h1=g
+ÌQ$|i\(âÓîâøwÑ$
+ºÌY=±b&4[¡N]Åè¥rR®\
+ºí£_áqyüÆéÑ¥ÚAš°ÙD=áä\8W÷`¯üÃœlò
+`b$n%ÿ
+ÛÉ_!p±Ü÷ñ-ÿžÈ¿ü°ÂkëÁâ§Ç
+:%+Cïbx4±ªó>"D÷sh'Ý/þúÕebBRÚ*·ÙYÝ+ýî[È³ÞLÕT;®zÄ
+@ç¿ÝÐ]
+ÀšÛý	MÇßí1Á.3hÿ<RÇô-»×Užhì
+œet}è9Yìk
+=ðÆgFáÏOÎ¹ Í;Ã?(þu'=ÄÆúÁú^%«?_¯wþý÷I(v€ü
+ú»ÑHj·îWÈœüIª
+õhtÑ`n	Vµ
+*ÍÔ«ðÌŸÞp×eAR¬'ßSê.ZiT|cè*ø}Kh¬~lûjõ
+;{<Z5=%fO/ÌÏb&|Z
+}¡Å |ÎXäQÓE"&
+fy[Çxõe B3YbKõi	F¶$~|dGÇzÐòAt÷#Å.Í
+öe}y­
+
+/F1DBÚu~~ß
+~0[|c~Iq8å"UI]×ÁQü^Ygè%|Û^èPUjo·	ùZñOô.
+×?W Å
+EÁgiB!Í€~ÌDÛ	rÈ|PjR
+5Ñ":!yn
+`<e8¯9ql-€È!6
+ZQ6v1fXÀAl×c9>=9ã»]gnn
+á  y61s^+S{e!Îa°8WÖFMWÅ}3ì?¡è.\{fe®^Ñ8ýýW	?Î0Û
+é'
+aqß~WÓ	¥]bXE(4]·4"Ž??Ú
+_€Tcn45I*Ÿ<÷;â9;tÚÛyÀ`Xô%?U<bXŽVãz	"ÏiŸ`	uZLdÀo:$p
+Ù:>
+e${jž
+é
+<~
+VXä*mð@P¿+Mìi&#}oWfÅZ³tO`
+
+±a¢j²X
+`I)
+;{
+åi^\&­=0nz
+žeL
+~$662YgeáVN2^=ÄTyzÿ&
+W
+¬aqu©_VJid3h;
+,E
+óxskò*P\!Yï3Sa4=2h\&ä
+]M
+*$Z
+C}M/è%¹@~ºoÝÌ2ïy[+£.B
+r6yÏY4y72
+#
+	}l(I¥pGS6hÅz)X?
+
+ç5³þ
+,t°:gY7÷Aø;|*Â]Js
+4,oÐñ"0z1ß
+v
+=²hiKø
+
+_wö{H!r<2¢@Ñ;4s>ë
+zp>±nb;|gnyÀ^xc]ºpÕOktœ/âoÑ€^ÆAœc³¯WFJ~~
+GB
+E
+âx!n#
+t|t@C0»8Ä]$?ëC>9:`ocEó
+4`]Z
+©ô0ema0!!p³æ3tç.Ñ,6Ypdo{\ÂódÆ,cñ<$]1M
+ì¹;!VW
+#Ss×xj\µvï+xv<N%"i>~ÎAÏ9'I;Mâf\U%z_ëm!|ÿvÕ[Å&EøËg"D[/·gT*DîÁkTn¥kD{31øeÃ>­dn ð,UK4Ýx
+Ñ	í\Q4&5K9s~L
+@MñdÜ+yÓ"4V|f9ä@9^y|#mWç>(4`
+%g3v,Ïv#øYe8ë[9Û6e,VÒ
+>M}|¯)«SÎ;Ð;å@;éC<{úxsZ93yFj°(cc|¡d,d
+
+uálŠš5®Wœ{O¥|²|H{!.ha{10uXñpësi%fä=N8~>zU[!íh¿-
+SUe{zÎ=îði
+$y{gièPÅy?VMQß
+ÜiŒ?
+XßjX]>i	Q.JÖIè]1	
+kÃTIs÷åfMuTk¯&¹rBbT"ÑbÖ!?ë9Švœí } ÷x·ä
+`
+:q+XÀRvGN9ó{*dvÌ|d}k4]ªZå8${ï0-qzŽBv
+F|o«Q`ÝÈwfú]A|g
+ëf~	O{
+UA%erÉKe
+Ü?
+i7Xö8&A]¢áB DvZ*
+â!sÍ$M7H?|ÿ{
+E_ÕL:#	_ötŠgVÿ$í"	M9ãcý2!)	Ñ[ŽcPF¶m
+="%uY3
+|wU
+Ç-@Tºb?`I!'KQ{8Ý*Ú=©
+û
+dE7b{€|zvÒOØ"@A
+ŠPï3ìkš}ö,ä ºt4êzh|ìQèkPQzñ#z
+i'
+ý
+Øb)
+ÈPI70;3
+yHA»/a(gÈÚ*ì
+»í
+@ÿI3P#Nõ~V/E:ÿòNþI±	ª
+c
+
+G>/\ÒF;\à"(kÀGqŒL€te	ýUÄD^9b~{9Mâ<=#
+
+ãPG_^W@TE^û<¥~·D&büyb=èAVhÜÄmëdC-WÏ,b>Z!q9E·  \\÷ojUÏÄKK{@ð+Å}ªuÞ{L
++rù
+Cs
+^J¿vt']ÅBÁ!ñ{œŒ||-FRjú~
+
+#i»Oµ.7 ¬z€U
+B|Lu$
+4/TGËU;ö?ºgU
+Xù
+ò	òS[
+Y
+Ao
+QBý\w^í@ÅR}Ušù{ÊuPFùrz ðu)5.-
+T
+{
+Ïiöd:æí
+ñÔf3N}uæ&
+Vâ"RÈzdp,.F!<"k:>À|géQÝG¢\ö	þ*Å7Û~þC}u+eL=<cµ
+å"F#PI8mi`¢E?&]U
+{DR]ên7^5:
+"ÉoÈbvZP P.âQ+Ô/ï\lŒ=G±A@TœtZ(zmáîB
+{z]~kULýg	6pg1ì0ð}²{ï×FÀRÎ
+²^#Êr¹mŸÔ	|ž%õ65`ÜdIS×? ð8
+K-rh¿euPz(­&
+à3 JY&_^F^@|
+ biy€ËLÕµGjsÿ3c/Wgt|0@
+ÐU;(jè%§78GnmúÅO#
+Þ#")Ó|ÜeR~¯
+g'è~
+ Š^ÌFmoãd©
+û=@p
+ÉÌ\3+!;êåtœE·?5ZjÔÈo÷[éŠ&óDü;àœxt;C¡T-&";Q^ÆøõÚ:Çú<j¡}uP`ÀŠO-^	smœVêhj%nÞ
+
+lz
+²XctöúzÔ	Üô|
+)nðzz,ê/IFYG fü]AQWPè
+xw3è%ÂhËÛ9Q
+
+r°ó	rÿ'
+§ôj
+ûEC[Èzaí[x¡±Ë{4D$t':tÅ*Õ{eÖ	¬z#Â%f%Ü{5õTKâ^ßB	úRŸ&7%5S\J@ IB0
+_Šk¶m)Gkm
+I
+Ÿa&njg1JM
+4 /Ï
+>²yYJ*^
+3("'a»2
+>ã,Ÿ+Gpsã,ÇB_3Ÿ&éLç&r<	
+I¿q3fŽVLÇBÙLE
+7²õW
+\&±øHû01kÿX¹-Â%1'If@²ÞUAT[ÐK!ŠH»{¹{ãÐO8G>/@-w4U_A
+'\
+ôbLÖ
+
+SüOñe
+±ñs){
+PtšSî3	¶oðzÐF^9OÒM
+hv]Wê
+46_Y t"ÏFÃdS
+3['YCò)qØy©&
+Ð&ü!!DÏZ;
+±oDymÂl» }oöy±:>0zh]Û.TÆ`VNK?ðjxlnÈáWÅ'ör
+@m
+WcGá:€DamY °:ÞA,q@qWIø;'/l¶OŠPn
+õ\±7'XWñoPzq/¹$gk³|RThs®©GÊ°mA"o¹m¢K9|I\
+C{ù ôSùkòß»gÆ@7ËZR«I1OÖ×é|ÔZyrwfètv!ø~0
+³@Ù{z
+iTŒb
+?°&§)1hhÈpA\Mm*
+¬yUfùwgB
+
+ðr@%]bÞå/6U8
+
+.-^i7ÆzJ-&PéaVJ
+K9
+ic*ÛNß"d!í8P:PBÓvàGS-M
+"b
+¥zU@ö 8
+â\¬`E
+hyF]iy~ÞSH`k%×R^
+Bîg2T
+ã ËdÝ8×
+qmýD?7ÙN	Ö	\ey"!j
+5N9>óKb~zRmlRë4"8
+
+Ä+WK'ú9-åQª
+)Ev
+Ï|%tà+Z³PŒu>ðd€@9zäd#Ë0OáZ¬"q
+{î6f~æ{9TÔ~M/P;ÿ"v)PŽVF#Ò
+;-:
+pv>ÿ1zìuVöV·
+úSE
+W~ y07êg}-äyu£q
+Zoh{/k4sA 	ÿgE÷/Eµï]iBF-#S/i|Š\(¥6W79°Psµ^?{°8TÂ`Ö8$ìYo
+
+ŒÁV%~4>I%
+Ô	NB6ÄR1**æ}î(QKn¥|¯]{v$õ	^do%ÆIÝ&ÛÞ.|-PŒ8€abUf:X7Ø9
+Ès94;2`Uÿ
+Wª 
+Â9Ð	x.|/iR»?D5@
+ùn=#
+MÕ93^0QG2I|ü¬teVÁ/ÀTÉÞfA`?T+DÖOØG?\'
+8mIs^©:=ÀUi"Ò'·¬@¡j«uMc2o1-¡J9S/Íz³;
+
+%r]Þ'GA(×RÔ	â3®"{qnÎŽ
+z
+o(Ò!¢
++6c&/:Ë]Õ3 t=lLW
+©gÆm]­Ä^J(y'HTjP@šÀG{WtbµAvF\@o\â^ÉE¬Š,qc:Ä9÷&hNdyc í\-ÛDÔdÞ$ÞYDÜp}ò%O?BL.m
+Yd§zÍ0<?ê)c_-v #@J»MÑo
+,
+Š]E5eYº
+j&5 »!Ì>qrŽ5
+yíbÎ2?ü5Ñ'>!QJm9+`?à=@jVô4¡yK\Ž|'
+iU
+
+HE³
+,Ê³.Ip6jpLë1
+
+]V­ ù
+©_LR«?>tûxQp<«ÅUýzxKÊÏ
+Ô9Ò1íA_7ñtvzø
+šd<ç&û)t
+a#3f3®*ñ'J*	[6]Ÿ9
+R5aXK
+a×
+®SKhú
+$
+ÙLäs°_õ8>)€{ìc+Hu-){4gAVhbJÅ4WY^-r%Ip0
+/YÏ?[Ác__OpalQ6
+@ÃæIXb^ôfxd1Ã>/ÁZoÊ&/jP#49lI
+Q(Eû	uÒw9/k
+uMùjC
+HY¹
+v Î}OdJCÔE?\K
+Ë{s[Àkh]·Ät	C3"§n &Î
+¶,[ÀÓhz$qqy~ÀHT+è)e-X5€F
+²YSl0z²
+U.mx
+Ì2,7¢œëHD¢dóö_9Q
+90$ý[sÔvÜ-/ÊQ%M	ýabl­;ñ'(c­;A cyW4Dz__L¿5hC%uÄ+
+ØGÛfK
+â%Ì
+ë6°Çpb÷=53Üö^enõ*dä
+û~Y14<K¬.{, 	õ{ôH#¢=èr;° (¢SÏ;d\wŽ^{
+Å
+áf
+'$­Nvi=kTW$T5n8^=)Ý[ç=}Ìò:Ìg$?Z,ÁHIJFt*Uë$ZGEÜŽ/©Mdp\lÈz¯°0lÎK{`~Hœjç@ñat"
+	H\Ÿ
+
+È
+¿
+Rp
+K!f)!¢HŽ>ù	å:°ö~üp5qàkC+Ž»!dXïX  #s§.
+9
+9-
+fwóXíyWIt\	I/ýrúcÚKù3:)#	n0.°+V2://ç6nª	0¢Æ
+>jü
+Ín<
+u
+L@}ª$
+.oa_~ö&1pÛKz *~iÙNÐrø.ñ/\,
+fZüü?æaI1DÇw,FA 0lÉÅ
+x{œ%Dül1!KYX
+ ŒXd
+¢
+D}\$	OKa:.eNÆ]šaØ
+
+?cßxhrVv!
+Ëxõ85l7lêÂ2RÒ$;ueÅ#€O5Wt
+#Y^4
+9í
+q¹nbNÓO.ySj0!Ï
+ÈSBY5Œ5×	Tf¬k
+ë5 *>TFB#(«QÏ·	I,at:è-û
+
+l<]«
+þ9IL#!)î
+P-1©3%¢O4weúJI¥
+¹¥»T-
+
+¯Yº/rŽ28î
+ÒTÈl­/kX ?+;%))Ü}UIN	ø
+wHüBþ
+Ÿ"
+È-òý"š'¹
+ /\
+z
Index: /main/tags/3.01/indexers/mg/WIN32cfg.h
===================================================================
--- /main/tags/3.01/indexers/mg/WIN32cfg.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/WIN32cfg.h	(revision 21112)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /main/tags/3.01/indexers/mg/about_mg.html
===================================================================
--- /main/tags/3.01/indexers/mg/about_mg.html	(revision 21112)
+++ /main/tags/3.01/indexers/mg/about_mg.html	(revision 21112)
@@ -0,0 +1,75 @@
+<HTML>
+<HEAD>
+<TITLE>About MG</TITLE>
+</HEAD>
+<BODY>
+<H1>About MG</H1>
+ 
+<HR SIZE=4>
+
+MG is a public domain
+indexing and retrieval system for text, images, and textual images.
+MG is covered by a 
+<a href="http://deimos.kbs.citri.edu.au/mg/intro/copying.html">GNU 
+public licence</a>.
+<P>
+MG is available for ftp <a href="ftp://munnari.oz.au/pub/mg">here</a>
+
+<H2>Developers</H2>
+
+The MG system is an embodiment of ideas developed primarily by:
+<ul>
+<li> <a href="http://www.cosc.canterbury.ac.nz/~tim">Tim C. Bell</a>,
+	University of Canterbury;
+<li> <a href="http://www.cs.mu.oz.au/~alistair">Alistair Moffat</a>,
+	University of Melbourne;
+<li> <a href="http://www.cs.waikato.ac.nz/~ihw">Ian Witten</a>,
+	University of Waikato; and
+<li> <a href="http://www.cs.rmit.edu.au/~jz">Justin Zobel</a>,
+	RMIT.
+</ul>
+
+<H2>Programmers</H2>
+
+The bulk of the programming work has been undertaken by:
+<ul>
+<li><A HREF="http://www.cs.waikato.ac.nz/~singlis/">Stuart Inglis</A>,
+       University of Waikato;
+<li><A HREF="http://www.cs.waikato.ac.nz/~cgn/">Craig Nevill-Manning</A>,
+       University of Waikato;
+<li> Neil Sharman, University of Melbourne; and
+<li> Tim Shimmin, Collaborative Information Technology Research Institute (CITRI).
+</ul>
+A full list of contributors appears
+<A HREF="http://deimos.kbs.citri.edu.au/mg/intro/mgintro.html#xtocid2063010">
+here</a>.
+
+<H2>Institutional Support</H2>
+
+The MG project has been supported by the following organisations:
+<ul>
+<li>	The Australian Research Council;
+<li>	<a HREF="http://www.citri.edu.au/">CITRI</a>, Melbourne;
+<li>	The <a HREF="http://www.cpsc.ucalgary.ca/">University
+		of Calgary</a>, Canada;
+<li>	The <a HREF="http://www.cosc.canterbury.ac.nz/">University
+		of Canterbury</a>, New Zealand;
+<li>	The <a HREF="http://www.cs.mu.oz.au/">University
+		of Melbourne</a>, Australia;
+<li>	<a HREF="http://www.cs.rmit.edu.au/">RMIT</a>, Australia; and
+<li>	The <a HREF="http://www.cs.waikato.ac.nz/">University
+		of Waikato</a>, New Zealand.
+</ul>
+
+<H2>Backpointer</H2>
+
+If you didn't just come from there, the MG home page is accessible
+<a href="http://www.kbs.citri.edu.au/mg/">here</a>.
+
+<HR SIZE=4>
+<ADDRESS>alistair@cs.mu.oz.au</ADDRESS><BR>
+ 
+</BODY>
+</HTML>
+
+
Index: /main/tags/3.01/indexers/mg/acconfig.h
===================================================================
--- /main/tags/3.01/indexers/mg/acconfig.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/acconfig.h	(revision 21112)
@@ -0,0 +1,35 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
Index: /main/tags/3.01/indexers/mg/aclocal.m4
===================================================================
--- /main/tags/3.01/indexers/mg/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/indexers/mg/aclocal.m4	(revision 21112)
@@ -0,0 +1,140 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ([regex])
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , [AC_LIBOBJ([rx])])
+fi
+])
Index: /main/tags/3.01/indexers/mg/build_options.in
===================================================================
--- /main/tags/3.01/indexers/mg/build_options.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/build_options.in	(revision 21112)
@@ -0,0 +1,2 @@
+set stem_method = 3
+set do_indexes = 1
Index: /main/tags/3.01/indexers/mg/config.guess
===================================================================
--- /main/tags/3.01/indexers/mg/config.guess	(revision 21112)
+++ /main/tags/3.01/indexers/mg/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/indexers/mg/config.h.in
===================================================================
--- /main/tags/3.01/indexers/mg/config.h.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/config.h.in	(revision 21112)
@@ -0,0 +1,177 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#define HAVE_STRERROR 1
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
Index: /main/tags/3.01/indexers/mg/config.sub
===================================================================
--- /main/tags/3.01/indexers/mg/config.sub	(revision 21112)
+++ /main/tags/3.01/indexers/mg/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/indexers/mg/configure
===================================================================
--- /main/tags/3.01/indexers/mg/configure	(revision 21112)
+++ /main/tags/3.01/indexers/mg/configure	(revision 21112)
@@ -0,0 +1,7753 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="src/text/mgquery.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS JNIINC JNISUFFIX JNIFLAGS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mg
+VERSION=1.2
+
+LDFLAGS=
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -single_module -framework JavaVM"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+
+
+
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+fi
+
+fi
+
+
+# text for endianness
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+
+                                                                      ac_config_files="$ac_config_files Makefile src/text/Makefile src/images/Makefile src/scripts/Makefile lib/Makefile jni/Makefile java/org/greenstone/mg/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "src/text/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/text/Makefile" ;;
+  "src/images/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/images/Makefile" ;;
+  "src/scripts/Makefile" ) CONFIG_FILES="$CONFIG_FILES src/scripts/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "jni/Makefile" ) CONFIG_FILES="$CONFIG_FILES jni/Makefile" ;;
+  "java/org/greenstone/mg/Makefile" ) CONFIG_FILES="$CONFIG_FILES java/org/greenstone/mg/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@JNIINC@,$JNIINC,;t t
+s,@JNISUFFIX@,$JNISUFFIX,;t t
+s,@JNIFLAGS@,$JNIFLAGS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/indexers/mg/configure.in
===================================================================
--- /main/tags/3.01/indexers/mg/configure.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/configure.in	(revision 21112)
@@ -0,0 +1,123 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(src/text/mgquery.c)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mg
+VERSION=1.2
+
+LDFLAGS=
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -single_module -framework JavaVM"
+else
+  AC_MSG_RESULT(no)
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+AC_SUBST(JNIINC)
+AC_SUBST(JNISUFFIX)
+AC_SUBST(JNIFLAGS)
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# text for endianness
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile
+	  jni/Makefile java/org/greenstone/mg/Makefile ], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/indexers/mg/docs/Makefile
===================================================================
--- /main/tags/3.01/indexers/mg/docs/Makefile	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/Makefile	(revision 21112)
@@ -0,0 +1,5 @@
+DISTFILES = Makefile mgmerge.README mg_invf_dump.changes \
+  README.man mg.1 mgintro.1 mgintro++.1
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /main/tags/3.01/indexers/mg/docs/README.man
===================================================================
--- /main/tags/3.01/indexers/mg/docs/README.man	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/README.man	(revision 21112)
@@ -0,0 +1,1 @@
+The rest of the man files are in the respective src directories.
Index: /main/tags/3.01/indexers/mg/docs/mg.1
===================================================================
--- /main/tags/3.01/indexers/mg/docs/mg.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/mg.1	(revision 21112)
@@ -0,0 +1,462 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg 1 \*(Dt CITRI
+.\"=====================================================================
+.\" Author:
+.\"     Nelson H. F. Beebe
+.\"     Center for Scientific Computing
+.\"     Department of Mathematics
+.\"     University of Utah
+.\"     Salt Lake City, UT 84112
+.\"     USA
+.\"     Email: beebe@math.utah.edu (Internet)
+.\"=====================================================================
+.if t .ds Bi B\s-2IB\s+2T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Bi BibTeX
+.if t .ds Te T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Te TeX
+.\"=====================================================================
+.SH NAME
+mg \- full-text inverted index support
+.\"=====================================================================
+.SH DESCRIPTION
+.B mg
+is a suite of programs that can be used to create
+and query full-text inverted indexes for a
+collection of documents.
+.PP
+An inverted index is essentially a list of
+pointers to occurrences of every word in a
+collection of documents, so that, for example, in
+a collection of Shakespeare's works, one can pose
+questions like
+.I "How often is a fool mentioned in the plays?"
+and
+.I "Are Romeo and Juliet ever mentioned in the same sentence?"
+While search utilities like the UNIX
+.BR grep (1)
+family can be used for simple queries, they suffer
+from several limitations:
+.TP \w'\(bu'u+2n
+\(bu
+Each invocation requires a separate pass over the
+documents, which becomes impossibly slow if the
+document collection is very large.
+.TP
+\(bu
+It is difficult to compose Boolean queries like
+.IR "Caesar AND Brutus AND NOT Antony" .
+.TP
+\(bu
+They provide no mechanism for ranking matches
+according to importance, which is extremely
+important when queries are complex and numerous
+matches are found.
+.TP
+\(bu
+They provide no mechanism for displaying
+surrounding context (although the Free Software
+Foundation
+implementations for the GNU Project remedy this
+with their
+.BI \-A " num"
+(after)
+and
+.BI \-B " num"
+(before)
+switches).
+.TP
+\(bu
+They provide no mechanism for searching for
+occurrences in the same paragraph, unless
+preprocessing is done on the files to wrap
+paragraphs into long lines.  Even that will often
+fail, because input buffer sizes may be limited to
+a few hundred characters.
+.TP
+\(bu
+They provide no easy way to deal with grammatical
+word-ending variants, except by explicit
+enumeration, such as searching for
+.IR compress ,
+.IR compressed ,
+.IR compresses ,
+.IR compressing ,
+and
+.IR compression .
+.PP
+Most computer users have been faced with the
+problem of finding information from a large
+collection of files, such as electronic mail,
+on-line documentation, or source code.  Inverted
+indices provide a highly-effective solution to
+this problem.
+.PP
+The
+.B mg
+software is described in Appendix A of the book
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994
+.fi
+.RE
+.PP
+Many of the algorithms implemented in
+.B mg
+represent significant advances over previous work,
+both in speed, and in storage requirements.  On a
+fast workstation, in tens of minutes, or a few
+hours,
+.BR mgbuild (1)
+can create an index to
+.I all
+of the words in hundreds of thousands of documents
+occupying hundreds of megabytes, or even more than
+a gigabyte, of disk space.
+.BR mgquery (1)
+can then be used to answer complex queries, with
+responses often returned in a second or less.
+.B mg
+also contains algorithms to deal with images, so
+that with a small amount of descriptive text for
+each image, it is possible to do searches in
+collections of images, and to have retrievals
+display the images using a viewer like
+.BR xv (1).
+.PP
+.B mg
+can deal with compressed text and image files and
+surprisingly, it usually runs faster than it would
+if the files were not compressed!  Thus, the
+considerable disk space savings possible from
+file compression are not lost because of the need
+for fast document search and retrieval.
+.PP
+The Free Software Foundation GNU Project
+compression utilities
+.BR gzip (1)
+and
+.BR gunzip (1)
+are recommended for general use over older
+alternatives, like
+.BR compress (1),
+because of their speed and high compression
+ratios.
+.\"=====================================================================
+.SH AVAILABILITY
+The
+.B mg
+software can be obtained via anonymous ftp to the
+Australian archive host
+.I "munnari.oz.au [128.250.1.21]"
+from the directory
+.IR "/pub/mg" .
+.\"=====================================================================
+.SH "TYPICAL USE OF mg"
+Although
+.B mg
+consists of more than 20 separate programs, many
+of which have complicated command-line options,
+take heart: most users require only two or three
+of these programs, and nothing more than a
+document name on the command line.
+.PP
+A
+.I document
+for
+.B mg
+is a fragment of text suitable for retrieval as a
+unit when it is found to contain a requested word,
+or words.  In a collection of poetry, a document
+might be a stanza, while in a novel, it could be a
+paragraph.  In an index of first lines of poems, a
+document would likely be just a single line.
+.PP
+Just what constitutes a document is decided by a
+user-modifiable UNIX shell script,
+.BR mg_get (1).
+The default script provided with the
+.B mg
+source distribution knows about these named
+document collections:
+.TP \w'mailfiles'u+2n
+.I alice
+Lewis Carroll's
+.I "Alice in Wonderland"
+book.
+.TP
+.I allfiles
+all mail files in the directory tree
+.IR $HOME/Mail ,
+including all of its nested subdirectories.
+.TP
+.I mailfiles
+individual mail messages in
+.IR $HOME/mbox
+and
+.IR $HOME/.sentmail.
+.TP
+.I davinci
+A small collection of text and images from the
+work of Leonardo da Vinci.
+.PP
+A document collection name is used by
+.B mg
+as a
+.BR csh (1)
+.I case
+statement selector, and as a subdirectory name in the
+.B $MGDATA
+directory, or the current directory, if
+.B MGDATA
+is not defined.
+.\"=====================================================================
+.SH "EXTENDING mg_get"
+This section describes how to extend
+.BR mg_get (1)
+to handle a new document collection.
+.PP
+Let us take two examples: all \*(Bi
+.I .bib
+files, and all \*(Te files, contained in
+subdirectories under the login directory.
+.PP
+For \*(Bi, each bibliographic entry will be
+considered a separate document.  In order to
+facilitate easy identification of entries, we
+shall require them to begin at the start of a
+line; the
+.BR bibclean (1)
+utility can be used to standardize the format of
+.I .bib
+files, and to validate their string values, so
+that this requirement is met.
+.PP
+For \*(Te, each paragraph will be a separate
+document, and we assume that paragraphs are
+separated by blank lines.  We assume that files
+with extensions
+.IR .atx ,
+.IR .ltx ,
+.IR .stx ,
+and
+.I .tex
+contain input to common \*(Te macro package
+variants.
+.PP
+Make a personal copy of the
+.I mg_get
+script, using the one in the
+.B mg
+source distribution
+.RI ( mg-1.0/mg/mg_get.sh ),
+or the one in the local binary program directory,
+at many sites called
+.IR /usr/local/bin/mg_get .
+.PP
+Examination of the
+.I mg_get
+script shows that each document collection name is
+used in a
+.BR csh (1)
+.I case
+statement selector, and that most of work is done
+by very simple
+.BR awk (1)
+programs that extract documents from files.  In
+your private copy of the
+.I mg_get
+file, after the line
+.PP
+.nf
+  breaksw #davinci
+.fi
+.PP
+and before the line
+.PP
+.nf
+  default:
+.fi
+.PP
+insert this new code:
+.PP
+.nf
+  case bibfiles:
+# Takes a list of files that contain BibTeX entries, and splits them up
+#   by putting ^B after each entry. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*.bib' -print | \e
+        sort | \e
+        xargs -l100 awk \e
+          '/^@/&&NR!=1{print "^B"} {print $0} END{print "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bibfiles
+
+  case texfiles:
+# Takes a list of TeX files and split them up
+#   by putting ^B after each paragraph. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*x' -print | \e
+        egrep '[.]tex$|[.]ltx$|[.]atx$|[.]stx$' | \e
+        sort | \e
+        xargs -l100 nawk '   /^ *$/ {if (b!=1) printf "^B";b=1} \e
+            \e!/^ *$/ {print;b=0} \e
+            END {printf "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #texfiles
+.fi
+.PP
+The ^B characters here are Control-B characters,
+.I not
+caret-B pairs.
+.PP
+If you have a large number of \*(Bi or \*(Te
+files, it is likely that a list of them would be
+too long for the UNIX shell to hold in a single
+variable, or on a single command line.  Thus,
+instead of storing the output of
+.BR find (1)
+in a variable, we proceed more cautiously, and
+employ it to produce a list of the required files,
+then pipe them to
+.BR xargs (1),
+which in turn passes up to 100 filenames at a time
+to
+.BR nawk (1)
+for document selection.
+.PP
+Install this modified
+.I mg_get
+script in your private directory for executable
+programs (e.g.
+.IR $HOME/bin ),
+create a directory
+.I $HOME/mgdata
+to hold the index, issue a
+.B rehash
+command if you are using
+.BR csh (1)
+or
+.BR tcsh (1),
+ensure that
+.I mg_get
+occurs in your search path
+.I before
+any system-wide one (the command
+.BI which " mg_get"
+will tell you which version will be selected),
+then create the inverted indexes by
+.BI mgbuild " bibfiles"
+and
+.B mgbuild
+.IR texfiles .
+These commands may take several minutes to run if
+you have a lot of \*(Bi or \*(Te files, or a large
+home directory tree.  Once they are complete, you
+can then query the index with the commands
+.BI mgquery " bibfiles"
+and
+.B mgquery
+.IR texfiles .
+These should respond very rapidly.
+.PP
+In order to keep your index up-to-date, you should
+arrange for it to be recreated automatically and
+regularly, probably every night.  You can do this
+with
+.BR cron (1).
+Use the command
+.B "crontab \-e"
+to edit your
+.I crontab
+file and add two lines like this:
+.PP
+.nf
+00 04 * * * mgbuild bibfiles >$HOME/mgdata/bibfiles.log 2>&1
+15 04 * * * mgbuild texfiles >$HOME/mgdata/texfiles.log 2>&1
+.fi
+.PP
+Save the file and exit the editor.  Now, every
+night at 4am and 4:15am,
+.BR mgbuild (1)
+will reconstruct your inverted indexes, and the
+results of the builds will be saved in log files
+in your
+.I $HOME/mgdata
+directory.
+.\"=====================================================================
+.SH "SEE ALSO"
+.na
+.BR awk (1),
+.BR bibclean (1),
+.BR bibtex (1),
+.BR compress (1),
+.BR csh (1),
+.BR grep (1),
+.BR gunzip (1),
+.BR gzip (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR nawk (1),
+.BR tcsh (1),
+.BR tex (1),
+.BR xargs (1),
+.BR xmg (1),
+.BR xv (1).
+.\"=====================================================================
+.\" This is for GNU Emacs file-specific customization:
+.\" Local Variables:
+.\" fill-column: 50
+.\" End:
Index: /main/tags/3.01/indexers/mg/docs/mgintro++.1
===================================================================
--- /main/tags/3.01/indexers/mg/docs/mgintro++.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/mgintro++.1	(revision 21112)
@@ -0,0 +1,212 @@
+.\"------------------------------------------------------------  
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4 
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------  
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO++ 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro++ \- extended introduction to the MG system
+.\"-------------------------------------------------------------
+.SH DESCRIPTION
+This manual assumes the reader has already read
+.BR mgintro (1).
+.\"-------------------------------------------------------------
+.SS Creating Different Databases
+If a user wants to build databases other than for some
+predefined ones, such as "alice", "davinci", "mailfiles", "allfiles",
+then the user has a couple of choices.
+Ultimately (s)he must produce a text file with control-Bs 
+terminating the documents. 
+To do this one can produce one or more such files, or write a
+"get" command (typically in the form of a script or c program).
+.\"-------------------------------------------------------------
+.SS Using Input Files for mgbuild
+If you don't want to write a "get" script and just want to use
+one or more text files as input, then you must first generate
+the file with control-Bs. For a simple example, you could take
+any text file(s) such as "test1.txt" and "test2.txt", and use 
+.BR vi (1)
+to insert control-Bs by typing "control-V b".
+Next you should create a file with "set" statements 
+in the following form:
+.PP
+.IP
+\fBset pipe = 0 # do not use pipe - use file instead 
+.br
+\fBset input_files = 'test1.txt test2.txt'
+.LP
+Let's call this file, "build_options".
+Now issue the command:
+.IP
+.B mgbuild -s build_options test
+.LP
+This should build a database called "test" in the $MGDATA directory,
+based on the source data of "test1.txt" and "test2.txt".
+The build_options file is simply sourced by 
+.BR mgbuild (1)
+after it has set up its variables. 
+Therefore, any settings one makes in the
+build_options file will override the standard settings.
+See 
+.BR mgbuild (1) 
+for more information.
+.\"-------------------------------------------------------------
+.SS Writing A Get Program
+Instead of using files as input, it is often more convenient to 
+write a "get" program. This program is called by 
+.BR mgbuild (1)
+to get the text data with control-Bs as document terminators.
+It should take three options:
+.br
+(i) -init; (ii) -text; (iii) -cleanup.
+.br
+Get will be called with "init" first and with "cleanup" at the end.
+It will call get with "text" when it wants the text and it should
+write the text to stdout.
+.br
+See 
+.BR mg_get (1)
+for an example.
+.\"-------------------------------------------------------------
+.SS Regular Builds
+The MG system provides a static database; 
+there are no update commands.
+So if one wants to keep one's database reasonably up-to-date
+then one can have this done automatically on a regular basis by 
+.BR cron (1).
+A crontab file can be created using:
+crontab -e
+A crontab file contains lines of the form:
+.nf
+.IP
+\fBminute hour day-of-month month day-of-week shell-command.
+.LP
+.fi
+See
+.BR crontab (1)
+for more information.
+.nf
+An example crontab entry is:
+.IP
+\fB15 02 * * * mgbuild allfiles >$MGDATA/allfiles/allfiles.log 2>&1
+.LP
+.fi
+This will build up the mg database for "allfiles", your mail in
+the folders, every morning at 2:15am. 
+.\"
+.\"-------------------------------------------------------------
+.SS Command Structure
+There are 22 commands that make up the mg system. However,
+a user may only need to be aware of a few: 
+.BR mgbuild (1),
+.BR mgquery (1), 
+and perhaps
+.BR mg_get (1). 
+Many of the commands are called by 
+.BR mgbuild(1).
+The commands can be broken up into a hierarchy.
+.PP
+--------------------------------------
+.br
+MG--+--image compression
+    |  |
+    |  +--mgbilevel
+    |  |
+    |  +--mgfelics
+    |  |
+    |  +--mgtic
+    |  |
+    |  +--mgticbuild
+    |  |
+    |  +--mgticdump
+    |  |
+    |  +--mgticprune
+    |  |
+    |  +--mgticstat
+    |
+    +--text 
+       |
+       +--compression
+       |  |
+       |  +--mg_passes -T1
+       |  |
+       |  +--mg_passes -T2
+       |  |
+       |  +--mg_compression_dict
+       |  |
+       |  +--mg_fast_comp_dict
+       |
+       +--indexing
+       |  |
+       |  +--mg_passes -N1
+       |  |
+       |  +--mg_passes -N2
+       |  |
+       |  +--mg_perf_hash_build
+       |  |
+       |  +--mg_invf_dict
+       |  |
+       |  +--mg_invf_rebuild
+       |  
+       +--weights
+       |  |
+       |  +--mg_weights_build
+       |
+       +--query
+       |  |
+       |  +--mgquery
+       |
+       +--tools
+          |
+          +--mg_invf_dump
+          |
+          +--mg_text_estimate
+          |
+          +--mgdictlist
+          |
+          +--mgstat
+.br
+--------------------------------------
+.PP
+.nf
+.BR mgbuild (1)
+calls the following commands:
+.RS
+.BR mg_passes (1), mg_compression_dict (1)
+.BR mg_perf_hash_build (1), mg_invf_dict (1), mg_invf_rebuild (1)
+.BR mg_weights_build (1)
+.RE
+.fi
+.\"--------------------------------------------
+.SH SEE ALSO
+.BR mgintro (1),
+.BR mgbuild (1),
+.BR mg_get (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95 
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
+
Index: /main/tags/3.01/indexers/mg/docs/mgintro.1
===================================================================
--- /main/tags/3.01/indexers/mg/docs/mgintro.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/mgintro.1	(revision 21112)
@@ -0,0 +1,155 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro \- introduction to the MG system
+.\"--------------------------------------------------------------
+.SH DESCRIPTION
+The MG (Managing Gigabytes) system is a collection of
+programs which comprise a full-text retrieval system.
+A full-text retrieval system allows one to create a
+database out of some given documents and then do queries
+upon it to retrieve any relevant documents.
+It is "full-text" in the sense that every word in the
+text is indexed and the query operates only on this index 
+to do the searching.
+.PP
+For example, one could have a database on the book,
+"Alice in Wonderland." A document could be represented by
+each paragraph in the book.
+Having built up the "Alice" database, one could do queries
+such as "cat alice grin" and retrieve any paragraphs which
+match the query. The matching could either be boolean, that
+is the retrieved paragraphs contain a boolean expression of
+the query terms e.g. "cat alice grin"; or the matching
+could be ranked i.e. the most relevant documents to the query
+in relevance order, using some standard heuristic measure.
+.\"--------------------------------------------------------------
+.SS Motivation
+If one wants to find some particular information which
+is stored in a computer text file then one has a few alternative
+courses of action. One can operate directly on the text files
+with utilities such as grep or can process the text files into
+some form of database. Grep is generally limited to identifying
+lines by matching on regular expressions. If the collection 
+of files which grep operates on becomes large, then continual
+passes over the entire text on each query becomes expensive.
+However, its usage is simple as no auxiliary files must be created.
+.PP
+A database consists of some data and indexes into that data. By having
+indexes one can query a large database quickly. Standard databases
+divide the data up into records of fields. This means that the granularity
+of search is a field. In a full-text system, such as \*r,
+there are no fields 
+(or there is an arbitrary sized list of word fields per document) 
+and instead every word is indexed. 
+Using this method, we can except free-form information and yet be fast on searches.
+The next question is what is the overhead of this database.
+In \*r most files which are produced are in a compressed form. The
+two notable compressed files being the given data and the index, called
+an "inverted file". By compressing the files it is possible to have the
+size of the database smaller than the size of the source data.
+.\"--------------------------------------------------------------
+.SS Typical Usage
+The most common use for \*r
+has been as a search database on unix mail files.
+However, any set of text data can be used, one just needs to determine
+what constitutes a document (see 
+.BR mgintro++ (1)
+).
+\*r has also been used on large collections such
+as Comact (Commonwealth Acts of Australia) which is around 132 megabytes 
+and also on sizes up to around 2 gigabytes for TREC 
+(a mixture of collections such as the Wall Street Journal 
+and Associated Press).
+.\"--------------------------------------------------------------
+.SS Getting Started with \*r
+The first thing to do is install the package; 
+please follow the INSTALL instructions.
+Having done this, it is necessary to set a couple of environment variables.
+MGDATA should be set to a directory which is to hold subdirectories for
+each database that you build. For example: 
+.IP
+.B mkdir ~/mgdata; setenv MGDATA ~/mgdata.
+.LP
+If you want to try out building some sample databases then there is
+some sample data such as the "Alice In Wonderland" book. To make sure
+this is accessible you should set the environment variable MGSAMPLE.
+For example: 
+.IP
+.B setenv MGSAMPLE ~/mg/SampleData
+.LP
+Here, "~/mg/SampleData" should contain alice.z .
+.PP
+To build the Alice database (to be contained in $MGDATA/alice subdirectory), 
+type the command 
+.IP
+.B mgbuild alice 
+.LP
+Assuming all went well and some status messages 
+were printed indicating the build was completed, then type 
+.IP
+.B mgquery alice 
+.LP
+to query the database. 
+You can type a few words at the prompt, hit return and
+some relevant documents, Alice paragraphs, should be retrieved.
+Type ".set query ranked" to do ranking queries. Please refer to the 
+.BR mgquery (1)
+man-page for more information on the commands and options of 
+.BR mgquery (1).
+.PP
+The next thing to do is to use \*r
+on a more personal database. If you have
+your mail stored in subdirectories of ~/Mail, such as is done if you use the
+typical set up of 
+.BR elm (1), 
+then type 
+.IP
+.B mgbuild allfiles 
+.LP
+If, however, you keep all your mail in ~/mbox or ~/sentmail, then type 
+.IP
+.B mgbuild mailfiles
+.LP
+.\"--------------------------------------------------------------
+.SH AVAILABILITY
+The \*r software for SunOS 4, Solaris, HPUX, and MIPS,
+can be ftped from: munnari.oz.au [128.250.1.21] in the directory
+/pub/mg. 
+.\"--------------------------------------------------------------
+.SH SEE ALSO
+.na
+.BR mgintro++ (1),
+.BR mgbuild (1),
+.BR mgquery (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
Index: /main/tags/3.01/indexers/mg/docs/mgmerge.README
===================================================================
--- /main/tags/3.01/indexers/mg/docs/mgmerge.README	(revision 21112)
+++ /main/tags/3.01/indexers/mg/docs/mgmerge.README	(revision 21112)
@@ -0,0 +1,230 @@
+A CRASH COURSE GUIDE TO MGMERGE
+===============================
+
+Shane Hudson
+15 November 1994
+
+
+This document is intended as a note to the maintainers of mgmerge
+outlining the mgmerge utility, the changes
+that had to be made to the existing mg code to make mgmerge
+work, and the new source code for mgmerge.
+
+
+****  NOTE:    ****
+All the NEW/CHANGED files are in the file "new_mg.tar" which
+you should have received with this; files that were
+not changed are NOT in the tar file  so it will have to be
+extracted "on top of" the existing code. You'll figure it out :)
+
+
+****  DESCRIPTION:   ****
+mgmerge adds documents to an existing mg database
+without the need to rebuild the database from scratch.
+It works by building a temporary new database from the
+new documents, then merging the "old" and "new" databases
+to give one merged database.
+
+
+The source consists of:
+    mgmerge.sh		The main script, a lot like mgbuild.sh
+    mg_get_merge.sh	Script to retrieve new documents, a lot like
+			mg_get.sh
+    mg_merge.h		header file used by mg_invf_merge.c
+			and mg_text_merge.c
+    mg_text_merge.c	Program to append one compressed text file
+			to another
+    mg_invf_merge.c	Program to merge two inverted files and
+			stemmed dictionaries
+
+See the man pages for mgmerge, mg_get_merge, mg_text_merge and
+mg_invf_merge for a brief overview.
+
+mg_get_merge.sh should really be put into mg_get.sh, with a "-merge"
+option if it is being called by the mgmerge utility, but I was
+too lazy to change it to that.
+
+
+The files updated by mgmerge are:
+
+*.text			with mg_text_merge
+*.text.idx		with mg_text_merge
+*.text.idx.wgt		with mg_weights_build (after merging)
+*.invf			with mg_invf_merge
+*.invf.idx		with      "
+*.invf.dict		with      "
+*.weight		with mg_invf_merge or mg_weights_build
+*.invf.dict.blocked	with mg_invf_dict (after merging)
+
+The *.text.stats file and *.text.dict file remain the same.
+Any other database files (ie, *.invf.dict.hash)
+will be out of date after a merge and will need to be recomputed.
+
+--------------------------------------------------------------------------
+
+MG_TEXT_MERGE
+=============
+
+The two "merging" utilities are mg_text_merge and mg_invf_merge
+
+mg_text_merge simply appends the compressed text for the new
+documents to the old compressed text file.
+For this to succeed, the two files should have been created
+with the same parameters to mg_passes and the same compression
+model must be used. So *.text.dict from the old (static) database
+is used as the model for compressing the new documents.
+The default "-C" option to mg_compression_dict in mgbuild was
+changed to "-S" so novel words in the new documents can be coded.
+The option must be the same in mgmerge -- it is "-S" there too.
+
+---------------
+
+EFFECT on FILE COMPRESSION PERFORMANCE
+
+Since the compression model is not being updated,compression
+performance for the text will slowly degrade. The only solution
+is a periodic rebuild from scratch.
+
+---------------------------------------------------------------------------
+
+MG_INVF_MERGE
+=============
+
+mg_invf_merge updates the stemmed dictionary and
+inverted file.
+For reasons of simplicity, the stemmed dictionaries merged
+are in the ".invf.dict" format, NOT the ".invf.dict.blocked"
+format. 
+So if a database is to be merged the .invf.dict file should
+not be deleted after the .invf.dict.blocked file used by mgquery
+has been created.
+Also for simplicity, level 3 inverted files are not supported, nor
+are skipped format files.
+
+
+
+Merging the inverted files requires every entry in each file
+to be decoded and merged if an entry for the same term appears
+in the other inverted file.
+This can be a slow process if the old inverted file is very large.
+Especially since each entry is decoded bit-by-bit with
+the bit-level I/O rouutines.
+As was noted on page 94 of the book "Managing Gigabytes" (hereafter 
+called "MG"), switching from bernoulli coding (called "Bblock" in
+the mg source code) to gamma or delta coding for the
+inter-document gaps would be an advantage since they require no
+parameters.
+With Bblock coding, N (the number of documents) is a parameter and
+as N changes whenever documents are added, every entry must
+be decoded and recoded.
+My approach was to keep Bblock coding, but keep the N parameter
+constant by recording the artificial N  used to code document
+gaps (called "Nstatic") as well as the real number of documents.
+
+Then, any entry in the old inverted file (called "IFold") that
+is not merged with an entry in the new inverted file (called "IFnew")
+can be copied directly to the merged inverted file (called "IFmerge").
+This can give a significant increase in speed, since usually the
+size of IFnew is very small if only a few documents are being added.
+
+To store Nstatic, a field "static_num_of_docs" was added to
+the invf_dict_header and stem_dict_header structs defined
+in invf.h
+Any program that decodes inverted file entries also had to be changed.
+(The existing mg source code that I changed has been returned with
+this document.)
+Most changes were simply altering a line that
+called BIO_Bblock_Init() so it used the static_num_of_docs
+value rather than num_of_docs.
+
+So since the file format has changed, any existing collection will
+have to be rebuilt first even using mgmerge with it is not
+intended.
+
+-------------
+
+EFFECT on INVERTED FILE COMPRESSION PERFORMANCE
+
+As more merges are accumulatively performed on a database, the
+compression performance will decline.
+The solution is an option in mg_invf_merge to do a "slow"
+merge where every inverted file entry is decoded and recoded using
+the real value of N.
+This is not a lot slower than a fast merge when compared to the
+cost of completely rebuilding the collection from scratch, and 
+the option can be used periodically (when, say, 40% of the
+inverted file has been added since the previous slow merge).
+
+-----------------
+
+THE EXACT WEIGHTS FILE
+
+Recomputing exact document weights requires a complete scan over the
+merged inverted file. This is a waste since the weights for old
+documents will (hopefully) not change much anyway, and the weights
+for new documents can be computed exactly when merging the inverted files,
+at no extra time cost.
+So mg_invf_merge computes the weights for new documents and
+updates the .weight file.
+The weights for old documents are left untouched.
+mgmerge has an option to recompute the weights file.
+This is not much more expensive in terms of time, and is only
+periodically needed.
+
+The effect of leaving the old weights unchanged is difficult to
+asess since the "correct" ranking of documents for a query is
+subjective.
+The frequency with which a rebuild of the weights file with the
+"-w" option should be done is hard to guess.
+But if most merges involves only a few documents, it makes sense
+to not bother recomputing it since the change in old weights values
+is very small and the weights only stray from their "true"
+values slowly over accumulated merging.
+
+-------------------------------------------------------------------------
+
+EXPERIMENTAL RESULTS
+====================
+
+With some small test collections (only a few Mb of text)
+and a slow merge, and a weights file rebuild,
+mgmerge typically took under 20% of the time to completely
+rebuild. Using the default options (fast merge, dont rebuild weights)
+took around 10% or more.
+For larger collections, the savings were greater, especially if the
+added text is very small.
+For example, when the last 6Kb of text of the GNUBib collection was
+added to the rest of it (14Mb) using mgmerge, the total mgmerge time
+was around 20 seconds whereas a complete mgbuild took 280 seconds.
+Only one larger collection was tested (resource contraints prevented
+testing on any really large collections): two short (one-line)
+documents were added to the Gutenberg collection, which comprised
+of nearly 74Mb of source text and had an inverted file size of 9Mb.
+mgmerge took 42 seconds (or 142 seconds with a slow merge and rebuilding the
+weights file), and I'd estimate that an mgbuild on the same machine would 
+have taken at least 20 minutes.
+
+To summarise, mgmerge is not as fast as some sophisticated method
+such as the fixed-length blocks described in section 5.7 of "MG",
+but its huge advantage is that it required almost no changes to
+the existing mg code and is still far better than using mgbuild
+every time a document needs to be added to a collection.
+
+----------------------------------------------------------------------------
+
+OTHER NOTES
+===========
+
+Since it works like mgbuild, mgmerge needs mg_get_merge
+to return the SAME text each time "mg_get_merge -text .." is called.
+
+----
+
+One other thing discovered (but not fixed) was a bug in "invf.pass2.c":
+It crashes on parsed words longer than 128 characters.
+So if any text with words >128 chars is piped to mg_passes,
+the "-N1" and "-N2" options (the memory-efficient inversion method)
+had better be used and not the "-I1"/"-I2" option (which would only
+work on small collections anyway, being the memory-inefficient
+method).
+
Index: /main/tags/3.01/indexers/mg/install-sh
===================================================================
--- /main/tags/3.01/indexers/mg/install-sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGDocInfo.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGDocInfo.java	(revision 21112)
@@ -0,0 +1,42 @@
+/*
+ *    MGDocInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** DocInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @See MGQueryResult
+ */
+public class MGDocInfo
+{
+    public long num_ = 0;
+    public float rank_ = 0;
+
+    public MGDocInfo(long doc, float rank)
+    {
+	num_ = doc;
+	rank_ = rank;
+    }
+
+    public String toString()
+    {
+	return "" + num_ + " (" + rank_ + ")";
+    }
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGEquivTermInfo.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGEquivTermInfo.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGEquivTermInfo.java	(revision 21112)
@@ -0,0 +1,53 @@
+/*
+ *    MGEquivTermInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** EquivTermInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @see MGQueryResult
+ */
+public class MGEquivTermInfo
+{
+    /** the term itself */
+    public String term_ = null;
+
+    /** the number of documents containing this term */
+    public long match_docs_ = 0;
+
+    /** overall term freq - word level */
+    public long term_freq_ = 0;
+
+
+    public MGEquivTermInfo(String term, long match, long freq)
+    {
+	term_ = term;
+	match_docs_ = match;
+	term_freq_ = freq;
+    }
+
+
+    public String toString()
+    {
+	String result = "\"" + term_ + "\"";
+	result += " (" + match_docs_ + "/" + term_freq_ + ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGPassesWrapper.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGPassesWrapper.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGPassesWrapper.java	(revision 21112)
@@ -0,0 +1,154 @@
+/*
+ *    MGPassesWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** java wrapper class for access to gs3_mg_passes in C
+ *
+ * the native side implemented in MGPassesWrapperImpl.c
+ */
+
+public class MGPassesWrapper
+{
+    static {
+	System.loadLibrary("mgpassjni");
+	initIDs();
+    }
+    
+    static final public char INVF_LEVEL_1 = '1';
+    static final public char INVF_LEVEL_2 = '2';
+    static final public char INVF_LEVEL_3 = '3';
+    
+    static final public int TEXT_PASS_1 = 0;
+    static final public int TEXT_PASS_2 = 1;
+    static final public int INDEX_PASS_1 = 2;
+    static final public int INDEX_PASS_2 = 3;
+    static final public int SPECIAL_PASS = 4;
+
+    static final public int NO_STEM_OR_CASE = 0;
+    static final public int CASE_ONLY = 1;
+    static final public int STEM_ONLY = 2;
+    static final public int STEM_AND_CASE = 3;
+
+    static final public String STEMMER_ENGLISH = "english";
+    static final public String STEMMER_FRENCH = "french";
+    static final public String STEMMER_LOVIN = "lovin";
+    static final public String STEMMER_SIMPLE_FRENCH = "simple-french";
+
+    static final private char END_OF_DOCUMENT = (char) 2;
+
+    public MGPassesWrapper() {
+	initCSide();
+    }
+
+    /** initialise the pass through the documents */
+    public native boolean init();
+    
+    /** add a pass declaration */
+    public void addPass(int pass) {
+	switch (pass) {
+	case TEXT_PASS_1:
+	    addPass('T','1');
+	    break;
+	case TEXT_PASS_2:
+	    addPass('T','2');	    
+	    break;
+	case INDEX_PASS_1:
+	    addPass('I','1');
+	    break;
+	case INDEX_PASS_2:
+	    addPass('I','2');
+	    break;
+	case SPECIAL_PASS:
+	    addPass('S','1');
+	    break;
+	}
+    }
+    /** set the base path */
+    public native void setBasePath(String basepath);
+   /** set the file name */
+    public native void setFileName(String filename);
+
+    public native void setStemOptions(String stemmer_type, int stem_method);
+
+    public native void setInvfLevel(char level);
+
+    /** Specify  the size of the document buffer in kilobytes.
+	If any document is larger than  bufsize,  the  program
+	will  abort with an error message. 
+     */
+    public native void setBufferSize(long bufsize);
+    
+    /** Maximum amount of memory to use for  the index  pass-2  file
+	inversion  in  megabytes.
+    */
+    public native void setInversionMemLimit(int limit);
+
+    /** If true, treat   SGML  tags  as  non-words  when  building  the
+	inverted file.
+    */
+    public native void ignoreSGMLTags(boolean ignore);
+    
+    /** if mg_passes fails, the document that caused teh failure will be
+	output to teh trace file or STDERR. 
+    */
+    public native void dumpFailedDocument(boolean dump);
+    
+    /** output statistics on the compression performance to  a  file  
+	called  *.compression.stats.    frequency  specifies  the  interval
+	(in kilobytes of source text) between outputting each line of 
+	statistics. 
+    */
+    public native void outputCompStats(int frequency);
+
+    /** activate tracing, a line will be output every tracepos input bytes */
+    public native void enableTracing(int tracepos);
+    /** process a Greenstone document, which may consist of many MG documents (seeparated by ^B */
+    public boolean processDocument(String docs_text) {
+	// need -1 in the following to keep empty strings at the end
+	String [] docs = docs_text.split(String.valueOf(END_OF_DOCUMENT), -1);
+	System.err.println("GS document split into "+docs.length+" mg documents");
+	for (int i=0; i<docs.length; i++) {
+	    try {
+		processMGDocument(docs[i].getBytes("UTF-8"));
+	    } catch (Exception e) {
+		e.printStackTrace();
+	    }
+	}
+	return true;
+    }
+   
+    /** finalise the pass through the documents */
+    public native boolean finish();
+
+    /** get the exit value once finished */
+    public native int exitValue();
+ 
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises any C side stuff */
+    private native boolean initCSide();
+
+    private native void addPass(char pass_type, char pass_num);
+
+    /** process a MG document */
+    private native boolean processMGDocument(byte[] text);
+
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGQueryResult.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGQueryResult.java	(revision 21112)
@@ -0,0 +1,155 @@
+/*
+ *    MGQueryResult.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** a java version of QueryResult for mg
+ *
+ * contains get methods for the java side, and simple
+ * set methods to be called from c side
+ *
+ */
+public class MGQueryResult
+{
+    /** the list of MGDocInfo */
+    protected Vector docs_ = null;
+    /** the list of MGTermInfo */
+    protected Vector terms_ = null;
+    /** the total number of docs found - not likely to be the size of docs_ */
+    protected long total_num_docs_ = 0;
+
+
+    MGQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+    }
+
+
+    /** clear the info from the last query
+	should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_ = 0;
+	docs_.clear();
+	terms_.clear();
+    }
+
+    public boolean isClear() {
+	return (total_num_docs_ == 0 && docs_.isEmpty() && terms_.isEmpty());
+    }
+
+    /** returns the result as a String - useful for printing out results */
+    public String toString()
+    {
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i = 0; i < docs_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGDocInfo) docs_.elementAt(i)).toString();
+	}
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = " + total_num_docs_;
+	return result;
+    }
+
+
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i = 0; i < terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += ((MGTermInfo) terms_.elementAt(i)).toString();
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+
+
+    // set methods used by c++ code
+
+    public void setTotalDocs(long num) {
+	total_num_docs_=num;
+    }
+
+
+    public void addDoc(long doc, float rank) {
+	MGDocInfo doc_info = new MGDocInfo(doc, rank);
+	///ystem.out.println("(Java) Added doc " + doc_info);
+
+	docs_.add(doc_info);
+    }
+
+
+    public void addTerm(String term, int stem)
+    {
+	MGTermInfo ti = new MGTermInfo();
+	ti.term_ = term;
+	ti.stem_method_ = stem;
+	terms_.add(ti);
+	///ystem.out.println("(Java) Added term " + ti);
+    }
+
+
+    public void addEquivTerm(String term, String equivTerm,
+			     long match, long freq)
+    {
+	// Find the term to add the equivalent to
+	MGTermInfo ti = null;
+	for (int i = (terms_.size() - 1); i >= 0; i--) {
+	    ti = (MGTermInfo) terms_.elementAt(i);
+	    // Found
+	    if (ti.term_ == term) {
+		break;
+	    }
+	}
+
+	if (ti == null) {
+	    System.err.println("Internal error: No term exists to add to.\n");
+	}
+	else {
+	    ti.addEquivTerm(equivTerm, match, freq);
+	    ///ystem.out.println("(Java) Added equivalent term " + equivTerm + ", match: " + match + ", freq: " + freq);
+	}
+    }
+
+
+    // Get methods for the java side - GS2MGSearch.java, GS2MGRetrieve.java
+    public Vector getDocs() {
+	return docs_;
+    }
+
+
+    public Vector getTerms() {
+	return terms_;
+    }
+
+
+    public long getTotalDocs() {
+	return total_num_docs_;
+    }
+}
+
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGTermInfo.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGTermInfo.java	(revision 21112)
@@ -0,0 +1,72 @@
+/*
+ *    MGTermInfo.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+import java.util.Vector;
+
+
+/** TermInfo 'struct' class for a search result from mg
+ *  used by MGQueryResult
+ *
+ * @see MGQueryResult
+ */
+public class MGTermInfo
+{
+    /** the term itself */
+    public String term_ = null;
+
+    /** the stem and casefold method used
+	0 = none
+	1 = casefold only
+	2 = stem only
+	3 = casefold and stem */
+    public int stem_method_ = 0;
+
+    /** list of stemmed and casefolded equivalent terms - if stem_method_ is non-zero 
+      - Vector of Strings */
+    public Vector equiv_terms_ = null;
+
+
+    public MGTermInfo()
+    {
+	equiv_terms_ = new Vector();
+    }
+
+
+    public void addEquivTerm(String equivTerm, long match, long freq)
+    {
+	equiv_terms_.add(new MGEquivTermInfo(equivTerm, match, freq));
+    }
+
+
+    /** output the class as a string */
+    public String toString()
+    {
+	String result = "\"" + term_ + "\"";
+	result += " stem(" + stem_method_ + ")";
+	result += " equiv terms(";
+	for (int i = 0; i < equiv_terms_.size(); i++) {
+	    if (i > 0) result += ", ";
+	    result += equiv_terms_.elementAt(i);
+	}
+	result += ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGWrapper.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/MGWrapper.java	(revision 21112)
@@ -0,0 +1,99 @@
+/*
+ *    MGWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+
+/** java wrapper class for access to mg in C
+ *
+ * the native side implemented in MGWrapperImpl.c
+ * uses MGQueryResult to hold the result of a query.
+ * the result of a getDocument is a String
+ * uses the jni
+ *
+ *@see MGQueryResult
+ */
+public class MGWrapper
+{
+    /** the query result, filled in by runQuery */
+    protected MGQueryResult mg_query_result_ = null;
+
+    /** pointer to c MGWrapperData class - cached indexData and queryInfo */
+    protected long mg_data_ptr_ = 0;
+
+    static {
+	System.loadLibrary("mgjni");
+	initIDs();
+    }
+
+    public MGWrapper() {
+	mg_query_result_ = new MGQueryResult();
+	initCSide();
+    }
+
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises the mg_data_ptr_ */
+    private native boolean initCSide();
+
+    /** returns a document: number docnum at level level 
+     * the base_dir and text_path paths should join together to provide
+     * the absolute location of the mg text files eg ..../index/text/demo
+     * returns the doc in utf-8
+     */
+    public native String getDocument(String base_dir, String text_dir, long docnum);
+
+    /** unloads the data */
+    public native boolean unloadIndexData();
+
+    // query param methods
+
+    /** if on=true, sets default casefolding on - it's off by default */
+    public native void setCase(boolean on);
+    /** if on=true, sets default stemming on - it's off by default */
+    public native void setStem(boolean on);
+    /** default is 50 */
+    public native void setMaxDocs(int num);
+    /** if on=true, a query returns term freq info - default is on */
+    public native void setReturnTerms(boolean on);
+    /** sets the index to search - default is 'dtx' */
+    public native void setIndex(String index);
+    /** sets the default boolean operator - AND(=1)/OR(=0) */
+    public native void setMatchMode(int mode);
+
+    /** returns a string with all the current query param settings */
+    public native String getQueryParams();
+
+    /** actually carry out the query.
+	Use the set methods to set query results.
+	Writes the result to query_result.
+     * - maintains state between requests as can be slow  
+     * base_dir and index_path should join together to provide
+     * the absolute location of the mg index files eg ..../index/dtx/demo 
+     * base_dir must end with a file separator (OS dependant)
+     */
+    public native void runQuery(String base_dir, String text_dir, String query_string);
+
+
+    /** get the result out of the wrapper */
+    public MGQueryResult getQueryResult()
+    {
+	return mg_query_result_;
+    }
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Makefile.in	(revision 21112)
@@ -0,0 +1,62 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = java/org/greenstone/mg
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+JAVAC = $(JAVA_HOME)/bin/javac
+JAVAH = $(JAVA_HOME)/bin/javah
+JAR = $(JAVA_HOME)/bin/jar
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS =  -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+MGHOME=../../../..
+JAVACLASSDIR=./classes
+JAVASRCDIR=../../..
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) *.java
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGHOME)/jni org.greenstone.mg.MGWrapper
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGHOME)/jni org.greenstone.mg.MGPassesWrapper
+	$(JAR) cf $(MGHOME)/mg.jar -C $(JAVACLASSDIR) org
+
+clean:
+	rm -rf $(JAVACLASSDIR) $(MGHOME)/mg.jar $(MGHOME)/jni/org_greenstone_mg_MGWrapper.h $(MGHOME)/jni/org_greenstone_mg_MGPassesWrapper.h
+
+install:
+
+distclean: clean
+	rm -f Makefile
+
+
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Queryer.java
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/Queryer.java	(revision 21112)
@@ -0,0 +1,175 @@
+/*
+ *    Queryer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mg;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+/** Queryer - java port of c++ Queryer 
+ * - uses MGWrapper to access mg
+ * only has get document and search - full text browse not implemented
+ *
+ * run as:
+ * java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>
+ *@see MGWrapper
+ */
+public class Queryer {
+  
+    /** outputs to std out a help message describing the commands used by 
+     * Queryer 
+     */
+    public void printHelp()
+    {
+	System.out.println( "commands available are:\n"+
+			    "\t.q\t\tquit\n"+
+			    "\t.h\t\tprint this help message\n"+
+			    "\t.d\t\tprint the current query parameter settings\n" +
+			    "\t.i<index>\tsearch index <index>\n"+
+			    "\t.t0/.t1\t\tquery type some/all\n"+
+			    "\t.c0/.c1\t\tcasefolding off/on\n"+
+			    "\t.s0/.s1\t\tstemming off/on\n"+
+			    "\t.o0/.o1\t\tshort output off/on\n"+
+			    "\t.m<num>\t\tset max docs to return to num\n\n"+
+			    "\t.p<docnum>\tprint document docnum\n"+
+			    "\t<query string>\tdo a query\n");
+    }
+
+
+    public static void main(String[] args)
+    {
+	if (args.length != 3) {
+	    System.out.println("Usage: java org.greenstone.mg.Queryer <basedir> <indexdir> <textdir>");
+	    return;
+	}
+
+	Queryer self = new Queryer();
+
+	String base_dir = args[0];
+	String index_path = args[1];
+	String text_path = args[2];
+
+	// the jni class to access mg stuff
+	MGWrapper wrapper = new MGWrapper();
+	wrapper.setIndex(index_path);
+
+	System.out.println("Welcome to Java Queryer :-)");
+	self.printHelp();
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	boolean shortOutput = false;
+
+	String command; // user input
+	char x; // the command letter
+	String data = null; // any auxiliary input to a command
+	while (true) {
+	    System.out.print(">"); // the prompt
+            try {
+                command = br.readLine();
+                command = command.trim();
+		if (command.startsWith(".")) {
+		    // a system command
+		    x = command.charAt(1);
+		    if (command.length() > 2) {
+			data = command.substring(2);
+			data = data.trim();
+		    }
+
+		    switch (x) {
+		    case 'q': // clean up and exit
+			wrapper.unloadIndexData();
+			return;
+		    case 'h': // print help message
+			self.printHelp();
+			break;
+		    case 'd': // print query param settings
+			String info = wrapper.getQueryParams();
+			System.out.println(info);
+			break;
+		    case 'p': // print doc
+			int docnum = Integer.parseInt(data);
+			String doc = wrapper.getDocument(base_dir, text_path, docnum);
+			System.out.println(doc);
+			break;
+		    case 'm': //match docs
+			int match = Integer.parseInt(data);
+			wrapper.setMaxDocs(match);
+			break;
+		    case 's': // set stem on/off
+			int stem = Integer.parseInt(data);
+			if (stem==0 ){
+			    wrapper.setStem(false);
+			} else if(stem==1) {
+			    wrapper.setStem(true);
+			} else {
+			    System.err.println("Error: stem should be 0 or 1");
+			}
+			break;
+		    case 'c': // set case on/off
+			int casef = Integer.parseInt(data);
+			if (casef==0) {
+			    wrapper.setCase(false);
+			} else if (casef==1) {
+			    wrapper.setCase(true);
+			} else {
+			    System.err.println("Error: case should be 0 or 1");
+			}
+			break;
+		    case 'i': // set index
+			wrapper.setIndex(data);
+			break;
+		    case 't': // set query type some/all
+			int type = Integer.parseInt(data);
+			if (type==0 || type==1) {
+			    wrapper.setMatchMode(type);
+			} else {
+			    System.err.println("Error: type should be 0 (some) or 1 (all)");
+			}
+			break;
+		    case 'o': // set output short/long
+			int output = Integer.parseInt(data);
+			if (output==0) {
+			    shortOutput = false;
+			} else if (output==1) {
+			    shortOutput = true;
+			} else {
+			    System.err.println("Error: output should be 0 or 1");
+			}
+			break;
+		    }
+		}
+		else {
+		    // a query
+		    wrapper.runQuery(base_dir, text_path, command);
+		    MGQueryResult res = wrapper.getQueryResult();
+		    System.out.println("(Java) Matching documents: " + res.getTotalDocs());
+		    if (shortOutput) {
+			System.out.println(res.toShortString());
+		    } else {
+			System.out.println(res.toString());
+		    }
+		}
+
+	    } catch (Exception e) {
+		System.out.println("Queryer error: "+e.getClass() + " "+e.getMessage());
+		e.printStackTrace();
+	    }
+	}
+    }
+}
Index: /main/tags/3.01/indexers/mg/java/org/greenstone/mg/winMake.bat
===================================================================
--- /main/tags/3.01/indexers/mg/java/org/greenstone/mg/winMake.bat	(revision 21112)
+++ /main/tags/3.01/indexers/mg/java/org/greenstone/mg/winMake.bat	(revision 21112)
@@ -0,0 +1,47 @@
+@echo off
+
+set MGHOME=..\..\..\..
+
+rem ---- Where to put class files ----
+set JAVACLASSDIR=classes
+
+rem ---- Name and location of java programs ----
+set JAVAC="%JAVA_HOME%\bin\javac"
+set JAVAH="%JAVA_HOME%\bin\javah"
+set JAVA="%JAVA_HOME%\bin\java"
+set JAVADOC="%JAVA_HOME%\bin\javadoc"
+set JAR="%JAVA_HOME%\bin\jar"
+
+set JAVACOPTIONS= -deprecation -g -O
+
+
+if ""%1"" == """" goto compile
+if ""%1"" == ""compile"" goto compile
+if ""%1"" == ""install"" goto install
+if ""%1"" == ""clean"" goto clean
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:compile
+	if not exist %JAVACLASSDIR% mkdir %JAVACLASSDIR%
+	echo Compiling...
+        %JAVAC% -d %JAVACLASSDIR% %JAVACOPTIONS% *.java
+	%JAVAH% -classpath %JAVACLASSDIR% -d %MGHOME%\jni org.greenstone.mg.MGWrapper
+	%JAVAH% -classpath %JAVACLASSDIR% -d %MGHOME%\jni org.greenstone.mg.MGPassesWrapper
+	%JAR% cf %MGHOME%\mg.jar -C %JAVACLASSDIR% org
+	goto done
+
+:install
+	goto done
+
+:clean
+	echo Cleaning up...
+	if exist %JAVACLASSDIR% rmdir /S /Q %JAVACLASSDIR%
+        if exist %MGHOME%\mg.jar del %MGHOME%\mg.jar
+        if exist %MGHOME%\jni\org_greenstone_mg_MGWrapper.h del %MGHOME%\jni\org_greenstone_mg_MGWrapper.h
+        if exist %MGHOME%\jni\org_greenstone_mg_MGPassesWrapper.h del %MGHOME%\jni\org_greenstone_mg_MGPassesWrapper.h
+	goto done
+
+:done
Index: /main/tags/3.01/indexers/mg/jni/MGPassesWrapperImpl.c
===================================================================
--- /main/tags/3.01/indexers/mg/jni/MGPassesWrapperImpl.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/MGPassesWrapperImpl.c	(revision 21112)
@@ -0,0 +1,243 @@
+/*
+ *    MGPassesWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <jni.h>
+#include <assert.h>
+#include "sysfuncs.h"
+#include "org_greenstone_mg_MGPassesWrapper.h"
+
+#include "mg_passes_4jni.h"
+#include "mg_files.h"
+
+/* if we need to use java objects, we should initialise their field ids here*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  return;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  clear_variables();
+  return 1;  /* true - no errors */
+}  
+
+/* add a pass type T1, T2, I1, I2, S */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_addPass(JNIEnv *j_env, 
+					       jobject j_obj, 
+					       jchar j_pass_type,
+					       jchar j_pass_num)
+{
+  /* get the level as a c char */
+  const char pass_type = j_pass_type;
+  const char pass_num = j_pass_num;
+  add_pass(pass_type, pass_num);
+  
+}
+
+/* Set the filename */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setFileName(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_filename)
+{
+  /* Get the filename as a C string */
+  const char* filename = (*j_env)->GetStringUTFChars(j_env, j_filename, NULL);
+ 
+  assert(filename != NULL);
+  set_filename(filename);
+  
+  /* Release the string */
+ (*j_env)->ReleaseStringUTFChars(j_env, j_filename, filename);
+
+}
+
+/* Set the base path */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setBasePath(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_basepath)
+{
+  /* Get the base_path as a C string */
+  const char* basepath = (*j_env)->GetStringUTFChars(j_env, j_basepath, NULL);
+  assert(basepath != NULL);
+ 
+  set_basepath(basepath);
+  
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_basepath, basepath);
+
+}
+
+/* set the level for the inverted file */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setInvfLevel(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jchar j_level)
+{
+  /* get the level as a c char */
+  const char level = j_level;
+  set_invf_level(level);
+  
+}
+
+/* set the stemmer and stem method */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_setStemOptions(JNIEnv *j_env, 
+						      jobject j_obj,
+						      jstring j_stemmer, 
+						      jint j_method)
+{
+
+  const char* stemmer = (*j_env)->GetStringUTFChars(j_env, j_stemmer, NULL);
+  int method = j_method;
+
+  assert(stemmer != NULL);
+  set_stem_options(stemmer, method);
+  
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_stemmer, stemmer);
+ }
+
+/** Specify  the size of the document buffer in kilobytes.
+    If any document is larger than  bufsize,  the  program
+    will  abort with an error message. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setBufferSize(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jlong j_bufsize){
+  long buffer = j_bufsize;
+  set_buffer_size(buffer);
+}
+    
+/** Maximum amount of memory to use for  the index  pass-2  file
+    inversion  in  megabytes.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setInversionMemLimit(JNIEnv *j_env, 
+							    jobject j_obj, 
+							    jint j_limit) {
+  int limit = j_limit;
+  set_inversion_limit(limit);
+}
+
+/** If true, treat   SGML  tags  as  non-words  when  building  the
+    inverted file.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_ignoreSGMLTags(JNIEnv *j_env, 
+						      jobject j_obj, 
+						      jboolean j_ignore){
+  int ignore = j_ignore;
+  ignore_sgml_tags(ignore);
+}
+
+/** if mg_passes fails, the document that caused the failure will be
+    output to the trace file or STDERR. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_dumpFailedDocument(JNIEnv *j_env, 
+							  jobject j_obj, 
+							  jboolean j_dump) {
+  int dump = j_dump;
+  dump_failed_document(dump);
+}
+
+/** output statistics on the compression performance to  a  file  
+    called  *.compression.stats.    frequency  specifies  the  interval
+    (in kilobytes of source text) between outputting each line of 
+    statistics. 
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_outputCompStats(JNIEnv *j_env, 
+						       jobject j_obj, 
+						       jint j_frequency){
+  int comp_stat_point = j_frequency;
+  set_comp_stat_point(comp_stat_point);
+  
+}
+/** activate tracing, a line will be output every tracepos input bytes */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_enableTracing(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jint j_tracepos){
+  int tracepos = j_tracepos;
+  set_trace_point(tracepos);
+}
+
+/** specify the name of the trace file */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGPassesWrapper_setTraceFile(JNIEnv *j_env, 
+						     jobject j_obj, 
+						     jstring j_tracefile){
+
+  const char* tracefile = (*j_env)->GetStringUTFChars(j_env, j_tracefile, NULL);
+  assert(tracefile != NULL);
+  set_trace_file(tracefile);
+  /* Release the string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_tracefile, tracefile);
+}
+
+/* initialise the pass through the documents. must be called after all
+   the set methods 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_init(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  init_driver();
+  return 1;
+}
+
+
+/* process one document */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_processMGDocument(JNIEnv *j_env, 
+						       jobject j_obj,
+						       jbyteArray j_doc_text) {
+  /* Get the text as a C string */
+  int length = (*j_env)->GetArrayLength(j_env, j_doc_text);
+  u_char * text_buffer = (u_char *)(*j_env)->GetByteArrayElements(j_env, j_doc_text, NULL);
+  process_document(text_buffer, length);
+  /* Release the string */
+  (*j_env)->ReleaseByteArrayElements(j_env, j_doc_text, text_buffer,0);
+  return 1;
+}
+
+/* finalise the pass through the documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_finish(JNIEnv *j_env, 
+					    jobject j_obj) {
+  
+  finalise_driver();
+  return 1;
+}
+
+/** get the exit value once finished */
+JNIEXPORT jint JNICALL 
+Java_org_greenstone_mg_MGPassesWrapper_exitValue(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  return get_exit_value();
+}
+
Index: /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.c
===================================================================
--- /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.c	(revision 21112)
@@ -0,0 +1,725 @@
+/*
+ *    MGWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "MGWrapperImpl.h"
+#include <jni.h>
+#include "org_greenstone_mg_MGWrapper.h"
+
+#include "backend.h"
+#include "environment.h"
+#include "text_get.h"
+#include "stemmer.h"
+
+/*************************************************************************
+   NOTES
+
+ - Features supported by MGPP but not by MG (AFAIK)
+
+   - Sorting results by rank.
+     Done here as a post-processing operation. Could be more efficient in
+     some cases: the current solution is not very good when the number of
+     matching documents is large and the number of desired matches is
+     small. In this case it would be better to iterate through the array
+     picking out the best documents rather than sorting them all.
+
+   - Asking for query term frequencies to be returned.
+     This cannot be turned off in MG. If the term frequencies are not
+     required, they are simply not passed back to the Java side.
+
+   - Choosing the index for queries.
+     It is possible for MG to build Section and Document indexes (for
+     example), but these are two separate indexes, and must be loaded
+     separately. This module can load more than one index at a time, thus
+     processing queries to different indexes more quickly.
+     NOTE: This replaces TWO options in the MGPP version: returnLevel and
+           queryLevel.
+
+ *************************************************************************/
+
+
+#define MAX_INDEXES_CACHED 3
+
+/* copied from mgquery, needed to convert paragraph numbers to document numbers
+   for greenstone */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+static int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+        low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+        high = mid-1;
+      else
+        return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+  return 0;
+}
+#endif
+
+
+/*********************************************
+   initialisation stuff
+ *********************************************/
+
+/* cached ids for java stuff */
+jfieldID FID_mg_data = NULL; /* MGWrapperData */
+jfieldID FID_query_result = NULL; /* MGQueryResult */
+jmethodID MID_addDoc = NULL; /* MGQueryResult.addDoc() */
+jmethodID MID_addTerm = NULL; /* MGQueryResult.addTerm() */
+jmethodID MID_addEquivTerm = NULL; /* MGQueryResult.addEquivTerm() */
+jmethodID MID_setTotalDocs = NULL; /* MGQueryResult.setTotalDocs() */
+jmethodID MID_clearResult = NULL; /* MGQueryResult.clear() */
+
+
+/* to access objects and methods on java side, need their field/method ids - 
+ this initialises them at the start to avoid recalculating them each time they
+ are needed
+Note: the descriptors need to be exactly right, otherwise you get an error 
+saying "no such field" but no reference to the fact that it has the right 
+name but the wrong type.
+Note: apparently the jclass is a local ref and should only work 
+in the method that created it. It seems to work ok, but I'll make it
+global cos the book said I should, and it may avoid future hassles.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  jclass JC_MGQueryResult;
+
+  /* a long-"J" */
+  FID_mg_data = (*j_env)->GetFieldID(j_env, j_cls, "mg_data_ptr_", "J");
+  assert(FID_mg_data != NULL);
+
+  /* an object -"L<class name>;" */
+  FID_query_result = (*j_env)->GetFieldID(j_env, j_cls, "mg_query_result_",
+					  "Lorg/greenstone/mg/MGQueryResult;");
+  assert(FID_query_result != NULL);
+
+  /* the methods we want to use */
+  JC_MGQueryResult = (*j_env)->FindClass(j_env, "org/greenstone/mg/MGQueryResult");
+
+  /* addDoc(long doc, float rank) */
+  MID_addDoc = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addDoc", "(JF)V");
+  assert(MID_addDoc != NULL);
+
+  /* addTerm(String term, int stem) */
+  MID_addTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addTerm", "(Ljava/lang/String;I)V");
+  assert(MID_addTerm != NULL);
+
+  /* addEquivTerm(String term, String equivTerm, long match, long freq) */
+  MID_addEquivTerm = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "addEquivTerm", "(Ljava/lang/String;Ljava/lang/String;JJ)V");
+  assert(MID_addEquivTerm != NULL);
+
+  /* setTotalDocs(long) */
+  MID_setTotalDocs = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "setTotalDocs", "(J)V");
+  assert(MID_setTotalDocs != NULL);
+
+  /* clear(void) */
+  MID_clearResult = (*j_env)->GetMethodID(j_env, JC_MGQueryResult, "clear", "()V");
+  assert(MID_clearResult != NULL);
+}
+
+
+/* the java side MGWrapper has a pointer to a C object - MGWrapperData
+   initialise this and set the pointer 
+*/
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  /* Allocate a MGWrapperData object to store query parameters */
+  MGWrapperData* data = (MGWrapperData*) malloc(sizeof(MGWrapperData));
+  assert(data != NULL);
+
+  /* Set default values - no stemming, no case-folding, boolean OR queries */
+  data->defaultStemMethod = 0;
+  data->defaultBoolCombine = 0;
+
+  /* Allocate a QueryInfo object to store more query parameters */
+  data->queryInfo = (QueryInfo*) malloc(sizeof(QueryInfo));
+  assert(data->queryInfo != NULL);
+
+  /* Set default values - 50 documents max, return term freqs, sort by rank */
+  data->queryInfo->index = NULL;
+  data->queryInfo->maxDocs = 50;
+  data->queryInfo->needTermFreqs = 1;
+
+  /* Save the object on the Java side */
+  (*j_env)->SetIntField(j_env, j_obj, FID_mg_data, (long) data);
+
+  /* Initialise MG environment variables */
+  InitEnv();
+  SetEnv("expert", "true", NULL);
+  SetEnv("mode", "docnums", NULL);
+  return 1;  /* true - no errors */
+}
+
+
+/*******************************************
+   Index caching
+ *******************************************/
+
+query_data* cached_indexes[MAX_INDEXES_CACHED] = { NULL };
+
+
+/* Get the index data necessary to perform a query or document retrieval */
+query_data*
+loadIndexData(char* base_dir, char* index_path, char* text_path)
+{
+  char* index_path_name;
+  char* text_path_name;
+  query_data* qd;
+  int i = 0;
+
+  /* Form the path name of the desired indexes */
+  index_path_name = (char*) malloc(strlen(base_dir) + strlen(index_path) + 1);
+  assert(index_path_name != NULL);
+  strcpy(index_path_name, base_dir);
+  strcat(index_path_name, index_path);
+  printf("Index pathname: %s\n", index_path_name);
+
+  text_path_name = (char*) malloc(strlen(base_dir) + strlen(text_path) + 1);
+  assert(text_path_name != NULL);
+  strcpy(text_path_name, base_dir);
+  strcat(text_path_name, text_path);
+  printf("Text pathname: %s\n", text_path_name);
+
+  /* Search through the cached indexes for the desired one */
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    printf("(Cached) Pathname: %s\n", cached_indexes[i]->pathname);
+    printf("(Cached) Textpathname: %s\n", cached_indexes[i]->textpathname);
+
+    /* Check if the index has already been loaded */
+    if ((strcmp(index_path_name, cached_indexes[i]->pathname) == 0) &&
+	(strcmp(text_path_name, cached_indexes[i]->textpathname) == 0)) {
+      /* Index has already been loaded and cached, so return it */
+      printf("Found index!\n");
+      free(index_path_name);
+      free(text_path_name);
+      return cached_indexes[i];
+    }
+
+    i++;
+  }
+
+  /* Text strings no longer needed */
+  free(index_path_name);
+  free(text_path_name);
+
+  /* The index is not cached, so load it now */
+  qd = InitQuerySystem(base_dir, index_path, text_path, NULL);
+  if (!qd) {
+    printf("Error: Could not InitQuerySystem()...\n");
+    return NULL;
+  }
+
+  /* The index loaded OK, so cache it */
+  /* This could be a little more sophisticated, eg. replace least frequently used index */
+  if (i >= MAX_INDEXES_CACHED)
+    i = MAX_INDEXES_CACHED - 1;
+
+  /* Free the index being replaced */
+  if (cached_indexes[i] != NULL)
+    FinishQuerySystem(cached_indexes[i]);
+
+  /* Cache the loaded index, and return it */
+  cached_indexes[i] = qd;
+  return cached_indexes[i];
+}
+
+
+/* Clean up by unloading all cached indexes */
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mg_MGWrapper_unloadIndexData(JNIEnv* j_env, jobject j_obj)
+{
+  /* Free all the loaded indexes */
+  int i = 0;
+  while (i < MAX_INDEXES_CACHED && cached_indexes[i] != NULL) {
+    FinishQuerySystem(cached_indexes[i]);
+    cached_indexes[i] = NULL;
+    i++;
+  }
+
+  return 1;  /* true - no errors */
+}
+
+
+/****************************************************
+   retrieve a document
+ ****************************************************/
+
+/* Returns a document from mg as a string */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getDocument(JNIEnv *j_env, jobject j_obj,
+					     jstring j_base_dir, jstring j_text_path,
+					     jlong j_docnum)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  u_long pos, len;
+  u_char* c_buffer = NULL;
+  u_char* uc_buffer = NULL;
+  int ULen;
+
+  jstring result;
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return NULL;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return NULL;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  printf("Document retrieval, index path: %s\n", index_path);
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  if (qd==NULL) {
+    return NULL;
+  }
+  /*assert(qd != NULL);*/
+
+  /* Get the document position and length in the text file */
+  printf("Fetching document number %ld...\n", (unsigned long) j_docnum);
+  FetchDocStart(qd, (unsigned long) j_docnum, &pos, &len);
+  printf("Fetched document start. Pos: %ld, Len: %ld\n", pos, len);
+
+  /* Allocate memory for the document text (from mg/src/text/mgquery.c:RawDocOutput()) */
+  c_buffer = (u_char*) malloc(len);
+  assert(c_buffer != NULL);
+  uc_buffer = (u_char*) malloc((int) (qd->td->cth.ratio * 1.01 * len) + 100);
+  assert(uc_buffer != NULL);
+
+  /* Seek to the correct position in the file and read the document text */
+  Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+
+  /* Decompress the document text into another buffer, and terminate it */
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  uc_buffer[ULen] = '\0';
+
+  /* Load the document text into a Java string */
+  result = (*j_env)->NewStringUTF(j_env, uc_buffer);
+  assert(result != NULL);
+
+  /* Free C buffers */
+  free(c_buffer);
+  free(uc_buffer);
+
+  /* Return the document text */
+  return result;
+}
+
+
+/*******************************************
+   do a query
+ *******************************************/
+
+/* do the actual query - the results are written to query_result held on the Java side */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_runQuery(JNIEnv *j_env, jobject j_obj,
+					  jstring j_base_dir, jstring j_text_path,
+					  jstring j_query)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  char* index_path;
+  const char* base_dir;
+  const char* text_path;
+  query_data* qd;
+
+  jobject result_ptr;
+  char* query;
+  int i, j;
+
+  jthrowable exc;
+
+  /* First of all, clear the previous result */
+  /* The result to write to */
+  result_ptr = (*j_env)->GetObjectField(j_env, j_obj, FID_query_result);
+  assert(result_ptr != NULL);
+
+  /* Clear any previous query results */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_clearResult);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Make sure an index has been specified */
+  index_path = data->queryInfo->index;
+  assert(index_path != NULL);
+
+  /* Obtain C versions of the two string parameters */
+  base_dir = (*j_env)->GetStringUTFChars(j_env, j_base_dir, NULL);
+  if (base_dir == NULL) {
+    return;
+  }
+  text_path = (*j_env)->GetStringUTFChars(j_env, j_text_path, NULL);
+  if (text_path == NULL) {
+    (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+    return;
+  }
+
+  /* Load the appropriate index for satisfying this request */
+  qd = loadIndexData((char*) base_dir, (char*) index_path, (char*) text_path);
+
+  /* The C text strings are no longer needed */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_base_dir, base_dir);
+  (*j_env)->ReleaseStringUTFChars(j_env, j_text_path, text_path);
+
+  /* Check that the index was loaded successfully */
+  if (qd == NULL) {
+    return;
+  }
+  /*assert(qd != NULL);*/
+
+  /* Remove anything hanging around from last time */
+  FreeQueryDocs(qd);
+
+  /* Obtain a C version of the query string */
+  query = (char*) (*j_env)->GetStringUTFChars(j_env, j_query, NULL);
+  assert(query != NULL);
+  printf("Searching for query \"%s\"...\n", query);
+
+  /* Make sure the query isn't empty */
+  if (strlen(query) == 0) {
+    printf("Warning: Empty query.\n");
+    return;
+  }
+
+  /* "Some" queries are done as ranked queries */
+  if (data->defaultBoolCombine == 0) {
+    RankedQueryInfo rqi;
+    rqi.QueryFreqs = 1;  /* Use the frequency of each query term in the query - OK? */
+    rqi.Exact = 1;  /* Perform exact ranking */
+    rqi.MaxDocsToRetrieve = data->queryInfo->maxDocs;  /* Get only the desired number */
+    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;  /* OK? */
+    /* we may need to get more paragraphs to get enough documents. I copied the following from mgquery. it seems to work, not sure why - kjdon */
+    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas")) {
+      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+    }
+    
+    rqi.Sort = 1;  /* Sort the query terms by frequency before ranking */
+    rqi.AccumMethod = 'L';  /* Use a list when accumulating (has bugs though...) */
+    /* rqi.MaxAccums = -1; */ /* Use as many accumulators as necessary - CRASHES with list */
+    rqi.MaxAccums = 100000;
+    rqi.MaxTerms = -1;  /* Use all the query terms */
+    /* rqi.StopAtMaxAccum = 0;*/  /* Don't care (using as many accumulators as necessary) */
+    rqi.StopAtMaxAccum = 1;
+    rqi.HashTblSize = 1000;  /* Don't care (not using a hash table) */
+    rqi.skip_dump = NULL;  /* Don't dump skip information */
+
+    /* RankedQuery() reads 'casefold' and 'stem' parameters from the environment */
+    SetEnv("casefold", ((data->defaultStemMethod & 1) ? "on" : "off"), NULL);
+    SetEnv("stem", ((data->defaultStemMethod & 2) ? "on" : "off"), NULL);
+
+    RankedQuery(qd, query, &rqi);
+  }
+  /* "All" queries are done as boolean queries */
+  else {
+    BooleanQueryInfo bqi;
+    bqi.MaxDocsToRetrieve = data->queryInfo->maxDocs;
+
+    /* Had to add "words$o" to LIB_OBJS in mg/src/text/Makefile and recompile mg for this */
+    BooleanQuery(qd, query, &bqi, data->defaultStemMethod);
+  }
+
+  /* Finished with the C query string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_query, query);
+
+  /* Check the query was processed successfully */
+  assert(qd->DL != NULL);
+  assert(qd->QTL != NULL);
+  assert(qd->TL != NULL);
+
+  /* Record the total number of matching documents */
+  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_setTotalDocs, (jlong) qd->DL->num);
+  exc = (*j_env)->ExceptionOccurred(j_env);
+  if (exc) {
+    (*j_env)->ExceptionDescribe(j_env);
+    return;
+  }
+
+  /* Record the matching documents, but only the number requested */
+  printf("Number of matching documents: %d\n", qd->DL->num);
+  
+  for (i = 0; (i < qd->DL->num && i < data->queryInfo->maxDocs); i++) {
+    int doc_num = qd->DL->DE[i].DocNum;
+    float doc_weight = qd->DL->DE[i].Weight;
+
+#if defined(PARADOCNUM) || defined(NZDL)
+    if (qd->id->ifh.InvfLevel == 3) {
+      /* pararaph level, need to convert to doc level*/
+      doc_num = GetDocNumFromParaNum(qd, doc_num);
+    }
+#endif
+    
+    
+    /* Call the addDoc function (Java side) to record a matching document */
+    (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addDoc,
+			     (jlong) doc_num, (jfloat) doc_weight);
+    exc = (*j_env)->ExceptionOccurred(j_env);
+    if (exc) {
+      (*j_env)->ExceptionDescribe(j_env);
+      return;
+    }
+  }
+
+  /* Record the term information, if desired */
+  if (data->queryInfo->needTermFreqs) {
+    /* The following code is a lot more complicated than it could be, but it is necessary
+       to compensate for an oddity in MG. */
+    unsigned char** stemmed_terms = malloc(sizeof(unsigned char*) * qd->TL->num);
+
+    printf("Number of terms: %d\n", qd->TL->num);
+    printf("Number of query terms: %d\n", qd->QTL->num);
+
+    /* Generate the stemmed form of each of the relevant terms */
+    for (i = 0; i < qd->TL->num; i++) {
+      u_char* raw_term = qd->TL->TE[i].Word;
+      unsigned int term_length = raw_term[0];
+
+      u_char* raw_stemmed_term = malloc(term_length + 1);
+      unsigned int stemmed_term_length;
+
+      /* Copy the term, and stem it */
+      for (j = 0; j <= term_length; j++)
+	raw_stemmed_term[j] = raw_term[j];
+      stemmer(data->defaultStemMethod, qd->sd->sdh.stemmer_num, raw_stemmed_term);
+
+      /* Allocate memory to store the stemmed term, and fill it */
+      stemmed_term_length = raw_stemmed_term[0];
+      stemmed_terms[i] = malloc(stemmed_term_length + 1);
+      assert(stemmed_terms[i] != NULL);
+      strncpy(stemmed_terms[i], &(raw_stemmed_term[1]), stemmed_term_length);
+      stemmed_terms[i][stemmed_term_length] = '\0';
+    }
+
+    /* Record every query term, along with their equivalent terms */
+    for (i = 0; i < qd->QTL->num; i++) {
+      u_char* raw_query_term = qd->QTL->QTE[i].Term;
+      unsigned int query_term_length = raw_query_term[0];
+      unsigned char* query_term;
+      jstring j_query_term;
+
+      u_char* raw_stemmed_query_term = malloc(query_term_length + 1);
+      unsigned int stemmed_query_term_length;
+      unsigned char* stemmed_query_term;
+
+      /* Allocate memory to store the query term, and fill it */
+      query_term = malloc(query_term_length + 1);
+      assert(query_term != NULL);
+      strncpy(query_term, &(raw_query_term[1]), query_term_length);
+      query_term[query_term_length] = '\0';
+
+      /* Allocate a new jstring for the query term */
+      j_query_term = (*j_env)->NewStringUTF(j_env, query_term);
+      assert(j_query_term != NULL);
+
+      /* Call the addTerm function (Java side) to record the query term */
+      (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addTerm,
+			       j_query_term, (jint) data->defaultStemMethod);
+      exc = (*j_env)->ExceptionOccurred(j_env);
+      if (exc) {
+	(*j_env)->ExceptionDescribe(j_env);
+	return;
+      }
+
+      /* Copy the query term, and stem it */
+      for (j = 0; j <= query_term_length; j++)
+	raw_stemmed_query_term[j] = raw_query_term[j];
+      stemmer(data->defaultStemMethod, qd->sd->sdh.stemmer_num, raw_stemmed_query_term);
+
+      /* Allocate memory to store the stemmed query term, and fill it */
+      stemmed_query_term_length = raw_stemmed_query_term[0];
+      stemmed_query_term = malloc(stemmed_query_term_length + 1);
+      assert(stemmed_query_term != NULL);
+      strncpy(stemmed_query_term, &(raw_stemmed_query_term[1]), stemmed_query_term_length);
+      stemmed_query_term[stemmed_query_term_length] = '\0';
+
+      /* Find all the terms equivalent to the query term */
+      for (j = 0; j < qd->TL->num; j++) {
+	/* Check if the stemmed query term matches the stemmed term */
+	if (strcmp(stemmed_query_term, stemmed_terms[j]) == 0) {
+	  u_char* raw_term = qd->TL->TE[j].Word;
+	  unsigned int term_length = raw_term[0];
+	  unsigned char* term;
+	  jstring j_term;
+
+	  /* Allocate memory to store the query term, and fill it */
+	  term = malloc(term_length + 1);
+	  assert(term != NULL);
+	  strncpy(term, &(raw_term[1]), term_length);
+	  term[term_length] = '\0';
+
+	  /* Allocate a new jstring for the query term */
+	  j_term = (*j_env)->NewStringUTF(j_env, term);
+	  assert(j_term != NULL);
+
+	  /* Call the addEquivTerm function (Java side) to record the equivalent term */
+	  (*j_env)->CallVoidMethod(j_env, result_ptr, MID_addEquivTerm,
+				   j_query_term, j_term,
+				   (jlong) qd->TL->TE[j].WE.doc_count,
+				   (jlong) qd->TL->TE[j].WE.count);
+	  exc = (*j_env)->ExceptionOccurred(j_env);
+	  if (exc) {
+	    (*j_env)->ExceptionDescribe(j_env);
+	    return;
+	  }
+	}
+      }
+    }
+  }
+}
+
+
+/*******************************************
+   set query options
+ *******************************************/
+
+/* Turn casefolding on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setCase(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 1;
+  } else {
+    data->defaultStemMethod &= 0xe;
+  }
+}
+
+
+/* Turn stemming on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setStem(JNIEnv *j_env, jobject j_obj, 
+					 jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 2;
+  } else {
+    data->defaultStemMethod &= 0xd;
+  }
+}
+
+
+/* Set the maximum number of documents to return from a query */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setMaxDocs(JNIEnv *j_env, jobject j_obj, 
+					    jint j_max)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->maxDocs = j_max;
+}
+
+
+/* Turn term frequency recording on or off */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setReturnTerms(JNIEnv *j_env, jobject j_obj, 
+						jboolean j_on)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->queryInfo->needTermFreqs = j_on;
+}
+
+
+/* Choose MG index to search */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mg_MGWrapper_setIndex(JNIEnv *j_env, jobject j_obj,
+					  jstring j_index)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+
+  /* Get the index name as a C string */
+  const char* index = (*j_env)->GetStringUTFChars(j_env, j_index, NULL);
+  assert(index != NULL);
+  printf("Choosing index %s...\n", index);
+
+  /* Free the previous index name */
+  if (data->queryInfo->index)
+    free(data->queryInfo->index);
+
+  /* Allocate memory for the index name, and fill it */
+  data->queryInfo->index = (char*) malloc(strlen(index) + 1);
+  assert(data->queryInfo->index != NULL);
+  strcpy(data->queryInfo->index, index);
+
+  /* Release the index string */
+  (*j_env)->ReleaseStringUTFChars(j_env, j_index, index);
+}
+
+
+/* Choose boolean AND or boolean OR queries */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mg_MGWrapper_setMatchMode(JNIEnv *j_env, jobject j_obj,
+					      jint j_mode)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  data->defaultBoolCombine = j_mode;
+}
+
+
+/* Get a text representation of the current parameter values */
+JNIEXPORT jstring JNICALL
+Java_org_greenstone_mg_MGWrapper_getQueryParams(JNIEnv *j_env, jobject j_obj)
+{
+  MGWrapperData* data = (MGWrapperData*) (*j_env)->GetIntField(j_env, j_obj, FID_mg_data);
+  char result[512];  /* Assume this is big enough */
+
+  /* Print the data to a character array */
+  sprintf(result, "Query params:\nindex\t\t%s\ncasefold\t%d\nstem\t\t%d\nquery type\t%s\nmax docs\t%ld\n",
+	  (data->queryInfo->index == NULL ? "<none loaded>" : data->queryInfo->index),
+	  (data->defaultStemMethod & 1),
+	  (data->defaultStemMethod & 2),
+	  (data->defaultBoolCombine == 1 ? "all" : "some"),
+	  (data->queryInfo->maxDocs));
+
+  /* Convert to a jstring, and return */
+  return (*j_env)->NewStringUTF(j_env, result);
+}
Index: /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.h
===================================================================
--- /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/MGWrapperImpl.h	(revision 21112)
@@ -0,0 +1,52 @@
+/*
+ *    MGWrapperImpl.h
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include "backend.h"
+
+
+/*************************************************************************
+   NOTES
+
+ - These data structures are based on the MGPP ones but are modified
+   to reflect the different capabilities of the MG system.
+
+ *************************************************************************/
+
+ 
+typedef struct QueryInfo
+{
+  /* Name of index to use */
+  char* index;
+  /* Maximum number of documents to retrieve */
+  unsigned long maxDocs;
+  /* Whether term frequency information is desired (boolean value) */
+  int needTermFreqs;
+} QueryInfo;
+
+
+typedef struct MGWrapperData
+{
+  /* Information about a query, see above */
+  QueryInfo* queryInfo;
+  /* Whether to perform stemming and case-folding */
+  int defaultStemMethod;
+  /* Whether to perform boolean AND (1) or boolean OR (0) queries */
+  int defaultBoolCombine;
+} MGWrapperData;
Index: /main/tags/3.01/indexers/mg/jni/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,67 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = jni
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX -DNZDL -DQUIET -DPARADOCNUM
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#NOTE: libtextin.a needs to come before libmg.a !!!
+QUERY_LIBS = ../src/text/libtextin.a ../lib/libmg.a @LIBS@
+PASSES_LIBS = ../src/text/libmgpass.a ../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+JNIFLAGS=@JNIFLAGS@
+JNISUFFIX=@JNISUFFIX@
+JNIINC=@JNIINC@
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+CCOMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CFLAGS)
+
+.SUFFIXES:
+
+.SUFFIXES: .c .o
+.c.o:
+	$(CCOMPILE) $<
+
+# can we get rid of JAVA_HOME from here??
+INCLUDES = -I.. -I../lib -I../src/text -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JNIINC)
+
+all: compile link
+
+compile:
+	$(CCOMPILE) MGWrapperImpl.c MGPassesWrapperImpl.c
+
+link:
+	$(CC) $(LDFLAGS) $(JNIFLAGS) -o libmgjni.$(JNISUFFIX) MGWrapperImpl.o $(QUERY_LIBS)
+	$(CC) $(LDFLAGS)  $(JNIFLAGS) -o libmgpassjni.$(JNISUFFIX)  MGPassesWrapperImpl.o  $(PASSES_LIBS)
+
+clean:
+	rm -rf *.o libmgjni.$(JNISUFFIX) libmgpassjni.$(JNISUFFIX) 
+
+distclean: clean
+	rm -f Makefile
+
+install:
+	../mkinstalldirs $(libdir); \
+	cp libmgjni.$(JNISUFFIX) libmgpassjni.$(JNISUFFIX)  $(libdir)/;
+
+
+
Index: /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGPassesWrapper.h
===================================================================
--- /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGPassesWrapper.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGPassesWrapper.h	(revision 21112)
@@ -0,0 +1,175 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mg_MGPassesWrapper */
+
+#ifndef _Included_org_greenstone_mg_MGPassesWrapper
+#define _Included_org_greenstone_mg_MGPassesWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_1
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_1 49L
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_2
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_2 50L
+#undef org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_3
+#define org_greenstone_mg_MGPassesWrapper_INVF_LEVEL_3 51L
+#undef org_greenstone_mg_MGPassesWrapper_TEXT_PASS_1
+#define org_greenstone_mg_MGPassesWrapper_TEXT_PASS_1 0L
+#undef org_greenstone_mg_MGPassesWrapper_TEXT_PASS_2
+#define org_greenstone_mg_MGPassesWrapper_TEXT_PASS_2 1L
+#undef org_greenstone_mg_MGPassesWrapper_INDEX_PASS_1
+#define org_greenstone_mg_MGPassesWrapper_INDEX_PASS_1 2L
+#undef org_greenstone_mg_MGPassesWrapper_INDEX_PASS_2
+#define org_greenstone_mg_MGPassesWrapper_INDEX_PASS_2 3L
+#undef org_greenstone_mg_MGPassesWrapper_SPECIAL_PASS
+#define org_greenstone_mg_MGPassesWrapper_SPECIAL_PASS 4L
+#undef org_greenstone_mg_MGPassesWrapper_NO_STEM_OR_CASE
+#define org_greenstone_mg_MGPassesWrapper_NO_STEM_OR_CASE 0L
+#undef org_greenstone_mg_MGPassesWrapper_CASE_ONLY
+#define org_greenstone_mg_MGPassesWrapper_CASE_ONLY 1L
+#undef org_greenstone_mg_MGPassesWrapper_STEM_ONLY
+#define org_greenstone_mg_MGPassesWrapper_STEM_ONLY 2L
+#undef org_greenstone_mg_MGPassesWrapper_STEM_AND_CASE
+#define org_greenstone_mg_MGPassesWrapper_STEM_AND_CASE 3L
+#undef org_greenstone_mg_MGPassesWrapper_END_OF_DOCUMENT
+#define org_greenstone_mg_MGPassesWrapper_END_OF_DOCUMENT 2L
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    init
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_init
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setBasePath
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setBasePath
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setFileName
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setFileName
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setStemOptions
+ * Signature: (Ljava/lang/String;I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setStemOptions
+  (JNIEnv *, jobject, jstring, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setInvfLevel
+ * Signature: (C)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setInvfLevel
+  (JNIEnv *, jobject, jchar);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setBufferSize
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setBufferSize
+  (JNIEnv *, jobject, jlong);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    setInversionMemLimit
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_setInversionMemLimit
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    ignoreSGMLTags
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_ignoreSGMLTags
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    dumpFailedDocument
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_dumpFailedDocument
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    outputCompStats
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_outputCompStats
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    enableTracing
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_enableTracing
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    finish
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_finish
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    exitValue
+ * Signature: ()I
+ */
+JNIEXPORT jint JNICALL Java_org_greenstone_mg_MGPassesWrapper_exitValue
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    addPass
+ * Signature: (CC)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGPassesWrapper_addPass
+  (JNIEnv *, jobject, jchar, jchar);
+
+/*
+ * Class:     org_greenstone_mg_MGPassesWrapper
+ * Method:    processMGDocument
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGPassesWrapper_processMGDocument
+  (JNIEnv *, jobject, jbyteArray);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGWrapper.h
===================================================================
--- /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/org_greenstone_mg_MGWrapper.h	(revision 21112)
@@ -0,0 +1,109 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mg_MGWrapper */
+
+#ifndef _Included_org_greenstone_mg_MGWrapper
+#define _Included_org_greenstone_mg_MGWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getDocument
+ * Signature: (Ljava/lang/String;Ljava/lang/String;J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getDocument
+  (JNIEnv *, jobject, jstring, jstring, jlong);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    unloadIndexData
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mg_MGWrapper_unloadIndexData
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setCase
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setCase
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setStem
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setStem
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMaxDocs
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMaxDocs
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setReturnTerms
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setReturnTerms
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setIndex
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setIndex
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    setMatchMode
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_setMatchMode
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    getQueryParams
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mg_MGWrapper_getQueryParams
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mg_MGWrapper
+ * Method:    runQuery
+ * Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mg_MGWrapper_runQuery
+  (JNIEnv *, jobject, jstring, jstring, jstring);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/indexers/mg/jni/win32.mak
===================================================================
--- /main/tags/3.01/indexers/mg/jni/win32.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mg/jni/win32.mak	(revision 21112)
@@ -0,0 +1,81 @@
+###########################################################################
+#
+# win32 makefile -- mg\jni
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN -DPARADOCNUM \
+       -DSHORT_SUFFIX
+INCLUDES = -I"$(MGHOME)" -I"$(MGHOME)\lib" -I"$(MGHOME)\src\text" \
+           -I"C:\j2sdk1~1.1_0\include" -I"C:\j2sdk1~1.1_0\include\win32"           
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    MGWrapperImpl.h   org_greenstone_mg_MGWrapper.h  \
+    org_greenstone_mg_MGPassesWrapper.h
+
+
+SOURCES = \
+    MGWrapperImpl.c MGPassesWrapperImpl.c
+
+OBJECTS = \
+    MGWrapperImpl$o   $(MGHOME)\src\text\libmgtextin.lib \
+    $(MGHOME)\lib\libmg.lib
+PASSES_OBJECTS = \
+    MGPassesWrapperImpl$o   $(MGHOME)\src\text\libmgpass.lib \
+    $(MGHOME)\lib\libmg.lib
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : compile link
+
+compile:
+        $(COMPILE) MGWrapperImpl.c MGPassesWrapperImpl.c
+
+link:
+        $(CC) -MD -LD $(OBJECTS) -Femgjni.dll
+        $(CC) -MD -LD $(PASSES_OBJECTS) -Femgpassjni.dll
+
+
+install: 
+       copy mgjni.dll "$(GSDL3HOME)\lib\jni"
+       copy mgpassjni.dll "$(GSDL3HOME)\lib\jni"
+clean:
+        if exist *$o del *$o 
+	if exist mgjni.dll del mgjni.dll
+	if exist mgpassjni.dll del mgpassjni.dll
Index: /main/tags/3.01/indexers/mg/lib/GCC.mak
===================================================================
--- /main/tags/3.01/indexers/mg/lib/GCC.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/GCC.mak	(revision 21112)
@@ -0,0 +1,84 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+
+CPPFLAGS = -g
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I".." -I"."
+
+o = .o
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+
+
+HEADERS = \
+    bitio_m_random.h filestats.h local_strings.h lovinstem.h \
+    bitio_m_stdio.h getopt.h memlib.h bitio_gen.h bitio_mem.h \
+    mgheap.h messages.h bitio_m.h bitio_mems.h huffman.h \
+    timing.h bitio_m_mem.h bitio_random.h huffman_mem.h \
+    perf_hash.h bitio_m_mems.h bitio_stdio.h huffman_stdio.h \
+    sptree.h rx.h pathmax.h getpagesize.h random.h \
+    win32in.h simplefrenchstem.h unitool.h
+
+SOURCES = \
+    bitio_random.c mgheap.c memlib.c bitio_stdio.c huffman.c \
+    messages.c bitio_gen.c filestats.c huffman_mem.c \
+    perf_hash.c bitio_mem.c getopt.c huffman_stdio.c \
+    sptree.c bitio_mems.c getopt1.c local_strings.c \
+    lovinstem.c timing.c rx.c alloca.c error.c xmalloc.c \
+    strstr.c ftruncate.c strcasecmp.c random.c win32in.c \
+    simplefrenchstem.c unitool.h
+
+OBJECTS = \
+    rx$o bitio_random$o mgheap$o memlib$o bitio_stdio$o \
+    huffman$o messages$o bitio_gen$o filestats$o huffman_mem$o \
+    perf_hash$o bitio_mem$o getopt$o huffman_stdio$o sptree$o \
+    bitio_mems$o getopt1$o local_strings$o lovinstem$o \
+    timing$o error$o xmalloc$o random$o win32in$o \
+    simplefrenchstem$o unitool$o
+ 
+all : libmg.a
+
+libmg.a : $(OBJECTS)
+	$(DELETE) libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+install: 
+ 
+clean: 
+	$(DELETE) *$o
+	$(DELETE) libmg.a
Index: /main/tags/3.01/indexers/mg/lib/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,174 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DQUIET
+##########################################
+
+ 
+prefix = @prefix@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+ 
+INCLUDES = -I../../.. -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+    pathmax.h          getpagesize.h \
+    random.h           simplefrenchstem.h unitool.h  longlong.h
+# regex.h            rx.h               
+
+
+SOURCES = \
+    bitio_random.c    mgheap.c            memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c  \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    gmalloc.c         ftruncate.c       strcasecmp.c      random.c \
+    simplefrenchstem.c                  unitool.c
+# regex.c           rx.c              
+
+OBJECTS = @ALLOCA@ @LIBOBJS@ \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+#   gmalloc$o regex$o		rx$o 
+
+LIB_OBJECTS =   \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o          \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.a gsdllibmg.a
+
+libmg.a : $(OBJECTS)
+	rm -f libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+$(OBJECTS): ../config.h
+
+gsdllibmg.a: $(LIB_OBJECTS)
+	rm -f gsdllibmg.a
+	$(AR) cru gsdllibmg.a $(LIB_OBJECTS)
+	$(RANLIB) gsdllibmg.a
+ 
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmg.a gsdllibmg.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/indexers/mg/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/indexers/mg/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,91 @@
+###########################################################################
+#
+# win32 makefile -- mg\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN -DPARADOCNUM \
+       -DSHORT_SUFFIX
+
+INCLUDES = -I"." -I"$(MGHOME)"
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+ 
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+o = .obj
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+                       rx.h               pathmax.h          getpagesize.h \
+    random.h           win32in.h          simplefrenchstem.h unitool.h \
+    longlong.h	
+
+SOURCES = \
+    bitio_random.c    mgheap.c          memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c                            rx.c              \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    ftruncate.c       strcasecmp.c      random.c          win32in.c \
+    simplefrenchstem.c                  unitool.h
+
+OBJECTS =   rx$o \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o           \
+    error$o           xmalloc$o \
+    random$o          win32in$o         simplefrenchstem$o \
+    unitool$o
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.lib
+
+libmg.lib : $(OBJECTS)
+        if exist libmg.lib del libmg.lib
+	$(AR) /out:libmg.lib $(OBJECTS)
+
+install: 
+
+clean:
+        if exist *$o del *$o 
+        if exist libmg.lib del libmg.lib
Index: /main/tags/3.01/indexers/mg/lib/alloca.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/indexers/mg/lib/bitio_gen.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_gen.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_gen.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+
+
+
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BINARY_LENGTH (val, b, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Gamma_Length (unsigned long val)
+{
+  register unsigned long num;
+  GAMMA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  DELTA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  ELIAS_LENGTH (val, b, s, num);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BBLOCK_LENGTH (val, b, num);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  FLOORLOG_2 ((N - p) / p, logb);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  CEILLOG_2 (b, clogb);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+/* copied from mg-1.3f, 17-09-01 kjm18 */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  /* return ((int) (p * (2 * log2 ((double) N / p) + 1))); */
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  FLOORLOG_2 (b, logb);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  CEILLOG_2 (b, logb);
+  return logb;
+}
Index: /main/tags/3.01/indexers/mg/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,63 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_m.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_m.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_m.h	(revision 21112)
@@ -0,0 +1,543 @@
+/**************************************************************************
+ *
+ * bitio_m.h -- Macros for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_M
+#define H_BITIO_M
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#ifndef BIO_ENCODE_PROLOGUE
+#define BIO_ENCODE_PROLOGUE
+#endif
+
+#ifndef BIO_DECODE_PROLOGUE
+#define BIO_DECODE_PROLOGUE
+#endif
+
+#ifndef BIO_ENCODE_EPILOGUE
+#define BIO_ENCODE_EPILOGUE
+#endif
+
+#ifndef BIO_DECODE_EPILOGUE
+#define BIO_DECODE_EPILOGUE
+#endif
+
+#ifndef DECODE_ADD
+#define DECODE_ADD(b) (b) += (b) + DECODE_BIT
+#endif
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#define POSITIVE(f, x)							\
+  if ((x)<=0) 								\
+    fprintf(stderr,"Error: Cannot "#f" encode %lu\n",(unsigned long)x),exit(1);
+
+
+#define CEILLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x) - 1;						\
+  (v) = 0;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+#define FLOORLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x);						\
+  (v) = -1;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+/****************************************************************************/
+
+
+
+#define UNARY_ENCODE(x)							\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  (count) += _B_x;							\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE(x)							\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE_L(x, count)					\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  (count) += (x);							\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+
+#define UNARY_LENGTH(x, count)						\
+do {									\
+  POSITIVE(unary, x);							\
+  (count) = (x);							\
+} while(0)
+
+
+/****************************************************************************/
+
+
+#define BINARY_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  (count) += _B_nbits;							\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+
+#define BINARY_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      (count) += _B_logofb;					       	\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+          (count)++;							\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    _B_nbits = _B_logofb;						\
+  (count) = _B_nbits;							\
+} while(0)
+
+/****************************************************************************/
+
+
+
+
+#define GAMMA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  (count) += _B_logofb*2+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  (count) += _B_nbits*2+1;						\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  (count) = _B_logofb*2+1;						\
+} while (0)
+
+
+
+/****************************************************************************/
+
+
+#define DELTA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE(_B_logx+1);						\
+  BINARY_ENCODE(_B_xxx+1, 1<<_B_logx);					\
+} while (0)
+
+#define DELTA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE_L(_B_logx+1, count);					\
+  BINARY_ENCODE_L(_B_xxx+1, 1<<_B_logx, count);				\
+} while (0)
+
+
+#define DELTA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE(_B_logx); _B_logx--;					\
+  BINARY_DECODE(_B_xxx, 1<<_B_logx); _B_xxx--;				\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE_L(_B_logx, count); _B_logx--;				\
+  BINARY_DECODE_L(_B_xxx, 1<<_B_logx, count); _B_xxx--;			\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx, _B_dcount;					\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_LENGTH(_B_logx+1, count);					\
+  BINARY_LENGTH(_B_xxx+1, 1<<_B_logx, _B_dcount);			\
+  (count) += _B_dcount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+
+
+#define ELIAS_ENCODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE(_B_k+1);							\
+  BINARY_ENCODE(_B_xx-_B_lower+1, _B_upper-_B_lower+1);			\
+} while (0)
+
+#define ELIAS_ENCODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE_L(_B_k+1, count);					\
+  BINARY_ENCODE_L(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);	\
+} while (0)
+
+#define ELIAS_DECODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE(_B_k); _B_k--;						\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE(_B_xx, _B_upper-_B_lower+1); _B_xx--;			\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_DECODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE_L(_B_k, count); _B_k--;					\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE_L(_B_xx, _B_upper-_B_lower+1, count); _B_xx--;		\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_LENGTH(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0, _B_ecount;					\
+  register double _B_pow=1.0;						\
+  POSITIVE(gamma, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_LENGTH(_B_k+1, count);						\
+  BINARY_LENGTH(_B_xx-_B_lower+1, _B_upper-_B_lower+1, _B_ecount);	\
+  (count) += _B_ecount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+#define BBLOCK_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE(_B_xdivb+1);						\
+  BINARY_ENCODE(_B_xx+1, _B_bb);					\
+} while (0)
+
+#define BBLOCK_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE_L(_B_xdivb+1, count);					\
+  BINARY_ENCODE_L(_B_xx+1, _B_bb, count);				\
+} while (0)
+
+#define BBLOCK_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE(_B_xdivb); _B_xdivb--;					\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE(_B_x1, _B_bb);						\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE_L(_B_xdivb, count); _B_xdivb--;				\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE_L(_B_x1, _B_bb, count);					\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_bcount, _B_xx = (x);			\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_LENGTH(_B_xdivb+1, count);					\
+  BINARY_LENGTH(_B_xx+1, _B_bb, _B_bcount);				\
+  (count) += _B_bcount;  						\
+} while (0)
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,191 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+	    *__pos++ = __buff;						\
+	    __remaining--;						\
+	  }  								\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_FLUSH							\
+  do {									\
+    if (__btg != sizeof(__buff)*8)					\
+      if (__remaining)							\
+	{								\
+          *__pos++ = __buff;						\
+          __remaining--;						\
+	}  								\
+    __btg = sizeof(__buff)*8;						\
+  } while (0)
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0xff;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0x00;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (!__remaining ? 							\
+    (DECODE_ERROR, 0) :							\
+    (__buff = *__pos++, __remaining--, __btg = sizeof(__buff)*8, 	\
+     (__buff >> --__btg) & 1)))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    __pos = __base + (__new_pos >> 3);					\
+    __buff = *__pos++;							\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+#define ENCODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __base[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __base[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __pos++;								\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define ENCODE_FLUSH
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    (b) += (b) + (__base[__pos>>3] & (0x80 >> (__pos&7)) != 0);		\
+    __pos++;								\
+  } while(0)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_BIT 							\
+    (__pos++, ((__base[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define DECODE_SEEK(pos) __pos = (pos)
+
+#define ENCODE_SEEK(pos) __pos = (pos)
+
+
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,168 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+#define ENCODE_START(f,l)						\
+  {									\
+    register FILE *__file = f;						\
+    register unsigned char *__buf;					\
+    register unsigned long __pos = 0;					\
+    register unsigned long __base = 0;					\
+    register unsigned long __used = 0;					\
+    register unsigned long __len = (l)-1;				\
+    register unsigned long __sft = 0;					\
+    while (__len) { __sft++; __len >>=1; }				\
+    __len = 1<<__sft;							\
+    __buf = Xmalloc(__len);						\
+    fseek(__file, 0, 0);						\
+    fread(__buf, 1, __len, __file);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register FILE *__file = (b).File;					\
+    register unsigned char *__buf = (b).Buf;				\
+    register unsigned long __pos = (b).pos;				\
+    register unsigned long __base = (b).Base;				\
+    register unsigned long __used = (b).Used;				\
+    register unsigned long __len = (b).len;				\
+    register unsigned long __sft = (b).sft;
+
+#define SEEK fprintf(stderr, "Seek to %d\n",__base)
+#define READ fprintf(stderr, "Read of %d\n",__len)
+#define WRITE fprintf(stderr, "Write of %d\n",__used)
+
+#define WRITE_READ 							\
+	(__used ? (fseek(__file, __base, 0),				\
+                   fwrite(__buf, 1, __len, __file)) : 0,		\
+        __base += __len,						\
+	fseek(__file, __base, 0),					\
+        fread(__buf, 1, __len, __file),					\
+	__pos = 0, __used = 0)
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __buf[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __buf[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __used = 1;								\
+    __pos++;								\
+    if ((__pos>>3) >= __len)						\
+      (void)WRITE_READ;							\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __file;							\
+    (b).Buf = __buf;							\
+    (b).pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Used = __used;							\
+    (b).len = __len;							\
+    (b).sft = __sft;							\
+  }
+
+#define ENCODE_FLUSH							\
+  if (__used)								\
+    {									\
+      fseek(__file, __base, 0);						\
+      fwrite(__buf, 1, __len, __file);					\
+      __used = 0;							\
+    }
+
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+    free(__buf);							\
+  }
+
+
+#define DECODE_START(f,l) ENCODE_START(f,l)
+
+#define DECODE_CONTINUE(b) ENCODE_CONTINUE(b)
+
+#define DECODE_BIT 							\
+    (__pos++, (((__pos-1)>>3) >= __len ? WRITE_READ, __pos=1 : 0),	\
+     ((__buf[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_ADD_FF(b) ((b) += (b) + DECODE_BIT)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_DONE ENCODE_DONE
+
+#define DECODE_PAUSE(b)	ENCODE_PAUSE(b)
+
+#define ENCODE_SEEK(pos) 						\
+  do {									\
+    if ((((pos)>>3) >= __base) && ((((pos)+7)>>3) < (__base+__len)))   	\
+      __pos = (pos)-(__base << 3); 					\
+    else								\
+      {									\
+        ENCODE_FLUSH;							\
+        __base = (pos) >> 3;						\
+        __base = ((pos)  >> (__sft+3)) << __sft;			\
+        fseek(__file, __base, 0);					\
+        fread(__buf, 1, __len, __file);					\
+        __pos = (pos)&7; 						\
+        __pos = (pos)&((8 << __sft)-1);					\
+      }									\
+  } while(0)
+
+
+#define DECODE_SEEK(pos) ENCODE_SEEK(pos)
+
+#define ENCODE_TELL ((__base << 3) + __pos)
+
+#define DECODE_TELL ENCODE_TELL
+
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,147 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(f)							\
+  {									\
+    FILE *__outfile = f;						\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    FILE *__outfile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	putc(__buff, __outfile);					\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __outfile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_DONE							\
+    if (__btg != sizeof(__buff)*8)					\
+      putc(__buff,__outfile);						\
+  }
+
+
+#define DECODE_START(f)							\
+  {									\
+    FILE *__infile = f;							\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    FILE *__infile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (__buff = getc(__infile), (feof(__infile) ?				\
+    (DECODE_ERROR, 0) :							\
+    (__btg = sizeof(__buff)*8, (__buff >> --__btg) & 1))))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).File = __infile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    fseek(__infile, __new_pos >> 3, 0);					\
+    __buff = getc(__infile);						\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_mem.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_mem.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_mem.c	(revision 21112)
@@ -0,0 +1,259 @@
+/**************************************************************************
+ *
+ * bitio_mem.c -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  ENCODE_START (buf, rem)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mem_Encode_Done (mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  DECODE_START (buf, rem)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Elias_Decode (unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/indexers/mg/lib/bitio_mem.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_mem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_mem.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * bitio_mem.h -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are faster than the ones 
+ *  in "mems" files. But with these routines you cannot mix reads and writes 
+ *  on the array of chars, or multiple write, at the same time and guarantee
+ *  them to work, also you cannot seek to a point and do a write. The decode
+ *  routine can detect when you run off the end of the array and will produce
+ *  an approate error message, and the encode routine will stop when it gets
+ *  to the end of the character array. 
+ *
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_MEM
+#define H_BITIO_MEM
+
+
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs);
+void BIO_Mem_Encode_Done (mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+				    unsigned long *bits);
+
+
+
+void BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+			   mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Elias_Decode (unsigned long b, double s,
+				 mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+void BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_mems.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_mems.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_mems.c	(revision 21112)
@@ -0,0 +1,268 @@
+/**************************************************************************
+ *
+ * bitio_mems.c -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mems.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mems_Encode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  ENCODE_START (buf, pos)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Done (mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mems_Decode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  DECODE_START (buf, pos)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Elias_Decode (unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/indexers/mg/lib/bitio_mems.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_mems.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_mems.h	(revision 21112)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * bitio_mems.h -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_MEMS
+#define H_BITIO_MEMS
+
+
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mems_Encode_Start (char *buf, unsigned long pos, mems_bitio_state * bs);
+void BIO_Mems_Encode_Done (mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Decode_Start (void *buf, unsigned long pos, mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+			    mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Elias_Decode (unsigned long b, double s,
+				mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_random.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_random.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_random.c	(revision 21112)
@@ -0,0 +1,331 @@
+/*************************************************************************
+ *
+ * bitio_random.c -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:45:03  kjm18
+   updated mg to be in line with version mg-1.3f. Now uses long long for
+   some variables to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:16:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:50  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:40  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "longlong.h"
+
+#include "bitio_m_random.h"
+#include "bitio_m.h"
+
+
+void 
+BIO_Random_Start (FILE * f, unsigned long len,
+		  random_bitio_state * bs)
+{
+  ENCODE_START (f, len)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Random_Done (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+void 
+BIO_Random_Encode_Bit (int bit, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+int 
+BIO_Random_Decode_Bit (random_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+void 
+BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Unary_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Binary_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Gamma_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Delta_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			 random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Elias_Decode (unsigned long b, double s,
+			 random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Bblock_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Seek (unsigned long pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
+void 
+BIO_Random_Flush (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_FLUSH;
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Random_Tell (random_bitio_state * bs)
+{
+  register unsigned long t;
+  ENCODE_CONTINUE (*bs)
+    t = ENCODE_TELL;
+  ENCODE_PAUSE (*bs)
+    return (t);
+}
+
+#ifdef USE_LONG_LONG
+
+void 
+BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE(*bs)
+
+    if ((((pos) >> 3) >= (mg_ullong)__base) &&
+	((((pos)+7) >> 3) < (mg_ullong)(__base + __len)))
+      {
+	__pos = (long)((pos) - (mg_ullong)(__base << 3));
+      }
+    else
+      {
+        ENCODE_FLUSH;
+	__base = (long)(((pos) >> (__sft+3)) << __sft);
+	
+	fseek(__file,__base,0);
+	fread(__buf,1,__len,__file);
+	__pos = (long)((pos) & ((8 << __sft)-1));
+      }
+  
+  ENCODE_PAUSE(*bs)
+}
+
+mg_ullong 
+BIO_Random_Tell_LL (random_bitio_state * bs)
+{
+  mg_ullong t;
+  ENCODE_CONTINUE(*bs)
+    t = (((mg_ullong)__base) << 3ull) + __pos;
+  ENCODE_PAUSE(*bs)
+  return(t);  
+}
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_random.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_random.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_random.h -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ *  Modified:
+ *   - long long seek and tell ops
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ **************************************************************************/
+
+#ifndef H_BITIO_RANDOM
+#define H_BITIO_RANDOM
+
+#include "longlong.h"
+
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Random_Start (FILE * f, unsigned long len,
+		       random_bitio_state * bs);
+void BIO_Random_Done (random_bitio_state * bs);
+
+
+
+void BIO_Random_Decode_Start (void *buf, unsigned long pos,
+			      random_bitio_state * bs);
+
+void BIO_Random_Encode_Bit (int bit, random_bitio_state * bs);
+
+int BIO_Random_Decode_Bit (random_bitio_state * bs);
+
+
+void BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Unary_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Binary_Decode (unsigned long b, random_bitio_state * bs,
+					unsigned long *bits);
+
+
+
+void BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Gamma_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Delta_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			      random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Elias_Decode (unsigned long b, double s,
+				       random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Bblock_Decode (unsigned long b,
+					random_bitio_state * bs,
+					unsigned long *bits);
+
+
+void BIO_Random_Seek (unsigned long pos, random_bitio_state * bs);
+
+void BIO_Random_Flush (random_bitio_state * bs);
+
+unsigned long BIO_Random_Tell (random_bitio_state * bs);
+
+#ifdef USE_LONG_LONG
+
+void BIO_Random_Seek_LL (mg_ullong pos, random_bitio_state * bs);
+mg_ullong BIO_Random_Tell_LL (random_bitio_state * bs);
+
+#endif /* USE_LONG_LONG */
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/bitio_stdio.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_stdio.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_stdio.c	(revision 21112)
@@ -0,0 +1,284 @@
+/**************************************************************************
+ *
+ * bitio_stdio.c -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:41  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_stdio.h"
+#include "bitio_m.h"
+
+
+
+void 
+BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  ENCODE_START (f)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Done (stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  DECODE_START (f)
+    DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+
+int 
+BIO_Stdio_Decode_Bit (stdio_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Elias_Decode (unsigned long b, double s,
+			stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+void 
+BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /main/tags/3.01/indexers/mg/lib/bitio_stdio.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/bitio_stdio.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/bitio_stdio.h	(revision 21112)
@@ -0,0 +1,106 @@
+/**************************************************************************
+ *
+ * bitio_stdio.h -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of number on a FILE*. With these function you cannot mix reads and writes
+ *  on the FILE, or multiple writes, at the same time and guarantee them to 
+ *  work, also you  cannot seek to a point and do a write. The decode function
+ *  can detect when you run off the end of the file and will produce an 
+ *  approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_STDIO
+#define H_BITIO_STDIO
+
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs);
+void BIO_Stdio_Encode_Done (stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs);
+int BIO_Stdio_Decode_Bit (stdio_bitio_state * bs);
+
+
+void BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			     stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Elias_Decode (unsigned long b, double s,
+				      stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/error.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/error.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/error.c	(revision 21112)
@@ -0,0 +1,127 @@
+/* error.c -- error handler for noninteractive utilities
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by David MacKenzie <djm@gnu.ai.mit.edu>.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <stdio.h>
+
+#if HAVE_VPRINTF || HAVE_DOPRNT
+# if __STDC__
+#  include <stdarg.h>
+#  define VA_START(args, lastarg) va_start(args, lastarg)
+# else
+#  include <varargs.h>
+#  define VA_START(args, lastarg) va_start(args)
+# endif
+#else
+# define va_alist a1, a2, a3, a4, a5, a6, a7, a8
+# define va_dcl char *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8;
+#endif
+
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <string.h>
+#else
+void exit ();
+#endif
+
+/* If NULL, error will flush stdout, then print on stderr the program
+   name, a colon and a space.  Otherwise, error will call this
+   function without parameters instead.  */
+void (*error_print_progname) () = NULL;
+
+/* The calling program should define program_name and set it to the
+   name of the executing program.  */
+#if 0 /* [RB/TS:10/95] do not use program_name */
+extern char *program_name;
+#endif
+
+#if HAVE_STRERROR
+char *strerror (int errnum);
+#else
+static char *
+private_strerror (errnum)
+     int errnum;
+{
+  extern char *sys_errlist[];
+  extern int sys_nerr;
+
+  if (errnum > 0 && errnum <= sys_nerr)
+    return sys_errlist[errnum];
+  return "Unknown system error";
+}
+#define strerror private_strerror
+#endif
+
+/* Print the program name and error message MESSAGE, which is a printf-style
+   format string with optional args.
+   If ERRNUM is nonzero, print its corresponding system error message.
+   Exit with status STATUS if it is nonzero.  */
+/* VARARGS */
+
+void
+#if defined(VA_START) && __STDC__
+error (int status, int errnum, const char *message, ...)
+#else
+error (status, errnum, message, va_alist)
+     int status;
+     int errnum;
+     char *message;
+     va_dcl
+#endif
+{
+#ifdef VA_START
+  va_list args;
+#endif
+
+  if (error_print_progname)
+    (*error_print_progname) ();
+  else
+    {
+      fflush (stdout);
+#if 0 /* [RB/TS:10/95] do not use program_name */
+      fprintf (stderr, "%s: ", program_name);
+#endif
+    }
+
+#ifdef VA_START
+  VA_START (args, message);
+# if HAVE_VPRINTF
+  vfprintf (stderr, message, args);
+# else
+  _doprnt (message, args, stderr);
+# endif
+  va_end (args);
+#else
+  fprintf (stderr, message, a1, a2, a3, a4, a5, a6, a7, a8);
+#endif
+
+  if (errnum)
+    fprintf (stderr, ": %s", strerror (errnum));
+  putc ('\n', stderr);
+  fflush (stderr);
+  if (status)
+    exit (status);
+}
Index: /main/tags/3.01/indexers/mg/lib/filestats.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/filestats.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/filestats.c	(revision 21112)
@@ -0,0 +1,136 @@
+/**************************************************************************
+ *
+ * filestats.c -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:54  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:42  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+File *
+Fopen (char *name, char *mode, unsigned long magic)
+{
+  FILE *f;
+  File *F;
+  unsigned long m;
+  if (!(f = fopen (name, mode)))
+    return (NULL);
+  if (magic)
+    switch (*mode)
+      {
+      case 'r':
+	fread ((char *) &m, sizeof (m), 1, f);
+	NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (m != magic)
+	  {
+	    fclose (f);
+	    return (NULL);
+	  }
+	break;
+      case 'w':
+	HTONUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+	fwrite ((char *) &magic, sizeof (magic), 1, f);
+      }
+  if (!(F = Xmalloc (sizeof (File))))
+    {
+      fclose (f);
+      return (NULL);
+    }
+  F->pathname = Xstrdup (name);
+  F->name = strrchr (F->pathname, '/');
+  F->name = F->name ? F->name + 1 : F->pathname;
+  F->f = f;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+  F->Cumulative = F->Current;
+  return (F);
+}
+
+
+int 
+Fclose (File * F)
+{
+  int num;
+  if (!F)
+    return (0);
+  num = fclose (F->f);
+  if (F->pathname)
+    Xfree (F->pathname);
+  Xfree (F);
+  return (num);
+}
+
+size_t 
+Fread (void *ptr, size_t size, size_t nitems, File * F)
+{
+  int num;
+  num = fread ((char *) ptr, size, nitems, F->f);
+  F->Current.NumReads++;
+  F->Current.NumBytes += num * size;
+  return (num);
+}
+
+int 
+Fseek (File * F, long offset, int ptrname)
+{
+  int num;
+  num = fseek (F->f, offset, ptrname);
+  F->Current.NumSeeks++;
+  return (num);
+}
+
+void 
+Rewind (File * F)
+{
+  rewind (F->f);
+  F->Current.NumSeeks++;
+}
+
+
+void 
+ZeroFileStats (File * F)
+{
+  F->Cumulative.NumSeeks += F->Current.NumSeeks;
+  F->Cumulative.NumReads += F->Current.NumReads;
+  F->Cumulative.NumBytes += F->Current.NumBytes;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+}
Index: /main/tags/3.01/indexers/mg/lib/filestats.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/filestats.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/filestats.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filestats.h -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_FILESTATS
+#define H_FILESTATS
+
+#include "sysfuncs.h"
+
+typedef struct FileStats
+  {
+    unsigned long NumSeeks;
+    unsigned long NumReads;
+    unsigned long NumBytes;
+  }
+FileStats;
+
+
+
+typedef struct
+  {
+    FILE *f;
+    char *pathname;
+    char *name;
+    FileStats Current;
+    FileStats Cumulative;
+  }
+File;
+
+/* if magic is 0 no magic number is read or written */
+File *Fopen (char *name, char *mode, unsigned long magic);
+
+size_t Fread (void *ptr, size_t size, size_t nitems, File * F);
+
+int Fseek (File * F, long offset, int ptrname);
+
+void Rewind (File * F);
+
+int Fclose (File * F);
+
+#define Getc(F) (F->Current.NumReads++, F->Current.NumBytes++, getc(F->f))
+
+/* This adds the Current file stats to the Cumulative stats and then zeros
+   the Current stats */
+void ZeroFileStats (File * F);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/ftruncate.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/ftruncate.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/ftruncate.c	(revision 21112)
@@ -0,0 +1,107 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/indexers/mg/lib/getopt.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/getopt.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/getopt.c	(revision 21112)
@@ -0,0 +1,750 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "getopt.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/indexers/mg/lib/getopt.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/getopt.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/getopt.h	(revision 21112)
@@ -0,0 +1,130 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  extern int getopt ();
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
Index: /main/tags/3.01/indexers/mg/lib/getopt1.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/getopt1.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/getopt1.c	(revision 21112)
@@ -0,0 +1,178 @@
+/* getopt_long and getopt_long_only entry points for GNU getopt.
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#include "getopt.h"
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef __GNU_LIBRARY__
+#include <stdlib.h>
+#else
+char *getenv ();
+#endif
+
+#ifndef	NULL
+#define NULL 0
+#endif
+
+int
+getopt_long (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 0);
+}
+
+/* Like getopt_long, but '-' as well as '--' can indicate a long option.
+   If an option that starts with '-' (not '--') doesn't match a long option,
+   but does match a short option, it is parsed as a short option
+   instead.  */
+
+int
+getopt_long_only (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 1);
+}
+
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+#include <stdio.h>
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+      int option_index = 0;
+      static struct option long_options[] =
+      {
+	{"add", 1, 0, 0},
+	{"append", 0, 0, 0},
+	{"delete", 1, 0, 0},
+	{"verbose", 0, 0, 0},
+	{"create", 0, 0, 0},
+	{"file", 1, 0, 0},
+	{0, 0, 0, 0}
+      };
+
+      c = getopt_long (argc, argv, "abc:d:0123456789",
+		       long_options, &option_index);
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case 0:
+	  printf ("option %s", long_options[option_index].name);
+	  if (optarg)
+	    printf (" with arg %s", optarg);
+	  printf ("\n");
+	  break;
+
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case 'd':
+	  printf ("option d with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/indexers/mg/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/indexers/mg/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/indexers/mg/lib/huffman.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman.c	(revision 21112)
@@ -0,0 +1,426 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:44  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = Xmalloc (sizeof (huff_data))))
+	goto error0;
+    }
+  if (!(heap = Xmalloc (num * 2 * sizeof (*heap))))
+    goto error1;
+  if (!(hd->clens = Xmalloc (num * sizeof (char))))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  bcopy ((char *) freqs, (char *) &heap[num], num * sizeof (*heap));
+  bzero ((char *) heap, num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; i++)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; i--)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    child++;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; i++)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		child++;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	HNum++;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; i++)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  bzero ((char *) hd->lencount, sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; i++)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      hd->lencount[codelen]++;
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; i--)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  free (heap);
+  return (hd);
+
+error2:
+  free (heap);
+error1:
+  if (!data)
+    free (hd);
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = Xmalloc (data->num_codes * sizeof (*codes))))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  bcopy ((char *) data->min_code, (char *) mc, sizeof (mc));
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = Xmalloc (data->num_codes * sizeof (*vals))))
+    return (NULL);
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *))))
+    {
+      free (vals);
+      return (NULL);
+    }
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; i++)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; i++)
+    size += counts[i] * hd.clens[i];
+  Xfree (hd.clens);
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+
+static int 
+General_Read_Huffman_Data (size_t (*rd) (), void *f,
+			   huff_data * hd, u_long * mem, u_long * disk)
+{
+  if (rd (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      bzero ((char *) (hd->lencount), sizeof (hd->lencount));
+      if (rd (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      bzero ((char *) (hd->min_code), sizeof (hd->min_code));
+
+      if (rd (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = Xmalloc (sizeof (*hd->clens) * hd->num_codes)))
+	return -1;
+
+
+      if (rd (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
+
+
+
+int 
+Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (fread, f, hd, mem, disk);
+}
+
+
+
+
+int 
+F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (Fread, f, hd, mem, disk);
+}
Index: /main/tags/3.01/indexers/mg/lib/huffman.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+
+
+#define MAX_HUFFCODE_LEN 31
+
+typedef struct huff_data
+  {
+    int num_codes;
+    int mincodelen;
+    int maxcodelen;
+    int lencount[MAX_HUFFCODE_LEN + 1];
+    unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+    char *clens;
+  }
+huff_data;
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+int F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+
+#define HUFF_ENCODE(x, codes, lens)					\
+  do {									\
+    register int __i;							\
+    register int __clen = (lens)[x];					\
+    register unsigned long __code = (codes)[x];				\
+    for (__i=__clen-1; __i>=0; __i--)					\
+      ENCODE_BIT((__code >> __i) & 1);					\
+  } while(0)
+
+#define HUFF_ENCODE_L(x, codes, lens, count)				\
+  do {									\
+    HUFF_ENCODE(x, codes, lens);		   			\
+    (count) += (lens)[x];						\
+  } while(0)
+
+
+#define HUFF_DECODE(x, mcodes, values)					\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+#define HUFF_DECODE_L(x, mcodes, values, count)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+	(count)++;							\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/huffman_mem.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman_mem.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman_mem.c	(revision 21112)
@@ -0,0 +1,69 @@
+/**************************************************************************
+ *
+ * huffman_mem.c -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:45  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+#include "bitio_m_mem.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+#endif
+
+void 
+BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+		     char *clens, mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Mem_Huff_Decode (unsigned long *mincodes,
+		     unsigned long **values, mem_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/indexers/mg/lib/huffman_mem.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman_mem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman_mem.h	(revision 21112)
@@ -0,0 +1,36 @@
+/**************************************************************************
+ *
+ * huffman_mem.h -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_MEM
+#define H_HUFFMAN_MEM
+
+
+
+void BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+			  char *clens, mem_bitio_state * bs);
+
+unsigned long BIO_Mem_Huff_Decode (unsigned long *mincodes,
+			      unsigned long **values, mem_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/huffman_stdio.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman_stdio.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman_stdio.c	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * huffman_stdio.c -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:52  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:07  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:46  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+int _filbuf (FILE *);
+int _flsbuf (unsigned, FILE *);
+#endif
+
+void 
+BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+		       char *clens, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+		       unsigned long **values,
+		       stdio_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /main/tags/3.01/indexers/mg/lib/huffman_stdio.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/huffman_stdio.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/huffman_stdio.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * huffman_stdio.h -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_STDIO
+#define H_HUFFMAN_STDIO
+
+
+
+void BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+			    char *clens, stdio_bitio_state * bs);
+
+
+unsigned long BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+				     unsigned long **values,
+				     stdio_bitio_state * bs);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/local_strings.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/local_strings.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/local_strings.c	(revision 21112)
@@ -0,0 +1,415 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = Xmalloc (src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  bcopy ((char *) src_str, (char *) (copy), src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    i++;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  bzero (buf, sizeof (buf));
+  for (i = 0; i < len; i++)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  s++;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      s--;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; i++)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/indexers/mg/lib/local_strings.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/longlong.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/longlong.h	(revision 21112)
@@ -0,0 +1,81 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifndef __WIN32__
+#include "config.h"
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+	mgstat.c
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/lovinstem.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/lovinstem.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/lovinstem.c	(revision 21112)
@@ -0,0 +1,1187 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();
+
+static int 
+aio (ch)			/* Rule number 9     */
+     char ch;
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (ch)				/* Rule Number 24    */
+     char ch;
+{
+  return (ch != 's');
+}
+
+static int 
+pt (ch)				/* Rule number 28    */
+     char ch;
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (ch)				/* Rule number 30    */
+     char ch;
+{
+  return (ch != 'm');
+}
+
+static int 
+n (ch)				/* Rule number 32    */
+     char ch;
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+
+static int 
+B (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'e');
+}
+
+static int 
+F (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'c');
+}
+
+static int 
+W (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  bcopy ((char *) word, (char *) copy, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  bcopy ((char *) copy, (char *) word, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) ();		/* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  rule++;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) ();		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      suffix_start++;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/indexers/mg/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,56 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:36  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:32:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:20:10  tes
+   * Changes made for 1.1
+   *
+   *
+ */
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/memlib.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/memlib.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/memlib.c	(revision 21112)
@@ -0,0 +1,66 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/11/29 03:15:13  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:47  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+/* change the name so we do not have problems with other libs */
+char *
+my_mg_strdup(const char *str)
+{
+  char *ret_str = malloc(strlen(str)+1);
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_mg_strdup;
Index: /main/tags/3.01/indexers/mg/lib/memlib.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/memlib.h	(revision 21112)
@@ -0,0 +1,51 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+#ifdef STD_MEM
+
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+#define Xstrdup strdup
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/messages.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/messages.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/messages.c	(revision 21112)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/23 03:38:06  rjmcnab
+
+   combined unix and windows versions of mg
+
+   Revision 1.1  1998/11/19 03:00:32  sjboddie
+   Added windows gdbm and mg versions
+
+   * Revision 1.1  1994/08/22  00:24:48  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef QUIET
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/indexers/mg/lib/messages.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/messages.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+
+void Message (const char *fmt,...);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/mgheap.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/mgheap.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/mgheap.c	(revision 21112)
@@ -0,0 +1,173 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1999/01/15 03:05:50  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.1  1998/11/17 09:32:01  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:43  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; __i--)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; __i--)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	child++;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; i--)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; i--)
+    {
+      SWAP (size, heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)--;
+  SWAP (size, heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)++;
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/indexers/mg/lib/mgheap.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,84 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/indexers/mg/lib/netorder.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/netorder.h	(revision 21112)
@@ -0,0 +1,178 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &hd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &hd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &nd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &nd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &nd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &hd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &hd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define NTOHF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define HTONF2(hf, nf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(hf), (char *) &tmp, sizeof (float));                \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(nf), sizeof (float));                \
+	}while(0)
+#define NTOHF2(nf, hf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(nf), (char *) &tmp, sizeof (float));                \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(hf), sizeof (float));                \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/indexers/mg/lib/pathmax.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/indexers/mg/lib/perf_hash.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/perf_hash.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/perf_hash.c	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/02/22 01:35:03  jrm21
+   Fixed bug that caused perf_hash to fail for lexicon of < 4 words.
+   Added a #define to increase size of n (for vertices in the graph) as we
+   need to allocate more memory somewhere else before calling tree_builder()
+
+   Revision 1.1  1999/08/10 21:16:57  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:17  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/08/22  00:26:30  tes
+   * Made perf_hash take the ceiling of 1.23 * num
+   * in lieu of the floor.
+   * This was done so that perf_hash works ok on
+   * a small number of keys.
+   *
+   * Revision 1.1  1994/08/22  00:24:49  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define STRUCT
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+/* An arbitrary amount to add to both MAX_N and n for building tree/acyclic
+   graph -- John McPherson - jrm21 (22-02-2001) */
+#define MAX_N_HEADROOM  100
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  StackPos++;
+  while (StackPos)
+    {
+      int m, b;
+      StackPos--;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; e++)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; v++)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      /* Added variable - need to increase MAX_N on allocate as well */
+      *n = (int) (num * c + MAX_N_HEADROOM); /* john mcpherson 22-02-2001 */
+
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; i++)
+	for (j = 0; j < MAX_L; j++)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; e++)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      w++;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; e++)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; i++)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  bzero ((char *) translate, sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; i++)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, s++)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; i++)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; i++)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = Xmalloc (sizeof (int) * MAX_M);
+  mk = Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    if (tb)
+      if (!(tb[i] = Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = Xmalloc (sizeof (u_char) * 256);
+  bzero ((char *) translate, sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  /* For extremely small lexicons (<4 words), this isn't good enough! 
+   -- John McPherson jrm21 22-02-2001 */
+  /* MAX_N = MAX_M + MAX_M / 4; */
+  MAX_N = (int)(ceil (1.25 * MAX_M)); 
+  /* Add some space for "experimentation" of n in tree_builder() */
+  MAX_N += MAX_N_HEADROOM;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  bcopy ((char *) &g[1], (char *) g, n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = Xmalloc (sizeof (u_char) * 256);
+  phd->g = Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb)
+      if (!(phd->tb[i] = Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      l--;
+      s++;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/indexers/mg/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+
+
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/random.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/random.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/random.c	(revision 21112)
@@ -0,0 +1,228 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:14:16  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:16:57  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:20  rjmcnab
+*** empty log message ***
+
+*/
+
+static char *RCSID = "$Id$";
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/indexers/mg/lib/random.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/random.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/random.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:14:16  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:16:58  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:21  rjmcnab
+*** empty log message ***
+
+*/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/regex.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/regex.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/regex.c	(revision 21112)
@@ -0,0 +1,5426 @@
+/* Extended regular expression matching and search library,
+   version 0.12.
+   (Implements POSIX draft P10003.2/D11.2, except for
+   internationalization features.)
+
+   Copyright (C) 1993, 1994, 1995 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* AIX requires this to be the first thing in the file. */
+#if defined (_AIX) && !defined (REGEX_MALLOC)
+  #pragma alloca
+#endif
+
+#define _GNU_SOURCE
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+/* We need this for `regex.h', and perhaps for the Emacs include files.  */
+#include <sys/types.h>
+
+/* This is for other GNU distributions with internationalized messages.  */
+#if HAVE_LIBINTL_H || defined (_LIBC)
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+/* The `emacs' switch turns on certain matching commands
+   that make sense only in Emacs. */
+#ifdef emacs
+
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+
+#else  /* not emacs */
+
+/* If we are not linking with Emacs proper,
+   we can't use the relocating allocator
+   even if sysfuncs.h says that we can.  */
+#undef REL_ALLOC
+
+#if defined (STDC_HEADERS) || defined (_LIBC)
+#include <stdlib.h>
+#else
+char *malloc ();
+char *realloc ();
+#endif
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+   `BSTRING', as far as I know, and neither of them use this code.  */
+#ifndef INHIBIT_STRING_HEADER
+#if HAVE_STRING_H || STDC_HEADERS || defined (_LIBC)
+#include <string.h>
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+#else
+#include <strings.h>
+#endif
+#endif
+
+/* Define the syntax stuff for \<, \>, etc.  */
+
+/* This must be nonzero for the wordchar and notwordchar pattern
+   commands in re_match_2.  */
+#ifndef Sword 
+#define Sword 1
+#endif
+
+#ifdef SWITCH_ENUM_BUG
+#define SWITCH_ENUM_CAST(x) ((int)(x))
+#else
+#define SWITCH_ENUM_CAST(x) (x)
+#endif
+
+#ifdef SYNTAX_TABLE
+
+extern char *re_syntax_table;
+
+#else /* not SYNTAX_TABLE */
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE 256
+
+static char re_syntax_table[CHAR_SET_SIZE];
+
+static void
+init_syntax_once ()
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+
+#endif /* not SYNTAX_TABLE */
+
+#define SYNTAX(c) re_syntax_table[c]
+
+#endif /* not emacs */
+
+
+/* Get the interface, including the syntax bits.  */
+#include "regex.h"
+
+/* isalpha etc. are used for the character classes.  */
+#include <ctype.h>
+
+/* Jim Meyering writes:
+
+   "... Some ctype macros are valid only for character codes that
+   isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when
+   using /bin/cc or gcc but without giving an ansi option).  So, all
+   ctype uses should be through macros like ISPRINT...  If
+   STDC_HEADERS is defined, then autoconf has verified that the ctype
+   macros don't need to be guarded with references to isascii. ...
+   Defining isascii to 1 should let any compiler worth its salt
+   eliminate the && through constant folding."  */
+
+#ifndef ISASCII
+#  if defined (STDC_HEADERS) || (!defined (isascii) && !defined (HAVE_ISASCII))
+#    define ISASCII(c) 1
+#  else
+#    define ISASCII(c) isascii(c)
+#  endif
+#endif
+
+#ifdef isblank
+#define ISBLANK(c) (ISASCII (c) && isblank (c))
+#else
+#define ISBLANK(c) ((c) == ' ' || (c) == '\t')
+#endif
+#ifdef isgraph
+#define ISGRAPH(c) (ISASCII (c) && isgraph (c))
+#else
+#define ISGRAPH(c) (ISASCII (c) && isprint (c) && !isspace (c))
+#endif
+
+#define ISPRINT(c) (ISASCII (c) && isprint (c))
+#define ISDIGIT(c) (ISASCII (c) && isdigit (c))
+#define ISALNUM(c) (ISASCII (c) && isalnum (c))
+#define ISALPHA(c) (ISASCII (c) && isalpha (c))
+#define ISCNTRL(c) (ISASCII (c) && iscntrl (c))
+#define ISLOWER(c) (ISASCII (c) && islower (c))
+#define ISPUNCT(c) (ISASCII (c) && ispunct (c))
+#define ISSPACE(c) (ISASCII (c) && isspace (c))
+#define ISUPPER(c) (ISASCII (c) && isupper (c))
+#define ISXDIGIT(c) (ISASCII (c) && isxdigit (c))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* We remove any previous definition of `SIGN_EXTEND_CHAR',
+   since ours (we hope) works properly with all combinations of
+   machines, compilers, `char' and `unsigned char' argument types.
+   (Per Bothner suggested the basic approach.)  */
+#undef SIGN_EXTEND_CHAR
+#if __STDC__
+#define SIGN_EXTEND_CHAR(c) ((signed char) (c))
+#else  /* not __STDC__ */
+/* As in Harbison and Steele.  */
+#define SIGN_EXTEND_CHAR(c) ((((unsigned char) (c)) ^ 128) - 128)
+#endif
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+   use `alloca' instead of `malloc'.  This is because using malloc in
+   re_search* or re_match* could cause memory leaks when C-g is used in
+   Emacs; also, malloc is slower and causes storage fragmentation.  On
+   the other hand, malloc is more portable, and easier to debug.  
+   
+   Because we sometimes use alloca, some routines have to be macros,
+   not functions -- `alloca'-allocated space disappears at the end of the
+   function it is called in.  */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE malloc
+#define REGEX_REALLOCATE(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE free
+
+#else /* not REGEX_MALLOC  */
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+#define REGEX_ALLOCATE alloca
+
+/* Assumes a `char *destination' variable.  */
+#define REGEX_REALLOCATE(source, osize, nsize)				\
+  (destination = (char *) alloca (nsize),				\
+   bcopy (source, destination, osize),					\
+   destination)
+
+/* No need to do anything to free, after alloca.  */
+#define REGEX_FREE(arg) ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+
+#endif /* not REGEX_MALLOC */
+
+/* Define how to allocate the failure stack.  */
+
+#ifdef REL_ALLOC
+#define REGEX_ALLOCATE_STACK(size)				\
+  r_alloc (&failure_stack_ptr, (size))
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)		\
+  r_re_alloc (&failure_stack_ptr, (nsize))
+#define REGEX_FREE_STACK(ptr)					\
+  r_alloc_free (&failure_stack_ptr)
+
+#else /* not REL_ALLOC */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE_STACK malloc
+#define REGEX_REALLOCATE_STACK(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE_STACK free
+
+#else /* not REGEX_MALLOC */
+
+#define REGEX_ALLOCATE_STACK alloca
+
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)			\
+   REGEX_REALLOCATE (source, osize, nsize)
+/* No need to explicitly free anything.  */
+#define REGEX_FREE_STACK(arg)
+
+#endif /* not REGEX_MALLOC */
+#endif /* not REL_ALLOC */
+
+
+/* True if `size1' is non-NULL and PTR is pointing anywhere inside
+   `string1' or just past its end.  This works if PTR is NULL, which is
+   a good thing.  */
+#define FIRST_STRING_P(ptr) 					\
+  (size1 && string1 <= (ptr) && (ptr) <= string1 + size1)
+
+/* (Re)Allocate N items of type T using malloc, or fail.  */
+#define TALLOC(n, t) ((t *) malloc ((n) * sizeof (t)))
+#define RETALLOC(addr, n, t) ((addr) = (t *) realloc (addr, (n) * sizeof (t)))
+#define RETALLOC_IF(addr, n, t) \
+  if (addr) RETALLOC((addr), (n), t); else (addr) = TALLOC ((n), t)
+#define REGEX_TALLOC(n, t) ((t *) REGEX_ALLOCATE ((n) * sizeof (t)))
+
+#define BYTEWIDTH 8 /* In bits.  */
+
+#define STREQ(s1, s2) ((strcmp (s1, s2) == 0))
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+static int re_match_2_internal ();
+
+
+/* These are the command codes that appear in compiled regular
+   expressions.  Some opcodes are followed by argument bytes.  A
+   command code can specify any interpretation whatsoever for its
+   arguments.  Zero bytes may appear in the compiled regular expression.  */
+
+typedef enum
+{
+  no_op = 0,
+
+  /* Succeed right away--no more backtracking.  */
+  succeed,
+
+        /* Followed by one byte giving n, then by n literal bytes.  */
+  exactn,
+
+        /* Matches any (more or less) character.  */
+  anychar,
+
+        /* Matches any one char belonging to specified set.  First
+           following byte is number of bitmap bytes.  Then come bytes
+           for a bitmap saying which chars are in.  Bits in each byte
+           are ordered low-bit-first.  A character is in the set if its
+           bit is 1.  A character too large to have a bit in the map is
+           automatically not in the set.  */
+  charset,
+
+        /* Same parameters as charset, but match any character that is
+           not one of those specified.  */
+  charset_not,
+
+        /* Start remembering the text that is matched, for storing in a
+           register.  Followed by one byte with the register number, in
+           the range 0 to one less than the pattern buffer's re_nsub
+           field.  Then followed by one byte with the number of groups
+           inner to this one.  (This last has to be part of the
+           start_memory only because we need it in the on_failure_jump
+           of re_match_2.)  */
+  start_memory,
+
+        /* Stop remembering the text that is matched and store it in a
+           memory register.  Followed by one byte with the register
+           number, in the range 0 to one less than `re_nsub' in the
+           pattern buffer, and one byte with the number of inner groups,
+           just like `start_memory'.  (We need the number of inner
+           groups here because we don't have any easy way of finding the
+           corresponding start_memory when we're at a stop_memory.)  */
+  stop_memory,
+
+        /* Match a duplicate of something remembered. Followed by one
+           byte containing the register number.  */
+  duplicate,
+
+        /* Fail unless at beginning of line.  */
+  begline,
+
+        /* Fail unless at end of line.  */
+  endline,
+
+        /* Succeeds if at beginning of buffer (if emacs) or at beginning
+           of string to be matched (if not).  */
+  begbuf,
+
+        /* Analogously, for end of buffer/string.  */
+  endbuf,
+ 
+        /* Followed by two byte relative address to which to jump.  */
+  jump, 
+
+	/* Same as jump, but marks the end of an alternative.  */
+  jump_past_alt,
+
+        /* Followed by two-byte relative address of place to resume at
+           in case of failure.  */
+  on_failure_jump,
+	
+        /* Like on_failure_jump, but pushes a placeholder instead of the
+           current string position when executed.  */
+  on_failure_keep_string_jump,
+  
+        /* Throw away latest failure point and then jump to following
+           two-byte relative address.  */
+  pop_failure_jump,
+
+        /* Change to pop_failure_jump if know won't have to backtrack to
+           match; otherwise change to jump.  This is used to jump
+           back to the beginning of a repeat.  If what follows this jump
+           clearly won't match what the repeat does, such that we can be
+           sure that there is no use backtracking out of repetitions
+           already matched, then we change it to a pop_failure_jump.
+           Followed by two-byte address.  */
+  maybe_pop_jump,
+
+        /* Jump to following two-byte address, and push a dummy failure
+           point. This failure point will be thrown away if an attempt
+           is made to use it for a failure.  A `+' construct makes this
+           before the first repeat.  Also used as an intermediary kind
+           of jump when compiling an alternative.  */
+  dummy_failure_jump,
+
+	/* Push a dummy failure point and continue.  Used at the end of
+	   alternatives.  */
+  push_dummy_failure,
+
+        /* Followed by two-byte relative address and two-byte number n.
+           After matching N times, jump to the address upon failure.  */
+  succeed_n,
+
+        /* Followed by two-byte relative address, and two-byte number n.
+           Jump to the address N times, then fail.  */
+  jump_n,
+
+        /* Set the following two-byte relative address to the
+           subsequent two-byte number.  The address *includes* the two
+           bytes of number.  */
+  set_number_at,
+
+  wordchar,	/* Matches any word-constituent character.  */
+  notwordchar,	/* Matches any char that is not a word-constituent.  */
+
+  wordbeg,	/* Succeeds if at word beginning.  */
+  wordend,	/* Succeeds if at word end.  */
+
+  wordbound,	/* Succeeds if at a word boundary.  */
+  notwordbound	/* Succeeds if not at a word boundary.  */
+
+#ifdef emacs
+  ,before_dot,	/* Succeeds if before point.  */
+  at_dot,	/* Succeeds if at point.  */
+  after_dot,	/* Succeeds if after point.  */
+
+	/* Matches any character whose syntax is specified.  Followed by
+           a byte which contains a syntax code, e.g., Sword.  */
+  syntaxspec,
+
+	/* Matches any character whose syntax is not that specified.  */
+  notsyntaxspec
+#endif /* emacs */
+} re_opcode_t;
+
+
+/* Common operations on the compiled pattern.  */
+
+/* Store NUMBER in two contiguous bytes starting at DESTINATION.  */
+
+#define STORE_NUMBER(destination, number)				\
+  do {									\
+    (destination)[0] = (number) & 0377;					\
+    (destination)[1] = (number) >> 8;					\
+  } while (0)
+
+/* Same as STORE_NUMBER, except increment DESTINATION to
+   the byte after where the number is stored.  Therefore, DESTINATION
+   must be an lvalue.  */
+
+#define STORE_NUMBER_AND_INCR(destination, number)			\
+  do {									\
+    STORE_NUMBER (destination, number);					\
+    (destination) += 2;							\
+  } while (0)
+
+/* Put into DESTINATION a number stored in two contiguous bytes starting
+   at SOURCE.  */
+
+#define EXTRACT_NUMBER(destination, source)				\
+  do {									\
+    (destination) = *(source) & 0377;					\
+    (destination) += SIGN_EXTEND_CHAR (*((source) + 1)) << 8;		\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number (dest, source)
+    int *dest;
+    unsigned char *source;
+{
+  int temp = SIGN_EXTEND_CHAR (*(source + 1)); 
+  *dest = *source & 0377;
+  *dest += temp << 8;
+}
+
+#ifndef EXTRACT_MACROS /* To debug the macros.  */
+#undef EXTRACT_NUMBER
+#define EXTRACT_NUMBER(dest, src) extract_number (&dest, src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+/* Same as EXTRACT_NUMBER, except increment SOURCE to after the number.
+   SOURCE must be an lvalue.  */
+
+#define EXTRACT_NUMBER_AND_INCR(destination, source)			\
+  do {									\
+    EXTRACT_NUMBER (destination, source);				\
+    (source) += 2; 							\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number_and_incr (destination, source)
+    int *destination;
+    unsigned char **source;
+{ 
+  extract_number (destination, *source);
+  *source += 2;
+}
+
+#ifndef EXTRACT_MACROS
+#undef EXTRACT_NUMBER_AND_INCR
+#define EXTRACT_NUMBER_AND_INCR(dest, src) \
+  extract_number_and_incr (&dest, &src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+
+/* If DEBUG is defined, Regex prints many voluminous messages about what
+   it is doing (if the variable `debug' is nonzero).  If linked with the
+   main program in `iregex.c', you can enter patterns and strings
+   interactively.  And if linked with the main program in `main.c' and
+   the other test files, you can run the already-written tests.  */
+
+#ifdef DEBUG
+
+/* We use standard I/O for debugging.  */
+#include <stdio.h>
+
+/* It is useful to test things that ``must'' be true when debugging.  */
+#include <assert.h>
+
+static int debug = 0;
+
+#define DEBUG_STATEMENT(e) e
+#define DEBUG_PRINT1(x) if (debug) printf (x)
+#define DEBUG_PRINT2(x1, x2) if (debug) printf (x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3) if (debug) printf (x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4) if (debug) printf (x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e) 				\
+  if (debug) print_partial_compiled_pattern (s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)			\
+  if (debug) print_double_string (w, s1, sz1, s2, sz2)
+
+
+/* Print the fastmap in human-readable form.  */
+
+void
+print_fastmap (fastmap)
+    char *fastmap;
+{
+  unsigned was_a_range = 0;
+  unsigned i = 0;  
+  
+  while (i < (1 << BYTEWIDTH))
+    {
+      if (fastmap[i++])
+	{
+	  was_a_range = 0;
+          putchar (i - 1);
+          while (i < (1 << BYTEWIDTH)  &&  fastmap[i])
+            {
+              was_a_range = 1;
+              i++;
+            }
+	  if (was_a_range)
+            {
+              printf ("-");
+              putchar (i - 1);
+            }
+        }
+    }
+  putchar ('\n'); 
+}
+
+
+/* Print a compiled pattern string in human-readable form, starting at
+   the START pointer into it and ending just before the pointer END.  */
+
+void
+print_partial_compiled_pattern (start, end)
+    unsigned char *start;
+    unsigned char *end;
+{
+  int mcnt, mcnt2;
+  unsigned char *p = start;
+  unsigned char *pend = end;
+
+  if (start == NULL)
+    {
+      printf ("(null)\n");
+      return;
+    }
+    
+  /* Loop over pattern commands.  */
+  while (p < pend)
+    {
+      printf ("%d:\t", p - start);
+
+      switch ((re_opcode_t) *p++)
+	{
+        case no_op:
+          printf ("/no_op");
+          break;
+
+	case exactn:
+	  mcnt = *p++;
+          printf ("/exactn/%d", mcnt);
+          do
+	    {
+              putchar ('/');
+	      putchar (*p++);
+            }
+          while (--mcnt);
+          break;
+
+	case start_memory:
+          mcnt = *p++;
+          printf ("/start_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case stop_memory:
+          mcnt = *p++;
+	  printf ("/stop_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case duplicate:
+	  printf ("/duplicate/%d", *p++);
+	  break;
+
+	case anychar:
+	  printf ("/anychar");
+	  break;
+
+	case charset:
+        case charset_not:
+          {
+            register int c, last = -100;
+	    register int in_range = 0;
+
+	    printf ("/charset [%s",
+	            (re_opcode_t) *(p - 1) == charset_not ? "^" : "");
+            
+            assert (p + *p < pend);
+
+            for (c = 0; c < 256; c++)
+	      if (c / 8 < *p
+		  && (p[1 + (c/8)] & (1 << (c % 8))))
+		{
+		  /* Are we starting a range?  */
+		  if (last + 1 == c && ! in_range)
+		    {
+		      putchar ('-');
+		      in_range = 1;
+		    }
+		  /* Have we broken a range?  */
+		  else if (last + 1 != c && in_range)
+              {
+		      putchar (last);
+		      in_range = 0;
+		    }
+                
+		  if (! in_range)
+		    putchar (c);
+
+		  last = c;
+              }
+
+	    if (in_range)
+	      putchar (last);
+
+	    putchar (']');
+
+	    p += 1 + *p;
+	  }
+	  break;
+
+	case begline:
+	  printf ("/begline");
+          break;
+
+	case endline:
+          printf ("/endline");
+          break;
+
+	case on_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case on_failure_keep_string_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_keep_string_jump to %d", p + mcnt - start);
+          break;
+
+	case dummy_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/dummy_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case push_dummy_failure:
+          printf ("/push_dummy_failure");
+          break;
+          
+        case maybe_pop_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/maybe_pop_jump to %d", p + mcnt - start);
+	  break;
+
+        case pop_failure_jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/pop_failure_jump to %d", p + mcnt - start);
+	  break;          
+          
+        case jump_past_alt:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump_past_alt to %d", p + mcnt - start);
+	  break;          
+          
+        case jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump to %d", p + mcnt - start);
+	  break;
+
+        case succeed_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/succeed_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case jump_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/jump_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case set_number_at: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/set_number_at location %d to %d", p + mcnt - start, mcnt2);
+          break;
+        
+        case wordbound:
+	  printf ("/wordbound");
+	  break;
+
+	case notwordbound:
+	  printf ("/notwordbound");
+          break;
+
+	case wordbeg:
+	  printf ("/wordbeg");
+	  break;
+          
+	case wordend:
+	  printf ("/wordend");
+          
+#ifdef emacs
+	case before_dot:
+	  printf ("/before_dot");
+          break;
+
+	case at_dot:
+	  printf ("/at_dot");
+          break;
+
+	case after_dot:
+	  printf ("/after_dot");
+          break;
+
+	case syntaxspec:
+          printf ("/syntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+          break;
+	  
+	case notsyntaxspec:
+          printf ("/notsyntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+	  break;
+#endif /* emacs */
+
+	case wordchar:
+	  printf ("/wordchar");
+          break;
+	  
+	case notwordchar:
+	  printf ("/notwordchar");
+          break;
+
+	case begbuf:
+	  printf ("/begbuf");
+          break;
+
+	case endbuf:
+	  printf ("/endbuf");
+          break;
+
+        default:
+          printf ("?%d", *(p-1));
+	}
+
+      putchar ('\n');
+    }
+
+  printf ("%d:\tend of pattern.\n", p - start);
+}
+
+
+void
+print_compiled_pattern (bufp)
+    struct re_pattern_buffer *bufp;
+{
+  unsigned char *buffer = bufp->buffer;
+
+  print_partial_compiled_pattern (buffer, buffer + bufp->used);
+  printf ("%d bytes used/%d bytes allocated.\n", bufp->used, bufp->allocated);
+
+  if (bufp->fastmap_accurate && bufp->fastmap)
+    {
+      printf ("fastmap: ");
+      print_fastmap (bufp->fastmap);
+    }
+
+  printf ("re_nsub: %d\t", bufp->re_nsub);
+  printf ("regs_alloc: %d\t", bufp->regs_allocated);
+  printf ("can_be_null: %d\t", bufp->can_be_null);
+  printf ("newline_anchor: %d\n", bufp->newline_anchor);
+  printf ("no_sub: %d\t", bufp->no_sub);
+  printf ("not_bol: %d\t", bufp->not_bol);
+  printf ("not_eol: %d\t", bufp->not_eol);
+  printf ("syntax: %d\n", bufp->syntax);
+  /* Perhaps we should print the translate table?  */
+}
+
+
+void
+print_double_string (where, string1, size1, string2, size2)
+    const char *where;
+    const char *string1;
+    const char *string2;
+    int size1;
+    int size2;
+{
+  unsigned this_char;
+  
+  if (where == NULL)
+    printf ("(null)");
+  else
+    {
+      if (FIRST_STRING_P (where))
+        {
+          for (this_char = where - string1; this_char < size1; this_char++)
+            putchar (string1[this_char]);
+
+          where = string2;    
+        }
+
+      for (this_char = where - string2; this_char < size2; this_char++)
+        putchar (string2[this_char]);
+    }
+}
+
+#else /* not DEBUG */
+
+#undef assert
+#define assert(e)
+
+#define DEBUG_STATEMENT(e)
+#define DEBUG_PRINT1(x)
+#define DEBUG_PRINT2(x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)
+
+#endif /* not DEBUG */
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+/* This has no initializer because initialized variables in Emacs
+   become read-only after dumping.  */
+reg_syntax_t re_syntax_options;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+{
+  reg_syntax_t ret = re_syntax_options;
+  
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.
+   POSIX doesn't require that we do anything for REG_NOERROR,
+   but why not be nice?  */
+
+static const char *re_error_msgid[] =
+  { "Success",					/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+  };
+
+
+/* Avoiding alloca during matching, to placate r_alloc.  */
+
+/* Define MATCH_MAY_ALLOCATE unless we need to make sure that the
+   searching and matching functions should not call alloca.  On some
+   systems, alloca is implemented in terms of malloc, and if we're
+   using the relocating allocator routines, then malloc could cause a
+   relocation, which might (if the strings being searched are in the
+   ralloc heap) shift the data out from underneath the regexp
+   routines.
+
+   Here's another reason to avoid allocation: Emacs 
+   processes input from X in a signal handler; processing X input may
+   call malloc; if input arrives while a matching routine is calling
+   malloc, then we're scrod.  But Emacs can't just block input while
+   calling matching routines; then we don't notice interrupts when
+   they come in.  So, Emacs blocks input around all regexp calls
+   except the matching calls, which it leaves unprotected, in the
+   faith that they will not malloc.  */
+
+/* Normally, this is fine.  */
+#define MATCH_MAY_ALLOCATE
+
+/* When using GNU C, we are not REALLY using the C alloca, no matter
+   what sysfuncs.h may say.  So don't take precautions for it.  */
+#ifdef __GNUC__
+#undef C_ALLOCA
+#endif
+
+/* The match routines may not allocate if (1) they would do it with malloc
+   and (2) it's not safe for them to use malloc.
+   Note that if REL_ALLOC is defined, matching would not use malloc for the
+   failure stack, but we would still use it for the register vectors;
+   so REL_ALLOC should not affect this.  */
+#if (defined (C_ALLOCA) || defined (REGEX_MALLOC)) && defined (emacs)
+#undef MATCH_MAY_ALLOCATE
+#endif
+
+
+
+/* Failure stack declarations and macros; both re_compile_fastmap and
+   re_match_2 use a failure stack.  These have to be macros because of
+   REGEX_ALLOCATE_STACK.  */
+   
+
+/* Number of failure points for which to initially allocate space
+   when matching.  If this number is exceeded, we allocate more
+   space, so it is not a hard limit.  */
+#ifndef INIT_FAILURE_ALLOC
+#define INIT_FAILURE_ALLOC 5
+#endif
+
+/* Roughly the maximum number of failure points on the stack.  Would be
+   exactly that if always used MAX_FAILURE_SPACE each time we failed.
+   This is a variable only so users of regex can assign to it; we never
+   change it ourselves.  */
+#if defined (MATCH_MAY_ALLOCATE)
+int re_max_failures = 200000;
+#else
+int re_max_failures = 2000;
+#endif
+
+union fail_stack_elt
+{
+  unsigned char *pointer;
+  int integer;
+};
+
+typedef union fail_stack_elt fail_stack_elt_t;
+
+typedef struct
+{
+  fail_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} fail_stack_type;
+
+#define FAIL_STACK_EMPTY()     (fail_stack.avail == 0)
+#define FAIL_STACK_PTR_EMPTY() (fail_stack_ptr->avail == 0)
+#define FAIL_STACK_FULL()      (fail_stack.avail == fail_stack.size)
+
+
+/* Define macros to initialize and free the failure stack.
+   Do `return -2' if the alloc fails.  */
+
+#ifdef MATCH_MAY_ALLOCATE
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.stack = (fail_stack_elt_t *)				\
+      REGEX_ALLOCATE_STACK (INIT_FAILURE_ALLOC * sizeof (fail_stack_elt_t));	\
+									\
+    if (fail_stack.stack == NULL)					\
+      return -2;							\
+									\
+    fail_stack.size = INIT_FAILURE_ALLOC;				\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()  REGEX_FREE_STACK (fail_stack.stack)
+#else
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()
+#endif
+
+
+/* Double the size of FAIL_STACK, up to approximately `re_max_failures' items.
+
+   Return 1 if succeeds, and 0 if either ran out of memory
+   allocating space for it or it was already too large.  
+   
+   REGEX_REALLOCATE_STACK requires `destination' be declared.   */
+
+#define DOUBLE_FAIL_STACK(fail_stack)					\
+  ((fail_stack).size > re_max_failures * MAX_FAILURE_ITEMS		\
+   ? 0									\
+   : ((fail_stack).stack = (fail_stack_elt_t *)				\
+        REGEX_REALLOCATE_STACK ((fail_stack).stack, 			\
+          (fail_stack).size * sizeof (fail_stack_elt_t),		\
+          ((fail_stack).size << 1) * sizeof (fail_stack_elt_t)),	\
+									\
+      (fail_stack).stack == NULL					\
+      ? 0								\
+      : ((fail_stack).size <<= 1, 					\
+         1)))
+
+
+/* Push pointer POINTER on FAIL_STACK. 
+   Return 1 if was able to do so and 0 if ran out of memory allocating
+   space to do so.  */
+#define PUSH_PATTERN_OP(POINTER, FAIL_STACK)				\
+  ((FAIL_STACK_FULL ()							\
+    && !DOUBLE_FAIL_STACK (FAIL_STACK))					\
+   ? 0									\
+   : ((FAIL_STACK).stack[(FAIL_STACK).avail++].pointer = POINTER,	\
+      1))
+
+/* Push a pointer value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_POINTER(item)					\
+  fail_stack.stack[fail_stack.avail++].pointer = (unsigned char *) (item)
+
+/* This pushes an integer-valued item onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_INT(item)					\
+  fail_stack.stack[fail_stack.avail++].integer = (item)
+
+/* Push a fail_stack_elt_t value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_ELT(item)					\
+  fail_stack.stack[fail_stack.avail++] =  (item)
+
+/* These three POP... operations complement the three PUSH... operations.
+   All assume that `fail_stack' is nonempty.  */
+#define POP_FAILURE_POINTER() fail_stack.stack[--fail_stack.avail].pointer
+#define POP_FAILURE_INT() fail_stack.stack[--fail_stack.avail].integer
+#define POP_FAILURE_ELT() fail_stack.stack[--fail_stack.avail]
+
+/* Used to omit pushing failure point id's when we're not debugging.  */
+#ifdef DEBUG
+#define DEBUG_PUSH PUSH_FAILURE_INT
+#define DEBUG_POP(item_addr) *(item_addr) = POP_FAILURE_INT ()
+#else
+#define DEBUG_PUSH(item)
+#define DEBUG_POP(item_addr)
+#endif
+
+
+/* Push the information about the state we will need
+   if we ever fail back to it.  
+   
+   Requires variables fail_stack, regstart, regend, reg_info, and
+   num_regs be declared.  DOUBLE_FAIL_STACK requires `destination' be
+   declared.
+   
+   Does `return FAILURE_CODE' if runs out of memory.  */
+
+#define PUSH_FAILURE_POINT(pattern_place, string_place, failure_code)	\
+  do {									\
+    char *destination;							\
+    /* Must be int, so when we don't save any registers, the arithmetic	\
+       of 0 + -1 isn't done as unsigned.  */				\
+    int this_reg;							\
+    									\
+    DEBUG_STATEMENT (failure_id++);					\
+    DEBUG_STATEMENT (nfailure_points_pushed++);				\
+    DEBUG_PRINT2 ("\nPUSH_FAILURE_POINT #%u:\n", failure_id);		\
+    DEBUG_PRINT2 ("  Before push, next avail: %d\n", (fail_stack).avail);\
+    DEBUG_PRINT2 ("                     size: %d\n", (fail_stack).size);\
+									\
+    DEBUG_PRINT2 ("  slots needed: %d\n", NUM_FAILURE_ITEMS);		\
+    DEBUG_PRINT2 ("     available: %d\n", REMAINING_AVAIL_SLOTS);	\
+									\
+    /* Ensure we have enough space allocated for what we will push.  */	\
+    while (REMAINING_AVAIL_SLOTS < NUM_FAILURE_ITEMS)			\
+      {									\
+        if (!DOUBLE_FAIL_STACK (fail_stack))				\
+          return failure_code;						\
+									\
+        DEBUG_PRINT2 ("\n  Doubled stack; size now: %d\n",		\
+		       (fail_stack).size);				\
+        DEBUG_PRINT2 ("  slots available: %d\n", REMAINING_AVAIL_SLOTS);\
+      }									\
+									\
+    /* Push the info, starting with the registers.  */			\
+    DEBUG_PRINT1 ("\n");						\
+									\
+    for (this_reg = lowest_active_reg; this_reg <= highest_active_reg;	\
+         this_reg++)							\
+      {									\
+	DEBUG_PRINT2 ("  Pushing reg: %d\n", this_reg);			\
+        DEBUG_STATEMENT (num_regs_pushed++);				\
+									\
+	DEBUG_PRINT2 ("    start: 0x%x\n", regstart[this_reg]);		\
+        PUSH_FAILURE_POINTER (regstart[this_reg]);			\
+                                                                        \
+	DEBUG_PRINT2 ("    end: 0x%x\n", regend[this_reg]);		\
+        PUSH_FAILURE_POINTER (regend[this_reg]);			\
+									\
+	DEBUG_PRINT2 ("    info: 0x%x\n      ", reg_info[this_reg]);	\
+        DEBUG_PRINT2 (" match_null=%d",					\
+                      REG_MATCH_NULL_STRING_P (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" active=%d", IS_ACTIVE (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" matched_something=%d",				\
+                      MATCHED_SOMETHING (reg_info[this_reg]));		\
+        DEBUG_PRINT2 (" ever_matched=%d",				\
+                      EVER_MATCHED_SOMETHING (reg_info[this_reg]));	\
+	DEBUG_PRINT1 ("\n");						\
+        PUSH_FAILURE_ELT (reg_info[this_reg].word);			\
+      }									\
+									\
+    DEBUG_PRINT2 ("  Pushing  low active reg: %d\n", lowest_active_reg);\
+    PUSH_FAILURE_INT (lowest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing high active reg: %d\n", highest_active_reg);\
+    PUSH_FAILURE_INT (highest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing pattern 0x%x: ", pattern_place);		\
+    DEBUG_PRINT_COMPILED_PATTERN (bufp, pattern_place, pend);		\
+    PUSH_FAILURE_POINTER (pattern_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing string 0x%x: `", string_place);		\
+    DEBUG_PRINT_DOUBLE_STRING (string_place, string1, size1, string2,   \
+				 size2);				\
+    DEBUG_PRINT1 ("'\n");						\
+    PUSH_FAILURE_POINTER (string_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing failure id: %u\n", failure_id);		\
+    DEBUG_PUSH (failure_id);						\
+  } while (0)
+
+/* This is the number of items that are pushed and popped on the stack
+   for each register.  */
+#define NUM_REG_ITEMS  3
+
+/* Individual items aside from the registers.  */
+#ifdef DEBUG
+#define NUM_NONREG_ITEMS 5 /* Includes failure point id.  */
+#else
+#define NUM_NONREG_ITEMS 4
+#endif
+
+/* We push at most this many items on the stack.  */
+#define MAX_FAILURE_ITEMS ((num_regs - 1) * NUM_REG_ITEMS + NUM_NONREG_ITEMS)
+
+/* We actually push this many items.  */
+#define NUM_FAILURE_ITEMS						\
+  ((highest_active_reg - lowest_active_reg + 1) * NUM_REG_ITEMS 	\
+    + NUM_NONREG_ITEMS)
+
+/* How many items can still be added to the stack without overflowing it.  */
+#define REMAINING_AVAIL_SLOTS ((fail_stack).size - (fail_stack).avail)
+
+
+/* Pops what PUSH_FAIL_STACK pushes.
+
+   We restore into the parameters, all of which should be lvalues:
+     STR -- the saved data position.
+     PAT -- the saved pattern position.
+     LOW_REG, HIGH_REG -- the highest and lowest active registers.
+     REGSTART, REGEND -- arrays of string positions.
+     REG_INFO -- array of information about each subexpression.
+   
+   Also assumes the variables `fail_stack' and (if debugging), `bufp',
+   `pend', `string1', `size1', `string2', and `size2'.  */
+
+#define POP_FAILURE_POINT(str, pat, low_reg, high_reg, regstart, regend, reg_info)\
+{									\
+  DEBUG_STATEMENT (fail_stack_elt_t failure_id;)			\
+  int this_reg;								\
+  const unsigned char *string_temp;					\
+									\
+  assert (!FAIL_STACK_EMPTY ());					\
+									\
+  /* Remove failure points and point to how many regs pushed.  */	\
+  DEBUG_PRINT1 ("POP_FAILURE_POINT:\n");				\
+  DEBUG_PRINT2 ("  Before pop, next avail: %d\n", fail_stack.avail);	\
+  DEBUG_PRINT2 ("                    size: %d\n", fail_stack.size);	\
+									\
+  assert (fail_stack.avail >= NUM_NONREG_ITEMS);			\
+									\
+  DEBUG_POP (&failure_id);						\
+  DEBUG_PRINT2 ("  Popping failure id: %u\n", failure_id);		\
+									\
+  /* If the saved string location is NULL, it came from an		\
+     on_failure_keep_string_jump opcode, and we want to throw away the	\
+     saved NULL, thus retaining our current position in the string.  */	\
+  string_temp = POP_FAILURE_POINTER ();					\
+  if (string_temp != NULL)						\
+    str = (const char *) string_temp;					\
+									\
+  DEBUG_PRINT2 ("  Popping string 0x%x: `", str);			\
+  DEBUG_PRINT_DOUBLE_STRING (str, string1, size1, string2, size2);	\
+  DEBUG_PRINT1 ("'\n");							\
+									\
+  pat = (unsigned char *) POP_FAILURE_POINTER ();			\
+  DEBUG_PRINT2 ("  Popping pattern 0x%x: ", pat);			\
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, pat, pend);			\
+									\
+  /* Restore register info.  */						\
+  high_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping high active reg: %d\n", high_reg);		\
+									\
+  low_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping  low active reg: %d\n", low_reg);		\
+									\
+  for (this_reg = high_reg; this_reg >= low_reg; this_reg--)		\
+    {									\
+      DEBUG_PRINT2 ("    Popping reg: %d\n", this_reg);			\
+									\
+      reg_info[this_reg].word = POP_FAILURE_ELT ();			\
+      DEBUG_PRINT2 ("      info: 0x%x\n", reg_info[this_reg]);		\
+									\
+      regend[this_reg] = (const char *) POP_FAILURE_POINTER ();		\
+      DEBUG_PRINT2 ("      end: 0x%x\n", regend[this_reg]);		\
+									\
+      regstart[this_reg] = (const char *) POP_FAILURE_POINTER ();	\
+      DEBUG_PRINT2 ("      start: 0x%x\n", regstart[this_reg]);		\
+    }									\
+									\
+  set_regs_matched_done = 0;						\
+  DEBUG_STATEMENT (nfailure_points_popped++);				\
+} /* POP_FAILURE_POINT */
+
+
+
+
+/* Structure for per-register (a.k.a. per-group) information.
+   Other register information, such as the
+   starting and ending positions (which are addresses), and the list of
+   inner groups (which is a bits list) are maintained in separate
+   variables.  
+   
+   We are making a (strictly speaking) nonportable assumption here: that
+   the compiler will pack our bit fields into something that fits into
+   the type of `word', i.e., is something that fits into one item on the
+   failure stack.  */
+
+typedef union
+{
+  fail_stack_elt_t word;
+  struct
+  {
+      /* This field is one if this group can match the empty string,
+         zero if not.  If not yet determined,  `MATCH_NULL_UNSET_VALUE'.  */
+#define MATCH_NULL_UNSET_VALUE 3
+    unsigned match_null_string_p : 2;
+    unsigned is_active : 1;
+    unsigned matched_something : 1;
+    unsigned ever_matched_something : 1;
+  } bits;
+} register_info_type;
+
+#define REG_MATCH_NULL_STRING_P(R)  ((R).bits.match_null_string_p)
+#define IS_ACTIVE(R)  ((R).bits.is_active)
+#define MATCHED_SOMETHING(R)  ((R).bits.matched_something)
+#define EVER_MATCHED_SOMETHING(R)  ((R).bits.ever_matched_something)
+
+
+/* Call this when have matched a real character; it sets `matched' flags
+   for the subexpressions which we are currently inside.  Also records
+   that those subexprs have matched.  */
+#define SET_REGS_MATCHED()						\
+  do									\
+    {									\
+      if (!set_regs_matched_done)					\
+	{								\
+	  unsigned r;							\
+	  set_regs_matched_done = 1;					\
+	  for (r = lowest_active_reg; r <= highest_active_reg; r++)	\
+	    {								\
+	      MATCHED_SOMETHING (reg_info[r])				\
+		= EVER_MATCHED_SOMETHING (reg_info[r])			\
+		= 1;							\
+	    }								\
+	}								\
+    }									\
+  while (0)
+
+/* Registers are set to a sentinel when they haven't yet matched.  */
+static char reg_unset_dummy;
+#define REG_UNSET_VALUE (&reg_unset_dummy)
+#define REG_UNSET(e) ((e) == REG_UNSET_VALUE)
+
+
+/* Subroutine declarations and macros for regex_compile.  */
+
+static void store_op1 (), store_op2 ();
+static void insert_op1 (), insert_op2 ();
+static boolean at_begline_loc_p (), at_endline_loc_p ();
+static boolean group_in_compile_stack ();
+static reg_errcode_t compile_range ();
+
+/* Fetch the next character in the uncompiled pattern---translating it 
+   if necessary.  Also cast from a signed character in the constant
+   string passed to us by the user to an unsigned char that we can use
+   as an array index (in, e.g., `translate').  */
+#define PATFETCH(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    if (translate) c = translate[c]; 					\
+  } while (0)
+
+/* Fetch the next character in the uncompiled pattern, with no
+   translation.  */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+/* If `translate' is non-null, return translate[D], else just D.  We
+   cast the subscript to translate because some data is declared as
+   `char *', to avoid warnings when a string constant is passed.  But
+   when we use a character as a subscript we must make it unsigned.  */
+#define TRANSLATE(d) (translate ? translate[(unsigned char) (d)] : (d))
+
+
+/* Macros for outputting the compiled pattern into `buffer'.  */
+
+/* If the buffer isn't allocated when it comes in, use this.  */
+#define INIT_BUF_SIZE  32
+
+/* Make sure we have at least N more bytes of space in buffer.  */
+#define GET_BUFFER_SPACE(n)						\
+    while (b - bufp->buffer + (n) > bufp->allocated)			\
+      EXTEND_BUFFER ()
+
+/* Make sure we have one more byte of buffer space and then add C to it.  */
+#define BUF_PUSH(c)							\
+  do {									\
+    GET_BUFFER_SPACE (1);						\
+    *b++ = (unsigned char) (c);						\
+  } while (0)
+
+
+/* Ensure we have two more bytes of buffer space and then append C1 and C2.  */
+#define BUF_PUSH_2(c1, c2)						\
+  do {									\
+    GET_BUFFER_SPACE (2);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+  } while (0)
+
+
+/* As with BUF_PUSH_2, except for three bytes.  */
+#define BUF_PUSH_3(c1, c2, c3)						\
+  do {									\
+    GET_BUFFER_SPACE (3);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+    *b++ = (unsigned char) (c3);					\
+  } while (0)
+
+
+/* Store a jump with opcode OP at LOC to location TO.  We store a
+   relative address offset by the three bytes the jump itself occupies.  */
+#define STORE_JUMP(op, loc, to) \
+  store_op1 (op, loc, (to) - (loc) - 3)
+
+/* Likewise, for a two-argument jump.  */
+#define STORE_JUMP2(op, loc, to, arg) \
+  store_op2 (op, loc, (to) - (loc) - 3, arg)
+
+/* Like `STORE_JUMP', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP(op, loc, to) \
+  insert_op1 (op, loc, (to) - (loc) - 3, b)
+
+/* Like `STORE_JUMP2', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP2(op, loc, to, arg) \
+  insert_op2 (op, loc, (to) - (loc) - 3, arg, b)
+
+
+/* This is not an arbitrary limit: the arguments which represent offsets
+   into the pattern are two bytes long.  So if 2^16 bytes turns out to
+   be too small, many things would have to change.  */
+#define MAX_BUF_SIZE (1L << 16)
+
+
+/* Extend the buffer by twice its current size via realloc and
+   reset the pointers that pointed into the old block to point to the
+   correct places in the new one.  If extending the buffer results in it
+   being larger than MAX_BUF_SIZE, then flag memory exhausted.  */
+#define EXTEND_BUFFER()							\
+  do { 									\
+    unsigned char *old_buffer = bufp->buffer;				\
+    if (bufp->allocated == MAX_BUF_SIZE) 				\
+      return REG_ESIZE;							\
+    bufp->allocated <<= 1;						\
+    if (bufp->allocated > MAX_BUF_SIZE)					\
+      bufp->allocated = MAX_BUF_SIZE; 					\
+    bufp->buffer = (unsigned char *) realloc (bufp->buffer, bufp->allocated);\
+    if (bufp->buffer == NULL)						\
+      return REG_ESPACE;						\
+    /* If the buffer moved, move all the pointers into it.  */		\
+    if (old_buffer != bufp->buffer)					\
+      {									\
+        b = (b - old_buffer) + bufp->buffer;				\
+        begalt = (begalt - old_buffer) + bufp->buffer;			\
+        if (fixup_alt_jump)						\
+          fixup_alt_jump = (fixup_alt_jump - old_buffer) + bufp->buffer;\
+        if (laststart)							\
+          laststart = (laststart - old_buffer) + bufp->buffer;		\
+        if (pending_exact)						\
+          pending_exact = (pending_exact - old_buffer) + bufp->buffer;	\
+      }									\
+  } while (0)
+
+
+/* Since we have one byte reserved for the register number argument to
+   {start,stop}_memory, the maximum number of groups we can report
+   things about is what fits in that byte.  */
+#define MAX_REGNUM 255
+
+/* But patterns can have more than `MAX_REGNUM' registers.  We just
+   ignore the excess.  */
+typedef unsigned regnum_t;
+
+
+/* Macros for the compile stack.  */
+
+/* Since offsets can go either forwards or backwards, this type needs to
+   be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.  */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  pattern_offset_t begalt_offset;
+  pattern_offset_t fixup_alt_jump;
+  pattern_offset_t inner_group_offset;
+  pattern_offset_t laststart_offset;  
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / BYTEWIDTH]               \
+   |= 1 << (((unsigned char) c) % BYTEWIDTH))
+
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (ISDIGIT (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (STREQ (string, "alpha") || STREQ (string, "upper")			\
+    || STREQ (string, "lower") || STREQ (string, "digit")		\
+    || STREQ (string, "alnum") || STREQ (string, "xdigit")		\
+    || STREQ (string, "space") || STREQ (string, "print")		\
+    || STREQ (string, "punct") || STREQ (string, "graph")		\
+    || STREQ (string, "cntrl") || STREQ (string, "blank"))
+
+
+#ifndef MATCH_MAY_ALLOCATE
+
+/* If we cannot allocate large objects within re_match_2_internal,
+   we make the fail stack and register vectors global.
+   The fail stack, we grow to the maximum size when a regexp
+   is compiled.
+   The register vectors, we adjust in size each time we
+   compile a regexp, according to the number of registers it needs.  */
+
+static fail_stack_type fail_stack;
+
+/* Size with which the following vectors are currently allocated.
+   That is so we can make them bigger as needed,
+   but never make them smaller.  */
+static int regs_allocated_size;
+
+static const char **     regstart, **     regend;
+static const char ** old_regstart, ** old_regend;
+static const char **best_regstart, **best_regend;
+static register_info_type *reg_info; 
+static const char **reg_dummy;
+static register_info_type *reg_info_dummy;
+
+/* Make the register vectors big enough for NUM_REGS registers,
+   but don't make them smaller.  */
+
+static
+regex_grow_registers (num_regs)
+     int num_regs;
+{
+  if (num_regs > regs_allocated_size)
+    {
+      RETALLOC_IF (regstart,	 num_regs, const char *);
+      RETALLOC_IF (regend,	 num_regs, const char *);
+      RETALLOC_IF (old_regstart, num_regs, const char *);
+      RETALLOC_IF (old_regend,	 num_regs, const char *);
+      RETALLOC_IF (best_regstart, num_regs, const char *);
+      RETALLOC_IF (best_regend,	 num_regs, const char *);
+      RETALLOC_IF (reg_info,	 num_regs, register_info_type);
+      RETALLOC_IF (reg_dummy,	 num_regs, const char *);
+      RETALLOC_IF (reg_info_dummy, num_regs, register_info_type);
+
+      regs_allocated_size = num_regs;
+    }
+}
+
+#endif /* not MATCH_MAY_ALLOCATE */
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is zero;
+     `re_nsub' is the number of subexpressions in PATTERN;
+     `not_bol' and `not_eol' are zero;
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+/* Return, freeing storage we allocated.  */
+#define FREE_STACK_RETURN(value)		\
+  return (free (compile_stack.stack), value)
+
+static reg_errcode_t
+regex_compile (pattern, size, syntax, bufp)
+     const char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer *bufp;
+{
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random temporary spot in PATTERN.  */
+  const char *p1;
+
+  /* Points to the end of the buffer, where we should append.  */
+  register unsigned char *b;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  const char *p = pattern;
+  const char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  char *translate = bufp->translate;
+
+  /* Address of the count-byte of the most recently inserted `exactn'
+     command.  This makes it possible to tell if a new exact-match
+     character can be added to that command or if the character requires
+     a new `exactn' command.  */
+  unsigned char *pending_exact = 0;
+
+  /* Address of start of the most recently finished expression.
+     This tells, e.g., postfix * where to find the start of its
+     operand.  Reset at the beginning of groups and alternatives.  */
+  unsigned char *laststart = 0;
+
+  /* Address of beginning of regexp, or inside of last group.  */
+  unsigned char *begalt;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+     which to go back if the interval is invalid.  */
+  const char *beg_interval;
+                
+  /* Address of the place where a forward jump should go to the end of
+     the containing expression.  Each alternative of an `or' -- except the
+     last -- ends with a forward jump of this sort.  */
+  unsigned char *fixup_alt_jump = 0;
+
+  /* Counts open-groups as they are encountered.  Remembered for the
+     matching close-group on the compile stack, so the same register
+     number is put in the stop_memory as the start_memory.  */
+  regnum_t regnum = 0;
+
+#ifdef DEBUG
+  DEBUG_PRINT1 ("\nCompiling pattern: ");
+  if (debug)
+    {
+      unsigned debug_count;
+      
+      for (debug_count = 0; debug_count < size; debug_count++)
+        putchar (pattern[debug_count]);
+      putchar ('\n');
+    }
+#endif /* DEBUG */
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack = TALLOC (INIT_COMPILE_STACK_SIZE, compile_stack_elt_t);
+  if (compile_stack.stack == NULL)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  bufp->syntax = syntax;
+  bufp->fastmap_accurate = 0;
+  bufp->not_bol = bufp->not_eol = 0;
+
+  /* Set `used' to zero, so that if we return an error, the pattern
+     printer (for debugging) will think there's no pattern.  We reset it
+     at the end.  */
+  bufp->used = 0;
+  
+  /* Always count groups, whether or not bufp->no_sub is set.  */
+  bufp->re_nsub = 0;				
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  if (bufp->allocated == 0)
+    {
+      if (bufp->buffer)
+	{ /* If zero allocated, but buffer is non-null, try to realloc
+             enough space.  This loses if buffer's address is bogus, but
+             that is the user's responsibility.  */
+          RETALLOC (bufp->buffer, INIT_BUF_SIZE, unsigned char);
+        }
+      else
+        { /* Caller did not allocate a buffer.  Do it for them.  */
+          bufp->buffer = TALLOC (INIT_BUF_SIZE, unsigned char);
+        }
+      if (!bufp->buffer) FREE_STACK_RETURN (REG_ESPACE);
+
+      bufp->allocated = INIT_BUF_SIZE;
+    }
+
+  begalt = b = bufp->buffer;
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+              BUF_PUSH (begline);
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+               BUF_PUSH (endline);
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (!laststart)
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                FREE_STACK_RETURN (REG_BADRPT);
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* Are we optimizing this jump?  */
+            boolean keep_string_p = false;
+            
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!laststart)  
+              break;
+
+            /* Now we know whether or not zero matches is allowed
+               and also whether or not two or more matches is allowed.  */
+            if (many_times_ok)
+              { /* More than one repetition is allowed, so put in at the
+                   end a backward relative jump from `b' to before the next
+                   jump we're going to put in below (which jumps from
+                   laststart to after this jump).  
+
+                   But if we are at the `*' in the exact sequence `.*\n',
+                   insert an unconditional jump backwards to the .,
+                   instead of the beginning of the loop.  This way we only
+                   push a failure point once, instead of every time
+                   through the loop.  */
+                assert (p - 1 > pattern);
+
+                /* Allocate the space for the jump.  */
+                GET_BUFFER_SPACE (3);
+
+                /* We know we are not at the first character of the pattern,
+                   because laststart was nonzero.  And we've already
+                   incremented `p', by the way, to be the character after
+                   the `*'.  Do we have to do something analogous here
+                   for null bytes, because of RE_DOT_NOT_NULL?  */
+                if (TRANSLATE (*(p - 2)) == TRANSLATE ('.')
+		    && zero_times_ok
+                    && p < pend && TRANSLATE (*p) == TRANSLATE ('\n')
+                    && !(syntax & RE_DOT_NEWLINE))
+                  { /* We have .*\n.  */
+                    STORE_JUMP (jump, b, laststart);
+                    keep_string_p = true;
+                  }
+                else
+                  /* Anything else.  */
+                  STORE_JUMP (maybe_pop_jump, b, laststart - 3);
+
+                /* We've added more stuff to the buffer.  */
+                b += 3;
+              }
+
+            /* On failure, jump from laststart to b + 3, which will be the
+               end of the buffer after this jump is inserted.  */
+            GET_BUFFER_SPACE (3);
+            INSERT_JUMP (keep_string_p ? on_failure_keep_string_jump
+                                       : on_failure_jump,
+                         laststart, b + 3);
+            pending_exact = 0;
+            b += 3;
+
+            if (!zero_times_ok)
+              {
+                /* At least one repetition is required, so insert a
+                   `dummy_failure_jump' before the initial
+                   `on_failure_jump' instruction of the loop. This
+                   effects a skip over that instruction the first time
+                   we hit that loop.  */
+                GET_BUFFER_SPACE (3);
+                INSERT_JUMP (dummy_failure_jump, laststart, laststart + 6);
+                b += 3;
+              }
+            }
+	  break;
+
+
+	case '.':
+          laststart = b;
+          BUF_PUSH (anychar);
+          break;
+
+
+        case '[':
+          {
+            boolean had_char_class = false;
+
+            if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+            /* Ensure that we have enough space to push a charset: the
+               opcode, the length count, and the bitset; 34 bytes in all.  */
+	    GET_BUFFER_SPACE (34);
+
+            laststart = b;
+
+            /* We test `*p == '^' twice, instead of using an if
+               statement, so we only need one BUF_PUSH.  */
+            BUF_PUSH (*p == '^' ? charset_not : charset); 
+            if (*p == '^')
+              p++;
+
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+
+            /* Push the number of bytes in the bitmap.  */
+            BUF_PUSH ((1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* Clear the whole map.  */
+            bzero (b, (1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* charset_not matches newline according to a syntax bit.  */
+            if ((re_opcode_t) b[-2] == charset_not
+                && (syntax & RE_HAT_LISTS_NOT_NEWLINE))
+              SET_LIST_BIT ('\n');
+
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                PATFETCH (c);
+
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    SET_LIST_BIT (c1);
+                    continue;
+                  }
+
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  break;
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  FREE_STACK_RETURN (REG_ERANGE);
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                /* See if we're at the beginning of a possible character
+                   class.  */
+
+                else if (syntax & RE_CHAR_CLASSES && c == '[' && *p == ':')
+                  { /* Leave room for the null.  */
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+
+                    PATFETCH (c);
+                    c1 = 0;
+
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+                          break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = STREQ (str, "alnum");
+                        boolean is_alpha = STREQ (str, "alpha");
+                        boolean is_blank = STREQ (str, "blank");
+                        boolean is_cntrl = STREQ (str, "cntrl");
+                        boolean is_digit = STREQ (str, "digit");
+                        boolean is_graph = STREQ (str, "graph");
+                        boolean is_lower = STREQ (str, "lower");
+                        boolean is_print = STREQ (str, "print");
+                        boolean is_punct = STREQ (str, "punct");
+                        boolean is_space = STREQ (str, "space");
+                        boolean is_upper = STREQ (str, "upper");
+                        boolean is_xdigit = STREQ (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str))
+			  FREE_STACK_RETURN (REG_ECTYPE);
+
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+
+                        if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                        for (ch = 0; ch < 1 << BYTEWIDTH; ch++)
+                          {
+			    /* This was split into 3 if's to
+			       avoid an arbitrary limit in some compiler.  */
+                            if (   (is_alnum  && ISALNUM (ch))
+                                || (is_alpha  && ISALPHA (ch))
+                                || (is_blank  && ISBLANK (ch))
+                                || (is_cntrl  && ISCNTRL (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_digit  && ISDIGIT (ch))
+                                || (is_graph  && ISGRAPH (ch))
+                                || (is_lower  && ISLOWER (ch))
+                                || (is_print  && ISPRINT (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_punct  && ISPUNCT (ch))
+                                || (is_space  && ISSPACE (ch))
+                                || (is_upper  && ISUPPER (ch))
+                                || (is_xdigit && ISXDIGIT (ch)))
+			      SET_LIST_BIT (ch);
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+                        SET_LIST_BIT ('[');
+                        SET_LIST_BIT (':');
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+                    SET_LIST_BIT (c);
+                  }
+              }
+
+            /* Discard any (non)matching list bytes that are all 0 at the
+               end of the map.  Decrease the map-length byte too.  */
+            while ((int) b[-1] > 0 && b[b[-1] - 1] == 0) 
+              b[-1]--; 
+            b += b[-1];
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+           if (syntax & RE_INTERVALS && syntax & RE_NO_BK_BRACES)
+             goto handle_interval;
+           else
+             goto normal_char;
+
+
+        case '\\':
+          if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              bufp->re_nsub++;
+              regnum++;
+
+              if (COMPILE_STACK_FULL)
+                { 
+                  RETALLOC (compile_stack.stack, compile_stack.size << 1,
+                            compile_stack_elt_t);
+                  if (compile_stack.stack == NULL) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+              /* These are the values to restore when we hit end of this
+                 group.  They are all relative offsets, so that if the
+                 whole pattern moves because of realloc, they will still
+                 be valid.  */
+              COMPILE_STACK_TOP.begalt_offset = begalt - bufp->buffer;
+              COMPILE_STACK_TOP.fixup_alt_jump 
+                = fixup_alt_jump ? fixup_alt_jump - bufp->buffer + 1 : 0;
+              COMPILE_STACK_TOP.laststart_offset = b - bufp->buffer;
+              COMPILE_STACK_TOP.regnum = regnum;
+
+              /* We will eventually replace the 0 with the number of
+                 groups inner to this one.  But do not push a
+                 start_memory for groups beyond the last one we can
+                 represent in the compiled pattern.  */
+              if (regnum <= MAX_REGNUM)
+                {
+                  COMPILE_STACK_TOP.inner_group_offset = b - bufp->buffer + 2;
+                  BUF_PUSH_3 (start_memory, regnum, 0);
+                }
+                
+              compile_stack.avail++;
+
+              fixup_alt_jump = 0;
+              laststart = 0;
+              begalt = b;
+	      /* If we've reached MAX_REGNUM groups, then this open
+		 won't actually generate any code, so we'll have to
+		 clear pending_exact explicitly.  */
+	      pending_exact = 0;
+              break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_backslash;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+            handle_close:
+              if (fixup_alt_jump)
+                { /* Push a dummy failure point at the end of the
+                     alternative for a possible future
+                     `pop_failure_jump' to pop.  See comments at
+                     `push_dummy_failure' in `re_match_2'.  */
+                  BUF_PUSH (push_dummy_failure);
+                  
+                  /* We allocated space for this jump when we assigned
+                     to `fixup_alt_jump', in the `handle_alt' case below.  */
+                  STORE_JUMP (jump_past_alt, fixup_alt_jump, b - 1);
+                }
+
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+              assert (compile_stack.avail != 0);
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+
+                compile_stack.avail--;		
+                begalt = bufp->buffer + COMPILE_STACK_TOP.begalt_offset;
+                fixup_alt_jump
+                  = COMPILE_STACK_TOP.fixup_alt_jump
+                    ? bufp->buffer + COMPILE_STACK_TOP.fixup_alt_jump - 1 
+                    : 0;
+                laststart = bufp->buffer + COMPILE_STACK_TOP.laststart_offset;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		/* If we've reached MAX_REGNUM groups, then this open
+		   won't actually generate any code, so we'll have to
+		   clear pending_exact explicitly.  */
+		pending_exact = 0;
+
+                /* We're at the end of the group, so now we know how many
+                   groups were inside this one.  */
+                if (this_group_regnum <= MAX_REGNUM)
+                  {
+                    unsigned char *inner_group_loc
+                      = bufp->buffer + COMPILE_STACK_TOP.inner_group_offset;
+                    
+                    *inner_group_loc = regnum - this_group_regnum;
+                    BUF_PUSH_3 (stop_memory, this_group_regnum,
+                                regnum - this_group_regnum);
+                  }
+              }
+              break;
+
+
+            case '|':					/* `\|'.  */
+              if (syntax & RE_LIMITED_OPS || syntax & RE_NO_BK_VBAR)
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+              /* Insert before the previous alternative a jump which
+                 jumps to this alternative if the former fails.  */
+              GET_BUFFER_SPACE (3);
+              INSERT_JUMP (on_failure_jump, begalt, b + 6);
+              pending_exact = 0;
+              b += 3;
+
+              /* The alternative before this one has a jump after it
+                 which gets executed if it gets matched.  Adjust that
+                 jump so it will jump to this alternative's analogous
+                 jump (put in below, which in turn will jump to the next
+                 (if any) alternative's such jump, etc.).  The last such
+                 jump jumps to the correct final destination.  A picture:
+                          _____ _____ 
+                          |   | |   |   
+                          |   v |   v 
+                         a | b   | c   
+
+                 If we are at `b', then fixup_alt_jump right now points to a
+                 three-byte space after `a'.  We'll put in the jump, set
+                 fixup_alt_jump to right after `b', and leave behind three
+                 bytes which we'll fill in when we get to after `c'.  */
+
+              if (fixup_alt_jump)
+                STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+              /* Mark and leave space for a jump after this alternative,
+                 to be filled in later either by next alternative or
+                 when know we're at the end of a series of alternatives.  */
+              fixup_alt_jump = b;
+              GET_BUFFER_SPACE (3);
+              b += 3;
+
+              laststart = 0;
+              begalt = b;
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      FREE_STACK_RETURN (REG_EBRACE);
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') FREE_STACK_RETURN (REG_EBRACE);
+
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (!laststart)
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      FREE_STACK_RETURN (REG_BADRPT);
+                    else if (syntax & RE_CONTEXT_INDEP_OPS)
+                      laststart = b;
+                    else
+                      goto unfetch_interval;
+                  }
+
+                /* If the upper bound is zero, don't want to succeed at
+                   all; jump from `laststart' to `b + 3', which will be
+                   the end of the buffer after we insert the jump.  */
+                 if (upper_bound == 0)
+                   {
+                     GET_BUFFER_SPACE (3);
+                     INSERT_JUMP (jump, laststart, b + 3);
+                     b += 3;
+                   }
+
+                 /* Otherwise, we have a nontrivial interval.  When
+                    we're all done, the pattern will look like:
+                      set_number_at <jump count> <upper bound>
+                      set_number_at <succeed_n count> <lower bound>
+                      succeed_n <after jump addr> <succeed_n count>
+                      <body of loop>
+                      jump_n <succeed_n addr> <jump count>
+                    (The upper bound and `jump_n' are omitted if
+                    `upper_bound' is 1, though.)  */
+                 else 
+                   { /* If the upper bound is > 1, we need to insert
+                        more at the end of the loop.  */
+                     unsigned nbytes = 10 + (upper_bound > 1) * 10;
+
+                     GET_BUFFER_SPACE (nbytes);
+
+                     /* Initialize lower bound of the `succeed_n', even
+                        though it will be set during matching by its
+                        attendant `set_number_at' (inserted next),
+                        because `re_compile_fastmap' needs to know.
+                        Jump to the `jump_n' we might insert below.  */
+                     INSERT_JUMP2 (succeed_n, laststart,
+                                   b + 5 + (upper_bound > 1) * 5,
+                                   lower_bound);
+                     b += 5;
+
+                     /* Code to initialize the lower bound.  Insert 
+                        before the `succeed_n'.  The `5' is the last two
+                        bytes of this `set_number_at', plus 3 bytes of
+                        the following `succeed_n'.  */
+                     insert_op2 (set_number_at, laststart, 5, lower_bound, b);
+                     b += 5;
+
+                     if (upper_bound > 1)
+                       { /* More than one repetition is allowed, so
+                            append a backward jump to the `succeed_n'
+                            that starts this interval.
+                            
+                            When we've reached this during matching,
+                            we'll have matched the interval once, so
+                            jump back only `upper_bound - 1' times.  */
+                         STORE_JUMP2 (jump_n, b, laststart + 5,
+                                      upper_bound - 1);
+                         b += 5;
+
+                         /* The location we want to set is the second
+                            parameter of the `jump_n'; that is `b-2' as
+                            an absolute address.  `laststart' will be
+                            the `set_number_at' we're about to insert;
+                            `laststart+3' the number to set, the source
+                            for the relative address.  But we are
+                            inserting into the middle of the pattern --
+                            so everything is getting moved up by 5.
+                            Conclusion: (b - 2) - (laststart + 3) + 5,
+                            i.e., b - laststart.
+                            
+                            We insert this at the beginning of the loop
+                            so that if we fail during matching, we'll
+                            reinitialize the bounds.  */
+                         insert_op2 (set_number_at, laststart, b - laststart,
+                                     upper_bound - 1, b);
+                         b += 5;
+                       }
+                   }
+                pending_exact = 0;
+                beg_interval = NULL;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               assert (beg_interval);
+               p = beg_interval;
+               beg_interval = NULL;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+              BUF_PUSH (at_dot);
+              break;
+
+            case 's':	
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (syntaxspec, syntax_spec_code[c]);
+              break;
+
+            case 'S':
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (notsyntaxspec, syntax_spec_code[c]);
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+              laststart = b;
+              BUF_PUSH (wordchar);
+              break;
+
+
+            case 'W':
+              laststart = b;
+              BUF_PUSH (notwordchar);
+              break;
+
+
+            case '<':
+              BUF_PUSH (wordbeg);
+              break;
+
+            case '>':
+              BUF_PUSH (wordend);
+              break;
+
+            case 'b':
+              BUF_PUSH (wordbound);
+              break;
+
+            case 'B':
+              BUF_PUSH (notwordbound);
+              break;
+
+            case '`':
+              BUF_PUSH (begbuf);
+              break;
+
+            case '\'':
+              BUF_PUSH (endbuf);
+              break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                FREE_STACK_RETURN (REG_ESUBREG);
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+                goto normal_char;
+
+              laststart = b;
+              BUF_PUSH_2 (duplicate, c1);
+              break;
+
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	      /* If no exactn currently being built.  */
+          if (!pending_exact 
+
+              /* If last exactn not at current position.  */
+              || pending_exact + *pending_exact + 1 != b
+              
+              /* We have only one byte following the exactn for the count.  */
+	      || *pending_exact == (1 << BYTEWIDTH) - 1
+
+              /* If followed by a repetition operator.  */
+              || *p == '*' || *p == '^'
+	      || ((syntax & RE_BK_PLUS_QM)
+		  ? *p == '\\' && (p[1] == '+' || p[1] == '?')
+		  : (*p == '+' || *p == '?'))
+	      || ((syntax & RE_INTERVALS)
+                  && ((syntax & RE_NO_BK_BRACES)
+		      ? *p == '{'
+                      : (p[0] == '\\' && p[1] == '{'))))
+	    {
+	      /* Start building a new exactn.  */
+              
+              laststart = b;
+
+	      BUF_PUSH_2 (exactn, 0);
+	      pending_exact = b - 1;
+            }
+            
+	  BUF_PUSH (c);
+          (*pending_exact)++;
+	  break;
+        } /* switch (c) */
+    } /* while p != pend */
+
+  
+  /* Through the pattern now.  */
+  
+  if (fixup_alt_jump)
+    STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+  if (!COMPILE_STACK_EMPTY) 
+    FREE_STACK_RETURN (REG_EPAREN);
+
+  /* If we don't want backtracking, force success
+     the first time we reach the end of the compiled pattern.  */
+  if (syntax & RE_NO_POSIX_BACKTRACKING)
+    BUF_PUSH (succeed);
+
+  free (compile_stack.stack);
+
+  /* We have succeeded; set the length of the buffer.  */
+  bufp->used = b - bufp->buffer;
+
+#ifdef DEBUG
+  if (debug)
+    {
+      DEBUG_PRINT1 ("\nCompiled pattern: \n");
+      print_compiled_pattern (bufp);
+    }
+#endif /* DEBUG */
+
+#ifndef MATCH_MAY_ALLOCATE
+  /* Initialize the failure stack to the largest possible stack.  This
+     isn't necessary unless we're trying to avoid calling alloca in
+     the search and match routines.  */
+  {
+    int num_regs = bufp->re_nsub + 1;
+
+    /* Since DOUBLE_FAIL_STACK refuses to double only if the current size
+       is strictly greater than re_max_failures, the largest possible stack
+       is 2 * re_max_failures failure points.  */
+    if (fail_stack.size < (2 * re_max_failures * MAX_FAILURE_ITEMS))
+      {
+	fail_stack.size = (2 * re_max_failures * MAX_FAILURE_ITEMS);
+
+#ifdef emacs
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xmalloc (fail_stack.size 
+					    * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xrealloc (fail_stack.stack,
+					     (fail_stack.size
+					      * sizeof (fail_stack_elt_t)));
+#else /* not emacs */
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) malloc (fail_stack.size 
+					   * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) realloc (fail_stack.stack,
+					    (fail_stack.size
+					     * sizeof (fail_stack_elt_t)));
+#endif /* not emacs */
+      }
+
+    regex_grow_registers (num_regs);
+  }
+#endif /* not MATCH_MAY_ALLOCATE */
+
+  return REG_NOERROR;
+} /* regex_compile */
+
+
+/* Subroutines for `regex_compile'.  */
+
+/* Store OP at LOC followed by two-byte integer parameter ARG.  */
+
+static void
+store_op1 (op, loc, arg)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg);
+}
+
+
+/* Like `store_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+store_op2 (op, loc, arg1, arg2)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg1);
+  STORE_NUMBER (loc + 3, arg2);
+}
+
+
+/* Copy the bytes from LOC to END to open up three bytes of space at LOC
+   for OP followed by two-byte integer parameter ARG.  */
+
+static void
+insert_op1 (op, loc, arg, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 3;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op1 (op, loc, arg);
+}
+
+
+/* Like `insert_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+insert_op2 (op, loc, arg1, arg2, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 5;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op2 (op, loc, arg1, arg2);
+}
+
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+   after an alternative or a begin-subexpression.  We assume there is at
+   least one character before the ^.  */
+
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+    const char *pattern, *p;
+    reg_syntax_t syntax;
+{
+  const char *prev = p - 2;
+  boolean prev_prev_backslash = prev > pattern && prev[-1] == '\\';
+  
+  return
+       /* After a subexpression?  */
+       (*prev == '(' && (syntax & RE_NO_BK_PARENS || prev_prev_backslash))
+       /* After an alternative?  */
+    || (*prev == '|' && (syntax & RE_NO_BK_VBAR || prev_prev_backslash));
+}
+
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+   at least one character after the $, i.e., `P < PEND'.  */
+
+static boolean
+at_endline_loc_p (p, pend, syntax)
+    const char *p, *pend;
+    int syntax;
+{
+  const char *next = p;
+  boolean next_backslash = *next == '\\';
+  const char *next_next = p + 1 < pend ? p + 1 : NULL;
+  
+  return
+       /* Before a subexpression?  */
+       (syntax & RE_NO_BK_PARENS ? *next == ')'
+        : next_backslash && next_next && *next_next == ')')
+       /* Before an alternative?  */
+    || (syntax & RE_NO_BK_VBAR ? *next == '|'
+        : next_backslash && next_next && *next_next == '|');
+}
+
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/* Read the ending character of a range (in a bracket expression) from the
+   uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+   starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+   Then we set the translation of all bits between the starting and
+   ending characters (inclusive) in the compiled pattern B.
+   
+   Return an error code.
+   
+   We use these short variable names so we can use the same macros as
+   `regex_compile' itself.  */
+
+static reg_errcode_t
+compile_range (p_ptr, pend, translate, syntax, b)
+    const char **p_ptr, *pend;
+    char *translate;
+    reg_syntax_t syntax;
+    unsigned char *b;
+{
+  unsigned this_char;
+
+  const char *p = *p_ptr;
+  int range_start, range_end;
+  
+  if (p == pend)
+    return REG_ERANGE;
+
+  /* Even though the pattern is a signed `char *', we need to fetch
+     with unsigned char *'s; if the high bit of the pattern character
+     is set, the range endpoints will be negative if we fetch using a
+     signed char *.
+
+     We also want to fetch the endpoints without translating them; the 
+     appropriate translation is done in the bit-setting loop below.  */
+  /* The SVR4 compiler on the 3B2 had trouble with unsigned const char *.  */
+  range_start = ((const unsigned char *) p)[-2];
+  range_end   = ((const unsigned char *) p)[0];
+
+  /* Have to increment the pointer into the pattern string, so the
+     caller isn't still at the ending character.  */
+  (*p_ptr)++;
+
+  /* If the start is after the end, the range is empty.  */
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  /* Here we see why `this_char' has to be larger than an `unsigned
+     char' -- the range is inclusive, so if `range_end' == 0xff
+     (assuming 8-bit characters), we would otherwise go into an infinite
+     loop, since all characters <= 0xff.  */
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      SET_LIST_BIT (TRANSLATE (this_char));
+    }
+  
+  return REG_NOERROR;
+}
+
+
+/* re_compile_fastmap computes a ``fastmap'' for the compiled pattern in
+   BUFP.  A fastmap records which of the (1 << BYTEWIDTH) possible
+   characters can start a string that matches the pattern.  This fastmap
+   is used by re_search to skip quickly over impossible starting points.
+
+   The caller must supply the address of a (1 << BYTEWIDTH)-byte data
+   area as BUFP->fastmap.
+   
+   We set the `fastmap', `fastmap_accurate', and `can_be_null' fields in
+   the pattern buffer.
+
+   Returns 0 if we succeed, -2 if an internal error.   */
+
+int
+re_compile_fastmap (bufp)
+     struct re_pattern_buffer *bufp;
+{
+  int j, k;
+#ifdef MATCH_MAY_ALLOCATE
+  fail_stack_type fail_stack;
+#endif
+#ifndef REGEX_MALLOC
+  char *destination;
+#endif
+  /* We don't push any register information onto the failure stack.  */
+  unsigned num_regs = 0;
+  
+  register char *fastmap = bufp->fastmap;
+  unsigned char *pattern = bufp->buffer;
+  unsigned long size = bufp->used;
+  unsigned char *p = pattern;
+  register unsigned char *pend = pattern + size;
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* Assume that each path through the pattern can be null until
+     proven otherwise.  We set this false at the bottom of switch
+     statement, to which we get only if a particular path doesn't
+     match the empty string.  */
+  boolean path_can_be_null = true;
+
+  /* We aren't doing a `succeed_n' to begin with.  */
+  boolean succeed_n_p = false;
+
+  assert (fastmap != NULL && p != NULL);
+  
+  INIT_FAIL_STACK ();
+  bzero (fastmap, 1 << BYTEWIDTH);  /* Assume nothing's valid.  */
+  bufp->fastmap_accurate = 1;	    /* It will be when we're done.  */
+  bufp->can_be_null = 0;
+      
+  while (1)
+    {
+      if (p == pend || *p == succeed)
+	{
+	  /* We have reached the (effective) end of pattern.  */
+	  if (!FAIL_STACK_EMPTY ())
+	    {
+	      bufp->can_be_null |= path_can_be_null;
+
+	      /* Reset for next path.  */
+	      path_can_be_null = true;
+
+	      p = fail_stack.stack[--fail_stack.avail].pointer;
+
+	      continue;
+	    }
+	  else
+	    break;
+	}
+
+      /* We should never be about to go beyond the end of the pattern.  */
+      assert (p < pend);
+      
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+
+        /* I guess the idea here is to simply not bother with a fastmap
+           if a backreference is used, since it's too hard to figure out
+           the fastmap for the corresponding group.  Setting
+           `can_be_null' stops `re_search_2' from using the fastmap, so
+           that is all we do.  */
+	case duplicate:
+	  bufp->can_be_null = 1;
+          goto done;
+
+
+      /* Following are the cases which match a character.  These end
+         with `break'.  */
+
+	case exactn:
+          fastmap[p[1]] = 1;
+	  break;
+
+
+        case charset:
+          for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH)))
+              fastmap[j] = 1;
+	  break;
+
+
+	case charset_not:
+	  /* Chars beyond end of map must be allowed.  */
+	  for (j = *p * BYTEWIDTH; j < (1 << BYTEWIDTH); j++)
+            fastmap[j] = 1;
+
+	  for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (!(p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH))))
+              fastmap[j] = 1;
+          break;
+
+
+	case wordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notwordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+        case anychar:
+	  {
+	    int fastmap_newline = fastmap['\n'];
+
+	    /* `.' matches anything ...  */
+	    for (j = 0; j < (1 << BYTEWIDTH); j++)
+	      fastmap[j] = 1;
+
+	    /* ... except perhaps newline.  */
+	    if (!(bufp->syntax & RE_DOT_NEWLINE))
+	      fastmap['\n'] = fastmap_newline;
+
+	    /* Return if we have already set `can_be_null'; if we have,
+	       then the fastmap is irrelevant.  Something's wrong here.  */
+	    else if (bufp->can_be_null)
+	      goto done;
+
+	    /* Otherwise, have to check alternative paths.  */
+	    break;
+	  }
+
+#ifdef emacs
+        case syntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notsyntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+      /* All cases after this match the empty string.  These end with
+         `continue'.  */
+
+
+	case before_dot:
+	case at_dot:
+	case after_dot:
+          continue;
+#endif /* not emacs */
+
+
+        case no_op:
+        case begline:
+        case endline:
+	case begbuf:
+	case endbuf:
+	case wordbound:
+	case notwordbound:
+	case wordbeg:
+	case wordend:
+        case push_dummy_failure:
+          continue;
+
+
+	case jump_n:
+        case pop_failure_jump:
+	case maybe_pop_jump:
+	case jump:
+        case jump_past_alt:
+	case dummy_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+	  p += j;	
+	  if (j > 0)
+	    continue;
+            
+          /* Jump backward implies we just went through the body of a
+             loop and matched nothing.  Opcode jumped to should be
+             `on_failure_jump' or `succeed_n'.  Just treat it like an
+             ordinary jump.  For a * loop, it has pushed its failure
+             point already; if so, discard that as redundant.  */
+          if ((re_opcode_t) *p != on_failure_jump
+	      && (re_opcode_t) *p != succeed_n)
+	    continue;
+
+          p++;
+          EXTRACT_NUMBER_AND_INCR (j, p);
+          p += j;		
+	  
+          /* If what's on the stack is where we are now, pop it.  */
+          if (!FAIL_STACK_EMPTY () 
+	      && fail_stack.stack[fail_stack.avail - 1].pointer == p)
+            fail_stack.avail--;
+
+          continue;
+
+
+        case on_failure_jump:
+        case on_failure_keep_string_jump:
+	handle_on_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+
+          /* For some patterns, e.g., `(a?)?', `p+j' here points to the
+             end of the pattern.  We don't want to push such a point,
+             since when we restore it above, entering the switch will
+             increment `p' past the end of the pattern.  We don't need
+             to push such a point since we obviously won't find any more
+             fastmap entries beyond `pend'.  Such a pattern can match
+             the null string, though.  */
+          if (p + j < pend)
+            {
+              if (!PUSH_PATTERN_OP (p + j, fail_stack))
+		{
+		  RESET_FAIL_STACK ();
+		  return -2;
+		}
+            }
+          else
+            bufp->can_be_null = 1;
+
+          if (succeed_n_p)
+            {
+              EXTRACT_NUMBER_AND_INCR (k, p);	/* Skip the n.  */
+              succeed_n_p = false;
+	    }
+
+          continue;
+
+
+	case succeed_n:
+          /* Get to the number of times to succeed.  */
+          p += 2;		
+
+          /* Increment p past the n for when k != 0.  */
+          EXTRACT_NUMBER_AND_INCR (k, p);
+          if (k == 0)
+	    {
+              p -= 4;
+  	      succeed_n_p = true;  /* Spaghetti code alert.  */
+              goto handle_on_failure_jump;
+            }
+          continue;
+
+
+	case set_number_at:
+          p += 4;
+          continue;
+
+
+	case start_memory:
+        case stop_memory:
+	  p += 2;
+	  continue;
+
+
+	default:
+          abort (); /* We have listed all the cases.  */
+        } /* switch *p++ */
+
+      /* Getting here means we have found the possible starting
+         characters for one path of the pattern -- and that the empty
+         string does not match.  We need not follow this path further.
+         Instead, look at the next alternative (remembered on the
+         stack), or quit if no more.  The test at the top of the loop
+         does these things.  */
+      path_can_be_null = false;
+      p = pend;
+    } /* while p */
+
+  /* Set `can_be_null' for the last path (also the first path, if the
+     pattern is empty).  */
+  bufp->can_be_null |= path_can_be_null;
+
+ done:
+  RESET_FAIL_STACK ();
+  return 0;
+} /* re_compile_fastmap */
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+    struct re_pattern_buffer *bufp;
+    struct re_registers *regs;
+    unsigned num_regs;
+    regoff_t *starts, *ends;
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t *) 0;
+    }
+}
+
+
+/* Searching routines.  */
+
+/* Like re_search_2, below, but only one string is specified, and
+   doesn't let you say where to stop matching. */
+
+int
+re_search (bufp, string, size, startpos, range, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, startpos, range;
+     struct re_registers *regs;
+{
+  return re_search_2 (bufp, NULL, 0, string, size, startpos, range, 
+		      regs, size);
+}
+
+
+/* Using the compiled pattern in BUFP->buffer, first tries to match the
+   virtual concatenation of STRING1 and STRING2, starting first at index
+   STARTPOS, then at STARTPOS + 1, and so on.
+   
+   STRING1 and STRING2 have length SIZE1 and SIZE2, respectively.
+   
+   RANGE is how far to scan while trying to match.  RANGE = 0 means try
+   only at STARTPOS; in general, the last start tried is STARTPOS +
+   RANGE.
+   
+   In REGS, return the indices of the virtual concatenation of STRING1
+   and STRING2 that matched the entire BUFP->buffer and its contained
+   subexpressions.
+   
+   Do not consider matching one past the index STOP in the virtual
+   concatenation of STRING1 and STRING2.
+
+   We return either the position in the strings at which the match was
+   found, -1 if no match, or -2 if error (such as failure
+   stack overflow).  */
+
+int
+re_search_2 (bufp, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+{
+  int val;
+  register char *fastmap = bufp->fastmap;
+  register char *translate = bufp->translate;
+  int total_size = size1 + size2;
+  int endpos = startpos + range;
+
+  /* Check for out-of-range STARTPOS.  */
+  if (startpos < 0 || startpos > total_size)
+    return -1;
+    
+  /* Fix up RANGE if it might eventually take us outside
+     the virtual concatenation of STRING1 and STRING2.  */
+  if (endpos < -1)
+    range = -1 - startpos;
+  else if (endpos > total_size)
+    range = total_size - startpos;
+
+  /* If the search isn't to be a backwards one, don't waste time in a
+     search for a pattern that must be anchored.  */
+  if (bufp->used > 0 && (re_opcode_t) bufp->buffer[0] == begbuf && range > 0)
+    {
+      if (startpos > 0)
+	return -1;
+      else
+	range = 1;
+    }
+
+  /* Update the fastmap now if not correct already.  */
+  if (fastmap && !bufp->fastmap_accurate)
+    if (re_compile_fastmap (bufp) == -2)
+      return -2;
+  
+  /* Loop through the string, looking for a place to start matching.  */
+  for (;;)
+    { 
+      /* If a fastmap is supplied, skip quickly over characters that
+         cannot be the start of a match.  If the pattern can match the
+         null string, however, we don't need to skip characters; we want
+         the first null string.  */
+      if (fastmap && startpos < total_size && !bufp->can_be_null)
+	{
+	  if (range > 0)	/* Searching forwards.  */
+	    {
+	      register const char *d;
+	      register int lim = 0;
+	      int irange = range;
+
+              if (startpos < size1 && startpos + range >= size1)
+                lim = range - (size1 - startpos);
+
+	      d = (startpos >= size1 ? string2 - size1 : string1) + startpos;
+   
+              /* Written out as an if-else to avoid testing `translate'
+                 inside the loop.  */
+	      if (translate)
+                while (range > lim
+                       && !fastmap[(unsigned char)
+				   translate[(unsigned char) *d++]])
+                  range--;
+	      else
+                while (range > lim && !fastmap[(unsigned char) *d++])
+                  range--;
+
+	      startpos += irange - range;
+	    }
+	  else				/* Searching backwards.  */
+	    {
+	      register char c = (size1 == 0 || startpos >= size1
+                                 ? string2[startpos - size1] 
+                                 : string1[startpos]);
+
+	      if (!fastmap[(unsigned char) TRANSLATE (c)])
+		goto advance;
+	    }
+	}
+
+      /* If can't match the null string, and that's all we have left, fail.  */
+      if (range >= 0 && startpos == total_size && fastmap
+          && !bufp->can_be_null)
+	return -1;
+
+      val = re_match_2_internal (bufp, string1, size1, string2, size2,
+				 startpos, regs, stop);
+#ifndef REGEX_MALLOC
+#ifdef C_ALLOCA
+      alloca (0);
+#endif
+#endif
+
+      if (val >= 0)
+	return startpos;
+        
+      if (val == -2)
+	return -2;
+
+    advance:
+      if (!range) 
+        break;
+      else if (range > 0) 
+        {
+          range--; 
+          startpos++;
+        }
+      else
+        {
+          range++; 
+          startpos--;
+        }
+    }
+  return -1;
+} /* re_search_2 */
+
+
+/* Declarations and macros for re_match_2.  */
+
+static int bcmp_translate ();
+static boolean alt_match_null_string_p (),
+               common_op_match_null_string_p (),
+               group_match_null_string_p ();
+
+/* This converts PTR, a pointer into one of the search strings `string1'
+   and `string2' into an offset from the beginning of that string.  */
+#define POINTER_TO_OFFSET(ptr)			\
+  (FIRST_STRING_P (ptr)				\
+   ? ((regoff_t) ((ptr) - string1))		\
+   : ((regoff_t) ((ptr) - string2 + size1)))
+
+/* Macros for dealing with the split strings in re_match_2.  */
+
+#define MATCHING_IN_FIRST_STRING  (dend == end_match_1)
+
+/* Call before fetching a character with *d.  This switches over to
+   string2 if necessary.  */
+#define PREFETCH()							\
+  while (d == dend)						    	\
+    {									\
+      /* End of string2 => fail.  */					\
+      if (dend == end_match_2) 						\
+        goto fail;							\
+      /* End of string1 => advance to string2.  */ 			\
+      d = string2;						        \
+      dend = end_match_2;						\
+    }
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+   of `string1' and `string2'.  If only one string, it's `string2'.  */
+#define AT_STRINGS_BEG(d) ((d) == (size1 ? string1 : string2) || !size2)
+#define AT_STRINGS_END(d) ((d) == end2)	
+
+
+/* Test if D points to a character which is word-constituent.  We have
+   two special cases to check for: if past the end of string1, look at
+   the first character in string2; and if before the beginning of
+   string2, look at the last character in string1.  */
+#define WORDCHAR_P(d)							\
+  (SYNTAX ((d) == end1 ? *string2					\
+           : (d) == string2 - 1 ? *(end1 - 1) : *(d))			\
+   == Sword)
+
+/* Test if the character before D and the one at D differ with respect
+   to being word-constituent.  */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG (d) || AT_STRINGS_END (d)				\
+   || WORDCHAR_P (d - 1) != WORDCHAR_P (d))
+
+
+/* Free everything we malloc.  */
+#ifdef MATCH_MAY_ALLOCATE
+#define FREE_VAR(var) if (var) REGEX_FREE (var); var = NULL
+#define FREE_VARIABLES()						\
+  do {									\
+    REGEX_FREE_STACK (fail_stack.stack);				\
+    FREE_VAR (regstart);						\
+    FREE_VAR (regend);							\
+    FREE_VAR (old_regstart);						\
+    FREE_VAR (old_regend);						\
+    FREE_VAR (best_regstart);						\
+    FREE_VAR (best_regend);						\
+    FREE_VAR (reg_info);						\
+    FREE_VAR (reg_dummy);						\
+    FREE_VAR (reg_info_dummy);						\
+  } while (0)
+#else
+#define FREE_VARIABLES() ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+#endif /* not MATCH_MAY_ALLOCATE */
+
+/* These values must meet several constraints.  They must not be valid
+   register values; since we have a limit of 255 registers (because
+   we use only one byte in the pattern for the register number), we can
+   use numbers larger than 255.  They must differ by 1, because of
+   NUM_FAILURE_ITEMS above.  And the value for the lowest register must
+   be larger than the value for the highest register, so we do not try
+   to actually save any registers when none are active.  */
+#define NO_HIGHEST_ACTIVE_REG (1 << BYTEWIDTH)
+#define NO_LOWEST_ACTIVE_REG (NO_HIGHEST_ACTIVE_REG + 1)
+
+
+/* Matching routines.  */
+
+#ifndef emacs   /* Emacs never uses this.  */
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+int
+re_match (bufp, string, size, pos, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, pos;
+     struct re_registers *regs;
+{
+  int result = re_match_2_internal (bufp, NULL, 0, string, size,
+				    pos, regs, size);
+  alloca (0);
+  return result;
+}
+#endif /* not emacs */
+
+
+/* re_match_2 matches the compiled pattern in BUFP against the
+   the (virtual) concatenation of STRING1 and STRING2 (of length SIZE1
+   and SIZE2, respectively).  We start matching at POS, and stop
+   matching at STOP.
+   
+   If REGS is non-null and the `no_sub' field of BUFP is nonzero, we
+   store offsets for the substring each group matched in REGS.  See the
+   documentation for exactly how many groups we fill.
+
+   We return -1 if no match, -2 if an internal error (such as the
+   failure stack overflowing).  Otherwise, we return the length of the
+   matched substring.  */
+
+int
+re_match_2 (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  int result = re_match_2_internal (bufp, string1, size1, string2, size2,
+				    pos, regs, stop);
+  alloca (0);
+  return result;
+}
+
+/* This is a separate function so that we can force an alloca cleanup
+   afterwards.  */
+static int
+re_match_2_internal (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  /* General temporaries.  */
+  int mcnt;
+  unsigned char *p1;
+
+  /* Just past the end of the corresponding string.  */
+  const char *end1, *end2;
+
+  /* Pointers into string1 and string2, just past the last characters in
+     each to consider matching.  */
+  const char *end_match_1, *end_match_2;
+
+  /* Where we are in the data, and the end of the current string.  */
+  const char *d, *dend;
+  
+  /* Where we are in the pattern, and the end of the pattern.  */
+  unsigned char *p = bufp->buffer;
+  register unsigned char *pend = p + bufp->used;
+
+  /* Mark the opcode just after a start_memory, so we can test for an
+     empty subpattern when we get to the stop_memory.  */
+  unsigned char *just_past_start_mem = 0;
+
+  /* We use this to map every character in the string.  */
+  char *translate = bufp->translate;
+
+  /* Failure point stack.  Each place that can handle a failure further
+     down the line pushes a failure point on this stack.  It consists of
+     restart, regend, and reg_info for all registers corresponding to
+     the subexpressions we're currently inside, plus the number of such
+     registers, and, finally, two char *'s.  The first char * is where
+     to resume scanning the pattern; the second one is where to resume
+     scanning the strings.  If the latter is zero, the failure point is
+     a ``dummy''; if a failure happens and the failure point is a dummy,
+     it gets discarded and the next next one is tried.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  fail_stack_type fail_stack;
+#endif
+#ifdef DEBUG
+  static unsigned failure_id = 0;
+  unsigned nfailure_points_pushed = 0, nfailure_points_popped = 0;
+#endif
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* We fill all the registers internally, independent of what we
+     return, for use in backreferences.  The number here includes
+     an element for register zero.  */
+  unsigned num_regs = bufp->re_nsub + 1;
+  
+  /* The currently active registers.  */
+  unsigned lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+  unsigned highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+
+  /* Information on the contents of registers. These are pointers into
+     the input strings; they record just what was matched (on this
+     attempt) by a subexpression part of the pattern, that is, the
+     regnum-th regstart pointer points to where in the pattern we began
+     matching and the regnum-th regend points to right after where we
+     stopped matching the regnum-th subexpression.  (The zeroth register
+     keeps track of what the whole pattern matches.)  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **regstart, **regend;
+#endif
+
+  /* If a group that's operated upon by a repetition operator fails to
+     match anything, then the register for its start will need to be
+     restored because it will have been set to wherever in the string we
+     are when we last see its open-group operator.  Similarly for a
+     register's end.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **old_regstart, **old_regend;
+#endif
+
+  /* The is_active field of reg_info helps us keep track of which (possibly
+     nested) subexpressions we are currently in. The matched_something
+     field of reg_info[reg_num] helps us tell whether or not we have
+     matched any of the pattern so far this time through the reg_num-th
+     subexpression.  These two fields get reset each time through any
+     loop their register is in.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  register_info_type *reg_info; 
+#endif
+
+  /* The following record the register info as found in the above
+     variables when we find a match better than any we've seen before. 
+     This happens as we backtrack through the failure points, which in
+     turn happens only if we have not yet matched the entire string. */
+  unsigned best_regs_set = false;
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **best_regstart, **best_regend;
+#endif
+  
+  /* Logically, this is `best_regend[0]'.  But we don't want to have to
+     allocate space for that if we're not allocating space for anything
+     else (see below).  Also, we never need info about register 0 for
+     any of the other register vectors, and it seems rather a kludge to
+     treat `best_regend' differently than the rest.  So we keep track of
+     the end of the best match so far in a separate variable.  We
+     initialize this to NULL so that when we backtrack the first time
+     and need to test it, it's not garbage.  */
+  const char *match_end = NULL;
+
+  /* This helps SET_REGS_MATCHED avoid doing redundant work.  */
+  int set_regs_matched_done = 0;
+
+  /* Used when we pop values we don't care about.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **reg_dummy;
+  register_info_type *reg_info_dummy;
+#endif
+
+#ifdef DEBUG
+  /* Counts the total number of registers pushed.  */
+  unsigned num_regs_pushed = 0; 	
+#endif
+
+  DEBUG_PRINT1 ("\n\nEntering re_match_2.\n");
+  
+  INIT_FAIL_STACK ();
+  
+#ifdef MATCH_MAY_ALLOCATE
+  /* Do not bother to initialize all the register variables if there are
+     no groups in the pattern, as it takes a fair amount of time.  If
+     there are groups, we include space for register 0 (the whole
+     pattern), even though we never use it, since it simplifies the
+     array indexing.  We should fix this.  */
+  if (bufp->re_nsub)
+    {
+      regstart = REGEX_TALLOC (num_regs, const char *);
+      regend = REGEX_TALLOC (num_regs, const char *);
+      old_regstart = REGEX_TALLOC (num_regs, const char *);
+      old_regend = REGEX_TALLOC (num_regs, const char *);
+      best_regstart = REGEX_TALLOC (num_regs, const char *);
+      best_regend = REGEX_TALLOC (num_regs, const char *);
+      reg_info = REGEX_TALLOC (num_regs, register_info_type);
+      reg_dummy = REGEX_TALLOC (num_regs, const char *);
+      reg_info_dummy = REGEX_TALLOC (num_regs, register_info_type);
+
+      if (!(regstart && regend && old_regstart && old_regend && reg_info 
+            && best_regstart && best_regend && reg_dummy && reg_info_dummy)) 
+        {
+          FREE_VARIABLES ();
+          return -2;
+        }
+    }
+  else
+    {
+      /* We must initialize all our variables to NULL, so that
+         `FREE_VARIABLES' doesn't try to free them.  */
+      regstart = regend = old_regstart = old_regend = best_regstart
+        = best_regend = reg_dummy = NULL;
+      reg_info = reg_info_dummy = (register_info_type *) NULL;
+    }
+#endif /* MATCH_MAY_ALLOCATE */
+
+  /* The starting position is bogus.  */
+  if (pos < 0 || pos > size1 + size2)
+    {
+      FREE_VARIABLES ();
+      return -1;
+    }
+    
+  /* Initialize subexpression text positions to -1 to mark ones that no
+     start_memory/stop_memory has been seen for. Also initialize the
+     register information struct.  */
+  for (mcnt = 1; mcnt < num_regs; mcnt++)
+    {
+      regstart[mcnt] = regend[mcnt] 
+        = old_regstart[mcnt] = old_regend[mcnt] = REG_UNSET_VALUE;
+        
+      REG_MATCH_NULL_STRING_P (reg_info[mcnt]) = MATCH_NULL_UNSET_VALUE;
+      IS_ACTIVE (reg_info[mcnt]) = 0;
+      MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+      EVER_MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+    }
+  
+  /* We move `string1' into `string2' if the latter's empty -- but not if
+     `string1' is null.  */
+  if (size2 == 0 && string1 != NULL)
+    {
+      string2 = string1;
+      size2 = size1;
+      string1 = 0;
+      size1 = 0;
+    }
+  end1 = string1 + size1;
+  end2 = string2 + size2;
+
+  /* Compute where to stop matching, within the two strings.  */
+  if (stop <= size1)
+    {
+      end_match_1 = string1 + stop;
+      end_match_2 = string2;
+    }
+  else
+    {
+      end_match_1 = end1;
+      end_match_2 = string2 + stop - size1;
+    }
+
+  /* `p' scans through the pattern as `d' scans through the data. 
+     `dend' is the end of the input string that `d' points within.  `d'
+     is advanced into the following input string whenever necessary, but
+     this happens before fetching; therefore, at the beginning of the
+     loop, `d' can be pointing at the end of a string, but it cannot
+     equal `string2'.  */
+  if (size1 > 0 && pos <= size1)
+    {
+      d = string1 + pos;
+      dend = end_match_1;
+    }
+  else
+    {
+      d = string2 + pos - size1;
+      dend = end_match_2;
+    }
+
+  DEBUG_PRINT1 ("The compiled pattern is: ");
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, p, pend);
+  DEBUG_PRINT1 ("The string to match is: `");
+  DEBUG_PRINT_DOUBLE_STRING (d, string1, size1, string2, size2);
+  DEBUG_PRINT1 ("'\n");
+  
+  /* This loops over pattern commands.  It exits by returning from the
+     function if the match is complete, or it drops through if the match
+     fails at this starting point in the input data.  */
+  for (;;)
+    {
+      DEBUG_PRINT2 ("\n0x%x: ", p);
+
+      if (p == pend)
+	{ /* End of pattern means we might have succeeded.  */
+          DEBUG_PRINT1 ("end of pattern ... ");
+          
+	  /* If we haven't matched the entire string, and we want the
+             longest match, try backtracking.  */
+          if (d != end_match_2)
+	    {
+	      /* 1 if this match ends in the same string (string1 or string2)
+		 as the best previous match.  */
+	      boolean same_str_p = (FIRST_STRING_P (match_end) 
+				    == MATCHING_IN_FIRST_STRING);
+	      /* 1 if this match is the best seen so far.  */
+	      boolean best_match_p;
+
+	      /* AIX compiler got confused when this was combined
+		 with the previous declaration.  */
+	      if (same_str_p)
+		best_match_p = d > match_end;
+	      else
+		best_match_p = !MATCHING_IN_FIRST_STRING;
+
+              DEBUG_PRINT1 ("backtracking.\n");
+              
+              if (!FAIL_STACK_EMPTY ())
+                { /* More failure points to try.  */
+
+                  /* If exceeds best match so far, save it.  */
+                  if (!best_regs_set || best_match_p)
+                    {
+                      best_regs_set = true;
+                      match_end = d;
+                      
+                      DEBUG_PRINT1 ("\nSAVING match as best so far.\n");
+                      
+                      for (mcnt = 1; mcnt < num_regs; mcnt++)
+                        {
+                          best_regstart[mcnt] = regstart[mcnt];
+                          best_regend[mcnt] = regend[mcnt];
+                        }
+                    }
+                  goto fail;	       
+                }
+
+              /* If no failure points, don't restore garbage.  And if
+                 last match is real best match, don't restore second
+                 best one. */
+              else if (best_regs_set && !best_match_p)
+                {
+  	        restore_best_regs:
+                  /* Restore best match.  It may happen that `dend ==
+                     end_match_1' while the restored d is in string2.
+                     For example, the pattern `x.*y.*z' against the
+                     strings `x-' and `y-z-', if the two strings are
+                     not consecutive in memory.  */
+                  DEBUG_PRINT1 ("Restoring best registers.\n");
+                  
+                  d = match_end;
+                  dend = ((d >= string1 && d <= end1)
+		           ? end_match_1 : end_match_2);
+
+		  for (mcnt = 1; mcnt < num_regs; mcnt++)
+		    {
+		      regstart[mcnt] = best_regstart[mcnt];
+		      regend[mcnt] = best_regend[mcnt];
+		    }
+                }
+            } /* d != end_match_2 */
+
+	succeed_label:
+          DEBUG_PRINT1 ("Accepting match.\n");
+
+          /* If caller wants register contents data back, do it.  */
+          if (regs && !bufp->no_sub)
+	    {
+              /* Have the register data arrays been allocated?  */
+              if (bufp->regs_allocated == REGS_UNALLOCATED)
+                { /* No.  So allocate them with malloc.  We need one
+                     extra element beyond `num_regs' for the `-1' marker
+                     GNU code uses.  */
+                  regs->num_regs = MAX (RE_NREGS, num_regs + 1);
+                  regs->start = TALLOC (regs->num_regs, regoff_t);
+                  regs->end = TALLOC (regs->num_regs, regoff_t);
+                  if (regs->start == NULL || regs->end == NULL)
+		    {
+		      FREE_VARIABLES ();
+		      return -2;
+		    }
+                  bufp->regs_allocated = REGS_REALLOCATE;
+                }
+              else if (bufp->regs_allocated == REGS_REALLOCATE)
+                { /* Yes.  If we need more elements than were already
+                     allocated, reallocate them.  If we need fewer, just
+                     leave it alone.  */
+                  if (regs->num_regs < num_regs + 1)
+                    {
+                      regs->num_regs = num_regs + 1;
+                      RETALLOC (regs->start, regs->num_regs, regoff_t);
+                      RETALLOC (regs->end, regs->num_regs, regoff_t);
+                      if (regs->start == NULL || regs->end == NULL)
+			{
+			  FREE_VARIABLES ();
+			  return -2;
+			}
+                    }
+                }
+              else
+		{
+		  /* These braces fend off a "empty body in an else-statement"
+		     warning under GCC when assert expands to nothing.  */
+		  assert (bufp->regs_allocated == REGS_FIXED);
+		}
+
+              /* Convert the pointer data in `regstart' and `regend' to
+                 indices.  Register zero has to be set differently,
+                 since we haven't kept track of any info for it.  */
+              if (regs->num_regs > 0)
+                {
+                  regs->start[0] = pos;
+                  regs->end[0] = (MATCHING_IN_FIRST_STRING
+				  ? ((regoff_t) (d - string1))
+			          : ((regoff_t) (d - string2 + size1)));
+                }
+              
+              /* Go through the first `min (num_regs, regs->num_regs)'
+                 registers, since that is all we initialized.  */
+	      for (mcnt = 1; mcnt < MIN (num_regs, regs->num_regs); mcnt++)
+		{
+                  if (REG_UNSET (regstart[mcnt]) || REG_UNSET (regend[mcnt]))
+                    regs->start[mcnt] = regs->end[mcnt] = -1;
+                  else
+                    {
+		      regs->start[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regstart[mcnt]);
+                      regs->end[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regend[mcnt]);
+                    }
+		}
+              
+              /* If the regs structure we return has more elements than
+                 were in the pattern, set the extra elements to -1.  If
+                 we (re)allocated the registers, this is the case,
+                 because we always allocate enough to have at least one
+                 -1 at the end.  */
+              for (mcnt = num_regs; mcnt < regs->num_regs; mcnt++)
+                regs->start[mcnt] = regs->end[mcnt] = -1;
+	    } /* regs && !bufp->no_sub */
+
+          DEBUG_PRINT4 ("%u failure points pushed, %u popped (%u remain).\n",
+                        nfailure_points_pushed, nfailure_points_popped,
+                        nfailure_points_pushed - nfailure_points_popped);
+          DEBUG_PRINT2 ("%u registers pushed.\n", num_regs_pushed);
+
+          mcnt = d - pos - (MATCHING_IN_FIRST_STRING 
+			    ? string1 
+			    : string2 - size1);
+
+          DEBUG_PRINT2 ("Returning %d from re_match_2.\n", mcnt);
+
+          FREE_VARIABLES ();
+          return mcnt;
+        }
+
+      /* Otherwise match next pattern command.  */
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+        /* Ignore these.  Used to ignore the n of succeed_n's which
+           currently have n == 0.  */
+        case no_op:
+          DEBUG_PRINT1 ("EXECUTING no_op.\n");
+          break;
+
+	case succeed:
+          DEBUG_PRINT1 ("EXECUTING succeed.\n");
+	  goto succeed_label;
+
+        /* Match the next n pattern characters exactly.  The following
+           byte in the pattern defines n, and the n bytes after that
+           are the characters to match.  */
+	case exactn:
+	  mcnt = *p++;
+          DEBUG_PRINT2 ("EXECUTING exactn %d.\n", mcnt);
+
+          /* This is written out as an if-else so we don't waste time
+             testing `translate' inside the loop.  */
+          if (translate)
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (translate[(unsigned char) *d++] != (char) *p++)
+                    goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  else
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (*d++ != (char) *p++) goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  SET_REGS_MATCHED ();
+          break;
+
+
+        /* Match any character except possibly a newline or a null.  */
+	case anychar:
+          DEBUG_PRINT1 ("EXECUTING anychar.\n");
+
+          PREFETCH ();
+
+          if ((!(bufp->syntax & RE_DOT_NEWLINE) && TRANSLATE (*d) == '\n')
+              || (bufp->syntax & RE_DOT_NOT_NULL && TRANSLATE (*d) == '\000'))
+	    goto fail;
+
+          SET_REGS_MATCHED ();
+          DEBUG_PRINT2 ("  Matched `%d'.\n", *d);
+          d++;
+	  break;
+
+
+	case charset:
+	case charset_not:
+	  {
+	    register unsigned char c;
+	    boolean not = (re_opcode_t) *(p - 1) == charset_not;
+
+            DEBUG_PRINT2 ("EXECUTING charset%s.\n", not ? "_not" : "");
+
+	    PREFETCH ();
+	    c = TRANSLATE (*d); /* The character to match.  */
+
+            /* Cast to `unsigned' instead of `unsigned char' in case the
+               bit list is a full 32 bytes long.  */
+	    if (c < (unsigned) (*p * BYTEWIDTH)
+		&& p[1 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+	      not = !not;
+
+	    p += 1 + *p;
+
+	    if (!not) goto fail;
+            
+	    SET_REGS_MATCHED ();
+            d++;
+	    break;
+	  }
+
+
+        /* The beginning of a group is represented by start_memory.
+           The arguments are the register number in the next byte, and the
+           number of groups inner to this one in the next.  The text
+           matched within the group is recorded (in the internal
+           registers data structure) under the register number.  */
+        case start_memory:
+	  DEBUG_PRINT3 ("EXECUTING start_memory %d (%d):\n", *p, p[1]);
+
+          /* Find out if this group can match the empty string.  */
+	  p1 = p;		/* To send to group_match_null_string_p.  */
+          
+          if (REG_MATCH_NULL_STRING_P (reg_info[*p]) == MATCH_NULL_UNSET_VALUE)
+            REG_MATCH_NULL_STRING_P (reg_info[*p]) 
+              = group_match_null_string_p (&p1, pend, reg_info);
+
+          /* Save the position in the string where we were the last time
+             we were at this open-group operator in case the group is
+             operated upon by a repetition operator, e.g., with `(a*)*b'
+             against `ab'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regstart[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                             ? REG_UNSET (regstart[*p]) ? d : regstart[*p]
+                             : regstart[*p];
+	  DEBUG_PRINT2 ("  old_regstart: %d\n", 
+			 POINTER_TO_OFFSET (old_regstart[*p]));
+
+          regstart[*p] = d;
+	  DEBUG_PRINT2 ("  regstart: %d\n", POINTER_TO_OFFSET (regstart[*p]));
+
+          IS_ACTIVE (reg_info[*p]) = 1;
+          MATCHED_SOMETHING (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+          
+          /* This is the new highest active register.  */
+          highest_active_reg = *p;
+          
+          /* If nothing was active before, this is the new lowest active
+             register.  */
+          if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+            lowest_active_reg = *p;
+
+          /* Move past the register number and inner group count.  */
+          p += 2;
+	  just_past_start_mem = p;
+
+          break;
+
+
+        /* The stop_memory opcode represents the end of a group.  Its
+           arguments are the same as start_memory's: the register
+           number, and the number of inner groups.  */
+	case stop_memory:
+	  DEBUG_PRINT3 ("EXECUTING stop_memory %d (%d):\n", *p, p[1]);
+             
+          /* We need to save the string position the last time we were at
+             this close-group operator in case the group is operated
+             upon by a repetition operator, e.g., with `((a*)*(b*)*)*'
+             against `aba'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regend[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                           ? REG_UNSET (regend[*p]) ? d : regend[*p]
+			   : regend[*p];
+	  DEBUG_PRINT2 ("      old_regend: %d\n", 
+			 POINTER_TO_OFFSET (old_regend[*p]));
+
+          regend[*p] = d;
+	  DEBUG_PRINT2 ("      regend: %d\n", POINTER_TO_OFFSET (regend[*p]));
+
+          /* This register isn't active anymore.  */
+          IS_ACTIVE (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+
+          /* If this was the only register active, nothing is active
+             anymore.  */
+          if (lowest_active_reg == highest_active_reg)
+            {
+              lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+              highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+            }
+          else
+            { /* We must scan for the new highest active register, since
+                 it isn't necessarily one less than now: consider
+                 (a(b)c(d(e)f)g).  When group 3 ends, after the f), the
+                 new highest active register is 1.  */
+              unsigned char r = *p - 1;
+              while (r > 0 && !IS_ACTIVE (reg_info[r]))
+                r--;
+              
+              /* If we end up at register zero, that means that we saved
+                 the registers as the result of an `on_failure_jump', not
+                 a `start_memory', and we jumped to past the innermost
+                 `stop_memory'.  For example, in ((.)*) we save
+                 registers 1 and 2 as a result of the *, but when we pop
+                 back to the second ), we are at the stop_memory 1.
+                 Thus, nothing is active.  */
+	      if (r == 0)
+                {
+                  lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+                  highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+                }
+              else
+                highest_active_reg = r;
+            }
+          
+          /* If just failed to match something this time around with a
+             group that's operated on by a repetition operator, try to
+             force exit from the ``loop'', and restore the register
+             information for this group that we had before trying this
+             last match.  */
+          if ((!MATCHED_SOMETHING (reg_info[*p])
+               || just_past_start_mem == p - 1)
+	      && (p + 2) < pend)              
+            {
+              boolean is_a_jump_n = false;
+              
+              p1 = p + 2;
+              mcnt = 0;
+              switch ((re_opcode_t) *p1++)
+                {
+                  case jump_n:
+		    is_a_jump_n = true;
+                  case pop_failure_jump:
+		  case maybe_pop_jump:
+		  case jump:
+		  case dummy_failure_jump:
+                    EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+		    if (is_a_jump_n)
+		      p1 += 2;
+                    break;
+                  
+                  default:
+                    /* do nothing */ ;
+                }
+	      p1 += mcnt;
+        
+              /* If the next operation is a jump backwards in the pattern
+	         to an on_failure_jump right before the start_memory
+                 corresponding to this stop_memory, exit from the loop
+                 by forcing a failure after pushing on the stack the
+                 on_failure_jump's jump in the pattern, and d.  */
+              if (mcnt < 0 && (re_opcode_t) *p1 == on_failure_jump
+                  && (re_opcode_t) p1[3] == start_memory && p1[4] == *p)
+		{
+                  /* If this group ever matched anything, then restore
+                     what its registers were before trying this last
+                     failed match, e.g., with `(a*)*b' against `ab' for
+                     regstart[1], and, e.g., with `((a*)*(b*)*)*'
+                     against `aba' for regend[3].
+                     
+                     Also restore the registers for inner groups for,
+                     e.g., `((a*)(b*))*' against `aba' (register 3 would
+                     otherwise get trashed).  */
+                     
+                  if (EVER_MATCHED_SOMETHING (reg_info[*p]))
+		    {
+		      unsigned r; 
+        
+                      EVER_MATCHED_SOMETHING (reg_info[*p]) = 0;
+                      
+		      /* Restore this and inner groups' (if any) registers.  */
+                      for (r = *p; r < *p + *(p + 1); r++)
+                        {
+                          regstart[r] = old_regstart[r];
+
+                          /* xx why this test?  */
+                          if (old_regend[r] >= regstart[r])
+                            regend[r] = old_regend[r];
+                        }     
+                    }
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  PUSH_FAILURE_POINT (p1 + mcnt, d, -2);
+
+                  goto fail;
+                }
+            }
+          
+          /* Move past the register number and the inner group count.  */
+          p += 2;
+          break;
+
+
+	/* \<digit> has been turned into a `duplicate' command which is
+           followed by the numeric value of <digit> as the register number.  */
+        case duplicate:
+	  {
+	    register const char *d2, *dend2;
+	    int regno = *p++;   /* Get which register to match against.  */
+	    DEBUG_PRINT2 ("EXECUTING duplicate %d.\n", regno);
+
+	    /* Can't back reference a group which we've never matched.  */
+            if (REG_UNSET (regstart[regno]) || REG_UNSET (regend[regno]))
+              goto fail;
+              
+            /* Where in input to try to start matching.  */
+            d2 = regstart[regno];
+            
+            /* Where to stop matching; if both the place to start and
+               the place to stop matching are in the same string, then
+               set to the place to stop, otherwise, for now have to use
+               the end of the first string.  */
+
+            dend2 = ((FIRST_STRING_P (regstart[regno]) 
+		      == FIRST_STRING_P (regend[regno]))
+		     ? regend[regno] : end_match_1);
+	    for (;;)
+	      {
+		/* If necessary, advance to next segment in register
+                   contents.  */
+		while (d2 == dend2)
+		  {
+		    if (dend2 == end_match_2) break;
+		    if (dend2 == regend[regno]) break;
+
+                    /* End of string1 => advance to string2. */
+                    d2 = string2;
+                    dend2 = regend[regno];
+		  }
+		/* At end of register contents => success */
+		if (d2 == dend2) break;
+
+		/* If necessary, advance to next segment in data.  */
+		PREFETCH ();
+
+		/* How many characters left in this segment to match.  */
+		mcnt = dend - d;
+                
+		/* Want how many consecutive characters we can match in
+                   one shot, so, if necessary, adjust the count.  */
+                if (mcnt > dend2 - d2)
+		  mcnt = dend2 - d2;
+                  
+		/* Compare that many; failure if mismatch, else move
+                   past them.  */
+		if (translate 
+                    ? bcmp_translate (d, d2, mcnt, translate) 
+                    : bcmp (d, d2, mcnt))
+		  goto fail;
+		d += mcnt, d2 += mcnt;
+
+		/* Do this because we've match some characters.  */
+		SET_REGS_MATCHED ();
+	      }
+	  }
+	  break;
+
+
+        /* begline matches the empty string at the beginning of the string
+           (unless `not_bol' is set in `bufp'), and, if
+           `newline_anchor' is set, after newlines.  */
+	case begline:
+          DEBUG_PRINT1 ("EXECUTING begline.\n");
+          
+          if (AT_STRINGS_BEG (d))
+            {
+              if (!bufp->not_bol) break;
+            }
+          else if (d[-1] == '\n' && bufp->newline_anchor)
+            {
+              break;
+            }
+          /* In all other cases, we fail.  */
+          goto fail;
+
+
+        /* endline is the dual of begline.  */
+	case endline:
+          DEBUG_PRINT1 ("EXECUTING endline.\n");
+
+          if (AT_STRINGS_END (d))
+            {
+              if (!bufp->not_eol) break;
+            }
+          
+          /* We have to ``prefetch'' the next character.  */
+          else if ((d == end1 ? *string2 : *d) == '\n'
+                   && bufp->newline_anchor)
+            {
+              break;
+            }
+          goto fail;
+
+
+	/* Match at the very beginning of the data.  */
+        case begbuf:
+          DEBUG_PRINT1 ("EXECUTING begbuf.\n");
+          if (AT_STRINGS_BEG (d))
+            break;
+          goto fail;
+
+
+	/* Match at the very end of the data.  */
+        case endbuf:
+          DEBUG_PRINT1 ("EXECUTING endbuf.\n");
+	  if (AT_STRINGS_END (d))
+	    break;
+          goto fail;
+
+
+        /* on_failure_keep_string_jump is used to optimize `.*\n'.  It
+           pushes NULL as the value for the string on the stack.  Then
+           `pop_failure_point' will keep the current value for the
+           string, instead of restoring it.  To see why, consider
+           matching `foo\nbar' against `.*\n'.  The .* matches the foo;
+           then the . fails against the \n.  But the next thing we want
+           to do is match the \n against the \n; if we restored the
+           string value, we would be back at the foo.
+           
+           Because this is used only in specific cases, we don't need to
+           check all the things that `on_failure_jump' does, to make
+           sure the right things get saved on the stack.  Hence we don't
+           share its code.  The only reason to push anything on the
+           stack at all is that otherwise we would have to change
+           `anychar's code to do something besides goto fail in this
+           case; that seems worse than this.  */
+        case on_failure_keep_string_jump:
+          DEBUG_PRINT1 ("EXECUTING on_failure_keep_string_jump");
+          
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x):\n", mcnt, p + mcnt);
+
+          PUSH_FAILURE_POINT (p + mcnt, NULL, -2);
+          break;
+
+
+	/* Uses of on_failure_jump:
+        
+           Each alternative starts with an on_failure_jump that points
+           to the beginning of the next alternative.  Each alternative
+           except the last ends with a jump that in effect jumps past
+           the rest of the alternatives.  (They really jump to the
+           ending jump of the following alternative, because tensioning
+           these jumps is a hassle.)
+
+           Repeats start with an on_failure_jump that points past both
+           the repetition text and either the following jump or
+           pop_failure_jump back to this on_failure_jump.  */
+	case on_failure_jump:
+        on_failure:
+          DEBUG_PRINT1 ("EXECUTING on_failure_jump");
+
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x)", mcnt, p + mcnt);
+
+          /* If this on_failure_jump comes right before a group (i.e.,
+             the original * applied to a group), save the information
+             for that group and all inner ones, so that if we fail back
+             to this point, the group's information will be correct.
+             For example, in \(a*\)*\1, we need the preceding group,
+             and in \(\(a*\)b*\)\2, we need the inner group.  */
+
+          /* We can't use `p' to check ahead because we push
+             a failure point to `p + mcnt' after we do this.  */
+          p1 = p;
+
+          /* We need to skip no_op's before we look for the
+             start_memory in case this on_failure_jump is happening as
+             the result of a completed succeed_n, as in \(a\)\{1,3\}b\1
+             against aba.  */
+          while (p1 < pend && (re_opcode_t) *p1 == no_op)
+            p1++;
+
+          if (p1 < pend && (re_opcode_t) *p1 == start_memory)
+            {
+              /* We have a new highest active register now.  This will
+                 get reset at the start_memory we are about to get to,
+                 but we will have saved all the registers relevant to
+                 this repetition op, as described above.  */
+              highest_active_reg = *(p1 + 1) + *(p1 + 2);
+              if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+                lowest_active_reg = *(p1 + 1);
+            }
+
+          DEBUG_PRINT1 (":\n");
+          PUSH_FAILURE_POINT (p + mcnt, d, -2);
+          break;
+
+
+        /* A smart repeat ends with `maybe_pop_jump'.
+	   We change it to either `pop_failure_jump' or `jump'.  */
+        case maybe_pop_jump:
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT2 ("EXECUTING maybe_pop_jump %d.\n", mcnt);
+          {
+	    register unsigned char *p2 = p;
+
+            /* Compare the beginning of the repeat with what in the
+               pattern follows its end. If we can establish that there
+               is nothing that they would both match, i.e., that we
+               would have to backtrack because of (as in, e.g., `a*a')
+               then we can change to pop_failure_jump, because we'll
+               never have to backtrack.
+               
+               This is not true in the case of alternatives: in
+               `(a|ab)*' we do need to backtrack to the `ab' alternative
+               (e.g., if the string was `ab').  But instead of trying to
+               detect that here, the alternative has put on a dummy
+               failure point which is what we will end up popping.  */
+
+	    /* Skip over open/close-group commands.
+	       If what follows this loop is a ...+ construct,
+	       look at what begins its body, since we will have to
+	       match at least one of that.  */
+	    while (1)
+	      {
+		if (p2 + 2 < pend
+		    && ((re_opcode_t) *p2 == stop_memory
+			|| (re_opcode_t) *p2 == start_memory))
+		  p2 += 3;
+		else if (p2 + 6 < pend
+			 && (re_opcode_t) *p2 == dummy_failure_jump)
+		  p2 += 6;
+		else
+		  break;
+	      }
+
+	    p1 = p + mcnt;
+	    /* p1[0] ... p1[2] are the `on_failure_jump' corresponding
+	       to the `maybe_finalize_jump' of this case.  Examine what 
+	       follows.  */
+
+            /* If we're at the end of the pattern, we can change.  */
+            if (p2 == pend)
+	      {
+		/* Consider what happens when matching ":\(.*\)"
+		   against ":/".  I don't really understand this code
+		   yet.  */
+  	        p[-3] = (unsigned char) pop_failure_jump;
+                DEBUG_PRINT1
+                  ("  End of pattern: change to `pop_failure_jump'.\n");
+              }
+
+            else if ((re_opcode_t) *p2 == exactn
+		     || (bufp->newline_anchor && (re_opcode_t) *p2 == endline))
+	      {
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+
+                if ((re_opcode_t) p1[3] == exactn && p1[5] != c)
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset
+			 || (re_opcode_t) p1[3] == charset_not)
+		  {
+		    int not = (re_opcode_t) p1[3] == charset_not;
+                    
+		    if (c < (unsigned char) (p1[4] * BYTEWIDTH)
+			&& p1[5 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+		      not = !not;
+
+                    /* `not' is equal to 1 if c would match, which means
+                        that we can't change to pop_failure_jump.  */
+		    if (!not)
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+            else if ((re_opcode_t) *p2 == charset)
+	      {
+#ifdef DEBUG
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+#endif
+
+                if ((re_opcode_t) p1[3] == exactn
+		    && ! ((int) p2[1] * BYTEWIDTH > (int) p1[4]
+			  && (p2[1 + p1[4] / BYTEWIDTH]
+			      & (1 << (p1[4] % BYTEWIDTH)))))
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset_not)
+		  {
+		    int idx;
+		    /* We win if the charset_not inside the loop
+		       lists every character listed in the charset after.  */
+		    for (idx = 0; idx < (int) p2[1]; idx++)
+		      if (! (p2[2 + idx] == 0
+			     || (idx < (int) p1[4]
+				 && ((p2[2 + idx] & ~ p1[5 + idx]) == 0))))
+			break;
+
+		    if (idx == p2[1])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+		else if ((re_opcode_t) p1[3] == charset)
+		  {
+		    int idx;
+		    /* We win if the charset inside the loop
+		       has no overlap with the one after the loop.  */
+		    for (idx = 0;
+			 idx < (int) p2[1] && idx < (int) p1[4];
+			 idx++)
+		      if ((p2[2 + idx] & p1[5 + idx]) != 0)
+			break;
+
+		    if (idx == p2[1] || idx == p1[4])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+	  }
+	  p -= 2;		/* Point at relative address again.  */
+	  if ((re_opcode_t) p[-1] != pop_failure_jump)
+	    {
+	      p[-1] = (unsigned char) jump;
+              DEBUG_PRINT1 ("  Match => jump.\n");
+	      goto unconditional_jump;
+	    }
+        /* Note fall through.  */
+
+
+	/* The end of a simple repeat has a pop_failure_jump back to
+           its matching on_failure_jump, where the latter will push a
+           failure point.  The pop_failure_jump takes off failure
+           points put on by this pop_failure_jump's matching
+           on_failure_jump; we got through the pattern to here from the
+           matching on_failure_jump, so didn't fail.  */
+        case pop_failure_jump:
+          {
+            /* We need to pass separate storage for the lowest and
+               highest registers, even though we don't care about the
+               actual values.  Otherwise, we will restore only one
+               register from the stack, since lowest will == highest in
+               `pop_failure_point'.  */
+            unsigned dummy_low_reg, dummy_high_reg;
+            unsigned char *pdummy;
+            const char *sdummy;
+
+            DEBUG_PRINT1 ("EXECUTING pop_failure_jump.\n");
+            POP_FAILURE_POINT (sdummy, pdummy,
+                               dummy_low_reg, dummy_high_reg,
+                               reg_dummy, reg_dummy, reg_info_dummy);
+          }
+          /* Note fall through.  */
+
+          
+        /* Unconditionally jump (without popping any failure points).  */
+        case jump:
+	unconditional_jump:
+	  EXTRACT_NUMBER_AND_INCR (mcnt, p);	/* Get the amount to jump.  */
+          DEBUG_PRINT2 ("EXECUTING jump %d ", mcnt);
+	  p += mcnt;				/* Do the jump.  */
+          DEBUG_PRINT2 ("(to 0x%x).\n", p);
+	  break;
+
+	
+        /* We need this opcode so we can detect where alternatives end
+           in `group_match_null_string_p' et al.  */
+        case jump_past_alt:
+          DEBUG_PRINT1 ("EXECUTING jump_past_alt.\n");
+          goto unconditional_jump;
+
+
+        /* Normally, the on_failure_jump pushes a failure point, which
+           then gets popped at pop_failure_jump.  We will end up at
+           pop_failure_jump, also, and with a pattern of, say, `a+', we
+           are skipping over the on_failure_jump, so we have to push
+           something meaningless for pop_failure_jump to pop.  */
+        case dummy_failure_jump:
+          DEBUG_PRINT1 ("EXECUTING dummy_failure_jump.\n");
+          /* It doesn't matter what we push for the string here.  What
+             the code at `fail' tests is the value for the pattern.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          goto unconditional_jump;
+
+
+        /* At the end of an alternative, we need to push a dummy failure
+           point in case we are followed by a `pop_failure_jump', because
+           we don't want the failure point for the alternative to be
+           popped.  For example, matching `(a|ab)*' against `aab'
+           requires that we match the `ab' alternative.  */
+        case push_dummy_failure:
+          DEBUG_PRINT1 ("EXECUTING push_dummy_failure.\n");
+          /* See comments just above at `dummy_failure_jump' about the
+             two zeroes.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          break;
+
+        /* Have to succeed matching what follows at least n times.
+           After that, handle like `on_failure_jump'.  */
+        case succeed_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING succeed_n %d.\n", mcnt);
+
+          assert (mcnt >= 0);
+          /* Originally, this is how many times we HAVE to succeed.  */
+          if (mcnt > 0)
+            {
+               mcnt--;
+	       p += 2;
+               STORE_NUMBER_AND_INCR (p, mcnt);
+               DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p, mcnt);
+            }
+	  else if (mcnt == 0)
+            {
+              DEBUG_PRINT2 ("  Setting two bytes from 0x%x to no_op.\n", p+2);
+	      p[2] = (unsigned char) no_op;
+              p[3] = (unsigned char) no_op;
+              goto on_failure;
+            }
+          break;
+        
+        case jump_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING jump_n %d.\n", mcnt);
+
+          /* Originally, this is how many times we CAN jump.  */
+          if (mcnt)
+            {
+               mcnt--;
+               STORE_NUMBER (p + 2, mcnt);
+	       goto unconditional_jump;	     
+            }
+          /* If don't have to jump any more, skip over the rest of command.  */
+	  else      
+	    p += 4;		     
+          break;
+        
+	case set_number_at:
+	  {
+            DEBUG_PRINT1 ("EXECUTING set_number_at.\n");
+
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            p1 = p + mcnt;
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p1, mcnt);
+	    STORE_NUMBER (p1, mcnt);
+            break;
+          }
+
+        case wordbound:
+          DEBUG_PRINT1 ("EXECUTING wordbound.\n");
+          if (AT_WORD_BOUNDARY (d))
+	    break;
+          goto fail;
+
+	case notwordbound:
+          DEBUG_PRINT1 ("EXECUTING notwordbound.\n");
+	  if (AT_WORD_BOUNDARY (d))
+	    goto fail;
+          break;
+
+	case wordbeg:
+          DEBUG_PRINT1 ("EXECUTING wordbeg.\n");
+	  if (WORDCHAR_P (d) && (AT_STRINGS_BEG (d) || !WORDCHAR_P (d - 1)))
+	    break;
+          goto fail;
+
+	case wordend:
+          DEBUG_PRINT1 ("EXECUTING wordend.\n");
+	  if (!AT_STRINGS_BEG (d) && WORDCHAR_P (d - 1)
+              && (!WORDCHAR_P (d) || AT_STRINGS_END (d)))
+	    break;
+          goto fail;
+
+#ifdef emacs
+  	case before_dot:
+          DEBUG_PRINT1 ("EXECUTING before_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) >= point)
+  	    goto fail;
+  	  break;
+  
+  	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) != point)
+  	    goto fail;
+  	  break;
+  
+  	case after_dot:
+          DEBUG_PRINT1 ("EXECUTING after_dot.\n");
+          if (PTR_CHAR_POS ((unsigned char *) d) <= point)
+  	    goto fail;
+  	  break;
+#if 0 /* not emacs19 */
+	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+	  if (PTR_CHAR_POS ((unsigned char *) d) + 1 != point)
+	    goto fail;
+	  break;
+#endif /* not emacs19 */
+
+	case syntaxspec:
+          DEBUG_PRINT2 ("EXECUTING syntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchsyntax;
+
+        case wordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs wordchar.\n");
+	  mcnt = (int) Sword;
+        matchsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) != (enum syntaxcode) mcnt)
+	    goto fail;
+          SET_REGS_MATCHED ();
+	  break;
+
+	case notsyntaxspec:
+          DEBUG_PRINT2 ("EXECUTING notsyntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchnotsyntax;
+
+        case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs notwordchar.\n");
+	  mcnt = (int) Sword;
+        matchnotsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) == (enum syntaxcode) mcnt)
+	    goto fail;
+	  SET_REGS_MATCHED ();
+          break;
+
+#else /* not emacs */
+	case wordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs wordchar.\n");
+	  PREFETCH ();
+          if (!WORDCHAR_P (d))
+            goto fail;
+	  SET_REGS_MATCHED ();
+          d++;
+	  break;
+	  
+	case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs notwordchar.\n");
+	  PREFETCH ();
+	  if (WORDCHAR_P (d))
+            goto fail;
+          SET_REGS_MATCHED ();
+          d++;
+	  break;
+#endif /* not emacs */
+          
+        default:
+          abort ();
+	}
+      continue;  /* Successfully executed one pattern command; keep going.  */
+
+
+    /* We goto here if a matching operation fails. */
+    fail:
+      if (!FAIL_STACK_EMPTY ())
+	{ /* A restart point is known.  Restore to that state.  */
+          DEBUG_PRINT1 ("\nFAIL:\n");
+          POP_FAILURE_POINT (d, p,
+                             lowest_active_reg, highest_active_reg,
+                             regstart, regend, reg_info);
+
+          /* If this failure point is a dummy, try the next one.  */
+          if (!p)
+	    goto fail;
+
+          /* If we failed to the end of the pattern, don't examine *p.  */
+	  assert (p <= pend);
+          if (p < pend)
+            {
+              boolean is_a_jump_n = false;
+              
+              /* If failed to a backwards jump that's part of a repetition
+                 loop, need to pop this failure point and use the next one.  */
+              switch ((re_opcode_t) *p)
+                {
+                case jump_n:
+                  is_a_jump_n = true;
+                case maybe_pop_jump:
+                case pop_failure_jump:
+                case jump:
+                  p1 = p + 1;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  p1 += mcnt;	
+
+                  if ((is_a_jump_n && (re_opcode_t) *p1 == succeed_n)
+                      || (!is_a_jump_n
+                          && (re_opcode_t) *p1 == on_failure_jump))
+                    goto fail;
+                  break;
+                default:
+                  /* do nothing */ ;
+                }
+            }
+
+          if (d >= string1 && d <= end1)
+	    dend = end_match_1;
+        }
+      else
+        break;   /* Matching at this starting point really fails.  */
+    } /* for (;;) */
+
+  if (best_regs_set)
+    goto restore_best_regs;
+
+  FREE_VARIABLES ();
+
+  return -1;         			/* Failure to match.  */
+} /* re_match_2 */
+
+
+/* Subroutine definitions for re_match_2.  */
+
+
+/* We are passed P pointing to a register number after a start_memory.
+   
+   Return true if the pattern up to the corresponding stop_memory can
+   match the empty string, and false otherwise.
+   
+   If we find the matching stop_memory, sets P to point to one past its number.
+   Otherwise, sets P to an undefined byte less than or equal to END.
+
+   We don't handle duplicates properly (yet).  */
+
+static boolean
+group_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  /* Point to after the args to the start_memory.  */
+  unsigned char *p1 = *p + 2;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and return true or
+	 false, as appropriate, when we get to one that can't, or to the
+         matching stop_memory.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+        /* Could be either a loop or a series of alternatives.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          
+          /* If the next operation is not a jump backwards in the
+	     pattern.  */
+
+	  if (mcnt >= 0)
+	    {
+              /* Go through the on_failure_jumps of the alternatives,
+                 seeing if any of the alternatives cannot match nothing.
+                 The last alternative starts with only a jump,
+                 whereas the rest start with on_failure_jump and end
+                 with a jump, e.g., here is the pattern for `a|b|c':
+
+                 /on_failure_jump/0/6/exactn/1/a/jump_past_alt/0/6
+                 /on_failure_jump/0/6/exactn/1/b/jump_past_alt/0/3
+                 /exactn/1/c						
+
+                 So, we have to first go through the first (n-1)
+                 alternatives and then deal with the last one separately.  */
+
+
+              /* Deal with the first (n-1) alternatives, which start
+                 with an on_failure_jump (see above) that jumps to right
+                 past a jump_past_alt.  */
+
+              while ((re_opcode_t) p1[mcnt-3] == jump_past_alt)
+                {
+                  /* `mcnt' holds how many bytes long the alternative
+                     is, including the ending `jump_past_alt' and
+                     its number.  */
+
+                  if (!alt_match_null_string_p (p1, p1 + mcnt - 3, 
+				                      reg_info))
+                    return false;
+
+                  /* Move to right after this alternative, including the
+		     jump_past_alt.  */
+                  p1 += mcnt;	
+
+                  /* Break if it's the beginning of an n-th alternative
+                     that doesn't begin with an on_failure_jump.  */
+                  if ((re_opcode_t) *p1 != on_failure_jump)
+                    break;
+		
+		  /* Still have to check that it's not an n-th
+		     alternative that starts with an on_failure_jump.  */
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  if ((re_opcode_t) p1[mcnt-3] != jump_past_alt)
+                    {
+		      /* Get to the beginning of the n-th alternative.  */
+                      p1 -= 3;
+                      break;
+                    }
+                }
+
+              /* Deal with the last alternative: go back and get number
+                 of the `jump_past_alt' just before it.  `mcnt' contains
+                 the length of the alternative.  */
+              EXTRACT_NUMBER (mcnt, p1 - 2);
+
+              if (!alt_match_null_string_p (p1, p1 + mcnt, reg_info))
+                return false;
+
+              p1 += mcnt;	/* Get past the n-th alternative.  */
+            } /* if mcnt > 0 */
+          break;
+
+          
+        case stop_memory:
+	  assert (p1[1] == **p);
+          *p = p1 + 2;
+          return true;
+
+        
+        default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    } /* while p1 < end */
+
+  return false;
+} /* group_match_null_string_p */
+
+
+/* Similar to group_match_null_string_p, but doesn't deal with alternatives:
+   It expects P to be the first byte of a single alternative and END one
+   byte past the last. The alternative can contain groups.  */
+   
+static boolean
+alt_match_null_string_p (p, end, reg_info)
+    unsigned char *p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  unsigned char *p1 = p;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and break when we get 
+         to one that can't.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+	/* It's a loop.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+          break;
+          
+	default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    }  /* while p1 < end */
+
+  return true;
+} /* alt_match_null_string_p */
+
+
+/* Deals with the ops common to group_match_null_string_p and
+   alt_match_null_string_p.  
+   
+   Sets P to one after the op and its arguments, if any.  */
+
+static boolean
+common_op_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  boolean ret;
+  int reg_no;
+  unsigned char *p1 = *p;
+
+  switch ((re_opcode_t) *p1++)
+    {
+    case no_op:
+    case begline:
+    case endline:
+    case begbuf:
+    case endbuf:
+    case wordbeg:
+    case wordend:
+    case wordbound:
+    case notwordbound:
+#ifdef emacs
+    case before_dot:
+    case at_dot:
+    case after_dot:
+#endif
+      break;
+
+    case start_memory:
+      reg_no = *p1;
+      assert (reg_no > 0 && reg_no <= MAX_REGNUM);
+      ret = group_match_null_string_p (&p1, end, reg_info);
+      
+      /* Have to set this here in case we're checking a group which
+         contains a group and a back reference to it.  */
+
+      if (REG_MATCH_NULL_STRING_P (reg_info[reg_no]) == MATCH_NULL_UNSET_VALUE)
+        REG_MATCH_NULL_STRING_P (reg_info[reg_no]) = ret;
+
+      if (!ret)
+        return false;
+      break;
+          
+    /* If this is an optimized succeed_n for zero times, make the jump.  */
+    case jump:
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+      if (mcnt >= 0)
+        p1 += mcnt;
+      else
+        return false;
+      break;
+
+    case succeed_n:
+      /* Get to the number of times to succeed.  */
+      p1 += 2;		
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+
+      if (mcnt == 0)
+        {
+          p1 -= 4;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+        }
+      else
+        return false;
+      break;
+
+    case duplicate: 
+      if (!REG_MATCH_NULL_STRING_P (reg_info[*p1]))
+        return false;
+      break;
+
+    case set_number_at:
+      p1 += 4;
+
+    default:
+      /* All other opcodes mean we cannot match the empty string.  */
+      return false;
+  }
+
+  *p = p1;
+  return true;
+} /* common_op_match_null_string_p */
+
+
+/* Return zero if TRANSLATE[S1] and TRANSLATE[S2] are identical for LEN
+   bytes; nonzero otherwise.  */
+   
+static int
+bcmp_translate (s1, s2, len, translate)
+     unsigned char *s1, *s2;
+     register int len;
+     char *translate;
+{
+  register unsigned char *p1 = s1, *p2 = s2;
+  while (len)
+    {
+      if (translate[*p1++] != translate[*p2++]) return 1;
+      len--;
+    }
+  return 0;
+}
+
+
+/* Entry points for GNU code.  */
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in BUFP.
+   Returns 0 if the pattern was valid, otherwise an error string.
+   
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in BUFP on entry.
+   
+   We call regex_compile to do the actual compilation.  */
+
+const char *
+re_compile_pattern (pattern, length, bufp)
+     const char *pattern;
+     int length;
+     struct re_pattern_buffer *bufp;
+{
+  reg_errcode_t ret;
+  
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  bufp->regs_allocated = REGS_UNALLOCATED;
+  
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  bufp->no_sub = 0;
+  
+  /* Match anchors at newline.  */
+  bufp->newline_anchor = 1;
+  
+  ret = regex_compile (pattern, length, re_syntax_options, bufp);
+
+  if (!ret)
+    return NULL;
+  return gettext (re_error_msgid[(int) ret]);
+}     
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them unless specifically requested.  */
+
+#ifdef _REGEX_RE_COMP
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer re_comp_buf;
+
+char *
+re_comp (s)
+    const char *s;
+{
+  reg_errcode_t ret;
+  
+  if (!s)
+    {
+      if (!re_comp_buf.buffer)
+	return gettext ("No previous regular expression");
+      return 0;
+    }
+
+  if (!re_comp_buf.buffer)
+    {
+      re_comp_buf.buffer = (unsigned char *) malloc (200);
+      if (re_comp_buf.buffer == NULL)
+        return gettext (re_error_msgid[(int) REG_ESPACE]);
+      re_comp_buf.allocated = 200;
+
+      re_comp_buf.fastmap = (char *) malloc (1 << BYTEWIDTH);
+      if (re_comp_buf.fastmap == NULL)
+	return gettext (re_error_msgid[(int) REG_ESPACE]);
+    }
+
+  /* Since `re_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  re_comp_buf.newline_anchor = 1;
+
+  ret = regex_compile (s, strlen (s), re_syntax_options, &re_comp_buf);
+  
+  if (!ret)
+    return NULL;
+
+  /* Yes, we're discarding `const' here if !HAVE_LIBINTL.  */
+  return (char *) gettext (re_error_msgid[(int) ret]);
+}
+
+
+int
+re_exec (s)
+    const char *s;
+{
+  const int len = strlen (s);
+  return
+    0 <= re_search (&re_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* _REGEX_RE_COMP */
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#ifndef emacs
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+int
+regcomp (preg, pattern, cflags)
+    regex_t *preg;
+    const char *pattern; 
+    int cflags;
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = (cflags & REG_EXTENDED) ?
+      RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->used = 0;
+  
+  /* Don't bother to use a fastmap when searching.  This simplifies the
+     REG_NEWLINE case: if we used a fastmap, we'd have to put all the
+     characters after newlines into the fastmap.  This way, we just try
+     every character.  */
+  preg->fastmap = 0;
+  
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+      
+      preg->translate = (char *) malloc (CHAR_SET_SIZE);
+      if (preg->translate == NULL)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = ISUPPER (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = NULL;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we 
+     can use strlen here in compiling the pattern.  */
+  ret = regex_compile (pattern, strlen (pattern), syntax, preg);
+  
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+  
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+   
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+   
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+   
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    const regex_t *preg;
+    const char *string; 
+    size_t nmatch; 
+    regmatch_t pmatch[]; 
+    int eflags;
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+  
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+  
+  /* The user has told us exactly how many registers to return
+     information about, via `nmatch'.  We have to pass that on to the
+     matching routines.  */
+  private_preg.regs_allocated = REGS_FIXED;
+  
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start = TALLOC (nmatch, regoff_t);
+      regs.end = TALLOC (nmatch, regoff_t);
+      if (regs.start == NULL || regs.end == NULL)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg, string, len,
+                   /* start: */ 0, /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+  
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   We don't use PREG here.  */
+
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    const regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+{
+  const char *msg;
+  size_t msg_size;
+
+  if (errcode < 0
+      || errcode >= (sizeof (re_error_msgid) / sizeof (re_error_msgid[0])))
+    /* Only error codes returned by the rest of the code should be passed 
+       to this routine.  If we are given anything else, or if other regex
+       code generates an invalid error code, then the program has a bug.
+       Dump core so we can fix it.  */
+    abort ();
+
+  msg = gettext (re_error_msgid[errcode]);
+
+  msg_size = strlen (msg) + 1; /* Includes the null.  */
+  
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+void
+regfree (preg)
+    regex_t *preg;
+{
+  if (preg->buffer != NULL)
+    free (preg->buffer);
+  preg->buffer = NULL;
+  
+  preg->allocated = 0;
+  preg->used = 0;
+
+  if (preg->fastmap != NULL)
+    free (preg->fastmap);
+  preg->fastmap = NULL;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != NULL)
+    free (preg->translate);
+  preg->translate = NULL;
+}
+
+#endif /* not emacs  */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/indexers/mg/lib/regex.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/regex.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/regex.h	(revision 21112)
@@ -0,0 +1,497 @@
+/* Definitions for data structures and routines for the regular
+   expression library, version 0.12.
+
+   Copyright (C) 1985, 89, 90, 91, 92, 93, 95 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef __REGEXP_LIBRARY_H__
+#define __REGEXP_LIBRARY_H__
+
+/* POSIX says that <sys/types.h> must be included (by the caller) before
+   <regex.h>.  */
+
+#if !defined (_POSIX_C_SOURCE) && !defined (_POSIX_SOURCE) && defined (VMS)
+/* VMS doesn't have `size_t' in <sys/types.h>, even though POSIX says it
+   should be there.  */
+#include <stddef.h>
+#endif
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+typedef unsigned reg_syntax_t;
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* If this bit is set, succeed as soon as we match the whole pattern,
+   without further backtracking.  */
+#define RE_NO_POSIX_BACKTRACKING (RE_UNMATCHED_RIGHT_PAREN_ORD << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+/* P1003.2/D11.2, section 4.20.7.1, lines 5078ff.  */
+#define RE_SYNTAX_ED RE_SYNTAX_POSIX_BASIC
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+
+/* This data structure represents a compiled pattern.  Before calling
+   the pattern compiler, the fields `buffer', `allocated', `fastmap',
+   `translate', and `no_sub' can be set.  After the pattern has been
+   compiled, the `re_nsub' field is available.  All other fields are
+   private to the regex routines.  */
+
+struct re_pattern_buffer
+{
+/* [[[begin pattern_buffer]]] */
+	/* Space that holds the compiled pattern.  It is declared as
+          `unsigned char *' because its elements are
+           sometimes used as array indexes.  */
+  unsigned char *buffer;
+
+	/* Number of bytes to which `buffer' points.  */
+  unsigned long allocated;
+
+	/* Number of bytes actually used in `buffer'.  */
+  unsigned long used;	
+
+        /* Syntax setting with which the pattern was compiled.  */
+  reg_syntax_t syntax;
+
+        /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+           the fastmap, if there is one, to skip over impossible
+           starting points for matches.  */
+  char *fastmap;
+
+        /* Either a translate table to apply to all characters before
+           comparing them, or zero for no translation.  The translation
+           is applied to a pattern when it is compiled and to a string
+           when it is matched.  */
+  char *translate;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+        /* Zero if this pattern cannot match the empty string, one else.
+           Well, in truth it's used only in `re_search_2', to see
+           whether or not we should use the fastmap, so we don't set
+           this absolutely perfectly; see `re_compile_fastmap' (the
+           `duplicate' case).  */
+  unsigned can_be_null : 1;
+
+        /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+             for `max (RE_NREGS, re_nsub + 1)' groups.
+           If REGS_REALLOCATE, reallocate space if necessary.
+           If REGS_FIXED, use what's there.  */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned regs_allocated : 2;
+
+        /* Set to zero when `regex_compile' compiles a pattern; set to one
+           by `re_compile_fastmap' if it updates the fastmap.  */
+  unsigned fastmap_accurate : 1;
+
+        /* If set, `re_match_2' does not return information about
+           subexpressions.  */
+  unsigned no_sub : 1;
+
+        /* If set, a beginning-of-line anchor doesn't match at the
+           beginning of the string.  */ 
+  unsigned not_bol : 1;
+
+        /* Similarly for an end-of-line anchor.  */
+  unsigned not_eol : 1;
+
+        /* If true, an anchor at a newline matches.  */
+  unsigned newline_anchor : 1;
+
+/* [[[end pattern_buffer]]] */
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+   `re_match_2' returns information about at least this many registers
+   the first time a `regs' structure is passed.  */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+/* Declarations for routines.  */
+
+/* To avoid duplicating every routine declaration -- once with a
+   prototype (if we are ANSI), and once without (if we aren't) -- we
+   use the following macro to declare argument types.  This
+   unfortunately clutters up the declarations a bit, but I think it's
+   worth it.  */
+
+#if __STDC__
+
+#define _RE_ARGS(args) args
+
+#else /* not __STDC__ */
+
+#define _RE_ARGS(args) ()
+
+#endif /* not __STDC__ */
+
+/* Sets the current default syntax to SYNTAX, and return the old syntax.
+   You can also simply assign to the `re_syntax_options' variable.  */
+extern reg_syntax_t re_set_syntax _RE_ARGS ((reg_syntax_t syntax));
+
+/* Compile the regular expression PATTERN, with length LENGTH
+   and syntax given by the global `re_syntax_options', into the buffer
+   BUFFER.  Return NULL if successful, and an error string if not.  */
+extern const char *re_compile_pattern
+  _RE_ARGS ((const char *pattern, int length,
+             struct re_pattern_buffer *buffer));
+
+
+/* Compile a fastmap for the compiled pattern in BUFFER; used to
+   accelerate searches.  Return 0 if successful and -2 if was an
+   internal error.  */
+extern int re_compile_fastmap _RE_ARGS ((struct re_pattern_buffer *buffer));
+
+
+/* Search in the string STRING (with length LENGTH) for the pattern
+   compiled into BUFFER.  Start searching at position START, for RANGE
+   characters.  Return the starting position of the match, -1 for no
+   match, or -2 for an internal error.  Also return register
+   information in REGS (if REGS and BUFFER->no_sub are nonzero).  */
+extern int re_search
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+            int length, int start, int range, struct re_registers *regs));
+
+
+/* Like `re_search', but search in the concatenation of STRING1 and
+   STRING2.  Also, stop searching at index START + STOP.  */
+extern int re_search_2
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, int range, struct re_registers *regs, int stop));
+
+
+/* Like `re_search', but return how many characters in STRING the regexp
+   in BUFFER matched, starting at position START.  */
+extern int re_match
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+             int length, int start, struct re_registers *regs));
+
+
+/* Relates to `re_match' as `re_search_2' relates to `re_search'.  */
+extern int re_match_2 
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, struct re_registers *regs, int stop));
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using BUFFER and REGS will use this memory
+   for recording register information.  STARTS and ENDS must be
+   allocated with malloc, and must each be at least `NUM_REGS * sizeof
+   (regoff_t)' bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+extern void re_set_registers
+  _RE_ARGS ((struct re_pattern_buffer *buffer, struct re_registers *regs,
+             unsigned num_regs, regoff_t *starts, regoff_t *ends));
+
+#ifdef _REGEX_RE_COMP
+/* 4.2 bsd compatibility.  */
+extern char *re_comp _RE_ARGS ((const char *));
+extern int re_exec _RE_ARGS ((const char *));
+#endif
+
+/* POSIX compatibility.  */
+extern int regcomp _RE_ARGS ((regex_t *preg, const char *pattern, int cflags));
+extern int regexec
+  _RE_ARGS ((const regex_t *preg, const char *string, size_t nmatch,
+             regmatch_t pmatch[], int eflags));
+extern size_t regerror
+  _RE_ARGS ((int errcode, const regex_t *preg, char *errbuf,
+             size_t errbuf_size));
+extern void regfree _RE_ARGS ((regex_t *preg));
+
+#endif /* not __REGEXP_LIBRARY_H__ */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /main/tags/3.01/indexers/mg/lib/rx.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/rx.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/rx.c	(revision 21112)
@@ -0,0 +1,7271 @@
+/*	Copyright (C) 1992, 1993, 1994, 1995 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+
+/* NOTE!!!  AIX is so losing it requires this to be the first thing in the 
+ * file. 
+ * Do not put ANYTHING before it!  
+ */
+#if !defined (__GNUC__) && defined (_AIX)
+ #pragma alloca
+#endif
+
+/* To make linux happy? */
+#ifndef	_GNU_SOURCE
+#define	_GNU_SOURCE
+#endif
+
+#if HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+#  include <malloc.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+
+
+char rx_version_string[] = "GNU Rx version 0.07.1";
+
+			/* ``Too hard!''
+			 *	    -- anon.
+			 */
+
+
+
+#include <stdio.h>
+#include <ctype.h>
+#ifndef isgraph
+#define isgraph(c) (isprint (c) && !isspace (c))
+#endif
+#ifndef isblank
+#define isblank(c) ((c) == ' ' || (c) == '\t')
+#endif
+
+#include <sys/types.h>
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+#ifndef __GCC__
+#undef __inline__
+#define __inline__
+#endif
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+/* Memory management and stuff for emacs. */
+
+#define CHARBITS 8
+#define remalloc(M, S) (M ? realloc (M, S) : malloc (S))
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+ * use `alloca' instead of `malloc' for the backtracking stack.
+ *
+ * Emacs will die miserably if we don't do this.
+ */
+
+#ifdef REGEX_MALLOC
+#define REGEX_ALLOCATE malloc
+#else /* not REGEX_MALLOC  */
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#define REGEX_ALLOCATE _alloca
+#else
+#define REGEX_ALLOCATE alloca
+#endif /* __WIN32__ */
+#endif /* not REGEX_MALLOC */
+
+
+#ifdef RX_WANT_RX_DEFS
+#define RX_DECL extern
+#define RX_DEF_QUAL 
+#else
+#define RX_WANT_RX_DEFS
+#define RX_DECL static
+#define RX_DEF_QUAL static
+#endif
+#include "rx.h"
+#undef RX_DECL
+#define RX_DECL RX_DEF_QUAL
+
+
+#ifndef emacs
+
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else /* not SYNTAX_TABLE */
+
+#ifdef __STDC__
+static void
+init_syntax_once (void)
+#else
+static void
+init_syntax_once ()
+#endif
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+#endif /* not SYNTAX_TABLE */
+#endif /* not emacs */
+
+
+/* Compile with `-DRX_DEBUG' and use the following flags.
+ *
+ * Debugging flags:
+ *   	rx_debug - print information as a regexp is compiled
+ * 	rx_debug_trace - print information as a regexp is executed
+ */
+
+#ifdef RX_DEBUG
+
+int rx_debug_compile = 0;
+int rx_debug_trace = 0;
+static struct re_pattern_buffer * dbug_rxb = 0;
+
+#ifdef __STDC__
+typedef void (*side_effect_printer) (struct rx *, void *, FILE *);
+#else
+typedef void (*side_effect_printer) ();
+#endif
+
+#ifdef __STDC__
+static void print_cset (struct rx *rx, rx_Bitset cset, FILE * fp);
+#else
+static void print_cset ();
+#endif
+
+#ifdef __STDC__
+static void
+print_rexp (struct rx *rx,
+	    struct rexp_node *node, int depth,
+	    side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_rexp (rx, node, depth, seprint, fp)
+     struct rx *rx;
+     struct rexp_node *node;
+     int depth;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  if (!node)
+    return;
+  else
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  {
+	    fprintf (fp, "%*s", depth, "");
+	    print_cset (rx, node->params.cset, fp);
+	    fputc ('\n', fp);
+	    break;
+	  }
+
+ 	case r_opt:
+	case r_star:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_opt ? "opt" : "star");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+	case r_2phase_star:
+	  fprintf (fp, "%*s2phase star\n", depth, "");
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+
+	case r_alternate:
+	case r_concat:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_alternate ? "alt" : "concat");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  break;
+	case r_side_effect:
+	  fprintf (fp, "%*sSide effect: ", depth, "");
+	  seprint (rx, node->params.side_effect, fp);
+	  fputc ('\n', fp);
+	}
+    }
+}
+
+#ifdef __STDC__
+static void
+print_nfa (struct rx * rx,
+	   struct rx_nfa_state * n,
+	   side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_nfa (rx, n, seprint, fp)
+     struct rx * rx;
+     struct rx_nfa_state * n;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      fprintf (fp, "node %d %s\n", n->id,
+	       n->is_final ? "final" : (n->is_start ? "start" : ""));
+      while (e)
+	{
+	  fprintf (fp, "   edge to %d, ", e->dest->id);
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	      fprintf (fp, "epsilon\n");
+	      break;
+	    case ne_side_effect:
+	      fprintf (fp, "side effect ");
+	      seprint (rx, e->params.side_effect, fp);
+	      fputc ('\n', fp);
+	      break;
+	    case ne_cset:
+	      fprintf (fp, "cset ");
+	      print_cset (rx, e->params.cset, fp);
+	      fputc ('\n', fp);
+	      break;
+	    }
+	  e = e->next;
+	}
+
+      while (ec)
+	{
+	  int x;
+	  struct rx_nfa_state_set * s;
+	  struct rx_se_list * l;
+	  fprintf (fp, "   eclosure to {");
+	  for (s = ec->destset; s; s = s->cdr)
+	    fprintf (fp, "%d ", s->car->id);
+	  fprintf (fp, "} (");
+	  for (l = ec->effects; l; l = l->cdr)
+	    {
+	      seprint (rx, l->car, fp);
+	      fputc (' ', fp);
+	    }
+	  fprintf (fp, ")\n");
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+}
+
+static char * efnames [] =
+{
+  "bogon",
+  "re_se_try",
+  "re_se_pushback",
+  "re_se_push0",
+  "re_se_pushpos",
+  "re_se_chkpos",
+  "re_se_poppos",
+  "re_se_at_dot",
+  "re_se_syntax",
+  "re_se_not_syntax",
+  "re_se_begbuf",
+  "re_se_hat",
+  "re_se_wordbeg",
+  "re_se_wordbound",
+  "re_se_notwordbound",
+  "re_se_wordend",
+  "re_se_endbuf",
+  "re_se_dollar",
+  "re_se_fail",
+};
+
+static char * efnames2[] =
+{
+  "re_se_win",
+  "re_se_lparen",
+  "re_se_rparen",
+  "re_se_backref",
+  "re_se_iter",
+  "re_se_end_iter",
+  "re_se_tv"
+};
+
+static char * inx_names[] = 
+{
+  "rx_backtrack_point",
+  "rx_do_side_effects",
+  "rx_cache_miss",
+  "rx_next_char",
+  "rx_backtrack",
+  "rx_error_inx",
+  "rx_num_instructions"
+};
+
+
+#ifdef __STDC__
+static void
+re_seprint (struct rx * rx, void * effect, FILE * fp)
+#else
+static void
+re_seprint (rx, effect, fp)
+     struct rx * rx;
+     void * effect;
+     FILE * fp;
+#endif
+{
+  if ((int)effect < 0)
+    fputs (efnames[-(int)effect], fp);
+  else if (dbug_rxb)
+    {
+      struct re_se_params * p = &dbug_rxb->se_params[(int)effect];
+      fprintf (fp, "%s(%d,%d)", efnames2[p->se], p->op1, p->op2);
+    }
+  else
+    fprintf (fp, "[complex op # %d]", (int)effect);
+}
+
+
+/* These are so the regex.c regression tests will compile. */
+void
+print_compiled_pattern (rxb)
+     struct re_pattern_buffer * rxb;
+{
+}
+
+void
+print_fastmap (fm)
+     char * fm;
+{
+}
+
+#endif /* RX_DEBUG */
+
+
+
+
+/* This page: Bitsets.  Completely unintersting. */
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_equal (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  RX_subset s = b[0];
+  b[0] = ~a[0];
+
+  for (x = rx_bitset_numb_subsets(size) - 1; a[x] == b[x]; --x)
+    ;
+
+  b[0] = s;
+  return !x && s == a[0];
+}
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_subset (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets(size) - 1;
+  while (x-- && (a[x] & b[x]) == a[x]);
+  return x == -1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_empty (int size, rx_Bitset set)
+#else
+RX_DECL int
+rx_bitset_empty (size, set)
+     int size;
+     rx_Bitset set;
+#endif
+{
+  int x;
+  RX_subset s = set[0];
+  set[0] = 1;
+  for (x = rx_bitset_numb_subsets(size) - 1; !set[x]; --x)
+    ;
+  set[0] = s;
+  return !s;
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_null (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_null (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  bzero (b, rx_sizeof_bitset(size));
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_universe (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_universe (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    *b++ = ~(RX_subset)0;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_complement (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_complement (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    {
+      *b = ~*b;
+      ++b;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_assign (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_union (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_union (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] |= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_intersection (int size,
+			rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_intersection (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_difference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &=  ~ b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_revdifference (int size,
+			 rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_revdifference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = ~a[x] & b[x];
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_xor (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] ^= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL unsigned long
+rx_bitset_hash (int size, rx_Bitset b)
+#else
+RX_DECL unsigned long
+rx_bitset_hash (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  unsigned long hash = (unsigned long)rx_bitset_hash;
+
+  for (x = rx_bitset_numb_subsets(size) - 1; x >= 0; --x)
+    hash ^= rx_bitset_subset_val(b, x);
+
+  return hash;
+}
+
+
+RX_DECL RX_subset rx_subset_singletons [RX_subset_bits] = 
+{
+  0x1,
+  0x2,
+  0x4,
+  0x8,
+  0x10,
+  0x20,
+  0x40,
+  0x80,
+  0x100,
+  0x200,
+  0x400,
+  0x800,
+  0x1000,
+  0x2000,
+  0x4000,
+  0x8000,
+  0x10000,
+  0x20000,
+  0x40000,
+  0x80000,
+  0x100000,
+  0x200000,
+  0x400000,
+  0x800000,
+  0x1000000,
+  0x2000000,
+  0x4000000,
+  0x8000000,
+  0x10000000,
+  0x20000000,
+  0x40000000,
+  0x80000000
+};
+
+#ifdef RX_DEBUG
+
+#ifdef __STDC__
+static void
+print_cset (struct rx *rx, rx_Bitset cset, FILE * fp)
+#else
+static void
+print_cset (rx, cset, fp)
+     struct rx *rx;
+     rx_Bitset cset;
+     FILE * fp;
+#endif
+{
+  int x;
+  fputc ('[', fp);
+  for (x = 0; x < rx->local_cset_size; ++x)
+    if (RX_bitset_member (cset, x))
+      {
+	if (isprint(x))
+	  fputc (x, fp);
+	else
+	  fprintf (fp, "\\0%o ", x);
+      }
+  fputc (']', fp);
+}
+
+#endif /*  RX_DEBUG */
+
+
+
+
+static unsigned long rx_hash_masks[4] =
+{
+  0x12488421,
+  0x96699669,
+  0xbe7dd7eb,
+  0xffffffff
+};
+
+
+/* Hash tables */
+#ifdef __STDC__
+RX_DECL struct rx_hash_item * 
+rx_hash_find (struct rx_hash * table,
+	      unsigned long hash,
+	      void * value,
+	      struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item * 
+rx_hash_find (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks [0];
+  int bucket = (hash & mask) % 13;
+
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+    }
+
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+
+  return 0;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_hash_item *
+rx_hash_store (struct rx_hash * table,
+	       unsigned long hash,
+	       void * value,
+	       struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item *
+rx_hash_store (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks[0];
+  int bucket = (hash & mask) % 13;
+  int depth = 0;
+  
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+      ++depth;
+    }
+  
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+  
+  {
+    if (   (depth < 3)
+	&& (table->bucket_size [bucket] >= 4))
+      {
+	struct rx_hash * newtab = ((struct rx_hash *)
+				   rules->hash_alloc (rules));
+	if (!newtab)
+	  goto add_to_bucket;
+	bzero (newtab, sizeof (*newtab));
+	newtab->parent = table;
+	{
+	  struct rx_hash_item * them = table->buckets[bucket];
+	  unsigned long newmask = rx_hash_masks[maskc + 1];
+	  while (them)
+	    {
+	      struct rx_hash_item * save = them->next_same_hash;
+	      int new_buck = (them->hash & newmask) % 13;
+	      them->next_same_hash = newtab->buckets[new_buck];
+	      newtab->buckets[new_buck] = them;
+	      them->table = newtab;
+	      them = save;
+	      ++newtab->bucket_size[new_buck];
+	      ++newtab->refs;
+	    }
+	  table->refs = (table->refs - table->bucket_size[bucket] + 1);
+	  table->bucket_size[bucket] = 0;
+	  table->buckets[bucket] = 0;
+	  table->children[bucket] = newtab;
+	  table = newtab;
+	  bucket = (hash & newmask) % 13;
+	}
+      }
+  }
+ add_to_bucket:
+  {
+    struct rx_hash_item  * it = ((struct rx_hash_item *)
+				 rules->hash_item_alloc (rules, value));
+    if (!it)
+      return 0;
+    it->hash = hash;
+    it->table = table;
+    /* DATA and BINDING are to be set in hash_item_alloc */
+    it->next_same_hash = table->buckets [bucket];
+    table->buckets[bucket] = it;
+    ++table->bucket_size [bucket];
+    ++table->refs;
+    return it;
+  }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_hash_free (it, rules)
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+  if (it)
+    {
+      struct rx_hash * table = it->table;
+      unsigned long hash = it->hash;
+      int depth = (table->parent
+		   ? (table->parent->parent
+		      ? (table->parent->parent->parent
+			 ? 3
+			 : 2)
+		      : 1)
+		   : 0);
+      int bucket = (hash & rx_hash_masks [depth]) % 13;
+      struct rx_hash_item ** pos = &table->buckets [bucket];
+      
+      while (*pos != it)
+	pos = &(*pos)->next_same_hash;
+      *pos = it->next_same_hash;
+      rules->free_hash_item (it, rules);
+      --table->bucket_size[bucket];
+      --table->refs;
+      while (!table->refs && depth)
+	{
+	  struct rx_hash * save = table;
+	  table = table->parent;
+	  --depth;
+	  bucket = (hash & rx_hash_masks [depth]) % 13;
+	  --table->refs;
+	  table->children[bucket] = 0;
+	  rules->free_hash (save, rules);
+	}
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+		    struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_free_hash_table (tab, freefn, rules)
+     struct rx_hash * tab;
+     rx_hash_freefn freefn;
+     struct rx_hash_rules * rules;
+#endif
+{
+  int x;
+
+  for (x = 0; x < 13; ++x)
+    if (tab->children[x])
+      {
+	rx_free_hash_table (tab->children[x], freefn, rules);
+	rules->free_hash (tab->children[x], rules);
+      }
+    else
+      {
+	struct rx_hash_item * them = tab->buckets[x];
+	while (them)
+	  {
+	    struct rx_hash_item * that = them;
+	    them = that->next_same_hash;
+	    freefn (that);
+	    rules->free_hash_item (that, rules);
+	  }
+      }
+}
+
+
+
+
+/* Utilities for manipulating bitset represntations of characters sets. */
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_cset (struct rx *rx)
+#else
+RX_DECL rx_Bitset
+rx_cset (rx)
+     struct rx *rx;
+#endif
+{
+  rx_Bitset b = (rx_Bitset) malloc (rx_sizeof_bitset (rx->local_cset_size));
+  if (b)
+    rx_bitset_null (rx->local_cset_size, b);
+  return b;
+}
+
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_copy_cset (struct rx *rx, rx_Bitset a)
+#else
+RX_DECL rx_Bitset
+rx_copy_cset (rx, a)
+     struct rx *rx;
+     rx_Bitset a;
+#endif
+{
+  rx_Bitset cs = rx_cset (rx);
+
+  if (cs)
+    rx_bitset_union (rx->local_cset_size, cs, a);
+
+  return cs;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_cset (struct rx * rx, rx_Bitset c)
+#else
+RX_DECL void
+rx_free_cset (rx, c)
+     struct rx * rx;
+     rx_Bitset c;
+#endif
+{
+  if (c)
+    free ((char *)c);
+}
+
+
+
+/* Hash table memory allocation policy for the regexp compiler */
+
+#ifdef __STDC__
+static struct rx_hash *
+compiler_hash_alloc (struct rx_hash_rules * rules)
+#else
+static struct rx_hash *
+compiler_hash_alloc (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  return (struct rx_hash *)malloc (sizeof (struct rx_hash));
+}
+
+
+#ifdef __STDC__
+static struct rx_hash_item *
+compiler_hash_item_alloc (struct rx_hash_rules * rules, void * value)
+#else
+static struct rx_hash_item *
+compiler_hash_item_alloc (rules, value)
+     struct rx_hash_rules * rules;
+     void * value;
+#endif
+{
+  struct rx_hash_item * it;
+  it = (struct rx_hash_item *)malloc (sizeof (*it));
+  if (it)
+    {
+      it->data = value;
+      it->binding = 0;
+    }
+  return it;
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash (struct rx_hash * tab,
+		    struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash (tab, rules)
+     struct rx_hash * tab;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)tab);
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash_item (struct rx_hash_item * item,
+			 struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash_item (item, rules)
+     struct rx_hash_item * item;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)item);
+}
+
+
+
+/* This page: REXP_NODE (expression tree) structures. */
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rexp_node (struct rx *rx,
+	   enum rexp_node_type type)
+#else
+RX_DECL struct rexp_node *
+rexp_node (rx, type)
+     struct rx *rx;
+     enum rexp_node_type type;
+#endif
+{
+  struct rexp_node *n;
+
+  n = (struct rexp_node *)malloc (sizeof (*n));
+  bzero (n, sizeof (*n));
+  if (n)
+    n->type = type;
+  return n;
+}
+
+
+/* free_rexp_node assumes that the bitset passed to rx_mk_r_cset
+ * can be freed using rx_free_cset.
+ */
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_cset (struct rx * rx,
+	      rx_Bitset b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_cset (rx, b)
+     struct rx * rx;
+     rx_Bitset b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_cset);
+  if (n)
+    n->params.cset = b;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_concat (struct rx * rx,
+		struct rexp_node * a,
+		struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_concat (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_concat);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (struct rx * rx,
+		   struct rexp_node * a,
+		   struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_alternate);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_opt (struct rx * rx,
+	     struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_opt (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_opt);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_star (struct rx * rx,
+	      struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_star (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (struct rx * rx,
+		     struct rexp_node * a,
+		     struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_2phase_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (struct rx * rx,
+		     rx_side_effect a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (rx, a)
+     struct rx * rx;
+     rx_side_effect a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_side_effect);
+  if (n)
+    {
+      n->params.side_effect = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_data  (struct rx * rx,
+	       void * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_data  (rx, a)
+     struct rx * rx;
+     void * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_data);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_rexp (struct rx * rx, struct rexp_node * node)
+#else
+RX_DECL void
+rx_free_rexp (rx, node)
+     struct rx * rx;
+     struct rexp_node * node;
+#endif
+{
+  if (node)
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  if (node->params.cset)
+	    rx_free_cset (rx, node->params.cset);
+
+	case r_side_effect:
+	  break;
+	  
+	case r_concat:
+	case r_alternate:
+	case r_2phase_star:
+	case r_opt:
+	case r_star:
+	  rx_free_rexp (rx, node->params.pair.left);
+	  rx_free_rexp (rx, node->params.pair.right);
+	  break;
+
+	case r_data:
+	  /* This shouldn't occur. */
+	  break;
+	}
+      free ((char *)node);
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node * 
+rx_copy_rexp (struct rx *rx,
+	   struct rexp_node *node)
+#else
+RX_DECL struct rexp_node * 
+rx_copy_rexp (rx, node)
+     struct rx *rx;
+     struct rexp_node *node;
+#endif
+{
+  if (!node)
+    return 0;
+  else
+    {
+      struct rexp_node *n = rexp_node (rx, node->type);
+      if (!n)
+	return 0;
+      switch (node->type)
+	{
+	case r_cset:
+	  n->params.cset = rx_copy_cset (rx, node->params.cset);
+	  if (!n->params.cset)
+	    {
+	      rx_free_rexp (rx, n);
+	      return 0;
+	    }
+	  break;
+
+	case r_side_effect:
+	  n->params.side_effect = node->params.side_effect;
+	  break;
+
+	case r_concat:
+	case r_alternate:
+	case r_opt:
+	case r_2phase_star:
+	case r_star:
+	  n->params.pair.left =
+	    rx_copy_rexp (rx, node->params.pair.left);
+	  n->params.pair.right =
+	    rx_copy_rexp (rx, node->params.pair.right);
+	  if (   (node->params.pair.left && !n->params.pair.left)
+	      || (node->params.pair.right && !n->params.pair.right))
+	    {
+	      rx_free_rexp  (rx, n);
+	      return 0;
+	    }
+	  break;
+	case r_data:
+	  /* shouldn't happen */
+	  break;
+	}
+      return n;
+    }
+}
+
+
+
+
+/* This page: functions to build and destroy graphs that describe nfa's */
+
+/* Constructs a new nfa node. */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (struct rx *rx)
+#else
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state * n = (struct rx_nfa_state *)malloc (sizeof (*n));
+  if (!n)
+    return 0;
+  bzero (n, sizeof (*n));
+  n->next = rx->nfa_states;
+  rx->nfa_states = n;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_state (struct rx_nfa_state * n)
+#else
+RX_DECL void
+rx_free_nfa_state (n)
+  struct rx_nfa_state * n;
+#endif
+{
+  free ((char *)n);
+}
+
+
+/* This looks up an nfa node, given a numeric id.  Numeric id's are
+ * assigned after the nfa has been built.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (struct rx * rx,
+		    int id)
+#else
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (rx, id)
+     struct rx * rx;
+     int id;
+#endif
+{
+  struct rx_nfa_state * n;
+  for (n = rx->nfa_states; n; n = n->next)
+    if (n->id == id)
+      return n;
+  return 0;
+}
+
+
+/* This adds an edge between two nodes, but doesn't initialize the 
+ * edge label.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (struct rx *rx,
+	     enum rx_nfa_etype type,
+	     struct rx_nfa_state *start,
+	     struct rx_nfa_state *dest)
+#else
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (rx, type, start, dest)
+     struct rx *rx;
+     enum rx_nfa_etype type;
+     struct rx_nfa_state *start;
+     struct rx_nfa_state *dest;
+#endif
+{
+  struct rx_nfa_edge *e;
+  e = (struct rx_nfa_edge *)malloc (sizeof (*e));
+  if (!e)
+    return 0;
+  e->next = start->edges;
+  start->edges = e;
+  e->type = type;
+  e->dest = dest;
+  return e;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_edge (struct rx_nfa_edge * e)
+#else
+RX_DECL void
+rx_free_nfa_edge (e)
+     struct rx_nfa_edge * e;
+#endif
+{
+  free ((char *)e);
+}
+
+
+/* This constructs a POSSIBLE_FUTURE, which is a kind epsilon-closure
+ * of an NFA.  These are added to an nfa automaticly by eclose_nfa.
+ */  
+
+#ifdef __STDC__
+static struct rx_possible_future * 
+rx_possible_future (struct rx * rx,
+		 struct rx_se_list * effects)
+#else
+static struct rx_possible_future * 
+rx_possible_future (rx, effects)
+     struct rx * rx;
+     struct rx_se_list * effects;
+#endif
+{
+  struct rx_possible_future *ec;
+  ec = (struct rx_possible_future *) malloc (sizeof (*ec));
+  if (!ec)
+    return 0;
+  ec->destset = 0;
+  ec->next = 0;
+  ec->effects = effects;
+  return ec;
+}
+
+
+#ifdef __STDC__
+static void
+rx_free_possible_future (struct rx_possible_future * pf)
+#else
+static void
+rx_free_possible_future (pf)
+     struct rx_possible_future * pf;
+#endif
+{
+  free ((char *)pf);
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa (struct rx *rx)
+#else
+RX_DECL void
+rx_free_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  while (rx->nfa_states)
+    {
+      while (rx->nfa_states->edges)
+	{
+	  switch (rx->nfa_states->edges->type)
+	    {
+	    case ne_cset:
+	      rx_free_cset (rx, rx->nfa_states->edges->params.cset);
+	      break;
+	    default:
+	      break;
+	    }
+	  {
+	    struct rx_nfa_edge * e;
+	    e = rx->nfa_states->edges;
+	    rx->nfa_states->edges = rx->nfa_states->edges->next;
+	    rx_free_nfa_edge (e);
+	  }
+	} /* while (rx->nfa_states->edges) */
+      {
+	/* Iterate over the partial epsilon closures of rx->nfa_states */
+	struct rx_possible_future * pf = rx->nfa_states->futures;
+	while (pf)
+	  {
+	    struct rx_possible_future * pft = pf;
+	    pf = pf->next;
+	    rx_free_possible_future (pft);
+	  }
+      }
+      {
+	struct rx_nfa_state *n;
+	n = rx->nfa_states;
+	rx->nfa_states = rx->nfa_states->next;
+	rx_free_nfa_state (n);
+      }
+    }
+}
+
+
+
+
+/* This page: translating a pattern expression into an nfa and doing the 
+ * static part of the nfa->super-nfa translation.
+ */
+
+/* This is the thompson regexp->nfa algorithm. 
+ * It is modified to allow for `side-effect epsilons.'  Those are
+ * edges that are taken whenever a similar epsilon edge would be,
+ * but which imply that some side effect occurs when the edge 
+ * is taken.
+ *
+ * Side effects are used to model parts of the pattern langauge 
+ * that are not regular (in the formal sense).
+ */
+
+#ifdef __STDC__
+RX_DECL int
+rx_build_nfa (struct rx *rx,
+	      struct rexp_node *rexp,
+	      struct rx_nfa_state **start,
+	      struct rx_nfa_state **end)
+#else
+RX_DECL int
+rx_build_nfa (rx, rexp, start, end)
+     struct rx *rx;
+     struct rexp_node *rexp;
+     struct rx_nfa_state **start;
+     struct rx_nfa_state **end;
+#endif
+{
+  struct rx_nfa_edge *edge;
+
+  /* Start & end nodes may have been allocated by the caller. */
+  *start = *start ? *start : rx_nfa_state (rx);
+
+  if (!*start)
+    return 0;
+
+  if (!rexp)
+    {
+      *end = *start;
+      return 1;
+    }
+
+  *end = *end ? *end : rx_nfa_state (rx);
+
+  if (!*end)
+    {
+      rx_free_nfa_state (*start);
+      return 0;
+    }
+
+  switch (rexp->type)
+    {
+    case r_data:
+      return 0;
+
+    case r_cset:
+      edge = rx_nfa_edge (rx, ne_cset, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.cset = rx_copy_cset (rx, rexp->params.cset);
+      if (!edge->params.cset)
+	{
+	  rx_free_nfa_edge (edge);
+	  return 0;
+	}
+      return 1;
+ 
+    case r_opt:
+      return (rx_build_nfa (rx, rexp->params.pair.left, start, end)
+	      && rx_nfa_edge (rx, ne_epsilon, *start, *end));
+
+    case r_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& star_start
+		&& star_end
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, star_end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, star_start));
+      }
+
+    case r_2phase_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	struct rx_nfa_state * loop_exp_start = 0;
+	struct rx_nfa_state * loop_exp_end = 0;
+
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& rx_build_nfa (rx, rexp->params.pair.right,
+				 &loop_exp_start, &loop_exp_end)
+		&& star_start
+		&& star_end
+		&& loop_exp_end
+		&& loop_exp_start
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, loop_exp_start)
+		&& rx_nfa_edge (rx, ne_epsilon, loop_exp_end, star_start));
+      }
+
+
+    case r_concat:
+      {
+	struct rx_nfa_state *shared = 0;
+	return
+	  (rx_build_nfa (rx, rexp->params.pair.left, start, &shared)
+	   && rx_build_nfa (rx, rexp->params.pair.right, &shared, end));
+      }
+
+    case r_alternate:
+      {
+	struct rx_nfa_state *ls = 0;
+	struct rx_nfa_state *le = 0;
+	struct rx_nfa_state *rs = 0;
+	struct rx_nfa_state *re = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left, &ls, &le)
+		&& rx_build_nfa (rx, rexp->params.pair.right, &rs, &re)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, ls)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, rs)
+		&& rx_nfa_edge (rx, ne_epsilon, le, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, re, *end));
+      }
+
+    case r_side_effect:
+      edge = rx_nfa_edge (rx, ne_side_effect, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.side_effect = rexp->params.side_effect;
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+/* RX_NAME_NFA_STATES identifies all nodes with outgoing non-epsilon
+ * transitions.  Only these nodes can occur in super-states.  
+ * All nodes are given an integer id. 
+ * The id is non-negative if the node has non-epsilon out-transitions, negative
+ * otherwise (this is because we want the non-negative ids to be used as 
+ * array indexes in a few places).
+ */
+
+#ifdef __STDC__
+RX_DECL void
+rx_name_nfa_states (struct rx *rx)
+#else
+RX_DECL void
+rx_name_nfa_states (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+
+  rx->nodec = 0;
+  rx->epsnodec = -1;
+
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+
+      if (n->is_start)
+	n->eclosure_needed = 1;
+
+      while (e)
+	{
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      break;
+
+	    case ne_cset:
+	      n->id = rx->nodec++;
+	      {
+		struct rx_nfa_edge *from_n = n->edges;
+		while (from_n)
+		  {
+		    from_n->dest->eclosure_needed = 1;
+		    from_n = from_n->next;
+		  }
+	      }
+	      goto cont;
+	    }
+	  e = e->next;
+	}
+      n->id = rx->epsnodec--;
+    cont:
+      n = n->next;
+    }
+  rx->epsnodec = -rx->epsnodec;
+}
+
+
+
+/* This page: data structures for the static part of the nfa->supernfa
+ * translation.
+ *
+ * There are side effect lists -- lists of side effects occuring
+ * along an uninterrupted, acyclic path of side-effect epsilon edges.
+ * Such paths are collapsed to single edges in the course of computing
+ * epsilon closures.  Such single edges are labled with a list of all
+ * the side effects entailed in crossing them.  Like lists of side
+ * effects are made == by the constructors below.
+ *
+ * There are also nfa state sets.  These are used to hold a list of all
+ * states reachable from a starting state for a given type of transition
+ * and side effect list.   These are also hash-consed.
+ */
+
+/* The next several functions compare, construct, etc. lists of side
+ * effects.  See ECLOSE_NFA (below) for details.
+ */
+
+/* Ordering of rx_se_list
+ * (-1, 0, 1 return value convention).
+ */
+
+#ifdef __STDC__
+static int 
+se_list_cmp (void * va, void * vb)
+#else
+static int 
+se_list_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_se_list * a = (struct rx_se_list *)va;
+  struct rx_se_list * b = (struct rx_se_list *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: ((long)a->car < (long)b->car
+		   ? 1
+		   : ((long)a->car > (long)b->car
+		      ? -1
+		      : se_list_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+
+#ifdef __STDC__
+static int 
+se_list_equal (void * va, void * vb)
+#else
+static int 
+se_list_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !(se_list_cmp (va, vb));
+}
+
+static struct rx_hash_rules se_list_hash_rules =
+{
+  se_list_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_se_list * 
+side_effect_cons (struct rx * rx,
+		  void * se, struct rx_se_list * list)
+#else
+static struct rx_se_list * 
+side_effect_cons (rx, se, list)
+     struct rx * rx;
+     void * se;
+     struct rx_se_list * list;
+#endif
+{
+  struct rx_se_list * l;
+  l = ((struct rx_se_list *) malloc (sizeof (*l)));
+  if (!l)
+    return 0;
+  l->car = se;
+  l->cdr = list;
+  return l;
+}
+
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_cons_se_prog (struct rx * rx,
+		   struct rx_hash * memo,
+		   void * car, struct rx_se_list * cdr)
+#else
+static struct rx_se_list *
+hash_cons_se_prog (rx, memo, car, cdr)
+     struct rx * rx;
+     struct rx_hash * memo;
+     void * car;
+     struct rx_se_list * cdr;
+#endif
+{
+  long hash = (long)car ^ (long)cdr;
+  struct rx_se_list template;
+
+  template.car = car;
+  template.cdr = cdr;
+  {
+    struct rx_hash_item * it = rx_hash_store (memo, hash,
+					      (void *)&template,
+					      &se_list_hash_rules);
+    if (!it)
+      return 0;
+    if (it->data == (void *)&template)
+      {
+	struct rx_se_list * consed;
+	consed = (struct rx_se_list *) malloc (sizeof (*consed));
+	*consed = template;
+	it->data = (void *)consed;
+      }
+    return (struct rx_se_list *)it->data;
+  }
+}
+     
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_se_prog (struct rx * rx, struct rx_hash * memo, struct rx_se_list * prog)
+#else
+static struct rx_se_list *
+hash_se_prog (rx, memo, prog)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_se_list * prog;
+#endif
+{
+  struct rx_se_list * answer = 0;
+  while (prog)
+    {
+      answer = hash_cons_se_prog (rx, memo, prog->car, answer);
+      if (!answer)
+	return 0;
+      prog = prog->cdr;
+    }
+  return answer;
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_cmp (void * va, void * vb)
+#else
+static int 
+nfa_set_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state_set * a = (struct rx_nfa_state_set *)va;
+  struct rx_nfa_state_set * b = (struct rx_nfa_state_set *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: (a->car->id < b->car->id
+		   ? 1
+		   : (a->car->id > b->car->id
+		      ? -1
+		      : nfa_set_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_equal (void * va, void * vb)
+#else
+static int 
+nfa_set_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !nfa_set_cmp (va, vb);
+}
+
+static struct rx_hash_rules nfa_set_hash_rules =
+{
+  nfa_set_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_cons (struct rx * rx,
+	      struct rx_hash * memo, struct rx_nfa_state * state,
+	      struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_cons (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  struct rx_nfa_state_set template;
+  struct rx_hash_item * node;
+  template.car = state;
+  template.cdr = set;
+  node = rx_hash_store (memo,
+			(((long)state) >> 8) ^ (long)set,
+			&template, &nfa_set_hash_rules);
+  if (!node)
+    return 0;
+  if (node->data == &template)
+    {
+      struct rx_nfa_state_set * l;
+      l = (struct rx_nfa_state_set *) malloc (sizeof (*l));
+      node->data = (void *) l;
+      if (!l)
+	return 0;
+      *l = template;
+    }
+  return (struct rx_nfa_state_set *)node->data;
+}
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (struct rx * rx,
+		struct rx_hash * memo, struct rx_nfa_state * state,
+		struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  if (!set || state->id < set->car->id)
+    return nfa_set_cons (rx, memo, state, set);
+  if (state->id == set->car->id)
+    return set;
+  else
+    {
+      struct rx_nfa_state_set * newcdr
+	= nfa_set_enjoin (rx, memo, state, set->cdr);
+      if (newcdr != set->cdr)
+	set = nfa_set_cons (rx, memo, set->car, newcdr);
+      return set;
+    }
+}
+
+
+
+
+/* This page: computing epsilon closures.  The closures aren't total.
+ * Each node's closures are partitioned according to the side effects entailed
+ * along the epsilon edges.  Return true on success.
+ */ 
+
+struct eclose_frame
+{
+  struct rx_se_list *prog_backwards;
+};
+
+
+#ifdef __STDC__
+static int 
+eclose_node (struct rx *rx, struct rx_nfa_state *outnode,
+	     struct rx_nfa_state *node, struct eclose_frame *frame)
+#else
+static int 
+eclose_node (rx, outnode, node, frame)
+     struct rx *rx;
+     struct rx_nfa_state *outnode;
+     struct rx_nfa_state *node;
+     struct eclose_frame *frame;
+#endif
+{
+  struct rx_nfa_edge *e = node->edges;
+
+  /* For each node, we follow all epsilon paths to build the closure.
+   * The closure omits nodes that have only epsilon edges.
+   * The closure is split into partial closures -- all the states in
+   * a partial closure are reached by crossing the same list of
+   * of side effects (though not necessarily the same path).
+   */
+  if (node->mark)
+    return 1;
+  node->mark = 1;
+
+  if (node->id >= 0 || node->is_final)
+    {
+      struct rx_possible_future **ec;
+      struct rx_se_list * prog_in_order
+	= ((struct rx_se_list *)hash_se_prog (rx,
+					      &rx->se_list_memo,
+					      frame->prog_backwards));
+      int cmp;
+
+      ec = &outnode->futures;
+
+      while (*ec)
+	{
+	  cmp = se_list_cmp ((void *)(*ec)->effects, (void *)prog_in_order);
+	  if (cmp <= 0)
+	    break;
+	  ec = &(*ec)->next;
+	}
+      if (!*ec || (cmp < 0))
+	{
+	  struct rx_possible_future * saved = *ec;
+	  *ec = rx_possible_future (rx, prog_in_order);
+	  (*ec)->next = saved;
+	  if (!*ec)
+	    return 0;
+	}
+      if (node->id >= 0)
+	{
+	  (*ec)->destset = nfa_set_enjoin (rx, &rx->set_list_memo,
+					   node, (*ec)->destset);
+	  if (!(*ec)->destset)
+	    return 0;
+	}
+    }
+
+  while (e)
+    {
+      switch (e->type)
+	{
+	case ne_epsilon:
+	  if (!eclose_node (rx, outnode, e->dest, frame))
+	    return 0;
+	  break;
+	case ne_side_effect:
+	  {
+	    frame->prog_backwards = side_effect_cons (rx, 
+						      e->params.side_effect,
+						      frame->prog_backwards);
+	    if (!frame->prog_backwards)
+	      return 0;
+	    if (!eclose_node (rx, outnode, e->dest, frame))
+	      return 0;
+	    {
+	      struct rx_se_list * dying = frame->prog_backwards;
+	      frame->prog_backwards = frame->prog_backwards->cdr;
+	      free ((char *)dying);
+	    }
+	    break;
+	  }
+	default:
+	  break;
+	}
+      e = e->next;
+    }
+  node->mark = 0;
+  return 1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int 
+rx_eclose_nfa (struct rx *rx)
+#else
+RX_DECL int 
+rx_eclose_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct eclose_frame frame;
+  static int rx_id = 0;
+  
+  frame.prog_backwards = 0;
+  rx->rx_id = rx_id++;
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+  while (n)
+    {
+      n->futures = 0;
+      if (n->eclosure_needed && !eclose_node (rx, n, n, &frame))
+	return 0;
+      /* clear_marks (rx); */
+      n = n->next;
+    }
+  return 1;
+}
+
+
+/* This deletes epsilon edges from an NFA.  After running eclose_node,
+ * we have no more need for these edges.  They are removed to simplify
+ * further operations on the NFA.
+ */
+
+#ifdef __STDC__
+RX_DECL void 
+rx_delete_epsilon_transitions (struct rx *rx)
+#else
+RX_DECL void 
+rx_delete_epsilon_transitions (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct rx_nfa_edge **e;
+
+  while (n)
+    {
+      e = &n->edges;
+      while (*e)
+	{
+	  struct rx_nfa_edge *t;
+	  switch ((*e)->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      t = *e;
+	      *e = t->next;
+	      rx_free_nfa_edge (t);
+	      break;
+
+	    default:
+	      e = &(*e)->next;
+	      break;
+	    }
+	}
+      n = n->next;
+    }
+}
+
+
+
+/* This page: storing the nfa in a contiguous region of memory for
+ * subsequent conversion to a super-nfa.
+ */
+
+/* This is for qsort on an array of nfa_states. The order
+ * is based on state ids and goes 
+ *		[0...MAX][MIN..-1] where (MAX>=0) and (MIN<0)
+ * This way, positive ids double as array indices.
+ */
+
+#ifdef __STDC__
+static int 
+nfacmp (void * va, void * vb)
+#else
+static int 
+nfacmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state **a = (struct rx_nfa_state **)va;
+  struct rx_nfa_state **b = (struct rx_nfa_state **)vb;
+  return (*a == *b		/* &&&& 3.18 */
+	  ? 0
+	  : (((*a)->id < 0) == ((*b)->id < 0)
+	     ? (((*a)->id  < (*b)->id) ? -1 : 1)
+	     : (((*a)->id < 0)
+		? 1 : -1)));
+}
+
+#ifdef __STDC__
+static int 
+count_hash_nodes (struct rx_hash * st)
+#else
+static int 
+count_hash_nodes (st)
+     struct rx_hash * st;
+#endif
+{
+  int x;
+  int count = 0;
+  for (x = 0; x < 13; ++x)
+    count += ((st->children[x])
+	      ? count_hash_nodes (st->children[x])
+	      : st->bucket_size[x]);
+  
+  return count;
+}
+
+
+#ifdef __STDC__
+static void 
+se_memo_freer (struct rx_hash_item * node)
+#else
+static void 
+se_memo_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+#ifdef __STDC__
+static void 
+nfa_set_freer (struct rx_hash_item * node)
+#else
+static void 
+nfa_set_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+/* This copies an entire NFA into a single malloced block of memory.
+ * Mostly this is for compatability with regex.c, though it is convenient
+ * to have the nfa nodes in an array.
+ */
+
+#ifdef __STDC__
+RX_DECL int 
+rx_compactify_nfa (struct rx *rx,
+		   void **mem, unsigned long *size)
+#else
+RX_DECL int 
+rx_compactify_nfa (rx, mem, size)
+     struct rx *rx;
+     void **mem;
+     unsigned long *size;
+#endif
+{
+  int total_nodec;
+  struct rx_nfa_state *n;
+  int edgec = 0;
+  int eclosec = 0;
+  int se_list_consc = count_hash_nodes (&rx->se_list_memo);
+  int nfa_setc = count_hash_nodes (&rx->set_list_memo);
+  unsigned long total_size;
+
+  /* This takes place in two stages.   First, the total size of the
+   * nfa is computed, then structures are copied.  
+   */   
+  n = rx->nfa_states;
+  total_nodec = 0;
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      ++total_nodec;
+      while (e)
+	{
+	  ++edgec;
+	  e = e->next;
+	}
+      while (ec)
+	{
+	  ++eclosec;
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+
+  total_size = (total_nodec * sizeof (struct rx_nfa_state)
+		+ edgec * rx_sizeof_bitset (rx->local_cset_size)
+		+ edgec * sizeof (struct rx_nfa_edge)
+		+ nfa_setc * sizeof (struct rx_nfa_state_set)
+		+ eclosec * sizeof (struct rx_possible_future)
+		+ se_list_consc * sizeof (struct rx_se_list)
+		+ rx->reserved);
+
+  if (total_size > *size)
+    {
+      *mem = remalloc (*mem, total_size);
+      if (*mem)
+	*size = total_size;
+      else
+	return 0;
+    }
+  /* Now we've allocated the memory; this copies the NFA. */
+  {
+    static struct rx_nfa_state **scratch = 0;
+    static int scratch_alloc = 0;
+    struct rx_nfa_state *state_base = (struct rx_nfa_state *) * mem;
+    struct rx_nfa_state *new_state = state_base;
+    struct rx_nfa_edge *new_edge =
+      (struct rx_nfa_edge *)
+	((char *) state_base + total_nodec * sizeof (struct rx_nfa_state));
+    struct rx_se_list * new_se_list =
+      (struct rx_se_list *)
+	((char *)new_edge + edgec * sizeof (struct rx_nfa_edge));
+    struct rx_possible_future *new_close =
+      ((struct rx_possible_future *)
+       ((char *) new_se_list
+	+ se_list_consc * sizeof (struct rx_se_list)));
+    struct rx_nfa_state_set * new_nfa_set =
+      ((struct rx_nfa_state_set *)
+       ((char *)new_close + eclosec * sizeof (struct rx_possible_future)));
+    char *new_bitset =
+      ((char *) new_nfa_set + nfa_setc * sizeof (struct rx_nfa_state_set));
+    int x;
+    struct rx_nfa_state *n;
+
+    if (scratch_alloc < total_nodec)
+      {
+	scratch = ((struct rx_nfa_state **)
+		   remalloc (scratch, total_nodec * sizeof (*scratch)));
+	if (scratch)
+	  scratch_alloc = total_nodec;
+	else
+	  {
+	    scratch_alloc = 0;
+	    return 0;
+	  }
+      }
+
+    for (x = 0, n = rx->nfa_states; n; n = n->next)
+      scratch[x++] = n;
+
+    qsort (scratch, total_nodec,
+	   sizeof (struct rx_nfa_state *), (int (*)())nfacmp);
+
+    for (x = 0; x < total_nodec; ++x)
+      {
+	struct rx_possible_future *eclose = scratch[x]->futures;
+	struct rx_nfa_edge *edge = scratch[x]->edges;
+	struct rx_nfa_state *cn = new_state++;
+	cn->futures = 0;
+	cn->edges = 0;
+	cn->next = (x == total_nodec - 1) ? 0 : (cn + 1);
+	cn->id = scratch[x]->id;
+	cn->is_final = scratch[x]->is_final;
+	cn->is_start = scratch[x]->is_start;
+	cn->mark = 0;
+	while (edge)
+	  {
+	    int indx = (edge->dest->id < 0
+			 ? (total_nodec + edge->dest->id)
+			 : edge->dest->id);
+	    struct rx_nfa_edge *e = new_edge++;
+	    rx_Bitset cset = (rx_Bitset) new_bitset;
+	    new_bitset += rx_sizeof_bitset (rx->local_cset_size);
+	    rx_bitset_null (rx->local_cset_size, cset);
+	    rx_bitset_union (rx->local_cset_size, cset, edge->params.cset);
+	    e->next = cn->edges;
+	    cn->edges = e;
+	    e->type = edge->type;
+	    e->dest = state_base + indx;
+	    e->params.cset = cset;
+	    edge = edge->next;
+	  }
+	while (eclose)
+	  {
+	    struct rx_possible_future *ec = new_close++;
+	    struct rx_hash_item * sp;
+	    struct rx_se_list ** sepos;
+	    struct rx_se_list * sesrc;
+	    struct rx_nfa_state_set * destlst;
+	    struct rx_nfa_state_set ** destpos;
+	    ec->next = cn->futures;
+	    cn->futures = ec;
+	    for (sepos = &ec->effects, sesrc = eclose->effects;
+		 sesrc;
+		 sesrc = sesrc->cdr, sepos = &(*sepos)->cdr)
+	      {
+		sp = rx_hash_find (&rx->se_list_memo,
+				   (long)sesrc->car ^ (long)sesrc->cdr,
+				   sesrc, &se_list_hash_rules);
+		if (sp->binding)
+		  {
+		    sesrc = (struct rx_se_list *)sp->binding;
+		    break;
+		  }
+		*new_se_list = *sesrc;
+		sp->binding = (void *)new_se_list;
+		*sepos = new_se_list;
+		++new_se_list;
+	      }
+	    *sepos = sesrc;
+	    for (destpos = &ec->destset, destlst = eclose->destset;
+		 destlst;
+		 destpos = &(*destpos)->cdr, destlst = destlst->cdr)
+	      {
+		sp = rx_hash_find (&rx->set_list_memo,
+				   ((((long)destlst->car) >> 8)
+				    ^ (long)destlst->cdr),
+				   destlst, &nfa_set_hash_rules);
+		if (sp->binding)
+		  {
+		    destlst = (struct rx_nfa_state_set *)sp->binding;
+		    break;
+		  }
+		*new_nfa_set = *destlst;
+		new_nfa_set->car = state_base + destlst->car->id;
+		sp->binding = (void *)new_nfa_set;
+		*destpos = new_nfa_set;
+		++new_nfa_set;
+	      }
+	    *destpos = destlst;
+	    eclose = eclose->next;
+	  }
+      }
+  }
+  rx_free_hash_table (&rx->se_list_memo, se_memo_freer, &se_list_hash_rules);
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  rx_free_hash_table (&rx->set_list_memo, nfa_set_freer, &nfa_set_hash_rules);
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+
+  rx_free_nfa (rx);
+  rx->nfa_states = (struct rx_nfa_state *)*mem;
+  return 1;
+}
+
+
+
+/* The functions in the next several pages define the lazy-NFA-conversion used
+ * by matchers.  The input to this construction is an NFA such as 
+ * is built by compactify_nfa (rx.c).  The output is the superNFA.
+ */
+
+/* Match engines can use arbitrary values for opcodes.  So, the parse tree 
+ * is built using instructions names (enum rx_opcode), but the superstate
+ * nfa is populated with mystery opcodes (void *).
+ *
+ * For convenience, here is an id table.  The opcodes are == to their inxs
+ *
+ * The lables in re_search_2 would make good values for instructions.
+ */
+
+void * rx_id_instruction_table[rx_num_instructions] =
+{
+  (void *) rx_backtrack_point,
+  (void *) rx_do_side_effects,
+  (void *) rx_cache_miss,
+  (void *) rx_next_char,
+  (void *) rx_backtrack,
+  (void *) rx_error_inx
+};
+
+
+
+
+/* Memory mgt. for superstate graphs. */
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc (struct rx_cache * cache, int bytes)
+#else
+static char *
+rx_cache_malloc (cache, bytes)
+     struct rx_cache * cache;
+     int bytes;
+#endif
+{
+  while (cache->bytes_left < bytes)
+    {
+      if (cache->memory_pos)
+	cache->memory_pos = cache->memory_pos->next;
+      if (!cache->memory_pos)
+	{
+	  cache->morecore (cache);
+	  if (!cache->memory_pos)
+	    return 0;
+	}
+      cache->bytes_left = cache->memory_pos->bytes;
+      cache->memory_addr = ((char *)cache->memory_pos
+			    + sizeof (struct rx_blocklist));
+    }
+  cache->bytes_left -= bytes;
+  {
+    char * addr = cache->memory_addr;
+    cache->memory_addr += bytes;
+    return addr;
+  }
+}
+
+#ifdef __STDC__
+static void
+rx_cache_free (struct rx_cache * cache,
+	       struct rx_freelist ** freelist, char * mem)
+#else
+static void
+rx_cache_free (cache, freelist, mem)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     char * mem;
+#endif
+{
+  struct rx_freelist * it = (struct rx_freelist *)mem;
+  it->next = *freelist;
+  *freelist = it;
+}
+
+
+/* The partially instantiated superstate graph has a transition 
+ * table at every node.  There is one entry for every character.
+ * This fills in the transition for a set.
+ */
+#ifdef __STDC__
+static void 
+install_transition (struct rx_superstate *super,
+		    struct rx_inx *answer, rx_Bitset trcset) 
+#else
+static void 
+install_transition (super, answer, trcset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     rx_Bitset trcset;
+#endif
+{
+  struct rx_inx * transitions = super->transitions;
+  int chr;
+  for (chr = 0; chr < 256; )
+    if (!*trcset)
+      {
+	++trcset;
+	chr += 32;
+      }
+    else
+      {
+	RX_subset sub = *trcset;
+	RX_subset mask = 1;
+	int bound = chr + 32;
+	while (chr < bound)
+	  {
+	    if (sub & mask)
+	      transitions [chr] = *answer;
+	    ++chr;
+	    mask <<= 1;
+	  }
+	++trcset;
+      }
+}
+
+
+#ifdef __STDC__
+static int
+qlen (struct rx_superstate * q)
+#else
+static int
+qlen (q)
+     struct rx_superstate * q;
+#endif
+{
+  int count = 1;
+  struct rx_superstate * it;
+  if (!q)
+    return 0;
+  for (it = q->next_recyclable; it != q; it = it->next_recyclable)
+    ++count;
+  return count;
+}
+
+#ifdef __STDC__
+static void
+check_cache (struct rx_cache * cache)
+#else
+static void
+check_cache (cache)
+     struct rx_cache * cache;
+#endif
+{
+  struct rx_cache * you_fucked_up = 0;
+  int total = cache->superstates;
+  int semi = cache->semifree_superstates;
+  if (semi != qlen (cache->semifree_superstate))
+    check_cache (you_fucked_up);
+  if ((total - semi) != qlen (cache->lru_superstate))
+    check_cache (you_fucked_up);
+}
+
+/* When a superstate is old and neglected, it can enter a 
+ * semi-free state.  A semi-free state is slated to die.
+ * Incoming transitions to a semi-free state are re-written
+ * to cause an (interpreted) fault when they are taken.
+ * The fault handler revives the semi-free state, patches
+ * incoming transitions back to normal, and continues.
+ *
+ * The idea is basicly to free in two stages, aborting 
+ * between the two if the state turns out to be useful again.
+ * When a free is aborted, the rescued superstate is placed
+ * in the most-favored slot to maximize the time until it
+ * is next semi-freed.
+ */
+
+#ifdef __STDC__
+static void
+semifree_superstate (struct rx_cache * cache)
+#else
+static void
+semifree_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int disqualified = cache->semifree_superstates;
+  if (disqualified == cache->superstates)
+    return;
+  while (cache->lru_superstate->locks)
+    {
+      cache->lru_superstate = cache->lru_superstate->next_recyclable;
+      ++disqualified;
+      if (disqualified == cache->superstates)
+	return;
+    }
+  {
+    struct rx_superstate * it = cache->lru_superstate;
+    it->next_recyclable->prev_recyclable = it->prev_recyclable;
+    it->prev_recyclable->next_recyclable = it->next_recyclable;
+    cache->lru_superstate = (it == it->next_recyclable
+			     ? 0
+			     : it->next_recyclable);
+    if (!cache->semifree_superstate)
+      {
+	cache->semifree_superstate = it;
+	it->next_recyclable = it;
+	it->prev_recyclable = it;
+      }
+    else
+      {
+	it->prev_recyclable = cache->semifree_superstate->prev_recyclable;
+	it->next_recyclable = cache->semifree_superstate;
+	it->prev_recyclable->next_recyclable = it;
+	it->next_recyclable->prev_recyclable = it;
+      }
+    {
+      struct rx_distinct_future *df;
+      it->is_semifree = 1;
+      ++cache->semifree_superstates;
+      df = it->transition_refs;
+      if (df)
+	{
+	  df->prev_same_dest->next_same_dest = 0;
+	  for (df = it->transition_refs; df; df = df->next_same_dest)
+	    {
+	      df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	      df->future_frame.data = 0;
+	      df->future_frame.data_2 = (void *) df;
+	      /* If there are any NEXT-CHAR instruction frames that
+	       * refer to this state, we convert them to CACHE-MISS frames.
+	       */
+	      if (!df->effects
+		  && (df->edge->options->next_same_super_edge[0]
+		      == df->edge->options))
+		install_transition (df->present, &df->future_frame,
+				    df->edge->cset);
+	    }
+	  df = it->transition_refs;
+	  df->prev_same_dest->next_same_dest = df;
+	}
+    }
+  }
+}
+
+
+#ifdef __STDC__
+static void 
+refresh_semifree_superstate (struct rx_cache * cache,
+			     struct rx_superstate * super)
+#else
+static void 
+refresh_semifree_superstate (cache, super)
+     struct rx_cache * cache;
+     struct rx_superstate * super;
+#endif
+{
+  struct rx_distinct_future *df;
+
+  if (super->transition_refs)
+    {
+      super->transition_refs->prev_same_dest->next_same_dest = 0; 
+      for (df = super->transition_refs; df; df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_next_char];
+	  df->future_frame.data = (void *) super->transitions;
+	  /* CACHE-MISS instruction frames that refer to this state,
+	   * must be converted to NEXT-CHAR frames.
+	   */
+	  if (!df->effects
+	      && (df->edge->options->next_same_super_edge[0]
+		  == df->edge->options))
+	    install_transition (df->present, &df->future_frame,
+				df->edge->cset);
+	}
+      super->transition_refs->prev_same_dest->next_same_dest
+	= super->transition_refs;
+    }
+  if (cache->semifree_superstate == super)
+    cache->semifree_superstate = (super->prev_recyclable == super
+				  ? 0
+				  : super->prev_recyclable);
+  super->next_recyclable->prev_recyclable = super->prev_recyclable;
+  super->prev_recyclable->next_recyclable = super->next_recyclable;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = super->next_recyclable
+     = super->prev_recyclable
+     = super);
+  else
+    {
+      super->next_recyclable = cache->lru_superstate;
+      super->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      super->next_recyclable->prev_recyclable = super;
+      super->prev_recyclable->next_recyclable = super;
+    }
+  super->is_semifree = 0;
+  --cache->semifree_superstates;
+}
+
+#ifdef __STDC__
+static void
+rx_refresh_this_superstate (struct rx_cache * cache, struct rx_superstate * superstate)
+#else
+static void
+rx_refresh_this_superstate (cache, superstate)
+     struct rx_cache * cache;
+     struct rx_superstate * superstate;
+#endif
+{
+  if (superstate->is_semifree)
+    refresh_semifree_superstate (cache, superstate);
+  else if (cache->lru_superstate == superstate)
+    cache->lru_superstate = superstate->next_recyclable;
+  else if (superstate != cache->lru_superstate->prev_recyclable)
+    {
+      superstate->next_recyclable->prev_recyclable
+	= superstate->prev_recyclable;
+      superstate->prev_recyclable->next_recyclable
+	= superstate->next_recyclable;
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      superstate->next_recyclable->prev_recyclable = superstate;
+      superstate->prev_recyclable->next_recyclable = superstate;
+    }
+}
+
+#ifdef __STDC__
+static void 
+release_superset_low (struct rx_cache * cache,
+		     struct rx_superset *set)
+#else
+static void 
+release_superset_low (cache, set)
+     struct rx_cache * cache;
+     struct rx_superset *set;
+#endif
+{
+  if (!--set->refs)
+    {
+      if (set->cdr)
+	release_superset_low (cache, set->cdr);
+
+      set->starts_for = 0;
+
+      rx_hash_free
+	(rx_hash_find
+	 (&cache->superset_table,
+	  (unsigned long)set->car ^ set->id ^ (unsigned long)set->cdr,
+	  (void *)set,
+	  &cache->superset_hash_rules),
+	 &cache->superset_hash_rules);
+      rx_cache_free (cache, &cache->free_supersets, (char *)set);
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void 
+rx_release_superset (struct rx *rx,
+		     struct rx_superset *set)
+#else
+RX_DECL void 
+rx_release_superset (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  release_superset_low (rx->cache, set);
+}
+
+/* This tries to add a new superstate to the superstate freelist.
+ * It might, as a result, free some edge pieces or hash tables.
+ * If nothing can be freed because too many locks are being held, fail.
+ */
+
+#ifdef __STDC__
+static int
+rx_really_free_superstate (struct rx_cache * cache)
+#else
+static int
+rx_really_free_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int locked_superstates = 0;
+  struct rx_superstate * it;
+
+  if (!cache->superstates)
+    return 0;
+
+  {
+    /* This is a total guess.  The idea is that we should expect as
+     * many misses as we've recently experienced.  I.e., cache->misses
+     * should be the same as cache->semifree_superstates.
+     */
+    while ((cache->hits + cache->misses) > cache->superstates_allowed)
+      {
+	cache->hits >>= 1;
+	cache->misses >>= 1;
+      }
+    if (  ((cache->hits + cache->misses) * cache->semifree_superstates)
+	< (cache->superstates		 * cache->misses))
+      {
+	semifree_superstate (cache);
+	semifree_superstate (cache);
+      }
+  }
+
+  while (cache->semifree_superstate && cache->semifree_superstate->locks)
+    {
+      refresh_semifree_superstate (cache, cache->semifree_superstate);
+      ++locked_superstates;
+      if (locked_superstates == cache->superstates)
+	return 0;
+    }
+
+  if (cache->semifree_superstate)
+    {
+      it = cache->semifree_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->semifree_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+      --cache->semifree_superstates;
+    }
+  else
+    {
+      while (cache->lru_superstate->locks)
+	{
+	  cache->lru_superstate = cache->lru_superstate->next_recyclable;
+	  ++locked_superstates;
+	  if (locked_superstates == cache->superstates)
+	    return 0;
+	}
+      it = cache->lru_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->lru_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+    }
+
+  if (it->transition_refs)
+    {
+      struct rx_distinct_future *df;
+      for (df = it->transition_refs,
+	   df->prev_same_dest->next_same_dest = 0;
+	   df;
+	   df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	  df->future_frame.data = 0;
+	  df->future_frame.data_2 = (void *) df;
+	  df->future = 0;
+	}
+      it->transition_refs->prev_same_dest->next_same_dest =
+	it->transition_refs;
+    }
+  {
+    struct rx_super_edge *tc = it->edges;
+    while (tc)
+      {
+	struct rx_distinct_future * df;
+	struct rx_super_edge *tct = tc->next;
+	df = tc->options;
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+	while (df)
+	  {
+	    struct rx_distinct_future *dft = df;
+	    df = df->next_same_super_edge[0];
+	    
+	    
+	    if (dft->future && dft->future->transition_refs == dft)
+	      {
+		dft->future->transition_refs = dft->next_same_dest;
+		if (dft->future->transition_refs == dft)
+		  dft->future->transition_refs = 0;
+	      }
+	    dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+	    dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+	    rx_cache_free (cache, &cache->free_discernable_futures,
+			   (char *)dft);
+	  }
+	rx_cache_free (cache, &cache->free_transition_classes, (char *)tc);
+	tc = tct;
+      }
+  }
+  
+  if (it->contents->superstate == it)
+    it->contents->superstate = 0;
+  release_superset_low (cache, it->contents);
+  rx_cache_free (cache, &cache->free_superstates, (char *)it);
+  --cache->superstates;
+  return 1;
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get (struct rx_cache * cache,
+	      struct rx_freelist ** freelist)
+#else
+static char *
+rx_cache_get (cache, freelist)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+#endif
+{
+  while (!*freelist && rx_really_free_superstate (cache))
+    ;
+  if (!*freelist)
+    return 0;
+  {
+    struct rx_freelist * it = *freelist;
+    *freelist = it->next;
+    return (char *)it;
+  }
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc_or_get (struct rx_cache * cache,
+			struct rx_freelist ** freelist, int bytes)
+#else
+static char *
+rx_cache_malloc_or_get (cache, freelist, bytes)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     int bytes;
+#endif
+{
+  if (!*freelist)
+    {
+      char * answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	return answer;
+    }
+
+  return rx_cache_get (cache, freelist);
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get_superstate (struct rx_cache * cache)
+#else
+static char *
+rx_cache_get_superstate (cache)
+	  struct rx_cache * cache;
+#endif
+{
+  char * answer;
+  int bytes = (   sizeof (struct rx_superstate)
+	       +  cache->local_cset_size * sizeof (struct rx_inx));
+  if (!cache->free_superstates
+      && (cache->superstates < cache->superstates_allowed))
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	{
+	  ++cache->superstates;
+	  return answer;
+	}
+    }
+  answer = rx_cache_get (cache, &cache->free_superstates);
+  if (!answer)
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	++cache->superstates_allowed;
+    }
+  ++cache->superstates;
+  return answer;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+supersetcmp (void * va, void * vb)
+#else
+static int
+supersetcmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_superset * a = (struct rx_superset *)va;
+  struct rx_superset * b = (struct rx_superset *)vb;
+  return (   (a == b)
+	  || (a && b && (a->car == b->car) && (a->cdr == b->cdr)));
+}
+
+#ifdef __STDC__
+static struct rx_hash_item *
+superset_allocator (struct rx_hash_rules * rules, void * val)
+#else
+static struct rx_hash_item *
+superset_allocator (rules, val)
+     struct rx_hash_rules * rules;
+     void * val;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  struct rx_superset * template = (struct rx_superset *)val;
+  struct rx_superset * newset
+    = ((struct rx_superset *)
+       rx_cache_malloc_or_get (cache,
+			       &cache->free_supersets,
+			       sizeof (*template)));
+  if (!newset)
+    return 0;
+  newset->refs = 0;
+  newset->car = template->car;
+  newset->id = template->car->id;
+  newset->cdr = template->cdr;
+  newset->superstate = 0;
+  rx_protect_superset (rx, template->cdr);
+  newset->hash_item.data = (void *)newset;
+  newset->hash_item.binding = 0;
+  return &newset->hash_item;
+}
+
+#ifdef __STDC__
+static struct rx_hash * 
+super_hash_allocator (struct rx_hash_rules * rules)
+#else
+static struct rx_hash * 
+super_hash_allocator (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  return ((struct rx_hash *)
+	  rx_cache_malloc_or_get (cache,
+				  &cache->free_hash, sizeof (struct rx_hash)));
+}
+
+
+#ifdef __STDC__
+static void
+super_hash_liberator (struct rx_hash * hash, struct rx_hash_rules * rules)
+#else
+static void
+super_hash_liberator (hash, rules)
+     struct rx_hash * hash;
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       (char *)rules - (long)(&((struct rx_cache *)0)->superset_hash_rules));
+  rx_cache_free (cache, &cache->free_hash, (char *)hash);
+}
+
+#ifdef __STDC__
+static void
+superset_hash_item_liberator (struct rx_hash_item * it,
+			      struct rx_hash_rules * rules)
+#else
+static void
+superset_hash_item_liberator (it, rules) /* Well, it does ya know. */
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+}
+
+int rx_cache_bound = 128;
+static int rx_default_cache_got = 0;
+
+#ifdef __STDC__
+static int
+bytes_for_cache_size (int supers, int cset_size)
+#else
+static int
+bytes_for_cache_size (supers, cset_size)
+     int supers;
+     int cset_size;
+#endif
+{
+  /* What the hell is this? !!!*/
+  return (int)
+    ((float)supers *
+     (  (1.03 * (float) (  rx_sizeof_bitset (cset_size)
+			 + sizeof (struct rx_super_edge)))
+      + (1.80 * (float) sizeof (struct rx_possible_future))
+      + (float) (  sizeof (struct rx_superstate)
+		 + cset_size * sizeof (struct rx_inx))));
+}
+
+#ifdef __STDC__
+static void
+rx_morecore (struct rx_cache * cache)
+#else
+static void
+rx_morecore (cache)
+     struct rx_cache * cache;
+#endif
+{
+  if (rx_default_cache_got >= rx_cache_bound)
+    return;
+
+  rx_default_cache_got += 16;
+  cache->superstates_allowed = rx_cache_bound;
+  {
+    struct rx_blocklist ** pos = &cache->memory;
+    int size = bytes_for_cache_size (16, cache->local_cset_size);
+    while (*pos)
+      pos = &(*pos)->next;
+    *pos = ((struct rx_blocklist *)
+	    malloc (size + sizeof (struct rx_blocklist))); 
+    if (!*pos)
+      return;
+
+    (*pos)->next = 0;
+    (*pos)->bytes = size;
+    cache->memory_pos = *pos;
+    cache->memory_addr = (char *)*pos + sizeof (**pos);
+    cache->bytes_left = size;
+  }
+}
+
+static struct rx_cache default_cache = 
+{
+  {
+    supersetcmp,
+    super_hash_allocator,
+    super_hash_liberator,
+    superset_allocator,
+    superset_hash_item_liberator,
+  },
+  0,
+  0,
+  0,
+  0,
+  rx_morecore,
+
+  0,
+  0,
+  0,
+  0,
+  0,
+
+  0,
+  0,
+
+  0,
+
+  0,
+  0,
+  0,
+  0,
+  128,
+
+  256,
+  rx_id_instruction_table,
+
+  {
+    0,
+    0,
+    {0},
+    {0},
+    {0}
+  }
+};
+
+/* This adds an element to a superstate set.  These sets are lists, such
+ * that lists with == elements are ==.  The empty set is returned by
+ * superset_cons (rx, 0, 0) and is NOT equivelent to 
+ * (struct rx_superset)0.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superset_cons (struct rx * rx,
+		  struct rx_nfa_state *car, struct rx_superset *cdr)
+#else
+RX_DECL struct rx_superset *
+rx_superset_cons (rx, car, cdr)
+     struct rx * rx;
+     struct rx_nfa_state *car;
+     struct rx_superset *cdr;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  if (!car && !cdr)
+    {
+      if (!cache->empty_superset)
+	{
+	  cache->empty_superset
+	    = ((struct rx_superset *)
+	       rx_cache_malloc_or_get (cache, &cache->free_supersets,
+				       sizeof (struct rx_superset)));
+	  if (!cache->empty_superset)
+	    return 0;
+	  bzero (cache->empty_superset, sizeof (struct rx_superset));
+	  cache->empty_superset->refs = 1000;
+	}
+      return cache->empty_superset;
+    }
+  {
+    struct rx_superset template;
+    struct rx_hash_item * hit;
+    template.car = car;
+    template.cdr = cdr;
+    template.id = car->id;
+    hit = rx_hash_store (&cache->superset_table,
+			 (unsigned long)car ^ car->id ^ (unsigned long)cdr,
+			 (void *)&template,
+			 &cache->superset_hash_rules);
+    return (hit
+	    ?  (struct rx_superset *)hit->data
+	    : 0);
+  }
+}
+
+/* This computes a union of two NFA state sets.  The sets do not have the
+ * same representation though.  One is a RX_SUPERSET structure (part
+ * of the superstate NFA) and the other is an NFA_STATE_SET (part of the NFA).
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl) 
+#else
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union (rx, set, ecl)
+     struct rx * rx;
+     struct rx_superset *set;
+     struct rx_nfa_state_set *ecl;
+#endif
+{
+  if (!ecl)
+    return set;
+
+  if (!set->car)
+    return rx_superset_cons (rx, ecl->car,
+			     rx_superstate_eclosure_union (rx, set, ecl->cdr));
+  if (set->car == ecl->car)
+    return rx_superstate_eclosure_union (rx, set, ecl->cdr);
+
+  {
+    struct rx_superset * tail;
+    struct rx_nfa_state * first;
+
+    if (set->car > ecl->car)
+      {
+	tail = rx_superstate_eclosure_union (rx, set->cdr, ecl);
+	first = set->car;
+      }
+    else
+      {
+	tail = rx_superstate_eclosure_union (rx, set, ecl->cdr);
+	first = ecl->car;
+      }
+    if (!tail)
+      return 0;
+    else
+      {
+	struct rx_superset * answer;
+	answer = rx_superset_cons (rx, first, tail);
+	if (!answer)
+	  {
+	    rx_protect_superset (rx, tail);
+	    rx_release_superset (rx, tail);
+	    return 0;
+	  }
+	else
+	  return answer;
+      }
+  }
+}
+
+
+
+
+
+/*
+ * This makes sure that a list of rx_distinct_futures contains
+ * a future for each possible set of side effects in the eclosure
+ * of a given state.  This is some of the work of filling in a
+ * superstate transition. 
+ */
+
+#ifdef __STDC__
+static struct rx_distinct_future *
+include_futures (struct rx *rx,
+		 struct rx_distinct_future *df, struct rx_nfa_state
+		 *state, struct rx_superstate *superstate) 
+#else
+static struct rx_distinct_future *
+include_futures (rx, df, state, superstate)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+     struct rx_nfa_state *state;
+     struct rx_superstate *superstate;
+#endif
+{
+  struct rx_possible_future *future;
+  struct rx_cache * cache = rx->cache;
+  for (future = state->futures; future; future = future->next)
+    {
+      struct rx_distinct_future *dfp;
+      struct rx_distinct_future *insert_before = 0;
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      for (dfp = df; dfp; dfp = dfp->next_same_super_edge[0])
+	if (dfp->effects == future->effects)
+	  break;
+	else
+	  {
+	    int order = rx->se_list_cmp (rx, dfp->effects, future->effects);
+	    if (order > 0)
+	      {
+		insert_before = dfp;
+		dfp = 0;
+		break;
+	      }
+	  }
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+      if (!dfp)
+	{
+	  dfp
+	    = ((struct rx_distinct_future *)
+	       rx_cache_malloc_or_get (cache, &cache->free_discernable_futures,
+				       sizeof (struct rx_distinct_future)));
+	  if (!dfp)
+	    return 0;
+	  if (!df)
+	    {
+	      df = insert_before = dfp;
+	      df->next_same_super_edge[0] = df->next_same_super_edge[1] = df;
+	    }
+	  else if (!insert_before)
+	    insert_before = df;
+	  else if (insert_before == df)
+	    df = dfp;
+
+	  dfp->next_same_super_edge[0] = insert_before;
+	  dfp->next_same_super_edge[1]
+	    = insert_before->next_same_super_edge[1];
+	  dfp->next_same_super_edge[1]->next_same_super_edge[0] = dfp;
+	  dfp->next_same_super_edge[0]->next_same_super_edge[1] = dfp;
+	  dfp->next_same_dest = dfp->prev_same_dest = dfp;
+	  dfp->future = 0;
+	  dfp->present = superstate;
+	  dfp->future_frame.inx = rx->instruction_table[rx_cache_miss];
+	  dfp->future_frame.data = 0;
+	  dfp->future_frame.data_2 = (void *) dfp;
+	  dfp->side_effects_frame.inx
+	    = rx->instruction_table[rx_do_side_effects];
+	  dfp->side_effects_frame.data = 0;
+	  dfp->side_effects_frame.data_2 = (void *) dfp;
+	  dfp->effects = future->effects;
+	}
+    }
+  return df;
+}
+
+
+
+
+/* This constructs a new superstate from its state set.  The only 
+ * complexity here is memory management.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_superstate *
+rx_superstate (struct rx *rx,
+	       struct rx_superset *set)
+#else
+RX_DECL struct rx_superstate *
+rx_superstate (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  struct rx_superstate * superstate = 0;
+
+  /* Does the superstate already exist in the cache? */
+  if (set->superstate)
+    {
+      if (set->superstate->rx_id != rx->rx_id)
+	{
+	  /* Aha.  It is in the cache, but belongs to a superstate
+	   * that refers to an NFA that no longer exists.
+	   * (We know it no longer exists because it was evidently
+	   *  stored in the same region of memory as the current nfa
+	   *  yet it has a different id.)
+	   */
+	  superstate = set->superstate;
+	  if (!superstate->is_semifree)
+	    {
+	      if (cache->lru_superstate == superstate)
+		{
+		  cache->lru_superstate = superstate->next_recyclable;
+		  if (cache->lru_superstate == superstate)
+		    cache->lru_superstate = 0;
+		}
+	      {
+		superstate->next_recyclable->prev_recyclable
+		  = superstate->prev_recyclable;
+		superstate->prev_recyclable->next_recyclable
+		  = superstate->next_recyclable;
+		if (!cache->semifree_superstate)
+		  {
+		    (cache->semifree_superstate
+		     = superstate->next_recyclable
+		     = superstate->prev_recyclable
+		     = superstate);
+		  }
+		else
+		  {
+		    superstate->next_recyclable = cache->semifree_superstate;
+		    superstate->prev_recyclable
+		      = cache->semifree_superstate->prev_recyclable;
+		    superstate->next_recyclable->prev_recyclable
+		      = superstate;
+		    superstate->prev_recyclable->next_recyclable
+		      = superstate;
+		    cache->semifree_superstate = superstate;
+		  }
+		++cache->semifree_superstates;
+	      }
+	    }
+	  set->superstate = 0;
+	  goto handle_cache_miss;
+	}
+      ++cache->hits;
+      superstate = set->superstate;
+
+      rx_refresh_this_superstate (cache, superstate);
+      return superstate;
+    }
+
+ handle_cache_miss:
+
+  /* This point reached only for cache misses. */
+  ++cache->misses;
+#if RX_DEBUG
+  if (rx_debug_trace > 1)
+    {
+      struct rx_superset * setp = set;
+      fprintf (stderr, "Building a superstet %d(%d): ", rx->rx_id, set);
+      while (setp)
+	{
+	  fprintf (stderr, "%d ", setp->id);
+	  setp = setp->cdr;
+	}
+      fprintf (stderr, "(%d)\n", set);
+    }
+#endif
+  superstate = (struct rx_superstate *)rx_cache_get_superstate (cache);
+  if (!superstate)
+    return 0;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = superstate->next_recyclable
+     = superstate->prev_recyclable
+     = superstate);
+  else
+    {
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      (  superstate->prev_recyclable->next_recyclable
+       = superstate->next_recyclable->prev_recyclable
+       = superstate);
+    }
+  superstate->rx_id = rx->rx_id;
+  superstate->transition_refs = 0;
+  superstate->locks = 0;
+  superstate->is_semifree = 0;
+  set->superstate = superstate;
+  superstate->contents = set;
+  rx_protect_superset (rx, set);
+  superstate->edges = 0;
+  {
+    int x;
+    /* None of the transitions from this superstate are known yet. */
+    for (x = 0; x < rx->local_cset_size; ++x) /* &&&&& 3.8 % */
+      {
+	struct rx_inx * ifr = &superstate->transitions[x];
+	ifr->inx = rx->instruction_table [rx_cache_miss];
+	ifr->data = ifr->data_2 = 0;
+      }
+  }
+  return superstate;
+}
+
+
+
+/* This computes the destination set of one edge of the superstate NFA.
+ * Note that a RX_DISTINCT_FUTURE is a superstate edge.
+ * Returns 0 on an allocation failure.
+ */
+
+#ifdef __STDC__
+static int 
+solve_destination (struct rx *rx, struct rx_distinct_future *df)
+#else
+static int 
+solve_destination (rx, df)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc = df->edge;
+  struct rx_superset *nfa_state;
+  struct rx_superset *nil_set = rx_superset_cons (rx, 0, 0);
+  struct rx_superset *solution = nil_set;
+  struct rx_superstate *dest;
+
+  rx_protect_superset (rx, solution);
+  /* Iterate over all NFA states in the state set of this superstate. */
+  for (nfa_state = df->present->contents;
+       nfa_state->car;
+       nfa_state = nfa_state->cdr)
+    {
+      struct rx_nfa_edge *e;
+      /* Iterate over all edges of each NFA state. */
+      for (e = nfa_state->car->edges; e; e = e->next)
+        /* If we find an edge that is labeled with 
+	 * the characters we are solving for.....
+	 */
+	if (rx_bitset_is_subset (rx->local_cset_size,
+				 tc->cset, e->params.cset))
+	  {
+	    struct rx_nfa_state *n = e->dest;
+	    struct rx_possible_future *pf;
+	    /* ....search the partial epsilon closures of the destination
+	     * of that edge for a path that involves the same set of
+	     * side effects we are solving for.
+	     * If we find such a RX_POSSIBLE_FUTURE, we add members to the
+	     * stateset we are computing.
+	     */
+	    for (pf = n->futures; pf; pf = pf->next)
+	      if (pf->effects == df->effects)
+		{
+		  struct rx_superset * old_sol;
+		  old_sol = solution;
+		  solution = rx_superstate_eclosure_union (rx, solution,
+							   pf->destset);
+		  if (!solution)
+		    return 0;
+		  rx_protect_superset (rx, solution);
+		  rx_release_superset (rx, old_sol);
+		}
+	  }
+    }
+  /* It is possible that the RX_DISTINCT_FUTURE we are working on has 
+   * the empty set of NFA states as its definition.  In that case, this
+   * is a failure point.
+   */
+  if (solution == nil_set)
+    {
+      df->future_frame.inx = (void *) rx_backtrack;
+      df->future_frame.data = 0;
+      df->future_frame.data_2 = 0;
+      return 1;
+    }
+  dest = rx_superstate (rx, solution);
+  rx_release_superset (rx, solution);
+  if (!dest)
+    return 0;
+
+  {
+    struct rx_distinct_future *dft;
+    dft = df;
+    df->prev_same_dest->next_same_dest = 0;
+    while (dft)
+      {
+	dft->future = dest;
+	dft->future_frame.inx = rx->instruction_table[rx_next_char];
+	dft->future_frame.data = (void *) dest->transitions;
+	dft = dft->next_same_dest;
+      }
+    df->prev_same_dest->next_same_dest = df;
+  }
+  if (!dest->transition_refs)
+    dest->transition_refs = df;
+  else
+    {
+      struct rx_distinct_future *dft = dest->transition_refs->next_same_dest;
+      dest->transition_refs->next_same_dest = df->next_same_dest;
+      df->next_same_dest->prev_same_dest = dest->transition_refs;
+      df->next_same_dest = dft;
+      dft->prev_same_dest = df;
+    }
+  return 1;
+}
+
+
+/* This takes a superstate and a character, and computes some edges
+ * from the superstate NFA.  In particular, this computes all edges
+ * that lead from SUPERSTATE given CHR.   This function also 
+ * computes the set of characters that share this edge set.
+ * This returns 0 on allocation error.
+ * The character set and list of edges are returned through 
+ * the paramters CSETOUT and DFOUT.
+} */
+
+#ifdef __STDC__
+static int 
+compute_super_edge (struct rx *rx, struct rx_distinct_future **dfout,
+			  rx_Bitset csetout, struct rx_superstate *superstate,
+			  unsigned char chr)  
+#else
+static int 
+compute_super_edge (rx, dfout, csetout, superstate, chr)
+     struct rx *rx;
+     struct rx_distinct_future **dfout;
+     rx_Bitset csetout;
+     struct rx_superstate *superstate;
+     unsigned char chr;
+#endif
+{
+  struct rx_superset *stateset = superstate->contents;
+
+  /* To compute the set of characters that share edges with CHR, 
+   * we start with the full character set, and subtract.
+   */
+  rx_bitset_universe (rx->local_cset_size, csetout);
+  *dfout = 0;
+
+  /* Iterate over the NFA states in the superstate state-set. */
+  while (stateset->car)
+    {
+      struct rx_nfa_edge *e;
+      for (e = stateset->car->edges; e; e = e->next)
+	if (RX_bitset_member (e->params.cset, chr))
+	  {
+	    /* If we find an NFA edge that applies, we make sure there
+	     * are corresponding edges in the superstate NFA.
+	     */
+	    {
+	      struct rx_distinct_future * saved;
+	      saved = *dfout;
+	      *dfout = include_futures (rx, *dfout, e->dest, superstate);
+	      if (!*dfout)
+		{
+		  struct rx_distinct_future * df;
+		  df = saved;
+		  df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+		  while (df)
+		    {
+		      struct rx_distinct_future *dft;
+		      dft = df;
+		      df = df->next_same_super_edge[0];
+
+		      if (dft->future && dft->future->transition_refs == dft)
+			{
+			  dft->future->transition_refs = dft->next_same_dest;
+			  if (dft->future->transition_refs == dft)
+			    dft->future->transition_refs = 0;
+			}
+		      dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+		      dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+		      rx_cache_free (rx->cache,
+				     &rx->cache->free_discernable_futures,
+				     (char *)dft);
+		    }
+		  return 0;
+		}
+	    }
+	    /* We also trim the character set a bit. */
+	    rx_bitset_intersection (rx->local_cset_size,
+				    csetout, e->params.cset);
+	  }
+	else
+	  /* An edge that doesn't apply at least tells us some characters
+	   * that don't share the same edge set as CHR.
+	   */
+	  rx_bitset_difference (rx->local_cset_size, csetout, e->params.cset);
+      stateset = stateset->cdr;
+    }
+  return 1;
+}
+
+
+
+/* This is a constructor for RX_SUPER_EDGE structures.  These are
+ * wrappers for lists of superstate NFA edges that share character sets labels.
+ * If a transition class contains more than one rx_distinct_future (superstate
+ * edge), then it represents a non-determinism in the superstate NFA.
+ */
+
+#ifdef __STDC__
+static struct rx_super_edge *
+rx_super_edge (struct rx *rx,
+	       struct rx_superstate *super, rx_Bitset cset,
+	       struct rx_distinct_future *df) 
+#else
+static struct rx_super_edge *
+rx_super_edge (rx, super, cset, df)
+     struct rx *rx;
+     struct rx_superstate *super;
+     rx_Bitset cset;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc =
+    (struct rx_super_edge *)rx_cache_malloc_or_get
+      (rx->cache, &rx->cache->free_transition_classes,
+       sizeof (struct rx_super_edge) + rx_sizeof_bitset (rx->local_cset_size));
+
+  if (!tc)
+    return 0;
+  tc->next = super->edges;
+  super->edges = tc;
+  tc->rx_backtrack_frame.inx = rx->instruction_table[rx_backtrack_point];
+  tc->rx_backtrack_frame.data = 0;
+  tc->rx_backtrack_frame.data_2 = (void *) tc;
+  tc->options = df;
+  tc->cset = (rx_Bitset) ((char *) tc + sizeof (*tc));
+  rx_bitset_assign (rx->local_cset_size, tc->cset, cset);
+  if (df)
+    {
+      struct rx_distinct_future * dfp = df;
+      df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      while (dfp)
+	{
+	  dfp->edge = tc;
+	  dfp = dfp->next_same_super_edge[0];
+	}
+      df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+    }
+  return tc;
+}
+
+
+/* There are three kinds of cache miss.  The first occurs when a
+ * transition is taken that has never been computed during the
+ * lifetime of the source superstate.  That cache miss is handled by
+ * calling COMPUTE_SUPER_EDGE.  The second kind of cache miss
+ * occurs when the destination superstate of a transition doesn't
+ * exist.  SOLVE_DESTINATION is used to construct the destination superstate.
+ * Finally, the third kind of cache miss occurs when the destination
+ * superstate of a transition is in a `semi-free state'.  That case is
+ * handled by UNFREE_SUPERSTATE.
+ *
+ * The function of HANDLE_CACHE_MISS is to figure out which of these
+ * cases applies.
+ */
+
+#ifdef __STDC__
+static void
+install_partial_transition  (struct rx_superstate *super,
+			     struct rx_inx *answer,
+			     RX_subset set, int offset)
+#else
+static void
+install_partial_transition  (super, answer, set, offset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     RX_subset set;
+     int offset;
+#endif
+{
+  int start = offset;
+  int end = start + 32;
+  RX_subset pos = 1;
+  struct rx_inx * transitions = super->transitions;
+  
+  while (start < end)
+    {
+      if (set & pos)
+	transitions[start] = *answer;
+      pos <<= 1;
+      ++start;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_inx *
+rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data) 
+#else
+RX_DECL struct rx_inx *
+rx_handle_cache_miss (rx, super, chr, data)
+     struct rx *rx;
+     struct rx_superstate *super;
+     unsigned char chr;
+     void *data;
+#endif
+{
+  int offset = chr / RX_subset_bits;
+  struct rx_distinct_future *df = data;
+
+  if (!df)			/* must be the shared_cache_miss_frame */
+    {
+      /* Perhaps this is just a transition waiting to be filled. */
+      struct rx_super_edge *tc;
+      RX_subset mask = rx_subset_singletons [chr % RX_subset_bits];
+
+      for (tc = super->edges; tc; tc = tc->next)
+	if (tc->cset[offset] & mask)
+	  {
+	    struct rx_inx * answer;
+	    df = tc->options;
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	    install_partial_transition (super, answer,
+					tc->cset [offset], offset * 32);
+	    return answer;
+	  }
+      /* Otherwise, it's a flushed or  newly encountered edge. */
+      {
+	char cset_space[1024];	/* this limit is far from unreasonable */
+	rx_Bitset trcset;
+	struct rx_inx *answer;
+
+	if (rx_sizeof_bitset (rx->local_cset_size) > sizeof (cset_space))
+	  return 0;		/* If the arbitrary limit is hit, always fail */
+				/* cleanly. */
+	trcset = (rx_Bitset)cset_space;
+	rx_lock_superstate (rx, super);
+	if (!compute_super_edge (rx, &df, trcset, super, chr))
+	  {
+	    rx_unlock_superstate (rx, super);
+	    return 0;
+	  }
+	if (!df)		/* We just computed the fail transition. */
+	  {
+	    static struct rx_inx
+	      shared_fail_frame = { 0, 0, (void *)rx_backtrack, 0 };
+	    answer = &shared_fail_frame;
+	  }
+	else
+	  {
+	    tc = rx_super_edge (rx, super, trcset, df);
+	    if (!tc)
+	      {
+		rx_unlock_superstate (rx, super);
+		return 0;
+	      }
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	  }
+	install_partial_transition (super, answer,
+				    trcset[offset], offset * 32);
+	rx_unlock_superstate (rx, super);
+	return answer;
+      }
+    }
+  else if (df->future) /* A cache miss on an edge with a future? Must be
+			* a semi-free destination. */
+    {				
+      if (df->future->is_semifree)
+	refresh_semifree_superstate (rx->cache, df->future);
+      return &df->future_frame;
+    }
+  else
+    /* no future superstate on an existing edge */
+    {
+      rx_lock_superstate (rx, super);
+      if (!solve_destination (rx, df))
+	{
+	  rx_unlock_superstate (rx, super);
+	  return 0;
+	}
+      if (!df->effects
+	  && (df->edge->options->next_same_super_edge[0] == df->edge->options))
+	install_partial_transition (super, &df->future_frame,
+				    df->edge->cset[offset], offset * 32);
+      rx_unlock_superstate (rx, super);
+      return &df->future_frame;
+    }
+}
+
+
+
+
+
+/* The rest of the code provides a regex.c compatable interface. */
+
+
+__const__ char *re_error_msg[] =
+{
+  0,						/* REG_NOUT */
+  "No match",					/* REG_NOMATCH */
+  "Invalid regular expression",			/* REG_BADPAT */
+  "Invalid collation character",		/* REG_ECOLLATE */
+  "Invalid character class name",		/* REG_ECTYPE */
+  "Trailing backslash",				/* REG_EESCAPE */
+  "Invalid back reference",			/* REG_ESUBREG */
+  "Unmatched [ or [^",				/* REG_EBRACK */
+  "Unmatched ( or \\(",				/* REG_EPAREN */
+  "Unmatched \\{",				/* REG_EBRACE */
+  "Invalid content of \\{\\}",			/* REG_BADBR */
+  "Invalid range end",				/* REG_ERANGE */
+  "Memory exhausted",				/* REG_ESPACE */
+  "Invalid preceding regular expression",	/* REG_BADRPT */
+  "Premature end of regular expression",	/* REG_EEND */
+  "Regular expression too big",			/* REG_ESIZE */
+  "Unmatched ) or \\)",				/* REG_ERPAREN */
+};
+
+
+
+
+/* 
+ * Macros used while compiling patterns.
+ *
+ * By convention, PEND points just past the end of the uncompiled pattern,
+ * P points to the read position in the pattern.  `translate' is the name
+ * of the translation table (`TRANSLATE' is the name of a macro that looks
+ * things up in `translate').
+ */
+
+
+/*
+ * Fetch the next character in the uncompiled pattern---translating it 
+ * if necessary. *Also cast from a signed character in the constant
+ * string passed to us by the user to an unsigned char that we can use
+ * as an array index (in, e.g., `translate').
+ */
+#define PATFETCH(c)							\
+ do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    c = translate[c];		 					\
+ } while (0)
+
+/* 
+ * Fetch the next character in the uncompiled pattern, with no
+ * translation.
+ */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+#define TRANSLATE(d) translate[(unsigned char) (d)]
+
+typedef unsigned regnum_t;
+
+/* Since offsets can go either forwards or backwards, this type needs to
+ * be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.
+ */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  struct rexp_node ** top_expression; /* was begalt */
+  struct rexp_node ** last_expression; /* was laststart */
+  pattern_offset_t inner_group_offset;
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / CHARBITS]               \
+   |= 1 << (((unsigned char) c) % CHARBITS))
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (isdigit (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (!strcmp (string, "alpha") || !strcmp (string, "upper")		\
+    || !strcmp (string, "lower") || !strcmp (string, "digit")		\
+    || !strcmp (string, "alnum") || !strcmp (string, "xdigit")		\
+    || !strcmp (string, "space") || !strcmp (string, "print")		\
+    || !strcmp (string, "punct") || !strcmp (string, "graph")		\
+    || !strcmp (string, "cntrl") || !strcmp (string, "blank"))
+
+
+
+/* These predicates are used in regex_compile. */
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+ * after an alternative or a begin-subexpression.  We assume there is at
+ * least one character before the ^.  
+ */
+
+#ifdef __STDC__
+static boolean
+at_begline_loc_p (__const__ char *pattern, __const__ char * p, reg_syntax_t syntax)
+#else
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+     __const__ char *pattern;
+     __const__ char * p;
+     reg_syntax_t syntax;
+#endif
+{
+  __const__ char *prev = p - 2;
+  boolean prev_prev_backslash = ((prev > pattern) && (prev[-1] == '\\'));
+  
+    return
+      
+      (/* After a subexpression?  */
+       ((*prev == '(') && ((syntax & RE_NO_BK_PARENS) || prev_prev_backslash))
+       ||
+       /* After an alternative?  */
+       ((*prev == '|') && ((syntax & RE_NO_BK_VBAR) || prev_prev_backslash))
+       );
+}
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+ * at least one character after the $, i.e., `P < PEND'.
+ */
+
+#ifdef __STDC__
+static boolean
+at_endline_loc_p (__const__ char *p, __const__ char *pend, int syntax)
+#else
+static boolean
+at_endline_loc_p (p, pend, syntax)
+     __const__ char *p;
+     __const__ char *pend;
+     int syntax;
+#endif
+{
+  __const__ char *next = p;
+  boolean next_backslash = (*next == '\\');
+  __const__ char *next_next = (p + 1 < pend) ? (p + 1) : 0;
+  
+  return
+    (
+     /* Before a subexpression?  */
+     ((syntax & RE_NO_BK_PARENS)
+      ? (*next == ')')
+      : (next_backslash && next_next && (*next_next == ')')))
+    ||
+     /* Before an alternative?  */
+     ((syntax & RE_NO_BK_VBAR)
+      ? (*next == '|')
+      : (next_backslash && next_next && (*next_next == '|')))
+     );
+}
+
+
+
+unsigned char rx_id_translation[256] =
+{
+  0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+ 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+ 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+ 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+ 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+ 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
+ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
+ 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
+
+ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
+ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249,
+ 250, 251, 252, 253, 254, 255
+};
+
+/* The compiler keeps an inverted translation table.
+ * This looks up/inititalize elements.
+ * VALID is an array of booleans that validate CACHE.
+ */
+
+#ifdef __STDC__
+static rx_Bitset
+inverse_translation (struct re_pattern_buffer * rxb,
+		     char * valid, rx_Bitset cache,
+		     unsigned char * translate, int c)
+#else
+static rx_Bitset
+inverse_translation (rxb, valid, cache, translate, c)
+     struct re_pattern_buffer * rxb;
+     char * valid;
+     rx_Bitset cache;
+     unsigned char * translate;
+     int c;
+#endif
+{
+  rx_Bitset cs
+    = cache + c * rx_bitset_numb_subsets (rxb->rx.local_cset_size); 
+
+  if (!valid[c])
+    {
+      int x;
+      int c_tr = TRANSLATE(c);
+      rx_bitset_null (rxb->rx.local_cset_size, cs);
+      for (x = 0; x < 256; ++x)	/* &&&& 13.37 */
+	if (TRANSLATE(x) == c_tr)
+	  RX_bitset_enjoin (cs, x);
+      valid[c] = 1;
+    }
+  return cs;
+}
+
+
+
+
+
+/* More subroutine declarations and macros for regex_compile.  */
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+#ifdef __STDC__
+static boolean
+group_in_compile_stack (compile_stack_type compile_stack, regnum_t regnum)
+#else
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+#endif
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/*
+ * Read the ending character of a range (in a bracket expression) from the
+ * uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+ * starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+ * Then we set the translation of all bits between the starting and
+ * ending characters (inclusive) in the compiled pattern B.
+ * 
+ * Return an error code.
+ * 
+ * We use these short variable names so we can use the same macros as
+ * `regex_compile' itself.  
+ */
+
+#ifdef __STDC__
+static reg_errcode_t
+compile_range (struct re_pattern_buffer * rxb, rx_Bitset cs,
+	       __const__ char ** p_ptr, __const__ char * pend,
+	       unsigned char * translate, reg_syntax_t syntax,
+	       rx_Bitset inv_tr,  char * valid_inv_tr)
+#else
+static reg_errcode_t
+compile_range (rxb, cs, p_ptr, pend, translate, syntax, inv_tr, valid_inv_tr)
+     struct re_pattern_buffer * rxb;
+     rx_Bitset cs;
+     __const__ char ** p_ptr;
+     __const__ char * pend;
+     unsigned char * translate;
+     reg_syntax_t syntax;
+     rx_Bitset inv_tr;
+     char * valid_inv_tr;
+#endif
+{
+  unsigned this_char;
+
+  __const__ char *p = *p_ptr;
+
+  unsigned char range_end;
+  unsigned char range_start = TRANSLATE(p[-2]);
+
+  if (p == pend)
+    return REG_ERANGE;
+
+  PATFETCH (range_end);
+
+  (*p_ptr)++;
+
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      rx_Bitset it =
+	inverse_translation (rxb, valid_inv_tr, inv_tr, translate, this_char);
+      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+    }
+  
+  return REG_NOERROR;
+}
+
+
+
+/* This searches a regexp for backreference side effects.
+ * It fills in the array OUT with 1 at the index of every register pair
+ * referenced by a backreference.
+ *
+ * This is used to help optimize patterns for searching.  The information is
+ * useful because, if the caller doesn't want register values, backreferenced
+ * registers are the only registers for which we need rx_backtrack.
+ */
+
+#ifdef __STDC__
+static void
+find_backrefs (char * out, struct rexp_node * rexp,
+	       struct re_se_params * params)
+#else
+static void
+find_backrefs (out, rexp, params)
+     char * out;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (rexp)
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return;
+      case r_alternate:
+      case r_concat:
+      case r_opt:
+      case r_star:
+      case r_2phase_star:
+	find_backrefs (out, rexp->params.pair.left, params);
+	find_backrefs (out, rexp->params.pair.right, params);
+	return;
+      case r_side_effect:
+	if (   ((long)rexp->params.side_effect >= 0)
+	    && (params [(long)rexp->params.side_effect].se == re_se_backref))
+	  out[ params [(long)rexp->params.side_effect].op1] = 1;
+	return;
+      }
+}
+
+
+
+
+/* Returns 0 unless the pattern can match the empty string. */
+
+#ifdef __STDC__
+static int
+compute_fastset (struct re_pattern_buffer * rxb, struct rexp_node * rexp)
+#else
+static int
+compute_fastset (rxb, rexp)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 1;
+  switch (rexp->type)
+    {
+    case r_data:
+      return 1;
+    case r_cset:
+      {
+	rx_bitset_union (rxb->rx.local_cset_size,
+			 rxb->fastset, rexp->params.cset);
+      }
+      return 0;
+    case r_concat:
+      return (compute_fastset (rxb, rexp->params.pair.left)
+	      && compute_fastset (rxb, rexp->params.pair.right));
+    case r_2phase_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      /* compute_fastset (rxb, rexp->params.pair.right);  nope... */
+      return 1;
+    case r_alternate:
+      return !!(compute_fastset (rxb, rexp->params.pair.left)
+		+ compute_fastset (rxb, rexp->params.pair.right));
+    case r_opt:
+    case r_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      return 1;
+    case r_side_effect:
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* returns
+ *  1 -- yes, definately anchored by the given side effect.
+ *  2 -- maybe anchored, maybe the empty string.
+ *  0 -- definately not anchored
+ *  There is simply no other possibility.
+ */
+
+#ifdef __STDC__
+static int
+is_anchored (struct rexp_node * rexp, rx_side_effect se)
+#else
+static int
+is_anchored (rexp, se)
+     struct rexp_node * rexp;
+     rx_side_effect se;
+#endif
+{
+  if (!rexp)
+    return 2;
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+    case r_concat:
+    case r_2phase_star:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	return (l == 2 ? is_anchored (rexp->params.pair.right, se) : l);
+      }
+    case r_alternate:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	int r = l ? is_anchored (rexp->params.pair.right, se) : 0;
+
+	if (l == r)
+	  return l;
+	else if ((l == 0) || (r == 0))
+	  return 0;
+	else
+	  return 2;
+      }
+    case r_opt:
+    case r_star:
+      return is_anchored (rexp->params.pair.left, se) ? 2 : 0;
+      
+    case r_side_effect:
+      return ((rexp->params.side_effect == se)
+	      ? 1 : 2);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This removes register assignments that aren't required by backreferencing.
+ * This can speed up explore_future, especially if it eliminates
+ * non-determinism in the superstate NFA.
+ * 
+ * NEEDED is an array of characters, presumably filled in by FIND_BACKREFS.
+ * The non-zero elements of the array indicate which register assignments
+ * can NOT be removed from the expression.
+ */
+
+#ifdef __STDC__
+static struct rexp_node *
+remove_unecessary_side_effects (struct rx * rx, char * needed,
+				struct rexp_node * rexp,
+				struct re_se_params * params)
+#else
+static struct rexp_node *
+remove_unecessary_side_effects (rx, needed, rexp, params)
+     struct rx * rx;
+     char * needed;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  struct rexp_node * l;
+  struct rexp_node * r;
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return rexp;
+      case r_alternate:
+      case r_concat:
+      case r_2phase_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	r = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.right, params);
+	if ((l && r) || (rexp->type != r_concat))
+	  {
+	    rexp->params.pair.left = l;
+	    rexp->params.pair.right = r;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = rexp->params.pair.right = 0;
+	    rx_free_rexp (rx, rexp);
+	    return l ? l : r;
+	  }
+      case r_opt:
+      case r_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	if (l)
+	  {
+	    rexp->params.pair.left = l;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = 0;
+	    rx_free_rexp (rx, rexp);
+	    return 0;
+	  }
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   (se >= 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen))
+	      && (params [se].op1 > 0)
+	      && (!needed [params [se].op1]))
+	    {
+	      rx_free_rexp (rx, rexp);
+	      return 0;
+	    }
+	  else
+	    return rexp;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+pointless_if_repeated (struct rexp_node * node, struct re_se_params * params)
+#else
+static int
+pointless_if_repeated (node, params)
+     struct rexp_node * node;
+     struct re_se_params * params;
+#endif
+{
+  if (!node)
+    return 1;
+  switch (node->type)
+    {
+    case r_cset:
+      return 0;
+    case r_alternate:
+    case r_concat:
+    case r_2phase_star:
+      return (pointless_if_repeated (node->params.pair.left, params)
+	      && pointless_if_repeated (node->params.pair.right, params));
+    case r_opt:
+    case r_star:
+      return pointless_if_repeated (node->params.pair.left, params);
+    case r_side_effect:
+      switch ((enum re_side_effects) (((long)node->params.side_effect < 0)
+	      ? (enum re_side_effects) (long) node->params.side_effect
+	      : (enum re_side_effects)params[(long)node->params.side_effect].se))
+	{
+	case re_se_try:
+	case re_se_at_dot:
+	case re_se_begbuf:
+	case re_se_hat:
+	case re_se_wordbeg:
+	case re_se_wordbound:
+	case re_se_notwordbound:
+	case re_se_wordend:
+	case re_se_endbuf:
+	case re_se_dollar:
+	case re_se_fail:
+	case re_se_win:
+	  return 1;
+	case re_se_lparen:
+	case re_se_rparen:
+	case re_se_iter:
+	case re_se_end_iter:
+	case re_se_syntax:
+	case re_se_not_syntax:
+	case re_se_backref:
+	  return 0;
+	}
+    case r_data:
+    default:
+      return 0;
+    }
+}
+
+
+
+
+#ifdef __STDC__
+static int
+registers_on_stack (struct re_pattern_buffer * rxb,
+		    struct rexp_node * rexp, int in_danger,
+		    struct re_se_params * params)
+#else
+static int
+registers_on_stack (rxb, rexp, in_danger, params)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+     int in_danger;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return 0;
+      case r_alternate:
+      case r_concat:
+	return (   registers_on_stack (rxb, rexp->params.pair.left,
+				       in_danger, params)
+		|| (registers_on_stack
+		    (rxb, rexp->params.pair.right,
+		     in_danger, params)));
+      case r_opt:
+	return registers_on_stack (rxb, rexp->params.pair.left, 0, params);
+      case r_star:
+	return registers_on_stack (rxb, rexp->params.pair.left, 1, params);
+      case r_2phase_star:
+	return
+	  (   registers_on_stack (rxb, rexp->params.pair.left, 1, params)
+	   || registers_on_stack (rxb, rexp->params.pair.right, 1, params));
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   in_danger
+	      && (se >= 0)
+	      && (params [se].op1 > 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen)))
+	    return 1;
+	  else
+	    return 0;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+static char idempotent_complex_se[] =
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     IDEM,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  23
+};
+
+static char idempotent_se[] =
+{
+  13,
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      IDEM,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  42
+};
+
+
+
+
+
+#ifdef __STDC__
+static int
+has_any_se (struct rx * rx,
+	    struct rexp_node * rexp)
+#else
+static int
+has_any_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return 1;
+      
+    case r_2phase_star:
+    case r_concat:
+    case r_alternate:
+      return
+	(   has_any_se (rx, rexp->params.pair.left)
+	 || has_any_se (rx, rexp->params.pair.right));
+
+    case r_opt:
+    case r_star:
+      return has_any_se (rx, rexp->params.pair.left);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This must be called AFTER `convert_hard_loops' for a given REXP. */
+#ifdef __STDC__
+static int
+has_non_idempotent_epsilon_path (struct rx * rx,
+				 struct rexp_node * rexp,
+				 struct re_se_params * params)
+#else
+static int
+has_non_idempotent_epsilon_path (rx, rexp, params)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_star:
+      return 0;
+
+    case r_side_effect:
+      return
+	!((long)rexp->params.side_effect > 0
+	  ? idempotent_complex_se [ params [(long)rexp->params.side_effect].se ]
+	  : idempotent_se [-(long)rexp->params.side_effect]);
+      
+    case r_alternate:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 || has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_2phase_star:
+    case r_concat:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 && has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_opt:
+      return has_non_idempotent_epsilon_path (rx,
+					      rexp->params.pair.left, params);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This computes rougly what it's name suggests.   It can (and does) go wrong 
+ * in the direction of returning spurious 0 without causing disasters.
+ */
+#ifdef __STDC__
+static int
+begins_with_complex_se (struct rx * rx, struct rexp_node * rexp)
+#else
+static int
+begins_with_complex_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return ((long)rexp->params.side_effect >= 0);
+      
+    case r_alternate:
+      return
+	(   begins_with_complex_se (rx, rexp->params.pair.left)
+	 && begins_with_complex_se (rx, rexp->params.pair.right));
+
+
+    case r_concat:
+      return has_any_se (rx, rexp->params.pair.left);
+    case r_opt:
+    case r_star:
+    case r_2phase_star:
+      return 0;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This destructively removes some of the re_se_tv side effects from 
+ * a rexp tree.  In particular, during parsing re_se_tv was inserted on the
+ * right half of every | to guarantee that posix path preference could be 
+ * honored.  This function removes some which it can be determined aren't 
+ * needed.  
+ */
+
+#ifdef __STDC__
+static void
+speed_up_alt (struct rx * rx,
+	      struct rexp_node * rexp,
+	      int unposix)
+#else
+static void
+speed_up_alt (rx, rexp, unposix)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     int unposix;
+#endif
+{
+  if (!rexp)
+    return;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_side_effect:
+      return;
+
+    case r_opt:
+    case r_star:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      return;
+
+    case r_2phase_star:
+    case r_concat:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right, unposix);
+      return;
+
+    case r_alternate:
+      /* the right child is guaranteed to be (concat re_se_tv <subexp>) */
+
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right->params.pair.right, unposix);
+      
+      if (   unposix
+	  || (begins_with_complex_se
+	      (rx, rexp->params.pair.right->params.pair.right))
+	  || !(   has_any_se (rx, rexp->params.pair.right->params.pair.right)
+	       || has_any_se (rx, rexp->params.pair.left)))
+	{
+	  struct rexp_node * conc = rexp->params.pair.right;
+	  rexp->params.pair.right = conc->params.pair.right;
+	  conc->params.pair.right = 0;
+	  rx_free_rexp (rx, conc);
+	}
+    }
+}
+
+
+
+
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is set to zero;
+     `re_nsub' is set to the number of groups in PATTERN;
+     `not_bol' and `not_eol' are set to zero.
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+
+
+#ifdef __STDC__
+RX_DECL reg_errcode_t
+rx_compile (__const__ char *pattern, int size,
+	    reg_syntax_t syntax,
+	    struct re_pattern_buffer * rxb) 
+#else
+RX_DECL reg_errcode_t
+rx_compile (pattern, size, syntax, rxb)
+     __const__ char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  RX_subset
+    inverse_translate [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+  char
+    validate_inv_tr [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random tempory spot in PATTERN.  */
+  __const__ char *p1;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  __const__ char *p = pattern;
+  __const__ char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  unsigned char *translate = (rxb->translate
+			      ? rxb->translate
+			      : rx_id_translation);
+
+  /* When parsing is done, this will hold the expression tree. */
+  struct rexp_node * rexp = 0;
+
+  /* In the midst of compilation, this holds onto the regexp 
+   * first parst while rexp goes on to aquire additional constructs.
+   */
+  struct rexp_node * orig_rexp = 0;
+  struct rexp_node * fewer_side_effects = 0;
+
+  /* This and top_expression are saved on the compile stack. */
+  struct rexp_node ** top_expression = &rexp;
+  struct rexp_node ** last_expression = top_expression;
+  
+  /* Parameter to `goto append_node' */
+  struct rexp_node * append;
+
+  /* Counts open-groups as they are encountered.  This is the index of the
+   * innermost group being compiled.
+   */
+  regnum_t regnum = 0;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+   * which to go back if the interval is invalid.  
+   */
+  __const__ char *beg_interval;
+
+  struct re_se_params * params = 0;
+  int paramc = 0;		/* How many complex side effects so far? */
+
+  rx_side_effect side;		/* param to `goto add_side_effect' */
+
+  bzero (validate_inv_tr, sizeof (validate_inv_tr));
+
+  rxb->rx.instruction_table = rx_id_instruction_table;
+
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack =  (( compile_stack_elt_t *) malloc ((INIT_COMPILE_STACK_SIZE) * sizeof ( compile_stack_elt_t)));
+  if (compile_stack.stack == 0)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  rxb->rx.cache = &default_cache;
+  rxb->syntax = syntax;
+  rxb->fastmap_accurate = 0;
+  rxb->not_bol = rxb->not_eol = 0;
+  rxb->least_subs = 0;
+  
+  /* Always count groups, whether or not rxb->no_sub is set.  
+   * The whole pattern is implicitly group 0, so counting begins
+   * with 1.
+   */
+  rxb->re_nsub = 0;
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_hat);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_dollar);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (pointless_if_repeated (*last_expression, params))
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                return REG_BADRPT;
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!last_expression)
+              break;
+
+	    /* Now we know whether or not zero matches is allowed
+	     * and also whether or not two or more matches is allowed.
+	     */
+
+	    {
+	      struct rexp_node * inner_exp = *last_expression;
+	      int need_sync = 0;
+
+	      if (many_times_ok
+		  && has_non_idempotent_epsilon_path (&rxb->rx,
+						      inner_exp, params))
+		{
+		  struct rexp_node * pusher
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushpos);
+		  struct rexp_node * checker
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_chkpos);
+		  struct rexp_node * pushback
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushback);
+		  rx_Bitset cs = rx_cset (&rxb->rx);
+		  struct rexp_node * lit_t = rx_mk_r_cset (&rxb->rx, cs);
+		  struct rexp_node * fake_state
+		    = rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		  struct rexp_node * phase2
+		    = rx_mk_r_concat (&rxb->rx, checker, fake_state);
+		  struct rexp_node * popper
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_poppos);
+		  struct rexp_node * star
+		    = rx_mk_r_2phase_star (&rxb->rx, inner_exp, phase2);
+		  struct rexp_node * a
+		    = rx_mk_r_concat (&rxb->rx, pusher, star);
+		  struct rexp_node * whole_thing
+		    = rx_mk_r_concat (&rxb->rx, a, popper);
+		  if (!(pusher && star && pushback && lit_t && fake_state
+			&& lit_t && phase2 && checker && popper
+			&& a && whole_thing))
+		    return REG_ESPACE;
+		  RX_bitset_enjoin (cs, 't');
+		  *last_expression = whole_thing;
+		}
+	      else
+		{
+		  struct rexp_node * star =
+		    (many_times_ok ? rx_mk_r_star : rx_mk_r_opt)
+		      (&rxb->rx, *last_expression);
+		  if (!star)
+		    return REG_ESPACE;
+		  *last_expression = star;
+		  need_sync = has_any_se (&rxb->rx, *last_expression);
+		}
+	      if (!zero_times_ok)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, inner_exp,
+				      rx_copy_rexp (&rxb->rx,
+						    *last_expression));
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		}
+	      if (need_sync)
+		{
+		  int sync_se = paramc;
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					sizeof (*params) * (1 + paramc)))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (*params))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+		  params [sync_se].se = re_se_tv;
+		  side = (rx_side_effect)sync_se;
+		  goto add_side_effect;
+		}
+	    }
+	    /* The old regex.c used to optimize `.*\n'.  
+	     * Maybe rx should too?
+	     */
+	  }
+	  break;
+
+
+	case '.':
+	  {
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * n = rx_mk_r_cset (&rxb->rx, cs);
+	    if (!(cs && n))
+	      return REG_ESPACE;
+
+	    rx_bitset_universe (rxb->rx.local_cset_size, cs);
+	    if (!(rxb->syntax & RE_DOT_NEWLINE))
+	      RX_bitset_remove (cs, '\n');
+	    if (!(rxb->syntax & RE_DOT_NOT_NULL))
+	      RX_bitset_remove (cs, 0);
+
+	    append = n;
+	    goto append_node;
+	    break;
+	  }
+
+
+        case '[':
+	  if (p == pend) return REG_EBRACK;
+          {
+            boolean had_char_class = false;
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * node = rx_mk_r_cset (&rxb->rx, cs);
+	    int is_inverted = *p == '^';
+	    
+	    if (!(node && cs))
+	      return REG_ESPACE;
+	    
+	    /* This branch of the switch is normally exited with
+	     *`goto append_node'
+	     */
+	    append = node;
+	    
+            if (is_inverted)
+	      p++;
+	    
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+	    
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) return REG_EBRACK;
+		
+                PATFETCH (c);
+		
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+		    
+                    PATFETCH (c1);
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c1);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                    continue;
+                  }
+		
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  goto finalize_class_and_append;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  return REG_ERANGE;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (rxb, cs, &p, pend, translate, syntax,
+				       inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+		    
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (rxb, cs, &p, pend, translate, syntax,
+					 inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                /* See if we're at the beginning of a possible character
+                   class.  */
+		
+		else if ((syntax & RE_CHAR_CLASSES)
+			 && (c == '[') && (*p == ':'))
+                  {
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+		    
+                    PATFETCH (c);
+                    c1 = 0;
+		    
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) return REG_EBRACK;
+		    
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+			  break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+		    
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = !strcmp (str, "alnum");
+                        boolean is_alpha = !strcmp (str, "alpha");
+                        boolean is_blank = !strcmp (str, "blank");
+                        boolean is_cntrl = !strcmp (str, "cntrl");
+                        boolean is_digit = !strcmp (str, "digit");
+                        boolean is_graph = !strcmp (str, "graph");
+                        boolean is_lower = !strcmp (str, "lower");
+                        boolean is_print = !strcmp (str, "print");
+                        boolean is_punct = !strcmp (str, "punct");
+                        boolean is_space = !strcmp (str, "space");
+                        boolean is_upper = !strcmp (str, "upper");
+                        boolean is_xdigit = !strcmp (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str)) return REG_ECTYPE;
+			
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+			
+                        if (p == pend) return REG_EBRACK;
+			
+                        for (ch = 0; ch < 1 << CHARBITS; ch++)
+                          {
+                            if (   (is_alnum  && isalnum (ch))
+                                || (is_alpha  && isalpha (ch))
+                                || (is_blank  && isblank (ch))
+                                || (is_cntrl  && iscntrl (ch))
+                                || (is_digit  && isdigit (ch))
+                                || (is_graph  && isgraph (ch))
+                                || (is_lower  && islower (ch))
+                                || (is_print  && isprint (ch))
+                                || (is_punct  && ispunct (ch))
+                                || (is_space  && isspace (ch))
+                                || (is_upper  && isupper (ch))
+                                || (is_xdigit && isxdigit (ch)))
+			      {
+				rx_Bitset it =
+				  inverse_translation (rxb, 
+						       validate_inv_tr,
+						       inverse_translate,
+						       translate,
+						       ch);
+				rx_bitset_union (rxb->rx.local_cset_size,
+						 cs, it);
+			      }
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 '[');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 ':');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                  }
+              }
+
+	  finalize_class_and_append:
+	    if (is_inverted)
+	      {
+		rx_bitset_complement (rxb->rx.local_cset_size, cs);
+		if (syntax & RE_HAT_LISTS_NOT_NEWLINE)
+		  RX_bitset_remove (cs, '\n');
+	      }
+	    goto append_node;
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+	  if ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+	    goto handle_interval;
+	  else
+	    goto normal_char;
+
+
+        case '\\':
+          if (p == pend) return REG_EESCAPE;
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              rxb->re_nsub++;
+              regnum++;
+              if (COMPILE_STACK_FULL)
+                { 
+                  ((compile_stack.stack) =
+		   (compile_stack_elt_t *) realloc (compile_stack.stack, ( compile_stack.size << 1) * sizeof (
+													      compile_stack_elt_t)));
+                  if (compile_stack.stack == 0) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, 0);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+
+              /*
+	       * These are the values to restore when we hit end of this
+               * group.  
+	       */
+	      COMPILE_STACK_TOP.top_expression = top_expression;
+	      COMPILE_STACK_TOP.last_expression = last_expression;
+              COMPILE_STACK_TOP.regnum = regnum;
+	      
+              compile_stack.avail++;
+	      
+	      top_expression = last_expression;
+	      break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+            handle_close:
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  return REG_ERPAREN;
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+		struct rexp_node ** inner = top_expression;
+
+                compile_stack.avail--;
+		top_expression = COMPILE_STACK_TOP.top_expression;
+		last_expression = COMPILE_STACK_TOP.last_expression;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		{
+		  int left_se = paramc;
+		  int right_se = paramc + 1;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 2) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (2 * sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  paramc += 2;
+
+		  params[left_se].se = re_se_lparen;
+		  params[left_se].op1 = this_group_regnum;
+		  params[right_se].se = re_se_rparen;
+		  params[right_se].op1 = this_group_regnum;
+		  {
+		    struct rexp_node * left
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)left_se);
+		    struct rexp_node * right
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)right_se);
+		    struct rexp_node * c1
+		      = (*inner
+			 ? rx_mk_r_concat (&rxb->rx, left, *inner) : left);
+		    struct rexp_node * c2
+		      = rx_mk_r_concat (&rxb->rx, c1, right);
+		    if (!(left && right && c1 && c2))
+		      return REG_ESPACE;
+		    *inner = c2;
+		  }
+		}
+		break;
+	      }
+
+            case '|':					/* `\|'.  */
+              if ((syntax & RE_LIMITED_OPS) || (syntax & RE_NO_BK_VBAR))
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+	      {
+		struct rexp_node * alt
+		  = rx_mk_r_alternate (&rxb->rx, *top_expression, 0);
+		if (!alt)
+		  return REG_ESPACE;
+		*top_expression = alt;
+		last_expression = &alt->params.pair.right;
+		{
+		  int sync_se = paramc;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 1) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+
+		  params[sync_se].se = re_se_tv;
+		  {
+		    struct rexp_node * sync
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)sync_se);
+		    struct rexp_node * conc
+		      = rx_mk_r_concat (&rxb->rx, sync, 0);
+
+		    if (!sync || !conc)
+		      return REG_ESPACE;
+
+		    *last_expression = conc;
+		    last_expression = &conc->params.pair.right;
+		  }
+		}
+	      }
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      return REG_EBRACE;
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') return REG_EBRACE;
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (pointless_if_repeated (*last_expression, params))
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      return REG_BADRPT;
+                    else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                      goto unfetch_interval;
+		    /* was: else laststart = b; */
+                  }
+
+                /* If the upper bound is zero, don't want to iterate
+                 * at all.
+		 */
+                 if (upper_bound == 0)
+		   {
+		     if (*last_expression)
+		       {
+			 rx_free_rexp (&rxb->rx, *last_expression);
+			 *last_expression = 0;
+		       }
+		   }
+		else
+		  /* Otherwise, we have a nontrivial interval. */
+		  {
+		    int iter_se = paramc;
+		    int end_se = paramc + 1;
+		    params = (params
+			      ? ((struct re_se_params *)
+				 realloc (params,
+					  sizeof (*params) * (2 + paramc)))
+			      : ((struct re_se_params *)
+				 malloc (2 * sizeof (*params))));
+		    if (!params)
+		      return REG_ESPACE;
+		    paramc += 2;
+		    params [iter_se].se = re_se_iter;
+		    params [iter_se].op1 = lower_bound;
+		    params[iter_se].op2 = upper_bound;
+
+		    params[end_se].se = re_se_end_iter;
+		    params[end_se].op1 = lower_bound;
+		    params[end_se].op2 = upper_bound;
+		    {
+		      struct rexp_node * push0
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_push0);
+		      struct rexp_node * start_one_iter
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)iter_se);
+		      struct rexp_node * phase1
+			= rx_mk_r_concat (&rxb->rx, start_one_iter,
+					  *last_expression);
+		      struct rexp_node * pushback
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_pushback);
+		      rx_Bitset cs = rx_cset (&rxb->rx);
+		      struct rexp_node * lit_t
+			= rx_mk_r_cset (&rxb->rx, cs);
+		      struct rexp_node * phase2
+			= rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		      struct rexp_node * loop
+			= rx_mk_r_2phase_star (&rxb->rx, phase1, phase2);
+		      struct rexp_node * push_n_loop
+			= rx_mk_r_concat (&rxb->rx, push0, loop);
+		      struct rexp_node * final_test
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)end_se);
+		      struct rexp_node * full_exp
+			= rx_mk_r_concat (&rxb->rx, push_n_loop, final_test);
+
+		      if (!(push0 && start_one_iter && phase1
+			    && pushback && lit_t && phase2
+			    && loop && push_n_loop && final_test && full_exp))
+			return REG_ESPACE;
+
+		      RX_bitset_enjoin(cs, 't');
+
+		      *last_expression = full_exp;
+		    }
+		  }
+                beg_interval = 0;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               p = beg_interval;
+               beg_interval = 0;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+	      side = (rx_side_effect)rx_se_at_dot;
+	      goto add_side_effect;
+              break;
+
+            case 's':
+	    case 'S':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * set = rx_mk_r_cset (&rxb->rx, cs);
+		if (!(cs && set))
+		  return REG_ESPACE;
+		if (c == 'S')
+		  rx_bitset_universe (rxb->rx.local_cset_size, cs);
+
+		PATFETCH (c);
+		{
+		  int x;
+		  enum syntaxcode code = syntax_spec_code [c];
+		  for (x = 0; x < 256; ++x)
+		    {
+		      
+		      if (SYNTAX (x) == code)
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, validate_inv_tr,
+						 inverse_translate,
+						 translate, x);
+			  rx_bitset_xor (rxb->rx.local_cset_size, cs, it);
+			}
+		    }
+		}
+		append = set;
+		goto append_node;
+	      }
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+            case 'W':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * n = (cs ? rx_mk_r_cset (&rxb->rx, cs) : 0);
+		if (!(cs && n))
+		  return REG_ESPACE;
+		if (c == 'W')
+		  rx_bitset_universe (rxb->rx.local_cset_size ,cs);
+		{
+		  int x;
+		  for (x = rxb->rx.local_cset_size - 1; x > 0; --x)
+		    if (SYNTAX(x) & Sword)
+		      RX_bitset_toggle (cs, x);
+		}
+		append = n;
+		goto append_node;
+	      }
+              break;
+
+/* With a little extra work, some of these side effects could be optimized
+ * away (basicly by looking at what we already know about the surrounding
+ * chars).  
+ */
+            case '<':
+	      side = (rx_side_effect)re_se_wordbeg;
+	      goto add_side_effect;
+              break;
+
+            case '>':
+              side = (rx_side_effect)re_se_wordend;
+	      goto add_side_effect;
+              break;
+
+            case 'b':
+              side = (rx_side_effect)re_se_wordbound;
+	      goto add_side_effect;
+              break;
+
+            case 'B':
+              side = (rx_side_effect)re_se_notwordbound;
+	      goto add_side_effect;
+              break;
+
+            case '`':
+	      side = (rx_side_effect)re_se_begbuf;
+	      goto add_side_effect;
+	      break;
+	      
+            case '\'':
+	      side = (rx_side_effect)re_se_endbuf;
+	      goto add_side_effect;
+              break;
+
+	    add_side_effect:
+	      {
+		struct rexp_node * se
+		  = rx_mk_r_side_effect (&rxb->rx, side);
+		if (!se)
+		  return REG_ESPACE;
+		append = se;
+		goto append_node;
+	      }
+	      break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                return REG_ESUBREG;
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+		return REG_ESUBREG;
+
+	      {
+		int backref_se = paramc;
+		params = (params
+			  ? ((struct re_se_params *)
+			     realloc (params,
+				      sizeof (*params) * (1 + paramc)))
+			  : ((struct re_se_params *)
+			     malloc (sizeof (*params))));
+		if (!params)
+		  return REG_ESPACE;
+		++paramc;
+		params[backref_se].se = re_se_backref;
+		params[backref_se].op1 = c1;
+		side = (rx_side_effect)backref_se;
+		goto add_side_effect;
+	      }
+              break;
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	    {
+	      rx_Bitset cs = rx_cset(&rxb->rx);
+	      struct rexp_node * match = rx_mk_r_cset (&rxb->rx, cs);
+	      rx_Bitset it;
+	      if (!(cs && match))
+		return REG_ESPACE;
+	      it = inverse_translation (rxb, validate_inv_tr,
+					inverse_translate, translate, c);
+	      rx_bitset_union (CHAR_SET_SIZE, cs, it);
+	      append = match;
+
+	    append_node:
+	      /* This genericly appends the rexp APPEND to *LAST_EXPRESSION
+	       * and then parses the next character normally.
+	       */
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, append);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+	      else
+		*last_expression = append;
+	    }
+	} /* switch (c) */
+    } /* while p != pend */
+
+  
+  {
+    int win_se = paramc;
+    params = (params
+	      ? ((struct re_se_params *)
+		 realloc (params,
+			  sizeof (*params) * (1 + paramc)))
+	      : ((struct re_se_params *)
+		 malloc (sizeof (*params))));
+    if (!params)
+      return REG_ESPACE;
+    ++paramc;
+    params[win_se].se = re_se_win;
+    {
+      struct rexp_node * se
+	= rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)win_se);
+      struct rexp_node * concat
+	= rx_mk_r_concat (&rxb->rx, rexp, se);
+      if (!(se && concat))
+	return REG_ESPACE;
+      rexp = concat;
+    }
+  }
+
+
+  /* Through the pattern now.  */
+
+  if (!COMPILE_STACK_EMPTY) 
+    return REG_EPAREN;
+
+      free (compile_stack.stack);
+
+  orig_rexp = rexp;
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      dbug_rxb = rxb;
+      fputs ("\n\nCompiling ", stdout);
+      fwrite (pattern, 1, size, stdout);
+      fputs (":\n", stdout);
+      rxb->se_params = params;
+      print_rexp (&rxb->rx, orig_rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    rx_Bitset cs = rx_cset(&rxb->rx);
+    rx_Bitset cs2 = rx_cset(&rxb->rx);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+    char * se_map = (char *) _alloca (paramc);
+#else
+    char * se_map = (char *) alloca (paramc);
+#endif
+    struct rexp_node * new_rexp = 0;
+
+
+    bzero (se_map, paramc);
+    find_backrefs (se_map, rexp, params);
+    fewer_side_effects =
+      remove_unecessary_side_effects (&rxb->rx, se_map,
+				      rx_copy_rexp (&rxb->rx, rexp), params);
+
+    speed_up_alt (&rxb->rx, rexp, 0);
+    speed_up_alt (&rxb->rx, fewer_side_effects, 1);
+
+    {
+      char * syntax_parens = rxb->syntax_parens;
+      if (syntax_parens == (char *)0x1)
+	rexp = remove_unecessary_side_effects
+	  (&rxb->rx, se_map, rexp, params);
+      else if (syntax_parens)
+	{
+	  int x;
+	  for (x = 0; x < paramc; ++x)
+	    if ((   (params[x].se == re_se_lparen)
+		 || (params[x].se == re_se_rparen))
+		&& (!syntax_parens [params[x].op1]))
+	      se_map [x] = 1;
+	  rexp = remove_unecessary_side_effects
+	    (&rxb->rx, se_map, rexp, params);
+	}
+    }
+
+    /* At least one more optimization would be nice to have here but i ran out 
+     * of time.  The idea would be to delay side effects.  
+     * For examle, `(abc)' is the same thing as `abc()' except that the
+     * left paren is offset by 3 (which we know at compile time).
+     * (In this comment, write that second pattern `abc(:3:)' 
+     * where `(:3:' is a syntactic unit.)
+     *
+     * Trickier:  `(abc|defg)'  is the same as `(abc(:3:|defg(:4:))'
+     * (The paren nesting may be hard to follow -- that's an alternation
+     *	of `abc(:3:' and `defg(:4:' inside (purely syntactic) parens
+     *  followed by the closing paren from the original expression.)
+     *
+     * Neither the expression tree representation nor the the nfa make
+     * this very easy to write. :(
+     */
+
+  /* What we compile is different than what the parser returns.
+   * Suppose the parser returns expression R.
+   * Let R' be R with unnecessary register assignments removed 
+   * (see REMOVE_UNECESSARY_SIDE_EFFECTS, above).
+   *
+   * What we will compile is the expression:
+   *
+   *    m{try}R{win}\|s{try}R'{win}
+   *
+   * {try} and {win} denote side effect epsilons (see EXPLORE_FUTURE).
+   * 
+   * When trying a match, we insert an `m' at the beginning of the 
+   * string if the user wants registers to be filled, `s' if not.
+   */
+    new_rexp =
+      rx_mk_r_alternate
+	(&rxb->rx,
+	 rx_mk_r_concat (&rxb->rx, rx_mk_r_cset (&rxb->rx, cs2), rexp),
+	 rx_mk_r_concat (&rxb->rx,
+			 rx_mk_r_cset (&rxb->rx, cs), fewer_side_effects));
+
+    if (!(new_rexp && cs && cs2))
+      return REG_ESPACE;
+    RX_bitset_enjoin (cs2, '\0'); /* prefixed to the rexp used for matching. */
+    RX_bitset_enjoin (cs, '\1'); /* prefixed to the rexp used for searching. */
+    rexp = new_rexp;
+  }
+
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      fputs ("\n...which is compiled as:\n", stdout);
+      print_rexp (&rxb->rx, rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    struct rx_nfa_state *start = 0;
+    struct rx_nfa_state *end = 0;
+
+    if (!rx_build_nfa (&rxb->rx, rexp, &start, &end))
+      return REG_ESPACE;	/*  */
+    else
+      {
+	void * mem = (void *)rxb->buffer;
+	unsigned long size = rxb->allocated;
+	int start_id;
+	char * perm_mem;
+	int iterator_size = paramc * sizeof (params[0]);
+
+	end->is_final = 1;
+	start->is_start = 1;
+	rx_name_nfa_states (&rxb->rx);
+	start_id = start->id;
+#ifdef RX_DEBUG
+	if (rx_debug_compile)
+	  {
+	    fputs ("...giving the NFA: \n", stdout);
+	    dbug_rxb = rxb;
+	    print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	  }
+#endif
+	if (!rx_eclose_nfa (&rxb->rx))
+	  return REG_ESPACE;
+	else
+	  {
+	    rx_delete_epsilon_transitions (&rxb->rx);
+	    
+	    /* For compatability reasons, we need to shove the
+	     * compiled nfa into one chunk of malloced memory.
+	     */
+	    rxb->rx.reserved = (   sizeof (params[0]) * paramc
+				+  rx_sizeof_bitset (rxb->rx.local_cset_size));
+#ifdef RX_DEBUG
+	    if (rx_debug_compile)
+	      {
+		dbug_rxb = rxb;
+		fputs ("...which cooks down (uncompactified) to: \n", stdout);
+		print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	      }
+#endif
+	    if (!rx_compactify_nfa (&rxb->rx, &mem, &size))
+	      return REG_ESPACE;
+	    rxb->buffer = mem;
+	    rxb->allocated = size;
+	    rxb->rx.buffer = mem;
+	    rxb->rx.allocated = size;
+	    perm_mem = ((char *)rxb->rx.buffer
+			+ rxb->rx.allocated - rxb->rx.reserved);
+	    rxb->se_params = ((struct re_se_params *)perm_mem);
+	    bcopy (params, rxb->se_params, iterator_size);
+	    perm_mem += iterator_size;
+	    rxb->fastset = (rx_Bitset) perm_mem;
+	    rxb->start = rx_id_to_nfa_state (&rxb->rx, start_id);
+	  }
+	rx_bitset_null (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->can_match_empty = compute_fastset (rxb, orig_rexp);
+	rxb->match_regs_on_stack =
+	  registers_on_stack (rxb, orig_rexp, 0, params); 
+	rxb->search_regs_on_stack =
+	  registers_on_stack (rxb, fewer_side_effects, 0, params);
+	if (rxb->can_match_empty)
+	  rx_bitset_universe (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->is_anchored = is_anchored (orig_rexp, (rx_side_effect) re_se_hat);
+	rxb->begbuf_only = is_anchored (orig_rexp,
+					(rx_side_effect) re_se_begbuf);
+      }
+    rx_free_rexp (&rxb->rx, rexp);
+    if (params)
+      free (params);
+#ifdef RX_DEBUG
+    if (rx_debug_compile)
+      {
+	dbug_rxb = rxb;
+	fputs ("...which cooks down to: \n", stdout);
+	print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+      }
+#endif
+  }
+  return REG_NOERROR;
+}
+
+
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.  */
+
+__const__ char * rx_error_msg[] =
+{ 0,						/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+};
+
+
+
+
+
+char rx_slowmap [256] =
+{
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+};
+
+#ifdef __STDC__
+RX_DECL void
+rx_blow_up_fastmap (struct re_pattern_buffer * rxb)
+#else
+RX_DECL void
+rx_blow_up_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  int x;
+  for (x = 0; x < 256; ++x)	/* &&&& 3.6 % */
+    rxb->fastmap [x] = !!RX_bitset_member (rxb->fastset, x);
+  rxb->fastmap_accurate = 1;
+}
+
+
+
+
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#define RE_SEARCH_2_FN	inner_re_search_2
+#define RE_S2_QUAL static
+#else
+#define RE_SEARCH_2_FN	re_search_2
+#define RE_S2_QUAL 
+#endif
+
+struct re_search_2_closure
+{
+  __const__ char * string1;
+  int size1;
+  __const__ char * string2;
+  int size2;
+};
+
+
+static __inline__ enum rx_get_burst_return
+re_search_2_get_burst (pos, vclosure, stop)
+     struct rx_string_position * pos;
+     void * vclosure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)vclosure;
+  if (!closure->string2)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      if ((inset < -1) || (inset > closure->size1))
+	return rx_get_burst_no_more;
+      else
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+  else if (!closure->string1)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      pos->pos = (__const__ unsigned char *) closure->string2 + inset;
+      pos->string = (__const__ unsigned char *) closure->string2;
+      pos->size = closure->size2;
+      pos->end = ((__const__ unsigned char *)
+		  MIN(closure->string2 + closure->size2,
+		      closure->string2 + stop));
+      pos->offset = 0;
+      return ((pos->pos < pos->end)
+	      ? rx_get_burst_ok
+	      :  rx_get_burst_no_more);
+    }
+  else
+    {
+      int inset;
+
+      inset = pos->pos - pos->string + pos->offset;
+      if (inset < closure->size1)
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return rx_get_burst_ok;
+	}
+      else
+	{
+	  pos->pos = ((__const__ unsigned char *)
+		      closure->string2 + inset - closure->size1);
+	  pos->string = (__const__ unsigned char *) closure->string2;
+	  pos->size = closure->size2;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string2 + closure->size2,
+			  closure->string2 + stop - closure->size1));
+	  pos->offset = closure->size1;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+}
+
+
+static __inline__ enum rx_back_check_return
+re_search_2_back_check (pos, lparen, rparen, translate, vclosure, stop)
+     struct rx_string_position * pos;
+     int lparen;
+     int rparen;
+     unsigned char * translate;
+     void * vclosure;
+     int stop;
+{
+  struct rx_string_position there;
+  struct rx_string_position past;
+
+  there = *pos;
+  there.pos = there.string + lparen - there.offset;
+  re_search_2_get_burst (&there, vclosure, stop);
+
+  past = *pos;
+  past.pos = past.string + rparen - there.offset;
+  re_search_2_get_burst (&past, vclosure, stop);
+
+  ++pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+
+  while (   (there.pos != past.pos)
+	 && (pos->pos != pos->end))
+    if (TRANSLATE(*there.pos) != TRANSLATE(*pos->pos))
+      return rx_back_check_fail;
+    else
+      {
+	++there.pos;
+	++pos->pos;
+	if (there.pos == there.end)
+	  re_search_2_get_burst (&there, vclosure, stop);
+	if (pos->pos == pos->end)
+	  re_search_2_get_burst (pos, vclosure, stop);
+      }
+
+  if (there.pos != past.pos)
+    return rx_back_check_fail;
+  --pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+  return rx_back_check_pass;
+}
+
+static __inline__ int
+re_search_2_fetch_char (pos, offset, app_closure, stop)
+     struct rx_string_position * pos;
+     int offset;
+     void * app_closure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)app_closure;
+  if (offset == 0)
+    {
+      if (pos->pos >= pos->string)
+	return *pos->pos;
+      else
+	{
+	  if (   (pos->string == (__const__ unsigned char *) closure->string2)
+	      && (closure->string1)
+	      && (closure->size1))
+	    return closure->string1[closure->size1 - 1];
+	  else
+	    return 0;		/* sure, why not. */
+	}
+    }
+  if (pos->pos == pos->end)
+    return *closure->string2;
+  else
+    return pos->pos[1];
+}
+     
+
+#ifdef __STDC__
+RE_S2_QUAL int
+RE_SEARCH_2_FN (struct re_pattern_buffer *rxb,
+		__const__ char * string1, int size1,
+		__const__ char * string2, int size2,
+		int startpos, int range,
+		struct re_registers *regs,
+		int stop)
+#else
+RE_S2_QUAL int
+RE_SEARCH_2_FN (rxb,
+		string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int answer;
+  struct re_search_2_closure closure;
+  closure.string1 = string1;
+  closure.size1 = size1;
+  closure.string2 = string2;
+  closure.size2 = size2;
+  answer = rx_search (rxb, startpos, range, stop, size1 + size2,
+		      re_search_2_get_burst,
+		      re_search_2_back_check,
+		      re_search_2_fetch_char,
+		      (void *)&closure,
+		      regs,
+		      0,
+		      0);
+  switch (answer)
+    {
+    case rx_search_continuation:
+      abort ();
+    case rx_search_error:
+      return -2;
+    case rx_search_soft_fail:
+    case rx_search_fail:
+      return -1;
+    default:
+      return answer;
+    }
+}
+
+/* Export rx_search to callers outside this file.  */
+
+int
+re_rx_search (rxb, startpos, range, stop, total_size,
+	      get_burst, back_check, fetch_char,
+	      app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+{
+  return rx_search (rxb, startpos, range, stop, total_size,
+		    get_burst, back_check, fetch_char, app_closure,
+		    regs, resume_state, save_state);
+}
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#ifdef __STDC__
+int
+re_search_2 (struct re_pattern_buffer *rxb,
+	     __const__ char * string1, int size1,
+	     __const__ char * string2, int size2,
+	     int startpos, int range,
+	     struct re_registers *regs,
+	     int stop)
+#else
+int
+re_search_2 (rxb, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int ret;
+  ret = inner_re_search_2 (rxb, string1, size1, string2, size2, startpos,
+			   range, regs, stop);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return ret;
+}
+#endif
+
+
+/* Like re_search_2, above, but only one string is specified, and
+ * doesn't let you say where to stop matching.
+ */
+
+#ifdef __STDC__
+int
+re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+	   int size, int startpos, int range,
+	   struct re_registers *regs)
+#else
+int
+re_search (rxb, string, size, startpos, range, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string;
+     int size;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+#endif
+{
+  return re_search_2 (rxb, 0, 0, string, size, startpos, range, regs, size);
+}
+
+#ifdef __STDC__
+int
+re_match_2 (struct re_pattern_buffer * rxb,
+	    __const__ char * string1, int size1,
+	    __const__ char * string2, int size2,
+	    int pos, struct re_registers *regs, int stop)
+#else
+int
+re_match_2 (rxb, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  struct re_registers some_regs;
+  regoff_t start;
+  regoff_t end;
+  int srch;
+  int save = rxb->regs_allocated;
+  struct re_registers * regs_to_pass = regs;
+
+  if (!regs)
+    {
+      some_regs.start = &start;
+      some_regs.end = &end;
+      some_regs.num_regs = 1;
+      regs_to_pass = &some_regs;
+      rxb->regs_allocated = REGS_FIXED;
+    }
+
+  srch = re_search_2 (rxb, string1, size1, string2, size2,
+		      pos, 1, regs_to_pass, stop);
+  if (regs_to_pass != regs)
+    rxb->regs_allocated = save;
+  if (srch < 0)
+    return srch;
+  return regs_to_pass->end[0] - regs_to_pass->start[0];
+}
+
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+#ifdef __STDC__
+int
+re_match (struct re_pattern_buffer * rxb,
+	  __const__ char * string,
+	  int size, int pos,
+	  struct re_registers *regs)
+#else
+int
+re_match (rxb, string, size, pos, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char *string;
+     int size;
+     int pos;
+     struct re_registers *regs;
+#endif
+{
+  return re_match_2 (rxb, string, size, 0, 0, pos, regs, size);
+}
+
+
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+reg_syntax_t re_syntax_options = RE_SYNTAX_EMACS;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+#ifdef __STDC__
+reg_syntax_t
+re_set_syntax (reg_syntax_t syntax)
+#else
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+#endif
+{
+  reg_syntax_t ret = re_syntax_options;
+
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+#ifdef __STDC__
+void
+re_set_registers (struct re_pattern_buffer *bufp,
+		  struct re_registers *regs,
+		  unsigned num_regs,
+		  regoff_t * starts, regoff_t * ends)
+#else
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+     struct re_pattern_buffer *bufp;
+     struct re_registers *regs;
+     unsigned num_regs;
+     regoff_t * starts;
+     regoff_t * ends;
+#endif
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t) 0;
+    }
+}
+
+
+
+
+
+#ifdef __STDC__
+static int 
+cplx_se_sublist_len (struct rx_se_list * list)
+#else
+static int 
+cplx_se_sublist_len (list)
+     struct rx_se_list * list;
+#endif
+{
+  int x = 0;
+  while (list)
+    {
+      if ((long)list->car >= 0)
+	++x;
+      list = list->cdr;
+    }
+  return x;
+}
+
+
+/* For rx->se_list_cmp */
+
+#ifdef __STDC__
+static int 
+posix_se_list_order (struct rx * rx,
+		     struct rx_se_list * a, struct rx_se_list * b)
+#else
+static int 
+posix_se_list_order (rx, a, b)
+     struct rx * rx;
+     struct rx_se_list * a;
+     struct rx_se_list * b;
+#endif
+{
+  int al = cplx_se_sublist_len (a);
+  int bl = cplx_se_sublist_len (b);
+
+  if (!al && !bl)
+    return ((a == b)
+	    ? 0
+	    : ((a < b) ? -1 : 1));
+  
+  else if (!al)
+    return -1;
+
+  else if (!bl)
+    return 1;
+
+  else
+    {
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+      rx_side_effect * av = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (bl + 1)));
+#else
+      rx_side_effect * av = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (bl + 1)));
+#endif
+      struct rx_se_list * ap = a;
+      struct rx_se_list * bp = b;
+      int ai, bi;
+      
+      for (ai = al - 1; ai >= 0; --ai)
+	{
+	  while ((long)ap->car < 0)
+	    ap = ap->cdr;
+	  av[ai] = ap->car;
+	  ap = ap->cdr;
+	}
+      av[al] = (rx_side_effect)-2;
+      for (bi = bl - 1; bi >= 0; --bi)
+	{
+	  while ((long)bp->car < 0)
+	    bp = bp->cdr;
+	  bv[bi] = bp->car;
+	  bp = bp->cdr;
+	}
+      bv[bl] = (rx_side_effect)-1;
+
+      {
+	int ret;
+	int x = 0;
+	while (av[x] == bv[x])
+	  ++x;
+ 	ret = (((unsigned *)(av[x]) < (unsigned *)(bv[x])) ? -1 : 1);
+	return ret;
+      }
+    }
+}
+
+
+
+
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in RXB.
+   Returns 0 if the pattern was valid, otherwise an error string.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in RXB on entry.
+
+   We call rx_compile to do the actual compilation.  */
+
+#ifdef __STDC__
+__const__ char *
+re_compile_pattern (__const__ char *pattern,
+		    int length,
+		    struct re_pattern_buffer * rxb)
+#else
+__const__ char *
+re_compile_pattern (pattern, length, rxb)
+     __const__ char *pattern;
+     int length;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  reg_errcode_t ret;
+
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  rxb->regs_allocated = REGS_UNALLOCATED;
+
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  rxb->no_sub = 0;
+
+  rxb->rx.local_cset_size = 256;
+
+  /* Match anchors at newline.  */
+  rxb->newline_anchor = 1;
+ 
+  rxb->re_nsub = 0;
+  rxb->start = 0;
+  rxb->se_params = 0;
+  rxb->rx.nodec = 0;
+  rxb->rx.epsnodec = 0;
+  rxb->rx.instruction_table = 0;
+  rxb->rx.nfa_states = 0;
+  rxb->rx.se_list_cmp = posix_se_list_order;
+  rxb->rx.start_set = 0;
+
+  ret = rx_compile (pattern, length, re_syntax_options, rxb);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return rx_error_msg[(int) ret];
+}
+
+
+
+#ifdef __STDC__
+int
+re_compile_fastmap (struct re_pattern_buffer * rxb)
+#else
+int
+re_compile_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  rx_blow_up_fastmap (rxb);
+  return 0;
+}
+
+
+
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them if this is an Emacs or POSIX compilation.  */
+
+#if (!defined (emacs) && !defined (_POSIX_SOURCE)) || defined(USE_BSD_REGEX)
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer rx_comp_buf;
+
+#ifdef __STDC__
+char *
+re_comp (__const__ char *s)
+#else
+char *
+re_comp (s)
+    __const__ char *s;
+#endif
+{
+  reg_errcode_t ret;
+
+  if (!s || (*s == '\0'))
+    {
+      if (!rx_comp_buf.buffer)
+	return "No previous regular expression";
+      return 0;
+    }
+
+  if (!rx_comp_buf.fastmap)
+    {
+      rx_comp_buf.fastmap = (char *) malloc (1 << CHARBITS);
+      if (!rx_comp_buf.fastmap)
+	return "Memory exhausted";
+    }
+
+  /* Since `rx_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  rx_comp_buf.newline_anchor = 1;
+
+  rx_comp_buf.re_nsub = 0;
+  rx_comp_buf.start = 0;
+  rx_comp_buf.se_params = 0;
+  rx_comp_buf.rx.nodec = 0;
+  rx_comp_buf.rx.epsnodec = 0;
+  rx_comp_buf.rx.instruction_table = 0;
+  rx_comp_buf.rx.nfa_states = 0;
+  rx_comp_buf.rx.start = 0;
+  rx_comp_buf.rx.se_list_cmp = posix_se_list_order;
+  rx_comp_buf.rx.local_cset_size = 256;
+
+  ret = rx_compile (s, strlen (s), re_syntax_options, &rx_comp_buf);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* Yes, we're discarding `__const__' here.  */
+  return (char *) rx_error_msg[(int) ret];
+}
+
+
+#ifdef __STDC__
+int
+re_exec (__const__ char *s)
+#else
+int
+re_exec (s)
+    __const__ char *s;
+#endif
+{
+  __const__ int len = strlen (s);
+  return
+    0 <= re_search (&rx_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* not emacs and not _POSIX_SOURCE */
+
+
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#if !defined(emacs)
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+
+#ifdef __STDC__
+int
+regcomp (regex_t * preg, __const__ char * pattern, int cflags)
+#else
+int
+regcomp (preg, pattern, cflags)
+    regex_t * preg;
+    __const__ char * pattern;
+    int cflags;
+#endif
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = cflags & REG_EXTENDED ? RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->fastmap = malloc (256);
+  if (!preg->fastmap)
+    return REG_ESPACE;
+  preg->fastmap_accurate = 0;
+
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+
+      preg->translate = (unsigned char *) malloc (256);
+      if (!preg->translate)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = isupper (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = 0;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we
+     can use strlen here in compiling the pattern.  */
+  preg->re_nsub = 0;
+  preg->start = 0;
+  preg->se_params = 0;
+  preg->syntax_parens = 0;
+  preg->rx.nodec = 0;
+  preg->rx.epsnodec = 0;
+  preg->rx.instruction_table = 0;
+  preg->rx.nfa_states = 0;
+  preg->rx.local_cset_size = 256;
+  preg->rx.start = 0;
+  preg->rx.se_list_cmp = posix_se_list_order;
+  preg->rx.start_set = 0;
+  ret = rx_compile (pattern, strlen (pattern), syntax, preg);
+
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+#ifdef __STDC__
+int
+regexec (__const__ regex_t *preg, __const__ char *string,
+	 size_t nmatch, regmatch_t pmatch[],
+	 int eflags)
+#else
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    __const__ regex_t *preg;
+    __const__ char *string;
+    size_t nmatch;
+    regmatch_t pmatch[];
+    int eflags;
+#endif
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+
+  /* The user has told us exactly how many registers to return
+   * information about, via `nmatch'.  We have to pass that on to the
+   * matching routines.
+   */
+  private_preg.regs_allocated = REGS_FIXED;
+
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      regs.end =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      if (regs.start == 0 || regs.end == 0)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg,
+		   string, len,
+                   /* start: */ 0,
+		   /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   */
+
+#ifdef __STDC__
+size_t
+regerror (int errcode, __const__ regex_t *preg,
+	  char *errbuf, size_t errbuf_size)
+#else
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    __const__ regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+#endif
+{
+  __const__ char *msg
+    = rx_error_msg[errcode] == 0 ? "Success" : rx_error_msg[errcode];
+  size_t msg_size = strlen (msg) + 1; /* Includes the 0.  */
+
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+#ifdef __STDC__
+void
+regfree (regex_t *preg)
+#else
+void
+regfree (preg)
+    regex_t *preg;
+#endif
+{
+  if (preg->buffer != 0)
+    free (preg->buffer);
+  preg->buffer = 0;
+  preg->allocated = 0;
+
+  if (preg->fastmap != 0)
+    free (preg->fastmap);
+  preg->fastmap = 0;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != 0)
+    free (preg->translate);
+  preg->translate = 0;
+}
+
+#endif /* not emacs  */
+
+
+
+
+
Index: /main/tags/3.01/indexers/mg/lib/rx.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/rx.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/rx.h	(revision 21112)
@@ -0,0 +1,3755 @@
+#if !defined(RXH) || defined(RX_WANT_SE_DEFS)
+#define RXH
+
+/*	Copyright (C) 1992, 1993 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+/*  t. lord	Wed Sep 23 18:20:57 1992	*/
+
+
+
+
+
+
+
+
+
+#ifndef RX_WANT_SE_DEFS
+
+/* This page: Bitsets */
+
+#ifndef RX_subset
+typedef unsigned int RX_subset;
+#define RX_subset_bits	(32)
+#define RX_subset_mask	(RX_subset_bits - 1)
+#endif
+
+typedef RX_subset * rx_Bitset;
+
+#ifdef __STDC__
+typedef void (*rx_bitset_iterator) (void *, int member_index);
+#else
+typedef void (*rx_bitset_iterator) ();
+#endif
+
+#define rx_bitset_subset(N)  ((N) / RX_subset_bits)
+#define rx_bitset_subset_val(B,N)  ((B)[rx_bitset_subset(N)])
+#define RX_bitset_access(B,N,OP) \
+  ((B)[rx_bitset_subset(N)] OP rx_subset_singletons[(N) & RX_subset_mask])
+#define RX_bitset_member(B,N)   RX_bitset_access(B, N, &)
+#define RX_bitset_enjoin(B,N)   RX_bitset_access(B, N, |=)
+#define RX_bitset_remove(B,N)   RX_bitset_access(B, N, &= ~)
+#define RX_bitset_toggle(B,N)   RX_bitset_access(B, N, ^= )
+#define rx_bitset_numb_subsets(N) (((N) + RX_subset_bits - 1) / RX_subset_bits)
+#define rx_sizeof_bitset(N)	(rx_bitset_numb_subsets(N) * sizeof(RX_subset))
+
+
+
+
+/* This page: Splay trees. */
+
+#ifdef __STDC__
+typedef int (*rx_sp_comparer) (void * a, void * b);
+#else
+typedef int (*rx_sp_comparer) ();
+#endif
+
+struct rx_sp_node 
+{
+  void * key;
+  void * data;
+  struct rx_sp_node * kids[2];
+};
+
+#ifdef __STDC__
+typedef void (*rx_sp_key_data_freer) (struct rx_sp_node *);
+#else
+typedef void (*rx_sp_key_data_freer) ();
+#endif
+
+
+
+/* giant inflatable hash trees */
+
+struct rx_hash_item
+{
+  struct rx_hash_item * next_same_hash;
+  struct rx_hash * table;
+  unsigned long hash;
+  void * data;
+  void * binding;
+};
+
+struct rx_hash
+{
+  struct rx_hash * parent;
+  int refs;
+  struct rx_hash * children[13];
+  struct rx_hash_item * buckets [13];
+  int bucket_size [13];
+};
+
+struct rx_hash_rules;
+
+#ifdef __STDC__
+/* should return like == */
+typedef int (*rx_hash_eq)(void *, void *);
+typedef struct rx_hash * (*rx_alloc_hash)(struct rx_hash_rules *);
+typedef void (*rx_free_hash)(struct rx_hash *,
+			    struct rx_hash_rules *);
+typedef struct rx_hash_item * (*rx_alloc_hash_item)(struct rx_hash_rules *,
+						    void *);
+typedef void (*rx_free_hash_item)(struct rx_hash_item *,
+				 struct rx_hash_rules *);
+#else
+typedef int (*rx_hash_eq)();
+typedef struct rx_hash * (*rx_alloc_hash)();
+typedef void (*rx_free_hash)();
+typedef struct rx_hash_item * (*rx_alloc_hash_item)();
+typedef void (*rx_free_hash_item)();
+#endif
+
+struct rx_hash_rules
+{
+  rx_hash_eq eq;
+  rx_alloc_hash hash_alloc;
+  rx_free_hash free_hash;
+  rx_alloc_hash_item hash_item_alloc;
+  rx_free_hash_item free_hash_item;
+};
+
+
+
+/* Forward declarations */
+
+struct rx_cache;
+struct rx_superset;
+struct rx;
+struct rx_se_list;
+
+
+
+
+/* 
+ * GLOSSARY
+ *
+ * regexp
+ * regular expression
+ * expression
+ * pattern - a `regular' expression.  The expression
+ *       need not be formally regular -- it can contain
+ *       constructs that don't correspond to purely regular
+ *       expressions.
+ *
+ * buffer
+ * string - the string (or strings) being searched or matched.
+ *
+ * pattern buffer - a structure of type `struct re_pattern_buffer'
+ *       This in turn contains a `struct rx', which holds the
+ *       NFA compiled from a pattern, as well as some of the state
+ *       of a matcher using the pattern.
+ *
+ * NFA - nondeterministic finite automata.  Some people
+ *       use this term to a member of the class of 
+ *       regular automata (those corresponding to a regular
+ *       language).  However, in this code, the meaning is
+ *       more general.  The automata used by Rx are comperable
+ *       in power to what are usually called `push down automata'.
+ *
+ *       Two NFA are built by rx for every pattern.  One is built
+ *       by the compiler.  The other is built from the first, on
+ *       the fly, by the matcher.  The latter is called the `superstate
+ *       NFA' because its states correspond to sets of states from
+ *       the first NFA.  (Joe Keane gets credit for the name
+ *       `superstate NFA').
+ *
+ * NFA edges
+ * epsilon edges
+ * side-effect edges - The NFA compiled from a pattern can have three
+ *       kinds of edges.  Epsilon edges can be taken freely anytime
+ *       their source state is reached.  Character set edges can be
+ *       taken when their source state is reached and when the next 
+ *       character in the buffer is a member of the set.  Side effect
+ *       edges imply a transition that can only be taken after the
+ *       indicated side effect has been successfully accomplished.
+ *       Some examples of side effects are:
+ *
+ *		Storing the current match position to record the
+ *              location of a parentesized subexpression.
+ *
+ *              Advancing the matcher over N characters if they
+ *              match the N characters previously matched by a 
+ *              parentesized subexpression.
+ *
+ *       Both of those kinds of edges occur in the NFA generated
+ *       by the pattern:  \(.\)\1
+ *
+ *       Epsilon and side effect edges are similar.  Unfortunately,
+ *       some of the code uses the name `epsilon edge' to mean
+ *       both epsilon and side effect edges.  For example,  the
+ *       function has_non_idempotent_epsilon_path computes the existance
+ *       of a non-trivial path containing only a mix of epsilon and
+ *       side effect edges.  In that case `nonidempotent epsilon' is being
+ *       used to mean `side effect'.
+ */
+
+
+
+
+
+
+/* LOW LEVEL PATTERN BUFFERS */
+
+/* Suppose that from some NFA state, more than one path through
+ * side-effect edges is possible.  In what order should the paths
+ * be tried?  A function of type rx_se_list_order answers that
+ * question.  It compares two lists of side effects, and says
+ * which list comes first.
+ */
+ 
+#ifdef __STDC__
+typedef int (*rx_se_list_order) (struct rx *,
+				 struct rx_se_list *, 
+				 struct rx_se_list *);
+#else
+typedef int (*rx_se_list_order) ();
+#endif
+
+
+
+/* Struct RX holds a compiled regular expression - that is, an nfa
+ * ready to be converted on demand to a more efficient superstate nfa.
+ * This is for the low level interface.  The high-level interfaces enclose
+ * this in a `struct re_pattern_buffer'.  
+ */
+struct rx
+{
+  /* The compiler assigns a unique id to every pattern.
+   * Like sequence numbers in X, there is a subtle bug here
+   * if you use Rx in a system that runs for a long time.
+   * But, because of the way the caches work out, it is almost
+   * impossible to trigger the Rx version of this bug.
+   *
+   * The id is used to validate superstates found in a cache
+   * of superstates.  It isn't sufficient to let a superstate
+   * point back to the rx for which it was compiled -- the caller
+   * may be re-using a `struct rx' in which case the superstate
+   * is not really valid.  So instead, superstates are validated
+   * by checking the sequence number of the pattern for which
+   * they were built.
+   */
+  int rx_id;
+
+  /* This is memory mgt. state for superstates.  This may be 
+   * shared by more than one struct rx.
+   */
+  struct rx_cache * cache;
+
+  /* Every regex defines the size of its own character set. 
+   * A superstate has an array of this size, with each element
+   * a `struct rx_inx'.  So, don't make this number too large.
+   * In particular, don't make it 2^16.
+   */
+  int local_cset_size;
+
+  /* After the NFA is built, it is copied into a contiguous region
+   * of memory (mostly for compatability with GNU regex).
+   * Here is that region, and it's size:
+   */
+  void * buffer;
+  unsigned long allocated;
+
+  /* Clients of RX can ask for some extra storage in the space pointed
+   * to by BUFFER.  The field RESERVED is an input parameter to the
+   * compiler.  After compilation, this much space will be available 
+   * at (buffer + allocated - reserved)
+   */
+  unsigned long reserved;
+
+  /* --------- The remaining fields are for internal use only. --------- */
+  /* --------- But! they must be initialized to 0.	       --------- */
+
+  /* NODEC is the number of nodes in the NFA with non-epsilon
+   * transitions. 
+   */
+  int nodec;
+
+  /* EPSNODEC is the number of nodes with only epsilon transitions. */
+  int epsnodec;
+
+  /* The sum (NODEC + EPSNODEC) is the total number of states in the
+   * compiled NFA.
+   */
+
+  /* Lists of side effects as stored in the NFA are `hash consed'..meaning
+   * that lists with the same elements are ==.  During compilation, 
+   * this table facilitates hash-consing.
+   */
+  struct rx_hash se_list_memo;
+
+  /* Lists of NFA states are also hashed. 
+   */
+  struct rx_hash set_list_memo;
+
+
+
+
+  /* The compiler and matcher must build a number of instruction frames.
+   * The format of these frames is fixed (c.f. struct rx_inx).  The values
+   * of the instructions is not fixed.
+   *
+   * An enumerated type (enum rx_opcode) defines the set of instructions
+   * that the compiler or matcher might generate.  When filling an instruction
+   * frame, the INX field is found by indexing this instruction table
+   * with an opcode:
+   */
+  void ** instruction_table;
+
+  /* The list of all states in an NFA.
+   * During compilation, the NEXT field of NFA states links this list.
+   * After compilation, all the states are compacted into an array,
+   * ordered by state id numbers.  At that time, this points to the base 
+   * of that array.
+   */
+  struct rx_nfa_state *nfa_states;
+
+  /* Every nfa begins with one distinguished starting state:
+   */
+  struct rx_nfa_state *start;
+
+  /* This orders the search through super-nfa paths.
+   * See the comment near the typedef of rx_se_list_order.
+   */
+  rx_se_list_order se_list_cmp;
+
+  struct rx_superset * start_set;
+};
+
+
+
+
+
+/* SYNTAX TREES */
+
+/* Compilation is in stages.  
+ *
+ * In the first stage, a pattern specified by a string is 
+ * translated into a syntax tree.  Later stages will convert
+ * the syntax tree into an NFA optimized for conversion to a
+ * superstate-NFA.
+ *
+ * This page is about syntax trees.
+ */
+
+enum rexp_node_type
+{
+  r_cset,			/* Match from a character set. `a' or `[a-z]'*/
+  r_concat,			/* Concat two subexpressions.   `ab' */
+  r_alternate,			/* Choose one of two subexpressions. `a\|b' */
+  r_opt,			/* Optional subexpression. `a?' */
+  r_star,			/* Repeated subexpression. `a*' */
+
+
+  /* A 2phase-star is a variation on a repeated subexpression.
+   * In this case, there are two subexpressions.  The first, if matched,
+   * begins a repitition (otherwise, the whole expression is matches the
+   * empth string).  
+   * 
+   * After matching the first subexpression, a 2phase star either finishes,
+   * or matches the second subexpression.  If the second subexpression is
+   * matched, then the whole construct repeats.
+   *
+   * 2phase stars are used in two circumstances.  First, they
+   * are used as part of the implementation of POSIX intervals (counted
+   * repititions).  Second, they are used to implement proper star
+   * semantics when the repeated subexpression contains paths of
+   * only side effects.  See rx_compile for more information.
+   */
+  r_2phase_star,
+
+
+  /* c.f. "typedef void * rx_side_effect" */
+  r_side_effect,
+
+  /* This is an extension type:  It is for transient use in source->source
+   * transformations (implemented over syntax trees).
+   */
+  r_data
+};
+
+/* A side effect is a matcher-specific action associated with
+ * transitions in the NFA.  The details of side effects are up
+ * to the matcher.  To the compiler and superstate constructors
+ * side effects are opaque:
+ */
+
+typedef void * rx_side_effect;
+
+/* Nodes in a syntax tree are of this type:
+ */
+struct rexp_node
+{
+  enum rexp_node_type type;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+    struct
+      {
+	struct rexp_node *left;
+	struct rexp_node *right;
+      } pair;
+    void * data;
+  } params;
+};
+
+
+
+
+/* NFA
+ *
+ * A syntax tree is compiled into an NFA.  This page defines the structure
+ * of that NFA.
+ */
+
+struct rx_nfa_state
+{
+  /* These are kept in a list as the NFA is being built. */
+  struct rx_nfa_state *next;
+
+  /* After the NFA is built, states are given integer id's.
+   * States whose outgoing transitions are all either epsilon or 
+   * side effect edges are given ids less than 0.  Other states
+   * are given successive non-negative ids starting from 0.
+   */
+  int id;
+
+  /* The list of NFA edges that go from this state to some other. */
+  struct rx_nfa_edge *edges;
+
+  /* If you land in this state, then you implicitly land
+   * in all other states reachable by only epsilon translations.
+   * Call the set of maximal paths to such states the epsilon closure
+   * of this state.
+   *
+   * There may be other states that are reachable by a mixture of
+   * epsilon and side effect edges.  Consider the set of maximal paths
+   * of that sort from this state.  Call it the epsilon-side-effect
+   * closure of the state.
+   * 
+   * The epsilon closure of the state is a subset of the epsilon-side-
+   * effect closure.  It consists of all the paths that contain 
+   * no side effects -- only epsilon edges.
+   * 
+   * The paths in the epsilon-side-effect closure  can be partitioned
+   * into equivalance sets. Two paths are equivalant if they have the
+   * same set of side effects, in the same order.  The epsilon-closure
+   * is one of these equivalance sets.  Let's call these equivalance
+   * sets: observably equivalant path sets.  That name is chosen
+   * because equivalance of two paths means they cause the same side
+   * effects -- so they lead to the same subsequent observations other
+   * than that they may wind up in different target states.
+   *
+   * The superstate nfa, which is derived from this nfa, is based on
+   * the observation that all of the paths in an observably equivalant
+   * path set can be explored at the same time, provided that the
+   * matcher keeps track not of a single nfa state, but of a set of
+   * states.   In particular, after following all the paths in an
+   * observably equivalant set, you wind up at a set of target states.
+   * That set of target states corresponds to one state in the
+   * superstate NFA.
+   *
+   * Staticly, before matching begins, it is convenient to analyze the
+   * nfa.  Each state is labeled with a list of the observably
+   * equivalant path sets who's union covers all the
+   * epsilon-side-effect paths beginning in this state.  This list is
+   * called the possible futures of the state.
+   *
+   * A trivial example is this NFA:
+   *             s1
+   *         A  --->  B
+   *
+   *             s2  
+   *            --->  C
+   *
+   *             epsilon           s1
+   *            --------->  D   ------> E
+   * 
+   * 
+   * In this example, A has two possible futures.
+   * One invokes the side effect `s1' and contains two paths,
+   * one ending in state B, the other in state E.
+   * The other invokes the side effect `s2' and contains only
+   * one path, landing in state C.
+   */
+  struct rx_possible_future *futures;
+
+
+  /* There are exactly two distinguished states in every NFA: */
+  unsigned int is_final:1;
+  unsigned int is_start:1;
+
+  /* These are used during NFA construction... */
+  unsigned int eclosure_needed:1;
+  unsigned int mark:1;
+};
+
+
+/* An edge in an NFA is typed: */
+enum rx_nfa_etype
+{
+  /* A cset edge is labled with a set of characters one of which
+   * must be matched for the edge to be taken.
+   */
+  ne_cset,
+
+  /* An epsilon edge is taken whenever its starting state is
+   * reached. 
+   */
+  ne_epsilon,
+
+  /* A side effect edge is taken whenever its starting state is
+   * reached.  Side effects may cause the match to fail or the
+   * position of the matcher to advance.
+   */
+  ne_side_effect		/* A special kind of epsilon. */
+};
+
+struct rx_nfa_edge
+{
+  struct rx_nfa_edge *next;
+  enum rx_nfa_etype type;
+  struct rx_nfa_state *dest;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+  } params;
+};
+
+
+
+/* A possible future consists of a list of side effects
+ * and a set of destination states.  Below are their
+ * representations.  These structures are hash-consed which
+ * means that lists with the same elements share a representation
+ * (their addresses are ==).
+ */
+
+struct rx_nfa_state_set
+{
+  struct rx_nfa_state * car;
+  struct rx_nfa_state_set * cdr;
+};
+
+struct rx_se_list
+{
+  rx_side_effect car;
+  struct rx_se_list * cdr;
+};
+
+struct rx_possible_future
+{
+  struct rx_possible_future *next;
+  struct rx_se_list * effects;
+  struct rx_nfa_state_set * destset;
+};
+
+
+
+
+/* This begins the description of the superstate NFA.
+ *
+ * The superstate NFA corresponds to the NFA in these ways:
+ *
+ * Every superstate NFA states SUPER correspond to sets of NFA states,
+ * nfa_states(SUPER).
+ *
+ * Superstate edges correspond to NFA paths.
+ *
+ * The superstate has no epsilon transitions;
+ * every edge has a character label, and a (possibly empty) side
+ * effect label.   The side effect label corresponds to a list of
+ * side effects that occur in the NFA.  These parts are referred
+ * to as:   superedge_character(EDGE) and superedge_sides(EDGE).
+ *
+ * For a superstate edge EDGE starting in some superstate SUPER,
+ * the following is true (in pseudo-notation :-):
+ *
+ *       exists DEST in nfa_states s.t. 
+ *         exists nfaEDGE in nfa_edges s.t.
+ *                 origin (nfaEDGE) == DEST
+ *              && origin (nfaEDGE) is a member of nfa_states(SUPER)
+ *              && exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *
+ * also:
+ *
+ *      let SUPER2 := superedge_destination(EDGE)
+ *          nfa_states(SUPER2)
+ *           == union of all nfa state sets S s.t.
+ *                          exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                       sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *                          && S == dests_of_possible_future (PF) }
+ *
+ * Or in english, every superstate is a set of nfa states.  A given
+ * character and a superstate implies many transitions in the NFA --
+ * those that begin with an edge labeled with that character from a
+ * state in the set corresponding to the superstate.
+ * 
+ * The destinations of those transitions each have a set of possible
+ * futures.  A possible future is a list of side effects and a set of
+ * destination NFA states.  Two sets of possible futures can be
+ * `merged' by combining all pairs of possible futures that have the
+ * same side effects.  A pair is combined by creating a new future
+ * with the same side effect but the union of the two destination sets.
+ * In this way, all the possible futures suggested by a superstate
+ * and a character can be merged into a set of possible futures where
+ * no two elements of the set have the same set of side effects.
+ *
+ * The destination of a possible future, being a set of NFA states, 
+ * corresponds to a supernfa state.  So, the merged set of possible
+ * futures we just created can serve as a set of edges in the
+ * supernfa.
+ *
+ * The representation of the superstate nfa and the nfa is critical.
+ * The nfa has to be compact, but has to facilitate the rapid
+ * computation of missing superstates.  The superstate nfa has to 
+ * be fast to interpret, lazilly constructed, and bounded in space.
+ *
+ * To facilitate interpretation, the superstate data structures are 
+ * peppered with `instruction frames'.  There is an instruction set
+ * defined below which matchers using the supernfa must be able to
+ * interpret.
+ *
+ * We'd like to make it possible but not mandatory to use code
+ * addresses to represent instructions (c.f. gcc's computed goto).
+ * Therefore, we define an enumerated type of opcodes, and when
+ * writing one of these instructions into a data structure, use
+ * the opcode as an index into a table of instruction values.
+ * 
+ * Here are the opcodes that occur in the superstate nfa:
+ */
+ 
+
+/* Every superstate contains a table of instruction frames indexed 
+ * by characters.  A normal `move' in a matcher is to fetch the next
+ * character and use it as an index into a superstates transition
+ * table.
+ *
+ * In the fasted case, only one edge follows from that character.
+ * In other cases there is more work to do.
+ * 
+ * The descriptions of the opcodes refer to data structures that are
+ * described further below. 
+ */
+
+enum rx_opcode
+{
+  /* 
+   * BACKTRACK_POINT is invoked when a character transition in 
+   * a superstate leads to more than one edge.  In that case,
+   * the edges have to be explored independently using a backtracking
+   * strategy.
+   *
+   * A BACKTRACK_POINT instruction is stored in a superstate's 
+   * transition table for some character when it is known that that
+   * character crosses more than one edge.  On encountering this
+   * instruction, the matcher saves enough state to backtrack to this
+   * point in the match later.
+   */
+  rx_backtrack_point = 0,	/* data is (struct transition_class *) */
+
+  /* 
+   * RX_DO_SIDE_EFFECTS evaluates the side effects of an epsilon path.
+   * There is one occurence of this instruction per rx_distinct_future.
+   * This instruction is skipped if a rx_distinct_future has no side effects.
+   */
+  rx_do_side_effects = rx_backtrack_point + 1,
+
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_CACHE_MISS instructions are stored in rx_distinct_futures whose
+   * destination superstate has been reclaimed (or was never built).
+   * It recomputes the destination superstate.
+   * RX_CACHE_MISS is also stored in a superstate transition table before
+   * any of its edges have been built.
+   */
+  rx_cache_miss = rx_do_side_effects + 1,
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_NEXT_CHAR is called to consume the next character and take the
+   * corresponding transition.  This is the only instruction that uses 
+   * the DATA field of the instruction frame instead of DATA_2.
+   * (see EXPLORE_FUTURE in regex.c).
+   */
+  rx_next_char = rx_cache_miss + 1, /* data is (struct superstate *) */
+
+  /* RX_BACKTRACK indicates that a transition fails.
+   */
+  rx_backtrack = rx_next_char + 1, /* no data */
+
+  /* 
+   * RX_ERROR_INX is stored only in places that should never be executed.
+   */
+  rx_error_inx = rx_backtrack + 1, /* Not supposed to occur. */
+
+  rx_num_instructions = rx_error_inx + 1
+};
+
+/* An id_instruction_table holds the values stored in instruction
+ * frames.  The table is indexed by the enums declared above.
+ */
+extern void * rx_id_instruction_table[rx_num_instructions];
+
+/* The heart of the matcher is a `word-code-interpreter' 
+ * (like a byte-code interpreter, except that instructions
+ * are a full word wide).
+ *
+ * Instructions are not stored in a vector of code, instead,
+ * they are scattered throughout the data structures built
+ * by the regexp compiler and the matcher.  One word-code instruction,
+ * together with the arguments to that instruction, constitute
+ * an instruction frame (struct rx_inx).
+ *
+ * This structure type is padded by hand to a power of 2 because
+ * in one of the dominant cases, we dispatch by indexing a table
+ * of instruction frames.  If that indexing can be accomplished
+ * by just a shift of the index, we're happy.
+ *
+ * Instructions take at most one argument, but there are two
+ * slots in an instruction frame that might hold that argument.
+ * These are called data and data_2.  The data slot is only
+ * used for one instruction (RX_NEXT_CHAR).  For all other 
+ * instructions, data should be set to 0.
+ *
+ * RX_NEXT_CHAR is the most important instruction by far.
+ * By reserving the data field for its exclusive use, 
+ * instruction dispatch is sped up in that case.  There is
+ * no need to fetch both the instruction and the data,
+ * only the data is needed.  In other words, a `cycle' begins
+ * by fetching the field data.  If that is non-0, then it must
+ * be the destination state of a next_char transition, so
+ * make that value the current state, advance the match position
+ * by one character, and start a new cycle.  On the other hand,
+ * if data is 0, fetch the instruction and do a more complicated
+ * dispatch on that.
+ */
+
+struct rx_inx 
+{
+  void * data;
+  void * data_2;
+  void * inx;
+  void * fnord;
+};
+
+#ifndef RX_TAIL_ARRAY
+#define RX_TAIL_ARRAY  1
+#endif
+
+/* A superstate corresponds to a set of nfa states.  Those sets are
+ * represented by STRUCT RX_SUPERSET.  The constructors
+ * guarantee that only one (shared) structure is created for a given set.
+ */
+struct rx_superset
+{
+  int refs;			/* This is a reference counted structure. */
+
+  /* We keep these sets in a cache because (in an unpredictable way),
+   * the same set is often created again and again.  But that is also
+   * problematic -- compatibility with POSIX and GNU regex requires
+   * that we not be able to tell when a program discards a particular
+   * NFA (thus invalidating the supersets created from it).
+   *
+   * But when a cache hit appears to occur, we will have in hand the
+   * nfa for which it may have happened.  That is why every nfa is given
+   * its own sequence number.  On a cache hit, the cache is validated
+   * by comparing the nfa sequence number to this field:
+   */
+  int id;
+
+  struct rx_nfa_state * car;	/* May or may not be a valid addr. */
+  struct rx_superset * cdr;
+
+  /* If the corresponding superstate exists: */
+  struct rx_superstate * superstate;
+
+
+  /* There is another bookkeeping problem.  It is expensive to 
+   * compute the starting nfa state set for an nfa.  So, once computed,
+   * it is cached in the `struct rx'.
+   *
+   * But, the state set can be flushed from the superstate cache.
+   * When that happens, we can't know if the corresponding `struct rx'
+   * is still alive or if it has been freed or re-used by the program.
+   * So, the cached pointer to this set in a struct rx might be invalid
+   * and we need a way to validate it.
+   *
+   * Fortunately, even if this set is flushed from the cache, it is
+   * not freed.  It just goes on the free-list of supersets.
+   * So we can still examine it.  
+   *
+   * So to validate a starting set memo, check to see if the
+   * starts_for field still points back to the struct rx in question,
+   * and if the ID matches the rx sequence number.
+   */
+  struct rx * starts_for;
+
+  /* This is used to link into a hash bucket so these objects can
+   * be `hash-consed'.
+   */
+  struct rx_hash_item hash_item;
+};
+
+#define rx_protect_superset(RX,CON) (++(CON)->refs)
+
+/* The terminology may be confusing (rename this structure?).
+ * Every character occurs in at most one rx_super_edge per super-state.
+ * But, that structure might have more than one option, indicating a point
+ * of non-determinism. 
+ *
+ * In other words, this structure holds a list of superstate edges
+ * sharing a common starting state and character label.  The edges
+ * are in the field OPTIONS.  All superstate edges sharing the same
+ * starting state and character are in this list.
+ */
+struct rx_super_edge
+{
+  struct rx_super_edge *next;
+  struct rx_inx rx_backtrack_frame;
+  int cset_size;
+  rx_Bitset cset;
+  struct rx_distinct_future *options;
+};
+
+/* A superstate is a set of nfa states (RX_SUPERSET) along
+ * with a transition table.  Superstates are built on demand and reclaimed
+ * without warning.  To protect a superstate from this ghastly fate,
+ * use LOCK_SUPERSTATE. 
+ */
+struct rx_superstate
+{
+  int rx_id;			/* c.f. the id field of rx_superset */
+  int locks;			/* protection from reclamation */
+
+  /* Within a superstate cache, all the superstates are kept in a big
+   * queue.  The tail of the queue is the state most likely to be
+   * reclaimed.  The *recyclable fields hold the queue position of 
+   * this state.
+   */
+  struct rx_superstate * next_recyclable;
+  struct rx_superstate * prev_recyclable;
+
+  /* The supernfa edges that exist in the cache and that have
+   * this state as their destination are kept in this list:
+   */
+  struct rx_distinct_future * transition_refs;
+
+  /* The list of nfa states corresponding to this superstate: */
+  struct rx_superset * contents;
+
+  /* The list of edges in the cache beginning from this state. */
+  struct rx_super_edge * edges;
+
+  /* A tail of the recyclable queue is marked as semifree.  A semifree
+   * state has no incoming next_char transitions -- any transition
+   * into a semifree state causes a complex dispatch with the side
+   * effect of rescuing the state from its semifree state.
+   *
+   * An alternative to this might be to make next_char more expensive,
+   * and to move a state to the head of the recyclable queue whenever
+   * it is entered.  That way, popular states would never be recycled.
+   *
+   * But unilaterally making next_char more expensive actually loses.
+   * So, incoming transitions are only made expensive for states near
+   * the tail of the recyclable queue.  The more cache contention
+   * there is, the more frequently a state will have to prove itself
+   * and be moved back to the front of the queue.  If there is less 
+   * contention, then popular states just aggregate in the front of 
+   * the queue and stay there.
+   */
+  int is_semifree;
+
+
+  /* This keeps track of the size of the transition table for this
+   * state.  There is a half-hearted attempt to support variable sized
+   * superstates.
+   */
+  int trans_size;
+
+  /* Indexed by characters... */
+  struct rx_inx transitions[RX_TAIL_ARRAY];
+};
+
+
+/* A list of distinct futures define the edges that leave from a 
+ * given superstate on a given character.  c.f. rx_super_edge.
+ */
+
+struct rx_distinct_future
+{
+  struct rx_distinct_future * next_same_super_edge[2];
+  struct rx_distinct_future * next_same_dest;
+  struct rx_distinct_future * prev_same_dest;
+  struct rx_superstate * present;	/* source state */
+  struct rx_superstate * future;	/* destination state */
+  struct rx_super_edge * edge;
+
+
+  /* The future_frame holds the instruction that should be executed
+   * after all the side effects are done, when it is time to complete
+   * the transition to the next state.
+   *
+   * Normally this is a next_char instruction, but it may be a
+   * cache_miss instruction as well, depending on whether or not
+   * the superstate is in the cache and semifree.
+   * 
+   * If this is the only future for a given superstate/char, and
+   * if there are no side effects to be performed, this frame is
+   * not used (directly) at all.  Instead, its contents are copied
+   * into the transition table of the starting state of this dist. future.
+   */
+  struct rx_inx future_frame;
+
+  struct rx_inx side_effects_frame;
+  struct rx_se_list * effects;
+};
+
+#define rx_lock_superstate(R,S)  ((S)->locks++)
+#define rx_unlock_superstate(R,S) (--(S)->locks)
+
+
+
+/* This page destined for rx.h */
+
+struct rx_blocklist
+{
+  struct rx_blocklist * next;
+  int bytes;
+};
+
+struct rx_freelist
+{
+  struct rx_freelist * next;
+};
+
+struct rx_cache;
+
+#ifdef __STDC__
+typedef void (*rx_morecore_fn)(struct rx_cache *);
+#else
+typedef void (*rx_morecore_fn)();
+#endif
+
+/* You use this to control the allocation of superstate data 
+ * during matching.  Most of it should be initialized to 0.
+ *
+ * A MORECORE function is necessary.  It should allocate
+ * a new block of memory or return 0.
+ * A default that uses malloc is called `rx_morecore'.
+ *
+ * The number of SUPERSTATES_ALLOWED indirectly limits how much memory
+ * the system will try to allocate.  The default is 128.  Batch style
+ * applications that are very regexp intensive should use as high a number
+ * as possible without thrashing.
+ * 
+ * The LOCAL_CSET_SIZE is the number of characters in a character set.
+ * It is therefore the number of entries in a superstate transition table.
+ * Generally, it should be 256.  If your character set has 16 bits, 
+ * it is better to translate your regexps into equivalent 8 bit patterns.
+ */
+
+struct rx_cache
+{
+  struct rx_hash_rules superset_hash_rules;
+
+  /* Objects are allocated by incrementing a pointer that 
+   * scans across rx_blocklists.
+   */
+  struct rx_blocklist * memory;
+  struct rx_blocklist * memory_pos;
+  int bytes_left;
+  char * memory_addr;
+  rx_morecore_fn morecore;
+
+  /* Freelists. */
+  struct rx_freelist * free_superstates;
+  struct rx_freelist * free_transition_classes;
+  struct rx_freelist * free_discernable_futures;
+  struct rx_freelist * free_supersets;
+  struct rx_freelist * free_hash;
+
+  /* Two sets of superstates -- those that are semifreed, and those
+   * that are being used.
+   */
+  struct rx_superstate * lru_superstate;
+  struct rx_superstate * semifree_superstate;
+
+  struct rx_superset * empty_superset;
+
+  int superstates;
+  int semifree_superstates;
+  int hits;
+  int misses;
+  int superstates_allowed;
+
+  int local_cset_size;
+  void ** instruction_table;
+
+  struct rx_hash superset_table;
+};
+
+
+
+
+/* The lowest-level search function supports arbitrarily fragmented
+ * strings and (optionally) suspendable/resumable searches.
+ *
+ * Callers have to provide a few hooks.
+ */
+
+#ifndef __GNUC__
+#ifdef __STDC__
+#define __const__ const
+#else
+#define __const__
+#endif
+#endif
+
+/* This holds a matcher position */
+struct rx_string_position
+{
+  __const__ unsigned char * pos;	/* The current pos. */
+  __const__ unsigned char * string; /* The current string burst. */
+  __const__ unsigned char * end;	/* First invalid position >= POS. */
+  int offset;			/* Integer address of the current burst. */
+  int size;			/* Current string's size. */
+  int search_direction;		/* 1 or -1 */
+  int search_end;		/* First position to not try. */
+};
+
+
+enum rx_get_burst_return
+{
+  rx_get_burst_continuation,
+  rx_get_burst_error,
+  rx_get_burst_ok,
+  rx_get_burst_no_more
+};
+
+
+/* A call to get burst should make POS valid.  It might be invalid
+ * if the STRING field doesn't point to a burst that actually
+ * contains POS.
+ *
+ * GET_BURST should take a clue from SEARCH_DIRECTION (1 or -1) as to
+ * whether or not to pad to the left.  Padding to the right is always
+ * appropriate, but need not go past the point indicated by STOP.
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the get_burst.
+ */
+
+#ifdef __STDC__
+typedef enum rx_get_burst_return
+  (*rx_get_burst_fn) (struct rx_string_position * pos,
+		      void * app_closure,
+		      int stop);
+					       
+#else
+typedef enum rx_get_burst_return (*rx_get_burst_fn) ();
+#endif
+
+
+enum rx_back_check_return
+{
+  rx_back_check_continuation,
+  rx_back_check_error,
+  rx_back_check_pass,
+  rx_back_check_fail
+};
+
+/* Back_check should advance the position it is passed 
+ * over rparen - lparen characters and return pass iff
+ * the characters starting at POS match those indexed
+ * by [LPAREN..RPAREN].
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the back_check.
+ */
+
+#ifdef __STDC__
+typedef enum rx_back_check_return
+  (*rx_back_check_fn) (struct rx_string_position * pos,
+		       int lparen,
+		       int rparen,
+		       unsigned char * translate,
+		       void * app_closure,
+		       int stop);
+					       
+#else
+typedef enum rx_back_check_return (*rx_back_check_fn) ();
+#endif
+
+
+
+
+/* A call to fetch_char should return the character at POS or POS + 1.
+ * Returning continuations here isn't supported.  OFFSET is either 0 or 1
+ * and indicates which characters is desired.
+ */
+
+#ifdef __STDC__
+typedef int (*rx_fetch_char_fn) (struct rx_string_position * pos,
+				 int offset,
+				 void * app_closure,
+				 int stop);
+#else
+typedef int (*rx_fetch_char_fn) ();
+#endif
+
+
+enum rx_search_return
+{
+  rx_search_continuation = -4,
+  rx_search_error = -3,
+  rx_search_soft_fail = -2,	/* failed by running out of string */
+  rx_search_fail = -1		/* failed only by reaching failure states */
+  /* return values >= 0 indicate the position of a successful match */
+};
+
+
+
+
+
+
+
+/* regex.h
+ * 
+ * The remaining declarations replace regex.h.
+ */
+
+/* This is an array of error messages corresponding to the error codes.
+ */
+extern __const__ char *re_error_msg[];
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+/* The regex.c support, as a client of rx, defines a set of possible
+ * side effects that can be added to the edge lables of nfa edges.
+ * Here is the list of sidef effects in use.
+ */
+
+enum re_side_effects
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      NAME VALUE,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     NAME VALUE,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+   re_floogle_flap = 65533
+};
+
+/* These hold paramaters for the kinds of side effects that are possible
+ * in the supported pattern languages.  These include things like the 
+ * numeric bounds of {} operators and the index of paren registers for 
+ * subexpression measurement or backreferencing.
+ */
+struct re_se_params
+{
+  enum re_side_effects se;
+  int op1;
+  int op2;
+};
+
+typedef unsigned reg_syntax_t;
+
+struct re_pattern_buffer
+{
+  struct rx rx;
+  reg_syntax_t syntax;		/* See below for syntax bit definitions. */
+
+  unsigned int no_sub:1;	/* If set, don't  return register offsets. */
+  unsigned int not_bol:1;	/* If set, the anchors ('^' and '$') don't */
+  unsigned int not_eol:1;	/*     match at the ends of the string.  */  
+  unsigned int newline_anchor:1;/* If true, an anchor at a newline matches.*/
+  unsigned int least_subs:1;	/* If set, and returning registers, return
+				 * as few values as possible.  Only 
+				 * backreferenced groups and group 0 (the whole
+				 * match) will be returned.
+				 */
+
+  /* If true, this says that the matcher should keep registers on its
+   * backtracking stack.  For many patterns, we can easily determine that
+   * this isn't necessary.
+   */
+  unsigned int match_regs_on_stack:1;
+  unsigned int search_regs_on_stack:1;
+
+  /* is_anchored and begbuf_only are filled in by rx_compile. */
+  unsigned int is_anchored:1;	/* Anchorded by ^? */
+  unsigned int begbuf_only:1;	/* Anchored to char position 0? */
+
+  
+  /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+   * for `max (RE_NREGS, re_nsub + 1)' groups.
+   * If REGS_REALLOCATE, reallocate space if necessary.
+   * If REGS_FIXED, use what's there.  
+   */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned int regs_allocated:2;
+
+  
+  /* Either a translate table to apply to all characters before
+   * comparing them, or zero for no translation.  The translation
+   * is applied to a pattern when it is compiled and to a string
+   * when it is matched.
+   */
+  unsigned char * translate;
+
+  /* If this is a valid pointer, it tells rx not to store the extents of 
+   * certain subexpressions (those corresponding to non-zero entries).
+   * Passing 0x1 is the same as passing an array of all ones.  Passing 0x0
+   * is the same as passing an array of all zeros.
+   * The array should contain as many entries as their are subexps in the 
+   * regexp.
+   *
+   * For POSIX compatability, when using regcomp and regexec this field
+   * is zeroed and ignored.
+   */
+  char * syntax_parens;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+  void * buffer;		/* Malloced memory for the nfa. */
+  unsigned long allocated;	/* Size of that memory. */
+
+  /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+   * the fastmap, if there is one, to skip over impossible
+   * starting points for matches.  */
+  char *fastmap;
+
+  unsigned int fastmap_accurate:1; /* These three are internal. */
+  unsigned int can_match_empty:1;  
+  struct rx_nfa_state * start;	/* The nfa starting state. */
+
+  /* This is the list of iterator bounds for {lo,hi} constructs.
+   * The memory pointed to is part of the rx->buffer.
+   */
+  struct re_se_params *se_params;
+
+  /* This is a bitset representation of the fastmap.
+   * This is a true fastmap that already takes the translate
+   * table into account.
+   */
+  rx_Bitset fastset;
+};
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+ * `re_match_2' returns information about at least this many registers
+ * the first time a `regs' structure is passed. 
+ *
+ * Also, this is the greatest number of backreferenced subexpressions
+ * allowed in a pattern being matched without caller-supplied registers.
+ */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+extern int rx_cache_bound;
+extern char rx_version_string[];
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+/* This is decls to the interesting subsystems and lower layers
+ * of rx.  Everything which doesn't have a public counterpart in 
+ * regex.c is declared here.
+ */
+
+
+#ifdef __STDC__
+typedef void (*rx_hash_freefn) (struct rx_hash_item * it);
+#else /* ndef __STDC__ */
+typedef void (*rx_hash_freefn) ();
+#endif /* ndef __STDC__ */
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL int rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_empty (int size, rx_Bitset set);
+RX_DECL void rx_bitset_null (int size, rx_Bitset b);
+RX_DECL void rx_bitset_universe (int size, rx_Bitset b);
+RX_DECL void rx_bitset_complement (int size, rx_Bitset b);
+RX_DECL void rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_union (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_intersection (int size,
+				     rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_revdifference (int size,
+				      rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL unsigned long rx_bitset_hash (int size, rx_Bitset b);
+RX_DECL struct rx_hash_item * rx_hash_find (struct rx_hash * table,
+					    unsigned long hash,
+					    void * value,
+					    struct rx_hash_rules * rules);
+RX_DECL struct rx_hash_item * rx_hash_store (struct rx_hash * table,
+					     unsigned long hash,
+					     void * value,
+					     struct rx_hash_rules * rules);
+RX_DECL void rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules);
+RX_DECL void rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+				 struct rx_hash_rules * rules);
+RX_DECL rx_Bitset rx_cset (struct rx *rx);
+RX_DECL rx_Bitset rx_copy_cset (struct rx *rx, rx_Bitset a);
+RX_DECL void rx_free_cset (struct rx * rx, rx_Bitset c);
+RX_DECL struct rexp_node * rexp_node (struct rx *rx,
+				      enum rexp_node_type type);
+RX_DECL struct rexp_node * rx_mk_r_cset (struct rx * rx,
+					 rx_Bitset b);
+RX_DECL struct rexp_node * rx_mk_r_concat (struct rx * rx,
+					   struct rexp_node * a,
+					   struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_alternate (struct rx * rx,
+					      struct rexp_node * a,
+					      struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_opt (struct rx * rx,
+					struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_star (struct rx * rx,
+					 struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_2phase_star (struct rx * rx,
+						struct rexp_node * a,
+						struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_side_effect (struct rx * rx,
+						rx_side_effect a);
+RX_DECL struct rexp_node * rx_mk_r_data  (struct rx * rx,
+					  void * a);
+RX_DECL void rx_free_rexp (struct rx * rx, struct rexp_node * node);
+RX_DECL struct rexp_node * rx_copy_rexp (struct rx *rx,
+					 struct rexp_node *node);
+RX_DECL struct rx_nfa_state * rx_nfa_state (struct rx *rx);
+RX_DECL void rx_free_nfa_state (struct rx_nfa_state * n);
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state (struct rx * rx,
+						  int id);
+RX_DECL struct rx_nfa_edge * rx_nfa_edge (struct rx *rx,
+					  enum rx_nfa_etype type,
+					  struct rx_nfa_state *start,
+					  struct rx_nfa_state *dest);
+RX_DECL void rx_free_nfa_edge (struct rx_nfa_edge * e);
+RX_DECL void rx_free_nfa (struct rx *rx);
+RX_DECL int rx_build_nfa (struct rx *rx,
+			  struct rexp_node *rexp,
+			  struct rx_nfa_state **start,
+			  struct rx_nfa_state **end);
+RX_DECL void rx_name_nfa_states (struct rx *rx);
+RX_DECL int rx_eclose_nfa (struct rx *rx);
+RX_DECL void rx_delete_epsilon_transitions (struct rx *rx);
+RX_DECL int rx_compactify_nfa (struct rx *rx,
+			       void **mem, unsigned long *size);
+RX_DECL void rx_release_superset (struct rx *rx,
+				  struct rx_superset *set);
+RX_DECL struct rx_superset * rx_superset_cons (struct rx * rx,
+					       struct rx_nfa_state *car, struct rx_superset *cdr);
+RX_DECL struct rx_superset * rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl);
+RX_DECL struct rx_superstate * rx_superstate (struct rx *rx,
+					      struct rx_superset *set);
+RX_DECL struct rx_inx * rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data);
+RX_DECL reg_errcode_t rx_compile (__const__ char *pattern, int size,
+				  reg_syntax_t syntax,
+				  struct re_pattern_buffer * rxb);
+RX_DECL void rx_blow_up_fastmap (struct re_pattern_buffer * rxb);
+#else /* STDC */
+RX_DECL int rx_bitset_is_equal ();
+RX_DECL int rx_bitset_is_subset ();
+RX_DECL int rx_bitset_empty ();
+RX_DECL void rx_bitset_null ();
+RX_DECL void rx_bitset_universe ();
+RX_DECL void rx_bitset_complement ();
+RX_DECL void rx_bitset_assign ();
+RX_DECL void rx_bitset_union ();
+RX_DECL void rx_bitset_intersection ();
+RX_DECL void rx_bitset_difference ();
+RX_DECL void rx_bitset_revdifference ();
+RX_DECL void rx_bitset_xor ();
+RX_DECL unsigned long rx_bitset_hash ();
+RX_DECL struct rx_hash_item * rx_hash_find ();
+RX_DECL struct rx_hash_item * rx_hash_store ();
+RX_DECL void rx_hash_free ();
+RX_DECL void rx_free_hash_table ();
+RX_DECL rx_Bitset rx_cset ();
+RX_DECL rx_Bitset rx_copy_cset ();
+RX_DECL void rx_free_cset ();
+RX_DECL struct rexp_node * rexp_node ();
+RX_DECL struct rexp_node * rx_mk_r_cset ();
+RX_DECL struct rexp_node * rx_mk_r_concat ();
+RX_DECL struct rexp_node * rx_mk_r_alternate ();
+RX_DECL struct rexp_node * rx_mk_r_opt ();
+RX_DECL struct rexp_node * rx_mk_r_star ();
+RX_DECL struct rexp_node * rx_mk_r_2phase_star ();
+RX_DECL struct rexp_node * rx_mk_r_side_effect ();
+RX_DECL struct rexp_node * rx_mk_r_data  ();
+RX_DECL void rx_free_rexp ();
+RX_DECL struct rexp_node * rx_copy_rexp ();
+RX_DECL struct rx_nfa_state * rx_nfa_state ();
+RX_DECL void rx_free_nfa_state ();
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state ();
+RX_DECL struct rx_nfa_edge * rx_nfa_edge ();
+RX_DECL void rx_free_nfa_edge ();
+RX_DECL void rx_free_nfa ();
+RX_DECL int rx_build_nfa ();
+RX_DECL void rx_name_nfa_states ();
+RX_DECL int rx_eclose_nfa ();
+RX_DECL void rx_delete_epsilon_transitions ();
+RX_DECL int rx_compactify_nfa ();
+RX_DECL void rx_release_superset ();
+RX_DECL struct rx_superset * rx_superset_cons ();
+RX_DECL struct rx_superset * rx_superstate_eclosure_union ();
+RX_DECL struct rx_superstate * rx_superstate ();
+RX_DECL struct rx_inx * rx_handle_cache_miss ();
+RX_DECL reg_errcode_t rx_compile ();
+RX_DECL void rx_blow_up_fastmap ();
+#endif /* STDC */
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+
+#ifdef __STDC__
+extern int re_search_2 (struct re_pattern_buffer *rxb,
+			__const__ char * string1, int size1,
+			__const__ char * string2, int size2,
+			int startpos, int range,
+			struct re_registers *regs,
+			int stop);
+extern int re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+		      int size, int startpos, int range,
+		      struct re_registers *regs);
+extern int re_match_2 (struct re_pattern_buffer * rxb,
+		       __const__ char * string1, int size1,
+		       __const__ char * string2, int size2,
+		       int pos, struct re_registers *regs, int stop);
+extern int re_match (struct re_pattern_buffer * rxb,
+		     __const__ char * string,
+		     int size, int pos,
+		     struct re_registers *regs);
+extern reg_syntax_t re_set_syntax (reg_syntax_t syntax);
+extern void re_set_registers (struct re_pattern_buffer *bufp,
+			      struct re_registers *regs,
+			      unsigned num_regs,
+			      regoff_t * starts, regoff_t * ends);
+extern __const__ char * re_compile_pattern (__const__ char *pattern,
+					int length,
+					struct re_pattern_buffer * rxb);
+extern int re_compile_fastmap (struct re_pattern_buffer * rxb);
+extern char * re_comp (__const__ char *s);
+extern int re_exec (__const__ char *s);
+extern int regcomp (regex_t * preg, __const__ char * pattern, int cflags);
+extern int regexec (__const__ regex_t *preg, __const__ char *string,
+		    size_t nmatch, regmatch_t pmatch[],
+		    int eflags);
+extern size_t regerror (int errcode, __const__ regex_t *preg,
+			char *errbuf, size_t errbuf_size);
+extern void regfree (regex_t *preg);
+
+#else /* STDC */
+extern int re_search_2 ();
+extern int re_search ();
+extern int re_match_2 ();
+extern int re_match ();
+extern reg_syntax_t re_set_syntax ();
+extern void re_set_registers ();
+extern __const__ char * re_compile_pattern ();
+extern int re_compile_fastmap ();
+extern char * re_comp ();
+extern int re_exec ();
+extern int regcomp ();
+extern int regexec ();
+extern size_t regerror ();
+extern void regfree ();
+
+#endif /* STDC */
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+struct rx_counter_frame
+{
+  int tag;
+  int val;
+  struct rx_counter_frame * inherited_from; /* If this is a copy. */
+  struct rx_counter_frame * cdr;
+};
+
+struct rx_backtrack_frame
+{
+  char * counter_stack_sp;
+
+  /* A frame is used to save the matchers state when it crosses a 
+   * backtracking point.  The `stk_' fields correspond to variables
+   * in re_search_2 (just strip off thes `stk_').  They are documented
+   * tere.
+   */
+  struct rx_superstate * stk_super;
+  unsigned int stk_c;
+  struct rx_string_position stk_test_pos;
+  int stk_last_l;
+  int stk_last_r;
+  int stk_test_ret;
+
+  /* This is the list of options left to explore at the backtrack
+   * point for which this frame was created. 
+   */
+  struct rx_distinct_future * df;
+  struct rx_distinct_future * first_df;
+
+#ifdef RX_DEBUG
+   int stk_line_no;
+#endif
+};
+
+struct rx_stack_chunk
+{
+  struct rx_stack_chunk * next_chunk;
+  int bytes_left;
+  char * sp;
+};
+
+enum rx_outer_entry
+{
+  rx_outer_start,
+  rx_outer_fastmap,
+  rx_outer_test,
+  rx_outer_restore_pos
+};
+
+enum rx_fastmap_return
+{
+  rx_fastmap_continuation,
+  rx_fastmap_error,
+  rx_fastmap_ok,
+  rx_fastmap_fail
+};
+
+enum rx_fastmap_entry
+{
+  rx_fastmap_start,
+  rx_fastmap_string_break
+};
+
+enum rx_test_return
+{
+  rx_test_continuation,
+  rx_test_error,
+  rx_test_fail,
+  rx_test_ok
+};
+
+enum rx_test_internal_return
+{
+  rx_test_internal_error,
+  rx_test_found_first,
+  rx_test_line_finished
+};
+
+enum rx_test_match_entry
+{
+  rx_test_start,
+  rx_test_cache_hit_loop,
+  rx_test_backreference_check,
+  rx_test_backtrack_return
+};
+
+struct rx_search_state
+{
+  /* Two groups of registers are kept.  The group with the register state
+   * of the current test match, and the group that holds the state at the end
+   * of the best known match, if any.
+   *
+   * For some patterns, there may also be registers saved on the stack.
+   */
+  unsigned num_regs;		/* Includes an element for register zero. */
+  regoff_t * lparen;		/* scratch space for register returns */
+  regoff_t * rparen;
+  regoff_t * best_lpspace;	/* in case the user doesn't want these */
+  regoff_t * best_rpspace;	/* values, we still need space to store
+				 * them.  Normally, this memoryis unused
+				 * and the space pointed to by REGS is 
+				 * used instead.
+				 */
+  
+  int last_l;			/* Highest index of a valid lparen. */
+  int last_r;			/* It's dual. */
+  
+  int * best_lparen;		/* This contains the best known register */
+  int * best_rparen;		/* assignments. 
+				 * This may point to the same mem as
+				 * best_lpspace, or it might point to memory
+				 * passed by the caller.
+				 */
+  int best_last_l;		/* best_last_l:best_lparen::last_l:lparen */
+  int best_last_r;
+
+
+  unsigned char * translate;  
+
+  struct rx_string_position outer_pos;
+
+  struct rx_superstate * start_super;
+  int nfa_choice;
+  int first_found;		/* If true, return after finding any match. */
+  int ret_val;
+
+  /* For continuations... */
+  enum rx_outer_entry outer_search_resume_pt;
+  struct re_pattern_buffer * saved_rxb;
+  int saved_startpos;
+  int saved_range;
+  int saved_stop;
+  int saved_total_size;
+  rx_get_burst_fn saved_get_burst;
+  rx_back_check_fn saved_back_check;
+  struct re_registers * saved_regs;
+  
+  /**
+   ** state for fastmap
+   **/
+  char * fastmap;
+  int fastmap_chr;
+  int fastmap_val;
+
+  /* for continuations in the fastmap procedure: */
+  enum rx_fastmap_entry fastmap_resume_pt;
+
+  /**
+   ** state for test_match 
+   **/
+
+  /* The current superNFA position of the matcher. */
+  struct rx_superstate * super;
+  
+  /* The matcher interprets a series of instruction frames.
+   * This is the `instruction counter' for the interpretation.
+   */
+  struct rx_inx * ifr;
+  
+  /* We insert a ghost character in the string to prime
+   * the nfa.  test_pos.pos, test_pos.str_half, and test_pos.end_half
+   * keep track of the test-match position and string-half.
+   */
+  unsigned char c;
+  
+  /* Position within the string. */
+  struct rx_string_position test_pos;
+
+  struct rx_stack_chunk * counter_stack;
+  struct rx_stack_chunk * backtrack_stack;
+  int backtrack_frame_bytes;
+  int chunk_bytes;
+  struct rx_stack_chunk * free_chunks;
+
+  /* To return from this function, set test_ret and 
+   * `goto test_do_return'.
+   *
+   * Possible return values are:
+   *     1   --- end of string while the superNFA is still going
+   *     0   --- internal error (out of memory)
+   *	-1   --- search completed by reaching the superNFA fail state
+   *    -2   --- a match was found, maybe not the longest.
+   *
+   * When the search is complete (-1), best_last_r indicates whether
+   * a match was found.
+   *
+   * -2 is return only if search_state.first_found is non-zero.
+   *
+   * if search_state.first_found is non-zero, a return of -1 indicates no match,
+   * otherwise, best_last_r has to be checked.
+   */
+  int test_ret;
+
+  int could_have_continued;
+  
+#ifdef RX_DEBUG
+  int backtrack_depth;
+  /* There is a search tree with every node as set of deterministic
+   * transitions in the super nfa.  For every branch of a 
+   * backtrack point is an edge in the tree.
+   * This counts up a pre-order of nodes in that tree.
+   * It's saved on the search stack and printed when debugging. 
+   */
+  int line_no;
+  int lines_found;
+#endif
+
+
+  /* For continuations within the match tester */
+  enum rx_test_match_entry test_match_resume_pt;
+  struct rx_inx * saved_next_tr_table;
+  struct rx_inx * saved_this_tr_table;
+  int saved_reg;
+  struct rx_backtrack_frame * saved_bf;
+  
+};
+
+
+
+extern char rx_slowmap[];
+extern unsigned char rx_id_translation[];
+
+static __inline__ void
+init_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  search_state->fastmap = (rxb->fastmap
+			   ? (char *)rxb->fastmap
+			   : (char *)rx_slowmap);
+  /* Update the fastmap now if not correct already. 
+   * When the regexp was compiled, the fastmap was computed
+   * and stored in a bitset.  This expands the bitset into a
+   * character array containing 1s and 0s.
+   */
+  if ((search_state->fastmap == rxb->fastmap) && !rxb->fastmap_accurate)
+    rx_blow_up_fastmap (rxb);
+  search_state->fastmap_chr = -1;
+  search_state->fastmap_val = 0;
+  search_state->fastmap_resume_pt = rx_fastmap_start;
+}
+
+static __inline__ void
+uninit_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  /* Unset the fastmap sentinel */
+  if (search_state->fastmap_chr >= 0)
+    search_state->fastmap[search_state->fastmap_chr]
+      = search_state->fastmap_val;
+}
+
+static __inline__ int
+fastmap_search (rxb, stop, get_burst, app_closure, search_state)
+     struct re_pattern_buffer * rxb;
+     int stop;
+     rx_get_burst_fn get_burst;
+     void * app_closure;
+     struct rx_search_state * search_state;
+{
+  enum rx_fastmap_entry pc;
+
+  if (0)
+    {
+    return_continuation:
+      search_state->fastmap_resume_pt = pc;
+      return rx_fastmap_continuation;
+    }
+
+  pc = search_state->fastmap_resume_pt;
+
+  switch (pc)
+    {
+    default:
+      return rx_fastmap_error;
+    case rx_fastmap_start:
+    init_fastmap_sentinal:
+      /* For the sake of fast fastmapping, set a sentinal in the fastmap.
+       * This sentinal will trap the fastmap loop when it reaches the last
+       * valid character in a string half.
+       *
+       * This must be reset when the fastmap/search loop crosses a string 
+       * boundry, and before returning to the caller.  So sometimes,
+       * the fastmap loop is restarted with `continue', othertimes by
+       * `goto init_fastmap_sentinal'.
+       */
+      if (search_state->outer_pos.size)
+	{
+	  search_state->fastmap_chr = ((search_state->outer_pos.search_direction == 1)
+				       ? *(search_state->outer_pos.end - 1)
+				       : *search_state->outer_pos.string);
+	  search_state->fastmap_val
+	    = search_state->fastmap[search_state->fastmap_chr];
+	  search_state->fastmap[search_state->fastmap_chr] = 1;
+	}
+      else
+	{
+	  search_state->fastmap_chr = -1;
+	  search_state->fastmap_val = 0;
+	}
+      
+      if (search_state->outer_pos.pos >= search_state->outer_pos.end)
+	goto fastmap_hit_bound;
+      else
+	{
+	  if (search_state->outer_pos.search_direction == 1)
+	    {
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      if (*search_state->outer_pos.pos != search_state->fastmap_chr)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  ++search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == search_state->outer_pos.end)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	  else
+	    {
+	      __const__ unsigned char * bound;
+	      bound = search_state->outer_pos.string - 1;
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      if ((*search_state->outer_pos.pos != search_state->fastmap_chr) || search_state->fastmap_val)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  --search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == bound)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	}
+      
+    case rx_fastmap_string_break:
+    fastmap_hit_bound:
+      {
+	/* If we hit a bound, it may be time to fetch another burst
+	 * of string, or it may be time to return a continuation to 
+ 	 * the caller, or it might be time to fail.
+	 */
+
+	int burst_state;
+	burst_state = get_burst (&search_state->outer_pos, app_closure, stop);
+	switch (burst_state)
+	  {
+	  default:
+	  case rx_get_burst_error:
+	    return rx_fastmap_error;
+	  case rx_get_burst_continuation:
+	    {
+	      pc = rx_fastmap_string_break;
+	      goto return_continuation;
+	    }
+	  case rx_get_burst_ok:
+	    goto init_fastmap_sentinal;
+	  case rx_get_burst_no_more:
+	    /* ...not a string split, simply no more string. 
+	     *
+	     * When searching backward, running out of string
+	     * is reason to quit.
+	     *
+	     * When searching forward, we allow the possibility
+	     * of an (empty) match after the last character in the
+	     * virtual string.  So, fall through to the matcher
+	     */
+	    return (  (search_state->outer_pos.search_direction == 1)
+		    ? rx_fastmap_ok
+		    : rx_fastmap_fail);
+	  }
+      }
+    }
+
+}
+
+
+
+
+#ifdef emacs
+/* The `emacs' switch turns on certain matching commands
+ * that make sense only in Emacs. 
+ */
+#include "sysfuncs.h"
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+#endif /* emacs */
+
+/* Setting RX_MEMDBUG is useful if you have dbmalloc.  Maybe with similar
+ * packages too.
+ */
+#ifdef RX_MEMDBUG
+#include <malloc.h>
+#endif /* RX_RX_MEMDBUG */
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+ * `BSTRING', as far as I know, and neither of them use this code.  
+ */
+#if HAVE_STRING_H || STDC_HEADERS
+#include <string.h>
+
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+
+#else /*  HAVE_STRING_H || STDC_HEADERS */
+#include <strings.h>
+#endif   /* not (HAVE_STRING_H || STDC_HEADERS) */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else /* not STDC_HEADERS */
+char *malloc ();
+char *realloc ();
+#endif /* not STDC_HEADERS */
+
+
+
+
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE (1 << CHARBITS)
+
+#ifndef emacs
+/* Define the syntax basics for \<, \>, etc.
+ * This must be nonzero for the wordchar and notwordchar pattern
+ * commands in re_match_2.
+ */
+#ifndef Sword 
+#define Sword 1
+#endif
+#define SYNTAX(c) re_syntax_table[c]
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else
+RX_DECL char re_syntax_table[CHAR_SET_SIZE];
+#endif
+#endif /* not emacs */
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+ *  of `string1' and `string2'.  If only one string, it's `string2'.  
+ */
+
+#define AT_STRINGS_BEG() \
+  (   -1		 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+#define AT_STRINGS_END() \
+  (   (total_size - 1)	 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+
+/* Test if POS + 1 points to a character which is word-constituent.  We have
+ * two special cases to check for: if past the end of string1, look at
+ * the first character in string2; and if before the beginning of
+ * string2, look at the last character in string1.
+ *
+ * Assumes `string1' exists, so use in conjunction with AT_STRINGS_BEG ().  
+ */
+#define LETTER_P(POS,OFF)						\
+  (   SYNTAX (fetch_char(POS, OFF, app_closure, stop))			\
+   == Sword)
+
+/* Test if the character at D and the one after D differ with respect
+ * to being word-constituent.  
+ */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG () || AT_STRINGS_END () || LETTER_P (d,0) != LETTER_P (d, 1))
+
+
+#ifdef RX_SUPPORT_CONTINUATIONS
+#define RX_STACK_ALLOC(BYTES) malloc(BYTES)
+#define RX_STACK_FREE(MEM) free(MEM)
+#else
+#define RX_STACK_ALLOC(BYTES) alloca(BYTES)
+#define RX_STACK_FREE(MEM) \
+      ((struct rx_stack_chunk *)MEM)->next_chunk = search_state.free_chunks; \
+      search_state.free_chunks = ((struct rx_stack_chunk *)MEM);
+
+#endif
+
+#define PUSH(CHUNK_VAR,BYTES)   \
+  if (!CHUNK_VAR || (CHUNK_VAR->bytes_left < (BYTES)))  \
+    {					\
+      struct rx_stack_chunk * new_chunk;	\
+      if (search_state.free_chunks)			\
+	{				\
+	  new_chunk = search_state.free_chunks;	\
+	  search_state.free_chunks = search_state.free_chunks->next_chunk; \
+	}				\
+      else				\
+	{				\
+	  new_chunk = (struct rx_stack_chunk *)RX_STACK_ALLOC(search_state.chunk_bytes); \
+	  if (!new_chunk)		\
+	    {				\
+	      search_state.ret_val = 0;		\
+	      goto test_do_return;	\
+	    }				\
+	}				\
+      new_chunk->sp = (char *)new_chunk + sizeof (struct rx_stack_chunk); \
+      new_chunk->bytes_left = (search_state.chunk_bytes \
+			       - (BYTES) \
+			       - sizeof (struct rx_stack_chunk)); \
+      new_chunk->next_chunk = CHUNK_VAR; \
+      CHUNK_VAR = new_chunk;		\
+    } \
+  else \
+    (CHUNK_VAR->sp += (BYTES)), (CHUNK_VAR->bytes_left -= (BYTES))
+
+#define POP(CHUNK_VAR,BYTES) \
+  if (CHUNK_VAR->sp == ((char *)CHUNK_VAR + sizeof(*CHUNK_VAR))) \
+    { \
+      struct rx_stack_chunk * new_chunk = CHUNK_VAR->next_chunk; \
+      RX_STACK_FREE(CHUNK_VAR); \
+      CHUNK_VAR = new_chunk; \
+    } \
+  else \
+    (CHUNK_VAR->sp -= BYTES), (CHUNK_VAR->bytes_left += BYTES)
+
+
+
+#define SRCH_TRANSLATE(C)  search_state.translate[(unsigned char) (C)]
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL __inline__ int
+rx_search  (struct re_pattern_buffer * rxb,
+	    int startpos,
+	    int range,
+	    int stop,
+	    int total_size,
+	    rx_get_burst_fn get_burst,
+	    rx_back_check_fn back_check,
+	    rx_fetch_char_fn fetch_char,
+	    void * app_closure,
+	    struct re_registers * regs,
+	    struct rx_search_state * resume_state,
+	    struct rx_search_state * save_state)
+#else
+RX_DECL __inline__ int
+rx_search  (rxb, startpos, range, stop, total_size,
+	    get_burst, back_check, fetch_char,
+	    app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+#endif
+{
+  int pc;
+  int test_state;
+  struct rx_search_state search_state;
+
+  search_state.free_chunks = 0;
+  if (!resume_state)
+    pc = rx_outer_start;
+  else
+    {
+      search_state = *resume_state;
+      regs = search_state.saved_regs;
+      rxb = search_state.saved_rxb;
+      startpos = search_state.saved_startpos;
+      range = search_state.saved_range;
+      stop = search_state.saved_stop;
+      total_size = search_state.saved_total_size;
+      get_burst = search_state.saved_get_burst;
+      back_check = search_state.saved_back_check;
+      pc = search_state.outer_search_resume_pt;
+      if (0)
+	{
+	return_continuation:
+	  if (save_state)
+	    {
+	      *save_state = search_state;
+	      save_state->saved_regs = regs;
+	      save_state->saved_rxb = rxb;
+	      save_state->saved_startpos = startpos;
+	      save_state->saved_range = range;
+	      save_state->saved_stop = stop;
+	      save_state->saved_total_size = total_size;
+	      save_state->saved_get_burst = get_burst;
+	      save_state->saved_back_check = back_check;
+	      save_state->outer_search_resume_pt = pc;
+	    }
+	  return rx_search_continuation;
+	}
+    }
+
+  switch (pc)
+    {
+    case rx_outer_start:
+      search_state.ret_val = rx_search_fail;
+      (  search_state.lparen
+       = search_state.rparen
+       = search_state.best_lpspace
+       = search_state.best_rpspace
+       = 0);
+      
+      /* figure the number of registers we may need for use in backreferences.
+       * the number here includes an element for register zero.  
+       */
+      search_state.num_regs = rxb->re_nsub + 1;
+      
+      
+      /* check for out-of-range startpos.  */
+      if ((startpos < 0) || (startpos > total_size))
+	return rx_search_fail;
+      
+      /* fix up range if it might eventually take us outside the string. */
+      {
+	int endpos;
+	endpos = startpos + range;
+	if (endpos < -1)
+	  range = (-1 - startpos);
+	else if (endpos > (total_size + 1))
+	  range = total_size - startpos;
+      }
+      
+      /* if the search isn't to be a backwards one, don't waste time in a
+       * long search for a pattern that says it is anchored.
+       */
+      if (rxb->begbuf_only && (range > 0))
+	{
+	  if (startpos > 0)
+	    return rx_search_fail;
+	  else
+	    range = 1;
+	}
+      
+      /* decide whether to use internal or user-provided reg buffers. */
+      if (!regs || rxb->no_sub)
+	{
+	  search_state.best_lpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_rpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_lparen = search_state.best_lpspace;
+	  search_state.best_rparen = search_state.best_rpspace;
+	}
+      else
+	{	
+	  /* have the register data arrays been allocated?  */
+	  if (rxb->regs_allocated == REGS_UNALLOCATED)
+	    { /* no.  so allocate them with malloc.  we need one
+		 extra element beyond `search_state.num_regs' for the `-1' marker
+		 gnu code uses.  */
+	      regs->num_regs = MAX (RE_NREGS, rxb->re_nsub + 1);
+	      regs->start = ((regoff_t *)
+			     malloc (regs->num_regs * sizeof ( regoff_t)));
+	      regs->end = ((regoff_t *)
+			   malloc (regs->num_regs * sizeof ( regoff_t)));
+	      if (regs->start == 0 || regs->end == 0)
+		return rx_search_error;
+	      rxb->regs_allocated = REGS_REALLOCATE;
+	    }
+	  else if (rxb->regs_allocated == REGS_REALLOCATE)
+	    { /* yes.  if we need more elements than were already
+		 allocated, reallocate them.  if we need fewer, just
+		 leave it alone.  */
+	      if (regs->num_regs < search_state.num_regs + 1)
+		{
+		  regs->num_regs = search_state.num_regs + 1;
+		  regs->start = ((regoff_t *)
+				 realloc (regs->start,
+					  regs->num_regs * sizeof (regoff_t)));
+		  regs->end = ((regoff_t *)
+			       realloc (regs->end,
+					regs->num_regs * sizeof ( regoff_t)));
+		  if (regs->start == 0 || regs->end == 0)
+		    return rx_search_error;
+		}
+	    }
+	  else if (rxb->regs_allocated != REGS_FIXED)
+	    return rx_search_error;
+	  
+	  if (regs->num_regs < search_state.num_regs + 1)
+	    {
+	      search_state.best_lpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_rpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_lparen = search_state.best_lpspace;
+	      search_state.best_rparen = search_state.best_rpspace;
+	    }
+	  else
+	    {
+	      search_state.best_lparen = regs->start;
+	      search_state.best_rparen = regs->end;
+	    }
+	}
+      
+      search_state.lparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+      search_state.rparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)); 
+      
+      if (! (   search_state.best_rparen
+	     && search_state.best_lparen
+	     && search_state.lparen && search_state.rparen))
+	return rx_search_error;
+      
+      search_state.best_last_l = search_state.best_last_r = -1;
+      
+      search_state.translate = (rxb->translate
+				? rxb->translate
+				: rx_id_translation);
+      
+      
+      
+      /*
+       * two nfa's were compiled.  
+       * `0' is complete.
+       * `1' faster but gets registers wrong and ends too soon.
+       */
+      search_state.nfa_choice = (regs && !rxb->least_subs) ? '\0' : '\1';
+      
+      /* we have the option to look for the best match or the first
+       * one we can find.  if the user isn't asking for register information,
+       * we don't need to find the best match.
+       */
+      search_state.first_found = !regs;
+      
+      if (range >= 0)
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = 1;
+	}
+      else
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = -1;
+	}
+      
+      /* the vacuous search always turns up nothing. */
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos > search_state.outer_pos.search_end)
+	  : (startpos < search_state.outer_pos.search_end))
+	return rx_search_fail;
+      
+      /* now we build the starting state of the supernfa. */
+      {
+	struct rx_superset * start_contents;
+	struct rx_nfa_state_set * start_nfa_set;
+	
+	/* we presume here that the nfa start state has only one
+	 * possible future with no side effects.  
+	 */
+	start_nfa_set = rxb->start->futures->destset;
+	if (   rxb->rx.start_set
+	    && (rxb->rx.start_set->starts_for == &rxb->rx))
+	  start_contents = rxb->rx.start_set;
+	else
+	  {
+	    start_contents =
+	      rx_superstate_eclosure_union (&rxb->rx,
+					    rx_superset_cons (&rxb->rx, 0, 0),
+					    start_nfa_set);
+	    
+	    if (!start_contents)
+	      return rx_search_fail;
+	    
+	    start_contents->starts_for = &rxb->rx;
+	    rxb->rx.start_set = start_contents;
+	  }
+	if (   start_contents->superstate
+	    && (start_contents->superstate->rx_id == rxb->rx.rx_id))
+	  {
+	    search_state.start_super = start_contents->superstate;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	  }
+	else
+	  {
+	    rx_protect_superset (&rxb->rx, start_contents);
+	    
+	    search_state.start_super = rx_superstate (&rxb->rx, start_contents);
+	    if (!search_state.start_super)
+	      return rx_search_fail;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	    rx_release_superset (&rxb->rx, start_contents);
+	  }
+      }
+      
+      
+      /* The outer_pos tracks the position within the strings
+       * as seen by loop that calls fastmap_search.
+       *
+       * The caller supplied get_burst function actually 
+       * gives us pointers to chars.
+       * 
+       * Communication with the get_burst function is through an
+       * rx_string_position structure.  Here, the structure for
+       * outer_pos is initialized.   It is set to point to the
+       * NULL string, at an offset of STARTPOS.  STARTPOS is out
+       * of range of the NULL string, so the first call to 
+       * getburst will patch up the rx_string_position to point
+       * to valid characters.
+       */
+
+      (  search_state.outer_pos.string
+       = search_state.outer_pos.end
+       = 0);
+
+      search_state.outer_pos.offset = 0;
+      search_state.outer_pos.size = 0;
+      search_state.outer_pos.pos = (unsigned char *)startpos;
+      init_fastmap (rxb, &search_state);
+
+      search_state.fastmap_resume_pt = rx_fastmap_start;
+    case rx_outer_fastmap:
+      /* do { */
+    pseudo_do:
+      {
+	{
+	  int fastmap_state;
+	  fastmap_state = fastmap_search (rxb, stop, get_burst, app_closure,
+					  &search_state);
+	  switch (fastmap_state)
+	    {
+	    case rx_fastmap_continuation:
+	      pc = rx_outer_fastmap;
+	      goto return_continuation;
+	    case rx_fastmap_fail:
+	      goto finish;
+	    case rx_fastmap_ok:
+	      break;
+	    }
+	}
+	
+	/* now the fastmap loop has brought us to a plausible 
+	 * starting point for a match.  so, it's time to run the
+	 * nfa and see if a match occured.
+	 */
+	startpos = (  search_state.outer_pos.pos
+		    - search_state.outer_pos.string
+		    + search_state.outer_pos.offset);
+/*|*/	if ((range > 0) && (startpos == search_state.outer_pos.search_end))
+/*|*/	  goto finish;
+      }
+
+      search_state.test_match_resume_pt = rx_test_start;
+      /* do interrupted for entry point... */
+    case rx_outer_test:
+      /* ...do continued */
+      {
+	goto test_match;
+      test_returns_to_search:
+	switch (test_state)
+	  {
+	  case rx_test_continuation:
+	    pc = rx_outer_test;
+	    goto return_continuation;
+	  case rx_test_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_test_fail:
+	    break;
+	  case rx_test_ok:
+	    goto finish;
+	  }
+	search_state.outer_pos.pos += search_state.outer_pos.search_direction;
+	startpos += search_state.outer_pos.search_direction;
+#if 0
+/*|*/	if (search_state.test_pos.pos < search_state.test_pos.end)
+/*|*/	  break;
+#endif
+      }
+      /* do interrupted for entry point... */
+    case rx_outer_restore_pos:
+      {
+	int x;
+	x = get_burst (&search_state.outer_pos, app_closure, stop);
+	switch (x)
+	  {
+	  case rx_get_burst_continuation:
+	    pc = rx_outer_restore_pos;
+	    goto return_continuation;
+	  case rx_get_burst_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_get_burst_no_more:
+	    if (rxb->can_match_empty)
+	      break;
+	    goto finish;
+	  case rx_get_burst_ok:
+	    break;
+	  }
+      } /* } while (...see below...) */
+
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos < search_state.outer_pos.search_end)
+	  : (startpos > search_state.outer_pos.search_end))
+	goto pseudo_do;
+
+	
+    finish:
+      uninit_fastmap (rxb, &search_state);
+      if (search_state.start_super)
+	rx_unlock_superstate (&rxb->rx, search_state.start_super);
+      
+#ifdef regex_malloc
+      if (search_state.lparen) free (search_state.lparen);
+      if (search_state.rparen) free (search_state.rparen);
+      if (search_state.best_lpspace) free (search_state.best_lpspace);
+      if (search_state.best_rpspace) free (search_state.best_rpspace);
+#endif
+      return search_state.ret_val;
+    }
+
+
+ test_match:
+  {
+    enum rx_test_match_entry test_pc;
+    int inx;
+    test_pc = search_state.test_match_resume_pt;
+    if (test_pc == rx_test_start)
+      {
+#ifdef RX_DEBUG
+	search_state.backtrack_depth = 0;
+#endif
+	search_state.last_l = search_state.last_r = 0;
+	search_state.lparen[0] = startpos;
+	search_state.super = search_state.start_super;
+	search_state.c = search_state.nfa_choice;
+	search_state.test_pos.pos = search_state.outer_pos.pos - 1;    
+	search_state.test_pos.string = search_state.outer_pos.string;
+	search_state.test_pos.end = search_state.outer_pos.end;
+	search_state.test_pos.offset = search_state.outer_pos.offset;
+	search_state.test_pos.size = search_state.outer_pos.size;
+	search_state.test_pos.search_direction = 1;
+	search_state.counter_stack = 0;
+	search_state.backtrack_stack = 0;
+	search_state.backtrack_frame_bytes =
+	  (sizeof (struct rx_backtrack_frame)
+	   + (rxb->match_regs_on_stack
+	      ? sizeof (regoff_t) * (search_state.num_regs + 1) * 2
+	      : 0));
+	search_state.chunk_bytes = search_state.backtrack_frame_bytes * 64;
+	search_state.test_ret = rx_test_line_finished;
+	search_state.could_have_continued = 0;
+      }  
+    /* This is while (1)...except that the body of the loop is interrupted 
+     * by some alternative entry points.
+     */
+  pseudo_while_1:
+    switch (test_pc)
+      {
+      case rx_test_cache_hit_loop:
+	goto resume_continuation_1;
+      case rx_test_backreference_check:
+	goto resume_continuation_2;
+      case rx_test_backtrack_return:
+	goto resume_continuation_3;
+      case rx_test_start:
+#ifdef RX_DEBUG
+	/* There is a search tree with every node as set of deterministic
+	 * transitions in the super nfa.  For every branch of a 
+	 * backtrack point is an edge in the tree.
+	 * This counts up a pre-order of nodes in that tree.
+	 * It's saved on the search stack and printed when debugging. 
+	 */
+	search_state.line_no = 0;
+	search_state.lines_found = 0;
+#endif
+	
+      top_of_cycle:
+	/* A superstate is basicly a transition table, indexed by 
+	 * characters from the string being tested, and containing 
+	 * RX_INX (`instruction frame') structures.
+	 */
+	search_state.ifr = &search_state.super->transitions [search_state.c];
+	
+      recurse_test_match:
+	/* This is the point to which control is sent when the
+	 * test matcher `recurses'.  Before jumping here, some variables
+	 * need to be saved on the stack and the next instruction frame
+	 * has to be computed.
+	 */
+	
+      restart:
+	/* Some instructions don't advance the matcher, but just
+	 * carry out some side effects and fetch a new instruction.
+	 * To dispatch that new instruction, `goto restart'.
+	 */
+	
+	{
+	  struct rx_inx * next_tr_table;
+	  struct rx_inx * this_tr_table;
+	  /* The fastest route through the loop is when the instruction 
+	   * is RX_NEXT_CHAR.  This case is detected when SEARCH_STATE.IFR->DATA
+	   * is non-zero.  In that case, it points to the next
+	   * superstate. 
+	   *
+	   * This allows us to not bother fetching the bytecode.
+	   */
+	  next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	  this_tr_table = search_state.super->transitions;
+	  while (next_tr_table)
+	    {
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		{
+		  struct rx_superset * setp;
+		  
+		  fprintf (stderr, "%d %d>> re_next_char @ %d (%d)",
+			   search_state.line_no,
+			   search_state.backtrack_depth,
+			   (search_state.test_pos.pos - search_state.test_pos.string
+			    + search_state.test_pos.offset), search_state.c);
+		  
+		  search_state.super =
+		    ((struct rx_superstate *)
+		     ((char *)this_tr_table
+		      - ((unsigned long)
+			 ((struct rx_superstate *)0)->transitions)));
+		  
+		  setp = search_state.super->contents;
+		  fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+			   rxb->rx.rx_id, setp);
+		  while (setp)
+		    {
+		      fprintf (stderr, "%d ", setp->id);
+		      setp = setp->cdr;
+		    }
+		  fprintf (stderr, "\n");
+		}
+#endif
+	      this_tr_table = next_tr_table;
+	      ++search_state.test_pos.pos;
+	      if (search_state.test_pos.pos == search_state.test_pos.end)
+		{
+		  int burst_state;
+		try_burst_1:
+		  burst_state = get_burst (&search_state.test_pos,
+					   app_closure, stop);
+		  switch (burst_state)
+		    {
+		    case rx_get_burst_continuation:
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      test_pc = rx_test_cache_hit_loop;
+		      goto test_return_continuation;
+		      
+		    resume_continuation_1:
+		      /* Continuation one jumps here to do its work: */
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      goto try_burst_1;
+		      
+		    case rx_get_burst_ok:
+		      /* get_burst succeeded...keep going */
+		      break;
+		      
+		    case rx_get_burst_no_more:
+		      search_state.test_ret = rx_test_line_finished;
+		      search_state.could_have_continued = 1;
+		      goto test_do_return;
+		      
+		    case rx_get_burst_error:
+		      /* An error... */
+		      search_state.test_ret = rx_test_internal_error;
+		      goto test_do_return;
+		    }
+		}
+	      search_state.c = *search_state.test_pos.pos;
+	      search_state.ifr = this_tr_table + search_state.c;
+	      next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	    } /* Fast loop through cached transition tables */
+	  
+	  /* Here when we ran out of cached next-char transitions. 
+	   * So, it will be necessary to do a more expensive
+	   * dispatch on the current instruction.  The superstate
+	   * pointer is allowed to become invalid during next-char
+	   * transitions -- now we must bring it up to date.
+	   */
+	  search_state.super =
+	    ((struct rx_superstate *)
+	     ((char *)this_tr_table
+	      - ((unsigned long)
+		 ((struct rx_superstate *)0)->transitions)));
+	}
+	
+	/* We've encountered an instruction other than next-char.
+	 * Dispatch that instruction:
+	 */
+	inx = (int)search_state.ifr->inx;
+#ifdef RX_DEBUG_0
+	if (rx_debug_trace)
+	  {
+	    struct rx_superset * setp = search_state.super->contents;
+	    
+	    fprintf (stderr, "%d %d>> %s @ %d (%d)", search_state.line_no,
+		     search_state.backtrack_depth,
+		     inx_names[inx],
+		     (search_state.test_pos.pos - search_state.test_pos.string
+		      + (test_pos.half == 0 ? 0 : size1)), search_state.c);
+	    
+	    fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+		     rxb->rx.rx_id, setp);
+	    while (setp)
+	      {
+		fprintf (stderr, "%d ", setp->id);
+		setp = setp->cdr;
+	      }
+	    fprintf (stderr, "\n");
+	  }
+#endif
+	switch ((enum rx_opcode)inx)
+	  {
+	  case rx_do_side_effects:
+	    
+	    /*  RX_DO_SIDE_EFFECTS occurs when we cross epsilon 
+	     *  edges associated with parentheses, backreferencing, etc.
+	     */
+	    {
+	      struct rx_distinct_future * df =
+		(struct rx_distinct_future *)search_state.ifr->data_2;
+	      struct rx_se_list * el = df->effects;
+	      /* Side effects come in lists.  This walks down
+	       * a list, dispatching.
+	       */
+	      while (el)
+		{
+		  long effect;
+		  effect = (long)el->car;
+		  if (effect < 0)
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			{
+			  struct rx_superset * setp = search_state.super->contents;
+			  
+			  fprintf (stderr, "....%d %d>> %s\n", search_state.line_no,
+				   search_state.backtrack_depth,
+				   efnames[-effect]);
+			}
+#endif
+		      switch ((enum re_side_effects) effect)
+
+			{
+			case re_se_pushback:
+			  search_state.ifr = &df->future_frame;
+			  if (!search_state.ifr->data)
+			    {
+			      struct rx_superstate * sup;
+			      sup = search_state.super;
+			      rx_lock_superstate (rx, sup);
+			      if (!rx_handle_cache_miss (&rxb->rx,
+							 search_state.super,
+							 search_state.c,
+							 (search_state.ifr
+							  ->data_2)))
+				{
+				  rx_unlock_superstate (rx, sup);
+				  search_state.test_ret = rx_test_internal_error;
+				  goto test_do_return;
+				}
+			      rx_unlock_superstate (rx, sup);
+			    }
+			  /* --search_state.test_pos.pos; */
+			  search_state.c = 't';
+			  search_state.super
+			    = ((struct rx_superstate *)
+			       ((char *)search_state.ifr->data
+				- (long)(((struct rx_superstate *)0)
+					 ->transitions)));
+			  goto top_of_cycle;
+			  break;
+			case re_se_push0:
+			  {
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH (search_state.counter_stack,
+				  sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_iter;
+			    cf->val = 0;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			case re_se_fail:
+			  goto test_do_return;
+			case re_se_begbuf:
+			  if (!AT_STRINGS_BEG ())
+			    goto test_do_return;
+			  break;
+			case re_se_endbuf:
+			  if (!AT_STRINGS_END ())
+			    goto test_do_return;
+			  break;
+			case re_se_wordbeg:
+			  if (   LETTER_P (&search_state.test_pos, 1)
+			      && (   AT_STRINGS_BEG()
+				  || !LETTER_P (&search_state.test_pos, 0)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordend:
+			  if (   !AT_STRINGS_BEG ()
+			      && LETTER_P (&search_state.test_pos, 0)
+			      && (AT_STRINGS_END ()
+				  || !LETTER_P (&search_state.test_pos, 1)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordbound:
+			  if (AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_notwordbound:
+			  if (!AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_hat:
+			  if (AT_STRINGS_BEG ())
+			    {
+			      if (rxb->not_bol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      char pos_c = *search_state.test_pos.pos;
+			      if (   (SRCH_TRANSLATE (pos_c)
+				      == SRCH_TRANSLATE('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			case re_se_dollar:
+			  if (AT_STRINGS_END ())
+			    {
+			      if (rxb->not_eol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      if (   (   SRCH_TRANSLATE (fetch_char
+						    (&search_state.test_pos, 1,
+						     app_closure, stop))
+				      == SRCH_TRANSLATE ('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			  
+			case re_se_try:
+			  /* This is the first side effect in every
+			   * expression.
+			   *
+			   *  FOR NO GOOD REASON...get rid of it...
+			   */
+			  break;
+			  
+			case re_se_pushpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH(search_state.counter_stack,
+				 sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_pushpos;
+			    cf->val = urhere;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			  
+			case re_se_chkpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * cf
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (cf->val == urhere)
+			      goto test_do_return;
+			    cf->val = urhere;
+			    break;
+			  }
+			  break;
+			  
+			case re_se_poppos:
+			  POP(search_state.counter_stack,
+			      sizeof (struct rx_counter_frame));
+			  break;
+			  
+			  
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+#ifdef emacs
+			  /* 
+			   * this release lacks emacs support
+			   */
+#endif
+			  break;
+			case re_se_win:
+			case re_se_lparen:
+			case re_se_rparen:
+			case re_se_backref:
+			case re_se_iter:
+			case re_se_end_iter:
+			case re_se_tv:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  else
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			fprintf (stderr, "....%d %d>> %s %d %d\n", search_state.line_no,
+				 search_state.backtrack_depth,
+				 efnames2[rxb->se_params [effect].se],
+				 rxb->se_params [effect].op1,
+				 rxb->se_params [effect].op2);
+#endif
+		      switch (rxb->se_params [effect].se)
+			{
+			case re_se_win:
+			  /* This side effect indicates that we've 
+			   * found a match, though not necessarily the 
+			   * best match.  This is a fancy assignment to 
+			   * register 0 unless the caller didn't 
+			   * care about registers.  In which case,
+			   * this stops the match.
+			   */
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    if (   (search_state.best_last_r < 0)
+				|| (urhere + 1 > search_state.best_rparen[0]))
+			      {
+				/* Record the best known and keep
+				 * looking.
+				 */
+				int x;
+				for (x = 0; x <= search_state.last_l; ++x)
+				  search_state.best_lparen[x] = search_state.lparen[x];
+				search_state.best_last_l = search_state.last_l;
+				for (x = 0; x <= search_state.last_r; ++x)
+				  search_state.best_rparen[x] = search_state.rparen[x];
+				search_state.best_rparen[0] = urhere + 1;
+				search_state.best_last_r = search_state.last_r;
+			      }
+			    /* If we're not reporting the match-length 
+			     * or other register info, we need look no
+			     * further.
+			     */
+			    if (search_state.first_found)
+			      {
+				search_state.test_ret = rx_test_found_first;
+				goto test_do_return;
+			      }
+			  }
+			  break;
+			case re_se_lparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    int reg = rxb->se_params [effect].op1;
+#if 0
+			    if (reg > search_state.last_l)
+#endif
+			      {
+				search_state.lparen[reg] = urhere + 1;
+				/* In addition to making this assignment,
+				 * we now know that lower numbered regs
+				 * that haven't already been assigned,
+				 * won't be.  We make sure they're
+				 * filled with -1, so they can be
+				 * recognized as unassigned.
+				 */
+				if (search_state.last_l < reg)
+				  while (++search_state.last_l < reg)
+				    search_state.lparen[search_state.last_l] = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_rparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    int reg = rxb->se_params [effect].op1;
+			    search_state.rparen[reg] = urhere + 1;
+			    if (search_state.last_r < reg)
+			      {
+				while (++search_state.last_r < reg)
+				  search_state.rparen[search_state.last_r]
+				    = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_backref:
+			  {
+			    int reg = rxb->se_params [effect].op1;
+			    if (   reg > search_state.last_r
+				|| search_state.rparen[reg] < 0)
+			      goto test_do_return;
+			    
+			    {
+			      int backref_status;
+			    check_backreference:
+			      backref_status
+				= back_check (&search_state.test_pos,
+					      search_state.lparen[reg],
+					      search_state.rparen[reg],
+					      search_state.translate,
+					      app_closure,
+					      stop);
+			      switch (backref_status)
+				{
+				case rx_back_check_continuation:
+				  search_state.saved_reg = reg;
+				  test_pc = rx_test_backreference_check;
+				  goto test_return_continuation;
+				resume_continuation_2:
+				  reg = search_state.saved_reg;
+				  goto check_backreference;
+				case rx_back_check_fail:
+				  /* Fail */
+				  goto test_do_return;
+				case rx_back_check_pass:
+				  /* pass --
+				   * test_pos now advanced to last
+				   * char matched by backref
+				   */
+				  break;
+				}
+			    }
+			    break;
+			  }
+			case re_se_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val == rxb->se_params[effect].op2)
+			      goto test_do_return;
+			    else
+			      ++csp->val;
+			    break;
+			  }
+			case re_se_end_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val < rxb->se_params[effect].op1)
+			      goto test_do_return;
+			    else
+			      {
+				struct rx_counter_frame * source = csp;
+				while (source->inherited_from)
+				  source = source->inherited_from;
+				if (!source || !source->cdr)
+				  {
+				    POP(search_state.counter_stack,
+					sizeof(struct rx_counter_frame));
+				  }
+				else
+				  {
+				    source = source->cdr;
+				    csp->val = source->val;
+				    csp->tag = source->tag;
+				    csp->cdr = 0;
+				    csp->inherited_from = source;
+				  }
+			      }
+			    break;
+			  }
+			case re_se_tv:
+			  /* is a noop */
+			  break;
+			case re_se_try:
+			case re_se_pushback:
+			case re_se_push0:
+			case re_se_pushpos:
+			case re_se_chkpos:
+			case re_se_poppos:
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+			case re_se_begbuf:
+			case re_se_hat:
+			case re_se_wordbeg:
+			case re_se_wordbound:
+			case re_se_notwordbound:
+			case re_se_wordend:
+			case re_se_endbuf:
+			case re_se_dollar:
+			case re_se_fail:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  el = el->cdr;
+		}
+	      /* Now the side effects are done,
+	       * so get the next instruction.
+	       * and move on.
+	       */
+	      search_state.ifr = &df->future_frame;
+	      goto restart;
+	    }
+	    
+	  case rx_backtrack_point:
+	    {
+	      /* A backtrack point indicates that we've reached a
+	       * non-determinism in the superstate NFA.  This is a
+	       * loop that exhaustively searches the possibilities.
+	       *
+	       * A backtracking strategy is used.  We keep track of what
+	       * registers are valid so we can erase side effects.
+	       *
+	       * First, make sure there is some stack space to hold 
+	       * our state.
+	       */
+	      
+	      struct rx_backtrack_frame * bf;
+	      
+	      PUSH(search_state.backtrack_stack,
+		   search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG_0
+	      ++search_state.backtrack_depth;
+#endif
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+	      {
+		bf->stk_super = search_state.super;
+		/* We prevent the current superstate from being
+		 * deleted from the superstate cache.
+		 */
+		rx_lock_superstate (&rxb->rx, search_state.super);
+#ifdef RX_DEBUG_0
+		bf->stk_search_state.line_no = search_state.line_no;
+#endif
+		bf->stk_c = search_state.c;
+		bf->stk_test_pos = search_state.test_pos;
+		bf->stk_last_l = search_state.last_l;
+		bf->stk_last_r = search_state.last_r;
+		bf->df = ((struct rx_super_edge *)
+			  search_state.ifr->data_2)->options;
+		bf->first_df = bf->df;
+		bf->counter_stack_sp = (search_state.counter_stack
+					? search_state.counter_stack->sp
+					: 0);
+		bf->stk_test_ret = search_state.test_ret;
+		if (rxb->match_regs_on_stack)
+		  {
+		    int x;
+		    regoff_t * stk =
+		      (regoff_t *)((char *)bf + sizeof (*bf));
+		    for (x = 0; x <= search_state.last_l; ++x)
+		      stk[x] = search_state.lparen[x];
+		    stk += x;
+		    for (x = 0; x <= search_state.last_r; ++x)
+		      stk[x] = search_state.rparen[x];
+		  }
+	      }
+	      
+	      /* Here is a while loop whose body is mainly a function
+	       * call and some code to handle a return from that
+	       * function.
+	       *
+	       * From here on for the rest of `case backtrack_point' it
+	       * is unsafe to assume that the search_state copies of 
+	       * variables saved on the backtracking stack are valid
+	       * -- so read their values from the backtracking stack.
+	       *
+	       * This lets us use one generation fewer stack saves in
+	       * the call-graph of a search.
+	       */
+	      
+	    while_non_det_options:
+#ifdef RX_DEBUG_0
+	      ++search_state.lines_found;
+	      if (rx_debug_trace)
+		fprintf (stderr, "@@@ %d calls %d @@@\n",
+			 search_state.line_no, search_state.lines_found);
+	      
+	      search_state.line_no = search_state.lines_found;
+#endif
+	      
+	      if (bf->df->next_same_super_edge[0] == bf->first_df)
+		{
+		  /* This is a tail-call optimization -- we don't recurse
+		   * for the last of the possible futures.
+		   */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  
+		  rx_unlock_superstate (&rxb->rx, search_state.super);
+		  POP(search_state.backtrack_stack,
+		      search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG
+		  --search_state.backtrack_depth;
+#endif
+		  goto restart;
+		}
+	      else
+		{
+		  if (search_state.counter_stack)
+		    {
+		      struct rx_counter_frame * old_cf
+			= ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      struct rx_counter_frame * cf;
+		      PUSH(search_state.counter_stack, sizeof (struct rx_counter_frame));
+		      cf = ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      cf->tag = old_cf->tag;
+		      cf->val = old_cf->val;
+		      cf->inherited_from = old_cf;
+		      cf->cdr = 0;
+		    }			
+		  /* `Call' this test-match block */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  goto recurse_test_match;
+		}
+	      
+	      /* Returns in this block are accomplished by
+	       * goto test_do_return.  There are two cases.
+	       * If there is some search-stack left,
+	       * then it is a return from a `recursive' call.
+	       * If there is no search-stack left, then
+	       * we should return to the fastmap/search loop.
+	       */
+	      
+	    test_do_return:
+	      
+	      if (!search_state.backtrack_stack)
+		{
+#ifdef RX_DEBUG_0
+		  if (rx_debug_trace)
+		    fprintf (stderr, "!!! %d bails returning %d !!!\n",
+			     search_state.line_no, search_state.test_ret);
+#endif
+		  
+		  /* No more search-stack -- this test is done. */
+		  if (search_state.test_ret)
+		    goto return_from_test_match;
+		  else
+		    goto error_in_testing_match;
+		}
+	      
+	      /* Returning from a recursive call to 
+	       * the test match block:
+	       */
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		fprintf (stderr, "+++ %d returns %d (to %d)+++\n",
+			 search_state.line_no,
+			 search_state.test_ret,
+			 bf->stk_search_state.line_no);
+#endif
+	      
+	      while (search_state.counter_stack
+		     && (!bf->counter_stack_sp
+			 || (bf->counter_stack_sp
+			     != search_state.counter_stack->sp)))
+		{
+		  POP(search_state.counter_stack,
+		      sizeof (struct rx_counter_frame));
+		}
+	      
+	      if (search_state.test_ret == rx_test_error)
+		{
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      /* If a non-longest match was found and that is good 
+	       * enough, return immediately.
+	       */
+	      if (   (search_state.test_ret == rx_test_found_first)
+		  && search_state.first_found)
+		{
+		  rx_unlock_superstate (&rxb->rx, bf->stk_super);
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      search_state.test_ret = bf->stk_test_ret;
+	      search_state.last_l = bf->stk_last_l;
+	      search_state.last_r = bf->stk_last_r;
+	      bf->df = bf->df->next_same_super_edge[0];
+	      search_state.super = bf->stk_super;
+	      search_state.c = bf->stk_c;
+#ifdef RX_DEBUG_0
+	      search_state.line_no = bf->stk_search_state.line_no;
+#endif
+	      
+	      if (rxb->match_regs_on_stack)
+		{
+		  int x;
+		  regoff_t * stk =
+		    (regoff_t *)((char *)bf + sizeof (*bf));
+		  for (x = 0; x <= search_state.last_l; ++x)
+		    search_state.lparen[x] = stk[x];
+		  stk += x;
+		  for (x = 0; x <= search_state.last_r; ++x)
+		    search_state.rparen[x] = stk[x];
+		}
+	      
+	      {
+		int x;
+	      try_burst_2:
+		x = get_burst (&bf->stk_test_pos, app_closure, stop);
+		switch (x)
+		  {
+		  case rx_get_burst_continuation:
+		    search_state.saved_bf = bf;
+		    test_pc = rx_test_backtrack_return;
+		    goto test_return_continuation;
+		  resume_continuation_3:
+		    bf = search_state.saved_bf;
+		    goto try_burst_2;
+		  case rx_get_burst_no_more:
+		    /* Since we've been here before, it is some kind of
+		     * error that we can't return.
+		     */
+		  case rx_get_burst_error:
+		    search_state.test_ret = rx_test_internal_error;
+		    goto test_do_return;
+		  case rx_get_burst_ok:
+		    break;
+		  }
+	      }
+	      search_state.test_pos = bf->stk_test_pos;
+	      goto while_non_det_options;
+	    }
+	    
+	    
+	  case rx_cache_miss:
+	    /* Because the superstate NFA is lazily constructed,
+	     * and in fact may erode from underneath us, we sometimes
+	     * have to construct the next instruction from the hard way.
+	     * This invokes one step in the lazy-conversion.
+	     */
+	    search_state.ifr = rx_handle_cache_miss (&rxb->rx,
+						     search_state.super,
+						     search_state.c,
+						     search_state.ifr->data_2);
+	    if (!search_state.ifr)
+	      {
+		search_state.test_ret = rx_test_internal_error;
+		goto test_do_return;
+	      }
+	    goto restart;
+	    
+	  case rx_backtrack:
+	    /* RX_BACKTRACK means that we've reached the empty
+	     * superstate, indicating that match can't succeed
+	     * from this point.
+	     */
+	    goto test_do_return;
+	    
+	  case rx_next_char:
+	  case rx_error_inx:
+	  case rx_num_instructions:
+	    search_state.ret_val = 0;
+	    goto test_do_return;
+	  }
+	goto pseudo_while_1;
+      }
+    
+    /* Healthy exits from the test-match loop do a 
+     * `goto return_from_test_match'   On the other hand, 
+     * we might end up here.
+     */
+  error_in_testing_match:
+    test_state = rx_test_error;
+    goto test_returns_to_search;
+    
+    /***** fastmap/search loop body
+     *	      considering the results testing for a match
+     */
+    
+  return_from_test_match:
+    
+    if (search_state.best_last_l >= 0)
+      {
+	if (regs && (regs->start != search_state.best_lparen))
+	  {
+	    bcopy (search_state.best_lparen, regs->start,
+		   regs->num_regs * sizeof (int));
+	    bcopy (search_state.best_rparen, regs->end,
+		   regs->num_regs * sizeof (int));
+	  }
+	if (regs && !rxb->no_sub)
+	  {
+	    int q;
+	    int bound = (regs->num_regs > search_state.num_regs
+			 ? regs->num_regs
+			 : search_state.num_regs);
+	    regoff_t * s = regs->start;
+	    regoff_t * e = regs->end;
+	    for (q = search_state.best_last_l + 1;  q < bound; ++q)
+	      s[q] = e[q] = -1;
+	  }
+	search_state.ret_val = search_state.best_lparen[0];
+	test_state = rx_test_ok;
+	goto test_returns_to_search;
+      }
+    else
+      {
+	test_state = rx_test_fail;
+	goto test_returns_to_search;
+      }
+    
+  test_return_continuation:
+    search_state.test_match_resume_pt = test_pc;
+    test_state = rx_test_continuation;
+    goto test_returns_to_search;
+  }
+}
+
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+#else /* RX_WANT_SE_DEFS */
+  /* Integers are used to represent side effects.
+   *
+   * Simple side effects are given negative integer names by these enums.
+   * 
+   * Non-negative names are reserved for complex effects.
+   *
+   * Complex effects are those that take arguments.  For example, 
+   * a register assignment associated with a group is complex because
+   * it requires an argument to tell which group is being matched.
+   * 
+   * The integer name of a complex effect is an index into rxb->se_params.
+   */
+ 
+  RX_DEF_SE(1, re_se_try, = -1)		/* Epsilon from start state */
+
+  RX_DEF_SE(0, re_se_pushback, = re_se_try - 1)
+  RX_DEF_SE(0, re_se_push0, = re_se_pushback -1)
+  RX_DEF_SE(0, re_se_pushpos, = re_se_push0 - 1)
+  RX_DEF_SE(0, re_se_chkpos, = re_se_pushpos -1)
+  RX_DEF_SE(0, re_se_poppos, = re_se_chkpos - 1)
+
+  RX_DEF_SE(1, re_se_at_dot, = re_se_poppos - 1)	/* Emacs only */
+  RX_DEF_SE(0, re_se_syntax, = re_se_at_dot - 1) /* Emacs only */
+  RX_DEF_SE(0, re_se_not_syntax, = re_se_syntax - 1) /* Emacs only */
+
+  RX_DEF_SE(1, re_se_begbuf, = re_se_not_syntax - 1) /* match beginning of buffer */
+  RX_DEF_SE(1, re_se_hat, = re_se_begbuf - 1) /* match beginning of line */
+
+  RX_DEF_SE(1, re_se_wordbeg, = re_se_hat - 1) 
+  RX_DEF_SE(1, re_se_wordbound, = re_se_wordbeg - 1)
+  RX_DEF_SE(1, re_se_notwordbound, = re_se_wordbound - 1)
+
+  RX_DEF_SE(1, re_se_wordend, = re_se_notwordbound - 1)
+  RX_DEF_SE(1, re_se_endbuf, = re_se_wordend - 1)
+
+  /* This fails except at the end of a line. 
+   * It deserves to go here since it is typicly one of the last steps 
+   * in a match.
+   */
+  RX_DEF_SE(1, re_se_dollar, = re_se_endbuf - 1)
+
+  /* Simple effects: */
+  RX_DEF_SE(1, re_se_fail, = re_se_dollar - 1)
+
+  /* Complex effects.  These are used in the 'se' field of 
+   * a struct re_se_params.  Indexes into the se array
+   * are stored as instructions on nfa edges.
+   */
+  RX_DEF_CPLX_SE(1, re_se_win, = 0)
+  RX_DEF_CPLX_SE(1, re_se_lparen, = re_se_win + 1)
+  RX_DEF_CPLX_SE(1, re_se_rparen, = re_se_lparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_backref, = re_se_rparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_iter, = re_se_backref + 1) 
+  RX_DEF_CPLX_SE(0, re_se_end_iter, = re_se_iter + 1)
+  RX_DEF_CPLX_SE(0, re_se_tv, = re_se_end_iter + 1)
+
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/simplefrenchstem.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/simplefrenchstem.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/simplefrenchstem.c	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      last--;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') last--;
+      if (last>=0 && wordstart[last]=='r') last--;
+      if (last>=0 && wordstart[last]=='e') last--;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	last--;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) last--;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/indexers/mg/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:16  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:37  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/24 06:37:53  rjmcnab
+
+   Incorporated the french stemmer better.
+
+
+ */
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/sptree.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/sptree.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/sptree.c	(revision 21112)
@@ -0,0 +1,394 @@
+/**************************************************************************
+ *
+ * sptree.c -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include	"sysfuncs.h"
+#include	"sptree.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:17  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:50  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define TRACE(w)
+
+#define POOL_SIZE 1024
+
+
+typedef struct node_rec node;
+
+struct node_rec
+  {
+    node *left, *rght;
+    node *par;
+    char *datsun;
+  };
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    node nodes[POOL_SIZE];
+  };
+
+static node *SP_splay ();
+
+#define	CNULL	((char *) NULL)
+#define	NNULL	((node *) NULL)
+#define	SNULL	((Splay_Tree *) NULL)
+
+
+/*
+ *    createset():
+ *              allocates memory for and initialises a set.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+Splay_Tree *
+SP_createset (int (*cmp) (void *, void *))
+{
+  Splay_Tree *S;
+
+  if (!(S = (Splay_Tree *) malloc (sizeof (Splay_Tree))))
+    return (SNULL);
+  S->cmp = cmp;
+  S->root = CNULL;
+  S->no_of_items = 0;
+  S->mem_in_use = sizeof (Splay_Tree);
+  S->pool = NULL;
+  S->pool_chunks = 1024;
+  return S;
+}				/* createset() */
+
+
+
+static node *
+SP_get_node (Splay_Tree * S)
+{
+  struct pool *p = S->pool;
+  if (!p || p->pos == p->length)
+    {
+      p = malloc (sizeof (struct pool));
+      if (!p)
+	return NULL;
+      S->mem_in_use += sizeof (struct pool);
+      p->length = POOL_SIZE;
+      p->pos = 0;
+      p->next = S->pool;
+      S->pool = p;
+    }
+  return &(p->nodes[p->pos++]);
+}
+
+
+
+/*
+ *    SP_freeset():
+ *              free the memory allocated to a splay tree. Note this Does not
+ *              free the data in the tree you must do this yourself.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+void 
+SP_freeset (Splay_Tree * S)
+{
+  while (S->pool)
+    {
+      struct pool *p = S->pool;
+      S->pool = p->next;
+      free (p);
+    }
+  free (S);
+}
+
+
+/*
+ *    insert():
+ *              inserts the item given into the set given in the order
+ *              specified by the comparison function in the set 
+ *              definition.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+
+void *
+SP_insert (void *item, Splay_Tree * S)
+{
+  int dir = 0;
+  register node *curr, *prev;
+  node *tmp;
+
+  prev = NNULL;
+  curr = (node *) S->root;
+  for (; curr;)
+    {
+      prev = curr;
+      dir = S->cmp (item, curr->datsun);
+      if (dir < 0)
+	curr = curr->left;
+      else
+	curr = curr->rght;
+    }
+  if (!(tmp = SP_get_node (S)))
+    return NULL;
+  tmp->datsun = item;
+
+  if (prev == NNULL)
+    S->root = (char *) tmp;
+  else if (dir < 0)
+    prev->left = tmp;
+  else
+    prev->rght = tmp;
+
+  tmp->par = prev;
+  tmp->left = NNULL;
+  tmp->rght = NNULL;
+  S->no_of_items++;
+  S->root = (char *) SP_splay (tmp);
+  return (tmp->datsun);
+}				/* insert() */
+
+/*
+ *    get_first():
+ *              returns a pointer to the first item in the set. If the set 
+ *              is empty a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_first (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  for (curr = (node *) S->root, prev = NNULL; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_first() */
+
+/*
+ *    get_next():
+ *              returns the in-order successor of the currently fingered
+ *              item of the set. If no item is fingered or if there is 
+ *              no in-order successor a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_next (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  curr = (node *) S->root;
+  if (!curr || !curr->rght)
+    return (CNULL);
+
+  for (prev = curr, curr = curr->rght; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_next() */
+
+/*
+ *    deletemin():
+ *              removes the first element in the set and returns a
+ *              pointer to it. If the set is empty a NULL pointer 
+ *              is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_deletemin (Splay_Tree * S)
+{
+  node *tmp;
+  char *dat;
+
+  dat = SP_get_first (S);
+  if (dat)
+    {
+      tmp = (node *) S->root;
+      S->root = (char *) tmp->rght;
+      if (S->root)
+	((node *) S->root)->par = NULL;
+    }
+  S->no_of_items--;
+  return dat;
+}				/* deletemin() */
+
+
+/*
+ *    member():
+ *              searches the given set for an item which matches (has
+ *              the same key) and returns a pointer to a copy of that item.
+ *              If no matching item is found a NULL pointer is 
+ *              returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_member (void *dat, Splay_Tree * S)
+{
+  register node *curr;
+  register int dir;
+  static int calls = 0;
+
+  TRACE ("in member");
+  for (curr = (node *) S->root; curr;)
+    {
+      TRACE ("call to cmp");
+      dir = S->cmp (dat, curr->datsun);
+      TRACE ("out of cmp");
+      if (dir < 0)
+	curr = curr->left;
+      else if (dir > 0)
+	curr = curr->rght;
+      else
+	{
+	  /* splay occasionaly to speed up execution */
+	  if ((calls++ % 4) == 0 && curr)
+	    S->root = (char *) SP_splay (curr);
+	  TRACE ("out of member");
+	  return (curr->datsun);
+	}
+    }
+  TRACE ("out of member");
+  return (CNULL);
+}				/* member() */
+
+#define	ROTATEL(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->rght = p->left;			\
+		if(q->rght)				\
+			q->rght->par = q;		\
+		p->left = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+#define	ROTATER(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->left = p->rght;			\
+		if(q->left)				\
+			q->left->par = q;		\
+		p->rght = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+
+/*
+ *    splay():
+ *              i
+ *
+ *      Accesses outside variables:     i
+ *
+ *      Return value...:                i
+ */
+
+static node *
+SP_splay (p)
+     register node *p;
+{
+  register node *q, *g;
+
+  q = p->par;
+  /* splay the tree about p */
+  while (q != NNULL)
+    {
+      g = q->par;
+      if (p == q->left)
+	{
+	  if (g == NNULL)
+	    ROTATER (p, q);
+	  else if (q == g->left)
+	    {
+	      ROTATER (q, g);
+	      ROTATER (p, q);
+	    }
+	  else
+	    {
+	      ROTATER (p, q);
+	      ROTATEL (p, g);
+	    }
+	}
+      else
+	{
+	  if (g == NNULL)
+	    ROTATEL (p, q);
+	  else if (q == g->rght)
+	    {
+	      ROTATEL (q, g);
+	      ROTATEL (p, q);
+	    }
+	  else
+	    {
+	      ROTATEL (p, q);
+	      ROTATER (p, g);
+	    }
+	}
+      q = p->par;
+    }
+  return (p);
+}
Index: /main/tags/3.01/indexers/mg/lib/sptree.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/sptree.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/sptree.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * sptree.h -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef SPTREE_H
+#define SPTREE_H
+
+typedef struct set_rec Splay_Tree;
+
+struct set_rec
+  {
+    int no_of_items;
+    unsigned long mem_in_use;
+    int (*cmp) (void *, void *);
+    void *root;
+    void *pool;
+    int pool_chunks;
+  };
+
+Splay_Tree *SP_createset (int (*cmp) (void *, void *));
+void *SP_insert (void *item, Splay_Tree * S);
+void *SP_get_first (Splay_Tree * S);
+void *SP_get_next (Splay_Tree * S);
+void *SP_deletemin (Splay_Tree * S);
+void *SP_member (void *dat, Splay_Tree * S);
+void SP_freeset (Splay_Tree * S);
+#endif
Index: /main/tags/3.01/indexers/mg/lib/strcasecmp.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/strcasecmp.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/strcasecmp.c	(revision 21112)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(s1, s2)
+    char *s1, *s2;
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/indexers/mg/lib/strstr.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/strstr.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/strstr.c	(revision 21112)
@@ -0,0 +1,44 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (s1, s2)
+     char *s1;
+     char *s2;
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, i++, s++)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; p1++, p2++)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/lib/timing.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/timing.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/timing.c	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * timing.c -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:14:17  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:05  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:53  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "timing.h"
+
+
+/*
+ * Return the time in seconds since 00:00:00 GMT, Jan. 1, 1970 to the
+ * best precision possible
+ *
+ */
+double 
+RealTime (void)
+{
+  return ((double) time (NULL));
+}
+
+
+
+/*
+ * Return the amount of system and user CPU used by the process to date to 
+ * the best precision possible. If user is non-null then it is initialised to 
+ * the user time. If sys is non-null then it is initialised to the system time.
+ *
+ */
+double 
+CPUTime (double *user, double *sys)
+{
+
+#if defined(HAVE_TIMES)
+
+  struct tms buffer;
+  static double clk_tck = 0;
+  double u, s;
+
+  times (&buffer);
+
+  if (clk_tck == 0)
+    clk_tck = CLK_TCK;
+
+  u = (double) buffer.tms_utime / clk_tck;
+  s = (double) buffer.tms_stime / clk_tck;
+  if (user)
+    *user = u;
+  if (sys)
+    *sys = s;
+  return u + s;
+
+#elif defined(HAVE_GETRUSAGE)
+
+  struct rusage ruse;
+  getrusage (RUSAGE_SELF, &ruse);
+  if (user)
+    *user = (double) ruse.ru_utime.tv_sec +
+      (double) ruse.ru_utime.tv_usec / 1000000;
+  if (sys)
+    *sys = (double) ruse.ru_stime.tv_sec +
+      (double) ruse.ru_stime.tv_usec / 1000000;
+  return ((double) ruse.ru_utime.tv_sec + ruse.ru_stime.tv_sec +
+	((double) ruse.ru_utime.tv_usec + ruse.ru_stime.tv_usec) / 1000000);
+
+#else
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (user) *user = 0.0;
+  if (sys) *sys = 0.0;
+  return 0.0;
+#else
+  -->Can NOT find a system time routine to use ! !!
+#endif  /* __WIN32__ */
+
+#endif
+
+}
+
+
+
+
+/*
+ * Get the Real and CPU time and store them in a ProgTime structure
+ */
+void 
+GetTime (ProgTime * StartTime)
+{
+  StartTime->RealTime = RealTime ();
+  StartTime->CPUTime = CPUTime (NULL, NULL);
+}
+
+
+
+
+/*
+ * Display the Real and CPU time elapsed since the StartTime anf FinishTime 
+ * structures were initialised. If FinishTime is NULL then FinishTime is
+ * Now.
+ */
+char *
+ElapsedTime (ProgTime * StartTime,
+	     ProgTime * FinishTime)
+{
+  static char buf[50];
+  double Real, CPU, hour, min, sec;
+  if (!FinishTime)
+    {
+      Real = RealTime () - StartTime->RealTime;
+      CPU = CPUTime (NULL, NULL) - StartTime->CPUTime;
+    }
+  else
+    {
+      Real = FinishTime->RealTime - StartTime->RealTime;
+      CPU = FinishTime->CPUTime - StartTime->CPUTime;
+    }
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f cpu, %02d:%02d:%02d elapsed.",
+	   hour, min, sec,
+	   ((int) ceil (Real)) / 3600,
+	   (((int) ceil (Real)) % 3600) / 60, ((int) ceil (Real)) % 60);
+  return (buf);
+}
+
+
+#define MILLION 1000000
+
+/* ===============================================================================
+ * Function: cputime_string
+ * Description:
+ *      Prints out given cpu time into a string buffer.
+ * Input: cpu-time in clock_t or timeval format
+ * Output: pointer to internal buffer 
+ * =============================================================================== */
+
+#ifdef HAVE_TIMES
+char *
+cputime_string (clock_t clk)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) clk / (double) CLK_TCK;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#else
+char *
+cputime_string (struct timeval *t)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) t->tv_sec + (double) t->tv_usec / MILLION;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#endif
+
+
+
+/* ===============================================================================
+ * Function: time_normalise
+ * Description:
+ *      If the micro-second component is negative or over a million,
+ *      then must take the over/under flow and add it to the second component.
+ * Usage:
+ *      Use this routine when you have just been doing straight addition and
+ *      subtraction on the micro-second component. For this could lead to a
+ *      negative number or over a million.
+ *
+ * =============================================================================== */
+
+#ifndef HAVE_TIMES
+
+#ifndef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+static 
+#endif
+void
+time_normalise (struct timeval *t)
+{
+  while (t->tv_usec < 0)
+    {
+      t->tv_usec += MILLION;
+      t->tv_sec--;
+    }
+  while (t->tv_usec > MILLION)
+    {
+      t->tv_usec -= MILLION;
+      t->tv_sec++;
+    }
+}
+#endif
Index: /main/tags/3.01/indexers/mg/lib/timing.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/timing.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/timing.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * timing.h -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TIMING
+#define H_TIMING
+
+
+#ifdef HAVE_TIMES
+# if TIME_WITH_SYS_TIME
+#  include <sys/times.h>
+#  include <time.h>
+#  ifndef CLK_TCK
+#   define CLK_TCK sysconf(_SC_CLK_TCK)
+#  endif
+# else
+#  if HAVE_SYS_TIME_H
+#   include <sys/time.h>
+#  else
+#   include <time.h>
+#  endif
+# endif
+#else
+/* [RPAP - Feb 97: WIN32 Port] */
+# ifdef __WIN32__
+   struct timeval { long tv_sec, tv_usec; };
+# else
+#  include <sys/time.h>
+#  include <sys/resource.h>
+# endif
+#endif
+
+
+typedef struct
+  {
+    double RealTime, CPUTime;
+  }
+ProgTime;
+
+double RealTime (void);
+
+double CPUTime (double *user, double *sys);
+
+void GetTime (ProgTime * StartTime);
+
+/* FinishTime may be NULL */
+char *ElapsedTime (ProgTime * StartTime, ProgTime * FinishTime);
+
+#ifdef HAVE_TIMES
+char *cputime_string (clock_t clk);
+#else
+void time_normalise (struct timeval *t);
+char *cputime_string (struct timeval *t);
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/unitool.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/unitool.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/unitool.c	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; i--) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; i++) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      curpos++;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data;
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) letter_here++;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) digit_here++;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) space_here++;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	i++;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      outhere++;
+      outlen++;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    in_i++;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/indexers/mg/lib/unitool.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mg/lib/win32in.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/win32in.c	(revision 21112)
@@ -0,0 +1,86 @@
+#include "longlong.h"
+
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/indexers/mg/lib/win32in.h
===================================================================
--- /main/tags/3.01/indexers/mg/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/win32in.h	(revision 21112)
@@ -0,0 +1,10 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+#include "longlong.h"
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
Index: /main/tags/3.01/indexers/mg/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/indexers/mg/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif;
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/indexers/mg/man/man1/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_get.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_get.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/indexers/mg/man/man1/mg_get_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/indexers/mg/man/man1/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_passes.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_text_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mg_weights_build.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgbilevel.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgbuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgbuild_4.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/indexers/mg/man/man1/mgdictlist.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgfelics.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgmerge.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgquery.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgquery.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgstat.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgstat.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgtic.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgtic.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgticbuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgticdump.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgticprune.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/man/man1/mgticstat.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/indexers/mg/man/man1/xmg.1
===================================================================
--- /main/tags/3.01/indexers/mg/man/man1/xmg.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/man/man1/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/mkinstalldirs
===================================================================
--- /main/tags/3.01/indexers/mg/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/indexers/mg/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/indexers/mg/src/images/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/Makefile.in	(revision 21112)
@@ -0,0 +1,217 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/images
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+GSDLOS = @gsdlos@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics mgbilevel mgtic mgticbuild mgticprune mgticstat \
+                mgticdump
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/indexers/mg/src/images/WIN32.MAK
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/WIN32.MAK	(revision 21112)
@@ -0,0 +1,187 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+MGHOME = ..\..
+
+CC = cl
+CPPFLAGS = 
+DEFS = -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I"." -I"$(MGHOME)" -I"$(MGHOME)\lib"
+LDFLAGS =
+LIBS = $(MGHOME)\lib\libmg.lib 
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+ 
+o = .obj
+e = .exe
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics$e mgbilevel$e mgtic$e mgticbuild$e mgticprune$e mgticstat$e \
+                mgticdump$e
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics$e: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel$e: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic$e: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild$e: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune$e: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat$e: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump$e: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#ansi2knr: ansi2knr.o
+#	$(LINK) ansi2knr.o $(LIBS)
+#
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+#execinstall: $(EXEC)
+#	$(top_srcdir)/mkinstalldirs $(bindir); \
+#	for name in $(EXEC); do \
+#	  $(INSTALL_PROGRAM) $$name \
+#	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+#execuninstall:
+#	for name in $(EXEC); do \
+#	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	if exist *$o del *$o
+	if exist  _*.c del  _*.c
+	if exist _*.o  del  _*.o 
+	if exist *._c  del *._c
+	if exist *._o  del *._o
+	if exist core  del  core
+	if exist core.* del  core.*
+ 
+clean: mostlyclean
+	if exist *$e del *$e
+ 
+distclean: clean
+	del ansi2knr
+	 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#	for file in $(DISTFILES); do \
+#	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#	  || cp -p $(srcdir)/$$file $(distdir); \
+#	done
+ 
+#Makefile: Makefile.in ../../config.status
+#	cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /main/tags/3.01/indexers/mg/src/images/arithcode.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/arithcode.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/arithcode.c	(revision 21112)
@@ -0,0 +1,471 @@
+/**************************************************************************
+ *
+ * arithcode.c -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *	Arithmetic encoding. Taken largely from Witten, Neal, Cleary,
+ *		CACM, 1987, p520.
+ *	Includes bitfiddling routines.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "arithcode.h"
+
+
+FILE *arith_in, *arith_out;
+
+/*
+   $log$
+ */
+
+static char *RCSID = "$Id$";
+
+#undef BECAREFUL
+#define INTEGRATED
+
+
+codevalue S_low = 0, S_high = 0, S_value = 0;
+long S_bitstofollow = 0;
+int S_buffer = 0, S_bitstogo = 0;
+
+long cmpbytes = 0, rawbytes = 0;
+
+#ifdef INTEGRATED
+long CountOfBitsOut;
+#endif
+
+
+/*==================================*/
+
+#define BITPLUSFOLLOW(b)		\
+{	OUTPUTBIT((b));			\
+	while (bitstofollow)		\
+	{	OUTPUTBIT(!(b));	\
+		bitstofollow -= 1;	\
+	}				\
+}
+
+#ifdef INTEGRATED
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+	CountOfBitsOut++;		\
+}
+#else
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+}
+#endif
+
+#define ADDNEXTINPUTBIT(v)		\
+{	if (bitstogo==0)		\
+	{	buffer = getc(arith_in);	\
+		bitstogo = 8;		\
+	}				\
+	v = (v<<1)+(buffer&1);		\
+	buffer >>= 1;			\
+	bitstogo -=1;			\
+}
+
+/*==================================*/
+
+void
+arithmetic_encode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue H;
+    register long bitstofollow;
+    register int buffer, bitstogo;
+
+    bitstofollow = S_bitstofollow;
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+
+    for (;;)
+      {
+	if (high < H)
+	  {
+	    BITPLUSFOLLOW (0);
+	  }
+	else if (low >= H)
+	  {
+	    BITPLUSFOLLOW (1);
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    bitstofollow++;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high++;
+      }
+    S_bitstofollow = bitstofollow;
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+  }
+}
+
+/*==================================*/
+
+/* Made into a macro 
+
+   codevalue
+   arithmetic_decode_target(totl)
+   register int totl;
+   {    return (((S_value-S_low+1)*totl-1) / (S_high -S_low+1));
+   } 
+ */
+
+/*==================================*/
+
+void
+arithmetic_decode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue value, H;
+    register int buffer, bitstogo;
+
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+    value = S_value;
+
+    for (;;)
+      {
+	if (high < H)
+	  {			/* nothing */
+	  }
+	else if (low >= H)
+	  {
+	    value -= H;
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    value -= firstqtr;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high += 1;
+	ADDNEXTINPUTBIT (value);
+      }
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+    S_value = value;
+  }
+}
+
+/*==================================*/
+
+
+static void
+startoutputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 8;
+}
+
+static void
+startinputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 0;
+}
+
+static void
+doneoutputingbits ()
+{				/* write another byte if necessary */
+  if (S_bitstogo < 8)
+    {
+      putc (S_buffer >> S_bitstogo, arith_out);
+      cmpbytes += 1;
+    }
+}
+
+/*==================================*/
+
+static void
+startencoding ()
+{
+  S_low = 0;
+  S_high = topvalue;
+  S_bitstofollow = 0;
+}
+
+static void
+startdecoding ()
+{
+  register int i;
+  register int buffer, bitstogo;
+  S_low = 0;
+  S_high = topvalue;
+  S_value = 0;
+  bitstogo = S_bitstogo;
+  buffer = S_buffer;
+  for (i = 0; i < codevaluebits; i++)
+    ADDNEXTINPUTBIT (S_value);
+  S_bitstogo = bitstogo;
+  S_buffer = buffer;
+}
+
+static void
+doneencoding ()
+{
+  register long bitstofollow;
+  register int buffer, bitstogo;
+  bitstofollow = S_bitstofollow;
+  buffer = S_buffer;
+  bitstogo = S_bitstogo;
+  bitstofollow += 1;
+  if (S_low < firstqtr)
+    {
+      BITPLUSFOLLOW (0);
+    }
+  else
+    {
+      BITPLUSFOLLOW (1);
+    }
+  S_bitstofollow = bitstofollow;
+  S_buffer = buffer;
+  S_bitstogo = bitstogo;
+}
+
+
+
+void 
+EncodeGammaDist (int Off)
+/* Encode Off with probability distribution derived from the Elias gamma distribution. */
+{
+  int Len, i;
+
+  Off++;
+  for (Len = 31; Len >= 0 && !(Off & (1 << Len)); Len--);
+
+  for (i = Len; i >= 0; i--)
+    arithmetic_encode (0, 1, 2);
+  arithmetic_encode (1, 2, 2);
+
+  for (i = Len - 1; i >= 0; i--)
+    if (Off & (1 << i))
+      arithmetic_encode (1, 2, 2);
+    else
+      arithmetic_encode (0, 1, 2);
+}
+
+
+
+int 
+DecodeGammaDist ()
+/* Decode next Off using probability distribution derived from the Elias gamma distribution. Inverse of EncodeGammaDist. */
+{
+  int i = 1, Len = 0, bit;
+
+  do
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	{
+	  arithmetic_decode (0, 1, 2);
+	  Len++;
+	}
+    }
+  while (!bit);
+
+  Len--;
+  while (Len-- > 0)
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	arithmetic_decode (0, 1, 2);
+      i = (i << 1) | bit;
+    }
+  return (i - 1);
+}
+
+
+
+
+
+void 
+EncodeGammaSigned (int snum, int *pos, int *neg)
+{
+  if (snum >= 0)
+    arithmetic_encode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_encode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++;
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+  EncodeGammaDist (abs (snum));
+}
+
+
+int 
+DecodeGammaSigned (int *pos, int *neg)
+{
+  int signedpos = 1;
+  if (arithmetic_decode_target (*pos + (*neg)) < *pos)
+    arithmetic_decode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_decode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++, signedpos = 0;
+
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+
+  if (signedpos)
+    return DecodeGammaDist ();
+  else
+    return (-DecodeGammaDist ());
+}
+
+
+void 
+InitArithEncoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startoutputingbits ();
+  startencoding ();
+}
+
+void 
+InitArithDecoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startinputingbits ();
+  startdecoding ();
+}
+
+void 
+CloseDownArithEncoding (void)
+{
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void 
+CloseDownArithDecoding (void)
+{
+}
+
+
+
+
+
+/* encode that famous number, 42 uses 12 bits */
+void 
+EncodeChecksum ()
+{
+  EncodeGammaDist (42);
+}
+
+void 
+DecodeChecksum (char str[])
+{
+  if (DecodeGammaDist () != 42)
+    {
+      fprintf (stderr, "%s: checksum error, file corrupt.\n", str);
+      exit (1);
+    }
+}
Index: /main/tags/3.01/indexers/mg/src/images/arithcode.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/arithcode.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/arithcode.h	(revision 21112)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * arithcode.h -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef _arithcode_h
+#define _arithcode_h
+
+extern FILE *arith_in, *arith_out;
+
+#define codevaluebits 16
+#if (codevaluebits<16)
+typedef unsigned short codevalue;
+#else
+typedef unsigned long codevalue;
+#endif
+
+#define topvalue ((codevalue)((1<<codevaluebits)-1))
+#define maxfrequency ((topvalue+1)/4 + 1)
+
+#define firstqtr (topvalue/4+1)
+#define half     (2*firstqtr)
+#define thirdqtr (3*firstqtr)
+
+#define	escape_event	U->totalcnt-U->notfound, U->totalcnt, U->totalcnt
+
+void arithmetic_encode ();
+codevalue arithmetic_decode_target ();
+
+#define	arithmetic_decode_target(totl)	\
+	(  ((S_value-S_low+1)*(totl)-1) / (S_high-S_low+1)  )
+
+void arithmetic_decode ();
+
+
+extern codevalue S_low, S_high, S_value;
+extern long S_bitstofollow;
+extern int S_buffer, S_bitstogo;
+
+extern long cmpbytes, rawbytes;
+
+extern long CountOfBitsOut;
+
+void EncodeGammaDist (int Off);
+int DecodeGammaDist ();
+
+void EncodeGammaSigned (int snum, int *pos, int *neg);
+int DecodeGammaSigned (int *pos, int *neg);
+
+
+void InitArithEncoding (void);
+void InitArithDecoding (void);
+
+void CloseDownArithEncoding (void);
+void CloseDownArithDecoding (void);
+
+void EncodeChecksum ();
+void DecodeChecksum (char str[]);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/basictypes.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/basictypes.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/basictypes.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * basictypes.h -- base data types
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file contains information relating to the base data types
+ *
+ **************************************************************************/
+
+#ifndef __basictypes_h
+#define __basictypes_h
+
+typedef unsigned long Pixel;
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/bilevel.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/bilevel.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/bilevel.c	(revision 21112)
@@ -0,0 +1,747 @@
+/**************************************************************************
+ *
+ * bilevel.c -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  
+ *  
+ *  This is the code used to compress a bitmap.  The template is given in
+ *  the following format. 
+ *   '.'  = pixel not used
+ *   'p'  = pixel used in 1st order context (the superset)
+ *   '2'  = pixel used in 2nd order context (the subset)
+ *   ';'  = marker used to denote the end of a row in the template
+ *   '*'  = the current context pixel.
+ *  for instance, the common 22/10 template would be shown as:
+ *     ".ppppp.;"
+ *     "pp222pp;"
+ *     "p22222p;"
+ *     "p22*...;"
+ * 
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "bilevel.h"
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+
+
+#define INITVALUE 2
+#define INCREMENT 4
+
+#define MAX_T_W 32
+
+/* note memory usage is 2*TEMPLATESIZE*sizeof(prob), which is currently 4.9 Mb */
+#define TEMPLATESIZE 612317
+
+
+#define HASH(x) ((x)%TEMPLATESIZE)
+
+
+typedef struct
+  {
+    unsigned short s, n;
+  }
+prob;				/* set and notset */
+typedef struct
+  {
+    short int x, y;
+  }
+coordinate;
+
+coordinate MM[MAX_T_W][MAX_T_W];	/* the actual (x,y) positions */
+unsigned int MMcount[MAX_T_W];	/* number of bits in this row of the template */
+unsigned int MMmask[MAX_T_W];	/* the "current" mask value */
+unsigned int MMbits[MAX_T_W];	/* the bitmask to mask out higher bits */
+unsigned int MMrows;		/* total number of rows */
+unsigned int MM2level[MAX_T_W];	/* bitmask to calculate the 2nd level mask */
+int twolevel, curr_row, surr;
+
+prob *P = NULL;			/* probability array, up to N bits */
+prob *P2 = NULL;
+
+void 
+bl_clearmodel ()
+{
+  int i;
+
+  P = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  P2 = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P2)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  for (i = 0; i < TEMPLATESIZE; i++)
+    {
+      P[i].s = P[i].n = INITVALUE;
+      P2[i].s = P2[i].n = INITVALUE;
+    }
+}
+
+
+void 
+bl_freemodel ()
+{
+  free (P);
+  free (P2);
+}
+
+
+
+
+#define update_counts(p, pixel)                        \
+    pixel ? (p->s+=INCREMENT) : (p->n+=INCREMENT);     \
+                                                       \
+    if(p->s+p->n >= (unsigned)(maxfrequency)) {        \
+       p->s=(p->s+1)/(unsigned)2;                      \
+       p->n=(p->n+1)/(unsigned)2;                      \
+    }
+
+
+
+
+/* mask is the bigger one, mask2l is smaller one */
+static void 
+bl_encode_in_context (unsigned long mask, unsigned long mask2l, unsigned long pixel)
+{
+  register int twol, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+  twol = twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT));
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_encode (0, p->s, t);
+      else
+	arithmetic_encode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_encode (0, p2->s, t);
+      else
+	arithmetic_encode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+}
+
+
+static void 
+bl_decode_in_context (unsigned long mask, unsigned long mask2l, unsigned long *val)
+{
+  register int twol, pixel, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+
+  if (twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT)))
+    {
+      pixel = arithmetic_decode_target (p2->s + p2->n) < p2->s, twol = 1;
+    }
+  else
+    {
+      pixel = arithmetic_decode_target (p->s + p->n) < p->s, twol = 0;
+    }
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_decode (0, p->s, t);
+      else
+	arithmetic_decode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_decode (0, p2->s, t);
+      else
+	arithmetic_decode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+
+  *val = pixel;
+}
+
+static void 
+bl_string_to_array (char inputstr[], char parsestr[], int *w, int *h)
+{
+  int r, c;
+  int i, j, lastc, p, l;
+  int xc, yc;
+  int foundstatus;
+
+  surr = twolevel = 0;
+  for (i = 0; i < MAX_T_W; i++)
+    MMcount[i] = MM2level[i] = 0;
+  MMrows = 0;
+
+  for (i = 0; i < (int) strlen (inputstr); i++)
+    {
+      p = inputstr[i];
+      if ((p != '.') && (p != '2') && (p != '*') && (p != 'p') && (p != ';'))
+	{
+	  fprintf (stderr, "Template string can only contain [.p2*;], not \"%c\".\n", inputstr[i]);
+	  exit (1);
+	}
+    }
+
+  /* get the width and height, test rectangularity... */
+  parsestr[l = 0] = 0;
+  for (c = 0, r = 0, lastc = -1, p = 0; p < (int) strlen (inputstr); p++)
+    {
+      if ((inputstr[p] == ';'))
+	{
+	  r++;
+	  if (lastc != -1)
+	    {			/* this is the 2nd row at least */
+	      if (lastc != c)
+		{
+		  fprintf (stderr, "The rows have to be the same width. "
+			   "Row %d has width %d (previous row has width %d)\n", r, c, lastc);
+		  exit (1);
+		}
+	    }
+	  lastc = c;
+	  c = 0;
+	}
+      else
+	{
+	  c++;
+	  parsestr[l] = inputstr[p];
+	  parsestr[++l] = 0;
+	}
+    }
+
+  if (c != 0)
+    lastc = c;
+  *w = lastc;
+  *h = r;
+  if ((*w) <= 0)
+    {
+      fprintf (stderr, "Must have at least one column!\n");
+      exit (1);
+    }
+  if ((*h) <= 0)
+    {
+      fprintf (stderr, "Must have at least one row! Terminate each row with a ';'\n");
+      exit (1);
+    }
+  if ((*w > MAX_T_W) || (*h > MAX_T_W))
+    {
+      fprintf (stderr, "template size bigger than internal threshold of %d\n", MAX_T_W);
+      exit (1);
+    }
+
+  /* get the (x,y) position of the current (*) pixel */
+  for (xc = -1, yc = 0, i = 0; i < *w; i++)
+    for (j = 0; j < *h; j++)
+      if (parsestr[j * (*w) + i] == '*')
+	{
+	  if (xc >= 0)
+	    {
+	      fprintf (stderr, "There can only be ONE current ('*') pixel.\n");
+	      exit (1);
+	    }
+	  xc = i;
+	  yc = j;
+	  curr_row = j + 1;
+	}
+  if (xc < 0)
+    error_msg ("bl_string_to_array", "Error in mask, no current ('*') pixel.", "");
+  /* found the position (xc,yc) */
+
+
+  /* convert parsestr[] into a MM[][] structure */
+  for (j = 0; j < *h; j++)
+    {
+      foundstatus = 0;
+      for (i = 0; i < *w; i++)
+	{
+	  p = j * (*w) + i;
+	  if ((parsestr[p] == 'p') || (parsestr[p] == '2'))
+	    {
+	      if (!foundstatus)
+		foundstatus = 1;
+	      MM2level[j] = MM2level[j] * 2 | (parsestr[p] == '2');
+	      if (parsestr[p] == '2')
+		twolevel = 1;
+
+	      MM[j][MMcount[j]].x = i - xc;
+	      MM[j][MMcount[j]++].y = j - yc;
+	      if (((i - xc) >= 0) && ((j - yc) >= 0))
+		surr = 1;
+	    }
+	  else
+	    {
+	      if (foundstatus == 1)
+		foundstatus = 2;
+	    }
+
+	}
+      if (foundstatus)
+	MMrows++;
+      MMbits[j] = (1 << MMcount[j]) - 1;
+    }
+
+  /* reverse matrix, so [][0] is the closest to the right */
+  {
+    coordinate temp;
+    for (i = 0; i < MMrows; i++)
+      for (j = 0; j < MMcount[i] / (unsigned) 2; j++)
+	{
+	  temp = MM[i][MMcount[i] - 1 - j];
+	  MM[i][MMcount[i] - 1 - j] = MM[i][j];
+	  MM[i][j] = temp;
+	}
+  }
+}
+
+
+
+
+
+
+
+
+void 
+bl_writetemplate (char inputstr[])
+{
+  register int i, j;
+  int w, h;
+  char ind[255];
+  char *str;
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+
+  bl_string_to_array (inputstr, str, &w, &h);
+
+  EncodeGammaDist (surr);
+  EncodeGammaDist (w);
+  EncodeGammaDist (h);
+  ind['.'] = 0;
+  ind['p'] = 1;
+  ind['2'] = 2;
+  ind['*'] = 3;
+  for (j = 0; j < h; j++)
+    for (i = 0; i < w; i++)
+      EncodeGammaDist (ind[(int) str[j * w + i]]);
+
+  free (str);
+}
+
+
+void 
+bl_readtemplate ()
+{
+  register int i, j;
+  int w, h, p;
+  char ind[4];
+  char *inputstr, *str;
+
+  surr = DecodeGammaDist (surr);
+  if (surr)
+    {
+      fprintf (stderr, "It's fine compressing an image using a 'surrounding' template, but I can't "
+	       "decompress it!\n");
+      exit (1);
+    }
+  w = DecodeGammaDist ();
+  h = DecodeGammaDist ();
+
+  inputstr = (char *) malloc ((w + 2) * h * sizeof (char) + 2);
+  ind[0] = '.';
+  ind[1] = 'p';
+  ind[2] = '2';
+  ind[3] = '*';
+  for (p = 0, j = 0; j < h; j++)
+    {
+      for (i = 0; i < w; i++)
+	{
+	  inputstr[p++] = ind[DecodeGammaDist ()];
+	}
+      inputstr[p++] = ';';
+      inputstr[p] = 0;
+    }
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+  bl_string_to_array (inputstr, str, &w, &h);
+}
+
+
+
+
+
+/* Currently if there are more than 32 bits, the topmost 32 will be discarded, if the 
+   topbits are to be used in compression, define _MORETHAN32 */
+#undef _MORETHAN32
+
+void 
+bl_compress_mark (marktype d)
+{
+  register int r, c;
+  register int i, j;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d.w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  EncodeGammaDist (d.w);
+  EncodeGammaDist (d.h);
+
+  for (r = 0; r < min (curr_row, d.h); r++)
+    for (c = 0; c < d.w; c++)
+      array[r % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r);
+
+  for (r = 0; r < d.h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+
+      if (r + curr_row < d.h)
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r + curr_row);
+      else
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = 0;
+
+      for (c = 0; c < d.w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (d.bitmap, c, r));
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_decompress_mark (marktype * d)
+{
+  register int r, c;
+  register int i, j;
+  unsigned long pix;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  d->w = DecodeGammaDist ();
+  d->h = DecodeGammaDist ();
+
+  d->bitmap = pbm_allocarray (d->w, d->h);
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d->w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  for (r = 0; r < d->h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+      for (c = 0; c < d->w; c++)
+	array[r % MAX_T_W][c] = pbm_getpixel (d->bitmap, c, r);
+      for (c = 0; c < d->w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)	/* get all pixels */
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_decode_in_context (mask, mask2level, &pix);
+	  if (pix)
+	    {
+	      pbm_putpixel (d->bitmap, c, r, pix);
+	      array[r % MAX_T_W][c] = pix;
+	    }
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_compress (marktype d, char inputstr[])
+{
+  bl_clearmodel ();
+  bl_writetemplate (inputstr);
+  bl_compress_mark (d);
+  bl_freemodel ();
+}
+
+
+void 
+bl_decompress (marktype * d)
+{
+  bl_clearmodel ();
+  bl_readtemplate ();
+  bl_decompress_mark (d);
+  bl_freemodel ();
+}
+
+
+
+
+
+
+/* 
+   The bitmap in d2, can be examined clairvoyantly.  The ordering of the
+   params is therefore non-clairvoyant, and clairvoyant in that order.
+
+ */
+/*
+   coordinate M1[6]={        { 0,-2},
+   {-1,-1},{ 0,-1}, {1,-1},
+   {-2,0},  {-1, 0}};
+   };
+   coordinate M2[13]={       { 0,-2},
+   {-1,-1}, { 0,-1},   {1, -1},
+   {-2,0}, {-1, 0}, { 0, 0},   { 1, 0},  {2,0},
+   {-1, 1}, { 0, 1},   { 1, 1},
+   { 0, 2}
+ */
+
+coordinate M1[6] =
+{
+  {0, -2},
+  {-2, 0},
+  {1, -1},
+  {-1, -1},
+  {0, -1},
+  {-1, 0}};
+coordinate M2[13] =
+{
+  {0, -2},
+  {1, -1},
+  {-2, 0},
+  {2, 0},
+  {-1, 1},
+  {1, 1},
+  {0, 2},
+  {-1, -1},
+  {0, -1},
+  {1, 0},
+  {-1, 0},
+  {0, 1},
+  {0, 0}
+};
+#define M1S 6
+#define M1S_2level 3
+#define M2S 13
+#define M2S_2level 6
+
+
+void 
+bl_clair_compress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  EncodeGammaDist (cols);
+  EncodeGammaDist (rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (b1, c, r));
+
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
+
+
+void 
+bl_clair_decompress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  unsigned long p;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  i = DecodeGammaDist ();
+  j = DecodeGammaDist ();
+  if ((i != cols) || (j != rows))
+    error_msg ("bl_clair_decompress", "incorrect size of residue image", "");
+
+  if (V)
+    fprintf (stderr, "cols=%d rows=%d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+
+	  bl_decode_in_context (mask, mask2level, &p);
+	  if (p)
+	    pbm_putpixel (b1, c, r, 1);
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
Index: /main/tags/3.01/indexers/mg/src/images/bilevel.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/bilevel.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/bilevel.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * bilevel.h -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __BILEVEL_H
+#define __BILEVEL_H
+
+#include "marklist.h"
+
+/* must be called before the start of de/compression */
+void bl_clearmodel ();
+void bl_freemodel ();
+
+/* writes the template to the stream, and sets up the de/compression params */
+void bl_writetemplate (char inputstr[]);
+void bl_readtemplate ();
+
+/* actually performs the de/compression */
+void bl_compress (marktype d, char str[]);
+void bl_compress_mark (marktype d);
+void bl_decompress (marktype * d);
+void bl_decompress_mark (marktype * d);
+
+/* clairvoyantly de/compress d1 with respect to clairvoyantly viewable d2 */
+void bl_clair_compress (marktype d1, marktype d2);
+void bl_clair_decompress (marktype d1, marktype d2);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/codeoffsets.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/codeoffsets.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/codeoffsets.c	(revision 21112)
@@ -0,0 +1,257 @@
+/**************************************************************************
+ *
+ * codeoffsets.c -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+#define TOP_INC 2
+#define MID_INC 1
+#define BOT_INC 1
+
+
+void 
+EncodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int found;
+  int dox = 0, leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;	/* all previous offsets */
+  int p, f, t, xoff, yoff, symnum, count;
+  elemlist *xcond, *ycond;	/* previous offsets conditioned on symbol num */
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (t = 0; t < nsyms; t++)
+    {
+      xcond[t].head = ycond[t].head = NULL;
+      xcond[t].tot = ycond[t].tot = 0;
+    }
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      xoff = step->data.xoffset;
+      yoff = step->data.yoffset;
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* encode the offsets, x first then y */
+
+	  if (dox)
+	    found = elem_find (&xcond[symnum], xoff, &p, &f, &t);
+	  else
+	    found = elem_find (&ycond[symnum], yoff, &p, &f, &t);
+
+	  if (found)
+	    {
+	      arithmetic_encode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+	      arithmetic_encode (p, f, t);
+	    }
+	  else
+	    {
+	      if (dox)
+		found = elem_find (&xlist, xoff, &p, &f, &t);
+	      else
+		found = elem_find (&ylist, yoff, &p, &f, &t);
+
+	      if (found)
+		{
+		  arithmetic_encode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+		  levelmid += MID_INC;
+		  arithmetic_encode (p, f, t);
+		}
+	      else
+		{
+		  arithmetic_encode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+		  levelbot += BOT_INC;
+		  if (dox)
+		    EncodeGammaSigned (xoff, &xpos, &xneg);
+		  else
+		    EncodeGammaSigned (yoff, &ypos, &yneg);
+		}
+	    }
+	  if (dox)
+	    {
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+  if (V)
+    fprintf (stderr, "offset models: top %d, mid %d, bot %d\n", (leveltop - 1) / TOP_INC, (levelmid - 1) / MID_INC, (levelbot - 1) / BOT_INC);
+}
+
+
+
+
+
+void 
+DecodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int i, target, dox;
+  int leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;
+  int p, f, t, xoff = 0, yoff = 0, symnum, count = 0;
+  elemlist *xcond, *ycond;
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (i = 0; i < nsyms; i++)
+    {
+      xcond[i].head = ycond[i].head = NULL;
+      xcond[i].tot = ycond[i].tot = 0;
+    }
+
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* decode x offset first, then y */
+	  target = arithmetic_decode_target (leveltop + levelmid + levelbot);
+	  if (target < leveltop)
+	    {
+	      arithmetic_decode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xcond[symnum].tot);
+	      else
+		i = arithmetic_decode_target (ycond[symnum].tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xcond[symnum], i, &xoff);
+		  elem_find (&xcond[symnum], xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ycond[symnum], i, &yoff);
+		  elem_find (&ycond[symnum], yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else if (target < leveltop + levelmid)
+	    {
+	      arithmetic_decode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+	      levelmid += MID_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xlist.tot);
+	      else
+		i = arithmetic_decode_target (ylist.tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xlist, i, &xoff);
+		  elem_find (&xlist, xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ylist, i, &yoff);
+		  elem_find (&ylist, yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else
+	    {
+	      arithmetic_decode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+	      levelbot += BOT_INC;
+	      if (dox)
+		xoff = DecodeGammaSigned (&xpos, &xneg);
+	      else
+		yoff = DecodeGammaSigned (&ypos, &yneg);
+	    }
+
+	  if (dox)
+	    {
+	      step->data.xoffset = xoff;
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      step->data.yoffset = yoff;
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+}
Index: /main/tags/3.01/indexers/mg/src/images/codeoffsets.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/codeoffsets.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/codeoffsets.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * codeoffsets.h -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __CODEOFFSETS_H
+#define __CODEOFFSETS_H
+
+#include "marklist.h"
+
+void EncodeOffsets (marklistptr MarkList, int num_symbols);
+void DecodeOffsets (marklistptr MarkList, int num_symbols);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/codesyms.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/codesyms.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/codesyms.c	(revision 21112)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * codesyms.c -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Hugh Emberson
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+#include "codesyms.h"
+
+int charsetsize = 256;
+
+void 
+EncodeSymbols (marklistptr MarkList, int NSyms)
+{
+  marklistptr step;
+  unsigned short Events[PPM_CHUNK];
+  int j = 0;
+
+  charsetsize = max (NSyms, 256);
+
+  step = MarkList;
+  while (step)
+    {
+      Events[j] = step->data.symnum;
+
+      assert (Events[j] > 0 || Events[j] < (unsigned) NSyms);
+
+      j++;
+
+      if (j == PPM_CHUNK)
+	{
+	  encodestring (Events, PPM_CHUNK);
+	  j = 0;
+	}
+      step = step->next;
+    };
+
+  if (j != 0)
+    encodestring (Events, j);
+
+  encodestring (Events, 0);
+}
+
+
+marklistptr 
+DecodeSymbols (int NSyms)
+{
+  marklistptr MarkList = NULL;
+  unsigned short Events[PPM_CHUNK];
+  int j, n;
+  marktype d;
+
+  charsetsize = max (NSyms, 256);
+
+  do
+    {
+      decodestring (Events, &n);
+
+      for (j = 0; j < n; j++)
+	{
+	  d.symnum = Events[j];
+	  marklist_add (&MarkList, d);
+	}
+    }
+  while (n != 0);
+
+  return MarkList;
+}
+
+void 
+InitPPM (void)
+{
+  kbytes = 896;
+}
Index: /main/tags/3.01/indexers/mg/src/images/codesyms.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/codesyms.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/codesyms.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * codesyms.h -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef _CodeSymbols_h_
+#define _CodeSymbols_h_
+
+#include "marklist.h"
+#include "arithcode.h"
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+#define PPM_CHUNK	1024
+
+extern int kbytes;
+extern void decodestring ();
+extern void encodestring ();
+
+void EncodeSymbols (marklistptr MarkList, int NSyms);
+marklistptr DecodeSymbols (int NSyms);
+
+void InitPPM (void);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/extractor.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/extractor.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/extractor.c	(revision 21112)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * extractor.c -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "utils.h"
+
+
+/* 
+   'splitjoined' defines whether the extractor modules tries to break
+   "seemingly" merged/joined marks.
+ */
+int splitjoined = 1;
+
+#define MAX_FILL_STACK_SIZE 10000
+
+/* RunRegionFill uses line conherence to implement a better fill
+   routine.  The stack has currently got 10000 elements, although the
+   biggest I have encountered used a maximum stack size of 1294.  usage:
+   RunRegionFill(bitmap,mark,i,j,xpos,ypos); xpos and ypos are the top
+   left corners of the mark.  */
+
+int maxst = 0, st = 0;
+struct
+  {
+    int x, y;
+  }
+a[MAX_FILL_STACK_SIZE];
+
+#define push(i,j) a[st].x=i;a[st].y=j;st++;if(st>MAX_FILL_STACK_SIZE){error_msg ("extractor/flood_fill","woooo! Fill stack overflow!","");}
+#define pop(i,j) st--;i=a[st].x;j=a[st].y
+
+void 
+RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows)
+{
+  int l, r, t, start_up, start_down;
+
+  push (i, j);
+  while (st > 0)
+    {
+      if (st > maxst)
+	maxst = st;
+
+      pop (l, j);
+      r = l;
+
+      while (pbm_getpixel (bitmap, l - 1, j))
+	l--;
+      while (pbm_getpixel (bitmap, r + 1, j))
+	r++;
+      start_up = start_down = 1;
+      for (t = min (r + 1, cols - 1); t >= max (l - 1, 0); t--)
+	{
+	  if ((t >= l) && (t <= r))
+	    {
+	      pbm_putpixel (bitmap, t, j, 0);
+
+	      if (mark)
+		pbm_putpixel (mark->bitmap, t - xl, j - yt, 1);
+	    }
+
+	  if ((j - 1 >= 0) && (pbm_getpixel (bitmap, t, j - 1)))
+	    {
+	      if (start_up)
+		{
+		  push (t, j - 1);
+		  start_up = 0;
+		}
+	    }
+	  else
+	    start_up = 1;
+	  if ((j + 1 < rows) && (pbm_getpixel (bitmap, t, j + 1)))
+	    {
+	      if (start_down)
+		{
+		  push (t, j + 1);
+		  start_down = 0;
+		}
+	    }
+	  else
+	    start_down = 1;
+	}
+    }
+}
+
+
+
+
+/*
+
+   ExtractNextMark starts at (tx,ty) and looks for a set (==1) pixel.
+   The mark is then boundary traced to give an outline {The process is a
+   4-pixel exterior check -- giving 8 way connectivity}, and then
+   RunRegionFill is used to remove the mark and set the pixels in the
+   mark.
+
+   usage: ExtractNextMark(bitmap,&marklist,&startx,&starty,int cols,int rows);
+   Defines:
+   gp - gets a pixel, but returns 0 for outside regions.
+   boundary_{x,y}comp - returns the {x,y} component of the direction.
+   dir 0=right, 1=down, 2=left, 3=up.
+ */
+
+#define gp(i,j) ( (((i)<0)||((j)<0)||((i)>=cols)||((j)>=rows))? 0 : pbm_getpixel(bitmap,(i),(j)))
+
+#define boundary_xcomp(x) (x==0?1:(x==2?-1 : 0))
+#define boundary_ycomp(x) (x==1?1:(x==3?-1 : 0))
+
+int 
+ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows)
+{
+  int i, j, xl, xr, yt, yb;
+  register int dir, tempdir;
+  marktype d;
+
+  xl = INT_MAX;
+  xr = (-1);
+  yt = INT_MAX;
+  yb = (-1);
+
+  i = *tx;
+  j = *ty;
+  if ((i < 0) || (j < 0))
+    error_msg ("ExtractNextMark()", "Stupid values for x,y in findnextmark()", "");
+  while (!pbm_getpixel (bitmap, i, j))
+    {
+      i++;
+      if (i >= cols)
+	{
+	  i = 0;
+	  j++;
+	  if (j >= rows)
+	    return 0;
+	}
+    }
+
+  *tx = i;
+  *ty = j;
+  dir = 0;
+  j = j - 1;			/* start above */
+  do
+    {
+      if (i > xr)
+	xr = i - 1;
+      if (i < xl)
+	xl = i + 1;
+      if (j < yt)
+	yt = j + 1;
+      if (j > yb)
+	yb = j - 1;
+      tempdir = (dir + 1) % 4;
+      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	dir = tempdir;
+      else
+	{
+	  tempdir = dir;
+	  if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	    dir = tempdir;
+	  else
+	    {
+	      tempdir = (dir + 3) % 4;
+	      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+		dir = tempdir;
+	      else
+		dir = (dir + 2) % 4;
+	    }
+	}
+      i += boundary_xcomp (dir);
+      j += boundary_ycomp (dir);
+    }
+  while ((i != (*tx)) || (j != (*ty - 1)));
+
+  d.w = (xr - xl + 1);
+  d.h = (yb - yt + 1);
+  d.xpos = xl;
+  d.ypos = yt;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+  RunRegionFill (bitmap, &d, *tx, *ty, xl, yt, cols, rows);
+
+  marktype_calc_features (&d, "?");
+  marklist_add (marklist, d);
+
+  *tx = *tx + 1;
+  return 1;
+}
+
+
+
+
+static void 
+normalise (float *p, int w)
+{
+  int x;
+  float mx;
+
+  mx = p[0];
+  for (x = 0; x < w; x++)
+    if (p[x] > mx)
+      mx = p[x];
+  if (mx != 0)
+    {
+      for (x = 0; x < w; x++)
+	p[x] = p[x] / mx;
+    }
+}
+
+
+
+#define MAXPROFILE 1000
+float profile[MAXPROFILE], p2[MAXPROFILE];
+
+
+static int 
+possibly_joined (marktype * d)
+{
+  float aspect;
+  int i, n, x, y;
+  int leftable, l2;
+
+
+  aspect = d->w / (float) d->h;
+
+  if ((aspect >= 1.1) || (d->w < MAXPROFILE))
+    {
+      /* examine it more closely */
+
+
+      /* set up profile[] with a vertical profile */
+      for (x = 0; x < d->w; x++)
+	{
+	  profile[x] = 0;
+	  for (y = 0; y < d->h; y++)
+	    if (pbm_getpixel (d->bitmap, x, y))
+	      profile[x]++;
+	}
+
+      /* average the profile from (up to) 3 either way */
+      for (x = 0; x < d->w; x++)
+	{
+	  n = 0;
+	  p2[x] = 0;
+	  for (i = x - 3; i <= x + 3; i++)
+	    if ((i >= 0) && (i < d->w))
+	      {
+		n++;
+		p2[x] += profile[x];
+	      }
+	  p2[x] = p2[x] / (float) n;
+	}
+      for (x = 0; x < d->w; x++)
+	profile[x] = p2[x];
+
+      /* normalise profile[] */
+      normalise (profile, d->w);
+
+      /* find a little local minimum, with profile <0.25, make p2[] either 0 or 1 */
+      for (x = 0; x < d->w; x++)
+	p2[x] = 0;
+      for (x = 3; x < d->w - 3; x++)
+	{
+	  if (((profile[x - 2] - profile[x]) > 0.00) &&
+	      ((profile[x + 2] - profile[x]) > 0.00) &&
+	      (profile[x] < .25))
+	    p2[x] = 1;
+	}
+
+      /* travel rightwards, checking aspect ratio, looking at p2 now */
+      leftable = l2 = 0;
+      for (x = 0; x < d->w; x++)
+	if (p2[x] == 1)
+	  {
+	    if ((fabs ((abs (x - leftable) / (float) d->h) - 0.7) > 0.3) &&
+		(fabs ((abs (x - l2) / (float) d->h) - 0.7) > 0.3))
+	      p2[x] = 0;
+	    if (p2[x] == 1)
+	      {
+		leftable = l2;
+		l2 = x;
+	      }
+	  }
+
+      /* travel inwards, checking widths of chars, looking at p2 */
+      /* if a char has an aspect ratio of <= ? kill it */
+      l2 = 0;
+      leftable = d->w - 1;
+      x = 0;
+      y = d->w - 1;
+      while (x <= y)
+	{
+	  if (p2[x])
+	    {
+	      if (abs (x - l2) < max (d->h * 0.6, 5))
+		p2[x] = 0;
+	      if (p2[x])
+		l2 = x;
+	    }
+	  if (p2[y])
+	    {
+	      if (abs (y - leftable) < max (d->h * 0.6, 5))
+		p2[y] = 0;
+	      if (p2[y])
+		leftable = y;
+	    }
+	  x++;
+	  y--;
+	}
+
+
+      for (x = 0; x < d->w; x++)
+	if (p2[x])
+	  return 1;
+    }
+  return 0;
+}
+
+
+static void 
+split_mark (marklistptr * list, marktype * joinedmark)
+{
+  marktype new, old;
+  int x, found = 0, left = 0;
+  int i, j;
+  int oldw, oldh;
+  int overwrite = 1;
+
+  old = marktype_copy (*joinedmark);
+  oldw = joinedmark->w;
+  oldh = joinedmark->h;
+  for (x = 0; x < old.w; x++)
+    if (p2[x])
+      {
+	found = x;
+	break;
+      }
+  do
+    {
+      /* split at (found) position */
+      new.w = found + 1 - left;
+      new.h = old.h;
+      new.xpos = old.xpos + left;
+      new.ypos = old.ypos;
+      new.bitmap = pbm_allocarray (new.w, new.h);
+
+      for (i = 0; i < new.w; i++)
+	for (j = 0; j < new.h; j++)
+	  pbm_putpixel (new.bitmap, i, j, pbm_getpixel (old.bitmap, left + i, j));
+
+      marktype_adj_bound (&new);
+
+      if (overwrite)
+	{
+	  overwrite = 0;
+	  pbm_freearray (&(joinedmark->bitmap), oldh);
+	  *joinedmark = new;
+	}
+      else
+	marklist_add (list, new);
+
+
+      p2[found] = 0;
+      left = found;
+      found = 0;
+      for (x = left; x < old.w; x++)
+	if (p2[x])
+	  {
+	    found = x;
+	    break;
+	  }
+      if (left == old.w - 1)
+	break;			/* if we've got to the right */
+      if (found == 0)
+	found = old.w - 1;
+    }
+  while (found);
+  pbm_freearray (&(old.bitmap), oldh);
+}
+
+
+
+
+
+void 
+ExtractAllMarks (Pixel ** bitmap, marklistptr * list, int cols, int rows)
+{
+  int startx, starty, count;
+  marklistptr step;
+
+  startx = starty = 0;
+  while (ExtractNextMark (bitmap, list, &startx, &starty, cols, rows))
+    ;
+
+  if (splitjoined)
+    for (step = (*list), count = 0; step; step = step->next, count++)
+      if (possibly_joined (&(step->data)))
+	split_mark (list, &(step->data));
+
+  for (step = (*list), count = 0; step; step = step->next, count++)
+    step->data.symnum = count;
+}
Index: /main/tags/3.01/indexers/mg/src/images/extractor.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/extractor.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/extractor.h	(revision 21112)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * extractor.h -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _EXTRACTOR_H_
+#define _EXTRACTOR_H_
+
+#include "marklist.h"
+#include "pbmtools.h"
+
+extern int splitjoined;
+
+void RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows);
+
+int ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows);
+void ExtractAllMarks (Pixel ** bitmap, marklistptr * marklist, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/felics.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/felics.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/felics.c	(revision 21112)
@@ -0,0 +1,340 @@
+/**************************************************************************
+ *
+ * felics.c -- The main guts of the felics algorithm
+ * Copyright (C) 1994  Neil Sharman and Kerry Guise
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+
+
+#undef putw
+#define putw(x)	{					\
+	short z;					\
+	byteptr = (unsigned char *)(x);			\
+	for (z = 0; z < sizeof(int); z++)		\
+		(void) putc((int) byteptr[z], fp_out);	\
+}
+#define INTSIZE sizeof(int) * 8
+#define BOOLEAN short
+#define TRUE 1
+#define FALSE 0
+
+#define IN_RANGE 0
+#define OUT_OF_RANGE 1
+#define BELOW_RANGE 0
+#define ABOVE_RANGE 1
+
+
+void 
+encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out, int (*get) (FILE *))
+{
+  int x, y;
+  unsigned long P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  ENCODE_START (fp_out)
+
+    for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		line[x] = get (fp_in);	/* +------- */
+		BINARY_ENCODE (line[x] + 1, maxval + 1);	/* |P****** */
+		continue;	/* |******* */
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	line[x] = P = get (fp_in);
+
+	if ((L <= P) && (P <= H))
+	  {
+	    /* IN-RANGE */
+	    unsigned long range, logofrange, thresh, numlong;
+	    int nbits;
+
+	    ENCODE_BIT (IN_RANGE);
+
+	    P -= L;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P -= (numlong >> 1);
+		if ((long) P < 0)
+		  P += range;
+
+		if (P < thresh)
+		  nbits = logofrange - 1;
+		else
+		  {
+		    nbits = logofrange;
+		    P += thresh;
+		  }
+		while (--nbits >= 0)
+		  ENCODE_BIT ((P >> nbits) & 1);
+	      }
+	  }
+	else
+	  {
+	    /* OUT_OF_RANGE */
+	    unsigned long diff, parm, i, min;
+
+	    ENCODE_BIT (OUT_OF_RANGE);
+
+	    if (P < L)
+	      {
+		ENCODE_BIT (BELOW_RANGE);
+		diff = L - P - 1;
+	      }
+	    else
+	      {
+		ENCODE_BIT (ABOVE_RANGE);
+		diff = P - H - 1;
+	      }
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_ENCODE ((diff >> parm) + 1);
+	    BINARY_ENCODE ((diff & ((1 << parm) - 1)) + 1, 1 << parm);
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+	  }
+
+      }
+
+  ENCODE_DONE
+
+}
+
+
+
+
+
+
+void 
+decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out)
+{
+  int x, y;
+  unsigned int P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  DECODE_START (fp_in);
+
+  for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		BINARY_DECODE (line[x], maxval + 1);
+		line[x]--;
+		if (maxval < 256)
+		  putc (line[x], fp_out);	/* +------- */
+		else		/* |P****** */
+		  fprintf (fp_out, "%u ", line[x]);	/* |******* */
+		continue;
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	if (DECODE_BIT == IN_RANGE)
+	  {
+	    unsigned long i, range, logofrange, thresh, numlong;
+	    unsigned xx, l;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+		for (P = i = 0; i < logofrange - 1; i++)
+		  DECODE_ADD (P);
+		xx = P;
+		l = logofrange - 1;
+		if (P >= thresh)
+		  {
+		    DECODE_ADD (P);
+		    xx = P;
+		    l++;
+		    P -= thresh;
+		  }
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P += (numlong >> 1);
+		if ((long) P >= range)
+		  P -= range;
+
+		P += L;
+	      }
+	    else
+	      P = L;
+	  }
+	else
+	  {
+	    int out_of_range = DECODE_BIT;
+	    unsigned long diff, unary, binary, i, min, parm;
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_DECODE (unary);
+	    diff = (unary - 1) << parm;
+	    BINARY_DECODE (binary, 1 << parm);
+	    diff |= binary - 1;
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+
+	    if (out_of_range == BELOW_RANGE)
+	      {
+		P = L - diff - 1;
+	      }
+	    else
+	      {
+		P = diff + H + 1;
+	      }
+	  }
+
+	line[x] = P;
+
+	if (maxval < 256)
+	  (void) putc (P, fp_out);
+	else
+	  fprintf (fp_out, "%u ", P);
+      }
+
+  DECODE_DONE
+}
Index: /main/tags/3.01/indexers/mg/src/images/lstevent.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/lstevent.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/lstevent.c	(revision 21112)
@@ -0,0 +1,424 @@
+/**************************************************************************
+ *
+ * lstevent.c -- Routines to implement "events" using MTF lists.
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:16:56  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:40  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+void
+write_method ()
+{
+  fprintf (stderr, "C");
+}
+
+
+/*===========================================*/
+
+/*
+ * [Rodney Brown - Sep/95]
+ * On IBM RS/6000 AIX version 3.2 the compiler defines NULL as (void *)0
+ * when in ANSI mode. Hence the (event) and (point) casts
+ */
+
+
+#define GETNEWNODE(e,node)		\
+do 					\
+{	node = p32(numnodes) ;		\
+	node->eventnum = (event)e ;	\
+	node->count = increment ;	\
+	node->foll.totalcnt = 0 ;	\
+	node->foll.notfound = 0 ;	\
+	node->foll.list = (point)NULL ;	\
+	numnodes += 1 ;			\
+	nfreenodes -= 1 ;		\
+}					\
+while (false)
+
+/*===========================================*/
+
+#define INSERTNODE(U,node)		\
+do					\
+{	node->next = U->list ;		\
+	U->list = p16(node) ;		\
+	U->totalcnt += increment;	\
+}					\
+while (false)
+
+/*===========================================*/
+
+int
+halvelist (p)
+     register eventnode *p;
+{
+  register int sum = 0;
+  while (p != ENULL)
+    {
+      sum += p->count = (p->count + 1) / 2;
+      p = p32 (p->next);
+    }
+  return (sum);
+}
+
+/*===========================================*/
+
+void
+halveallcounts (U)
+     register eventset *U;
+{
+  U->notfound = (U->notfound + 1) / 2;
+  U->totalcnt = U->notfound;
+  U->totalcnt += halvelist (p32 (U->list));
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event_noex (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (e == node->eventnum)
+	break;
+      lbnd += node->count;
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  totl = U->totalcnt;
+
+  if (node == ENULL)
+    {				/* event not found in list */
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event_noex (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = U->totalcnt;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (tget < (lbnd += node->count))
+	    break;
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node, *p;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	{
+	  if (e == node->eventnum)
+	    break;
+	  lbnd += node->count;
+	}
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  if (node == ENULL)
+    {				/* event not found in list, and whole list has been scanned */
+      totl = lbnd + U->notfound;
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      /* first, total up rest of list */
+      totl = lbnd + node->count;
+      p = p32 (node->next);
+      while (p != ENULL)
+	{
+	  if (!excluded[p->eventnum])
+	    totl += p->count;
+	  p = p32 (p->next);
+	}
+      totl += U->notfound;
+
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = 0;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	totl += node->count;
+      node = p32 (node->next);
+    }
+  totl += U->notfound;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (!excluded[node->eventnum])
+	    {
+	      lbnd += node->count;
+	      if (tget < lbnd)
+		break;
+	    }
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+newnode (e)
+     event e;
+{
+  register eventnode *node;
+  GETNEWNODE (e, node);
+  node->next = (point)NULL;
+  node->prev = (point)NULL;
+  return (node);
+}
+
+/*===========================================*/
+
+/* not required in this version */
+
+#if 0
+eventnode *
+addevent (U, e)
+     register eventset *U;
+     event e;
+{
+  register eventnode *node;
+  /* get an event node */
+  GETNEWNODE (e, node);
+  INSERTNODE (U, node);
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/marklist.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/marklist.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/marklist.c	(revision 21112)
@@ -0,0 +1,827 @@
+/**************************************************************************
+ *
+ * marklist.c -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+void 
+marktype_dump (marktype d)
+{
+  fprintf (stderr, "symnum: %d\n", d.symnum);
+  fprintf (stderr, "xpos: %d, ypos: %d\n", d.xpos, d.ypos);
+  fprintf (stderr, "w: %d, h: %d\n", d.w, d.h);
+  fprintf (stderr, "xcen: %d, ycen: %d\n", d.xcen, d.ycen);
+  write_library_mark (stderr, d);
+}
+
+
+/* copy, including make a copy of the bitmap */
+marktype 
+marktype_copy (marktype d)
+{
+  marktype d2;
+  int i, j;
+
+  d2 = d;
+  d2.bitmap = pbm_allocarray (d2.w, d2.h);
+  for (i = 0; i < d2.w; i++)
+    for (j = 0; j < d2.h; j++)
+      pbm_putpixel (d2.bitmap, i, j, pbm_getpixel (d.bitmap, i, j));
+  return d2;
+}
+
+
+float 
+marktype_density (marktype d)
+{
+  return d.set / (float) (d.w * d.h);
+}
+
+
+
+void 
+marktype_calc_centroid (marktype * b)
+{
+  int r, c;
+  int count = 0;
+  int xtot = 0, ytot = 0;
+
+  for (r = 0; r < b->h; r++)
+    for (c = 0; c < b->w; c++)
+      if (pbm_getpixel (b->bitmap, c, r))
+	{
+	  count++;
+	  xtot += c;
+	  ytot += r;
+	}
+  if (count)
+    {
+      xtot /= count;
+      ytot /= count;
+    }
+  b->xcen = xtot;
+  b->ycen = ytot;
+}
+
+
+
+static void 
+marktype_area (marktype * d)
+{
+  int x, y;
+  int set = 0;
+
+  for (y = 0; y < d->h; y++)
+    for (x = 0; x < d->w; x++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	set++;
+  d->set = set;
+}
+
+
+static void 
+marktype_h_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+  for (y = 0; y < d->h; y++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (x = 0; x < d->w; x++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->h_runs = num;
+}
+
+static void 
+marktype_v_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+
+  for (x = 0; x < d->w; x++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (y = 0; y < d->h; y++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->v_runs = num;
+}
+
+
+/* assume we already have the centroid! */
+static void 
+marktype_prog_cent (marktype * d)
+{
+  int x, y;
+  int xtot = 0, ytot = 0;
+  int num = 0;
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen1 = -1;
+      d->ycen1 = -1;
+    }
+  else
+    {
+      d->xcen1 = xtot / num;
+      d->ycen1 = ytot / num;
+    }
+
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen2 = -1;
+      d->ycen2 = -1;
+    }
+  else
+    {
+      d->xcen2 = xtot / num;
+      d->ycen2 = ytot / num;
+    }
+
+
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+
+
+  if (num == 0)
+    {
+      d->xcen3 = -1;
+      d->ycen3 = -1;
+    }
+  else
+    {
+      d->xcen3 = xtot / num;
+      d->ycen3 = ytot / num;
+    }
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen4 = -1;
+      d->ycen4 = -1;
+    }
+  else
+    {
+      d->xcen4 = xtot / num;
+      d->ycen4 = ytot / num;
+    }
+}
+
+
+
+/* call this to get all features */
+void 
+marktype_calc_features (marktype * d, char str[])
+{
+  if (strlen (str))
+    strcpy (d->name, str);
+  marktype_calc_centroid (d);
+  marktype_area (d);
+  marktype_prog_cent (d);
+  marktype_h_runs (d);
+  marktype_v_runs (d);
+}
+
+
+
+
+
+void 
+marktype_adj_bound (marktype * m)
+{
+  int x, y, w, h;
+  int xpos, ypos;
+  int top = 0, left = 0, right = 0, bot = 0;
+  marktype temp;
+
+  w = m->w;
+  h = m->h;
+  xpos = m->xpos;
+  ypos = m->ypos;
+
+  for (y = 0; y < h; y++)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  top = y;
+	  x = w;
+	  y = h;
+	}
+
+  for (y = h - 1; y >= 0; y--)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  bot = y;
+	  x = w;
+	  y = -1;
+	}
+
+  for (x = 0; x < w; x++)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  left = x;
+	  x = w;
+	  y = h;
+	}
+
+  for (x = w - 1; x >= 0; x--)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  right = x;
+	  x = -1;
+	  y = h;
+	}
+
+  temp.w = right - left + 1;
+  temp.h = bot - top + 1;
+  temp.xpos = xpos + left;
+  temp.ypos = ypos + top;
+  temp.bitmap = pbm_allocarray (temp.w, temp.h);
+
+  for (x = 0; x < temp.w; x++)
+    for (y = 0; y < temp.h; y++)
+      pbm_putpixel (temp.bitmap, x, y, pbm_getpixel (m->bitmap, x + left, y + top));
+
+  pbm_freearray (&(m->bitmap), h);
+
+  marktype_calc_features (&temp, "?");
+  *m = temp;
+}
+
+
+
+
+
+
+/* returns a newly created mark reprenting the average of the marklist */
+void 
+marklist_average (marklistptr list, marktype * avgmark)
+{
+  int w = 0, h = 0;
+  int t = INT_MAX, b = 0, l = INT_MAX, r = 0, matched = 0;
+  marktype d, temp;
+  marklistptr step;
+  static int a[100][100];
+  int x, y;
+
+  if (list == NULL)
+    error_msg ("marklist_average", "Can't average 'nothing'!", "");
+
+  for (step = list; step; step = step->next)
+    {
+      matched++;
+      temp = step->data;
+      w = max (w, temp.w);
+      h = max (h, temp.h);
+      r = max (r, temp.xcen);
+      l = min (l, temp.xcen);
+      b = max (b, temp.ycen);
+      t = min (t, temp.ycen);
+    }
+  w = w + (r - l);
+  h = h + (b - t);
+  if ((w >= 100) || (h >= 100))
+    matched = 0;
+
+  if (matched > 1)
+    {
+      int x_off, y_off;
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  a[x][y] = 0;
+
+      for (step = list; step; step = step->next)
+	{
+	  x_off = r - step->data.xcen;
+	  y_off = b - step->data.ycen;
+	  for (x = 0; x < step->data.w; x++)
+	    for (y = 0; y < step->data.h; y++)
+	      if (pbm_getpixel (step->data.bitmap, x, y))
+		a[x + x_off][y + y_off]++;
+	}
+
+      d.w = w;
+      d.h = h;
+      d.bitmap = pbm_allocarray (w, h);
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  if (a[x][y] > max (1, (matched / 2)))		/* > gives better CR than >= */
+	    pbm_putpixel (d.bitmap, x, y, 1);
+      marktype_adj_bound (&d);
+      strcpy (d.name, "?");
+      marktype_calc_centroid (&d);
+    }
+  else
+    d = marktype_copy (list->data);
+
+  *avgmark = d;
+}
+
+
+
+
+
+
+void 
+marklist_setnull (marklistptr * list)
+{
+  *list = NULL;
+}
+
+
+marklistptr 
+marklist_add (marklistptr * list, marktype m)
+{
+  marklistptr p;
+
+  if (*list == NULL)
+    {
+      *list = (marklistptr) malloc (sizeof (marklisttype));
+      if ((*list) == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = *list;
+    }
+  else
+    {
+      p = *list;
+      while (p->next != NULL)
+	p = p->next;
+      p->next = (marklistptr) malloc (sizeof (marklisttype));
+      if (p->next == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = p->next;
+    }
+  p->data = m;
+  p->next = NULL;
+  return p;
+
+}
+
+
+
+marklistptr 
+marklist_addcopy (marklistptr * list, marktype m)
+{
+  marktype d;
+
+  d = marktype_copy (m);
+  return marklist_add (list, d);
+}
+
+
+
+int 
+marklist_length (marklistptr list)
+{
+  int count = 0;
+
+  while (list)
+    {
+      count++;
+      list = list->next;
+    }
+  return count;
+}
+
+
+
+
+
+void 
+marklist_dump (marklistptr list)
+{
+  int count = 0;
+  marktype d;
+
+  while (list)
+    {
+      d = list->data;
+      marktype_dump (d);
+
+      count++;
+      list = list->next;
+    }
+}
+
+
+
+
+/* pos starts from 0, to up marklist_length-1 */
+marklistptr 
+marklist_getat (marklistptr list, int pos, marktype * d)
+{
+  int count = 0;
+
+  while (list)
+    {
+      if (count == pos)
+	{
+	  *d = list->data;
+	  return list;
+	}
+      count++;
+      list = list->next;
+    }
+  fprintf (stderr, "marklist_getat(): access off ends of list: pos %d\n", pos);
+  return NULL;
+}
+
+
+
+void 
+marklist_free (marklistptr * list)
+{
+  marklistptr n;
+
+  while ((*list) != NULL)
+    {
+      n = (*list)->next;
+      pbm_freearray (&((*list)->data.bitmap), (*list)->data.h);
+      free ((*list));
+      (*list) = n;
+    }
+}
+
+
+
+int 
+marklist_removeat (marklistptr * list, int pos)
+{
+  marklistptr c, n, p = NULL;
+  int count = 0;
+
+  if ((pos == 0) && (*list))
+    {				/* if removing the 1st element */
+      *list = (*list)->next;
+      return 1;
+    }
+  else
+    {
+      c = *list;
+      while (c)
+	{
+	  if (count == pos)
+	    {			/* two cases, no previous, >=1 previous */
+	      if (p == NULL)	/* no previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  c = n;
+		  return 1;
+		}
+	      else
+		/* >=1 previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  p->next = n;
+		  return 1;
+		}
+	    }
+	  count++;
+	  p = c;
+	  c = c->next;
+	}
+    }
+  fprintf (stderr, "marklist_removeat(): access off ends of list: pos %d\n", pos);
+  return 0;			/* nothing was removed */
+}
+
+
+
+marklistptr 
+marklist_next (marklistptr list)
+{
+  return list->next;
+}
+
+
+/* only writes out ascii form */
+void 
+write_library_mark (FILE * fp, marktype d)
+{
+  int r, c, rows, cols;
+  int xtot = d.xcen, ytot = d.ycen;
+
+  rows = d.h;
+  cols = d.w;
+
+  fprintf (fp, "Mark: %d\n", d.symnum);
+  fprintf (fp, "Char: %s\n", d.name);
+  fprintf (fp, "Cols: %d Rows: %d\n", d.w, d.h);
+  for (r = 0; r < d.h; r++)
+    {
+      for (c = 0; c < d.w; c++)
+	if ((r == ytot) && (c == xtot))
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'C' : '!', fp);
+	else
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'X' : '.', fp);
+      putc ('\n', fp);
+    }
+}
+
+
+
+/* only reads in ascii library */
+int 
+read_library_mark (FILE * fp, marktype * d)
+{
+  int r, c, rows, cols, numread, ch1 = 'C';
+  char str1[100], str2[100];
+
+  numread = fscanf (fp, "%s %d\n", str1, &(d->symnum));
+  if ((numread != 2) || (strcmp (str1, "Mark:") != 0))
+    return 0;
+  fscanf (fp, "%s %s\n", str1, (d->name));
+  fscanf (fp, "%s %d %s %d\n", str1, &(d->w), str2, &(d->h));
+  cols = d->w;
+  rows = d->h;
+  d->bitmap = pbm_allocarray (cols, rows);
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c++)
+	{
+	  ch1 = getc (fp);
+	  if ((ch1 == 'X') || (ch1 == 'C'))
+	    pbm_putpixel (d->bitmap, c, r, 1);
+	  else
+	    pbm_putpixel (d->bitmap, c, r, 0);
+	}
+      ch1 = getc (fp);
+    }
+  marktype_calc_features (d, "?");
+
+  return 1;
+}
+
+
+
+
+
+
+
+int 
+read_library (char libraryname[], marklistptr * library)
+{
+  int err, count;
+  FILE *lib;
+  marktype d;
+  marklistptr step;
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg ("read_library()", "Trouble opening library file", "");
+  *library = NULL;
+  count = 0;
+  while (!isEOF (lib))
+    {
+      err = read_library_mark (lib, &d);
+      if (!err)
+	error_msg ("read_library()", "unknown format of the library file.", "");
+      d.symnum = count;
+      count++;
+      if ((*library) == NULL)
+	step = marklist_add (library, d);
+      else
+	step = marklist_add (&step, d);
+    }
+  count = marklist_length (*library);
+  fclose (lib);
+
+  return count;
+}
+
+
+
+
+
+
+/**********************************************************************
+
+'elem' routines for coding offsets
+
+the elem datatype contains the 3 params for arithmetic coding,
+stored as a no-duplicates order linked list.
+
+************************************************************************/
+
+
+
+
+void 
+elem_add (elemlist * list, int num)
+{
+  elemptr temp, nn;
+  int p, f;
+
+  list->tot++;
+  if (list->head == NULL)
+    {
+      list->head = (elemptr) malloc (sizeof (elem));
+      if ((list->head) == NULL)
+	error_msg ("", "ekk", "");
+      temp = list->head;
+      temp->next = NULL;
+      temp->num = num;
+      temp->pos = 0;
+      temp->freq = 1;
+    }
+  else
+    {				/* add to list */
+      temp = list->head;
+      p = 0;
+      f = 0;
+      while ((num > temp->num) && (temp->next != NULL))
+	{
+	  p = temp->pos;
+	  f = temp->freq;
+	  temp = temp->next;
+	}
+
+      if (temp->num == num)
+	temp->freq++;
+      else
+	{			/* insert it */
+	  nn = (elemptr) malloc (sizeof (elem));
+	  if (nn == NULL)
+	    error_msg ("", "ekk", "");
+
+	  if (num < temp->num)
+	    {
+	      nn->num = temp->num;
+	      nn->pos = temp->pos;
+	      nn->freq = temp->freq;
+	      nn->next = temp->next;
+	      temp->num = num;
+	      temp->pos = p + f;
+	      temp->freq = 1;
+	      temp->next = nn;
+	    }
+	  else
+	    {
+	      nn->next = temp->next;
+	      temp->next = nn;
+	      nn->pos = temp->pos + temp->freq;
+	      nn->num = num;
+	      nn->freq = 1;
+	      temp = nn;
+	    }
+	}
+    }
+  /* update frequencies */
+  temp = temp->next;
+  if (temp)
+    do
+      {
+	temp->pos++;
+	temp = temp->next;
+      }
+    while (temp);
+}
+
+
+void 
+elem_free (elemlist * list)
+{
+  elemptr n, p;
+
+  p = list->head;
+  while (p != NULL)
+    {
+      n = p->next;
+      free (p);
+      p = n;
+    }
+  list->head = NULL;
+}
+
+int 
+elem_find (elemlist * list, int n, int *p, int *f, int *t)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if (temp->num == n)
+	{
+	  *p = temp->pos;
+	  *f = temp->freq + temp->pos;
+	  *t = list->tot;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
+
+
+int 
+elem_getrange (elemlist * list, int range, int *num)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if ((range >= temp->pos) && (range < (temp->pos + temp->freq)))
+	{
+	  *num = temp->num;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/images/marklist.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/marklist.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/marklist.h	(revision 21112)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * marklist.h -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MARKLIST_H
+#define __MARKLIST_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+typedef struct
+  {
+    int symnum;
+    int xpos, ypos;
+    int w, h;
+    int xcen, ycen;
+    int xoffset, yoffset;
+    char name[10];
+    Pixel **bitmap;
+
+    int xcen1, ycen1;
+    int xcen2, ycen2;
+    int xcen3, ycen3;
+    int xcen4, ycen4;
+    int set;
+    int h_runs;
+    int v_runs;
+  }
+marktype;
+
+
+typedef struct marklisttype *marklistptr;
+typedef struct marklisttype
+  {
+    marktype data;
+    marklistptr next;
+  }
+marklisttype;
+
+
+void marktype_dump (marktype d);
+marktype marktype_copy (marktype d);
+
+float marktype_density (marktype d);
+void marktype_calc_centroid (marktype * b);
+void marktype_calc_features (marktype * d, char str[]);
+void marktype_adj_bound (marktype * m);
+
+void marklist_average (marklistptr list, marktype * avgmark);
+void marklist_setnull (marklistptr * list);
+marklistptr marklist_add (marklistptr * list, marktype m);
+marklistptr marklist_addcopy (marklistptr * list, marktype m);
+int marklist_length (marklistptr list);
+void marklist_dump (marklistptr list);
+marklistptr marklist_getat (marklistptr list, int pos, marktype * d);
+void marklist_free (marklistptr * list);
+int marklist_removeat (marklistptr * list, int pos);
+marklistptr marklist_next (marklistptr list);
+
+void write_library_mark (FILE * fp, marktype d);
+int read_library_mark (FILE * fp, marktype * d);
+int read_library (char libraryname[], marklistptr * library);
+
+
+
+
+
+
+/* 
+   The 'elem' structure is an ordered linked list, with fields corresponding
+   directly to the arithmetic coding routines.  
+ */
+typedef struct elem *elemptr;
+typedef struct elem
+  {
+    int num;
+    int pos;
+    int freq;
+    elemptr next;
+  }
+elem;
+
+typedef struct elemlist
+  {
+    elemptr head;
+    int tot;
+  }
+elemlist;
+
+void elem_add (elemlist * list, int num);
+void elem_free (elemlist * list);
+int elem_find (elemlist * list, int n, int *p, int *f, int *t);
+int elem_getrange (elemlist * list, int range, int *num);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/match.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/match.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/match.c	(revision 21112)
@@ -0,0 +1,417 @@
+/**************************************************************************
+ *
+ * match.c -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/* error mask goes from 0..8, single pixel errors are worth nothing */
+int 
+WXOR_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+  marktype d;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  d.w = maxc - minc + 1;
+  d.h = maxr - minr + 1;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 = back ground colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;		/* background colour */
+	pbm_putpixel (d.bitmap, c - minc, r - minr, p1 != p2);
+      }
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (pbm_getpixel (d.bitmap, c - minc, r - minr))
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= 0) && (c + i >= 0) && (r + j < d.h) && (c + i < d.w) && ((i != 0) || (j != 0)) && (pbm_getpixel (d.bitmap, c + i - minc, r + j - minr)))
+	      count++;
+  pbm_freearray (&d.bitmap, d.h);
+  return count;
+}
+
+
+
+/* define an array for most matches, if it is a massive mark, then WXOR is called */
+#define S_SIZE 500
+
+static unsigned char scratch[S_SIZE][S_SIZE], sc2[S_SIZE][S_SIZE];
+
+
+
+
+/* Weighted AND-NOT */
+
+int 
+WAN_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (b1, b2);
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	scratch[r - minr][c - minc] = sc2[r - minr][c - minc] = 0;
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 == background colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;
+
+	if ((p1 == 0) && (p2 == 1))	/* p1 not set, p2 = set */
+	  scratch[r - minr][c - minc] = 1;
+	if ((p1 == 1) && (p2 == 0))	/* p1 set, p2 not set */
+	  sc2[r - minr][c - minc] = 1;
+      }
+
+  count = 0;
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && scratch[r - minr + j][c - minc + i])
+	      count++;
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (sc2[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && sc2[r - minr + j][c - minc + i])
+	      count++;
+
+  return count;
+}
+
+/* oc 8 must be in a clock wise order, with oc[0] a corner */
+struct
+{
+  int x, y;
+}
+oc[8] =
+{
+  {
+    -1, -1
+  }
+  ,
+  {
+    0, -1
+  }
+  ,
+  {
+    1, -1
+  }
+  ,
+  {
+    1, 0
+  }
+  ,
+  {
+    1, 1
+  }
+  ,
+  {
+    0, 1
+  }
+  ,
+  {
+    -1, 1
+  }
+  ,
+  {
+    -1, 0
+  }
+};
+
+
+
+static void 
+xormatchscratch (marktype * b1, marktype * b2, int minr, int maxr, int minc, int maxc, int offx, int offy)
+{
+  int r, c, rmm, roo, coo;
+  int p1, p2;
+  Pixel **bm1 = b1->bitmap, **bm2 = b2->bitmap;
+  int w1 = b1->w, h1 = b1->h, w2 = b2->w, h2 = b2->h;
+
+  for (r = minr; r < maxr; r++)
+    {
+      rmm = r - minr;
+      roo = r - offy;
+      p1 = 0;
+      for (c = minc; c < 0; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+      for (c = 0; c < maxc; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((r >= 0) && (c < w1) && (r < h1))
+	    p1 = pbm_getpixel (bm1, c, r);
+	  else
+	    p1 = 0;		/* background */
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+    }
+}
+
+
+
+int 
+CSIS_match (marktype * b1, marktype * b2)
+{
+  int r, c, i;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy, fepcount;
+  int totscratch = 0;
+
+
+  offx = b1->xcen - b2->xcen;
+  offy = b1->ycen - b2->ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1->w, b2->w + offx);
+  maxr = max (b1->h, b2->h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (*b1, *b2);
+
+  /* set the scratch array to 1 if there is an XOR match */
+
+  xormatchscratch (b1, b2, minr, maxr, minc, maxc, offx, offy);
+  /* We how have the XOR map. Scratch contains the number of neighbours, 1=1 neighbour set...etc */
+
+  /* CSIS rule 1, if FEP (four error pels) then exit */
+  fepcount = 0;
+  for (r = minr; r < maxr - 1; r++)
+    for (c = minc; c < maxc - 1; c++)
+      if ((scratch[r - minr][c - minc]) && (scratch[r - minr + 1][c - minc]) &&
+	  (scratch[r - minr][c - minc + 1]) && (scratch[r - minr + 1][c - minc + 1]))
+	{
+	  fepcount++;
+	  if (fepcount == 1)
+	    return 10001;
+	}
+
+  /* foreach set pixel, update scratch with the number of pels around in five dir's. 
+     FPN (five-pel neighbour hood) */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	{
+	  sc2[r - minr][c - minc] = 1;	/* itself */
+	  for (i = 1; i < 8; i += 3)
+	    if (((r + oc[i].y) >= minr) && ((c + oc[i].x) >= minc) && ((r + oc[i].y) < maxr) && ((c + oc[i].x) < maxc))
+	      if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])
+		sc2[r - minr][c - minc]++;
+	}
+  /* now scratch has the value of the FPN - 1=single error, 5=4 neighbouring errors etc... */
+
+  /* set scratch and sc2 to be the same */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      totscratch += scratch[r - minr][c - minc] = sc2[r - minr][c - minc];
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	/* CSIS rule 2i: an error pel has 2 or more neighbours---this num is the same as PMS */
+	if (scratch[r - minr][c - minc] >= 3)
+	  if ((r - minr >= 1) && (c - minc >= 1) && (r - minr < (maxr - 1)) && (c - minc < (maxc - 1)))
+	    {
+	      int i, numsep = 0;
+	      /* find how many neighbours a set pixel has, only check the 4 edges-FPN */
+	      for (i = 1; i < 8; i += 2)
+		if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])	/* if a scratch pixel is set */
+		  {
+		    if ((scratch[r + oc[(i + 2) % 8].y - minr][c + oc[(i + 2) % 8].x - minc] == 0) &&
+			(scratch[r + oc[(i + 6) % 8].y - minr][c + oc[(i + 6) % 8].x - minc] == 0)
+		      )
+		      numsep++;
+		  }
+
+	      if ((b1->w >= 12) || (b2->w >= 12) || (b1->h >= 12) || (b2->h >= 12))
+		{
+
+		  /* PMS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, j, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i >= 0) && (r + j >= 0) && (c + i < b1->w) && (r + j < b1->h))
+				  p2 = pbm_getpixel (b1->bitmap, c + i, r + j);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10002;
+			    }
+			}	/* (*b1) in range */
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i - offx >= 0) && (r + j - offy >= 0) && (c + i - offx < b2->w) && (r + j - offy < b2->h))
+				  p2 = pbm_getpixel (b2->bitmap, c + i - offx, r + j - offy);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	      else
+		{		/* do the CSIS rule */
+		  /* CSIS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x >= 0) && (r + oc[i].y >= 0) && (c + oc[i].x < b1->w) && (r + oc[i].y < b1->h))
+			      p2 = pbm_getpixel (b1->bitmap, c + oc[i].x, r + oc[i].y);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10002;
+			    }
+			}
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x - offx >= 0) && (r + oc[i].y - offy >= 0) && (c + oc[i].x - offx < b2->w) && (r + oc[i].y - offy < b2->h))
+			      p2 = pbm_getpixel (b2->bitmap, c + oc[i].x - offx, r + oc[i].y - offy);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	    }
+      }
+
+  return totscratch;
+}
Index: /main/tags/3.01/indexers/mg/src/images/match.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/match.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/match.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * match.h -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MATCH_H
+#define __MATCH_H
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/*
+   #define NOT_SCREENED(b1,b2) ((abs(b1.w-b2.w)<=4) && (abs(b1.h-b2.h)<=4))
+ */
+
+#define NOT_SCREENED(b1,b2) ((abs(b1.xcen-b2.xcen)+abs(b1.ycen-b2.ycen))<=3)
+
+#define MATCH_VAL 1000
+#define MATCH(b1,b2) (CSIS_match(b1,b2))
+
+int XOR_match (marktype b1, marktype b2);
+int WXOR_match (marktype b1, marktype b2);
+int WAN_match (marktype b1, marktype b2);
+int PMS_match (marktype b1, marktype b2);
+int CSIS_match (marktype * b1, marktype * b2);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/mgbilevel.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgbilevel.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgbilevel.1	(revision 21112)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgbilevel.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgbilevel.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgbilevel.c	(revision 21112)
@@ -0,0 +1,149 @@
+/**************************************************************************
+ *
+ * mgbilevel.c -- Program to compress/decompress bilevel images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "utils.h"
+#include "pbmtools.h"
+#include "arithcode.h"
+#include "bilevel.h"
+
+
+
+static char default_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+Encode_bitmap (marktype b, char template[])
+{
+  InitArithEncoding ();
+
+  bl_compress (b, template);
+
+  CloseDownArithEncoding ();
+}
+
+void 
+Decode_bitmap (marktype * b)
+{
+  InitArithDecoding ();
+
+  bl_decompress (b);
+
+  CloseDownArithDecoding ();
+}
+
+
+void 
+usage ()
+{
+  fprintf (stderr,
+	   "usage:\n"
+	   "       mgbilevel -e  [-t \"string\"]  infile      >compressed\n"
+	   "or\n"
+	   "       mgbilevel -d                 compressed  >outfile\n");
+  exit (1);
+}
+
+#define BUF_SIZE 65536
+
+void 
+main (int argc, char *args[])
+{
+  marktype d;
+  char *filename = NULL, *template = NULL;
+  Pixel **bitmap;
+  int encode = 0, decode = 0, i;
+  char buffer[BUF_SIZE];
+
+  if (argc < 2)
+    usage ();
+  template = default_template;
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp (args[i], "-e") == 0)
+	encode = 1;
+      else if (strcmp (args[i], "-d") == 0)
+	decode = 1;
+      else if (strcmp (args[i], "-h") == 0)
+	usage ();
+      else if (strcmp (args[i], "-t") == 0)
+	{
+	  template = args[i + 1];
+	  i++;
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!filename)
+	filename = args[i];
+      else
+	error_msg (args[0], "only specify 1 filename", "");
+    }
+  if (encode == decode)
+    {
+      fprintf (stderr, "You must specify either encode XOR decode.\n");
+      exit (1);
+    }
+
+  if (!filename)
+    {
+      fprintf (stderr, "You must specify a filename.\n");
+      exit (1);
+    }
+
+  if (encode)
+    {
+      bitmap = pbm_readnamedfile (filename, &d.w, &d.h);
+      if (bitmap == NULL)
+	error_msg (args[0], "problem reading bitmap", "");
+
+      arith_out = stdout;
+      setbuffer (stdout, buffer, BUF_SIZE);
+      magic_write (stdout, MAGIC_BILEVEL);
+      if (bitmap)
+	{
+	  d.bitmap = bitmap;
+	  Encode_bitmap (d, template);
+	}
+    }
+  else if (decode)
+    {
+      FILE *tpin;
+      tpin = fopen (filename, "rb");
+      if (!tpin)
+	error_msg ("mgbilevel", "Can't open file:", filename);
+      arith_in = tpin;
+      setbuffer (tpin, buffer, BUF_SIZE);
+      magic_check (tpin, MAGIC_BILEVEL);
+      Decode_bitmap (&d);
+      fclose (tpin);
+
+      setbuffer (stdout, buffer, BUF_SIZE);
+      pbm_writefile (stdout, d.bitmap, d.w, d.h);
+    }
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgfelics.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgfelics.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgfelics.1	(revision 21112)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgfelics.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgfelics.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgfelics.c	(revision 21112)
@@ -0,0 +1,155 @@
+/**************************************************************************
+ *
+ * mgfelics.c -- Program to compress/decompress grey scale images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "utils.h"
+
+
+#define MAXKVALUE 32
+
+
+
+void encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out, int (*get) (FILE *));
+
+void decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out);
+
+
+void 
+usage (char *prog)
+{
+  fprintf (stderr, "usage:\n");
+  fprintf (stderr, "\t%s -e [-k n]   infile    [>compressed]\n"
+	   "or\n"
+	   "\t%s -d  compressed-infile   [>outfile]\n", prog, prog);
+  exit (1);
+}
+
+
+
+
+int 
+main (int argc, char **argv)
+{
+  FILE *fp_in = (stdin), *fp_out = (stdout);
+  int width, height, maxval;
+  u_long magic;
+  char *filename = NULL;
+  unsigned int *line;
+  int maxk = 0;
+  int i;
+  int encode_flag = 0, decode_flag = 0;
+
+  if (argc < 2)
+    usage (argv[0]);
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (argv[i], "-h"))
+	usage (argv[0]);
+      else if (!strcmp (argv[i], "-e"))
+	encode_flag = 1;
+      else if (!strcmp (argv[i], "-d"))
+	decode_flag = 1;
+      else if (!strcmp (argv[i], "-k"))
+	{
+	  maxk = atoi (argv[++i]);
+	}
+      else if (argv[i][0] == '-')
+	error_msg (argv[0], "unknown switch:", argv[i]);
+      else if (!filename)
+	filename = argv[i];
+      else
+	error_msg (argv[0], "too many filenames specified.", "");
+    }
+
+  if (encode_flag == decode_flag)
+    error_msg (argv[0], "please specify either encode XOR decode.", "");
+
+  if (!filename)
+    error_msg (argv[0], "please specify a filename.", "");
+
+  if ((encode_flag) && (maxk >= MAXKVALUE))
+    {				/* to fit into 32 bit integers */
+      fprintf (stderr, "%s: k value of %d is too large; it must be less than %d\n",
+	       argv[0], maxk, MAXKVALUE);
+      exit (1);
+    }
+
+
+  if ((fp_in = fopen (filename, "rb")) == NULL)
+    error_msg (argv[0], "unable to open file:", filename);
+
+  /* read file header to determine the type and dimensions */
+
+
+  magic = magic_read (fp_in);
+  if (magic != MAGIC_FELICS)
+    {
+      rewind (fp_in);
+      magic = getmagicno_short_pop (fp_in);
+    }
+
+  if ((decode_flag && (magic != MAGIC_FELICS)) ||
+      (encode_flag && ((magic != MAGIC_P2) && (magic != MAGIC_P5))))
+    {
+      fclose (fp_in);
+      error_msg (argv[0], encode_flag ? "only PGM files can be used." : "only felics compressed files can be used.", "");
+    }
+
+  width = gethint (fp_in);
+  height = gethint (fp_in);
+  maxval = gethint (fp_in);
+
+  if ((line = (unsigned int *) malloc (width * sizeof (int *))) == NULL)
+    {
+      fclose (fp_in);
+      error_msg (argv[0], "unable to allocate memory,", "");
+    }
+
+
+  if (encode_flag)
+    {
+      magic_write (fp_out, MAGIC_FELICS);
+      fprintf (fp_out, "%d\n%d\n%d\n%d\n", width, height, maxval, maxk);
+      encode (line, width, height, maxval, maxk, fp_in, fp_out,
+	      magic == MAGIC_P2 ? getint : fgetc);
+    }
+  else if (decode_flag)
+    {
+      if (maxk)
+	warn (argv[0], "k value not required for decompression, ignored.", "");
+
+      maxk = gethint (fp_in);
+      fprintf (fp_out, "P%d\n%d %d\n%d\n", maxval < 256 ? 5 : 2,
+	       width, height, maxval);
+      decode (line, width, height, maxval, maxk, fp_in, fp_out);
+    }
+
+
+  fclose (fp_in);
+  fclose (fp_out);
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgtic.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgtic.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgtic.1	(revision 21112)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgtic.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgtic.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgtic.c	(revision 21112)
@@ -0,0 +1,596 @@
+/**************************************************************************
+ *
+ * mgtic.c -- Program to compress/decompress textual images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file is the basis of mgtic, which compresses the components of an
+ * image. 
+ * The compressed file contains the following sections:
+ *         mgtic magic_no
+ *         mgtic version
+ *         lossy/lossless flag 
+ *         internal/external library flag 
+ *         cols, rows, number of marks, library size
+ *         [library sequence]
+ *         symbol sequence
+ *         offset sequence
+ *         [residue bitmap]
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+#include "sortmarks.h"
+#include "codesyms.h"
+#include "arithcode.h"
+#include "codeoffsets.h"
+#include "bilevel.h"
+
+static char library_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgtic -e [-L] [-Q] [-X]  libraryfile   infile    [>compressed]\n"
+	   "or\n"
+	   "\tmgtic -d [-L] [-X libraryfile]  compressed  [>outfile]\n");
+  exit (1);
+}
+
+
+/* encode the list of marks 'list', w.r.t. the library 'library' */
+void 
+match_sequence (marklistptr list, marklistptr library, marklistptr * symbol_list, Pixel ** recon, int quick)
+{
+  int carryx, carryy, gap;
+  int lastx = 0, lasty = 0;
+  int i, j;
+  marktype d, d2;
+  marklistptr liststep, librarystep;
+  int score, found;
+  int r, c;
+  int p;
+
+  carryx = carryy = 0;
+  gap = max (1, marklist_length (list) / 10);
+
+  for (liststep = list, i = 0; liststep; liststep = liststep->next, i++)
+    {
+      if (V)
+	if (!(i % gap))
+	  fprintf (stderr, ".");
+      d = liststep->data;
+
+      found = -1;
+      score = INT_MAX;
+      for (librarystep = library, j = 0; librarystep; librarystep = librarystep->next, j++)
+	{
+	  d2 = librarystep->data;
+
+	  if (NOT_SCREENED (d2, d))
+	    if ((p = MATCH (&d, &d2)) < MATCH_VAL)
+	      {
+		if (p < score)
+		  {
+		    score = p;
+		    found = j;
+		    if (quick)
+		      break;
+		  }
+	      }
+	}
+      /*
+         if(found<0){
+         for(librarystep=library,j=0; librarystep; librarystep=librarystep->next,j++){
+         d2=librarystep->data;
+
+         if((p=MATCH(&d,&d2))<MATCH_VAL){
+         if(p<score){
+         score=p;found=j; if(quick) break;
+         }
+         }
+         }
+         }
+       */
+      if (found >= 0)
+	{
+	  liststep->data.symnum = found;
+	  marklist_getat (library, found, &d2);
+	  d2.symnum = found;
+
+	  d2.xoffset = d.xpos - lastx + carryx;		/* propagate changes to next mark */
+	  d2.yoffset = d.ypos - lasty + carryy;
+	  carryx = (d2.xcen - d.xcen);	/* offet centroids because thats where we match them! */
+	  carryy = (d2.ycen - d.ycen);
+	  d2.xoffset -= carryx;	/* make the changes to this mark */
+	  d2.yoffset -= carryy;
+	  lastx = d.xpos + d2.w;
+	  lasty = d.ypos;
+
+	  marklist_add (symbol_list, d2);
+
+	  /* add to the reconstructed image--this is the lossy version */
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, d.xpos + c - carryx, d.ypos + r - carryy, 1, d2.w, d2.h);
+	}
+      else
+	{
+	  /* didn't match at all..hopefully noise */
+	  if ((liststep->data.w >= 10) || (liststep->data.w >= 10))
+	    {
+	      fprintf (stderr, "mgtic: warning, unable to match a char.\n");
+	      write_library_mark (stderr, liststep->data);
+	    }
+	  if (V)
+	    fprintf (stderr, "x");
+	  liststep->data.symnum = -1;
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+}
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i, n, r, c, ch;
+  FILE *inf;
+  marklistptr library = NULL, list = NULL, step = NULL;
+  marklistptr symbol_list = NULL;
+  marktype d, d2;
+  Pixel **bitmap, **recon, **bitmapcopy;
+  char *splitfilename = NULL;
+  int cols, rows, count, librarysize, lossy = 1;
+  int quick = 0;
+  int ticencode = 0, ticdecode = 0, external = 0;
+  char *libraryname = NULL, *infile = NULL;
+  char *s1 = NULL, *s2 = NULL;
+  char bufferin[BUFSIZ], bufferout[BUFSIZ];
+
+  if (argc < 2)
+    usage ();
+
+  while ((ch = getopt (argc, args, "edlLQhvXR:")) != -1)
+    switch (ch)
+      {
+      case 'e':
+	ticencode = 1;
+	break;
+      case 'd':
+	ticdecode = 1;
+	break;
+      case 'l':
+	lossy = 1;
+	break;
+      case 'L':
+	lossy = 0;
+	break;
+      case 'Q':
+	quick = 1;
+	break;
+      case 'v':
+	V = 1;
+	break;
+      case 'X':
+	external = 1;
+	break;
+      case 'R':
+	splitfilename = optarg;
+	break;
+      case 'h':
+      case '?':
+	usage ();
+      }
+
+  for (i = 1; i < argc; i++)
+    if (args[i][0] != '-')
+      if (strcmp (args[i - 1], "-R"))
+	{			/* ignore arg following -R, ie. -R <filename> */
+	  if (!s1)
+	    s1 = args[i];
+	  else if (!s2)
+	    s2 = args[i];
+	  else
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+  if (ticencode == ticdecode)
+    error_msg (args[0], "please specify either encode XOR decode", "");
+
+  if (ticencode)
+    {
+	  /*****************
+	    ENCODING STAGE
+	  *****************/
+      long PrevBits = 0;
+      long BITS_header = 0, BITS_symbols = 0, BITS_offsets = 0, BITS_residue = 0,
+        BITS_footer = 0, BITS_library = 0;
+
+      libraryname = s1;
+      if (!libraryname)
+	error_msg (args[0], "please specify a library file", "");
+      infile = s2;
+      if (!infile)
+	error_msg (args[0], "please specify a file name", "");
+
+      inf = fopen (infile, "rb");
+      if (inf == NULL)
+	error_msg (args[0], "Trouble opening file:", infile);
+      setbuf (inf, bufferin);
+
+      count = librarysize = read_library (libraryname, &library);
+
+      if (V)
+	fprintf (stderr, "%s: processing...\n", args[0]);
+
+      if (pbm_isapbmfile (inf))
+	{
+	  if (V)
+	    fprintf (stderr, "reading file %s...\n", infile);
+	  bitmap = pbm_readfile (inf, &cols, &rows);
+	  fclose (inf);
+
+	  bitmapcopy = pbm_copy (bitmap, cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "extracting...");
+	  ExtractAllMarks (bitmap, &list, cols, rows);
+	  if (V)
+	    fprintf (stderr, "(%d marks)\n", marklist_length (list));
+
+	  /* sort into reading order */
+	  if (V)
+	    fprintf (stderr, "sorting...\n");
+	  list = sortmarks (list);
+
+	  pbm_freearray (&bitmap, rows);	/* clear old version */
+	  bitmap = bitmapcopy;	/* point to the copy */
+	  recon = pbm_allocarray (cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "matching...");
+
+	  match_sequence (list, library, &symbol_list, recon, quick);
+
+
+	  PrevBits = 0;
+	  setbuf (stdout, bufferout);
+	  /* start output */
+	  magic_write (stdout, MAGIC_TIC);
+	  InitArithEncoding ();
+
+	  EncodeGammaDist (1);	/* version 1 of the program */
+	  EncodeGammaDist (lossy);	/* lossy=1 = no residue */
+	  EncodeGammaDist (external);	/* 1== external file */
+
+	  count = marklist_length (symbol_list);
+	  if (V)
+	    fprintf (stderr, "encoding cols, rows, and number of symbols=%d\n", count);
+
+	  EncodeGammaDist (cols);
+	  EncodeGammaDist (rows);
+	  EncodeGammaDist (count);
+	  EncodeGammaDist (librarysize);
+
+	  BITS_header = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the library sequence */
+	  if (external == 0)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding library\n");
+
+	      bl_clearmodel ();
+	      bl_writetemplate (library_template);
+	      for (step = library; step; step = step->next)
+		bl_compress_mark (step->data);
+	      bl_freemodel ();
+	      BITS_library = CountOfBitsOut - PrevBits;
+	      PrevBits = CountOfBitsOut;
+	    }
+
+	  /* output the symbol sequence */
+	  if (V)
+	    fprintf (stderr, "encoding symbol sequence\n");
+	  InitPPM ();
+	  EncodeSymbols (symbol_list, count);
+	  BITS_symbols = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the offset sequence */
+	  if (V)
+	    fprintf (stderr, "encoding offset sequence\n");
+	  EncodeOffsets (symbol_list, count);
+	  BITS_offsets = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+	  EncodeChecksum ();	/* code lossy checksum */
+
+	  /* calculate the residue...and compress it---if need be! */
+	  if (!lossy)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding residue...\n");
+
+	      d.bitmap = bitmap;
+	      d.h = rows;
+	      d.w = cols;
+	      d2.bitmap = recon;
+	      d2.h = rows;
+	      d2.w = cols;
+
+	      if (splitfilename)
+		{
+		  FILE *temp;
+
+		  CloseDownArithEncoding ();
+		  fclose (stdout);
+		  /* no residue result */
+		  BITS_footer = CountOfBitsOut - PrevBits;
+
+		  temp = fopen (splitfilename, "wb");
+		  if (temp == NULL)
+		    error_msg (args[0], "Trouble creating file:", splitfilename);
+
+		  arith_out = temp;
+		  InitArithEncoding ();
+		  bl_clair_compress (d, d2);
+		  CloseDownArithEncoding ();
+		  BITS_residue = CountOfBitsOut;
+		  fclose (temp);
+		}
+	      else
+		{
+		  bl_clair_compress (d, d2);
+		  BITS_residue = CountOfBitsOut - PrevBits;
+		  PrevBits = CountOfBitsOut;
+		  EncodeChecksum ();	/* code lossless checksum */
+		  CloseDownArithEncoding ();
+		  BITS_footer = CountOfBitsOut - PrevBits;
+		}
+	    }
+	  else
+	    {
+	      if (V)
+		fprintf (stderr, "not encoding residue..lossy mode\n");
+	      CloseDownArithEncoding ();
+	      BITS_footer = CountOfBitsOut - PrevBits;
+	    }
+
+	  /* because we edit the values above */
+	  CountOfBitsOut = BITS_header + BITS_library + BITS_symbols + BITS_offsets + BITS_residue + BITS_footer;
+
+	  fprintf (stderr, "bits: header=%ld, library=%ld, "
+		   "symbols=%ld, offsets=%ld, residue=%ld, footer=%ld\n",
+		   BITS_header, BITS_library, BITS_symbols,
+		   BITS_offsets, BITS_residue, BITS_footer);
+	  fprintf (stderr, "total bits: %ld, ", CountOfBitsOut);
+	  fprintf (stderr, "Lossy CR: %4.2f", (cols * rows) / (float) (CountOfBitsOut - BITS_residue));
+	  if (external)
+	    fprintf (stderr, " (excluding external lib)");
+	  fprintf (stderr, ", Lossless CR: %4.2f\n", (!lossy) * (cols * rows) / (float) (CountOfBitsOut));
+	}
+      else
+	error_msg (args[0], "unknown format of bitmap--expecting PBM.", "");
+    }
+  else
+    {
+	  /*****************
+	    DECODING STAGE
+	  *****************/
+      int lastx, lasty;
+      librarysize = 0;
+
+      if (external)
+	{
+	  libraryname = s1;
+	  infile = s2;
+	  count = librarysize = read_library (libraryname, &library);
+	}
+      else
+	{
+	  infile = s1;
+	  if (infile && s2)
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+      if (!freopen (infile, "rb", stdin))
+	error_msg (args[0], "Trouble opening file:", infile);
+
+
+      if (V)
+	fprintf (stderr, "decompressing...\n");
+
+      setbuf (stdin, bufferin);
+      magic_check (stdin, MAGIC_TIC);
+
+      InitArithDecoding ();
+
+      {
+	int version = DecodeGammaDist ();
+	if (version != 1)
+	  error_msg (args[0], "Need later version of decompressor.", "");
+      }
+
+      {
+	int templossy = DecodeGammaDist ();
+	if (!lossy)
+	  lossy = templossy;	/* can only choose if encoded file is lossless */
+	if (V)
+	  {
+	    if (lossy)
+	      fprintf (stderr, "lossy mode\n");
+	    else
+	      fprintf (stderr, "lossless mode\n");
+	  }
+      }
+
+      if (DecodeGammaDist ())
+	{			/* if compressed file doesn't contain library */
+	  if (!external)
+	    error_msg (args[0], "compressed file doesn't contain library, specify externally", "");
+	  external = 1;
+	}
+      else
+	{			/* if compressed file contains library */
+	  if (external)
+	    fprintf (stderr, "ignoring external library file\n");
+	  external = 0;
+	}
+
+      cols = DecodeGammaDist ();
+      rows = DecodeGammaDist ();
+      count = DecodeGammaDist ();
+
+      i = DecodeGammaDist ();	/* librarysize */
+      if (external)
+	{
+	  if (i > librarysize)
+	    error_msg (args[0], "external library file is too small!", "");
+	  else if (i < librarysize)
+	    fprintf (stderr, "%s: warning, expecting a different (smaller) library.\n", args[0]);
+	}
+      librarysize = i;
+
+      if (V)
+	fprintf (stderr, "cols %d, rows %d, num syms %d, library size %d\n", cols, rows, count, librarysize);
+
+      /* decode library */
+      if (external == 0)
+	{
+	  if (V)
+	    fprintf (stderr, "reading library\n");
+	  bl_clearmodel ();
+	  bl_readtemplate ();
+	  for (n = 0; n < librarysize; n++)
+	    {
+	      bl_decompress_mark (&d);
+	      d.symnum = n;
+	      if (library == NULL)
+		step = marklist_add (&library, d);
+	      else
+		step = marklist_add (&step, d);
+	    }
+	  bl_freemodel ();
+	  if (V)
+	    fprintf (stderr, "read %d marks from library\n", marklist_length (library));
+	}
+
+      recon = pbm_allocarray (cols, rows);
+
+      /* decode symbols */
+      InitPPM ();
+      if (V)
+	fprintf (stderr, "decompressing %d symbols\n", count);
+      symbol_list = DecodeSymbols (count);
+
+      /* decode offsets */
+      if (V)
+	fprintf (stderr, "reading offsets...\n");
+      DecodeOffsets (symbol_list, count);
+      lastx = lasty = 0;
+      for (step = symbol_list; step; step = step->next)
+	{
+	  lastx = lastx + step->data.xoffset;
+	  lasty = lasty + step->data.yoffset;
+
+	  marklist_getat (library, step->data.symnum, &d2);
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, lastx + c, lasty + r, 1, cols, rows);	/* we don't care, already warned them! */
+	  lastx += d2.w;
+	}
+
+      DecodeChecksum (args[0]);
+
+      /* decode the residue */
+      if (!lossy)
+	{
+	  if (V)
+	    fprintf (stderr, "decoding residue...\n");
+
+	  bitmap = pbm_allocarray (cols, rows);
+	  d.bitmap = bitmap;
+	  d.w = d2.w = cols;
+	  d.h = d2.h = rows;
+
+	  d2.bitmap = recon;
+
+	  /* NOTE: the 2nd argument is clairvoyantly compressed */
+	  if (splitfilename)
+	    {
+	      FILE *temp;
+
+	      CloseDownArithDecoding ();
+	      fclose (stdin);
+
+	      temp = fopen (splitfilename, "rb");
+	      if (temp == NULL)
+		error_msg (args[0], "Trouble opening file:", splitfilename);
+
+	      arith_in = temp;
+	      InitArithDecoding ();
+	      bl_clair_decompress (d, d2);
+	      CloseDownArithDecoding ();
+	      fclose (temp);
+	    }
+	  else
+	    {
+	      bl_clair_decompress (d, d2);
+	      DecodeChecksum (args[0]);
+	      CloseDownArithDecoding ();
+	    }
+	  pbm_freearray (&recon, rows);
+	  recon = bitmap;	/* point to the bitmap */
+	}
+      else
+	{
+	  CloseDownArithDecoding ();
+	}
+
+      if (V)
+	fprintf (stderr, "writing pbm file...\n");
+      setbuf (stdout, bufferout);
+      pbm_writefile (stdout, recon, cols, rows);
+      pbm_freearray (&recon, rows);
+    }				/* end decoding */
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgticbuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticbuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticbuild.1	(revision 21112)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgticbuild.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticbuild.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticbuild.c	(revision 21112)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * mgticbuild.c -- Program to build the textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgticbuild   libraryfile   infile.pbm  [infile2 infile3 ...]\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib = NULL, *inf = NULL;
+  marklistptr list = NULL, step;
+  Pixel **bitmap;
+  int cols, rows;
+  int count, pos = 0;
+  char *libraryname = NULL;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	pos = i;
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+  if (pos < 2)
+    usage ();
+
+  count = 0;
+  for (i = pos; i < argc; i++)
+    if (args[i][0] != '-')
+      {
+	inf = fopen (args[i], "rb");
+	count = 0;
+	if (inf == NULL)
+	  fprintf (stderr, "Trouble opening file: %s\n", args[i]);
+	else
+	  {
+	    if (pbm_isapbmfile (inf))
+	      {
+		if (V)
+		  fprintf (stderr, "%s: processing...\n", args[0]);
+		if (V)
+		  fprintf (stderr, "reading file %s...", args[i]);
+		bitmap = pbm_readfile (inf, &cols, &rows);
+		if (V && (count == 0))
+		  fprintf (stderr, "building...");
+
+		list = NULL;
+
+		ExtractAllMarks (bitmap, &list, cols, rows);
+
+		if (V)
+		  fprintf (stderr, "writing...");
+
+		if ((!lib) && (marklist_length (list)))
+		  {
+		    lib = fopen (libraryname, "r");
+		    if (lib)
+		      {
+			fclose (lib);
+			lib = fopen (libraryname, "ab");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble appending library file", "");
+			fprintf (stderr, "\nwarning: appending library file: %s\n", libraryname);
+		      }
+		    else
+		      {
+			lib = fopen (libraryname, "wb");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble creating library file", "");
+		      }
+		  }		/* if !lib */
+
+		step = list;
+		while (step)
+		  {
+		    step->data.symnum = count;
+		    write_library_mark (lib, step->data);
+		    count++;
+		    step = step->next;
+		  }
+		pbm_freearray (&bitmap, rows);
+	      }
+	    else
+	      fprintf (stderr, "warning: %s is not a PBM file.\n", args[i]);
+	    fclose (inf);
+	  }
+	if (count)
+	  {
+	    if (V)
+	      fprintf (stderr, "%d marks written.\n", count);
+	  }
+	else
+	  fprintf (stderr, "No marks written!\n");
+      }
+
+  if (lib)
+    fclose (lib);
+  marklist_free (&list);
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgticdump.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticdump.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticdump.1	(revision 21112)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgticdump.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticdump.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticdump.c	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mgticdump.c -- Program to dump out a library in a human readable form
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticdump   libraryfile\n");
+  exit (1);
+}
+
+
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib;
+  int count, i;
+  marktype d;
+  char *libraryname = NULL;
+  FILE *outf = (stdout);
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble opening library file:", libraryname);
+
+  count = 0;
+  while (!isEOF (lib))
+    {
+      read_library_mark (lib, &d);
+
+      write_library_mark (outf, d);
+      count++;
+
+      pbm_freearray (&d.bitmap, d.h);
+    }
+  fprintf (stderr, "count=%d", count);
+  fclose (lib);
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgticprune.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticprune.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticprune.1	(revision 21112)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgticprune.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticprune.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticprune.c	(revision 21112)
@@ -0,0 +1,393 @@
+/**************************************************************************
+ *
+ * mgticprune.c -- Program to prune a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+
+marklistptr *avglib = NULL;
+int avglibsize;
+int libsize = 0;
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+    "\tmgticbuild [-a n]  [-s n]  [-Q]   libraryfile   [pruned-library]\n");
+  exit (1);
+}
+
+
+
+
+/* foreach list in the avglib[] array, an average mark is returned */
+marklistptr 
+average_library ()
+{
+  marklistptr librarylist;
+  int i;
+  marktype avgmark;
+
+  librarylist = NULL;
+  for (i = 0; i < libsize; i++)
+    {
+      if (avglib[i])
+	{
+	  marklist_average (avglib[i], &avgmark);
+	  avgmark.symnum = i;
+	  marklist_addcopy (&librarylist, avgmark);
+	}
+    }
+  return librarylist;
+}
+
+
+void 
+average_space_enlarge ()
+{
+  int j;
+
+  if (libsize >= avglibsize)
+    {
+      avglib = (marklistptr *) realloc (avglib, (avglibsize + 100) * sizeof (marklistptr));
+      if (!avglib)
+	error_msg ("mgticprune", "out of memory in reallocating..", "");
+      for (j = avglibsize; j < avglibsize + 100; j++)
+	avglib[j] = NULL;
+      avglibsize += 100;
+    }
+}
+
+
+/* for each mark in 'origlist', if it can't be matched with one in 
+   'librarylist', then add to the librarylist, and update the relevant
+   avglib[] position */
+void 
+greedy_add (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, step_i;
+  int i, j, notfound;
+  marktype d_i;
+
+  for (i = 0, step_i = origlist; step_i; i++, step_i = step_i->next)
+    {
+      d_i = step_i->data;
+
+      notfound = 1;
+      for (j = 0, step = *librarylist; step; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d_i, step->data))
+	    if (MATCH (&d_i, &step->data) < MATCH_VAL)
+	      {
+		d_i.symnum = j;
+		marklist_addcopy (&avglib[j], d_i);
+		notfound = 0;
+		break;
+	      }
+	}
+
+      if (notfound)
+	{
+	  d_i.symnum = libsize;
+	  marklist_addcopy (librarylist, d_i);
+	  marklist_addcopy (&avglib[libsize], d_i);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+/* for each mark in 'library', if another matches with it, remove
+   one copy from the library, and erase the relevant avglib[] */
+void 
+greedy_prune (marklistptr * library)
+{
+  int count;
+  int i, j;
+  marklistptr step;
+  marktype d;
+
+  count = marklist_length (*library);
+  /* an invariant, all marks <i have been kept in the pruned library */
+  for (i = 0; i < count; i++)
+    {
+      marklist_getat (*library, i, &d);
+
+      for (j = 0, step = *library; j < i; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d, step->data))
+	    if (MATCH (&d, &step->data) < MATCH_VAL)
+	      {
+		/* [i,d] has matched with library element [j,d2]. */
+		marklist_removeat (library, i);		/* because we kill it here */
+		marklist_free (&avglib[d.symnum]);
+		avglib[d.symnum] = NULL;
+		i--, count--;
+		break;
+	      }
+	}
+    }
+}
+
+
+void 
+prune_on_size (marklistptr * list, int sizethres)
+{
+  int count, i;
+  marktype d;
+
+  if (sizethres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if ((d.w <= sizethres) && (d.h <= sizethres))
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+
+void 
+prune_on_area (marklistptr * list, int areathres)
+{
+  int count, i;
+  marktype d;
+
+  if (areathres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if (d.set <= areathres)
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+void 
+averagedout_addagain (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, stepi;
+  int notfound;
+  int i, j;
+
+  for (i = 0, step = origlist; step; i++, step = step->next)
+    {
+      notfound = 1;
+
+      for (j = 0, stepi = *librarylist; stepi; j++, stepi = stepi->next)
+	if (NOT_SCREENED (step->data, stepi->data))
+	  if (MATCH (&step->data, &stepi->data) < MATCH_VAL)
+	    {
+	      notfound = 0;
+	      break;
+	    }
+
+      if (notfound)
+	{
+	  marklist_addcopy (librarylist, step->data);
+	  marklist_addcopy (&avglib[libsize], step->data);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib;
+
+  marklistptr origlist = NULL, librarylist = NULL;
+  marklistptr step;
+  char tempn[1000], sysstr[1000];
+  int count;
+  char *libraryname = NULL, *prunename = NULL;
+  int sizethres = 3, areathres = 0;
+  int pass;
+  int num_passes = 3;
+  int overwrite = 0;
+
+  V = 0;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (!strcmp (args[i], "-Q"))
+	{
+	  num_passes = 0;
+	}
+      else if (!strcmp (args[i], "-s"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    sizethres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -s option with a numeric arg >=0", "");
+	}
+      else if (!strcmp (args[i], "-a"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    areathres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -a option with a numeric arg >=0", "");
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else if (!prunename)
+	prunename = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+  if (V)
+    fprintf (stderr, "%s: processing...\n", args[0]);
+
+  overwrite = 0;
+  if (!prunename)
+    {
+      prunename = libraryname;
+      if (V)
+	fprintf (stderr, "overwriting previous library file.\n");
+      overwrite = 1;
+    }
+
+  if (overwrite)
+    {
+      tmpnam (tempn);
+    }
+  else
+    strcpy (tempn, prunename);
+
+
+  avglibsize = count = read_library (libraryname, &origlist);
+  if (V)
+    fprintf (stderr, "%d marks...", count);
+
+  avglib = (marklistptr *) malloc (count * sizeof (marklistptr));
+  if (!avglib)
+    error_msg (args[0], "out of memory", "");
+  for (i = 0; i < avglibsize; i++)
+    avglib[i] = NULL;
+
+
+  prune_on_size (&origlist, sizethres);
+
+  prune_on_area (&origlist, areathres);
+  if (V)
+    fprintf (stderr, "after threshold pruning, %d marks\n", count = marklist_length (origlist));
+
+
+  if (V)
+    fprintf (stderr, "starting greedy library creation...\n");
+
+  greedy_add (origlist, &librarylist);
+
+
+  for (pass = 0; pass < num_passes; pass++)
+    {
+      marklist_free (&librarylist);
+      if (V)
+	fprintf (stderr, "averaging...\n");
+      librarylist = average_library ();
+      averagedout_addagain (origlist, &librarylist);
+      greedy_prune (&librarylist);
+      if (V)
+	fprintf (stderr, "after pass %d greedy prune %d marks\n", 1 + pass, marklist_length (librarylist));
+    }
+
+
+  lib = fopen (tempn, "wb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble creating prune-library file.", "");
+
+
+  if (V)
+    fprintf (stderr, "writing to file: %s...", prunename);
+  for (step = librarylist, count = 0; step; step = step->next, count++)
+    {
+      step->data.symnum = count;
+      write_library_mark (lib, step->data);
+    }
+
+  if (V)
+    fprintf (stderr, "%d marks.\n", count);
+  fclose (lib);
+
+  if (overwrite)
+    {
+      sprintf (sysstr, "mv %s %s", tempn, prunename);
+      system (sysstr);		/* move temp to proper name */
+    }
+
+  for (i = 0; i < avglibsize; i++)
+    {
+      marklist_free (&avglib[i]);
+      free (avglib[i]);
+    }
+}
Index: /main/tags/3.01/indexers/mg/src/images/mgticstat.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticstat.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticstat.1	(revision 21112)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /main/tags/3.01/indexers/mg/src/images/mgticstat.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/mgticstat.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/mgticstat.c	(revision 21112)
@@ -0,0 +1,130 @@
+/**************************************************************************
+ *
+ * mgticstat.c -- Program to print out stats about a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticstat   libraryfile\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib, *outf = (stdout);
+  int count;
+  int small1, small2, small4, small9, small16, small25, small36, small49,
+    small64, small81, small100;
+  marktype d;
+  int avx, avy;
+  int hhh, www, i;
+  int avruns_h = 0, avruns_v = 0;
+  int av_area = 0;
+  char *libraryname = NULL;
+
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "r");
+  if (lib == NULL)
+    error_msg (args[0], "Trouble opening library file", "");
+
+  count = 0;
+  small1 = small2 = small4 = small9 = small16 = small25 = small36 = small49 = small64 = small81 = small100 = 0;
+  avx = avy = 0;
+  www = hhh = 0;
+  while (!isEOF (lib))
+    {
+      if (!read_library_mark (lib, &d))
+	error_msg (args[0], "error in library file", "");
+
+      av_area += d.set;
+      if (d.set <= 1)
+	small1++;
+      if (d.set <= 2)
+	small2++;
+      if (d.set <= 4)
+	small4++;
+      if (d.set <= 9)
+	small9++;
+      if (d.set <= 16)
+	small16++;
+      if (d.set <= 25)
+	small25++;
+      if (d.set <= 36)
+	small36++;
+      if (d.set <= 49)
+	small49++;
+      if (d.set <= 64)
+	small64++;
+      if (d.set <= 81)
+	small81++;
+      if (d.set <= 100)
+	small100++;
+      avx += d.xcen;
+      avy += d.ycen;
+      www += d.w;
+      hhh += d.h;
+      avruns_h += d.h_runs;
+      avruns_v += d.v_runs;
+      pbm_freearray (&d.bitmap, d.h);
+      count++;
+      if (count % 5000 == 0)
+	fprintf (outf, "%dk ", count / 1000);
+    }
+  fprintf (outf, "number of marks=%d\n", count);
+  fprintf (outf, "average area=%d\n", av_area / count);
+  fprintf (outf, "area <=1:%d, <=2:%d, <=4:%d, <=9:%d, <=16:%d, <=25:%d, <=36:%d, <=49:%d, <=64:%d, <=81:%d, <=100:%d\n", small1, small2, small4, small9, small16, small25, small36, small49, small64, small81, small100);
+
+  fprintf (outf, "average num runs, horizontal=%d, vertical=%d\n", avruns_h / count, avruns_v / count);
+  fprintf (outf, "average centroid position=(%d,%d)\n", avx / count, avy / count);
+  fprintf (outf, "average width=%d, average height=%d\n", www / count, hhh / count);
+  fclose (lib);
+}
Index: /main/tags/3.01/indexers/mg/src/images/model.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/model.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/model.h	(revision 21112)
@@ -0,0 +1,92 @@
+/**************************************************************************
+ *
+ * model.h -- Part pf PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#define defaultkbytes	896
+#define minkbytes	56
+#define maxkbytes	((65536/1024)*sizeof(eventnode))
+
+#ifndef increment
+#define	increment	8
+#endif
+#ifndef halveat
+#define halveat		64
+#endif
+#define maxtotalcnt 	(increment*halveat)
+
+typedef int boolean;
+#define false           0
+#define true            1
+
+typedef short event;
+typedef unsigned short point;
+
+typedef struct
+  {
+    short totalcnt;
+    short notfound;
+    point list;			/* list storing the event records */
+  }
+eventset;
+
+typedef struct
+  {
+    event eventnum;
+    short count;
+    point next, prev;
+    eventset foll;
+  }
+eventnode, *eventptr;
+
+#define memsize		(kbytes*1024)
+#define maxnodes	(memsize/sizeof(eventnode))
+/* #define charsetsize     256 -- MAHE */
+#define nchars          (charsetsize+1)
+
+extern int charsetsize;		/* -- MAHE */
+
+extern eventptr E;
+extern long kbytes;
+extern point numnodes, nfreenodes;
+extern boolean excluded[];	/* -- MAHE */
+
+#define ENULL		(E)
+
+typedef unsigned u;
+#define p32(p)	(E+p)
+#define p16(p)	(p-E)
+
+/* in ppm.c */
+void write_model ();
+void encodestring ();
+void decodestring ();
+void encodefile ();
+void decodefile ();
+
+/* in lstevent.c */
+void write_method ();
+eventnode *encode_event_noex ();
+eventnode *decode_event_noex ();
+eventnode *encode_event ();
+eventnode *decode_event ();
+/* eventnode *addevent(); */
+eventnode *newnode ();
Index: /main/tags/3.01/indexers/mg/src/images/pbmtools.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/pbmtools.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/pbmtools.c	(revision 21112)
@@ -0,0 +1,288 @@
+/**************************************************************************
+ *
+ * pbmtools.c -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+
+void 
+pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    pbm_putpixel (bitmap, c, r, val);
+  else
+    fprintf (stderr, "putpitxel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+}
+
+int 
+pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    return pbm_getpixel (bitmap, c, r);
+  else
+    {
+      fprintf (stderr, "getpixel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+      return 0;
+    }
+}
+
+
+
+
+
+/*
+   Returns 1 if *fp points to a PBM (magic is P1 or P4) file.
+   leaves the fp in it's original position.
+   usage: if(pbm_isapbmfile(fp)) process(fp);
+ */
+int 
+pbm_isapbmfile (FILE * fp)
+{
+  u_long magic;
+
+  magic = getmagicno_short (fp);
+  if ((magic == MAGIC_P4) || (magic == MAGIC_P1))
+    return 1;
+  else
+    return 0;
+}
+
+
+/*
+   Frees the memory used in the bitmap structure.
+   usage: pbm_freearray(&bitmap);
+   Sets bitmap to be NULL on return.
+ */
+void 
+pbm_freearray (Pixel *** bitmap, int rows)
+{
+  int r;
+  if (rows == 0)
+    rows++;
+  for (r = 0; r < rows; r++)
+    free ((*bitmap)[r]);
+  free (*bitmap);
+  (*bitmap) = NULL;
+}
+
+
+/*
+   This function allocates the bitmap.
+   usage: bitmap=pbm_allocarray(cols,rows);
+ */
+Pixel **
+pbm_allocarray (int cols, int rows)
+{
+  Pixel **p;
+  int i, newc = (cols + PBITS) / PBITS;
+
+  if (cols == 0)
+    cols++;
+  if (rows == 0)
+    rows++;
+  p = (Pixel **) malloc (rows * sizeof (Pixel *));
+  if (p == NULL)
+    error_msg ("pbm_allocarray", "out of memory", "");
+  for (i = 0; i < rows; i++)
+    {
+      p[i] = (Pixel *) calloc (newc, sizeof (Pixel));
+      if (p[i] == NULL)
+	error_msg ("pbm_allocarray", "out of memory", "");
+    }
+  return p;
+}
+
+
+
+Pixel **
+pbm_copy (Pixel ** bitmap, int cols, int rows)
+{
+  Pixel **p;
+  int newc = (cols + PBITS) / PBITS;
+  int x, y;
+
+  p = pbm_allocarray (cols, rows);
+  for (x = 0; x < newc; x++)
+    for (y = 0; y < rows; y++)
+      p[y][x] = bitmap[y][x];
+  return p;
+}
+
+
+
+
+/*
+   Reads in a pbm file and returns a pointer to the allocated bitmap, 
+   in the P4 format.
+   Assumes fp was opened and keeps it that way.
+   usage: bitmap=pbm_readfile(fp,&cols,&rows);
+ */
+Pixel **
+pbm_readfile (FILE * fp, int *cols, int *rows)
+{
+  Pixel **p = NULL;
+  char s[100];
+  unsigned char *buf;
+  int r, c, ii = 0;
+  int p1 = 0;
+  int arg;
+
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  if (strcmp (s, "P1") == 0)
+    p1 = 1;
+  else if (strcmp (s, "P4") == 0)
+    p1 = 0;
+  else
+    error_msg ("pbm_readfile", "only pbm files are supported", "");
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  arg = sscanf (s, "%d %d", cols, rows);
+  if (arg == 1)
+    {
+      readline (s, fp);
+      arg = sscanf (s, "%d", rows);
+      if (arg != 1)
+	error_msg ("pbm_readfile", "bizzare pbm format", "");
+    }
+  p = pbm_allocarray (*cols, *rows);
+  buf = (unsigned char *) malloc ((*cols + 2) * sizeof (unsigned char));
+  if (!buf)
+    error_msg ("pbm_readfile", "out of memory", "");
+
+  if (p1)
+    {
+      for (r = 0; r < *rows; r++)
+	for (c = 0; c < *cols; c++)
+	  {
+	    fscanf (fp, "%d", &ii);
+	    pbm_putpixel (p, c, r, ii);
+	  }
+    }
+  else
+    {
+      for (r = 0; r < *rows; r++)
+	{
+	  c = 0;
+	  fread (buf, 1, (*cols + 7) / 8, fp);
+	  for (c = 0; c < *cols; c++)
+	    {
+	      if (!(c % 8))
+		ii = buf[c / 8];
+	      if (ii < 0)
+		error_msg ("pbm_readfile", "file reading error", "");
+	      pbm_putpixel (p, c, r, (ii & 128) ? 1 : 0);
+	      ii <<= 1;
+	    }
+	}
+    }				/* p4 type */
+  free (buf);
+  return p;
+}
+
+
+/*
+   Writes a pbm file given a file pointer and bitmap. Assumes that
+   fp is already open, and keeps in that way.
+   usage: pbm_writefile(fp,bitmap,cols,rows);
+ */
+void 
+pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows)
+{
+  int r, c, ii = 0, j;
+  unsigned char *buf;
+
+  buf = (unsigned char *) malloc (((cols + 14) / 8) * sizeof (unsigned char));
+
+  fprintf (fp, "%s\n", "P4");
+  fprintf (fp, "%d %d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c += 8)
+	{
+	  ii = 0;
+	  for (j = 0; j < 8; j++)
+	    if (c + j < cols)
+	      {
+		if (pbm_getpixel (bitmap, c + j, r))
+		  ii |= (128 >> j);
+	      }
+	  buf[c / 8] = ii;
+	}
+      fwrite (buf, 1, (cols + 7) / 8, fp);
+    }
+  free (buf);
+/*      putc('\n',fp);   if this line is included programs don't say things like 'short image' &c, but then again,
+   in most cases a 'cmp' between the original will fail because of the last byte...sigh, "standards". */
+}
+
+#define BUF_SIZE 131072
+char buffer[BUF_SIZE];
+
+void 
+pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows)
+{
+  FILE *tempfp;
+
+  tempfp = fopen (fn, "wb");
+  if (tempfp == NULL)
+    error_msg ("pbm_writenamedfile", "can't open file:", fn);
+  setbuffer (tempfp, buffer, BUF_SIZE);
+  pbm_writefile (tempfp, bitmap, cols, rows);
+  fclose (tempfp);
+}
+
+
+
+Pixel **
+pbm_readnamedfile (char fn[], int *cols, int *rows)
+{
+  FILE *tempfp;
+  Pixel **temp;
+
+  tempfp = fopen (fn, "rb");
+  if (tempfp == NULL)
+    return NULL;
+  setbuffer (tempfp, buffer, BUF_SIZE);
+
+  temp = pbm_readfile (tempfp, cols, rows);
+
+  fclose (tempfp);
+  return temp;
+}
Index: /main/tags/3.01/indexers/mg/src/images/pbmtools.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/pbmtools.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/pbmtools.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * pbmtools.h -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __PBMTOOLS_H
+#define __PBMTOOLS_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+#include "utils.h"
+#include "marklist.h"
+
+#define PBITS 32
+#define PSH 5
+
+
+/* The next two are ordered pixels col0=left, cols31=right */
+
+/*
+   #define pbm_getpixel(bitmap,c,r)  \
+   ((bitmap[r][(c)>>PSH] >> ((PBITS-1)-((c)&(PBITS-1))))&1)
+   #define pbm_putpixel(bitmap,c,r,val) \
+   if((val))  \
+   bitmap[r][(c)>>PSH] |=  (((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))); \
+   else \
+   bitmap[r][(c)>>PSH] &= ~(((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))) 
+ */
+
+
+
+
+/* The next two defines are ordered pixel cols0=right, cols31=left */
+
+#define pbm_getpixel(bitmap,c,r)                            \
+         ((bitmap[r][(c)>>PSH] >> ((c)&(PBITS-1)))&1)
+#define pbm_putpixel(bitmap,c,r,val)                        \
+         if((val))                                          \
+              bitmap[r][(c)>>PSH] |= (1<<((c)&(PBITS-1)));  \
+         else                                               \
+              bitmap[r][(c)>>PSH] &= ~(1<<((c)&(PBITS-1)))
+
+
+#define pbm_putpixel_trunc(bitmap,c,r,val,cols,rows) \
+       if(((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) \
+             pbm_putpixel(bitmap,c,r,val)
+
+#define pbm_getpixel_trunc(bitmap,c,r,cols,rows)       \
+       ((((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) ?  \
+             pbm_getpixel(bitmap,c,r) : 0)
+
+
+
+
+int pbm_isapbmfile (FILE * fp);
+void pbm_freearray (Pixel *** bitmap, int rows);
+Pixel **pbm_copy (Pixel ** bitmap, int cols, int rows);
+Pixel **pbm_allocarray (int cols, int rows);
+Pixel **pbm_readfile (FILE * fp, int *cols, int *rows);
+Pixel **pbm_readnamedfile (char fn[], int *cols, int *rows);
+void pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows);
+void pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows);
+
+void pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows);
+int pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/ppm.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/ppm.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/ppm.c	(revision 21112)
@@ -0,0 +1,588 @@
+/**************************************************************************
+ *
+ * ppm.c -- PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:16:56  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:33  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:55  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:02  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* FIXEDORDER sets the maximum context length that will be used.
+   Making FIXEDORDER large increases the compression if there is enough
+   space to store the model, but when space limit is reached increasing
+   FIXEDORDER will decrease compression because of model thrashing.
+   Third order (FIXEDORDER=3) is suitable for a few hundred Kb of model
+   space, FIXEDORDER=1 is suitable for a few tens of kB.
+   This version requires that FIXEDORDER be a preprocessor rather
+   than runtime variable to allow loops to be unrolled and make for faster
+   execution. Maximum in this implementation is 3 */
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+/* global variables */
+eventptr E = NULL;		/* array of eventnodes */
+point numnodes = 0, nfreenodes = 0;
+long kbytes = defaultkbytes;
+/*boolean               excluded[nchars]={false};  - MAHE */
+boolean excluded[16384] =
+{false};			/* excluded characters */
+
+/* local variables */
+#define defaultchunksize        4096	/* compression chunks */
+
+#define EOSchar         (charsetsize)
+
+unsigned chunksize = defaultchunksize;
+eventptr s[FIXEDORDER + 1] =
+{NULL},				/* current contexts */
+  t[FIXEDORDER + 1] =
+{NULL};				/* new contexts */
+
+#if (FIXEDORDER==3)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1] ; s[3] = t[2]
+#else
+#if (FIXEDORDER==2)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1]
+#else
+#if (FIXEDORDER==1)
+#define SHIFT_ST	s[1] = t[0]
+#else
+#define SHIFT_ST
+#endif
+#endif
+#endif
+
+/* follow a character list and set the exclusions */
+#define SET_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = true;	\
+}						\
+while (false)
+
+/* clear the exclusions */
+#define CLEAR_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = false;	\
+}						\
+while (false)
+
+/*==================================*/
+
+void
+write_model ()
+{
+  fprintf (stderr, "ppm");
+  write_method ();
+#ifdef NOEXCLUSIONS
+  fprintf (stderr, "nx");
+#endif
+  fprintf (stderr, "-o%1d", FIXEDORDER);
+  fprintf (stderr, "-k%ld", kbytes);
+}
+
+/*==================================*/
+
+void
+encode_in_order (n, e)
+     int n;
+     event e;
+/* encode character e in order n */
+{
+  boolean encoded;
+  int i, c;
+  t[n] = encode_event_noex (&s[n]->foll, e, &encoded);
+  if (encoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = encode_event_noex (&s[c]->foll, e, &encoded);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = encode_event (&s[c]->foll, e, &encoded);
+#endif
+	if (encoded)
+	  break;
+      }
+  if (!encoded)
+    arithmetic_encode (e, e + 1, nchars);
+  for (i = c + 1; i <= n; i++)
+    if (i > 0)
+      t[i]->prev = p16 (t[i - 1]);
+    else
+      t[0]->prev = p16 (s[0]);
+  for (i = c - 1; i >= 0; i--)
+    t[i] = p32 (t[i + 1]->prev);
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+/*==================================*/
+
+/* not required in this version */
+
+#if 0
+
+void
+decode_in_order (n, e)
+     int n;
+     event *e;
+{
+  boolean decoded;
+  int i, c;
+  t[n] = decode_event_noex (&s[n]->foll, &decoded, e);
+  if (decoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = decode_event_noex (&s[c]->foll, &decoded, e);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = decode_event (&s[c]->foll, &decoded, e);
+#endif
+	if (decoded)
+	  break;
+      }
+  if (!decoded)
+    {
+      *e = arithmetic_decode_target (nchars);
+      arithmetic_decode (*e, *e + 1, nchars);
+    }
+  for (i = c + 1; i <= n; i++)
+    {
+      t[i]->eventnum = *e;
+      if (i > 0)
+	t[i]->prev = p16 (t[i - 1]);
+      else
+	t[0]->prev = p16 (s[0]);
+    }
+  {
+    register int i;
+    for (i = c - 1; i >= 0; i--)
+      t[i] = p32 (t[i + 1]->prev);
+  }
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+#endif
+
+/*==================================*/
+
+void
+start_model ()
+{
+  int i;
+  /* is this the first call? */
+  if (E == NULL)
+    E = (eventptr) malloc ((unsigned) kbytes * 1024);
+  /* start off with no information */
+  numnodes = 0;
+  nfreenodes = maxnodes;
+  for (i = 0; i <= FIXEDORDER; i++)
+    {
+      s[i] = newnode (NULL);
+      if (i > 0)
+	s[i]->prev = p16 (s[i - 1]);
+    }
+}
+
+/*==================================*/
+
+void
+encodestring (str, n)
+     unsigned short *str;
+     int n;
+/* encode all n characters of the supplied string str */
+{
+  event e;
+  boolean encoded;
+  register eventptr S, T;
+  int i;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  if (E == NULL)
+    start_model ();
+  /* set the initial context node */
+  S = s[FIXEDORDER];
+
+  for (i = 0; i < n; i++)
+    {
+      /* encode one character */
+      e = *str++;
+      /* this next monstrosity is logically equivalent to 
+         encode_in_order(FIXEDORDER,e)
+         the loop has been unrolled to make it all go faster */
+      for (;;)
+	{
+	  T = encode_event_noex (&S->foll, e, &encoded);
+	  if (encoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = encode_event_noex (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = encode_event_noex (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = encode_event_noex (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    break;
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = encode_event (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = encode_event (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = encode_event (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  arithmetic_encode (e, e + 1, nchars);
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      /* phew! Update the current context to the latest node */
+      S = p32 (T->prev);
+
+      /* is there a danger of running out of space? */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+    }
+
+  /* save the current contexts for the various order submodels */
+  s[FIXEDORDER] = S;
+  for (i = FIXEDORDER; i > 0; i--)
+    s[i - 1] = p32 (s[i]->prev);
+  /* and send the terminator character */
+  encode_in_order (FIXEDORDER, EOSchar);
+  /* and check the space again */
+  if (nfreenodes <= FIXEDORDER)
+    start_model ();
+}
+
+/*==================================*/
+
+void
+decodestring (str, n)
+     unsigned short *str;
+     int *n;
+{
+  event e;
+  boolean decoded;
+  register eventptr S, T;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  /* just mimic the corresponding encoding actions */
+  if (E == NULL)
+    start_model ();
+  S = s[FIXEDORDER];
+
+  *n = 0;
+  for (;;)
+    {				/* logically equivalent to "decode_in_order(FIXEDORDER,&e)" */
+
+      for (;;)
+	{
+	  T = decode_event_noex (&S->foll, &decoded, &e);
+	  if (decoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      break;
+	    }
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = decode_event (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = decode_event (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = decode_event (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  e = arithmetic_decode_target (nchars);
+	  arithmetic_decode (e, e + 1, nchars);
+#if (FIXEDORDER>2)
+	  t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+	    t[2]->eventnum =
+#endif
+#if (FIXEDORDER>0)
+	    t[1]->eventnum =
+#endif
+	    t[0]->eventnum = e;
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      S = p32 (T->prev);
+
+      /* check the space requirements */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+      /* are we done?  or do we save the char */
+      if (e == EOSchar)
+	break;
+      *str++ = e;
+      (*n)++;
+    }
+  s[FIXEDORDER] = S;
+}
+
+
+#if 0				/* Writing my own version of this - MAHE */
+void
+encodefile ()
+/* encodes a file by breaking into a sequence of fixed length chunks */
+{
+  unsigned char *str;
+  int c, n = 0;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startoutputingbits ();
+  startencoding ();
+
+  /* to encode the file, break it into chunks and encode the
+     chunks one by one */
+  while ((c = getchar ()) != EOF)
+    {
+      str[n++] = c;
+      if (n == chunksize)
+	{
+	  encodestring (str, n);
+	  rawbytes += n;
+	  n = 0;
+	}
+    }
+  /* send final partial chunk */
+  if (n > 0)
+    {
+      encodestring (str, n);
+      rawbytes += n;
+    }
+  /* send EOF as an empty string */
+  encodestring (str, 0);
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void
+decodefile ()
+{
+  unsigned char *str;
+  int n, i;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startinputingbits ();
+  startdecoding ();
+  for (;;)
+    {
+      decodestring (str, &n);
+      if (n == 0)
+	break;
+      for (i = 0; i < n; i++)
+	putchar ((int) str[i]);
+      rawbytes += n;
+    }
+}
+
+#endif /* #if 0 */
Index: /main/tags/3.01/indexers/mg/src/images/sortmarks.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/sortmarks.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/sortmarks.c	(revision 21112)
@@ -0,0 +1,93 @@
+/**************************************************************************
+ *
+ * sortmarks.c -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "sortmarks.h"
+
+#define GAP 4
+
+
+
+static int 
+CmpOnY (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->ycen * 0 + ((marktype *) e1)->ypos) -
+    (((marktype *) e2)->ycen * 0 + ((marktype *) e2)->ypos);
+}
+
+static int 
+CmpOnX (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->xcen * 0 + ((marktype *) e1)->xpos) -
+    (((marktype *) e2)->xcen * 0 + ((marktype *) e2)->xpos);
+}
+
+
+
+
+
+
+
+
+marklistptr 
+sortmarks (marklistptr listofmarks)
+{
+  marktype *table;
+  int count = 0;
+  int i, s, e;
+
+
+  table = calloc ((unsigned int) marklist_length (listofmarks), sizeof (marklisttype));
+
+  while (listofmarks)
+    {
+      table[count] = listofmarks->data;
+      marklist_removeat (&listofmarks, 0);
+      count++;
+    }
+
+  /* sort first on the y position of their centroids */
+  qsort ((void *) table, (unsigned int) (count), sizeof (*table), CmpOnY);
+
+  for (s = 0; s < count;)
+    {
+      int c;
+
+      for (e = s + 1, c = 0; (e < count) &&
+	   ((table[e].ypos + table[e].ycen * 0) - (table[e - 1].ypos + table[e - 1].ycen * 0) <= GAP); c++, e++);
+
+      if (e < count && c >= 1)
+	qsort (&table[s], (unsigned int) (e - s), sizeof (*table), CmpOnX);
+      s = e;
+    }
+
+  for (i = 0; i < count; i++)
+    {
+      table[i].symnum = i;
+      marklist_add (&listofmarks, table[i]);
+    }
+
+  return listofmarks;
+}
Index: /main/tags/3.01/indexers/mg/src/images/sortmarks.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/sortmarks.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/sortmarks.h	(revision 21112)
@@ -0,0 +1,30 @@
+/**************************************************************************
+ *
+ * sortmarks.h -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _SortMarks_h_
+#define _SortMarks_h_
+
+marklistptr sortmarks (marklistptr listofmarks);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/images/utils.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/utils.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/utils.c	(revision 21112)
@@ -0,0 +1,270 @@
+/**************************************************************************
+ *
+ * utils.c -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "utils.h"
+
+
+int V = 0;
+
+
+int 
+isEOF (FILE * fp)
+{
+  int ch;
+
+  ch = getc (fp);
+  ungetc (ch, fp);
+  if (ch == EOF)
+    return 1;
+  return 0;
+}
+
+
+int 
+getmagicno_short (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 8) | ch2);
+}
+
+int 
+getmagicno_short_pop (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    return 0;
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    return 0;
+
+  return ((ch1 << 8) | ch2);  
+}
+
+int 
+getmagicno_byte (FILE * fp)
+{
+  int ch1;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ungetc (ch1, fp);
+  return (ch1);
+}
+
+
+int 
+getmagicno_long (FILE * fp)
+{
+  int ch1, ch2, ch3, ch4;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ch3 = getc (fp);
+  if (ch3 == EOF)
+    {
+      ungetc (ch3, fp);
+      return 0;
+    }
+  ch4 = getc (fp);
+  if (ch4 == EOF)
+    {
+      ungetc (ch4, fp);
+      return 0;
+    }
+  ungetc (ch4, fp);
+  ungetc (ch3, fp);
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 24) | (ch2 << 16) | (ch3 << 8) | (ch4));
+}
+
+
+
+
+void 
+error_msg (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+  exit (1);
+}
+
+void 
+warn (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+}
+
+
+
+void 
+readline (char str[], FILE * fp)
+{
+  int i = 0, ch;
+
+  while (((ch = fgetc (fp)) != '\n') && (!feof (fp)))
+    str[i++] = ch;
+  str[i] = '\0';
+}
+
+
+
+
+
+
+
+int 
+getint (FILE * fp)
+{
+  register char ch;
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+/* get_header_int i.e. get an integer from a PNM header ; basically  
+   the same as
+   above except for comment checking ; above kept because of lower  
+   overhead */
+
+unsigned int 
+gethint (FILE * fp)
+{
+  register char ch, prev = '\0';
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if ((ch == '#') && (prev == '\n'))
+	while (((ch = getc (fp)) != '\n') && !(feof (fp)));
+      prev = ch;
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+
+
+
+void 
+magic_write (FILE * fp, u_long magic_num)
+{
+  HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fwrite (&magic_num, sizeof (magic_num), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't write magic number.", "");
+}
+
+
+void 
+magic_check (FILE * fp, u_long magic_num)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1 || NTOHUL(magic) != magic_num)  /* [RPAP - Jan 97: Endian Ordering] */
+    error_msg ("magic num", "Incorrect magic number.", "");
+}
+
+
+u_long 
+magic_read (FILE * fp)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't read magic number.", "");
+  return NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+isinteger (char s[])
+{
+  int i = 0;
+
+  for (i = 0; s[i] != '\0'; i++)
+    if (!isdigit (s[i]))
+      return 0;
+  return 1;
+}
Index: /main/tags/3.01/indexers/mg/src/images/utils.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/images/utils.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/images/utils.h	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * utils.h -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __UTILS_H
+#define __UTILS_H
+
+#include "sysfuncs.h"
+
+extern int V;
+
+#define DEBUG
+
+#define min(a,b) ((a)<(b) ? (a) : (b))
+#define max(a,b) ((a)>(b) ? (a) : (b))
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+                                            ((c)<<8) + (d)))
+ 
+/* --- MG magic file codes [TS:Aug/95] --- */
+#define MAGIC_FELICS            GEN_MAGIC('M','G','F','X')
+#define MAGIC_BILEVEL           GEN_MAGIC('M','G','B','I')
+#define MAGIC_TIC               GEN_MAGIC('M','G','T','C')
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+/* --------------------------------------- */
+
+#define MAGIC_P1      0x5031	/* P1   - pbm */
+#define MAGIC_P2      0x5032	/* P2   - pgm */
+#define MAGIC_P3      0x5033	/* P3   - ppm */
+#define MAGIC_P4      0x5034	/* P4   - rawbits pbm */
+#define MAGIC_P5      0x5035	/* P5   - rawbits pgm */
+#define MAGIC_P6      0x5036	/* P6   - rawbits ppm */
+
+int isEOF (FILE * fp);
+int getmagicno_byte (FILE * fp);
+int getmagicno_short (FILE * fp);
+int getmagicno_short_pop (FILE * fp);
+int getmagicno_long (FILE * fp);
+
+
+
+void error_msg (char *prog, char *message, char *extra);
+void warn (char *prog, char *message, char *extra);
+
+void readline (char str[], FILE * fp);
+
+void magic_write (FILE * fp, u_long magic_num);
+void magic_check (FILE * fp, u_long magic_num);
+u_long magic_read (FILE * fp);
+/* kerry's code */
+
+int getint (FILE * fp);
+unsigned int gethint (FILE * fp);
+int isinteger (char s[]);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/scripts/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/Makefile.in	(revision 21112)
@@ -0,0 +1,152 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/scripts
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+GSDLOS = @gsdlos@
+ 
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+SHFILES       =	mgbuild.sh mgbuild_4.sh mg_get.sh xmg.sh  mg_get_merge.sh mgmerge.sh
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SHFILES) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgbuild mg_get xmg mg_get_merge mgmerge mgbuild_4
+
+MAN           = mgbuild.1 mgbuild_4.1 mg_get.1 xmg.1 mg_get_merge.1 mgmerge.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+mgbuild : mgbuild.sh
+	@rm -f $@
+	cat mgbuild.sh >$@
+	chmod +x $@
+
+mgbuild_4 : mgbuild_4.sh
+	@rm -f $@
+	cat mgbuild_4.sh >$@
+	chmod +x $@
+
+mg_get : mg_get.sh
+	@rm -f $@
+	cat mg_get.sh >$@
+	chmod +x $@
+
+mgmerge : mgmerge.sh
+	@rm -f $@
+	cat mgmerge.sh >$@
+	chmod +x $@
+
+mg_get_merge : mg_get_merge.sh
+	@rm -f $@
+	cat mg_get_merge.sh >$@
+	chmod +x $@
+
+xmg : xmg.sh
+	@rm -f $@
+	cat xmg.sh >$@
+	chmod +x $@
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+mostlyclean:
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+# screws up distclean (jrm21) 
+#Makefile: Makefile.in ../../../../config.status
+#	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/indexers/mg/src/scripts/mg_get.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mg_get.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mg_get.1	(revision 21112)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /main/tags/3.01/indexers/mg/src/scripts/mg_get.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mg_get.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mg_get.sh	(revision 21112)
@@ -0,0 +1,447 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell
+# Changed to allow it to be driven from a file ~/.mg_getrc
+# by Bruce McKenzie, Oct 1994
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mg_get.sh	1.11 25 Mar 1994
+#
+###########################################################################
+#
+# "mg_get" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+# The file ~/.mg_get contains lines of the form:
+#     name	TYPE	files and directories
+# where name is the name supplied to mg_get, TYPE is one of
+# PARA, MAIL, DIR, DIR2, BIB or TXTIMG and says how mg_get should process the
+# named files and directories
+#
+###########################################################################
+
+if ($?MG_GETRC) then
+  set mg_getrc = $MG_GETRC
+else
+  set mg_getrc = ~/.mg_getrc
+endif
+
+# Added by [TS:May/95]
+# if we don't have an mg_getrc file in existence then create a default one
+if !(-e $mg_getrc) then 
+  echo "Can not find a .mg_getrc, so creating a default one: $mg_getrc"
+
+  if ($?MGSAMPLE) then
+    set SampleData = $MGSAMPLE
+  else
+    set SampleData = ./SampleData
+  endif
+
+  cat > $mg_getrc << END
+alice	PARA	$SampleData/alice13a.txt.Z
+davinci	TXTIMG	$SampleData/davinci
+mailfiles	MAIL	~/mbox ~/.sentmail
+allfiles	DIR	~/Mail
+mods	LINESEP	$SampleData/MODIFICATIONS
+bible	PLAIN	$SampleData/bible/genesis.txt.Z $SampleData/bible/revelation.txt.Z
+END
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: mg_get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set COLLECTION = $1
+set TYPE  = `grep \^$1 $mg_getrc | cut -f2`
+set FILES = `grep \^$1 $mg_getrc | cut -f3`
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($TYPE)
+  case PLAIN:
+# Just outputs the given text, assumes ^Bs already inserted 
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case LINESEP:
+# Assumes that the documents are separated by a line of =====
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f |  sed -e '/\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/s/.*//' < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case PARA:
+# Splits the file at blank lines, which corresponds to paragraphs.
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '   /^ *$/ {if (b!=1) printf "";b=1} \
+		\!/^ *$/ {print;b=0} \
+		 END {printf ""}'
+       end #foreach
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case BIB:
+# Takes a list of files that contain bibliographies, and splits them up 
+#   by putting ^B after each reference. Assumes that each reference
+#   begins with a line '^@'.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '/^@/&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #BIB
+
+  case MAIL:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f| awk '/xbtoa Begin/,/xbtoa End/ {next} /^From /&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #MAIL
+
+  case DIR:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz' -exec echo '<' {} '>' \;		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' -exec echo '<' {} '>' \;		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)' -exec echo '<' {} \
+#			'>' \; -exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec echo '<' {} '>' \; \
+			-exec cat {} \; -exec echo -n '' \;
+
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR
+
+
+
+  case DIR2:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory. Does not include filename
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz'  		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' 	 		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)'	\
+#			-exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR2
+
+
+
+
+  case TXTIMG:
+# compress and index the collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# Files that are related have the same prefix. For example,
+#  monaLisa.pgm might be a gray-level image, and
+#  monaLisa.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt	ascii text
+#    .ptm	scanned text stored as a bilevel image
+#    .pbm	a black and white image (typically a line drawing)
+#    .pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $FILES
+  set targetDir = $MGDATA/$COLLECTION
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   # create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null     # in case it already existed
+    
+      # take care if no match to foreach statements
+      set nonomatch
+
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+        if ($f == "$sourceDir/*.ptm") then
+                break
+        endif
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+        if ($f == "$sourceDir/*.pgm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+        if ($f == "$sourceDir/*.pbm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+      unset nonomatch
+    breaksw #-init
+
+    case '-text'
+      # take care if no match to foreach statements
+      set nonomatch
+
+      #output each text file and tmp.txt, with names of other associated files
+
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+        if ($f == "$sourceDir/*.txt") then
+           continue
+        endif
+        if ($f == "$sourceDir/*.tmp") then
+           continue
+        endif
+	set r = $f:r	#root name of file
+	set base = $r:t
+        set first_image = 1 # set so that I print out image header once
+                            # also signifies if header printed out at all
+
+	foreach d ($targetDir/$base.*)
+          if ($d == "$targetDir/$base.*") then
+                break
+          endif
+
+          if ($first_image) then
+            set first_image = 0
+            echo '::::::::::'  #separate the image information from the text
+            echo 'Image(s) available:'
+          endif
+
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+
+	end #foreach d (associated file)
+
+        if (! $first_image) then
+	  echo '::::::::::'  #separate the image information from the text
+        endif
+
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+      unset nonomatch
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #IMGTXT
+
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mg_get_merge.sh	(revision 21112)
@@ -0,0 +1,313 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get_merge.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# "mg_get_merge" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+###########################################################################
+
+
+
+# The directory containing the data files is stored in the environment
+#   variable MGSAMPLE
+
+if ($?MGSAMPLE) then
+  set SampleData = $MGSAMPLE
+else
+  set SampleData = ./SampleData
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($1)
+  case alice:
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+	echo "This is a test document 1 2 3 4 5 6 7 8 9 "
+	echo ""
+	echo "A B C"
+     breaksw
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case bible:
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+       # use uncompressed file if it exists
+       # revelation.txt already contains '^B' characters between verses
+       if (-e $SampleData/bible/revelation.txt) then
+          cat $SampleData/bible/revelation.txt
+       else
+          uncompress <$SampleData/bible/revelation.txt.Z
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bible
+
+
+
+  case mailfiles:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+# If messages are already in individual files, use "get allfiles"
+#   rather than this one.
+# The EXTRA documents added are in files in the directory mailextra
+#### The variable 'mailfiles' contains a list of files to be split up
+  set mailfiles = '~/mbox ~/.sentmail'
+  set mailextra = '~/extra/*'
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      foreach f ($mailextra)
+	awk '/^From /&&NR!=1{print ""} {print $0} END{print ""}' $f
+      end # foreach
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #mailfiles
+
+
+
+  case allfiles:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+# The variable 'directory' on the next line contains the directory to be used
+  set directory = ~/Mail
+
+# The date of the file ".mgbuildtime" in 'directory' is the last
+# build or merge. At the start of the merge ("-init" argument) the
+# file ".mgmergetime" is touched. Only files newer than ".mgbuildtime"
+# but older than ".mgmergetime" are output by mg_get_merge
+
+  switch ($flag)
+    case '-init':
+        touch $directory/.mgmergetime
+    breaksw
+
+    case '-text'
+      find $directory -type f -newer $directory/.mgbuildtime \! -newer $directory/.mgmergetime \! -name '.*' -exec echo '<' {} '>' \; -exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+        touch $directory/.mgmergetime
+        touch $directory/.mgbuildtime
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #allfiles
+
+
+
+
+  case davinci:
+# compress and index the DaVinci collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# It assumes that all files are in one directory ($sourceDir).
+# Files that are related have the same prefix. For example,
+#  monaLisa.img.pgm might be a gray-level image, and
+#  monaLisa.txt.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt.txt	ascii text
+#    .txt.pbm	scanned text stored as a bilevel image
+#    .img.pbm	a black and white image (typically a line drawing)
+#    .img.pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $SampleData/davinci
+  set targetDir = $MGDATA/davinci
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   #create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null #in case it already existed
+    
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+    breaksw #-init
+
+    case '-text'
+      #output each text file and tmp.txt, with names of other associated files
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+	set r = $f:r	#root name of file
+	set base = $r:t
+	set n = `/bin/ls $sourceDir/$base.* | wc -l`
+	if ($n != 1) then # more files than just the text one
+	 echo '::::::::::'  #separate the image information from the text
+	 echo 'Image(s) available:'
+	 foreach d ($targetDir/$base.*)
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+	 end #foreach d (associated file)
+	echo '::::::::::'  #separate the image information from the text
+	endif # if more than one file
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #davinci
+
+
+#---------------------------
+  case mods:
+# Splits the modifications file at the mod separator "--------"
+  switch ($flag)
+     case '-init':
+     breaksw
+ 
+     case '-text':
+       sed -e 's/^---.*//g' < ${SampleData}/MODIFICATIONS
+     breaksw
+ 
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw #mods
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /main/tags/3.01/indexers/mg/src/scripts/mgbuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgbuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgbuild.1	(revision 21112)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/scripts/mgbuild.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgbuild.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgbuild.sh	(revision 21112)
@@ -0,0 +1,339 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+###########################################################################
+#  mg.build
+# Creates a Full Text Retrieval database
+#
+#  @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+
+set complex = ""
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-d") then
+    shift
+    if ($#argv >= 1) then
+      set mgdata = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-d mgdata dir] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get
+endif
+
+if ($?mgdata) then
+  setenv MGDATA $mgdata
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# If do_text == 1 then do the text component of building a mg database. 
+# Otherwise don't do the text component.
+set do_text = 1
+
+
+# If do_invf == 1 then do the inversion component of building a mg database. 
+# Otherwise don't do the inversion component.
+set do_invf = 1
+
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+set do_pass2 = 1
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+#### Note: Set -S so novel words can be encoded, as this will happen if
+#### mgmerge is used.
+set mcd = -S
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+
+#build up the command lines for pass 1 and 2
+set pass1 = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${bname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+if ($do_text) then
+  set pass1 = (${pass1} -T1)
+  set pass2 = (${pass2} -T2)
+endif
+
+if ($do_invf) then
+  set pass1 = (${pass1} -I1)
+  set pass2 = (${pass2} -I2)
+endif
+  
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1}"
+      $get $text $complex| $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1} ${input_files}
+    $bindir/mg_passes ${pass1} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2}"
+      $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2}"
+      $get $text $complex | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2} ${input_files}
+    $bindir/mg_passes ${pass2} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+echo "mg_weights_build -f ${bname} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+echo "mg_invf_dict -f ${bname} -b 4096"
+$bindir/mg_invf_dict -f ${bname} -b 4096
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+# [RPAP - Jan 97: Stem Index Change]
+if ("$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+echo "-- The fast-loading compression dictionary has not been built."
+echo "-- If you wish to build it, execute the following command:"
+echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+echo "-----------------------------------"
+
+echo ""
+
Index: /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.1	(revision 21112)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgbuild_4.sh	(revision 21112)
@@ -0,0 +1,435 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild_4.sh -- Script used to build mg text collection in 4 passes.
+# Copyright (C) 1994  Neil Sharman; Modified by tes@kbs 1995
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# Does an mgbuild in 4 passes instead of 2 passes.
+# This is useful if one does not have as much RAM and a large collection.
+#
+
+set complex = ""
+
+# ************ parse the command line arguments ************
+
+while ($#argv >= 1)
+  if ("$1" == "-s" || "$1" == "--source") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g" || "$1" == "--get") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c" || "$1" == "--complex") then
+    set complex = "-text"
+    shift
+  else
+    if (! $?text ) then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if (! $?text ) then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = `which $0` # find which one we are executing
+set bindir = $bindir:h # use it as the bindir path
+
+# ************ set up shell variables ************
+# Override these by providing a script to source using the -s option
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+set get_options = ""
+
+
+if (! $?get ) then
+  set complex = "-text"
+  set get = $bindir/mg_get $get_options
+endif
+
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# whether to build weights or not
+set do_weights = 1
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+# Text 1st pass
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+# Text 2nd pass
+set do_pass2 = 1
+
+# If do_pass3 == 1 then do pass3 of building a mg database. 
+# Otherwise don't do pass3.
+# Invf 1st pass
+set do_pass3 = 1
+
+# If do_pass4 == 1 then do pass4 of building a mg database. 
+# Otherwise don't do pass4.
+# Invf 2nd pass
+set do_pass4 = 1
+
+# Buffer size used by mg_passes
+# Should be big enough to hold largest document
+# In Kilobytes
+set buf_size = 3072
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 5
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -C
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+# ************ set up pass parameters ************
+
+if ($do_pass1) then
+  set pass1_text = (-f ${bname} -T1)
+endif
+
+if ($do_pass2) then
+  set pass2_text = (-f ${bname} -T2)
+endif
+
+if ($do_pass3) then
+  set pass3_invf = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+  set pass3_invf = (${pass3_invf} -N1)
+endif
+
+if ($do_pass4) then
+  set pass4_invf = (-f ${bname} -${invf_level} -N2)
+endif
+
+if ($strip_sgml) then
+  set pass1_text = (${pass1_text} -G)
+  set pass2_text = (${pass2_text} -G)
+  set pass3_invf = (${pass3_invf} -G)
+  set pass4_invf = (${pass4_invf} -G)
+endif
+
+if ($?trace) then
+  set pass1_text = (${pass1_text} -t ${trace})
+  set pass2_text = (${pass2_text} -t ${trace})
+  set pass3_invf = (${pass3_invf} -t ${trace})
+  set pass4_invf = (${pass4_invf} -t ${trace})
+endif
+
+if ($?trace_name) then
+  set pass1_text = (${pass1_text} -n ${trace_name})
+  set pass2_text = (${pass2_text} -n ${trace_name})
+  set pass3_invf = (${pass3_invf} -n ${trace_name})
+  set pass4_invf = (${pass4_invf} -n ${trace_name})
+endif
+
+if ($buf_size) then
+  set pass1_text = (${pass1_text} -b ${buf_size})
+  set pass2_text = (${pass2_text} -b ${buf_size})
+  set pass3_invf = (${pass3_invf} -b ${buf_size})
+  set pass4_invf = (${pass4_invf} -b ${buf_size})
+endif
+  
+if ($?comp_stats) then
+  set pass2_text = (${pass2_text} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+
+# ************ init get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+# ************************************* T E X T **********************************
+
+# ************ pass 1_text ************
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1_text}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1_text}"
+      $get $text $complex| $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1_text} ${input_files}
+    $bindir/mg_passes ${pass1_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ compression dict  ************
+if ($do_pass2) then 
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ************ pass 2_text ************
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2_text}"
+      $get $text $complex| $pass2filter | $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2_text}"
+      $get $text $complex| $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2_text} ${input_files}
+    $bindir/mg_passes ${pass2_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************************************* I N V F **********************************
+
+# ************ pass 3_invf ************
+if (${do_pass3}) then
+  if ($pipe) then
+    if ($?pass3filter) then
+      echo "$get $text $complex | $pass3filter | mg_passes ${pass3_invf}"
+      $get $text $complex| $pass3filter | $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass3_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass3_invf} ${input_files}
+    $bindir/mg_passes ${pass3_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ perfect hash  ************
+if ($do_pass4) then
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ pass 4_invf ************
+if (${do_pass4}) then
+  if ($pipe) then
+    if ($?pass4filter) then
+      echo "$get $text $complex | $pass4filter | mg_passes ${pass4_invf}"
+      $get $text $complex| $pass4filter | $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass4_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass4_invf} ${input_files}
+    $bindir/mg_passes ${pass4_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+
+# ************ build invf dictionary ************
+
+if ($do_pass4) then
+  echo "mg_invf_dict -f ${bname} -b 4096"
+  $bindir/mg_invf_dict -f ${bname} -b 4096
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ******** build indexes to stem dict *********
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_pass4 && "$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ build weights ************
+
+if ($do_pass4 && $do_weights) then
+  echo "mg_weights_build -f ${bname} -b ${weight_bits} "
+  $bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# *******************************************************************************
+
+
+# ************ print out statistics ************
+if ($do_pass1 && $do_pass2 && $do_pass3 && $do_pass4) then
+  echo "mgstat -f ${bname} -E"
+  $bindir/mgstat -f ${bname} -E
+  if ("$status" != "0") exit 1 
+endif
+
+
+# ************ cleanup get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+if ($do_pass2) then
+  echo "-- The fast-loading compression dictionary has not been built."
+  echo "-- If you wish to build it, execute the following command:"
+  echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+  echo "-----------------------------------"
+endif
+
+echo ""
+
Index: /main/tags/3.01/indexers/mg/src/scripts/mgmerge.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgmerge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgmerge.1	(revision 21112)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /main/tags/3.01/indexers/mg/src/scripts/mgmerge.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/mgmerge.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/mgmerge.sh	(revision 21112)
@@ -0,0 +1,417 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgmerge.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+
+set complex = ""
+set slow_merge = ""
+set guess_weights = "-w "
+# [RPAP - Feb 97: Level 3 Merge]
+set text_pass = 1
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else if ("$1" == "-S") then
+# set slow merge option in mg_invf_merge
+    set slow_merge = "-s "
+    shift
+  else if ("$1" == "-w") then
+# turn weights guessing off (rebuild weights file from scratch)
+    set guess_weights = ""
+    shift
+
+# [RPAP - Feb 97: Level 3 Merge]
+  else if ("$1" == "-T") then
+# turn text pass off
+    set text_pass = 0
+    shift
+
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get_merge
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the inversion method this may be either "I" or "N"
+# N is memory-efficient, I is faster for small collections
+set invf_method = N
+
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+# This only has an effect if $invf_method is "N".
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+# This only has an effect if $invf_method is "N".
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+# Note: The value *MUST* be the same as was used to build the collection
+# that is being merged.
+# In the current mg_invf_merge implementation the maximum level is 2. 
+# Paragraph-level inversion (invf_level = 3) is NOT supported.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -S
+
+# $merge_name is the name of the subdirectory under ${MGDATA} where
+# the mergeing steps will be performed. There should NOT be an mg
+# collection with this name! The default is "MERGE"
+set merge = "MERGE"
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ($do_indexes) then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+
+# Generate the directory where merging will take place
+if (-e $MGDATA/${merge}) then
+else
+  mkdir $MGDATA/${merge}
+endif
+
+# Generate the base name for the collection.
+# [RPAP - Feb 97: Level 3 Merge]
+set bname = ${coll_name}
+
+# generate the old, new and merge names in the $merge directory
+set oldname = ${merge}/${coll_name}.old
+set newname = ${merge}/${coll_name}.new
+set mergename = ${merge}/${coll_name}
+
+# build up the command lines for pass 1 and 2 to build a
+# dummy collection for the new documents
+set pass1 = (-f ${newname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${newname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+# Note that a -T1 pass isnt done since the old .text.stats file is used
+# as the compression model for compressing the new documents.
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Do not perform -T2 pass if text_pass == 0
+if ($text_pass) then
+  set pass2 = (${pass2} -T2)
+endif
+
+set pass1 = (${pass1} -${invf_method}1)
+set pass2 = (${pass2} -${invf_method}2)
+
+
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+########################################################################
+# Here is where mgmerge goes to work.
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "MGMERGE collection: ${bname}"
+echo "--------------------------------------------------------------"
+echo "FIRST PHASE: build new collection from new documents"
+echo "--------------------------------------------------------------"
+
+# move and rename the text dictionary file
+# [RPAP - Feb 97: Level 3 Merge] - only if performing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text.dict $MGDATA/${newname}.text.dict
+endif
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo ">> $get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if ($pipe) then
+  if ($?pass1filter) then
+    echo ">> $get $text $complex | $pass1filter | mg_passes ${pass1}"
+    $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass1}"
+    $get $text $complex| $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass1} ${input_files}"
+  $bindir/mg_passes ${pass1} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+echo "mg_perf_hash_build -f ${newname}"
+$bindir/mg_perf_hash_build -f ${newname}
+if ("$status" != "0") exit 1 
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ($?pass2filter) then
+    echo ">> $get $text $complex | $pass2filter | mg_passes ${pass2}"
+    $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass2}"
+    $get $text $complex | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass2} ${input_files}"
+  $bindir/mg_passes ${pass2} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo ">> $get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "---------------------------------------"
+date
+echo "--------------------------------------------------------------"
+echo "SECOND PHASE: merge the two collections"
+echo "--------------------------------------------------------------"
+
+# move the appropriate files from the base directory to the merge dir
+echo "(moving files....)"
+# [RPAP - Feb 97: Level 3 Merge]
+# Move files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text       $MGDATA/${oldname}.text
+  mv $MGDATA/${bname}.text.idx   $MGDATA/${oldname}.text.idx
+else
+# Need this file if building weights
+  cp -d $MGDATA/${bname}.text.idx   $MGDATA/${mergename}.text.idx
+endif
+mv $MGDATA/${bname}.invf       $MGDATA/${oldname}.invf
+mv $MGDATA/${bname}.invf.idx   $MGDATA/${oldname}.invf.idx
+mv $MGDATA/${bname}.invf.dict  $MGDATA/${oldname}.invf.dict
+# the old weight file may be used for the new file
+mv $MGDATA/${bname}.weight     $MGDATA/${mergename}.weight
+# EOC: the old invf.paragraph file may be used for the new file for Level 3 Invf
+if (-e $MGDATA/${bname}.invf.paragraph) then
+  mv $MGDATA/${bname}.invf.paragraph  $MGDATA/${mergename}.invf.paragraph
+endif
+echo "---------------------------------------"
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Only merge text if doing text pass
+if ($text_pass) then
+  echo ">> mg_text_merge -f ${mergename}"
+  $bindir/mg_text_merge -f $mergename
+  echo "---------------------------------------"
+endif
+
+echo ">> mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}"
+$bindir/mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}
+echo "---------------------------------------"
+
+#remove .weight file, so a new one will be created by mg_weights_build
+if ("$guess_weights" == "") then
+  echo "rm ${mergename}.weight"
+  rm $MGDATA/${mergename}.weight
+endif
+
+# mv $MGDATA/${mergename}.weight $MGDATA/${mergename}.w2
+
+echo ">> mg_weights_build -f ${mergename} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${mergename} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "---------------------------------------"
+
+echo ">> mg_invf_dict -f ${mergename} -b 4096"
+$bindir/mg_invf_dict -f ${mergename} -b 4096
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  echo "-----------------------------------"
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+endif
+
+echo "--------------------------------------------------------------"
+
+# move files back to base dir
+# [RPAP - Feb 97: Level 3 Merge]
+# mv the text files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${oldname}.text         $MGDATA/${bname}.text
+  mv $MGDATA/${mergename}.text.idx   $MGDATA/${bname}.text.idx
+  mv $MGDATA/${newname}.text.dict    $MGDATA/${bname}.text.dict
+endif
+mv $MGDATA/${mergename}.invf       $MGDATA/${bname}.invf
+mv $MGDATA/${mergename}.invf.idx   $MGDATA/${bname}.invf.idx
+mv $MGDATA/${mergename}.invf.dict  $MGDATA/${bname}.invf.dict
+mv $MGDATA/${mergename}.invf.dict.blocked  $MGDATA/${bname}.invf.dict.blocked
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  mv $MGDATA/${mergename}.invf.dict.blocked.1 $MGDATA/${bname}.invf.dict.blocked.1
+  mv $MGDATA/${mergename}.invf.dict.blocked.2 $MGDATA/${bname}.invf.dict.blocked.2
+  mv $MGDATA/${mergename}.invf.dict.blocked.3 $MGDATA/${bname}.invf.dict.blocked.3
+endif
+
+mv $MGDATA/${mergename}.weight     $MGDATA/${bname}.weight
+mv $MGDATA/${mergename}.weight.approx $MGDATA/${bname}.weight.approx
+mv $MGDATA/${mergename}.text.idx.wgt $MGDATA/${bname}.text.idx.wgt
+
+# EOC: mv .invf.paragraph file if exists
+if (-e $MGDATA/${mergename}.invf.paragraph) then
+  mv $MGDATA/${mergename}.invf.paragraph   $MGDATA/${bname}.invf.paragraph
+endif
+
+# remove old files in the MERGE directory
+rm -f $MGDATA/${merge}/*
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "--------------------------------------------------------------"
+
+echo ""
Index: /main/tags/3.01/indexers/mg/src/scripts/xmg.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/xmg.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/xmg.1	(revision 21112)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/scripts/xmg.sh
===================================================================
--- /main/tags/3.01/indexers/mg/src/scripts/xmg.sh	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/scripts/xmg.sh	(revision 21112)
@@ -0,0 +1,1376 @@
+#!/usr/local/bin/wish -f
+# Edit the line above to point to your version of wish
+#
+###############################################################################
+#
+# xmg -- X interface to the mg information retrieval system
+# Copyright (C) 1994  Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)xmg.sh	1.4 25 Mar 1994
+#
+###############################################################################
+#
+# "xmg" is an X interface to the mg information retrieval system.
+#
+# Type a query in the "Query" box.
+# The first line or so of each matching document is listed.
+# Click on any line to show the complete document.
+# Shift-click to show the document in a new window.
+# Press Next and Previous to cycle through the list of documents (Shift works
+#   here too). 
+# If you change the query, existing windows continue to refer to the original
+#   query. 
+# 
+# You can select either Boolean or Ranked queries.
+# Boolean queries: use "&" for AND, "|" for OR, "!" for NOT, and parentheses
+# for grouping 
+# Ranked queries:  - documents are listed in order of relevance; a weight is
+# 		     shown alongside each 
+#                  - weights are normalized to 100 for the most relevant
+#		     document 
+#                  - choose Approximate Ranking for greater retrieval speed. 
+# 
+# In document windows (other than the main window): 
+#		Typing 'n' = next document
+#		Typing 'N' = next document in new window
+#		Typing 'p' = previous document
+#		Typing 'P' = previous document in new window
+#		Typing 'q' = close window
+#
+###############################################################################
+
+# Date: 13 March 1994
+# Authors: Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+
+###############################################################################
+# Customizing xmg
+###############################################################################
+
+# To override the default font options, use RESOURCE_MANAGER properties like
+# the following in your .Xdefaults file
+#
+#	xmg.textfont: fixed
+#	xmg.buttonfont: fixed
+#
+# To override the default query type of ranked, use
+#
+#	xmg.querytype: boolean
+#
+# To override the default collection name (the first one listed by /bin/ls from
+# your MGDATA environment variable), use
+#
+#	xmg.collection: mail93
+
+###############################################################################
+# Global variables
+###############################################################################
+
+#   genuine constants:
+#
+#	env		the environment variables
+#	Terminator	terminator for mgquery
+#	MaxWindowHeight	the maximum height permitted for a document window
+#	ListBoxLength	the number of items in the document list
+#	HeadLength	the length document head displayed in the list box
+#	ImageString	the string at the start of a document associated
+#			with an image
+#
+#   things that rarely get changed:
+#
+#	querytype	the current query type (boolean, ranked, approx-ranked)
+#	booleanquery	says whether the query is boolean or ranked
+#	approxquery	says whether ranking is to be approximate or not
+#	collection	the name of the document collection being used
+#	fileId		file identifer for a pipe to the mgquery process
+#       errorOccurred	flag to say that there has been an I/O error on the
+#			current collection
+#
+#   things that reflect items in the main panel
+#
+#	query		the current query
+#	hits		number of hits for the current query
+#	display		shell command to display the image associated with
+#			the last document retrieved
+#	scrolledto	document prefixes have been obtained for all documents
+#			up to this position in the document list
+#
+#   static variable needed in just one procedure
+#
+#	top_rank	weight associated with the first document retrieved
+#	windowname	unique id for each document window
+
+###############################################################################
+# High-level procedures
+###############################################################################
+
+proc SaveDocument {w} {
+  set file [FSBox "Save document to file:"]
+
+  if {$file != ""} {
+    if [file exists $file] {
+      toplevel .r
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text "The file already exists"
+      button .r.replace -text "Replace" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"w\" $w" 
+      button .r.append -text "Append" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"a\" $w"
+      button .r.cancel -text "Cancel" -command {destroy .r} -width 10 -height 2
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.replace .r.append .r.cancel -side left -pady 10 -padx 20
+
+    } else {
+       WriteDocument $file "w" $w
+    }
+  }
+}
+
+proc Dialog {text command} {
+      if {![winfo exists .r] } { toplevel .r }
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text $text
+      button .r.ok -text "OK" -width 10 -height 2 \
+          -command "destroy .r $command"
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.ok -side bottom -pady 10 -padx 20
+}
+
+proc WriteDocument {file mode w} {
+    if {[catch {set f [open $file $mode]}] == 0} {
+        if [winfo exists $w] {
+            puts $f [$w get 0.0 end]
+        } else {
+            Dialog "Unable to save text" ""
+        }
+        flush $f
+        close $f
+    } else {
+        Dialog "Unable to open file for writing" "; SaveDocument $w"
+    }
+}
+
+# Send a query to mg and enter the response in the document list
+#
+proc sendQuery {window} {
+    global query hits querytype errorOccurred scrolledto
+
+    .m.t.q.retrieved.label configure -foreground Gray
+    .m.t.q.retrieved.hits configure -foreground Gray
+
+    sendto ".set query \"$querytype\""
+    sendto ".set mode \"docnums\""
+    foreach f "sdocnums sdocs srank" { $window.$f delete 0 end }
+    disableButton .m.d.buttons.previous
+    disableButton .m.d.buttons.next
+    if {[string index $query 0] == "." || [string length $query] > 255} {
+        set doclist ""
+    } else { set doclist [sendandget $query] }
+
+    if {$errorOccurred} {
+        enterText .m.d.text [mgqueryFailure]
+        .m.t.q.retrieved.label configure -foreground Black
+        .m.t.q.retrieved.hits configure -foreground Black 
+        update
+        return
+    }
+    sendto ".set query \"DocNums\""
+    sendto ".set mode  \"text\""
+    .m.t.q.retrieved.label configure -foreground Black
+    .m.t.q.retrieved.hits configure -foreground Black 
+    set hits [llength $doclist]
+    set i 0
+    global top_rank
+
+    scan [lindex $doclist 0] "%d %f %d" d rank size
+    set top_rank $rank
+    if {$top_rank == 0} {set top_rank 1}
+
+    while {$i < $hits} {
+	scan [lindex $doclist $i] "%d %f %d" d rank size
+        if {$rank == "Inf"} {set rank 1}
+	if {$querytype == "boolean"} {
+	    $window.srank insert end ""
+	} else {
+	    $window.srank insert end \
+		[format "%3.0f" [expr $rank / $top_rank * 99 + 1]]
+	}
+	$window.sdocnums insert end [format " %d" $d]
+	incr i
+    }
+
+    set scrolledto 0
+    scroll $window 0
+    bind $window.sdocs <ButtonRelease-1> \
+	"showDocument $window.sdocs %y .m.d \"$doclist\" \"$query\""
+    bind $window.sdocs <Shift-ButtonRelease-1> \
+	"showDocument $window.sdocs %y New \"$doclist\" \"$query\""
+    enterText .m.d.text ""
+}
+
+# Scrolling routine for the document list
+#
+proc scroll {window n} {
+    global hits scrolledto ListBoxLength Terminator fileId ImageString
+
+    $window.sdocnums yview $n
+    $window.sdocs yview $n
+    $window.srank yview $n
+    set last $n
+    if {$last <= 0} {set last 0}
+    incr last $ListBoxLength
+    if {$last > $hits} {set last $hits}
+
+#    .m config -cursor {watch}
+#    update
+    sendto ".set mode \"heads\""
+    sendto ".set terminator \"\""
+    while {$scrolledto < $last} {
+	if {[scan [$window.sdocnums get $scrolledto] "%d" d] == 1} {
+            sendto $d
+            gets $fileId line
+            set i [string first " " $line]
+            if {$i != -1} {set line [string range $line [expr $i + 1] end]}
+            if {[string compare $ImageString [string range $line 0 9]] == 0} {
+                set line [getImageDocumentHead $d]
+            }
+	    $window.sdocs insert $scrolledto $line
+	}
+	incr scrolledto
+    }
+    sendto ".set terminator \"$Terminator\\n\""
+    sendto ".set mode text"
+#   .m config -cursor {}
+}
+
+proc getImageDocumentHead {d} {
+    global HeadLength ImageString fileId
+
+    sendto ".set heads_length 1000"
+    sendto $d
+    gets $fileId line
+
+    set line [string range $line 11 end]
+
+    set s [string first $ImageString $line]
+    if {$s != -1} {
+        incr s 11
+        set line [string range $line $s [expr $s + $HeadLength]]
+    }
+  
+    sendto ".set heads_length $HeadLength"
+
+    return "* $line"
+}
+
+# Show document from the document list
+#
+proc showDocument {window y newWindow doclist query} {
+    showThisDocument [$window nearest $y] $doclist $newWindow $query
+}
+
+proc adjacentDocument {index doclist window q button} {
+    global ListBoxLength
+
+    showThisDocument $index $doclist $window $q
+
+    if {$window == ".m.d"} {
+      set first [.m.t.q.r.sdocs nearest 0]
+      if {$index < $first || $index >= $first + $ListBoxLength} {
+          scroll .m.t.q.r [expr $index - $ListBoxLength / 2]
+      }
+      .m.t.q.r.sdocs select from $index
+    } 
+}
+
+
+# Show document with this index in the document list
+#
+proc showThisDocument {index doclist window q} {
+    global collection MaxWindowHeight display windowname
+
+    set document ""
+    scan [lindex $doclist $index] "%d %f" document rank
+    if {$document == ""} { return }
+    set textlist [sendandget $document]
+	
+    if {[string length $display] != 0} { exec csh -c $display >& /dev/null & }
+	
+# throw away the first line
+    set text [join [lrange $textlist 1 end] "\n"]
+
+# if necessary, create a new window for this document
+    if {$window == "New"} {
+        .m.t.q.r.sdocs select clear
+	set window .m.document$windowname
+        incr windowname
+        set existing ""
+        set raised 0
+        foreach w [lrange [winfo children .m] 2 end] {
+           if {[lindex [$w.docnum configure -text] 4] == $index && \
+               [lindex [$w.query configure -text] 4] == $q} {
+              if {$raised} {
+                 closeDocumentWindow $w
+              } else {
+                  raise $w
+		  getFocus $w
+                  set window $w
+                  set raised 1
+              }
+           }
+        }
+  
+        if {!$raised} {
+   	    toplevel $window
+            $window configure -borderwidth 2 -relief raised
+            wm protocol $window WM_TAKE_FOCUS "getFocus $window"
+	    getFocus $window
+	    makeDocumentPanel $window
+            bind $window q "$window.buttons.close invoke"
+            label $window.docnum -text $index
+            label $window.query -text $q
+       }
+    }
+
+# configure the window for this document
+    set rows [expr [llength $textlist] - 1]
+    if {$window != ".m.d"} {
+	wm title $window "$collection \"$q\""
+	wm iconname $window "$document"
+	$window.text configure -height $rows
+	if {$rows > $MaxWindowHeight} {
+	    $window.text configure -height $MaxWindowHeight
+	}
+    }
+
+# show the document text in the window
+    enterText $window.text $text
+
+# configure the window's Previous and Next buttons
+    if {$index == 0} {
+        disableButton $window.buttons.previous
+    } else {
+	set newindex [expr $index-1]
+	bindBoth $window previous $newindex $doclist $q
+        bind $window p [bind $window.buttons.previous <1>]
+        bind $window P [bind $window.buttons.previous <Shift-1>]
+    }
+    if {$index == [expr [llength $doclist]-1]} {
+	disableButton $window.buttons.next 
+    } else {
+	set newindex [expr $index+1]
+	bindBoth $window next $newindex $doclist $q
+        bind $window n [bind $window.buttons.next <1>]
+        bind $window N [bind $window.buttons.next <Shift-1>]
+    }
+
+    if {$window != ".m.d"} { $window.docnum configure -text $index }
+
+    if {[string length $display] != 0} { 
+        set on "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground Black
+                 update\}
+        }"
+        set off "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground [lindex [.m.d configure -background] 4]
+                 update\}
+        }"
+        set t 0
+        while {$t < 5} {
+            after [expr $t * 1000] eval $on
+            after [expr $t * 1000 + 500] eval $off
+            incr t
+        }   
+    }
+}
+
+# Bind both Button-1 and Shift-Button-1 commands
+#
+proc bindBoth {window button index doclist query} {
+    $window.buttons.$button configure -state normal
+    bind $window.buttons.$button <Button-1> \
+	"adjacentDocument $index \"$doclist\" $window \"$query\" $button"
+    bind $window.buttons.$button <Shift-Button-1> \
+	"adjacentDocument $index \"$doclist\" New \"$query\" $button"
+}
+
+# Change the type of query
+#
+proc changeQuerytype {name element op} {
+    global querytype booleanquery approxquery
+
+    if {$booleanquery} {
+	.m.t.b.type.approx configure -state disabled
+	.m.t.q.retrieved.weight configure \
+             -foreground [lindex [.m configure -background] 4]
+	set querytype boolean
+    } else {
+	.m.t.b.type.approx configure -state normal
+	.m.t.q.retrieved.weight configure -foreground Black
+	if {$approxquery} {
+	    set querytype "approx-ranked"
+	} else {
+	    set querytype "ranked"
+	}
+    }
+}
+
+proc enterText {window text} {
+    $window configure -state normal
+    $window delete 0.0 end
+    $window insert 0.0 $text
+    $window configure -state disabled
+}
+
+###############################################################################
+# IO and initialization
+###############################################################################
+
+# Send a message to mg and return the reply
+#
+proc sendandget {message} {
+    global fileId Terminator display errorOccurred ImageString
+    set display ""
+
+    .m config -cursor {watch}
+    update idletasks
+    set reply ""
+    sendto "$message\n"
+    if {$message != "" && !$errorOccurred} {
+        gets $fileId line
+    	while {[string compare $Terminator $line] != 0} {
+            lappend reply $line;
+            gets $fileId line
+        }
+
+        if {[string compare $ImageString [lindex $reply 1]] == 0 } {
+          if {[string compare "MGDATA" [string range [lindex $reply 3] 0 5]] == 0} {
+             set display [string range [lindex $reply 3] 7 end];
+          }
+          set reply [lrange $reply 4 end]
+        }
+    }
+
+    .m config -cursor {}
+    return $reply
+}
+    
+# Send a message to mg
+#
+proc sendto {message} {
+    global fileId errorOccurred
+
+    puts $fileId $message
+    set errorOccurred [catch {flush $fileId}]
+}
+
+# Reinitialise mgquery to work with a new document collection
+#
+proc reInitMG {c} {
+    global fileId
+    catch {close $fileId}
+    initMG $c
+}
+
+# Initialize mgquery to work with a specified text collection
+#
+proc initMG {c} {
+    global query hits fileId collection Terminator env errorOccurred
+    global HeadLength
+
+    set collection $c
+    set query ""
+    set hits 0
+    set errorOccurred 0
+
+    foreach f "sdocs srank" { .m.t.q.r.$f delete 0 end }
+    enterText .m.d.text ""
+    .m.t.b.collect.m configure -text $collection
+    foreach w [winfo children .m] {
+	if {[string first ".m.document" $w] == 0} { closeDocumentWindow $w }
+    }
+
+    disableButton .m.d.buttons.previous 
+    disableButton .m.d.buttons.next
+
+    if {$c != ""} {
+        set f $env(MGDATA)/$c
+
+        if {[file exists $f] && [file readable $f]} {
+    	    set fileId [open "| mgquery $collection" RDWR]
+	    sendto ".set expert \"true\""
+	    sendto ".set terminator \"$Terminator\\n\""
+	    sendto ".set pager \"cat\""
+            sendto ".set heads_length $HeadLength"
+	    return
+        }
+    }
+    bind .m.t.q.e.entry <Return> {null}
+    enterText .m.d.text [initializationFailure]
+    update
+}
+
+# Routine to obtain values from RESOURCE_MANAGER
+#
+proc query_resource {option class default} {
+	set val [option get . $option $class]
+	if {[string length $val] == 0} { return $default }
+	return $val
+}
+
+
+proc disableButton {b} {
+    bind $b <Button-1> {null}
+    bind $b <Shift-Button-1> {null}
+    $b configure -state disabled
+}
+
+proc null {} {}
+
+###############################################################################
+# Setting up windows
+###############################################################################
+
+proc removeFromWindowList {window} {
+    global stacking
+
+    set i 0
+    while {$i < [llength $stacking]} {
+      if {$window == [lindex $stacking $i]} {
+         set stacking [lreplace $stacking $i $i]
+      }
+      incr i
+    }
+}
+
+proc getFocus {window} {
+    global stacking
+
+    focus $window
+    removeFromWindowList $window
+    set stacking [linsert $stacking 0 $window]
+}
+
+proc closeDocumentWindow {w} {
+   global stacking
+   removeFromWindowList $w
+   set t [lindex $stacking 0]
+
+   if {$t != ""} {
+     focus $t
+     raise $t
+  }
+  destroy $w
+}
+
+# Make a "document" panel, including Previous, Next, and Close buttons
+#
+proc makeDocumentPanel {window} {
+    frame $window.buttons
+    button $window.buttons.previous -text "Previous" -width 10 -state disabled
+    button $window.buttons.next -text "Next" -width 10 -state disabled
+    button $window.buttons.close -text "Close" -width 10 \
+	-command "closeDocumentWindow $window"
+    button $window.buttons.save -text "Save" -width 10 \
+        -command "SaveDocument $window.text"
+
+    label $window.buttons.image -text "Fetching image..." \
+            -foreground [lindex [$window configure -background] 4]
+    pack $window.buttons.previous $window.buttons.next -side left
+    pack $window.buttons.image -side left -padx 10
+    pack $window.buttons.close $window.buttons.save -side right 
+    pack $window.buttons -side top -fill x 
+
+    text $window.text -borderwidth 2 -relief raised \
+             -setgrid true -width 80 -state disabled
+
+    $window.text configure -yscrollcommand "$window.sy set" 
+    scrollbar $window.sy -relief flat -command "$window.text yview"
+    pack $window.sy -side right -fill y 
+    pack $window.text -side top -expand yes -fill both
+}
+
+# Make the "buttons" panel, with the query type and collection name
+#
+proc mkButtons {window} {
+    global collection booleanquery approxquery env
+    frame $window.type
+    label $window.type.label -text "Query type"
+
+    radiobutton $window.type.boolean -text "Boolean" -variable booleanquery \
+	  -value 1 -relief flat
+
+    radiobutton $window.type.ranked -text "Ranked" -variable booleanquery \
+	  -value 0 -relief flat
+
+    checkbutton $window.type.approx -text "Approximate" -variable approxquery \
+	  -relief flat
+
+    trace variable booleanquery w changeQuerytype
+    trace variable approxquery w changeQuerytype
+
+    pack $window.type.label   -side top -anchor w
+    pack $window.type.boolean -side top -anchor w -padx 12
+    pack $window.type.ranked  -side top -anchor w -padx 12
+    pack $window.type.approx  -side top -anchor w -padx 36
+
+    pack $window.type -side top -anchor w -pady 12
+
+    frame $window.collect
+    label $window.collect.label -text "Collection: "
+    pack $window.collect.label -side left -anchor w
+
+    getViewer
+
+    set dirs [getMGDATA]
+    set collection ""
+    if {$dirs != ""} { set collection [lindex $dirs 0] }
+   
+    menubutton $window.collect.m -menu $window.collect.m.menu \
+	-relief sunken
+    menu $window.collect.m.menu
+    foreach c $dirs {
+        if {[file readable $env(MGDATA)/$c] && 
+            [file isdirectory $env(MGDATA)/$c]} {
+    	    $window.collect.m.menu add command -label $c \
+	        -command "reInitMG $c"
+        }
+    }
+    pack $window.collect.m -side left
+    pack $window.collect -side bottom -anchor w
+
+
+}
+
+# Check that the MGDATA environment variable is set, and is non-null
+#
+proc getMGDATA {} {
+    global env
+
+    set MGDATA 0
+    foreach i [array names env] {
+	if {$i == "MGDATA"} {
+	    set dirs [exec /bin/ls $env(MGDATA)]
+	    if {[info exists dirs]} { return $dirs }
+	}
+    }
+
+    return ""
+}
+
+proc getViewer {} {
+    global env
+
+    set viewer 0
+    foreach i [array names env] {
+	if {$i == "MGIMAGEVIEWER"} {set viewer $env(MGIMAGEVIEWER)}
+    }
+
+    if {$viewer == 0} {set env(MGIMAGEVIEWER) "xloadimage -quiet -fit stdin"}
+}
+
+# Make the "query" panel, with the query and list of retrieved documents
+#
+proc mkQuery {window} {
+    global hits ListBoxLength
+
+# at the top of the "query" panel is the "entry" frame
+    frame $window.e
+    label $window.e.label -text "Query"
+    entry $window.e.entry -relief sunk -width 47 -textvariable query
+    pack $window.e.label -side left
+    pack $window.e.entry -side right -expand yes -fill x
+    focus default $window.e.entry
+    focus $window.e.entry
+
+    bind $window.e.entry <Control-Key> {null}
+    bind $window.e.entry <Return> "sendQuery $window.r"
+    bind $window.e.entry <Escape> { set query "" }
+    bind $window.e.entry <2> {
+	%W insert insert [selection get STRING]
+	tk_entrySeeCaret %W
+    }
+
+# next is the "retrieved" frame
+    frame $window.retrieved
+    label $window.retrieved.weight -text "Weight"
+    label $window.retrieved.label -text "   Documents retrieved: "
+    label $window.retrieved.hits -width 4 -relief sunk -textvariable hits
+    set documentsRetrieved 0
+    pack $window.retrieved.weight $window.retrieved.label \
+        $window.retrieved.hits -side left
+
+# next is the "results" frame (note that the docnums listbox doesn't appear on
+# the screen)
+    frame $window.r
+    scrollbar $window.r.sscroll -relief sunken -command "scroll $window.r"
+#    bind $window.r.sscroll <ButtonRelease-1> "updateSdocs $window.r.sdocs $window.r.sdocnums"
+    listbox $window.r.sdocnums
+    listbox $window.r.sdocs -relief sunken \
+            -setgrid 1 -geometry 50x$ListBoxLength
+    listbox $window.r.srank -yscroll "$window.r.sscroll set" \
+            -setgrid 1 -geometry 4x$ListBoxLength
+  
+    bind $window.r.srank <Any-1> {null}
+    bind $window.r.srank <Any-2> {null}
+    bind $window.r.srank <Any-3> {null}
+
+    bind $window.r.srank <Any-B1-Motion> {null}
+    bind $window.r.srank <Any-B2-Motion> {null}
+    bind $window.r.srank <Any-B3-Motion> {null}
+
+    bind $window.r.sdocs <Any-2> {null}
+    bind $window.r.sdocs <Any-B2-Motion> {null}
+
+    pack $window.r.sscroll -side right -fill y
+    pack $window.r.srank -side left 
+    pack $window.r.sdocs -side left -fill x -expand yes
+
+# pack them up
+    pack $window.e -side top -anchor e -pady 10 -fill x -expand yes
+    pack $window.retrieved -anchor w
+    pack $window.r -side bottom -anchor e -pady 10 -fill x -expand yes
+}
+
+# Make a "help" window
+#
+proc mkHelp {} {
+    if [winfo exists .help] { destroy .help }
+    toplevel .help
+    wm title .help "X interface to the mg information retrieval system"
+    wm iconname .help "About xmg"
+    frame .help.buttons
+    button .help.buttons.close -text Close -command "destroy .help" -width 10
+    button .help.buttons.help -text Help -width 10
+    button .help.buttons.conditions -text Conditions -width 10
+    button .help.buttons.warranty -text Warranty -width 10
+    button .help.buttons.bugs -text Bugs -width 10
+
+    frame .help.t
+    text .help.t.text -relief raised -bd 2 -setgrid true -height 24 -width 95
+    .help.t.text configure -yscrollcommand ".help.t.sy set" 
+    scrollbar .help.t.sy -relief flat -command ".help.t.text yview"
+    
+    pack .help.buttons.close .help.buttons.help -side right -fill x
+    pack .help.buttons.conditions .help.buttons.warranty \
+         .help.buttons.bugs -side left
+    pack .help.buttons -side bottom -fill x
+
+    pack .help.t.text -side left -expand yes -fill both
+    pack .help.t.sy -side right -fill y 
+    pack .help.t -side top -fill both -expand yes
+    enterText .help.t.text [HelpText]
+    grayButton help
+}
+
+proc grayButton {button} {
+    bind .help.buttons.help <1> "enterText .help.t.text \[HelpText\]
+         grayButton help" 
+    .help.buttons.help configure -state normal
+
+    bind .help.buttons.bugs <1> "enterText .help.t.text \[Bugs\]
+         grayButton bugs" 
+    .help.buttons.bugs configure -state normal
+
+    bind .help.buttons.warranty <1> "enterText .help.t.text \[Warranty\]
+         grayButton warranty"
+    .help.buttons.warranty configure -state normal
+
+    bind .help.buttons.conditions <1> "enterText .help.t.text \[Conditions\]
+         grayButton conditions" 
+    .help.buttons.conditions configure -state normal
+    disableButton .help.buttons.$button
+}
+
+# The text for errors in the MG environment variable
+#
+
+
+proc bookDetails {} {
+  return { \
+
+ This X interface was written by Bruce McKenzie, Ian Witten and Craig Nevill-Manning.  
+ For information on mg, see
+             Witten, I.H.,  Moffat, A. and Bell, T.C. (1994) "Managing gigabytes: compressing
+             and indexing documents and images." Van Nostrand Reinhold, New York.
+             ISBN 0-442-01863-0, US $54.95; call 1 (800) 544-0550 to order.
+}
+}
+
+proc initializationFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ Your MGDATA variable is unset or does not specify a directory with files in; therefore XMG
+ cannot proceed.
+} [bookDetails]]
+}
+
+proc mgqueryFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ There is a problem with the collection that you have chosen. Please re-index the collection before
+ you use it again.
+} [bookDetails]]
+
+}
+
+# The text for the help message
+#
+proc HelpText {} {
+    return [format "%s\n%s" {\
+This is an X interface to the mg information retrieval system.
+
+ Type a query in the "Query" box.
+ The first line or so of each matching document is listed. 
+ Click on any line to show the complete document.
+ Shift-click to show the document in its own window
+ Press Next and Previous to cycle through the list of documents (Shift works here too).
+ In document windows (other than the main window), typing n or p shows the next and previous 
+ documents; N or P creates a new window for them. q closes the most recently created window.
+ If you change the query, existing windows continue to refer to the original query.
+ 
+ You can select either Boolean or Ranked queries.
+ Boolean queries: use & for AND, | for OR, ! for NOT, and parentheses for grouping
+ Ranked queries:  documents are listed in order of relevance; a weight is shown alongside each
+                                  weights are normalized to 100 for the most relevant document
+                                  choose Approximate Ranking for greater retrieval speed. 
+ } [bookDetails]]
+}
+
+proc reformat {text} {
+    regsub -all "\n *\n *" $text "BLEQUEWFI" text
+    regsub -all "\[ \t\n\r\]+" $text " " text
+    regsub -all "BLEQUEWFI" $text "\n\n" text
+    return $text
+}
+
+proc Conditions {} {
+    return [reformat [join [sendandget ".conditions\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Warranty {} {
+    return [reformat [join [sendandget ".warranty\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Bugs {} {
+    return [reformat {
+Known shortcomings of xmg include:
+
+    Slow picture viewing.  Pictures are decompressed in their entirety before a window is created 
+        for the picture.  This often results in a long delay with no feedback. Moreover, when 
+        an image's document is displayed repeatedly, multiple identical images are shown.
+
+    Sequential viewing.  There is no way of going to the next document in the collection to see the
+        context of documents that have been retrieved.  This would be trivial to implement but
+        conflicts with the interpretation of the "previous" and "next" buttons.
+    }]
+}
+
+###############################################################################
+# The main program
+###############################################################################
+
+scan [info tclversion] "%d.%d" major minor
+
+if {$major < 7} {
+    puts "xmg requires version 7.0 of the tcl interpreter. You are using version $major.$minor"
+    exit
+}
+
+# Default font settings
+set defaulttextfont   -adobe-times-medium-r-normal--14-140-75-75-p-74-iso8859-1
+set defaultbuttonfont -adobe-times-bold-r-normal--14-140-75-75-p-77-iso8859-1
+set textfont   [query_resource textfont Textfont $defaulttextfont]
+set buttonfont [query_resource buttonfont Buttonfont $defaultbuttonfont]
+
+set MaxWindowHeight 10
+set HeadLength 100
+set scrolledto 0
+set ListBoxLength 12
+set windowname 0
+set Terminator "RECORD_END"
+set fileId ""
+set query ""
+set ImageString "::::::::::"
+set stacking ".m.t"
+
+tk_listboxSingleSelect Listbox
+option add *Text.wrap word
+
+option add *Text.font $textfont
+option add *Listbox.font $textfont
+option add *Entry.font $textfont
+
+option add *Button.font $buttonfont
+option add *Menubutton.font $buttonfont
+option add *Radiobutton.font $buttonfont
+option add *Checkbutton.font $buttonfont
+option add *Label.font $buttonfont
+option add *Menu.font $buttonfont
+
+wm title . "Managing Gigabytes"
+wm iconname . "xmg"
+frame .m -borderwidth 5 -relief raised
+
+# top part of frame
+frame .m.t
+
+# on the left side of the top part is the "buttons" panel
+frame .m.t.b
+mkButtons .m.t.b
+
+# on the right side is the "query" panel
+frame .m.t.q
+mkQuery .m.t.q
+
+set q [query_resource querytype Querytype approx-ranked]
+set booleanquery [expr \"$q\" == \"boolean\"]
+set approxquery [expr \"$q\" == \"approx-ranked\"]
+# the above lines will trigger changeQuerytype
+
+pack .m.t.b -side left -anchor nw
+pack .m.t.q -side right -anchor ne -fill both -expand yes
+
+# at the bottom is the "document" panel
+frame .m.d
+makeDocumentPanel .m.d
+
+.m.d.buttons.close configure -text "Exit" \
+    -command "destroy ."
+button .m.d.buttons.help -text "Help" -width 10 \
+	-command "mkHelp"
+
+pack .m.d.buttons.help -side right
+pack .m.t -side top -fill x -expand yes
+pack .m.d -side bottom -fill both -expand yes
+
+pack .m -fill both -expand yes
+wm geometry . +300+85
+
+set collection [query_resource collection Collection $collection]
+if {$argc>0} { set collection $argv }
+
+enterText .m.d.text [HelpText]
+
+initMG $collection
+
+###############################################################################
+###############################################################################
+# The following file selection box procedure was written by Andrew Donkin
+# of the University of Waikato. 
+###############################################################################
+
+proc FSBox { {fsBoxMessage "Select file:"}} {
+# xf ignore me 5
+  global fsBox
+
+  set fsBox(name) ""
+  set fsBox(internalPath) [pwd]
+  set fsBox(pattern) "*"
+  set fsBox(all) 0
+
+  # build widget structure
+
+  # start build of toplevel
+  if {"[info commands XFDestroy]" != ""} {
+    catch {XFDestroy .fsBox}
+  } {
+    catch {destroy .fsBox}
+  }
+  toplevel .fsBox -borderwidth 0
+  wm geometry .fsBox 350x300 
+  wm title .fsBox {File select box}
+  wm maxsize .fsBox 1000 1000
+  wm minsize .fsBox 100 100
+  # end build of toplevel
+
+  label .fsBox.message1 -anchor c -text "$fsBoxMessage"
+  frame .fsBox.frame1  -borderwidth 0  -relief raised
+
+  button .fsBox.frame1.ok  -text "OK"  -command "
+      global fsBox
+      set fsBox(name) \[.fsBox.file.file get\]
+      set fsBox(path) \[.fsBox.path.path get\]
+
+      set fsBox(internalPath) \[.fsBox.path.path get\]
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  button .fsBox.frame1.rescan -text "Rescan" -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+  }
+
+  button .fsBox.frame1.cancel -text "Cancel" -command "
+      global fsBox
+      set fsBox(name) {}
+      set fsBox(path) {}
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  frame .fsBox.path -borderwidth 0 -relief raised
+  frame .fsBox.path.paths 
+  label .fsBox.path.paths.paths -borderwidth 0 -text "Pathname:"
+
+  entry .fsBox.path.path -relief sunken
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+  frame .fsBox.pattern -borderwidth 0 -relief raised
+  frame .fsBox.pattern.patterns 
+
+  label .fsBox.pattern.patterns.patterns -borderwidth 0 -text "Selection pattern:"
+  entry .fsBox.pattern.pattern -relief sunken
+  .fsBox.pattern.pattern insert 0 $fsBox(pattern)
+  
+  frame .fsBox.files -borderwidth 0 -relief raised
+  scrollbar .fsBox.files.vscroll -relief raised -command ".fsBox.files.files yview"
+  scrollbar .fsBox.files.hscroll -orient horiz -relief raised -command ".fsBox.files.files xview" 
+  listbox .fsBox.files.files -exportselection false -relief raised -xscrollcommand \
+        ".fsBox.files.hscroll set" -yscrollcommand ".fsBox.files.vscroll set"
+  frame .fsBox.file -borderwidth 0 -relief raised
+  label .fsBox.file.labelfile -text "Filename:"
+  entry .fsBox.file.file -relief sunken
+  focus .fsBox.file.file
+
+  .fsBox.file.file delete 0 end
+  .fsBox.file.file insert 0 $fsBox(name)
+  
+  checkbutton .fsBox.pattern.all -offvalue 0 -onvalue 1 -text "Show all files" -variable fsBox(all) -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    }
+
+  FSBoxFSShow $fsBox(internalPath) $fsBox(pattern) $fsBox(all)
+
+  # bindings
+  bind .fsBox.files.files <Double-Button-1> "FSBoxFSFileSelectDouble %W %y"
+  bind .fsBox.files.files <ButtonPress-1> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-ButtonPress-1> "FSBoxFSFileSelect %W %y"
+
+  bind .fsBox.path.path <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.path.path <Up> {}"
+  bind .fsBox.path.path <Down> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+
+  bind .fsBox.file.file <Return> "
+    global fsBox
+    set fsBox(name) \[.fsBox.file.file get\]
+    set fsBox(path) \[.fsBox.path.path get\]
+    set fsBox(internalPath) \[.fsBox.path.path get\]
+
+    if {\"\[info commands XFDestroy\]\" != \"\"} {
+      catch {XFDestroy .fsBox}
+    } {
+      catch {destroy .fsBox}
+    }"
+  bind .fsBox.file.file <Up> {.fsBox.path.path icursor end
+    focus .fsBox.path.path}
+  bind .fsBox.file.file <Down> {.fsBox.pattern.pattern icursor end
+    focus .fsBox.pattern.pattern}
+
+  bind .fsBox.pattern.pattern <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)}
+  bind .fsBox.pattern.pattern <Up> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.pattern.pattern <Down> {}"
+
+  # packing
+  pack .fsBox.files.vscroll -side left -fill y 
+  pack .fsBox.files.hscroll -side bottom -fill x
+  pack .fsBox.files.files -side left -fill both -expand yes
+
+  pack .fsBox.file.labelfile -side left
+  pack .fsBox.file.file -side left -fill  both -expand yes
+
+  pack .fsBox.frame1.ok -side left -fill both -expand yes
+  pack .fsBox.frame1.rescan -side left -fill both -expand yes
+  pack .fsBox.frame1.cancel -side left -fill both -expand yes
+
+  pack .fsBox.path.paths.paths -side left
+
+  pack .fsBox.pattern.patterns.patterns -side left
+
+  pack .fsBox.path.paths -side left
+  pack .fsBox.path.path -side left -fill both -expand yes
+
+  pack .fsBox.pattern.patterns -side left
+  pack .fsBox.pattern.all -side right -fill both
+  pack .fsBox.pattern.pattern -side left -fill both -expand yes
+
+  pack .fsBox.message1 -side top -fill both
+  pack .fsBox.frame1 -side bottom -fill both
+  pack .fsBox.pattern -side bottom -fill both
+  pack .fsBox.file -side bottom -fill both
+  pack .fsBox.path -side bottom -fill both
+  pack .fsBox.files -side left -fill both -expand yes
+
+  update idletask
+  grab .fsBox
+  tkwait window .fsBox
+
+  if {"[string trim $fsBox(path)]" != "" ||
+      "[string trim $fsBox(name)]" != ""} {
+    if {"[string trimleft [string trim $fsBox(name)] /]" == ""} {
+      return [string trimright [string trim $fsBox(path)] /]
+    } {
+      return [string trimright [string trim $fsBox(path)] /]/[string trimleft [string trim $fsBox(name)] /]
+    }
+  }
+}
+
+proc FSBoxBindSelectOne { fsBoxW fsBoxY} {
+# xf ignore me 6
+
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    $fsBoxW select from $fsBoxNearest
+    $fsBoxW select to $fsBoxNearest
+  }
+}
+
+proc FSBoxFSFileSelect { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+        "[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+      set fsBoxFileName [string range $fsBoxTmpEntry 0             [expr [string length $fsBoxTmpEntry]-2]]
+      if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+          ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    } {
+      if {"[string index $fsBoxTmpEntry             [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0           [expr [string length $fsBoxTmpEntry]-2]]
+        if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    }
+    if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+      set fsBox(name) $fsBoxFileName
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 $fsBox(name)
+    }
+  }
+}
+
+proc FSBoxFSFileSelectDouble { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"$fsBoxTmpEntry" == "../"} {
+      set fsBoxTmpEntry [string trimright [string trim $fsBox(internalPath)] "@/"]
+      if {"$fsBoxTmpEntry" == ""} {
+        return
+      }
+      FSBoxFSShow [file dirname $fsBoxTmpEntry]         [.fsBox.pattern.pattern get] $fsBox(all)
+      .fsBox.path.path delete 0 end
+      .fsBox.path.path insert 0 $fsBox(internalPath)
+    } {
+      if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+          "[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0               [expr [string length $fsBoxTmpEntry]-2]]
+        if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+            ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+          set fsBoxFileName [string range $fsBoxTmpEntry 0                 [expr [string length $fsBoxTmpEntry]-2]]
+          if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+            set fsBoxFileName $fsBoxTmpEntry
+          }
+        } {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      }
+      if {[IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBox(internalPath) "[string trimright $fsBox(internalPath) {/@}]/$fsBoxFileName"
+        FSBoxFSShow $fsBox(internalPath) [.fsBox.pattern.pattern get] $fsBox(all)
+        .fsBox.path.path delete 0 end
+        .fsBox.path.path insert 0 $fsBox(internalPath)
+      } {
+        set fsBox(name) $fsBoxFileName
+        set fsBox(path) $fsBox(internalPath)
+        if {"[info commands XFDestroy]" != ""} {
+          catch {XFDestroy .fsBox}
+        } {
+          catch {destroy .fsBox}
+        }
+      }
+    }
+  }
+}
+
+proc FSBoxFSShow { fsBoxPath fsBoxPattern fsBoxAll} {
+# xf ignore me 6
+  global fsBox
+
+  set fsBox(pattern) $fsBoxPattern
+
+  if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+      [IsADir $fsBoxPath]} {
+    set fsBox(internalPath) $fsBoxPath
+  } {
+    if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+        [IsAFile $fsBoxPath]} {
+      set fsBox(internalPath) [file dirname $fsBoxPath]
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 [file tail $fsBoxPath]
+      set fsBoxPath $fsBox(internalPath)
+    } {
+      while {"$fsBoxPath" != "" && "$fsBoxPath" != "/" &&
+             ![file isdirectory $fsBoxPath]} {
+        set fsBox(internalPath) [file dirname $fsBoxPath]
+         set fsBoxPath $fsBox(internalPath)
+      }
+    }
+  }
+  if {"$fsBoxPath" == ""} {
+    set fsBoxPath "/"
+    set fsBox(internalPath) "/"
+  }
+  .fsBox.path.path delete 0 end
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+
+  if {[.fsBox.files.files size] > 0} {
+    .fsBox.files.files delete 0 end
+  }
+  if {$fsBoxAll} {
+    if {[catch "exec ls -F -a $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  } {
+    if {[catch "exec ls -F $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  }
+  set fsBoxElementList [lsort $fsBoxResult]
+
+  foreach fsBoxCounter [winfo children .fsBox.pattern.patterns.patterns] {
+    if {[string length [info commands XFDestroy]] > 0} {
+      catch {XFDestroy $fsBoxCounter}
+    } {
+      catch {destroy $fsBoxCounter}
+    }
+  }
+  menu .fsBox.pattern.patterns.patterns.menu
+
+  if {"$fsBoxPath" != "/"} {
+    .fsBox.files.files insert end "../"
+  }
+  foreach fsBoxCounter $fsBoxElementList {
+    if {[string match $fsBoxPattern $fsBoxCounter] ||
+        [IsADir [string trimright $fsBoxPath/$fsBoxCounter "/@"]]} {
+      if {"$fsBoxCounter" != "../" &&
+          "$fsBoxCounter" != "./"} {
+        .fsBox.files.files insert end $fsBoxCounter
+      }
+    }
+  }
+}
+
+proc IsADir { pathName} {
+# xf ignore me 5
+  if {[file isdirectory $pathName]} {
+    return 1
+  } {
+    catch "file type $pathName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $pathName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isdirectory $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsAFile { fileName} {
+# xf ignore me 5
+  if {[file isfile $fileName]} {
+    return 1
+  } {
+    catch "file type $fileName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $fileName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isfile $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsASymlink { fileName} {
+# xf ignore me 5
+  catch "file type $fileName" fileType
+  if {"$fileType" == "link"} {
+    return 1
+  }
+  return 0
+}
Index: /main/tags/3.01/indexers/mg/src/text/GCC.mak
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/GCC.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/GCC.mak	(revision 21112)
@@ -0,0 +1,205 @@
+###########################################################################
+#
+# gcc win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2000  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+RANLIB = ranlib
+CC = gcc
+DELETE = del
+INSTALL = copy
+
+o = .o
+e = .exe
+/ = \\
+
+CPPFLAGS = -g
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D_LITTLE_ENDIAN
+LIBS = ..$/..$/lib$/libmg.a
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c $o
+.c$o:
+	$(COMPILE) $<
+ 
+INCLUDES = -I"..$/.." -I"..$/..$/lib"
+
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+EXEC = \
+  mgquery$e mg_weights_build$e mgstat$e mg_invf_dump$e \
+  mg_invf_dict$e mg_invf_rebuild$e mgdictlist$e \
+  mg_passes$e mg_perf_hash_build$e mg_compression_dict$e \
+  mg_text_estimate$e mg_fast_comp_dict$e mg_hilite_words$e \
+  mg_invf_merge$e mg_text_merge$e bool_tester$e \
+  mg_stem_idx$e  mgstemidxlist$e mg_decompress_text$e
+
+all:    $(EXEC) libtextin.a
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(QUERY_OBJS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+install: 
+	$(INSTALL) *$e ..$/..$/..$/..$/bin$/windows
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	$(DELETE) libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+clean:
+	$(DELETE) *$o 
+	$(DELETE) *$e
Index: /main/tags/3.01/indexers/mg/src/text/MGPASS.C
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/MGPASS.C	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/MGPASS.C	(revision 21112)
@@ -0,0 +1,510 @@
+/**************************************************************************
+ *
+ * mgpass.cpp -- Driver for the various passes -
+
+	V1 -  removed all the pipe processing and replaced with
+	      code to directly explore a directory of files.
+	V2 -  rebuilt to extract non text files from web browser
+	      'catch' file.  Also to display progress count
+			   GH/WJR
+
+ * Copyright (C) 1994  Neil Sharman, ..
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <malloc.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <io.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include "memlib.h"
+#include "messages.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/20 02:32:00  cs025
+   a few fixes for compiling under windows with new long long stuff in mg
+
+   Revision 1.1  1999/08/10 21:17:39  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.5  1998/12/02 01:32:40  sjboddie
+   fixed 1 missing semi-colon..
+
+   Revision 1.4  1998/12/02 00:57:10  rjmcnab
+
+   Fixed to compile with the changes to the stemming.
+
+   Revision 1.3  1998/11/24 03:39:23  rjmcnab
+
+   Fixed a small error in the windows compilation
+
+   Revision 1.2  1998/11/24 01:29:39  rjmcnab
+
+   Fixed a few problems with the windows build
+
+   Revision 1.1  1998/11/17 09:34:12  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define path_length 256
+
+unsigned long buf_size = 3 * 1024 * 1024;       /* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;       /* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+int num_docs = 0;
+mg_ullong bytes_received = 0;
+int stem_method = 0;
+int stemmer_num = 0; /* default to the Lovin stemmer */
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+static char dirname[path_length], wildname[path_length];
+static int by_para = 0, recurse = 0, html_catch = 0;
+static char *buffer;
+char *line_start, *data_end, *base, *scan;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+  }
+pass_data;
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method] -f doc-collection-name\n"
+"  %*s [source directory\\] [source file]\n";
+
+static void usage (char *err)
+{
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+    strlen (msg_prefix), "", strlen (msg_prefix), "");
+  exit (1);
+}
+
+void do_process(char *buffer, int num)
+{
+  int pass;
+  if (num == 0) Message ("Warning : Processing zero length document");
+  num_docs++;
+  bytes_processed += num;
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+	if (PassData[pass].process((u_char *)buffer, num) == COMPERROR)
+	   FatalError(1, "Error during file processing");
+}
+
+int refill(int in_fd)
+{
+  int num, bitleft;
+  bitleft = data_end - base;
+  memmove(buffer, base, bitleft);
+  if (buf_size - bitleft < MIN_BUF)
+    FatalError(1, "Paragraph too big for buffer");
+  num = read(in_fd, &buffer[bitleft], buf_size - bitleft);
+  line_start -= (base - buffer);
+  scan -= (base - buffer);
+  base = buffer;
+  data_end = buffer + bitleft + num;
+  if (num > 0) return 1; else return 0;
+}
+
+char *scanpara(int in_fd)
+{
+  int num, blank, in_blank, at_end;
+  num = read(in_fd,buffer,buf_size);
+  in_blank = 1;  base = buffer;  line_start = buffer;  scan = buffer;
+  data_end = buffer + num;  at_end = 0;
+  for (;;) {
+    blank = 1;  line_start = scan;                   // Get a line
+    while (scan < data_end  &&  *scan != '\n')
+      if (*scan++ > ' ') blank = 0;
+    if (scan >= data_end) {
+      if (refill(in_fd)) {
+	while (scan < data_end  &&  *scan != '\n')
+	  if (*scan++ > ' ') blank = 0;
+	if (scan < data_end) scan++;
+	}
+      else at_end = 1;
+      }
+    else scan++;
+    if (line_start < scan) {                         // If we have a line
+      if (in_blank) {
+	if (!blank) { in_blank = 0; base = line_start; }
+	}
+      else {
+	if (blank) {
+	  do_process(base, line_start - base);
+	  in_blank = 1;  base = line_start;
+	  }
+	}
+      }
+    if (at_end) break;
+    }
+  if (in_blank) base = scan;
+  if (scan + 2 <= buffer + buf_size) {
+    *scan++ = 26;  *scan++ = 10;
+    }
+  if (base < scan) do_process(base, scan - base);
+  return NULL;
+}
+
+char *scanfile(int in_fd)
+{
+  int num = read(in_fd,buffer,buf_size);  /*expect to read the whole file*/
+  if (num < 0) return "file locked";
+  if (num >= buf_size-1) return "file too long";
+  do_process(buffer, num);
+  return NULL;
+}
+
+void search(char *dname, char *fname)
+{
+  long dirtag;  struct _finddata_t dirinfo;  int in_fd;  char *res;
+  char search_name[path_length], found_name[path_length];
+
+  sprintf(search_name, "%s%s", dname, fname);    /*Scan files*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag >= 0) {
+    do {
+      if ((dirinfo.attrib & (_A_SUBDIR | _A_HIDDEN | _A_SYSTEM)) == 0) {
+	sprintf(found_name,"%s%s",dname,dirinfo.name);
+	in_fd = open(found_name,O_RDONLY|O_BINARY);
+	if (in_fd >= 0) {
+	  if (by_para)
+	    res = scanpara(in_fd);
+	  else
+	    res = scanfile(in_fd);
+	  if (res != NULL) {
+	    Message("Error %s in processing file %s\n", res, found_name);
+	    exit(1);
+	    }
+	  close(in_fd);
+	  }
+	}
+      } while (_findnext(dirtag, &dirinfo) == 0);
+    _findclose(dirtag);
+    }
+
+  if (recurse == 0) return;
+
+  sprintf(search_name, "%s*.*", dname);          /*Look for subdirs*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag < 0) return;
+  do {
+    if ( ((dirinfo.attrib & (_A_HIDDEN | _A_SYSTEM)) == 0)  &&
+	 ((dirinfo.attrib & _A_SUBDIR) != 0)  &&
+	 strcmp(dirinfo.name,".") != 0 &&
+	 strcmp(dirinfo.name,"..") != 0) {
+      sprintf(found_name,"%s%s",dname,dirinfo.name);
+      strcat(found_name,"\\");
+      search(found_name,fname);
+      }
+    } while (_findnext(dirtag, &dirinfo) == 0);
+  _findclose(dirtag);
+}
+
+static int toobig(int n)
+{
+  if (n > path_length) {
+    printf("Cannot handle urls > %d characters in length\n", path_length);
+    exit(1);
+    }
+  return 0;
+}
+
+void scan_catch(char *dname, char *fname)
+{
+  int in_fd, urllen, conlen, filesize;  char catch_name[path_length];
+  int filecount = 0;
+  enum { filekind_redirected, filekind_text, filekind_other } filekind;
+  sprintf(catch_name, "%s%s", dname, fname);
+  in_fd = open(catch_name,O_RDONLY|O_BINARY);
+  if (in_fd < 0)
+    FatalError(1, "Couldn't open catch file \"%s\"", catch_name);
+  for (;;) {
+    filecount++;  if (filecount%100 == 0) { printf("%d\r", filecount);  fflush(stdout); }
+    if (read(in_fd, &urllen, sizeof(int)) != sizeof(int) || toobig(urllen) ||
+	read(in_fd, catch_name, urllen) != urllen ||
+	read(in_fd, &conlen, sizeof(int)) != sizeof(int) || toobig(conlen) ||
+	read(in_fd, catch_name, conlen) != conlen) break;
+    if (conlen >= 1 && *catch_name == '@')
+      filekind = filekind_redirected;
+    else {
+      if (conlen >= 4 && strncmp(catch_name, "text", 4) == 0)
+	filekind = filekind_text;
+      else
+	filekind = filekind_other;
+      if (read(in_fd, &filesize, sizeof(int)) != sizeof(int))
+	FatalError(1, "File read failed for size field");
+      if (filesize > buf_size)
+	FatalError(1, "File too large (%d > %d)", filesize, buf_size);
+      if (read(in_fd, buffer, filesize) != filesize)
+	FatalError(1, "Failed to read file data");
+      }
+    if (filekind == filekind_text) do_process(buffer, filesize);
+    }
+  close(in_fd);
+}
+
+static void driver (FILE * Trace, char *file_name)
+{
+  int pass;
+
+  buffer = (char *)Xmalloc (buf_size);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"",PassData[pass].name);
+      }
+
+  if (html_catch == 0) search(dirname, wildname);
+		  else scan_catch(dirname, wildname);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+
+
+  free (buffer);
+}
+
+void main (int argc, char **argv)
+{
+  int ch;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WHGpSD123f:d:b:T:I:t:m:N:c:n:s:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'H':
+	  html_catch = 1;
+	  break;
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'p':
+	  by_para = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (argc - optind == 1) {
+    strcpy(dirname,"");
+    strcpy(wildname,argv[optind]);
+    }
+  else if (argc - optind == 2) {
+    strcpy(dirname,argv[optind]);
+    strcpy(wildname,argv[optind+1]);
+    }
+  else FatalError(1, "Finder code requires directory and filespec.");
+
+  if (strrchr(wildname,'*') != NULL  ||  strrchr(wildname,'?') != NULL)
+    recurse = 1;
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  exit (0);
+}
Index: /main/tags/3.01/indexers/mg/src/text/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/Makefile.in	(revision 21112)
@@ -0,0 +1,388 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+YACC = @YACC@
+GSDLOS = @gsdlos@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c                 mg_passes_4jni.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+
+MISC          = Makefile.in help.mg.src bool_parser.y warranty.src conditions.src
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgquery mg_weights_build mgstat mg_invf_dump mg_invf_dict \
+                mg_invf_rebuild mgdictlist mg_passes mg_perf_hash_build \
+                mg_compression_dict mg_text_estimate mg_fast_comp_dict \
+                mg_hilite_words mg_invf_merge mg_text_merge bool_tester \
+                mg_stem_idx mgstemidxlist mg_decompress_text 
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+                mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+                mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+                mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+                mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a libmgpass.a
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+help.mg.h : help.mg.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+warranty.h : warranty.src
+	sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+conditions.h : conditions.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+
+mg_hilite_words: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+bool_parser.c: bool_parser.y
+	$(YACC) $?
+	mv y.tab.c $@
+
+BOOL_OBJS = bool_tester$o bool_tree$o bool_parser$o bool_optimiser$o  \
+            term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o environment$o
+
+bool_tester: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+
+mgquery: $(QUERY_OBJS)	
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat:	$(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+
+mg_invf_rebuild: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+
+mgdictlist: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = \
+                mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+
+mgstemidxlist: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text.o query_term_list.o mg_files.o mg_errors.o \
+                text_get.o locallib.o backend.o stem_search.o \
+                term_lists.o stemmer.o weights.o invf_get.o lists.o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+
+mg_invf_merge: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall #maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+JNI_LIB_OBJS = \
+	mg_passes_4jni$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o  words$o environment$o\
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o 
+
+libmgpass.a: $(JNI_LIB_OBJS)
+	rm -f libmgpass.a
+	$(AR) cru libmgpass.a $(JNI_LIB_OBJS)
+	$(RANLIB) libmgpass.a
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC) libtextin.a libmgpass.a
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+  
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
+
+
+
+
Index: /main/tags/3.01/indexers/mg/src/text/backend.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/backend.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/backend.c	(revision 21112)
@@ -0,0 +1,838 @@
+/**************************************************************************
+ *
+ * backend.c -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "filestats.h"
+#include "sptree.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+
+static File *
+OpenFile (char *base, char *suffix, unsigned long magic, int *ok)
+{
+  char FileName[512];
+  File *F;
+  sprintf (FileName, "%s%s", base, suffix);
+  if (!(F = Fopen (FileName, "rb", 0)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      mg_errno = MG_NOFILE;
+      MgErrorData (FileName);
+      if (ok)
+	*ok = 0;
+      return (NULL);
+    }
+  if (magic)
+    {
+      unsigned long m;
+      if (fread ((char *) &m, sizeof (m), 1, F->f) == 0)
+	{
+	  mg_errno = MG_READERR;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+      NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (m != magic)
+	{
+	  mg_errno = MG_BADMAGIC;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+    }
+  return (F);
+}
+
+
+static int 
+open_all_files (query_data * qd)
+{
+  int ok = 1;
+
+  qd->File_text = OpenFile (qd->textpathname, TEXT_SUFFIX, /* [RJM 06/97: text filename] */
+			    MAGIC_TEXT, &ok);
+  qd->File_fast_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				      TEXT_DICT_FAST_SUFFIX, MAGIC_FAST_DICT, NULL);
+  if (!qd->File_fast_comp_dict)
+    {
+      qd->File_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				     TEXT_DICT_SUFFIX, MAGIC_DICT, &ok);
+      qd->File_aux_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				    TEXT_DICT_AUX_SUFFIX, MAGIC_AUX_DICT, NULL);
+    }
+  else
+    qd->File_comp_dict = qd->File_aux_dict = NULL;
+
+  qd->File_stem = OpenFile (qd->pathname, INVF_DICT_BLOCKED_SUFFIX,
+			    MAGIC_STEM, &ok);
+
+  /* [RPAP - Jan 97: Stem Index Change]
+     These will fail if collection not built with stem indexes */
+  qd->File_stem1 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_1_SUFFIX,
+			     MAGIC_STEM_1, NULL);
+  qd->File_stem2 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_2_SUFFIX,
+			     MAGIC_STEM_2, NULL);
+  qd->File_stem3 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_3_SUFFIX,
+			     MAGIC_STEM_3, NULL);
+
+  qd->File_invf = OpenFile (qd->pathname, INVF_SUFFIX,
+			    MAGIC_INVF, &ok);
+
+  /* These will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_text_idx_wgt = OpenFile (qd->pathname, TEXT_IDX_WGT_SUFFIX,
+				    MAGIC_TEXI_WGT, NULL);
+  qd->File_weight_approx = OpenFile (qd->pathname, APPROX_WEIGHTS_SUFFIX,
+				     MAGIC_WGHT_APPROX, NULL);
+  if (qd->File_text_idx_wgt == NULL && qd->File_weight_approx == NULL)
+    qd->File_text_idx = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				  TEXT_IDX_SUFFIX, MAGIC_TEXI, NULL);
+  else
+    qd->File_text_idx = NULL;
+
+
+  if (!ok)
+    {
+      Fclose (qd->File_text);
+      if (qd->File_fast_comp_dict)
+	Fclose (qd->File_fast_comp_dict);
+      if (qd->File_comp_dict)
+	Fclose (qd->File_comp_dict);
+      Fclose (qd->File_stem);
+
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+
+      Fclose (qd->File_invf);
+      if (qd->File_text_idx_wgt)
+	Fclose (qd->File_text_idx_wgt);
+      if (qd->File_weight_approx)
+	Fclose (qd->File_weight_approx);
+      if (qd->File_text_idx)
+	Fclose (qd->File_text_idx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+static void 
+close_all_files (query_data * qd)
+{
+  Fclose (qd->File_text);
+  if (qd->File_fast_comp_dict)
+    Fclose (qd->File_fast_comp_dict);
+  if (qd->File_aux_dict)
+    Fclose (qd->File_aux_dict);
+  if (qd->File_comp_dict)
+    Fclose (qd->File_comp_dict);
+  Fclose (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    Fclose (qd->File_stem1);
+  if (qd->File_stem2)
+    Fclose (qd->File_stem2);
+  if (qd->File_stem3)
+    Fclose (qd->File_stem3);
+
+  Fclose (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    Fclose (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    Fclose (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    Fclose (qd->File_text_idx);
+}
+
+/* If textname is equal to null then name will be used instead */
+/* [RJM 06/97: text filename] */
+query_data *
+InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{
+  query_data *qd;
+  char *s;
+
+  if (textname == NULL) textname = name; /* [RJM 06/97: text filename] */
+
+  if (!(qd = Xmalloc (sizeof (query_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) qd, sizeof (*qd));
+
+  qd->mem_in_use = qd->max_mem_in_use = 0;
+
+  qd->doc_pos = qd->buf_in_use = 0;
+  qd->TextBufferLen = 0;
+  qd->DL = NULL;
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  qd->TL = NULL;
+  qd->QTL = NULL;
+
+  qd->TextBuffer = NULL;
+
+  qd->tot_hops_taken = 0;
+  qd->tot_num_of_ptrs = 0;
+  qd->tot_num_of_accum = 0;
+  qd->tot_num_of_terms = 0;
+  qd->tot_num_of_ans = 0;
+  qd->tot_text_idx_lookups = 0;
+
+  qd->hops_taken = 0;
+  qd->num_of_ptrs = 0;
+  qd->num_of_accum = 0;
+  qd->num_of_terms = 0;
+  qd->num_of_ans = 0;
+  qd->text_idx_lookups = 0;
+
+  qd->pathname = NULL; /* RJM 06/97: text filename] */
+  qd->textpathname = NULL; /* RJM 06/97: text filename] */
+
+  s = strrchr (dir, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 1)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 1)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      sprintf (qd->pathname, "%s%s", dir, name);
+      sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+    }
+
+  else
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 2)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 2)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      if (dir == NULL || dir[0] == '\0') {
+	sprintf (qd->pathname, "%s", name);
+	sprintf (qd->textpathname, "%s", textname); /* [RJM 06/97: text filename] */
+      } else {
+	sprintf (qd->pathname, "%s%s", dir, name);
+	sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+      }
+#else
+      sprintf (qd->pathname, "%s/%s", dir, name);
+      sprintf (qd->textpathname, "%s/%s", dir, textname); /* [RJM 06/97: text filename] */
+#endif
+    }
+
+  if (open_all_files (qd) == -1)
+    {
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->Start);
+
+  /* Initialise the stemmed dictionary system */
+  if (!(qd->sd = ReadStemDictBlk (qd->File_stem)))
+    {
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if ((qd->sd->sdh.indexed & 7) && qd->File_stem1 && qd->File_stem2 && qd->File_stem3)
+    {
+      if (!(qd->sd->stem1 = ReadStemIdxBlk (qd->File_stem1)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem2 = ReadStemIdxBlk (qd->File_stem2)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem3 = ReadStemIdxBlk (qd->File_stem3)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      }
+  else if (qd->sd->sdh.indexed != 0)
+    {
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  else
+    {
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+      qd->File_stem1 = NULL;
+      qd->File_stem2 = NULL;
+      qd->File_stem3 = NULL;
+      qd->sd->stem1 = NULL;
+      qd->sd->stem2 = NULL;
+      qd->sd->stem3 = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->StemDict);
+  if (qd->File_weight_approx)
+    {
+      if (!(qd->awd = LoadDocWeights (qd->File_weight_approx,
+				      qd->sd->sdh.num_of_docs)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+    }
+  else
+    qd->awd = NULL;
+
+
+  if (iqt)
+    GetTime (&iqt->ApproxWeights);
+
+  if (!(qd->cd = LoadCompDict (qd->File_comp_dict, qd->File_aux_dict,
+			       qd->File_fast_comp_dict)))
+    {
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->CompDict);
+
+  if (!(qd->id = InitInvfFile (qd->File_invf, qd->sd)))
+    {
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  if ((qd->File_text_idx_wgt == NULL || qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      mg_errno = MG_INVERSION;
+      return (NULL);
+    }
+  if (iqt)
+    GetTime (&iqt->Invf);
+
+  if (!(qd->td = LoadTextData (qd->File_text, qd->File_text_idx_wgt,
+			       qd->File_text_idx)))
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+
+/*
+
+This code is based on the TREC_MODE code below to read the .paragraph
+file to determine what document numbers correspond to what paragraphs.
+This code is more space efficient, reading in the .paragraph file
+into memory as an accumulate docnum array.  Eg.  the .paragraph may contain
+
+	[5 3 6 4 7 9 4]
+
+indicating the first document has 5 paragraphs, the next 3, etc.
+This will be stored in memory as
+
+	[0 5 8 14 18 25 34 38]
+
+so a binary search can be performed.  The first 0 is for convenience;
+it prevents testing boundary conditions.
+
+
+The TREC_MODE code does this differently; it stores the array
+
+	[1 1 1 1 1 2 2 2 3 3 3 3 3 3 ....]
+
+allowing directy paragraph to docnum conversion, at the expense
+of memory.
+
+*/ 
+  qd->paragraph = NULL;
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      unsigned long magic;
+      FILE *paragraph;
+      int i;
+      char paraFile[512];
+
+      sprintf(paraFile, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+      paragraph = fopen(paraFile, "rb");
+      if (!paragraph)
+	FatalError(1, "Unable to open 'paraFile'.", paraFile);
+
+      fread((void *)&magic, sizeof(magic), 1, paragraph);
+      qd->paragraph = Xmalloc((qd->td->cth.num_of_docs+1)*sizeof(int));
+      qd->paragraph[0] = 0;
+      for (i = 1; i <= qd->td->cth.num_of_docs; i++)
+	{
+	  int count;
+
+	  if (fread((void *)&count, sizeof(count), 1, paragraph) != 1)
+	    FatalError(1, "Unexpected EOF while reading '%s'.", paraFile);
+	  NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  qd->paragraph[i] = qd->paragraph[i-1]+count;
+	}
+
+      fclose (paragraph); /* [RJM 07/98: Memory Leak] */
+    }
+  
+  
+#endif
+
+#ifdef TREC_MODE
+  {
+    extern char *trec_ids;
+    extern long *trec_paras;
+    int size;
+    char FileName[512];
+    FILE *f;
+    if (!strstr (qd->pathname, "trec"))
+      goto error;
+    sprintf (FileName, "%s%s", qd->pathname, ".DOCIDS");
+    if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+      {
+	Message ("Unable to open \"%s\"", FileName);
+	goto error;
+      }
+    fseek (f, 0, 2);
+    size = ftell (f);
+    fseek (f, 0, 0);
+    trec_ids = Xmalloc (size);
+    if (!trec_ids)
+      {
+	fclose (f);
+	goto error;
+      }
+    fread (trec_ids, 1, size, f);
+    fclose (f);
+    if (qd->id->ifh.InvfLevel == 3)
+      {
+	int i, d;
+	unsigned long magic;
+	trec_paras = Xmalloc (qd->sd->sdh.num_of_docs * sizeof (long));
+	if (!trec_paras)
+	  {
+	    Xfree (trec_ids);
+	    trec_ids = NULL;
+	    goto error;
+	  }
+	sprintf (FileName, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+	if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	  {
+	    Message ("Unable to open \"%s\"", FileName);
+	    goto error;
+	  }
+	if (fread ((char *) &magic, sizeof (magic), 1, f) != 1 ||
+	    NTOHUL(magic) != MAGIC_PARAGRAPH)  /* [RPAP - Jan 97: Endian Ordering] */
+	  {
+	    fclose (f);
+	    Message ("Bad magic number in \"%s\"", FileName);
+	    goto error;
+	  }
+
+	for (d = i = 0; i < qd->td->cth.num_of_docs; i++)
+	  {
+	    int count;
+	    if (fread ((char *) &count, sizeof (count), 1, f) != 1)
+	      {
+		fclose (f);
+		goto error;
+	      }
+	    NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	    while (count--)
+	      trec_paras[d++] = i;
+	  }
+	fclose (f);
+      }
+    goto ok;
+  error:
+    if (trec_ids)
+      Xfree (trec_ids);
+    if (trec_paras)
+      Xfree (trec_paras);
+    trec_ids = NULL;
+    trec_paras = NULL;
+  ok:
+    ;
+  }
+#endif
+
+  if (iqt)
+    GetTime (&iqt->Text);
+
+  return (qd);
+}
+
+
+
+
+
+
+/*
+ * Change the amount of memory currently in use
+ *
+ */
+void 
+ChangeMemInUse (query_data * qd, long delta)
+{
+  qd->mem_in_use += delta;
+  if (qd->mem_in_use > qd->max_mem_in_use)
+    qd->max_mem_in_use = qd->mem_in_use;
+}
+
+
+void 
+FinishQuerySystem (query_data * qd)
+{
+/* [RJM 07/98: Memory Leak] */
+#if defined(PARADOCNUM) || defined(NZDL)
+  if (qd->paragraph != NULL) {
+    Xfree (qd->paragraph);
+    qd->paragraph = NULL;
+  }
+#endif
+
+  FreeTextData (qd->td);
+  FreeInvfData (qd->id);
+  FreeCompDict (qd->cd);
+  if (qd->awd)
+    FreeWeights (qd->awd);
+  FreeStemDict (qd->sd);
+  close_all_files (qd);
+  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+  Xfree (qd->pathname);
+  FreeQueryDocs (qd);
+  if (qd->TL != NULL) FreeTermList(&qd->TL);         /* [RJM 07/98: Memory Leak] */
+  if (qd->QTL != NULL) FreeQueryTermList(&qd->QTL);  /* [RJM 07/98: Memory Leak] */
+  Xfree (qd);
+
+  /* other global stuff hanging around */
+  MgErrorDeinit ();
+}
+
+
+void 
+ResetFileStats (query_data * qd)
+{
+  ZeroFileStats (qd->File_text);
+  if (qd->File_comp_dict)
+    ZeroFileStats (qd->File_comp_dict);
+  if (qd->File_fast_comp_dict)
+    ZeroFileStats (qd->File_fast_comp_dict);
+  ZeroFileStats (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    ZeroFileStats (qd->File_stem1);
+  if (qd->File_stem2)
+    ZeroFileStats (qd->File_stem2);
+  if (qd->File_stem3)
+    ZeroFileStats (qd->File_stem3);
+
+  ZeroFileStats (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    ZeroFileStats (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    ZeroFileStats (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    ZeroFileStats (qd->File_text_idx);
+}
+
+
+void 
+TransFileStats (query_data * qd)
+{
+  qd->File_text->Current = qd->File_text->Cumulative;
+  if (qd->File_comp_dict)
+    qd->File_comp_dict->Current = qd->File_comp_dict->Cumulative;
+  if (qd->File_fast_comp_dict)
+    qd->File_fast_comp_dict->Current = qd->File_fast_comp_dict->Cumulative;
+  qd->File_stem->Current = qd->File_stem->Cumulative;
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    qd->File_stem1->Current = qd->File_stem1->Cumulative;
+  if (qd->File_stem2)
+    qd->File_stem2->Current = qd->File_stem2->Cumulative;
+  if (qd->File_stem3)
+    qd->File_stem3->Current = qd->File_stem3->Cumulative;
+
+  qd->File_invf->Current = qd->File_invf->Cumulative;
+  if (qd->File_text_idx_wgt)
+    qd->File_text_idx_wgt->Current = qd->File_text_idx_wgt->Cumulative;
+  if (qd->File_weight_approx)
+    qd->File_weight_approx->Current = qd->File_weight_approx->Cumulative;
+  if (qd->File_text_idx)
+    qd->File_text_idx->Current = qd->File_text_idx->Cumulative;
+}
+
+
+void 
+FreeTextBuffer (query_data * qd)
+{
+  if (qd->TextBuffer)
+    {
+      Xfree (qd->TextBuffer);
+      ChangeMemInUse (qd, -qd->TextBufferLen);
+    }
+  qd->TextBuffer = NULL;
+  qd->TextBufferLen = 0;
+}
+
+void 
+FreeQueryDocs (query_data * qd)
+{
+  qd->doc_pos = 0;
+  qd->buf_in_use = 0;
+  if (qd->DL)
+    {
+      int i;
+      for (i = 0; i < qd->DL->num; i++)
+	if (qd->DL->DE[i].CompTextBuffer)
+	  {
+	    Xfree (qd->DL->DE[i].CompTextBuffer);
+	    qd->DL->DE[i].CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -qd->DL->DE[i].Len);
+	  }
+      Xfree (qd->DL);
+    }
+  qd->DL = NULL;
+  FreeTextBuffer (qd);
+}
+
+int 
+LoadCompressedText (query_data * qd, int max_mem)
+{
+  DocEntry *DE;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+  if (!DE->CompTextBuffer)
+    {
+      int i;
+      DocEntry *de;
+      for (i = 0, de = qd->DL->DE; i < qd->DL->num; i++, de++)
+	if (de->CompTextBuffer)
+	  {
+	    Xfree (de->CompTextBuffer);
+	    de->CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -de->Len);
+	  }
+      if (LoadBuffers (qd, &qd->DL->DE[qd->doc_pos], max_mem,
+		       qd->DL->num - qd->doc_pos) == -1)
+	return -1;
+    }
+  return 0;
+}
+
+int 
+GetDocNum (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].DocNum;
+}
+
+DocEntry *
+GetDocChain (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+  return &(qd->DL->DE[qd->doc_pos]);
+}
+
+float 
+GetDocWeight (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Weight;
+}
+
+long 
+GetDocCompLength (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Len;
+}
+
+
+u_char *
+GetDocText (query_data * qd, unsigned long *len)
+{
+  DocEntry *DE;
+  int ULen;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+
+  if (!DE->CompTextBuffer)
+    {
+      fprintf (stderr, "The compressed text buffer is NULL\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  FreeTextBuffer (qd);
+
+  qd->TextBufferLen = (int) (qd->td->cth.ratio * 1.01 *
+			     DE->Len) + 100;
+  if (!(qd->TextBuffer = Xmalloc (qd->TextBufferLen)))
+    {
+      fprintf (stderr, "No memory for TextBuffer\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  DecodeText (qd->cd, (u_char *) (DE->CompTextBuffer), DE->Len,
+	      (u_char *) (qd->TextBuffer), &ULen);
+  qd->TextBuffer[ULen] = '\0';
+
+  if (ULen >= qd->TextBufferLen)
+    {
+      fprintf (stderr, "%d >= %d\n", ULen, qd->TextBufferLen);
+      mg_errno = MG_BUFTOOSMALL;
+      return (NULL);
+    }
+
+  if (len)
+    *len = ULen;
+
+  return qd->TextBuffer;
+}
+
+int 
+NextDoc (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return 0;
+  qd->doc_pos++;
+  return qd->doc_pos < qd->DL->num;
+}
Index: /main/tags/3.01/indexers/mg/src/text/backend.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/backend.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/backend.h	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * backend.h -- Underlying routines and datastructures for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef BACKEND_H
+#define BACKEND_H
+
+#include "sysfuncs.h"
+
+#include "timing.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	/* N parameter for decoding inverted file entries */
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+     *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    u_char *word_data[2];
+    u_char **words[2];
+    int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+						   for the hybrid methods */
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    u_char ***values[2];
+    u_char *escape[2];
+    huff_data *chars_huff[2];
+    u_long **chars_vals[2];
+    huff_data *lens_huff[2];
+    u_long **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx   /* [RPAP - Jan 97: Stem Index Change] */
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemDict;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+
+typedef struct RankedQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;           /* use exact weights for ranking or not */
+    long MaxDocsToRetrieve;	/* may be -1 for all */
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		/* 'A' = array,  'S' = splay tree,  'H' = hash_table */
+    long MaxAccums;		/* may be -1 for all */
+    long MaxTerms;		/* may be -1 for all */
+    int StopAtMaxAccum; /* Stop at maximum accumulator or not */
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo
+  {
+    long MaxDocsToRetrieve;
+  }
+BooleanQueryInfo;
+
+
+/* [TS:24/Aug/94] - maximum number of characters in term string */
+#define MAXTERMSTRLEN 1023
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+#if defined(PARADOCNUM) || defined(NZDL)
+    int *paragraph;
+#endif
+    char *pathname;
+    char *textpathname; /* [RJM 06/97: text filename] */
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		/* [TS:Oct/94] - so term list for query can easily be accessed */
+    u_char *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;    /* [RPAP - Feb 97: Term Frequency] */
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+/* [RJM 06/97: text filename] */
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void FreeTextBuffer (query_data * qd);
+
+void FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+u_char *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/bool_optimiser.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_optimiser.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_optimiser.c	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994,95  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:20  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1995/07/27  04:54:55  tes
+   * Committing optimiser before I add the code
+   * to find diff-nodes in any kind of tree.
+   *
+   * Revision 1.3  1994/10/20  00:02:55  tes
+   * Fixed bug with bool_optimiser:
+   *   - it couldn't handle "false | false"
+   *   - it couldn't handle "true & true"
+   * These would collapse into an empty list and cause an error.
+   *
+   * Revision 1.2  1994/10/18  06:11:02  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:28  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bool_tree.h"
+#include "term_lists.h"
+#include "messages.h"
+
+typedef int (*CompFunc) (const void *, const void *);
+typedef void (*TreeFunc) (bool_tree_node *);
+
+/* --- routines --- */
+static void PermeateNots (bool_tree_node * tree);
+static void DoubleNeg (bool_tree_node * not_tree);
+static void AndDeMorgan (bool_tree_node * not_tree);
+static void OrDeMorgan (bool_tree_node * not_tree);
+static int AndDistribute (bool_tree_node * tree);
+static void DNF_Distribute (bool_tree_node * tree);
+static void ReplaceWithChildren (bool_tree_node * tree);
+static void CollapseTree (bool_tree_node * tree);
+static void AndTrueFalse (bool_tree_node * and_tree);
+static void OrTrueFalse (bool_tree_node * or_tree);
+static void TF_Idempotent (bool_tree_node * tree);
+static void AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func);
+static void DiffDNF (bool_tree_node * or_node);
+static void DiffTree (bool_tree_node * tree);
+static void AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp);
+static void AndOrSort (bool_tree_node * tree, TermList * tl);
+static void AndSortDNF (bool_tree_node * or_node, TermList * tl);
+static void MarkOrTerms (bool_tree_node * tree, TermList * tl);
+static int calc_sum_ft (bool_tree_node * term_tree, TermList * tl);
+static int AndOrSort_comp (const void *A, const void *B);
+static int AndSort_comp (const void *A, const void *B);
+
+/* =========================================================================
+ * Function: OptimiseBoolTree
+ * Description: 
+ *      For case 2:
+ *        Do three major steps:
+ *        (i) put into standard form
+ *            -> put into DNF (disjunctive normal form - or of ands)
+ *            Use DeMorgan's, Double-negative, Distributive rules
+ *        (ii) simplify
+ *             apply idempotency rules
+ *        (iii) ameliorate
+ *              convert &! to diff nodes, order terms by frequency,...
+ *     Could also do the matching idempotency laws i.e. ...
+ *     (A | A), (A | !A), (A & !A), (A & A), (A & (A | B)), (A | (A & B)) 
+ *     Job for future.... ;-) 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+OptimiseBoolTree (bool_tree_node * parse_tree, TermList * tl, int type)
+{
+  switch (type)
+    {
+    case 1:
+      /* --- sorts for and of or-terms --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      CollapseTree (parse_tree);
+      DiffTree (parse_tree);
+      MarkOrTerms (parse_tree, tl);
+      AndOrSort (parse_tree, tl);
+      break;
+    case 2:
+      /* --- put into DNF --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      DNF_Distribute (parse_tree);
+      CollapseTree (parse_tree);
+      AndSortDNF (parse_tree, tl);
+      DiffDNF (parse_tree);
+      break;
+    default:
+      /* --- do nothing --- */
+      break;
+    }				/*switch */
+}				/*OptimiseBoolTree */
+
+/* =========================================================================
+ * Function: DoubleNeg
+ * Description: 
+ *      !(!(a) = a
+ *      Assumes binary tree.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+DoubleNeg (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *sibling = BOOL_SIBLING (not_tree);
+
+  /* copy a into not tree */
+  bcopy (a, not_tree, sizeof (bool_tree_node));
+
+  BOOL_SIBLING (not_tree) = sibling;
+
+  Xfree (child);
+  Xfree (a);
+
+}
+
+/* =========================================================================
+ * Function: AndDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'and'  i.e. !(a & b) <=> (!a) | (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+AndDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'or' */
+  BOOL_TAG (not_tree) = N_or;
+
+  /* change 'and' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: OrDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'or' i.e. !(a | b) <=> (!a) & (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+OrDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'and' */
+  BOOL_TAG (not_tree) = N_and;
+
+  /* change 'or' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: PermeateNots
+ * Description: 
+ *      Use DeMorgan's and Double-negative
+ *      Assumes tree in binary form (i.e. No ands/ors collapsed)
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PermeateNots (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if (BOOL_TAG (tree) == N_not)
+    {
+      switch (BOOL_TAG (BOOL_CHILD (tree)))
+	{
+	case N_not:
+	  DoubleNeg (tree);
+/** NOTE: I do not go on to do the children **/
+	  /* I permeate the nots on the tree here & leave it at that */
+	  /* In this case I do not know what type 'tree' will end up */
+	  PermeateNots (tree);
+	  return;
+	case N_and:
+	  AndDeMorgan (tree);
+	  break;
+	case N_or:
+	  OrDeMorgan (tree);
+	  break;
+	default:
+	  /* leave alone */
+	  break;
+	}
+    }
+
+  /* Permeate Nots for children */
+  if (BOOL_HAS_CHILD (tree))
+    {
+      PermeateNots (BOOL_CHILD (tree));
+      PermeateNots (BOOL_SIBLING (BOOL_CHILD (tree)));
+    }
+
+}
+
+/* =========================================================================
+ * Function: DNF_Distribute
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+#define ITER_LIMIT 500		/* if reach limit then probably in trouble */
+
+static void
+DNF_Distribute (bool_tree_node * tree)
+{
+  int i = 0;
+
+  /* multiple passes until no change */
+  /* not very smart :-( */
+  while (1)
+    {
+      if (AndDistribute (tree) == 0)
+	return;
+      if (i == ITER_LIMIT)
+	FatalError (1, "DNF infinite loop");
+      i++;
+    }				/*while */
+
+}
+
+/* =========================================================================
+ * Function: AndDistribute
+ * Description: 
+ *      (a | b) & A <=> (a & A) | (b & A)
+ * Input: 
+ *      binary tree of "AND" , "OR"s.
+ * Output: 
+ *      return 1 if changed the tree
+ *      return 0 if there was NO change (no distributive rule to apply)
+ * ========================================================================= */
+
+static int
+AndDistribute (bool_tree_node * tree)
+{
+  if (!tree)
+    return 0;
+
+  if (BOOL_TAG (tree) != N_and)
+    {
+      /* try the children if got some */
+      if (BOOL_TAG (tree) == N_or)
+	{
+	  bool_tree_node *left_child = BOOL_CHILD (tree);
+	  int left = AndDistribute (left_child);
+	  int right = AndDistribute (BOOL_SIBLING (left_child));
+	  return (left || right);
+	}
+
+      /* no children, then no change */
+      return 0;
+    }
+
+  /* AND case */
+  {
+    /* we have to check the children to see if can distribute */
+    bool_tree_node *left_child = BOOL_CHILD (tree);
+    bool_tree_node *right_child = BOOL_SIBLING (left_child);
+    int swap = 0;
+
+    /* If "or" on the right side then do a swap */
+    /* This is done for code simplicity */
+    if (BOOL_TAG (right_child) == N_or)
+      {
+	/* do a swap */
+	bool_tree_node *temp;
+	temp = right_child;
+	right_child = left_child;
+	left_child = temp;
+
+	BOOL_SIBLING (left_child) = right_child;
+	BOOL_SIBLING (right_child) = NULL;
+	BOOL_CHILD (tree) = left_child;
+
+	swap = 1;
+      }
+
+    /* do the actual distribution */
+    if (swap || BOOL_TAG (left_child) == N_or)
+      {
+	bool_tree_node *A = right_child;
+	bool_tree_node *A_copy = CopyBoolTree (A);
+	bool_tree_node *a = BOOL_CHILD (left_child);
+	bool_tree_node *b = BOOL_SIBLING (a);
+
+	/* then can distribute away ... */
+	BOOL_TAG (tree) = N_or;
+	BOOL_TAG (left_child) = N_and;
+	BOOL_SIBLING (left_child) = CreateBoolTreeNode (N_and, b, A);
+	BOOL_SIBLING (a) = A_copy;
+
+	AndDistribute (tree);
+	return 1;
+      }
+
+    /* if "and" of something other than "or" s then try the children */
+    {
+      int left = AndDistribute (left_child);
+      int right = AndDistribute (right_child);
+      return (left || right);
+    }
+  }				/* AND case */
+
+}				/*AndDistribute */
+
+/* =========================================================================
+ * Function: CollapseTree
+ * Description: 
+ *      Collapse "and" and "or"s.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+CollapseTree (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if ((BOOL_TAG (tree) == N_and) || (BOOL_TAG (tree) == N_or))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  while (BOOL_TAG (child) == BOOL_TAG (tree))
+	    {
+	      ReplaceWithChildren (child);
+	    }
+	  CollapseTree (child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+  /* assume "not"s permeated */
+  return;
+}
+
+/* =========================================================================
+ * Function: ReplaceWithChildren
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+ReplaceWithChildren (bool_tree_node * tree)
+{
+  bool_tree_node *next = BOOL_SIBLING (tree);
+  bool_tree_node *child = BOOL_CHILD (tree);
+  bool_tree_node *last = LastBoolSibling (child);
+
+  bcopy ((char *) child, (char *) tree, sizeof (bool_tree_node));
+  Xfree (child);
+  BOOL_SIBLING (last) = next;
+
+}
+
+/* =========================================================================
+ * Function: TF_Idempotent
+ * Description: 
+ *      Applies Idempotency laws which refer to True or False (hence "TF")
+ *      uses the true or false idempotent rules
+ * Input: 
+ *      operates on n-ary trees (not restricted to binary)
+ * Output: 
+ * ========================================================================= */
+
+static void
+TF_Idempotent (bool_tree_node * tree)
+{
+  bool_tree_node *child = NULL;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	{
+	  child = BOOL_CHILD (tree);
+	  TF_Idempotent (child);
+	}
+
+      /* --- process node --- */
+      switch (BOOL_TAG (tree))
+	{
+	case N_not:
+	  child = BOOL_CHILD (tree);
+
+	  /* if true then falsify */
+	  if (BOOL_TAG (child) == N_all)
+	    {
+	      BOOL_TAG (tree) = N_none;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  /* if false then trueify */
+	  if (BOOL_TAG (child) == N_none)
+	    {
+	      BOOL_TAG (tree) = N_all;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  break;
+	case N_and:
+	  AndTrueFalse (tree);
+	  break;
+	case N_or:
+	  OrTrueFalse (tree);
+	  break;
+
+	default:
+	  /* do nothing */
+	  break;
+	}			/*switch */
+
+      tree = BOOL_SIBLING (tree);
+    }				/*while */
+}				/*TF_Idempotent */
+
+/* =========================================================================
+ * Function: AndTrueFalse
+ * Description: 
+ *      If any "false" then falsify whole tree 
+ *      If any "true" then delete them
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndTrueFalse (bool_tree_node * and_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_none)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (and_tree)));
+	  BOOL_TAG (and_tree) = N_none;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_all)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+
+	  /* --- link over child --- */
+	  if (!prev)
+	    {
+	      BOOL_CHILD (and_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+
+	  Xfree (child);
+	  child = sibling;
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (and_tree);
+
+  /* if no children ... */
+  /* => all have been true */
+  if (!child)
+    {
+      BOOL_TAG (and_tree) = N_all;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (and_tree);
+      bcopy ((char *) child, (char *) and_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (and_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: OrTrueFalse
+ * Description: 
+ *      If any "true" then trueify whole tree 
+ *      If any "false" then delete them 
+ *      ***** NOTE: This is a reflection of AndTrueFalse *****
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+OrTrueFalse (bool_tree_node * or_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (or_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_all)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (or_tree)));
+	  BOOL_TAG (or_tree) = N_all;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_none)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+	  if (!prev)
+	    {
+	      BOOL_CHILD (or_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+	  Xfree (child);
+	  child = sibling;
+
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (or_tree);
+
+  /* if no children ... */
+  /* => all have been false */
+  if (!child)
+    {
+      BOOL_TAG (or_tree) = N_none;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (or_tree);
+      bcopy ((char *) child, (char *) or_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (or_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: AndNotToDiff
+ * Description: 
+ *      Convert an "And" tree with possibly some negated terms into
+ *      a "And - neg-terms". 
+ * Input: 
+ *      Expects "And" of only literals - as tree should be in DNF form.
+ *      This means that no recursion is necessary.
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func)
+{
+
+  /* -------------------------------------------------- */
+  /* --- Divide the children up into positive and negative literals --- */
+  /* Preserves ordering of children */
+  bool_tree_node *pos = NULL;
+  bool_tree_node *neg = NULL;
+  bool_tree_node *last_pos = NULL;
+  bool_tree_node *last_neg = NULL;
+
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+  while (child)
+    {
+      bool_tree_node *next = BOOL_SIBLING (child);
+      if (BOOL_TAG (child) == N_not)
+	{
+	  /* add to end of neg list */
+	  if (!neg)
+	    neg = child;
+	  else
+	    BOOL_SIBLING (last_neg) = child;
+	  last_neg = child;
+	}
+      else
+	{
+	  /* add to end of pos list */
+	  if (!pos)
+	    pos = child;
+	  else
+	    BOOL_SIBLING (last_pos) = child;
+	  last_pos = child;
+	}
+      child = next;
+    }				/*while */
+  if (last_pos)
+    BOOL_SIBLING (last_pos) = NULL;
+  if (last_neg)
+    BOOL_SIBLING (last_neg) = NULL;
+  /* -------------------------------------------------- */
+
+  if (!neg)			/* no negtives => no subtraction */
+    {
+      /* fix up parent's child link */
+      BOOL_CHILD (and_tree) = pos;
+      return;
+    }
+
+  if (!pos)			/* no positives = all negatives */
+    {
+      /* Choose one of the negs for the left-hand-side of the subtraction */
+      /* In particular, choose the first */
+      pos = neg;
+      neg = BOOL_SIBLING (neg);
+      BOOL_SIBLING (pos) = NULL;
+    }
+
+  /* Get rid of all '!' from not nodes */
+  /* Make a list out of the children */
+  {
+    bool_tree_node *n = neg;
+    bool_tree_node *next = NULL;
+
+    neg = BOOL_CHILD (neg);
+    while (n)
+      {
+	bool_tree_node *c = BOOL_CHILD (n);
+	next = BOOL_SIBLING (n);
+	Xfree (n);
+	if (next)
+	  BOOL_SIBLING (c) = BOOL_CHILD (next);
+	n = next;
+      }
+  }
+
+  /* --- restructure the tree with "and" and "diff" --- */
+  BOOL_TAG (and_tree) = N_diff;
+  if (!BOOL_SIBLING (pos))	/* i.e. only one pos */
+    {
+      /* (A & !B & !C & !D & ...) ==> A - (B,C,D,...) */
+
+      BOOL_CHILD (and_tree) = pos;
+      BOOL_SIBLING (pos) = neg;
+    }
+  else
+    {
+      /* (A & !C & B & !D & ...) ==> (A & B & ....) - (C, D, ...) */
+
+      bool_tree_node *n = CreateBoolNode (N_and);
+      BOOL_CHILD (and_tree) = n;
+      BOOL_CHILD (n) = pos;
+      BOOL_SIBLING (n) = neg;
+    }				/*if */
+
+  if (recurs_func)
+    {
+      recurs_func (BOOL_CHILD (and_tree));
+    }
+
+}				/*AndNotToDiff */
+
+/* =========================================================================
+ * Function: DiffTree
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ *      Unlike DiffDNF, this routine will search throughout the tree for
+ *      "and"-nodes. It will call AndNotToNeg with this as its recursive 
+ *      routine.
+ * Date: 27/July/95
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+DiffTree (bool_tree_node * tree)
+{
+  bool_tree_node *t = tree;
+
+  while (t)
+    {
+      if (BOOL_TAG (t) == N_and)
+	{
+	  AndNotToDiff (t, DiffTree);
+	}
+      t = BOOL_SIBLING (t);
+    }				/*while */
+
+}				/*DiffTree */
+
+/* =========================================================================
+ * Function: DiffDNF
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ * Input: 
+ *      A DNF tree
+ * Output: 
+ *      A tree possibly with some diffs in it.
+ * ========================================================================= */
+
+static void
+DiffDNF (bool_tree_node * DNF_tree)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndNotToDiff (child, NULL);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndNotToDiff (DNF_tree, NULL);
+    }
+}
+
+/* =========================================================================
+ * Function: AndSort_comp 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+/* Module variable, "term_list",  used as parameter for comparison function */
+/* The comparison function only allows certain fixed parameters */
+static TermList *term_list;
+
+static int
+AndSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) != N_term)
+    return -1;
+
+  if (BOOL_TAG (*a) != N_term && BOOL_TAG (*b) == N_term)
+    return 1;
+
+  return 0;
+}				/*AndSort_comp */
+
+/* =========================================================================
+ * Function: AndSort
+ * Description: 
+ *      Sort the list of nodes by increasing doc_count 
+ *      Using some Neil Sharman code - pretty straight forward.
+ *      Note: not-terms are sent to the end of the list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp)
+{
+  bool_tree_node **list = NULL;
+  bool_tree_node *child = NULL;
+  int count = 0;
+  int i = 0;
+
+  if (BOOL_TAG (and_tree) != N_and)
+    return;
+
+  /* --- Convert linked list into an array for sorting --- */
+  child = BOOL_CHILD (and_tree);
+  count = 0;
+  while (child)
+    {
+      count++;
+      child = BOOL_SIBLING (child);
+    }
+
+  if (!(list = Xmalloc (sizeof (bool_tree_node *) * count)))
+    return;
+
+  child = BOOL_CHILD (and_tree);
+  for (i = 0; i < count; i++)
+    {
+      list[i] = child;
+      child = BOOL_SIBLING (child);
+    }
+  /* ----------------------------------------------------- */
+
+  term_list = tl;
+  qsort (list, count, sizeof (bool_tree_node *), comp);
+
+  /* --- Convert array back into linked list --- */
+  for (i = 0; i < count - 1; i++)
+    BOOL_SIBLING (list[i]) = list[i + 1];
+  BOOL_SIBLING (list[count - 1]) = NULL;
+  BOOL_CHILD (and_tree) = list[0];
+  Xfree (list);
+  /* ----------------------------------------------------- */
+
+}				/*AndSort */
+
+/* =========================================================================
+ * Function: AndSortDNF
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+static void
+AndSortDNF (bool_tree_node * DNF_tree, TermList * tl)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndSort (child, tl, AndSort_comp);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndSort (DNF_tree, tl, AndSort_comp);
+    }
+}
+
+/* =========================================================================
+ * Function: MarkOrTerms
+ * Description: 
+ *      Mark or-nodes whose children are all terms.
+ *      Also store the sum of the ft's of these nodes.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+MarkOrTerms (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_TAG (tree) == N_or)
+    {
+      int found_non_term = 0;
+      bool_tree_node *child = BOOL_CHILD (tree);
+
+      /* --- check to see if got all terms --- */
+      while (child)
+	{
+	  if (BOOL_TAG (child) != N_term)
+	    {
+	      found_non_term = 1;
+	    }
+	  if (BOOL_HAS_CHILD (tree))
+	    {
+	      MarkOrTerms (child, tl);
+	    }
+	  child = BOOL_SIBLING (child);
+	}			/*while */
+
+      if (!found_non_term)
+	{
+	  /* Must have all terms for the OR node */
+	  /* So change its tag and calculate its sum */
+	  BOOL_TAG (tree) = N_or_terms;
+	  BOOL_SUM_FT (tree) = calc_sum_ft (tree, tl);
+	}			/*if */
+      /* ------------------------------------- */
+
+    }				/*if OR node */
+
+  else if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  MarkOrTerms (child, tl);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+}				/*MarkOrTerms */
+
+/* =========================================================================
+ * Function: calc_sum_ft
+ * Description: 
+ *      Sums up the ft's of all the terms 
+ * Input: 
+ *      Assumes that the tree only contains terms in it 
+ * Output: 
+ *      the ft sum
+ * ========================================================================= */
+static int
+calc_sum_ft (bool_tree_node * term_tree, TermList * tl)
+{
+  int sum_ft = 0;
+  bool_tree_node *child = BOOL_CHILD (term_tree);
+  while (child)
+    {
+      sum_ft += GetNthWE (tl, BOOL_TERM (child))->doc_count;
+      child = BOOL_SIBLING (child);
+    }
+  return sum_ft;
+}
+
+/* =========================================================================
+ * Function: AndOrSort
+ * Description: 
+ *      This is like an AndSort of just terms. 
+ *      The difference is that we are now also 
+ *      considering sorting with <or of terms> by using the sum of f_t's.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+AndOrSort (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+
+      if (BOOL_TAG (tree) == N_and)
+	{
+	  AndSort (tree, tl, AndOrSort_comp);
+	}
+
+      /* Check out any children down the tree */
+      /* Probably a waste of time */
+      {
+	bool_tree_node *child = BOOL_CHILD (tree);
+	while (child)
+	  {
+	    AndOrSort (child, tl);
+	    child = BOOL_SIBLING (child);
+	  }
+      }				/*do the tree */
+
+    }				/*if has children */
+}				/*AndOrSort */
+
+/* =========================================================================
+ * Function: AndOrSort_comp
+ * Description: 
+ *      Comparison function used to order "term"s and "or_terms"s.
+ *      It's purpose is to be used by a sort routine.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int
+AndOrSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if ((BOOL_TAG (*a) == N_term || BOOL_TAG (*a) == N_or_terms) &&
+      (BOOL_TAG (*b) != N_term && BOOL_TAG (*b) != N_or_terms))
+    return -1;
+
+  if ((BOOL_TAG (*a) != N_term && BOOL_TAG (*a) != N_or_terms) &&
+      (BOOL_TAG (*b) == N_term || BOOL_TAG (*b) == N_or_terms))
+    return 1;
+
+  return 0;
+}				/*AndOrSort_comp */
Index: /main/tags/3.01/indexers/mg/src/text/bool_optimiser.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_optimiser.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_optimiser.h	(revision 21112)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:23  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:21  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.1  1994/10/20  03:56:32  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/10/18  06:11:03  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:29  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_OPTIMISER_H
+#define BOOL_OPTIMISER_H
+
+#include "bool_tree.h"
+#include "term_lists.h"
+
+void OptimiseBoolTree (bool_tree_node * parse_tree, TermList * term_list, int type);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/bool_parser.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_parser.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_parser.c	(revision 21112)
@@ -0,0 +1,1196 @@
+
+/*  A Bison parser, made from bool_parser.y
+ by  GNU Bison version 1.27
+  */
+
+#define YYBISON 1  /* Identify Bison output.  */
+
+#define	TERM	257
+
+#line 24 "bool_parser.y"
+
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+
+#line 66 "bool_parser.y"
+typedef union {
+  char *text;
+  bool_tree_node *node;
+} YYSTYPE;
+#include <stdio.h>
+
+#ifndef __cplusplus
+#ifndef __STDC__
+#define const
+#endif
+#endif
+
+
+
+#define	YYFINAL		20
+#define	YYFLAG		-32768
+#define	YYNTBASE	11
+
+#define YYTRANSLATE(x) ((unsigned)(x) <= 257 ? yytranslate[x] : 16)
+
+static const char yytranslate[] = {     0,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     8,     2,     2,     2,     2,     9,     2,     4,
+     5,     6,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     7,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,    10,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+     2,     2,     2,     2,     2,     1,     3
+};
+
+#if YYDEBUG != 0
+static const short yyprhs[] = {     0,
+     0,     2,     4,     8,    10,    12,    14,    17,    21,    24,
+    26,    30
+};
+
+static const short yyrhs[] = {    15,
+     0,     3,     0,     4,    15,     5,     0,     6,     0,     7,
+     0,    12,     0,     8,    13,     0,    14,     9,    13,     0,
+    14,    13,     0,    13,     0,    15,    10,    14,     0,    14,
+     0
+};
+
+#endif
+
+#if YYDEBUG != 0
+static const short yyrline[] = { 0,
+    76,    80,    81,    82,    83,    86,    87,    90,    91,    92,
+    95,    96
+};
+#endif
+
+
+#if YYDEBUG != 0 || defined (YYERROR_VERBOSE)
+
+static const char * const yytname[] = {   "$","error","$undefined.","TERM","'('",
+"')'","'*'","'_'","'!'","'&'","'|'","query","term","not","and","or", NULL
+};
+#endif
+
+static const short yyr1[] = {     0,
+    11,    12,    12,    12,    12,    13,    13,    14,    14,    14,
+    15,    15
+};
+
+static const short yyr2[] = {     0,
+     1,     1,     3,     1,     1,     1,     2,     3,     2,     1,
+     3,     1
+};
+
+static const short yydefact[] = {     0,
+     2,     0,     4,     5,     0,     6,    10,    12,     1,     0,
+     7,     0,     9,     0,     3,     8,    11,     0,     0,     0
+};
+
+static const short yydefgoto[] = {    18,
+     6,     7,     8,     9
+};
+
+static const short yypact[] = {    10,
+-32768,    10,-32768,-32768,    10,-32768,-32768,     2,    -9,    14,
+-32768,    10,-32768,    10,-32768,-32768,     2,     4,    15,-32768
+};
+
+static const short yypgoto[] = {-32768,
+-32768,    -5,   -12,    18
+};
+
+
+#define	YYLAST		24
+
+
+static const short yytable[] = {    11,
+    14,    17,    13,    19,     1,     2,    16,     3,     4,     5,
+    12,    13,     1,     2,    20,     3,     4,     5,    15,    10,
+     0,     0,     0,    14
+};
+
+static const short yycheck[] = {     5,
+    10,    14,     8,     0,     3,     4,    12,     6,     7,     8,
+     9,    17,     3,     4,     0,     6,     7,     8,     5,     2,
+    -1,    -1,    -1,    10
+};
+/* -*-C-*-  Note some compilers choke on comments on `#line' lines.  */
+#line 3 "/usr/lib/bison.simple"
+/* This file comes from bison-1.27.  */
+
+/* Skeleton output parser for bison,
+   Copyright (C) 1984, 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place - Suite 330,
+   Boston, MA 02111-1307, USA.  */
+
+/* As a special exception, when this file is copied by Bison into a
+   Bison output file, you may use that output file without restriction.
+   This special exception was added by the Free Software Foundation
+   in version 1.24 of Bison.  */
+
+/* This is the parser code that is written into each bison parser
+  when the %semantic_parser declaration is not specified in the grammar.
+  It was written by Richard Stallman by simplifying the hairy parser
+  used when %semantic_parser is specified.  */
+
+#ifndef YYSTACK_USE_ALLOCA
+#ifdef alloca
+#define YYSTACK_USE_ALLOCA
+#else /* alloca not defined */
+#ifdef __GNUC__
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#else /* not GNU C.  */
+#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386))
+#define YYSTACK_USE_ALLOCA
+#include <alloca.h>
+#else /* not sparc */
+/* We think this test detects Watcom and Microsoft C.  */
+/* This used to test MSDOS, but that is a bad idea
+   since that symbol is in the user namespace.  */
+#if (defined (_MSDOS) || defined (_MSDOS_)) && !defined (__TURBOC__)
+#if 0 /* No need for malloc.h, which pollutes the namespace;
+	 instead, just don't use alloca.  */
+#include <malloc.h>
+#endif
+#else /* not MSDOS, or __TURBOC__ */
+#if defined(_AIX)
+/* I don't know what this was needed for, but it pollutes the namespace.
+   So I turned it off.   rms, 2 May 1997.  */
+/* #include <malloc.h>  */
+ #pragma alloca
+#define YYSTACK_USE_ALLOCA
+#else /* not MSDOS, or __TURBOC__, or _AIX */
+#if 0
+#ifdef __hpux /* haible@ilog.fr says this works for HPUX 9.05 and up,
+		 and on HPUX 10.  Eventually we can turn this on.  */
+#define YYSTACK_USE_ALLOCA
+#define alloca __builtin_alloca
+#endif /* __hpux */
+#endif
+#endif /* not _AIX */
+#endif /* not MSDOS, or __TURBOC__ */
+#endif /* not sparc */
+#endif /* not GNU C */
+#endif /* alloca not defined */
+#endif /* YYSTACK_USE_ALLOCA not defined */
+
+#ifdef YYSTACK_USE_ALLOCA
+#define YYSTACK_ALLOC alloca
+#else
+#define YYSTACK_ALLOC malloc
+#endif
+
+/* Note: there must be only one dollar sign in this file.
+   It is replaced by the list of actions, each action
+   as one case of the switch.  */
+
+#define yyerrok		(yyerrstatus = 0)
+#define yyclearin	(yychar = YYEMPTY)
+#define YYEMPTY		-2
+#define YYEOF		0
+#define YYACCEPT	goto yyacceptlab
+#define YYABORT 	goto yyabortlab
+#define YYERROR		goto yyerrlab1
+/* Like YYERROR except do call yyerror.
+   This remains here temporarily to ease the
+   transition to the new meaning of YYERROR, for GCC.
+   Once GCC version 2 has supplanted version 1, this can go.  */
+#define YYFAIL		goto yyerrlab
+#define YYRECOVERING()  (!!yyerrstatus)
+#define YYBACKUP(token, value) \
+do								\
+  if (yychar == YYEMPTY && yylen == 1)				\
+    { yychar = (token), yylval = (value);			\
+      yychar1 = YYTRANSLATE (yychar);				\
+      YYPOPSTACK;						\
+      goto yybackup;						\
+    }								\
+  else								\
+    { yyerror ("syntax error: cannot back up"); YYERROR; }	\
+while (0)
+
+#define YYTERROR	1
+#define YYERRCODE	256
+
+#ifndef YYPURE
+#define YYLEX		yylex()
+#endif
+
+#ifdef YYPURE
+#ifdef YYLSP_NEEDED
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, &yylloc, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval, &yylloc)
+#endif
+#else /* not YYLSP_NEEDED */
+#ifdef YYLEX_PARAM
+#define YYLEX		yylex(&yylval, YYLEX_PARAM)
+#else
+#define YYLEX		yylex(&yylval)
+#endif
+#endif /* not YYLSP_NEEDED */
+#endif
+
+/* If nonreentrant, generate the variables here */
+
+#ifndef YYPURE
+
+int	yychar;			/*  the lookahead symbol		*/
+YYSTYPE	yylval;			/*  the semantic value of the		*/
+				/*  lookahead symbol			*/
+
+#ifdef YYLSP_NEEDED
+YYLTYPE yylloc;			/*  location data for the lookahead	*/
+				/*  symbol				*/
+#endif
+
+int yynerrs;			/*  number of parse errors so far       */
+#endif  /* not YYPURE */
+
+#if YYDEBUG != 0
+int yydebug;			/*  nonzero means print parse trace	*/
+/* Since this is uninitialized, it does not stop multiple parsers
+   from coexisting.  */
+#endif
+
+/*  YYINITDEPTH indicates the initial size of the parser's stacks	*/
+
+#ifndef	YYINITDEPTH
+#define YYINITDEPTH 200
+#endif
+
+/*  YYMAXDEPTH is the maximum size the stacks can grow to
+    (effective only if the built-in stack extension method is used).  */
+
+#if YYMAXDEPTH == 0
+#undef YYMAXDEPTH
+#endif
+
+#ifndef YYMAXDEPTH
+#define YYMAXDEPTH 10000
+#endif
+
+
+/* Define __yy_memcpy.  Note that the size argument
+   should be passed with type unsigned int, because that is what the non-GCC
+   definitions require.  With GCC, __builtin_memcpy takes an arg
+   of type size_t, but it can handle unsigned int.  */
+
+#if __GNUC__ > 1		/* GNU C and GNU C++ define this.  */
+#define __yy_memcpy(TO,FROM,COUNT)	__builtin_memcpy(TO,FROM,COUNT)
+#else				/* not GNU C or C++ */
+#ifndef __cplusplus
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (to, from, count)
+     char *to;
+     char *from;
+     unsigned int count;
+{
+  register char *f = from;
+  register char *t = to;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#else /* __cplusplus */
+
+/* This is the most reliable way to avoid incompatibilities
+   in available built-in functions on various systems.  */
+static void
+__yy_memcpy (char *to, char *from, unsigned int count)
+{
+  register char *t = to;
+  register char *f = from;
+  register int i = count;
+
+  while (i-- > 0)
+    *t++ = *f++;
+}
+
+#endif
+#endif
+
+
+#line 216 "/usr/lib/bison.simple"
+
+/* The user can define YYPARSE_PARAM as the name of an argument to be passed
+   into yyparse.  The argument should have type void *.
+   It should actually point to an object.
+   Grammar actions can access the variable by casting it
+   to the proper pointer type.  */
+
+#ifdef YYPARSE_PARAM
+#ifdef __cplusplus
+#define YYPARSE_PARAM_ARG void *YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL
+#else /* not __cplusplus */
+#define YYPARSE_PARAM_ARG YYPARSE_PARAM
+#define YYPARSE_PARAM_DECL void *YYPARSE_PARAM;
+#endif /* not __cplusplus */
+#else /* not YYPARSE_PARAM */
+#define YYPARSE_PARAM_ARG
+#define YYPARSE_PARAM_DECL
+#endif /* not YYPARSE_PARAM */
+
+/* Prevent warning if -Wstrict-prototypes.  */
+#ifdef __GNUC__
+#ifdef YYPARSE_PARAM
+int yyparse (void *);
+#else
+int yyparse (void);
+#endif
+#endif
+
+int
+yyparse(YYPARSE_PARAM_ARG)
+     YYPARSE_PARAM_DECL
+{
+  register int yystate;
+  register int yyn;
+  register short *yyssp;
+  register YYSTYPE *yyvsp;
+  int yyerrstatus;	/*  number of tokens to shift before error messages enabled */
+  int yychar1 = 0;		/*  lookahead token as an internal (translated) token number */
+
+  short	yyssa[YYINITDEPTH];	/*  the state stack			*/
+  YYSTYPE yyvsa[YYINITDEPTH];	/*  the semantic value stack		*/
+
+  short *yyss = yyssa;		/*  refer to the stacks thru separate pointers */
+  YYSTYPE *yyvs = yyvsa;	/*  to allow yyoverflow to reallocate them elsewhere */
+
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylsa[YYINITDEPTH];	/*  the location stack			*/
+  YYLTYPE *yyls = yylsa;
+  YYLTYPE *yylsp;
+
+#define YYPOPSTACK   (yyvsp--, yyssp--, yylsp--)
+#else
+#define YYPOPSTACK   (yyvsp--, yyssp--)
+#endif
+
+  int yystacksize = YYINITDEPTH;
+  int yyfree_stacks = 0;
+
+#ifdef YYPURE
+  int yychar;
+  YYSTYPE yylval;
+  int yynerrs;
+#ifdef YYLSP_NEEDED
+  YYLTYPE yylloc;
+#endif
+#endif
+
+  YYSTYPE yyval;		/*  the variable used to return		*/
+				/*  semantic values from the action	*/
+				/*  routines				*/
+
+  int yylen;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Starting parse\n");
+#endif
+
+  yystate = 0;
+  yyerrstatus = 0;
+  yynerrs = 0;
+  yychar = YYEMPTY;		/* Cause a token to be read.  */
+
+  /* Initialize stack pointers.
+     Waste one element of value and location stack
+     so that they stay on the same level as the state stack.
+     The wasted elements are never initialized.  */
+
+  yyssp = yyss - 1;
+  yyvsp = yyvs;
+#ifdef YYLSP_NEEDED
+  yylsp = yyls;
+#endif
+
+/* Push a new state, which is found in  yystate  .  */
+/* In all cases, when you get here, the value and location stacks
+   have just been pushed. so pushing a state here evens the stacks.  */
+yynewstate:
+
+  *++yyssp = yystate;
+
+  if (yyssp >= yyss + yystacksize - 1)
+    {
+      /* Give user a chance to reallocate the stack */
+      /* Use copies of these so that the &'s don't force the real ones into memory. */
+      YYSTYPE *yyvs1 = yyvs;
+      short *yyss1 = yyss;
+#ifdef YYLSP_NEEDED
+      YYLTYPE *yyls1 = yyls;
+#endif
+
+      /* Get the current used size of the three stacks, in elements.  */
+      int size = yyssp - yyss + 1;
+
+#ifdef yyoverflow
+      /* Each stack pointer address is followed by the size of
+	 the data in use in that stack, in bytes.  */
+#ifdef YYLSP_NEEDED
+      /* This used to be a conditional around just the two extra args,
+	 but that might be undefined if yyoverflow is a macro.  */
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yyls1, size * sizeof (*yylsp),
+		 &yystacksize);
+#else
+      yyoverflow("parser stack overflow",
+		 &yyss1, size * sizeof (*yyssp),
+		 &yyvs1, size * sizeof (*yyvsp),
+		 &yystacksize);
+#endif
+
+      yyss = yyss1; yyvs = yyvs1;
+#ifdef YYLSP_NEEDED
+      yyls = yyls1;
+#endif
+#else /* no yyoverflow */
+      /* Extend the stack our own way.  */
+      if (yystacksize >= YYMAXDEPTH)
+	{
+	  yyerror("parser stack overflow");
+	  if (yyfree_stacks)
+	    {
+	      free (yyss);
+	      free (yyvs);
+#ifdef YYLSP_NEEDED
+	      free (yyls);
+#endif
+	    }
+	  return 2;
+	}
+      yystacksize *= 2;
+      if (yystacksize > YYMAXDEPTH)
+	yystacksize = YYMAXDEPTH;
+#ifndef YYSTACK_USE_ALLOCA
+      yyfree_stacks = 1;
+#endif
+      yyss = (short *) YYSTACK_ALLOC (yystacksize * sizeof (*yyssp));
+      __yy_memcpy ((char *)yyss, (char *)yyss1,
+		   size * (unsigned int) sizeof (*yyssp));
+      yyvs = (YYSTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yyvsp));
+      __yy_memcpy ((char *)yyvs, (char *)yyvs1,
+		   size * (unsigned int) sizeof (*yyvsp));
+#ifdef YYLSP_NEEDED
+      yyls = (YYLTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yylsp));
+      __yy_memcpy ((char *)yyls, (char *)yyls1,
+		   size * (unsigned int) sizeof (*yylsp));
+#endif
+#endif /* no yyoverflow */
+
+      yyssp = yyss + size - 1;
+      yyvsp = yyvs + size - 1;
+#ifdef YYLSP_NEEDED
+      yylsp = yyls + size - 1;
+#endif
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Stack size increased to %d\n", yystacksize);
+#endif
+
+      if (yyssp >= yyss + yystacksize - 1)
+	YYABORT;
+    }
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Entering state %d\n", yystate);
+#endif
+
+  goto yybackup;
+ yybackup:
+
+/* Do appropriate processing given the current state.  */
+/* Read a lookahead token if we need one and don't already have one.  */
+/* yyresume: */
+
+  /* First try to decide what to do without reference to lookahead token.  */
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yydefault;
+
+  /* Not known => get a lookahead token if don't already have one.  */
+
+  /* yychar is either YYEMPTY or YYEOF
+     or a valid token in external form.  */
+
+  if (yychar == YYEMPTY)
+    {
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Reading a token: ");
+#endif
+      yychar = YYLEX;
+    }
+
+  /* Convert token to internal form (in yychar1) for indexing tables with */
+
+  if (yychar <= 0)		/* This means end of input. */
+    {
+      yychar1 = 0;
+      yychar = YYEOF;		/* Don't call YYLEX any more */
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Now at end of input.\n");
+#endif
+    }
+  else
+    {
+      yychar1 = YYTRANSLATE(yychar);
+
+#if YYDEBUG != 0
+      if (yydebug)
+	{
+	  fprintf (stderr, "Next token is %d (%s", yychar, yytname[yychar1]);
+	  /* Give the individual parser a way to print the precise meaning
+	     of a token, for further debugging info.  */
+#ifdef YYPRINT
+	  YYPRINT (stderr, yychar, yylval);
+#endif
+	  fprintf (stderr, ")\n");
+	}
+#endif
+    }
+
+  yyn += yychar1;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != yychar1)
+    goto yydefault;
+
+  yyn = yytable[yyn];
+
+  /* yyn is what to do for this token type in this state.
+     Negative => reduce, -yyn is rule number.
+     Positive => shift, yyn is new state.
+       New state is final state => don't bother to shift,
+       just return success.
+     0, or most negative number => error.  */
+
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrlab;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrlab;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+  /* Shift the lookahead token.  */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting token %d (%s), ", yychar, yytname[yychar1]);
+#endif
+
+  /* Discard the token being shifted unless it is eof.  */
+  if (yychar != YYEOF)
+    yychar = YYEMPTY;
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  /* count tokens shifted since error; after three, turn off error status.  */
+  if (yyerrstatus) yyerrstatus--;
+
+  yystate = yyn;
+  goto yynewstate;
+
+/* Do the default action for the current state.  */
+yydefault:
+
+  yyn = yydefact[yystate];
+  if (yyn == 0)
+    goto yyerrlab;
+
+/* Do a reduction.  yyn is the number of a rule to reduce with.  */
+yyreduce:
+  yylen = yyr2[yyn];
+  if (yylen > 0)
+    yyval = yyvsp[1-yylen]; /* implement default value of the action */
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      int i;
+
+      fprintf (stderr, "Reducing via rule %d (line %d), ",
+	       yyn, yyrline[yyn]);
+
+      /* Print the symbols being reduced, and their result.  */
+      for (i = yyprhs[yyn]; yyrhs[i] > 0; i++)
+	fprintf (stderr, "%s ", yytname[yyrhs[i]]);
+      fprintf (stderr, " -> %s\n", yytname[yyr1[yyn]]);
+    }
+#endif
+
+
+  switch (yyn) {
+
+case 1:
+#line 76 "bool_parser.y"
+{ tree_base = yyvsp[0].node;;
+    break;}
+case 2:
+#line 80 "bool_parser.y"
+{ yyval.node = CreateBoolTermNode(term_list, yyvsp[0].text, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); ;
+    break;}
+case 3:
+#line 81 "bool_parser.y"
+{ yyval.node = yyvsp[-1].node; ;
+    break;}
+case 4:
+#line 82 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_all, NULL, NULL); ;
+    break;}
+case 5:
+#line 83 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_none, NULL, NULL); ;
+    break;}
+case 7:
+#line 87 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_not, yyvsp[0].node, NULL); ;
+    break;}
+case 8:
+#line 90 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+case 9:
+#line 91 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-1].node, yyvsp[0].node); ;
+    break;}
+case 11:
+#line 95 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_or, yyvsp[-2].node, yyvsp[0].node); ;
+    break;}
+}
+   /* the action file gets copied in in place of this dollarsign */
+#line 542 "/usr/lib/bison.simple"
+
+
+  yyvsp -= yylen;
+  yyssp -= yylen;
+#ifdef YYLSP_NEEDED
+  yylsp -= yylen;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+  *++yyvsp = yyval;
+
+#ifdef YYLSP_NEEDED
+  yylsp++;
+  if (yylen == 0)
+    {
+      yylsp->first_line = yylloc.first_line;
+      yylsp->first_column = yylloc.first_column;
+      yylsp->last_line = (yylsp-1)->last_line;
+      yylsp->last_column = (yylsp-1)->last_column;
+      yylsp->text = 0;
+    }
+  else
+    {
+      yylsp->last_line = (yylsp+yylen-1)->last_line;
+      yylsp->last_column = (yylsp+yylen-1)->last_column;
+    }
+#endif
+
+  /* Now "shift" the result of the reduction.
+     Determine what state that goes to,
+     based on the state we popped back to
+     and the rule number reduced by.  */
+
+  yyn = yyr1[yyn];
+
+  yystate = yypgoto[yyn - YYNTBASE] + *yyssp;
+  if (yystate >= 0 && yystate <= YYLAST && yycheck[yystate] == *yyssp)
+    yystate = yytable[yystate];
+  else
+    yystate = yydefgoto[yyn - YYNTBASE];
+
+  goto yynewstate;
+
+yyerrlab:   /* here on detecting error */
+
+  if (! yyerrstatus)
+    /* If not already recovering from an error, report this error.  */
+    {
+      ++yynerrs;
+
+#ifdef YYERROR_VERBOSE
+      yyn = yypact[yystate];
+
+      if (yyn > YYFLAG && yyn < YYLAST)
+	{
+	  int size = 0;
+	  char *msg;
+	  int x, count;
+
+	  count = 0;
+	  /* Start X at -yyn if nec to avoid negative indexes in yycheck.  */
+	  for (x = (yyn < 0 ? -yyn : 0);
+	       x < (sizeof(yytname) / sizeof(char *)); x++)
+	    if (yycheck[x + yyn] == x)
+	      size += strlen(yytname[x]) + 15, count++;
+	  msg = (char *) malloc(size + 15);
+	  if (msg != 0)
+	    {
+	      strcpy(msg, "parse error");
+
+	      if (count < 5)
+		{
+		  count = 0;
+		  for (x = (yyn < 0 ? -yyn : 0);
+		       x < (sizeof(yytname) / sizeof(char *)); x++)
+		    if (yycheck[x + yyn] == x)
+		      {
+			strcat(msg, count == 0 ? ", expecting `" : " or `");
+			strcat(msg, yytname[x]);
+			strcat(msg, "'");
+			count++;
+		      }
+		}
+	      yyerror(msg);
+	      free(msg);
+	    }
+	  else
+	    yyerror ("parse error; also virtual memory exceeded");
+	}
+      else
+#endif /* YYERROR_VERBOSE */
+	yyerror("parse error");
+    }
+
+  goto yyerrlab1;
+yyerrlab1:   /* here on error raised explicitly by an action */
+
+  if (yyerrstatus == 3)
+    {
+      /* if just tried and failed to reuse lookahead token after an error, discard it.  */
+
+      /* return failure if at end of input */
+      if (yychar == YYEOF)
+	YYABORT;
+
+#if YYDEBUG != 0
+      if (yydebug)
+	fprintf(stderr, "Discarding token %d (%s).\n", yychar, yytname[yychar1]);
+#endif
+
+      yychar = YYEMPTY;
+    }
+
+  /* Else will try to reuse lookahead token
+     after shifting the error token.  */
+
+  yyerrstatus = 3;		/* Each real token shifted decrements this */
+
+  goto yyerrhandle;
+
+yyerrdefault:  /* current state does not do anything special for the error token. */
+
+#if 0
+  /* This is wrong; only states that explicitly want error tokens
+     should shift them.  */
+  yyn = yydefact[yystate];  /* If its default is to accept any token, ok.  Otherwise pop it.*/
+  if (yyn) goto yydefault;
+#endif
+
+yyerrpop:   /* pop the current state because it cannot handle the error token */
+
+  if (yyssp == yyss) YYABORT;
+  yyvsp--;
+  yystate = *--yyssp;
+#ifdef YYLSP_NEEDED
+  yylsp--;
+#endif
+
+#if YYDEBUG != 0
+  if (yydebug)
+    {
+      short *ssp1 = yyss - 1;
+      fprintf (stderr, "Error: state stack now");
+      while (ssp1 != yyssp)
+	fprintf (stderr, " %d", *++ssp1);
+      fprintf (stderr, "\n");
+    }
+#endif
+
+yyerrhandle:
+
+  yyn = yypact[yystate];
+  if (yyn == YYFLAG)
+    goto yyerrdefault;
+
+  yyn += YYTERROR;
+  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != YYTERROR)
+    goto yyerrdefault;
+
+  yyn = yytable[yyn];
+  if (yyn < 0)
+    {
+      if (yyn == YYFLAG)
+	goto yyerrpop;
+      yyn = -yyn;
+      goto yyreduce;
+    }
+  else if (yyn == 0)
+    goto yyerrpop;
+
+  if (yyn == YYFINAL)
+    YYACCEPT;
+
+#if YYDEBUG != 0
+  if (yydebug)
+    fprintf(stderr, "Shifting error token, ");
+#endif
+
+  *++yyvsp = yylval;
+#ifdef YYLSP_NEEDED
+  *++yylsp = yylloc;
+#endif
+
+  yystate = yyn;
+  goto yynewstate;
+
+ yyacceptlab:
+  /* YYACCEPT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 0;
+
+ yyabortlab:
+  /* YYABORT comes here.  */
+  if (yyfree_stacks)
+    {
+      free (yyss);
+      free (yyvs);
+#ifdef YYLSP_NEEDED
+      free (yyls);
+#endif
+    }
+  return 1;
+}
+#line 99 "bool_parser.y"
+
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+ 
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/indexers/mg/src/text/bool_parser.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_parser.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_parser.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:43  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:40  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:26  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:33  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:31  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_PARSER_H
+#define BOOL_PARSER_H
+
+#include "backend.h"
+#include "term_lists.h"
+#include "invf.h"
+
+bool_tree_node *ParseBool (char *query_line, int query_len,
+			   TermList ** the_term_list, int the_stemmer_num, 
+			   int the_stem_method, int *res,
+			   stemmed_dict * the_sd, int is_indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			   QueryTermList ** the_query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/bool_parser.y
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_parser.y	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_parser.y	(revision 21112)
@@ -0,0 +1,316 @@
+/**************************************************************************
+ *
+ * bool_parser - boolean query parser
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+ 
+/**************************************************************************/
+%{
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+%}
+ 
+ 
+%union {
+  char *text;
+  bool_tree_node *node;
+}
+ 
+%token <text> TERM
+%type <node> query term not and or
+ 
+%%
+ 
+query: or  { tree_base = $1;}
+;
+ 
+ 
+term:     TERM  { $$ = CreateBoolTermNode(term_list, $1, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+        | '(' or ')' { $$ = $2; }
+        | '*' { $$ = CreateBoolTreeNode(N_all, NULL, NULL); }
+        | '_' { $$ = CreateBoolTreeNode(N_none, NULL, NULL); }
+;
+ 
+not:      term
+        | '!' not { $$ = CreateBoolTreeNode(N_not, $2, NULL); }
+;
+ 
+and:      and '&' not { $$ = CreateBoolTreeNode(N_and, $1, $3); }
+        | and not { $$ = CreateBoolTreeNode(N_and, $1, $2); }
+        | not
+;
+ 
+or:       or '|' and { $$ = CreateBoolTreeNode(N_or, $1, $3); }
+        | and
+;
+ 
+%%
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      buf_ptr = skipspace(buf_ptr, end);
+
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /main/tags/3.01/indexers/mg/src/text/bool_query.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_query.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_query.c	(revision 21112)
@@ -0,0 +1,353 @@
+/**************************************************************************
+ *
+ * bool_query.c -- Boolean query evaluation
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "sptree.h"
+
+
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "backend.h"
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "read_line.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "environment.h"
+
+
+/**********************
+  [RPAP - Feb 97: Term Frequency]
+
+  Shifted to bool_parser.y
+
+  *********************/
+
+/* =========================================================================
+ * Function: ReadInTermInfo
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+/**************************
+static void 
+ReadInTermInfo (query_data * qd)
+{
+  int i;
+  TermList *tl = qd->TL;
+
+  for (i = 0; i < tl->num; i++)
+    {
+      TermEntry *te = &(tl->TE[i]);
+      WordEntry *we = &(te->WE);
+
+       we->word_num = FindWord (qd->sd, te->Word,
+			       &we->count, &we->doc_count,
+			       &we->invf_ptr, &we->invf_len);
+      if (we->word_num == -1)
+	we->count = we->doc_count = 0;
+    }
+}
+
+***********************/
+
+
+
+/* =========================================================================
+ * Function: FindNoneTerms 
+ * Description: 
+ *      Set any terms which are not in dictionary to N_none (i.e. false)
+ *      Assumes a prior call to ReadInTermInfo
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FindNoneTerms (query_data * qd, bool_tree_node * tree)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  FindNoneTerms (qd, child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (tree) == N_term)
+    {
+      int doc_count = qd->TL->TE[BOOL_TERM (tree)].WE.doc_count;
+      if (!doc_count)
+	BOOL_TAG (tree) = N_none;
+    }
+
+}				/*FindNoneTerms */
+
+/* =========================================================================
+ * Function: ReduceList
+ * Description: 
+ *      Go thru doc-set and get rid of all docs which are unmarked
+ *      The motivation for this routine is in the case of CNF queries.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ReduceList (DocList * cset)
+{
+  DocEntry *src;
+  DocEntry *dest;
+
+  src = dest = cset->DE;
+  for (src = dest = cset->DE; src < cset->DE + cset->num; src++)
+    {
+      if (src->or_included)
+	{
+	  src->or_included = 0;
+	  *dest++ = *src;
+	}
+    }
+  cset->num = dest - cset->DE;
+
+}				/*ReduceList */
+
+/* =========================================================================
+ * Function: BooleanGet
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static DocList *
+BooleanGet (query_data * qd, bool_tree_node * n,
+	    BooleanQueryInfo * bqi)
+{
+  if (n)
+    {
+      DocList *res = NULL;
+      switch (BOOL_TAG (n))
+	{
+	case N_all:
+	  Message ("Unexpected \"all\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_not:
+	  Message ("Unexpected \"not\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_none:
+	  {
+	    res = MakeDocList (0);
+	    break;
+	  }
+	case N_diff:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		if (BOOL_TAG (n) == N_term)
+		  {
+		    WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		    res = GetDocsOp (qd, we, op_diff_terms, res);
+		  }
+		else
+		  res = DiffLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_and:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);		/* initial fill of c-set */
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		switch (BOOL_TAG (n))
+		  {
+		  case N_term:
+		    {
+		      WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		      res = GetDocsOp (qd, we, op_and_terms, res);
+		    }
+		    break;
+		  case N_or_terms:
+		    {
+		      bool_tree_node *child = BOOL_CHILD (n);
+		      /* note: all children are terms - predetermined */
+		      while (child)
+			{
+			  WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (child));
+			  res = GetDocsOp (qd, we, op_and_or_terms, res);
+			  child = BOOL_SIBLING (child);
+			}
+		      ReduceList (res);
+		    }
+		    break;
+		  default:
+		    res = IntersectLists (res, BooleanGet (qd, n, bqi));
+		  }		/*switch */
+		n = next;
+	      }
+	    break;
+	  }
+	case N_or:
+	case N_or_terms:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		res = MergeLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_term:
+	  {
+	    res = GetDocsOp (qd, GetNthWE (qd->TL, BOOL_TERM (n)), op_term, NULL);
+	    break;
+	  }
+	}
+      return (res ? res : MakeDocList (0));
+    }
+  else
+    return MakeDocList (0);
+}
+
+/* =========================================================================
+ * Function: AdjustParaDocs
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+static void
+AdjustParaDocs (query_data * qd, DocList * Docs)
+{
+  DocEntry *DE = Docs->DE;
+  int s, d;
+  Splay_Tree *SP;
+
+
+  /* Make the doc-list contain only references to unique
+   * real documents. So can get rid of cases where there
+   * are multiple paragraphs for a real document.
+   */
+
+  /* Note: [TS: 29/Sep/94] */
+  /* reads in seek-pos for docnums so can compare docs */
+  /* seek-pos used as real doc identifier */
+  /* whereas doc-num entries refer to paragraphs -- I think */
+  for (s = 0; s < Docs->num; s++)
+    FetchDocStart (qd, DE[s].DocNum, &DE[s].SeekPos, &DE[s].Len);
+
+  Message ("Original Number = %d\n", Docs->num);
+  SP = SP_createset (DE_comp);
+  d = 0;
+  for (s = 0; s < Docs->num; s++)
+    if (!SP_member (&DE[s], SP))
+      {
+	DE[d] = DE[s];
+	SP_insert (&DE[d], SP);
+	d++;
+      }
+  SP_freeset (SP);
+  Docs->num = d;
+  Message ("Final Number = %d\n", Docs->num);
+}
+
+/* =========================================================================
+ * Function: BooleanQuery 
+ * Description: 
+ *      The main control function for the boolean query.
+ * Input: 
+ * Output: 
+ *      qd->DL
+ * ========================================================================= */
+
+void 
+BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+	      int stem_method)
+{
+  bool_tree_node *tree;
+  DocList *Docs;
+  int res = 0;
+
+  tree = ParseBool (Query, MAXLINEBUFFERLEN, &(qd->TL), 
+		    qd->sd->sdh.stemmer_num, stem_method, &res,
+		    qd->sd, qd->sd->sdh.indexed,         /* [RPAP - Jan 97: Stem Index Change] */
+		    &(qd->QTL));                         /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [TS:Aug/94] no longer tries to continue after parse failure */
+  if (res != 0)			/* failed the parse */
+    {
+      qd->DL = NULL;		/* have empty doc-list */
+      return;
+    }
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  /*  ReadInTermInfo (qd); */
+
+  FindNoneTerms (qd, tree);
+
+  OptimiseBoolTree (tree, qd->TL, atoi (GetEnv ("optimise_type")));
+
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_terms = 0;
+
+  Docs = BooleanGet (qd, tree, bqi);
+
+/* [RPAP - Feb 97: NZDL Specifics] */
+#if !defined(PARABOOLEANMATCHES) && !defined(NZDL)
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      AdjustParaDocs (qd, Docs);
+    }
+#endif
+
+  if (bqi->MaxDocsToRetrieve != -1 && Docs->num > bqi->MaxDocsToRetrieve)
+    Docs->num = bqi->MaxDocsToRetrieve;
+
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/indexers/mg/src/text/bool_query.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_query.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_query.h	(revision 21112)
@@ -0,0 +1,55 @@
+/**************************************************************************
+ *
+ * bool_query.h -- boolean query processing
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:28  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:36  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_QUERY_H
+#define BOOL_QUERY_H
+
+#include "backend.h"
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/bool_tester.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_tester.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_tester.c	(revision 21112)
@@ -0,0 +1,185 @@
+/**************************************************************************
+ *
+ * bool_tester -- used to test bool_tree, bool_parser, bool_optimiser
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2000/03/05 22:58:24  sjboddie
+   compiler complained about using stdin and stdout to do global
+   initialization - variables were never changed so they were
+   replaced with #defines
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:41  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:23  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:05  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#define MAX_LINE_LEN  255
+#define STEMMER_NUM 0 /* Lovin's stemmer */
+#define STEM_METHOD 3
+
+/* Newer compilers complain when you use stdin and stdout to do global 
+initialization. Since they're static we'll just use #defines instead
+static FILE *file_in = stdin;
+static FILE *file_out = stdout;
+*/
+#define FILE_IN stdin
+#define FILE_OUT stdout
+
+static char line[MAX_LINE_LEN + 1];
+
+/* --- prototypes --- */
+static char *prompt (char *str);
+
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int main (int argc, char *argv[])
+{
+  bool_tree_node *tree = NULL;
+  TermList *term_list = NULL;
+  QueryTermList *query_term_list = NULL;  /* [RPAP - Feb 97: Term Frequency] */
+  int opt_type = 0;
+
+  while (1)
+    {
+      int res = 0;
+      int len = 0;
+
+      /* get a line */
+      if (!prompt ("Please enter boolean expression..."))
+	break;
+
+      len = strlen (line) - 1;	/* -1 => ignore the \n */
+
+      tree = ParseBool (line, len, &term_list, STEMMER_NUM, STEM_METHOD, &res,
+			NULL, 0,             /* [RPAP - Jan 97: Stem Index Change] */
+			&query_term_list);   /* [RPAP - Feb 97: Term Frequency] */
+
+      {
+	int done = 0;
+	while (!done)
+	  {
+	    if (!prompt ("Which type of optimisation 1 or 2 ?"))
+	      break;
+	    opt_type = atoi (line);
+	    done = (opt_type == 1 || opt_type == 2);
+	  }
+      }
+
+      if (!prompt ("Do you want to assign doc_counts to terms ?"))
+	break;
+
+      if (toupper (line[0]) == 'Y')
+	{
+	  /* cycle thru terms asking for doc_counts */
+	  int i;
+	  for (i = 0; i < term_list->num; i++)
+	    {
+	      char str[80];
+	      TermEntry *te = &(term_list->TE[i]);
+
+	      sprintf (str, "Please enter doc count for term '%s'",
+		       str255_to_string (te->Word, NULL));
+	      prompt (str);
+	      te->WE.doc_count = atoi (line);
+	      printf ("doc_count = %ld\n", te->WE.doc_count);
+	    }			/*for */
+	}			/*if */
+
+
+      if (res == 0)
+	{
+	  fprintf (FILE_OUT, "\n***Parsed Expression***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	  OptimiseBoolTree (tree, term_list, opt_type);
+	  fprintf (FILE_OUT, "\n***Optimised Expression ***\n");
+	  PrintBoolTree (tree, FILE_OUT);
+	  fputc ('\n', FILE_OUT);
+	}
+
+    }
+  return 0;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static char *
+prompt (char *str)
+{
+  fprintf (FILE_OUT, "\n%s\n", str);
+  return fgets (line, MAX_LINE_LEN, FILE_IN);
+}
Index: /main/tags/3.01/indexers/mg/src/text/bool_tree.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_tree.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_tree.c	(revision 21112)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * bool_tree.c -- Boolean parse tree ADT
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *       @(#)query.bool.y	1.9 16 Mar 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bool_tree.h"
+
+static char *RCSID = "$Id$";
+
+
+/* =========================================================================
+ * Function: CreateBoolNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolNode (N_Tag tag)
+{
+  bool_tree_node *n;
+  if (!(n = Xmalloc (sizeof (bool_tree_node))))
+    FatalError (1, "Can not create bool_tree_node for boolean query");
+  n->tag = tag;
+  BOOL_SIBLING (n) = NULL;
+  return n;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+		    u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len,  /* [RPAP - Feb 97: Term Frequency] */
+		    int stemmer_num) 
+{
+  bool_tree_node *n = NULL;
+
+  /* allocate bool_tree_node */
+  n = CreateBoolNode (N_term);
+
+  BOOL_TERM (n) = AddTerm (tl, (u_char *) text, Count, word_num, count, doc_count, 
+			   invf_ptr, invf_len, stemmer_num);  /* [RPAP - Feb 97: Term Frequency] */
+
+  return n;
+}
+
+
+/* =========================================================================
+ * Function: CreateTreeNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CreateBoolTreeNode (N_Tag tag, bool_tree_node * left, bool_tree_node * right)
+{
+  bool_tree_node *n = CreateBoolNode (tag);
+
+  BOOL_CHILD (n) = left;
+  if (left)
+    BOOL_SIBLING (left) = right;
+  if (right)
+    BOOL_SIBLING (right) = NULL;
+
+  return (n);
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+EqualBoolNodes (bool_tree_node * a, bool_tree_node * b)
+{
+  if (BOOL_TAG (a) == N_term && BOOL_TAG (a) == N_term)
+    return BOOL_TERM (a) == BOOL_TERM (b);
+  return 0;
+}
+
+/* =========================================================================
+ * Function: FreeBoolTree
+ * Description: 
+ *      Notes: does not fix up any parent or sibling links around given tree
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FreeTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	FreeTree (BOOL_CHILD (tree));
+      sibling = BOOL_SIBLING (tree);
+      Xfree (tree);
+      tree = sibling;
+    }
+}
+
+void 
+FreeBoolTree (bool_tree_node ** the_tree)
+{
+  bool_tree_node *tree = *the_tree;
+
+  if (!tree)
+    return;
+
+  FreeTree (tree);
+  *the_tree = NULL;
+/*********************************/
+  /* THIS IS POTENTIALLY DANGEROUS */
+  /* FIX UP LATER */
+  /* Need to correct the links for parent/siblings */
+/*********************************/
+
+}
+
+/* =========================================================================
+ * Function: PrintBoolTree
+ * Description: 
+ *      Print out a text version of the tree to the file
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PrintBinaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+  assert (BOOL_HAS_CHILD (tree));
+
+  child = BOOL_CHILD (tree);
+  while (child)
+    {
+      PrintBoolTree (child, file);
+      child = BOOL_SIBLING (child);
+      if (child)
+	fprintf (file, " %c ", op);
+    }
+}
+
+static void
+PrintUnaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+
+  fprintf (file, " %c ", op);
+  assert (BOOL_HAS_CHILD (tree));
+  child = BOOL_CHILD (tree);
+  PrintBoolTree (child, file);
+}
+
+void
+PrintBoolTree (bool_tree_node * tree, FILE * file)
+{
+  if (!tree)
+    return;
+
+  fprintf (file, "(");
+  switch (BOOL_TAG (tree))
+    {
+    case N_term:
+      fprintf (file, "term %d", BOOL_TERM (tree));
+      break;
+    case N_and:
+      PrintBinaryOp (tree, file, '&');
+      break;
+    case N_or:
+      PrintBinaryOp (tree, file, '|');
+      break;
+    case N_or_terms:
+      PrintBinaryOp (tree, file, '+');
+      break;
+    case N_diff:
+      PrintBinaryOp (tree, file, '-');
+      break;
+    case N_not:
+      PrintUnaryOp (tree, file, '!');
+      break;
+    case N_all:
+      fprintf (file, " TRUE ");
+      break;
+    case N_none:
+      fprintf (file, " FALSE ");
+      break;
+    default:
+      break;
+    }
+
+  fprintf (file, ")");
+}
+
+
+/* =========================================================================
+ * Function: CopyBoolTree
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CopyBoolTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling = NULL;
+  bool_tree_node *tree_copy = NULL;
+  bool_tree_node *prev = NULL;
+
+  if (!tree)
+    return NULL;
+
+  sibling = tree;
+  prev = NULL;
+  while (sibling)
+    {
+      /* copy sibling */
+      bool_tree_node *sibling_copy = (bool_tree_node *) malloc (sizeof (bool_tree_node));
+      if (!sibling_copy)
+	FatalError (1, "No memory for bool tree copy");
+      bcopy ((char *) sibling, (char *) sibling_copy, sizeof (bool_tree_node));
+
+      if (BOOL_HAS_CHILD (sibling))
+	{
+	  BOOL_CHILD (sibling_copy) = CopyBoolTree (BOOL_CHILD (sibling));
+	}
+
+      /* do the link up of copies */
+      if (prev)
+	BOOL_SIBLING (prev) = sibling_copy;
+      else
+	tree_copy = sibling_copy;
+      prev = sibling_copy;
+
+      sibling = BOOL_SIBLING (sibling);
+    }
+  return tree_copy;
+}
+
+
+/* =========================================================================
+ * Function: LastBoolSibling
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+LastBoolSibling (bool_tree_node * tree)
+{
+
+  if (!tree)
+    return NULL;
+
+  do
+    {
+      tree = BOOL_SIBLING (tree);
+    }
+  while (BOOL_SIBLING (tree));
+
+  return tree;
+}
Index: /main/tags/3.01/indexers/mg/src/text/bool_tree.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/bool_tree.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/bool_tree.h	(revision 21112)
@@ -0,0 +1,113 @@
+/**************************************************************************
+ *
+ * bool_tree.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:42  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:25  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:07  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:35  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_TREE_H
+#define BOOL_TREE_H
+
+#include "sysfuncs.h"
+#include "term_lists.h"
+
+typedef enum
+  {
+    N_term, N_and, N_or, N_diff, N_none, N_all,
+    N_not, N_or_terms
+  }
+N_Tag;
+
+typedef struct bool_tree_node
+  {
+    N_Tag tag;
+    struct bool_tree_node *sibling;
+    union
+      {
+	struct bool_tree_node *child;
+	int term_id;
+      }
+    fields;
+    int sum_ft;			/* for or_terms nodes */
+  }
+bool_tree_node;
+
+
+#define BOOL_HAS_CHILD(n) (\
+			(n)->tag==N_and || \
+			(n)->tag==N_or || (n)->tag==N_or_terms || \
+			(n)->tag==N_not || (n)->tag==N_diff)
+#define BOOL_TAG(n) ((n)->tag)
+#define BOOL_SIBLING(n) ((n)->sibling)
+#define BOOL_TERM(n) ((n)->fields.term_id)
+#define BOOL_CHILD(n) ((n)->fields.child)
+#define BOOL_SUM_FT(n) ((n)->sum_ft)
+
+bool_tree_node *CreateBoolNode (N_Tag tag);
+bool_tree_node *CopyBoolTree (bool_tree_node * tree);
+bool_tree_node *CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+				    u_long count, u_long doc_count, u_long invf_ptr, 
+				    u_long invf_len, int stemmer_num);
+bool_tree_node *CreateBoolTreeNode (N_Tag tag,
+			     bool_tree_node * left, bool_tree_node * right);
+void FreeBoolTree (bool_tree_node ** tree);
+void PrintBoolTree (bool_tree_node * tree, FILE * file);
+bool_tree_node *LastBoolSibling (bool_tree_node * tree);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/build.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/build.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/build.h	(revision 21112)
@@ -0,0 +1,171 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#define	TERMPARAGRAPH	'\003'
+
+
+int init_special (char *file_name);
+int init_text_1 (char *file_name);
+int init_text_2 (char *file_name);
+int init_invf_1 (char *file_name);
+int init_invf_2 (char *file_name);
+int init_ivf_1 (char *file_name);
+int init_ivf_2 (char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (u_char * s_in, int l_in);
+int process_text_1 (u_char * s_in, int l_in);
+int process_text_2 (u_char * s_in, int l_in);
+int process_invf_1 (u_char * s_in, int l_in);
+int process_invf_2 (u_char * s_in, int l_in);
+int process_ivf_1 (u_char * s_in, int l_in);
+int process_ivf_2 (u_char * s_in, int l_in);
+/*
+ * s_in               IN      The binary string to be compressed
+ * l_in         IN      The number of characters in s_in
+ *
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (char *filename);
+int done_text_1 (char *filename);
+int done_text_2 (char *filename);
+int done_invf_1 (char *filename);
+int done_invf_2 (char *filename);
+int done_ivf_1 (char *filename);
+int done_ivf_2 (char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char InvfLevel;
+/*
+ * This will determine the level of the inverted file it can take on the
+ * values 1, 2, or 3.
+ *
+ * Level 1: The inverted file contains only document numbers making it possible
+ *          to do only boolean queries.
+ * 
+ * Level 2: The inverted file also contains word counts per document making it
+ *          possible to do cosine ranked queries.
+ *
+ * Level 3: The inverted file contains word positions.
+ */
+
+
+extern unsigned long buf_size;
+/*
+ * The size of the document input buffer.
+ */
+
+extern unsigned long ChunkLimit;
+/*
+ * The maximum number of chunks that can be written to disk.
+ */
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+extern char SkipSGML;
+/*
+ * 1 if SGML tags are to be considered non-words when building the 
+ * inverted file.
+ */
+
+extern char MakeWeights;
+/*
+ * 1 if the weights file should be generated.
+ */
+
+extern FILE *Comp_Stats;
+/*
+ * Contains a file pointer to the file where compression stats should be sent
+ */
+
+extern int comp_stat_point;
+/*
+ * Generate a compression stat entry every comp_stat_point bytes 
+ */
+
+extern mg_ullong bytes_processed;
+/*
+ * The number of bytes processed. NOTE: This excludes document separators.
+ */
+
+extern mg_ullong bytes_received;
+/*
+ * The number of bytes processed. NOTE: This includes document separators.
+ */
+
+
+extern int stemmer_num;
+/*
+ * The stemmer to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+
+extern int stem_method;
+/*
+ * The method to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/commands.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/commands.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/commands.c	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * commands.c -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:32  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:22  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "environment.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"		/* for qd */
+#include "term_lists.h"		/* for MAXTERMSTRLEN */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define OPENPIPE        _popen
+# define CLOSEPIPE       _pclose
+#else
+# define OPENPIPE        popen
+# define CLOSEPIPE       pclose
+#endif
+
+extern char *PathName;
+extern FILE *OutFile, *InFile;
+extern int OutPipe, InPipe;
+extern int Quitting;
+
+char *CommandsErrorStr = NULL;
+
+/* [TS: June/95] */
+/* current qd for use by any commands */
+static query_data *the_qd = NULL;
+
+
+#include "help.mg.h"
+#include "warranty.h"
+#include "conditions.h"
+
+static void 
+CmdQuit (void)
+{
+  Quitting = 1;
+}
+
+static void 
+CmdPush (void)
+{
+  PushEnv ();
+}
+
+
+static void 
+CmdPop (void)
+{
+  PopEnv ();
+  if (EnvStackHeight () == 1)
+    PushEnv ();
+}
+
+static void 
+CmdHelp (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (help_str) / sizeof (help_str[0]); i++)
+    fputs (help_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdWarranty (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (warranty_str) / sizeof (warranty_str[0]); i++)
+    fputs (warranty_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdConditions (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (cond_str) / sizeof (cond_str[0]); i++)
+    fputs (cond_str[i], more);
+
+  CLOSEPIPE (more);
+}
+
+static void 
+CmdReset (void)
+{
+  while (EnvStackHeight () > 1)
+    PopEnv ();
+  PushEnv ();
+}
+
+static void 
+CmdDisplay (void)
+{
+  int i;
+  size_t l = 0;
+  char *name;
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    if (strlen (name) > l)
+      l = strlen (name);
+
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+}
+
+static void 
+CmdQueryTerms (void)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+
+  if (the_qd && the_qd->TL)
+    {
+      ConvertTermsToString (the_qd->TL, terms_str);
+      printf ("%s\n", terms_str);
+      fflush (stdout);
+    }
+
+}
+
+
+static void 
+CmdSet (char *name, char *value)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  switch (SetEnv (name, value, NULL))
+    {
+    case -1:
+      CommandsErrorStr = "Out of memory\n";
+      break;
+    case -2:
+      CommandsErrorStr = ConstraintErrorStr;
+      break;
+    }
+}
+
+static void 
+CmdUnset (char *name)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  if (UnsetEnv (name, 0) == -1)
+    CommandsErrorStr = "Specified name does not exist";
+}
+
+
+
+
+
+
+/*
+ * Do the .input command 
+ */
+static void 
+CmdInput (char *param1, char *param2)
+{
+  enum
+  {
+    READ, APPEND, PIPEFROM
+  }
+  mode;
+  FILE *NewInput;
+  char *buf, *s;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '<')
+    {
+      s++;
+      mode = READ;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPEFROM;
+      s++;
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == READ)
+	NewInput = fopen (s, "r");
+      else
+	NewInput = OPENPIPE (s, "r");  /* [RPAP - Feb 97: WIN32 Port] */
+
+      if (NewInput)
+	{
+	  if (InPipe)
+	    CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (InFile != stdin)
+	    fclose (InFile);
+	  InPipe = (mode == PIPEFROM);
+	  InFile = NewInput;
+	}
+      else
+	CommandsErrorStr = "Bad input file/pipe";
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+    }
+  Xfree (buf);
+}
+
+
+
+static void 
+CmdOutput (char *param1, char *param2)
+{
+  enum
+  {
+    WRITE, APPEND, PIPETO
+  }
+  mode;
+  char *buf, *s;
+  FILE *NewOutput;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '>')
+    {
+      s++;
+      mode = APPEND;
+      if (*s == '>')
+	s++;
+      else
+	mode = WRITE;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPETO;
+      s++;
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == PIPETO)
+	NewOutput = OPENPIPE (s, "w");  /* [RPAP - Feb 97: WIN32 Port] */
+      else
+	NewOutput = fopen (s, mode == WRITE ? "w" : "a");
+      if (NewOutput)
+	{
+	  if (OutPipe)
+	    CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (OutFile != stdout)
+	    fclose (OutFile);
+	  OutPipe = (mode == PIPETO);
+	  OutFile = NewOutput;
+	}
+      else
+	CommandsErrorStr = "Bad output file/pipe";
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+    }
+  Xfree (buf);
+}
+
+
+
+static char *
+ParseArg (char *line, char *buffer)
+{
+  if (*line == '\"')
+    {
+      line++;
+      while (*line != '\"' && *line != '\0')
+	{
+	  if (*line == '\\')
+	    *buffer++ = *line++;
+	  if (*line != '\0')
+	    *buffer++ = *line++;
+	}
+      if (*line == '\"')
+	line++;
+    }
+  else
+    while (*line != ' ' && *line != '\t' && *line != '\0')
+      {
+	if (*line == '\\')
+	  line++;
+	if (*line != '\0')
+	  *buffer++ = *line++;
+      }
+  *buffer = '\0';
+  while (*line == ' ' || *line == '\t')
+    line++;
+  return (line);
+}
+
+
+
+char *
+ProcessCommands (char *line, query_data * qd)
+{
+  struct
+  {
+    char *Command;
+    void (*action) (void);
+    void (*action1) (char *);
+    void (*action2) (char *, char *);
+  }
+   *dc, dot_command[] =
+  {
+    {
+      "set", NULL, NULL, CmdSet
+    }
+    ,
+    {
+      "input", NULL, NULL, CmdInput
+    }
+    ,
+    {
+      "output", NULL, NULL, CmdOutput
+    }
+    ,
+    {
+      "unset", NULL, CmdUnset, NULL
+    }
+    ,
+    {
+      "help", CmdHelp, NULL, NULL
+    }
+    ,
+    {
+      "reset", CmdReset, NULL, NULL
+    }
+    ,
+    {
+      "display", CmdDisplay, NULL, NULL
+    }
+    ,
+    {
+      "push", CmdPush, NULL, NULL
+    }
+    ,
+    {
+      "quit", CmdQuit, NULL, NULL
+    }
+    ,
+    {
+      "pop", CmdPop, NULL, NULL
+    }
+    ,
+    {
+      "warranty", CmdWarranty, NULL, NULL
+    }
+    ,
+    {
+      "conditions", CmdConditions, NULL, NULL
+    }
+    ,
+    {
+      "queryterms", CmdQueryTerms, NULL, NULL
+    }
+  };
+
+
+  char command[512];
+  char param1[512], param2[512];
+  CommandsErrorStr = NULL;
+  the_qd = qd;			/* set globally to be accessed by command functions if needed */
+  while (*line == ' ' || *line == '\t')
+    line++;
+  while (*line == '.')
+    {
+      int i;
+      line++;
+      for (i = 0; isalpha (*line) && i < 512; i++, line++)
+	command[i] = tolower (*line);
+      while (*line == ' ' || *line == '\t')
+	line++;
+      if (i != 512)
+	{
+	  command[i] = '\0';
+	  for (i = 0, dc = dot_command; i < NUMOF (dot_command); i++, dc++)
+	    if (!strcmp (dc->Command, command))
+	      {
+		if (dc->action)
+		  dc->action ();
+		else if (dc->action1)
+		  {
+		    line = ParseArg (line, param1);
+		    dc->action1 (param1);
+		    break;
+		  }
+		else if (dc->action2)
+		  {
+		    line = ParseArg (line, param1);
+		    line = ParseArg (line, param2);
+		    dc->action2 (param1, param2);
+		    break;
+		  }
+		if (CommandsErrorStr)
+		  *line = '\0';
+		break;
+	      }
+	  if (i == NUMOF (dot_command))
+	    i = 100;
+	}
+      if (i == 100)
+	{
+	  CommandsErrorStr = "Illegal command";
+	  *line = '\0';
+	}
+      while (*line == ' ' || *line == '\t')
+	line++;
+    }
+  return (line);
+}
+
+
+
+
+
+void 
+read_mgrc_file (void)
+{
+  char line[1000];
+  char FileName[256];
+  int LineNum = 0;
+  FILE *mgrc;
+
+  strcpy (FileName, ".mgrc");
+  mgrc = fopen (FileName, "r");
+
+  if (!mgrc)
+    {
+      char *home_path = getenv ("HOME");
+      if (home_path)
+	{
+	  sprintf (FileName, "%s/.mgrc", home_path);
+	  mgrc = fopen (FileName, "r");
+	}
+    }
+
+  if (!mgrc)
+    return;
+
+  while (fgets (line, sizeof (line), mgrc))
+    {
+      char *s, linebuf[1000];
+      LineNum++;
+      if ((s = strchr (line, '\n')) != NULL)
+	*s = '\0';
+      strcpy (linebuf, line);
+      if ((s = strchr (line, '#')) != NULL)
+	*s = '\0';
+      if (!*line)
+	continue;
+      ProcessCommands (line, NULL);
+      if (CommandsErrorStr)
+	Message ("%s\n%s : ERROR in line %d of %s\n%s : %s\n",
+		 linebuf,
+		 msg_prefix, LineNum, FileName,
+		 msg_prefix, CommandsErrorStr);
+    }
+  fclose (mgrc);
+}
Index: /main/tags/3.01/indexers/mg/src/text/commands.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/commands.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/commands.h	(revision 21112)
@@ -0,0 +1,33 @@
+/**************************************************************************
+ *
+ * commands.h -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef COMMANDS_H
+#define COMMANDS_H
+
+#include "backend.h"
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/comp_dict.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/comp_dict.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/comp_dict.c	(revision 21112)
@@ -0,0 +1,258 @@
+/**************************************************************************
+ *
+ * comp_dict.c -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:41  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:24  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = Xmalloc (sizeof (dict_hash_table) +
+		      (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  bzero ((char *) ht->table, ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; i++, words++)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register int tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+	    goto error;
+
+	  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+	    goto error;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(char_codes[which] =
+		Generate_Huffman_Codes (&char_huff[which], NULL)))
+	    goto error;
+	  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(lens_codes[which] =
+		Generate_Huffman_Codes (&lens_huff[which], NULL)))
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(char_codes[which] =
+		    Generate_Huffman_Codes (&char_huff[which], NULL)))
+		goto error;
+	      if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(lens_codes[which] =
+		    Generate_Huffman_Codes (&lens_huff[which], NULL)))
+		goto error;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/indexers/mg/src/text/comp_dict.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+
+#define HASH_RATIO 2
+
+
+
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+
+
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/indexers/mg/src/text/conditions.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/conditions.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/conditions.h	(revision 21112)
@@ -0,0 +1,226 @@
+static char *cond_str[] = {
+"		    GNU GENERAL PUBLIC LICENSE\n",
+"   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n",
+"\n",
+"  0. This License applies to any program or other work which contains\n",
+"a notice placed by the copyright holder saying it may be distributed\n",
+"under the terms of this General Public License.  The \"Program\", below,\n",
+"refers to any such program or work, and a \"work based on the Program\"\n",
+"means either the Program or any derivative work under copyright law:\n",
+"that is to say, a work containing the Program or a portion of it,\n",
+"either verbatim or with modifications and/or translated into another\n",
+"language.  (Hereinafter, translation is included without limitation in\n",
+"the term \"modification\".)  Each licensee is addressed as \"you\".\n",
+"\n",
+"Activities other than copying, distribution and modification are not\n",
+"covered by this License; they are outside its scope.  The act of\n",
+"running the Program is not restricted, and the output from the Program\n",
+"is covered only if its contents constitute a work based on the\n",
+"Program (independent of having been made by running the Program).\n",
+"Whether that is true depends on what the Program does.\n",
+"\n",
+"  1. You may copy and distribute verbatim copies of the Program's\n",
+"source code as you receive it, in any medium, provided that you\n",
+"conspicuously and appropriately publish on each copy an appropriate\n",
+"copyright notice and disclaimer of warranty; keep intact all the\n",
+"notices that refer to this License and to the absence of any warranty;\n",
+"and give any other recipients of the Program a copy of this License\n",
+"along with the Program.\n",
+"\n",
+"You may charge a fee for the physical act of transferring a copy, and\n",
+"you may at your option offer warranty protection in exchange for a fee.\n",
+"\n",
+"  2. You may modify your copy or copies of the Program or any portion\n",
+"of it, thus forming a work based on the Program, and copy and\n",
+"distribute such modifications or work under the terms of Section 1\n",
+"above, provided that you also meet all of these conditions:\n",
+"\n",
+"    a) You must cause the modified files to carry prominent notices\n",
+"    stating that you changed the files and the date of any change.\n",
+"\n",
+"    b) You must cause any work that you distribute or publish, that in\n",
+"    whole or in part contains or is derived from the Program or any\n",
+"    part thereof, to be licensed as a whole at no charge to all third\n",
+"    parties under the terms of this License.\n",
+"\n",
+"    c) If the modified program normally reads commands interactively\n",
+"    when run, you must cause it, when started running for such\n",
+"    interactive use in the most ordinary way, to print or display an\n",
+"    announcement including an appropriate copyright notice and a\n",
+"    notice that there is no warranty (or else, saying that you provide\n",
+"    a warranty) and that users may redistribute the program under\n",
+"    these conditions, and telling the user how to view a copy of this\n",
+"    License.  (Exception: if the Program itself is interactive but\n",
+"    does not normally print such an announcement, your work based on\n",
+"    the Program is not required to print an announcement.)\n",
+"
+\n",
+"These requirements apply to the modified work as a whole.  If\n",
+"identifiable sections of that work are not derived from the Program,\n",
+"and can be reasonably considered independent and separate works in\n",
+"themselves, then this License, and its terms, do not apply to those\n",
+"sections when you distribute them as separate works.  But when you\n",
+"distribute the same sections as part of a whole which is a work based\n",
+"on the Program, the distribution of the whole must be on the terms of\n",
+"this License, whose permissions for other licensees extend to the\n",
+"entire whole, and thus to each and every part regardless of who wrote it.\n",
+"\n",
+"Thus, it is not the intent of this section to claim rights or contest\n",
+"your rights to work written entirely by you; rather, the intent is to\n",
+"exercise the right to control the distribution of derivative or\n",
+"collective works based on the Program.\n",
+"\n",
+"In addition, mere aggregation of another work not based on the Program\n",
+"with the Program (or with a work based on the Program) on a volume of\n",
+"a storage or distribution medium does not bring the other work under\n",
+"the scope of this License.\n",
+"\n",
+"  3. You may copy and distribute the Program (or a work based on it,\n",
+"under Section 2) in object code or executable form under the terms of\n",
+"Sections 1 and 2 above provided that you also do one of the following:\n",
+"\n",
+"    a) Accompany it with the complete corresponding machine-readable\n",
+"    source code, which must be distributed under the terms of Sections\n",
+"    1 and 2 above on a medium customarily used for software interchange; or,\n",
+"\n",
+"    b) Accompany it with a written offer, valid for at least three\n",
+"    years, to give any third party, for a charge no more than your\n",
+"    cost of physically performing source distribution, a complete\n",
+"    machine-readable copy of the corresponding source code, to be\n",
+"    distributed under the terms of Sections 1 and 2 above on a medium\n",
+"    customarily used for software interchange; or,\n",
+"\n",
+"    c) Accompany it with the information you received as to the offer\n",
+"    to distribute corresponding source code.  (This alternative is\n",
+"    allowed only for noncommercial distribution and only if you\n",
+"    received the program in object code or executable form with such\n",
+"    an offer, in accord with Subsection b above.)\n",
+"\n",
+"The source code for a work means the preferred form of the work for\n",
+"making modifications to it.  For an executable work, complete source\n",
+"code means all the source code for all modules it contains, plus any\n",
+"associated interface definition files, plus the scripts used to\n",
+"control compilation and installation of the executable.  However, as a\n",
+"special exception, the source code distributed need not include\n",
+"anything that is normally distributed (in either source or binary\n",
+"form) with the major components (compiler, kernel, and so on) of the\n",
+"operating system on which the executable runs, unless that component\n",
+"itself accompanies the executable.\n",
+"\n",
+"If distribution of executable or object code is made by offering\n",
+"access to copy from a designated place, then offering equivalent\n",
+"access to copy the source code from the same place counts as\n",
+"distribution of the source code, even though third parties are not\n",
+"compelled to copy the source along with the object code.\n",
+"
+\n",
+"  4. You may not copy, modify, sublicense, or distribute the Program\n",
+"except as expressly provided under this License.  Any attempt\n",
+"otherwise to copy, modify, sublicense or distribute the Program is\n",
+"void, and will automatically terminate your rights under this License.\n",
+"However, parties who have received copies, or rights, from you under\n",
+"this License will not have their licenses terminated so long as such\n",
+"parties remain in full compliance.\n",
+"\n",
+"  5. You are not required to accept this License, since you have not\n",
+"signed it.  However, nothing else grants you permission to modify or\n",
+"distribute the Program or its derivative works.  These actions are\n",
+"prohibited by law if you do not accept this License.  Therefore, by\n",
+"modifying or distributing the Program (or any work based on the\n",
+"Program), you indicate your acceptance of this License to do so, and\n",
+"all its terms and conditions for copying, distributing or modifying\n",
+"the Program or works based on it.\n",
+"\n",
+"  6. Each time you redistribute the Program (or any work based on the\n",
+"Program), the recipient automatically receives a license from the\n",
+"original licensor to copy, distribute or modify the Program subject to\n",
+"these terms and conditions.  You may not impose any further\n",
+"restrictions on the recipients' exercise of the rights granted herein.\n",
+"You are not responsible for enforcing compliance by third parties to\n",
+"this License.\n",
+"\n",
+"  7. If, as a consequence of a court judgment or allegation of patent\n",
+"infringement or for any other reason (not limited to patent issues),\n",
+"conditions are imposed on you (whether by court order, agreement or\n",
+"otherwise) that contradict the conditions of this License, they do not\n",
+"excuse you from the conditions of this License.  If you cannot\n",
+"distribute so as to satisfy simultaneously your obligations under this\n",
+"License and any other pertinent obligations, then as a consequence you\n",
+"may not distribute the Program at all.  For example, if a patent\n",
+"license would not permit royalty-free redistribution of the Program by\n",
+"all those who receive copies directly or indirectly through you, then\n",
+"the only way you could satisfy both it and this License would be to\n",
+"refrain entirely from distribution of the Program.\n",
+"\n",
+"If any portion of this section is held invalid or unenforceable under\n",
+"any particular circumstance, the balance of the section is intended to\n",
+"apply and the section as a whole is intended to apply in other\n",
+"circumstances.\n",
+"\n",
+"It is not the purpose of this section to induce you to infringe any\n",
+"patents or other property right claims or to contest validity of any\n",
+"such claims; this section has the sole purpose of protecting the\n",
+"integrity of the free software distribution system, which is\n",
+"implemented by public license practices.  Many people have made\n",
+"generous contributions to the wide range of software distributed\n",
+"through that system in reliance on consistent application of that\n",
+"system; it is up to the author/donor to decide if he or she is willing\n",
+"to distribute software through any other system and a licensee cannot\n",
+"impose that choice.\n",
+"\n",
+"This section is intended to make thoroughly clear what is believed to\n",
+"be a consequence of the rest of this License.\n",
+"
+\n",
+"  8. If the distribution and/or use of the Program is restricted in\n",
+"certain countries either by patents or by copyrighted interfaces, the\n",
+"original copyright holder who places the Program under this License\n",
+"may add an explicit geographical distribution limitation excluding\n",
+"those countries, so that distribution is permitted only in or among\n",
+"countries not thus excluded.  In such case, this License incorporates\n",
+"the limitation as if written in the body of this License.\n",
+"\n",
+"  9. The Free Software Foundation may publish revised and/or new versions\n",
+"of the General Public License from time to time.  Such new versions will\n",
+"be similar in spirit to the present version, but may differ in detail to\n",
+"address new problems or concerns.\n",
+"\n",
+"Each version is given a distinguishing version number.  If the Program\n",
+"specifies a version number of this License which applies to it and \"any\n",
+"later version\", you have the option of following the terms and conditions\n",
+"either of that version or of any later version published by the Free\n",
+"Software Foundation.  If the Program does not specify a version number of\n",
+"this License, you may choose any version ever published by the Free Software\n",
+"Foundation.\n",
+"\n",
+"  10. If you wish to incorporate parts of the Program into other free\n",
+"programs whose distribution conditions are different, write to the author\n",
+"to ask for permission.  For software which is copyrighted by the Free\n",
+"Software Foundation, write to the Free Software Foundation; we sometimes\n",
+"make exceptions for this.  Our decision will be guided by the two goals\n",
+"of preserving the free status of all derivatives of our free software and\n",
+"of promoting the sharing and reuse of software generally.\n",
+"\n",
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/indexers/mg/src/text/conditions.src
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/conditions.src	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/conditions.src	(revision 21112)
@@ -0,0 +1,225 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/indexers/mg/src/text/environment.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/environment.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/environment.c	(revision 21112)
@@ -0,0 +1,691 @@
+/**************************************************************************
+ *
+ * environment.c -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/04/25 23:01:18  kjdon
+   added a new -M option to mg_passes, allowing maxnumeric to be altered - made this change to keep gsdl3 mg inline with gsdl2 mg.
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:17:50  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.5  1995/03/14  05:15:26  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.4  1994/11/25  03:47:43  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:56:43  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:25  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+
+#include "environment.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define STRCASECMP      stricmp
+#else
+# define STRCASECMP      strcasecmp
+#endif
+
+typedef struct
+  {
+    char *Name, *Data;
+    char *(*Constraint) (char *Old, char *New);
+  }
+EEntry;
+
+typedef struct Env
+  {
+    EEntry *Environment;
+    int NumEnv;
+    struct Env *Next;
+  }
+Env;
+
+
+char *ConstraintErrorStr;
+
+static Env Base =
+{NULL, 0, NULL};
+
+
+/*
+ * Add name and data to the environment (data may be NULL)
+ * All names are converted to lower case
+ * returns 0 on success, -1 on memory failure or -2 on constraint failure
+ */
+int 
+SetEnv (char *name, char *data, char *(*Constraint) (char *, char *))
+{
+  int i;
+  EEntry *newenv, *env = Base.Environment;
+
+  if (data && *data == '\0')
+    data = NULL;
+
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* if not found the increase the environment size */
+  if (i >= Base.NumEnv)
+    {
+      if (Constraint)
+	{
+	  data = Constraint (NULL, data);
+	  if (!data)
+	    return (-2);
+	}
+      /* allocate memory for the new environment */
+      newenv = Base.Environment ?
+	Xrealloc (Base.Environment, sizeof (EEntry) * (Base.NumEnv + 1)) :
+	Xmalloc (sizeof (EEntry) * (Base.NumEnv + 1));
+      if (!newenv)
+	return (-1);
+      Base.Environment = newenv;
+      env = &newenv[Base.NumEnv];
+      env->Data = NULL;
+      env->Name = NULL;
+      env->Constraint = Constraint;
+      env->Name = Xstrdup (name);
+      if (!env->Name)
+	return (-1);
+      Base.NumEnv++;
+      for (i = 0; env->Name[i]; i++)
+	env->Name[i] = tolower (env->Name[i]);
+    }
+  if (env->Constraint)
+    {
+      data = env->Constraint (env->Data, data);
+      if (!data)
+	return (-2);
+    }
+  if (env->Data)
+    Xfree (env->Data);
+  env->Data = NULL;
+  if (data && *data != '\0')
+    {
+      env->Data = Xstrdup (data);
+      if (!env->Data)
+	return (-1);
+    }
+  return (0);
+}
+
+
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns NULL.
+ */
+char *
+GetEnv (char *name)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv)
+    return (NULL);
+
+  return (env->Data ? env->Data : "");
+}
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns default.
+ */
+char *
+GetDefEnv (char *name, char *def)
+{
+  char *data = GetEnv (name);
+  return (data ? data : def);
+}
+
+
+
+/*
+ * This function pushes the environment on to a stack and duplicated it in 
+ * the current stack.
+ *
+ */
+int 
+PushEnv (void)
+{
+  Env *env = Xmalloc (sizeof (Env));
+  int i;
+  *env = Base;
+  Base.Next = env;
+  Base.Environment = NULL;
+  Base.NumEnv = 0;
+  for (i = 0; i < env->NumEnv; i++)
+    if (SetEnv (env->Environment[i].Name, env->Environment[i].Data,
+		env->Environment[i].Constraint))
+      return (-1);
+  return (0);
+}
+
+int 
+PopEnv (void)
+{
+  Env *env;
+  int i;
+  if (!Base.Next)
+    return (-1);
+  for (i = 0; i < Base.NumEnv; i++)
+    {
+      Xfree (Base.Environment[i].Name);
+      if (Base.Environment[i].Data)
+	Xfree (Base.Environment[i].Data);
+    }
+  if (Base.Environment)
+    Xfree (Base.Environment);
+  env = Base.Next;
+  Base = *env;
+  Xfree (env);
+  return (0);
+}
+
+
+/*
+ * Count the number of environments on the stack, including the current one.
+ *
+ */
+int 
+EnvStackHeight (void)
+{
+  int i = 1;
+  Env *env = &Base;
+  while (env->Next)
+    {
+      i++;
+      env = env->Next;
+    }
+  return (i);
+}
+
+/*
+ * Delete environment variable name. If name does not exist it returns -1
+ * otherwise it returns 0
+ */
+int 
+UnsetEnv (char *name, int Force)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv ||
+      (Base.Environment[i].Constraint && !Force))
+    return (-1);
+  Xfree (Base.Environment[i].Name);
+  if (Base.Environment[i].Data)
+    Xfree (Base.Environment[i].Data);
+  for (i++; i < Base.NumEnv; i++)
+    Base.Environment[i - 1] = Base.Environment[i];
+  Base.NumEnv--;
+  if (Base.NumEnv == 0)
+    {
+      Xfree (Base.Environment);
+      Base.Environment = NULL;
+    }
+  return (0);
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns the name of environment variable number i or NULL if i is greater 
+ * than or equal to the number of environment variables. Names start from
+ * zero. 
+ */
+char *
+GetEnvName (int i)
+{
+  return (i < Base.NumEnv ? Base.Environment[i].Name : NULL);
+}
+
+static char *BooleanStrs[] =
+{"false", "true", "no", "yes", "off", "on"};
+
+static char *
+BooleanCons (char *Old, char *New)
+{
+  int i;
+  int old = -1, new = -1;
+  if (Old)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (Old, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  old = i;
+	  break;
+	}
+  if (New)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  if (!New)
+    return (BooleanStrs[Old ? old ^ 1 : 0]);
+  ConstraintErrorStr = "Invalid argument [true|false|yes|no|on|off] required";
+  return (NULL);
+}
+
+/*
+ * returns 0 or 1  for a boolean string
+ * or default on a error
+ */
+int 
+BooleanEnv (char *data, int def)
+{
+  int i;
+  if (!data)
+    return (def);
+  for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+    if (!STRCASECMP (data, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (i & 1);
+  return (def);
+}
+
+/*
+ * returns the value for a integer string
+ * or default on a error
+ */
+long 
+IntEnv (char *data, long def)
+{
+  long val;
+  char *ptr;
+  if (!data)
+    return (def);
+  val = strtol (data, &ptr, 10);
+  return *ptr ? def : val;
+}
+
+static char *
+NumberCmp (char *num, int min, int max)
+{
+  char *err;
+  int val;
+  static char Err[100];
+  ConstraintErrorStr = "Not a valid number";
+  if (!num)
+    return (NULL);
+  val = strtol (num, &err, 10);
+  if (*err)
+    return (NULL);
+  sprintf (Err, "Not in legal range [%d <= num <= %d]", min, max);
+  ConstraintErrorStr = Err;
+  if (val < min || val > max)
+    return (NULL);
+  return (num);
+}
+
+/*ARGSUSED */
+static char *
+MaxDocsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/*ARGSUSED */
+static char *
+MaxTermsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/* ARGSUSED */
+static char *
+BufferCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 16 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxNodesCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHashCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxParasCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHeadsCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 1000));
+}
+
+/* ARGSUSED */
+char *
+OptimiseCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 2));
+}
+
+/* ARGSUSED */
+char *
+MaxNumericCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 4, 512));
+}
+
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+static char *
+QueryCons (char *Old, char *New)
+{
+  static char *QueryStrs[] =
+  {
+    "boolean", "ranked",
+    "docnums", "approx-ranked"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (QueryStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, QueryStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [boolean|ranked|docnums|approx-ranked] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid accum type
+ *
+ */
+/* ARGSUSED */
+static char *
+AccumCons (char *Old, char *New)
+{
+  static char *AccumStrs[] =
+  {
+    "array", "splay_tree", "hash_table", "list"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (AccumStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, AccumStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument "
+    "[array|splay_tree|hash_table|list] required";
+  return (NULL);
+}
+
+
+
+/*
+ * Makes sure that New is a valid output type type
+ *
+ */
+/* ARGSUSED */
+static char *
+OutputTypeCons (char *Old, char *New)
+{
+  static char *OutputTypeStrs[] =
+  {"text", "silent", "docnums",
+   "count", "heads", "hilite"
+#ifdef TREC_MODE
+   ,"extras_for_trec"
+#endif
+  };
+
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (OutputTypeStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, OutputTypeStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [text|silent|docnums|heads|count|hilite"
+#ifdef TREC_MODE
+    "|extras_for_trec"
+#endif
+    "] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+char *
+HiliteStyleCons (char *Old, char *New)
+{
+  static char *StyleStrs[] =
+  {"bold", "underline"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (StyleStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, StyleStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [bold|underline] required";
+  return (NULL);
+}
+
+/* =========================================================================
+ * Function: get_output_type
+ * Description:
+ *      Map a mode string onto a mode char e.g. OUTPUT_????
+ * Input:
+ * Output:
+ *      The mode char which is a subrange value.
+ * ========================================================================= */
+
+char
+get_output_type (void)
+{
+  char *env_str = GetDefEnv ("mode", "text");
+  char ch1 = toupper (env_str[0]);
+  char ch2 = toupper (env_str[1]);
+
+  switch (ch1)
+    {
+    case 'T':
+      return OUTPUT_TEXT;
+    case 'S':
+      return OUTPUT_SILENT;
+    case 'E':
+      return OUTPUT_EXTRAS;
+    case 'C':
+      return OUTPUT_COUNT;
+    case 'D':
+      return OUTPUT_DOCNUMS;
+    case 'H':
+      if (ch2 == 'E')
+	return OUTPUT_HEADERS;
+      else
+	return OUTPUT_HILITE;
+    default:
+      FatalError (1, "Problem in output type switch");
+    }
+
+  return '\0';			/*shouldn't reach here -- keep compiler happy */
+}
+
+/* =========================================================================
+ * Function: get_query_type
+ * Description:
+ *      Map a query type string onto a char.
+ * Input:
+ * Output:
+ *      Query type char.
+ * ========================================================================= */
+
+char
+get_query_type (void)
+{
+  char *env_str = GetDefEnv ("query", "boolean");
+  char ch1 = toupper (env_str[0]);
+
+  switch (ch1)
+    {
+    case 'R':
+      return QUERY_RANKED;
+    case 'A':
+      return QUERY_APPROX;
+    case 'B':
+      return QUERY_BOOLEAN;
+    case 'D':
+      return QUERY_DOCNUMS;
+    default:
+      FatalError (1, "Problem in query type switch");
+    }
+
+  return '\0';			/*shouldn't reach here */
+}
+
+
+
+/*
+ * This initialises certain environment variables
+ *
+ */
+void 
+InitEnv (void)
+{
+  SetEnv ("hilite_style", "underline", HiliteStyleCons);	/*[TS:Sep/94] */
+  SetEnv ("briefstats", "off", BooleanCons);
+  SetEnv ("diskstats", "off", BooleanCons);
+  SetEnv ("expert", "false", BooleanCons);
+  SetEnv ("mgdir", getenv ("MGDATA") ? getenv ("MGDATA") : ".", NULL);
+  SetEnv ("mgname", "", NULL);
+  SetEnv ("maxdocs", "all", MaxDocsCons);
+  SetEnv ("memstats", "off", BooleanCons);
+  SetEnv ("mode", "text", OutputTypeCons);
+  SetEnv ("pager", getenv ("PAGER") ? getenv ("PAGER") : "more", NULL);
+  SetEnv ("qfreq", "true", BooleanCons);
+  SetEnv ("query", "boolean", QueryCons);
+  SetEnv ("sizestats", "off", BooleanCons);
+  SetEnv ("timestats", "off", BooleanCons);
+  SetEnv ("verbatim", "off", BooleanCons);
+  SetEnv ("sorted_terms", "on", BooleanCons);
+  SetEnv ("accumulator_method", "array", AccumCons);
+  SetEnv ("stop_at_max_accum", "off", BooleanCons);
+  SetEnv ("buffer", "1048576", BufferCons);
+  SetEnv ("max_accumulators", "50000", MaxNodesCons);
+  SetEnv ("max_terms", "all", MaxTermsCons);
+  SetEnv ("maxparas", "1000", MaxParasCons);
+  SetEnv ("hash_tbl_size", "1000", MaxHashCons);
+  SetEnv ("skip_dump", "skips.%d", NULL);
+  SetEnv ("ranked_doc_sepstr", "---------------------------------- %n %w\\n", NULL);
+  SetEnv ("doc_sepstr", "---------------------------------- %n\\n", NULL);
+  SetEnv ("para_sepstr", "\\n######## PARAGRAPH %n ########\\n", NULL);
+  SetEnv ("para_start", "***** Weight = %w *****\\n", NULL);
+  SetEnv ("terminator", "", NULL);
+  SetEnv ("heads_length", "50", MaxHeadsCons);
+  SetEnv ("optimise_type", "1", OptimiseCons);	/*[TS:Mar/95] */
+  SetEnv ("casefold", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("stem", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("term_freq", "off", BooleanCons);  /* [RPAP - Feb 97: Term Frequency] */
+  SetEnv ("maxnumeric", "4", MaxNumericCons);  /* [sjboddie - Jun 2002: Max Numeric word length] */
+}
+
+
+void 
+UninitEnv (void)
+{
+  char *name;
+  while (PopEnv () == 0);
+  while ((name = GetEnvName (0)) != 0)
+    UnsetEnv (name, 1);
+}
Index: /main/tags/3.01/indexers/mg/src/text/environment.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/environment.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/environment.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * environment.h -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef ENVIRONMENT_H
+#define ENVIRONMENT_H
+
+#define OUTPUT_TEXT     'T'
+#define OUTPUT_HEADERS  'H'
+#define OUTPUT_SILENT   'S'
+#define OUTPUT_EXTRAS   'E'
+#define OUTPUT_HILITE   'I'
+#define OUTPUT_COUNT    'C'
+#define OUTPUT_DOCNUMS  'D'
+
+#define QUERY_RANKED    'R'
+#define QUERY_APPROX    'A'
+#define QUERY_BOOLEAN   'B'
+#define QUERY_DOCNUMS   'D'
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/globals.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/globals.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/globals.h	(revision 21112)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * globals.h -- Some global stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_GLOBALS
+#define H_GLOBALS
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/hash.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/hash.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/hash.h	(revision 21112)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_HASH
+#define H_HASH
+
+
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; len--)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/help.mg.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/help.mg.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/help.mg.h	(revision 21112)
@@ -0,0 +1,320 @@
+static char *help_str[] = {
+"\n",
+"			  HELP for mgquery\n",
+"		   ================================\n",
+"\n",
+"The text is a summary of the information in the \"mgquery\" manual pages.\n",
+"\n",
+"The input to 'mgquery' consists of a series of input lines. The backslash\n",
+"character (\"\\\") is used at the end of lines to indicate that input \n",
+"continues on the next line.\n",
+"\n",
+"Input lines on which the first character is a dot (\".\") are commands to\n",
+"the mgquery program. Input lines that do not start with a dot are queries.\n",
+"\n",
+"A query consists of two parts. One part is a boolean or ranked query that\n",
+"identifies documents. The second part is a post-processing pattern matching\n",
+"operation. Any text between the first speech mark (\") and the last speech\n",
+"mark is considered to the a post-processing pattern. \n",
+"\n",
+"\n",
+"The following command are available :-\n",
+"\n",
+"    .help	     - displays this text.\n",
+"    .quit            - quits the program.\n",
+"    .set name value  - sets parameter \"name\" to \"value\" . If the parameter\n",
+"                       is a boolean parameter and value is omitted the\n",
+"	    	       parameter will be inverted (i.e. if it is true it\n",
+"		       will change to false, if it is false it will change\n",
+"		       to true).\n",
+"    .unset name      - deletes parameter \"name\"\n",
+"    .reset           - sets all the parameters to their initial state.\n",
+"    .display         - displays the values of all the current parameters.\n",
+"    .push            - pushes the current parameters on to a stack.\n",
+"    .pop             - destroys the current parameters and pops a new set\n",
+"                       of parameters off the stack.\n",
+"    .output arg      - This is used to specify where to send the documents.\n",
+"                       Arg may one be of the following:\n",
+"		       > filename  : Send output to the specified file.\n",
+"		       >> filename : Append output to the specified file.\n",
+"                       | command   : The output is piped into command, \n",
+"                                     which is executed by sh.\n",
+"    .input arg       - This is used to specify where input comes from.\n",
+"                       Arg may one be of the following:\n",
+"		       < filename  : Get the input from the specified\n",
+"                                     file.\n",
+"                       | command   : The input comes from the standard\n",
+"                                     output of command, which is executed\n",
+"                                     by sh.\n",
+"\n",
+"On startup the mgquery program reads from the file .mgrc a sequence \n",
+"of commands (NOTE: The .mgrc file may not contain any queries). mgquery \n",
+"first looks for .mgrc in the current directory and then in the users home \n",
+"directory. Lines starting with a '#' in the .mgrc file are considered to\n",
+"be comments and are ignored.\n",
+"\n",
+"The following parameters (used in the .set and .unset commands) are \n",
+"predefined and have special significance :-\n",
+"\n",
+"accumulator_method = `array'\n",
+"	This parameter is used during ranking, and specifies how the\n",
+"	weight for each document should be accumulated. The following\n",
+"	methods are available `array', `splay_tree', `hash_table', and\n",
+"	`list'.\n",
+"\n",
+"briefstats = `off'\n",
+"	This is a boolean parameter that determines whether the the\n",
+"	totals for disk, memory and time usage statistics will be \n",
+"	displayed. at the end of each query. \n",
+"	NOTE: this takes precedence over the parameters \"diskstats\",\n",
+"	\"memstats\" and \"timestats\". This parameter may take the values \n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"buffer = `1048576'\n",
+"	When the documents are being read in they are read into a \n",
+"	buffer of this size and then displayed from this buffer. If \n",
+"        the documents are larger than this buffer the buffer is\n",
+"	expanded automatically. Having a large buffer gives a very\n",
+"	slight performance improvement because it allows the order of \n",
+"	disk operations to be optimised. The buffer size is measured\n",
+"	in bytes.\n",
+"\n",
+"diskstats = `off'\n",
+"	This is a boolean parameter that determines whether the disk\n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"doc_sepstr = `---------------------------------- %n\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `boolean' or `docnums'\n",
+"	queries. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the MG document number use\n",
+"	the sequence `%n'.\n",
+"\n",
+"expert = `false'\n",
+"	If this is true then a lot of the waffle that the program \n",
+"	spits out is suppressed. This parameter may take the values\n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"hash_tbl_size = `1000'\n",
+"	One of the options during ranking queries is to use a hash \n",
+"	table to accumulate the weights for each document. The hash \n",
+"	table is a simple chained type. This parameter specifies the \n",
+"	size of the hash table and may take any value between 8 and\n",
+"	268435456. \n",
+"\n",
+"heads_length = `50'\n",
+"	When the mode is `heads' this specifies the number of \n",
+"	characters that	will be output for each document.\n",
+"\n",
+"maxdocs = `all'\n",
+"	The maximum number of documents to display in response to a\n",
+"	query. This parameter may take on a numeric value between 1 \n",
+"	and 429467295 or the word `all'.\n",
+"\n",
+"maxparas = `1000'\n",
+"	The maximum number of paragraphs to identify during a ranked\n",
+"	query with paragraph indexing. After the paragraphs have been\n",
+"	identified the paragraphs are converted into documents, and \n",
+"	because some of the paragraphs may refer to the same documents \n",
+"	the final number of answers may be less that maxparas. The \n",
+"	maxdocs parameter will then be applied. This parameter may \n",
+"	take on a numeric value between 1 and 429467295.\n",
+"\n",
+"max_accumulators = `50000'\n",
+"	This parameter limits the number of different paragraph/\n",
+"	document numbers to be accumulated during ranked queries when \n",
+"	the parameter `accumulator_method` is set to `splay_tree',\n",
+"	`hash_table', or `list'.  This parameter may take any value \n",
+"	between 8 and 268435456.\n",
+"\n",
+"max_terms = `all'\n",
+"	This parameter limits the number of terms that will actually\n",
+"	be used during a ranked query. If more terms than the number\n",
+"	specified by max_terms are entered, then the extra terms will\n",
+"	be discarded. If `sorted_terms' is on then the limiting will \n",
+"	be done after the terms have been sorted. This parameter may\n",
+"	take any value between 1 and 429467295 or the word `all'.\n",
+"\n",
+"memstats = `off'\n",
+"	This is a boolean parameter that determines whether the memory \n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes', \n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"mgdir = `.'\n",
+"	This specifies the directory where the MG files may be found.\n",
+"	If the environment variable `MGDATA' is set then `mgdir' is\n",
+"	initialised to the value in `MGDATA'.\n",
+"\n",
+"mgname = `'\n",
+"	This specifies the name of the MG database to process.\n",
+"\n",
+"mode = `text'\n",
+"	This specifies how documents should be displayed when they\n",
+"	are retrieved it may take four different values `text', \n",
+"	`docnums', `silent', `heads' or `count'. `text' displays \n",
+"	the contents of the document. `docnums' displays only the\n",
+"	document numbers. `Silent' retrieves all the documents but\n",
+"	displays nothing except how many documents were retrieved.\n",
+"	This mode is intended to be used in timing experiments. \n",
+"	`Heads` is used to print out the head of each document.\n",
+"	`Count' does the minimum amount of work required to determine\n",
+"	how many documents would be retrieved, but does not retrieve\n",
+"	them.\n",
+"\n",
+"pager = `more'\n",
+"	This is the name of the program that will be used to display\n",
+"	the help and the retrieved documents. If the environment \n",
+"	variable \"PAGER\" is defined then `pager' takes on that value.\n",
+"\n",
+"para_sepstr = `\\n######## PARAGRAPH %n ########\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	paragraphs. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the paragraph number within\n",
+"	the document use the sequence `%n'.\n",
+"\n",
+"para_start = `***** Weight = %w *****\\n'\n",
+"	This specifies the string that will be used at the head of \n",
+"	paragraphs for a paraghaph level index following a ranked query.\n",
+"	The standard C escape character sequences (see the man page)\n",
+"	may be used to place special characters in the string. For \n",
+"	example, a newline would the `\\n'. To include a `%' use the\n",
+"	sequence `%%'. To include the paragraph weight use the \n",
+"	sequence `%w'.\n",
+"\n",
+"qfreq = `true'\n",
+"	This determine whether the ranked queries will take into \n",
+"	account the number of times each query term is specified.\n",
+"	When this is `true' the number of times a term appears in\n",
+"	the query is used in the ranking. When this is `false' all \n",
+"	query term are assumed to occur only once. This parameter\n",
+"	may take the values `yes', `no', `true', `false', `on' or\n",
+"	`off'.\n",
+"\n",
+"query = `boolean'\n",
+"	This specifies the type of queries that are to be specified.\n",
+"	It can take four different values `boolean', `ranked', \n",
+"	`docnums' or `approx-ranked'. \n",
+"\n",
+"	`boolean' is for boolean queries. \n",
+"		The yacc grammar for boolean queries is as follows :-\n",
+"\n",
+"		query   : or;\n",
+" \n",
+"		or      : or '|' and\n",
+"		        | and ;	\n",
+" \n",
+"		and     : and '&' not\n",
+"        		| and not\n",
+"        		| not ;\n",
+"\n",
+"		not     : term\n",
+"		        | '!' not ;\n",
+" \n",
+"		term    : TERM\n",
+"		        | '(' or ')' ;\n",
+" \n",
+"\n",
+"\n",
+"\n",
+"	`ranked' and `approx-ranked' are for queries ranked by the\n",
+"		cosine measure. `approx-ranked' uses only the low\n",
+"		precision document lengths, and therefore only\n",
+"		produces an approximation to full cosine ranking.\n",
+"\n",
+"		query   : TERM\n",
+"      			| query TERM ;\n",
+"	\n",
+"	`docnums' allows the entry of document numbers. Multiple \n",
+"		numbers separated by spaces may be specified\n",
+"		or ranges separated by hyphens.\n",
+"\n",
+"\n",
+"		query   : range\n",
+"			| query range ;\n",
+"\n",
+"\n",
+"		range   : num\n",
+"			| num '-' num ;\n",
+"\n",
+"\n",
+"ranked_doc_sepstr = `---------------------------------- %n %w\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `ranked' or \n",
+"	`approx-ranked'	queries. The standard C escape character \n",
+"	sequences (see the man page) may be used to place special\n",
+"	characters in the string. For example, a newline would the\n",
+"	`\\n'. To include a `%' use the sequence `%%'. To include the\n",
+"	MG document number use the sequence `%n'. To include the\n",
+"	document weight use the sequence `%w'.\n",
+"\n",
+"sizestats = `false'\n",
+"	If this is true then various numbers are output at the end\n",
+"	of each query indicating what went on during the query. \n",
+"	This parameter may take the values `yes', `no', `true', \n",
+"	`false', `on' or `off'.\n",
+"\n",
+"skip_dump = `skips.%d'\n",
+"	If this parameter is set then during ranked queries on skipped\n",
+"	inverted files when `accumulator_method' is set to `splay_tree',\n",
+"	`hash_table', or `list` a file will be produced in the current\n",
+"	directory. The name of the file is the value of this parameter,\n",
+"	a `%d' in the file name will be replaced with the process id of\n",
+"	mgquery. This file will contain information about the usage of\n",
+"	skips during the query processing. This option is expensive; \n",
+"	use `.unset skip_dump' to obtain optimal performance.\n",
+"\n",
+"sorted_terms = `on'\n",
+"	This specifies whether of not the terms should be sorted into\n",
+"	decreasing occurrence in documents so that the least often\n",
+"	occurring terms are processed first when ranked queries are\n",
+"	being done. When this is true the terms are sorted. When this\n",
+"	is false the terms are not sorted and are instead processed in\n",
+"	order of occurrence. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"\n",
+"stop_at_max_accum = `on'\n",
+"	This specifies what should happen when the maximum number of\n",
+"	accumulators set by `max_accumulators' is reached. When this\n",
+"	is true the the processing of terms is stopped at the completion\n",
+"	of the current term. When this is false processing continues but\n",
+"	no new accumulators are created. This parameter may take the \n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"terminator = `'\n",
+"	This specifies the string that will be output after the	last\n",
+"	document from the previous query has been output. The standard\n",
+"	C escape character sequences (see the man page) may be used to\n",
+"	place special characters in the string. For example, a newline\n",
+"	would the `\\n'. To include a `%' use the sequence `%%'.\n",
+"	\n",
+"\n",
+"timestats = `false'\n",
+"	If this is true then the time to process a query is displayed\n",
+"	in both real time and CPU time. This parameter may take the\n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"verbatim = `off'\n",
+"	This is a boolean parameter that determines whether the program\n",
+"	should attempt to do a regular expression match on the retrieved\n",
+"	text. If verbatim is `on' and a post-processing strng is specified\n",
+"	with the query then the post-processing string will be searched for\n",
+"	in the documents just before they are displayed. If the string is\n",
+"	found the document will be displayed, if not the document will not\n",
+"	be displayed. If verbatim is `off' the post-processing string will\n",
+"	be considered a regular expression like in `vi' or `egrep'.\n",
+"	E.G. If verbatim is `on', \"and.*the\" will look for the 8 character\n",
+"	     sequence \"and.*the\". If verbatim is `off', \"and.*the\" will\n",
+"	     look for the sequence \"and\" followed somewhere later in the\n",
+"	     document by the sequence \"the\".\n",
+"	This parameter may take the values `yes', `no', `true', `false',\n",
+"	`on' or `off'.\n",
+"\n",
+"\n"};
Index: /main/tags/3.01/indexers/mg/src/text/help.mg.src
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/help.mg.src	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/help.mg.src	(revision 21112)
@@ -0,0 +1,347 @@
+###########################################################################
+#
+# help.mg.src -- Source for the help command
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)help.mg.src	1.8 21 Mar 1994
+#
+###########################################################################
+#
+# The help file for mgquery.  
+# Lines starting with '#' are treated as comments and are discarded when
+# the 'help.mg.h' file is produced
+#
+###########################################################################
+
+			  HELP for mgquery
+		   ================================
+
+The text is a summary of the information in the "mgquery" manual pages.
+
+The input to 'mgquery' consists of a series of input lines. The backslash
+character ("\") is used at the end of lines to indicate that input 
+continues on the next line.
+
+Input lines on which the first character is a dot (".") are commands to
+the mgquery program. Input lines that do not start with a dot are queries.
+
+A query consists of two parts. One part is a boolean or ranked query that
+identifies documents. The second part is a post-processing pattern matching
+operation. Any text between the first speech mark (") and the last speech
+mark is considered to the a post-processing pattern. 
+
+
+The following command are available :-
+
+    .help	     - displays this text.
+    .quit            - quits the program.
+    .set name value  - sets parameter "name" to "value" . If the parameter
+                       is a boolean parameter and value is omitted the
+	    	       parameter will be inverted (i.e. if it is true it
+		       will change to false, if it is false it will change
+		       to true).
+    .unset name      - deletes parameter "name"
+    .reset           - sets all the parameters to their initial state.
+    .display         - displays the values of all the current parameters.
+    .push            - pushes the current parameters on to a stack.
+    .pop             - destroys the current parameters and pops a new set
+                       of parameters off the stack.
+    .output arg      - This is used to specify where to send the documents.
+                       Arg may one be of the following:
+		       > filename  : Send output to the specified file.
+		       >> filename : Append output to the specified file.
+                       | command   : The output is piped into command, 
+                                     which is executed by sh.
+    .input arg       - This is used to specify where input comes from.
+                       Arg may one be of the following:
+		       < filename  : Get the input from the specified
+                                     file.
+                       | command   : The input comes from the standard
+                                     output of command, which is executed
+                                     by sh.
+
+On startup the mgquery program reads from the file .mgrc a sequence 
+of commands (NOTE: The .mgrc file may not contain any queries). mgquery 
+first looks for .mgrc in the current directory and then in the users home 
+directory. Lines starting with a '#' in the .mgrc file are considered to
+be comments and are ignored.
+
+The following parameters (used in the .set and .unset commands) are 
+predefined and have special significance :-
+
+accumulator_method = `array'
+	This parameter is used during ranking, and specifies how the
+	weight for each document should be accumulated. The following
+	methods are available `array', `splay_tree', `hash_table', and
+	`list'.
+
+briefstats = `off'
+	This is a boolean parameter that determines whether the the
+	totals for disk, memory and time usage statistics will be 
+	displayed. at the end of each query. 
+	NOTE: this takes precedence over the parameters "diskstats",
+	"memstats" and "timestats". This parameter may take the values 
+	`yes', `no', `true', `false', `on' or `off'.
+
+buffer = `1048576'
+	When the documents are being read in they are read into a 
+	buffer of this size and then displayed from this buffer. If 
+        the documents are larger than this buffer the buffer is
+	expanded automatically. Having a large buffer gives a very
+	slight performance improvement because it allows the order of 
+	disk operations to be optimised. The buffer size is measured
+	in bytes.
+
+diskstats = `off'
+	This is a boolean parameter that determines whether the disk
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+doc_sepstr = `---------------------------------- %n\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `boolean' or `docnums'
+	queries. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the MG document number use
+	the sequence `%n'.
+
+expert = `false'
+	If this is true then a lot of the waffle that the program 
+	spits out is suppressed. This parameter may take the values
+	`yes', `no', `true', `false', `on' or `off'.
+
+hash_tbl_size = `1000'
+	One of the options during ranking queries is to use a hash 
+	table to accumulate the weights for each document. The hash 
+	table is a simple chained type. This parameter specifies the 
+	size of the hash table and may take any value between 8 and
+	268435456. 
+
+heads_length = `50'
+	When the mode is `heads' this specifies the number of 
+	characters that	will be output for each document.
+
+maxdocs = `all'
+	The maximum number of documents to display in response to a
+	query. This parameter may take on a numeric value between 1 
+	and 429467295 or the word `all'.
+
+maxparas = `1000'
+	The maximum number of paragraphs to identify during a ranked
+	query with paragraph indexing. After the paragraphs have been
+	identified the paragraphs are converted into documents, and 
+	because some of the paragraphs may refer to the same documents 
+	the final number of answers may be less that maxparas. The 
+	maxdocs parameter will then be applied. This parameter may 
+	take on a numeric value between 1 and 429467295.
+
+max_accumulators = `50000'
+	This parameter limits the number of different paragraph/
+	document numbers to be accumulated during ranked queries when 
+	the parameter `accumulator_method` is set to `splay_tree',
+	`hash_table', or `list'.  This parameter may take any value 
+	between 8 and 268435456.
+
+max_terms = `all'
+	This parameter limits the number of terms that will actually
+	be used during a ranked query. If more terms than the number
+	specified by max_terms are entered, then the extra terms will
+	be discarded. If `sorted_terms' is on then the limiting will 
+	be done after the terms have been sorted. This parameter may
+	take any value between 1 and 429467295 or the word `all'.
+
+memstats = `off'
+	This is a boolean parameter that determines whether the memory 
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes', 
+	`no', `true', `false', `on' or `off'.
+
+mgdir = `.'
+	This specifies the directory where the MG files may be found.
+	If the environment variable `MGDATA' is set then `mgdir' is
+	initialised to the value in `MGDATA'.
+
+mgname = `'
+	This specifies the name of the MG database to process.
+
+mode = `text'
+	This specifies how documents should be displayed when they
+	are retrieved it may take four different values `text', 
+	`docnums', `silent', `heads' or `count'. `text' displays 
+	the contents of the document. `docnums' displays only the
+	document numbers. `Silent' retrieves all the documents but
+	displays nothing except how many documents were retrieved.
+	This mode is intended to be used in timing experiments. 
+	`Heads` is used to print out the head of each document.
+	`Count' does the minimum amount of work required to determine
+	how many documents would be retrieved, but does not retrieve
+	them.
+
+pager = `more'
+	This is the name of the program that will be used to display
+	the help and the retrieved documents. If the environment 
+	variable "PAGER" is defined then `pager' takes on that value.
+
+para_sepstr = `\n######## PARAGRAPH %n ########\n'
+	This specifies the string that will be used to separate 
+	paragraphs. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the paragraph number within
+	the document use the sequence `%n'.
+
+para_start = `***** Weight = %w *****\n'
+	This specifies the string that will be used at the head of 
+	paragraphs for a paraghaph level index following a ranked query.
+	The standard C escape character sequences (see the man page)
+	may be used to place special characters in the string. For 
+	example, a newline would the `\n'. To include a `%' use the
+	sequence `%%'. To include the paragraph weight use the 
+	sequence `%w'.
+
+qfreq = `true'
+	This determine whether the ranked queries will take into 
+	account the number of times each query term is specified.
+	When this is `true' the number of times a term appears in
+	the query is used in the ranking. When this is `false' all 
+	query term are assumed to occur only once. This parameter
+	may take the values `yes', `no', `true', `false', `on' or
+	`off'.
+
+query = `boolean'
+	This specifies the type of queries that are to be specified.
+	It can take four different values `boolean', `ranked', 
+	`docnums' or `approx-ranked'. 
+
+	`boolean' is for boolean queries. 
+		The yacc grammar for boolean queries is as follows :-
+
+		query   : or;
+ 
+		or      : or '|' and
+		        | and ;	
+ 
+		and     : and '&' not
+        		| and not
+        		| not ;
+
+		not     : term
+		        | '!' not ;
+ 
+		term    : TERM
+		        | '(' or ')' ;
+ 
+
+
+
+	`ranked' and `approx-ranked' are for queries ranked by the
+		cosine measure. `approx-ranked' uses only the low
+		precision document lengths, and therefore only
+		produces an approximation to full cosine ranking.
+
+		query   : TERM
+      			| query TERM ;
+	
+	`docnums' allows the entry of document numbers. Multiple 
+		numbers separated by spaces may be specified
+		or ranges separated by hyphens.
+
+
+		query   : range
+			| query range ;
+
+
+		range   : num
+			| num '-' num ;
+
+
+ranked_doc_sepstr = `---------------------------------- %n %w\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `ranked' or 
+	`approx-ranked'	queries. The standard C escape character 
+	sequences (see the man page) may be used to place special
+	characters in the string. For example, a newline would the
+	`\n'. To include a `%' use the sequence `%%'. To include the
+	MG document number use the sequence `%n'. To include the
+	document weight use the sequence `%w'.
+
+sizestats = `false'
+	If this is true then various numbers are output at the end
+	of each query indicating what went on during the query. 
+	This parameter may take the values `yes', `no', `true', 
+	`false', `on' or `off'.
+
+skip_dump = `skips.%d'
+	If this parameter is set then during ranked queries on skipped
+	inverted files when `accumulator_method' is set to `splay_tree',
+	`hash_table', or `list` a file will be produced in the current
+	directory. The name of the file is the value of this parameter,
+	a `%d' in the file name will be replaced with the process id of
+	mgquery. This file will contain information about the usage of
+	skips during the query processing. This option is expensive; 
+	use `.unset skip_dump' to obtain optimal performance.
+
+sorted_terms = `on'
+	This specifies whether of not the terms should be sorted into
+	decreasing occurrence in documents so that the least often
+	occurring terms are processed first when ranked queries are
+	being done. When this is true the terms are sorted. When this
+	is false the terms are not sorted and are instead processed in
+	order of occurrence. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+
+stop_at_max_accum = `on'
+	This specifies what should happen when the maximum number of
+	accumulators set by `max_accumulators' is reached. When this
+	is true the the processing of terms is stopped at the completion
+	of the current term. When this is false processing continues but
+	no new accumulators are created. This parameter may take the 
+	values `yes', `no', `true', `false', `on' or `off'.
+
+terminator = `'
+	This specifies the string that will be output after the	last
+	document from the previous query has been output. The standard
+	C escape character sequences (see the man page) may be used to
+	place special characters in the string. For example, a newline
+	would the `\n'. To include a `%' use the sequence `%%'.
+	
+
+timestats = `false'
+	If this is true then the time to process a query is displayed
+	in both real time and CPU time. This parameter may take the
+	values `yes', `no', `true', `false', `on' or `off'.
+
+verbatim = `off'
+	This is a boolean parameter that determines whether the program
+	should attempt to do a regular expression match on the retrieved
+	text. If verbatim is `on' and a post-processing strng is specified
+	with the query then the post-processing string will be searched for
+	in the documents just before they are displayed. If the string is
+	found the document will be displayed, if not the document will not
+	be displayed. If verbatim is `off' the post-processing string will
+	be considered a regular expression like in `vi' or `egrep'.
+	E.G. If verbatim is `on', "and.*the" will look for the 8 character
+	     sequence "and.*the". If verbatim is `off', "and.*the" will
+	     look for the sequence "and" followed somewhere later in the
+	     document by the sequence "the".
+	This parameter may take the values `yes', `no', `true', `false',
+	`on' or `off'.
+
+
Index: /main/tags/3.01/indexers/mg/src/text/invf.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/invf.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/invf.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_INVF
+#define H_INVF
+
+/* NOTE: This does not include the magic number */
+struct invf_dict_header
+  {
+    unsigned long lookback;
+    unsigned long dict_size;
+    unsigned long total_bytes;
+    unsigned long index_string_bytes;
+    double input_bytes; /* [RJM 07/97: 4G limit] */
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+  };
+
+struct stem_dict_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+    unsigned long indexed;       /* [RPAP - Jan 97: Stem Index Change] */
+  };
+
+struct invf_file_header
+  {
+    unsigned long no_of_words;
+    unsigned long no_of_ptrs;
+    unsigned long skip_mode;
+    unsigned long params[16];
+    unsigned long InvfLevel;
+  };
+
+/* [RPAP - Jan 97: Stem Index Change] */
+struct stem_idx_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_words;
+  };
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/invf_get.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/invf_get.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/invf_get.c	(revision 21112)
@@ -0,0 +1,1666 @@
+/**************************************************************************
+ *
+ * invf_get.c -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "bitio_gen.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "sptree.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "invf_get.h"
+#include "mg_errors.h"
+
+#define SKIPPING
+#define SHORT_CUT
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    Invf_Doc_Entry ide[1];
+  };
+
+static int skip_header = 0;
+
+static char skip_header_text0[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of docnum/word_count entries decoded.\n"
+"# 6\tNumber of hits.\n"
+"# 7\tNumber of entries in the splay tree or hash table.\n"
+"# 8\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char skip_header_text1[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of skips taken in evaluating the inverted file entry.\n"
+"# 6\tNumber of docnum/word_count entries decoded.\n"
+"# 7\tNumber of hits.\n"
+"# 8\tNumber of entries in the splay tree or hash table.\n"
+"# 9\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+
+static char skip_header_text2[] =
+"#######################################################################\n"
+"#\n"
+"#  1\tWord number.\n"
+"#  2\tNumber of time the word occurs in the collection.\n"
+"#  3\tNumber of documents the word occurs in.\n"
+"#  4\tThe word.\n"
+"#  5\tSkip size.\n"
+"#  6\tNumber of skips taken in evaluating the inverted file entry.\n"
+"#  7\tNumber of docnum/word_count entries decoded.\n"
+"#  8\tNumber of hits.\n"
+"#  9\tNumber of entries in the splay tree or hash table.\n"
+"# 10\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char *skip_header_text[] =
+{skip_header_text0,
+ skip_header_text1,
+ skip_header_text2};
+
+
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static Hash_Table *
+HT_create (query_data * qd, unsigned max_entries,
+	   unsigned max_size)
+{
+  unsigned es, hts;
+  Hash_Table *HT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * (max_entries - 1);
+  hts = sizeof (Invf_Doc_EntryH *) * max_size;
+
+  HT = Xmalloc (es);
+  if (!HT)
+    return NULL;
+
+  HT->HashTable = Xmalloc (hts);
+  if (!(HT->HashTable))
+    {
+      Xfree (HT);
+      return NULL;
+    }
+
+  ChangeMemInUse (qd, es + hts);
+  bzero ((char *) (HT->HashTable), hts);
+  HT->max = max_entries;
+  HT->num = 0;
+  HT->Suplimentary_Entries = NULL;
+  HT->Suplimentary_Size = 0;
+  HT->Suplimentary_Num = 0;
+  HT->Head = NULL;
+  HT->size = max_size;
+  return HT;
+}
+
+
+static void 
+HT_Sup_create (query_data * qd, Hash_Table * HT, unsigned max_size)
+{
+  int es = sizeof (Invf_Doc_EntryH) * max_size;
+  HT->Suplimentary_Entries = Xmalloc (es);
+  if (!HT->Suplimentary_Entries)
+    return;
+  HT->Suplimentary_Size = max_size;
+  HT->Suplimentary_Num = 0;
+  ChangeMemInUse (qd, es);
+}
+
+static void 
+HT_Sup_Add (Hash_Table * HT, unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH *IDEH;
+  IDEH = &HT->Suplimentary_Entries[HT->Suplimentary_Num++];
+  IDEH->next = NULL;
+  IDEH->IDE.DocNum = DN;
+  IDEH->IDE.Sum = Sum;
+}
+
+Invf_Doc_Entry *
+HT_find (Hash_Table * HT, unsigned long DN)
+{
+  Invf_Doc_EntryH *IDEH;
+  if (!HT->HashTable)
+    FatalError (1, "Accessing the hash_table through HT_find after HT_sort");
+  IDEH = HT->HashTable[DN % HT->size];
+  while (IDEH)
+    {
+      if (IDEH->IDE.DocNum == DN)
+	return &(IDEH->IDE);
+      IDEH = IDEH->next;
+    }
+  return NULL;
+}
+
+
+static int 
+HT_insert (Hash_Table * HT,
+	   unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH **IDEH;
+  if (HT->num == HT->max)
+    return 0;
+  IDEH = &(HT->HashTable[DN % HT->size]);
+  while (*IDEH)
+    IDEH = &((*IDEH)->next);
+  *IDEH = &(HT->IDEH[HT->num++]);
+  (*IDEH)->next = NULL;
+  (*IDEH)->IDE.DocNum = DN;
+  (*IDEH)->IDE.Sum = Sum;
+  return 1;
+}
+
+
+void 
+HT_free (query_data * qd, Hash_Table * HT)
+{
+  unsigned es;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * HT->max +
+    sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  if (HT->Suplimentary_Entries)
+    {
+      Xfree (HT->Suplimentary_Entries);
+      es += sizeof (Invf_Doc_EntryH) * HT->Suplimentary_Size;
+    }
+  Xfree (HT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDEH_comp (const void *a, const void *b)
+{
+  const Invf_Doc_EntryH *A = a;
+  const Invf_Doc_EntryH *B = b;
+  return A->IDE.DocNum - B->IDE.DocNum;
+}
+
+static void 
+HT_sort (query_data * qd, Hash_Table * HT)
+{
+  int i;
+  Invf_Doc_EntryH *l, *r, **b;
+  unsigned es = sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  HT->size = 0;
+  HT->HashTable = NULL;
+  ChangeMemInUse (qd, -es);
+
+  qsort (HT->IDEH, HT->num, sizeof (Invf_Doc_EntryH), IDEH_comp);
+
+  for (i = 0; i < HT->num - 1; i++)
+    HT->IDEH[i].next = &HT->IDEH[i + 1];
+  HT->IDEH[i].next = NULL;
+  l = HT->IDEH;
+
+  if (HT->Suplimentary_Entries)
+    {
+      for (i = 0; i < HT->Suplimentary_Num - 1; i++)
+	HT->Suplimentary_Entries[i].next = &HT->Suplimentary_Entries[i + 1];
+      HT->Suplimentary_Entries[i].next = NULL;
+    }
+  r = HT->Suplimentary_Entries;
+
+  b = &HT->Head;
+
+  while (l || r)
+    {
+      if (r && (!l || (l->IDE.DocNum > r->IDE.DocNum)))
+	{
+	  *b = r;
+	  b = &r->next;
+	  r = *b;
+	}
+      else
+	{
+	  *b = l;
+	  b = &l->next;
+	  l = *b;
+	}
+    }
+  *b = NULL;
+}
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static List_Table *
+LT_create (query_data * qd, unsigned max_entries)
+{
+  unsigned es;
+  List_Table *LT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * (max_entries - 1);
+
+  LT = Xmalloc (es);
+  if (!LT)
+    return NULL;
+
+  ChangeMemInUse (qd, es);
+  LT->max = max_entries;
+  LT->num = 0;
+  return LT;
+}
+
+
+
+Invf_Doc_Entry *
+LT_find (List_Table * LT, unsigned long DN)
+{
+  int l = 0;
+  int r = LT->num - 1;
+  int m, c;
+  while (l <= r)
+    {
+      m = (l + r) / 2;
+      c = DN - LT->IDE[m].DocNum;
+      if (c < 0)
+	r = m - 1;
+      else if (c > 0)
+	l = m + 1;
+      else
+	return &LT->IDE[m];
+    }
+  return NULL;
+}
+
+static List_Table *
+LT_add (query_data * qd, List_Table * LT,
+	unsigned long DN, float Sum)
+{
+  Invf_Doc_Entry *ide;
+
+  /* increase the size of the list if there is not enough */
+  /* room for this entry */
+  if (LT->num == LT->max)
+    {
+      int old, new, max_entries;
+      old = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+      max_entries = LT->max + (LT->max >> 1);
+      new = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * max_entries;
+
+      LT = Xrealloc (LT, new);
+      if (!LT)
+	return NULL;
+
+      ChangeMemInUse (qd, new - old);
+      LT->max = max_entries;
+    }
+
+  /* add the entry to the end of the list */
+  ide = &LT->IDE[LT->num];
+  ide->DocNum = DN;
+  ide->Sum = Sum;
+  LT->num++;
+
+  return LT;
+}
+
+
+void 
+LT_free (query_data * qd, List_Table * LT)
+{
+  unsigned es;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+  Xfree (LT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDE_comp (const void *a, const void *b)
+{
+  const Invf_Doc_Entry *A = a;
+  const Invf_Doc_Entry *B = b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+LT_sort (List_Table * LT)
+{
+  qsort (LT->IDE, LT->num, sizeof (Invf_Doc_Entry), IDE_comp);
+}
+
+
+/* [RJM 07/97: Ranked Required Terms] */
+/* accumulators with a sum less than or equal to 0.0 are to be deleted */
+static void 
+LT_pack (List_Table * LT)
+{
+  int s, d = 0;
+  for (s = 0; s < LT->num; s++) {
+      if (LT->IDE[s].Sum <= 0.0) {
+	/* ignore this accumulator */
+
+      } else if (d > 0 && LT->IDE[s].DocNum == LT->IDE[d-1].DocNum) {
+	LT->IDE[d-1].Sum += LT->IDE[s].Sum;
+
+      } else {
+	if (d != s) LT->IDE[d] = LT->IDE[s];
+	d++;
+      }
+  }
+
+  if (LT->num) LT->num = d;
+}
+
+
+/**************************************************************************/
+
+
+
+invf_data *
+InitInvfFile (File * InvfFile, stemmed_dict * sd)
+{
+  invf_data *id;
+  if (!(id = Xmalloc (sizeof (invf_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return NULL;
+    }
+
+  id->InvfFile = InvfFile;
+
+  Fread (&id->ifh, sizeof (id->ifh), 1, InvfFile);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  {
+    int i;
+    
+    NTOHUL(id->ifh.no_of_words);
+    NTOHUL(id->ifh.no_of_ptrs);
+    NTOHUL(id->ifh.skip_mode);
+    for (i = 0; i < 16; i++)
+      NTOHUL(id->ifh.params[i]);
+    NTOHUL(id->ifh.InvfLevel);
+  }
+
+  id->N = sd->sdh.num_of_docs;
+  id->Nstatic = sd->sdh.static_num_of_docs;
+
+  return (id);
+}
+
+
+void 
+FreeInvfData (invf_data * id)
+{
+  Xfree (id);
+}
+
+/****************************************************************************/
+
+
+static void 
+CalcBlks (query_data * qd, WordEntry * we, int *blk,
+	  int *dn_blk, int *len_blk, int *k, int *p)
+{
+  *p = we->doc_count;
+
+  *blk = BIO_Bblock_Init (qd->id->Nstatic, *p);
+  switch (qd->id->ifh.skip_mode)
+    {
+    case 1:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[0])
+	  *k = 0;
+	else
+	  {
+	    *k = qd->id->ifh.params[0];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs, (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    case 2:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[1])
+	  *k = 0;
+	else
+	  {
+	    *k = (int) (2 * sqrt (((double) (*p)) / qd->id->ifh.params[0]));
+	    if (*k <= qd->id->ifh.params[1])
+	      *k = qd->id->ifh.params[1];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs,
+				       (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    default:
+      *k = 0;
+    }
+}
+
+
+/* =========================================================================
+ * Function: GetDocsOp
+ * Description: 
+ *      if op == op_term then 
+ *        look up term in invf and decode all its entries/documents
+ *      if op == op_and_terms then 
+ *        go thru L, testing if it exists in invf using skips  
+ *        if it does exist then overwrite the entry nearer the start of the list
+ *      if op == op_and_or_terms then
+ *        go thru L, testing if it exists in invf using skips  
+ *        BUT only mark ones which are found - keep list intact
+ *      if op == op_diff_terms then     >>>> NOT tested <<<<
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+DocList *
+GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L)
+{
+  u_char *buffer;
+  DocEntry *src = NULL, *dest = NULL;
+  int i;			/* index counter into inverted list */
+  unsigned long pos;
+  int next_start, next_doc_num, next_i;
+  int block_counter;		/* within counter in block */
+  int dn_blk = 0, len_blk = 0;
+  int skip_size, have_skipping;
+  int blk;			/* bblock control parameter */
+  int ft;			/* The number of documents the word occurs in */
+  unsigned long CurrDoc = 0;	/* NOTE: Document numbers start at 1 */
+  float Weight;
+
+  /* ... AND empty */
+  if (op == op_and_terms && (!L || !L->num))
+    {
+      if (L)
+	Xfree (L);
+      return MakeDocList (0);
+    }
+
+  if (op == op_term)
+    L = MakeDocList (we->doc_count);
+
+  CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &skip_size, &ft);
+  have_skipping = (skip_size != 0);
+  if (!(buffer = Xmalloc (we->invf_len)))
+    return MakeDocList (0);
+
+  ChangeMemInUse (qd, we->invf_len);
+
+  qd->num_of_terms++;
+
+  /* --- Read an inverted file entry for the given word --- */
+  Fseek (qd->File_invf, we->invf_ptr, 0);
+  Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+  /* Read from the buffer */
+  DECODE_START (buffer, we->invf_len);
+
+  Weight = qd->sd->sdh.num_of_words / (100 * log2 ((float) we->count + 1));
+
+  if (op != op_term)
+    dest = src = L->DE;
+  i = pos = 0;
+  next_i = next_start = next_doc_num = 0;
+  block_counter = skip_size - 1;
+
+  /* Go thru the elements of an inverted list for a term */
+  /* May not go sequentially if skipping is done */
+  while (i < ft && ((op == op_term) || src - L->DE < L->num))
+    {
+      unsigned long diff, Temp;
+
+      if (have_skipping)
+	{
+
+	  /* --- Skip to next block --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    {
+	    ShortCut:
+	      i = next_i;
+	      DECODE_SEEK (next_start);
+	      pos = next_start;
+	      CurrDoc = next_doc_num;
+	      qd->hops_taken++;
+	      block_counter = skip_size - 1;
+	    }
+
+	  /* --- Decode the skip block header --- */
+	  if (block_counter == skip_size - 1)
+	    {
+	      if (i + skip_size < ft)
+		{
+		  /* --- Decode the skip block header --- */
+		  int doc_diff, pos_diff;
+		  BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		  next_doc_num += doc_diff;
+		  next_i += skip_size;
+		  BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		  next_start = pos + pos_diff;
+		}
+	      else
+		next_doc_num = -1;
+	    }
+
+	  /* --- Do a skip if c-set doc is higher than block header doc --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    goto ShortCut;
+
+
+	}			/*if skipping */
+
+      /* --- increment CurrDoc --- */
+      if (have_skipping && block_counter == 0 && i != ft - 1)
+	CurrDoc = next_doc_num;
+      else
+	{
+	  BBLOCK_DECODE_L (diff, blk, pos);
+	  CurrDoc += diff;
+	}
+
+      /* --- If have f_dt's then eat them up --- */
+      if (qd->id->ifh.InvfLevel >= 2)
+	GAMMA_DECODE_L (Temp, pos);	/* discard word count */
+
+      /* --- Increment counters --- */
+      qd->num_of_ptrs++;
+      i++;
+      block_counter--;
+      if (block_counter < 0)	/* ??? Why would this happen ? */
+	block_counter = skip_size - 1;
+
+      switch (op)
+	{
+	case op_term:
+	  /* --- Add the doc from invf onto the c-set list --- */
+	  L->DE[L->num].DocNum = CurrDoc;
+	  L->DE[L->num].Weight = Weight;
+	  L->DE[L->num].SeekPos = 0;
+	  L->DE[L->num].Len = 0;
+	  L->DE[L->num].CompTextBuffer = NULL;
+	  L->DE[L->num].Next = NULL;
+	  L->DE[L->num].or_included = 0;
+	  L->num++;
+	  break;
+
+	case op_and_terms:
+	case op_and_or_terms:
+	  /* --- see if have a match and save it --- */
+
+	  while (src < (L->DE + L->num) && CurrDoc > src->DocNum)
+	    {
+	      src++;
+	    }
+	  /* Unless at end of list, assert(CurrDoc <= src->DocNum) */
+
+	  /* --- Accept the doc as a candidate --- */
+	  if (src < (L->DE + L->num) && CurrDoc == src->DocNum)
+	    {
+	      if (op == op_and_terms)
+		{
+		  *dest++ = *src;
+		}
+	      else
+		{
+		  src->or_included = 1;
+		}
+	      src++;		/* Move onto next one */
+	    }
+
+	  /* If the next one to try has already been tried then move on again */
+	  if (op == op_and_or_terms)
+	    {
+	      while (src < (L->DE + L->num) && src->or_included)
+		src++;
+	    }
+
+	  break;
+
+	case op_diff_terms:
+	  while (src < (L->DE + L->num) && src->DocNum < CurrDoc)
+	    *dest++ = *src++;
+	  if (src < (L->DE + L->num) && src->DocNum == CurrDoc)
+	    src++;
+	  break;
+
+	default:
+	  break;
+	}			/*switch */
+
+    }				/*while traversing invf */
+
+  if (op == op_and_terms || op == op_diff_terms)
+    {
+      if (op == op_diff_terms)
+	while (src - L->DE < L->num)
+	  *dest++ = *src++;
+
+      /* update size of c-set */
+      L->num = dest - L->DE;
+    }
+  DECODE_DONE
+
+    Xfree (buffer);
+  ChangeMemInUse (qd, -we->invf_len);
+
+  return (L);
+}
+
+
+
+
+
+
+
+float *
+CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi)
+{
+  float *AccumulatedWeights;
+  int j, num_docs, num_terms;
+  int seen_must_match = 0; /* [RJM 07/97: Ranked Required Terms] says whether to 'and' queries or not */
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  if (!(AccumulatedWeights = Xmalloc (num_docs * sizeof (float))))
+      return (NULL);
+
+  bzero ((char *) AccumulatedWeights, sizeof (float) * num_docs);
+  ChangeMemInUse (qd, sizeof (float) * num_docs);
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      u_char *buffer;
+      int i, kd;
+      unsigned long next_mjr_dn = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long TempDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Terms] */
+      WordEntry *we = &Terms->TE[j].WE;
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      kd = 0;
+      for (i = 0; i < p; i++, kd++)
+	{
+	  int Count;		/* The number of times the word occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  if (kd == k)
+	    {
+	      kd = 0;
+	    }
+	  if (k && kd == 0)
+	    if (i + k < p)
+	      {
+		int temp;
+		BBLOCK_DECODE (next_mjr_dn, dn_blk);
+		next_mjr_dn += CurrDocNum;
+		BBLOCK_DECODE (temp, len_blk);
+	      }
+	  if (k && kd == k - 1 && i != p - 1)
+	    CurrDocNum = next_mjr_dn;
+	  else
+	    {
+	      BBLOCK_DECODE (diff, blk);
+	      CurrDocNum += diff;
+	    }
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE (Count);
+	  else
+	    Count = (double) we->count / p;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* zero the weights of intermediate documents if this term is required to match */
+	  if (require_match == 1) {
+	    for (TempDocNum = LastDocNum+1; TempDocNum < CurrDocNum; TempDocNum++)
+	      AccumulatedWeights[TempDocNum - 1] = 0.0;
+	  }
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if a term has been required to match before, the weight we are */
+	  /* adding to must be non-zero */
+	  if (!seen_must_match || (AccumulatedWeights[CurrDocNum - 1] > 0.0)) {
+	      AccumulatedWeights[CurrDocNum - 1] += Wqt * Wdt;
+	  }
+
+	  qd->num_of_ptrs++;
+	  LastDocNum = CurrDocNum; /* [RJM 07/97: Ranked Required Terms] */
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* zero the remaining weights if this term is required to match */
+      if (require_match == 1) {
+	for (TempDocNum = LastDocNum+1; TempDocNum <= num_docs; TempDocNum++)
+	  AccumulatedWeights[TempDocNum - 1] = 0.0;
+	seen_must_match = 1;
+      }
+    }
+  return AccumulatedWeights;
+}
+
+
+
+static int 
+doc_comp (void *a, void *b)
+{
+  return ((Invf_Doc_Entry *) a)->DocNum - ((Invf_Doc_Entry *) b)->DocNum;
+}
+
+
+static Invf_Doc_Entry *
+get_ide (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  struct pool *p = pool->pool;
+  if (!p || p->pos == p->length)
+    {
+      register unsigned size = sizeof (struct pool) +
+      (pool->pool_chunks - 1) * sizeof (Invf_Doc_Entry);
+      p = malloc (size);
+      if (!p)
+	return NULL;
+      ChangeMemInUse (qd, size);
+      p->length = pool->pool_chunks;
+      p->pos = 0;
+      p->next = pool->pool;
+      pool->pool = p;
+    }
+  return &(p->ide[p->pos++]);
+}
+
+
+void 
+free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  while (pool->pool)
+    {
+      struct pool *p = pool->pool;
+      pool->pool = p->next;
+      ChangeMemInUse (qd, -(sizeof (struct pool) + (p->length - 1) *
+			    sizeof (Invf_Doc_Entry)));
+      free (p);
+    }
+  pool->pool = NULL;
+}
+
+
+FILE *
+open_skip (char *sd)
+{
+  char buf[256];
+  if (sd == NULL || *sd == '\0')
+    return NULL;
+  sprintf (buf, sd, getpid ());
+  return fopen (buf, "a");
+}
+
+
+
+Splay_Tree *
+CosineDecodeSplay (query_data * qd, TermList * Terms,
+		   RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool)
+{
+  Splay_Tree *Doc_Set;
+  int j, num_docs, num_terms, MoreAccum;
+  int Anding = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  Doc_Set = SP_createset (doc_comp);
+
+  pool->pool = NULL;
+  pool->pool_chunks = 1024;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos, i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && Doc_Set->no_of_items >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  next = SP_get_first (Doc_Set);
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry ent, *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->DocNum < CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      if (next && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->DocNum <= CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      goto NextDocNum;
+	    }
+	  ent.DocNum = CurrDocNum - 1;
+	  if ((mem = SP_member (&ent, Doc_Set)) == NULL)
+	    {
+	      hits++;
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      if ((mem = get_ide (qd, pool)) == NULL)
+		return NULL;
+	      ChangeMemInUse (qd, sizeof (*mem));
+	      mem->DocNum = CurrDocNum - 1;
+	      mem->Sum = 0;
+	      ChangeMemInUse (qd, -Doc_Set->mem_in_use);
+	      if (SP_insert (mem, Doc_Set) == NULL)
+		{
+		  Message ("Unable to allocate memory for a splay node");
+		  Abort = 1;
+		  goto FastAbort;
+		}
+	      ChangeMemInUse (qd, Doc_Set->mem_in_use);
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+
+	  mem->Sum += Wqt * Wdt;
+	  total_added += Wqt * Wdt;
+
+	NextDocNum:
+	  ;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, Doc_Set->no_of_items, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  return Doc_Set;
+}
+
+
+
+
+Hash_Table *
+CosineDecodeHash (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  Hash_Table *HT;
+  int Anding = 0;
+  int Sorted = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  HT = HT_create (qd, rqi->MaxAccums, rqi->HashTblSize);
+  if (!HT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_EntryH *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  if (!Sorted)
+	    {
+	      HT_sort (qd, HT);
+	      Sorted = 1;
+	    }
+	  next = HT->Head;
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+#ifdef SHORT_CUT
+		ShortCut:
+#endif
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		{
+		  if (i + k < p)
+		    {
+		      int doc_diff, pos_diff;
+		      BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		      next_doc_num += doc_diff;
+		      next_i += k;
+		      BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		      next_start = pos + pos_diff;
+		    }
+		  else
+		    next_doc_num = -1;
+		}
+#ifdef SHORT_CUT
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		goto ShortCut;
+#else
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+#endif
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    {
+	      CurrDocNum = next_doc_num;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->IDE.DocNum < CurrDocNum - 1)
+		next = next->next;
+	      if (next && next->IDE.DocNum == CurrDocNum - 1)
+		{
+		  Wdt = Count * WordLog;
+		  next->IDE.Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->IDE.DocNum <= CurrDocNum - 1)
+		next = next->next;
+	      goto NextDocNum;
+	    }
+	  if ((mem = HT_find (HT, CurrDocNum - 1)) == NULL)
+	    {
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      Wdt = Count * WordLog;
+	      if (!HT_insert (HT, CurrDocNum - 1, Wqt * Wdt))
+		{
+		  if (!HT->Suplimentary_Entries)
+		    HT_Sup_create (qd, HT, we->doc_count - i + 1);
+		  HT_Sup_Add (HT, CurrDocNum - 1, Wqt * Wdt);
+		}
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+	  else
+	    {
+	      Wdt = Count * WordLog;
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+	  ;
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+	       ptrs_dec, hits, HT->num + HT->Suplimentary_Num, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  if (!HT->Head)
+    HT_sort (qd, HT);
+  return HT;
+}
+
+
+
+List_Table *
+CosineDecodeList (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  List_Table *LT;
+  int Anding = 0;
+  int Sorted = 0;
+  int has_grown = 0; /* [RJM 07/97: Ranked Required Match] whether elements have been added lately */
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+
+  /* Create the list for the documents */
+  LT = LT_create (qd, rqi->MaxAccums);
+  if (!LT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i, n = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0;	/* [RJM 07/97: Ranked Required Match] */
+      int this_item = 0; /* [RJM 07/97: Ranked Required Match] points to the next possible LT to delete */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Match] */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums)
+	MoreAccum = 0;
+
+      if ((rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums) || Anding)
+	{
+	  Anding = 1;
+	  if (!Sorted || has_grown)
+	    {
+	      LT_sort (LT);
+	      LT_pack (LT);
+	      Sorted = 1;
+	      has_grown = 0;
+	    }
+	  next = LT->IDE;
+	  n = 0;
+	}
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* make sure that the document list is set up correctly for */
+      /* requiring a term to match */
+      if (require_match == 1) {
+	if (!Sorted || has_grown) {
+	  LT_sort (LT);
+	  LT_pack (LT);
+	  Sorted = 1;
+	  has_grown = 0;
+	}
+	/* this_item is already set to zero */
+      }
+
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || n < LT->num))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+
+	  /* deal with skipping */
+	  if (k)
+	    {
+	      /* see if we should take a skip */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+
+	      /* should we take another skip? */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if this is a required match we must remove all accumulators */
+	  /* of intermediate documents */
+	  if (require_match == 1) {
+	    while (this_item < LT->num && LT->IDE[this_item].DocNum < CurrDocNum - 1) {
+	      if (LastDocNum==0 ||(LT->IDE[this_item].DocNum > LastDocNum - 1))
+		LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	      this_item++;
+	    }
+	  }
+
+	  if (Anding)
+	    {
+	      while (n < LT->num && next->DocNum < CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      if (n < LT->num && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (n < LT->num && next->DocNum <= CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      goto NextDocNum;
+	    }
+	  if (!Anding)
+	    {
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      LT = LT_add (qd, LT, CurrDocNum - 1, Wqt * Wdt);
+	      if (!LT)
+		{
+		  Abort = 1;
+		  goto FastAbort;
+		}
+
+	      /* the list of accumulators just got bigger */
+	      has_grown = 1;
+
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+
+
+	    }
+	  else if ((mem = LT_find (LT, CurrDocNum - 1)))
+	    {
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+
+	  LastDocNum = CurrDocNum;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, LT->num, total_added);
+	}
+      if (Abort)
+	break;
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* remove the remaining accumulators if this term is required to match */
+      if (require_match == 1) {
+	while (this_item < LT->num) {
+	  if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	    LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	  this_item++;
+	}
+	Anding = 1; /* no more documents should be added */
+
+	/* Get rid of any documents which have been deleted */
+	LT_pack (LT);
+      }
+    }
+
+  if (sk)
+    fclose (sk);
+  return LT;
+}
Index: /main/tags/3.01/indexers/mg/src/text/invf_get.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/invf_get.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/invf_get.h	(revision 21112)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * invf_get.h -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF_GET
+#define H_INVF_GET
+
+typedef struct Invf_Doc_Entry
+  {
+    int DocNum;
+    float Sum;
+  }
+Invf_Doc_Entry;
+
+typedef struct Invf_Doc_Entry_Pool
+  {
+    void *pool;
+    unsigned pool_chunks;
+  }
+Invf_Doc_Entry_Pool;
+
+typedef struct Invf_Doc_EntryH
+  {
+    Invf_Doc_Entry IDE;
+    struct Invf_Doc_EntryH *next;
+  }
+Invf_Doc_EntryH;
+
+typedef struct Hash_Table
+  {
+    Invf_Doc_EntryH *Suplimentary_Entries;
+    int Suplimentary_Size;
+    int Suplimentary_Num;
+    unsigned max;
+    unsigned num;
+    unsigned size;
+    Invf_Doc_EntryH **HashTable;
+    Invf_Doc_EntryH *Head;
+    Invf_Doc_EntryH IDEH[1];
+  }
+Hash_Table;
+
+typedef struct List_Table
+  {
+    unsigned max;
+    unsigned num;
+    Invf_Doc_Entry IDE[1];
+  }
+List_Table;
+
+typedef enum
+  {
+    op_term, op_and_terms, op_and_or_terms, op_diff_terms
+  }
+Op_types;
+
+/*Invf_Doc_Entry *HT_find(Hash_Table *HT, unsigned long DN); */
+void HT_free (query_data * qd, Hash_Table * HT);
+void LT_free (query_data * qd, List_Table * LT);
+
+invf_data *InitInvfFile (File * InvfFile, stemmed_dict * sd);
+void FreeInvfData (invf_data * id);
+
+DocList *GetDocs (query_data * qd, WordEntry * we, BooleanQueryInfo * bqi);
+
+float *CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi);
+
+void free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool);
+
+Splay_Tree *CosineDecodeSplay (query_data * qd, TermList * Terms,
+			 RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool);
+
+Hash_Table *CosineDecodeHash (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+List_Table *CosineDecodeList (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+
+
+DocList *GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L);
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/ivf.pass1.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/ivf.pass1.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/ivf.pass1.c	(revision 21112)
@@ -0,0 +1,866 @@
+/**************************************************************************
+ *
+ * ivf.pass1.c -- Memory efficient pass 1 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:50  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.6  1995/01/16  03:57:05  tes
+   * Fixed bug for index_string_bytes which was calculated
+   * by adding the suffix lengths except for every lookback (block)
+   * number of words. This was incorrect as invf.dict is fully
+   * front coded - perhaps it was partially coded in the past.
+   * This would only affect a statistic - hence the reason why the
+   * bug could exist ;-)
+   *
+   * Revision 1.5  1994/11/29  00:31:59  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:48  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:34  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long bit counts for inverted file
+ *     (1999-08-05 Tim Bell <bhat@cs.mu.oz.au>)
+ */
+static char *RCSID = "$Id$";
+
+#define LOGLOOKBACK	2
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define PARA_DIV 1.5
+#define NORM_DIV 1.0
+
+
+typedef struct hash_rec
+  {
+    unsigned long fcnt;		/* fragment frequency */
+    unsigned long lfcnt;	/* local fragment frequency */
+    unsigned long fnum;		/* last fragment to use stem */
+    unsigned long wcnt;		/* stem frequency */
+    unsigned long lwcnt;	/* local stem frequency */
+    u_char *word;
+  }
+hash_rec;
+
+static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0;
+static unsigned long outputbytes = 0;
+static unsigned long inputbytes = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static unsigned long ChunksWritten = 0;
+
+
+static FILE *ic;		/* The invf block file */
+static stdio_bitio_state sbs;
+
+static hash_rec **HashTable;
+static unsigned long HashSize=0;
+static unsigned long HashUsed=0;
+static u_char *Pool;
+static int PoolLeft=0;
+
+static hash_rec *hr_pool;
+static int hr_PoolLeft=0;
+
+static hash_rec **first_occr;
+static int max_first_occr=0;
+
+static mg_ullong L1_bits = 0;
+static mg_ullong L2_bits = 0;
+static mg_ullong L3_bits = 0;
+static unsigned long L1_ohead = 0;
+static unsigned long L2_ohead = 0;
+static unsigned long L3_ohead = 0;
+static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0;
+static unsigned long ptrcnt = 0;
+static unsigned long checknum=0;
+static long max_mem = 0;
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+void
+ResetStaticI1Vars () 
+{
+  words_read = 0;
+  words_diff = 0; 
+  bytes_diff = 0;
+  outputbytes = 0;
+  inputbytes = 0;
+  MaxMemInUse = 0;
+  MemInUse = 0;
+  ChunksWritten = 0;
+  
+  HashSize=0;
+  HashUsed=0;
+  PoolLeft=0;
+  
+  hr_PoolLeft=0;
+  
+  max_first_occr=0;
+  
+  L1_bits = 0;
+  L2_bits = 0;
+  L3_bits = 0;
+  L1_ohead = 0;
+  L2_ohead = 0;
+  L3_ohead = 0;
+  callnum = 0;
+  lcallnum = 0;
+  wordnum = 0;
+  lwordnum = 0;
+  ptrcnt = 0;
+  checknum=0;
+  max_mem = 0;
+  
+}
+int 
+init_ivf_1 (char *file_name)
+{
+  ResetStaticI1Vars();
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fwrite ("    ", sizeof (u_long), 1, ic);	/* Space for the maxmem */
+  ENCODE_START (ic)
+    ENCODE_PAUSE (sbs)
+
+    if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (hash_rec))))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+
+  HashSize = INITIAL_HASH_SIZE;
+  HashUsed = 0;
+  if (!(HashTable = Xmalloc (sizeof (hash_rec *) * HashSize)))
+    {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * HashSize);
+  bzero ((char *) HashTable, sizeof (hash_rec *) * HashSize);
+  max_first_occr = 8192;
+  if (!(first_occr = Xmalloc (sizeof (hash_rec *) * max_first_occr)))
+    {
+      Message ("Unable to allocate memory for first_occur");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * max_first_occr);
+
+  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+
+  return (COMPALLOK);
+}
+
+
+
+static unsigned long 
+mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? lwordnum : lcallnum; */
+  register unsigned long N = lcallnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->lwcnt : ent->lfcnt; */
+      register unsigned long p = ent->lfcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->lwcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+static unsigned long 
+max_mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? wordnum : callnum; */
+  register unsigned long N = callnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->wcnt : ent->fcnt; */
+      register unsigned long p = ent->fcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->wcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+static void 
+dump_dict (unsigned long mem)
+{
+  int i;
+
+  ChunksWritten++;
+
+  ENCODE_CONTINUE (sbs)
+
+    GAMMA_ENCODE (lcallnum + 1);
+  if (mem > max_mem)
+    max_mem = mem;
+  GAMMA_ENCODE (mem + 1);
+  lwordnum = lcallnum = 0;
+  GAMMA_ENCODE (HashUsed + 1);
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *ent = first_occr[i];
+      GAMMA_ENCODE (ent->lwcnt + 1);
+      if (ent->lwcnt >= 2)
+	GAMMA_ENCODE (ent->lfcnt);
+      ent->lwcnt = ent->lfcnt = 0;
+    }
+  ptrcnt = 0;
+  ENCODE_PAUSE (sbs)
+}
+
+
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  u_char *end = s_in + l_in - 1;
+
+  callnum++;
+  lcallnum++;
+  inputbytes += l_in;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      
+      stemmer (stem_method, stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      words_read++;
+      wordnum++;
+      lwordnum++;
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = HashTable[hashval];
+	    if (!ent)
+	      {
+		int len = *Word + 1;
+		if (!hr_PoolLeft)
+		  {
+		    if (!(hr_pool = Xmalloc (HASH_POOL_SIZE *
+					     sizeof (hash_rec))))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    hr_PoolLeft = HASH_POOL_SIZE;
+		    ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+		  }
+		ent = hr_pool++;
+		hr_PoolLeft--;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->wcnt = 1;
+		ent->fcnt = 1;
+		ent->lwcnt = 1;
+		ent->lfcnt = 1;
+		ent->fnum = callnum;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		if (HashUsed == max_first_occr - 1)
+		  {
+		    ChangeMem (-sizeof (hash_rec *) * max_first_occr);
+		    max_first_occr *= 2;
+		    if (!(first_occr = Xrealloc (first_occr, sizeof (hash_rec *) *
+						 max_first_occr)))
+		      {
+			Message ("Unable to allocate memory for first_occr");
+			return (COMPERROR);
+		      }
+		    ChangeMem (sizeof (hash_rec *) * max_first_occr);
+		  }
+		first_occr[HashUsed] = ent;
+		HashUsed++;
+		HashTable[hashval] = ent;
+		bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		ent->lwcnt++;
+		if (callnum > ent->fnum)
+		  {
+		    ptrcnt++;
+		    ent->fcnt++;
+		    ent->lfcnt++;
+		    ent->fnum = callnum;
+		  }
+		break;
+	      }
+	  }
+      }
+
+      if (HashUsed >= HashSize >> 1)
+	{
+	  hash_rec **ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec *) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  bzero ((char *) ht, sizeof (hash_rec *) * size);
+	  ChangeMem (sizeof (hash_rec *) * size);
+
+	  for (i = 0; i < HashSize; i++)
+	    if (HashTable[i])
+	      {
+		register u_char *wptr;
+		hash_rec *ent;
+		register unsigned long hashval, step;
+
+		wptr = HashTable[i]->word;
+		HASH (hashval, step, wptr, size);
+		ent = ht[hashval];
+		while (ent)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    ent = ht[hashval];
+		  }
+		ht[hashval] = HashTable[i];
+	      }
+	  Xfree (HashTable);
+	  ChangeMem (-sizeof (hash_rec *) * HashSize);
+	  HashTable = ht;
+	  HashSize = size;
+	}
+
+    }
+
+  if (ptrcnt >= checknum)
+    {
+      unsigned long mem;
+      /*fprintf(stderr, "Checking at %u . . . ", ptrcnt); */
+      mem = mem_reqd ();
+      if (mem >= invf_buffer_size * CHECK_CLOSE)
+	{
+	  /*fprintf(stderr, "Got a match\n"); */
+	  dump_dict (mem);
+	  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+	    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+	}
+      else
+	{
+	  checknum = checknum * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	    checknum;
+	  if (checknum <= ptrcnt)
+	    checknum = ptrcnt + 1;
+	  /*fprintf(stderr, "Next check at %u\n", checknum); */
+	}
+    }
+  return (COMPALLOK);
+
+}				/* encode */
+
+
+int 
+process_ivf_1 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	return process_doc (start, pos + s_in - start);
+    }
+  return COMPALLOK;
+}
+
+static int 
+PackHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < HashSize; s++)
+    if (HashTable[s])
+      HashTable[d++] = HashTable[s];
+  ChangeMem (-sizeof (hash_rec *) * HashSize);
+  ChangeMem (sizeof (hash_rec *) * HashUsed);
+  if (!(HashTable = Xrealloc (HashTable, sizeof (hash_rec *) * HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  HashSize = HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((hash_rec **) A))->word;
+  u_char *s2 = (*((hash_rec **) B))->word;
+
+  return (casecompare (s1, s2)); /* [RPAP - Jan 97: Stem Index Change] */
+}				/* stem_comp */
+
+
+
+
+
+/*
+ * void count_text()
+ *
+ * The maths used in this function is described in the paper "Coding for 
+ * Compression in Full-Text Retrieval Systems"
+ *
+ */
+static void 
+count_text ()
+{
+  int i;
+  mg_ullong oldL12_bits = 0;
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *wrd = HashTable[i];
+      /* estimate size of a level 1 inverted file */
+      L1_bits += (int) (0.99 + wrd->fcnt * (1.6 + log2 (1.0 * callnum / wrd->fcnt)));
+      L1_ohead += BIO_Gamma_Length (wrd->fcnt);
+
+      L2_bits += BIO_Unary_Length (wrd->wcnt);
+      L2_ohead += BIO_Gamma_Length (wrd->wcnt - wrd->fcnt + 1);
+
+      L3_bits += (int) (0.99 + wrd->wcnt *
+		  (1.6 + log2 (1.0 * words_read / (wrd->wcnt + callnum))));
+      L3_ohead += 0;
+
+      /* check for overflow */
+      if (L1_bits + L2_bits < oldL12_bits) {
+	fprintf(stderr, "ERROR: Inverted file size will probably overflow %d byte unsigned integer\n", sizeof (mg_ullong));
+	fprintf(stderr, "       counter in pass 2.\n");
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+      oldL12_bits = L1_bits + L2_bits;
+    }
+  L3_bits = (L3_bits + L2_bits + L1_bits + 7) / 8;
+  L3_ohead = (L3_ohead + L2_ohead + L1_ohead + 7) / 8;
+  L2_bits = (L2_bits + L1_bits + 7) / 8;
+  L2_ohead = (L2_ohead + L1_ohead + 7) / 8;
+  L1_bits = (L1_bits + 7) / 8;
+  L1_ohead = (L1_ohead + 7) / 8;
+}				/* count_text */
+
+
+
+/*
+ *    write_stem_file():
+ *              writes out the stemmed dictionary file
+ *              in the following format 
+ *                      lookback value (int)
+ *                      totalbytes value (int)
+ *                      indexstringbytes (int)
+ *                      for each word 
+ *                        wordlen (4 bits)
+ *                        prefix match (4 bits)
+ *                        word (wordlen bytes)
+ *                        word frequency (int)
+ *                        word count (int)
+ *
+ *      Accesses outside variables:     
+ *
+ *      Return value...:                
+ */
+
+static void 
+write_stem_file (char *file_name)
+{
+  long j;
+  struct invf_dict_header idh;
+  long lookback = (1 << LOGLOOKBACK);	/* ???? */
+  long totalbytes = 0;		/* The sum of the length of all words, including 
+				   the length byte */
+  long indexstringbytes = 0;	/* The amount of space required to store the 
+				   words in the diction, this takes into account 
+				   the prefixes */
+  u_char *lastword = NULL;	/* A pointer to the last word processed */
+  FILE *sp;
+
+
+  /* Calculate the size of certain things */
+  for (j = 0; j < HashSize; j++)
+    {
+      u_char *word = HashTable[j]->word;
+      indexstringbytes += word[0] + 2;
+      totalbytes += word[0] + 1;
+      if (lastword)
+	indexstringbytes -= prefixlen (lastword, word);
+      lastword = word;
+    }
+
+  lastword = NULL;
+
+  if (!(sp = create_file (file_name, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(lookback, idh.lookback);
+  HTONUL2(HashSize, idh.dict_size);
+  HTONUL2(totalbytes, idh.total_bytes);
+  HTONUL2(indexstringbytes, idh.index_string_bytes);
+  HTOND2(inputbytes, idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL2(callnum, idh.num_of_docs);
+  HTONUL2(callnum, idh.static_num_of_docs);
+  HTONUL2(words_read, idh.num_of_words);
+  HTONUL2(stemmer_num, idh.stemmer_num);
+  HTONUL2(stem_method, idh.stem_method);
+
+  fwrite ((char *) &idh, sizeof (idh), 1, sp);
+  outputbytes += sizeof (idh);
+
+  for (j = 0; j < HashSize; j++)
+    {
+      int i;
+      unsigned long wcnt, fcnt;  /* [RPAP - Jan 97: Endian Ordering] */
+      hash_rec *ent = HashTable[j];
+      if (lastword != NULL)
+	/* look for prefix match with prev string */
+	i = prefixlen (lastword, ent->word);
+      else
+	i = 0;
+      fputc (i, sp);
+      fputc (ent->word[0] - i, sp);
+      fwrite ((char *) ent->word + i + 1, sizeof (u_char), ent->word[0] - i, sp);
+      outputbytes += ent->word[0] - i + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->fcnt, fcnt);
+      fwrite ((char *) &(fcnt), sizeof (fcnt), 1, sp);
+
+      outputbytes += sizeof (ent->fcnt);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->wcnt, wcnt);
+      fwrite ((char *) &(wcnt), sizeof (wcnt), 1, sp);
+
+      outputbytes += sizeof (ent->wcnt);
+      lastword = ent->word;
+    }
+
+  fclose (sp);
+}				/* write_stem_file() */
+
+
+
+/*
+ *    write_codes():
+ *              calls functions to assign and write out codes and
+ *              then prints out stats about execution results.
+ *
+ *      Accesses outside variables:     z
+ *
+ *      Return value...:                z
+ */
+
+static void 
+write_codes (char *file_name)
+{
+  unsigned long dicts = 0;
+  outputbytes = 0;
+  write_stem_file (file_name);
+  dicts = outputbytes;
+
+#ifndef SILENT
+  Message ("Chunks written      : %d\n",
+	   ChunksWritten);
+  if (InvfLevel == 3)
+    Message ("Paragraphs          : %8d\n", callnum);
+  Message ("Peak memory usage   : %10.1f Mb\n",
+	   (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("Stems size          : %10.1f kB  %5.2f%%\n",
+	   (double) dicts / 1024, (double) 100 * dicts / inputbytes);
+  Message ("Lev 1 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L1_bits + L1_ohead) / 1024,
+	   (double) 100 * (L1_bits + L1_ohead) / inputbytes);
+  Message ("Lev 2 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L2_bits + L2_ohead) / 1024,
+	   (double) 100 * (L2_bits + L2_ohead) / inputbytes);
+  Message ("Lev 3 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L3_bits + L3_ohead) / 1024,
+	   (double) 100 * (L3_bits + L3_ohead) / inputbytes);
+  Message ("Record addresses    : %10.1f kB  %5.2f%%\n",
+	   (double) words_diff * 4 / 1024, (double) 100 * words_diff * 4 / inputbytes);
+#endif
+}				/* write_codes() */
+
+
+
+void 
+write_num_file (char *file_name)
+{
+
+  int i;
+  FILE *f;
+
+  for (i = 0; i < HashSize; i++)
+    HashTable[i]->fnum = i;
+
+  if (!(f = create_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+#if 1
+  ENCODE_START (f)
+
+    for (i = 0; i < HashSize; i++)
+    BINARY_ENCODE (first_occr[i]->fnum + 1, HashSize + 1);
+
+  ENCODE_DONE;
+#else
+  for (i = 0; i < HashSize; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      unsigned long fnum;
+      HTONUL2(first_occur[i]->fnum, fnum);
+      fwrite ((char *) &fnum, sizeof (unsigned long), 1, f);
+    }
+#endif
+
+  fclose (f);
+}
+
+int 
+done_ivf_1 (char *FileName)
+{
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+#ifndef SILENT
+  Message ("Mem reqd for 1 chunk: %8u bytes\n",
+	   max_mem_reqd ());
+#endif
+  if (lcallnum)
+    dump_dict (mem_reqd ());
+
+  if (PackHashTable () == COMPERROR)
+    return COMPERROR;
+
+  qsort (HashTable, HashUsed, sizeof (hash_rec *), ent_comp);
+
+  count_text ();
+  write_codes (FileName);
+
+  ENCODE_CONTINUE (sbs)
+    GAMMA_ENCODE (1);
+  ENCODE_DONE
+    fseek (ic, sizeof (long), 0);
+  
+  HTONSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&max_mem, sizeof (max_mem), 1, ic);
+  NTOHSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (ic);
+
+  write_num_file (FileName);
+
+  msg_prefix = temp_str;
+
+  /* free the memory */
+  Xfree(HashTable);
+  if (Pool) {
+    Pool -= (POOL_SIZE-PoolLeft);
+    Xfree(Pool);
+  }
+  if (hr_pool) {
+    hr_pool -= (HASH_POOL_SIZE-hr_PoolLeft);
+    Xfree(hr_pool);
+  }
+  if (first_occr) {
+    Xfree(first_occr);
+  }
+
+  return (COMPALLOK);
+}				/* done_encode */
Index: /main/tags/3.01/indexers/mg/src/text/ivf.pass2.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/ivf.pass2.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/ivf.pass2.c	(revision 21112)
@@ -0,0 +1,1191 @@
+/**************************************************************************
+ *
+ * ivf.pass2.c -- Memory efficient pass 2 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  2004/06/10 05:07:43  kjdon
+   have to declare vars before calling functions!!
+
+   Revision 1.2  2004/06/10 03:02:05  kjdon
+   fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2
+
+   Revision 1.1  2003/02/20 21:18:23  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+ * Revision 1.2  1997/08/02  05:01:57  wew
+ * changed literal values of 32 for the bit size of magic numbers of
+ * files to sizeof (unsigned long) * 8, increased the gap at the start
+ * of the invf during processing to 200 bytes
+
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:51  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:49  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:35  tes
+   * For version 1.1
+   *
+ */
+
+/*
+ *  Modified:
+ *   - long long disk pointers and bit counts for inverted file
+ *     (1999-08-03 Tim Bell <bhat@cs.mu.oz.au>)
+ *     Code provided by Owen de Kretser <oldk@cs.mu.oz.au>
+ */
+
+static char *RCSID = "$Id$";
+
+#include "local_strings.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "stemmer.h"
+#include "perf_hash.h"
+#include "bitio_m.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include "bitio_random.h"
+#include "bitio_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "mg.h"
+#include "build.h"
+#include "words.h"
+#include "hash.h"
+
+#include "longlong.h"
+
+#ifdef USE_LONG_LONG
+#define BIO_Random_Seek_X BIO_Random_Seek_LL
+#define BIO_Random_Tell_X BIO_Random_Tell_LL
+#else
+#define BIO_Random_Seek_X BIO_Random_Seek
+#define BIO_Random_Tell_X BIO_Random_Tell
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include <io.h>
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+/*#define RND_BUF_SIZE 128 */
+#endif
+
+#define print_fsize(file)\
+do\
+  {\
+    struct stat file_state;\
+    fstat(fileno(invf_out), &file_state);\
+    Message("len(invf) = %ld", file_state.st_size);\
+  }while(0)
+
+typedef struct word_rec
+  {
+    unsigned long ptr;
+    unsigned long last;
+  }
+word_rec;
+
+typedef struct invf_state_rec
+  {
+    mg_ullong Disk_Ptr;
+    mg_ullong Disk_Last;
+    unsigned long Disk_B;
+  }
+invf_state_rec;
+
+typedef struct chunk
+  {
+    unsigned long start_doc;
+    unsigned long params_pos;
+    unsigned long disk_pos;
+    unsigned long N;
+  }
+chunk;
+
+
+static FILE *dict;		/* Stemmed dictionary file */
+static FILE *hash;		/* Stemmed dictionary hash file */
+static FILE *invf;		/* Inverted file */
+static FILE *invf_in;		/* Inverted file */
+static FILE *invf_out;		/* Inverted file */
+static FILE *invf_idx;		/* Inverted index file */
+static FILE *count;		/* Count file */
+static FILE *count_trans;	/* Count translation file */
+static FILE *invf_state;	/* Inverted file State */
+static FILE *chunk_state;	/* Chunk state */
+static FILE *chunks;		/* Chunk state */
+static FILE *invf_para = NULL;	/* Paragraph counts file */
+static FILE *weights = NULL;	/* Weights file */
+
+static stdio_bitio_state sbs;
+static random_bitio_state crbs;
+static chunk *chunk_data = NULL;
+static random_bitio_state rbs, rbsp;
+
+static int docs_left = 0, next_docs_left = 0;
+static unsigned long N;
+
+static word_rec *WordRecs;
+static u_char *lg_bs;
+static float *idf = NULL;
+
+static char *MemoryBuffer = NULL;
+static unsigned long MemBufSize;
+static unsigned long BufToUse;
+static struct invf_dict_header idh;
+
+static perf_hash_data *phd;
+
+static unsigned long *word_list = NULL;
+static unsigned long wl_size = 0;
+
+static unsigned long dict_size;
+static unsigned long no_of_ptrs = 0;
+static unsigned long chunks_read = 0;
+static unsigned long Disk_pos = 0;
+static unsigned long callnum = 0;
+static unsigned long wordnum = 0;
+
+static unsigned long totalIbytes = 0;
+static unsigned long totalDbytes = 0;
+static unsigned long totalHbytes = 0;
+
+static unsigned long MemInUse = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long max_buffer_len;
+
+void 
+ChangeMemInUse (int mem)
+{
+  MemInUse += mem;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+void 
+ResetStaticI2Vars() 
+{
+  docs_left = 0;
+  next_docs_left = 0;
+  N = 0;
+  MemBufSize=0;
+  BufToUse=0;
+  memset(&idh, 0, sizeof(idh));
+  wl_size = 0;
+  
+  dict_size = 0;
+  no_of_ptrs = 0;
+  chunks_read = 0;
+  Disk_pos = 0;
+  callnum = 0;
+  wordnum = 0;
+  
+  totalIbytes = 0;
+  totalDbytes = 0;
+  totalHbytes = 0;
+  
+  MemInUse = 0;
+  MaxMemInUse = 0;
+  max_buffer_len = 0;
+
+}
+
+
+static int 
+open_files (char *file_name)
+{
+  char FName[200];
+
+  if (!(dict = open_file (file_name, INVF_DICT_SUFFIX, "rb",
+			  MAGIC_STEM_BUILD, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(hash = open_file (file_name, INVF_DICT_HASH_SUFFIX, "rb",
+			  MAGIC_HASH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(count = open_file (file_name, INVF_CHUNK_SUFFIX, "rb",
+			   MAGIC_CHUNK, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fread (&max_buffer_len, sizeof (max_buffer_len), 1, count);
+  NTOHUL(max_buffer_len);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  BIO_Stdio_Decode_Start (count, &sbs);
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  if (!(count_trans = open_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "rb",
+				 MAGIC_CHUNK_TRANS, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(invf = create_file (file_name, INVF_SUFFIX, "w+b",
+			    MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fflush (invf);
+  if (!(invf_in = open_file (file_name, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  if (!(invf_out = create_file (file_name, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbs);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbsp);
+  ChangeMemInUse (RND_BUF_SIZE * 2);
+
+  if (!(invf_idx = create_file (file_name, INVF_IDX_SUFFIX, "wb",
+				MAGIC_INVI, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (InvfLevel == 3)
+    if (!(invf_para = create_file (file_name, INVF_PARAGRAPH_SUFFIX, "wb",
+				   MAGIC_PARAGRAPH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (COMPERROR);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".invf.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(invf_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunk.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunk_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+  BIO_Random_Start (chunk_state, RND_BUF_SIZE, &crbs);
+  ChangeMemInUse (RND_BUF_SIZE);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunks", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunks = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  return (COMPALLOK);
+}
+
+
+
+
+
+#define ISR_CACHE 1024
+#define ISR_ENTRY_SIZE (sizeof(mg_ullong)*2 + sizeof(unsigned long))
+
+invf_state_rec *
+in_cache (int pos)
+{
+  static char isr_data[ISR_CACHE * ISR_ENTRY_SIZE];
+  static invf_state_rec isr;
+  static int isr_base = 0, isr_num = -1, isr_pos = -1;
+  if (isr_pos >= 0)
+    bcopy ((char *) &isr, &isr_data[isr_pos * ISR_ENTRY_SIZE], ISR_ENTRY_SIZE);
+  if (pos < isr_base || pos >= isr_base + isr_num)
+    {
+      if (isr_num >= 0)
+	{
+	  fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+	  fwrite (isr_data, 1, ISR_ENTRY_SIZE * isr_num, invf_state);
+	}
+      isr_base = pos;
+      fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+      fread (isr_data, 1, ISR_ENTRY_SIZE * ISR_CACHE, invf_state);
+      isr_num = ISR_CACHE;
+    }
+  isr_pos = pos - isr_base;
+  bcopy (&isr_data[isr_pos * ISR_ENTRY_SIZE], (char *) &isr, ISR_ENTRY_SIZE);
+  return &isr;
+}
+
+
+
+
+
+unsigned long 
+occur_to_lexical (long occ, int clear_state)
+{
+  static long pos = -1;
+  static random_bitio_state rbs;
+  static int val = 0;
+  if (clear_state) {
+    pos = -1;
+    val = 0;
+    return 0;
+  }
+  if (pos == -1)
+    {
+      BIO_Random_Start (count_trans, RND_BUF_SIZE, &rbs);
+      pos = 0x7fffffff;
+    }
+  if (occ < pos)
+    {
+      if (occ == -1)
+	{
+	  BIO_Random_Done (&rbs);
+	  return 0;
+	}
+      BIO_Random_Seek_X (sizeof (unsigned long) * 8, &rbs);
+      pos = 0;
+    }
+  while (pos <= occ)
+    {
+      val = BIO_Random_Binary_Decode (dict_size + 1, &rbs, NULL) - 1;
+      pos++;
+    }
+  return (val);
+}
+
+
+void 
+add_chunk_state (unsigned long pos, unsigned long start_doc,
+		 unsigned long N)
+{
+  chunk_data[chunks_read].params_pos = pos;
+  chunk_data[chunks_read].start_doc = start_doc;
+  chunk_data[chunks_read].N = N;
+  chunks_read++;
+}
+
+
+int 
+init_ivf_2 (char *file_name)
+{
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+  mg_ullong totalIbits;
+  mg_ullong lasttotalIbits;
+  double logN = 0.0;
+
+  ResetStaticI2Vars(); /* clear the global statics */
+  occur_to_lexical(0, 1); /* clear the statics in here*/
+
+  if (open_files (file_name) == COMPERROR)
+    return COMPERROR;
+
+
+  /* Read in the stemmed dictionary file header */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  dict_size = idh.dict_size;
+
+  N = idh.num_of_docs;
+
+  if (!(phd = read_perf_hash_data (hash)))
+    {
+      Message ("Unable to read in hash data");
+      return COMPERROR;
+    }
+  totalHbytes = sizeof (perf_hash_data) + sizeof (u_char) * 256 +
+    sizeof (int) * (phd->MAX_N + 1) + sizeof (int *) * 3 * phd->MAX_CH +
+    sizeof (long) * phd->MAX_CH * phd->MAX_L;
+
+  if (!(WordRecs = Xmalloc (sizeof (word_rec) * idh.dict_size)))
+    {
+      Message ("No memory for word entries");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (word_rec) * idh.dict_size;
+  /* separate storage for the log(b) values, one byte each */
+  if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size)))
+    {
+      Message ("No memory for lg b's");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (u_char) * idh.dict_size;
+
+  if (MakeWeights)
+    {
+      /* separate storage for the idf values, one single each */
+      if (!(idf = Xmalloc (sizeof (float) * idh.dict_size)))
+	{
+	  Message ("No memory for idf's");
+	  return COMPERROR;
+	}
+      totalDbytes += sizeof (float) * idh.dict_size;
+
+      if (!(weights = create_file (file_name, WEIGHTS_SUFFIX, "wb",
+				 MAGIC_WGHT, MG_CONTINUE))) { /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Couldn't open weights file for writing");
+	return (COMPERROR);
+      }
+    }
+  else
+    {
+      unlink (make_name (file_name, WEIGHTS_SUFFIX, NULL));
+    }
+
+  chunk_data = Xmalloc (sizeof (chunk) * (ChunkLimit + 2));
+  totalDbytes += sizeof (chunk) * (ChunkLimit + 2);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  if (MakeWeights)
+    {
+      wl_size = 1024;
+      if (!(word_list = Xmalloc (sizeof (*word_list) * wl_size)))
+	{
+	  Message ("No memory for word_list");
+	  return COMPERROR;
+	}
+
+      logN = log ((double) N);
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      invf_state_rec *isr;
+      register unsigned long copy, suff, p;
+      unsigned long fcnt, wcnt;
+
+      lasttotalIbits = totalIbits;
+      
+      copy = fgetc (dict);
+      suff = fgetc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      WordRecs[i].last = 0;
+      WordRecs[i].ptr = 0;
+
+      p = fcnt;
+
+      if (MakeWeights)
+	idf[i] = logN - log ((double) fcnt);
+
+
+      isr = in_cache (i);
+
+      isr->Disk_Last = 0;
+      isr->Disk_Ptr = totalIbits;
+
+      isr->Disk_B = BIO_Bblock_Init (N, p);
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+      if (totalIbits < lasttotalIbits) {
+	fprintf(stderr, "ERROR: The totalIbits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+	if (sizeof (mg_ullong) < 8) {
+	  fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+	}
+	fprintf(stderr, "       Build aborted.\n");
+	exit(1);
+      }
+    }
+
+
+  /* now convert to bytes, and actually get the space */
+#ifdef USE_LONG_LONG
+  totalIbytes = (totalIbits + 7ull) >> 3ull;
+#else
+  totalIbytes = (totalIbits + 7ul) >> 3ul;
+#endif
+  return (COMPALLOK);
+
+}
+
+
+
+
+
+static void 
+LoadCounts (void)
+{
+  unsigned long numwords, i, last_total;
+  static unsigned long local_N = 0;
+  unsigned long totalIbits, crbs_pos;
+  word_rec *wr;
+  unsigned long *counts;
+
+  if (MemoryBuffer == NULL)
+    {
+      MemBufSize = sizeof (unsigned long) * dict_size;
+      if (max_buffer_len > MemBufSize) 
+	MemBufSize = max_buffer_len;
+      if (!(MemoryBuffer = Xmalloc (MemBufSize)))
+	FatalError (1, "Unable to allocate memory for buffer");
+      ChangeMemInUse (MemBufSize);
+    }
+  counts = (unsigned long *) MemoryBuffer;
+/*   bzero ((char *) counts, sizeof (unsigned long) * dict_size); */
+  bzero ((char *) counts, MemBufSize);
+  docs_left = next_docs_left; 
+  if (!docs_left)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  BufToUse = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  numwords = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  local_N = docs_left;
+
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    wr->ptr = 0; 
+
+  bzero ((char *) lg_bs, dict_size);
+
+  for (i = 0; i < numwords; i++)
+    {
+      unsigned long word_num, wcnt, fcnt, p;
+      word_num = occur_to_lexical (i,0);
+      
+      wr = &WordRecs[word_num];
+      
+      wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+      if (wcnt >= 2)
+	fcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL);
+      else
+	fcnt = wcnt;
+
+
+      p = fcnt;
+      if (wcnt)
+	{
+	  register unsigned long length;
+	  counts[word_num] = p;
+	  length = BIO_Bblock_Bound (local_N, p);
+	  if (InvfLevel >= 2)
+	    length += wcnt;
+	  wr->ptr = length;
+	  lg_bs[word_num] = floorlog_2 (BIO_Bblock_Init_W (local_N, p));
+	}
+
+
+  }  
+  crbs_pos = BIO_Random_Tell (&crbs);
+  totalIbits = 0;
+  last_total = 0;
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    {
+      register unsigned long length;
+      length = wr->ptr;
+      wr->last = callnum;
+      BIO_Random_Gamma_Encode (counts[i] + 1, &crbs, NULL);
+      if (counts[i])
+	{
+	  if (i)
+	    BIO_Random_Delta_Encode (totalIbits - last_total + 1, &crbs, NULL);
+	  else
+	    BIO_Random_Delta_Encode (1, &crbs, NULL);
+
+	  last_total = totalIbits;
+	}
+      wr->ptr = totalIbits;
+      totalIbits += length;
+    }
+  add_chunk_state (crbs_pos, callnum, local_N);
+
+  if ((totalIbits + 7ul) >> 3ul > BufToUse)
+    FatalError (1, "Pointers exceed buffer size");
+
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+}
+
+
+
+
+static void 
+DumpChunk (void)
+{
+  chunk_data[chunks_read - 1].disk_pos = Disk_pos << 3;
+  fseek (chunks, Disk_pos, 0);
+  fwrite (MemoryBuffer, sizeof (char), BufToUse, chunks);
+  Disk_pos += BufToUse;
+}
+
+
+
+
+static void 
+DiskMerge (void)
+{
+  random_bitio_state *rbsi;
+  random_bitio_state *chks = NULL;
+  unsigned long *chunk_ptrs;
+  int i;
+
+  BIO_Random_Flush (&crbs);
+
+  chunk_ptrs = Xmalloc (chunks_read * sizeof (unsigned long));
+  ChangeMemInUse (chunks_read * sizeof (unsigned long));
+  bzero ((char *) chunk_ptrs, chunks_read * sizeof (unsigned long));
+
+  rbsi = Xmalloc (chunks_read * sizeof (random_bitio_state));
+  ChangeMemInUse (chunks_read * sizeof (random_bitio_state));
+  for (i = 0; i < chunks_read; i++)
+    {
+      rbsi[i] = crbs;
+      rbsi[i].Buf = Xmalloc (rbsi[i].len);
+      ChangeMemInUse (rbsi[i].len);
+      bcopy ((char *) (crbs.Buf), (char *) (rbsi[i].Buf), rbsi[i].len);
+      BIO_Random_Seek (chunk_data[i].params_pos, &rbsi[i]);
+    }
+
+  if (chunks_read > 1)
+    {
+      int j;
+      chks = Xmalloc ((chunks_read - 1) * sizeof (random_bitio_state));
+      ChangeMemInUse ((chunks_read - 1) * sizeof (random_bitio_state));
+      BIO_Random_Start (chunks, RND_BUF_SIZE, &chks[0]);
+      ChangeMemInUse (RND_BUF_SIZE);
+      for (j = 1; j < chunks_read - 1; j++)
+	{
+	  chks[j] = chks[0];
+	  chks[j].Buf = Xmalloc (chks[0].len);
+	  ChangeMemInUse (chks[0].len);
+	  bcopy ((char *) (chks[0].Buf), (char *) (chks[j].Buf), chks[0].len);
+	}
+    }
+  for (i = 0; i < dict_size; i++)
+    {
+      int j;
+      invf_state_rec *isr = in_cache (i);
+      register int B;
+
+      BIO_Random_Seek_X (isr->Disk_Ptr, &rbs);	/* Position in invf file */
+
+      B = isr->Disk_B;
+
+      for (j = 0; j < chunks_read; j++)
+	{
+	  int p;
+	  p = BIO_Random_Gamma_Decode (&rbsi[j], NULL) - 1;
+
+	  if (p)
+	    {
+	      int ptr, b;
+	      chunk_ptrs[j] += BIO_Random_Delta_Decode (&rbsi[j], NULL) - 1;
+	      ptr = chunk_ptrs[j];
+	      b = 1 << floorlog_2 (BIO_Bblock_Init_W (chunk_data[j].N, p));
+
+	      if (j == chunks_read - 1)
+		{
+		  int k, CurrDoc;
+		  DECODE_START ((u_char *) MemoryBuffer, ptr)
+		    CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      BBLOCK_DECODE (x, b);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  UNARY_DECODE (tf);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  DECODE_DONE
+		    isr->Disk_Last = CurrDoc;
+		}
+	      else
+		{
+		  int k, CurrDoc;
+		  random_bitio_state *Chks = chks + j;
+		  BIO_Random_Seek (chunk_data[j].disk_pos + ptr, Chks);
+		  CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      x = BIO_Random_Bblock_Decode (b, Chks, NULL);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  tf = BIO_Random_Unary_Decode (Chks, NULL);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  isr->Disk_Last = CurrDoc;
+		}
+	    }
+	}
+
+      isr->Disk_Ptr = BIO_Random_Tell_X (&rbs);
+
+    }
+  if (chunks_read > 1)
+    {
+      int j;
+      for (j = 0; j < chunks_read - 1; j++)
+	{
+	  Xfree (chks[j].Buf);
+	  ChangeMemInUse (-chks[j].len);
+	}
+      Xfree (chks);
+      ChangeMemInUse (-(chunks_read - 1) * sizeof (random_bitio_state));
+    }
+
+  for (i = 0; i < chunks_read; i++)
+    {
+      Xfree (rbsi[i].Buf);
+      ChangeMemInUse (-rbsi[i].len);
+    }
+  Xfree (rbsi);
+  ChangeMemInUse (-chunks_read * sizeof (random_bitio_state));
+/*   chunks_read = 0; */
+  Xfree (chunk_ptrs);
+  ChangeMemInUse (-chunks_read * sizeof (unsigned long));
+  chunks_read = 0;
+  Disk_pos = 0;
+  BIO_Random_Seek (0, &crbs);
+}
+
+static void 
+MergeIn (void)
+{
+  static int disk_chunks = 0;
+  static header = 0;
+  if (!header)
+    {
+      fprintf (stderr, "ivf.pass2 : ");
+      header = 1;
+    }
+  if (disk_chunks == ChunkLimit || next_docs_left == 0)
+    {
+      fprintf (stderr, "M");
+      DiskMerge ();
+      disk_chunks = 0;
+    }
+  else
+    {
+      fprintf (stderr, "-");
+      DumpChunk ();
+      disk_chunks++;
+    }
+  if (next_docs_left == 0)
+    fprintf (stderr, "\n");
+}
+
+
+static int 
+wl_comp (const void *a, const void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  int res;
+  u_char *end = s_in + l_in - 1;
+  unsigned long tocode;
+  unsigned long wl_pos = 0;
+
+  if (!docs_left)
+    LoadCounts ();
+
+  callnum++;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      stemmer (idh.stem_method, idh.stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      if (*Word == 0)
+	continue;
+
+      res = perf_hash (phd, Word);
+
+      {
+	word_rec *arr = &WordRecs[res];
+	int b = 1 << lg_bs[res];
+	wordnum++;
+
+	tocode = callnum;
+
+	ENCODE_START ((u_char *) MemoryBuffer, arr->ptr)
+
+	  if (tocode > arr->last)
+	  {
+	    register int x;
+	    x = tocode - arr->last - 1;
+	    BBLOCK_ENCODE (x + 1, b);
+	    if (InvfLevel >= 2)
+	      ENCODE_BIT (1);
+	    no_of_ptrs++;
+	    arr->last = tocode;
+	  }
+	else if (InvfLevel >= 2)
+	  {
+	    __pos--;
+	    ENCODE_BIT (0);
+	    ENCODE_BIT (1);
+	  }
+	arr->ptr = __pos;
+	ENCODE_DONE
+      }
+
+      if (MakeWeights)
+	{
+	  if (wl_pos >= wl_size)
+	    {
+	      wl_size += (wl_size >> 1);
+	      word_list = Xrealloc (word_list, sizeof (*word_list) * wl_size);
+	    }
+	  word_list[wl_pos++] = res;
+	}
+    }
+  if (MakeWeights)
+    {
+      float doc_weight = 0.0;
+      if (wl_pos)
+	{
+	  unsigned long *wl = word_list;
+	  unsigned long i, count, val;
+	  qsort (wl, wl_pos, sizeof (*wl), wl_comp);
+	  count = 1;
+	  val = *wl++;
+	  for (i = 1; i <= wl_pos; i++, wl++)
+	    if (i == wl_pos || val != *wl)
+	      {
+		double weight = count * idf[val];
+		doc_weight += weight * weight;
+		count = 1;
+		val = *wl;
+	      }
+	    else
+	      count++;
+	}
+      HTONF(doc_weight);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &doc_weight, sizeof (doc_weight), 1, weights);
+    }
+  docs_left--;
+  if (!docs_left)
+    MergeIn ();
+
+  return COMPALLOK;
+}
+
+int 
+process_ivf_2 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int count = 0;
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	      count++;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	{
+	  if (process_doc (start, pos + s_in - start) != COMPALLOK)
+	    return (COMPERROR);
+	  count++;
+	}
+      HTONSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &count, sizeof (count), 1, invf_para);
+    }
+  return COMPALLOK;
+}
+
+
+
+
+
+static void 
+stats (unsigned long len)
+{
+#ifndef SILENT
+  fseek (count, 0, 2);
+  fseek (count_trans, 0, 2);
+  fseek (invf_state, 0, 2);
+  fseek (invf, 0, 0);
+  fseek (invf, 0, 2);
+  fseek (chunks, 0, 2);
+  fseek (chunk_state, 0, 2);
+  Message ("File sizes\n");
+  Message ("  Chunk desc    : %10u bytes\n", ftell (count));
+  Message ("  Chunk trans   : %10u bytes\n", ftell (count_trans));
+  Message ("  Chunks        : %10u bytes\n", ftell (chunks));
+  Message ("  Chunk state   : %10u bytes\n", ftell (chunk_state));
+  Message ("  Invf state    : %10u bytes\n", ftell (invf_state));
+  Message ("  Peak invf     : %10u bytes\n", len);
+  Message ("  Final invf    : %10u bytes\n", ftell (invf));
+  Message ("Peak disk usage : %10.2f %%\n",
+	   (double) (ftell (count) + ftell (count_trans) +
+		     ftell (invf_state) + ftell (chunks) +
+		     ftell (chunk_state) + len) / ftell (invf) * 100.0);
+#endif
+}
+
+
+/* ARGSUSED */
+int 
+done_ivf_2 (char *FileName)
+{
+  long i;
+  mg_ullong totalIbits;
+  unsigned long invf_len;
+  unsigned long bytes_output;
+  struct invf_file_header ifh;
+
+  if (weights)
+    fclose (weights);
+  if (invf_para)
+    fclose (invf_para);
+
+  free_perf_hash (phd);
+  phd = NULL;
+
+  Xfree (MemoryBuffer);
+  MemoryBuffer = NULL;
+  ChangeMemInUse (-MemBufSize);
+
+  BIO_Random_Done (&rbs);
+  BIO_Random_Done (&rbsp);
+  fflush (invf);
+
+  fseek (invf, 0, 2);
+  invf_len = ftell (invf);
+
+  fseek (invf_out, sizeof (long), 0);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(dict_size, ifh.no_of_words);
+  HTONUL2(no_of_ptrs, ifh.no_of_ptrs);
+  ifh.skip_mode = 0;
+  bzero ((char *) ifh.params, sizeof (ifh.params));
+  HTONUL2(InvfLevel, ifh.InvfLevel);
+  fwrite ((char *) &ifh, sizeof (ifh), 1, invf_out);
+
+  bytes_output = ftell (invf_out);
+
+  totalIbits = sizeof (unsigned long) * 8;		/* The magic number */
+  totalIbits += 8 * 200;	/* A 200 byte gap */
+
+  /* find the right place in the file to start reading p values */
+  fseek (dict, sizeof (unsigned long) + sizeof (struct invf_dict_header), 0);
+  for (i = 0; i < dict_size; i++)
+    {
+      invf_state_rec *isr;
+      unsigned long fcnt, wcnt, s, e;
+      register unsigned long p;
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+
+      /* output location to the invf_idx  */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      isr = in_cache (i);
+
+      e = (isr->Disk_Ptr + 7ul) >> 3ul;
+      s = totalIbits >> 3;
+
+      fseek (invf_in, s, 0);
+      while (s < e)
+	{
+	  u_char c = getc (invf_in);
+	  if (s == e - 1)
+	    {
+	      u_char ands[8] =
+	      {0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	      c &= ands[isr->Disk_Ptr & 7ul];
+	    }
+	  putc (c, invf_out);
+	  bytes_output++;
+	  s++;
+	}
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+#ifdef USE_LONG_LONG
+      totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull;
+#else
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+#endif
+
+    }
+
+  fclose (invf_in);
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (!(_chsize (_fileno (invf_out), bytes_output)))
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (invf_out), bytes_output);
+#endif
+
+  fclose (invf_out);
+
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (invf_idx);
+
+#ifndef SILENT
+  {
+    char *temp_str = msg_prefix;
+    unsigned long total;
+    msg_prefix = "ivf.pass2";
+    stats (invf_len);
+    Message ("Pass two data structures      : %6.3f Mbyte\n",
+	     (double) totalDbytes / 1024 / 1024);
+    total = totalDbytes;
+    Message ("Pass two hash structure(s)    : %6.3f Mbyte\n",
+	     (double) totalHbytes / 1024 / 1024);
+    total += totalHbytes;
+    Message ("Peak extra memory in use      : %6.3f Mbyte\n",
+	     (double) MaxMemInUse / 1024 / 1024);
+    total += MaxMemInUse;
+    Message ("Peak total memory in use      : %6.3f Mbyte\n",
+	     (double) total / 1024 / 1024);
+    msg_prefix = temp_str;
+  }
+#endif
+
+  Xfree(chunk_data);
+  chunk_data = NULL;
+  Xfree (WordRecs);
+  WordRecs = NULL;
+  Xfree (lg_bs);
+  lg_bs = NULL;
+  Xfree (idf);
+  idf = NULL;
+  Xfree (word_list);
+  word_list = NULL;
+  /* Free the memory allocated for the BIO_Random */
+  occur_to_lexical (-1,1);
+
+  BIO_Random_Done (&crbs);
+
+  fclose (invf);
+  fclose (dict);
+  fclose (hash);
+  fclose (count);
+  fclose (count_trans);
+  fclose (chunk_state);
+  fclose (chunks);
+  fclose (invf_state);
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/indexers/mg/src/text/lists.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/lists.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/lists.c	(revision 21112)
@@ -0,0 +1,217 @@
+/**************************************************************************
+ *
+ * lists.c -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+
+#include "lists.h"
+#include "local_strings.h"
+#include "locallib.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:01  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1999/07/01 09:30:48  rjmcnab
+   Changes for better match reporting.
+
+   Revision 1.1  1998/11/17 09:34:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:50  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:37  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/*
+ * Make a DocList large enough to hold num entries 
+ */
+DocList *
+MakeDocList (int num)
+{
+  DocList *d;
+  if (!(d = Xmalloc (sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  d->num = 0;
+  d->total_retrieved = 0;
+  d->is_approx = 0;
+  return (d);
+}
+
+
+/*
+ * Change the size of a DocList so that it can hold num entries 
+ */
+DocList *
+ResizeDocList (DocList * d, int num)
+{
+  if (!(d = Xrealloc (d, sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  return (d);
+}
+
+
+/* IntersectLists () */
+/* Returns the intersection of List1 and List2 by deleting from List1 all */
+/* entries not in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the intersection of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+IntersectLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List2 || !List1)
+    {
+      if (List1)
+	Xfree (List1);
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num && s2 - List2->DE < List2->num)
+    {
+      if (s1->DocNum == s2->DocNum)
+	*d1++ = *s1++;
+      if (s1 - List1->DE < List1->num)
+	while (s2 - List2->DE < List2->num && s2->DocNum < s1->DocNum)
+	  s2++;
+      if (s2 - List2->DE < List2->num)
+	while (s1 - List1->DE < List1->num && s1->DocNum < s2->DocNum)
+	  s1++;
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+
+/* DiffLists () */
+/* Returns the difference of List1 and List2 by deleting from List1 all */
+/* entries in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the differrence of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DiffLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num)
+    {
+      if (s2 - List2->DE == List2->num)
+	*d1++ = *s1++;
+      else
+	{
+	  if (s1->DocNum == s2->DocNum)
+	    {
+	      s1++;
+	      s2++;
+	    }
+	  else if (s1->DocNum < s2->DocNum)
+	    *d1++ = *s1++;
+	  else
+	    s2++;
+	}
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+/* MergeLists () */
+/* Finds the union of two lists and returns the head of the new list formed. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+MergeLists (DocList * List1, DocList * List2)
+{
+  DocList *NewList;
+  DocEntry *d, *s1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	return (List2);
+      else
+	return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  if (!(NewList = MakeDocList (List1->num + List2->num)))
+    {
+      Xfree (List1);
+      Xfree (List2);
+      return (NULL);
+    }
+
+  s1 = List1->DE;
+  s2 = List2->DE;
+  d = NewList->DE;
+  while (s1 - List1->DE < List1->num || s2 - List2->DE < List2->num)
+    {
+      if (s1 - List1->DE >= List1->num)
+	*d++ = *s2++;
+      else if (s2 - List2->DE >= List2->num)
+	*d++ = *s1++;
+      else if (s1->DocNum == s2->DocNum)
+	{
+	  *d = *s1++;
+	  (d++)->Weight += (s2++)->Weight;
+	}
+      else if (s1->DocNum < s2->DocNum)
+	*d++ = *s1++;
+      else
+	*d++ = *s2++;
+    }
+  NewList->num = d - NewList->DE;
+  Xfree (List1);
+  Xfree (List2);
+  return (NewList);
+}
Index: /main/tags/3.01/indexers/mg/src/text/lists.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/lists.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/lists.h	(revision 21112)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * lists.h -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_LISTS
+#define H_LISTS
+
+
+
+typedef struct DocEntry
+  {
+    float Weight;
+    int DocNum;
+    unsigned long SeekPos;	/* position in the text file in bytes */
+    unsigned long Len;		/* length of the document in bytes */
+    char *CompTextBuffer;
+    struct DocEntry *Next;
+    short or_included;		/*[TS:Mar/95] whether included in an AND of ORed-terms */
+  }
+DocEntry;
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved; /* valid if > 0 */
+    int is_approx;       /* whether total_retrieved is approximate */
+    DocEntry DE[1];
+  }
+DocList;
+
+DocList *MakeDocList (int num);
+DocList *ResizeDocList (DocList * d, int num);
+DocList *IntersectLists (DocList * List1, DocList * List2);
+DocList *DiffLists (DocList * List1, DocList * List2);
+DocList *MergeLists (DocList * List1, DocList * List2);
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/locallib.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/locallib.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/locallib.c	(revision 21112)
@@ -0,0 +1,307 @@
+/**************************************************************************
+ *
+ * locallib.c -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:02  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:51  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:38  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; i++)
+    total += A[i];
+  for (i = 0; i < n; i++)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; i--)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      n--;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  N++;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	limit++;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int 
+Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem,
+	  u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+	    u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (Fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
+
+
+int 
+F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (F_Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (Fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (Fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/locallib.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/locallib.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+int F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+		u_long * disk);
+
+int F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/mg.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+
+
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/mg.special.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg.special.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg.special.c	(revision 21112)
@@ -0,0 +1,118 @@
+/**************************************************************************
+ *
+ * mg.special.c -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg.h"
+#include "build.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:03  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:53  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:44  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+FILE *fp;
+
+
+int 
+init_special (char *FileName)
+{
+  char FName[200];
+  sprintf (FName, "%s%s", FileName, ".docnums");
+  if (!(fp = fopen (FName, "w")))
+    return (COMPERROR);
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_special (u_char * s_in, int l_in)
+{
+  static int un = 1;
+  char *s_done = (char *) (s_in + l_in);
+  for (; (char *) s_in != s_done; s_in++)
+    if (*s_in == '<')
+      {
+	if (strncmp ((char *) s_in, "<DOCNO>", 7) == 0)
+	  {
+	    char *s_end = (char *) (s_in + 1);
+	    for (; s_end != s_done; s_end++)
+	      if (*s_end == '<')
+		if (strncmp (s_end, "</DOCNO>", 8) == 0)
+		  break;
+	    if (s_end == s_done)
+	      {
+		fprintf (fp, "NO DOC ID %d\n", un++);
+		return COMPALLOK;
+	      }
+	    s_in += 7;
+	    while (s_end != (char *) s_in &&
+		   (*(s_end - 1) == ' ' || *(s_end - 1) == '\n'))
+	      s_end--;
+	    while ((char *) s_in != s_end &&
+		   (*s_in == ' ' || *s_in == '\n'))
+	      s_in++;
+	    for (; (char *) s_in != s_end; s_in++)
+	      putc (*s_in, fp);
+	    putc ('\n', fp);
+	    return (COMPALLOK);
+	  }
+      }
+  fprintf (fp, "NO DOC ID %d\n", un++);
+  return (COMPALLOK);
+}
+
+
+
+
+
+
+
+
+int 
+done_special (char *FileName)
+{
+  fclose (fp);
+  return COMPALLOK;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.1	(revision 21112)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_compression_dict.c	(revision 21112)
@@ -0,0 +1,1087 @@
+/**************************************************************************
+ *
+ * mg_compression_dict.c -- Routines for reading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  1999/12/10 15:04:25  cs025
+   Incorrect use of NTOHUL on a double value (num_bytes).
+
+   Revision 1.1  1999/08/10 21:18:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:52  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:54  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:45  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* #define DEBUG1 */
+
+/* #define DEBUG */
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct WordData
+  {
+    u_long freq, occur_num;
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+WordData;
+
+static WordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    WordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HBDYM")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'B':
+	novel_method = MG_NOVEL_BINARY;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'M':
+	novel_method = MG_NOVEL_HYBRID_MTF;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; i++)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      WordData *wd;
+      int j;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = Xmalloc (sizeof (WordData *) * Num[i] + 1);
+
+      buf = Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = Xmalloc (sizeof (WordData) * Num[i]);
+      for (j = 0; j < Num[i]; j++, wd++)
+	{
+	  int len;
+	  fread (&wd->freq, sizeof (wd->freq), 1, f);
+	  NTOHUL(wd->freq);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fread (&wd->occur_num, sizeof (wd->occur_num), 1, f);
+	  NTOHUL(wd->occur_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  int i;
+  WordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (WordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; i++, wd++)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; i++)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; i++)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->freq > bb->freq)
+    return -1;
+  else if (aa->freq < bb->freq)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->occur_num > bb->occur_num)
+    return 1;
+  else if (aa->occur_num < bb->occur_num)
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  WordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = Xmalloc (num * sizeof (WordData *));
+  for (i = 0; i < Num[0]; i++)
+    wd[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      WordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->freq, word->occur_num, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; i++)
+    {
+      WordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (WordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  bzero ((char *) char_freqs, sizeof (char_freqs));
+  bzero ((char *) len_freqs, sizeof (len_freqs));
+  for (i = 0; i < num; i++, wd++)
+    {
+      u_long freq = (*wd)->freq;
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; len--, buf++)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+
+
+
+
+void 
+CalcBitCost (WordData ** discard_word, int discard_num,
+	     WordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  int j;
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  for (j = 0; j < discard_num; j++, discard_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*discard_word)->word;
+      int len = *buf++;
+      u_long freq = (*discard_word)->freq;
+      int idx = KIND (*discard_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*discard_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*discard_word)->saving = ((*discard_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*discard_word)->word[0]);
+
+      (*discard_word)->num_trans = num_trans;
+    }
+  for (j = 0; j < keep_num; j++, keep_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*keep_word)->word;
+      int len = *buf++;
+      u_long freq = (*keep_word)->freq;
+      int idx = KIND (*keep_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*keep_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*keep_word)->saving = ((*keep_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*keep_word)->word[0]);
+
+      (*keep_word)->num_trans = num_trans;
+    }
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  WordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+  keep_num = 0;
+  keep_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+
+  for (i = 0; i < Num[0]; i++)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      WordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->freq;
+      num_trans++;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (keep_heap, sizeof (word), keep_num, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "DISCARD \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (discard_heap, sizeof (word),
+				discard_num, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      discard_heap[discard_num] = word;
+	      heap_additem (discard_heap, sizeof (word), &discard_num,
+			    BigSaving);
+	      freqs_trans[idx] -= word->freq;
+	      mem -= sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      keep_to_discard++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (discard_heap, sizeof (word),
+			       &discard_num, BigSaving);
+	      keep_heap[keep_num] = word;
+	      heap_additem (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      freqs_trans[idx] += word->freq;
+	      mem += sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      discard_to_keep++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "DISCARD -> KEEP %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  recalcs++;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; i++)
+    {
+      WordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; i++)
+    {
+      WordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  int i, us;
+  for (us = i = 0; i < dd->num_wds; i++)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = Xmalloc ((dd->num_wds) * sizeof (*freqs))))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      freqs[i] = dd->wd[i]->freq;
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  Xfree (freqs), freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  bzero ((char *) &huff_words_size, sizeof (huff_words_size));
+  bzero ((char *) &lencounts, sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      lencounts[codelen]++;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  Xfree (hd->clens);
+  Xfree (hd);
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  int j;
+  long freqs[256];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; len--, buf++)
+	freqs[(u_long) (*buf)] += (*wd)->freq;
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; j++)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  int j;
+  long freqs[16];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    freqs[(*wd)->word[0]] += (*wd)->freq;
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; j++)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  bzero((char *) &cdh, sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; i++)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < discard[i].num_wds; j++)
+		esc.freq += discard[i].wd[j]->freq;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < all[i].num_wds; j++)
+		if (all[i].wd[j]->freq == 1)
+		  esc.freq++;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.1	(revision 21112)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_decompress_text.c	(revision 21112)
@@ -0,0 +1,167 @@
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "netorder.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "backend.h"  /* compressed_dict struct */
+#include "text_get.h"
+#include "words.h"
+
+#define EOD         '\002'
+#define EOP         '\003'
+
+/* These are defined in mgquery, and they need to be here for backend.o */
+/* Messy I know but this was meant to be a quick hack :-) */
+#if defined(PARADOCNUM) || defined(NZDL)
+int *Paragraph = NULL;
+int Documents = 0;
+#endif
+
+int
+main (int argc, char **argv)
+{
+  FILE *text, *text_idx, *para;
+  File *text_comp_dict = NULL, *text_aux_dict = NULL, *text_fast_comp_dict = NULL;
+
+  int ch;
+  unsigned long start, finish, len;
+  u_long total_chars = 0;
+  u_char *c_buffer = 0;
+  int buf_len = -1;
+  u_char *uc_buffer = 0;
+  int ULen;
+  compression_dict *cd = NULL;
+  compressed_text_header cth;
+  u_long doc;
+  int count;
+  char *filename = "";
+  char path[512];
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-d directory] -f name\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  /* Open files */
+  text = open_file (filename, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text);
+
+  text_idx = open_file (filename, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text_idx);
+
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  para = open_file (filename, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH, MG_CONTINUE);
+
+  sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_FAST_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(text_fast_comp_dict = Fopen (path, "rb", MAGIC_FAST_DICT)))
+    {
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_comp_dict = Fopen (path, "rb", MAGIC_DICT);
+      if (!text_comp_dict)
+	FatalError (1, "Could not open compressed dictionary");
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_AUX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_aux_dict = Fopen (path, "rb", MAGIC_AUX_DICT);
+    }
+
+  cd = LoadCompDict (text_comp_dict, text_aux_dict, text_fast_comp_dict);
+
+
+  /* Uncompress text */
+
+  doc = 1;
+  fread ((char *)&start, sizeof (start), 1, text_idx);
+  if (para)
+    fread ((char *)&count, sizeof (count), 1, para);
+  else
+    count = 1;
+
+  for (;;)
+    {
+      fread ((char *) &finish, sizeof (finish), 1, text_idx);
+      len = finish - start;
+      
+      if ((int) len > buf_len)
+	{
+	  if (c_buffer)
+	    {
+	      Xfree (c_buffer);
+	      Xfree (uc_buffer);
+	    }
+	  if (!(c_buffer = Xmalloc (len)))
+	    FatalError (1, "Cannot allocate memory for compressed buffer");
+	  if (!(uc_buffer = Xmalloc ((int) (cth.ratio * 1.01 * len) + 100)))
+	    FatalError (1, "Cannot allocate memory for uncompressed buffer");
+	  buf_len = len;
+	}
+
+      fread (c_buffer, 1, len, text);
+      DecodeText (cd, c_buffer, len, uc_buffer, &ULen);
+      fwrite (uc_buffer, ULen, sizeof (u_char), stdout);
+      fflush (stdout);
+
+      total_chars += ULen;
+
+      if (++doc > cth.num_of_docs)
+	{
+	  if (para)
+	    fputc (EOP, stdout);
+	  fputc (EOD, stdout);
+	  fflush (stdout);
+	  break;
+	}
+
+      if (!--count)
+	{
+	  /* End of document */
+	  fputc (EOD, stdout);
+	  if (para)
+	    fread ((char *)&count, sizeof (count), 1, para);
+	  else
+	    count = 1;
+	}
+      else
+	fputc (EOP, stdout);
+      fflush (stdout);
+      start = finish;
+    }
+
+  /* Close files */
+
+  fclose (text);
+  fclose (text_idx);
+  if (para)
+    fclose (para);
+
+#if 0
+  printf ("\n#Total chars output = %lu\n", total_chars);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_errors.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_errors.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_errors.c	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * mg_errors.c -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_errors.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_fast_comp_dict.c	(revision 21112)
@@ -0,0 +1,801 @@
+/**************************************************************************
+ *
+ * mg_fast_comp_dict.c -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:06  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:57  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:55  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:47  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "huffman.h"
+#include "timing.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "backend.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; i++)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  bzero (buffer, mem);
+  fixup = Xmalloc (fixup_mem);
+  bzero (fixup, fixup_mem);
+
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+
+  load_comp_dict (filename);
+
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; which++)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+	cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+      for (which = 0; which <= 1; which++)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; j++)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; i++)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = buffer; (u_long) p < (u_long) cur; p++)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * cdh, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  u_long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; val--)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem += sizeof (comp_frags_header);
+	  Read_cfh (dict, &cfh, NULL, NULL);
+
+	  /* Don't need to count the space for the clens of the huffman data */
+
+	  mem += mem_for_words (dict, &cdh, &cfh);
+	  if (cfh.hd.clens)
+	    Xfree (cfh.hd.clens);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY ||
+      cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID ||
+      cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((long) cur > (long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID ||
+	  cdh->novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; i++)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+	  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+	  FIXUP (&cd->cfh[which]);
+	  FIXUP (&cd->values[which]);
+	  cd->escape[which] = NULL;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->chars_huff[which] = hd;
+	  FIXUP (&cd->chars_huff[which]);
+	  cd->chars_vals[which] = vals;
+	  FIXUP (&cd->chars_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->lens_huff[which] = hd;
+	  FIXUP (&cd->lens_huff[which]);
+	  cd->lens_vals[which] = vals;
+	  FIXUP (&cd->lens_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->chars_huff[which] = hd;
+	      FIXUP (&cd->chars_huff[which]);
+	      cd->chars_vals[which] = vals;
+	      FIXUP (&cd->chars_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->lens_huff[which] = hd;
+	      FIXUP (&cd->lens_huff[which]);
+	      cd->lens_vals[which] = vals;
+	      FIXUP (&cd->lens_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = buffer; (u_long) p < (u_long) cur; p++)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_files.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_files.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_files.c	(revision 21112)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * mg_files.c -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      Xfree (basepath);
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = Xmalloc (strlen (bp) + 2);
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_files.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_files.h	(revision 21112)
@@ -0,0 +1,329 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_hilite_words.c	(revision 21112)
@@ -0,0 +1,693 @@
+/**************************************************************************
+ *
+ * mg_hilite_words -- display text and highlight particular words which 
+ *                    it contains
+ * Copyright (C) 1994 Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "getopt.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "stemmer.h"
+#include "words.h"
+
+
+/*
+ * Description
+ * -----------
+ *   Hilite_words reads text from stdin and outputs it into a pager such as
+ *   less. It's command arguments include a list of stemmed words 
+ *   which if extracted from the text will be highlighted on the output.
+ *
+ * Implementation
+ * --------------
+ *   Extracting of words - using PARSE_STEM_WORD & stemmer
+ *   
+ *   Highlighting of words - for standard pager:
+ *                             using back-space character code
+ *                             e.g. bolded a = "a\ba", underlined a = "_\ba"
+ *                         - for pager==html:
+ *                             use some of the standard html character 
+ *                             formatting tags
+ *
+ *   Storage of words - using hashtable (set) of size equalling a constant
+ *                      times the number of words
+ * 
+ * Usage
+ * -----
+ *   mg_hilite_words --stem_method [0-3] 
+ *                   --style [bold|underline|italic|emphasis|strong]
+ *                   --pager [less|more|html|???]
+ *                   --terminator [terminator-string]
+ *                   list-of-words-to-highlight
+ */
+
+/*
+ * Modifications:
+ *
+ * 21/Apr/95: To handle outputting html tags to stdout
+ *
+ */
+
+/* --- constants --- */
+
+/* highlighting styles */
+#define HILITE_MAX      5	/* the number of styles */
+#define BOLD 		0
+#define UNDERLINE	1
+#define ITALIC     	2
+#define EMPHASIS     	3
+#define STRONG     	4
+
+/* maximum length of line buffer */
+#define MAX_LINE_BUFFER 200
+
+/* set pager to this to get html hilited text to stdout */
+#define HTML_OUT "html"
+
+/* output types */
+#define PAGER 0
+#define HTML  1
+
+/* --- types --- */
+
+typedef u_char *Word;
+
+/* --- globals --- */
+
+/* keep in synch with constants */
+static char *hilite_names[] =
+{"bold", "underline", "italic", "emphasis", "strong"};
+static char *hilite_tags[] =
+{"B", "", "I", "EM", "STRONG"};
+static short hilite_style = BOLD;
+static char *pager = "less";
+static int stemmer_num = 0;	/* Lovin's stemmer */
+static int stem_method = 3;	/* fold & stem */
+static char **word_list;
+static int num_of_words = 0;
+static int output_type = PAGER;
+static char *terminator = NULL;
+
+/* --- prototypes --- */
+
+static Word copy_c_word (char *w);
+static void process_words (char **words, int num_of_words);
+static int get_line (u_char * line, int n, FILE * stream);
+static void process_text (FILE * input_file, FILE * output_file);
+static void copy_word (u_char * w1, u_char * w2);
+static void process_buffer (u_char * s_in, int len, FILE * output_file);
+static void output_word (u_char * s_start, u_char * s_finish, FILE * output_file);
+static void output_hilite_word (u_char * s_start, u_char * s_finish,
+				FILE * output_file);
+static void process_args (int argc, char *argv[]);
+static void print_line (u_char * line, int n, FILE * stream);
+
+
+/******************************** word set ************************************/
+
+#include "hash.h"
+#include "locallib.h"
+
+#define SIZE_FACTOR 2
+
+typedef Word *WordSet;
+static WordSet set_of_words = NULL;
+static int hash_size = 0;
+
+/* prototypes - set routines */
+static void set_create (int num_of_words);
+static void set_add (Word word);
+static int set_member (Word word);
+static void set_print (void);
+
+/* =========================================================================
+ * Function: set_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_print (void)
+{
+  int i = 0;
+
+  for (i = 0; i < hash_size; i++)
+    {
+      Word word = set_of_words[i];
+
+      if (word)
+	{
+	  int len = *word++;
+
+	  fprintf (stderr, "[%d] = ", i);
+	  while (len--)
+	    fputc (*word++, stderr);
+	  fputc ('\n', stderr);
+	}
+    }
+
+}
+
+
+/* =========================================================================
+ * Function: set_create
+ * Description: 
+ *      Allocate memory for the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_create (int num_of_words)
+{
+  WordSet set;
+
+  hash_size = prime (num_of_words * SIZE_FACTOR);
+  set = (WordSet) malloc (hash_size * sizeof (Word));
+  if (!set)
+    FatalError (1, "Runout of memory for word hashtable");
+  bzero ((char *) set, hash_size * sizeof (Word));  /* [RPAP - Feb 97: WIN32 Port] */
+
+  set_of_words = set;
+
+}
+
+/* =========================================================================
+ * Function: set_add
+ * Description: 
+ *      Add a string element to the set.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_add (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	{
+	  set_of_words[hash_val] = word;
+	  break;
+	}
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	break;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+
+/* =========================================================================
+ * Function: set_member
+ * Description: 
+ *      Tests whether a string is a member of the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+set_member (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	return 0;
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	return 1;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+/******************************** end of word set ******************************/
+
+
+
+/* =========================================================================
+ * Function: copy_c_word
+ * Description: 
+ *      Allocate enough memory and copy word over
+ * Input: 
+ *      w = null terminated string (c_word)
+ * Output: 
+ *      word with length in 1st byte
+ * ========================================================================= */
+
+static Word
+copy_c_word (char *w)
+{
+  int len = strlen (w);
+  Word w_copy = (Word) malloc (len + 1);
+  Word w_ptr = NULL;
+  int j = 0;
+
+  if (!w_copy)
+    FatalError (1, "Not enough memory to copy a word");
+
+  w_copy[0] = len;
+  w_ptr = w_copy + 1;
+  for (j = 0; j < len; j++)
+    *w_ptr++ = *w++;
+
+  return w_copy;
+}
+
+/* =========================================================================
+ * Function: process_words
+ * Description: 
+ *      Go through the stemmed words and add to word set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_words (char **words, int num_of_words)
+{
+  int i = 0;
+
+  set_create (num_of_words);
+
+  for (i = 0; i < num_of_words; i++)
+    {
+      Word word = copy_c_word (words[i]);
+      set_add (word);
+    }
+
+}
+
+/* =========================================================================
+ * Function: get_line
+ * Description: 
+ *      Equivalent of fgets for u_char*.
+ *      But returns length of read-in line.
+ *      Expects to see a '\n' before an EOF
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+get_line (u_char * line, int n, FILE * stream)
+{
+  int i = 0;
+  int ch = '\0';
+
+  while (1)
+    {
+      if (i == n)
+	return i;
+
+      ch = fgetc (stream);
+
+      if (ch == EOF)
+	{
+	  if (!feof (stream))
+	    FatalError (1, "Error on reading a line from stdin");
+	  return EOF;
+	}
+
+      if (ch == '\n')
+	return i;
+
+      *line++ = ch;
+      i++;
+    }
+
+}
+
+/* =========================================================================
+ * Function: print_line
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+print_line (u_char * line, int n, FILE * stream)
+{
+
+  while (n--)
+    {
+      fputc (*line++, stream);
+    }
+  fputc ('\n', stream);
+
+}
+
+/* =========================================================================
+ * Function: process_text
+ * Description: 
+ *      Go through the text from input_file and highlight to output_file 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_text (FILE * input_file, FILE * output_file)
+{
+  static u_char line_buffer[MAX_LINE_BUFFER];
+
+
+  while (1)
+    {
+      int len = get_line (line_buffer, MAX_LINE_BUFFER, input_file);
+
+      if (len == EOF)
+	break;
+      process_buffer (line_buffer, len, output_file);
+
+    }
+}
+
+/* =========================================================================
+ * Function: copy_word
+ * Description: 
+ *      Copies w2 into w1. Assumes both have storage allocated.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+copy_word (u_char * w1, u_char * w2)
+{
+  int i;
+  int len = w2[0];
+
+  for (i = 0; i <= len; i++)
+    *w1++ = *w2++;
+
+}
+
+/* =========================================================================
+ * Function: process_buffer
+ * Description: 
+ *      Parse & stem words of line buffer
+ *      Based on the usage of PARSEing in other mg files.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_buffer (u_char * s_in, int len, FILE * output_file)
+{
+  u_char *end = s_in + len - 1;
+  u_char *s_start = NULL;
+
+  if (!inaword (s_in, end))
+    {
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+    }
+
+  while (s_in <= end)
+    {
+      u_char word[MAXSTEMLEN + 1];
+
+      s_start = s_in;
+      PARSE_STEM_WORD (word, s_in, end);
+
+      stemmer (stem_method, stemmer_num, word);
+
+      if (set_member (word))	/* output with highlighting */
+	{
+	  output_hilite_word (s_start, s_start + word[0] - 1, output_file);
+	  s_start += word[0];	/* step over hilited output */
+	}
+      output_word (s_start, s_in - 1, output_file);
+
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+
+    }				/*while */
+
+  fputc ('\n', output_file);
+  fflush (output_file);
+
+}				/*process_buffer */
+
+/* =========================================================================
+ * Function: output_word
+ * Description: 
+ *      Output a word which lies from s_start to s_finish in buffer
+ * Input: 
+ *      s_start = ptr to 1st char
+ *      s_finish = ptr to last char
+ * Output: 
+ * ========================================================================= */
+
+static void
+output_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+  while (s_start <= s_finish)
+    {
+      fputc (*s_start++, output_file);
+    }
+}
+
+
+/* =========================================================================
+ * Function: output_hilite_word
+ * Description: 
+ *      Highlight a word (with length in 1st byte)
+ *      Pager highlighting:
+ *        Highlighting is either by bolding or underlining using
+ *        the method used by UNIX utilities More(1) and Less(1)
+ *      HTML highlighting:
+ *        use the appropriate start and end tags around the word
+ * ========================================================================= */
+
+static void
+output_hilite_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+
+  if (output_type == HTML)
+    {
+      char *hilite_tag = hilite_tags[hilite_style];
+
+      /* print start tag */
+      fprintf (output_file, "<%s>", hilite_tag);
+
+      output_word (s_start, s_finish, output_file);
+
+      /* print end tag */
+      fprintf (output_file, "</%s>", hilite_tag);
+    }
+
+  else
+    /* PAGER */
+    {
+      /* use backspaces around each letter */
+      while (s_start <= s_finish)
+	{
+	  switch (hilite_style)
+	    {
+	    case BOLD:
+	      fputc (*s_start, output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    case UNDERLINE:
+	      fputc ('_', output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    default:
+	      fputc (*s_start, output_file);
+	    }
+	  s_start++;
+	}			/*while */
+    }
+}
+
+/* =========================================================================
+ * Function: process_args
+ * Description: 
+ *      sets the global variables:
+ *              hilite_style, pager, num_of_words, word_list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+struct option long_opts[] =
+{
+  {"style", required_argument, 0, 's'},
+  {"terminator", required_argument, 0, 't'},
+  {"pager", required_argument, 0, 'p'},
+  {"stem_method", required_argument, 0, 'm'},
+  {"stemmer", required_argument, 0, 'a'},
+  {0, 0, 0, 0}
+};
+
+static void
+process_args (int argc, char *argv[])
+{
+  int ch;
+
+
+  opterr = 0;
+  while ((ch = getopt_long (argc, argv, "s:p:t:m:a:", long_opts, (int *) 0)) != -1)
+    {
+      switch (ch)
+	{
+	case 's':
+	  {
+	    int i;
+	    for (i = 0; i < HILITE_MAX; i++)
+	      if (strcmp (optarg, hilite_names[i]) == 0)
+		break;
+
+	    if (i < HILITE_MAX)
+	      hilite_style = i;
+	  }
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 't':
+	  terminator = optarg;
+	  break;
+	case 'm':
+	  stem_method = atoi (optarg);
+	  break;
+	case 'p':
+	  pager = optarg;
+	  break;
+	default:
+	  FatalError (1, "Usage: \n"
+		      "mg_hilite_words --stem_method [0-3]\n"
+	  "                --stemmer [english|lovin|french|simplefrench]\n"
+	  "                --style [bold|underline|italic|emphasis|strong]\n"
+		      "                --pager [less|more|html|???]\n");
+	}
+    }
+
+  num_of_words = argc - optind;
+
+  word_list = &argv[optind];
+
+  /* fix up output type */
+  if (strcmp (pager, HTML_OUT) == 0)
+    output_type = HTML;
+  else
+    output_type = PAGER;
+
+}
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+main (int argc, char *argv[])
+{
+  FILE *output = NULL;
+
+  process_args (argc, argv);
+
+  /* set output file */
+  if (output_type == PAGER)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    output = _popen (pager, "w");
+#else
+    output = popen (pager, "w");
+#endif
+  else
+    output = stdout;
+
+  if (!output)
+    FatalError (1, "Unable to run \"%s\"\n", pager);
+
+  if (num_of_words < 1)
+    {
+      int ch;
+
+      /* just echo the input */
+      /* better not to call this program at all ;-) */
+      while ((ch = fgetc (stdin)) != EOF)
+	{
+	  fputc (ch, output);
+	}
+    }
+  else
+    {
+      /* set up hash table for words */
+      process_words (word_list, num_of_words);
+
+
+      /* Go thru lines of text from stdin and 
+       * output words with hilite info if
+       * words parse into existence in the hash table 
+       */
+      process_text (stdin, output);
+    }
+
+  if (terminator)
+    fprintf (output, "%s\n", terminator);
+
+  if (output != stdout)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (output);
+#else
+    pclose (output);
+#endif
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.1	(revision 21112)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_dict.c	(revision 21112)
@@ -0,0 +1,347 @@
+/**************************************************************************
+ *
+ * mg_invf_dict.c -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:08  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:45  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:00  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:49  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static void process_files (char *filename);
+
+int main (int argc, char **argv)
+{
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:b:Fh")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b num] [-F] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  process_files (file_name);
+  return 0;
+}
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *id, *idb, *tmp, *ii;
+  unsigned long i, pos, num, First_word, invf_ptr, invf_len;
+  unsigned long last_ptr = 0;
+  char *FName;
+  struct invf_dict_header idh;
+  struct stem_dict_header sdh;
+  u_char prev[MAXSTEMLEN + 1];
+  u_char *buffer;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+
+  id = open_file (filename, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ii = open_file (filename, INVF_IDX_SUFFIX, "rb", MAGIC_INVI, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  idb = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "w+b", MAGIC_STEM,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  fread (&idh, sizeof (idh), 1, id);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  sdh.lookback = idh.lookback;
+  sdh.block_size = block_size;
+  sdh.num_blocks = 0;
+  sdh.index_chars = 0;
+  sdh.blocks_start = 0;
+  sdh.num_of_docs = idh.num_of_docs;
+  sdh.static_num_of_docs = idh.static_num_of_docs;
+  sdh.num_of_words = idh.num_of_words;
+  sdh.stemmer_num = idh.stemmer_num;
+  sdh.stem_method = idh.stem_method;
+  sdh.indexed = 0;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long extra, copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (id);
+      suff = getc (id);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, id);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, id);
+      fread (&wcnt, sizeof (wcnt), 1, id);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      /* read in the inverted file position */
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (word_num % idh.lookback == 0)
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + suff + 1 + sizeof (fcnt) + sizeof (wcnt) +
+	  sizeof (First_word) + sizeof (invf_ptr) + sizeof (invf_len) > block_size)
+	{
+	  int chunk;
+	  invf_len = invf_ptr - last_ptr;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUL(First_word);
+	  HTONUL(invf_len);
+	  HTONUS(word_num);
+
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sdh.num_blocks++;
+	}
+
+      if (word_num % idh.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (prev + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUL(fcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &fcnt, (char *) (buffer + buf_in_use), sizeof (fcnt));
+      buf_in_use += sizeof (fcnt);
+      HTONUL(wcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &wcnt, (char *) (buffer + buf_in_use), sizeof (wcnt));
+      buf_in_use += sizeof (wcnt);
+      last_ptr = invf_ptr;
+      HTONUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &invf_ptr, (char *) (buffer + buf_in_use), sizeof (invf_ptr));
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (invf_ptr);
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 34876234\n");
+      if (word_num == 0)
+	{
+	  fwrite (prev, sizeof (u_char), *prev + 1, idb);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idb);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sdh.index_chars += *prev + 1;
+	  First_word = i;
+	}
+      word_num++;
+    }
+  if (buf_in_use)
+    {
+      int chunk;
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      invf_len = invf_ptr - last_ptr;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUL(invf_len);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sdh.num_blocks++;
+    }
+  fclose (id);
+  fclose (ii);
+
+  rewind (tmp);
+  sdh.blocks_start = sdh.index_chars + sizeof (u_long) + sizeof (sdh) +
+    sdh.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sdh.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idb);
+	  sdh.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idb);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idb, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sdh.lookback);
+  HTONUL(sdh.block_size);
+  HTONUL(sdh.num_blocks);
+  HTONUL(sdh.blocks_start);
+  HTONUL(sdh.index_chars);
+  HTONUL(sdh.num_of_docs);
+  HTONUL(sdh.static_num_of_docs);
+  HTONUL(sdh.num_of_words);
+  HTONUL(sdh.stemmer_num);
+  HTONUL(sdh.stem_method);
+  HTONUL(sdh.indexed);
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+
+
+  Message ("Block size = %d\n", block_size);
+  Message ("Number of blocks written = %d\n", NTOHUL(sdh.num_blocks));
+
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.1	(revision 21112)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_dump.c	(revision 21112)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * mg_invf_dump.c -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+
+extern unsigned long S_btg;
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/06/12 01:41:39  kjm18
+   changed the help message
+
+   Revision 1.1  1999/08/10 21:18:09  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/11/29  00:32:01  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.2  1994/09/20  04:41:50  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+static void process_files (char *filename);
+
+int binary = 0;
+int word_counts = 0;
+int term_dump = 0;
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hbwtf:d:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+        set_basepath(optarg);
+	break;
+      case 'b':
+	binary = 1;
+	break;
+      case 'w':
+	word_counts = 1;
+	break;
+      case 't':
+	term_dump = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-b] [-w] [-t] [-f input_file]"
+		 "[-d data directory]\n"
+		 "(b - binary mode, w - wordcount, t - term dump)\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+static void 
+process_files (char *name)
+{
+  unsigned long N, Nstatic, k;
+  FILE *dict;
+  FILE *invf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (!(invf = open_file (name, INVF_SUFFIX ".ORG", "rb", MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    invf = open_file (name, INVF_SUFFIX, "rb", MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  if (ifh.skip_mode != 0)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  DECODE_START (invf)
+    N = idh.num_of_docs;
+  Nstatic = idh.static_num_of_docs;
+  if (binary)
+    {
+      fwrite ((char *) &N, sizeof (N), 1, stdout);
+      fwrite ((char *) &ifh.no_of_words, sizeof (ifh.no_of_words), 1, stdout);
+    }
+  else
+    printf ("%ld %ld\n", N, ifh.no_of_words);
+  for (k = 0; k < ifh.no_of_words; k++)
+    {
+      int i, blk, doc;
+      register unsigned long suff, prefix;
+      unsigned long fcnt, wcnt;
+      char term[MAXSTEMLEN + 1];
+
+      prefix = fgetc (dict);
+      suff = fgetc (dict);
+      fread (&term[prefix], sizeof (char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+      term[prefix + suff] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (binary)
+	fwrite ((char *) &fcnt, sizeof (fcnt), 1, stdout);
+      else
+	{
+	  if (term_dump)
+	    printf ("%ld    \"%s\"\n", fcnt, term);
+	  else
+	    printf ("%ld\n", fcnt);
+	}
+      blk = BIO_Bblock_Init (Nstatic, fcnt);
+      for (doc = i = 0; i < fcnt; i++)
+	{
+	  int num;
+	  BBLOCK_DECODE (num, blk);
+	  doc += num;
+	  if (binary)
+	    fwrite ((char *) &doc, sizeof (doc), 1, stdout);
+	  else
+	    printf (" %d", doc);
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      int count;
+	      GAMMA_DECODE (count);
+	      if (word_counts)
+		if (binary)
+		  fwrite ((char *) &count, sizeof (count), 1, stdout);
+		else
+		  printf (" %d", count);
+	    }
+	  if (!binary)
+	    putchar ('\n');
+	}
+      while (__btg)
+	DECODE_BIT;
+    }
+  DECODE_DONE
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_merge.c	(revision 21112)
@@ -0,0 +1,852 @@
+/**************************************************************************
+ *
+ * mg_invf_merge.c -- description
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Last edited:  31 March, 1995 
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "bitio_m.h"
+#include "bitio_gen.h"
+#include "bitio_stdio.h"
+
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+
+typedef char FileName[256];
+
+
+/*
+ *  The merge_info struct contains all the data for an inverted file and
+ *  lexicon. We declare 3 merge_info variables, m[OLD] m[NEW] and m[MERGE].
+ */
+typedef struct merge_info_type
+  {
+    FileName fname;
+    FILE *invf;
+    FILE *dict;
+    FILE *idx;
+    FILE *para;  /* [RPAP - Feb 97: Level 3 Merge] */
+    struct invf_dict_header idh;
+    struct invf_file_header ifh;
+    u_long nDocs;
+
+    /* Lexicon processing variables */
+    u_long fcnt, wcnt;
+    int suff, pref;
+    unsigned char term[MAXSTEMLEN + 1];
+    u_long term_count;
+    int done;
+
+    /* Variables used for merging */
+    struct stdio_bitio_state sbs;	/* for bit-based I/O on inverted files */
+    u_long nTerms;		/* = number of invf entries */
+    int *fcntlist;		/* store fcnt values read in from the lexicons */
+
+  }
+merge_info;
+
+
+/***************************************************
+ **** GLOBALS ****
+ ***************************************************/
+
+merge_info m[3];		/*  m[OLD], m[NEW] and m[MERGE] */
+u_long magicsize;		/* how big is magic number? currently always 4 */
+u_long Nstatic;			/* static # of documents in file */
+
+float *DocWeightBuffer;
+int weightOption = 0;		/* true if weights file is to be appended */
+
+/*** Globals for processing the lexicons -- see readTerm() and writeTerm() 
+ ***/
+ProgTime start;			/* for printing elapsed time */
+unsigned char prevTerm[MAXSTEMLEN + 1];
+
+/*** Globals for merging -- see processEntry() 
+ ***/
+char *mergedata;		/* stores value OLD/NEW/MERGE for each term */
+int fastMerge = 1;		/* True if fast merge selected. Default = true */
+
+u_long oldIdxValue;		/* stores last read value from m[OLD].idx
+				   Used to calculate entry lengths */
+u_long bytes_output;		/* for writing .invf.idx file */
+u_long oldOnlyBits = 0;		/* entries in old only */
+u_long oldMergeBits = 0;	/* entries in old that are merged with 
+				   a new entry */
+
+/**************************************************************************/
+
+
+/*=======================================================================
+ * init_merge_invf(): open files, initialise globals, etc
+ *=======================================================================*/
+int 
+init_merge_invf ()
+{
+  prevTerm[0] = prevTerm[1] = 0;
+
+    /***
+     *  open .dict files
+     ***/
+  m[OLD].dict = open_file (m[OLD].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (m[OLD].dict);
+  fread (&(m[OLD].idh), sizeof (m[OLD].idh), 1, m[OLD].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].idh.lookback);
+  NTOHUL(m[OLD].idh.dict_size);
+  NTOHUL(m[OLD].idh.total_bytes);
+  NTOHUL(m[OLD].idh.index_string_bytes);
+  NTOHD(m[OLD].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[OLD].idh.num_of_docs);
+  NTOHUL(m[OLD].idh.static_num_of_docs);
+  NTOHUL(m[OLD].idh.num_of_words);
+  NTOHUL(m[OLD].idh.stemmer_num);
+  NTOHUL(m[OLD].idh.stem_method);
+
+  m[NEW].dict = open_file (m[NEW].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[NEW].idh), sizeof (m[NEW].idh), 1, m[NEW].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].idh.lookback);
+  NTOHUL(m[NEW].idh.dict_size);
+  NTOHUL(m[NEW].idh.total_bytes);
+  NTOHUL(m[NEW].idh.index_string_bytes);
+  NTOHD(m[NEW].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[NEW].idh.num_of_docs);
+  NTOHUL(m[NEW].idh.static_num_of_docs);
+  NTOHUL(m[NEW].idh.num_of_words);
+  NTOHUL(m[NEW].idh.stemmer_num);
+  NTOHUL(m[NEW].idh.stem_method);
+
+  m[MERGE].dict = create_file (m[MERGE].fname, INVF_DICT_SUFFIX, "wb",
+			       MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header */
+  fwrite ((char *) &(m[OLD].idh), sizeof (m[OLD].idh), 1, m[MERGE].dict);
+
+  m[OLD].nDocs = m[OLD].idh.num_of_docs;
+  m[NEW].nDocs = m[NEW].idh.num_of_docs;
+  Nstatic = m[OLD].idh.static_num_of_docs;
+  m[OLD].nTerms = m[OLD].idh.dict_size;
+  m[NEW].nTerms = m[NEW].idh.dict_size;
+  m[MERGE].nDocs = m[OLD].nDocs + m[NEW].nDocs;
+  m[MERGE].nTerms = 0;
+
+  /* Set up weight buffer for weights of new documents */
+  if (!(DocWeightBuffer = Xmalloc (m[NEW].nDocs * sizeof (*DocWeightBuffer))))
+    {
+      Message ("Insufficient memory\n");
+      return COMPERROR;
+    }
+  bzero ((char *) DocWeightBuffer, m[NEW].nDocs * sizeof (*DocWeightBuffer));
+
+    /***
+     *  open .invf files
+     *  Try ".ORG" extension first for m[OLD].invf since an inverted file
+     *  with skips may have been created.
+     ****/
+  if (!(m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX ".ORG", "rb",
+				 MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[OLD].ifh), sizeof (m[OLD].ifh), 1, m[OLD].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].ifh.no_of_words);
+  NTOHUL(m[OLD].ifh.no_of_ptrs);
+  NTOHUL(m[OLD].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[OLD].ifh.params[i]);
+  }
+  NTOHUL(m[OLD].ifh.InvfLevel);
+
+  m[NEW].invf = open_file (m[NEW].fname, INVF_SUFFIX, "rb",
+			   MAGIC_INVF, MG_ABORT);
+  fread (&(m[NEW].ifh), sizeof (m[NEW].ifh), 1, m[NEW].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].ifh.no_of_words);
+  NTOHUL(m[NEW].ifh.no_of_ptrs);
+  NTOHUL(m[NEW].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[NEW].ifh.params[i]);
+  }
+  NTOHUL(m[NEW].ifh.InvfLevel);
+
+  if (m[OLD].ifh.skip_mode != 0)
+    FatalError (1, "The old invf file contains skips. Unable to merge.");
+
+  if (m[OLD].ifh.InvfLevel != m[NEW].ifh.InvfLevel)
+    FatalError (1, "The two invf files have different inversion levels!");
+
+  /********** [RPAP - Feb 97: Level 3 Merge]
+
+  if ((m[OLD].ifh.InvfLevel > 2) || (m[NEW].ifh.InvfLevel > 2))
+    FatalError (1, "mgmerge cannot handle level 3 inverted files!");
+
+    *************/
+
+  m[MERGE].invf = create_file (m[MERGE].fname, INVF_SUFFIX, "wb",
+			       MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header in inverted file */
+  m[MERGE].ifh.no_of_words = m[MERGE].ifh.no_of_ptrs = 0;
+  m[MERGE].ifh.skip_mode = 0;
+  m[MERGE].ifh.InvfLevel = 2;
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = 0;
+  }
+  fwrite ((char *) &(m[MERGE].ifh), sizeof (m[MERGE].ifh), 1, m[MERGE].invf);
+
+    /***
+     *  open .invf.idx files
+     ***/
+  m[OLD].idx = open_file (m[OLD].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+  fread (&oldIdxValue, sizeof (u_long), 1, m[OLD].idx);
+  NTOHUL(oldIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  m[NEW].idx = open_file (m[NEW].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+
+  m[MERGE].idx = create_file (m[MERGE].fname, INVF_IDX_SUFFIX, "wb",
+			      MAGIC_INVI, MG_ABORT);
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+
+    /***
+     *  open .invf.parargraph files
+     ***/
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = open_file (m[NEW].fname, INVF_PARAGRAPH_SUFFIX, "rb",
+			       MAGIC_PARAGRAPH, MG_ABORT);
+      m[MERGE].para = open_file (m[MERGE].fname, INVF_PARAGRAPH_SUFFIX, "rb+",
+				 MAGIC_PARAGRAPH, MG_ABORT);
+    }
+  else
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = NULL;
+      m[MERGE].para = NULL;
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * procEntry_slow(): merge an entry, slow method
+ *=======================================================================*/
+void 
+procEntry_slow (int tnum, int type, u_long oldN, u_long newN, u_long mergeN)
+/* 
+ *  Process an invf entry. 
+ *  Slow Merge. (Always decode/re-encode).
+ *
+ *  tnum is the entry number, type = OLD or NEW or MERGE.
+ *  oldN, newN and mergeN are the N parameters for Bblock coding.
+ *  They are passed so other procEntry_fast can call this one
+ *  when a decode/recode is needed to save repeating code.
+ */
+{
+  int doc = 0;
+  int fcntOLD = m[OLD].fcntlist[tnum];
+  int fcntNEW = m[NEW].fcntlist[tnum];
+  int inblk, outblk;
+  u_long inbits = 0, outbits = 0;
+
+  /* write .invf.idx pointer */
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  outblk = BIO_Bblock_Init (mergeN, (fcntOLD + fcntNEW));
+
+/*** OLD entry -> merge entry ***/
+  if (type != NEW)
+    {
+      int i;
+      inblk = BIO_Bblock_Init (oldN, fcntOLD);
+      for (i = 0; i < fcntOLD; i++)
+	{
+	  int num;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[OLD].sbs), &inbits);
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[OLD].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	    }
+	}
+      if (type == OLD)
+	oldOnlyBits += (inbits + m[OLD].sbs.Btg);
+      else
+	oldMergeBits += (inbits + m[OLD].sbs.Btg);
+    }
+
+/*** NEW entry -> merge entry ***/
+  if (type != OLD)
+    {
+      int i;
+      int offset;
+      double logN = log ((double) m[MERGE].nDocs);
+      double idf = logN - log ((double) (fcntOLD + fcntNEW));
+
+      inblk = BIO_Bblock_Init (newN, fcntNEW);
+      offset = (m[OLD].nDocs - doc);	/* amount to be added to 1ST pointer */
+      for (i = 0; i < fcntNEW; i++)
+	{
+	  register int num;
+	  register double weight;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[NEW].sbs), &inbits);
+	  if (i == 0)
+	    num += offset;
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[NEW].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	      weight = num * idf;
+	      DocWeightBuffer[doc - m[OLD].nDocs - 1] += weight * weight;
+	    }
+	}
+    }
+/*** Now the padding bits ***/
+  while (m[OLD].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[OLD].sbs));
+  while (m[NEW].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[NEW].sbs));
+  while (m[MERGE].sbs.Btg != 8)
+    {
+      BIO_Stdio_Encode_Bit (0, &(m[MERGE].sbs));
+      outbits++;
+    }
+  bytes_output += (outbits >> 3);
+  return;
+}
+
+
+/*=======================================================================
+ * procEntry_fast(): merge an entry, WITHOUT decoding if it's in the
+ *                   old inverted file only
+ *=======================================================================*/
+void 
+procEntry_fast (int tnum, int type)
+/*
+ *  Merge an invf entry. 
+ *  Faster Method : Copy (don't re-code) entries for terms only in IFold.
+ *  The N parameter is Nstatic, from m[OLD].idh.static_num_of_docs.
+ *  If entry type isnt OLD, just call procEntry_slow()
+ */
+{
+  u_long newIdxValue, len = 0;
+
+  /* Calculate IFold entry length in bytes if not NEW type */
+  if (type != NEW)		/* read in an index number from m[OLD].idx */
+    {
+      fread (&newIdxValue, sizeof (u_long), 1, m[OLD].idx);
+      NTOHUL(newIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+      len = newIdxValue - oldIdxValue;
+      oldIdxValue = newIdxValue;
+    }
+
+  if (type != OLD)		/* must decode/recode */
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, Nstatic);
+  else
+    {
+      /* write .invf.idx pointer */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* copy entry */
+      {
+	int i;
+	char c;
+	for (i = 0; i < len; i++)
+	  {
+	    c = fgetc (m[OLD].invf);
+	    fputc (c, m[MERGE].invf);
+	  }
+      }
+      bytes_output += len;
+      oldOnlyBits += (len * 8);
+    }
+  return;
+}
+
+
+/*=======================================================================
+ * processEntry(): merge an entry. Call either procEntry_slow() or
+ *                 procEntry_fast.
+ *=======================================================================*/
+void 
+processEntry (int tnum, int type)
+/* 
+ *  Read an entry from IFold and/or IFnew, and write to IFmerge.
+ *  tnum is the term number, type is OLD or NEW or MERGE
+ */
+{
+  if (fastMerge)
+    procEntry_fast (tnum, type);
+  else
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, m[MERGE].nDocs);
+  return;
+}
+
+
+/*=======================================================================
+ * readTerm()
+ *=======================================================================*/
+void 
+readTerm (int x)
+/*
+ *  Read the next term from the appropriate ".invf.dict" file.
+ *  x should be either OLD or NEW.
+ *  Sets m[x].done to true when that lexicon has been completely read.
+ */
+{
+  int i;
+
+  if (m[x].term_count >= m[x].nTerms)
+    {
+      m[x].term[0] = 0;
+      m[x].done = 1;
+      return;
+    }
+  m[x].pref = fgetc (m[x].dict);
+  m[x].suff = fgetc (m[x].dict);
+  m[x].term[0] = m[x].pref + m[x].suff;
+
+  for (i = 0; i < m[x].suff; i++)
+    {
+      m[x].term[m[x].pref + i + 1] = fgetc (m[x].dict);
+    }
+  m[x].term_count = m[x].term_count + 1;
+
+  fread ((char *) &(m[x].fcnt), sizeof (u_long), 1, m[x].dict);
+  fread ((char *) &(m[x].wcnt), sizeof (u_long), 1, m[x].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[x].fcnt);
+  NTOHUL(m[x].wcnt);
+
+  return;
+}
+
+
+/*=======================================================================
+ * writeTerm()
+ *=======================================================================*/
+void 
+writeTerm (int x)
+/*
+ *  Write the current term to m[MERGE].dict file.
+ *  x is OLD, NEW or MERGE.
+ */
+{
+  unsigned char i, prefix, suffix;
+  if (x == MERGE)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt + m[NEW].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt + m[NEW].wcnt;
+      m[MERGE].pref = m[OLD].pref;
+      m[MERGE].suff = m[OLD].suff;
+      memcpy (m[MERGE].term, m[OLD].term, m[OLD].term[0] + 1);
+    }
+  if (x == OLD)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt;
+    }
+  if (x == NEW)
+    {
+      m[MERGE].fcnt = m[NEW].fcnt;
+      m[MERGE].wcnt = m[NEW].wcnt;
+    }
+  prefix = prefixlen (prevTerm, m[x].term);
+  suffix = m[x].term[0] - prefix;
+  fputc (prefix, m[MERGE].dict);	/* prefix length */
+  fputc (suffix, m[MERGE].dict);	/* suffix length */
+  for (i = 0; i < suffix; i++)
+    fputc (m[x].term[prefix + i + 1], m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].fcnt);
+  HTONUL(m[MERGE].wcnt);
+
+  fwrite ((char *) &(m[MERGE].fcnt), sizeof (m[MERGE].fcnt), 1, m[MERGE].dict);
+  fwrite ((char *) &(m[MERGE].wcnt), sizeof (m[MERGE].wcnt), 1, m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[MERGE].fcnt);
+  NTOHUL(m[MERGE].wcnt);
+
+  memcpy (prevTerm, m[x].term, prefix + suffix + 1);
+  m[MERGE].idh.total_bytes += m[x].term[0] + 1;
+  m[MERGE].idh.index_string_bytes += m[x].term[0] + 2 - prefix;
+  fflush (m[MERGE].dict);
+}
+
+
+/*=======================================================================
+ * process_merge_invf(): The main loop
+ *=======================================================================*/
+int 
+process_merge_invf (void)
+/*
+ *  The main function to merge the lexicons and inverted files.
+ */
+{
+/*** Some extra Initialisation stuff first ***/
+
+  m[OLD].term[0] = m[NEW].term[0] = '\0';
+  m[OLD].term_count = m[NEW].term_count = 0;
+  m[OLD].done = m[NEW].done = 0;
+  m[MERGE].idh.total_bytes = m[MERGE].idh.index_string_bytes = 0;
+
+/*** malloc arrays: mergedata, m[OLD].fcntlist, m[NEW].fcntlist ***/
+  mergedata = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (char));
+  if (mergedata == 0)
+    {
+      fprintf (stderr, "MALLOC error!\n");
+      exit (1);
+    }
+  {
+    int i;
+    for (i = 0; i < 2; i++)
+      {
+	m[i].fcntlist = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (int));
+	if (m[i].fcntlist == 0)
+	  {
+	    fprintf (stderr, "MALLOC error!\n");
+	    exit (1);
+	  }
+      }
+  }
+
+/*========================*/
+/***    LEXICON PASS    ***/
+/*========================*/
+
+/*** read first terms ***/
+  readTerm (OLD);
+  readTerm (NEW);
+
+  m[MERGE].nTerms = 0;
+  while (m[OLD].done == 0 || m[NEW].done == 0)
+    {
+      int i;
+      if (m[OLD].done)
+	i = 1; /* NEW will always be greater */
+      else if (m[NEW].done)
+	i = -1; /* OLD will always be greater */
+      else
+	i = casecompare (m[OLD].term, m[NEW].term);
+
+      if (i < 0)
+	{			/* term in OLD only */
+	  mergedata[m[MERGE].nTerms] = (char) OLD;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (OLD);
+	  readTerm (OLD);
+	}
+      if (i == 0)
+	{			/* term in both lexions */
+	  mergedata[m[MERGE].nTerms] = (char) MERGE;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[MERGE].nTerms++;
+	  writeTerm (MERGE);
+	  readTerm (OLD);
+	  readTerm (NEW);
+	}
+      if (i > 0)
+	{			/* term in NEW only */
+	  mergedata[m[MERGE].nTerms] = (char) NEW;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (NEW);
+	  readTerm (NEW);
+	}
+    }/*while*/
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  /* print some results about terms */
+  {
+    fprintf (stderr, "   Terms: OLD = %ld,  NEW = %ld,  MERGE = %ld\n",
+	     m[OLD].nTerms, m[NEW].nTerms, m[MERGE].nTerms);
+    fprintf (stderr, "   In OLD only: %ld,  In NEW only: %ld,  BOTH: %ld\n",
+	     (m[MERGE].nTerms - m[NEW].nTerms),
+	     (m[MERGE].nTerms - m[OLD].nTerms),
+	     (m[OLD].nTerms + m[NEW].nTerms - m[MERGE].nTerms));
+  }
+
+
+/*========================*/
+/***      INVF PASS     ***/
+/*========================*/
+
+  bytes_output = ftell (m[MERGE].invf);
+  BIO_Stdio_Decode_Start (m[OLD].invf, &m[OLD].sbs);
+  BIO_Stdio_Decode_Start (m[NEW].invf, &m[NEW].sbs);
+  BIO_Stdio_Encode_Start (m[MERGE].invf, &m[MERGE].sbs);
+  {
+    int i;
+    for (i = 0; i < m[MERGE].nTerms; i++)
+      {
+	processEntry (i, mergedata[i]);
+      }
+    /* write final .invf.idx pointer */
+    HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+    fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+    NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+    fprintf (stderr, "Old invf: old only bits: %ld, merged bits: %ld\n",
+	     oldOnlyBits, oldMergeBits);
+  }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * done_merge_invf(): write headers, close files, append weights to
+ *                     weights file
+ *=======================================================================*/
+int 
+done_merge_invf (void)
+{
+/*** write new lexicon header ***/
+  m[MERGE].idh.dict_size = m[MERGE].nTerms;
+  m[MERGE].idh.lookback = m[OLD].idh.lookback;
+  m[MERGE].idh.input_bytes = m[OLD].idh.input_bytes + m[NEW].idh.input_bytes;
+  m[MERGE].idh.total_bytes = m[OLD].idh.total_bytes + m[NEW].idh.total_bytes;
+  m[MERGE].idh.num_of_docs = m[MERGE].nDocs;
+  m[MERGE].idh.num_of_words = m[OLD].idh.num_of_words + m[NEW].idh.num_of_words;
+  m[MERGE].idh.stemmer_num = m[OLD].idh.stemmer_num;
+  m[MERGE].idh.stem_method = m[OLD].idh.stem_method;
+
+  /* if fastMerge, static num of docs stays the same! */
+  if (fastMerge)
+    m[MERGE].idh.static_num_of_docs = m[OLD].idh.static_num_of_docs;
+  else
+    m[MERGE].idh.static_num_of_docs = m[MERGE].nDocs;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].idh.lookback);
+  HTONUL(m[MERGE].idh.dict_size);
+  HTONUL(m[MERGE].idh.total_bytes);
+  HTONUL(m[MERGE].idh.index_string_bytes);
+  HTOND(m[MERGE].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(m[MERGE].idh.num_of_docs);
+  HTONUL(m[MERGE].idh.static_num_of_docs);
+  HTONUL(m[MERGE].idh.num_of_words);
+  HTONUL(m[MERGE].idh.stemmer_num);
+  HTONUL(m[MERGE].idh.stem_method);
+
+  fseek (m[MERGE].dict, magicsize, 0);
+  fwrite (&(m[MERGE].idh), sizeof (struct invf_dict_header), 1, m[MERGE].dict);
+
+/*** write new inverted file header ***/
+  m[MERGE].ifh.no_of_words = m[MERGE].nTerms;
+  m[MERGE].ifh.no_of_ptrs = m[OLD].ifh.no_of_ptrs + m[NEW].ifh.no_of_ptrs;
+  m[MERGE].ifh.skip_mode = m[OLD].ifh.skip_mode;
+  m[MERGE].ifh.InvfLevel = m[OLD].ifh.InvfLevel;
+
+  /* ifh.params[16] -- I have NO IDEA what these do!!!!! */
+  /* only set if a inverted file is in skipped format, I think */
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = m[OLD].ifh.params[i];
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].ifh.no_of_words);
+  HTONUL(m[MERGE].ifh.no_of_ptrs);
+  HTONUL(m[MERGE].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      HTONUL(m[MERGE].ifh.params[i]);
+  }
+  HTONUL(m[MERGE].ifh.InvfLevel);
+
+  fseek (m[MERGE].invf, magicsize, 0);	/* go to start of header */
+  fwrite (&(m[MERGE].ifh), sizeof (struct invf_file_header), 1, m[MERGE].invf);
+
+/*** Write new document weights --- leave old weights alone ***/
+  if ((weightOption) && (m[OLD].ifh.InvfLevel >= 2))
+    {
+      FILE *weightfile = open_file (m[MERGE].fname, WEIGHTS_SUFFIX, "r+b",
+				    MAGIC_WGHT, MG_CONTINUE);  /* [RPAP - Feb 97: WIN32 Port] */
+      if (weightfile)
+	{
+	  u_long i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+	  fprintf (stderr, "writing new weights: Nnew = %ld\n", m[NEW].nDocs);
+	  fseek (weightfile, 0, 2);	/* end of file */
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (i = 0; i < m[NEW].nDocs; i++)
+	    HTONF(DocWeightBuffer[i]);
+
+	  fwrite ((char *) DocWeightBuffer, sizeof (float), m[NEW].nDocs,
+		  weightfile);
+	}
+      fclose (weightfile);
+    }
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      /* Update paragraph file */
+      int buffer_size = 1000;
+      u_char *buffer;
+      int num;
+
+      if (!(buffer = (u_char *) Xmalloc (buffer_size)))
+	FatalError (1, "Could not allocate memory for paragraph buffer");
+
+      fseek (m[MERGE].para, 0, SEEK_END);
+      while ((num = fread ((char *) buffer, sizeof (*buffer), buffer_size, m[NEW].para)) != 0)
+	fwrite ((char *) buffer, sizeof (*buffer), num, m[MERGE].para);
+
+      Xfree (buffer);
+    }
+
+/*** close files ***/
+  fclose (m[MERGE].invf);
+  fclose (m[OLD].invf);
+  fclose (m[NEW].invf);
+  fclose (m[MERGE].dict);
+  fclose (m[OLD].dict);
+  fclose (m[NEW].dict);
+  fclose (m[MERGE].idx);
+  fclose (m[OLD].idx);
+  fclose (m[NEW].idx);
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      fclose (m[NEW].para);
+      fclose (m[MERGE].para);
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s [-s] [-w] [-d directory] -f mg_collection\n",
+	   progname);
+  fprintf (stderr, "  example: %s -s -w -f MERGE/mailfiles\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int 
+main (int argc, char *argv[])
+{
+  int ch;
+  char *progname;
+  progname = argv[0];
+  m[MERGE].fname[0] = '\0';
+
+  /* message and timing information */
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:swh")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (m[MERGE].fname, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	fastMerge = 0;
+	break;
+      case 'w':
+	weightOption = 1;
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (m[MERGE].fname[0] == '\0')
+    usage (progname);
+  strcpy (m[OLD].fname, m[MERGE].fname);
+  strcat (m[OLD].fname, ".old");
+  strcpy (m[NEW].fname, m[MERGE].fname);
+  strcat (m[NEW].fname, ".new");
+
+  GetTime (&start);
+  init_merge_invf ();
+  process_merge_invf ();
+  done_merge_invf ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  exit (0);
+}
+
+/*************************************************************************/
+/*   EOF: mg_invf_merge_new.c                                            */
+/*************************************************************************/
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.1	(revision 21112)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_invf_rebuild.c	(revision 21112)
@@ -0,0 +1,567 @@
+/**************************************************************************
+ *
+ * mg_invf_rebuild.c -- Program to rebuild an inverted file with skipping
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define LEN 40
+#define WRD 1
+
+typedef struct invf_info
+  {
+    unsigned long doc_num, count, bits_so_far, count_bits;
+  }
+invf_info;
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:11  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:03  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:51  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+static char pathname[256];
+
+static void process_files (char *filename);
+
+static int k = -1;
+static int mode = 0;
+static int max_nodes = -1;
+static int mins = -1;
+
+void 
+usage (char *pgname)
+{
+  fprintf (stderr, "usage: %s [-f input_file]"
+       "[-d data directory] -0|-1|-2 [-k num] [-m num] [-s num]\n", pgname);
+  printf ("Mode 0: (the default)\n");
+  printf ("\tk, m, and s have no meaning and if specified produce an error\n");
+  printf ("Mode 1:\n");
+  printf ("\tm, and s have no meaning and if specified produce an error\n");
+  printf ("\tk The size of skips\n");
+  printf ("Mode 2:\n");
+  printf ("\tk has no meaning and if specified will produce an error\n");
+  printf ("\tm is the the number of accumulators that will be used for the\n");
+  printf ("\t  ranking. The program builds an inverted file that is \n");
+  printf ("\t  \"optimal\" for that number of accumulators.\n");
+  printf ("\ts is the minimum size for skips.\n");
+  exit (1);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  strcpy (pathname, dir_name ? dir_name : ".");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "012hf:d:k:b:s:m:")) != -1)
+    switch (ch)
+      {
+      case '0':
+	mode = 0;
+	break;
+      case '1':
+	mode = 1;
+	break;
+      case '2':
+	mode = 2;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	strcpy (pathname, optarg);
+	break;
+      case 'k':
+	k = atoi (optarg);
+	break;
+      case 'm':
+	max_nodes = atoi (optarg);
+	break;
+      case 's':
+	mins = atoi (optarg);
+	if (mins < 1)
+	  FatalError (1, "The number for the -m option must be greater"
+		      " than or equal to 1");
+	break;
+      case 'h':
+      case '?':
+	usage (argv[0]);
+      }
+
+  if ((mode == 0 && (k != -1 || max_nodes != -1 || mins != -1)) ||
+      (mode == 1 && (max_nodes != -1 || mins != -1)) ||
+      (mode == 2 && (k != -1)))
+    {
+      Message ("Illegal parameters for mode");
+      usage (argv[0]);
+    }
+
+  if (mode == 1 && k == -1)
+    {
+      k = 8;
+      Message ("k is required for mode 1: defaulting k to %d", k);
+    }
+  if (mode == 2 && max_nodes == -1)
+    {
+      max_nodes = 1024;
+      Message ("m is required for mode 2: defaulting m to %d", max_nodes);
+    }
+  if (mode == 2 && mins == -1)
+    {
+      mins = 1;
+      Message ("s is required for mode 2: defaulting s to %d", mins);
+    }
+
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  Message ("**** Don\'t forget to rebuild the stemmed dictionary with mg_invf_dict. ****\n");
+  Message ("**** If the collection was built with stem indexes don\'t forget to     ****\n");
+  Message ("**** rebuild them with mg_stem_idx.                                     ****\n");
+
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *in, *out, *idx, *odx, *dict;
+  unsigned long magic, outmode, N, in_k, out_k;
+  unsigned long bits_out, bytes_out, i, j;
+  stdio_bitio_state out_buf, in_buf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh_in, ifh_out;
+  char FName[256];
+
+  outmode = mode;
+
+  /* open .invf.ORG, rename .invf if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      rename (fname, FName);
+      if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.ORG */
+  if (fread (&magic, sizeof (magic), 1, in) != 1 || NTOHUL(magic) != MAGIC_INVF)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+
+  /* open .invf.idx.ORG, rename .invf.idx if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      rename (fname, FName);
+      if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.idx.ORG */
+  if (fread (&magic, sizeof (magic), 1, idx) != 1 || NTOHUL(magic) != MAGIC_INVI)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(out = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(odx = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Opening \"%s\"\n", FName);
+  if (!(dict = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  if (fread (&magic, sizeof (magic), 1, dict) != 1 || NTOHUL(magic) != MAGIC_STEM_BUILD)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  HTONUL2(MAGIC_INVF, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, out);
+
+  fread ((char *) &ifh_in, sizeof (ifh_in), 1, in);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_in.no_of_words);
+  NTOHUL(ifh_in.no_of_ptrs);
+  NTOHUL(ifh_in.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_in.params[i]);
+  NTOHUL(ifh_in.InvfLevel);
+
+  ifh_out = ifh_in;
+  ifh_out.skip_mode = outmode;
+  bzero ((char *) ifh_out.params, sizeof (ifh_out.params));
+  switch (outmode)
+    {
+    case 0:
+      break;
+    case 1:
+      ifh_out.params[0] = k;
+      break;
+    case 2:
+      ifh_out.params[0] = max_nodes;
+      ifh_out.params[1] = mins;
+      break;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(ifh_out.no_of_words);
+  HTONUL(ifh_out.no_of_ptrs);
+  HTONUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    HTONUL(ifh_out.params[i]);
+  HTONUL(ifh_out.InvfLevel);
+
+  fwrite ((char *) &ifh_out, sizeof (ifh_out), 1, out);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_out.no_of_words);
+  NTOHUL(ifh_out.no_of_ptrs);
+  NTOHUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_out.params[i]);
+  NTOHUL(ifh_out.InvfLevel);
+
+  Message ("The file is a level %d inverted file.\n", ifh_in.InvfLevel);
+
+  bits_out = ftell (out) * 8;
+
+
+  HTONUL2(MAGIC_INVI, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, odx);
+
+  DECODE_START (in)
+    DECODE_PAUSE (in_buf)
+
+    ENCODE_START (out)
+    ENCODE_PAUSE (out_buf)
+
+    N = idh.num_of_docs;
+
+  for (i = 0; i < ifh_in.no_of_words; i++)
+    {
+      unsigned long blk, p;
+      unsigned long odn_blk = 0, olen_blk = 0;
+      unsigned long idn_blk = 0, ilen_blk = 0;
+      register unsigned long suff;
+      unsigned long fcnt, wcnt, doc_num, bits_so_far, last_major;
+      unsigned long next_mjr_dn, kd;
+      char dummy2[MAXSTEMLEN + 1];
+      invf_info *ii;
+
+      fgetc (dict);
+      suff = fgetc (dict);
+      fread (dummy2, sizeof (u_char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+      NTOHUL(bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      p = fcnt;
+      blk = BIO_Bblock_Init (idh.static_num_of_docs, p);
+      switch (outmode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_out.params[0])
+	      out_k = 0;
+	    else
+	      {
+		out_k = ifh_out.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= mins)
+	      out_k = 0;
+	    else
+	      {
+		out_k = (int) (2 * sqrt ((double) p / max_nodes));
+		if (out_k <= mins)
+		  out_k = mins;
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	default:
+	  out_k = 0;
+	}
+
+      switch (ifh_in.skip_mode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[0])
+	      in_k = 0;
+	    else
+	      {
+		in_k = ifh_in.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[1])
+	      {
+		in_k = 0;
+	      }
+	    else
+	      {
+		in_k = (int) (2 * sqrt ((double) p / ifh_in.params[0]));
+		if (in_k <= ifh_in.params[1])
+		  in_k = ifh_in.params[1];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	default:
+	  in_k = 0;
+	}
+
+      if (!(ii = Xmalloc (sizeof (invf_info) * p)))
+	FatalError (1, "Unable to allocate memory for \"ii\"\n");
+
+      doc_num = bits_so_far = 0;
+      next_mjr_dn = 0;
+      kd = 0;
+      DECODE_CONTINUE (in_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  unsigned long doc_diff, count = 0;
+	  if (kd == in_k)
+	    kd = 0;
+	  if (in_k && kd == 0 && j + in_k < p)
+	    {
+	      int temp;
+	      BBLOCK_DECODE (next_mjr_dn, idn_blk);
+	      next_mjr_dn += doc_num;
+	      BBLOCK_DECODE (temp, ilen_blk);
+	    }
+	  ii[j].bits_so_far = bits_so_far;
+	  if (in_k && kd == in_k - 1 && j != p - 1)
+	    {
+	      int count;
+	      BBLOCK_LENGTH (next_mjr_dn - doc_num, blk, count);
+	      bits_so_far += count;
+	      doc_num = next_mjr_dn;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (doc_diff, blk, bits_so_far);
+	      doc_num += doc_diff;
+	    }
+	  ii[j].doc_num = doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    {
+	      int count_bits = 0;
+	      GAMMA_DECODE_L (count, count_bits);
+	      ii[j].count_bits = count_bits;
+	      bits_so_far += count_bits;
+	      ii[j].count = count;
+	    }
+	}
+
+      /* read till a byte boundary */
+      while (__btg)
+	{
+	  DECODE_BIT;
+	  bits_so_far++;
+	}
+
+      DECODE_PAUSE (in_buf)
+
+	doc_num = bits_so_far = 0;
+      last_major = 0;
+      kd = 0;
+      ENCODE_CONTINUE (out_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  if (kd == out_k)
+	    kd = 0;
+	  if (out_k && kd == 0)
+	    {
+	      if (j + out_k < p)
+		{
+		  int num = ii[j + out_k - 1].doc_num - last_major;
+		  BBLOCK_ENCODE_L (num, odn_blk, bits_out);
+		  last_major = ii[j + out_k - 1].doc_num;
+
+		  num = ii[j + out_k - 1].bits_so_far + ii[j + out_k - 1].count_bits -
+		    bits_so_far;
+		  BBLOCK_ENCODE_L (num, olen_blk, bits_out);
+		  bits_so_far = ii[j + out_k].bits_so_far;
+		}
+	    }
+	  if (!(out_k && kd == out_k - 1 && j != p - 1))
+	    BBLOCK_ENCODE_L (ii[j].doc_num - doc_num, blk, bits_out);
+	  doc_num = ii[j].doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    GAMMA_ENCODE_L (ii[j].count, bits_out);
+	}
+
+      /* write till a byte boundary */
+      while (__btg != 8)
+	{
+	  ENCODE_BIT (0);
+	  bits_out++;
+	}
+      ENCODE_PAUSE (out_buf)
+
+	Xfree (ii);
+
+    }
+  ENCODE_CONTINUE (out_buf)
+    ENCODE_DONE
+
+    HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+  NTOHUL(bytes_out);
+
+  fclose (idx);
+  fclose (odx);
+  fclose (in);
+  fclose (out);
+  fclose (dict);
+
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_merge.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_merge.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_merge.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mgmerge.h --- mgmerge #defines, etc
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edtited: November 12 1994
+ *
+ **************************************************************************/
+
+#ifndef MGMERGE_H
+#define MGMERGE_H
+
+typedef unsigned char byte;
+
+#define OLD 0
+#define NEW 1
+#define MERGE 2
+
+#define OK 0
+#define ERROR_MGMERGE 1
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/mg_passes.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_passes.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_passes.1	(revision 21112)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_passes.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_passes.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_passes.c	(revision 21112)
@@ -0,0 +1,641 @@
+/**************************************************************************
+ *
+ * mg_passes.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+#include <stdlib.h>
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.3  2004/11/29 03:15:13  kjdon
+   added some changes made by Emanuel Dejanu (Simple Words)
+
+   Revision 1.2  2004/04/25 23:01:18  kjdon
+   added a new -M option to mg_passes, allowing maxnumeric to be altered - made this change to keep gsdl3 mg inline with gsdl2 mg.
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.3  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.2  2001/06/12 23:23:42  jrm21
+   fixed a bug where mg_passes segfaults when trying to print the usage message.
+
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
+"  %*s [-a stemmer] [-M max-numeric] -f doc-collection-name\n";
+
+
+static void 
+usage (char *err)
+{
+  if (err)
+    Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "",strlen (msg_prefix), "",
+	   strlen (msg_prefix),"");
+  exit (1);
+}
+
+
+
+
+#if 0
+static char *
+str_comma (unsigned long u)
+{
+  static char buf[20];
+  unsigned long a, b, c, d;
+  a = u / 1000000000;
+  u -= a * 1000000000;
+  b = u / 1000000;
+  u -= b * 1000000;
+  c = u / 1000;
+  u -= c * 1000;
+  d = u;
+
+  if (a)
+    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
+  else if (b)
+    sprintf (buf, "%u,%03u,%03u", b, c, d);
+  else if (c)
+    sprintf (buf, "%u,%03u", c, d);
+  else
+    sprintf (buf, "%u", d);
+  return (buf);
+}
+#endif
+
+
+
+
+int 
+open_next_file (int in_fd)
+{
+  if (in_fd > 0)
+    close (in_fd);
+  if (num_files == 0)
+    return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  files++;
+  num_files--;
+  return (in_fd);
+}
+
+
+static void 
+driver (int in_fd, FILE * Trace, char *file_name)
+{
+  int pass, num = 1;
+
+  char *buffer = Xmalloc (buf_size);
+  unsigned long num_docs = 0;
+  unsigned long block_bytes = 0;
+  register int buf_left = buf_size;
+  register char *look_pos = buffer;
+  register char *end_pos = buffer;
+
+  ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+  GetTime (&StartTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->init_time);
+#endif
+      }
+
+  GetTime (&InitTime);
+  for (;;)
+    {
+      int len = 0;
+      char *base = look_pos;
+
+      while (look_pos != end_pos && *look_pos != TERMRECORD)
+	look_pos++;
+
+      while (look_pos == end_pos)
+	{
+	  if (buf_left < MIN_BUF)
+	    {
+	      bcopy (base, buffer, end_pos - base);
+	      look_pos = buffer + (end_pos - base);
+	      buf_left = buf_size - (end_pos - base);
+	      end_pos = look_pos;
+	      base = buffer;
+	    }
+	  if (buf_left)
+	    {
+    	      num = read (in_fd, end_pos, buf_left);
+	      if (num < 0) num = 0; /* RJM - quick hack :-) */
+	      if (num == 0)
+		if ((in_fd = open_next_file (in_fd)) != -1)
+		  num = read (in_fd, end_pos, buf_left);
+	      bytes_received += num;
+	      buf_left -= num;
+	      end_pos += num;
+	    }
+	  while (look_pos < end_pos && *look_pos != TERMRECORD)
+	    look_pos++;
+	  if (buf_left == 0 && base == buffer && look_pos == end_pos)
+	    {
+	      Message ("Unable to find document terminator (i.e ^B)"
+		       " in the document");
+	      FatalError (1, "The document is in excess of %d chars long",
+			  look_pos - base);
+	    }
+	  if (!num)
+	    break;
+	}
+      len = look_pos++ - base;
+
+      if (!num && base == end_pos)
+	break;
+
+      bytes_processed += len;
+
+#ifndef QUIET
+      if (!len)
+	Message ("Warning : Processing zero length document");
+#endif
+
+      for (pass = 0; pass < MAX_PASSES; pass++)
+	if (Passes & (1 << pass))
+	  {
+	    register pass_data *pd = &PassData[pass];
+
+#ifdef HAVE_TIMES
+	    struct tms tims;
+	    times (&tims);
+	    pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    struct rusage ru;
+	    register struct timeval *tv = &pd->process_time;
+
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+
+	    if (pd->process ((u_char *) base, len) == COMPERROR)
+	      {
+		Message ("Error during processing of \"%s\"", pd->name);
+		if (Dump || Trace)
+		  {
+		    int i;
+		    FILE *f = Trace ? Trace : stderr;
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		    for (i = 0; i < len; i++)
+		      {
+			char ch = base[i];
+			if (ch == '\1' || ch == '\2')
+			  ch = '\n';
+			putc (ch, f);
+		      }
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		  }
+		if (Trace)
+		  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+			   bytes_processed, num_docs,
+			   ElapsedTime (&StartTime, NULL));
+		exit (1);
+	      }
+
+#ifdef HAVE_TIMES
+	    times (&tims);
+	    pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	  }
+      num_docs++;
+      if (Trace)
+	{
+	  block_bytes += (look_pos - base);
+	  if (block_bytes >= trace)
+	    {
+#ifdef HAVE_MALLINFO
+	      struct mallinfo mi;
+	      mi = mallinfo ();
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		       ElapsedTime (&StartTime, NULL));
+#else
+	      block_bytes -= trace;
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+#endif
+	    }
+	}
+      if (!num && look_pos - 1 == end_pos)
+	break;
+    }
+
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  free (buffer);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  int ch, in_fd;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:M:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'M':
+	  SetEnv ("maxnumeric", optarg, NULL);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc)
+    {
+      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+	FatalError (1, "Cannot open %s", argv[optind]);
+      files = &argv[optind + 1];
+      num_files = argc - (optind + 1);
+    }
+  else
+    in_fd = 0;			/* stdin */
+
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (in_fd, Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.c	(revision 21112)
@@ -0,0 +1,546 @@
+/**************************************************************************
+ *
+ * mg_passes_4jni.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+/* this needs to come first */
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include "mg_passes_4jni.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "longlong.h"
+#include "stemmer.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+
+#include "words.h"
+#include "environment.h"
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+mg_ullong bytes_processed = 0;
+mg_ullong bytes_received = 0;
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+FILE * Trace;
+char * filename;
+unsigned long num_docs = 0;
+unsigned long block_bytes = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+int mg_passes_exit_value = 0;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+
+/* clear all the settings from one mg_passes run to the next */
+void clear_variables() {
+
+  buf_size = 3 * 1024 * 1024;	/* 3Mb */
+  invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+  ChunkLimit = 0;
+  InvfLevel = 2;
+  SkipSGML = 0;
+  MakeWeights = 0;
+  Comp_Stats = NULL;
+  comp_stat_point = 0;
+  bytes_processed = 0;
+  bytes_received = 0;
+  stemmer_num = 0; /* default to the lovin stemmer */
+  stem_method = 0;
+  Trace = NULL;
+  filename = NULL;
+  num_docs = 0;
+  block_bytes = 0;
+  
+  Passes = 0;
+  trace = 0;
+  Dump = 0;
+  files = NULL;
+  num_files = 0;
+  trace_name = NULL;
+  
+  mg_passes_exit_value = 0;
+}
+
+/* ################################################## */
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num) {
+
+  switch(pass_type) {
+  case 'S':
+    Passes |= SPECIAL;
+    break;
+  case 'I':
+  case 'N':
+    if (pass_num == '1')
+      Passes |= IVF_PASS_1;
+    else if (pass_num == '2')
+      Passes |= IVF_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  case 'T':
+    if (pass_num == '1')
+      Passes |= TEXT_PASS_1;
+    else if (pass_num == '2')
+      Passes |= TEXT_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  }
+}
+
+/* -D arg to mg_passes */
+void dump_failed_document(int dump) {
+  Dump = dump;
+}
+
+/* -G arg to mg_passes */
+void ignore_sgml_tags(int ignore) {
+  if (ignore) {
+    SkipSGML = 1;
+  } else {
+    SkipSGML = 0;
+  }
+}
+
+/* -b arg to mg_passes */
+void set_buffer_size(long size) {
+  buf_size = size * 1024;
+  if (buf_size <  MIN_BUF) {
+    buf_size = MIN_BUF;
+  }
+}
+
+/* -c arg to mg_passes */
+void set_chunk_limit(long chunk_limit) {
+  ChunkLimit = chunk_limit;
+}
+
+/* -C arg to mg_passes */
+void set_comp_stat_point(int stat_point) {
+  comp_stat_point = stat_point * 1024;
+}
+
+/* -f arg to mg_passes */
+void set_filename(const char * filen) {
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+  filename = Xstrdup (filen);  
+}
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit) {
+  invf_buffer_size = limit * 1024 * 1024;
+}
+
+/* -1, -2, -3 args to mg_passes */
+void set_invf_level(char level) {
+  switch (level) {
+  case '1':
+    InvfLevel = 1;
+    break;
+  case '2':
+    InvfLevel = 2;
+    break;
+  case '3':
+    InvfLevel = 3;
+    break;
+  }
+}
+
+/* -W arg to mg_passes */
+void set_make_weights(int make_w) {
+  MakeWeights = make_w;
+}
+
+/* -M arg to mg_passes */
+void set_max_numeric(int max_numeric) {
+  char data[99];
+  sprintf(data, "%d", max_numeric);
+  SetEnv ("maxnumeric", data, NULL);
+}
+
+/* -a, -s args to mg_passes */
+void set_stem_options(const char * stemmer, int method) {
+  stemmer_num = stemmernumber (stemmer);
+  stem_method = method & STEMMER_MASK;
+}
+
+/* -t arg to mg_passes */
+void set_trace_point(int tracepos) {
+  trace = (unsigned long) (tracepos * 1024 * 1024);
+}
+
+/* -n arg to mg_passes */
+void set_trace_file(const char * filen) {
+  if (trace_name) {
+    Xfree (trace_name);
+    trace_name = NULL;
+  }
+  trace_name = Xstrdup (filen);
+}
+
+/* ############################################### */
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+
+ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+void 
+init_driver ()
+{
+  int pass;
+  if (!filename || *filename == '\0') {
+    mg_passes_exit_value = 1;
+    FatalError (1, "A document collection name must be specified.");
+  }
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2)) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  }
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2)) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+  }
+  if (!Passes) {
+    mg_passes_exit_value = 1;
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+  }
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+  if (Trace)
+    {
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      /* print out the args to mg_passes */
+      fprintf (Trace, "\n\n");
+    }
+
+
+  GetTime (&StartTime);
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+      struct tms tims;
+      times (&tims);
+      pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      struct rusage ru;
+      
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      if (pd->init (filename) == COMPERROR) {
+	mg_passes_exit_value = 1;
+	FatalError (1, "Error during init of \"%s\"", pd->name);
+      }
+#ifdef HAVE_TIMES
+      times (&tims);
+      pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+      getrusage (RUSAGE_SELF, &ru);
+      pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+      pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+      time_normalise (&pd->init_time);
+#endif
+    }
+  }
+  GetTime (&InitTime);
+}
+
+
+void process_document(const u_char *buffer, int len) {
+  int pass;
+  bytes_processed += len;
+
+#ifndef QUIET
+  if (!len)
+    Message ("Warning : Processing zero length document");
+#endif
+  
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass))
+      {
+	register pass_data *pd = &PassData[pass];
+	
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	register struct timeval *tv = &pd->process_time;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->process ((u_char *) buffer, len) == COMPERROR)
+	  {
+	    Message ("Error during processing of \"%s\"", pd->name);
+	    if (Dump || Trace)
+	      {
+		int i;
+		FILE *f = Trace ? Trace : stderr;
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		for (i = 0; i < len; i++)
+		  {
+		    char ch = buffer[i];
+		    if (ch == '\1' || ch == '\2')
+		      ch = '\n';
+		    putc (ch, f);
+		  }
+		fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+	      }
+	    if (Trace)
+	      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+	    exit (1);
+	  }
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+      }
+  }
+  num_docs++;
+  if (Trace)
+    {
+      block_bytes += len;
+      if (block_bytes >= trace)
+	{
+#ifdef HAVE_MALLINFO
+	  struct mallinfo mi;
+	  mi = mallinfo ();
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+		   bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		   ElapsedTime (&StartTime, NULL));
+#else
+	  block_bytes -= trace;
+	  fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+		   bytes_processed, num_docs,
+		   ElapsedTime (&StartTime, NULL));
+#endif
+	}
+    }
+}
+
+void finalise_driver() {
+  int pass;
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+	
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (filename) == COMPERROR) {
+	  mg_passes_exit_value = 1;
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+	}
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%11" ULL_FS " bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %" ULL_FS, bytes_received);
+  Message ("Bytes processed : %" ULL_FS, bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  //free (buffer);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+}
+
+int get_exit_value() {
+  return mg_passes_exit_value;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_passes_4jni.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mg_passes_4jni.h -- modified mg_passes for use from jni
+ * Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef H_MG_PASSES_4JNI
+#define H_MG_PASSES_4JNI
+
+/* clear all the settings from one mg_passes run to the next */
+void clear_variables();
+
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass(char pass_type, char pass_num);
+
+/* -D arg to mg_passes */
+void dump_failed_document(int dump);
+
+/* -G arg to mg_passes */
+void ignore_sgml_tags(int ignore);
+
+/* -b arg to mg_passes */
+void set_buffer_size(long size);
+
+/* -c arg to mg_passes */
+void set_chunk_limit(long chunk_limit);
+
+/* -C arg to mg_passes */
+void set_comp_stat_point(int stat_point);
+
+/* -f arg to mg_passes */
+void set_filename(const char * filen);
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit);
+
+/* -1, -2, -3 args to mg_passes */
+void set_invf_level(char level);
+
+/* -W arg to mg_passes */
+void set_make_weights(int make_w);
+
+/* -M arg to mg_passes */
+void set_max_numeric(int max_numeric);
+
+/* -a, -s args to mg_passes */
+void set_stem_options(const char * stemmer, int method);
+
+/* -t arg to mg_passes */
+void set_trace_point(int tracepos);
+
+/* -n arg to mg_passes */
+void set_trace_file(const char * filen);
+
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver ();
+
+void process_document(const u_char *buffer, int len);
+
+void finalise_driver();
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_perf_hash_build.c	(revision 21112)
@@ -0,0 +1,186 @@
+/**************************************************************************
+ *
+ * mg_perf_hash_build.c -- Program to build a perfect hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:13  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:15  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:58  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:53  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#define POOL_SIZE 1024*1024
+
+static void process_files (char *filename);
+
+int r = -1;
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'r':
+	r = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *dict, *hash;
+  unsigned long i;
+  u_char prev[MAXSTEMLEN + 1];
+  struct invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  int pool_left;
+  u_char **starts;
+
+
+  dict = open_file (filename, INVF_DICT_SUFFIX, "rb",
+		    MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  hash = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+		      MAGIC_HASH, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (!(pool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = Xmalloc (sizeof (u_char *) * idh.dict_size)))
+    FatalError (1, "Out of memory");
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff, l;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (dict);
+      suff = getc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, dict);
+      fread (&wcnt, sizeof (wcnt), 1, dict);
+
+      l = *prev + 1;
+      if (pool_left < l)
+	{
+	  pool = Xmalloc (POOL_SIZE);
+	  pool_left = POOL_SIZE;
+	}
+      starts[i] = pool;
+      bcopy ((char *) prev, (char *) pool, l);
+      pool += l;
+      pool_left -= l;
+    }
+  if (!(phd = gen_hash_func (idh.dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hash, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (dict);
+  fclose (hash);
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_stem_idx.c	(revision 21112)
@@ -0,0 +1,892 @@
+/**************************************************************************
+ *
+ * mg_stem_idx.c -- Memory efficient stem index builder
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"     /* For struct stemmed_dict */
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+#include "local_strings.h"
+
+typedef struct PosEntry
+{
+  unsigned int num_cases;
+  unsigned int blk;
+  unsigned short blk_index;
+  unsigned short offset;
+}
+PosEntry;
+
+
+typedef struct PosList
+{
+  unsigned int list_size;
+  unsigned int num_entries;
+  PosEntry PE[1];
+}
+PosList;
+
+
+typedef struct idx_hash_rec
+{
+  u_char *word;
+  PosList *PL;
+}
+idx_hash_rec;
+
+
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+
+static idx_hash_rec **IdxHashTable;
+static unsigned long IdxHashSize;
+static unsigned long IdxHashUsed;
+static u_char *IdxPool;
+static int IdxPoolLeft;
+
+static idx_hash_rec *idx_hr_pool;
+static int idx_hr_PoolLeft;
+
+static idx_hash_rec **idx_first_occr;
+static int idx_max_first_occr;
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static long lookback = 4;
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+/* =========================================================================
+ * Function: MakePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+PosList *
+MakePosList (int n)
+{
+  PosList *pl;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  pl = Xmalloc (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+  if (!pl)
+    FatalError (1, "Unable to allocate term list");
+  ChangeMem (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+
+  pl->num_entries = n;
+  pl->list_size = list_size;
+
+  return pl;
+}
+
+/* =========================================================================
+ * Function: ResizePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizePosList (PosList ** pos_list)
+{
+  PosList *pl = *pos_list;
+
+  ChangeMem (-(sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)));
+  if (pl->num_entries > pl->list_size)
+    {
+      if (pl->list_size)
+	pl->list_size *= GROWTH_FACTOR;
+      else
+	pl->list_size = MIN_SIZE;
+    }
+  pl = Xrealloc (pl, sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  if (!pl)
+    FatalError (1, "Unable to resize pos list");
+  ChangeMem (sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  *pos_list = pl;
+}
+
+
+/* =========================================================================
+ * Function: FreePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreePosList (PosList ** the_pl)
+{
+  PosList *pl = *the_pl;
+
+  ChangeMem (-(sizeof(PosList) + sizeof (PosEntry) * (pl->list_size - 1)));
+  Xfree (pl);
+
+  *the_pl = NULL;
+}
+
+
+/* =========================================================================
+ * Function: ResetPosList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetPosList (PosList ** pl)
+{
+  if (*pl)
+    FreePosList (pl);
+  *pl = MakePosList (0);
+}
+
+/* =========================================================================
+ * Function: AddPosEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddPosEntry (PosList ** pos_list, PosEntry * pe)
+{
+  PosList *pl = *pos_list;
+
+  pl->num_entries++;
+  ResizePosList (pos_list);
+  pl = *pos_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) pe, (char *) &(pl->PE[pl->num_entries - 1]), sizeof (PosEntry));
+
+  return pl->num_entries - 1;
+}
+
+
+/* Modified from stem_search.c */
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    FatalError (1, "Could not allocate memory for stemmed dict");
+
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stemmer_num);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  return sd;
+}
+
+
+void
+mg_init_process ()
+{
+  /* Allocate memory for idx hash table */
+
+  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  IdxPoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec))))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  idx_hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+
+  IdxHashSize = INITIAL_HASH_SIZE;
+  IdxHashUsed = 0;
+  if (!(IdxHashTable = Xmalloc (sizeof (idx_hash_rec *) * IdxHashSize)))
+    FatalError (1, "Unable to allocate memory for idx table");
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashSize);
+  bzero ((char *) IdxHashTable, sizeof (idx_hash_rec *) * IdxHashSize);
+  idx_max_first_occr = 8192;
+  if (!(idx_first_occr = Xmalloc (sizeof (idx_hash_rec *) * idx_max_first_occr)))
+    FatalError (1, "Unable to allocate memory for idx_first_occur");
+  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+}
+
+
+void
+PackIdxHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < IdxHashSize; s++)
+    if (IdxHashTable[s])
+      IdxHashTable[d++] = IdxHashTable[s];
+  ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashUsed);
+  if (!(IdxHashTable = Xrealloc (IdxHashTable, sizeof (idx_hash_rec *) * IdxHashUsed)))
+    FatalError (1, "Out of memory");
+  IdxHashSize = IdxHashUsed;
+}
+
+
+void
+process_stem_dict (stemmed_dict * sd, int stem_method)
+{
+  int block;
+  short blk_index = -1;
+  int wordnum = -1;
+  PosEntry *prevPE = NULL;
+  idx_hash_rec *prevIdx = NULL;
+  u_char word[MAXSTEMLEN + 1];
+  u_char prev[MAXSTEMLEN + 1];
+
+  /* For each block in stem dict... */
+  for (block = 0; block < sd->sdh.num_blocks; block++)
+    {
+      register unsigned int res;
+      int num_indexes;
+      unsigned long *first_word, *last_invf_len;
+      unsigned short *num_words;
+      u_char *base;
+      unsigned short *index;
+
+      /* Read block into buffer */
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* Move through block header */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      {
+	/* [RPAP - Jan 97: Endian Ordering] */
+	int i;
+	for (i = 0; i < num_indexes; i++)
+	  NTOHUS(index[i]);
+      }
+
+      base = (u_char *) (index + num_indexes);
+      blk_index = -1;
+
+      /* For each word in block... */
+      for (res = 0; res < *num_words; res++)
+	{
+	  unsigned copy, suff;
+	  register idx_hash_rec *idx_ent = 0;
+
+	  /* Update blk_index */
+	  if (!(res % sd->sdh.lookback))
+	    blk_index++;
+
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+
+	  /* Skip irrelevant word info */
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+
+	  /* Stem word */
+	  bcopy ((char *) prev, (char *) word, *prev + 1);
+	  stemmer (stem_method, sd->sdh.stemmer_num, word);
+
+	  /* Check if word follows straight on from previous word */
+	  if (prevIdx && !compare (word, prevIdx->word))
+	    prevPE->num_cases++;
+
+	  else
+	    {
+	      /* Search the idx hash table for word */
+	      register unsigned long hashval, step;
+	      register int hsize = IdxHashSize;
+	      HASH (hashval, step, word, hsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  idx_ent = IdxHashTable[hashval];
+		  if (!idx_ent)
+		    {
+		      /* Create a next entry in the hash table */
+		      int len = *word + 1;
+		      if (!idx_hr_PoolLeft)
+			{
+			  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE *
+						       sizeof (idx_hash_rec))))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  idx_hr_PoolLeft = HASH_POOL_SIZE;
+			  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+			}
+		      idx_ent = idx_hr_pool++;
+		      idx_hr_PoolLeft--;
+		      if (len > IdxPoolLeft)
+			{
+			  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  IdxPoolLeft = POOL_SIZE;
+			  ChangeMem (POOL_SIZE);
+			}
+		      wordnum++;
+
+		      idx_ent->word = IdxPool;
+		      idx_ent->PL = MakePosList (0);
+		      {
+			PosEntry PE;
+			PE.num_cases = 1;
+			PE.blk = block;
+			PE.blk_index = (unsigned short) blk_index;
+			PE.offset = res % sd->sdh.lookback;
+			AddPosEntry (&(idx_ent->PL), &PE);
+		      }
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+
+		      bcopy ((char *) word, (char *) IdxPool, len);
+		      IdxPool += len;
+		      IdxPoolLeft -= len;
+		      if (IdxHashUsed == idx_max_first_occr - 1)
+			{
+			  ChangeMem (-sizeof (idx_hash_rec *) * idx_max_first_occr);
+			  idx_max_first_occr *= 2;
+			  if (!(idx_first_occr = Xrealloc (idx_first_occr, sizeof (idx_hash_rec *) *
+							   idx_max_first_occr)))
+			    FatalError (1, "Unable to allocate memory for idx_first_occr");
+			  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+			}
+		      idx_first_occr[IdxHashUsed] = idx_ent;
+		      IdxHashUsed++;
+		      IdxHashTable[hashval] = idx_ent;
+		      break;
+		    }
+		  
+		  /* Compare the words */
+		  s1 = word;
+		  s2 = idx_ent->word;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+		  
+		  if (len)
+		    {
+		      /* Entry is not the right one - move to next hash index */
+		      hashval = (hashval + step);
+		      if (hashval >= hsize)
+			hashval -= hsize;
+		    }
+		  else
+		    {
+		      /* Entry is correct - added PosEntry to word */
+		      PosEntry PE;
+		      PE.num_cases = 1;
+		      PE.blk = block;
+		      PE.blk_index = (unsigned short) blk_index;
+		      PE.offset = res % sd->sdh.lookback;
+		      AddPosEntry (&(idx_ent->PL), &PE);
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+		      break;
+		    }
+		}      
+	    }
+
+	  if (IdxHashUsed >= IdxHashSize >> 1)
+	    {
+	      idx_hash_rec **ht;
+	      unsigned long size;
+	      unsigned long i;
+	      size = prime (IdxHashSize * 2);
+	      if (!(ht = Xmalloc (sizeof (idx_hash_rec *) * size)))
+		FatalError (1, "Unable to allocate memory for idx table");
+	      bzero ((char *) ht, sizeof (idx_hash_rec *) * size);
+	      ChangeMem (sizeof (idx_hash_rec *) * size);
+
+	      for (i = 0; i < IdxHashSize; i++)
+		if (IdxHashTable[i])
+		  {
+		    register u_char *wptr;
+		    idx_hash_rec *ent;
+		    register unsigned long hashval, step;
+		    
+		    wptr = IdxHashTable[i]->word;
+		    HASH (hashval, step, wptr, size);
+		    ent = ht[hashval];
+		    while (ent)
+		      {
+			hashval += step;
+			if (hashval >= size)
+			  hashval -= size;
+			ent = ht[hashval];
+		      }
+		    ht[hashval] = IdxHashTable[i];
+		  }
+	      Xfree (IdxHashTable);
+	      ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+	      IdxHashTable = ht;
+	      IdxHashSize = size;
+	    }
+	  
+	} /* end for each word */
+
+    } /* end for each block */
+
+}
+ 
+static int 
+idx_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((idx_hash_rec **) A))->word;
+  u_char *s2 = (*((idx_hash_rec **) B))->word;
+  return (casecompare (s1, s2));
+}
+
+
+void
+save_idx (char * filename, int stem_method)
+{
+  char *FName;
+  unsigned long i, j, pos, First_word, num;
+  struct stem_idx_header sih;
+  u_char *buffer, *last_word = NULL;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+  FILE *idbi = NULL, *tmp = NULL;
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  /* Create appropriate stem index file */
+  switch (stem_method)
+    {
+    case (1):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "wb", MAGIC_STEM_1,
+			    MG_ABORT);
+	break;
+      }
+
+    case (2):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "wb", MAGIC_STEM_2,
+			    MG_ABORT);
+	break;
+      }
+    case (3):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "wb", MAGIC_STEM_3,
+			    MG_ABORT);
+	break;
+      }
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT create .invf.blocked.%d file", stem_method);
+
+  PackIdxHashTable();
+  qsort (IdxHashTable, IdxHashUsed, sizeof (idx_hash_rec *), idx_comp);
+
+  sih.lookback = lookback;
+  sih.block_size = block_size;
+  sih.num_blocks = 0;
+  sih.blocks_start = 0;
+  sih.index_chars = 0;
+  sih.num_of_words = IdxHashUsed;
+
+  fwrite ((char *) &sih, sizeof (sih), 1, idbi);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+
+  /* For each word in the hashtable... */
+  for (i = 0; i < IdxHashUsed; i++)
+    {
+      register unsigned long extra, copy, suff;
+      register struct idx_hash_rec *ent = IdxHashTable[i];
+
+      /* build a new word on top of prev */
+      if (last_word != NULL)
+	copy = prefixlen (last_word, ent->word);
+      else
+	copy = 0;
+      suff = *(ent->word) - copy;
+      last_word = ent->word;
+
+      if (word_num % sih.lookback == 0)
+	/* Will need copy chars to add + a pointer in index */
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + 
+	  ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	{
+	  /* Dump buffer to tmp file */
+	  int chunk;
+	  HTONUL(First_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUS(word_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sih.num_blocks++;
+
+	  /* Check that entry will fit into new block */
+	  if (sizeof (*pointers) + sizeof (ptrs_in_use) + extra + sizeof (First_word) +
+	      suff + 1 + sizeof (ent->PL->num_entries) + 
+	      ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	    FatalError (1, "Block size to small");
+
+	}
+
+      if (word_num % sih.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+
+      /* Output Word information */
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (ent->word + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &(ent->PL->num_entries), (char *) (buffer + buf_in_use), sizeof (ent->PL->num_entries));
+      NTOHUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (ent->PL->num_entries);
+
+      for (j = 0; j < ent->PL->num_entries; j++)
+	{
+	  register PosEntry *pe = &(ent->PL->PE[j]);
+	  HTONUI(pe->num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->num_cases), (char *) (buffer + buf_in_use), sizeof (pe->num_cases));
+	  buf_in_use += sizeof (pe->num_cases);
+	  HTONUI(pe->blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk), (char *) (buffer + buf_in_use), sizeof (pe->blk));
+	  buf_in_use += sizeof (pe->blk);
+	  HTONUS(pe->blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk_index), (char *) (buffer + buf_in_use), sizeof (pe->blk_index));
+	  buf_in_use += sizeof (pe->blk_index);
+	  HTONUS(pe->offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->offset), (char *) (buffer + buf_in_use), sizeof (pe->offset));
+	  buf_in_use += sizeof (pe->offset);
+	}
+
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 64209258\n");
+
+      if (word_num == 0)
+	{
+	  /* Write word to main index */
+	  fwrite (ent->word, sizeof (u_char), *(ent->word) + 1, idbi);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idbi);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sih.index_chars += *(ent->word) + 1;
+	  First_word = i;
+	}
+      word_num++;
+    } /* end for each word */
+
+  if (buf_in_use)
+    {
+      /* Write last buffer to tmp file */
+      int chunk;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sih.num_blocks++;
+    }
+
+  rewind (tmp);
+  sih.blocks_start = sih.index_chars + sizeof (u_long) + sizeof (sih) +
+    sih.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sih.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idbi);
+	  sih.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idbi);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idbi, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sih.lookback);
+  HTONUL(sih.block_size);
+  HTONUL(sih.num_blocks);
+  HTONUL(sih.blocks_start);
+  HTONUL(sih.index_chars);
+  HTONUL(sih.num_of_words);
+
+  fwrite (&sih, sizeof (sih), 1, idbi);
+  fclose (idbi);
+
+#ifndef SILENT
+  Message ("Stem %d:\n", stem_method);
+  Message ("     Block size   : %10d\n", block_size);
+  Message ("     Num_blocks   : %10d\n", NTOHUL(sih.num_blocks));  /* [RPAP - Jan 97: Endian Ordering] */
+  Message ("     Max mem used : %10.1f Mb\n", (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("     Num_of_words : %10d\n", NTOHUL(sih.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+#endif
+}
+
+
+void
+UpdateStemDict (char * filename, int stem_method)
+{
+  FILE *idb;
+  struct stem_dict_header sdh;
+
+  if (!(idb = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "r+b",
+			 MAGIC_STEM, MG_CONTINUE)))
+    FatalError (1, "Could not update stemmed dict");
+
+  fread ((char *) &sdh, sizeof (sdh), 1, idb);
+  NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+  sdh.indexed |= 1 << (stem_method - 1);
+  HTONUL(sdh.indexed);
+  fseek (idb, sizeof (u_long), 0);
+  fwrite ((char *) &sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+}
+
+
+
+/* Main */
+int main (int argc, char **argv)
+{
+  File *idb;    /* File to .invf.dict.blocked */
+  char *filename = "";
+  stemmed_dict *sd;   /* Stemmed dictionary */
+  int ch;
+  char path[512];
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:b:hFs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d directory] "
+		 "[-b num] [-F] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  if (stem_method < 1 || stem_method > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  /* Open required stem dict file */
+  sprintf (path, FILE_NAME_FORMAT, get_basepath (), filename, INVF_DICT_BLOCKED_SUFFIX);
+  if (!(idb = Fopen (path, "rb", MAGIC_STEM)))
+    FatalError (1, "Unable to open \"%s\"", path);
+
+  /* Read in idb header and index to blocks */
+  if (!(sd = ReadStemDictBlk (idb)))
+    FatalError (1, "Could not read stemmed dictionary");
+
+  /* Process stemmed dictionary */
+  mg_init_process ();
+  process_stem_dict (sd, stem_method);
+  save_idx (filename, stem_method);
+ 
+  /* Close stemmed dict */
+  Fclose (idb);
+
+  /* Update stemmed dict */
+  UpdateStemDict (filename, stem_method);
+
+  return 0;
+}
+
Index: /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_text_estimate.c	(revision 21112)
@@ -0,0 +1,584 @@
+/**************************************************************************
+ *
+ * mg_text_estimate.c -- Program to estimate the compressed text size
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "comp_dict.h"
+#include "hash.h"
+
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:14  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:48  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:19  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:59  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:54  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+typedef struct stats_t
+  {
+    u_long freq, occur;
+    u_long len;
+  }
+stats_t;
+
+static u_long huff_count = 0;
+static u_long escape_count = 0;
+
+
+char mode = 'H';
+
+/* [RJM 07/97: 4G limit] */
+static double ReadInWordsStandard (char *name, double *num_bytes);
+static double ReadInWordsSpecial (char *name, double *num_bytes,
+				  double *num_extra_bytes,
+				  u_long *aux_compressed);
+
+
+
+
+int main (int argc, char **argv)
+{
+  char *stats_dict = NULL, *comp_dict = NULL;
+  ProgTime StartTime;
+  double num_bytes, num_extra_bytes = 0; /* [RJM 07/97: 4G limit] */
+  u_long aux_compressed;
+  int ch;
+  double bytes;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "BMDYHh")) != -1)
+    switch (ch)
+      {
+      case 'B':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = ch;
+	break;
+      case 'M':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = 'Y';
+	break;
+      case 'D':
+      case 'Y':
+      case 'H':
+	mode = ch;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    stats_dict = argv[optind++];
+  else
+    stats_dict = NULL;
+  if (optind < argc)
+    comp_dict = argv[optind++];
+  else
+    {
+      fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+	       argv[0]);
+      exit (1);
+    }
+
+
+
+  GetTime (&StartTime);
+
+  if (LoadCompressionDictionary (comp_dict) == COMPERROR)
+    exit (1);
+
+  if (mode == 'H')
+    bytes = ReadInWordsStandard (stats_dict, &num_bytes);
+  else
+    bytes = ReadInWordsSpecial (stats_dict, &num_bytes, &num_extra_bytes,
+				&aux_compressed);
+
+
+  Message ("Compressed Text %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   bytes / 1024.0 / 1024, bytes, bytes * 100 / num_bytes);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / num_bytes);
+  if (mode == 'H')
+    Message ("Huffman info for chars in dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	     Chars_disk / 1024.0 / 1024, Chars_disk * 1.0,
+	     (Chars_disk * 100.0) / num_bytes);
+  else
+    {
+      Message ("Aux dictionary (Uncompressed) %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       num_extra_bytes / 1024.0 / 1024, num_extra_bytes * 1.0,
+	       (num_extra_bytes * 100.0) / num_bytes);
+      Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	       (aux_compressed * 100.0) / num_bytes);
+    }
+  Message ("Words and non-words in the text %u", huff_count + escape_count);
+  Message ("Words and non-words coded by escapes %u ( %0.3f %%)", escape_count,
+	   escape_count * 100.0 / huff_count);
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+
+static double 
+ReadInWordsStandard (char *name, double *num_bytes)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long which;
+  double bytes, bits = 0;
+
+  f = open_named_file (name, "rb", MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      frags_stats_header fsh;
+      int j;
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, freq, res, occur_num;
+	  u_char Word[16];
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(freq);
+	  NTOHSI(occur_num);
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      escape_count += freq;
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	      if (cdh.dict_type == MG_SEED_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * freq;
+	      huff_count += freq;
+	    }
+	}
+    }
+  fclose (f);
+
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
+
+
+static int 
+stats_comp (const void *a, const void *b)
+{
+  return ((stats_t *) a)->occur - ((stats_t *) b)->occur;
+}
+
+
+
+
+
+static double 
+ReadInWordsSpecial (char *name, double *num_bytes,
+		    double *num_extra_bytes,
+		    u_long *aux_compressed)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long magic, which;
+  double bytes, bits = 0;
+  double size[2];
+
+  if (!(f = fopen (name, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open file \"%s\"\n", name);
+
+  if (fread (&magic, sizeof (magic), 1, f) != 1)
+    FatalError (1, "Unable to read magic number");
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (magic != MAGIC_STATS_DICT)
+    FatalError (1, "Bad magic number");
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      stats_t *stats;
+      frags_stats_header fsh;
+      int j, num = 0;
+      long lens[16], chars[256];
+
+      bzero ((char *) lens, sizeof (lens));
+      bzero ((char *) chars, sizeof (chars));
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      stats = Xmalloc (fsh.num_frags * sizeof (*stats));
+      if (!stats)
+	FatalError (1, "Unable to allocate memory for stats");
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, res;
+	  u_char Word[16];
+	  fread (&stats[num].freq, sizeof (stats[num].freq), 1, f);
+	  fread (&stats[num].occur, sizeof (stats[num].occur), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(stats[num].freq);
+	  NTOHSI(stats[num].occur);
+
+	  stats[num].len = len + 1;
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  assert (stats[num].freq > 0);
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if ((cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		   cdh.dict_type == MG_SEED_DICTIONARY) &&
+		  ht[which])
+		{
+		  int k;
+		  res = ht[which]->hd->num_codes - 1;
+		  bits += ht[which]->hd->clens[res] * stats[num].freq;
+		  lens[Word[0]]++;
+		  for (k = 0; k < (int) Word[0]; k++)
+		    chars[Word[k + 1]]++;
+		}
+	      num++;
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * stats[num].freq;
+	    }
+	}
+
+      {
+	int j, num_buckets = 0;
+	int start[10], end[10];
+	long flens[16], fchars[256];
+
+	for (j = 0; j < 256; j++)
+	  if (!chars[j] && PESINAWORD (j) == which)
+	    fchars[j] = 1;
+	  else
+	    fchars[j] = chars[j];
+	for (j = 0; j < 16; j++)
+	  if (!lens[j])
+	    flens[j] = 1;
+	  else
+	    flens[j] = lens[j];
+
+	size[which] = Calculate_Huffman_Size (16, flens, lens) +
+	  Calculate_Huffman_Size (256, fchars, chars);
+
+
+	qsort (stats, num, sizeof (*stats), stats_comp);
+
+
+	if (mode == 'Y')
+	  {
+	    int k;
+	    num_buckets = 1;
+	    start[0] = 0;
+	    end[0] = cdh.num_words[which] - 1;
+	    Message ("There are %d novel %swords", num, which ? "" : "non-");
+	    while (end[num_buckets - 1] < num)
+	      {
+		start[num_buckets] = end[num_buckets - 1] + 1;
+		end[num_buckets] = start[num_buckets] +
+		  (end[num_buckets - 1] - start[num_buckets - 1]) * 2;
+		num_buckets++;
+	      }
+	    for (k = 0; k < num_buckets; k++)
+	      Message ("Bucket %d   %d <= x <= %d\n", k + 1, start[k], end[k]);
+	  }
+
+	for (j = 0; j < num; j++)
+	  {
+	    (*num_extra_bytes) += stats[j].len;
+	    switch (mode)
+	      {
+	      case 'D':
+		bits += BIO_Delta_Length (j + 1) * stats[j].freq;
+		break;
+	      case 'B':
+		bits += BIO_Binary_Length (j + 1, num) * stats[j].freq;
+		break;
+	      case 'Y':
+		{
+		  int k = 0;
+		  while (j > end[k])
+		    k++;
+		  if (!(j - start[k] + 1 >= 1 &&
+			j - start[k] + 1 <= end[k] - start[k] + 1))
+		    {
+		      Message ("k = %d, j = %d", k, j);
+		      assert (j - start[k] + 1 >= 1 &&
+			      j - start[k] + 1 <= end[k] - start[k] + 1);
+		    }
+		  else
+		    {
+		      bits += (BIO_Gamma_Length (k + 1) +
+			       BIO_Binary_Length (j - start[k] + 1,
+						  end[k] - start[k] + 1)) *
+			stats[j].freq;
+		    }
+		}
+	      }
+	  }
+	if (mode == 'B' && num)
+	  bits += BIO_Delta_Length (num) * csh.num_docs;
+      }
+      Xfree (stats);
+    }
+  fclose (f);
+
+  *aux_compressed = (size[0] + size[1]) / 8;
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_text_merge.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_text_merge.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_text_merge.1	(revision 21112)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_text_merge.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_text_merge.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_text_merge.c	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * mg_text_merge.c --- merge  *.text, *.text.idx files
+ *                     part of the mgmerge utility
+ * Copyright (C) 1995 Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edited:  November 11 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "text.h"
+
+
+/**** GLOBALS ****/
+FILE *text[3], *idx[3];
+
+typedef char FileName[256];
+FileName old_name, new_name, merge_name;
+
+long magicsize;			/* == where the header in a file begins */
+
+compressed_text_header cth[3];
+
+/*=======================================================================
+ * init_merge_text(): open files, set up global variables, etc
+ *=======================================================================*/
+int 
+init_merge_text ()
+{
+
+  /* open .text files */
+  text[OLD] = open_file (old_name, TEXT_SUFFIX, "r+b",
+			 MAGIC_TEXT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (text[OLD]);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, text[OLD]);
+
+  text[NEW] = open_file (new_name, TEXT_SUFFIX, "rb+",
+			 MAGIC_TEXT, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, text[NEW]);
+
+  /* open .text.idx files */
+  idx[OLD] = open_file (old_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, idx[OLD]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[OLD].num_of_docs);
+  NTOHD(cth[OLD].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[OLD].num_of_words);
+  NTOHUL(cth[OLD].length_of_longest_doc);
+  NTOHD(cth[OLD].ratio);
+
+  idx[NEW] = open_file (new_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, idx[NEW]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[NEW].num_of_docs);
+  NTOHD(cth[NEW].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[NEW].num_of_words);
+  NTOHUL(cth[NEW].length_of_longest_doc);
+  NTOHD(cth[NEW].ratio);
+
+  idx[MERGE] = create_file (merge_name, TEXT_IDX_SUFFIX, "wb",
+			    MAGIC_TEXI, MG_ABORT);
+  return OK;
+}
+
+
+/*=======================================================================
+ * process_merge_text(): merge the files
+ *=======================================================================*/
+int 
+process_merge_text (void)
+{
+  int i;
+  u_long data, offset;
+  byte c;
+
+  /* update and write merged header to .text and .text.idx files */
+  /* they have the exact same header */
+  cth[MERGE].num_of_docs = cth[OLD].num_of_docs
+    + cth[NEW].num_of_docs;
+  cth[MERGE].num_of_bytes = cth[OLD].num_of_bytes
+    + cth[NEW].num_of_bytes;
+  cth[MERGE].num_of_words = cth[OLD].num_of_words;
+  cth[MERGE].length_of_longest_doc =
+    (cth[OLD].length_of_longest_doc > cth[NEW].length_of_longest_doc
+     ? cth[OLD].length_of_longest_doc
+     : cth[NEW].length_of_longest_doc);
+  cth[MERGE].ratio = ((cth[OLD].num_of_bytes * cth[OLD].ratio) +
+		      (cth[NEW].num_of_bytes * cth[NEW].ratio))
+    / cth[MERGE].num_of_bytes;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth[MERGE].num_of_docs);
+  HTOND(cth[MERGE].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth[MERGE].num_of_words);
+  HTONUL(cth[MERGE].length_of_longest_doc);
+  HTOND(cth[MERGE].ratio);
+
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, idx[MERGE]);
+  fseek (text[OLD], magicsize, 0);
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, text[OLD]);
+
+  /*
+   *  Update *.text.idx:  need to know where each new doc starts
+   *  in the appended .text file
+   */
+  for (i = 0; i < cth[OLD].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[OLD]);
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+
+  /* offset is the amount to add to each entry from idx[NEW] */
+  fread (&offset, sizeof (u_long), 1, idx[OLD]);
+  NTOHUL(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  offset -= (4 + sizeof (cth[OLD]));	/* 4 for the magic number */
+
+  for (i = 0; i < cth[NEW].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[NEW]);
+      NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      data += offset;
+      HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+  /* write last u_long in idx[MERGE] (= length of file) */
+  fread (&data, sizeof (u_long), 1, idx[NEW]);
+  NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  data += offset;
+  HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+
+/******* update .text *******/
+  /* simply cat's the files together, except for the headers 
+   * and magic numbers, of course
+   */
+  fseek (text[OLD], 0L, 2);
+  while (!feof (text[NEW]))
+    {
+      fread (&c, sizeof (c), 1, text[NEW]);
+      if (!feof (text[NEW]))
+	fwrite (&c, sizeof (c), 1, text[OLD]);
+    }
+
+  return OK;
+}
+
+
+
+/*=======================================================================
+ * done_merge_text(): close files.
+ *=======================================================================*/
+int 
+done_merge_text (void)
+{
+  fclose (idx[MERGE]);
+  fclose (text[OLD]);
+  fclose (idx[OLD]);
+  fclose (text[NEW]);
+  fclose (idx[NEW]);
+
+  fprintf (stderr, "mg_text_merge: %ld documents added to %s\n",
+	   cth[NEW].num_of_docs, merge_name);
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s -f collection_name\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int main (int argc, char *argv[])
+{
+  char *progname;
+  ProgTime start;
+  int ch;			/* for command line processing */
+
+  progname = argv[0];
+  msg_prefix = argv[0];
+  merge_name[0] = '\0';
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (merge_name, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (merge_name[0] == '\0')
+    usage (progname);
+  strcpy (old_name, merge_name);
+  strcat (old_name, ".old");
+  strcpy (new_name, merge_name);
+  strcat (new_name, ".new");
+
+  GetTime (&start);
+  init_merge_text ();
+  process_merge_text ();
+  done_merge_text ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mg_weights_build.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_weights_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_weights_build.1	(revision 21112)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mg_weights_build.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mg_weights_build.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mg_weights_build.c	(revision 21112)
@@ -0,0 +1,514 @@
+/**************************************************************************
+ *
+ * mg_weights_build.c -- Program to build the document weights file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.2  2004/05/24 21:12:18  kjdon
+   changed a message
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:16  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:22  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:05  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:00  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:55  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+unsigned char bits = 8;
+static char *file_name = "";
+static char *text_file_name = "";
+static unsigned long NumPara = 0;
+static unsigned long StaticNumOfDocs = 0;
+
+unsigned long get_NumPara (void);
+unsigned long get_StaticNumOfDocs (void);
+void GenerateWeights (void);
+void Make_weight_approx (void);
+void Make_text_idx_wgt (void);
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:t:d:b:sh")) != -1) /* [RJM 10/98 - Text Filename] */
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	if (strlen(text_file_name) == 0) text_file_name = optarg;
+	break;
+      /* [RJM 10/98 - Text Filename] */
+      case 't':		/* text input file */
+	text_file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	bits = atoi (optarg);
+	if (bits > 32)
+	  {
+	    fprintf (stderr, "b may only take values 0-32\n");
+	    exit (1);
+	  }
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b bits] [-s] [-h]\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  GenerateWeights ();
+
+  Make_weight_approx ();
+
+  Make_text_idx_wgt ();
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+
+  return 0;
+}
+
+
+
+
+unsigned long 
+get_NumPara (void)
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (NumPara)
+    return (NumPara);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.num_of_docs, NumPara);  /* [RPAP - Jan 97: Endian Ordering] */
+  return NumPara;
+}
+
+
+
+unsigned long 
+get_StaticNumOfDocs (void)
+/* the static number of documents is the N parameter used to
+ * decode document gaps in the inverted file encoded using
+ * the Bblock method.
+ */
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (StaticNumOfDocs)
+    return (StaticNumOfDocs);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.static_num_of_docs, StaticNumOfDocs);  /* [RPAP - Jan 97: Endian Ordering] */
+  return StaticNumOfDocs;
+}
+
+
+
+void GenerateWeights (void) {
+  FILE *dict, *invf, *f, *idx;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;
+  double logN;
+  float *DocWeights;
+
+  /* make sure the globals NumPara and StaticNumOfDocs are loaded */
+  get_NumPara ();
+  get_StaticNumOfDocs ();
+
+  /* check to see if the weights file has already been built */
+  if ((f = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		      MG_CONTINUE)) != NULL) {
+      fclose (f);
+      return;
+  }
+  Message ("The file \"%s.weight\" does not exist.", file_name);
+  Message ("Building the weight data from the file \"%s.invf\".", file_name);
+
+  logN = log ((double) NumPara);
+
+  /* allocate memory for the weights */
+  if (!(DocWeights = Xmalloc (sizeof (float) * (NumPara + 1))))
+      FatalError (1, "No memory for doc weights");
+  bzero ((char *) DocWeights, sizeof (float) * (NumPara + 1));
+
+  /* open the .invf.dict file and read in its header */
+  dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+		    MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  /* open .invf.idx */
+  idx = open_file (file_name, INVF_IDX_SUFFIX, "rb", MAGIC_INVI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* open .invf and read in its header */
+  invf = open_file (file_name, INVF_SUFFIX, "rb", MAGIC_INVF,
+		    MG_ABORT);
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i <= 15; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  /* make sure the inverted file does not contain skips and is not level 1 */
+  if (ifh.skip_mode != 0)
+    FatalError (0, "Can\'t make weights file from a skipped inverted file.");
+  if (ifh.InvfLevel == 1)
+    FatalError (0, "Can\'t make weights file from level 1 inverted file.");
+
+  DECODE_START (invf)
+
+    /* process each word adding its contributions to the document weights */
+    for (i = 0; i < ifh.no_of_words; i++)
+    {
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+      unsigned long fcnt, wcnt, blk, CurrDoc, p, j;
+      float idf;
+
+      /* give a little feedback every 4096 words */
+      if ((i & 0xfff) == 0)
+	fprintf (stderr, ".");
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      dummy2[dummy1] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      idf = logN - log ((double) fcnt);
+      blk = BIO_Bblock_Init (StaticNumOfDocs, p);
+      CurrDoc = 0;
+
+      /* check the inverted file index entry for this word */
+      {
+	unsigned long loc;
+	fread ((char *) &loc, sizeof (loc), 1, idx);
+	NTOHUL(loc);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (ftell (invf) != loc)
+	  {
+	    FatalError (1, "Word %d  %d != %d", i, ftell (invf), loc);
+	  }
+      }
+
+      for (j = 0; j < p; j++)
+	{
+	  unsigned long x, tf;
+	  BBLOCK_DECODE (x, blk);
+	  CurrDoc += x;
+
+	  if (CurrDoc > idh.num_of_docs) {
+	    FatalError (1, "CurrDoc = %d, number of documents = %d", 
+			CurrDoc, idh.num_of_docs);
+	  }	  
+
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      double weight;
+	      GAMMA_DECODE (tf);
+	      weight = tf * idf;
+	      DocWeights[CurrDoc - 1] += weight * weight;
+	    }
+	}
+      
+      while (__btg)
+	DECODE_BIT;
+    }
+
+  DECODE_DONE
+
+  fclose (dict);
+  fclose (invf);
+  fprintf (stderr, "\n");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < NumPara; i++)
+    HTONF(DocWeights[i]);
+
+  f = create_file (file_name, WEIGHTS_SUFFIX, "wb", MAGIC_WGHT,
+		   MG_ABORT);
+
+  fwrite ((char *) DocWeights, sizeof (float), NumPara, f);
+  fclose (f);
+  Xfree (DocWeights);
+}
+
+
+
+
+
+
+
+
+
+
+
+void 
+Make_weight_approx (void)
+{
+  int i, pos, max;
+  unsigned long buf;
+  double U, L, B;
+  FILE *approx, *exact;
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* calculate U and L */
+  L = 1e300;
+  U = 0;
+  for (i = 0; i < NumPara; i++)
+    {
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt > U)
+	U = wgt;
+      if (wgt > 0 && wgt < L)
+	L = wgt;
+
+    }
+  fseek (exact, sizeof (u_long), SEEK_SET);
+
+  B = pow (U / L, pow (2.0, -(double) bits));
+
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+
+
+
+  approx = create_file (file_name, APPROX_WEIGHTS_SUFFIX, "wb",
+			MAGIC_WGHT_APPROX, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fwrite ((char *) &bits, sizeof (bits), 1, approx);
+  HTOND(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  HTOND(B);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &L, sizeof (L), 1, approx);
+  fwrite ((char *) &B, sizeof (B), 1, approx);
+  NTOHD(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHD(B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  max = bits == 32 ? 0xffffffff : (1 << bits) - 1;
+  for (buf = pos = i = 0; i < NumPara; i++)
+    {
+      unsigned long fx;
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt == 0)
+	{
+	  wgt = L;
+#ifndef QUIET
+	  Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+	}
+      fx = (int) floor (log (wgt / L) / log (B));
+
+      if (fx > max)
+	fx = max;
+
+      buf |= (fx << pos);
+      pos += bits;
+
+      if (pos >= MAXBITS)
+	{
+	  HTONUL(buf);
+	  fwrite ((char *) &buf, sizeof (buf), 1, approx);
+	  buf = fx >> (bits - (pos - MAXBITS));
+	  pos = pos - MAXBITS;
+	}
+    }
+  if (pos > 0)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(buf);
+      fwrite ((char *) &buf, sizeof (buf), 1, approx);
+    }
+
+  fclose (approx);
+  fclose (exact);
+}
+
+
+
+
+
+void 
+Make_text_idx_wgt (void)
+{
+  compressed_text_header cth;
+  int i;
+  FILE *idx_wgt, *idx, *para, *exact;
+
+  idx_wgt = create_file (file_name, TEXT_IDX_WGT_SUFFIX, "wb", MAGIC_TEXI_WGT,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* [RJM 10/98 - Text Filename] */
+  idx = open_file (text_file_name, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (fread (&cth, sizeof (cth), 1, idx) != 1)
+    FatalError (1, "Unable to read header of index file");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  get_NumPara ();
+  if (cth.num_of_docs != NumPara)
+    {
+      Message ("The number of documents %d does not equal "
+	       "the number of paragraphs %d.", cth.num_of_docs, NumPara);
+      Message ("Using the \"%s.invf.paragraph\" file\n", file_name);
+      para = open_file (file_name, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH,
+			MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+    }
+  else
+    para = NULL;
+
+  {
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+    data;
+    for (i = 0; i < cth.num_of_docs; i++)
+      {
+	int count;
+	fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+	if (para && i < cth.num_of_docs)
+	  {
+	    /* [RPAP - Jan 97: Endian Ordering] */
+	    fread ((char *) &count, sizeof (count), 1, para);
+	    NTOHSI(count);
+	  }
+	else
+	  count = 1;
+	while (count--)
+	  {
+	    fread ((char *) &data.Weight, sizeof (float), 1, exact);
+	    NTOHF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    data.Weight = sqrt (data.Weight);
+	    HTONF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    fwrite ((char *) &data, sizeof (data), 1, idx_wgt);
+	  }
+      }
+    /* Write out the extra entry for the idx file */
+    fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+    data.Weight = 0;
+    fwrite((char*)&data, sizeof(data), 1, idx_wgt);
+  }
+
+  fclose (idx_wgt);
+  fclose (idx);
+  fclose (exact);
+  if (para)
+    fclose (para);
+}
Index: /main/tags/3.01/indexers/mg/src/text/mgdictlist.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgdictlist.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgdictlist.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mgdictlist.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgdictlist.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgdictlist.c	(revision 21112)
@@ -0,0 +1,408 @@
+/**************************************************************************
+ *
+ * mgdictlist.c -- Program to list a dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "text.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:17  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:24  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:07  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:01  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:56  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int quick = 0;
+int no_of_words[2];
+u_long maxcodelen[2];
+
+char *dictname = "";
+
+
+
+
+void 
+DumpStemDict (FILE * f)
+{
+  struct invf_dict_header idh;
+  int i;
+  u_char prev[MAXSTEMLEN + 1];
+
+  fread (&idh, sizeof (idh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (quick)
+    printf ("%ld\n", idh.dict_size);
+  else
+    {
+      printf ("# lookback           = %lu\n", idh.lookback);
+      printf ("# dict size          = %lu\n", idh.dict_size);
+      printf ("# total bytes        = %lu\n", idh.total_bytes);
+      printf ("# index string bytes = %lu\n", idh.index_string_bytes);
+      printf ("# input bytes        = %.0f\n", idh.input_bytes); /* [RJM 07/97: 4G limit] */
+      printf ("# num of docs        = %lu\n", idh.num_of_docs);
+      printf ("# static num of docs = %lu\n", idh.static_num_of_docs);
+      printf ("# num of words       = %lu\n", idh.num_of_words);
+      printf ("#\n");
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (f);
+      suff = getc (f);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, f);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, f);
+      fread (&wcnt, sizeof (wcnt), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (!quick)
+	{
+	  printf ("%d: %8ld ", i, wcnt);
+	  printf ("/ %5ld ", fcnt);
+	  printf ("%2d %2ld\t\"", *prev, copy);
+	}
+      printf ("%s", word2str (prev));
+      if (quick)
+	printf (" %ld %ld\n", wcnt, fcnt);
+      else
+	{
+	  putchar ('"');
+	  putchar ('\n');
+	}
+    }
+}
+
+
+
+
+void 
+ReadInWords (FILE * f)
+{
+  comp_frags_header cfh;
+  u_long *codes;
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+
+  if (Read_cfh (f, &cfh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read in the dictionary");
+
+  printf ("#\n");
+  printf ("#   max code len       = %u\n", cfh.hd.maxcodelen);
+  printf ("#   total bytes        = %lu\n", cfh.uncompressed_size);
+  printf ("#\n");
+
+  if (!(codes = Generate_Huffman_Codes (&cfh.hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  for (i = 0; i < cfh.hd.num_codes; i++)
+    {
+      register int val, copy, j, k;
+      char code[33];
+      val = fgetc (f);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (prev + copy + 1, sizeof (u_char), val, f);
+      *prev = val + copy;
+
+      for (k = 0, j = cfh.hd.clens[i] - 1; j >= 0; j--, k++)
+	code[k] = '0' + ((codes[i] >> j) & 1);
+      code[k] = '\0';
+
+      printf ("%d: %2d : %*s : \"%s\"\n", i, cfh.hd.clens[i],
+	      cfh.hd.maxcodelen, code, word2str (prev));
+    }
+  Xfree (codes);
+  Xfree (cfh.hd.clens);
+}
+
+
+void 
+ReadCharHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : \"%s\"\n", hd.clens[i],
+		hd.maxcodelen, code, char2str (i));
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+void 
+ReadLenHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : %d\n", hd.clens[i],
+		hd.maxcodelen, code, i);
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+
+
+
+void 
+DumpTextDict (FILE * f)
+{
+  struct compression_dict_header cdh;
+  int which;
+
+  if (Read_cdh (f, &cdh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read dictionary header");
+  switch (cdh.dict_type)
+    {
+    case MG_COMPLETE_DICTIONARY:
+      printf ("# COMPLETE DICTIONARY\n");
+      break;
+    case MG_PARTIAL_DICTIONARY:
+      printf ("# PARTIAL DICTIONARY\n");
+      break;
+    case MG_SEED_DICTIONARY:
+      printf ("# SEED DICTIONARY\n");
+      break;
+    }
+  printf ("# num words          = %lu\n", cdh.num_words[1]);
+  printf ("# num word chars     = %lu\n", cdh.num_word_chars[1]);
+  printf ("# num non-words      = %lu\n", cdh.num_words[0]);
+  printf ("# num non-word chars = %lu\n", cdh.num_word_chars[0]);
+  printf ("# lookback           = %lu\n", cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  ReadInWords (f);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      }
+}
+
+
+
+
+void 
+DumpStatsDict (FILE * f)
+{
+  int i;
+  compression_stats_header csh;
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  for (i = 0; i < 2; i++)
+    {
+      int j;
+      frags_stats_header fsh;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      if (!quick)
+	printf ("#\n# num %9s      = %lu\n#\n", i ? "words" : "non-words",
+		fsh.num_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  u_char Word[16];
+	  u_long freq, occur_num;
+
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHUL(freq);
+	  NTOHUL(occur_num);
+
+	  Word[0] = fgetc (f);
+	  fread (Word + 1, Word[0], 1, f);
+	  printf ("%d: %7ld : %7ld : \"%s\"\n", j, freq,
+		  occur_num, word2str (Word));
+	}
+    }
+}
+
+
+int main (int argc, char **argv)
+{
+  FILE *fp;
+  unsigned long magic = 0;
+
+  if (argc < 2)
+    FatalError (1, "A file name must be specified");
+  dictname = argv[1];
+  if (strcmp (dictname, "-q") == 0)
+    {
+      quick = 1;
+      if (argc < 3)
+	FatalError (1, "A file name must be specified");
+      dictname = argv[2];
+    }
+  if (!(fp = fopen (dictname, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\"", dictname);
+
+  fread (&magic, sizeof (magic), 1, fp);
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  switch (magic)
+    {
+    case MAGIC_STEM_BUILD:
+      if (!quick)
+	printf ("# Contents of STEM file \"%s\"\n#\n", dictname);
+      DumpStemDict (fp);
+      break;
+    case MAGIC_DICT:
+      if (!quick)
+	printf ("# Contents of DICT file \"%s\"\n#\n", dictname);
+      DumpTextDict (fp);
+      break;
+    case MAGIC_STATS_DICT:
+      if (!quick)
+	printf ("# Contents of STATS file \"%s\"\n#\n", dictname);
+      DumpStatsDict (fp);
+      break;
+    default:
+      FatalError (1, "Bad magic number. \"%s\" cannot be dumped", dictname);
+    }
+  fclose (fp);
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mgquery.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgquery.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgquery.1	(revision 21112)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /main/tags/3.01/indexers/mg/src/text/mgquery.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgquery.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgquery.c	(revision 21112)
@@ -0,0 +1,1502 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  2004/06/22 04:17:51  kjdon
+   fixed a couple of compiler warnings
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+ 
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+
+
+#include <stdarg.h>
+#include <signal.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F) 
+    {
+      if ((int) F != -1)
+	{
+	  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	    Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		       (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		       F->Current.NumReads);
+	  NumBytes += F->Current.NumBytes;
+	  NumSeeks += F->Current.NumSeeks;
+	  NumReads += F->Current.NumReads;
+	}
+      else
+	{
+	  Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		     (double) NumBytes / 1024, NumSeeks, NumReads);
+	  NumSeeks = NumReads = NumBytes = 0;
+	}
+    }
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+      if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+	{
+	  char *s;
+	  s = re_comp (post_proc);
+	  if (!s)
+	    {
+	      Xfree (post_proc);
+	      post_proc = NULL;
+	    }
+	}
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  if (verbatim) {
+    return (strstr (UDoc, post_proc) != NULL);
+  }
+  return re_exec ((char *) UDoc);
+}
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static u_long last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{
+	  /* load the compressed text */
+	  if (LoadCompressedText (qd, max_buf))
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+
+	  /* uncompress the loaded text */
+	  UDoc = GetDocText (qd, &ULen);
+	  if (UDoc == NULL)
+	    FatalError (1, "UDoc is unexpectedly NULL");
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->Weight);
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%7d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->Weight);
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->Weight);
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->num;
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      ConvertTermsToString (qd->TL, terms_str);
+      pager = Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      ConvertTermsToString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->num > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->num;
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->num,
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->num == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+  start_up_stats (qd, iqt);
+
+
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					     (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	    else
+	      BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFREG) != 0)
+	{
+	  /* The name is a directory */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    {
+      query ();
+    }
+  else
+    {
+      int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/mgstat.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgstat.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgstat.1	(revision 21112)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /main/tags/3.01/indexers/mg/src/text/mgstat.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgstat.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgstat.c	(revision 21112)
@@ -0,0 +1,406 @@
+/**************************************************************************
+ *
+ * mgstat.c -- Program to generate statistics on a text collection
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "mg.h"
+#include "words.h"
+#include "invf.h"
+#include "text.h"
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2001/09/21 12:46:42  kjm18
+   updated mg to be in line with mg_1.3f. Now uses long long for some variables
+   to enable indexing of very large collections.
+
+   Revision 1.1  1999/08/10 21:18:19  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:59  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+char *search_for_collection (char *name);
+int process_file (char *, char *, int);
+void ProcessDict (char *);
+void ProcessStem (char *);
+long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */
+
+static mg_ullong inputbytes = 0;
+static mg_ullong total = 0;
+
+
+
+
+
+int main (int argc, char **argv)
+{
+  mg_ullong sub_total;
+  int fast;
+  char *file_name = "";
+  int ch;
+  int exact = 0;
+  long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
+    switch (ch)
+      {
+      case 'E':
+	exact = 1;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    file_name = search_for_collection (argv[optind]);
+
+  ProcessDict (file_name);
+  ProcessStem (file_name);
+  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */
+
+  {
+    char Name[256];
+    char *s;
+    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
+    s = strrchr (Name, '/');
+    if (s)
+      *s = '\0';
+    printf ("\nThe collection is in \"%s\"\n", Name);
+  }
+
+
+  printf ("\n\t\tFiles required by mgquery\n");
+  process_file (file_name, TEXT_SUFFIX, exact);
+  process_file (file_name, INVF_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
+  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (indexed)
+    {
+      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
+    }
+
+  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
+  if (!fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+
+
+  process_file (NULL, "SUB TOTAL", exact);
+  sub_total = total;
+  total = 0;
+
+  printf ("\n\t\tFiles NOT required by mgquery\n");
+  if (fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+  process_file (file_name, INVF_DICT_SUFFIX, exact);
+  process_file (file_name, INVF_IDX_SUFFIX, exact);
+  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_SUFFIX, exact);
+  process_file (file_name, WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
+  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
+  process_file (NULL, "SUB TOTAL", exact);
+  total += sub_total;
+  printf ("\n");
+  process_file (NULL, "TOTAL", exact);
+  return 0;
+
+}
+
+
+
+char *
+search_for_collection (char *name)
+{
+  char *dir = get_basepath ();
+  static char buffer[512];
+  struct stat stat_buf;
+
+  /* Look in the current directory first */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  set_basepath (".");
+	  return buffer;
+	}
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  set_basepath (".");
+	  return name;
+	}
+    }
+  sprintf (buffer, "%s/%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  return buffer;
+	}
+    }
+  return name;
+}
+
+
+
+
+
+
+void 
+ProcessDict (char *name)
+{
+  FILE *f;
+  compression_dict_header cdh;
+  int have_cdh = 0;
+  compressed_text_header cth;
+  int have_cth = 0;
+
+  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Read_cdh (f, &cdh, NULL, NULL);
+      fclose (f);
+      have_cdh = 1;
+    }
+
+  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fread ((char *) &cth, sizeof (cth), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(cth.num_of_docs);
+      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+      NTOHUL(cth.num_of_words);
+      NTOHUL(cth.length_of_longest_doc);
+      NTOHD(cth.ratio);
+
+      fclose (f);
+      have_cth = 1;
+    }
+
+  if (have_cth)
+    {
+      inputbytes = cth.num_of_bytes;
+      printf ("Input bytes                        : %10" ULL_FS ", %8.2f Mbyte\n",
+	      cth.num_of_bytes, (double) cth.num_of_bytes / 1024 / 1024);
+      printf ("Documents                          : %10lu\n", cth.num_of_docs);
+      printf ("Words in collection [dict]         : %10lu\n", cth.num_of_words);
+      printf ("Longest doc in collection [dict]   : %10lu characters\n",
+	      cth.length_of_longest_doc);
+      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
+    }
+
+  if (have_cdh)
+    {
+      printf ("Words in dict                      : %10lu\n", cdh.num_words[1]);
+      printf ("Non-words in dict                  : %10lu\n", cdh.num_words[0]);
+      printf ("Total chars of distinct words      : %10lu\n", cdh.num_word_chars[1]);
+      printf ("Total chars of distinct non-words  : %10lu\n", cdh.num_word_chars[0]);
+    }
+
+}
+
+
+
+
+
+
+
+
+void 
+ProcessStem (char *name)
+{
+  FILE *f;
+  struct invf_dict_header idh;
+
+  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
+		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+  fread ((char *) &idh, sizeof (idh), 1, f);
+  printf ("Words in collection [stem]         : %10ld\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Words in stem                      : %10ld\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Indexed fragments                  : %10ld\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Total chars of stem words          : %10ld\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
+  fclose (f);
+}
+
+
+
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+long 
+ProcessStemBlk (char *name)
+{
+  FILE *f;
+  struct stem_dict_header sdh;
+
+  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
+		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return 0;
+  fread ((char *) &sdh, sizeof (sdh), 1, f);
+  fclose (f);
+  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+process_file (char *name, char *ext, int exact)
+{
+  static double scale = 0;
+  static char *units;
+  struct stat buf;
+  if (scale == 0)
+    {
+      /* So can get output in Mb or Kb, */
+      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
+      /* Note: if inputbytes==0, then use 1024 as default */
+      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
+      units = scale == 1024 ? "Kb" : "Mb";
+    }
+  if (name)
+    {
+      char Name[256];
+      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
+      if (!stat (Name, &buf))
+	{
+	  char fname[256];
+	  char *nam = strrchr (name, '/');
+	  nam = nam ? nam + 1 : name;
+	  sprintf (fname, "%s%s", nam, ext);
+
+	  if (inputbytes == 0.0)
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size,
+			100.0 * buf.st_size / inputbytes);
+	      else
+		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units,
+			100.0 * buf.st_size / inputbytes);
+	    }
+	  else
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size);
+	      else
+		printf ("%s%*s: %8.2f %s\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units);
+	    }
+	  total += buf.st_size;
+	  return 1;
+	}
+      else
+	return 0;
+    }
+  else
+    {
+      if (inputbytes == 0.0)
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes   %7.3f%%\n", ext,
+		    total,
+		    100.0 * total / inputbytes);
+	  else
+	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
+		    total / scale, units,
+		    100.0 * total / inputbytes);
+	}
+      else
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes\n", ext, total);
+	  else
+	    printf ("%-34s : %8.2f %s\n", ext,
+		    total / scale, units);
+	}
+    }
+  return 1;
+
+}
Index: /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.1
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.1	(revision 21112)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/mgstemidxlist.c	(revision 21112)
@@ -0,0 +1,245 @@
+/**************************************************************************
+ *
+ * mgstemidxlist.c -- Text dumper for the stem indexes
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+
+void
+read_3_in_4 (FILE * idbi)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+  int block = 0;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      return;
+    }
+
+  si->MemForStemIdx = 0;
+
+  fread (&(si->sih), sizeof (si->sih), 1, idbi);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = fgetc (idbi);
+      *buffer++ = len;
+      fread (buffer, sizeof (u_char), len, idbi);
+      buffer += len;
+      fread (&si->pos[i], sizeof (*si->pos), 1, idbi);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  printf ("# lookback     = %lu\n", si->sih.lookback);
+  printf ("# block_size   = %lu\n", si->sih.block_size);
+  printf ("# num_blocks   = %lu\n", si->sih.num_blocks);
+  printf ("# blocks_start = %lu\n", si->sih.blocks_start);
+  printf ("# index_chars  = %lu\n", si->sih.index_chars);
+  printf ("# num_of_words = %lu\n", si->sih.num_of_words);
+
+  block = 0;
+  while (block < si->sih.num_blocks)
+    {
+      unsigned long *first_word;
+      unsigned short *num_words;
+      unsigned short *index;
+      long res;
+      u_char *base;
+      int num_indexes;
+
+      /* Read in next block */
+      fseek (idbi, si->pos[block] + si->sih.blocks_start, 0);
+      fread (si->buffer, si->sih.block_size, sizeof (u_char), idbi);
+      si->active = si->pos[block];
+
+      first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (first_word + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+
+      base = (u_char *) (index + num_indexes);
+      base += index[0];
+
+      printf ("\n# block      = %d\n", block);
+      printf ("# first_word = %lu\n", *first_word);
+      printf ("# num_words  = %u\n", *num_words);
+
+      res = 0;
+      while (res < *num_words)
+	{
+	  unsigned copy, suff;
+	  u_char prev[MAXSTEMLEN + 1];
+	  unsigned int num_entries, num_cases, blk;
+	  unsigned short blk_index, offset;
+
+	  /* Read word entry */
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  *prev = copy + suff;
+	  base += suff;
+	  bcopy ((char *) base, (char *) &num_entries, sizeof (num_entries));
+	  base += sizeof (num_entries);
+	  NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+	  printf ("%u \"%s\"\n", num_entries, word2str (prev));
+
+	  /* For all the PosEntries for the word... */
+	  for (i = 0; i < num_entries; i++)
+	    {
+	      bcopy ((char *) base, (char *) &num_cases, sizeof (num_cases));
+	      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (num_cases);
+	      bcopy ((char *) base, (char *) &blk, sizeof (blk));
+	      NTOHUI(blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk);
+	      bcopy ((char *) base, (char *) &blk_index, sizeof (blk_index));
+	      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk_index);
+	      bcopy ((char *) base, (char *) &offset, sizeof (offset));
+	      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (offset);
+
+	      printf ("    -> %4u %4u %4u %4u\n", num_cases, blk, blk_index, offset);
+	    }
+	  res++;
+	}
+      block++;
+    }
+  fclose (idbi);
+}
+
+int
+main (int argc, char **argv)
+{
+  FILE *idbi;
+  char *filename = "";
+  int ch;
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:hs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d data directory] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  /* Open required files */
+  switch (stem_method)
+    {
+    case (1):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "rb", MAGIC_STEM_1,
+			MG_ABORT);
+      break;
+    case (2):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "rb", MAGIC_STEM_2,
+			MG_ABORT);
+      break;
+    case (3):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "rb", MAGIC_STEM_3,
+			MG_ABORT);
+      break;
+    default:
+      FatalError (1, "Stem method must be 1, 2 or 3\n");
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT open file");
+
+  read_3_in_4 (idbi);
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/query.docnums.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/query.docnums.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/query.docnums.c	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * query.docnums.c -- Docnums query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "filestats.h"
+#include "timing.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+
+
+static DocList *
+AddToDocList (DocList * Docs, int res)
+{
+  int j;
+  /* Search the list for the word */
+  for (j = 0; j < Docs->num; j++)
+    if (Docs->DE[j].DocNum == res)
+      break;
+
+  /* Increment the weight if the word is in the list */
+  if (j < Docs->num)
+    Docs->DE[j].Weight++;
+  else
+    {
+      /* Create a new entry in the list for the new document */
+      Docs = ResizeDocList (Docs, Docs->num + 1);
+      Docs->DE[Docs->num].DocNum = res;
+      Docs->DE[Docs->num].Weight = 1;
+      Docs->DE[Docs->num].SeekPos = 0;
+      Docs->DE[Docs->num].Len = 0;
+      Docs->DE[Docs->num].CompTextBuffer = NULL;
+      Docs->DE[Docs->num].Next = NULL;
+      Docs->num++;
+    }
+  return (Docs);
+}
+
+void 
+DocnumsQuery (query_data * qd, char *QueryLine)
+{
+  DocList *Docs = MakeDocList (0);
+  unsigned long start = 0, finish;
+  int range = 0; /* is there a range or not ? */
+
+  /* find the start of the first word */
+  while (*QueryLine)
+    {
+      char *end;
+      finish = strtol (QueryLine, &end, 10);
+      if (end != (char *) QueryLine && finish >= 1 &&
+	  finish <= qd->sd->sdh.num_of_docs)
+	{
+	  unsigned long i;
+	  if (!range)
+	    start = finish;
+	  else
+	    range = 0;
+	  for (i = start; i <= finish; i++)
+	    Docs = AddToDocList (Docs, i);
+	  while (*end == ' ' || *end == '\t' || *end == '\n')
+	    end++;
+	  if (*end == '-')
+	    {
+	      start = finish;
+	      range = 1;
+	      end++;
+	    }
+	}
+      else
+	break;
+      QueryLine = end;
+    }
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /main/tags/3.01/indexers/mg/src/text/query.ranked.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/query.ranked.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/query.ranked.c	(revision 21112)
@@ -0,0 +1,994 @@
+/**************************************************************************
+ *
+ * query.ranked.c -- Ranked query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "messages.h"
+#include "timing.h"
+#include "sptree.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "weights.h"
+#include "text_get.h"
+#include "invf_get.h"
+#include "words.h"
+#include "stemmer.h"
+#include "locallib.h"
+#include "environment.h"
+#include "term_lists.h"
+#include "local_strings.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:20  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:46  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:03  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:04  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/*************************************************************************/
+
+typedef struct HeapEntry
+  {
+    float Weight;
+    float *OrgWeight;
+    int DocNum;
+    u_long SeekPos;		/* position in the text file in bytes */
+    u_long Len;			/* length of the document in bytes */
+  }
+HeapEntry;
+
+typedef int (*HeapComp) (HeapEntry *, HeapEntry *);
+
+typedef struct Heap
+  {
+    int NumItems;
+    int MaxSize;
+    HeapComp HC;
+    HeapEntry HE[1];
+  }
+Heap;
+
+
+Heap *
+Heap_Make (int size, HeapComp hc)
+{
+  Heap *H;
+  H = Xmalloc (sizeof (Heap) + (size - 1) * sizeof (HeapEntry));
+  if (!H)
+    return NULL;
+  H->NumItems = 0;
+  H->MaxSize = size;
+  H->HC = hc;
+  return H;
+}
+
+int 
+Heap_Size (Heap * H)
+{
+  return sizeof (Heap) + H->MaxSize * sizeof (HeapEntry);
+}
+
+HeapEntry *
+Heap_GetHead (Heap * H)
+{
+  if (H && H->NumItems)
+    return &H->HE[0];
+  else
+    return NULL;
+}
+
+
+
+void 
+Heap_Heapify (Heap * H, int i)
+{
+  register int curr, child;
+  curr = i;
+  child = curr * 2;
+  while (child <= H->NumItems)
+    {
+      if (child < H->NumItems && H->HC (&H->HE[child], &H->HE[child - 1]) > 0)
+	child++;
+      if (H->HC (&H->HE[curr - 1], &H->HE[child - 1]) < 0)
+	{
+	  HeapEntry temp = H->HE[child - 1];
+	  H->HE[child - 1] = H->HE[curr - 1];
+	  H->HE[curr - 1] = temp;
+	  curr = child;
+	  child = 2 * child;
+	}
+      else
+	break;
+    }
+}
+
+
+void 
+Heap_Build (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems / 2; i > 0; i--)
+    Heap_Heapify (H, i);
+}
+
+
+void 
+Heap_Sort (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems; i > 1; i--)
+    {
+      HeapEntry temp = H->HE[0];
+      H->HE[0] = H->HE[i - 1];
+      H->HE[i - 1] = temp;
+      H->NumItems--;
+      Heap_Heapify (H, 1);
+    }
+}
+
+
+void 
+Heap_DeleteHead (Heap * H)
+{
+  H->HE[0] = H->HE[--H->NumItems];
+  Heap_Heapify (H, 1);
+}
+
+int 
+Heap_Lesser (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? -1 :
+	  (a->Weight == b->Weight ? 0 : 1));
+}
+
+
+int 
+Heap_Greater (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? 1 :
+	  (a->Weight == b->Weight ? 0 : -1));
+}
+
+int 
+Make_Exact_Root (query_data * qd, Heap * H)
+{
+  int num = 0;
+  HeapEntry *he = H->HE;
+  while (he->SeekPos == 0)
+    {
+      he->Weight = he->Weight *
+	GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+      Heap_Heapify (H, 1);
+      num++;
+    }
+  return num;
+}
+
+void 
+Heap_Dump (Heap * H, int num)
+{
+  int i, l, lines, r;
+  if (num > H->NumItems)
+    num = H->NumItems;
+  lines = (num + 3) / 4;
+  for (l = 0; l < lines; l++)
+    for (r = 0; r < 4; r++)
+      {
+	i = lines * r + l;
+	if (i < num)
+	  fprintf (stderr, "[%2d] %7.4f", i, H->HE[i].Weight);
+	fprintf (stderr, r == 3 ? "\n" : "        ");
+      }
+  fprintf (stderr, "\n");
+}
+
+/*************************************************************************/
+
+
+
+
+
+
+int 
+doc_count_comp (const void *A, const void *B)
+{
+  const TermEntry *a = A;
+  const TermEntry *b = B;
+  return (a->WE.doc_count - b->WE.doc_count);
+}
+
+/* =========================================================================
+ * Function: ParseRankedQuery
+ * Description:
+ *      Takes a string query line and extracts the terms in the query
+ *      which exist in the stemmed dictionary.
+ *      Optionally sorts the terms into ascending order by doc. number.
+ * Input:
+ *      stemmed dictionary, query line, sort-flag
+ * Output:
+ *      A list of terms.
+ * ========================================================================= */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+static TermList *
+ParseRankedQuery (stemmed_dict * sd, char *QueryLine, int Sort, int indexed,
+		  QueryTermList **query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  u_char Word[MAXSTEMLEN + 1];
+  u_char sWord[MAXSTEMLEN + 1];
+  u_char *end, *s_in;
+  int default_stem_method = 0;
+  TermList *Terms = MakeTermList(0);
+
+  s_in = (u_char *) QueryLine;
+  end = s_in + strlen ((char *) s_in) - 1;
+  *query_term_list = MakeQueryTermList(0);  /* [RPAP - Feb 97: Term Frequency] */
+
+  if (indexed)
+    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+  else
+    default_stem_method = sd->sdh.stem_method;
+
+  while (s_in <= end)
+    {
+      int j;
+      long num_entries, word_num;
+      unsigned long count, doc_count, invf_ptr, invf_len;
+      int weight_to_apply, stem_to_apply;
+      int method_using = -1;
+
+      /* 0=optional, 1=mustmatch */
+      int require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+      /* Skip over the non word separator taking note of any parameters */
+      PARSE_RANKED_NON_STEM_WORD (require_match, s_in, end); /* [RJM 07/97: Ranked Required Terms] */
+      if (s_in > end) break;
+
+      /* Get a word and stem it */
+      PARSE_STEM_WORD (Word, s_in, end);
+
+      /* Extract any parameters */
+      weight_to_apply = 1;
+      stem_to_apply = default_stem_method;
+      while (s_in <= end)
+	{
+	  int stem_param, weight_param, param_type;
+	  char param[MAXPARAMLEN + 1];
+
+	  param_type = 0;
+	  PARSE_OPT_TERM_PARAM (param, param_type, s_in, end);
+	  if (!param_type)
+	    break;
+
+	  switch (param_type)
+	    {
+	    case (WEIGHTPARAM):
+	      weight_param = atoi (param);
+	      if (errno != ERANGE && weight_param > 0)
+		weight_to_apply = weight_param;
+	      break;
+
+	    case (STEMPARAM):
+	      stem_param = atoi (param);
+	      if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		method_using = stem_to_apply = stem_param;
+	      break;
+	    }
+	}
+
+      bcopy ((char *) Word, (char *) sWord, *Word + 1);
+      stemmer (stem_to_apply, sd->sdh.stemmer_num, sWord);
+
+      if (!indexed || stem_to_apply == 0)
+	{
+	  /* Look for the word in the already identified terms */
+	  for (j = 0; j < Terms->num; j++)
+	    if (compare (Terms->TE[j].Word, Word) == 0)
+	      break;
+
+	  /* Increment the weight if the word is in the list */
+	  /* Update the require match attribute */
+	  if (j < Terms->num)
+	    {
+	      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+				    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+	      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+	      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+	    }
+	  else
+	    {
+	      /* Look for it in the stemmed dictionary */
+	      if ((word_num = FindWord (sd, sWord, &count, &doc_count,
+					&invf_ptr, &invf_len)) != -1)
+		{
+		  /* Search the list for the word */
+		  for (j = 0; j < Terms->num; j++)
+		    if (Terms->TE[j].WE.word_num == word_num)
+		      break;
+
+		  /* Increment the weight if the word is in the list */
+		  if (j < Terms->num)
+		    {
+		      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+					    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+		      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+		      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+		    }
+		  else
+		    {
+		      /* Create a new entry in the list for the new word */
+		      TermEntry te;
+
+		      te.WE.word_num = word_num;
+		      te.WE.count = count;
+		      te.WE.doc_count = doc_count;
+		      te.WE.max_doc_count = doc_count;
+		      te.WE.invf_ptr = invf_ptr;
+		      te.WE.invf_len = invf_len;
+		      te.Count = weight_to_apply;
+		      te.Word = copy_string (Word);
+		      if (!te.Word)
+			FatalError (1, "Could NOT create memory to add term");
+		      te.Stem = NULL;
+		      te.require_match = require_match;
+
+		      AddTermEntry (&Terms, &te);
+
+		      /* [RPAP - Feb 97: Term Frequency] */
+		      AddQueryTerm (query_term_list, Word, count, method_using);
+		    }
+		}
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      else
+		AddQueryTerm (query_term_list, Word, 0, method_using);
+	    }
+	}
+      else
+	{
+	  int total_count = 0;  /* [RPAP - Feb 97: Term Frequency] */
+	  TermList *tempList = MakeTermList (0);
+	  if ((num_entries = FindWords (sd, sWord, stem_to_apply, &tempList)) > 0)
+	    {
+	      int i;
+	      u_long max_doc_count = 0;
+
+	      /* get the maximum doc count */
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  if (tempList->TE[i].WE.doc_count > max_doc_count)
+		    max_doc_count = tempList->TE[i].WE.doc_count;
+		  total_count += tempList->TE[i].WE.count;  /* [RPAP - Feb 97: Term Frequency] */
+		}
+
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  /* Look for the word(s) in the already identified terms */
+		  for (j = 0; j < Terms->num; j++) 
+		    {
+		      if (compare (Terms->TE[j].Word, tempList->TE[i].Word) == 0)
+			{
+			  /* found the word */
+			  /* Modify weight */
+			  Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+						INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+			  if (Terms->TE[j].WE.max_doc_count < max_doc_count)
+			    Terms->TE[j].WE.max_doc_count = max_doc_count;
+			  break;
+			}
+		    }
+
+		  if (j == Terms->num)
+		    {
+		      /* word was not found */
+		      tempList->TE[i].WE.max_doc_count = max_doc_count;
+		      tempList->TE[i].Count = weight_to_apply;
+
+		      /* We cannot require a term to match if it is expanded */
+		      /* into multiple terms :-( */
+		      tempList->TE[i].require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+		      AddTermEntry (&Terms, &tempList->TE[i]);
+		    }
+		}
+	    }
+	  /* [RPAP - Feb 97: Term Frequency] */
+	  AddQueryTerm (query_term_list, Word, total_count, method_using);
+
+	  if (tempList != NULL) Xfree(tempList); /* [RJM 07/98: Memory Leak] */
+	} /* end indexed */
+    } /* end while */
+
+  if (Sort)
+    /* Sort the terms in ascending order by doc_count */
+    qsort (Terms->TE, Terms->num, sizeof (TermEntry), doc_count_comp);
+  return (Terms);
+}
+
+
+
+
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+
+
+
+
+/*
+ * This function is given a list of term numbers and it returns a list
+ * of document numbers based on the cosine document weighting system.
+ * This puts the entries in an array.
+ * inverted file.
+ * If MaxDocs == -1 then it means all 
+ */
+static DocList *
+CosineGet (query_data * qd, TermList * Terms, RankedQueryInfo * rqi) {
+  DocList *Docs;
+  float *AccumulatedWeights = NULL;
+  Splay_Tree *ST = NULL;
+  Splay_Tree *Para_ST = NULL;
+  Hash_Table *HT = NULL;
+  List_Table *LT = NULL;
+  Heap *H;
+  HeapEntry *he;
+  register float *fptr = NULL;
+  register Invf_Doc_Entry *ide = NULL;
+  register Invf_Doc_EntryH *ideh = NULL;
+  int BackEnd, NumExact, MaxExact, NumParas;
+  int MaxDocs = 0, MaxParas = 0;
+  int i;
+  Invf_Doc_Entry_Pool ide_pool;
+  ide_pool.pool = NULL;
+
+  qd->hops_taken = qd->num_of_ptrs = qd->num_of_accum = 0;
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      ST = CosineDecodeSplay (qd, Terms, rqi, &ide_pool);
+      if (!ST)
+	return NULL;
+      break;
+    case 'A':
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      if (!AccumulatedWeights)
+	return NULL;
+      break;
+    case 'H':
+      HT = CosineDecodeHash (qd, Terms, rqi);
+      if (!HT)
+	return NULL;
+      break;
+    case 'L':
+      LT = CosineDecodeList (qd, Terms, rqi);
+      if (!LT)
+	return NULL;
+      break;
+    }
+
+#if 0
+  if (rqi->UseSplayTree)
+    {
+
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      fptr = AccumulatedWeights;
+      ide = SP_get_first (ST);
+      for (i = 0; i < qd->sd->sdh.num_of_docs; i++)
+	{
+	  if (AccumulatedWeights[i] != 0)
+	    {
+	      if (i != ide->DocNum)
+		fprintf (stderr, "Sum mismatch for %d %f %d %f\n", i + 1,
+			 AccumulatedWeights[i], ide->DocNum + 1, ide->Sum);
+	      ide = SP_get_next (ST);
+	    }
+	}
+    }
+#endif
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      MaxParas = ST->no_of_items;
+      break;
+    case 'A':
+      {				/* count the number of non-zero document weights */
+	register int i = qd->sd->sdh.num_of_docs;
+	register float *d;
+	MaxParas = 0;
+	for (d = AccumulatedWeights; i; i--, d++)
+	  if (*d)
+	    MaxParas++;
+      }
+      break;
+    case 'H':
+      MaxParas = HT->num + HT->Suplimentary_Num;
+      break;
+    case 'L':
+      MaxParas = LT->num;
+      break;
+    }
+
+  if (rqi->MaxParasToRetrieve != -1 && MaxParas > rqi->MaxParasToRetrieve)
+    MaxParas = rqi->MaxParasToRetrieve;
+  MaxDocs = MaxParas;
+
+  /* Allocate memory for the heap */
+  Docs = MakeDocList (MaxDocs);
+  ChangeMemInUse (qd, sizeof (DocEntry) * MaxDocs);
+
+  H = Heap_Make (MaxDocs, Heap_Lesser);
+
+
+  /* Get the sums from the array divide the sums by the 
+     document weights which we retrieve from the ".idx.wgt" file and put
+     the resulting data into a heap */
+
+
+  he = H->HE;
+  H->NumItems = MaxDocs;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	ide = SP_get_first (ST);
+	for (i = 0; i < H->NumItems; i++, ide = SP_get_next (ST), he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'A':
+      {
+	fptr = AccumulatedWeights;
+	for (i = 0; i < H->NumItems; i++, fptr++, he++)
+	  {
+	    he->DocNum = i + 1;
+	    he->OrgWeight = fptr;
+	    if (*fptr)
+	      qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'H':
+      {
+	ideh = HT->Head;
+	for (i = 0; i < H->NumItems; i++, ideh = ideh->next, he++)
+	  {
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'L':
+      {
+	ide = LT->IDE;
+	for (i = 0; i < H->NumItems; i++, ide++, he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    }
+
+  he = H->HE;
+  for (i = 0; i < H->NumItems; i++, he++)
+    {
+      *he->OrgWeight /= GetLowerApproxDocWeight (qd->awd, he->DocNum - 1);
+      he->Weight = *he->OrgWeight;
+      *he->OrgWeight = 0;
+      he->SeekPos = he->Len = 0;
+    }
+
+  Heap_Build (H);
+
+  he = H->HE;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	for (i = MaxDocs; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	  {
+	    ide->Sum /= GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'A':
+      {
+	for (i = MaxDocs; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	  {
+	    if (!*fptr)
+	      continue;
+	    qd->num_of_accum++;
+	    *fptr /= GetLowerApproxDocWeight (qd->awd, i);
+	    if (*fptr <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = i + 1;
+	    he->Weight = *fptr;
+	    he->OrgWeight = fptr;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'H':
+      {
+	for (; ideh; ideh = ideh->next)
+	  {
+	    qd->num_of_accum++;
+	    ideh->IDE.Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum);
+	    if (ideh->IDE.Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->Weight = ideh->IDE.Sum;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'L':
+      {
+	for (i = MaxDocs; i < LT->num; i++, ide++)
+	  {
+	    qd->num_of_accum++;
+	    ide->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    }
+
+
+  if (rqi->Exact && qd->id->ifh.InvfLevel != 3)
+    {
+      HeapEntry *he = H->HE;
+
+      for (i = 0; i < H->NumItems; i++, he++)
+	{
+	  he->Weight = he->Weight *
+	    GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	    FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+	}
+
+      Heap_Build (H);
+
+      he = H->HE;
+
+      switch (rqi->AccumMethod)
+	{
+	case 'S':
+	  {
+	    ide = SP_get_first (ST);
+	    for (i = 0; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+
+	  /* up to here */
+
+	case 'A':
+	  {
+	    fptr = AccumulatedWeights;
+	    for (i = 0; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!*fptr)
+		  continue;
+		if (*fptr <= he->Weight)
+		  continue;
+		Weight = *fptr *
+		  GetLowerApproxDocWeight (qd->awd, i) /
+		  FetchDocStart (qd, i + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = i + 1;
+		he->OrgWeight = fptr;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		*fptr = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'H':
+	  {
+	    ideh = HT->Head;
+	    for (ideh = HT->Head; ideh; ideh = ideh->next)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ideh->IDE.Sum)
+		  continue;
+		if (ideh->IDE.Sum <= he->Weight)
+		  continue;
+		Weight = ideh->IDE.Sum *
+		  GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum) /
+		  FetchDocStart (qd, ideh->IDE.DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ideh->IDE.DocNum + 1;
+		he->OrgWeight = &ideh->IDE.Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ideh->IDE.Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'L':
+	  {
+	    ide = LT->IDE;
+	    for (i = 0; i < LT->num; i++, ide++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	}
+    }
+
+
+
+  H->HC = Heap_Greater;
+  Heap_Build (H);
+
+
+  MaxDocs = H->NumItems;
+  if (rqi->MaxDocsToRetrieve != -1 && MaxDocs > rqi->MaxDocsToRetrieve)
+    MaxDocs = rqi->MaxDocsToRetrieve;
+
+  /* Alarm */
+
+  he = H->HE;
+  BackEnd = H->NumItems - 1;
+  NumExact = 0;
+  MaxExact = H->NumItems;
+  NumParas = 0;
+  Para_ST = SP_createset (DE_comp);
+  while (H->NumItems && Docs->num < MaxDocs)
+    {
+      DocEntry DocEnt;
+      DocEntry *mem;
+
+      if (rqi->Exact)
+	{
+	  if (H->HE[0].SeekPos == 0)
+	    NumExact += Make_Exact_Root (qd, H);
+	}
+      else
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+
+      NumParas++;
+
+      DocEnt.DocNum = he->DocNum;
+      DocEnt.Weight = he->Weight;
+      DocEnt.Len = he->Len;
+      DocEnt.SeekPos = he->SeekPos;
+      DocEnt.CompTextBuffer = NULL;
+      DocEnt.Next = NULL;
+
+      Heap_DeleteHead (H);
+
+      if (!(mem = SP_member (&DocEnt, Para_ST)))
+	{
+	  Docs->DE[Docs->num] = DocEnt;
+	  SP_insert (&Docs->DE[Docs->num], Para_ST);
+	  Docs->num++;
+	}
+      else
+	{
+	  DocEnt.Next = mem->Next;
+	  Docs->DE[BackEnd] = DocEnt;
+	  mem->Next = &Docs->DE[BackEnd--];
+	}
+    }
+  SP_freeset (Para_ST);
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      Message ("%d Paragraphs were required to get %d documents",
+	       NumParas, Docs->num);
+      if (NumExact == MaxExact)
+	{
+	  Message ("The exact weights of all %d paragraphs had to be calculated", MaxExact);
+	  Message ("to obtain %d paragraphs. This may mean that the the documents", NumParas);
+	  Message ("returned do not necessarly represent an exact cosine ranking.");
+	  Message ("This problem may be corrected by increasing \'maxparas\'.");
+	}
+    }
+#if 0
+  {
+    int i;
+    FILE *f = fopen ("top.paras", "w");
+    fprintf (f, "=========================\nTop Paragraphs\n");
+    for (i = 0; i < Docs->num; i++)
+      {
+	DocEntry *d;
+	fprintf (f, "<%d(%f)>  ", Heap[i].DocNum, Heap[i].Weight);
+	for (d = Heap[i].Next; d; d = d->Next)
+	  fprintf (f, "%d(%f)  ", d->DocNum, d->Weight);
+	fprintf (f, "\n");
+      }
+    fprintf (f, "=========================\n");
+    fclose (f);
+  }
+#endif
+
+  if (AccumulatedWeights)
+    {
+      Xfree (AccumulatedWeights);
+      ChangeMemInUse (qd, -sizeof (float) * qd->sd->sdh.num_of_docs);
+    }
+  if (ST)
+    {
+      int mem = ST->mem_in_use;
+      SP_freeset (ST);
+      ChangeMemInUse (qd, -mem);
+      free_ide_pool (qd, &ide_pool);
+    }
+  if (HT)
+    HT_free (qd, HT);
+
+  if (LT)
+    LT_free (qd, LT);
+
+  if (H) Xfree (H); /* [RJM 07/98: Memory Leak] */
+
+  return (Docs);
+}
+
+
+
+
+/* if MaxDocs == -1 it means all */
+void 
+RankedQuery (query_data *qd, char *Query, RankedQueryInfo *rqi)
+{
+  DocList *dl;
+
+  if (qd->TL)
+    FreeTermList (&(qd->TL));
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  if (qd->QTL)
+    FreeQueryTermList (&(qd->QTL));
+
+  qd->TL = ParseRankedQuery (qd->sd, Query, rqi->Sort, qd->sd->sdh.indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			     &(qd->QTL));  /* [RPAP - Feb 97: Term Frequency] */
+
+  /*  PrintTermList (qd->TL, stderr); */
+
+  dl = CosineGet (qd, qd->TL, rqi);
+
+  if (!dl)
+    FatalError (1, "Out of memory\n");
+
+  FreeQueryDocs (qd);
+
+  qd->DL = dl;
+}
Index: /main/tags/3.01/indexers/mg/src/text/query_term_list.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/query_term_list.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/query_term_list.c	(revision 21112)
@@ -0,0 +1,228 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:36  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "query_term_list.h"
+#include "term_lists.h"  /* for MAXTERMSTRLEN */
+#include "messages.h"
+
+
+/* =========================================================================
+ * Function: MakeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+QueryTermList *
+MakeQueryTermList (int n)
+{
+  QueryTermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (QueryTermList) + (list_size - 1) * sizeof (QueryTermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate query term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeQueryTermList (QueryTermList ** query_list)
+{
+  QueryTermList *qtl = *query_list;
+
+  if (qtl->num > qtl->list_size)
+    {
+      if (qtl->list_size)
+	qtl->list_size *= GROWTH_FACTOR;
+      else
+	qtl->list_size = MIN_SIZE;
+    }
+  qtl = Xrealloc (qtl, sizeof (QueryTermList) + (qtl->list_size - 1) * sizeof (QueryTermEntry));
+
+  if (!qtl)
+    FatalError (1, "Unable to resize query term list");
+
+  *query_list = qtl;
+}
+
+/* =========================================================================
+ * Function: ConvertQueryTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = query list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertQueryTermsToString (QueryTermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->QTE[i].Term;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (query_term_list->QTE[i].stem_method >= 0)
+	total_len += 2;         /* '#' and the stem number */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (query_term_list->QTE[i].stem_method >= 0)
+	{
+	  *str++ = '#';
+	  *str++ = '0' + query_term_list->QTE[i].stem_method;
+	}
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetQueryTermList (QueryTermList ** qtl)
+{
+  if (*qtl)
+    FreeQueryTermList (qtl);
+  *qtl = MakeQueryTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddQueryTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTermEntry (QueryTermList ** query_term_list, QueryTermEntry * qte)
+{
+  QueryTermList *qtl = *query_term_list;
+
+  qtl->num++;
+  ResizeQueryTermList (query_term_list);
+  qtl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) qte, (char *) &(qtl->QTE[qtl->num - 1]), sizeof (QueryTermEntry));
+
+  return qtl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddQueryTerm
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method)
+{
+  /* Create a new entry in the list for the new word */
+  QueryTermEntry qte;
+
+  qte.Count = Count;
+  qte.stem_method = stem_method;
+  qte.Term = copy_string (Term);
+  if (!qte.Term)
+    FatalError (1, "Could NOT create memory to add term");
+
+  return AddQueryTermEntry (query_term_list, &qte);
+}
+
+/* =========================================================================
+ * Function: FreeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeQueryTermList (QueryTermList ** the_qtl)
+{
+  int j;
+  QueryTermList *qtl = *the_qtl;
+
+  for (j = 0; j < qtl->num; j++)
+    if (qtl->QTE[j].Term)
+      Xfree (qtl->QTE[j].Term);
+  Xfree (qtl);
+
+  *the_qtl = NULL;
+}
Index: /main/tags/3.01/indexers/mg/src/text/query_term_list.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/query_term_list.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/query_term_list.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef QUERY_TERM_LIST_H
+#define QUERY_TERM_LIST_H
+
+#include "sysfuncs.h"
+
+typedef struct QueryTermEntry
+  {
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Term;		/* The query term */
+    int stem_method;            /* The steming method to apply to the term : -1 is default */
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+/* --- prototypes --- */
+void ConvertQueryTermsToString (QueryTermList * query_term_list, char *str);
+int AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method);
+void ResetQueryTermList (QueryTermList ** tl);
+void FreeQueryTermList (QueryTermList ** the_qtl);
+QueryTermList *MakeQueryTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/read_line.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/read_line.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/read_line.c	(revision 21112)
@@ -0,0 +1,214 @@
+/**************************************************************************
+ *
+ * read_line.c -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#ifdef GNU_READLINE
+#include "readline.h"
+#include "chardefs.h"
+#include "history.h"
+#endif
+
+
+#include "globals.h"
+#include "environment.h"
+#include "read_line.h"
+
+
+
+#ifdef GNU_READLINE
+int rl_bind_key (int, int (*)());
+
+
+void 
+Init_ReadLine (void)
+{
+  /* Make TAB just insert itself rather than do a file name completion */
+  rl_bind_key (TAB, rl_insert);
+}
+#else
+
+static FILE *rl_instream = NULL;
+static FILE *rl_outstream = NULL;
+
+void 
+Init_ReadLine (void)
+{
+  rl_instream = stdin;
+  rl_outstream = stdout;
+}
+
+static char *
+readline (char *pmt)
+{
+  static char buf[MAXLINEBUFFERLEN + 1];
+  char *s;
+
+  fprintf (rl_outstream, "%s", pmt);
+  s = fgets (buf, sizeof (buf), rl_instream);
+  if (s)
+    {
+      char *s1 = strrchr (s, '\n');
+      if (s1 && *(s1 + 1) == '\0')
+	*s1 = '\0';
+    }
+  return s ? Xstrdup (s) : NULL;
+}
+#endif
+
+/* WritePrompt() */
+/* Write out a prompt if user is a TTY */
+void 
+WritePrompt (void)
+{
+  if (isatty (fileno (InFile)))
+    {
+      if (!BooleanEnv (GetEnv ("expert"), 0))
+	fprintf (stderr, "Enter a command or query (.quit to terminate, .help for assistance).\n");
+    }
+}
+
+#ifdef GNU_READLINE
+static void memory_error_and_abort ();
+
+char *
+xmalloc (bytes)
+     int bytes;
+{
+  char *temp = (char *) Xmalloc (bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+  return (temp);
+}
+
+char *
+xrealloc (pointer, bytes)
+     char *pointer;
+     int bytes;
+{
+  char *temp;
+
+  if (!pointer)
+    temp = (char *) xmalloc (bytes);
+  else
+    temp = (char *) Xrealloc (pointer, bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+
+  return (temp);
+}
+
+static void
+memory_error_and_abort ()
+{
+  fprintf (stderr, "history: Out of virtual memory!\n");
+  abort ();
+}
+#endif
+
+
+/*
+ * This routine returns a pointer to the users entered line 
+ *
+ */
+static char *
+GetLine (char *pmt)
+{
+  static char *the_line = NULL;
+  if (the_line)
+    Xfree (the_line);
+  the_line = NULL;
+  rl_instream = InFile;
+  if (isatty (fileno (InFile)))
+    {
+      fputc ('\r', stderr);
+      if (!isatty (fileno (OutFile)))
+	rl_outstream = stderr;
+      else
+	rl_outstream = OutFile;
+      the_line = readline (pmt);
+    }
+  else
+    {
+      if (isatty (fileno (OutFile)))
+	{
+	  the_line = readline (pmt);
+	  fprintf (stderr, "%s\n", the_line ? the_line : "");
+	}
+      else
+	{
+	  the_line = readline ("");
+	  if (the_line)
+	    fprintf (stderr, "%s%s\n", pmt, the_line);
+	}
+    }
+#ifdef GNU_READLINE
+  if (the_line && *the_line)
+    add_history (the_line);
+#endif
+  return (the_line);
+}
+
+
+
+
+
+
+char *
+GetMultiLine (void)
+{
+  static char *line = NULL;
+  char *s;
+  if (line)
+    Xfree (line);
+  line = NULL;
+  if (!(s = GetLine ("> ")))
+    return (NULL);
+  if (!(line = Xstrdup (s)))
+    {
+      fprintf (stderr, "Unable to allocate memory for the line\n");
+      abort ();
+    }
+
+  while ((s = strrchr (line, '\\')) && *(s + 1) == '\0')
+    {
+      char *new;
+      *strrchr (line, '\\') = '\0';
+      if (!(s = GetLine ("? ")))
+	return (NULL);
+      if (!(new = Xmalloc (strlen (line) + strlen (s) + 2)))
+	{
+	  fprintf (stderr, "Unable to allocate memory for the line\n");
+	  abort ();
+	}
+      strcpy (new, line);
+      strcat (new, "\n");
+      strcat (new, s);
+      Xfree (line);
+      line = new;
+    }
+  return (line);
+}
Index: /main/tags/3.01/indexers/mg/src/text/read_line.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/read_line.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/read_line.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * read_line.h -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef READ_LINE_H
+#define READ_LINE_H
+
+#define MAXLINEBUFFERLEN  65535
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/stem_search.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/stem_search.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/stem_search.c	(revision 21112)
@@ -0,0 +1,775 @@
+/**************************************************************************
+ *
+ * stem_search.c -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "words.h"
+#include "locallib.h"
+#include "stem_search.h"
+#include "mg_errors.h"
+#include "term_lists.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:22  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/07/02 00:18:55  rjmcnab
+   Changed so FindWords could be used in new ways.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:39  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:04  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:08  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem dict = %i\n", sd->MemForStemDict); */
+
+  return sd;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_idx *
+ReadStemIdxBlk (File * stem_idx_file)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  si->stem_idx_file = stem_idx_file;
+  si->MemForStemIdx = 0;
+
+  Fread (&si->sih, sizeof (si->sih), 1, stem_idx_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = Getc (stem_idx_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_idx_file);
+      buffer += len;
+      Fread (&si->pos[i], sizeof (*si->pos), 1, stem_idx_file);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem idx = %i\n", si->MemForStemIdx); */
+
+  return si;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* word should be appropriately stemed */
+static int
+GetIdxBlock (stemmed_idx * si, u_char * word)
+{
+  register int lo = 0, hi = si->sih.num_blocks - 1;
+  register int mid = 0, c = 0;
+
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, si->index[mid]);
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+static int 
+GetBlock (stemmed_dict * sd, u_char * Word)
+{
+  register int lo = 0, hi = sd->sdh.num_blocks - 1;
+  register int mid = 0, c = 0;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, sd->index[mid]);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+/*
+ * This function looks up a word in the stemmed dictionary, it returns -1
+ * if the word cound not be found, and 0 if it successfully finds the word.
+ * If count is non-null the ulong it is pointing to is set to the number of 
+ * occurances of the stemmed word in the collection. i.e wcnt.
+ * If doc_count is non-null the ulong it is pointing to is set to the number
+ * of documents that the word occurs in. i.e fcnt
+ * If invf_ptr is non-null the ulong it is pointing to is set to the position
+ * of the inverted file where the entry for this word start.
+ */
+int 
+FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	  unsigned long *doc_count, unsigned long *invf_ptr,
+	  unsigned long *invf_len)
+{
+  register int lo, hi, mid, c;
+  register unsigned int res;
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  block = GetBlock (sd, Word);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (sd->active != sd->pos[block])
+    {
+      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+    }
+  else
+    {
+      first_word = (unsigned long *) (sd->buffer);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+    }
+  base = (u_char *) (index + num_indexes);
+
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, base + index[mid] + 1);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+
+  res = hi * sd->sdh.lookback;
+  base += index[hi];
+
+  for (;;)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      if (res >= *num_words)
+	return (-1);
+      copy = *base++;
+      suff = *base++;
+      bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+      base += suff;
+      *prev = copy + suff;
+
+      c = casecompare (Word, prev);   /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	return (-1);
+
+      if (c == 0 && doc_count)
+	{
+	  bcopy ((char *) base, (char *) doc_count, sizeof (*doc_count));
+	  NTOHUL(*doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*doc_count);
+
+      if (c == 0 && count)
+	{
+	  bcopy ((char *) base, (char *) count, sizeof (*count));
+	  NTOHUL(*count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*count);
+
+      if (c == 0 && invf_ptr)
+	{
+	  bcopy ((char *) base, (char *) &invfp, sizeof (invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_ptr = invfp;
+	}
+      base += sizeof (*invf_ptr);
+
+      if (c == 0)
+	{
+	  /* Calculate invf_len is necessary */
+	  unsigned long next_invfp;
+	  if (!invf_len)
+	    return (*first_word + res);
+
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    {
+	      *invf_len = *last_invf_len;
+	      return (*first_word + res);
+	    }
+
+	  /* Skip over most of the next word to get to the invf_ptr */
+	  base++;
+	  suff = *base++;
+	  base += suff + sizeof (unsigned long) * 2;
+	  bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	  NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_len = next_invfp - invfp;
+	  return (*first_word + res);
+	}
+      res++;
+    }
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int 
+FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl)
+{
+  register unsigned int res;
+  unsigned int idx_res;
+  unsigned copy, suff;
+  int j, k;
+
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  int idx_block, idx_num_indexes;
+  unsigned long *idx_first_word;
+  unsigned short *idx_num_words;
+  u_char *idx_base;
+  unsigned short *idx_index;
+  u_char idx_prev[MAXSTEMLEN + 1];
+
+  unsigned int num_entries, num_cases;
+  unsigned short blk_index, offset;
+  stemmed_idx * si = NULL;
+
+  /* handle stem_method 0 seperately */
+  if (stem_method == 0) {
+    TermEntry te;
+    if ((te.WE.word_num = FindWord (sd, sWord, &te.WE.count, &te.WE.doc_count,
+				   &te.WE.invf_ptr, &te.WE.invf_len)) != -1) {
+      te.WE.max_doc_count = te.WE.doc_count;
+
+      te.Count = 1;
+      te.Word = copy_string (sWord);
+      if (!te.Word)
+	FatalError (1, "Could NOT create memory to add term");
+      te.Stem = copy_string (sWord);
+      if (!te.Stem)
+	FatalError (1, "Could NOT create memory to add term");
+      /*      te.query_mask = NULL;*/
+      
+      AddTermEntry (tl, &te);
+      return (*tl)->num;
+      
+    } else {
+      /* didn't match */
+      return 0;
+    }
+  }
+  
+  if (stem_method == 1)
+    si = sd->stem1;
+  else if (stem_method == 2)
+    si = sd->stem2;
+  else
+    si = sd->stem3;
+
+  /* Locate block */
+  idx_block = GetIdxBlock (si, sWord);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (si->active != si->pos[idx_block])
+    {
+      Fseek (si->stem_idx_file, si->pos[idx_block] + si->sih.blocks_start, 0);
+      Fread (si->buffer, si->sih.block_size, sizeof (u_char), si->stem_idx_file);
+      si->active = si->pos[idx_block];
+
+      idx_first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*idx_first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      NTOHUS(*idx_num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < idx_num_indexes; j++)
+	NTOHUS(idx_index[j]);
+    }
+  else
+    {
+      idx_first_word = (unsigned long *) (si->buffer);
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+    }
+  idx_base = (u_char *) (idx_index + idx_num_indexes);
+  
+  {
+    /* Locate 3-in-4 block */
+    register int lo, hi, mid, c;
+    lo = 0;
+    hi = idx_num_indexes - 1;
+    while (lo <= hi)
+      {
+	mid = (lo + hi) / 2;
+	c = casecompare (sWord, idx_base + idx_index[mid] + 1);
+	if (c < 0)
+	  hi = mid - 1;
+	else if (c > 0)
+	  lo = mid + 1;
+	else
+	  {
+	    hi = mid;
+	    break;
+	  }
+      }
+    if (hi < 0)
+      hi = 0;
+
+    idx_res = hi * si->sih.lookback;
+    idx_base += idx_index[hi];
+  }
+
+  /* Locate actual word entry */
+  for (;;)
+    {
+      int c;
+      if (idx_res >= *idx_num_words)
+	return (-1);
+      copy = *idx_base++;
+      suff = *idx_base++;
+      bcopy ((char *) idx_base, (char *) (idx_prev + copy + 1), suff);
+      idx_base += suff;
+      *idx_prev = copy + suff;
+
+      c = casecompare (sWord, idx_prev);
+      if (c < 0)
+	return (-1);
+     
+      bcopy ((char *) idx_base, (char *) &num_entries, sizeof (num_entries));
+      NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_entries);
+      
+      if (c > 0)
+	idx_base += num_entries * (sizeof (num_cases) + sizeof (block) + 
+				   sizeof (blk_index) + sizeof (offset));
+
+      else
+	break;
+
+      idx_res++;
+    }
+
+  for (k = 0; k < num_entries; k++)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      /* Read next stem index pos */
+      bcopy ((char *) idx_base, (char *) &num_cases, sizeof (num_cases));
+      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_cases);
+      bcopy ((char *) idx_base, (char *) &block, sizeof (block));
+      NTOHUI(block);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (block);
+      bcopy ((char *) idx_base, (char *) &blk_index, sizeof (blk_index));
+      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (blk_index);
+      bcopy ((char *) idx_base, (char *) &offset, sizeof (offset));
+      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (offset);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      if (sd->active != sd->pos[block])
+	{
+	  Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	  Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	  sd->active = sd->pos[block];
+
+	  first_word = (unsigned long *) (sd->buffer);
+	  NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (j = 0; j < num_indexes; j++)
+	    NTOHUS(index[j]);
+	}
+      else
+	{
+	  first_word = (unsigned long *) (sd->buffer);
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+	}
+      base = (u_char *) (index + num_indexes);
+      
+      res = blk_index * sd->sdh.lookback;
+      base += index[blk_index];
+      
+      for (j = 0; j < offset; j++)
+	{
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  base += sizeof (unsigned long);   /* skip doc_count */
+	  base += sizeof (unsigned long);   /* skip count */
+	  base += sizeof (unsigned long);   /* skip invf_ptr */
+	  res++;
+	}
+
+      for (j = 0; j < num_cases; j++)
+	{
+	  TermEntry te;
+	  
+	  if (res >= *num_words)
+	    return (-1);
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	      
+	  te.Word = copy_string (prev);
+	  if (!te.Word)
+	    FatalError (1, "Could NOT create memory to add term");
+	  te.Stem = copy_string (prev);
+	  if (!te.Stem)
+	    FatalError (1, "Could NOT create memory to add term");
+	  stemmer (2, sd->sdh.stemmer_num, te.Stem);
+
+	  te.Count = 1;
+	  te.WE.word_num = *first_word + res;
+	  bcopy ((char *) base, (char *) &te.WE.doc_count, sizeof (te.WE.doc_count));
+	  NTOHUL(te.WE.doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.max_doc_count = te.WE.doc_count;
+	  base += sizeof (te.WE.doc_count);
+	      
+	  bcopy ((char *) base, (char *) &te.WE.count, sizeof (te.WE.count));
+	  NTOHUL(te.WE.count);
+	  base += sizeof (te.WE.count);
+	      
+	  bcopy ((char *) base, (char *) &invfp, sizeof (te.WE.invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.invf_ptr = invfp;
+	  base += sizeof (te.WE.invf_ptr);
+		
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    te.WE.invf_len = *last_invf_len;
+	  else
+	    {
+	      unsigned long next_invfp;
+	      u_char *oldbase = base;
+		  
+	      /* Skip over most of the next word to get to the invf_ptr */
+	      base++;
+	      suff = *base++;
+	      base += suff + sizeof (unsigned long) * 2;
+	      bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	      NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	      te.WE.invf_len = next_invfp - invfp;
+	      base = oldbase;
+	    }
+	      
+	  /* Add term entry to term list */
+	  /*	  te.query_mask = NULL;*/
+	  AddTermEntry (tl, &te);
+	      
+	  if (res == *num_words - 1 && j + 1 < num_cases)
+	    {
+	      int ii;
+	      /* Read in next block */
+	      block++;
+	      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	      sd->active = sd->pos[block];
+
+	      first_word = (unsigned long *) (sd->buffer);
+	      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	      last_invf_len = (unsigned long *) (first_word + 1);
+	      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	      num_words = (unsigned short *) (last_invf_len + 1);
+	      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	      index = num_words + 1;
+	      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	      /* [RPAP - Jan 97: Endian Ordering] */
+	      for (ii = 0; ii < num_indexes; ii++)
+		NTOHUS(index[ii]);
+
+	      base = (u_char *) (index + num_indexes);
+	      base += index[0];
+	      res = 0;
+	      blk_index = 0;
+	    }
+	  else
+	    res++;
+	} /* end for num_cases */
+    } /* end for num_entries */
+  return (*tl)->num;
+}
+
+
+void 
+FreeStemDict (stemmed_dict * sd)
+{
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (sd->stem1)
+    FreeStemIdx (sd->stem1);
+  if (sd->stem2)
+    FreeStemIdx (sd->stem2);
+  if (sd->stem3)
+    FreeStemIdx (sd->stem3);
+
+  Xfree (sd->index[0]);
+  Xfree (sd->index);
+  Xfree (sd->buffer);
+  Xfree (sd->pos);
+  Xfree (sd);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void
+FreeStemIdx (stemmed_idx * si)
+{
+  Xfree (si->index[0]);
+  Xfree (si->index);
+  Xfree (si->buffer);
+  Xfree (si->pos);
+  Xfree (si);
+}
Index: /main/tags/3.01/indexers/mg/src/text/stem_search.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/stem_search.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/stem_search.h	(revision 21112)
@@ -0,0 +1,43 @@
+/**************************************************************************
+ *
+ * stem_search.h -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_STEM_SEARCH
+#define H_STEM_SEARCH
+
+stemmed_dict *ReadStemDictBlk (File * stem_file);
+
+stemmed_idx *ReadStemIdxBlk (File * stem_idx_file);
+
+int FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	      unsigned long *doc_count, unsigned long *invf_ptr,
+	      unsigned long *invf_len);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl);
+
+void FreeStemDict (stemmed_dict * sd);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void FreeStemIdx (stemmed_idx * si);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/stemmer.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/stemmer.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/stemmer.c	(revision 21112)
@@ -0,0 +1,148 @@
+/**************************************************************************
+ *
+ * stemmer.c -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+/*
+   $Log$
+   Revision 1.2  2004/06/22 04:17:06  kjdon
+   make stemmerdescription a const
+
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:23  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:42  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:05  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:10  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; i++) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (const u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); i++)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp (descript, "0") == 0) ||
+      (strcmp (descript, "english") == 0) ||
+      (strcmp (descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp (descript, "1") == 1) ||
+      (strcmp (descript, "french") == 0) ||
+      (strcmp (descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/indexers/mg/src/text/stemmer.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/stemmer.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (const u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/term_lists.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/term_lists.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/term_lists.c	(revision 21112)
@@ -0,0 +1,334 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:43  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "term_lists.h"
+#include "messages.h"
+#include "stemmer.h"
+
+TermList *query_term_list = NULL;
+
+/* =========================================================================
+ * Function: MakeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+TermList *
+MakeTermList (int n)
+{
+  TermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (TermList) + (list_size - 1) * sizeof (TermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeTermList (TermList ** term_list)
+{
+  TermList *tl = *term_list;
+
+  if (tl->num > tl->list_size)
+    {
+      if (tl->list_size)
+	tl->list_size *= GROWTH_FACTOR;
+      else
+	tl->list_size = MIN_SIZE;
+    }
+  tl = Xrealloc (tl, sizeof (TermList) + (tl->list_size - 1) * sizeof (TermEntry));
+
+  if (!tl)
+    FatalError (1, "Unable to resize term list");
+
+  *term_list = tl;
+}
+
+/* =========================================================================
+ * Function: ConvertTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = term list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertTermsToString (TermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->TE[i].Word;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetTermList (TermList ** tl)
+{
+  if (*tl)
+    FreeTermList (tl);
+  *tl = MakeTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTermEntry (TermList ** query_term_list, TermEntry * te)
+{
+  TermList *tl = *query_term_list;
+
+  tl->num++;
+  ResizeTermList (query_term_list);
+  tl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) te, (char *) &(tl->TE[tl->num - 1]), sizeof (TermEntry));
+
+  return tl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddTerm
+ * Description: Used in boolean parser - see bool_tree   [RPAP - Feb 97: Term Frequency] 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	 u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, int stemmer_num)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  int j;
+  TermList *tl = *query_term_list;
+
+  /* Look for the word in the already identified terms */
+  for (j = 0; j < tl->num; j++)
+    {
+      TermEntry *te = &(tl->TE[j]);
+      if (compare (te->Word, Word) == 0)
+	{
+	  te->Count++;
+	  return j;
+	}
+    }
+
+
+  {
+    /* Create a new entry in the list for the new word */
+    TermEntry te;
+
+    /* [RPAP - Feb 97: Term Frequency] */
+    te.WE.word_num = word_num;
+    te.WE.count = count;
+    te.WE.doc_count = doc_count;
+    te.WE.max_doc_count = doc_count;   /* [RPAP - Jan 97: Stem Index Change] */
+    te.WE.invf_ptr = invf_ptr;
+    te.WE.invf_len = invf_len;
+    te.Count = Count;
+    te.Word = copy_string (Word);
+    if (!te.Word)
+      FatalError (1, "Could NOT create memory to add term");
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    te.Stem = copy_string (Word);
+    if (!te.Stem)
+      FatalError (1, "Could NOT create memory to add term");
+    stemmer (2, stemmer_num, te.Stem);
+
+    te.require_match = 0;  /* [RJM - 07/97: Ranked Required Terms] */
+
+    return AddTermEntry (query_term_list, &te);
+  }
+
+}
+
+/* =========================================================================
+ * Function: FreeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeTermList (TermList ** the_tl)
+{
+  int j;
+  TermList *tl = *the_tl;
+
+  for (j = 0; j < tl->num; j++)
+    {
+      if (tl->TE[j].Word)
+	Xfree (tl->TE[j].Word);
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (tl->TE[j].Stem)
+	Xfree (tl->TE[j].Stem);
+    }
+  Xfree (tl);
+
+  *the_tl = NULL;
+}
+
+/* =========================================================================
+ * Function: PrintWordEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintWordEntry (WordEntry * we, FILE * file)
+{
+  fprintf (file, "we->word_num = %d\n", we->word_num);
+  fprintf (file, "we->count = %ld\n", we->count);
+  fprintf (file, "we->doc_count = %ld\n", we->doc_count);
+  fprintf (file, "we->max_doc_count = %ld\n", we->max_doc_count);
+  fprintf (file, "we->invf_ptr = %ld\n", we->invf_ptr);
+  fprintf (file, "we->invf_len = %ld\n", we->invf_len);
+}
+
+/* =========================================================================
+ * Function: PrintTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermEntry (TermEntry * te, FILE * file)
+{
+
+  fprintf (file, "Term Entry\n");
+  fprintf (file, "te->Count = %d\n", te->Count);
+  fprintf (file, "te->Word = %s\n", str255_to_string (te->Word, NULL));
+  if (te->Stem != NULL)
+    fprintf (file, "te->Stem = %s\n", str255_to_string (te->Stem, NULL));  /* [RPAP - Jan 97: Stem Index Change] */
+  fprintf (file, "te->require_match = %i\n", te->require_match);  /* [RJM 07/97: Ranked Required Terms] */
+  PrintWordEntry (&(te->WE), file);
+
+}
+
+/* =========================================================================
+ * Function: PrintTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermList (TermList * tl, FILE * file)
+{
+  int i;
+
+  fprintf (file, "Term List\n");
+  fprintf (file, "tl->list_size = %d\n", tl->list_size);
+  fprintf (file, "tl->num = %d\n", tl->num);
+
+  for (i = 0; i < tl->num; i++)
+    {
+      fprintf (file, "[%d]\n", i);
+      PrintTermEntry (&(tl->TE[i]), file);
+    }
+}
Index: /main/tags/3.01/indexers/mg/src/text/term_lists.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/term_lists.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/term_lists.h	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef TERM_LISTS_H
+#define TERM_LISTS_H
+
+#include "sysfuncs.h"
+
+#define MAXTERMSTRLEN 1023	/* maximum number of characters in term string */
+
+typedef struct WordEntry
+  {
+    int word_num;		/* Unique number for each different word */
+    u_long count;		/* Number of times the word occurs in the text */
+    u_long doc_count;		/* Number of documents that contain the word */
+    u_long max_doc_count;       /* [RPAP - Jan 97: Stem Index Change] */
+    u_long invf_ptr;		/* This is a byte position of the  
+				   inverted file entry corresponding to the word */
+    u_long invf_len;		/* This is the length of the inverted 
+				   file entry in bytes */
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Word;		/* The word. */
+    u_char *Stem;               /* [RPAP - Jan 97: Stem Index Change] - The stemmed non-cf version of Word */
+    int require_match;          /* [RJM 07/97: Ranked Required Terms] 0=optional match, 1=must match */
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry TE[1];
+  }
+TermList;
+
+#define GetNthWE(term_list, n) (&((term_list)->TE[(n)].WE))
+
+/* --- prototypes --- */
+void ConvertTermsToString (TermList * query_term_list, char *str);
+int AddTermEntry (TermList ** query_term_list, TermEntry * te);
+int AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	     u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, /* [RPAP - Feb 97: Term Frequency] */
+	     int stemmer_num);  
+void ResetTermList (TermList ** tl);
+void FreeTermList (TermList ** the_tl);
+void PrintWordEntry (WordEntry * we, FILE * file);
+void PrintTermEntry (TermEntry * te, FILE * file);
+void PrintTermList (TermList * tl, FILE * file);
+TermList *MakeTermList (int n);
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/text.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/text.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/text.h	(revision 21112)
@@ -0,0 +1,176 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_TEXT
+#define H_TEXT
+
+#include "huffman.h"
+
+
+
+
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using binary codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_BINARY 1
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+/* This method codes novel words using hybrid version of delta and a MTF
+   operation. The novel words are stored in a auxillary dictionary which
+   is built by pass two. */
+#define MG_NOVEL_HYBRID_MTF 4
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+typedef struct compression_dict_header
+  {
+    u_long dict_type;
+    u_long novel_method;
+    u_long params[TEXT_PARAMS];
+    u_long num_words[2];
+    u_long num_word_chars[2];
+    u_long lookback;
+  }
+compression_dict_header;
+
+
+typedef struct comp_frags_header
+  {
+    huff_data hd;
+    u_long uncompressed_size;
+    u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  }
+comp_frags_header;
+
+
+
+typedef struct compressed_text_header
+  {
+    u_long num_of_docs;
+    u_long num_of_words;	/* number of words in collection */
+    double num_of_bytes; /* [RJM 07/97: 4G limit] */
+    double ratio;
+    u_long length_of_longest_doc;	/* (characters) */
+    u_long dummy;    /* added to make the structure the same on different architectures */
+  }
+compressed_text_header;
+
+
+typedef struct compression_stats_header
+  {
+    u_long num_docs;
+    u_long dummy;   /* added to make the structure the same on different architectures */
+    double num_bytes; /* [RJM 07/97: 4G limit] */
+  }
+compression_stats_header;
+
+
+typedef struct frags_stats_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+frags_stats_header;
+
+
+typedef struct aux_frags_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+aux_frags_header;
+
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/text.pass1.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/text.pass1.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/text.pass1.c	(revision 21112)
@@ -0,0 +1,405 @@
+/**************************************************************************
+ *
+ * text.pass1.c -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long LongestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh =
+{0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+int 
+init_text_1 (char *FileName)
+{
+  int which;
+  if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  for (which = 1; which >= 0; which--)
+    {
+      u_char *word;
+      hash_rec *ent;
+      dict_data *dd = &DictData[which];
+
+      dd->wordnum = 0;
+      dd->words_read = 0;
+      dd->bytes_diff = 0;
+      dd->HashSize = INITIAL_HASH_SIZE;
+      dd->HashUsed = 0;
+      if (!(dd->HashTable = Xmalloc (sizeof (hash_rec) * dd->HashSize)))
+	{
+	  Message ("Unable to allocate memory for table");
+	  return (COMPERROR);
+	}
+      ChangeMem (sizeof (hash_rec) * dd->HashSize);
+      bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);
+
+      word = Pool;
+      *Pool++ = '\0';
+      PoolLeft--;
+      {
+	register u_char *wptr;
+	register int hsize = dd->HashSize;
+	register unsigned long hashval, step;
+
+	HASH (hashval, step, word, hsize);
+	wptr = (dd->HashTable + hashval)->word;
+	while (wptr)
+	  {
+	    hashval += step;
+	    if (hashval >= hsize)
+	      hashval -= hsize;
+	    wptr = (dd->HashTable + hashval)->word;
+	  }
+	ent = dd->HashTable + hashval;
+      }
+      ent->wcnt = 1;
+      ent->word = word;
+      dd->HashUsed = 1;
+    }
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_text_1 (u_char * s_in, int l_in)
+{
+  int which;
+  u_char *end = s_in + l_in - 1;
+  if (l_in > LongestDoc)
+    LongestDoc = l_in;
+
+  csh.num_docs++;
+  csh.num_bytes += l_in;
+
+  which = inaword (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      dict_data *dd = &DictData[which];
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+      dd->wordnum++;
+      inputbytes += *Word;
+      dd->words_read++;
+
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = dd->HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = dd->HashTable + hashval;
+	    if (!ent->word)
+	      {
+		int len = *Word + 1;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->occurance_num = occurance_num++;
+		ent->wcnt = 1;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		dd->HashUsed++;
+		dd->bytes_diff += Word[0];
+		break;
+	      }
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		break;
+	      }
+	  }
+      }
+
+  
+      if (dd->HashUsed >= dd->HashSize >> 1)
+	{
+	  hash_rec *ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (dd->HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  ChangeMem (sizeof (hash_rec) * size);
+	  bzero ((char *) ht, sizeof (hash_rec) * size);
+
+	  for (i = 0; i < dd->HashSize; i++)
+	    if (dd->HashTable[i].word)
+	      {
+		register u_char *wptr;
+		register unsigned long hashval, step;
+
+		wptr = dd->HashTable[i].word;
+		HASH (hashval, step, wptr, size);
+		wptr = (ht + hashval)->word;
+		while (wptr)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    wptr = (ht + hashval)->word;
+		  }
+		ht[hashval] = dd->HashTable[i];
+	      }
+	  Xfree (dd->HashTable);
+	  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+	  dd->HashTable = ht;
+	  dd->HashSize = size;
+
+
+	}
+    }
+  return (COMPALLOK);
+}				/* encode */
+
+
+
+static int 
+PackHashTable (dict_data * dd)
+{
+  int s, d;
+  for (s = d = 0; s < dd->HashSize; s++)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  if (!(dd->HashTable = Xrealloc (dd->HashTable,
+				  sizeof (hash_rec) * dd->HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *s1, const void *s2)
+{
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+
+static void 
+WriteHashTable (FILE * fp, dict_data * dd)
+{
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR)
+    return;
+
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; j++)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+
+  for (j = 0; j < dd->HashSize; j++)
+    {
+      curr = dd->HashTable[j].word;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(dd->HashTable[j].wcnt);
+      HTONUL(dd->HashTable[j].occurance_num);
+
+      fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+      fwrite (&dd->HashTable[j].occurance_num,
+	      sizeof (dd->HashTable[j].occurance_num), 1, fp);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(dd->HashTable[j].wcnt);
+      NTOHUL(dd->HashTable[j].occurance_num);
+
+      fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+    }
+}
+
+
+int 
+done_text_1 (char *file_name)
+{
+  char *temp_str;
+  FILE *fp;
+
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+	      file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+	      strerror(errno)
+#else
+	      " "
+#endif
+	      );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  fclose(fp);
+  return COMPALLOK;
+}				/* done_encode */
Index: /main/tags/3.01/indexers/mg/src/text/text.pass2.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/text.pass2.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/text.pass2.c	(revision 21112)
@@ -0,0 +1,791 @@
+/**************************************************************************
+ *
+ * text.pass2.c -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "huffman.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.2  2002/01/22 02:41:14  jrm21
+   Try to print out an error if the aux file couldn't be written to.
+
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:10  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:14  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define POOL_SIZE 1024*256
+
+typedef struct char_pool
+  {
+    struct char_pool *next;
+    u_long left;
+    u_char *ptr;
+    u_char pool[POOL_SIZE];
+  }
+char_pool;
+
+typedef struct novel_hash_rec
+  {
+    u_long ordinal_num;
+    u_char *word;
+  }
+novel_hash_rec;
+
+
+#define INITIAL_HASH_SIZE 7927
+#define MAX_SWAPS 10000
+
+typedef struct novel_hash_table
+  {
+    novel_hash_rec *HashTable;
+    u_long HashSize, HashUsed;
+    char_pool *first_pool;
+    char_pool *pool;
+    u_long next_num, binary_start;
+    novel_hash_rec **code_to_nhr;
+  }
+novel_hash_table;
+
+
+static FILE *text, *text_idx;
+
+static u_char *comp_buffer;
+
+static u_long text_length;
+
+/* [RJM 07/97: 4G limit] */
+static double stats_in_tot_bytes = 0.0;
+static double stats_in_bytes = 0.0;
+static double stats_out_bytes = 0.0;
+
+
+static novel_hash_table nht[2];
+
+static u_long prefix_len = 0;
+
+int blk_start[2][33], blk_end[2][33];
+
+
+static char_pool *
+new_pool (char_pool * pool)
+{
+  char_pool *p = Xmalloc (sizeof (char_pool));
+  if (!p)
+    FatalError (1, "Unable to allocate memory for pool");
+  if (pool)
+    pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+
+int 
+init_text_2 (char *file_name)
+{
+  char path[512];
+  int i;
+
+  if (LoadCompressionDictionary (make_name (file_name, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  if (!(text = create_file (file_name, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  bzero ((char *) &cth, sizeof (cth));
+
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  text_length = sizeof (u_long) + sizeof (cth);
+
+  if (!(text_idx = create_file (file_name, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  if (fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+
+  if (!(comp_buffer = Xmalloc (sizeof (u_char) * buf_size)))
+    {
+      Message ("No memory for compression buffer");
+      return (COMPERROR);
+    }
+
+#if 0
+  MaxMemInUse += sizeof (u_char) * buf_size;
+#endif
+
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; i++)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	bzero ((char *) nht[i].HashTable,
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  nht[i].code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+					((nht[i].HashSize >> 1) + 2));
+	else
+	  nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID ||
+	    cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		num++;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+
+int 
+ic (void *a, void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+
+
+/* #define DOCDUMP 477 */
+
+int 
+process_text_2 (u_char * s_in, int l_in)
+{
+  int which, byte_length;
+  u_char *end = s_in + l_in - 1;
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+
+  which = inaword (s_in, end);
+
+  ENCODE_START (comp_buffer, buf_size)
+
+    ENCODE_BIT (which);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_ENCODE_L (nht[0].binary_start, prefix_len);
+      DELTA_ENCODE_L (nht[1].binary_start, prefix_len);
+    }
+
+  novels_used[0] = novels_used[1] = 0;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("---------------------------------------------------\n");
+      printf ("which = %d\n", which);
+    }
+#endif
+
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      int res;
+
+      if (which)
+	cth.num_of_words++;
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+#ifdef DOCDUMP
+      if (cth.num_of_docs == DOCDUMP)
+	{
+	  printf ("%sword : \"%.*s\"", which ? "    " : "non-", Word[0], Word + 1);
+	}
+#endif
+
+      /* Search the hash table for Word */
+      if (ht[which])
+	{
+	  register unsigned long hashval, step;
+	  register int tsize = ht[which]->size;
+	  register u_char **wptr;
+	  HASH (hashval, step, Word, tsize);
+	  for (;;)
+	    {
+	      register u_char *s1;
+	      register u_char *s2;
+	      register int len;
+	      wptr = ht[which]->table[hashval];
+	      if (wptr == NULL)
+		{
+		  res = COMPERROR;
+		  break;
+		}
+
+	      /* Compare the words */
+	      s1 = Word;
+	      s2 = *wptr;
+	      len = *s1 + 1;
+	      for (; len; len--)
+		if (*s1++ != *s2++)
+		  break;
+
+	      if (len)
+		{
+		  hashval += step;
+		  if (hashval >= tsize)
+		    hashval -= tsize;
+		}
+	      else
+		{
+		  res = ht[which]->table[hashval] - ht[which]->words;
+		  break;
+		}
+	    }
+	}
+      else
+	res = COMPERROR;
+      /* Check that the word was found in the dictionary */
+      if (res == COMPERROR)
+	{
+	  if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	    {
+	      Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      return (COMPERROR);
+	    }
+	  if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	    {
+	      u_long i;
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      HUFF_ENCODE (Word[0], lens_codes[which], lens_huff[which].clens);
+	      for (i = 0; i < Word[0]; i++)
+		HUFF_ENCODE (Word[i + 1], char_codes[which],
+			     char_huff[which].clens);
+	    }
+	  if (cdh.dict_type == MG_SEED_DICTIONARY)
+	    {
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      switch (cdh.novel_method)
+		{
+		case MG_NOVEL_HUFFMAN_CHARS:
+		  {
+		    u_long i;
+		    HUFF_ENCODE (Word[0], lens_codes[which],
+				 lens_huff[which].clens);
+		    for (i = 0; i < Word[0]; i++)
+		      HUFF_ENCODE (Word[i + 1], char_codes[which],
+				   char_huff[which].clens);
+		  }
+		  break;
+		case MG_NOVEL_BINARY:
+		case MG_NOVEL_DELTA:
+		case MG_NOVEL_HYBRID:
+		case MG_NOVEL_HYBRID_MTF:
+		  {
+		    register unsigned long hashval, step;
+		    register novel_hash_table *h = &nht[which];
+		    register int hsize = h->HashSize;
+		    register novel_hash_rec *ent;
+		    HASH (hashval, step, Word, hsize);
+		    for (;;)
+		      {
+			register u_char *s1, *s2;
+			register int len;
+			ent = h->HashTable + hashval;
+			if (!ent->word)
+			  {
+			    int len = *Word + 1;
+			    if (len > h->pool->left)
+			      h->pool = new_pool (h->pool);
+			    ent->word = h->pool->ptr;
+			    ent->ordinal_num = h->next_num++;
+			    if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			      h->code_to_nhr[ent->ordinal_num - 1] = ent;
+			    memcpy (h->pool->ptr, Word, len);
+			    h->pool->ptr += len;
+			    h->pool->left -= len;
+			    h->HashUsed++;
+			    break;
+			  }
+			/* Compare the words */
+			s1 = Word;
+			s2 = ent->word;
+			len = *s1 + 1;
+			for (; len; len--)
+			  if (*s1++ != *s2++)
+			    break;
+
+			if (!len)
+			  break;
+
+			hashval = (hashval + step);
+			if (hashval >= hsize)
+			  hashval -= hsize;
+		      }
+
+		    switch (cdh.novel_method)
+		      {
+		      case MG_NOVEL_BINARY:
+			{
+			  BINARY_ENCODE (ent->ordinal_num, h->binary_start);
+			  if (ent->ordinal_num == h->binary_start)
+			    h->binary_start++;
+			}
+			break;
+		      case MG_NOVEL_DELTA:
+			{
+			  DELTA_ENCODE (ent->ordinal_num);
+			}
+			break;
+		      case MG_NOVEL_HYBRID:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+			}
+			break;
+		      case MG_NOVEL_HYBRID_MTF:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+
+			  if (ent->ordinal_num - 1 >= novels_used[which])
+			    {
+			      int a = novels_used[which];
+			      int b = ent->ordinal_num - 1;
+			      novel_hash_rec *temp;
+
+
+/*                            fprintf(stderr, "a = %d , b = %d\n", a, b);
+ */
+			      temp = h->code_to_nhr[a];
+			      h->code_to_nhr[a] = h->code_to_nhr[b];
+			      h->code_to_nhr[b] = temp;
+			      h->code_to_nhr[a]->ordinal_num = a + 1;
+			      h->code_to_nhr[b]->ordinal_num = b + 1;
+			      if (novels_used[which] == MAX_SWAPS)
+				FatalError (1, "Not enough mem for swapping");
+			      swaps[which][novels_used[which]] = b;
+			      novels_used[which]++;
+			    }
+			}
+			break;
+		      }
+		    if (h->HashUsed >= h->HashSize >> 1)
+		      {
+			novel_hash_rec *ht;
+			unsigned long size;
+			unsigned long i;
+			size = prime (h->HashSize * 2);
+			if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			  {
+			    Xfree (h->code_to_nhr);
+			    h->code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+						      ((size >> 1) + 2));
+			  }
+			if (!(ht = Xmalloc (sizeof (novel_hash_rec) * size)))
+			  {
+			    Message ("Unable to allocate memory for table");
+			    return (COMPERROR);
+			  }
+			bzero ((char *) ht, sizeof (novel_hash_rec) * size);
+
+			for (i = 0; i < h->HashSize; i++)
+			  if (h->HashTable[i].word)
+			    {
+			      register u_char *wptr;
+			      register unsigned long hashval, step;
+
+			      wptr = h->HashTable[i].word;
+			      HASH (hashval, step, wptr, size);
+			      wptr = (ht + hashval)->word;
+			      while (wptr)
+				{
+				  hashval += step;
+				  if (hashval >= size)
+				    hashval -= size;
+				  wptr = (ht + hashval)->word;
+				}
+			      ht[hashval] = h->HashTable[i];
+			      if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+				h->code_to_nhr[ht[hashval].ordinal_num - 1] =
+				  &ht[hashval];
+			    }
+			Xfree (h->HashTable);
+			h->HashTable = ht;
+			h->HashSize = size;
+		      }
+		  }
+		  break;
+		}
+	    }
+	}
+      else
+	{
+	  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+#ifdef DOCDUMP
+	  if (cth.num_of_docs == DOCDUMP)
+	    {
+	      printf ("   %d %d\n", ht[which]->hd->clens[res],
+		      ht[which]->codes[res]);
+	    }
+#endif
+	}
+    }
+
+
+  /* Add a 1 bit onto the end of the buffer the remaining bits in the last
+     byte will all be zero */
+
+  ENCODE_BIT (1);
+
+  ENCODE_FLUSH;
+
+  byte_length = __pos - __base;
+  if (!__remaining)
+    {
+      Message ("The end of the buffer was probably overrun");
+      return COMPERROR;
+    }
+
+  ENCODE_DONE
+
+#ifdef DOCDUMP
+    if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("unused bits = %d\n", bits_unused);
+    }
+#endif
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  text_length += byte_length;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      int i;
+      for (i = 0; i < byte_length; i++)
+	printf ("%02x ", comp_buffer[i]);
+      printf ("\n");
+    }
+#endif
+
+  if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  novel_hash_rec *temp;
+	  temp = nht[which].code_to_nhr[a];
+	  nht[which].code_to_nhr[a] = nht[which].code_to_nhr[b];
+	  nht[which].code_to_nhr[b] = temp;
+	  nht[which].code_to_nhr[a]->ordinal_num = a + 1;
+	  nht[which].code_to_nhr[b]->ordinal_num = b + 1;
+	}
+
+
+  fwrite (comp_buffer, sizeof (*comp_buffer), byte_length, text);
+
+  if ((double) l_in / (double) byte_length > cth.ratio)
+    cth.ratio = (double) l_in / (double) byte_length;
+
+  cth.num_of_docs++;
+  if (l_in > cth.length_of_longest_doc)
+    cth.length_of_longest_doc = l_in;
+
+  cth.num_of_bytes += l_in;
+
+  if (Comp_Stats)
+    {
+      stats_in_tot_bytes += l_in;
+      stats_in_bytes += l_in;
+      stats_out_bytes += byte_length;
+      if (stats_in_bytes >= comp_stat_point)
+	{
+	  fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+		   stats_in_bytes, stats_out_bytes,
+		   (double) stats_out_bytes / (double) stats_in_bytes);
+	  stats_in_bytes = 0.0;
+	  stats_out_bytes = 0.0;
+	}
+    }
+
+  return COMPALLOK;
+}
+
+
+
+
+
+
+int 
+write_aux_dict (char *FileName)
+{
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (FileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              FileName, TEXT_DICT_AUX_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+              " "
+#endif
+              );
+      return COMPERROR;
+    }
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+
+      fwrite (&afh, sizeof (afh), 1, aux);
+
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+void 
+estimate_compressed_aux_dict (void)
+{
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      bzero ((char *) chars, sizeof (chars));
+      bzero ((char *) lens, sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      lens[len]++;
+	      total_uncomp += len + 4;
+	      for (; len; len--)
+		chars[*buf++]++;
+	    }
+	}
+      for (j = 0; j < 256; j++)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; j++)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+
+      aux_compressed += (Calculate_Huffman_Size (16, flens, lens) +
+			 Calculate_Huffman_Size (256, fchars, chars)) / 8;
+
+    }
+
+  Message ("Aux dictionary (Uncompressed) %.2f Mb ( %u bytes   %0.3f %%)",
+	   total_uncomp / 1024.0 / 1024, total_uncomp,
+	   (total_uncomp * 100.0) / bytes_processed);
+  Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	   (aux_compressed * 100.0) / bytes_processed);
+}
+
+
+
+
+
+
+int 
+done_text_2 (char *FileName)
+{
+  if (Comp_Stats)
+    fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+	     stats_in_bytes, stats_out_bytes,
+	     (double) stats_out_bytes / (double) stats_in_bytes);
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth.num_of_docs);
+  HTOND(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth.num_of_words);
+  HTONUL(cth.length_of_longest_doc);
+  HTOND(cth.ratio);
+
+  if (fseek (text_idx, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+  fclose (text_idx);
+
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+  fclose (text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+  
+
+  Message ("Compressed Text %.2f Mb ( %u bytes   %0.3f %%)",
+	   text_length / 1024.0 / 1024.0, text_length,
+	   (text_length * 100.0) / bytes_processed);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / bytes_processed);
+
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_BINARY ||
+       cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID ||
+       cdh.novel_method == MG_NOVEL_HYBRID_MTF))
+    {
+      if (write_aux_dict (FileName) == COMPERROR)
+	return COMPERROR;
+      estimate_compressed_aux_dict ();
+    }
+  else
+    {
+      if (cdh.dict_type != MG_COMPLETE_DICTIONARY)
+	Message ("Huffman info for chars in dictionary %.2f Mb"
+		 " ( %u bytes   %0.3f %%)",
+		 Chars_disk / 1024.0 / 1024, Chars_disk,
+		 (Chars_disk * 100.0) / bytes_processed);
+      unlink (make_name (FileName, TEXT_DICT_AUX_SUFFIX, NULL));
+    }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/indexers/mg/src/text/text_get.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/text_get.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/text_get.c	(revision 21112)
@@ -0,0 +1,1090 @@
+/**************************************************************************
+ *
+ * text_get.c -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "text_get.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg_errors.h"
+#include "local_strings.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:11  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:15  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+
+
+
+
+static double 
+FetchDocStartLev1 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  unsigned long data[2];
+  /* [TS:Sep/94] Fixed up the seek call to give the correct offset */
+  Fseek (td->TextIdxFile,
+	 sizeof (unsigned long) * (DN - 1) +	/* the doc offsets */
+	 sizeof (unsigned long) +	/* the magic number */
+	 sizeof (compressed_text_header),	/* the header */
+	 0);
+  Fread ((char *) &data, sizeof (data), 1, td->TextIdxFile);
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(data[0]);
+  NTOHUL(data[1]);
+
+  *seek_pos = data[0];
+  *len = data[1] - data[0];
+  return (1.0);
+}
+
+#define MG_PAGE_SIZE 2048
+
+static int 
+LoadIdx (text_data * td, unsigned long DN)
+{
+  if (!td->idx_data)
+    {
+      td->idx_data = Xmalloc (sizeof (*(td->idx_data)) * MG_PAGE_SIZE);
+      if (!td->idx_data)
+	FatalError (1, "Out of memory in FDSL2");
+    }
+  if (td->current_pos == -1 || DN >= td->current_pos + MG_PAGE_SIZE - 1 ||
+      DN < td->current_pos)
+    {
+      int i, num;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      long rn = (long) DN - (MG_PAGE_SIZE >> 1);
+      if (rn < 1)
+	rn = 1;
+      Fseek (td->TextIdxWgtFile, (sizeof (unsigned long) + sizeof (float)) *
+	       (rn - 1) + sizeof (unsigned long), 0);
+      num = Fread ((char *) td->idx_data, sizeof (*(td->idx_data)), MG_PAGE_SIZE, /* [RPAP - Jan 97: Endian Ordering] */
+		   td->TextIdxWgtFile);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num; i++)
+	{
+	  NTOHUL(td->idx_data[i].Start);
+	  NTOHF(td->idx_data[i].Weight);
+	}
+
+      td->current_pos = rn;
+    }
+  return DN - td->current_pos;
+}
+
+static double 
+FDSL2 (text_data * td, unsigned long DN, unsigned long *Pos)
+{
+  unsigned long pos = LoadIdx (td, DN);
+  *Pos = td->idx_data[pos].Start;
+  return (td->idx_data[pos].Weight);
+}
+
+
+static double 
+FetchDocStartLev2 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  double Weight;
+  unsigned long s1, s2;
+  Weight = FDSL2 (td, DN, &s1);
+  do
+    {
+      DN++;
+      FDSL2 (td, DN, &s2);
+    }
+  while (s2 == s1);
+  *seek_pos = s1;
+  *len = s2 - s1;
+  return (Weight);
+}
+
+
+
+
+double 
+FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len)
+{
+  qd->text_idx_lookups++;
+  if (qd->td->TextIdxWgtFile)
+    return FetchDocStartLev2 (qd->td, DN, seek_pos, len);
+  else
+    return FetchDocStartLev1 (qd->td, DN, seek_pos, len);
+}
+
+unsigned long 
+FetchInitialParagraph (text_data * td, unsigned long ParaNum)
+{
+  if (td->TextIdxWgtFile)
+    {
+      unsigned long pos;
+      unsigned long start;
+      int PN = ParaNum - 1;
+      pos = LoadIdx (td, ParaNum);
+      start = td->idx_data[pos].Start;
+      while (PN > 0)
+	{
+	  pos = LoadIdx (td, PN);
+	  if (td->idx_data[pos].Start != start)
+	    return PN + 1;
+	  PN--;
+	}
+      return PN + 1;
+    }
+  else
+    return ParaNum;
+}
+
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int 
+FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt)
+{
+  if (!DocEnt->SeekPos)
+    FetchDocStart (qd, DocEnt->DocNum, &DocEnt->SeekPos, &DocEnt->Len);
+  if (!(*DocBuff = Xmalloc (DocEnt->Len)))
+    return (-1);
+
+  if (Fseek (qd->td->TextFile, DocEnt->SeekPos, 0) == -1)
+    FatalError (1, "Error when seeking into text file");
+#if 0
+  printf ("Loading compressed text %d %d\n", DocEnt->SeekPos, DocEnt->Len);
+#endif
+  if (Fread (*DocBuff, 1, DocEnt->Len, qd->td->TextFile) != DocEnt->Len)
+    FatalError (1, "Error when reading data");
+
+  return (DocEnt->Len);
+
+}
+
+
+text_data *
+LoadTextData (File * text, File * text_idx_wgt, File * text_idx)
+{
+  text_data *td;
+
+  if (!(td = Xmalloc (sizeof (text_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  td->TextFile = text;
+  td->TextIdxWgtFile = text_idx_wgt;
+  td->TextIdxFile = text_idx;
+  td->current_pos = -1;
+  td->idx_data = NULL;
+  Fread (&td->cth, sizeof (td->cth), 1, text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(td->cth.num_of_docs);
+  NTOHD(td->cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(td->cth.num_of_words);
+  NTOHUL(td->cth.length_of_longest_doc);
+  NTOHD(td->cth.ratio);
+
+  return (td);
+}
+
+
+void 
+FreeTextData (text_data * td)
+{
+  if (td)
+    {
+      if (td->idx_data)
+	Xfree (td->idx_data);
+      Xfree (td);
+    }
+}
+
+
+static int 
+pts_comp (const void *A, const void *B)
+{
+  const DocEntry *const *a = A;
+  const DocEntry *const *b = B;
+  return (*a)->DocNum - (*b)->DocNum;
+}
+
+
+
+
+int 
+GetPosLens (query_data * qd, DocEntry * Docs, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+  for (i = j = 0; i < num; i++, Docs++)
+    if (!Docs->SeekPos)
+      pts[j++] = Docs;
+
+  if (j)
+    {
+      qsort (pts, j, sizeof (DocEntry *), pts_comp);
+      for (i = 0; i < j; i++)
+	FetchDocStart (qd, pts[i]->DocNum, &pts[i]->SeekPos, &pts[i]->Len);
+    }
+
+  Xfree (pts);
+  return (0);
+}
+
+
+
+
+
+int 
+LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  int mem;
+
+  if (!num)
+    return (0);
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+
+  mem = i = 0;
+  do
+    {
+      pts[i] = Docs;
+      mem += Docs->Len;
+      i++;
+      Docs++;
+    }
+  while (i < num && mem < max_mem);
+  if (i > 1)
+    qsort (pts, i, sizeof (DocEntry *), pts_comp);
+  for (j = 0; j < i; j++)
+    {
+      if (FetchCompressed (qd, &pts[j]->CompTextBuffer, pts[j]) == -1)
+	return (-1);
+      ChangeMemInUse (qd, pts[j]->Len);
+    }
+
+  Xfree (pts);
+
+  return (i);
+}
+
+
+
+
+
+void 
+FreeBuffers (query_data * qd, DocEntry * Docs, int num)
+{
+  int i;
+  for (i = 0; i < num; i++, Docs++)
+    if (Docs->CompTextBuffer)
+      {
+	Xfree (Docs->CompTextBuffer);
+	Docs->CompTextBuffer = NULL;
+	ChangeMemInUse (qd, -Docs->Len);
+      }
+}
+
+
+
+/****************************************************************************/
+
+static void 
+FreeAuxDict (auxiliary_dict * ad)
+{
+  if (!ad)
+    return;
+  if (ad->word_data[0])
+    Xfree (ad->word_data[0]);
+  if (ad->word_data[1])
+    Xfree (ad->word_data[1]);
+  if (ad->words[0])
+    Xfree (ad->words[0]);
+  if (ad->words[1])
+    Xfree (ad->words[1]);
+  Xfree (ad);
+}
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict * cd, File * text_aux_dict)
+{
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = Xmalloc (sizeof (auxiliary_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) ad, sizeof (*ad));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      Fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = Xmalloc (ad->afh[i].mem_for_frags)))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+      if (!(ad->words[i] = Xmalloc (ad->afh[i].num_frags * sizeof (u_char *))))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+
+      Fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	  cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd->cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+
+
+
+
+static u_char ***
+ReadInWords (File * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  if (!(vals = Xmalloc (ptrs_reqd * sizeof (*vals))))
+    return (NULL);
+
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **))))
+    return (NULL);
+
+  if (!(next_word[0] = Xmalloc (mem_reqd))) return (NULL);
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = Getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      Fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static compression_dict *
+Load_Comp_Dict (File * dict, File * aux_dict)
+{
+  int which;
+  compression_dict *cd;
+
+  if (!(cd = Xmalloc (sizeof (compression_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) cd, sizeof (compression_dict));
+
+  cd->MemForCompDict = sizeof (compression_dict);
+
+  if (F_Read_cdh (dict, &cd->cdh, &cd->MemForCompDict, NULL) == -1)
+    return NULL;
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+
+	  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						 NULL)))
+	    return NULL;
+	  cd->escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  cd->chars_huff[which] = hd;
+	  cd->chars_vals[which] = vals;
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  cd->lens_huff[which] = hd;
+	  cd->lens_vals[which] = vals;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->chars_huff[which] = hd;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict,
+				       NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->chars_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      cd->lens_huff[which] = hd;
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict
+				       ,NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->lens_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+
+      if (!(cd->ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+    }
+
+
+  mg_errno = MG_NOERROR;
+
+  cd->fast_loaded = 0;
+  return (cd);
+}
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p, cd) & 7))) != 0)
+
+
+static compression_dict *
+Load_Fast_Comp_Dict (File * text_fast_comp_dict)
+{
+  compression_dict *cd;
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  Fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (!(cd = Xmalloc (mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  Fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = Xmalloc (fixup_mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; p++)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (i = 0; i <= 1; i++)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; j++)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  Xfree (fixup);
+  return (cd);
+}
+
+
+
+
+compression_dict *
+LoadCompDict (File * text_comp_dict,
+	      File * text_aux_dict,
+	      File * text_fast_comp_dict)
+{
+  return text_fast_comp_dict ?
+    Load_Fast_Comp_Dict (text_fast_comp_dict) :
+    Load_Comp_Dict (text_comp_dict, text_aux_dict);
+}
+
+
+
+
+void 
+FreeCompDict (compression_dict * cd)
+{
+  int which;
+  if (cd->fast_loaded)
+    {
+      Xfree (cd);
+      return;
+    }
+  for (which = 0; which < 2; which++)
+    {
+      if (cd->cfh[which])
+	Xfree (cd->cfh[which]);
+      if (cd->chars_huff[which])
+	Xfree (cd->chars_huff[which]);
+      if (cd->lens_huff[which])
+	Xfree (cd->lens_huff[which]);
+      if (cd->values[which])
+	{
+	  Xfree (cd->values[which][0][0]); /* [RJM 07/98: Memory Leak] */
+	  Xfree (cd->values[which][0]);
+	  Xfree (cd->values[which]);
+	}
+      if (cd->chars_vals[which])
+	{
+	  Xfree (cd->chars_vals[which][0]);
+	  Xfree (cd->chars_vals[which]);
+	}
+      if (cd->lens_vals[which])
+	{
+	  Xfree (cd->lens_vals[which][0]);
+	  Xfree (cd->lens_vals[which]);
+	}
+    }
+  if (cd->ad)
+    FreeAuxDict (cd->ad);
+  Xfree (cd);
+}
+
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+/*#define DUMPDOC */
+
+#define MAX_SWAPS 10000
+
+int 
+DecodeText (compression_dict * cd,
+	    u_char * s_in, int l_in, u_char * s_out, int *l_out)
+{
+  auxiliary_dict *ad = cd->ad;
+  int which;
+  u_long num_bits, bits;
+  u_char *ptr = s_out;
+  static int num = 0;
+  u_long binary_start[2];
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+  novels_used[0] = novels_used[1] = 0;
+
+  {
+    unsigned char bf = s_in[l_in - 1];
+    num_bits = 1;
+    while ((bf & 1) != 1)
+      {
+	num_bits++;
+	bf >>= 1;
+      }
+    num_bits = l_in * 8 - num_bits;
+  }
+
+  DECODE_START (s_in, l_in)
+
+    which = DECODE_BIT;
+  bits = 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_DECODE_L (binary_start[0], bits);
+      DELTA_DECODE_L (binary_start[1], bits);
+    }
+
+
+  while (bits < num_bits)
+    {
+      register unsigned code, len;
+      register int r;
+      register u_char *t, *b = NULL;
+      u_char word[MAXWORDLEN + 1];
+
+#ifdef DUMPDOC
+      printf ("\n%d %d  ", bits, num_bits);
+#endif
+      if (cd->cfh[which])
+	{
+	  MY_HUFF_DECODE (len, code, cd->cfh[which]->hd.min_code);
+	  bits += len;
+
+	  r = code & ((1 << cd->cdh.lookback) - 1);
+	  t = cd->values[which][len][code >> cd->cdh.lookback];
+
+	  /* step through from base pointer */
+	  b = word + 1;
+	  while (r--)
+	    {
+	      register int copy = *t >> 4;
+	      memcpy (word + copy + 1, t + 1, *t & 0xf);
+	      word[0] = copy + (*t & 0xf);
+	      t += ((*t) & 0xf) + 1;
+	    }
+	}
+      else
+	t = NULL;
+      if (t == cd->escape[which])
+	{
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      {
+		int len, i;
+		int c;
+		HUFF_DECODE_L (len, cd->lens_huff[which]->min_code,
+			       cd->lens_vals[which], bits);
+		for (i = 0; i < len; i++)
+		  {
+		    HUFF_DECODE_L (c, cd->chars_huff[which]->min_code,
+				   cd->chars_vals[which], bits);
+		    *ptr++ = c;
+		  }
+	      }
+	      break;
+	    case MG_NOVEL_BINARY:
+	    case MG_NOVEL_DELTA:
+	    case MG_NOVEL_HYBRID:
+	    case MG_NOVEL_HYBRID_MTF:
+	      {
+		int idx = 0, len;
+		u_char *base;
+		switch (cd->cdh.novel_method)
+		  {
+		  case MG_NOVEL_BINARY:
+		    {
+		      BINARY_DECODE_L (idx, binary_start[which], bits);
+		      if (idx == binary_start[which])
+			binary_start[which]++;
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_DELTA:
+		    {
+		      DELTA_DECODE_L (idx, bits);
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID_MTF:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		      if (idx >= novels_used[which])
+			{
+			  u_char *temp;
+			  temp = ad->words[which][idx];
+			  ad->words[which][idx] =
+			    ad->words[which][novels_used[which]];
+			  ad->words[which][novels_used[which]] = temp;
+			  swaps[which][novels_used[which]] = idx;
+			  idx = novels_used[which]++;
+			}
+		    }
+		    break;
+		  }
+		base = ad->words[which][idx];
+		len = *base++;
+#ifdef DUMPDOC
+		printf ("[[");
+#endif
+		for (; len; len--)
+		  {
+		    *ptr++ = *base++;
+#ifdef DUMPDOC
+		    putchar (*(base - 1));
+#endif
+		  }
+#ifdef DUMPDOC
+		printf ("]]");
+#endif
+	      }
+	      break;
+	    }
+	}
+      else
+	{
+	  /* copy over the matching prefix */
+	  r = (*t >> 4);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *b++;
+#else
+	    {
+	      *ptr = *b++;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+
+	  /* and the stored suffix */
+	  r = ((*t) & 0xf);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *++t;
+#else
+	    {
+	      *ptr = *++t;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+	}
+      which = !which;
+    }
+
+  DECODE_DONE
+
+    * l_out = ptr - s_out;
+  num += *l_out + 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  u_char *temp;
+	  temp = ad->words[which][a];
+	  ad->words[which][a] = ad->words[which][b];
+	  ad->words[which][b] = temp;
+	}
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/indexers/mg/src/text/text_get.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/text_get.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/text_get.h	(revision 21112)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * text_get.h -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT_GET
+#define H_TEXT_GET
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+double FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		u_char * s_in, int l_in, u_char * s_out, int *l_out);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/warranty.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/warranty.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/warranty.h	(revision 21112)
@@ -0,0 +1,24 @@
+static char *warranty_str[] = {
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /main/tags/3.01/indexers/mg/src/text/warranty.src
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/warranty.src	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/warranty.src	(revision 21112)
@@ -0,0 +1,23 @@
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /main/tags/3.01/indexers/mg/src/text/weights.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/weights.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/weights.c	(revision 21112)
@@ -0,0 +1,161 @@
+/**************************************************************************
+ *
+ * weights.c -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  2003/02/20 21:18:24  mdewsnip
+   Addition of MG package for search and retrieval
+
+   Revision 1.1  1999/08/10 21:18:27  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:18  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+approx_weights_data *
+LoadDocWeights (File * weight_file,
+		unsigned long num_of_docs)
+{
+  approx_weights_data *awd;
+  int num;
+
+  if (!(awd = Xmalloc (sizeof (*awd))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fseek (weight_file, sizeof (long), 0);
+  Fread (&awd->bits, sizeof (awd->bits), 1, weight_file);
+  Fread (&awd->L, sizeof (awd->L), 1, weight_file);
+  NTOHD(awd->L);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&awd->B, sizeof (awd->B), 1, weight_file);
+  NTOHD(awd->B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  awd->mask = awd->bits == 32 ? 0xffffffff : (1 << awd->bits) - 1;
+
+  num = (num_of_docs * awd->bits + 31) / 32;
+  if (!(awd->DocWeights = Xmalloc (sizeof (unsigned long) * num)))
+    {
+      Xfree (awd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (awd->DocWeights, sizeof (unsigned long), num, weight_file);
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    unsigned long i;
+    for (i = 0; i < num; i++)
+      NTOHUL(awd->DocWeights[i]);
+  }
+
+  awd->MemForWeights = num * sizeof (unsigned long);
+  awd->num_of_docs = num_of_docs;
+
+  mg_errno = MG_NOERROR;
+
+  if (awd->bits <= 12)
+    {
+      int i, size = (1 << awd->bits);
+      if (!(awd->table = (float *) Xmalloc (size * sizeof (float))))
+	  return (awd);
+      awd->table[0] = awd->L;
+      for (i = 1; i < size; i++)
+	awd->table[i] = awd->table[i - 1] * awd->B;
+    }
+  else
+    awd->table = NULL;
+  return awd;
+}
+
+
+
+
+
+
+/*
+ * the first document in the collection has DocNum = 0
+ */
+float 
+GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum)
+{
+  register unsigned long c, Pos;
+  register unsigned long *dw;
+  if (awd == NULL)
+    return 1.0;
+#if 0
+  if (DocNum < 0 || DocNum >= awd->num_of_docs)
+    FatalError (1, "Something is wrong in \"GetDocWeight\" DocNum = %d\n",
+		DocNum);
+#endif
+  Pos = DocNum * awd->bits;
+  dw = &(awd->DocWeights[Pos / MAXBITS]);
+  Pos &= (MAXBITS - 1);
+  c = *dw >> Pos;
+  if (Pos + awd->bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - Pos);
+  c &= awd->mask;
+  if (awd->table)
+    return (awd->table[c]);
+  else
+    return (awd->L * pow (awd->B, (double) c));
+
+}
+
+void 
+FreeWeights (approx_weights_data * awd)
+{
+  Xfree (awd->DocWeights);
+  if (awd->table)
+    Xfree (awd->table);
+  Xfree (awd);
+}
Index: /main/tags/3.01/indexers/mg/src/text/weights.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/weights.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/weights.h	(revision 21112)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * weights.h -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_WEIGHTS
+#define H_WEIGHTS
+
+approx_weights_data *LoadDocWeights (File * weight_file,
+				     unsigned long num_of_docs);
+
+float GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum);
+
+void FreeWeights (approx_weights_data * awd);
+
+#endif
Index: /main/tags/3.01/indexers/mg/src/text/win32.mak
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/win32.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/win32.mak	(revision 21112)
@@ -0,0 +1,241 @@
+###########################################################################
+#
+# win32 makefile -- mg\src\text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGHOME = ..\..
+
+AR = lib
+
+CC = cl
+CPPFLAGS = 
+# DEFS = -DQUIET -D__WIN32__ -DSHORT_SUFFIX -DHAVE_STRING_H -DHAVE_FCNTL_H \
+#        -DSTDC_HEADERS -DHAVE_CONFIG_H
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H \
+       -D__WIN32__ -D_LITTLE_ENDIAN
+INCLUDES = -I"." -I"$(MGHOME)" -I"$(MGHOME)\lib"
+COMPILE = $(CC) $(CPPFLAGS) -c $(DEFS) $(INCLUDES)
+
+LDFLAGS =
+LINK = $(CC) $(LDFLAGS)
+
+LIBS = "$(MGHOME)\lib\libmg.lib"
+
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h          query_term_list.h
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  mgpass.c                query_term_list.c       words.c              \
+  mg_passes_4jni.c
+
+EXEC = \
+  mgquery$e mg_weights_build$e mgstat$e \
+  mg_invf_dump$e     mg_invf_dict$e \
+  mg_invf_rebuild$e     mgdictlist$e \
+  mg_passes$e     mg_perf_hash_build$e \
+  mg_compression_dict$e     mg_text_estimate$e \
+  mg_fast_comp_dict$e \
+  mg_hilite_words$e     mg_invf_merge$e \
+  mg_text_merge$e     bool_tester$e  mgpass$e \
+  mg_stem_idx$e  mgstemidxlist$e \
+  mg_decompress_text$e 
+
+all: $(EXEC) libmgtextin.lib libmgpass.lib
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o environment$o
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+BOOL_OBJS = bool_tester$o bool_tree$o bool_parser$o bool_optimiser$o  \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o \
+	    query_term_list$o words$o environment$o
+
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o environment$o
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+PASS_OBJS = \
+	 mgpass$o words$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o environment$o
+mgpass$e: $(PASS_OBJS)
+	$(LINK) $(PASS_OBJS) $(LIBS)
+
+LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
+ advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
+ odbccp32.lib
+
+BUILDER_OBJS = \
+	builder$o
+builder$e: $(BUILDER_OBJS)
+	$(LINK) $(BUILDER_OBJS) $(LIBS) $(LINK32_FLAGS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o \
+		       mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+        bool_query$o words$o
+
+JNI_LIB_OBJS = \
+	mg_passes_4jni$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o  words$o environment$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o 
+
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+
+libmgtextin.lib : $(LIB_OBJS)
+        if exist libmgtextin.lib del libmgtextin.lib
+	$(AR) /out:libmgtextin.lib $(LIB_OBJS)
+
+libmgpass.lib : $(JNI_LIB_OBJS)
+ 	  if exist libmgpass.lib del libmgpass.lib
+	$(AR) /out:libmgpass.lib $(JNI_LIB_OBJS)
+
+install: 
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDL3HOME)\bin"
+
+clean:
+        if exist *$o del *$o 
+	if exist *$e del *$e
+        if exist libmgtextin.lib del libmgtextin.lib
+        if exist libmgpass.lib del libmgpass.lib
Index: /main/tags/3.01/indexers/mg/src/text/words.c
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/words.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/words.c	(revision 21112)
@@ -0,0 +1,34 @@
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skipped. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+
+/* It determines whether a given place in a UTF-8 encoded 
+   Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
Index: /main/tags/3.01/indexers/mg/src/text/words.h
===================================================================
--- /main/tags/3.01/indexers/mg/src/text/words.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/src/text/words.h	(revision 21112)
@@ -0,0 +1,394 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+#include "unitool.h"
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+/*#define MAXNUMERIC	4*/
+
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded 
+	   Unicode string is a unicode space. */
+
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with 
+	   begining unicode spaces skipped. */ 
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+  /*  
+#define PARSE_WORD(Word, s_in, end)                                  \
+  do {                                                               \
+	  register u_char  *wptr = (Word)+1;                         \
+	  register int    length = 0;                                \
+	  register int    c = *(s_in);                               \
+	  register int	numeric = 0;                                 \
+                                                                     \
+	  while( length < MAXWORDLEN && INAWORD(c) && (s_in)<=(end)) \
+	    {                                                        \
+	      if ((numeric += INNUMBER(c)) > MAXNUMERIC)             \
+		break;                                               \
+	      *wptr++ = c;                                           \
+	      length++;                                              \
+	      c = *++(s_in);                                         \
+	    }                                                        \
+	  *(Word) = length;                                          \
+  }while(0)
+  */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+	  register u_char  *wptr = (Word)+1;                       \
+	  register int    length = 0;                              \
+	  register int    c = *(s_in);                             \
+                                                                   \
+	  while( length < MAXWORDLEN && !INAWORD(c) && (s_in)<=(end) ) \
+	    {                                                      \
+	      *wptr++ = c;                                         \
+	      length++;                                            \
+	      c = *++(s_in);                                       \
+	    }                                                      \
+	  *(Word) = length;                                        \
+  }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_STEM_WORD 
+ * Description: 
+ *      Extracts out Word.      
+ * Input: 
+ *      s_in points to 1st letter in buffer to test
+ *      end points to last letter in buffer
+ * Output: 
+ *      s_in is modified to move to next word
+ *      Returns Word filled in with length in 1st byte.
+ * ========================================================================= */
+#define PARSE_STEM_WORD(Word, s_in, end)                           \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+    register int maxnumeric = IntEnv (GetEnv ("maxnumeric"), 4);   \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXSTEMLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= maxnumeric))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+    /*
+#define PARSE_STEM_WORD(Word, s_in, end)                      \
+  do                                                          \
+    {                                                         \
+      register u_char  *wptr = (Word)+1;                      \
+      register int    length = 0;                             \
+      register int    c = *(s_in);                            \
+      register int    numeric = 0;                            \
+                                                              \
+      while ( length < MAXSTEMLEN && INAWORD(c) && (s_in)<=(end)) \
+        {                                                     \
+ 	  if ((numeric += INNUMBER(c)) > MAXNUMERIC)          \
+	    break;                                            \
+	  *wptr++ = c;                                        \
+	  length++;                                           \
+	  c = *++(s_in);                                      \
+	}                                                     \
+      *(Word) = length;                                       \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD 
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD(s_in, end)                             \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD(s_in, end)           \
+  do                                             \
+    {                                            \
+      while (!INAWORD(*(s_in)) && (s_in)<=(end)) \
+	(s_in)++;                                \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD_OR_SGML_TAG 
+ * Description: 
+ *      Like PARSE_NON_STEM_WORD but also eats up SGML tags
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end)                 \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == '<') {                                              \
+	while (charlength > 0 && c != '>') {                       \
+	  (s_in) += charlength;                                    \
+	  charlength = parse_utf8_char((s_in),(end),&c);           \
+	}                                                          \
+      }                                                            \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end) \
+  do                                               \
+    {                                              \
+      register int    c = *(s_in);                 \
+                                                   \
+      while (!INAWORD(c) && (s_in)<=(end))         \
+        {                                          \
+	  if (c == '<')                            \
+            {                                      \
+	      while (c != '>' && (s_in)<=(end))    \
+		c = *++(s_in);                     \
+            }                                      \
+	  if ((s_in)<=(end))                       \
+	    c = *++(s_in);                         \
+	}                                          \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_OPT_TERM_PARAM     [RPAP - Jan 97: Stem Index Change]
+ * Description: 
+ *      Extracts out optional paramater for query term.
+ *      Needed only in parsing the query line !
+ *
+ *      Note: that this function has not been converted to use UTF-8
+ *            as it should still work as it is (only uses ascii
+ *            characters)
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_OPT_TERM_PARAM(Param, type, s_in, end)                       \
+  do                                                                       \
+    {                                                                      \
+	  register u_char  *wptr = (Param);                                \
+	  register int    length = 0;                                      \
+	  register int    c = *(s_in);                                     \
+                                                                           \
+          if (c == WEIGHTPARAM || c == STEMPARAM)                          \
+	    {                                                              \
+	      type = c;                                                    \
+	      c = *++(s_in);                                               \
+	      while( length < MAXPARAMLEN && isdigit(c) && (s_in)<=(end))  \
+		{                                                          \
+	           *wptr++ = c;                                            \
+	           length++;                                               \
+	           c = *++(s_in);                                          \
+	        }                                                          \
+	      *wptr = '\0';                                                \
+              for (; isdigit(c) && (s_in)<=(end); c = *++(s_in))           \
+                ;                                                          \
+            }							           \
+    }while(0)
+
+/* =========================================================================
+ * Macro: PARSE_RANKED_NON_STEM_WORD    [RJM 07/97: Ranked Required Terms]
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      If come across a match requirement store it in require_match
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above
+ * Output: the requirement mode for the next term. -1=must not match,
+ *      0=optional match, 1=must match
+ * ========================================================================= */
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)       \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+    (require_match) = 0;                                           \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == MUSTMATCHPARAM) (require_match) = 1;                \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)         \
+  do {                                            \
+    (require_match) = 0;                          \
+    while (!INAWORD(*(s_in)) && (s_in)<=(end)) {  \
+      if (*(s_in) == MUSTMATCHPARAM) {            \
+        (require_match) = 1;                      \
+      }                                           \
+      (s_in)++;                                   \
+    }                                             \
+  } while (0)
+    */
Index: /main/tags/3.01/indexers/mg/stamp-h.in
===================================================================
--- /main/tags/3.01/indexers/mg/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/indexers/mg/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/indexers/mg/stemmerinterface.c
===================================================================
--- /main/tags/3.01/indexers/mg/stemmerinterface.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/stemmerinterface.c	(revision 21112)
@@ -0,0 +1,39 @@
+#include "lib/lovinstem.h"
+#include <stdio.h>
+#include <ctype.h>
+
+int main () {
+  char word[256];
+  char c;
+  int len;
+
+  c = fgetc (stdin);
+  while (!feof (stdin)) {
+    /* output non words */
+    while (!feof (stdin) && !isalnum (c)) {
+      fputc (c, stdout);
+      c = fgetc (stdin);
+    }
+
+    /* get the word */
+    len = 0;
+    while (!feof (stdin) && isalnum (c) && len < 255) {
+      len++;
+      word[len] = c;
+      c = fgetc (stdin);
+    }
+    word[0] = len;
+
+    /* if we found a word stem and output it */
+    if (len > 0) {
+      lovinstem (word);
+      len = 0;
+      while (len < word[0]) {
+        len++;
+        fputc(word[len], stdout);
+      }
+    }
+  }
+}
+
+
Index: /main/tags/3.01/indexers/mg/sysfuncs.h
===================================================================
--- /main/tags/3.01/indexers/mg/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/indexers/mg/sysfuncs.h	(revision 21112)
@@ -0,0 +1,361 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#include <sys/stat.h>
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#if defined(__MSDOS__)
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#if defined(__GNUC__) && defined(__WIN32__)
+#include <limits.h>
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#if defined(__WIN32__)
+#include <io.h>
+#define lseek _lseek
+#elif defined(__MSDOS__)
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#include <pathmax.h>
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+#include <getopt.h>
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#if (HAVE_VPRINTF || HAVE_DOPRNT) && __STDC__
+/*void error __P ((int, int, const char *,...));*/
+void error (int, int, const char *,...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/indexers/mg/test/Makefile
===================================================================
--- /main/tags/3.01/indexers/mg/test/Makefile	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/Makefile	(revision 21112)
@@ -0,0 +1,10 @@
+DISTFILES = Makefile q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum simple_sum.c alice.len
+
+all: simple_sum
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+simple_sum: simple_sum.c
+	cc -o $@ $?
Index: /main/tags/3.01/indexers/mg/test/alice.len
===================================================================
--- /main/tags/3.01/indexers/mg/test/alice.len	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/alice.len	(revision 21112)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19321
Index: /main/tags/3.01/indexers/mg/test/alice.sum
===================================================================
--- /main/tags/3.01/indexers/mg/test/alice.sum	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/alice.sum	(revision 21112)
@@ -0,0 +1,8 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict.blocked: 1030626 33214
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /main/tags/3.01/indexers/mg/test/mgrc1
===================================================================
--- /main/tags/3.01/indexers/mg/test/mgrc1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/mgrc1	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query boolean
Index: /main/tags/3.01/indexers/mg/test/mgrc2
===================================================================
--- /main/tags/3.01/indexers/mg/test/mgrc2	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/mgrc2	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode text
+.set query ranked
Index: /main/tags/3.01/indexers/mg/test/mgrc3
===================================================================
--- /main/tags/3.01/indexers/mg/test/mgrc3	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/mgrc3	(revision 21112)
@@ -0,0 +1,2 @@
+.set mode docnums
+.set query boolean
Index: /main/tags/3.01/indexers/mg/test/q1
===================================================================
--- /main/tags/3.01/indexers/mg/test/q1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/q1	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/indexers/mg/test/q2
===================================================================
--- /main/tags/3.01/indexers/mg/test/q2	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/q2	(revision 21112)
@@ -0,0 +1,1 @@
+flamingo
Index: /main/tags/3.01/indexers/mg/test/q3
===================================================================
--- /main/tags/3.01/indexers/mg/test/q3	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/q3	(revision 21112)
@@ -0,0 +1,2 @@
+Rabbit & !(Alice | she)
+laser
Index: /main/tags/3.01/indexers/mg/test/r1
===================================================================
--- /main/tags/3.01/indexers/mg/test/r1	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/r1	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /main/tags/3.01/indexers/mg/test/r2
===================================================================
--- /main/tags/3.01/indexers/mg/test/r2	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/r2	(revision 21112)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /main/tags/3.01/indexers/mg/test/r3
===================================================================
--- /main/tags/3.01/indexers/mg/test/r3	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/r3	(revision 21112)
@@ -0,0 +1,21 @@
+> Rabbit & !(Alice | she)
+Rabbit Alice she
+     14.14     
+    126.126    
+    143.143    
+    478.478    
+    701.701    
+    704.704    
+    705.705    
+    755.755    
+    774.774    
+    786.786    
+    788.788    
+    791.791    
+    793.793    
+    801.801    
+    803.803    
+15 documents retrieved.
+> laser
+laser
+No entries correspond to that query.
Index: /main/tags/3.01/indexers/mg/test/simple_sum.c
===================================================================
--- /main/tags/3.01/indexers/mg/test/simple_sum.c	(revision 21112)
+++ /main/tags/3.01/indexers/mg/test/simple_sum.c	(revision 21112)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * sum.c -- byte summing program
+ * Copyright (C) 1995  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  2003/02/20 21:20:24  mdewsnip
+Addition of MG package for search and retrieval
+
+Revision 1.1  1999/08/10 21:18:33  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:36:00  rjmcnab
+*** empty log message ***
+
+*/
+
+/*
+ * If given -l option then just prints the length.
+ * Normally prints the sum and the length for the file.
+ */
+
+static int length_only = 0;
+
+#include <stdio.h>
+
+void 
+output_sum(fname, input_file, output_file)
+char *fname;
+FILE *input_file;
+FILE *output_file;
+{
+  int ch = '\0';
+  unsigned long sum = 0;
+  unsigned long num_bytes = 0;
+
+  while((ch = getc(input_file))!=EOF){
+    sum += ch;
+    num_bytes++;
+  }
+
+  if (length_only)
+    fprintf(output_file, "%s: %ld\n", fname, num_bytes);
+  else
+    fprintf(output_file, "%s: %ld %ld\n", fname, sum, num_bytes);
+}
+
+void
+main(argc, argv)
+int argc;
+char *argv[];
+{
+  int i = 0;
+
+  /* use stdin */
+  if (argc == 1){
+    output_sum("-", stdin, stdout);
+    exit(0);
+  }
+
+  /* use args as file names for input */
+  for (i=1;i<argc; i++){
+    char *fname = argv[i];
+    FILE *input_file = NULL;
+    
+    if (!length_only && (argv[i][0] == '-') && (argv[i][1] =='l') ) 
+      { 
+        length_only = 1; 
+        continue;
+      }
+
+    input_file = fopen(fname, "r");
+    if (!input_file){
+      fprintf(stderr,"Could not open %s\n", fname);
+      continue;
+    }
+    else{
+      output_sum(fname, input_file, stdout);
+      fclose(input_file);
+    }
+  }/*for*/
+  exit(0);
+
+}
Index: /main/tags/3.01/indexers/mg/win32.mak
===================================================================
--- /main/tags/3.01/indexers/mg/win32.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mg/win32.mak	(revision 21112)
@@ -0,0 +1,76 @@
+###########################################################################
+#
+# win32 makefile -- mg
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+AR = lib
+CC = cl
+CPPFLAGS =
+
+MGHOME = $(MAKEDIR)
+DLL = 0
+DLLDEBUG = 0
+
+MDEFINES = /f win32.mak
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+SOURCES = 
+
+OBJECTS = 
+
+OTHERDIRS = lib 
+
+INSTALLDIRS = src\text jni 
+
+MAKECMD = $(MAKE) $(MDEFINES)
+!IF $(DLL)
+MAKECMD = $(MAKECMD) DLL=1
+!ENDIF
+!IF $(DLLDEBUG)
+MAKECMD = $(MAKECMD) DLLDEBUG=1
+!ENDIF
+
+all: win32.mak $(OTHERDIRS) $(INSTALLDIRS)
+
+java:	FORCE
+	cd "java\org\greenstone\mg"
+	call winMake.bat
+	call winMake.bat install 
+	cd "$(MGHOME)"
+
+
+install: win32.mak $(INSTALLDIRS)
+
+$(OTHERDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	cd "$(MGHOME)"
+
+$(INSTALLDIRS): FORCE
+	cd "$@"
+	$(MAKECMD)
+	$(MAKECMD) install
+	cd "$(MGHOME)"
+
+FORCE:
Index: /main/tags/3.01/indexers/mg/winMake.bat
===================================================================
--- /main/tags/3.01/indexers/mg/winMake.bat	(revision 21112)
+++ /main/tags/3.01/indexers/mg/winMake.bat	(revision 21112)
@@ -0,0 +1,47 @@
+@echo off
+
+set MAKE=nmake
+set MAKE_OPTIONS=/f
+
+if ""%1"" == """" goto all
+if ""%1"" == ""compile"" goto go
+if ""%1"" == ""install"" goto go
+if ""%1"" == ""clean"" goto go
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:go
+	if ""%2"" == ""javaonly"" goto java
+	
+:all
+	cd lib
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd src\images
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..\..
+
+	cd src\text
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..\..
+
+	cd java\org\greenstone\mg
+	call winMake.bat %1
+	cd ..\..\..\..
+
+	cd jni
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+:java
+	cd java\org\greenstone\mg
+	call winMake.bat %1
+	cd ..\..\..\..
+	goto done
+
+:done
+
+
Index: /main/tags/3.01/indexers/mgpp/COPYING
===================================================================
--- /main/tags/3.01/indexers/mgpp/COPYING	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/indexers/mgpp/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/Makefile.in	(revision 21112)
@@ -0,0 +1,111 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CPP = @CXX@
+CFLAGS = @CFLAGS@ -ansi
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib text java/org/greenstone/mgpp jni
+
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall:
+#config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+distclean: distclean-recursive distclean-local
+	rm config.status
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive \
+maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile config.cache config.h config.log stamp-h
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in
+
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
+
Index: /main/tags/3.01/indexers/mgpp/acconfig.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/acconfig.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/acconfig.h	(revision 21112)
@@ -0,0 +1,41 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
Index: /main/tags/3.01/indexers/mgpp/aclocal.m4
===================================================================
--- /main/tags/3.01/indexers/mgpp/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/aclocal.m4	(revision 21112)
@@ -0,0 +1,224 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ([regex])	
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , 
+   [AC_CHECK_SIZEOF(unsigned char *, unsigned char *)  
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      AC_MSG_WARN(I'm forcing you to use regex because I can't 
+        find a local rx library and the one included with this 
+        distribution doesn't work on 64-bit machines like yours)
+      AC_LIBOBJ([regex])
+    else
+      AC_LIBOBJ([rx])
+    fi]
+  )
+fi
+])
+
+
+## --------------------------------------- ##
+## Check if --with-gnu-readline was given. ##
+## --------------------------------------- ##
+
+AC_DEFUN(fp_WITH_GNU_READLINE,
+[AC_MSG_CHECKING(whether GNU readline requested)
+  AC_ARG_WITH(gnu_readline,
+    [  --with-gnu-readline     compile with GNU readline support],
+    [if test "$withval" = yes; then
+      AC_MSG_RESULT(yes)
+      ac_with_gnu_readline=1
+    else
+      AC_MSG_RESULT(no)
+    fi],
+    AC_MSG_RESULT(no))
+
+  if test -n "$ac_with_gnu_readline"; then
+    AC_CHECK_HEADER(readline/readline.h, ac_mg_readline_header_found=1,
+	AC_MSG_WARN(Can't find GNU readline headers; configuring without \
+GNU readline support))
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      AC_CHECK_LIB(readline, main, 
+        [ac_mg_readline_lib_found=1
+         AC_DEFINE(WITH_GNU_READLINE)
+         LIBS="$LIBS -lreadline"],
+        AC_MSG_WARN(Can't find GNU readline library; configuring without \
+GNU readline support))
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to 
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	AC_CHECK_LIB(curses, main,
+	  LIBS="$LIBS -lcurses")
+      fi
+    fi
+  fi
+])	
+
+
+
+#AC_DEFUN(fp_WITH_GNU_READLINE,
+#[AC_MSG_CHECKING(whether GNU readline requested)
+#  AC_ARG_WITH(gnu-readline,
+#    [  --with-gnu-readline   compile with GNU readline support],
+#    [if test "$withval" = yes; then
+#      AC_MSG_RESULT(yes)
+#      ac_with_gnu_readline=1
+#    else
+#      AC_MSG_RESULT(no)
+#    fi],
+#    AC_MSG_RESULT(no))
+#  if test -n "$ac_with_gnu_readline"; then
+#    AC_CHECK_LIB(readline, main, 
+#      AC_CHECK_HEADER(readline/readline.h,
+#	[AC_DEFINE(WITH_GNU_READLINE)
+## curses required on some systems (DEC OSF/1), but not on others (GNU/Linux,
+##  Solaris 2.6).  Forcing its inclusion is unsophisticated; if it causes 
+##  any actual problems on any UNIX platforms, let the maintainer know.
+#	LIBS="$LIBS -lreadline -lcurses"],
+#        AC_MSG_WARN(Can't find GNU readline headers: configuring \
+#without GNU readline support)
+#      ),
+#      AC_MSG_WARN(Can't find GNU readline library: configuring \
+#without GNU readline support)
+#    )
+#  fi 
+#])
Index: /main/tags/3.01/indexers/mgpp/config.guess
===================================================================
--- /main/tags/3.01/indexers/mgpp/config.guess	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/indexers/mgpp/config.h.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/config.h.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/config.h.in	(revision 21112)
@@ -0,0 +1,209 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+#ifndef _MGPP_CONFIG_H
+#define _MGPP_CONFIG_H
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
+
+/* The number of bytes in a unsigned char *.  */
+#undef SIZEOF_UNSIGNED_CHAR_P
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#undef HAVE_STRERROR
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <crypt.h> header file.  */
+#undef HAVE_CRYPT_H
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the crypt library (-lcrypt).  */
+#undef HAVE_LIBCRYPT
+
+/* Define if you have the g library (-lg).  */
+#undef HAVE_LIBG
+
+/* Define if you have the gdbm library (-lgdbm).  */
+#undef HAVE_LIBGDBM
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if you have the stdc++ library (-lstdc++).  */
+#undef HAVE_LIBSTDC__
+
+/* Define if you have gettext (eg glibc) */
+#undef ENABLE_NLS
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
+
+#endif /* _MGPP_CONFIG_H */
Index: /main/tags/3.01/indexers/mgpp/config.sub
===================================================================
--- /main/tags/3.01/indexers/mgpp/config.sub	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/indexers/mgpp/configure
===================================================================
--- /main/tags/3.01/indexers/mgpp/configure	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/configure	(revision 21112)
@@ -0,0 +1,8822 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="text/MGQuery.cpp"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS JNIINC JNISUFFIX JNIFLAGS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Features:
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --enable-override-longlong
+                          overide use of long long with GCC
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+  --with-gnu-readline     compile with GNU readline support
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mgpp
+VERSION=1.3
+
+LDFLAGS=
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lg" >&5
+echo $ECHO_N "checking for main in -lg... $ECHO_C" >&6
+if test "${ac_cv_lib_g_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lg  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_g_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_g_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_g_main" >&5
+echo "${ECHO_T}$ac_cv_lib_g_main" >&6
+if test $ac_cv_lib_g_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBG 1
+_ACEOF
+
+  LIBS="-lg $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lstdc++" >&5
+echo $ECHO_N "checking for main in -lstdc++... $ECHO_C" >&6
+if test "${ac_cv_lib_stdcpp_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lstdc++  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_stdcpp_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_stdcpp_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_stdcpp_main" >&5
+echo "${ECHO_T}$ac_cv_lib_stdcpp_main" >&6
+if test $ac_cv_lib_stdcpp_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSTDC__ 1
+_ACEOF
+
+  LIBS="-lstdc++ $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lcrypt" >&5
+echo $ECHO_N "checking for main in -lcrypt... $ECHO_C" >&6
+if test "${ac_cv_lib_crypt_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcrypt  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_crypt_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_crypt_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_crypt_main" >&5
+echo "${ECHO_T}$ac_cv_lib_crypt_main" >&6
+if test $ac_cv_lib_crypt_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBCRYPT 1
+_ACEOF
+
+  LIBS="-lcrypt $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+echo "$as_me:$LINENO: checking for textdomain" >&5
+echo $ECHO_N "checking for textdomain... $ECHO_C" >&6
+if test "${ac_cv_func_textdomain+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define textdomain to an innocuous variant, in case <limits.h> declares textdomain.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define textdomain innocuous_textdomain
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char textdomain (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef textdomain
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char textdomain ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_textdomain) || defined (__stub___textdomain)
+choke me
+#else
+char (*f) () = textdomain;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != textdomain;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_textdomain=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_textdomain=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_textdomain" >&5
+echo "${ECHO_T}$ac_cv_func_textdomain" >&6
+if test $ac_cv_func_textdomain = yes; then
+  cat >>confdefs.h <<\_ACEOF
+#define ENABLE_NLS  1
+_ACEOF
+
+fi
+
+
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  echo "$as_me:$LINENO: checking for unsigned char *" >&5
+echo $ECHO_N "checking for unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_type_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((unsigned char * *) 0)
+  return 0;
+if (sizeof (unsigned char *))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_unsigned_char_p=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_unsigned_char_p=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_type_unsigned_char_p" >&6
+
+echo "$as_me:$LINENO: checking size of unsigned char *" >&5
+echo $ECHO_N "checking size of unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_sizeof_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_unsigned_char_p" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_unsigned_char_p=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (unsigned char *)); }
+unsigned long ulongval () { return (long) (sizeof (unsigned char *)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (unsigned char *))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_unsigned_char_p=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_unsigned_char_p=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_sizeof_unsigned_char_p" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_UNSIGNED_CHAR_P $ac_cv_sizeof_unsigned_char_p
+_ACEOF
+
+
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      { echo "$as_me:$LINENO: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&5
+echo "$as_me: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&2;}
+      case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+    else
+      case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+    fi
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking whether GNU readline requested" >&5
+echo $ECHO_N "checking whether GNU readline requested... $ECHO_C" >&6
+
+# Check whether --with-gnu_readline or --without-gnu_readline was given.
+if test "${with_gnu_readline+set}" = set; then
+  withval="$with_gnu_readline"
+  if test "$withval" = yes; then
+      echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+      ac_with_gnu_readline=1
+    else
+      echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+    fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+  if test -n "$ac_with_gnu_readline"; then
+    if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking readline/readline.h usability" >&5
+echo $ECHO_N "checking readline/readline.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <readline/readline.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking readline/readline.h presence" >&5
+echo $ECHO_N "checking readline/readline.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <readline/readline.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: readline/readline.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: readline/readline.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_readline_readline_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+
+fi
+if test $ac_cv_header_readline_readline_h = yes; then
+  ac_mg_readline_header_found=1
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&2;}
+fi
+
+
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      echo "$as_me:$LINENO: checking for main in -lreadline" >&5
+echo $ECHO_N "checking for main in -lreadline... $ECHO_C" >&6
+if test "${ac_cv_lib_readline_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lreadline  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_readline_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_readline_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_readline_main" >&5
+echo "${ECHO_T}$ac_cv_lib_readline_main" >&6
+if test $ac_cv_lib_readline_main = yes; then
+  ac_mg_readline_lib_found=1
+         cat >>confdefs.h <<\_ACEOF
+#define WITH_GNU_READLINE 1
+_ACEOF
+
+         LIBS="$LIBS -lreadline"
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&2;}
+fi
+
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	echo "$as_me:$LINENO: checking for main in -lcurses" >&5
+echo $ECHO_N "checking for main in -lcurses... $ECHO_C" >&6
+if test "${ac_cv_lib_curses_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurses  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_curses_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_curses_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_curses_main" >&5
+echo "${ECHO_T}$ac_cv_lib_curses_main" >&6
+if test $ac_cv_lib_curses_main = yes; then
+  LIBS="$LIBS -lcurses"
+fi
+
+      fi
+    fi
+  fi
+
+
+# text for endianness
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+# Check whether --enable-override-longlong or --disable-override-longlong was given.
+if test "${enable_override_longlong+set}" = set; then
+  enableval="$enable_override_longlong"
+  cat >>confdefs.h <<\_ACEOF
+#define DISABLE_LONG_LONG 1
+_ACEOF
+
+fi;
+
+
+# ---------------------------------------------------------------------------
+
+                                                  ac_config_files="$ac_config_files Makefile text/Makefile lib/Makefile jni/Makefile java/org/greenstone/mgpp/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "text/Makefile" ) CONFIG_FILES="$CONFIG_FILES text/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "jni/Makefile" ) CONFIG_FILES="$CONFIG_FILES jni/Makefile" ;;
+  "java/org/greenstone/mgpp/Makefile" ) CONFIG_FILES="$CONFIG_FILES java/org/greenstone/mgpp/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@JNIINC@,$JNIINC,;t t
+s,@JNISUFFIX@,$JNISUFFIX,;t t
+s,@JNIFLAGS@,$JNIFLAGS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/indexers/mgpp/configure.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/configure.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/configure.in	(revision 21112)
@@ -0,0 +1,141 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(text/MGQuery.cpp)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mgpp
+VERSION=1.3
+
+LDFLAGS=
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lg:
+AC_CHECK_LIB(g, main)
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+dnl Replace `main' with a function in -lstdc++:
+AC_CHECK_LIB(stdc++, main)
+dnl Replace `main' with a function in -lcrypt:
+AC_CHECK_LIB(crypt, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+dnl see if gettext is in libc. (try the textdomain() function).
+AC_CHECK_FUNC(textdomain, AC_DEFINE(ENABLE_NLS ,1) )
+
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  AC_MSG_RESULT(no)
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+AC_SUBST(JNIINC)
+AC_SUBST(JNISUFFIX)
+AC_SUBST(JNIFLAGS)
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+fp_WITH_GNU_READLINE
+
+# text for endianness
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+AC_ARG_ENABLE(override-longlong,
+[  --enable-override-longlong
+                          overide use of long long with GCC],
+  AC_DEFINE(DISABLE_LONG_LONG))
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile text/Makefile lib/Makefile 
+   jni/Makefile java/org/greenstone/mgpp/Makefile], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/indexers/mgpp/install-sh
===================================================================
--- /main/tags/3.01/indexers/mgpp/install-sh	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.mgpp;
+
+/** DocInfo 'struct' class for a search result from mgpp
+ * used by MGPPQueryResult
+ *
+ * @See MGPPQueryResult
+ */
+public class MGPPDocInfo {
+
+    public long num_=0;
+    public float rank_=0;
+
+    public MGPPDocInfo(long doc, float rank) {
+	num_=doc;
+	rank_=rank;
+    }
+    public String toString(){
+	return ""+num_+"("+rank_+")";
+    }
+}
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java	(revision 21112)
@@ -0,0 +1,146 @@
+/*
+ *    MGPassesWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mgpp;
+
+
+/** java wrapper class for access to gs3_mgpp_passes in C
+ *
+ * the native side implemented in MGPPPassesWrapperImpl.c
+ */
+
+public class MGPPPassesWrapper
+{
+    static {
+	System.loadLibrary("mgpppassjni");
+	initIDs();
+    }
+    
+    //static final public char INVF_LEVEL_1 = '1';
+    //static final public char INVF_LEVEL_2 = '2';
+    //static final public char INVF_LEVEL_3 = '3';
+    
+    static final public int TEXT_PASS_1 = 0;
+    static final public int TEXT_PASS_2 = 1;
+    static final public int INDEX_PASS_1 = 2;
+    static final public int INDEX_PASS_2 = 3;
+    static final public int SPECIAL_PASS = 4;
+
+    //static final public int NO_STEM_OR_CASE = 0;
+    //static final public int CASE_ONLY = 1;
+    //static final public int STEM_ONLY = 2;
+    //static final public int STEM_AND_CASE = 3;
+
+    //static final public String STEMMER_ENGLISH = "english";
+    //static final public String STEMMER_FRENCH = "french";
+    //static final public String STEMMER_LOVIN = "lovin";
+    //static final public String STEMMER_SIMPLE_FRENCH = "simple-french";
+
+    //static final private char END_OF_DOCUMENT = (char) 2;
+
+    public MGPPPassesWrapper() {
+	initCSide();
+    }
+
+    /** initialise the pass through the documents */
+    public native boolean init();
+    
+    /** add a pass declaration */
+    public void addPass(int pass) {
+	switch (pass) {
+	case TEXT_PASS_1:
+	    addPass('T','1');
+	    break;
+	case TEXT_PASS_2:
+	    addPass('T','2');	    
+	    break;
+	case INDEX_PASS_1:
+	    addPass('I','1');
+	    break;
+	case INDEX_PASS_2:
+	    addPass('I','2');
+	    break;
+	case SPECIAL_PASS:
+	    addPass('S','1');
+	    break;
+	}
+    }
+    /** set the base path */
+    public native void setBasePath(String basepath);
+    /** set the file name */
+    public native void setFileName(String filename);
+
+    /** set the Document tag */
+    public native void setDocumentTag(String tag);
+
+    /** add a level tag */
+    public native void addLevelTag(String tag);
+
+    /** set the index level (default word level) */
+    public native void setIndexLevel(String tag);
+    
+    /** Maximum amount of memory to use for  the index  pass-2  file
+	inversion  in  megabytes.
+    */
+    public native void setInversionMemLimit(int limit);
+
+    /** process one or more MGPP documents */
+    public boolean processDocument(String doc_text) {
+	try {
+	    processMGPPDocument(doc_text.getBytes("UTF-8"));
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    return false;
+	}
+	return true;
+    }
+   
+    /** finalise the pass through the documents */
+    public native boolean finish();
+
+    /** get the exit value once finished */
+    public native int exitValue();
+
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises any C side stuff */
+    private native boolean initCSide();
+
+    private native void addPass(char pass_type, char pass_num);
+
+    /** process a MGPP document */
+    private native boolean processMGPPDocument(byte[] text);
+
+    public static void main(String []args) {
+
+	String doc = "<Document>hello there kath<Title>this is a title</Title></Document><Document>my name is helen</Document>";
+
+	MGPPPassesWrapper wrapper = new MGPPPassesWrapper();
+	wrapper.addPass(TEXT_PASS_1);
+	wrapper.addPass(TEXT_PASS_2);
+	wrapper.setDocumentTag("Document");
+	wrapper.setBasePath("/tmp/kjdon/index");
+	wrapper.setFileName("idx");
+
+	wrapper.init();
+	wrapper.processDocument(doc);
+	wrapper.finish();
+
+    }
+}
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java	(revision 21112)
@@ -0,0 +1,108 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** a java version of QueryResult for mgpp
+ *
+ * contains get methods for the java side, and simple
+ * set methods to be called from c++ side
+ *
+ */
+public class MGPPQueryResult {
+
+    /** the list of MGPPDocInfo */
+    protected Vector docs_=null;
+    /** the list of MGPPTermInfo */
+    protected Vector terms_=null;
+    /** the total number of docs found - not likely to be the size of docs_*/
+    protected long total_num_docs_=0;
+    /** whether a syntax error has occurred */
+    protected boolean syntax_error_ = false;
+    
+    MGPPQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+    }
+    /** clear the info from the last query - should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_=0;
+	docs_.clear();
+	terms_.clear();
+	syntax_error_ = false;
+    }
+    /** returns the result as a String - useful for printing out results */
+    public String toString() {
+	
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i=0; i<docs_.size(); i++) {
+	    result += ((MGPPDocInfo)docs_.elementAt(i)).toString()+", ";
+	}
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((MGPPTermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+
+	return result;
+    }
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((MGPPTermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+
+    // set methods used by c++ code
+    
+    public void setTotalDocs(long num) {
+	total_num_docs_=num;
+    }
+
+    public void addDoc(long doc, float rank) {
+	MGPPDocInfo doc_info = new MGPPDocInfo(doc, rank);
+	docs_.add(doc_info);
+    }
+
+    public void addTerm(String term, String tag, int stem, long match, long freq, String[] equivs) {
+	MGPPTermInfo ti = new MGPPTermInfo();
+	ti.term_=term;
+	ti.tag_=tag;
+	ti.stem_method_=stem;
+	ti.match_docs_=match;
+	ti.term_freq_=freq;
+	if (equivs!=null) {
+	  for (int i=0; i<equivs.length; i++) {
+	    ti.equiv_terms_.add(equivs[i]);
+	  }
+	}
+	terms_.add(ti);
+    }
+
+    // get methods for the java side
+    public Vector getDocs() {
+	return docs_;
+    }
+
+    public Vector getTerms() {
+	return terms_;
+    }
+
+    public long getTotalDocs() {
+	return total_num_docs_;
+    }
+     
+    public void setSyntaxError(boolean error) {
+	syntax_error_ = error;
+    }
+
+    public boolean hasSyntaxError() {
+	return syntax_error_;
+    }
+}
+
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java	(revision 21112)
@@ -0,0 +1,46 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** TermInfo 'struct' class for a search result from mgpp
+ * used by MGPPQueryResult
+ * 
+ *
+ *@see MGPPQueryResult
+ */
+public class MGPPTermInfo {
+
+    /** the term itself */
+    public String term_=null;
+    /** the tag - level or metadata - for which the query was done */
+    public String tag_=null;
+    /** what stem and casefold method was used
+	0 = none
+	1 = casefold only
+	2 = stem only
+	3 = casefold and stem */
+    public int stem_method_=0;
+    /** the number of documents containing this term (where document is defined by the tag data */
+    public long match_docs_=0;
+    /** overall term freq  - word level*/
+    public long term_freq_=0;
+    /** list of stemmed and casefolded equivalent terms - if stem_method_ is non-zero 
+      - Vector of Strings */
+    public Vector equiv_terms_=null;
+
+    public MGPPTermInfo() {
+	equiv_terms_ = new Vector();
+    }
+	
+    /** output the class as a string */
+    public String toString() {
+	String result="";
+	result +="<"+tag_+">\""+term_+"\"stem("+stem_method_+")docs("+match_docs_;
+	result +=")freq("+term_freq_+")equiv terms(";
+	for (int i=0; i<equiv_terms_.size(); i++) {
+	    result += (String)equiv_terms_.elementAt(i)+", ";
+	}
+	result += ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java	(revision 21112)
@@ -0,0 +1,102 @@
+
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** java wrapper class for access to mgpp in C++
+ *
+ * the native side implemented in MGPPWrapperImpl.cpp
+ * uses MGPPQueryResult to hold the result of a query. the 
+ * result of a getDocument is a String
+ * uses the jni
+ *
+ *@see MGPPQueryResult
+ */
+public class MGPPWrapper {
+  
+    /** the query result
+     * will be filled in by runQuery
+     */
+    protected MGPPQueryResult mgpp_query_result_=null; 
+    /** pointer to c++ MGPPWrapperData class - cached indexData and queryInfo
+     */
+    protected long mgpp_data_ptr_=0;
+
+    static {
+	System.loadLibrary("mgppjni");
+	initIDs();
+    }
+    
+    public MGPPWrapper() {
+	mgpp_query_result_ = new MGPPQueryResult();
+	initCppSide();
+    }
+
+    /** initialises field and method IDs for java side to enable access on
+	C++ side */
+    private static native void initIDs();
+
+    /** initialises the mgpp_data_ptr_ */
+    private native boolean initCppSide();
+
+    /** returns a document: number docnum at level level 
+     * text_path path should provide
+     * the absolute location of the mgpp text files eg ..../index/text/demo
+     * returns the doc in utf-8
+     */
+    public native String getDocument(String text_path,
+				     String level,
+				     long docnum );
+
+    /** the public method - converts between utf-8 and unicode
+     */
+    // public String getDocument(String text_path,
+    //		      String level,
+    //		      long docnum ) {
+
+    //String utf8_doc = getUtF8Document(base_dir, text_path, level, docnum);
+	
+    /** load up the data structure for index 
+     * - maintains state between requests as can be slow  
+     * index_path should provide
+     * the absolute location of the mgpp index files eg ..../index/tt/demo 
+     */
+    public native boolean loadIndexData(String index_path);
+    /** unloads the data */
+    public native boolean unloadIndexData();
+
+    // query param methods
+
+    /** if on=true, sets default stemming on  - its off by default*/
+    public native void setStem(boolean on);
+    /** if on=true, sets default casefolding on - its off by default */
+    public native void setCase(boolean on);
+    /** default is 50 */
+    public native void setMaxDocs(int num);
+    /** if on=true, sorts by rank, otherwise returns in build order - 
+	default is on */
+    public native void setSortByRank(boolean on);
+    /** if on=true, a query returns term freq info - default is on */
+    public native void setReturnTerms(boolean on);
+    /** sets the granularity of the  query  - default 'Document' */
+    public native void setQueryLevel(String level);
+    /** sets the granularity of the result - default 'Document' */
+    public native void setReturnLevel(String level);
+    /** sets the default boolean operator - AND(=1)/OR(=0) */
+    public native void setMatchMode(int mode);
+
+    /** returns a string with all the current query param settings */
+    public native String getQueryParams();
+
+    /** actually carry out the query. Use the set methods to set any query
+	params 
+    * writes the result to query_result
+    */
+    public native void runQuery(String query_string); 
+
+    /** get the result out of the wrapper */
+    public MGPPQueryResult getQueryResult() {
+	return mgpp_query_result_;
+    }
+    
+}
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Makefile.in	(revision 21112)
@@ -0,0 +1,63 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = java/org/greenstone/mgpp
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+JAVAC = $(JAVA_HOME)/bin/javac
+JAVAH = $(JAVA_HOME)/bin/javah
+JAR = $(JAVA_HOME)/bin/jar
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS =  -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+MGPPHOME=../../../..
+JAVACLASSDIR=./classes
+JAVASRCDIR=../../..
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) *.java
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGPPHOME)/jni org.greenstone.mgpp.MGPPWrapper
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGPPHOME)/jni org.greenstone.mgpp.MGPPPassesWrapper
+
+	$(JAR) cf $(MGPPHOME)/mgpp.jar -C $(JAVACLASSDIR) org
+
+clean:
+	rm -rf $(JAVACLASSDIR) $(MGPPHOME)/mgpp.jar $(MGPPHOME)/jni/org_greenstone_mgpp_MGPPWrapper.h $(MGPPHOME)/jni/org_greenstone_mgpp_MGPPPassesWrapper.h
+
+install:
+
+distclean: clean
+	rm -f Makefile
+
+
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Queryer.java
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Queryer.java	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/Queryer.java	(revision 21112)
@@ -0,0 +1,184 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+/** Queryer - java port of c++ Queryer 
+ * - uses MGPPWrapper to access mgpp
+ * only has get document and search - full text browse not implemented
+ *
+ * run as:
+ * java org.greenstone.mgpp.Queryer <basedir> <indexdir> <textdir>
+ *@see MGPPWrapper
+ */
+public class Queryer {
+  
+  /** outputs to std out a help message describing the commands used by 
+   * Queryer 
+   */
+  public void printHelp() {
+	
+	System.out.println( "commands available are:\n"+
+			    "\t.q\t\tquit\n"+
+			    "\t.h\t\tprint this help message\n"+
+			    "\t.d\t\tprint the current query parameter settings\n" +
+			    "\t.i<level>\tchange the search level to level\n"+
+			    "\t.l<level>\tchange the result level to level\n"+
+			    "\t.r0/.r1\t\tranking off/on\n"+
+			    "\t.t0/.t1\t\tquery type some/all\n"+
+			    "\t.c0/.c1\t\tcasefolding off/on\n"+
+			    "\t.s0/.s1\t\tstemming off/on\n"+
+			    "\t.o0/.o1\t\tshort output off/on\n"+
+			    "\t.m<num>\t\tset max docs to return to num\n\n"+
+			    "\t.p<docnum>\tprint document docnum\n"+
+			    "\t<query string>\t do a query\n");
+    }
+
+   
+    public static void main(String[] args) {
+	if (args.length != 3) {
+	    System.out.println("Usage: java org.greenstone.mgpp.Queryer <basedir> <indexdir> <textdir>");
+	    return;
+	}
+	
+	Queryer self = new Queryer();
+	
+	String base_dir = args[0];
+	String text_path = args[2];
+	String index_path = args[1];
+
+	text_path = base_dir + File.separatorChar+text_path;
+	index_path = base_dir  + File.separatorChar+ index_path;
+	
+	// the jni class to access mgpp stuff
+	MGPPWrapper wrapper = new MGPPWrapper();
+	wrapper.loadIndexData(index_path);
+	
+	// the return level
+	String level = "Section";
+
+	System.out.println("Welcome to Java Queryer :-)");
+	self.printHelp();
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	boolean shortOutput=false;
+
+	String command; // user input
+	char x; // the command letter
+	String data=null; // any auxiliary input to a command
+	while (true) {
+	  System.out.print(">"); // the prompt
+            try {
+                command = br.readLine();
+                command=command.trim();
+		if (command.startsWith(".")) {
+		    // a system command
+		    x = command.charAt(1);
+		    if (command.length() > 2) {
+			data = command.substring(2);
+			data = data.trim();
+			
+		    }
+		    switch (x) {
+		    case 'q': // clean up and exit
+			wrapper.unloadIndexData();
+			return;  
+		    case 'h': // print help message
+			self.printHelp();
+			break;
+		    case 'd': // print query param settings
+			String info = wrapper.getQueryParams();
+			System.out.println(info);
+			break;
+		    case 'p': // print doc
+			int docnum = Integer.parseInt(data);
+			String doc = wrapper.getDocument(text_path, 
+							 level, docnum);
+			System.out.println(doc);
+			break;
+		    case 'm': //match docs
+			int match = Integer.parseInt(data);
+			wrapper.setMaxDocs(match);
+			break;
+		    case 's': // set stem on/off
+			int stem = Integer.parseInt(data);
+			if (stem==0 ){
+			    wrapper.setStem(false);
+			} else if(stem==1) {
+			    wrapper.setStem(true);
+			} else {
+			    System.err.println("Error: stem should be 0 or 1");
+			}
+			break;
+		    case 'c': // set case on/off
+			int casef = Integer.parseInt(data);
+			if (casef==0) {
+			    wrapper.setCase(false);
+			} else if (casef==1) {
+			    wrapper.setCase(true);
+			} else {
+			    System.err.println("Error: case should be 0 or 1");
+			}
+			break;
+		    case 'i': // set search level
+			wrapper.setQueryLevel(data);
+			break;
+		    case 'l': // set result level
+			wrapper.setReturnLevel(data);
+			level = data;
+			break; 
+		    case 'r': // set rank on/off
+			int rank = Integer.parseInt(data);
+			if (rank==0) {
+			    wrapper.setSortByRank(false);
+			} else if (rank==1) {
+			    wrapper.setSortByRank(true);
+			} else {
+			    System.err.println("Error: rank should be 0 or 1");
+			}
+			break;
+		    case 't': // set query type some/all
+			int type = Integer.parseInt(data);
+			if (type==0 || type==1) {
+			    wrapper.setMatchMode(type);
+			} else {
+			    System.err.println("Error: type should be 0 (some) or 1 (all)");
+			}
+			break;
+		    case 'o': // set output short/long
+			int output = Integer.parseInt(data);
+			if (output==0) {
+			    shortOutput = false;
+			} else if (output==1) {
+			    shortOutput = true;
+			}else {
+			    System.err.println("Error: output should be 0 or 1");
+			}
+			break;
+		    }
+		}
+		else {
+		    // a query
+		    wrapper.runQuery(command);
+		    MGPPQueryResult res = wrapper.getQueryResult();
+		    if (res.hasSyntaxError()) {
+		      System.out.println("invalid syntax error\n");
+		    } else {
+		      if (shortOutput) {
+			System.out.println(res.toShortString());
+		      } else {
+			System.out.println(res.toString());
+		      }
+		    }
+		}
+		    
+	    } catch (Exception e) {
+		System.out.println("Queryer error: "+e.getClass() + " "+e.getMessage());
+	    }
+
+	}
+    }
+
+
+}
Index: /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/winMake.bat
===================================================================
--- /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/winMake.bat	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/java/org/greenstone/mgpp/winMake.bat	(revision 21112)
@@ -0,0 +1,45 @@
+@echo off
+
+set MGPPHOME=..\..\..\..
+
+rem ---- Where to put class files ----
+set JAVACLASSDIR=classes
+
+rem ---- Name and location of java programs ----
+set JAVAC="%JAVA_HOME%\bin\javac"
+set JAVAH="%JAVA_HOME%\bin\javah"
+set JAVA="%JAVA_HOME%\bin\java"
+set JAVADOC="%JAVA_HOME%\bin\javadoc"
+set JAR="%JAVA_HOME%\bin\jar"
+
+set JAVACOPTIONS= -deprecation -g -O
+
+
+if ""%1"" == """" goto compile
+if ""%1"" == ""compile"" goto compile
+if ""%1"" == ""install"" goto install
+if ""%1"" == ""clean"" goto clean
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:compile
+	if not exist %JAVACLASSDIR% mkdir %JAVACLASSDIR%
+	echo Compiling...
+        %JAVAC% -d %JAVACLASSDIR% %JAVACOPTIONS% *.java
+        %JAVAH% -classpath %JAVACLASSDIR% -d %MGPPHOME%\jni org.greenstone.mgpp.MGPPWrapper
+        %JAR% cf %MGPPHOME%\mgpp.jar -C %JAVACLASSDIR% org
+	goto done
+
+:install
+	goto done
+
+:clean
+	echo Cleaning up...
+	if exist %JAVACLASSDIR% rmdir /S /Q %JAVACLASSDIR%
+        if exist %MGPPHOME%\mgpp.jar del %MGPPHOME%\mgpp.jar
+        if exist %MGPPHOME%\jni\org_greenstone_mgpp_MGPPWrapper.h del %MGPPHOME%\jni\org_greenstone_mgpp_MGPPWrapper.h
+	goto done
+
+:done
Index: /main/tags/3.01/indexers/mgpp/jni/MGPPPassesWrapperImpl.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/MGPPPassesWrapperImpl.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/MGPPPassesWrapperImpl.cpp	(revision 21112)
@@ -0,0 +1,190 @@
+/*
+ *    MGPassesWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <jni.h>
+#include <assert.h>
+#include "sysfuncs.h"
+#include "org_greenstone_mgpp_MGPPPassesWrapper.h"
+
+#include "mgpp_passes_4jni.h"
+#include "mg_files.h"
+
+/* if we need to use java objects, we should initialise their field ids here*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  return;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  clear_variables();
+  return true;  
+}  
+
+/* add a pass type T1, T2, I1, I2, S */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_addPass(JNIEnv *j_env, 
+					       jobject j_obj, 
+					       jchar j_pass_type,
+					       jchar j_pass_num)
+{
+  /* get the level as a c char */
+  const char pass_type = j_pass_type;
+  const char pass_num = j_pass_num;
+  add_pass(pass_type, pass_num);
+  
+}
+
+/* Set the filename */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setFileName(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_filename)
+{
+  /* Get the filename as a C string */
+  const char* filename = j_env->GetStringUTFChars(j_filename, NULL);
+ 
+  assert(filename != NULL);
+  set_filename(filename);
+  
+  /* Release the string */
+ j_env->ReleaseStringUTFChars(j_filename, filename);
+
+}
+
+/* Set the base path */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setBasePath(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_basepath)
+{
+  /* Get the base_path as a C string */
+  const char* basepath = j_env->GetStringUTFChars(j_basepath, NULL);
+  assert(basepath != NULL);
+ 
+  set_basepath(basepath);
+  
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_basepath, basepath);
+
+}
+
+/* set the Document tag */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setDocumentTag(JNIEnv *j_env, 
+						      jobject j_obj,
+						      jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+
+  set_document_tag(tag);
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+
+}
+
+/* add a level tag */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_addLevelTag(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+  add_level_tag(tag);
+
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+}
+
+/* set the index level (default word level) */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setIndexLevel(JNIEnv *j_env, 
+						     jobject j_obj,
+						     jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+  set_index_level(tag);
+
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+  
+}
+
+
+/** Maximum amount of memory to use for  the index  pass-2  file
+    inversion  in  megabytes.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setInversionMemLimit(JNIEnv *j_env, 
+							    jobject j_obj, 
+							    jint j_limit) {
+  int limit = j_limit;
+  set_inversion_limit(limit);
+}
+
+
+/* initialise the pass through the documents. must be called after all
+   the set methods 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_init(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  init_driver();
+  return true;
+}
+
+
+/* process one or more documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_processMGPPDocument(JNIEnv *j_env, 
+						       jobject j_obj,
+						       jbyteArray j_doc_text) {
+  /* Get the text as a C string */
+  int length = j_env->GetArrayLength(j_doc_text);
+  u_char * text_buffer = (u_char *)j_env->GetByteArrayElements(j_doc_text, NULL);
+  process_document(text_buffer, length);
+  /* Release the string */
+  // why doesn't this work in c++ when it works in C?????
+  //j_env->ReleaseByteArrayElements(j_doc_text, text_buffer, 0);
+  return true;
+}
+
+/* finalise the pass through the documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_finish(JNIEnv *j_env, 
+					    jobject j_obj) {
+  
+  finalise_driver();
+  return true;
+}
+
+/** get the exit value once finished */
+JNIEXPORT jint JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_exitValue(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  return get_exit_value();
+}
Index: /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.cpp	(revision 21112)
@@ -0,0 +1,503 @@
+#ifdef __WIN32__
+#include <WIN32cfg.h>
+#include <strstream>
+#include <sstream>
+#else
+#ifdef __APPLE__
+#include <strstream>
+#include <sstream>
+#else
+#include <sstream>
+#endif
+#endif
+
+#include <jni.h>
+#include "org_greenstone_mgpp_MGPPWrapper.h"
+#include "MGPPWrapperImpl.h"
+#include "TextGet.h"
+#include "GSDLQueryParser.h"
+#include "MGQuery.h"
+  
+MGPPWrapperData::MGPPWrapperData() {
+  indexData = new IndexData();
+  queryInfo = new QueryInfo();
+
+  if (queryInfo==NULL) {
+    cerr<<"couldn't allocate new query info\n";
+    if (indexData!=NULL) {
+      delete indexData;
+    }
+  }
+
+  // set all the default params
+  SetCStr(queryInfo->docLevel, "Document"); // the level to search at
+  queryInfo->maxDocs = 50;
+  queryInfo->sortByRank = true;
+  queryInfo->exactWeights = false;
+  queryInfo->needRankInfo = true;
+  queryInfo->needTermFreqs = true;
+
+  UCArrayClear(level);
+  SetCStr(level, "Document"); // the level to return docs at
+  defaultStemMethod=0;
+  defaultBoolCombine=0;
+  
+}
+
+MGPPWrapperData::~MGPPWrapperData() { 
+  if (indexData !=NULL) {
+    delete indexData;
+  }
+  if (queryInfo !=NULL) {
+    delete queryInfo;
+  }
+}
+
+// ********************************************
+// initialisation stuff
+// ********************************************
+
+// cached ids for java stuff 
+jfieldID FID_mgpp_data = NULL; // MGPPWrapperData
+jfieldID FID_query_result = NULL; // MGPPQueryResult
+jmethodID MID_addDoc=NULL; // MGPPQueryResult.addDoc()
+jmethodID MID_addTerm=NULL; // MGPPQueryResult.addTerm()
+jmethodID MID_setTotalDocs=NULL; // MGPPQueryResult.setTotalDocs()
+jmethodID MID_clearResult=NULL; //MGPPQueryResult.clear()
+jmethodID MID_setSyntaxError=NULL; // MGPPQueryResult.setSyntaxError()
+jclass CID_String=NULL; // class ID of String
+
+/* to access objects and methods on java side, need their field/method ids - 
+ this initialises them at the start to avoid recalculating them each time they
+ are needed 
+Note: the descriptors need to be exactly right, otherwise you get an error 
+saying "no such field" but no reference to the fact that it has the right 
+name but the wrong type.
+Note: apparently the jclass is a local ref and should only work 
+in the method that created it. It seems to work ok, but I'll make it
+ global cos the book said I should, and it may avoid future hassles.
+*/
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_initIDs (JNIEnv *j_env, jclass j_cls) {
+  
+  FID_mgpp_data = j_env->GetFieldID(j_cls, "mgpp_data_ptr_", "J"); //a long-"J"
+  if (FID_mgpp_data==NULL) {
+      cerr <<"MGPP JNI: field mgpp_data_ptr_ not found"<<endl;
+  } 
+
+  FID_query_result = j_env->GetFieldID(j_cls, "mgpp_query_result_", "Lorg/greenstone/mgpp/MGPPQueryResult;"); // an object -"L<class name>;"
+  if (FID_query_result==NULL) {
+      cerr <<"MGPP JNI: field mgpp_query_result_ not found"<<endl;
+  }
+  // the methods we want to use
+
+  // addDoc(long doc, float rank)
+  jclass JC_MGPPQueryResult = j_env->FindClass("org/greenstone/mgpp/MGPPQueryResult");
+  MID_addDoc = j_env->GetMethodID(JC_MGPPQueryResult, "addDoc", "(JF)V");
+  if (MID_addDoc==NULL) {
+      cerr <<"MGPP JNI: addDoc method not found"<<endl;
+  }
+  // addTerm(String term, String tag, int stem_method, long match_docs, 
+  // long term_freq, String[] equiv_terms)
+  MID_addTerm = j_env->GetMethodID(JC_MGPPQueryResult, "addTerm", "(Ljava/lang/String;Ljava/lang/String;IJJ[Ljava/lang/String;)V");
+  if (MID_addTerm==NULL) {
+      cerr <<"MGPP JNI: method addTerm not found"<<endl;
+  }
+  
+  // setTotalDocs(long)
+  MID_setTotalDocs = j_env->GetMethodID(JC_MGPPQueryResult, "setTotalDocs", "(J)V");
+  if (MID_setTotalDocs==NULL) {
+      cerr <<"MGPP JNI: method setTotalDocs not found"<<endl;
+  }
+  
+  MID_clearResult = j_env->GetMethodID(JC_MGPPQueryResult, "clear", "()V");
+  if (MID_clearResult==NULL) {
+      cerr <<"MGPP JNI: method clear not found"<<endl;
+  }
+  MID_setSyntaxError = j_env->GetMethodID(JC_MGPPQueryResult, "setSyntaxError", "(Z)V");
+  if (MID_clearResult==NULL) {
+      cerr <<"MGPP JNI: method setSyntaxError not found"<<endl;
+  }
+
+  // get the class for String to use in NewObjectArray in runQuery()
+  // FindClass returns a local reference - have to convert it to a global one
+  jclass local_CID_String = j_env->FindClass("java/lang/String");
+  if (local_CID_String==NULL) {
+    cerr <<"MGPP JNI: java String class not found"<<endl;
+  } else { 
+    /* create a global ref */
+    CID_String = (jclass)j_env->NewGlobalRef(local_CID_String);
+    /* The local reference is no longer useful */
+    j_env->DeleteLocalRef(local_CID_String);
+ 
+    /* Is the global reference created successfully? */
+    if (CID_String == NULL) {
+      return; /* out of memory exception thrown */
+    }
+  }
+  
+}
+
+/* the java side MGPPWrapper has a pointer to a C++ object - MGPPWrapperData
+   initialise this and set the pointer 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_initCppSide (JNIEnv *j_env, jobject j_obj){
+
+  MGPPWrapperData * data = new MGPPWrapperData();
+  j_env->SetIntField(j_obj, FID_mgpp_data, (long)data);
+
+  return true;
+
+}
+
+//****************************************************
+// retrieve a document
+//****************************************************
+
+/* returns a document from mgpp as a string
+Note: TextData isn't cached - just reloaded each time
+*/
+JNIEXPORT jstring JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_getDocument (JNIEnv *j_env, 
+ jobject j_obj, jstring j_text_name, jstring j_level, jlong j_docnum) {
+  
+#ifdef __WIN32__
+  const char* base_dir = "";
+#else
+  const char* base_dir = "/";
+#endif
+
+  const char * text_name = j_env->GetStringUTFChars(j_text_name, NULL);
+  if (text_name==NULL) {
+   return NULL;
+  }
+
+  const char * level = j_env->GetStringUTFChars( j_level, NULL);
+  if (level==NULL) {
+   j_env->ReleaseStringUTFChars(j_text_name, text_name);
+   return NULL;
+  }
+    
+  // does this work alright? j_docnum is a long (64 bit)
+  unsigned long docnum = j_docnum;
+  TextData td;
+
+  // cast to char* otherwise complains about const
+  td.LoadData((char *)base_dir, (char *)text_name);
+
+  UCArray mg_level;
+  SetCStr(mg_level, level);
+  UCArray docText;
+  docText.clear();
+  // get the actual text
+  if (!GetDocText(td, mg_level, docnum, docText)) {
+    cerr <<"MGPP JNI: couldn't retrieve doc text"<<endl;
+  }
+
+  td.UnloadData();
+  
+  char * doc = GetCStr(docText); // do I need to free this char *??
+  jstring result = j_env->NewStringUTF(doc);
+  // release any gets
+  j_env->ReleaseStringUTFChars(j_text_name, text_name);
+  j_env->ReleaseStringUTFChars(j_level, level);
+
+  // free any C++ stuff
+  delete doc;
+
+  return result;
+ 
+}
+
+//******************************************
+// do a query
+// ****************************************
+
+/* load the IndexData - cached for querying
+ */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_loadIndexData (JNIEnv *j_env, jobject j_obj,  jstring j_index_name) {
+
+  jint data_ptr = j_env->GetIntField(j_obj, FID_mgpp_data);
+  MGPPWrapperData * data = (MGPPWrapperData *)data_ptr;
+
+#ifdef __WIN32__
+  const char* base_dir = "";
+#else
+  const char* base_dir = "/";
+#endif
+
+  const char * index_name = j_env->GetStringUTFChars( j_index_name, NULL);
+  if (index_name==NULL) {
+    return false;
+  }
+  
+  jboolean j_result=false;
+  
+  // why doesn't this complain about const??
+  if (data->indexData->LoadData(base_dir, index_name)) {
+    j_result=true;
+  }
+  
+  // release any gets
+  j_env->ReleaseStringUTFChars(j_index_name, index_name);
+  
+  return j_result;
+}
+
+/* unload the data 
+ */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_unloadIndexData (JNIEnv *j_env, jobject j_obj) {
+
+  jint data_ptr = j_env->GetIntField(j_obj, FID_mgpp_data);
+  MGPPWrapperData * data = (MGPPWrapperData *)data_ptr;
+
+  data->indexData->UnloadData();
+  return true;
+
+}
+
+/* do the actual query - the results are written to query_result held on the 
+   java side */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_runQuery (JNIEnv *j_env, jobject j_obj, jstring j_query){
+  
+  jthrowable exc; // an exception - check if something funny has happened
+  const char *query = j_env->GetStringUTFChars(j_query, NULL);
+  if (query==NULL) {
+    return; // exception already thrown
+  }
+  // turn to UCArray for mgpp and then release the string
+  UCArray queryArray;
+  SetCStr(queryArray, query);
+  j_env->ReleaseStringUTFChars(j_query, query);
+  
+  // the query data
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  // the result to write to
+  jobject result_ptr = j_env->GetObjectField(j_obj, FID_query_result);
+  if (result_ptr==NULL) {
+    cerr <<"couldn't access the result to write to"<<endl;
+    return;
+  }
+  
+  // clear the result
+  j_env->CallVoidMethod(result_ptr, MID_clearResult);
+  exc = j_env->ExceptionOccurred(); // this catches teh exception I think - it
+  //wont be thrown any further
+  if (exc) {
+    j_env->ExceptionDescribe();
+    return;
+  }
+  // the mgpp QueryResult that we will use
+  ExtQueryResult queryResult;
+
+  QueryNode * queryTree = NULL;
+  // parse the query string into a tree structure
+  queryTree = ParseQuery(queryArray, data->defaultBoolCombine, 
+			 data->defaultStemMethod);
+  if (queryTree == NULL) {
+    // invalid syntax
+    j_env->CallVoidMethod(result_ptr, MID_setSyntaxError, true);
+    cerr << "MGPPWrapperImpl: invalid query syntax!!\n";
+    return; 
+  }
+  // print the query
+  PrintNode (cout, queryTree);
+  // finally, do the query
+  MGQuery(*(data->indexData), *(data->queryInfo), queryTree, queryResult, data->level);
+
+  delete queryTree;
+
+  // convert queryResult to the java side version
+  // use levels rather than docs of ExtQueryResult
+  // CallVoidMethod(obj, method id, args to method)
+  for (int i=0; i<queryResult.levels.size(); i++) {
+    jlong doc = queryResult.levels[i];
+    jfloat rank = queryResult.ranks[i];
+    j_env->CallVoidMethod(result_ptr, MID_addDoc, doc, rank);
+    exc = j_env->ExceptionOccurred();
+    if (exc) {
+      j_env->ExceptionDescribe();
+      return;
+    }
+    
+  }
+
+  // actual num of docs
+  jlong total = queryResult.actualNumDocs;
+  j_env->CallVoidMethod(result_ptr, MID_setTotalDocs, total);
+  exc = j_env->ExceptionOccurred();
+  if (exc) {
+    j_env->ExceptionDescribe();
+    return;
+  }
+
+  // the terms
+  for (int j=0; j<queryResult.termFreqs.size(); j++) {
+    
+    TermFreqData tf = queryResult.termFreqs[j];
+    jstring term = j_env->NewStringUTF(GetCStr(tf.term));
+    jstring tag = j_env->NewStringUTF(GetCStr(tf.tag));
+    jint stem = tf.stemMethod;
+    jlong match = tf.matchDocs;
+    jlong freq = tf.termFreq;
+    
+    jobjectArray equivs=NULL;
+    jstring empty = j_env->NewStringUTF(""); // the initial object to fill the array
+    jint num_equivs = tf.equivTerms.size();
+    equivs = j_env->NewObjectArray(num_equivs, CID_String, empty);
+    if (equivs==NULL) {
+      cerr<<"couldn't create object array"<<endl;
+      
+    } else {
+      for (int k=0; k<num_equivs;k++) {
+	jstring equiv = j_env->NewStringUTF(GetCStr(tf.equivTerms[k]));
+	j_env->SetObjectArrayElement(equivs, k, equiv);
+      }
+      
+      
+      j_env->CallVoidMethod(result_ptr, MID_addTerm, term, tag, stem, match, freq, equivs);
+      exc = j_env->ExceptionOccurred();
+      if (exc) {
+	j_env->ExceptionDescribe();
+	return;
+      }
+    }
+  }
+
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setStem (JNIEnv *j_env, 
+					      jobject j_obj, 
+					      jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  if (j_on) {
+    data->defaultStemMethod |= 2;
+  } else {
+    data->defaultStemMethod &= 0xd;
+  }
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setCase (JNIEnv *j_env, 
+					      jobject j_obj, 
+					      jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 1;
+  } else {
+    data->defaultStemMethod &= 0xe;
+  }
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setMaxDocs (JNIEnv *j_env, 
+						 jobject j_obj, 
+						 jint j_max) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->queryInfo->maxDocs=j_max;
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setSortByRank (JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  data->queryInfo->sortByRank=j_on;
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setReturnTerms(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->queryInfo->needTermFreqs = j_on;
+ 
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setQueryLevel(JNIEnv *j_env, 
+						   jobject j_obj, 
+						   jstring j_level){
+  
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  const char * level = j_env->GetStringUTFChars(j_level, NULL);
+  if (level==NULL) {
+    return; // exception already thrown
+  }
+  
+  data->queryInfo->docLevel.clear();
+  SetCStr(data->queryInfo->docLevel, level);
+  
+  // release the java stuff
+  j_env->ReleaseStringUTFChars(j_level, level);
+ 
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setReturnLevel(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jstring j_level){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  const char * level = j_env->GetStringUTFChars(j_level, NULL);
+  if (level==NULL) {
+    return; // exception already thrown
+  }
+  
+  data->level.clear();
+  SetCStr(data->level, level);
+  
+  // release the java stuff
+  j_env->ReleaseStringUTFChars(j_level, level);
+    
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setMatchMode (JNIEnv *j_env,
+						   jobject j_obj,
+						   jint j_mode){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->defaultBoolCombine=j_mode;
+
+}
+
+JNIEXPORT jstring JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_getQueryParams (JNIEnv *j_env,
+						     jobject j_obj){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  // print the data to a stringstream, then convert to char*, then to 
+  //jstring
+
+  stringstream output;
+  output << "Query params:"<<endl
+    // need to change this to use platform specific separator for niceness
+	 << "index\t\t"<<data->indexData->basePath<<"/"<<data->indexData->filename<<endl
+	 <<"search level\t"<<GetCStr(data->queryInfo->docLevel)<<endl
+	 <<"result level\t"<<GetCStr(data->level)<<endl
+	 <<"casefold\t"<<(data->defaultStemMethod&1)<<endl
+	 <<"stem\t\t"<<(data->defaultStemMethod&2)<<endl
+	 <<"order by rank\t"<<data->queryInfo->sortByRank<<endl
+	 <<"query type\t"<<(data->defaultBoolCombine==1?"all":"some")<<endl
+         <<"max docs\t"<<data->queryInfo->maxDocs<<endl<<ends;
+
+  const char *result = output.str().c_str();
+  jstring j_result = j_env->NewStringUTF(result);
+  delete (char *)result;
+  return j_result;
+}
Index: /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/MGPPWrapperImpl.h	(revision 21112)
@@ -0,0 +1,15 @@
+#include "MGQuery.h"
+
+// the global stuff we need to do a query
+class MGPPWrapperData {
+ public:
+  IndexData * indexData;
+  QueryInfo * queryInfo;
+  int defaultStemMethod;
+  int defaultBoolCombine;
+  UCArray level; // the return level
+
+  MGPPWrapperData();
+  ~MGPPWrapperData();
+
+};
Index: /main/tags/3.01/indexers/mgpp/jni/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,69 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = jni
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#NOTE: libmgpptext.a needs to come before libmgpplib.a !!!
+QUERY_LIBS = ../text/libmgpptext.a ../lib/libmgpplib.a -lstdc++ @LIBS@
+PASSES_LIBS = ../text/libmgpppass.a ../lib/libmgpplib.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+JNIFLAGS=@JNIFLAGS@
+JNISUFFIX=@JNISUFFIX@
+JNIINC=@JNIINC@
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+CXXCOMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS)
+
+.SUFFIXES:
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) $<
+
+# can we get rid of JAVA_HOME from here??
+INCLUDES = -I.. -I../lib -I../text -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JNIINC)
+
+all: compile link
+
+compile:
+	$(CXXCOMPILE) MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+link:
+	$(CXX) $(LDFLAGS) $(JNIFLAGS) -o libmgppjni.$(JNISUFFIX) MGPPWrapperImpl.o $(QUERY_LIBS)
+	$(CXX) $(LDFLAGS) $(JNIFLAGS) -o libmgpppassjni.$(JNISUFFIX) MGPPPassesWrapperImpl.o $(PASSES_LIBS)
+
+clean:
+	rm -rf *.o libmgppjni.$(JNISUFFIX) libmgpppassjni.$(JNISUFFIX)
+
+distclean: clean
+	rm -f Makefile
+
+install:
+	../mkinstalldirs $(libdir); \
+	cp libmgppjni.$(JNISUFFIX)  libmgpppassjni.$(JNISUFFIX)  $(libdir)/;
+
+
+
Index: /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h	(revision 21112)
@@ -0,0 +1,127 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mgpp_MGPPPassesWrapper */
+
+#ifndef _Included_org_greenstone_mgpp_MGPPPassesWrapper
+#define _Included_org_greenstone_mgpp_MGPPPassesWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+#undef org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_1
+#define org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_1 0L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_2
+#define org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_2 1L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_1
+#define org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_1 2L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_2
+#define org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_2 3L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_SPECIAL_PASS
+#define org_greenstone_mgpp_MGPPPassesWrapper_SPECIAL_PASS 4L
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    init
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_init
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setBasePath
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setBasePath
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setFileName
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setFileName
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setDocumentTag
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setDocumentTag
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    addLevelTag
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_addLevelTag
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setIndexLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setIndexLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setInversionMemLimit
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setInversionMemLimit
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    finish
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_finish
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    exitValue
+ * Signature: ()I
+ */
+JNIEXPORT jint JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_exitValue
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    addPass
+ * Signature: (CC)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_addPass
+  (JNIEnv *, jobject, jchar, jchar);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    processMGPPDocument
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_processMGPPDocument
+  (JNIEnv *, jobject, jbyteArray);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h	(revision 21112)
@@ -0,0 +1,133 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mgpp_MGPPWrapper */
+
+#ifndef _Included_org_greenstone_mgpp_MGPPWrapper
+#define _Included_org_greenstone_mgpp_MGPPWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    initCppSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_initCppSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    getDocument
+ * Signature: (Ljava/lang/String;Ljava/lang/String;J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mgpp_MGPPWrapper_getDocument
+  (JNIEnv *, jobject, jstring, jstring, jlong);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    loadIndexData
+ * Signature: (Ljava/lang/String;)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_loadIndexData
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    unloadIndexData
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_unloadIndexData
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setStem
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setStem
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setCase
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setCase
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setMaxDocs
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setMaxDocs
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setSortByRank
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setSortByRank
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setReturnTerms
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setReturnTerms
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setQueryLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setQueryLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setReturnLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setReturnLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setMatchMode
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setMatchMode
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    getQueryParams
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mgpp_MGPPWrapper_getQueryParams
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    runQuery
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_runQuery
+  (JNIEnv *, jobject, jstring);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/indexers/mgpp/jni/win32.mak
===================================================================
--- /main/tags/3.01/indexers/mgpp/jni/win32.mak	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/jni/win32.mak	(revision 21112)
@@ -0,0 +1,75 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\jni
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGPPHOME = ..
+GSDL3HOME = ..\..\..
+AR = lib
+CC = cl
+CPPFLAGS = -GX
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -DSHORT_SUFFIX -DHAVE_STRING_H -D__WIN32__
+INCLUDES = -I"$(MGPPHOME)" -I"$(MGPPHOME)\lib" -I"$(MGPPHOME)\text" \
+           -I"C:\j2sdk1~1.1_0\include" -I"C:\j2sdk1~1.1_0\include\win32"           
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES) 
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    MGPPWrapperImpl.h   org_greenstone_mgpp_MGPPWrapper.h org_greenstone_mgpp_MGMGPPPassesWrapperImpl.h
+
+SOURCES = \
+    MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+
+OBJECTS = \
+    MGPPWrapperImpl$o  $(MGPPHOME)\text\libtextin.lib \
+    $(MGPPHOME)\lib\libmgpp.lib
+PASSES_OBJECTS = \
+    MGPPPassesWrapperImpl$o $(MGPPHOME)\text\libmgpppass.lib \
+    $(MGPPHOME)\lib\libmgpp.lib
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : compile link
+
+compile:
+        $(COMPILE) MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+
+link:
+        $(CC) -MD -LD $(OBJECTS) -Femgppjni.dll
+        $(CC) -MD -LD $(PASSES_OBJECTS) -Femgpppassjni.dll
+
+install: 
+	copy mgppjni.dll $(GSDL3HOME)\lib\jni
+	copy mgpppassjni.dll $(GSDL3HOME)\lib\jni
+clean:
+	del *$o mgppjni.dll mgppjni.exp mgppjni.lib mgpppassjni.dll mgpppassjni.exp mgpppassjni.lib 
Index: /main/tags/3.01/indexers/mgpp/lib/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,156 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CXX = @CXX@
+ifneq ($(NZDL), 1)
+CFLAGS = @CFLAGS@ -ansi
+else 
+CFLAGS = @CFLAGS@ -ansi -DSILENT
+endif
+CXXFLAGS = @CXXFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+libdir = @libdir@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)  $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) -D__cplusplus $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+INCLUDES = -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h      \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h
+
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp
+# memlib.cpp removed
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+
+all : libmgpplib.a
+
+libmgpplib.a : $(OBJECTS)
+	rm -f libmgpplib.a
+	$(AR) cru libmgpplib.a $(OBJECTS)
+	$(RANLIB) libmgpplib.a
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmgpplib.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# breaks make distclean (jrm21)
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/indexers/mgpp/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,96 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+MGPPHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -DSHORT_SUFFIX
+INCLUDES = -I"." -I"$(MGPPHOME)"
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h      \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h         getopt_old.h       win32in.h
+
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp                getopt_old.c      win32in.c
+# memlib.cpp removed
+
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o         getopt_old$o      win32in$o
+
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmgpp.lib
+
+libmgpp.lib : $(OBJECTS)
+	if EXIST libmgpp.lib del libmgpp.lib
+	$(AR) /out:libmgpp.lib $(OBJECTS)
+
+$(OBJECTS): "$(MGPPHOME)\win32cfg.h"
+
+install: 
+clean:
+	del *$o 
+	del libmgpp.lib
Index: /main/tags/3.01/indexers/mgpp/lib/alloca.c
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_gen.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_gen.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_gen.cpp	(revision 21112)
@@ -0,0 +1,134 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::unary_length(val);
+  //  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::binary_length (val, b);
+  return (num);
+}
+
+
+unsigned long BIO_Gamma_Length (unsigned long val) {
+  register unsigned long num;
+  num = bitio_buffer::gamma_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::delta_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  num = bitio_buffer::elias_length(val, b, s);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::bblock_length (val, b);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 ((N - p) / p);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  clogb = bitio_buffer::ceil_log2 (b);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 (b);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::ceil_log2 (b);
+  return logb;
+}
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+
+
+
+
+
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_abstract.h	(revision 21112)
@@ -0,0 +1,340 @@
+#ifndef H_BITIO_ABSTRACT
+#define H_BITIO_ABSTRACT
+
+class bitio_buffer {
+public:
+  virtual int bit() = 0;
+  virtual void encodeBit(int bit) = 0;
+  virtual void seek(unsigned long topos) = 0;
+  
+  static inline void check_positive(char *type, unsigned long x) {
+    if (x <= 0) {
+      fprintf(stderr, "Error: Cannot %s encode %lu\n", type, x);
+      exit(1);
+    }
+  }
+
+  static inline int floor_log2(unsigned long x) {
+    unsigned long  _B_x = x;
+    int reply;
+    
+    reply = -1;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+  
+  static inline int ceil_log2(unsigned long x) {
+    register int _B_x;
+    unsigned long reply;
+    
+    _B_x = x - 1;
+    reply = 0;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+
+  inline unsigned long unary_decode(unsigned long *count) {
+    unsigned long x;
+    
+    x = 1;
+    while (!this->bit()) ++x;
+    if (count != NULL) {
+      *count += x;
+    }
+    return x;
+  }
+
+  inline void unary_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_x = x;
+    check_positive("unary", _B_x);
+    if (count != NULL) {
+      *count += _B_x;
+    }
+    while(--_B_x) 
+      encodeBit(0);
+    encodeBit(1);
+  }
+
+
+  inline unsigned long binary_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = 0;
+    register unsigned long _B_b = b;
+    register int _B_i, _B_logofb, _B_thresh;
+    register unsigned long reply;
+    
+    if (_B_b != 1) {					
+      _B_logofb = ceil_log2(_B_b);
+      _B_thresh = (1<<_B_logofb) - _B_b;
+      --_B_logofb;
+      
+      if (count != NULL) {
+	*count += _B_logofb;
+      }
+      
+      for (_B_i=0; _B_i < _B_logofb; ++_B_i) {
+	_B_x += _B_x + this->bit();
+      }
+
+      if ((int)_B_x >= _B_thresh) {
+	_B_x += _B_x + this->bit();
+	_B_x -= _B_thresh;
+	if (count != NULL) {
+	  *count = *count + 1;
+	}
+      }
+      reply = _B_x+1;
+      
+    } else reply = 1;
+    
+    return reply;
+  }
+
+  inline void binary_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else {
+      _B_nbits = _B_logofb;
+      _B_x += _B_thresh;
+    }
+    if (count != NULL) {
+      *count += _B_nbits;
+    }
+    while (--_B_nbits>=0)
+      encodeBit((_B_x>>_B_nbits) & 0x1);
+  }
+
+  inline unsigned long bblock_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x1, _B_xx = 0;
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb;
+    
+    _B_xdivb = this->unary_decode(count);
+    --_B_xdivb;				       
+    while (_B_xdivb--) {
+      _B_xx += _B_bb;
+    }
+    _B_x1 = this->binary_decode(_B_bb, count);
+    return (_B_xx + _B_x1);
+  }
+  
+  inline void bblock_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    unary_encode(_B_xdivb+1, count);
+    binary_encode(_B_xx+1, _B_bb, count);
+  }
+
+  inline unsigned long elias_decode(unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k;
+    register double _B_pow=1.0;
+    
+    _B_k = this->unary_decode(count);
+    --_B_k;
+    while (_B_k--) {
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    _B_xx = this->binary_decode(_B_upper-_B_lower+1, count);
+    --_B_xx;
+    check_positive("gamma", _B_xx+_B_lower);
+    return _B_xx+_B_lower;
+  }
+  
+  inline void elias_encode(unsigned long x, unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0;
+    register double _B_pow=1.0;
+    
+    check_positive("elias", _B_xx);
+    
+    while ((int)_B_xx>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    unary_encode(_B_k+1, count);
+    binary_encode(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);
+  }
+  
+  inline unsigned long gamma_decode(unsigned long *count) {
+    register unsigned long _B_xx = 1;
+    register int _B_nbits = 0;
+    
+    while(!this->bit()) ++_B_nbits;
+    if (count != NULL) {
+      *count += _B_nbits*2+1;
+    }
+    while (_B_nbits-- > 0)
+      _B_xx += _B_xx + this->bit();
+    return _B_xx;
+  }
+
+  inline void gamma_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    register int _B_nbits;
+    
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    _B_nbits = _B_logofb+1;
+    
+    if (count != NULL) { 
+      *count += _B_logofb*2+1;
+    }
+    while(_B_logofb--) encodeBit(0);
+    while (--_B_nbits>=0) encodeBit((_B_xx>>_B_nbits) & 0x1);
+  }
+  
+  inline unsigned long delta_decode(unsigned long *count) {
+    register unsigned long _B_xxx;
+    register int _B_logx;
+    _B_logx = gamma_decode(count);
+    --_B_logx;
+    _B_xxx = binary_decode(1<<_B_logx, count); 
+    --_B_xxx;
+    return (_B_xxx + (1<<_B_logx));
+  }
+
+  inline void delta_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx;
+    _B_logx = floor_log2(_B_xxx);
+    gamma_encode(_B_logx+1, count);
+    binary_encode(_B_xxx+1, 1<<_B_logx, count);
+  }
+
+  inline unsigned long huff_decode(unsigned long *mcodes, unsigned long **values,
+				   unsigned long *count = NULL) {
+    register unsigned long *__min_code = mcodes;
+    register unsigned long *__mclen    = mcodes;
+    register unsigned long **__values  = values;
+    register unsigned long __code = 0;
+    do {
+      __code += __code + bit();
+      if (count != NULL) ++(*count);
+    } while (__code < *++__mclen);
+    
+    return __values[__mclen - __min_code][__code - *__mclen];
+  }
+
+  inline void huff_encode(unsigned long x, unsigned long *codes, char *lens,
+			  unsigned long *count = NULL) {
+    register int __i;
+    register int __clen = lens[x];
+    register unsigned long __code = codes[x];
+    for (__i=__clen-1; __i>=0; --__i) {
+      encodeBit((__code >> __i) & 1);
+    }
+    if (count != NULL) {
+      *count += lens[x];
+    }
+  }
+
+  static inline unsigned long unary_length(unsigned long x) {
+    check_positive("unary", x);
+    return x;
+  }
+
+  static inline int binary_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else _B_nbits = _B_logofb;
+    return _B_nbits;
+  }
+
+  static inline unsigned long bblock_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_bcount, count, _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    count = unary_length(_B_xdivb+1);
+    _B_bcount = binary_length(_B_xx+1, _B_bb);
+    count += _B_bcount;
+    return count;
+  }
+  
+  static inline unsigned long elias_length(unsigned long x, unsigned long b, 
+					   double s) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0, _B_ecount;
+    register double _B_pow=1.0;
+    register unsigned long count;
+    check_positive("gamma", _B_xx);
+    
+    while ((int)(_B_xx)>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    count = unary_length(_B_k+1);
+    _B_ecount = binary_length(_B_xx-_B_lower+1, _B_upper-_B_lower+1);
+    count += _B_ecount;
+    return count;
+  }
+  
+  static inline unsigned long gamma_length(unsigned long x) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    return _B_logofb*2+1;
+  }
+
+  static inline unsigned long delta_length(unsigned long x) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx, _B_dcount;
+    register unsigned long count;
+    _B_logx = floor_log2 (_B_xxx);
+    count = gamma_length(_B_logx+1);
+    _B_dcount = binary_length (_B_xxx+1, 1<<_B_logx);
+    count += _B_dcount;
+    return count;
+  }
+};
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
@@ -0,0 +1,128 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mem.h"
+
+mem_bitio_buffer::mem_bitio_buffer() {
+}
+
+mem_bitio_buffer::mem_bitio_buffer(unsigned char *readbuffer, int size) { 
+  pos       = readbuffer;
+  base      = readbuffer;
+  remaining = size;
+  buffer    = 0;
+  btg       = 0;
+}
+
+
+void mem_bitio_buffer::error() {
+  fprintf (stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+void mem_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+
+void mem_bitio_buffer::seek(unsigned long toPos) { 
+  register long _newPos;
+  do {
+    _newPos = toPos;
+    
+    pos    = base + (_newPos >> 3);
+    buffer = *pos++;
+    btg    = 8 - (_newPos & 0x07);
+  } while (0);
+}
+
+long mem_bitio_buffer::addff(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0xff;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+
+  return b;
+}
+
+long mem_bitio_buffer::add00(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0x00;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+  
+  return b;
+}
+
+int mem_bitio_buffer::bit() {
+  if (btg == 0) {
+
+    if (remaining == 0) { 
+      this->error();
+      return 0;
+
+    } else {
+      buffer = *pos++;
+      --remaining;
+      btg = sizeof(buffer) * 8;
+    }
+    
+  }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void mem_bitio_buffer::encodeBit(int b) {
+  --btg;
+  if (b) buffer |= (1 << btg);
+  
+  if (btg == 0) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+    buffer = 0;
+    btg = sizeof(buffer)*8;
+  }
+}
+
+void mem_bitio_buffer::flush() {
+  if (btg != sizeof(buffer)*8) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+  }
+  btg = sizeof(buffer)*8;
+}
+
+void mem_bitio_buffer::done() {
+  // do nothing
+}
+
+int mem_bitio_buffer::encodeLength() {
+  return pos - base;
+}
+
+int mem_bitio_buffer::encodeVerify() {
+  return (remaining != 0);
+}
+
+
+
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,104 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+#include "bitio_m_abstract.h"
+
+
+class mem_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *pos;
+  unsigned char *base;
+  int            remaining;
+  unsigned char  buffer;
+  unsigned char  btg;
+  
+public:
+  mem_bitio_buffer ();
+  mem_bitio_buffer (unsigned char *buffer, int size);
+  void seek (unsigned long topos);
+  void error ();
+  long add00 (long b);
+  long addff (long b);
+  int  bit ();
+  void done ();
+
+  void encodeStart ();
+  void encodeBit (int bit);
+  void encodeDone () { flush(); }
+  int  encodeVerify ();
+  int  encodeLength ();
+  void flush ();
+};
+
+
+/*
+#define ENCODE_START(p,r)						\
+  {									\
+    mem_bitio_buffer buffer(p, r);                                     \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush()
+
+#define ENCODE_DONE buffer.flush();}
+
+
+#define DECODE_START(p,r)						\
+  {	                                                                \
+    mem_bitio_buffer buffer(p,r);
+
+#define DECODE_CONTINUE(b)						\
+  {	                                                                \
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
@@ -0,0 +1,89 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mems.h"
+
+/**
+ * Create decoding "stream"/"file"
+ */
+mems_bitio_buffer::mems_bitio_buffer()
+{
+}
+
+mems_bitio_buffer::mems_bitio_buffer(unsigned char *readbuffer, long _pos)
+{ 
+  base = readbuffer;
+  pos  = _pos;
+}
+
+void mems_bitio_buffer::encodeStart()
+{
+  // Do nothing
+}
+
+
+void mems_bitio_buffer::error()
+{
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+
+void mems_bitio_buffer::seek(unsigned long toPos)
+{ 
+  pos = toPos;
+}
+
+unsigned long mems_bitio_buffer::position()
+{
+  return pos;
+}
+
+long mems_bitio_buffer::addff(long b)
+{
+  do
+    {
+      b = b + b + (base[pos>>3] & (0x80 >> (pos & 0x07)) != 0);
+      ++pos;
+    }
+  while(0);
+
+  return b;
+}
+
+long mems_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int mems_bitio_buffer::bit()
+{
+  ++pos;
+  return ((base[(pos-1)>>3] & (0x80 >> ((pos-1) & 0x07))) != 0);
+}
+
+void mems_bitio_buffer::encodeBit(int bit)
+{
+  if (bit)
+    base[pos>>3] |= 0x80 >> (pos & 0x07);
+  else
+    base[pos>>3] &= 0xff7f >> (pos & 0x07);
+  ++pos;
+}
+
+void mems_bitio_buffer::flush()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::done()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::encodeDone()
+{
+  flush();
+}
+
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+#include "bitio_m_abstract.h"
+
+
+class mems_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *base;
+  unsigned long  pos;
+  
+public:
+  mems_bitio_buffer();
+  mems_bitio_buffer(unsigned char *readbuffer, long _pos);
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  //  int  bitOffset();
+  void done();
+
+  unsigned long position();
+
+  void encodeStart();
+  void encodeBit(int b);
+  void encodeDone();
+  void flush();
+};
+
+
+/*#define ENCODE_START(b,p)						\
+  {									\
+    mems_bitio_buffer buffer(b, p);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+
+#define DECODE_START(b,p)						\
+  {		                                                        \
+    mems_bitio_buffer buffer(b, p);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b);
+
+#define DECODE_ADD_00(b) b = buffer.add00(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "longlong.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_random.h"
+
+random_bitio_buffer::random_bitio_buffer(FILE *f, unsigned long length)
+{
+  buffer = NULL;
+  attachFile (f, length);
+}
+
+random_bitio_buffer::~random_bitio_buffer() {
+  if (buffer != NULL) delete []buffer;
+}
+
+void random_bitio_buffer::attachFile (FILE *f, unsigned long length) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  // reset all the values
+  file   = f;
+  pos    = 0;
+  base   = 0;
+  used   = 0;
+  len    = length - 1;
+  shift  = 0;
+
+  // make sure the length is a power of 2
+  while (len != 0) {
+    ++shift;
+    len = len >> 1;
+  }
+  len = 1 << shift;
+
+  // allocate a new buffer
+  buffer = new u_char[len];
+
+  // fill up the buffer
+  if (file != NULL) {
+    fseek(file, 0, 0);
+    fread(buffer, 1, len, file);
+  }
+}
+
+random_bitio_buffer &random_bitio_buffer::operator= (const random_bitio_buffer &_rbb) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  file = _rbb.file;
+
+  base = _rbb.base;
+  used = _rbb.used;
+  pos = _rbb.pos;
+  len = _rbb.len;
+  shift = _rbb.shift;
+
+  // copy the buffer
+  buffer = new u_char [len];
+  //  memcpy (buffer, _rbb.buffer, len);
+memcpy (buffer, _rbb.buffer, len);
+  
+  return (*this);
+}
+
+void random_bitio_buffer::error() {
+}
+
+void random_bitio_buffer::writeRead()
+{
+  if (used)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+    }
+  //  else
+  //    {
+      base += len;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = 0;
+      used = 0;
+  //    }
+}
+
+void random_bitio_buffer::encodeBit(int bit)
+{
+  do
+    {
+      if (bit != 0)
+	{
+	  buffer[pos >> 3] |= 0x80 >> (pos & 0x07);
+	}
+      else
+	{
+	  buffer[pos >> 3] &= 0xff7f >> (pos & 0x07);
+	}
+      used = 1;
+      ++pos;
+
+      if ((pos >> 3) >= len)
+	{
+	  writeRead();
+	}
+    }
+  while (0);
+}
+
+void random_bitio_buffer::flush()
+{
+  if (used != 0)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+      used = 0;
+    }
+}
+
+void random_bitio_buffer::encodeDone()
+{
+  flush();
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+}
+
+unsigned long random_bitio_buffer::tell()
+{
+  return ((base << 3) + pos);
+}
+
+void random_bitio_buffer::seek(unsigned long toPos)
+{
+  if (((toPos >> 3) >= base) && (((toPos+7) >> 3) < (base + len)))
+    {
+      pos = toPos - (base << 3);
+    }
+  else
+    {
+      flush();
+      base = toPos >> 3;
+      base = (toPos >> (shift + 3)) << shift;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = toPos & 0x07;
+      pos = toPos & ((8 << shift) - 1);
+    }
+}
+
+int random_bitio_buffer::bit()
+{
+  //  fprintf(stderr, "Bit in");
+  ++pos;
+  if (((pos - 1) >> 3) >= len)
+    {
+      writeRead();
+      pos = 1;
+    }
+  return (buffer[(pos - 1) >> 3] & (0x80 >> ((pos - 1) & 0x07))) != 0;
+}
+
+long random_bitio_buffer::addff(long b)
+{
+  return b + b + bit();
+}
+
+long random_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int random_bitio_buffer::bitOffset()
+{
+  return 0;
+}
+
+void random_bitio_buffer::done()
+{
+  encodeDone();
+}
+
+
+#ifdef USE_LONG_LONG
+void random_bitio_buffer::seek_LL(mg_ullong toPos)
+{
+  if (((toPos >> 3) >= (mg_ullong) base) &&
+      (((toPos + 7) >> 3) < (mg_ullong) (base + len)))
+    {
+      pos = (long) (toPos - (mg_ullong) (base << 3));
+    }
+  else
+    {
+      flush();
+      base = (long) ((toPos >> (shift + 3)) << shift);
+
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = (long) (toPos & ((8 << shift) - 1));
+    }
+}
+
+mg_ullong random_bitio_buffer::tell_LL()
+{
+  return (((mg_ullong) base) << 3ull) + pos;
+}
+#endif
+
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+#include "longlong.h"
+#include "bitio_m_abstract.h"
+
+
+class random_bitio_buffer : public bitio_buffer { 
+public:
+  FILE *file;
+  unsigned char *buffer;
+  unsigned long  base;
+  unsigned long  used;
+  unsigned long  pos;
+  unsigned long  len;
+  unsigned long  shift;
+  void writeRead();
+
+  random_bitio_buffer(FILE *f = 0,
+		      unsigned long length = 8*1024);
+  virtual ~random_bitio_buffer();
+  
+  void attachFile (FILE *f, unsigned long length);
+
+  random_bitio_buffer &operator= (const random_bitio_buffer &_rbb);
+  
+  void encodeDone();
+  void encodeBit(int b);
+  void flush();
+  unsigned long tell();
+
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset();
+  void done();
+
+#ifdef USE_LONG_LONG
+  // extras for longlong.h
+  void seek_LL(mg_ullong pos);
+  mg_ullong tell_LL();
+#endif
+};
+
+
+/*#define ENCODE_START(f,l)						\
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define SEEK fprintf(stderr, "Seek to %d\n",buffer.base)
+#define READ fprintf(stderr, "Read of %d\n",buffer.len)
+#define WRITE fprintf(stderr, "Write of %d\n",buffer.used)
+
+#define WRITE_READ buffer.writeRead()
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+#define ENCODE_DONE buffer.done();}							
+
+#define DECODE_START(f,l) \
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define DECODE_CONTINUE(b) \
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_TELL buffer.tell()
+
+#define DECODE_TELL buffer.tell()
+*/
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
@@ -0,0 +1,125 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_stdio.h"
+
+stdio_bitio_buffer::stdio_bitio_buffer(FILE *f) {
+  attachFile (f);
+}
+
+void stdio_bitio_buffer::attachFile (FILE *f) {
+  file = f;
+  buffer = 0;
+  btg = 0;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+void stdio_bitio_buffer::error() {
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d", __FILE__, __LINE__);
+  exit(1);
+}
+
+void stdio_bitio_buffer::seek (unsigned long byte, unsigned char bit) {
+  if (fseek (file, byte, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = bit;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::seek(unsigned long topos) {
+  if (fseek(file, topos >> 3, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = 8 - (topos & 0x07);
+  bytesWritten = 0;
+}
+
+long stdio_bitio_buffer::addff(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0x0ff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+
+  return b;
+}
+
+long stdio_bitio_buffer::add00(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0xff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+  
+  return b;
+}
+
+void stdio_bitio_buffer::encodeBit(int b)
+{
+  --btg;
+  if (b != 0)
+    buffer |= (1 << btg);
+
+  if (btg == 0)
+    {
+      putc(buffer, file);
+      bytesWritten += sizeof (buffer);
+      buffer = 0;
+      btg = sizeof(buffer)*8;
+    }
+}
+
+int stdio_bitio_buffer::bit()
+{
+  if (btg == 0)
+    {
+      buffer = getc(file);
+      if (feof(file))
+	{
+	  error();
+	  return 0;
+	}
+      else
+	{
+	  btg = sizeof(buffer) * 8;
+	}
+    }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void stdio_bitio_buffer::done() {
+  // no tidying required
+}
+
+void stdio_bitio_buffer::encodeDone() {
+  if (btg != sizeof(buffer) * 8) {
+    putc(buffer, file);
+    bytesWritten += sizeof (buffer);
+  }
+}
+
Index: /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,101 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+#include "bitio_m_abstract.h"
+
+
+class stdio_bitio_buffer : public bitio_buffer {
+protected:
+  FILE *file;
+  unsigned char buffer;
+  unsigned char btg;
+  unsigned long bytesWritten;
+  
+public:
+  stdio_bitio_buffer (FILE *f = NULL);
+
+  void attachFile (FILE *f);
+
+  unsigned char GetBtg () { return btg; }
+  unsigned long GetBytesWritten () { return bytesWritten; }
+  
+  void seek (unsigned long byte, unsigned char bit);
+  void seek (unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset() { return btg; }
+  void done();
+
+  void encodeStart();
+  void encodeBit(int bit);
+  void encodeDone();
+};
+
+
+/*#define ENCODE_START(f)							\
+  {									\
+    stdio_bitio_buffer buffer(f);                                       \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b) buffer.pause(&b);}
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+#define DECODE_START(f)							\
+  {		                                                        \
+    stdio_bitio_buffer buffer(f);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+     buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/ftruncate.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/ftruncate.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/ftruncate.cpp	(revision 21112)
@@ -0,0 +1,99 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/indexers/mgpp/lib/getopt_old.c
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/getopt_old.c	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/getopt_old.c	(revision 21112)
@@ -0,0 +1,748 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+/* jrm21 - GSDL: getopt.h -> getopt_old.h */
+#include "getopt_old.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/indexers/mgpp/lib/getopt_old.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/getopt_old.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/getopt_old.h	(revision 21112)
@@ -0,0 +1,134 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef HAVE_DECL_GETOPT
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  int getopt (int, char * const *, const char *);
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/indexers/mgpp/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/indexers/mgpp/lib/huffman.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/huffman.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/huffman.cpp	(revision 21112)
@@ -0,0 +1,385 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = new huff_data))
+	goto error0;
+    }
+  if (!(heap = new unsigned long[num * 2]))
+    goto error1;
+  if (!(hd->clens = new char[num]))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  memcpy (&heap[num], freqs, num * sizeof (*heap));
+  memset (heap, '\0', num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; ++i)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; --i)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    ++child;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; ++i)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		++child;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	++HNum;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; ++i)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  memset (hd->lencount, '\0', sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; ++i)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      ++hd->lencount[codelen];
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; --i)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  delete []heap;
+  return (hd);
+
+error2:
+  delete []heap;
+error1:
+  if (!data)
+    delete hd;
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  memcpy (mc, data->min_code, sizeof (mc));
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (!(values = new unsigned long *[MAX_HUFFCODE_LEN + 1]))
+    {
+      delete []vals;
+      return (NULL);
+    }
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; ++i)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; ++i)
+    size += counts[i] * hd.clens[i];
+  delete []hd.clens;
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk) {
+  if (fread (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      memset (hd->lencount, '\0', sizeof (hd->lencount));
+      if (fread (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      memset (hd->min_code, '\0', sizeof (hd->min_code));
+
+      if (fread (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = new char[hd->num_codes]))
+	return -1;
+
+
+      if (fread (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/lib/huffman.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/huffman.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+#define MAX_HUFFCODE_LEN 31
+
+struct huff_data {
+  int num_codes;
+  int mincodelen;
+  int maxcodelen;
+  int lencount[MAX_HUFFCODE_LEN + 1];
+  unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+  char *clens;
+};
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/local_strings.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/local_strings.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/local_strings.cpp	(revision 21112)
@@ -0,0 +1,411 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = new u_char(src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  memcpy (copy, src_str, src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    ++i;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  memset (buf, '\0', sizeof(buf));
+  for (i = 0; i < len; ++i)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  ++s;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      --s;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; ++i)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/indexers/mgpp/lib/local_strings.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+class mgString
+{
+  u_char *data;
+ public:
+  inline int length()
+    {
+      return data[0];
+    }
+  inline u_char *rawdata()
+    {
+      return data;
+    }
+  inline void read(u_char **base)
+    {
+      unsigned int copy, suff;
+      
+      copy  = **base;
+      *base = *base + 1;
+      suff  = **base;
+      *base = *base + 1;
+      memcpy( (data + copy + 1), *base, suff);
+      *data = copy + suff;
+      *base += suff;
+    }
+  inline void read(FILE *dict)
+    {
+      unsigned int copy, suff;
+      copy = fgetc(dict);
+      suff = fgetc(dict);
+      fread(data + copy + 1, sizeof(u_char), suff, dict);
+      *data = copy + suff;
+    }
+  inline static void readString(u_char **base, u_char *prev)
+    {
+      ((mgString *) &prev)->read(base);
+    }
+  inline static void readString(FILE *dict, u_char *prev)
+    { 
+      ((mgString *) &prev)->read(dict);
+    }
+};
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/longlong.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/longlong.h	(revision 21112)
@@ -0,0 +1,85 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+    
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/lovinstem.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/lovinstem.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/lovinstem.cpp	(revision 21112)
@@ -0,0 +1,1172 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+/*static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();*/
+
+static int 
+aio (char ch)			/* Rule number 9     */
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (char ch)				/* Rule Number 24    */
+{
+  return (ch != 's');
+}
+
+static int 
+pt (char ch)				/* Rule number 28    */
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (char ch)				/* Rule number 30    */
+{
+  return (ch != 'm');
+}
+
+static int 
+n (char ch)				/* Rule number 32    */
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+/*static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+*/
+static int 
+B (int stem_length, char *end)
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (int stem_length, char *end)
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (int stem_length, char *end)
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (int stem_length, char *end)
+
+     
+{
+  return (*end != 'e');
+}
+
+static int 
+F (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'c');
+}
+
+static int 
+W (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (int stem_length, char *end)
+     
+     
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  memcpy (copy, word, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  memcpy (word, copy, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) (char);	        /* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  ++rule;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) (int length, char *end);		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      ++suffix_start;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/indexers/mgpp/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,28 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/memlib.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/memlib.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/memlib.cpp	(revision 21112)
@@ -0,0 +1,41 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+char *
+my_strdup(const char *str)
+{
+  char *ret_str = new char[strlen(str)+1];
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_strdup;
Index: /main/tags/3.01/indexers/mgpp/lib/memlib.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/memlib.h	(revision 21112)
@@ -0,0 +1,55 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+/* changed default behaviour - we now assume std memory. - john mcp. */
+/* #ifdef STD_MEM */
+#if !defined(NOT_STD_MEM)
+#include <string.h>
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+
+#if defined (__WIN32__)
+# define Xstrdup _strdup
+#else
+# define Xstrdup strdup
+#endif
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/messages.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/messages.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/messages.cpp	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef SILENT
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/indexers/mgpp/lib/messages.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/messages.h	(revision 21112)
@@ -0,0 +1,49 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+void Message (const char *fmt,...);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
+
+
+
+
Index: /main/tags/3.01/indexers/mgpp/lib/mgheap.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/mgheap.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/mgheap.cpp	(revision 21112)
@@ -0,0 +1,148 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; --__i)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; --__i)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	++child;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; --i)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; --i)
+    {
+      SWAP (size, (char *) heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  --(*num);
+  SWAP (size, (char *) heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  ++(*num);
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/indexers/mgpp/lib/mgheap.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/indexers/mgpp/lib/netorder.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/netorder.h	(revision 21112)
@@ -0,0 +1,177 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy ( (void *) &d, (const void *) &tmpl, sizeof (double) >> 1);                           \
+	     memcpy ((void *) ((unsigned int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &d, (const void *) &tmpl, sizeof (double) >> 1); \
+	     memcpy ((void *) ((int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &hd, sizeof(double) >> 1);\
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &hd + (sizeof(double) >> 1)), sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy((void *) &nd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) ((unsigned int) &nd + (sizeof(double) >> 1)), (void *) &tmph, sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &nd, (void *) &tmph, sizeof(double) >> 1);      \
+	     memcpy ((void *) &tmpl, (const void *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &hd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) &hd + (sizeof(double) >> 1), (const void *) &tmph, sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define NTOHF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define HTONF2(hf, nf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(hf), sizeof (float));    \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(nf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+#define NTOHF2(nf, hf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(nf), sizeof (float));    \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(hf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/indexers/mgpp/lib/non_ansi.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/non_ansi.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/non_ansi.h	(revision 21112)
@@ -0,0 +1,24 @@
+#if !defined(_NON_ANSI_H)
+#define _NON_ANSI_H 1
+
+#include "config.h" 
+# if !defined(__STDC__)
+
+#include <stdlib.h>  /* for FILE def */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+int fileno(FILE *);
+int ftruncate(int, long);
+int pclose(FILE *);
+FILE *popen(const char *command, const char *type);
+int getpagesize();
+int strcasecmp(const char *, const char *);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* !defined STDC, ... */
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/pathmax.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/indexers/mgpp/lib/perf_hash.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/perf_hash.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/perf_hash.cpp	(revision 21112)
@@ -0,0 +1,1074 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+#define STRUCT
+
+#include <time.h>  /* for call to time () in SEED_RANDOM() */
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+
+long irandm (long is[2]);  /* from random.c */
+
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+_delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    _delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    _delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    _delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = (struct Stack *)Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = (struct Stack *) Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  ++StackPos;
+  while (StackPos)
+    {
+      int m, b;
+      --StackPos;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      _delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      _delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      _delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; ++e)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; ++v)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      *n = (int) (num * c + 1);
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; ++i)
+	for (j = 0; j < MAX_L; ++j)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; ++e)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      ++w;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int _xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ _xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; ++e)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; ++i)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  memset (translate, '\0', sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; ++i)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, ++s)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; ++i)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = (int *) Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = (int *)Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = (int *)Xmalloc (sizeof (int) * MAX_M);
+  mk = (char *)Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = (tb_entry **)Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb)
+      if (!(tb[i] = (tb_entry *)Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  memset(translate, '\0', sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  MAX_N = MAX_M + MAX_M / 4;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  memcpy ((void *) g, (const void *) &g[1], n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  phd->g = (int *)Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = (struct tb_entry **)Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb)
+      if (!(phd->tb[i] = (struct tb_entry *)Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      --l;
+      ++s;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/indexers/mgpp/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/random.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/random.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/random.cpp	(revision 21112)
@@ -0,0 +1,211 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/indexers/mgpp/lib/random.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/random.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/random.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      --last;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') --last;
+      if (last>=0 && wordstart[last]=='r') --last;
+      if (last>=0 && wordstart[last]=='e') --last;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	--last;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) --last;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,26 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/strcasecmp.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/strcasecmp.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/strcasecmp.cpp	(revision 21112)
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(char *s1, char *s2)
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/indexers/mgpp/lib/strstr.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/strstr.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/strstr.cpp	(revision 21112)
@@ -0,0 +1,42 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (char *s1, char *s2)
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, ++i, ++s)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; ++p1, ++p2)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/lib/sysfuncs.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/sysfuncs.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/sysfuncs.cpp	(revision 21112)
@@ -0,0 +1,6 @@
+#include <stdlib.h>
+
+void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+
Index: /main/tags/3.01/indexers/mgpp/lib/sysfuncs.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/sysfuncs.h	(revision 21112)
@@ -0,0 +1,388 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+#include <sys/stat.h>
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+#ifdef __CYGWIN__ 
+/* Cygwin has setbuffer in stdio.h, but can't link against it!! */
+#include <stdio.h>
+#endif
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+
+#ifdef __cplusplus /* compiling for C++ */
+#include <stdlib.h>
+#ifndef __bcopy
+inline void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+#define __bcopy 1
+#endif
+#ifndef __bcmp
+inline int bcmp(void *s1, void *s2, int n)
+{ return memcmp(s1, s2, n);
+}
+#define __bcmp 1
+#endif
+#ifndef __bzero
+inline void bzero(void *m, int n)
+{ memset(m, 0, n);
+}
+#define __bzero 1
+#endif
+#else              /* compiling for C */
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#endif
+
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#ifdef __MSDOS__
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#ifdef __MSDOS__
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#ifndef __WIN32__
+#include <pathmax.h>
+#endif
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+/*#include <getopt.h>*/
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+/*** setlocale is ansi and posix...(jrm21)
+#if !HAVE_SETLOCALE
+#define setlocale(Category, Locale)
+#endif
+***/
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#ifdef HAVE_VPRINTF
+void error __P ((int, int, const char *,...));
+#else
+void error ();
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/indexers/mgpp/lib/unitool.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/unitool.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/unitool.cpp	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; --i) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; ++i) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      ++curpos;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+/*unsigned long *unicode_cat_data;*/
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = (unsigned long *) malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) ++letter_here;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) ++digit_here;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) ++space_here;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	++i;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      ++outhere;
+      ++outlen;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    ++in_i;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/indexers/mgpp/lib/unitool.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/win32in.c
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/win32in.c	(revision 21112)
@@ -0,0 +1,85 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#include "win32in.h"
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/indexers/mgpp/lib/win32in.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/win32in.h	(revision 21112)
@@ -0,0 +1,20 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#ifdef __WIN32__
+#define _LITTLE_ENDIAN
+#endif
+
+#include "longlong.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
+
+#ifdef __cplusplus
+}
+#endif
Index: /main/tags/3.01/indexers/mgpp/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/indexers/mgpp/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/indexers/mgpp/mkinstalldirs
===================================================================
--- /main/tags/3.01/indexers/mgpp/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/indexers/mgpp/stamp-h.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
@@ -0,0 +1,129 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.cpp -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FIvfLevelInfo.h"
+
+
+IvfLevelInfo::IvfLevelInfo () {
+  Clear();
+}
+
+void IvfLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  numEntries = 0;
+  approxWeightsDiskPtr = 0;
+  exactWeightsDiskPtr = 0;
+  workInLevel = false;
+}
+
+bool IvfLevelInfo::Read (FILE *f) {
+  workInLevel = false;
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, numEntries) &&
+	  ReadUL (f, approxWeightsDiskPtr) &&
+	  ReadUL (f, exactWeightsDiskPtr));
+}
+
+bool IvfLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, numEntries) &&
+	  WriteUL (f, approxWeightsDiskPtr) &&
+	  WriteUL (f, exactWeightsDiskPtr));
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  numEntries: " << l.numEntries << "\n"
+    << "  approxWeightsDiskPtr: " << l.approxWeightsDiskPtr << "\n"
+    << "  exactWeightsDiskPtr: " << l.exactWeightsDiskPtr << "\n"
+    << "  workInLevel: " << ((l.workInLevel) ? "true" : "false") << "\n\n";
+  
+  return s;
+}
+
+
+
+FIvfLevel::FIvfLevel () {
+  Clear ();
+}
+
+void FIvfLevel::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FIvfLevel::Read (FILE *f) {
+  if (!ReadUCArray (f, docTag) ||
+      !ReadUCArray (f, indexLevel))
+    return false;
+  
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+  
+  IvfLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FIvfLevel::Write (FILE *f) const {
+  if (!WriteUCArray (f, docTag) ||
+      !WriteUCArray (f, indexLevel))
+    return false;
+  
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  IvfLevelInfoMap::const_iterator here = levelInfo.begin();
+  IvfLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l) {
+  s << "docTag: \"" << l.docTag << "\"\n"
+    << "indexLevel: \"" << l.indexLevel << "\"\n";
+  
+  IvfLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  IvfLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
Index: /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FIvfLevelInfo.h	(revision 21112)
@@ -0,0 +1,77 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.h -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FIVFLEVELINFO_H
+#define FIVFLEVELINFO_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+class IvfLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long numEntries; // N for level
+  unsigned long approxWeightsDiskPtr;
+  unsigned long exactWeightsDiskPtr;
+  bool workInLevel;  // this is just a working variable,
+                     // it is not saved or read from the file
+  
+  IvfLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l);
+
+
+typedef map<UCArray, IvfLevelInfo, LTUCArray> IvfLevelInfoMap;
+
+
+class FIvfLevel {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  IvfLevelInfoMap levelInfo;
+  
+  FIvfLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/FText.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FText.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FText.cpp	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * FText.cpp -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FText.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+BitAddr::BitAddr () {
+  Clear ();  
+}
+
+void BitAddr::Clear () {
+  byte = 0;
+  bit = 0;
+}
+
+bool BitAddr::Read (FILE *f) {
+  return (ReadUL (f, byte) && ReadUC (f, bit));
+}
+
+bool BitAddr::Write (FILE *f) const {
+  return (WriteUL (f, byte) && WriteUC (f, bit));
+}
+
+#define BITADDRLEN (sizeof(unsigned long) + sizeof(unsigned char))
+
+  
+TextLevelInfo::TextLevelInfo () {
+  Clear();
+}
+
+void TextLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  textIdxPtr = 0;
+  numEntries = 0;
+}
+
+bool TextLevelInfo::Read (FILE *f) {
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, textIdxPtr) &&
+	  ReadUL (f, numEntries));
+}
+
+bool TextLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, textIdxPtr) &&
+	  WriteUL (f, numEntries));
+}
+
+ostream &operator<<(ostream &s, const TextLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  textIdxPtr: " << l.textIdxPtr << "\n"
+    << "  numEntries: " << l.numEntries << "\n\n";
+  
+  return s;
+}
+
+
+FTextLevel::FTextLevel () {
+  Clear();
+}
+
+void FTextLevel::Clear () {
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FTextLevel::Read (FILE *f) {
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+
+  TextLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FTextLevel::Write (FILE *f) const {
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  TextLevelInfoMap::const_iterator here = levelInfo.begin();
+  TextLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+ostream &operator<<(ostream &s, const FTextLevel &l) {
+  TextLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  TextLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
+
+
+TextIdx::TextIdx () {
+  Clear ();
+}
+
+void TextIdx::Clear () {
+  start.Clear();
+  end.Clear();
+  which = 0;
+}
+
+#define TEXTIDXLEN (BITADDRLEN*2 + sizeof(char))
+
+bool TextIdx::Read (FILE *f, const TextLevelInfo &levelInfo,
+		    unsigned long docNum) {
+  if (!SeekTextIdx (f, levelInfo, docNum)) return false;
+  return Read (f);
+}
+
+bool TextIdx::Read (FILE *f) {
+  return (start.Read (f) && end.Read (f) && ReadUC (f, which));
+}
+
+bool TextIdx::Write (FILE *f) const {
+  return (start.Write (f) && end.Write (f) && WriteUC (f, which));
+}
+
+ostream &operator<<(ostream &s, const TextIdx &t) {
+  s << "start byte: " << t.start.byte << "\n";
+  s << "start bit:  " << (int)t.start.bit << "\n";
+  s << "end byte:   " << t.end.byte << "\n";
+  s << "end bit:    " << (int)t.end.bit << "\n";
+  s << "which:      " << (int)t.which << "\n\n";
+  
+  return s;
+}
+
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum) {
+  if (docNum == 0 || docNum > levelInfo.numEntries) return false;
+  
+  unsigned long seekPos = levelInfo.textIdxPtr + (docNum-1) * TEXTIDXLEN;
+  if (fseek (f, seekPos, SEEK_SET) != 0) return false;
+  
+  return true;
+}
+
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize) {
+  // clear the array
+  a.erase (a.begin(), a.end());
+
+  // read in each element
+  TextIdx idx;
+  while (arrSize > 0) {
+    if (!idx.Read(f)) return false;
+    a.push_back (idx);
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a) {
+  TextIdxArray::const_iterator here = a.begin();
+  TextIdxArray::const_iterator end = a.end();
+  while (here != end) {
+    if (!(*here).Write(f)) return false;
+    ++here;
+  }
+  
+  return true;
+}
+
+
+CompressTextInfo::CompressTextInfo () {
+  Clear();
+}
+
+void CompressTextInfo::Clear () {
+  ResetStart();
+  docPtrs.erase (docPtrs.begin(), docPtrs.end());
+}
+
+void CompressTextInfo::ResetStart () {
+  inDoc = false;
+  start.Clear();
+  which = 0;
+}
+  
+void CompressTextInfo::SetStart (unsigned long startPos,
+				 unsigned char startBit,
+				 unsigned char startWhich) {
+  // place an imaginary end tag if needed
+  // note: the document tag always needs a closing tag
+  if (inDoc) SetEnd (startPos, startBit);
+
+  // remember start of level
+  inDoc = true;
+  start.byte = startPos;
+  start.bit = startBit;
+  which = startWhich;
+}
+
+void CompressTextInfo::SetEnd (unsigned long endPos,
+			       unsigned char endBit) {
+  if (inDoc) {
+    // add completed entry to list of ptrs
+    TextIdx textIdx;
+    textIdx.start = start;
+    textIdx.end.byte = endPos;
+    textIdx.end.bit = endBit;
+    textIdx.which = which;
+    docPtrs.push_back (textIdx);
+    
+    // reset this entry
+    ResetStart();
+  }
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/FText.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FText.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FText.h	(revision 21112)
@@ -0,0 +1,150 @@
+/**************************************************************************
+ *
+ * FText.h -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FTEXT_H
+#define FTEXT_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+// holds a bit address
+class BitAddr {
+public:
+  unsigned long byte;
+  unsigned char bit;
+
+  BitAddr ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+class TextLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long textIdxPtr;
+  unsigned long numEntries;
+
+  TextLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextLevelInfo &l);
+
+
+typedef map<UCArray, TextLevelInfo, LTUCArray> TextLevelInfoMap;
+
+
+class FTextLevel {
+public:
+  TextLevelInfoMap levelInfo;
+  
+  FTextLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FTextLevel &l);
+
+
+// the text.idx file points into the compressed text
+class TextIdx {
+public:
+  BitAddr start;       // the first bit of the compressed text
+  BitAddr end;         // the first bit past the end of the text
+  unsigned char which; // 0 = non-word, 1 = word
+  
+  TextIdx ();
+  void Clear ();
+
+  // does a seek and reads the appropriate record
+  bool Read (FILE *f, const TextLevelInfo &levelInfo,
+	     unsigned long docNum);
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextIdx &t);
+
+
+// note: document numbers start at 1
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum);
+
+
+// used to store the pointers while compressing the text
+typedef vector<TextIdx> TextIdxArray;
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize);
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a);
+
+
+// used to store information about a level while compressing the text
+class CompressTextInfo {
+public:
+  bool inDoc;
+  BitAddr start;
+  unsigned char which;
+  TextIdxArray docPtrs;
+
+  CompressTextInfo ();
+  void Clear ();
+
+  // clears inDoc, start, and which (not the docPtrs)
+  void ResetStart ();
+  
+  // set the start of a level (closing off the last opening
+  // tag if needed)
+  void SetStart (unsigned long startPos,
+		 unsigned char startBit,
+		 unsigned char startWhich);
+
+  // if in a document, it will set the end, add the document to the
+  // list of document ptrs, and then call ResetStart
+  void SetEnd (unsigned long endPos,
+	       unsigned char endBit);
+};
+
+typedef map<UCArray, CompressTextInfo, LTUCArray> CompressTextInfoMap;
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/FileBuf.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FileBuf.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FileBuf.cpp	(revision 21112)
@@ -0,0 +1,50 @@
+/**************************************************************************
+ *
+ * FileBuf.cpp -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FileBuf.h"
+#include "sysfuncs.h"
+
+#ifdef __WIN32__
+# include <io.h>
+# define read _read
+#endif
+
+bool FileBuf::NextBuf () {
+  if (fd < 0) return false;
+  int numbytes = read (fd, buf, FILEBUFSIZE);
+  if (numbytes < 0) numbytes = 0;
+  
+  here = buf;
+  end = here + numbytes;
+
+  return (numbytes > 0);
+}
+
+FileBuf::FileBuf () {
+  here = end = buf;
+  fd = -1;
+}
+
+void FileBuf::SetFD (int _fd) {
+  here = end = buf;
+  fd = _fd;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/FileBuf.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FileBuf.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FileBuf.h	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * FileBuf.h -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FILEBUF_H
+#define FILEBUF_H
+
+#define FILEBUFSIZE 65536
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+class FileBuf {
+protected:
+  unsigned char buf[FILEBUFSIZE];
+  unsigned char *here;
+  unsigned char *end;
+
+  int fd;
+
+  // gets a new buffer if need be, returns true if
+  // more could be got. if true is returned the caller
+  // can assume that here != end
+  bool NextBuf ();
+  
+public:
+  FileBuf ();
+
+  void SetFD (int _fd);
+
+  inline bool Get (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here++; return true; }
+    return false;
+  }
+  
+  inline bool Peek (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here; return true; }
+    return false;
+  }
+
+  inline bool Next () {
+    if (here != end || NextBuf()) { ++here; return true; }
+    return false;
+  }
+};
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.cpp -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FragLevelConvert.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+
+
+FragLevelConvert::FragLevelConvert () {
+  numFrags = 0;
+  numLevelDocs = 0;
+}
+
+bool FragLevelConvert::Read (FILE *invfFile,
+			     unsigned long levelPtr,
+			     unsigned long _numFrags,
+			     unsigned long _numLevelDocs) {
+  levelStarts.erase (levelStarts.begin(), levelStarts.end());
+  numFrags = _numFrags;
+  numLevelDocs = _numLevelDocs;
+
+  // seek to the appropriate place in the inverted file
+  fseek (invfFile, levelPtr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(invfFile);
+  
+  unsigned long pTag = numLevelDocs*2;
+  unsigned long B = BIO_Bblock_Init (numFrags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  for (i=0; i<numLevelDocs; ++i) {
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    levelStarts.push_back (fragNum);
+
+    // ignore end, assume every fragment is in a level
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+  }
+
+  // fragNum would be the starting fragment of the next document
+  // if there was one
+  levelStarts.push_back (fragNum);
+  
+  if (fragNum != _numFrags) {
+    cerr << "last frag num:"<<fragNum<<" != num frags:"<<_numFrags<<endl;
+  } 
+  buffer.done();
+
+  return true;
+}
+
+bool FragLevelConvert::FragToLevel (unsigned long fragNum,
+				    unsigned long &levelDocNum) const {
+  // do binary search for something containing this fragment number
+  unsigned long low = 1, high = numLevelDocs;
+  unsigned long mid;
+
+  while ((mid = (low+high)/2) >= 1 && low <= high) {
+    if (fragNum > levelStarts[mid]) low = mid+1;
+    else if (fragNum <= levelStarts[mid-1]) high = mid-1;
+    else {
+      levelDocNum = mid;
+      return true;
+    }
+  }
+  return false;
+}
+
+
+bool FragLevelConvert::LevelToLevel (FragLevelConvert sectionLevelConvert,
+		 unsigned long levelNum, unsigned long &DocNum) {
+  
+  if (levelNum==1) {
+    DocNum=1;
+    return true;
+  }
+  unsigned long levelfragnum = sectionLevelConvert.levelStarts[levelNum-1]+1;
+
+  if (FragToLevel(levelfragnum, DocNum)) {
+    return true;
+  } 
+  return false;
+  
+
+}
Index: /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/FragLevelConvert.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.h -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FRAGLEVELCONVERT_H
+#define FRAGLEVELCONVERT_H
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+/*Notes about levelStarts (kjm18) - this is my interpretation of
+whats going on, feel free to correct this if its wrong.
+
+say you have 5 docs, consisting of fragments:
+1:  1-55
+2: 56-83
+3: 84-106
+4: 107-152
+5: 153-211
+
+then levelStarts would have the values
+0  1  2   3   4   5 
+0  55 83 106 152 211
+
+*/
+class FragLevelConvert {
+protected:
+  // note fragment start is one before before real start
+  vector<unsigned long> levelStarts;
+  unsigned long numFrags;
+  unsigned long numLevelDocs;
+  
+public:
+  FragLevelConvert ();
+
+  bool Read (FILE *invfFile, unsigned long levelPtr,
+	     unsigned long _numFrags, unsigned long _numLevelDocs);
+
+  bool FragToLevel (unsigned long fragNum, unsigned long &levelDocNum) const;
+
+  // this function converts from one level number to another
+  // the FragLevelConvert thats calling the method should be loaded with
+  // the level that you want to convert to, while the sectionLevelConvert
+  // should be loaded with the level converting from.
+  // level Num has the old level, DocNum will be returned with the new
+  // coresponding level
+  bool LevelToLevel (FragLevelConvert sectionLevelConvert,
+		   unsigned long levelNum, unsigned long &DocNum);
+};
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
@@ -0,0 +1,275 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size() + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //  UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParsePotentialInteger(UCArray::const_iterator &here,
+				  UCArray::const_iterator end,
+				  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen=0;
+  int length=0;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  // check the next character -if it is a letter, then have a term, not an integer
+  if (!is_unicode_letter(c)) {
+    // this was just an integer
+    return (!el.text.empty());
+  }
+  // else its a term
+  el.lexType = TermE;
+  el.num = 0;
+  /* this bit taken from ParseIndexWord in words.h*/
+  while (length+charLen<=MAXSTEMLEN && charLen > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  return (!el.text.empty());
+}
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+  const unsigned char* new_here = ParseIndexWord (&*here, endMinus1, text);
+  here += (new_here - &*here); // advance iterator by number of chars advanced
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+
+  } else if (c =='[') {
+    el.lexType = OpenSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c ==']') {
+    el.lexType = CloseSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '/') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '*') {
+    el.lexType = StarE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c=='&') {
+    el.lexType = AndOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '|') {
+    el.lexType = OrOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '!') {
+    el.lexType = NotOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ) {
+    return  ParseInteger (here, end, el);
+  }
+
+  else if (c >= '0' && c <= '9') {
+    return ParsePotentialInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text))  {
+    // parse term returns false if it hasn't parsed anything that is a term
+    // here should be the same as it was before 
+    el.lexType = UnknownE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  }
+  //return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+  UCArray NEAR; SetCStr (NEAR, "NEAR", 4);
+  if (PrefixLen(el.text, NEAR)==4) {
+    el.lexType = NearOpE;
+    return true;
+  }
+  UCArray WITHIN; SetCStr (WITHIN, "WITHIN", 6);
+  if (PrefixLen(el.text, WITHIN)==6) {
+    el.lexType = WithinOpE;
+    return true;
+  }
+  
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/GSDLQueryLex.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.h -- Lexical analyser for a simple query language, for GSDL
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYLEX_H
+#define GSDLQUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, NearOpE, WithinOpE, QuoteE, IntegerE, TermWeightE, 
+	      StemMethodE, RangeE, AtE, TagE, OpenSquareBracketE, 
+	      CloseSquareBracketE, StarE, UnknownE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
@@ -0,0 +1,515 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryParser.h"
+#include "GSDLQueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end, 
+				   int defaultBoolCombine,
+				   int defaultStemMethod);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end, 
+					  int defaultBoolCombine,
+					  int defaultStemMethod) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end, defaultBoolCombine,
+					defaultStemMethod);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+// default is within 20 words
+static void SetRangeValues (TermNode &termNode,
+			    UCArray &nearby,
+			    bool reverse) {
+  UCArray NEARBY; SetCStr(NEARBY, "NEAR", 4);
+  UCArray WITHIN; SetCStr(WITHIN, "WITHIN", 6);
+  
+  if (nearby == NEARBY) { // no modifier
+    termNode.startRange = (NEAR_DEFAULT+1)*-1;
+    termNode.endRange = NEAR_DEFAULT;
+
+  } else if (nearby == WITHIN) { // no modifier
+    if (reverse) {
+      termNode.startRange = (NEAR_DEFAULT+1)*-1;
+      termNode.endRange = -1;
+    } else {
+      termNode.startRange = NEAR_DEFAULT;
+      termNode.endRange = 0;
+    }
+  }
+  else { // extract number
+    UCArray::const_iterator here;
+    bool within = false;
+    if (PrefixLen(nearby, WITHIN)==6) {
+      within=true;
+      here = nearby.begin()+6;
+    } else {
+      here = nearby.begin()+4;
+    }
+    UCArray::const_iterator end = nearby.end();
+    int size=0;
+    while (here != end) {
+      size = size*10 + (*here-'0');
+      ++here;
+    }
+    if (within) {
+      if (reverse) {
+	termNode.startRange = size;
+	termNode.endRange = 0;
+      } else {
+	termNode.startRange = -1 * (size+1);
+	termNode.endRange = -1;
+      }
+    } else {
+      termNode.startRange = -1 * (size+1);
+      termNode.endRange = size;
+    }
+  }
+}
+
+static unsigned long GetStemMethod(LexEl &el, int defaultStemMethod) {
+  // here expect el to contain some of c,s,i,u
+  // stem method 0 = c,u  00
+  // stem method 1 = i,u  01  - default for DL
+  // stem method 2 = c, s  10
+  // stem method 3 = i,s  11
+
+  unsigned long stem = (unsigned long)defaultStemMethod;
+      
+  UCArray::const_iterator here = el.text.begin();
+  UCArray::const_iterator end = el.text.end();
+
+  unsigned char c1 = *here; 
+  if (!(c1 == 'c'|| c1 == 'i' || c1 == 'u' || c1 == 's')) 
+    return 4; // incorrect format
+  
+  ++here;
+  unsigned char c2 = 'a';
+  if (here !=end) {
+    c2 = *here;
+    if (!(c2 == 'c'|| c2 == 'i' || c2 == 'u' || c2 == 's')) 
+      return 4; // incorrect format
+  }
+  
+  if (c1 == 'i'|| c2=='i') stem |= 1; // set bit 0 to 1
+  if (c1 == 'c' || c2 == 'c') stem &=0xe; //set bit 0 to 0
+  if (c1 == 's'|| c2 == 's') stem |= 2; // set bit 1 to 1
+  if (c1 == 'u' || c2 =='u') stem &=0xd; // set bit 1 to 0
+  return stem;
+}
+  
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode,
+				int defaultStemMethod) {
+ 
+  termNode.stemMethod = defaultStemMethod;
+  bool partial_match = false;
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      oldHere = here;
+      LexEl stem;
+      if (ParseLexEl (here, end, stem) && stem.lexType == TermE) {
+	termNode.stemMethod = GetStemMethod(stem, defaultStemMethod);
+	if (termNode.stemMethod == 4) { // error so backtrack
+	  here = oldHere;
+	  termNode.stemMethod = (unsigned long)defaultStemMethod;
+	} 
+      }else here = oldHere; //ignore - wrong syntax
+
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+    } else if (el.lexType == StarE) {
+      partial_match = true;
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    if (partial_match) {
+      termNode.stemMethod |= 4; // set bit 2 to 1
+      termNode.stemMethod &=0xd; // set bit 1 to 0 // we dont have stemming on if doing partial matching.
+    }
+    oldHere = here;
+  }  
+}
+
+static void ParseProxModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				ProxMatchQueryNode *proxNode) {
+  // so far only have one - the tag stuff
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TagE) {
+      oldHere = here; // don't backtrack past here
+      if (ParseLexEl (here, end, el) && el.lexType == TermE) {
+	proxNode->tagNodePtr = new TagNode;
+	proxNode->tagNodePtr->tagName = el.text;
+	
+      }
+      else { // error in tag
+	here = oldHere;
+      }
+    } // TagE
+    // add in other cases here
+    else {
+      // no modifiers
+      here = oldHere;
+      break;
+    }
+    oldHere = here;
+  }//while
+
+
+}
+
+// expects starting brackets to have been parsed
+// sets error to true if something has gone wrong
+static ProxMatchQueryNode *ParseSquareBrackets(UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				/*ProxMatchQueryNode *proxNode,*/
+				int defaultStemMethod,
+				bool & error) {
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+  LexEl el;
+  bool phrase=false;
+  bool first=true;
+  bool prox = false;
+  UCArray near_string;
+  while (ParseLexEl (here, end, el)) {
+    // cant have AND, OR, NOT in square brackets, so assume they are words
+    if (el.lexType == TermE || el.lexType == IntegerE || el.lexType == AndOpE || el.lexType == OrOpE || el.lexType == NotOpE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (phrase) {
+        if (first) first=false;
+        else {
+          termNode.startRange = -2;
+          termNode.endRange = -1;
+        }
+      } else if (prox) {
+	SetRangeValues(termNode, near_string, false);
+	prox = false;
+      }
+      proxNode->terms.push_back(termNode);
+    }
+    else if (el.lexType == CloseSquareBracketE) {
+      break;
+    }
+    else if (el.lexType == QuoteE) {
+      // phrase inside square brackets
+      if (phrase) { // end of phrase
+	phrase=false;
+	first = true;
+      } else {
+	phrase=true; // start of phrase
+      }
+    } else if (el.lexType == NearOpE || el.lexType == WithinOpE) {
+      if (phrase) {
+	// cant have proximity  op in a phrase - just assume its an actual word
+	TermNode termNode;
+	termNode.term = el.text;
+	ParseTermModifiers (here, end, termNode, defaultStemMethod);
+	proxNode->terms.push_back(termNode);
+      } else {
+	// its a NEAR or within op
+	prox = true;
+	near_string = el.text;
+      }
+	  
+    } 
+    else if (el.lexType == UnknownE) {
+      // just ignore it 
+    }
+    else {
+      //error - we set the proxNode to NULL,
+      cerr <<"GSDLQueryParser: bad syntax inside []\n";
+      error = true;
+      return NULL;
+    }
+  } // while
+  return proxNode;
+}
+// expects the starting quote to have been parsed
+// and discarded
+// now phrases use the case and stem preference options
+// ie can search for a phrase ignoring case
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode, 
+			 int defaultStemMethod,
+			 bool &error) {
+  LexEl el;
+  bool first = true;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      //termNode.stemMethod = defaultStemMethod;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (first) {
+	first = false;
+      }
+      else {
+	termNode.startRange = -2;
+	termNode.endRange = -1;
+      }
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it
+    } else {
+      // error 
+      error = true;
+      return;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end,
+			     int defaultBoolCombine,
+			     int defaultStemMethod) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end, defaultBoolCombine,
+				   defaultStemMethod);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode, defaultStemMethod);
+    oldHere = here;  // dont backtrack past here
+    if (ParseLexEl(here, end, el) && (el.lexType == NearOpE || el.lexType == WithinOpE )) {
+	delete proxNode;
+	oldHere = here;
+      // this is calling ParseTerm again, but only a subset of the things accepted by ParseTerm are appropriate here. add in some hacks to avoid segmentation faults - kjdon, 04/2003
+      
+      // if the next element is a '(' have a syntax error, return NULL 
+      LexEl temp_el;
+      if (ParseLexEl(here, end, temp_el) && temp_el.lexType == OpenBracketE) {
+	cerr << "GSDLQueryParser: NEAR/WITHIN cannot be followed by a '('\n";
+	return NULL;
+      }
+      here = oldHere; // else backtrack
+	
+      proxNode = (ProxMatchQueryNode *)ParseTerm(here, end, defaultBoolCombine,
+						 defaultStemMethod);
+      SetRangeValues(termNode, el.text, true);
+      proxNode->terms.push_back (termNode);
+      return proxNode;
+      
+    } else {
+      here = oldHere; // backtrack
+      proxNode->terms.push_back (termNode);
+      ParseProxModifiers(here, end, proxNode);
+      return proxNode;
+    }
+  } else if (el.lexType == QuoteE) {
+    bool error = false;
+    ParsePhrase (here, end, *proxNode, defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    return proxNode;
+  }
+  else if (el.lexType == OpenSquareBracketE) {
+    bool error = false;
+    proxNode = ParseSquareBrackets (here, end, /*proxNode, */defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    ParseProxModifiers (here, end, proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end,
+				   int defaultBoolCombine,
+				   int defaultStemMethod) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) {
+      // parsebracketexpression is waiting for the last bracket, so put it back
+      here = oldHere;
+      break;
+      
+    } else if (el.lexType == OpenSquareBracketE ||
+	       el.lexType == OpenBracketE ||
+	       el.lexType == TermE ||
+	       el.lexType == QuoteE ||
+	       el.lexType == IntegerE ) { 
+      
+      // some type of term, back track and parse it
+      here = oldHere;
+
+      //  parse the term
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine, 
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+
+      // if default==1, AND, else if==0, OR
+      if (defaultBoolCombine) {
+	curTree = AndAdd (curTree, newTerm);
+      }
+      else {
+	curTree = OrAdd (curTree, newTerm); 
+      }
+      
+    } else if (el.lexType == AndOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = AndAdd (curTree, newTerm);
+      
+    } else if (el.lexType == OrOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = OrAdd (curTree, newTerm);
+      
+    } else if (el.lexType == NotOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = NotAdd (curTree, newTerm); 
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it 
+    } else {
+
+      // syntax error, return NUll
+      delete curTree;
+      return NULL;
+    }
+
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end, defaultBoolCombine, defaultStemMethod);
+}
Index: /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/GSDLQueryParser.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYPARSER_H
+#define GSDLQUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+#define NEAR_DEFAULT 20
+
+// returns NULL if the query could not be parsed
+// defaultBoolCombine determines if individual terms are combined using
+// AND or OR in the absence of specified operators.
+// defaultStemMethod used to set stemming and casefolding for terms where
+// its not set explicitly in the query string. THe defaultStemMethod value
+// should be set using the values from the preferences page
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/IndexData.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/IndexData.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/IndexData.cpp	(revision 21112)
@@ -0,0 +1,207 @@
+/**************************************************************************
+ *
+ * IndexData.cpp -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "IndexData.h"
+#include "string.h"
+#include "mg_files.h"
+
+IndexData::IndexData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  dictFile = NULL;
+
+  stem1File = NULL;
+  stem2File = NULL;
+  stem3File = NULL;
+
+  invfFile = NULL;
+
+  approxWeightsFile = NULL;
+  exactWeightsFile = NULL;
+
+  curLevelNum = 0;
+}
+
+IndexData::~IndexData () {
+  UnloadData ();
+}
+
+bool IndexData::LoadData (const char *_basePath, const char *_filename) {
+  if (_filename[0] == '\0') return false;
+  
+  // make sure this data has not already been loaded
+  if (strcmp (_basePath, basePath) == 0 &&
+      strcmp (_filename, filename) == 0) return true;
+  
+  // make sure there is nothing else loaded
+  UnloadData ();
+
+  strcpy (basePath, _basePath);
+  strcpy (filename, _filename);
+
+  set_basepath (basePath);
+
+  // load in the level information
+  FILE *ivfLevelFile;
+  ivfLevelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			    MAGIC_INVF_LEVELS, MG_ABORT);
+  levels.Read (ivfLevelFile);
+  fclose (ivfLevelFile);
+
+  // blocked dictionary
+  dictFile = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "rb",
+			MAGIC_STEM, MG_ABORT);
+  if (!bdh.Read (dictFile)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.wblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biWords)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.tblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biTags)) { UnloadData (); return false; }
+
+  // blocked stem index 1
+  stem1File = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+			 "rb", MAGIC_STEM_1, MG_ABORT);
+  if (!sih1.Read (stem1File)) { UnloadData (); return false; }
+
+  fseek (stem1File, sih1.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem1File, sii1)) { UnloadData (); return false; }
+  
+  // blocked stem index 2
+  stem2File = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+			 "rb", MAGIC_STEM_2, MG_ABORT);
+  if (!sih2.Read (stem2File)) { UnloadData (); return false; }
+
+  fseek (stem2File, sih2.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem2File, sii2)) { UnloadData (); return false; }
+  
+  // blocked stem index 3
+  stem3File = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+			 "rb", MAGIC_STEM_3, MG_ABORT);
+  if (!sih3.Read (stem3File)) { UnloadData (); return false; }
+
+  fseek (stem3File, sih3.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem3File, sii3)) { UnloadData (); return false; }
+
+  // inverted file
+  invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  ifh.Read (invfFile);
+
+  // weights
+  approxWeightsFile = open_file (filename, APPROX_WEIGHTS_SUFFIX, "rb",
+				 MAGIC_WGHT_APPROX, MG_ABORT);
+  exactWeightsFile = open_file (filename, WEIGHTS_SUFFIX, "rb",
+				MAGIC_WGHT, MG_ABORT);
+
+  return true;
+}
+
+bool IndexData::UnloadData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  if (dictFile != NULL) {
+    fclose (dictFile); dictFile = NULL;
+  }
+
+  if (stem1File != NULL) {
+    fclose (stem1File); stem1File = NULL;
+  }
+  if (stem2File != NULL) {
+    fclose (stem2File); stem2File = NULL;
+  }
+  if (stem3File != NULL) {
+    fclose (stem3File); stem3File = NULL;
+  }
+
+  if (invfFile != NULL) {
+    fclose (invfFile); invfFile = NULL;
+  }
+
+  if (approxWeightsFile != NULL) {
+    fclose (approxWeightsFile); approxWeightsFile = NULL;
+  }
+  if (exactWeightsFile != NULL) {
+    fclose (exactWeightsFile); exactWeightsFile = NULL;
+  }
+
+  UnloadLevel();
+  
+  return true;
+}
+
+
+bool IndexData::LoadLevel (const UCArray &level) {
+  // make sure this level is not already loaded
+  if (level == curLevel) return true;
+
+  // unload any levels currently loaded
+  UnloadLevel();
+
+  // make sure the required files are open
+  if (dictFile == NULL || invfFile == NULL ||
+      approxWeightsFile == NULL || exactWeightsFile == NULL)
+    return false;
+  
+  // read in the information from the dictionary
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (dictFile, biTags, bdh.entries_per_tblk,
+			  bdh.tag_dict_size, level, tagEl, tagElNum))
+    return false;
+
+  // read in the level conversion information
+  if (!levelConverter.Read (invfFile, tagEl.invf_ptr,
+			    bdh.num_frags, tagEl.frag_occur))
+    return false;
+
+  // read in the approximate weights
+  if (!weightData.Read (approxWeightsFile,
+			levels.levelInfo[level].approxWeightsDiskPtr,
+			levels.levelInfo[level].numEntries))
+    return false;
+  
+  // find the level number
+  curLevelNum = 0;
+  IvfLevelInfoMap::const_iterator levelHere, levelEnd;
+  for (levelHere=levels.levelInfo.begin(), levelEnd=levels.levelInfo.end();
+       levelHere!=levelEnd && (*levelHere).first != level; ++levelHere)
+    ++curLevelNum;
+
+  // make sure we found the level
+  if (levelHere == levelEnd) return false;
+
+  curLevel = level;
+
+  return true;
+}
+
+bool IndexData::UnloadLevel () {
+  curLevel.erase (curLevel.begin(), curLevel.end());
+  curLevelNum = 0;
+  
+  weightData.Free ();
+    
+  return true;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/IndexData.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/IndexData.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/IndexData.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * IndexData.h -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef INDEXDATA_H
+#define INDEXDATA_H
+
+#include <stdio.h>
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+#include "Weights.h"
+
+
+class IndexData {
+public:
+  // general information
+  char basePath[FILENAME_MAX];
+  char filename[FILENAME_MAX];
+  FIvfLevel levels;
+  
+  // blocked dictionary
+  FILE *dictFile;
+  block_dict_header bdh;
+  block_idx biWords;
+  block_idx biTags;
+
+  // stem indexes
+  FILE *stem1File;
+  FILE *stem2File;
+  FILE *stem3File;
+  stem_idx_header sih1;
+  stem_idx_header sih2;
+  stem_idx_header sih3;
+  block_idx sii1;
+  block_idx sii2;
+  block_idx sii3;
+
+  // inverted file
+  FILE *invfFile;
+  invf_file_header ifh;
+
+  // weights
+  FILE *approxWeightsFile;
+  FILE *exactWeightsFile;
+
+  // level specific information
+  UCArray curLevel; // last level that was loaded
+  unsigned long curLevelNum;
+  FragLevelConvert levelConverter;
+  ApproxWeightsData weightData;
+  
+  IndexData ();
+  ~IndexData ();
+
+  // LoadData will only load the data if it has not been loaded before
+  bool LoadData (const char *_basePath, const char *_filename);
+  bool UnloadData ();
+
+  bool LoadLevel (const UCArray &level);
+  bool UnloadLevel ();
+};
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/MGQuery.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/MGQuery.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/MGQuery.cpp	(revision 21112)
@@ -0,0 +1,760 @@
+/**************************************************************************
+ *
+ * MGQuery.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "Terms.h"
+#include "QueryResultsSort.h"
+#include <assert.h>
+
+
+void PrintIndent (ostream &s, int indent) {
+  while (indent-- > 0) s << " ";
+}
+
+
+void PrintIndentText (ostream &s, char *text, int indent) {
+  PrintIndent (s, indent);
+  s << text;
+}
+
+void PrintNode (ostream &s, QueryNode *node, int indent) {
+  if (node == NULL) {
+    PrintIndentText (s, "NULL\n", indent);
+  } else {
+    node->Print (s, indent+2);
+  }
+}
+
+  
+
+QueryNode::QueryNode () {
+}
+
+QueryNode::~QueryNode () {
+}
+
+void QueryNode::Calculate (IndexData &/*indexData*/,
+			   const QueryInfo &/*queryInfo*/,
+			   QueryResult &result) const {
+  result.Clear();
+}
+
+void QueryNode::Free () {
+}
+
+void QueryNode::Print (ostream &/*s*/, int /*indent*/) const {
+}
+
+
+
+AndQueryNode::AndQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+AndQueryNode::~AndQueryNode () {
+  Free ();
+}
+
+void AndQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // an And between nothing and something is nothing...
+  if (leftNode == NULL || rightNode == NULL) return;
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult rightResult;
+  leftNode->Calculate (indexData, queryInfo, result);
+  rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with an And
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long outI = 0;
+  while (leftI < result.docs.size() &&
+	 rightI < rightResult.docs.size()) {
+    if (result.docs[leftI] < rightResult.docs[rightI]) {
+      ++leftI;
+    } else if (result.docs[leftI] > rightResult.docs[rightI]) {
+      ++rightI;
+    } else {
+      // the documents are equal
+      result.docs[outI] = result.docs[leftI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[leftI] + rightResult.ranks[rightI];
+      ++leftI;
+      ++rightI;
+      ++outI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+    
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+}
+
+void AndQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void AndQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "AND\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+OrQueryNode::OrQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+OrQueryNode::~OrQueryNode () {
+  Free ();
+}
+
+void OrQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult leftResult;
+  QueryResult rightResult;
+  if (leftNode != NULL)
+    leftNode->Calculate (indexData, queryInfo, leftResult);
+  if (rightNode != NULL)
+    rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results
+  
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (leftResult.ranks.size() != leftResult.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftSize = leftResult.docs.size();
+  unsigned long rightSize = rightResult.docs.size();
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long leftDocNum = 0;
+  unsigned long rightDocNum = 0;
+  while (leftI < leftSize || rightI < rightSize) {
+    // check leftI
+    if (leftI < leftResult.docs.size())
+      leftDocNum = leftResult.docs[leftI];
+    else leftDocNum = ULONG_MAX;
+    
+    // check rightI
+    if (rightI < rightResult.docs.size())
+      rightDocNum = rightResult.docs[rightI];
+    else rightDocNum = ULONG_MAX;
+    
+    // combine
+    if (leftDocNum < rightDocNum) {
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI]);
+      ++leftI;
+      
+    } else if (leftDocNum > rightDocNum) {
+      result.docs.push_back (rightDocNum);
+      if (haveAccum)
+	result.ranks.push_back (rightResult.ranks[rightI]);
+      ++rightI;
+      
+    } else { // equal
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI] +
+				rightResult.ranks[rightI]);
+      ++leftI;
+      ++rightI;
+    }
+  }
+
+  // combine term frequency information
+  if (queryInfo.needTermFreqs) {
+    result.termFreqs.insert (result.termFreqs.end(),
+			     leftResult.termFreqs.begin(),
+			     leftResult.termFreqs.end());
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+  }
+}
+
+void OrQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void OrQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "OR\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+
+NotQueryNode::NotQueryNode () {
+  queryNode = NULL;
+  notNode = NULL;
+}
+
+NotQueryNode::~NotQueryNode () {
+  Free ();
+}
+
+void NotQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // check for nothing
+  if (queryNode == NULL) return;
+  if (notNode == NULL) {
+    queryNode->Calculate (indexData, queryInfo, result);
+    return;
+  }
+  
+  // calculate the result from the query tree and the result
+  // from the not tree
+  QueryResult notResult;
+  queryNode->Calculate (indexData, queryInfo, result);
+  notNode->Calculate (indexData, queryInfo, notResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with a Not
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    notResult.ranks.size() != notResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long queryI = 0;
+  unsigned long notI = 0;
+  unsigned long outI = 0;
+  while (queryI < result.docs.size() &&
+	 notI < notResult.docs.size()) {
+    if (result.docs[queryI] < notResult.docs[notI]) {
+      // found a document not in the notResults
+      result.docs[outI] = result.docs[queryI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[queryI];
+      ++queryI;
+      ++outI;
+    } else if (result.docs[queryI] > notResult.docs[notI]) {
+      ++notI;
+    } else {
+      // the documents are equal, ignore both
+      ++queryI;
+      ++notI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+  
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     notResult.termFreqs.begin(),
+			     notResult.termFreqs.end());
+}
+
+void NotQueryNode::Free () {
+  if (queryNode != NULL) {
+    delete queryNode;
+    queryNode = NULL;
+  }
+  if (notNode != NULL) {
+    delete notNode;
+    notNode = NULL;
+  }
+}
+
+void NotQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "queryNode:\n", indent);
+  PrintNode (s, queryNode, indent+2);
+  PrintIndentText (s, "NOT\n", indent);
+  PrintIndentText (s, "notNode:\n", indent);
+  PrintNode (s, notNode, indent+2);
+}
+
+
+void TagNode::Clear () {
+  UCArrayClear (tagName);
+}
+
+void TagNode::Calculate (IndexData &indexData,
+			 FragRangeArray &fragRange) const {
+  fragRange.erase (fragRange.begin(), fragRange.end());
+  if (tagName.empty()) return;
+
+  // get information about this tag
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (indexData.dictFile, indexData.biTags,
+			  indexData.bdh.entries_per_tblk,
+			  indexData.bdh.tag_dict_size,
+			  tagName, tagEl, tagElNum))
+    return;
+
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, tagEl.invf_ptr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(indexData.invfFile);
+  
+  unsigned long pTag = tagEl.frag_occur*2;
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  FragRange thisFrag;
+  for (i=0; i<tagEl.frag_occur; ++i) {
+    // get start
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeStart = fragNum;
+
+    // get end
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeEnd = fragNum;
+    fragRange.push_back (thisFrag);
+  }
+
+  buffer.done();
+}
+
+void TagNode::Free () {
+  Clear ();
+}
+
+void TagNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TAG: \"" << tagName << "\"\n";
+}
+
+
+void TermNode::Clear () {
+  UCArrayClear (term);
+  termWeight = 1;
+  stemMethod = 0;
+  startRange = NO_TERM_RANGE_START;
+  endRange = NO_TERM_RANGE_END;
+}
+
+TermNode::TermNode () {
+  Clear ();
+}
+
+void TermNode::Calculate (IndexData &indexData,
+			  bool needFragFreqs,
+			  FragRangeArray *fragLimits,
+			  FragData &fragData,
+			  UCArrayVector &equivTerms) const {
+  fragData.Clear ();
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+
+  // get a list of term numbers
+  vector<unsigned long> equivNums;
+  FindWordNumbers (indexData, term, stemMethod, equivNums);
+
+  // get the information for each word and merge it with
+  // previous results
+  FragData tempFragData1;
+  FragData tempFragData2;
+  UCArray equivWord;
+  vector<unsigned long>::iterator here = equivNums.begin();
+  vector<unsigned long>::iterator end = equivNums.end();
+  while (here != end) {
+    // get the information for this word
+    ReadTermFragData (indexData, needFragFreqs, *here,
+		      tempFragData1, fragLimits, equivWord);
+    equivTerms.push_back(equivWord);
+    // combine with last results
+    tempFragData2 = fragData;
+    CombineFragData (needFragFreqs, tempFragData1, tempFragData2, fragData);
+    
+    ++here;
+  }
+}
+
+void TermNode::Free () {
+  Clear ();
+}
+
+void TermNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TERM: \"" << term << "\"\n";
+  PrintIndent (s, indent+2);
+  s << "termWeight: " << termWeight << "\n";
+  PrintIndent (s, indent+2);
+  s << "stemMethod: " << stemMethod << "\n";
+  PrintIndent (s, indent+2);
+  s << "startRange: " << startRange << "\n";
+  PrintIndent (s, indent+2);
+  s << "endRange: " << endRange << "\n";
+}
+
+
+ProxMatchQueryNode::ProxMatchQueryNode () {
+  tagNodePtr = NULL;
+}
+
+ProxMatchQueryNode::~ProxMatchQueryNode () {
+  Free ();
+}
+
+void ProxMatchQueryNode::Calculate (IndexData &indexData,
+				    const QueryInfo &queryInfo,
+				    QueryResult &result) const {
+  result.Clear ();
+
+  bool needFragFreqs = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  // read in the tag if needed
+  FragRangeArray fragLimits;
+  FragRangeArray *fragLimitsPtr = NULL;
+  if (tagNodePtr == NULL && terms.size() > 1) {
+    // multiple terms must be compared relative to some tag
+    // otherwise phrase matches could span documents
+    TagNode tempTagNode;
+    tempTagNode.tagName = indexData.curLevel;
+    tempTagNode.Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+    
+  } else if (tagNodePtr != NULL) {
+    (*tagNodePtr).Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+  }
+
+  UCArray tagOrLevel = indexData.curLevel;
+  if (tagNodePtr != NULL) tagOrLevel = (*tagNodePtr).tagName;
+
+  // read in the first term
+  FragData termData;
+  UCArrayVector equivTerms;
+  TermNodeArray::const_iterator termHere=terms.begin(), termEnd = terms.end();
+  if (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs, fragLimitsPtr, 
+			   termData, equivTerms);
+
+    // convert initial fragment information
+    FragsToQueryResult (indexData,
+			queryInfo,
+			termData,
+			tagOrLevel,
+			(*termHere).term,
+			(*termHere).stemMethod,
+			(*termHere).termWeight,
+			equivTerms,
+			result);
+  
+    ++termHere;
+
+    if (termHere == termEnd) return; // nothing more to do
+  }
+
+  // read and combine the rest of the terms
+  FragData comTermData;
+  while (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs,
+			   fragLimitsPtr, comTermData, equivTerms);
+
+    AndFragsToQueryResult (indexData,
+			   queryInfo,
+			   comTermData,
+			   tagOrLevel,
+			   (*termHere).term,
+			   (*termHere).stemMethod,
+			   (*termHere).termWeight,
+			   equivTerms,
+			   result);
+    
+    AndCombineFragData (needFragFreqs, termData, comTermData,
+			(*termHere).startRange,
+			(*termHere).endRange,
+			fragLimitsPtr);
+    ++termHere;
+  }
+
+  // remove unwanted document numbers
+  RemoveUnwantedResults (indexData,
+			 queryInfo,
+			 termData,
+			 result);
+}
+
+void ProxMatchQueryNode::Free () {
+  if (tagNodePtr != NULL) {
+    delete tagNodePtr;
+    tagNodePtr = NULL;
+  }
+  terms.erase (terms.begin(), terms.end());
+}
+
+void ProxMatchQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "PROXMATCH\n", indent);
+  if (tagNodePtr != NULL) tagNodePtr->Print (s, indent+2);
+  
+  TermNodeArray::const_iterator here = terms.begin();
+  TermNodeArray::const_iterator end = terms.end();
+  while (here != end) {
+    (*here).Print (s, indent+2);
+    ++here;
+  }
+}
+
+
+void BrowseQueryNode::Clear () {
+  UCArrayClear(term);
+}
+
+void BrowseQueryNode::Calculate (IndexData &indexData, BrowseQueryResult &result) const {
+
+  unsigned long number=0;
+  FindNearestWordNumber(indexData, term, number);
+  if (number + startPosition > 0 ) {
+    number = number+startPosition;
+  }
+  else {
+    number = 1;
+  }
+
+  GetTermList (indexData, number, numTerms, result.termFreqs);
+
+}
+
+
+
+void BrowseQueryNode::Free () {
+  Clear();
+}
+
+
+void BrowseQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText(s, "BROWSEQUERYNODE\n", indent);
+  PrintIndent (s, indent+2);
+  s << "TERM:"<<term<<"\n";
+  PrintIndent (s, indent+2);
+  s << "Start position: "<< startPosition<<", Num terms: "<< numTerms<<"\n";
+
+
+}
+
+
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result) {
+  result.Clear ();
+
+  // make sure level is current
+  if (!indexData.LoadLevel (queryInfo.docLevel)) {
+    return;
+  }
+  // do query
+  if (queryTree == NULL) return;
+  
+  (*queryTree).Calculate (indexData, queryInfo, result);
+
+  // make weights into ranks if needed
+  unsigned long i;
+  if (queryInfo.sortByRank || queryInfo.needRankInfo) {
+    for (i=0; i<result.ranks.size(); ++i) {
+      result.ranks[i] /=
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]);
+    }
+  }
+  
+  unsigned long resultsSize = queryInfo.maxDocs;
+  if (resultsSize == 0 || resultsSize > result.docs.size())
+    resultsSize = result.docs.size();
+
+  result.actualNumDocs = result.docs.size(); // the total number of docs 
+                                   //returned, before pruning based on maxDocs
+
+  // sort results by rank if needed
+  GTRank gtRank;
+  if (queryInfo.sortByRank) {
+    // need in ascending order for SelectAddHeap
+    MakeHeap (result.docs.begin(), result.ranks.begin(),
+	      resultsSize, gtRank);
+    SelectAddHeap (result.docs.begin(), result.ranks.begin(), resultsSize,
+		   result.docs.begin()+resultsSize,
+		   result.ranks.begin()+resultsSize,
+		   (unsigned long) result.ranks.size()-resultsSize,
+		   gtRank);
+
+    // sort into descending order
+    SortHeap (result.docs.begin(), result.ranks.begin(), resultsSize, gtRank);
+  }
+  
+  // get exact weights if needed
+  if (queryInfo.exactWeights &&
+      (queryInfo.sortByRank || queryInfo.needRankInfo)) {
+    unsigned long exactDiskPtr =
+      indexData.levels.levelInfo[indexData.curLevel].exactWeightsDiskPtr;
+    
+    for (i=0; i<resultsSize; ++i) {
+      result.ranks[i] =  result.ranks[i] *
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]) /
+	GetExactDocWeight (indexData.exactWeightsFile, exactDiskPtr,
+			   result.docs[i]);
+    }
+
+    // re-sort top candidates based on exact weights
+    if (queryInfo.sortByRank) {
+      MakeHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+      SortHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+    }
+  }
+
+  // get rid of unwanted ranking results
+  if (result.ranks.empty()) {
+    // do nothing
+  } else if (!queryInfo.needRankInfo) {
+    result.ranks.erase(result.ranks.begin(), result.ranks.end());
+  } else {
+    result.ranks.erase(result.ranks.begin()+resultsSize, result.ranks.end());
+  }
+
+  // remove extra docs that are unwanted
+  result.docs.erase(result.docs.begin()+resultsSize, result.docs.end());
+}
+
+// new MGQuery to retrieve doc and section nums
+// this will return doc nums for the level queried at (set in queryInfo)
+// in QueryResult.docs and if a second level is specified, 
+// it will return corresponding docnums for that level in QueryResult.levels
+// If there is no level specified, or that level is invalid, the query
+// level is used
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &realresult, UCArray &level) {
+  realresult.Clear ();
+  QueryResult result; // temp result
+
+  // do the normal query
+  MGQuery (indexData, queryInfo, queryTree, result);
+
+  // now that have the final result stuff, convert to ExtQueryResult,
+  // add in level nums if needed
+
+  realresult.docs = result.docs;
+  realresult.ranks = result.ranks;
+  realresult.termFreqs = result.termFreqs;
+  realresult.actualNumDocs = result.actualNumDocs;
+
+  if (queryInfo.docLevel == level || level.empty()) {
+    realresult.levels = result.docs;
+    return;
+  }
+
+  // else need to convert from queryInfo.docLevel to level
+
+  // the original level info
+  FragLevelConvert sectionLevelConverter = indexData.levelConverter;
+
+  // the new level info
+  if (!indexData.LoadLevel(level)) {
+    realresult.levels = result.docs;
+    return;
+  }
+  
+  unsigned long DocNum = 0;
+  
+  for (unsigned long i=0; i<realresult.docs.size(); ++i) {
+
+    // do an if ! here????
+    indexData.levelConverter.LevelToLevel(sectionLevelConverter, realresult.docs[i], DocNum);
+    realresult.levels.push_back(DocNum);
+  }
+  
+}
+
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result) {
+
+  indexData.LoadLevel(level);
+  node.Calculate(indexData, result); 
+
+}
Index: /main/tags/3.01/indexers/mgpp/text/MGQuery.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/MGQuery.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/MGQuery.h	(revision 21112)
@@ -0,0 +1,197 @@
+/**************************************************************************
+ *
+ * MGQuery.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MGQUERY_H
+#define MGQUERY_H
+
+#include "IndexData.h"
+#include <limits.h>
+#include "Terms.h"
+
+// forms base for defining and calculating the boolean
+// part of a query
+class QueryNode {
+public:
+  QueryNode ();
+  virtual ~QueryNode ();
+
+  virtual void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			  QueryResult &result) const;
+  virtual void Free ();
+  virtual void Print (ostream &s, int indent=0) const;
+};
+
+
+
+void PrintIndent (ostream &s, int indent);
+void PrintIndentText (ostream &s, char *text, int indent);
+void PrintNode (ostream &s, QueryNode *node, int indent=0);
+
+
+
+// basic boolean nodes
+
+class AndQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  AndQueryNode ();
+  ~AndQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class OrQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  OrQueryNode ();
+  ~OrQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class NotQueryNode : public QueryNode {
+public:
+  QueryNode *queryNode;
+  QueryNode *notNode;
+  
+  NotQueryNode ();
+  ~NotQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+
+class TagNode {
+public:
+  UCArray tagName;
+
+  void Clear ();
+  TagNode () { Clear (); }
+
+  void Calculate (IndexData &indexData, FragRangeArray &fragRange) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+#define NO_TERM_RANGE_START (LONG_MIN/2)
+#define NO_TERM_RANGE_END   (LONG_MAX/2)
+
+/* NOTE: range stuff - the range values are for the previous term relative
+to the current term. So if searching for the phrase "the cat", 'the' doesn't 
+need range limits, but 'cat' has a range of -2 to -1. ie, if have found 'cat'
+then 'the' has to be at position between -2 and -1 relative to 'cat'.
+"the cat" could also be searched for by 'cat' with no range limits, then 'the' with range 0 to 1.
+range values are relative to the gaps between words:
+   x   y   z   X   a   b   c
+    -3   -2  -1  0   1   2   3
+
+ */
+class TermNode {
+public:
+  UCArray term;
+  unsigned long termWeight;
+  unsigned long stemMethod;
+  signed long startRange; // range relative to last term
+  signed long endRange;
+  
+  void Clear ();
+  TermNode ();
+
+  // fragLimits can be NULL if not neede
+  void Calculate (IndexData &indexData,
+		  bool needFragFreqs,
+		  FragRangeArray *fragLimits,
+		  FragData &fragData,
+		  UCArrayVector &equivTerms) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+typedef vector<TermNode> TermNodeArray;
+
+
+class ProxMatchQueryNode : public QueryNode {
+public:
+  TagNode *tagNodePtr; // optional field
+  TermNodeArray terms;
+  
+  ProxMatchQueryNode ();
+  ~ProxMatchQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class BrowseQueryNode :public QueryNode {
+ public:
+  UCArray term;
+  signed long startPosition;
+  unsigned long numTerms;
+
+  void Clear();
+  BrowseQueryNode () { Clear(); }
+  //  ~BrowseQueryNode ();
+
+  void Calculate (IndexData &indexData, BrowseQueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+
+
+
+
+};
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result);
+
+// this function for retrieving results with both section doc nums
+// and Document docnums
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &result, UCArray &level);
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/Makefile.in
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Makefile.in	(revision 21112)
@@ -0,0 +1,318 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MGPP system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../lib/libmgpplib.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+GSDLOS = @gsdlos@
+
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CXXFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) $<
+
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+INCLUDES = -I.. -I../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp \
+	mgpp_passes_4jni.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h  mgpp_passes_4jni.h
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgpp_passes mgpp_decompress_text mgpp_perf_hash_build \
+                mgpp_invf_dump mgpp_weights_build mgpp_invf_dict mgpp_stem_idx \
+                mgpp_invf_dict_dump mgpp_compression_dict mgpp_fast_comp_dict \
+                QueryTester Queryer
+
+MAN           =  mgpp_weights_build.1  mgpp_invf_dict.1 \
+                 mgpp_passes.1 mgpp_perf_hash_build.1 \
+                 mgpp_compression_dict.1  mgpp_fast_comp_dict.1 \
+                 mgpp_stem_idx.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libmgpptext.a libmgpppass.a
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CXXFLAGS) $(CFLAGS) -- $(SOURCES)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall
+uninstall: execuninstall
+
+execinstall: $(EXEC)
+	../mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done 
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libmgpptext.a: $(LIB_OBJS)
+	rm -f libmgpptext.a
+	$(AR) cru libmgpptext.a $(LIB_OBJS)
+	$(RANLIB) libmgpptext.a
+
+JNI_LIB_OBJS = \
+	mgpp_passes_4jni$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+libmgpppass.a: $(JNI_LIB_OBJS)
+	rm -f libmgpppass.a
+	$(AR) cru libmgpppass.a $(JNI_LIB_OBJS)
+	$(RANLIB) libmgpppass.a
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+
+clean: mostlyclean
+	rm -f $(EXEC) libmgpptext.a libmgpppass.a
+
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+movclean:
+	for file in $(SOURCES); do \
+          test = $$file; \
+          echo $("test.c"##*.c); \
+        done 
+
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
Index: /main/tags/3.01/indexers/mgpp/text/QueryLex.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryLex.cpp	(revision 21112)
@@ -0,0 +1,175 @@
+/**************************************************************************
+ *
+ * QueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  // read in number part
+  el.num = 0;
+  el.lexType = IntegerE;
+  while (c >= '0' && c <= '9') {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  UCArray::const_iterator endMinus1 = end-1;
+  here = ParseIndexWord (here, endMinus1, text);
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+  
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '$') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ||
+	     (c >= '0' && c <= '9')) {
+    return ParseInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text)) return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {  
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/QueryLex.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryLex.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryLex.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * QueryLex.h -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYLEX_H
+#define QUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, QuoteE, IntegerE, TermWeightE, StemMethodE,
+	      RangeE, AtE, TagE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/QueryParser.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryParser.cpp	(revision 21112)
@@ -0,0 +1,219 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryParser.h"
+#include "QueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      termNode.stemMethod = ParseInt (here, end);
+      
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+      
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    oldHere = here;
+  }  
+}
+
+// expects the starting quote to have been parsed
+// and discarded
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode) {
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode);
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else {
+      // error
+      break;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  // check for a tag
+  if (el.lexType == TagE) {
+    oldHere = here; // don't backtrack past here
+    if (!ParseLexEl (here, end, el)) return NULL;
+    if (el.lexType == TermE) {
+      proxNode->tagNodePtr = new TagNode;
+      proxNode->tagNodePtr->tagName = el.text;
+      if (!ParseLexEl (here, end, el)) return NULL;
+    }
+  }
+  
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode);
+    proxNode->terms.push_back (termNode);
+    return proxNode;
+    
+  } else if (el.lexType == QuoteE) {
+    ParsePhrase (here, end, *proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == OpenBracketE ||
+	el.lexType == TermE ||
+	el.lexType == QuoteE ||
+	el.lexType == IntegerE ||
+	el.lexType == TagE) {
+      // some type of term, back track and parse it
+      here = oldHere;
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == AndOpE) {
+      curTree = AndAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == OrOpE) {
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == NotOpE) {
+      curTree = NotAdd (curTree, ParseTerm (here, end));
+      
+    } else break;
+    
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end);
+}
Index: /main/tags/3.01/indexers/mgpp/text/QueryParser.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryParser.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryParser.h	(revision 21112)
@@ -0,0 +1,31 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYPARSER_H
+#define QUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+// returns NULL if the query could not be parsed
+QueryNode *ParseQuery (const UCArray &queryStr);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/QueryResultsSort.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryResultsSort.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryResultsSort.h	(revision 21112)
@@ -0,0 +1,140 @@
+/**************************************************************************
+ *
+ * QueryResultsSort.h -- Functions to sort query results and parallel data
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYRESULTSSORT_H
+#define QUERYRESULTSSORT_H
+
+#include "Terms.h"
+
+
+struct LTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 < f2); }
+};
+
+struct GTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 > f2); }
+};
+
+
+
+// assumes the rank vector is the same size as the document
+// number vector. assumes the value to be pushed is in
+// docNumHeap + heapSize - 1 and rankHeap + heapSize - 1
+// topIndex can be set
+template <class RandomAccessIterator, class Compare>
+void PushHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long heapSize,
+	       const Compare &comp) {
+  unsigned long holeIndex = heapSize-1;
+  unsigned long parent = (holeIndex - 1) / 2;
+  while (holeIndex > 0 && comp (*(parallelHeap+parent),
+				*(parallelHeap+holeIndex))) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+parent));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+parent));
+    holeIndex = parent;
+    parent = (holeIndex - 1) / 2;
+  }
+}
+
+// AdjustHeap is used to add a element to a heap after one
+// element in the heap has been replaced with a new one
+template <class RandomAccessIterator, class Compare>
+void AdjustHeap (DocNumArray::iterator docNumHeap,
+		     RandomAccessIterator parallelHeap,
+		     unsigned long holeIndex,
+		     unsigned long heapSize,
+		     const Compare &comp) {
+  unsigned long secondChild = 2 * holeIndex + 2;
+  while (secondChild < heapSize) {
+    if (comp(*(parallelHeap + secondChild),
+	     *(parallelHeap + (secondChild - 1))))
+      --secondChild;
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+secondChild));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+secondChild));
+    holeIndex = secondChild;
+    secondChild = 2 * secondChild + 2;
+  }
+  if (secondChild == heapSize) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+(secondChild-1)));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+(secondChild-1)));
+    holeIndex = secondChild - 1;
+  }
+  PushHeap (docNumHeap, parallelHeap, holeIndex+1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void PopHeap (DocNumArray::iterator docNumHeap,
+	      RandomAccessIterator parallelHeap,
+	      unsigned long heapSize,
+	      const Compare &comp) {
+  swap (*(parallelHeap), *(parallelHeap+heapSize-1));
+  swap (*(docNumHeap), *(docNumHeap+heapSize-1));
+  AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize-1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void MakeHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+  
+  unsigned long i = 0;
+  for (i=1; i<=size; ++i)
+    PushHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+ 
+template <class RandomAccessIterator, class Compare>
+void SortHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+
+  unsigned long i;
+  for (i=size; i>1; --i)
+    PopHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+// uses the heap to get the top "size" elements
+template <class RandomAccessIterator, class Compare>
+void SelectAddHeap (DocNumArray::iterator docNumHeap,
+		    RandomAccessIterator parallelHeap,
+		    unsigned long heapSize,
+		    DocNumArray::iterator docNumAdd,
+		    RandomAccessIterator parallelAdd,
+		    unsigned long addSize,
+		    const Compare &comp) {
+  unsigned long i;
+  for (i=0; i<addSize; ++i) {
+    if (comp(*(parallelAdd+i), *(parallelHeap))) {
+      swap (*(docNumHeap), *(docNumAdd+i));
+      swap (*(parallelHeap), *(parallelAdd+i));
+      AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize, comp);
+    }
+  }
+}
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/QueryTester.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/QueryTester.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/QueryTester.cpp	(revision 21112)
@@ -0,0 +1,447 @@
+/**************************************************************************
+ *
+ * QueryTester.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+
+
+class SetQueryNode : public QueryNode {
+public:
+  QueryResult queryResult;
+  
+  SetQueryNode () {}
+  ~SetQueryNode () {}
+
+  void Calculate (IndexData &/*indexData*/, const QueryInfo &queryInfo,
+		  QueryResult &result) const {
+    result = queryResult;
+    if (!queryInfo.sortByRank && !queryInfo.needRankInfo)
+      result.ranks.erase (result.ranks.begin(), result.ranks.end());
+    if (!queryInfo.needTermFreqs)
+      result.termFreqs.erase (result.termFreqs.begin(),
+			      result.termFreqs.end());
+  }
+  void Free () { queryResult.Clear(); }
+};
+
+
+bool Test1 () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (15);  rcRankSet.push_back (0.2+0.6);
+  rcDocSet.push_back (19);  rcRankSet.push_back (1.4);
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test1b () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (15); 
+  rcDocSet.push_back (19); 
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2 () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (2);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (11); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (12); rcRankSet.push_back (0.3f);
+  rcDocSet.push_back (13); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (14); rcRankSet.push_back (0.5f);
+  rcDocSet.push_back (15); rcRankSet.push_back (0.2f+0.6f);
+  rcDocSet.push_back (16); rcRankSet.push_back (0.7f);
+  rcDocSet.push_back (17); rcRankSet.push_back (0.8f);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (19); rcRankSet.push_back (0.9f+0.5f);
+  rcDocSet.push_back (20); rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (21); rcRankSet.push_back (0.2f);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2b () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2);
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1);
+  rcDocSet.push_back (2); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (11);
+  rcDocSet.push_back (12);
+  rcDocSet.push_back (13);
+  rcDocSet.push_back (14);
+  rcDocSet.push_back (15);
+  rcDocSet.push_back (16);
+  rcDocSet.push_back (17);
+  rcDocSet.push_back (18);
+  rcDocSet.push_back (19);
+  rcDocSet.push_back (20);
+  rcDocSet.push_back (21);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3 () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3b () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (18);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+
+int main () {
+  if (!Test1 () ||
+      !Test2 () ||
+      !Test2b () ||
+      !Test3 () ||
+      !Test3b ()) return 1;
+  
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/Queryer.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Queryer.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Queryer.cpp	(revision 21112)
@@ -0,0 +1,271 @@
+/**************************************************************************
+ *
+ * Queryer.cpp -- simple interactive query program
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "MGQuery.h"
+#include "TextGet.h"
+
+#include "messages.h"
+#include "mg_files.h"
+
+#include "GSDLQueryParser.h"
+
+void printHelp() {
+
+  cout << "commands available are:\n"
+       << "\t.q\t\tquit\n"
+       << "\t.h\t\tprint the help message\n"
+       << "\t.i\t\tchange the search level (enter the new level at the prompt)\n"
+       << "\t.l\t\tchange the result level ( \"\"   \"\"  )\n"
+       << "\t.b\t\tfull text browse (enter a word or fragment at the prompt)\n"
+       << "\t.r0/.r1\t\tranking off/on\n"
+       << "\t.t0/.t1\t\tquery type some/all\n"
+       << "\t.c0/.c1\t\tcasefolding off/on\n"
+       << "\t.s0/.s1\t\tstemming off/on\n"
+       << "\t.o0/.o1\t\tshort output off/on\n\n"
+       << "\t.p\t\tprint a document (enter the docnum at the prompt)\n"
+       << "otherwise just enter a query\n\n";
+
+}
+
+int main (int argc, char **argv) {
+  int ch;
+  char *textfilename = "";
+  char *indexfilename = "";
+  char *basePath = "";
+  
+  opterr = 0;
+  msg_prefix = argv[0];
+  
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:t:d:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      indexfilename = optarg;
+      break;
+    case 't':
+      textfilename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (basePath);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-d directory] -f indexname -t textname\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  if (textfilename[0] == '\0' || indexfilename[0] == '\0') {
+    FatalError (1, "Index and text file names must be specified with -f and -t \n");
+  }
+  
+  // init the text system
+    TextData textData;
+  if (!textData.LoadData (basePath, textfilename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", textfilename);
+  }
+  
+  // init the query system
+  IndexData indexData;
+  if (!indexData.LoadData (basePath, indexfilename)) {
+    FatalError (1, "Couldn't load index information for \"%s\"", indexfilename);
+  }
+
+  // debug output
+  cerr << "num docs: "<<indexData.bdh.num_docs       
+       << "\nnum frags: "<<indexData.bdh.num_frags
+       << "\nnum words: "<<indexData.bdh.num_words
+       << "\ntotal bytes: "<<indexData.bdh.total_bytes
+       << "\nindex string bytes: "<<indexData.bdh.index_string_bytes
+       << "\nnum levels: "<<indexData.bdh.num_levels<<endl;
+  
+  // do querying
+  QueryInfo queryInfo;
+  SetCStr (queryInfo.docLevel, "Document", 8);
+  queryInfo.maxDocs = 50;
+  queryInfo.sortByRank = true;
+  queryInfo.exactWeights = false;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  ExtQueryResult queryResult;
+  char query[2048];
+  UCArray queryArray;
+  QueryNode *queryTree = NULL;
+
+
+  UCArray docLevel;
+  SetCStr(docLevel, "Document", 8);
+
+  UCArray level;
+  UCArrayClear(level);
+  //SetCStr(level, "");
+   
+  int defaultStemMethod = 0; // uncasefolded, unstemmed
+  int defaultBoolCombine = 0; // OR
+  bool shortOutput = false;
+  BrowseQueryNode browseNode;
+  browseNode.startPosition = -10;
+  browseNode.numTerms = 40;
+
+  BrowseQueryResult browseResult;
+
+  while (true) {
+    cout << "> ";
+    cin.getline(query, 2048, '\n');
+    SetCStr (queryArray, query, strlen(query));
+
+    // check for commands
+    if (queryArray.size() >= 2 && queryArray[0] == '.') {
+      if (queryArray[1] == 'q') break;  // quit
+
+      if (queryArray[1] == 'h') { // help
+	printHelp();
+      } else if (queryArray[1] == 'i') {
+	cout << "current index="<< queryInfo.docLevel << "\nchange to index:";
+	cin >> query;
+	UCArrayClear(queryInfo.docLevel);
+	SetCStr(queryInfo.docLevel, query, strlen(query));
+	cout << "index set to " << queryInfo.docLevel <<"\n";
+	cin.getline(query, 2048, '\n');
+      } else if (queryArray[1] == 'l') {
+	cout << "current level="<< level << "\nchange to level:";
+	cin >> query;
+	UCArrayClear(level);
+	SetCStr(level, query, strlen(query));
+	cout << "level set to " << level <<"\n";
+	cin.getline(query, 2048, '\n');
+      }
+
+
+      else if (queryArray[1] == 'p') {
+	// print
+	UCArray docText;
+	unsigned long docNum = 0;
+	cin >> docNum;
+	cin.getline(query, 2048, '\n'); // eat up return
+	
+	if (!GetDocText (textData, queryInfo.docLevel, docNum, docText)) {
+	  FatalError (1, "Error while trying to get document %u", docNum);
+	}
+	
+	cout << docText << "\n";
+      }
+      else if (queryArray[1] == 't') { // query type - all/some
+	if (queryArray[2] == '1') defaultBoolCombine = 1;
+	else if (queryArray[2] == '0') defaultBoolCombine = 0;
+	else {
+	  cout << "Error: please enter .t0 (some) or .t1 (all)\n";
+	}
+      }
+      else if (queryArray[1] == 'r') { // ranking - on/off
+	if (queryArray[2] == '1') queryInfo.sortByRank = true;
+	else if (queryArray[2] == '0') queryInfo.sortByRank = false;
+	else {
+	  cout << "Error: please enter .r0 (non-ranked) or .r1 (ranked)\n";
+	}
+      }
+      else if (queryArray[1] == 'c') { // casefolding - on/off
+	if (queryArray[2] == '1') defaultStemMethod |= 1;
+	else if (queryArray[2] == '0') defaultStemMethod &= 0xe;
+	else {
+	  cout << "Error: please enter .c0 (case sensitive) or .c1 (casefolded)\n";
+	}
+      }
+      else if (queryArray[1] == 's') { // stemming - on/off
+	if (queryArray[2] == '1') defaultStemMethod |=2;
+	else if (queryArray[2] == '0') defaultStemMethod &=0xd;
+	else {
+	  cout << "Error: please enter .s0 (unstemmed) or .s1 (stemmed)\n";
+	}
+      }      
+      else if (queryArray[1] == 'o') { // output - short/long
+	if (queryArray[2] == '1') shortOutput = true;
+	else if (queryArray[2] == '0') shortOutput = false;
+	else {
+	  cout << "Error: please enter .o0 (long output) or .o1 (short output)\n";
+	}
+      }
+      else if (queryArray[1] == 'b') {
+	// full text browse
+	cout<<"enter a few letters to start browsing from:";
+	cin>>query;
+	UCArrayClear(browseNode.term);
+	SetCStr(browseNode.term, query, strlen(query));
+	cin.getline(query, 2048, '\n'); // get rest of line
+
+	// print the query
+	PrintNode (cout, &browseNode);
+
+	MGBrowseQuery(indexData, docLevel, browseNode, browseResult);
+	cout << browseResult;
+	cout << "\n";
+
+      }
+      else { // bad option
+	cout << "bad command\n\n";
+	printHelp();
+      }
+    } // if a .x query
+    else {
+      // regular query
+      queryTree = ParseQuery (queryArray, defaultBoolCombine, defaultStemMethod);
+      if (queryTree == NULL) {
+	cout << "invalid syntax\n";
+      } else {
+	// print the query
+	PrintNode (cout, queryTree);
+	
+	MGQuery (indexData, queryInfo, queryTree, queryResult, level);
+	if (shortOutput) {
+	  queryResult.printShort(cout);
+	  cout << "\n";
+	} else {
+	  cout << queryResult;
+	  cout << "\n";
+	}
+	// delete the query
+	delete queryTree;
+	queryTree = NULL;
+      }
+    }
+  }
+
+
+  // clean up, everybody clean up
+    textData.UnloadData ();
+  indexData.UnloadData ();
+
+  return (0);
+}
+
+
Index: /main/tags/3.01/indexers/mgpp/text/TagInfo.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TagInfo.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TagInfo.cpp	(revision 21112)
@@ -0,0 +1,74 @@
+/**************************************************************************
+ *
+ * TagInfo.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TagInfo.h"
+
+
+void TagInfo::SetDocTag (const char *cStr) {
+  SetCStr (docTag, cStr, strlen(cStr));
+}
+
+void TagInfo::SetIndexLevel (const char *cStr) {
+  SetCStr (indexLevel, cStr, strlen(cStr));
+}
+void TagInfo::AddLevelTag (const char *cStr) {
+  // convert the string
+  UCArray cArr;
+  SetCStr (cArr, cStr, strlen(cStr));
+
+  // insert the tag
+  levelTags.insert (cArr);
+}
+
+
+TagInfo::TagInfo () {
+  Clear ();
+}
+
+void TagInfo::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelTags.erase (levelTags.begin(), levelTags.end());
+}
+
+ostream &operator<<(ostream &s, const TagInfo &t) {
+  // output docTag
+  s << "docTag: \"" << t.docTag << "\"\n";
+
+  // output indexLevel
+  s << "indexLevel: \"" << t.indexLevel << "\"\n";
+
+  // output levelTags
+  s << "levelTags: ";
+
+  UCArraySet::const_iterator here = t.levelTags.begin();
+  UCArraySet::const_iterator end = t.levelTags.end();
+  while (here != end) {
+    s << "\"" << (*here) << "\"";
+    ++here;
+    if (here != end) s << ", ";
+  }
+
+  s << "\n";
+  
+  return s;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/TagInfo.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TagInfo.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TagInfo.h	(revision 21112)
@@ -0,0 +1,46 @@
+/**************************************************************************
+ *
+ * TagInfo.h -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TAGINFO_H
+#define TAGINFO_H
+
+#include "UCArray.h"
+
+
+class TagInfo {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  UCArraySet levelTags;
+
+  void SetDocTag (const char *cStr);
+  void SetIndexLevel (const char *cStr);
+  void AddLevelTag (const char *cStr);
+  
+  TagInfo ();
+  void Clear ();
+};
+
+// stream operator to print TagInfo
+ostream &operator<<(ostream &s, const TagInfo &t);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/Terms.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Terms.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Terms.cpp	(revision 21112)
@@ -0,0 +1,797 @@
+/**************************************************************************
+ *
+ * Terms.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "Terms.h"
+#include "words.h"
+#include "stemmer.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+
+void QueryInfo::Clear () {
+  UCArrayClear (docLevel);
+  maxDocs = 0;
+  sortByRank = true;
+  exactWeights = false;
+  needRankInfo = false;
+  needTermFreqs = false;
+}
+
+
+
+void TermFreqData::Clear () {
+  UCArrayClear (tag);
+  UCArrayClear (term);
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+  stemMethod = 0;
+  matchDocs = 0;
+  termFreq = 0;
+}
+
+ostream &operator<< (ostream &s, const TermFreqData &t) {
+  s << "<" << t.tag << ">\"" << t.term << "\"stem("
+    << t.stemMethod << ")equiv terms(";
+  
+  unsigned long i;
+  for (i=0; i<t.equivTerms.size(); ++i) {
+    s << t.equivTerms[i] << ", ";
+  }
+  s <<")docs(" << t.matchDocs << ")"
+    << "count("<<t.termFreq<<")";
+  return s;
+}
+
+bool operator== (const TermFreqData &t1, const TermFreqData &t2) {
+  return ((t1.tag == t2.tag) &&
+	  (t1.term == t2.term) &&
+	  (t1.stemMethod == t2.stemMethod) &&
+	  (t1.equivTerms == t2.equivTerms) &&
+	  (t1.matchDocs == t2.matchDocs) &&
+	  (t1.termFreq == t2.termFreq));
+}
+
+
+void QueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+QueryResult::QueryResult () {
+  Clear ();
+}
+
+void QueryResult::printShort(ostream &s) {
+
+  s << "termFreqs: ";
+  for (unsigned long i=0; i<termFreqs.size(); ++i)
+    s << termFreqs[i] << ", ";
+  
+  s << "\nactual number of docs found: " << actualNumDocs;
+  s << "\n\n";
+
+}
+
+
+ostream &operator<< (ostream &s, const QueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const QueryResult &r1, const QueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//---------------------------------------------------
+// new ExtQueryResult stuff
+void ExtQueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  levels.erase (levels.begin(), levels.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+ExtQueryResult::ExtQueryResult () {
+  Clear ();
+}
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+
+  s << "\nlevels: ";
+  for (i=0; i<r.levels.size(); ++i)
+    s << r.levels[i] << ", ";
+
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.levels == r2.levels) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//-------------------------------------------------------
+// new BrowseQueryResult stuff
+void BrowseQueryResult::Clear () {
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+}
+
+BrowseQueryResult::BrowseQueryResult () {
+  Clear ();
+}
+
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r) {
+  s << "terms: ";
+  unsigned long i;
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+    s << "\n\n";
+  return s;
+}
+
+
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2) {
+  return ((r1.termFreqs == r2.termFreqs));
+	 
+}
+
+
+
+
+//--------------------------------------
+void FragData::Clear () {
+  matchDocs = 0;
+  fragNums.erase (fragNums.begin(), fragNums.end());
+  fragFreqs.erase (fragFreqs.begin(), fragFreqs.end());
+}
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (stemMethod == 0 || stemMethod==4 || stemMethod==5) {
+    // don't need to stem the word,
+    // find the word number(s) for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (stemMethod ==0) {
+      if (SearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels, term, wordDictEl, wordElNum))
+	equivWords.push_back (wordElNum);
+      
+      return;
+    } else {
+      // partial matching,
+      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod==5?true:false) );
+      return;
+    }
+  }
+		      
+  // need to stem this word and find it in the blocked stem index
+  
+  unsigned char  mgWord[MAXSTEMLEN + 1];
+  UCArray stemTerm;
+  unsigned long stemmerNum = 0;
+  if (stemMethod == 1) stemmerNum = indexData.sih1.stemmer_num;
+  else if (stemMethod == 2) stemmerNum = indexData.sih2.stemmer_num;
+  else if (stemMethod == 3) stemmerNum = indexData.sih3.stemmer_num;
+   
+  // convert the word to an "mg word"
+  mgWord[0] = term.size();
+  memcpy ((char *)&mgWord[1], &(term[0]), term.size());
+  
+  // stem the word
+  stemmer (stemMethod, stemmerNum, mgWord);
+
+  // convert the result back to a UCArray
+  stemTerm.insert (stemTerm.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+  // need to look up this term in the appropriate dictionary
+  stem_block_dict_el stemDictEl;
+  unsigned long stemElNum;
+  bool result = false;
+  if (stemMethod == 1) {
+    result = SearchStemBlockDictEl (indexData.stem1File,
+			   indexData.sii1,
+			   indexData.sih1.entries_per_block,
+			   indexData.sih1.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 2) {
+    result = SearchStemBlockDictEl (indexData.stem2File,
+			   indexData.sii2,
+			   indexData.sih2.entries_per_block,
+			   indexData.sih2.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 3) {
+    result = SearchStemBlockDictEl (indexData.stem3File,
+			   indexData.sii3,
+			   indexData.sih3.entries_per_block,
+			   indexData.sih3.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+  }
+
+  if (result) {
+    equivWords = stemDictEl.equivWords;  
+  }
+}
+
+
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray & termWord) {
+  fragData.Clear();
+
+  // look up the word in the dictionary
+  unsigned long numLevels = indexData.bdh.num_levels;
+  word_block_dict_el wordDictEl;
+  wordDictEl.SetNumLevels (numLevels);
+  if (!SearchWordBlockDictElNum (indexData.dictFile,
+				 indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels,
+				 termNum, wordDictEl))
+    return; // nothing more to do
+
+  fragData.matchDocs = wordDictEl.levelFreqs[indexData.curLevelNum];
+  termWord = wordDictEl.el;
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, wordDictEl.invf_ptr, SEEK_SET);
+  stdio_bitio_buffer buffer (indexData.invfFile);
+    
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags,
+				     wordDictEl.frag_occur);
+  unsigned long fragNum = 0;
+  unsigned long termFreq = 0;
+
+  unsigned long fragLimitI = 0;
+  unsigned long i;
+  for (i=0; i<wordDictEl.frag_occur; ++i) {
+    fragNum += buffer.bblock_decode (B, NULL);
+    if (!indexData.ifh.word_level_index) termFreq = buffer.gamma_decode (NULL);
+    else termFreq = 1;
+
+    // get the right fragment range
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < (*fragLimits).size() &&
+	     fragNum > (*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    // add the entry if it is within the limits
+    if ((fragLimits == NULL) ||
+	(fragLimitI < (*fragLimits).size() &&
+	 fragNum > (*fragLimits)[fragLimitI].rangeStart &&
+	 fragNum <= (*fragLimits)[fragLimitI].rangeEnd)) {
+      fragData.fragNums.push_back (fragNum);
+      if (needFragFreqs)
+	fragData.fragFreqs.push_back (termFreq);
+    }
+  }
+
+  buffer.done();
+}
+
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData) {
+  outFragData.Clear();
+
+  // the new number of matching documents is the maximum
+  // of the two input matching number of documents -- it
+  // is assumed that these are at the same document level
+  outFragData.matchDocs = (f1.matchDocs > f2.matchDocs) ?
+    f1.matchDocs : f2.matchDocs;
+
+  // do or
+  unsigned long f1I = 0, f1Size = f1.fragNums.size();
+  unsigned long f2I = 0, f2Size = f2.fragNums.size();
+  while (f1I < f1Size || f2I < f2Size) {
+    if (f2I < f2Size &&
+	(f1I >= f1Size ||
+	 f1.fragNums[f1I] > f2.fragNums[f2I])) {
+      // output f2I
+      outFragData.fragNums.push_back (f2.fragNums[f2I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f2.fragFreqs[f2I]);
+      ++f2I;
+      
+    } else if (f1I < f1Size &&
+	       (f2I >= f2Size ||
+		f1.fragNums[f1I] < f2.fragNums[f2I])) {
+      // output f1I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]);
+      ++f1I;
+      
+    } else {
+      // must be equal combine f1I and f2I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]+f2.fragFreqs[f2I]);
+      ++f1I;
+      ++f2I;
+    }
+  }
+}
+
+
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits) {
+  // sanity check on range
+  if (startRange > endRange) {
+    signed long temp = endRange;
+    endRange = startRange;
+    startRange = temp;
+  }
+
+  // get min matchdocs
+  if (comFragData.matchDocs < fragData.matchDocs)
+    fragData.matchDocs = comFragData.matchDocs;
+  
+  unsigned long fragDataI = 0;
+  unsigned long fragDataSize = fragData.fragNums.size();
+  unsigned long comFragDataI = 0;
+  unsigned long comFragDataSize = comFragData.fragNums.size();
+  unsigned long fragLimitI = 0;
+  unsigned long fragLimitSize = (fragLimits==NULL) ? 0 : (*fragLimits).size();
+  unsigned long outI = 0;
+
+  while (fragDataI < fragDataSize &&
+	 comFragDataI < comFragDataSize) {
+    signed long fragNum = (signed long)fragData.fragNums[fragDataI];
+    signed long comFragNum = (signed long)comFragData.fragNums[comFragDataI];
+    
+    // go to the right fragment limit (for the com frag)
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < fragLimitSize &&
+	     comFragNum > (signed long)(*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    if (fragNum <= comFragNum+startRange ||
+	(fragLimits!=NULL &&
+	 fragNum<=(signed long)(*fragLimits)[fragLimitI].rangeStart)) {
+      ++fragDataI;
+      
+    } else if (fragNum > comFragNum+endRange ||
+	       (fragLimits!=NULL &&
+		fragNum>(signed long)(*fragLimits)[fragLimitI].rangeEnd)) {
+      ++comFragDataI;
+      
+    } else {
+      // equal and within tag
+      fragData.fragNums[outI] = comFragNum;
+      if (needFragFreqs) {
+	fragData.fragFreqs[outI] =
+	  (fragData.fragFreqs[fragDataI] < comFragData.fragFreqs[comFragDataI]) ?
+	  fragData.fragFreqs[fragDataI] : comFragData.fragFreqs[comFragDataI];
+      }
+      ++fragDataI;
+      ++comFragDataI;
+      ++outI;
+    }
+  }
+
+  // erase unused part of fragData
+  fragData.fragNums.erase (fragData.fragNums.begin()+outI,
+			   fragData.fragNums.end());
+  if (needFragFreqs)
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin()+outI,
+			      fragData.fragFreqs.end());
+  else
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin(),
+			      fragData.fragFreqs.end());
+}
+
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  result.Clear();
+
+  // log (N / ft)
+  unsigned long N = indexData.levels.levelInfo[indexData.curLevel].numEntries;
+  float wordLog = log((double)N / (double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  if (needRanks) {
+	    Wdt = termDocFreq * wordLog;
+	    result.ranks.push_back (Wqt * Wdt);
+	  }
+	  result.docs.push_back (lastLevelDocNum);
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks){
+	termDocFreq += termData.fragFreqs[termDataI];
+	overallwordfreq += termData.fragFreqs[termDataI];
+      }
+    }
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    if (needRanks) {
+      Wdt = termDocFreq * wordLog;
+      result.ranks.push_back (Wqt * Wdt);
+    }
+    result.docs.push_back (lastLevelDocNum);
+  }
+
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq; // will be zero if needRankInfo 
+                                              //not true
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  // log (N / ft)
+  float wordLog =
+    log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries/
+	(double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  Wdt = termDocFreq * wordLog;
+	  
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks)
+	termDocFreq += termData.fragFreqs[termDataI];
+     overallwordfreq += termData.fragFreqs[termDataI]; 
+    }
+    
+    ++termDataI;
+  } // while
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    Wdt = termDocFreq * wordLog;
+
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks)
+	result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+  
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq;
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long lastLevelDocNum = 0;
+
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI];
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+      }
+    }
+    
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks) 
+	result.ranks[resultOutI] = result.ranks[resultI];
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+}
+
+
+
+//--------------------------------------------------------------
+// functions to support full text browse
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number) {
+
+    // find the word number for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (NearestSearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				      indexData.bdh.entries_per_wblk,
+				      indexData.bdh.word_dict_size,
+				      numLevels, term, wordDictEl, wordElNum))
+      number = wordElNum;
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms) {
+
+  word_block_dict_el_array wordBlocks; // = new word_block_dict_el_array();
+  TermFreqData termdata;
+
+  terms.erase(terms.begin(), terms.end());
+
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, wordBlocks);
+
+  word_block_dict_el_array::iterator here = wordBlocks.begin();
+  word_block_dict_el_array::iterator end = wordBlocks.end();
+
+  while (here != end) {
+    termdata.Clear();
+    termdata.term = (*here).el;
+    termdata.termFreq = (*here).freq;
+    terms.push_back(termdata);
+    ++here;
+  }
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms) {
+
+  
+  
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, terms);
+
+}
Index: /main/tags/3.01/indexers/mgpp/text/Terms.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Terms.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Terms.h	(revision 21112)
@@ -0,0 +1,215 @@
+/**************************************************************************
+ *
+ * Terms.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TERMS_H
+#define TERMS_H
+
+#include "IndexData.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+class QueryInfo {
+public:
+  // general query information
+  UCArray docLevel;
+  unsigned long maxDocs; // 0 = all
+  bool sortByRank;
+  bool exactWeights;
+
+  // information need to return
+  bool needRankInfo;
+  bool needTermFreqs;
+
+  void Clear ();
+  QueryInfo () { Clear (); }
+};
+
+
+class TermFreqData {
+public:
+  UCArray tag;  // level tag or query tag
+  UCArray term; // unstemmed term
+  int stemMethod;
+  UCArrayVector equivTerms; // the stemmed and casefolded variants of the term
+  unsigned long matchDocs; // tf for level - num levels
+			   // containing this term
+  unsigned long termFreq;  // overall term freq - num words that
+				// are this term
+  void Clear ();
+  TermFreqData () { Clear (); }
+};
+
+ostream &operator<< (ostream &s, const TermFreqData &t);
+bool operator== (const TermFreqData &t1, const TermFreqData &t2);
+
+typedef vector<TermFreqData> TermFreqArray;
+
+
+typedef vector<float> RankArray;
+typedef vector<unsigned long> DocNumArray;
+
+class QueryResult {
+public:
+  DocNumArray docs;
+  RankArray ranks; // used for accumulators during query
+  
+  TermFreqArray termFreqs;
+
+  unsigned long actualNumDocs;
+  void Clear ();
+  QueryResult ();
+  void printShort(ostream &s);
+};
+
+ostream &operator<< (ostream &s, const QueryResult &r);
+bool operator== (const QueryResult &r1, const QueryResult &r2);
+
+typedef vector<unsigned long> FragNumArray;
+typedef vector<unsigned long> FragFreqArray;
+
+class FragData {
+public:
+  unsigned long matchDocs;    // ft for level
+  FragNumArray fragNums;
+  FragFreqArray fragFreqs;
+
+  void Clear ();
+  FragData () { Clear (); }
+};
+
+
+class FragRange {
+public:
+  unsigned long rangeStart;
+  unsigned long rangeEnd;
+
+  void Clear () { rangeStart = rangeEnd = 0; }
+  FragRange () { Clear (); }
+};
+
+typedef vector<FragRange> FragRangeArray;
+
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords);
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray &termWord);
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData);
+
+// output will be in fragData (as this is an and operation)
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits);
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result);
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result);
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result);
+
+//-----------------------------------------------------------------
+// new QueryResult class to handle retrieval of doc and level nums.
+// Use this class with extended version of MGQuery
+
+class ExtQueryResult : public QueryResult {
+public:
+  DocNumArray levels; // used for returning a different granularity, eg
+  // search sections but return Document numbers, or search Documents, 
+  // return Section numbers.
+ 
+  void Clear ();
+  ExtQueryResult ();
+};
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r);
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2);
+
+//------------------------------------------------------------
+// new functions to handle full text browse
+
+class BrowseQueryResult {
+ public:
+  TermFreqArray termFreqs;
+  void Clear();
+  BrowseQueryResult ();
+
+};
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r);
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2);
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number);
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms);
+
+void GetTermList (IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/TextEl.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TextEl.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TextEl.cpp	(revision 21112)
@@ -0,0 +1,374 @@
+/**************************************************************************
+ *
+ * TextEl.cpp -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TextEl.h"
+#include "ctype.h"
+
+#define REC_TERM   '\002'
+#define PARA_TERM  '\003'
+
+
+void TextEl::Clear () {
+  elType = TextE;
+  tagName.erase (tagName.begin(), tagName.end());
+  text.erase (text.begin(), text.end());
+}
+
+
+static bool ReadTextTag (FileBuf &buf, TextEl &el) {
+  // we must have some sort of tag
+  unsigned char c;
+  if (!buf.Peek(c)) return false;
+  
+  // get the '<'
+  if (c != '<') return false;
+  el.text.push_back (c);
+  if (!buf.Next() || !buf.Peek (c)) return false;
+  
+  // see if we have a '/'
+  if (c == '/') {
+    // we have a closing tag
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+    el.elType = CloseTagE;
+  } else {
+    // we have an opening tag
+    el.elType = OpenTagE;
+  }
+
+  // get the tag name
+  while (isalnum(c)) {
+    el.text.push_back (c);
+    el.tagName.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+  
+  // get everything up to the closing '>'
+  while (c != '>') {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+
+  // get the closing '>'
+  el.text.push_back (c);
+  buf.Next();
+
+  // if there is no tag name then make this a text element
+  if (el.tagName.empty()) el.elType = TextE;
+  
+  return true;
+}
+
+static bool ReadTextTag (u_char ** buffer, TextEl &el) {
+  // we must have some sort of tag
+  u_char * buffer_pos = *buffer;
+  if (*buffer_pos == '\0' || *buffer_pos != '<') {
+     return false;
+  }
+  el.text.push_back('<');
+  buffer_pos ++;
+  if (*buffer_pos == '\0') {
+    *buffer = buffer_pos;
+    return false;
+  }
+  // see if we have a '/'
+  if (*buffer_pos == '/') {
+    el.text.push_back('/');
+    el.elType = CloseTagE;
+    buffer_pos++;
+    if (*buffer_pos == '\0') {    
+      *buffer = buffer_pos;
+      return false;
+    }
+  } else {
+    el.elType = OpenTagE;
+  }
+  // get the tag name
+  while (isalnum(*buffer_pos)) {
+    el.text.push_back (*buffer_pos);
+    el.tagName.push_back (*buffer_pos);
+    buffer_pos++;
+    
+    if (*buffer_pos == '\0') {
+      *buffer = buffer_pos;
+      return false;
+    }
+
+  }
+  cout << "tag name = "<<el.tagName<<endl;
+  // get everything up to closing '>'
+  while (*buffer_pos != '>') {
+    el.text.push_back(*buffer_pos);
+    buffer_pos++;
+    if (*buffer_pos == '\0') {
+      *buffer = buffer_pos;
+      return false;
+    }
+    
+  }
+  // get teh closing '>'
+  el.text.push_back('>');
+  buffer_pos++;
+  *buffer = buffer_pos;
+  cout << "at end of readtexttag, buffer is "<<*buffer<<endl;
+  // if there is no tag name then make this a text element
+  if (el.tagName.empty()) el.elType = TextE;
+  
+  return true;
+}
+
+static void ToggleParaTag (TextEl &el, bool &compatInPara) {
+  SetCStr (el.tagName, "Paragraph", 9);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (compatInPara) {
+    el.elType = CloseTagE;
+    el.text.push_back (PARA_TERM);
+  } else el.elType = OpenTagE;
+  compatInPara = !compatInPara;
+}
+
+static void SetRecTag (TextEl &el, TextElType elType) {
+  el.elType = elType;
+  SetCStr (el.tagName, "Document", 8);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (elType == CloseTagE)
+    el.text.push_back (REC_TERM);
+}
+
+
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara) {
+  el.Clear();
+
+  unsigned char c;
+  if (!buf.Peek (c)) return false;
+
+  if (compatMode) {
+    if (c == PARA_TERM) {
+      ToggleParaTag (el, compatInPara);
+      if (!compatInPara) buf.Next();
+      return true;
+    }
+    if (c == REC_TERM) {
+      if (compatInPara) {
+	// need to close this paragraph
+	ToggleParaTag (el, compatInPara);
+	return true;
+      }
+      // can close this document
+      buf.Next();
+      SetRecTag (el, CloseTagE);
+      return true;
+    }
+    if (!compatInPara) {
+      // need to open a paragraph
+      ToggleParaTag (el, compatInPara);
+      return true;
+    }
+  }
+  
+  // see if we have some sort of tag
+  if (c == '<') return ReadTextTag (buf, el);
+
+  // read in a text element
+  el.elType = TextE;
+  while (c != '<' && !(compatMode && (c == PARA_TERM || c == REC_TERM))) {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) break;
+  }
+
+  return true;
+}
+bool ReadTextEl (u_char **buffer, TextEl &el,
+		 bool compatMode, bool &compatInPara) {
+  el.Clear();
+  u_char * buffer_pos = *buffer;
+  if (*buffer_pos == '\0') return false;
+  if (compatMode) {
+    if (*buffer_pos == PARA_TERM) {
+      ToggleParaTag (el, compatInPara);
+      if (!compatInPara) buffer_pos++;
+      *buffer = buffer_pos;
+      return true;
+    }
+    if (*buffer_pos == REC_TERM) {
+      if (compatInPara) {
+	// need to close this paragraph
+	ToggleParaTag (el, compatInPara);
+	*buffer = buffer_pos;
+	return true;
+      }
+      // can close this document
+      buffer_pos++;
+      SetRecTag (el, CloseTagE);
+      *buffer = buffer_pos;
+      return true;
+    }
+    if (!compatInPara) {
+      // need to open a paragraph
+      ToggleParaTag (el, compatInPara);
+      *buffer = buffer_pos;
+      return true;
+    }
+  }
+  
+  // see if we have some sort of tag
+  if (*buffer_pos == '<') return ReadTextTag (buffer, el);
+
+  // read in a text element
+  el.elType = TextE;
+  while (*buffer_pos != '<' && !(compatMode && (*buffer_pos == PARA_TERM || *buffer_pos == REC_TERM))) {
+    el.text.push_back (*buffer_pos);
+    buffer_pos++;
+    if (*buffer_pos == '\0') break;
+  }
+  *buffer = buffer_pos;
+  cout << "text element: "<<el.text<<endl;
+  return true;
+}
+  
+static void AddTextEl (TextElArray &doc, unsigned long &docLen,
+		       const TextEl &el) {
+  doc.push_back (el);
+  docLen += el.text.size();
+
+//    cout << "elType:  " << el.elType << "\n";
+//    cout << "tagName: " << el.tagName << "\n";
+//    cout << "text:    \"" << el.text << "\"\n\n";
+}
+
+
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode) {
+  bool compatInPara = false;
+  bool foundDocEl = false;
+  TextEl el;
+  
+  doc.erase (doc.begin(), doc.end());
+  docLen = 0;
+
+  if (compatMode) {
+    // add <Document><Paragraph>
+    SetRecTag (el, OpenTagE);
+    AddTextEl (doc, docLen, el);
+    ToggleParaTag (el, compatInPara);
+    AddTextEl (doc, docLen, el);
+    
+  } else {
+    // look for an opening docTag
+    do {
+      if (!ReadTextEl (buf, el, compatMode, compatInPara)) return false;
+    } while (el.elType != OpenTagE || el.tagName != docTag);
+    
+    AddTextEl (doc, docLen, el);
+  }
+
+  
+  // get all elements until the closing docTag
+  while (ReadTextEl (buf, el, compatMode, compatInPara)) {
+    foundDocEl = true;
+    AddTextEl (doc, docLen, el);
+    if (el.elType == CloseTagE && el.tagName == docTag)
+      return true;
+  }
+
+  
+  if (compatMode) {
+    if (!foundDocEl) { // end of text
+      doc.erase (doc.begin(), doc.end());
+      return false;
+    }
+
+    // if we got here then the eof was encountered before
+    // the closing document tag
+    if (compatInPara) {
+      // need to close this paragraph
+      ToggleParaTag (el, compatInPara);
+      AddTextEl (doc, docLen, el);
+    }
+    // close this document
+    SetRecTag (el, CloseTagE);
+    AddTextEl (doc, docLen, el);
+  }
+  
+  return true;
+}
+
+// copy of readdoc to get TextElArray from u_char buffer
+bool ReadDoc(u_char ** buffer, const UCArray &docTag, 
+	     TextElArray &doc, unsigned long &docLen,
+	     bool compatMode) {
+  bool compatInPara = false;
+  bool foundDocEl = false;
+  TextEl el;
+  
+  doc.erase (doc.begin(), doc.end());
+  docLen = 0;
+
+  if (compatMode) {
+    // add <Document><Paragraph>
+    SetRecTag (el, OpenTagE);
+    AddTextEl (doc, docLen, el);
+    ToggleParaTag (el, compatInPara);
+    AddTextEl (doc, docLen, el);
+    
+  } else {
+    // look for an opening docTag
+    do {
+      if (!ReadTextEl (buffer, el, compatMode, compatInPara)) return false;
+    } while (el.elType != OpenTagE || el.tagName != docTag);
+    
+    AddTextEl (doc, docLen, el);
+  }
+
+  
+  // get all elements until the closing docTag
+  while (ReadTextEl (buffer, el, compatMode, compatInPara)) {
+    foundDocEl = true;
+    AddTextEl (doc, docLen, el);
+    if (el.elType == CloseTagE && el.tagName == docTag)
+      return true;
+  }
+
+  
+  if (compatMode) {
+    if (!foundDocEl) { // end of text
+      doc.erase (doc.begin(), doc.end());
+      return false;
+    }
+
+    // if we got here then the eof was encountered before
+    // the closing document tag
+    if (compatInPara) {
+      // need to close this paragraph
+      ToggleParaTag (el, compatInPara);
+      AddTextEl (doc, docLen, el);
+    }
+    // close this document
+    SetRecTag (el, CloseTagE);
+    AddTextEl (doc, docLen, el);
+  }
+  
+  return true;
+}
+
+
Index: /main/tags/3.01/indexers/mgpp/text/TextEl.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TextEl.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TextEl.h	(revision 21112)
@@ -0,0 +1,69 @@
+/**************************************************************************
+ *
+ * TextEl.h -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTEL_H
+#define TEXTEL_H
+
+#include "UCArray.h"
+#include "FileBuf.h"
+#include "sysfuncs.h"
+
+enum TextElType {OpenTagE, CloseTagE, TextE};
+
+class TextEl {
+protected:
+  bool ReadTextTag (FileBuf &buf, TextEl &el);
+  bool ReadTextTag (u_char ** buffer, TextEl &el);
+public:
+  TextElType elType;
+  UCArray tagName;
+  UCArray text; // complete text, including tag
+
+  TextEl () { Clear (); }
+  void Clear ();
+};
+
+
+typedef vector<TextEl> TextElArray;
+
+
+// compatMode parses documents similar to "old mg" -- ie with
+// C-b between documents and C-c between paragraphs
+
+// returns true if a text element could be read
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara);
+bool ReadTextEl (u_char **buffer, TextEl &el,
+		 bool compatMode, bool &compatInPara);
+
+// looks for an opening docTag and reads all text elements up
+// to a corresponding closing docTag. note that this does not
+// hierarchically parse the document so you can't nest documents.
+// returns true if a document could be found
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode=false);
+
+bool ReadDoc(u_char **buffer, const UCArray &docTag, 
+	     TextElArray &doc, unsigned long &docLen,
+	     bool compatMode=false);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/TextGet.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TextGet.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TextGet.cpp	(revision 21112)
@@ -0,0 +1,685 @@
+/**************************************************************************
+ *
+ * TextGet.cpp -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// is important to be first, so we escape the truncation warning on VC++
+#include "TextGet.h"
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "mg_files.h"
+#include "netorder.h"
+#include "mg_errors.h"
+#include "locallib.h"
+#include "words.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+
+typedef enum huff_type {lengths, chars};
+
+
+static auxiliary_dict *LoadAuxDict (compression_dict &cd, FILE *text_aux_dict) {
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = new auxiliary_dict))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  memset (ad, '\0', sizeof (*ad));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = new u_char[ad->afh[i].mem_for_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+      if (!(ad->words[i] = new u_char* [ad->afh[i].num_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < (int)ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd.cdh.novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd.cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+static u_char ***ReadInWords (FILE *dict, compression_dict &cd,
+			      comp_frags_header *cfh, u_char **escape) {
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd.cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i) {
+    ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+    mem_reqd += cfh->huff_words_size[i];
+  }
+
+  if (!(vals = new u_char* [ptrs_reqd]))
+    return (NULL);
+
+  if (!(values = new u_char** [MAX_HUFFCODE_LEN + 1]))
+    return (NULL);
+
+  if (!(next_word[0] = new u_char[mem_reqd])) 
+    return (NULL);
+
+  cd.MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+static int Load_Comp_HuffData(compression_dict &cd, int which, FILE *dict, 
+			      huff_type type) {
+  huff_data * hd;
+  u_long **   vals;
+
+  if (!(hd = new huff_data))
+    return 1;
+  cd.MemForCompDict += sizeof (huff_data);
+  if (Read_Huffman_Data (dict, hd, &cd.MemForCompDict, NULL) == -1)
+    return 2;
+  if (!(vals = Generate_Huffman_Vals (hd, &cd.MemForCompDict)))
+    return 3;
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+  if (type == chars)
+    {
+      cd.chars_huff[which] = hd;
+      cd.chars_vals[which] = vals;
+    }
+  else
+    {
+      cd.lens_huff[which] = hd;
+      cd.lens_vals[which] = vals;
+    }
+
+  return 0;
+}
+
+static int Load_Comp_FragsHeader(compression_dict &cd, int which, int getEscape,
+				 FILE *dict) {
+  if (!(cd.cfh[which] = new comp_frags_header))
+    return 1;
+  cd.MemForCompDict += sizeof (*cd.cfh[which]);
+  if (Read_cfh (dict, cd.cfh[which], &cd.MemForCompDict, NULL) == -1)
+    return 2;
+
+  if (!(cd.values[which] = ReadInWords (dict, cd, cd.cfh[which],
+					getEscape == 0 ? NULL : &cd.escape[which])))
+    return 3;
+  
+  return 0;
+}
+
+static bool LoadSlowCompDict (FILE *dict, FILE *aux_dict, compression_dict &cd) {
+  if (dict == NULL) return false;
+
+  int which;
+
+  memset (&cd, '\0', sizeof (compression_dict));
+
+  cd.MemForCompDict = sizeof (compression_dict);
+
+  if (Read_cdh (dict, &cd.cdh, &cd.MemForCompDict, NULL) == -1)
+    return false;
+
+  for (which = 0; which < 2; ++which)
+    switch (cd.cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Load_Comp_FragsHeader(cd, which, 0, dict) != 0)
+	    return false;
+	  cd.escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+
+	  if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+	    return false;
+
+	  if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+	    return false;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+		return false;
+
+	      if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+		return false;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd.cdh.novel_method == MG_NOVEL_DELTA ||
+      cd.cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  cd.Clear();
+	  return false;
+	}
+
+      if (!(cd.ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  cd.Clear();
+	  return false;
+	}
+    }
+
+  mg_errno = MG_NOERROR;
+
+  cd.fast_loaded = 0;
+  
+  return true;
+}
+
+
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p,cd) & 7))) != 0)
+
+// fast loading really needs to be totally re-writen. "Unloading" the
+// text data will currently cause a crash because memory is being
+// deleted multiple times (and probably a zillion other reasons).
+static bool LoadFastCompDict (FILE *text_fast_comp_dict, compression_dict &_cd) {
+  if (text_fast_comp_dict == NULL) return false;
+  
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  compression_dict *cd;
+  if (!(cd = (compression_dict *)malloc (mem))) {
+    mg_errno = MG_NOMEM;
+    return false;
+  }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = new u_char[fixup_mem]))
+    {
+      mg_errno = MG_NOMEM;
+      return false;
+    }
+
+  fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; ++p)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    for (i = 0; i <= 1; ++i)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; ++j)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  delete []fixup;
+
+  // the whole fast comp dict is a bit of a hack so I don't
+  // feel too bad about the next line :-) -- Rodger.
+  _cd = *cd;
+  
+  return true;
+}
+
+
+static bool LoadCompDict (FILE *compDictFile,
+			  FILE *auxDictFile,
+			  FILE *fastCompDictFile,
+			  compression_dict &cd) {
+  // see if we have a fast loading compression dictionary
+  if (fastCompDictFile != NULL)
+    return LoadFastCompDict (fastCompDictFile, cd);
+  
+  // slow compression dictionary
+  return LoadSlowCompDict (compDictFile, auxDictFile, cd);
+}
+
+
+// try to open the dictionary files and load the dictionary
+static bool OpenLoadCompDict (char *textname, compression_dict &cd) {
+  FILE *compDictFile = NULL;
+  FILE *auxDictFile = NULL;
+  FILE *fastCompDictFile = NULL;
+
+  fastCompDictFile = open_file (textname, TEXT_DICT_FAST_SUFFIX,
+				"rb", MAGIC_FAST_DICT, MG_CONTINUE);
+  
+  if (fastCompDictFile == NULL) {
+    compDictFile = open_file (textname, TEXT_DICT_SUFFIX,
+			      "rb", MAGIC_DICT, MG_MESSAGE);
+    auxDictFile =  open_file (textname, TEXT_DICT_AUX_SUFFIX,
+			      "rb", MAGIC_AUX_DICT, MG_CONTINUE);
+  }
+  
+  bool res = LoadCompDict (compDictFile, auxDictFile, fastCompDictFile, cd);
+
+  if (compDictFile != NULL) fclose (compDictFile);
+  if (auxDictFile != NULL) fclose (auxDictFile);
+  if (fastCompDictFile != NULL) fclose (fastCompDictFile);
+  
+  return res;
+}
+
+static bool LoadLevels (char *textname, FTextLevel &levels) {
+  FILE *levelFile = NULL;
+
+  // open the text level file
+  levelFile = open_file (textname, TEXT_LEVEL_SUFFIX,
+			 "rb", MAGIC_TEXT_LEVELS, MG_CONTINUE);
+  if (levelFile == NULL) return false;
+
+  // seek to the appropriate place and read the level information
+  bool res = ((fseek (levelFile, sizeof (u_long), SEEK_SET) == 0) &&
+	      levels.Read (levelFile));
+
+  // close the file
+  fclose (levelFile);
+  
+  return res;
+}
+
+
+TextData::TextData () {
+  // put file pointers in known state first
+  textFile = NULL;
+  textIdxFile = NULL;
+  Clear ();
+}
+
+void TextData::Clear () {
+  cd.Clear(); 
+  textFile = NULL;
+  textIdxFile = NULL;
+  cth.Clear();
+  levels.Clear();
+}
+
+bool TextData::LoadData (char *basepath, char *textname) {
+  
+  if (textname[0] == '\0') return false;
+  
+  // set the basepath
+  set_basepath(basepath);
+  
+  // load the compression dictionary
+  if (!OpenLoadCompDict (textname, cd)) return false;
+
+  // open the compressed text and text index file
+  textFile = open_file (textname, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_CONTINUE);
+  if (textFile == NULL) return false;
+
+  textIdxFile = open_file (textname, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_CONTINUE);
+  if (textIdxFile == NULL) return false;
+
+  // read in the compressed text header
+  if ((fseek (textFile, sizeof (u_long), SEEK_SET) != 0) || !cth.Read (textFile))
+    return false;
+
+  // read in the level information
+  if (!LoadLevels (textname, levels)) return false;
+  
+  return true;
+}
+
+bool TextData::UnloadData () {
+  // close any open files
+  if (textFile != NULL) {
+    fclose (textFile);
+    textFile = NULL;
+  }
+  if (textIdxFile != NULL) {
+    fclose (textIdxFile);
+    textIdxFile = NULL;
+  }
+
+  // do general clear
+  Clear ();
+  
+  return true;  
+}
+
+
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx) {
+  // make sure the text index file was opened successfully
+  if (td.textIdxFile == NULL) return false;
+
+  // read in the index
+  TextLevelInfo &levelInfo = td.levels.levelInfo[docLevel];
+  if (!docIdx.Read (td.textIdxFile, levelInfo, docNum)) return false;
+  
+  return true;
+}
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        __code += __code + buffer.bit();	                  	\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText) {
+  // erase the current text
+  docText.erase (docText.begin(), docText.end());
+
+  // look up the information about this document
+  TextIdx docIdx;
+  if (!GetDocIdx (td, docLevel, docNum, docIdx)) return false;
+
+  // do seek to appropriate position
+  stdio_bitio_buffer buffer (td.textFile);
+  buffer.seek (docIdx.start.byte, docIdx.start.bit);
+  
+  // decompress the document
+  compression_dict &cd = td.cd;
+  auxiliary_dict *ad = cd.ad;
+  int which = docIdx.which;
+  unsigned long num_bits = (docIdx.end.byte*8+(8-docIdx.end.bit)) -
+    (docIdx.start.byte*8+(8-docIdx.start.bit));
+  unsigned long bits = 0;
+
+  if (docText.capacity() < docText.size() + num_bits + 1) {
+    docText.reserve(docText.size() + num_bits + 1);
+  }
+  // keep decoding bits until enough bits have been decoded
+  while (bits < num_bits) {
+    register unsigned code, len;
+    register int r;
+    register u_char *t, *b = NULL;
+    u_char word[MAXWORDLEN + 1];
+    
+    if (cd.cfh[which]) {
+      MY_HUFF_DECODE (len, code, cd.cfh[which]->hd.min_code);
+      bits += len;
+      
+      r = code & ((1 << cd.cdh.lookback) - 1);
+      t = cd.values[which][len][code >> cd.cdh.lookback];
+      
+      /* step through from base pointer */
+      b = word + 1;
+      while (r--) {
+	register int copy = *t >> 4;
+	memcpy (word + copy + 1, t + 1, *t & 0xf);
+	word[0] = copy + (*t & 0xf);
+	t += ((*t) & 0xf) + 1;
+      }
+    } else t = NULL;
+    
+    if (t == cd.escape[which]) {
+      switch (cd.cdh.novel_method) {
+      case MG_NOVEL_HUFFMAN_CHARS:
+	{
+	  int len, i;
+	  int c;
+	  len = buffer.huff_decode(cd.lens_huff[which]->min_code,
+				   cd.lens_vals[which], &bits);
+	  for (i = 0; i < len; ++i) {
+	    c = buffer.huff_decode(cd.chars_huff[which]->min_code,
+				   cd.chars_vals[which], &bits);
+	    docText.push_back (c);
+	  }
+	}
+	break;
+      case MG_NOVEL_DELTA:
+      case MG_NOVEL_HYBRID:
+	{
+	  int idx = 0, len;
+	  u_char *base;
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_DELTA:
+	      {
+		idx = buffer.delta_decode (&bits);
+		--idx;
+	      }
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      {
+		int k;
+		k = buffer.gamma_decode (&bits);
+		--k;
+		idx = buffer.binary_decode(ad->blk_end[which][k] -
+					   ad->blk_start[which][k] + 1, 
+					   &bits);
+		idx += ad->blk_start[which][k] - 1;
+	      }
+	      break;
+	    }
+	  base = ad->words[which][idx];
+	  len = *base++;
+	  for (; len; --len)
+	    {
+	      docText.push_back (*base++);
+	    }
+	}
+	break;
+      }
+    }
+    else
+      {
+	/* copy over the matching prefix */
+	r = (*t >> 4);
+	while (r--) {
+	  docText.push_back (*b++);
+	}
+	
+	/* and the stored suffix */
+	r = ((*t) & 0xf);
+	while (r--) {
+	  docText.push_back (*++t);
+	}
+      }
+    which = !which;
+  }
+  
+  buffer.done();
+  
+  return true;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/TextGet.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/TextGet.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/TextGet.h	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * TextGet.h -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTGET_H
+#define TEXTGET_H
+
+#include "text.h"
+#include "FText.h"
+
+
+class TextData {
+public:
+  compression_dict cd;
+  FILE *textFile;
+  FILE *textIdxFile;
+  compressed_text_header cth;
+  FTextLevel levels;
+
+  TextData ();
+  void Clear ();
+
+  // loads compression dictionary, the compressed text header,
+  // and all level informaiton
+  bool LoadData (char *basepath, char *textname);
+  bool UnloadData ();
+};
+
+
+// note: document numbers start at 1
+
+// get information about where a document starts and finishes
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx);
+
+
+// decompress a document's text
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/UCArray.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/UCArray.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/UCArray.cpp	(revision 21112)
@@ -0,0 +1,360 @@
+/**************************************************************************
+ *
+ * UCArray.cpp -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+void SetCStr (UCArray &text, const char *cStr) {
+  text.erase(text.begin(), text.end());
+  
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint) {
+  text.erase(text.begin(), text.end());
+
+  // reserve the needed space in advance
+  if (text.capacity() < nSizeHint + 1) {
+    text.reserve(nSizeHint + 1);
+  }
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+char * GetCStr(const UCArray& text) {
+
+  char *cstr = new char[text.size()+1];
+  UCArray::const_iterator here = text.begin();
+  UCArray::const_iterator end = text.end();
+
+  int i = 0;
+  while (here != end) {
+    cstr[i] = text[i];
+    ++here; ++i;
+  }
+  cstr[i]='\0';
+  return cstr;
+}
+
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr)
+{
+  if ((cStr == NULL || *cStr == '\0') && text.empty()) return true;
+  UCArray::const_iterator thisUC = text.begin();
+  UCArray::const_iterator endUC = text.end();
+  while (thisUC != endUC && *cStr != '\0') {
+    if (*thisUC != *cStr) return false;
+    ++cStr; ++thisUC;
+  }
+  if (thisUC == endUC && *cStr == '\0') return true;
+  return false;
+}
+
+ostream &operator<<(ostream &s, const UCArray &a) {
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    s << *here;
+    ++here;
+  }
+
+  return s;
+}
+
+
+bool ReadVarLenUL (FILE *f, unsigned long &n) {
+  register unsigned long temp = 0;
+  register unsigned int bitPos = 0;
+  unsigned char b = 0;
+
+  do {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    temp |= (b & 0x7f) << bitPos;
+    bitPos += 7;
+  } while (b >= 0x80 && bitPos < 32);
+
+  n = temp;
+  
+  return true;
+}
+
+bool WriteVarLenUL (FILE *f, unsigned long n) {
+  register unsigned long temp = n;
+  register unsigned char b = 0;
+  do {
+    b = static_cast<unsigned char> (temp & 0x7f);
+    if (temp >= 0x80) b |= 0x80;
+    fputc (b, f);
+    if (ferror (f) != 0) return false;
+  } while ((temp = temp >> 7) > 0);
+  
+  return true;
+}
+
+
+bool ReadUL (FILE *f, unsigned long &n) {
+  if (fread (&n, sizeof (unsigned long), 1, f) <= 0) return false;
+  NTOHUL (n);
+  return true;
+}
+
+
+bool WriteUL (FILE *f, unsigned long n) {
+  HTONUL (n);
+  return (fwrite (&n, sizeof (unsigned long), 1, f) > 0);
+}
+
+bool ReadF (FILE *f, float &n) {
+  if (fread (&n, sizeof (float), 1, f) <= 0) return false;
+  NTOHF(n);
+  return true;
+}
+
+bool WriteF (FILE *f, float n) {
+  HTONF(n);
+  return (fwrite (&n, sizeof (float), 1, f) > 0);
+}
+
+bool ReadD (FILE *f, double &n) {
+  if (fread (&n, sizeof (double), 1, f) <= 0) return false;
+  NTOHD(n);
+  return true;
+}
+
+bool WriteD (FILE *f, double n) {
+  HTOND(n);
+  return (fwrite (&n, sizeof (double), 1, f) > 0);
+}
+
+bool ReadUCArray (FILE *f, UCArray &a) {
+  // clear the array in preparation
+  a.erase (a.begin(), a.end());
+
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+
+  // reserve the needed space in advance
+  if (a.capacity() < arraySize + 1) {
+    a.reserve(arraySize + 1);
+  }
+
+  // read in the array
+  unsigned char b = 0;
+  while (arraySize > 0) {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    a.push_back (b);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteUCArray (FILE *f, const UCArray &a) {
+  // write out the array size
+  if (!WriteVarLenUL (f, a.size())) return false;
+
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    fputc (*here, f);
+    if (ferror (f) != 0) return false;
+    
+    ++here;
+  }
+  
+  return true;
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+
+int DictCompare (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  unsigned int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((diff = casecharmap[*a1Here] - casecharmap[*a2Here]) != 0)
+      return diff;
+    if (pos == 0 && (diff = *a1Here - *a2Here) != 0)
+      pos = diff;
+
+    ++a1Here;
+    ++a2Here;
+  }
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+}
+
+// does the first string start with the second?
+bool StartsWith (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((*a1Here != *a2Here))
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if (casecharmap[*a1Here] != casecharmap[*a2Here]) 
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2) {
+  unsigned long l = (a1.size() < a2.size()) ? a1.size() : a2.size();
+  unsigned long i = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+
+  while (i < l && *a1Here == *a2Here) {
+    ++i; ++a1Here; ++a2Here;
+  }
+  
+  return i;
+}
+
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a) {
+  unsigned char preLen;
+  unsigned char sufLen;
+  
+  if (prev != NULL) preLen = PrefixLen (*prev, a);
+  else preLen = 0;
+  sufLen = a.size() - preLen;
+
+  // output the prefix length, suffix length, and the suffix
+  fputc (preLen, f);
+  if (ferror(f) != 0) return false;
+  fputc (sufLen, f);
+  if (ferror(f) != 0) return false;
+  char* tmp=GetCStr(a);
+  int ret=(fwrite (tmp+preLen, sizeof (char), sufLen, f) == sufLen);
+  delete []tmp;
+  return (ret != 0);
+}
+
+// a also used for prev
+bool ReadPreSufStr (FILE *f, UCArray &a) {
+  unsigned char preLen = 0;
+  unsigned char sufLen = 0;
+
+  preLen = fgetc(f);
+  sufLen = fgetc(f);
+
+  if (a.size() > preLen) a.erase (a.begin()+preLen, a.end());
+
+  // reserve the needed space in advance
+  if (a.capacity() < a.size() + sufLen + 1) {
+    a.reserve(a.size() + sufLen + 1);
+  }
+
+  while (sufLen > 0) {
+    unsigned char c = fgetc (f);
+    a.push_back (c);
+    --sufLen;
+  }
+
+  return true;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/UCArray.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/UCArray.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/UCArray.h	(revision 21112)
@@ -0,0 +1,131 @@
+/**************************************************************************
+ *
+ * UCArray.h -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef UCARRAY_H
+#define UCARRAY_H
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if !defined (GSDL_NAMESPACE_BROKEN) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+using namespace std;
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\set>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <set.h>
+#else
+#  include <vector>
+#  include <set>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#include <stdio.h>
+
+
+typedef vector<unsigned char> UCArray;
+
+// functions to manipulate UCArrays
+void SetCStr (UCArray &text, const char *cStr);
+// same as SetCStr but first tries to allocate nSizeHint space (only if needed)
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint);
+char * GetCStr(const UCArray& text);
+inline void UCArrayClear (UCArray &a) {
+  a.erase (a.begin(), a.end());
+}
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr);
+inline bool UCArrayCStrEquals(const UCArray &text, const char *cStr) { return UCArrayCStrEquals(text, (const unsigned char *)cStr); }
+
+// stream operator to print UCArray
+ostream &operator<<(ostream &s, const UCArray &a);
+
+// used for reading and writing unsigned characters
+inline bool ReadUC (FILE *f, unsigned char &c) {
+  return (fread (&c, sizeof (unsigned char), 1, f) == 1);
+}
+inline bool WriteUC (FILE *f, unsigned char c) {
+  return (fwrite (&c, sizeof (unsigned char), 1, f) == 1);
+}
+
+// used for reading and writing variable length unsigned longs
+bool ReadVarLenUL (FILE *f, unsigned long &n);
+bool WriteVarLenUL (FILE *f, unsigned long n);
+
+// used for reading and writing unsigned longs
+bool ReadUL (FILE *f, unsigned long &n);
+bool WriteUL (FILE *f, unsigned long n);
+
+// used for reading and writing floats
+bool ReadF (FILE *f, float &n);
+bool WriteF (FILE *f, float n);
+
+// used for reading and writing doubles
+bool ReadD (FILE *f, double &n);
+bool WriteD (FILE *f, double n);
+
+// used for reading and writing arrays to files
+bool ReadUCArray (FILE *f, UCArray &a);
+bool WriteUCArray (FILE *f, const UCArray &a);
+
+
+// compares the two strings in dictionary order
+int DictCompare (const UCArray &a1, const UCArray &a2);
+
+// does the first string start with the second?
+bool StartsWith(const UCArray &a1, const UCArray &a2);
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2);
+
+struct LTUCArray {
+  bool operator()(const UCArray &a1, const UCArray &a2) const
+    { return (a1 < a2); }
+};
+
+struct DictLTUCArray {
+  bool operator() (const UCArray &a1, const UCArray &a2) const
+    { return (DictCompare (a1, a2) < 0); }
+};
+
+
+typedef set<UCArray, LTUCArray> UCArraySet;
+typedef vector<UCArray> UCArrayVector;
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2);
+
+// prev == NULL if no previous string
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a);
+bool ReadPreSufStr (FILE *f, UCArray &a); // a also used for prev
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/WIN32.MAK
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/WIN32.MAK	(revision 21112)
@@ -0,0 +1,191 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+MGPPHOME = ..
+GSDL3HOME = ..\..\..
+AR = lib
+CC = cl
+CPPFLAGS = -GX
+LINK = cl
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -DSHORT_SUFFIX -D__STDC__ -D__GNU_LIBRARY__ -D__MSDOS__
+INCLUDES = -I"." -I"$(MGPPHOME)" -I"$(MGPPHOME)\lib"
+LIBS = $(MGPPHOME)\lib\libmgpp.lib
+
+COMPILE = $(CC) -c  $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+e = .exe
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h
+
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp
+
+
+EXEC          = mgpp_passes$e mgpp_decompress_text$e mgpp_perf_hash_build$e \
+                mgpp_invf_dump$e mgpp_weights_build$e mgpp_invf_dict$e \
+		mgpp_stem_idx$e mgpp_invf_dict_dump$e mgpp_compression_dict$e \
+		mgpp_fast_comp_dict$e QueryTester$e Queryer$e
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : $(EXEC) libtextin.lib libmgpppass.lib
+
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer$e: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester$e: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump$e: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libtextin.lib: $(LIB_OBJS)
+        if exist libtextin.lib del libtextin.lib
+        $(AR) /out:libtextin.lib $(LIB_OBJS)
+
+JNI_LIB_OBJS = \
+	mgpp_passes_4jni$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+libmgpppass.lib: $(JNI_LIB_OBJS)
+	if exist libmgpppass.lib del libmgpppass.lib
+	$(AR) /out:libmgpppass.lib $(JNI_LIB_OBJS)
+
+install: 
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDL3HOME)\bin"
+ 
+clean:
+	del *$o 
+        del libtextin.lib libmgpppass.lib
+
Index: /main/tags/3.01/indexers/mgpp/text/Weights.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Weights.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Weights.cpp	(revision 21112)
@@ -0,0 +1,126 @@
+/**************************************************************************
+ *
+ * Weights.cpp -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "Weights.h"
+#include "sysfuncs.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+
+ApproxWeightsData::ApproxWeightsData () {
+  weightBuf = NULL;
+  table = NULL;
+  Free();
+}
+
+ApproxWeightsData::~ApproxWeightsData () {
+  Free ();
+}
+
+void ApproxWeightsData::Free () {
+  bits = 0;
+  mask = 0;
+  L = 0.0;
+  B = 0.0;
+  numLevelDocs = 0;
+  if (weightBuf != NULL) {
+    delete [] weightBuf;
+    weightBuf = NULL;
+  }
+  if (table != NULL) {
+    delete [] table;
+    table = NULL;
+  }
+}
+
+bool ApproxWeightsData::Read (FILE *approxWeightsFile, unsigned long diskPtr,
+			      unsigned long _numLevelDocs) {
+  Free ();
+  
+  // go to the appropriate place in the weights file
+  fseek (approxWeightsFile, diskPtr, SEEK_SET);
+
+  // read in approx weights parameters
+  ReadUC (approxWeightsFile, bits);
+  ReadD (approxWeightsFile, L);
+  ReadD (approxWeightsFile, B);
+
+  mask = (bits == 32) ? 0xffffffff : (1 << bits) - 1;
+  numLevelDocs = _numLevelDocs;
+
+  // allocate buffer for the weights
+  unsigned long arrSize = (bits*numLevelDocs+sizeof(unsigned long)-1) / 32;
+  weightBuf = new unsigned long[arrSize];
+
+  // read in the weights
+  unsigned long i;
+  for (i=0; i<arrSize; ++i) {
+    ReadUL (approxWeightsFile, weightBuf[i]);
+  }
+
+  // create precomputed table of values if it will be small enough
+  // (if not small enough table == NULL
+  if (bits <= 12) {
+    unsigned long i, tableSize = (1 << bits);
+    table = new float[tableSize];
+    table[0] = L;
+    for (i = 1; i < tableSize; ++i)
+      table[i] = table[i - 1] * B;
+  }
+
+  return true;
+}
+
+float ApproxWeightsData::GetLowerApproxDocWeight (unsigned long levelDocNum) {
+  // sanity check
+  if (bits == 0 || weightBuf == NULL) return 1.0;
+
+  register unsigned long c, pos;
+  register unsigned long *dw;
+
+  // get the compressed version of the weight
+  pos = levelDocNum * bits;
+  dw = &weightBuf[pos / MAXBITS];
+  pos &= (MAXBITS - 1);
+  c = *dw >> pos;
+  if (pos + bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - pos);
+  c &= mask;
+
+  // decompress the weight
+  if (table != NULL) return table[c];
+  return (L * pow (B, (double) c));
+}
+
+
+
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum) {
+  // go to the appropriate place in the weights file
+  fseek (exactWeightsFile, diskPtr+sizeof(float)*levelDocNum, SEEK_SET);
+
+  // read in the exact weight
+  float weight;
+  ReadF (exactWeightsFile, weight);
+  
+  return weight;
+}
Index: /main/tags/3.01/indexers/mgpp/text/Weights.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/Weights.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/Weights.h	(revision 21112)
@@ -0,0 +1,56 @@
+/**************************************************************************
+ *
+ * Weights.h -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef WEIGHTS_H
+#define WEIGHTS_H
+
+#include <stdio.h>
+
+
+class ApproxWeightsData {
+public:
+  unsigned char bits;
+  unsigned long mask;
+  double L;
+  double B;
+  unsigned long numLevelDocs;
+  unsigned long *weightBuf;
+  float *table;   // precomputed translation table
+  
+  ApproxWeightsData ();
+  ~ApproxWeightsData ();
+
+  void Free (); // frees any allocated memory
+  
+  bool Read (FILE *approxWeightsFile, unsigned long diskPtr,
+	     unsigned long _numLevelDocs);
+
+  float GetLowerApproxDocWeight (unsigned long levelDocNum);
+};
+
+
+// diskPtr points to the start of the exact document weights
+// for this level
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/WordData.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/WordData.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/WordData.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * WordData.h -- 
+ * Copyright (C) 1999  NZDL
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+class WordData
+{
+
+ public:
+  unsigned long docCount;
+  unsigned long wordCount;
+
+  inline void read(FILE *f)
+    {
+      fread((char *) &docCount, sizeof(docCount), 1, f);
+      fread((char *) &wordCount, sizeof(wordCount), 1, f);
+
+      NTOHUL(docCount);
+      NTOHUL(wordCount);
+    }
+
+  inline unsigned long documents()
+    {
+      return docCount;
+    }
+
+  inline int words()
+    {
+      return wordCount;
+    }
+};
Index: /main/tags/3.01/indexers/mgpp/text/build.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/build.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/build.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#include "TextEl.h"
+#include "TagInfo.h"
+
+
+int init_special (const TagInfo &tagInfo, char *file_name);
+int init_text_1 (const TagInfo &tagInfo, char *file_name);
+int init_text_2 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_2 (const TagInfo &tagInfo, char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_2 (const TagInfo &tagInfo, const TextElArray &doc);
+/*
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (const TagInfo &tagInfo, char *filename);
+int done_text_1 (const TagInfo &tagInfo, char *filename);
+int done_text_2 (const TagInfo &tagInfo, char *filename);
+int done_ivf_1 (const TagInfo &tagInfo, char *filename);
+int done_ivf_2 (const TagInfo &tagInfo, char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/comp_dict.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/comp_dict.cpp	(revision 21112)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * comp_dict.cpp -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = (dict_hash_table *) Xmalloc (sizeof (dict_hash_table) +
+		                            (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = (u_char **) Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  memset (ht->table, '\0', ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = (u_char *) Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; ++i, ++words)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register unsigned long tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+inline int LoadWords(FILE *dict, int which)
+{
+  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+    return 1;
+  
+  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+    return 1;
+
+  return 0;
+}
+
+inline int LoadHuffmanCodes(FILE *dict, int which)
+{
+  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 1;
+  if (!(char_codes[which] =
+	Generate_Huffman_Codes (&char_huff[which], NULL)))
+    return 2;
+  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 3;
+  if (!(lens_codes[which] =
+	Generate_Huffman_Codes (&lens_huff[which], NULL)))
+    return 4;
+
+  return 0;
+}
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (LoadWords(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (LoadHuffmanCodes(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (LoadHuffmanCodes(dict, which) != 0)
+		goto error;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/indexers/mgpp/text/comp_dict.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,48 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define HASH_RATIO 2
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/indexers/mgpp/text/hash.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/hash.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/hash.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HASH
+#define H_HASH
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; --len)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/invf.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/invf.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/invf.cpp	(revision 21112)
@@ -0,0 +1,914 @@
+/**************************************************************************
+ *
+ * invf.cpp -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "invf.h"
+#include "UCArray.h"
+
+
+invf_dict_header::invf_dict_header () {
+  Clear();
+}
+
+invf_dict_header::~invf_dict_header () {
+}
+
+void invf_dict_header::Clear() {
+  lookback = 0;
+  word_dict_start = 0;
+  word_dict_size = 0;
+  tag_dict_start = 0;
+  tag_dict_size = 0;
+  num_docs = 0;
+  num_frags = 0;
+  num_words = 0;
+  total_bytes = 0;
+  index_string_bytes = 0;
+  num_levels = 0;
+}
+
+bool invf_dict_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, word_dict_start) &&
+	  ReadUL (f, word_dict_size) &&
+	  ReadUL (f, tag_dict_start) &&
+	  ReadUL (f, tag_dict_size) &&
+	  ReadUL (f, num_docs) &&
+	  ReadUL (f, num_frags) &&
+	  ReadUL (f, num_words) &&
+	  ReadUL (f, total_bytes) &&
+	  ReadUL (f, index_string_bytes) &&
+	  ReadUL (f, num_levels));
+}
+
+bool invf_dict_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, word_dict_start) &&
+	  WriteUL (f, word_dict_size) &&
+	  WriteUL (f, tag_dict_start) &&
+	  WriteUL (f, tag_dict_size) &&
+	  WriteUL (f, num_docs) &&
+	  WriteUL (f, num_frags) &&
+	  WriteUL (f, num_words) &&
+	  WriteUL (f, total_bytes) &&
+	  WriteUL (f, index_string_bytes) &&
+	  WriteUL (f, num_levels));
+}
+
+
+
+void dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+}
+
+bool dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq));
+}
+
+bool dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq));
+}
+
+
+void word_dict_el::Clear () {
+  dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_dict_el::~word_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+  
+bool word_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!dict_el::Read (f)) return false;
+
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_dict_el::Write (FILE *f, const UCArray *lastEl,
+			  unsigned long numLevels) const {
+  if (!dict_el::Write (f, lastEl)) return false;
+
+  if (levelFreqs == NULL) return false;
+
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+block_dict_header::block_dict_header () {
+  Clear();
+}
+
+void block_dict_header::Clear () {
+  invf_dict_header::Clear();
+
+  entries_per_wblk = 0;
+  num_wblks = 0;
+  max_wblk_size = 0;
+  wblk_start = 0;
+  wblk_idx_start = 0;
+  
+  entries_per_tblk = 0;
+  num_tblks = 0;
+  max_tblk_size = 0;
+  tblk_start = 0;
+  tblk_idx_start = 0;
+}
+
+bool block_dict_header::Read (FILE *f) {
+  return (invf_dict_header::Read (f) &&
+
+	  ReadUL (f, entries_per_wblk) &&
+	  ReadUL (f, num_wblks) &&
+	  ReadUL (f, max_wblk_size) &&
+	  ReadUL (f, wblk_start) &&
+	  ReadUL (f, wblk_idx_start) &&
+  
+	  ReadUL (f, entries_per_tblk) &&
+	  ReadUL (f, num_tblks) &&
+	  ReadUL (f, max_tblk_size) &&
+	  ReadUL (f, tblk_start) &&
+	  ReadUL (f, tblk_idx_start));
+}
+
+bool block_dict_header::Write (FILE *f) const {
+  return (invf_dict_header::Write (f) &&
+
+	  WriteUL (f, entries_per_wblk) &&
+	  WriteUL (f, num_wblks) &&
+	  WriteUL (f, max_wblk_size) &&
+	  WriteUL (f, wblk_start) &&
+	  WriteUL (f, wblk_idx_start) &&
+  
+	  WriteUL (f, entries_per_tblk) &&
+	  WriteUL (f, num_tblks) &&
+	  WriteUL (f, max_tblk_size) &&
+	  WriteUL (f, tblk_start) &&
+	  WriteUL (f, tblk_idx_start));
+}
+
+
+
+
+
+void block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+  invf_ptr = 0;
+}
+
+bool block_dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq) &&
+	  ReadUL (f, invf_ptr));
+}
+
+bool block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq) &&
+	  WriteUL (f, invf_ptr));
+}
+
+
+
+
+
+void word_block_dict_el::Clear () {
+  block_dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_block_dict_el::~word_block_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_block_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+
+bool word_block_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!block_dict_el::Read (f)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_block_dict_el::Write (FILE *f, const UCArray *lastEl,
+				unsigned long numLevels) const {
+  if (!block_dict_el::Write (f, lastEl)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+  
+  return true;
+}
+
+
+
+
+block_idx_info::block_idx_info () {
+  Clear ();
+}
+
+void block_idx_info::Clear () {
+  el.erase (el.begin(), el.end());
+  block_ptr = 0;
+}
+
+bool block_idx_info::Read (FILE *f) {
+  return (ReadUCArray (f, el) &&
+	  ReadUL (f, block_ptr));
+}
+
+bool block_idx_info::Write (FILE *f) const {
+  return (WriteUCArray (f, el) &&
+	  WriteUL (f, block_ptr));
+}
+
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx) {
+  blockIdx.erase (blockIdx.begin(), blockIdx.end());
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  block_idx_info bi;
+  while (arraySize > 0) {
+    if (!bi.Read (f)) return false;
+    blockIdx.push_back (bi);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx) {
+  // write out the array size
+  if (!WriteVarLenUL (f, blockIdx.size())) return false;
+
+  block_idx::const_iterator here = blockIdx.begin();
+  block_idx::const_iterator end = blockIdx.end();
+  while (here != end) {
+    if (!(*here).Write (f)) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+stem_idx_header::stem_idx_header () {
+  Clear ();
+}
+
+void stem_idx_header::Clear () {
+  lookback = 0;
+  dict_size = 0;
+
+  entries_per_block = 0;
+  num_blocks = 0;
+  max_block_size = 0;
+  blocks_start = 0;
+  block_idx_start = 0;
+  
+  stemmer_num = 0;
+  stem_method = 0;
+}
+
+bool stem_idx_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, dict_size) &&
+	  
+	  ReadUL (f, entries_per_block) &&
+	  ReadUL (f, num_blocks) &&
+	  ReadUL (f, max_block_size) &&
+	  ReadUL (f, blocks_start) &&
+	  ReadUL (f, block_idx_start) &&
+	  
+	  ReadUL (f, stemmer_num) &&
+	  ReadUL (f, stem_method));
+}
+
+bool stem_idx_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, dict_size) &&
+	  
+	  WriteUL (f, entries_per_block) &&
+	  WriteUL (f, num_blocks) &&
+	  WriteUL (f, max_block_size) &&
+	  WriteUL (f, blocks_start) &&
+	  WriteUL (f, block_idx_start) &&
+	  
+	  WriteUL (f, stemmer_num) &&
+	  WriteUL (f, stem_method));
+}
+
+
+
+stem_block_dict_el::stem_block_dict_el () {
+  Clear ();
+}
+
+void stem_block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  equivWords.erase (equivWords.begin(), equivWords.end());
+}
+
+bool stem_block_dict_el::Read (FILE *f) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (!ReadPreSufStr (f, el)) return false;
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  unsigned long wordNum;
+  while (arraySize > 0) {
+    if (!ReadUL (f, wordNum)) return false;
+    equivWords.push_back (wordNum);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool stem_block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  if (!WritePreSufStr (f, lastEl, el)) return false;
+
+  // write out the array size
+  if (!WriteVarLenUL (f, equivWords.size())) return false;
+
+  vector<unsigned long>::const_iterator here = equivWords.begin();
+  vector<unsigned long>::const_iterator end = equivWords.end();
+  while (here != end) {
+    if (!WriteUL (f, (*here))) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+invf_file_header::invf_file_header () {
+  Clear ();
+}
+
+void invf_file_header::Clear () {
+  no_of_words = 0;
+  no_of_tags = 0;
+  skip_mode = SKIP_MODE_NO_SKIPS;
+  word_level_index = 0;
+  
+  int i;
+  for (i=0; i<16; ++i) params[i] = 0;
+}
+  
+bool invf_file_header::Read (FILE *f) {
+  if (!ReadUL (f, no_of_words) ||
+      !ReadUL (f, no_of_tags) ||
+      !ReadUL (f, skip_mode) ||
+      !ReadUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!ReadUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+bool invf_file_header::Write (FILE *f) const {
+  if (!WriteUL (f, no_of_words) ||
+      !WriteUL (f, no_of_tags) ||
+      !WriteUL (f, skip_mode) ||
+      !WriteUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!WriteUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+
+
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  // make sure the element number isn't out of range
+  if (elNum > bIdx.size()*entriesPerBlock) return false;
+
+  blockIdxNum = elNum / entriesPerBlock;
+  blockStartElNum = blockIdxNum * entriesPerBlock;
+  
+  return true;
+}
+
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  unsigned long begin = 0;
+  unsigned long bIdxEnd = bIdx.size();
+  unsigned long end = bIdxEnd;
+  unsigned long mid;
+  while (begin < end) {
+    mid = (begin+end)/2;
+    if (DictCompare (el, bIdx[mid].el) < 0) {
+      end = mid;
+    } else if (mid+1 < bIdxEnd &&
+	       DictCompare (el, bIdx[mid+1].el) >= 0) {
+      begin = mid+1;
+    } else {
+      blockIdxNum = mid;
+      blockStartElNum = blockIdxNum * entriesPerBlock;
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+
+
+// use the block dictionary functions for tag entries, and word block dict
+// functions for word entries.
+
+
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+// ------------------------------------------
+// functions for partial term matching
+// ie find all words that start with xxx
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold) {
+
+  UCArrayClear (dictEl.el);
+  elNumList.erase (elNumList.begin(), elNumList.end());
+  unsigned long elNum;
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  // will this work??
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum)) {
+    return false;
+  }
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+  
+  bool still_looking = true;
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // test each element
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    if (StartsWithCasefold(dictEl.el, el)) {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+      }
+      still_looking=false; // we have found one now, so stop the next time we don't have a match
+    } else if (!still_looking) {
+      // we have found a match previously, and now this doesn't match
+      return true;
+    } // else keep looking
+    
+    ++elNum;
+  }
+  // if we get here, we are either still searching for the first
+  // case, or we are collecting terms.
+  if (still_looking) {
+    //we haven't found a match yet, just check the next element,
+    dictEl.Read (dictFile, numLevels);
+    if (!StartsWithCasefold(dictEl.el, el)) {
+      // the first element of the next block is not a match, so there are no matches
+      return false;
+    } else {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+	++elNum;
+      } 
+
+    }
+  }
+  // just keep accumulating matches until there are no more
+  dictEl.Read (dictFile, numLevels);
+  while (StartsWithCasefold(dictEl.el, el)) {
+    if (casefold || StartsWith(dictEl.el, el)) {
+      elNumList.push_back(elNum);
+    }
+    ++elNum;
+    dictEl.Read (dictFile, numLevels);
+  }
+  return true;
+  
+}
+
+
+//----------------------------------------------------------------
+// functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+				   const block_idx &bIdx,
+				   unsigned long entriesPerBlock,
+				   unsigned long dictSize,
+				   unsigned long numLevels,
+				   const UCArray &el,
+				   word_block_dict_el &dictEl,
+				   unsigned long &elNum) {
+
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (el, dictEl.el); // look for the first word that is
+                                     // greater or equal to the el
+    if (res <= 0) {
+      return true; // found one
+   } 
+    
+    ++elNum;
+  }
+  // it must be the last term
+  return true;
+
+
+}
+
+
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms) {
+
+  word_block_dict_el dictEl;  
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  terms.push_back(dictEl.el);
+  while (curElNum <= lastElNum ) {
+    dictEl.Read(dictFile, numLevels);
+    terms.push_back(dictEl.el);
+    ++curElNum;
+  }
+  
+
+  return true;
+}
+
+// NOte: before each addition of dictEl to the array, the level freqs array
+// is deleted, as this was causing problems - generating a seg fault, I think if
+// the vector had to be reallocated or something.
+// setNumLevels has to be called each time before a read, now, to set up the level
+//freqs array. this is necessary.
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms) {
+
+  word_block_dict_el dictEl;
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  block_dict_el elem;
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  dictEl.levelFreqs = NULL;
+  terms.push_back(dictEl);
+
+  while (curElNum <= lastElNum ) { 
+    dictEl.SetNumLevels(numLevels);
+    dictEl.Read(dictFile, numLevels);
+    dictEl.levelFreqs = NULL;
+    terms.push_back(dictEl);
+    ++curElNum;
+  }
+
+  return true;
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/indexers/mgpp/text/invf.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/invf.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/invf.h	(revision 21112)
@@ -0,0 +1,352 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF
+#define H_INVF
+
+#include <stdio.h>
+
+#include "UCArray.h"
+
+// NOTE: This does not include the magic number
+// header info for .invf.dict file
+struct invf_dict_header {
+  unsigned long lookback;
+  unsigned long word_dict_start;
+  unsigned long word_dict_size;
+  unsigned long tag_dict_start;
+  unsigned long tag_dict_size;
+  unsigned long num_docs;
+  unsigned long num_frags;
+  unsigned long num_words;
+  unsigned long total_bytes;
+  unsigned long index_string_bytes;
+  unsigned long num_levels;
+
+  invf_dict_header ();
+  virtual ~invf_dict_header ();
+  virtual void Clear();
+
+  virtual bool Read (FILE *f);
+  virtual bool Write (FILE *f) const;
+};
+
+
+struct dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur;
+  unsigned long freq;
+
+  virtual void Clear ();
+  dict_el () { Clear (); }
+  virtual ~dict_el () { }
+
+  // Read assumes that the last word is in el
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+struct word_dict_el : public dict_el {
+  unsigned long *levelFreqs;
+
+  void Clear ();
+  word_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+
+// wblk = word block
+// tblk = tag block
+// this version of the blocked dictionary uses a fixed number
+// of entries per block, not a fixed block size
+// info for .invf.dict.blocked file
+// blocked dict has a heap of blocks, some for words, some for tags
+// and an index into each set of blocks. The index has pointers to
+// the first entry in each block. Can do a binary search on the index
+// to find out which block an elemnet is in
+struct block_dict_header : public invf_dict_header {
+  // note: word_dict_start and tag_dict_start are undefined
+  // for blocked dictionaries
+
+  unsigned long entries_per_wblk; // word blocks
+  unsigned long num_wblks;
+  unsigned long max_wblk_size;
+  unsigned long wblk_start;
+  unsigned long wblk_idx_start;
+  
+  unsigned long entries_per_tblk; // tag blocks
+  unsigned long num_tblks;
+  unsigned long max_tblk_size;
+  unsigned long tblk_start;
+  unsigned long tblk_idx_start;
+  
+  block_dict_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct block_dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur; // # entries in invf file - if have a
+  // word level index, this is the same as freq, otherwise, its the number
+  // of fragments containing this word
+  unsigned long freq; // # of times this word occurs
+  unsigned long invf_ptr; // pointer into inverted file
+
+  virtual void Clear ();
+  block_dict_el () { Clear (); }
+  virtual ~block_dict_el () { }
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+struct word_block_dict_el : public block_dict_el {
+  unsigned long *levelFreqs; // freq of the word at each level
+
+  void Clear ();
+  word_block_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_block_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+typedef vector<word_block_dict_el> word_block_dict_el_array;
+
+struct block_idx_info {
+  UCArray el;
+  unsigned long block_ptr;
+
+  block_idx_info ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// used for an index into the word and tag blocks
+typedef vector<block_idx_info> block_idx;
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx);
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx);
+
+
+
+struct stem_idx_header {
+  unsigned long lookback;
+  unsigned long dict_size;
+
+  unsigned long entries_per_block;
+  unsigned long num_blocks;
+  unsigned long max_block_size;
+  unsigned long blocks_start;
+  unsigned long block_idx_start;
+  
+  unsigned long stemmer_num;
+  unsigned long stem_method;
+
+  stem_idx_header ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+struct stem_block_dict_el {
+  UCArray el; // word or tag
+  vector<unsigned long> equivWords;
+  
+  stem_block_dict_el ();
+  void Clear ();
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+
+#define SKIP_MODE_NO_SKIPS 0
+
+// invf file - has a list of frags for each word, but the word is not 
+//  stored in the invf file - the dictionaries store the words, along
+// with num entries, and a pointer into invf file
+struct invf_file_header {
+  unsigned long no_of_words;
+  unsigned long no_of_tags;
+  unsigned long skip_mode;
+  unsigned long word_level_index; // 1 if word level index
+  unsigned long params[16];
+
+  invf_file_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+
+
+
+
+// the search functions returns true if a block that could
+// satisfy the request is found. these functions assume that
+// the block index is sorted by DictCompare (or DictLTUCArray)
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum);
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum);
+
+
+// The next six functions use SearchElNum and SearchEl
+// for a particular type of dictionary (Block, WordBlock, or StemBlock)
+// and then look up the entry
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl);
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum);
+
+// assumes the numLevels has been set for dictEl
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl);
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl);
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+//-----------------------------------------------------
+// New functions for partial matching
+// returns a list of word numbers from the main block index whose
+// words start with el.
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold); 
+//----------------------------------------------------------
+
+// new functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+// returns a list of word_block_dict_el, with no levelfreqs
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms);
+				    
+// just returns a list of terms
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms);
+				    
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/ivf.pass1.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/ivf.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/ivf.pass1.cpp	(revision 21112)
@@ -0,0 +1,775 @@
+/**************************************************************************
+ *
+ * ivf.pass1.cpp -- Memory efficient pass 1 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+// a fragment corresponds to the index level (word-level is the
+// minimum index level)
+
+
+// structure to determine level information
+struct LevelWorker {
+  unsigned long lastLevelDocNum;
+  unsigned long count;
+
+  LevelWorker () {
+    lastLevelDocNum = count = 0;
+  }
+};
+
+
+// note: the word is stored in the map
+struct IvfWordInfo {
+  unsigned long wordCount;      // word frequency
+  unsigned long fragCount;      // number of fragments that contain the word
+  unsigned long lastFragNum;    // last fragment to contain the word
+  unsigned long chunkWordCount; // word frequency within this chunk
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the word
+
+  LevelWorker *levels;          // level info for this word
+  
+  IvfWordInfo ();
+  ~IvfWordInfo ();
+  void Clear ();                // will delete levels
+};
+
+typedef map<UCArray, IvfWordInfo, DictLTUCArray> IvfWordInfoMap;
+typedef vector<IvfWordInfoMap::iterator> IvfWordInfoItArray;
+
+// tags don't require as much information
+struct IvfTagInfo {
+  unsigned long tagCount;       // tag frequency
+  unsigned long fragCount;      // number of fragments that contain the tag
+  unsigned long lastFragNum;    // last fragment to contain the tag
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the tag
+
+  IvfTagInfo ();
+  void Clear ();
+};
+
+typedef map<UCArray, IvfTagInfo, DictLTUCArray> IvfTagInfoMap;
+typedef vector<IvfTagInfoMap::iterator> IvfTagInfoItArray;
+
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define CHECK_DIV 1.5
+
+
+static FILE *ic;		// the invf chunk file
+static stdio_bitio_buffer icb;
+
+IvfWordInfoMap ivfWordInfo;
+IvfWordInfoItArray ivfWordInfoOccurOrder;
+IvfTagInfoMap ivfTagInfo;
+IvfTagInfoItArray ivfTagInfoOccurOrder;
+
+static unsigned long chunksWritten;
+static unsigned long maxMemNeeded;
+
+static unsigned long numDocs;
+static unsigned long numChunkDocs;
+
+static unsigned long numFrags;
+static unsigned long numChunkFrags;
+
+static unsigned long numWords;
+
+// the number of document numbers in the inverted file
+static unsigned long numChunkEntries;
+
+// next entry in the inverted file to check memory
+// requirements for the current chunk
+static unsigned long entryCheckPoint;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+
+
+IvfWordInfo::IvfWordInfo () {
+  levels = NULL;
+  Clear();
+}
+
+IvfWordInfo::~IvfWordInfo () {
+  if (levels != NULL) delete [] levels;
+}
+
+void IvfWordInfo::Clear () {
+  wordCount = 0;
+  chunkWordCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+
+  if (levels != NULL) {
+    delete [] levels;
+    levels = NULL;
+  }
+}
+
+
+IvfTagInfo::IvfTagInfo () {
+  Clear();
+}
+
+void IvfTagInfo::Clear () {
+  tagCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+}
+
+
+
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name) {
+  // set up the chunk file
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb",
+			  MAGIC_CHUNK, MG_MESSAGE)))
+    return COMPERROR;
+  fwrite ("    ", sizeof (u_long), 1, ic);	// Space for the maxmem
+  icb.attachFile (ic);
+  icb.encodeStart();
+
+  // reset global variables
+  ivfWordInfo.erase (ivfWordInfo.begin(), ivfWordInfo.end());
+  ivfWordInfoOccurOrder.erase (ivfWordInfoOccurOrder.begin(), ivfWordInfoOccurOrder.end());
+  ivfTagInfo.erase (ivfTagInfo.begin(), ivfTagInfo.end());
+  ivfTagInfoOccurOrder.erase (ivfTagInfoOccurOrder.begin(), ivfTagInfoOccurOrder.end());
+  
+  chunksWritten = 0;
+  maxMemNeeded = 0;
+ 
+  numDocs = 0;
+  numChunkDocs = 0;
+
+  numFrags = 0;
+  numChunkFrags = 0;
+
+  numWords = 0;
+
+  numChunkEntries = 0;
+  entryCheckPoint = (unsigned long) ((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+
+  // init the level information
+  ivfLevel.Clear();
+  ivfLevel.docTag = tagInfo.docTag;
+  ivfLevel.indexLevel = tagInfo.indexLevel;
+  IvfLevelInfo blankLevel;
+  UCArraySet::const_iterator levelHere = tagInfo.levelTags.begin();
+  UCArraySet::const_iterator levelEnd = tagInfo.levelTags.end();
+  while (levelHere != levelEnd) {
+    blankLevel.levelTag = *levelHere;
+    ivfLevel.levelInfo[*levelHere] = blankLevel;
+    ++levelHere;
+  }
+  
+  return COMPALLOK;
+}
+
+static void ProcessOpenTag (const TagInfo &tagInfo, const TextEl &el,
+			    bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for start of next fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    ++numFrags;
+    ++numChunkFrags;
+    inFrag = true;
+  }
+  
+  // update tag stats
+  IvfTagInfo &i = ivfTagInfo[el.tagName];
+  if (i.tagCount == 0) {
+    // new tag, add to list of iterators
+    IvfTagInfoMap::iterator iIt = ivfTagInfo.find (el.tagName);
+    ivfTagInfoOccurOrder.push_back (iIt);
+  }
+  ++i.tagCount;
+
+  // all open tags count as new tags
+  ++numChunkEntries;
+  ++i.fragCount;
+  ++i.chunkFragCount;
+  i.lastFragNum = numFrags;
+  
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    ++levelIt->second.numEntries;
+    levelIt->second.workInLevel = true;
+  }
+}
+
+static void ProcessCloseTag (const TagInfo &tagInfo, const TextEl &el,
+			     bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for end of fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    inFrag = false;
+  }
+
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    (*levelIt).second.workInLevel = false;
+  }
+}
+
+static void ProcessText (const TagInfo &tagInfo, const TextEl &el,
+			 bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // make sure this text is to be indexed
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  UCArray word;
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+  // Each token is then inserted into the set if it does
+  // not exist or has it's frequency count incremented if it does.
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexWord (textHere, textEnd, word);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (!word.empty()) {
+      ++numWords;
+
+      if (wordLevelIndex) {
+	++numFrags;
+	++numChunkFrags;
+      }
+      
+      // update word stats
+
+      IvfWordInfo &i = ivfWordInfo[word];
+      if (i.wordCount == 0) {
+	// new word
+	// add to list of iterators
+	IvfWordInfoMap::iterator iIt = ivfWordInfo.find (word);
+	ivfWordInfoOccurOrder.push_back (iIt);
+
+	// add level information array
+	if (ivfLevel.levelInfo.size() > 0) 
+	  i.levels = new LevelWorker [ivfLevel.levelInfo.size()];
+      }
+
+      ++i.wordCount;
+      ++i.chunkWordCount;
+      if (numFrags > i.lastFragNum) {
+	++numChunkEntries;
+	++i.fragCount;
+	++i.chunkFragCount;
+	i.lastFragNum = numFrags;
+      }
+
+      // update level information for this word
+      if (i.levels != NULL) {
+	IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+	IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+	LevelWorker *levelWorkerPtr = i.levels;
+	while (levelHere != levelEnd) {
+
+	  // check to make sure the level encompases this fragment
+	  if (!(*levelHere).second.workInLevel) {
+	    cerr << "Level tag <" << (*levelHere).first
+		 << "> does not encompass all fragments\n";
+	    exit (1);
+	  }
+	  
+	  if ((*levelHere).second.numEntries > (*levelWorkerPtr).lastLevelDocNum) {
+	    (*levelWorkerPtr).lastLevelDocNum = (*levelHere).second.numEntries;
+	    ++levelWorkerPtr->count;
+	  }
+	  
+	  ++levelHere;
+	  ++levelWorkerPtr;
+	}
+      }
+    }
+  }
+}
+
+
+static unsigned long MemoryRequired (bool wordLevelIndex) {
+  register unsigned long total = 0;
+
+  // add memory required for word entries
+  IvfWordInfoMap::const_iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::const_iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    register const IvfWordInfo &info = (*wordHere).second;
+    if (info.chunkFragCount > 0) {
+      total += BIO_Bblock_Bound (numChunkFrags, info.chunkFragCount);
+      if (!wordLevelIndex) {
+	total += info.chunkWordCount;
+      }
+    }
+    
+    ++wordHere;
+  }
+
+  // add memory required for tag entries
+  IvfTagInfoMap::const_iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::const_iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    register const IvfTagInfo &info = (*tagHere).second;
+    if (info.chunkFragCount > 0) {
+      // two d entries for each frag entry
+      unsigned long pTag = info.chunkFragCount*2;
+      total += BIO_Bblock_Bound (numChunkFrags+pTag, pTag);
+    }
+    
+    ++tagHere;
+  }
+  
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+/*
+static void PrintChunkInfo (unsigned long mem) {
+  cout << "Chunk Number: " << chunksWritten << "\n";
+  cout << "numChunkDocs " << numChunkDocs << "\n";
+  cout << "numChunkFrags " << numChunkFrags << "\n";
+  cout << "mem " << mem << "\n";
+  cout << "numWords " << ivfWordInfo.size() << "\n";
+  cout << "numTags " << ivfTagInfo.size() << "\n\n";
+
+  // output debug tag information in dictionary order
+  IvfTagInfoMap::iterator tagMapHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagMapEnd = ivfTagInfo.end();
+  unsigned long tagNum = 0;
+  while (tagMapHere != tagMapEnd) {
+    cout << (*tagMapHere).first << " " << tagNum
+	 << " " << (*tagMapHere).second.chunkFragCount << "\n";
+    ++tagNum;
+    ++tagMapHere;
+  }
+}
+*/
+
+static void OutputChunkInfo (unsigned long mem, bool /*wordLevelIndex*/) {
+  ++chunksWritten;
+
+  // sanity check
+  if (ivfWordInfo.size() != ivfWordInfoOccurOrder.size()) {
+    Message ("ERROR: Word information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfWordInfo.size(),
+	     (unsigned int)ivfWordInfoOccurOrder.size());
+    exit (1);
+  }
+  if (ivfTagInfo.size() != ivfTagInfoOccurOrder.size()) {
+    Message ("ERROR: Tag information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfTagInfo.size(),
+	     (unsigned int)ivfTagInfoOccurOrder.size());
+    exit (1);
+  }
+
+  icb.gamma_encode (numChunkDocs + 1, NULL);
+  icb.gamma_encode (numChunkFrags + 1, NULL);
+  icb.gamma_encode (mem + 1, NULL);
+  icb.gamma_encode (ivfWordInfo.size() + 1, NULL);
+  icb.gamma_encode (ivfTagInfo.size() + 1, NULL);
+
+  /*  PrintChunkInfo (mem);*/
+
+  // output word information in occurance order
+  IvfWordInfoItArray::iterator wordHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordEnd = ivfWordInfoOccurOrder.end();
+  while (wordHere != wordEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordHere)).second;
+    
+    icb.gamma_encode (ivfWordInfo.chunkWordCount + 1, NULL);
+    if (ivfWordInfo.chunkWordCount >= 2) {
+      icb.gamma_encode (ivfWordInfo.chunkFragCount, NULL);
+    }
+
+    ivfWordInfo.lastFragNum = 0;
+    ivfWordInfo.chunkWordCount = 0;
+    ivfWordInfo.chunkFragCount = 0;
+    
+    ++wordHere;
+  }
+
+  // output tag information in occurance order
+  IvfTagInfoItArray::iterator tagHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagEnd = ivfTagInfoOccurOrder.end();
+  while (tagHere != tagEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagHere)).second;
+
+    icb.gamma_encode (ivfTagInfo.chunkFragCount + 1, NULL);
+
+    ivfTagInfo.lastFragNum = 0;
+    ivfTagInfo.chunkFragCount = 0;
+    
+    ++tagHere;
+  }
+
+  numChunkDocs = 0;
+  numChunkFrags = 0;
+  numChunkEntries = 0;
+}
+
+
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  ++numDocs;
+  ++numChunkDocs;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE)
+      ProcessOpenTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == CloseTagE)
+      ProcessCloseTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == TextE)
+      ProcessText (tagInfo, *here, inFrag);
+    
+    ++here;
+  }
+
+  // check the amount of memory needed for this chunk
+  if (numChunkEntries >= entryCheckPoint) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    if (mem >= invf_buffer_size * CHECK_CLOSE) {
+      if (mem > maxMemNeeded) maxMemNeeded = mem;
+      OutputChunkInfo (mem, wordLevelIndex);
+      entryCheckPoint = (unsigned long)
+	((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+      
+    } else {
+      entryCheckPoint = (unsigned long)
+	(entryCheckPoint * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	 entryCheckPoint);
+      if (entryCheckPoint <= numChunkEntries)
+	entryCheckPoint = numChunkEntries + 1;
+    }
+  }
+  
+  return COMPALLOK;
+}
+
+
+static void CalcInvfDictSize (unsigned long &totalBytes,
+			      unsigned long &indexStringBytes) {
+  totalBytes = 0;	// The sum of the length of all words, including 
+			// the length byte
+  indexStringBytes = 0; // The amount of space required to store the 
+			// words in the diction, this takes into account 
+			// the prefixes
+  const UCArray *lastWord = NULL;
+  
+  // calculate size of word information
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    unsigned long wordSize = (*wordHere).first.size();
+    totalBytes += wordSize + 1;
+    indexStringBytes += wordSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*wordHere).first);
+    lastWord = &((*wordHere).first);
+    
+    ++wordHere;
+  }
+
+  // calculate size of tag information
+  lastWord = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    unsigned long tagSize = (*tagHere).first.size();
+    totalBytes += tagSize + 1;
+    indexStringBytes += tagSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*tagHere).first);
+    lastWord = &((*tagHere).first);
+    
+    ++tagHere;
+  }
+}
+
+
+//      OutputInvfDict ():
+//                writes out the stemmed dictionary file
+//                in the following format 
+//                        lookback value (int)
+//                        totalbytes value (int)
+//                        indexstringbytes (int)
+//                        for each word 
+//                          wordlen (4 bits)
+//                          prefix match (4 bits)
+//                          word (wordlen bytes)
+//                          word frequency (int)
+//                          word count (int)
+//
+//        Accesses outside variables:     
+//
+//        Return value...:                
+static void OutputInvfDict (char *filename) {
+  // create the dictionary header
+  invf_dict_header idh;
+  idh.word_dict_size = ivfWordInfo.size();
+  idh.tag_dict_size = ivfTagInfo.size();
+  idh.num_docs = numDocs;
+  idh.num_frags = numFrags;
+  idh.num_words = numWords;
+  idh.num_levels = ivfLevel.levelInfo.size();
+  CalcInvfDictSize (idh.total_bytes, idh.index_string_bytes);
+
+  // create the inverted dictionary file
+  FILE *sp;
+  if (!(sp = create_file (filename, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))
+    return;
+
+  // write out the dictionary header
+  if (!idh.Write (sp)) { fclose (sp); return; }
+
+  // remember where the word dictionary starts
+  idh.word_dict_start = ftell (sp);
+  
+  // output the word dictionary
+  const UCArray *lastWord = NULL;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisWord = (*wordHere).first;
+    WritePreSufStr (sp, lastWord, thisWord);
+
+    // output the number of fragments the word appeared in and the
+    // number of times the word appeared
+    WriteUL (sp, (*wordHere).second.fragCount);
+    WriteUL (sp, (*wordHere).second.wordCount);
+
+    // output frequency information for each level
+    // note that we are expecting every word to have
+    // level information
+    LevelWorker *lwHere = (*wordHere).second.levels;
+    LevelWorker *lwEnd = lwHere + idh.num_levels;
+    while (lwHere != lwEnd) {
+      WriteUL (sp, (*lwHere).count);
+      ++lwHere;
+    }
+    
+    lastWord = &thisWord;
+    ++wordHere;
+  }
+
+  // remember where the tag dictionary starts
+  idh.tag_dict_start = ftell (sp);
+  
+  // output the tag dictionary
+  const UCArray *lastTag = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisTag = (*tagHere).first;
+    WritePreSufStr (sp, lastTag, thisTag);
+
+    // output the number of fragments the tag appeared in and the
+    // number of times the tag appeared
+    WriteUL (sp, (*tagHere).second.fragCount);
+    WriteUL (sp, (*tagHere).second.tagCount);
+    
+    lastTag = &thisTag;
+    ++tagHere;
+  }
+
+  // write out the updated header
+  fseek (sp, sizeof (u_long), SEEK_SET);
+  if (!idh.Write (sp)) { fclose (sp); return; }
+  
+  fclose (sp);
+}
+
+static void OutputLevelFile (char *filename) {
+  // create the level file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_LEVEL_SUFFIX, "wb", MAGIC_INVF_LEVELS,
+			 MG_MESSAGE)))
+    return;
+
+  // write out the information
+  ivfLevel.Write (f);
+  
+  // close the file
+  fclose (f);
+}
+
+static void OutputTransFile (char *filename) {
+  // save the word number (in lastFragNum :-/)
+  int i = 0;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    (*wordHere).second.lastFragNum = i;
+    ++i; ++wordHere;
+  }
+
+  // save the tag number (in lastFragNum :-/)
+  i = 0;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    (*tagHere).second.lastFragNum = i;
+    ++i; ++tagHere;
+  }
+  
+  // create the translation file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))
+    return;
+
+  stdio_bitio_buffer buffer(f);
+  buffer.encodeStart();
+
+  // write out the word translation table
+  unsigned long wordDictSize = ivfWordInfoOccurOrder.size();
+  IvfWordInfoItArray::iterator wordItHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordItEnd = ivfWordInfoOccurOrder.end();
+  unsigned long oN = 0;
+  while (wordItHere != wordItEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordItHere)).second;
+    buffer.binary_encode (ivfWordInfo.lastFragNum + 1, wordDictSize + 1, NULL);
+    ++oN;
+    ++wordItHere;
+  }
+
+  // write out the tag translation table
+  unsigned long tagDictSize = ivfTagInfoOccurOrder.size();
+  IvfTagInfoItArray::iterator tagItHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagItEnd = ivfTagInfoOccurOrder.end();
+  while (tagItHere != tagItEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagItHere)).second;
+    buffer.binary_encode (ivfTagInfo.lastFragNum + 1, tagDictSize + 1, NULL);
+    ++oN;
+    ++tagItHere;
+  }
+
+  // finish encoding and close the file
+  buffer.encodeDone();
+  fclose (f);
+}
+
+#ifndef SILENT
+static void PrintStats () {
+  Message ("Inverted buffer size:          %8u bytes\n", invf_buffer_size);
+  Message ("Max memory needed for 1 chunk: %8u bytes\n", maxMemNeeded);
+
+  Message ("Number of chunks written:      %8u\n", chunksWritten);
+  Message ("Number of documents:           %8u\n", numDocs);
+  Message ("Number of fragments:           %8u\n", numFrags);
+  Message ("Number of words:               %8u\n", numWords);
+
+  Message ("Size of word dictionary:       %8u\n", ivfWordInfo.size());
+  Message ("Size of tag dictionary:        %8u\n", ivfTagInfo.size());
+}
+#endif
+
+int done_ivf_1 (const TagInfo &tagInfo, char * filename) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+
+  // output the last chunk
+  if (numChunkDocs > 0) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    OutputChunkInfo (mem, wordLevelIndex);
+    if (mem > maxMemNeeded) maxMemNeeded = mem;
+  }
+  
+  // write out and encoded 1 to say there are no more chunks
+  icb.gamma_encode (1, NULL);
+  icb.encodeDone ();
+
+  // write out the maximum memory required and close the file
+  fseek (ic, sizeof (long), 0);
+  WriteUL (ic, maxMemNeeded);
+  fclose (ic);
+
+  // output the inverted dictionary
+  OutputInvfDict (filename);
+
+  // write out the translation file
+  OutputTransFile (filename);
+
+  // output the level information
+  OutputLevelFile (filename);
+  
+  // output statistics
+#ifndef SILENT
+  PrintStats ();
+#endif
+  
+  msg_prefix = temp_str;
+
+  return COMPALLOK;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/ivf.pass2.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/ivf.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/ivf.pass2.cpp	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * ivf.pass2.cpp -- Memory efficient pass 2 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include <stdio.h>
+
+#if defined __WIN32__
+# include <process.h>
+# include <io.h>
+# define getpid _getpid
+# define unlink _unlink
+#else
+# include <unistd.h>
+# include "non_ansi.h"
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_random.h"
+#include "bitio_m_stdio.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+#include "perf_hash.h"
+#include "string.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+#ifdef USE_LONG_LONG
+#define SEEK_X seek_LL
+#define TELL_X tell_LL
+#else
+#define SEEK_X seek
+#define TELL_X tell
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+#endif
+
+
+static unsigned long numDocs = 0;
+static unsigned long numChunkDocs = 0;
+static unsigned long numDocsInChunk = 0;
+
+static unsigned long numFrags = 0;
+static unsigned long numFragsInChunk = 0;
+static unsigned long chunkStartFragNum = 0;
+
+
+
+
+struct BitPtr {
+  unsigned long start;
+  unsigned long here;
+  unsigned long lastFragNum;
+  unsigned long lgB;
+  
+  void Clear () { start = here = lastFragNum = lgB = 0; }
+  BitPtr () { Clear(); }
+};
+
+class WordBitPtrs {
+protected:
+  unsigned long numWords;
+  unsigned long numTags;
+  unsigned long size;
+  BitPtr *wordBitPtrs;
+
+  void CheckBufOverrun (unsigned long num) {
+    if (wordBitPtrs[num].here > wordBitPtrs[num+1].start) {
+      cerr << "numDocs: " << numDocs << "\n";
+      cerr << "numChunkDocs: " << numChunkDocs << "\n";
+      cerr << "numDocsInChunk: " << numDocsInChunk << "\n";
+      cerr << "numFrags: " << numFrags << "\n";
+      cerr << "numFragsInChunk: " << numFragsInChunk << "\n";
+      cerr << "chunkStartFragNum: " << chunkStartFragNum << "\n";
+      cerr << "num: " << num << "\n";
+      cerr << "[num].start: " << wordBitPtrs[num].start << "\n";
+      cerr << "[num].here: " << wordBitPtrs[num].here << "\n";
+      cerr << "[num+1].start: " << wordBitPtrs[num+1].start << "\n";
+      FatalError (1, "Bit buffer overrun");
+    }
+  }
+  
+public:
+  void Clear ();
+  WordBitPtrs () { wordBitPtrs = NULL; Clear(); }
+  ~WordBitPtrs ();
+  void SetSize (unsigned long _numWords,
+		unsigned long _numTags);
+
+  void ResetPtrs () {
+    if (wordBitPtrs == NULL) return;
+    unsigned long i;
+    for (i=0; i<size; ++i) wordBitPtrs[i].Clear();
+  }
+  
+  BitPtr &GetWordBitPtr (unsigned long wordNum)
+    { return wordBitPtrs[wordNum]; }
+  unsigned long &GetWordStart (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].start; }
+  unsigned long &GetWordHere (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].here; }
+  void CheckWordBufOverrun (unsigned long wordNum)
+    { CheckBufOverrun (wordNum); }
+
+  BitPtr &GetTagBitPtr (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords]; }
+  unsigned long &GetTagStart (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].start; }
+  unsigned long &GetTagHere (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].here; }
+  void CheckTagBufOverrun (unsigned long tagNum)
+    { CheckBufOverrun (tagNum + numWords); }
+
+  BitPtr &GetEndBitPtr ()
+    { return wordBitPtrs[size-1]; }
+  unsigned long &GetEndStart ()
+    { return wordBitPtrs[size-1].start; }
+  unsigned long &GetEndHere ()
+    { return wordBitPtrs[size-1].here; }
+};
+
+
+struct IP2TagInfo {
+  bool inTag;
+  unsigned long startFrag;
+  unsigned long tagNum;
+
+  IP2TagInfo () {
+    inTag = false;
+    startFrag = 0;
+    tagNum = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, IP2TagInfo, DictLTUCArray> TagMapDict;
+
+
+// class to handle the translation of occurrence order
+// to dictionary order for words and tags
+class OccurToDictConverter {
+protected:
+  unsigned long pos;
+  unsigned long val;
+  FILE *transFile;
+  random_bitio_buffer rbs;
+
+  unsigned long wordDictSize;
+  unsigned long tagDictSize;
+
+  void SeekStart ();
+  unsigned long TranslateNum (unsigned long num);
+  
+public:
+  OccurToDictConverter ();
+  ~OccurToDictConverter ();
+
+  void Open (char *filename, unsigned long _wordDictSize,
+	     unsigned long _tagDictSize);
+
+  // Close frees all allocated memory
+  void Close ();
+
+  unsigned long TranslateWord (unsigned long occurNum)
+    { return TranslateNum (occurNum); }
+  unsigned long TranslateTag (unsigned long occurNum)
+    { return TranslateNum (occurNum+wordDictSize); }
+};
+
+
+struct InvfStateRec {
+  mg_ullong start;
+  mg_ullong here;
+  unsigned long lastFragNum;
+  unsigned long B;
+
+  void Clear () {
+    start = here = 0;
+    lastFragNum = B = 0;
+  }
+  InvfStateRec () { Clear (); }
+};
+
+
+#define ISR_SIZE 1024
+
+class InvfStateCache {
+protected:
+  InvfStateRec recCache [ISR_SIZE];
+  unsigned long startNum;
+
+  FILE *stateFile;
+
+  void ClearCache () {
+    unsigned int i = 0;
+    for (i=0; i<ISR_SIZE; ++i) recCache[i].Clear();
+  }
+  
+public:
+  InvfStateCache ();
+  ~InvfStateCache ();
+  
+  void Open (char *filename);
+  void Close ();
+
+  // previous references to state records may be
+  // invalidated calling GetRec
+  InvfStateRec &GetRec (unsigned long num);
+};
+
+
+static invf_dict_header idh;
+static WordBitPtrs bitPtrs;
+
+static FILE *chunkFile = NULL;
+static stdio_bitio_buffer chunkBuf;
+
+static unsigned long ivfMemBufSize = 0;
+static char *ivfMemBuf = NULL;
+
+// word and tag dictionaries. a map is used for the tag dictionary
+// as it should never be very big (and the perfect hash function
+// sometimes has trouble with small values).
+static perf_hash_data *wordHashDict = NULL;
+static TagMapDict tagMapDict;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+static OccurToDictConverter occurConvert;
+
+// information about the state of the inverted file
+static InvfStateCache invfState;
+
+static char collectFilename[512];
+
+
+void WordBitPtrs::Clear () {
+  numWords = 0;
+  numTags = 0;
+  size=0;
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+  wordBitPtrs = NULL;
+}
+
+WordBitPtrs::~WordBitPtrs () {
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+}
+
+void WordBitPtrs::SetSize (unsigned long _numWords,
+			   unsigned long _numTags){
+  Clear();
+  numWords = _numWords;
+  numTags = _numTags;
+  size = numWords + numTags + 1;
+  wordBitPtrs = new BitPtr [size];
+}
+
+
+void OccurToDictConverter::SeekStart () {
+  if (transFile == NULL) return;
+  rbs.SEEK_X (sizeof (unsigned long) * 8);
+  pos = 0;
+}
+
+unsigned long OccurToDictConverter::TranslateNum (unsigned long num) {
+  if (num < pos) SeekStart ();
+  while (pos <= num) {
+    if (pos < wordDictSize)
+      val = rbs.binary_decode (wordDictSize + 1, NULL) - 1;
+    else 
+      val = rbs.binary_decode (tagDictSize + 1, NULL) - 1;
+    ++pos;
+  }
+  return val;
+}
+
+OccurToDictConverter::OccurToDictConverter () {
+  pos = 0;
+  val = 0;
+  transFile = NULL;
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+OccurToDictConverter::~OccurToDictConverter () {
+  if (transFile != NULL) Close ();
+}
+
+void OccurToDictConverter::Open (char *filename, unsigned long _wordDictSize,
+				 unsigned long _tagDictSize) {
+  if (transFile != NULL) Close ();
+
+  wordDictSize = _wordDictSize;
+  tagDictSize = _tagDictSize;
+
+  transFile = open_file (filename, INVF_CHUNK_TRANS_SUFFIX, "rb",
+			 MAGIC_CHUNK_TRANS, MG_ABORT);
+  rbs.attachFile (transFile, RND_BUF_SIZE);
+  SeekStart ();
+  val = 0;
+}
+
+void OccurToDictConverter::Close () {
+  if (transFile == NULL) return;
+
+  rbs.done ();
+  fclose (transFile);
+  transFile = NULL;
+  pos = 0;
+  val = 0;
+
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+
+
+
+InvfStateCache::InvfStateCache () {
+  startNum = 0;
+  stateFile = NULL;
+}
+
+InvfStateCache::~InvfStateCache () {
+  if (stateFile != NULL) Close ();
+}
+  
+void InvfStateCache::Open (char *filename) {
+  if (stateFile != NULL) Close();
+
+  // open the state file
+  char path[512];
+  sprintf (path, FILE_NAME_FORMAT ".%ld", get_basepath (), filename,
+           ".invf.state", (long) getpid ());
+  if (!(stateFile = fopen (path, "wb+"))) {
+    Message ("Unable to create \"%s\"", path);
+    exit (1);
+  }
+  unlink (path); // file will be deleted after it is closed
+  // reset the buffer
+  startNum = 0;
+  ClearCache();
+}
+
+void InvfStateCache::Close () {
+  if (stateFile == NULL) return;
+  fclose (stateFile);
+  stateFile = NULL;
+  startNum = 0;
+}
+
+InvfStateRec &InvfStateCache::GetRec (unsigned long num) {
+  // see if cached
+  if ((num >= startNum) && (num < startNum + ISR_SIZE))
+    return recCache[num-startNum];
+
+  // not cached, write out this lot of records and read in
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fwrite ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  // read in the new set of records
+  ClearCache ();
+  startNum = num - (num % ISR_SIZE);
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fread ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  return recCache[num-startNum];
+}
+
+
+
+static void ClearCharBuf (char *buf, unsigned long size) {
+  char *end = buf + size;
+  while (buf != end) *buf++ = 0;
+}
+
+static void ReadWordDict (char *filename) {
+  // read in the perfect hash function for the word dictionary
+  FILE *wordHashFile = open_file (filename, INVF_DICT_HASH_SUFFIX, "rb",
+				  MAGIC_HASH, MG_ABORT);
+  if (!(wordHashDict = read_perf_hash_data (wordHashFile))) {
+    FatalError (1, "Unable to read in hash data for word dictionary");
+  }
+  fclose (wordHashFile);
+}
+
+static void ReadTagDict (char *filename, invf_dict_header &_idh) {
+  // open the file
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+  
+  unsigned long tagNum;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < _idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    tagMapDict[thisEl.el].tagNum = tagNum;
+  }
+  
+  fclose (dictFile);
+}
+
+static void ReadLevelFile (char *filename) {
+  FILE *f;
+  f = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+		 MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Read (f);
+  fclose (f);
+}
+
+void CheckIntOverflow (mg_ullong totalIBits, mg_ullong lastTotalIBits) {
+  if (totalIBits < lastTotalIBits) {
+    fprintf(stderr, "ERROR: The totalIBits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+    if (sizeof (mg_ullong) < 8) {
+      fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+    }
+    fprintf(stderr, "       Build aborted.\n");
+    exit(1);
+  }
+}
+
+// assumes the inverted file state file has been opened
+static void InitInvfState (char *filename,
+			   invf_dict_header &_idh,
+			   InvfStateCache &_invfState,
+			   mg_ullong &totalIBits,
+			   bool wordLevelIndex) {
+  // read in the dictionary, setting inverted state information
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the word dictionary
+  fseek (dictFile, _idh.word_dict_start, SEEK_SET);
+
+  // add the word entries
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (_idh.num_levels);
+  unsigned long dictWordNum, p;
+  mg_ullong lastTotalIBits;
+  unsigned long N = _idh.num_frags;
+  for (dictWordNum=0; dictWordNum<_idh.word_dict_size; ++dictWordNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next word and associated information
+    wordEl.Read (dictFile, _idh.num_levels);
+
+    // update the state record
+    p = wordEl.frag_occur;
+    InvfStateRec &wisr = _invfState.GetRec (dictWordNum);
+    wisr.start = totalIBits;
+    wisr.here = totalIBits;
+    wisr.B = BIO_Bblock_Init (N, p);
+    
+    // add the length of the fragment numbers
+    totalIBits += BIO_Bblock_Bound_b (N, p, wisr.B);
+
+    // if needed, add the length of the fragment frequency information
+    if (!wordLevelIndex)
+      totalIBits += BIO_Gamma_Bound (wordEl.freq, wordEl.frag_occur);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+
+  // add the tag entries
+  dict_el tagEl;
+  unsigned long dictTagNum;
+  N = _idh.num_frags;
+  for (dictTagNum=0; dictTagNum<_idh.tag_dict_size; ++dictTagNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next tag and associated information
+    tagEl.Read (dictFile);
+
+    // update the state record
+    p = tagEl.frag_occur*2;
+    InvfStateRec &tisr = _invfState.GetRec (dictTagNum + _idh.word_dict_size);
+    tisr.start = totalIBits;
+    tisr.here = totalIBits;
+    tisr.B = BIO_Bblock_Init (N+p, p);
+    
+    // add the length of the fragment numbers (two numbers for each
+    // tag, one for start and one for end)
+    totalIBits += BIO_Bblock_Bound_b (N+p, p, tisr.B);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  fclose (dictFile);
+}
+
+/*
+// assumes the chunk tag information has been placed in .first
+static void PrintChunkInfo (unsigned long chunkMem,
+			    unsigned long numChunkWords,
+			    unsigned long numChunkTags) {
+  static unsigned long chunksRead = 0;
+  ++chunksRead;
+  cout << "Chunk Number: " << chunksRead << "\n";
+  cout << "numChunkDocs " << numDocsInChunk << "\n";
+  cout << "numChunkFrags " << numFragsInChunk << "\n";
+  cout << "mem " << chunkMem << "\n";
+  cout << "numWords " << numChunkWords << "\n";
+  cout << "numTags " << numChunkTags << "\n\n";
+
+  TagMapDict::iterator tagMapHere = tagMapDict.begin();
+  TagMapDict::iterator tagMapEnd = tagMapDict.end();
+  while (tagMapHere != tagMapEnd) {
+    unsigned long tagMapNum = (*tagMapHere).second.tagNum;
+    cout << (*tagMapHere).first << " " << tagMapNum << " "
+	 << bitPtrs.GetTagBitPtr(tagMapNum).here << "\n";
+    ++tagMapHere;
+  }
+}
+*/
+
+void ReadChunk (invf_dict_header &_idh, bool wordLevelIndex) {
+  // reset globals
+  numChunkDocs = 0;
+  chunkStartFragNum = numFrags;
+
+  // read in information about this chunk
+  numDocsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  if (numDocsInChunk == 0)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  numFragsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long chunkMem = chunkBuf.gamma_decode (NULL) - 1;
+
+  if (chunkMem > ivfMemBufSize)
+    FatalError (1, "Chunk memory size is greater than maximum");
+  
+  unsigned long numChunkWords = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long numChunkTags = chunkBuf.gamma_decode (NULL) - 1;
+
+  
+  // reset stuff
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+  bitPtrs.ResetPtrs();  
+
+  // read in the entries in occurrence order storing the
+  // "chunkWordCount" in "start" and the "chunkFragCount"
+  // in "here"
+  unsigned long numOccur;
+  unsigned long wordNum;
+  for (numOccur=0; numOccur<numChunkWords; ++numOccur) {
+    wordNum = occurConvert.TranslateWord (numOccur);
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    wordPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    if (wordPtr.start >= 2)
+      wordPtr.here = chunkBuf.gamma_decode (NULL);
+    else wordPtr.here = wordPtr.start;
+  }
+  unsigned long tagNum;
+  for (numOccur=0; numOccur<numChunkTags; ++numOccur) {
+    tagNum = occurConvert.TranslateTag (numOccur);
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    // only chunkFragCount is encoded for tags
+    tagPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    tagPtr.here = tagPtr.start;
+  }
+
+  /*  PrintChunkInfo (chunkMem, numChunkWords, numChunkTags);*/
+  
+  // create the bit ptrs in dictionary order
+  unsigned long totalIBits = 0; // only dealing with memory
+  unsigned long chunkWordCount, chunkFragCount;
+  for (wordNum=0; wordNum<_idh.word_dict_size; ++wordNum) {
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    chunkWordCount = wordPtr.start;
+    chunkFragCount = wordPtr.here;
+    wordPtr.start = totalIBits;
+    wordPtr.here = totalIBits;
+    wordPtr.lastFragNum = chunkStartFragNum;
+    wordPtr.lgB = 0;
+    if (chunkWordCount > 0) {
+      wordPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk,
+						   chunkFragCount));
+      totalIBits += BIO_Bblock_Bound (numFragsInChunk, chunkFragCount);
+      // use unary encoding for memory buffer encoding of fragment freq
+      if (!wordLevelIndex) {
+	totalIBits += chunkWordCount;
+      }
+    }
+  }
+  for (tagNum=0; tagNum<_idh.tag_dict_size; ++tagNum) {
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    chunkFragCount = tagPtr.here;
+    tagPtr.start = totalIBits;
+    tagPtr.here = totalIBits;
+    tagPtr.lastFragNum = chunkStartFragNum;
+    tagPtr.lgB = 0;
+    if (chunkFragCount > 0) {
+      unsigned long pTag = chunkFragCount*2;
+      tagPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk+pTag,
+						  pTag));
+      unsigned long bLen = BIO_Bblock_Bound (numFragsInChunk+pTag,
+					     pTag);
+//        cout << tagNum + _idh.word_dict_size << " ";
+//        cout << "numFrags: " << numFragsInChunk
+//    	   << " chunkFragCount: " << chunkFragCount
+//    	   << " B: " << 1 << tagPtr.lgB
+//    	   << " blen: " << blen << "\n";
+      totalIBits += bLen;
+    }
+  }
+  bitPtrs.GetEndStart() = totalIBits;
+  bitPtrs.GetEndHere() = totalIBits;
+
+  if ((totalIBits + 7ul) >> 3ul > chunkMem) {
+    cerr << "totalIBits: " << totalIBits << "\n";
+    cerr << "bytes: " << ((totalIBits + 7ul) >> 3ul) << "\n";
+    cerr << "chunkMem: " << chunkMem << "\n";
+    FatalError (1, "Pointers exceed buffer size");
+  }
+}
+
+
+
+
+int init_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // read in compressed dictionary header
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  idh.Read (dictFile);
+  fclose (dictFile);
+
+  // set the size of the bit ptrs
+  bitPtrs.SetSize (idh.word_dict_size, idh.tag_dict_size);
+
+  // open the chunk file and read in the maximum memory needed
+  // for the inverted memory buffer
+  chunkFile = open_file (filename, INVF_CHUNK_SUFFIX, "rb",
+			 MAGIC_CHUNK, MG_ABORT);
+  ReadUL (chunkFile, ivfMemBufSize);
+  chunkBuf.attachFile (chunkFile);
+
+  // allocate memory for the inverted buffer
+  ivfMemBuf = new char [ivfMemBufSize];
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+
+  // read in the word dictionary
+  ReadWordDict (filename);
+
+  // read in the tag dictionary
+  ReadTagDict (filename, idh);
+
+  // read in the level information
+  ReadLevelFile (filename);
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  
+  // set up the translation table file
+  occurConvert.Open (filename, idh.word_dict_size, idh.tag_dict_size);
+
+  // reset some globals
+  numDocs = 0;
+  numChunkDocs = 0;
+  numDocsInChunk = 0;
+  numFrags = 0;
+  numFragsInChunk = 0;
+  chunkStartFragNum = 0;
+
+  strcpy (collectFilename, filename);
+  
+  
+  // create the inverted file
+  mg_ullong totalIBits = 0;
+  FILE *invfFile = create_file (filename, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_ABORT);
+  totalIBits += sizeof (unsigned long) * 8; // magic number
+  totalIBits += 8 * 200;                    // 200 byte gap -- why??????
+  fclose (invfFile);
+
+  // init the inverted file state cache
+  invfState.Open (filename);
+  InitInvfState (filename, idh, invfState, totalIBits, wordLevelIndex);
+
+  return COMPALLOK;
+}
+
+static void CloseTextTag (IP2TagInfo &tInfo, const UCArray &/*tagName*/) {
+  if (!tInfo.inTag) return;
+
+    // add this tag to the inverted list
+  BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tInfo.tagNum);
+  unsigned long endFrag = numFrags;
+  int b = 1 << tagBitPtr.lgB;
+
+  /*
+  cout << (tInfo.tagNum+idh.word_dict_size) << " \"<" << tagName << ">\" "
+       << tInfo.startFrag << " " << endFrag << "\n";
+  */
+    
+  mems_bitio_buffer buffer ((u_char *) ivfMemBuf, tagBitPtr.here);
+  buffer.bblock_encode (tInfo.startFrag - tagBitPtr.lastFragNum + 1,
+			b, NULL);
+  buffer.bblock_encode (endFrag - tInfo.startFrag + 1, b, NULL);
+  tagBitPtr.lastFragNum = endFrag;
+  tagBitPtr.here = buffer.position();
+  buffer.encodeDone();
+  
+  // check for buffer overrun
+  bitPtrs.CheckTagBufOverrun (tInfo.tagNum);
+  
+  // reset information about this tag
+  tInfo.inTag = false;
+  tInfo.startFrag = 0;
+}
+
+static void ProcessOpenTag (const TextEl &el, bool &inFrag) {
+  // close tag if open
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  if (tInfo.inTag) CloseTextTag (tInfo, el.tagName);
+
+  // open this tag
+  tInfo.inTag = true;
+  tInfo.startFrag = numFrags;
+
+  // check for start of next fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    ++numFrags;
+    inFrag = true;
+  }
+}
+
+static void ProcessCloseTag (const TextEl &el, bool &inFrag) {
+  // check for end of fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    inFrag = false;
+  }
+
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  CloseTextTag (tInfo, el.tagName);
+}
+
+static void ProcessText (const TextEl &el, bool &inFrag) {
+  // make sure this text is to be indexed
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  unsigned char mgWord[MAXSTEMLEN + 1];
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexMGWord (textHere, textEnd, mgWord);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (mgWord[0] > 0) {
+      if (wordLevelIndex) ++numFrags;
+
+      unsigned long wordNum = perf_hash (wordHashDict, mgWord);
+
+      /*
+      cout << wordNum << " \"";
+      cout.write (mgWord+1, *mgWord);
+      cout << "\" " << numFrags << "\n";
+      */
+      
+      // add this word to the inverted list
+      BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+      unsigned long fragNum = numFrags;
+      int b = 1 << wordBitPtr.lgB;
+  
+      mems_bitio_buffer buffer ((u_char *) ivfMemBuf, wordBitPtr.here);
+
+      // note: this assumes that fragments don't carry over between
+      // chunks (which they don't because all tags are closed at the
+      // end of each document and chunks are based on document
+      // boundaries), i.e. the first fragment number must be greater
+      // than the starting fragment number of the chunk.
+      if (fragNum > wordBitPtr.lastFragNum) {
+	buffer.bblock_encode ((fragNum - wordBitPtr.lastFragNum - 1) + 1,
+			      b, NULL);
+	if (!wordLevelIndex) buffer.encodeBit (1); // freq = 1
+	
+      } else if (!wordLevelIndex) {
+	// add one to the frequency count for this word
+	buffer.seek (buffer.position()-1);
+	buffer.encodeBit (0); // unary encoding -- last = 1
+	buffer.encodeBit (1);
+      }
+      
+      wordBitPtr.lastFragNum = fragNum;
+      wordBitPtr.here = buffer.position();
+      buffer.encodeDone();
+      
+      // check for buffer overrun
+      bitPtrs.CheckWordBufOverrun (wordNum);
+    }
+  }
+}
+
+// combine the in memory inverted buffer with the disk
+// based inverted file
+static void DiskMerge (char *filename) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+
+      // make sure we have something to process
+  if (numChunkDocs <= 0) return;
+    
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb+",
+			      MAGIC_INVF, MG_ABORT);
+  random_bitio_buffer invfOutBuf (invfFile);
+
+  // set up to decode the entries in memory
+  mems_bitio_buffer memInBuf ((u_char *) ivfMemBuf, 0);
+
+  // write out the word information
+  unsigned long wordNum;
+  int b;
+  unsigned long currFragNum;
+  unsigned long delta;
+  unsigned long currFreq;
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &wordDiskState = invfState.GetRec (wordNum);
+    invfOutBuf.SEEK_X (wordDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+    memInBuf.seek(wordBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currFragNum = chunkStartFragNum;
+    while (memInBuf.position() < wordBitPtr.here) {
+      // decode word entry
+      b = 1 << wordBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL);
+      currFragNum += delta;
+      if (!wordLevelIndex) currFreq = memInBuf.unary_decode (NULL);
+      else currFreq = 1;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currFragNum-wordDiskState.lastFragNum,
+				wordDiskState.B, NULL);
+      if (!wordLevelIndex) invfOutBuf.gamma_encode (currFreq, NULL);
+      wordDiskState.lastFragNum = currFragNum;
+    }
+
+    wordDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  // write out the tag information
+  unsigned long tagNum;
+  unsigned long currTagStart;
+  unsigned long currTagEnd;
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &tagDiskState = invfState.GetRec (tagNum+idh.word_dict_size);
+    invfOutBuf.SEEK_X (tagDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tagNum);
+    memInBuf.seek(tagBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currTagEnd = chunkStartFragNum;
+    while (memInBuf.position() < tagBitPtr.here) {
+      // decode tag entry
+      b = 1 << tagBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagStart = currTagEnd + delta;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagEnd = currTagStart + delta;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currTagStart-tagDiskState.lastFragNum+1,
+				tagDiskState.B, NULL);
+      invfOutBuf.bblock_encode (currTagEnd-currTagStart+1,
+				tagDiskState.B, NULL);
+
+      tagDiskState.lastFragNum = currTagEnd;
+    }
+
+    tagDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  memInBuf.done();
+
+  invfOutBuf.encodeDone();
+  fclose (invfFile);
+}
+
+
+int process_ivf_2 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  // get next chunk information if need to. the chunk information
+  // is needed before the first document is processed
+  if (numChunkDocs >= numDocsInChunk) ReadChunk (idh, wordLevelIndex);
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // process this element
+    if ((*here).elType == OpenTagE) ProcessOpenTag (*here, inFrag);
+    else if ((*here).elType == CloseTagE) ProcessCloseTag (*here, inFrag);
+    else ProcessText (*here, inFrag);
+    
+    ++here;
+  }
+
+  // close off any unclosed tags
+  TagMapDict::iterator tdHere = tagMapDict.begin();
+  TagMapDict::iterator tdEnd = tagMapDict.end();
+  while (tdHere != tdEnd) {
+    CloseTextTag ((*tdHere).second, (*tdHere).first);
+    ++tdHere;
+  }
+  
+  // we've processed one more document
+  ++numDocs;
+  ++numChunkDocs;
+
+  // merge the memory based inverted file with the one on
+  // disk if this is the end of this chunk
+  if (numChunkDocs >= numDocsInChunk) DiskMerge (collectFilename);
+  
+  return COMPALLOK;
+}
+
+
+static void CondenseInvfFile (char *filename, unsigned long &bytesOutput) {
+  FILE *inInvfFile = open_file (filename, INVF_SUFFIX, "rb",
+				MAGIC_INVF, MG_ABORT);
+  FILE *outInvfFile = open_file (filename, INVF_SUFFIX, "rb+",
+				 MAGIC_INVF, MG_ABORT);
+
+  // skip the magic number
+  fseek (outInvfFile, sizeof (unsigned long), SEEK_SET);
+  
+  // write the inverted file header -- use defaults for most things
+  invf_file_header ifh;
+  ifh.no_of_words = idh.word_dict_size;
+  ifh.no_of_tags = idh.tag_dict_size;
+  ifh.word_level_index = (ivfLevel.indexLevel.empty()) ? 1 : 0;
+  ifh.Write (outInvfFile);
+
+  bytesOutput = ftell (outInvfFile);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = ifh.no_of_words + ifh.no_of_tags;
+  unsigned long entryNum;
+  mg_ullong lastStart = 0;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // overrun check
+    if (stateRec.start < lastStart)
+      FatalError (1, "Inverted file Buffer overrun");
+    lastStart = stateRec.start;
+    
+    unsigned long oldEntryStart = stateRec.start >> 3;
+    unsigned long oldEntryStartOver = stateRec.start & 7; // should be 0
+    unsigned long oldEntryEnd = (stateRec.here + 7) >> 3;  // byte after end
+    unsigned long oldEntryEndOver = stateRec.here & 7;
+
+    fseek (inInvfFile, oldEntryStart, SEEK_SET);
+
+    stateRec.here -= stateRec.start;
+    stateRec.start = (mg_ullong)bytesOutput * 8 + oldEntryStartOver;
+    stateRec.here += stateRec.start;
+    while (oldEntryStart < oldEntryEnd) {
+      unsigned char c = getc (inInvfFile);
+      if (oldEntryStart == oldEntryEnd - 1) {
+	u_char ands[8] =
+	{0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	c &= ands[oldEntryEndOver];
+      }
+      putc (c, outInvfFile);
+      ++bytesOutput;
+      ++oldEntryStart;
+    }
+  }  
+
+  fclose (inInvfFile);
+
+#ifdef __WIN32__
+  if (_chsize (_fileno (outInvfFile), bytesOutput) != 0)
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (outInvfFile), bytesOutput);
+#endif
+
+  fclose (outInvfFile);
+}
+
+static void OutputInvfIdx (char *filename, unsigned long invfNumBytes) {
+  FILE *invfIdxFile = create_file (filename, INVF_IDX_SUFFIX, "wb",
+				   MAGIC_INVI, MG_ABORT);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = idh.word_dict_size + idh.tag_dict_size;
+  unsigned long entryNum;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // assumes that inverted entries start at beginning of each byte
+    if (!WriteUL (invfIdxFile, (stateRec.start >> 3))) break;
+  }
+
+  WriteUL (invfIdxFile, invfNumBytes);
+
+  fclose (invfIdxFile);
+}
+
+
+int done_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // close most open files
+  if (chunkFile != NULL) {
+    chunkBuf.done();
+    fclose (chunkFile);
+    chunkFile = NULL;
+  }
+  occurConvert.Close();
+  
+  // free allocated memory
+  bitPtrs.Clear();
+  if (ivfMemBuf != NULL) { delete [] ivfMemBuf; ivfMemBuf = NULL; }
+  free_perf_hash (wordHashDict);
+  wordHashDict = NULL;
+  tagMapDict.erase (tagMapDict.begin(), tagMapDict.end());
+
+  // condense the inverted file and truncate it
+  // this function also writes out the inverted header
+  unsigned long invfNumBytes = 0;
+  CondenseInvfFile (filename, invfNumBytes);
+
+  OutputInvfIdx (filename, invfNumBytes);
+
+  // close the rest of the open files
+  invfState.Close ();
+  
+  return COMPALLOK;
+}
Index: /main/tags/3.01/indexers/mgpp/text/locallib.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/locallib.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/locallib.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+/**************************************************************************
+ *
+ * locallib.cpp -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; ++i)
+    total += A[i];
+  for (i = 0; i < n; ++i)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; --i)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      --n;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  ++N;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	++limit;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * /*mem*/,
+	      u_long * disk) {
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; ++i)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  memset (cfh->huff_words_size, '\0', sizeof (cfh->huff_words_size));
+
+  if ((int)fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; ++i)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
Index: /main/tags/3.01/indexers/mgpp/text/locallib.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/locallib.h	(revision 21112)
@@ -0,0 +1,45 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/mg.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/mg.special.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg.special.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg.special.cpp	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * mg.special.cpp -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "mg.h"
+#include "build.h"
+
+int init_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
+
+int process_special (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  // print out the document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE) {
+      cout << "** OpenTag  \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n"; 
+    } else if ((*here).elType == CloseTagE) {
+      cout << "** CloseTag \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n";
+    } else {
+      cout << "** Text     \"" << (*here).text << "\"\n";
+    }
+    
+    ++here;
+  }
+  
+  return COMPALLOK;
+}
+
+int done_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mg_errors.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg_errors.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg_errors.cpp	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * mg_errors.cpp -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/indexers/mgpp/text/mg_errors.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/indexers/mgpp/text/mg_files.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg_files.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg_files.cpp	(revision 21112)
@@ -0,0 +1,259 @@
+/**************************************************************************
+ *
+ * mg_files.cpp -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      delete []basepath;
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = new char[strlen (bp) + 2];
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/indexers/mgpp/text/mg_files.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mg_files.h	(revision 21112)
@@ -0,0 +1,343 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXT_LEVELS	GEN_MAGIC('M','G','t','L')
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVF_LEVELS	GEN_MAGIC('M','G','I','L')
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The document levels in the text */
+#ifdef SHORT_SUFFIX
+# define TEXT_LEVEL_SUFFIX	 	".tl"
+#else
+# define TEXT_LEVEL_SUFFIX	 	".text.level"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The level information for the inverted text */
+#ifdef SHORT_SUFFIX
+# define INVF_LEVEL_SUFFIX              ".il"
+#else
+# define INVF_LEVEL_SUFFIX              ".invf.level"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.1	(revision 21112)
@@ -0,0 +1,188 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mgpp_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
@@ -0,0 +1,1021 @@
+/**************************************************************************
+ *
+ * mgpp_compression_dict.cpp -- Routines for creating compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "WordData.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct DictWordData : public WordData
+  {
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+DictWordData;
+
+static DictWordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    DictWordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HDY")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+#ifndef SILENT
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+#endif
+  return (0);
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; ++i)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      DictWordData *wd;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = (DictWordData **) Xmalloc (sizeof (DictWordData *) * Num[i] + 1);
+
+      buf = (char *) Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = (DictWordData *) Xmalloc (sizeof (DictWordData) * Num[i]);
+      unsigned int j;
+      for (j = 0; j < Num[i]; ++j, ++wd)
+	{
+	  int len;
+
+	  // read docCount and wordCount
+	  wd->read(f);
+
+	  // read an mgString
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  unsigned int i;
+  DictWordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (DictWordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; ++i, ++wd)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  unsigned int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; ++i)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; ++i)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->docCount > bb->docCount)
+    return -1;
+  else if (aa->docCount < bb->docCount)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->words() > bb->words())
+    return 1;
+  else if (aa->words() < bb->words())
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  DictWordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = (DictWordData **) Xmalloc (num * sizeof (DictWordData *));
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    wd[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      DictWordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->documents(), word->word, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; ++i)
+    {
+      DictWordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (DictWordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  memset (char_freqs, '\0', sizeof (char_freqs));
+  memset (len_freqs, '\0', sizeof (len_freqs));
+  for (i = 0; i < num; ++i, ++wd)
+    {
+      u_long freq = (*wd)->documents();
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; --len, ++buf)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+inline void CalcBitCostForWordData(DictWordData **word, int num,
+				   double freqs_trans[2], u_long escape[2],
+				   char * char_lens[2], char *len_lens[2], 
+				   double esc[2], int num_trans)
+{ 
+  int j;
+
+  for (j = 0; j < num; ++j, ++word)
+    {
+      float   cbc, wbc;
+      u_char *buf = (*word)->word;
+      int     len = *buf++;
+      u_long  freq = (*word)->documents();
+      int     idx = KIND (*word);
+
+      cbc = len_lens[idx][len];
+      for (; len; --len, ++buf)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*word)->saving = ((*word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*word)->word[0]);
+
+      (*word)->num_trans = num_trans;
+    }
+}
+
+
+
+void 
+CalcBitCost (DictWordData ** discard_word, int discard_num,
+	     DictWordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  CalcBitCostForWordData(discard_word, discard_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  CalcBitCostForWordData(keep_word, keep_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+inline void m3_transferWord(DictWordData **toHeap, int *toNum, heap_comp toSaving,
+			    DictWordData **fromHeap, int *fromNum, heap_comp fromSaving,
+			    int *num_trans, double freqs_trans[], int *mem, double *total,
+			    int *count)
+{
+  /* Transfer the word at the top of the keep heap to the
+     discard heap. */
+  DictWordData *word = fromHeap[0];
+  int idx = KIND (word);
+  heap_deletehead (fromHeap, sizeof (word), fromNum, fromSaving);
+  toHeap[*toNum] = word;
+  heap_additem (toHeap, sizeof (word), toNum, toSaving);
+  freqs_trans[idx] -= word->documents();
+  *mem = (*mem) - sizeof (u_char *) + word->word[0];
+  *num_trans += 1;
+  *total += word->saving;
+  *count += 1;
+#ifdef DEBUG
+  fprintf (stderr,
+	   "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+	   *mem, word->documents(), word->char_bit_cost,
+	   word->saving, word2str (word->word));
+#endif
+}
+
+inline void m3_storeWord(DictWordData **wordHeap, int wordNum, int num_trans, 
+			 double freqs_trans[], u_long escape[], heap_comp hc)
+{
+  DictWordData *word = wordHeap[0];
+  int idx = KIND (word);
+  float wbc;
+#ifdef DEBUG1
+  fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+	   word->saving);
+#endif
+  wbc = -log2 (word->documents() / (freqs_trans[idx] + escape[idx])) *
+    word->documents();
+  word->saving = (word->char_bit_cost - wbc) /
+    (sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+  fprintf (stderr, " %.2f\n", word->saving);
+#endif
+  word->num_trans = num_trans;
+  heap_changedhead (wordHeap, sizeof (word), wordNum, hc);
+}
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  DictWordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+  keep_num = 0;
+  keep_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      DictWordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->documents();
+      ++num_trans;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < (unsigned)num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(keep_heap, keep_num, num_trans, freqs_trans, escape, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(discard_heap, discard_num, num_trans, freqs_trans, escape, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > (unsigned)k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      m3_transferWord(discard_heap, &discard_num, BigSaving,
+			      keep_heap, &keep_num, SmallSaving,
+			      &num_trans, freqs_trans, &mem, &total, &keep_to_discard);
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      m3_transferWord(keep_heap, &keep_num, SmallSaving,
+			      discard_heap, &discard_num, BigSaving,
+			      &num_trans, freqs_trans, &mem, &total, &discard_to_keep);
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  ++recalcs;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; ++i)
+    {
+      DictWordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; ++i)
+    {
+      DictWordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+#ifndef SILENT
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+#endif
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  unsigned int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; ++i)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  unsigned int i, us;
+  for (us = i = 0; i < dd->num_wds; ++i)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = new long [dd->num_wds]))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      freqs[i] = dd->wd[i]->documents();
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  delete []freqs;
+  freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  memset (&huff_words_size, '\0', sizeof (huff_words_size));
+  memset (&lencounts, '\0', sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      ++lencounts[codelen];
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  delete []hd->clens;
+  delete hd;
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[256];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; --len, ++buf)
+	freqs[(u_long) (*buf)] += (*wd)->documents();
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; ++j)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[16];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    freqs[(*wd)->word[0]] += (*wd)->documents();
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; ++j)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  memset (&cdh, '\0', sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; ++i)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < discard[i].num_wds; ++j)
+		esc.docCount += discard[i].wd[j]->documents();
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < all[i].num_wds; ++j)
+		if (all[i].wd[j]->documents() == 1)
+		  ++esc.docCount;
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_decompress_text.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * mgpp_decompress_text.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "TextGet.h"
+#include "messages.h"
+#include "mg_files.h"
+
+int main (int argc, char **argv) {
+  int ch;
+  char *filename = "";
+  char *basePath = "";
+  UCArray level;
+  SetCStr (level, "Document", 8);
+
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:d:K:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      filename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (optarg);
+      break;
+    case 'K':
+      SetCStr (level, optarg, strlen(optarg));
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-K level] [-d directory] -f name\n",
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  // load up the text information
+  TextData td;
+  if (!td.LoadData (basePath, filename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", filename);
+  }
+  
+  // output each document in the level
+//    cout << td.levels << "\n";
+  
+  TextLevelInfo levelInfo = td.levels.levelInfo[level];
+  unsigned long docNum = 1;
+  UCArray docText;
+  while (docNum <= levelInfo.numEntries) {
+//      TextIdx docIdx;
+//      if (GetDocIdx (td, level, docNum, docIdx)) {
+//        cout << "doc: " << docNum << "\n";
+//        cout << docIdx;
+//      }
+    
+    if (!GetDocText (td, level, docNum, docText)) {
+      FatalError (1, "Error while trying to get document %u", docNum);
+    }
+
+    cout << docText << "\n";
+    
+    ++docNum;
+  }
+  
+  return 0;
+}
+
+
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,69 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mgpp_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgppquery (1)
+can load it in very quickly, making the startup time for
+.BR mgppquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
@@ -0,0 +1,707 @@
+/**************************************************************************
+ *
+ * mgpp_fast_comp_dict.cpp -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "huffman.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; ++i)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  memset (buffer, 0, mem);
+  fixup = (u_char *) Xmalloc (fixup_mem);
+  memset (fixup, 0, fixup_mem);
+
+#ifndef SILENT
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+#endif
+
+  load_comp_dict (filename);
+
+#ifndef SILENT
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+#endif
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; ++which)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID)
+      for (which = 0; which <= 1; ++which)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; ++j)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; ++i)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  return (0);
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * /*cdh*/, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; --val)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+static u_long mem_skip_hd(FILE *dict, u_long mem)
+{ huff_data hd;
+
+  mem += sizeof (hd);
+  Read_Huffman_Data (dict, &hd, NULL, NULL);
+  if (hd.clens)
+    delete []hd.clens;
+  mem += hd.num_codes * sizeof (unsigned long);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+  return mem;
+}
+
+static u_long mem_read_cfh(FILE *dict, compression_dict_header *cdh, comp_frags_header *cfh, 
+			   u_long mem)
+{
+  mem += sizeof (comp_frags_header);
+  Read_cfh (dict, cfh, NULL, NULL);
+
+  /* Don't need to count the space for the clens of the huffman data */
+
+  mem += mem_for_words (dict, cdh, cfh);
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+
+  return mem;
+}
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+
+	  mem = mem_skip_hd(dict, mem);
+	  mem = mem_skip_hd(dict, mem);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem = mem_skip_hd(dict, mem);
+	      mem = mem_skip_hd(dict, mem);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((u_long) cur > (u_long)buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = (auxiliary_dict *) getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      unsigned int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = (u_char *) getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = (u_char **) getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = (u_char **) getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = (u_char ***) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = (u_char *) getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = (unsigned long *) getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = (unsigned long **) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; ++i)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+static void load_read_huffman(FILE *dict, compression_dict *cd, int which)
+{
+  huff_data *hd;
+  u_long   **vals;
+
+  hd = (huff_data *) getmem (sizeof (huff_data), sizeof (char *));
+  cd->MemForCompDict += sizeof (huff_data);
+  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+  cd->chars_huff[which] = hd;
+  FIXUP (&cd->chars_huff[which]);
+  cd->chars_vals[which] = vals;
+  FIXUP (&cd->chars_vals[which]);
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+}
+
+static void load_read_words(FILE *dict, compression_dict *cd, int which, int fixescape)
+{
+  cd->cfh[which] = (comp_frags_header *) getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+  FIXUP (&cd->cfh[which]);
+  FIXUP (&cd->values[which]);
+  if (fixescape)
+    { FIXUP(&cd->escape[which]);
+    }
+  else
+    {
+      cd->escape[which] = NULL;
+    }
+}
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = (compression_dict *) getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; ++which)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  load_read_words(dict, cd, which, 0);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+
+	  load_read_huffman(dict, cd, which);
+	  load_read_huffman(dict, cd, which);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      load_read_huffman(dict, cd, which);
+	      load_read_huffman(dict, cd, which);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_invf_dict
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +13n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that can be used during querying.
+Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word. The dictionary is blocked with
+each block containing the same number of entries (not the same size).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to 
+specify the number of entries per block. The default is 16.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict.cpp -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename, unsigned long entriesPerBlock) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // create the blocked dictionary
+  FILE *blockDictFile = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "wb",
+				     MAGIC_STEM, MG_ABORT);
+  block_dict_header bdh;
+  bdh.lookback = idh.lookback;
+  bdh.word_dict_start = idh.word_dict_start;
+  bdh.word_dict_size = idh.word_dict_size;
+  bdh.tag_dict_start = idh.tag_dict_start;
+  bdh.tag_dict_size = idh.tag_dict_size;
+  bdh.num_docs = idh.num_docs;
+  bdh.num_frags = idh.num_frags;
+  bdh.num_words = idh.num_words;
+  bdh.total_bytes = idh.total_bytes;
+  bdh.index_string_bytes = idh.index_string_bytes;
+  bdh.num_levels = idh.num_levels;
+  bdh.Write (blockDictFile);
+
+  
+  // write out the word part of the dictionary
+
+  bdh.entries_per_wblk = entriesPerBlock;
+  bdh.max_wblk_size = 0;
+  bdh.wblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  block_idx wordIdx;
+  word_block_dict_el wordBlockEl;
+  wordBlockEl.SetNumLevels (idh.num_levels);
+  
+  
+  unsigned long wordNum;
+  unsigned long wordInvfPtr;
+  UCArray lastEl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word and inverted file pointer
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordInvfPtr);
+
+    // remember this word (and position) if this is the start
+    // of a new block
+    if (wordNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = wordEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (wordIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(wordIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_wblk_size) bdh.max_wblk_size = blockLen;
+      }
+      
+      wordIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this word
+    wordBlockEl.el = wordEl.el;
+    wordBlockEl.frag_occur = wordEl.frag_occur;
+    wordBlockEl.freq = wordEl.freq;
+    wordBlockEl.invf_ptr = wordInvfPtr;
+    unsigned long tempI;
+    for (tempI=0; tempI<idh.num_levels; ++tempI)
+      wordBlockEl.levelFreqs[tempI] = wordEl.levelFreqs[tempI];
+
+    // write out the word
+    wordBlockEl.Write (blockDictFile, &lastEl, idh.num_levels);
+
+    lastEl = wordBlockEl.el;
+  }
+
+
+  // write out the tag part of the dictionary
+
+  bdh.entries_per_tblk = entriesPerBlock;
+  bdh.max_tblk_size = 0;
+  bdh.tblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+
+  block_idx tagIdx;
+  block_dict_el tagBlockEl;
+  
+  unsigned long tagNum;
+  unsigned long tagInvfPtr;
+  dict_el tagEl;
+  lastEl.erase (lastEl.begin(), lastEl.end());
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // read in the next tag and inverted file pointer
+    tagEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagInvfPtr);
+
+    // remember this tag (and position) if this is the start
+    // of a new block
+    if (tagNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = tagEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (tagIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(tagIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_tblk_size) bdh.max_tblk_size = blockLen;
+      }
+      
+      tagIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this tag
+    tagBlockEl.el = tagEl.el;
+    tagBlockEl.frag_occur = tagEl.frag_occur;
+    tagBlockEl.freq = tagEl.freq;
+    tagBlockEl.invf_ptr = tagInvfPtr;
+
+    // write out the tag
+    tagBlockEl.Write (blockDictFile, &lastEl);
+
+    lastEl = tagBlockEl.el;
+  }
+
+
+  // write out the element indexes
+  bdh.num_wblks = wordIdx.size();
+  bdh.wblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, wordIdx);
+  
+  bdh.num_tblks = tagIdx.size();
+  bdh.tblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, tagIdx);
+
+  // write out the blocked dictionary header
+  fseek (blockDictFile, sizeof(unsigned long), SEEK_SET);
+  bdh.Write (blockDictFile);
+
+
+  // close open files
+  fclose (blockDictFile);
+  fclose (invfIdxFile);
+  fclose (dictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Max word block size = %d\n", bdh.max_wblk_size);
+  Message ("Max tag block size = %d\n", bdh.max_tblk_size);
+  Message ("Number of word blocks written = %d\n", bdh.num_wblks);
+  Message ("Number of tag blocks written = %d\n", bdh.num_tblks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file] "
+	       "[-d data directory] [-b entries-per-block] "
+	       "[-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, entriesPerBlock);
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict_dump.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
@@ -0,0 +1,141 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict_dump.cpp -- Program to printthe various inverted dictionaries
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // print the information from the inverted dictionary file header
+  if (printHeader) {
+    cout << "lookback: " << idh.lookback << "\n";
+    cout << "word_dict_start: " << idh.word_dict_start << "\n";
+    cout << "word_dict_size: " << idh.word_dict_size << "\n";
+    cout << "tag_dict_start: " << idh.tag_dict_start << "\n";
+    cout << "tag_dict_size: " << idh.tag_dict_size << "\n";
+    cout << "num_docs: " << idh.num_docs << "\n";
+    cout << "num_frags: " << idh.num_frags << "\n";
+    cout << "num_words: " << idh.num_words << "\n";
+    cout << "total_bytes: " << idh.total_bytes << "\n";
+    cout << "index_string_bytes: " << idh.index_string_bytes << "\n";
+    cout << "num_levels: " << idh.num_levels << "\n";
+
+    cout << "\n";
+  }
+
+
+  if (printWords) {
+    fseek (dictFile, idh.word_dict_start, SEEK_SET);
+    
+    unsigned long wordNum;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      cout << "\"" << wordEl.el << "\"\n";// (" << wordNum << ")\n";
+    }
+    
+    cout << "\n";
+  }
+
+  // write out the tag part of the dictionary
+  if (printTags) {
+    fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+    
+    unsigned long tagNum;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      // read in the next tag and inverted file pointer
+      tagEl.Read (dictFile);
+      
+      cout << "\"" << tagEl.el << "\" (" << tagNum << ")\n";
+    }
+  }
+  
+  // close open files
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+
+  while ((ch = getopt (argc, argv, "f:d:rwth")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-f input_file] "
+	       "[-d data directory]\n(-rwt:print header, words, tags)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, printHeader, printWords, printTags);
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dump.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
@@ -0,0 +1,244 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dump.cpp -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+#include "WordData.h"
+
+static void PrintInvfWord (FILE *invfFile,
+			   invf_dict_header &idh,
+			   invf_file_header &ifh,
+			   word_dict_el &wordEl,
+			   unsigned long wordStart,
+			   bool printFrags) {
+  cout << wordEl.frag_occur << " \"" << wordEl.el << "\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL);
+      fragNum += delta;
+      cout << " " << fragNum;
+      
+      if (!ifh.word_level_index ) {
+	unsigned long count = buffer.gamma_decode (NULL);
+	cout << "(" << count << ")";
+      } else {
+	cout << "(1)";
+      }
+    }
+    
+    cout << "\n";
+  
+    buffer.done();
+  }
+}
+
+static void PrintInvfTag (FILE *invfFile,
+			  invf_dict_header &idh,
+			  invf_file_header &/*ifh*/,
+			  dict_el &tagEl,
+			  unsigned long tagStart,
+			  bool printFrags) {
+  cout << tagEl.frag_occur << " \"<" << tagEl.el << ">\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, tagStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long pTag = tagEl.frag_occur*2;
+    unsigned long B = BIO_Bblock_Init (idh.num_frags+pTag, pTag);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<tagEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << " " << fragNum;
+      cout << "-";
+      delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << fragNum;
+    }
+    
+    cout << "\n";
+    
+    buffer.done();
+  }
+}
+
+static void PrintHeaderInfo (invf_dict_header &idh,
+			     invf_file_header &ifh) {
+  cerr << "Lookback:         " << idh.lookback << "\n";
+  cerr << "Word Dict Size:   " << idh.word_dict_size << "\n";
+  cerr << "Tag Dict Size:    " << idh.tag_dict_size << "\n";
+  cerr << "Num Documents:    " << idh.num_docs << "\n";
+  cerr << "Num Fragments:    " << idh.num_frags << "\n";
+  cerr << "Num Words:        " << idh.num_words << "\n";
+
+  cerr << "Skip Mode:        " << ifh.skip_mode << "\n";
+  cerr << "Word Level Index: " << ifh.word_level_index << "\n";
+
+  cerr << "\n";
+}
+
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags,
+			   bool printFrags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  // print out header information
+  if (printHeader) {
+    PrintHeaderInfo (idh, ifh);
+  }
+
+  // open the inverted index
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  // process all the words
+  if (printWords) {
+    unsigned long wordNum;
+    unsigned long wordStart;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      ReadUL (invfIdxFile, wordStart);
+      PrintInvfWord (invfFile, idh, ifh, wordEl, wordStart, printFrags);
+    }
+  }
+
+  // process all the tags
+  if (printTags) {
+    unsigned long tagNum;
+    unsigned long tagStart;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      tagEl.Read (dictFile);
+      ReadUL (invfIdxFile, tagStart);
+      PrintInvfTag (invfFile, idh, ifh, tagEl, tagStart, printFrags);
+    }
+  }
+  // close the open files
+  fclose (invfIdxFile);
+  fclose (invfFile);
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *dir_name, *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+  bool printFrags = false;
+  
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hrwtnf:d:")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath(optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 'n':
+      printFrags = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-n] [-f input_file]"
+	       "[-d data directory]\n(-rwnt:print header, words, tags, fragnums)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+  
+  process_files (filename, printHeader, printWords, printTags, printFrags);
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_passes.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_passes.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_passes.1	(revision 21112)
@@ -0,0 +1,323 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_passes 1 \*(Dt CITRI
+.SH NAME
+mgpp_passes \- builds mgpp databases
+.SH SYNOPSIS
+.B mgpp_passes
+[
+.BI \-J " doc-tag"
+]
+[
+.BI \-K " level-tag"
+]
+.if n .ti +10n
+[
+.BI \-L " index-level"
+]
+[
+.BI \-m " invf-mem-buffer"
+]
+.if n .ti +10n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+[
+.B \-S
+]
+[
+.B \-C
+]
+.if n .ti +10n
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mgpp_passes
+is the program that does most of the work when building mgpp
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  In general,
+.B mgpp_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+mgpp database.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+mgpp database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpointt\fP'u+2n"
+.BI \-J " doc-tag"
+Specifies the SGML tag that encloses each document.  Text appearing
+outside this tag is ignored.  The document tag defines the highest
+level document that can be queried and printed.  The default document
+tag is 'Document'.
+.TP
+.BI \-K " level-tag"
+Specifies the SGML tag of a sub document level.  A level tag must
+enclose all text enclosed by the document tag.  Levels can be
+queried and printed as if they were separate documents.  Multiple
+document levels can be specified (the document tag is always
+added as a document level).
+.TP
+.BI \-L " index-level"
+Specifies the SGML tag enclosing the smallest indexed element.  The
+index level should be no larger than the smallest document 
+level.  An empty string can be used to specify a word level index 
+(which is the default).
+.TP
+.BI \-m " invf-mem-buffer"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+.IR *.text.level ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mgpp_perf_hash_build (1)
+from the
+.I *.invf.dict
+file.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP 
+.B \-C
+This activates the compatibility parsing mode.  When using this
+mode documents are separated by control-B and paragraphs are separated
+by control-C.  Internally these are converted to documents surrounded
+by 'Document' tags and paragraphs surrounded by 'Paragraph' tags.
+.TP
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an mgpp document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats, *.invf.dict, *.invf.level
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mgpp_passes -T1 -I1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mgpp_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mgpp_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text, *.text.idx, *.text.level
+.I
+#   *.invf and *.invf.idx
+${source} | mgpp_passes -T2 -I2 -f ${text}
+.PP
+.I
+# Create *.text.weight and *.weight.approx
+mgpp_weights_build -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked
+mgpp_invf_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.1
+mgpp_stem_idx -s 1 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.2
+mgpp_stem_idx -s 2 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.3
+mgpp_stem_idx -s 3 -f ${text}
+.PP
+.I
+# Create *.text.dict.fast
+mgpp_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary with index into the dictionary.
+.TP
+.B *.invf.dict.blocked.n
+Transformation dictionary from words stemmed with method 
+.B n
+to unstemmed words.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.level
+Information about the document levels needed for querying.
+.TP
+.B *.text
+Compressed text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.fast
+A fast loading version of the compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.level
+Information about the document levels needed for text decompression.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+The approximate weights file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_passes.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_passes.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_passes.cpp	(revision 21112)
@@ -0,0 +1,268 @@
+/**************************************************************************
+ *
+ * mgpp_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#if defined __WIN32__
+# include <io.h>
+# include "getopt_old.h"
+# define close _close
+# define open _open
+#elif defined __CYGWIN__
+#include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "longlong.h"
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-J doc-tag] [-K level-tag] [-L index-level]\n"
+"  %*s [-m invf-memory] [-T1] [-T2] [-I1] [-I2] [-S]\n"
+"  %*s [-C] [-h] [-d directory] -f name\n\n";
+
+
+
+static void Usage (char *err) {
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "");
+  exit (1);
+}
+
+
+
+
+int OpenNextFile (int in_fd) {
+  if (in_fd > 0) close (in_fd);
+  if (num_files == 0) return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  ++files;
+  --num_files;
+  return (in_fd);
+}
+
+
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; ++pass) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      ++numDocs;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+int main (int argc, char **argv) {
+  int ch, in_fd;
+  char *filename = NULL;
+  bool compatMode = false;
+  TagInfo tagInfo;
+  tagInfo.SetDocTag ("Document");
+  
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch=getopt(argc, argv, "J:K:L:f:d:m:I:T:SCh"))!=-1){
+    switch (ch) {
+    case 'J':
+      tagInfo.SetDocTag (optarg);
+      break;
+    case 'K':
+      tagInfo.AddLevelTag (optarg);
+      break;
+    case 'L':
+      tagInfo.SetIndexLevel (optarg);
+      break;
+    case 'f':
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'm':
+      invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+      break;
+    case 'I':
+      if (*optarg == '1')
+	Passes |= IVF_PASS_1;
+      else if (*optarg == '2')
+	Passes |= IVF_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'T':
+      if (*optarg == '1')
+	Passes |= TEXT_PASS_1;
+      else if (*optarg == '2')
+	Passes |= TEXT_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'S':
+      Passes |= SPECIAL;
+      break;
+    case 'C':
+      compatMode = true;
+      break;
+    case 'h':
+    case '?':
+      Usage (NULL);
+    }
+  }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc) {
+    if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+      FatalError (1, "Cannot open %s", argv[optind]);
+    files = &argv[optind + 1];
+    num_files = argc - (optind + 1);
+    
+  } else in_fd = 0;  // stdin
+
+  
+  if (compatMode) tagInfo.SetDocTag ("Document");
+
+  // a document tag is also a level tag
+  tagInfo.levelTags.insert (tagInfo.docTag); 
+  
+  Driver (in_fd, filename, tagInfo, compatMode);
+
+  return (0);
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.cpp	(revision 21112)
@@ -0,0 +1,316 @@
+/**************************************************************************
+ *
+ * mgpp_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#if defined __WIN32__
+# include <io.h>
+# include "getopt_old.h"
+# define close _close
+# define open _open
+#elif defined __CYGWIN__
+#include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "longlong.h"
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+#include "mgpp_passes_4jni.h"
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+TagInfo tagInfo;
+char *filename = NULL;
+
+unsigned long numBytes = 0;
+unsigned long numDocs = 0;
+int mgpp_passes_exit_value = 0;
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+
+/* clear all the settings from one mgpp_passes run to the next */
+void clear_variables() {
+  tagInfo.Clear();
+  tagInfo.SetDocTag ("Document");
+  Passes = 0;
+  invf_buffer_size = 5 * 1024 * 1024;
+  numBytes = 0;
+  numDocs = 0;
+  mgpp_passes_exit_value = 0;
+}
+
+/* ################################################## */
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num) {
+
+  switch(pass_type) {
+  case 'S':
+    Passes |= SPECIAL;
+    break;
+  case 'I':
+    if (pass_num == '1')
+      Passes |= IVF_PASS_1;
+    else if (pass_num == '2')
+      Passes |= IVF_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  case 'T':
+    if (pass_num == '1')
+      Passes |= TEXT_PASS_1;
+    else if (pass_num == '2')
+      Passes |= TEXT_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  }
+}
+
+/* -m arg to mgpp_passes */
+void set_inversion_limit(int limit) {
+  invf_buffer_size = limit * 1024 * 1024;
+}
+
+/* -J arg to mgpp_passes */
+void set_document_tag(const char *tag_name) {
+  tagInfo.SetDocTag (tag_name);
+
+  // a doc tag is also a level tag
+  tagInfo.AddLevelTag(tag_name);
+
+}
+
+/* -K arg to mgpp_passes */
+void add_level_tag(const char *tag_nam) {
+  tagInfo.AddLevelTag(tag_nam);
+}
+
+/* -L arg to mgpp_passes */
+void set_index_level(const char *tag_name) {
+  tagInfo.SetIndexLevel(tag_name);
+}
+
+// is this enough???
+/* -f arg to mgpp_passes */
+void set_filename(const char * filen) {
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+  filename = Xstrdup (filen);  
+}
+ 
+
+/* ############################################### */
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver () {
+
+  if (!filename || *filename == '\0') {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "A document collection name must be specified.");
+  }
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2)) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+  }
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2)) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+  }
+  if (!Passes) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+  }
+  
+  // initialise all the passes
+  for (int pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, filename) == COMPERROR) {
+	mgpp_passes_exit_value = 1;
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+      }
+    }
+  }
+
+}
+
+void process_document(u_char *buffer, int len) {
+  
+  TextElArray doc;
+  unsigned long doc_len =  0;
+
+  while(ReadDoc(&buffer, tagInfo.docTag, doc, doc_len, false)) {
+    // give this document to each pass
+    for (int pass = 0; pass < MAX_PASSES; pass++) {
+      if (Passes & (1 << pass)) {
+	if (PassData[pass].process (tagInfo, doc) == COMPERROR) {
+	  mgpp_passes_exit_value = 1;
+	  FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+    }
+    
+    // another document has been processed
+    numBytes += doc_len;
+    numDocs++;
+    cout << "doc_len = "<<doc_len<<endl;
+  }
+}    
+
+void finalise_driver () {
+  
+  // do done for each pass
+  for (int pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, filename) == COMPERROR) {
+	mgpp_passes_exit_value = 1;
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+    }
+  }  
+}
+
+int get_exit_value() {
+  return mgpp_passes_exit_value;
+}
+  /*
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; pass++) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      numDocs++;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+  */
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_passes_4jni.h	(revision 21112)
@@ -0,0 +1,62 @@
+/**************************************************************************
+ *
+ * mgpp_passes_4jni.h -- modified mgpp_passes for use from jni
+ * Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef H_MGPP_PASSES_4JNI
+#define H_MGPP_PASSES_4JNI
+
+
+/* clear all the settings from one mgpp_passes run to the next */
+void clear_variables();
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num);
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit);
+
+/* -J arg to mgpp_passes */
+void set_document_tag(const char * tag_name);
+
+/* -K arg to mgpp_passes */
+void add_level_tag(const char * tag_name);
+
+/* -L arg to mgpp_passes */
+void set_index_level(const char * tag_name);
+
+/* -f arg to mgpp_passes */
+void set_filename(const char * filen);
+
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver ();
+
+void process_document(u_char *buffer, int len);
+
+void finalise_driver();
+
+int get_exit_value();
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mgpp_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * mgpp_perf_hash_build.cpp -- Program to build a perfect hash function
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define POOL_SIZE 1024*1024
+
+static void ProcessFiles (char *filename, int r) {
+  FILE *dictFile, *hashFile;
+  unsigned long i;
+  invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  unsigned long pool_left;
+  u_char **starts;
+
+  // read in the dictionary
+  dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  if (dictFile==NULL) {
+    FatalError(1, "unable to open file");
+  }
+  idh.Read (dictFile);
+
+  //cerr << idh.lookback<<" "<<idh.word_dict_start<<endl;
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  
+  if (!(pool = (u_char *) Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = (u_char **) Xmalloc (sizeof (u_char *) * idh.word_dict_size)))
+    FatalError (1, "Out of memory");
+  //cerr << "size= "<< idh.word_dict_size<<endl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (i = 0; i < idh.word_dict_size; ++i) {
+    // read the next word and associated information
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // push string onto pool data
+    register unsigned long l;
+    l = wordEl.el.size() + 1;
+    if (pool_left < l) {
+      pool = (u_char *) Xmalloc (POOL_SIZE);
+      pool_left = POOL_SIZE;
+    }
+    starts[i] = pool;
+    
+    *pool++ = wordEl.el.size();
+    memcpy ((char *) pool, &(wordEl.el[0]), wordEl.el.size());
+    //cerr << pool<<"   " <<starts[i]<<endl;
+    pool += wordEl.el.size();
+    pool_left -= l;
+    
+  }
+  fclose (dictFile);
+  //cerr << pool<<"   " <<starts[i-1]<<endl;
+  //cerr<<"starts  "<<starts[113529]<<endl;
+  //cerr << starts[17][1] << " "<<starts[25][4]<<endl;
+  // create perfect hash file
+  hashFile = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+			  MAGIC_HASH, MG_ABORT);
+  if (!(phd = gen_hash_func (idh.word_dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hashFile, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (hashFile);
+}
+
+
+
+int main (int argc, char **argv) {
+  int r = -1;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      r = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  ProcessFiles (filename, r);
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mgpp_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mgpp_stem_idx
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +12n
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +12n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_stem_idx
+generates a stem index file for a collection.
+This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to set
+the number of entries per block. The default is 16.
+.TP
+.BI \-a " stemmer"
+The name of the stemmer to use, the default is the Lovin stemmer.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_stem_idx.cpp -- stem index builder
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+#include "stemmer.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#  include <vector.h>
+#else
+#  include <map>
+#  include <vector>
+#endif
+
+typedef vector<unsigned long> WordNumList;
+typedef map<UCArray, WordNumList, DictLTUCArray> StemMapDict;
+
+void CreateStemDict (char *filename,
+		     StemMapDict &stemDict,
+		     int stemMethod,
+		     int stemmerNum) {
+  stemDict.erase (stemDict.begin(), stemDict.end());
+
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  unsigned long wordNum;
+  u_char mgWord[MAXSTEMLEN + 1];
+  word_dict_el wordEl;
+  UCArray stemEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // convert the word to an "mg word"
+    mgWord[0] = wordEl.el.size();
+    memcpy((char *)&mgWord[1], &(wordEl.el[0]), wordEl.el.size());
+
+    // stem the word
+    stemmer (stemMethod, stemmerNum, mgWord);
+
+    // convert the result back to a UCArray
+    stemEl.erase (stemEl.begin(), stemEl.end());
+    stemEl.insert (stemEl.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+//      cout << "\"" << stemEl << "\" -> \"" << wordEl.el << "\"\n";
+    
+    // add this word number to the list of word numbers for this word
+    stemDict[stemEl].push_back (wordNum);
+  }
+
+  fclose (dictFile);
+}
+
+
+void WriteStemDict (char *filename,
+		    StemMapDict &stemDict,
+		    int stemMethod,
+		    int stemmerNum,
+		    unsigned long entriesPerBlock) {
+  // Create appropriate stem index file
+  FILE *stemDictFile = NULL;
+  if (stemMethod == 1) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+				"wb", MAGIC_STEM_1, MG_ABORT);
+  } else if (stemMethod == 2) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+				"wb", MAGIC_STEM_2, MG_ABORT);
+  } else if (stemMethod == 3) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+				"wb", MAGIC_STEM_3, MG_ABORT);
+  } else {
+    FatalError (1, "Unknown stem method %d", stemMethod);
+  }
+
+  stem_idx_header sih;
+  sih.lookback = 0;
+  sih.dict_size = stemDict.size();
+  sih.entries_per_block = entriesPerBlock;
+  sih.max_block_size = 0;
+  
+  sih.stemmer_num = stemmerNum;
+  sih.stem_method = stemMethod;
+
+  // write out a place-holder version of the header
+  sih.Write (stemDictFile);
+
+  sih.blocks_start = ftell (stemDictFile);
+
+  block_idx stemIdx;
+  unsigned long stemNum = 0;
+  stem_block_dict_el stemEl;
+  UCArray lastEl;
+
+  StemMapDict::const_iterator here = stemDict.begin();
+  StemMapDict::const_iterator end = stemDict.end();
+  while (here != end) {
+    // remember this stem (and position) if this is the start
+    // of a new block
+    if (stemNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = (*here).first;
+      elIdx.block_ptr = ftell (stemDictFile);
+
+      // see if this block is the longest so far
+      if (stemIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(stemIdx.end()-1)).block_ptr;
+	if (blockLen > sih.max_block_size) sih.max_block_size = blockLen;
+      }
+      
+      stemIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this stem
+    stemEl.el = (*here).first;
+    stemEl.equivWords = (*here).second;
+
+    // write out the stem
+    stemEl.Write (stemDictFile, &lastEl);
+    
+    ++here; ++stemNum;
+  }
+
+
+  // write out the element indexes
+  sih.num_blocks = stemIdx.size();
+  sih.block_idx_start = ftell (stemDictFile);
+  WriteBlockIdx (stemDictFile, stemIdx);
+  
+  // write out the stem dictionary header
+  fseek (stemDictFile, sizeof(unsigned long), SEEK_SET);
+  sih.Write (stemDictFile);
+
+
+  // close open files
+  fclose (stemDictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Num word stems = %d\n", sih.dict_size);
+  Message ("Max stem block size = %d\n", sih.max_block_size);
+  Message ("Number of stem blocks written = %d\n", sih.num_blocks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  int stemMethod = 0; // illegal value (no translation)
+  int stemmerNum = 0; // English stemmer
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:s:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 's':
+      stemMethod = atoi (optarg);
+      break;
+    case 'a':
+      stemmerNum = stemmernumber ((unsigned char *) optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-d directory] "
+	       "[-b entries-per-block] [-h] -s 1|2|3 "
+	       "[-a stemmer-method] -f name\n", argv[0]);
+      exit (1);
+    }
+  }
+  
+  if (stemMethod < 1 || stemMethod > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  // read in the dictionary and create the in memory dictionary
+  StemMapDict stemDict;
+  CreateStemDict (filename, stemDict, stemMethod, stemmerNum);
+  
+  // write out the dictionary as a blocked file
+  WriteStemDict (filename, stemDict, stemMethod, stemmerNum, entriesPerBlock);
+  
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.1
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.1	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_weights_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mgpp_weights_build
+[
+.B \-h
+]
+[
+.BI \-b " num-bits"
+]
+.if n .ti +17n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_weights_build
+is the program that generates the exact and approximate weights 
+files.  To do this, it uses the inverted file
+and dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num-bits"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1)
Index: /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
@@ -0,0 +1,399 @@
+/**************************************************************************
+ *
+ * mgpp_weights_build.cpp -- Program to build the document weights file
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+struct WBTagPtr {
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  unsigned long fragOccur;
+  
+  WBTagPtr () {
+    tagNum = 0;
+    tagPtr = 0;
+    fragOccur = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, WBTagPtr, DictLTUCArray> WBTagDict;
+
+typedef vector<float> Weights;
+
+
+static void ReadTagDict (const invf_dict_header &idh,
+			 FILE *dictFile,
+			 FILE *invfIdxFile,
+			 WBTagDict &tagDict) {
+  tagDict.erase (tagDict.begin(), tagDict.end());
+
+  // seek to the start of the tag information
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof(unsigned long) +
+	 idh.word_dict_size*sizeof(unsigned long), SEEK_SET);
+  
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagPtr);
+    tagDict[thisEl.el].tagNum = tagNum;
+    tagDict[thisEl.el].tagPtr = tagPtr;
+    tagDict[thisEl.el].fragOccur = thisEl.frag_occur;
+  }
+}
+
+static void AddWeight (Weights &w,
+		       unsigned long levelDocNum,
+		       unsigned long termFreq,
+		       float idf) {
+  double weight = termFreq * idf;
+  w[levelDocNum-1] += weight * weight;
+}
+
+static void GenerateLevelWeights (const invf_dict_header &idh,
+				  const invf_file_header &ifh,
+				  unsigned long numLevelDocs,
+				  unsigned long levelNum,
+				  FILE *dictFile,
+				  FILE *invfFile,
+				  FILE *invfIdxFile,
+				  const FragLevelConvert &fragLevelConvert,
+				  Weights &w) {
+  // pre-allocate the right number of weights
+  w.erase (w.begin(), w.end());
+  w.insert (w.end(), (Weights::size_type)numLevelDocs, (float)0.0);
+
+  double logN = log ((double) numLevelDocs);
+
+  // reset the files
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof (unsigned long), SEEK_SET);
+  
+  // process each word adding its contributions to the document weights
+  unsigned long wordNum;
+  unsigned long wordStart;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+#ifndef SILENT
+    // give a little feedback every 4096 words
+    if ((wordNum & 0xfff) == 0) fprintf (stderr, ".");
+#endif
+    
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordStart);
+
+    float idf = logN - log ((double) wordEl.levelFreqs[levelNum]);
+
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    stdio_bitio_buffer buffer (invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long levelDocNum = 0;
+    unsigned long lastLevelDocNum = 0;
+    unsigned long termFreq = 0;
+    unsigned long checkLevelFreq = 0;
+    
+    unsigned long count, i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      fragNum += buffer.bblock_decode (B, NULL);
+      if (!ifh.word_level_index) count = buffer.gamma_decode (NULL);
+      else count = 1;
+
+      if (fragNum > idh.num_frags)
+	FatalError (1, "fragNum = %d, "
+		    "number of fragments = %d\n"
+		    "wordNum = %d\n"
+		    "i = %d, frag_occur = %d\n",
+		    fragNum, idh.num_frags, wordNum, i, wordEl.frag_occur);
+
+      if (!fragLevelConvert.FragToLevel (fragNum, levelDocNum))
+	FatalError (1, "could not convert fragment number %d in level %d", fragNum, levelNum);
+
+      if (levelDocNum == 0 || levelDocNum > numLevelDocs)
+	FatalError (1, "bad level document number %d in level %d", levelDocNum, levelNum);
+      
+      if (levelDocNum != lastLevelDocNum) {
+	// new level document
+	if (lastLevelDocNum > 0) {
+	  AddWeight (w, lastLevelDocNum, termFreq, idf);
+	  ++checkLevelFreq;
+	}
+	lastLevelDocNum = levelDocNum;
+	termFreq = 0;
+      }
+      termFreq += count;
+    }
+
+    if (lastLevelDocNum > 0) {
+      AddWeight (w, lastLevelDocNum, termFreq, idf);
+      ++checkLevelFreq;
+    }
+
+    if (checkLevelFreq != wordEl.levelFreqs[levelNum]) {
+      cerr << "bad level freq at level " <<levelNum<<" "<< checkLevelFreq << " != "
+	   << wordEl.levelFreqs[levelNum] << ", word \""
+	   << wordEl.el << "\" (" << wordNum << ")\n";
+      exit (1);
+    }
+    
+    buffer.done();
+  }
+
+  if (w.size() != numLevelDocs) 
+    FatalError (1, "wrong number of weights created %d != %d",
+		w.size(), numLevelDocs);
+}
+
+static void WriteExactWeights (FILE *weightsFile,
+			       unsigned long &diskPtr,
+			       const Weights &w) {
+  diskPtr = ftell(weightsFile);
+
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+//      cout << *here << "\n";
+    WriteF (weightsFile, sqrt (*here));
+    ++here;
+  }
+}
+
+static void WriteApproxWeights (FILE *approxWeightsFile,
+				unsigned long &diskPtr,
+				const Weights &w,
+				unsigned char bits) {
+  diskPtr = ftell(approxWeightsFile);
+
+  // calculate L, U and B
+  double L = 1e300;
+  double U = 0;
+  float wgt;
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+    wgt = sqrt (*here);
+    if (wgt > U) U = wgt;
+    if (wgt > 0 && wgt < L) L = wgt;
+    ++here;
+  }
+
+  double B = pow (U / L, pow (2.0, -(double) bits));
+
+#ifndef SILENT
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+#endif
+  
+  WriteUC (approxWeightsFile, bits);
+  WriteD (approxWeightsFile, L);
+  WriteD (approxWeightsFile, B);
+  
+
+  unsigned long max = (bits == 32) ? 0xffffffff : (1 << bits) - 1;;
+  unsigned long i=0, buf=0, pos=0;
+  here = w.begin();
+  end = w.end();
+  while (here != end) {
+    unsigned long fx;
+    wgt = sqrt (*here);
+    if (wgt == 0) {
+      wgt = L;
+#ifndef SILENT
+      Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+    }
+    fx = (unsigned long) floor (log (wgt / L) / log (B));
+
+    if (fx > max) fx = max;
+
+    buf |= (fx << pos);
+    pos += bits;
+
+    if (pos >= MAXBITS) {
+      WriteUL (approxWeightsFile, buf);
+      buf = fx >> (bits - (pos - MAXBITS));
+      pos = pos - MAXBITS;
+    }
+
+    ++here; ++i;
+  }
+
+  // write out the last bits
+  if (pos > 0) WriteUL (approxWeightsFile, buf);
+}
+
+int main (int argc, char **argv) {
+  unsigned char bits = 8;
+  char *filename = "";
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      bits = atoi (optarg);
+      if (bits > 32) {
+	fprintf (stderr, "b may only take values 0-32\n");
+	exit (1);
+      }
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-b bits] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to create weights.");
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+  
+  // read the level information
+  FILE *levelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			       MAGIC_INVF_LEVELS, MG_ABORT);
+  FIvfLevel ivfLevel;
+  ivfLevel.Read (levelFile);
+  fclose (levelFile);
+
+  // read in the tag dictionary and inverted file pointers
+  WBTagDict tagDict;
+  ReadTagDict (idh, dictFile, invfIdxFile, tagDict);
+
+  
+  // create the weights file
+  FILE *weightsFile = create_file (filename, WEIGHTS_SUFFIX, "wb",
+				   MAGIC_WGHT, MG_ABORT);
+
+  // create the approx weights file
+  FILE *approxWeightsFile = create_file (filename, APPROX_WEIGHTS_SUFFIX, "wb",
+					 MAGIC_WGHT_APPROX, MG_ABORT);
+
+  
+  // create weights for each document level
+  FragLevelConvert fragLevelConvert;
+  Weights w;
+  IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+  IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+  unsigned long levelNum = 0;
+  while (levelHere != levelEnd) {
+    const UCArray &levelName = (*levelHere).first;
+    
+    // read the tag information about this level
+    fragLevelConvert.Read(invfFile, tagDict[levelName].tagPtr,
+			  idh.num_frags, tagDict[levelName].fragOccur);
+    
+    // create the weights for this level
+    GenerateLevelWeights (idh, ifh, tagDict[levelName].fragOccur,
+			  levelNum, dictFile, invfFile, invfIdxFile,
+			  fragLevelConvert, w);
+
+    // write out the exact weights
+    WriteExactWeights (weightsFile,
+		       (*levelHere).second.exactWeightsDiskPtr,
+		       w);
+    
+    // write out the approximate weights
+    WriteApproxWeights (approxWeightsFile,
+			(*levelHere).second.approxWeightsDiskPtr,
+			w, bits);
+    
+    ++levelHere; ++levelNum;
+  }
+  
+
+  // close input files
+  fclose (dictFile);
+  fclose (invfFile);
+  fclose (invfIdxFile);
+
+  // update the level information
+  levelFile = create_file (filename, INVF_LEVEL_SUFFIX, "wb",
+			   MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Write (levelFile);
+  fclose (levelFile);
+
+  // close output files
+  fclose (weightsFile);
+  fclose (approxWeightsFile);
+
+
+  return 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/stemmer.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/stemmer.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/stemmer.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * stemmer.cpp -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; ++i) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); ++i)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp ((char *) descript,  "0") == 0) ||
+      (strcmp ((char *) descript, "english") == 0) ||
+      (strcmp ((char *) descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp ((char *) descript, "1") == 1) ||
+      (strcmp ((char *) descript, "french") == 0) ||
+      (strcmp ((char *) descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/indexers/mgpp/text/stemmer.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/stemmer.h	(revision 21112)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+#ifdef __cplusplus
+extern "C" 
+#endif
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/text.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/text.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/text.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// UCArray currently contains all the basic reading and writing functions
+#include "UCArray.h"
+#include "text.h"
+
+compressed_text_header::compressed_text_header () {
+  Clear();
+}
+
+void compressed_text_header::Clear () {
+  num_of_docs = 0;
+  num_of_words = 0;
+  num_of_bytes = 0.0;
+}
+
+
+bool compressed_text_header::Read (FILE *f) {
+  return (ReadUL (f, num_of_docs) &&
+	  ReadUL (f, num_of_words) &&
+	  ReadD (f, num_of_bytes));
+}
+
+bool compressed_text_header::Write (FILE *f) const {
+  return (WriteUL (f, num_of_docs) &&
+	  WriteUL (f, num_of_words) &&
+	  WriteD (f, num_of_bytes));
+}
+
+
+auxiliary_dict::auxiliary_dict () {
+  word_data[0] = NULL;
+  word_data[1] = NULL;
+  words[0] = NULL;
+  words[1] = NULL;
+}
+
+auxiliary_dict::~auxiliary_dict () {
+  Clear ();
+}
+
+void auxiliary_dict::Clear () {
+  if (word_data[0] != NULL) {
+    delete word_data[0];
+    word_data[0] = NULL;
+  }
+  if (word_data[1] != NULL) {
+    delete word_data[1];
+    word_data[1] = NULL;
+  }
+  if (words[0] != NULL) {
+    delete words[0];
+    words[0] = NULL;
+  }
+  if (words[1] != NULL) {
+    delete words[1];
+    words[1] = NULL;
+  }
+}
+
+
+compression_dict::compression_dict () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    cfh[which] = NULL;
+    values[which] = NULL;
+    escape[which] = NULL;
+    chars_huff[which] = NULL;
+    chars_vals[which] = NULL;
+    lens_huff[which] = NULL;
+    lens_vals[which] = NULL;
+  }
+
+  MemForCompDict = 0;
+  ad = NULL;
+  fast_loaded = 0;
+}
+
+compression_dict::~compression_dict () {
+  Clear ();
+}
+
+void compression_dict::Clear () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    if (cfh[which] != NULL) {
+      delete cfh[which];
+      cfh[which] = NULL;
+    }
+    if (values[which] != NULL) {
+      delete values[which][0][0];
+      delete values[which][0];
+      delete values[which];
+      values[which] = NULL;
+    }
+    if (escape[which] != NULL) { // don't know if needed
+      //delete escape[which]; // this was causing a seg fault
+      escape[which] = NULL;
+    }
+    if (chars_huff[which] != NULL) {
+      delete chars_huff[which];
+      chars_huff[which] = NULL;
+    }
+    if (chars_vals[which] != NULL) {
+      delete chars_vals[which][0];
+      delete chars_vals[which];
+      chars_vals[which] = NULL;
+    }
+    if (lens_huff[which] != NULL) {
+      delete lens_huff[which];
+      lens_huff[which] = NULL;
+    }
+    if (lens_vals[which] != NULL) {
+      delete lens_vals[which][0];
+      delete lens_vals[which];
+      lens_vals[which] = NULL;
+    }
+  }
+
+  if (ad != NULL) {
+    delete ad;
+    ad = NULL;
+  }
+
+  MemForCompDict = 0;
+  fast_loaded = 0;
+}
Index: /main/tags/3.01/indexers/mgpp/text/text.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/text.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/text.h	(revision 21112)
@@ -0,0 +1,199 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT
+#define H_TEXT
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+#include "huffman.h"
+#include <stdio.h>
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+struct compression_dict_header {
+  u_long dict_type;
+  u_long novel_method;
+  u_long params[TEXT_PARAMS];
+  u_long num_words[2];
+  u_long num_word_chars[2];
+  u_long lookback;
+};
+
+
+struct comp_frags_header {
+  huff_data hd;
+  u_long uncompressed_size;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+};
+
+// BOGUSTEXTLEN is used to replace the ratio
+// in the compressed_text_header while preparing
+// for UCArray
+#define BOGUSTEXTLEN 1000000
+
+struct compressed_text_header {
+  u_long num_of_docs;
+  u_long num_of_words; // number of words in collection
+  double num_of_bytes;
+
+  compressed_text_header ();
+  void Clear ();
+
+  // you must seek to the appropriate place before calling
+  // Read or Write
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct compression_stats_header {
+  u_long num_docs;
+  u_long dummy;     // alignment
+  double num_bytes; 
+};
+
+
+struct frags_stats_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct aux_frags_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct auxiliary_dict {
+  aux_frags_header afh[2];
+  u_char *word_data[2];
+  u_char **words[2];
+  int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+					   for the hybrid methods */
+
+  auxiliary_dict ();
+  ~auxiliary_dict ();
+
+  void Clear ();
+};
+
+
+struct compression_dict {
+  compression_dict_header cdh;
+  comp_frags_header *cfh[2];
+  unsigned long MemForCompDict;
+  u_char ***values[2];
+  u_char *escape[2];
+  huff_data *chars_huff[2];
+  u_long **chars_vals[2];
+  huff_data *lens_huff[2];
+  u_long **lens_vals[2];
+  auxiliary_dict *ad;
+  int fast_loaded;
+
+  compression_dict ();
+  ~compression_dict ();
+
+  void Clear ();
+};
+
+
+#endif
Index: /main/tags/3.01/indexers/mgpp/text/text.pass1.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/text.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/text.pass1.cpp	(revision 21112)
@@ -0,0 +1,388 @@
+/**************************************************************************
+ *
+ * text.pass1.cpp -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+#include "TextEl.h"
+
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long longestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh = {0, 0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void ChangeMem (int Change) {
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse;
+}
+
+
+int init_text_1 (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  int which;
+  
+  if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+    Message ("Unable to allocate memory for pool");
+    return (COMPERROR);
+  }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+  
+  for (which = 1; which >= 0; --which) {
+    u_char *word;
+    hash_rec *ent;
+    dict_data *dd = &DictData[which];
+    
+    dd->wordnum = 0;
+    dd->words_read = 0;
+    dd->bytes_diff = 0;
+    dd->HashSize = INITIAL_HASH_SIZE;
+    dd->HashUsed = 0;
+    
+    if (!(dd->HashTable = (hash_rec *) Xmalloc (sizeof (hash_rec) * dd->HashSize))) {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+    ChangeMem (sizeof (hash_rec) * dd->HashSize);
+    memset (dd->HashTable,'\0' , sizeof (hash_rec) * dd->HashSize);
+
+    word = Pool;
+    *Pool++ = '\0';
+    --PoolLeft;
+    {
+      register u_char *wptr;
+      register int hsize = dd->HashSize;
+      register unsigned long hashval, step;
+      
+      HASH (hashval, step, word, hsize);
+      wptr = (dd->HashTable + hashval)->word;
+      while (wptr) {
+	hashval += step;
+	if (hashval >= (unsigned long)hsize)
+	  hashval -= hsize;
+	wptr = (dd->HashTable + hashval)->word;
+      }
+      ent = dd->HashTable + hashval;
+    }
+    ent->wcnt = 1;
+    ent->word = word;
+    dd->HashUsed = 1;
+  }
+  return (COMPALLOK);
+}
+
+
+static int process_text_element (const u_char *s_in, int l_in) {
+  const u_char *end = s_in + l_in - 1;
+  
+   /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  
+  bool which = false; // non-word
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    dict_data *dd = &DictData[which];
+    
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+    
+    ++dd->wordnum;
+    inputbytes += *Word;
+    ++dd->words_read;
+    
+    /* Search the hash table for Word */
+    {
+      register unsigned long hashval, step;
+      register int hsize = dd->HashSize;
+      HASH (hashval, step, Word, hsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	register hash_rec *ent;
+	ent = dd->HashTable + hashval;
+	if (!ent->word) {
+	  int len = *Word + 1;
+	  if (len > PoolLeft) {
+	    if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+	      Message ("Unable to allocate memory for pool");
+	      return (COMPERROR);
+	    }
+	    PoolLeft = POOL_SIZE;
+	    ChangeMem (POOL_SIZE);
+	  }
+	  ent->occurance_num = occurance_num++;
+	  ent->wcnt = 1;
+	  ent->word = Pool;
+	  memcpy (Pool, Word, len);
+	  Pool += len;
+	  PoolLeft -= len;
+	  ++dd->HashUsed;
+	  dd->bytes_diff += Word[0];
+	  break;
+	}
+	
+	/* Compare the words */
+	s1 = Word;
+	s2 = ent->word;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval = (hashval + step);
+	  if (hashval >= (unsigned long)hsize) hashval -= hsize;
+	} else {
+	  ++ent->wcnt;
+	  break;
+	}
+      }
+    }
+
+    
+    if (dd->HashUsed >= dd->HashSize >> 1) {
+      hash_rec *ht;
+      unsigned long size;
+      unsigned long i;
+      size = prime (dd->HashSize * 2);
+      if (!(ht = (hash_rec *) Xmalloc (sizeof (hash_rec) * size))) {
+	Message ("Unable to allocate memory for table");
+	return (COMPERROR);
+      }
+      ChangeMem (sizeof (hash_rec) * size);
+      memset (ht, '\0', sizeof (hash_rec) * size);
+
+      for (i = 0; i < dd->HashSize; ++i)
+	if (dd->HashTable[i].word) {
+	  register u_char *wptr;
+	  register unsigned long hashval, step;
+	  
+	  wptr = dd->HashTable[i].word;
+	  HASH (hashval, step, wptr, size);
+	  wptr = (ht + hashval)->word;
+	  while (wptr) {
+	    hashval += step;
+	    if (hashval >= size) hashval -= size;
+	    wptr = (ht + hashval)->word;
+	  }
+	  ht[hashval] = dd->HashTable[i];
+	}
+      Xfree (dd->HashTable);
+      ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+      dd->HashTable = ht;
+      dd->HashSize = size;
+    }
+  }
+
+  return COMPALLOK;
+}
+
+
+int process_text_1 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  unsigned long textLen = 0;
+  unsigned long docLen = 0;
+  int retValue;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    textLen = (*here).text.size();
+    docLen += textLen;
+
+    retValue = process_text_element (&(here->text[0]), textLen);
+    if (retValue != COMPALLOK) return retValue;
+    
+    ++here;
+  }
+
+  // get max document length
+  if (docLen > longestDoc) longestDoc = docLen;
+
+  // update header information
+  ++csh.num_docs;
+  csh.num_bytes += docLen;
+  
+  return COMPALLOK;
+}
+
+
+static int PackHashTable (dict_data * dd) {
+  int s, d;
+  for (s = d = 0; (unsigned int)s < dd->HashSize; ++s)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  
+  if (!(dd->HashTable = (hash_rec *) Xrealloc (dd->HashTable,
+			   sizeof (hash_rec) * dd->HashUsed))) {
+    Message ("Out of memory");
+    return COMPERROR;
+  }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+static int ent_comp (const void *s1, const void *s2) {
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+static void WriteHashTable (FILE * fp, dict_data * dd) {
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR) return;
+  
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+  
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; ++j)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+  
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+  
+  for (j = 0; j < dd->HashSize; ++j) {
+    curr = dd->HashTable[j].word;
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    HTONUL(dd->HashTable[j].wcnt);
+    HTONUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+    fwrite (&dd->HashTable[j].occurance_num,
+	    sizeof (dd->HashTable[j].occurance_num), 1, fp);
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    NTOHUL(dd->HashTable[j].wcnt);
+    NTOHUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+  }
+}
+
+
+int done_text_1 (const TagInfo &/*tagInfo*/, char *file_name) {
+  char *temp_str;
+  FILE *fp;
+  
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+	      " "
+#endif
+              );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  fclose(fp);
+  return COMPALLOK;
+}
Index: /main/tags/3.01/indexers/mgpp/text/text.pass2.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/text.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/text.pass2.cpp	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * text.pass2.cpp -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#if defined (__WIN32__)
+# include <stdio.h>
+# define unlink _unlink
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+#include "FText.h"
+
+#define POOL_SIZE 1024*256
+
+
+struct char_pool {
+  struct char_pool *next;
+  u_long left;
+  u_char *ptr;
+  u_char pool[POOL_SIZE];
+};
+
+struct novel_hash_rec {
+  u_long ordinal_num;
+  u_char *word;
+};
+
+
+#define INITIAL_HASH_SIZE 7927
+
+struct novel_hash_table {
+  novel_hash_rec *HashTable;
+  u_long HashSize, HashUsed;
+  char_pool *first_pool;
+  char_pool *pool;
+  u_long next_num, binary_start;
+  novel_hash_rec **code_to_nhr;
+};
+
+
+static FILE *text;
+static stdio_bitio_buffer textOutBuf;
+static u_long headLen;
+
+static novel_hash_table nht[2];
+
+int blk_start[2][33], blk_end[2][33];
+
+// 0 = non-word, 1 = word
+unsigned char whichWordType = 0;
+
+CompressTextInfoMap textInfoMap;
+
+
+static char_pool *new_pool (char_pool * pool) {
+  char_pool *p = (char_pool *) Xmalloc (sizeof (char_pool));
+  if (!p) FatalError (1, "Unable to allocate memory for pool");
+  
+  if (pool) pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+int init_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  char path[512];
+
+  // load the compression dictionary
+  if (LoadCompressionDictionary (make_name (fileName, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  // create the compressed text file and attach it to the text buffer
+  if (!(text = create_file (fileName, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))
+    return COMPERROR;
+  textOutBuf.attachFile (text);
+  textOutBuf.encodeStart ();
+  
+  // write out the compressed text header
+  memset (&cth, '\0', sizeof (cth));
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  // the length of the compressed header is the size of the
+  // magic number plus the size of the compressed text header
+  headLen = sizeof (u_long) + sizeof (cth);
+
+  // start off with non-word
+  whichWordType = 0;
+
+  // clear the text pointers
+  textInfoMap.erase (textInfoMap.begin(), textInfoMap.end());
+  
+  // set up the hash table of novel words
+  int i;
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; ++i)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	memset (nht[i].HashTable, '\0',
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		++num;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+static int compress_text (bitio_buffer &buffer, const u_char *s_in,
+			  int l_in, unsigned char &which,
+			  unsigned long &numWords) {
+  const u_char *end = s_in + l_in - 1;
+
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    int res;
+    
+    if (which) ++numWords;
+
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+
+    /* Search the hash table for Word */
+    if (ht[which]) {
+      register unsigned long hashval, step;
+      register int tsize = ht[which]->size;
+      register u_char **wptr;
+      HASH (hashval, step, Word, tsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	wptr = ht[which]->table[hashval];
+	if (wptr == NULL) {
+	  res = COMPERROR;
+	  break;
+	}
+
+	/* Compare the words */
+	s1 = Word;
+	s2 = *wptr;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval += step;
+	  if (hashval >= (unsigned long)tsize) hashval -= tsize;
+	  
+	} else {
+	  res = ht[which]->table[hashval] - ht[which]->words;
+	  break;
+	}
+      }
+    }
+    else
+      res = COMPERROR;
+    /* Check that the word was found in the dictionary */
+    if (res == COMPERROR)
+      {
+	if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	  {
+	    Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	    return (COMPERROR);
+	  }
+	if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	  {
+	    u_long i;
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    buffer.huff_encode (Word[0], lens_codes[which], lens_huff[which].clens, NULL);
+	    for (i = 0; i < Word[0]; ++i)
+	      buffer.huff_encode (Word[i + 1], char_codes[which],
+				  char_huff[which].clens, NULL);
+	  }
+	if (cdh.dict_type == MG_SEED_DICTIONARY)
+	  {
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    switch (cdh.novel_method)
+	      {
+	      case MG_NOVEL_HUFFMAN_CHARS:
+		{
+		  u_long i;
+		  buffer.huff_encode (Word[0], lens_codes[which],
+				      lens_huff[which].clens, NULL);
+		  for (i = 0; i < Word[0]; ++i)
+		    buffer.huff_encode (Word[i + 1], char_codes[which],
+					char_huff[which].clens, NULL);
+		}
+		break;
+	      case MG_NOVEL_DELTA:
+	      case MG_NOVEL_HYBRID:
+		{
+		  register unsigned long hashval, step;
+		  register novel_hash_table *h = &nht[which];
+		  register int hsize = h->HashSize;
+		  register novel_hash_rec *ent;
+		  HASH (hashval, step, Word, hsize);
+		  for (;;)
+		    {
+		      register u_char *s1, *s2;
+		      register int len;
+		      ent = h->HashTable + hashval;
+		      if (!ent->word)
+			{
+			  int len = *Word + 1;
+			  if ((unsigned)len > h->pool->left)
+			    h->pool = new_pool (h->pool);
+			  ent->word = h->pool->ptr;
+			  ent->ordinal_num = h->next_num++;
+			  memcpy (h->pool->ptr, Word, len);
+			  h->pool->ptr += len;
+			  h->pool->left -= len;
+			  ++h->HashUsed;
+			  break;
+			}
+		      /* Compare the words */
+		      s1 = Word;
+		      s2 = ent->word;
+		      len = *s1 + 1;
+		      for (; len; --len)
+			if (*s1++ != *s2++)
+			  break;
+		      
+		      if (!len)
+			break;
+		      
+		      hashval = (hashval + step);
+		      if (hashval >= (unsigned)hsize) hashval -= hsize;
+		    }
+		  
+		  switch (cdh.novel_method) {
+		  case MG_NOVEL_DELTA:
+		    buffer.delta_encode (ent->ordinal_num, NULL);
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    int k = 0;
+		    int j = ent->ordinal_num - 1;
+		    while (j > blk_end[which][k])
+		      ++k;
+		    assert (j - blk_start[which][k] + 1 >= 1 &&
+			    j - blk_start[which][k] + 1 <=
+			    blk_end[which][k] - blk_start[which][k] + 1);
+		    
+		    buffer.gamma_encode (k + 1, NULL);
+		    buffer.binary_encode (j - blk_start[which][k] + 1,
+					  blk_end[which][k] - blk_start[which][k] + 1,
+					  NULL);
+		    break;
+		  }
+		  
+		  if (h->HashUsed >= h->HashSize >> 1)
+		    {
+		      novel_hash_rec *ht;
+		      unsigned long size;
+		      unsigned long i;
+		      size = prime (h->HashSize * 2);
+		      if (!(ht = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * size)))
+			{
+			  Message ("Unable to allocate memory for table");
+			  return (COMPERROR);
+			}
+		      memset (ht, '\0', sizeof (novel_hash_rec) * size);
+		      
+		      for (i = 0; i < h->HashSize; ++i)
+			if (h->HashTable[i].word)
+			  {
+			    register u_char *wptr;
+			    register unsigned long hashval, step;
+			    
+			    wptr = h->HashTable[i].word;
+			    HASH (hashval, step, wptr, size);
+			    wptr = (ht + hashval)->word;
+			    while (wptr)
+			      {
+				hashval += step;
+				if (hashval >= size)
+				  hashval -= size;
+				wptr = (ht + hashval)->word;
+			      }
+			    ht[hashval] = h->HashTable[i];
+			  }
+		      Xfree (h->HashTable);
+		      h->HashTable = ht;
+		      h->HashSize = size;
+		    }
+		}
+		break;
+	      }
+	  }
+      }
+    else
+      {
+	buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+      }
+  }
+  
+  return COMPALLOK;
+}
+
+
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc) {
+  // we need to remember where the last text element ends
+  unsigned char endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+  unsigned long endPos = textOutBuf.GetBytesWritten () + headLen;
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // remember current place in compressed text
+    unsigned char startWhich = whichWordType;
+    unsigned char startBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    unsigned long startPos = textOutBuf.GetBytesWritten () + headLen;
+
+    // compress the text
+    if (compress_text (textOutBuf,
+		       &(here->text[0]),
+		       (*here).text.size(),
+		       whichWordType,
+		       cth.num_of_words) != COMPALLOK)
+      return COMPERROR;
+
+    endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    endPos = textOutBuf.GetBytesWritten () + headLen;
+    
+    // update the pointers if this is a level tag
+    if (tagInfo.levelTags.find ((*here).tagName) != tagInfo.levelTags.end()) {
+      if ((*here).elType == OpenTagE) {
+	// set the start of the document
+	textInfoMap[(*here).tagName].SetStart (startPos, startBit, startWhich);
+	
+      } else if ((*here).elType == CloseTagE) {
+	// set the end of the document
+	textInfoMap[(*here).tagName].SetEnd (endPos, endBit);
+	
+      } else {
+	// TextE, nothing needs to be done
+      }
+    }
+    
+    cth.num_of_bytes += (*here).text.size();
+    ++here;
+  }
+
+  // close off any unclosed tags
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    if ((*tiHere).second.inDoc) (*tiHere).second.SetEnd (endPos, endBit);
+    ++tiHere;
+  }
+  
+  // we've processed one more document
+  ++cth.num_of_docs;
+
+  return COMPALLOK;
+}
+
+
+static int write_aux_dict (char *fileName) {
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (fileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))
+    return COMPERROR;
+  
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+      
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+      
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+      
+      fwrite (&afh, sizeof (afh), 1, aux);
+      
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+static void estimate_compressed_aux_dict (void) {
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      memset (chars, '\0', sizeof (chars));
+      memset (lens, '\0', sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      ++lens[len];
+	      total_uncomp += len + 4;
+	      for (; len; --len)
+		++chars[*buf++];
+	    }
+	}
+      for (j = 0; j < 256; ++j)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; ++j)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+      
+      aux_compressed += (long unsigned int) ((Calculate_Huffman_Size (16, flens, lens) +
+					      Calculate_Huffman_Size (256, fchars, chars)) / 8);
+      
+    }
+}
+
+
+
+static bool WriteTextIdx (char *fileName, FTextLevel &textLevels) {
+  // create the text index
+  FILE *textIdxFile;
+  if (!(textIdxFile = create_file (fileName, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    // remember the level information
+    TextLevelInfo levelInfo;
+    levelInfo.levelTag = (*tiHere).first;
+    levelInfo.textIdxPtr = ftell (textIdxFile);
+    levelInfo.numEntries = (*tiHere).second.docPtrs.size();
+    textLevels.levelInfo[levelInfo.levelTag] = levelInfo;
+
+    // write out the index array for this level
+    if (!WriteTextIdxArray (textIdxFile, (*tiHere).second.docPtrs))
+      return false;
+    
+    ++tiHere;
+  }
+
+  // close the file
+  fclose (textIdxFile);
+
+  return true;
+}
+
+
+bool WriteTextLevels (char *fileName, const FTextLevel &textLevels) {
+  //  cout << textLevels;
+  
+  // create the text index
+  FILE *textLevelFile;
+  if (!(textLevelFile = create_file (fileName, TEXT_LEVEL_SUFFIX, "w+b",
+				     MAGIC_TEXT_LEVELS, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  if (!textLevels.Write (textLevelFile)) return false;
+  
+  // close the file
+  fclose (textLevelFile);
+
+  return true;
+}
+
+
+int done_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  // write out any remaining bits
+  textOutBuf.encodeDone();
+
+  // write out the compressed text header (with updated statistics)
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 || !cth.Write (text))
+    return COMPERROR;
+  fclose (text);
+
+  // write out the text index file and get the level information
+  FTextLevel textLevels;
+  if (!WriteTextIdx (fileName, textLevels)) return COMPERROR;
+  
+  // write out the text level information
+  if (!WriteTextLevels (fileName, textLevels)) return COMPERROR;
+
+  // write out the auxiliary dictionary
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID)) {
+    if (write_aux_dict (fileName) == COMPERROR) return COMPERROR;
+    estimate_compressed_aux_dict ();
+    
+  } else {
+    unlink (make_name (fileName, TEXT_DICT_AUX_SUFFIX, NULL));
+  }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/indexers/mgpp/text/words.cpp
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/words.cpp	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/words.cpp	(revision 21112)
@@ -0,0 +1,122 @@
+/**************************************************************************
+ *
+ * words.cpp -- Functions for parsing out words from the source text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* It determines whether a given place in a UTF-8 encoded Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skippend. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+ 
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word) {
+  word.erase (word.begin(), word.end());
+
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      word.push_back (*textHere++); ++length;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
+
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord) {
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      mgWord[++length] = *textHere++;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+
+  mgWord[0] = length;
+  
+  return textHere;
+}
+
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd) {
+  register int charlength = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char(textHere, textEnd, &c);
+  
+  while (charlength > 0 && !is_unicode_letdig(c)) {
+    textHere += charlength;
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
Index: /main/tags/3.01/indexers/mgpp/text/words.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/text/words.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/text/words.h	(revision 21112)
@@ -0,0 +1,164 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "unitool.h"
+#include "UCArray.h"
+
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+#define MAXNUMERIC	4
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded Unicode string  is a unicode space. */
+
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with begining unicode spaces skippend. */
+
+#ifdef __cplusplus
+}
+#endif
+
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word);
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord);
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd);
+
+
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
Index: /main/tags/3.01/indexers/mgpp/win32cfg.h
===================================================================
--- /main/tags/3.01/indexers/mgpp/win32cfg.h	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/win32cfg.h	(revision 21112)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /main/tags/3.01/indexers/mgpp/winMake.bat
===================================================================
--- /main/tags/3.01/indexers/mgpp/winMake.bat	(revision 21112)
+++ /main/tags/3.01/indexers/mgpp/winMake.bat	(revision 21112)
@@ -0,0 +1,41 @@
+@echo off
+
+set MAKE=nmake
+set MAKE_OPTIONS=/f
+
+if ""%1"" == """" goto all
+if ""%1"" == ""compile"" goto go
+if ""%1"" == ""install"" goto go
+if ""%1"" == ""clean"" goto go
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:go
+	if ""%2"" == ""javaonly"" goto java
+
+:all
+	cd lib
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd text
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd java\org\greenstone\mgpp
+	call winMake.bat %1
+	cd ..\..\..\..
+
+	cd jni
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+:java
+	cd java\org\greenstone\mgpp
+	call winMake.bat %1
+	cd ..\..\..\..
+	goto done
+
+:done
Index: /main/tags/3.01/mgpp/COPYING
===================================================================
--- /main/tags/3.01/mgpp/COPYING	(revision 21112)
+++ /main/tags/3.01/mgpp/COPYING	(revision 21112)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /main/tags/3.01/mgpp/Makefile.in
===================================================================
--- /main/tags/3.01/mgpp/Makefile.in	(revision 21112)
+++ /main/tags/3.01/mgpp/Makefile.in	(revision 21112)
@@ -0,0 +1,111 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CPP = @CXX@
+CFLAGS = @CFLAGS@ -ansi
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib text java/org/greenstone/mgpp jni
+
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall:
+#config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+distclean: distclean-recursive distclean-local
+	rm config.status
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive \
+maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile config.cache config.h config.log stamp-h
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in
+
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
+
Index: /main/tags/3.01/mgpp/acconfig.h
===================================================================
--- /main/tags/3.01/mgpp/acconfig.h	(revision 21112)
+++ /main/tags/3.01/mgpp/acconfig.h	(revision 21112)
@@ -0,0 +1,41 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
Index: /main/tags/3.01/mgpp/aclocal.m4
===================================================================
--- /main/tags/3.01/mgpp/aclocal.m4	(revision 21112)
+++ /main/tags/3.01/mgpp/aclocal.m4	(revision 21112)
@@ -0,0 +1,224 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  AC_LIBOBJ([regex])	
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , 
+   [AC_CHECK_SIZEOF(unsigned char *, unsigned char *)  
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      AC_MSG_WARN(I'm forcing you to use regex because I can't 
+        find a local rx library and the one included with this 
+        distribution doesn't work on 64-bit machines like yours)
+      AC_LIBOBJ([regex])
+    else
+      AC_LIBOBJ([rx])
+    fi]
+  )
+fi
+])
+
+
+## --------------------------------------- ##
+## Check if --with-gnu-readline was given. ##
+## --------------------------------------- ##
+
+AC_DEFUN(fp_WITH_GNU_READLINE,
+[AC_MSG_CHECKING(whether GNU readline requested)
+  AC_ARG_WITH(gnu_readline,
+    [  --with-gnu-readline     compile with GNU readline support],
+    [if test "$withval" = yes; then
+      AC_MSG_RESULT(yes)
+      ac_with_gnu_readline=1
+    else
+      AC_MSG_RESULT(no)
+    fi],
+    AC_MSG_RESULT(no))
+
+  if test -n "$ac_with_gnu_readline"; then
+    AC_CHECK_HEADER(readline/readline.h, ac_mg_readline_header_found=1,
+	AC_MSG_WARN(Can't find GNU readline headers; configuring without \
+GNU readline support))
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      AC_CHECK_LIB(readline, main, 
+        [ac_mg_readline_lib_found=1
+         AC_DEFINE(WITH_GNU_READLINE)
+         LIBS="$LIBS -lreadline"],
+        AC_MSG_WARN(Can't find GNU readline library; configuring without \
+GNU readline support))
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to 
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	AC_CHECK_LIB(curses, main,
+	  LIBS="$LIBS -lcurses")
+      fi
+    fi
+  fi
+])	
+
+
+
+#AC_DEFUN(fp_WITH_GNU_READLINE,
+#[AC_MSG_CHECKING(whether GNU readline requested)
+#  AC_ARG_WITH(gnu-readline,
+#    [  --with-gnu-readline   compile with GNU readline support],
+#    [if test "$withval" = yes; then
+#      AC_MSG_RESULT(yes)
+#      ac_with_gnu_readline=1
+#    else
+#      AC_MSG_RESULT(no)
+#    fi],
+#    AC_MSG_RESULT(no))
+#  if test -n "$ac_with_gnu_readline"; then
+#    AC_CHECK_LIB(readline, main, 
+#      AC_CHECK_HEADER(readline/readline.h,
+#	[AC_DEFINE(WITH_GNU_READLINE)
+## curses required on some systems (DEC OSF/1), but not on others (GNU/Linux,
+##  Solaris 2.6).  Forcing its inclusion is unsophisticated; if it causes 
+##  any actual problems on any UNIX platforms, let the maintainer know.
+#	LIBS="$LIBS -lreadline -lcurses"],
+#        AC_MSG_WARN(Can't find GNU readline headers: configuring \
+#without GNU readline support)
+#      ),
+#      AC_MSG_WARN(Can't find GNU readline library: configuring \
+#without GNU readline support)
+#    )
+#  fi 
+#])
Index: /main/tags/3.01/mgpp/config.guess
===================================================================
--- /main/tags/3.01/mgpp/config.guess	(revision 21112)
+++ /main/tags/3.01/mgpp/config.guess	(revision 21112)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /main/tags/3.01/mgpp/config.h.in
===================================================================
--- /main/tags/3.01/mgpp/config.h.in	(revision 21112)
+++ /main/tags/3.01/mgpp/config.h.in	(revision 21112)
@@ -0,0 +1,209 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+#ifndef _MGPP_CONFIG_H
+#define _MGPP_CONFIG_H
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Define to 1 if GNU readline should be used. */
+#undef WITH_GNU_READLINE
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define to 1 to override the automatic selection of "long long" */
+#undef DISABLE_LONG_LONG
+
+/* The number of bytes in a unsigned char *.  */
+#undef SIZEOF_UNSIGNED_CHAR_P
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strerror function.  */
+#undef HAVE_STRERROR
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <crypt.h> header file.  */
+#undef HAVE_CRYPT_H
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the crypt library (-lcrypt).  */
+#undef HAVE_LIBCRYPT
+
+/* Define if you have the g library (-lg).  */
+#undef HAVE_LIBG
+
+/* Define if you have the gdbm library (-lgdbm).  */
+#undef HAVE_LIBGDBM
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if you have the stdc++ library (-lstdc++).  */
+#undef HAVE_LIBSTDC__
+
+/* Define if you have gettext (eg glibc) */
+#undef ENABLE_NLS
+
+/* Define if big endian */
+#undef WORDS_BIGENDIAN
+
+#endif /* _MGPP_CONFIG_H */
Index: /main/tags/3.01/mgpp/config.sub
===================================================================
--- /main/tags/3.01/mgpp/config.sub	(revision 21112)
+++ /main/tags/3.01/mgpp/config.sub	(revision 21112)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /main/tags/3.01/mgpp/configure
===================================================================
--- /main/tags/3.01/mgpp/configure	(revision 21112)
+++ /main/tags/3.01/mgpp/configure	(revision 21112)
@@ -0,0 +1,8822 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.59.
+#
+# Copyright (C) 2003 Free Software Foundation, Inc.
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+
+# Name of the host.
+# hostname on some systems (SVR3.2, Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+exec 6>&1
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_config_libobj_dir=.
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+
+# Maximum number of lines to put in a shell here document.
+# This variable seems obsolete.  It should probably be removed, and
+# only ac_max_sed_lines should be used.
+: ${ac_max_here_lines=38}
+
+# Identity of this package.
+PACKAGE_NAME=
+PACKAGE_TARNAME=
+PACKAGE_VERSION=
+PACKAGE_STRING=
+PACKAGE_BUGREPORT=
+
+ac_unique_file="text/MGQuery.cpp"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stdio.h>
+#if HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <stddef.h>
+#else
+# if HAVE_STDLIB_H
+#  include <stdlib.h>
+# endif
+#endif
+#if HAVE_STRING_H
+# if !STDC_HEADERS && HAVE_MEMORY_H
+#  include <memory.h>
+# endif
+# include <string.h>
+#endif
+#if HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#if HAVE_INTTYPES_H
+# include <inttypes.h>
+#else
+# if HAVE_STDINT_H
+#  include <stdint.h>
+# endif
+#endif
+#if HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS PACKAGE VERSION CXX CXXFLAGS LDFLAGS CPPFLAGS ac_ct_CXX EXEEXT OBJEXT AWK YACC CC CFLAGS ac_ct_CC INSTALL_PROGRAM INSTALL_SCRIPT INSTALL_DATA LN_S SET_MAKE RANLIB ac_ct_RANLIB CPP EGREP U ANSI2KNR ALLOCA LIBOBJS JNIINC JNISUFFIX JNIFLAGS LTLIBOBJS'
+ac_subst_files=''
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+ac_prev=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case $ac_option in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    eval "enable_$ac_feature=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_feature" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid feature name: $ac_feature" >&2
+   { (exit 1); exit 1; }; }
+    ac_feature=`echo $ac_feature | sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_$ac_feature='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_package=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case $ac_option in
+      *=*) ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_$ac_package='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_package" : ".*[^-_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid package name: $ac_package" >&2
+   { (exit 1); exit 1; }; }
+    ac_package=`echo $ac_package | sed 's/-/_/g'`
+    eval "with_$ac_package=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) { echo "$as_me: error: unrecognized option: $ac_option
+Try \`$0 --help' for more information." >&2
+   { (exit 1); exit 1; }; }
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_envvar" : ".*[^_$as_cr_alnum]" >/dev/null &&
+      { echo "$as_me: error: invalid variable name: $ac_envvar" >&2
+   { (exit 1); exit 1; }; }
+    ac_optarg=`echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"`
+    eval "$ac_envvar='$ac_optarg'"
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    echo "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      echo "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  { echo "$as_me: error: missing argument to $ac_option" >&2
+   { (exit 1); exit 1; }; }
+fi
+
+# Be sure to have absolute paths.
+for ac_var in exec_prefix prefix
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* | NONE | '' ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# Be sure to have absolute paths.
+for ac_var in bindir sbindir libexecdir datadir sysconfdir sharedstatedir \
+	      localstatedir libdir includedir oldincludedir infodir mandir
+do
+  eval ac_val=$`echo $ac_var`
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* ) ;;
+    *)  { echo "$as_me: error: expected an absolute directory name for --$ac_var: $ac_val" >&2
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+    echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host.
+    If a cross compiler is detected then cross compile mode will be used." >&2
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_confdir=`(dirname "$0") 2>/dev/null ||
+$as_expr X"$0" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$0" : 'X\(//\)[^/]' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$0" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $ac_confdir or .." >&2
+   { (exit 1); exit 1; }; }
+  else
+    { echo "$as_me: error: cannot find sources ($ac_unique_file) in $srcdir" >&2
+   { (exit 1); exit 1; }; }
+  fi
+fi
+(cd $srcdir && test -r ./$ac_unique_file) 2>/dev/null ||
+  { echo "$as_me: error: sources are in $srcdir, but \`cd $srcdir' does not work" >&2
+   { (exit 1); exit 1; }; }
+srcdir=`echo "$srcdir" | sed 's%\([^\\/]\)[\\/]*$%\1%'`
+ac_env_build_alias_set=${build_alias+set}
+ac_env_build_alias_value=$build_alias
+ac_cv_env_build_alias_set=${build_alias+set}
+ac_cv_env_build_alias_value=$build_alias
+ac_env_host_alias_set=${host_alias+set}
+ac_env_host_alias_value=$host_alias
+ac_cv_env_host_alias_set=${host_alias+set}
+ac_cv_env_host_alias_value=$host_alias
+ac_env_target_alias_set=${target_alias+set}
+ac_env_target_alias_value=$target_alias
+ac_cv_env_target_alias_set=${target_alias+set}
+ac_cv_env_target_alias_value=$target_alias
+ac_env_CXX_set=${CXX+set}
+ac_env_CXX_value=$CXX
+ac_cv_env_CXX_set=${CXX+set}
+ac_cv_env_CXX_value=$CXX
+ac_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_env_CXXFLAGS_value=$CXXFLAGS
+ac_cv_env_CXXFLAGS_set=${CXXFLAGS+set}
+ac_cv_env_CXXFLAGS_value=$CXXFLAGS
+ac_env_LDFLAGS_set=${LDFLAGS+set}
+ac_env_LDFLAGS_value=$LDFLAGS
+ac_cv_env_LDFLAGS_set=${LDFLAGS+set}
+ac_cv_env_LDFLAGS_value=$LDFLAGS
+ac_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_env_CPPFLAGS_value=$CPPFLAGS
+ac_cv_env_CPPFLAGS_set=${CPPFLAGS+set}
+ac_cv_env_CPPFLAGS_value=$CPPFLAGS
+ac_env_CC_set=${CC+set}
+ac_env_CC_value=$CC
+ac_cv_env_CC_set=${CC+set}
+ac_cv_env_CC_value=$CC
+ac_env_CFLAGS_set=${CFLAGS+set}
+ac_env_CFLAGS_value=$CFLAGS
+ac_cv_env_CFLAGS_set=${CFLAGS+set}
+ac_cv_env_CFLAGS_value=$CFLAGS
+ac_env_CPP_set=${CPP+set}
+ac_env_CPP_value=$CPP
+ac_cv_env_CPP_set=${CPP+set}
+ac_cv_env_CPP_value=$CPP
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures this package to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+_ACEOF
+
+  cat <<_ACEOF
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+			  [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+			  [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR           user executables [EPREFIX/bin]
+  --sbindir=DIR          system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR       program executables [EPREFIX/libexec]
+  --datadir=DIR          read-only architecture-independent data [PREFIX/share]
+  --sysconfdir=DIR       read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR   modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR    modifiable single-machine data [PREFIX/var]
+  --libdir=DIR           object code libraries [EPREFIX/lib]
+  --includedir=DIR       C header files [PREFIX/include]
+  --oldincludedir=DIR    C header files for non-gcc [/usr/include]
+  --infodir=DIR          info documentation [PREFIX/info]
+  --mandir=DIR           man documentation [PREFIX/man]
+_ACEOF
+
+  cat <<\_ACEOF
+
+Program names:
+  --program-prefix=PREFIX            prepend PREFIX to installed program names
+  --program-suffix=SUFFIX            append SUFFIX to installed program names
+  --program-transform-name=PROGRAM   run sed PROGRAM on installed program names
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+
+  cat <<\_ACEOF
+
+Optional Features:
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --enable-override-longlong
+                          overide use of long long with GCC
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz
+  --with-regex            use older regex in lieu of GNU rx for matching
+  --with-gnu-readline     compile with GNU readline support
+
+Some influential environment variables:
+  CXX         C++ compiler command
+  CXXFLAGS    C++ compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
+              headers in a nonstandard directory <include dir>
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  CPP         C preprocessor
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+_ACEOF
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  ac_popdir=`pwd`
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d $ac_dir || continue
+    ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+    cd $ac_dir
+    # Check for guested configure; otherwise get Cygnus style configure.
+    if test -f $ac_srcdir/configure.gnu; then
+      echo
+      $SHELL $ac_srcdir/configure.gnu  --help=recursive
+    elif test -f $ac_srcdir/configure; then
+      echo
+      $SHELL $ac_srcdir/configure  --help=recursive
+    elif test -f $ac_srcdir/configure.ac ||
+	   test -f $ac_srcdir/configure.in; then
+      echo
+      $ac_configure --help
+    else
+      echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi
+    cd $ac_popdir
+  done
+fi
+
+test -n "$ac_init_help" && exit 0
+if $ac_init_version; then
+  cat <<\_ACEOF
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+_ACEOF
+  exit 0
+fi
+exec 5>config.log
+cat >&5 <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  $ $0 $@
+
+_ACEOF
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+hostinfo               = `(hostinfo) 2>/dev/null               || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  echo "PATH: $as_dir"
+done
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_sep=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=`echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) ac_configure_args0="$ac_configure_args0 '$ac_arg'" ;;
+    2)
+      ac_configure_args1="$ac_configure_args1 '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      ac_configure_args="$ac_configure_args$ac_sep'$ac_arg'"
+      # Get rid of the leading space.
+      ac_sep=" "
+      ;;
+    esac
+  done
+done
+$as_unset ac_configure_args0 || test "${ac_configure_args0+set}" != set || { ac_configure_args0=; export ac_configure_args0; }
+$as_unset ac_configure_args1 || test "${ac_configure_args1+set}" != set || { ac_configure_args1=; export ac_configure_args1; }
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Be sure not to use single quotes in there, as some shells,
+# such as our DU 5.0 friend, will then `close' the trap.
+trap 'exit_status=$?
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    cat <<\_ASBOX
+## ---------------- ##
+## Cache variables. ##
+## ---------------- ##
+_ASBOX
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+{
+  (set) 2>&1 |
+    case `(ac_space='"'"' '"'"'; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      sed -n \
+	"s/'"'"'/'"'"'\\\\'"'"''"'"'/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='"'"'\\2'"'"'/p"
+      ;;
+    *)
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+}
+    echo
+
+    cat <<\_ASBOX
+## ----------------- ##
+## Output variables. ##
+## ----------------- ##
+_ASBOX
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=$`echo $ac_var`
+      echo "$ac_var='"'"'$ac_val'"'"'"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      cat <<\_ASBOX
+## ------------- ##
+## Output files. ##
+## ------------- ##
+_ASBOX
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=$`echo $ac_var`
+	echo "$ac_var='"'"'$ac_val'"'"'"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      cat <<\_ASBOX
+## ----------- ##
+## confdefs.h. ##
+## ----------- ##
+_ASBOX
+      echo
+      sed "/^$/d" confdefs.h | sort
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      echo "$as_me: caught signal $ac_signal"
+    echo "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core &&
+  rm -rf conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+     ' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; { (exit 1); exit 1; }' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo >confdefs.h
+
+# Predefined preprocessor variables.
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_NAME "$PACKAGE_NAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_TARNAME "$PACKAGE_TARNAME"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_VERSION "$PACKAGE_VERSION"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_STRING "$PACKAGE_STRING"
+_ACEOF
+
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT"
+_ACEOF
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    { echo "$as_me:$LINENO: loading site script $ac_site_file" >&5
+echo "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special
+  # files actually), so we avoid doing that.
+  if test -f "$cache_file"; then
+    { echo "$as_me:$LINENO: loading cache $cache_file" >&5
+echo "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . $cache_file;;
+      *)                      . ./$cache_file;;
+    esac
+  fi
+else
+  { echo "$as_me:$LINENO: creating cache $cache_file" >&5
+echo "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in `(set) 2>&1 |
+	       sed -n 's/^ac_env_\([a-zA-Z_0-9]*\)_set=.*/\1/p'`; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val="\$ac_cv_env_${ac_var}_value"
+  eval ac_new_val="\$ac_env_${ac_var}_value"
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { echo "$as_me:$LINENO: error: \`$ac_var' was not set in the previous run" >&5
+echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	{ echo "$as_me:$LINENO: error: \`$ac_var' has changed since the previous run:" >&5
+echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	{ echo "$as_me:$LINENO:   former value:  $ac_old_val" >&5
+echo "$as_me:   former value:  $ac_old_val" >&2;}
+	{ echo "$as_me:$LINENO:   current value: $ac_new_val" >&5
+echo "$as_me:   current value: $ac_new_val" >&2;}
+	ac_cache_corrupted=:
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?\"\']*)
+      ac_arg=$ac_var=`echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { echo "$as_me:$LINENO: error: changes in the environment can compromise the build" >&5
+echo "$as_me: error: changes in the environment can compromise the build" >&2;}
+  { { echo "$as_me:$LINENO: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&5
+echo "$as_me: error: run \`make distclean' and/or \`rm $cache_file' and start over" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+          ac_config_headers="$ac_config_headers config.h"
+
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,$program_prefix,;$program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$,$program_suffix,;$program_transform_name"
+# Double any \ or $.  echo might interpret backslashes.
+# By default was `s,x,x', remove it if useless.
+cat <<\_ACEOF >conftest.sed
+s/[\\$]/&&/g;s/;s,x,x,$//
+_ACEOF
+program_transform_name=`echo $program_transform_name | sed -f conftest.sed`
+rm conftest.sed
+
+
+PACKAGE=mgpp
+VERSION=1.3
+
+LDFLAGS=
+
+cat >>confdefs.h <<_ACEOF
+#define PACKAGE "$PACKAGE"
+_ACEOF
+
+cat >>confdefs.h <<_ACEOF
+#define VERSION "$VERSION"
+_ACEOF
+
+
+
+
+ac_ext=cc
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CXX="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CXX=$ac_cv_prog_CXX
+if test -n "$CXX"; then
+  echo "$as_me:$LINENO: result: $CXX" >&5
+echo "${ECHO_T}$CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CXX" && break
+  done
+fi
+if test -z "$CXX"; then
+  ac_ct_CXX=$CXX
+  for ac_prog in $CCC g++ c++ gpp aCC CC cxx cc++ cl FCC KCC RCC xlC_r xlC
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CXX+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CXX"; then
+  ac_cv_prog_ac_ct_CXX="$ac_ct_CXX" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CXX="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CXX=$ac_cv_prog_ac_ct_CXX
+if test -n "$ac_ct_CXX"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CXX" >&5
+echo "${ECHO_T}$ac_ct_CXX" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CXX" && break
+done
+test -n "$ac_ct_CXX" || ac_ct_CXX="g++"
+
+  CXX=$ac_ct_CXX
+fi
+
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C++ compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+echo "$as_me:$LINENO: checking for C++ compiler default output file name" >&5
+echo $ECHO_N "checking for C++ compiler default output file name... $ECHO_C" >&6
+ac_link_default=`echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+if { (eval echo "$as_me:$LINENO: \"$ac_link_default\"") >&5
+  (eval $ac_link_default) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # Find the output, starting from the most likely.  This scheme is
+# not robust to junk in `.', hence go to wildcards (a.*) only as a last
+# resort.
+
+# Be careful to initialize this variable, since it used to be cached.
+# Otherwise an old cache value of `no' led to `EXEEXT = no' in a Makefile.
+ac_cv_exeext=
+# b.out is created by i960 compilers.
+for ac_file in a_out.exe a.exe conftest.exe a.out conftest a.* conftest.* b.out
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj )
+	;;
+    conftest.$ac_ext )
+	# This is the source file.
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	# FIXME: I believe we export ac_cv_exeext for Libtool,
+	# but it would be cool to find out if it's true.  Does anybody
+	# maintain Libtool? --akim.
+	export ac_cv_exeext
+	break;;
+    * )
+	break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&5
+echo "$as_me: error: C++ compiler cannot create executables
+See \`config.log' for more details." >&2;}
+   { (exit 77); exit 77; }; }
+fi
+
+ac_exeext=$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_file" >&5
+echo "${ECHO_T}$ac_file" >&6
+
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether the C++ compiler works" >&5
+echo $ECHO_N "checking whether the C++ compiler works... $ECHO_C" >&6
+# FIXME: These cross compiler hacks should be removed for Autoconf 3.0
+# If not cross compiling, check that we can run a simple program.
+if test "$cross_compiling" != yes; then
+  if { ac_try='./$ac_file'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { echo "$as_me:$LINENO: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run C++ compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+    fi
+  fi
+fi
+echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+
+rm -f a.out a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+# Check the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+echo "$as_me:$LINENO: checking whether we are cross compiling" >&5
+echo $ECHO_N "checking whether we are cross compiling... $ECHO_C" >&6
+echo "$as_me:$LINENO: result: $cross_compiling" >&5
+echo "${ECHO_T}$cross_compiling" >&6
+
+echo "$as_me:$LINENO: checking for suffix of executables" >&5
+echo $ECHO_N "checking for suffix of executables... $ECHO_C" >&6
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  export ac_cv_exeext
+	  break;;
+    * ) break;;
+  esac
+done
+else
+  { { echo "$as_me:$LINENO: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest$ac_cv_exeext
+echo "$as_me:$LINENO: result: $ac_cv_exeext" >&5
+echo "${ECHO_T}$ac_cv_exeext" >&6
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+echo "$as_me:$LINENO: checking for suffix of object files" >&5
+echo $ECHO_N "checking for suffix of object files... $ECHO_C" >&6
+if test "${ac_cv_objext+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; then
+  for ac_file in `(ls conftest.o conftest.obj; ls conftest.*) 2>/dev/null`; do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { echo "$as_me:$LINENO: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute suffix of object files: cannot compile
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_objext" >&5
+echo "${ECHO_T}$ac_cv_objext" >&6
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+echo "$as_me:$LINENO: checking whether we are using the GNU C++ compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C++ compiler... $ECHO_C" >&6
+if test "${ac_cv_cxx_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_cxx_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_cxx_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_cxx_compiler_gnu" >&6
+GXX=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CXXFLAGS=${CXXFLAGS+set}
+ac_save_CXXFLAGS=$CXXFLAGS
+CXXFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CXX accepts -g" >&5
+echo $ECHO_N "checking whether $CXX accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cxx_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cxx_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cxx_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cxx_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cxx_g" >&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS=$ac_save_CXXFLAGS
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_cxx_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+for ac_prog in gawk mawk nawk awk
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_AWK+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_AWK="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+AWK=$ac_cv_prog_AWK
+if test -n "$AWK"; then
+  echo "$as_me:$LINENO: result: $AWK" >&5
+echo "${ECHO_T}$AWK" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_YACC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_YACC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+YACC=$ac_cv_prog_YACC
+if test -n "$YACC"; then
+  echo "$as_me:$LINENO: result: $YACC" >&5
+echo "${ECHO_T}$YACC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  CC=$ac_ct_CC
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  echo "$as_me:$LINENO: result: $CC" >&5
+echo "${ECHO_T}$CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_CC+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  echo "$as_me:$LINENO: result: $ac_ct_CC" >&5
+echo "${ECHO_T}$ac_ct_CC" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  test -n "$ac_ct_CC" && break
+done
+
+  CC=$ac_ct_CC
+fi
+
+fi
+
+
+test -z "$CC" && { { echo "$as_me:$LINENO: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&5
+echo "$as_me: error: no acceptable C compiler found in \$PATH
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+
+# Provide some information about the compiler.
+echo "$as_me:$LINENO:" \
+     "checking for C compiler version" >&5
+ac_compiler=`set X $ac_compile; echo $2`
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5
+  (eval $ac_compiler --version </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -v </dev/null >&5\"") >&5
+  (eval $ac_compiler -v </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+{ (eval echo "$as_me:$LINENO: \"$ac_compiler -V </dev/null >&5\"") >&5
+  (eval $ac_compiler -V </dev/null >&5) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }
+
+echo "$as_me:$LINENO: checking whether we are using the GNU C compiler" >&5
+echo $ECHO_N "checking whether we are using the GNU C compiler... $ECHO_C" >&6
+if test "${ac_cv_c_compiler_gnu+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_compiler_gnu=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_compiler_gnu=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_compiler_gnu" >&5
+echo "${ECHO_T}$ac_cv_c_compiler_gnu" >&6
+GCC=`test $ac_compiler_gnu = yes && echo yes`
+ac_test_CFLAGS=${CFLAGS+set}
+ac_save_CFLAGS=$CFLAGS
+CFLAGS="-g"
+echo "$as_me:$LINENO: checking whether $CC accepts -g" >&5
+echo $ECHO_N "checking whether $CC accepts -g... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_g+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_g=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_prog_cc_g=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_g" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_g" >&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+echo "$as_me:$LINENO: checking for $CC option to accept ANSI C" >&5
+echo $ECHO_N "checking for $CC option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CC=$CC
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdarg.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+/* Most of the following tests are stolen from RCS 5.7's src/conf.sh.  */
+struct buf { int x; };
+FILE * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not '\xHH' hex character constants.
+   These don't provoke an error unfortunately, instead are silently treated
+   as 'x'.  The following induces an error, until -std1 is added to get
+   proper ANSI mode.  Curiously '\x00'!='x' always comes out true, for an
+   array size at least.  It's necessary to write '\x00'==0 to get something
+   that's true only with -std1.  */
+int osf4_cc_array ['\x00' == 0 ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int);
+int argc;
+char **argv;
+int
+main ()
+{
+return f (e, argv, 0) != argv[0]  ||  f (e, argv, 1) != argv[1];
+  ;
+  return 0;
+}
+_ACEOF
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX 10.20 and later	-Ae
+# HP-UX older versions	-Aa -D_HPUX_SOURCE
+# SVR4			-Xc -D__EXTENSIONS__
+for ac_arg in "" -qlanglvl=ansi -std1 -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc=$ac_arg
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext
+done
+rm -f conftest.$ac_ext conftest.$ac_objext
+CC=$ac_save_CC
+
+fi
+
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno)
+    echo "$as_me:$LINENO: result: none needed" >&5
+echo "${ECHO_T}none needed" >&6 ;;
+  *)
+    echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+    CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+# Some people use a C++ compiler to compile C.  Since we use `exit',
+# in C++ we need to declare it.  In case someone uses the same compiler
+# for both compiling C and C++ we need to have the C++ compiler decide
+# the declaration of exit, since it's the most demanding environment.
+cat >conftest.$ac_ext <<_ACEOF
+#ifndef __cplusplus
+  choke me
+#endif
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  for ac_declaration in \
+   '' \
+   'extern "C" void std::exit (int) throw (); using std::exit;' \
+   'extern "C" void std::exit (int); using std::exit;' \
+   'extern "C" void exit (int) throw ();' \
+   'extern "C" void exit (int);' \
+   'void exit (int);'
+do
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+#include <stdlib.h>
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+continue
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_declaration
+int
+main ()
+{
+exit (42);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+rm -f conftest*
+if test -n "$ac_declaration"; then
+  echo '#ifdef __cplusplus' >>confdefs.h
+  echo $ac_declaration      >>confdefs.h
+  echo '#endif'             >>confdefs.h
+fi
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  elif test -f $ac_dir/shtool; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/shtool install -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { { echo "$as_me:$LINENO: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&5
+echo "$as_me: error: cannot find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+ac_config_guess="$SHELL $ac_aux_dir/config.guess"
+ac_config_sub="$SHELL $ac_aux_dir/config.sub"
+ac_configure="$SHELL $ac_aux_dir/configure" # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AmigaOS /C/install, which installs bootblocks on floppy discs
+# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# OS/2's system install, which has a completely different semantic
+# ./install, which can be erroneously created by make from ./install.sh.
+echo "$as_me:$LINENO: checking for a BSD-compatible install" >&5
+echo $ECHO_N "checking for a BSD-compatible install... $ECHO_C" >&6
+if test -z "$INSTALL"; then
+if test "${ac_cv_path_install+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  # Account for people who put trailing slashes in PATH elements.
+case $as_dir/ in
+  ./ | .// | /cC/* | \
+  /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \
+  ?:\\/os2\\/install\\/* | ?:\\/OS2\\/INSTALL\\/* | \
+  /usr/ucb/* ) ;;
+  *)
+    # OSF1 and SCO ODT 3.0 have their own names for install.
+    # Don't use installbsd from OSF since it installs stuff as root
+    # by default.
+    for ac_prog in ginstall scoinst install; do
+      for ac_exec_ext in '' $ac_executable_extensions; do
+	if $as_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then
+	  if test $ac_prog = install &&
+	    grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    :
+	  elif test $ac_prog = install &&
+	    grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then
+	    # program-specific install script used by HP pwplus--don't use.
+	    :
+	  else
+	    ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c"
+	    break 3
+	  fi
+	fi
+      done
+    done
+    ;;
+esac
+done
+
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL=$ac_cv_path_install
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL=$ac_install_sh
+  fi
+fi
+echo "$as_me:$LINENO: result: $INSTALL" >&5
+echo "${ECHO_T}$INSTALL" >&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo "$as_me:$LINENO: checking whether ln -s works" >&5
+echo $ECHO_N "checking whether ln -s works... $ECHO_C" >&6
+LN_S=$as_ln_s
+if test "$LN_S" = "ln -s"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+else
+  echo "$as_me:$LINENO: result: no, using $LN_S" >&5
+echo "${ECHO_T}no, using $LN_S" >&6
+fi
+
+echo "$as_me:$LINENO: checking whether ${MAKE-make} sets \$(MAKE)" >&5
+echo $ECHO_N "checking whether ${MAKE-make} sets \$(MAKE)... $ECHO_C" >&6
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y,:./+-,___p_,'`
+if eval "test \"\${ac_cv_prog_make_${ac_make}_set+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.make <<\_ACEOF
+all:
+	@echo 'ac_maketemp="$(MAKE)"'
+_ACEOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftest.make 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftest.make
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  SET_MAKE=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  echo "$as_me:$LINENO: result: $RANLIB" >&5
+echo "${ECHO_T}$RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6
+if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if $as_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+
+  test -z "$ac_cv_prog_ac_ct_RANLIB" && ac_cv_prog_ac_ct_RANLIB=":"
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  echo "$as_me:$LINENO: result: $ac_ct_RANLIB" >&5
+echo "${ECHO_T}$ac_ct_RANLIB" >&6
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+
+  RANLIB=$ac_ct_RANLIB
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+echo "$as_me:$LINENO: checking how to run the C preprocessor" >&5
+echo $ECHO_N "checking how to run the C preprocessor... $ECHO_C" >&6
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+  if test "${ac_cv_prog_CPP+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+      # Double quotes because CPP needs to be expanded
+    for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp"
+    do
+      ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  break
+fi
+
+    done
+    ac_cv_prog_CPP=$CPP
+
+fi
+  CPP=$ac_cv_prog_CPP
+else
+  ac_cv_prog_CPP=$CPP
+fi
+echo "$as_me:$LINENO: result: $CPP" >&5
+echo "${ECHO_T}$CPP" >&6
+ac_preproc_ok=false
+for ac_c_preproc_warn_flag in '' yes
+do
+  # Use a header file that comes with gcc, so configuring glibc
+  # with a fresh cross-compiler works.
+  # Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+  # <limits.h> exists even on freestanding compilers.
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp. "Syntax error" is here to catch this case.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+		     Syntax error
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Broken: fails on valid input.
+continue
+fi
+rm -f conftest.err conftest.$ac_ext
+
+  # OK, works on sane cases.  Now check whether non-existent headers
+  # can be detected and how.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ac_nonexistent.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  # Broken: success on invalid input.
+continue
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  # Passes both tests.
+ac_preproc_ok=:
+break
+fi
+rm -f conftest.err conftest.$ac_ext
+
+done
+# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped.
+rm -f conftest.err conftest.$ac_ext
+if $ac_preproc_ok; then
+  :
+else
+  { { echo "$as_me:$LINENO: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&5
+echo "$as_me: error: C preprocessor \"$CPP\" fails sanity check
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+echo "$as_me:$LINENO: checking for egrep" >&5
+echo $ECHO_N "checking for egrep... $ECHO_C" >&6
+if test "${ac_cv_prog_egrep+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if echo a | (grep -E '(a|b)') >/dev/null 2>&1
+    then ac_cv_prog_egrep='grep -E'
+    else ac_cv_prog_egrep='egrep'
+    fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_egrep" >&5
+echo "${ECHO_T}$ac_cv_prog_egrep" >&6
+ EGREP=$ac_cv_prog_egrep
+
+
+
+echo "$as_me:$LINENO: checking for AIX" >&5
+echo $ECHO_N "checking for AIX... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef _AIX
+  yes
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "yes" >/dev/null 2>&1; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+cat >>confdefs.h <<\_ACEOF
+#define _ALL_SOURCE 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+rm -f conftest*
+
+
+echo "$as_me:$LINENO: checking for library containing strerror" >&5
+echo $ECHO_N "checking for library containing strerror... $ECHO_C" >&6
+if test "${ac_cv_search_strerror+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_strerror=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_strerror" = no; then
+  for ac_lib in cposix; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char strerror ();
+int
+main ()
+{
+strerror ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_strerror="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_strerror" >&5
+echo "${ECHO_T}$ac_cv_search_strerror" >&6
+if test "$ac_cv_search_strerror" != no; then
+  test "$ac_cv_search_strerror" = "none required" || LIBS="$ac_cv_search_strerror $LIBS"
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+# On IRIX 5.3, sys/types and inttypes.h are conflicting.
+
+
+
+
+
+
+
+
+
+for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \
+		  inttypes.h stdint.h unistd.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking minix/config.h usability" >&5
+echo $ECHO_N "checking minix/config.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <minix/config.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking minix/config.h presence" >&5
+echo $ECHO_N "checking minix/config.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <minix/config.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: minix/config.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: minix/config.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: minix/config.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: minix/config.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: minix/config.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: minix/config.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: minix/config.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: minix/config.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for minix/config.h" >&5
+echo $ECHO_N "checking for minix/config.h... $ECHO_C" >&6
+if test "${ac_cv_header_minix_config_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_minix_config_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_minix_config_h" >&5
+echo "${ECHO_T}$ac_cv_header_minix_config_h" >&6
+
+fi
+if test $ac_cv_header_minix_config_h = yes; then
+  MINIX=yes
+else
+  MINIX=
+fi
+
+
+if test "$MINIX" = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_SOURCE 1
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _POSIX_1_SOURCE 2
+_ACEOF
+
+
+cat >>confdefs.h <<\_ACEOF
+#define _MINIX 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for ${CC-cc} option to accept ANSI C" >&5
+echo $ECHO_N "checking for ${CC-cc} option to accept ANSI C... $ECHO_C" >&6
+if test "${ac_cv_prog_cc_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif
+
+int
+main ()
+{
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$as_me:$LINENO: result: $ac_cv_prog_cc_stdc" >&5
+echo "${ECHO_T}$ac_cv_prog_cc_stdc" >&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo "$as_me:$LINENO: checking for function prototypes" >&5
+echo $ECHO_N "checking for function prototypes... $ECHO_C" >&6
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define PROTOTYPES 1
+_ACEOF
+
+  U= ANSI2KNR=
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
+if test "${ac_cv_c_const+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+int
+main ()
+{
+/* FIXME: Include the comments suggested by Paul. */
+#ifndef __cplusplus
+  /* Ultrix mips cc rejects this.  */
+  typedef int charset[2];
+  const charset x;
+  /* SunOS 4.1.1 cc rejects this.  */
+  char const *const *ccp;
+  char **p;
+  /* NEC SVR4.0.2 mips cc rejects this.  */
+  struct point {int x, y;};
+  static struct point const zero = {0,0};
+  /* AIX XL C 1.02.0.0 rejects this.
+     It does not let you subtract one const X* pointer from another in
+     an arm of an if-expression whose if-part is not a constant
+     expression */
+  const char *g = "string";
+  ccp = &g + (g ? g-g : 0);
+  /* HPUX 7.0 cc rejects these. */
+  ++ccp;
+  p = (char**) ccp;
+  ccp = (char const *const *) p;
+  { /* SCO 3.2v4 cc rejects this.  */
+    char *t;
+    char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+    *t++ = 0;
+  }
+  { /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+    int x[] = {25, 17};
+    const int *foo = &x[0];
+    ++foo;
+  }
+  { /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+    typedef const int *iptr;
+    iptr p = 0;
+    ++p;
+  }
+  { /* AIX XL C 1.02.0.0 rejects this saying
+       "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+    struct s { int j; const int *ap[3]; };
+    struct s *b; b->j = 5;
+  }
+  { /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+    const int foo = 10;
+  }
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_const=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_const=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_const" >&5
+echo "${ECHO_T}$ac_cv_c_const" >&6
+if test $ac_cv_c_const = no; then
+
+cat >>confdefs.h <<\_ACEOF
+#define const
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for off_t" >&5
+echo $ECHO_N "checking for off_t... $ECHO_C" >&6
+if test "${ac_cv_type_off_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((off_t *) 0)
+  return 0;
+if (sizeof (off_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_off_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_off_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_off_t" >&5
+echo "${ECHO_T}$ac_cv_type_off_t" >&6
+if test $ac_cv_type_off_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define off_t long
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for size_t" >&5
+echo $ECHO_N "checking for size_t... $ECHO_C" >&6
+if test "${ac_cv_type_size_t+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((size_t *) 0)
+  return 0;
+if (sizeof (size_t))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_size_t=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_size_t=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_size_t" >&5
+echo "${ECHO_T}$ac_cv_type_size_t" >&6
+if test $ac_cv_type_size_t = yes; then
+  :
+else
+
+cat >>confdefs.h <<_ACEOF
+#define size_t unsigned
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether time.h and sys/time.h may both be included" >&5
+echo $ECHO_N "checking whether time.h and sys/time.h may both be included... $ECHO_C" >&6
+if test "${ac_cv_header_time+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+
+int
+main ()
+{
+if ((struct tm *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_time=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_time=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_time" >&5
+echo "${ECHO_T}$ac_cv_header_time" >&6
+if test $ac_cv_header_time = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TIME_WITH_SYS_TIME 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking whether struct tm is in sys/time.h or time.h" >&5
+echo $ECHO_N "checking whether struct tm is in sys/time.h or time.h... $ECHO_C" >&6
+if test "${ac_cv_struct_tm+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <time.h>
+
+int
+main ()
+{
+struct tm *tp; tp->tm_sec;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_struct_tm=time.h
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_struct_tm" >&5
+echo "${ECHO_T}$ac_cv_struct_tm" >&6
+if test $ac_cv_struct_tm = sys/time.h; then
+
+cat >>confdefs.h <<\_ACEOF
+#define TM_IN_SYS_TIME 1
+_ACEOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+int
+main ()
+{
+ FILE *rubbish = popen("yes", "r");
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+
+
+echo "$as_me:$LINENO: checking for main in -lg" >&5
+echo $ECHO_N "checking for main in -lg... $ECHO_C" >&6
+if test "${ac_cv_lib_g_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lg  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_g_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_g_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_g_main" >&5
+echo "${ECHO_T}$ac_cv_lib_g_main" >&6
+if test $ac_cv_lib_g_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBG 1
+_ACEOF
+
+  LIBS="-lg $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lm" >&5
+echo $ECHO_N "checking for main in -lm... $ECHO_C" >&6
+if test "${ac_cv_lib_m_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lm  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_m_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_m_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_m_main" >&5
+echo "${ECHO_T}$ac_cv_lib_m_main" >&6
+if test $ac_cv_lib_m_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBM 1
+_ACEOF
+
+  LIBS="-lm $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lstdc++" >&5
+echo $ECHO_N "checking for main in -lstdc++... $ECHO_C" >&6
+if test "${ac_cv_lib_stdcpp_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lstdc++  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_stdcpp_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_stdcpp_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_stdcpp_main" >&5
+echo "${ECHO_T}$ac_cv_lib_stdcpp_main" >&6
+if test $ac_cv_lib_stdcpp_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBSTDC__ 1
+_ACEOF
+
+  LIBS="-lstdc++ $LIBS"
+
+fi
+
+
+echo "$as_me:$LINENO: checking for main in -lcrypt" >&5
+echo $ECHO_N "checking for main in -lcrypt... $ECHO_C" >&6
+if test "${ac_cv_lib_crypt_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcrypt  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_crypt_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_crypt_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_crypt_main" >&5
+echo "${ECHO_T}$ac_cv_lib_crypt_main" >&6
+if test $ac_cv_lib_crypt_main = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBCRYPT 1
+_ACEOF
+
+  LIBS="-lcrypt $LIBS"
+
+fi
+
+
+
+
+
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do
+  as_ac_Header=`echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_hdr that defines DIR" >&5
+echo $ECHO_N "checking for $ac_hdr that defines DIR... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <$ac_hdr>
+
+int
+main ()
+{
+if ((DIR *) 0)
+return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_Header=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_Header=no"
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_hdr" | $as_tr_cpp` 1
+_ACEOF
+
+ac_header_dirent=$ac_hdr; break
+fi
+
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in dir; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+else
+  echo "$as_me:$LINENO: checking for library containing opendir" >&5
+echo $ECHO_N "checking for library containing opendir... $ECHO_C" >&6
+if test "${ac_cv_search_opendir+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_func_search_save_LIBS=$LIBS
+ac_cv_search_opendir=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="none required"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_opendir" = no; then
+  for ac_lib in x; do
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char opendir ();
+int
+main ()
+{
+opendir ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_search_opendir="-l$ac_lib"
+break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+  done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_opendir" >&5
+echo "${ECHO_T}$ac_cv_search_opendir" >&6
+if test "$ac_cv_search_opendir" != no; then
+  test "$ac_cv_search_opendir" = "none required" || LIBS="$ac_cv_search_opendir $LIBS"
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking for ANSI C header files" >&5
+echo $ECHO_N "checking for ANSI C header files... $ECHO_C" >&6
+if test "${ac_cv_header_stdc+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_header_stdc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_header_stdc=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <string.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "memchr" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdlib.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "free" >/dev/null 2>&1; then
+  :
+else
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+  if test "$cross_compiling" = yes; then
+  :
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <ctype.h>
+#if ((' ' & 0x0FF) == 0x020)
+# define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#else
+# define ISLOWER(c) \
+		   (('a' <= (c) && (c) <= 'i') \
+		     || ('j' <= (c) && (c) <= 'r') \
+		     || ('s' <= (c) && (c) <= 'z'))
+# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c))
+#endif
+
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int
+main ()
+{
+  int i;
+  for (i = 0; i < 256; i++)
+    if (XOR (islower (i), ISLOWER (i))
+	|| toupper (i) != TOUPPER (i))
+      exit(2);
+  exit (0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  :
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_header_stdc=no
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_stdc" >&5
+echo "${ECHO_T}$ac_cv_header_stdc" >&6
+if test $ac_cv_header_stdc = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define STDC_HEADERS 1
+_ACEOF
+
+fi
+
+
+
+
+
+
+
+
+
+for ac_header in fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h
+do
+as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking $ac_header usability" >&5
+echo $ECHO_N "checking $ac_header usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <$ac_header>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking $ac_header presence" >&5
+echo $ECHO_N "checking $ac_header presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <$ac_header>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: $ac_header: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: $ac_header: present but cannot be compiled" >&5
+echo "$as_me: WARNING: $ac_header: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: $ac_header:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: $ac_header: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: $ac_header:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: $ac_header: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: $ac_header: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: $ac_header: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for $ac_header" >&5
+echo $ECHO_N "checking for $ac_header... $ECHO_C" >&6
+if eval "test \"\${$as_ac_Header+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  eval "$as_ac_Header=\$ac_header_preproc"
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_Header'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_Header'}'`" >&6
+
+fi
+if test `eval echo '${'$as_ac_Header'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fread" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FREAD_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <stdio.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "fgetc" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_FGETC_DECL 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/procfs.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "pr_brksize" >/dev/null 2>&1; then
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_PR_BRKSIZE 1
+_ACEOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo "$as_me:$LINENO: checking for working alloca.h" >&5
+echo $ECHO_N "checking for working alloca.h... $ECHO_C" >&6
+if test "${ac_cv_working_alloca_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <alloca.h>
+int
+main ()
+{
+char *p = (char *) alloca (2 * sizeof (int));
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_working_alloca_h=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_working_alloca_h=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_working_alloca_h" >&5
+echo "${ECHO_T}$ac_cv_working_alloca_h" >&6
+if test $ac_cv_working_alloca_h = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA_H 1
+_ACEOF
+
+fi
+
+echo "$as_me:$LINENO: checking for alloca" >&5
+echo $ECHO_N "checking for alloca... $ECHO_C" >&6
+if test "${ac_cv_func_alloca_works+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int
+main ()
+{
+char *p = (char *) alloca (1);
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_alloca_works=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_alloca_works=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_alloca_works" >&5
+echo "${ECHO_T}$ac_cv_func_alloca_works" >&6
+
+if test $ac_cv_func_alloca_works = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_ALLOCA 1
+_ACEOF
+
+else
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+# that cause trouble.  Some versions do not even contain alloca or
+# contain a buggy version.  If you still want to use their alloca,
+# use ar to extract alloca.o from them instead of compiling alloca.c.
+
+ALLOCA=alloca.$ac_objext
+
+cat >>confdefs.h <<\_ACEOF
+#define C_ALLOCA 1
+_ACEOF
+
+
+echo "$as_me:$LINENO: checking whether \`alloca.c' needs Cray hooks" >&5
+echo $ECHO_N "checking whether \`alloca.c' needs Cray hooks... $ECHO_C" >&6
+if test "${ac_cv_os_cray+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "webecray" >/dev/null 2>&1; then
+  ac_cv_os_cray=yes
+else
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+echo "$as_me:$LINENO: result: $ac_cv_os_cray" >&5
+echo "${ECHO_T}$ac_cv_os_cray" >&6
+if test $ac_cv_os_cray = yes; then
+  for ac_func in _getb67 GETB67 getb67; do
+    as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+
+cat >>confdefs.h <<_ACEOF
+#define CRAY_STACKSEG_END $ac_func
+_ACEOF
+
+    break
+fi
+
+  done
+fi
+
+echo "$as_me:$LINENO: checking stack direction for C alloca" >&5
+echo $ECHO_N "checking stack direction for C alloca... $ECHO_C" >&6
+if test "${ac_cv_c_stack_direction+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+
+int
+main ()
+{
+  exit (find_stack_direction () < 0);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_stack_direction=1
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_stack_direction=-1
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_stack_direction" >&5
+echo "${ECHO_T}$ac_cv_c_stack_direction" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+_ACEOF
+
+
+fi
+
+if test $ac_cv_c_compiler_gnu = yes; then
+    echo "$as_me:$LINENO: checking whether $CC needs -traditional" >&5
+echo $ECHO_N "checking whether $CC needs -traditional... $ECHO_C" >&6
+if test "${ac_cv_prog_gcc_traditional+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sgtty.h>
+Autoconf TIOCGETP
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+else
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <termio.h>
+Autoconf TCGETA
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "$ac_pattern" >/dev/null 2>&1; then
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_prog_gcc_traditional" >&5
+echo "${ECHO_T}$ac_cv_prog_gcc_traditional" >&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo "$as_me:$LINENO: checking return type of signal handlers" >&5
+echo $ECHO_N "checking return type of signal handlers... $ECHO_C" >&6
+if test "${ac_cv_type_signal+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+# undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int
+main ()
+{
+int i;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_signal=void
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_signal=int
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_signal" >&5
+echo "${ECHO_T}$ac_cv_type_signal" >&6
+
+cat >>confdefs.h <<_ACEOF
+#define RETSIGTYPE $ac_cv_type_signal
+_ACEOF
+
+
+
+for ac_func in vprintf
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+echo "$as_me:$LINENO: checking for _doprnt" >&5
+echo $ECHO_N "checking for _doprnt... $ECHO_C" >&6
+if test "${ac_cv_func__doprnt+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define _doprnt to an innocuous variant, in case <limits.h> declares _doprnt.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define _doprnt innocuous__doprnt
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef _doprnt
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char _doprnt ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+char (*f) () = _doprnt;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != _doprnt;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func__doprnt=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func__doprnt=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func__doprnt" >&5
+echo "${ECHO_T}$ac_cv_func__doprnt" >&6
+if test $ac_cv_func__doprnt = yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_DOPRNT 1
+_ACEOF
+
+fi
+
+fi
+done
+
+
+
+
+
+
+
+
+
+
+
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+
+
+
+for ac_func in ftruncate strstr strcasecmp
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+else
+  case $LIBOBJS in
+    "$ac_func.$ac_objext"   | \
+  *" $ac_func.$ac_objext"   | \
+    "$ac_func.$ac_objext "* | \
+  *" $ac_func.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;;
+esac
+
+fi
+done
+
+
+
+
+echo "$as_me:$LINENO: checking for textdomain" >&5
+echo $ECHO_N "checking for textdomain... $ECHO_C" >&6
+if test "${ac_cv_func_textdomain+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define textdomain to an innocuous variant, in case <limits.h> declares textdomain.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define textdomain innocuous_textdomain
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char textdomain (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef textdomain
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char textdomain ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_textdomain) || defined (__stub___textdomain)
+choke me
+#else
+char (*f) () = textdomain;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != textdomain;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_textdomain=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_textdomain=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_textdomain" >&5
+echo "${ECHO_T}$ac_cv_func_textdomain" >&6
+if test $ac_cv_func_textdomain = yes; then
+  cat >>confdefs.h <<\_ACEOF
+#define ENABLE_NLS  1
+_ACEOF
+
+fi
+
+
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+echo "$as_me:$LINENO: checking for Apple MacOS X/Darwin" >&5
+echo $ECHO_N "checking for Apple MacOS X/Darwin... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo "$as_me:$LINENO: checking for HP-UX needing gmalloc" >&5
+echo $ECHO_N "checking for HP-UX needing gmalloc... $ECHO_C" >&6
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  case $LIBOBJS in
+    "gmalloc.$ac_objext"   | \
+  *" gmalloc.$ac_objext"   | \
+    "gmalloc.$ac_objext "* | \
+  *" gmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS gmalloc.$ac_objext" ;;
+esac
+
+  cat >>confdefs.h <<\_ACEOF
+#define HAVE_VALLOC 1
+_ACEOF
+
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+
+for ac_func in valloc
+do
+as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
+echo "$as_me:$LINENO: checking for $ac_func" >&5
+echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6
+if eval "test \"\${$as_ac_var+set}\" = set"; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define $ac_func to an innocuous variant, in case <limits.h> declares $ac_func.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $ac_func innocuous_$ac_func
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef $ac_func
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char $ac_func ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+char (*f) () = $ac_func;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != $ac_func;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  eval "$as_ac_var=yes"
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+eval "$as_ac_var=no"
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: `eval echo '${'$as_ac_var'}'`" >&5
+echo "${ECHO_T}`eval echo '${'$as_ac_var'}'`" >&6
+if test `eval echo '${'$as_ac_var'}'` = yes; then
+  cat >>confdefs.h <<_ACEOF
+#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+done
+
+fi
+
+echo "$as_me:$LINENO: checking if malloc debugging is wanted" >&5
+echo $ECHO_N "checking if malloc debugging is wanted... $ECHO_C" >&6
+
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_DMALLOC 1
+_ACEOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+echo "$as_me:$LINENO: checking which of rx or regex is wanted" >&5
+echo $ECHO_N "checking which of rx or regex is wanted... $ECHO_C" >&6
+
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$as_me:$LINENO: result: regex" >&5
+echo "${ECHO_T}regex" >&6
+  cat >>confdefs.h <<\_ACEOF
+#define WITH_REGEX 1
+_ACEOF
+
+  case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+fi
+fi;
+if test -z "$ac_with_regex"; then
+  echo "$as_me:$LINENO: result: rx" >&5
+echo "${ECHO_T}rx" >&6
+  echo "$as_me:$LINENO: checking for re_rx_search" >&5
+echo $ECHO_N "checking for re_rx_search... $ECHO_C" >&6
+if test "${ac_cv_func_re_rx_search+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+/* Define re_rx_search to an innocuous variant, in case <limits.h> declares re_rx_search.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define re_rx_search innocuous_re_rx_search
+
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search (); below.
+    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
+    <limits.h> exists even on freestanding compilers.  */
+
+#ifdef __STDC__
+# include <limits.h>
+#else
+# include <assert.h>
+#endif
+
+#undef re_rx_search
+
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+/* We use char because int might match the return type of a gcc2
+   builtin and then its argument prototype would still apply.  */
+char re_rx_search ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+char (*f) () = re_rx_search;
+#endif
+#ifdef __cplusplus
+}
+#endif
+
+int
+main ()
+{
+return f != re_rx_search;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_func_re_rx_search=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_func_re_rx_search=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_func_re_rx_search" >&5
+echo "${ECHO_T}$ac_cv_func_re_rx_search" >&6
+if test $ac_cv_func_re_rx_search = yes; then
+  :
+else
+  echo "$as_me:$LINENO: checking for unsigned char *" >&5
+echo $ECHO_N "checking for unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_type_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+if ((unsigned char * *) 0)
+  return 0;
+if (sizeof (unsigned char *))
+  return 0;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_type_unsigned_char_p=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_type_unsigned_char_p=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_type_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_type_unsigned_char_p" >&6
+
+echo "$as_me:$LINENO: checking size of unsigned char *" >&5
+echo $ECHO_N "checking size of unsigned char *... $ECHO_C" >&6
+if test "${ac_cv_sizeof_unsigned_char_p+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test "$ac_cv_type_unsigned_char_p" = yes; then
+  # The cast to unsigned long works around a bug in the HP C Compiler
+  # version HP92453-01 B.11.11.23709.GP, which incorrectly rejects
+  # declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'.
+  # This bug is HP SR number 8606223364.
+  if test "$cross_compiling" = yes; then
+  # Depending upon the size, compute the lo and hi bounds.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=0 ac_mid=0
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr $ac_mid + 1`
+		    if test $ac_lo -le $ac_mid; then
+		      ac_lo= ac_hi=
+		      break
+		    fi
+		    ac_mid=`expr 2 '*' $ac_mid + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) < 0)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=-1 ac_mid=-1
+  while :; do
+    cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) >= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_lo=$ac_mid; break
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_hi=`expr '(' $ac_mid ')' - 1`
+		       if test $ac_mid -le $ac_hi; then
+			 ac_lo= ac_hi=
+			 break
+		       fi
+		       ac_mid=`expr 2 '*' $ac_mid`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+  done
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo= ac_hi=
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+# Binary search between lo and hi bounds.
+while test "x$ac_lo" != "x$ac_hi"; do
+  ac_mid=`expr '(' $ac_hi - $ac_lo ')' / 2 + $ac_lo`
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+int
+main ()
+{
+static int test_array [1 - 2 * !(((long) (sizeof (unsigned char *))) <= $ac_mid)];
+test_array [0] = 0
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_hi=$ac_mid
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_lo=`expr '(' $ac_mid ')' + 1`
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+done
+case $ac_lo in
+?*) ac_cv_sizeof_unsigned_char_p=$ac_lo;;
+'') { { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+else
+  if test "$cross_compiling" = yes; then
+  { { echo "$as_me:$LINENO: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot run test program while cross compiling
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+long longval () { return (long) (sizeof (unsigned char *)); }
+unsigned long ulongval () { return (long) (sizeof (unsigned char *)); }
+#include <stdio.h>
+#include <stdlib.h>
+int
+main ()
+{
+
+  FILE *f = fopen ("conftest.val", "w");
+  if (! f)
+    exit (1);
+  if (((long) (sizeof (unsigned char *))) < 0)
+    {
+      long i = longval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%ld\n", i);
+    }
+  else
+    {
+      unsigned long i = ulongval ();
+      if (i != ((long) (sizeof (unsigned char *))))
+	exit (1);
+      fprintf (f, "%lu\n", i);
+    }
+  exit (ferror (f) || fclose (f) != 0);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_sizeof_unsigned_char_p=`cat conftest.val`
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+{ { echo "$as_me:$LINENO: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&5
+echo "$as_me: error: cannot compute sizeof (unsigned char *), 77
+See \`config.log' for more details." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.val
+else
+  ac_cv_sizeof_unsigned_char_p=0
+fi
+fi
+echo "$as_me:$LINENO: result: $ac_cv_sizeof_unsigned_char_p" >&5
+echo "${ECHO_T}$ac_cv_sizeof_unsigned_char_p" >&6
+cat >>confdefs.h <<_ACEOF
+#define SIZEOF_UNSIGNED_CHAR_P $ac_cv_sizeof_unsigned_char_p
+_ACEOF
+
+
+    if test "$ac_cv_sizeof_unsigned_char_p" = 8
+    then
+      { echo "$as_me:$LINENO: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&5
+echo "$as_me: WARNING: I'm forcing you to use regex because I can't
+        find a local rx library and the one included with this
+        distribution doesn't work on 64-bit machines like yours" >&2;}
+      case $LIBOBJS in
+    "regex.$ac_objext"   | \
+  *" regex.$ac_objext"   | \
+    "regex.$ac_objext "* | \
+  *" regex.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS regex.$ac_objext" ;;
+esac
+
+    else
+      case $LIBOBJS in
+    "rx.$ac_objext"   | \
+  *" rx.$ac_objext"   | \
+    "rx.$ac_objext "* | \
+  *" rx.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS rx.$ac_objext" ;;
+esac
+
+    fi
+
+fi
+
+fi
+
+echo "$as_me:$LINENO: checking whether GNU readline requested" >&5
+echo $ECHO_N "checking whether GNU readline requested... $ECHO_C" >&6
+
+# Check whether --with-gnu_readline or --without-gnu_readline was given.
+if test "${with_gnu_readline+set}" = set; then
+  withval="$with_gnu_readline"
+  if test "$withval" = yes; then
+      echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6
+      ac_with_gnu_readline=1
+    else
+      echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+    fi
+else
+  echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6
+fi;
+
+  if test -n "$ac_with_gnu_readline"; then
+    if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+else
+  # Is the header compilable?
+echo "$as_me:$LINENO: checking readline/readline.h usability" >&5
+echo $ECHO_N "checking readline/readline.h usability... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <readline/readline.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_header_compiler=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_header_compiler=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+echo "${ECHO_T}$ac_header_compiler" >&6
+
+# Is the header present?
+echo "$as_me:$LINENO: checking readline/readline.h presence" >&5
+echo $ECHO_N "checking readline/readline.h presence... $ECHO_C" >&6
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <readline/readline.h>
+_ACEOF
+if { (eval echo "$as_me:$LINENO: \"$ac_cpp conftest.$ac_ext\"") >&5
+  (eval $ac_cpp conftest.$ac_ext) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null; then
+  if test -s conftest.err; then
+    ac_cpp_err=$ac_c_preproc_warn_flag
+    ac_cpp_err=$ac_cpp_err$ac_c_werror_flag
+  else
+    ac_cpp_err=
+  fi
+else
+  ac_cpp_err=yes
+fi
+if test -z "$ac_cpp_err"; then
+  ac_header_preproc=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+rm -f conftest.err conftest.$ac_ext
+echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+echo "${ECHO_T}$ac_header_preproc" >&6
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&5
+echo "$as_me: WARNING: readline/readline.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the compiler's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: present but cannot be compiled" >&5
+echo "$as_me: WARNING: readline/readline.h: present but cannot be compiled" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&5
+echo "$as_me: WARNING: readline/readline.h:     check for missing prerequisite headers?" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: see the Autoconf documentation" >&5
+echo "$as_me: WARNING: readline/readline.h: see the Autoconf documentation" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&5
+echo "$as_me: WARNING: readline/readline.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&5
+echo "$as_me: WARNING: readline/readline.h: proceeding with the preprocessor's result" >&2;}
+    { echo "$as_me:$LINENO: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&5
+echo "$as_me: WARNING: readline/readline.h: in the future, the compiler will take precedence" >&2;}
+    (
+      cat <<\_ASBOX
+## ------------------------------------------ ##
+## Report this to the AC_PACKAGE_NAME lists.  ##
+## ------------------------------------------ ##
+_ASBOX
+    ) |
+      sed "s/^/$as_me: WARNING:     /" >&2
+    ;;
+esac
+echo "$as_me:$LINENO: checking for readline/readline.h" >&5
+echo $ECHO_N "checking for readline/readline.h... $ECHO_C" >&6
+if test "${ac_cv_header_readline_readline_h+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_cv_header_readline_readline_h=$ac_header_preproc
+fi
+echo "$as_me:$LINENO: result: $ac_cv_header_readline_readline_h" >&5
+echo "${ECHO_T}$ac_cv_header_readline_readline_h" >&6
+
+fi
+if test $ac_cv_header_readline_readline_h = yes; then
+  ac_mg_readline_header_found=1
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline headers; configuring without \
+GNU readline support" >&2;}
+fi
+
+
+    if test -n "$ac_mg_readline_header_found"; then
+      # first check whether we can find the readline library itself
+      echo "$as_me:$LINENO: checking for main in -lreadline" >&5
+echo $ECHO_N "checking for main in -lreadline... $ECHO_C" >&6
+if test "${ac_cv_lib_readline_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lreadline  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_readline_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_readline_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_readline_main" >&5
+echo "${ECHO_T}$ac_cv_lib_readline_main" >&6
+if test $ac_cv_lib_readline_main = yes; then
+  ac_mg_readline_lib_found=1
+         cat >>confdefs.h <<\_ACEOF
+#define WITH_GNU_READLINE 1
+_ACEOF
+
+         LIBS="$LIBS -lreadline"
+else
+  { echo "$as_me:$LINENO: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&5
+echo "$as_me: WARNING: Can't find GNU readline library; configuring without \
+GNU readline support" >&2;}
+fi
+
+      if test -n "$ac_mg_readline_lib_found"; then
+        # on some systems, readline needs curses.  It is difficult to
+        #  determine whether this is necessary on the current system,
+        #  since other undefined symbols can be turned up by an
+        #  AC_CHECK_LIB(readline, readline) test that are actually defined
+	#  by mg itself (e.g. xmalloc, xrealloc).  So, if we find libcurses,
+	#  we just bung it on and hope for the best.
+	echo "$as_me:$LINENO: checking for main in -lcurses" >&5
+echo $ECHO_N "checking for main in -lcurses... $ECHO_C" >&6
+if test "${ac_cv_lib_curses_main+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurses  $LIBS"
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+main ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_lib_curses_main=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_lib_curses_main=no
+fi
+rm -f conftest.err conftest.$ac_objext \
+      conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_lib_curses_main" >&5
+echo "${ECHO_T}$ac_cv_lib_curses_main" >&6
+if test $ac_cv_lib_curses_main = yes; then
+  LIBS="$LIBS -lcurses"
+fi
+
+      fi
+    fi
+  fi
+
+
+# text for endianness
+echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  case $LIBOBJS in
+    "xmalloc.$ac_objext"   | \
+  *" xmalloc.$ac_objext"   | \
+    "xmalloc.$ac_objext "* | \
+  *" xmalloc.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS xmalloc.$ac_objext" ;;
+esac
+
+  case $LIBOBJS in
+    "error.$ac_objext"   | \
+  *" error.$ac_objext"   | \
+    "error.$ac_objext "* | \
+  *" error.$ac_objext "* ) ;;
+  *) LIBOBJS="$LIBOBJS error.$ac_objext" ;;
+esac
+
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+# Check whether --enable-override-longlong or --disable-override-longlong was given.
+if test "${enable_override_longlong+set}" = set; then
+  enableval="$enable_override_longlong"
+  cat >>confdefs.h <<\_ACEOF
+#define DISABLE_LONG_LONG 1
+_ACEOF
+
+fi;
+
+
+# ---------------------------------------------------------------------------
+
+                                                  ac_config_files="$ac_config_files Makefile text/Makefile lib/Makefile jni/Makefile java/org/greenstone/mgpp/Makefile"
+          ac_config_commands="$ac_config_commands default"
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+{
+  (set) 2>&1 |
+    case `(ac_space=' '; set | grep ac_space) 2>&1` in
+    *ac_space=\ *)
+      # `set' does not quote correctly, so add quotes (double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \).
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;;
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n \
+	"s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1=\\2/p"
+      ;;
+    esac;
+} |
+  sed '
+     t clear
+     : clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/
+     t end
+     /^ac_cv_env/!s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     : end' >>confcache
+if diff $cache_file confcache >/dev/null 2>&1; then :; else
+  if test -w $cache_file; then
+    test "x$cache_file" != "x/dev/null" && echo "updating cache $cache_file"
+    cat confcache >$cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# VPATH may cause trouble with some makes, so we remove $(srcdir),
+# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=/{
+s/:*\$(srcdir):*/:/;
+s/:*\${srcdir}:*/:/;
+s/:*@srcdir@:*/:/;
+s/^\([^=]*=[	 ]*\):*/\1/;
+s/:*$//;
+s/^[^=]*=[	 ]*$//;
+}'
+fi
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_i=`echo "$ac_i" |
+	 sed 's/\$U\././;s/\.o$//;s/\.obj$//'`
+  # 2. Add them.
+  ac_libobjs="$ac_libobjs $ac_i\$U.$ac_objext"
+  ac_ltlibobjs="$ac_ltlibobjs $ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: ${CONFIG_STATUS=./config.status}
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ echo "$as_me:$LINENO: creating $CONFIG_STATUS" >&5
+echo "$as_me: creating $CONFIG_STATUS" >&6;}
+cat >$CONFIG_STATUS <<_ACEOF
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+SHELL=\${CONFIG_SHELL-$SHELL}
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+## --------------------- ##
+## M4sh Initialization.  ##
+## --------------------- ##
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+elif test -n "${BASH_VERSION+set}" && (set -o posix) >/dev/null 2>&1; then
+  set -o posix
+fi
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# Support unset when possible.
+if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then
+  as_unset=unset
+else
+  as_unset=false
+fi
+
+
+# Work around bugs in pre-3.0 UWIN ksh.
+$as_unset ENV MAIL MAILPATH
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# NLS nuisances.
+for as_var in \
+  LANG LANGUAGE LC_ADDRESS LC_ALL LC_COLLATE LC_CTYPE LC_IDENTIFICATION \
+  LC_MEASUREMENT LC_MESSAGES LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER \
+  LC_TELEPHONE LC_TIME
+do
+  if (set +x; test -z "`(eval $as_var=C; export $as_var) 2>&1`"); then
+    eval $as_var=C; export $as_var
+  else
+    $as_unset $as_var
+  fi
+done
+
+# Required to use basename.
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename /) >/dev/null 2>&1 && test "X`basename / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+
+# Name of the executable.
+as_me=`$as_basename "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)$' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{ s//\1/; q; }
+  	  /^X\/\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\/\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+
+
+# PATH needs CR, and LINENO needs CR and PATH.
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+# The user is always right.
+if test "${PATH_SEPARATOR+set}" != set; then
+  echo "#! /bin/sh" >conf$$.sh
+  echo  "exit 0"   >>conf$$.sh
+  chmod +x conf$$.sh
+  if (PATH="/nonexistent;."; conf$$.sh) >/dev/null 2>&1; then
+    PATH_SEPARATOR=';'
+  else
+    PATH_SEPARATOR=:
+  fi
+  rm -f conf$$.sh
+fi
+
+
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2"  || {
+  # Find who we are.  Look in the path if we contain no path at all
+  # relative or not.
+  case $0 in
+    *[\\/]* ) as_myself=$0 ;;
+    *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break
+done
+
+       ;;
+  esac
+  # We did not find ourselves, most probably we were run as `sh COMMAND'
+  # in which case we are not to be found in the path.
+  if test "x$as_myself" = x; then
+    as_myself=$0
+  fi
+  if test ! -f "$as_myself"; then
+    { { echo "$as_me:$LINENO: error: cannot find myself; rerun with an absolute path" >&5
+echo "$as_me: error: cannot find myself; rerun with an absolute path" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+  case $CONFIG_SHELL in
+  '')
+    as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for as_base in sh bash ksh sh5; do
+	 case $as_dir in
+	 /*)
+	   if ("$as_dir/$as_base" -c '
+  as_lineno_1=$LINENO
+  as_lineno_2=$LINENO
+  as_lineno_3=`(expr $as_lineno_1 + 1) 2>/dev/null`
+  test "x$as_lineno_1" != "x$as_lineno_2" &&
+  test "x$as_lineno_3"  = "x$as_lineno_2" ') 2>/dev/null; then
+	     $as_unset BASH_ENV || test "${BASH_ENV+set}" != set || { BASH_ENV=; export BASH_ENV; }
+	     $as_unset ENV || test "${ENV+set}" != set || { ENV=; export ENV; }
+	     CONFIG_SHELL=$as_dir/$as_base
+	     export CONFIG_SHELL
+	     exec "$CONFIG_SHELL" "$0" ${1+"$@"}
+	   fi;;
+	 esac
+       done
+done
+;;
+  esac
+
+  # Create $as_me.lineno as a copy of $as_myself, but with $LINENO
+  # uniformly replaced by the line number.  The first 'sed' inserts a
+  # line-number line before each line; the second 'sed' does the real
+  # work.  The second script uses 'N' to pair each line-number line
+  # with the numbered line, and appends trailing '-' during
+  # substitution so that $LINENO is not a special case at line end.
+  # (Raja R Harinath suggested sed '=', and Paul Eggert wrote the
+  # second 'sed' script.  Blame Lee E. McMahon for sed's syntax.  :-)
+  sed '=' <$as_myself |
+    sed '
+      N
+      s,$,-,
+      : loop
+      s,^\(['$as_cr_digits']*\)\(.*\)[$]LINENO\([^'$as_cr_alnum'_]\),\1\2\1\3,
+      t loop
+      s,-$,,
+      s,^['$as_cr_digits']*\n,,
+    ' >$as_me.lineno &&
+  chmod +x $as_me.lineno ||
+    { { echo "$as_me:$LINENO: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&5
+echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2;}
+   { (exit 1); exit 1; }; }
+
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensible to this).
+  . ./$as_me.lineno
+  # Exit status is that of the last command.
+  exit
+}
+
+
+case `echo "testing\c"; echo 1,2,3`,`echo -n testing; echo 1,2,3` in
+  *c*,-n*) ECHO_N= ECHO_C='
+' ECHO_T='	' ;;
+  *c*,*  ) ECHO_N=-n ECHO_C= ECHO_T= ;;
+  *)       ECHO_N= ECHO_C='\c' ECHO_T= ;;
+esac
+
+if expr a : '\(a\)' >/dev/null 2>&1; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+rm -f conf$$ conf$$.exe conf$$.file
+echo >conf$$.file
+if ln -s conf$$.file conf$$ 2>/dev/null; then
+  # We could just check for DJGPP; but this test a) works b) is more generic
+  # and c) will remain valid once DJGPP supports symlinks (DJGPP 2.04).
+  if test -f conf$$.exe; then
+    # Don't use ln at all; we don't have any links
+    as_ln_s='cp -p'
+  else
+    as_ln_s='ln -s'
+  fi
+elif ln conf$$.file conf$$ 2>/dev/null; then
+  as_ln_s=ln
+else
+  as_ln_s='cp -p'
+fi
+rm -f conf$$ conf$$.exe conf$$.file
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p=:
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_executable_p="test -f"
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+# IFS
+# We need space, tab and new line, in precisely that order.
+as_nl='
+'
+IFS=" 	$as_nl"
+
+# CDPATH.
+$as_unset CDPATH
+
+exec 6>&1
+
+# Open the log real soon, to keep \$[0] and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.  Logging --version etc. is OK.
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+} >&5
+cat >&5 <<_CSEOF
+
+This file was extended by $as_me, which was
+generated by GNU Autoconf 2.59.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+_CSEOF
+echo "on `(hostname || uname -n) 2>/dev/null | sed 1q`" >&5
+echo >&5
+_ACEOF
+
+# Files that config.status was made for.
+if test -n "$ac_config_files"; then
+  echo "config_files=\"$ac_config_files\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_headers"; then
+  echo "config_headers=\"$ac_config_headers\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_links"; then
+  echo "config_links=\"$ac_config_links\"" >>$CONFIG_STATUS
+fi
+
+if test -n "$ac_config_commands"; then
+  echo "config_commands=\"$ac_config_commands\"" >>$CONFIG_STATUS
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+ac_cs_usage="\
+\`$as_me' instantiates files from templates according to the
+current configuration.
+
+Usage: $0 [OPTIONS] [FILE]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number, then exit
+  -q, --quiet      do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+  --file=FILE[:TEMPLATE]
+		   instantiate the configuration file FILE
+  --header=FILE[:TEMPLATE]
+		   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration commands:
+$config_commands
+
+Report bugs to <bug-autoconf@gnu.org>."
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+ac_cs_version="\\
+config.status
+configured by $0, generated by GNU Autoconf 2.59,
+  with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
+
+Copyright (C) 2003 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+srcdir=$srcdir
+INSTALL="$INSTALL"
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+# If no file are specified by the user, then we need to provide default
+# value.  By we need to know if files were specified by the user.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=*)
+    ac_option=`expr "x$1" : 'x\([^=]*\)='`
+    ac_optarg=`expr "x$1" : 'x[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  -*)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  *) # This is not an option, so the user has probably given explicit
+     # arguments.
+     ac_option=$1
+     ac_need_defaults=false;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --vers* | -V )
+    echo "$ac_cs_version"; exit 0 ;;
+  --he | --h)
+    # Conflict between --help and --header
+    { { echo "$as_me:$LINENO: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: ambiguous option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; };;
+  --help | --hel | -h )
+    echo "$ac_cs_usage"; exit 0 ;;
+  --debug | --d* | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    CONFIG_FILES="$CONFIG_FILES $ac_optarg"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    CONFIG_HEADERS="$CONFIG_HEADERS $ac_optarg"
+    ac_need_defaults=false;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) { { echo "$as_me:$LINENO: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&5
+echo "$as_me: error: unrecognized option: $1
+Try \`$0 --help' for more information." >&2;}
+   { (exit 1); exit 1; }; } ;;
+
+  *) ac_config_targets="$ac_config_targets $1" ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+if \$ac_cs_recheck; then
+  echo "running $SHELL $0 " $ac_configure_args \$ac_configure_extra_args " --no-create --no-recursion" >&6
+  exec $SHELL $0 $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+fi
+
+_ACEOF
+
+
+
+
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_config_target in $ac_config_targets
+do
+  case "$ac_config_target" in
+  # Handling of arguments.
+  "Makefile" ) CONFIG_FILES="$CONFIG_FILES Makefile" ;;
+  "text/Makefile" ) CONFIG_FILES="$CONFIG_FILES text/Makefile" ;;
+  "lib/Makefile" ) CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
+  "jni/Makefile" ) CONFIG_FILES="$CONFIG_FILES jni/Makefile" ;;
+  "java/org/greenstone/mgpp/Makefile" ) CONFIG_FILES="$CONFIG_FILES java/org/greenstone/mgpp/Makefile" ;;
+  "default" ) CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;;
+  "config.h" ) CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+  *) { { echo "$as_me:$LINENO: error: invalid argument: $ac_config_target" >&5
+echo "$as_me: error: invalid argument: $ac_config_target" >&2;}
+   { (exit 1); exit 1; }; };;
+  esac
+done
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files
+  test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers
+  test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason to put it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Create a temporary directory, and hook for its removal unless debugging.
+$debug ||
+{
+  trap 'exit_status=$?; rm -rf $tmp && exit $exit_status' 0
+  trap '{ (exit 1); exit 1; }' 1 2 13 15
+}
+
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d -q "./confstatXXXXXX") 2>/dev/null` &&
+  test -n "$tmp" && test -d "$tmp"
+}  ||
+{
+  tmp=./confstat$$-$RANDOM
+  (umask 077 && mkdir $tmp)
+} ||
+{
+   echo "$me: cannot create a temporary directory in ." >&2
+   { (exit 1); exit 1; }
+}
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<_ACEOF
+
+#
+# CONFIG_FILES section.
+#
+
+# No need to generate the scripts if there are no CONFIG_FILES.
+# This happens for instance when ./config.status config.h
+if test -n "\$CONFIG_FILES"; then
+  # Protect against being on the right side of a sed subst in config.status.
+  sed 's/,@/@@/; s/@,/@@/; s/,;t t\$/@;t t/; /@;t t\$/s/[\\\\&,]/\\\\&/g;
+   s/@@/,@/; s/@@/@,/; s/@;t t\$/,;t t/' >\$tmp/subs.sed <<\\CEOF
+s,@SHELL@,$SHELL,;t t
+s,@PATH_SEPARATOR@,$PATH_SEPARATOR,;t t
+s,@PACKAGE_NAME@,$PACKAGE_NAME,;t t
+s,@PACKAGE_TARNAME@,$PACKAGE_TARNAME,;t t
+s,@PACKAGE_VERSION@,$PACKAGE_VERSION,;t t
+s,@PACKAGE_STRING@,$PACKAGE_STRING,;t t
+s,@PACKAGE_BUGREPORT@,$PACKAGE_BUGREPORT,;t t
+s,@exec_prefix@,$exec_prefix,;t t
+s,@prefix@,$prefix,;t t
+s,@program_transform_name@,$program_transform_name,;t t
+s,@bindir@,$bindir,;t t
+s,@sbindir@,$sbindir,;t t
+s,@libexecdir@,$libexecdir,;t t
+s,@datadir@,$datadir,;t t
+s,@sysconfdir@,$sysconfdir,;t t
+s,@sharedstatedir@,$sharedstatedir,;t t
+s,@localstatedir@,$localstatedir,;t t
+s,@libdir@,$libdir,;t t
+s,@includedir@,$includedir,;t t
+s,@oldincludedir@,$oldincludedir,;t t
+s,@infodir@,$infodir,;t t
+s,@mandir@,$mandir,;t t
+s,@build_alias@,$build_alias,;t t
+s,@host_alias@,$host_alias,;t t
+s,@target_alias@,$target_alias,;t t
+s,@DEFS@,$DEFS,;t t
+s,@ECHO_C@,$ECHO_C,;t t
+s,@ECHO_N@,$ECHO_N,;t t
+s,@ECHO_T@,$ECHO_T,;t t
+s,@LIBS@,$LIBS,;t t
+s,@PACKAGE@,$PACKAGE,;t t
+s,@VERSION@,$VERSION,;t t
+s,@CXX@,$CXX,;t t
+s,@CXXFLAGS@,$CXXFLAGS,;t t
+s,@LDFLAGS@,$LDFLAGS,;t t
+s,@CPPFLAGS@,$CPPFLAGS,;t t
+s,@ac_ct_CXX@,$ac_ct_CXX,;t t
+s,@EXEEXT@,$EXEEXT,;t t
+s,@OBJEXT@,$OBJEXT,;t t
+s,@AWK@,$AWK,;t t
+s,@YACC@,$YACC,;t t
+s,@CC@,$CC,;t t
+s,@CFLAGS@,$CFLAGS,;t t
+s,@ac_ct_CC@,$ac_ct_CC,;t t
+s,@INSTALL_PROGRAM@,$INSTALL_PROGRAM,;t t
+s,@INSTALL_SCRIPT@,$INSTALL_SCRIPT,;t t
+s,@INSTALL_DATA@,$INSTALL_DATA,;t t
+s,@LN_S@,$LN_S,;t t
+s,@SET_MAKE@,$SET_MAKE,;t t
+s,@RANLIB@,$RANLIB,;t t
+s,@ac_ct_RANLIB@,$ac_ct_RANLIB,;t t
+s,@CPP@,$CPP,;t t
+s,@EGREP@,$EGREP,;t t
+s,@U@,$U,;t t
+s,@ANSI2KNR@,$ANSI2KNR,;t t
+s,@ALLOCA@,$ALLOCA,;t t
+s,@LIBOBJS@,$LIBOBJS,;t t
+s,@JNIINC@,$JNIINC,;t t
+s,@JNISUFFIX@,$JNISUFFIX,;t t
+s,@JNIFLAGS@,$JNIFLAGS,;t t
+s,@LTLIBOBJS@,$LTLIBOBJS,;t t
+CEOF
+
+_ACEOF
+
+  cat >>$CONFIG_STATUS <<\_ACEOF
+  # Split the substitutions into bite-sized pieces for seds with
+  # small command number limits, like on Digital OSF/1 and HP-UX.
+  ac_max_sed_lines=48
+  ac_sed_frag=1 # Number of current file.
+  ac_beg=1 # First line for current file.
+  ac_end=$ac_max_sed_lines # Line after last line for current file.
+  ac_more_lines=:
+  ac_sed_cmds=
+  while $ac_more_lines; do
+    if test $ac_beg -gt 1; then
+      sed "1,${ac_beg}d; ${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    else
+      sed "${ac_end}q" $tmp/subs.sed >$tmp/subs.frag
+    fi
+    if test ! -s $tmp/subs.frag; then
+      ac_more_lines=false
+    else
+      # The purpose of the label and of the branching condition is to
+      # speed up the sed processing (if there are no `@' at all, there
+      # is no need to browse any of the substitutions).
+      # These are the two extra sed commands mentioned above.
+      (echo ':t
+  /@[a-zA-Z_][a-zA-Z_0-9]*@/!b' && cat $tmp/subs.frag) >$tmp/subs-$ac_sed_frag.sed
+      if test -z "$ac_sed_cmds"; then
+	ac_sed_cmds="sed -f $tmp/subs-$ac_sed_frag.sed"
+      else
+	ac_sed_cmds="$ac_sed_cmds | sed -f $tmp/subs-$ac_sed_frag.sed"
+      fi
+      ac_sed_frag=`expr $ac_sed_frag + 1`
+      ac_beg=$ac_end
+      ac_end=`expr $ac_end + $ac_max_sed_lines`
+    fi
+  done
+  if test -z "$ac_sed_cmds"; then
+    ac_sed_cmds=cat
+  fi
+fi # test -n "$CONFIG_FILES"
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+for ac_file in : $CONFIG_FILES; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  # Compute @srcdir@, @top_srcdir@, and @INSTALL@ for subdirectories.
+  ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  case $INSTALL in
+  [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;;
+  *) ac_INSTALL=$ac_top_builddir$INSTALL ;;
+  esac
+
+  if test x"$ac_file" != x-; then
+    { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+    rm -f "$ac_file"
+  fi
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    configure_input=
+  else
+    configure_input="$ac_file.  "
+  fi
+  configure_input=$configure_input"Generated from `echo $ac_file_in |
+				     sed 's,.*/,,'` by configure."
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF
+  sed "$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s,@configure_input@,$configure_input,;t t
+s,@srcdir@,$ac_srcdir,;t t
+s,@abs_srcdir@,$ac_abs_srcdir,;t t
+s,@top_srcdir@,$ac_top_srcdir,;t t
+s,@abs_top_srcdir@,$ac_abs_top_srcdir,;t t
+s,@builddir@,$ac_builddir,;t t
+s,@abs_builddir@,$ac_abs_builddir,;t t
+s,@top_builddir@,$ac_top_builddir,;t t
+s,@abs_top_builddir@,$ac_abs_top_builddir,;t t
+s,@INSTALL@,$ac_INSTALL,;t t
+" $ac_file_inputs | (eval "$ac_sed_cmds") >$tmp/out
+  rm -f $tmp/stdin
+  if test x"$ac_file" != x-; then
+    mv $tmp/out $ac_file
+  else
+    cat $tmp/out
+    rm -f $tmp/out
+  fi
+
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_HEADER section.
+#
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s,^\([	 ]*\)#\([	 ]*define[	 ][	 ]*\)'
+ac_dB='[	 ].*$,\1#\2'
+ac_dC=' '
+ac_dD=',;t'
+# ac_u turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_uA='s,^\([	 ]*\)#\([	 ]*\)undef\([	 ][	 ]*\)'
+ac_uB='$,\1#\2define\3'
+ac_uC=' '
+ac_uD=',;t'
+
+for ac_file in : $CONFIG_HEADERS; do test "x$ac_file" = x: && continue
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case $ac_file in
+  - | *:- | *:-:* ) # input from stdin
+	cat >$tmp/stdin
+	ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  *:* ) ac_file_in=`echo "$ac_file" | sed 's,[^:]*:,,'`
+	ac_file=`echo "$ac_file" | sed 's,:.*,,'` ;;
+  * )   ac_file_in=$ac_file.in ;;
+  esac
+
+  test x"$ac_file" != x- && { echo "$as_me:$LINENO: creating $ac_file" >&5
+echo "$as_me: creating $ac_file" >&6;}
+
+  # First look for the input files in the build tree, otherwise in the
+  # src tree.
+  ac_file_inputs=`IFS=:
+    for f in $ac_file_in; do
+      case $f in
+      -) echo $tmp/stdin ;;
+      [\\/$]*)
+	 # Absolute (can't be DOS-style, as IFS=:)
+	 test -f "$f" || { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 # Do quote $f, to prevent DOS paths from being IFS'd.
+	 echo "$f";;
+      *) # Relative
+	 if test -f "$f"; then
+	   # Build tree
+	   echo "$f"
+	 elif test -f "$srcdir/$f"; then
+	   # Source tree
+	   echo "$srcdir/$f"
+	 else
+	   # /dev/null tree
+	   { { echo "$as_me:$LINENO: error: cannot find input file: $f" >&5
+echo "$as_me: error: cannot find input file: $f" >&2;}
+   { (exit 1); exit 1; }; }
+	 fi;;
+      esac
+    done` || { (exit 1); exit 1; }
+  # Remove the trailing spaces.
+  sed 's/[	 ]*$//' $ac_file_inputs >$tmp/in
+
+_ACEOF
+
+# Transform confdefs.h into two sed scripts, `conftest.defines' and
+# `conftest.undefs', that substitutes the proper values into
+# config.h.in to produce config.h.  The first handles `#define'
+# templates, and the second `#undef' templates.
+# And first: Protect against being on the right side of a sed subst in
+# config.status.  Protect against being in an unquoted here document
+# in config.status.
+rm -f conftest.defines conftest.undefs
+# Using a here document instead of a string reduces the quoting nightmare.
+# Putting comments in sed scripts is not portable.
+#
+# `end' is used to avoid that the second main sed command (meant for
+# 0-ary CPP macros) applies to n-ary macro definitions.
+# See the Autoconf documentation for `clear'.
+cat >confdef2sed.sed <<\_ACEOF
+s/[\\&,]/\\&/g
+s,[\\$`],\\&,g
+t clear
+: clear
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 (][^	 (]*\)\(([^)]*)\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1\2${ac_dC}\3${ac_dD},gp
+t end
+s,^[	 ]*#[	 ]*define[	 ][	 ]*\([^	 ][^	 ]*\)[	 ]*\(.*\)$,${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD},gp
+: end
+_ACEOF
+# If some macros were called several times there might be several times
+# the same #defines, which is useless.  Nevertheless, we may not want to
+# sort them, since we want the *last* AC-DEFINE to be honored.
+uniq confdefs.h | sed -n -f confdef2sed.sed >conftest.defines
+sed 's/ac_d/ac_u/g' conftest.defines >conftest.undefs
+rm -f confdef2sed.sed
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >>conftest.undefs <<\_ACEOF
+s,^[	 ]*#[	 ]*undef[	 ][	 ]*[a-zA-Z_][a-zA-Z_0-9]*,/* & */,
+_ACEOF
+
+# Break up conftest.defines because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #define templates only if necessary.' >>$CONFIG_STATUS
+echo '  if grep "^[	 ]*#[	 ]*define" $tmp/in >/dev/null; then' >>$CONFIG_STATUS
+echo '  # If there are no defines, we may have an empty if/fi' >>$CONFIG_STATUS
+echo '  :' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.defines >/dev/null
+do
+  # Write a limited-size here document to $tmp/defines.sed.
+  echo '  cat >$tmp/defines.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#define' lines.
+  echo '/^[	 ]*#[	 ]*define/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.defines >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/defines.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.defines >conftest.tail
+  rm -f conftest.defines
+  mv conftest.tail conftest.defines
+done
+rm -f conftest.defines
+echo '  fi # grep' >>$CONFIG_STATUS
+echo >>$CONFIG_STATUS
+
+# Break up conftest.undefs because some shells have a limit on the size
+# of here documents, and old seds have small limits too (100 cmds).
+echo '  # Handle all the #undef templates' >>$CONFIG_STATUS
+rm -f conftest.tail
+while grep . conftest.undefs >/dev/null
+do
+  # Write a limited-size here document to $tmp/undefs.sed.
+  echo '  cat >$tmp/undefs.sed <<CEOF' >>$CONFIG_STATUS
+  # Speed up: don't consider the non `#undef'
+  echo '/^[	 ]*#[	 ]*undef/!b' >>$CONFIG_STATUS
+  # Work around the forget-to-reset-the-flag bug.
+  echo 't clr' >>$CONFIG_STATUS
+  echo ': clr' >>$CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.undefs >>$CONFIG_STATUS
+  echo 'CEOF
+  sed -f $tmp/undefs.sed $tmp/in >$tmp/out
+  rm -f $tmp/in
+  mv $tmp/out $tmp/in
+' >>$CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.undefs >conftest.tail
+  rm -f conftest.undefs
+  mv conftest.tail conftest.undefs
+done
+rm -f conftest.undefs
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+  # Let's still pretend it is `configure' which instantiates (i.e., don't
+  # use $as_me), people would be surprised to read:
+  #    /* config.h.  Generated by config.status.  */
+  if test x"$ac_file" = x-; then
+    echo "/* Generated by configure.  */" >$tmp/config.h
+  else
+    echo "/* $ac_file.  Generated by configure.  */" >$tmp/config.h
+  fi
+  cat $tmp/in >>$tmp/config.h
+  rm -f $tmp/in
+  if test x"$ac_file" != x-; then
+    if diff $ac_file $tmp/config.h >/dev/null 2>&1; then
+      { echo "$as_me:$LINENO: $ac_file is unchanged" >&5
+echo "$as_me: $ac_file is unchanged" >&6;}
+    else
+      ac_dir=`(dirname "$ac_file") 2>/dev/null ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+      { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+      rm -f $ac_file
+      mv $tmp/config.h $ac_file
+    fi
+  else
+    cat $tmp/config.h
+    rm -f $tmp/config.h
+  fi
+done
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+#
+# CONFIG_COMMANDS section.
+#
+for ac_file in : $CONFIG_COMMANDS; do test "x$ac_file" = x: && continue
+  ac_dest=`echo "$ac_file" | sed 's,:.*,,'`
+  ac_source=`echo "$ac_file" | sed 's,[^:]*:,,'`
+  ac_dir=`(dirname "$ac_dest") 2>/dev/null ||
+$as_expr X"$ac_dest" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_dest" : 'X\(//\)[^/]' \| \
+	 X"$ac_dest" : 'X\(//\)$' \| \
+	 X"$ac_dest" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$ac_dest" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+  { if $as_mkdir_p; then
+    mkdir -p "$ac_dir"
+  else
+    as_dir="$ac_dir"
+    as_dirs=
+    while test ! -d "$as_dir"; do
+      as_dirs="$as_dir $as_dirs"
+      as_dir=`(dirname "$as_dir") 2>/dev/null ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| \
+	 .     : '\(.\)' 2>/dev/null ||
+echo X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ s//\1/; q; }
+  	  /^X\(\/\/\)[^/].*/{ s//\1/; q; }
+  	  /^X\(\/\/\)$/{ s//\1/; q; }
+  	  /^X\(\/\).*/{ s//\1/; q; }
+  	  s/.*/./; q'`
+    done
+    test ! -n "$as_dirs" || mkdir $as_dirs
+  fi || { { echo "$as_me:$LINENO: error: cannot create directory \"$ac_dir\"" >&5
+echo "$as_me: error: cannot create directory \"$ac_dir\"" >&2;}
+   { (exit 1); exit 1; }; }; }
+
+  ac_builddir=.
+
+if test "$ac_dir" != .; then
+  ac_dir_suffix=/`echo "$ac_dir" | sed 's,^\.[\\/],,'`
+  # A "../" for each directory in $ac_dir_suffix.
+  ac_top_builddir=`echo "$ac_dir_suffix" | sed 's,/[^\\/]*,../,g'`
+else
+  ac_dir_suffix= ac_top_builddir=
+fi
+
+case $srcdir in
+  .)  # No --srcdir option.  We are building in place.
+    ac_srcdir=.
+    if test -z "$ac_top_builddir"; then
+       ac_top_srcdir=.
+    else
+       ac_top_srcdir=`echo $ac_top_builddir | sed 's,/$,,'`
+    fi ;;
+  [\\/]* | ?:[\\/]* )  # Absolute path.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir ;;
+  *) # Relative path.
+    ac_srcdir=$ac_top_builddir$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_builddir$srcdir ;;
+esac
+
+# Do not use `cd foo && pwd` to compute absolute paths, because
+# the directories may not exist.
+case `pwd` in
+.) ac_abs_builddir="$ac_dir";;
+*)
+  case "$ac_dir" in
+  .) ac_abs_builddir=`pwd`;;
+  [\\/]* | ?:[\\/]* ) ac_abs_builddir="$ac_dir";;
+  *) ac_abs_builddir=`pwd`/"$ac_dir";;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_builddir=${ac_top_builddir}.;;
+*)
+  case ${ac_top_builddir}. in
+  .) ac_abs_top_builddir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_builddir=${ac_top_builddir}.;;
+  *) ac_abs_top_builddir=$ac_abs_builddir/${ac_top_builddir}.;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_srcdir=$ac_srcdir;;
+*)
+  case $ac_srcdir in
+  .) ac_abs_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_srcdir=$ac_srcdir;;
+  *) ac_abs_srcdir=$ac_abs_builddir/$ac_srcdir;;
+  esac;;
+esac
+case $ac_abs_builddir in
+.) ac_abs_top_srcdir=$ac_top_srcdir;;
+*)
+  case $ac_top_srcdir in
+  .) ac_abs_top_srcdir=$ac_abs_builddir;;
+  [\\/]* | ?:[\\/]* ) ac_abs_top_srcdir=$ac_top_srcdir;;
+  *) ac_abs_top_srcdir=$ac_abs_builddir/$ac_top_srcdir;;
+  esac;;
+esac
+
+
+  { echo "$as_me:$LINENO: executing $ac_dest commands" >&5
+echo "$as_me: executing $ac_dest commands" >&6;}
+  case $ac_dest in
+    default ) test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h ;;
+  esac
+done
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF
+
+{ (exit 0); exit 0; }
+_ACEOF
+chmod +x $CONFIG_STATUS
+ac_clean_files=$ac_clean_files_save
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || { (exit 1); exit 1; }
+fi
+
Index: /main/tags/3.01/mgpp/configure.in
===================================================================
--- /main/tags/3.01/mgpp/configure.in	(revision 21112)
+++ /main/tags/3.01/mgpp/configure.in	(revision 21112)
@@ -0,0 +1,141 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(text/MGQuery.cpp)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mgpp
+VERSION=1.3
+
+LDFLAGS=
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lg:
+AC_CHECK_LIB(g, main)
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+dnl Replace `main' with a function in -lstdc++:
+AC_CHECK_LIB(stdc++, main)
+dnl Replace `main' with a function in -lcrypt:
+AC_CHECK_LIB(crypt, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h crypt.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize strerror)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+dnl see if gettext is in libc. (try the textdomain() function).
+AC_CHECK_FUNC(textdomain, AC_DEFINE(ENABLE_NLS ,1) )
+
+
+# we need to set an extended JNI path for Mac OS/Darwin, as jni.h is on
+# a non-standard Path
+AC_MSG_CHECKING(for Apple MacOS X/Darwin)
+if test "`(uname -s) 2> /dev/null`" = 'Darwin'; then
+  AC_MSG_RESULT(yes)
+  JNIINC="-I/System/Library/Frameworks/JavaVM.framework/Headers/ "
+  JNISUFFIX="jnilib"
+  JNIFLAGS="-dynamiclib -fno-common -framework JavaVM"
+else
+  AC_MSG_RESULT(no)
+  JNIINC=""
+  JNISUFFIX="so"
+  JNIFLAGS="-shared"
+fi
+AC_SUBST(JNIINC)
+AC_SUBST(JNISUFFIX)
+AC_SUBST(JNIFLAGS)
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  AC_LIBOBJ(gmalloc)
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+fp_WITH_GNU_READLINE
+
+# text for endianness
+AC_C_BIGENDIAN
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  AC_LIBOBJ(xmalloc)
+  AC_LIBOBJ(error)
+fi
+
+
+# ---------------------------------------------------------------------------
+# see if the user wants to override use of long long
+AC_ARG_ENABLE(override-longlong,
+[  --enable-override-longlong
+                          overide use of long long with GCC],
+  AC_DEFINE(DISABLE_LONG_LONG))
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile text/Makefile lib/Makefile 
+   jni/Makefile java/org/greenstone/mgpp/Makefile], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /main/tags/3.01/mgpp/install-sh
===================================================================
--- /main/tags/3.01/mgpp/install-sh	(revision 21112)
+++ /main/tags/3.01/mgpp/install-sh	(revision 21112)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPDocInfo.java	(revision 21112)
@@ -0,0 +1,20 @@
+package org.greenstone.mgpp;
+
+/** DocInfo 'struct' class for a search result from mgpp
+ * used by MGPPQueryResult
+ *
+ * @See MGPPQueryResult
+ */
+public class MGPPDocInfo {
+
+    public long num_=0;
+    public float rank_=0;
+
+    public MGPPDocInfo(long doc, float rank) {
+	num_=doc;
+	rank_=rank;
+    }
+    public String toString(){
+	return ""+num_+"("+rank_+")";
+    }
+}
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPPassesWrapper.java	(revision 21112)
@@ -0,0 +1,146 @@
+/*
+ *    MGPassesWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.mgpp;
+
+
+/** java wrapper class for access to gs3_mgpp_passes in C
+ *
+ * the native side implemented in MGPPPassesWrapperImpl.c
+ */
+
+public class MGPPPassesWrapper
+{
+    static {
+	System.loadLibrary("mgpppassjni");
+	initIDs();
+    }
+    
+    //static final public char INVF_LEVEL_1 = '1';
+    //static final public char INVF_LEVEL_2 = '2';
+    //static final public char INVF_LEVEL_3 = '3';
+    
+    static final public int TEXT_PASS_1 = 0;
+    static final public int TEXT_PASS_2 = 1;
+    static final public int INDEX_PASS_1 = 2;
+    static final public int INDEX_PASS_2 = 3;
+    static final public int SPECIAL_PASS = 4;
+
+    //static final public int NO_STEM_OR_CASE = 0;
+    //static final public int CASE_ONLY = 1;
+    //static final public int STEM_ONLY = 2;
+    //static final public int STEM_AND_CASE = 3;
+
+    //static final public String STEMMER_ENGLISH = "english";
+    //static final public String STEMMER_FRENCH = "french";
+    //static final public String STEMMER_LOVIN = "lovin";
+    //static final public String STEMMER_SIMPLE_FRENCH = "simple-french";
+
+    //static final private char END_OF_DOCUMENT = (char) 2;
+
+    public MGPPPassesWrapper() {
+	initCSide();
+    }
+
+    /** initialise the pass through the documents */
+    public native boolean init();
+    
+    /** add a pass declaration */
+    public void addPass(int pass) {
+	switch (pass) {
+	case TEXT_PASS_1:
+	    addPass('T','1');
+	    break;
+	case TEXT_PASS_2:
+	    addPass('T','2');	    
+	    break;
+	case INDEX_PASS_1:
+	    addPass('I','1');
+	    break;
+	case INDEX_PASS_2:
+	    addPass('I','2');
+	    break;
+	case SPECIAL_PASS:
+	    addPass('S','1');
+	    break;
+	}
+    }
+    /** set the base path */
+    public native void setBasePath(String basepath);
+    /** set the file name */
+    public native void setFileName(String filename);
+
+    /** set the Document tag */
+    public native void setDocumentTag(String tag);
+
+    /** add a level tag */
+    public native void addLevelTag(String tag);
+
+    /** set the index level (default word level) */
+    public native void setIndexLevel(String tag);
+    
+    /** Maximum amount of memory to use for  the index  pass-2  file
+	inversion  in  megabytes.
+    */
+    public native void setInversionMemLimit(int limit);
+
+    /** process one or more MGPP documents */
+    public boolean processDocument(String doc_text) {
+	try {
+	    processMGPPDocument(doc_text.getBytes("UTF-8"));
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    return false;
+	}
+	return true;
+    }
+   
+    /** finalise the pass through the documents */
+    public native boolean finish();
+
+    /** get the exit value once finished */
+    public native int exitValue();
+
+    /** initialises field and method IDs for java side to enable access on C side */
+    private static native void initIDs();
+
+    /** initialises any C side stuff */
+    private native boolean initCSide();
+
+    private native void addPass(char pass_type, char pass_num);
+
+    /** process a MGPP document */
+    private native boolean processMGPPDocument(byte[] text);
+
+    public static void main(String []args) {
+
+	String doc = "<Document>hello there kath<Title>this is a title</Title></Document><Document>my name is helen</Document>";
+
+	MGPPPassesWrapper wrapper = new MGPPPassesWrapper();
+	wrapper.addPass(TEXT_PASS_1);
+	wrapper.addPass(TEXT_PASS_2);
+	wrapper.setDocumentTag("Document");
+	wrapper.setBasePath("/tmp/kjdon/index");
+	wrapper.setFileName("idx");
+
+	wrapper.init();
+	wrapper.processDocument(doc);
+	wrapper.finish();
+
+    }
+}
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPQueryResult.java	(revision 21112)
@@ -0,0 +1,108 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** a java version of QueryResult for mgpp
+ *
+ * contains get methods for the java side, and simple
+ * set methods to be called from c++ side
+ *
+ */
+public class MGPPQueryResult {
+
+    /** the list of MGPPDocInfo */
+    protected Vector docs_=null;
+    /** the list of MGPPTermInfo */
+    protected Vector terms_=null;
+    /** the total number of docs found - not likely to be the size of docs_*/
+    protected long total_num_docs_=0;
+    /** whether a syntax error has occurred */
+    protected boolean syntax_error_ = false;
+    
+    MGPPQueryResult() {
+	docs_ = new Vector();
+	terms_ = new Vector();
+    }
+    /** clear the info from the last query - should be called before setting any new docs/terms */
+    public void clear() {
+	total_num_docs_=0;
+	docs_.clear();
+	terms_.clear();
+	syntax_error_ = false;
+    }
+    /** returns the result as a String - useful for printing out results */
+    public String toString() {
+	
+	String result = "";
+	result += "docs (ranks): ";
+	for (int i=0; i<docs_.size(); i++) {
+	    result += ((MGPPDocInfo)docs_.elementAt(i)).toString()+", ";
+	}
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((MGPPTermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+
+	return result;
+    }
+    /** a shorter representation - just terms and total docs - not the 
+	individual docnums and ranks */
+    public String toShortString() {
+	String result = "";
+	result += "\nterms: ";
+	for (int i=0; i<terms_.size(); i++) {
+	    result += ((MGPPTermInfo)terms_.elementAt(i)).toString()+", ";
+	}
+	result += "\nactual number of docs found = "+total_num_docs_;
+	return result;
+    }
+
+    // set methods used by c++ code
+    
+    public void setTotalDocs(long num) {
+	total_num_docs_=num;
+    }
+
+    public void addDoc(long doc, float rank) {
+	MGPPDocInfo doc_info = new MGPPDocInfo(doc, rank);
+	docs_.add(doc_info);
+    }
+
+    public void addTerm(String term, String tag, int stem, long match, long freq, String[] equivs) {
+	MGPPTermInfo ti = new MGPPTermInfo();
+	ti.term_=term;
+	ti.tag_=tag;
+	ti.stem_method_=stem;
+	ti.match_docs_=match;
+	ti.term_freq_=freq;
+	if (equivs!=null) {
+	  for (int i=0; i<equivs.length; i++) {
+	    ti.equiv_terms_.add(equivs[i]);
+	  }
+	}
+	terms_.add(ti);
+    }
+
+    // get methods for the java side
+    public Vector getDocs() {
+	return docs_;
+    }
+
+    public Vector getTerms() {
+	return terms_;
+    }
+
+    public long getTotalDocs() {
+	return total_num_docs_;
+    }
+     
+    public void setSyntaxError(boolean error) {
+	syntax_error_ = error;
+    }
+
+    public boolean hasSyntaxError() {
+	return syntax_error_;
+    }
+}
+
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPTermInfo.java	(revision 21112)
@@ -0,0 +1,46 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** TermInfo 'struct' class for a search result from mgpp
+ * used by MGPPQueryResult
+ * 
+ *
+ *@see MGPPQueryResult
+ */
+public class MGPPTermInfo {
+
+    /** the term itself */
+    public String term_=null;
+    /** the tag - level or metadata - for which the query was done */
+    public String tag_=null;
+    /** what stem and casefold method was used
+	0 = none
+	1 = casefold only
+	2 = stem only
+	3 = casefold and stem */
+    public int stem_method_=0;
+    /** the number of documents containing this term (where document is defined by the tag data */
+    public long match_docs_=0;
+    /** overall term freq  - word level*/
+    public long term_freq_=0;
+    /** list of stemmed and casefolded equivalent terms - if stem_method_ is non-zero 
+      - Vector of Strings */
+    public Vector equiv_terms_=null;
+
+    public MGPPTermInfo() {
+	equiv_terms_ = new Vector();
+    }
+	
+    /** output the class as a string */
+    public String toString() {
+	String result="";
+	result +="<"+tag_+">\""+term_+"\"stem("+stem_method_+")docs("+match_docs_;
+	result +=")freq("+term_freq_+")equiv terms(";
+	for (int i=0; i<equiv_terms_.size(); i++) {
+	    result += (String)equiv_terms_.elementAt(i)+", ";
+	}
+	result += ")";
+	return result;
+    }
+}
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/MGPPWrapper.java	(revision 21112)
@@ -0,0 +1,102 @@
+
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+
+/** java wrapper class for access to mgpp in C++
+ *
+ * the native side implemented in MGPPWrapperImpl.cpp
+ * uses MGPPQueryResult to hold the result of a query. the 
+ * result of a getDocument is a String
+ * uses the jni
+ *
+ *@see MGPPQueryResult
+ */
+public class MGPPWrapper {
+  
+    /** the query result
+     * will be filled in by runQuery
+     */
+    protected MGPPQueryResult mgpp_query_result_=null; 
+    /** pointer to c++ MGPPWrapperData class - cached indexData and queryInfo
+     */
+    protected long mgpp_data_ptr_=0;
+
+    static {
+	System.loadLibrary("mgppjni");
+	initIDs();
+    }
+    
+    public MGPPWrapper() {
+	mgpp_query_result_ = new MGPPQueryResult();
+	initCppSide();
+    }
+
+    /** initialises field and method IDs for java side to enable access on
+	C++ side */
+    private static native void initIDs();
+
+    /** initialises the mgpp_data_ptr_ */
+    private native boolean initCppSide();
+
+    /** returns a document: number docnum at level level 
+     * text_path path should provide
+     * the absolute location of the mgpp text files eg ..../index/text/demo
+     * returns the doc in utf-8
+     */
+    public native String getDocument(String text_path,
+				     String level,
+				     long docnum );
+
+    /** the public method - converts between utf-8 and unicode
+     */
+    // public String getDocument(String text_path,
+    //		      String level,
+    //		      long docnum ) {
+
+    //String utf8_doc = getUtF8Document(base_dir, text_path, level, docnum);
+	
+    /** load up the data structure for index 
+     * - maintains state between requests as can be slow  
+     * index_path should provide
+     * the absolute location of the mgpp index files eg ..../index/tt/demo 
+     */
+    public native boolean loadIndexData(String index_path);
+    /** unloads the data */
+    public native boolean unloadIndexData();
+
+    // query param methods
+
+    /** if on=true, sets default stemming on  - its off by default*/
+    public native void setStem(boolean on);
+    /** if on=true, sets default casefolding on - its off by default */
+    public native void setCase(boolean on);
+    /** default is 50 */
+    public native void setMaxDocs(int num);
+    /** if on=true, sorts by rank, otherwise returns in build order - 
+	default is on */
+    public native void setSortByRank(boolean on);
+    /** if on=true, a query returns term freq info - default is on */
+    public native void setReturnTerms(boolean on);
+    /** sets the granularity of the  query  - default 'Document' */
+    public native void setQueryLevel(String level);
+    /** sets the granularity of the result - default 'Document' */
+    public native void setReturnLevel(String level);
+    /** sets the default boolean operator - AND(=1)/OR(=0) */
+    public native void setMatchMode(int mode);
+
+    /** returns a string with all the current query param settings */
+    public native String getQueryParams();
+
+    /** actually carry out the query. Use the set methods to set any query
+	params 
+    * writes the result to query_result
+    */
+    public native void runQuery(String query_string); 
+
+    /** get the result out of the wrapper */
+    public MGPPQueryResult getQueryResult() {
+	return mgpp_query_result_;
+    }
+    
+}
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Makefile.in
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Makefile.in	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Makefile.in	(revision 21112)
@@ -0,0 +1,63 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = java/org/greenstone/mgpp
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+JAVAC = $(JAVA_HOME)/bin/javac
+JAVAH = $(JAVA_HOME)/bin/javah
+JAR = $(JAVA_HOME)/bin/jar
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS =  -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+MGPPHOME=../../../..
+JAVACLASSDIR=./classes
+JAVASRCDIR=../../..
+JAVACOPTIONS= -deprecation -g -O
+JAVACLASSPATH=$(JAVACLASSDIR):
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+
+all: setup compile
+
+setup:
+	mkdir -p $(JAVACLASSDIR)
+
+compile: setup
+	$(JAVAC) -d $(JAVACLASSDIR) -sourcepath $(JAVASRCDIR) $(JAVACOPTIONS) *.java
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGPPHOME)/jni org.greenstone.mgpp.MGPPWrapper
+	$(JAVAH) -classpath $(JAVACLASSPATH) -d $(MGPPHOME)/jni org.greenstone.mgpp.MGPPPassesWrapper
+
+	$(JAR) cf $(MGPPHOME)/mgpp.jar -C $(JAVACLASSDIR) org
+
+clean:
+	rm -rf $(JAVACLASSDIR) $(MGPPHOME)/mgpp.jar $(MGPPHOME)/jni/org_greenstone_mgpp_MGPPWrapper.h $(MGPPHOME)/jni/org_greenstone_mgpp_MGPPPassesWrapper.h
+
+install:
+
+distclean: clean
+	rm -f Makefile
+
+
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Queryer.java
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Queryer.java	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/Queryer.java	(revision 21112)
@@ -0,0 +1,184 @@
+package org.greenstone.mgpp;
+
+import java.util.Vector;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+/** Queryer - java port of c++ Queryer 
+ * - uses MGPPWrapper to access mgpp
+ * only has get document and search - full text browse not implemented
+ *
+ * run as:
+ * java org.greenstone.mgpp.Queryer <basedir> <indexdir> <textdir>
+ *@see MGPPWrapper
+ */
+public class Queryer {
+  
+  /** outputs to std out a help message describing the commands used by 
+   * Queryer 
+   */
+  public void printHelp() {
+	
+	System.out.println( "commands available are:\n"+
+			    "\t.q\t\tquit\n"+
+			    "\t.h\t\tprint this help message\n"+
+			    "\t.d\t\tprint the current query parameter settings\n" +
+			    "\t.i<level>\tchange the search level to level\n"+
+			    "\t.l<level>\tchange the result level to level\n"+
+			    "\t.r0/.r1\t\tranking off/on\n"+
+			    "\t.t0/.t1\t\tquery type some/all\n"+
+			    "\t.c0/.c1\t\tcasefolding off/on\n"+
+			    "\t.s0/.s1\t\tstemming off/on\n"+
+			    "\t.o0/.o1\t\tshort output off/on\n"+
+			    "\t.m<num>\t\tset max docs to return to num\n\n"+
+			    "\t.p<docnum>\tprint document docnum\n"+
+			    "\t<query string>\t do a query\n");
+    }
+
+   
+    public static void main(String[] args) {
+	if (args.length != 3) {
+	    System.out.println("Usage: java org.greenstone.mgpp.Queryer <basedir> <indexdir> <textdir>");
+	    return;
+	}
+	
+	Queryer self = new Queryer();
+	
+	String base_dir = args[0];
+	String text_path = args[2];
+	String index_path = args[1];
+
+	text_path = base_dir + File.separatorChar+text_path;
+	index_path = base_dir  + File.separatorChar+ index_path;
+	
+	// the jni class to access mgpp stuff
+	MGPPWrapper wrapper = new MGPPWrapper();
+	wrapper.loadIndexData(index_path);
+	
+	// the return level
+	String level = "Section";
+
+	System.out.println("Welcome to Java Queryer :-)");
+	self.printHelp();
+
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	boolean shortOutput=false;
+
+	String command; // user input
+	char x; // the command letter
+	String data=null; // any auxiliary input to a command
+	while (true) {
+	  System.out.print(">"); // the prompt
+            try {
+                command = br.readLine();
+                command=command.trim();
+		if (command.startsWith(".")) {
+		    // a system command
+		    x = command.charAt(1);
+		    if (command.length() > 2) {
+			data = command.substring(2);
+			data = data.trim();
+			
+		    }
+		    switch (x) {
+		    case 'q': // clean up and exit
+			wrapper.unloadIndexData();
+			return;  
+		    case 'h': // print help message
+			self.printHelp();
+			break;
+		    case 'd': // print query param settings
+			String info = wrapper.getQueryParams();
+			System.out.println(info);
+			break;
+		    case 'p': // print doc
+			int docnum = Integer.parseInt(data);
+			String doc = wrapper.getDocument(text_path, 
+							 level, docnum);
+			System.out.println(doc);
+			break;
+		    case 'm': //match docs
+			int match = Integer.parseInt(data);
+			wrapper.setMaxDocs(match);
+			break;
+		    case 's': // set stem on/off
+			int stem = Integer.parseInt(data);
+			if (stem==0 ){
+			    wrapper.setStem(false);
+			} else if(stem==1) {
+			    wrapper.setStem(true);
+			} else {
+			    System.err.println("Error: stem should be 0 or 1");
+			}
+			break;
+		    case 'c': // set case on/off
+			int casef = Integer.parseInt(data);
+			if (casef==0) {
+			    wrapper.setCase(false);
+			} else if (casef==1) {
+			    wrapper.setCase(true);
+			} else {
+			    System.err.println("Error: case should be 0 or 1");
+			}
+			break;
+		    case 'i': // set search level
+			wrapper.setQueryLevel(data);
+			break;
+		    case 'l': // set result level
+			wrapper.setReturnLevel(data);
+			level = data;
+			break; 
+		    case 'r': // set rank on/off
+			int rank = Integer.parseInt(data);
+			if (rank==0) {
+			    wrapper.setSortByRank(false);
+			} else if (rank==1) {
+			    wrapper.setSortByRank(true);
+			} else {
+			    System.err.println("Error: rank should be 0 or 1");
+			}
+			break;
+		    case 't': // set query type some/all
+			int type = Integer.parseInt(data);
+			if (type==0 || type==1) {
+			    wrapper.setMatchMode(type);
+			} else {
+			    System.err.println("Error: type should be 0 (some) or 1 (all)");
+			}
+			break;
+		    case 'o': // set output short/long
+			int output = Integer.parseInt(data);
+			if (output==0) {
+			    shortOutput = false;
+			} else if (output==1) {
+			    shortOutput = true;
+			}else {
+			    System.err.println("Error: output should be 0 or 1");
+			}
+			break;
+		    }
+		}
+		else {
+		    // a query
+		    wrapper.runQuery(command);
+		    MGPPQueryResult res = wrapper.getQueryResult();
+		    if (res.hasSyntaxError()) {
+		      System.out.println("invalid syntax error\n");
+		    } else {
+		      if (shortOutput) {
+			System.out.println(res.toShortString());
+		      } else {
+			System.out.println(res.toString());
+		      }
+		    }
+		}
+		    
+	    } catch (Exception e) {
+		System.out.println("Queryer error: "+e.getClass() + " "+e.getMessage());
+	    }
+
+	}
+    }
+
+
+}
Index: /main/tags/3.01/mgpp/java/org/greenstone/mgpp/winMake.bat
===================================================================
--- /main/tags/3.01/mgpp/java/org/greenstone/mgpp/winMake.bat	(revision 21112)
+++ /main/tags/3.01/mgpp/java/org/greenstone/mgpp/winMake.bat	(revision 21112)
@@ -0,0 +1,45 @@
+@echo off
+
+set MGPPHOME=..\..\..\..
+
+rem ---- Where to put class files ----
+set JAVACLASSDIR=classes
+
+rem ---- Name and location of java programs ----
+set JAVAC="%JAVA_HOME%\bin\javac"
+set JAVAH="%JAVA_HOME%\bin\javah"
+set JAVA="%JAVA_HOME%\bin\java"
+set JAVADOC="%JAVA_HOME%\bin\javadoc"
+set JAR="%JAVA_HOME%\bin\jar"
+
+set JAVACOPTIONS= -deprecation -g -O
+
+
+if ""%1"" == """" goto compile
+if ""%1"" == ""compile"" goto compile
+if ""%1"" == ""install"" goto install
+if ""%1"" == ""clean"" goto clean
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:compile
+	if not exist %JAVACLASSDIR% mkdir %JAVACLASSDIR%
+	echo Compiling...
+        %JAVAC% -d %JAVACLASSDIR% %JAVACOPTIONS% *.java
+        %JAVAH% -classpath %JAVACLASSDIR% -d %MGPPHOME%\jni org.greenstone.mgpp.MGPPWrapper
+        %JAR% cf %MGPPHOME%\mgpp.jar -C %JAVACLASSDIR% org
+	goto done
+
+:install
+	goto done
+
+:clean
+	echo Cleaning up...
+	if exist %JAVACLASSDIR% rmdir /S /Q %JAVACLASSDIR%
+        if exist %MGPPHOME%\mgpp.jar del %MGPPHOME%\mgpp.jar
+        if exist %MGPPHOME%\jni\org_greenstone_mgpp_MGPPWrapper.h del %MGPPHOME%\jni\org_greenstone_mgpp_MGPPWrapper.h
+	goto done
+
+:done
Index: /main/tags/3.01/mgpp/jni/MGPPPassesWrapperImpl.cpp
===================================================================
--- /main/tags/3.01/mgpp/jni/MGPPPassesWrapperImpl.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/MGPPPassesWrapperImpl.cpp	(revision 21112)
@@ -0,0 +1,190 @@
+/*
+ *    MGPassesWrapperImpl.c
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <jni.h>
+#include <assert.h>
+#include "sysfuncs.h"
+#include "org_greenstone_mgpp_MGPPPassesWrapper.h"
+
+#include "mgpp_passes_4jni.h"
+#include "mg_files.h"
+
+/* if we need to use java objects, we should initialise their field ids here*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_initIDs(JNIEnv *j_env, jclass j_cls)
+{
+  return;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_initCSide(JNIEnv *j_env, jobject j_obj)
+{
+  clear_variables();
+  return true;  
+}  
+
+/* add a pass type T1, T2, I1, I2, S */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_addPass(JNIEnv *j_env, 
+					       jobject j_obj, 
+					       jchar j_pass_type,
+					       jchar j_pass_num)
+{
+  /* get the level as a c char */
+  const char pass_type = j_pass_type;
+  const char pass_num = j_pass_num;
+  add_pass(pass_type, pass_num);
+  
+}
+
+/* Set the filename */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setFileName(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_filename)
+{
+  /* Get the filename as a C string */
+  const char* filename = j_env->GetStringUTFChars(j_filename, NULL);
+ 
+  assert(filename != NULL);
+  set_filename(filename);
+  
+  /* Release the string */
+ j_env->ReleaseStringUTFChars(j_filename, filename);
+
+}
+
+/* Set the base path */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setBasePath(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_basepath)
+{
+  /* Get the base_path as a C string */
+  const char* basepath = j_env->GetStringUTFChars(j_basepath, NULL);
+  assert(basepath != NULL);
+ 
+  set_basepath(basepath);
+  
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_basepath, basepath);
+
+}
+
+/* set the Document tag */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setDocumentTag(JNIEnv *j_env, 
+						      jobject j_obj,
+						      jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+
+  set_document_tag(tag);
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+
+}
+
+/* add a level tag */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_addLevelTag(JNIEnv *j_env, 
+						   jobject j_obj,
+						   jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+  add_level_tag(tag);
+
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+}
+
+/* set the index level (default word level) */
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setIndexLevel(JNIEnv *j_env, 
+						     jobject j_obj,
+						     jstring j_tag)
+{
+  const char* tag = j_env->GetStringUTFChars(j_tag, NULL);
+  set_index_level(tag);
+
+  /* Release the string */
+  j_env->ReleaseStringUTFChars(j_tag, tag);
+
+  
+}
+
+
+/** Maximum amount of memory to use for  the index  pass-2  file
+    inversion  in  megabytes.
+*/
+JNIEXPORT void JNICALL
+Java_org_greenstone_mgpp_MGPPPassesWrapper_setInversionMemLimit(JNIEnv *j_env, 
+							    jobject j_obj, 
+							    jint j_limit) {
+  int limit = j_limit;
+  set_inversion_limit(limit);
+}
+
+
+/* initialise the pass through the documents. must be called after all
+   the set methods 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_init(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  init_driver();
+  return true;
+}
+
+
+/* process one or more documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_processMGPPDocument(JNIEnv *j_env, 
+						       jobject j_obj,
+						       jbyteArray j_doc_text) {
+  /* Get the text as a C string */
+  int length = j_env->GetArrayLength(j_doc_text);
+  u_char * text_buffer = (u_char *)j_env->GetByteArrayElements(j_doc_text, NULL);
+  process_document(text_buffer, length);
+  /* Release the string */
+  // why doesn't this work in c++ when it works in C?????
+  //j_env->ReleaseByteArrayElements(j_doc_text, text_buffer, 0);
+  return true;
+}
+
+/* finalise the pass through the documents */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_finish(JNIEnv *j_env, 
+					    jobject j_obj) {
+  
+  finalise_driver();
+  return true;
+}
+
+/** get the exit value once finished */
+JNIEXPORT jint JNICALL 
+Java_org_greenstone_mgpp_MGPPPassesWrapper_exitValue(JNIEnv *j_env, 
+					    jobject j_obj) {
+
+  return get_exit_value();
+}
Index: /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.cpp
===================================================================
--- /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.cpp	(revision 21112)
@@ -0,0 +1,503 @@
+#ifdef __WIN32__
+#include <WIN32cfg.h>
+#include <strstream>
+#include <sstream>
+#else
+#ifdef __APPLE__
+#include <strstream>
+#include <sstream>
+#else
+#include <sstream>
+#endif
+#endif
+
+#include <jni.h>
+#include "org_greenstone_mgpp_MGPPWrapper.h"
+#include "MGPPWrapperImpl.h"
+#include "TextGet.h"
+#include "GSDLQueryParser.h"
+#include "MGQuery.h"
+  
+MGPPWrapperData::MGPPWrapperData() {
+  indexData = new IndexData();
+  queryInfo = new QueryInfo();
+
+  if (queryInfo==NULL) {
+    cerr<<"couldn't allocate new query info\n";
+    if (indexData!=NULL) {
+      delete indexData;
+    }
+  }
+
+  // set all the default params
+  SetCStr(queryInfo->docLevel, "Document"); // the level to search at
+  queryInfo->maxDocs = 50;
+  queryInfo->sortByRank = true;
+  queryInfo->exactWeights = false;
+  queryInfo->needRankInfo = true;
+  queryInfo->needTermFreqs = true;
+
+  UCArrayClear(level);
+  SetCStr(level, "Document"); // the level to return docs at
+  defaultStemMethod=0;
+  defaultBoolCombine=0;
+  
+}
+
+MGPPWrapperData::~MGPPWrapperData() { 
+  if (indexData !=NULL) {
+    delete indexData;
+  }
+  if (queryInfo !=NULL) {
+    delete queryInfo;
+  }
+}
+
+// ********************************************
+// initialisation stuff
+// ********************************************
+
+// cached ids for java stuff 
+jfieldID FID_mgpp_data = NULL; // MGPPWrapperData
+jfieldID FID_query_result = NULL; // MGPPQueryResult
+jmethodID MID_addDoc=NULL; // MGPPQueryResult.addDoc()
+jmethodID MID_addTerm=NULL; // MGPPQueryResult.addTerm()
+jmethodID MID_setTotalDocs=NULL; // MGPPQueryResult.setTotalDocs()
+jmethodID MID_clearResult=NULL; //MGPPQueryResult.clear()
+jmethodID MID_setSyntaxError=NULL; // MGPPQueryResult.setSyntaxError()
+jclass CID_String=NULL; // class ID of String
+
+/* to access objects and methods on java side, need their field/method ids - 
+ this initialises them at the start to avoid recalculating them each time they
+ are needed 
+Note: the descriptors need to be exactly right, otherwise you get an error 
+saying "no such field" but no reference to the fact that it has the right 
+name but the wrong type.
+Note: apparently the jclass is a local ref and should only work 
+in the method that created it. It seems to work ok, but I'll make it
+ global cos the book said I should, and it may avoid future hassles.
+*/
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_initIDs (JNIEnv *j_env, jclass j_cls) {
+  
+  FID_mgpp_data = j_env->GetFieldID(j_cls, "mgpp_data_ptr_", "J"); //a long-"J"
+  if (FID_mgpp_data==NULL) {
+      cerr <<"MGPP JNI: field mgpp_data_ptr_ not found"<<endl;
+  } 
+
+  FID_query_result = j_env->GetFieldID(j_cls, "mgpp_query_result_", "Lorg/greenstone/mgpp/MGPPQueryResult;"); // an object -"L<class name>;"
+  if (FID_query_result==NULL) {
+      cerr <<"MGPP JNI: field mgpp_query_result_ not found"<<endl;
+  }
+  // the methods we want to use
+
+  // addDoc(long doc, float rank)
+  jclass JC_MGPPQueryResult = j_env->FindClass("org/greenstone/mgpp/MGPPQueryResult");
+  MID_addDoc = j_env->GetMethodID(JC_MGPPQueryResult, "addDoc", "(JF)V");
+  if (MID_addDoc==NULL) {
+      cerr <<"MGPP JNI: addDoc method not found"<<endl;
+  }
+  // addTerm(String term, String tag, int stem_method, long match_docs, 
+  // long term_freq, String[] equiv_terms)
+  MID_addTerm = j_env->GetMethodID(JC_MGPPQueryResult, "addTerm", "(Ljava/lang/String;Ljava/lang/String;IJJ[Ljava/lang/String;)V");
+  if (MID_addTerm==NULL) {
+      cerr <<"MGPP JNI: method addTerm not found"<<endl;
+  }
+  
+  // setTotalDocs(long)
+  MID_setTotalDocs = j_env->GetMethodID(JC_MGPPQueryResult, "setTotalDocs", "(J)V");
+  if (MID_setTotalDocs==NULL) {
+      cerr <<"MGPP JNI: method setTotalDocs not found"<<endl;
+  }
+  
+  MID_clearResult = j_env->GetMethodID(JC_MGPPQueryResult, "clear", "()V");
+  if (MID_clearResult==NULL) {
+      cerr <<"MGPP JNI: method clear not found"<<endl;
+  }
+  MID_setSyntaxError = j_env->GetMethodID(JC_MGPPQueryResult, "setSyntaxError", "(Z)V");
+  if (MID_clearResult==NULL) {
+      cerr <<"MGPP JNI: method setSyntaxError not found"<<endl;
+  }
+
+  // get the class for String to use in NewObjectArray in runQuery()
+  // FindClass returns a local reference - have to convert it to a global one
+  jclass local_CID_String = j_env->FindClass("java/lang/String");
+  if (local_CID_String==NULL) {
+    cerr <<"MGPP JNI: java String class not found"<<endl;
+  } else { 
+    /* create a global ref */
+    CID_String = (jclass)j_env->NewGlobalRef(local_CID_String);
+    /* The local reference is no longer useful */
+    j_env->DeleteLocalRef(local_CID_String);
+ 
+    /* Is the global reference created successfully? */
+    if (CID_String == NULL) {
+      return; /* out of memory exception thrown */
+    }
+  }
+  
+}
+
+/* the java side MGPPWrapper has a pointer to a C++ object - MGPPWrapperData
+   initialise this and set the pointer 
+*/
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_initCppSide (JNIEnv *j_env, jobject j_obj){
+
+  MGPPWrapperData * data = new MGPPWrapperData();
+  j_env->SetIntField(j_obj, FID_mgpp_data, (long)data);
+
+  return true;
+
+}
+
+//****************************************************
+// retrieve a document
+//****************************************************
+
+/* returns a document from mgpp as a string
+Note: TextData isn't cached - just reloaded each time
+*/
+JNIEXPORT jstring JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_getDocument (JNIEnv *j_env, 
+ jobject j_obj, jstring j_text_name, jstring j_level, jlong j_docnum) {
+  
+#ifdef __WIN32__
+  const char* base_dir = "";
+#else
+  const char* base_dir = "/";
+#endif
+
+  const char * text_name = j_env->GetStringUTFChars(j_text_name, NULL);
+  if (text_name==NULL) {
+   return NULL;
+  }
+
+  const char * level = j_env->GetStringUTFChars( j_level, NULL);
+  if (level==NULL) {
+   j_env->ReleaseStringUTFChars(j_text_name, text_name);
+   return NULL;
+  }
+    
+  // does this work alright? j_docnum is a long (64 bit)
+  unsigned long docnum = j_docnum;
+  TextData td;
+
+  // cast to char* otherwise complains about const
+  td.LoadData((char *)base_dir, (char *)text_name);
+
+  UCArray mg_level;
+  SetCStr(mg_level, level);
+  UCArray docText;
+  docText.clear();
+  // get the actual text
+  if (!GetDocText(td, mg_level, docnum, docText)) {
+    cerr <<"MGPP JNI: couldn't retrieve doc text"<<endl;
+  }
+
+  td.UnloadData();
+  
+  char * doc = GetCStr(docText); // do I need to free this char *??
+  jstring result = j_env->NewStringUTF(doc);
+  // release any gets
+  j_env->ReleaseStringUTFChars(j_text_name, text_name);
+  j_env->ReleaseStringUTFChars(j_level, level);
+
+  // free any C++ stuff
+  delete doc;
+
+  return result;
+ 
+}
+
+//******************************************
+// do a query
+// ****************************************
+
+/* load the IndexData - cached for querying
+ */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_loadIndexData (JNIEnv *j_env, jobject j_obj,  jstring j_index_name) {
+
+  jint data_ptr = j_env->GetIntField(j_obj, FID_mgpp_data);
+  MGPPWrapperData * data = (MGPPWrapperData *)data_ptr;
+
+#ifdef __WIN32__
+  const char* base_dir = "";
+#else
+  const char* base_dir = "/";
+#endif
+
+  const char * index_name = j_env->GetStringUTFChars( j_index_name, NULL);
+  if (index_name==NULL) {
+    return false;
+  }
+  
+  jboolean j_result=false;
+  
+  // why doesn't this complain about const??
+  if (data->indexData->LoadData(base_dir, index_name)) {
+    j_result=true;
+  }
+  
+  // release any gets
+  j_env->ReleaseStringUTFChars(j_index_name, index_name);
+  
+  return j_result;
+}
+
+/* unload the data 
+ */
+JNIEXPORT jboolean JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_unloadIndexData (JNIEnv *j_env, jobject j_obj) {
+
+  jint data_ptr = j_env->GetIntField(j_obj, FID_mgpp_data);
+  MGPPWrapperData * data = (MGPPWrapperData *)data_ptr;
+
+  data->indexData->UnloadData();
+  return true;
+
+}
+
+/* do the actual query - the results are written to query_result held on the 
+   java side */
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_runQuery (JNIEnv *j_env, jobject j_obj, jstring j_query){
+  
+  jthrowable exc; // an exception - check if something funny has happened
+  const char *query = j_env->GetStringUTFChars(j_query, NULL);
+  if (query==NULL) {
+    return; // exception already thrown
+  }
+  // turn to UCArray for mgpp and then release the string
+  UCArray queryArray;
+  SetCStr(queryArray, query);
+  j_env->ReleaseStringUTFChars(j_query, query);
+  
+  // the query data
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  // the result to write to
+  jobject result_ptr = j_env->GetObjectField(j_obj, FID_query_result);
+  if (result_ptr==NULL) {
+    cerr <<"couldn't access the result to write to"<<endl;
+    return;
+  }
+  
+  // clear the result
+  j_env->CallVoidMethod(result_ptr, MID_clearResult);
+  exc = j_env->ExceptionOccurred(); // this catches teh exception I think - it
+  //wont be thrown any further
+  if (exc) {
+    j_env->ExceptionDescribe();
+    return;
+  }
+  // the mgpp QueryResult that we will use
+  ExtQueryResult queryResult;
+
+  QueryNode * queryTree = NULL;
+  // parse the query string into a tree structure
+  queryTree = ParseQuery(queryArray, data->defaultBoolCombine, 
+			 data->defaultStemMethod);
+  if (queryTree == NULL) {
+    // invalid syntax
+    j_env->CallVoidMethod(result_ptr, MID_setSyntaxError, true);
+    cerr << "MGPPWrapperImpl: invalid query syntax!!\n";
+    return; 
+  }
+  // print the query
+  PrintNode (cout, queryTree);
+  // finally, do the query
+  MGQuery(*(data->indexData), *(data->queryInfo), queryTree, queryResult, data->level);
+
+  delete queryTree;
+
+  // convert queryResult to the java side version
+  // use levels rather than docs of ExtQueryResult
+  // CallVoidMethod(obj, method id, args to method)
+  for (int i=0; i<queryResult.levels.size(); i++) {
+    jlong doc = queryResult.levels[i];
+    jfloat rank = queryResult.ranks[i];
+    j_env->CallVoidMethod(result_ptr, MID_addDoc, doc, rank);
+    exc = j_env->ExceptionOccurred();
+    if (exc) {
+      j_env->ExceptionDescribe();
+      return;
+    }
+    
+  }
+
+  // actual num of docs
+  jlong total = queryResult.actualNumDocs;
+  j_env->CallVoidMethod(result_ptr, MID_setTotalDocs, total);
+  exc = j_env->ExceptionOccurred();
+  if (exc) {
+    j_env->ExceptionDescribe();
+    return;
+  }
+
+  // the terms
+  for (int j=0; j<queryResult.termFreqs.size(); j++) {
+    
+    TermFreqData tf = queryResult.termFreqs[j];
+    jstring term = j_env->NewStringUTF(GetCStr(tf.term));
+    jstring tag = j_env->NewStringUTF(GetCStr(tf.tag));
+    jint stem = tf.stemMethod;
+    jlong match = tf.matchDocs;
+    jlong freq = tf.termFreq;
+    
+    jobjectArray equivs=NULL;
+    jstring empty = j_env->NewStringUTF(""); // the initial object to fill the array
+    jint num_equivs = tf.equivTerms.size();
+    equivs = j_env->NewObjectArray(num_equivs, CID_String, empty);
+    if (equivs==NULL) {
+      cerr<<"couldn't create object array"<<endl;
+      
+    } else {
+      for (int k=0; k<num_equivs;k++) {
+	jstring equiv = j_env->NewStringUTF(GetCStr(tf.equivTerms[k]));
+	j_env->SetObjectArrayElement(equivs, k, equiv);
+      }
+      
+      
+      j_env->CallVoidMethod(result_ptr, MID_addTerm, term, tag, stem, match, freq, equivs);
+      exc = j_env->ExceptionOccurred();
+      if (exc) {
+	j_env->ExceptionDescribe();
+	return;
+      }
+    }
+  }
+
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setStem (JNIEnv *j_env, 
+					      jobject j_obj, 
+					      jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  if (j_on) {
+    data->defaultStemMethod |= 2;
+  } else {
+    data->defaultStemMethod &= 0xd;
+  }
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setCase (JNIEnv *j_env, 
+					      jobject j_obj, 
+					      jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  if (j_on) {
+    data->defaultStemMethod |= 1;
+  } else {
+    data->defaultStemMethod &= 0xe;
+  }
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setMaxDocs (JNIEnv *j_env, 
+						 jobject j_obj, 
+						 jint j_max) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->queryInfo->maxDocs=j_max;
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setSortByRank (JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  data->queryInfo->sortByRank=j_on;
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setReturnTerms(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jboolean j_on) {
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->queryInfo->needTermFreqs = j_on;
+ 
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setQueryLevel(JNIEnv *j_env, 
+						   jobject j_obj, 
+						   jstring j_level){
+  
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  const char * level = j_env->GetStringUTFChars(j_level, NULL);
+  if (level==NULL) {
+    return; // exception already thrown
+  }
+  
+  data->queryInfo->docLevel.clear();
+  SetCStr(data->queryInfo->docLevel, level);
+  
+  // release the java stuff
+  j_env->ReleaseStringUTFChars(j_level, level);
+ 
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setReturnLevel(JNIEnv *j_env, 
+						    jobject j_obj, 
+						    jstring j_level){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+
+  const char * level = j_env->GetStringUTFChars(j_level, NULL);
+  if (level==NULL) {
+    return; // exception already thrown
+  }
+  
+  data->level.clear();
+  SetCStr(data->level, level);
+  
+  // release the java stuff
+  j_env->ReleaseStringUTFChars(j_level, level);
+    
+
+}
+
+JNIEXPORT void JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_setMatchMode (JNIEnv *j_env,
+						   jobject j_obj,
+						   jint j_mode){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  data->defaultBoolCombine=j_mode;
+
+}
+
+JNIEXPORT jstring JNICALL 
+Java_org_greenstone_mgpp_MGPPWrapper_getQueryParams (JNIEnv *j_env,
+						     jobject j_obj){
+
+  MGPPWrapperData * data = (MGPPWrapperData *)j_env->GetIntField(j_obj, FID_mgpp_data);
+  
+  // print the data to a stringstream, then convert to char*, then to 
+  //jstring
+
+  stringstream output;
+  output << "Query params:"<<endl
+    // need to change this to use platform specific separator for niceness
+	 << "index\t\t"<<data->indexData->basePath<<"/"<<data->indexData->filename<<endl
+	 <<"search level\t"<<GetCStr(data->queryInfo->docLevel)<<endl
+	 <<"result level\t"<<GetCStr(data->level)<<endl
+	 <<"casefold\t"<<(data->defaultStemMethod&1)<<endl
+	 <<"stem\t\t"<<(data->defaultStemMethod&2)<<endl
+	 <<"order by rank\t"<<data->queryInfo->sortByRank<<endl
+	 <<"query type\t"<<(data->defaultBoolCombine==1?"all":"some")<<endl
+         <<"max docs\t"<<data->queryInfo->maxDocs<<endl<<ends;
+
+  const char *result = output.str().c_str();
+  jstring j_result = j_env->NewStringUTF(result);
+  delete (char *)result;
+  return j_result;
+}
Index: /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.h
===================================================================
--- /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.h	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/MGPPWrapperImpl.h	(revision 21112)
@@ -0,0 +1,15 @@
+#include "MGQuery.h"
+
+// the global stuff we need to do a query
+class MGPPWrapperData {
+ public:
+  IndexData * indexData;
+  QueryInfo * queryInfo;
+  int defaultStemMethod;
+  int defaultBoolCombine;
+  UCArray level; // the return level
+
+  MGPPWrapperData();
+  ~MGPPWrapperData();
+
+};
Index: /main/tags/3.01/mgpp/jni/Makefile.in
===================================================================
--- /main/tags/3.01/mgpp/jni/Makefile.in	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/Makefile.in	(revision 21112)
@@ -0,0 +1,69 @@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = jni
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+#NOTE: libmgpptext.a needs to come before libmgpplib.a !!!
+QUERY_LIBS = ../text/libmgpptext.a ../lib/libmgpplib.a -lstdc++ @LIBS@
+PASSES_LIBS = ../text/libmgpppass.a ../lib/libmgpplib.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+JNIFLAGS=@JNIFLAGS@
+JNISUFFIX=@JNISUFFIX@
+JNIINC=@JNIINC@
+
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+CXXCOMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS)
+
+.SUFFIXES:
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) $<
+
+# can we get rid of JAVA_HOME from here??
+INCLUDES = -I.. -I../lib -I../text -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JNIINC)
+
+all: compile link
+
+compile:
+	$(CXXCOMPILE) MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+link:
+	$(CXX) $(LDFLAGS) $(JNIFLAGS) -o libmgppjni.$(JNISUFFIX) MGPPWrapperImpl.o $(QUERY_LIBS)
+	$(CXX) $(LDFLAGS) $(JNIFLAGS) -o libmgpppassjni.$(JNISUFFIX) MGPPPassesWrapperImpl.o $(PASSES_LIBS)
+
+clean:
+	rm -rf *.o libmgppjni.$(JNISUFFIX) libmgpppassjni.$(JNISUFFIX)
+
+distclean: clean
+	rm -f Makefile
+
+install:
+	../mkinstalldirs $(libdir); \
+	cp libmgppjni.$(JNISUFFIX)  libmgpppassjni.$(JNISUFFIX)  $(libdir)/;
+
+
+
Index: /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h
===================================================================
--- /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPPassesWrapper.h	(revision 21112)
@@ -0,0 +1,127 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mgpp_MGPPPassesWrapper */
+
+#ifndef _Included_org_greenstone_mgpp_MGPPPassesWrapper
+#define _Included_org_greenstone_mgpp_MGPPPassesWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+#undef org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_1
+#define org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_1 0L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_2
+#define org_greenstone_mgpp_MGPPPassesWrapper_TEXT_PASS_2 1L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_1
+#define org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_1 2L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_2
+#define org_greenstone_mgpp_MGPPPassesWrapper_INDEX_PASS_2 3L
+#undef org_greenstone_mgpp_MGPPPassesWrapper_SPECIAL_PASS
+#define org_greenstone_mgpp_MGPPPassesWrapper_SPECIAL_PASS 4L
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    init
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_init
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setBasePath
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setBasePath
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setFileName
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setFileName
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setDocumentTag
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setDocumentTag
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    addLevelTag
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_addLevelTag
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setIndexLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setIndexLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    setInversionMemLimit
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_setInversionMemLimit
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    finish
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_finish
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    exitValue
+ * Signature: ()I
+ */
+JNIEXPORT jint JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_exitValue
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    initCSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_initCSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    addPass
+ * Signature: (CC)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_addPass
+  (JNIEnv *, jobject, jchar, jchar);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPPassesWrapper
+ * Method:    processMGPPDocument
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPPassesWrapper_processMGPPDocument
+  (JNIEnv *, jobject, jbyteArray);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h
===================================================================
--- /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/org_greenstone_mgpp_MGPPWrapper.h	(revision 21112)
@@ -0,0 +1,133 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include <jni.h>
+/* Header for class org_greenstone_mgpp_MGPPWrapper */
+
+#ifndef _Included_org_greenstone_mgpp_MGPPWrapper
+#define _Included_org_greenstone_mgpp_MGPPWrapper
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    initIDs
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_initIDs
+  (JNIEnv *, jclass);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    initCppSide
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_initCppSide
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    getDocument
+ * Signature: (Ljava/lang/String;Ljava/lang/String;J)Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mgpp_MGPPWrapper_getDocument
+  (JNIEnv *, jobject, jstring, jstring, jlong);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    loadIndexData
+ * Signature: (Ljava/lang/String;)Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_loadIndexData
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    unloadIndexData
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_org_greenstone_mgpp_MGPPWrapper_unloadIndexData
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setStem
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setStem
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setCase
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setCase
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setMaxDocs
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setMaxDocs
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setSortByRank
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setSortByRank
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setReturnTerms
+ * Signature: (Z)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setReturnTerms
+  (JNIEnv *, jobject, jboolean);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setQueryLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setQueryLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setReturnLevel
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setReturnLevel
+  (JNIEnv *, jobject, jstring);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    setMatchMode
+ * Signature: (I)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_setMatchMode
+  (JNIEnv *, jobject, jint);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    getQueryParams
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_org_greenstone_mgpp_MGPPWrapper_getQueryParams
+  (JNIEnv *, jobject);
+
+/*
+ * Class:     org_greenstone_mgpp_MGPPWrapper
+ * Method:    runQuery
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_org_greenstone_mgpp_MGPPWrapper_runQuery
+  (JNIEnv *, jobject, jstring);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
Index: /main/tags/3.01/mgpp/jni/win32.mak
===================================================================
--- /main/tags/3.01/mgpp/jni/win32.mak	(revision 21112)
+++ /main/tags/3.01/mgpp/jni/win32.mak	(revision 21112)
@@ -0,0 +1,75 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\jni
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+ 
+MGPPHOME = ..
+GSDL3HOME = ..\..\..
+AR = lib
+CC = cl
+CPPFLAGS = -GX
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -DSHORT_SUFFIX -DHAVE_STRING_H -D__WIN32__
+INCLUDES = -I"$(MGPPHOME)" -I"$(MGPPHOME)\lib" -I"$(MGPPHOME)\text" \
+           -I"C:\j2sdk1~1.1_0\include" -I"C:\j2sdk1~1.1_0\include\win32"           
+
+COMPILE = $(CC) -c $(CPPFLAGS) $(DEFS) $(INCLUDES) 
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+HEADERS = \
+    MGPPWrapperImpl.h   org_greenstone_mgpp_MGPPWrapper.h org_greenstone_mgpp_MGMGPPPassesWrapperImpl.h
+
+SOURCES = \
+    MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+
+OBJECTS = \
+    MGPPWrapperImpl$o  $(MGPPHOME)\text\libtextin.lib \
+    $(MGPPHOME)\lib\libmgpp.lib
+PASSES_OBJECTS = \
+    MGPPPassesWrapperImpl$o $(MGPPHOME)\text\libmgpppass.lib \
+    $(MGPPHOME)\lib\libmgpp.lib
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : compile link
+
+compile:
+        $(COMPILE) MGPPWrapperImpl.cpp MGPPPassesWrapperImpl.cpp
+
+link:
+        $(CC) -MD -LD $(OBJECTS) -Femgppjni.dll
+        $(CC) -MD -LD $(PASSES_OBJECTS) -Femgpppassjni.dll
+
+install: 
+	copy mgppjni.dll $(GSDL3HOME)\lib\jni
+	copy mgpppassjni.dll $(GSDL3HOME)\lib\jni
+clean:
+	del *$o mgppjni.dll mgppjni.exp mgppjni.lib mgpppassjni.dll mgpppassjni.exp mgpppassjni.lib 
Index: /main/tags/3.01/mgpp/lib/Makefile.in
===================================================================
--- /main/tags/3.01/mgpp/lib/Makefile.in	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/Makefile.in	(revision 21112)
@@ -0,0 +1,156 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CXX = @CXX@
+ifneq ($(NZDL), 1)
+CFLAGS = @CFLAGS@ -ansi
+else 
+CFLAGS = @CFLAGS@ -ansi -DSILENT
+endif
+CXXFLAGS = @CXXFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+libdir = @libdir@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES)  $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(INCLUDES) $(CXXFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) -D__cplusplus $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+INCLUDES = -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h      \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h
+
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp
+# memlib.cpp removed
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+
+all : libmgpplib.a
+
+libmgpplib.a : $(OBJECTS)
+	rm -f libmgpplib.a
+	$(AR) cru libmgpplib.a $(OBJECTS)
+	$(RANLIB) libmgpplib.a
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmgpplib.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# breaks make distclean (jrm21)
+#Makefile: Makefile.in ../../../config.status
+#	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /main/tags/3.01/mgpp/lib/WIN32.MAK
===================================================================
--- /main/tags/3.01/mgpp/lib/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/WIN32.MAK	(revision 21112)
@@ -0,0 +1,96 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+MGPPHOME = ..
+
+AR = lib
+CC = cl
+CPPFLAGS = 
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -DSHORT_SUFFIX
+INCLUDES = -I"." -I"$(MGPPHOME)"
+
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+
+HEADERS = \
+    bitio_m_random.h   local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    memlib.h           netorder.h \
+    bitio_gen.h        mgheap.h           messages.h \
+    bitio_m.h          huffman.h  \
+    bitio_m_mem.h      perf_hash.h \
+    bitio_m_mems.h      \
+    pathmax.h          getpagesize.h      sysfuncs.h         bitio_m_abstract.h  \
+    random.h           simplefrenchstem.h unitool.h	     longlong.h \
+    non_ansi.h         getopt_old.h       win32in.h
+
+
+SOURCES = \
+    mgheap.cpp         \
+    huffman.cpp       messages.cpp \
+    bitio_gen.cpp     perf_hash.cpp \
+    local_strings.cpp lovinstem.cpp \
+    bitio_m_mem.cpp   bitio_m_stdio.cpp bitio_m_mems.cpp \
+    alloca.c          xmalloc.c         strstr.cpp \
+    ftruncate.cpp     strcasecmp.cpp    random.cpp \
+    simplefrenchstem.cpp                unitool.cpp       bitio_m_random.cpp \
+    bitio_m_abstract.cpp                getopt_old.c      win32in.c
+# memlib.cpp removed
+
+
+OBJECTS =  mgheap$o   \
+    huffman$o         messages$o \
+    bitio_gen$o       perf_hash$o \
+     local_strings$o   lovinstem$o \
+    bitio_m_mem$o     bitio_m_stdio$o   bitio_m_mems$o \
+    xmalloc$o         bitio_m_random$o  bitio_m_abstract$o \
+     random$o          simplefrenchstem$o \
+    unitool$o         getopt_old$o      win32in$o
+
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmgpp.lib
+
+libmgpp.lib : $(OBJECTS)
+	if EXIST libmgpp.lib del libmgpp.lib
+	$(AR) /out:libmgpp.lib $(OBJECTS)
+
+$(OBJECTS): "$(MGPPHOME)\win32cfg.h"
+
+install: 
+clean:
+	del *$o 
+	del libmgpp.lib
Index: /main/tags/3.01/mgpp/lib/alloca.c
===================================================================
--- /main/tags/3.01/mgpp/lib/alloca.c	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/alloca.c	(revision 21112)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /main/tags/3.01/mgpp/lib/bitio_gen.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_gen.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_gen.cpp	(revision 21112)
@@ -0,0 +1,134 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::unary_length(val);
+  //  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::binary_length (val, b);
+  return (num);
+}
+
+
+unsigned long BIO_Gamma_Length (unsigned long val) {
+  register unsigned long num;
+  num = bitio_buffer::gamma_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  num = bitio_buffer::delta_length(val);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  num = bitio_buffer::elias_length(val, b, s);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  num = bitio_buffer::bblock_length (val, b);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 ((N - p) / p);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  clogb = bitio_buffer::ceil_log2 (b);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+/* adjustment is a hack to overcome inaccuracies in floating point,
+   where (int)(ln4/ln2) can equal 1, not 2. */
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  return ((int) (p * (2 * log2 (((double) N * 1.0001) / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::floor_log2 (b);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  logb = bitio_buffer::ceil_log2 (b);
+  return logb;
+}
Index: /main/tags/3.01/mgpp/lib/bitio_gen.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_gen.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_gen.h	(revision 21112)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/bitio_m_abstract.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_abstract.cpp	(revision 21112)
@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+
+
+
+
+
Index: /main/tags/3.01/mgpp/lib/bitio_m_abstract.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_abstract.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_abstract.h	(revision 21112)
@@ -0,0 +1,340 @@
+#ifndef H_BITIO_ABSTRACT
+#define H_BITIO_ABSTRACT
+
+class bitio_buffer {
+public:
+  virtual int bit() = 0;
+  virtual void encodeBit(int bit) = 0;
+  virtual void seek(unsigned long topos) = 0;
+  
+  static inline void check_positive(char *type, unsigned long x) {
+    if (x <= 0) {
+      fprintf(stderr, "Error: Cannot %s encode %lu\n", type, x);
+      exit(1);
+    }
+  }
+
+  static inline int floor_log2(unsigned long x) {
+    unsigned long  _B_x = x;
+    int reply;
+    
+    reply = -1;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+  
+  static inline int ceil_log2(unsigned long x) {
+    register int _B_x;
+    unsigned long reply;
+    
+    _B_x = x - 1;
+    reply = 0;
+    while (_B_x != 0) {
+      _B_x >>= 1;
+      ++reply;
+    }
+    return reply;
+  }
+
+  inline unsigned long unary_decode(unsigned long *count) {
+    unsigned long x;
+    
+    x = 1;
+    while (!this->bit()) ++x;
+    if (count != NULL) {
+      *count += x;
+    }
+    return x;
+  }
+
+  inline void unary_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_x = x;
+    check_positive("unary", _B_x);
+    if (count != NULL) {
+      *count += _B_x;
+    }
+    while(--_B_x) 
+      encodeBit(0);
+    encodeBit(1);
+  }
+
+
+  inline unsigned long binary_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = 0;
+    register unsigned long _B_b = b;
+    register int _B_i, _B_logofb, _B_thresh;
+    register unsigned long reply;
+    
+    if (_B_b != 1) {					
+      _B_logofb = ceil_log2(_B_b);
+      _B_thresh = (1<<_B_logofb) - _B_b;
+      --_B_logofb;
+      
+      if (count != NULL) {
+	*count += _B_logofb;
+      }
+      
+      for (_B_i=0; _B_i < _B_logofb; ++_B_i) {
+	_B_x += _B_x + this->bit();
+      }
+
+      if ((int)_B_x >= _B_thresh) {
+	_B_x += _B_x + this->bit();
+	_B_x -= _B_thresh;
+	if (count != NULL) {
+	  *count = *count + 1;
+	}
+      }
+      reply = _B_x+1;
+      
+    } else reply = 1;
+    
+    return reply;
+  }
+
+  inline void binary_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else {
+      _B_nbits = _B_logofb;
+      _B_x += _B_thresh;
+    }
+    if (count != NULL) {
+      *count += _B_nbits;
+    }
+    while (--_B_nbits>=0)
+      encodeBit((_B_x>>_B_nbits) & 0x1);
+  }
+
+  inline unsigned long bblock_decode(unsigned long b, unsigned long *count) {
+    register unsigned long _B_x1, _B_xx = 0;
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb;
+    
+    _B_xdivb = this->unary_decode(count);
+    --_B_xdivb;				       
+    while (_B_xdivb--) {
+      _B_xx += _B_bb;
+    }
+    _B_x1 = this->binary_decode(_B_bb, count);
+    return (_B_xx + _B_x1);
+  }
+  
+  inline void bblock_encode(unsigned long x, unsigned long b, unsigned long *count) {
+    register unsigned long _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    unary_encode(_B_xdivb+1, count);
+    binary_encode(_B_xx+1, _B_bb, count);
+  }
+
+  inline unsigned long elias_decode(unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k;
+    register double _B_pow=1.0;
+    
+    _B_k = this->unary_decode(count);
+    --_B_k;
+    while (_B_k--) {
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    _B_xx = this->binary_decode(_B_upper-_B_lower+1, count);
+    --_B_xx;
+    check_positive("gamma", _B_xx+_B_lower);
+    return _B_xx+_B_lower;
+  }
+  
+  inline void elias_encode(unsigned long x, unsigned long b, double s, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0;
+    register double _B_pow=1.0;
+    
+    check_positive("elias", _B_xx);
+    
+    while ((int)_B_xx>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    unary_encode(_B_k+1, count);
+    binary_encode(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);
+  }
+  
+  inline unsigned long gamma_decode(unsigned long *count) {
+    register unsigned long _B_xx = 1;
+    register int _B_nbits = 0;
+    
+    while(!this->bit()) ++_B_nbits;
+    if (count != NULL) {
+      *count += _B_nbits*2+1;
+    }
+    while (_B_nbits-- > 0)
+      _B_xx += _B_xx + this->bit();
+    return _B_xx;
+  }
+
+  inline void gamma_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    register int _B_nbits;
+    
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    _B_nbits = _B_logofb+1;
+    
+    if (count != NULL) { 
+      *count += _B_logofb*2+1;
+    }
+    while(_B_logofb--) encodeBit(0);
+    while (--_B_nbits>=0) encodeBit((_B_xx>>_B_nbits) & 0x1);
+  }
+  
+  inline unsigned long delta_decode(unsigned long *count) {
+    register unsigned long _B_xxx;
+    register int _B_logx;
+    _B_logx = gamma_decode(count);
+    --_B_logx;
+    _B_xxx = binary_decode(1<<_B_logx, count); 
+    --_B_xxx;
+    return (_B_xxx + (1<<_B_logx));
+  }
+
+  inline void delta_encode(unsigned long x, unsigned long *count) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx;
+    _B_logx = floor_log2(_B_xxx);
+    gamma_encode(_B_logx+1, count);
+    binary_encode(_B_xxx+1, 1<<_B_logx, count);
+  }
+
+  inline unsigned long huff_decode(unsigned long *mcodes, unsigned long **values,
+				   unsigned long *count = NULL) {
+    register unsigned long *__min_code = mcodes;
+    register unsigned long *__mclen    = mcodes;
+    register unsigned long **__values  = values;
+    register unsigned long __code = 0;
+    do {
+      __code += __code + bit();
+      if (count != NULL) ++(*count);
+    } while (__code < *++__mclen);
+    
+    return __values[__mclen - __min_code][__code - *__mclen];
+  }
+
+  inline void huff_encode(unsigned long x, unsigned long *codes, char *lens,
+			  unsigned long *count = NULL) {
+    register int __i;
+    register int __clen = lens[x];
+    register unsigned long __code = codes[x];
+    for (__i=__clen-1; __i>=0; --__i) {
+      encodeBit((__code >> __i) & 1);
+    }
+    if (count != NULL) {
+      *count += lens[x];
+    }
+  }
+
+  static inline unsigned long unary_length(unsigned long x) {
+    check_positive("unary", x);
+    return x;
+  }
+
+  static inline int binary_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_x = (x);
+    register unsigned long _B_b = (b);
+    register int _B_nbits, _B_logofb, _B_thresh;
+    
+    check_positive("binary", _B_x);
+    _B_logofb = ceil_log2(_B_b);
+    _B_thresh = (1<<_B_logofb) - _B_b;
+    if ((int)(--_B_x) < _B_thresh) _B_nbits = _B_logofb-1;
+    else _B_nbits = _B_logofb;
+    return _B_nbits;
+  }
+
+  static inline unsigned long bblock_length(unsigned long x, unsigned long b) {
+    register unsigned long _B_bcount, count, _B_xx = (x);
+    register unsigned long _B_bb = (b);
+    register int _B_xdivb = 0;
+    
+    check_positive("bblock", _B_xx);
+    --_B_xx;
+    while (_B_xx >= _B_bb) {
+      ++_B_xdivb;
+      _B_xx -= _B_bb;
+    }
+    count = unary_length(_B_xdivb+1);
+    _B_bcount = binary_length(_B_xx+1, _B_bb);
+    count += _B_bcount;
+    return count;
+  }
+  
+  static inline unsigned long elias_length(unsigned long x, unsigned long b, 
+					   double s) {
+    register unsigned long _B_xx = x;
+    register unsigned long _B_b = b;
+    register double _B_s = s;
+    register int _B_lower=1, _B_upper=1;
+    register int _B_k=0, _B_ecount;
+    register double _B_pow=1.0;
+    register unsigned long count;
+    check_positive("gamma", _B_xx);
+    
+    while ((int)(_B_xx)>_B_upper) {
+      ++_B_k;
+      _B_lower = _B_upper+1;
+      _B_pow *= _B_s;
+      _B_upper += (int) (_B_b*_B_pow);
+    }
+    count = unary_length(_B_k+1);
+    _B_ecount = binary_length(_B_xx-_B_lower+1, _B_upper-_B_lower+1);
+    count += _B_ecount;
+    return count;
+  }
+  
+  static inline unsigned long gamma_length(unsigned long x) {
+    register unsigned long _B_xx = x;
+    register int _B_logofb;
+    check_positive("gamma", _B_xx);
+    _B_logofb = floor_log2(_B_xx);
+    return _B_logofb*2+1;
+  }
+
+  static inline unsigned long delta_length(unsigned long x) {
+    register unsigned long _B_xxx = x;
+    register int _B_logx, _B_dcount;
+    register unsigned long count;
+    _B_logx = floor_log2 (_B_xxx);
+    count = gamma_length(_B_logx+1);
+    _B_dcount = binary_length (_B_xxx+1, 1<<_B_logx);
+    count += _B_dcount;
+    return count;
+  }
+};
+
+#endif
Index: /main/tags/3.01/mgpp/lib/bitio_m_mem.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_mem.cpp	(revision 21112)
@@ -0,0 +1,128 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mem.h"
+
+mem_bitio_buffer::mem_bitio_buffer() {
+}
+
+mem_bitio_buffer::mem_bitio_buffer(unsigned char *readbuffer, int size) { 
+  pos       = readbuffer;
+  base      = readbuffer;
+  remaining = size;
+  buffer    = 0;
+  btg       = 0;
+}
+
+
+void mem_bitio_buffer::error() {
+  fprintf (stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+void mem_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+
+void mem_bitio_buffer::seek(unsigned long toPos) { 
+  register long _newPos;
+  do {
+    _newPos = toPos;
+    
+    pos    = base + (_newPos >> 3);
+    buffer = *pos++;
+    btg    = 8 - (_newPos & 0x07);
+  } while (0);
+}
+
+long mem_bitio_buffer::addff(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0xff;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+
+  return b;
+}
+
+long mem_bitio_buffer::add00(long b) {
+  do {
+    if (btg == 0) {
+      if (remaining != 0) {
+	buffer = *pos++;
+	--remaining;
+      } else {
+	buffer = 0x00;
+      }
+      btg = sizeof(buffer) * 8;
+    }
+    b = b + b + ((buffer >> --btg) & 1);
+  } while(0);
+  
+  return b;
+}
+
+int mem_bitio_buffer::bit() {
+  if (btg == 0) {
+
+    if (remaining == 0) { 
+      this->error();
+      return 0;
+
+    } else {
+      buffer = *pos++;
+      --remaining;
+      btg = sizeof(buffer) * 8;
+    }
+    
+  }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void mem_bitio_buffer::encodeBit(int b) {
+  --btg;
+  if (b) buffer |= (1 << btg);
+  
+  if (btg == 0) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+    buffer = 0;
+    btg = sizeof(buffer)*8;
+  }
+}
+
+void mem_bitio_buffer::flush() {
+  if (btg != sizeof(buffer)*8) {
+    if (remaining) {
+      *pos++ = buffer;
+      --remaining;
+    }
+  }
+  btg = sizeof(buffer)*8;
+}
+
+void mem_bitio_buffer::done() {
+  // do nothing
+}
+
+int mem_bitio_buffer::encodeLength() {
+  return pos - base;
+}
+
+int mem_bitio_buffer::encodeVerify() {
+  return (remaining != 0);
+}
+
+
+
Index: /main/tags/3.01/mgpp/lib/bitio_m_mem.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_mem.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_mem.h	(revision 21112)
@@ -0,0 +1,104 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+#include "bitio_m_abstract.h"
+
+
+class mem_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *pos;
+  unsigned char *base;
+  int            remaining;
+  unsigned char  buffer;
+  unsigned char  btg;
+  
+public:
+  mem_bitio_buffer ();
+  mem_bitio_buffer (unsigned char *buffer, int size);
+  void seek (unsigned long topos);
+  void error ();
+  long add00 (long b);
+  long addff (long b);
+  int  bit ();
+  void done ();
+
+  void encodeStart ();
+  void encodeBit (int bit);
+  void encodeDone () { flush(); }
+  int  encodeVerify ();
+  int  encodeLength ();
+  void flush ();
+};
+
+
+/*
+#define ENCODE_START(p,r)						\
+  {									\
+    mem_bitio_buffer buffer(p, r);                                     \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush()
+
+#define ENCODE_DONE buffer.flush();}
+
+
+#define DECODE_START(p,r)						\
+  {	                                                                \
+    mem_bitio_buffer buffer(p,r);
+
+#define DECODE_CONTINUE(b)						\
+  {	                                                                \
+    mem_bitio_buffer buffer;                                           \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/mgpp/lib/bitio_m_mems.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_mems.cpp	(revision 21112)
@@ -0,0 +1,89 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_mems.h"
+
+/**
+ * Create decoding "stream"/"file"
+ */
+mems_bitio_buffer::mems_bitio_buffer()
+{
+}
+
+mems_bitio_buffer::mems_bitio_buffer(unsigned char *readbuffer, long _pos)
+{ 
+  base = readbuffer;
+  pos  = _pos;
+}
+
+void mems_bitio_buffer::encodeStart()
+{
+  // Do nothing
+}
+
+
+void mems_bitio_buffer::error()
+{
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d\n", __FILE__, __LINE__);
+  exit(1);
+}
+
+
+void mems_bitio_buffer::seek(unsigned long toPos)
+{ 
+  pos = toPos;
+}
+
+unsigned long mems_bitio_buffer::position()
+{
+  return pos;
+}
+
+long mems_bitio_buffer::addff(long b)
+{
+  do
+    {
+      b = b + b + (base[pos>>3] & (0x80 >> (pos & 0x07)) != 0);
+      ++pos;
+    }
+  while(0);
+
+  return b;
+}
+
+long mems_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int mems_bitio_buffer::bit()
+{
+  ++pos;
+  return ((base[(pos-1)>>3] & (0x80 >> ((pos-1) & 0x07))) != 0);
+}
+
+void mems_bitio_buffer::encodeBit(int bit)
+{
+  if (bit)
+    base[pos>>3] |= 0x80 >> (pos & 0x07);
+  else
+    base[pos>>3] &= 0xff7f >> (pos & 0x07);
+  ++pos;
+}
+
+void mems_bitio_buffer::flush()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::done()
+{
+  // do nothing
+}
+
+void mems_bitio_buffer::encodeDone()
+{
+  flush();
+}
+
Index: /main/tags/3.01/mgpp/lib/bitio_m_mems.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_mems.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_mems.h	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+#include "bitio_m_abstract.h"
+
+
+class mems_bitio_buffer : public bitio_buffer {
+protected:
+  unsigned char *base;
+  unsigned long  pos;
+  
+public:
+  mems_bitio_buffer();
+  mems_bitio_buffer(unsigned char *readbuffer, long _pos);
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  //  int  bitOffset();
+  void done();
+
+  unsigned long position();
+
+  void encodeStart();
+  void encodeBit(int b);
+  void encodeDone();
+  void flush();
+};
+
+
+/*#define ENCODE_START(b,p)						\
+  {									\
+    mems_bitio_buffer buffer(b, p);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+
+#define DECODE_START(b,p)						\
+  {		                                                        \
+    mems_bitio_buffer buffer(b, p);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    mems_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b);
+
+#define DECODE_ADD_00(b) b = buffer.add00(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/mgpp/lib/bitio_m_random.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_random.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_random.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "longlong.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "bitio_m_abstract.h"
+#include "bitio_m_random.h"
+
+random_bitio_buffer::random_bitio_buffer(FILE *f, unsigned long length)
+{
+  buffer = NULL;
+  attachFile (f, length);
+}
+
+random_bitio_buffer::~random_bitio_buffer() {
+  if (buffer != NULL) delete []buffer;
+}
+
+void random_bitio_buffer::attachFile (FILE *f, unsigned long length) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  // reset all the values
+  file   = f;
+  pos    = 0;
+  base   = 0;
+  used   = 0;
+  len    = length - 1;
+  shift  = 0;
+
+  // make sure the length is a power of 2
+  while (len != 0) {
+    ++shift;
+    len = len >> 1;
+  }
+  len = 1 << shift;
+
+  // allocate a new buffer
+  buffer = new u_char[len];
+
+  // fill up the buffer
+  if (file != NULL) {
+    fseek(file, 0, 0);
+    fread(buffer, 1, len, file);
+  }
+}
+
+random_bitio_buffer &random_bitio_buffer::operator= (const random_bitio_buffer &_rbb) {
+  // delete the old buffer
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+
+  file = _rbb.file;
+
+  base = _rbb.base;
+  used = _rbb.used;
+  pos = _rbb.pos;
+  len = _rbb.len;
+  shift = _rbb.shift;
+
+  // copy the buffer
+  buffer = new u_char [len];
+  //  memcpy (buffer, _rbb.buffer, len);
+memcpy (buffer, _rbb.buffer, len);
+  
+  return (*this);
+}
+
+void random_bitio_buffer::error() {
+}
+
+void random_bitio_buffer::writeRead()
+{
+  if (used)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+    }
+  //  else
+  //    {
+      base += len;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = 0;
+      used = 0;
+  //    }
+}
+
+void random_bitio_buffer::encodeBit(int bit)
+{
+  do
+    {
+      if (bit != 0)
+	{
+	  buffer[pos >> 3] |= 0x80 >> (pos & 0x07);
+	}
+      else
+	{
+	  buffer[pos >> 3] &= 0xff7f >> (pos & 0x07);
+	}
+      used = 1;
+      ++pos;
+
+      if ((pos >> 3) >= len)
+	{
+	  writeRead();
+	}
+    }
+  while (0);
+}
+
+void random_bitio_buffer::flush()
+{
+  if (used != 0)
+    {
+      fseek(file, base, 0);
+      fwrite(buffer, 1, len, file);
+      used = 0;
+    }
+}
+
+void random_bitio_buffer::encodeDone()
+{
+  flush();
+  if (buffer != NULL) {
+    delete []buffer;
+    buffer = NULL;
+  }
+}
+
+unsigned long random_bitio_buffer::tell()
+{
+  return ((base << 3) + pos);
+}
+
+void random_bitio_buffer::seek(unsigned long toPos)
+{
+  if (((toPos >> 3) >= base) && (((toPos+7) >> 3) < (base + len)))
+    {
+      pos = toPos - (base << 3);
+    }
+  else
+    {
+      flush();
+      base = toPos >> 3;
+      base = (toPos >> (shift + 3)) << shift;
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = toPos & 0x07;
+      pos = toPos & ((8 << shift) - 1);
+    }
+}
+
+int random_bitio_buffer::bit()
+{
+  //  fprintf(stderr, "Bit in");
+  ++pos;
+  if (((pos - 1) >> 3) >= len)
+    {
+      writeRead();
+      pos = 1;
+    }
+  return (buffer[(pos - 1) >> 3] & (0x80 >> ((pos - 1) & 0x07))) != 0;
+}
+
+long random_bitio_buffer::addff(long b)
+{
+  return b + b + bit();
+}
+
+long random_bitio_buffer::add00(long b)
+{
+  return addff(b);
+}
+
+int random_bitio_buffer::bitOffset()
+{
+  return 0;
+}
+
+void random_bitio_buffer::done()
+{
+  encodeDone();
+}
+
+
+#ifdef USE_LONG_LONG
+void random_bitio_buffer::seek_LL(mg_ullong toPos)
+{
+  if (((toPos >> 3) >= (mg_ullong) base) &&
+      (((toPos + 7) >> 3) < (mg_ullong) (base + len)))
+    {
+      pos = (long) (toPos - (mg_ullong) (base << 3));
+    }
+  else
+    {
+      flush();
+      base = (long) ((toPos >> (shift + 3)) << shift);
+
+      fseek(file, base, 0);
+      fread(buffer, 1, len, file);
+      pos = (long) (toPos & ((8 << shift) - 1));
+    }
+}
+
+mg_ullong random_bitio_buffer::tell_LL()
+{
+  return (((mg_ullong) base) << 3ull) + pos;
+}
+#endif
+
Index: /main/tags/3.01/mgpp/lib/bitio_m_random.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_random.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_random.h	(revision 21112)
@@ -0,0 +1,127 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+#include "longlong.h"
+#include "bitio_m_abstract.h"
+
+
+class random_bitio_buffer : public bitio_buffer { 
+public:
+  FILE *file;
+  unsigned char *buffer;
+  unsigned long  base;
+  unsigned long  used;
+  unsigned long  pos;
+  unsigned long  len;
+  unsigned long  shift;
+  void writeRead();
+
+  random_bitio_buffer(FILE *f = 0,
+		      unsigned long length = 8*1024);
+  virtual ~random_bitio_buffer();
+  
+  void attachFile (FILE *f, unsigned long length);
+
+  random_bitio_buffer &operator= (const random_bitio_buffer &_rbb);
+  
+  void encodeDone();
+  void encodeBit(int b);
+  void flush();
+  unsigned long tell();
+
+  void seek(unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset();
+  void done();
+
+#ifdef USE_LONG_LONG
+  // extras for longlong.h
+  void seek_LL(mg_ullong pos);
+  mg_ullong tell_LL();
+#endif
+};
+
+
+/*#define ENCODE_START(f,l)						\
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define SEEK fprintf(stderr, "Seek to %d\n",buffer.base)
+#define READ fprintf(stderr, "Read of %d\n",buffer.len)
+#define WRITE fprintf(stderr, "Write of %d\n",buffer.used)
+
+#define WRITE_READ buffer.writeRead()
+
+#define ENCODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_FLUSH buffer.flush();
+
+#define ENCODE_DONE buffer.done();}							
+
+#define DECODE_START(f,l) \
+  {									\
+    random_bitio_buffer buffer(f, l);
+
+#define DECODE_CONTINUE(b) \
+  {									\
+    random_bitio_buffer buffer;                                               \
+    buffer.unpause(b);
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b)	buffer.pause(&b);}
+
+#define ENCODE_TELL buffer.tell()
+
+#define DECODE_TELL buffer.tell()
+*/
+
+
+
+#endif
Index: /main/tags/3.01/mgpp/lib/bitio_m_stdio.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_stdio.cpp	(revision 21112)
@@ -0,0 +1,125 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "bitio_m_stdio.h"
+
+stdio_bitio_buffer::stdio_bitio_buffer(FILE *f) {
+  attachFile (f);
+}
+
+void stdio_bitio_buffer::attachFile (FILE *f) {
+  file = f;
+  buffer = 0;
+  btg = 0;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::encodeStart() {
+  btg = sizeof(buffer) * 8;
+}
+
+void stdio_bitio_buffer::error() {
+  fprintf(stderr, "Unexpected EOF in \"%s\" on line %d", __FILE__, __LINE__);
+  exit(1);
+}
+
+void stdio_bitio_buffer::seek (unsigned long byte, unsigned char bit) {
+  if (fseek (file, byte, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = bit;
+  bytesWritten = 0;
+}
+
+void stdio_bitio_buffer::seek(unsigned long topos) {
+  if (fseek(file, topos >> 3, SEEK_SET) != 0) return;
+  buffer = getc(file);
+  btg = 8 - (topos & 0x07);
+  bytesWritten = 0;
+}
+
+long stdio_bitio_buffer::addff(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0x0ff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+
+  return b;
+}
+
+long stdio_bitio_buffer::add00(long b)
+{
+  if (!btg)
+    {
+      register int val;
+      val = getc(file);
+      if (val == EOF)
+	{
+	  buffer = 0xff;
+	}
+      else
+	{
+	  buffer = val;
+	}
+      btg = sizeof(buffer) * 8;
+    }
+  b = b + b + ((buffer >> --btg) & 0x01);
+  
+  return b;
+}
+
+void stdio_bitio_buffer::encodeBit(int b)
+{
+  --btg;
+  if (b != 0)
+    buffer |= (1 << btg);
+
+  if (btg == 0)
+    {
+      putc(buffer, file);
+      bytesWritten += sizeof (buffer);
+      buffer = 0;
+      btg = sizeof(buffer)*8;
+    }
+}
+
+int stdio_bitio_buffer::bit()
+{
+  if (btg == 0)
+    {
+      buffer = getc(file);
+      if (feof(file))
+	{
+	  error();
+	  return 0;
+	}
+      else
+	{
+	  btg = sizeof(buffer) * 8;
+	}
+    }
+  return ((buffer >> --btg) & 0x01);
+}
+
+void stdio_bitio_buffer::done() {
+  // no tidying required
+}
+
+void stdio_bitio_buffer::encodeDone() {
+  if (btg != sizeof(buffer) * 8) {
+    putc(buffer, file);
+    bytesWritten += sizeof (buffer);
+  }
+}
+
Index: /main/tags/3.01/mgpp/lib/bitio_m_stdio.h
===================================================================
--- /main/tags/3.01/mgpp/lib/bitio_m_stdio.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/bitio_m_stdio.h	(revision 21112)
@@ -0,0 +1,101 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+#include "bitio_m_abstract.h"
+
+
+class stdio_bitio_buffer : public bitio_buffer {
+protected:
+  FILE *file;
+  unsigned char buffer;
+  unsigned char btg;
+  unsigned long bytesWritten;
+  
+public:
+  stdio_bitio_buffer (FILE *f = NULL);
+
+  void attachFile (FILE *f);
+
+  unsigned char GetBtg () { return btg; }
+  unsigned long GetBytesWritten () { return bytesWritten; }
+  
+  void seek (unsigned long byte, unsigned char bit);
+  void seek (unsigned long topos);
+  void error();
+  long add00(long b);
+  long addff(long b);
+  int  bit();
+  int  bitOffset() { return btg; }
+  void done();
+
+  void encodeStart();
+  void encodeBit(int bit);
+  void encodeDone();
+};
+
+
+/*#define ENCODE_START(f)							\
+  {									\
+    stdio_bitio_buffer buffer(f);                                       \
+    buffer.encodeStart();
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+    buffer.unpause(b);
+
+#define ENCODE_PAUSE(b) buffer.pause(&b);}
+
+#define ENCODE_DONE buffer.encodeDone();}
+
+#define DECODE_START(f)							\
+  {		                                                        \
+    stdio_bitio_buffer buffer(f);
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    stdio_bitio_buffer buffer;                                          \
+     buffer.unpause(b);
+
+#define DECODE_ADD_FF(b) b = buffer.addff(b)
+
+#define DECODE_ADD_00(b) b = buffer.add00(b)
+
+#define DECODE_BIT buffer.bit()
+
+#define DECODE_DONE buffer.done();}
+
+#define DECODE_PAUSE(b) buffer.pause(&b);}
+*/
+
+#endif
Index: /main/tags/3.01/mgpp/lib/ftruncate.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/ftruncate.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/ftruncate.cpp	(revision 21112)
@@ -0,0 +1,99 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (int fd, off_t length)
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (int fd, off_t length)
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /main/tags/3.01/mgpp/lib/getopt_old.c
===================================================================
--- /main/tags/3.01/mgpp/lib/getopt_old.c	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/getopt_old.c	(revision 21112)
@@ -0,0 +1,748 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+/* jrm21 - GSDL: getopt.h -> getopt_old.h */
+#include "getopt_old.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /main/tags/3.01/mgpp/lib/getopt_old.h
===================================================================
--- /main/tags/3.01/mgpp/lib/getopt_old.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/getopt_old.h	(revision 21112)
@@ -0,0 +1,134 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef HAVE_DECL_GETOPT
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  int getopt (int, char * const *, const char *);
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
+
+#endif
Index: /main/tags/3.01/mgpp/lib/getpagesize.h
===================================================================
--- /main/tags/3.01/mgpp/lib/getpagesize.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/getpagesize.h	(revision 21112)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /main/tags/3.01/mgpp/lib/gmalloc.c
===================================================================
--- /main/tags/3.01/mgpp/lib/gmalloc.c	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/gmalloc.c	(revision 21112)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /main/tags/3.01/mgpp/lib/huffman.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/huffman.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/huffman.cpp	(revision 21112)
@@ -0,0 +1,385 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = new huff_data))
+	goto error0;
+    }
+  if (!(heap = new unsigned long[num * 2]))
+    goto error1;
+  if (!(hd->clens = new char[num]))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  memcpy (&heap[num], freqs, num * sizeof (*heap));
+  memset (heap, '\0', num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; ++i)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; --i)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    ++child;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; ++i)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		++child;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	++HNum;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; ++i)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  memset (hd->lencount, '\0', sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; ++i)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      ++hd->lencount[codelen];
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; --i)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  delete []heap;
+  return (hd);
+
+error2:
+  delete []heap;
+error1:
+  if (!data)
+    delete hd;
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  memcpy (mc, data->min_code, sizeof (mc));
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = new unsigned long[data->num_codes]))
+    return (NULL);
+  if (!(values = new unsigned long *[MAX_HUFFCODE_LEN + 1]))
+    {
+      delete []vals;
+      return (NULL);
+    }
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; ++i)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; ++i)
+    size += counts[i] * hd.clens[i];
+  delete []hd.clens;
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk) {
+  if (fread (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fread (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      memset (hd->lencount, '\0', sizeof (hd->lencount));
+      if (fread (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      memset (hd->min_code, '\0', sizeof (hd->min_code));
+
+      if (fread (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; ++i)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = new char[hd->num_codes]))
+	return -1;
+
+
+      if (fread (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
Index: /main/tags/3.01/mgpp/lib/huffman.h
===================================================================
--- /main/tags/3.01/mgpp/lib/huffman.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/huffman.h	(revision 21112)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+#include "bitio_m_abstract.h"
+
+#define MAX_HUFFCODE_LEN 31
+
+struct huff_data {
+  int num_codes;
+  int mincodelen;
+  int maxcodelen;
+  int lencount[MAX_HUFFCODE_LEN + 1];
+  unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+  char *clens;
+};
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+#endif
Index: /main/tags/3.01/mgpp/lib/local_strings.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/local_strings.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/local_strings.cpp	(revision 21112)
@@ -0,0 +1,411 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = new u_char(src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  memcpy (copy, src_str, src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    ++i;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  memset (buf, '\0', sizeof(buf));
+  for (i = 0; i < len; ++i)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  ++s;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      --s;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; ++i)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /main/tags/3.01/mgpp/lib/local_strings.h
===================================================================
--- /main/tags/3.01/mgpp/lib/local_strings.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/local_strings.h	(revision 21112)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+class mgString
+{
+  u_char *data;
+ public:
+  inline int length()
+    {
+      return data[0];
+    }
+  inline u_char *rawdata()
+    {
+      return data;
+    }
+  inline void read(u_char **base)
+    {
+      unsigned int copy, suff;
+      
+      copy  = **base;
+      *base = *base + 1;
+      suff  = **base;
+      *base = *base + 1;
+      memcpy( (data + copy + 1), *base, suff);
+      *data = copy + suff;
+      *base += suff;
+    }
+  inline void read(FILE *dict)
+    {
+      unsigned int copy, suff;
+      copy = fgetc(dict);
+      suff = fgetc(dict);
+      fread(data + copy + 1, sizeof(u_char), suff, dict);
+      *data = copy + suff;
+    }
+  inline static void readString(u_char **base, u_char *prev)
+    {
+      ((mgString *) &prev)->read(base);
+    }
+  inline static void readString(FILE *dict, u_char *prev)
+    { 
+      ((mgString *) &prev)->read(dict);
+    }
+};
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/longlong.h
===================================================================
--- /main/tags/3.01/mgpp/lib/longlong.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/longlong.h	(revision 21112)
@@ -0,0 +1,85 @@
+/**************************************************************************
+ *
+ * longlong.h -- Use of GCC's long long integer types
+ * Copyright (C) 1999 Tim A.H. Bell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_LONGLONG
+#define H_LONGLONG
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/*
+  Use GCC's "long long" integer types for certain variables, to avoid
+  overflowing 32 bit integers.
+
+  Other files affected:
+  
+    lib:
+        bitio_random.c
+        bitio_random.h
+	
+    src/text:
+        ivf.pass1.c
+        ivf.pass2.c
+        build.h
+        mg_passes.c
+        text.h
+    
+*/
+
+/*#define TESTING_OVERFLOW*/
+
+#ifdef TESTING_OVERFLOW
+
+/* Test the overflow detection by using tiny (16-bit) types */
+typedef unsigned short int mg_ullong;
+typedef short int mg_llong;
+
+#define ULL_FS "u"
+#define LL_FS "d"
+
+#elif defined __GNUC__ && ! defined DISABLE_LONG_LONG
+
+/* We're using GCC, so it's okay to use "long long" (64-bit) types */
+#define USE_LONG_LONG
+
+typedef unsigned long long int mg_ullong;
+typedef long long int mg_llong;
+
+#define ULL_FS "llu"
+#define LL_FS "lld"
+
+#else
+
+/* Not using GCC, so fall back on plain "long" (32-bit) types */
+typedef unsigned long int mg_ullong;
+typedef long int mg_llong;
+
+#define ULL_FS "lu"
+#define LL_FS "ld"
+
+#endif /* __GNUC__ */
+
+#endif
Index: /main/tags/3.01/mgpp/lib/lovinstem.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/lovinstem.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/lovinstem.cpp	(revision 21112)
@@ -0,0 +1,1172 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+/*static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();*/
+
+static int 
+aio (char ch)			/* Rule number 9     */
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (char ch)				/* Rule Number 24    */
+{
+  return (ch != 's');
+}
+
+static int 
+pt (char ch)				/* Rule number 28    */
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (char ch)				/* Rule number 30    */
+{
+  return (ch != 'm');
+}
+
+static int 
+n (char ch)				/* Rule number 32    */
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+/*static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+*/
+static int 
+B (int stem_length, char *end)
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (int stem_length, char *end)
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (int stem_length, char *end)
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (int stem_length, char *end)
+
+     
+{
+  return (*end != 'e');
+}
+
+static int 
+F (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (int stem_length, char *end)
+     
+     
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'c');
+}
+
+static int 
+W (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (int stem_length, char *end)
+     
+     
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (int stem_length, char *end)
+     
+     
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (int stem_length, char *end)
+     
+     
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (int stem_length, char *end)
+     
+     
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (int stem_length, char *end)
+     
+     
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (int stem_length, char *end)
+     
+     
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  memcpy (copy, word, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  memcpy (word, copy, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) (char);	        /* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  ++rule;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) (int length, char *end);		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      ++suffix_start;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /main/tags/3.01/mgpp/lib/lovinstem.h
===================================================================
--- /main/tags/3.01/mgpp/lib/lovinstem.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/lovinstem.h	(revision 21112)
@@ -0,0 +1,28 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/memlib.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/memlib.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/memlib.cpp	(revision 21112)
@@ -0,0 +1,41 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+char *
+my_strdup(const char *str)
+{
+  char *ret_str = new char[strlen(str)+1];
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_strdup;
Index: /main/tags/3.01/mgpp/lib/memlib.h
===================================================================
--- /main/tags/3.01/mgpp/lib/memlib.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/memlib.h	(revision 21112)
@@ -0,0 +1,55 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+/* changed default behaviour - we now assume std memory. - john mcp. */
+/* #ifdef STD_MEM */
+#if !defined(NOT_STD_MEM)
+#include <string.h>
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+
+#if defined (__WIN32__)
+# define Xstrdup _strdup
+#else
+# define Xstrdup strdup
+#endif
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /main/tags/3.01/mgpp/lib/messages.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/messages.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/messages.cpp	(revision 21112)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef SILENT
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /main/tags/3.01/mgpp/lib/messages.h
===================================================================
--- /main/tags/3.01/mgpp/lib/messages.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/messages.h	(revision 21112)
@@ -0,0 +1,49 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+void Message (const char *fmt,...);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
+
+
+
+
Index: /main/tags/3.01/mgpp/lib/mgheap.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/mgheap.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/mgheap.cpp	(revision 21112)
@@ -0,0 +1,148 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; --__i)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; --__i)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	++child;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; --i)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; --i)
+    {
+      SWAP (size, (char *) heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  --(*num);
+  SWAP (size, (char *) heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  ++(*num);
+  heap_pullup (heap, size, *num, hc);
+}
Index: /main/tags/3.01/mgpp/lib/mgheap.h
===================================================================
--- /main/tags/3.01/mgpp/lib/mgheap.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/mgheap.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /main/tags/3.01/mgpp/lib/netorder.h
===================================================================
--- /main/tags/3.01/mgpp/lib/netorder.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/netorder.h	(revision 21112)
@@ -0,0 +1,177 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+#ifndef WORDS_BIGENDIAN
+
+/* double */
+#define HTOND(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy ( (void *) &d, (const void *) &tmpl, sizeof (double) >> 1);                           \
+	     memcpy ((void *) ((unsigned int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                              \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &d, sizeof(double) >> 1); \
+	     memcpy ((void *) &tmpl, (const void *) ((int) &d + (sizeof(double) >> 1)), sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &d, (const void *) &tmpl, sizeof (double) >> 1); \
+	     memcpy ((void *) ((int) &d + (sizeof(double) >> 1)), (const void *) &tmph, sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &tmph, (const void *) &hd, sizeof(double) >> 1);\
+	     memcpy ((void *) &tmpl, (const void *) ((unsigned int) &hd + (sizeof(double) >> 1)), sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                             \
+	     tmpl = htonl (tmpl);                                             \
+	     memcpy((void *) &nd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) ((unsigned int) &nd + (sizeof(double) >> 1)), (void *) &tmph, sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                        \
+        do {                                                                  \
+             unsigned long tmph, tmpl;                                        \
+	     memcpy ((void *) &nd, (void *) &tmph, sizeof(double) >> 1);      \
+	     memcpy ((void *) &tmpl, (const void *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                             \
+	     tmpl = ntohl (tmpl);                                             \
+	     memcpy((void *) &hd, (const void *) &tmpl, sizeof (double) >> 1);\
+	     memcpy ((void *) &hd + (sizeof(double) >> 1), (const void *) &tmph, sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define NTOHF(f)                                                              \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(f), sizeof (float));     \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(f), (const void *) &tmp, sizeof (float));     \
+	}while(0)
+#define HTONF2(hf, nf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(hf), sizeof (float));    \
+             HTONUL (tmp);                                                    \
+	     memcpy ((void *) &(nf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+#define NTOHF2(nf, hf)                                                        \
+        do {                                                                  \
+             unsigned long tmp;                                               \
+             memcpy ((void *) &tmp, (const void *) &(nf), sizeof (float));    \
+	     NTOHUL (tmp);                                                    \
+	     memcpy ((void *) &(hf), (const void *) &tmp, sizeof (float));    \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* WORDS_BIGENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /main/tags/3.01/mgpp/lib/non_ansi.h
===================================================================
--- /main/tags/3.01/mgpp/lib/non_ansi.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/non_ansi.h	(revision 21112)
@@ -0,0 +1,24 @@
+#if !defined(_NON_ANSI_H)
+#define _NON_ANSI_H 1
+
+#include "config.h" 
+# if !defined(__STDC__)
+
+#include <stdlib.h>  /* for FILE def */
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+int fileno(FILE *);
+int ftruncate(int, long);
+int pclose(FILE *);
+FILE *popen(const char *command, const char *type);
+int getpagesize();
+int strcasecmp(const char *, const char *);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* !defined STDC, ... */
+#endif
Index: /main/tags/3.01/mgpp/lib/pathmax.h
===================================================================
--- /main/tags/3.01/mgpp/lib/pathmax.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/pathmax.h	(revision 21112)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /main/tags/3.01/mgpp/lib/perf_hash.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/perf_hash.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/perf_hash.cpp	(revision 21112)
@@ -0,0 +1,1074 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+#define STRUCT
+
+#include <time.h>  /* for call to time () in SEED_RANDOM() */
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+
+long irandm (long is[2]);  /* from random.c */
+
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+_delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    _delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    _delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    _delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = (struct Stack *)Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = (struct Stack *) Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  ++StackPos;
+  while (StackPos)
+    {
+      int m, b;
+      --StackPos;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      _delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      _delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      _delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; ++e)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; ++v)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      *n = (int) (num * c + 1);
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; ++i)
+	for (j = 0; j < MAX_L; ++j)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; ++e)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      ++w;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int _xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	_xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ _xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; ++e)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; ++i)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; ++j)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; ++i)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; ++i)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  memset (translate, '\0', sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; ++i)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, ++s)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; ++i)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = (int *) Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = (int *)Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = (int *)Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = (int *)Xmalloc (sizeof (int) * MAX_M);
+  mk = (char *)Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = (int *)Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = (tb_entry **)Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; ++i)
+    if (tb)
+      if (!(tb[i] = (tb_entry *)Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  memset(translate, '\0', sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  MAX_N = MAX_M + MAX_M / 4;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  memcpy ((void *) g, (const void *) &g[1], n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = (perf_hash_data *)Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = (u_char *)Xmalloc (sizeof (u_char) * 256);
+  phd->g = (int *)Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = (struct tb_entry **)Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb)
+      if (!(phd->tb[i] = (struct tb_entry *)Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; ++i)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; ++i)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; ++i)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; ++j)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; ++i)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      --l;
+      ++s;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /main/tags/3.01/mgpp/lib/perf_hash.h
===================================================================
--- /main/tags/3.01/mgpp/lib/perf_hash.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/perf_hash.h	(revision 21112)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/random.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/random.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/random.cpp	(revision 21112)
@@ -0,0 +1,211 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /main/tags/3.01/mgpp/lib/random.h
===================================================================
--- /main/tags/3.01/mgpp/lib/random.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/random.h	(revision 21112)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/simplefrenchstem.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/simplefrenchstem.cpp	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      --last;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') --last;
+      if (last>=0 && wordstart[last]=='r') --last;
+      if (last>=0 && wordstart[last]=='e') --last;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	--last;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) --last;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /main/tags/3.01/mgpp/lib/simplefrenchstem.h
===================================================================
--- /main/tags/3.01/mgpp/lib/simplefrenchstem.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/simplefrenchstem.h	(revision 21112)
@@ -0,0 +1,26 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /main/tags/3.01/mgpp/lib/strcasecmp.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/strcasecmp.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/strcasecmp.cpp	(revision 21112)
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(char *s1, char *s2)
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /main/tags/3.01/mgpp/lib/strstr.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/strstr.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/strstr.cpp	(revision 21112)
@@ -0,0 +1,42 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (char *s1, char *s2)
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, ++i, ++s)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; ++p1, ++p2)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /main/tags/3.01/mgpp/lib/sysfuncs.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/sysfuncs.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/sysfuncs.cpp	(revision 21112)
@@ -0,0 +1,6 @@
+#include <stdlib.h>
+
+void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+
Index: /main/tags/3.01/mgpp/lib/sysfuncs.h
===================================================================
--- /main/tags/3.01/mgpp/lib/sysfuncs.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/sysfuncs.h	(revision 21112)
@@ -0,0 +1,388 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+#include <sys/stat.h>
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#ifndef alloca
+#define alloca __builtin_alloca
+#endif
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+#ifdef __CYGWIN__ 
+/* Cygwin has setbuffer in stdio.h, but can't link against it!! */
+#include <stdio.h>
+#endif
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+/*#define index strchr 
+ *
+ * this is stuffing up our code so its commented out */
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+
+#ifdef __cplusplus /* compiling for C++ */
+#include <stdlib.h>
+#ifndef __bcopy
+inline void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+#define __bcopy 1
+#endif
+#ifndef __bcmp
+inline int bcmp(void *s1, void *s2, int n)
+{ return memcmp(s1, s2, n);
+}
+#define __bcmp 1
+#endif
+#ifndef __bzero
+inline void bzero(void *m, int n)
+{ memset(m, 0, n);
+}
+#define __bzero 1
+#endif
+#else              /* compiling for C */
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#endif
+
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#ifdef __MSDOS__
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#ifdef __MSDOS__
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#ifndef __WIN32__
+#include <pathmax.h>
+#endif
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+/*#include <getopt.h>*/
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+/*** setlocale is ansi and posix...(jrm21)
+#if !HAVE_SETLOCALE
+#define setlocale(Category, Locale)
+#endif
+***/
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#ifdef HAVE_VPRINTF
+void error __P ((int, int, const char *,...));
+#else
+void error ();
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /main/tags/3.01/mgpp/lib/unitool.cpp
===================================================================
--- /main/tags/3.01/mgpp/lib/unitool.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/unitool.cpp	(revision 21112)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; --i) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; ++i) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      ++curpos;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+/*unsigned long *unicode_cat_data;*/
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = (unsigned long *) malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) ++letter_here;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) ++digit_here;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) ++space_here;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	++i;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    ++len;
+    ++str;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      ++outhere;
+      ++outlen;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    ++in_i;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /main/tags/3.01/mgpp/lib/unitool.h
===================================================================
--- /main/tags/3.01/mgpp/lib/unitool.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/unitool.h	(revision 21112)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /main/tags/3.01/mgpp/lib/win32in.c
===================================================================
--- /main/tags/3.01/mgpp/lib/win32in.c	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/win32in.c	(revision 21112)
@@ -0,0 +1,85 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#include "win32in.h"
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+mg_ullong ntohll(mg_ullong x)
+{
+#if defined (LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = 
+	  ((x >> 56) & 0x00000000000000FF) | ((x >> 40) & 0x000000000000FF00) |
+	  ((x >> 24) & 0x0000000000FF0000) | ((x >> 8)  & 0x00000000FF000000) |
+	  ((x << 8)  & 0x000000FF00000000) | ((x << 24) & 0x0000FF0000000000) |
+		((x << 40) & 0x00FF000000000000) | ((x << 56) & 0xFF00000000000000);
+#endif
+  return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /main/tags/3.01/mgpp/lib/win32in.h
===================================================================
--- /main/tags/3.01/mgpp/lib/win32in.h	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/win32in.h	(revision 21112)
@@ -0,0 +1,20 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+#ifdef __WIN32__
+#define _LITTLE_ENDIAN
+#endif
+
+#include "longlong.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+mg_ullong ntohll(mg_ullong x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
+
+#ifdef __cplusplus
+}
+#endif
Index: /main/tags/3.01/mgpp/lib/xmalloc.c
===================================================================
--- /main/tags/3.01/mgpp/lib/xmalloc.c	(revision 21112)
+++ /main/tags/3.01/mgpp/lib/xmalloc.c	(revision 21112)
@@ -0,0 +1,111 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if defined(__WIN32__)
+void error (int, int, const char *, ...);
+#else
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error (int, int, char *, char *, char *, char *, char *, char *, char *, char *, char *);
+#endif
+#endif
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /main/tags/3.01/mgpp/mkinstalldirs
===================================================================
--- /main/tags/3.01/mgpp/mkinstalldirs	(revision 21112)
+++ /main/tags/3.01/mgpp/mkinstalldirs	(revision 21112)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /main/tags/3.01/mgpp/stamp-h.in
===================================================================
--- /main/tags/3.01/mgpp/stamp-h.in	(revision 21112)
+++ /main/tags/3.01/mgpp/stamp-h.in	(revision 21112)
@@ -0,0 +1,1 @@
+timestamp
Index: /main/tags/3.01/mgpp/text/FIvfLevelInfo.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FIvfLevelInfo.cpp	(revision 21112)
@@ -0,0 +1,129 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.cpp -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FIvfLevelInfo.h"
+
+
+IvfLevelInfo::IvfLevelInfo () {
+  Clear();
+}
+
+void IvfLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  numEntries = 0;
+  approxWeightsDiskPtr = 0;
+  exactWeightsDiskPtr = 0;
+  workInLevel = false;
+}
+
+bool IvfLevelInfo::Read (FILE *f) {
+  workInLevel = false;
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, numEntries) &&
+	  ReadUL (f, approxWeightsDiskPtr) &&
+	  ReadUL (f, exactWeightsDiskPtr));
+}
+
+bool IvfLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, numEntries) &&
+	  WriteUL (f, approxWeightsDiskPtr) &&
+	  WriteUL (f, exactWeightsDiskPtr));
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  numEntries: " << l.numEntries << "\n"
+    << "  approxWeightsDiskPtr: " << l.approxWeightsDiskPtr << "\n"
+    << "  exactWeightsDiskPtr: " << l.exactWeightsDiskPtr << "\n"
+    << "  workInLevel: " << ((l.workInLevel) ? "true" : "false") << "\n\n";
+  
+  return s;
+}
+
+
+
+FIvfLevel::FIvfLevel () {
+  Clear ();
+}
+
+void FIvfLevel::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FIvfLevel::Read (FILE *f) {
+  if (!ReadUCArray (f, docTag) ||
+      !ReadUCArray (f, indexLevel))
+    return false;
+  
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+  
+  IvfLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FIvfLevel::Write (FILE *f) const {
+  if (!WriteUCArray (f, docTag) ||
+      !WriteUCArray (f, indexLevel))
+    return false;
+  
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  IvfLevelInfoMap::const_iterator here = levelInfo.begin();
+  IvfLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l) {
+  s << "docTag: \"" << l.docTag << "\"\n"
+    << "indexLevel: \"" << l.indexLevel << "\"\n";
+  
+  IvfLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  IvfLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
Index: /main/tags/3.01/mgpp/text/FIvfLevelInfo.h
===================================================================
--- /main/tags/3.01/mgpp/text/FIvfLevelInfo.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FIvfLevelInfo.h	(revision 21112)
@@ -0,0 +1,77 @@
+/**************************************************************************
+ *
+ * FIvfLevelInfo.h -- File functions for document levels
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FIVFLEVELINFO_H
+#define FIVFLEVELINFO_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+class IvfLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long numEntries; // N for level
+  unsigned long approxWeightsDiskPtr;
+  unsigned long exactWeightsDiskPtr;
+  bool workInLevel;  // this is just a working variable,
+                     // it is not saved or read from the file
+  
+  IvfLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const IvfLevelInfo &l);
+
+
+typedef map<UCArray, IvfLevelInfo, LTUCArray> IvfLevelInfoMap;
+
+
+class FIvfLevel {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  IvfLevelInfoMap levelInfo;
+  
+  FIvfLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FIvfLevel &l);
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/FText.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/FText.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FText.cpp	(revision 21112)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * FText.cpp -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FText.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+BitAddr::BitAddr () {
+  Clear ();  
+}
+
+void BitAddr::Clear () {
+  byte = 0;
+  bit = 0;
+}
+
+bool BitAddr::Read (FILE *f) {
+  return (ReadUL (f, byte) && ReadUC (f, bit));
+}
+
+bool BitAddr::Write (FILE *f) const {
+  return (WriteUL (f, byte) && WriteUC (f, bit));
+}
+
+#define BITADDRLEN (sizeof(unsigned long) + sizeof(unsigned char))
+
+  
+TextLevelInfo::TextLevelInfo () {
+  Clear();
+}
+
+void TextLevelInfo::Clear () {
+  levelTag.erase (levelTag.begin(), levelTag.end());
+  textIdxPtr = 0;
+  numEntries = 0;
+}
+
+bool TextLevelInfo::Read (FILE *f) {
+  return (ReadUCArray (f, levelTag) &&
+	  ReadUL (f, textIdxPtr) &&
+	  ReadUL (f, numEntries));
+}
+
+bool TextLevelInfo::Write (FILE *f) const {
+  return (WriteUCArray (f, levelTag) &&
+	  WriteUL (f, textIdxPtr) &&
+	  WriteUL (f, numEntries));
+}
+
+ostream &operator<<(ostream &s, const TextLevelInfo &l) {
+  s << "  Tag: \"" << l.levelTag << "\"\n"
+    << "  textIdxPtr: " << l.textIdxPtr << "\n"
+    << "  numEntries: " << l.numEntries << "\n\n";
+  
+  return s;
+}
+
+
+FTextLevel::FTextLevel () {
+  Clear();
+}
+
+void FTextLevel::Clear () {
+  levelInfo.erase (levelInfo.begin(), levelInfo.end());
+}
+
+bool FTextLevel::Read (FILE *f) {
+  // read in the array size
+  unsigned long arrSize = 0;
+  if (!ReadVarLenUL (f, arrSize)) return false;
+
+  TextLevelInfo thisLevel;
+  while (arrSize > 0) {
+    // read and insert the next level information
+    if (!thisLevel.Read (f)) return false;
+    levelInfo[thisLevel.levelTag] = thisLevel;
+    
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool FTextLevel::Write (FILE *f) const {
+  // write out the array size
+  if (!WriteVarLenUL (f, levelInfo.size())) return false;
+
+  // write out each level info
+  TextLevelInfoMap::const_iterator here = levelInfo.begin();
+  TextLevelInfoMap::const_iterator end = levelInfo.end();
+  while (here != end) {
+    if (!(*here).second.Write (f)) return false;
+    
+    ++here;
+  }
+
+  return true;
+}
+
+ostream &operator<<(ostream &s, const FTextLevel &l) {
+  TextLevelInfoMap::const_iterator here = l.levelInfo.begin ();
+  TextLevelInfoMap::const_iterator end = l.levelInfo.end ();
+  while (here != end) {
+    s << (*here).second;
+    ++here;
+  }
+  
+  return s;
+}
+
+
+TextIdx::TextIdx () {
+  Clear ();
+}
+
+void TextIdx::Clear () {
+  start.Clear();
+  end.Clear();
+  which = 0;
+}
+
+#define TEXTIDXLEN (BITADDRLEN*2 + sizeof(char))
+
+bool TextIdx::Read (FILE *f, const TextLevelInfo &levelInfo,
+		    unsigned long docNum) {
+  if (!SeekTextIdx (f, levelInfo, docNum)) return false;
+  return Read (f);
+}
+
+bool TextIdx::Read (FILE *f) {
+  return (start.Read (f) && end.Read (f) && ReadUC (f, which));
+}
+
+bool TextIdx::Write (FILE *f) const {
+  return (start.Write (f) && end.Write (f) && WriteUC (f, which));
+}
+
+ostream &operator<<(ostream &s, const TextIdx &t) {
+  s << "start byte: " << t.start.byte << "\n";
+  s << "start bit:  " << (int)t.start.bit << "\n";
+  s << "end byte:   " << t.end.byte << "\n";
+  s << "end bit:    " << (int)t.end.bit << "\n";
+  s << "which:      " << (int)t.which << "\n\n";
+  
+  return s;
+}
+
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum) {
+  if (docNum == 0 || docNum > levelInfo.numEntries) return false;
+  
+  unsigned long seekPos = levelInfo.textIdxPtr + (docNum-1) * TEXTIDXLEN;
+  if (fseek (f, seekPos, SEEK_SET) != 0) return false;
+  
+  return true;
+}
+
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize) {
+  // clear the array
+  a.erase (a.begin(), a.end());
+
+  // read in each element
+  TextIdx idx;
+  while (arrSize > 0) {
+    if (!idx.Read(f)) return false;
+    a.push_back (idx);
+    --arrSize;
+  }
+
+  return true;
+}
+
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a) {
+  TextIdxArray::const_iterator here = a.begin();
+  TextIdxArray::const_iterator end = a.end();
+  while (here != end) {
+    if (!(*here).Write(f)) return false;
+    ++here;
+  }
+  
+  return true;
+}
+
+
+CompressTextInfo::CompressTextInfo () {
+  Clear();
+}
+
+void CompressTextInfo::Clear () {
+  ResetStart();
+  docPtrs.erase (docPtrs.begin(), docPtrs.end());
+}
+
+void CompressTextInfo::ResetStart () {
+  inDoc = false;
+  start.Clear();
+  which = 0;
+}
+  
+void CompressTextInfo::SetStart (unsigned long startPos,
+				 unsigned char startBit,
+				 unsigned char startWhich) {
+  // place an imaginary end tag if needed
+  // note: the document tag always needs a closing tag
+  if (inDoc) SetEnd (startPos, startBit);
+
+  // remember start of level
+  inDoc = true;
+  start.byte = startPos;
+  start.bit = startBit;
+  which = startWhich;
+}
+
+void CompressTextInfo::SetEnd (unsigned long endPos,
+			       unsigned char endBit) {
+  if (inDoc) {
+    // add completed entry to list of ptrs
+    TextIdx textIdx;
+    textIdx.start = start;
+    textIdx.end.byte = endPos;
+    textIdx.end.bit = endBit;
+    textIdx.which = which;
+    docPtrs.push_back (textIdx);
+    
+    // reset this entry
+    ResetStart();
+  }
+}
+
Index: /main/tags/3.01/mgpp/text/FText.h
===================================================================
--- /main/tags/3.01/mgpp/text/FText.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FText.h	(revision 21112)
@@ -0,0 +1,150 @@
+/**************************************************************************
+ *
+ * FText.h -- File structures for text compression
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FTEXT_H
+#define FTEXT_H
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+// holds a bit address
+class BitAddr {
+public:
+  unsigned long byte;
+  unsigned char bit;
+
+  BitAddr ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+class TextLevelInfo {
+public:
+  UCArray levelTag;
+  unsigned long textIdxPtr;
+  unsigned long numEntries;
+
+  TextLevelInfo ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextLevelInfo &l);
+
+
+typedef map<UCArray, TextLevelInfo, LTUCArray> TextLevelInfoMap;
+
+
+class FTextLevel {
+public:
+  TextLevelInfoMap levelInfo;
+  
+  FTextLevel ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const FTextLevel &l);
+
+
+// the text.idx file points into the compressed text
+class TextIdx {
+public:
+  BitAddr start;       // the first bit of the compressed text
+  BitAddr end;         // the first bit past the end of the text
+  unsigned char which; // 0 = non-word, 1 = word
+  
+  TextIdx ();
+  void Clear ();
+
+  // does a seek and reads the appropriate record
+  bool Read (FILE *f, const TextLevelInfo &levelInfo,
+	     unsigned long docNum);
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// stream output for debugging purposes
+ostream &operator<<(ostream &s, const TextIdx &t);
+
+
+// note: document numbers start at 1
+
+bool SeekTextIdx (FILE *f, const TextLevelInfo &levelInfo,
+		  unsigned long docNum);
+
+
+// used to store the pointers while compressing the text
+typedef vector<TextIdx> TextIdxArray;
+
+bool ReadTextIdxArray (FILE *f, TextIdxArray &a, unsigned long arrSize);
+bool WriteTextIdxArray (FILE *f, const TextIdxArray &a);
+
+
+// used to store information about a level while compressing the text
+class CompressTextInfo {
+public:
+  bool inDoc;
+  BitAddr start;
+  unsigned char which;
+  TextIdxArray docPtrs;
+
+  CompressTextInfo ();
+  void Clear ();
+
+  // clears inDoc, start, and which (not the docPtrs)
+  void ResetStart ();
+  
+  // set the start of a level (closing off the last opening
+  // tag if needed)
+  void SetStart (unsigned long startPos,
+		 unsigned char startBit,
+		 unsigned char startWhich);
+
+  // if in a document, it will set the end, add the document to the
+  // list of document ptrs, and then call ResetStart
+  void SetEnd (unsigned long endPos,
+	       unsigned char endBit);
+};
+
+typedef map<UCArray, CompressTextInfo, LTUCArray> CompressTextInfoMap;
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/FileBuf.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/FileBuf.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FileBuf.cpp	(revision 21112)
@@ -0,0 +1,50 @@
+/**************************************************************************
+ *
+ * FileBuf.cpp -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FileBuf.h"
+#include "sysfuncs.h"
+
+#ifdef __WIN32__
+# include <io.h>
+# define read _read
+#endif
+
+bool FileBuf::NextBuf () {
+  if (fd < 0) return false;
+  int numbytes = read (fd, buf, FILEBUFSIZE);
+  if (numbytes < 0) numbytes = 0;
+  
+  here = buf;
+  end = here + numbytes;
+
+  return (numbytes > 0);
+}
+
+FileBuf::FileBuf () {
+  here = end = buf;
+  fd = -1;
+}
+
+void FileBuf::SetFD (int _fd) {
+  here = end = buf;
+  fd = _fd;
+}
+
Index: /main/tags/3.01/mgpp/text/FileBuf.h
===================================================================
--- /main/tags/3.01/mgpp/text/FileBuf.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FileBuf.h	(revision 21112)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * FileBuf.h -- File buffer
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FILEBUF_H
+#define FILEBUF_H
+
+#define FILEBUFSIZE 65536
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+class FileBuf {
+protected:
+  unsigned char buf[FILEBUFSIZE];
+  unsigned char *here;
+  unsigned char *end;
+
+  int fd;
+
+  // gets a new buffer if need be, returns true if
+  // more could be got. if true is returned the caller
+  // can assume that here != end
+  bool NextBuf ();
+  
+public:
+  FileBuf ();
+
+  void SetFD (int _fd);
+
+  inline bool Get (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here++; return true; }
+    return false;
+  }
+  
+  inline bool Peek (unsigned char &c) {
+    if (here != end || NextBuf()) { c = *here; return true; }
+    return false;
+  }
+
+  inline bool Next () {
+    if (here != end || NextBuf()) { ++here; return true; }
+    return false;
+  }
+};
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/FragLevelConvert.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/FragLevelConvert.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FragLevelConvert.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.cpp -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "FragLevelConvert.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+
+
+FragLevelConvert::FragLevelConvert () {
+  numFrags = 0;
+  numLevelDocs = 0;
+}
+
+bool FragLevelConvert::Read (FILE *invfFile,
+			     unsigned long levelPtr,
+			     unsigned long _numFrags,
+			     unsigned long _numLevelDocs) {
+  levelStarts.erase (levelStarts.begin(), levelStarts.end());
+  numFrags = _numFrags;
+  numLevelDocs = _numLevelDocs;
+
+  // seek to the appropriate place in the inverted file
+  fseek (invfFile, levelPtr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(invfFile);
+  
+  unsigned long pTag = numLevelDocs*2;
+  unsigned long B = BIO_Bblock_Init (numFrags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  for (i=0; i<numLevelDocs; ++i) {
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    levelStarts.push_back (fragNum);
+
+    // ignore end, assume every fragment is in a level
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+  }
+
+  // fragNum would be the starting fragment of the next document
+  // if there was one
+  levelStarts.push_back (fragNum);
+  
+  if (fragNum != _numFrags) {
+    cerr << "last frag num:"<<fragNum<<" != num frags:"<<_numFrags<<endl;
+  } 
+  buffer.done();
+
+  return true;
+}
+
+bool FragLevelConvert::FragToLevel (unsigned long fragNum,
+				    unsigned long &levelDocNum) const {
+  // do binary search for something containing this fragment number
+  unsigned long low = 1, high = numLevelDocs;
+  unsigned long mid;
+
+  while ((mid = (low+high)/2) >= 1 && low <= high) {
+    if (fragNum > levelStarts[mid]) low = mid+1;
+    else if (fragNum <= levelStarts[mid-1]) high = mid-1;
+    else {
+      levelDocNum = mid;
+      return true;
+    }
+  }
+  return false;
+}
+
+
+bool FragLevelConvert::LevelToLevel (FragLevelConvert sectionLevelConvert,
+		 unsigned long levelNum, unsigned long &DocNum) {
+  
+  if (levelNum==1) {
+    DocNum=1;
+    return true;
+  }
+  unsigned long levelfragnum = sectionLevelConvert.levelStarts[levelNum-1]+1;
+
+  if (FragToLevel(levelfragnum, DocNum)) {
+    return true;
+  } 
+  return false;
+  
+
+}
Index: /main/tags/3.01/mgpp/text/FragLevelConvert.h
===================================================================
--- /main/tags/3.01/mgpp/text/FragLevelConvert.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/FragLevelConvert.h	(revision 21112)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * FragLevelConvert.h -- converting between fragment and document numbers
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef FRAGLEVELCONVERT_H
+#define FRAGLEVELCONVERT_H
+
+#include <stdio.h>
+#include "UCArray.h"
+
+
+/*Notes about levelStarts (kjm18) - this is my interpretation of
+whats going on, feel free to correct this if its wrong.
+
+say you have 5 docs, consisting of fragments:
+1:  1-55
+2: 56-83
+3: 84-106
+4: 107-152
+5: 153-211
+
+then levelStarts would have the values
+0  1  2   3   4   5 
+0  55 83 106 152 211
+
+*/
+class FragLevelConvert {
+protected:
+  // note fragment start is one before before real start
+  vector<unsigned long> levelStarts;
+  unsigned long numFrags;
+  unsigned long numLevelDocs;
+  
+public:
+  FragLevelConvert ();
+
+  bool Read (FILE *invfFile, unsigned long levelPtr,
+	     unsigned long _numFrags, unsigned long _numLevelDocs);
+
+  bool FragToLevel (unsigned long fragNum, unsigned long &levelDocNum) const;
+
+  // this function converts from one level number to another
+  // the FragLevelConvert thats calling the method should be loaded with
+  // the level that you want to convert to, while the sectionLevelConvert
+  // should be loaded with the level converting from.
+  // level Num has the old level, DocNum will be returned with the new
+  // coresponding level
+  bool LevelToLevel (FragLevelConvert sectionLevelConvert,
+		   unsigned long levelNum, unsigned long &DocNum);
+};
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/GSDLQueryLex.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/GSDLQueryLex.cpp	(revision 21112)
@@ -0,0 +1,275 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size() + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //  UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParsePotentialInteger(UCArray::const_iterator &here,
+				  UCArray::const_iterator end,
+				  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  int charLen=0;
+  int length=0;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  // check the next character -if it is a letter, then have a term, not an integer
+  if (!is_unicode_letter(c)) {
+    // this was just an integer
+    return (!el.text.empty());
+  }
+  // else its a term
+  el.lexType = TermE;
+  el.num = 0;
+  /* this bit taken from ParseIndexWord in words.h*/
+  while (length+charLen<=MAXSTEMLEN && charLen > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+
+  return (!el.text.empty());
+}
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+  const unsigned char* new_here = ParseIndexWord (&*here, endMinus1, text);
+  here += (new_here - &*here); // advance iterator by number of chars advanced
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  //UCArray::const_iterator endMinus1 = end-1;
+  const unsigned char* endMinus1 = &(*end)-1;
+
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (&*here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (&*here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+
+  } else if (c =='[') {
+    el.lexType = OpenSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c ==']') {
+    el.lexType = CloseSquareBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '/') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '*') {
+    el.lexType = StarE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c=='&') {
+    el.lexType = AndOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '|') {
+    el.lexType = OrOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '!') {
+    el.lexType = NotOpE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ) {
+    return  ParseInteger (here, end, el);
+  }
+
+  else if (c >= '0' && c <= '9') {
+    return ParsePotentialInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text))  {
+    // parse term returns false if it hasn't parsed anything that is a term
+    // here should be the same as it was before 
+    el.lexType = UnknownE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  }
+  //return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+  UCArray NEAR; SetCStr (NEAR, "NEAR", 4);
+  if (PrefixLen(el.text, NEAR)==4) {
+    el.lexType = NearOpE;
+    return true;
+  }
+  UCArray WITHIN; SetCStr (WITHIN, "WITHIN", 6);
+  if (PrefixLen(el.text, WITHIN)==6) {
+    el.lexType = WithinOpE;
+    return true;
+  }
+  
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/mgpp/text/GSDLQueryLex.h
===================================================================
--- /main/tags/3.01/mgpp/text/GSDLQueryLex.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/GSDLQueryLex.h	(revision 21112)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * GSDLQueryLex.h -- Lexical analyser for a simple query language, for GSDL
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYLEX_H
+#define GSDLQUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, NearOpE, WithinOpE, QuoteE, IntegerE, TermWeightE, 
+	      StemMethodE, RangeE, AtE, TagE, OpenSquareBracketE, 
+	      CloseSquareBracketE, StarE, UnknownE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/mgpp/text/GSDLQueryParser.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/GSDLQueryParser.cpp	(revision 21112)
@@ -0,0 +1,515 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "GSDLQueryParser.h"
+#include "GSDLQueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end, 
+				   int defaultBoolCombine,
+				   int defaultStemMethod);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end, 
+					  int defaultBoolCombine,
+					  int defaultStemMethod) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end, defaultBoolCombine,
+					defaultStemMethod);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+// default is within 20 words
+static void SetRangeValues (TermNode &termNode,
+			    UCArray &nearby,
+			    bool reverse) {
+  UCArray NEARBY; SetCStr(NEARBY, "NEAR", 4);
+  UCArray WITHIN; SetCStr(WITHIN, "WITHIN", 6);
+  
+  if (nearby == NEARBY) { // no modifier
+    termNode.startRange = (NEAR_DEFAULT+1)*-1;
+    termNode.endRange = NEAR_DEFAULT;
+
+  } else if (nearby == WITHIN) { // no modifier
+    if (reverse) {
+      termNode.startRange = (NEAR_DEFAULT+1)*-1;
+      termNode.endRange = -1;
+    } else {
+      termNode.startRange = NEAR_DEFAULT;
+      termNode.endRange = 0;
+    }
+  }
+  else { // extract number
+    UCArray::const_iterator here;
+    bool within = false;
+    if (PrefixLen(nearby, WITHIN)==6) {
+      within=true;
+      here = nearby.begin()+6;
+    } else {
+      here = nearby.begin()+4;
+    }
+    UCArray::const_iterator end = nearby.end();
+    int size=0;
+    while (here != end) {
+      size = size*10 + (*here-'0');
+      ++here;
+    }
+    if (within) {
+      if (reverse) {
+	termNode.startRange = size;
+	termNode.endRange = 0;
+      } else {
+	termNode.startRange = -1 * (size+1);
+	termNode.endRange = -1;
+      }
+    } else {
+      termNode.startRange = -1 * (size+1);
+      termNode.endRange = size;
+    }
+  }
+}
+
+static unsigned long GetStemMethod(LexEl &el, int defaultStemMethod) {
+  // here expect el to contain some of c,s,i,u
+  // stem method 0 = c,u  00
+  // stem method 1 = i,u  01  - default for DL
+  // stem method 2 = c, s  10
+  // stem method 3 = i,s  11
+
+  unsigned long stem = (unsigned long)defaultStemMethod;
+      
+  UCArray::const_iterator here = el.text.begin();
+  UCArray::const_iterator end = el.text.end();
+
+  unsigned char c1 = *here; 
+  if (!(c1 == 'c'|| c1 == 'i' || c1 == 'u' || c1 == 's')) 
+    return 4; // incorrect format
+  
+  ++here;
+  unsigned char c2 = 'a';
+  if (here !=end) {
+    c2 = *here;
+    if (!(c2 == 'c'|| c2 == 'i' || c2 == 'u' || c2 == 's')) 
+      return 4; // incorrect format
+  }
+  
+  if (c1 == 'i'|| c2=='i') stem |= 1; // set bit 0 to 1
+  if (c1 == 'c' || c2 == 'c') stem &=0xe; //set bit 0 to 0
+  if (c1 == 's'|| c2 == 's') stem |= 2; // set bit 1 to 1
+  if (c1 == 'u' || c2 =='u') stem &=0xd; // set bit 1 to 0
+  return stem;
+}
+  
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode,
+				int defaultStemMethod) {
+ 
+  termNode.stemMethod = defaultStemMethod;
+  bool partial_match = false;
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      oldHere = here;
+      LexEl stem;
+      if (ParseLexEl (here, end, stem) && stem.lexType == TermE) {
+	termNode.stemMethod = GetStemMethod(stem, defaultStemMethod);
+	if (termNode.stemMethod == 4) { // error so backtrack
+	  here = oldHere;
+	  termNode.stemMethod = (unsigned long)defaultStemMethod;
+	} 
+      }else here = oldHere; //ignore - wrong syntax
+
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+    } else if (el.lexType == StarE) {
+      partial_match = true;
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    if (partial_match) {
+      termNode.stemMethod |= 4; // set bit 2 to 1
+      termNode.stemMethod &=0xd; // set bit 1 to 0 // we dont have stemming on if doing partial matching.
+    }
+    oldHere = here;
+  }  
+}
+
+static void ParseProxModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				ProxMatchQueryNode *proxNode) {
+  // so far only have one - the tag stuff
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TagE) {
+      oldHere = here; // don't backtrack past here
+      if (ParseLexEl (here, end, el) && el.lexType == TermE) {
+	proxNode->tagNodePtr = new TagNode;
+	proxNode->tagNodePtr->tagName = el.text;
+	
+      }
+      else { // error in tag
+	here = oldHere;
+      }
+    } // TagE
+    // add in other cases here
+    else {
+      // no modifiers
+      here = oldHere;
+      break;
+    }
+    oldHere = here;
+  }//while
+
+
+}
+
+// expects starting brackets to have been parsed
+// sets error to true if something has gone wrong
+static ProxMatchQueryNode *ParseSquareBrackets(UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				/*ProxMatchQueryNode *proxNode,*/
+				int defaultStemMethod,
+				bool & error) {
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+  LexEl el;
+  bool phrase=false;
+  bool first=true;
+  bool prox = false;
+  UCArray near_string;
+  while (ParseLexEl (here, end, el)) {
+    // cant have AND, OR, NOT in square brackets, so assume they are words
+    if (el.lexType == TermE || el.lexType == IntegerE || el.lexType == AndOpE || el.lexType == OrOpE || el.lexType == NotOpE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (phrase) {
+        if (first) first=false;
+        else {
+          termNode.startRange = -2;
+          termNode.endRange = -1;
+        }
+      } else if (prox) {
+	SetRangeValues(termNode, near_string, false);
+	prox = false;
+      }
+      proxNode->terms.push_back(termNode);
+    }
+    else if (el.lexType == CloseSquareBracketE) {
+      break;
+    }
+    else if (el.lexType == QuoteE) {
+      // phrase inside square brackets
+      if (phrase) { // end of phrase
+	phrase=false;
+	first = true;
+      } else {
+	phrase=true; // start of phrase
+      }
+    } else if (el.lexType == NearOpE || el.lexType == WithinOpE) {
+      if (phrase) {
+	// cant have proximity  op in a phrase - just assume its an actual word
+	TermNode termNode;
+	termNode.term = el.text;
+	ParseTermModifiers (here, end, termNode, defaultStemMethod);
+	proxNode->terms.push_back(termNode);
+      } else {
+	// its a NEAR or within op
+	prox = true;
+	near_string = el.text;
+      }
+	  
+    } 
+    else if (el.lexType == UnknownE) {
+      // just ignore it 
+    }
+    else {
+      //error - we set the proxNode to NULL,
+      cerr <<"GSDLQueryParser: bad syntax inside []\n";
+      error = true;
+      return NULL;
+    }
+  } // while
+  return proxNode;
+}
+// expects the starting quote to have been parsed
+// and discarded
+// now phrases use the case and stem preference options
+// ie can search for a phrase ignoring case
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode, 
+			 int defaultStemMethod,
+			 bool &error) {
+  LexEl el;
+  bool first = true;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      //termNode.stemMethod = defaultStemMethod;
+      ParseTermModifiers (here, end, termNode, defaultStemMethod);
+      if (first) {
+	first = false;
+      }
+      else {
+	termNode.startRange = -2;
+	termNode.endRange = -1;
+      }
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it
+    } else {
+      // error 
+      error = true;
+      return;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end,
+			     int defaultBoolCombine,
+			     int defaultStemMethod) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end, defaultBoolCombine,
+				   defaultStemMethod);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode, defaultStemMethod);
+    oldHere = here;  // dont backtrack past here
+    if (ParseLexEl(here, end, el) && (el.lexType == NearOpE || el.lexType == WithinOpE )) {
+	delete proxNode;
+	oldHere = here;
+      // this is calling ParseTerm again, but only a subset of the things accepted by ParseTerm are appropriate here. add in some hacks to avoid segmentation faults - kjdon, 04/2003
+      
+      // if the next element is a '(' have a syntax error, return NULL 
+      LexEl temp_el;
+      if (ParseLexEl(here, end, temp_el) && temp_el.lexType == OpenBracketE) {
+	cerr << "GSDLQueryParser: NEAR/WITHIN cannot be followed by a '('\n";
+	return NULL;
+      }
+      here = oldHere; // else backtrack
+	
+      proxNode = (ProxMatchQueryNode *)ParseTerm(here, end, defaultBoolCombine,
+						 defaultStemMethod);
+      SetRangeValues(termNode, el.text, true);
+      proxNode->terms.push_back (termNode);
+      return proxNode;
+      
+    } else {
+      here = oldHere; // backtrack
+      proxNode->terms.push_back (termNode);
+      ParseProxModifiers(here, end, proxNode);
+      return proxNode;
+    }
+  } else if (el.lexType == QuoteE) {
+    bool error = false;
+    ParsePhrase (here, end, *proxNode, defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    return proxNode;
+  }
+  else if (el.lexType == OpenSquareBracketE) {
+    bool error = false;
+    proxNode = ParseSquareBrackets (here, end, /*proxNode, */defaultStemMethod, error);
+    if (error) {
+      delete proxNode;
+      return NULL;
+    }
+    ParseProxModifiers (here, end, proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end,
+				   int defaultBoolCombine,
+				   int defaultStemMethod) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) {
+      // parsebracketexpression is waiting for the last bracket, so put it back
+      here = oldHere;
+      break;
+      
+    } else if (el.lexType == OpenSquareBracketE ||
+	       el.lexType == OpenBracketE ||
+	       el.lexType == TermE ||
+	       el.lexType == QuoteE ||
+	       el.lexType == IntegerE ) { 
+      
+      // some type of term, back track and parse it
+      here = oldHere;
+
+      //  parse the term
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine, 
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+
+      // if default==1, AND, else if==0, OR
+      if (defaultBoolCombine) {
+	curTree = AndAdd (curTree, newTerm);
+      }
+      else {
+	curTree = OrAdd (curTree, newTerm); 
+      }
+      
+    } else if (el.lexType == AndOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = AndAdd (curTree, newTerm);
+      
+    } else if (el.lexType == OrOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = OrAdd (curTree, newTerm);
+      
+    } else if (el.lexType == NotOpE) {
+      QueryNode * newTerm = ParseTerm (here, end, defaultBoolCombine,
+				       defaultStemMethod);
+      if (newTerm == NULL) {
+	delete curTree;
+	return NULL;
+      }
+      curTree = NotAdd (curTree, newTerm); 
+      
+    } else if (el.lexType == UnknownE) {
+      // just ignore it 
+    } else {
+
+      // syntax error, return NUll
+      delete curTree;
+      return NULL;
+    }
+
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end, defaultBoolCombine, defaultStemMethod);
+}
Index: /main/tags/3.01/mgpp/text/GSDLQueryParser.h
===================================================================
--- /main/tags/3.01/mgpp/text/GSDLQueryParser.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/GSDLQueryParser.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef GSDLQUERYPARSER_H
+#define GSDLQUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+#define NEAR_DEFAULT 20
+
+// returns NULL if the query could not be parsed
+// defaultBoolCombine determines if individual terms are combined using
+// AND or OR in the absence of specified operators.
+// defaultStemMethod used to set stemming and casefolding for terms where
+// its not set explicitly in the query string. THe defaultStemMethod value
+// should be set using the values from the preferences page
+QueryNode *ParseQuery (const UCArray &queryStr, int defaultBoolCombine,
+		       int defaultStemMethod);
+
+#endif
Index: /main/tags/3.01/mgpp/text/IndexData.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/IndexData.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/IndexData.cpp	(revision 21112)
@@ -0,0 +1,207 @@
+/**************************************************************************
+ *
+ * IndexData.cpp -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "IndexData.h"
+#include "string.h"
+#include "mg_files.h"
+
+IndexData::IndexData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  dictFile = NULL;
+
+  stem1File = NULL;
+  stem2File = NULL;
+  stem3File = NULL;
+
+  invfFile = NULL;
+
+  approxWeightsFile = NULL;
+  exactWeightsFile = NULL;
+
+  curLevelNum = 0;
+}
+
+IndexData::~IndexData () {
+  UnloadData ();
+}
+
+bool IndexData::LoadData (const char *_basePath, const char *_filename) {
+  if (_filename[0] == '\0') return false;
+  
+  // make sure this data has not already been loaded
+  if (strcmp (_basePath, basePath) == 0 &&
+      strcmp (_filename, filename) == 0) return true;
+  
+  // make sure there is nothing else loaded
+  UnloadData ();
+
+  strcpy (basePath, _basePath);
+  strcpy (filename, _filename);
+
+  set_basepath (basePath);
+
+  // load in the level information
+  FILE *ivfLevelFile;
+  ivfLevelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			    MAGIC_INVF_LEVELS, MG_ABORT);
+  levels.Read (ivfLevelFile);
+  fclose (ivfLevelFile);
+
+  // blocked dictionary
+  dictFile = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "rb",
+			MAGIC_STEM, MG_ABORT);
+  if (!bdh.Read (dictFile)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.wblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biWords)) { UnloadData (); return false; }
+
+  fseek (dictFile, bdh.tblk_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (dictFile, biTags)) { UnloadData (); return false; }
+
+  // blocked stem index 1
+  stem1File = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+			 "rb", MAGIC_STEM_1, MG_ABORT);
+  if (!sih1.Read (stem1File)) { UnloadData (); return false; }
+
+  fseek (stem1File, sih1.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem1File, sii1)) { UnloadData (); return false; }
+  
+  // blocked stem index 2
+  stem2File = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+			 "rb", MAGIC_STEM_2, MG_ABORT);
+  if (!sih2.Read (stem2File)) { UnloadData (); return false; }
+
+  fseek (stem2File, sih2.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem2File, sii2)) { UnloadData (); return false; }
+  
+  // blocked stem index 3
+  stem3File = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+			 "rb", MAGIC_STEM_3, MG_ABORT);
+  if (!sih3.Read (stem3File)) { UnloadData (); return false; }
+
+  fseek (stem3File, sih3.block_idx_start, SEEK_SET);
+  if (!ReadBlockIdx (stem3File, sii3)) { UnloadData (); return false; }
+
+  // inverted file
+  invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  ifh.Read (invfFile);
+
+  // weights
+  approxWeightsFile = open_file (filename, APPROX_WEIGHTS_SUFFIX, "rb",
+				 MAGIC_WGHT_APPROX, MG_ABORT);
+  exactWeightsFile = open_file (filename, WEIGHTS_SUFFIX, "rb",
+				MAGIC_WGHT, MG_ABORT);
+
+  return true;
+}
+
+bool IndexData::UnloadData () {
+  basePath[0] = '\0';
+  filename[0] = '\0';
+
+  if (dictFile != NULL) {
+    fclose (dictFile); dictFile = NULL;
+  }
+
+  if (stem1File != NULL) {
+    fclose (stem1File); stem1File = NULL;
+  }
+  if (stem2File != NULL) {
+    fclose (stem2File); stem2File = NULL;
+  }
+  if (stem3File != NULL) {
+    fclose (stem3File); stem3File = NULL;
+  }
+
+  if (invfFile != NULL) {
+    fclose (invfFile); invfFile = NULL;
+  }
+
+  if (approxWeightsFile != NULL) {
+    fclose (approxWeightsFile); approxWeightsFile = NULL;
+  }
+  if (exactWeightsFile != NULL) {
+    fclose (exactWeightsFile); exactWeightsFile = NULL;
+  }
+
+  UnloadLevel();
+  
+  return true;
+}
+
+
+bool IndexData::LoadLevel (const UCArray &level) {
+  // make sure this level is not already loaded
+  if (level == curLevel) return true;
+
+  // unload any levels currently loaded
+  UnloadLevel();
+
+  // make sure the required files are open
+  if (dictFile == NULL || invfFile == NULL ||
+      approxWeightsFile == NULL || exactWeightsFile == NULL)
+    return false;
+  
+  // read in the information from the dictionary
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (dictFile, biTags, bdh.entries_per_tblk,
+			  bdh.tag_dict_size, level, tagEl, tagElNum))
+    return false;
+
+  // read in the level conversion information
+  if (!levelConverter.Read (invfFile, tagEl.invf_ptr,
+			    bdh.num_frags, tagEl.frag_occur))
+    return false;
+
+  // read in the approximate weights
+  if (!weightData.Read (approxWeightsFile,
+			levels.levelInfo[level].approxWeightsDiskPtr,
+			levels.levelInfo[level].numEntries))
+    return false;
+  
+  // find the level number
+  curLevelNum = 0;
+  IvfLevelInfoMap::const_iterator levelHere, levelEnd;
+  for (levelHere=levels.levelInfo.begin(), levelEnd=levels.levelInfo.end();
+       levelHere!=levelEnd && (*levelHere).first != level; ++levelHere)
+    ++curLevelNum;
+
+  // make sure we found the level
+  if (levelHere == levelEnd) return false;
+
+  curLevel = level;
+
+  return true;
+}
+
+bool IndexData::UnloadLevel () {
+  curLevel.erase (curLevel.begin(), curLevel.end());
+  curLevelNum = 0;
+  
+  weightData.Free ();
+    
+  return true;
+}
+
Index: /main/tags/3.01/mgpp/text/IndexData.h
===================================================================
--- /main/tags/3.01/mgpp/text/IndexData.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/IndexData.h	(revision 21112)
@@ -0,0 +1,82 @@
+/**************************************************************************
+ *
+ * IndexData.h -- Information needed for querying
+ * Copyright (C) 1999 Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef INDEXDATA_H
+#define INDEXDATA_H
+
+#include <stdio.h>
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+#include "Weights.h"
+
+
+class IndexData {
+public:
+  // general information
+  char basePath[FILENAME_MAX];
+  char filename[FILENAME_MAX];
+  FIvfLevel levels;
+  
+  // blocked dictionary
+  FILE *dictFile;
+  block_dict_header bdh;
+  block_idx biWords;
+  block_idx biTags;
+
+  // stem indexes
+  FILE *stem1File;
+  FILE *stem2File;
+  FILE *stem3File;
+  stem_idx_header sih1;
+  stem_idx_header sih2;
+  stem_idx_header sih3;
+  block_idx sii1;
+  block_idx sii2;
+  block_idx sii3;
+
+  // inverted file
+  FILE *invfFile;
+  invf_file_header ifh;
+
+  // weights
+  FILE *approxWeightsFile;
+  FILE *exactWeightsFile;
+
+  // level specific information
+  UCArray curLevel; // last level that was loaded
+  unsigned long curLevelNum;
+  FragLevelConvert levelConverter;
+  ApproxWeightsData weightData;
+  
+  IndexData ();
+  ~IndexData ();
+
+  // LoadData will only load the data if it has not been loaded before
+  bool LoadData (const char *_basePath, const char *_filename);
+  bool UnloadData ();
+
+  bool LoadLevel (const UCArray &level);
+  bool UnloadLevel ();
+};
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/MGQuery.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/MGQuery.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/MGQuery.cpp	(revision 21112)
@@ -0,0 +1,760 @@
+/**************************************************************************
+ *
+ * MGQuery.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "Terms.h"
+#include "QueryResultsSort.h"
+#include <assert.h>
+
+
+void PrintIndent (ostream &s, int indent) {
+  while (indent-- > 0) s << " ";
+}
+
+
+void PrintIndentText (ostream &s, char *text, int indent) {
+  PrintIndent (s, indent);
+  s << text;
+}
+
+void PrintNode (ostream &s, QueryNode *node, int indent) {
+  if (node == NULL) {
+    PrintIndentText (s, "NULL\n", indent);
+  } else {
+    node->Print (s, indent+2);
+  }
+}
+
+  
+
+QueryNode::QueryNode () {
+}
+
+QueryNode::~QueryNode () {
+}
+
+void QueryNode::Calculate (IndexData &/*indexData*/,
+			   const QueryInfo &/*queryInfo*/,
+			   QueryResult &result) const {
+  result.Clear();
+}
+
+void QueryNode::Free () {
+}
+
+void QueryNode::Print (ostream &/*s*/, int /*indent*/) const {
+}
+
+
+
+AndQueryNode::AndQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+AndQueryNode::~AndQueryNode () {
+  Free ();
+}
+
+void AndQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // an And between nothing and something is nothing...
+  if (leftNode == NULL || rightNode == NULL) return;
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult rightResult;
+  leftNode->Calculate (indexData, queryInfo, result);
+  rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with an And
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long outI = 0;
+  while (leftI < result.docs.size() &&
+	 rightI < rightResult.docs.size()) {
+    if (result.docs[leftI] < rightResult.docs[rightI]) {
+      ++leftI;
+    } else if (result.docs[leftI] > rightResult.docs[rightI]) {
+      ++rightI;
+    } else {
+      // the documents are equal
+      result.docs[outI] = result.docs[leftI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[leftI] + rightResult.ranks[rightI];
+      ++leftI;
+      ++rightI;
+      ++outI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+    
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+}
+
+void AndQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void AndQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "AND\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+OrQueryNode::OrQueryNode () {
+  leftNode = NULL;
+  rightNode = NULL;
+}
+
+OrQueryNode::~OrQueryNode () {
+  Free ();
+}
+
+void OrQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+  
+  // calculate the result from the left tree and the result
+  // from the right tree
+  QueryResult leftResult;
+  QueryResult rightResult;
+  if (leftNode != NULL)
+    leftNode->Calculate (indexData, queryInfo, leftResult);
+  if (rightNode != NULL)
+    rightNode->Calculate (indexData, queryInfo, rightResult);
+
+  // merge the results
+  
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (leftResult.ranks.size() != leftResult.docs.size() ||
+		    rightResult.ranks.size() != rightResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long leftSize = leftResult.docs.size();
+  unsigned long rightSize = rightResult.docs.size();
+  unsigned long leftI = 0;
+  unsigned long rightI = 0;
+  unsigned long leftDocNum = 0;
+  unsigned long rightDocNum = 0;
+  while (leftI < leftSize || rightI < rightSize) {
+    // check leftI
+    if (leftI < leftResult.docs.size())
+      leftDocNum = leftResult.docs[leftI];
+    else leftDocNum = ULONG_MAX;
+    
+    // check rightI
+    if (rightI < rightResult.docs.size())
+      rightDocNum = rightResult.docs[rightI];
+    else rightDocNum = ULONG_MAX;
+    
+    // combine
+    if (leftDocNum < rightDocNum) {
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI]);
+      ++leftI;
+      
+    } else if (leftDocNum > rightDocNum) {
+      result.docs.push_back (rightDocNum);
+      if (haveAccum)
+	result.ranks.push_back (rightResult.ranks[rightI]);
+      ++rightI;
+      
+    } else { // equal
+      result.docs.push_back (leftDocNum);
+      if (haveAccum)
+	result.ranks.push_back (leftResult.ranks[leftI] +
+				rightResult.ranks[rightI]);
+      ++leftI;
+      ++rightI;
+    }
+  }
+
+  // combine term frequency information
+  if (queryInfo.needTermFreqs) {
+    result.termFreqs.insert (result.termFreqs.end(),
+			     leftResult.termFreqs.begin(),
+			     leftResult.termFreqs.end());
+    result.termFreqs.insert (result.termFreqs.end(),
+			     rightResult.termFreqs.begin(),
+			     rightResult.termFreqs.end());
+  }
+}
+
+void OrQueryNode::Free () {
+  if (leftNode != NULL) {
+    delete leftNode;
+    leftNode = NULL;
+  }
+  if (rightNode != NULL) {
+    delete rightNode;
+    rightNode = NULL;
+  }
+}
+
+void OrQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "leftNode:\n", indent);
+  PrintNode (s, leftNode, indent+2);
+  PrintIndentText (s, "OR\n", indent);
+  PrintIndentText (s, "rightNode:\n", indent);
+  PrintNode (s, rightNode, indent+2);
+}
+
+
+
+NotQueryNode::NotQueryNode () {
+  queryNode = NULL;
+  notNode = NULL;
+}
+
+NotQueryNode::~NotQueryNode () {
+  Free ();
+}
+
+void NotQueryNode::Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			      QueryResult &result) const {
+  result.Clear();
+
+  // check for nothing
+  if (queryNode == NULL) return;
+  if (notNode == NULL) {
+    queryNode->Calculate (indexData, queryInfo, result);
+    return;
+  }
+  
+  // calculate the result from the query tree and the result
+  // from the not tree
+  QueryResult notResult;
+  queryNode->Calculate (indexData, queryInfo, result);
+  notNode->Calculate (indexData, queryInfo, notResult);
+
+  // merge the results, this can be done in place as the results
+  // will always shrink with a Not
+
+  bool haveAccum = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  if (haveAccum && (result.ranks.size() != result.docs.size() ||
+		    notResult.ranks.size() != notResult.docs.size())) {
+    // shouldn't ever get here
+    haveAccum = false;
+    assert (0);
+  }
+
+  // combine document numbers and corresponding ranks
+  unsigned long queryI = 0;
+  unsigned long notI = 0;
+  unsigned long outI = 0;
+  while (queryI < result.docs.size() &&
+	 notI < notResult.docs.size()) {
+    if (result.docs[queryI] < notResult.docs[notI]) {
+      // found a document not in the notResults
+      result.docs[outI] = result.docs[queryI];
+      if (haveAccum)
+	result.ranks[outI] = result.ranks[queryI];
+      ++queryI;
+      ++outI;
+    } else if (result.docs[queryI] > notResult.docs[notI]) {
+      ++notI;
+    } else {
+      // the documents are equal, ignore both
+      ++queryI;
+      ++notI;
+    }
+  }
+
+  // erase unused document numbers and ranks
+  result.docs.erase(result.docs.begin()+outI, result.docs.end());
+  if (haveAccum)
+    result.ranks.erase(result.ranks.begin()+outI, result.ranks.end());
+  
+  // combine term frequency information
+  if (queryInfo.needTermFreqs)
+    result.termFreqs.insert (result.termFreqs.end(),
+			     notResult.termFreqs.begin(),
+			     notResult.termFreqs.end());
+}
+
+void NotQueryNode::Free () {
+  if (queryNode != NULL) {
+    delete queryNode;
+    queryNode = NULL;
+  }
+  if (notNode != NULL) {
+    delete notNode;
+    notNode = NULL;
+  }
+}
+
+void NotQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "queryNode:\n", indent);
+  PrintNode (s, queryNode, indent+2);
+  PrintIndentText (s, "NOT\n", indent);
+  PrintIndentText (s, "notNode:\n", indent);
+  PrintNode (s, notNode, indent+2);
+}
+
+
+void TagNode::Clear () {
+  UCArrayClear (tagName);
+}
+
+void TagNode::Calculate (IndexData &indexData,
+			 FragRangeArray &fragRange) const {
+  fragRange.erase (fragRange.begin(), fragRange.end());
+  if (tagName.empty()) return;
+
+  // get information about this tag
+  block_dict_el tagEl;
+  unsigned long tagElNum;
+  if (!SearchBlockDictEl (indexData.dictFile, indexData.biTags,
+			  indexData.bdh.entries_per_tblk,
+			  indexData.bdh.tag_dict_size,
+			  tagName, tagEl, tagElNum))
+    return;
+
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, tagEl.invf_ptr, SEEK_SET);
+
+  stdio_bitio_buffer buffer(indexData.invfFile);
+  
+  unsigned long pTag = tagEl.frag_occur*2;
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  FragRange thisFrag;
+  for (i=0; i<tagEl.frag_occur; ++i) {
+    // get start
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeStart = fragNum;
+
+    // get end
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+
+    thisFrag.rangeEnd = fragNum;
+    fragRange.push_back (thisFrag);
+  }
+
+  buffer.done();
+}
+
+void TagNode::Free () {
+  Clear ();
+}
+
+void TagNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TAG: \"" << tagName << "\"\n";
+}
+
+
+void TermNode::Clear () {
+  UCArrayClear (term);
+  termWeight = 1;
+  stemMethod = 0;
+  startRange = NO_TERM_RANGE_START;
+  endRange = NO_TERM_RANGE_END;
+}
+
+TermNode::TermNode () {
+  Clear ();
+}
+
+void TermNode::Calculate (IndexData &indexData,
+			  bool needFragFreqs,
+			  FragRangeArray *fragLimits,
+			  FragData &fragData,
+			  UCArrayVector &equivTerms) const {
+  fragData.Clear ();
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+
+  // get a list of term numbers
+  vector<unsigned long> equivNums;
+  FindWordNumbers (indexData, term, stemMethod, equivNums);
+
+  // get the information for each word and merge it with
+  // previous results
+  FragData tempFragData1;
+  FragData tempFragData2;
+  UCArray equivWord;
+  vector<unsigned long>::iterator here = equivNums.begin();
+  vector<unsigned long>::iterator end = equivNums.end();
+  while (here != end) {
+    // get the information for this word
+    ReadTermFragData (indexData, needFragFreqs, *here,
+		      tempFragData1, fragLimits, equivWord);
+    equivTerms.push_back(equivWord);
+    // combine with last results
+    tempFragData2 = fragData;
+    CombineFragData (needFragFreqs, tempFragData1, tempFragData2, fragData);
+    
+    ++here;
+  }
+}
+
+void TermNode::Free () {
+  Clear ();
+}
+
+void TermNode::Print (ostream &s, int indent) const {
+  PrintIndent (s, indent);
+  s << "TERM: \"" << term << "\"\n";
+  PrintIndent (s, indent+2);
+  s << "termWeight: " << termWeight << "\n";
+  PrintIndent (s, indent+2);
+  s << "stemMethod: " << stemMethod << "\n";
+  PrintIndent (s, indent+2);
+  s << "startRange: " << startRange << "\n";
+  PrintIndent (s, indent+2);
+  s << "endRange: " << endRange << "\n";
+}
+
+
+ProxMatchQueryNode::ProxMatchQueryNode () {
+  tagNodePtr = NULL;
+}
+
+ProxMatchQueryNode::~ProxMatchQueryNode () {
+  Free ();
+}
+
+void ProxMatchQueryNode::Calculate (IndexData &indexData,
+				    const QueryInfo &queryInfo,
+				    QueryResult &result) const {
+  result.Clear ();
+
+  bool needFragFreqs = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  // read in the tag if needed
+  FragRangeArray fragLimits;
+  FragRangeArray *fragLimitsPtr = NULL;
+  if (tagNodePtr == NULL && terms.size() > 1) {
+    // multiple terms must be compared relative to some tag
+    // otherwise phrase matches could span documents
+    TagNode tempTagNode;
+    tempTagNode.tagName = indexData.curLevel;
+    tempTagNode.Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+    
+  } else if (tagNodePtr != NULL) {
+    (*tagNodePtr).Calculate (indexData, fragLimits);
+    fragLimitsPtr = &fragLimits;
+  }
+
+  UCArray tagOrLevel = indexData.curLevel;
+  if (tagNodePtr != NULL) tagOrLevel = (*tagNodePtr).tagName;
+
+  // read in the first term
+  FragData termData;
+  UCArrayVector equivTerms;
+  TermNodeArray::const_iterator termHere=terms.begin(), termEnd = terms.end();
+  if (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs, fragLimitsPtr, 
+			   termData, equivTerms);
+
+    // convert initial fragment information
+    FragsToQueryResult (indexData,
+			queryInfo,
+			termData,
+			tagOrLevel,
+			(*termHere).term,
+			(*termHere).stemMethod,
+			(*termHere).termWeight,
+			equivTerms,
+			result);
+  
+    ++termHere;
+
+    if (termHere == termEnd) return; // nothing more to do
+  }
+
+  // read and combine the rest of the terms
+  FragData comTermData;
+  while (termHere != termEnd) {
+    (*termHere).Calculate (indexData, needFragFreqs,
+			   fragLimitsPtr, comTermData, equivTerms);
+
+    AndFragsToQueryResult (indexData,
+			   queryInfo,
+			   comTermData,
+			   tagOrLevel,
+			   (*termHere).term,
+			   (*termHere).stemMethod,
+			   (*termHere).termWeight,
+			   equivTerms,
+			   result);
+    
+    AndCombineFragData (needFragFreqs, termData, comTermData,
+			(*termHere).startRange,
+			(*termHere).endRange,
+			fragLimitsPtr);
+    ++termHere;
+  }
+
+  // remove unwanted document numbers
+  RemoveUnwantedResults (indexData,
+			 queryInfo,
+			 termData,
+			 result);
+}
+
+void ProxMatchQueryNode::Free () {
+  if (tagNodePtr != NULL) {
+    delete tagNodePtr;
+    tagNodePtr = NULL;
+  }
+  terms.erase (terms.begin(), terms.end());
+}
+
+void ProxMatchQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText (s, "PROXMATCH\n", indent);
+  if (tagNodePtr != NULL) tagNodePtr->Print (s, indent+2);
+  
+  TermNodeArray::const_iterator here = terms.begin();
+  TermNodeArray::const_iterator end = terms.end();
+  while (here != end) {
+    (*here).Print (s, indent+2);
+    ++here;
+  }
+}
+
+
+void BrowseQueryNode::Clear () {
+  UCArrayClear(term);
+}
+
+void BrowseQueryNode::Calculate (IndexData &indexData, BrowseQueryResult &result) const {
+
+  unsigned long number=0;
+  FindNearestWordNumber(indexData, term, number);
+  if (number + startPosition > 0 ) {
+    number = number+startPosition;
+  }
+  else {
+    number = 1;
+  }
+
+  GetTermList (indexData, number, numTerms, result.termFreqs);
+
+}
+
+
+
+void BrowseQueryNode::Free () {
+  Clear();
+}
+
+
+void BrowseQueryNode::Print (ostream &s, int indent) const {
+  PrintIndentText(s, "BROWSEQUERYNODE\n", indent);
+  PrintIndent (s, indent+2);
+  s << "TERM:"<<term<<"\n";
+  PrintIndent (s, indent+2);
+  s << "Start position: "<< startPosition<<", Num terms: "<< numTerms<<"\n";
+
+
+}
+
+
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result) {
+  result.Clear ();
+
+  // make sure level is current
+  if (!indexData.LoadLevel (queryInfo.docLevel)) {
+    return;
+  }
+  // do query
+  if (queryTree == NULL) return;
+  
+  (*queryTree).Calculate (indexData, queryInfo, result);
+
+  // make weights into ranks if needed
+  unsigned long i;
+  if (queryInfo.sortByRank || queryInfo.needRankInfo) {
+    for (i=0; i<result.ranks.size(); ++i) {
+      result.ranks[i] /=
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]);
+    }
+  }
+  
+  unsigned long resultsSize = queryInfo.maxDocs;
+  if (resultsSize == 0 || resultsSize > result.docs.size())
+    resultsSize = result.docs.size();
+
+  result.actualNumDocs = result.docs.size(); // the total number of docs 
+                                   //returned, before pruning based on maxDocs
+
+  // sort results by rank if needed
+  GTRank gtRank;
+  if (queryInfo.sortByRank) {
+    // need in ascending order for SelectAddHeap
+    MakeHeap (result.docs.begin(), result.ranks.begin(),
+	      resultsSize, gtRank);
+    SelectAddHeap (result.docs.begin(), result.ranks.begin(), resultsSize,
+		   result.docs.begin()+resultsSize,
+		   result.ranks.begin()+resultsSize,
+		   (unsigned long) result.ranks.size()-resultsSize,
+		   gtRank);
+
+    // sort into descending order
+    SortHeap (result.docs.begin(), result.ranks.begin(), resultsSize, gtRank);
+  }
+  
+  // get exact weights if needed
+  if (queryInfo.exactWeights &&
+      (queryInfo.sortByRank || queryInfo.needRankInfo)) {
+    unsigned long exactDiskPtr =
+      indexData.levels.levelInfo[indexData.curLevel].exactWeightsDiskPtr;
+    
+    for (i=0; i<resultsSize; ++i) {
+      result.ranks[i] =  result.ranks[i] *
+	indexData.weightData.GetLowerApproxDocWeight (result.docs[i]) /
+	GetExactDocWeight (indexData.exactWeightsFile, exactDiskPtr,
+			   result.docs[i]);
+    }
+
+    // re-sort top candidates based on exact weights
+    if (queryInfo.sortByRank) {
+      MakeHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+      SortHeap (result.docs.begin(), result.ranks.begin(),
+		resultsSize, gtRank);
+    }
+  }
+
+  // get rid of unwanted ranking results
+  if (result.ranks.empty()) {
+    // do nothing
+  } else if (!queryInfo.needRankInfo) {
+    result.ranks.erase(result.ranks.begin(), result.ranks.end());
+  } else {
+    result.ranks.erase(result.ranks.begin()+resultsSize, result.ranks.end());
+  }
+
+  // remove extra docs that are unwanted
+  result.docs.erase(result.docs.begin()+resultsSize, result.docs.end());
+}
+
+// new MGQuery to retrieve doc and section nums
+// this will return doc nums for the level queried at (set in queryInfo)
+// in QueryResult.docs and if a second level is specified, 
+// it will return corresponding docnums for that level in QueryResult.levels
+// If there is no level specified, or that level is invalid, the query
+// level is used
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &realresult, UCArray &level) {
+  realresult.Clear ();
+  QueryResult result; // temp result
+
+  // do the normal query
+  MGQuery (indexData, queryInfo, queryTree, result);
+
+  // now that have the final result stuff, convert to ExtQueryResult,
+  // add in level nums if needed
+
+  realresult.docs = result.docs;
+  realresult.ranks = result.ranks;
+  realresult.termFreqs = result.termFreqs;
+  realresult.actualNumDocs = result.actualNumDocs;
+
+  if (queryInfo.docLevel == level || level.empty()) {
+    realresult.levels = result.docs;
+    return;
+  }
+
+  // else need to convert from queryInfo.docLevel to level
+
+  // the original level info
+  FragLevelConvert sectionLevelConverter = indexData.levelConverter;
+
+  // the new level info
+  if (!indexData.LoadLevel(level)) {
+    realresult.levels = result.docs;
+    return;
+  }
+  
+  unsigned long DocNum = 0;
+  
+  for (unsigned long i=0; i<realresult.docs.size(); ++i) {
+
+    // do an if ! here????
+    indexData.levelConverter.LevelToLevel(sectionLevelConverter, realresult.docs[i], DocNum);
+    realresult.levels.push_back(DocNum);
+  }
+  
+}
+
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result) {
+
+  indexData.LoadLevel(level);
+  node.Calculate(indexData, result); 
+
+}
Index: /main/tags/3.01/mgpp/text/MGQuery.h
===================================================================
--- /main/tags/3.01/mgpp/text/MGQuery.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/MGQuery.h	(revision 21112)
@@ -0,0 +1,197 @@
+/**************************************************************************
+ *
+ * MGQuery.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MGQUERY_H
+#define MGQUERY_H
+
+#include "IndexData.h"
+#include <limits.h>
+#include "Terms.h"
+
+// forms base for defining and calculating the boolean
+// part of a query
+class QueryNode {
+public:
+  QueryNode ();
+  virtual ~QueryNode ();
+
+  virtual void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+			  QueryResult &result) const;
+  virtual void Free ();
+  virtual void Print (ostream &s, int indent=0) const;
+};
+
+
+
+void PrintIndent (ostream &s, int indent);
+void PrintIndentText (ostream &s, char *text, int indent);
+void PrintNode (ostream &s, QueryNode *node, int indent=0);
+
+
+
+// basic boolean nodes
+
+class AndQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  AndQueryNode ();
+  ~AndQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class OrQueryNode : public QueryNode {
+public:
+  QueryNode *leftNode;
+  QueryNode *rightNode;
+  
+  OrQueryNode ();
+  ~OrQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class NotQueryNode : public QueryNode {
+public:
+  QueryNode *queryNode;
+  QueryNode *notNode;
+  
+  NotQueryNode ();
+  ~NotQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+
+class TagNode {
+public:
+  UCArray tagName;
+
+  void Clear ();
+  TagNode () { Clear (); }
+
+  void Calculate (IndexData &indexData, FragRangeArray &fragRange) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+
+#define NO_TERM_RANGE_START (LONG_MIN/2)
+#define NO_TERM_RANGE_END   (LONG_MAX/2)
+
+/* NOTE: range stuff - the range values are for the previous term relative
+to the current term. So if searching for the phrase "the cat", 'the' doesn't 
+need range limits, but 'cat' has a range of -2 to -1. ie, if have found 'cat'
+then 'the' has to be at position between -2 and -1 relative to 'cat'.
+"the cat" could also be searched for by 'cat' with no range limits, then 'the' with range 0 to 1.
+range values are relative to the gaps between words:
+   x   y   z   X   a   b   c
+    -3   -2  -1  0   1   2   3
+
+ */
+class TermNode {
+public:
+  UCArray term;
+  unsigned long termWeight;
+  unsigned long stemMethod;
+  signed long startRange; // range relative to last term
+  signed long endRange;
+  
+  void Clear ();
+  TermNode ();
+
+  // fragLimits can be NULL if not neede
+  void Calculate (IndexData &indexData,
+		  bool needFragFreqs,
+		  FragRangeArray *fragLimits,
+		  FragData &fragData,
+		  UCArrayVector &equivTerms) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+typedef vector<TermNode> TermNodeArray;
+
+
+class ProxMatchQueryNode : public QueryNode {
+public:
+  TagNode *tagNodePtr; // optional field
+  TermNodeArray terms;
+  
+  ProxMatchQueryNode ();
+  ~ProxMatchQueryNode ();
+
+  void Calculate (IndexData &indexData, const QueryInfo &queryInfo,
+		  QueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+};
+
+class BrowseQueryNode :public QueryNode {
+ public:
+  UCArray term;
+  signed long startPosition;
+  unsigned long numTerms;
+
+  void Clear();
+  BrowseQueryNode () { Clear(); }
+  //  ~BrowseQueryNode ();
+
+  void Calculate (IndexData &indexData, BrowseQueryResult &result) const;
+  void Free ();
+  void Print (ostream &s, int indent=0) const;
+
+
+
+
+};
+
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      QueryResult &result);
+
+// this function for retrieving results with both section doc nums
+// and Document docnums
+void MGQuery (IndexData &indexData,
+	      const QueryInfo &queryInfo,
+	      const QueryNode *queryTree,
+	      ExtQueryResult &result, UCArray &level);
+
+
+// new function for full text browsing, 
+void MGBrowseQuery (IndexData &indexData, UCArray &level,
+		  const BrowseQueryNode &node,
+		  BrowseQueryResult &result);
+
+#endif
Index: /main/tags/3.01/mgpp/text/Makefile.in
===================================================================
--- /main/tags/3.01/mgpp/text/Makefile.in	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Makefile.in	(revision 21112)
@@ -0,0 +1,318 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MGPP system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+bindir = @bindir@
+srcdir = @srcdir@
+libdir = @libdir@
+subdir = text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CXX = @CXX@
+CFLAGS = @CFLAGS@ -ansi -DSILENT -DSHORT_SUFFIX
+CXXFLAGS = @CXXFLAGS@ -DSILENT -DSHORT_SUFFIX
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../lib/libmgpplib.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+GSDLOS = @gsdlos@
+
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CFLAGS)
+CXXCOMPILE = $(CXX) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CXXFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CXXCOMPILE) $<
+
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+INCLUDES = -I.. -I../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp \
+	mgpp_passes_4jni.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h  mgpp_passes_4jni.h
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgpp_passes mgpp_decompress_text mgpp_perf_hash_build \
+                mgpp_invf_dump mgpp_weights_build mgpp_invf_dict mgpp_stem_idx \
+                mgpp_invf_dict_dump mgpp_compression_dict mgpp_fast_comp_dict \
+                QueryTester Queryer
+
+MAN           =  mgpp_weights_build.1  mgpp_invf_dict.1 \
+                 mgpp_passes.1 mgpp_perf_hash_build.1 \
+                 mgpp_compression_dict.1  mgpp_fast_comp_dict.1 \
+                 mgpp_stem_idx.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libmgpptext.a libmgpppass.a
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CXXFLAGS) $(CFLAGS) -- $(SOURCES)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall
+uninstall: execuninstall
+
+execinstall: $(EXEC)
+	../mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done 
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libmgpptext.a: $(LIB_OBJS)
+	rm -f libmgpptext.a
+	$(AR) cru libmgpptext.a $(LIB_OBJS)
+	$(RANLIB) libmgpptext.a
+
+JNI_LIB_OBJS = \
+	mgpp_passes_4jni$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+libmgpppass.a: $(JNI_LIB_OBJS)
+	rm -f libmgpppass.a
+	$(AR) cru libmgpppass.a $(JNI_LIB_OBJS)
+	$(RANLIB) libmgpppass.a
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+
+clean: mostlyclean
+	rm -f $(EXEC) libmgpptext.a libmgpppass.a
+
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+movclean:
+	for file in $(SOURCES); do \
+          test = $$file; \
+          echo $("test.c"##*.c); \
+        done 
+
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
Index: /main/tags/3.01/mgpp/text/QueryLex.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/QueryLex.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryLex.cpp	(revision 21112)
@@ -0,0 +1,175 @@
+/**************************************************************************
+ *
+ * QueryLex.cpp -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryLex.h"
+#include "unitool.h"
+#include "words.h"
+
+inline void AddNChar (UCArray::const_iterator &here,
+		      UCArray &text,
+		      int len) {
+  if (text.capacity() < text.size() + len + 1) {
+    text.reserve(text.size + len + 1);
+  }
+  while (len > 0) {
+    text.push_back (*here++);
+    --len;
+  }
+}
+
+static bool ParseInteger (UCArray::const_iterator &here,
+			  UCArray::const_iterator end,
+			  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+
+  // check for positive or negative
+  bool neg = false;
+  if (c == '+') {
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  } else if (c == '-') {
+    neg = true;
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  // read in number part
+  el.num = 0;
+  el.lexType = IntegerE;
+  while (c >= '0' && c <= '9') {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  if (neg) el.num *= -1;
+
+  return (!el.text.empty());
+}
+
+static bool ParseTerm (UCArray::const_iterator &here,
+		       UCArray::const_iterator end,
+		       UCArray &text) {
+  UCArray::const_iterator endMinus1 = end-1;
+  here = ParseIndexWord (here, endMinus1, text);
+  return !text.empty();
+}
+
+
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el) {
+  el.Clear();
+
+  // strange things can happen if here == end == 0
+  if (here == end) return false;
+  
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+  
+  // ignore all white space
+  int charLen;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+  while (here != end && is_unicode_space (c)) {
+    here += charLen;
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+  if (here == end) return false;
+
+  if (c == '(') {
+    el.lexType = OpenBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ')') {
+    el.lexType = CloseBracketE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '\"') {
+    el.lexType = QuoteE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '#') {
+    el.lexType = TermWeightE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '$') {
+    el.lexType = StemMethodE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '^') {
+    el.lexType = RangeE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '@') {
+    el.lexType = AtE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == ':') {
+    el.lexType = TagE;
+    AddNChar (here, el.text, charLen);
+    return true;
+    
+  } else if (c == '+' || c == '-' ||
+	     (c >= '0' && c <= '9')) {
+    return ParseInteger (here, end, el);
+  }
+
+  // assume it is a term of some sort
+  if (!ParseTerm (here, end, el.text)) return false;
+
+  //UCArray AND; SetCStr (AND, "AND");
+  //if (el.text == AND) {
+  if (UCArrayCStrEquals(el.text, "AND")) {  
+    el.lexType = AndOpE;
+    return true;
+  }
+  //UCArray OR; SetCStr (OR, "OR");
+  //if (el.text == OR) {
+  if (UCArrayCStrEquals(el.text, "OR")) {
+    el.lexType = OrOpE;
+    return true;
+  }
+  //UCArray NOT; SetCStr (NOT, "NOT");
+  //if (el.text == NOT) {
+  if (UCArrayCStrEquals(el.text, "NOT")) {
+    el.lexType = NotOpE;
+    return true;
+  }
+
+  el.lexType = TermE;
+  return true;
+}
+
Index: /main/tags/3.01/mgpp/text/QueryLex.h
===================================================================
--- /main/tags/3.01/mgpp/text/QueryLex.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryLex.h	(revision 21112)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * QueryLex.h -- Lexical analyser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYLEX_H
+#define QUERYLEX_H
+
+#include "UCArray.h"
+
+
+// note: white space is ignored
+enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
+	      NotOpE, QuoteE, IntegerE, TermWeightE, StemMethodE,
+	      RangeE, AtE, TagE};
+
+struct LexEl {
+  LexType lexType;
+  UCArray text;
+  int num; // for IntegerE
+
+  void Clear () {
+    lexType = NoTypeE;
+    UCArrayClear (text);
+    num = 0;
+  }
+  LexEl () { Clear (); }
+};
+
+
+// returns true if another lexical element could be parsed.
+bool ParseLexEl (UCArray::const_iterator &here,
+		 UCArray::const_iterator end,
+		 LexEl &el);
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/QueryParser.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/QueryParser.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryParser.cpp	(revision 21112)
@@ -0,0 +1,219 @@
+/**************************************************************************
+ *
+ * QueryParser.cpp -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "QueryParser.h"
+#include "QueryLex.h"
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end);
+
+static QueryNode *AndAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  AndQueryNode *andNode = new AndQueryNode;
+  andNode->leftNode = t1;
+  andNode->rightNode = t2;
+  return andNode;
+}
+
+static QueryNode *OrAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  OrQueryNode *orNode = new OrQueryNode;
+  orNode->leftNode = t1;
+  orNode->rightNode = t2;
+  return orNode;
+}
+
+static QueryNode *NotAdd (QueryNode *t1, QueryNode *t2) {
+  if (t1 == NULL) return t2;
+  if (t2 == NULL) return t1;
+
+  NotQueryNode *notNode = new NotQueryNode;
+  notNode->queryNode = t1;
+  notNode->notNode = t2;
+  return notNode;
+}
+
+// expects the opening bracket to have already been parsed
+// and discarded
+static QueryNode *ParseBracketExpression (UCArray::const_iterator &here,
+					  UCArray::const_iterator end) {
+  // get everything in the expression
+  QueryNode *curTree = ParseExpression (here, end);
+  
+  // gobble up tokens until a closing bracket is found
+  // or the end of the string
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == CloseBracketE) break;
+  }
+
+  return curTree;
+}
+
+static int ParseInt (UCArray::const_iterator &here,
+		     UCArray::const_iterator end) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  if (ParseLexEl (here, end, el) && el.lexType == IntegerE)
+    return el.num;
+  
+  here = oldHere; // not an integer
+  return 0;
+}
+
+
+static void ParseTermModifiers (UCArray::const_iterator &here,
+				UCArray::const_iterator end,
+				TermNode &termNode) {
+  LexEl el;
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermWeightE) {
+      termNode.termWeight = ParseInt (here, end);
+      
+    } else if (el.lexType == StemMethodE) {
+      termNode.stemMethod = ParseInt (here, end);
+      
+    } else if (el.lexType == RangeE) {
+      termNode.startRange = ParseInt (here, end);
+      termNode.endRange = ParseInt (here, end);
+      
+    } else if (el.lexType == AtE) {
+      termNode.startRange = termNode.endRange = ParseInt (here, end);
+      
+    } else {
+      // no term modifiers
+      here = oldHere;
+      break;
+    }
+    
+    oldHere = here;
+  }  
+}
+
+// expects the starting quote to have been parsed
+// and discarded
+static void ParsePhrase (UCArray::const_iterator &here,
+			 UCArray::const_iterator end,
+			 ProxMatchQueryNode &proxNode) {
+  LexEl el;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == TermE || el.lexType == IntegerE) {
+      TermNode termNode;
+      termNode.term = el.text;
+      ParseTermModifiers (here, end, termNode);
+      proxNode.terms.push_back (termNode);
+      
+    } else if (el.lexType == QuoteE) {
+      break;
+      
+    } else {
+      // error
+      break;
+    }
+  }
+}
+
+static QueryNode *ParseTerm (UCArray::const_iterator &here,
+			     UCArray::const_iterator end) {
+  LexEl el;
+
+  UCArray::const_iterator oldHere = here;
+  if (!ParseLexEl (here, end, el)) return NULL;
+
+  if (el.lexType == OpenBracketE)
+    return ParseBracketExpression (here, end);
+
+  ProxMatchQueryNode *proxNode = new ProxMatchQueryNode;
+
+  // check for a tag
+  if (el.lexType == TagE) {
+    oldHere = here; // don't backtrack past here
+    if (!ParseLexEl (here, end, el)) return NULL;
+    if (el.lexType == TermE) {
+      proxNode->tagNodePtr = new TagNode;
+      proxNode->tagNodePtr->tagName = el.text;
+      if (!ParseLexEl (here, end, el)) return NULL;
+    }
+  }
+  
+  if (el.lexType == TermE || el.lexType == IntegerE) {
+    TermNode termNode;
+    termNode.term = el.text;
+    ParseTermModifiers (here, end, termNode);
+    proxNode->terms.push_back (termNode);
+    return proxNode;
+    
+  } else if (el.lexType == QuoteE) {
+    ParsePhrase (here, end, *proxNode);
+    return proxNode;
+  }
+
+  // not a term
+  here = oldHere;
+  delete proxNode;
+  return NULL;
+}
+
+
+static QueryNode *ParseExpression (UCArray::const_iterator &here,
+				   UCArray::const_iterator end) {
+  LexEl el;
+  QueryNode *curTree = NULL;
+
+  UCArray::const_iterator oldHere = here;
+  while (ParseLexEl (here, end, el)) {
+    if (el.lexType == OpenBracketE ||
+	el.lexType == TermE ||
+	el.lexType == QuoteE ||
+	el.lexType == IntegerE ||
+	el.lexType == TagE) {
+      // some type of term, back track and parse it
+      here = oldHere;
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == AndOpE) {
+      curTree = AndAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == OrOpE) {
+      curTree = OrAdd (curTree, ParseTerm (here, end));
+      
+    } else if (el.lexType == NotOpE) {
+      curTree = NotAdd (curTree, ParseTerm (here, end));
+      
+    } else break;
+    
+    oldHere = here;
+  }
+
+  return curTree;
+}
+
+QueryNode *ParseQuery (const UCArray &queryStr) {
+  UCArray::const_iterator here = queryStr.begin();
+  UCArray::const_iterator end = queryStr.end();
+  return ParseExpression (here, end);
+}
Index: /main/tags/3.01/mgpp/text/QueryParser.h
===================================================================
--- /main/tags/3.01/mgpp/text/QueryParser.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryParser.h	(revision 21112)
@@ -0,0 +1,31 @@
+/**************************************************************************
+ *
+ * QueryParser.h -- Query parser for a simple query language
+ * Copyright (C) 2000  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYPARSER_H
+#define QUERYPARSER_H
+
+#include "UCArray.h"
+#include "MGQuery.h"
+
+// returns NULL if the query could not be parsed
+QueryNode *ParseQuery (const UCArray &queryStr);
+
+#endif
Index: /main/tags/3.01/mgpp/text/QueryResultsSort.h
===================================================================
--- /main/tags/3.01/mgpp/text/QueryResultsSort.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryResultsSort.h	(revision 21112)
@@ -0,0 +1,140 @@
+/**************************************************************************
+ *
+ * QueryResultsSort.h -- Functions to sort query results and parallel data
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef QUERYRESULTSSORT_H
+#define QUERYRESULTSSORT_H
+
+#include "Terms.h"
+
+
+struct LTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 < f2); }
+};
+
+struct GTRank {
+  bool operator()(const float &f1, const float &f2) const
+    { return (f1 > f2); }
+};
+
+
+
+// assumes the rank vector is the same size as the document
+// number vector. assumes the value to be pushed is in
+// docNumHeap + heapSize - 1 and rankHeap + heapSize - 1
+// topIndex can be set
+template <class RandomAccessIterator, class Compare>
+void PushHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long heapSize,
+	       const Compare &comp) {
+  unsigned long holeIndex = heapSize-1;
+  unsigned long parent = (holeIndex - 1) / 2;
+  while (holeIndex > 0 && comp (*(parallelHeap+parent),
+				*(parallelHeap+holeIndex))) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+parent));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+parent));
+    holeIndex = parent;
+    parent = (holeIndex - 1) / 2;
+  }
+}
+
+// AdjustHeap is used to add a element to a heap after one
+// element in the heap has been replaced with a new one
+template <class RandomAccessIterator, class Compare>
+void AdjustHeap (DocNumArray::iterator docNumHeap,
+		     RandomAccessIterator parallelHeap,
+		     unsigned long holeIndex,
+		     unsigned long heapSize,
+		     const Compare &comp) {
+  unsigned long secondChild = 2 * holeIndex + 2;
+  while (secondChild < heapSize) {
+    if (comp(*(parallelHeap + secondChild),
+	     *(parallelHeap + (secondChild - 1))))
+      --secondChild;
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+secondChild));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+secondChild));
+    holeIndex = secondChild;
+    secondChild = 2 * secondChild + 2;
+  }
+  if (secondChild == heapSize) {
+    swap (*(parallelHeap+holeIndex), *(parallelHeap+(secondChild-1)));
+    swap (*(docNumHeap+holeIndex), *(docNumHeap+(secondChild-1)));
+    holeIndex = secondChild - 1;
+  }
+  PushHeap (docNumHeap, parallelHeap, holeIndex+1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void PopHeap (DocNumArray::iterator docNumHeap,
+	      RandomAccessIterator parallelHeap,
+	      unsigned long heapSize,
+	      const Compare &comp) {
+  swap (*(parallelHeap), *(parallelHeap+heapSize-1));
+  swap (*(docNumHeap), *(docNumHeap+heapSize-1));
+  AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize-1, comp);
+}
+
+template <class RandomAccessIterator, class Compare>
+void MakeHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+  
+  unsigned long i = 0;
+  for (i=1; i<=size; ++i)
+    PushHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+ 
+template <class RandomAccessIterator, class Compare>
+void SortHeap (DocNumArray::iterator docNumHeap,
+	       RandomAccessIterator parallelHeap,
+	       unsigned long size,
+	       const Compare &comp) {
+  if (size < 2) return;
+
+  unsigned long i;
+  for (i=size; i>1; --i)
+    PopHeap (docNumHeap, parallelHeap, i, comp);
+}
+
+// uses the heap to get the top "size" elements
+template <class RandomAccessIterator, class Compare>
+void SelectAddHeap (DocNumArray::iterator docNumHeap,
+		    RandomAccessIterator parallelHeap,
+		    unsigned long heapSize,
+		    DocNumArray::iterator docNumAdd,
+		    RandomAccessIterator parallelAdd,
+		    unsigned long addSize,
+		    const Compare &comp) {
+  unsigned long i;
+  for (i=0; i<addSize; ++i) {
+    if (comp(*(parallelAdd+i), *(parallelHeap))) {
+      swap (*(docNumHeap), *(docNumAdd+i));
+      swap (*(parallelHeap), *(parallelAdd+i));
+      AdjustHeap (docNumHeap, parallelHeap, (unsigned long)0, heapSize, comp);
+    }
+  }
+}
+
+#endif
Index: /main/tags/3.01/mgpp/text/QueryTester.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/QueryTester.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/QueryTester.cpp	(revision 21112)
@@ -0,0 +1,447 @@
+/**************************************************************************
+ *
+ * QueryTester.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "MGQuery.h"
+
+
+class SetQueryNode : public QueryNode {
+public:
+  QueryResult queryResult;
+  
+  SetQueryNode () {}
+  ~SetQueryNode () {}
+
+  void Calculate (IndexData &/*indexData*/, const QueryInfo &queryInfo,
+		  QueryResult &result) const {
+    result = queryResult;
+    if (!queryInfo.sortByRank && !queryInfo.needRankInfo)
+      result.ranks.erase (result.ranks.begin(), result.ranks.end());
+    if (!queryInfo.needTermFreqs)
+      result.termFreqs.erase (result.termFreqs.begin(),
+			      result.termFreqs.end());
+  }
+  void Free () { queryResult.Clear(); }
+};
+
+
+bool Test1 () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (15);  rcRankSet.push_back (0.2+0.6);
+  rcDocSet.push_back (19);  rcRankSet.push_back (1.4);
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test1b () {
+  AndQueryNode andNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "AND\n\n"
+       << setNode2->queryResult;
+  
+
+  andNode.leftNode = setNode1;
+  andNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  andNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (15); 
+  rcDocSet.push_back (19); 
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2 () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (2);  rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (11); rcRankSet.push_back (0.2f);
+  rcDocSet.push_back (12); rcRankSet.push_back (0.3f);
+  rcDocSet.push_back (13); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (14); rcRankSet.push_back (0.5f);
+  rcDocSet.push_back (15); rcRankSet.push_back (0.2f+0.6f);
+  rcDocSet.push_back (16); rcRankSet.push_back (0.7f);
+  rcDocSet.push_back (17); rcRankSet.push_back (0.8f);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4f);
+  rcDocSet.push_back (19); rcRankSet.push_back (0.9f+0.5f);
+  rcDocSet.push_back (20); rcRankSet.push_back (0.1f);
+  rcDocSet.push_back (21); rcRankSet.push_back (0.2f);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test2b () {
+  OrQueryNode orNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2);
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "OR\n\n"
+       << setNode2->queryResult;
+  
+
+  orNode.leftNode = setNode1;
+  orNode.rightNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  orNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1);
+  rcDocSet.push_back (2); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (11);
+  rcDocSet.push_back (12);
+  rcDocSet.push_back (13);
+  rcDocSet.push_back (14);
+  rcDocSet.push_back (15);
+  rcDocSet.push_back (16);
+  rcDocSet.push_back (17);
+  rcDocSet.push_back (18);
+  rcDocSet.push_back (19);
+  rcDocSet.push_back (20);
+  rcDocSet.push_back (21);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3 () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  RankArray &rankSet1 = setNode1->queryResult.ranks;
+  docSet1.push_back (1);  rankSet1.push_back (0.1f);
+  docSet1.push_back (10); rankSet1.push_back (0.2f);
+  docSet1.push_back (15); rankSet1.push_back (0.2f);
+  docSet1.push_back (18); rankSet1.push_back (0.4f);
+  docSet1.push_back (19); rankSet1.push_back (0.5f);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  RankArray &rankSet2 = setNode2->queryResult.ranks;
+  docSet2.push_back (2);  rankSet2.push_back (0.1f);
+  docSet2.push_back (11); rankSet2.push_back (0.2f);
+  docSet2.push_back (12); rankSet2.push_back (0.3f);
+  docSet2.push_back (13); rankSet2.push_back (0.4f);
+  docSet2.push_back (14); rankSet2.push_back (0.5f);
+  docSet2.push_back (15); rankSet2.push_back (0.6f);
+  docSet2.push_back (16); rankSet2.push_back (0.7f);
+  docSet2.push_back (17); rankSet2.push_back (0.8f);
+  docSet2.push_back (19); rankSet2.push_back (0.9f);
+  docSet2.push_back (20); rankSet2.push_back (0.1f);
+  docSet2.push_back (21); rankSet2.push_back (0.2f);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = true;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  RankArray &rcRankSet = resultCompare.ranks;
+  rcDocSet.push_back (1);  rcRankSet.push_back (0.1);
+  rcDocSet.push_back (10); rcRankSet.push_back (0.2);
+  rcDocSet.push_back (18); rcRankSet.push_back (0.4);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+bool Test3b () {
+  NotQueryNode notNode;
+
+  SetQueryNode *setNode1 = new SetQueryNode;
+  DocNumArray &docSet1 = setNode1->queryResult.docs;
+  docSet1.push_back (1); 
+  docSet1.push_back (10);
+  docSet1.push_back (15);
+  docSet1.push_back (18);
+  docSet1.push_back (19);
+
+  SetQueryNode *setNode2 = new SetQueryNode;
+  DocNumArray &docSet2 = setNode2->queryResult.docs;
+  docSet2.push_back (2); 
+  docSet2.push_back (11);
+  docSet2.push_back (12);
+  docSet2.push_back (13);
+  docSet2.push_back (14);
+  docSet2.push_back (15);
+  docSet2.push_back (16);
+  docSet2.push_back (17);
+  docSet2.push_back (19);
+  docSet2.push_back (20);
+  docSet2.push_back (21);
+
+  cout << "\n" << setNode1->queryResult << "NOT\n\n"
+       << setNode2->queryResult;
+  
+
+  notNode.queryNode = setNode1;
+  notNode.notNode = setNode2;
+
+  IndexData indexData; // this will not be used anywhere
+  QueryInfo queryInfo;
+  queryInfo.maxDocs = 0;
+  queryInfo.sortByRank = false;
+  queryInfo.needRankInfo = false;
+  queryInfo.needTermFreqs = true;
+  
+  QueryResult result;
+
+  notNode.Calculate(indexData, queryInfo, result);
+
+  cout << "EQUALS\n\n" << result << "\n";
+
+  QueryResult resultCompare;
+  DocNumArray &rcDocSet = resultCompare.docs;
+  rcDocSet.push_back (1); 
+  rcDocSet.push_back (10);
+  rcDocSet.push_back (18);
+  
+
+  if (!(result == resultCompare)) {
+    cout << "Test failed!!!\n";
+    cout << resultCompare;
+    return false;
+  }
+
+  return true;
+}
+
+
+int main () {
+  if (!Test1 () ||
+      !Test2 () ||
+      !Test2b () ||
+      !Test3 () ||
+      !Test3b ()) return 1;
+  
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/Queryer.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/Queryer.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Queryer.cpp	(revision 21112)
@@ -0,0 +1,271 @@
+/**************************************************************************
+ *
+ * Queryer.cpp -- simple interactive query program
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "MGQuery.h"
+#include "TextGet.h"
+
+#include "messages.h"
+#include "mg_files.h"
+
+#include "GSDLQueryParser.h"
+
+void printHelp() {
+
+  cout << "commands available are:\n"
+       << "\t.q\t\tquit\n"
+       << "\t.h\t\tprint the help message\n"
+       << "\t.i\t\tchange the search level (enter the new level at the prompt)\n"
+       << "\t.l\t\tchange the result level ( \"\"   \"\"  )\n"
+       << "\t.b\t\tfull text browse (enter a word or fragment at the prompt)\n"
+       << "\t.r0/.r1\t\tranking off/on\n"
+       << "\t.t0/.t1\t\tquery type some/all\n"
+       << "\t.c0/.c1\t\tcasefolding off/on\n"
+       << "\t.s0/.s1\t\tstemming off/on\n"
+       << "\t.o0/.o1\t\tshort output off/on\n\n"
+       << "\t.p\t\tprint a document (enter the docnum at the prompt)\n"
+       << "otherwise just enter a query\n\n";
+
+}
+
+int main (int argc, char **argv) {
+  int ch;
+  char *textfilename = "";
+  char *indexfilename = "";
+  char *basePath = "";
+  
+  opterr = 0;
+  msg_prefix = argv[0];
+  
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:t:d:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      indexfilename = optarg;
+      break;
+    case 't':
+      textfilename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (basePath);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-d directory] -f indexname -t textname\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  if (textfilename[0] == '\0' || indexfilename[0] == '\0') {
+    FatalError (1, "Index and text file names must be specified with -f and -t \n");
+  }
+  
+  // init the text system
+    TextData textData;
+  if (!textData.LoadData (basePath, textfilename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", textfilename);
+  }
+  
+  // init the query system
+  IndexData indexData;
+  if (!indexData.LoadData (basePath, indexfilename)) {
+    FatalError (1, "Couldn't load index information for \"%s\"", indexfilename);
+  }
+
+  // debug output
+  cerr << "num docs: "<<indexData.bdh.num_docs       
+       << "\nnum frags: "<<indexData.bdh.num_frags
+       << "\nnum words: "<<indexData.bdh.num_words
+       << "\ntotal bytes: "<<indexData.bdh.total_bytes
+       << "\nindex string bytes: "<<indexData.bdh.index_string_bytes
+       << "\nnum levels: "<<indexData.bdh.num_levels<<endl;
+  
+  // do querying
+  QueryInfo queryInfo;
+  SetCStr (queryInfo.docLevel, "Document", 8);
+  queryInfo.maxDocs = 50;
+  queryInfo.sortByRank = true;
+  queryInfo.exactWeights = false;
+  queryInfo.needRankInfo = true;
+  queryInfo.needTermFreqs = true;
+  
+  ExtQueryResult queryResult;
+  char query[2048];
+  UCArray queryArray;
+  QueryNode *queryTree = NULL;
+
+
+  UCArray docLevel;
+  SetCStr(docLevel, "Document", 8);
+
+  UCArray level;
+  UCArrayClear(level);
+  //SetCStr(level, "");
+   
+  int defaultStemMethod = 0; // uncasefolded, unstemmed
+  int defaultBoolCombine = 0; // OR
+  bool shortOutput = false;
+  BrowseQueryNode browseNode;
+  browseNode.startPosition = -10;
+  browseNode.numTerms = 40;
+
+  BrowseQueryResult browseResult;
+
+  while (true) {
+    cout << "> ";
+    cin.getline(query, 2048, '\n');
+    SetCStr (queryArray, query, strlen(query));
+
+    // check for commands
+    if (queryArray.size() >= 2 && queryArray[0] == '.') {
+      if (queryArray[1] == 'q') break;  // quit
+
+      if (queryArray[1] == 'h') { // help
+	printHelp();
+      } else if (queryArray[1] == 'i') {
+	cout << "current index="<< queryInfo.docLevel << "\nchange to index:";
+	cin >> query;
+	UCArrayClear(queryInfo.docLevel);
+	SetCStr(queryInfo.docLevel, query, strlen(query));
+	cout << "index set to " << queryInfo.docLevel <<"\n";
+	cin.getline(query, 2048, '\n');
+      } else if (queryArray[1] == 'l') {
+	cout << "current level="<< level << "\nchange to level:";
+	cin >> query;
+	UCArrayClear(level);
+	SetCStr(level, query, strlen(query));
+	cout << "level set to " << level <<"\n";
+	cin.getline(query, 2048, '\n');
+      }
+
+
+      else if (queryArray[1] == 'p') {
+	// print
+	UCArray docText;
+	unsigned long docNum = 0;
+	cin >> docNum;
+	cin.getline(query, 2048, '\n'); // eat up return
+	
+	if (!GetDocText (textData, queryInfo.docLevel, docNum, docText)) {
+	  FatalError (1, "Error while trying to get document %u", docNum);
+	}
+	
+	cout << docText << "\n";
+      }
+      else if (queryArray[1] == 't') { // query type - all/some
+	if (queryArray[2] == '1') defaultBoolCombine = 1;
+	else if (queryArray[2] == '0') defaultBoolCombine = 0;
+	else {
+	  cout << "Error: please enter .t0 (some) or .t1 (all)\n";
+	}
+      }
+      else if (queryArray[1] == 'r') { // ranking - on/off
+	if (queryArray[2] == '1') queryInfo.sortByRank = true;
+	else if (queryArray[2] == '0') queryInfo.sortByRank = false;
+	else {
+	  cout << "Error: please enter .r0 (non-ranked) or .r1 (ranked)\n";
+	}
+      }
+      else if (queryArray[1] == 'c') { // casefolding - on/off
+	if (queryArray[2] == '1') defaultStemMethod |= 1;
+	else if (queryArray[2] == '0') defaultStemMethod &= 0xe;
+	else {
+	  cout << "Error: please enter .c0 (case sensitive) or .c1 (casefolded)\n";
+	}
+      }
+      else if (queryArray[1] == 's') { // stemming - on/off
+	if (queryArray[2] == '1') defaultStemMethod |=2;
+	else if (queryArray[2] == '0') defaultStemMethod &=0xd;
+	else {
+	  cout << "Error: please enter .s0 (unstemmed) or .s1 (stemmed)\n";
+	}
+      }      
+      else if (queryArray[1] == 'o') { // output - short/long
+	if (queryArray[2] == '1') shortOutput = true;
+	else if (queryArray[2] == '0') shortOutput = false;
+	else {
+	  cout << "Error: please enter .o0 (long output) or .o1 (short output)\n";
+	}
+      }
+      else if (queryArray[1] == 'b') {
+	// full text browse
+	cout<<"enter a few letters to start browsing from:";
+	cin>>query;
+	UCArrayClear(browseNode.term);
+	SetCStr(browseNode.term, query, strlen(query));
+	cin.getline(query, 2048, '\n'); // get rest of line
+
+	// print the query
+	PrintNode (cout, &browseNode);
+
+	MGBrowseQuery(indexData, docLevel, browseNode, browseResult);
+	cout << browseResult;
+	cout << "\n";
+
+      }
+      else { // bad option
+	cout << "bad command\n\n";
+	printHelp();
+      }
+    } // if a .x query
+    else {
+      // regular query
+      queryTree = ParseQuery (queryArray, defaultBoolCombine, defaultStemMethod);
+      if (queryTree == NULL) {
+	cout << "invalid syntax\n";
+      } else {
+	// print the query
+	PrintNode (cout, queryTree);
+	
+	MGQuery (indexData, queryInfo, queryTree, queryResult, level);
+	if (shortOutput) {
+	  queryResult.printShort(cout);
+	  cout << "\n";
+	} else {
+	  cout << queryResult;
+	  cout << "\n";
+	}
+	// delete the query
+	delete queryTree;
+	queryTree = NULL;
+      }
+    }
+  }
+
+
+  // clean up, everybody clean up
+    textData.UnloadData ();
+  indexData.UnloadData ();
+
+  return (0);
+}
+
+
Index: /main/tags/3.01/mgpp/text/TagInfo.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/TagInfo.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TagInfo.cpp	(revision 21112)
@@ -0,0 +1,74 @@
+/**************************************************************************
+ *
+ * TagInfo.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TagInfo.h"
+
+
+void TagInfo::SetDocTag (const char *cStr) {
+  SetCStr (docTag, cStr, strlen(cStr));
+}
+
+void TagInfo::SetIndexLevel (const char *cStr) {
+  SetCStr (indexLevel, cStr, strlen(cStr));
+}
+void TagInfo::AddLevelTag (const char *cStr) {
+  // convert the string
+  UCArray cArr;
+  SetCStr (cArr, cStr, strlen(cStr));
+
+  // insert the tag
+  levelTags.insert (cArr);
+}
+
+
+TagInfo::TagInfo () {
+  Clear ();
+}
+
+void TagInfo::Clear () {
+  docTag.erase (docTag.begin(), docTag.end());
+  indexLevel.erase (indexLevel.begin(), indexLevel.end());
+  levelTags.erase (levelTags.begin(), levelTags.end());
+}
+
+ostream &operator<<(ostream &s, const TagInfo &t) {
+  // output docTag
+  s << "docTag: \"" << t.docTag << "\"\n";
+
+  // output indexLevel
+  s << "indexLevel: \"" << t.indexLevel << "\"\n";
+
+  // output levelTags
+  s << "levelTags: ";
+
+  UCArraySet::const_iterator here = t.levelTags.begin();
+  UCArraySet::const_iterator end = t.levelTags.end();
+  while (here != end) {
+    s << "\"" << (*here) << "\"";
+    ++here;
+    if (here != end) s << ", ";
+  }
+
+  s << "\n";
+  
+  return s;
+}
+
Index: /main/tags/3.01/mgpp/text/TagInfo.h
===================================================================
--- /main/tags/3.01/mgpp/text/TagInfo.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TagInfo.h	(revision 21112)
@@ -0,0 +1,46 @@
+/**************************************************************************
+ *
+ * TagInfo.h -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TAGINFO_H
+#define TAGINFO_H
+
+#include "UCArray.h"
+
+
+class TagInfo {
+public:
+  UCArray docTag;
+  UCArray indexLevel;    // empty for word level index
+  UCArraySet levelTags;
+
+  void SetDocTag (const char *cStr);
+  void SetIndexLevel (const char *cStr);
+  void AddLevelTag (const char *cStr);
+  
+  TagInfo ();
+  void Clear ();
+};
+
+// stream operator to print TagInfo
+ostream &operator<<(ostream &s, const TagInfo &t);
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/Terms.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/Terms.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Terms.cpp	(revision 21112)
@@ -0,0 +1,797 @@
+/**************************************************************************
+ *
+ * Terms.cpp -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "Terms.h"
+#include "words.h"
+#include "stemmer.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+
+void QueryInfo::Clear () {
+  UCArrayClear (docLevel);
+  maxDocs = 0;
+  sortByRank = true;
+  exactWeights = false;
+  needRankInfo = false;
+  needTermFreqs = false;
+}
+
+
+
+void TermFreqData::Clear () {
+  UCArrayClear (tag);
+  UCArrayClear (term);
+  equivTerms.erase(equivTerms.begin(), equivTerms.end());
+  stemMethod = 0;
+  matchDocs = 0;
+  termFreq = 0;
+}
+
+ostream &operator<< (ostream &s, const TermFreqData &t) {
+  s << "<" << t.tag << ">\"" << t.term << "\"stem("
+    << t.stemMethod << ")equiv terms(";
+  
+  unsigned long i;
+  for (i=0; i<t.equivTerms.size(); ++i) {
+    s << t.equivTerms[i] << ", ";
+  }
+  s <<")docs(" << t.matchDocs << ")"
+    << "count("<<t.termFreq<<")";
+  return s;
+}
+
+bool operator== (const TermFreqData &t1, const TermFreqData &t2) {
+  return ((t1.tag == t2.tag) &&
+	  (t1.term == t2.term) &&
+	  (t1.stemMethod == t2.stemMethod) &&
+	  (t1.equivTerms == t2.equivTerms) &&
+	  (t1.matchDocs == t2.matchDocs) &&
+	  (t1.termFreq == t2.termFreq));
+}
+
+
+void QueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+QueryResult::QueryResult () {
+  Clear ();
+}
+
+void QueryResult::printShort(ostream &s) {
+
+  s << "termFreqs: ";
+  for (unsigned long i=0; i<termFreqs.size(); ++i)
+    s << termFreqs[i] << ", ";
+  
+  s << "\nactual number of docs found: " << actualNumDocs;
+  s << "\n\n";
+
+}
+
+
+ostream &operator<< (ostream &s, const QueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const QueryResult &r1, const QueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//---------------------------------------------------
+// new ExtQueryResult stuff
+void ExtQueryResult::Clear () {
+  docs.erase (docs.begin(), docs.end());
+  levels.erase (levels.begin(), levels.end());
+  ranks.erase (ranks.begin(), ranks.end());
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+  actualNumDocs = 0;
+}
+
+ExtQueryResult::ExtQueryResult () {
+  Clear ();
+}
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r) {
+  s << "docs: ";
+  unsigned long i;
+  for (i=0; i<r.docs.size(); ++i)
+    s << r.docs[i] << ", ";
+
+  s << "\nlevels: ";
+  for (i=0; i<r.levels.size(); ++i)
+    s << r.levels[i] << ", ";
+
+  
+  s << "\nranks: ";
+  for (i=0; i<r.ranks.size(); ++i)
+    s << r.ranks[i] << ", ";
+
+  s << "\ntermFreqs: ";
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+  s << "\nactual number of docs found: " << r.actualNumDocs;
+  s << "\n\n";
+
+  return s;
+}
+
+
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2) {
+  return ((r1.docs == r2.docs) &&
+	  (r1.levels == r2.levels) &&
+	  (r1.ranks == r2.ranks) &&
+	  (r1.termFreqs == r2.termFreqs) &&
+	  (r1.actualNumDocs == r2.actualNumDocs));
+}
+
+//-------------------------------------------------------
+// new BrowseQueryResult stuff
+void BrowseQueryResult::Clear () {
+  termFreqs.erase (termFreqs.begin(), termFreqs.end());
+}
+
+BrowseQueryResult::BrowseQueryResult () {
+  Clear ();
+}
+
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r) {
+  s << "terms: ";
+  unsigned long i;
+  for (i=0; i<r.termFreqs.size(); ++i)
+    s << r.termFreqs[i] << ", ";
+    s << "\n\n";
+  return s;
+}
+
+
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2) {
+  return ((r1.termFreqs == r2.termFreqs));
+	 
+}
+
+
+
+
+//--------------------------------------
+void FragData::Clear () {
+  matchDocs = 0;
+  fragNums.erase (fragNums.begin(), fragNums.end());
+  fragFreqs.erase (fragFreqs.begin(), fragFreqs.end());
+}
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (stemMethod == 0 || stemMethod==4 || stemMethod==5) {
+    // don't need to stem the word,
+    // find the word number(s) for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (stemMethod ==0) {
+      if (SearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels, term, wordDictEl, wordElNum))
+	equivWords.push_back (wordElNum);
+      
+      return;
+    } else {
+      // partial matching,
+      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod==5?true:false) );
+      return;
+    }
+  }
+		      
+  // need to stem this word and find it in the blocked stem index
+  
+  unsigned char  mgWord[MAXSTEMLEN + 1];
+  UCArray stemTerm;
+  unsigned long stemmerNum = 0;
+  if (stemMethod == 1) stemmerNum = indexData.sih1.stemmer_num;
+  else if (stemMethod == 2) stemmerNum = indexData.sih2.stemmer_num;
+  else if (stemMethod == 3) stemmerNum = indexData.sih3.stemmer_num;
+   
+  // convert the word to an "mg word"
+  mgWord[0] = term.size();
+  memcpy ((char *)&mgWord[1], &(term[0]), term.size());
+  
+  // stem the word
+  stemmer (stemMethod, stemmerNum, mgWord);
+
+  // convert the result back to a UCArray
+  stemTerm.insert (stemTerm.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+  // need to look up this term in the appropriate dictionary
+  stem_block_dict_el stemDictEl;
+  unsigned long stemElNum;
+  bool result = false;
+  if (stemMethod == 1) {
+    result = SearchStemBlockDictEl (indexData.stem1File,
+			   indexData.sii1,
+			   indexData.sih1.entries_per_block,
+			   indexData.sih1.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 2) {
+    result = SearchStemBlockDictEl (indexData.stem2File,
+			   indexData.sii2,
+			   indexData.sih2.entries_per_block,
+			   indexData.sih2.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+
+  } else if (stemMethod == 3) {
+    result = SearchStemBlockDictEl (indexData.stem3File,
+			   indexData.sii3,
+			   indexData.sih3.entries_per_block,
+			   indexData.sih3.dict_size,
+			   stemTerm,
+			   stemDictEl,
+			   stemElNum);
+  }
+
+  if (result) {
+    equivWords = stemDictEl.equivWords;  
+  }
+}
+
+
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray & termWord) {
+  fragData.Clear();
+
+  // look up the word in the dictionary
+  unsigned long numLevels = indexData.bdh.num_levels;
+  word_block_dict_el wordDictEl;
+  wordDictEl.SetNumLevels (numLevels);
+  if (!SearchWordBlockDictElNum (indexData.dictFile,
+				 indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 numLevels,
+				 termNum, wordDictEl))
+    return; // nothing more to do
+
+  fragData.matchDocs = wordDictEl.levelFreqs[indexData.curLevelNum];
+  termWord = wordDictEl.el;
+  // seek to the appropriate place in the inverted file
+  fseek (indexData.invfFile, wordDictEl.invf_ptr, SEEK_SET);
+  stdio_bitio_buffer buffer (indexData.invfFile);
+    
+  unsigned long B = BIO_Bblock_Init (indexData.bdh.num_frags,
+				     wordDictEl.frag_occur);
+  unsigned long fragNum = 0;
+  unsigned long termFreq = 0;
+
+  unsigned long fragLimitI = 0;
+  unsigned long i;
+  for (i=0; i<wordDictEl.frag_occur; ++i) {
+    fragNum += buffer.bblock_decode (B, NULL);
+    if (!indexData.ifh.word_level_index) termFreq = buffer.gamma_decode (NULL);
+    else termFreq = 1;
+
+    // get the right fragment range
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < (*fragLimits).size() &&
+	     fragNum > (*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    // add the entry if it is within the limits
+    if ((fragLimits == NULL) ||
+	(fragLimitI < (*fragLimits).size() &&
+	 fragNum > (*fragLimits)[fragLimitI].rangeStart &&
+	 fragNum <= (*fragLimits)[fragLimitI].rangeEnd)) {
+      fragData.fragNums.push_back (fragNum);
+      if (needFragFreqs)
+	fragData.fragFreqs.push_back (termFreq);
+    }
+  }
+
+  buffer.done();
+}
+
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData) {
+  outFragData.Clear();
+
+  // the new number of matching documents is the maximum
+  // of the two input matching number of documents -- it
+  // is assumed that these are at the same document level
+  outFragData.matchDocs = (f1.matchDocs > f2.matchDocs) ?
+    f1.matchDocs : f2.matchDocs;
+
+  // do or
+  unsigned long f1I = 0, f1Size = f1.fragNums.size();
+  unsigned long f2I = 0, f2Size = f2.fragNums.size();
+  while (f1I < f1Size || f2I < f2Size) {
+    if (f2I < f2Size &&
+	(f1I >= f1Size ||
+	 f1.fragNums[f1I] > f2.fragNums[f2I])) {
+      // output f2I
+      outFragData.fragNums.push_back (f2.fragNums[f2I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f2.fragFreqs[f2I]);
+      ++f2I;
+      
+    } else if (f1I < f1Size &&
+	       (f2I >= f2Size ||
+		f1.fragNums[f1I] < f2.fragNums[f2I])) {
+      // output f1I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]);
+      ++f1I;
+      
+    } else {
+      // must be equal combine f1I and f2I
+      outFragData.fragNums.push_back (f1.fragNums[f1I]);
+      if (needFragFreqs)
+	outFragData.fragFreqs.push_back (f1.fragFreqs[f1I]+f2.fragFreqs[f2I]);
+      ++f1I;
+      ++f2I;
+    }
+  }
+}
+
+
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits) {
+  // sanity check on range
+  if (startRange > endRange) {
+    signed long temp = endRange;
+    endRange = startRange;
+    startRange = temp;
+  }
+
+  // get min matchdocs
+  if (comFragData.matchDocs < fragData.matchDocs)
+    fragData.matchDocs = comFragData.matchDocs;
+  
+  unsigned long fragDataI = 0;
+  unsigned long fragDataSize = fragData.fragNums.size();
+  unsigned long comFragDataI = 0;
+  unsigned long comFragDataSize = comFragData.fragNums.size();
+  unsigned long fragLimitI = 0;
+  unsigned long fragLimitSize = (fragLimits==NULL) ? 0 : (*fragLimits).size();
+  unsigned long outI = 0;
+
+  while (fragDataI < fragDataSize &&
+	 comFragDataI < comFragDataSize) {
+    signed long fragNum = (signed long)fragData.fragNums[fragDataI];
+    signed long comFragNum = (signed long)comFragData.fragNums[comFragDataI];
+    
+    // go to the right fragment limit (for the com frag)
+    if (fragLimits != NULL) {
+      while (fragLimitI+1 < fragLimitSize &&
+	     comFragNum > (signed long)(*fragLimits)[fragLimitI+1].rangeStart) {
+	++fragLimitI;
+      }
+    }
+
+    if (fragNum <= comFragNum+startRange ||
+	(fragLimits!=NULL &&
+	 fragNum<=(signed long)(*fragLimits)[fragLimitI].rangeStart)) {
+      ++fragDataI;
+      
+    } else if (fragNum > comFragNum+endRange ||
+	       (fragLimits!=NULL &&
+		fragNum>(signed long)(*fragLimits)[fragLimitI].rangeEnd)) {
+      ++comFragDataI;
+      
+    } else {
+      // equal and within tag
+      fragData.fragNums[outI] = comFragNum;
+      if (needFragFreqs) {
+	fragData.fragFreqs[outI] =
+	  (fragData.fragFreqs[fragDataI] < comFragData.fragFreqs[comFragDataI]) ?
+	  fragData.fragFreqs[fragDataI] : comFragData.fragFreqs[comFragDataI];
+      }
+      ++fragDataI;
+      ++comFragDataI;
+      ++outI;
+    }
+  }
+
+  // erase unused part of fragData
+  fragData.fragNums.erase (fragData.fragNums.begin()+outI,
+			   fragData.fragNums.end());
+  if (needFragFreqs)
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin()+outI,
+			      fragData.fragFreqs.end());
+  else
+    fragData.fragFreqs.erase (fragData.fragFreqs.begin(),
+			      fragData.fragFreqs.end());
+}
+
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  result.Clear();
+
+  // log (N / ft)
+  unsigned long N = indexData.levels.levelInfo[indexData.curLevel].numEntries;
+  float wordLog = log((double)N / (double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  if (needRanks) {
+	    Wdt = termDocFreq * wordLog;
+	    result.ranks.push_back (Wqt * Wdt);
+	  }
+	  result.docs.push_back (lastLevelDocNum);
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks){
+	termDocFreq += termData.fragFreqs[termDataI];
+	overallwordfreq += termData.fragFreqs[termDataI];
+      }
+    }
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    if (needRanks) {
+      Wdt = termDocFreq * wordLog;
+      result.ranks.push_back (Wqt * Wdt);
+    }
+    result.docs.push_back (lastLevelDocNum);
+  }
+
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq; // will be zero if needRankInfo 
+                                              //not true
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+  
+  // log (N / ft)
+  float wordLog =
+    log((double)indexData.levels.levelInfo[indexData.curLevel].numEntries/
+	(double)termData.matchDocs);
+
+  // Wqt = fqt * log (N / ft)
+  // note: terms are allowed to have a weight of zero so
+  // they can be excluded from the ranking
+  float Wqt = termWeight * wordLog;
+
+  // Wdt = fdt * log (N / ft)
+  float Wdt;
+  
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long termDocFreq = 0;
+  unsigned long lastLevelDocNum = 0;
+  unsigned long overallwordfreq = 0;
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // add this doc information
+	  Wdt = termDocFreq * wordLog;
+	  
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+	termDocFreq = 0;
+      }
+
+      if (needRanks)
+	termDocFreq += termData.fragFreqs[termDataI];
+     overallwordfreq += termData.fragFreqs[termDataI]; 
+    }
+    
+    ++termDataI;
+  } // while
+
+  if (lastLevelDocNum > 0) {
+    // add the last document information
+    Wdt = termDocFreq * wordLog;
+
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks)
+	result.ranks[resultOutI] = result.ranks[resultI] + Wqt * Wdt;
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+  
+  // add the term frequency information
+  if (queryInfo.needTermFreqs) {
+    TermFreqData termFreqData;
+    termFreqData.tag = tag;
+    termFreqData.term = term;
+    termFreqData.stemMethod = stemMethod;
+    termFreqData.equivTerms = equivTerms;
+    termFreqData.matchDocs = termData.matchDocs;
+    termFreqData.termFreq = overallwordfreq;
+    result.termFreqs.push_back (termFreqData);
+  }
+}
+
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result) {
+  bool needRanks = (queryInfo.sortByRank || queryInfo.needRankInfo);
+
+  unsigned long termDataI = 0;
+  unsigned long termDataSize = termData.fragNums.size();
+  unsigned long levelDocNum = 0;
+  
+  unsigned long lastLevelDocNum = 0;
+
+  unsigned long resultI = 0;
+  unsigned long resultSize = result.docs.size();
+  unsigned long resultOutI = 0;
+  
+  while (termDataI < termDataSize) {
+    if (indexData.levelConverter.FragToLevel (termData.fragNums[termDataI],
+					      levelDocNum)) {
+      if (levelDocNum != lastLevelDocNum) {
+	if (lastLevelDocNum > 0) {
+	  // find this document number
+	  while (resultI < resultSize &&
+		 result.docs[resultI] < lastLevelDocNum)
+	    ++resultI;
+	  
+	  // store the result
+	  if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+	    result.docs[resultOutI] = lastLevelDocNum;
+	    if (needRanks)
+	      result.ranks[resultOutI] = result.ranks[resultI];
+	    ++resultI;
+	    ++resultOutI;
+	  }
+	}
+	
+	lastLevelDocNum = levelDocNum;
+      }
+    }
+    
+    ++termDataI;
+  }
+
+  if (lastLevelDocNum > 0) {
+    // find this document number
+    while (resultI < resultSize &&
+	   result.docs[resultI] < lastLevelDocNum)
+      ++resultI;
+    
+    // store the result
+    if (resultI < resultSize && result.docs[resultI] == lastLevelDocNum) {
+      result.docs[resultOutI] = lastLevelDocNum;
+      if (needRanks) 
+	result.ranks[resultOutI] = result.ranks[resultI];
+      ++resultI;
+      ++resultOutI;
+    }
+  }
+
+  // remove unneeded entries
+  result.docs.erase (result.docs.begin()+resultOutI, result.docs.end());
+  if (needRanks)
+    result.ranks.erase (result.ranks.begin()+resultOutI, result.ranks.end());
+  else
+    result.ranks.erase (result.ranks.begin(), result.ranks.end());
+}
+
+
+
+//--------------------------------------------------------------
+// functions to support full text browse
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number) {
+
+    // find the word number for this term
+    unsigned long wordElNum = 0;
+    unsigned long numLevels = indexData.bdh.num_levels;
+    word_block_dict_el wordDictEl;
+    wordDictEl.SetNumLevels (numLevels);
+    if (NearestSearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
+				      indexData.bdh.entries_per_wblk,
+				      indexData.bdh.word_dict_size,
+				      numLevels, term, wordDictEl, wordElNum))
+      number = wordElNum;
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms) {
+
+  word_block_dict_el_array wordBlocks; // = new word_block_dict_el_array();
+  TermFreqData termdata;
+
+  terms.erase(terms.begin(), terms.end());
+
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, wordBlocks);
+
+  word_block_dict_el_array::iterator here = wordBlocks.begin();
+  word_block_dict_el_array::iterator end = wordBlocks.end();
+
+  while (here != end) {
+    termdata.Clear();
+    termdata.term = (*here).el;
+    termdata.termFreq = (*here).freq;
+    terms.push_back(termdata);
+    ++here;
+  }
+
+}
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms) {
+
+  
+  
+  SearchWordBlockDictElNumRange (indexData.dictFile, indexData.biWords,
+				 indexData.bdh.entries_per_wblk,
+				 indexData.bdh.word_dict_size,
+				 indexData.bdh.num_levels, startTerm,
+				 numTerms, terms);
+
+}
Index: /main/tags/3.01/mgpp/text/Terms.h
===================================================================
--- /main/tags/3.01/mgpp/text/Terms.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Terms.h	(revision 21112)
@@ -0,0 +1,215 @@
+/**************************************************************************
+ *
+ * Terms.h -- Query related functions
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TERMS_H
+#define TERMS_H
+
+#include "IndexData.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+class QueryInfo {
+public:
+  // general query information
+  UCArray docLevel;
+  unsigned long maxDocs; // 0 = all
+  bool sortByRank;
+  bool exactWeights;
+
+  // information need to return
+  bool needRankInfo;
+  bool needTermFreqs;
+
+  void Clear ();
+  QueryInfo () { Clear (); }
+};
+
+
+class TermFreqData {
+public:
+  UCArray tag;  // level tag or query tag
+  UCArray term; // unstemmed term
+  int stemMethod;
+  UCArrayVector equivTerms; // the stemmed and casefolded variants of the term
+  unsigned long matchDocs; // tf for level - num levels
+			   // containing this term
+  unsigned long termFreq;  // overall term freq - num words that
+				// are this term
+  void Clear ();
+  TermFreqData () { Clear (); }
+};
+
+ostream &operator<< (ostream &s, const TermFreqData &t);
+bool operator== (const TermFreqData &t1, const TermFreqData &t2);
+
+typedef vector<TermFreqData> TermFreqArray;
+
+
+typedef vector<float> RankArray;
+typedef vector<unsigned long> DocNumArray;
+
+class QueryResult {
+public:
+  DocNumArray docs;
+  RankArray ranks; // used for accumulators during query
+  
+  TermFreqArray termFreqs;
+
+  unsigned long actualNumDocs;
+  void Clear ();
+  QueryResult ();
+  void printShort(ostream &s);
+};
+
+ostream &operator<< (ostream &s, const QueryResult &r);
+bool operator== (const QueryResult &r1, const QueryResult &r2);
+
+typedef vector<unsigned long> FragNumArray;
+typedef vector<unsigned long> FragFreqArray;
+
+class FragData {
+public:
+  unsigned long matchDocs;    // ft for level
+  FragNumArray fragNums;
+  FragFreqArray fragFreqs;
+
+  void Clear ();
+  FragData () { Clear (); }
+};
+
+
+class FragRange {
+public:
+  unsigned long rangeStart;
+  unsigned long rangeEnd;
+
+  void Clear () { rangeStart = rangeEnd = 0; }
+  FragRange () { Clear (); }
+};
+
+typedef vector<FragRange> FragRangeArray;
+
+
+
+void FindWordNumbers (IndexData &indexData,
+		      const UCArray &term,
+		      unsigned long stemMethod,
+		      vector<unsigned long> &equivWords);
+
+void ReadTermFragData (IndexData &indexData,
+		       bool needFragFreqs,
+		       unsigned long termNum,
+		       FragData &fragData,
+		       FragRangeArray *fragLimits,
+		       UCArray &termWord);
+
+void CombineFragData (bool needFragFreqs,
+		      const FragData &f1,
+		      const FragData &f2,
+		      FragData &outFragData);
+
+// output will be in fragData (as this is an and operation)
+void AndCombineFragData (bool needFragFreqs,
+			 FragData &fragData,
+			 const FragData &comFragData,
+			 signed long startRange,
+			 signed long endRange,
+			 const FragRangeArray *fragLimits);
+
+void FragsToQueryResult (IndexData &indexData,
+			 const QueryInfo &queryInfo,
+			 const FragData &termData,
+			 const UCArray &tag,
+			 const UCArray &term,
+			 unsigned long stemMethod,
+			 unsigned long termWeight,
+			 UCArrayVector &equivTerms,
+			 QueryResult &result);
+
+void AndFragsToQueryResult (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    const UCArray &tag,
+			    const UCArray &term,
+			    unsigned long stemMethod,
+			    unsigned long termWeight,
+			    UCArrayVector &equivTerms,
+			    QueryResult &result);
+
+void RemoveUnwantedResults (IndexData &indexData,
+			    const QueryInfo &queryInfo,
+			    const FragData &termData,
+			    QueryResult &result);
+
+//-----------------------------------------------------------------
+// new QueryResult class to handle retrieval of doc and level nums.
+// Use this class with extended version of MGQuery
+
+class ExtQueryResult : public QueryResult {
+public:
+  DocNumArray levels; // used for returning a different granularity, eg
+  // search sections but return Document numbers, or search Documents, 
+  // return Section numbers.
+ 
+  void Clear ();
+  ExtQueryResult ();
+};
+
+ostream &operator<< (ostream &s, const ExtQueryResult &r);
+bool operator== (const ExtQueryResult &r1, const ExtQueryResult &r2);
+
+//------------------------------------------------------------
+// new functions to handle full text browse
+
+class BrowseQueryResult {
+ public:
+  TermFreqArray termFreqs;
+  void Clear();
+  BrowseQueryResult ();
+
+};
+
+
+ostream &operator<< (ostream &s, const BrowseQueryResult &r);
+bool operator== (const BrowseQueryResult &r1, const BrowseQueryResult &r2);
+
+void FindNearestWordNumber (IndexData &indexData,
+			    const UCArray &term,
+			    unsigned long &number);
+
+void GetTermList(IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 TermFreqArray &terms);
+
+void GetTermList (IndexData &indexData,
+		 unsigned long startTerm,
+		 unsigned long numTerms,
+		 UCArrayVector &terms);
+
+#endif
Index: /main/tags/3.01/mgpp/text/TextEl.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/TextEl.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TextEl.cpp	(revision 21112)
@@ -0,0 +1,374 @@
+/**************************************************************************
+ *
+ * TextEl.cpp -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "TextEl.h"
+#include "ctype.h"
+
+#define REC_TERM   '\002'
+#define PARA_TERM  '\003'
+
+
+void TextEl::Clear () {
+  elType = TextE;
+  tagName.erase (tagName.begin(), tagName.end());
+  text.erase (text.begin(), text.end());
+}
+
+
+static bool ReadTextTag (FileBuf &buf, TextEl &el) {
+  // we must have some sort of tag
+  unsigned char c;
+  if (!buf.Peek(c)) return false;
+  
+  // get the '<'
+  if (c != '<') return false;
+  el.text.push_back (c);
+  if (!buf.Next() || !buf.Peek (c)) return false;
+  
+  // see if we have a '/'
+  if (c == '/') {
+    // we have a closing tag
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+    el.elType = CloseTagE;
+  } else {
+    // we have an opening tag
+    el.elType = OpenTagE;
+  }
+
+  // get the tag name
+  while (isalnum(c)) {
+    el.text.push_back (c);
+    el.tagName.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+  
+  // get everything up to the closing '>'
+  while (c != '>') {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) return false;
+  }
+
+  // get the closing '>'
+  el.text.push_back (c);
+  buf.Next();
+
+  // if there is no tag name then make this a text element
+  if (el.tagName.empty()) el.elType = TextE;
+  
+  return true;
+}
+
+static bool ReadTextTag (u_char ** buffer, TextEl &el) {
+  // we must have some sort of tag
+  u_char * buffer_pos = *buffer;
+  if (*buffer_pos == '\0' || *buffer_pos != '<') {
+     return false;
+  }
+  el.text.push_back('<');
+  buffer_pos ++;
+  if (*buffer_pos == '\0') {
+    *buffer = buffer_pos;
+    return false;
+  }
+  // see if we have a '/'
+  if (*buffer_pos == '/') {
+    el.text.push_back('/');
+    el.elType = CloseTagE;
+    buffer_pos++;
+    if (*buffer_pos == '\0') {    
+      *buffer = buffer_pos;
+      return false;
+    }
+  } else {
+    el.elType = OpenTagE;
+  }
+  // get the tag name
+  while (isalnum(*buffer_pos)) {
+    el.text.push_back (*buffer_pos);
+    el.tagName.push_back (*buffer_pos);
+    buffer_pos++;
+    
+    if (*buffer_pos == '\0') {
+      *buffer = buffer_pos;
+      return false;
+    }
+
+  }
+  cout << "tag name = "<<el.tagName<<endl;
+  // get everything up to closing '>'
+  while (*buffer_pos != '>') {
+    el.text.push_back(*buffer_pos);
+    buffer_pos++;
+    if (*buffer_pos == '\0') {
+      *buffer = buffer_pos;
+      return false;
+    }
+    
+  }
+  // get teh closing '>'
+  el.text.push_back('>');
+  buffer_pos++;
+  *buffer = buffer_pos;
+  cout << "at end of readtexttag, buffer is "<<*buffer<<endl;
+  // if there is no tag name then make this a text element
+  if (el.tagName.empty()) el.elType = TextE;
+  
+  return true;
+}
+
+static void ToggleParaTag (TextEl &el, bool &compatInPara) {
+  SetCStr (el.tagName, "Paragraph", 9);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (compatInPara) {
+    el.elType = CloseTagE;
+    el.text.push_back (PARA_TERM);
+  } else el.elType = OpenTagE;
+  compatInPara = !compatInPara;
+}
+
+static void SetRecTag (TextEl &el, TextElType elType) {
+  el.elType = elType;
+  SetCStr (el.tagName, "Document", 8);
+  el.text.erase (el.text.begin(), el.text.end());
+  if (elType == CloseTagE)
+    el.text.push_back (REC_TERM);
+}
+
+
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara) {
+  el.Clear();
+
+  unsigned char c;
+  if (!buf.Peek (c)) return false;
+
+  if (compatMode) {
+    if (c == PARA_TERM) {
+      ToggleParaTag (el, compatInPara);
+      if (!compatInPara) buf.Next();
+      return true;
+    }
+    if (c == REC_TERM) {
+      if (compatInPara) {
+	// need to close this paragraph
+	ToggleParaTag (el, compatInPara);
+	return true;
+      }
+      // can close this document
+      buf.Next();
+      SetRecTag (el, CloseTagE);
+      return true;
+    }
+    if (!compatInPara) {
+      // need to open a paragraph
+      ToggleParaTag (el, compatInPara);
+      return true;
+    }
+  }
+  
+  // see if we have some sort of tag
+  if (c == '<') return ReadTextTag (buf, el);
+
+  // read in a text element
+  el.elType = TextE;
+  while (c != '<' && !(compatMode && (c == PARA_TERM || c == REC_TERM))) {
+    el.text.push_back (c);
+    if (!buf.Next() || !buf.Peek (c)) break;
+  }
+
+  return true;
+}
+bool ReadTextEl (u_char **buffer, TextEl &el,
+		 bool compatMode, bool &compatInPara) {
+  el.Clear();
+  u_char * buffer_pos = *buffer;
+  if (*buffer_pos == '\0') return false;
+  if (compatMode) {
+    if (*buffer_pos == PARA_TERM) {
+      ToggleParaTag (el, compatInPara);
+      if (!compatInPara) buffer_pos++;
+      *buffer = buffer_pos;
+      return true;
+    }
+    if (*buffer_pos == REC_TERM) {
+      if (compatInPara) {
+	// need to close this paragraph
+	ToggleParaTag (el, compatInPara);
+	*buffer = buffer_pos;
+	return true;
+      }
+      // can close this document
+      buffer_pos++;
+      SetRecTag (el, CloseTagE);
+      *buffer = buffer_pos;
+      return true;
+    }
+    if (!compatInPara) {
+      // need to open a paragraph
+      ToggleParaTag (el, compatInPara);
+      *buffer = buffer_pos;
+      return true;
+    }
+  }
+  
+  // see if we have some sort of tag
+  if (*buffer_pos == '<') return ReadTextTag (buffer, el);
+
+  // read in a text element
+  el.elType = TextE;
+  while (*buffer_pos != '<' && !(compatMode && (*buffer_pos == PARA_TERM || *buffer_pos == REC_TERM))) {
+    el.text.push_back (*buffer_pos);
+    buffer_pos++;
+    if (*buffer_pos == '\0') break;
+  }
+  *buffer = buffer_pos;
+  cout << "text element: "<<el.text<<endl;
+  return true;
+}
+  
+static void AddTextEl (TextElArray &doc, unsigned long &docLen,
+		       const TextEl &el) {
+  doc.push_back (el);
+  docLen += el.text.size();
+
+//    cout << "elType:  " << el.elType << "\n";
+//    cout << "tagName: " << el.tagName << "\n";
+//    cout << "text:    \"" << el.text << "\"\n\n";
+}
+
+
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode) {
+  bool compatInPara = false;
+  bool foundDocEl = false;
+  TextEl el;
+  
+  doc.erase (doc.begin(), doc.end());
+  docLen = 0;
+
+  if (compatMode) {
+    // add <Document><Paragraph>
+    SetRecTag (el, OpenTagE);
+    AddTextEl (doc, docLen, el);
+    ToggleParaTag (el, compatInPara);
+    AddTextEl (doc, docLen, el);
+    
+  } else {
+    // look for an opening docTag
+    do {
+      if (!ReadTextEl (buf, el, compatMode, compatInPara)) return false;
+    } while (el.elType != OpenTagE || el.tagName != docTag);
+    
+    AddTextEl (doc, docLen, el);
+  }
+
+  
+  // get all elements until the closing docTag
+  while (ReadTextEl (buf, el, compatMode, compatInPara)) {
+    foundDocEl = true;
+    AddTextEl (doc, docLen, el);
+    if (el.elType == CloseTagE && el.tagName == docTag)
+      return true;
+  }
+
+  
+  if (compatMode) {
+    if (!foundDocEl) { // end of text
+      doc.erase (doc.begin(), doc.end());
+      return false;
+    }
+
+    // if we got here then the eof was encountered before
+    // the closing document tag
+    if (compatInPara) {
+      // need to close this paragraph
+      ToggleParaTag (el, compatInPara);
+      AddTextEl (doc, docLen, el);
+    }
+    // close this document
+    SetRecTag (el, CloseTagE);
+    AddTextEl (doc, docLen, el);
+  }
+  
+  return true;
+}
+
+// copy of readdoc to get TextElArray from u_char buffer
+bool ReadDoc(u_char ** buffer, const UCArray &docTag, 
+	     TextElArray &doc, unsigned long &docLen,
+	     bool compatMode) {
+  bool compatInPara = false;
+  bool foundDocEl = false;
+  TextEl el;
+  
+  doc.erase (doc.begin(), doc.end());
+  docLen = 0;
+
+  if (compatMode) {
+    // add <Document><Paragraph>
+    SetRecTag (el, OpenTagE);
+    AddTextEl (doc, docLen, el);
+    ToggleParaTag (el, compatInPara);
+    AddTextEl (doc, docLen, el);
+    
+  } else {
+    // look for an opening docTag
+    do {
+      if (!ReadTextEl (buffer, el, compatMode, compatInPara)) return false;
+    } while (el.elType != OpenTagE || el.tagName != docTag);
+    
+    AddTextEl (doc, docLen, el);
+  }
+
+  
+  // get all elements until the closing docTag
+  while (ReadTextEl (buffer, el, compatMode, compatInPara)) {
+    foundDocEl = true;
+    AddTextEl (doc, docLen, el);
+    if (el.elType == CloseTagE && el.tagName == docTag)
+      return true;
+  }
+
+  
+  if (compatMode) {
+    if (!foundDocEl) { // end of text
+      doc.erase (doc.begin(), doc.end());
+      return false;
+    }
+
+    // if we got here then the eof was encountered before
+    // the closing document tag
+    if (compatInPara) {
+      // need to close this paragraph
+      ToggleParaTag (el, compatInPara);
+      AddTextEl (doc, docLen, el);
+    }
+    // close this document
+    SetRecTag (el, CloseTagE);
+    AddTextEl (doc, docLen, el);
+  }
+  
+  return true;
+}
+
+
Index: /main/tags/3.01/mgpp/text/TextEl.h
===================================================================
--- /main/tags/3.01/mgpp/text/TextEl.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TextEl.h	(revision 21112)
@@ -0,0 +1,69 @@
+/**************************************************************************
+ *
+ * TextEl.h -- Data structures for parsed documents
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTEL_H
+#define TEXTEL_H
+
+#include "UCArray.h"
+#include "FileBuf.h"
+#include "sysfuncs.h"
+
+enum TextElType {OpenTagE, CloseTagE, TextE};
+
+class TextEl {
+protected:
+  bool ReadTextTag (FileBuf &buf, TextEl &el);
+  bool ReadTextTag (u_char ** buffer, TextEl &el);
+public:
+  TextElType elType;
+  UCArray tagName;
+  UCArray text; // complete text, including tag
+
+  TextEl () { Clear (); }
+  void Clear ();
+};
+
+
+typedef vector<TextEl> TextElArray;
+
+
+// compatMode parses documents similar to "old mg" -- ie with
+// C-b between documents and C-c between paragraphs
+
+// returns true if a text element could be read
+bool ReadTextEl (FileBuf &buf, TextEl &el,
+		 bool compatMode, bool &compatInPara);
+bool ReadTextEl (u_char **buffer, TextEl &el,
+		 bool compatMode, bool &compatInPara);
+
+// looks for an opening docTag and reads all text elements up
+// to a corresponding closing docTag. note that this does not
+// hierarchically parse the document so you can't nest documents.
+// returns true if a document could be found
+bool ReadDoc (FileBuf &buf, const UCArray &docTag,
+	      TextElArray &doc, unsigned long &docLen,
+	      bool compatMode=false);
+
+bool ReadDoc(u_char **buffer, const UCArray &docTag, 
+	     TextElArray &doc, unsigned long &docLen,
+	     bool compatMode=false);
+
+#endif
Index: /main/tags/3.01/mgpp/text/TextGet.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/TextGet.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TextGet.cpp	(revision 21112)
@@ -0,0 +1,685 @@
+/**************************************************************************
+ *
+ * TextGet.cpp -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// is important to be first, so we escape the truncation warning on VC++
+#include "TextGet.h"
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "mg_files.h"
+#include "netorder.h"
+#include "mg_errors.h"
+#include "locallib.h"
+#include "words.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+
+typedef enum huff_type {lengths, chars};
+
+
+static auxiliary_dict *LoadAuxDict (compression_dict &cd, FILE *text_aux_dict) {
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = new auxiliary_dict))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  memset (ad, '\0', sizeof (*ad));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = new u_char[ad->afh[i].mem_for_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+      if (!(ad->words[i] = new u_char* [ad->afh[i].num_frags]))
+	{
+	  mg_errno = MG_NOMEM;
+	  delete ad;
+	  return (NULL);
+	}
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < (int)ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd.cdh.novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd.cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+static u_char ***ReadInWords (FILE *dict, compression_dict &cd,
+			      comp_frags_header *cfh, u_char **escape) {
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd.cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i) {
+    ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+    mem_reqd += cfh->huff_words_size[i];
+  }
+
+  if (!(vals = new u_char* [ptrs_reqd]))
+    return (NULL);
+
+  if (!(values = new u_char** [MAX_HUFFCODE_LEN + 1]))
+    return (NULL);
+
+  if (!(next_word[0] = new u_char[mem_reqd])) 
+    return (NULL);
+
+  cd.MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+static int Load_Comp_HuffData(compression_dict &cd, int which, FILE *dict, 
+			      huff_type type) {
+  huff_data * hd;
+  u_long **   vals;
+
+  if (!(hd = new huff_data))
+    return 1;
+  cd.MemForCompDict += sizeof (huff_data);
+  if (Read_Huffman_Data (dict, hd, &cd.MemForCompDict, NULL) == -1)
+    return 2;
+  if (!(vals = Generate_Huffman_Vals (hd, &cd.MemForCompDict)))
+    return 3;
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+  if (type == chars)
+    {
+      cd.chars_huff[which] = hd;
+      cd.chars_vals[which] = vals;
+    }
+  else
+    {
+      cd.lens_huff[which] = hd;
+      cd.lens_vals[which] = vals;
+    }
+
+  return 0;
+}
+
+static int Load_Comp_FragsHeader(compression_dict &cd, int which, int getEscape,
+				 FILE *dict) {
+  if (!(cd.cfh[which] = new comp_frags_header))
+    return 1;
+  cd.MemForCompDict += sizeof (*cd.cfh[which]);
+  if (Read_cfh (dict, cd.cfh[which], &cd.MemForCompDict, NULL) == -1)
+    return 2;
+
+  if (!(cd.values[which] = ReadInWords (dict, cd, cd.cfh[which],
+					getEscape == 0 ? NULL : &cd.escape[which])))
+    return 3;
+  
+  return 0;
+}
+
+static bool LoadSlowCompDict (FILE *dict, FILE *aux_dict, compression_dict &cd) {
+  if (dict == NULL) return false;
+
+  int which;
+
+  memset (&cd, '\0', sizeof (compression_dict));
+
+  cd.MemForCompDict = sizeof (compression_dict);
+
+  if (Read_cdh (dict, &cd.cdh, &cd.MemForCompDict, NULL) == -1)
+    return false;
+
+  for (which = 0; which < 2; ++which)
+    switch (cd.cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Load_Comp_FragsHeader(cd, which, 0, dict) != 0)
+	    return false;
+	  cd.escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+
+	  if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+	    return false;
+
+	  if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+	    return false;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd.cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return false;
+	    }
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Load_Comp_HuffData(cd, which, dict, chars) != 0)
+		return false;
+
+	      if (Load_Comp_HuffData(cd, which, dict, lengths) != 0)
+		return false;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd.cdh.novel_method == MG_NOVEL_DELTA ||
+      cd.cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  cd.Clear();
+	  return false;
+	}
+
+      if (!(cd.ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  cd.Clear();
+	  return false;
+	}
+    }
+
+  mg_errno = MG_NOERROR;
+
+  cd.fast_loaded = 0;
+  
+  return true;
+}
+
+
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p,cd) & 7))) != 0)
+
+// fast loading really needs to be totally re-writen. "Unloading" the
+// text data will currently cause a crash because memory is being
+// deleted multiple times (and probably a zillion other reasons).
+static bool LoadFastCompDict (FILE *text_fast_comp_dict, compression_dict &_cd) {
+  if (text_fast_comp_dict == NULL) return false;
+  
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  compression_dict *cd;
+  if (!(cd = (compression_dict *)malloc (mem))) {
+    mg_errno = MG_NOMEM;
+    return false;
+  }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = new u_char[fixup_mem]))
+    {
+      mg_errno = MG_NOMEM;
+      return false;
+    }
+
+  fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; ++p)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    for (i = 0; i <= 1; ++i)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; ++j)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  delete []fixup;
+
+  // the whole fast comp dict is a bit of a hack so I don't
+  // feel too bad about the next line :-) -- Rodger.
+  _cd = *cd;
+  
+  return true;
+}
+
+
+static bool LoadCompDict (FILE *compDictFile,
+			  FILE *auxDictFile,
+			  FILE *fastCompDictFile,
+			  compression_dict &cd) {
+  // see if we have a fast loading compression dictionary
+  if (fastCompDictFile != NULL)
+    return LoadFastCompDict (fastCompDictFile, cd);
+  
+  // slow compression dictionary
+  return LoadSlowCompDict (compDictFile, auxDictFile, cd);
+}
+
+
+// try to open the dictionary files and load the dictionary
+static bool OpenLoadCompDict (char *textname, compression_dict &cd) {
+  FILE *compDictFile = NULL;
+  FILE *auxDictFile = NULL;
+  FILE *fastCompDictFile = NULL;
+
+  fastCompDictFile = open_file (textname, TEXT_DICT_FAST_SUFFIX,
+				"rb", MAGIC_FAST_DICT, MG_CONTINUE);
+  
+  if (fastCompDictFile == NULL) {
+    compDictFile = open_file (textname, TEXT_DICT_SUFFIX,
+			      "rb", MAGIC_DICT, MG_MESSAGE);
+    auxDictFile =  open_file (textname, TEXT_DICT_AUX_SUFFIX,
+			      "rb", MAGIC_AUX_DICT, MG_CONTINUE);
+  }
+  
+  bool res = LoadCompDict (compDictFile, auxDictFile, fastCompDictFile, cd);
+
+  if (compDictFile != NULL) fclose (compDictFile);
+  if (auxDictFile != NULL) fclose (auxDictFile);
+  if (fastCompDictFile != NULL) fclose (fastCompDictFile);
+  
+  return res;
+}
+
+static bool LoadLevels (char *textname, FTextLevel &levels) {
+  FILE *levelFile = NULL;
+
+  // open the text level file
+  levelFile = open_file (textname, TEXT_LEVEL_SUFFIX,
+			 "rb", MAGIC_TEXT_LEVELS, MG_CONTINUE);
+  if (levelFile == NULL) return false;
+
+  // seek to the appropriate place and read the level information
+  bool res = ((fseek (levelFile, sizeof (u_long), SEEK_SET) == 0) &&
+	      levels.Read (levelFile));
+
+  // close the file
+  fclose (levelFile);
+  
+  return res;
+}
+
+
+TextData::TextData () {
+  // put file pointers in known state first
+  textFile = NULL;
+  textIdxFile = NULL;
+  Clear ();
+}
+
+void TextData::Clear () {
+  cd.Clear(); 
+  textFile = NULL;
+  textIdxFile = NULL;
+  cth.Clear();
+  levels.Clear();
+}
+
+bool TextData::LoadData (char *basepath, char *textname) {
+  
+  if (textname[0] == '\0') return false;
+  
+  // set the basepath
+  set_basepath(basepath);
+  
+  // load the compression dictionary
+  if (!OpenLoadCompDict (textname, cd)) return false;
+
+  // open the compressed text and text index file
+  textFile = open_file (textname, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_CONTINUE);
+  if (textFile == NULL) return false;
+
+  textIdxFile = open_file (textname, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_CONTINUE);
+  if (textIdxFile == NULL) return false;
+
+  // read in the compressed text header
+  if ((fseek (textFile, sizeof (u_long), SEEK_SET) != 0) || !cth.Read (textFile))
+    return false;
+
+  // read in the level information
+  if (!LoadLevels (textname, levels)) return false;
+  
+  return true;
+}
+
+bool TextData::UnloadData () {
+  // close any open files
+  if (textFile != NULL) {
+    fclose (textFile);
+    textFile = NULL;
+  }
+  if (textIdxFile != NULL) {
+    fclose (textIdxFile);
+    textIdxFile = NULL;
+  }
+
+  // do general clear
+  Clear ();
+  
+  return true;  
+}
+
+
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx) {
+  // make sure the text index file was opened successfully
+  if (td.textIdxFile == NULL) return false;
+
+  // read in the index
+  TextLevelInfo &levelInfo = td.levels.levelInfo[docLevel];
+  if (!docIdx.Read (td.textIdxFile, levelInfo, docNum)) return false;
+  
+  return true;
+}
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        __code += __code + buffer.bit();	                  	\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText) {
+  // erase the current text
+  docText.erase (docText.begin(), docText.end());
+
+  // look up the information about this document
+  TextIdx docIdx;
+  if (!GetDocIdx (td, docLevel, docNum, docIdx)) return false;
+
+  // do seek to appropriate position
+  stdio_bitio_buffer buffer (td.textFile);
+  buffer.seek (docIdx.start.byte, docIdx.start.bit);
+  
+  // decompress the document
+  compression_dict &cd = td.cd;
+  auxiliary_dict *ad = cd.ad;
+  int which = docIdx.which;
+  unsigned long num_bits = (docIdx.end.byte*8+(8-docIdx.end.bit)) -
+    (docIdx.start.byte*8+(8-docIdx.start.bit));
+  unsigned long bits = 0;
+
+  if (docText.capacity() < docText.size() + num_bits + 1) {
+    docText.reserve(docText.size() + num_bits + 1);
+  }
+  // keep decoding bits until enough bits have been decoded
+  while (bits < num_bits) {
+    register unsigned code, len;
+    register int r;
+    register u_char *t, *b = NULL;
+    u_char word[MAXWORDLEN + 1];
+    
+    if (cd.cfh[which]) {
+      MY_HUFF_DECODE (len, code, cd.cfh[which]->hd.min_code);
+      bits += len;
+      
+      r = code & ((1 << cd.cdh.lookback) - 1);
+      t = cd.values[which][len][code >> cd.cdh.lookback];
+      
+      /* step through from base pointer */
+      b = word + 1;
+      while (r--) {
+	register int copy = *t >> 4;
+	memcpy (word + copy + 1, t + 1, *t & 0xf);
+	word[0] = copy + (*t & 0xf);
+	t += ((*t) & 0xf) + 1;
+      }
+    } else t = NULL;
+    
+    if (t == cd.escape[which]) {
+      switch (cd.cdh.novel_method) {
+      case MG_NOVEL_HUFFMAN_CHARS:
+	{
+	  int len, i;
+	  int c;
+	  len = buffer.huff_decode(cd.lens_huff[which]->min_code,
+				   cd.lens_vals[which], &bits);
+	  for (i = 0; i < len; ++i) {
+	    c = buffer.huff_decode(cd.chars_huff[which]->min_code,
+				   cd.chars_vals[which], &bits);
+	    docText.push_back (c);
+	  }
+	}
+	break;
+      case MG_NOVEL_DELTA:
+      case MG_NOVEL_HYBRID:
+	{
+	  int idx = 0, len;
+	  u_char *base;
+	  switch (cd.cdh.novel_method)
+	    {
+	    case MG_NOVEL_DELTA:
+	      {
+		idx = buffer.delta_decode (&bits);
+		--idx;
+	      }
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      {
+		int k;
+		k = buffer.gamma_decode (&bits);
+		--k;
+		idx = buffer.binary_decode(ad->blk_end[which][k] -
+					   ad->blk_start[which][k] + 1, 
+					   &bits);
+		idx += ad->blk_start[which][k] - 1;
+	      }
+	      break;
+	    }
+	  base = ad->words[which][idx];
+	  len = *base++;
+	  for (; len; --len)
+	    {
+	      docText.push_back (*base++);
+	    }
+	}
+	break;
+      }
+    }
+    else
+      {
+	/* copy over the matching prefix */
+	r = (*t >> 4);
+	while (r--) {
+	  docText.push_back (*b++);
+	}
+	
+	/* and the stored suffix */
+	r = ((*t) & 0xf);
+	while (r--) {
+	  docText.push_back (*++t);
+	}
+      }
+    which = !which;
+  }
+  
+  buffer.done();
+  
+  return true;
+}
+
Index: /main/tags/3.01/mgpp/text/TextGet.h
===================================================================
--- /main/tags/3.01/mgpp/text/TextGet.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/TextGet.h	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * TextGet.h -- Decompressing the text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef TEXTGET_H
+#define TEXTGET_H
+
+#include "text.h"
+#include "FText.h"
+
+
+class TextData {
+public:
+  compression_dict cd;
+  FILE *textFile;
+  FILE *textIdxFile;
+  compressed_text_header cth;
+  FTextLevel levels;
+
+  TextData ();
+  void Clear ();
+
+  // loads compression dictionary, the compressed text header,
+  // and all level informaiton
+  bool LoadData (char *basepath, char *textname);
+  bool UnloadData ();
+};
+
+
+// note: document numbers start at 1
+
+// get information about where a document starts and finishes
+bool GetDocIdx (TextData &td, const UCArray &docLevel,
+		unsigned long docNum, TextIdx &docIdx);
+
+
+// decompress a document's text
+bool GetDocText (TextData &td, const UCArray &docLevel,
+		 unsigned long docNum, UCArray &docText);
+
+#endif
Index: /main/tags/3.01/mgpp/text/UCArray.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/UCArray.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/UCArray.cpp	(revision 21112)
@@ -0,0 +1,360 @@
+/**************************************************************************
+ *
+ * UCArray.cpp -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+void SetCStr (UCArray &text, const char *cStr) {
+  text.erase(text.begin(), text.end());
+  
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint) {
+  text.erase(text.begin(), text.end());
+
+  // reserve the needed space in advance
+  if (text.capacity() < nSizeHint + 1) {
+    text.reserve(nSizeHint + 1);
+  }
+  while (*cStr != '\0') {
+    text.push_back (*cStr);
+    ++cStr;
+  }
+}
+
+char * GetCStr(const UCArray& text) {
+
+  char *cstr = new char[text.size()+1];
+  UCArray::const_iterator here = text.begin();
+  UCArray::const_iterator end = text.end();
+
+  int i = 0;
+  while (here != end) {
+    cstr[i] = text[i];
+    ++here; ++i;
+  }
+  cstr[i]='\0';
+  return cstr;
+}
+
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr)
+{
+  if ((cStr == NULL || *cStr == '\0') && text.empty()) return true;
+  UCArray::const_iterator thisUC = text.begin();
+  UCArray::const_iterator endUC = text.end();
+  while (thisUC != endUC && *cStr != '\0') {
+    if (*thisUC != *cStr) return false;
+    ++cStr; ++thisUC;
+  }
+  if (thisUC == endUC && *cStr == '\0') return true;
+  return false;
+}
+
+ostream &operator<<(ostream &s, const UCArray &a) {
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    s << *here;
+    ++here;
+  }
+
+  return s;
+}
+
+
+bool ReadVarLenUL (FILE *f, unsigned long &n) {
+  register unsigned long temp = 0;
+  register unsigned int bitPos = 0;
+  unsigned char b = 0;
+
+  do {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    temp |= (b & 0x7f) << bitPos;
+    bitPos += 7;
+  } while (b >= 0x80 && bitPos < 32);
+
+  n = temp;
+  
+  return true;
+}
+
+bool WriteVarLenUL (FILE *f, unsigned long n) {
+  register unsigned long temp = n;
+  register unsigned char b = 0;
+  do {
+    b = static_cast<unsigned char> (temp & 0x7f);
+    if (temp >= 0x80) b |= 0x80;
+    fputc (b, f);
+    if (ferror (f) != 0) return false;
+  } while ((temp = temp >> 7) > 0);
+  
+  return true;
+}
+
+
+bool ReadUL (FILE *f, unsigned long &n) {
+  if (fread (&n, sizeof (unsigned long), 1, f) <= 0) return false;
+  NTOHUL (n);
+  return true;
+}
+
+
+bool WriteUL (FILE *f, unsigned long n) {
+  HTONUL (n);
+  return (fwrite (&n, sizeof (unsigned long), 1, f) > 0);
+}
+
+bool ReadF (FILE *f, float &n) {
+  if (fread (&n, sizeof (float), 1, f) <= 0) return false;
+  NTOHF(n);
+  return true;
+}
+
+bool WriteF (FILE *f, float n) {
+  HTONF(n);
+  return (fwrite (&n, sizeof (float), 1, f) > 0);
+}
+
+bool ReadD (FILE *f, double &n) {
+  if (fread (&n, sizeof (double), 1, f) <= 0) return false;
+  NTOHD(n);
+  return true;
+}
+
+bool WriteD (FILE *f, double n) {
+  HTOND(n);
+  return (fwrite (&n, sizeof (double), 1, f) > 0);
+}
+
+bool ReadUCArray (FILE *f, UCArray &a) {
+  // clear the array in preparation
+  a.erase (a.begin(), a.end());
+
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+
+  // reserve the needed space in advance
+  if (a.capacity() < arraySize + 1) {
+    a.reserve(arraySize + 1);
+  }
+
+  // read in the array
+  unsigned char b = 0;
+  while (arraySize > 0) {
+    b = fgetc (f);
+    if (feof(f)) return false;
+    a.push_back (b);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteUCArray (FILE *f, const UCArray &a) {
+  // write out the array size
+  if (!WriteVarLenUL (f, a.size())) return false;
+
+  UCArray::const_iterator here = a.begin();
+  UCArray::const_iterator end = a.end();
+  while (here != end) {
+    fputc (*here, f);
+    if (ferror (f) != 0) return false;
+    
+    ++here;
+  }
+  
+  return true;
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+
+int DictCompare (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  unsigned int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((diff = casecharmap[*a1Here] - casecharmap[*a2Here]) != 0)
+      return diff;
+    if (pos == 0 && (diff = *a1Here - *a2Here) != 0)
+      pos = diff;
+
+    ++a1Here;
+    ++a2Here;
+  }
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+}
+
+// does the first string start with the second?
+bool StartsWith (const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if ((*a1Here != *a2Here))
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2) {
+  unsigned int l1 = a1.size();
+  unsigned int l2 = a2.size();
+  if (l2 > l1) {
+    // if the prefix is longer than the string, it can't start with it
+    return false; 
+  }
+  unsigned int l =l2;
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+  
+  while (l--) {
+    if (casecharmap[*a1Here] != casecharmap[*a2Here]) 
+      return false;
+    ++a1Here;
+    ++a2Here;
+  }
+  return true; // we have successfully matched the whole way
+    
+}
+
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2) {
+  unsigned long l = (a1.size() < a2.size()) ? a1.size() : a2.size();
+  unsigned long i = 0;
+
+  UCArray::const_iterator a1Here = a1.begin();
+  UCArray::const_iterator a2Here = a2.begin();
+
+  while (i < l && *a1Here == *a2Here) {
+    ++i; ++a1Here; ++a2Here;
+  }
+  
+  return i;
+}
+
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a) {
+  unsigned char preLen;
+  unsigned char sufLen;
+  
+  if (prev != NULL) preLen = PrefixLen (*prev, a);
+  else preLen = 0;
+  sufLen = a.size() - preLen;
+
+  // output the prefix length, suffix length, and the suffix
+  fputc (preLen, f);
+  if (ferror(f) != 0) return false;
+  fputc (sufLen, f);
+  if (ferror(f) != 0) return false;
+  char* tmp=GetCStr(a);
+  int ret=(fwrite (tmp+preLen, sizeof (char), sufLen, f) == sufLen);
+  delete []tmp;
+  return (ret != 0);
+}
+
+// a also used for prev
+bool ReadPreSufStr (FILE *f, UCArray &a) {
+  unsigned char preLen = 0;
+  unsigned char sufLen = 0;
+
+  preLen = fgetc(f);
+  sufLen = fgetc(f);
+
+  if (a.size() > preLen) a.erase (a.begin()+preLen, a.end());
+
+  // reserve the needed space in advance
+  if (a.capacity() < a.size() + sufLen + 1) {
+    a.reserve(a.size() + sufLen + 1);
+  }
+
+  while (sufLen > 0) {
+    unsigned char c = fgetc (f);
+    a.push_back (c);
+    --sufLen;
+  }
+
+  return true;
+}
+
Index: /main/tags/3.01/mgpp/text/UCArray.h
===================================================================
--- /main/tags/3.01/mgpp/text/UCArray.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/UCArray.h	(revision 21112)
@@ -0,0 +1,131 @@
+/**************************************************************************
+ *
+ * UCArray.h -- vector based string class
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef UCARRAY_H
+#define UCARRAY_H
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if !defined (GSDL_NAMESPACE_BROKEN) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+using namespace std;
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\set>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <set.h>
+#else
+#  include <vector>
+#  include <set>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#include <stdio.h>
+
+
+typedef vector<unsigned char> UCArray;
+
+// functions to manipulate UCArrays
+void SetCStr (UCArray &text, const char *cStr);
+// same as SetCStr but first tries to allocate nSizeHint space (only if needed)
+void SetCStr (UCArray &text, const char *cStr, size_t nSizeHint);
+char * GetCStr(const UCArray& text);
+inline void UCArrayClear (UCArray &a) {
+  a.erase (a.begin(), a.end());
+}
+bool UCArrayCStrEquals(const UCArray &text, const unsigned char *cStr);
+inline bool UCArrayCStrEquals(const UCArray &text, const char *cStr) { return UCArrayCStrEquals(text, (const unsigned char *)cStr); }
+
+// stream operator to print UCArray
+ostream &operator<<(ostream &s, const UCArray &a);
+
+// used for reading and writing unsigned characters
+inline bool ReadUC (FILE *f, unsigned char &c) {
+  return (fread (&c, sizeof (unsigned char), 1, f) == 1);
+}
+inline bool WriteUC (FILE *f, unsigned char c) {
+  return (fwrite (&c, sizeof (unsigned char), 1, f) == 1);
+}
+
+// used for reading and writing variable length unsigned longs
+bool ReadVarLenUL (FILE *f, unsigned long &n);
+bool WriteVarLenUL (FILE *f, unsigned long n);
+
+// used for reading and writing unsigned longs
+bool ReadUL (FILE *f, unsigned long &n);
+bool WriteUL (FILE *f, unsigned long n);
+
+// used for reading and writing floats
+bool ReadF (FILE *f, float &n);
+bool WriteF (FILE *f, float n);
+
+// used for reading and writing doubles
+bool ReadD (FILE *f, double &n);
+bool WriteD (FILE *f, double n);
+
+// used for reading and writing arrays to files
+bool ReadUCArray (FILE *f, UCArray &a);
+bool WriteUCArray (FILE *f, const UCArray &a);
+
+
+// compares the two strings in dictionary order
+int DictCompare (const UCArray &a1, const UCArray &a2);
+
+// does the first string start with the second?
+bool StartsWith(const UCArray &a1, const UCArray &a2);
+// does the first string start with the second, ignoring case?
+bool StartsWithCasefold(const UCArray &a1, const UCArray &a2);
+
+struct LTUCArray {
+  bool operator()(const UCArray &a1, const UCArray &a2) const
+    { return (a1 < a2); }
+};
+
+struct DictLTUCArray {
+  bool operator() (const UCArray &a1, const UCArray &a2) const
+    { return (DictCompare (a1, a2) < 0); }
+};
+
+
+typedef set<UCArray, LTUCArray> UCArraySet;
+typedef vector<UCArray> UCArrayVector;
+
+unsigned long PrefixLen (const UCArray &a1, const UCArray &a2);
+
+// prev == NULL if no previous string
+bool WritePreSufStr (FILE *f, const UCArray *prev, const UCArray &a);
+bool ReadPreSufStr (FILE *f, UCArray &a); // a also used for prev
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/WIN32.MAK
===================================================================
--- /main/tags/3.01/mgpp/text/WIN32.MAK	(revision 21112)
+++ /main/tags/3.01/mgpp/text/WIN32.MAK	(revision 21112)
@@ -0,0 +1,191 @@
+###########################################################################
+#
+# win32 makefile -- mgpp\lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+MGPPHOME = ..
+GSDL3HOME = ..\..\..
+AR = lib
+CC = cl
+CPPFLAGS = -GX
+LINK = cl
+
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -DSHORT_SUFFIX -D__STDC__ -D__GNU_LIBRARY__ -D__MSDOS__
+INCLUDES = -I"." -I"$(MGPPHOME)" -I"$(MGPPHOME)\lib"
+LIBS = $(MGPPHOME)\lib\libmgpp.lib
+
+COMPILE = $(CC) -c  $(CPPFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .cpp .obj
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+e = .exe
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h \
+  GSDLQueryLex.h	GSDLQueryParser.h
+
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mgpp_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mgpp_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mgpp_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mgpp_stem_idx.cpp        \
+  IndexData.cpp            mgpp_compression_dict.cpp  mgpp_weights_build.cpp   \
+  TagInfo.cpp              mgpp_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mgpp_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mgpp_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mgpp_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp         \
+  GSDLQueryLex.cpp	   GSDLQueryParser.cpp      Queryer.cpp
+
+
+EXEC          = mgpp_passes$e mgpp_decompress_text$e mgpp_perf_hash_build$e \
+                mgpp_invf_dump$e mgpp_weights_build$e mgpp_invf_dict$e \
+		mgpp_stem_idx$e mgpp_invf_dict_dump$e mgpp_compression_dict$e \
+		mgpp_fast_comp_dict$e QueryTester$e Queryer$e
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : $(EXEC) libtextin.lib libmgpppass.lib
+
+
+PASSES_OBJS = \
+	mgpp_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mgpp_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               GSDLQueryParser$o GSDLQueryLex$o words$o
+
+Queryer$e: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester$e: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mgpp_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mgpp_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mgpp_compression_dict$o mg_files$o locallib$o
+
+mgpp_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mgpp_fast_comp_dict$o mg_files$o locallib$o
+
+mgpp_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mgpp_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mgpp_invf_dict$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mgpp_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mgpp_invf_dict_dump$e: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mgpp_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mgpp_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mgpp_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mgpp_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mgpp_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mgpp_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+
+LIB_OBJS = mg_files$o  mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           GSDLQueryParser$o GSDLQueryLex$o words$o
+
+libtextin.lib: $(LIB_OBJS)
+        if exist libtextin.lib del libtextin.lib
+        $(AR) /out:libtextin.lib $(LIB_OBJS)
+
+JNI_LIB_OBJS = \
+	mgpp_passes_4jni$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+libmgpppass.lib: $(JNI_LIB_OBJS)
+	if exist libmgpppass.lib del libmgpppass.lib
+	$(AR) /out:libmgpppass.lib $(JNI_LIB_OBJS)
+
+install: 
+	for %%i in ($(EXEC)) do \
+	  copy %%i "$(GSDL3HOME)\bin"
+ 
+clean:
+	del *$o 
+        del libtextin.lib libmgpppass.lib
+
Index: /main/tags/3.01/mgpp/text/Weights.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/Weights.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Weights.cpp	(revision 21112)
@@ -0,0 +1,126 @@
+/**************************************************************************
+ *
+ * Weights.cpp -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "Weights.h"
+#include "sysfuncs.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+
+ApproxWeightsData::ApproxWeightsData () {
+  weightBuf = NULL;
+  table = NULL;
+  Free();
+}
+
+ApproxWeightsData::~ApproxWeightsData () {
+  Free ();
+}
+
+void ApproxWeightsData::Free () {
+  bits = 0;
+  mask = 0;
+  L = 0.0;
+  B = 0.0;
+  numLevelDocs = 0;
+  if (weightBuf != NULL) {
+    delete [] weightBuf;
+    weightBuf = NULL;
+  }
+  if (table != NULL) {
+    delete [] table;
+    table = NULL;
+  }
+}
+
+bool ApproxWeightsData::Read (FILE *approxWeightsFile, unsigned long diskPtr,
+			      unsigned long _numLevelDocs) {
+  Free ();
+  
+  // go to the appropriate place in the weights file
+  fseek (approxWeightsFile, diskPtr, SEEK_SET);
+
+  // read in approx weights parameters
+  ReadUC (approxWeightsFile, bits);
+  ReadD (approxWeightsFile, L);
+  ReadD (approxWeightsFile, B);
+
+  mask = (bits == 32) ? 0xffffffff : (1 << bits) - 1;
+  numLevelDocs = _numLevelDocs;
+
+  // allocate buffer for the weights
+  unsigned long arrSize = (bits*numLevelDocs+sizeof(unsigned long)-1) / 32;
+  weightBuf = new unsigned long[arrSize];
+
+  // read in the weights
+  unsigned long i;
+  for (i=0; i<arrSize; ++i) {
+    ReadUL (approxWeightsFile, weightBuf[i]);
+  }
+
+  // create precomputed table of values if it will be small enough
+  // (if not small enough table == NULL
+  if (bits <= 12) {
+    unsigned long i, tableSize = (1 << bits);
+    table = new float[tableSize];
+    table[0] = L;
+    for (i = 1; i < tableSize; ++i)
+      table[i] = table[i - 1] * B;
+  }
+
+  return true;
+}
+
+float ApproxWeightsData::GetLowerApproxDocWeight (unsigned long levelDocNum) {
+  // sanity check
+  if (bits == 0 || weightBuf == NULL) return 1.0;
+
+  register unsigned long c, pos;
+  register unsigned long *dw;
+
+  // get the compressed version of the weight
+  pos = levelDocNum * bits;
+  dw = &weightBuf[pos / MAXBITS];
+  pos &= (MAXBITS - 1);
+  c = *dw >> pos;
+  if (pos + bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - pos);
+  c &= mask;
+
+  // decompress the weight
+  if (table != NULL) return table[c];
+  return (L * pow (B, (double) c));
+}
+
+
+
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum) {
+  // go to the appropriate place in the weights file
+  fseek (exactWeightsFile, diskPtr+sizeof(float)*levelDocNum, SEEK_SET);
+
+  // read in the exact weight
+  float weight;
+  ReadF (exactWeightsFile, weight);
+  
+  return weight;
+}
Index: /main/tags/3.01/mgpp/text/Weights.h
===================================================================
--- /main/tags/3.01/mgpp/text/Weights.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/Weights.h	(revision 21112)
@@ -0,0 +1,56 @@
+/**************************************************************************
+ *
+ * Weights.h -- Dealing with document weights
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef WEIGHTS_H
+#define WEIGHTS_H
+
+#include <stdio.h>
+
+
+class ApproxWeightsData {
+public:
+  unsigned char bits;
+  unsigned long mask;
+  double L;
+  double B;
+  unsigned long numLevelDocs;
+  unsigned long *weightBuf;
+  float *table;   // precomputed translation table
+  
+  ApproxWeightsData ();
+  ~ApproxWeightsData ();
+
+  void Free (); // frees any allocated memory
+  
+  bool Read (FILE *approxWeightsFile, unsigned long diskPtr,
+	     unsigned long _numLevelDocs);
+
+  float GetLowerApproxDocWeight (unsigned long levelDocNum);
+};
+
+
+// diskPtr points to the start of the exact document weights
+// for this level
+float GetExactDocWeight (FILE *exactWeightsFile, unsigned long diskPtr,
+			 unsigned long levelDocNum);
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/WordData.h
===================================================================
--- /main/tags/3.01/mgpp/text/WordData.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/WordData.h	(revision 21112)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * WordData.h -- 
+ * Copyright (C) 1999  NZDL
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+class WordData
+{
+
+ public:
+  unsigned long docCount;
+  unsigned long wordCount;
+
+  inline void read(FILE *f)
+    {
+      fread((char *) &docCount, sizeof(docCount), 1, f);
+      fread((char *) &wordCount, sizeof(wordCount), 1, f);
+
+      NTOHUL(docCount);
+      NTOHUL(wordCount);
+    }
+
+  inline unsigned long documents()
+    {
+      return docCount;
+    }
+
+  inline int words()
+    {
+      return wordCount;
+    }
+};
Index: /main/tags/3.01/mgpp/text/build.h
===================================================================
--- /main/tags/3.01/mgpp/text/build.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/build.h	(revision 21112)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#include "TextEl.h"
+#include "TagInfo.h"
+
+
+int init_special (const TagInfo &tagInfo, char *file_name);
+int init_text_1 (const TagInfo &tagInfo, char *file_name);
+int init_text_2 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_2 (const TagInfo &tagInfo, char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_2 (const TagInfo &tagInfo, const TextElArray &doc);
+/*
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (const TagInfo &tagInfo, char *filename);
+int done_text_1 (const TagInfo &tagInfo, char *filename);
+int done_text_2 (const TagInfo &tagInfo, char *filename);
+int done_ivf_1 (const TagInfo &tagInfo, char *filename);
+int done_ivf_2 (const TagInfo &tagInfo, char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/comp_dict.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/comp_dict.cpp	(revision 21112)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * comp_dict.cpp -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = (dict_hash_table *) Xmalloc (sizeof (dict_hash_table) +
+		                            (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = (u_char **) Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  memset (ht->table, '\0', ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = (u_char *) Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; ++i, ++words)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register unsigned long tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+inline int LoadWords(FILE *dict, int which)
+{
+  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+    return 1;
+  
+  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+    return 1;
+
+  return 0;
+}
+
+inline int LoadHuffmanCodes(FILE *dict, int which)
+{
+  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 1;
+  if (!(char_codes[which] =
+	Generate_Huffman_Codes (&char_huff[which], NULL)))
+    return 2;
+  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 3;
+  if (!(lens_codes[which] =
+	Generate_Huffman_Codes (&lens_huff[which], NULL)))
+    return 4;
+
+  return 0;
+}
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (LoadWords(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (LoadHuffmanCodes(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (LoadHuffmanCodes(dict, which) != 0)
+		goto error;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /main/tags/3.01/mgpp/text/comp_dict.h
===================================================================
--- /main/tags/3.01/mgpp/text/comp_dict.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/comp_dict.h	(revision 21112)
@@ -0,0 +1,48 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define HASH_RATIO 2
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /main/tags/3.01/mgpp/text/hash.h
===================================================================
--- /main/tags/3.01/mgpp/text/hash.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/hash.h	(revision 21112)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_HASH
+#define H_HASH
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; --len)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/invf.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/invf.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/invf.cpp	(revision 21112)
@@ -0,0 +1,914 @@
+/**************************************************************************
+ *
+ * invf.cpp -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "invf.h"
+#include "UCArray.h"
+
+
+invf_dict_header::invf_dict_header () {
+  Clear();
+}
+
+invf_dict_header::~invf_dict_header () {
+}
+
+void invf_dict_header::Clear() {
+  lookback = 0;
+  word_dict_start = 0;
+  word_dict_size = 0;
+  tag_dict_start = 0;
+  tag_dict_size = 0;
+  num_docs = 0;
+  num_frags = 0;
+  num_words = 0;
+  total_bytes = 0;
+  index_string_bytes = 0;
+  num_levels = 0;
+}
+
+bool invf_dict_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, word_dict_start) &&
+	  ReadUL (f, word_dict_size) &&
+	  ReadUL (f, tag_dict_start) &&
+	  ReadUL (f, tag_dict_size) &&
+	  ReadUL (f, num_docs) &&
+	  ReadUL (f, num_frags) &&
+	  ReadUL (f, num_words) &&
+	  ReadUL (f, total_bytes) &&
+	  ReadUL (f, index_string_bytes) &&
+	  ReadUL (f, num_levels));
+}
+
+bool invf_dict_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, word_dict_start) &&
+	  WriteUL (f, word_dict_size) &&
+	  WriteUL (f, tag_dict_start) &&
+	  WriteUL (f, tag_dict_size) &&
+	  WriteUL (f, num_docs) &&
+	  WriteUL (f, num_frags) &&
+	  WriteUL (f, num_words) &&
+	  WriteUL (f, total_bytes) &&
+	  WriteUL (f, index_string_bytes) &&
+	  WriteUL (f, num_levels));
+}
+
+
+
+void dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+}
+
+bool dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq));
+}
+
+bool dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq));
+}
+
+
+void word_dict_el::Clear () {
+  dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_dict_el::~word_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+  
+bool word_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!dict_el::Read (f)) return false;
+
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_dict_el::Write (FILE *f, const UCArray *lastEl,
+			  unsigned long numLevels) const {
+  if (!dict_el::Write (f, lastEl)) return false;
+
+  if (levelFreqs == NULL) return false;
+
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+block_dict_header::block_dict_header () {
+  Clear();
+}
+
+void block_dict_header::Clear () {
+  invf_dict_header::Clear();
+
+  entries_per_wblk = 0;
+  num_wblks = 0;
+  max_wblk_size = 0;
+  wblk_start = 0;
+  wblk_idx_start = 0;
+  
+  entries_per_tblk = 0;
+  num_tblks = 0;
+  max_tblk_size = 0;
+  tblk_start = 0;
+  tblk_idx_start = 0;
+}
+
+bool block_dict_header::Read (FILE *f) {
+  return (invf_dict_header::Read (f) &&
+
+	  ReadUL (f, entries_per_wblk) &&
+	  ReadUL (f, num_wblks) &&
+	  ReadUL (f, max_wblk_size) &&
+	  ReadUL (f, wblk_start) &&
+	  ReadUL (f, wblk_idx_start) &&
+  
+	  ReadUL (f, entries_per_tblk) &&
+	  ReadUL (f, num_tblks) &&
+	  ReadUL (f, max_tblk_size) &&
+	  ReadUL (f, tblk_start) &&
+	  ReadUL (f, tblk_idx_start));
+}
+
+bool block_dict_header::Write (FILE *f) const {
+  return (invf_dict_header::Write (f) &&
+
+	  WriteUL (f, entries_per_wblk) &&
+	  WriteUL (f, num_wblks) &&
+	  WriteUL (f, max_wblk_size) &&
+	  WriteUL (f, wblk_start) &&
+	  WriteUL (f, wblk_idx_start) &&
+  
+	  WriteUL (f, entries_per_tblk) &&
+	  WriteUL (f, num_tblks) &&
+	  WriteUL (f, max_tblk_size) &&
+	  WriteUL (f, tblk_start) &&
+	  WriteUL (f, tblk_idx_start));
+}
+
+
+
+
+
+void block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+  invf_ptr = 0;
+}
+
+bool block_dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq) &&
+	  ReadUL (f, invf_ptr));
+}
+
+bool block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq) &&
+	  WriteUL (f, invf_ptr));
+}
+
+
+
+
+
+void word_block_dict_el::Clear () {
+  block_dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_block_dict_el::~word_block_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_block_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+
+bool word_block_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!block_dict_el::Read (f)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_block_dict_el::Write (FILE *f, const UCArray *lastEl,
+				unsigned long numLevels) const {
+  if (!block_dict_el::Write (f, lastEl)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; ++i) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+  
+  return true;
+}
+
+
+
+
+block_idx_info::block_idx_info () {
+  Clear ();
+}
+
+void block_idx_info::Clear () {
+  el.erase (el.begin(), el.end());
+  block_ptr = 0;
+}
+
+bool block_idx_info::Read (FILE *f) {
+  return (ReadUCArray (f, el) &&
+	  ReadUL (f, block_ptr));
+}
+
+bool block_idx_info::Write (FILE *f) const {
+  return (WriteUCArray (f, el) &&
+	  WriteUL (f, block_ptr));
+}
+
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx) {
+  blockIdx.erase (blockIdx.begin(), blockIdx.end());
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  block_idx_info bi;
+  while (arraySize > 0) {
+    if (!bi.Read (f)) return false;
+    blockIdx.push_back (bi);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx) {
+  // write out the array size
+  if (!WriteVarLenUL (f, blockIdx.size())) return false;
+
+  block_idx::const_iterator here = blockIdx.begin();
+  block_idx::const_iterator end = blockIdx.end();
+  while (here != end) {
+    if (!(*here).Write (f)) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+stem_idx_header::stem_idx_header () {
+  Clear ();
+}
+
+void stem_idx_header::Clear () {
+  lookback = 0;
+  dict_size = 0;
+
+  entries_per_block = 0;
+  num_blocks = 0;
+  max_block_size = 0;
+  blocks_start = 0;
+  block_idx_start = 0;
+  
+  stemmer_num = 0;
+  stem_method = 0;
+}
+
+bool stem_idx_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, dict_size) &&
+	  
+	  ReadUL (f, entries_per_block) &&
+	  ReadUL (f, num_blocks) &&
+	  ReadUL (f, max_block_size) &&
+	  ReadUL (f, blocks_start) &&
+	  ReadUL (f, block_idx_start) &&
+	  
+	  ReadUL (f, stemmer_num) &&
+	  ReadUL (f, stem_method));
+}
+
+bool stem_idx_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, dict_size) &&
+	  
+	  WriteUL (f, entries_per_block) &&
+	  WriteUL (f, num_blocks) &&
+	  WriteUL (f, max_block_size) &&
+	  WriteUL (f, blocks_start) &&
+	  WriteUL (f, block_idx_start) &&
+	  
+	  WriteUL (f, stemmer_num) &&
+	  WriteUL (f, stem_method));
+}
+
+
+
+stem_block_dict_el::stem_block_dict_el () {
+  Clear ();
+}
+
+void stem_block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  equivWords.erase (equivWords.begin(), equivWords.end());
+}
+
+bool stem_block_dict_el::Read (FILE *f) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (!ReadPreSufStr (f, el)) return false;
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  unsigned long wordNum;
+  while (arraySize > 0) {
+    if (!ReadUL (f, wordNum)) return false;
+    equivWords.push_back (wordNum);
+    
+    --arraySize;
+  }
+  
+  return true;
+}
+
+bool stem_block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  if (!WritePreSufStr (f, lastEl, el)) return false;
+
+  // write out the array size
+  if (!WriteVarLenUL (f, equivWords.size())) return false;
+
+  vector<unsigned long>::const_iterator here = equivWords.begin();
+  vector<unsigned long>::const_iterator end = equivWords.end();
+  while (here != end) {
+    if (!WriteUL (f, (*here))) return false;
+    ++here;
+  }
+
+  return true;
+}
+
+
+
+
+invf_file_header::invf_file_header () {
+  Clear ();
+}
+
+void invf_file_header::Clear () {
+  no_of_words = 0;
+  no_of_tags = 0;
+  skip_mode = SKIP_MODE_NO_SKIPS;
+  word_level_index = 0;
+  
+  int i;
+  for (i=0; i<16; ++i) params[i] = 0;
+}
+  
+bool invf_file_header::Read (FILE *f) {
+  if (!ReadUL (f, no_of_words) ||
+      !ReadUL (f, no_of_tags) ||
+      !ReadUL (f, skip_mode) ||
+      !ReadUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!ReadUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+bool invf_file_header::Write (FILE *f) const {
+  if (!WriteUL (f, no_of_words) ||
+      !WriteUL (f, no_of_tags) ||
+      !WriteUL (f, skip_mode) ||
+      !WriteUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; ++i) {
+    if (!WriteUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+
+
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  // make sure the element number isn't out of range
+  if (elNum > bIdx.size()*entriesPerBlock) return false;
+
+  blockIdxNum = elNum / entriesPerBlock;
+  blockStartElNum = blockIdxNum * entriesPerBlock;
+  
+  return true;
+}
+
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  unsigned long begin = 0;
+  unsigned long bIdxEnd = bIdx.size();
+  unsigned long end = bIdxEnd;
+  unsigned long mid;
+  while (begin < end) {
+    mid = (begin+end)/2;
+    if (DictCompare (el, bIdx[mid].el) < 0) {
+      end = mid;
+    } else if (mid+1 < bIdxEnd &&
+	       DictCompare (el, bIdx[mid+1].el) >= 0) {
+      begin = mid+1;
+    } else {
+      blockIdxNum = mid;
+      blockStartElNum = blockIdxNum * entriesPerBlock;
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+
+
+// use the block dictionary functions for tag entries, and word block dict
+// functions for word entries.
+
+
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    ++elNum;
+  }
+
+  return false;
+}
+
+// ------------------------------------------
+// functions for partial term matching
+// ie find all words that start with xxx
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold) {
+
+  UCArrayClear (dictEl.el);
+  elNumList.erase (elNumList.begin(), elNumList.end());
+  unsigned long elNum;
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  // will this work??
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum)) {
+    return false;
+  }
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+  
+  bool still_looking = true;
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // test each element
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    if (StartsWithCasefold(dictEl.el, el)) {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+      }
+      still_looking=false; // we have found one now, so stop the next time we don't have a match
+    } else if (!still_looking) {
+      // we have found a match previously, and now this doesn't match
+      return true;
+    } // else keep looking
+    
+    ++elNum;
+  }
+  // if we get here, we are either still searching for the first
+  // case, or we are collecting terms.
+  if (still_looking) {
+    //we haven't found a match yet, just check the next element,
+    dictEl.Read (dictFile, numLevels);
+    if (!StartsWithCasefold(dictEl.el, el)) {
+      // the first element of the next block is not a match, so there are no matches
+      return false;
+    } else {
+      if (casefold || StartsWith(dictEl.el, el)) {
+	elNumList.push_back(elNum);
+	++elNum;
+      } 
+
+    }
+  }
+  // just keep accumulating matches until there are no more
+  dictEl.Read (dictFile, numLevels);
+  while (StartsWithCasefold(dictEl.el, el)) {
+    if (casefold || StartsWith(dictEl.el, el)) {
+      elNumList.push_back(elNum);
+    }
+    ++elNum;
+    dictEl.Read (dictFile, numLevels);
+  }
+  return true;
+  
+}
+
+
+//----------------------------------------------------------------
+// functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+				   const block_idx &bIdx,
+				   unsigned long entriesPerBlock,
+				   unsigned long dictSize,
+				   unsigned long numLevels,
+				   const UCArray &el,
+				   word_block_dict_el &dictEl,
+				   unsigned long &elNum) {
+
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (el, dictEl.el); // look for the first word that is
+                                     // greater or equal to the el
+    if (res <= 0) {
+      return true; // found one
+   } 
+    
+    ++elNum;
+  }
+  // it must be the last term
+  return true;
+
+
+}
+
+
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms) {
+
+  word_block_dict_el dictEl;  
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  terms.push_back(dictEl.el);
+  while (curElNum <= lastElNum ) {
+    dictEl.Read(dictFile, numLevels);
+    terms.push_back(dictEl.el);
+    ++curElNum;
+  }
+  
+
+  return true;
+}
+
+// NOte: before each addition of dictEl to the array, the level freqs array
+// is deleted, as this was causing problems - generating a seg fault, I think if
+// the vector had to be reallocated or something.
+// setNumLevels has to be called each time before a read, now, to set up the level
+//freqs array. this is necessary.
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms) {
+
+  word_block_dict_el dictEl;
+  dictEl.SetNumLevels (numLevels);
+  UCArrayClear(dictEl.el);
+
+  block_dict_el elem;
+  terms.erase(terms.begin(), terms.end());
+
+  if (elNum >= dictSize) return false;
+  
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  unsigned long lastElNum = elNum + numWords - 1;
+  if (lastElNum > dictSize) lastElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  // get the first term
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  dictEl.levelFreqs = NULL;
+  terms.push_back(dictEl);
+
+  while (curElNum <= lastElNum ) { 
+    dictEl.SetNumLevels(numLevels);
+    dictEl.Read(dictFile, numLevels);
+    dictEl.levelFreqs = NULL;
+    terms.push_back(dictEl);
+    ++curElNum;
+  }
+
+  return true;
+}
+
+
+
+
+
+
+
+
+
Index: /main/tags/3.01/mgpp/text/invf.h
===================================================================
--- /main/tags/3.01/mgpp/text/invf.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/invf.h	(revision 21112)
@@ -0,0 +1,352 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF
+#define H_INVF
+
+#include <stdio.h>
+
+#include "UCArray.h"
+
+// NOTE: This does not include the magic number
+// header info for .invf.dict file
+struct invf_dict_header {
+  unsigned long lookback;
+  unsigned long word_dict_start;
+  unsigned long word_dict_size;
+  unsigned long tag_dict_start;
+  unsigned long tag_dict_size;
+  unsigned long num_docs;
+  unsigned long num_frags;
+  unsigned long num_words;
+  unsigned long total_bytes;
+  unsigned long index_string_bytes;
+  unsigned long num_levels;
+
+  invf_dict_header ();
+  virtual ~invf_dict_header ();
+  virtual void Clear();
+
+  virtual bool Read (FILE *f);
+  virtual bool Write (FILE *f) const;
+};
+
+
+struct dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur;
+  unsigned long freq;
+
+  virtual void Clear ();
+  dict_el () { Clear (); }
+  virtual ~dict_el () { }
+
+  // Read assumes that the last word is in el
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+struct word_dict_el : public dict_el {
+  unsigned long *levelFreqs;
+
+  void Clear ();
+  word_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+
+// wblk = word block
+// tblk = tag block
+// this version of the blocked dictionary uses a fixed number
+// of entries per block, not a fixed block size
+// info for .invf.dict.blocked file
+// blocked dict has a heap of blocks, some for words, some for tags
+// and an index into each set of blocks. The index has pointers to
+// the first entry in each block. Can do a binary search on the index
+// to find out which block an elemnet is in
+struct block_dict_header : public invf_dict_header {
+  // note: word_dict_start and tag_dict_start are undefined
+  // for blocked dictionaries
+
+  unsigned long entries_per_wblk; // word blocks
+  unsigned long num_wblks;
+  unsigned long max_wblk_size;
+  unsigned long wblk_start;
+  unsigned long wblk_idx_start;
+  
+  unsigned long entries_per_tblk; // tag blocks
+  unsigned long num_tblks;
+  unsigned long max_tblk_size;
+  unsigned long tblk_start;
+  unsigned long tblk_idx_start;
+  
+  block_dict_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct block_dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur; // # entries in invf file - if have a
+  // word level index, this is the same as freq, otherwise, its the number
+  // of fragments containing this word
+  unsigned long freq; // # of times this word occurs
+  unsigned long invf_ptr; // pointer into inverted file
+
+  virtual void Clear ();
+  block_dict_el () { Clear (); }
+  virtual ~block_dict_el () { }
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+struct word_block_dict_el : public block_dict_el {
+  unsigned long *levelFreqs; // freq of the word at each level
+
+  void Clear ();
+  word_block_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_block_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+typedef vector<word_block_dict_el> word_block_dict_el_array;
+
+struct block_idx_info {
+  UCArray el;
+  unsigned long block_ptr;
+
+  block_idx_info ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// used for an index into the word and tag blocks
+typedef vector<block_idx_info> block_idx;
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx);
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx);
+
+
+
+struct stem_idx_header {
+  unsigned long lookback;
+  unsigned long dict_size;
+
+  unsigned long entries_per_block;
+  unsigned long num_blocks;
+  unsigned long max_block_size;
+  unsigned long blocks_start;
+  unsigned long block_idx_start;
+  
+  unsigned long stemmer_num;
+  unsigned long stem_method;
+
+  stem_idx_header ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+struct stem_block_dict_el {
+  UCArray el; // word or tag
+  vector<unsigned long> equivWords;
+  
+  stem_block_dict_el ();
+  void Clear ();
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+
+#define SKIP_MODE_NO_SKIPS 0
+
+// invf file - has a list of frags for each word, but the word is not 
+//  stored in the invf file - the dictionaries store the words, along
+// with num entries, and a pointer into invf file
+struct invf_file_header {
+  unsigned long no_of_words;
+  unsigned long no_of_tags;
+  unsigned long skip_mode;
+  unsigned long word_level_index; // 1 if word level index
+  unsigned long params[16];
+
+  invf_file_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+
+
+
+
+// the search functions returns true if a block that could
+// satisfy the request is found. these functions assume that
+// the block index is sorted by DictCompare (or DictLTUCArray)
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum);
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum);
+
+
+// The next six functions use SearchElNum and SearchEl
+// for a particular type of dictionary (Block, WordBlock, or StemBlock)
+// and then look up the entry
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl);
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum);
+
+// assumes the numLevels has been set for dictEl
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl);
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl);
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+//-----------------------------------------------------
+// New functions for partial matching
+// returns a list of word numbers from the main block index whose
+// words start with el.
+bool PartialMatchSearchWordBlockDictEl (FILE *dictFile,
+				 const block_idx &bIdx,
+				 unsigned long entriesPerBlock,
+				 unsigned long dictSize,
+				 unsigned long numLevels,
+				 const UCArray &el,
+				 word_block_dict_el &dictEl,
+				 vector<unsigned long> &elNumList,
+				 bool casefold); 
+//----------------------------------------------------------
+
+// new functions for full text browse
+
+bool NearestSearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+// returns a list of word_block_dict_el, with no levelfreqs
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    word_block_dict_el_array &terms);
+				    
+// just returns a list of terms
+bool SearchWordBlockDictElNumRange (FILE *dictFile,
+				    const block_idx &bIdx,
+				    unsigned long entriesPerBlock,
+				    unsigned long dictSize,
+				    unsigned long numLevels,
+				    unsigned long elNum,
+				    unsigned long numWords,
+				    UCArrayVector &terms);
+				    
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/ivf.pass1.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/ivf.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/ivf.pass1.cpp	(revision 21112)
@@ -0,0 +1,775 @@
+/**************************************************************************
+ *
+ * ivf.pass1.cpp -- Memory efficient pass 1 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+// a fragment corresponds to the index level (word-level is the
+// minimum index level)
+
+
+// structure to determine level information
+struct LevelWorker {
+  unsigned long lastLevelDocNum;
+  unsigned long count;
+
+  LevelWorker () {
+    lastLevelDocNum = count = 0;
+  }
+};
+
+
+// note: the word is stored in the map
+struct IvfWordInfo {
+  unsigned long wordCount;      // word frequency
+  unsigned long fragCount;      // number of fragments that contain the word
+  unsigned long lastFragNum;    // last fragment to contain the word
+  unsigned long chunkWordCount; // word frequency within this chunk
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the word
+
+  LevelWorker *levels;          // level info for this word
+  
+  IvfWordInfo ();
+  ~IvfWordInfo ();
+  void Clear ();                // will delete levels
+};
+
+typedef map<UCArray, IvfWordInfo, DictLTUCArray> IvfWordInfoMap;
+typedef vector<IvfWordInfoMap::iterator> IvfWordInfoItArray;
+
+// tags don't require as much information
+struct IvfTagInfo {
+  unsigned long tagCount;       // tag frequency
+  unsigned long fragCount;      // number of fragments that contain the tag
+  unsigned long lastFragNum;    // last fragment to contain the tag
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the tag
+
+  IvfTagInfo ();
+  void Clear ();
+};
+
+typedef map<UCArray, IvfTagInfo, DictLTUCArray> IvfTagInfoMap;
+typedef vector<IvfTagInfoMap::iterator> IvfTagInfoItArray;
+
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define CHECK_DIV 1.5
+
+
+static FILE *ic;		// the invf chunk file
+static stdio_bitio_buffer icb;
+
+IvfWordInfoMap ivfWordInfo;
+IvfWordInfoItArray ivfWordInfoOccurOrder;
+IvfTagInfoMap ivfTagInfo;
+IvfTagInfoItArray ivfTagInfoOccurOrder;
+
+static unsigned long chunksWritten;
+static unsigned long maxMemNeeded;
+
+static unsigned long numDocs;
+static unsigned long numChunkDocs;
+
+static unsigned long numFrags;
+static unsigned long numChunkFrags;
+
+static unsigned long numWords;
+
+// the number of document numbers in the inverted file
+static unsigned long numChunkEntries;
+
+// next entry in the inverted file to check memory
+// requirements for the current chunk
+static unsigned long entryCheckPoint;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+
+
+IvfWordInfo::IvfWordInfo () {
+  levels = NULL;
+  Clear();
+}
+
+IvfWordInfo::~IvfWordInfo () {
+  if (levels != NULL) delete [] levels;
+}
+
+void IvfWordInfo::Clear () {
+  wordCount = 0;
+  chunkWordCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+
+  if (levels != NULL) {
+    delete [] levels;
+    levels = NULL;
+  }
+}
+
+
+IvfTagInfo::IvfTagInfo () {
+  Clear();
+}
+
+void IvfTagInfo::Clear () {
+  tagCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+}
+
+
+
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name) {
+  // set up the chunk file
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb",
+			  MAGIC_CHUNK, MG_MESSAGE)))
+    return COMPERROR;
+  fwrite ("    ", sizeof (u_long), 1, ic);	// Space for the maxmem
+  icb.attachFile (ic);
+  icb.encodeStart();
+
+  // reset global variables
+  ivfWordInfo.erase (ivfWordInfo.begin(), ivfWordInfo.end());
+  ivfWordInfoOccurOrder.erase (ivfWordInfoOccurOrder.begin(), ivfWordInfoOccurOrder.end());
+  ivfTagInfo.erase (ivfTagInfo.begin(), ivfTagInfo.end());
+  ivfTagInfoOccurOrder.erase (ivfTagInfoOccurOrder.begin(), ivfTagInfoOccurOrder.end());
+  
+  chunksWritten = 0;
+  maxMemNeeded = 0;
+ 
+  numDocs = 0;
+  numChunkDocs = 0;
+
+  numFrags = 0;
+  numChunkFrags = 0;
+
+  numWords = 0;
+
+  numChunkEntries = 0;
+  entryCheckPoint = (unsigned long) ((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+
+  // init the level information
+  ivfLevel.Clear();
+  ivfLevel.docTag = tagInfo.docTag;
+  ivfLevel.indexLevel = tagInfo.indexLevel;
+  IvfLevelInfo blankLevel;
+  UCArraySet::const_iterator levelHere = tagInfo.levelTags.begin();
+  UCArraySet::const_iterator levelEnd = tagInfo.levelTags.end();
+  while (levelHere != levelEnd) {
+    blankLevel.levelTag = *levelHere;
+    ivfLevel.levelInfo[*levelHere] = blankLevel;
+    ++levelHere;
+  }
+  
+  return COMPALLOK;
+}
+
+static void ProcessOpenTag (const TagInfo &tagInfo, const TextEl &el,
+			    bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for start of next fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    ++numFrags;
+    ++numChunkFrags;
+    inFrag = true;
+  }
+  
+  // update tag stats
+  IvfTagInfo &i = ivfTagInfo[el.tagName];
+  if (i.tagCount == 0) {
+    // new tag, add to list of iterators
+    IvfTagInfoMap::iterator iIt = ivfTagInfo.find (el.tagName);
+    ivfTagInfoOccurOrder.push_back (iIt);
+  }
+  ++i.tagCount;
+
+  // all open tags count as new tags
+  ++numChunkEntries;
+  ++i.fragCount;
+  ++i.chunkFragCount;
+  i.lastFragNum = numFrags;
+  
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    ++levelIt->second.numEntries;
+    levelIt->second.workInLevel = true;
+  }
+}
+
+static void ProcessCloseTag (const TagInfo &tagInfo, const TextEl &el,
+			     bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for end of fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    inFrag = false;
+  }
+
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    (*levelIt).second.workInLevel = false;
+  }
+}
+
+static void ProcessText (const TagInfo &tagInfo, const TextEl &el,
+			 bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // make sure this text is to be indexed
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  UCArray word;
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+  // Each token is then inserted into the set if it does
+  // not exist or has it's frequency count incremented if it does.
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexWord (textHere, textEnd, word);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (!word.empty()) {
+      ++numWords;
+
+      if (wordLevelIndex) {
+	++numFrags;
+	++numChunkFrags;
+      }
+      
+      // update word stats
+
+      IvfWordInfo &i = ivfWordInfo[word];
+      if (i.wordCount == 0) {
+	// new word
+	// add to list of iterators
+	IvfWordInfoMap::iterator iIt = ivfWordInfo.find (word);
+	ivfWordInfoOccurOrder.push_back (iIt);
+
+	// add level information array
+	if (ivfLevel.levelInfo.size() > 0) 
+	  i.levels = new LevelWorker [ivfLevel.levelInfo.size()];
+      }
+
+      ++i.wordCount;
+      ++i.chunkWordCount;
+      if (numFrags > i.lastFragNum) {
+	++numChunkEntries;
+	++i.fragCount;
+	++i.chunkFragCount;
+	i.lastFragNum = numFrags;
+      }
+
+      // update level information for this word
+      if (i.levels != NULL) {
+	IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+	IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+	LevelWorker *levelWorkerPtr = i.levels;
+	while (levelHere != levelEnd) {
+
+	  // check to make sure the level encompases this fragment
+	  if (!(*levelHere).second.workInLevel) {
+	    cerr << "Level tag <" << (*levelHere).first
+		 << "> does not encompass all fragments\n";
+	    exit (1);
+	  }
+	  
+	  if ((*levelHere).second.numEntries > (*levelWorkerPtr).lastLevelDocNum) {
+	    (*levelWorkerPtr).lastLevelDocNum = (*levelHere).second.numEntries;
+	    ++levelWorkerPtr->count;
+	  }
+	  
+	  ++levelHere;
+	  ++levelWorkerPtr;
+	}
+      }
+    }
+  }
+}
+
+
+static unsigned long MemoryRequired (bool wordLevelIndex) {
+  register unsigned long total = 0;
+
+  // add memory required for word entries
+  IvfWordInfoMap::const_iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::const_iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    register const IvfWordInfo &info = (*wordHere).second;
+    if (info.chunkFragCount > 0) {
+      total += BIO_Bblock_Bound (numChunkFrags, info.chunkFragCount);
+      if (!wordLevelIndex) {
+	total += info.chunkWordCount;
+      }
+    }
+    
+    ++wordHere;
+  }
+
+  // add memory required for tag entries
+  IvfTagInfoMap::const_iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::const_iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    register const IvfTagInfo &info = (*tagHere).second;
+    if (info.chunkFragCount > 0) {
+      // two d entries for each frag entry
+      unsigned long pTag = info.chunkFragCount*2;
+      total += BIO_Bblock_Bound (numChunkFrags+pTag, pTag);
+    }
+    
+    ++tagHere;
+  }
+  
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+/*
+static void PrintChunkInfo (unsigned long mem) {
+  cout << "Chunk Number: " << chunksWritten << "\n";
+  cout << "numChunkDocs " << numChunkDocs << "\n";
+  cout << "numChunkFrags " << numChunkFrags << "\n";
+  cout << "mem " << mem << "\n";
+  cout << "numWords " << ivfWordInfo.size() << "\n";
+  cout << "numTags " << ivfTagInfo.size() << "\n\n";
+
+  // output debug tag information in dictionary order
+  IvfTagInfoMap::iterator tagMapHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagMapEnd = ivfTagInfo.end();
+  unsigned long tagNum = 0;
+  while (tagMapHere != tagMapEnd) {
+    cout << (*tagMapHere).first << " " << tagNum
+	 << " " << (*tagMapHere).second.chunkFragCount << "\n";
+    ++tagNum;
+    ++tagMapHere;
+  }
+}
+*/
+
+static void OutputChunkInfo (unsigned long mem, bool /*wordLevelIndex*/) {
+  ++chunksWritten;
+
+  // sanity check
+  if (ivfWordInfo.size() != ivfWordInfoOccurOrder.size()) {
+    Message ("ERROR: Word information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfWordInfo.size(),
+	     (unsigned int)ivfWordInfoOccurOrder.size());
+    exit (1);
+  }
+  if (ivfTagInfo.size() != ivfTagInfoOccurOrder.size()) {
+    Message ("ERROR: Tag information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfTagInfo.size(),
+	     (unsigned int)ivfTagInfoOccurOrder.size());
+    exit (1);
+  }
+
+  icb.gamma_encode (numChunkDocs + 1, NULL);
+  icb.gamma_encode (numChunkFrags + 1, NULL);
+  icb.gamma_encode (mem + 1, NULL);
+  icb.gamma_encode (ivfWordInfo.size() + 1, NULL);
+  icb.gamma_encode (ivfTagInfo.size() + 1, NULL);
+
+  /*  PrintChunkInfo (mem);*/
+
+  // output word information in occurance order
+  IvfWordInfoItArray::iterator wordHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordEnd = ivfWordInfoOccurOrder.end();
+  while (wordHere != wordEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordHere)).second;
+    
+    icb.gamma_encode (ivfWordInfo.chunkWordCount + 1, NULL);
+    if (ivfWordInfo.chunkWordCount >= 2) {
+      icb.gamma_encode (ivfWordInfo.chunkFragCount, NULL);
+    }
+
+    ivfWordInfo.lastFragNum = 0;
+    ivfWordInfo.chunkWordCount = 0;
+    ivfWordInfo.chunkFragCount = 0;
+    
+    ++wordHere;
+  }
+
+  // output tag information in occurance order
+  IvfTagInfoItArray::iterator tagHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagEnd = ivfTagInfoOccurOrder.end();
+  while (tagHere != tagEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagHere)).second;
+
+    icb.gamma_encode (ivfTagInfo.chunkFragCount + 1, NULL);
+
+    ivfTagInfo.lastFragNum = 0;
+    ivfTagInfo.chunkFragCount = 0;
+    
+    ++tagHere;
+  }
+
+  numChunkDocs = 0;
+  numChunkFrags = 0;
+  numChunkEntries = 0;
+}
+
+
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  ++numDocs;
+  ++numChunkDocs;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE)
+      ProcessOpenTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == CloseTagE)
+      ProcessCloseTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == TextE)
+      ProcessText (tagInfo, *here, inFrag);
+    
+    ++here;
+  }
+
+  // check the amount of memory needed for this chunk
+  if (numChunkEntries >= entryCheckPoint) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    if (mem >= invf_buffer_size * CHECK_CLOSE) {
+      if (mem > maxMemNeeded) maxMemNeeded = mem;
+      OutputChunkInfo (mem, wordLevelIndex);
+      entryCheckPoint = (unsigned long)
+	((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+      
+    } else {
+      entryCheckPoint = (unsigned long)
+	(entryCheckPoint * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	 entryCheckPoint);
+      if (entryCheckPoint <= numChunkEntries)
+	entryCheckPoint = numChunkEntries + 1;
+    }
+  }
+  
+  return COMPALLOK;
+}
+
+
+static void CalcInvfDictSize (unsigned long &totalBytes,
+			      unsigned long &indexStringBytes) {
+  totalBytes = 0;	// The sum of the length of all words, including 
+			// the length byte
+  indexStringBytes = 0; // The amount of space required to store the 
+			// words in the diction, this takes into account 
+			// the prefixes
+  const UCArray *lastWord = NULL;
+  
+  // calculate size of word information
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    unsigned long wordSize = (*wordHere).first.size();
+    totalBytes += wordSize + 1;
+    indexStringBytes += wordSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*wordHere).first);
+    lastWord = &((*wordHere).first);
+    
+    ++wordHere;
+  }
+
+  // calculate size of tag information
+  lastWord = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    unsigned long tagSize = (*tagHere).first.size();
+    totalBytes += tagSize + 1;
+    indexStringBytes += tagSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*tagHere).first);
+    lastWord = &((*tagHere).first);
+    
+    ++tagHere;
+  }
+}
+
+
+//      OutputInvfDict ():
+//                writes out the stemmed dictionary file
+//                in the following format 
+//                        lookback value (int)
+//                        totalbytes value (int)
+//                        indexstringbytes (int)
+//                        for each word 
+//                          wordlen (4 bits)
+//                          prefix match (4 bits)
+//                          word (wordlen bytes)
+//                          word frequency (int)
+//                          word count (int)
+//
+//        Accesses outside variables:     
+//
+//        Return value...:                
+static void OutputInvfDict (char *filename) {
+  // create the dictionary header
+  invf_dict_header idh;
+  idh.word_dict_size = ivfWordInfo.size();
+  idh.tag_dict_size = ivfTagInfo.size();
+  idh.num_docs = numDocs;
+  idh.num_frags = numFrags;
+  idh.num_words = numWords;
+  idh.num_levels = ivfLevel.levelInfo.size();
+  CalcInvfDictSize (idh.total_bytes, idh.index_string_bytes);
+
+  // create the inverted dictionary file
+  FILE *sp;
+  if (!(sp = create_file (filename, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))
+    return;
+
+  // write out the dictionary header
+  if (!idh.Write (sp)) { fclose (sp); return; }
+
+  // remember where the word dictionary starts
+  idh.word_dict_start = ftell (sp);
+  
+  // output the word dictionary
+  const UCArray *lastWord = NULL;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisWord = (*wordHere).first;
+    WritePreSufStr (sp, lastWord, thisWord);
+
+    // output the number of fragments the word appeared in and the
+    // number of times the word appeared
+    WriteUL (sp, (*wordHere).second.fragCount);
+    WriteUL (sp, (*wordHere).second.wordCount);
+
+    // output frequency information for each level
+    // note that we are expecting every word to have
+    // level information
+    LevelWorker *lwHere = (*wordHere).second.levels;
+    LevelWorker *lwEnd = lwHere + idh.num_levels;
+    while (lwHere != lwEnd) {
+      WriteUL (sp, (*lwHere).count);
+      ++lwHere;
+    }
+    
+    lastWord = &thisWord;
+    ++wordHere;
+  }
+
+  // remember where the tag dictionary starts
+  idh.tag_dict_start = ftell (sp);
+  
+  // output the tag dictionary
+  const UCArray *lastTag = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisTag = (*tagHere).first;
+    WritePreSufStr (sp, lastTag, thisTag);
+
+    // output the number of fragments the tag appeared in and the
+    // number of times the tag appeared
+    WriteUL (sp, (*tagHere).second.fragCount);
+    WriteUL (sp, (*tagHere).second.tagCount);
+    
+    lastTag = &thisTag;
+    ++tagHere;
+  }
+
+  // write out the updated header
+  fseek (sp, sizeof (u_long), SEEK_SET);
+  if (!idh.Write (sp)) { fclose (sp); return; }
+  
+  fclose (sp);
+}
+
+static void OutputLevelFile (char *filename) {
+  // create the level file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_LEVEL_SUFFIX, "wb", MAGIC_INVF_LEVELS,
+			 MG_MESSAGE)))
+    return;
+
+  // write out the information
+  ivfLevel.Write (f);
+  
+  // close the file
+  fclose (f);
+}
+
+static void OutputTransFile (char *filename) {
+  // save the word number (in lastFragNum :-/)
+  int i = 0;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    (*wordHere).second.lastFragNum = i;
+    ++i; ++wordHere;
+  }
+
+  // save the tag number (in lastFragNum :-/)
+  i = 0;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    (*tagHere).second.lastFragNum = i;
+    ++i; ++tagHere;
+  }
+  
+  // create the translation file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))
+    return;
+
+  stdio_bitio_buffer buffer(f);
+  buffer.encodeStart();
+
+  // write out the word translation table
+  unsigned long wordDictSize = ivfWordInfoOccurOrder.size();
+  IvfWordInfoItArray::iterator wordItHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordItEnd = ivfWordInfoOccurOrder.end();
+  unsigned long oN = 0;
+  while (wordItHere != wordItEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordItHere)).second;
+    buffer.binary_encode (ivfWordInfo.lastFragNum + 1, wordDictSize + 1, NULL);
+    ++oN;
+    ++wordItHere;
+  }
+
+  // write out the tag translation table
+  unsigned long tagDictSize = ivfTagInfoOccurOrder.size();
+  IvfTagInfoItArray::iterator tagItHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagItEnd = ivfTagInfoOccurOrder.end();
+  while (tagItHere != tagItEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagItHere)).second;
+    buffer.binary_encode (ivfTagInfo.lastFragNum + 1, tagDictSize + 1, NULL);
+    ++oN;
+    ++tagItHere;
+  }
+
+  // finish encoding and close the file
+  buffer.encodeDone();
+  fclose (f);
+}
+
+#ifndef SILENT
+static void PrintStats () {
+  Message ("Inverted buffer size:          %8u bytes\n", invf_buffer_size);
+  Message ("Max memory needed for 1 chunk: %8u bytes\n", maxMemNeeded);
+
+  Message ("Number of chunks written:      %8u\n", chunksWritten);
+  Message ("Number of documents:           %8u\n", numDocs);
+  Message ("Number of fragments:           %8u\n", numFrags);
+  Message ("Number of words:               %8u\n", numWords);
+
+  Message ("Size of word dictionary:       %8u\n", ivfWordInfo.size());
+  Message ("Size of tag dictionary:        %8u\n", ivfTagInfo.size());
+}
+#endif
+
+int done_ivf_1 (const TagInfo &tagInfo, char * filename) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+
+  // output the last chunk
+  if (numChunkDocs > 0) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    OutputChunkInfo (mem, wordLevelIndex);
+    if (mem > maxMemNeeded) maxMemNeeded = mem;
+  }
+  
+  // write out and encoded 1 to say there are no more chunks
+  icb.gamma_encode (1, NULL);
+  icb.encodeDone ();
+
+  // write out the maximum memory required and close the file
+  fseek (ic, sizeof (long), 0);
+  WriteUL (ic, maxMemNeeded);
+  fclose (ic);
+
+  // output the inverted dictionary
+  OutputInvfDict (filename);
+
+  // write out the translation file
+  OutputTransFile (filename);
+
+  // output the level information
+  OutputLevelFile (filename);
+  
+  // output statistics
+#ifndef SILENT
+  PrintStats ();
+#endif
+  
+  msg_prefix = temp_str;
+
+  return COMPALLOK;
+}
+
Index: /main/tags/3.01/mgpp/text/ivf.pass2.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/ivf.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/ivf.pass2.cpp	(revision 21112)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * ivf.pass2.cpp -- Memory efficient pass 2 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include <stdio.h>
+
+#if defined __WIN32__
+# include <process.h>
+# include <io.h>
+# define getpid _getpid
+# define unlink _unlink
+#else
+# include <unistd.h>
+# include "non_ansi.h"
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "bitio_m_random.h"
+#include "bitio_m_stdio.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+#include "perf_hash.h"
+#include "string.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+#ifdef USE_LONG_LONG
+#define SEEK_X seek_LL
+#define TELL_X tell_LL
+#else
+#define SEEK_X seek
+#define TELL_X tell
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+#endif
+
+
+static unsigned long numDocs = 0;
+static unsigned long numChunkDocs = 0;
+static unsigned long numDocsInChunk = 0;
+
+static unsigned long numFrags = 0;
+static unsigned long numFragsInChunk = 0;
+static unsigned long chunkStartFragNum = 0;
+
+
+
+
+struct BitPtr {
+  unsigned long start;
+  unsigned long here;
+  unsigned long lastFragNum;
+  unsigned long lgB;
+  
+  void Clear () { start = here = lastFragNum = lgB = 0; }
+  BitPtr () { Clear(); }
+};
+
+class WordBitPtrs {
+protected:
+  unsigned long numWords;
+  unsigned long numTags;
+  unsigned long size;
+  BitPtr *wordBitPtrs;
+
+  void CheckBufOverrun (unsigned long num) {
+    if (wordBitPtrs[num].here > wordBitPtrs[num+1].start) {
+      cerr << "numDocs: " << numDocs << "\n";
+      cerr << "numChunkDocs: " << numChunkDocs << "\n";
+      cerr << "numDocsInChunk: " << numDocsInChunk << "\n";
+      cerr << "numFrags: " << numFrags << "\n";
+      cerr << "numFragsInChunk: " << numFragsInChunk << "\n";
+      cerr << "chunkStartFragNum: " << chunkStartFragNum << "\n";
+      cerr << "num: " << num << "\n";
+      cerr << "[num].start: " << wordBitPtrs[num].start << "\n";
+      cerr << "[num].here: " << wordBitPtrs[num].here << "\n";
+      cerr << "[num+1].start: " << wordBitPtrs[num+1].start << "\n";
+      FatalError (1, "Bit buffer overrun");
+    }
+  }
+  
+public:
+  void Clear ();
+  WordBitPtrs () { wordBitPtrs = NULL; Clear(); }
+  ~WordBitPtrs ();
+  void SetSize (unsigned long _numWords,
+		unsigned long _numTags);
+
+  void ResetPtrs () {
+    if (wordBitPtrs == NULL) return;
+    unsigned long i;
+    for (i=0; i<size; ++i) wordBitPtrs[i].Clear();
+  }
+  
+  BitPtr &GetWordBitPtr (unsigned long wordNum)
+    { return wordBitPtrs[wordNum]; }
+  unsigned long &GetWordStart (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].start; }
+  unsigned long &GetWordHere (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].here; }
+  void CheckWordBufOverrun (unsigned long wordNum)
+    { CheckBufOverrun (wordNum); }
+
+  BitPtr &GetTagBitPtr (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords]; }
+  unsigned long &GetTagStart (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].start; }
+  unsigned long &GetTagHere (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].here; }
+  void CheckTagBufOverrun (unsigned long tagNum)
+    { CheckBufOverrun (tagNum + numWords); }
+
+  BitPtr &GetEndBitPtr ()
+    { return wordBitPtrs[size-1]; }
+  unsigned long &GetEndStart ()
+    { return wordBitPtrs[size-1].start; }
+  unsigned long &GetEndHere ()
+    { return wordBitPtrs[size-1].here; }
+};
+
+
+struct IP2TagInfo {
+  bool inTag;
+  unsigned long startFrag;
+  unsigned long tagNum;
+
+  IP2TagInfo () {
+    inTag = false;
+    startFrag = 0;
+    tagNum = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, IP2TagInfo, DictLTUCArray> TagMapDict;
+
+
+// class to handle the translation of occurrence order
+// to dictionary order for words and tags
+class OccurToDictConverter {
+protected:
+  unsigned long pos;
+  unsigned long val;
+  FILE *transFile;
+  random_bitio_buffer rbs;
+
+  unsigned long wordDictSize;
+  unsigned long tagDictSize;
+
+  void SeekStart ();
+  unsigned long TranslateNum (unsigned long num);
+  
+public:
+  OccurToDictConverter ();
+  ~OccurToDictConverter ();
+
+  void Open (char *filename, unsigned long _wordDictSize,
+	     unsigned long _tagDictSize);
+
+  // Close frees all allocated memory
+  void Close ();
+
+  unsigned long TranslateWord (unsigned long occurNum)
+    { return TranslateNum (occurNum); }
+  unsigned long TranslateTag (unsigned long occurNum)
+    { return TranslateNum (occurNum+wordDictSize); }
+};
+
+
+struct InvfStateRec {
+  mg_ullong start;
+  mg_ullong here;
+  unsigned long lastFragNum;
+  unsigned long B;
+
+  void Clear () {
+    start = here = 0;
+    lastFragNum = B = 0;
+  }
+  InvfStateRec () { Clear (); }
+};
+
+
+#define ISR_SIZE 1024
+
+class InvfStateCache {
+protected:
+  InvfStateRec recCache [ISR_SIZE];
+  unsigned long startNum;
+
+  FILE *stateFile;
+
+  void ClearCache () {
+    unsigned int i = 0;
+    for (i=0; i<ISR_SIZE; ++i) recCache[i].Clear();
+  }
+  
+public:
+  InvfStateCache ();
+  ~InvfStateCache ();
+  
+  void Open (char *filename);
+  void Close ();
+
+  // previous references to state records may be
+  // invalidated calling GetRec
+  InvfStateRec &GetRec (unsigned long num);
+};
+
+
+static invf_dict_header idh;
+static WordBitPtrs bitPtrs;
+
+static FILE *chunkFile = NULL;
+static stdio_bitio_buffer chunkBuf;
+
+static unsigned long ivfMemBufSize = 0;
+static char *ivfMemBuf = NULL;
+
+// word and tag dictionaries. a map is used for the tag dictionary
+// as it should never be very big (and the perfect hash function
+// sometimes has trouble with small values).
+static perf_hash_data *wordHashDict = NULL;
+static TagMapDict tagMapDict;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+static OccurToDictConverter occurConvert;
+
+// information about the state of the inverted file
+static InvfStateCache invfState;
+
+static char collectFilename[512];
+
+
+void WordBitPtrs::Clear () {
+  numWords = 0;
+  numTags = 0;
+  size=0;
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+  wordBitPtrs = NULL;
+}
+
+WordBitPtrs::~WordBitPtrs () {
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+}
+
+void WordBitPtrs::SetSize (unsigned long _numWords,
+			   unsigned long _numTags){
+  Clear();
+  numWords = _numWords;
+  numTags = _numTags;
+  size = numWords + numTags + 1;
+  wordBitPtrs = new BitPtr [size];
+}
+
+
+void OccurToDictConverter::SeekStart () {
+  if (transFile == NULL) return;
+  rbs.SEEK_X (sizeof (unsigned long) * 8);
+  pos = 0;
+}
+
+unsigned long OccurToDictConverter::TranslateNum (unsigned long num) {
+  if (num < pos) SeekStart ();
+  while (pos <= num) {
+    if (pos < wordDictSize)
+      val = rbs.binary_decode (wordDictSize + 1, NULL) - 1;
+    else 
+      val = rbs.binary_decode (tagDictSize + 1, NULL) - 1;
+    ++pos;
+  }
+  return val;
+}
+
+OccurToDictConverter::OccurToDictConverter () {
+  pos = 0;
+  val = 0;
+  transFile = NULL;
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+OccurToDictConverter::~OccurToDictConverter () {
+  if (transFile != NULL) Close ();
+}
+
+void OccurToDictConverter::Open (char *filename, unsigned long _wordDictSize,
+				 unsigned long _tagDictSize) {
+  if (transFile != NULL) Close ();
+
+  wordDictSize = _wordDictSize;
+  tagDictSize = _tagDictSize;
+
+  transFile = open_file (filename, INVF_CHUNK_TRANS_SUFFIX, "rb",
+			 MAGIC_CHUNK_TRANS, MG_ABORT);
+  rbs.attachFile (transFile, RND_BUF_SIZE);
+  SeekStart ();
+  val = 0;
+}
+
+void OccurToDictConverter::Close () {
+  if (transFile == NULL) return;
+
+  rbs.done ();
+  fclose (transFile);
+  transFile = NULL;
+  pos = 0;
+  val = 0;
+
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+
+
+
+InvfStateCache::InvfStateCache () {
+  startNum = 0;
+  stateFile = NULL;
+}
+
+InvfStateCache::~InvfStateCache () {
+  if (stateFile != NULL) Close ();
+}
+  
+void InvfStateCache::Open (char *filename) {
+  if (stateFile != NULL) Close();
+
+  // open the state file
+  char path[512];
+  sprintf (path, FILE_NAME_FORMAT ".%ld", get_basepath (), filename,
+           ".invf.state", (long) getpid ());
+  if (!(stateFile = fopen (path, "wb+"))) {
+    Message ("Unable to create \"%s\"", path);
+    exit (1);
+  }
+  unlink (path); // file will be deleted after it is closed
+  // reset the buffer
+  startNum = 0;
+  ClearCache();
+}
+
+void InvfStateCache::Close () {
+  if (stateFile == NULL) return;
+  fclose (stateFile);
+  stateFile = NULL;
+  startNum = 0;
+}
+
+InvfStateRec &InvfStateCache::GetRec (unsigned long num) {
+  // see if cached
+  if ((num >= startNum) && (num < startNum + ISR_SIZE))
+    return recCache[num-startNum];
+
+  // not cached, write out this lot of records and read in
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fwrite ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  // read in the new set of records
+  ClearCache ();
+  startNum = num - (num % ISR_SIZE);
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fread ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  return recCache[num-startNum];
+}
+
+
+
+static void ClearCharBuf (char *buf, unsigned long size) {
+  char *end = buf + size;
+  while (buf != end) *buf++ = 0;
+}
+
+static void ReadWordDict (char *filename) {
+  // read in the perfect hash function for the word dictionary
+  FILE *wordHashFile = open_file (filename, INVF_DICT_HASH_SUFFIX, "rb",
+				  MAGIC_HASH, MG_ABORT);
+  if (!(wordHashDict = read_perf_hash_data (wordHashFile))) {
+    FatalError (1, "Unable to read in hash data for word dictionary");
+  }
+  fclose (wordHashFile);
+}
+
+static void ReadTagDict (char *filename, invf_dict_header &_idh) {
+  // open the file
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+  
+  unsigned long tagNum;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < _idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    tagMapDict[thisEl.el].tagNum = tagNum;
+  }
+  
+  fclose (dictFile);
+}
+
+static void ReadLevelFile (char *filename) {
+  FILE *f;
+  f = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+		 MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Read (f);
+  fclose (f);
+}
+
+void CheckIntOverflow (mg_ullong totalIBits, mg_ullong lastTotalIBits) {
+  if (totalIBits < lastTotalIBits) {
+    fprintf(stderr, "ERROR: The totalIBits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+    if (sizeof (mg_ullong) < 8) {
+      fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+    }
+    fprintf(stderr, "       Build aborted.\n");
+    exit(1);
+  }
+}
+
+// assumes the inverted file state file has been opened
+static void InitInvfState (char *filename,
+			   invf_dict_header &_idh,
+			   InvfStateCache &_invfState,
+			   mg_ullong &totalIBits,
+			   bool wordLevelIndex) {
+  // read in the dictionary, setting inverted state information
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the word dictionary
+  fseek (dictFile, _idh.word_dict_start, SEEK_SET);
+
+  // add the word entries
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (_idh.num_levels);
+  unsigned long dictWordNum, p;
+  mg_ullong lastTotalIBits;
+  unsigned long N = _idh.num_frags;
+  for (dictWordNum=0; dictWordNum<_idh.word_dict_size; ++dictWordNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next word and associated information
+    wordEl.Read (dictFile, _idh.num_levels);
+
+    // update the state record
+    p = wordEl.frag_occur;
+    InvfStateRec &wisr = _invfState.GetRec (dictWordNum);
+    wisr.start = totalIBits;
+    wisr.here = totalIBits;
+    wisr.B = BIO_Bblock_Init (N, p);
+    
+    // add the length of the fragment numbers
+    totalIBits += BIO_Bblock_Bound_b (N, p, wisr.B);
+
+    // if needed, add the length of the fragment frequency information
+    if (!wordLevelIndex)
+      totalIBits += BIO_Gamma_Bound (wordEl.freq, wordEl.frag_occur);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+
+  // add the tag entries
+  dict_el tagEl;
+  unsigned long dictTagNum;
+  N = _idh.num_frags;
+  for (dictTagNum=0; dictTagNum<_idh.tag_dict_size; ++dictTagNum) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next tag and associated information
+    tagEl.Read (dictFile);
+
+    // update the state record
+    p = tagEl.frag_occur*2;
+    InvfStateRec &tisr = _invfState.GetRec (dictTagNum + _idh.word_dict_size);
+    tisr.start = totalIBits;
+    tisr.here = totalIBits;
+    tisr.B = BIO_Bblock_Init (N+p, p);
+    
+    // add the length of the fragment numbers (two numbers for each
+    // tag, one for start and one for end)
+    totalIBits += BIO_Bblock_Bound_b (N+p, p, tisr.B);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  fclose (dictFile);
+}
+
+/*
+// assumes the chunk tag information has been placed in .first
+static void PrintChunkInfo (unsigned long chunkMem,
+			    unsigned long numChunkWords,
+			    unsigned long numChunkTags) {
+  static unsigned long chunksRead = 0;
+  ++chunksRead;
+  cout << "Chunk Number: " << chunksRead << "\n";
+  cout << "numChunkDocs " << numDocsInChunk << "\n";
+  cout << "numChunkFrags " << numFragsInChunk << "\n";
+  cout << "mem " << chunkMem << "\n";
+  cout << "numWords " << numChunkWords << "\n";
+  cout << "numTags " << numChunkTags << "\n\n";
+
+  TagMapDict::iterator tagMapHere = tagMapDict.begin();
+  TagMapDict::iterator tagMapEnd = tagMapDict.end();
+  while (tagMapHere != tagMapEnd) {
+    unsigned long tagMapNum = (*tagMapHere).second.tagNum;
+    cout << (*tagMapHere).first << " " << tagMapNum << " "
+	 << bitPtrs.GetTagBitPtr(tagMapNum).here << "\n";
+    ++tagMapHere;
+  }
+}
+*/
+
+void ReadChunk (invf_dict_header &_idh, bool wordLevelIndex) {
+  // reset globals
+  numChunkDocs = 0;
+  chunkStartFragNum = numFrags;
+
+  // read in information about this chunk
+  numDocsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  if (numDocsInChunk == 0)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  numFragsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long chunkMem = chunkBuf.gamma_decode (NULL) - 1;
+
+  if (chunkMem > ivfMemBufSize)
+    FatalError (1, "Chunk memory size is greater than maximum");
+  
+  unsigned long numChunkWords = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long numChunkTags = chunkBuf.gamma_decode (NULL) - 1;
+
+  
+  // reset stuff
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+  bitPtrs.ResetPtrs();  
+
+  // read in the entries in occurrence order storing the
+  // "chunkWordCount" in "start" and the "chunkFragCount"
+  // in "here"
+  unsigned long numOccur;
+  unsigned long wordNum;
+  for (numOccur=0; numOccur<numChunkWords; ++numOccur) {
+    wordNum = occurConvert.TranslateWord (numOccur);
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    wordPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    if (wordPtr.start >= 2)
+      wordPtr.here = chunkBuf.gamma_decode (NULL);
+    else wordPtr.here = wordPtr.start;
+  }
+  unsigned long tagNum;
+  for (numOccur=0; numOccur<numChunkTags; ++numOccur) {
+    tagNum = occurConvert.TranslateTag (numOccur);
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    // only chunkFragCount is encoded for tags
+    tagPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    tagPtr.here = tagPtr.start;
+  }
+
+  /*  PrintChunkInfo (chunkMem, numChunkWords, numChunkTags);*/
+  
+  // create the bit ptrs in dictionary order
+  unsigned long totalIBits = 0; // only dealing with memory
+  unsigned long chunkWordCount, chunkFragCount;
+  for (wordNum=0; wordNum<_idh.word_dict_size; ++wordNum) {
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    chunkWordCount = wordPtr.start;
+    chunkFragCount = wordPtr.here;
+    wordPtr.start = totalIBits;
+    wordPtr.here = totalIBits;
+    wordPtr.lastFragNum = chunkStartFragNum;
+    wordPtr.lgB = 0;
+    if (chunkWordCount > 0) {
+      wordPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk,
+						   chunkFragCount));
+      totalIBits += BIO_Bblock_Bound (numFragsInChunk, chunkFragCount);
+      // use unary encoding for memory buffer encoding of fragment freq
+      if (!wordLevelIndex) {
+	totalIBits += chunkWordCount;
+      }
+    }
+  }
+  for (tagNum=0; tagNum<_idh.tag_dict_size; ++tagNum) {
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    chunkFragCount = tagPtr.here;
+    tagPtr.start = totalIBits;
+    tagPtr.here = totalIBits;
+    tagPtr.lastFragNum = chunkStartFragNum;
+    tagPtr.lgB = 0;
+    if (chunkFragCount > 0) {
+      unsigned long pTag = chunkFragCount*2;
+      tagPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk+pTag,
+						  pTag));
+      unsigned long bLen = BIO_Bblock_Bound (numFragsInChunk+pTag,
+					     pTag);
+//        cout << tagNum + _idh.word_dict_size << " ";
+//        cout << "numFrags: " << numFragsInChunk
+//    	   << " chunkFragCount: " << chunkFragCount
+//    	   << " B: " << 1 << tagPtr.lgB
+//    	   << " blen: " << blen << "\n";
+      totalIBits += bLen;
+    }
+  }
+  bitPtrs.GetEndStart() = totalIBits;
+  bitPtrs.GetEndHere() = totalIBits;
+
+  if ((totalIBits + 7ul) >> 3ul > chunkMem) {
+    cerr << "totalIBits: " << totalIBits << "\n";
+    cerr << "bytes: " << ((totalIBits + 7ul) >> 3ul) << "\n";
+    cerr << "chunkMem: " << chunkMem << "\n";
+    FatalError (1, "Pointers exceed buffer size");
+  }
+}
+
+
+
+
+int init_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // read in compressed dictionary header
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  idh.Read (dictFile);
+  fclose (dictFile);
+
+  // set the size of the bit ptrs
+  bitPtrs.SetSize (idh.word_dict_size, idh.tag_dict_size);
+
+  // open the chunk file and read in the maximum memory needed
+  // for the inverted memory buffer
+  chunkFile = open_file (filename, INVF_CHUNK_SUFFIX, "rb",
+			 MAGIC_CHUNK, MG_ABORT);
+  ReadUL (chunkFile, ivfMemBufSize);
+  chunkBuf.attachFile (chunkFile);
+
+  // allocate memory for the inverted buffer
+  ivfMemBuf = new char [ivfMemBufSize];
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+
+  // read in the word dictionary
+  ReadWordDict (filename);
+
+  // read in the tag dictionary
+  ReadTagDict (filename, idh);
+
+  // read in the level information
+  ReadLevelFile (filename);
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  
+  // set up the translation table file
+  occurConvert.Open (filename, idh.word_dict_size, idh.tag_dict_size);
+
+  // reset some globals
+  numDocs = 0;
+  numChunkDocs = 0;
+  numDocsInChunk = 0;
+  numFrags = 0;
+  numFragsInChunk = 0;
+  chunkStartFragNum = 0;
+
+  strcpy (collectFilename, filename);
+  
+  
+  // create the inverted file
+  mg_ullong totalIBits = 0;
+  FILE *invfFile = create_file (filename, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_ABORT);
+  totalIBits += sizeof (unsigned long) * 8; // magic number
+  totalIBits += 8 * 200;                    // 200 byte gap -- why??????
+  fclose (invfFile);
+
+  // init the inverted file state cache
+  invfState.Open (filename);
+  InitInvfState (filename, idh, invfState, totalIBits, wordLevelIndex);
+
+  return COMPALLOK;
+}
+
+static void CloseTextTag (IP2TagInfo &tInfo, const UCArray &/*tagName*/) {
+  if (!tInfo.inTag) return;
+
+    // add this tag to the inverted list
+  BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tInfo.tagNum);
+  unsigned long endFrag = numFrags;
+  int b = 1 << tagBitPtr.lgB;
+
+  /*
+  cout << (tInfo.tagNum+idh.word_dict_size) << " \"<" << tagName << ">\" "
+       << tInfo.startFrag << " " << endFrag << "\n";
+  */
+    
+  mems_bitio_buffer buffer ((u_char *) ivfMemBuf, tagBitPtr.here);
+  buffer.bblock_encode (tInfo.startFrag - tagBitPtr.lastFragNum + 1,
+			b, NULL);
+  buffer.bblock_encode (endFrag - tInfo.startFrag + 1, b, NULL);
+  tagBitPtr.lastFragNum = endFrag;
+  tagBitPtr.here = buffer.position();
+  buffer.encodeDone();
+  
+  // check for buffer overrun
+  bitPtrs.CheckTagBufOverrun (tInfo.tagNum);
+  
+  // reset information about this tag
+  tInfo.inTag = false;
+  tInfo.startFrag = 0;
+}
+
+static void ProcessOpenTag (const TextEl &el, bool &inFrag) {
+  // close tag if open
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  if (tInfo.inTag) CloseTextTag (tInfo, el.tagName);
+
+  // open this tag
+  tInfo.inTag = true;
+  tInfo.startFrag = numFrags;
+
+  // check for start of next fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    ++numFrags;
+    inFrag = true;
+  }
+}
+
+static void ProcessCloseTag (const TextEl &el, bool &inFrag) {
+  // check for end of fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    inFrag = false;
+  }
+
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  CloseTextTag (tInfo, el.tagName);
+}
+
+static void ProcessText (const TextEl &el, bool &inFrag) {
+  // make sure this text is to be indexed
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = &(el.text[0]);
+  const unsigned char *textEnd = &(el.text[el.text.size() - 1]);
+  unsigned char mgWord[MAXSTEMLEN + 1];
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexMGWord (textHere, textEnd, mgWord);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (mgWord[0] > 0) {
+      if (wordLevelIndex) ++numFrags;
+
+      unsigned long wordNum = perf_hash (wordHashDict, mgWord);
+
+      /*
+      cout << wordNum << " \"";
+      cout.write (mgWord+1, *mgWord);
+      cout << "\" " << numFrags << "\n";
+      */
+      
+      // add this word to the inverted list
+      BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+      unsigned long fragNum = numFrags;
+      int b = 1 << wordBitPtr.lgB;
+  
+      mems_bitio_buffer buffer ((u_char *) ivfMemBuf, wordBitPtr.here);
+
+      // note: this assumes that fragments don't carry over between
+      // chunks (which they don't because all tags are closed at the
+      // end of each document and chunks are based on document
+      // boundaries), i.e. the first fragment number must be greater
+      // than the starting fragment number of the chunk.
+      if (fragNum > wordBitPtr.lastFragNum) {
+	buffer.bblock_encode ((fragNum - wordBitPtr.lastFragNum - 1) + 1,
+			      b, NULL);
+	if (!wordLevelIndex) buffer.encodeBit (1); // freq = 1
+	
+      } else if (!wordLevelIndex) {
+	// add one to the frequency count for this word
+	buffer.seek (buffer.position()-1);
+	buffer.encodeBit (0); // unary encoding -- last = 1
+	buffer.encodeBit (1);
+      }
+      
+      wordBitPtr.lastFragNum = fragNum;
+      wordBitPtr.here = buffer.position();
+      buffer.encodeDone();
+      
+      // check for buffer overrun
+      bitPtrs.CheckWordBufOverrun (wordNum);
+    }
+  }
+}
+
+// combine the in memory inverted buffer with the disk
+// based inverted file
+static void DiskMerge (char *filename) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+
+      // make sure we have something to process
+  if (numChunkDocs <= 0) return;
+    
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb+",
+			      MAGIC_INVF, MG_ABORT);
+  random_bitio_buffer invfOutBuf (invfFile);
+
+  // set up to decode the entries in memory
+  mems_bitio_buffer memInBuf ((u_char *) ivfMemBuf, 0);
+
+  // write out the word information
+  unsigned long wordNum;
+  int b;
+  unsigned long currFragNum;
+  unsigned long delta;
+  unsigned long currFreq;
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &wordDiskState = invfState.GetRec (wordNum);
+    invfOutBuf.SEEK_X (wordDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+    memInBuf.seek(wordBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currFragNum = chunkStartFragNum;
+    while (memInBuf.position() < wordBitPtr.here) {
+      // decode word entry
+      b = 1 << wordBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL);
+      currFragNum += delta;
+      if (!wordLevelIndex) currFreq = memInBuf.unary_decode (NULL);
+      else currFreq = 1;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currFragNum-wordDiskState.lastFragNum,
+				wordDiskState.B, NULL);
+      if (!wordLevelIndex) invfOutBuf.gamma_encode (currFreq, NULL);
+      wordDiskState.lastFragNum = currFragNum;
+    }
+
+    wordDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  // write out the tag information
+  unsigned long tagNum;
+  unsigned long currTagStart;
+  unsigned long currTagEnd;
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &tagDiskState = invfState.GetRec (tagNum+idh.word_dict_size);
+    invfOutBuf.SEEK_X (tagDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tagNum);
+    memInBuf.seek(tagBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currTagEnd = chunkStartFragNum;
+    while (memInBuf.position() < tagBitPtr.here) {
+      // decode tag entry
+      b = 1 << tagBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagStart = currTagEnd + delta;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagEnd = currTagStart + delta;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currTagStart-tagDiskState.lastFragNum+1,
+				tagDiskState.B, NULL);
+      invfOutBuf.bblock_encode (currTagEnd-currTagStart+1,
+				tagDiskState.B, NULL);
+
+      tagDiskState.lastFragNum = currTagEnd;
+    }
+
+    tagDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  memInBuf.done();
+
+  invfOutBuf.encodeDone();
+  fclose (invfFile);
+}
+
+
+int process_ivf_2 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  // get next chunk information if need to. the chunk information
+  // is needed before the first document is processed
+  if (numChunkDocs >= numDocsInChunk) ReadChunk (idh, wordLevelIndex);
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // process this element
+    if ((*here).elType == OpenTagE) ProcessOpenTag (*here, inFrag);
+    else if ((*here).elType == CloseTagE) ProcessCloseTag (*here, inFrag);
+    else ProcessText (*here, inFrag);
+    
+    ++here;
+  }
+
+  // close off any unclosed tags
+  TagMapDict::iterator tdHere = tagMapDict.begin();
+  TagMapDict::iterator tdEnd = tagMapDict.end();
+  while (tdHere != tdEnd) {
+    CloseTextTag ((*tdHere).second, (*tdHere).first);
+    ++tdHere;
+  }
+  
+  // we've processed one more document
+  ++numDocs;
+  ++numChunkDocs;
+
+  // merge the memory based inverted file with the one on
+  // disk if this is the end of this chunk
+  if (numChunkDocs >= numDocsInChunk) DiskMerge (collectFilename);
+  
+  return COMPALLOK;
+}
+
+
+static void CondenseInvfFile (char *filename, unsigned long &bytesOutput) {
+  FILE *inInvfFile = open_file (filename, INVF_SUFFIX, "rb",
+				MAGIC_INVF, MG_ABORT);
+  FILE *outInvfFile = open_file (filename, INVF_SUFFIX, "rb+",
+				 MAGIC_INVF, MG_ABORT);
+
+  // skip the magic number
+  fseek (outInvfFile, sizeof (unsigned long), SEEK_SET);
+  
+  // write the inverted file header -- use defaults for most things
+  invf_file_header ifh;
+  ifh.no_of_words = idh.word_dict_size;
+  ifh.no_of_tags = idh.tag_dict_size;
+  ifh.word_level_index = (ivfLevel.indexLevel.empty()) ? 1 : 0;
+  ifh.Write (outInvfFile);
+
+  bytesOutput = ftell (outInvfFile);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = ifh.no_of_words + ifh.no_of_tags;
+  unsigned long entryNum;
+  mg_ullong lastStart = 0;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // overrun check
+    if (stateRec.start < lastStart)
+      FatalError (1, "Inverted file Buffer overrun");
+    lastStart = stateRec.start;
+    
+    unsigned long oldEntryStart = stateRec.start >> 3;
+    unsigned long oldEntryStartOver = stateRec.start & 7; // should be 0
+    unsigned long oldEntryEnd = (stateRec.here + 7) >> 3;  // byte after end
+    unsigned long oldEntryEndOver = stateRec.here & 7;
+
+    fseek (inInvfFile, oldEntryStart, SEEK_SET);
+
+    stateRec.here -= stateRec.start;
+    stateRec.start = (mg_ullong)bytesOutput * 8 + oldEntryStartOver;
+    stateRec.here += stateRec.start;
+    while (oldEntryStart < oldEntryEnd) {
+      unsigned char c = getc (inInvfFile);
+      if (oldEntryStart == oldEntryEnd - 1) {
+	u_char ands[8] =
+	{0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	c &= ands[oldEntryEndOver];
+      }
+      putc (c, outInvfFile);
+      ++bytesOutput;
+      ++oldEntryStart;
+    }
+  }  
+
+  fclose (inInvfFile);
+
+#ifdef __WIN32__
+  if (_chsize (_fileno (outInvfFile), bytesOutput) != 0)
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (outInvfFile), bytesOutput);
+#endif
+
+  fclose (outInvfFile);
+}
+
+static void OutputInvfIdx (char *filename, unsigned long invfNumBytes) {
+  FILE *invfIdxFile = create_file (filename, INVF_IDX_SUFFIX, "wb",
+				   MAGIC_INVI, MG_ABORT);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = idh.word_dict_size + idh.tag_dict_size;
+  unsigned long entryNum;
+  for (entryNum = 0; entryNum < numEntries; ++entryNum) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // assumes that inverted entries start at beginning of each byte
+    if (!WriteUL (invfIdxFile, (stateRec.start >> 3))) break;
+  }
+
+  WriteUL (invfIdxFile, invfNumBytes);
+
+  fclose (invfIdxFile);
+}
+
+
+int done_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // close most open files
+  if (chunkFile != NULL) {
+    chunkBuf.done();
+    fclose (chunkFile);
+    chunkFile = NULL;
+  }
+  occurConvert.Close();
+  
+  // free allocated memory
+  bitPtrs.Clear();
+  if (ivfMemBuf != NULL) { delete [] ivfMemBuf; ivfMemBuf = NULL; }
+  free_perf_hash (wordHashDict);
+  wordHashDict = NULL;
+  tagMapDict.erase (tagMapDict.begin(), tagMapDict.end());
+
+  // condense the inverted file and truncate it
+  // this function also writes out the inverted header
+  unsigned long invfNumBytes = 0;
+  CondenseInvfFile (filename, invfNumBytes);
+
+  OutputInvfIdx (filename, invfNumBytes);
+
+  // close the rest of the open files
+  invfState.Close ();
+  
+  return COMPALLOK;
+}
Index: /main/tags/3.01/mgpp/text/locallib.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/locallib.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/locallib.cpp	(revision 21112)
@@ -0,0 +1,218 @@
+/**************************************************************************
+ *
+ * locallib.cpp -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; ++i)
+    total += A[i];
+  for (i = 0; i < n; ++i)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; --i)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      --n;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; ++i)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  ++N;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	++limit;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * /*mem*/,
+	      u_long * disk) {
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; ++i)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  memset (cfh->huff_words_size, '\0', sizeof (cfh->huff_words_size));
+
+  if ((int)fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; ++i)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
Index: /main/tags/3.01/mgpp/text/locallib.h
===================================================================
--- /main/tags/3.01/mgpp/text/locallib.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/locallib.h	(revision 21112)
@@ -0,0 +1,45 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/mg.h
===================================================================
--- /main/tags/3.01/mgpp/text/mg.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /main/tags/3.01/mgpp/text/mg.special.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mg.special.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg.special.cpp	(revision 21112)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * mg.special.cpp -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "mg.h"
+#include "build.h"
+
+int init_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
+
+int process_special (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  // print out the document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE) {
+      cout << "** OpenTag  \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n"; 
+    } else if ((*here).elType == CloseTagE) {
+      cout << "** CloseTag \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n";
+    } else {
+      cout << "** Text     \"" << (*here).text << "\"\n";
+    }
+    
+    ++here;
+  }
+  
+  return COMPALLOK;
+}
+
+int done_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
Index: /main/tags/3.01/mgpp/text/mg_errors.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mg_errors.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg_errors.cpp	(revision 21112)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * mg_errors.cpp -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete []mg_error_data;
+    mg_error_data = null_data;
+  }
+}
Index: /main/tags/3.01/mgpp/text/mg_errors.h
===================================================================
--- /main/tags/3.01/mgpp/text/mg_errors.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg_errors.h	(revision 21112)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /main/tags/3.01/mgpp/text/mg_files.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mg_files.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg_files.cpp	(revision 21112)
@@ -0,0 +1,259 @@
+/**************************************************************************
+ *
+ * mg_files.cpp -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      delete []basepath;
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = new char[strlen (bp) + 2];
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /main/tags/3.01/mgpp/text/mg_files.h
===================================================================
--- /main/tags/3.01/mgpp/text/mg_files.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mg_files.h	(revision 21112)
@@ -0,0 +1,343 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXT_LEVELS	GEN_MAGIC('M','G','t','L')
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVF_LEVELS	GEN_MAGIC('M','G','I','L')
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The document levels in the text */
+#ifdef SHORT_SUFFIX
+# define TEXT_LEVEL_SUFFIX	 	".tl"
+#else
+# define TEXT_LEVEL_SUFFIX	 	".text.level"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The level information for the inverted text */
+#ifdef SHORT_SUFFIX
+# define INVF_LEVEL_SUFFIX              ".il"
+#else
+# define INVF_LEVEL_SUFFIX              ".invf.level"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/mgpp_compression_dict.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_compression_dict.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_compression_dict.1	(revision 21112)
@@ -0,0 +1,188 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mgpp_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_compression_dict.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_compression_dict.cpp	(revision 21112)
@@ -0,0 +1,1021 @@
+/**************************************************************************
+ *
+ * mgpp_compression_dict.cpp -- Routines for creating compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "WordData.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct DictWordData : public WordData
+  {
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+DictWordData;
+
+static DictWordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    DictWordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HDY")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+#ifndef SILENT
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+#endif
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+#ifndef SILENT
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+#endif
+  return (0);
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; ++i)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      DictWordData *wd;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = (DictWordData **) Xmalloc (sizeof (DictWordData *) * Num[i] + 1);
+
+      buf = (char *) Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = (DictWordData *) Xmalloc (sizeof (DictWordData) * Num[i]);
+      unsigned int j;
+      for (j = 0; j < Num[i]; ++j, ++wd)
+	{
+	  int len;
+
+	  // read docCount and wordCount
+	  wd->read(f);
+
+	  // read an mgString
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  unsigned int i;
+  DictWordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (DictWordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; ++i, ++wd)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  unsigned int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; ++i)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; ++i)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->docCount > bb->docCount)
+    return -1;
+  else if (aa->docCount < bb->docCount)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->words() > bb->words())
+    return 1;
+  else if (aa->words() < bb->words())
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  DictWordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = (DictWordData **) Xmalloc (num * sizeof (DictWordData *));
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    wd[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      DictWordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->documents(), word->word, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; ++i)
+    {
+      DictWordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (DictWordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  memset (char_freqs, '\0', sizeof (char_freqs));
+  memset (len_freqs, '\0', sizeof (len_freqs));
+  for (i = 0; i < num; ++i, ++wd)
+    {
+      u_long freq = (*wd)->documents();
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; --len, ++buf)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+inline void CalcBitCostForWordData(DictWordData **word, int num,
+				   double freqs_trans[2], u_long escape[2],
+				   char * char_lens[2], char *len_lens[2], 
+				   double esc[2], int num_trans)
+{ 
+  int j;
+
+  for (j = 0; j < num; ++j, ++word)
+    {
+      float   cbc, wbc;
+      u_char *buf = (*word)->word;
+      int     len = *buf++;
+      u_long  freq = (*word)->documents();
+      int     idx = KIND (*word);
+
+      cbc = len_lens[idx][len];
+      for (; len; --len, ++buf)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*word)->saving = ((*word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*word)->word[0]);
+
+      (*word)->num_trans = num_trans;
+    }
+}
+
+
+
+void 
+CalcBitCost (DictWordData ** discard_word, int discard_num,
+	     DictWordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  CalcBitCostForWordData(discard_word, discard_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  CalcBitCostForWordData(keep_word, keep_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+inline void m3_transferWord(DictWordData **toHeap, int *toNum, heap_comp toSaving,
+			    DictWordData **fromHeap, int *fromNum, heap_comp fromSaving,
+			    int *num_trans, double freqs_trans[], int *mem, double *total,
+			    int *count)
+{
+  /* Transfer the word at the top of the keep heap to the
+     discard heap. */
+  DictWordData *word = fromHeap[0];
+  int idx = KIND (word);
+  heap_deletehead (fromHeap, sizeof (word), fromNum, fromSaving);
+  toHeap[*toNum] = word;
+  heap_additem (toHeap, sizeof (word), toNum, toSaving);
+  freqs_trans[idx] -= word->documents();
+  *mem = (*mem) - sizeof (u_char *) + word->word[0];
+  *num_trans += 1;
+  *total += word->saving;
+  *count += 1;
+#ifdef DEBUG
+  fprintf (stderr,
+	   "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+	   *mem, word->documents(), word->char_bit_cost,
+	   word->saving, word2str (word->word));
+#endif
+}
+
+inline void m3_storeWord(DictWordData **wordHeap, int wordNum, int num_trans, 
+			 double freqs_trans[], u_long escape[], heap_comp hc)
+{
+  DictWordData *word = wordHeap[0];
+  int idx = KIND (word);
+  float wbc;
+#ifdef DEBUG1
+  fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+	   word->saving);
+#endif
+  wbc = -log2 (word->documents() / (freqs_trans[idx] + escape[idx])) *
+    word->documents();
+  word->saving = (word->char_bit_cost - wbc) /
+    (sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+  fprintf (stderr, " %.2f\n", word->saving);
+#endif
+  word->num_trans = num_trans;
+  heap_changedhead (wordHeap, sizeof (word), wordNum, hc);
+}
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  DictWordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+  keep_num = 0;
+  keep_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+
+  for (i = 0; (unsigned int)i < Num[0]; ++i)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; ++i)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      DictWordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->documents();
+      ++num_trans;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < (unsigned)num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(keep_heap, keep_num, num_trans, freqs_trans, escape, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(discard_heap, discard_num, num_trans, freqs_trans, escape, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > (unsigned)k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      m3_transferWord(discard_heap, &discard_num, BigSaving,
+			      keep_heap, &keep_num, SmallSaving,
+			      &num_trans, freqs_trans, &mem, &total, &keep_to_discard);
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      m3_transferWord(keep_heap, &keep_num, SmallSaving,
+			      discard_heap, &discard_num, BigSaving,
+			      &num_trans, freqs_trans, &mem, &total, &discard_to_keep);
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  ++recalcs;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; ++i)
+    {
+      DictWordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; ++i)
+    {
+      DictWordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+#ifndef SILENT
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+#endif
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  unsigned int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; ++i)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  unsigned int i, us;
+  for (us = i = 0; i < dd->num_wds; ++i)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = new long [dd->num_wds]))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      freqs[i] = dd->wd[i]->documents();
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  delete []freqs;
+  freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  memset (&huff_words_size, '\0', sizeof (huff_words_size));
+  memset (&lencounts, '\0', sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; (unsigned)i < dd->num_wds; ++i)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      ++lencounts[codelen];
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; ++i)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  delete []hd->clens;
+  delete hd;
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[256];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; --len, ++buf)
+	freqs[(u_long) (*buf)] += (*wd)->documents();
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; ++j)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[16];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  memset (freqs, '\0', sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; ++j, ++wd)
+    freqs[(*wd)->word[0]] += (*wd)->documents();
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; ++j)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  delete []hd->clens;
+  delete hd;
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  memset (&cdh, '\0', sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; ++i)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < discard[i].num_wds; ++j)
+		esc.docCount += discard[i].wd[j]->documents();
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < all[i].num_wds; ++j)
+		if (all[i].wd[j]->documents() == 1)
+		  ++esc.docCount;
+	      if (!esc.docCount)
+		++esc.docCount;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_decompress_text.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_decompress_text.cpp	(revision 21112)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * mgpp_decompress_text.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "TextGet.h"
+#include "messages.h"
+#include "mg_files.h"
+
+int main (int argc, char **argv) {
+  int ch;
+  char *filename = "";
+  char *basePath = "";
+  UCArray level;
+  SetCStr (level, "Document", 8);
+
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:d:K:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      filename = optarg;
+      break;
+    case 'd':
+      basePath = optarg;
+      set_basepath (optarg);
+      break;
+    case 'K':
+      SetCStr (level, optarg, strlen(optarg));
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-K level] [-d directory] -f name\n",
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  // load up the text information
+  TextData td;
+  if (!td.LoadData (basePath, filename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", filename);
+  }
+  
+  // output each document in the level
+//    cout << td.levels << "\n";
+  
+  TextLevelInfo levelInfo = td.levels.levelInfo[level];
+  unsigned long docNum = 1;
+  UCArray docText;
+  while (docNum <= levelInfo.numEntries) {
+//      TextIdx docIdx;
+//      if (GetDocIdx (td, level, docNum, docIdx)) {
+//        cout << "doc: " << docNum << "\n";
+//        cout << docIdx;
+//      }
+    
+    if (!GetDocText (td, level, docNum, docText)) {
+      FatalError (1, "Error while trying to get document %u", docNum);
+    }
+
+    cout << docText << "\n";
+    
+    ++docNum;
+  }
+  
+  return 0;
+}
+
+
Index: /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.1	(revision 21112)
@@ -0,0 +1,69 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mgpp_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgppquery (1)
+can load it in very quickly, making the startup time for
+.BR mgppquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_fast_comp_dict.cpp	(revision 21112)
@@ -0,0 +1,707 @@
+/**************************************************************************
+ *
+ * mgpp_fast_comp_dict.cpp -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "huffman.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; ++i)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  memset (buffer, 0, mem);
+  fixup = (u_char *) Xmalloc (fixup_mem);
+  memset (fixup, 0, fixup_mem);
+
+#ifndef SILENT
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+#endif
+
+  load_comp_dict (filename);
+
+#ifndef SILENT
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+#endif
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; ++which)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; ++j)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID)
+      for (which = 0; which <= 1; ++which)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; ++j)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; ++i)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  return (0);
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * /*cdh*/, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; --val)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+static u_long mem_skip_hd(FILE *dict, u_long mem)
+{ huff_data hd;
+
+  mem += sizeof (hd);
+  Read_Huffman_Data (dict, &hd, NULL, NULL);
+  if (hd.clens)
+    delete []hd.clens;
+  mem += hd.num_codes * sizeof (unsigned long);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+  return mem;
+}
+
+static u_long mem_read_cfh(FILE *dict, compression_dict_header *cdh, comp_frags_header *cfh, 
+			   u_long mem)
+{
+  mem += sizeof (comp_frags_header);
+  Read_cfh (dict, cfh, NULL, NULL);
+
+  /* Don't need to count the space for the clens of the huffman data */
+
+  mem += mem_for_words (dict, cdh, cfh);
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+
+  return mem;
+}
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; ++i)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; ++which)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+
+	  mem = mem_skip_hd(dict, mem);
+	  mem = mem_skip_hd(dict, mem);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem = mem_skip_hd(dict, mem);
+	      mem = mem_skip_hd(dict, mem);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((u_long) cur > (u_long)buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = (auxiliary_dict *) getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; ++i)
+    {
+      unsigned int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = (u_char *) getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = (u_char **) getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; ++j)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      ++num;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; ++i)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = (u_char **) getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = (u_char ***) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = (u_char *) getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; ++i)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  memset (num_set, '\0', sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; ++i)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      ++num_set[len];
+    }
+  if (cfh->hd.clens)
+    delete []cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = (unsigned long *) getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = (unsigned long **) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  memset (values, '\0', (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; ++i)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; ++i)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+static void load_read_huffman(FILE *dict, compression_dict *cd, int which)
+{
+  huff_data *hd;
+  u_long   **vals;
+
+  hd = (huff_data *) getmem (sizeof (huff_data), sizeof (char *));
+  cd->MemForCompDict += sizeof (huff_data);
+  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+  cd->chars_huff[which] = hd;
+  FIXUP (&cd->chars_huff[which]);
+  cd->chars_vals[which] = vals;
+  FIXUP (&cd->chars_vals[which]);
+  if (hd->clens)
+    delete []hd->clens;
+  hd->clens = NULL;
+}
+
+static void load_read_words(FILE *dict, compression_dict *cd, int which, int fixescape)
+{
+  cd->cfh[which] = (comp_frags_header *) getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+  FIXUP (&cd->cfh[which]);
+  FIXUP (&cd->values[which]);
+  if (fixescape)
+    { FIXUP(&cd->escape[which]);
+    }
+  else
+    {
+      cd->escape[which] = NULL;
+    }
+}
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = (compression_dict *) getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; ++which)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  load_read_words(dict, cd, which, 0);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+
+	  load_read_huffman(dict, cd, which);
+	  load_read_huffman(dict, cd, which);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      load_read_huffman(dict, cd, which);
+	      load_read_huffman(dict, cd, which);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = (u_long *) buffer; (u_long) p < (u_long) cur; ++p)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /main/tags/3.01/mgpp/text/mgpp_invf_dict.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_invf_dict.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_invf_dict.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mgpp_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_invf_dict
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +13n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that can be used during querying.
+Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word. The dictionary is blocked with
+each block containing the same number of entries (not the same size).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to 
+specify the number of entries per block. The default is 16.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_invf_dict.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_invf_dict.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict.cpp -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename, unsigned long entriesPerBlock) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // create the blocked dictionary
+  FILE *blockDictFile = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "wb",
+				     MAGIC_STEM, MG_ABORT);
+  block_dict_header bdh;
+  bdh.lookback = idh.lookback;
+  bdh.word_dict_start = idh.word_dict_start;
+  bdh.word_dict_size = idh.word_dict_size;
+  bdh.tag_dict_start = idh.tag_dict_start;
+  bdh.tag_dict_size = idh.tag_dict_size;
+  bdh.num_docs = idh.num_docs;
+  bdh.num_frags = idh.num_frags;
+  bdh.num_words = idh.num_words;
+  bdh.total_bytes = idh.total_bytes;
+  bdh.index_string_bytes = idh.index_string_bytes;
+  bdh.num_levels = idh.num_levels;
+  bdh.Write (blockDictFile);
+
+  
+  // write out the word part of the dictionary
+
+  bdh.entries_per_wblk = entriesPerBlock;
+  bdh.max_wblk_size = 0;
+  bdh.wblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  block_idx wordIdx;
+  word_block_dict_el wordBlockEl;
+  wordBlockEl.SetNumLevels (idh.num_levels);
+  
+  
+  unsigned long wordNum;
+  unsigned long wordInvfPtr;
+  UCArray lastEl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word and inverted file pointer
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordInvfPtr);
+
+    // remember this word (and position) if this is the start
+    // of a new block
+    if (wordNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = wordEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (wordIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(wordIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_wblk_size) bdh.max_wblk_size = blockLen;
+      }
+      
+      wordIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this word
+    wordBlockEl.el = wordEl.el;
+    wordBlockEl.frag_occur = wordEl.frag_occur;
+    wordBlockEl.freq = wordEl.freq;
+    wordBlockEl.invf_ptr = wordInvfPtr;
+    unsigned long tempI;
+    for (tempI=0; tempI<idh.num_levels; ++tempI)
+      wordBlockEl.levelFreqs[tempI] = wordEl.levelFreqs[tempI];
+
+    // write out the word
+    wordBlockEl.Write (blockDictFile, &lastEl, idh.num_levels);
+
+    lastEl = wordBlockEl.el;
+  }
+
+
+  // write out the tag part of the dictionary
+
+  bdh.entries_per_tblk = entriesPerBlock;
+  bdh.max_tblk_size = 0;
+  bdh.tblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+
+  block_idx tagIdx;
+  block_dict_el tagBlockEl;
+  
+  unsigned long tagNum;
+  unsigned long tagInvfPtr;
+  dict_el tagEl;
+  lastEl.erase (lastEl.begin(), lastEl.end());
+  for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+    // read in the next tag and inverted file pointer
+    tagEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagInvfPtr);
+
+    // remember this tag (and position) if this is the start
+    // of a new block
+    if (tagNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = tagEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (tagIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(tagIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_tblk_size) bdh.max_tblk_size = blockLen;
+      }
+      
+      tagIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this tag
+    tagBlockEl.el = tagEl.el;
+    tagBlockEl.frag_occur = tagEl.frag_occur;
+    tagBlockEl.freq = tagEl.freq;
+    tagBlockEl.invf_ptr = tagInvfPtr;
+
+    // write out the tag
+    tagBlockEl.Write (blockDictFile, &lastEl);
+
+    lastEl = tagBlockEl.el;
+  }
+
+
+  // write out the element indexes
+  bdh.num_wblks = wordIdx.size();
+  bdh.wblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, wordIdx);
+  
+  bdh.num_tblks = tagIdx.size();
+  bdh.tblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, tagIdx);
+
+  // write out the blocked dictionary header
+  fseek (blockDictFile, sizeof(unsigned long), SEEK_SET);
+  bdh.Write (blockDictFile);
+
+
+  // close open files
+  fclose (blockDictFile);
+  fclose (invfIdxFile);
+  fclose (dictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Max word block size = %d\n", bdh.max_wblk_size);
+  Message ("Max tag block size = %d\n", bdh.max_tblk_size);
+  Message ("Number of word blocks written = %d\n", bdh.num_wblks);
+  Message ("Number of tag blocks written = %d\n", bdh.num_tblks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file] "
+	       "[-d data directory] [-b entries-per-block] "
+	       "[-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, entriesPerBlock);
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_invf_dict_dump.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_invf_dict_dump.cpp	(revision 21112)
@@ -0,0 +1,141 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dict_dump.cpp -- Program to printthe various inverted dictionaries
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // print the information from the inverted dictionary file header
+  if (printHeader) {
+    cout << "lookback: " << idh.lookback << "\n";
+    cout << "word_dict_start: " << idh.word_dict_start << "\n";
+    cout << "word_dict_size: " << idh.word_dict_size << "\n";
+    cout << "tag_dict_start: " << idh.tag_dict_start << "\n";
+    cout << "tag_dict_size: " << idh.tag_dict_size << "\n";
+    cout << "num_docs: " << idh.num_docs << "\n";
+    cout << "num_frags: " << idh.num_frags << "\n";
+    cout << "num_words: " << idh.num_words << "\n";
+    cout << "total_bytes: " << idh.total_bytes << "\n";
+    cout << "index_string_bytes: " << idh.index_string_bytes << "\n";
+    cout << "num_levels: " << idh.num_levels << "\n";
+
+    cout << "\n";
+  }
+
+
+  if (printWords) {
+    fseek (dictFile, idh.word_dict_start, SEEK_SET);
+    
+    unsigned long wordNum;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      cout << "\"" << wordEl.el << "\"\n";// (" << wordNum << ")\n";
+    }
+    
+    cout << "\n";
+  }
+
+  // write out the tag part of the dictionary
+  if (printTags) {
+    fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+    
+    unsigned long tagNum;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      // read in the next tag and inverted file pointer
+      tagEl.Read (dictFile);
+      
+      cout << "\"" << tagEl.el << "\" (" << tagNum << ")\n";
+    }
+  }
+  
+  // close open files
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+
+  while ((ch = getopt (argc, argv, "f:d:rwth")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-f input_file] "
+	       "[-d data directory]\n(-rwt:print header, words, tags)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, printHeader, printWords, printTags);
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_invf_dump.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_invf_dump.cpp	(revision 21112)
@@ -0,0 +1,244 @@
+/**************************************************************************
+ *
+ * mgpp_invf_dump.cpp -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "messages.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+#include "WordData.h"
+
+static void PrintInvfWord (FILE *invfFile,
+			   invf_dict_header &idh,
+			   invf_file_header &ifh,
+			   word_dict_el &wordEl,
+			   unsigned long wordStart,
+			   bool printFrags) {
+  cout << wordEl.frag_occur << " \"" << wordEl.el << "\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL);
+      fragNum += delta;
+      cout << " " << fragNum;
+      
+      if (!ifh.word_level_index ) {
+	unsigned long count = buffer.gamma_decode (NULL);
+	cout << "(" << count << ")";
+      } else {
+	cout << "(1)";
+      }
+    }
+    
+    cout << "\n";
+  
+    buffer.done();
+  }
+}
+
+static void PrintInvfTag (FILE *invfFile,
+			  invf_dict_header &idh,
+			  invf_file_header &/*ifh*/,
+			  dict_el &tagEl,
+			  unsigned long tagStart,
+			  bool printFrags) {
+  cout << tagEl.frag_occur << " \"<" << tagEl.el << ">\"\n";
+  
+  if (printFrags) {
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, tagStart, SEEK_SET);
+    
+    stdio_bitio_buffer buffer(invfFile);
+    
+    unsigned long pTag = tagEl.frag_occur*2;
+    unsigned long B = BIO_Bblock_Init (idh.num_frags+pTag, pTag);
+    unsigned long fragNum = 0;
+    unsigned long i;
+    for (i=0; i<tagEl.frag_occur; ++i) {
+      unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << " " << fragNum;
+      cout << "-";
+      delta = buffer.bblock_decode (B, NULL)-1;
+      fragNum += delta;
+      cout << fragNum;
+    }
+    
+    cout << "\n";
+    
+    buffer.done();
+  }
+}
+
+static void PrintHeaderInfo (invf_dict_header &idh,
+			     invf_file_header &ifh) {
+  cerr << "Lookback:         " << idh.lookback << "\n";
+  cerr << "Word Dict Size:   " << idh.word_dict_size << "\n";
+  cerr << "Tag Dict Size:    " << idh.tag_dict_size << "\n";
+  cerr << "Num Documents:    " << idh.num_docs << "\n";
+  cerr << "Num Fragments:    " << idh.num_frags << "\n";
+  cerr << "Num Words:        " << idh.num_words << "\n";
+
+  cerr << "Skip Mode:        " << ifh.skip_mode << "\n";
+  cerr << "Word Level Index: " << ifh.word_level_index << "\n";
+
+  cerr << "\n";
+}
+
+
+static void process_files (char *filename,
+			   bool printHeader,
+			   bool printWords,
+			   bool printTags,
+			   bool printFrags) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  // print out header information
+  if (printHeader) {
+    PrintHeaderInfo (idh, ifh);
+  }
+
+  // open the inverted index
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  // process all the words
+  if (printWords) {
+    unsigned long wordNum;
+    unsigned long wordStart;
+    word_dict_el wordEl;
+    wordEl.SetNumLevels (idh.num_levels);
+    for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+      wordEl.Read (dictFile, idh.num_levels);
+      ReadUL (invfIdxFile, wordStart);
+      PrintInvfWord (invfFile, idh, ifh, wordEl, wordStart, printFrags);
+    }
+  }
+
+  // process all the tags
+  if (printTags) {
+    unsigned long tagNum;
+    unsigned long tagStart;
+    dict_el tagEl;
+    for (tagNum=0; tagNum<idh.tag_dict_size; ++tagNum) {
+      tagEl.Read (dictFile);
+      ReadUL (invfIdxFile, tagStart);
+      PrintInvfTag (invfFile, idh, ifh, tagEl, tagStart, printFrags);
+    }
+  }
+  // close the open files
+  fclose (invfIdxFile);
+  fclose (invfFile);
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *dir_name, *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+
+  bool printHeader = false;
+  bool printWords = false;
+  bool printTags = false;
+  bool printFrags = false;
+  
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hrwtnf:d:")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath(optarg);
+      break;
+    case 'r':
+      printHeader = true;
+      break;
+    case 'w':
+      printWords = true;
+      break;
+    case 'n':
+      printFrags = true;
+      break;
+    case 't':
+      printTags = true;
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-r] [-w] [-t] [-n] [-f input_file]"
+	       "[-d data directory]\n(-rwnt:print header, words, tags, fragnums)\n", 
+	       argv[0]);
+      exit (1);
+    }
+  }
+  
+  process_files (filename, printHeader, printWords, printTags, printFrags);
+
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_passes.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_passes.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_passes.1	(revision 21112)
@@ -0,0 +1,323 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_passes 1 \*(Dt CITRI
+.SH NAME
+mgpp_passes \- builds mgpp databases
+.SH SYNOPSIS
+.B mgpp_passes
+[
+.BI \-J " doc-tag"
+]
+[
+.BI \-K " level-tag"
+]
+.if n .ti +10n
+[
+.BI \-L " index-level"
+]
+[
+.BI \-m " invf-mem-buffer"
+]
+.if n .ti +10n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+[
+.B \-S
+]
+[
+.B \-C
+]
+.if n .ti +10n
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mgpp_passes
+is the program that does most of the work when building mgpp
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  In general,
+.B mgpp_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+mgpp database.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+mgpp database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpointt\fP'u+2n"
+.BI \-J " doc-tag"
+Specifies the SGML tag that encloses each document.  Text appearing
+outside this tag is ignored.  The document tag defines the highest
+level document that can be queried and printed.  The default document
+tag is 'Document'.
+.TP
+.BI \-K " level-tag"
+Specifies the SGML tag of a sub document level.  A level tag must
+enclose all text enclosed by the document tag.  Levels can be
+queried and printed as if they were separate documents.  Multiple
+document levels can be specified (the document tag is always
+added as a document level).
+.TP
+.BI \-L " index-level"
+Specifies the SGML tag enclosing the smallest indexed element.  The
+index level should be no larger than the smallest document 
+level.  An empty string can be used to specify a word level index 
+(which is the default).
+.TP
+.BI \-m " invf-mem-buffer"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+.IR *.text.level ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mgpp_perf_hash_build (1)
+from the
+.I *.invf.dict
+file.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP 
+.B \-C
+This activates the compatibility parsing mode.  When using this
+mode documents are separated by control-B and paragraphs are separated
+by control-C.  Internally these are converted to documents surrounded
+by 'Document' tags and paragraphs surrounded by 'Paragraph' tags.
+.TP
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an mgpp document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats, *.invf.dict, *.invf.level
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mgpp_passes -T1 -I1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mgpp_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mgpp_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text, *.text.idx, *.text.level
+.I
+#   *.invf and *.invf.idx
+${source} | mgpp_passes -T2 -I2 -f ${text}
+.PP
+.I
+# Create *.text.weight and *.weight.approx
+mgpp_weights_build -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked
+mgpp_invf_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.1
+mgpp_stem_idx -s 1 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.2
+mgpp_stem_idx -s 2 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.3
+mgpp_stem_idx -s 3 -f ${text}
+.PP
+.I
+# Create *.text.dict.fast
+mgpp_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary with index into the dictionary.
+.TP
+.B *.invf.dict.blocked.n
+Transformation dictionary from words stemmed with method 
+.B n
+to unstemmed words.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.level
+Information about the document levels needed for querying.
+.TP
+.B *.text
+Compressed text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.fast
+A fast loading version of the compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.level
+Information about the document levels needed for text decompression.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+The approximate weights file.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_passes.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_passes.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_passes.cpp	(revision 21112)
@@ -0,0 +1,268 @@
+/**************************************************************************
+ *
+ * mgpp_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#if defined __WIN32__
+# include <io.h>
+# include "getopt_old.h"
+# define close _close
+# define open _open
+#elif defined __CYGWIN__
+#include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "longlong.h"
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-J doc-tag] [-K level-tag] [-L index-level]\n"
+"  %*s [-m invf-memory] [-T1] [-T2] [-I1] [-I2] [-S]\n"
+"  %*s [-C] [-h] [-d directory] -f name\n\n";
+
+
+
+static void Usage (char *err) {
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "");
+  exit (1);
+}
+
+
+
+
+int OpenNextFile (int in_fd) {
+  if (in_fd > 0) close (in_fd);
+  if (num_files == 0) return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  ++files;
+  --num_files;
+  return (in_fd);
+}
+
+
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; ++pass) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      ++numDocs;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; ++pass) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+int main (int argc, char **argv) {
+  int ch, in_fd;
+  char *filename = NULL;
+  bool compatMode = false;
+  TagInfo tagInfo;
+  tagInfo.SetDocTag ("Document");
+  
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch=getopt(argc, argv, "J:K:L:f:d:m:I:T:SCh"))!=-1){
+    switch (ch) {
+    case 'J':
+      tagInfo.SetDocTag (optarg);
+      break;
+    case 'K':
+      tagInfo.AddLevelTag (optarg);
+      break;
+    case 'L':
+      tagInfo.SetIndexLevel (optarg);
+      break;
+    case 'f':
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'm':
+      invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+      break;
+    case 'I':
+      if (*optarg == '1')
+	Passes |= IVF_PASS_1;
+      else if (*optarg == '2')
+	Passes |= IVF_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'T':
+      if (*optarg == '1')
+	Passes |= TEXT_PASS_1;
+      else if (*optarg == '2')
+	Passes |= TEXT_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'S':
+      Passes |= SPECIAL;
+      break;
+    case 'C':
+      compatMode = true;
+      break;
+    case 'h':
+    case '?':
+      Usage (NULL);
+    }
+  }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc) {
+    if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+      FatalError (1, "Cannot open %s", argv[optind]);
+    files = &argv[optind + 1];
+    num_files = argc - (optind + 1);
+    
+  } else in_fd = 0;  // stdin
+
+  
+  if (compatMode) tagInfo.SetDocTag ("Document");
+
+  // a document tag is also a level tag
+  tagInfo.levelTags.insert (tagInfo.docTag); 
+  
+  Driver (in_fd, filename, tagInfo, compatMode);
+
+  return (0);
+}
Index: /main/tags/3.01/mgpp/text/mgpp_passes_4jni.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_passes_4jni.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_passes_4jni.cpp	(revision 21112)
@@ -0,0 +1,316 @@
+/**************************************************************************
+ *
+ * mgpp_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#if defined __WIN32__
+# include <io.h>
+# include "getopt_old.h"
+# define close _close
+# define open _open
+#elif defined __CYGWIN__
+#include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "longlong.h"
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+#include "mgpp_passes_4jni.h"
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+TagInfo tagInfo;
+char *filename = NULL;
+
+unsigned long numBytes = 0;
+unsigned long numDocs = 0;
+int mgpp_passes_exit_value = 0;
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+
+/* clear all the settings from one mgpp_passes run to the next */
+void clear_variables() {
+  tagInfo.Clear();
+  tagInfo.SetDocTag ("Document");
+  Passes = 0;
+  invf_buffer_size = 5 * 1024 * 1024;
+  numBytes = 0;
+  numDocs = 0;
+  mgpp_passes_exit_value = 0;
+}
+
+/* ################################################## */
+/* the following are methods to set all the variables that used to be
+   set by command line args */
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num) {
+
+  switch(pass_type) {
+  case 'S':
+    Passes |= SPECIAL;
+    break;
+  case 'I':
+    if (pass_num == '1')
+      Passes |= IVF_PASS_1;
+    else if (pass_num == '2')
+      Passes |= IVF_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  case 'T':
+    if (pass_num == '1')
+      Passes |= TEXT_PASS_1;
+    else if (pass_num == '2')
+      Passes |= TEXT_PASS_2;
+    else
+      fprintf(stderr, "Invalid pass number %c for pass type %c\n", pass_num, pass_type);
+    break;
+  }
+}
+
+/* -m arg to mgpp_passes */
+void set_inversion_limit(int limit) {
+  invf_buffer_size = limit * 1024 * 1024;
+}
+
+/* -J arg to mgpp_passes */
+void set_document_tag(const char *tag_name) {
+  tagInfo.SetDocTag (tag_name);
+
+  // a doc tag is also a level tag
+  tagInfo.AddLevelTag(tag_name);
+
+}
+
+/* -K arg to mgpp_passes */
+void add_level_tag(const char *tag_nam) {
+  tagInfo.AddLevelTag(tag_nam);
+}
+
+/* -L arg to mgpp_passes */
+void set_index_level(const char *tag_name) {
+  tagInfo.SetIndexLevel(tag_name);
+}
+
+// is this enough???
+/* -f arg to mgpp_passes */
+void set_filename(const char * filen) {
+  if (filename) {
+    Xfree (filename);
+    filename = NULL;
+  }
+  filename = Xstrdup (filen);  
+}
+ 
+
+/* ############################################### */
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver () {
+
+  if (!filename || *filename == '\0') {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "A document collection name must be specified.");
+  }
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2)) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+  }
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2)) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+  }
+  if (!Passes) {
+    mgpp_passes_exit_value = 1;
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+  }
+  
+  // initialise all the passes
+  for (int pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, filename) == COMPERROR) {
+	mgpp_passes_exit_value = 1;
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+      }
+    }
+  }
+
+}
+
+void process_document(u_char *buffer, int len) {
+  
+  TextElArray doc;
+  unsigned long doc_len =  0;
+
+  while(ReadDoc(&buffer, tagInfo.docTag, doc, doc_len, false)) {
+    // give this document to each pass
+    for (int pass = 0; pass < MAX_PASSES; pass++) {
+      if (Passes & (1 << pass)) {
+	if (PassData[pass].process (tagInfo, doc) == COMPERROR) {
+	  mgpp_passes_exit_value = 1;
+	  FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+    }
+    
+    // another document has been processed
+    numBytes += doc_len;
+    numDocs++;
+    cout << "doc_len = "<<doc_len<<endl;
+  }
+}    
+
+void finalise_driver () {
+  
+  // do done for each pass
+  for (int pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, filename) == COMPERROR) {
+	mgpp_passes_exit_value = 1;
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+    }
+  }  
+}
+
+int get_exit_value() {
+  return mgpp_passes_exit_value;
+}
+  /*
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; pass++) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      numDocs++;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+  */
Index: /main/tags/3.01/mgpp/text/mgpp_passes_4jni.h
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_passes_4jni.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_passes_4jni.h	(revision 21112)
@@ -0,0 +1,62 @@
+/**************************************************************************
+ *
+ * mgpp_passes_4jni.h -- modified mgpp_passes for use from jni
+ * Copyright (C) 2004 New Zealand Digital Library, http://www.nzdl.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef H_MGPP_PASSES_4JNI
+#define H_MGPP_PASSES_4JNI
+
+
+/* clear all the settings from one mgpp_passes run to the next */
+void clear_variables();
+
+/* -S, -T1, -T2, -I1, -I2, args to mg_passes */
+void add_pass (char pass_type, char pass_num);
+
+/* -m arg to mg_passes */
+void set_inversion_limit(int limit);
+
+/* -J arg to mgpp_passes */
+void set_document_tag(const char * tag_name);
+
+/* -K arg to mgpp_passes */
+void add_level_tag(const char * tag_name);
+
+/* -L arg to mgpp_passes */
+void set_index_level(const char * tag_name);
+
+/* -f arg to mgpp_passes */
+void set_filename(const char * filen);
+
+/* The old driver method has been split into 3:
+init_driver, process_document (called numdocs times), 
+finalise_driver.
+The above set vars methods should all be called before init_driver.
+*/
+
+void init_driver ();
+
+void process_document(u_char *buffer, int len);
+
+void finalise_driver();
+
+int get_exit_value();
+
+#endif
Index: /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.1	(revision 21112)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mgpp_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mgpp_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_stem_idx (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_perf_hash_build.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * mgpp_perf_hash_build.cpp -- Program to build a perfect hash function
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define POOL_SIZE 1024*1024
+
+static void ProcessFiles (char *filename, int r) {
+  FILE *dictFile, *hashFile;
+  unsigned long i;
+  invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  unsigned long pool_left;
+  u_char **starts;
+
+  // read in the dictionary
+  dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  if (dictFile==NULL) {
+    FatalError(1, "unable to open file");
+  }
+  idh.Read (dictFile);
+
+  //cerr << idh.lookback<<" "<<idh.word_dict_start<<endl;
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  
+  if (!(pool = (u_char *) Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = (u_char **) Xmalloc (sizeof (u_char *) * idh.word_dict_size)))
+    FatalError (1, "Out of memory");
+  //cerr << "size= "<< idh.word_dict_size<<endl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (i = 0; i < idh.word_dict_size; ++i) {
+    // read the next word and associated information
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // push string onto pool data
+    register unsigned long l;
+    l = wordEl.el.size() + 1;
+    if (pool_left < l) {
+      pool = (u_char *) Xmalloc (POOL_SIZE);
+      pool_left = POOL_SIZE;
+    }
+    starts[i] = pool;
+    
+    *pool++ = wordEl.el.size();
+    memcpy ((char *) pool, &(wordEl.el[0]), wordEl.el.size());
+    //cerr << pool<<"   " <<starts[i]<<endl;
+    pool += wordEl.el.size();
+    pool_left -= l;
+    
+  }
+  fclose (dictFile);
+  //cerr << pool<<"   " <<starts[i-1]<<endl;
+  //cerr<<"starts  "<<starts[113529]<<endl;
+  //cerr << starts[17][1] << " "<<starts[25][4]<<endl;
+  // create perfect hash file
+  hashFile = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+			  MAGIC_HASH, MG_ABORT);
+  if (!(phd = gen_hash_func (idh.word_dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hashFile, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (hashFile);
+}
+
+
+
+int main (int argc, char **argv) {
+  int r = -1;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      r = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  ProcessFiles (filename, r);
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_stem_idx.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_stem_idx.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_stem_idx.1	(revision 21112)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mgpp_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mgpp_stem_idx
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +12n
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +12n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_stem_idx
+generates a stem index file for a collection.
+This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to set
+the number of entries per block. The default is 16.
+.TP
+.BI \-a " stemmer"
+The name of the stemmer to use, the default is the Lovin stemmer.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_weights_build (1)
Index: /main/tags/3.01/mgpp/text/mgpp_stem_idx.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_stem_idx.cpp	(revision 21112)
@@ -0,0 +1,237 @@
+/**************************************************************************
+ *
+ * mgpp_stem_idx.cpp -- stem index builder
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "messages.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+#include "stemmer.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#  include <vector.h>
+#else
+#  include <map>
+#  include <vector>
+#endif
+
+typedef vector<unsigned long> WordNumList;
+typedef map<UCArray, WordNumList, DictLTUCArray> StemMapDict;
+
+void CreateStemDict (char *filename,
+		     StemMapDict &stemDict,
+		     int stemMethod,
+		     int stemmerNum) {
+  stemDict.erase (stemDict.begin(), stemDict.end());
+
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  unsigned long wordNum;
+  u_char mgWord[MAXSTEMLEN + 1];
+  word_dict_el wordEl;
+  UCArray stemEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+    // read in the next word
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // convert the word to an "mg word"
+    mgWord[0] = wordEl.el.size();
+    memcpy((char *)&mgWord[1], &(wordEl.el[0]), wordEl.el.size());
+
+    // stem the word
+    stemmer (stemMethod, stemmerNum, mgWord);
+
+    // convert the result back to a UCArray
+    stemEl.erase (stemEl.begin(), stemEl.end());
+    stemEl.insert (stemEl.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+//      cout << "\"" << stemEl << "\" -> \"" << wordEl.el << "\"\n";
+    
+    // add this word number to the list of word numbers for this word
+    stemDict[stemEl].push_back (wordNum);
+  }
+
+  fclose (dictFile);
+}
+
+
+void WriteStemDict (char *filename,
+		    StemMapDict &stemDict,
+		    int stemMethod,
+		    int stemmerNum,
+		    unsigned long entriesPerBlock) {
+  // Create appropriate stem index file
+  FILE *stemDictFile = NULL;
+  if (stemMethod == 1) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+				"wb", MAGIC_STEM_1, MG_ABORT);
+  } else if (stemMethod == 2) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+				"wb", MAGIC_STEM_2, MG_ABORT);
+  } else if (stemMethod == 3) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+				"wb", MAGIC_STEM_3, MG_ABORT);
+  } else {
+    FatalError (1, "Unknown stem method %d", stemMethod);
+  }
+
+  stem_idx_header sih;
+  sih.lookback = 0;
+  sih.dict_size = stemDict.size();
+  sih.entries_per_block = entriesPerBlock;
+  sih.max_block_size = 0;
+  
+  sih.stemmer_num = stemmerNum;
+  sih.stem_method = stemMethod;
+
+  // write out a place-holder version of the header
+  sih.Write (stemDictFile);
+
+  sih.blocks_start = ftell (stemDictFile);
+
+  block_idx stemIdx;
+  unsigned long stemNum = 0;
+  stem_block_dict_el stemEl;
+  UCArray lastEl;
+
+  StemMapDict::const_iterator here = stemDict.begin();
+  StemMapDict::const_iterator end = stemDict.end();
+  while (here != end) {
+    // remember this stem (and position) if this is the start
+    // of a new block
+    if (stemNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = (*here).first;
+      elIdx.block_ptr = ftell (stemDictFile);
+
+      // see if this block is the longest so far
+      if (stemIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(stemIdx.end()-1)).block_ptr;
+	if (blockLen > sih.max_block_size) sih.max_block_size = blockLen;
+      }
+      
+      stemIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this stem
+    stemEl.el = (*here).first;
+    stemEl.equivWords = (*here).second;
+
+    // write out the stem
+    stemEl.Write (stemDictFile, &lastEl);
+    
+    ++here; ++stemNum;
+  }
+
+
+  // write out the element indexes
+  sih.num_blocks = stemIdx.size();
+  sih.block_idx_start = ftell (stemDictFile);
+  WriteBlockIdx (stemDictFile, stemIdx);
+  
+  // write out the stem dictionary header
+  fseek (stemDictFile, sizeof(unsigned long), SEEK_SET);
+  sih.Write (stemDictFile);
+
+
+  // close open files
+  fclose (stemDictFile);
+  
+  // print out information
+#ifndef SILENT
+  Message ("Num word stems = %d\n", sih.dict_size);
+  Message ("Max stem block size = %d\n", sih.max_block_size);
+  Message ("Number of stem blocks written = %d\n", sih.num_blocks);
+#endif
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  int stemMethod = 0; // illegal value (no translation)
+  int stemmerNum = 0; // English stemmer
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:s:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 's':
+      stemMethod = atoi (optarg);
+      break;
+    case 'a':
+      stemmerNum = stemmernumber ((unsigned char *) optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-d directory] "
+	       "[-b entries-per-block] [-h] -s 1|2|3 "
+	       "[-a stemmer-method] -f name\n", argv[0]);
+      exit (1);
+    }
+  }
+  
+  if (stemMethod < 1 || stemMethod > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  // read in the dictionary and create the in memory dictionary
+  StemMapDict stemDict;
+  CreateStemDict (filename, stemDict, stemMethod, stemmerNum);
+  
+  // write out the dictionary as a blocked file
+  WriteStemDict (filename, stemDict, stemMethod, stemmerNum, entriesPerBlock);
+  
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/mgpp_weights_build.1
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_weights_build.1	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_weights_build.1	(revision 21112)
@@ -0,0 +1,79 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.\"------------------------------------------------------------
+.TH mgpp_weights_build 1 \*(Dt CITRI
+.SH NAME
+mgpp_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mgpp_weights_build
+[
+.B \-h
+]
+[
+.BI \-b " num-bits"
+]
+.if n .ti +17n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgpp_weights_build
+is the program that generates the exact and approximate weights 
+files.  To do this, it uses the inverted file
+and dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num-bits"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mgpp
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mgpp_compression_dict (1),
+.BR mgpp_fast_comp_dict (1),
+.BR mgpp_invf_dict (1),
+.BR mgpp_passes (1),
+.BR mgpp_perf_hash_build (1),
+.BR mgpp_stem_idx (1)
Index: /main/tags/3.01/mgpp/text/mgpp_weights_build.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/mgpp_weights_build.cpp	(revision 21112)
@@ -0,0 +1,399 @@
+/**************************************************************************
+ *
+ * mgpp_weights_build.cpp -- Program to build the document weights file
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#define _XOPEN_SOURCE 1
+#define _XOPEN_SOURCE_EXTENDED 1
+
+/* getopt is in posix.2, so cygwin should have it in unistd, but doesn't */
+#if defined (__WIN32__) || defined (__CYGWIN__)
+# include "getopt_old.h"
+#else
+# include <unistd.h>
+#endif
+
+#include "UCArray.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+struct WBTagPtr {
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  unsigned long fragOccur;
+  
+  WBTagPtr () {
+    tagNum = 0;
+    tagPtr = 0;
+    fragOccur = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, WBTagPtr, DictLTUCArray> WBTagDict;
+
+typedef vector<float> Weights;
+
+
+static void ReadTagDict (const invf_dict_header &idh,
+			 FILE *dictFile,
+			 FILE *invfIdxFile,
+			 WBTagDict &tagDict) {
+  tagDict.erase (tagDict.begin(), tagDict.end());
+
+  // seek to the start of the tag information
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof(unsigned long) +
+	 idh.word_dict_size*sizeof(unsigned long), SEEK_SET);
+  
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < idh.tag_dict_size; ++tagNum) {
+    thisEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagPtr);
+    tagDict[thisEl.el].tagNum = tagNum;
+    tagDict[thisEl.el].tagPtr = tagPtr;
+    tagDict[thisEl.el].fragOccur = thisEl.frag_occur;
+  }
+}
+
+static void AddWeight (Weights &w,
+		       unsigned long levelDocNum,
+		       unsigned long termFreq,
+		       float idf) {
+  double weight = termFreq * idf;
+  w[levelDocNum-1] += weight * weight;
+}
+
+static void GenerateLevelWeights (const invf_dict_header &idh,
+				  const invf_file_header &ifh,
+				  unsigned long numLevelDocs,
+				  unsigned long levelNum,
+				  FILE *dictFile,
+				  FILE *invfFile,
+				  FILE *invfIdxFile,
+				  const FragLevelConvert &fragLevelConvert,
+				  Weights &w) {
+  // pre-allocate the right number of weights
+  w.erase (w.begin(), w.end());
+  w.insert (w.end(), (Weights::size_type)numLevelDocs, (float)0.0);
+
+  double logN = log ((double) numLevelDocs);
+
+  // reset the files
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof (unsigned long), SEEK_SET);
+  
+  // process each word adding its contributions to the document weights
+  unsigned long wordNum;
+  unsigned long wordStart;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; ++wordNum) {
+#ifndef SILENT
+    // give a little feedback every 4096 words
+    if ((wordNum & 0xfff) == 0) fprintf (stderr, ".");
+#endif
+    
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordStart);
+
+    float idf = logN - log ((double) wordEl.levelFreqs[levelNum]);
+
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    stdio_bitio_buffer buffer (invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long levelDocNum = 0;
+    unsigned long lastLevelDocNum = 0;
+    unsigned long termFreq = 0;
+    unsigned long checkLevelFreq = 0;
+    
+    unsigned long count, i;
+    for (i=0; i<wordEl.frag_occur; ++i) {
+      fragNum += buffer.bblock_decode (B, NULL);
+      if (!ifh.word_level_index) count = buffer.gamma_decode (NULL);
+      else count = 1;
+
+      if (fragNum > idh.num_frags)
+	FatalError (1, "fragNum = %d, "
+		    "number of fragments = %d\n"
+		    "wordNum = %d\n"
+		    "i = %d, frag_occur = %d\n",
+		    fragNum, idh.num_frags, wordNum, i, wordEl.frag_occur);
+
+      if (!fragLevelConvert.FragToLevel (fragNum, levelDocNum))
+	FatalError (1, "could not convert fragment number %d in level %d", fragNum, levelNum);
+
+      if (levelDocNum == 0 || levelDocNum > numLevelDocs)
+	FatalError (1, "bad level document number %d in level %d", levelDocNum, levelNum);
+      
+      if (levelDocNum != lastLevelDocNum) {
+	// new level document
+	if (lastLevelDocNum > 0) {
+	  AddWeight (w, lastLevelDocNum, termFreq, idf);
+	  ++checkLevelFreq;
+	}
+	lastLevelDocNum = levelDocNum;
+	termFreq = 0;
+      }
+      termFreq += count;
+    }
+
+    if (lastLevelDocNum > 0) {
+      AddWeight (w, lastLevelDocNum, termFreq, idf);
+      ++checkLevelFreq;
+    }
+
+    if (checkLevelFreq != wordEl.levelFreqs[levelNum]) {
+      cerr << "bad level freq at level " <<levelNum<<" "<< checkLevelFreq << " != "
+	   << wordEl.levelFreqs[levelNum] << ", word \""
+	   << wordEl.el << "\" (" << wordNum << ")\n";
+      exit (1);
+    }
+    
+    buffer.done();
+  }
+
+  if (w.size() != numLevelDocs) 
+    FatalError (1, "wrong number of weights created %d != %d",
+		w.size(), numLevelDocs);
+}
+
+static void WriteExactWeights (FILE *weightsFile,
+			       unsigned long &diskPtr,
+			       const Weights &w) {
+  diskPtr = ftell(weightsFile);
+
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+//      cout << *here << "\n";
+    WriteF (weightsFile, sqrt (*here));
+    ++here;
+  }
+}
+
+static void WriteApproxWeights (FILE *approxWeightsFile,
+				unsigned long &diskPtr,
+				const Weights &w,
+				unsigned char bits) {
+  diskPtr = ftell(approxWeightsFile);
+
+  // calculate L, U and B
+  double L = 1e300;
+  double U = 0;
+  float wgt;
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+    wgt = sqrt (*here);
+    if (wgt > U) U = wgt;
+    if (wgt > 0 && wgt < L) L = wgt;
+    ++here;
+  }
+
+  double B = pow (U / L, pow (2.0, -(double) bits));
+
+#ifndef SILENT
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+#endif
+  
+  WriteUC (approxWeightsFile, bits);
+  WriteD (approxWeightsFile, L);
+  WriteD (approxWeightsFile, B);
+  
+
+  unsigned long max = (bits == 32) ? 0xffffffff : (1 << bits) - 1;;
+  unsigned long i=0, buf=0, pos=0;
+  here = w.begin();
+  end = w.end();
+  while (here != end) {
+    unsigned long fx;
+    wgt = sqrt (*here);
+    if (wgt == 0) {
+      wgt = L;
+#ifndef SILENT
+      Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+    }
+    fx = (unsigned long) floor (log (wgt / L) / log (B));
+
+    if (fx > max) fx = max;
+
+    buf |= (fx << pos);
+    pos += bits;
+
+    if (pos >= MAXBITS) {
+      WriteUL (approxWeightsFile, buf);
+      buf = fx >> (bits - (pos - MAXBITS));
+      pos = pos - MAXBITS;
+    }
+
+    ++here; ++i;
+  }
+
+  // write out the last bits
+  if (pos > 0) WriteUL (approxWeightsFile, buf);
+}
+
+int main (int argc, char **argv) {
+  unsigned char bits = 8;
+  char *filename = "";
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      bits = atoi (optarg);
+      if (bits > 32) {
+	fprintf (stderr, "b may only take values 0-32\n");
+	exit (1);
+      }
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-b bits] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to create weights.");
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+  
+  // read the level information
+  FILE *levelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			       MAGIC_INVF_LEVELS, MG_ABORT);
+  FIvfLevel ivfLevel;
+  ivfLevel.Read (levelFile);
+  fclose (levelFile);
+
+  // read in the tag dictionary and inverted file pointers
+  WBTagDict tagDict;
+  ReadTagDict (idh, dictFile, invfIdxFile, tagDict);
+
+  
+  // create the weights file
+  FILE *weightsFile = create_file (filename, WEIGHTS_SUFFIX, "wb",
+				   MAGIC_WGHT, MG_ABORT);
+
+  // create the approx weights file
+  FILE *approxWeightsFile = create_file (filename, APPROX_WEIGHTS_SUFFIX, "wb",
+					 MAGIC_WGHT_APPROX, MG_ABORT);
+
+  
+  // create weights for each document level
+  FragLevelConvert fragLevelConvert;
+  Weights w;
+  IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+  IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+  unsigned long levelNum = 0;
+  while (levelHere != levelEnd) {
+    const UCArray &levelName = (*levelHere).first;
+    
+    // read the tag information about this level
+    fragLevelConvert.Read(invfFile, tagDict[levelName].tagPtr,
+			  idh.num_frags, tagDict[levelName].fragOccur);
+    
+    // create the weights for this level
+    GenerateLevelWeights (idh, ifh, tagDict[levelName].fragOccur,
+			  levelNum, dictFile, invfFile, invfIdxFile,
+			  fragLevelConvert, w);
+
+    // write out the exact weights
+    WriteExactWeights (weightsFile,
+		       (*levelHere).second.exactWeightsDiskPtr,
+		       w);
+    
+    // write out the approximate weights
+    WriteApproxWeights (approxWeightsFile,
+			(*levelHere).second.approxWeightsDiskPtr,
+			w, bits);
+    
+    ++levelHere; ++levelNum;
+  }
+  
+
+  // close input files
+  fclose (dictFile);
+  fclose (invfFile);
+  fclose (invfIdxFile);
+
+  // update the level information
+  levelFile = create_file (filename, INVF_LEVEL_SUFFIX, "wb",
+			   MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Write (levelFile);
+  fclose (levelFile);
+
+  // close output files
+  fclose (weightsFile);
+  fclose (approxWeightsFile);
+
+
+  return 0;
+}
Index: /main/tags/3.01/mgpp/text/stemmer.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/stemmer.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/stemmer.cpp	(revision 21112)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * stemmer.cpp -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; ++i) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); ++i)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp ((char *) descript,  "0") == 0) ||
+      (strcmp ((char *) descript, "english") == 0) ||
+      (strcmp ((char *) descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp ((char *) descript, "1") == 1) ||
+      (strcmp ((char *) descript, "french") == 0) ||
+      (strcmp ((char *) descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /main/tags/3.01/mgpp/text/stemmer.h
===================================================================
--- /main/tags/3.01/mgpp/text/stemmer.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/stemmer.h	(revision 21112)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+#ifdef __cplusplus
+extern "C" 
+#endif
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /main/tags/3.01/mgpp/text/text.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/text.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/text.cpp	(revision 21112)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// UCArray currently contains all the basic reading and writing functions
+#include "UCArray.h"
+#include "text.h"
+
+compressed_text_header::compressed_text_header () {
+  Clear();
+}
+
+void compressed_text_header::Clear () {
+  num_of_docs = 0;
+  num_of_words = 0;
+  num_of_bytes = 0.0;
+}
+
+
+bool compressed_text_header::Read (FILE *f) {
+  return (ReadUL (f, num_of_docs) &&
+	  ReadUL (f, num_of_words) &&
+	  ReadD (f, num_of_bytes));
+}
+
+bool compressed_text_header::Write (FILE *f) const {
+  return (WriteUL (f, num_of_docs) &&
+	  WriteUL (f, num_of_words) &&
+	  WriteD (f, num_of_bytes));
+}
+
+
+auxiliary_dict::auxiliary_dict () {
+  word_data[0] = NULL;
+  word_data[1] = NULL;
+  words[0] = NULL;
+  words[1] = NULL;
+}
+
+auxiliary_dict::~auxiliary_dict () {
+  Clear ();
+}
+
+void auxiliary_dict::Clear () {
+  if (word_data[0] != NULL) {
+    delete word_data[0];
+    word_data[0] = NULL;
+  }
+  if (word_data[1] != NULL) {
+    delete word_data[1];
+    word_data[1] = NULL;
+  }
+  if (words[0] != NULL) {
+    delete words[0];
+    words[0] = NULL;
+  }
+  if (words[1] != NULL) {
+    delete words[1];
+    words[1] = NULL;
+  }
+}
+
+
+compression_dict::compression_dict () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    cfh[which] = NULL;
+    values[which] = NULL;
+    escape[which] = NULL;
+    chars_huff[which] = NULL;
+    chars_vals[which] = NULL;
+    lens_huff[which] = NULL;
+    lens_vals[which] = NULL;
+  }
+
+  MemForCompDict = 0;
+  ad = NULL;
+  fast_loaded = 0;
+}
+
+compression_dict::~compression_dict () {
+  Clear ();
+}
+
+void compression_dict::Clear () {
+  int which;
+  for (which = 0; which < 2; ++which) {
+    if (cfh[which] != NULL) {
+      delete cfh[which];
+      cfh[which] = NULL;
+    }
+    if (values[which] != NULL) {
+      delete values[which][0][0];
+      delete values[which][0];
+      delete values[which];
+      values[which] = NULL;
+    }
+    if (escape[which] != NULL) { // don't know if needed
+      //delete escape[which]; // this was causing a seg fault
+      escape[which] = NULL;
+    }
+    if (chars_huff[which] != NULL) {
+      delete chars_huff[which];
+      chars_huff[which] = NULL;
+    }
+    if (chars_vals[which] != NULL) {
+      delete chars_vals[which][0];
+      delete chars_vals[which];
+      chars_vals[which] = NULL;
+    }
+    if (lens_huff[which] != NULL) {
+      delete lens_huff[which];
+      lens_huff[which] = NULL;
+    }
+    if (lens_vals[which] != NULL) {
+      delete lens_vals[which][0];
+      delete lens_vals[which];
+      lens_vals[which] = NULL;
+    }
+  }
+
+  if (ad != NULL) {
+    delete ad;
+    ad = NULL;
+  }
+
+  MemForCompDict = 0;
+  fast_loaded = 0;
+}
Index: /main/tags/3.01/mgpp/text/text.h
===================================================================
--- /main/tags/3.01/mgpp/text/text.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/text.h	(revision 21112)
@@ -0,0 +1,199 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT
+#define H_TEXT
+
+// some elderly windows compilers need to include <vector> (or something
+// similar) here so that they know what a "bool" is
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+
+#include "huffman.h"
+#include <stdio.h>
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+struct compression_dict_header {
+  u_long dict_type;
+  u_long novel_method;
+  u_long params[TEXT_PARAMS];
+  u_long num_words[2];
+  u_long num_word_chars[2];
+  u_long lookback;
+};
+
+
+struct comp_frags_header {
+  huff_data hd;
+  u_long uncompressed_size;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+};
+
+// BOGUSTEXTLEN is used to replace the ratio
+// in the compressed_text_header while preparing
+// for UCArray
+#define BOGUSTEXTLEN 1000000
+
+struct compressed_text_header {
+  u_long num_of_docs;
+  u_long num_of_words; // number of words in collection
+  double num_of_bytes;
+
+  compressed_text_header ();
+  void Clear ();
+
+  // you must seek to the appropriate place before calling
+  // Read or Write
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct compression_stats_header {
+  u_long num_docs;
+  u_long dummy;     // alignment
+  double num_bytes; 
+};
+
+
+struct frags_stats_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct aux_frags_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct auxiliary_dict {
+  aux_frags_header afh[2];
+  u_char *word_data[2];
+  u_char **words[2];
+  int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+					   for the hybrid methods */
+
+  auxiliary_dict ();
+  ~auxiliary_dict ();
+
+  void Clear ();
+};
+
+
+struct compression_dict {
+  compression_dict_header cdh;
+  comp_frags_header *cfh[2];
+  unsigned long MemForCompDict;
+  u_char ***values[2];
+  u_char *escape[2];
+  huff_data *chars_huff[2];
+  u_long **chars_vals[2];
+  huff_data *lens_huff[2];
+  u_long **lens_vals[2];
+  auxiliary_dict *ad;
+  int fast_loaded;
+
+  compression_dict ();
+  ~compression_dict ();
+
+  void Clear ();
+};
+
+
+#endif
Index: /main/tags/3.01/mgpp/text/text.pass1.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/text.pass1.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/text.pass1.cpp	(revision 21112)
@@ -0,0 +1,388 @@
+/**************************************************************************
+ *
+ * text.pass1.cpp -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+#include "TextEl.h"
+
+/* for more meaningful messages - jrm21 (gsdl) */
+#ifdef HAVE_STRERROR
+#include <string.h>
+#include <errno.h>
+#endif
+
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long longestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh = {0, 0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void ChangeMem (int Change) {
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse;
+}
+
+
+int init_text_1 (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  int which;
+  
+  if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+    Message ("Unable to allocate memory for pool");
+    return (COMPERROR);
+  }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+  
+  for (which = 1; which >= 0; --which) {
+    u_char *word;
+    hash_rec *ent;
+    dict_data *dd = &DictData[which];
+    
+    dd->wordnum = 0;
+    dd->words_read = 0;
+    dd->bytes_diff = 0;
+    dd->HashSize = INITIAL_HASH_SIZE;
+    dd->HashUsed = 0;
+    
+    if (!(dd->HashTable = (hash_rec *) Xmalloc (sizeof (hash_rec) * dd->HashSize))) {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+    ChangeMem (sizeof (hash_rec) * dd->HashSize);
+    memset (dd->HashTable,'\0' , sizeof (hash_rec) * dd->HashSize);
+
+    word = Pool;
+    *Pool++ = '\0';
+    --PoolLeft;
+    {
+      register u_char *wptr;
+      register int hsize = dd->HashSize;
+      register unsigned long hashval, step;
+      
+      HASH (hashval, step, word, hsize);
+      wptr = (dd->HashTable + hashval)->word;
+      while (wptr) {
+	hashval += step;
+	if (hashval >= (unsigned long)hsize)
+	  hashval -= hsize;
+	wptr = (dd->HashTable + hashval)->word;
+      }
+      ent = dd->HashTable + hashval;
+    }
+    ent->wcnt = 1;
+    ent->word = word;
+    dd->HashUsed = 1;
+  }
+  return (COMPALLOK);
+}
+
+
+static int process_text_element (const u_char *s_in, int l_in) {
+  const u_char *end = s_in + l_in - 1;
+  
+   /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  
+  bool which = false; // non-word
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    dict_data *dd = &DictData[which];
+    
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+    
+    ++dd->wordnum;
+    inputbytes += *Word;
+    ++dd->words_read;
+    
+    /* Search the hash table for Word */
+    {
+      register unsigned long hashval, step;
+      register int hsize = dd->HashSize;
+      HASH (hashval, step, Word, hsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	register hash_rec *ent;
+	ent = dd->HashTable + hashval;
+	if (!ent->word) {
+	  int len = *Word + 1;
+	  if (len > PoolLeft) {
+	    if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+	      Message ("Unable to allocate memory for pool");
+	      return (COMPERROR);
+	    }
+	    PoolLeft = POOL_SIZE;
+	    ChangeMem (POOL_SIZE);
+	  }
+	  ent->occurance_num = occurance_num++;
+	  ent->wcnt = 1;
+	  ent->word = Pool;
+	  memcpy (Pool, Word, len);
+	  Pool += len;
+	  PoolLeft -= len;
+	  ++dd->HashUsed;
+	  dd->bytes_diff += Word[0];
+	  break;
+	}
+	
+	/* Compare the words */
+	s1 = Word;
+	s2 = ent->word;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval = (hashval + step);
+	  if (hashval >= (unsigned long)hsize) hashval -= hsize;
+	} else {
+	  ++ent->wcnt;
+	  break;
+	}
+      }
+    }
+
+    
+    if (dd->HashUsed >= dd->HashSize >> 1) {
+      hash_rec *ht;
+      unsigned long size;
+      unsigned long i;
+      size = prime (dd->HashSize * 2);
+      if (!(ht = (hash_rec *) Xmalloc (sizeof (hash_rec) * size))) {
+	Message ("Unable to allocate memory for table");
+	return (COMPERROR);
+      }
+      ChangeMem (sizeof (hash_rec) * size);
+      memset (ht, '\0', sizeof (hash_rec) * size);
+
+      for (i = 0; i < dd->HashSize; ++i)
+	if (dd->HashTable[i].word) {
+	  register u_char *wptr;
+	  register unsigned long hashval, step;
+	  
+	  wptr = dd->HashTable[i].word;
+	  HASH (hashval, step, wptr, size);
+	  wptr = (ht + hashval)->word;
+	  while (wptr) {
+	    hashval += step;
+	    if (hashval >= size) hashval -= size;
+	    wptr = (ht + hashval)->word;
+	  }
+	  ht[hashval] = dd->HashTable[i];
+	}
+      Xfree (dd->HashTable);
+      ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+      dd->HashTable = ht;
+      dd->HashSize = size;
+    }
+  }
+
+  return COMPALLOK;
+}
+
+
+int process_text_1 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  unsigned long textLen = 0;
+  unsigned long docLen = 0;
+  int retValue;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    textLen = (*here).text.size();
+    docLen += textLen;
+
+    retValue = process_text_element (&(here->text[0]), textLen);
+    if (retValue != COMPALLOK) return retValue;
+    
+    ++here;
+  }
+
+  // get max document length
+  if (docLen > longestDoc) longestDoc = docLen;
+
+  // update header information
+  ++csh.num_docs;
+  csh.num_bytes += docLen;
+  
+  return COMPALLOK;
+}
+
+
+static int PackHashTable (dict_data * dd) {
+  int s, d;
+  for (s = d = 0; (unsigned int)s < dd->HashSize; ++s)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  
+  if (!(dd->HashTable = (hash_rec *) Xrealloc (dd->HashTable,
+			   sizeof (hash_rec) * dd->HashUsed))) {
+    Message ("Out of memory");
+    return COMPERROR;
+  }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+static int ent_comp (const void *s1, const void *s2) {
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+static void WriteHashTable (FILE * fp, dict_data * dd) {
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR) return;
+  
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+  
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; ++j)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+  
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+  
+  for (j = 0; j < dd->HashSize; ++j) {
+    curr = dd->HashTable[j].word;
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    HTONUL(dd->HashTable[j].wcnt);
+    HTONUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+    fwrite (&dd->HashTable[j].occurance_num,
+	    sizeof (dd->HashTable[j].occurance_num), 1, fp);
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    NTOHUL(dd->HashTable[j].wcnt);
+    NTOHUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+  }
+}
+
+
+int done_text_1 (const TagInfo &/*tagInfo*/, char *file_name) {
+  char *temp_str;
+  FILE *fp;
+  
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf(stderr,"Couldn't create file %s%s:%s\n",
+              file_name, TEXT_STATS_DICT_SUFFIX,
+#if defined(HAVE_STRERROR) || defined(__WIN32__)
+              strerror(errno)
+#else
+	      " "
+#endif
+              );
+      return COMPERROR;
+    }
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  fclose(fp);
+  return COMPALLOK;
+}
Index: /main/tags/3.01/mgpp/text/text.pass2.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/text.pass2.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/text.pass2.cpp	(revision 21112)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * text.pass2.cpp -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__)
+#pragma warning(disable:4786)
+#endif
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#if defined (__WIN32__)
+# include <stdio.h>
+# define unlink _unlink
+#endif
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+#include "FText.h"
+
+#define POOL_SIZE 1024*256
+
+
+struct char_pool {
+  struct char_pool *next;
+  u_long left;
+  u_char *ptr;
+  u_char pool[POOL_SIZE];
+};
+
+struct novel_hash_rec {
+  u_long ordinal_num;
+  u_char *word;
+};
+
+
+#define INITIAL_HASH_SIZE 7927
+
+struct novel_hash_table {
+  novel_hash_rec *HashTable;
+  u_long HashSize, HashUsed;
+  char_pool *first_pool;
+  char_pool *pool;
+  u_long next_num, binary_start;
+  novel_hash_rec **code_to_nhr;
+};
+
+
+static FILE *text;
+static stdio_bitio_buffer textOutBuf;
+static u_long headLen;
+
+static novel_hash_table nht[2];
+
+int blk_start[2][33], blk_end[2][33];
+
+// 0 = non-word, 1 = word
+unsigned char whichWordType = 0;
+
+CompressTextInfoMap textInfoMap;
+
+
+static char_pool *new_pool (char_pool * pool) {
+  char_pool *p = (char_pool *) Xmalloc (sizeof (char_pool));
+  if (!p) FatalError (1, "Unable to allocate memory for pool");
+  
+  if (pool) pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+int init_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  char path[512];
+
+  // load the compression dictionary
+  if (LoadCompressionDictionary (make_name (fileName, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  // create the compressed text file and attach it to the text buffer
+  if (!(text = create_file (fileName, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))
+    return COMPERROR;
+  textOutBuf.attachFile (text);
+  textOutBuf.encodeStart ();
+  
+  // write out the compressed text header
+  memset (&cth, '\0', sizeof (cth));
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  // the length of the compressed header is the size of the
+  // magic number plus the size of the compressed text header
+  headLen = sizeof (u_long) + sizeof (cth);
+
+  // start off with non-word
+  whichWordType = 0;
+
+  // clear the text pointers
+  textInfoMap.erase (textInfoMap.begin(), textInfoMap.end());
+  
+  // set up the hash table of novel words
+  int i;
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; ++i)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	memset (nht[i].HashTable, '\0',
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		++num;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+static int compress_text (bitio_buffer &buffer, const u_char *s_in,
+			  int l_in, unsigned char &which,
+			  unsigned long &numWords) {
+  const u_char *end = s_in + l_in - 1;
+
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    int res;
+    
+    if (which) ++numWords;
+
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+
+    /* Search the hash table for Word */
+    if (ht[which]) {
+      register unsigned long hashval, step;
+      register int tsize = ht[which]->size;
+      register u_char **wptr;
+      HASH (hashval, step, Word, tsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	wptr = ht[which]->table[hashval];
+	if (wptr == NULL) {
+	  res = COMPERROR;
+	  break;
+	}
+
+	/* Compare the words */
+	s1 = Word;
+	s2 = *wptr;
+	len = *s1 + 1;
+	for (; len; --len)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval += step;
+	  if (hashval >= (unsigned long)tsize) hashval -= tsize;
+	  
+	} else {
+	  res = ht[which]->table[hashval] - ht[which]->words;
+	  break;
+	}
+      }
+    }
+    else
+      res = COMPERROR;
+    /* Check that the word was found in the dictionary */
+    if (res == COMPERROR)
+      {
+	if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	  {
+	    Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	    return (COMPERROR);
+	  }
+	if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	  {
+	    u_long i;
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    buffer.huff_encode (Word[0], lens_codes[which], lens_huff[which].clens, NULL);
+	    for (i = 0; i < Word[0]; ++i)
+	      buffer.huff_encode (Word[i + 1], char_codes[which],
+				  char_huff[which].clens, NULL);
+	  }
+	if (cdh.dict_type == MG_SEED_DICTIONARY)
+	  {
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    switch (cdh.novel_method)
+	      {
+	      case MG_NOVEL_HUFFMAN_CHARS:
+		{
+		  u_long i;
+		  buffer.huff_encode (Word[0], lens_codes[which],
+				      lens_huff[which].clens, NULL);
+		  for (i = 0; i < Word[0]; ++i)
+		    buffer.huff_encode (Word[i + 1], char_codes[which],
+					char_huff[which].clens, NULL);
+		}
+		break;
+	      case MG_NOVEL_DELTA:
+	      case MG_NOVEL_HYBRID:
+		{
+		  register unsigned long hashval, step;
+		  register novel_hash_table *h = &nht[which];
+		  register int hsize = h->HashSize;
+		  register novel_hash_rec *ent;
+		  HASH (hashval, step, Word, hsize);
+		  for (;;)
+		    {
+		      register u_char *s1, *s2;
+		      register int len;
+		      ent = h->HashTable + hashval;
+		      if (!ent->word)
+			{
+			  int len = *Word + 1;
+			  if ((unsigned)len > h->pool->left)
+			    h->pool = new_pool (h->pool);
+			  ent->word = h->pool->ptr;
+			  ent->ordinal_num = h->next_num++;
+			  memcpy (h->pool->ptr, Word, len);
+			  h->pool->ptr += len;
+			  h->pool->left -= len;
+			  ++h->HashUsed;
+			  break;
+			}
+		      /* Compare the words */
+		      s1 = Word;
+		      s2 = ent->word;
+		      len = *s1 + 1;
+		      for (; len; --len)
+			if (*s1++ != *s2++)
+			  break;
+		      
+		      if (!len)
+			break;
+		      
+		      hashval = (hashval + step);
+		      if (hashval >= (unsigned)hsize) hashval -= hsize;
+		    }
+		  
+		  switch (cdh.novel_method) {
+		  case MG_NOVEL_DELTA:
+		    buffer.delta_encode (ent->ordinal_num, NULL);
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    int k = 0;
+		    int j = ent->ordinal_num - 1;
+		    while (j > blk_end[which][k])
+		      ++k;
+		    assert (j - blk_start[which][k] + 1 >= 1 &&
+			    j - blk_start[which][k] + 1 <=
+			    blk_end[which][k] - blk_start[which][k] + 1);
+		    
+		    buffer.gamma_encode (k + 1, NULL);
+		    buffer.binary_encode (j - blk_start[which][k] + 1,
+					  blk_end[which][k] - blk_start[which][k] + 1,
+					  NULL);
+		    break;
+		  }
+		  
+		  if (h->HashUsed >= h->HashSize >> 1)
+		    {
+		      novel_hash_rec *ht;
+		      unsigned long size;
+		      unsigned long i;
+		      size = prime (h->HashSize * 2);
+		      if (!(ht = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * size)))
+			{
+			  Message ("Unable to allocate memory for table");
+			  return (COMPERROR);
+			}
+		      memset (ht, '\0', sizeof (novel_hash_rec) * size);
+		      
+		      for (i = 0; i < h->HashSize; ++i)
+			if (h->HashTable[i].word)
+			  {
+			    register u_char *wptr;
+			    register unsigned long hashval, step;
+			    
+			    wptr = h->HashTable[i].word;
+			    HASH (hashval, step, wptr, size);
+			    wptr = (ht + hashval)->word;
+			    while (wptr)
+			      {
+				hashval += step;
+				if (hashval >= size)
+				  hashval -= size;
+				wptr = (ht + hashval)->word;
+			      }
+			    ht[hashval] = h->HashTable[i];
+			  }
+		      Xfree (h->HashTable);
+		      h->HashTable = ht;
+		      h->HashSize = size;
+		    }
+		}
+		break;
+	      }
+	  }
+      }
+    else
+      {
+	buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+      }
+  }
+  
+  return COMPALLOK;
+}
+
+
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc) {
+  // we need to remember where the last text element ends
+  unsigned char endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+  unsigned long endPos = textOutBuf.GetBytesWritten () + headLen;
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // remember current place in compressed text
+    unsigned char startWhich = whichWordType;
+    unsigned char startBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    unsigned long startPos = textOutBuf.GetBytesWritten () + headLen;
+
+    // compress the text
+    if (compress_text (textOutBuf,
+		       &(here->text[0]),
+		       (*here).text.size(),
+		       whichWordType,
+		       cth.num_of_words) != COMPALLOK)
+      return COMPERROR;
+
+    endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    endPos = textOutBuf.GetBytesWritten () + headLen;
+    
+    // update the pointers if this is a level tag
+    if (tagInfo.levelTags.find ((*here).tagName) != tagInfo.levelTags.end()) {
+      if ((*here).elType == OpenTagE) {
+	// set the start of the document
+	textInfoMap[(*here).tagName].SetStart (startPos, startBit, startWhich);
+	
+      } else if ((*here).elType == CloseTagE) {
+	// set the end of the document
+	textInfoMap[(*here).tagName].SetEnd (endPos, endBit);
+	
+      } else {
+	// TextE, nothing needs to be done
+      }
+    }
+    
+    cth.num_of_bytes += (*here).text.size();
+    ++here;
+  }
+
+  // close off any unclosed tags
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    if ((*tiHere).second.inDoc) (*tiHere).second.SetEnd (endPos, endBit);
+    ++tiHere;
+  }
+  
+  // we've processed one more document
+  ++cth.num_of_docs;
+
+  return COMPALLOK;
+}
+
+
+static int write_aux_dict (char *fileName) {
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (fileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))
+    return COMPERROR;
+  
+  for (i = 0; i <= 1; ++i)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+      
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+      
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+      
+      fwrite (&afh, sizeof (afh), 1, aux);
+      
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+static void estimate_compressed_aux_dict (void) {
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; ++i)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      memset (chars, '\0', sizeof (chars));
+      memset (lens, '\0', sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      ++lens[len];
+	      total_uncomp += len + 4;
+	      for (; len; --len)
+		++chars[*buf++];
+	    }
+	}
+      for (j = 0; j < 256; ++j)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; ++j)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+      
+      aux_compressed += (long unsigned int) ((Calculate_Huffman_Size (16, flens, lens) +
+					      Calculate_Huffman_Size (256, fchars, chars)) / 8);
+      
+    }
+}
+
+
+
+static bool WriteTextIdx (char *fileName, FTextLevel &textLevels) {
+  // create the text index
+  FILE *textIdxFile;
+  if (!(textIdxFile = create_file (fileName, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    // remember the level information
+    TextLevelInfo levelInfo;
+    levelInfo.levelTag = (*tiHere).first;
+    levelInfo.textIdxPtr = ftell (textIdxFile);
+    levelInfo.numEntries = (*tiHere).second.docPtrs.size();
+    textLevels.levelInfo[levelInfo.levelTag] = levelInfo;
+
+    // write out the index array for this level
+    if (!WriteTextIdxArray (textIdxFile, (*tiHere).second.docPtrs))
+      return false;
+    
+    ++tiHere;
+  }
+
+  // close the file
+  fclose (textIdxFile);
+
+  return true;
+}
+
+
+bool WriteTextLevels (char *fileName, const FTextLevel &textLevels) {
+  //  cout << textLevels;
+  
+  // create the text index
+  FILE *textLevelFile;
+  if (!(textLevelFile = create_file (fileName, TEXT_LEVEL_SUFFIX, "w+b",
+				     MAGIC_TEXT_LEVELS, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  if (!textLevels.Write (textLevelFile)) return false;
+  
+  // close the file
+  fclose (textLevelFile);
+
+  return true;
+}
+
+
+int done_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  // write out any remaining bits
+  textOutBuf.encodeDone();
+
+  // write out the compressed text header (with updated statistics)
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 || !cth.Write (text))
+    return COMPERROR;
+  fclose (text);
+
+  // write out the text index file and get the level information
+  FTextLevel textLevels;
+  if (!WriteTextIdx (fileName, textLevels)) return COMPERROR;
+  
+  // write out the text level information
+  if (!WriteTextLevels (fileName, textLevels)) return COMPERROR;
+
+  // write out the auxiliary dictionary
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID)) {
+    if (write_aux_dict (fileName) == COMPERROR) return COMPERROR;
+    estimate_compressed_aux_dict ();
+    
+  } else {
+    unlink (make_name (fileName, TEXT_DICT_AUX_SUFFIX, NULL));
+  }
+
+  return (COMPALLOK);
+}
Index: /main/tags/3.01/mgpp/text/words.cpp
===================================================================
--- /main/tags/3.01/mgpp/text/words.cpp	(revision 21112)
+++ /main/tags/3.01/mgpp/text/words.cpp	(revision 21112)
@@ -0,0 +1,122 @@
+/**************************************************************************
+ *
+ * words.cpp -- Functions for parsing out words from the source text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+// need this to avoid bizarre compiler problems under VC++ 6.0
+#if defined (__WIN32__) && !defined (GSDL_USE_IOS_H)
+# include <iostream>
+#endif
+
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+/* It determines whether a given place in a UTF-8 encoded Unicode string is a unicode space. */
+int isaspace (const u_char *here, const u_char *end)
+{
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_space(c);
+  return 0;
+}
+
+/* Return a the UTF-8 encoded Unicode string with begining
+   unicode spaces skippend. */
+u_char *skipspace(u_char *here, u_char *end)
+{
+  unsigned short c;	
+  int length;
+  while(here != end) {
+    length = parse_utf8_char(here, end, &c);
+    if (length == 0 || !is_unicode_space(c)) break;
+    here += length;
+  }
+  return here;
+}
+ 
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word) {
+  word.erase (word.begin(), word.end());
+
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      word.push_back (*textHere++); ++length;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
+
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord) {
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      mgWord[++length] = *textHere++;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+
+  mgWord[0] = length;
+  
+  return textHere;
+}
+
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd) {
+  register int charlength = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char(textHere, textEnd, &c);
+  
+  while (charlength > 0 && !is_unicode_letdig(c)) {
+    textHere += charlength;
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
Index: /main/tags/3.01/mgpp/text/words.h
===================================================================
--- /main/tags/3.01/mgpp/text/words.h	(revision 21112)
+++ /main/tags/3.01/mgpp/text/words.h	(revision 21112)
@@ -0,0 +1,164 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "unitool.h"
+#include "UCArray.h"
+
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+#define MAXNUMERIC	4
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+int isaspace (const u_char *here, const u_char *end);
+        /* It determines whether a given place in a UTF-8 encoded Unicode string  is a unicode space. */
+
+u_char *skipspace(u_char *here, u_char *end);
+        /* Return a the UTF-8 encoded Unicode string with begining unicode spaces skippend. */
+
+#ifdef __cplusplus
+}
+#endif
+
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word);
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord);
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd);
+
+
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; ++length;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
Index: /main/tags/3.01/mgpp/win32cfg.h
===================================================================
--- /main/tags/3.01/mgpp/win32cfg.h	(revision 21112)
+++ /main/tags/3.01/mgpp/win32cfg.h	(revision 21112)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /main/tags/3.01/mgpp/winMake.bat
===================================================================
--- /main/tags/3.01/mgpp/winMake.bat	(revision 21112)
+++ /main/tags/3.01/mgpp/winMake.bat	(revision 21112)
@@ -0,0 +1,41 @@
+@echo off
+
+set MAKE=nmake
+set MAKE_OPTIONS=/f
+
+if ""%1"" == """" goto all
+if ""%1"" == ""compile"" goto go
+if ""%1"" == ""install"" goto go
+if ""%1"" == ""clean"" goto go
+
+:unknown
+	echo Error: Unrecognized argument %1.
+	goto done
+
+:go
+	if ""%2"" == ""javaonly"" goto java
+
+:all
+	cd lib
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd text
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+	cd java\org\greenstone\mgpp
+	call winMake.bat %1
+	cd ..\..\..\..
+
+	cd jni
+	%MAKE% %MAKE_OPTIONS% win32.mak %1
+	cd ..
+
+:java
+	cd java\org\greenstone\mgpp
+	call winMake.bat %1
+	cd ..\..\..\..
+	goto done
+
+:done
